From 3adaf096758a6015ca4f733e2e49ee5528ac3cd5 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Thu, 24 Mar 2016 11:54:05 -0700
Subject: [PATCH 0001/1311] Settings: Cleanup placeholder replacement

This change moves placeholder replacement to a pkg private class for
settings. It also adds a null check when calling replacement, as
settings objects can still contain null values, because we only prohibit
nulls on file loading. Finally, this cleans up file and stream loading a
bit to not have unnecessary exception wrapping.
---
 .../common/logging/LogConfigurator.java       |  2 +-
 .../PropertyPlaceholder.java                  | 37 ++++------
 .../common/settings/Settings.java             | 35 +++++----
 .../indices/analysis/HunspellService.java     |  2 +-
 .../internal/InternalSettingsPreparer.java    |  6 +-
 .../PropertyPlaceholderTests.java             | 62 ++++++----------
 .../loader/YamlSettingsLoaderTests.java       | 60 ++++++++--------
 .../index/analysis/AnalysisModuleTests.java   |  2 +-
 .../index/analysis/CompoundAnalysisTests.java |  4 +-
 .../cloud/azure/AbstractAzureTestCase.java    | 72 -------------------
 .../cloud/aws/AbstractAwsTestCase.java        |  7 +-
 .../AbstractAzureWithThirdPartyTestCase.java  |  7 +-
 .../cloud/aws/AbstractAwsTestCase.java        |  7 +-
 13 files changed, 110 insertions(+), 193 deletions(-)
 rename core/src/main/java/org/elasticsearch/common/{property => settings}/PropertyPlaceholder.java (83%)
 rename core/src/test/java/org/elasticsearch/common/{property => settings}/PropertyPlaceholderTests.java (78%)
 delete mode 100644 plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureTestCase.java

diff --git a/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java b/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
index da628b09d2b..5ccbfe4a2dd 100644
--- a/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
+++ b/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
@@ -167,7 +167,7 @@ public class LogConfigurator {
     static void loadConfig(Path file, Settings.Builder settingsBuilder) {
         try {
             settingsBuilder.loadFromPath(file);
-        } catch (SettingsException | NoClassDefFoundError e) {
+        } catch (IOException | SettingsException | NoClassDefFoundError e) {
             // ignore
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/common/property/PropertyPlaceholder.java b/core/src/main/java/org/elasticsearch/common/settings/PropertyPlaceholder.java
similarity index 83%
rename from core/src/main/java/org/elasticsearch/common/property/PropertyPlaceholder.java
rename to core/src/main/java/org/elasticsearch/common/settings/PropertyPlaceholder.java
index 70e6807cb92..2eb7e2b8e70 100644
--- a/core/src/main/java/org/elasticsearch/common/property/PropertyPlaceholder.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/PropertyPlaceholder.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.common.property;
+package org.elasticsearch.common.settings;
 
 import org.elasticsearch.common.Strings;
 
@@ -34,23 +34,12 @@ import java.util.Set;
  * Values for substitution can be supplied using a {@link Properties} instance or using a
  * {@link PlaceholderResolver}.
  */
-public class PropertyPlaceholder {
+class PropertyPlaceholder {
 
     private final String placeholderPrefix;
     private final String placeholderSuffix;
     private final boolean ignoreUnresolvablePlaceholders;
 
-    /**
-     * Creates a new <code>PropertyPlaceholderHelper</code> that uses the supplied prefix and suffix. Unresolvable
-     * placeholders are ignored.
-     *
-     * @param placeholderPrefix the prefix that denotes the start of a placeholder.
-     * @param placeholderSuffix the suffix that denotes the end of a placeholder.
-     */
-    public PropertyPlaceholder(String placeholderPrefix, String placeholderSuffix) {
-        this(placeholderPrefix, placeholderSuffix, true);
-    }
-
     /**
      * Creates a new <code>PropertyPlaceholderHelper</code> that uses the supplied prefix and suffix.
      *
@@ -59,12 +48,10 @@ public class PropertyPlaceholder {
      * @param ignoreUnresolvablePlaceholders indicates whether unresolvable placeholders should be ignored
      *                                       (<code>true</code>) or cause an exception (<code>false</code>).
      */
-    public PropertyPlaceholder(String placeholderPrefix, String placeholderSuffix,
+    PropertyPlaceholder(String placeholderPrefix, String placeholderSuffix,
                                boolean ignoreUnresolvablePlaceholders) {
-        Objects.requireNonNull(placeholderPrefix, "Argument 'placeholderPrefix' must not be null.");
-        Objects.requireNonNull(placeholderSuffix, "Argument 'placeholderSuffix' must not be null.");
-        this.placeholderPrefix = placeholderPrefix;
-        this.placeholderSuffix = placeholderSuffix;
+        this.placeholderPrefix = Objects.requireNonNull(placeholderPrefix);
+        this.placeholderSuffix = Objects.requireNonNull(placeholderSuffix);
         this.ignoreUnresolvablePlaceholders = ignoreUnresolvablePlaceholders;
     }
 
@@ -75,15 +62,15 @@ public class PropertyPlaceholder {
      * @param value               the value containing the placeholders to be replaced.
      * @param placeholderResolver the <code>PlaceholderResolver</code> to use for replacement.
      * @return the supplied value with placeholders replaced inline.
+     * @throws NullPointerException if value is null
      */
-    public String replacePlaceholders(String key, String value, PlaceholderResolver placeholderResolver) {
-        Objects.requireNonNull(key);
-        Objects.requireNonNull(value, "value can not be null for [" + key + "]");
-        return parseStringValue(value, placeholderResolver, new HashSet<String>());
+    String replacePlaceholders(String value, PlaceholderResolver placeholderResolver) {
+        Objects.requireNonNull(value);
+        return parseStringValue(value, placeholderResolver, new HashSet<>());
     }
 
-    protected String parseStringValue(String strVal, PlaceholderResolver placeholderResolver,
-                                      Set<String> visitedPlaceholders) {
+    private String parseStringValue(String strVal, PlaceholderResolver placeholderResolver,
+                                    Set<String> visitedPlaceholders) {
         StringBuilder buf = new StringBuilder(strVal);
 
         int startIndex = strVal.indexOf(this.placeholderPrefix);
@@ -164,7 +151,7 @@ public class PropertyPlaceholder {
      *
      * @see PropertyPlaceholder
      */
-    public interface PlaceholderResolver {
+    interface PlaceholderResolver {
 
         /**
          * Resolves the supplied placeholder name into the replacement value.
diff --git a/core/src/main/java/org/elasticsearch/common/settings/Settings.java b/core/src/main/java/org/elasticsearch/common/settings/Settings.java
index ce79bf92d20..887edf48443 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/Settings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/Settings.java
@@ -25,7 +25,6 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.property.PropertyPlaceholder;
 import org.elasticsearch.common.settings.loader.SettingsLoader;
 import org.elasticsearch.common.settings.loader.SettingsLoaderFactory;
 import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -1114,26 +1113,20 @@ public final class Settings implements ToXContent {
          * Loads settings from a url that represents them using the
          * {@link SettingsLoaderFactory#loaderFromSource(String)}.
          */
-        public Builder loadFromPath(Path path) throws SettingsException {
-            try {
-                return loadFromStream(path.getFileName().toString(), Files.newInputStream(path));
-            } catch (IOException e) {
-                throw new SettingsException("Failed to open stream for url [" + path + "]", e);
-            }
+        public Builder loadFromPath(Path path) throws IOException {
+            // NOTE: loadFromStream will close the input stream
+            return loadFromStream(path.getFileName().toString(), Files.newInputStream(path));
         }
 
         /**
          * Loads settings from a stream that represents them using the
          * {@link SettingsLoaderFactory#loaderFromSource(String)}.
          */
-        public Builder loadFromStream(String resourceName, InputStream is) throws SettingsException {
+        public Builder loadFromStream(String resourceName, InputStream is) throws IOException {
             SettingsLoader settingsLoader = SettingsLoaderFactory.loaderFromResource(resourceName);
-            try {
-                Map<String, String> loadedSettings = settingsLoader.load(Streams.copyToString(new InputStreamReader(is, StandardCharsets.UTF_8)));
-                put(loadedSettings);
-            } catch (Exception e) {
-                throw new SettingsException("Failed to load settings from [" + resourceName + "]", e);
-            }
+            // NOTE: copyToString will close the input stream
+            Map<String, String> loadedSettings = settingsLoader.load(Streams.copyToString(new InputStreamReader(is, StandardCharsets.UTF_8)));
+            put(loadedSettings);
             return this;
         }
 
@@ -1220,14 +1213,20 @@ public final class Settings implements ToXContent {
                         return true;
                     }
                 };
-            for (Map.Entry<String, String> entry : new HashMap<>(map).entrySet()) {
-                String value = propertyPlaceholder.replacePlaceholders(entry.getKey(), entry.getValue(), placeholderResolver);
+            Iterator<Map.Entry<String, String>> entryItr = map.entrySet().iterator();
+            while (entryItr.hasNext()) {
+                Map.Entry<String, String> entry = entryItr.next();
+                if (entry.getValue() == null) {
+                    // a null value obviously can't be replaced
+                    continue;
+                }
+                String value = propertyPlaceholder.replacePlaceholders(entry.getValue(), placeholderResolver);
                 // if the values exists and has length, we should maintain it  in the map
                 // otherwise, the replace process resolved into removing it
                 if (Strings.hasLength(value)) {
-                    map.put(entry.getKey(), value);
+                    entry.setValue(value);
                 } else {
-                    map.remove(entry.getKey());
+                    entryItr.remove();
                 }
             }
             return this;
diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java b/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java
index 75c15f09778..4450bd557b6 100644
--- a/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java
+++ b/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java
@@ -210,7 +210,7 @@ public class HunspellService extends AbstractComponent {
      * @param defaults The default settings for this dictionary
      * @return The resolved settings.
      */
-    private static Settings loadDictionarySettings(Path dir, Settings defaults) {
+    private static Settings loadDictionarySettings(Path dir, Settings defaults) throws IOException {
         Path file = dir.resolve("settings.yml");
         if (Files.exists(file)) {
             return Settings.settingsBuilder().loadFromPath(file).put(defaults).build();
diff --git a/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java b/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java
index 8864a70ccdc..f9539f7c363 100644
--- a/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java
+++ b/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java
@@ -92,7 +92,11 @@ public class InternalSettingsPreparer {
             Path path = environment.configFile().resolve("elasticsearch" + allowedSuffix);
             if (Files.exists(path)) {
                 if (!settingsFileFound) {
-                    output.loadFromPath(path);
+                    try {
+                        output.loadFromPath(path);
+                    } catch (IOException e) {
+                        throw new SettingsException("Failed to settings from " + path.toString(), e);
+                    }
                 }
                 settingsFileFound = true;
                 foundSuffixes.add(allowedSuffix);
diff --git a/core/src/test/java/org/elasticsearch/common/property/PropertyPlaceholderTests.java b/core/src/test/java/org/elasticsearch/common/settings/PropertyPlaceholderTests.java
similarity index 78%
rename from core/src/test/java/org/elasticsearch/common/property/PropertyPlaceholderTests.java
rename to core/src/test/java/org/elasticsearch/common/settings/PropertyPlaceholderTests.java
index 405ac566686..78176bc1d80 100644
--- a/core/src/test/java/org/elasticsearch/common/property/PropertyPlaceholderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/PropertyPlaceholderTests.java
@@ -17,14 +17,13 @@
  * under the License.
  */
 
-package org.elasticsearch.common.property;
-
-import org.elasticsearch.test.ESTestCase;
+package org.elasticsearch.common.settings;
 
 import java.util.LinkedHashMap;
 import java.util.Map;
 
-import static org.hamcrest.Matchers.hasToString;
+import org.elasticsearch.test.ESTestCase;
+
 import static org.hamcrest.Matchers.is;
 
 public class PropertyPlaceholderTests extends ESTestCase {
@@ -34,10 +33,10 @@ public class PropertyPlaceholderTests extends ESTestCase {
         map.put("foo1", "bar1");
         map.put("foo2", "bar2");
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true);
-        assertEquals("bar1", propertyPlaceholder.replacePlaceholders("key", "{foo1}", placeholderResolver));
-        assertEquals("a bar1b", propertyPlaceholder.replacePlaceholders("key", "a {foo1}b", placeholderResolver));
-        assertEquals("bar1bar2", propertyPlaceholder.replacePlaceholders("key", "{foo1}{foo2}", placeholderResolver));
-        assertEquals("a bar1 b bar2 c", propertyPlaceholder.replacePlaceholders("key", "a {foo1} b {foo2} c", placeholderResolver));
+        assertEquals("bar1", propertyPlaceholder.replacePlaceholders("{foo1}", placeholderResolver));
+        assertEquals("a bar1b", propertyPlaceholder.replacePlaceholders("a {foo1}b", placeholderResolver));
+        assertEquals("bar1bar2", propertyPlaceholder.replacePlaceholders("{foo1}{foo2}", placeholderResolver));
+        assertEquals("a bar1 b bar2 c", propertyPlaceholder.replacePlaceholders("a {foo1} b {foo2} c", placeholderResolver));
     }
 
     public void testVariousPrefixSuffix() {
@@ -48,24 +47,24 @@ public class PropertyPlaceholderTests extends ESTestCase {
         Map<String, String> map = new LinkedHashMap<>();
         map.put("foo", "bar");
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true);
-        assertEquals("bar", ppEqualsPrefix.replacePlaceholders("key", "{foo}", placeholderResolver));
-        assertEquals("bar", ppLongerPrefix.replacePlaceholders("key", "${foo}", placeholderResolver));
-        assertEquals("bar", ppShorterPrefix.replacePlaceholders("key", "{foo}}", placeholderResolver));
+        assertEquals("bar", ppEqualsPrefix.replacePlaceholders("{foo}", placeholderResolver));
+        assertEquals("bar", ppLongerPrefix.replacePlaceholders("${foo}", placeholderResolver));
+        assertEquals("bar", ppShorterPrefix.replacePlaceholders("{foo}}", placeholderResolver));
     }
 
     public void testDefaultValue() {
         PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false);
         Map<String, String> map = new LinkedHashMap<>();
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true);
-        assertEquals("bar", propertyPlaceholder.replacePlaceholders("key", "${foo:bar}", placeholderResolver));
-        assertEquals("", propertyPlaceholder.replacePlaceholders("key", "${foo:}", placeholderResolver));
+        assertEquals("bar", propertyPlaceholder.replacePlaceholders("${foo:bar}", placeholderResolver));
+        assertEquals("", propertyPlaceholder.replacePlaceholders("${foo:}", placeholderResolver));
     }
 
     public void testIgnoredUnresolvedPlaceholder() {
         PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", true);
         Map<String, String> map = new LinkedHashMap<>();
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true);
-        assertEquals("${foo}", propertyPlaceholder.replacePlaceholders("key", "${foo}", placeholderResolver));
+        assertEquals("${foo}", propertyPlaceholder.replacePlaceholders("${foo}", placeholderResolver));
     }
 
     public void testNotIgnoredUnresolvedPlaceholder() {
@@ -73,7 +72,7 @@ public class PropertyPlaceholderTests extends ESTestCase {
         Map<String, String> map = new LinkedHashMap<>();
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true);
         try {
-            propertyPlaceholder.replacePlaceholders("key", "${foo}", placeholderResolver);
+            propertyPlaceholder.replacePlaceholders("${foo}", placeholderResolver);
             fail("Expected IllegalArgumentException");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), is("Could not resolve placeholder 'foo'"));
@@ -84,7 +83,7 @@ public class PropertyPlaceholderTests extends ESTestCase {
         PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false);
         Map<String, String> map = new LinkedHashMap<>();
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, true, true);
-        assertEquals("bar", propertyPlaceholder.replacePlaceholders("key", "bar${foo}", placeholderResolver));
+        assertEquals("bar", propertyPlaceholder.replacePlaceholders("bar${foo}", placeholderResolver));
     }
 
     public void testRecursive() {
@@ -94,8 +93,8 @@ public class PropertyPlaceholderTests extends ESTestCase {
         map.put("foo1", "${foo2}");
         map.put("foo2", "bar");
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true);
-        assertEquals("bar", propertyPlaceholder.replacePlaceholders("key", "${foo}", placeholderResolver));
-        assertEquals("abarb", propertyPlaceholder.replacePlaceholders("key", "a${foo}b", placeholderResolver));
+        assertEquals("bar", propertyPlaceholder.replacePlaceholders("${foo}", placeholderResolver));
+        assertEquals("abarb", propertyPlaceholder.replacePlaceholders("a${foo}b", placeholderResolver));
     }
 
     public void testNestedLongerPrefix() {
@@ -106,7 +105,7 @@ public class PropertyPlaceholderTests extends ESTestCase {
         map.put("foo2", "bar");
         map.put("barbar", "baz");
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true);
-        assertEquals("baz", propertyPlaceholder.replacePlaceholders("key", "${bar${foo}}", placeholderResolver));
+        assertEquals("baz", propertyPlaceholder.replacePlaceholders("${bar${foo}}", placeholderResolver));
     }
 
     public void testNestedSameLengthPrefixSuffix() {
@@ -117,7 +116,7 @@ public class PropertyPlaceholderTests extends ESTestCase {
         map.put("foo2", "bar");
         map.put("barbar", "baz");
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true);
-        assertEquals("baz", propertyPlaceholder.replacePlaceholders("key", "{bar{foo}}", placeholderResolver));
+        assertEquals("baz", propertyPlaceholder.replacePlaceholders("{bar{foo}}", placeholderResolver));
     }
 
     public void testNestedShorterPrefix() {
@@ -128,7 +127,7 @@ public class PropertyPlaceholderTests extends ESTestCase {
         map.put("foo2", "bar");
         map.put("barbar", "baz");
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true);
-        assertEquals("baz", propertyPlaceholder.replacePlaceholders("key", "{bar{foo}}}}", placeholderResolver));
+        assertEquals("baz", propertyPlaceholder.replacePlaceholders("{bar{foo}}}}", placeholderResolver));
     }
 
     public void testCircularReference() {
@@ -138,7 +137,7 @@ public class PropertyPlaceholderTests extends ESTestCase {
         map.put("bar", "${foo}");
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, false, true);
         try {
-            propertyPlaceholder.replacePlaceholders("key", "${foo}", placeholderResolver);
+            propertyPlaceholder.replacePlaceholders("${foo}", placeholderResolver);
             fail("Expected IllegalArgumentException");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), is("Circular placeholder reference 'foo' in property definitions"));
@@ -149,24 +148,7 @@ public class PropertyPlaceholderTests extends ESTestCase {
         PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false);
         Map<String, String> map = new LinkedHashMap<>();
         PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, true, false);
-        assertEquals("bar${foo}", propertyPlaceholder.replacePlaceholders("key", "bar${foo}", placeholderResolver));
-    }
-
-    public void testNullKey() {
-        final PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false);
-        final Map<String, String> map = new LinkedHashMap<>();
-        final PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, true, false);
-        expectThrows(NullPointerException.class, () -> propertyPlaceholder.replacePlaceholders(null, "value", placeholderResolver));
-    }
-
-    public void testNullValue() {
-        final PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false);
-        final Map<String, String> map = new LinkedHashMap<>();
-        final PropertyPlaceholder.PlaceholderResolver placeholderResolver = new SimplePlaceholderResolver(map, true, false);
-        final String key = randomAsciiOfLength(10);
-        NullPointerException e =
-                expectThrows(NullPointerException.class, () -> propertyPlaceholder.replacePlaceholders(key, null, placeholderResolver));
-        assertThat(e, hasToString("java.lang.NullPointerException: value can not be null for [" + key + "]"));
+        assertEquals("bar${foo}", propertyPlaceholder.replacePlaceholders("bar${foo}", placeholderResolver));
     }
 
     private class SimplePlaceholderResolver implements PropertyPlaceholder.PlaceholderResolver {
diff --git a/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java b/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java
index 2e2a187da0b..67db756cb3d 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java
@@ -19,6 +19,11 @@
 
 package org.elasticsearch.common.settings.loader;
 
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
@@ -48,42 +53,39 @@ public class YamlSettingsLoaderTests extends ESTestCase {
         assertThat(settings.getAsArray("test1.test3")[1], equalTo("test3-2"));
     }
 
-    public void testIndentation() {
-        final String yaml = "/org/elasticsearch/common/settings/loader/indentation-settings.yml";
-        final SettingsException e =
-                expectThrows(
-                        SettingsException.class,
-                        () -> settingsBuilder().loadFromStream(yaml, getClass().getResourceAsStream(yaml)).build());
-        assertThat(e.getMessage(), containsString("Failed to load settings"));
+    public void testIndentation() throws Exception {
+        String yaml = "/org/elasticsearch/common/settings/loader/indentation-settings.yml";
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> {
+            settingsBuilder().loadFromStream(yaml, getClass().getResourceAsStream(yaml));
+        });
+        assertTrue(e.getMessage(), e.getMessage().contains("malformed"));
     }
 
-    public void testIndentationWithExplicitDocumentStart() {
-        final String yaml = "/org/elasticsearch/common/settings/loader/indentation-with-explicit-document-start-settings.yml";
-        final SettingsException e =
-                expectThrows(
-                        SettingsException.class,
-                        () -> settingsBuilder().loadFromStream(yaml, getClass().getResourceAsStream(yaml)).build());
-        assertThat(e.getMessage(), containsString("Failed to load settings"));
+    public void testIndentationWithExplicitDocumentStart() throws Exception {
+        String yaml = "/org/elasticsearch/common/settings/loader/indentation-with-explicit-document-start-settings.yml";
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> {
+            settingsBuilder().loadFromStream(yaml, getClass().getResourceAsStream(yaml));
+        });
+        assertTrue(e.getMessage(), e.getMessage().contains("malformed"));
     }
 
     public void testDuplicateKeysThrowsException() {
-        final String yaml = "foo: bar\nfoo: baz";
-        final SettingsException e = expectThrows(SettingsException.class, () -> settingsBuilder().loadFromSource(yaml).build());
+        String yaml = "foo: bar\nfoo: baz";
+        SettingsException e = expectThrows(SettingsException.class, () -> {
+                settingsBuilder().loadFromSource(yaml);
+        });
         assertEquals(e.getCause().getClass(), ElasticsearchParseException.class);
-        assertThat(
-                e.toString(),
-                containsString("duplicate settings key [foo] " +
-                        "found at line number [2], " +
-                        "column number [6], " +
-                        "previous value [bar], " +
-                        "current value [baz]"));
+        String msg = e.getCause().getMessage();
+        assertTrue(msg, msg.contains("duplicate settings key [foo] found"));
+        assertTrue(msg, msg.contains("previous value [bar], current value [baz]"));
     }
 
-    public void testNullValuedSettingThrowsException() {
-        final String yaml = "foo:";
-        final ElasticsearchParseException e =
-                expectThrows(ElasticsearchParseException.class, () -> new YamlSettingsLoader(false).load(yaml));
-        assertThat(e.toString(), containsString("null-valued setting found for key [foo] found at line number [1], column number [5]"));
+    public void testMissingValue() throws Exception {
+        Path tmp = createTempFile("test", ".yaml");
+        Files.write(tmp, Collections.singletonList("foo: # missing value\n"), StandardCharsets.UTF_8);
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> {
+            settingsBuilder().loadFromPath(tmp);
+        });
+        assertTrue(e.getMessage(), e.getMessage().contains("null-valued setting found for key [foo]"));
     }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisModuleTests.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisModuleTests.java
index 6468fae9397..e1d35039f02 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisModuleTests.java
@@ -79,7 +79,7 @@ public class AnalysisModuleTests extends ModuleTestCase {
                 Collections.emptyMap(), Collections.singletonMap("myfilter", MyFilterTokenFilterFactory::new), Collections.emptyMap(), Collections.emptyMap());
     }
 
-    private Settings loadFromClasspath(String path) {
+    private Settings loadFromClasspath(String path) throws IOException {
         return settingsBuilder().loadFromStream(path, getClass().getResourceAsStream(path))
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/CompoundAnalysisTests.java b/core/src/test/java/org/elasticsearch/index/analysis/CompoundAnalysisTests.java
index fbedf42d083..fe5b0855798 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/CompoundAnalysisTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/CompoundAnalysisTests.java
@@ -91,7 +91,7 @@ public class CompoundAnalysisTests extends ESTestCase {
         return terms;
     }
 
-    private Settings getJsonSettings() {
+    private Settings getJsonSettings() throws IOException {
         String json = "/org/elasticsearch/index/analysis/test1.json";
         return settingsBuilder()
                 .loadFromStream(json, getClass().getResourceAsStream(json))
@@ -100,7 +100,7 @@ public class CompoundAnalysisTests extends ESTestCase {
                 .build();
     }
 
-    private Settings getYamlSettings() {
+    private Settings getYamlSettings() throws IOException {
         String yaml = "/org/elasticsearch/index/analysis/test1.yml";
         return settingsBuilder()
                 .loadFromStream(yaml, getClass().getResourceAsStream(yaml))
diff --git a/plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureTestCase.java b/plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureTestCase.java
deleted file mode 100644
index ad7140f5020..00000000000
--- a/plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureTestCase.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.azure;
-
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.io.PathUtils;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsException;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.plugin.discovery.azure.AzureDiscoveryPlugin;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.ESIntegTestCase.ThirdParty;
-
-import java.util.Collection;
-
-/**
- * Base class for Azure tests that require credentials.
- * <p>
- * You must specify {@code -Dtests.thirdparty=true -Dtests.config=/path/to/config}
- * in order to run these tests.
- */
-@ThirdParty
-public abstract class AbstractAzureTestCase extends ESIntegTestCase {
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder()
-                .put(super.nodeSettings(nodeOrdinal))
-                .put(readSettingsFromFile())
-                .build();
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return pluginList(AzureDiscoveryPlugin.class);
-    }
-
-    protected Settings readSettingsFromFile() {
-        Settings.Builder settings = Settings.builder();
-        settings.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir());
-
-        // if explicit, just load it and don't load from env
-        try {
-            if (Strings.hasText(System.getProperty("tests.config"))) {
-                settings.loadFromPath(PathUtils.get((System.getProperty("tests.config"))));
-            } else {
-                throw new IllegalStateException("to run integration tests, you need to set -Dtests.thirdparty=true and -Dtests.config=/path/to/elasticsearch.yml");
-            }
-        } catch (SettingsException exception) {
-          throw new IllegalStateException("your test configuration file is incorrect: " + System.getProperty("tests.config"), exception);
-        }
-        return settings.build();
-    }
-}
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTestCase.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTestCase.java
index cc9b0897600..dc794038598 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTestCase.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTestCase.java
@@ -29,6 +29,7 @@ import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ThirdParty;
 
+import java.io.IOException;
 import java.util.Collection;
 
 /**
@@ -52,7 +53,11 @@ public abstract class AbstractAwsTestCase extends ESIntegTestCase {
         // if explicit, just load it and don't load from env
         try {
             if (Strings.hasText(System.getProperty("tests.config"))) {
-                settings.loadFromPath(PathUtils.get(System.getProperty("tests.config")));
+                try {
+                    settings.loadFromPath(PathUtils.get(System.getProperty("tests.config")));
+                } catch (IOException e) {
+                    throw new IllegalArgumentException("could not load aws tests config", e);
+                }
             } else {
                 throw new IllegalStateException("to run integration tests, you need to set -Dtests.thirdparty=true and -Dtests.config=/path/to/elasticsearch.yml");
             }
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureWithThirdPartyTestCase.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureWithThirdPartyTestCase.java
index b7c2d373a58..7136befeaff 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureWithThirdPartyTestCase.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureWithThirdPartyTestCase.java
@@ -27,6 +27,7 @@ import org.elasticsearch.plugin.repository.azure.AzureRepositoryPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase.ThirdParty;
 
+import java.io.IOException;
 import java.util.Collection;
 
 /**
@@ -58,7 +59,11 @@ public abstract class AbstractAzureWithThirdPartyTestCase extends AbstractAzureT
         // if explicit, just load it and don't load from env
         try {
             if (Strings.hasText(System.getProperty("tests.config"))) {
-                settings.loadFromPath(PathUtils.get((System.getProperty("tests.config"))));
+                try {
+                    settings.loadFromPath(PathUtils.get((System.getProperty("tests.config"))));
+                } catch (IOException e) {
+                    throw new IllegalArgumentException("could not load azure tests config", e);
+                }
             } else {
                 throw new IllegalStateException("to run integration tests, you need to set -Dtests.thirdparty=true and -Dtests.config=/path/to/elasticsearch.yml");
             }
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTestCase.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTestCase.java
index ec8fb902d66..9d1768db58b 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTestCase.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTestCase.java
@@ -29,6 +29,7 @@ import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ThirdParty;
 
+import java.io.IOException;
 import java.util.Collection;
 
 /**
@@ -52,7 +53,11 @@ public abstract class AbstractAwsTestCase extends ESIntegTestCase {
         // if explicit, just load it and don't load from env
         try {
             if (Strings.hasText(System.getProperty("tests.config"))) {
-                settings.loadFromPath(PathUtils.get(System.getProperty("tests.config")));
+                try {
+                    settings.loadFromPath(PathUtils.get(System.getProperty("tests.config")));
+                } catch (IOException e) {
+                    throw new IllegalArgumentException("could not load aws tests config", e);
+                }
             } else {
                 throw new IllegalStateException("to run integration tests, you need to set -Dtests.thirdparty=true and -Dtests.config=/path/to/elasticsearch.yml");
             }

From 6380560dbb989242e48f68719fe6045c51f7017e Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Mon, 11 Apr 2016 14:35:37 -0700
Subject: [PATCH 0002/1311] Check more complete exception message for missing
 setting value

---
 .../common/settings/loader/YamlSettingsLoaderTests.java         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java b/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java
index f3926ebe07e..618209cf114 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java
@@ -85,6 +85,6 @@ public class YamlSettingsLoaderTests extends ESTestCase {
         ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> {
             Settings.builder().loadFromPath(tmp);
         });
-        assertTrue(e.getMessage(), e.getMessage().contains("null-valued setting found for key [foo]"));
+        assertTrue(e.getMessage(), e.getMessage().contains("null-valued setting found for key [foo] found at line"));
     }
 }

From f2ee759ad56a11ee42236b13d7edf687149b7d94 Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Fri, 15 Apr 2016 11:51:47 +0200
Subject: [PATCH 0003/1311] Upgrade AWS SDK to 1.10.69

* Moving from JSON.org to Jackson for request marshallers.
* The Java SDK now supports retry throttling to limit the rate of retries during periods of reduced availability. This throttling behavior can be enabled via ClientConfiguration or via the system property "-Dcom.amazonaws.sdk.enableThrottledRetry".
* Fixed String case conversion issues when running with non English locales.
* AWS SDK for Java introduces a new dynamic endpoint system that can compute endpoints for services in new regions.
* Introducing a new AWS region, ap-northeast-2.
* Added a new metric, HttpSocketReadTime, that records socket read latency. You can enable this metric by adding enableHttpSocketReadMetric to the system property com.amazonaws.sdk.enableDefaultMetrics. For more information, see [Enabling Metrics with the AWS SDK for Java](https://java.awsblog.com/post/Tx3C0RV4NRRBKTG/Enabling-Metrics-with-the-AWS-SDK-for-Java).
* New Client Execution timeout feature to set a limit spent across retries, backoffs, ummarshalling, etc. This new timeout can be specified at the client level or per request.
  Also included in this release is the ability to specify the existing HTTP Request timeout per request rather than just per client.

* Added support for RequesterPays for all operations.
* Ignore the 'Connection' header when generating S3 responses.
* Allow users to generate an AmazonS3URI from a string without using URL encoding.
* Fixed issue that prevented creating buckets when using a client configured for the s3-external-1 endpoint.
* Amazon S3 bucket lifecycle configuration supports two new features: the removal of expired object delete markers and an action to abort incomplete multipart uploads.
* Allow TransferManagerConfiguration to accept integer values for multipart upload threshold.
* Copy the list of ETags before sorting https://github.com/aws/aws-sdk-java/pull/589.
* Option to disable chunked encoding https://github.com/aws/aws-sdk-java/pull/586.
* Adding retry on InternalErrors in CompleteMultipartUpload operation. https://github.com/aws/aws-sdk-java/issues/538
* Deprecated two APIs : AmazonS3#changeObjectStorageClass and AmazonS3#setObjectRedirectLocation.
* Added support for the aws-exec-read canned ACL. Owner gets FULL_CONTROL. Amazon EC2 gets READ access to GET an Amazon Machine Image (AMI) bundle from Amazon S3.

* Added support for referencing security groups in peered Virtual Private Clouds (VPCs). For more information see the service announcement at https://aws.amazon.com/about-aws/whats-new/2016/03/announcing-support-for-security-group-references-in-a-peered-vpc/ .
* Fixed a bug in AWS SDK for Java - Amazon EC2 module that returns NPE for dry run requests.
* Regenerated client with new implementation of code generator.
* This feature enables support for DNS resolution of public hostnames to private IP addresses when queried over ClassicLink. Additionally, you can now access private hosted zones associated with your VPC from a linked EC2-Classic instance. ClassicLink DNS support makes it easier for EC2-Classic instances to communicate with VPC resources using public DNS hostnames.
* You can now use Network Address Translation (NAT) Gateway, a highly available AWS managed service that makes it easy to connect to the Internet from instances within a private subnet in an AWS Virtual Private Cloud (VPC). Previously, you needed to launch a NAT instance to enable NAT for instances in a private subnet. Amazon VPC NAT Gateway is available in the US East (N. Virginia), US West (Oregon), US West (N. California), EU (Ireland), Asia Pacific (Tokyo), Asia Pacific (Singapore), and Asia Pacific (Sydney) regions. To learn more about Amazon VPC NAT, see [New - Managed NAT (Network Address Translation) Gateway for AWS](https://aws.amazon.com/blogs/aws/new-managed-nat-network-address-translation-gateway-for-aws/)
* A default read timeout is now applied when querying data from EC2 metadata service.
---
 plugins/discovery-ec2/build.gradle            |   7 +-
 .../aws-java-sdk-core-1.10.33.jar.sha1        |   1 -
 .../aws-java-sdk-core-1.10.69.jar.sha1        |   1 +
 .../aws-java-sdk-ec2-1.10.33.jar.sha1         |   1 -
 .../aws-java-sdk-ec2-1.10.69.jar.sha1         |   1 +
 .../discovery/ec2/AmazonEC2Mock.java          | 128 ++++++++++++++++++
 plugins/repository-s3/build.gradle            |  10 +-
 .../aws-java-sdk-core-1.10.33.jar.sha1        |   1 -
 .../aws-java-sdk-core-1.10.69.jar.sha1        |   1 +
 .../aws-java-sdk-kms-1.10.33.jar.sha1         |   1 -
 .../aws-java-sdk-kms-1.10.69.jar.sha1         |   1 +
 .../licenses/aws-java-sdk-s3-1.10.33.jar.sha1 |   1 -
 .../licenses/aws-java-sdk-s3-1.10.69.jar.sha1 |   1 +
 .../cloud/aws/AmazonS3Wrapper.java            |  18 +++
 14 files changed, 156 insertions(+), 17 deletions(-)
 delete mode 100644 plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.33.jar.sha1
 create mode 100644 plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.69.jar.sha1
 delete mode 100644 plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.33.jar.sha1
 create mode 100644 plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.69.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.10.33.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-core-1.10.69.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.33.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.69.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.33.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.69.jar.sha1

diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle
index e8baa1ec502..15cce1aa32b 100644
--- a/plugins/discovery-ec2/build.gradle
+++ b/plugins/discovery-ec2/build.gradle
@@ -23,7 +23,7 @@ esplugin {
 }
 
 versions << [
-  'aws': '1.10.33'
+  'aws': '1.10.69'
 ]
 
 dependencies {
@@ -48,11 +48,6 @@ test {
 }
 
 thirdPartyAudit.excludes = [
-  // uses internal java api: com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl
-  // uses internal java api: com.sun.org.apache.xml.internal.dtm.ref.DTMManagerDefault
-  // uses internal java api: com.sun.org.apache.xpath.internal.XPathContext
-  'com.amazonaws.util.XpathUtils',
-
   // classes are missing
   'javax.servlet.ServletContextEvent', 
   'javax.servlet.ServletContextListener', 
diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.33.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.33.jar.sha1
deleted file mode 100644
index 332a8f01035..00000000000
--- a/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.33.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fabedbbe2b834b1add150b6a38395c5ef7380168
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.69.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.69.jar.sha1
new file mode 100644
index 00000000000..2971a33d7d9
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/aws-java-sdk-core-1.10.69.jar.sha1
@@ -0,0 +1 @@
+a1f02d5f26ba1d8c37e2bf9c847db3c6729dda00
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.33.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.33.jar.sha1
deleted file mode 100644
index 4737b80b3f2..00000000000
--- a/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.33.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-202f6b5dbc196e355d50c131b0fd34969bfd89e6
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.69.jar.sha1 b/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.69.jar.sha1
new file mode 100644
index 00000000000..a1a493d3b8f
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/aws-java-sdk-ec2-1.10.69.jar.sha1
@@ -0,0 +1 @@
+afbff1ece8365859eb4cfe0d3ba543d68b154d26
\ No newline at end of file
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java
index 88d87a2d8c2..881cb98e1d1 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java
@@ -29,6 +29,8 @@ import com.amazonaws.services.ec2.model.AcceptVpcPeeringConnectionRequest;
 import com.amazonaws.services.ec2.model.AcceptVpcPeeringConnectionResult;
 import com.amazonaws.services.ec2.model.AllocateAddressRequest;
 import com.amazonaws.services.ec2.model.AllocateAddressResult;
+import com.amazonaws.services.ec2.model.AllocateHostsRequest;
+import com.amazonaws.services.ec2.model.AllocateHostsResult;
 import com.amazonaws.services.ec2.model.AssignPrivateIpAddressesRequest;
 import com.amazonaws.services.ec2.model.AssociateAddressRequest;
 import com.amazonaws.services.ec2.model.AssociateAddressResult;
@@ -80,6 +82,8 @@ import com.amazonaws.services.ec2.model.CreateInternetGatewayRequest;
 import com.amazonaws.services.ec2.model.CreateInternetGatewayResult;
 import com.amazonaws.services.ec2.model.CreateKeyPairRequest;
 import com.amazonaws.services.ec2.model.CreateKeyPairResult;
+import com.amazonaws.services.ec2.model.CreateNatGatewayRequest;
+import com.amazonaws.services.ec2.model.CreateNatGatewayResult;
 import com.amazonaws.services.ec2.model.CreateNetworkAclEntryRequest;
 import com.amazonaws.services.ec2.model.CreateNetworkAclRequest;
 import com.amazonaws.services.ec2.model.CreateNetworkAclResult;
@@ -120,6 +124,8 @@ import com.amazonaws.services.ec2.model.DeleteFlowLogsRequest;
 import com.amazonaws.services.ec2.model.DeleteFlowLogsResult;
 import com.amazonaws.services.ec2.model.DeleteInternetGatewayRequest;
 import com.amazonaws.services.ec2.model.DeleteKeyPairRequest;
+import com.amazonaws.services.ec2.model.DeleteNatGatewayRequest;
+import com.amazonaws.services.ec2.model.DeleteNatGatewayResult;
 import com.amazonaws.services.ec2.model.DeleteNetworkAclEntryRequest;
 import com.amazonaws.services.ec2.model.DeleteNetworkAclRequest;
 import com.amazonaws.services.ec2.model.DeleteNetworkInterfaceRequest;
@@ -161,6 +167,10 @@ import com.amazonaws.services.ec2.model.DescribeExportTasksRequest;
 import com.amazonaws.services.ec2.model.DescribeExportTasksResult;
 import com.amazonaws.services.ec2.model.DescribeFlowLogsRequest;
 import com.amazonaws.services.ec2.model.DescribeFlowLogsResult;
+import com.amazonaws.services.ec2.model.DescribeHostsRequest;
+import com.amazonaws.services.ec2.model.DescribeHostsResult;
+import com.amazonaws.services.ec2.model.DescribeIdFormatRequest;
+import com.amazonaws.services.ec2.model.DescribeIdFormatResult;
 import com.amazonaws.services.ec2.model.DescribeImageAttributeRequest;
 import com.amazonaws.services.ec2.model.DescribeImageAttributeResult;
 import com.amazonaws.services.ec2.model.DescribeImagesRequest;
@@ -181,6 +191,8 @@ import com.amazonaws.services.ec2.model.DescribeKeyPairsRequest;
 import com.amazonaws.services.ec2.model.DescribeKeyPairsResult;
 import com.amazonaws.services.ec2.model.DescribeMovingAddressesRequest;
 import com.amazonaws.services.ec2.model.DescribeMovingAddressesResult;
+import com.amazonaws.services.ec2.model.DescribeNatGatewaysRequest;
+import com.amazonaws.services.ec2.model.DescribeNatGatewaysResult;
 import com.amazonaws.services.ec2.model.DescribeNetworkAclsRequest;
 import com.amazonaws.services.ec2.model.DescribeNetworkAclsResult;
 import com.amazonaws.services.ec2.model.DescribeNetworkInterfaceAttributeRequest;
@@ -203,6 +215,10 @@ import com.amazonaws.services.ec2.model.DescribeReservedInstancesRequest;
 import com.amazonaws.services.ec2.model.DescribeReservedInstancesResult;
 import com.amazonaws.services.ec2.model.DescribeRouteTablesRequest;
 import com.amazonaws.services.ec2.model.DescribeRouteTablesResult;
+import com.amazonaws.services.ec2.model.DescribeScheduledInstanceAvailabilityRequest;
+import com.amazonaws.services.ec2.model.DescribeScheduledInstanceAvailabilityResult;
+import com.amazonaws.services.ec2.model.DescribeScheduledInstancesRequest;
+import com.amazonaws.services.ec2.model.DescribeScheduledInstancesResult;
 import com.amazonaws.services.ec2.model.DescribeSecurityGroupsRequest;
 import com.amazonaws.services.ec2.model.DescribeSecurityGroupsResult;
 import com.amazonaws.services.ec2.model.DescribeSnapshotAttributeRequest;
@@ -233,6 +249,8 @@ import com.amazonaws.services.ec2.model.DescribeVolumesRequest;
 import com.amazonaws.services.ec2.model.DescribeVolumesResult;
 import com.amazonaws.services.ec2.model.DescribeVpcAttributeRequest;
 import com.amazonaws.services.ec2.model.DescribeVpcAttributeResult;
+import com.amazonaws.services.ec2.model.DescribeVpcClassicLinkDnsSupportRequest;
+import com.amazonaws.services.ec2.model.DescribeVpcClassicLinkDnsSupportResult;
 import com.amazonaws.services.ec2.model.DescribeVpcClassicLinkRequest;
 import com.amazonaws.services.ec2.model.DescribeVpcClassicLinkResult;
 import com.amazonaws.services.ec2.model.DescribeVpcEndpointServicesRequest;
@@ -255,6 +273,8 @@ import com.amazonaws.services.ec2.model.DetachVolumeRequest;
 import com.amazonaws.services.ec2.model.DetachVolumeResult;
 import com.amazonaws.services.ec2.model.DetachVpnGatewayRequest;
 import com.amazonaws.services.ec2.model.DisableVgwRoutePropagationRequest;
+import com.amazonaws.services.ec2.model.DisableVpcClassicLinkDnsSupportRequest;
+import com.amazonaws.services.ec2.model.DisableVpcClassicLinkDnsSupportResult;
 import com.amazonaws.services.ec2.model.DisableVpcClassicLinkRequest;
 import com.amazonaws.services.ec2.model.DisableVpcClassicLinkResult;
 import com.amazonaws.services.ec2.model.DisassociateAddressRequest;
@@ -263,6 +283,8 @@ import com.amazonaws.services.ec2.model.DryRunResult;
 import com.amazonaws.services.ec2.model.DryRunSupportedRequest;
 import com.amazonaws.services.ec2.model.EnableVgwRoutePropagationRequest;
 import com.amazonaws.services.ec2.model.EnableVolumeIORequest;
+import com.amazonaws.services.ec2.model.EnableVpcClassicLinkDnsSupportRequest;
+import com.amazonaws.services.ec2.model.EnableVpcClassicLinkDnsSupportResult;
 import com.amazonaws.services.ec2.model.EnableVpcClassicLinkRequest;
 import com.amazonaws.services.ec2.model.EnableVpcClassicLinkResult;
 import com.amazonaws.services.ec2.model.Filter;
@@ -283,8 +305,13 @@ import com.amazonaws.services.ec2.model.ImportVolumeResult;
 import com.amazonaws.services.ec2.model.Instance;
 import com.amazonaws.services.ec2.model.InstanceState;
 import com.amazonaws.services.ec2.model.InstanceStateName;
+import com.amazonaws.services.ec2.model.ModifyHostsRequest;
+import com.amazonaws.services.ec2.model.ModifyHostsResult;
+import com.amazonaws.services.ec2.model.ModifyIdFormatRequest;
 import com.amazonaws.services.ec2.model.ModifyImageAttributeRequest;
 import com.amazonaws.services.ec2.model.ModifyInstanceAttributeRequest;
+import com.amazonaws.services.ec2.model.ModifyInstancePlacementRequest;
+import com.amazonaws.services.ec2.model.ModifyInstancePlacementResult;
 import com.amazonaws.services.ec2.model.ModifyNetworkInterfaceAttributeRequest;
 import com.amazonaws.services.ec2.model.ModifyReservedInstancesRequest;
 import com.amazonaws.services.ec2.model.ModifyReservedInstancesResult;
@@ -302,12 +329,16 @@ import com.amazonaws.services.ec2.model.MoveAddressToVpcRequest;
 import com.amazonaws.services.ec2.model.MoveAddressToVpcResult;
 import com.amazonaws.services.ec2.model.PurchaseReservedInstancesOfferingRequest;
 import com.amazonaws.services.ec2.model.PurchaseReservedInstancesOfferingResult;
+import com.amazonaws.services.ec2.model.PurchaseScheduledInstancesRequest;
+import com.amazonaws.services.ec2.model.PurchaseScheduledInstancesResult;
 import com.amazonaws.services.ec2.model.RebootInstancesRequest;
 import com.amazonaws.services.ec2.model.RegisterImageRequest;
 import com.amazonaws.services.ec2.model.RegisterImageResult;
 import com.amazonaws.services.ec2.model.RejectVpcPeeringConnectionRequest;
 import com.amazonaws.services.ec2.model.RejectVpcPeeringConnectionResult;
 import com.amazonaws.services.ec2.model.ReleaseAddressRequest;
+import com.amazonaws.services.ec2.model.ReleaseHostsRequest;
+import com.amazonaws.services.ec2.model.ReleaseHostsResult;
 import com.amazonaws.services.ec2.model.ReplaceNetworkAclAssociationRequest;
 import com.amazonaws.services.ec2.model.ReplaceNetworkAclAssociationResult;
 import com.amazonaws.services.ec2.model.ReplaceNetworkAclEntryRequest;
@@ -330,6 +361,8 @@ import com.amazonaws.services.ec2.model.RevokeSecurityGroupEgressRequest;
 import com.amazonaws.services.ec2.model.RevokeSecurityGroupIngressRequest;
 import com.amazonaws.services.ec2.model.RunInstancesRequest;
 import com.amazonaws.services.ec2.model.RunInstancesResult;
+import com.amazonaws.services.ec2.model.RunScheduledInstancesRequest;
+import com.amazonaws.services.ec2.model.RunScheduledInstancesResult;
 import com.amazonaws.services.ec2.model.StartInstancesRequest;
 import com.amazonaws.services.ec2.model.StartInstancesResult;
 import com.amazonaws.services.ec2.model.StopInstancesRequest;
@@ -519,6 +552,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public DeleteNatGatewayResult deleteNatGateway(DeleteNatGatewayRequest deleteNatGatewayRequest) {
+        return null;
+    }
+
     @Override
     public UnmonitorInstancesResult unmonitorInstances(UnmonitorInstancesRequest unmonitorInstancesRequest) throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -639,6 +677,16 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public ModifyHostsResult modifyHosts(ModifyHostsRequest modifyHostsRequest) {
+        return null;
+    }
+
+    @Override
+    public void modifyIdFormat(ModifyIdFormatRequest modifyIdFormatRequest) {
+
+    }
+
     @Override
     public DescribeSecurityGroupsResult describeSecurityGroups(DescribeSecurityGroupsRequest describeSecurityGroupsRequest) throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -964,6 +1012,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public DisableVpcClassicLinkDnsSupportResult disableVpcClassicLinkDnsSupport(DisableVpcClassicLinkDnsSupportRequest disableVpcClassicLinkDnsSupportRequest) {
+        return null;
+    }
+
     @Override
     public DescribeInstanceAttributeResult describeInstanceAttribute(DescribeInstanceAttributeRequest describeInstanceAttributeRequest) throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -989,6 +1042,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public RunScheduledInstancesResult runScheduledInstances(RunScheduledInstancesRequest runScheduledInstancesRequest) {
+        return null;
+    }
+
     @Override
     public DescribeSubnetsResult describeSubnets(DescribeSubnetsRequest describeSubnetsRequest) throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -1024,6 +1082,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public ModifyInstancePlacementResult modifyInstancePlacement(ModifyInstancePlacementRequest modifyInstancePlacementRequest) {
+        return null;
+    }
+
     @Override
     public CancelReservedInstancesListingResult cancelReservedInstancesListing(CancelReservedInstancesListingRequest cancelReservedInstancesListingRequest) throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -1159,6 +1222,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public PurchaseScheduledInstancesResult purchaseScheduledInstances(PurchaseScheduledInstancesRequest purchaseScheduledInstancesRequest) {
+        return null;
+    }
+
     @Override
     public void modifySnapshotAttribute(ModifySnapshotAttributeRequest modifySnapshotAttributeRequest) throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -1334,6 +1402,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public ReleaseHostsResult releaseHosts(ReleaseHostsRequest releaseHostsRequest) {
+        return null;
+    }
+
     @Override
     public void resetInstanceAttribute(ResetInstanceAttributeRequest resetInstanceAttributeRequest) throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -1344,6 +1417,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public CreateNatGatewayResult createNatGateway(CreateNatGatewayRequest createNatGatewayRequest) {
+        return null;
+    }
+
     @Override
     public void replaceNetworkAclEntry(ReplaceNetworkAclEntryRequest replaceNetworkAclEntryRequest) throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -1374,6 +1452,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public EnableVpcClassicLinkDnsSupportResult enableVpcClassicLinkDnsSupport(EnableVpcClassicLinkDnsSupportRequest enableVpcClassicLinkDnsSupportRequest) {
+        return null;
+    }
+
     @Override
     public void createVpnConnectionRoute(CreateVpnConnectionRouteRequest createVpnConnectionRouteRequest) throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -1489,6 +1572,16 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public DescribeScheduledInstanceAvailabilityResult describeScheduledInstanceAvailability(DescribeScheduledInstanceAvailabilityRequest describeScheduledInstanceAvailabilityRequest) {
+        return null;
+    }
+
+    @Override
+    public DescribeScheduledInstancesResult describeScheduledInstances(DescribeScheduledInstancesRequest describeScheduledInstancesRequest) {
+        return null;
+    }
+
     @Override
     public DescribeDhcpOptionsResult describeDhcpOptions() throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -1529,6 +1622,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public DescribeNatGatewaysResult describeNatGateways(DescribeNatGatewaysRequest describeNatGatewaysRequest) {
+        return null;
+    }
+
     @Override
     public DescribeConversionTasksResult describeConversionTasks() throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -1544,6 +1642,26 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public DescribeHostsResult describeHosts(DescribeHostsRequest describeHostsRequest) {
+        return null;
+    }
+
+    @Override
+    public DescribeHostsResult describeHosts() {
+        return null;
+    }
+
+    @Override
+    public DescribeIdFormatResult describeIdFormat(DescribeIdFormatRequest describeIdFormatRequest) {
+        return null;
+    }
+
+    @Override
+    public DescribeIdFormatResult describeIdFormat() {
+        return null;
+    }
+
     @Override
     public DescribeVpcPeeringConnectionsResult describeVpcPeeringConnections() throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -1584,6 +1702,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public DescribeVpcClassicLinkDnsSupportResult describeVpcClassicLinkDnsSupport(DescribeVpcClassicLinkDnsSupportRequest describeVpcClassicLinkDnsSupportRequest) {
+        return null;
+    }
+
     @Override
     public DescribeClassicLinkInstancesResult describeClassicLinkInstances() throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
@@ -1649,6 +1772,11 @@ public class AmazonEC2Mock implements AmazonEC2 {
         throw new UnsupportedOperationException("Not supported in mock");
     }
 
+    @Override
+    public AllocateHostsResult allocateHosts(AllocateHostsRequest allocateHostsRequest) {
+        return null;
+    }
+
     @Override
     public DescribeSnapshotsResult describeSnapshots() throws AmazonServiceException, AmazonClientException {
         throw new UnsupportedOperationException("Not supported in mock");
diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle
index a083309891e..d01ebd29c12 100644
--- a/plugins/repository-s3/build.gradle
+++ b/plugins/repository-s3/build.gradle
@@ -23,7 +23,7 @@ esplugin {
 }
 
 versions << [
-  'aws': '1.10.33'
+  'aws': '1.10.69'
 ]
 
 dependencies {
@@ -49,11 +49,6 @@ test {
 }
 
 thirdPartyAudit.excludes = [
-  // uses internal java api: com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl
-  // uses internal java api: com.sun.org.apache.xml.internal.dtm.ref.DTMManagerDefault
-  // uses internal java api: com.sun.org.apache.xpath.internal.XPathContext
-  'com.amazonaws.util.XpathUtils',
-
   // classes are missing
   'javax.servlet.ServletContextEvent', 
   'javax.servlet.ServletContextListener', 
@@ -61,3 +56,6 @@ thirdPartyAudit.excludes = [
   'org.apache.log.Hierarchy', 
   'org.apache.log.Logger',
 ]
+
+// AWS SDK is exposing some deprecated methods which we call using a delegate
+compileTestJava.options.compilerArgs << "-Xlint:-deprecation"
diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.33.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.33.jar.sha1
deleted file mode 100644
index 332a8f01035..00000000000
--- a/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.33.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fabedbbe2b834b1add150b6a38395c5ef7380168
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.69.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.69.jar.sha1
new file mode 100644
index 00000000000..2971a33d7d9
--- /dev/null
+++ b/plugins/repository-s3/licenses/aws-java-sdk-core-1.10.69.jar.sha1
@@ -0,0 +1 @@
+a1f02d5f26ba1d8c37e2bf9c847db3c6729dda00
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.33.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.33.jar.sha1
deleted file mode 100644
index 0d7ab9f8381..00000000000
--- a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.33.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-35881245894ecc4d893c074eacdf2e6b56820fda
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.69.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.69.jar.sha1
new file mode 100644
index 00000000000..22e1f924664
--- /dev/null
+++ b/plugins/repository-s3/licenses/aws-java-sdk-kms-1.10.69.jar.sha1
@@ -0,0 +1 @@
+ed74ff3872193b4704a751f0b72ab2cf0db0651b
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.33.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.33.jar.sha1
deleted file mode 100644
index 3328f01c658..00000000000
--- a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.33.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5665cf77102a932a16e99ebf41d197e03ddbf25c
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.69.jar.sha1 b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.69.jar.sha1
new file mode 100644
index 00000000000..64e7336c2bc
--- /dev/null
+++ b/plugins/repository-s3/licenses/aws-java-sdk-s3-1.10.69.jar.sha1
@@ -0,0 +1 @@
+6fa48bf0bff43f26436956b88d8d3764b6cf109e
\ No newline at end of file
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java
index 97829f9d689..7cda9ee0947 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AmazonS3Wrapper.java
@@ -50,6 +50,7 @@ import com.amazonaws.services.s3.model.CreateBucketRequest;
 import com.amazonaws.services.s3.model.DeleteBucketCrossOriginConfigurationRequest;
 import com.amazonaws.services.s3.model.DeleteBucketLifecycleConfigurationRequest;
 import com.amazonaws.services.s3.model.DeleteBucketPolicyRequest;
+import com.amazonaws.services.s3.model.DeleteBucketReplicationConfigurationRequest;
 import com.amazonaws.services.s3.model.DeleteBucketRequest;
 import com.amazonaws.services.s3.model.DeleteBucketTaggingConfigurationRequest;
 import com.amazonaws.services.s3.model.DeleteBucketWebsiteConfigurationRequest;
@@ -69,6 +70,7 @@ import com.amazonaws.services.s3.model.GetBucketReplicationConfigurationRequest;
 import com.amazonaws.services.s3.model.GetBucketTaggingConfigurationRequest;
 import com.amazonaws.services.s3.model.GetBucketVersioningConfigurationRequest;
 import com.amazonaws.services.s3.model.GetBucketWebsiteConfigurationRequest;
+import com.amazonaws.services.s3.model.GetObjectAclRequest;
 import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
 import com.amazonaws.services.s3.model.GetObjectRequest;
 import com.amazonaws.services.s3.model.GetS3AccountOwnerRequest;
@@ -253,6 +255,11 @@ public class AmazonS3Wrapper implements AmazonS3 {
         return delegate.getObjectAcl(bucketName, key, versionId);
     }
 
+    @Override
+    public AccessControlList getObjectAcl(GetObjectAclRequest getObjectAclRequest) throws AmazonClientException, AmazonServiceException {
+        return delegate.getObjectAcl(getObjectAclRequest);
+    }
+
     @Override
     public void setObjectAcl(String bucketName, String key, AccessControlList acl) throws AmazonClientException, AmazonServiceException {
         delegate.setObjectAcl(bucketName, key, acl);
@@ -358,6 +365,17 @@ public class AmazonS3Wrapper implements AmazonS3 {
         delegate.deleteBucketReplicationConfiguration(bucketName);
     }
 
+    @Override
+    public void deleteBucketReplicationConfiguration(DeleteBucketReplicationConfigurationRequest request) throws AmazonServiceException,
+        AmazonClientException {
+        delegate.deleteBucketReplicationConfiguration(request);
+    }
+
+    @Override
+    public boolean doesObjectExist(String bucketName, String objectName) throws AmazonServiceException, AmazonClientException {
+        return delegate.doesObjectExist(bucketName, objectName);
+    }
+
     @Override
     public PutObjectResult putObject(PutObjectRequest putObjectRequest) throws AmazonClientException, AmazonServiceException {
         return delegate.putObject(putObjectRequest);

From 44080a007f08a90d7eb266429a3826b074ab88bf Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Fri, 15 Apr 2016 14:52:27 +0200
Subject: [PATCH 0004/1311] Add cloud.aws.s3.throttle_retries setting

Defaults to `true`.

If anyone is having trouble with this option, you could disable it with `cloud.aws.s3.throttle_retries: false` in `elasticsearch.yml` file.
---
 .../main/java/org/elasticsearch/cloud/aws/AwsS3Service.java   | 4 ++++
 .../org/elasticsearch/cloud/aws/InternalAwsS3Service.java     | 1 +
 .../plugin/repository/s3/S3RepositoryPlugin.java              | 1 +
 3 files changed, 6 insertions(+)

diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
index 427c454fa28..383be6825c2 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
@@ -151,6 +151,10 @@ public interface AwsS3Service extends LifecycleComponent<AwsS3Service> {
          * cloud.aws.s3.endpoint: Endpoint. If not set, endpoint will be guessed based on region setting.
          */
         Setting<String> ENDPOINT_SETTING = Setting.simpleString("cloud.aws.s3.endpoint", Property.NodeScope);
+        /**
+         * cloud.aws.s3.throttle_retries: Set to `true` if you want to throttle retries. Defaults to `true`.
+         */
+        Setting<Boolean> THROTTLE_RETRIES_SETTING = Setting.boolSetting("cloud.aws.s3.throttle_retries", true, Property.NodeScope);
     }
 
     AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries);
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
index 81b6463a746..67fae30657d 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
@@ -101,6 +101,7 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
         if (maxRetries != null) {
             // If not explicitly set, default to 3 with exponential backoff policy
             clientConfiguration.setMaxErrorRetry(maxRetries);
+            clientConfiguration.setUseThrottleRetries(AwsS3Service.CLOUD_S3.THROTTLE_RETRIES_SETTING.get(settings));
         }
 
         // #155: we might have 3rd party users using older S3 API version
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
index d07d8c174c5..4da73bf3353 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
@@ -112,6 +112,7 @@ public class S3RepositoryPlugin extends Plugin {
         settingsModule.registerSetting(AwsS3Service.CLOUD_S3.SIGNER_SETTING);
         settingsModule.registerSetting(AwsS3Service.CLOUD_S3.REGION_SETTING);
         settingsModule.registerSetting(AwsS3Service.CLOUD_S3.ENDPOINT_SETTING);
+        settingsModule.registerSetting(AwsS3Service.CLOUD_S3.THROTTLE_RETRIES_SETTING);
 
         // Register S3 repositories settings: repositories.s3
         settingsModule.registerSetting(S3Repository.Repositories.KEY_SETTING);

From a8a33a1a942f3aad7fe9addc8587bfefcb2a8abb Mon Sep 17 00:00:00 2001
From: Alexander Kazakov <alexshadow007@gmail.com>
Date: Wed, 20 Apr 2016 19:10:01 +0300
Subject: [PATCH 0005/1311] Row-centric output for _cat/fielddata

---
 .../rest/action/cat/RestFielddataAction.java  | 63 +++++--------------
 docs/reference/cat/fielddata.asciidoc         | 33 +++++-----
 .../test/cat.fielddata/10_basic.yaml          | 37 ++++++-----
 3 files changed, 54 insertions(+), 79 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestFielddataAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestFielddataAction.java
index 46e75ae7de5..8febf04e892 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestFielddataAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestFielddataAction.java
@@ -19,8 +19,7 @@
 
 package org.elasticsearch.rest.action.cat;
 
-import com.carrotsearch.hppc.ObjectLongHashMap;
-import com.carrotsearch.hppc.ObjectLongMap;
+import com.carrotsearch.hppc.cursors.ObjectLongCursor;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
@@ -36,11 +35,6 @@ import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.support.RestResponseListener;
 import org.elasticsearch.rest.action.support.RestTable;
 
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 /**
@@ -57,7 +51,6 @@ public class RestFielddataAction extends AbstractCatAction {
 
     @Override
     protected void doRequest(final RestRequest request, final RestChannel channel, final Client client) {
-
         final NodesStatsRequest nodesStatsRequest = new NodesStatsRequest("data:true");
         nodesStatsRequest.clear();
         nodesStatsRequest.indices(true);
@@ -86,56 +79,30 @@ public class RestFielddataAction extends AbstractCatAction {
                 .addCell("host", "alias:h;desc:host name")
                 .addCell("ip", "desc:ip address")
                 .addCell("node", "alias:n;desc:node name")
-                .addCell("total", "text-align:right;desc:total field data usage")
+                .addCell("field", "alias:f;desc:field name")
+                .addCell("size", "text-align:right;alias:s;desc:field data usage")
                 .endHeaders();
         return table;
     }
 
     private Table buildTable(final RestRequest request, final NodesStatsResponse nodeStatses) {
-        Set<String> fieldNames = new HashSet<>();
-        Map<NodeStats, ObjectLongMap<String>> nodesFields = new HashMap<>();
+        Table table = getTableWithHeader(request);
 
-        // Collect all the field names so a new table can be built
-        for (NodeStats ns : nodeStatses.getNodes()) {
-            ObjectLongHashMap<String> fields = ns.getIndices().getFieldData().getFields();
-            nodesFields.put(ns, fields);
-            if (fields != null) {
-                for (String key : fields.keys().toArray(String.class)) {
-                    fieldNames.add(key);
+        for (NodeStats nodeStats: nodeStatses.getNodes()) {
+            if (nodeStats.getIndices().getFieldData().getFields() != null) {
+                for (ObjectLongCursor<String> cursor : nodeStats.getIndices().getFieldData().getFields()) {
+                    table.startRow();
+                    table.addCell(nodeStats.getNode().getId());
+                    table.addCell(nodeStats.getNode().getHostName());
+                    table.addCell(nodeStats.getNode().getHostAddress());
+                    table.addCell(nodeStats.getNode().getName());
+                    table.addCell(cursor.key);
+                    table.addCell(new ByteSizeValue(cursor.value));
+                    table.endRow();
                 }
             }
         }
 
-        // The table must be rebuilt because it has dynamic headers based on the fields
-        Table table = new Table();
-        table.startHeaders()
-                .addCell("id", "desc:node id")
-                .addCell("host", "alias:h;desc:host name")
-                .addCell("ip", "desc:ip address")
-                .addCell("node", "alias:n;desc:node name")
-                .addCell("total", "text-align:right;desc:total field data usage");
-        // The table columns must be built dynamically since the number of fields is unknown
-        for (String fieldName : fieldNames) {
-            table.addCell(fieldName, "text-align:right;desc:" + fieldName + " field");
-        }
-        table.endHeaders();
-
-        for (Map.Entry<NodeStats, ObjectLongMap<String>> statsEntry : nodesFields.entrySet()) {
-            table.startRow();
-            // add the node info and field data total before each individual field
-            NodeStats ns = statsEntry.getKey();
-            table.addCell(ns.getNode().getId());
-            table.addCell(ns.getNode().getHostName());
-            table.addCell(ns.getNode().getHostAddress());
-            table.addCell(ns.getNode().getName());
-            table.addCell(ns.getIndices().getFieldData().getMemorySize());
-            ObjectLongMap<String> fields = statsEntry.getValue();
-            for (String fieldName : fieldNames) {
-                table.addCell(new ByteSizeValue(fields == null ? 0L : fields.getOrDefault(fieldName, 0L)));
-            }
-            table.endRow();
-        }
-
         return table;
     }
 }
diff --git a/docs/reference/cat/fielddata.asciidoc b/docs/reference/cat/fielddata.asciidoc
index 250d0b39eaf..854853e4d39 100644
--- a/docs/reference/cat/fielddata.asciidoc
+++ b/docs/reference/cat/fielddata.asciidoc
@@ -7,10 +7,13 @@ on every data node in the cluster.
 [source,sh]
 --------------------------------------------------
 % curl '192.168.56.10:9200/_cat/fielddata?v'
-id                     host    ip            node          total   body    text
-c223lARiSGeezlbrcugAYQ myhost1 10.20.100.200 Jessica Jones 385.6kb 159.8kb 225.7kb
-waPCbitNQaCL6xC8VxjAwg myhost2 10.20.100.201 Adversary     435.2kb 159.8kb 275.3kb
-yaDkp-G3R0q1AJ-HUEvkSQ myhost3 10.20.100.202 Microchip     284.6kb 109.2kb 175.3kb
+id                     host    ip            node          field   size
+c223lARiSGeezlbrcugAYQ myhost1 10.20.100.200 Jessica Jones body    159.8kb
+c223lARiSGeezlbrcugAYQ myhost1 10.20.100.200 Jessica Jones text    225.7kb
+waPCbitNQaCL6xC8VxjAwg myhost2 10.20.100.201 Adversary     body    159.8kb
+waPCbitNQaCL6xC8VxjAwg myhost2 10.20.100.201 Adversary     text    275.3kb
+yaDkp-G3R0q1AJ-HUEvkSQ myhost3 10.20.100.202 Microchip     body    109.2kb
+yaDkp-G3R0q1AJ-HUEvkSQ myhost3 10.20.100.202 Microchip     text    175.3kb
 --------------------------------------------------
 
 Fields can be specified either as a query parameter, or in the URL path:
@@ -18,17 +21,19 @@ Fields can be specified either as a query parameter, or in the URL path:
 [source,sh]
 --------------------------------------------------
 % curl '192.168.56.10:9200/_cat/fielddata?v&fields=body'
-id                     host    ip            node          total   body
-c223lARiSGeezlbrcugAYQ myhost1 10.20.100.200 Jessica Jones 385.6kb 159.8kb
-waPCbitNQaCL6xC8VxjAwg myhost2 10.20.100.201 Adversary     435.2kb 159.8kb
-yaDkp-G3R0q1AJ-HUEvkSQ myhost3 10.20.100.202 Microchip     284.6kb 109.2kb
+id                     host    ip            node          field   size
+c223lARiSGeezlbrcugAYQ myhost1 10.20.100.200 Jessica Jones body    159.8kb
+waPCbitNQaCL6xC8VxjAwg myhost2 10.20.100.201 Adversary     body    159.8kb
+yaDkp-G3R0q1AJ-HUEvkSQ myhost3 10.20.100.202 Microchip     body    109.2kb
 
 % curl '192.168.56.10:9200/_cat/fielddata/body,text?v'
-id                     host    ip            node          total   body    text
-c223lARiSGeezlbrcugAYQ myhost1 10.20.100.200 Jessica Jones 385.6kb 159.8kb 225.7kb
-waPCbitNQaCL6xC8VxjAwg myhost2 10.20.100.201 Adversary     435.2kb 159.8kb 275.3kb
-yaDkp-G3R0q1AJ-HUEvkSQ myhost3 10.20.100.202 Microchip     284.6kb 109.2kb 175.3kb
+id                     host    ip            node          field   size
+c223lARiSGeezlbrcugAYQ myhost1 10.20.100.200 Jessica Jones body    159.8kb
+c223lARiSGeezlbrcugAYQ myhost1 10.20.100.200 Jessica Jones text    225.7kb
+waPCbitNQaCL6xC8VxjAwg myhost2 10.20.100.201 Adversary     body    159.8kb
+waPCbitNQaCL6xC8VxjAwg myhost2 10.20.100.201 Adversary     text    275.3kb
+yaDkp-G3R0q1AJ-HUEvkSQ myhost3 10.20.100.202 Microchip     body    109.2kb
+yaDkp-G3R0q1AJ-HUEvkSQ myhost3 10.20.100.202 Microchip     text    175.3kb
 --------------------------------------------------
 
-The output shows the total fielddata and then the individual fielddata for the
-`body` and `text` fields.
+The output shows the individual fielddata for the`body` and `text` fields, one row per field per node.
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.fielddata/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.fielddata/10_basic.yaml
index cd3a53a103c..363ff9b477c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.fielddata/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.fielddata/10_basic.yaml
@@ -10,7 +10,8 @@
                    host .+ \n
                    ip .+ \n
                    node .+ \n
-                   total .+ \n
+                   field .+ \n
+                   size .+ \n
                $/
 
 ---
@@ -38,39 +39,41 @@
         type: type
         body: { foo: bar }
         refresh: true
+
   - do:
       search:
         index: index
         body:
           query: { match_all: {} }
           sort: foo
+
   - do:
       cat.fielddata:
-        h: total
+        h: field,size
         v: true
 
   - match:
       $body: |
-               /^   total               \n
-                    (\s*\d+(\.\d+)?[gmk]?b  \n)+ $/
+               /^ field \s+   size \n
+                  foo     \s+ (\d+(\.\d+)?[gmk]?b  \n)+ $/
 
   - do:
       cat.fielddata:
-        h: total,foo
-        v: true
-
-  - match:
-      $body: |
-               /^   total \s+              foo \n
-                    (\s*\d+(\.\d+)?[gmk]?b \s+ \d+(\.\d+)?[gmk]?b \n)+ $/
-
-  - do:
-      cat.fielddata:
-        h: total,foo
+        h: field,size
         fields: notfoo,foo
         v: true
 
   - match:
       $body: |
-               /^   total \s+              foo \n
-                    (\s*\d+(\.\d+)?[gmk]?b \s+ \d+(\.\d+)?[gmk]?b \n)+ $/
+               /^ field \s+   size \n
+                  foo     \s+ (\d+(\.\d+)?[gmk]?b  \n)+ $/
+
+  - do:
+      cat.fielddata:
+        h: field,size
+        fields: notfoo
+        v: true
+
+  - match:
+      $body: |
+               /^ field \s+   size \n $/

From 6ef81c5dcdad0a65c350379dd12daadcc7ffaae5 Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Wed, 27 Apr 2016 14:11:17 +0200
Subject: [PATCH 0006/1311] S3 repositories credentials should be filtered

When working on #18008 I found while reading the code that we don't filter anymore `repositories.s3.access_key` and `repositories.s3.secret_key`.

Also fixed a typo in REST test
---
 .../java/org/elasticsearch/repositories/s3/S3Repository.java  | 4 ++--
 .../rest-api-spec/test/repository_s3/20_repository.yaml       | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
index a09d57ebc93..8fef5f8e893 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
@@ -67,13 +67,13 @@ public class S3Repository extends BlobStoreRepository {
          * @see CLOUD_S3#KEY_SETTING
          */
         Setting<String> KEY_SETTING =
-            new Setting<>("repositories.s3.access_key", CLOUD_S3.KEY_SETTING, Function.identity(), Property.NodeScope);
+            new Setting<>("repositories.s3.access_key", CLOUD_S3.KEY_SETTING, Function.identity(), Property.NodeScope, Property.Filtered);
         /**
          * repositories.s3.secret_key: AWS Secret key specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.secret_key.
          * @see CLOUD_S3#SECRET_SETTING
          */
         Setting<String> SECRET_SETTING =
-            new Setting<>("repositories.s3.secret_key", CLOUD_S3.SECRET_SETTING, Function.identity(), Property.NodeScope);
+            new Setting<>("repositories.s3.secret_key", CLOUD_S3.SECRET_SETTING, Function.identity(), Property.NodeScope, Property.Filtered);
         /**
          * repositories.s3.region: Region specific for all S3 Repositories API calls. Defaults to cloud.aws.s3.region.
          * @see CLOUD_S3#REGION_SETTING
diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml
index 34384653a74..eb909f91f1d 100644
--- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml
+++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml
@@ -1,6 +1,6 @@
 # Integration tests for Repository S3 component
 #
-"S3 repository can be registereed":
+"S3 repository can be registered":
     - do:
         snapshot.create_repository:
           repository: test_repo_s3_1

From 78ff4f52d605c109efd60039b9ef600a072c24ff Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Thu, 28 Apr 2016 12:10:56 +0200
Subject: [PATCH 0007/1311] Introduces GeoValidationMethod to
 GeoDistanceSortBuilder

Previously like in other geo related query parsers we were using
a combination of two booleans for coerce and ignore_malformed
which was error prone and not very clear.

Switched to using GeoValidationMethod instead as we already do
e.g. in GeoBoundingBoxQueryBuilder.

Left support for both, coerce and ignore_malformed in the parser
but deprecated the two in favour of validation method.

Introduced the same deprecation in geo bounding box query builder.
---
 .../query/GeoBoundingBoxQueryBuilder.java     |   6 +-
 .../search/sort/GeoDistanceSortBuilder.java   | 102 +++++++++---------
 .../search/sort/GeoDistanceSortBuilderIT.java |   3 +-
 .../sort/GeoDistanceSortBuilderTests.java     |  74 ++++++++++---
 4 files changed, 114 insertions(+), 71 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
index db743b971c7..7f5d5e2ec8d 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
@@ -64,10 +64,12 @@ public class GeoBoundingBoxQueryBuilder extends AbstractQueryBuilder<GeoBounding
      */
     public static final boolean DEFAULT_IGNORE_UNMAPPED = false;
 
-    private static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed");
     private static final ParseField TYPE_FIELD = new ParseField("type");
     private static final ParseField VALIDATION_METHOD_FIELD = new ParseField("validation_method");
-    private static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize");
+    private static final ParseField COERCE_FIELD =new ParseField("coerce", "normalize")
+            .withAllDeprecated("use field validation_method instead");
+    private static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed")
+            .withAllDeprecated("use field validation_method instead");
     private static final ParseField FIELD_FIELD = new ParseField("field");
     private static final ParseField TOP_FIELD = new ParseField("top");
     private static final ParseField BOTTOM_FIELD = new ParseField("bottom");
diff --git a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
index e6570543c83..2e08e38a96a 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
@@ -47,6 +47,7 @@ import org.elasticsearch.index.fielddata.MultiGeoPointValues;
 import org.elasticsearch.index.fielddata.NumericDoubleValues;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.query.GeoValidationMethod;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
@@ -65,16 +66,18 @@ import java.util.Objects;
 public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder> {
     public static final String NAME = "_geo_distance";
     public static final String ALTERNATIVE_NAME = "_geoDistance";
-    public static final boolean DEFAULT_COERCE = false;
-    public static final boolean DEFAULT_IGNORE_MALFORMED = false;
-    public static final ParseField UNIT_FIELD = new ParseField("unit");
-    public static final ParseField DISTANCE_TYPE_FIELD = new ParseField("distance_type");
-    public static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize");
-    public static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed");
-    public static final ParseField SORTMODE_FIELD = new ParseField("mode", "sort_mode");
-    public static final ParseField NESTED_PATH_FIELD = new ParseField("nested_path");
-    public static final ParseField NESTED_FILTER_FIELD = new ParseField("nested_filter");
-    public static final ParseField REVERSE_FORBIDDEN = new ParseField("reverse");
+    public static final GeoValidationMethod DEFAULT_VALIDATION = GeoValidationMethod.DEFAULT;
+
+    private static final ParseField UNIT_FIELD = new ParseField("unit");
+    private static final ParseField DISTANCE_TYPE_FIELD = new ParseField("distance_type");
+    private static final ParseField VALIDATION_METHOD_FIELD = new ParseField("validation_method");
+    private static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed")
+            .withAllDeprecated("use validation_method instead");
+    private static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize")
+            .withAllDeprecated("use validation_method instead");
+    private static final ParseField SORTMODE_FIELD = new ParseField("mode", "sort_mode");
+    private static final ParseField NESTED_PATH_FIELD = new ParseField("nested_path");
+    private static final ParseField NESTED_FILTER_FIELD = new ParseField("nested_filter");
 
     private final String fieldName;
     private final List<GeoPoint> points = new ArrayList<>();
@@ -87,9 +90,7 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
     private QueryBuilder nestedFilter;
     private String nestedPath;
 
-    // TODO switch to GeoValidationMethod enum
-    private boolean coerce = DEFAULT_COERCE;
-    private boolean ignoreMalformed = DEFAULT_IGNORE_MALFORMED;
+    private GeoValidationMethod validation = DEFAULT_VALIDATION;
 
     /**
      * Constructs a new distance based sort on a geo point like field.
@@ -144,8 +145,7 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
         this.sortMode = original.sortMode;
         this.nestedFilter = original.nestedFilter;
         this.nestedPath = original.nestedPath;
-        this.coerce = original.coerce;
-        this.ignoreMalformed = original.ignoreMalformed;
+        this.validation = original.validation;
     }
 
     /**
@@ -161,8 +161,7 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
         sortMode = in.readOptionalWriteable(SortMode::readFromStream);
         nestedFilter = in.readOptionalNamedWriteable(QueryBuilder.class);
         nestedPath = in.readOptionalString();
-        coerce = in.readBoolean();
-        ignoreMalformed =in.readBoolean();
+        validation = GeoValidationMethod.readFromStream(in);
     }
 
     @Override
@@ -175,8 +174,7 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
         out.writeOptionalWriteable(sortMode);
         out.writeOptionalNamedWriteable(nestedFilter);
         out.writeOptionalString(nestedPath);
-        out.writeBoolean(coerce);
-        out.writeBoolean(ignoreMalformed);
+        validation.writeTo(out);
     }
 
     /**
@@ -257,6 +255,21 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
         return this.unit;
     }
 
+    /**
+     * Sets validation method for this sort builder. 
+     */
+    public GeoDistanceSortBuilder validation(GeoValidationMethod method) {
+        this.validation = method;
+        return this;
+    }
+
+    /**
+     * Returns the validation method to use for this sort builder. 
+     */
+    public GeoValidationMethod validation() {
+        return validation;
+    }
+
     /**
      * Defines which distance to use for sorting in the case a document contains multiple geo points.
      * Possible values: min and max
@@ -309,26 +322,6 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
         return this.nestedPath;
     }
 
-    public GeoDistanceSortBuilder coerce(boolean coerce) {
-        this.coerce = coerce;
-        return this;
-    }
-
-    public boolean coerce() {
-        return this.coerce;
-    }
-
-    public GeoDistanceSortBuilder ignoreMalformed(boolean ignoreMalformed) {
-        if (coerce == false) {
-            this.ignoreMalformed = ignoreMalformed;
-        }
-        return this;
-    }
-
-    public boolean ignoreMalformed() {
-        return this.ignoreMalformed;
-    }
-
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
@@ -354,8 +347,7 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
         if (nestedFilter != null) {
             builder.field(NESTED_FILTER_FIELD.getPreferredName(), nestedFilter, params);
         }
-        builder.field(COERCE_FIELD.getPreferredName(), coerce);
-        builder.field(IGNORE_MALFORMED_FIELD.getPreferredName(), ignoreMalformed);
+        builder.field(VALIDATION_METHOD_FIELD.getPreferredName(), validation);
 
         builder.endObject();
         builder.endObject();
@@ -386,14 +378,14 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
                 Objects.equals(order, other.order) &&
                 Objects.equals(nestedFilter, other.nestedFilter) &&
                 Objects.equals(nestedPath, other.nestedPath) &&
-                Objects.equals(coerce, other.coerce) &&
-                Objects.equals(ignoreMalformed, other.ignoreMalformed);
+                Objects.equals(validation, other.validation);
     }
 
     @Override
     public int hashCode() {
         return Objects.hash(this.fieldName, this.points, this.geoDistance,
-                this.unit, this.sortMode, this.order, this.nestedFilter, this.nestedPath, this.coerce, this.ignoreMalformed);
+                this.unit, this.sortMode, this.order, this.nestedFilter,
+                this.nestedPath, this.validation);
     }
 
     /**
@@ -417,8 +409,9 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
         QueryBuilder<?> nestedFilter = null;
         String nestedPath = null;
 
-        boolean coerce = GeoDistanceSortBuilder.DEFAULT_COERCE;
-        boolean ignoreMalformed = GeoDistanceSortBuilder.DEFAULT_IGNORE_MALFORMED;
+        boolean coerce = GeoValidationMethod.DEFAULT_LENIENT_PARSING;
+        boolean ignoreMalformed = GeoValidationMethod.DEFAULT_LENIENT_PARSING;
+        GeoValidationMethod validation = null;
 
         XContentParser.Token token;
         String currentName = parser.currentName();
@@ -463,6 +456,8 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
                     if (coerce == false) {
                         ignoreMalformed = ignore_malformed_value;
                     }
+                } else if (parseFieldMatcher.match(currentName, VALIDATION_METHOD_FIELD)) {
+                    validation = GeoValidationMethod.fromString(parser.text());
                 } else if (parseFieldMatcher.match(currentName, SORTMODE_FIELD)) {
                     sortMode = SortMode.fromString(parser.text());
                 } else if (parseFieldMatcher.match(currentName, NESTED_PATH_FIELD)) {
@@ -498,8 +493,13 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
         }
         result.setNestedFilter(nestedFilter);
         result.setNestedPath(nestedPath);
-        result.coerce(coerce);
-        result.ignoreMalformed(ignoreMalformed);
+        if (validation == null) {
+            // looks like either validation was left unset or we are parsing old validation json
+            result.validation(GeoValidationMethod.infer(coerce, ignoreMalformed));
+        } else {
+            // ignore deprecated coerce/ignore_malformed
+            result.validation(validation);
+        }
         return result;
     }
 
@@ -512,7 +512,7 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
             localPoints.add(new GeoPoint(geoPoint));
         }
 
-        if (!indexCreatedBeforeV2_0 && !ignoreMalformed) {
+        if (!indexCreatedBeforeV2_0 && !GeoValidationMethod.isIgnoreMalformed(validation)) {
             for (GeoPoint point : localPoints) {
                 if (GeoUtils.isValidLatitude(point.lat()) == false) {
                     throw new ElasticsearchParseException(
@@ -529,9 +529,9 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
             }
         }
 
-        if (coerce) {
+        if (GeoValidationMethod.isCoerce(validation)) {
             for (GeoPoint point : localPoints) {
-                GeoUtils.normalizePoint(point, coerce, coerce);
+                GeoUtils.normalizePoint(point, true, true);
             }
         }
 
diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java
index 31c1d5793e0..5229c905bf8 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java
@@ -28,6 +28,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.GeoValidationMethod;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -314,7 +315,7 @@ public class GeoDistanceSortBuilderIT extends ESIntegTestCase {
                 .setSource(
                         new SearchSourceBuilder().sort(SortBuilders.geoDistanceSort(LOCATION_FIELD, 2.0, 2.0)
                                 .unit(DistanceUnit.KILOMETERS).geoDistance(GeoDistance.PLANE)
-                                .ignoreMalformed(true).coerce(true))).execute().actionGet();
+                                .validation(GeoValidationMethod.COERCE))).execute().actionGet();
         checkCorrectSortOrderForGeoSort(searchResponse);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
index 87fd183f1ce..f65050ca9ce 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
@@ -33,6 +33,7 @@ import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
+import org.elasticsearch.index.query.GeoValidationMethod;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.test.geo.RandomGeoGenerator;
@@ -94,10 +95,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
             result.setNestedPath(RandomSortDataGenerator.randomAscii(result.getNestedPath()));
         }
         if (randomBoolean()) {
-            result.coerce(! result.coerce());
-        }
-        if (randomBoolean()) {
-            result.ignoreMalformed(! result.ignoreMalformed());
+            result.validation(randomFrom(GeoValidationMethod.values()));
         }
 
         return result;
@@ -118,6 +116,14 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
         return result;
     }
 
+    private static GeoValidationMethod validation(GeoValidationMethod original) {
+        GeoValidationMethod result;
+        do {
+            result = randomFrom(GeoValidationMethod.values());
+        } while (result == original);
+        return result;
+    }
+
     private static DistanceUnit unit(DistanceUnit original) {
         int id = -1;
         while (id == -1 || (original != null && original.ordinal() == id)) {
@@ -149,7 +155,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
     @Override
     protected GeoDistanceSortBuilder mutate(GeoDistanceSortBuilder original) throws IOException {
         GeoDistanceSortBuilder result = new GeoDistanceSortBuilder(original);
-        int parameter = randomIntBetween(0, 9);
+        int parameter = randomIntBetween(0, 8);
         switch (parameter) {
         case 0:
             while (Arrays.deepEquals(original.points(), result.points())) {
@@ -179,12 +185,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
             result.setNestedPath(RandomSortDataGenerator.randomAscii(original.getNestedPath()));
             break;
         case 8:
-            result.coerce(! original.coerce());
-            break;
-        case 9:
-            // ignore malformed will only be set if coerce is set to true
-            result.coerce(false);
-            result.ignoreMalformed(! original.ignoreMalformed());
+            result.validation(validation(original.validation()));
             break;
         }
         return result;
@@ -262,7 +263,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
 
         try {
           GeoDistanceSortBuilder item = GeoDistanceSortBuilder.fromXContent(context, "");
-          item.ignoreMalformed(false);
+          item.validation(GeoValidationMethod.STRICT);
           item.build(createMockShardContext());
 
           fail("adding reverse sorting option should fail with an exception");
@@ -270,6 +271,48 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
             assertEquals("illegal latitude value [269.384765625] for [GeoDistanceSort] for field [reverse].", e.getMessage());
         }
     }
+    
+    public void testCoerceIsDeprecated() throws IOException {
+        String json = "{\n" +
+                "  \"testname\" : [ {\n" +
+                "    \"lat\" : -6.046997540714173,\n" +
+                "    \"lon\" : -51.94128329747579\n" +
+                "  } ],\n" +
+                "  \"unit\" : \"m\",\n" +
+                "  \"distance_type\" : \"sloppy_arc\",\n" +
+                "  \"mode\" : \"SUM\",\n" +
+                "  \"coerce\" : true\n" +
+                "}";
+        XContentParser itemParser = XContentHelper.createParser(new BytesArray(json));
+        itemParser.nextToken();
+
+        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> GeoDistanceSortBuilder.fromXContent(context, ""));
+        assertTrue(e.getMessage().startsWith("Deprecated field "));
+        
+    }
+
+    public void testIgnoreMalformedIsDeprecated() throws IOException {
+        String json = "{\n" +
+                "  \"testname\" : [ {\n" +
+                "    \"lat\" : -6.046997540714173,\n" +
+                "    \"lon\" : -51.94128329747579\n" +
+                "  } ],\n" +
+                "  \"unit\" : \"m\",\n" +
+                "  \"distance_type\" : \"sloppy_arc\",\n" +
+                "  \"mode\" : \"SUM\",\n" +
+                "  \"ignore_malformed\" : true\n" +
+                "}";
+        XContentParser itemParser = XContentHelper.createParser(new BytesArray(json));
+        itemParser.nextToken();
+
+        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, itemParser, ParseFieldMatcher.STRICT);
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> GeoDistanceSortBuilder.fromXContent(context, ""));
+        assertTrue(e.getMessage().startsWith("Deprecated field "));
+        
+    }
 
     public void testSortModeSumIsRejectedInJSON() throws IOException {
         String json = "{\n" +
@@ -279,9 +322,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
                 "  } ],\n" +
                 "  \"unit\" : \"m\",\n" +
                 "  \"distance_type\" : \"sloppy_arc\",\n" +
-                "  \"mode\" : \"SUM\",\n" +
-                "  \"coerce\" : false,\n" +
-                "  \"ignore_malformed\" : false\n" +
+                "  \"mode\" : \"SUM\"\n" +
                 "}";
         XContentParser itemParser = XContentHelper.createParser(new BytesArray(json));
         itemParser.nextToken();
@@ -306,8 +347,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
                 "        \"boost\" : 5.711116\n" +
                 "      }\n" +
                 "    },\n" +
-                "    \"coerce\" : false,\n" +
-                "    \"ignore_malformed\" : true\n" +
+                "    \"validation_method\" : \"STRICT\"\n" +
                 "  }";
         XContentParser itemParser = XContentHelper.createParser(new BytesArray(json));
         itemParser.nextToken();

From 5306de3ce32f0682cde398090d5875cc5478dcbc Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Thu, 28 Apr 2016 13:56:50 +0200
Subject: [PATCH 0008/1311] Deprecate coerce/ignore_malformed for
 GeoPolygonQueryBuilder

Includes update to parsing code, tests, migration docs and reference
docs.
---
 .../index/query/GeoPolygonQueryBuilder.java   | 10 ++---
 .../query/GeoPolygonQueryBuilderTests.java    | 37 +++++++++++++++++--
 .../migration/migrate_5_0/search.asciidoc     |  1 +
 .../query-dsl/geo-polygon-query.asciidoc      |  5 ++-
 4 files changed, 43 insertions(+), 10 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java
index d415c87b5ab..35ab98e25de 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java
@@ -54,8 +54,10 @@ public class GeoPolygonQueryBuilder extends AbstractQueryBuilder<GeoPolygonQuery
      */
     public static final boolean DEFAULT_IGNORE_UNMAPPED = false;
 
-    private static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize");
-    private static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed");
+    private static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize")
+            .withAllDeprecated("use validation_method instead");
+    private static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed")
+            .withAllDeprecated("use validation_method instead");
     private static final ParseField VALIDATION_METHOD = new ParseField("validation_method");
     private static final ParseField POINTS_FIELD = new ParseField("points");
     private static final ParseField IGNORE_UNMAPPED_FIELD = new ParseField("ignore_unmapped");
@@ -232,9 +234,7 @@ public class GeoPolygonQueryBuilder extends AbstractQueryBuilder<GeoPolygonQuery
         builder.endArray();
         builder.endObject();
 
-        builder.field(COERCE_FIELD.getPreferredName(), GeoValidationMethod.isCoerce(validationMethod));
-        builder.field(IGNORE_MALFORMED_FIELD.getPreferredName(),
-                GeoValidationMethod.isIgnoreMalformed(validationMethod));
+        builder.field(VALIDATION_METHOD.getPreferredName(), validationMethod);
         builder.field(IGNORE_UNMAPPED_FIELD.getPreferredName(), ignoreUnmapped);
 
         printBoostAndQueryName(builder);
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
index 55b7f24770c..9313b4278b0 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
@@ -207,7 +207,7 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase<GeoPolygo
             parseQuery(builder.string());
             fail("normalize is deprecated");
         } catch (IllegalArgumentException ex) {
-            assertEquals("Deprecated field [normalize] used, expected [coerce] instead", ex.getMessage());
+            assertEquals("Deprecated field [normalize] used, replaced by [use validation_method instead]", ex.getMessage());
         }
     }
 
@@ -342,8 +342,7 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase<GeoPolygo
                 "    \"person.location\" : {\n" +
                 "      \"points\" : [ [ -70.0, 40.0 ], [ -80.0, 30.0 ], [ -90.0, 20.0 ], [ -70.0, 40.0 ] ]\n" +
                 "    },\n" +
-                "    \"coerce\" : false,\n" +
-                "    \"ignore_malformed\" : false,\n" +
+                "    \"validation_method\" : \"STRICT\",\n" +
                 "    \"ignore_unmapped\" : false,\n" +
                 "    \"boost\" : 1.0\n" +
                 "  }\n" +
@@ -353,6 +352,38 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase<GeoPolygo
         assertEquals(json, 4, parsed.points().size());
     }
 
+    public void testFromJsonIgnoreMalformedDeprecated() throws IOException {
+        String json =
+                "{\n" +
+                "  \"geo_polygon\" : {\n" +
+                "    \"person.location\" : {\n" +
+                "      \"points\" : [ [ -70.0, 40.0 ], [ -80.0, 30.0 ], [ -90.0, 20.0 ], [ -70.0, 40.0 ] ]\n" +
+                "    },\n" +
+                "    \"ignore_malformed\" : false,\n" +
+                "    \"boost\" : 1.0\n" +
+                "  }\n" +
+                "}";
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
+        assertTrue(e.getMessage().startsWith("Deprecated field "));
+
+    }
+
+    public void testFromJsonCoerceDeprecated() throws IOException {
+        String json =
+                "{\n" +
+                "  \"geo_polygon\" : {\n" +
+                "    \"person.location\" : {\n" +
+                "      \"points\" : [ [ -70.0, 40.0 ], [ -80.0, 30.0 ], [ -90.0, 20.0 ], [ -70.0, 40.0 ] ]\n" +
+                "    },\n" +
+                "    \"coerce\" : false,\n" +
+                "    \"ignore_unmapped\" : false,\n" +
+                "    \"boost\" : 1.0\n" +
+                "  }\n" +
+                "}";
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
+        assertTrue(e.getMessage().startsWith("Deprecated field "));
+    }
+
     @Override
     public void testMustRewrite() throws IOException {
         assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
diff --git a/docs/reference/migration/migrate_5_0/search.asciidoc b/docs/reference/migration/migrate_5_0/search.asciidoc
index 2d22ee1c1c3..a2c56996c2b 100644
--- a/docs/reference/migration/migrate_5_0/search.asciidoc
+++ b/docs/reference/migration/migrate_5_0/search.asciidoc
@@ -126,6 +126,7 @@ in favour of `query` and `no_match_query`.
 
 * The `exists` query will now fail if the `_field_names` field is disabled.
 
+* Deprecated support for the coerce, normalize, ignore_malformed parameters in GeoPolygonQuery. Use parameter validation_method instead.
 
 ==== Top level `filter` parameter
 
diff --git a/docs/reference/query-dsl/geo-polygon-query.asciidoc b/docs/reference/query-dsl/geo-polygon-query.asciidoc
index 938a0b3dc87..ef80de82079 100644
--- a/docs/reference/query-dsl/geo-polygon-query.asciidoc
+++ b/docs/reference/query-dsl/geo-polygon-query.asciidoc
@@ -34,8 +34,9 @@ points. Here is an example:
 |Option |Description
 |`_name` |Optional name field to identify the filter
 
-|`ignore_malformed` |Set to `true` to accept geo points with invalid latitude or
-longitude (default is `false`).
+|`validation_method` |Set to `IGNORE_MALFORMED` to accept geo points with
+invalid latitude or longitude, `COERCE` to try and infer correct latitude
+or longitude, or `STRICT` (default is `STRICT`).
 |=======================================================================
 
 [float]

From 3160798084f61e4c613e1bf87f12708a270a8a6a Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Thu, 28 Apr 2016 14:01:54 +0200
Subject: [PATCH 0009/1311] Deprecate coerce/ignore_malformed for
 GeoDistanceRangeQuery

---
 .../query/GeoDistanceRangeQueryBuilder.java   |  6 ++-
 .../query/GeoDistanceRangeQueryTests.java     | 42 +++++++++++++++++++
 .../migration/migrate_5_0/search.asciidoc     |  1 +
 3 files changed, 47 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java
index c23ee423a55..7afbecad4d3 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java
@@ -78,8 +78,10 @@ public class GeoDistanceRangeQueryBuilder extends AbstractQueryBuilder<GeoDistan
     private static final ParseField NAME_FIELD = new ParseField("_name");
     private static final ParseField BOOST_FIELD = new ParseField("boost");
     private static final ParseField OPTIMIZE_BBOX_FIELD = new ParseField("optimize_bbox");
-    private static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize");
-    private static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed");
+    private static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize")
+            .withAllDeprecated("use validation_method instead");
+    private static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed")
+            .withAllDeprecated("use validation_method instead");
     private static final ParseField VALIDATION_METHOD = new ParseField("validation_method");
     private static final ParseField IGNORE_UNMAPPED_FIELD = new ParseField("ignore_unmapped");
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java
index d9aef7270b3..3b3ffa18cab 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java
@@ -357,6 +357,48 @@ public class GeoDistanceRangeQueryTests extends AbstractQueryTestCase<GeoDistanc
         assertEquals(json, -70.0, parsed.point().lon(), 0.0001);
     }
 
+    public void testFromJsonCoerceFails() throws IOException {
+        String json =
+                "{\n" +
+                "  \"geo_distance_range\" : {\n" +
+                "    \"pin.location\" : [ -70.0, 40.0 ],\n" +
+                "    \"from\" : \"200km\",\n" +
+                "    \"to\" : \"400km\",\n" +
+                "    \"include_lower\" : true,\n" +
+                "    \"include_upper\" : true,\n" +
+                "    \"unit\" : \"m\",\n" +
+                "    \"distance_type\" : \"sloppy_arc\",\n" +
+                "    \"optimize_bbox\" : \"memory\",\n" +
+                "    \"coerce\" : true,\n" +
+                "    \"ignore_unmapped\" : false,\n" +
+                "    \"boost\" : 1.0\n" +
+                "  }\n" +
+                "}";
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
+        assertTrue(e.getMessage().startsWith("Deprecated field "));
+    }
+
+    public void testFromJsonIgnoreMalformedFails() throws IOException {
+        String json =
+                "{\n" +
+                "  \"geo_distance_range\" : {\n" +
+                "    \"pin.location\" : [ -70.0, 40.0 ],\n" +
+                "    \"from\" : \"200km\",\n" +
+                "    \"to\" : \"400km\",\n" +
+                "    \"include_lower\" : true,\n" +
+                "    \"include_upper\" : true,\n" +
+                "    \"unit\" : \"m\",\n" +
+                "    \"distance_type\" : \"sloppy_arc\",\n" +
+                "    \"optimize_bbox\" : \"memory\",\n" +
+                "    \"ignore_malformed\" : true,\n" +
+                "    \"ignore_unmapped\" : false,\n" +
+                "    \"boost\" : 1.0\n" +
+                "  }\n" +
+                "}";
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
+        assertTrue(e.getMessage().startsWith("Deprecated field "));
+    }
+
     @Override
     public void testMustRewrite() throws IOException {
         assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
diff --git a/docs/reference/migration/migrate_5_0/search.asciidoc b/docs/reference/migration/migrate_5_0/search.asciidoc
index a2c56996c2b..58435256abb 100644
--- a/docs/reference/migration/migrate_5_0/search.asciidoc
+++ b/docs/reference/migration/migrate_5_0/search.asciidoc
@@ -128,6 +128,7 @@ in favour of `query` and `no_match_query`.
 
 * Deprecated support for the coerce, normalize, ignore_malformed parameters in GeoPolygonQuery. Use parameter validation_method instead.
 
+* Deprecated support for the coerce, normalize, ignore_malformed parameters in GeoDistanceRangeQuery. Use parameter validation_method instead.
 ==== Top level `filter` parameter
 
 Removed support for the deprecated top level `filter` in the search api,

From 3f743a30cfba4aa27de5a468bbbb920fa3a3b800 Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Thu, 28 Apr 2016 14:06:27 +0200
Subject: [PATCH 0010/1311] Deprecate coerce/ignore_malformed in
 GeoDistanceQueryBuilder

---
 .../index/query/GeoDistanceQueryBuilder.java  |  6 ++--
 .../query/GeoDistanceQueryBuilderTests.java   | 34 +++++++++++++++++++
 .../migration/migrate_5_0/search.asciidoc     |  2 ++
 .../query-dsl/geo-distance-query.asciidoc     |  7 ++--
 4 files changed, 44 insertions(+), 5 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java
index a422b6eb6f3..0f2b4694e27 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java
@@ -73,8 +73,10 @@ public class GeoDistanceQueryBuilder extends AbstractQueryBuilder<GeoDistanceQue
     public static final boolean DEFAULT_IGNORE_UNMAPPED = false;
 
     private static final ParseField VALIDATION_METHOD_FIELD = new ParseField("validation_method");
-    private static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed");
-    private static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize");
+    private static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed")
+            .withAllDeprecated("use validation_method instead");
+    private static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize")
+            .withAllDeprecated("use validation_method instead");
     private static final ParseField OPTIMIZE_BBOX_FIELD = new ParseField("optimize_bbox");
     private static final ParseField DISTANCE_TYPE_FIELD = new ParseField("distance_type");
     private static final ParseField UNIT_FIELD = new ParseField("unit");
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java
index 13036ce168d..b57dc61ee4a 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java
@@ -420,6 +420,40 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
         assertEquals(json, 12000.0, parsed.distance(), 0.0001);
     }
 
+    public void testFromCoerceFails() throws IOException {
+        String json =
+                "{\n" +
+                "  \"geo_distance\" : {\n" +
+                "    \"pin.location\" : [ -70.0, 40.0 ],\n" +
+                "    \"distance\" : 12000.0,\n" +
+                "    \"distance_type\" : \"sloppy_arc\",\n" +
+                "    \"optimize_bbox\" : \"memory\",\n" +
+                "    \"coerce\" : true,\n" +
+                "    \"ignore_unmapped\" : false,\n" +
+                "    \"boost\" : 1.0\n" +
+                "  }\n" +
+                "}";
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
+        assertTrue(e.getMessage().startsWith("Deprecated field "));
+    }
+
+    public void testFromJsonIgnoreMalformedFails() throws IOException {
+        String json =
+                "{\n" +
+                "  \"geo_distance\" : {\n" +
+                "    \"pin.location\" : [ -70.0, 40.0 ],\n" +
+                "    \"distance\" : 12000.0,\n" +
+                "    \"distance_type\" : \"sloppy_arc\",\n" +
+                "    \"optimize_bbox\" : \"memory\",\n" +
+                "    \"ignore_malformed\" : true,\n" +
+                "    \"ignore_unmapped\" : false,\n" +
+                "    \"boost\" : 1.0\n" +
+                "  }\n" +
+                "}";
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
+        assertTrue(e.getMessage().startsWith("Deprecated field "));
+    }
+
     @Override
     public void testMustRewrite() throws IOException {
         assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
diff --git a/docs/reference/migration/migrate_5_0/search.asciidoc b/docs/reference/migration/migrate_5_0/search.asciidoc
index 58435256abb..f5efde6403d 100644
--- a/docs/reference/migration/migrate_5_0/search.asciidoc
+++ b/docs/reference/migration/migrate_5_0/search.asciidoc
@@ -129,6 +129,8 @@ in favour of `query` and `no_match_query`.
 * Deprecated support for the coerce, normalize, ignore_malformed parameters in GeoPolygonQuery. Use parameter validation_method instead.
 
 * Deprecated support for the coerce, normalize, ignore_malformed parameters in GeoDistanceRangeQuery. Use parameter validation_method instead.
+
+* Deprecated support for the coerce, normalize, ignore_malformed parameters in GeoDistanceQuery. Use parameter validation_method instead.
 ==== Top level `filter` parameter
 
 Removed support for the deprecated top level `filter` in the search api,
diff --git a/docs/reference/query-dsl/geo-distance-query.asciidoc b/docs/reference/query-dsl/geo-distance-query.asciidoc
index f053fd3a2dd..2b80241dfc5 100644
--- a/docs/reference/query-dsl/geo-distance-query.asciidoc
+++ b/docs/reference/query-dsl/geo-distance-query.asciidoc
@@ -162,10 +162,11 @@ The following are options allowed on the filter:
 
     Optional name field to identify the query
 
-`ignore_malformed`::
+`validation_method`::
 
-    Set to `true` to accept geo points with invalid latitude or
-    longitude (default is `false`).
+    Set to `IGNORE_MALFORMED` to accept geo points with invalid latitude or
+    longitude, set to `COERCE` to additionally try and infer correct
+    coordinates (default is `STRICT`).
 
 [float]
 ==== geo_point Type

From a19c426e0f354f3a91bdd98d36ded091fa619054 Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Thu, 28 Apr 2016 14:10:59 +0200
Subject: [PATCH 0011/1311] Deprecate coerce/ignore_malformed for
 GeoBoundingBoxQuery

---
 .../GeoBoundingBoxQueryBuilderTests.java      | 36 +++++++++++++++++++
 .../migration/migrate_5_0/search.asciidoc     |  2 ++
 .../query-dsl/geo-bounding-box-query.asciidoc |  5 +--
 3 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java
index 558d8b0e3f1..8ba48839aba 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java
@@ -505,6 +505,42 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBo
         }
     }
 
+    public void testFromJsonCoerceFails() throws IOException {
+        String json =
+                "{\n" +
+                "  \"geo_bounding_box\" : {\n" +
+                "    \"pin.location\" : {\n" +
+                "      \"top_left\" : [ -74.1, 40.73 ],\n" +
+                "      \"bottom_right\" : [ -71.12, 40.01 ]\n" +
+                "    },\n" +
+                "    \"coerce\" : true,\n" +
+                "    \"type\" : \"MEMORY\",\n" +
+                        "    \"ignore_unmapped\" : false,\n" +
+                "    \"boost\" : 1.0\n" +
+                "  }\n" +
+                "}";
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
+        assertTrue(e.getMessage().startsWith("Deprecated field "));
+    }
+
+    public void testFromJsonIgnoreMalformedFails() throws IOException {
+        String json =
+                "{\n" +
+                "  \"geo_bounding_box\" : {\n" +
+                "    \"pin.location\" : {\n" +
+                "      \"top_left\" : [ -74.1, 40.73 ],\n" +
+                "      \"bottom_right\" : [ -71.12, 40.01 ]\n" +
+                "    },\n" +
+                "    \"ignore_malformed\" : true,\n" +
+                "    \"type\" : \"MEMORY\",\n" +
+                        "    \"ignore_unmapped\" : false,\n" +
+                "    \"boost\" : 1.0\n" +
+                "  }\n" +
+                "}";
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> parseQuery(json));
+        assertTrue(e.getMessage().startsWith("Deprecated field "));
+    }
+
     @Override
     public void testMustRewrite() throws IOException {
         assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
diff --git a/docs/reference/migration/migrate_5_0/search.asciidoc b/docs/reference/migration/migrate_5_0/search.asciidoc
index f5efde6403d..f1892573f1d 100644
--- a/docs/reference/migration/migrate_5_0/search.asciidoc
+++ b/docs/reference/migration/migrate_5_0/search.asciidoc
@@ -131,6 +131,8 @@ in favour of `query` and `no_match_query`.
 * Deprecated support for the coerce, normalize, ignore_malformed parameters in GeoDistanceRangeQuery. Use parameter validation_method instead.
 
 * Deprecated support for the coerce, normalize, ignore_malformed parameters in GeoDistanceQuery. Use parameter validation_method instead.
+
+* Deprecated support for the coerce, normalize, ignore_malformed parameters in GeoBoundingBoxQuery. Use parameter validation_method instead.
 ==== Top level `filter` parameter
 
 Removed support for the deprecated top level `filter` in the search api,
diff --git a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
index 593086f2a1a..2630e6cf048 100644
--- a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
+++ b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
@@ -52,8 +52,9 @@ Then the following simple query can be executed with a
 |Option |Description
 |`_name` |Optional name field to identify the filter
 
-|`ignore_malformed` |Set to `true` to
-accept geo points with invalid latitude or longitude (default is `false`).
+|`validation_method` |Set to `IGNORE_MALFORMED` to
+accept geo points with invalid latitude or longitude, set to
+`COERCE` to also try to infer correct latitude or longitude. (default is `STRICT`).
 
 |`type` |Set to one of `indexed` or `memory` to defines whether this filter will
 be executed in memory or indexed. See <<geo-bbox-type,Type>> below for further details

From c1fa9cd18e221357f7c21708ee631db72bef503e Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Thu, 28 Apr 2016 14:13:58 +0200
Subject: [PATCH 0012/1311] Add note that coerce and ignore_malformed are
 deprecated for geo distance sorting

---
 docs/reference/migration/migrate_5_0/search.asciidoc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/reference/migration/migrate_5_0/search.asciidoc b/docs/reference/migration/migrate_5_0/search.asciidoc
index f1892573f1d..dac80ad05ed 100644
--- a/docs/reference/migration/migrate_5_0/search.asciidoc
+++ b/docs/reference/migration/migrate_5_0/search.asciidoc
@@ -155,6 +155,8 @@ vectors don't support distributed document frequencies anymore.
 The `reverse` parameter has been removed, in favour of explicitly
 specifying the sort order with the `order` option.
 
+The `coerce` and `ignore_malformed` parameters were deprecated in favour of `validation_method`.
+
 ==== Inner hits
 
 * The format of top level inner hits has been changed to be more readable. All options are now set on the same level.

From 3ab006e42272507202dfaa0f00a92ede7f06bbc6 Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Thu, 28 Apr 2016 14:27:22 +0200
Subject: [PATCH 0013/1311] Switch to new random value supplier for test.

---
 .../sort/GeoDistanceSortBuilderTests.java     | 39 +++++--------------
 .../org/elasticsearch/test/ESTestCase.java    | 11 ++++++
 2 files changed, 20 insertions(+), 30 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
index f65050ca9ce..77d04d6cae2 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
@@ -36,6 +36,7 @@ import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
 import org.elasticsearch.index.query.GeoValidationMethod;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.geo.RandomGeoGenerator;
 
 import java.io.IOException;
@@ -80,13 +81,13 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
             result.geoDistance(geoDistance(result.geoDistance()));
         }
         if (randomBoolean()) {
-            result.unit(unit(result.unit()));
+            result.unit(ESTestCase.randomValueOtherThan(result.unit(), () -> randomFrom(DistanceUnit.values())));
         }
         if (randomBoolean()) {
             result.order(RandomSortDataGenerator.order(null));
         }
         if (randomBoolean()) {
-            result.sortMode(mode(result.sortMode()));
+            result.sortMode(ESTestCase.randomValueOtherThan(SortMode.SUM, () -> randomFrom(SortMode.values())));
         }
         if (randomBoolean()) {
             result.setNestedFilter(RandomSortDataGenerator.nestedFilter(result.getNestedFilter()));
@@ -95,7 +96,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
             result.setNestedPath(RandomSortDataGenerator.randomAscii(result.getNestedPath()));
         }
         if (randomBoolean()) {
-            result.validation(randomFrom(GeoValidationMethod.values()));
+            result.validation(ESTestCase.randomValueOtherThan(result.validation(), () -> randomFrom(GeoValidationMethod.values())));
         }
 
         return result;
@@ -108,30 +109,6 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
         return clone;
     }
 
-    private static SortMode mode(SortMode original) {
-        SortMode result;
-        do {
-            result = randomFrom(SortMode.values());
-        } while (result == SortMode.SUM || result == original);
-        return result;
-    }
-
-    private static GeoValidationMethod validation(GeoValidationMethod original) {
-        GeoValidationMethod result;
-        do {
-            result = randomFrom(GeoValidationMethod.values());
-        } while (result == original);
-        return result;
-    }
-
-    private static DistanceUnit unit(DistanceUnit original) {
-        int id = -1;
-        while (id == -1 || (original != null && original.ordinal() == id)) {
-            id = randomIntBetween(0, DistanceUnit.values().length - 1);
-        }
-        return DistanceUnit.values()[id];
-    }
-
     private static GeoPoint[] points(GeoPoint[] original) {
         GeoPoint[] result = null;
         while (result == null || Arrays.deepEquals(original, result)) {
@@ -170,13 +147,15 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
             result.geoDistance(geoDistance(original.geoDistance()));
             break;
         case 3:
-            result.unit(unit(original.unit()));
+            result.unit(ESTestCase.randomValueOtherThan(result.unit(), () -> randomFrom(DistanceUnit.values())));
             break;
         case 4:
             result.order(RandomSortDataGenerator.order(original.order()));
             break;
         case 5:
-            result.sortMode(mode(original.sortMode()));
+            result.sortMode(ESTestCase.randomValueOtherThanMany(
+                    Arrays.asList(SortMode.SUM, result.sortMode()),
+                    () -> randomFrom(SortMode.values())));
             break;
         case 6:
             result.setNestedFilter(RandomSortDataGenerator.nestedFilter(original.getNestedFilter()));
@@ -185,7 +164,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
             result.setNestedPath(RandomSortDataGenerator.randomAscii(original.getNestedPath()));
             break;
         case 8:
-            result.validation(validation(original.validation()));
+            result.validation(ESTestCase.randomValueOtherThan(result.validation(), () -> randomFrom(GeoValidationMethod.values())));
             break;
         }
         return result;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 8bbc95fe2ad..d8a3221c88f 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -417,6 +417,17 @@ public abstract class ESTestCase extends LuceneTestCase {
         return randomValue;
     }
 
+    /**
+     * helper to get a random value in a certain range that's different from the input
+     */
+    public static <T> T randomValueOtherThanMany(Collection<T> input, Supplier<T> randomSupplier) {
+        T randomValue = null;
+        do {
+            randomValue = randomSupplier.get();
+        } while (input.contains(randomValue));
+        return randomValue;
+    }
+
     /**
      * Runs the code block for 10 seconds waiting for no assertion to trip.
      */

From 47fefdd273f7784e9c166eedb2419011395a7574 Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Thu, 28 Apr 2016 14:45:56 +0200
Subject: [PATCH 0014/1311] Switch from separate sort_mode to more general
 randomValueOtherThan

... for sort tests only ...
---
 .../search/sort/FieldSortBuilderTests.java         |  5 +++--
 .../search/sort/RandomSortDataGenerator.java       | 14 --------------
 .../search/sort/ScriptSortBuilderTests.java        |  7 ++++---
 .../java/org/elasticsearch/test/ESTestCase.java    | 11 +++++++++++
 4 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
index 5ca9aef658c..f209ebefdef 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 
@@ -51,7 +52,7 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
         }
 
         if (randomBoolean()) {
-            builder.sortMode(RandomSortDataGenerator.mode(builder.sortMode()));
+            builder.sortMode(ESTestCase.randomValueOtherThan(builder.sortMode(), () -> randomFrom(SortMode.values())));
         }
 
         if (randomBoolean()) {
@@ -77,7 +78,7 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
             mutated.setNestedFilter(RandomSortDataGenerator.nestedFilter(mutated.getNestedFilter()));
             break;
         case 2:
-            mutated.sortMode(RandomSortDataGenerator.mode(mutated.sortMode()));
+            mutated.sortMode(ESTestCase.randomValueOtherThan(mutated.sortMode(), () -> randomFrom(SortMode.values())));
             break;
         case 3:
             mutated.unmappedType(RandomSortDataGenerator.randomAscii(mutated.unmappedType()));
diff --git a/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java b/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java
index a31de756bc9..3a40e9b56c6 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java
@@ -63,20 +63,6 @@ public class RandomSortDataGenerator {
         return nestedPath;
     }
 
-    public static SortMode mode(SortMode original) {
-        Set<SortMode> set = new HashSet<>();
-        set.add(original);
-        return mode(set);
-    }
-
-    public static SortMode mode(Set<SortMode> except) {
-        SortMode mode = ESTestCase.randomFrom(SortMode.values());
-        while (except.contains(mode)) {
-            mode = ESTestCase.randomFrom(SortMode.values());
-        }
-        return mode;
-    }
-
     public static Object missing(Object original) {
         Object missing = null;
         Object otherMissing = original;
diff --git a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
index a1a24632f09..85ee079ffe4 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
@@ -29,6 +29,7 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
+import org.elasticsearch.test.ESTestCase;
 import org.junit.Rule;
 import org.junit.rules.ExpectedException;
 
@@ -52,13 +53,13 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild
         }
         if (randomBoolean()) {
             if (type == ScriptSortType.NUMBER) {
-                builder.sortMode(RandomSortDataGenerator.mode(builder.sortMode()));
+                builder.sortMode(ESTestCase.randomValueOtherThan(builder.sortMode(), () -> randomFrom(SortMode.values())));
             } else {
                 Set<SortMode> exceptThis = new HashSet<>();
                 exceptThis.add(SortMode.SUM);
                 exceptThis.add(SortMode.AVG);
                 exceptThis.add(SortMode.MEDIAN);
-                builder.sortMode(RandomSortDataGenerator.mode(exceptThis));
+                builder.sortMode(ESTestCase.randomValueOtherThanMany(exceptThis, () -> randomFrom(SortMode.values())));
             }
         }
         if (randomBoolean()) {
@@ -101,7 +102,7 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild
                 break;
             case 1:
                 if (original.type() == ScriptSortType.NUMBER) {
-                    result.sortMode(RandomSortDataGenerator.mode(original.sortMode()));
+                    result.sortMode(ESTestCase.randomValueOtherThan(result.sortMode(), () -> randomFrom(SortMode.values())));
                 } else {
                     // script sort type String only allows MIN and MAX, so we only switch
                     if (original.sortMode() == SortMode.MIN) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 8bbc95fe2ad..d8a3221c88f 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -417,6 +417,17 @@ public abstract class ESTestCase extends LuceneTestCase {
         return randomValue;
     }
 
+    /**
+     * helper to get a random value in a certain range that's different from the input
+     */
+    public static <T> T randomValueOtherThanMany(Collection<T> input, Supplier<T> randomSupplier) {
+        T randomValue = null;
+        do {
+            randomValue = randomSupplier.get();
+        } while (input.contains(randomValue));
+        return randomValue;
+    }
+
     /**
      * Runs the code block for 10 seconds waiting for no assertion to trip.
      */

From 8dc0610eaa86b0b2bf5db0e6387fe98ded94bc55 Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Thu, 28 Apr 2016 15:07:55 +0200
Subject: [PATCH 0015/1311] Get rid of duplicated random ascii generation

---
 .../search/sort/FieldSortBuilderTests.java       | 16 ++++++++++++----
 .../search/sort/GeoDistanceSortBuilderTests.java | 10 ++++++++--
 .../search/sort/RandomSortDataGenerator.java     |  8 --------
 .../search/sort/ScriptSortBuilderTests.java      |  4 +++-
 4 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
index f209ebefdef..c557c714d0f 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
@@ -48,7 +48,9 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
         }
 
         if (randomBoolean()) {
-            builder.unmappedType(RandomSortDataGenerator.randomAscii(builder.unmappedType()));
+            builder.unmappedType(ESTestCase.randomValueOtherThan(
+                    builder.unmappedType(),
+                    () -> ESTestCase.randomAsciiOfLengthBetween(1, 10)));
         }
 
         if (randomBoolean()) {
@@ -60,7 +62,9 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
         }
 
         if (randomBoolean()) {
-            builder.setNestedPath(RandomSortDataGenerator.randomAscii(builder.getNestedPath()));
+            builder.setNestedPath(ESTestCase.randomValueOtherThan(
+                    builder.getNestedPath(),
+                    () -> ESTestCase.randomAsciiOfLengthBetween(1, 10)));
         }
 
         return builder;
@@ -72,7 +76,9 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
         int parameter = randomIntBetween(0, 5);
         switch (parameter) {
         case 0:
-            mutated.setNestedPath(RandomSortDataGenerator.randomAscii(mutated.getNestedPath()));
+            mutated.setNestedPath(ESTestCase.randomValueOtherThan(
+                    mutated.getNestedPath(),
+                    () -> ESTestCase.randomAsciiOfLengthBetween(1, 10)));
             break;
         case 1:
             mutated.setNestedFilter(RandomSortDataGenerator.nestedFilter(mutated.getNestedFilter()));
@@ -81,7 +87,9 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
             mutated.sortMode(ESTestCase.randomValueOtherThan(mutated.sortMode(), () -> randomFrom(SortMode.values())));
             break;
         case 3:
-            mutated.unmappedType(RandomSortDataGenerator.randomAscii(mutated.unmappedType()));
+            mutated.unmappedType(ESTestCase.randomValueOtherThan(
+                    mutated.unmappedType(),
+                    () -> ESTestCase.randomAsciiOfLengthBetween(1, 10)));
             break;
         case 4:
             mutated.missing(RandomSortDataGenerator.missing(mutated.missing()));
diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
index 87fd183f1ce..a43f9c54bb3 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
@@ -35,6 +35,7 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.geo.RandomGeoGenerator;
 
 import java.io.IOException;
@@ -91,7 +92,10 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
             result.setNestedFilter(RandomSortDataGenerator.nestedFilter(result.getNestedFilter()));
         }
         if (randomBoolean()) {
-            result.setNestedPath(RandomSortDataGenerator.randomAscii(result.getNestedPath()));
+            result.setNestedPath(
+                    ESTestCase.randomValueOtherThan(
+                            result.getNestedPath(),
+                            () -> ESTestCase.randomAsciiOfLengthBetween(1, 10)));
         }
         if (randomBoolean()) {
             result.coerce(! result.coerce());
@@ -176,7 +180,9 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
             result.setNestedFilter(RandomSortDataGenerator.nestedFilter(original.getNestedFilter()));
             break;
         case 7:
-            result.setNestedPath(RandomSortDataGenerator.randomAscii(original.getNestedPath()));
+            result.setNestedPath(ESTestCase.randomValueOtherThan(
+                    result.getNestedPath(),
+                    () -> ESTestCase.randomAsciiOfLengthBetween(1, 10)));
             break;
         case 8:
             result.coerce(! original.coerce());
diff --git a/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java b/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java
index 3a40e9b56c6..86a12f7ad84 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java
@@ -55,14 +55,6 @@ public class RandomSortDataGenerator {
         return nested;
     }
 
-    public static String randomAscii(String original) {
-        String nestedPath = ESTestCase.randomAsciiOfLengthBetween(1, 10);
-        while (nestedPath.equals(original)) {
-            nestedPath = ESTestCase.randomAsciiOfLengthBetween(1, 10);
-        }
-        return nestedPath;
-    }
-
     public static Object missing(Object original) {
         Object missing = null;
         Object otherMissing = original;
diff --git a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
index 85ee079ffe4..b14b82aa092 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
@@ -66,7 +66,9 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild
             builder.setNestedFilter(RandomSortDataGenerator.nestedFilter(builder.getNestedFilter()));
         }
         if (randomBoolean()) {
-            builder.setNestedPath(RandomSortDataGenerator.randomAscii(builder.getNestedPath()));
+            builder.setNestedPath(ESTestCase.randomValueOtherThan(
+                    builder.getNestedPath(),
+                    () -> ESTestCase.randomAsciiOfLengthBetween(1, 10)));
         }
         return builder;
     }

From 0c7e58de93c33d4d64073abd583978cd82a8e451 Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Thu, 28 Apr 2016 15:14:15 +0200
Subject: [PATCH 0016/1311] Get rid of duplicate random sort order generation

---
 .../search/sort/FieldSortBuilderTests.java        |  4 ++--
 .../search/sort/GeoDistanceSortBuilderTests.java  |  4 ++--
 .../search/sort/RandomSortDataGenerator.java      | 15 ---------------
 .../search/sort/ScoreSortBuilderTests.java        |  3 ++-
 .../search/sort/ScriptSortBuilderTests.java       |  2 +-
 5 files changed, 7 insertions(+), 21 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
index c557c714d0f..ebfa3bbb76f 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
@@ -40,7 +40,7 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
         String fieldName = rarely() ? FieldSortBuilder.DOC_FIELD_NAME : randomAsciiOfLengthBetween(1, 10);
         FieldSortBuilder builder = new FieldSortBuilder(fieldName);
         if (randomBoolean()) {
-            builder.order(RandomSortDataGenerator.order(null));
+            builder.order(randomFrom(SortOrder.values()));
         }
 
         if (randomBoolean()) {
@@ -95,7 +95,7 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
             mutated.missing(RandomSortDataGenerator.missing(mutated.missing()));
             break;
         case 5:
-            mutated.order(RandomSortDataGenerator.order(mutated.order()));
+            mutated.order(ESTestCase.randomValueOtherThan(mutated.order(), () -> randomFrom(SortOrder.values())));
             break;
         default:
             throw new IllegalStateException("Unsupported mutation.");
diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
index a43f9c54bb3..723e569bdb5 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
@@ -83,7 +83,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
             result.unit(unit(result.unit()));
         }
         if (randomBoolean()) {
-            result.order(RandomSortDataGenerator.order(null));
+            result.order(randomFrom(SortOrder.values()));
         }
         if (randomBoolean()) {
             result.sortMode(mode(result.sortMode()));
@@ -171,7 +171,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
             result.unit(unit(original.unit()));
             break;
         case 4:
-            result.order(RandomSortDataGenerator.order(original.order()));
+            result.order(ESTestCase.randomValueOtherThan(original.order(), () -> randomFrom(SortOrder.values())));
             break;
         case 5:
             result.sortMode(mode(original.sortMode()));
diff --git a/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java b/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java
index 86a12f7ad84..bad5ff83f76 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java
@@ -84,19 +84,4 @@ public class RandomSortDataGenerator {
         }
         return missing;
     }
-
-    /**
-     * return a random {@link SortOrder} settings, except the one provided by parameter if set
-     */
-    public static SortOrder order(@Nullable SortOrder original) {
-        if (original == null) {
-            return ESTestCase.randomBoolean() ? SortOrder.ASC : SortOrder.DESC;
-        }
-        if (original.equals(SortOrder.ASC)) {
-            return SortOrder.DESC;
-        } else {
-            return SortOrder.ASC;
-        }
-    }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java
index 142a88c4036..222240a9ac4 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java
@@ -26,6 +26,7 @@ import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.test.ESTestCase;
 import org.junit.Rule;
 import org.junit.rules.ExpectedException;
 
@@ -45,7 +46,7 @@ public class ScoreSortBuilderTests extends AbstractSortTestCase<ScoreSortBuilder
     @Override
     protected ScoreSortBuilder mutate(ScoreSortBuilder original) throws IOException {
         ScoreSortBuilder result = new ScoreSortBuilder();
-        result.order(RandomSortDataGenerator.order(original.order()));
+        result.order(ESTestCase.randomValueOtherThan(original.order(), () -> randomFrom(SortOrder.values())));
         return result;
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
index b14b82aa092..9637df30772 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
@@ -49,7 +49,7 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild
         ScriptSortBuilder builder = new ScriptSortBuilder(new Script(randomAsciiOfLengthBetween(5, 10)),
                 type);
         if (randomBoolean()) {
-                builder.order(RandomSortDataGenerator.order(null));
+                builder.order(randomFrom(SortOrder.values()));
         }
         if (randomBoolean()) {
             if (type == ScriptSortType.NUMBER) {

From 3490daa7eaa501ea21de2b2beb21cbbe96ccf326 Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Thu, 28 Apr 2016 15:27:36 +0200
Subject: [PATCH 0017/1311] Get rid of no longer needed missing generation
 method

---
 .../search/sort/FieldSortBuilderTests.java    | 15 +++++++--
 .../search/sort/RandomSortDataGenerator.java  | 33 -------------------
 2 files changed, 12 insertions(+), 36 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
index ebfa3bbb76f..27f87681777 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
@@ -28,6 +28,8 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
 
 public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder> {
 
@@ -36,7 +38,14 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
         return randomFieldSortBuilder();
     }
 
-    public static FieldSortBuilder randomFieldSortBuilder() {
+    private List<Object> missingContent = Arrays.asList(
+            "_last",
+            "_first",
+            ESTestCase.randomAsciiOfLength(10), ESTestCase.randomUnicodeOfCodepointLengthBetween(5, 15),
+            ESTestCase.randomInt());
+
+
+    public FieldSortBuilder randomFieldSortBuilder() {
         String fieldName = rarely() ? FieldSortBuilder.DOC_FIELD_NAME : randomAsciiOfLengthBetween(1, 10);
         FieldSortBuilder builder = new FieldSortBuilder(fieldName);
         if (randomBoolean()) {
@@ -44,7 +53,7 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
         }
 
         if (randomBoolean()) {
-            builder.missing(RandomSortDataGenerator.missing(builder.missing()));
+            builder.missing(ESTestCase.randomValueOtherThan(builder.missing(), () -> randomFrom(missingContent)));
         }
 
         if (randomBoolean()) {
@@ -92,7 +101,7 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
                     () -> ESTestCase.randomAsciiOfLengthBetween(1, 10)));
             break;
         case 4:
-            mutated.missing(RandomSortDataGenerator.missing(mutated.missing()));
+            mutated.missing(ESTestCase.randomValueOtherThan(mutated.missing(), () -> randomFrom(missingContent)));
             break;
         case 5:
             mutated.order(ESTestCase.randomValueOtherThan(mutated.order(), () -> randomFrom(SortOrder.values())));
diff --git a/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java b/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java
index bad5ff83f76..43ff2428d81 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java
@@ -19,16 +19,12 @@
 
 package org.elasticsearch.search.sort;
 
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.query.IdsQueryBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.test.ESTestCase;
 
-import java.util.HashSet;
-import java.util.Set;
-
 public class RandomSortDataGenerator {
     private RandomSortDataGenerator() {
         // this is a helper class only, doesn't need a constructor
@@ -55,33 +51,4 @@ public class RandomSortDataGenerator {
         return nested;
     }
 
-    public static Object missing(Object original) {
-        Object missing = null;
-        Object otherMissing = original;
-
-        while (missing == null || missing.equals(otherMissing)) {
-          int missingId = ESTestCase.randomIntBetween(0, 4);
-          switch (missingId) {
-          case 0:
-              missing = ("_last");
-              break;
-          case 1:
-              missing = ("_first");
-              break;
-          case 2:
-              missing = ESTestCase.randomAsciiOfLength(10);
-              break;
-          case 3:
-              missing = ESTestCase.randomUnicodeOfCodepointLengthBetween(5, 15);
-              break;
-          case 4:
-              missing = ESTestCase.randomInt();
-              break;
-          default:
-              throw new IllegalStateException("Unknown missing type.");
-
-          }
-        }
-        return missing;
-    }
 }

From a03b131319dd815c826c211100db5efacf8a7cb4 Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Thu, 28 Apr 2016 15:31:56 +0200
Subject: [PATCH 0018/1311] Shorten nested test query builder generation.

---
 .../search/sort/FieldSortBuilderTests.java    |  4 +--
 .../sort/GeoDistanceSortBuilderTests.java     |  4 +--
 ....java => NestedQueryBuilderGenerator.java} | 33 ++++++++-----------
 .../search/sort/ScriptSortBuilderTests.java   |  4 +--
 4 files changed, 19 insertions(+), 26 deletions(-)
 rename core/src/test/java/org/elasticsearch/search/sort/{RandomSortDataGenerator.java => NestedQueryBuilderGenerator.java} (61%)

diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
index 27f87681777..abcab363dcc 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
@@ -67,7 +67,7 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
         }
 
         if (randomBoolean()) {
-            builder.setNestedFilter(RandomSortDataGenerator.nestedFilter(builder.getNestedFilter()));
+            builder.setNestedFilter(NestedQueryBuilderGenerator.nestedFilter(builder.getNestedFilter()));
         }
 
         if (randomBoolean()) {
@@ -90,7 +90,7 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
                     () -> ESTestCase.randomAsciiOfLengthBetween(1, 10)));
             break;
         case 1:
-            mutated.setNestedFilter(RandomSortDataGenerator.nestedFilter(mutated.getNestedFilter()));
+            mutated.setNestedFilter(NestedQueryBuilderGenerator.nestedFilter(mutated.getNestedFilter()));
             break;
         case 2:
             mutated.sortMode(ESTestCase.randomValueOtherThan(mutated.sortMode(), () -> randomFrom(SortMode.values())));
diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
index 723e569bdb5..9f1b1a325ef 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
@@ -89,7 +89,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
             result.sortMode(mode(result.sortMode()));
         }
         if (randomBoolean()) {
-            result.setNestedFilter(RandomSortDataGenerator.nestedFilter(result.getNestedFilter()));
+            result.setNestedFilter(NestedQueryBuilderGenerator.nestedFilter(result.getNestedFilter()));
         }
         if (randomBoolean()) {
             result.setNestedPath(
@@ -177,7 +177,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
             result.sortMode(mode(original.sortMode()));
             break;
         case 6:
-            result.setNestedFilter(RandomSortDataGenerator.nestedFilter(original.getNestedFilter()));
+            result.setNestedFilter(NestedQueryBuilderGenerator.nestedFilter(original.getNestedFilter()));
             break;
         case 7:
             result.setNestedPath(ESTestCase.randomValueOtherThan(
diff --git a/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java b/core/src/test/java/org/elasticsearch/search/sort/NestedQueryBuilderGenerator.java
similarity index 61%
rename from core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java
rename to core/src/test/java/org/elasticsearch/search/sort/NestedQueryBuilderGenerator.java
index 43ff2428d81..7e6f88cffd5 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/NestedQueryBuilderGenerator.java
@@ -25,30 +25,23 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.test.ESTestCase;
 
-public class RandomSortDataGenerator {
-    private RandomSortDataGenerator() {
+import java.util.Arrays;
+import java.util.List;
+
+public class NestedQueryBuilderGenerator {
+    private NestedQueryBuilderGenerator() {
         // this is a helper class only, doesn't need a constructor
     }
 
-    public static QueryBuilder nestedFilter(QueryBuilder original) {
+    private static List<QueryBuilder<?>> builders = Arrays.asList(
+            new MatchAllQueryBuilder(),
+            new IdsQueryBuilder(),
+            new TermQueryBuilder(ESTestCase.randomAsciiOfLengthBetween(1, 10), ESTestCase.randomDouble()));
+
+    public static QueryBuilder<?> nestedFilter(QueryBuilder<?> original) {
         @SuppressWarnings("rawtypes")
-        QueryBuilder nested = null;
-        while (nested == null || nested.equals(original)) {
-            switch (ESTestCase.randomInt(2)) {
-            case 0:
-                nested = new MatchAllQueryBuilder();
-                break;
-            case 1:
-                nested = new IdsQueryBuilder();
-                break;
-            default:
-            case 2:
-                nested = new TermQueryBuilder(ESTestCase.randomAsciiOfLengthBetween(1, 10), ESTestCase.randomDouble());
-                break;
-            }
-            nested.boost((float) ESTestCase.randomDoubleBetween(0, 10, false));
-        }
+        QueryBuilder nested = ESTestCase.randomValueOtherThan(original, () -> ESTestCase.randomFrom(builders));
+        nested.boost((float) ESTestCase.randomDoubleBetween(0, 10, false));
         return nested;
     }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
index 9637df30772..ba9d608f610 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
@@ -63,7 +63,7 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild
             }
         }
         if (randomBoolean()) {
-            builder.setNestedFilter(RandomSortDataGenerator.nestedFilter(builder.getNestedFilter()));
+            builder.setNestedFilter(NestedQueryBuilderGenerator.nestedFilter(builder.getNestedFilter()));
         }
         if (randomBoolean()) {
             builder.setNestedPath(ESTestCase.randomValueOtherThan(
@@ -115,7 +115,7 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild
                 }
                 break;
             case 2:
-                result.setNestedFilter(RandomSortDataGenerator.nestedFilter(original.getNestedFilter()));
+                result.setNestedFilter(NestedQueryBuilderGenerator.nestedFilter(original.getNestedFilter()));
                 break;
             case 3:
                 result.setNestedPath(original.getNestedPath() + "_some_suffix");

From faa3c6ef3c9b437360e183032c9ea3f049c70d1e Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Fri, 29 Apr 2016 14:40:02 +0200
Subject: [PATCH 0019/1311] Add new UnsupportedException for EC Mock

---
 .../discovery/ec2/AmazonEC2Mock.java          | 38 +++++++++----------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java
index 881cb98e1d1..49f4f885419 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Mock.java
@@ -554,7 +554,7 @@ public class AmazonEC2Mock implements AmazonEC2 {
 
     @Override
     public DeleteNatGatewayResult deleteNatGateway(DeleteNatGatewayRequest deleteNatGatewayRequest) {
-        return null;
+        throw new UnsupportedOperationException("Not supported in mock");
     }
 
     @Override
@@ -679,12 +679,12 @@ public class AmazonEC2Mock implements AmazonEC2 {
 
     @Override
     public ModifyHostsResult modifyHosts(ModifyHostsRequest modifyHostsRequest) {
-        return null;
+        throw new UnsupportedOperationException("Not supported in mock");
     }
 
     @Override
     public void modifyIdFormat(ModifyIdFormatRequest modifyIdFormatRequest) {
-
+        throw new UnsupportedOperationException("Not supported in mock");
     }
 
     @Override
@@ -1014,7 +1014,7 @@ public class AmazonEC2Mock implements AmazonEC2 {
 
     @Override
     public DisableVpcClassicLinkDnsSupportResult disableVpcClassicLinkDnsSupport(DisableVpcClassicLinkDnsSupportRequest disableVpcClassicLinkDnsSupportRequest) {
-        return null;
+        throw new UnsupportedOperationException("Not supported in mock");
     }
 
     @Override
@@ -1044,7 +1044,7 @@ public class AmazonEC2Mock implements AmazonEC2 {
 
     @Override
     public RunScheduledInstancesResult runScheduledInstances(RunScheduledInstancesRequest runScheduledInstancesRequest) {
-        return null;
+        throw new UnsupportedOperationException("Not supported in mock");
     }
 
     @Override
@@ -1084,7 +1084,7 @@ public class AmazonEC2Mock implements AmazonEC2 {
 
     @Override
     public ModifyInstancePlacementResult modifyInstancePlacement(ModifyInstancePlacementRequest modifyInstancePlacementRequest) {
-        return null;
+        throw new UnsupportedOperationException("Not supported in mock");
     }
 
     @Override
@@ -1224,7 +1224,7 @@ public class AmazonEC2Mock implements AmazonEC2 {
 
     @Override
     public PurchaseScheduledInstancesResult purchaseScheduledInstances(PurchaseScheduledInstancesRequest purchaseScheduledInstancesRequest) {
-        return null;
+        throw new UnsupportedOperationException("Not supported in mock");
     }
 
     @Override
@@ -1404,7 +1404,7 @@ public class AmazonEC2Mock implements AmazonEC2 {
 
     @Override
     public ReleaseHostsResult releaseHosts(ReleaseHostsRequest releaseHostsRequest) {
-        return null;
+        throw new UnsupportedOperationException("Not supported in mock");
     }
 
     @Override
@@ -1419,7 +1419,7 @@ public class AmazonEC2Mock implements AmazonEC2 {
 
     @Override
     public CreateNatGatewayResult createNatGateway(CreateNatGatewayRequest createNatGatewayRequest) {
-        return null;
+        throw new UnsupportedOperationException("Not supported in mock");
     }
 
     @Override
@@ -1454,7 +1454,7 @@ public class AmazonEC2Mock implements AmazonEC2 {
 
     @Override
     public EnableVpcClassicLinkDnsSupportResult enableVpcClassicLinkDnsSupport(EnableVpcClassicLinkDnsSupportRequest enableVpcClassicLinkDnsSupportRequest) {
-        return null;
+        throw new UnsupportedOperationException("Not supported in mock");
     }
 
     @Override
@@ -1574,12 +1574,12 @@ public class AmazonEC2Mock implements AmazonEC2 {
 
     @Override
     public DescribeScheduledInstanceAvailabilityResult describeScheduledInstanceAvailability(DescribeScheduledInstanceAvailabilityRequest describeScheduledInstanceAvailabilityRequest) {
-        return null;
+        throw new UnsupportedOperationException("Not supported in mock");
     }
 
     @Override
     public DescribeScheduledInstancesResult describeScheduledInstances(DescribeScheduledInstancesRequest describeScheduledInstancesRequest) {
-        return null;
+        throw new UnsupportedOperationException("Not supported in mock");
     }
 
     @Override
@@ -1624,7 +1624,7 @@ public class AmazonEC2Mock implements AmazonEC2 {
 
     @Override
     public DescribeNatGatewaysResult describeNatGateways(DescribeNatGatewaysRequest describeNatGatewaysRequest) {
-        return null;
+        throw new UnsupportedOperationException("Not supported in mock");
     }
 
     @Override
@@ -1644,22 +1644,22 @@ public class AmazonEC2Mock implements AmazonEC2 {
 
     @Override
     public DescribeHostsResult describeHosts(DescribeHostsRequest describeHostsRequest) {
-        return null;
+        throw new UnsupportedOperationException("Not supported in mock");
     }
 
     @Override
     public DescribeHostsResult describeHosts() {
-        return null;
+        throw new UnsupportedOperationException("Not supported in mock");
     }
 
     @Override
     public DescribeIdFormatResult describeIdFormat(DescribeIdFormatRequest describeIdFormatRequest) {
-        return null;
+        throw new UnsupportedOperationException("Not supported in mock");
     }
 
     @Override
     public DescribeIdFormatResult describeIdFormat() {
-        return null;
+        throw new UnsupportedOperationException("Not supported in mock");
     }
 
     @Override
@@ -1704,7 +1704,7 @@ public class AmazonEC2Mock implements AmazonEC2 {
 
     @Override
     public DescribeVpcClassicLinkDnsSupportResult describeVpcClassicLinkDnsSupport(DescribeVpcClassicLinkDnsSupportRequest describeVpcClassicLinkDnsSupportRequest) {
-        return null;
+        throw new UnsupportedOperationException("Not supported in mock");
     }
 
     @Override
@@ -1774,7 +1774,7 @@ public class AmazonEC2Mock implements AmazonEC2 {
 
     @Override
     public AllocateHostsResult allocateHosts(AllocateHostsRequest allocateHostsRequest) {
-        return null;
+        throw new UnsupportedOperationException("Not supported in mock");
     }
 
     @Override

From 8c77399f1c56be28f31d47cc1f50aa5b2d8ef647 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Fri, 29 Apr 2016 15:47:03 +0200
Subject: [PATCH 0020/1311] test: changed test's expectation that a
 BooleanQuery is returned. A NoMatchDocsQuery is returned instead now.

---
 .../index/query/ExistsQueryBuilderTests.java   | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java
index 6bac6392e14..d46cce420e1 100644
--- a/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java
@@ -24,10 +24,12 @@ import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 
 import java.io.IOException;
 import java.util.Collection;
 
+import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
 
@@ -56,9 +58,9 @@ public class ExistsQueryBuilderTests extends AbstractQueryTestCase<ExistsQueryBu
         String fieldPattern = queryBuilder.fieldName();
         Collection<String> fields = context.simpleMatchToIndexNames(fieldPattern);
         if (getCurrentTypes().length == 0) {
-            assertThat(query, instanceOf(BooleanQuery.class));
-            BooleanQuery booleanQuery = (BooleanQuery) query;
-            assertThat(booleanQuery.clauses().size(), equalTo(0));
+            assertThat(query, instanceOf(MatchNoDocsQuery.class));
+            MatchNoDocsQuery matchNoDocsQuery = (MatchNoDocsQuery) query;
+            assertThat(matchNoDocsQuery.toString(null), containsString("Missing types in \"exists\" query."));
         } else {
             assertThat(query, instanceOf(ConstantScoreQuery.class));
             ConstantScoreQuery constantScoreQuery = (ConstantScoreQuery) query;
@@ -79,11 +81,11 @@ public class ExistsQueryBuilderTests extends AbstractQueryTestCase<ExistsQueryBu
 
     public void testFromJson() throws IOException {
         String json =
-                "{\n" + 
-                "  \"exists\" : {\n" + 
-                "    \"field\" : \"user\",\n" + 
-                "    \"boost\" : 42.0\n" + 
-                "  }\n" + 
+                "{\n" +
+                "  \"exists\" : {\n" +
+                "    \"field\" : \"user\",\n" +
+                "    \"boost\" : 42.0\n" +
+                "  }\n" +
                 "}";
 
         ExistsQueryBuilder parsed = (ExistsQueryBuilder) parseQuery(json);

From 37382ecfb2f1c25e274ea3d667c0d6ec3205737a Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Fri, 29 Apr 2016 15:54:15 +0200
Subject: [PATCH 0021/1311] Add Azure discovery tests mocking Azure management
 endpoint (#18004)

---
 plugins/discovery-azure/build.gradle          |  31 ++
 .../azure/management/AzureComputeService.java |  26 +-
 .../management/AzureComputeServiceImpl.java   |  28 +-
 .../azure/AzureUnicastHostsProvider.java      |  21 +-
 .../discovery/azure/AzureDiscoveryPlugin.java |   3 +
 .../AzureDiscoveryClusterFormationTests.java  | 285 ++++++++++++++++++
 .../Ec2DiscoveryClusterFormationTests.java    |   2 +-
 7 files changed, 360 insertions(+), 36 deletions(-)
 create mode 100644 plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryClusterFormationTests.java

diff --git a/plugins/discovery-azure/build.gradle b/plugins/discovery-azure/build.gradle
index 1dd2aa26f23..ceddbc2ff44 100644
--- a/plugins/discovery-azure/build.gradle
+++ b/plugins/discovery-azure/build.gradle
@@ -1,3 +1,5 @@
+import org.elasticsearch.gradle.LoggedExec
+
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -49,6 +51,35 @@ dependencies {
   compile 'org.codehaus.jackson:jackson-xc:1.9.2'  
 }
 
+// needed to be consistent with ssl host checking
+String host = InetAddress.getLoopbackAddress().getHostAddress();
+
+// location of keystore and files to generate it
+File keystore = new File(project.buildDir, 'keystore/test-node.jks')
+
+// generate the keystore
+task createKey(type: LoggedExec) {
+  doFirst {
+    project.delete(keystore.parentFile)
+    keystore.parentFile.mkdirs()
+  }
+  executable = 'keytool'
+  standardInput = new ByteArrayInputStream('FirstName LastName\nUnit\nOrganization\nCity\nState\nNL\nyes\n\n'.getBytes('UTF-8'))
+  args '-genkey',
+          '-alias', 'test-node',
+          '-keystore', keystore,
+          '-keyalg', 'RSA',
+          '-keysize', '2048',
+          '-validity', '712',
+          '-dname', 'CN=' + host,
+          '-keypass', 'keypass',
+          '-storepass', 'keypass'
+}
+
+// add keystore to test classpath: it expects it there
+sourceSets.test.resources.srcDir(keystore.parentFile)
+processTestResources.dependsOn(createKey)
+
 dependencyLicenses {
   mapping from: /azure-.*/, to: 'azure'
   mapping from: /jackson-.*/, to: 'jackson'
diff --git a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeService.java b/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeService.java
index acc1e76bde4..526f98025b7 100644
--- a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeService.java
+++ b/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeService.java
@@ -25,6 +25,11 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.discovery.azure.AzureUnicastHostsProvider;
+import org.elasticsearch.discovery.azure.AzureUnicastHostsProvider.Deployment;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.function.Function;
 
 public interface AzureComputeService {
 
@@ -43,19 +48,30 @@ public interface AzureComputeService {
         public static final Setting<KeyStoreType> KEYSTORE_TYPE_SETTING =
             new Setting<>("cloud.azure.management.keystore.type", KeyStoreType.pkcs12.name(), KeyStoreType::fromString,
                 Property.NodeScope, Property.Filtered);
+
+        // so that it can overridden for tests
+        public static final Setting<URI> ENDPOINT_SETTING = new Setting<URI>("cloud.azure.management.endpoint",
+            "https://management.core.windows.net/", s -> {
+                try {
+                    return new URI(s);
+                } catch (URISyntaxException e) {
+                    throw new IllegalArgumentException(e);
+                }
+            }, Property.NodeScope);
     }
 
     final class Discovery {
         public static final Setting<TimeValue> REFRESH_SETTING =
             Setting.positiveTimeSetting("discovery.azure.refresh_interval", TimeValue.timeValueSeconds(0), Property.NodeScope);
-
         public static final Setting<AzureUnicastHostsProvider.HostType> HOST_TYPE_SETTING =
             new Setting<>("discovery.azure.host.type", AzureUnicastHostsProvider.HostType.PRIVATE_IP.name(),
                 AzureUnicastHostsProvider.HostType::fromString, Property.NodeScope);
-
-        public static final String ENDPOINT_NAME = "discovery.azure.endpoint.name";
-        public static final String DEPLOYMENT_NAME = "discovery.azure.deployment.name";
-        public static final String DEPLOYMENT_SLOT = "discovery.azure.deployment.slot";
+        public static final Setting<String> ENDPOINT_NAME_SETTING = new Setting<>("discovery.azure.endpoint.name", "elasticsearch",
+            Function.identity(), Property.NodeScope);
+        public static final Setting<String> DEPLOYMENT_NAME_SETTING = Setting.simpleString("discovery.azure.deployment.name",
+            Property.NodeScope);
+        public static final Setting<Deployment> DEPLOYMENT_SLOT_SETTING = new Setting<>("discovery.azure.deployment.slot",
+            Deployment.PRODUCTION.name(), Deployment::fromString, Property.NodeScope);
     }
 
     HostedServiceGetDetailedResponse getServiceDetails();
diff --git a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeServiceImpl.java b/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeServiceImpl.java
index 04b4f32ea92..0764ec99c12 100644
--- a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeServiceImpl.java
+++ b/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeServiceImpl.java
@@ -33,8 +33,6 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 
 import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
 
 /**
  *
@@ -42,10 +40,6 @@ import java.net.URISyntaxException;
 public class AzureComputeServiceImpl extends AbstractLifecycleComponent<AzureComputeServiceImpl>
     implements AzureComputeService {
 
-    static final class Azure {
-        private static final String ENDPOINT = "https://management.core.windows.net/";
-    }
-
     private final ComputeManagementClient computeManagementClient;
     private final String serviceName;
 
@@ -59,18 +53,18 @@ public class AzureComputeServiceImpl extends AbstractLifecycleComponent<AzureCom
         String keystorePassword = Management.KEYSTORE_PASSWORD_SETTING.get(settings);
         KeyStoreType keystoreType = Management.KEYSTORE_TYPE_SETTING.get(settings);
 
-        // Check that we have all needed properties
-        Configuration configuration;
-        try {
-            configuration = ManagementConfiguration.configure(new URI(Azure.ENDPOINT),
-                    subscriptionId, keystorePath, keystorePassword, keystoreType);
-        } catch (IOException|URISyntaxException e) {
-            logger.error("can not start azure client: {}", e.getMessage());
-            computeManagementClient = null;
-            return;
-        }
         logger.trace("creating new Azure client for [{}], [{}]", subscriptionId, serviceName);
-        computeManagementClient = ComputeManagementService.create(configuration);
+        ComputeManagementClient result;
+        try {
+            // Check that we have all needed properties
+            Configuration configuration = ManagementConfiguration.configure(Management.ENDPOINT_SETTING.get(settings),
+                subscriptionId, keystorePath, keystorePassword, keystoreType);
+            result = ComputeManagementService.create(configuration);
+        } catch (IOException e) {
+            logger.error("can not start azure client: {}", e.getMessage());
+            result = null;
+        }
+        this.computeManagementClient = result;
     }
 
     @Override
diff --git a/plugins/discovery-azure/src/main/java/org/elasticsearch/discovery/azure/AzureUnicastHostsProvider.java b/plugins/discovery-azure/src/main/java/org/elasticsearch/discovery/azure/AzureUnicastHostsProvider.java
index b1bf5515195..869fda5f8e2 100644
--- a/plugins/discovery-azure/src/main/java/org/elasticsearch/discovery/azure/AzureUnicastHostsProvider.java
+++ b/plugins/discovery-azure/src/main/java/org/elasticsearch/discovery/azure/AzureUnicastHostsProvider.java
@@ -30,8 +30,10 @@ import org.elasticsearch.cloud.azure.AzureServiceRemoteException;
 import org.elasticsearch.cloud.azure.management.AzureComputeService;
 import org.elasticsearch.cloud.azure.management.AzureComputeService.Discovery;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
@@ -92,7 +94,7 @@ public class AzureUnicastHostsProvider extends AbstractComponent implements Unic
                     return deployment;
                 }
             }
-            return null;
+            throw new IllegalArgumentException("invalid value for deployment type [" + string + "]");
         }
     }
 
@@ -123,21 +125,14 @@ public class AzureUnicastHostsProvider extends AbstractComponent implements Unic
         this.refreshInterval = Discovery.REFRESH_SETTING.get(settings);
 
         this.hostType = Discovery.HOST_TYPE_SETTING.get(settings);
-        this.publicEndpointName = settings.get(Discovery.ENDPOINT_NAME, "elasticsearch");
+        this.publicEndpointName = Discovery.ENDPOINT_NAME_SETTING.get(settings);
 
         // Deployment name could be set with discovery.azure.deployment.name
         // Default to cloud.azure.management.cloud.service.name
-        this.deploymentName = settings.get(Discovery.DEPLOYMENT_NAME);
+        this.deploymentName = Discovery.DEPLOYMENT_NAME_SETTING.get(settings);
 
         // Reading deployment_slot
-        String strDeployment = settings.get(Discovery.DEPLOYMENT_SLOT, Deployment.PRODUCTION.deployment);
-        Deployment tmpDeployment = Deployment.fromString(strDeployment);
-        if (tmpDeployment == null) {
-            logger.warn("wrong value for [{}]: [{}]. falling back to [{}]...", Discovery.DEPLOYMENT_SLOT, strDeployment,
-                    Deployment.PRODUCTION.deployment);
-            tmpDeployment = Deployment.PRODUCTION;
-        }
-        this.deploymentSlot = tmpDeployment.slot;
+        this.deploymentSlot = Discovery.DEPLOYMENT_SLOT_SETTING.get(settings).slot;
     }
 
     /**
@@ -191,7 +186,7 @@ public class AzureUnicastHostsProvider extends AbstractComponent implements Unic
             }
 
             // If provided, we check the deployment name
-            if (deploymentName != null && !deploymentName.equals(deployment.getName())) {
+            if (Strings.hasLength(deploymentName) && !deploymentName.equals(deployment.getName())) {
                 logger.debug("current deployment name [{}] different from [{}]. skipping...",
                         deployment.getName(), deploymentName);
                 continue;
@@ -219,7 +214,7 @@ public class AzureUnicastHostsProvider extends AbstractComponent implements Unic
                             if (privateIp.equals(ipAddress)) {
                                 logger.trace("adding ourselves {}", NetworkAddress.format(ipAddress));
                             }
-                            networkAddress = NetworkAddress.format(privateIp);
+                            networkAddress = InetAddresses.toUriString(privateIp);
                         } else {
                             logger.trace("no private ip provided. ignoring [{}]...", instance.getInstanceName());
                         }
diff --git a/plugins/discovery-azure/src/main/java/org/elasticsearch/plugin/discovery/azure/AzureDiscoveryPlugin.java b/plugins/discovery-azure/src/main/java/org/elasticsearch/plugin/discovery/azure/AzureDiscoveryPlugin.java
index e5de7a0cb02..757ff359d42 100644
--- a/plugins/discovery-azure/src/main/java/org/elasticsearch/plugin/discovery/azure/AzureDiscoveryPlugin.java
+++ b/plugins/discovery-azure/src/main/java/org/elasticsearch/plugin/discovery/azure/AzureDiscoveryPlugin.java
@@ -75,5 +75,8 @@ public class AzureDiscoveryPlugin extends Plugin {
         settingsModule.registerSetting(AzureComputeService.Management.SUBSCRIPTION_ID_SETTING);
         settingsModule.registerSetting(AzureComputeService.Management.SERVICE_NAME_SETTING);
         settingsModule.registerSetting(AzureComputeService.Discovery.HOST_TYPE_SETTING);
+        settingsModule.registerSetting(AzureComputeService.Discovery.DEPLOYMENT_NAME_SETTING);
+        settingsModule.registerSetting(AzureComputeService.Discovery.DEPLOYMENT_SLOT_SETTING);
+        settingsModule.registerSetting(AzureComputeService.Discovery.ENDPOINT_NAME_SETTING);
     }
 }
diff --git a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryClusterFormationTests.java b/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryClusterFormationTests.java
new file mode 100644
index 00000000000..6c2bec3ba43
--- /dev/null
+++ b/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryClusterFormationTests.java
@@ -0,0 +1,285 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.azure;
+
+import com.microsoft.windowsazure.management.compute.models.DeploymentSlot;
+import com.microsoft.windowsazure.management.compute.models.DeploymentStatus;
+import com.sun.net.httpserver.Headers;
+import com.sun.net.httpserver.HttpsConfigurator;
+import com.sun.net.httpserver.HttpsServer;
+import org.elasticsearch.cloud.azure.management.AzureComputeService;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.io.FileSystemUtils;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsModule;
+import org.elasticsearch.discovery.DiscoveryModule;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.plugin.discovery.azure.AzureDiscoveryPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.transport.TransportSettings;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManagerFactory;
+import javax.xml.XMLConstants;
+import javax.xml.stream.XMLOutputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.StringWriter;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.KeyStore;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout;
+
+@ESIntegTestCase.SuppressLocalMode
+@ESIntegTestCase.ClusterScope(numDataNodes = 2, numClientNodes = 0)
+@SuppressForbidden(reason = "use http server")
+// TODO this should be a IT but currently all ITs in this project run against a real cluster
+public class AzureDiscoveryClusterFormationTests extends ESIntegTestCase {
+
+    public static class TestPlugin extends Plugin {
+
+        @Override
+        public String name() {
+            return AzureDiscoveryClusterFormationTests.class.getName();
+        }
+
+        @Override
+        public String description() {
+            return AzureDiscoveryClusterFormationTests.class.getName();
+        }
+
+        public void onModule(SettingsModule settingsModule) {
+            settingsModule.registerSetting(AzureComputeService.Management.ENDPOINT_SETTING);
+        }
+    }
+
+    private static HttpsServer httpsServer;
+    private static Path logDir;
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return pluginList(AzureDiscoveryPlugin.class, TestPlugin.class);
+    }
+
+    private static Path keyStoreFile;
+
+    @BeforeClass
+    public static void setupKeyStore() throws IOException {
+        Path tempDir = createTempDir();
+        keyStoreFile = tempDir.resolve("test-node.jks");
+        try (InputStream stream = AzureDiscoveryClusterFormationTests.class.getResourceAsStream("/test-node.jks")) {
+            assertNotNull("can't find keystore file", stream);
+            Files.copy(stream, keyStoreFile);
+        }
+    }
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        Path resolve = logDir.resolve(Integer.toString(nodeOrdinal));
+        try {
+            Files.createDirectory(resolve);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        return Settings.builder().put(super.nodeSettings(nodeOrdinal))
+            .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), AzureDiscoveryPlugin.AZURE)
+            .put(Environment.PATH_LOGS_SETTING.getKey(), resolve)
+            .put(TransportSettings.PORT.getKey(), 0)
+            .put(Node.WRITE_PORTS_FIELD_SETTING.getKey(), "true")
+            .put(AzureComputeService.Management.ENDPOINT_SETTING.getKey(), "https://" + InetAddress.getLoopbackAddress().getHostAddress() +
+                ":" + httpsServer.getAddress().getPort())
+            .put(Environment.PATH_CONF_SETTING.getKey(), keyStoreFile.getParent().toAbsolutePath())
+            .put(AzureComputeService.Management.KEYSTORE_PATH_SETTING.getKey(), keyStoreFile.toAbsolutePath())
+            .put(AzureComputeService.Discovery.HOST_TYPE_SETTING.getKey(), AzureUnicastHostsProvider.HostType.PUBLIC_IP.name())
+            .put(AzureComputeService.Management.KEYSTORE_PASSWORD_SETTING.getKey(), "keypass")
+            .put(AzureComputeService.Management.KEYSTORE_TYPE_SETTING.getKey(), "jks")
+            .put(AzureComputeService.Management.SERVICE_NAME_SETTING.getKey(), "myservice")
+            .put(AzureComputeService.Management.SUBSCRIPTION_ID_SETTING.getKey(), "subscription")
+            .put(AzureComputeService.Discovery.DEPLOYMENT_NAME_SETTING.getKey(), "mydeployment")
+            .put(AzureComputeService.Discovery.ENDPOINT_NAME_SETTING.getKey(), "myendpoint")
+            .put(AzureComputeService.Discovery.DEPLOYMENT_SLOT_SETTING.getKey(), AzureUnicastHostsProvider.Deployment.PRODUCTION.name())
+            .build();
+    }
+
+    /**
+     * Creates mock EC2 endpoint providing the list of started nodes to the DescribeInstances API call
+     */
+    @BeforeClass
+    public static void startHttpd() throws Exception {
+        logDir = createTempDir();
+        SSLContext sslContext = getSSLContext();
+        httpsServer = HttpsServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 0), 0);
+        httpsServer.setHttpsConfigurator(new HttpsConfigurator(sslContext));
+        httpsServer.createContext("/subscription/services/hostedservices/myservice", (s) -> {
+            Headers headers = s.getResponseHeaders();
+            headers.add("Content-Type", "text/xml; charset=UTF-8");
+            XMLOutputFactory xmlOutputFactory = XMLOutputFactory.newFactory();
+            xmlOutputFactory.setProperty(XMLOutputFactory.IS_REPAIRING_NAMESPACES, true);
+            StringWriter out = new StringWriter();
+            XMLStreamWriter sw;
+            try {
+                sw = xmlOutputFactory.createXMLStreamWriter(out);
+                sw.writeStartDocument();
+
+                String namespace = "http://schemas.microsoft.com/windowsazure";
+                sw.setDefaultNamespace(namespace);
+                sw.writeStartElement(XMLConstants.DEFAULT_NS_PREFIX, "HostedService", namespace);
+                {
+                    sw.writeStartElement("Deployments");
+                    {
+                        Path[] files = FileSystemUtils.files(logDir);
+                        for (int i = 0; i < files.length; i++) {
+                            Path resolve = files[i].resolve("transport.ports");
+                            if (Files.exists(resolve)) {
+                                List<String> addresses = Files.readAllLines(resolve);
+                                Collections.shuffle(addresses, random());
+                                String address = addresses.get(0);
+                                int indexOfLastColon = address.lastIndexOf(':');
+                                String host = address.substring(0, indexOfLastColon);
+                                String port = address.substring(indexOfLastColon + 1);
+
+                                sw.writeStartElement("Deployment");
+                                {
+                                    sw.writeStartElement("Name");
+                                    sw.writeCharacters("mydeployment");
+                                    sw.writeEndElement();
+
+                                    sw.writeStartElement("DeploymentSlot");
+                                    sw.writeCharacters(DeploymentSlot.Production.name());
+                                    sw.writeEndElement();
+
+                                    sw.writeStartElement("Status");
+                                    sw.writeCharacters(DeploymentStatus.Running.name());
+                                    sw.writeEndElement();
+
+                                    sw.writeStartElement("RoleInstanceList");
+                                    {
+                                        sw.writeStartElement("RoleInstance");
+                                        {
+                                            sw.writeStartElement("RoleName");
+                                            sw.writeCharacters(UUID.randomUUID().toString());
+                                            sw.writeEndElement();
+
+                                            sw.writeStartElement("IpAddress");
+                                            sw.writeCharacters(host);
+                                            sw.writeEndElement();
+
+                                            sw.writeStartElement("InstanceEndpoints");
+                                            {
+                                                sw.writeStartElement("InstanceEndpoint");
+                                                {
+                                                    sw.writeStartElement("Name");
+                                                    sw.writeCharacters("myendpoint");
+                                                    sw.writeEndElement();
+
+                                                    sw.writeStartElement("Vip");
+                                                    sw.writeCharacters(host);
+                                                    sw.writeEndElement();
+
+                                                    sw.writeStartElement("PublicPort");
+                                                    sw.writeCharacters(port);
+                                                    sw.writeEndElement();
+                                                }
+                                                sw.writeEndElement();
+                                            }
+                                            sw.writeEndElement();
+                                        }
+                                        sw.writeEndElement();
+                                    }
+                                    sw.writeEndElement();
+                                }
+                                sw.writeEndElement();
+                            }
+                        }
+                    }
+                    sw.writeEndElement();
+                }
+                sw.writeEndElement();
+
+                sw.writeEndDocument();
+                sw.flush();
+
+                final byte[] responseAsBytes = out.toString().getBytes(StandardCharsets.UTF_8);
+                s.sendResponseHeaders(200, responseAsBytes.length);
+                OutputStream responseBody = s.getResponseBody();
+                responseBody.write(responseAsBytes);
+                responseBody.close();
+            } catch (XMLStreamException e) {
+                Loggers.getLogger(AzureDiscoveryClusterFormationTests.class).error("Failed serializing XML", e);
+                throw new RuntimeException(e);
+            }
+        });
+
+        httpsServer.start();
+    }
+
+    private static SSLContext getSSLContext() throws Exception {
+        char[] passphrase = "keypass".toCharArray();
+        KeyStore ks = KeyStore.getInstance("JKS");
+        try (InputStream stream = AzureDiscoveryClusterFormationTests.class.getResourceAsStream("/test-node.jks")) {
+            assertNotNull("can't find keystore file", stream);
+            ks.load(stream, passphrase);
+        }
+        KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509");
+        kmf.init(ks, passphrase);
+        TrustManagerFactory tmf = TrustManagerFactory.getInstance("SunX509");
+        tmf.init(ks);
+        SSLContext ssl = SSLContext.getInstance("TLS");
+        ssl.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null);
+        return ssl;
+    }
+
+    @AfterClass
+    public static void stopHttpd() throws IOException {
+        for (int i = 0; i < internalCluster().size(); i++) {
+            // shut them all down otherwise we get spammed with connection refused exceptions
+            internalCluster().stopRandomDataNode();
+        }
+        httpsServer.stop(0);
+        httpsServer = null;
+        logDir = null;
+    }
+
+    public void testJoin() throws ExecutionException, InterruptedException {
+        // only wait for the cluster to form
+        assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(2)).get());
+        // add one more node and wait for it to join
+        internalCluster().startDataOnlyNodeAsync().get();
+        assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(3)).get());
+    }
+}
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java
index d458fc2c73a..6d1104ce6e2 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java
@@ -248,7 +248,7 @@ public class Ec2DiscoveryClusterFormationTests extends ESIntegTestCase {
         logDir = null;
     }
 
-    public void testJoin() throws ExecutionException, InterruptedException, XMLStreamException {
+    public void testJoin() throws ExecutionException, InterruptedException {
         // only wait for the cluster to form
         assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(2)).get());
         // add one more node and wait for it to join

From f349c4f1356f9365d39edf414f7e9950e3734c46 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 29 Apr 2016 10:26:34 -0400
Subject: [PATCH 0022/1311] Return null for "same" thread pool info

This commit modifes the EsThreadPoolTestCase#info helper method to
return null when info for the thread pool can not be found. This really
should only happen for the "same" thread pool, and so we also assert
that we only get to a place where there is no info if the thread pool
that info was requested for is in fact the "same" thread pool. Not
returning null here and instead throwing an exception would fail tests
that tried to lookup info on the "same" thread pool.
---
 .../org/elasticsearch/threadpool/ESThreadPoolTestCase.java     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/threadpool/ESThreadPoolTestCase.java b/core/src/test/java/org/elasticsearch/threadpool/ESThreadPoolTestCase.java
index 7fbd3ccd31b..21035811905 100644
--- a/core/src/test/java/org/elasticsearch/threadpool/ESThreadPoolTestCase.java
+++ b/core/src/test/java/org/elasticsearch/threadpool/ESThreadPoolTestCase.java
@@ -32,7 +32,8 @@ public abstract class ESThreadPoolTestCase extends ESTestCase {
                 return info;
             }
         }
-        throw new IllegalArgumentException(name);
+        assert "same".equals(name);
+        return null;
     }
 
     protected final ThreadPoolStats.Stats stats(final ThreadPool threadPool, final String name) {

From 07c2fbf83a6786835462630f640584d1932a4a29 Mon Sep 17 00:00:00 2001
From: Tal Levy <JubBoy333@gmail.com>
Date: Fri, 29 Apr 2016 07:58:27 -0700
Subject: [PATCH 0023/1311] Validate properties values according to database
 type (#17940)

Fixes #17683.
---
 .../ingest/geoip/GeoIpProcessor.java          | 64 +++++++++++++------
 .../geoip/GeoIpProcessorFactoryTests.java     | 41 +++++++++++-
 .../test/ingest_geoip/20_geoip_processor.yaml |  2 +-
 3 files changed, 86 insertions(+), 21 deletions(-)

diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
index 4b153d468ae..c524bf15a3d 100644
--- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
+++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
@@ -58,6 +58,8 @@ import static org.elasticsearch.ingest.core.ConfigurationUtils.readStringPropert
 public final class GeoIpProcessor extends AbstractProcessor {
 
     public static final String TYPE = "geoip";
+    private static final String CITY_DB_TYPE = "GeoLite2-City";
+    private static final String COUNTRY_DB_TYPE = "GeoLite2-Country";
 
     private final String field;
     private final String targetField;
@@ -79,14 +81,14 @@ public final class GeoIpProcessor extends AbstractProcessor {
 
         Map<String, Object> geoData;
         switch (dbReader.getMetadata().getDatabaseType()) {
-            case "GeoLite2-City":
+            case CITY_DB_TYPE:
                 try {
                     geoData = retrieveCityGeoData(ipAddress);
                 } catch (AddressNotFoundRuntimeException e) {
                     geoData = Collections.emptyMap();
                 }
                 break;
-            case "GeoLite2-Country":
+            case COUNTRY_DB_TYPE:
                 try {
                     geoData = retrieveCountryGeoData(ipAddress);
                 } catch (AddressNotFoundRuntimeException e) {
@@ -215,10 +217,11 @@ public final class GeoIpProcessor extends AbstractProcessor {
     }
 
     public static final class Factory extends AbstractProcessorFactory<GeoIpProcessor> implements Closeable {
-
-        static final Set<Property> DEFAULT_PROPERTIES = EnumSet.of(
-                Property.CONTINENT_NAME, Property.COUNTRY_ISO_CODE, Property.REGION_NAME, Property.CITY_NAME, Property.LOCATION
+        static final Set<Property> DEFAULT_CITY_PROPERTIES = EnumSet.of(
+            Property.CONTINENT_NAME, Property.COUNTRY_ISO_CODE, Property.REGION_NAME,
+            Property.CITY_NAME, Property.LOCATION
         );
+        static final Set<Property> DEFAULT_COUNTRY_PROPERTIES = EnumSet.of(Property.CONTINENT_NAME, Property.COUNTRY_ISO_CODE);
 
         private final Map<String, DatabaseReader> databaseReaders;
 
@@ -233,24 +236,33 @@ public final class GeoIpProcessor extends AbstractProcessor {
             String databaseFile = readStringProperty(TYPE, processorTag, config, "database_file", "GeoLite2-City.mmdb");
             List<String> propertyNames = readOptionalList(TYPE, processorTag, config, "properties");
 
+            DatabaseReader databaseReader = databaseReaders.get(databaseFile);
+            if (databaseReader == null) {
+                throw newConfigurationException(TYPE, processorTag, "database_file", "database file [" + databaseFile + "] doesn't exist");
+            }
+
+            String databaseType = databaseReader.getMetadata().getDatabaseType();
+
             final Set<Property> properties;
             if (propertyNames != null) {
                 properties = EnumSet.noneOf(Property.class);
                 for (String fieldName : propertyNames) {
                     try {
-                        properties.add(Property.parse(fieldName));
-                    } catch (Exception e) {
-                        throw newConfigurationException(TYPE, processorTag, "properties", "illegal field option [" + fieldName + "]. valid values are [" + Arrays.toString(Property.values()) + "]");
+                        properties.add(Property.parseProperty(databaseType, fieldName));
+                    } catch (IllegalArgumentException e) {
+                        throw newConfigurationException(TYPE, processorTag, "properties", e.getMessage());
                     }
                 }
             } else {
-                properties = DEFAULT_PROPERTIES;
+                if (CITY_DB_TYPE.equals(databaseType)) {
+                    properties = DEFAULT_CITY_PROPERTIES;
+                } else if (COUNTRY_DB_TYPE.equals(databaseType)) {
+                    properties = DEFAULT_COUNTRY_PROPERTIES;
+                } else {
+                    throw newConfigurationException(TYPE, processorTag, "database_file", "Unsupported database type [" + databaseType + "]");
+                }
             }
 
-            DatabaseReader databaseReader = databaseReaders.get(databaseFile);
-            if (databaseReader == null) {
-                throw newConfigurationException(TYPE, processorTag, "database_file", "database file [" + databaseFile + "] doesn't exist");
-            }
             return new GeoIpProcessor(processorTag, ipField, databaseReader, targetField, properties);
         }
 
@@ -279,13 +291,29 @@ public final class GeoIpProcessor extends AbstractProcessor {
         REGION_NAME,
         CITY_NAME,
         TIMEZONE,
-        LATITUDE,
-        LONGITUDE,
         LOCATION;
 
-        public static Property parse(String value) {
-            return valueOf(value.toUpperCase(Locale.ROOT));
+        static final EnumSet<Property> ALL_CITY_PROPERTIES = EnumSet.allOf(Property.class);
+        static final EnumSet<Property> ALL_COUNTRY_PROPERTIES = EnumSet.of(Property.IP, Property.CONTINENT_NAME,
+            Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE);
+
+        public static Property parseProperty(String databaseType, String value) {
+            Set<Property> validProperties = EnumSet.noneOf(Property.class);
+            if (CITY_DB_TYPE.equals(databaseType)) {
+                validProperties = ALL_CITY_PROPERTIES;
+            } else if (COUNTRY_DB_TYPE.equals(databaseType)) {
+                validProperties = ALL_COUNTRY_PROPERTIES;
+            }
+
+            try {
+                Property property = valueOf(value.toUpperCase(Locale.ROOT));
+                if (validProperties.contains(property) == false) {
+                    throw new IllegalArgumentException("invalid");
+                }
+                return property;
+            } catch (IllegalArgumentException e) {
+                throw new IllegalArgumentException("illegal property value [" + value + "]. valid values are " + Arrays.toString(validProperties.toArray()));
+            }
         }
     }
-
 }
diff --git a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
index 794f64bf01f..0840b10b004 100644
--- a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
+++ b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
@@ -19,8 +19,10 @@
 
 package org.elasticsearch.ingest.geoip;
 
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import com.maxmind.geoip2.DatabaseReader;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.Randomness;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.StreamsUtils;
@@ -79,7 +81,25 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("geoip"));
         assertThat(processor.getDbReader().getMetadata().getDatabaseType(), equalTo("GeoLite2-City"));
-        assertThat(processor.getProperties(), sameInstance(GeoIpProcessor.Factory.DEFAULT_PROPERTIES));
+        assertThat(processor.getProperties(), sameInstance(GeoIpProcessor.Factory.DEFAULT_CITY_PROPERTIES));
+    }
+
+    public void testCountryBuildDefaults() throws Exception {
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders);
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("database_file", "GeoLite2-Country.mmdb");
+
+        String processorTag = randomAsciiOfLength(10);
+        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
+
+        GeoIpProcessor processor = factory.create(config);
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertThat(processor.getField(), equalTo("_field"));
+        assertThat(processor.getTargetField(), equalTo("geoip"));
+        assertThat(processor.getDbReader().getMetadata().getDatabaseType(), equalTo("GeoLite2-Country"));
+        assertThat(processor.getProperties(), sameInstance(GeoIpProcessor.Factory.DEFAULT_COUNTRY_PROPERTIES));
     }
 
     public void testBuildTargetField() throws Exception {
@@ -101,6 +121,23 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("geoip"));
         assertThat(processor.getDbReader().getMetadata().getDatabaseType(), equalTo("GeoLite2-Country"));
+        assertThat(processor.getProperties(), sameInstance(GeoIpProcessor.Factory.DEFAULT_COUNTRY_PROPERTIES));
+    }
+
+    public void testBuildWithCountryDbAndCityFields() throws Exception {
+        GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders);
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("database_file", "GeoLite2-Country.mmdb");
+        EnumSet<GeoIpProcessor.Property> cityOnlyProperties = EnumSet.complementOf(GeoIpProcessor.Property.ALL_COUNTRY_PROPERTIES);
+        String cityProperty = RandomPicks.randomFrom(Randomness.get(), cityOnlyProperties).toString();
+        config.put("properties", Collections.singletonList(cityProperty));
+        try {
+            factory.create(config);
+            fail("Exception expected");
+        } catch (ElasticsearchParseException e) {
+            assertThat(e.getMessage(), equalTo("[properties] illegal property value [" + cityProperty + "]. valid values are [IP, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_NAME]"));
+        }
     }
 
     public void testBuildNonExistingDbFile() throws Exception {
@@ -146,7 +183,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
             factory.create(config);
             fail("exception expected");
         } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), equalTo("[properties] illegal field option [invalid]. valid values are [[IP, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_NAME, REGION_NAME, CITY_NAME, TIMEZONE, LATITUDE, LONGITUDE, LOCATION]]"));
+            assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [IP, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_NAME, REGION_NAME, CITY_NAME, TIMEZONE, LOCATION]"));
         }
 
         config = new HashMap<>();
diff --git a/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yaml b/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yaml
index 219497915d7..5e8a3e7c1ff 100644
--- a/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yaml
+++ b/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yaml
@@ -54,7 +54,7 @@
               {
                 "geoip" : {
                   "field" : "field1",
-                  "properties" : ["city_name", "country_iso_code", "ip", "latitude", "longitude", "location", "timezone", "country_name", "region_name", "continent_name"]
+                  "properties" : ["city_name", "country_iso_code", "ip", "location", "timezone", "country_name", "region_name", "continent_name"]
                 }
               }
             ]

From 7aca1389e2c57d12cb3f62cc293ef91b4817d1fe Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Mon, 25 Apr 2016 23:22:32 +0200
Subject: [PATCH 0024/1311] ingest: Add `date_index_name` processor.

Closes #17814
---
 .../processor/DateIndexNameProcessor.java     | 156 ++++++++++++++++++
 .../org/elasticsearch/node/NodeModule.java    |   2 +
 .../processor/DateIndexNameFactoryTests.java  |  99 +++++++++++
 .../DateIndexNameProcessorTests.java          |  77 +++++++++
 docs/reference/ingest/ingest-node.asciidoc    |  61 +++++++
 .../test/ingest_grok/10_basic.yaml            |  25 +--
 .../test/ingest_attachment/10_basic.yaml      |  23 +--
 .../test/ingest_geoip/10_basic.yaml           |  25 +--
 .../ingest/90_date_index_name_processor.yaml  |  29 ++++
 9 files changed, 462 insertions(+), 35 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/ingest/processor/DateIndexNameProcessor.java
 create mode 100644 core/src/test/java/org/elasticsearch/ingest/processor/DateIndexNameFactoryTests.java
 create mode 100644 core/src/test/java/org/elasticsearch/ingest/processor/DateIndexNameProcessorTests.java
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/ingest/90_date_index_name_processor.yaml

diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/DateIndexNameProcessor.java b/core/src/main/java/org/elasticsearch/ingest/processor/DateIndexNameProcessor.java
new file mode 100644
index 00000000000..2de44addaf5
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/ingest/processor/DateIndexNameProcessor.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.processor;
+
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.ingest.core.AbstractProcessor;
+import org.elasticsearch.ingest.core.AbstractProcessorFactory;
+import org.elasticsearch.ingest.core.ConfigurationUtils;
+import org.elasticsearch.ingest.core.IngestDocument;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.IllformedLocaleException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.function.Function;
+
+public final class DateIndexNameProcessor extends AbstractProcessor {
+
+    public static final String TYPE = "date_index_name";
+
+    private final String field;
+    private final String indexNamePrefix;
+    private final String dateRounding;
+    private final String indexNameFormat;
+    private final DateTimeZone timezone;
+    private final List<Function<String, DateTime>> dateFormats;
+
+    DateIndexNameProcessor(String tag, String field, List<Function<String, DateTime>> dateFormats, DateTimeZone timezone,
+                           String indexNamePrefix, String dateRounding, String indexNameFormat) {
+        super(tag);
+        this.field = field;
+        this.timezone = timezone;
+        this.dateFormats = dateFormats;
+        this.indexNamePrefix = indexNamePrefix;
+        this.dateRounding = dateRounding;
+        this.indexNameFormat = indexNameFormat;
+    }
+
+    @Override
+    public void execute(IngestDocument ingestDocument) throws Exception {
+        String date = ingestDocument.getFieldValue(field, String.class);
+
+        DateTime dateTime = null;
+        Exception lastException = null;
+        for (Function<String, DateTime> dateParser : dateFormats) {
+            try {
+                dateTime = dateParser.apply(date);
+            } catch (Exception e) {
+                //try the next parser and keep track of the exceptions
+                lastException = ExceptionsHelper.useOrSuppress(lastException, e);
+            }
+        }
+
+        if (dateTime == null) {
+            throw new IllegalArgumentException("unable to parse date [" + date + "]", lastException);
+        }
+
+        DateTimeFormatter formatter = DateTimeFormat.forPattern(indexNameFormat);
+        StringBuilder builder = new StringBuilder()
+                .append('<')
+                .append(indexNamePrefix)
+                    .append('{')
+                        .append(formatter.print(dateTime)).append("||/").append(dateRounding)
+                            .append('{').append(indexNameFormat).append('|').append(timezone).append('}')
+                    .append('}')
+                .append('>');
+        String dynamicIndexName  = builder.toString();
+        ingestDocument.setFieldValue(IngestDocument.MetaData.INDEX.getFieldName(), dynamicIndexName);
+    }
+
+    @Override
+    public String getType() {
+        return TYPE;
+    }
+
+    String getField() {
+        return field;
+    }
+
+    String getIndexNamePrefix() {
+        return indexNamePrefix;
+    }
+
+    String getDateRounding() {
+        return dateRounding;
+    }
+
+    String getIndexNameFormat() {
+        return indexNameFormat;
+    }
+
+    DateTimeZone getTimezone() {
+        return timezone;
+    }
+
+    List<Function<String, DateTime>> getDateFormats() {
+        return dateFormats;
+    }
+
+    public static final class Factory extends AbstractProcessorFactory<DateIndexNameProcessor> {
+
+        @Override
+        protected DateIndexNameProcessor doCreate(String tag, Map<String, Object> config) throws Exception {
+            String localeString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "locale");
+            String timezoneString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "timezone");
+            DateTimeZone timezone = timezoneString == null ? DateTimeZone.UTC : DateTimeZone.forID(timezoneString);
+            Locale locale = Locale.ENGLISH;
+            if (localeString != null) {
+                try {
+                    locale = (new Locale.Builder()).setLanguageTag(localeString).build();
+                } catch (IllformedLocaleException e) {
+                    throw new IllegalArgumentException("Invalid language tag specified: " + localeString);
+                }
+            }
+            List<String> dateFormatStrings = ConfigurationUtils.readOptionalList(TYPE, tag, config, "date_formats");
+            if (dateFormatStrings == null) {
+                dateFormatStrings = Collections.singletonList("yyyy-MM-dd'T'HH:mm:ss.SSSZ");
+            }
+            List<Function<String, DateTime>> dateFormats = new ArrayList<>(dateFormatStrings.size());
+            for (String format : dateFormatStrings) {
+                DateFormat dateFormat = DateFormat.fromString(format);
+                dateFormats.add(dateFormat.getFunction(format, timezone, locale));
+            }
+
+            String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field");
+            String indexNamePrefix = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_prefix", "");
+            String dateRounding = ConfigurationUtils.readStringProperty(TYPE, tag, config, "date_rounding");
+            String indexNameFormat = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_format", "yyyy-MM-dd");
+            return new DateIndexNameProcessor(tag, field, dateFormats, timezone, indexNamePrefix, dateRounding, indexNameFormat);
+        }
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/node/NodeModule.java b/core/src/main/java/org/elasticsearch/node/NodeModule.java
index 935b240b3d9..8565b14be8e 100644
--- a/core/src/main/java/org/elasticsearch/node/NodeModule.java
+++ b/core/src/main/java/org/elasticsearch/node/NodeModule.java
@@ -28,6 +28,7 @@ import org.elasticsearch.ingest.core.TemplateService;
 import org.elasticsearch.ingest.processor.AppendProcessor;
 import org.elasticsearch.ingest.processor.ConvertProcessor;
 import org.elasticsearch.ingest.processor.DateProcessor;
+import org.elasticsearch.ingest.processor.DateIndexNameProcessor;
 import org.elasticsearch.ingest.processor.FailProcessor;
 import org.elasticsearch.ingest.processor.ForEachProcessor;
 import org.elasticsearch.ingest.processor.GsubProcessor;
@@ -76,6 +77,7 @@ public class NodeModule extends AbstractModule {
         registerProcessor(GsubProcessor.TYPE, (templateService, registry) -> new GsubProcessor.Factory());
         registerProcessor(FailProcessor.TYPE, (templateService, registry) -> new FailProcessor.Factory(templateService));
         registerProcessor(ForEachProcessor.TYPE, (templateService, registry) -> new ForEachProcessor.Factory(registry));
+        registerProcessor(DateIndexNameProcessor.TYPE, (templateService, registry) -> new DateIndexNameProcessor.Factory());
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/DateIndexNameFactoryTests.java b/core/src/test/java/org/elasticsearch/ingest/processor/DateIndexNameFactoryTests.java
new file mode 100644
index 00000000000..5bb674e950a
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/ingest/processor/DateIndexNameFactoryTests.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.processor;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matchers;
+import org.joda.time.DateTimeZone;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+public class DateIndexNameFactoryTests extends ESTestCase {
+
+    public void testDefaults() throws Exception {
+        DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory();
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("date_rounding", "y");
+
+        DateIndexNameProcessor processor = factory.create(config);
+        assertThat(processor.getDateFormats().size(), Matchers.equalTo(1));
+        assertThat(processor.getField(), Matchers.equalTo("_field"));
+        assertThat(processor.getIndexNamePrefix(), Matchers.equalTo(""));
+        assertThat(processor.getDateRounding(), Matchers.equalTo("y"));
+        assertThat(processor.getIndexNameFormat(), Matchers.equalTo("yyyy-MM-dd"));
+        assertThat(processor.getTimezone(), Matchers.equalTo(DateTimeZone.UTC));
+    }
+
+    public void testSpecifyOptionalSettings() throws Exception {
+        DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory();
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("index_name_prefix", "_prefix");
+        config.put("date_rounding", "y");
+        config.put("date_formats", Arrays.asList("UNIX", "UNIX_MS"));
+
+        DateIndexNameProcessor processor = factory.create(config);
+        assertThat(processor.getDateFormats().size(), Matchers.equalTo(2));
+
+        config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("index_name_prefix", "_prefix");
+        config.put("date_rounding", "y");
+        config.put("index_name_format", "yyyyMMdd");
+
+        processor = factory.create(config);
+        assertThat(processor.getIndexNameFormat(), Matchers.equalTo("yyyyMMdd"));
+
+        config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("index_name_prefix", "_prefix");
+        config.put("date_rounding", "y");
+        config.put("timezone", "+02:00");
+
+        processor = factory.create(config);
+        assertThat(processor.getTimezone(), Matchers.equalTo(DateTimeZone.forOffsetHours(2)));
+
+        config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("index_name_prefix", "_prefix");
+        config.put("date_rounding", "y");
+
+        processor = factory.create(config);
+        assertThat(processor.getIndexNamePrefix(), Matchers.equalTo("_prefix"));
+    }
+
+    public void testRequiredFields() throws Exception {
+        DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory();
+        Map<String, Object> config = new HashMap<>();
+        config.put("date_rounding", "y");
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        assertThat(e.getMessage(), Matchers.equalTo("[field] required property is missing"));
+
+        config.clear();
+        config.put("field", "_field");
+        e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        assertThat(e.getMessage(), Matchers.equalTo("[date_rounding] required property is missing"));
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/DateIndexNameProcessorTests.java b/core/src/test/java/org/elasticsearch/ingest/processor/DateIndexNameProcessorTests.java
new file mode 100644
index 00000000000..2dd5934387c
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/ingest/processor/DateIndexNameProcessorTests.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.ingest.processor;
+
+import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.test.ESTestCase;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+import java.util.Collections;
+import java.util.Locale;
+import java.util.function.Function;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class DateIndexNameProcessorTests extends ESTestCase {
+
+    public void testJodaPattern() throws Exception {
+        Function<String, DateTime> function = DateFormat.Joda.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZ", DateTimeZone.UTC, Locale.ROOT);
+        DateIndexNameProcessor processor = new DateIndexNameProcessor(
+                "_tag", "_field", Collections.singletonList(function), DateTimeZone.UTC,
+                "events-", "y", "yyyyMMdd"
+        );
+
+        IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, null,
+                Collections.singletonMap("_field", "2016-04-25T12:24:20.101Z"));
+        processor.execute(document);
+        assertThat(document.getSourceAndMetadata().get("_index"), equalTo("<events-{20160425||/y{yyyyMMdd|UTC}}>"));
+    }
+
+    public void testTAI64N()throws Exception {
+        Function<String, DateTime> function = DateFormat.Tai64n.getFunction(null, DateTimeZone.UTC, null);
+        DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function),
+                DateTimeZone.UTC, "events-", "m", "yyyyMMdd");
+        IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, null,
+                Collections.singletonMap("_field", (randomBoolean() ? "@" : "") + "4000000050d506482dbdf024"));
+        dateProcessor.execute(document);
+        assertThat(document.getSourceAndMetadata().get("_index"), equalTo("<events-{20121222||/m{yyyyMMdd|UTC}}>"));
+    }
+
+    public void testUnixMs()throws Exception {
+        Function<String, DateTime> function = DateFormat.UnixMs.getFunction(null, DateTimeZone.UTC, null);
+        DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function),
+                DateTimeZone.UTC, "events-", "m", "yyyyMMdd");
+        IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, null,
+                Collections.singletonMap("_field", "1000500"));
+        dateProcessor.execute(document);
+        assertThat(document.getSourceAndMetadata().get("_index"), equalTo("<events-{19700101||/m{yyyyMMdd|UTC}}>"));
+    }
+
+    public void testUnix()throws Exception {
+        Function<String, DateTime> function = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null);
+        DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function),
+                DateTimeZone.UTC, "events-", "m", "yyyyMMdd");
+        IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, null,
+                Collections.singletonMap("_field", "1000.5"));
+        dateProcessor.execute(document);
+        assertThat(document.getSourceAndMetadata().get("_index"), equalTo("<events-{19700101||/m{yyyyMMdd|UTC}}>"));
+    }
+
+}
diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc
index 93dabbd9122..7831ee86290 100644
--- a/docs/reference/ingest/ingest-node.asciidoc
+++ b/docs/reference/ingest/ingest-node.asciidoc
@@ -739,6 +739,67 @@ Here is an example that adds the parsed date to the `timestamp` field based on t
 }
 --------------------------------------------------
 
+[[date-index-name-processor]]
+=== Date Index Name Processor
+
+The purpose of this processor is to point documents to the right time based index based
+on a date or timestamp field in a document by using the <<date-math-index-names, date math index name support>>.
+
+The processor sets the `_index` meta field with a date math index name expression based on the provided index name
+prefix, a date or timestamp field in the documents being processed and the provided date rounding.
+
+First this processor fetches the date or timestamp from a field in the document being processed. Optionally
+date formatting can be configured on how the field's value should be parsed into a date. Then this date,
+the provided index name prefix and the provided date rounding get formatted into a date math index name expression.
+Also here optionally date formatting can be specified on how the date should be formatted into a date math index name
+expression.
+
+An example pipeline that points documents to a monthly index that starts with a `myindex-` prefix based on a
+date in the `date1` field:
+
+[source,js]
+--------------------------------------------------
+PUT _ingest/pipeline/1
+{
+  "processors" : [
+    {
+      "date_index_name" : {
+        "field" : "date1",
+        "index_name_prefix" : "myindex-",
+        "date_rounding" : "m"
+      }
+    }
+  ]
+}
+--------------------------------------------------
+
+Using that pipeline for an index request:
+
+[source,js]
+--------------------------------------------------
+PUT /myindex/type/1?pipeline=1
+{
+  "date1" : "2016-04-25T12:02:01.789Z"
+}
+--------------------------------------------------
+
+The above request will not index this document into the `myindex` index, but into the `myindex-2016-04-01` index.
+This is because the date is being rounded by month.
+
+[[date-index-name-options]]
+.Date index name options
+[options="header"]
+|======
+| Name                   | Required  | Default                      | Description
+| `field`                | yes       | -                            | The field to get the date or timestamp from.
+| `index_name_prefix`    | no        | -                            | A prefix of the index name to be prepended before the printed date.
+| `date_rounding`        | yes       | -                            | How to round the date when formatting the date into the index name. Valid values are: `y` (year), `m` (month), `w` (week), `d` (day), `h` (hour), `m` (minute) and `s` (second).
+| `date_formats `        | no        | yyyy-MM-dd'T'HH:mm:ss.SSSZ   | An array of the expected date formats for parsing dates / timestamps in the document being preprocessed. Can be a Joda pattern or one of the following formats: ISO8601, UNIX, UNIX_MS, or TAI64N.
+| `timezone`             | no        | UTC                          | The timezone to use when parsing the date and when date math index supports resolves expressions into concrete index names.
+| `locale`               | no        | ENGLISH                      | The locale to use when parsing the date from the document being preprocessed, relevant when parsing month names or week days.
+| `index_name_format`    | no        | yyyy-MM-dd                   | The format to be used when printing the parsed date into the index name. An valid Joda pattern is expected here.
+|======
+
 [[fail-processor]]
 === Fail Processor
 Raises an exception. This is useful for when
diff --git a/modules/ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok/10_basic.yaml b/modules/ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok/10_basic.yaml
index ebb310ecf7a..f61027c3de3 100644
--- a/modules/ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok/10_basic.yaml
+++ b/modules/ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok/10_basic.yaml
@@ -12,15 +12,16 @@
     - match:  { nodes.$master.ingest.processors.0.type: append }
     - match:  { nodes.$master.ingest.processors.1.type: convert }
     - match:  { nodes.$master.ingest.processors.2.type: date }
-    - match:  { nodes.$master.ingest.processors.3.type: fail }
-    - match:  { nodes.$master.ingest.processors.4.type: foreach }
-    - match:  { nodes.$master.ingest.processors.5.type: grok }
-    - match:  { nodes.$master.ingest.processors.6.type: gsub }
-    - match:  { nodes.$master.ingest.processors.7.type: join }
-    - match:  { nodes.$master.ingest.processors.8.type: lowercase }
-    - match:  { nodes.$master.ingest.processors.9.type: remove }
-    - match:  { nodes.$master.ingest.processors.10.type: rename }
-    - match:  { nodes.$master.ingest.processors.11.type: set }
-    - match:  { nodes.$master.ingest.processors.12.type: split }
-    - match:  { nodes.$master.ingest.processors.13.type: trim }
-    - match:  { nodes.$master.ingest.processors.14.type: uppercase }
+    - match:  { nodes.$master.ingest.processors.3.type: date_index_name }
+    - match:  { nodes.$master.ingest.processors.4.type: fail }
+    - match:  { nodes.$master.ingest.processors.5.type: foreach }
+    - match:  { nodes.$master.ingest.processors.6.type: grok }
+    - match:  { nodes.$master.ingest.processors.7.type: gsub }
+    - match:  { nodes.$master.ingest.processors.8.type: join }
+    - match:  { nodes.$master.ingest.processors.9.type: lowercase }
+    - match:  { nodes.$master.ingest.processors.10.type: remove }
+    - match:  { nodes.$master.ingest.processors.11.type: rename }
+    - match:  { nodes.$master.ingest.processors.12.type: set }
+    - match:  { nodes.$master.ingest.processors.13.type: split }
+    - match:  { nodes.$master.ingest.processors.14.type: trim }
+    - match:  { nodes.$master.ingest.processors.15.type: uppercase }
diff --git a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/10_basic.yaml b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/10_basic.yaml
index 7c789b9c2ca..697ad558bb7 100644
--- a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/10_basic.yaml
+++ b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/10_basic.yaml
@@ -12,15 +12,16 @@
     - match:  { nodes.$master.ingest.processors.1.type: attachment }
     - match:  { nodes.$master.ingest.processors.2.type: convert }
     - match:  { nodes.$master.ingest.processors.3.type: date }
-    - match:  { nodes.$master.ingest.processors.4.type: fail }
-    - match:  { nodes.$master.ingest.processors.5.type: foreach }
-    - match:  { nodes.$master.ingest.processors.6.type: gsub }
-    - match:  { nodes.$master.ingest.processors.7.type: join }
-    - match:  { nodes.$master.ingest.processors.8.type: lowercase }
-    - match:  { nodes.$master.ingest.processors.9.type: remove }
-    - match:  { nodes.$master.ingest.processors.10.type: rename }
-    - match:  { nodes.$master.ingest.processors.11.type: set }
-    - match:  { nodes.$master.ingest.processors.12.type: split }
-    - match:  { nodes.$master.ingest.processors.13.type: trim }
-    - match:  { nodes.$master.ingest.processors.14.type: uppercase }
+    - match:  { nodes.$master.ingest.processors.4.type: date_index_name }
+    - match:  { nodes.$master.ingest.processors.5.type: fail }
+    - match:  { nodes.$master.ingest.processors.6.type: foreach }
+    - match:  { nodes.$master.ingest.processors.7.type: gsub }
+    - match:  { nodes.$master.ingest.processors.8.type: join }
+    - match:  { nodes.$master.ingest.processors.9.type: lowercase }
+    - match:  { nodes.$master.ingest.processors.10.type: remove }
+    - match:  { nodes.$master.ingest.processors.11.type: rename }
+    - match:  { nodes.$master.ingest.processors.12.type: set }
+    - match:  { nodes.$master.ingest.processors.13.type: split }
+    - match:  { nodes.$master.ingest.processors.14.type: trim }
+    - match:  { nodes.$master.ingest.processors.15.type: uppercase }
 
diff --git a/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/10_basic.yaml b/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/10_basic.yaml
index cf86f4c7f4c..e405bbb5a6c 100644
--- a/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/10_basic.yaml
+++ b/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/10_basic.yaml
@@ -11,15 +11,16 @@
     - match:  { nodes.$master.ingest.processors.0.type: append }
     - match:  { nodes.$master.ingest.processors.1.type: convert }
     - match:  { nodes.$master.ingest.processors.2.type: date }
-    - match:  { nodes.$master.ingest.processors.3.type: fail }
-    - match:  { nodes.$master.ingest.processors.4.type: foreach }
-    - match:  { nodes.$master.ingest.processors.5.type: geoip }
-    - match:  { nodes.$master.ingest.processors.6.type: gsub }
-    - match:  { nodes.$master.ingest.processors.7.type: join }
-    - match:  { nodes.$master.ingest.processors.8.type: lowercase }
-    - match:  { nodes.$master.ingest.processors.9.type: remove }
-    - match:  { nodes.$master.ingest.processors.10.type: rename }
-    - match:  { nodes.$master.ingest.processors.11.type: set }
-    - match:  { nodes.$master.ingest.processors.12.type: split }
-    - match:  { nodes.$master.ingest.processors.13.type: trim }
-    - match:  { nodes.$master.ingest.processors.14.type: uppercase }
+    - match:  { nodes.$master.ingest.processors.3.type: date_index_name }
+    - match:  { nodes.$master.ingest.processors.4.type: fail }
+    - match:  { nodes.$master.ingest.processors.5.type: foreach }
+    - match:  { nodes.$master.ingest.processors.6.type: geoip }
+    - match:  { nodes.$master.ingest.processors.7.type: gsub }
+    - match:  { nodes.$master.ingest.processors.8.type: join }
+    - match:  { nodes.$master.ingest.processors.9.type: lowercase }
+    - match:  { nodes.$master.ingest.processors.10.type: remove }
+    - match:  { nodes.$master.ingest.processors.11.type: rename }
+    - match:  { nodes.$master.ingest.processors.12.type: set }
+    - match:  { nodes.$master.ingest.processors.13.type: split }
+    - match:  { nodes.$master.ingest.processors.14.type: trim }
+    - match:  { nodes.$master.ingest.processors.15.type: uppercase }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/90_date_index_name_processor.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/90_date_index_name_processor.yaml
new file mode 100644
index 00000000000..c99c1025c10
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/90_date_index_name_processor.yaml
@@ -0,0 +1,29 @@
+---
+"Test date index name processor with defaults":
+  - do:
+      ingest.put_pipeline:
+        id: "1"
+        body:  >
+          {
+            "processors": [
+              {
+                "date_index_name" : {
+                  "field" : "date",
+                  "index_name_prefix": "events-",
+                  "date_rounding": "d"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: events
+        type: event
+        id: 1
+        pipeline: "1"
+        body: {
+          date: "2016-04-22T16:32:14.968Z"
+        }
+  - match: { _index: "events-2016-04-22"}

From a31b94e23706a44e1ece9c270021787fa255afd8 Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Thu, 28 Apr 2016 23:50:49 -0400
Subject: [PATCH 0025/1311] When checking if an index tombstone can be applied,
 use both the index name and uuid because the cluster state may contain an
 active index of the same name (but different uuid).

Closes #18058
Closes #18054
---
 .../elasticsearch/indices/IndicesService.java |  4 +--
 .../indices/IndicesServiceTests.java          | 32 +++++++++++++++++++
 2 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index 48973bca5bd..61aaf551610 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -681,8 +681,8 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
      */
     @Nullable
     public IndexMetaData verifyIndexIsDeleted(final Index index, final ClusterState clusterState) {
-        // this method should only be called when we know the index is not part of the cluster state
-        if (clusterState.metaData().hasIndex(index.getName())) {
+        // this method should only be called when we know the index (name + uuid) is not part of the cluster state
+        if (clusterState.metaData().index(index) != null) {
             throw new IllegalStateException("Cannot delete index [" + index + "], it is still part of the cluster state.");
         }
         if (nodeEnv.hasNodeFile() && FileSystemUtils.exists(nodeEnv.indexPaths(index))) {
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
index 02c415c247a..76f7a30e078 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
@@ -21,8 +21,10 @@ package org.elasticsearch.indices;
 import org.apache.lucene.store.LockObtainFailedException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.AliasAction;
+import org.elasticsearch.cluster.metadata.IndexGraveyard;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -283,6 +285,36 @@ public class IndicesServiceTests extends ESSingleNodeTestCase {
         indicesService.deleteIndex(test.index(), "finished with test");
     }
 
+    /**
+     * This test checks an edge case where, if a node had an index (lets call it A with UUID 1), then
+     * deleted it (so a tombstone entry for A will exist in the cluster state), then created
+     * a new index A with UUID 2, then shutdown, when the node comes back online, it will look at the
+     * tombstones for deletions, and it should proceed with trying to delete A with UUID 1 and not
+     * throw any errors that the index still exists in the cluster state.  This is a case of ensuring
+     * that tombstones that have the same name as current valid indices don't cause confusion by
+     * trying to delete an index that exists.
+     * See https://github.com/elastic/elasticsearch/issues/18054
+     */
+    public void testIndexAndTombstoneWithSameNameOnStartup() throws Exception {
+        final String indexName = "test";
+        final Index index = new Index(indexName, UUIDs.randomBase64UUID());
+        final IndicesService indicesService = getIndicesService();
+        final Settings idxSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                                         .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID())
+                                         .build();
+        final IndexMetaData indexMetaData = new IndexMetaData.Builder(index.getName())
+                                                .settings(idxSettings)
+                                                .numberOfShards(1)
+                                                .numberOfReplicas(0)
+                                                .build();
+        final Index tombstonedIndex = new Index(indexName, UUIDs.randomBase64UUID());
+        final IndexGraveyard graveyard = IndexGraveyard.builder().addTombstone(tombstonedIndex).build();
+        final MetaData metaData = MetaData.builder().put(indexMetaData, true).indexGraveyard(graveyard).build();
+        final ClusterState clusterState = new ClusterState.Builder(new ClusterName("testCluster")).metaData(metaData).build();
+        // if all goes well, this won't throw an exception, otherwise, it will throw an IllegalStateException
+        indicesService.verifyIndexIsDeleted(tombstonedIndex, clusterState);
+    }
+
     private static class DanglingListener implements LocalAllocateDangledIndices.Listener {
         final CountDownLatch latch = new CountDownLatch(1);
 

From e322903f2c849e52abfed07a45a96de4e8f67e26 Mon Sep 17 00:00:00 2001
From: Robin Joseph <robin.joseph@me.com>
Date: Fri, 29 Apr 2016 09:02:59 -0700
Subject: [PATCH 0026/1311] Fix typo in include-in-all.asciidoc (#18055)

---
 docs/reference/mapping/params/include-in-all.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/mapping/params/include-in-all.asciidoc b/docs/reference/mapping/params/include-in-all.asciidoc
index 76a0d14527c..18c454ae70c 100644
--- a/docs/reference/mapping/params/include-in-all.asciidoc
+++ b/docs/reference/mapping/params/include-in-all.asciidoc
@@ -30,7 +30,7 @@ PUT my_index
 --------------------------------
 // AUTOSENSE
 
-<1> The `title` and `content` fields with be included in the `_all` field.
+<1> The `title` and `content` fields will be included in the `_all` field.
 <2> The `date` field will not be included in the `_all` field.
 
 TIP: The `include_in_all` setting is allowed to have different settings for

From b2ce2f5afab8cb5d7c8125fb2e933fd8c1138d82 Mon Sep 17 00:00:00 2001
From: Josef Salyer <josef.salyer@gmail.com>
Date: Fri, 29 Apr 2016 12:07:23 -0400
Subject: [PATCH 0027/1311] Fixed the documentation formatting. (#17990)

The source code example for the initial example was missing the correct JSON object formatting and syntax.  That has been fixed with my change.
---
 docs/reference/indices/create-index.asciidoc | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/docs/reference/indices/create-index.asciidoc b/docs/reference/indices/create-index.asciidoc
index 11216fa4c01..e225b2f15e8 100644
--- a/docs/reference/indices/create-index.asciidoc
+++ b/docs/reference/indices/create-index.asciidoc
@@ -14,13 +14,14 @@ associated with it.
 
 [source,js]
 --------------------------------------------------
-$ curl -XPUT 'http://localhost:9200/twitter/'
-
-$ curl -XPUT 'http://localhost:9200/twitter/' -d '
-index :
-    number_of_shards : 3 <1>
-    number_of_replicas : 2 <2>
-'
+$ curl -XPUT 'http://localhost:9200/twitter/' -d '{
+    "settings" : {
+        "index" : {
+            "number_of_shards" : 3 <1>
+            "number_of_replicas" : 2 <2>
+        }
+    }
+}'
 --------------------------------------------------
 <1> Default for `number_of_shards` is 5
 <2> Default for `number_of_replicas` is 1 (ie one replica for each primary shard)

From 262a814c8dcf984b96db2b945d9470f5196856bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Fri, 29 Apr 2016 19:29:25 +0200
Subject: [PATCH 0028/1311] Tests: Fix TermsQueryBuilderTests expectations when
 lookup returns no terms

When the termslookup (mocked in this case) doesn't return any terms, the
query used to rewrite to an empty boolean query. Now it rewrites to a
MatchNoDocsQuery. This changes the test expectation accordingly.

Closes #18071
---
 .../index/query/TermsQueryBuilderTests.java          | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
index baf7a1e528c..a16f4e2f2ea 100644
--- a/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
@@ -96,16 +96,22 @@ public class TermsQueryBuilderTests extends AbstractQueryTestCase<TermsQueryBuil
     protected void doAssertLuceneQuery(TermsQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
         if (queryBuilder.termsLookup() == null && (queryBuilder.values() == null || queryBuilder.values().isEmpty())) {
             assertThat(query, instanceOf(MatchNoDocsQuery.class));
+            MatchNoDocsQuery matchNoDocsQuery = (MatchNoDocsQuery) query;
+            assertThat(matchNoDocsQuery.toString(), containsString("No terms supplied for \"terms\" query."));
+        } else if (queryBuilder.termsLookup() != null && randomTerms.size() == 0){
+            assertThat(query, instanceOf(MatchNoDocsQuery.class));
+            MatchNoDocsQuery matchNoDocsQuery = (MatchNoDocsQuery) query;
+            assertThat(matchNoDocsQuery.toString(), containsString("No terms supplied for \"terms\" query."));
         } else {
             assertThat(query, instanceOf(BooleanQuery.class));
             BooleanQuery booleanQuery = (BooleanQuery) query;
-    
+
             // we only do the check below for string fields (otherwise we'd have to decode the values)
             if (queryBuilder.fieldName().equals(INT_FIELD_NAME) || queryBuilder.fieldName().equals(DOUBLE_FIELD_NAME)
                     || queryBuilder.fieldName().equals(BOOLEAN_FIELD_NAME) || queryBuilder.fieldName().equals(DATE_FIELD_NAME)) {
                 return;
             }
-    
+
             // expected returned terms depending on whether we have a terms query or a terms lookup query
             List<Object> terms;
             if (queryBuilder.termsLookup() != null) {
@@ -113,7 +119,7 @@ public class TermsQueryBuilderTests extends AbstractQueryTestCase<TermsQueryBuil
             } else {
                 terms = queryBuilder.values();
             }
-    
+
             // compare whether we have the expected list of terms returned
             final List<Term> booleanTerms = new ArrayList<>();
             for (BooleanClause booleanClause : booleanQuery) {

From c82e5648865d2d732fd4172c6a42b4176b63f93e Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 29 Apr 2016 16:15:45 -0400
Subject: [PATCH 0029/1311] Remove racy but unnecessary assertion

This commit removes a racy but unnecessary assertion in scaling thread
pool idle test. Namely, the main test thread can reach the removed
assertion before the last few threads in the thread pool have completed
their tasks and caused the completed tasks count on the underlying
executor to be updated. But this assertion is unnecessary. The main test
thread already waits on a latch that is only decremented immediately
before a task completes. This ensures that it was in fact the case that
every submitted task was executed.

Closes #18072
---
 .../org/elasticsearch/threadpool/ScalingThreadPoolTests.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java b/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
index d27b43eeb8b..16c0a89e6c4 100644
--- a/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
+++ b/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
@@ -27,6 +27,7 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Executor;
+import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.function.BiConsumer;
 
@@ -193,7 +194,6 @@ public class ScalingThreadPoolTests extends ESThreadPoolTestCase {
             } catch (InterruptedException e) {
                 throw new RuntimeException(e);
             }
-            assertThat(stats(threadPool, threadPoolName).getCompleted(), equalTo(128L));
         }));
     }
 

From 0b415806564585b2605a692d6e978fffcf0052aa Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 29 Apr 2016 16:21:59 -0400
Subject: [PATCH 0030/1311] Remove unused import from ScalingThreadPoolTests

---
 .../org/elasticsearch/threadpool/ScalingThreadPoolTests.java     | 1 -
 1 file changed, 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java b/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
index 16c0a89e6c4..94d6d075589 100644
--- a/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
+++ b/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
@@ -27,7 +27,6 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Executor;
-import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.function.BiConsumer;
 

From 9162fd27a01fd6c82e85fd03f2b610ef32bab4f0 Mon Sep 17 00:00:00 2001
From: Todd Dicken <tdicken73@gmail.com>
Date: Mon, 2 May 2016 06:06:07 -0500
Subject: [PATCH 0031/1311] Change qoutes to quotes (#18078)

Updated misspelling in documentation
---
 docs/reference/setup/install/windows.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/setup/install/windows.asciidoc b/docs/reference/setup/install/windows.asciidoc
index 8fd8a19ce71..3e297a9b487 100644
--- a/docs/reference/setup/install/windows.asciidoc
+++ b/docs/reference/setup/install/windows.asciidoc
@@ -53,7 +53,7 @@ name, as follows:
 ./bin/elasticsearch -E es.cluster.name=my_cluster -E es.node.name=node_1
 --------------------------------------------
 
-NOTE: Values that contain spaces must be surrounded with qoutes.  For instance `-E es.path.logs="C:\My Logs\logs"`.
+NOTE: Values that contain spaces must be surrounded with quotes.  For instance `-E es.path.logs="C:\My Logs\logs"`.
 
 TIP: Typically, any cluster-wide settings (like `cluster.name`) should be
 added to the `elasticsearch.yml` config file, while any node-specific settings

From 7c8397d99b624a77be29995dd0d1534ae16d78ee Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Mon, 2 May 2016 13:47:14 +0200
Subject: [PATCH 0032/1311] Update keyword.asciidoc

`ignore_above` doesn't apply to analyzed `text` fields
---
 docs/reference/mapping/types/keyword.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/mapping/types/keyword.asciidoc b/docs/reference/mapping/types/keyword.asciidoc
index 8de965eea70..48fbf3cd75b 100644
--- a/docs/reference/mapping/types/keyword.asciidoc
+++ b/docs/reference/mapping/types/keyword.asciidoc
@@ -62,7 +62,7 @@ The following parameters are accepted by `keyword` fields:
 
 <<ignore-above,`ignore_above`>>::
 
-    Do not index or analyze any string longer than this value.  Defaults to
+    Do not index any string longer than this value.  Defaults to
     `2147483647` so that all values would be accepted.
 
 <<include-in-all,`include_in_all`>>::

From 4cf5385e4d4df9e652f5951f5ac0266b3467b926 Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Mon, 2 May 2016 13:50:00 +0200
Subject: [PATCH 0033/1311] Use plain random values for generating
 *SortBuilders

---
 .../search/sort/FieldSortBuilderTests.java    | 28 +++++++++----------
 .../sort/GeoDistanceSortBuilderTests.java     |  6 ++--
 .../sort/NestedQueryBuilderGenerator.java     |  7 ++---
 .../search/sort/ScriptSortBuilderTests.java   | 11 ++++----
 4 files changed, 25 insertions(+), 27 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
index abcab363dcc..cab40588777 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
@@ -53,27 +53,23 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
         }
 
         if (randomBoolean()) {
-            builder.missing(ESTestCase.randomValueOtherThan(builder.missing(), () -> randomFrom(missingContent)));
+            builder.missing(randomFrom(missingContent));
         }
 
         if (randomBoolean()) {
-            builder.unmappedType(ESTestCase.randomValueOtherThan(
-                    builder.unmappedType(),
-                    () -> ESTestCase.randomAsciiOfLengthBetween(1, 10)));
+            builder.unmappedType(ESTestCase.randomAsciiOfLengthBetween(1, 10));
         }
 
         if (randomBoolean()) {
-            builder.sortMode(ESTestCase.randomValueOtherThan(builder.sortMode(), () -> randomFrom(SortMode.values())));
+            builder.sortMode(randomFrom(SortMode.values()));
         }
 
         if (randomBoolean()) {
-            builder.setNestedFilter(NestedQueryBuilderGenerator.nestedFilter(builder.getNestedFilter()));
+            builder.setNestedFilter(NestedQueryBuilderGenerator.randomNestedFilter());
         }
 
         if (randomBoolean()) {
-            builder.setNestedPath(ESTestCase.randomValueOtherThan(
-                    builder.getNestedPath(),
-                    () -> ESTestCase.randomAsciiOfLengthBetween(1, 10)));
+            builder.setNestedPath(ESTestCase.randomAsciiOfLengthBetween(1, 10));
         }
 
         return builder;
@@ -86,25 +82,27 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
         switch (parameter) {
         case 0:
             mutated.setNestedPath(ESTestCase.randomValueOtherThan(
-                    mutated.getNestedPath(),
+                    original.getNestedPath(),
                     () -> ESTestCase.randomAsciiOfLengthBetween(1, 10)));
             break;
         case 1:
-            mutated.setNestedFilter(NestedQueryBuilderGenerator.nestedFilter(mutated.getNestedFilter()));
+            mutated.setNestedFilter(ESTestCase.randomValueOtherThan(
+                    original.getNestedFilter(),
+                    () -> NestedQueryBuilderGenerator.randomNestedFilter()));
             break;
         case 2:
-            mutated.sortMode(ESTestCase.randomValueOtherThan(mutated.sortMode(), () -> randomFrom(SortMode.values())));
+            mutated.sortMode(ESTestCase.randomValueOtherThan(original.sortMode(), () -> randomFrom(SortMode.values())));
             break;
         case 3:
             mutated.unmappedType(ESTestCase.randomValueOtherThan(
-                    mutated.unmappedType(),
+                    original.unmappedType(),
                     () -> ESTestCase.randomAsciiOfLengthBetween(1, 10)));
             break;
         case 4:
-            mutated.missing(ESTestCase.randomValueOtherThan(mutated.missing(), () -> randomFrom(missingContent)));
+            mutated.missing(ESTestCase.randomValueOtherThan(original.missing(), () -> randomFrom(missingContent)));
             break;
         case 5:
-            mutated.order(ESTestCase.randomValueOtherThan(mutated.order(), () -> randomFrom(SortOrder.values())));
+            mutated.order(ESTestCase.randomValueOtherThan(original.order(), () -> randomFrom(SortOrder.values())));
             break;
         default:
             throw new IllegalStateException("Unsupported mutation.");
diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
index 9f1b1a325ef..3da59f79de6 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
@@ -89,7 +89,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
             result.sortMode(mode(result.sortMode()));
         }
         if (randomBoolean()) {
-            result.setNestedFilter(NestedQueryBuilderGenerator.nestedFilter(result.getNestedFilter()));
+            result.setNestedFilter(NestedQueryBuilderGenerator.randomNestedFilter());
         }
         if (randomBoolean()) {
             result.setNestedPath(
@@ -177,7 +177,9 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
             result.sortMode(mode(original.sortMode()));
             break;
         case 6:
-            result.setNestedFilter(NestedQueryBuilderGenerator.nestedFilter(original.getNestedFilter()));
+            result.setNestedFilter(ESTestCase.randomValueOtherThan(
+                    original.getNestedFilter(),
+                    () -> NestedQueryBuilderGenerator.randomNestedFilter()));
             break;
         case 7:
             result.setNestedPath(ESTestCase.randomValueOtherThan(
diff --git a/core/src/test/java/org/elasticsearch/search/sort/NestedQueryBuilderGenerator.java b/core/src/test/java/org/elasticsearch/search/sort/NestedQueryBuilderGenerator.java
index 7e6f88cffd5..de3318036e8 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/NestedQueryBuilderGenerator.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/NestedQueryBuilderGenerator.java
@@ -38,10 +38,7 @@ public class NestedQueryBuilderGenerator {
             new IdsQueryBuilder(),
             new TermQueryBuilder(ESTestCase.randomAsciiOfLengthBetween(1, 10), ESTestCase.randomDouble()));
 
-    public static QueryBuilder<?> nestedFilter(QueryBuilder<?> original) {
-        @SuppressWarnings("rawtypes")
-        QueryBuilder nested = ESTestCase.randomValueOtherThan(original, () -> ESTestCase.randomFrom(builders));
-        nested.boost((float) ESTestCase.randomDoubleBetween(0, 10, false));
-        return nested;
+    public static QueryBuilder<?> randomNestedFilter() {
+        return ESTestCase.randomFrom(builders).boost(ESTestCase.randomFloat());
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
index ba9d608f610..1d7247ba1e4 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService.ScriptType;
@@ -63,12 +64,10 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild
             }
         }
         if (randomBoolean()) {
-            builder.setNestedFilter(NestedQueryBuilderGenerator.nestedFilter(builder.getNestedFilter()));
+            builder.setNestedFilter(NestedQueryBuilderGenerator.randomNestedFilter());
         }
         if (randomBoolean()) {
-            builder.setNestedPath(ESTestCase.randomValueOtherThan(
-                    builder.getNestedPath(),
-                    () -> ESTestCase.randomAsciiOfLengthBetween(1, 10)));
+            builder.setNestedPath(ESTestCase.randomAsciiOfLengthBetween(1, 10));
         }
         return builder;
     }
@@ -115,7 +114,9 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild
                 }
                 break;
             case 2:
-                result.setNestedFilter(NestedQueryBuilderGenerator.nestedFilter(original.getNestedFilter()));
+                result.setNestedFilter(ESTestCase.randomValueOtherThan(
+                        original.getNestedFilter(),
+                        () -> NestedQueryBuilderGenerator.randomNestedFilter()));
                 break;
             case 3:
                 result.setNestedPath(original.getNestedPath() + "_some_suffix");

From 9fee8c76af39e471926a30685722cba7533f7a73 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Mon, 2 May 2016 14:21:59 +0200
Subject: [PATCH 0034/1311] Added release notes for 5.0.0-alpha2

---
 docs/reference/release-notes.asciidoc         |   2 +
 .../release-notes/5.0.0-alpha2.asciidoc       | 255 ++++++++++++++++++
 2 files changed, 257 insertions(+)
 create mode 100644 docs/reference/release-notes/5.0.0-alpha2.asciidoc

diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc
index f9391cece06..76fe55d3d34 100644
--- a/docs/reference/release-notes.asciidoc
+++ b/docs/reference/release-notes.asciidoc
@@ -5,9 +5,11 @@
 --
 This section summarizes the changes in each release.
 
+* <<release-notes-5.0.0-alpha2>>
 * <<release-notes-5.0.0-alpha1>>
 * <<release-notes-5.0.0-alpha1-2x>>
 
 --
+include::release-notes/5.0.0-alpha2.asciidoc[]
 include::release-notes/5.0.0-alpha1.asciidoc[]
 include::release-notes/5.0.0-alpha1-2x.asciidoc[]
diff --git a/docs/reference/release-notes/5.0.0-alpha2.asciidoc b/docs/reference/release-notes/5.0.0-alpha2.asciidoc
new file mode 100644
index 00000000000..ad6c62653f0
--- /dev/null
+++ b/docs/reference/release-notes/5.0.0-alpha2.asciidoc
@@ -0,0 +1,255 @@
+[[release-notes-5.0.0-alpha2]]
+== 5.0.0-alpha2 Release Notes
+
+Also see <<breaking-changes-5.0>>.
+
+[[breaking-5.0.0-alpha2]]
+[float]
+=== Breaking changes
+
+Analysis::
+* Analyze API : Rename filters/token_filters/char_filter in Analyze API in master {pull}17843[#17843] (issue: {issue}15189[#15189])
+
+Cluster::
+* Remove validation errors from cluster health response {pull}17773[#17773] (issue: {issue}16979[#16979])
+
+Indexed Scripts/Templates::
+* Store indexed scripts in the cluster state instead of the `.scripts` index {pull}17650[#17650] (issue: {issue}16651[#16651])
+
+Packaging::
+* Add JVM options configuration file {pull}17675[#17675] (issue: {issue}17121[#17121])
+
+Percolator::
+* Remove `.percolator` type in favour of `percolator` field type {pull}17560[#17560]
+
+REST::
+* Remove camelCase support {pull}17933[#17933] (issue: {issue}8988[#8988])
+* Remove 'case' parameter from rest apis {pull}17774[#17774] (issue: {issue}8988[#8988])
+* Disallow unquoted field names {pull}15351[#15351] (issue: {issue}9800[#9800])
+
+Settings::
+* Remove `action.get.realtime` setting {pull}17857[#17857] (issue: {issue}12543[#12543])
+* Remove Settings.settingsBuilder. {pull}17619[#17619]
+
+
+
+[[deprecation-5.0.0-alpha2]]
+[float]
+=== Deprecations
+
+Query DSL::
+* Deprecate Indices query {pull}17710[#17710] (issue: {issue}12017[#12017])
+* Deprecate mlt, in and geo_bbox query name shortcuts {pull}17507[#17507]
+
+Query Refactoring::
+* Splits `phrase` and `phrase_prefix` in match query into `MatchPhraseQueryBuilder` and `MatchPhrasePrefixQueryBuilder` {pull}17508[#17508]
+
+
+
+[[feature-5.0.0-alpha2]]
+[float]
+=== New features
+
+Analysis::
+* Add `fingerprint` token filter and `fingerprint` analyzer {pull}17873[#17873] (issue: {issue}13325[#13325])
+
+Plugin Analysis ICU::
+* Adding support for customizing the rule file in ICU tokenizer {pull}13651[#13651] (issue: {issue}13146[#13146])
+
+
+
+[[enhancement-5.0.0-alpha2]]
+[float]
+=== Enhancements
+
+CAT API::
+* Add _cat/tasks {pull}17551[#17551]
+* Cat health supports ts=0 option {pull}13508[#13508] (issue: {issue}10109[#10109])
+
+Cache::
+* Allow the query cache to be disabled. {pull}16268[#16268] (issue: {issue}15802[#15802])
+
+Cluster::
+* Adds tombstones to cluster state for index deletions {pull}17265[#17265] (issues: {issue}16358[#16358], {issue}17435[#17435])
+* Enable acked indexing {pull}17038[#17038] (issue: {issue}7572[#7572])
+
+Core::
+* Kill thread local leak {pull}17921[#17921] (issues: {issue}283[#283], {issue}630[#630])
+* Add heap size bootstrap check {pull}17728[#17728] (issue: {issue}17490[#17490])
+* Remove hostname from NetworkAddress.format {pull}17601[#17601] (issue: {issue}17604[#17604])
+* Bootstrapping bootstrap checks {pull}17595[#17595] (issues: {issue}17474[#17474], {issue}17570[#17570])
+* Add max map count check {pull}16944[#16944]
+
+Geo::
+* Enhanced lat/long error handling {pull}16833[#16833] (issue: {issue}16137[#16137])
+
+Index APIs::
+* Fail hot_threads in a better way if unsupported by JDK {pull}15909[#15909]
+
+Ingest::
+* Streamline option naming for several processors {pull}17892[#17892] (issue: {issue}17835[#17835])
+
+Internal::
+* Makes Script type writeable {pull}17908[#17908] (issue: {issue}17753[#17753])
+* FiltersAggregatorBuilder: Don't create new context for inner parsing {pull}17851[#17851]
+* Clean up serialization on some stats {pull}17832[#17832] (issue: {issue}17085[#17085])
+* 	Normalize registration for SignificanceHeuristics {pull}17830[#17830] (issue: {issue}17085[#17085])
+* Make (read|write)NamedWriteable public {pull}17829[#17829] (issue: {issue}17682[#17682])
+* Use try-with-resource when creating new parser instances where possible {pull}17822[#17822]
+* Don't pass XContentParser to ParseFieldRegistry#lookup {pull}17794[#17794]
+* Internal: Remove threadlocal from document parser {pull}17764[#17764]
+* Cut range aggregations to registerAggregation {pull}17757[#17757] (issue: {issue}17085[#17085])
+* Remove ParseFieldMatcher from AbstractXContentParser {pull}17756[#17756] (issue: {issue}17417[#17417])
+* Remove parser argument from methods where we already pass in a parse context {pull}17738[#17738]
+* Switch SearchAfterBuilder to writeGenericValue {pull}17735[#17735] (issue: {issue}17085[#17085])
+* Remove StreamableReader {pull}17729[#17729] (issue: {issue}17085[#17085])
+* Cleanup nested, has_child & has_parent query builders for inner hits construction {pull}17719[#17719] (issue: {issue}11118[#11118])
+* Make AllocationCommands NamedWriteables {pull}17661[#17661]
+* Isolate StreamableReader {pull}17656[#17656] (issue: {issue}17085[#17085])
+* Create registration methods for aggregations similar to those for queries {pull}17653[#17653] (issues: {issue}17085[#17085], {issue}17389[#17389])
+* Turn RestChannel into an interface {pull}17643[#17643] (issue: {issue}17133[#17133])
+* Remove PROTOTYPEs from QueryBuilders {pull}17632[#17632] (issue: {issue}17085[#17085])
+* Remove registerQueryParser {pull}17608[#17608]
+* ParseField#getAllNamesIncludedDeprecated to not return duplicate names {pull}17504[#17504]
+* Rework a query parser and improve registration {pull}17458[#17458]
+* Clean up QueryParseContext and don't hold it inside QueryRewrite/ShardContext {pull}17417[#17417]
+
+Mapping::
+* Automatically upgrade analyzed strings with an analyzer to `text`. {pull}17861[#17861]
+* Support dots in field names when mapping already exists {pull}17759[#17759] (issue: {issue}15951[#15951])
+* Use the new points API to index numeric fields. {pull}17746[#17746] (issues: {issue}11513[#11513], {issue}16751[#16751], {issue}17007[#17007], {issue}17700[#17700])
+* Simplify AllEntries, AllField and AllFieldMapper: {pull}17613[#17613]
+
+Network::
+* Limit request size {pull}17133[#17133] (issue: {issue}16011[#16011])
+
+Packaging::
+* Remove unnecessary sleep from init script restart {pull}17966[#17966]
+* Explicitly set packaging permissions {pull}17912[#17912] (issue: {issue}17634[#17634])
+* Allow configuring Windows service name, description and user {pull}17312[#17312]
+* rpm uses non-portable `--system` flag to `useradd` {pull}14596[#14596] (issue: {issue}14211[#14211])
+
+Percolator::
+* PercolatorQueryBuilder cleanup by using MemoryIndex#fromDocument(...) helper {pull}17669[#17669] (issue: {issue}9386[#9386])
+
+Plugins::
+* Cli: Improve output for usage errors {pull}17938[#17938]
+* Cli: Add verbose output with zip url when installing plugin {pull}17662[#17662] (issue: {issue}17529[#17529])
+
+Query DSL::
+* Add MatchNoDocsQuery, a query that matches no documents and prints the reason why in the toString method. {pull}17780[#17780]
+* Adds `ignore_unmapped` option to geo queries {pull}17751[#17751]
+* Adds `ignore_unmapped` option to nested and P/C queries {pull}17748[#17748]
+* SimpleQueryParser should call MappedFieldType.termQuery when appropriate. {pull}17678[#17678]
+
+REST::
+* Allow JSON with unquoted field names by enabling system property {pull}17801[#17801] (issue: {issue}17674[#17674])
+
+Recovery::
+* TransportNodesListGatewayStartedShards should fall back to disk based index metadata if not found in cluster state {pull}17663[#17663] (issue: {issue}17630[#17630])
+
+Reindex API::
+* Properly mark reindex's child tasks as child tasks {pull}17770[#17770]
+
+Search::
+* Fail query if it contains very large rescores {pull}17917[#17917] (issue: {issue}17522[#17522])
+
+Settings::
+* Switch to registered Settings for all IndexingMemoryController settings {pull}17778[#17778] (issue: {issue}17442[#17442])
+
+Stats::
+* Add points to SegmentStats. {pull}17775[#17775] (issue: {issue}16974[#16974])
+* Remove FieldStats.Float. {pull}17749[#17749]
+* Show configured and remaining delay for an unassigned shard. {pull}17515[#17515] (issue: {issue}17372[#17372])
+
+Store::
+* Use `mmapfs` by default. {pull}17616[#17616] (issue: {issue}16983[#16983])
+
+Suggesters::
+* Add bwc support for reading  pre-5.0 completion index {pull}17602[#17602]
+
+Task Manager::
+* Move parentTaskId into TransportRequest  {pull}17872[#17872]
+* Shorten the serialization of the empty TaskId {pull}17870[#17870]
+* Expose whether a task is cancellable in the _tasks list API {pull}17464[#17464] (issue: {issue}17369[#17369])
+
+
+
+[[bug-5.0.0-alpha2]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Adds serialisation of sigma to extended_stats_bucket pipeline aggregation {pull}17703[#17703] (issue: {issue}17701[#17701])
+* Fixes NPE when no window is specified in moving average request {pull}17556[#17556] (issue: {issue}17516[#17516])
+* Fixes Filter and FiltersAggregation to work with empty query {pull}17542[#17542] (issue: {issue}17518[#17518])
+* ExtendedStatsAggregator should also pass sigma to emtpy aggs. {pull}17388[#17388] (issue: {issue}17362[#17362])
+
+Allocation::
+* Rebalancing policy shouldn't prevent hard allocation decisions {pull}17698[#17698] (issues: {issue}14057[#14057], {issue}14259[#14259])
+* When considering the size of shadow replica shards, set size to 0 {pull}17509[#17509] (issue: {issue}17460[#17460])
+
+Core::
+* Refactor UUID-generating methods out of Strings {pull}17837[#17837] (issue: {issue}17819[#17819])
+* Node names cleanup {pull}17723[#17723] (issue: {issue}17718[#17718])
+* NullPointerException from IndexingMemoryController when a version conflict happens during recovery {pull}17569[#17569]
+
+Ingest::
+* Ingest does not close its factories {pull}17626[#17626] (issue: {issue}17625[#17625])
+
+Internal::
+* Fix BulkItemResponse.Failure.toString {pull}17871[#17871]
+
+Logging::
+* Add missing index name to search slow log. {pull}17818[#17818] (issue: {issue}17025[#17025])
+
+Mapping::
+* Fix cross type mapping updates for `boolean` fields. {pull}17882[#17882] (issue: {issue}17879[#17879])
+* Fix dynamic check to properly handle parents {pull}17864[#17864] (issues: {issue}17644[#17644], {issue}17854[#17854])
+* Fix array parsing to remove its context when finished parsing {pull}17768[#17768]
+* Disallow fielddata loading on text fields that are not indexed. {pull}17747[#17747]
+* Fail if an object is added after a field with the same name. {pull}17568[#17568] (issue: {issue}17567[#17567])
+
+Packaging::
+* Fix exit code {pull}17082[#17082]
+
+Plugin Discovery EC2::
+* Fix EC2 Discovery settings {pull}17651[#17651] (issue: {issue}16602[#16602])
+
+Plugins::
+* Quote path to java binary {pull}17496[#17496] (issue: {issue}17495[#17495])
+
+Query DSL::
+* Apply the default operator on analyzed wildcard in simple_query_string builder {pull}17776[#17776]
+* Apply the default operator on analyzed wildcard in query_string builder: {pull}17711[#17711] (issue: {issue}2183[#2183])
+
+REST::
+* Fixes reading of CORS pre-flight headers and methods {pull}17523[#17523] (issue: {issue}17483[#17483])
+* index is a required url part for update by query {pull}17503[#17503]
+
+Reindex API::
+* Reindex should never report negative throttled_until {pull}17799[#17799] (issue: {issue}17783[#17783])
+* Reindex should gracefully handle when _source is disabled {pull}17667[#17667] (issue: {issue}17666[#17666])
+
+Settings::
+* convert settings for ResourceWatcherService to new infrastructure {pull}17948[#17948]
+
+Snapshot/Restore::
+* Fix the semantics for the BlobContainer interface {pull}17878[#17878] (issues: {issue}15579[#15579], {issue}15580[#15580])
+* On restore, selecting concrete indices can select wrong index {pull}17715[#17715]
+
+Task Manager::
+* Shard level tasks in Bulk Action lose reference to their parent tasks {pull}17743[#17743]
+
+Term Vectors::
+* Fix calculation of took time of term vectors request {pull}17817[#17817] (issue: {issue}12565[#12565])
+
+
+
+[[upgrade-5.0.0-alpha2]]
+[float]
+=== Upgrades
+
+Core::
+* Upgrade to lucene 6 release {pull}17657[#17657]
+

From 28409e45090a74a4d480dfcd34e399f7f39ed06e Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 2 May 2016 09:07:25 -0400
Subject: [PATCH 0035/1311] Add support for .empty to expressions, and some
 docs improvements

Closes #18077
---
 .../modules/scripting/scripting.asciidoc      | 19 ++++-
 .../expression/CountMethodValueSource.java    |  2 +-
 .../expression/EmptyMemberValueSource.java    | 83 +++++++++++++++++++
 .../ExpressionScriptEngineService.java        | 26 +++++-
 .../expression/MoreExpressionTests.java       | 26 +++++-
 5 files changed, 145 insertions(+), 11 deletions(-)
 create mode 100644 modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java

diff --git a/docs/reference/modules/scripting/scripting.asciidoc b/docs/reference/modules/scripting/scripting.asciidoc
index 047ad0aa136..5935554a368 100644
--- a/docs/reference/modules/scripting/scripting.asciidoc
+++ b/docs/reference/modules/scripting/scripting.asciidoc
@@ -455,11 +455,25 @@ for details on what operators and functions are available.
 
 Variables in `expression` scripts are available to access:
 
-* Single valued document fields, e.g. `doc['myfield'].value`
-* Single valued document fields can also be accessed without `.value` e.g. `doc['myfield']`
+* document fields, e.g. `doc['myfield'].value` or just `doc['myfield']`.
+* whether the field is empty, e.g. `doc['myfield'].empty`
 * Parameters passed into the script, e.g. `mymodifier`
 * The current document's score, `_score` (only available when used in a `script_score`)
 
+When a document is missing the field completely, by default the value will be treated as `0`.
+You can treat it as another value instead, e.g. `doc['myfield'].empty ? 100 : doc['myfield'].value`
+
+When a document has multiple values for the field, by default the minimum value is returned.
+You can choose a different value instead, e.g. `doc['myfield'].sum()`. The following methods are available
+for any field:
+
+* min()
+* max()
+* avg()
+* median()
+* sum()
+* count() 
+
 Variables in `expression` scripts that are of type `date` may use the following member methods:
 
 * getYear()
@@ -477,7 +491,6 @@ There are a few limitations relative to other script languages:
 
 * Only numeric fields may be accessed
 * Stored fields are not available
-* If a field is sparse (only some documents contain a value), documents missing the field will have a value of `0`
 
 [float]
 === Score
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java
index 043a11eebad..6f397c02bd3 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java
@@ -63,7 +63,7 @@ public class CountMethodValueSource extends ValueSource {
 
     @Override
     public int hashCode() {
-        return fieldData.hashCode();
+        return 31 * getClass().hashCode() + fieldData.hashCode();
     }
 
     @Override
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java
new file mode 100644
index 00000000000..b8c101e8abc
--- /dev/null
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script.expression;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
+
+/**
+ * ValueSource to return non-zero if a field is missing.
+ * <p>
+ * This is essentially sugar over !count()
+ */
+public class EmptyMemberValueSource extends ValueSource {
+    protected IndexFieldData<?> fieldData;
+
+    protected EmptyMemberValueSource(IndexFieldData<?> fieldData) {
+        this.fieldData = Objects.requireNonNull(fieldData);
+    }
+
+    @Override
+    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
+    public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
+        AtomicNumericFieldData leafData = (AtomicNumericFieldData) fieldData.load(leaf);
+        final SortedNumericDoubleValues values = leafData.getDoubleValues();
+        return new DoubleDocValues(this) {
+            @Override
+            public double doubleVal(int doc) {
+                values.setDocument(doc);
+                if (values.count() == 0) {
+                    return 1;
+                } else {
+                    return 0;
+                }
+            }
+        };
+    }
+
+    @Override
+    public int hashCode() {
+        return 31 * getClass().hashCode() + fieldData.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null) return false;
+        if (getClass() != obj.getClass()) return false;
+        EmptyMemberValueSource other = (EmptyMemberValueSource) obj;
+        if (!fieldData.equals(other.fieldData)) return false;
+        return true;
+    }
+
+    @Override
+    public String description() {
+        return "empty: field(" + fieldData.getFieldName() + ")";
+    }
+}
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
index d78f80bfe49..2bfd2928d57 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
@@ -65,6 +65,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
 
     public static final List<String> TYPES = Collections.singletonList(NAME);
 
+    // these methods only work on dates, e.g. doc['datefield'].getYear()
     protected static final String GET_YEAR_METHOD         = "getYear";
     protected static final String GET_MONTH_METHOD        = "getMonth";
     protected static final String GET_DAY_OF_MONTH_METHOD = "getDayOfMonth";
@@ -72,6 +73,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
     protected static final String GET_MINUTES_METHOD      = "getMinutes";
     protected static final String GET_SECONDS_METHOD      = "getSeconds";
 
+    // these methods work on any field, e.g. doc['field'].sum()
     protected static final String MINIMUM_METHOD          = "min";
     protected static final String MAXIMUM_METHOD          = "max";
     protected static final String AVERAGE_METHOD          = "avg";
@@ -79,6 +81,10 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
     protected static final String SUM_METHOD              = "sum";
     protected static final String COUNT_METHOD            = "count";
 
+    // these variables work on any field, e.g. doc['field'].value
+    protected static final String VALUE_VARIABLE          = "value";
+    protected static final String EMPTY_VARIABLE          = "empty";
+
     @Inject
     public ExpressionScriptEngineService(Settings settings) {
         super(settings);
@@ -169,6 +175,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                 } else {
                     String fieldname = null;
                     String methodname = null;
+                    String variablename = VALUE_VARIABLE; // .value is the default for doc['field'], its optional.
                     VariableContext[] parts = VariableContext.parse(variable);
                     if (parts[0].text.equals("doc") == false) {
                         throw new ScriptException("Unknown variable [" + parts[0].text + "] in expression");
@@ -181,8 +188,10 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                     if (parts.length == 3) {
                         if (parts[2].type == VariableContext.Type.METHOD) {
                             methodname = parts[2].text;
-                        } else if (parts[2].type != VariableContext.Type.MEMBER || !"value".equals(parts[2].text)) {
-                            throw new ScriptException("Only the member variable [value] or member methods may be accessed on a field when not accessing the field directly");
+                        } else if (parts[2].type == VariableContext.Type.MEMBER) {
+                            variablename = parts[2].text;
+                        } else {
+                            throw new ScriptException("Only member variables or member methods may be accessed on a field when not accessing the field directly");
                         }
                     }
                     if (parts.length > 3) {
@@ -201,7 +210,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                         throw new ScriptException("Field [" + fieldname + "] used in expression must be numeric");
                     }
                     if (methodname == null) {
-                        bindings.add(variable, new FieldDataValueSource(fieldData, MultiValueMode.MIN));
+                        bindings.add(variable, getVariableValueSource(fieldType, fieldData, fieldname, variablename));
                     } else {
                         bindings.add(variable, getMethodValueSource(fieldType, fieldData, fieldname, methodname));
                     }
@@ -245,6 +254,17 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                 throw new IllegalArgumentException("Member method [" + methodName + "] does not exist.");
         }
     }
+    
+    protected ValueSource getVariableValueSource(MappedFieldType fieldType, IndexFieldData<?> fieldData, String fieldName, String memberName) {
+        switch (memberName) {
+            case VALUE_VARIABLE:
+                return new FieldDataValueSource(fieldData, MultiValueMode.MIN);
+            case EMPTY_VARIABLE:
+                return new EmptyMemberValueSource(fieldData);
+            default:
+                throw new IllegalArgumentException("Member variable [" + memberName + "] does not exist.");
+        }
+    }
 
     protected ValueSource getDateMethodValueSource(MappedFieldType fieldType, IndexFieldData<?> fieldData, String fieldName, String methodName, int calendarType) {
         if (fieldType instanceof LegacyDateFieldMapper.DateFieldType == false
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
index b53245fda20..50a9900e3d9 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
@@ -164,10 +164,10 @@ public class MoreExpressionTests extends ESIntegTestCase {
     }
 
     public void testMultiValueMethods() throws Exception {
-        ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "double0", "type=double", "double1", "type=double"));
+        ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "double0", "type=double", "double1", "type=double", "double2", "type=double"));
         ensureGreen("test");
         indexRandom(true,
-                client().prepareIndex("test", "doc", "1").setSource("double0", "5.0", "double0", "1.0", "double0", "1.5", "double1", "1.2", "double1", "2.4"),
+                client().prepareIndex("test", "doc", "1").setSource("double0", "5.0", "double0", "1.0", "double0", "1.5", "double1", "1.2", "double1", "2.4", "double2", "3.0"),
                 client().prepareIndex("test", "doc", "2").setSource("double0", "5.0", "double1", "3.0"),
                 client().prepareIndex("test", "doc", "3").setSource("double0", "5.0", "double0", "1.0", "double0", "1.5", "double0", "-1.5", "double1", "4.0"));
 
@@ -227,6 +227,24 @@ public class MoreExpressionTests extends ESIntegTestCase {
         assertEquals(2.5, hits.getAt(0).field("foo").getValue(), 0.0D);
         assertEquals(5.0, hits.getAt(1).field("foo").getValue(), 0.0D);
         assertEquals(1.5, hits.getAt(2).field("foo").getValue(), 0.0D);
+        
+        // make sure count() works for missing
+        rsp = buildRequest("doc['double2'].count()").get();
+        assertSearchResponse(rsp);
+        hits = rsp.getHits();
+        assertEquals(3, hits.getTotalHits());
+        assertEquals(1.0, hits.getAt(0).field("foo").getValue(), 0.0D);
+        assertEquals(0.0, hits.getAt(1).field("foo").getValue(), 0.0D);
+        assertEquals(0.0, hits.getAt(2).field("foo").getValue(), 0.0D);
+        
+        // make sure .empty works in the same way
+        rsp = buildRequest("doc['double2'].empty ? 5.0 : 2.0").get();
+        assertSearchResponse(rsp);
+        hits = rsp.getHits();
+        assertEquals(3, hits.getTotalHits());
+        assertEquals(2.0, hits.getAt(0).field("foo").getValue(), 0.0D);
+        assertEquals(5.0, hits.getAt(1).field("foo").getValue(), 0.0D);
+        assertEquals(5.0, hits.getAt(2).field("foo").getValue(), 0.0D);
     }
 
     public void testInvalidDateMethodCall() throws Exception {
@@ -363,8 +381,8 @@ public class MoreExpressionTests extends ESIntegTestCase {
         } catch (SearchPhaseExecutionException e) {
             assertThat(e.toString() + "should have contained ScriptException",
                     e.toString().contains("ScriptException"), equalTo(true));
-            assertThat(e.toString() + "should have contained member variable [value] or member methods may be accessed",
-                    e.toString().contains("member variable [value] or member methods may be accessed"), equalTo(true));
+            assertThat(e.toString() + "should have contained member variable [bogus] does not exist",
+                    e.toString().contains("Member variable [bogus] does not exist"), equalTo(true));
         }
     }
 

From 372eceb8542d6928db8193649716ca9a6a3ac501 Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Mon, 2 May 2016 15:41:05 +0200
Subject: [PATCH 0036/1311] Switch to using predicate for testing existing
 value

---
 .../search/sort/FieldSortBuilderTests.java        |  1 +
 .../search/sort/ScriptSortBuilderTests.java       |  2 +-
 .../java/org/elasticsearch/test/ESTestCase.java   | 15 ++++++++-------
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
index cab40588777..ebd5403489b 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
@@ -30,6 +30,7 @@ import org.elasticsearch.test.ESTestCase;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
+import java.util.function.Supplier;
 
 public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder> {
 
diff --git a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
index 1d7247ba1e4..b809adcb27b 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
@@ -60,7 +60,7 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild
                 exceptThis.add(SortMode.SUM);
                 exceptThis.add(SortMode.AVG);
                 exceptThis.add(SortMode.MEDIAN);
-                builder.sortMode(ESTestCase.randomValueOtherThanMany(exceptThis, () -> randomFrom(SortMode.values())));
+                builder.sortMode(ESTestCase.randomValueOtherThanMany(exceptThis::contains, () -> randomFrom(SortMode.values())));
             }
         }
         if (randomBoolean()) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index d8a3221c88f..01f0d5b8151 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -87,6 +87,7 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.function.BooleanSupplier;
 import java.util.function.Consumer;
+import java.util.function.Predicate;
 import java.util.function.Supplier;
 
 import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList;
@@ -410,21 +411,21 @@ public abstract class ESTestCase extends LuceneTestCase {
      * helper to get a random value in a certain range that's different from the input
      */
     public static <T> T randomValueOtherThan(T input, Supplier<T> randomSupplier) {
-        T randomValue = null;
-        do {
-            randomValue = randomSupplier.get();
-        } while (randomValue.equals(input));
-        return randomValue;
+        if (input != null) {
+            return randomValueOtherThanMany(input::equals, randomSupplier);
+        }
+        
+        return(randomSupplier.get());
     }
 
     /**
      * helper to get a random value in a certain range that's different from the input
      */
-    public static <T> T randomValueOtherThanMany(Collection<T> input, Supplier<T> randomSupplier) {
+    public static <T> T randomValueOtherThanMany(Predicate<T> input, Supplier<T> randomSupplier) {
         T randomValue = null;
         do {
             randomValue = randomSupplier.get();
-        } while (input.contains(randomValue));
+        } while (input.test(randomValue));
         return randomValue;
     }
 

From 8e178c4f8e0321929b4147c1c6e1d22e488bea5d Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 2 May 2016 10:59:50 -0400
Subject: [PATCH 0037/1311] Add system bootstrap checks escape hatch

Today when running in production mode the bootstrap checks are
completely unforgiving. But there are cases where an end-user might not
have the ability to modify some of the system-level settings that cause
the bootstrap checks to trip (e.g., guest settings that are inherited
from a host and can not be modified). This commit adds a setting that
allows system-level bootstrap checks to be ignored for these
end-users. We classify certain bootstrap checks into system-level checks
and only those bootstrap checks will be ignored if this flag is
enabled. All other bootstrap checks are still subject to being enforced
if the user is in production mode. We will still log warnings for these
bootstrap checks because the end-user does still need to be made aware
that they are running in a configuration that is less-than-ideal from a
resiliency perspective.

Relates #18088
---
 .../bootstrap/BootstrapCheck.java             | 121 +++++++++++++++---
 .../bootstrap/BootstrapSettings.java          |   2 +
 .../common/settings/ClusterSettings.java      |   1 +
 .../bootstrap/BootstrapCheckTests.java        |  90 ++++++++++---
 4 files changed, 175 insertions(+), 39 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
index 2f87086ede4..25bdbe3fa8a 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
@@ -41,7 +41,6 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
-import java.util.stream.Collectors;
 
 /**
  * We enforce limits once any network host is configured. In this case we assume the node is running in production
@@ -63,42 +62,80 @@ final class BootstrapCheck {
      * @param boundTransportAddress the node network bindings
      */
     static void check(final Settings settings, final BoundTransportAddress boundTransportAddress) {
-        check(enforceLimits(boundTransportAddress), checks(settings), Node.NODE_NAME_SETTING.get(settings));
+        check(
+                enforceLimits(boundTransportAddress),
+                BootstrapSettings.IGNORE_SYSTEM_BOOTSTRAP_CHECKS.get(settings),
+                checks(settings),
+                Node.NODE_NAME_SETTING.get(settings));
     }
 
     /**
      * executes the provided checks and fails the node if
      * enforceLimits is true, otherwise logs warnings
      *
-     * @param enforceLimits true if the checks should be enforced or
-     *                      warned
-     * @param checks        the checks to execute
-     * @param nodeName      the node name to be used as a logging prefix
+     * @param enforceLimits      true if the checks should be enforced or
+     *                           otherwise warned
+     * @param ignoreSystemChecks true if system checks should be enforced
+     *                           or otherwise warned
+     * @param checks             the checks to execute
+     * @param nodeName           the node name to be used as a logging prefix
      */
     // visible for testing
-    static void check(final boolean enforceLimits, final List<Check> checks, final String nodeName) {
-        final ESLogger logger = Loggers.getLogger(BootstrapCheck.class, nodeName);
+    static void check(final boolean enforceLimits, final boolean ignoreSystemChecks, final List<Check> checks, final String nodeName) {
+        check(enforceLimits, ignoreSystemChecks, checks, Loggers.getLogger(BootstrapCheck.class, nodeName));
+    }
 
-        final List<String> errors =
-                checks.stream()
-                        .filter(BootstrapCheck.Check::check)
-                        .map(BootstrapCheck.Check::errorMessage)
-                        .collect(Collectors.toList());
+    /**
+     * executes the provided checks and fails the node if
+     * enforceLimits is true, otherwise logs warnings
+     *
+     * @param enforceLimits      true if the checks should be enforced or
+     *                           otherwise warned
+     * @param ignoreSystemChecks true if system checks should be enforced
+     *                           or otherwise warned
+     * @param checks             the checks to execute
+     * @param logger             the logger to
+     */
+    static void check(
+            final boolean enforceLimits,
+            final boolean ignoreSystemChecks,
+            final List<Check> checks,
+            final ESLogger logger) {
+        final List<String> errors = new ArrayList<>();
+        final List<String> ignoredErrors = new ArrayList<>();
 
-        if (!errors.isEmpty()) {
-            final List<String> messages = new ArrayList<>(1 + errors.size());
-            messages.add("bootstrap checks failed");
-            messages.addAll(errors);
-            if (enforceLimits) {
+        for (final Check check : checks) {
+            if (check.check()) {
+                if (!enforceLimits || (check.isSystemCheck() && ignoreSystemChecks)) {
+                    ignoredErrors.add(check.errorMessage());
+                } else {
+                    errors.add(check.errorMessage());
+                }
+            }
+        }
+
+        if (!errors.isEmpty() || !ignoredErrors.isEmpty()) {
+
+            if (!ignoredErrors.isEmpty()) {
+                ignoredErrors.forEach(error -> log(logger, error));
+            }
+
+            if (!errors.isEmpty()) {
+                final List<String> messages = new ArrayList<>(1 + errors.size());
+                messages.add("bootstrap checks failed");
+                messages.addAll(errors);
                 final RuntimeException re = new RuntimeException(String.join("\n", messages));
                 errors.stream().map(IllegalStateException::new).forEach(re::addSuppressed);
                 throw re;
-            } else {
-                messages.forEach(message -> logger.warn(message));
             }
+
         }
     }
 
+    static void log(final ESLogger logger, final String error) {
+        logger.warn(error);
+    }
+
     /**
      * Tests if the checks should be enforced
      *
@@ -151,6 +188,14 @@ final class BootstrapCheck {
          */
         String errorMessage();
 
+        /**
+         * test if the check is a system-level check
+         *
+         * @return true if the check is a system-level check as opposed
+         * to an Elasticsearch-level check
+         */
+        boolean isSystemCheck();
+
     }
 
     static class HeapSizeCheck implements BootstrapCheck.Check {
@@ -183,6 +228,11 @@ final class BootstrapCheck {
             return JvmInfo.jvmInfo().getConfiguredMaxHeapSize();
         }
 
+        @Override
+        public final boolean isSystemCheck() {
+            return false;
+        }
+
     }
 
     static class OsXFileDescriptorCheck extends FileDescriptorCheck {
@@ -233,6 +283,11 @@ final class BootstrapCheck {
             return ProcessProbe.getInstance().getMaxFileDescriptorCount();
         }
 
+        @Override
+        public final boolean isSystemCheck() {
+            return true;
+        }
+
     }
 
     // visible for testing
@@ -259,6 +314,11 @@ final class BootstrapCheck {
             return Natives.isMemoryLocked();
         }
 
+        @Override
+        public final boolean isSystemCheck() {
+            return true;
+        }
+
     }
 
     static class MinMasterNodesCheck implements Check {
@@ -279,6 +339,12 @@ final class BootstrapCheck {
             return "please set [" + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() +
                 "] to a majority of the number of master eligible nodes in your cluster.";
         }
+
+        @Override
+        public final boolean isSystemCheck() {
+            return false;
+        }
+
     }
 
     static class MaxNumberOfThreadsCheck implements Check {
@@ -305,6 +371,11 @@ final class BootstrapCheck {
             return JNANatives.MAX_NUMBER_OF_THREADS;
         }
 
+        @Override
+        public final boolean isSystemCheck() {
+            return true;
+        }
+
     }
 
     static class MaxSizeVirtualMemoryCheck implements Check {
@@ -333,6 +404,11 @@ final class BootstrapCheck {
             return JNANatives.MAX_SIZE_VIRTUAL_MEMORY;
         }
 
+        @Override
+        public final boolean isSystemCheck() {
+            return true;
+        }
+
     }
 
     static class MaxMapCountCheck implements Check {
@@ -396,6 +472,11 @@ final class BootstrapCheck {
             return Long.parseLong(procSysVmMaxMapCount);
         }
 
+        @Override
+        public boolean isSystemCheck() {
+            return true;
+        }
+
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java
index 4e9dffc995b..b2059380181 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java
@@ -37,5 +37,7 @@ public final class BootstrapSettings {
         Setting.boolSetting("bootstrap.seccomp", true, Property.NodeScope);
     public static final Setting<Boolean> CTRLHANDLER_SETTING =
         Setting.boolSetting("bootstrap.ctrlhandler", true, Property.NodeScope);
+    public static final Setting<Boolean> IGNORE_SYSTEM_BOOTSTRAP_CHECKS =
+            Setting.boolSetting("bootstrap.ignore_system_bootstrap_checks", false, Property.NodeScope);
 
 }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index 55aa58ca588..dad5f48ce27 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -408,6 +408,7 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     BootstrapSettings.MLOCKALL_SETTING,
                     BootstrapSettings.SECCOMP_SETTING,
                     BootstrapSettings.CTRLHANDLER_SETTING,
+                    BootstrapSettings.IGNORE_SYSTEM_BOOTSTRAP_CHECKS,
                     IndexingMemoryController.INDEX_BUFFER_SIZE_SETTING,
                     IndexingMemoryController.MIN_INDEX_BUFFER_SIZE_SETTING,
                     IndexingMemoryController.MAX_INDEX_BUFFER_SIZE_SETTING,
diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
index 9f6a4e25eb5..235957ac18b 100644
--- a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
+++ b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.bootstrap;
 
 import org.apache.lucene.util.Constants;
+import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
@@ -38,6 +39,8 @@ import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.not;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 public class BootstrapCheckTests extends ESTestCase {
@@ -113,6 +116,11 @@ public class BootstrapCheckTests extends ESTestCase {
                     public String errorMessage() {
                         return "first";
                     }
+
+                    @Override
+                    public boolean isSystemCheck() {
+                        return false;
+                    }
                 },
                 new BootstrapCheck.Check() {
                     @Override
@@ -124,11 +132,16 @@ public class BootstrapCheckTests extends ESTestCase {
                     public String errorMessage() {
                         return "second";
                     }
+
+                    @Override
+                    public boolean isSystemCheck() {
+                        return false;
+                    }
                 }
         );
 
         final RuntimeException e =
-                expectThrows(RuntimeException.class, () -> BootstrapCheck.check(true, checks, "testExceptionAggregation"));
+                expectThrows(RuntimeException.class, () -> BootstrapCheck.check(true, false, checks, "testExceptionAggregation"));
         assertThat(e, hasToString(allOf(containsString("bootstrap checks failed"), containsString("first"), containsString("second"))));
         final Throwable[] suppressed = e.getSuppressed();
         assertThat(suppressed.length, equalTo(2));
@@ -159,7 +172,7 @@ public class BootstrapCheckTests extends ESTestCase {
         final RuntimeException e =
                 expectThrows(
                         RuntimeException.class,
-                        () -> BootstrapCheck.check(true, Collections.singletonList(check), "testHeapSizeCheck"));
+                        () -> BootstrapCheck.check(true, false, Collections.singletonList(check), "testHeapSizeCheck"));
         assertThat(
                 e.getMessage(),
                 containsString("initial heap size [" + initialHeapSize.get() + "] " +
@@ -167,7 +180,7 @@ public class BootstrapCheckTests extends ESTestCase {
 
         initialHeapSize.set(maxHeapSize.get());
 
-        BootstrapCheck.check(true, Collections.singletonList(check), "testHeapSizeCheck");
+        BootstrapCheck.check(true, false, Collections.singletonList(check), "testHeapSizeCheck");
 
         // nothing should happen if the initial heap size or the max
         // heap size is not available
@@ -176,7 +189,7 @@ public class BootstrapCheckTests extends ESTestCase {
         } else {
             maxHeapSize.set(0);
         }
-        BootstrapCheck.check(true, Collections.singletonList(check), "testHeapSizeCheck");
+        BootstrapCheck.check(true, false, Collections.singletonList(check), "testHeapSizeCheck");
     }
 
     public void testFileDescriptorLimits() {
@@ -202,17 +215,17 @@ public class BootstrapCheckTests extends ESTestCase {
 
         final RuntimeException e =
                 expectThrows(RuntimeException.class,
-                        () -> BootstrapCheck.check(true, Collections.singletonList(check), "testFileDescriptorLimits"));
+                        () -> BootstrapCheck.check(true, false, Collections.singletonList(check), "testFileDescriptorLimits"));
         assertThat(e.getMessage(), containsString("max file descriptors"));
 
         maxFileDescriptorCount.set(randomIntBetween(limit + 1, Integer.MAX_VALUE));
 
-        BootstrapCheck.check(true, Collections.singletonList(check), "testFileDescriptorLimits");
+        BootstrapCheck.check(true, false, Collections.singletonList(check), "testFileDescriptorLimits");
 
         // nothing should happen if current file descriptor count is
         // not available
         maxFileDescriptorCount.set(-1);
-        BootstrapCheck.check(true, Collections.singletonList(check), "testFileDescriptorLimits");
+        BootstrapCheck.check(true, false, Collections.singletonList(check), "testFileDescriptorLimits");
     }
 
     public void testFileDescriptorLimitsThrowsOnInvalidLimit() {
@@ -255,13 +268,17 @@ public class BootstrapCheckTests extends ESTestCase {
             if (testCase.shouldFail) {
                 final RuntimeException e = expectThrows(
                         RuntimeException.class,
-                        () -> BootstrapCheck.check(true, Collections.singletonList(check), "testFileDescriptorLimitsThrowsOnInvalidLimit"));
+                        () -> BootstrapCheck.check(
+                                true,
+                                false,
+                                Collections.singletonList(check),
+                                "testFileDescriptorLimitsThrowsOnInvalidLimit"));
                 assertThat(
                         e.getMessage(),
                         containsString("memory locking requested for elasticsearch process but memory is not locked"));
             } else {
                 // nothing should happen
-                BootstrapCheck.check(true, Collections.singletonList(check), "testFileDescriptorLimitsThrowsOnInvalidLimit");
+                BootstrapCheck.check(true, false, Collections.singletonList(check), "testFileDescriptorLimitsThrowsOnInvalidLimit");
             }
         }
     }
@@ -278,17 +295,17 @@ public class BootstrapCheckTests extends ESTestCase {
 
         final RuntimeException e = expectThrows(
                 RuntimeException.class,
-                () -> BootstrapCheck.check(true, Collections.singletonList(check), "testMaxNumberOfThreadsCheck"));
+                () -> BootstrapCheck.check(true, false, Collections.singletonList(check), "testMaxNumberOfThreadsCheck"));
         assertThat(e.getMessage(), containsString("max number of threads"));
 
         maxNumberOfThreads.set(randomIntBetween(limit + 1, Integer.MAX_VALUE));
 
-        BootstrapCheck.check(true, Collections.singletonList(check), "testMaxNumberOfThreadsCheck");
+        BootstrapCheck.check(true, false, Collections.singletonList(check), "testMaxNumberOfThreadsCheck");
 
         // nothing should happen if current max number of threads is
         // not available
         maxNumberOfThreads.set(-1);
-        BootstrapCheck.check(true, Collections.singletonList(check), "testMaxNumberOfThreadsCheck");
+        BootstrapCheck.check(true, false, Collections.singletonList(check), "testMaxNumberOfThreadsCheck");
     }
 
     public void testMaxSizeVirtualMemory() {
@@ -309,17 +326,17 @@ public class BootstrapCheckTests extends ESTestCase {
 
         final RuntimeException e = expectThrows(
                 RuntimeException.class,
-                () -> BootstrapCheck.check(true, Collections.singletonList(check), "testMaxSizeVirtualMemory"));
+                () -> BootstrapCheck.check(true, false, Collections.singletonList(check), "testMaxSizeVirtualMemory"));
         assertThat(e.getMessage(), containsString("max size virtual memory"));
 
         maxSizeVirtualMemory.set(rlimInfinity);
 
-        BootstrapCheck.check(true, Collections.singletonList(check), "testMaxSizeVirtualMemory");
+        BootstrapCheck.check(true, false, Collections.singletonList(check), "testMaxSizeVirtualMemory");
 
         // nothing should happen if max size virtual memory is not
         // available
         maxSizeVirtualMemory.set(Long.MIN_VALUE);
-        BootstrapCheck.check(true, Collections.singletonList(check), "testMaxSizeVirtualMemory");
+        BootstrapCheck.check(true, false, Collections.singletonList(check), "testMaxSizeVirtualMemory");
     }
 
     public void testMaxMapCountCheck() {
@@ -334,17 +351,17 @@ public class BootstrapCheckTests extends ESTestCase {
 
         RuntimeException e = expectThrows(
                 RuntimeException.class,
-                () -> BootstrapCheck.check(true, Collections.singletonList(check), "testMaxMapCountCheck"));
+                () -> BootstrapCheck.check(true, false, Collections.singletonList(check), "testMaxMapCountCheck"));
         assertThat(e.getMessage(), containsString("max virtual memory areas vm.max_map_count"));
 
         maxMapCount.set(randomIntBetween(limit + 1, Integer.MAX_VALUE));
 
-        BootstrapCheck.check(true, Collections.singletonList(check), "testMaxMapCountCheck");
+        BootstrapCheck.check(true, false, Collections.singletonList(check), "testMaxMapCountCheck");
 
         // nothing should happen if current vm.max_map_count is not
         // available
         maxMapCount.set(-1);
-        BootstrapCheck.check(true, Collections.singletonList(check), "testMaxMapCountCheck");
+        BootstrapCheck.check(true, false, Collections.singletonList(check), "testMaxMapCountCheck");
     }
 
     public void testMinMasterNodes() {
@@ -353,7 +370,42 @@ public class BootstrapCheckTests extends ESTestCase {
         assertThat(check.check(), not(equalTo(isSet)));
         List<BootstrapCheck.Check> defaultChecks = BootstrapCheck.checks(Settings.EMPTY);
 
-        expectThrows(RuntimeException.class, () -> BootstrapCheck.check(true, defaultChecks, "testMinMasterNodes"));
+        expectThrows(RuntimeException.class, () -> BootstrapCheck.check(true, false, defaultChecks, "testMinMasterNodes"));
+    }
+
+    public void testIgnoringSystemChecks() {
+        BootstrapCheck.Check check = new BootstrapCheck.Check() {
+            @Override
+            public boolean check() {
+                return true;
+            }
+
+            @Override
+            public String errorMessage() {
+                return "error";
+            }
+
+            @Override
+            public boolean isSystemCheck() {
+                return true;
+            }
+        };
+
+        final RuntimeException notIgnored = expectThrows(
+                RuntimeException.class,
+                () -> BootstrapCheck.check(true, false, Collections.singletonList(check), "testIgnoringSystemChecks"));
+        assertThat(notIgnored, hasToString(containsString("error")));
+
+        final ESLogger logger = mock(ESLogger.class);
+
+        // nothing should happen if we ignore system checks
+        BootstrapCheck.check(true, true, Collections.singletonList(check), logger);
+        verify(logger).warn("error");
+        reset(logger);
+
+        // nothing should happen if we ignore all checks
+        BootstrapCheck.check(false, randomBoolean(), Collections.singletonList(check), logger);
+        verify(logger).warn("error");
     }
 
 }

From 8bdda49f907a45f9704377bc81dcd798ec1f48d2 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Mon, 2 May 2016 10:58:45 +0200
Subject: [PATCH 0038/1311] Fail automatic string upgrade if the value of
 `index` is not recognized. #18082

Closes #18062
---
 .../index/mapper/core/StringFieldMapper.java     |  6 ++++--
 .../mapper/core/StringMappingUpgradeTests.java   | 16 ++++++++++++++++
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java
index 65d11b45475..0c98dd10c3e 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java
@@ -29,7 +29,6 @@ import org.apache.lucene.search.RegexpQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
@@ -204,6 +203,9 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
         public Mapper.Builder parse(String fieldName, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
             if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0_alpha1)) {
                 final Object index = node.get("index");
+                if (Arrays.asList(null, "no", "not_analyzed", "analyzed").contains(index) == false) {
+                    throw new IllegalArgumentException("Can't parse [index] value [" + index + "] for field [" + fieldName + "], expected [no], [not_analyzed] or [analyzed]");
+                }
                 final boolean keyword = index != null && "analyzed".equals(index) == false;
 
                 // Automatically upgrade simple mappings for ease of upgrade, otherwise fail
@@ -283,7 +285,7 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
                     node.put("index", false);
                     break;
                 default:
-                    throw new IllegalArgumentException("Can't parse [index] value [" + index + "] for field [" + fieldName + "], expected [true], [false], [no], [not_analyzed] or [analyzed]");
+                    throw new IllegalArgumentException("Can't parse [index] value [" + index + "] for field [" + fieldName + "], expected [no], [not_analyzed] or [analyzed]");
                 }
             }
             final Object fielddataObject = node.get("fielddata");
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/StringMappingUpgradeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/StringMappingUpgradeTests.java
index a8dd1b65fcf..41094fec58a 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/core/StringMappingUpgradeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/core/StringMappingUpgradeTests.java
@@ -103,6 +103,22 @@ public class StringMappingUpgradeTests extends ESSingleNodeTestCase {
         assertEquals(IndexOptions.NONE, field.fieldType().indexOptions());
     }
 
+    public void testIllegalIndexValue() throws IOException {
+        IndexService indexService = createIndex("test");
+        DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties")
+                    .startObject("field")
+                        .field("type", "string")
+                        .field("index", false)
+                    .endObject()
+                .endObject() .endObject().endObject().string();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> parser.parse("type", new CompressedXContent(mapping)));
+        assertThat(e.getMessage(),
+                containsString("Can't parse [index] value [false] for field [field], expected [no], [not_analyzed] or [analyzed]"));
+    }
+
     public void testNotSupportedUpgrade() throws IOException {
         IndexService indexService = createIndex("test");
         DocumentMapperParser parser = indexService.mapperService().documentMapperParser();

From 226679c606098888e233309e9816aaeb0fbea1ef Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 2 May 2016 11:48:35 -0400
Subject: [PATCH 0039/1311] Mark MaxMapCountCheck#isSystemCheck as final

This commit marks the method MaxMapCountCheck#isSystemCheck as final as
this method should not be modifiable by inheriting classes (used in
tests).
---
 .../main/java/org/elasticsearch/bootstrap/BootstrapCheck.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
index 25bdbe3fa8a..72f5a9a6bdc 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
@@ -473,7 +473,7 @@ final class BootstrapCheck {
         }
 
         @Override
-        public boolean isSystemCheck() {
+        public final boolean isSystemCheck() {
             return true;
         }
 

From 15f33e61b74d4c65291f0ad157f89bd898f3ab96 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 2 May 2016 12:22:40 -0400
Subject: [PATCH 0040/1311] Kill redundant conditional in BootstrapCheck#check

This commit removes an unnecessary if statement in Bootstrap#check. The
removed if statement was duplicating the conditionals in the nested if
statements and was merely an artifact of an earlier refactoring.
---
 .../bootstrap/BootstrapCheck.java             | 27 +++++++++----------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
index 72f5a9a6bdc..626e274076c 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
@@ -114,22 +114,19 @@ final class BootstrapCheck {
             }
         }
 
-        if (!errors.isEmpty() || !ignoredErrors.isEmpty()) {
-
-            if (!ignoredErrors.isEmpty()) {
-                ignoredErrors.forEach(error -> log(logger, error));
-            }
-
-            if (!errors.isEmpty()) {
-                final List<String> messages = new ArrayList<>(1 + errors.size());
-                messages.add("bootstrap checks failed");
-                messages.addAll(errors);
-                final RuntimeException re = new RuntimeException(String.join("\n", messages));
-                errors.stream().map(IllegalStateException::new).forEach(re::addSuppressed);
-                throw re;
-            }
-
+        if (!ignoredErrors.isEmpty()) {
+            ignoredErrors.forEach(error -> log(logger, error));
         }
+
+        if (!errors.isEmpty()) {
+            final List<String> messages = new ArrayList<>(1 + errors.size());
+            messages.add("bootstrap checks failed");
+            messages.addAll(errors);
+            final RuntimeException re = new RuntimeException(String.join("\n", messages));
+            errors.stream().map(IllegalStateException::new).forEach(re::addSuppressed);
+            throw re;
+        }
+
     }
 
     static void log(final ESLogger logger, final String error) {

From 16c4c251f5d94a34957d735647fdd665b97f4406 Mon Sep 17 00:00:00 2001
From: Michael McCandless <mike@elastic.co>
Date: Mon, 2 May 2016 14:40:08 -0400
Subject: [PATCH 0041/1311] Merge pull request #18094 from
 mikemccand/completion_stats_once

Don't try to compute completion stats on a reader after we already closed it

Conflicts:
	core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
---
 .../main/java/org/elasticsearch/index/shard/IndexShard.java    | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 8739c265701..ae6f1b6bb82 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -680,9 +680,6 @@ public class IndexShard extends AbstractIndexShardComponent {
         CompletionStats completionStats = new CompletionStats();
         try (final Engine.Searcher currentSearcher = acquireSearcher("completion_stats")) {
             completionStats.add(CompletionFieldStats.completionStats(currentSearcher.reader(), fields));
-            Completion090PostingsFormat postingsFormat = ((Completion090PostingsFormat)
-                PostingsFormat.forName(Completion090PostingsFormat.CODEC_NAME));
-            completionStats.add(postingsFormat.completionStats(currentSearcher.reader(), fields));
         }
         return completionStats;
     }

From 0c6d6a5495d6a99866c5ea7acb99b821e93223e1 Mon Sep 17 00:00:00 2001
From: Mike McCandless <mikemccand@apache.org>
Date: Mon, 2 May 2016 16:14:05 -0400
Subject: [PATCH 0042/1311] also compute completion stats for 2.x shards

---
 .../main/java/org/elasticsearch/index/shard/IndexShard.java   | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index ae6f1b6bb82..c1610e9bb1f 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -680,6 +680,10 @@ public class IndexShard extends AbstractIndexShardComponent {
         CompletionStats completionStats = new CompletionStats();
         try (final Engine.Searcher currentSearcher = acquireSearcher("completion_stats")) {
             completionStats.add(CompletionFieldStats.completionStats(currentSearcher.reader(), fields));
+            // Necessary for 2.x shards:
+            Completion090PostingsFormat postingsFormat = ((Completion090PostingsFormat)
+                PostingsFormat.forName(Completion090PostingsFormat.CODEC_NAME));
+            completionStats.add(postingsFormat.completionStats(currentSearcher.reader(), fields));
         }
         return completionStats;
     }

From 693c1f667142f322e6faaf1abacee79c66660b37 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 2 May 2016 17:49:21 -0400
Subject: [PATCH 0043/1311] Support geo_point fields in lucene expressions.

Closes #18096
---
 .../modules/scripting/scripting.asciidoc      |  91 +++++++++++---
 .../expression/CountMethodFunctionValues.java |  44 -------
 .../expression/CountMethodValueSource.java    |  21 ++--
 .../script/expression/DateField.java          |  94 +++++++++++++++
 .../expression/DateMethodFunctionValues.java  |  47 --------
 .../expression/DateMethodValueSource.java     |  26 ++--
 .../expression/EmptyMemberValueSource.java    |   6 +-
 .../ExpressionScriptEngineService.java        | 113 +++++-------------
 .../expression/FieldDataFunctionValues.java   |  43 -------
 .../expression/FieldDataValueSource.java      |  27 +++--
 .../expression/GeoEmptyValueSource.java       |  81 +++++++++++++
 .../script/expression/GeoField.java           |  53 ++++++++
 .../expression/GeoLatitudeValueSource.java    |  81 +++++++++++++
 .../expression/GeoLongitudeValueSource.java   |  81 +++++++++++++
 .../script/expression/NumericField.java       |  75 ++++++++++++
 .../ReplaceableConstFunctionValues.java       |   4 +-
 .../ReplaceableConstValueSource.java          |   4 +-
 .../expression/MoreExpressionTests.java       |  44 ++++++-
 18 files changed, 666 insertions(+), 269 deletions(-)
 delete mode 100644 modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodFunctionValues.java
 create mode 100644 modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateField.java
 delete mode 100644 modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodFunctionValues.java
 delete mode 100644 modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataFunctionValues.java
 create mode 100644 modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java
 create mode 100644 modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoField.java
 create mode 100644 modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java
 create mode 100644 modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java
 create mode 100644 modules/lang-expression/src/main/java/org/elasticsearch/script/expression/NumericField.java

diff --git a/docs/reference/modules/scripting/scripting.asciidoc b/docs/reference/modules/scripting/scripting.asciidoc
index 5935554a368..b1f779b48a0 100644
--- a/docs/reference/modules/scripting/scripting.asciidoc
+++ b/docs/reference/modules/scripting/scripting.asciidoc
@@ -23,7 +23,7 @@ to specify the language of the script. Plugins are available for following langu
 |groovy     |no        |built-in
 |expression |yes       |built-in
 |mustache   |yes       |built-in
-/painless   /yes       /built-in (module)
+|painless   |yes       |built-in (module)
 |javascript |no        |{plugins}/lang-javascript.html[elasticsearch-lang-javascript]
 |python     |no        |{plugins}/lang-python.html[elasticsearch-lang-python]
 |=======================================================================
@@ -455,41 +455,94 @@ for details on what operators and functions are available.
 
 Variables in `expression` scripts are available to access:
 
-* document fields, e.g. `doc['myfield'].value` or just `doc['myfield']`.
-* whether the field is empty, e.g. `doc['myfield'].empty`
+* document fields, e.g. `doc['myfield'].value` 
+* variables and methods that the field supports, e.g. `doc['myfield'].empty`
 * Parameters passed into the script, e.g. `mymodifier`
 * The current document's score, `_score` (only available when used in a `script_score`)
 
+[float]
+=== Expressions API for numeric fields
+[cols="<,<",options="header",]
+|=======================================================================
+|Expression |Description
+|`doc['field_name'].value` |The native value of the field. For example,
+if its a short type, it will be short.
+
+|`doc['field_name'].empty` |A boolean indicating if the field has no
+values within the doc.
+
+|`doc['field_name'].min()` |The minimum value of the field in this document.
+
+|`doc['field_name'].max()` |The maximum value of the field in this document.
+
+|`doc['field_name'].median()` |The median value of the field in this document.
+
+|`doc['field_name'].avg()` |The average of the values in this document.
+
+|`doc['field_name'].sum()` |The sum of the values in this document.
+
+|`doc['field_name'].count()` |The number of values in this document.
+|=======================================================================
+
 When a document is missing the field completely, by default the value will be treated as `0`.
 You can treat it as another value instead, e.g. `doc['myfield'].empty ? 100 : doc['myfield'].value`
 
 When a document has multiple values for the field, by default the minimum value is returned.
-You can choose a different value instead, e.g. `doc['myfield'].sum()`. The following methods are available
-for any field:
+You can choose a different value instead, e.g. `doc['myfield'].sum()`.
 
-* min()
-* max()
-* avg()
-* median()
-* sum()
-* count() 
+When a document is missing the field completely, by default the value will be treated as `0`.
 
-Variables in `expression` scripts that are of type `date` may use the following member methods:
+[float]
+=== Additional methods for date fields
+Date fields are treated as the number of milliseconds since January 1, 1970 and 
+support the numeric API above, with these additional methods:
 
-* getYear()
-* getMonth()
-* getDayOfMonth()
-* getHourOfDay()
-* getMinutes()
-* getSeconds()
+[cols="<,<",options="header",]
+|=======================================================================
+|Expression |Description
+|`doc['field_name'].getYear()` |Year component, e.g. `1970`.
+
+|`doc['field_name'].getMonth()` |Month component (0-11), e.g. `0` for January.
+
+|`doc['field_name'].getDayOfMonth()` |Day component, e.g. `1` for the first of the month.
+
+|`doc['field_name'].getHourOfDay()` |Hour component (0-23)
+
+|`doc['field_name'].getMinutes()` |Minutes component (0-59)
+
+|`doc['field_name'].getSeconds()` |Seconds component (0-59)
+|=======================================================================
 
 The following example shows the difference in years between the `date` fields date0 and date1:
 
 `doc['date1'].getYear() - doc['date0'].getYear()`
 
+[float]
+=== Expressions API for `geo_point` fields
+[cols="<,<",options="header",]
+|=======================================================================
+|Expression |Description
+|`doc['field_name'].empty` |A boolean indicating if the field has no
+values within the doc.
+
+|`doc['field_name'].lat` |The latitude of the geo point.
+
+|`doc['field_name'].lon` |The longitude of the geo point.
+|=======================================================================
+
+The following example computes distance in kilometers from Washington, DC:
+
+`haversin(38.9072, 77.0369, doc['field_name'].lat, doc['field_name'].lon)`
+
+In this example the coordinates could have been passed as parameters to the script,
+e.g. based on geolocation of the user.
+
+[float]
+=== Expressions limitations
+
 There are a few limitations relative to other script languages:
 
-* Only numeric fields may be accessed
+* Only numeric, date, and geo_point fields may be accessed
 * Stored fields are not available
 
 [float]
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodFunctionValues.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodFunctionValues.java
deleted file mode 100644
index 818404e98e2..00000000000
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodFunctionValues.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.script.expression;
-
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
-import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
-import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
-
-/**
- * FunctionValues to get the count of the number of values in a field for a document.
- */
-public class CountMethodFunctionValues extends DoubleDocValues {
-    SortedNumericDoubleValues values;
-
-    CountMethodFunctionValues(ValueSource parent, AtomicNumericFieldData fieldData) {
-        super(parent);
-
-        values = fieldData.getDoubleValues();
-    }
-
-    @Override
-    public double doubleVal(int doc) {
-        values.setDocument(doc);
-        return values.count();
-    }
-}
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java
index 6f397c02bd3..1aa85ef685b 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/CountMethodValueSource.java
@@ -26,17 +26,18 @@ import java.util.Objects;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.queries.function.FunctionValues;
 import org.apache.lucene.queries.function.ValueSource;
-import org.elasticsearch.index.fielddata.AtomicFieldData;
+import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
 import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 
 /**
  * A ValueSource to create FunctionValues to get the count of the number of values in a field for a document.
  */
-public class CountMethodValueSource extends ValueSource {
-    protected IndexFieldData<?> fieldData;
+final class CountMethodValueSource extends ValueSource {
+    IndexFieldData<?> fieldData;
 
-    protected CountMethodValueSource(IndexFieldData<?> fieldData) {
+    CountMethodValueSource(IndexFieldData<?> fieldData) {
         Objects.requireNonNull(fieldData);
 
         this.fieldData = fieldData;
@@ -45,10 +46,16 @@ public class CountMethodValueSource extends ValueSource {
     @Override
     @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
     public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
-        AtomicFieldData leafData = fieldData.load(leaf);
-        assert(leafData instanceof AtomicNumericFieldData);
+        AtomicNumericFieldData leafData = (AtomicNumericFieldData) fieldData.load(leaf);
+        final SortedNumericDoubleValues values = leafData.getDoubleValues();
 
-        return new CountMethodFunctionValues(this, (AtomicNumericFieldData)leafData);
+        return new DoubleDocValues(this) {
+          @Override
+          public double doubleVal(int doc) {
+            values.setDocument(doc);
+            return values.count();
+          }
+        };
     }
 
     @Override
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateField.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateField.java
new file mode 100644
index 00000000000..e4648887772
--- /dev/null
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateField.java
@@ -0,0 +1,94 @@
+package org.elasticsearch.script.expression;
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.Calendar;
+
+import org.apache.lucene.queries.function.ValueSource;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.search.MultiValueMode;
+
+/**
+ * Expressions API for date fields.
+ */
+final class DateField {
+    // no instance
+    private DateField() {}
+    
+    // supported variables
+    static final String VALUE_VARIABLE          = "value";
+    static final String EMPTY_VARIABLE          = "empty";
+    
+    // supported methods
+    static final String MINIMUM_METHOD          = "min";
+    static final String MAXIMUM_METHOD          = "max";
+    static final String AVERAGE_METHOD          = "avg";
+    static final String MEDIAN_METHOD           = "median";
+    static final String SUM_METHOD              = "sum";
+    static final String COUNT_METHOD            = "count";
+    static final String GET_YEAR_METHOD         = "getYear";
+    static final String GET_MONTH_METHOD        = "getMonth";
+    static final String GET_DAY_OF_MONTH_METHOD = "getDayOfMonth";
+    static final String GET_HOUR_OF_DAY_METHOD  = "getHourOfDay";
+    static final String GET_MINUTES_METHOD      = "getMinutes";
+    static final String GET_SECONDS_METHOD      = "getSeconds";
+    
+    static ValueSource getVariable(IndexFieldData<?> fieldData, String fieldName, String variable) {
+        switch (variable) {
+            case VALUE_VARIABLE:
+                return new FieldDataValueSource(fieldData, MultiValueMode.MIN);
+            case EMPTY_VARIABLE:
+                return new EmptyMemberValueSource(fieldData);
+            default:
+                throw new IllegalArgumentException("Member variable [" + variable + "] does not exist for date field [" + fieldName + "].");
+        }
+    }
+    
+    static ValueSource getMethod(IndexFieldData<?> fieldData, String fieldName, String method) {
+        switch (method) {
+            case MINIMUM_METHOD:
+                return new FieldDataValueSource(fieldData, MultiValueMode.MIN);
+            case MAXIMUM_METHOD:
+                return new FieldDataValueSource(fieldData, MultiValueMode.MAX);
+            case AVERAGE_METHOD:
+                return new FieldDataValueSource(fieldData, MultiValueMode.AVG);
+            case MEDIAN_METHOD:
+                return new FieldDataValueSource(fieldData, MultiValueMode.MEDIAN);
+            case SUM_METHOD:
+                return new FieldDataValueSource(fieldData, MultiValueMode.SUM);
+            case COUNT_METHOD:
+                return new CountMethodValueSource(fieldData);
+            case GET_YEAR_METHOD:
+                return new DateMethodValueSource(fieldData, MultiValueMode.MIN, method, Calendar.YEAR);
+            case GET_MONTH_METHOD:
+                return new DateMethodValueSource(fieldData, MultiValueMode.MIN, method, Calendar.MONTH);
+            case GET_DAY_OF_MONTH_METHOD:
+                return new DateMethodValueSource(fieldData, MultiValueMode.MIN, method, Calendar.DAY_OF_MONTH);
+            case GET_HOUR_OF_DAY_METHOD:
+                return new DateMethodValueSource(fieldData, MultiValueMode.MIN, method, Calendar.HOUR_OF_DAY);
+            case GET_MINUTES_METHOD:
+                return new DateMethodValueSource(fieldData, MultiValueMode.MIN, method, Calendar.MINUTE);
+            case GET_SECONDS_METHOD:
+                return new DateMethodValueSource(fieldData, MultiValueMode.MIN, method, Calendar.SECOND);
+            default:
+                throw new IllegalArgumentException("Member method [" + method + "] does not exist for date field [" + fieldName + "].");
+        }
+    }
+}
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodFunctionValues.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodFunctionValues.java
deleted file mode 100644
index 3ed2ed1f0b5..00000000000
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodFunctionValues.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.script.expression;
-
-import org.apache.lucene.queries.function.ValueSource;
-import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
-import org.elasticsearch.search.MultiValueMode;
-
-import java.util.Calendar;
-import java.util.Locale;
-import java.util.TimeZone;
-
-class DateMethodFunctionValues extends FieldDataFunctionValues {
-    private final int calendarType;
-    private final Calendar calendar;
-
-    DateMethodFunctionValues(ValueSource parent, MultiValueMode multiValueMode,  AtomicNumericFieldData data, int calendarType) {
-        super(parent, multiValueMode, data);
-
-        this.calendarType = calendarType;
-        calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT);
-    }
-
-    @Override
-    public double doubleVal(int docId) {
-        long millis = (long)dataAccessor.get(docId);
-        calendar.setTimeInMillis(millis);
-        return calendar.get(calendarType);
-    }
-}
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java
index e6c9dcddc78..98909f4401a 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateMethodValueSource.java
@@ -20,20 +20,25 @@
 package org.elasticsearch.script.expression;
 
 import java.io.IOException;
+import java.util.Calendar;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.TimeZone;
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.queries.function.FunctionValues;
-import org.elasticsearch.index.fielddata.AtomicFieldData;
+import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
 import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.NumericDoubleValues;
 import org.elasticsearch.search.MultiValueMode;
 
+/** Extracts a portion of a date field with {@code Calendar.get()} */
 class DateMethodValueSource extends FieldDataValueSource {
 
-    protected final String methodName;
-    protected final int calendarType;
+    final String methodName;
+    final int calendarType;
 
     DateMethodValueSource(IndexFieldData<?> indexFieldData, MultiValueMode multiValueMode, String methodName, int calendarType) {
         super(indexFieldData, multiValueMode);
@@ -47,10 +52,17 @@ class DateMethodValueSource extends FieldDataValueSource {
     @Override
     @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
     public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
-        AtomicFieldData leafData = fieldData.load(leaf);
-        assert(leafData instanceof AtomicNumericFieldData);
-
-        return new DateMethodFunctionValues(this, multiValueMode, (AtomicNumericFieldData)leafData, calendarType);
+        AtomicNumericFieldData leafData = (AtomicNumericFieldData) fieldData.load(leaf);
+        final Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT);
+        NumericDoubleValues docValues = multiValueMode.select(leafData.getDoubleValues(), 0d);
+        return new DoubleDocValues(this) {
+          @Override
+          public double doubleVal(int docId) {
+            long millis = (long)docValues.get(docId);
+            calendar.setTimeInMillis(millis);
+            return calendar.get(calendarType);
+          }
+        };
     }
 
     @Override
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java
index b8c101e8abc..b4c8582e0d6 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/EmptyMemberValueSource.java
@@ -36,10 +36,10 @@ import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
  * <p>
  * This is essentially sugar over !count()
  */
-public class EmptyMemberValueSource extends ValueSource {
-    protected IndexFieldData<?> fieldData;
+final class EmptyMemberValueSource extends ValueSource {
+    final IndexFieldData<?> fieldData;
 
-    protected EmptyMemberValueSource(IndexFieldData<?> fieldData) {
+    EmptyMemberValueSource(IndexFieldData<?> fieldData) {
         this.fieldData = Objects.requireNonNull(fieldData);
     }
 
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
index 2bfd2928d57..a6ed6253263 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
@@ -37,20 +37,19 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.core.DateFieldMapper;
 import org.elasticsearch.index.mapper.core.LegacyDateFieldMapper;
+import org.elasticsearch.index.mapper.geo.BaseGeoPointFieldMapper;
 import org.elasticsearch.script.ClassPermission;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.ScriptEngineService;
 import org.elasticsearch.script.ScriptException;
 import org.elasticsearch.script.SearchScript;
-import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.security.AccessControlContext;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.text.ParseException;
-import java.util.Calendar;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -65,26 +64,6 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
 
     public static final List<String> TYPES = Collections.singletonList(NAME);
 
-    // these methods only work on dates, e.g. doc['datefield'].getYear()
-    protected static final String GET_YEAR_METHOD         = "getYear";
-    protected static final String GET_MONTH_METHOD        = "getMonth";
-    protected static final String GET_DAY_OF_MONTH_METHOD = "getDayOfMonth";
-    protected static final String GET_HOUR_OF_DAY_METHOD  = "getHourOfDay";
-    protected static final String GET_MINUTES_METHOD      = "getMinutes";
-    protected static final String GET_SECONDS_METHOD      = "getSeconds";
-
-    // these methods work on any field, e.g. doc['field'].sum()
-    protected static final String MINIMUM_METHOD          = "min";
-    protected static final String MAXIMUM_METHOD          = "max";
-    protected static final String AVERAGE_METHOD          = "avg";
-    protected static final String MEDIAN_METHOD           = "median";
-    protected static final String SUM_METHOD              = "sum";
-    protected static final String COUNT_METHOD            = "count";
-
-    // these variables work on any field, e.g. doc['field'].value
-    protected static final String VALUE_VARIABLE          = "value";
-    protected static final String EMPTY_VARIABLE          = "empty";
-
     @Inject
     public ExpressionScriptEngineService(Settings settings) {
         super(settings);
@@ -175,7 +154,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                 } else {
                     String fieldname = null;
                     String methodname = null;
-                    String variablename = VALUE_VARIABLE; // .value is the default for doc['field'], its optional.
+                    String variablename = "value"; // .value is the default for doc['field'], its optional.
                     VariableContext[] parts = VariableContext.parse(variable);
                     if (parts[0].text.equals("doc") == false) {
                         throw new ScriptException("Unknown variable [" + parts[0].text + "] in expression");
@@ -205,15 +184,38 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                     }
 
                     IndexFieldData<?> fieldData = lookup.doc().fieldDataService().getForField(fieldType);
-                    if (fieldData instanceof IndexNumericFieldData == false) {
-                        // TODO: more context (which expression?)
-                        throw new ScriptException("Field [" + fieldname + "] used in expression must be numeric");
-                    }
-                    if (methodname == null) {
-                        bindings.add(variable, getVariableValueSource(fieldType, fieldData, fieldname, variablename));
+                    
+                    // delegate valuesource creation based on field's type
+                    // there are three types of "fields" to expressions, and each one has a different "api" of variables and methods.
+                    
+                    final ValueSource valueSource;
+                    if (fieldType instanceof BaseGeoPointFieldMapper.GeoPointFieldType) {
+                        // geo
+                        if (methodname == null) {
+                            valueSource = GeoField.getVariable(fieldData, fieldname, variablename);
+                        } else {
+                            valueSource = GeoField.getMethod(fieldData, fieldname, methodname);
+                        }
+                    } else if (fieldType instanceof LegacyDateFieldMapper.DateFieldType || 
+                            fieldType instanceof DateFieldMapper.DateFieldType) {
+                        // date
+                        if (methodname == null) {
+                            valueSource = DateField.getVariable(fieldData, fieldname, variablename);
+                        } else {
+                            valueSource = DateField.getMethod(fieldData, fieldname, methodname);
+                        }
+                    } else if (fieldData instanceof IndexNumericFieldData) {
+                        // number
+                        if (methodname == null) {
+                            valueSource = NumericField.getVariable(fieldData, fieldname, variablename);
+                        } else {
+                            valueSource = NumericField.getMethod(fieldData, fieldname, methodname);
+                        }
                     } else {
-                        bindings.add(variable, getMethodValueSource(fieldType, fieldData, fieldname, methodname));
+                        throw new ScriptException("Field [" + fieldname + "] used in expression must be numeric, date, or geopoint");
                     }
+                    
+                    bindings.add(variable, valueSource);
                 }
             }
 
@@ -224,57 +226,6 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
         }
     }
 
-    protected ValueSource getMethodValueSource(MappedFieldType fieldType, IndexFieldData<?> fieldData, String fieldName, String methodName) {
-        switch (methodName) {
-            case GET_YEAR_METHOD:
-                return getDateMethodValueSource(fieldType, fieldData, fieldName, methodName, Calendar.YEAR);
-            case GET_MONTH_METHOD:
-                return getDateMethodValueSource(fieldType, fieldData, fieldName, methodName, Calendar.MONTH);
-            case GET_DAY_OF_MONTH_METHOD:
-                return getDateMethodValueSource(fieldType, fieldData, fieldName, methodName, Calendar.DAY_OF_MONTH);
-            case GET_HOUR_OF_DAY_METHOD:
-                return getDateMethodValueSource(fieldType, fieldData, fieldName, methodName, Calendar.HOUR_OF_DAY);
-            case GET_MINUTES_METHOD:
-                return getDateMethodValueSource(fieldType, fieldData, fieldName, methodName, Calendar.MINUTE);
-            case GET_SECONDS_METHOD:
-                return getDateMethodValueSource(fieldType, fieldData, fieldName, methodName, Calendar.SECOND);
-            case MINIMUM_METHOD:
-                return new FieldDataValueSource(fieldData, MultiValueMode.MIN);
-            case MAXIMUM_METHOD:
-                return new FieldDataValueSource(fieldData, MultiValueMode.MAX);
-            case AVERAGE_METHOD:
-                return new FieldDataValueSource(fieldData, MultiValueMode.AVG);
-            case MEDIAN_METHOD:
-                return new FieldDataValueSource(fieldData, MultiValueMode.MEDIAN);
-            case SUM_METHOD:
-                return new FieldDataValueSource(fieldData, MultiValueMode.SUM);
-            case COUNT_METHOD:
-                return new CountMethodValueSource(fieldData);
-            default:
-                throw new IllegalArgumentException("Member method [" + methodName + "] does not exist.");
-        }
-    }
-    
-    protected ValueSource getVariableValueSource(MappedFieldType fieldType, IndexFieldData<?> fieldData, String fieldName, String memberName) {
-        switch (memberName) {
-            case VALUE_VARIABLE:
-                return new FieldDataValueSource(fieldData, MultiValueMode.MIN);
-            case EMPTY_VARIABLE:
-                return new EmptyMemberValueSource(fieldData);
-            default:
-                throw new IllegalArgumentException("Member variable [" + memberName + "] does not exist.");
-        }
-    }
-
-    protected ValueSource getDateMethodValueSource(MappedFieldType fieldType, IndexFieldData<?> fieldData, String fieldName, String methodName, int calendarType) {
-        if (fieldType instanceof LegacyDateFieldMapper.DateFieldType == false
-                && fieldType instanceof DateFieldMapper.DateFieldType == false) {
-            throw new IllegalArgumentException("Member method [" + methodName + "] can only be used with a date field type, not the field [" + fieldName + "].");
-        }
-
-        return new DateMethodValueSource(fieldData, MultiValueMode.MIN, methodName, calendarType);
-    }
-
     @Override
     public ExecutableScript executable(CompiledScript compiledScript, Map<String, Object> vars) {
         return new ExpressionExecutableScript(compiledScript, vars);
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataFunctionValues.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataFunctionValues.java
deleted file mode 100644
index b3e06d6b9f2..00000000000
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataFunctionValues.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.script.expression;
-
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
-import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
-import org.elasticsearch.index.fielddata.NumericDoubleValues;
-import org.elasticsearch.search.MultiValueMode;
-
-/**
- * A {@link org.apache.lucene.queries.function.FunctionValues} which wrap field data.
- */
-class FieldDataFunctionValues extends DoubleDocValues {
-    NumericDoubleValues dataAccessor;
-
-    FieldDataFunctionValues(ValueSource parent, MultiValueMode m, AtomicNumericFieldData d) {
-        super(parent);
-        dataAccessor = m.select(d.getDoubleValues(), 0d);
-    }
-
-    @Override
-    public double doubleVal(int i) {
-        return dataAccessor.get(i);
-    }
-}
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java
index ae84a5cbfe7..3ac885e49dc 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java
@@ -26,9 +26,10 @@ import java.util.Objects;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.queries.function.FunctionValues;
 import org.apache.lucene.queries.function.ValueSource;
-import org.elasticsearch.index.fielddata.AtomicFieldData;
+import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
 import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.NumericDoubleValues;
 import org.elasticsearch.search.MultiValueMode;
 
 /**
@@ -36,15 +37,12 @@ import org.elasticsearch.search.MultiValueMode;
  */
 class FieldDataValueSource extends ValueSource {
 
-    protected IndexFieldData<?> fieldData;
-    protected MultiValueMode multiValueMode;
+    final IndexFieldData<?> fieldData;
+    final MultiValueMode multiValueMode;
 
-    protected FieldDataValueSource(IndexFieldData<?> d, MultiValueMode m) {
-        Objects.requireNonNull(d);
-        Objects.requireNonNull(m);
-
-        fieldData = d;
-        multiValueMode = m;
+    protected FieldDataValueSource(IndexFieldData<?> fieldData, MultiValueMode multiValueMode) {
+        this.fieldData = Objects.requireNonNull(fieldData);
+        this.multiValueMode = Objects.requireNonNull(multiValueMode);
     }
 
     @Override
@@ -69,9 +67,14 @@ class FieldDataValueSource extends ValueSource {
     @Override
     @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
     public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
-        AtomicFieldData leafData = fieldData.load(leaf);
-        assert(leafData instanceof AtomicNumericFieldData);
-        return new FieldDataFunctionValues(this, multiValueMode, (AtomicNumericFieldData)leafData);
+        AtomicNumericFieldData leafData = (AtomicNumericFieldData) fieldData.load(leaf);
+        NumericDoubleValues docValues = multiValueMode.select(leafData.getDoubleValues(), 0d);
+        return new DoubleDocValues(this) {
+          @Override
+          public double doubleVal(int doc) {
+            return docValues.get(doc);
+          }
+        };
     }
 
     @Override
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java
new file mode 100644
index 00000000000..2a9b09ba10d
--- /dev/null
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script.expression;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.MultiGeoPointValues;
+
+/**
+ * ValueSource to return non-zero if a field is missing.
+ */
+final class GeoEmptyValueSource extends ValueSource {
+    IndexFieldData<?> fieldData;
+
+    GeoEmptyValueSource(IndexFieldData<?> fieldData) {
+        this.fieldData = Objects.requireNonNull(fieldData);
+    }
+
+    @Override
+    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
+    public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
+        AtomicGeoPointFieldData leafData = (AtomicGeoPointFieldData) fieldData.load(leaf);
+        final MultiGeoPointValues values = leafData.getGeoPointValues();
+        return new DoubleDocValues(this) {
+            @Override
+            public double doubleVal(int doc) {
+                values.setDocument(doc);
+                if (values.count() == 0) {
+                    return 1;
+                } else {
+                    return 0;
+                }
+            }
+        };
+    }
+
+    @Override
+    public int hashCode() {
+        return 31 * getClass().hashCode() + fieldData.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null) return false;
+        if (getClass() != obj.getClass()) return false;
+        GeoEmptyValueSource other = (GeoEmptyValueSource) obj;
+        if (!fieldData.equals(other.fieldData)) return false;
+        return true;
+    }
+
+    @Override
+    public String description() {
+        return "empty: field(" + fieldData.getFieldName() + ")";
+    }
+}
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoField.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoField.java
new file mode 100644
index 00000000000..e830813e3a3
--- /dev/null
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoField.java
@@ -0,0 +1,53 @@
+package org.elasticsearch.script.expression;
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.lucene.queries.function.ValueSource;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+
+/**
+ * Expressions API for geo_point fields.
+ */
+final class GeoField {
+    // no instance
+    private GeoField() {}
+    
+    // supported variables
+    static final String EMPTY_VARIABLE        = "empty";
+    static final String LAT_VARIABLE          = "lat";
+    static final String LON_VARIABLE          = "lon";
+    
+    static ValueSource getVariable(IndexFieldData<?> fieldData, String fieldName, String variable) {
+        switch (variable) {
+            case EMPTY_VARIABLE:
+                return new GeoEmptyValueSource(fieldData);
+            case LAT_VARIABLE:
+                return new GeoLatitudeValueSource(fieldData);
+            case LON_VARIABLE:
+                return new GeoLongitudeValueSource(fieldData);
+            default:
+                throw new IllegalArgumentException("Member variable [" + variable + "] does not exist for geo field [" + fieldName + "].");
+        }
+    }
+    
+    static ValueSource getMethod(IndexFieldData<?> fieldData, String fieldName, String method) {
+        throw new IllegalArgumentException("Member method [" + method + "] does not exist for geo field [" + fieldName + "].");
+    }
+}
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java
new file mode 100644
index 00000000000..d23eceda2fe
--- /dev/null
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script.expression;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.MultiGeoPointValues;
+
+/**
+ * ValueSource to return latitudes as a double "stream" for geopoint fields
+ */
+final class GeoLatitudeValueSource extends ValueSource {
+    final IndexFieldData<?> fieldData;
+
+    GeoLatitudeValueSource(IndexFieldData<?> fieldData) {
+        this.fieldData = Objects.requireNonNull(fieldData);
+    }
+
+    @Override
+    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
+    public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
+        AtomicGeoPointFieldData leafData = (AtomicGeoPointFieldData) fieldData.load(leaf);
+        final MultiGeoPointValues values = leafData.getGeoPointValues();
+        return new DoubleDocValues(this) {
+            @Override
+            public double doubleVal(int doc) {
+                values.setDocument(doc);
+                if (values.count() == 0) {
+                    return 0.0;
+                } else {
+                    return values.valueAt(0).getLat();
+                }
+            }
+        };
+    }
+
+    @Override
+    public int hashCode() {
+        return 31 * getClass().hashCode() + fieldData.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null) return false;
+        if (getClass() != obj.getClass()) return false;
+        GeoLatitudeValueSource other = (GeoLatitudeValueSource) obj;
+        if (!fieldData.equals(other.fieldData)) return false;
+        return true;
+    }
+
+    @Override
+    public String description() {
+        return "lat: field(" + fieldData.getFieldName() + ")";
+    }
+}
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java
new file mode 100644
index 00000000000..4fbc8fd936c
--- /dev/null
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script.expression;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.MultiGeoPointValues;
+
+/**
+ * ValueSource to return longitudes as a double "stream" for geopoint fields
+ */
+final class GeoLongitudeValueSource extends ValueSource {
+    final IndexFieldData<?> fieldData;
+
+    GeoLongitudeValueSource(IndexFieldData<?> fieldData) {
+        this.fieldData = Objects.requireNonNull(fieldData);
+    }
+
+    @Override
+    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
+    public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
+        AtomicGeoPointFieldData leafData = (AtomicGeoPointFieldData) fieldData.load(leaf);
+        final MultiGeoPointValues values = leafData.getGeoPointValues();
+        return new DoubleDocValues(this) {
+            @Override
+            public double doubleVal(int doc) {
+                values.setDocument(doc);
+                if (values.count() == 0) {
+                    return 0.0;
+                } else {
+                    return values.valueAt(0).getLon();
+                }
+            }
+        };
+    }
+
+    @Override
+    public int hashCode() {
+        return 31 * getClass().hashCode() + fieldData.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null) return false;
+        if (getClass() != obj.getClass()) return false;
+        GeoLongitudeValueSource other = (GeoLongitudeValueSource) obj;
+        if (!fieldData.equals(other.fieldData)) return false;
+        return true;
+    }
+
+    @Override
+    public String description() {
+        return "lon: field(" + fieldData.getFieldName() + ")";
+    }
+}
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/NumericField.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/NumericField.java
new file mode 100644
index 00000000000..4147d0e97d9
--- /dev/null
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/NumericField.java
@@ -0,0 +1,75 @@
+package org.elasticsearch.script.expression;
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.lucene.queries.function.ValueSource;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.search.MultiValueMode;
+
+/**
+ * Expressions API for numeric fields.
+ */
+final class NumericField {
+    // no instance
+    private NumericField() {}
+    
+    // supported variables
+    static final String VALUE_VARIABLE          = "value";
+    static final String EMPTY_VARIABLE          = "empty";
+    
+    // supported methods
+    static final String MINIMUM_METHOD          = "min";
+    static final String MAXIMUM_METHOD          = "max";
+    static final String AVERAGE_METHOD          = "avg";
+    static final String MEDIAN_METHOD           = "median";
+    static final String SUM_METHOD              = "sum";
+    static final String COUNT_METHOD            = "count";
+    
+    static ValueSource getVariable(IndexFieldData<?> fieldData, String fieldName, String variable) {
+        switch (variable) {
+            case VALUE_VARIABLE:
+                return new FieldDataValueSource(fieldData, MultiValueMode.MIN);
+            case EMPTY_VARIABLE:
+                return new EmptyMemberValueSource(fieldData);
+            default:
+                throw new IllegalArgumentException("Member variable [" + variable + "] does not exist for " + 
+                                                   "numeric field [" + fieldName + "].");
+        }
+    }
+    
+    static ValueSource getMethod(IndexFieldData<?> fieldData, String fieldName, String method) {
+        switch (method) {
+            case MINIMUM_METHOD:
+                return new FieldDataValueSource(fieldData, MultiValueMode.MIN);
+            case MAXIMUM_METHOD:
+                return new FieldDataValueSource(fieldData, MultiValueMode.MAX);
+            case AVERAGE_METHOD:
+                return new FieldDataValueSource(fieldData, MultiValueMode.AVG);
+            case MEDIAN_METHOD:
+                return new FieldDataValueSource(fieldData, MultiValueMode.MEDIAN);
+            case SUM_METHOD:
+                return new FieldDataValueSource(fieldData, MultiValueMode.SUM);
+            case COUNT_METHOD:
+                return new CountMethodValueSource(fieldData);
+            default:
+                throw new IllegalArgumentException("Member method [" + method + "] does not exist for numeric field [" + fieldName + "].");
+        }
+    }
+}
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstFunctionValues.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstFunctionValues.java
index b3ebfab66ed..eab9131bf10 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstFunctionValues.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstFunctionValues.java
@@ -25,10 +25,10 @@ import org.apache.lucene.queries.function.FunctionValues;
  * A support class for an executable expression script that allows the double returned
  * by a {@link FunctionValues} to be modified.
  */
-public class ReplaceableConstFunctionValues extends FunctionValues {
+final class ReplaceableConstFunctionValues extends FunctionValues {
     private double value = 0;
 
-    public void setValue(double value) {
+    void setValue(double value) {
         this.value = value;
     }
 
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java
index bb05ef2325d..ae0f862ca70 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java
@@ -29,10 +29,10 @@ import org.apache.lucene.queries.function.ValueSource;
 /**
  * A {@link ValueSource} which has a stub {@link FunctionValues} that holds a dynamically replaceable constant double.
  */
-class ReplaceableConstValueSource extends ValueSource {
+final class ReplaceableConstValueSource extends ValueSource {
     final ReplaceableConstFunctionValues fv;
 
-    public ReplaceableConstValueSource() {
+    ReplaceableConstValueSource() {
         fv = new ReplaceableConstFunctionValues();
     }
 
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
index 50a9900e3d9..dbbddfc1424 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
@@ -27,12 +27,17 @@ import java.util.Map;
 
 import org.apache.lucene.expressions.Expression;
 import org.apache.lucene.expressions.js.JavascriptCompiler;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.action.update.UpdateRequestBuilder;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.lucene.search.function.CombineFunction;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
@@ -51,8 +56,10 @@ import org.elasticsearch.search.aggregations.pipeline.SimpleValue;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
 
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.bucketScript;
@@ -257,8 +264,8 @@ public class MoreExpressionTests extends ESIntegTestCase {
         } catch (SearchPhaseExecutionException e) {
             assertThat(e.toString() + "should have contained IllegalArgumentException",
                     e.toString().contains("IllegalArgumentException"), equalTo(true));
-            assertThat(e.toString() + "should have contained can only be used with a date field type",
-                    e.toString().contains("can only be used with a date field type"), equalTo(true));
+            assertThat(e.toString() + "should have contained does not exist for numeric field",
+                    e.toString().contains("does not exist for numeric field"), equalTo(true));
         }
     }
 
@@ -586,4 +593,37 @@ public class MoreExpressionTests extends ESIntegTestCase {
             }
         }
     }
+    
+    public void testGeo() throws Exception {
+      XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type1")
+              .startObject("properties").startObject("location").field("type", "geo_point");
+      xContentBuilder.endObject().endObject().endObject().endObject();
+      assertAcked(prepareCreate("test").addMapping("type1", xContentBuilder));
+      ensureGreen();
+      client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject()
+          .field("name", "test")
+          .startObject("location").field("lat", 61.5240).field("lon", 105.3188).endObject()
+          .endObject()).execute().actionGet();
+      refresh();
+      // access .lat
+      SearchResponse rsp = buildRequest("doc['location'].lat").get();
+      assertSearchResponse(rsp);
+      assertEquals(1, rsp.getHits().getTotalHits());
+      assertEquals(61.5240, rsp.getHits().getAt(0).field("foo").getValue(), 1.0D);
+      // access .lon
+      rsp = buildRequest("doc['location'].lon").get();
+      assertSearchResponse(rsp);
+      assertEquals(1, rsp.getHits().getTotalHits());
+      assertEquals(105.3188, rsp.getHits().getAt(0).field("foo").getValue(), 1.0D);
+      // access .empty
+      rsp = buildRequest("doc['location'].empty ? 1 : 0").get();
+      assertSearchResponse(rsp);
+      assertEquals(1, rsp.getHits().getTotalHits());
+      assertEquals(0, rsp.getHits().getAt(0).field("foo").getValue(), 1.0D);
+      // call haversin
+      rsp = buildRequest("haversin(38.9072, 77.0369, doc['location'].lat, doc['location'].lon)").get();
+      assertSearchResponse(rsp);
+      assertEquals(1, rsp.getHits().getTotalHits());
+      assertEquals(3170D, rsp.getHits().getAt(0).field("foo").getValue(), 50D);
+  }
 }

From fff82db6816cd9b9dea918ce6c230881a4c2b51b Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 2 May 2016 18:13:03 -0400
Subject: [PATCH 0044/1311] Add tests/doc for boolean fields with expressions

---
 .../modules/scripting/scripting.asciidoc      |  5 +-
 .../expression/MoreExpressionTests.java       | 96 +++++++++++++------
 2 files changed, 69 insertions(+), 32 deletions(-)

diff --git a/docs/reference/modules/scripting/scripting.asciidoc b/docs/reference/modules/scripting/scripting.asciidoc
index b1f779b48a0..fad96978be9 100644
--- a/docs/reference/modules/scripting/scripting.asciidoc
+++ b/docs/reference/modules/scripting/scripting.asciidoc
@@ -492,6 +492,9 @@ You can choose a different value instead, e.g. `doc['myfield'].sum()`.
 
 When a document is missing the field completely, by default the value will be treated as `0`.
 
+Boolean fields are exposed as numerics, with `true` mapped to `1` and `false` mapped to `0`. 
+For example: `doc['on_sale'] ? doc['price'] * 0.5 : doc['price']`
+
 [float]
 === Additional methods for date fields
 Date fields are treated as the number of milliseconds since January 1, 1970 and 
@@ -542,7 +545,7 @@ e.g. based on geolocation of the user.
 
 There are a few limitations relative to other script languages:
 
-* Only numeric, date, and geo_point fields may be accessed
+* Only numeric, boolean, date, and geo_point fields may be accessed
 * Stored fields are not available
 
 [float]
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
index dbbddfc1424..f023ec9f38a 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
@@ -595,35 +595,69 @@ public class MoreExpressionTests extends ESIntegTestCase {
     }
     
     public void testGeo() throws Exception {
-      XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type1")
-              .startObject("properties").startObject("location").field("type", "geo_point");
-      xContentBuilder.endObject().endObject().endObject().endObject();
-      assertAcked(prepareCreate("test").addMapping("type1", xContentBuilder));
-      ensureGreen();
-      client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject()
-          .field("name", "test")
-          .startObject("location").field("lat", 61.5240).field("lon", 105.3188).endObject()
-          .endObject()).execute().actionGet();
-      refresh();
-      // access .lat
-      SearchResponse rsp = buildRequest("doc['location'].lat").get();
-      assertSearchResponse(rsp);
-      assertEquals(1, rsp.getHits().getTotalHits());
-      assertEquals(61.5240, rsp.getHits().getAt(0).field("foo").getValue(), 1.0D);
-      // access .lon
-      rsp = buildRequest("doc['location'].lon").get();
-      assertSearchResponse(rsp);
-      assertEquals(1, rsp.getHits().getTotalHits());
-      assertEquals(105.3188, rsp.getHits().getAt(0).field("foo").getValue(), 1.0D);
-      // access .empty
-      rsp = buildRequest("doc['location'].empty ? 1 : 0").get();
-      assertSearchResponse(rsp);
-      assertEquals(1, rsp.getHits().getTotalHits());
-      assertEquals(0, rsp.getHits().getAt(0).field("foo").getValue(), 1.0D);
-      // call haversin
-      rsp = buildRequest("haversin(38.9072, 77.0369, doc['location'].lat, doc['location'].lon)").get();
-      assertSearchResponse(rsp);
-      assertEquals(1, rsp.getHits().getTotalHits());
-      assertEquals(3170D, rsp.getHits().getAt(0).field("foo").getValue(), 50D);
-  }
+        XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type1")
+                .startObject("properties").startObject("location").field("type", "geo_point");
+        xContentBuilder.endObject().endObject().endObject().endObject();
+        assertAcked(prepareCreate("test").addMapping("type1", xContentBuilder));
+        ensureGreen();
+        client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject()
+                .field("name", "test")
+                .startObject("location").field("lat", 61.5240).field("lon", 105.3188).endObject()
+                .endObject()).execute().actionGet();
+        refresh();
+        // access .lat
+        SearchResponse rsp = buildRequest("doc['location'].lat").get();
+        assertSearchResponse(rsp);
+        assertEquals(1, rsp.getHits().getTotalHits());
+        assertEquals(61.5240, rsp.getHits().getAt(0).field("foo").getValue(), 1.0D);
+        // access .lon
+        rsp = buildRequest("doc['location'].lon").get();
+        assertSearchResponse(rsp);
+        assertEquals(1, rsp.getHits().getTotalHits());
+        assertEquals(105.3188, rsp.getHits().getAt(0).field("foo").getValue(), 1.0D);
+        // access .empty
+        rsp = buildRequest("doc['location'].empty ? 1 : 0").get();
+        assertSearchResponse(rsp);
+        assertEquals(1, rsp.getHits().getTotalHits());
+        assertEquals(0, rsp.getHits().getAt(0).field("foo").getValue(), 1.0D);
+        // call haversin
+        rsp = buildRequest("haversin(38.9072, 77.0369, doc['location'].lat, doc['location'].lon)").get();
+        assertSearchResponse(rsp);
+        assertEquals(1, rsp.getHits().getTotalHits());
+        assertEquals(3170D, rsp.getHits().getAt(0).field("foo").getValue(), 50D);
+    }
+    
+    public void testBoolean() throws Exception {
+        XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type1")
+                .startObject("properties").startObject("vip").field("type", "boolean");
+        xContentBuilder.endObject().endObject().endObject().endObject();
+        assertAcked(prepareCreate("test").addMapping("type1", xContentBuilder));
+        ensureGreen();
+        indexRandom(true,
+                client().prepareIndex("test", "doc", "1").setSource("price", 1.0, "vip", true),
+                client().prepareIndex("test", "doc", "2").setSource("price", 2.0, "vip", false),
+                client().prepareIndex("test", "doc", "3").setSource("price", 2.0, "vip", false));
+        // access .value
+        SearchResponse rsp = buildRequest("doc['vip'].value").get();
+        assertSearchResponse(rsp);
+        assertEquals(3, rsp.getHits().getTotalHits());
+        assertEquals(1.0D, rsp.getHits().getAt(0).field("foo").getValue(), 1.0D);
+        assertEquals(0.0D, rsp.getHits().getAt(1).field("foo").getValue(), 1.0D);
+        assertEquals(0.0D, rsp.getHits().getAt(2).field("foo").getValue(), 1.0D);
+        // access .empty
+        rsp = buildRequest("doc['vip'].empty ? 1 : 0").get();
+        assertSearchResponse(rsp);
+        assertEquals(3, rsp.getHits().getTotalHits());
+        assertEquals(0.0D, rsp.getHits().getAt(0).field("foo").getValue(), 1.0D);
+        assertEquals(0.0D, rsp.getHits().getAt(1).field("foo").getValue(), 1.0D);
+        assertEquals(1.0D, rsp.getHits().getAt(2).field("foo").getValue(), 1.0D);
+        // ternary operator
+        // vip's have a 50% discount
+        rsp = buildRequest("doc['vip'] ? doc['price']/2 : doc['price']").get();
+        assertSearchResponse(rsp);
+        assertEquals(3, rsp.getHits().getTotalHits());
+        assertEquals(0.5D, rsp.getHits().getAt(0).field("foo").getValue(), 1.0D);
+        assertEquals(2.0D, rsp.getHits().getAt(1).field("foo").getValue(), 1.0D);
+        assertEquals(2.0D, rsp.getHits().getAt(2).field("foo").getValue(), 1.0D);
+    }
 }

From 0a6f40c7f5f3af4e04a8875216e059025e1bbdf3 Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@elastic.co>
Date: Tue, 3 May 2016 08:53:15 +0200
Subject: [PATCH 0045/1311] Enable HTTP compression by default with compression
 level 3

With this commit we compress HTTP responses provided the client
supports it (as indicated by the HTTP header 'Accept-Encoding').

We're also able to process compressed HTTP requests if needed.

The default compression level is lowered from 6 to 3 as benchmarks
have indicated that this reduces query latency with a negligible
increase in network traffic.

Closes #7309
---
 .../http/HttpTransportSettings.java           |   6 +-
 .../http/netty/ESHttpContentDecompressor.java |  51 ------
 .../http/netty/NettyHttpServerTransport.java  |   9 +-
 .../http/netty/NettyHttpCompressionIT.java    | 146 ++++++++++++++++++
 docs/reference/migration/migrate_5_0.asciidoc |   3 +
 .../migration/migrate_5_0/http.asciidoc       |   9 ++
 docs/reference/modules/http.asciidoc          |   6 +-
 .../elasticsearch/test/ESIntegTestCase.java   |   8 +-
 8 files changed, 177 insertions(+), 61 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/http/netty/ESHttpContentDecompressor.java
 create mode 100644 core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java
 create mode 100644 docs/reference/migration/migrate_5_0/http.asciidoc

diff --git a/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java b/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java
index 403c543041d..72f8f380df8 100644
--- a/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java
+++ b/core/src/main/java/org/elasticsearch/http/HttpTransportSettings.java
@@ -50,9 +50,11 @@ public final class HttpTransportSettings {
     public static final Setting<Integer> SETTING_PIPELINING_MAX_EVENTS =
         Setting.intSetting("http.pipelining.max_events", 10000, Property.NodeScope);
     public static final Setting<Boolean> SETTING_HTTP_COMPRESSION =
-        Setting.boolSetting("http.compression", false, Property.NodeScope);
+        Setting.boolSetting("http.compression", true, Property.NodeScope);
+    // we intentionally use a different compression level as Netty here as our benchmarks have shown that a compression level of 3 is the
+    // best compromise between reduction in network traffic and added latency. For more details please check #7309.
     public static final Setting<Integer> SETTING_HTTP_COMPRESSION_LEVEL =
-        Setting.intSetting("http.compression_level", 6, Property.NodeScope);
+        Setting.intSetting("http.compression_level", 3, Property.NodeScope);
     public static final Setting<List<String>> SETTING_HTTP_HOST =
         listSetting("http.host", emptyList(), Function.identity(), Property.NodeScope);
     public static final Setting<List<String>> SETTING_HTTP_PUBLISH_HOST =
diff --git a/core/src/main/java/org/elasticsearch/http/netty/ESHttpContentDecompressor.java b/core/src/main/java/org/elasticsearch/http/netty/ESHttpContentDecompressor.java
deleted file mode 100644
index 0d1e46cf06b..00000000000
--- a/core/src/main/java/org/elasticsearch/http/netty/ESHttpContentDecompressor.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.http.netty;
-
-import org.elasticsearch.transport.TransportException;
-import org.jboss.netty.buffer.ChannelBuffer;
-import org.jboss.netty.handler.codec.embedder.DecoderEmbedder;
-import org.jboss.netty.handler.codec.http.HttpContentDecompressor;
-import org.jboss.netty.handler.codec.http.HttpHeaders;
-
-public class ESHttpContentDecompressor extends HttpContentDecompressor {
-    private final boolean compression;
-
-    public ESHttpContentDecompressor(boolean compression) {
-        super();
-        this.compression = compression;
-    }
-
-    @Override
-    protected DecoderEmbedder<ChannelBuffer> newContentDecoder(String contentEncoding) throws Exception {
-        if (compression) {
-            // compression is enabled so handle the request according to the headers (compressed and uncompressed)
-            return super.newContentDecoder(contentEncoding);
-        } else {
-            // if compression is disabled only allow "identity" (uncompressed) requests
-            if (HttpHeaders.Values.IDENTITY.equals(contentEncoding)) {
-                // nothing to handle here
-                return null;
-            } else {
-                throw new TransportException("Support for compressed content is disabled. You can enable it with http.compression=true");
-            }
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java
index e90a3710889..22852b7c0ff 100644
--- a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java
+++ b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java
@@ -70,6 +70,7 @@ import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;
 import org.jboss.netty.channel.socket.oio.OioServerSocketChannelFactory;
 import org.jboss.netty.handler.codec.http.HttpChunkAggregator;
 import org.jboss.netty.handler.codec.http.HttpContentCompressor;
+import org.jboss.netty.handler.codec.http.HttpContentDecompressor;
 import org.jboss.netty.handler.codec.http.HttpMethod;
 import org.jboss.netty.handler.codec.http.HttpRequestDecoder;
 import org.jboss.netty.handler.timeout.ReadTimeoutException;
@@ -544,19 +545,19 @@ public class NettyHttpServerTransport extends AbstractLifecycleComponent<HttpSer
                 requestDecoder.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents);
             }
             pipeline.addLast("decoder", requestDecoder);
-            pipeline.addLast("decoder_compress", new ESHttpContentDecompressor(transport.compression));
+            pipeline.addLast("decoder_compress", new HttpContentDecompressor());
             HttpChunkAggregator httpChunkAggregator = new HttpChunkAggregator((int) transport.maxContentLength.bytes());
             if (transport.maxCompositeBufferComponents != -1) {
                 httpChunkAggregator.setMaxCumulationBufferComponents(transport.maxCompositeBufferComponents);
             }
             pipeline.addLast("aggregator", httpChunkAggregator);
-            if (SETTING_CORS_ENABLED.get(transport.settings())) {
-                pipeline.addLast("cors", new CorsHandler(transport.getCorsConfig()));
-            }
             pipeline.addLast("encoder", new ESHttpResponseEncoder());
             if (transport.compression) {
                 pipeline.addLast("encoder_compress", new HttpContentCompressor(transport.compressionLevel));
             }
+            if (SETTING_CORS_ENABLED.get(transport.settings())) {
+                pipeline.addLast("cors", new CorsHandler(transport.getCorsConfig()));
+            }
             if (transport.pipelining) {
                 pipeline.addLast("pipelining", new HttpPipeliningHandler(transport.pipeliningMaxEvents));
             }
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java
new file mode 100644
index 00000000000..bb7a78ab43f
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.http.netty;
+
+import org.apache.http.Header;
+import org.apache.http.HttpException;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpResponseInterceptor;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.protocol.HttpContext;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.http.HttpTransportSettings;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.rest.client.http.HttpResponse;
+
+import java.io.IOException;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 1, numClientNodes = 1)
+public class NettyHttpCompressionIT extends ESIntegTestCase {
+    private static final String GZIP_ENCODING = "gzip";
+
+    private static final String SAMPLE_DOCUMENT = "{\n" +
+        "   \"name\": {\n" +
+        "      \"first name\": \"Steve\",\n" +
+        "      \"last name\": \"Jobs\"\n" +
+        "   }\n" +
+        "}";
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal))
+            .put(NetworkModule.HTTP_ENABLED.getKey(), true)
+            .put(HttpTransportSettings.SETTING_HTTP_COMPRESSION.getKey(), true)
+            .build();
+    }
+
+    public void testCompressesResponseIfRequested() throws Exception {
+        ensureGreen();
+
+        // we need to intercept early, otherwise internal logic in HttpClient will just remove the header and we cannot verify it
+        ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor();
+        CloseableHttpClient internalClient = HttpClients.custom().addInterceptorFirst(headerExtractor).build();
+
+        HttpResponse response = httpClient(internalClient).path("/").addHeader(HttpHeaders.ACCEPT_ENCODING, GZIP_ENCODING).execute();
+        assertEquals(200, response.getStatusCode());
+        assertTrue(headerExtractor.hasContentEncodingHeader());
+        assertEquals(GZIP_ENCODING, headerExtractor.getContentEncodingHeader().getValue());
+    }
+
+    public void testUncompressedResponseByDefault() throws Exception {
+        ensureGreen();
+
+        ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor();
+        CloseableHttpClient internalClient = HttpClients
+            .custom()
+            .disableContentCompression()
+            .addInterceptorFirst(headerExtractor)
+            .build();
+
+        HttpResponse response = httpClient(internalClient).path("/").execute();
+        assertEquals(200, response.getStatusCode());
+        assertFalse(headerExtractor.hasContentEncodingHeader());
+    }
+
+    public void testCanInterpretUncompressedRequest() throws Exception {
+        ensureGreen();
+
+        ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor();
+        CloseableHttpClient internalClient = HttpClients
+            .custom()
+            // this disable content compression in both directions (request and response)
+            .disableContentCompression()
+            .addInterceptorFirst(headerExtractor)
+            .build();
+
+        HttpResponse response = httpClient(internalClient)
+            .path("/company/employees/1")
+            .method("POST")
+            .body(SAMPLE_DOCUMENT)
+            .execute();
+
+        assertEquals(201, response.getStatusCode());
+        assertFalse(headerExtractor.hasContentEncodingHeader());
+    }
+
+    public void testCanInterpretCompressedRequest() throws Exception {
+        ensureGreen();
+
+        ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor();
+        // we don't call #disableContentCompression() hence the client will send the content compressed
+        CloseableHttpClient internalClient = HttpClients.custom().addInterceptorFirst(headerExtractor).build();
+
+        HttpResponse response = httpClient(internalClient)
+            .path("/company/employees/2")
+            .method("POST")
+            .body(SAMPLE_DOCUMENT)
+            .execute();
+
+        assertEquals(201, response.getStatusCode());
+        assertTrue(headerExtractor.hasContentEncodingHeader());
+        assertEquals(GZIP_ENCODING, headerExtractor.getContentEncodingHeader().getValue());
+    }
+
+    private static class ContentEncodingHeaderExtractor implements HttpResponseInterceptor {
+        private Header contentEncodingHeader;
+
+        @Override
+        public void process(org.apache.http.HttpResponse response, HttpContext context) throws HttpException, IOException {
+            final Header[] headers = response.getHeaders(HttpHeaders.CONTENT_ENCODING);
+            if (headers.length == 1) {
+                this.contentEncodingHeader = headers[0];
+            } else if (headers.length > 1) {
+                throw new AssertionError("Expected none or one content encoding header but got " + headers.length + " headers.");
+            }
+        }
+
+        public boolean hasContentEncodingHeader() {
+            return contentEncodingHeader != null;
+        }
+
+        public Header getContentEncodingHeader() {
+            return contentEncodingHeader;
+        }
+    }
+
+
+}
diff --git a/docs/reference/migration/migrate_5_0.asciidoc b/docs/reference/migration/migrate_5_0.asciidoc
index 5f2d30e290e..e2dae648485 100644
--- a/docs/reference/migration/migrate_5_0.asciidoc
+++ b/docs/reference/migration/migrate_5_0.asciidoc
@@ -36,6 +36,7 @@ way to do this is to upgrade to Elasticsearch 2.3 or later and to use the
 * <<breaking_50_index_apis>>
 * <<breaking_50_settings_changes>>
 * <<breaking_50_allocation>>
+* <<breaking_50_http_changes>>
 * <<breaking_50_rest_api_changes>>
 * <<breaking_50_cat_api>>
 * <<breaking_50_java_api_changes>>
@@ -60,6 +61,8 @@ include::migrate_5_0/settings.asciidoc[]
 
 include::migrate_5_0/allocation.asciidoc[]
 
+include::migrate_5_0/http.asciidoc[]
+
 include::migrate_5_0/rest.asciidoc[]
 
 include::migrate_5_0/cat.asciidoc[]
diff --git a/docs/reference/migration/migrate_5_0/http.asciidoc b/docs/reference/migration/migrate_5_0/http.asciidoc
new file mode 100644
index 00000000000..68a3f2841f0
--- /dev/null
+++ b/docs/reference/migration/migrate_5_0/http.asciidoc
@@ -0,0 +1,9 @@
+[[breaking_50_http_changes]]
+=== HTTP changes
+
+==== Compressed HTTP requests are always accepted
+
+Before 5.0, Elasticsearch accepted compressed HTTP requests only if the setting
+ `http.compressed` was set to `true`. Elasticsearch accepts compressed requests
+ now but will continue to send compressed responses only if `http.compressed`
+ is set to `true`.
\ No newline at end of file
diff --git a/docs/reference/modules/http.asciidoc b/docs/reference/modules/http.asciidoc
index cf3780e53fd..24f11f0c378 100644
--- a/docs/reference/modules/http.asciidoc
+++ b/docs/reference/modules/http.asciidoc
@@ -48,10 +48,10 @@ to `4kb`
 
 
 |`http.compression` |Support for compression when possible (with
-Accept-Encoding). Defaults to `false`.
+Accept-Encoding). Defaults to `true`.
 
-|`http.compression_level` |Defines the compression level to use.
-Defaults to `6`.
+|`http.compression_level` |Defines the compression level to use for HTTP responses. Valid values are in the range of 1 (minimum compression)
+and 9 (maximum compression). Defaults to `3`.
 
 |`http.cors.enabled` |Enable or disable cross-origin resource sharing,
 i.e. whether a browser on another origin can do requests to
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index d347f001e79..a31ef76272e 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -23,6 +23,7 @@ import com.carrotsearch.randomizedtesting.RandomizedTest;
 import com.carrotsearch.randomizedtesting.annotations.TestGroup;
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
@@ -2040,15 +2041,20 @@ public abstract class ESIntegTestCase extends ESTestCase {
     }
 
     protected HttpRequestBuilder httpClient() {
+        return httpClient(HttpClients.createDefault());
+    }
+
+    protected HttpRequestBuilder httpClient(CloseableHttpClient httpClient) {
         final NodesInfoResponse nodeInfos = client().admin().cluster().prepareNodesInfo().get();
         final NodeInfo[] nodes = nodeInfos.getNodes();
         assertTrue(nodes.length > 0);
         TransportAddress publishAddress = randomFrom(nodes).getHttp().address().publishAddress();
         assertEquals(1, publishAddress.uniqueAddressTypeId());
         InetSocketAddress address = ((InetSocketTransportAddress) publishAddress).address();
-        return new HttpRequestBuilder(HttpClients.createDefault()).host(NetworkAddress.format(address.getAddress())).port(address.getPort());
+        return new HttpRequestBuilder(httpClient).host(NetworkAddress.format(address.getAddress())).port(address.getPort());
     }
 
+
     /**
      * This method is executed iff the test is annotated with {@link SuiteScopeTestCase}
      * before the first test of this class is executed.

From 98dba262cf3e81d405c8a372983942d59f993557 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Tue, 3 May 2016 11:31:56 +0200
Subject: [PATCH 0046/1311] Use object equality to compare versions in
 IndexSettings (#18103)

Fixes an issue where updating index metadata on a index from a version that it does not have in its static list of known versions fails.
---
 .../main/java/org/elasticsearch/index/IndexSettings.java | 2 +-
 .../java/org/elasticsearch/index/IndexSettingsTests.java | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/index/IndexSettings.java b/core/src/main/java/org/elasticsearch/index/IndexSettings.java
index 31e10e4d7ec..7c8cb4ff8c8 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexSettings.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexSettings.java
@@ -382,7 +382,7 @@ public final class IndexSettings {
      */
     synchronized boolean updateIndexMetaData(IndexMetaData indexMetaData) {
         final Settings newSettings = indexMetaData.getSettings();
-        if (Version.indexCreated(newSettings) != version) {
+        if (version.equals(Version.indexCreated(newSettings)) == false) {
             throw new IllegalArgumentException("version mismatch on settings update expected: " + version + " but was: " + Version.indexCreated(newSettings));
         }
         final String newUUID = newSettings.get(IndexMetaData.SETTING_INDEX_UUID, IndexMetaData.INDEX_UUID_NA_VALUE);
diff --git a/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
index 07962153d13..9c5040589ae 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
@@ -110,6 +110,15 @@ public class IndexSettingsTests extends ESTestCase {
             assertTrue(ex.getMessage(), ex.getMessage().startsWith("version mismatch on settings update expected: "));
         }
 
+        // use version number that is unknown
+        metaData = newIndexMeta("index", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.fromId(999999))
+            .build());
+        settings = new IndexSettings(metaData, Settings.EMPTY);
+        assertEquals(Version.fromId(999999), settings.getIndexVersionCreated());
+        assertEquals("_na_", settings.getUUID());
+        settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED,
+            Version.fromId(999999)).put("index.test.setting.int", 42).build()));
+
         metaData = newIndexMeta("index", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
             .put(IndexMetaData.SETTING_INDEX_UUID, "0xdeadbeef").build());
         settings = new IndexSettings(metaData, Settings.EMPTY);

From 1385e25542c78f9122f43727743e54a3e2a545a3 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 3 May 2016 09:20:19 -0400
Subject: [PATCH 0047/1311] Add note on configuring assertions in IDEs

This commit adds a note to the contributing docs on how to configure
assertions inside Eclipse and IntelliJ.

Relates #18107
---
 CONTRIBUTING.md | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 9afcd34fad7..9f7264acdae 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -76,7 +76,31 @@ Contributing to the Elasticsearch codebase
 
 **Repository:** [https://github.com/elastic/elasticsearch](https://github.com/elastic/elasticsearch)
 
-Make sure you have [Gradle](http://gradle.org) installed, as Elasticsearch uses it as its build system. Integration with IntelliJ and Eclipse should work out of the box. Eclipse users can automatically configure their IDE: `gradle eclipse` then `File: Import: Existing Projects into Workspace`. Select the option `Search for nested projects`. Additionally you will want to ensure that Eclipse is using 2048m of heap by modifying `eclipse.ini` accordingly to avoid GC overhead errors.
+Make sure you have [Gradle](http://gradle.org) installed, as
+Elasticsearch uses it as its build system.
+
+Eclipse users can automatically configure their IDE: `gradle eclipse`
+then `File: Import: Existing Projects into Workspace`. Select the
+option `Search for nested projects`. Additionally you will want to
+ensure that Eclipse is using 2048m of heap by modifying `eclipse.ini`
+accordingly to avoid GC overhead errors.
+
+IntelliJ users acn automatically configure their IDE: `gradle idea`
+then `File->New Project From Existing Sources`. Point to the root of
+the source directory, select
+`Import project from external model->Gradle`, enable
+`Use auto-import`.
+
+The Elasticsearch codebase makes heavy use of Java `assert`s and the
+test runner requires that assertions be enabled within the JVM. This
+can be accomplished by passing the flag `-ea` to the JVM on startup.
+
+For IntelliJ, go to
+`Run->Edit Configurations...->Defaults->JUnit->VM options` and input
+`-ea`.
+
+For Eclipse, go to `Preferences->Java->Installed JREs` and add `-ea` to
+`VM Arguments`.
 
 Please follow these formatting guidelines:
 

From d01526b6aba778c8c232460ab2840a3e18ff972c Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Tue, 3 May 2016 15:35:33 +0200
Subject: [PATCH 0048/1311] Bumped docs version

---
 docs/reference/index.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc
index 9b99b56997a..8812a2bed95 100644
--- a/docs/reference/index.asciidoc
+++ b/docs/reference/index.asciidoc
@@ -1,7 +1,7 @@
 [[elasticsearch-reference]]
 = Elasticsearch Reference
 
-:version:       5.0.0-alpha1
+:version:       5.0.0-alpha2
 :major-version: 5.x
 :branch:        master
 :jdk:           1.8.0_73

From 85d2fc0e38c873bdd4092fcf64c0aef3568cfc24 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 3 May 2016 10:05:44 -0400
Subject: [PATCH 0049/1311] Avoid string concatentation in
 IngestDocument.FieldPath

Today, the constructor for IngestDocument#FieldPath does a string
concatentation and two object allocation on every field path. This
commit removes these unnecessary operations.

Relates #18108
---
 .../elasticsearch/ingest/core/IngestDocument.java    | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/ingest/core/IngestDocument.java b/core/src/main/java/org/elasticsearch/ingest/core/IngestDocument.java
index 7b7199b657a..05682991713 100644
--- a/core/src/main/java/org/elasticsearch/ingest/core/IngestDocument.java
+++ b/core/src/main/java/org/elasticsearch/ingest/core/IngestDocument.java
@@ -51,6 +51,8 @@ import static java.nio.charset.StandardCharsets.UTF_8;
 public final class IngestDocument {
 
     public final static String INGEST_KEY = "_ingest";
+    private static final String INGEST_KEY_PREFIX = INGEST_KEY + ".";
+    private static final String SOURCE_PREFIX = SourceFieldMapper.NAME + ".";
 
     static final String TIMESTAMP = "timestamp";
 
@@ -600,6 +602,7 @@ public final class IngestDocument {
     }
 
     private class FieldPath {
+
         private final String[] pathElements;
         private final Object initialContext;
 
@@ -608,13 +611,13 @@ public final class IngestDocument {
                 throw new IllegalArgumentException("path cannot be null nor empty");
             }
             String newPath;
-            if (path.startsWith(INGEST_KEY + ".")) {
+            if (path.startsWith(INGEST_KEY_PREFIX)) {
                 initialContext = ingestMetadata;
-                newPath = path.substring(8, path.length());
+                newPath = path.substring(INGEST_KEY_PREFIX.length(), path.length());
             } else {
                 initialContext = sourceAndMetadata;
-                if (path.startsWith(SourceFieldMapper.NAME + ".")) {
-                    newPath = path.substring(8, path.length());
+                if (path.startsWith(SOURCE_PREFIX)) {
+                    newPath = path.substring(SOURCE_PREFIX.length(), path.length());
                 } else {
                     newPath = path;
                 }
@@ -624,5 +627,6 @@ public final class IngestDocument {
                 throw new IllegalArgumentException("path [" + path + "] is not valid");
             }
         }
+
     }
 }

From 3ca02d647eafcd875802109fcbd009e80d37e5bb Mon Sep 17 00:00:00 2001
From: Chris Earle <pickypg@users.noreply.github.com>
Date: Tue, 3 May 2016 11:37:34 -0400
Subject: [PATCH 0050/1311] Removing old documentation mentioning NodeBuilder.

---
 docs/groovy-api/client.asciidoc | 59 ---------------------------------
 1 file changed, 59 deletions(-)
 delete mode 100644 docs/groovy-api/client.asciidoc

diff --git a/docs/groovy-api/client.asciidoc b/docs/groovy-api/client.asciidoc
deleted file mode 100644
index c0a6d688415..00000000000
--- a/docs/groovy-api/client.asciidoc
+++ /dev/null
@@ -1,59 +0,0 @@
-[[client]]
-== Client
-
-Obtaining an elasticsearch Groovy `GClient` (a `GClient` is a simple
-wrapper on top of the Java `Client`) is simple. The most common way to
-get a client is by starting an embedded `Node` which acts as a node
-within the cluster.
-
-
-[[node-client]]
-=== Node Client
-
-A Node based client is the simplest form to get a `GClient` to start
-executing operations against elasticsearch.
-
-[source,js]
---------------------------------------------------
-import org.elasticsearch.groovy.client.GClient
-import org.elasticsearch.groovy.node.GNode
-import static org.elasticsearch.groovy.node.GNodeBuilder.nodeBuilder
-
-// on startup
-
-GNode node = nodeBuilder().node();
-GClient client = node.client();
-
-// on shutdown
-
-node.close();
---------------------------------------------------
-
-Since elasticsearch allows to configure it using JSON based settings,
-the configuration itself can be done using a closure that represent the
-JSON:
-
-[source,js]
---------------------------------------------------
-import org.elasticsearch.groovy.node.GNode
-import org.elasticsearch.groovy.node.GNodeBuilder
-import static org.elasticsearch.groovy.node.GNodeBuilder.*
-
-// on startup
-
-GNodeBuilder nodeBuilder = nodeBuilder();
-nodeBuilder.settings {
-    node {
-        client = true
-    }
-    cluster {
-        name = "test"
-    }
-}
-
-GNode node = nodeBuilder.node()
-
-// on shutdown
-
-node.stop().close()
---------------------------------------------------

From 5648253d452e2b2d07ee04574c2521440ff7d973 Mon Sep 17 00:00:00 2001
From: Lee Hinman <lee@writequit.org>
Date: Tue, 3 May 2016 09:51:15 -0600
Subject: [PATCH 0051/1311] Add documentation for shard store output in
 allocation explain API

Relates to #17689
---
 .../cluster/allocation-explain.asciidoc       | 54 ++++++++++++++-----
 1 file changed, 42 insertions(+), 12 deletions(-)

diff --git a/docs/reference/cluster/allocation-explain.asciidoc b/docs/reference/cluster/allocation-explain.asciidoc
index dcca80dd61f..7a8a3e4a8ce 100644
--- a/docs/reference/cluster/allocation-explain.asciidoc
+++ b/docs/reference/cluster/allocation-explain.asciidoc
@@ -34,15 +34,21 @@ The response looks like:
     "reason" : "INDEX_CREATED",                   <2>
     "at" : "2016-03-22T20:04:23.620Z"
   },
-  "nodes" : {                                     <3>
+  "allocation_delay_ms" : 0,                      <3>
+  "remaining_delay_ms" : 0,                       <4>
+  "nodes" : {
     "V-Spi0AyRZ6ZvKbaI3691w" : {
       "node_name" : "node1",
-      "node_attributes" : {                       <4>
+      "node_attributes" : {                       <5>
         "bar" : "baz"
       },
-      "final_decision" : "NO",                    <5>
-      "weight" : 0.06666675,                      <6>
-      "decisions" : [ {                           <7>
+      "store" : {
+        "shard_copy" : "NONE"                     <6>
+      },
+      "final_decision" : "NO",                    <7>
+      "final_explanation" : "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
+      "weight" : 0.06666675,                      <8>
+      "decisions" : [ {                           <9>
         "decider" : "filter",
         "decision" : "NO",
         "explanation" : "node does not match index include filters [foo:\"bar\"]"
@@ -54,7 +60,11 @@ The response looks like:
         "bar" : "baz",
         "foo" : "bar"
       },
+      "store" : {
+        "shard_copy" : "AVAILABLE"
+      },
       "final_decision" : "NO",
+      "final_explanation" : "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
       "weight" : -1.3833332,
       "decisions" : [ {
         "decider" : "same_shard",
@@ -65,7 +75,11 @@ The response looks like:
     "PzdyMZGXQdGhqTJHF_hGgA" : {
       "node_name" : "node3",
       "node_attributes" : { },
+      "store" : {
+        "shard_copy" : "NONE"
+      },
       "final_decision" : "NO",
+      "final_explanation" : "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
       "weight" : 2.3166666,
       "decisions" : [ {
         "decider" : "filter",
@@ -78,11 +92,13 @@ The response looks like:
 --------------------------------------------------
 <1> Whether the shard is assigned or unassigned
 <2> Reason for the shard originally becoming unassigned
-<3> List of node decisions about the shard
-<4> User-added attributes the node has
-<5> Final decision for whether the shard is allowed to be allocated to this node
-<6> Weight for how much the allocator would like to allocate the shard to this node
-<7> List of decisions factoring into final decision
+<3> Configured delay before the shard can be allocated
+<4> Remaining delay before the shard can be allocated
+<5> User-added attributes the node has
+<6> The shard copy information for this node and error (if applicable)
+<7> Final decision and explanation of whether the shard can be allocated to this node
+<8> Weight for how much the allocator would like to allocate the shard to this node
+<9> List of node decisions factoring into final decision about the shard
 
 For a shard that is already assigned, the output looks similar to:
 
@@ -97,13 +113,19 @@ For a shard that is already assigned, the output looks similar to:
   },
   "assigned" : true,
   "assigned_node_id" : "Qc6VL8c5RWaw1qXZ0Rg57g",      <1>
+  "allocation_delay_ms" : 0,
+  "remaining_delay_ms" : 0,
   "nodes" : {
     "V-Spi0AyRZ6ZvKbaI3691w" : {
       "node_name" : "Susan Storm",
       "node_attributes" : {
         "bar" : "baz"
       },
+      "store" : {
+        "shard_copy" : "NONE"
+      },
       "final_decision" : "NO",
+      "final_explanation" : "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
       "weight" : 1.4499999,
       "decisions" : [ {
         "decider" : "filter",
@@ -117,7 +139,11 @@ For a shard that is already assigned, the output looks similar to:
         "bar" : "baz",
         "foo" : "bar"
       },
-      "final_decision" : "CURRENTLY_ASSIGNED",        <2>
+      "store" : {
+        "shard_copy" : "AVAILABLE"
+      },
+      "final_decision" : "ALREADY_ASSIGNED",        <2>
+      "final_explanation" : "the shard is already assigned to this node",
       "weight" : 0.0,
       "decisions" : [ {
         "decider" : "same_shard",
@@ -128,7 +154,11 @@ For a shard that is already assigned, the output looks similar to:
     "PzdyMZGXQdGhqTJHF_hGgA" : {
       "node_name" : "The Symbiote",
       "node_attributes" : { },
+      "store" : {
+        "shard_copy" : "NONE"
+      },
       "final_decision" : "NO",
+      "final_explanation" : "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
       "weight" : 3.6999998,
       "decisions" : [ {
         "decider" : "filter",
@@ -140,7 +170,7 @@ For a shard that is already assigned, the output looks similar to:
 }
 --------------------------------------------------
 <1> Node the shard is currently assigned to
-<2> The decision is "CURRENTLY_ASSIGNED" because the shard is currently assigned to this node
+<2> The decision is "ALREADY_ASSIGNED" because the shard is currently assigned to this node
 
 You can also have Elasticsearch explain the allocation of the first unassigned
 shard it finds by sending an empty body, such as:

From 21e94e6054354f59fd1bdca400bb8a2229aecbea Mon Sep 17 00:00:00 2001
From: Chris Earle <chris.earle@elastic.co>
Date: Tue, 3 May 2016 12:03:00 -0400
Subject: [PATCH 0052/1311] Revert "Removing old documentation mentioning
 NodeBuilder."

This reverts commit 3ca02d647eafcd875802109fcbd009e80d37e5bb.
---
 docs/groovy-api/client.asciidoc | 59 +++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 docs/groovy-api/client.asciidoc

diff --git a/docs/groovy-api/client.asciidoc b/docs/groovy-api/client.asciidoc
new file mode 100644
index 00000000000..c0a6d688415
--- /dev/null
+++ b/docs/groovy-api/client.asciidoc
@@ -0,0 +1,59 @@
+[[client]]
+== Client
+
+Obtaining an elasticsearch Groovy `GClient` (a `GClient` is a simple
+wrapper on top of the Java `Client`) is simple. The most common way to
+get a client is by starting an embedded `Node` which acts as a node
+within the cluster.
+
+
+[[node-client]]
+=== Node Client
+
+A Node based client is the simplest form to get a `GClient` to start
+executing operations against elasticsearch.
+
+[source,js]
+--------------------------------------------------
+import org.elasticsearch.groovy.client.GClient
+import org.elasticsearch.groovy.node.GNode
+import static org.elasticsearch.groovy.node.GNodeBuilder.nodeBuilder
+
+// on startup
+
+GNode node = nodeBuilder().node();
+GClient client = node.client();
+
+// on shutdown
+
+node.close();
+--------------------------------------------------
+
+Since elasticsearch allows to configure it using JSON based settings,
+the configuration itself can be done using a closure that represent the
+JSON:
+
+[source,js]
+--------------------------------------------------
+import org.elasticsearch.groovy.node.GNode
+import org.elasticsearch.groovy.node.GNodeBuilder
+import static org.elasticsearch.groovy.node.GNodeBuilder.*
+
+// on startup
+
+GNodeBuilder nodeBuilder = nodeBuilder();
+nodeBuilder.settings {
+    node {
+        client = true
+    }
+    cluster {
+        name = "test"
+    }
+}
+
+GNode node = nodeBuilder.node()
+
+// on shutdown
+
+node.stop().close()
+--------------------------------------------------

From ca21aa0cb592decaf8a5f69b49dd778081f8456e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Tue, 3 May 2016 17:17:43 +0200
Subject: [PATCH 0053/1311] Make reset() in QueryShardContext private

The query shard reset() method resets some internal state in the
query shard context, like clearing query names, the filter flag
or named queries. The problem with this method being public is
that it currently (miss?) used for modifying an existing context
for recursive invocatiob, but the contexts that have been reseted
that way cannot be properly set back to their previous state.

This PR is a step towards removing reset() entirely by first making
it only be used internally in QueryShardContext. In places where
reset() was used we can either create new QueryShardContexts or
modify the existing context because it is discarded afterwards anyway.
---
 .../cluster/metadata/AliasValidator.java       | 14 ++++----------
 .../percolator/PercolatorFieldMapper.java      |  7 +------
 .../index/query/AbstractQueryBuilder.java      |  6 +++---
 .../index/query/QueryShardContext.java         |  9 ++++++---
 .../messy/tests/TemplateQueryParserTests.java  | 18 ++++++++++--------
 5 files changed, 24 insertions(+), 30 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java
index a3d776a4edb..64da04e6511 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java
@@ -139,15 +139,9 @@ public class AliasValidator extends AbstractComponent {
         }
     }
 
-    private void validateAliasFilter(XContentParser parser, QueryShardContext queryShardContext) throws IOException {
-        try {
-            queryShardContext.reset();
-            QueryParseContext queryParseContext = queryShardContext.newParseContext(parser);
-            QueryBuilder<?> queryBuilder = QueryBuilder.rewriteQuery(queryParseContext.parseInnerQueryBuilder(), queryShardContext);
-            queryBuilder.toFilter(queryShardContext);
-        } finally {
-            queryShardContext.reset();
-            parser.close();
-        }
+    private static void validateAliasFilter(XContentParser parser, QueryShardContext queryShardContext) throws IOException {
+        QueryParseContext queryParseContext = queryShardContext.newParseContext(parser);
+        QueryBuilder<?> queryBuilder = QueryBuilder.rewriteQuery(queryParseContext.parseInnerQueryBuilder(), queryShardContext);
+        queryBuilder.toFilter(queryShardContext);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
index 0a993c63a35..6637dd8b762 100644
--- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
@@ -207,7 +207,6 @@ public class PercolatorFieldMapper extends FieldMapper {
     }
 
     static Query toQuery(QueryShardContext context, boolean mapUnmappedFieldsAsString, QueryBuilder<?> queryBuilder) throws IOException {
-        context.reset();
         // This means that fields in the query need to exist in the mapping prior to registering this query
         // The reason that this is required, is that if a field doesn't exist then the query assumes defaults, which may be undesired.
         //
@@ -222,11 +221,7 @@ public class PercolatorFieldMapper extends FieldMapper {
         // as an analyzed string.
         context.setAllowUnmappedFields(false);
         context.setMapUnmappedFieldAsString(mapUnmappedFieldsAsString);
-        try {
-            return queryBuilder.toQuery(context);
-        } finally {
-            context.reset();
-        }
+        return queryBuilder.toQuery(context);
     }
 
     static QueryBuilder<?> parseQueryBuilder(QueryParseContext context, XContentLocation location) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java
index 635daab4142..ab04c1aff44 100644
--- a/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java
@@ -109,12 +109,12 @@ public abstract class AbstractQueryBuilder<QB extends AbstractQueryBuilder<QB>>
     @Override
     public final Query toFilter(QueryShardContext context) throws IOException {
         Query result = null;
-            final boolean originalIsFilter = context.isFilter;
+            final boolean originalIsFilter = context.isFilter();
             try {
-                context.isFilter = true;
+                context.setIsFilter(true);
                 result = toQuery(context);
             } finally {
-                context.isFilter = originalIsFilter;
+                context.setIsFilter(originalIsFilter);
             }
         return result;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
index 2c0b90f059c..af87a986243 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
@@ -91,7 +91,7 @@ public class QueryShardContext extends QueryRewriteContext {
     private boolean allowUnmappedFields;
     private boolean mapUnmappedFieldAsString;
     private NestedScope nestedScope;
-    boolean isFilter; // pkg private for testing
+    private boolean isFilter;
 
     public QueryShardContext(IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache, IndexFieldDataService indexFieldDataService,
                              MapperService mapperService, SimilarityService similarityService, ScriptService scriptService,
@@ -116,7 +116,7 @@ public class QueryShardContext extends QueryRewriteContext {
         this.types = source.getTypes();
     }
 
-    public void reset() {
+    private void reset() {
         allowUnmappedFields = indexSettings.isDefaultAllowUnmappedFields();
         this.lookup = null;
         this.namedQueries.clear();
@@ -183,6 +183,10 @@ public class QueryShardContext extends QueryRewriteContext {
         return isFilter;
     }
 
+    void setIsFilter(boolean isFilter) {
+        this.isFilter = isFilter;
+    }
+
     public Collection<String> simpleMatchToIndexNames(String pattern) {
         return mapperService.simpleMatchToIndexNames(pattern);
     }
@@ -369,5 +373,4 @@ public class QueryShardContext extends QueryRewriteContext {
     public final Index index() {
         return indexSettings.getIndex();
     }
-
 }
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
index 0815e27ce76..73a63088bf6 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
@@ -74,6 +74,7 @@ import org.junit.Before;
 import java.io.IOException;
 import java.lang.reflect.Proxy;
 import java.util.Collections;
+import java.util.function.Supplier;
 
 import static org.hamcrest.Matchers.containsString;
 
@@ -84,7 +85,7 @@ import static org.hamcrest.Matchers.containsString;
 public class TemplateQueryParserTests extends ESTestCase {
 
     private Injector injector;
-    private QueryShardContext context;
+    private Supplier<QueryShardContext> contextFactory;
 
     @Before
     public void setup() throws IOException {
@@ -134,7 +135,8 @@ public class TemplateQueryParserTests extends ESTestCase {
         ScriptService scriptService = injector.getInstance(ScriptService.class);
         SimilarityService similarityService = new SimilarityService(idxSettings, Collections.emptyMap());
         MapperRegistry mapperRegistry = new IndicesModule().getMapperRegistry();
-        MapperService mapperService = new MapperService(idxSettings, analysisService, similarityService, mapperRegistry, () -> context);
+        MapperService mapperService = new MapperService(idxSettings, analysisService, similarityService, mapperRegistry, () ->
+            contextFactory.get());
         IndicesFieldDataCache cache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() {});
         IndexFieldDataService indexFieldDataService =new IndexFieldDataService(idxSettings, cache, injector.getInstance(CircuitBreakerService.class), mapperService);
         BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, new BitsetFilterCache.Listener() {
@@ -149,7 +151,7 @@ public class TemplateQueryParserTests extends ESTestCase {
             }
         });
         IndicesQueriesRegistry indicesQueriesRegistry = injector.getInstance(IndicesQueriesRegistry.class);
-        context = new QueryShardContext(idxSettings, bitsetFilterCache, indexFieldDataService, mapperService,
+        contextFactory =  () -> new QueryShardContext(idxSettings, bitsetFilterCache, indexFieldDataService, mapperService,
                 similarityService, scriptService, indicesQueriesRegistry, proxy, null, null, null);
     }
 
@@ -164,7 +166,7 @@ public class TemplateQueryParserTests extends ESTestCase {
         String templateString = "{" + "\"query\":{\"match_{{template}}\": {}}," + "\"params\":{\"template\":\"all\"}" + "}";
 
         XContentParser templateSourceParser = XContentFactory.xContent(templateString).createParser(templateString);
-        context.reset();
+        QueryShardContext context = contextFactory.get();
         templateSourceParser.nextToken();
 
         Query query = QueryBuilder.rewriteQuery(TemplateQueryBuilder.fromXContent(context.newParseContext(templateSourceParser)),
@@ -176,7 +178,7 @@ public class TemplateQueryParserTests extends ESTestCase {
         String templateString = "{" + "  \"inline\" : \"{ \\\"match_{{#use_it}}{{template}}{{/use_it}}\\\":{} }\"," + "  \"params\":{"
                 + "    \"template\":\"all\"," + "    \"use_it\": true" + "  }" + "}";
         XContentParser templateSourceParser = XContentFactory.xContent(templateString).createParser(templateString);
-        context.reset();
+        QueryShardContext context = contextFactory.get();
 
         Query query = QueryBuilder.rewriteQuery(TemplateQueryBuilder.fromXContent(context.newParseContext(templateSourceParser)), context).toQuery(context);
         assertTrue("Parsing template query failed.", query instanceof MatchAllDocsQuery);
@@ -192,7 +194,7 @@ public class TemplateQueryParserTests extends ESTestCase {
                 + "  \"params\":{" + "    \"size\":2" + "  }\n" + "}";
 
         XContentParser templateSourceParser = XContentFactory.xContent(templateString).createParser(templateString);
-        context.reset();
+        QueryShardContext context = contextFactory.get();
 
         try {
             TemplateQueryBuilder.fromXContent(context.newParseContext(templateSourceParser)).rewrite(context);
@@ -206,7 +208,7 @@ public class TemplateQueryParserTests extends ESTestCase {
         String templateString = "{ \"file\": \"storedTemplate\" ,\"params\":{\"template\":\"all\" } } ";
 
         XContentParser templateSourceParser = XContentFactory.xContent(templateString).createParser(templateString);
-        context.reset();
+        QueryShardContext context = contextFactory.get();
         templateSourceParser.nextToken();
 
 
@@ -219,7 +221,7 @@ public class TemplateQueryParserTests extends ESTestCase {
         String templateString = "{ \"file\": \"storedTemplate\" ,\"params\":{\"template\":\"all\" } } ";
 
         XContentParser templateSourceParser = XContentFactory.xContent(templateString).createParser(templateString);
-        context.reset();
+        QueryShardContext context = contextFactory.get();
         templateSourceParser.nextToken();
         try {
             TemplateQueryBuilder.fromXContent(context.newParseContext(templateSourceParser)).toQuery(context);

From 929187c71b6125d4eeb014bb6e7bea1c0ddf5368 Mon Sep 17 00:00:00 2001
From: Kelsey DeJesus-Banos <kdejesusbano2013@my.fit.edu>
Date: Tue, 3 May 2016 13:29:24 -0400
Subject: [PATCH 0054/1311] Update function-score-query.asciidoc (#18115)

---
 docs/reference/query-dsl/function-score-query.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/query-dsl/function-score-query.asciidoc b/docs/reference/query-dsl/function-score-query.asciidoc
index 1e736efa11d..402eeb44550 100644
--- a/docs/reference/query-dsl/function-score-query.asciidoc
+++ b/docs/reference/query-dsl/function-score-query.asciidoc
@@ -148,7 +148,7 @@ you wish to inhibit this, set `"boost_mode": "replace"`
 The `weight` score allows you to multiply the score by the provided
 `weight`. This can sometimes be desired since boost value set on
 specific queries gets normalized, while for this score function it does
-not.
+not. The number value is of type float. 
 
 [source,js]
 --------------------------------------------------

From d63362ce43cbe40ff5490d67bc5b6d6151508dcd Mon Sep 17 00:00:00 2001
From: Jihun No <jeesim2@gmail.com>
Date: Wed, 4 May 2016 14:07:28 +0900
Subject: [PATCH 0055/1311] Update client.asciidoc

as of https://github.com/elastic/elasticsearch/commit/42526ac28e07da0055faafca1de6f8c5ec96cd85
5.0.0 alpha2 have no settingsBuilder() method.
---
 docs/java-api/client.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/java-api/client.asciidoc b/docs/java-api/client.asciidoc
index 75a58f3c6b5..c71bd5f6ece 100644
--- a/docs/java-api/client.asciidoc
+++ b/docs/java-api/client.asciidoc
@@ -51,7 +51,7 @@ Note that you have to set the cluster name if you use one different than
 
 [source,java]
 --------------------------------------------------
-Settings settings = Settings.settingsBuilder()
+Settings settings = Settings.builder()
         .put("cluster.name", "myClusterName").build();
 Client client = TransportClient.builder().settings(settings).build();
 //Add transport addresses and do something with the client...

From 80dbe31d59c6cfdc41b5a3c3971d91699ab42f7e Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Wed, 4 May 2016 08:30:28 +0200
Subject: [PATCH 0056/1311] Add note about using ipv6 addresses in
 `query_string`.

---
 docs/reference/mapping/types/ip.asciidoc | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/docs/reference/mapping/types/ip.asciidoc b/docs/reference/mapping/types/ip.asciidoc
index 9786ebf4fd8..624bf63660f 100644
--- a/docs/reference/mapping/types/ip.asciidoc
+++ b/docs/reference/mapping/types/ip.asciidoc
@@ -108,3 +108,20 @@ GET my_index/_search
   }
 }
 --------------------------------------------------
+
+Also beware that colons are special characters to the
+<<query-dsl-query-string-query,`query_string`>> query, so ipv6 addresses will
+need to be escaped. The easiest way to do so is to put quotes around the
+searched value:
+
+[source,js]
+--------------------------------------------------
+GET t/_search
+{
+  "query": {
+    "query_string" : {
+      "query": "ip_addr:\"2001:db8::/48\""
+    }
+  }
+}
+--------------------------------------------------

From 51a53c55cb2b30a3db85cd3342b3e3a4e5520903 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Wed, 4 May 2016 08:51:37 +0200
Subject: [PATCH 0057/1311] Update store documentation after #17616.

---
 docs/reference/index-modules/store.asciidoc | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/docs/reference/index-modules/store.asciidoc b/docs/reference/index-modules/store.asciidoc
index 79712fbc266..54080e365c2 100644
--- a/docs/reference/index-modules/store.asciidoc
+++ b/docs/reference/index-modules/store.asciidoc
@@ -8,9 +8,8 @@ The store module allows you to control how index data is stored and accessed on
 === File system storage types
 
 There are different file system implementations or _storage types_. The best
-one for the operating environment will be automatically chosen: `mmapfs` on
-Windows 64bit, `simplefs` on Windows 32bit, and `default` (hybrid `niofs` and
-`mmapfs`) for the rest.
+one for the operating environment will be automatically chosen: `simplefs` on
+Windows 32bit, `niofs` on other 32bit systems and `mmapfs` on 64bit systems.
 
 This can be overridden for all indices by adding this to the
 `config/elasticsearch.yml` file:
@@ -61,12 +60,13 @@ process equal to the size of the file being mapped. Before using this
 class, be sure you have allowed plenty of
 <<vm-max-map-count,virtual address space>>.
 
-[[default_fs]]`default_fs`::
+[[default_fs]]`default_fs` deprecated[5.0.0, The `default_fs` store type is deprecated - use `mmapfs` instead]::
 
 The `default` type is a hybrid of NIO FS and MMapFS, which chooses the best
-file system for each type of file. Currently only the Lucene term dictionary
-and doc values files are memory mapped to reduce the impact on the operating
-system. All other files are opened using Lucene `NIOFSDirectory`. Address
-space settings (<<vm-max-map-count>>) might also apply if your term
-dictionaries are large.
+file system for each type of file. Currently only the Lucene term dictionary,
+doc values and points files are memory mapped to reduce the impact on the
+operating system. All other files are opened using Lucene `NIOFSDirectory`.
+Address space settings (<<vm-max-map-count>>) might also apply if your term
+dictionary are large, if you index many fields that use points (numerics, dates
+and ip addresses) or if you have many fields with doc values.
 

From ad8bf53bbdfd1d90e52cfd3c4c2e04b7525d0f23 Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Wed, 4 May 2016 11:25:23 +0200
Subject: [PATCH 0058/1311] Fold helper class into abstract sort test class.

Folds the helper class for random object generation into the
abstract sort test class. Removes a few references to ESTestCase
that were not needed due to inheriting from it along the way.
---
 .../search/sort/AbstractSortTestCase.java     | 16 +++++++
 .../search/sort/FieldSortBuilderTests.java    | 30 ++++++-------
 .../sort/GeoDistanceSortBuilderTests.java     |  4 +-
 .../sort/NestedQueryBuilderGenerator.java     | 44 -------------------
 .../search/sort/ScriptSortBuilderTests.java   |  5 +--
 5 files changed, 34 insertions(+), 65 deletions(-)
 delete mode 100644 core/src/test/java/org/elasticsearch/search/sort/NestedQueryBuilderGenerator.java

diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
index a8562493c7a..76d5eba12f9 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
@@ -45,8 +45,12 @@ import org.elasticsearch.index.mapper.Mapper.BuilderContext;
 import org.elasticsearch.index.mapper.core.LegacyDoubleFieldMapper.DoubleFieldType;
 import org.elasticsearch.index.mapper.object.ObjectMapper;
 import org.elasticsearch.index.mapper.object.ObjectMapper.Nested;
+import org.elasticsearch.index.query.IdsQueryBuilder;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
@@ -252,6 +256,18 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST
         return doubleFieldType;
     }
 
+    protected static QueryBuilder<?> randomNestedFilter() {
+        int id = randomIntBetween(0, 2);
+        switch(id) {
+            case 0: return (new MatchAllQueryBuilder()).boost(randomFloat());
+            case 1: return (new IdsQueryBuilder()).boost(randomFloat());
+            case 2: return (new TermQueryBuilder(
+                    randomAsciiOfLengthBetween(1, 10),
+                    randomDouble()).boost(randomFloat()));
+            default: throw new IllegalStateException("Only three query builders supported for testing sort");
+        }
+    }
+
     @SuppressWarnings("unchecked")
     private T copyItem(T original) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
index ebd5403489b..4b6eb82304a 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
@@ -25,12 +25,10 @@ import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
-import java.util.function.Supplier;
 
 public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder> {
 
@@ -42,8 +40,8 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
     private List<Object> missingContent = Arrays.asList(
             "_last",
             "_first",
-            ESTestCase.randomAsciiOfLength(10), ESTestCase.randomUnicodeOfCodepointLengthBetween(5, 15),
-            ESTestCase.randomInt());
+            randomAsciiOfLength(10), randomUnicodeOfCodepointLengthBetween(5, 15),
+            randomInt());
 
 
     public FieldSortBuilder randomFieldSortBuilder() {
@@ -58,7 +56,7 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
         }
 
         if (randomBoolean()) {
-            builder.unmappedType(ESTestCase.randomAsciiOfLengthBetween(1, 10));
+            builder.unmappedType(randomAsciiOfLengthBetween(1, 10));
         }
 
         if (randomBoolean()) {
@@ -66,11 +64,11 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
         }
 
         if (randomBoolean()) {
-            builder.setNestedFilter(NestedQueryBuilderGenerator.randomNestedFilter());
+            builder.setNestedFilter(randomNestedFilter());
         }
 
         if (randomBoolean()) {
-            builder.setNestedPath(ESTestCase.randomAsciiOfLengthBetween(1, 10));
+            builder.setNestedPath(randomAsciiOfLengthBetween(1, 10));
         }
 
         return builder;
@@ -82,28 +80,28 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
         int parameter = randomIntBetween(0, 5);
         switch (parameter) {
         case 0:
-            mutated.setNestedPath(ESTestCase.randomValueOtherThan(
+            mutated.setNestedPath(randomValueOtherThan(
                     original.getNestedPath(),
-                    () -> ESTestCase.randomAsciiOfLengthBetween(1, 10)));
+                    () -> randomAsciiOfLengthBetween(1, 10)));
             break;
         case 1:
-            mutated.setNestedFilter(ESTestCase.randomValueOtherThan(
+            mutated.setNestedFilter(randomValueOtherThan(
                     original.getNestedFilter(),
-                    () -> NestedQueryBuilderGenerator.randomNestedFilter()));
+                    () -> randomNestedFilter()));
             break;
         case 2:
-            mutated.sortMode(ESTestCase.randomValueOtherThan(original.sortMode(), () -> randomFrom(SortMode.values())));
+            mutated.sortMode(randomValueOtherThan(original.sortMode(), () -> randomFrom(SortMode.values())));
             break;
         case 3:
-            mutated.unmappedType(ESTestCase.randomValueOtherThan(
+            mutated.unmappedType(randomValueOtherThan(
                     original.unmappedType(),
-                    () -> ESTestCase.randomAsciiOfLengthBetween(1, 10)));
+                    () -> randomAsciiOfLengthBetween(1, 10)));
             break;
         case 4:
-            mutated.missing(ESTestCase.randomValueOtherThan(original.missing(), () -> randomFrom(missingContent)));
+            mutated.missing(randomValueOtherThan(original.missing(), () -> randomFrom(missingContent)));
             break;
         case 5:
-            mutated.order(ESTestCase.randomValueOtherThan(original.order(), () -> randomFrom(SortOrder.values())));
+            mutated.order(randomValueOtherThan(original.order(), () -> randomFrom(SortOrder.values())));
             break;
         default:
             throw new IllegalStateException("Unsupported mutation.");
diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
index 3da59f79de6..817427f9ecc 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
@@ -89,7 +89,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
             result.sortMode(mode(result.sortMode()));
         }
         if (randomBoolean()) {
-            result.setNestedFilter(NestedQueryBuilderGenerator.randomNestedFilter());
+            result.setNestedFilter(randomNestedFilter());
         }
         if (randomBoolean()) {
             result.setNestedPath(
@@ -179,7 +179,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
         case 6:
             result.setNestedFilter(ESTestCase.randomValueOtherThan(
                     original.getNestedFilter(),
-                    () -> NestedQueryBuilderGenerator.randomNestedFilter()));
+                    () -> randomNestedFilter()));
             break;
         case 7:
             result.setNestedPath(ESTestCase.randomValueOtherThan(
diff --git a/core/src/test/java/org/elasticsearch/search/sort/NestedQueryBuilderGenerator.java b/core/src/test/java/org/elasticsearch/search/sort/NestedQueryBuilderGenerator.java
deleted file mode 100644
index de3318036e8..00000000000
--- a/core/src/test/java/org/elasticsearch/search/sort/NestedQueryBuilderGenerator.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.sort;
-
-import org.elasticsearch.index.query.IdsQueryBuilder;
-import org.elasticsearch.index.query.MatchAllQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.TermQueryBuilder;
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.Arrays;
-import java.util.List;
-
-public class NestedQueryBuilderGenerator {
-    private NestedQueryBuilderGenerator() {
-        // this is a helper class only, doesn't need a constructor
-    }
-
-    private static List<QueryBuilder<?>> builders = Arrays.asList(
-            new MatchAllQueryBuilder(),
-            new IdsQueryBuilder(),
-            new TermQueryBuilder(ESTestCase.randomAsciiOfLengthBetween(1, 10), ESTestCase.randomDouble()));
-
-    public static QueryBuilder<?> randomNestedFilter() {
-        return ESTestCase.randomFrom(builders).boost(ESTestCase.randomFloat());
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
index b809adcb27b..38a20a129aa 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
@@ -25,7 +25,6 @@ import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService.ScriptType;
@@ -64,7 +63,7 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild
             }
         }
         if (randomBoolean()) {
-            builder.setNestedFilter(NestedQueryBuilderGenerator.randomNestedFilter());
+            builder.setNestedFilter(randomNestedFilter());
         }
         if (randomBoolean()) {
             builder.setNestedPath(ESTestCase.randomAsciiOfLengthBetween(1, 10));
@@ -116,7 +115,7 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild
             case 2:
                 result.setNestedFilter(ESTestCase.randomValueOtherThan(
                         original.getNestedFilter(),
-                        () -> NestedQueryBuilderGenerator.randomNestedFilter()));
+                        () -> randomNestedFilter()));
                 break;
             case 3:
                 result.setNestedPath(original.getNestedPath() + "_some_suffix");

From 9a90bdce01592927c61956c3324cab69722abbdb Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Wed, 4 May 2016 12:04:28 +0200
Subject: [PATCH 0059/1311] Add deprecation warnings for ignore_malformed to
 reference docs

---
 docs/reference/query-dsl/geo-bounding-box-query.asciidoc | 3 +++
 docs/reference/query-dsl/geo-distance-query.asciidoc     | 5 +++++
 docs/reference/query-dsl/geo-polygon-query.asciidoc      | 3 +++
 3 files changed, 11 insertions(+)

diff --git a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
index 2630e6cf048..00120a98fc9 100644
--- a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
+++ b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
@@ -52,6 +52,9 @@ Then the following simple query can be executed with a
 |Option |Description
 |`_name` |Optional name field to identify the filter
 
+|`ignore_malformed` |deprecated[5.0.0,Use `validation_method` instead] Set to `true` to
+accept geo points with invalid latitude or longitude (default is `false`).
+
 |`validation_method` |Set to `IGNORE_MALFORMED` to
 accept geo points with invalid latitude or longitude, set to
 `COERCE` to also try to infer correct latitude or longitude. (default is `STRICT`).
diff --git a/docs/reference/query-dsl/geo-distance-query.asciidoc b/docs/reference/query-dsl/geo-distance-query.asciidoc
index 2b80241dfc5..c29391e5b57 100644
--- a/docs/reference/query-dsl/geo-distance-query.asciidoc
+++ b/docs/reference/query-dsl/geo-distance-query.asciidoc
@@ -162,6 +162,11 @@ The following are options allowed on the filter:
 
     Optional name field to identify the query
 
+`ignore_malformed`::
+   
+    deprecated[5.0.0,Use `validation_method` instead] Set to `true` to accept geo points with invalid latitude or
+    longitude (default is `false`).
+
 `validation_method`::
 
     Set to `IGNORE_MALFORMED` to accept geo points with invalid latitude or
diff --git a/docs/reference/query-dsl/geo-polygon-query.asciidoc b/docs/reference/query-dsl/geo-polygon-query.asciidoc
index ef80de82079..5717c5dc924 100644
--- a/docs/reference/query-dsl/geo-polygon-query.asciidoc
+++ b/docs/reference/query-dsl/geo-polygon-query.asciidoc
@@ -34,6 +34,9 @@ points. Here is an example:
 |Option |Description
 |`_name` |Optional name field to identify the filter
 
+|`ignore_malformed` |deprecated[5.0.0,Use `validation_method` instead] Set to `true` to accept geo points with invalid latitude or
+longitude (default is `false`).
+
 |`validation_method` |Set to `IGNORE_MALFORMED` to accept geo points with
 invalid latitude or longitude, `COERCE` to try and infer correct latitude
 or longitude, or `STRICT` (default is `STRICT`).

From 6f15f35819f26763a5b3f427af6b6a68fff3cebe Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Wed, 4 May 2016 12:14:39 +0200
Subject: [PATCH 0060/1311] Remove left over references to ESTestCase

---
 .../search/sort/GeoDistanceSortBuilderTests.java    | 13 ++++++-------
 .../search/sort/ScoreSortBuilderTests.java          |  3 +--
 .../search/sort/ScriptSortBuilderTests.java         | 11 +++++------
 3 files changed, 12 insertions(+), 15 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
index 817427f9ecc..a41138e2093 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
@@ -35,7 +35,6 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.geo.RandomGeoGenerator;
 
 import java.io.IOException;
@@ -93,9 +92,9 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
         }
         if (randomBoolean()) {
             result.setNestedPath(
-                    ESTestCase.randomValueOtherThan(
+                    randomValueOtherThan(
                             result.getNestedPath(),
-                            () -> ESTestCase.randomAsciiOfLengthBetween(1, 10)));
+                            () -> randomAsciiOfLengthBetween(1, 10)));
         }
         if (randomBoolean()) {
             result.coerce(! result.coerce());
@@ -171,20 +170,20 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
             result.unit(unit(original.unit()));
             break;
         case 4:
-            result.order(ESTestCase.randomValueOtherThan(original.order(), () -> randomFrom(SortOrder.values())));
+            result.order(randomValueOtherThan(original.order(), () -> randomFrom(SortOrder.values())));
             break;
         case 5:
             result.sortMode(mode(original.sortMode()));
             break;
         case 6:
-            result.setNestedFilter(ESTestCase.randomValueOtherThan(
+            result.setNestedFilter(randomValueOtherThan(
                     original.getNestedFilter(),
                     () -> randomNestedFilter()));
             break;
         case 7:
-            result.setNestedPath(ESTestCase.randomValueOtherThan(
+            result.setNestedPath(randomValueOtherThan(
                     result.getNestedPath(),
-                    () -> ESTestCase.randomAsciiOfLengthBetween(1, 10)));
+                    () -> randomAsciiOfLengthBetween(1, 10)));
             break;
         case 8:
             result.coerce(! original.coerce());
diff --git a/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java
index 222240a9ac4..605573e9b71 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java
@@ -26,7 +26,6 @@ import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.test.ESTestCase;
 import org.junit.Rule;
 import org.junit.rules.ExpectedException;
 
@@ -46,7 +45,7 @@ public class ScoreSortBuilderTests extends AbstractSortTestCase<ScoreSortBuilder
     @Override
     protected ScoreSortBuilder mutate(ScoreSortBuilder original) throws IOException {
         ScoreSortBuilder result = new ScoreSortBuilder();
-        result.order(ESTestCase.randomValueOtherThan(original.order(), () -> randomFrom(SortOrder.values())));
+        result.order(randomValueOtherThan(original.order(), () -> randomFrom(SortOrder.values())));
         return result;
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
index 38a20a129aa..6491c36bf6d 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
@@ -29,7 +29,6 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
-import org.elasticsearch.test.ESTestCase;
 import org.junit.Rule;
 import org.junit.rules.ExpectedException;
 
@@ -53,20 +52,20 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild
         }
         if (randomBoolean()) {
             if (type == ScriptSortType.NUMBER) {
-                builder.sortMode(ESTestCase.randomValueOtherThan(builder.sortMode(), () -> randomFrom(SortMode.values())));
+                builder.sortMode(randomValueOtherThan(builder.sortMode(), () -> randomFrom(SortMode.values())));
             } else {
                 Set<SortMode> exceptThis = new HashSet<>();
                 exceptThis.add(SortMode.SUM);
                 exceptThis.add(SortMode.AVG);
                 exceptThis.add(SortMode.MEDIAN);
-                builder.sortMode(ESTestCase.randomValueOtherThanMany(exceptThis::contains, () -> randomFrom(SortMode.values())));
+                builder.sortMode(randomValueOtherThanMany(exceptThis::contains, () -> randomFrom(SortMode.values())));
             }
         }
         if (randomBoolean()) {
             builder.setNestedFilter(randomNestedFilter());
         }
         if (randomBoolean()) {
-            builder.setNestedPath(ESTestCase.randomAsciiOfLengthBetween(1, 10));
+            builder.setNestedPath(randomAsciiOfLengthBetween(1, 10));
         }
         return builder;
     }
@@ -102,7 +101,7 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild
                 break;
             case 1:
                 if (original.type() == ScriptSortType.NUMBER) {
-                    result.sortMode(ESTestCase.randomValueOtherThan(result.sortMode(), () -> randomFrom(SortMode.values())));
+                    result.sortMode(randomValueOtherThan(result.sortMode(), () -> randomFrom(SortMode.values())));
                 } else {
                     // script sort type String only allows MIN and MAX, so we only switch
                     if (original.sortMode() == SortMode.MIN) {
@@ -113,7 +112,7 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild
                 }
                 break;
             case 2:
-                result.setNestedFilter(ESTestCase.randomValueOtherThan(
+                result.setNestedFilter(randomValueOtherThan(
                         original.getNestedFilter(),
                         () -> randomNestedFilter()));
                 break;

From 283a6d27f1aa59e38739b19a058ba798742f09ef Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Wed, 4 May 2016 13:34:11 +0200
Subject: [PATCH 0061/1311] Clean up merge commit changes

---
 .../search/sort/GeoDistanceSortBuilderTests.java  | 15 +++++++--------
 .../java/org/elasticsearch/test/ESTestCase.java   | 11 -----------
 2 files changed, 7 insertions(+), 19 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
index 89379e6b428..3c94f69eba5 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
@@ -36,7 +36,6 @@ import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
 import org.elasticsearch.index.query.GeoValidationMethod;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.geo.RandomGeoGenerator;
 
 import java.io.IOException;
@@ -81,13 +80,13 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
             result.geoDistance(geoDistance(result.geoDistance()));
         }
         if (randomBoolean()) {
-            result.unit(ESTestCase.randomValueOtherThan(result.unit(), () -> randomFrom(DistanceUnit.values())));
+            result.unit(randomValueOtherThan(result.unit(), () -> randomFrom(DistanceUnit.values())));
         }
         if (randomBoolean()) {
             result.order(randomFrom(SortOrder.values()));
         }
         if (randomBoolean()) {
-            result.sortMode(ESTestCase.randomValueOtherThan(SortMode.SUM, () -> randomFrom(SortMode.values())));
+            result.sortMode(randomValueOtherThan(SortMode.SUM, () -> randomFrom(SortMode.values())));
         }
         if (randomBoolean()) {
             result.setNestedFilter(randomNestedFilter());
@@ -99,7 +98,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
                             () -> randomAsciiOfLengthBetween(1, 10)));
         }
         if (randomBoolean()) {
-            result.validation(ESTestCase.randomValueOtherThan(result.validation(), () -> randomFrom(GeoValidationMethod.values())));
+            result.validation(randomValueOtherThan(result.validation(), () -> randomFrom(GeoValidationMethod.values())));
         }
 
         return result;
@@ -150,14 +149,14 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
             result.geoDistance(geoDistance(original.geoDistance()));
             break;
         case 3:
-            result.unit(ESTestCase.randomValueOtherThan(result.unit(), () -> randomFrom(DistanceUnit.values())));
+            result.unit(randomValueOtherThan(result.unit(), () -> randomFrom(DistanceUnit.values())));
             break;
         case 4:
             result.order(randomValueOtherThan(original.order(), () -> randomFrom(SortOrder.values())));
             break;
         case 5:
-            result.sortMode(ESTestCase.randomValueOtherThanMany(
-                    Arrays.asList(SortMode.SUM, result.sortMode()),
+            result.sortMode(randomValueOtherThanMany(
+                    Arrays.asList(SortMode.SUM, result.sortMode())::contains,
                     () -> randomFrom(SortMode.values())));
             break;
         case 6:
@@ -171,7 +170,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
                     () -> randomAsciiOfLengthBetween(1, 10)));
             break;
         case 8:
-            result.validation(ESTestCase.randomValueOtherThan(result.validation(), () -> randomFrom(GeoValidationMethod.values())));
+            result.validation(randomValueOtherThan(result.validation(), () -> randomFrom(GeoValidationMethod.values())));
             break;
         }
         return result;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 7e33f494d70..01f0d5b8151 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -429,17 +429,6 @@ public abstract class ESTestCase extends LuceneTestCase {
         return randomValue;
     }
 
-    /**
-     * helper to get a random value in a certain range that's different from the input
-     */
-    public static <T> T randomValueOtherThanMany(Collection<T> input, Supplier<T> randomSupplier) {
-        T randomValue = null;
-        do {
-            randomValue = randomSupplier.get();
-        } while (input.contains(randomValue));
-        return randomValue;
-    }
-
     /**
      * Runs the code block for 10 seconds waiting for no assertion to trip.
      */

From 2dea44994922dacec38f81cc68c42360bdc449d7 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 3 May 2016 09:12:28 -0400
Subject: [PATCH 0062/1311] Remove Strings#splitStringToArray

This commit removes the method Strings#splitStringToArray and replaces
the call sites with invocations to String#split. There are only two
explanations for the existence of this method. The first is that
String#split is slightly tricky in that it accepts a regular expression
rather than a character to split on. This means that if s is a string,
s.split(".")  does not split on the character '.', but rather splits on
the regular expression '.' which splits on every character (of course,
this is easily fixed by invoking s.split("\\.") instead). The second
possible explanation is that (again) String#split accepts a regular
expression. This means that there could be a performance concern
compared to just splitting on a single character. However, it turns out
that String#split has a fast path for the case of splitting on a single
character and microbenchmarks show that String#split has 1.5x--2x the
throughput of Strings#splitStringToArray. There is a slight behavior
difference between Strings#splitStringToArray and String#split: namely,
the former would return an empty array in cases when the input string
was null or empty but String#split will just NPE at the call site on
null and return a one-element array containing the empty string when the
input string is empty. There was only one place relying on this behavior
and the call site has been modified accordingly.
---
 .../action/search/TransportSearchHelper.java  |  3 +-
 .../org/elasticsearch/common/Strings.java     | 40 +------------------
 .../java/org/elasticsearch/common/Table.java  |  2 +-
 .../elasticsearch/common/path/PathTrie.java   | 13 +++---
 .../common/settings/Settings.java             | 13 ------
 .../common/util/BloomFilter.java              |  3 +-
 .../xcontent/support/XContentMapValues.java   |  4 +-
 .../index/mapper/DocumentParser.java          | 17 ++++----
 .../index/query/GeoShapeQueryBuilder.java     |  3 +-
 .../ingest/core/IngestDocument.java           |  4 +-
 .../admin/cluster/node/tasks/TasksIT.java     |  2 +-
 .../common/settings/SettingsTests.java        | 18 ---------
 .../cloud/gce/network/GceNameResolver.java    |  2 +-
 .../repositories/azure/AzureRepository.java   |  2 +-
 .../repositories/s3/S3Repository.java         |  2 +-
 .../ingest/RandomDocumentPicks.java           |  4 +-
 16 files changed, 29 insertions(+), 103 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java b/core/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
index 49d4c65add1..8da195b739b 100644
--- a/core/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
+++ b/core/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
@@ -24,7 +24,6 @@ import org.apache.lucene.util.CharsRefBuilder;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Base64;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
@@ -89,7 +88,7 @@ final class TransportSearchHelper {
         } catch (Exception e) {
             throw new IllegalArgumentException("Failed to decode scrollId", e);
         }
-        String[] elements = Strings.splitStringToArray(spare.get(), ';');
+        String[] elements = spare.get().toString().split(";");
         if (elements.length < 2) {
             throw new IllegalArgumentException("Malformed scrollId [" + scrollId + "]");
         }
diff --git a/core/src/main/java/org/elasticsearch/common/Strings.java b/core/src/main/java/org/elasticsearch/common/Strings.java
index 151c53e2007..6b6c31c1522 100644
--- a/core/src/main/java/org/elasticsearch/common/Strings.java
+++ b/core/src/main/java/org/elasticsearch/common/Strings.java
@@ -38,7 +38,6 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Properties;
-import java.util.Random;
 import java.util.Set;
 import java.util.StringTokenizer;
 import java.util.TreeSet;
@@ -557,7 +556,8 @@ public class Strings {
     }
 
     public static String[] splitStringByCommaToArray(final String s) {
-        return splitStringToArray(s, ',');
+        if (s == null || s.isEmpty()) return Strings.EMPTY_ARRAY;
+        else return s.split(",");
     }
 
     public static Set<String> splitStringToSet(final String s, final char c) {
@@ -588,42 +588,6 @@ public class Strings {
         return result;
     }
 
-    public static String[] splitStringToArray(final CharSequence s, final char c) {
-        if (s == null || s.length() == 0) {
-            return Strings.EMPTY_ARRAY;
-        }
-        int count = 1;
-        for (int i = 0; i < s.length(); i++) {
-            if (s.charAt(i) == c) {
-                count++;
-            }
-        }
-        final String[] result = new String[count];
-        final StringBuilder builder = new StringBuilder();
-        int res = 0;
-        for (int i = 0; i < s.length(); i++) {
-            if (s.charAt(i) == c) {
-                if (builder.length() > 0) {
-                    result[res++] = builder.toString();
-                    builder.setLength(0);
-                }
-
-            } else {
-                builder.append(s.charAt(i));
-            }
-        }
-        if (builder.length() > 0) {
-            result[res++] = builder.toString();
-        }
-        if (res != count) {
-            // we have empty strings, copy over to a new array
-            String[] result1 = new String[res];
-            System.arraycopy(result, 0, result1, 0, res);
-            return result1;
-        }
-        return result;
-    }
-
     /**
      * Split a String at the first occurrence of the delimiter.
      * Does not include the delimiter in the result.
diff --git a/core/src/main/java/org/elasticsearch/common/Table.java b/core/src/main/java/org/elasticsearch/common/Table.java
index 0d4a827202d..ab0252b11dc 100644
--- a/core/src/main/java/org/elasticsearch/common/Table.java
+++ b/core/src/main/java/org/elasticsearch/common/Table.java
@@ -149,7 +149,7 @@ public class Table {
                 // get the attributes of the header cell we are going to add
                 mAttr.putAll(headers.get(currentCells.size()).attr);
             }
-            String[] sAttrs = Strings.splitStringToArray(attributes, ';');
+            String[] sAttrs = attributes.split(";");
             for (String sAttr : sAttrs) {
                 if (sAttr.length() == 0) {
                     continue;
diff --git a/core/src/main/java/org/elasticsearch/common/path/PathTrie.java b/core/src/main/java/org/elasticsearch/common/path/PathTrie.java
index 704468f7533..3c25a9caa52 100644
--- a/core/src/main/java/org/elasticsearch/common/path/PathTrie.java
+++ b/core/src/main/java/org/elasticsearch/common/path/PathTrie.java
@@ -19,8 +19,6 @@
 
 package org.elasticsearch.common.path;
 
-import org.elasticsearch.common.Strings;
-
 import java.util.HashMap;
 import java.util.Map;
 
@@ -38,7 +36,7 @@ public class PathTrie<T> {
 
     private final Decoder decoder;
     private final TrieNode root;
-    private final char separator;
+    private final String separator;
     private T rootValue;
 
     public PathTrie(Decoder decoder) {
@@ -47,8 +45,9 @@ public class PathTrie<T> {
 
     public PathTrie(char separator, String wildcard, Decoder decoder) {
         this.decoder = decoder;
-        this.separator = separator;
-        root = new TrieNode(new String(new char[]{separator}), null, wildcard);
+        final String separatorAsString = new String(new char[]{separator});
+        this.separator = separatorAsString;
+        root = new TrieNode(separatorAsString, null, wildcard);
     }
 
     public class TrieNode {
@@ -196,7 +195,7 @@ public class PathTrie<T> {
     }
 
     public void insert(String path, T value) {
-        String[] strings = Strings.splitStringToArray(path, separator);
+        String[] strings = path.split(separator);
         if (strings.length == 0) {
             rootValue = value;
             return;
@@ -217,7 +216,7 @@ public class PathTrie<T> {
         if (path.length() == 0) {
             return rootValue;
         }
-        String[] strings = Strings.splitStringToArray(path, separator);
+        String[] strings = path.split(separator);
         if (strings.length == 0) {
             return rootValue;
         }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/Settings.java b/core/src/main/java/org/elasticsearch/common/settings/Settings.java
index 1c243255454..8488ca75c73 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/Settings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/Settings.java
@@ -899,19 +899,6 @@ public final class Settings implements ToXContent {
             return this;
         }
 
-        public Builder loadFromDelimitedString(String value, char delimiter) {
-            String[] values = Strings.splitStringToArray(value, delimiter);
-            for (String s : values) {
-                int index = s.indexOf('=');
-                if (index == -1) {
-                    throw new IllegalArgumentException(
-                            "value [" + s + "] for settings loaded with delimiter [" + delimiter + "] is malformed, missing =");
-                }
-                map.put(s.substring(0, index), s.substring(index + 1));
-            }
-            return this;
-        }
-
         /**
          * Loads settings from the actual string content that represents them using the
          * {@link SettingsLoaderFactory#loaderFromSource(String)}.
diff --git a/core/src/main/java/org/elasticsearch/common/util/BloomFilter.java b/core/src/main/java/org/elasticsearch/common/util/BloomFilter.java
index b9dd6859ce0..6c471cddb55 100644
--- a/core/src/main/java/org/elasticsearch/common/util/BloomFilter.java
+++ b/core/src/main/java/org/elasticsearch/common/util/BloomFilter.java
@@ -24,7 +24,6 @@ import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.RamUsageEstimator;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.hash.MurmurHash3;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -75,7 +74,7 @@ public class BloomFilter {
             if (config == null) {
                 return buildDefault();
             }
-            String[] sEntries = Strings.splitStringToArray(config, ',');
+            String[] sEntries = config.split(",");
             if (sEntries.length == 0) {
                 if (config.length() > 0) {
                     return new Factory(new Entry[]{new Entry(0, Double.parseDouble(config))});
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java b/core/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java
index 4612d3f05d0..a8c120f424b 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java
@@ -40,7 +40,7 @@ public class XContentMapValues {
      */
     public static List<Object> extractRawValues(String path, Map<String, Object> map) {
         List<Object> values = new ArrayList<>();
-        String[] pathElements = Strings.splitStringToArray(path, '.');
+        String[] pathElements = path.split("\\.");
         if (pathElements.length == 0) {
             return values;
         }
@@ -93,7 +93,7 @@ public class XContentMapValues {
     }
 
     public static Object extractValue(String path, Map<String, Object> map) {
-        String[] pathElements = Strings.splitStringToArray(path, '.');
+        String[] pathElements = path.split("\\.");
         if (pathElements.length == 0) {
             return null;
         }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
index 3d191e4d80a..a666ffc0ed5 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
@@ -19,13 +19,6 @@
 
 package org.elasticsearch.index.mapper;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.Version;
@@ -51,6 +44,13 @@ import org.elasticsearch.index.mapper.internal.UidFieldMapper;
 import org.elasticsearch.index.mapper.object.ArrayValueMapperParser;
 import org.elasticsearch.index.mapper.object.ObjectMapper;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+
 /** A parser for documents, given mappings from a DocumentMapper */
 final class DocumentParser {
 
@@ -829,8 +829,7 @@ final class DocumentParser {
             // The path of the dest field might be completely different from the current one so we need to reset it
             context = context.overridePath(new ContentPath(0));
 
-            // TODO: why Strings.splitStringToArray instead of String.split?
-            final String[] paths = Strings.splitStringToArray(field, '.');
+            final String[] paths = field.split("\\.");
             final String fieldName = paths[paths.length-1];
             ObjectMapper mapper = context.root();
             ObjectMapper[] mappers = new ObjectMapper[paths.length-1];
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
index 8ff2f697b45..d3c47d5bc70 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
@@ -33,7 +33,6 @@ import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.geo.ShapesAvailability;
 import org.elasticsearch.common.geo.SpatialStrategy;
@@ -379,7 +378,7 @@ public class GeoShapeQueryBuilder extends AbstractQueryBuilder<GeoShapeQueryBuil
             throw new IllegalArgumentException("Shape with ID [" + getRequest.id() + "] in type [" + getRequest.type() + "] not found");
         }
 
-        String[] pathElements = Strings.splitStringToArray(path, '.');
+        String[] pathElements = path.split("\\.");
         int currentPathSlot = 0;
 
         try (XContentParser parser = XContentHelper.createParser(response.getSourceAsBytesRef())) {
diff --git a/core/src/main/java/org/elasticsearch/ingest/core/IngestDocument.java b/core/src/main/java/org/elasticsearch/ingest/core/IngestDocument.java
index 05682991713..8be307d776c 100644
--- a/core/src/main/java/org/elasticsearch/ingest/core/IngestDocument.java
+++ b/core/src/main/java/org/elasticsearch/ingest/core/IngestDocument.java
@@ -622,8 +622,8 @@ public final class IngestDocument {
                     newPath = path;
                 }
             }
-            this.pathElements = Strings.splitStringToArray(newPath, '.');
-            if (pathElements.length == 0) {
+            this.pathElements = newPath.split("\\.");
+            if (pathElements.length == 1 && pathElements[0].isEmpty()) {
                 throw new IllegalArgumentException("path [" + path + "] is not valid");
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
index cfc10c926c3..cfa937f7dc1 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
@@ -492,7 +492,7 @@ public class TasksIT extends ESIntegTestCase {
     private void registerTaskManageListeners(String actionMasks) {
         for (String nodeName : internalCluster().getNodeNames()) {
             DiscoveryNode node = internalCluster().getInstance(ClusterService.class, nodeName).localNode();
-            RecordingTaskManagerListener listener = new RecordingTaskManagerListener(node, Strings.splitStringToArray(actionMasks, ','));
+            RecordingTaskManagerListener listener = new RecordingTaskManagerListener(node, actionMasks.split(","));
             ((MockTaskManager) internalCluster().getInstance(TransportService.class, nodeName).getTaskManager()).addListener(listener);
             RecordingTaskManagerListener oldListener = listeners.put(new Tuple<>(node.getName(), actionMasks), listener);
             assertNull(oldListener);
diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
index ced961a70b8..3539e54d943 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
@@ -38,24 +38,6 @@ import static org.hamcrest.Matchers.nullValue;
 
 public class SettingsTests extends ESTestCase {
 
-    public void testLoadFromDelimitedString() {
-        Settings settings = Settings.builder()
-                .loadFromDelimitedString("key1=value1;key2=value2", ';')
-                .build();
-        assertThat(settings.get("key1"), equalTo("value1"));
-        assertThat(settings.get("key2"), equalTo("value2"));
-        assertThat(settings.getAsMap().size(), equalTo(2));
-        assertThat(settings.toDelimitedString(';'), equalTo("key1=value1;key2=value2;"));
-
-        settings = Settings.builder()
-                .loadFromDelimitedString("key1=value1;key2=value2;", ';')
-                .build();
-        assertThat(settings.get("key1"), equalTo("value1"));
-        assertThat(settings.get("key2"), equalTo("value2"));
-        assertThat(settings.getAsMap().size(), equalTo(2));
-        assertThat(settings.toDelimitedString(';'), equalTo("key1=value1;key2=value2;"));
-    }
-
     public void testReplacePropertiesPlaceholderSystemProperty() {
         String value = System.getProperty("java.home");
         assertFalse(value.isEmpty());
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/network/GceNameResolver.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/network/GceNameResolver.java
index 22d79fb1614..0bd5e07da91 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/network/GceNameResolver.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/network/GceNameResolver.java
@@ -92,7 +92,7 @@ public class GceNameResolver extends AbstractComponent implements CustomNameReso
         } else if (value.startsWith(GceAddressResolverType.PRIVATE_IP.configName)) {
             // We extract the network interface from gce:privateIp:XX
             String network = "0";
-            String[] privateIpConfig = Strings.splitStringToArray(value, ':');
+            String[] privateIpConfig = value.split(":");
             if (privateIpConfig != null && privateIpConfig.length == 3) {
                 network = privateIpConfig[2];
             }
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
index 66db57fdd92..8af614df605 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
@@ -116,7 +116,7 @@ public class AzureRepository extends BlobStoreRepository {
             // Remove starting / if any
             basePath = Strings.trimLeadingCharacter(basePath, '/');
             BlobPath path = new BlobPath();
-            for(String elem : Strings.splitStringToArray(basePath, '/')) {
+            for(String elem : basePath.split("/")) {
                 path = path.add(elem);
             }
             this.basePath = path;
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
index a09d57ebc93..5861893d6b2 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
@@ -287,7 +287,7 @@ public class S3Repository extends BlobStoreRepository {
         String basePath = getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING);
         if (Strings.hasLength(basePath)) {
             BlobPath path = new BlobPath();
-            for(String elem : Strings.splitStringToArray(basePath, '/')) {
+            for(String elem : basePath.split("/")) {
                 path = path.add(elem);
             }
             this.basePath = path;
diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java b/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java
index c42d51b7abd..6dc920c6ff8 100644
--- a/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java
+++ b/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java
@@ -22,8 +22,6 @@ package org.elasticsearch.ingest;
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.ingest.core.IngestDocument;
 
 import java.util.ArrayList;
@@ -105,7 +103,7 @@ public final class RandomDocumentPicks {
      * that each node of the tree either doesn't exist or is a map, otherwise new fields cannot be added.
      */
     public static boolean canAddField(String path, IngestDocument ingestDocument) {
-        String[] pathElements = Strings.splitStringToArray(path, '.');
+        String[] pathElements = path.split("\\.");
         Map<String, Object> innerMap = ingestDocument.getSourceAndMetadata();
         if (pathElements.length > 1) {
             for (int i = 0; i < pathElements.length - 1; i++) {

From 9fe5ce934279a274abdf67b417aae8d1566cde63 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 4 May 2016 09:18:30 -0400
Subject: [PATCH 0063/1311] Remove arbitrary separator/wildcard from PathTrie

PathTrie has a constructor that allows for an arbitrary separtor and
wildcard, but this constructor is unused and internally we always use
'/' as the separator and '*' as the wildcard. There are no tests for the
case where the separator differs from the default separator and
wildcard. This commit removes this constructor and now all instances of
PathTrie have the default separator and wildcard.
---
 .../elasticsearch/common/path/PathTrie.java   | 19 ++++++-------------
 1 file changed, 6 insertions(+), 13 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/path/PathTrie.java b/core/src/main/java/org/elasticsearch/common/path/PathTrie.java
index 3c25a9caa52..c711bfa2a61 100644
--- a/core/src/main/java/org/elasticsearch/common/path/PathTrie.java
+++ b/core/src/main/java/org/elasticsearch/common/path/PathTrie.java
@@ -25,9 +25,6 @@ import java.util.Map;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.unmodifiableMap;
 
-/**
- *
- */
 public class PathTrie<T> {
 
     public interface Decoder {
@@ -36,18 +33,14 @@ public class PathTrie<T> {
 
     private final Decoder decoder;
     private final TrieNode root;
-    private final String separator;
     private T rootValue;
 
-    public PathTrie(Decoder decoder) {
-        this('/', "*", decoder);
-    }
+    private static final String SEPARATOR = "/";
+    private static final String WILDCARD = "*";
 
-    public PathTrie(char separator, String wildcard, Decoder decoder) {
+    public PathTrie(Decoder decoder) {
         this.decoder = decoder;
-        final String separatorAsString = new String(new char[]{separator});
-        this.separator = separatorAsString;
-        root = new TrieNode(separatorAsString, null, wildcard);
+        root = new TrieNode(SEPARATOR, null, WILDCARD);
     }
 
     public class TrieNode {
@@ -195,7 +188,7 @@ public class PathTrie<T> {
     }
 
     public void insert(String path, T value) {
-        String[] strings = path.split(separator);
+        String[] strings = path.split(SEPARATOR);
         if (strings.length == 0) {
             rootValue = value;
             return;
@@ -216,7 +209,7 @@ public class PathTrie<T> {
         if (path.length() == 0) {
             return rootValue;
         }
-        String[] strings = path.split(separator);
+        String[] strings = path.split(SEPARATOR);
         if (strings.length == 0) {
             return rootValue;
         }

From 052191f2a2a76cb214c74340eede8d193ae24d24 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Wed, 4 May 2016 10:55:24 +0200
Subject: [PATCH 0064/1311] Add the ability to use the breadth_first mode with
 nested aggregations (such as `top_hits`) which require access to score
 information. The score is recomputed lazily for each document belonging to a
 top bucket. Relates to #9825

---
 .../elasticsearch/common/lucene/Lucene.java   | 25 -------------
 .../search/aggregations/AggregatorBase.java   |  2 +-
 .../bucket/BestBucketsDeferringCollector.java | 33 +++++++++++++----
 .../bucket/terms/TermsAggregator.java         |  1 -
 .../aggregations/metrics/TopHitsIT.java       | 35 +++++++++++++++++--
 .../search/profile/ProfileTests.java          |  2 +-
 .../bucket/terms-aggregation.asciidoc         |  5 ++-
 7 files changed, 64 insertions(+), 39 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
index 8508a8a2e40..fcc443c57b6 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
@@ -638,31 +638,6 @@ public class Lucene {
         }
     }
 
-    /**
-     * Return a Scorer that throws an ElasticsearchIllegalStateException
-     * on all operations with the given message.
-     */
-    public static Scorer illegalScorer(final String message) {
-        return new Scorer(null) {
-            @Override
-            public float score() throws IOException {
-                throw new IllegalStateException(message);
-            }
-            @Override
-            public int freq() throws IOException {
-                throw new IllegalStateException(message);
-            }
-            @Override
-            public int docID() {
-                throw new IllegalStateException(message);
-            }
-            @Override
-            public DocIdSetIterator iterator() {
-                throw new IllegalStateException(message);
-            }
-        };
-    }
-
     private static final class CommitPoint extends IndexCommit {
         private String segmentsFileName;
         private final Collection<String> files;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java
index e70780ec48c..04023b04977 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java
@@ -165,7 +165,7 @@ public abstract class AggregatorBase extends Aggregator {
     public DeferringBucketCollector getDeferringCollector() {
         // Default impl is a collector that selects the best buckets
         // but an alternative defer policy may be based on best docs.
-        return new BestBucketsDeferringCollector();
+        return new BestBucketsDeferringCollector(context());
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java
index 8a379d1ad82..43cefdca290 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java
@@ -20,6 +20,9 @@
 package org.elasticsearch.search.aggregations.bucket;
 
 import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.packed.PackedInts;
 import org.apache.lucene.util.packed.PackedLongValues;
 import org.elasticsearch.common.lucene.Lucene;
@@ -30,6 +33,7 @@ import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
 import org.elasticsearch.search.aggregations.BucketCollector;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
+import org.elasticsearch.search.aggregations.support.AggregationContext;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -56,6 +60,7 @@ public class BestBucketsDeferringCollector extends DeferringBucketCollector {
 
     final List<Entry> entries = new ArrayList<>();
     BucketCollector collector;
+    final AggregationContext aggContext;
     LeafReaderContext context;
     PackedLongValues.Builder docDeltas;
     PackedLongValues.Builder buckets;
@@ -64,7 +69,8 @@ public class BestBucketsDeferringCollector extends DeferringBucketCollector {
     LongHash selectedBuckets;
 
     /** Sole constructor. */
-    public BestBucketsDeferringCollector() {
+    public BestBucketsDeferringCollector(AggregationContext context) {
+        this.aggContext = context;
     }
 
     @Override
@@ -139,19 +145,34 @@ public class BestBucketsDeferringCollector extends DeferringBucketCollector {
         this.selectedBuckets = hash;
 
         collector.preCollection();
-        if (collector.needsScores()) {
-            throw new IllegalStateException("Cannot defer if scores are needed");
+        boolean needsScores = collector.needsScores();
+        Weight weight = null;
+        if (needsScores) {
+            weight = aggContext.searchContext().searcher()
+                        .createNormalizedWeight(aggContext.searchContext().query(), true);
         }
-
         for (Entry entry : entries) {
             final LeafBucketCollector leafCollector = collector.getLeafCollector(entry.context);
-            leafCollector.setScorer(Lucene.illegalScorer("A limitation of the " + SubAggCollectionMode.BREADTH_FIRST
-                    + " collection mode is that scores cannot be buffered along with document IDs"));
+            DocIdSetIterator docIt = null;
+            if (needsScores && entry.docDeltas.size() > 0) {
+                Scorer scorer = weight.scorer(entry.context);
+                // We don't need to check if the scorer is null
+                // since we are sure that there are documents to replay (entry.docDeltas it not empty).
+                docIt = scorer.iterator();
+                leafCollector.setScorer(scorer);
+            }
             final PackedLongValues.Iterator docDeltaIterator = entry.docDeltas.iterator();
             final PackedLongValues.Iterator buckets = entry.buckets.iterator();
             int doc = 0;
             for (long i = 0, end = entry.docDeltas.size(); i < end; ++i) {
                 doc += docDeltaIterator.next();
+                if (needsScores) {
+                    if (docIt.docID() < doc) {
+                        docIt.advance(doc);
+                    }
+                    // aggregations should only be replayed on matching documents
+                    assert docIt.docID() == doc;
+                }
                 final long bucket = buckets.next();
                 final long rebasedBucket = hash.find(bucket);
                 if (rebasedBucket != -1) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java
index 1c59711b646..eefaf63c62b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java
@@ -199,7 +199,6 @@ public abstract class TermsAggregator extends BucketsAggregator {
     @Override
     protected boolean shouldDefer(Aggregator aggregator) {
         return collectMode == SubAggCollectionMode.BREADTH_FIRST
-                && aggregator.needsScores() == false
                 && !aggsUsedForSorting.contains(aggregator);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
index 36d021a5f3f..2cd1fed7fbe 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
@@ -351,8 +351,7 @@ public class TopHitsIT extends ESIntegTestCase {
     }
 
 
-    public void testBreadthFirst() throws Exception {
-        // breadth_first will be ignored since we need scores
+    public void testBreadthFirstWithScoreNeeded() throws Exception {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .addAggregation(terms("terms")
                         .executionHint(randomExecutionHint())
@@ -382,6 +381,38 @@ public class TopHitsIT extends ESIntegTestCase {
         }
     }
 
+    public void testBreadthFirstWithAggOrderAndScoreNeeded() throws Exception {
+        SearchResponse response = client().prepareSearch("idx").setTypes("type")
+            .addAggregation(terms("terms")
+                .executionHint(randomExecutionHint())
+                .collectMode(SubAggCollectionMode.BREADTH_FIRST)
+                .field(TERMS_AGGS_FIELD)
+                .order(Terms.Order.aggregation("max", false))
+                .subAggregation(max("max").field(SORT_FIELD))
+                .subAggregation(topHits("hits").size(3))
+            ).get();
+
+        assertSearchResponse(response);
+
+        Terms terms = response.getAggregations().get("terms");
+        assertThat(terms, notNullValue());
+        assertThat(terms.getName(), equalTo("terms"));
+        assertThat(terms.getBuckets().size(), equalTo(5));
+        int id = 4;
+        for (Terms.Bucket bucket : terms.getBuckets()) {
+            assertThat(bucket, notNullValue());
+            assertThat(key(bucket), equalTo("val" + id));
+            assertThat(bucket.getDocCount(), equalTo(10L));
+            TopHits topHits = bucket.getAggregations().get("hits");
+            SearchHits hits = topHits.getHits();
+            assertThat(hits.totalHits(), equalTo(10L));
+            assertThat(hits.getHits().length, equalTo(3));
+
+            assertThat(hits.getAt(0).sourceAsMap().size(), equalTo(4));
+            id --;
+        }
+    }
+
     public void testBasicsGetProperty() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery())
                 .addAggregation(global("global").subAggregation(topHits("hits"))).execute().actionGet();
diff --git a/core/src/test/java/org/elasticsearch/search/profile/ProfileTests.java b/core/src/test/java/org/elasticsearch/search/profile/ProfileTests.java
index 83f6efaa150..afc1c7b3cbb 100644
--- a/core/src/test/java/org/elasticsearch/search/profile/ProfileTests.java
+++ b/core/src/test/java/org/elasticsearch/search/profile/ProfileTests.java
@@ -164,7 +164,7 @@ public class ProfileTests extends ESTestCase {
         final LeafCollector leafCollector = profileCollector.getLeafCollector(reader.leaves().get(0));
         assertThat(profileCollector.getTime(), greaterThan(0L));
         long time = profileCollector.getTime();
-        leafCollector.setScorer(Lucene.illegalScorer("dummy scorer"));
+        leafCollector.setScorer(null);
         assertThat(profileCollector.getTime(), greaterThan(time));
         time = profileCollector.getTime();
         leafCollector.collect(0);
diff --git a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
index 5d79c5580cb..14dec039435 100644
--- a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
@@ -635,9 +635,8 @@ elasticsearch will always use the `depth_first` collect_mode unless explicitly i
 Note that the `order` parameter can still be used to refer to data from a child aggregation when using the `breadth_first` setting - the parent
 aggregation understands that this child aggregation will need to be called first before any of the other child aggregations.
 
-WARNING: It is not possible to nest aggregations such as `top_hits` which require access to match score information under an aggregation that uses
-the `breadth_first` collection mode. This is because this would require a RAM buffer to hold the float score value for every document and
-this would typically be too costly in terms of RAM.
+WARNING: Nested aggregations such as `top_hits` which require access to score information under an aggregation that uses the `breadth_first`
+collection mode need to replay the query on the second pass but only for the documents belonging to the top buckets.
 
 [[search-aggregations-bucket-terms-aggregation-execution-hint]]
 ==== Execution hint

From c4554fedad100cb82a5cab9c7efcd127d4c4d205 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Wed, 4 May 2016 16:46:44 +0200
Subject: [PATCH 0065/1311] Fix build: restore illegalScorer still in use in
 ExpressionSearchScript

---
 .../elasticsearch/common/lucene/Lucene.java   | 50 +++++++++++++++++++
 1 file changed, 50 insertions(+)

diff --git a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
index fcc443c57b6..3540b3712ae 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
@@ -638,6 +638,56 @@ public class Lucene {
         }
     }
 
+    /**
+     * Return a Scorer that throws an ElasticsearchIllegalStateException
+     * on all operations with the given message.
+     */
+     public static Scorer illegalScorer(final String message) {
+         return new Scorer(null) {
+             @Override
+             public float score() throws IOException {
+                 throw new IllegalStateException(message);
+             }
+             @Override
+             public int freq() throws IOException {
+                 throw new IllegalStateException(message);
+             }
+             @Override
+             public int docID() {
+                 throw new IllegalStateException(message);
+             }
+             @Override
+             public DocIdSetIterator iterator() {
+                 throw new IllegalStateException(message);
+             }
+         };
+    }
+
+    /**
+     * Return a Scorer that throws an ElasticsearchIllegalStateException
+     * on all operations with the given message.
+     */
+    public static Scorer illegalScorer(final String message) {
+        return new Scorer(null) {
+            @Override
+            public float score() throws IOException {
+                throw new IllegalStateException(message);
+            }
+            @Override
+            public int freq() throws IOException {
+                throw new IllegalStateException(message);
+            }
+            @Override
+            public int docID() {
+                throw new IllegalStateException(message);
+            }
+            @Override
+            public DocIdSetIterator iterator() {
+                throw new IllegalStateException(message);
+            }
+        };
+    }
+
     private static final class CommitPoint extends IndexCommit {
         private String segmentsFileName;
         private final Collection<String> files;

From 8d3427b44dc67625f2e8d0e38bbe41e3fe6d3f05 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Wed, 4 May 2016 16:48:08 +0200
Subject: [PATCH 0066/1311] Fix build: restore illegalScorer still in use in
 ExpressionSearchScript (2)

---
 .../elasticsearch/common/lucene/Lucene.java   | 25 -------------------
 1 file changed, 25 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
index 3540b3712ae..8508a8a2e40 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
@@ -638,31 +638,6 @@ public class Lucene {
         }
     }
 
-    /**
-     * Return a Scorer that throws an ElasticsearchIllegalStateException
-     * on all operations with the given message.
-     */
-     public static Scorer illegalScorer(final String message) {
-         return new Scorer(null) {
-             @Override
-             public float score() throws IOException {
-                 throw new IllegalStateException(message);
-             }
-             @Override
-             public int freq() throws IOException {
-                 throw new IllegalStateException(message);
-             }
-             @Override
-             public int docID() {
-                 throw new IllegalStateException(message);
-             }
-             @Override
-             public DocIdSetIterator iterator() {
-                 throw new IllegalStateException(message);
-             }
-         };
-    }
-
     /**
      * Return a Scorer that throws an ElasticsearchIllegalStateException
      * on all operations with the given message.

From 223d67df4aa49f8f7b2c62b4ded2a0824713bc38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Tue, 3 May 2016 16:16:29 +0200
Subject: [PATCH 0067/1311] Consolidate query generation in QueryShardContext

Currently we have a lot of methods left in QueryShardContext that
take parsers or BytesReference arguments to do some xContent
parsing on the shard. While this still seems necessary in some cases
(e.g. percolation, phrase suggester), the shard context should only
be concerned with generating lucene queries from QueryBuilders.

This change removes all of the parseX() methods in favour of two
public methods toQuery(QueryBuilder) and toFilter(QueryBuilder) that
either call the query builders toFilter() or toQuery() method and
move all code required for parsing out to the respective callers.
---
 .../org/elasticsearch/index/IndexService.java | 10 ++-
 .../percolator/PercolatorFieldMapper.java     |  2 +-
 .../index/query/AbstractQueryBuilder.java     |  4 +-
 .../index/query/QueryShardContext.java        | 90 +++++--------------
 .../search/query/PostFilterParseElement.java  |  4 +-
 .../search/query/QueryParseElement.java       |  4 +-
 .../search/rescore/QueryRescorer.java         | 14 ---
 .../search/rescore/Rescorer.java              | 12 ---
 .../suggest/phrase/PhraseSuggester.java       | 14 ++-
 .../index/query/AbstractQueryTestCase.java    |  6 ++
 10 files changed, 55 insertions(+), 105 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java
index bd96727d28f..ea2cfe9f106 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexService.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexService.java
@@ -431,6 +431,7 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
         return nodeServicesProvider;
     }
 
+    @Override
     public IndexSettings getIndexSettings() {
         return indexSettings;
     }
@@ -598,18 +599,18 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
         }
     }
 
-    private Query parse(AliasMetaData alias, QueryShardContext parseContext) {
+    private Query parse(AliasMetaData alias, QueryShardContext shardContext) {
         if (alias.filter() == null) {
             return null;
         }
         try {
             byte[] filterSource = alias.filter().uncompressed();
             try (XContentParser parser = XContentFactory.xContent(filterSource).createParser(filterSource)) {
-                ParsedQuery parsedFilter = parseContext.parseInnerFilter(parser);
+                ParsedQuery parsedFilter = shardContext.toFilter(shardContext.newParseContext(parser).parseInnerQueryBuilder());
                 return parsedFilter == null ? null : parsedFilter.query();
             }
         } catch (IOException ex) {
-            throw new AliasFilterParsingException(parseContext.index(), alias.getAlias(), "Invalid alias filter", ex);
+            throw new AliasFilterParsingException(shardContext.index(), alias.getAlias(), "Invalid alias filter", ex);
         }
     }
 
@@ -759,6 +760,7 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
             return scheduledFuture != null;
         }
 
+        @Override
         public final void run() {
             try {
                 runInternal();
@@ -824,6 +826,7 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
             super(indexService, indexService.getIndexSettings().getTranslogSyncInterval());
         }
 
+        @Override
         protected String getThreadPool() {
             return ThreadPool.Names.FLUSH;
         }
@@ -849,6 +852,7 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
             indexService.maybeRefreshEngine();
         }
 
+        @Override
         protected String getThreadPool() {
             return ThreadPool.Names.REFRESH;
         }
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
index 6637dd8b762..10c7e46e353 100644
--- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
@@ -224,7 +224,7 @@ public class PercolatorFieldMapper extends FieldMapper {
         return queryBuilder.toQuery(context);
     }
 
-    static QueryBuilder<?> parseQueryBuilder(QueryParseContext context, XContentLocation location) {
+    private static QueryBuilder parseQueryBuilder(QueryParseContext context, XContentLocation location) {
         try {
             return context.parseInnerQueryBuilder();
         } catch (IOException e) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java
index ab04c1aff44..6e82e7059d8 100644
--- a/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java
@@ -239,14 +239,14 @@ public abstract class AbstractQueryBuilder<QB extends AbstractQueryBuilder<QB>>
         return getWriteableName();
     }
 
-    protected final void writeQueries(StreamOutput out, List<? extends QueryBuilder<?>> queries) throws IOException {
+    protected final static void writeQueries(StreamOutput out, List<? extends QueryBuilder> queries) throws IOException {
         out.writeVInt(queries.size());
         for (QueryBuilder<?> query : queries) {
             out.writeNamedWriteable(query);
         }
     }
 
-    protected final List<QueryBuilder<?>> readQueries(StreamInput in) throws IOException {
+    protected final static List<QueryBuilder<?>> readQueries(StreamInput in) throws IOException {
         List<QueryBuilder<?>> queries = new ArrayList<>();
         int size = in.readVInt();
         for (int i = 0; i < size; i++) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
index af87a986243..4aa72728bc8 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
@@ -26,7 +26,6 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
-
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.queryparser.classic.MapperQueryParser;
@@ -37,13 +36,9 @@ import org.apache.lucene.search.similarities.Similarity;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalysisService;
@@ -292,84 +287,45 @@ public class QueryShardContext extends QueryRewriteContext {
         return false;
     }
 
-    public ParsedQuery parse(BytesReference source) {
-        XContentParser parser = null;
-        try {
-            parser = XContentFactory.xContent(source).createParser(source);
-            return innerParse(parser);
-        } catch (ParsingException e) {
-            throw e;
-        } catch (Exception e) {
-            throw new ParsingException(parser == null ? null : parser.getTokenLocation(), "Failed to parse", e);
-        } finally {
-            if (parser != null) {
-                parser.close();
-            }
-        }
-    }
-
-    public ParsedQuery parse(XContentParser parser) {
-        try {
-            return innerParse(parser);
-        } catch(IOException e) {
-            throw new ParsingException(parser.getTokenLocation(), "Failed to parse", e);
-        }
-    }
-
-    /**
-     * Parses an inner filter, returning null if the filter should be ignored.
-     */
-    @Nullable
-    public ParsedQuery parseInnerFilter(XContentParser parser) throws IOException {
-        reset();
-        try {
-            Query filter = QueryBuilder.rewriteQuery(newParseContext(parser).parseInnerQueryBuilder(), this).toFilter(this);
+    public ParsedQuery toFilter(QueryBuilder queryBuilder) {
+        return toQuery(queryBuilder, q -> {
+            Query filter = q.toFilter(this);
             if (filter == null) {
                 return null;
             }
-            return new ParsedQuery(filter, copyNamedQueries());
-        } finally {
-            reset();
-        }
+            return filter;
+        });
     }
 
+    public ParsedQuery toQuery(QueryBuilder queryBuilder) {
+        return toQuery(queryBuilder, q -> {
+            Query query = q.toQuery(this);
+            if (query == null) {
+                query = Queries.newMatchNoDocsQuery("No query left after rewrite.");
+            }
+            return query;
+        });
+    }
 
-    private ParsedQuery innerParse(XContentParser parser) throws IOException, QueryShardException {
+    @FunctionalInterface
+    private interface CheckedFunction<T, R> {
+       R apply(T t) throws IOException;
+    }
+
+    private ParsedQuery toQuery(QueryBuilder queryBuilder, CheckedFunction<QueryBuilder, Query> filterOrQuery) {
         reset();
         try {
-            Query query = parseInnerQuery(parser);
-            return new ParsedQuery(query, copyNamedQueries());
-        } finally {
-            reset();
-        }
-    }
-
-    public Query parseInnerQuery(XContentParser parser) throws IOException {
-        return toQuery(this.newParseContext(parser).parseInnerQueryBuilder(), this);
-    }
-
-    public ParsedQuery toQuery(QueryBuilder<?> queryBuilder) {
-        reset();
-        try {
-            Query query = toQuery(queryBuilder, this);
-            return new ParsedQuery(query, copyNamedQueries());
+            QueryBuilder rewriteQuery = QueryBuilder.rewriteQuery(queryBuilder, this);
+            return new ParsedQuery(filterOrQuery.apply(rewriteQuery), copyNamedQueries());
         } catch(QueryShardException | ParsingException e ) {
             throw e;
         } catch(Exception e) {
             throw new QueryShardException(this, "failed to create query: {}", e, queryBuilder);
         } finally {
-            this.reset();
+            reset();
         }
     }
 
-    private static Query toQuery(final QueryBuilder<?> queryBuilder, final QueryShardContext context) throws IOException {
-        final Query query = QueryBuilder.rewriteQuery(queryBuilder, context).toQuery(context);
-        if (query == null) {
-            return Queries.newMatchNoDocsQuery("No query left after rewrite.");
-        }
-        return query;
-    }
-
     public final Index index() {
         return indexSettings.getIndex();
     }
diff --git a/core/src/main/java/org/elasticsearch/search/query/PostFilterParseElement.java b/core/src/main/java/org/elasticsearch/search/query/PostFilterParseElement.java
index 6995b6ff8a7..1b9fee22d76 100644
--- a/core/src/main/java/org/elasticsearch/search/query/PostFilterParseElement.java
+++ b/core/src/main/java/org/elasticsearch/search/query/PostFilterParseElement.java
@@ -20,6 +20,7 @@ package org.elasticsearch.search.query;
 
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.ParsedQuery;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -30,7 +31,8 @@ public class PostFilterParseElement implements SearchParseElement {
 
     @Override
     public void parse(XContentParser parser, SearchContext context) throws Exception {
-        ParsedQuery postFilter = context.getQueryShardContext().parseInnerFilter(parser);
+        QueryShardContext shardContext = context.getQueryShardContext();
+        ParsedQuery postFilter = shardContext.toFilter(shardContext.newParseContext(parser).parseInnerQueryBuilder());
         if (postFilter != null) {
             context.parsedPostFilter(postFilter);
         }
diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryParseElement.java b/core/src/main/java/org/elasticsearch/search/query/QueryParseElement.java
index 094a29cd6b1..cfa4ea21747 100644
--- a/core/src/main/java/org/elasticsearch/search/query/QueryParseElement.java
+++ b/core/src/main/java/org/elasticsearch/search/query/QueryParseElement.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.search.query;
 
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -30,6 +31,7 @@ public class QueryParseElement implements SearchParseElement {
 
     @Override
     public void parse(XContentParser parser, SearchContext context) throws Exception {
-        context.parsedQuery(context.getQueryShardContext().parse(parser));
+        QueryShardContext queryShardContext = context.getQueryShardContext();
+        context.parsedQuery(queryShardContext.toQuery(queryShardContext.newParseContext(parser).parseInnerQueryBuilder()));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java
index 319055639ac..29d62f3c7d3 100644
--- a/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java
+++ b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java
@@ -120,20 +120,6 @@ public final class QueryRescorer implements Rescorer {
         }
     }
 
-    private static final ObjectParser<QueryRescoreContext, QueryShardContext> RESCORE_PARSER = new ObjectParser<>("query", null);
-
-    static {
-        RESCORE_PARSER.declareObject(QueryRescoreContext::setQuery, (p, c) -> c.parse(p).query(), new ParseField("rescore_query"));
-        RESCORE_PARSER.declareFloat(QueryRescoreContext::setQueryWeight, new ParseField("query_weight"));
-        RESCORE_PARSER.declareFloat(QueryRescoreContext::setRescoreQueryWeight, new ParseField("rescore_query_weight"));
-        RESCORE_PARSER.declareString(QueryRescoreContext::setScoreMode, new ParseField("score_mode"));
-    }
-
-    @Override
-    public RescoreSearchContext parse(XContentParser parser, QueryShardContext context) throws IOException {
-        return RESCORE_PARSER.parse(parser, new QueryRescoreContext(this), context);
-    }
-
     private final static Comparator<ScoreDoc> SCORE_DOC_COMPARATOR = new Comparator<ScoreDoc>() {
         @Override
         public int compare(ScoreDoc o1, ScoreDoc o2) {
diff --git a/core/src/main/java/org/elasticsearch/search/rescore/Rescorer.java b/core/src/main/java/org/elasticsearch/search/rescore/Rescorer.java
index b475ca90db1..5e824aadc6c 100644
--- a/core/src/main/java/org/elasticsearch/search/rescore/Rescorer.java
+++ b/core/src/main/java/org/elasticsearch/search/rescore/Rescorer.java
@@ -23,8 +23,6 @@ import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.TopDocs;
 import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
@@ -65,16 +63,6 @@ public interface Rescorer {
     public Explanation explain(int topLevelDocId, SearchContext context, RescoreSearchContext rescoreContext,
             Explanation sourceExplanation) throws IOException;
 
-    /**
-     * Parses the {@link RescoreSearchContext} for this implementation
-     *
-     * @param parser  the parser to read the context from
-     * @param context the current shard context
-     * @return the parsed {@link RescoreSearchContext}
-     * @throws IOException if an {@link IOException} occurs while parsing the context
-     */
-    public RescoreSearchContext parse(XContentParser parser, QueryShardContext context) throws IOException;
-
     /**
      * Extracts all terms needed to execute this {@link Rescorer}. This method
      * is executed in a distributed frequency collection roundtrip for
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
index 11a0c10ee0e..9aed90f55e5 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
@@ -32,8 +32,11 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.ScriptService;
@@ -115,11 +118,14 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
                     // from the index for a correction, collateMatch is updated
                     final Map<String, Object> vars = suggestion.getCollateScriptParams();
                     vars.put(SUGGESTION_TEMPLATE_VAR_NAME, spare.toString());
-                    ScriptService scriptService = suggestion.getShardContext().getScriptService();
+                    QueryShardContext shardContext = suggestion.getShardContext();
+                    ScriptService scriptService = shardContext.getScriptService();
                     final ExecutableScript executable = scriptService.executable(collateScript, vars);
                     final BytesReference querySource = (BytesReference) executable.run();
-                    final ParsedQuery parsedQuery = suggestion.getShardContext().parse(querySource);
-                    collateMatch = Lucene.exists(searcher, parsedQuery.query());
+                    try (XContentParser parser = XContentFactory.xContent(querySource).createParser(querySource)) {
+                        final ParsedQuery parsedQuery = shardContext.toQuery(shardContext.newParseContext(parser).parseInnerQueryBuilder());
+                        collateMatch = Lucene.exists(searcher, parsedQuery.query());
+                    }
                 }
                 if (!collateMatch && !collatePrune) {
                     continue;
@@ -142,7 +148,7 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
         return response;
     }
 
-    private PhraseSuggestion.Entry buildResultEntry(SuggestionContext suggestion, CharsRefBuilder spare, double cutoffScore) {
+    private static PhraseSuggestion.Entry buildResultEntry(SuggestionContext suggestion, CharsRefBuilder spare, double cutoffScore) {
         spare.copyUTF8Bytes(suggestion.getText());
         return new PhraseSuggestion.Entry(new Text(spare.toString()), 0, spare.length(), cutoffScore);
     }
diff --git a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
index ba55413a27b..fac11ab1f78 100644
--- a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
@@ -593,6 +593,12 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
                 assertNotEquals("modifying the boost doesn't affect the corresponding lucene query", rewrite(firstLuceneQuery),
                         rewrite(thirdLuceneQuery));
             }
+
+            // check that context#isFilter is not changed by invoking toQuery/rewrite
+            boolean filterFlag = randomBoolean();
+            context.setIsFilter(filterFlag);
+            rewriteQuery(firstQuery, context).toQuery(context);
+            assertEquals("isFilter should be unchanged", filterFlag, context.isFilter());
         }
     }
 

From fad0705e3049bd1c3a74593e9adbfbfadbd82944 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 4 May 2016 10:23:49 -0400
Subject: [PATCH 0068/1311] [docs] Modernize README.textile

* camel_case all the things
* `?pretty` all the things (we tell people to do this in bug reports)

Closes #18136
---
 README.textile | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/README.textile b/README.textile
index 5c75844b108..e48495a5524 100644
--- a/README.textile
+++ b/README.textile
@@ -50,19 +50,19 @@ h3. Indexing
 Let's try and index some twitter like information. First, let's create a twitter user, and add some tweets (the @twitter@ index will be created automatically):
 
 <pre>
-curl -XPUT 'http://localhost:9200/twitter/user/kimchy' -d '{ "name" : "Shay Banon" }'
+curl -XPUT 'http://localhost:9200/twitter/user/kimchy?pretty' -d '{ "name" : "Shay Banon" }'
 
-curl -XPUT 'http://localhost:9200/twitter/tweet/1' -d '
+curl -XPUT 'http://localhost:9200/twitter/tweet/1?pretty' -d '
 {
     "user": "kimchy",
-    "postDate": "2009-11-15T13:12:00",
+    "post_date": "2009-11-15T13:12:00",
     "message": "Trying out Elasticsearch, so far so good?"
 }'
 
-curl -XPUT 'http://localhost:9200/twitter/tweet/2' -d '
+curl -XPUT 'http://localhost:9200/twitter/tweet/2?pretty' -d '
 {
     "user": "kimchy",
-    "postDate": "2009-11-15T14:12:12",
+    "post_date": "2009-11-15T14:12:12",
     "message": "Another tweet, will it be indexed?"
 }'
 </pre>
@@ -101,7 +101,7 @@ Just for kicks, let's get all the documents stored (we should see the user as we
 curl -XGET 'http://localhost:9200/twitter/_search?pretty=true' -d '
 {
     "query" : {
-        "matchAll" : {}
+        "match_all" : {}
     }
 }'
 </pre>
@@ -113,7 +113,7 @@ curl -XGET 'http://localhost:9200/twitter/_search?pretty=true' -d '
 {
     "query" : {
         "range" : {
-            "postDate" : { "from" : "2009-11-15T13:00:00", "to" : "2009-11-15T14:00:00" }
+            "post_date" : { "from" : "2009-11-15T13:00:00", "to" : "2009-11-15T14:00:00" }
         }
     }
 }'
@@ -130,19 +130,19 @@ Elasticsearch supports multiple indices, as well as multiple types per index. In
 Another way to define our simple twitter system is to have a different index per user (note, though that each index has an overhead). Here is the indexing curl's in this case:
 
 <pre>
-curl -XPUT 'http://localhost:9200/kimchy/info/1' -d '{ "name" : "Shay Banon" }'
+curl -XPUT 'http://localhost:9200/kimchy/info/1?pretty' -d '{ "name" : "Shay Banon" }'
 
-curl -XPUT 'http://localhost:9200/kimchy/tweet/1' -d '
+curl -XPUT 'http://localhost:9200/kimchy/tweet/1?pretty' -d '
 {
     "user": "kimchy",
-    "postDate": "2009-11-15T13:12:00",
+    "post_date": "2009-11-15T13:12:00",
     "message": "Trying out Elasticsearch, so far so good?"
 }'
 
-curl -XPUT 'http://localhost:9200/kimchy/tweet/2' -d '
+curl -XPUT 'http://localhost:9200/kimchy/tweet/2?pretty' -d '
 {
     "user": "kimchy",
-    "postDate": "2009-11-15T14:12:12",
+    "post_date": "2009-11-15T14:12:12",
     "message": "Another tweet, will it be indexed?"
 }'
 </pre>
@@ -152,11 +152,11 @@ The above will index information into the @kimchy@ index, with two types, @info@
 Complete control on the index level is allowed. As an example, in the above case, we would want to change from the default 5 shards with 1 replica per index, to only 1 shard with 1 replica per index (== per twitter user). Here is how this can be done (the configuration can be in yaml as well):
 
 <pre>
-curl -XPUT http://localhost:9200/another_user/ -d '
+curl -XPUT http://localhost:9200/another_user?pretty -d '
 {
     "index" : {
-        "numberOfShards" : 1,
-        "numberOfReplicas" : 1
+        "number_of_shards" : 1,
+        "number_of_replicas" : 1
     }
 }'
 </pre>
@@ -168,7 +168,7 @@ index (twitter user), for example:
 curl -XGET 'http://localhost:9200/kimchy,another_user/_search?pretty=true' -d '
 {
     "query" : {
-        "matchAll" : {}
+        "match_all" : {}
     }
 }'
 </pre>
@@ -179,7 +179,7 @@ Or on all the indices:
 curl -XGET 'http://localhost:9200/_search?pretty=true' -d '
 {
     "query" : {
-        "matchAll" : {}
+        "match_all" : {}
     }
 }'
 </pre>

From 5a0cfdd6af7365f5af80d9e2cb85a1d4421cf5c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Mon, 2 May 2016 18:40:52 +0200
Subject: [PATCH 0069/1311] Change scriptFields member in InnerHitBuilder to
 set

Adding random shuffling of xContent to InnterHitBuilderTests shows
that the scriptFields are stored in order as a list internally although
they are an unordered json objects in the query dsl.

This changes the internal representation to a set and updates
serialization accordingly.
---
 .../index/query/InnerHitBuilder.java          | 25 +++++++++++++------
 .../index/query/InnerHitBuilderTests.java     | 21 ++++++++--------
 2 files changed, 28 insertions(+), 18 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
index 2734dbc0e6b..61663713c8e 100644
--- a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
@@ -47,10 +47,12 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
 
 import static org.elasticsearch.common.xcontent.XContentParser.Token.END_OBJECT;
 
@@ -72,7 +74,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         PARSER.declareStringArray(InnerHitBuilder::setFieldDataFields, SearchSourceBuilder.FIELDDATA_FIELDS_FIELD);
         PARSER.declareField((p, i, c) -> {
             try {
-                List<ScriptField> scriptFields = new ArrayList<>();
+                Set<ScriptField> scriptFields = new HashSet<>();
                 for (XContentParser.Token token = p.nextToken(); token != END_OBJECT; token = p.nextToken()) {
                     scriptFields.add(new ScriptField(c));
                 }
@@ -132,7 +134,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
     private QueryBuilder<?> query = new MatchAllQueryBuilder();
     private List<SortBuilder<?>> sorts;
     private List<String> fieldDataFields;
-    private List<ScriptField> scriptFields;
+    private Set<ScriptField> scriptFields;
     private HighlightBuilder highlightBuilder;
     private FetchSourceContext fetchSourceContext;
     private Map<String, InnerHitBuilder> childInnerHits;
@@ -155,7 +157,11 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         fieldNames = (List<String>) in.readGenericValue();
         fieldDataFields = (List<String>) in.readGenericValue();
         if (in.readBoolean()) {
-            scriptFields = in.readList(ScriptField::new);
+            int size = in.readVInt();
+            scriptFields = new HashSet<>(size);
+            for (int i = 0; i < size; i++) {
+                scriptFields.add(new ScriptField(in));
+            }
         }
         fetchSourceContext = in.readOptionalStreamable(FetchSourceContext::new);
         if (in.readBoolean()) {
@@ -190,7 +196,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
             fieldDataFields = new ArrayList<>(other.fieldDataFields);
         }
         if (other.scriptFields != null) {
-            scriptFields = new ArrayList<>(other.scriptFields);
+            scriptFields = new HashSet<>(other.scriptFields);
         }
         if (other.fetchSourceContext != null) {
             fetchSourceContext = new FetchSourceContext(
@@ -240,7 +246,10 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         boolean hasScriptFields = scriptFields != null;
         out.writeBoolean(hasScriptFields);
         if (hasScriptFields) {
-            out.writeList(scriptFields);
+            out.writeVInt(scriptFields.size());
+            for (ScriptField scriptField : scriptFields) {
+                scriptField.writeTo(out);;
+            }
         }
         out.writeOptionalStreamable(fetchSourceContext);
         boolean hasSorts = sorts != null;
@@ -350,18 +359,18 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         return this;
     }
 
-    public List<ScriptField> getScriptFields() {
+    public Set<ScriptField> getScriptFields() {
         return scriptFields;
     }
 
-    public InnerHitBuilder setScriptFields(List<ScriptField> scriptFields) {
+    public InnerHitBuilder setScriptFields(Set<ScriptField> scriptFields) {
         this.scriptFields = scriptFields;
         return this;
     }
 
     public InnerHitBuilder addScriptField(String name, Script script) {
         if (scriptFields == null) {
-            scriptFields = new ArrayList<>();
+            scriptFields = new HashSet<>();
         }
         scriptFields.add(new ScriptField(name, script, false));
         return this;
diff --git a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
index a05093761d2..d2cbec890e8 100644
--- a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
@@ -22,11 +22,11 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.sameInstance;
-import static org.hamcrest.Matchers.nullValue;
-
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Supplier;
@@ -91,12 +91,13 @@ public class InnerHitBuilderTests extends ESTestCase {
         for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
             InnerHitBuilder innerHit = randomInnerHits(true, false);
             XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
-            if (randomBoolean()) {
-                builder.prettyPrint();
-            }
             innerHit.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            XContentBuilder shuffled = shuffleXContent(builder, Collections.emptySet());
+            if (randomBoolean()) {
+                shuffled.prettyPrint();
+            }
 
-            XContentParser parser = XContentHelper.createParser(builder.bytes());
+            XContentParser parser = XContentHelper.createParser(shuffled.bytes());
             QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.EMPTY);
             InnerHitBuilder secondInnerHits = InnerHitBuilder.fromXContent(context);
             assertThat(innerHit, not(sameInstance(secondInnerHits)));
@@ -202,7 +203,7 @@ public class InnerHitBuilderTests extends ESTestCase {
                 .innerHit(leafInnerHits);
         FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(nestedQueryBuilder);
         Map<String, InnerHitBuilder> innerHitBuilders = new HashMap<>();
-        ((AbstractQueryBuilder) functionScoreQueryBuilder).extractInnerHitBuilders(innerHitBuilders);
+        ((AbstractQueryBuilder<?>) functionScoreQueryBuilder).extractInnerHitBuilders(innerHitBuilders);
         assertThat(innerHitBuilders.get(leafInnerHits.getName()), notNullValue());
     }
 
@@ -220,7 +221,7 @@ public class InnerHitBuilderTests extends ESTestCase {
         innerHits.setTrackScores(randomBoolean());
         innerHits.setFieldNames(randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16)));
         innerHits.setFieldDataFields(randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16)));
-        innerHits.setScriptFields(randomListStuff(16, InnerHitBuilderTests::randomScript));
+        innerHits.setScriptFields(new HashSet<>(randomListStuff(16, InnerHitBuilderTests::randomScript)));
         FetchSourceContext randomFetchSourceContext;
         if (randomBoolean()) {
             randomFetchSourceContext = new FetchSourceContext(randomBoolean());
@@ -245,7 +246,7 @@ public class InnerHitBuilderTests extends ESTestCase {
         }
 
         if (includeQueryTypeOrPath) {
-            QueryBuilder query = new MatchQueryBuilder(randomAsciiOfLengthBetween(1, 16), randomAsciiOfLengthBetween(1, 16));
+            QueryBuilder<?> query = new MatchQueryBuilder(randomAsciiOfLengthBetween(1, 16), randomAsciiOfLengthBetween(1, 16));
             if (randomBoolean()) {
                 return new InnerHitBuilder(innerHits, randomAsciiOfLength(8), query);
             } else {
@@ -299,7 +300,7 @@ public class InnerHitBuilderTests extends ESTestCase {
             case 7:
                 if (randomBoolean()) {
                     instance.setScriptFields(randomValueOtherThan(instance.getScriptFields(), () -> {
-                        return randomListStuff(16, InnerHitBuilderTests::randomScript);}));
+                        return new HashSet<>(randomListStuff(16, InnerHitBuilderTests::randomScript));}));
                 } else {
                     SearchSourceBuilder.ScriptField script = randomScript();
                     instance.addScriptField(script.fieldName(), script.script());

From 34d90b041f4cd6232c83c1732c53a93f46d57d66 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Wed, 4 May 2016 18:17:10 +0200
Subject: [PATCH 0070/1311] Reorganise scripting docs (#18132)

* Reorganize scripting documentation

* Further changes to tidy up scripting docs

Closes #18116

* Add note about .lat/lon potentially returning null

* Added .value to expressions example

* Fixed two bad ASCIIDOC links
---
 docs/reference/modules.asciidoc               |   2 -
 docs/reference/modules/scripting.asciidoc     | 103 ++-
 .../scripting/advanced-scripting.asciidoc     |  20 +-
 .../modules/scripting/expression.asciidoc     | 120 +++
 .../modules/scripting/fields.asciidoc         | 232 ++++++
 .../modules/scripting/groovy.asciidoc         | 181 +++++
 .../modules/scripting/native.asciidoc         |  84 ++
 .../modules/{ => scripting}/painless.asciidoc |   8 +-
 .../modules/scripting/scripting.asciidoc      | 761 ------------------
 .../modules/scripting/security.asciidoc       | 136 +++-
 .../modules/scripting/using.asciidoc          | 238 ++++++
 11 files changed, 1108 insertions(+), 777 deletions(-)
 create mode 100644 docs/reference/modules/scripting/expression.asciidoc
 create mode 100644 docs/reference/modules/scripting/fields.asciidoc
 create mode 100644 docs/reference/modules/scripting/groovy.asciidoc
 create mode 100644 docs/reference/modules/scripting/native.asciidoc
 rename docs/reference/modules/{ => scripting}/painless.asciidoc (98%)
 delete mode 100644 docs/reference/modules/scripting/scripting.asciidoc
 create mode 100644 docs/reference/modules/scripting/using.asciidoc

diff --git a/docs/reference/modules.asciidoc b/docs/reference/modules.asciidoc
index b71d1224e7e..5a39cdfd790 100644
--- a/docs/reference/modules.asciidoc
+++ b/docs/reference/modules.asciidoc
@@ -94,8 +94,6 @@ include::modules/network.asciidoc[]
 
 include::modules/node.asciidoc[]
 
-include::modules/painless.asciidoc[]
-
 include::modules/plugins.asciidoc[]
 
 include::modules/scripting.asciidoc[]
diff --git a/docs/reference/modules/scripting.asciidoc b/docs/reference/modules/scripting.asciidoc
index 114740306ee..feb8113a6da 100644
--- a/docs/reference/modules/scripting.asciidoc
+++ b/docs/reference/modules/scripting.asciidoc
@@ -1,5 +1,104 @@
-include::scripting/scripting.asciidoc[]
+[[modules-scripting]]
+== Scripting
+
+The scripting module enables you to use scripts to evaluate custom
+expressions. For example, you could use a script to return "script fields"
+as part of a search request or evaluate a custom score for a query.
+
+TIP: Elasticsearch now has a built-in scripting language called _Painless_
+that provides a more secure alternative for implementing
+scripts for Elasticsearch. We encourage you to try it out --
+for more information, see <<modules-scripting-painless, Painless Scripting Language>>.
+
+The default scripting language is http://groovy-lang.org/[groovy].
+Additional `lang` plugins enable you to run scripts written in other languages.
+Everywhere a script can be used, you can include a `lang` parameter
+to specify the language of the script.
+
+[float]
+=== General-purpose languages:
+
+These languages can be used for any purpose in the scripting APIs,
+and give the most flexibility.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Language
+    |Sandboxed
+    |Required plugin
+
+|<<modules-scripting-painless, `painless`>>
+    |yes
+    |built-in
+
+|<<modules-scripting-groovy, `groovy`>>
+    |<<modules-scripting-security, no>>
+    |built-in
+
+|{plugins}/lang-javascript.html[`javascript`]
+    |<<modules-scripting-security, no>>
+    |{plugins}/lang-javascript.html[`lang-javascript`]
+
+|{plugins}/lang-python.html[`python`]
+    |<<modules-scripting-security, no>>
+    |{plugins}/lang-python.html[`lang-python`]
+
+|=======================================================================
+
+[float]
+=== Special-purpose languages:
+
+These languages are less flexible, but typically have higher performance for
+certain tasks.
+
+[cols="<,<,<,<",options="header",]
+|=======================================================================
+|Language
+    |Sandboxed
+    |Required plugin
+    |Purpose
+
+|<<modules-scripting-expression, `expression`>>
+    |yes
+    |built-in
+    |fast custom ranking and sorting
+
+|<<search-template, `mustache`>>
+    |yes
+    |built-in
+    |templates
+
+|<<modules-scripting-native, `java`>>
+    |n/a
+    |you write it!
+    |expert API
+
+|=======================================================================
+
+[WARNING]
+.Scripts and security
+=================================================
+
+Languages that are sandboxed are designed with security in mind. However, non-
+sandboxed languages can be a security issue, please read
+<<modules-scripting-security, Scripting and security>> for more details.
+
+=================================================
+
+
+include::scripting/using.asciidoc[]
+
+include::scripting/fields.asciidoc[]
+
+include::scripting/security.asciidoc[]
+
+include::scripting/groovy.asciidoc[]
+
+include::scripting/painless.asciidoc[]
+
+include::scripting/expression.asciidoc[]
+
+include::scripting/native.asciidoc[]
 
 include::scripting/advanced-scripting.asciidoc[]
 
-include::scripting/security.asciidoc[]
diff --git a/docs/reference/modules/scripting/advanced-scripting.asciidoc b/docs/reference/modules/scripting/advanced-scripting.asciidoc
index 206a2fec50d..b4053d76598 100644
--- a/docs/reference/modules/scripting/advanced-scripting.asciidoc
+++ b/docs/reference/modules/scripting/advanced-scripting.asciidoc
@@ -1,13 +1,17 @@
 [[modules-advanced-scripting]]
-=== Text scoring in scripts
+=== Advanced text scoring in scripts
 
+experimental[The functionality described on this page is considered experimental and may be changed or removed in a future release]
 
-Text features, such as term or document frequency for a specific term can be accessed in scripts (see <<modules-scripting, scripting documentation>> ) with the `_index` variable. This can be useful if, for example, you want to implement your own scoring model using for example a script inside a <<query-dsl-function-score-query,function score query>>.
+Text features, such as term or document frequency for a specific term can be
+accessed in scripts with the `_index` variable. This can be useful if, for
+example, you want to implement your own scoring model using for example a
+script inside a <<query-dsl-function-score-query,function score query>>.
 Statistics over the document collection are computed *per shard*, not per
 index.
 
 [float]
-==== Nomenclature:
+=== Nomenclature:
 
 
 [horizontal]
@@ -33,7 +37,7 @@ depending on the shard the current document resides in.
 
 
 [float]
-==== Shard statistics:
+=== Shard statistics:
 
 `_index.numDocs()`::
 
@@ -49,7 +53,7 @@ depending on the shard the current document resides in.
 
 
 [float]
-==== Field statistics:
+=== Field statistics:
 
 Field statistics can be accessed with a subscript operator like this:
 `_index['FIELD']`.
@@ -74,7 +78,7 @@ depending on the shard the current document resides in.
 The number of terms in a field cannot be accessed using the `_index` variable. See <<token-count>> for how to do that.
 
 [float]
-==== Term statistics:
+=== Term statistics:
 
 Term statistics for a field can be accessed with a subscript operator like
 this: `_index['FIELD']['TERM']`. This will never return null, even if term or field does not exist.
@@ -101,7 +105,7 @@ affect is your set the <<index-options,`index_options`>> to `docs`.
 
 
 [float]
-==== Term positions, offsets and payloads:
+=== Term positions, offsets and payloads:
 
 If you need information on the positions of terms in a field, call
 `_index['FIELD'].get('TERM', flag)` where flag can be
@@ -174,7 +178,7 @@ return score;
 
 
 [float]
-==== Term vectors:
+=== Term vectors:
 
 The `_index` variable can only be used to gather statistics for single terms. If you want to use information on all terms in a field, you must store the term vectors (see <<term-vector>>). To access them, call
 `_index.termVectors()` to get a
diff --git a/docs/reference/modules/scripting/expression.asciidoc b/docs/reference/modules/scripting/expression.asciidoc
new file mode 100644
index 00000000000..d941fa2f4a4
--- /dev/null
+++ b/docs/reference/modules/scripting/expression.asciidoc
@@ -0,0 +1,120 @@
+[[modules-scripting-expression]]
+=== Lucene Expressions Language
+
+Lucene's expressions compile a `javascript` expression to bytecode. They are
+designed for high-performance custom ranking and sorting functions and are
+enabled for `inline` and `stored` scripting by default.
+
+[float]
+=== Performance
+
+Expressions were designed to have competitive performance with custom Lucene code.
+This performance is due to having low per-document overhead as opposed to other
+scripting engines: expressions do more "up-front".
+
+This allows for very fast execution, even faster than if you had written a `native` script.
+
+[float]
+=== Syntax
+
+Expressions support a subset of javascript syntax: a single expression.
+
+See the link:http://lucene.apache.org/core/6_0_0/expressions/index.html?org/apache/lucene/expressions/js/package-summary.html[expressions module documentation]
+for details on what operators and functions are available.
+
+Variables in `expression` scripts are available to access:
+
+* document fields, e.g. `doc['myfield'].value`
+* variables and methods that the field supports, e.g. `doc['myfield'].empty`
+* Parameters passed into the script, e.g. `mymodifier`
+* The current document's score, `_score` (only available when used in a `script_score`)
+
+You can use Expressions scripts for `script_score`, `script_fields`, sort scripts, and numeric aggregation
+scripts, simply set the `lang` parameter to `expression`.
+
+[float]
+=== Numeric field API
+[cols="<,<",options="header",]
+|=======================================================================
+|Expression |Description
+|`doc['field_name'].value` |The value of the field, as a `double`
+
+|`doc['field_name'].empty` |A boolean indicating if the field has no
+values within the doc.
+
+|`doc['field_name'].min()` |The minimum value of the field in this document.
+
+|`doc['field_name'].max()` |The maximum value of the field in this document.
+
+|`doc['field_name'].median()` |The median value of the field in this document.
+
+|`doc['field_name'].avg()` |The average of the values in this document.
+
+|`doc['field_name'].sum()` |The sum of the values in this document.
+
+|`doc['field_name'].count()` |The number of values in this document.
+|=======================================================================
+
+When a document is missing the field completely, by default the value will be treated as `0`.
+You can treat it as another value instead, e.g. `doc['myfield'].empty ? 100 : doc['myfield'].value`
+
+When a document has multiple values for the field, by default the minimum value is returned.
+You can choose a different value instead, e.g. `doc['myfield'].sum()`.
+
+When a document is missing the field completely, by default the value will be treated as `0`.
+
+Boolean fields are exposed as numerics, with `true` mapped to `1` and `false` mapped to `0`.
+For example: `doc['on_sale'].value ? doc['price'].value * 0.5 : doc['price'].value`
+
+[float]
+=== Date field API
+Date fields are treated as the number of milliseconds since January 1, 1970 and
+support the Numeric Fields API above, with these additional methods:
+
+[cols="<,<",options="header",]
+|=======================================================================
+|Expression |Description
+|`doc['field_name'].getYear()` |Year component, e.g. `1970`.
+
+|`doc['field_name'].getMonth()` |Month component (0-11), e.g. `0` for January.
+
+|`doc['field_name'].getDayOfMonth()` |Day component, e.g. `1` for the first of the month.
+
+|`doc['field_name'].getHourOfDay()` |Hour component (0-23)
+
+|`doc['field_name'].getMinutes()` |Minutes component (0-59)
+
+|`doc['field_name'].getSeconds()` |Seconds component (0-59)
+|=======================================================================
+
+The following example shows the difference in years between the `date` fields date0 and date1:
+
+`doc['date1'].getYear() - doc['date0'].getYear()`
+
+[float]
+=== `geo_point` field API
+[cols="<,<",options="header",]
+|=======================================================================
+|Expression |Description
+|`doc['field_name'].empty` |A boolean indicating if the field has no
+values within the doc.
+
+|`doc['field_name'].lat` |The latitude of the geo point, or `null`.
+
+|`doc['field_name'].lon` |The longitude of the geo point, or `null`.
+|=======================================================================
+
+The following example computes distance in kilometers from Washington, DC:
+
+`haversin(38.9072, 77.0369, doc['field_name'].lat, doc['field_name'].lon)`
+
+In this example the coordinates could have been passed as parameters to the script,
+e.g. based on geolocation of the user.
+
+[float]
+=== Limitations
+
+There are a few limitations relative to other script languages:
+
+* Only numeric, boolean, date, and geo_point fields may be accessed
+* Stored fields are not available
diff --git a/docs/reference/modules/scripting/fields.asciidoc b/docs/reference/modules/scripting/fields.asciidoc
new file mode 100644
index 00000000000..9ddf8267db6
--- /dev/null
+++ b/docs/reference/modules/scripting/fields.asciidoc
@@ -0,0 +1,232 @@
+[[modules-scripting-fields]]
+=== Accessing document fields and special variables
+
+Depending on where a script is used, it will have access to certain special
+variables and document fields.
+
+[float]
+== Update scripts
+
+A script used in the <<docs-update,update>>,
+<<docs-update-by-query,update-by-query>>, or <<docs-reindex,reindex>>
+API will have access to the `ctx` variable which exposes:
+
+[horizontal]
+`ctx._source`::     Access to the document <<mapping-source-field,`_source` field>>.
+`ctx.op`::          The operation that should be applied to the document: `index` or `delete`.
+`ctx._index` etc::  Access to <<mapping-fields,document meta-fields>>, some of which may be read-only.
+
+[float]
+== Search and Aggregation scripts
+
+With the exception of <<search-request-script-fields,script fields>> which are
+executed once per search hit, scripts used in search and aggregations will be
+executed once for every document which might match a query or an aggregation.
+Depending on how many documents you have, this could mean millions or billions
+of executions: these scripts need to be fast!
+
+Field values can be accessed from a script using
+<<modules-scripting-doc-vals,doc-values>>, or
+<<modules-scripting-stored,stored fields or `_source` field>>, which are explained below.
+
+Scripts may also have access to the document's relevance
+<<scripting-score,`_score`>> and, via the experimental `_index` variable,
+to term statistics for <<modules-advanced-scripting,advanced text scoring>>.
+
+[[scripting-score]]
+[float]
+=== Accessing the score of a document within a script
+
+Scripts used in the <<query-dsl-function-score-query,`function_score` query>>,
+in <<search-request-sort,script-based sorting>>, or in
+<<search-aggregations,aggregations>> have access to the `_score` variable which
+represents the current relevance score of a document.
+
+Here's an example of using a script in a
+<<query-dsl-function-score-query,`function_score` query>> to alter the
+relevance `_score` of each document:
+
+[source,js]
+-------------------------------------
+PUT my_index/my_type/1
+{
+  "text": "quick brown fox",
+  "popularity": 1
+}
+
+PUT my_index/my_type/2
+{
+  "text": "quick fox",
+  "popularity": 5
+}
+
+GET my_index/_search
+{
+  "query": {
+    "function_score": {
+      "query": {
+        "match": {
+          "text": "quick brown fox"
+        }
+      },
+      "script_score": {
+        "script": {
+          "lang": "expression",
+          "inline": "_score * doc['popularity']"
+        }
+      }
+    }
+  }
+}
+-------------------------------------
+// AUTOSENSE
+
+
+[float]
+[[modules-scripting-doc-vals]]
+=== Doc Values
+
+By far the fastest most efficient way to access a field value from a
+script is to use the `doc['field_name']` syntax, which retrieves the field
+value from <<doc-values,doc values>>. Doc values are a columnar field value
+store, enabled by default on all fields except for <<text,analyzed `text` fields>>.
+
+[source,js]
+-------------------------------
+PUT my_index/my_type/1
+{
+  "cost_price": 100
+}
+
+GET my_index/_search
+{
+  "script_fields": {
+    "sales_price": {
+      "script": {
+        "lang":   "expression",
+        "inline": "doc['cost_price'] * markup",
+        "params": {
+          "markup": 0.2
+        }
+      }
+    }
+  }
+}
+-------------------------------
+// AUTOSENSE
+
+Doc-values can only return "simple" field values like numbers, dates, geo-
+points, terms, etc, or arrays of these values if the field is multi-valued.
+It cannot return JSON objects.
+
+[NOTE]
+.Doc values and `text` fields
+===================================================
+
+The `doc['field']` syntax can also be used for <<text,analyzed `text` fields>>
+if <<fielddata,`fielddata`>> is enabled, but *BEWARE*: enabling fielddata on a
+`text` field requires loading all of the terms into the JVM heap, which can be
+very expensive both in terms of memory and CPU.  It seldom makes sense to
+access `text` fields from scripts.
+
+===================================================
+
+[float]
+[[modules-scripting-stored]]
+=== Stored Fields and `_source`
+
+_Stored fields_ -- fields explicitly marked as
+<<mapping-store,`"store": true`>> -- can be accessed using the
+`_fields['field_name'].value` or `_fields['field_name'].values` syntax.
+
+The document <<mapping-source-field,`_source`>>, which is really just a
+special stored field,  can be accessed using the `_source.field_name` syntax.
+The `_source` is loaded as a map-of-maps, so properties within object fields
+can be accessed as, for example, `_source.name.first`.
+
+[IMPORTANT]
+.Prefer doc-values to stored fields
+=========================================================
+
+Stored fields (which includes the stored `_source` field) are much slower than
+doc-values.  They are  optimised for returning several fields per result,
+while doc values are optimised for accessing the value of a specific field in
+many documents.
+
+
+It makes sense to use `_source` or stored fields when generating a
+<<search-request-script-fields,script field>> for the top ten hits from a search
+result but, for other search and aggregation use cases, always prefer using
+doc values.
+=========================================================
+
+
+For instance:
+
+[source,js]
+-------------------------------
+PUT my_index
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "title": { <1>
+          "type": "text"
+        },
+        "first_name": {
+          "type": "text",
+          "store": true
+        },
+        "last_name": {
+          "type": "text",
+          "store": true
+        }
+      }
+    }
+  }
+}
+
+PUT my_index/my_type/1
+{
+  "title": "Mr",
+  "first_name": "Barry",
+  "last_name": "White"
+}
+
+GET my_index/_search
+{
+  "script_fields": {
+    "source": {
+      "script": {
+        "lang": "groovy",
+        "inline": "_source.title + ' ' + _source.first_name + ' ' + _source.last_name" <2>
+      }
+    },
+    "stored_fields": {
+      "script": {
+        "lang": "groovy",
+        "inline": "_fields['first_name'].value + ' ' + _fields['last_name'].value"
+      }
+    }
+  }
+}
+-------------------------------
+// AUTOSENSE
+<1> The `title` field is not stored and so cannot be used with the `_fields[]` syntax.
+<2> The `title` field can still be accessed from the `_source`.
+
+[TIP]
+.Stored vs `_source`
+=======================================================
+
+The `_source` field is just a special stored field, so the performance is
+similar to that of other stored fields.  The `_source` provides access to the
+original document body that was indexed (including the ability to distinguish
+`null` values from empty fields, single-value arrays from plain scalars, etc).
+
+The only time it really makes sense to use stored fields instead of the
+`_source` field is when the `_source` is very large and it is less costly to
+access a few small stored fields instead of the entire `_source`.
+
+=======================================================
+
diff --git a/docs/reference/modules/scripting/groovy.asciidoc b/docs/reference/modules/scripting/groovy.asciidoc
new file mode 100644
index 00000000000..60b64e0131a
--- /dev/null
+++ b/docs/reference/modules/scripting/groovy.asciidoc
@@ -0,0 +1,181 @@
+[[modules-scripting-groovy]]
+=== Groovy Scripting Language
+
+Groovy is the default scripting language available in Elasticsearch.  Although
+limited by the <<java-security-manager,Java Security Manager>>, it is not a
+sandboxed language and only `file` scripts may be used by default.
+
+Enabling `inline` or `stored` Groovy scripting is a security risk and should
+only be considered if your Elasticsearch cluster is protected from the outside
+world. Even a simple `while (true) { }` loop could behave as a denial-of-
+service attack on your cluster.
+
+See <<modules-scripting-security, Scripting and Security>> for details
+on security issues with scripts, including how to customize class
+whitelisting.
+
+[float]
+=== Doc value properties and methods
+
+Doc values in Groovy support the following properties and methods (depending
+on the underlying field type):
+
+`doc['field_name'].value`::
+    The native value of the field. For example, if its a short type, it will be short.
+
+`doc['field_name'].values`::
+    The native array values of the field. For example, if its a short type,
+     it will be short[]. Remember, a field can have several values within a
+     single doc. Returns an empty array if the field has no values.
+
+`doc['field_name'].empty`::
+    A boolean indicating if the field has no values within the doc.
+
+`doc['field_name'].multiValued`::
+    A boolean indicating that the field has several values within the corpus.
+
+`doc['field_name'].lat`::
+    The latitude of a geo point type, or `null`.
+
+`doc['field_name'].lon`::
+    The longitude of a geo point type, or `null`.
+
+`doc['field_name'].lats`::
+    The latitudes of a geo point type, or an empty array.
+
+`doc['field_name'].lons`::
+    The longitudes of a geo point type, or an empty array.
+
+`doc['field_name'].distance(lat, lon)`::
+    The `plane` distance (in meters) of this geo point field from the provided lat/lon.
+
+`doc['field_name'].distanceWithDefault(lat, lon, default)`::
+    The `plane` distance (in meters) of this geo point field from the provided lat/lon with a default value.
+
+`doc['field_name'].distanceInMiles(lat, lon)`::
+    The `plane` distance (in miles) of this geo point field from the provided lat/lon.
+
+`doc['field_name'].distanceInMilesWithDefault(lat, lon, default)`::
+    The `plane` distance (in miles) of this geo point field from the provided lat/lon with a default value.
+
+`doc['field_name'].distanceInKm(lat, lon)`::
+    The `plane` distance (in km) of this geo point field from the provided lat/lon.
+
+`doc['field_name'].distanceInKmWithDefault(lat, lon, default)`::
+    The `plane` distance (in km) of this geo point field from the provided lat/lon with a default value.
+
+`doc['field_name'].arcDistance(lat, lon)`::
+    The `arc` distance (in meters) of this geo point field from the provided lat/lon.
+
+`doc['field_name'].arcDistanceWithDefault(lat, lon, default)`::
+    The `arc` distance (in meters) of this geo point field from the provided lat/lon with a default value.
+
+`doc['field_name'].arcDistanceInMiles(lat, lon)`::
+    The `arc` distance (in miles) of this geo point field from the provided lat/lon.
+
+`doc['field_name'].arcDistanceInMilesWithDefault(lat, lon, default)`::
+    The `arc` distance (in miles) of this geo point field from the provided lat/lon with a default value.
+
+`doc['field_name'].arcDistanceInKm(lat, lon)`::
+    The `arc` distance (in km) of this geo point field from the provided lat/lon.
+
+`doc['field_name'].arcDistanceInKmWithDefault(lat, lon, default)`::
+    The `arc` distance (in km) of this geo point field from the provided lat/lon with a default value.
+
+`doc['field_name'].factorDistance(lat, lon)`::
+    The distance factor of this geo point field from the provided lat/lon.
+
+`doc['field_name'].factorDistance(lat, lon, default)`::
+    The distance factor of this geo point field from the provided lat/lon with a default value.
+
+`doc['field_name'].geohashDistance(geohash)`::
+    The `arc` distance (in meters) of this geo point field from the provided geohash.
+
+`doc['field_name'].geohashDistanceInKm(geohash)`::
+    The `arc` distance (in km) of this geo point field from the provided geohash.
+
+`doc['field_name'].geohashDistanceInMiles(geohash)`::
+    The `arc` distance (in miles) of this geo point field from the provided geohash.
+
+
+[float]
+=== Groovy Built In Functions
+
+There are several built in functions that can be used within scripts.
+They include:
+
+[cols="<,<",options="header",]
+|=======================================================================
+|Function |Description
+|`sin(a)` |Returns the trigonometric sine of an angle.
+
+|`cos(a)` |Returns the trigonometric cosine of an angle.
+
+|`tan(a)` |Returns the trigonometric tangent of an angle.
+
+|`asin(a)` |Returns the arc sine of a value.
+
+|`acos(a)` |Returns the arc cosine of a value.
+
+|`atan(a)` |Returns the arc tangent of a value.
+
+|`toRadians(angdeg)` |Converts an angle measured in degrees to an
+approximately equivalent angle measured in radians
+
+|`toDegrees(angrad)` |Converts an angle measured in radians to an
+approximately equivalent angle measured in degrees.
+
+|`exp(a)` |Returns Euler's number _e_ raised to the power of value.
+
+|`log(a)` |Returns the natural logarithm (base _e_) of a value.
+
+|`log10(a)` |Returns the base 10 logarithm of a value.
+
+|`sqrt(a)` |Returns the correctly rounded positive square root of a
+value.
+
+|`cbrt(a)` |Returns the cube root of a double value.
+
+|`IEEEremainder(f1, f2)` |Computes the remainder operation on two
+arguments as prescribed by the IEEE 754 standard.
+
+|`ceil(a)` |Returns the smallest (closest to negative infinity) value
+that is greater than or equal to the argument and is equal to a
+mathematical integer.
+
+|`floor(a)` |Returns the largest (closest to positive infinity) value
+that is less than or equal to the argument and is equal to a
+mathematical integer.
+
+|`rint(a)` |Returns the value that is closest in value to the argument
+and is equal to a mathematical integer.
+
+|`atan2(y, x)` |Returns the angle _theta_ from the conversion of
+rectangular coordinates (_x_, _y_) to polar coordinates (r,_theta_).
+
+|`pow(a, b)` |Returns the value of the first argument raised to the
+power of the second argument.
+
+|`round(a)` |Returns the closest _int_ to the argument.
+
+|`random()` |Returns a random _double_ value.
+
+|`abs(a)` |Returns the absolute value of a value.
+
+|`max(a, b)` |Returns the greater of two values.
+
+|`min(a, b)` |Returns the smaller of two values.
+
+|`ulp(d)` |Returns the size of an ulp of the argument.
+
+|`signum(d)` |Returns the signum function of the argument.
+
+|`sinh(x)` |Returns the hyperbolic sine of a value.
+
+|`cosh(x)` |Returns the hyperbolic cosine of a value.
+
+|`tanh(x)` |Returns the hyperbolic tangent of a value.
+
+|`hypot(x, y)` |Returns sqrt(_x2_ + _y2_) without intermediate overflow
+or underflow.
+|=======================================================================
diff --git a/docs/reference/modules/scripting/native.asciidoc b/docs/reference/modules/scripting/native.asciidoc
new file mode 100644
index 00000000000..1b99673a90f
--- /dev/null
+++ b/docs/reference/modules/scripting/native.asciidoc
@@ -0,0 +1,84 @@
+[[modules-scripting-native]]
+=== Native (Java) Scripts
+
+Sometimes `groovy` and <<modules-scripting-expression, expression>> aren't enough. For those times you can
+implement a native script.
+
+The best way to implement a native script is to write a plugin and install it.
+The plugin {plugins}/plugin-authors.html[documentation] has more information on
+how to write a plugin so that Elasticsearch will properly load it.
+
+To register the actual script you'll need to implement `NativeScriptFactory`
+to construct the script. The actual script will extend either
+`AbstractExecutableScript` or `AbstractSearchScript`. The second one is likely
+the most useful and has several helpful subclasses you can extend like
+`AbstractLongSearchScript`, `AbstractDoubleSearchScript`, and
+`AbstractFloatSearchScript`. Finally, your plugin should register the native
+script by declaring the `onModule(ScriptModule)` method.
+
+If you squashed the whole thing into one class it'd look like:
+
+[source,java]
+--------------------------------------------------
+public class MyNativeScriptPlugin extends Plugin {
+    @Override
+    public String name() {
+        return "my-native-script";
+    }
+    @Override
+    public String description() {
+        return "my native script that does something great";
+    }
+    public void onModule(ScriptModule scriptModule) {
+        scriptModule.registerScript("my_script", MyNativeScriptFactory.class);
+    }
+
+    public static class MyNativeScriptFactory implements NativeScriptFactory {
+        @Override
+        public ExecutableScript newScript(@Nullable Map<String, Object> params) {
+            return new MyNativeScript();
+        }
+        @Override
+        public boolean needsScores() {
+            return false;
+        }
+    }
+
+    public static class MyNativeScript extends AbstractFloatSearchScript {
+        @Override
+        public float runAsFloat() {
+            float a = (float) source().get("a");
+            float b = (float) source().get("b");
+            return a * b;
+        }
+    }
+}
+--------------------------------------------------
+
+You can execute the script by specifying its `lang` as `native`, and the name
+of the script as the `id`:
+
+[source,js]
+--------------------------------------------------
+curl -XPOST localhost:9200/_search -d '{
+  "query": {
+    "function_score": {
+      "query": {
+        "match": {
+          "body": "foo"
+        }
+      },
+      "functions": [
+        {
+          "script_score": {
+            "script": {
+                "id": "my_script",
+                "lang" : "native"
+            }
+          }
+        }
+      ]
+    }
+  }
+}'
+--------------------------------------------------
diff --git a/docs/reference/modules/painless.asciidoc b/docs/reference/modules/scripting/painless.asciidoc
similarity index 98%
rename from docs/reference/modules/painless.asciidoc
rename to docs/reference/modules/scripting/painless.asciidoc
index e1234777efa..1937bd9947b 100644
--- a/docs/reference/modules/painless.asciidoc
+++ b/docs/reference/modules/scripting/painless.asciidoc
@@ -1,10 +1,12 @@
 [[modules-scripting-painless]]
-== Painless Scripting Language
+=== Painless Scripting Language
 
 experimental[The Painless scripting language is new and is still marked as experimental. The syntax or API may be changed in the future in non-backwards compatible ways if required.]
 
-_Painless_ is a simple, secure scripting language built in to Elasticsearch as a module.
-It is designed specifically for use with Elasticsearch and can safely be used dynamically.
+_Painless_ is a simple, secure scripting language available in Elasticsearch
+by default. It is designed specifically for use with Elasticsearch and can
+safely be used with `inline` and `stored` scripting, which is enabled by
+default.
 
 A Painless script is essentially a single function. Painless does not provide support
 for defining multiple functions within a script. The Painless syntax is similar to
diff --git a/docs/reference/modules/scripting/scripting.asciidoc b/docs/reference/modules/scripting/scripting.asciidoc
deleted file mode 100644
index fad96978be9..00000000000
--- a/docs/reference/modules/scripting/scripting.asciidoc
+++ /dev/null
@@ -1,761 +0,0 @@
-[[modules-scripting]]
-== Scripting
-
-The scripting module enables you to use scripts to evaluate custom
-expressions. For example, you could use a script to return "script fields"
-as part of a search request or evaluate a custom score for a query.
-
-TIP: Elasticsearch now has a built-in scripting language called _Painless_
-that provides a more secure alternative for implementing
-scripts for Elasticsearch. We encourage you to try it out--
-for more information, see <<modules-scripting-painless, Painless Scripting Language>>.
-
-The default scripting language is http://groovy-lang.org/[groovy] 
-(http://mvel.codehaus.org/[mvel] was the default in 1.3.x and earlier). 
-
-Additional `lang` plugins enable you to run scripts written in other languages. 
-Everywhere a script can be used, you can include a `lang` parameter
-to specify the language of the script. Plugins are available for following languages:
-
-[cols="<,<,<",options="header",]
-|=======================================================================
-|Language   |Sandboxed |Required plugin
-|groovy     |no        |built-in
-|expression |yes       |built-in
-|mustache   |yes       |built-in
-|painless   |yes       |built-in (module)
-|javascript |no        |{plugins}/lang-javascript.html[elasticsearch-lang-javascript]
-|python     |no        |{plugins}/lang-python.html[elasticsearch-lang-python]
-|=======================================================================
-
-.Groovy dynamic scripting off by default from v1.4.3
-[IMPORTANT]
-===================================================
-
-Groovy dynamic scripting is off by default. This prevents Groovy scripts
-from being accepted as part of a request or retrieved from the 
-`.scripts` index. You can still use Groovy file scripts stored in 
-the `config/scripts/` directory on every node.
-
-To convert an inline script to a file-based script, save the contents
-of the `inline` field to a file with the `.groovy` extension and
-store it in the `config/scripts` directory on every data node in your
-cluster. 
-
-For example, if you have the following inline script:
-
-[source,js]
------------------------------------
-GET /_search
-{
-    "script_fields": {
-        "my_field": {
-            "inline": "1 + my_var",
-            "params": {
-              "my_var": 2
-            }
-        }
-    }
-}
------------------------------------
-
-Save `1 + my_var` in a file called `config/scripts/my_script.groovy`.
-
-To use the script in a request, specify its name (without the `.groovy` extension) in the `file` field:
-
-[source,js]
------------------------------------
-GET /_search
-{
-    "script_fields": {
-        "my_field": {
-            "script": {
-                "file": "my_script",
-                "params": {
-                    "my_var": 2
-                }
-            }
-        }
-    }
-}
------------------------------------
-
-===================================================
-
-[float]
-=== File-based Scripts
-
-To increase security, Elasticsearch does not allow you to specify scripts for
-non-sandboxed languages with a request. Instead, scripts must be placed in the
-`scripts` directory inside the configuration directory (the directory where
-elasticsearch.yml is). The default location of this `scripts` directory can be
-changed by setting `path.scripts` in elasticsearch.yml. Scripts placed into
-this directory will automatically be picked up and be available to be used.
-Once a script has been placed in this directory, it can be referenced by name.
-For example, a script called `calculate-score.groovy` can be referenced in a
-request like this:
-
-[source,sh]
---------------------------------------------------
-$ tree config
-config
-├── elasticsearch.yml
-├── logging.yml
-└── scripts
-    └── calculate-score.groovy
---------------------------------------------------
-
-[source,sh]
---------------------------------------------------
-$ cat config/scripts/calculate-score.groovy
-log(_score * 2) + my_modifier
---------------------------------------------------
-
-[source,js]
---------------------------------------------------
-curl -XPOST localhost:9200/_search -d '{
-  "query": {
-    "function_score": {
-      "query": {
-        "match": {
-          "body": "foo"
-        }
-      },
-      "functions": [
-        {
-          "script_score": {
-            "script": {
-              "lang": "groovy",
-              "file": "calculate-score",
-              "params": {
-                "my_modifier": 8
-              }
-            }
-          }
-        }
-      ]
-    }
-  }
-}'
---------------------------------------------------
-
-The name of the script is derived from the hierarchy of directories it
-exists under, and the file name without the lang extension. For example,
-a script placed under `config/scripts/group1/group2/test.py` will be
-named `group1_group2_test`.
-
-[float]
-[[modules-scripting-stored-scripts]]
-=== Stored Scripts
-Elasticsearch allows you to store scripts in the cluster state.
-There are REST endpoints to manage stored scripts as follows:
-
-Requests to the scripts endpoint look like :
-[source,js]
------------------------------------
-/_scripts/{lang}/{id}
------------------------------------
-Where the `lang` part is the language the script is in and the `id` part is the id
-of the script.
-
-[source,js]
------------------------------------
-curl -XPOST localhost:9200/_scripts/groovy/calculateScore -d '{
-     "script": "log(_score * 2) + my_modifier"
-}'
------------------------------------
-
-This will store the script under the `calculateScore` in the cluster
-state.
-
-This script can be accessed at query time by using the `id` and `lang` script parameters:
-
-[source,js]
---------------------------------------------------
-curl -XPOST localhost:9200/_search -d '{
-  "query": {
-    "function_score": {
-      "query": {
-        "match": {
-          "body": "foo"
-        }
-      },
-      "functions": [
-        {
-          "script_score": {
-            "script": {
-              "id": "calculateScore",
-              "lang" : "groovy",
-              "params": {
-                "my_modifier": 8
-              }
-            }
-          }
-        }
-      ]
-    }
-  }
-}'
---------------------------------------------------
-
-The script can be viewed by:
-[source,js]
------------------------------------
-curl -XGET localhost:9200/_scripts/groovy/calculateScore
------------------------------------
-
-This is rendered as:
-
-[source,js]
------------------------------------
-'{
-     "script": "log(_score * 2) + my_modifier"
-}'
------------------------------------
-
-Stored scripts can be deleted by:
-[source,js]
------------------------------------
-curl -XDELETE localhost:9200/_scripts/groovy/calculateScore
------------------------------------
-
-NOTE: The size of stored scripts is limited to 65535 bytes. This can be changed by setting `script.max_size_in_bytes`
-setting to increase that soft limit, but if scripts are really large then alternatives like native scripts should be considered.
-
-[float]
-[[enable-dynamic-scripting]]
-=== Enabling dynamic scripting
-
-We recommend running Elasticsearch behind an application or proxy, which
-protects Elasticsearch from the outside world. If users are allowed to run
-inline scripts (even in a search request) or indexed scripts, then they have
-the same access to your box as the user that Elasticsearch is running as. For
-this reason dynamic scripting is allowed only for sandboxed languages by default.
-
-First, you should not run Elasticsearch as the `root` user, as this would allow
-a script to access or do *anything* on your server, without limitations. Second,
-you should not expose Elasticsearch directly to users, but instead have a proxy
-application inbetween. If you *do* intend to expose Elasticsearch directly to
-your users, then you have to decide whether you trust them enough to run scripts
-on your box or not.
-
-It is possible to enable scripts based on their source, for
-every script engine, through the following settings that need to be added to the
-`config/elasticsearch.yml` file on every node.
-
-[source,yaml]
------------------------------------
-script.inline: true
-script.stored: true
-
------------------------------------
-
-While this still allows execution of named scripts provided in the config, or
-_native_ Java scripts registered through plugins, it also allows users to run
-arbitrary scripts via the API. Instead of sending the name of the file as the
-script, the body of the script can be sent instead or retrieved from the
-cluster state if previously stored.
-
-There are three possible configuration values for any of the fine-grained
-script settings:
-
-[cols="<,<",options="header",]
-|=======================================================================
-|Value |Description
-| `false` |scripting is turned off completely, in the context of the setting being set.
-| `true`  |scripting is turned on, in the context of the setting being set.
-| `sandbox` |scripts may be executed only for languages that are sandboxed
-|=======================================================================
-
-The default values are the following:
-
-[source,yaml]
------------------------------------
-script.inline: sandbox
-script.stored: sandbox
-script.file: true
-
------------------------------------
-
-NOTE: Global scripting settings affect the `mustache` scripting language.
-<<search-template,Search templates>> internally use the `mustache` language,
-and will still be enabled by default as the `mustache` engine is sandboxed,
-but they will be enabled/disabled according to fine-grained settings
-specified in `elasticsearch.yml`.
-
-It is also possible to control which operations can execute scripts. The
-supported operations are:
-
-[cols="<,<",options="header",]
-|=======================================================================
-|Value |Description
-| `aggs`    |Aggregations (wherever they may be used)
-| `search`  |Search api, Percolator api and Suggester api (e.g filters, script_fields)
-| `update`  |Update api
-| `plugin`  |Any plugin that makes use of scripts under the generic `plugin` category
-|=======================================================================
-
-Plugins can also define custom operations that they use scripts for instead
-of using the generic `plugin` category. Those operations can be referred to
-in the following form: `${pluginName}_${operation}`.
-
-The following example disables scripting for `update` and `mapping` operations,
-regardless of the script source, for any engine. Scripts can still be
-executed from sandboxed languages as part of `aggregations`, `search`
-and plugins execution though, as the above defaults still get applied.
-
-[source,yaml]
------------------------------------
-script.update: false
-script.mapping: false
-
------------------------------------
-
-Generic settings get applied in order, operation based ones have precedence
-over source based ones. Language specific settings are supported too. They
-need to be prefixed with the `script.engine.<engine>` prefix and have
-precedence over any other generic settings.
-
-[source,yaml]
------------------------------------
-script.engine.groovy.file.aggs: true
-script.engine.groovy.file.mapping: true
-script.engine.groovy.file.search: true
-script.engine.groovy.file.update: true
-script.engine.groovy.file.plugin: true
-script.engine.groovy.stored.aggs: true
-script.engine.groovy.stored.mapping: false
-script.engine.groovy.stored.search: true
-script.engine.groovy.stored.update: false
-script.engine.groovy.stored.plugin: false
-script.engine.groovy.inline.aggs: true
-script.engine.groovy.inline.mapping: false
-script.engine.groovy.inline.search: false
-script.engine.groovy.inline.update: false
-script.engine.groovy.inline.plugin: false
-
------------------------------------
-
-[float]
-=== Default Scripting Language
-
-The default scripting language (assuming no `lang` parameter is provided) is
-`groovy`. In order to change it, set the `script.default_lang` to the
-appropriate language.
-
-[float]
-=== Automatic Script Reloading
-
-The `config/scripts` directory is scanned periodically for changes.
-New and changed scripts are reloaded and deleted script are removed
-from preloaded scripts cache. The reload frequency can be specified
-using `resource.reload.interval` setting, which defaults to `60s`.
-To disable script reloading completely set `script.auto_reload_enabled`
-to `false`.
-
-[[native-java-scripts]]
-[float]
-=== Native (Java) Scripts
-
-Sometimes `groovy` and `expressions` aren't enough. For those times you can
-implement a native script.
-
-The best way to implement a native script is to write a plugin and install it.
-The plugin {plugins}/plugin-authors.html[documentation] has more information on
-how to write a plugin so that Elasticsearch will properly load it.
-
-To register the actual script you'll need to implement `NativeScriptFactory`
-to construct the script. The actual script will extend either
-`AbstractExecutableScript` or `AbstractSearchScript`. The second one is likely
-the most useful and has several helpful subclasses you can extend like
-`AbstractLongSearchScript`, `AbstractDoubleSearchScript`, and
-`AbstractFloatSearchScript`. Finally, your plugin should register the native
-script by declaring the `onModule(ScriptModule)` method.
-
-If you squashed the whole thing into one class it'd look like:
-
-[source,java]
---------------------------------------------------
-public class MyNativeScriptPlugin extends Plugin {
-    @Override
-    public String name() {
-        return "my-native-script";
-    }
-    @Override
-    public String description() {
-        return "my native script that does something great";
-    }
-    public void onModule(ScriptModule scriptModule) {
-        scriptModule.registerScript("my_script", MyNativeScriptFactory.class);
-    }
-
-    public static class MyNativeScriptFactory implements NativeScriptFactory {
-        @Override
-        public ExecutableScript newScript(@Nullable Map<String, Object> params) {
-            return new MyNativeScript();
-        }
-        @Override
-        public boolean needsScores() {
-            return false;
-        }
-    }
-
-    public static class MyNativeScript extends AbstractFloatSearchScript {
-        @Override
-        public float runAsFloat() {
-            float a = (float) source().get("a");
-            float b = (float) source().get("b");
-            return a * b;
-        }
-    }
-}
---------------------------------------------------
-
-You can execute the script by specifying its `lang` as `native`, and the name
-of the script as the `id`:
-
-[source,js]
---------------------------------------------------
-curl -XPOST localhost:9200/_search -d '{
-  "query": {
-    "function_score": {
-      "query": {
-        "match": {
-          "body": "foo"
-        }
-      },
-      "functions": [
-        {
-          "script_score": {
-            "script": {
-                "id": "my_script",
-                "lang" : "native"
-            }
-          }
-        }
-      ]
-    }
-  }
-}'
---------------------------------------------------
-
-
-[float]
-=== Lucene Expressions Scripts
-
-experimental[The Lucene expressions module is undergoing significant development and the exposed functionality is likely to change in the future]
-
-Lucene's expressions module provides a mechanism to compile a
-`javascript` expression to bytecode.  This allows very fast execution,
-as if you had written a `native` script.  Expression scripts can be
-used in `script_score`, `script_fields`, sort scripts and numeric aggregation scripts.
-
-See the link:http://lucene.apache.org/core/4_9_0/expressions/index.html?org/apache/lucene/expressions/js/package-summary.html[expressions module documentation]
-for details on what operators and functions are available.
-
-Variables in `expression` scripts are available to access:
-
-* document fields, e.g. `doc['myfield'].value` 
-* variables and methods that the field supports, e.g. `doc['myfield'].empty`
-* Parameters passed into the script, e.g. `mymodifier`
-* The current document's score, `_score` (only available when used in a `script_score`)
-
-[float]
-=== Expressions API for numeric fields
-[cols="<,<",options="header",]
-|=======================================================================
-|Expression |Description
-|`doc['field_name'].value` |The native value of the field. For example,
-if its a short type, it will be short.
-
-|`doc['field_name'].empty` |A boolean indicating if the field has no
-values within the doc.
-
-|`doc['field_name'].min()` |The minimum value of the field in this document.
-
-|`doc['field_name'].max()` |The maximum value of the field in this document.
-
-|`doc['field_name'].median()` |The median value of the field in this document.
-
-|`doc['field_name'].avg()` |The average of the values in this document.
-
-|`doc['field_name'].sum()` |The sum of the values in this document.
-
-|`doc['field_name'].count()` |The number of values in this document.
-|=======================================================================
-
-When a document is missing the field completely, by default the value will be treated as `0`.
-You can treat it as another value instead, e.g. `doc['myfield'].empty ? 100 : doc['myfield'].value`
-
-When a document has multiple values for the field, by default the minimum value is returned.
-You can choose a different value instead, e.g. `doc['myfield'].sum()`.
-
-When a document is missing the field completely, by default the value will be treated as `0`.
-
-Boolean fields are exposed as numerics, with `true` mapped to `1` and `false` mapped to `0`. 
-For example: `doc['on_sale'] ? doc['price'] * 0.5 : doc['price']`
-
-[float]
-=== Additional methods for date fields
-Date fields are treated as the number of milliseconds since January 1, 1970 and 
-support the numeric API above, with these additional methods:
-
-[cols="<,<",options="header",]
-|=======================================================================
-|Expression |Description
-|`doc['field_name'].getYear()` |Year component, e.g. `1970`.
-
-|`doc['field_name'].getMonth()` |Month component (0-11), e.g. `0` for January.
-
-|`doc['field_name'].getDayOfMonth()` |Day component, e.g. `1` for the first of the month.
-
-|`doc['field_name'].getHourOfDay()` |Hour component (0-23)
-
-|`doc['field_name'].getMinutes()` |Minutes component (0-59)
-
-|`doc['field_name'].getSeconds()` |Seconds component (0-59)
-|=======================================================================
-
-The following example shows the difference in years between the `date` fields date0 and date1:
-
-`doc['date1'].getYear() - doc['date0'].getYear()`
-
-[float]
-=== Expressions API for `geo_point` fields
-[cols="<,<",options="header",]
-|=======================================================================
-|Expression |Description
-|`doc['field_name'].empty` |A boolean indicating if the field has no
-values within the doc.
-
-|`doc['field_name'].lat` |The latitude of the geo point.
-
-|`doc['field_name'].lon` |The longitude of the geo point.
-|=======================================================================
-
-The following example computes distance in kilometers from Washington, DC:
-
-`haversin(38.9072, 77.0369, doc['field_name'].lat, doc['field_name'].lon)`
-
-In this example the coordinates could have been passed as parameters to the script,
-e.g. based on geolocation of the user.
-
-[float]
-=== Expressions limitations
-
-There are a few limitations relative to other script languages:
-
-* Only numeric, boolean, date, and geo_point fields may be accessed
-* Stored fields are not available
-
-[float]
-=== Score
-
-In all scripts that can be used in aggregations, the current
-document's score is accessible in `_score`.
-
-[float]
-=== Computing scores based on terms in scripts
-
-see <<modules-advanced-scripting, advanced scripting documentation>>
-
-[float]
-=== Document Fields
-
-Most scripting revolve around the use of specific document fields data.
-The `doc['field_name']` can be used to access specific field data within
-a document (the document in question is usually derived by the context
-the script is used). Document fields are very fast to access since they
-end up being loaded into memory (all the relevant field values/tokens
-are loaded to memory). Note, however, that the `doc[...]` notation only
-allows for simple valued fields (can’t return a json object from it)
-and makes sense only on non-analyzed or single term based fields.
-
-The following data can be extracted from a field:
-
-[cols="<,<",options="header",]
-|=======================================================================
-|Expression |Description
-|`doc['field_name'].value` |The native value of the field. For example,
-if its a short type, it will be short.
-
-|`doc['field_name'].values` |The native array values of the field. For
-example, if its a short type, it will be short[]. Remember, a field can
-have several values within a single doc. Returns an empty array if the
-field has no values.
-
-|`doc['field_name'].empty` |A boolean indicating if the field has no
-values within the doc.
-
-|`doc['field_name'].multiValued` |A boolean indicating that the field
-has several values within the corpus.
-
-|`doc['field_name'].lat` |The latitude of a geo point type.
-
-|`doc['field_name'].lon` |The longitude of a geo point type.
-
-|`doc['field_name'].lats` |The latitudes of a geo point type.
-
-|`doc['field_name'].lons` |The longitudes of a geo point type.
-
-|`doc['field_name'].distance(lat, lon)` |The `plane` distance (in meters)
-of this geo point field from the provided lat/lon.
-
-|`doc['field_name'].distanceWithDefault(lat, lon, default)` |The `plane` distance (in meters)
-of this geo point field from the provided lat/lon with a default value.
-
-|`doc['field_name'].distanceInMiles(lat, lon)` |The `plane` distance (in
-miles) of this geo point field from the provided lat/lon.
-
-|`doc['field_name'].distanceInMilesWithDefault(lat, lon, default)` |The `plane` distance (in
-miles) of this geo point field from the provided lat/lon with a default value.
-
-|`doc['field_name'].distanceInKm(lat, lon)` |The `plane` distance (in
-km) of this geo point field from the provided lat/lon.
-
-|`doc['field_name'].distanceInKmWithDefault(lat, lon, default)` |The `plane` distance (in
-km) of this geo point field from the provided lat/lon with a default value.
-
-|`doc['field_name'].arcDistance(lat, lon)` |The `arc` distance (in
-meters) of this geo point field from the provided lat/lon.
-
-|`doc['field_name'].arcDistanceWithDefault(lat, lon, default)` |The `arc` distance (in
-meters) of this geo point field from the provided lat/lon with a default value.
-
-|`doc['field_name'].arcDistanceInMiles(lat, lon)` |The `arc` distance (in
-miles) of this geo point field from the provided lat/lon.
-
-|`doc['field_name'].arcDistanceInMilesWithDefault(lat, lon, default)` |The `arc` distance (in
-miles) of this geo point field from the provided lat/lon with a default value.
-
-|`doc['field_name'].arcDistanceInKm(lat, lon)` |The `arc` distance (in
-km) of this geo point field from the provided lat/lon.
-
-|`doc['field_name'].arcDistanceInKmWithDefault(lat, lon, default)` |The `arc` distance (in
-km) of this geo point field from the provided lat/lon with a default value.
-
-|`doc['field_name'].factorDistance(lat, lon)` |The distance factor of this geo point field from the provided lat/lon.
-
-|`doc['field_name'].factorDistance(lat, lon, default)` |The distance factor of this geo point field from the provided lat/lon with a default value.
-
-|`doc['field_name'].geohashDistance(geohash)` |The `arc` distance (in meters)
-of this geo point field from the provided geohash.
-
-|`doc['field_name'].geohashDistanceInKm(geohash)` |The `arc` distance (in km)
-of this geo point field from the provided geohash.
-
-|`doc['field_name'].geohashDistanceInMiles(geohash)` |The `arc` distance (in
-miles) of this geo point field from the provided geohash.
-|=======================================================================
-
-[float]
-=== Stored Fields
-
-Stored fields can also be accessed when executing a script. Note, they
-are much slower to access compared with document fields, as they are not
-loaded into memory. They can be simply accessed using
-`_fields['my_field_name'].value` or `_fields['my_field_name'].values`.
-
-[float]
-=== Accessing the score of a document within a script
-
-When using scripting for calculating the score of a document (for instance, with
-the `function_score` query), you can access the score using the `_score`
-variable inside of a Groovy script.
-
-[float]
-=== Source Field
-
-The source field can also be accessed when executing a script. The
-source field is loaded per doc, parsed, and then provided to the script
-for evaluation. The `_source` forms the context under which the source
-field can be accessed, for example `_source.obj2.obj1.field3`.
-
-Accessing `_source` is much slower compared to using `doc`
-but the data is not loaded into memory. For a single field access `_fields` may be
-faster than using `_source` due to the extra overhead of potentially parsing large documents.
-However, `_source` may be faster if you access multiple fields or if the source has already been
-loaded for other purposes.
-
-
-[float]
-=== Groovy Built In Functions
-
-There are several built in functions that can be used within scripts.
-They include:
-
-[cols="<,<",options="header",]
-|=======================================================================
-|Function |Description
-|`sin(a)` |Returns the trigonometric sine of an angle.
-
-|`cos(a)` |Returns the trigonometric cosine of an angle.
-
-|`tan(a)` |Returns the trigonometric tangent of an angle.
-
-|`asin(a)` |Returns the arc sine of a value.
-
-|`acos(a)` |Returns the arc cosine of a value.
-
-|`atan(a)` |Returns the arc tangent of a value.
-
-|`toRadians(angdeg)` |Converts an angle measured in degrees to an
-approximately equivalent angle measured in radians
-
-|`toDegrees(angrad)` |Converts an angle measured in radians to an
-approximately equivalent angle measured in degrees.
-
-|`exp(a)` |Returns Euler's number _e_ raised to the power of value.
-
-|`log(a)` |Returns the natural logarithm (base _e_) of a value.
-
-|`log10(a)` |Returns the base 10 logarithm of a value.
-
-|`sqrt(a)` |Returns the correctly rounded positive square root of a
-value.
-
-|`cbrt(a)` |Returns the cube root of a double value.
-
-|`IEEEremainder(f1, f2)` |Computes the remainder operation on two
-arguments as prescribed by the IEEE 754 standard.
-
-|`ceil(a)` |Returns the smallest (closest to negative infinity) value
-that is greater than or equal to the argument and is equal to a
-mathematical integer.
-
-|`floor(a)` |Returns the largest (closest to positive infinity) value
-that is less than or equal to the argument and is equal to a
-mathematical integer.
-
-|`rint(a)` |Returns the value that is closest in value to the argument
-and is equal to a mathematical integer.
-
-|`atan2(y, x)` |Returns the angle _theta_ from the conversion of
-rectangular coordinates (_x_, _y_) to polar coordinates (r,_theta_).
-
-|`pow(a, b)` |Returns the value of the first argument raised to the
-power of the second argument.
-
-|`round(a)` |Returns the closest _int_ to the argument.
-
-|`random()` |Returns a random _double_ value.
-
-|`abs(a)` |Returns the absolute value of a value.
-
-|`max(a, b)` |Returns the greater of two values.
-
-|`min(a, b)` |Returns the smaller of two values.
-
-|`ulp(d)` |Returns the size of an ulp of the argument.
-
-|`signum(d)` |Returns the signum function of the argument.
-
-|`sinh(x)` |Returns the hyperbolic sine of a value.
-
-|`cosh(x)` |Returns the hyperbolic cosine of a value.
-
-|`tanh(x)` |Returns the hyperbolic tangent of a value.
-
-|`hypot(x, y)` |Returns sqrt(_x2_ + _y2_) without intermediate overflow
-or underflow.
-|=======================================================================
diff --git a/docs/reference/modules/scripting/security.asciidoc b/docs/reference/modules/scripting/security.asciidoc
index 3c9144984d5..c5ad89538e3 100644
--- a/docs/reference/modules/scripting/security.asciidoc
+++ b/docs/reference/modules/scripting/security.asciidoc
@@ -1,5 +1,139 @@
 [[modules-scripting-security]]
-=== Scripting and the Java Security Manager
+=== Scripting and security
+
+You should never run Elasticsearch as the `root` user, as this would allow a
+script to access or do *anything* on your server, without limitations.
+
+You should not expose Elasticsearch directly to users, but instead have a
+proxy application inbetween. If you *do* intend to expose Elasticsearch
+directly to your users, then you have to decide whether you trust them enough
+to run scripts on your box or not, and apply the appropriate safety measures.
+
+[[enable-dynamic-scripting]]
+[float]
+=== Enabling dynamic scripting
+
+The `script.*` settings allow for <<security-script-fine,fine-grained>>
+control of which script languages (e.g `groovy`, `painless`) are allowed to
+run in which context ( e.g. `search`, `aggs`, `update`), and where the script
+source is allowed to come from (i.e. `inline`, `stored`, `file`).
+
+For instance, the following setting enables `stored` `update` scripts for
+`groovy`:
+
+[source,yaml]
+----------------
+script.engine.groovy.inline.update: true
+----------------
+
+Less fine-grained settings exist which allow you to enable or disable scripts
+for all sources, all languages, or all contexts.  The following settings
+enable `inline` and `stored` scripts for all languages in all contexts:
+
+[source,yaml]
+-----------------------------------
+script.inline: true
+script.stored: true
+-----------------------------------
+
+WARNING:  The above settings mean that anybody who can send requests to your
+Elasticsearch instance can run whatever scripts they choose! This is a
+security risk and may well lead to your Elasticsearch cluster being
+compromised.
+
+[[security-script-source]]
+[float]
+=== Script source settings
+
+Scripts may be enabled or disabled depending on their source: `inline`,
+`stored` in the cluster state, or from a `file` on each node in the cluster.
+Each of these settings takes one of these values:
+
+
+[horizontal]
+`false`::   Scripting is enabled.
+`true`::    Scripting is disabled.
+`sandbox`:: Scripting is enabled only for sandboxed languages.
+
+The default values are the following:
+
+[source,yaml]
+-----------------------------------
+script.inline: sandbox
+script.stored: sandbox
+script.file:   true
+-----------------------------------
+
+NOTE: Global scripting settings affect the `mustache` scripting language.
+<<search-template,Search templates>> internally use the `mustache` language,
+and will still be enabled by default as the `mustache` engine is sandboxed,
+but they will be enabled/disabled according to fine-grained settings
+specified in `elasticsearch.yml`.
+
+[[security-script-context]]
+[float]
+=== Script context settings
+
+Scripting may also be enabled or disabled in different contexts in the
+Elasticsearch API. The supported contexts are:
+
+[horizontal]
+`aggs`::    Aggregations
+`search`::  Search api, Percolator API and Suggester API
+`update`::  Update api
+`plugin`::  Any plugin that makes use of scripts under the generic `plugin` category
+
+Plugins can also define custom operations that they use scripts for instead
+of using the generic `plugin` category. Those operations can be referred to
+in the following form: `${pluginName}_${operation}`.
+
+The following example disables scripting for `update` and `plugin` operations,
+regardless of the script source or language. Scripts can still be executed
+from sandboxed languages as part of `aggregations`, `search` and plugins
+execution though, as the above defaults still get applied.
+
+[source,yaml]
+-----------------------------------
+script.update: false
+script.plugin: false
+-----------------------------------
+
+[[security-script-fine]]
+[float]
+=== Fine-grained script settings
+
+First, the high-level script settings described above are applied in order
+(context settings have precedence over source settings).  Then,  fine-grained
+settings which include the script language take precedence over any high-level
+settings.
+
+Fine-grained settings have the form:
+
+[source,yaml]
+------------------------
+script.engine.{lang}.{source}.{context}: true|false
+------------------------
+
+For example:
+
+[source,yaml]
+-----------------------------------
+script.inline: false <1>
+script.stored: false <1>
+script.file:   false <1>
+
+script.engine.groovy.stored.search:   true <2>
+script.engine.groovy.stored.aggs:     true <2>
+
+script.engine.mustache.stored.search: true <3>
+-----------------------------------
+<1> Disable all scripting from any source.
+<2> Allow stored Groovy scripts to be used for search and aggregations.
+<3> Allow stored Mustache templates to be used for search.
+
+[[java-security-manager]]
+[float]
+=== Java Security Manager
 
 Elasticsearch runs with the https://docs.oracle.com/javase/tutorial/essential/environment/security.html[Java Security Manager]
 enabled by default.  The security policy in Elasticsearch locks down the
diff --git a/docs/reference/modules/scripting/using.asciidoc b/docs/reference/modules/scripting/using.asciidoc
new file mode 100644
index 00000000000..71eba9386e3
--- /dev/null
+++ b/docs/reference/modules/scripting/using.asciidoc
@@ -0,0 +1,238 @@
+[[modules-scripting-using]]
+=== How to use scripts
+
+Wherever scripting is supported in the Elasticsearch API, the syntax follows
+the same pattern:
+
+[source,js]
+-------------------------------------
+  "script": {
+    "lang":   "...",  <1>
+    "inline" | "id" | "file": "...", <2>
+    "params": { ... } <3>
+  }
+-------------------------------------
+<1> The language the script is written in, which defaults to `groovy`.
+<2> The script itself which may be specfied as `inline`, `id`, or `file`.
+<3> Any named parameters that should be passed into the script.
+
+For example, the following script is used in a search request to return a
+<<search-request-script-fields, scripted field>>:
+
+[source,js]
+-------------------------------------
+PUT my_index/my_type/1
+{
+  "my_field": 5
+}
+
+GET my_index/_search
+{
+  "script_fields": {
+    "my_doubled_field": {
+      "script": {
+        "lang":   "expression",
+        "inline": "doc['my_field'] * multiplier",
+        "params": {
+          "multiplier": 2
+        }
+      }
+    }
+  }
+}
+-------------------------------------
+// AUTOSENSE
+
+
+[float]
+=== Script Parameters
+
+`lang`::
+
+    Specifies the language the script is written in.  Defaults to `groovy` but
+    may be set to any of languages listed in <<modules-scripting>>. The
+    default language may be changed in the `elasticsearch.yml` config file by
+    setting `script.default_lang` to the appropriate language.
+
+
+`inline`, `id`, `file`::
+
+    Specifies the source of the script.  An `inline` script is specified
+    `inline` as in the example above, a stored script with the specified `id`
+    is retrieved from the cluster state (see <<modules-scripting-stored-scripts,Stored Scripts>>),
+    and a `file` script is retrieved from a file in the `config/scripts`
+    directory (see <<modules-scripting-file-scripts, File Scripts>>).
++
+While languages like `expression` and `painless` can be used out of the box as
+inline or stored scripts, other languages like `groovy` can only be
+specified as `file` unless you first adjust the default
+<<modules-scripting-security,scripting security settings>>.
+
+`params`::
+
+    Specifies any named parameters that are passed into the script as
+    variables.
+
+[IMPORTANT]
+.Prefer parameters
+========================================
+
+The first time Elasticsearch sees a new script, it compiles it and stores the
+compiled version in a cache. Compilation can be a heavy process.
+
+If you need to pass variables into the script, you should pass them in as
+named `params` instead of hard-coding values into the script itself.  For
+example, if you want to be able to multiply a field value by different
+multipliers, don't hard-code the multiplier into the script:
+
+[source,js]
+----------------------
+  "inline": "doc['my_field'] * 2"
+----------------------
+
+Instead, pass it in as a named parameter:
+
+[source,js]
+----------------------
+  "inline": "doc['my_field'] * multiplier",
+  "params": {
+    "multiplier": 2
+  }
+----------------------
+
+The first version has to be recompiled every time the multiplier changes.  The
+second version is only compiled once.
+
+========================================
+
+
+[float]
+[[modules-scripting-file-scripts]]
+=== File-based Scripts
+
+To increase security, non-sandboxed languages can only be specified in script
+files stored on every node in the cluster.  File scripts must be saved in the
+`scripts` directory whose default location depends on whether you use  the
+<<zip-targz-layout,`zip`/`tar.gz`>> (`$ES_HOME/config/scripts/`),
+<<rpm-layout,RPM>>, or <<deb-layout,Debian>> package.  The default may be
+changed with the `path.script` setting.
+
+Any files placed in the `scripts` directory will be compiled automatically
+when the node starts up and then <<reload-scripts,every 60 seconds thereafter>>.
+
+The file should be named as follows: `{script-name}.{lang}`.  For instance,
+the following example creates a Groovy script called `calculate-score`:
+
+[source,sh]
+--------------------------------------------------
+cat "log(_score * 2) + my_modifier" > config/scripts/calculate-score.groovy
+--------------------------------------------------
+
+This script can be used as follows:
+
+[source,js]
+--------------------------------------------------
+GET my_index/_search
+{
+  "query": {
+    "script": {
+      "script": {
+        "lang":   "groovy", <1>
+        "file":   "calculate-score", <2>
+        "params": {
+          "my_modifier": 2
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+<1> The language of the script, which should correspond with the script file suffix.
+<2> The name of the script, which should be the name of the file.
+
+The `script` directory may contain sub-directories, in which case the
+hierarchy of directories is flattened and concatenated with underscores.  A
+script in `group1/group2/my_script.groovy` should use `group1_group2_myscript`
+as the `file` name.
+
+
+[[reload-scripts]]
+[float]
+==== Automatic script reloading
+
+The `scripts` directory will be rescanned every `60s` (configurable with the
+`resource.reload.interval` setting) and new, changed, or removed scripts will
+be compiled, updated, or deleted from the script cache.
+
+Script reloading can be completely disabled by setting
+`script.auto_reload_enabled` to `false`.
+
+[float]
+[[modules-scripting-stored-scripts]]
+=== Stored Scripts
+
+Scripts may be stored in and retrieved from the cluster state using the
+`_scripts` end-point:
+
+[source,js]
+-----------------------------------
+/_scripts/{lang}/{id} <1> <2>
+-----------------------------------
+<1> The `lang` represents the script language.
+<2> The `id` is a unique identifier or script name.
+
+This example stores a Groovy script called `calculate-score` in the cluster
+state:
+
+[source,js]
+-----------------------------------
+POST /_scripts/groovy/calculate-score
+{
+  "script": "log(_score * 2) + my_modifier"
+}
+-----------------------------------
+// AUTOSENSE
+
+
+This same script can be retrieved with:
+
+[source,js]
+-----------------------------------
+GET /_scripts/groovy/calculate-score
+-----------------------------------
+// AUTOSENSE
+
+or deleted with:
+
+[source,js]
+-----------------------------------
+DELETE /_scripts/groovy/calculate-score
+-----------------------------------
+// AUTOSENSE
+
+Stored scripts can be used by specifying the `lang` and `id` parameters as follows:
+
+[source,js]
+--------------------------------------------------
+GET my_index/_search
+{
+  "query": {
+    "script": {
+      "script": {
+        "lang": "groovy",
+        "id":   "calculate-score",
+        "params": {
+          "my_modifier": 2
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+
+
+NOTE: The size of stored scripts is limited to 65,535 bytes. This can be
+changed by setting `script.max_size_in_bytes` setting to increase that soft
+limit, but if scripts are really large then alternatives like
+<<modules-scripting-native,native>> scripts should be considered instead.
+

From 4c6cf7ee88bc75a161e3752014071d4f9ab8d8d5 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 4 May 2016 12:48:39 -0400
Subject: [PATCH 0071/1311] Pass ES_JAVA_OPTS to JVM for plugins script

This commit adds support for ES_JAVA_OPTS to the elasticsearch-plugin
script.

Relates #18140
---
 .../src/main/resources/bin/elasticsearch-plugin      |  2 +-
 .../src/main/resources/bin/elasticsearch-plugin.bat  |  2 +-
 .../scripts/module_and_plugin_test_cases.bash        | 12 ++++++++++++
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/distribution/src/main/resources/bin/elasticsearch-plugin b/distribution/src/main/resources/bin/elasticsearch-plugin
index f686ed054f2..2730a562907 100755
--- a/distribution/src/main/resources/bin/elasticsearch-plugin
+++ b/distribution/src/main/resources/bin/elasticsearch-plugin
@@ -110,4 +110,4 @@ fi
 HOSTNAME=`hostname | cut -d. -f1`
 export HOSTNAME
 
-eval "\"$JAVA\"" -client -Delasticsearch -Des.path.home="\"$ES_HOME\"" $properties -cp "\"$ES_HOME/lib/*\"" org.elasticsearch.plugins.PluginCli $args
+eval "\"$JAVA\"" "$ES_JAVA_OPTS" -client -Delasticsearch -Des.path.home="\"$ES_HOME\"" $properties -cp "\"$ES_HOME/lib/*\"" org.elasticsearch.plugins.PluginCli $args
diff --git a/distribution/src/main/resources/bin/elasticsearch-plugin.bat b/distribution/src/main/resources/bin/elasticsearch-plugin.bat
index 6c6be019fc6..c4286aa7b1c 100644
--- a/distribution/src/main/resources/bin/elasticsearch-plugin.bat
+++ b/distribution/src/main/resources/bin/elasticsearch-plugin.bat
@@ -48,7 +48,7 @@ GOTO loop
 
 SET HOSTNAME=%COMPUTERNAME%
 
-"%JAVA_HOME%\bin\java" -client -Des.path.home="%ES_HOME%" !properties! -cp "%ES_HOME%/lib/*;" "org.elasticsearch.plugins.PluginCli" !args!
+"%JAVA_HOME%\bin\java" %ES_JAVA_OPTS% -client -Des.path.home="%ES_HOME%" !properties! -cp "%ES_HOME%/lib/*;" "org.elasticsearch.plugins.PluginCli" !args!
 goto finally
 
 
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
index 382492e245e..85d829718e2 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
@@ -476,3 +476,15 @@ fi
     # restore JAVA_HOME
     export JAVA_HOME=$java_home
 }
+
+@test "[$GROUP] test ES_JAVA_OPTS" {
+    # preserve ES_JAVA_OPTS
+    local es_java_opts=$ES_JAVA_OPTS
+
+    export ES_JAVA_OPTS="-XX:+PrintFlagsFinal"
+    # this will fail if ES_JAVA_OPTS is not passed through
+    "$ESHOME/bin/elasticsearch-plugin" list | grep MaxHeapSize
+
+    # restore ES_JAVA_OPTS
+    export ES_JAVA_OPTS=$es_java_opts
+}

From 7656d7ea7396cb291d913411724add972b5f01d1 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Wed, 4 May 2016 12:49:16 -0400
Subject: [PATCH 0072/1311] docs: remove null from expressions case.

Expressions don't have nulls, only doubles. If the field is missing, then its
treated as 0.0. You can query .empty to see if its missing and substitute something else.

See https://github.com/elastic/elasticsearch/pull/18132#discussion_r62068494
---
 docs/reference/modules/scripting/expression.asciidoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/reference/modules/scripting/expression.asciidoc b/docs/reference/modules/scripting/expression.asciidoc
index d941fa2f4a4..61e28e05298 100644
--- a/docs/reference/modules/scripting/expression.asciidoc
+++ b/docs/reference/modules/scripting/expression.asciidoc
@@ -99,9 +99,9 @@ The following example shows the difference in years between the `date` fields da
 |`doc['field_name'].empty` |A boolean indicating if the field has no
 values within the doc.
 
-|`doc['field_name'].lat` |The latitude of the geo point, or `null`.
+|`doc['field_name'].lat` |The latitude of the geo point.
 
-|`doc['field_name'].lon` |The longitude of the geo point, or `null`.
+|`doc['field_name'].lon` |The longitude of the geo point.
 |=======================================================================
 
 The following example computes distance in kilometers from Washington, DC:

From 124e8e5a6dd3d23cf1182bec63063670e915152c Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 4 May 2016 14:11:29 -0400
Subject: [PATCH 0073/1311] Remove client option for JVM for plugins script

Today we specify the client option for the JVM when executing plugin
commands. Yet, this option does nothing on a 64-bit capable JDK as such
JDKs always select the Hotspot server VM. And for 32-bit JDKs, running
plugin commands with the server VM is okay. Thus, we should just remove
this unnecessary flag and just let the default VM be selected.

Relates #18142
---
 distribution/src/main/resources/bin/elasticsearch-plugin     | 2 +-
 distribution/src/main/resources/bin/elasticsearch-plugin.bat | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/distribution/src/main/resources/bin/elasticsearch-plugin b/distribution/src/main/resources/bin/elasticsearch-plugin
index 2730a562907..eec162dab94 100755
--- a/distribution/src/main/resources/bin/elasticsearch-plugin
+++ b/distribution/src/main/resources/bin/elasticsearch-plugin
@@ -110,4 +110,4 @@ fi
 HOSTNAME=`hostname | cut -d. -f1`
 export HOSTNAME
 
-eval "\"$JAVA\"" "$ES_JAVA_OPTS" -client -Delasticsearch -Des.path.home="\"$ES_HOME\"" $properties -cp "\"$ES_HOME/lib/*\"" org.elasticsearch.plugins.PluginCli $args
+eval "\"$JAVA\"" "$ES_JAVA_OPTS" -Delasticsearch -Des.path.home="\"$ES_HOME\"" $properties -cp "\"$ES_HOME/lib/*\"" org.elasticsearch.plugins.PluginCli $args
diff --git a/distribution/src/main/resources/bin/elasticsearch-plugin.bat b/distribution/src/main/resources/bin/elasticsearch-plugin.bat
index c4286aa7b1c..5604b57dbef 100644
--- a/distribution/src/main/resources/bin/elasticsearch-plugin.bat
+++ b/distribution/src/main/resources/bin/elasticsearch-plugin.bat
@@ -48,7 +48,7 @@ GOTO loop
 
 SET HOSTNAME=%COMPUTERNAME%
 
-"%JAVA_HOME%\bin\java" %ES_JAVA_OPTS% -client -Des.path.home="%ES_HOME%" !properties! -cp "%ES_HOME%/lib/*;" "org.elasticsearch.plugins.PluginCli" !args!
+"%JAVA_HOME%\bin\java" %ES_JAVA_OPTS% -Des.path.home="%ES_HOME%" !properties! -cp "%ES_HOME%/lib/*;" "org.elasticsearch.plugins.PluginCli" !args!
 goto finally
 
 

From 67c0734bf3d1f6748be56e83c292ad4910339c38 Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Wed, 4 May 2016 15:21:47 -0400
Subject: [PATCH 0074/1311] Update misc.asciidoc

Added documentation for the cluster.indices.tombstones.size property for maximum tombstones in the cluster state.
---
 docs/reference/modules/cluster/misc.asciidoc | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/docs/reference/modules/cluster/misc.asciidoc b/docs/reference/modules/cluster/misc.asciidoc
index b6c50f36f92..57b6e8ea5c9 100644
--- a/docs/reference/modules/cluster/misc.asciidoc
+++ b/docs/reference/modules/cluster/misc.asciidoc
@@ -17,6 +17,23 @@ user with access to the <<cluster-update-settings,cluster-update-settings>>
 API can make the cluster read-write again.
 
 
+[[cluster-max-tombstones]]
+==== Index Tombstones
+
+The cluster state maintains index tombstones to explicitly denote indices that 
+have been deleted.  The number of tombstones maintained in the cluster state is 
+controlled by the following property, which cannot be updated dynamically:
+
+`cluster.indices.tombstones.size`::
+
+Index tombstones prevent nodes that are not part of the cluster when a delete 
+occurs from joining the cluster and reimporting the index as though the delete 
+was never issued. To keep the cluster state from growing huge we only keep the 
+last `cluster.indices.tombstones.size` deletes, which defaults to 500. You can 
+increase it if you expect nodes to be absent from the cluster and miss more 
+than 500 deletes. We think that is rare, thus the default. Tombstones don't take 
+up much space, but we also think that a number like 50,000 is probably too big.
+
 [[cluster-logger]]
 ==== Logger
 

From 230697c20220937ae5ffed4de29af73b5124d56d Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 27 Apr 2016 18:10:57 -0400
Subject: [PATCH 0075/1311] [reindex] Switch throttle to
 Float.POSITIVE_INFITINTY/"unlimited"

All other values are errors.

Add java test for throttling. We had a REST test but it only ran against
one node so it didn't catch serialization errors.

Add Simple round trip test for rethrottle request
---
 docs/reference/docs/reindex.asciidoc          |  47 +++++++-
 docs/reference/docs/update-by-query.asciidoc  |  51 ++++++++-
 .../AbstractAsyncBulkByScrollAction.java      |   2 +-
 .../AbstractBaseReindexRestHandler.java       |  23 ++--
 .../reindex/AbstractBulkByScrollRequest.java  |  20 +++-
 .../AbstractBulkByScrollRequestBuilder.java   |  10 ++
 ...stractBulkIndexByScrollRequestBuilder.java |   3 +-
 .../index/reindex/BulkByScrollTask.java       |   5 +-
 .../index/reindex/RethrottleRequest.java      |  35 +++++-
 .../reindex/RethrottleRequestBuilder.java     |   4 +-
 .../reindex/AsyncBulkByScrollActionTests.java |   2 +-
 .../index/reindex/BulkByScrollTaskTests.java  |  12 +-
 .../index/reindex/CancelTestUtils.java        |   5 +-
 .../index/reindex/ReindexBasicTests.java      |  12 +-
 .../index/reindex/ReindexCancelTests.java     |   2 +-
 .../index/reindex/ReindexFailureTests.java    |   4 +-
 .../reindex/ReindexParentChildTests.java      |   8 +-
 .../index/reindex/ReindexTestCase.java        |  24 +++-
 .../index/reindex/ReindexVersioningTests.java |  18 +--
 .../index/reindex/RethrottleTests.java        |  64 +++++++++++
 .../index/reindex/RoundTripTests.java         |  22 +++-
 .../reindex/UpdateByQueryBasicTests.java      |  14 ++-
 .../reindex/UpdateByQueryCancelTests.java     |   6 +-
 .../index/reindex/UpdateByQueryTestCase.java  |  52 ---------
 .../UpdateByQueryWhileModifyingTests.java     |   7 +-
 .../test/reindex/20_validation.yaml           |  26 ++++-
 .../test/reindex/80_throttle.yaml             | 108 ++++++++----------
 .../test/update_by_query/20_validation.yaml   |  38 +++---
 .../test/update_by_query/70_throttle.yaml     |  88 ++++++--------
 .../rest-api-spec/api/reindex.rethrottle.json |   2 +-
 30 files changed, 458 insertions(+), 256 deletions(-)
 create mode 100644 modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
 delete mode 100644 modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryTestCase.java

diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index 488d3c53b89..8742e52c5c8 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -342,14 +342,15 @@ request. `timeout` controls how long each write request waits for unavailable
 shards to become available. Both work exactly how they work in the
 <<docs-bulk,Bulk API>>.
 
-`requests_per_second` can be set to any decimal number (1.4, 6, 1000, etc) and
-throttle the number of requests per second that the reindex issues. The
+`requests_per_second` can be set to any decimal number (`1.4`, `6`, `1000`, etc)
+and throttles the number of requests per second that the reindex issues. The
 throttling is done waiting between bulk batches so that it can manipulate the
 scroll timeout. The wait time is the difference between the time it took the
 batch to complete and the time `requests_per_second * requests_in_the_batch`.
 Since the batch isn't broken into multiple bulk requests large batch sizes will
 cause Elasticsearch to create many requests and then wait for a while before
-starting the next set. This is "bursty" instead of "smooth".
+starting the next set. This is "bursty" instead of "smooth". The default is
+`unlimited` which is also the only non-number value that it accepts.
 
 [float]
 === Response body
@@ -464,6 +465,46 @@ progress by adding the `updated`, `created`, and `deleted` fields. The request
 will finish when their sum is equal to the `total` field.
 
 
+[float]
+[[docs-reindex-cancel-task-api]]
+=== Works with the Cancel Task API
+
+Any Reindex can be canceled using the <<tasks,Task Cancel API>>:
+
+[source,js]
+--------------------------------------------------
+POST /_tasks/{task_id}/_cancel
+--------------------------------------------------
+// AUTOSENSE
+
+The `task_id` can be found using the tasks API above.
+
+Cancelation should happen quickly but might take a few seconds. The task status
+API above will continue to list the task until it is wakes to cancel itself.
+
+
+[float]
+[[docs-reindex-rethrottle]]
+=== Rethrottling
+
+The value of `requests_per_second` can be changed on a running reindex using
+the `_rethrottle` API:
+
+[source,js]
+--------------------------------------------------
+POST /_reindex/{task_id}/_rethrottle?requests_per_second=unlimited
+--------------------------------------------------
+// AUTOSENSE
+
+The `task_id` can be found using the tasks API above.
+
+Just like when setting it on the `_reindex` API `requests_per_second` can be
+either `unlimited` to disable throttling or any decimal number like `1.7` or
+`12` to throttle to that level. Rethrottling that speeds up the query takes
+effect immediately but rethrotting that slows down the query will take effect
+on after completing the current batch. This prevents scroll timeouts.
+
+
 [float]
 === Reindex to change the name of a field
 
diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc
index 84bd61f3e9b..e307d53e8f4 100644
--- a/docs/reference/docs/update-by-query.asciidoc
+++ b/docs/reference/docs/update-by-query.asciidoc
@@ -172,14 +172,15 @@ request. `timeout` controls how long each write request waits for unavailable
 shards to become available. Both work exactly how they work in the
 <<docs-bulk,Bulk API>>.
 
-`requests_per_second` can be set to any decimal number (1.4, 6, 1000, etc) and
-throttle the number of requests per second that the update by query issues. The
-throttling is done waiting between bulk batches so that it can manipulate the
-scroll timeout. The wait time is the difference between the time it took the
+`requests_per_second` can be set to any decimal number (`1.4`, `6`, `1000`, etc)
+and throttles the number of requests per second that the update by query issues.
+The throttling is done waiting between bulk batches so that it can manipulate
+the scroll timeout. The wait time is the difference between the time it took the
 batch to complete and the time `requests_per_second * requests_in_the_batch`.
 Since the batch isn't broken into multiple bulk requests large batch sizes will
 cause Elasticsearch to create many requests and then wait for a while before
-starting the next set. This is "bursty" instead of "smooth".
+starting the next set. This is "bursty" instead of "smooth". The default is
+`unlimited` which is also the only non-number value that it accepts.
 
 [float]
 === Response body
@@ -290,6 +291,46 @@ progress by adding the `updated`, `created`, and `deleted` fields. The request
 will finish when their sum is equal to the `total` field.
 
 
+[float]
+[[docs-update-by-query-cancel-task-api]]
+=== Works with the Cancel Task API
+
+Any Update By Query can be canceled using the <<tasks,Task Cancel API>>:
+
+[source,js]
+--------------------------------------------------
+POST /_tasks/{task_id}/_cancel
+--------------------------------------------------
+// AUTOSENSE
+
+The `task_id` can be found using the tasks API above.
+
+Cancelation should happen quickly but might take a few seconds. The task status
+API above will continue to list the task until it is wakes to cancel itself.
+
+
+[float]
+[[docs-update-by-query-rethrottle]]
+=== Rethrottling
+
+The value of `requests_per_second` can be changed on a running update by query
+using the `_rethrottle` API:
+
+[source,js]
+--------------------------------------------------
+POST /_update_by_query/{task_id}/_rethrottle?requests_per_second=unlimited
+--------------------------------------------------
+// AUTOSENSE
+
+The `task_id` can be found using the tasks API above.
+
+Just like when setting it on the `_update_by_query` API `requests_per_second`
+can be either `unlimited` to disable throttling or any decimal number like `1.7`
+or `12` to throttle to that level. Rethrottling that speeds up the query takes
+effect immediately but rethrotting that slows down the query will take effect
+on after completing the current batch. This prevents scroll timeouts.
+
+
 [float]
 [[picking-up-a-new-property]]
 === Pick up a new property
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
index f403178037b..efa01d78790 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
@@ -339,7 +339,7 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
      * How many nanoseconds should a batch of lastBatchSize have taken if it were perfectly throttled? Package private for testing.
      */
     float perfectlyThrottledBatchTime(int lastBatchSize) {
-        if (task.getRequestsPerSecond() == 0) {
+        if (task.getRequestsPerSecond() == Float.POSITIVE_INFINITY) {
             return 0;
         }
         //       requests
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
index a8b8f999c09..665ad37f1d8 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
@@ -48,17 +48,26 @@ public abstract class AbstractBaseReindexRestHandler<
      * @return requests_per_second from the request as a float if it was on the request, null otherwise
      */
     public static Float parseRequestsPerSecond(RestRequest request) {
-        String requestsPerSecond = request.param("requests_per_second");
-        if (requestsPerSecond == null) {
+        String requestsPerSecondString = request.param("requests_per_second");
+        if (requestsPerSecondString == null) {
             return null;
         }
-        if ("".equals(requestsPerSecond)) {
-            throw new IllegalArgumentException("requests_per_second cannot be an empty string");
+        if ("unlimited".equals(requestsPerSecondString)) {
+            return Float.POSITIVE_INFINITY;
         }
-        if ("unlimited".equals(requestsPerSecond)) {
-            return 0f;
+        float requestsPerSecond;
+        try {
+            requestsPerSecond = Float.parseFloat(requestsPerSecondString);
+        } catch (NumberFormatException e) {
+            throw new IllegalArgumentException(
+                    "[requests_per_second] must be a float greater than 0. Use \"unlimited\" to disable throttling.", e);
         }
-        return Float.parseFloat(requestsPerSecond);
+        if (requestsPerSecond <= 0) {
+            // We validate here and in the setters because the setters use "Float.POSITIVE_INFINITY" instead of "unlimited"
+            throw new IllegalArgumentException(
+                    "[requests_per_second] must be a float greater than 0. Use \"unlimited\" to disable throttling.");
+        }
+        return requestsPerSecond;
     }
 
     protected final IndicesQueriesRegistry indicesQueriesRegistry;
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
index 85236aeaf22..a5ec1962a55 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
@@ -87,11 +87,11 @@ public abstract class AbstractBulkByScrollRequest<Self extends AbstractBulkByScr
     private int maxRetries = 11;
 
     /**
-     * The throttle for this request in sub-requests per second. 0 means set no throttle and that is the default. Throttling is done between
-     * batches, as we start the next scroll requests. That way we can increase the scroll's timeout to make sure that it contains any time
-     * that we might wait.
+     * The throttle for this request in sub-requests per second. {@link Float#POSITIVE_INFINITY} means set no throttle and that is the
+     * default. Throttling is done between batches, as we start the next scroll requests. That way we can increase the scroll's timeout to
+     * make sure that it contains any time that we might wait.
      */
-    private float requestsPerSecond = 0;
+    private float requestsPerSecond = Float.POSITIVE_INFINITY;
 
     public AbstractBulkByScrollRequest() {
     }
@@ -264,16 +264,24 @@ public abstract class AbstractBulkByScrollRequest<Self extends AbstractBulkByScr
     }
 
     /**
-     * The throttle for this request in sub-requests per second. 0 means set no throttle and that is the default.
+     * The throttle for this request in sub-requests per second. {@link Float#POSITIVE_INFINITY} means set no throttle and that is the
+     * default. Throttling is done between batches, as we start the next scroll requests. That way we can increase the scroll's timeout to
+     * make sure that it contains any time that we might wait.
      */
     public float getRequestsPerSecond() {
         return requestsPerSecond;
     }
 
     /**
-     * Set the throttle for this request in sub-requests per second. 0 means set no throttle and that is the default.
+     * Set the throttle for this request in sub-requests per second. {@link Float#POSITIVE_INFINITY} means set no throttle and that is the
+     * default. Throttling is done between batches, as we start the next scroll requests. That way we can increase the scroll's timeout to
+     * make sure that it contains any time that we might wait.
      */
     public Self setRequestsPerSecond(float requestsPerSecond) {
+        if (requestsPerSecond <= 0) {
+            throw new IllegalArgumentException(
+                    "[requests_per_second] must be greater than 0. Use Float.POSITIVE_INFINITY to disable throttling.");
+        }
         this.requestsPerSecond = requestsPerSecond;
         return self();
     }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
index 18ebe42c44d..e93a7ba7e42 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
@@ -106,4 +106,14 @@ public abstract class AbstractBulkByScrollRequestBuilder<
         request.setConsistency(consistency);
         return self();
     }
+
+    /**
+     * Set the throttle for this request in sub-requests per second. {@link Float#POSITIVE_INFINITY} means set no throttle and that is the
+     * default. Throttling is done between batches, as we start the next scroll requests. That way we can increase the scroll's timeout to
+     * make sure that it contains any time that we might wait.
+     */
+    public Self setRequestsPerSecond(float requestsPerSecond) {
+        request.setRequestsPerSecond(requestsPerSecond);
+        return self();
+    }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java
index e5d39569236..fd86b488180 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java
@@ -20,14 +20,13 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.Action;
-import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.script.Script;
 
 public abstract class AbstractBulkIndexByScrollRequestBuilder<
                 Request extends AbstractBulkIndexByScrollRequest<Request>,
-                Response extends ActionResponse,
+                Response extends BulkIndexByScrollResponse,
                 Self extends AbstractBulkIndexByScrollRequestBuilder<Request, Response, Self>>
         extends AbstractBulkByScrollRequestBuilder<Request, Response, Self> {
 
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
index 888bfeb30ad..39a254bf757 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
@@ -190,7 +190,7 @@ public class BulkByScrollTask extends CancellableTask {
             builder.field("noops", noops);
             builder.field("retries", retries);
             builder.timeValueField("throttled_millis", "throttled", throttled);
-            builder.field("requests_per_second", requestsPerSecond == 0 ? "unlimited" : requestsPerSecond);
+            builder.field("requests_per_second", requestsPerSecond == Float.POSITIVE_INFINITY ? "unlimited" : requestsPerSecond);
             if (reasonCancelled != null) {
                 builder.field("canceled", reasonCancelled);
             }
@@ -393,9 +393,6 @@ public class BulkByScrollTask extends CancellableTask {
     }
 
     private void setRequestsPerSecond(float requestsPerSecond) {
-        if (requestsPerSecond == -1) {
-            requestsPerSecond = 0;
-        }
         this.requestsPerSecond = requestsPerSecond;
     }
 
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RethrottleRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RethrottleRequest.java
index 387b074f8c4..93f36d7034b 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RethrottleRequest.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RethrottleRequest.java
@@ -21,6 +21,10 @@ package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.tasks.BaseTasksRequest;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
@@ -29,11 +33,11 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
  */
 public class RethrottleRequest extends BaseTasksRequest<RethrottleRequest> {
     /**
-     * The throttle to apply to all matching requests in sub-requests per second. 0 means set no throttle and that is the default.
-     * Throttling is done between batches, as we start the next scroll requests. That way we can increase the scroll's timeout to make sure
-     * that it contains any time that we might wait.
+     * The throttle to apply to all matching requests in sub-requests per second. 0 means set no throttle. Throttling is done between
+     * batches, as we start the next scroll requests. That way we can increase the scroll's timeout to make sure that it contains any time
+     * that we might wait.
      */
-    private float requestsPerSecond = 0;
+    private Float requestsPerSecond;
 
     /**
      * The throttle to apply to all matching requests in sub-requests per second. 0 means set no throttle and that is the default.
@@ -43,9 +47,15 @@ public class RethrottleRequest extends BaseTasksRequest<RethrottleRequest> {
     }
 
     /**
-     * Set the throttle to apply to all matching requests in sub-requests per second. 0 means set no throttle and that is the default.
+     * Set the throttle to apply to all matching requests in sub-requests per second. {@link Float#POSITIVE_INFINITY} means set no throttle.
+     * Throttling is done between batches, as we start the next scroll requests. That way we can increase the scroll's timeout to make sure
+     * that it contains any time that we might wait.
      */
     public RethrottleRequest setRequestsPerSecond(float requestsPerSecond) {
+        if (requestsPerSecond <= 0) {
+            throw new IllegalArgumentException(
+                    "[requests_per_second] must be greater than 0. Use Float.POSITIVE_INFINITY to disable throttling.");
+        }
         this.requestsPerSecond = requestsPerSecond;
         return this;
     }
@@ -53,6 +63,9 @@ public class RethrottleRequest extends BaseTasksRequest<RethrottleRequest> {
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = super.validate();
+        if (requestsPerSecond == null) {
+            validationException = addValidationError("requests_per_second must be set", validationException);
+        }
         for (String action : getActions()) {
             switch (action) {
             case ReindexAction.NAME:
@@ -65,4 +78,16 @@ public class RethrottleRequest extends BaseTasksRequest<RethrottleRequest> {
         }
         return validationException;
     }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        requestsPerSecond = in.readFloat();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeFloat(requestsPerSecond);
+    }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RethrottleRequestBuilder.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RethrottleRequestBuilder.java
index e04ee055718..d5cc3d9c5d3 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RethrottleRequestBuilder.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RethrottleRequestBuilder.java
@@ -34,7 +34,9 @@ public class RethrottleRequestBuilder extends TasksRequestBuilder<RethrottleRequ
     }
 
     /**
-     * Set the throttle to apply to all matching requests in sub-requests per second. 0 means set no throttle and that is the default.
+     * Set the throttle to apply to all matching requests in sub-requests per second. {@link Float#POSITIVE_INFINITY} means set no throttle.
+     * Throttling is done between batches, as we start the next scroll requests. That way we can increase the scroll's timeout to make sure
+     * that it contains any time that we might wait.
      */
     public RethrottleRequestBuilder setRequestsPerSecond(float requestsPerSecond) {
         request.setRequestsPerSecond(requestsPerSecond);
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
index 048f95b1075..2409a99faf2 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
@@ -369,7 +369,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
 
     public void testPerfectlyThrottledBatchTime() {
         DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction();
-        testRequest.setRequestsPerSecond(0);
+        testRequest.setRequestsPerSecond(Float.POSITIVE_INFINITY);
         assertThat((double) action.perfectlyThrottledBatchTime(randomInt()), closeTo(0f, 0f));
 
         int total = between(0, 1000000);
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
index 440f712f352..ee07c4a9278 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
@@ -21,6 +21,9 @@ package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -40,6 +43,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 import static org.hamcrest.Matchers.both;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
@@ -49,7 +53,7 @@ public class BulkByScrollTaskTests extends ESTestCase {
 
     @Before
     public void createTask() {
-        task = new BulkByScrollTask(1, "test_type", "test_action", "test", TaskId.EMPTY_TASK_ID, 0);
+        task = new BulkByScrollTask(1, "test_type", "test_action", "test", TaskId.EMPTY_TASK_ID, Float.POSITIVE_INFINITY);
     }
 
     public void testBasicData() {
@@ -270,4 +274,10 @@ public class BulkByScrollTaskTests extends ESTestCase {
             threadPool.shutdown();
         }
     }
+
+    public void testXContentRepresentationOfUnlimitedRequestsPerSecon() throws IOException {
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        task.getStatus().toXContent(builder, ToXContent.EMPTY_PARAMS);
+        assertThat(builder.string(), containsString("\"requests_per_second\":\"unlimited\""));
+    }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTestUtils.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTestUtils.java
index e727431de17..763cca8c906 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTestUtils.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTestUtils.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ListenableActionFuture;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
@@ -51,7 +50,7 @@ import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThat;
 
 /**
- * Utilities for testing reindex and update-by-query cancellation. This whole class isn't thread safe. Luckily we run out tests in separate
+ * Utilities for testing reindex and update-by-query cancellation. This whole class isn't thread safe. Luckily we run our tests in separate
  * jvms.
  */
 public class CancelTestUtils {
@@ -62,7 +61,7 @@ public class CancelTestUtils {
     private static final CyclicBarrier barrier = new CyclicBarrier(2);
 
     public static <Request extends AbstractBulkIndexByScrollRequest<Request>,
-                    Response extends ActionResponse,
+                    Response extends BulkIndexByScrollResponse,
                     Builder extends AbstractBulkIndexByScrollRequestBuilder<Request, Response, Builder>>
             Response testCancel(ESIntegTestCase test, Builder request, String actionToCancel) throws Exception {
 
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java
index c169f6819ea..1ff7d6d4492 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java
@@ -37,22 +37,22 @@ public class ReindexBasicTests extends ReindexTestCase {
 
         // Copy all the docs
         ReindexRequestBuilder copy = reindex().source("source").destination("dest", "all").refresh(true);
-        assertThat(copy.get(), responseMatcher().created(4));
+        assertThat(copy.get(), reindexResponseMatcher().created(4));
         assertHitCount(client().prepareSearch("dest").setTypes("all").setSize(0).get(), 4);
 
         // Now none of them
         copy = reindex().source("source").destination("all", "none").filter(termQuery("foo", "no_match")).refresh(true);
-        assertThat(copy.get(), responseMatcher().created(0));
+        assertThat(copy.get(), reindexResponseMatcher().created(0));
         assertHitCount(client().prepareSearch("dest").setTypes("none").setSize(0).get(), 0);
 
         // Now half of them
         copy = reindex().source("source").destination("dest", "half").filter(termQuery("foo", "a")).refresh(true);
-        assertThat(copy.get(), responseMatcher().created(2));
+        assertThat(copy.get(), reindexResponseMatcher().created(2));
         assertHitCount(client().prepareSearch("dest").setTypes("half").setSize(0).get(), 2);
 
         // Limit with size
         copy = reindex().source("source").destination("dest", "size_one").size(1).refresh(true);
-        assertThat(copy.get(), responseMatcher().created(1));
+        assertThat(copy.get(), reindexResponseMatcher().created(1));
         assertHitCount(client().prepareSearch("dest").setTypes("size_one").setSize(0).get(), 1);
     }
 
@@ -70,7 +70,7 @@ public class ReindexBasicTests extends ReindexTestCase {
         ReindexRequestBuilder copy = reindex().source("source").destination("dest", "all").refresh(true);
         // Use a small batch size so we have to use more than one batch
         copy.source().setSize(5);
-        assertThat(copy.get(), responseMatcher().created(max).batches(max, 5));
+        assertThat(copy.get(), reindexResponseMatcher().created(max).batches(max, 5));
         assertHitCount(client().prepareSearch("dest").setTypes("all").setSize(0).get(), max);
 
         // Copy some of the docs
@@ -79,7 +79,7 @@ public class ReindexBasicTests extends ReindexTestCase {
         // Use a small batch size so we have to use more than one batch
         copy.source().setSize(5);
         copy.size(half); // The real "size" of the request.
-        assertThat(copy.get(), responseMatcher().created(half).batches(half, 5));
+        assertThat(copy.get(), reindexResponseMatcher().created(half).batches(half, 5));
         assertHitCount(client().prepareSearch("dest").setTypes("half").setSize(0).get(), half);
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexCancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexCancelTests.java
index 590957237f8..44f5a3296cb 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexCancelTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexCancelTests.java
@@ -35,7 +35,7 @@ public class ReindexCancelTests extends ReindexTestCase {
     public void testCancel() throws Exception {
         ReindexResponse response = CancelTestUtils.testCancel(this, reindex().destination("dest", "test"), ReindexAction.NAME);
 
-        assertThat(response, responseMatcher().created(1).reasonCancelled(equalTo("by user request")));
+        assertThat(response, reindexResponseMatcher().created(1).reasonCancelled(equalTo("by user request")));
         refresh("dest");
         assertHitCount(client().prepareSearch("dest").setSize(0).get(), 1);
     }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java
index 7aaec014d3e..196fa6c7e57 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java
@@ -58,7 +58,7 @@ public class ReindexFailureTests extends ReindexTestCase {
         copy.source().setSize(1);
 
         ReindexResponse response = copy.get();
-        assertThat(response, responseMatcher()
+        assertThat(response, reindexResponseMatcher()
                 .batches(1)
                 .failures(both(greaterThan(0)).and(lessThanOrEqualTo(maximumNumberOfShards()))));
         for (Failure failure: response.getIndexingFailures()) {
@@ -78,7 +78,7 @@ public class ReindexFailureTests extends ReindexTestCase {
         copy.destination().setOpType(CREATE);
 
         ReindexResponse response = copy.get();
-        assertThat(response, responseMatcher().batches(1).versionConflicts(1).failures(1).created(99));
+        assertThat(response, reindexResponseMatcher().batches(1).versionConflicts(1).failures(1).created(99));
         for (Failure failure: response.getIndexingFailures()) {
             assertThat(failure.getMessage(), containsString("VersionConflictEngineException[[test]["));
         }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParentChildTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParentChildTests.java
index b971e125b94..c2f0b5625d8 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParentChildTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParentChildTests.java
@@ -43,18 +43,18 @@ public class ReindexParentChildTests extends ReindexTestCase {
 
         // Copy parent to the new index
         ReindexRequestBuilder copy = reindex().source("source").destination("dest").filter(findsCountry).refresh(true);
-        assertThat(copy.get(), responseMatcher().created(1));
+        assertThat(copy.get(), reindexResponseMatcher().created(1));
 
         // Copy the child to a new index
         copy = reindex().source("source").destination("dest").filter(findsCity).refresh(true);
-        assertThat(copy.get(), responseMatcher().created(1));
+        assertThat(copy.get(), reindexResponseMatcher().created(1));
 
         // Make sure parent/child is intact on that index
         assertSearchHits(client().prepareSearch("dest").setQuery(findsCity).get(), "pittsburgh");
 
         // Copy the grandchild to a new index
         copy = reindex().source("source").destination("dest").filter(findsNeighborhood).refresh(true);
-        assertThat(copy.get(), responseMatcher().created(1));
+        assertThat(copy.get(), reindexResponseMatcher().created(1));
 
         // Make sure parent/child is intact on that index
         assertSearchHits(client().prepareSearch("dest").setQuery(findsNeighborhood).get(),
@@ -63,7 +63,7 @@ public class ReindexParentChildTests extends ReindexTestCase {
         // Copy the parent/child/grandchild structure all at once to a third index
         createParentChildIndex("dest_all_at_once");
         copy = reindex().source("source").destination("dest_all_at_once").refresh(true);
-        assertThat(copy.get(), responseMatcher().created(3));
+        assertThat(copy.get(), reindexResponseMatcher().created(3));
 
         // Make sure parent/child/grandchild is intact there too
         assertSearchHits(client().prepareSearch("dest_all_at_once").setQuery(findsNeighborhood).get(),
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java
index 8abdb39b6ae..d5426723b85 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java
@@ -41,11 +41,23 @@ public abstract class ReindexTestCase extends ESIntegTestCase {
         return ReindexAction.INSTANCE.newRequestBuilder(client());
     }
 
-    public IndexBySearchResponseMatcher responseMatcher() {
+    protected IndexBySearchResponseMatcher reindexResponseMatcher() {
         return new IndexBySearchResponseMatcher();
     }
 
-    public static class IndexBySearchResponseMatcher
+    protected UpdateByQueryRequestBuilder updateByQuery() {
+        return UpdateByQueryAction.INSTANCE.newRequestBuilder(client());
+    }
+
+    protected BulkIndexbyScrollResponseMatcher updateByQueryResponseMatcher() {
+        return new BulkIndexbyScrollResponseMatcher();
+    }
+
+    protected RethrottleRequestBuilder rethrottle() {
+        return RethrottleAction.INSTANCE.newRequestBuilder(client());
+    }
+
+    protected static class IndexBySearchResponseMatcher
             extends AbstractBulkIndexByScrollResponseMatcher<ReindexResponse, IndexBySearchResponseMatcher> {
         private Matcher<Long> createdMatcher = equalTo(0L);
 
@@ -74,4 +86,12 @@ public abstract class ReindexTestCase extends ESIntegTestCase {
             return this;
         }
     }
+
+    protected static class BulkIndexbyScrollResponseMatcher
+            extends AbstractBulkIndexByScrollResponseMatcher<BulkIndexByScrollResponse, BulkIndexbyScrollResponseMatcher> {
+        @Override
+        protected BulkIndexbyScrollResponseMatcher self() {
+            return this;
+        }
+    }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexVersioningTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexVersioningTests.java
index 725b55d76bb..0ce5b8740f7 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexVersioningTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexVersioningTests.java
@@ -33,55 +33,55 @@ public class ReindexVersioningTests extends ReindexTestCase {
 
     public void testExternalVersioningCreatesWhenAbsentAndSetsVersion() throws Exception {
         setupSourceAbsent();
-        assertThat(reindexExternal(), responseMatcher().created(1));
+        assertThat(reindexExternal(), reindexResponseMatcher().created(1));
         assertDest("source", SOURCE_VERSION);
     }
 
     public void testExternalVersioningUpdatesOnOlderAndSetsVersion() throws Exception {
         setupDestOlder();
-        assertThat(reindexExternal(), responseMatcher().updated(1));
+        assertThat(reindexExternal(), reindexResponseMatcher().updated(1));
         assertDest("source", SOURCE_VERSION);
     }
 
     public void testExternalVersioningVersionConflictsOnNewer() throws Exception {
         setupDestNewer();
-        assertThat(reindexExternal(), responseMatcher().versionConflicts(1));
+        assertThat(reindexExternal(), reindexResponseMatcher().versionConflicts(1));
         assertDest("dest", NEWER_VERSION);
     }
 
     public void testInternalVersioningCreatesWhenAbsent() throws Exception {
         setupSourceAbsent();
-        assertThat(reindexInternal(), responseMatcher().created(1));
+        assertThat(reindexInternal(), reindexResponseMatcher().created(1));
         assertDest("source", 1);
     }
 
     public void testInternalVersioningUpdatesOnOlder() throws Exception {
         setupDestOlder();
-        assertThat(reindexInternal(), responseMatcher().updated(1));
+        assertThat(reindexInternal(), reindexResponseMatcher().updated(1));
         assertDest("source", OLDER_VERSION + 1);
     }
 
     public void testInternalVersioningUpdatesOnNewer() throws Exception {
         setupDestNewer();
-        assertThat(reindexInternal(), responseMatcher().updated(1));
+        assertThat(reindexInternal(), reindexResponseMatcher().updated(1));
         assertDest("source", NEWER_VERSION + 1);
     }
 
     public void testCreateCreatesWhenAbsent() throws Exception {
         setupSourceAbsent();
-        assertThat(reindexCreate(), responseMatcher().created(1));
+        assertThat(reindexCreate(), reindexResponseMatcher().created(1));
         assertDest("source", 1);
     }
 
     public void testCreateVersionConflictsOnOlder() throws Exception {
         setupDestOlder();
-        assertThat(reindexCreate(), responseMatcher().versionConflicts(1));
+        assertThat(reindexCreate(), reindexResponseMatcher().versionConflicts(1));
         assertDest("dest", OLDER_VERSION);
     }
 
     public void testCreateVersionConflictsOnNewer() throws Exception {
         setupDestNewer();
-        assertThat(reindexCreate(), responseMatcher().versionConflicts(1));
+        assertThat(reindexCreate(), reindexResponseMatcher().versionConflicts(1));
         assertDest("dest", NEWER_VERSION);
     }
 
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
new file mode 100644
index 00000000000..ab4b76ef23f
--- /dev/null
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.ListenableActionFuture;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
+
+import static org.hamcrest.Matchers.hasSize;
+
+/**
+ * Tests that you can set requests_per_second over the Java API and that you can rethrottle running requests. There are REST tests for this
+ * too but this is the only place that tests running against multiple nodes so it is the only integration tests that checks for
+ * serialization.
+ */
+public class RethrottleTests extends ReindexTestCase {
+
+    public void testReindex() throws Exception {
+        testCase(reindex().source("test").destination("dest"), ReindexAction.NAME);
+    }
+
+    public void testUpdateByQuery() throws Exception {
+        testCase(updateByQuery().source("test"), UpdateByQueryAction.NAME);
+    }
+
+    private void testCase(AbstractBulkIndexByScrollRequestBuilder<?, ? extends BulkIndexByScrollResponse, ?> request, String actionName)
+            throws Exception {
+        // Use a single shard so the reindex has to happen in multiple batches
+        client().admin().indices().prepareCreate("test").setSettings("index.number_of_shards", 1).get();
+        indexRandom(true,
+                client().prepareIndex("test", "test", "1").setSource("foo", "bar"),
+                client().prepareIndex("test", "test", "2").setSource("foo", "bar"),
+                client().prepareIndex("test", "test", "3").setSource("foo", "bar"));
+
+        // Start a request that will never finish unless we rethrottle it
+        request.setRequestsPerSecond(.000001f);  // Throttle forever
+        request.source().setSize(1);             // Make sure we use multiple batches
+        ListenableActionFuture<? extends BulkIndexByScrollResponse> responseListener = request.execute();
+
+        // Now rethrottle it so it'll finish
+        ListTasksResponse rethrottleResponse = rethrottle().setActions(actionName).setRequestsPerSecond(Float.POSITIVE_INFINITY).get();
+        assertThat(rethrottleResponse.getTasks(), hasSize(1));
+
+        // Now the response should come back quickly because we've rethrottled the request
+        BulkIndexByScrollResponse response = responseListener.get();
+        assertEquals("Batches didn't match, this may invalidate the test as throttling is done between batches", 3, response.getBatches());
+    }
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
index 06bc0dba8ed..cbbe77e6479 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
@@ -33,6 +33,7 @@ import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.search.SearchShardTarget;
+import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -120,9 +121,24 @@ public class RoundTripTests extends ESTestCase {
         assertResponseEquals(response, tripped);
     }
 
+    public void testRethrottleRequest() throws IOException {
+        RethrottleRequest request = new RethrottleRequest();
+        request.setRequestsPerSecond((float) randomDoubleBetween(0, Float.POSITIVE_INFINITY, false));
+        if (randomBoolean()) {
+            request.setActions(randomFrom(UpdateByQueryAction.NAME, ReindexAction.NAME));
+        } else {
+            request.setTaskId(new TaskId(randomAsciiOfLength(5), randomLong()));
+        }
+        RethrottleRequest tripped = new RethrottleRequest();
+        roundTrip(request, tripped);
+        assertEquals(request.getRequestsPerSecond(), tripped.getRequestsPerSecond(), 0.00001);
+        assertArrayEquals(request.getActions(), tripped.getActions());
+        assertEquals(request.getTaskId(), tripped.getTaskId());
+    }
+
     private BulkByScrollTask.Status randomStatus() {
         return new BulkByScrollTask.Status(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
-                randomPositiveInt(), randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
+                randomInt(Integer.MAX_VALUE), randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
                 parseTimeValue(randomPositiveTimeValue(), "test"), abs(random().nextFloat()),
                 random().nextBoolean() ? null : randomSimpleString(random()), parseTimeValue(randomPositiveTimeValue(), "test"));
     }
@@ -163,10 +179,6 @@ public class RoundTripTests extends ESTestCase {
         return l;
     }
 
-    private int randomPositiveInt() {
-        return randomInt(Integer.MAX_VALUE);
-    }
-
     private void assertResponseEquals(BulkIndexByScrollResponse expected, BulkIndexByScrollResponse actual) {
         assertEquals(expected.getTook(), actual.getTook());
         assertTaskStatusEquals(expected.getStatus(), actual.getStatus());
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java
index 75a7ee2284b..3d70d71a9bd 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java
@@ -24,7 +24,7 @@ import org.elasticsearch.search.sort.SortOrder;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 
-public class UpdateByQueryBasicTests extends UpdateByQueryTestCase {
+public class UpdateByQueryBasicTests extends ReindexTestCase {
     public void testBasics() throws Exception {
         indexRandom(true, client().prepareIndex("test", "test", "1").setSource("foo", "a"),
                 client().prepareIndex("test", "test", "2").setSource("foo", "a"),
@@ -35,26 +35,28 @@ public class UpdateByQueryBasicTests extends UpdateByQueryTestCase {
         assertEquals(1, client().prepareGet("test", "test", "4").get().getVersion());
 
         // Reindex all the docs
-        assertThat(request().source("test").refresh(true).get(), responseMatcher().updated(4));
+        assertThat(updateByQuery().source("test").refresh(true).get(), updateByQueryResponseMatcher().updated(4));
         assertEquals(2, client().prepareGet("test", "test", "1").get().getVersion());
         assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion());
 
         // Now none of them
-        assertThat(request().source("test").filter(termQuery("foo", "no_match")).refresh(true).get(), responseMatcher().updated(0));
+        assertThat(updateByQuery().source("test").filter(termQuery("foo", "no_match")).refresh(true).get(),
+                updateByQueryResponseMatcher().updated(0));
         assertEquals(2, client().prepareGet("test", "test", "1").get().getVersion());
         assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion());
 
         // Now half of them
-        assertThat(request().source("test").filter(termQuery("foo", "a")).refresh(true).get(), responseMatcher().updated(2));
+        assertThat(updateByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).get(),
+                updateByQueryResponseMatcher().updated(2));
         assertEquals(3, client().prepareGet("test", "test", "1").get().getVersion());
         assertEquals(3, client().prepareGet("test", "test", "2").get().getVersion());
         assertEquals(2, client().prepareGet("test", "test", "3").get().getVersion());
         assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion());
 
         // Limit with size
-        UpdateByQueryRequestBuilder request = request().source("test").size(3).refresh(true);
+        UpdateByQueryRequestBuilder request = updateByQuery().source("test").size(3).refresh(true);
         request.source().addSort("foo.keyword", SortOrder.ASC);
-        assertThat(request.get(), responseMatcher().updated(3));
+        assertThat(request.get(), updateByQueryResponseMatcher().updated(3));
         // Only the first three documents are updated because of sort
         assertEquals(4, client().prepareGet("test", "test", "1").get().getVersion());
         assertEquals(4, client().prepareGet("test", "test", "2").get().getVersion());
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryCancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryCancelTests.java
index 069049765b8..dd63f90322e 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryCancelTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryCancelTests.java
@@ -32,11 +32,11 @@ import static org.hamcrest.Matchers.equalTo;
  * places - that is the responsibility of {@link AsyncBulkByScrollActionTests} which have more precise control to simulate failures but do
  * not exercise important portion of the stack like transport and task management.
  */
-public class UpdateByQueryCancelTests extends UpdateByQueryTestCase {
+public class UpdateByQueryCancelTests extends ReindexTestCase {
     public void testCancel() throws Exception {
-        BulkIndexByScrollResponse response = CancelTestUtils.testCancel(this, request(), UpdateByQueryAction.NAME);
+        BulkIndexByScrollResponse response = CancelTestUtils.testCancel(this, updateByQuery(), UpdateByQueryAction.NAME);
 
-        assertThat(response, responseMatcher().updated(1).reasonCancelled(equalTo("by user request")));
+        assertThat(response, updateByQueryResponseMatcher().updated(1).reasonCancelled(equalTo("by user request")));
         refresh("source");
         assertHitCount(client().prepareSearch("source").setSize(0).setQuery(matchQuery("giraffes", "giraffes")).get(), 1);
     }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryTestCase.java
deleted file mode 100644
index 09613eaffeb..00000000000
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryTestCase.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.reindex;
-
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-
-import java.util.Collection;
-
-import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
-
-@ClusterScope(scope = SUITE, transportClientRatio = 0)
-public abstract class UpdateByQueryTestCase extends ESIntegTestCase {
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return pluginList(ReindexPlugin.class);
-    }
-
-    protected UpdateByQueryRequestBuilder request() {
-        return UpdateByQueryAction.INSTANCE.newRequestBuilder(client());
-    }
-
-    public BulkIndexbyScrollResponseMatcher responseMatcher() {
-        return new BulkIndexbyScrollResponseMatcher();
-    }
-
-    public static class BulkIndexbyScrollResponseMatcher extends
-            AbstractBulkIndexByScrollResponseMatcher<BulkIndexByScrollResponse, BulkIndexbyScrollResponseMatcher> {
-        @Override
-        protected BulkIndexbyScrollResponseMatcher self() {
-            return this;
-        }
-    }
-}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java
index e2776d4d5d1..c36cbe974dc 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java
@@ -35,7 +35,7 @@ import static org.hamcrest.Matchers.equalTo;
  * Mutates a document while update-by-query-ing it and asserts that the mutation
  * always sticks. Update-by-query should never revert documents.
  */
-public class UpdateByQueryWhileModifyingTests extends UpdateByQueryTestCase {
+public class UpdateByQueryWhileModifyingTests extends ReindexTestCase {
     private static final int MAX_MUTATIONS = 50;
     private static final int MAX_ATTEMPTS = 10;
 
@@ -48,8 +48,9 @@ public class UpdateByQueryWhileModifyingTests extends UpdateByQueryTestCase {
         Thread updater = new Thread(() -> {
             while (keepUpdating.get()) {
                 try {
-                    assertThat(request().source("test").refresh(true).abortOnVersionConflict(false).get(), responseMatcher()
-                            .updated(either(equalTo(0L)).or(equalTo(1L))).versionConflicts(either(equalTo(0L)).or(equalTo(1L))));
+                    BulkIndexByScrollResponse response = updateByQuery().source("test").refresh(true).abortOnVersionConflict(false).get();
+                    assertThat(response, updateByQueryResponseMatcher().updated(either(equalTo(0L)).or(equalTo(1L)))
+                            .versionConflicts(either(equalTo(0L)).or(equalTo(1L))));
                 } catch (Throwable t) {
                     failure.set(t);
                 }
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
index 542eb3bd932..dc54c0d91cf 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
@@ -152,7 +152,7 @@
 ---
 "requests_per_second cannot be an empty string":
   - do:
-      catch: /requests_per_second cannot be an empty string/
+      catch: /\[requests_per_second\] must be a float greater than 0. Use "unlimited" to disable throttling./
       reindex:
         requests_per_second: ""
         body:
@@ -161,6 +161,30 @@
           dest:
             index: dest
 
+---
+"requests_per_second cannot be negative":
+  - do:
+      catch: /\[requests_per_second\] must be a float greater than 0. Use "unlimited" to disable throttling./
+      reindex:
+        requests_per_second: -12
+        body:
+          source:
+            index: test
+          dest:
+            index: dest
+
+---
+"requests_per_second cannot be zero":
+  - do:
+      catch: /\[requests_per_second\] must be a float greater than 0. Use "unlimited" to disable throttling./
+      reindex:
+        requests_per_second: 0
+        body:
+          source:
+            index: test
+          dest:
+            index: dest
+
 ---
 "reindex without source gives useful error message":
   - do:
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_throttle.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_throttle.yaml
index 9b818268d4a..597cfa4240f 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_throttle.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_throttle.yaml
@@ -52,6 +52,56 @@
   - gte: { took: 1000 }
   - is_false: task
 
+---
+"requests_per_second supports unlimited to turn off throttling":
+  - do:
+      indices.create:
+        index: source
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+  - do:
+      cluster.health:
+          wait_for_status: yellow
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      2
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      3
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      reindex:
+        requests_per_second: unlimited
+        body:
+          source:
+            index: source
+            size: 1
+          dest:
+            index: dest
+  - match: {created: 3}
+  - match: {updated: 0}
+  - match: {version_conflicts: 0}
+  - match: {batches: 3}
+  - match: {failures: []}
+  - match: {throttled_millis: 0}
+  - is_false: task
+
 ---
 "Rethrottle":
   # Throttling happens between each scroll batch so we need to control the size of the batch by using a single shard
@@ -110,64 +160,6 @@
         wait_for_completion: true
         task_id: $task
 
----
-"Rethrottle to -1 which also means unlimited":
-  # Throttling happens between each scroll batch so we need to control the size of the batch by using a single shard
-  # and a small batch size on the request
-  - do:
-      indices.create:
-        index: source
-        body:
-          settings:
-            number_of_shards: "1"
-            number_of_replicas: "0"
-  - do:
-      cluster.health:
-          wait_for_status: yellow
-  - do:
-      index:
-        index:   source
-        type:    foo
-        id:      1
-        body:    { "text": "test" }
-  - do:
-      index:
-        index:   source
-        type:    foo
-        id:      2
-        body:    { "text": "test" }
-  - do:
-      index:
-        index:   source
-        type:    foo
-        id:      3
-        body:    { "text": "test" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      reindex:
-        requests_per_second: .00000001 # About 9.5 years to complete the request
-        wait_for_completion: false
-        body:
-          source:
-            index: source
-            size: 1
-          dest:
-            index: dest
-  - match: {task: '/.+:\d+/'}
-  - set: {task: task}
-
-  - do:
-      reindex.rethrottle:
-        requests_per_second: -1
-        task_id: $task
-
-  - do:
-      tasks.list:
-        wait_for_completion: true
-        task_id: $task
-
 ---
 "Rethrottle but not unlimited":
   # Throttling happens between each scroll batch so we need to control the size of the batch by using a single shard
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml
index f237a711f41..ea487eb54e0 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml
@@ -40,20 +40,6 @@
         index: test
         scroll_size: asdf
 
----
-"requests_per_second cannot be an empty string":
-  - do:
-      index:
-        index:   test
-        type:    test
-        id:      1
-        body:    { "text": "test" }
-  - do:
-      catch: /requests_per_second cannot be an empty string/
-      update_by_query:
-        index: test
-        requests_per_second: ''
-
 ---
 "update_by_query without source gives useful error message":
   - do:
@@ -86,3 +72,27 @@
         index: test
         body:
           fields: [_id]
+
+---
+"requests_per_second cannot be an empty string":
+  - do:
+      catch: /\[requests_per_second\] must be a float greater than 0. Use "unlimited" to disable throttling./
+      update_by_query:
+        requests_per_second: ""
+        index: test
+
+---
+"requests_per_second cannot be negative":
+  - do:
+      catch: /\[requests_per_second\] must be a float greater than 0. Use "unlimited" to disable throttling./
+      update_by_query:
+        requests_per_second: -12
+        index: test
+
+---
+"requests_per_second cannot be zero":
+  - do:
+      catch: /\[requests_per_second\] must be a float greater than 0. Use "unlimited" to disable throttling./
+      update_by_query:
+        requests_per_second: 0
+        index: test
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/70_throttle.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/70_throttle.yaml
index 4d6377a5b64..3179ce32a19 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/70_throttle.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/70_throttle.yaml
@@ -38,6 +38,44 @@
   - gt: {throttled_millis: 1000}
   - lt: {throttled_millis: 4000}
 
+---
+"requests_per_second supports unlimited which turns off throttling":
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 1
+  - do:
+      cluster.health:
+          wait_for_status: yellow
+  - do:
+      index:
+        index:   test
+        type:    foo
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      update_by_query:
+        index: test
+        scroll_size: 1
+        requests_per_second: unlimited
+  - match: {batches: 3}
+  - match: {updated: 3}
+  - match: {throttled_millis: 0}
+
 ---
 "Rethrottle":
   # Throttling happens between each scroll batch so we need to control the size of the batch by using a single shard
@@ -88,56 +126,6 @@
         wait_for_completion: true
         task_id: $task
 
----
-"Rethrottle to -1 which also means unlimited":
-  # Throttling happens between each scroll batch so we need to control the size of the batch by using a single shard
-  # and a small batch size on the request
-  - do:
-      indices.create:
-        index: test
-        body:
-          settings:
-            number_of_shards: 1
-  - do:
-      cluster.health:
-          wait_for_status: yellow
-  - do:
-      index:
-        index:   test
-        type:    foo
-        body:    { "text": "test" }
-  - do:
-      index:
-        index:   test
-        type:    foo
-        body:    { "text": "test" }
-  - do:
-      index:
-        index:   test
-        type:    foo
-        body:    { "text": "test" }
-  - do:
-      indices.refresh: {}
-
-  - do:
-      update_by_query:
-        requests_per_second: .00000001 # About 9.5 years to complete the request
-        wait_for_completion: false
-        index: test
-        scroll_size: 1
-  - match: {task: '/.+:\d+/'}
-  - set: {task: task}
-
-  - do:
-      reindex.rethrottle:
-        requests_per_second: -1
-        task_id: $task
-
-  - do:
-      tasks.list:
-        wait_for_completion: true
-        task_id: $task
-
 ---
 "Rethrottle but not unlimited":
   # Throttling happens between each scroll batch so we need to control the size of the batch by using a single shard
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.rethrottle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.rethrottle.json
index 7b8705727a2..4903c7598c3 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.rethrottle.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.rethrottle.json
@@ -14,7 +14,7 @@
       "params": {
         "requests_per_second": {
           "type": "float",
-          "default": 0,
+          "required": true,
           "description": "The throttle to set on this request in sub-requests per second. 0 means set no throttle. As does \"unlimited\". Otherwise it must be a float."
         }
       }

From e49d21bdd6f9a50fd1356fe804c486ee2ee3debc Mon Sep 17 00:00:00 2001
From: debadair <deb.adair@elasticsearch.com>
Date: Wed, 4 May 2016 13:38:57 -0700
Subject: [PATCH 0076/1311] Docs: Fixed broken cross doc links to script
 topics.

---
 docs/plugins/lang-javascript.asciidoc | 12 ++++++------
 docs/plugins/lang-python.asciidoc     | 12 ++++++------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/docs/plugins/lang-javascript.asciidoc b/docs/plugins/lang-javascript.asciidoc
index b3e811209b2..856e4bf4f03 100644
--- a/docs/plugins/lang-javascript.asciidoc
+++ b/docs/plugins/lang-javascript.asciidoc
@@ -48,7 +48,7 @@ Scripting is available in many APIs, but we will use an example with the
 WARNING: Enabling inline scripting on an unprotected Elasticsearch cluster is dangerous.
 See <<lang-javascript-file>> for a safer option.
 
-If you have enabled {ref}/modules-scripting.html#enable-dynamic-scripting[inline scripts],
+If you have enabled {ref}/modules-scripting-security.html#enable-dynamic-scripting[inline scripts],
 you can use JavaScript as follows:
 
 [source,json]
@@ -84,14 +84,14 @@ GET test/_search
 ----
 // AUTOSENSE
 
-[[lang-javascript-indexed]]
+[[lang-javascript-stored]]
 [float]
-=== Indexed scripts
+=== Stored scripts
 
-WARNING: Enabling indexed scripting on an unprotected Elasticsearch cluster is dangerous.
+WARNING: Enabling stored scripts on an unprotected Elasticsearch cluster is dangerous.
 See <<lang-javascript-file>> for a safer option.
 
-If you have enabled {ref}/modules-scripting.html#enable-dynamic-scripting[indexed scripts],
+If you have enabled {ref}/modules-scripting-security.html#enable-dynamic-scripting[stored scripts],
 you can use JavaScript as follows:
 
 [source,json]
@@ -133,7 +133,7 @@ GET test/_search
 ----
 // AUTOSENSE
 
-<1> We index the script under the id `my_script`.
+<1> We store the script under the id `my_script`.
 <2> The function score query retrieves the script with id `my_script`.
 
 
diff --git a/docs/plugins/lang-python.asciidoc b/docs/plugins/lang-python.asciidoc
index 9146af03381..49494828b13 100644
--- a/docs/plugins/lang-python.asciidoc
+++ b/docs/plugins/lang-python.asciidoc
@@ -47,7 +47,7 @@ Scripting is available in many APIs, but we will use an example with the
 WARNING: Enabling inline scripting on an unprotected Elasticsearch cluster is dangerous.
 See <<lang-python-file>> for a safer option.
 
-If you have enabled {ref}/modules-scripting.html#enable-dynamic-scripting[inline scripts],
+If you have enabled {ref}/modules-scripting-security.html#enable-dynamic-scripting[inline scripts],
 you can use Python as follows:
 
 [source,json]
@@ -83,14 +83,14 @@ GET test/_search
 ----
 // AUTOSENSE
 
-[[lang-python-indexed]]
+[[lang-python-stored]]
 [float]
-=== Indexed scripts
+=== Stored scripts
 
-WARNING: Enabling indexed scripting on an unprotected Elasticsearch cluster is dangerous.
+WARNING: Enabling stored scripts on an unprotected Elasticsearch cluster is dangerous.
 See <<lang-python-file>> for a safer option.
 
-If you have enabled {ref}/modules-scripting.html#enable-dynamic-scripting[indexed scripts],
+If you have enabled {ref}/modules-scripting-security.html#enable-dynamic-scripting[stored scripts],
 you can use Python as follows:
 
 [source,json]
@@ -132,7 +132,7 @@ GET test/_search
 ----
 // AUTOSENSE
 
-<1> We index the script under the id `my_script`.
+<1> We store the script under the id `my_script`.
 <2> The function score query retrieves the script with id `my_script`.
 
 

From 07d8b3eb2b0146e3962ebe88d1b290745c5dbffa Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Wed, 4 May 2016 22:48:51 +0200
Subject: [PATCH 0077/1311] Fixed a bad asciidoc link

---
 docs/plugins/mapper-size.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/plugins/mapper-size.asciidoc b/docs/plugins/mapper-size.asciidoc
index 3213d502792..708c8bbd8f7 100644
--- a/docs/plugins/mapper-size.asciidoc
+++ b/docs/plugins/mapper-size.asciidoc
@@ -104,5 +104,5 @@ GET my_index/_search
 <1> Querying on the `_size` field
 <2> Aggregating on the `_size` field
 <3> Sorting on the `_size` field
-<4> Accessing the `_size` field in scripts (inline scripts must be modules-scripting.html#enable-dynamic-scripting[enabled] for this example to work)
+<4> Accessing the `_size` field in scripts (inline scripts must be modules-security-scripting.html#enable-dynamic-scripting[enabled] for this example to work)
 

From 784c9e5fb9c055f5bf25a7ba214d6c8644fd463c Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 4 May 2016 20:06:47 -0400
Subject: [PATCH 0078/1311] Introduce node handshake

This commit introduces a handshake when initiating a light
connection. During this handshake, node information, cluster name, and
version are received from the target node of the connection. This
information can be used to immediately validate that the target node is
a member of the same cluster, and used to set the version on the
stream. This will allow us to extend APIs that are used during initial
cluster recovery without a major version change.

Relates #15971
---
 .../TransportClientNodesService.java          |   5 +-
 .../zen/ping/unicast/UnicastZenPing.java      |   8 +-
 .../transport/TransportService.java           | 133 ++++++++++-
 .../node/tasks/TaskManagerTestCase.java       |   5 +-
 .../bulk/TransportBulkActionTookTests.java    |   2 +-
 .../TransportBroadcastByNodeActionTests.java  |   2 +-
 .../TransportMasterNodeActionTests.java       |   2 +-
 .../nodes/TransportNodesActionTests.java      |   2 +-
 .../BroadcastReplicationTests.java            |   2 +-
 .../TransportReplicationActionTests.java      |   2 +-
 ...ortInstanceSingleOperationActionTests.java |   2 +-
 .../transport/FailAndRetryMockTransport.java  |  13 +-
 .../TransportClientHeadersTests.java          |   4 +-
 .../TransportClientNodesServiceTests.java     |  18 +-
 .../cluster/NodeConnectionsServiceTests.java  |   2 +-
 .../action/shard/ShardStateActionTests.java   |   2 +-
 .../health/ClusterStateHealthTests.java       |   2 +-
 .../common/network/NetworkModuleTests.java    |   2 +-
 .../discovery/ZenFaultDetectionTests.java     |   8 +-
 .../zen/ping/unicast/UnicastZenPingIT.java    | 138 ++++++++++--
 .../PublishClusterStateActionTests.java       |  21 +-
 .../mapper/DynamicMappingDisabledTests.java   |   6 +-
 .../indices/store/IndicesStoreTests.java      |   2 +-
 .../AbstractSimpleTransportTestCase.java      |  19 +-
 .../NettySizeHeaderFrameDecoderTests.java     |   4 +-
 .../NettyTransportServiceHandshakeTests.java  | 206 ++++++++++++++++++
 .../transport/TransportModuleTests.java       |   5 +-
 .../local/SimpleLocalTransportTests.java      |   5 +-
 .../netty/NettyScheduledPingTests.java        |   7 +-
 .../netty/SimpleNettyTransportTests.java      |   5 +-
 .../messy/tests/IndicesRequestTests.java      |   5 +-
 .../discovery/ec2/Ec2DiscoveryTests.java      |   3 +-
 .../discovery/gce/GceDiscoveryTests.java      |   3 +-
 .../test/transport/MockTransportService.java  |  19 +-
 34 files changed, 558 insertions(+), 106 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java

diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
index e407a2e7ada..68ed7c927ac 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
@@ -360,9 +360,10 @@ public class TransportClientNodesService extends AbstractComponent {
                     try {
                         // its a listed node, light connect to it...
                         logger.trace("connecting to listed node (light) [{}]", listedNode);
-                        transportService.connectToNodeLight(listedNode);
+                        transportService.connectToNodeLight(listedNode, pingTimeout, !ignoreClusterName);
                     } catch (Throwable e) {
                         logger.debug("failed to connect to node [{}], removed from nodes list", e, listedNode);
+                        newFilteredNodes.add(listedNode);
                         continue;
                     }
                 }
@@ -434,7 +435,7 @@ public class TransportClientNodesService extends AbstractComponent {
                                     } else {
                                         // its a listed node, light connect to it...
                                         logger.trace("connecting to listed node (light) [{}]", listedNode);
-                                        transportService.connectToNodeLight(listedNode);
+                                        transportService.connectToNodeLight(listedNode, pingTimeout, !ignoreClusterName);
                                     }
                                 } catch (Exception e) {
                                     logger.debug("failed to connect to node [{}], ignoring...", e, listedNode);
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
index 640582af226..cc37504360c 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
@@ -402,7 +402,7 @@ public class UnicastZenPing extends AbstractLifecycleComponent<ZenPing> implemen
                             // connect to the node, see if we manage to do it, if not, bail
                             if (!nodeFoundByAddress) {
                                 logger.trace("[{}] connecting (light) to {}", sendPingsHandler.id(), finalNodeToSend);
-                                transportService.connectToNodeLight(finalNodeToSend);
+                                transportService.connectToNodeLight(finalNodeToSend, timeout.getMillis());
                             } else {
                                 logger.trace("[{}] connecting to {}", sendPingsHandler.id(), finalNodeToSend);
                                 transportService.connectToNode(finalNodeToSend);
@@ -473,12 +473,6 @@ public class UnicastZenPing extends AbstractLifecycleComponent<ZenPing> implemen
                             // that's us, ignore
                             continue;
                         }
-                        if (!pingResponse.clusterName().equals(clusterName)) {
-                            // not part of the cluster
-                            logger.debug("[{}] filtering out response from {}, not same cluster_name [{}]", id, pingResponse.node(),
-                                    pingResponse.clusterName().value());
-                            continue;
-                        }
                         SendPingsHandler sendPingsHandler = receivedResponses.get(response.id);
                         if (sendPingsHandler == null) {
                             if (!closed) {
diff --git a/core/src/main/java/org/elasticsearch/transport/TransportService.java b/core/src/main/java/org/elasticsearch/transport/TransportService.java
index b9663da72c2..89cc68debfd 100644
--- a/core/src/main/java/org/elasticsearch/transport/TransportService.java
+++ b/core/src/main/java/org/elasticsearch/transport/TransportService.java
@@ -19,12 +19,16 @@
 
 package org.elasticsearch.transport;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.metrics.MeanMetric;
@@ -50,6 +54,7 @@ import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ScheduledFuture;
@@ -67,10 +72,12 @@ import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
 public class TransportService extends AbstractLifecycleComponent<TransportService> {
 
     public static final String DIRECT_RESPONSE_PROFILE = ".direct";
+    private static final String HANDSHAKE_ACTION_NAME = "internal:transport/handshake";
 
     private final CountDownLatch blockIncomingRequestsLatch = new CountDownLatch(1);
     protected final Transport transport;
     protected final ThreadPool threadPool;
+    private final ClusterName clusterName;
     protected final TaskManager taskManager;
 
     volatile Map<String, RequestHandlerRegistry> requestHandlers = Collections.emptyMap();
@@ -110,15 +117,16 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
     /** if set will call requests sent to this id to shortcut and executed locally */
     volatile DiscoveryNode localNode = null;
 
-    public TransportService(Transport transport, ThreadPool threadPool) {
-        this(EMPTY_SETTINGS, transport, threadPool);
+    public TransportService(Transport transport, ThreadPool threadPool, ClusterName clusterName) {
+        this(EMPTY_SETTINGS, transport, threadPool, clusterName);
     }
 
     @Inject
-    public TransportService(Settings settings, Transport transport, ThreadPool threadPool) {
+    public TransportService(Settings settings, Transport transport, ThreadPool threadPool, ClusterName clusterName) {
         super(settings);
         this.transport = transport;
         this.threadPool = threadPool;
+        this.clusterName = clusterName;
         setTracerLogInclude(TRACE_LOG_INCLUDE_SETTING.get(settings));
         setTracerLogExclude(TRACE_LOG_EXCLUDE_SETTING.get(settings));
         tracerLog = Loggers.getLogger(logger, ".tracer");
@@ -178,6 +186,12 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
                 logger.info("profile [{}]: {}", entry.getKey(), entry.getValue());
             }
         }
+        registerRequestHandler(
+            HANDSHAKE_ACTION_NAME,
+            () -> HandshakeRequest.INSTANCE,
+            ThreadPool.Names.SAME,
+            (request, channel) -> channel.sendResponse(
+                    new HandshakeResponse(localNode, clusterName, localNode != null ? localNode.getVersion() : Version.CURRENT)));
     }
 
     @Override
@@ -263,11 +277,120 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
         transport.connectToNode(node);
     }
 
-    public void connectToNodeLight(DiscoveryNode node) throws ConnectTransportException {
+    /**
+     * Lightly connect to the specified node, and handshake cluster
+     * name and version
+     *
+     * @param node             the node to connect to
+     * @param handshakeTimeout handshake timeout
+     * @return the connected node with version set
+     * @throws ConnectTransportException if the connection or the
+     *                                   handshake failed
+     */
+    public DiscoveryNode connectToNodeLight(final DiscoveryNode node, final long handshakeTimeout) throws ConnectTransportException {
+        return connectToNodeLight(node, handshakeTimeout, true);
+    }
+
+    /**
+     * Lightly connect to the specified node, returning updated node
+     * information. The handshake will fail if the cluster name on the
+     * target node mismatches the local cluster name and
+     * {@code checkClusterName} is {@code true}.
+     *
+     * @param node             the node to connect to
+     * @param handshakeTimeout handshake timeout
+     * @param checkClusterName whether or not to ignore cluster name
+     *                         mismatches
+     * @return the connected node
+     * @throws ConnectTransportException if the connection or the
+     *                                   handshake failed
+     */
+    public DiscoveryNode connectToNodeLight(final DiscoveryNode node, final long handshakeTimeout, final boolean checkClusterName) {
         if (node.equals(localNode)) {
-            return;
+            return localNode;
         }
         transport.connectToNodeLight(node);
+        try {
+            return handshake(node, handshakeTimeout, checkClusterName);
+        } catch (ConnectTransportException e) {
+            transport.disconnectFromNode(node);
+            throw e;
+        }
+    }
+
+    private DiscoveryNode handshake(
+            final DiscoveryNode node,
+            final long handshakeTimeout,
+            final boolean checkClusterName) throws ConnectTransportException {
+        final HandshakeResponse response;
+        try {
+            response = this.submitRequest(
+                node,
+                HANDSHAKE_ACTION_NAME,
+                HandshakeRequest.INSTANCE,
+                TransportRequestOptions.builder().withTimeout(handshakeTimeout).build(),
+                new FutureTransportResponseHandler<HandshakeResponse>() {
+                    @Override
+                    public HandshakeResponse newInstance() {
+                        return new HandshakeResponse();
+                    }
+                }).txGet();
+        } catch (Exception e) {
+            throw new ConnectTransportException(node, "handshake failed", e);
+        }
+
+        if (checkClusterName && !Objects.equals(clusterName, response.clusterName)) {
+            throw new ConnectTransportException(node, "handshake failed, mismatched cluster name [" + response.clusterName + "]");
+        } else if (!isVersionCompatible(response.version)) {
+            throw new ConnectTransportException(node, "handshake failed, incompatible version [" + response.version + "]");
+        }
+
+        return response.discoveryNode;
+    }
+
+    private boolean isVersionCompatible(Version version) {
+        return version.minimumCompatibilityVersion().equals(
+                localNode != null ? localNode.getVersion().minimumCompatibilityVersion() : Version.CURRENT.minimumCompatibilityVersion());
+    }
+
+    public static class HandshakeRequest extends TransportRequest {
+
+        public static final HandshakeRequest INSTANCE = new HandshakeRequest();
+
+        private HandshakeRequest() {
+        }
+
+    }
+
+    public static class HandshakeResponse extends TransportResponse {
+        private DiscoveryNode discoveryNode;
+        private ClusterName clusterName;
+        private Version version;
+
+        public HandshakeResponse() {
+        }
+
+        public HandshakeResponse(DiscoveryNode discoveryNode, ClusterName clusterName, Version version) {
+            this.discoveryNode = discoveryNode;
+            this.version = version;
+            this.clusterName = clusterName;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            discoveryNode = in.readOptionalWriteable(DiscoveryNode::new);
+            clusterName = ClusterName.readClusterName(in);
+            version = Version.readVersion(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeOptionalWriteable(discoveryNode);
+            clusterName.writeTo(out);
+            Version.writeVersion(version, out);
+        }
     }
 
     public void disconnectFromNode(DiscoveryNode node) {
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
index 0490dfd96aa..9c0e2bfcafd 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
@@ -184,9 +184,11 @@ public abstract class TaskManagerTestCase extends ESTestCase {
 
     public static class TestNode implements Releasable {
         public TestNode(String name, ThreadPool threadPool, Settings settings) {
+            clusterService = createClusterService(threadPool);
+            ClusterName clusterName = clusterService.state().getClusterName();
             transportService = new TransportService(settings,
                     new LocalTransport(settings, threadPool, Version.CURRENT, new NamedWriteableRegistry(),
-                        new NoneCircuitBreakerService()), threadPool) {
+                        new NoneCircuitBreakerService()), threadPool, clusterName) {
                 @Override
                 protected TaskManager createTaskManager() {
                     if (MockTaskManager.USE_MOCK_TASK_MANAGER_SETTING.get(settings)) {
@@ -197,7 +199,6 @@ public abstract class TaskManagerTestCase extends ESTestCase {
                 }
             };
             transportService.start();
-            clusterService = createClusterService(threadPool);
             clusterService.add(transportService.getTaskManager());
             discoveryNode = new DiscoveryNode(name, transportService.boundAddress().publishAddress(),
                     emptyMap(), emptySet(), Version.CURRENT);
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
index 39202fcc43a..db1d09b3aa7 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
@@ -86,7 +86,7 @@ public class TransportBulkActionTookTests extends ESTestCase {
 
     private TransportBulkAction createAction(boolean controlled, AtomicLong expected) {
         CapturingTransport capturingTransport = new CapturingTransport();
-        TransportService transportService = new TransportService(capturingTransport, threadPool);
+        TransportService transportService = new TransportService(capturingTransport, threadPool, clusterService.state().getClusterName());
         transportService.start();
         transportService.acceptIncomingRequests();
         IndexNameExpressionResolver resolver = new Resolver(Settings.EMPTY);
diff --git a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
index f76fdf4fd20..23b0df27480 100644
--- a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
@@ -190,7 +190,7 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase {
         super.setUp();
         transport = new CapturingTransport();
         clusterService = createClusterService(THREAD_POOL);
-        final TransportService transportService = new TransportService(transport, THREAD_POOL);
+        final TransportService transportService = new TransportService(transport, THREAD_POOL, clusterService.state().getClusterName());
         transportService.start();
         transportService.acceptIncomingRequests();
         setClusterState(clusterService, TEST_INDEX);
diff --git a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
index a35accc5fc5..ccdb13f710a 100644
--- a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
@@ -85,7 +85,7 @@ public class TransportMasterNodeActionTests extends ESTestCase {
         super.setUp();
         transport = new CapturingTransport();
         clusterService = createClusterService(threadPool);
-        transportService = new TransportService(transport, threadPool);
+        transportService = new TransportService(transport, threadPool, clusterService.state().getClusterName());
         transportService.start();
         transportService.acceptIncomingRequests();
         localNode = new DiscoveryNode("local_node", DummyTransportAddress.INSTANCE, Collections.emptyMap(),
diff --git a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
index ad2326c3148..6a7f7ac3398 100644
--- a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
@@ -119,7 +119,7 @@ public class TransportNodesActionTests extends ESTestCase {
         super.setUp();
         transport = new CapturingTransport();
         clusterService = createClusterService(THREAD_POOL);
-        final TransportService transportService = new TransportService(transport, THREAD_POOL);
+        final TransportService transportService = new TransportService(transport, THREAD_POOL, clusterService.state().getClusterName());
         transportService.start();
         transportService.acceptIncomingRequests();
         int numNodes = randomIntBetween(3, 10);
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
index 0bd7f9bf18a..5253097818e 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
@@ -91,7 +91,7 @@ public class BroadcastReplicationTests extends ESTestCase {
         super.setUp();
         LocalTransport transport = new LocalTransport(Settings.EMPTY, threadPool, Version.CURRENT, new NamedWriteableRegistry(), circuitBreakerService);
         clusterService = createClusterService(threadPool);
-        transportService = new TransportService(transport, threadPool);
+        transportService = new TransportService(transport, threadPool, clusterService.state().getClusterName());
         transportService.start();
         transportService.acceptIncomingRequests();
         broadcastReplicationAction = new TestBroadcastReplicationAction(Settings.EMPTY, threadPool, clusterService, transportService, new ActionFilters(new HashSet<ActionFilter>()), new IndexNameExpressionResolver(Settings.EMPTY), null);
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
index a10ce35ca41..e9a92ff25b9 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
@@ -120,7 +120,7 @@ public class TransportReplicationActionTests extends ESTestCase {
         super.setUp();
         transport = new CapturingTransport();
         clusterService = createClusterService(threadPool);
-        transportService = new TransportService(transport, threadPool);
+        transportService = new TransportService(transport, threadPool, clusterService.state().getClusterName());
         transportService.start();
         transportService.acceptIncomingRequests();
         action = new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool);
diff --git a/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
index 2dd31548cb9..86165461c84 100644
--- a/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
@@ -141,7 +141,7 @@ public class TransportInstanceSingleOperationActionTests extends ESTestCase {
         super.setUp();
         transport = new CapturingTransport();
         clusterService = createClusterService(THREAD_POOL);
-        transportService = new TransportService(transport, THREAD_POOL);
+        transportService = new TransportService(transport, THREAD_POOL, clusterService.state().getClusterName());
         transportService.start();
         transportService.acceptIncomingRequests();
         action = new TestTransportInstanceSingleOperationAction(
diff --git a/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java b/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java
index 22b04a19a26..0aaec63fef0 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java
@@ -20,7 +20,9 @@
 package org.elasticsearch.client.transport;
 
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.node.liveness.LivenessResponse;
+import org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.component.Lifecycle;
@@ -34,6 +36,7 @@ import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportResponse;
 import org.elasticsearch.transport.TransportResponseHandler;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.TransportServiceAdapter;
 
 import java.io.IOException;
@@ -47,6 +50,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 abstract class FailAndRetryMockTransport<Response extends TransportResponse> implements Transport {
 
     private final Random random;
+    private final ClusterName clusterName;
 
     private boolean connectMode = true;
 
@@ -57,8 +61,9 @@ abstract class FailAndRetryMockTransport<Response extends TransportResponse> imp
     private final AtomicInteger successes = new AtomicInteger();
     private final Set<DiscoveryNode> triedNodes = new CopyOnWriteArraySet<>();
 
-    FailAndRetryMockTransport(Random random) {
+    FailAndRetryMockTransport(Random random, ClusterName clusterName) {
         this.random = new Random(random.nextLong());
+        this.clusterName = clusterName;
     }
 
     @Override
@@ -69,7 +74,11 @@ abstract class FailAndRetryMockTransport<Response extends TransportResponse> imp
         //we make sure that nodes get added to the connected ones when calling addTransportAddress, by returning proper nodes info
         if (connectMode) {
             TransportResponseHandler transportResponseHandler = transportServiceAdapter.onResponseReceived(requestId);
-            transportResponseHandler.handleResponse(new LivenessResponse(ClusterName.DEFAULT, node));
+            if (action.equals(TransportLivenessAction.NAME)) {
+                transportResponseHandler.handleResponse(new LivenessResponse(clusterName, node));
+            } else {
+                transportResponseHandler.handleResponse(new TransportService.HandshakeResponse(node, clusterName, Version.CURRENT));
+            }
             return;
         }
 
diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java
index a7e472da3b4..ec0a9e5cc08 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java
@@ -123,8 +123,8 @@ public class TransportClientHeadersTests extends AbstractClientHeadersTestCase {
         CountDownLatch clusterStateLatch = new CountDownLatch(1);
 
         @Inject
-        public InternalTransportService(Settings settings, Transport transport, ThreadPool threadPool) {
-            super(settings, transport, threadPool);
+        public InternalTransportService(Settings settings, Transport transport, ThreadPool threadPool, ClusterName clusterName) {
+            super(settings, transport, threadPool, clusterName);
         }
 
         @Override @SuppressWarnings("unchecked")
diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
index 0246ec227dd..b566d764231 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
@@ -62,8 +62,9 @@ public class TransportClientNodesServiceTests extends ESTestCase {
         private final int nodesCount;
 
         TestIteration() {
+            ClusterName clusterName = new ClusterName("test");
             threadPool = new ThreadPool("transport-client-nodes-service-tests");
-            transport = new FailAndRetryMockTransport<TestResponse>(random()) {
+            transport = new FailAndRetryMockTransport<TestResponse>(random(), clusterName) {
                 @Override
                 public List<String> getLocalAddresses() {
                     return Collections.emptyList();
@@ -74,12 +75,12 @@ public class TransportClientNodesServiceTests extends ESTestCase {
                     return  new TestResponse();
                 }
             };
-            transportService = new TransportService(Settings.EMPTY, transport, threadPool) {
+            transportService = new TransportService(Settings.EMPTY, transport, threadPool, clusterName) {
                 @Override
                 public <T extends TransportResponse> void sendRequest(DiscoveryNode node, String action,
                                                                       TransportRequest request, final TransportResponseHandler<T> handler) {
                     if (TransportLivenessAction.NAME.equals(action)) {
-                        super.sendRequest(node, action, request, wrapLivenessResponseHandler(handler, node));
+                        super.sendRequest(node, action, request, wrapLivenessResponseHandler(handler, node, clusterName));
                     } else {
                         super.sendRequest(node, action, request, handler);
                     }
@@ -90,7 +91,7 @@ public class TransportClientNodesServiceTests extends ESTestCase {
                                                                       TransportRequestOptions options,
                                                                       TransportResponseHandler<T> handler) {
                     if (TransportLivenessAction.NAME.equals(action)) {
-                        super.sendRequest(node, action, request, options, wrapLivenessResponseHandler(handler, node));
+                        super.sendRequest(node, action, request, options, wrapLivenessResponseHandler(handler, node, clusterName));
                     } else {
                         super.sendRequest(node, action, request, options, handler);
                     }
@@ -98,8 +99,8 @@ public class TransportClientNodesServiceTests extends ESTestCase {
             };
             transportService.start();
             transportService.acceptIncomingRequests();
-            transportClientNodesService = new TransportClientNodesService(Settings.EMPTY, ClusterName.DEFAULT, transportService, threadPool,
-                    Version.CURRENT);
+            transportClientNodesService =
+                    new TransportClientNodesService(Settings.EMPTY, clusterName, transportService, threadPool, Version.CURRENT);
             this.nodesCount = randomIntBetween(1, 10);
             for (int i = 0; i < nodesCount; i++) {
                 transportClientNodesService.addTransportAddresses(new LocalTransportAddress("node" + i));
@@ -108,7 +109,8 @@ public class TransportClientNodesServiceTests extends ESTestCase {
         }
 
         private <T extends TransportResponse> TransportResponseHandler wrapLivenessResponseHandler(TransportResponseHandler<T> handler,
-                                                                                                   DiscoveryNode node) {
+                                                                                                   DiscoveryNode node,
+                                                                                                   ClusterName clusterName) {
             return new TransportResponseHandler<T>() {
                 @Override
                 public T newInstance() {
@@ -118,7 +120,7 @@ public class TransportClientNodesServiceTests extends ESTestCase {
                 @Override
                 @SuppressWarnings("unchecked")
                 public void handleResponse(T response) {
-                    LivenessResponse livenessResponse = new LivenessResponse(ClusterName.DEFAULT,
+                    LivenessResponse livenessResponse = new LivenessResponse(clusterName,
                             new DiscoveryNode(node.getName(), node.getId(), "liveness-hostname" + node.getId(),
                                     "liveness-hostaddress" + node.getId(),
                                     new LocalTransportAddress("liveness-address-" + node.getId()), node.getAttributes(), node.getRoles(),
diff --git a/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
index c263bcbcf37..a3bb73977e7 100644
--- a/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
@@ -149,7 +149,7 @@ public class NodeConnectionsServiceTests extends ESTestCase {
     public void setUp() throws Exception {
         super.setUp();
         this.transport = new MockTransport();
-        transportService = new TransportService(transport, THREAD_POOL);
+        transportService = new TransportService(transport, THREAD_POOL, ClusterName.DEFAULT);
         transportService.start();
         transportService.acceptIncomingRequests();
     }
diff --git a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
index c79d198d350..6085bf92c32 100644
--- a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
@@ -106,7 +106,7 @@ public class ShardStateActionTests extends ESTestCase {
         super.setUp();
         this.transport = new CapturingTransport();
         clusterService = createClusterService(THREAD_POOL);
-        transportService = new TransportService(transport, THREAD_POOL);
+        transportService = new TransportService(transport, THREAD_POOL, clusterService.state().getClusterName());
         transportService.start();
         transportService.acceptIncomingRequests();
         shardStateAction = new TestShardStateAction(Settings.EMPTY, clusterService, transportService, null, null);
diff --git a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
index bf465d8d44e..298175c41ee 100644
--- a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
@@ -80,7 +80,7 @@ public class ClusterStateHealthTests extends ESTestCase {
     public void setUp() throws Exception {
         super.setUp();
         clusterService = createClusterService(threadPool);
-        transportService = new TransportService(new CapturingTransport(), threadPool);
+        transportService = new TransportService(new CapturingTransport(), threadPool, clusterService.state().getClusterName());
         transportService.start();
         transportService.acceptIncomingRequests();
     }
diff --git a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
index cbc517ceb64..d5fc4630bcf 100644
--- a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
@@ -52,7 +52,7 @@ public class NetworkModuleTests extends ModuleTestCase {
 
     static class FakeTransportService extends TransportService {
         public FakeTransportService() {
-            super(null, null);
+            super(null, null, null);
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java b/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
index ab834ca6ae2..e5199048c35 100644
--- a/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
@@ -115,8 +115,12 @@ public class ZenFaultDetectionTests extends ESTestCase {
 
     protected MockTransportService build(Settings settings, Version version) {
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
-        MockTransportService transportService = new MockTransportService(Settings.EMPTY,
-                new LocalTransport(settings, threadPool, version, namedWriteableRegistry, circuitBreakerService), threadPool);
+        MockTransportService transportService =
+                new MockTransportService(
+                        Settings.EMPTY,
+                        new LocalTransport(settings, threadPool, version, namedWriteableRegistry, circuitBreakerService),
+                        threadPool,
+                        ClusterName.DEFAULT);
         transportService.start();
         transportService.acceptIncomingRequests();
         return transportService;
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java b/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java
index 00e3daf1fc8..7847d7027d0 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java
@@ -28,6 +28,7 @@ import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.discovery.zen.elect.ElectMasterService;
@@ -35,16 +36,22 @@ import org.elasticsearch.discovery.zen.ping.PingContextProvider;
 import org.elasticsearch.discovery.zen.ping.ZenPing;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportConnectionListener;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.TransportSettings;
 import org.elasticsearch.transport.netty.NettyTransport;
+import org.jboss.netty.util.internal.ConcurrentHashMap;
 
 import java.net.InetSocketAddress;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 
 public class UnicastZenPingIT extends ESTestCase {
     public void testSimplePings() throws InterruptedException {
@@ -54,36 +61,31 @@ public class UnicastZenPingIT extends ESTestCase {
         settings = Settings.builder().put(settings).put(TransportSettings.PORT.getKey(), startPort + "-" + endPort).build();
 
         ThreadPool threadPool = new ThreadPool(getClass().getName());
-        ClusterName clusterName = new ClusterName("test");
+        ClusterName test = new ClusterName("test");
+        ClusterName mismatch = new ClusterName("mismatch");
         NetworkService networkService = new NetworkService(settings);
         ElectMasterService electMasterService = new ElectMasterService(settings, Version.CURRENT);
 
-        NettyTransport transportA = new NettyTransport(settings, threadPool, networkService, BigArrays.NON_RECYCLING_INSTANCE, Version.CURRENT, new NamedWriteableRegistry(), new NoneCircuitBreakerService());
-        final TransportService transportServiceA = new TransportService(transportA, threadPool).start();
-        transportServiceA.acceptIncomingRequests();
-        final DiscoveryNode nodeA = new DiscoveryNode("UZP_A", transportServiceA.boundAddress().publishAddress(),
-                emptyMap(), emptySet(), Version.CURRENT);
-
-        InetSocketTransportAddress addressA = (InetSocketTransportAddress) transportA.boundAddress().publishAddress();
-
-        NettyTransport transportB = new NettyTransport(settings, threadPool, networkService, BigArrays.NON_RECYCLING_INSTANCE, Version.CURRENT, new NamedWriteableRegistry(), new NoneCircuitBreakerService());
-        final TransportService transportServiceB = new TransportService(transportB, threadPool).start();
-        transportServiceB.acceptIncomingRequests();
-        final DiscoveryNode nodeB = new DiscoveryNode("UZP_B", transportServiceA.boundAddress().publishAddress(),
-                emptyMap(), emptySet(), Version.CURRENT);
-
-        InetSocketTransportAddress addressB = (InetSocketTransportAddress) transportB.boundAddress().publishAddress();
+        NetworkHandle handleA = startServices(settings, threadPool, networkService, "UZP_A", test, Version.CURRENT);
+        NetworkHandle handleB = startServices(settings, threadPool, networkService, "UZP_B", test, Version.CURRENT);
+        NetworkHandle handleC = startServices(settings, threadPool, networkService, "UZP_C", new ClusterName("mismatch"), Version.CURRENT);
+        // just fake that no versions are compatible with this node
+        Version previousVersion = VersionUtils.getPreviousVersion(Version.CURRENT.minimumCompatibilityVersion());
+        Version versionD = VersionUtils.randomVersionBetween(random(), previousVersion.minimumCompatibilityVersion(), previousVersion);
+        NetworkHandle handleD = startServices(settings, threadPool, networkService, "UZP_D", test, versionD);
 
         Settings hostsSettings = Settings.builder().putArray("discovery.zen.ping.unicast.hosts",
-                NetworkAddress.format(new InetSocketAddress(addressA.address().getAddress(), addressA.address().getPort())),
-                NetworkAddress.format(new InetSocketAddress(addressB.address().getAddress(), addressB.address().getPort())))
+                NetworkAddress.format(new InetSocketAddress(handleA.address.address().getAddress(), handleA.address.address().getPort())),
+                NetworkAddress.format(new InetSocketAddress(handleB.address.address().getAddress(), handleB.address.address().getPort())),
+                NetworkAddress.format(new InetSocketAddress(handleC.address.address().getAddress(), handleC.address.address().getPort())),
+                NetworkAddress.format(new InetSocketAddress(handleD.address.address().getAddress(), handleD.address.address().getPort())))
                 .build();
 
-        UnicastZenPing zenPingA = new UnicastZenPing(hostsSettings, threadPool, transportServiceA, clusterName, Version.CURRENT, electMasterService, null);
+        UnicastZenPing zenPingA = new UnicastZenPing(hostsSettings, threadPool, handleA.transportService, test, Version.CURRENT, electMasterService, null);
         zenPingA.setPingContextProvider(new PingContextProvider() {
             @Override
             public DiscoveryNodes nodes() {
-                return DiscoveryNodes.builder().put(nodeA).localNodeId("UZP_A").build();
+                return DiscoveryNodes.builder().put(handleA.node).localNodeId("UZP_A").build();
             }
 
             @Override
@@ -93,11 +95,11 @@ public class UnicastZenPingIT extends ESTestCase {
         });
         zenPingA.start();
 
-        UnicastZenPing zenPingB = new UnicastZenPing(hostsSettings, threadPool, transportServiceB, clusterName, Version.CURRENT, electMasterService, null);
+        UnicastZenPing zenPingB = new UnicastZenPing(hostsSettings, threadPool, handleB.transportService, test, Version.CURRENT, electMasterService, null);
         zenPingB.setPingContextProvider(new PingContextProvider() {
             @Override
             public DiscoveryNodes nodes() {
-                return DiscoveryNodes.builder().put(nodeB).localNodeId("UZP_B").build();
+                return DiscoveryNodes.builder().put(handleB.node).localNodeId("UZP_B").build();
             }
 
             @Override
@@ -107,12 +109,41 @@ public class UnicastZenPingIT extends ESTestCase {
         });
         zenPingB.start();
 
+        UnicastZenPing zenPingC = new UnicastZenPing(hostsSettings, threadPool, handleC.transportService, mismatch, versionD, electMasterService, null);
+        zenPingC.setPingContextProvider(new PingContextProvider() {
+            @Override
+            public DiscoveryNodes nodes() {
+                return DiscoveryNodes.builder().put(handleC.node).localNodeId("UZP_C").build();
+            }
+
+            @Override
+            public boolean nodeHasJoinedClusterOnce() {
+                return false;
+            }
+        });
+        zenPingC.start();
+
+        UnicastZenPing zenPingD = new UnicastZenPing(hostsSettings, threadPool, handleD.transportService, mismatch, Version.CURRENT, electMasterService, null);
+        zenPingD.setPingContextProvider(new PingContextProvider() {
+            @Override
+            public DiscoveryNodes nodes() {
+                return DiscoveryNodes.builder().put(handleD.node).localNodeId("UZP_D").build();
+            }
+
+            @Override
+            public boolean nodeHasJoinedClusterOnce() {
+                return false;
+            }
+        });
+        zenPingD.start();
+
         try {
             logger.info("ping from UZP_A");
             ZenPing.PingResponse[] pingResponses = zenPingA.pingAndWait(TimeValue.timeValueSeconds(10));
             assertThat(pingResponses.length, equalTo(1));
             assertThat(pingResponses[0].node().getId(), equalTo("UZP_B"));
             assertTrue(pingResponses[0].hasJoinedOnce());
+            assertCounters(handleA, handleA, handleB, handleC, handleD);
 
             // ping again, this time from B,
             logger.info("ping from UZP_B");
@@ -120,13 +151,72 @@ public class UnicastZenPingIT extends ESTestCase {
             assertThat(pingResponses.length, equalTo(1));
             assertThat(pingResponses[0].node().getId(), equalTo("UZP_A"));
             assertFalse(pingResponses[0].hasJoinedOnce());
+            assertCounters(handleB, handleA, handleB, handleC, handleD);
 
+            logger.info("ping from UZP_C");
+            pingResponses = zenPingC.pingAndWait(TimeValue.timeValueSeconds(10));
+            assertThat(pingResponses.length, equalTo(0));
+            assertCounters(handleC, handleA, handleB, handleC, handleD);
+
+            logger.info("ping from UZP_D");
+            pingResponses = zenPingD.pingAndWait(TimeValue.timeValueSeconds(10));
+            assertThat(pingResponses.length, equalTo(0));
+            assertCounters(handleD, handleA, handleB, handleC, handleD);
         } finally {
             zenPingA.close();
             zenPingB.close();
-            transportServiceA.close();
-            transportServiceB.close();
+            zenPingC.close();
+            zenPingD.close();
+            handleA.transportService.close();
+            handleB.transportService.close();
+            handleC.transportService.close();
+            handleD.transportService.close();
             terminate(threadPool);
         }
     }
+
+    // assert that we tried to ping each of the configured nodes at least once
+    private void assertCounters(NetworkHandle that, NetworkHandle...handles) {
+        for (NetworkHandle handle : handles) {
+            if (handle != that) {
+                assertThat(that.counters.get(handle.address).get(), greaterThan(0));
+            }
+        }
+    }
+
+    private NetworkHandle startServices(Settings settings, ThreadPool threadPool, NetworkService networkService, String nodeId, ClusterName clusterName, Version version) {
+        NettyTransport transport = new NettyTransport(settings, threadPool, networkService, BigArrays.NON_RECYCLING_INSTANCE, version, new NamedWriteableRegistry(), new NoneCircuitBreakerService());
+        final TransportService transportService = new TransportService(transport, threadPool, clusterName);
+        transportService.start();
+        transportService.acceptIncomingRequests();
+        ConcurrentMap<TransportAddress, AtomicInteger> counters = new ConcurrentHashMap<>();
+        transportService.addConnectionListener(new TransportConnectionListener() {
+            @Override
+            public void onNodeConnected(DiscoveryNode node) {
+                counters.computeIfAbsent(node.getAddress(), k -> new AtomicInteger());
+                counters.get(node.getAddress()).incrementAndGet();
+            }
+
+            @Override
+            public void onNodeDisconnected(DiscoveryNode node) {
+            }
+        });
+        final DiscoveryNode node = new DiscoveryNode(nodeId, transportService.boundAddress().publishAddress(), emptyMap(), emptySet(), version);
+        transportService.setLocalNode(node);
+        return new NetworkHandle((InetSocketTransportAddress)transport.boundAddress().publishAddress(), transportService, node, counters);
+    }
+
+    private static class NetworkHandle {
+        public final InetSocketTransportAddress address;
+        public final TransportService transportService;
+        public final DiscoveryNode node;
+        public final ConcurrentMap<TransportAddress, AtomicInteger> counters;
+
+        public NetworkHandle(InetSocketTransportAddress address, TransportService transportService, DiscoveryNode discoveryNode, ConcurrentMap<TransportAddress, AtomicInteger> counters) {
+            this.address = address;
+            this.transportService = transportService;
+            this.node = discoveryNode;
+            this.counters = counters;
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java
index 335d1b3e8fd..e6b160eabf8 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java
@@ -83,6 +83,9 @@ import static org.hamcrest.Matchers.nullValue;
 
 @TestLogging("discovery.zen.publish:TRACE")
 public class PublishClusterStateActionTests extends ESTestCase {
+
+    private static final ClusterName CLUSTER_NAME = ClusterName.DEFAULT;
+
     protected ThreadPool threadPool;
     protected Map<String, MockNode> nodes = new HashMap<>();
 
@@ -101,7 +104,7 @@ public class PublishClusterStateActionTests extends ESTestCase {
             this.service = service;
             this.listener = listener;
             this.logger = logger;
-            this.clusterState = ClusterState.builder(ClusterName.DEFAULT).nodes(DiscoveryNodes.builder().put(discoveryNode).localNodeId(discoveryNode.getId()).build()).build();
+            this.clusterState = ClusterState.builder(CLUSTER_NAME).nodes(DiscoveryNodes.builder().put(discoveryNode).localNodeId(discoveryNode.getId()).build()).build();
         }
 
         public MockNode setAsMaster() {
@@ -229,7 +232,7 @@ public class PublishClusterStateActionTests extends ESTestCase {
     }
 
     protected MockTransportService buildTransportService(Settings settings, Version version) {
-        MockTransportService transportService = MockTransportService.local(Settings.EMPTY, version, threadPool);
+        MockTransportService transportService = MockTransportService.local(Settings.EMPTY, version, threadPool, CLUSTER_NAME);
         transportService.start();
         transportService.acceptIncomingRequests();
         return transportService;
@@ -249,7 +252,7 @@ public class PublishClusterStateActionTests extends ESTestCase {
                 clusterStateSupplier,
                 listener,
                 discoverySettings,
-                ClusterName.DEFAULT);
+                CLUSTER_NAME);
     }
 
     public void testSimpleClusterStatePublishing() throws Exception {
@@ -343,7 +346,7 @@ public class PublishClusterStateActionTests extends ESTestCase {
 
         // Initial cluster state with both states - the second node still shouldn't get diff even though it's present in the previous cluster state
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder(nodeA.nodes()).put(nodeB.discoveryNode).build();
-        ClusterState previousClusterState = ClusterState.builder(ClusterName.DEFAULT).nodes(discoveryNodes).build();
+        ClusterState previousClusterState = ClusterState.builder(CLUSTER_NAME).nodes(discoveryNodes).build();
         ClusterState clusterState = ClusterState.builder(previousClusterState).incrementVersion().build();
         publishStateAndWait(nodeA.action, clusterState, previousClusterState);
         assertSameStateFromFull(nodeB.clusterState, clusterState);
@@ -374,7 +377,7 @@ public class PublishClusterStateActionTests extends ESTestCase {
 
         // Initial cluster state
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().put(nodeA.discoveryNode).localNodeId(nodeA.discoveryNode.getId()).masterNodeId(nodeA.discoveryNode.getId()).build();
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).nodes(discoveryNodes).build();
+        ClusterState clusterState = ClusterState.builder(CLUSTER_NAME).nodes(discoveryNodes).build();
 
         // cluster state update - add nodeB
         discoveryNodes = DiscoveryNodes.builder(discoveryNodes).put(nodeB.discoveryNode).build();
@@ -417,7 +420,7 @@ public class PublishClusterStateActionTests extends ESTestCase {
         AssertingAckListener[] listeners = new AssertingAckListener[numberOfIterations];
         DiscoveryNodes discoveryNodes = discoveryNodesBuilder.build();
         MetaData metaData = MetaData.EMPTY_META_DATA;
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).build();
+        ClusterState clusterState = ClusterState.builder(CLUSTER_NAME).metaData(metaData).build();
         ClusterState previousState;
         for (int i = 0; i < numberOfIterations; i++) {
             previousState = clusterState;
@@ -451,7 +454,7 @@ public class PublishClusterStateActionTests extends ESTestCase {
 
         // Initial cluster state with both states - the second node still shouldn't get diff even though it's present in the previous cluster state
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder(nodeA.nodes()).put(nodeB.discoveryNode).build();
-        ClusterState previousClusterState = ClusterState.builder(ClusterName.DEFAULT).nodes(discoveryNodes).build();
+        ClusterState previousClusterState = ClusterState.builder(CLUSTER_NAME).nodes(discoveryNodes).build();
         ClusterState clusterState = ClusterState.builder(previousClusterState).incrementVersion().build();
         publishStateAndWait(nodeA.action, clusterState, previousClusterState);
         assertSameStateFromFull(nodeB.clusterState, clusterState);
@@ -503,7 +506,7 @@ public class PublishClusterStateActionTests extends ESTestCase {
         discoveryNodesBuilder.localNodeId(master.discoveryNode.getId()).masterNodeId(master.discoveryNode.getId());
         DiscoveryNodes discoveryNodes = discoveryNodesBuilder.build();
         MetaData metaData = MetaData.EMPTY_META_DATA;
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).nodes(discoveryNodes).build();
+        ClusterState clusterState = ClusterState.builder(CLUSTER_NAME).metaData(metaData).nodes(discoveryNodes).build();
         ClusterState previousState = master.clusterState;
         try {
             publishState(master.action, clusterState, previousState, masterNodes + randomIntBetween(1, 5));
@@ -580,7 +583,7 @@ public class PublishClusterStateActionTests extends ESTestCase {
         discoveryNodesBuilder.localNodeId(master.discoveryNode.getId()).masterNodeId(master.discoveryNode.getId());
         DiscoveryNodes discoveryNodes = discoveryNodesBuilder.build();
         MetaData metaData = MetaData.EMPTY_META_DATA;
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).nodes(discoveryNodes).build();
+        ClusterState clusterState = ClusterState.builder(CLUSTER_NAME).metaData(metaData).nodes(discoveryNodes).build();
         ClusterState previousState = master.clusterState;
         try {
             publishState(master.action, clusterState, previousState, minMasterNodes);
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
index a7ceec92a61..7f9c4a3bbfe 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
@@ -74,9 +74,9 @@ public class DynamicMappingDisabledTests extends ESSingleNodeTestCase {
                 .put(MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey(), false)
                 .build();
         clusterService = createClusterService(THREAD_POOL);
-        transport = new LocalTransport(settings, THREAD_POOL, Version.CURRENT, new NamedWriteableRegistry(),
-            new NoneCircuitBreakerService());
-        transportService = new TransportService(transport, THREAD_POOL);
+        transport =
+                new LocalTransport(settings, THREAD_POOL, Version.CURRENT, new NamedWriteableRegistry(), new NoneCircuitBreakerService());
+        transportService = new TransportService(transport, THREAD_POOL, clusterService.state().getClusterName());
         indicesService = getInstanceFromNode(IndicesService.class);
         shardStateAction = new ShardStateAction(settings, clusterService, transportService, null, null, THREAD_POOL);
         actionFilters = new ActionFilters(Collections.emptySet());
diff --git a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
index 721c66c04e1..cf28de64b87 100644
--- a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
@@ -87,7 +87,7 @@ public class IndicesStoreTests extends ESTestCase {
     public void before() {
         localNode = new DiscoveryNode("abc", new LocalTransportAddress("abc"), emptyMap(), emptySet(), Version.CURRENT);
         clusterService = createClusterService(threadPool);
-        indicesStore = new IndicesStore(Settings.EMPTY, null, clusterService, new TransportService(null, null), null);
+        indicesStore = new IndicesStore(Settings.EMPTY, null, clusterService, new TransportService(null, null, clusterService.state().getClusterName()), null);
     }
 
     @After
diff --git a/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
index edeecd61d8e..45966606d31 100644
--- a/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
+++ b/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.transport;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -65,7 +66,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
     protected DiscoveryNode nodeB;
     protected MockTransportService serviceB;
 
-    protected abstract MockTransportService build(Settings settings, Version version, NamedWriteableRegistry namedWriteableRegistry);
+    protected abstract MockTransportService build(Settings settings, Version version, ClusterName clusterName);
 
     @Override
     @Before
@@ -78,8 +79,8 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
                     .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "")
                     .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
                     .build(),
-                version0, new NamedWriteableRegistry()
-        );
+                version0,
+                ClusterName.DEFAULT);
         serviceA.acceptIncomingRequests();
         nodeA = new DiscoveryNode("TS_A", serviceA.boundAddress().publishAddress(), emptyMap(), emptySet(), version0);
         serviceB = build(
@@ -88,8 +89,8 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
                     .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "")
                     .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
                     .build(),
-                version1, new NamedWriteableRegistry()
-        );
+                version1,
+                ClusterName.DEFAULT);
         serviceB.acceptIncomingRequests();
         nodeB = new DiscoveryNode("TS_B", serviceB.boundAddress().publishAddress(), emptyMap(), emptySet(), version1);
 
@@ -1178,7 +1179,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         }
 
         try {
-            serviceB.connectToNodeLight(nodeA);
+            serviceB.connectToNodeLight(nodeA, 100);
             fail("exception should be thrown");
         } catch (ConnectTransportException e) {
             // all is well
@@ -1238,7 +1239,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         }
 
         try {
-            serviceB.connectToNodeLight(nodeA);
+            serviceB.connectToNodeLight(nodeA, 100);
             fail("exception should be thrown");
         } catch (ConnectTransportException e) {
             // all is well
@@ -1298,8 +1299,8 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
                     .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "")
                     .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
                     .build(),
-                version0, new NamedWriteableRegistry()
-        );
+                version0,
+                ClusterName.DEFAULT);
         AtomicBoolean requestProcessed = new AtomicBoolean();
         service.registerRequestHandler("action", TestRequest::new, ThreadPool.Names.SAME,
                 (request, channel) -> {
diff --git a/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java b/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java
index 46c3cdbe3aa..d2d88b87593 100644
--- a/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.transport;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cache.recycler.MockPageCacheRecycler;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -70,7 +71,7 @@ public class NettySizeHeaderFrameDecoderTests extends ESTestCase {
         nettyTransport = new NettyTransport(settings, threadPool, networkService, bigArrays, Version.CURRENT, new NamedWriteableRegistry(),
             new NoneCircuitBreakerService());
         nettyTransport.start();
-        TransportService transportService = new TransportService(nettyTransport, threadPool);
+        TransportService transportService = new TransportService(nettyTransport, threadPool, ClusterName.DEFAULT);
         nettyTransport.transportServiceAdapter(transportService.createAdapter());
 
         TransportAddress[] boundAddresses = nettyTransport.boundAddress().boundAddresses();
@@ -84,6 +85,7 @@ public class NettySizeHeaderFrameDecoderTests extends ESTestCase {
     public void terminateThreadPool() throws InterruptedException {
         nettyTransport.stop();
         terminate(threadPool);
+        threadPool = null;
     }
 
     public void testThatTextMessageIsReturnedOnHTTPLikeRequest() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java b/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java
new file mode 100644
index 00000000000..b376a55af73
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.transport;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.network.NetworkService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.netty.NettyTransport;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.emptySet;
+import static org.hamcrest.Matchers.containsString;
+
+public class NettyTransportServiceHandshakeTests extends ESTestCase {
+    private static ThreadPool threadPool;
+
+    @BeforeClass
+    public static void startThreadPool() {
+        threadPool = new ThreadPool(NettyTransportServiceHandshakeTests.class.getSimpleName());
+    }
+
+    private List<TransportService> transportServices = new ArrayList<>();
+
+    private NetworkHandle startServices(String nodeNameAndId, Settings settings, Version version, ClusterName clusterName) {
+        NettyTransport transport =
+                new NettyTransport(
+                        settings,
+                        threadPool,
+                        new NetworkService(settings),
+                        BigArrays.NON_RECYCLING_INSTANCE,
+                        Version.CURRENT,
+                        new NamedWriteableRegistry(),
+                        new NoneCircuitBreakerService());
+        TransportService transportService = new MockTransportService(settings, transport, threadPool, clusterName);
+        transportService.start();
+        transportService.acceptIncomingRequests();
+        DiscoveryNode node =
+                new DiscoveryNode(
+                        nodeNameAndId,
+                        nodeNameAndId,
+                        transportService.boundAddress().publishAddress(),
+                        emptyMap(),
+                        emptySet(),
+                        version);
+        transportService.setLocalNode(node);
+        transportServices.add(transportService);
+        return new NetworkHandle(transportService, node);
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        for (TransportService transportService : transportServices) {
+            transportService.close();
+        }
+        super.tearDown();
+    }
+
+    @AfterClass
+    public static void terminateThreadPool() {
+        ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS);
+        // since static must set to null to be eligible for collection
+        threadPool = null;
+    }
+
+    public void testConnectToNodeLight() {
+        Settings settings = Settings.EMPTY;
+
+        ClusterName test = new ClusterName("test");
+
+        NetworkHandle handleA = startServices("TS_A", settings, Version.CURRENT, test);
+        NetworkHandle handleB =
+                startServices(
+                        "TS_B",
+                        settings,
+                        VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.CURRENT),
+                        test);
+
+        DiscoveryNode connectedNode =
+                handleA.transportService.connectToNodeLight(
+                        new DiscoveryNode(
+                                "",
+                                handleB.discoveryNode.getAddress(),
+                                emptyMap(),
+                                emptySet(),
+                                Version.CURRENT.minimumCompatibilityVersion()),
+                        100);
+        assertNotNull(connectedNode);
+
+        // the name and version should be updated
+        assertEquals(connectedNode.getName(), "TS_B");
+        assertEquals(connectedNode.getVersion(), handleB.discoveryNode.getVersion());
+    }
+
+    public void testMismatchedClusterName() {
+        Settings settings = Settings.EMPTY;
+
+        NetworkHandle handleA = startServices("TS_A", settings, Version.CURRENT, new ClusterName("a"));
+        NetworkHandle handleB = startServices("TS_B", settings, Version.CURRENT, new ClusterName("b"));
+
+        try {
+            handleA.transportService.connectToNodeLight(
+                    new DiscoveryNode(
+                            "",
+                            handleB.discoveryNode.getAddress(),
+                            emptyMap(),
+                            emptySet(),
+                            Version.CURRENT.minimumCompatibilityVersion()),
+                    100);
+            fail("expected handshake to fail from mismatched cluster names");
+        } catch (ConnectTransportException e) {
+            assertThat(e.getMessage(), containsString("handshake failed, mismatched cluster name [Cluster [b]]"));
+        }
+    }
+
+    public void testIncompatibleVersions() {
+        Settings settings = Settings.EMPTY;
+
+        ClusterName test = new ClusterName("test");
+        NetworkHandle handleA = startServices("TS_A", settings, Version.CURRENT, test);
+        NetworkHandle handleB =
+                startServices("TS_B", settings, VersionUtils.getPreviousVersion(Version.CURRENT.minimumCompatibilityVersion()), test);
+
+        try {
+            handleA.transportService.connectToNodeLight(
+                    new DiscoveryNode(
+                            "",
+                            handleB.discoveryNode.getAddress(),
+                            emptyMap(),
+                            emptySet(),
+                            Version.CURRENT.minimumCompatibilityVersion()),
+                    100);
+            fail("expected handshake to fail from incompatible versions");
+        } catch (ConnectTransportException e) {
+            assertThat(e.getMessage(), containsString("handshake failed, incompatible version"));
+        }
+    }
+
+    public void testIgnoreMismatchedClusterName() {
+        Settings settings = Settings.EMPTY;
+
+        NetworkHandle handleA = startServices("TS_A", settings, Version.CURRENT, new ClusterName("a"));
+        NetworkHandle handleB =
+                startServices(
+                        "TS_B",
+                        settings,
+                        VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.CURRENT),
+                        new ClusterName("b")
+                );
+
+        DiscoveryNode connectedNode = handleA.transportService.connectToNodeLight(
+                new DiscoveryNode(
+                        "",
+                        handleB.discoveryNode.getAddress(),
+                        emptyMap(),
+                        emptySet(),
+                        Version.CURRENT.minimumCompatibilityVersion()),
+                100,
+                false);
+        assertNotNull(connectedNode);
+        assertEquals(connectedNode.getName(), "TS_B");
+        assertEquals(connectedNode.getVersion(), handleB.discoveryNode.getVersion());
+    }
+
+    private static class NetworkHandle {
+        private TransportService transportService;
+        private DiscoveryNode discoveryNode;
+
+        public NetworkHandle(TransportService transportService, DiscoveryNode discoveryNode) {
+            this.transportService = transportService;
+            this.discoveryNode = discoveryNode;
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/transport/TransportModuleTests.java b/core/src/test/java/org/elasticsearch/transport/TransportModuleTests.java
index 888a73c9386..4eb14014cfc 100644
--- a/core/src/test/java/org/elasticsearch/transport/TransportModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/TransportModuleTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.transport;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.inject.ModuleTestCase;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -43,8 +44,8 @@ public class TransportModuleTests extends ModuleTestCase {
 
     static class FakeTransportService extends TransportService {
         @Inject
-        public FakeTransportService(Settings settings, Transport transport, ThreadPool threadPool) {
-            super(settings, transport, threadPool);
+        public FakeTransportService(Settings settings, Transport transport, ThreadPool threadPool, ClusterName clusterName) {
+            super(settings, transport, threadPool, clusterName);
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/transport/local/SimpleLocalTransportTests.java b/core/src/test/java/org/elasticsearch/transport/local/SimpleLocalTransportTests.java
index f071d56e3b9..2907a0a6ca8 100644
--- a/core/src/test/java/org/elasticsearch/transport/local/SimpleLocalTransportTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/local/SimpleLocalTransportTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.transport.local;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.transport.MockTransportService;
@@ -28,8 +29,8 @@ import org.elasticsearch.transport.AbstractSimpleTransportTestCase;
 public class SimpleLocalTransportTests extends AbstractSimpleTransportTestCase {
 
     @Override
-    protected MockTransportService build(Settings settings, Version version, NamedWriteableRegistry namedWriteableRegistry) {
-        MockTransportService transportService = MockTransportService.local(settings, version, threadPool);
+    protected MockTransportService build(Settings settings, Version version, ClusterName clusterName) {
+        MockTransportService transportService = MockTransportService.local(settings, version, threadPool, clusterName);
         transportService.start();
         return transportService;
     }
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java
index 49f86b909a6..8c00ae01b74 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.transport.netty;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.lease.Releasables;
@@ -63,14 +64,16 @@ public class NettyScheduledPingTests extends ESTestCase {
         NamedWriteableRegistry registryA = new NamedWriteableRegistry();
         final NettyTransport nettyA = new NettyTransport(settings, threadPool, new NetworkService(settings),
             BigArrays.NON_RECYCLING_INSTANCE, Version.CURRENT, registryA, circuitBreakerService);
-        MockTransportService serviceA = new MockTransportService(settings, nettyA, threadPool);
+        ClusterName test = new ClusterName("test");
+        MockTransportService serviceA = new MockTransportService(settings, nettyA, threadPool, test);
         serviceA.start();
         serviceA.acceptIncomingRequests();
 
         NamedWriteableRegistry registryB = new NamedWriteableRegistry();
         final NettyTransport nettyB = new NettyTransport(settings, threadPool, new NetworkService(settings),
             BigArrays.NON_RECYCLING_INSTANCE, Version.CURRENT, registryB, circuitBreakerService);
-        MockTransportService serviceB = new MockTransportService(settings, nettyB, threadPool);
+        MockTransportService serviceB = new MockTransportService(settings, nettyB, threadPool, test);
+
         serviceB.start();
         serviceB.acceptIncomingRequests();
 
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/SimpleNettyTransportTests.java b/core/src/test/java/org/elasticsearch/transport/netty/SimpleNettyTransportTests.java
index 7ba50b9c997..dac416128e5 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/SimpleNettyTransportTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/SimpleNettyTransportTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.transport.netty;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
@@ -39,9 +40,9 @@ import static org.hamcrest.Matchers.containsString;
 public class SimpleNettyTransportTests extends AbstractSimpleTransportTestCase {
 
     @Override
-    protected MockTransportService build(Settings settings, Version version, NamedWriteableRegistry namedWriteableRegistry) {
+    protected MockTransportService build(Settings settings, Version version, ClusterName clusterName) {
         settings = Settings.builder().put(settings).put(TransportSettings.PORT.getKey(), "0").build();
-        MockTransportService transportService = MockTransportService.nettyFromThreadPool(settings, version, threadPool);
+        MockTransportService transportService = MockTransportService.nettyFromThreadPool(settings, version, threadPool, clusterName);
         transportService.start();
         return transportService;
     }
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java
index b3d01e62fae..c2bc877b903 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java
@@ -77,6 +77,7 @@ import org.elasticsearch.action.termvectors.TermVectorsRequest;
 import org.elasticsearch.action.update.UpdateAction;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.network.NetworkModule;
@@ -725,8 +726,8 @@ public class IndicesRequestTests extends ESIntegTestCase {
         private final Map<String, List<TransportRequest>> requests = new HashMap<>();
 
         @Inject
-        public InterceptingTransportService(Settings settings, Transport transport, ThreadPool threadPool) {
-            super(settings, transport, threadPool);
+        public InterceptingTransportService(Settings settings, Transport transport, ThreadPool threadPool, ClusterName clusterName) {
+            super(settings, transport, threadPool, clusterName);
         }
 
         synchronized List<TransportRequest> consumeRequests(String action) {
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
index d86efaa2a85..c4863680613 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
@@ -23,6 +23,7 @@ import com.amazonaws.services.ec2.model.Tag;
 import org.elasticsearch.Version;
 import org.elasticsearch.cloud.aws.AwsEc2Service;
 import org.elasticsearch.cloud.aws.AwsEc2Service.DISCOVERY_EC2;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
@@ -65,7 +66,7 @@ public class Ec2DiscoveryTests extends ESTestCase {
 
     @Before
     public void createTransportService() {
-        transportService = MockTransportService.local(Settings.EMPTY, Version.CURRENT, threadPool);
+        transportService = MockTransportService.local(Settings.EMPTY, Version.CURRENT, threadPool, ClusterName.DEFAULT);
     }
 
     protected List<DiscoveryNode> buildDynamicNodes(Settings nodeSettings, int nodes) {
diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java
index 3b265d6a067..4525b1ece1d 100644
--- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java
+++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.discovery.gce;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cloud.gce.GceComputeService;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
@@ -93,7 +94,7 @@ public class GceDiscoveryTests extends ESTestCase {
 
     @Before
     public void createTransportService() {
-        transportService = MockTransportService.local(Settings.EMPTY, Version.CURRENT, threadPool);
+        transportService = MockTransportService.local(Settings.EMPTY, Version.CURRENT, threadPool, ClusterName.DEFAULT);
     }
 
     @Before
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
index 8e85134763d..05b8b23e90c 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
@@ -20,6 +20,9 @@
 package org.elasticsearch.test.transport;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.transport.TransportService;
+
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.component.LifecycleListener;
@@ -97,25 +100,29 @@ public class MockTransportService extends TransportService {
         }
     }
 
-    public static MockTransportService local(Settings settings, Version version, ThreadPool threadPool) {
+    public static MockTransportService local(Settings settings, Version version, ThreadPool threadPool, ClusterName clusterName) {
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
         Transport transport = new LocalTransport(settings, threadPool, version, namedWriteableRegistry, new NoneCircuitBreakerService());
-        return new MockTransportService(settings, transport, threadPool);
+        return new MockTransportService(settings, transport, threadPool, clusterName);
     }
 
-    public static MockTransportService nettyFromThreadPool(Settings settings, Version version, ThreadPool threadPool) {
+    public static MockTransportService nettyFromThreadPool(
+            Settings settings,
+            Version version,
+            ThreadPool threadPool,
+            ClusterName clusterName) {
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
         Transport transport = new NettyTransport(settings, threadPool, new NetworkService(settings), BigArrays.NON_RECYCLING_INSTANCE,
                 version, namedWriteableRegistry, new NoneCircuitBreakerService());
-        return new MockTransportService(Settings.EMPTY, transport, threadPool);
+        return new MockTransportService(Settings.EMPTY, transport, threadPool, clusterName);
     }
 
 
     private final Transport original;
 
     @Inject
-    public MockTransportService(Settings settings, Transport transport, ThreadPool threadPool) {
-        super(settings, new LookupTestTransport(transport), threadPool);
+    public MockTransportService(Settings settings, Transport transport, ThreadPool threadPool, ClusterName clusterName) {
+        super(settings, new LookupTestTransport(transport), threadPool, clusterName);
         this.original = transport;
     }
 

From 8fc51380de681f15a0467c0c484897f6c2044fa2 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Wed, 4 May 2016 17:29:23 -0700
Subject: [PATCH 0079/1311] Tests: improve logging for vagrant to emit entire
 output on failure

This change makes the vagrant tasks extend LoggedExec, so that the
entire vagrant output can be dumped on failure (and completely logged
when using --info). It should help for debugging issues like #18122.
---
 .../elasticsearch/gradle/LoggedExec.groovy    |   9 +-
 .../gradle/vagrant/BatsOverVagrantTask.groovy |  50 ++-----
 .../vagrant/TapLoggerOutputStream.groovy      |   4 +-
 .../gradle/vagrant/VagrantCommandTask.groovy  |  64 ++++----
 .../vagrant/VagrantLoggerOutputStream.groovy  | 138 +++++++++---------
 qa/vagrant/build.gradle                       |   4 +-
 6 files changed, 122 insertions(+), 147 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/LoggedExec.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/LoggedExec.groovy
index 1896cdf1b67..b1b04a2ded6 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/LoggedExec.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/LoggedExec.groovy
@@ -26,14 +26,17 @@ import org.gradle.api.tasks.Exec
  * A wrapper around gradle's Exec task to capture output and log on error.
  */
 class LoggedExec extends Exec {
+
+    protected ByteArrayOutputStream output = new ByteArrayOutputStream()
+
     LoggedExec() {
         if (logger.isInfoEnabled() == false) {
-            standardOutput = new ByteArrayOutputStream()
-            errorOutput = standardOutput
+            standardOutput = output
+            errorOutput = output
             ignoreExitValue = true
             doLast {
                 if (execResult.exitValue != 0) {
-                    standardOutput.toString('UTF-8').eachLine { line -> logger.error(line) }
+                    output.toString('UTF-8').eachLine { line -> logger.error(line) }
                     throw new GradleException("Process '${executable} ${args.join(' ')}' finished with non-zero exit value ${execResult.exitValue}")
                 }
             }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy
index 2f2030f6cd2..c68e0528c9b 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy
@@ -19,6 +19,7 @@
 package org.elasticsearch.gradle.vagrant
 
 import org.gradle.api.DefaultTask
+import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.TaskAction
 import org.gradle.logging.ProgressLoggerFactory
 import org.gradle.process.internal.ExecAction
@@ -30,41 +31,22 @@ import javax.inject.Inject
  * Runs bats over vagrant. Pretty much like running it using Exec but with a
  * nicer output formatter.
  */
-class BatsOverVagrantTask extends DefaultTask {
-  String command
-  String boxName
-  ExecAction execAction
+public class BatsOverVagrantTask extends VagrantCommandTask {
 
-  BatsOverVagrantTask() {
-    execAction = getExecActionFactory().newExecAction()
-  }
+    @Input
+    String command
 
-  @Inject
-  ProgressLoggerFactory getProgressLoggerFactory() {
-    throw new UnsupportedOperationException();
-  }
+    BatsOverVagrantTask() {
+        project.afterEvaluate {
+            args 'ssh', boxName, '--command', command
+        }
+    }
 
-  @Inject
-  ExecActionFactory getExecActionFactory() {
-    throw new UnsupportedOperationException();
-  }
-
-  void boxName(String boxName) {
-    this.boxName = boxName
-  }
-
-  void command(String command) {
-    this.command = command
-  }
-
-  @TaskAction
-  void exec() {
-    // It'd be nice if --machine-readable were, well, nice
-    execAction.commandLine(['vagrant', 'ssh', boxName, '--command', command])
-    execAction.setStandardOutput(new TapLoggerOutputStream(
-      command: command,
-      factory: getProgressLoggerFactory(),
-      logger: logger))
-    execAction.execute();
-  }
+    @Override
+    protected OutputStream createLoggerOutputStream() {
+        return new TapLoggerOutputStream(
+                command: commandLine.join(' '),
+                factory: getProgressLoggerFactory(),
+                logger: logger)
+    }
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.groovy
index 5f4a5e0a0c4..6b87ceeaf96 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.groovy
@@ -47,9 +47,9 @@ class TapLoggerOutputStream extends LoggingOutputStream {
   TapLoggerOutputStream(Map args) {
     logger = args.logger
     progressLogger = args.factory.newOperation(VagrantLoggerOutputStream)
-    progressLogger.setDescription("TAP output for $args.command")
+    progressLogger.setDescription("TAP output for `$args.command`")
     progressLogger.started()
-    progressLogger.progress("Starting $args.command...")
+    progressLogger.progress("Starting `$args.command`...")
   }
 
   void flush() {
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy
index 92b4a575eba..d79c2533fab 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy
@@ -18,11 +18,10 @@
  */
 package org.elasticsearch.gradle.vagrant
 
-import org.gradle.api.DefaultTask
-import org.gradle.api.tasks.TaskAction
+import org.apache.commons.io.output.TeeOutputStream
+import org.elasticsearch.gradle.LoggedExec
+import org.gradle.api.tasks.Input
 import org.gradle.logging.ProgressLoggerFactory
-import org.gradle.process.internal.ExecAction
-import org.gradle.process.internal.ExecActionFactory
 
 import javax.inject.Inject
 
@@ -30,43 +29,30 @@ import javax.inject.Inject
  * Runs a vagrant command. Pretty much like Exec task but with a nicer output
  * formatter and defaults to `vagrant` as first part of commandLine.
  */
-class VagrantCommandTask extends DefaultTask {
-  List<Object> commandLine
-  String boxName
-  ExecAction execAction
+public class VagrantCommandTask extends LoggedExec {
 
-  VagrantCommandTask() {
-    execAction = getExecActionFactory().newExecAction()
-  }
+    @Input
+    String boxName
 
-  @Inject
-  ProgressLoggerFactory getProgressLoggerFactory() {
-    throw new UnsupportedOperationException();
-  }
+    public VagrantCommandTask() {
+        executable = 'vagrant'
+        project.afterEvaluate {
+            // It'd be nice if --machine-readable were, well, nice
+            standardOutput = new TeeOutputStream(standardOutput, createLoggerOutputStream())
+        }
+    }
 
-  @Inject
-  ExecActionFactory getExecActionFactory() {
-    throw new UnsupportedOperationException();
-  }
+    protected OutputStream createLoggerOutputStream() {
+        return new VagrantLoggerOutputStream(
+            command: commandLine.join(' '),
+            factory: getProgressLoggerFactory(),
+            /* Vagrant tends to output a lot of stuff, but most of the important
+              stuff starts with ==> $box */
+            squashedPrefix: "==> $boxName: ")
+    }
 
-  void boxName(String boxName) {
-    this.boxName = boxName
-  }
-
-  void commandLine(Object... commandLine) {
-    this.commandLine = commandLine
-  }
-
-  @TaskAction
-  void exec() {
-    // It'd be nice if --machine-readable were, well, nice
-    execAction.commandLine(['vagrant'] + commandLine)
-    execAction.setStandardOutput(new VagrantLoggerOutputStream(
-      command: commandLine.join(' '),
-      factory: getProgressLoggerFactory(),
-      /* Vagrant tends to output a lot of stuff, but most of the important
-        stuff starts with ==> $box */
-      squashedPrefix: "==> $boxName: "))
-    execAction.execute();
-  }
+    @Inject
+    ProgressLoggerFactory getProgressLoggerFactory() {
+        throw new UnsupportedOperationException();
+    }
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.groovy
index 488c4511b1f..22008fa1a0e 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.groovy
@@ -20,6 +20,7 @@ package org.elasticsearch.gradle.vagrant
 
 import com.carrotsearch.gradle.junit4.LoggingOutputStream
 import org.gradle.logging.ProgressLogger
+import org.gradle.logging.ProgressLoggerFactory
 
 /**
  * Adapts an OutputStream being written to by vagrant into a ProcessLogger. It
@@ -42,79 +43,82 @@ import org.gradle.logging.ProgressLogger
  * to catch so it can render the output like
  * "Heading text > stdout from the provisioner".
  */
-class VagrantLoggerOutputStream extends LoggingOutputStream {
-  static final String HEADING_PREFIX = '==> '
+public class VagrantLoggerOutputStream extends LoggingOutputStream {
+    private static final String HEADING_PREFIX = '==> '
 
-  ProgressLogger progressLogger
-  String squashedPrefix
-  String lastLine = ''
-  boolean inProgressReport = false
-  String heading = ''
+    ProgressLoggerFactory progressLoggerFactory
 
-  VagrantLoggerOutputStream(Map args) {
-    progressLogger = args.factory.newOperation(VagrantLoggerOutputStream)
-    progressLogger.setDescription("Vagrant $args.command")
-    progressLogger.started()
-    progressLogger.progress("Starting vagrant $args.command...")
-    squashedPrefix = args.squashedPrefix
-  }
 
-  void flush() {
-    if (end == start) return
-    line(new String(buffer, start, end - start))
-    start = end
-  }
+    private ProgressLogger progressLogger
+    String squashedPrefix
+    String lastLine = ''
+    boolean inProgressReport = false
+    String heading = ''
 
-  void line(String line) {
-    // debugPrintLine(line) // Uncomment me to log every incoming line
-    if (line.startsWith('\r\u001b')) {
-      /* We don't want to try to be a full terminal emulator but we want to
-        keep the escape sequences from leaking and catch _some_ of the
-        meaning. */
-      line = line.substring(2)
-      if ('[K' == line) {
-        inProgressReport = true
-      }
-      return
+    VagrantLoggerOutputStream(Map args) {
+        progressLogger = args.factory.newOperation(VagrantLoggerOutputStream)
+        progressLogger.setDescription("Vagrant output for `$args.command`")
+        progressLogger.started()
+        progressLogger.progress("Starting `$args.command`...")
+        squashedPrefix = args.squashedPrefix
     }
-    if (line.startsWith(squashedPrefix)) {
-      line = line.substring(squashedPrefix.length())
-      inProgressReport = false
-      lastLine = line
-      if (line.startsWith(HEADING_PREFIX)) {
-        line = line.substring(HEADING_PREFIX.length())
-        heading = line + ' > '
-      } else {
-        line = heading + line
-      }
-    } else if (inProgressReport) {
-      inProgressReport = false
-      line = lastLine + line
-    } else {
-      return
-    }
-    // debugLogLine(line) // Uncomment me to log every line we add to the logger
-    progressLogger.progress(line)
-  }
 
-  void debugPrintLine(line) {
-    System.out.print '----------> '
-    for (int i = start; i < end; i++) {
-      switch (buffer[i] as char) {
-      case ' '..'~':
-        System.out.print buffer[i] as char
-        break
-      default:
-        System.out.print '%'
-        System.out.print Integer.toHexString(buffer[i])
-      }
+    void flush() {
+        if (end == start) return
+        line(new String(buffer, start, end - start))
+        start = end
     }
-    System.out.print '\n'
-  }
 
-  void debugLogLine(line) {
-    System.out.print '>>>>>>>>>>> '
-    System.out.print line
-    System.out.print '\n'
-  }
+    void line(String line) {
+        // debugPrintLine(line) // Uncomment me to log every incoming line
+        if (line.startsWith('\r\u001b')) {
+            /* We don't want to try to be a full terminal emulator but we want to
+              keep the escape sequences from leaking and catch _some_ of the
+              meaning. */
+            line = line.substring(2)
+            if ('[K' == line) {
+                inProgressReport = true
+            }
+            return
+        }
+        if (line.startsWith(squashedPrefix)) {
+            line = line.substring(squashedPrefix.length())
+            inProgressReport = false
+            lastLine = line
+            if (line.startsWith(HEADING_PREFIX)) {
+                line = line.substring(HEADING_PREFIX.length())
+                heading = line + ' > '
+            } else {
+                line = heading + line
+            }
+        } else if (inProgressReport) {
+            inProgressReport = false
+            line = lastLine + line
+        } else {
+            return
+        }
+        // debugLogLine(line) // Uncomment me to log every line we add to the logger
+        progressLogger.progress(line)
+    }
+
+    void debugPrintLine(line) {
+        System.out.print '----------> '
+        for (int i = start; i < end; i++) {
+            switch (buffer[i] as char) {
+                case ' '..'~':
+                    System.out.print buffer[i] as char
+                    break
+                default:
+                    System.out.print '%'
+                    System.out.print Integer.toHexString(buffer[i])
+            }
+        }
+        System.out.print '\n'
+    }
+
+    void debugLogLine(line) {
+        System.out.print '>>>>>>>>>>> '
+        System.out.print line
+        System.out.print '\n'
+    }
 }
diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle
index 735a401e542..211f8ffa5d1 100644
--- a/qa/vagrant/build.gradle
+++ b/qa/vagrant/build.gradle
@@ -156,7 +156,7 @@ for (String box : availableBoxes) {
   // always add a halt task for all boxes, so clean makes sure they are all shutdown
   Task halt = tasks.create("vagrant${boxTask}#halt", VagrantCommandTask) {
     boxName box
-    commandLine 'halt', box
+    args 'halt', box
   }
   stop.dependsOn(halt)
   if (boxes.contains(box) == false) {
@@ -176,7 +176,7 @@ for (String box : availableBoxes) {
       vagrant's default but its possible to change that default and folks do.
       But the boxes that we use are unlikely to work properly with other
       virtualization providers. Thus the lock. */
-    commandLine 'up', box, '--provision', '--provider', 'virtualbox'
+    args 'up', box, '--provision', '--provider', 'virtualbox'
     /* It'd be possible to check if the box is already up here and output
       SKIPPED but that would require running vagrant status which is slow! */
     dependsOn checkVagrantVersion

From 928e2b904db713d8d85e7bce0391f53f55928af9 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Thu, 5 May 2016 03:42:14 -0400
Subject: [PATCH 0080/1311] painless: optimize/simplify dynamic field and
 method access

---
 .../java/org/elasticsearch/painless/Def.java  | 52 +++++--------------
 1 file changed, 12 insertions(+), 40 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 71c7de23c85..b8cf7fb6069 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -242,18 +242,16 @@ public class Def {
     }
 
     public static Method getMethod(final Object owner, final String name, final Definition definition) {
-        Struct struct = null;
         Class<?> clazz = owner.getClass();
-        Method method = null;
 
         while (clazz != null) {
-            struct = definition.classes.get(clazz);
+            Struct struct = definition.classes.get(clazz);
 
             if (struct != null) {
-                method = struct.methods.get(name);
+                Method method = struct.methods.get(name);
 
                 if (method != null) {
-                    break;
+                    return method;
                 }
             }
 
@@ -261,45 +259,31 @@ public class Def {
                 struct = definition.classes.get(iface);
 
                 if (struct != null) {
-                    method = struct.methods.get(name);
+                    Method method = struct.methods.get(name);
 
                     if (method != null) {
-                        break;
+                        return method;
                     }
                 }
             }
 
-            if (struct != null) {
-                method = struct.methods.get(name);
-
-                if (method != null) {
-                    break;
-                }
-            }
-
             clazz = clazz.getSuperclass();
         }
 
-        if (struct == null) {
-            throw new IllegalArgumentException("Unable to find a dynamic struct for class [" + owner.getClass() + "].");
-        }
-
-        return method;
+        return null;
     }
 
     public static Field getField(final Object owner, final String name, final Definition definition) {
-        Struct struct = null;
         Class<?> clazz = owner.getClass();
-        Field field = null;
 
         while (clazz != null) {
-            struct = definition.classes.get(clazz);
+            Struct struct = definition.classes.get(clazz);
 
             if (struct != null) {
-                field = struct.members.get(name);
+                Field field = struct.members.get(name);
 
                 if (field != null) {
-                    break;
+                    return field;
                 }
             }
 
@@ -307,30 +291,18 @@ public class Def {
                 struct = definition.classes.get(iface);
 
                 if (struct != null) {
-                    field = struct.members.get(name);
+                    Field field = struct.members.get(name);
 
                     if (field != null) {
-                        break;
+                        return field;
                     }
                 }
             }
 
-            if (struct != null) {
-                field = struct.members.get(name);
-
-                if (field != null) {
-                    break;
-                }
-            }
-
             clazz = clazz.getSuperclass();
         }
 
-        if (struct == null) {
-            throw new IllegalArgumentException("Unable to find a dynamic struct for class [" + owner.getClass() + "].");
-        }
-
-        return field;
+        return null;
     }
 
     public static Transform getTransform(Class<?> fromClass, Class<?> toClass, final Definition definition) {

From 59c135b58d7c2fd788667790ca7cae48f2f49c28 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Thu, 5 May 2016 03:47:56 -0400
Subject: [PATCH 0081/1311] make internal Def methods private and add basic
 javadocs

---
 .../src/main/java/org/elasticsearch/painless/Def.java       | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index b8cf7fb6069..4e170c93e49 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -241,7 +241,8 @@ public class Def {
         }
     }
 
-    public static Method getMethod(final Object owner, final String name, final Definition definition) {
+    /** Method lookup for owner.name(), returns null if no matching method was found */ 
+    private static Method getMethod(final Object owner, final String name, final Definition definition) {
         Class<?> clazz = owner.getClass();
 
         while (clazz != null) {
@@ -273,7 +274,8 @@ public class Def {
         return null;
     }
 
-    public static Field getField(final Object owner, final String name, final Definition definition) {
+    /** Field lookup for owner.name, returns null if no matching field was found */ 
+    private static Field getField(final Object owner, final String name, final Definition definition) {
         Class<?> clazz = owner.getClass();
 
         while (clazz != null) {

From e11b96ca9c26957374a1928c605845cc4f678f51 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 5 May 2016 10:36:21 -0400
Subject: [PATCH 0082/1311] Default to server VM and add client VM check

Today we softly warn about running with the client VM. However, we
should really refuse to start in production mode if running with the
client VM as the performance of the client VM is too devastating for a
server application. This commit adds an option to jvm.options to ensure
that we are starting with the server VM (on all 32-bit non-Windows
platforms on server-class machines (2+ CPUs, 2+ GB physical RAM) this is
the default and on all 64-bit platforms this is the only option) and
adds a bootstrap check for the client VM.

Relates #18155
---
 .../elasticsearch/bootstrap/Bootstrap.java    |  6 ----
 .../bootstrap/BootstrapCheck.java             | 28 +++++++++++++++++++
 .../bootstrap/BootstrapCheckTests.java        | 22 +++++++++++++++
 .../src/main/resources/config/jvm.options     |  3 ++
 4 files changed, 53 insertions(+), 6 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
index 1131a4a99ec..6d35cafd088 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
@@ -246,12 +246,6 @@ final class Bootstrap {
             PidFile.create(environment.pidFile(), true);
         }
 
-        // warn if running using the client VM
-        if (JvmInfo.jvmInfo().getVmName().toLowerCase(Locale.ROOT).contains("client")) {
-            ESLogger logger = Loggers.getLogger(Bootstrap.class);
-            logger.warn("jvm uses the client vm, make sure to run `java` with the server vm for best performance by adding `-server` to the command line");
-        }
-
         try {
             if (!foreground) {
                 Loggers.disableConsoleLogging();
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
index 626e274076c..37a89531184 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
@@ -163,6 +163,7 @@ final class BootstrapCheck {
         if (Constants.LINUX) {
             checks.add(new MaxMapCountCheck());
         }
+        checks.add(new ClientJvmCheck());
         return Collections.unmodifiableList(checks);
     }
 
@@ -476,4 +477,31 @@ final class BootstrapCheck {
 
     }
 
+    static class ClientJvmCheck implements BootstrapCheck.Check {
+
+        @Override
+        public boolean check() {
+            return getVmName().toLowerCase(Locale.ROOT).contains("client");
+        }
+
+        // visible for testing
+        String getVmName() {
+            return JvmInfo.jvmInfo().getVmName();
+        }
+
+        @Override
+        public String errorMessage() {
+            return String.format(
+                    Locale.ROOT,
+                    "JVM is using the client VM [%s] but should be using a server VM for the best performance",
+                    getVmName());
+        }
+
+        @Override
+        public final boolean isSystemCheck() {
+            return false;
+        }
+
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
index 235957ac18b..d354adc7544 100644
--- a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
+++ b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
@@ -31,6 +31,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
 
 import static org.hamcrest.CoreMatchers.allOf;
 import static org.hamcrest.CoreMatchers.containsString;
@@ -373,6 +374,27 @@ public class BootstrapCheckTests extends ESTestCase {
         expectThrows(RuntimeException.class, () -> BootstrapCheck.check(true, false, defaultChecks, "testMinMasterNodes"));
     }
 
+    public void testClientJvmCheck() {
+        final AtomicReference<String> vmName = new AtomicReference<>("Java HotSpot(TM) 32-Bit Client VM");
+        final BootstrapCheck.Check check = new BootstrapCheck.ClientJvmCheck() {
+            @Override
+            String getVmName() {
+                return vmName.get();
+            }
+        };
+
+        RuntimeException e = expectThrows(
+                RuntimeException.class,
+                () -> BootstrapCheck.check(true, false, Collections.singletonList(check), "testClientJvmCheck"));
+        assertThat(
+                e.getMessage(),
+                containsString("JVM is using the client VM [Java HotSpot(TM) 32-Bit Client VM] " +
+                        "but should be using a server VM for the best performance"));
+
+        vmName.set("Java HotSpot(TM) 32-Bit Server VM");
+        BootstrapCheck.check(true, false, Collections.singletonList(check), "testClientJvmCheck");
+    }
+
     public void testIgnoringSystemChecks() {
         BootstrapCheck.Check check = new BootstrapCheck.Check() {
             @Override
diff --git a/distribution/src/main/resources/config/jvm.options b/distribution/src/main/resources/config/jvm.options
index 726f2772db6..b27b0a0a0f0 100644
--- a/distribution/src/main/resources/config/jvm.options
+++ b/distribution/src/main/resources/config/jvm.options
@@ -48,6 +48,9 @@
 
 ## basic
 
+# force the server VM
+-server
+
 # set to headless, just in case
 -Djava.awt.headless=true
 

From 2cae575f538af8cea473fa69788645c39c157307 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Thu, 5 May 2016 09:26:02 -0700
Subject: [PATCH 0083/1311] Added single-quoted strings.

Closes #18150
---
 .../modules/scripting/painless.asciidoc       |  14 +-
 .../src/main/antlr/PainlessLexer.g4           |   3 +-
 .../src/main/antlr/PainlessParser.g4          |   1 -
 .../org/elasticsearch/painless/Analyzer.java  |   8 -
 .../painless/AnalyzerCaster.java              |  15 +-
 .../painless/AnalyzerExpression.java          |  12 -
 .../painless/AnalyzerExternal.java            |   3 +-
 .../elasticsearch/painless/Definition.java    |   9 +
 .../elasticsearch/painless/PainlessLexer.java | 342 ++++++------
 .../painless/PainlessParser.java              | 495 +++++++++---------
 .../painless/PainlessParserBaseVisitor.java   |   7 -
 .../painless/PainlessParserVisitor.java       |   7 -
 .../org/elasticsearch/painless/Utility.java   |  28 +
 .../org/elasticsearch/painless/Writer.java    |   8 -
 .../painless/WriterExpression.java            |  15 -
 .../painless/BasicExpressionTests.java        |   2 +-
 .../elasticsearch/painless/EqualsTests.java   |  44 +-
 .../elasticsearch/painless/StringTests.java   | 114 ++++
 .../test/plan_a/20_scriptfield.yaml           |   2 +-
 .../rest-api-spec/test/plan_a/30_search.yaml  |  12 +-
 20 files changed, 614 insertions(+), 527 deletions(-)

diff --git a/docs/reference/modules/scripting/painless.asciidoc b/docs/reference/modules/scripting/painless.asciidoc
index 1937bd9947b..b96e1f099de 100644
--- a/docs/reference/modules/scripting/painless.asciidoc
+++ b/docs/reference/modules/scripting/painless.asciidoc
@@ -86,7 +86,7 @@ GET /hockey-stats/_search
       "script_score": {
         "script": {
           "lang": "painless",
-          "inline": "int total = 0; for (int i = 0; i < input.doc.goals.size(); ++i) { total += input.doc.goals[i]; } return total;"
+          "inline": "int total = 0; for (int i = 0; i < input.doc['goals'].size(); ++i) { total += input.doc['goals'][i]; } return total;"
         }
       }
     }
@@ -108,7 +108,7 @@ GET /hockey-stats/_search
     "total_goals": {
       "script": {
         "lang": "painless",
-        "inline": "int total = 0; for (int i = 0; i < input.doc.goals.size(); ++i) { total += input.doc.goals[i]; } return total;"
+        "inline": "int total = 0; for (int i = 0; i < input.doc['goals'].size(); ++i) { total += input.doc['goals'][i]; } return total;"
       }
     }
   }
@@ -118,7 +118,7 @@ GET /hockey-stats/_search
 
 You must always specify the index of the field value you want, even if there's only a single item in the field.
 All fields in Elasticsearch are multi-valued and Painless does not provide a `.value` shortcut. The following example uses a Painless script to sort the players by their combined first and last names. The names are accessed using
-`input.doc.first.0` and `input.doc.last.0`.
+`input.doc['first'].0` and `input.doc['last'].0`.
 
 [source,sh]
 ----------------------------------------------------------------
@@ -133,7 +133,7 @@ GET /hockey-stats/_search
       "order": "asc",
       "script": {
         "lang": "painless",
-        "inline": "input.doc.first.0 + \" \" + input.doc.last.0"
+        "inline": "input.doc['first'].0 + ' ' + input.doc['last'].0"
       }
     }
   }
@@ -219,13 +219,13 @@ GET /hockey-stats/_search
     "full_name_dynamic": {
       "script": {
         "lang": "painless",
-        "inline": "def first = input.doc.first.0; def last = input.doc.last.0; return first + \" \" + last;"
+        "inline": "def first = input.doc['first'].0; def last = input.doc['last'].0; return first + ' ' + last;"
       }
     },
     "full_name_static": {
       "script": {
         "lang": "painless",
-        "inline": "String first = (String)((List)((Map)input.get(\"doc\")).get(\"first\")).get(0); String last = (String)((List)((Map)input.get(\"doc\")).get(\"last\")).get(0); return first + \" \" + last;"
+        "inline": "String first = (String)((List)((Map)input.get('doc')).get('first')).get(0); String last = (String)((List)((Map)input.get('doc')).get('last')).get(0); return first + ' ' + last;"
       }
     }
   }
@@ -727,4 +727,4 @@ Def
     static Long defToLong(def)
     static Float defToFloat(def)
     static Double defToDouble(def)
------
\ No newline at end of file
+-----
diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.g4 b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
index f1e40f93d02..eab0670f973 100644
--- a/modules/lang-painless/src/main/antlr/PainlessLexer.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
@@ -90,8 +90,7 @@ HEX: '0' [xX] [0-9a-fA-F]+ [lL]?;
 INTEGER: ( '0' | [1-9] [0-9]* ) [lLfFdD]?;
 DECIMAL: ( '0' | [1-9] [0-9]* ) DOT [0-9]* ( [eE] [+\-]? [0-9]+ )? [fF]?;
 
-STRING: '"' ( '\\"' | '\\\\' | ~[\\"] )*? '"';
-CHAR: '\'' . '\'';
+STRING: ( '"' ( '\\"' | '\\\\' | ~[\\"] )*? '"' ) | ( '\'' ( '\\\'' | '\\\\' | ~[\\"] )*? '\'' );
 
 TRUE:  'true';
 FALSE: 'false';
diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index c9e0de72210..1ecd72528e0 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -89,7 +89,6 @@ generic
 expression
     :               LP expression RP                                    # precedence
     |               ( OCTAL | HEX | INTEGER | DECIMAL )                 # numeric
-    |               CHAR                                                # char
     |               TRUE                                                # true
     |               FALSE                                               # false
     |               NULL                                                # null
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
index 50473ea4e88..f411db00802 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
@@ -26,7 +26,6 @@ import org.elasticsearch.painless.PainlessParser.BinaryContext;
 import org.elasticsearch.painless.PainlessParser.BoolContext;
 import org.elasticsearch.painless.PainlessParser.BreakContext;
 import org.elasticsearch.painless.PainlessParser.CastContext;
-import org.elasticsearch.painless.PainlessParser.CharContext;
 import org.elasticsearch.painless.PainlessParser.CompContext;
 import org.elasticsearch.painless.PainlessParser.ConditionalContext;
 import org.elasticsearch.painless.PainlessParser.ContinueContext;
@@ -276,13 +275,6 @@ class Analyzer extends PainlessParserBaseVisitor<Void> {
         return null;
     }
 
-    @Override
-    public Void visitChar(final CharContext ctx) {
-        expression.processChar(ctx);
-
-        return null;
-    }
-
     @Override
     public Void visitTrue(final TrueContext ctx) {
         expression.processTrue(ctx);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
index 46a510bc6bb..dddb3491f64 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
@@ -159,6 +159,7 @@ class AnalyzerCaster {
                         return checkTransform(source, cast);
                     case BYTE_OBJ:
                     case SHORT_OBJ:
+                    case STRING:
                         if (explicit)
                             return checkTransform(source, cast);
 
@@ -371,6 +372,7 @@ class AnalyzerCaster {
                     case SHORT:
                     case BYTE_OBJ:
                     case SHORT_OBJ:
+                    case STRING:
                         if (explicit)
                             return checkTransform(source, cast);
 
@@ -470,6 +472,15 @@ class AnalyzerCaster {
                 }
 
                 break;
+            case STRING:
+                switch (to.sort) {
+                    case CHAR:
+                    case CHAR_OBJ:
+                        if (explicit)
+                            return checkTransform(source, cast);
+
+                        break;
+                }
         }
 
         try {
@@ -556,8 +567,8 @@ class AnalyzerCaster {
         } catch (IllegalAccessException | IllegalArgumentException |
             java.lang.reflect.InvocationTargetException | NullPointerException |
             ExceptionInInitializerError exception) {
-            throw new IllegalStateException(AnalyzerUtility.error(source) + "Unable to invoke transform to cast constant from " +
-                "[" + transform.from.name + "] to [" + transform.to.name + "].");
+            throw new IllegalArgumentException(AnalyzerUtility.error(source) +
+                "Cannot cast constant from [" + transform.from.name + "] to [" + transform.to.name + "].");
         }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExpression.java
index 029e7e530b5..3db42f88696 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExpression.java
@@ -27,7 +27,6 @@ import org.elasticsearch.painless.PainlessParser.AssignmentContext;
 import org.elasticsearch.painless.PainlessParser.BinaryContext;
 import org.elasticsearch.painless.PainlessParser.BoolContext;
 import org.elasticsearch.painless.PainlessParser.CastContext;
-import org.elasticsearch.painless.PainlessParser.CharContext;
 import org.elasticsearch.painless.PainlessParser.CompContext;
 import org.elasticsearch.painless.PainlessParser.ConditionalContext;
 import org.elasticsearch.painless.PainlessParser.DecltypeContext;
@@ -161,17 +160,6 @@ class AnalyzerExpression {
         }
     }
 
-    void processChar(final CharContext ctx) {
-        final ExpressionMetadata charemd = metadata.getExpressionMetadata(ctx);
-
-        if (ctx.CHAR() == null) {
-            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-        }
-
-        charemd.preConst = ctx.CHAR().getText().charAt(1);
-        charemd.from = definition.charType;
-    }
-
     void processTrue(final TrueContext ctx) {
         final ExpressionMetadata trueemd = metadata.getExpressionMetadata(ctx);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java
index 2ce5fa14d22..32ab8f43c9e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java
@@ -584,8 +584,7 @@ class AnalyzerExternal {
                                     "Illegal list get shortcut [" + value + "] for type [" + struct.name + "].");
                             }
 
-                            if (setter != null && (setter.rtn.sort != Sort.VOID || setter.arguments.size() != 2 ||
-                                setter.arguments.get(0).sort != Sort.INT)) {
+                            if (setter != null && (setter.arguments.size() != 2 || setter.arguments.get(0).sort != Sort.INT)) {
                                 throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
                                     "Illegal list set shortcut [" + value + "] for type [" + struct.name + "].");
                             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 6cb4125f026..450f2aa6fa4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -801,6 +801,7 @@ class Definition {
         addMethod("Utility", "charToLong", null, true, longobjType, new Type[] {charType}, null, null);
         addMethod("Utility", "charToFloat", null, true, floatobjType, new Type[] {charType}, null, null);
         addMethod("Utility", "charToDouble", null, true, doubleobjType, new Type[] {charType}, null, null);
+        addMethod("Utility", "charToString", null, true, stringType, new Type[] {charType}, null, null);
         addMethod("Utility", "CharacterToboolean", null, true, booleanType, new Type[] {charobjType}, null, null);
         addMethod("Utility", "CharacterTobyte", null, true, byteType, new Type[] {charobjType}, null, null);
         addMethod("Utility", "CharacterToshort", null, true, shortType, new Type[] {charobjType}, null, null);
@@ -815,6 +816,7 @@ class Definition {
         addMethod("Utility", "CharacterToLong", null, true, longobjType, new Type[] {charobjType}, null, null);
         addMethod("Utility", "CharacterToFloat", null, true, floatobjType, new Type[] {charobjType}, null, null);
         addMethod("Utility", "CharacterToDouble", null, true, doubleobjType, new Type[] {charobjType}, null, null);
+        addMethod("Utility", "CharacterToString", null, true, stringType, new Type[] {charobjType}, null, null);
         addMethod("Utility", "intToboolean", null, true, booleanType, new Type[] {intType}, null, null);
         addMethod("Utility", "intToByte", null, true, byteobjType, new Type[] {intType}, null, null);
         addMethod("Utility", "intToShort", null, true, shortobjType, new Type[] {intType}, null, null);
@@ -851,6 +853,8 @@ class Definition {
         addMethod("Utility", "doubleToFloat", null, true, floatobjType, new Type[] {doubleType}, null, null);
         addMethod("Utility", "DoubleToboolean", null, true, booleanType, new Type[] {doubleobjType}, null, null);
         addMethod("Utility", "DoubleTochar", null, true, charType, new Type[] {doubleobjType}, null, null);
+        addMethod("Utility", "StringTochar", null, true, charType, new Type[] {stringType}, null, null);
+        addMethod("Utility", "StringToCharacter", null, true, charobjType, new Type[] {stringType}, null, null);
 
         addMethod("Math", "abs", null, true, doubleType, new Type[] {doubleType}, null, null);
         addMethod("Math", "fabs", "abs", true, floatType, new Type[] {floatType}, null, null);
@@ -1125,6 +1129,7 @@ class Definition {
         addTransform(charType, longobjType, "Utility", "charToLong", true);
         addTransform(charType, floatobjType, "Utility", "charToFloat", true);
         addTransform(charType, doubleobjType, "Utility", "charToDouble", true);
+        addTransform(charType, stringType, "Utility", "charToString", true);
 
         addTransform(intType, booleanType, "Utility", "intToboolean", true);
         addTransform(intType, objectType, "Integer", "valueOf", true);
@@ -1281,6 +1286,7 @@ class Definition {
         addTransform(charobjType, longobjType, "Utility", "CharacterToLong", true);
         addTransform(charobjType, floatobjType, "Utility", "CharacterToFloat", true);
         addTransform(charobjType, doubleobjType, "Utility", "CharacterToDouble", true);
+        addTransform(charobjType, stringType, "Utility", "CharacterToString", true);
 
         addTransform(intobjType, booleanType, "Utility", "IntegerToboolean", true);
         addTransform(intobjType, byteType, "Integer", "byteValue", false);
@@ -1345,6 +1351,9 @@ class Definition {
         addTransform(doubleobjType, intobjType, "Utility", "NumberToInteger", true);
         addTransform(doubleobjType, longobjType, "Utility", "NumberToLong", true);
         addTransform(doubleobjType, floatobjType, "Utility", "NumberToFloat", true);
+
+        addTransform(stringType, charType, "Utility", "StringTochar", true);
+        addTransform(stringType, charobjType, "Utility", "StringToCharacter", true);
     }
 
     private void addDefaultBounds() {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessLexer.java
index ad71e155bb0..6951c67605a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessLexer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessLexer.java
@@ -25,7 +25,7 @@ class PainlessLexer extends Lexer {
     BWOR=44, BOOLAND=45, BOOLOR=46, COND=47, COLON=48, INCR=49, DECR=50, ASSIGN=51, 
     AADD=52, ASUB=53, AMUL=54, ADIV=55, AREM=56, AAND=57, AXOR=58, AOR=59, 
     ALSH=60, ARSH=61, AUSH=62, OCTAL=63, HEX=64, INTEGER=65, DECIMAL=66, STRING=67, 
-    CHAR=68, TRUE=69, FALSE=70, NULL=71, ID=72, EXTINTEGER=73, EXTID=74;
+    TRUE=68, FALSE=69, NULL=70, ID=71, EXTINTEGER=72, EXTID=73;
   public static final int EXT = 1;
   public static String[] modeNames = {
     "DEFAULT_MODE", "EXT"
@@ -39,8 +39,8 @@ class PainlessLexer extends Lexer {
     "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "BWXOR", "BWOR", "BOOLAND", 
     "BOOLOR", "COND", "COLON", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", 
     "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", 
-    "HEX", "INTEGER", "DECIMAL", "STRING", "CHAR", "TRUE", "FALSE", "NULL", 
-    "ID", "EXTINTEGER", "EXTID"
+    "HEX", "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "ID", 
+    "EXTINTEGER", "EXTID"
   };
 
   private static final String[] _LITERAL_NAMES = {
@@ -51,7 +51,7 @@ class PainlessLexer extends Lexer {
     "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", "'^'", "'|'", "'&&'", 
     "'||'", "'?'", "':'", "'++'", "'--'", "'='", "'+='", "'-='", "'*='", "'/='", 
     "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", "'>>>='", null, null, 
-    null, null, null, null, "'true'", "'false'", "'null'"
+    null, null, null, "'true'", "'false'", "'null'"
   };
   private static final String[] _SYMBOLIC_NAMES = {
     null, "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", 
@@ -61,8 +61,8 @@ class PainlessLexer extends Lexer {
     "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "BWXOR", "BWOR", "BOOLAND", 
     "BOOLOR", "COND", "COLON", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", 
     "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", 
-    "HEX", "INTEGER", "DECIMAL", "STRING", "CHAR", "TRUE", "FALSE", "NULL", 
-    "ID", "EXTINTEGER", "EXTID"
+    "HEX", "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "ID", 
+    "EXTINTEGER", "EXTID"
   };
   public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
 
@@ -119,7 +119,7 @@ class PainlessLexer extends Lexer {
   public ATN getATN() { return _ATN; }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2L\u01f4\b\1\b\1\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2K\u01fb\b\1\b\1\4"+
     "\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n"+
     "\4\13\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22"+
     "\t\22\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31"+
@@ -128,168 +128,172 @@ class PainlessLexer extends Lexer {
     "+\4,\t,\4-\t-\4.\t.\4/\t/\4\60\t\60\4\61\t\61\4\62\t\62\4\63\t\63\4\64"+
     "\t\64\4\65\t\65\4\66\t\66\4\67\t\67\48\t8\49\t9\4:\t:\4;\t;\4<\t<\4=\t"+
     "=\4>\t>\4?\t?\4@\t@\4A\tA\4B\tB\4C\tC\4D\tD\4E\tE\4F\tF\4G\tG\4H\tH\4"+
-    "I\tI\4J\tJ\4K\tK\3\2\6\2\u009a\n\2\r\2\16\2\u009b\3\2\3\2\3\3\3\3\3\3"+
-    "\3\3\7\3\u00a4\n\3\f\3\16\3\u00a7\13\3\3\3\3\3\3\3\3\3\3\3\7\3\u00ae\n"+
-    "\3\f\3\16\3\u00b1\13\3\3\3\3\3\5\3\u00b5\n\3\3\3\3\3\3\4\3\4\3\5\3\5\3"+
-    "\6\3\6\3\7\3\7\3\b\3\b\3\t\3\t\3\n\3\n\3\n\3\n\3\13\3\13\3\f\3\f\3\r\3"+
-    "\r\3\r\3\16\3\16\3\16\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3\17\3\20\3\20"+
-    "\3\20\3\21\3\21\3\21\3\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22"+
-    "\3\23\3\23\3\23\3\23\3\23\3\23\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\25"+
-    "\3\25\3\25\3\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\27\3\27\3\27\3\30"+
-    "\3\30\3\30\3\30\3\30\3\30\3\31\3\31\3\32\3\32\3\33\3\33\3\34\3\34\3\35"+
-    "\3\35\3\36\3\36\3\37\3\37\3 \3 \3 \3!\3!\3!\3\"\3\"\3\"\3\"\3#\3#\3$\3"+
-    "$\3$\3%\3%\3&\3&\3&\3\'\3\'\3\'\3(\3(\3(\3(\3)\3)\3)\3*\3*\3*\3*\3+\3"+
-    "+\3,\3,\3-\3-\3.\3.\3.\3/\3/\3/\3\60\3\60\3\61\3\61\3\62\3\62\3\62\3\63"+
-    "\3\63\3\63\3\64\3\64\3\65\3\65\3\65\3\66\3\66\3\66\3\67\3\67\3\67\38\3"+
-    "8\38\39\39\39\3:\3:\3:\3;\3;\3;\3<\3<\3<\3=\3=\3=\3=\3>\3>\3>\3>\3?\3"+
-    "?\3?\3?\3?\3@\3@\6@\u017b\n@\r@\16@\u017c\3@\5@\u0180\n@\3A\3A\3A\6A\u0185"+
-    "\nA\rA\16A\u0186\3A\5A\u018a\nA\3B\3B\3B\7B\u018f\nB\fB\16B\u0192\13B"+
-    "\5B\u0194\nB\3B\5B\u0197\nB\3C\3C\3C\7C\u019c\nC\fC\16C\u019f\13C\5C\u01a1"+
-    "\nC\3C\3C\7C\u01a5\nC\fC\16C\u01a8\13C\3C\3C\5C\u01ac\nC\3C\6C\u01af\n"+
-    "C\rC\16C\u01b0\5C\u01b3\nC\3C\5C\u01b6\nC\3D\3D\3D\3D\3D\3D\7D\u01be\n"+
-    "D\fD\16D\u01c1\13D\3D\3D\3E\3E\3E\3E\3F\3F\3F\3F\3F\3G\3G\3G\3G\3G\3G"+
-    "\3H\3H\3H\3H\3H\3I\3I\7I\u01db\nI\fI\16I\u01de\13I\3J\3J\3J\7J\u01e3\n"+
-    "J\fJ\16J\u01e6\13J\5J\u01e8\nJ\3J\3J\3K\3K\7K\u01ee\nK\fK\16K\u01f1\13"+
-    "K\3K\3K\5\u00a5\u00af\u01bf\2L\4\3\6\4\b\5\n\6\f\7\16\b\20\t\22\n\24\13"+
-    "\26\f\30\r\32\16\34\17\36\20 \21\"\22$\23&\24(\25*\26,\27.\30\60\31\62"+
-    "\32\64\33\66\348\35:\36<\37> @!B\"D#F$H%J&L\'N(P)R*T+V,X-Z.\\/^\60`\61"+
-    "b\62d\63f\64h\65j\66l\67n8p9r:t;v<x=z>|?~@\u0080A\u0082B\u0084C\u0086"+
-    "D\u0088E\u008aF\u008cG\u008eH\u0090I\u0092J\u0094K\u0096L\4\2\3\21\5\2"+
-    "\13\f\17\17\"\"\4\2\f\f\17\17\3\2\629\4\2NNnn\4\2ZZzz\5\2\62;CHch\3\2"+
-    "\63;\3\2\62;\b\2FFHHNNffhhnn\4\2GGgg\4\2--//\4\2HHhh\4\2$$^^\5\2C\\aa"+
-    "c|\6\2\62;C\\aac|\u020b\2\4\3\2\2\2\2\6\3\2\2\2\2\b\3\2\2\2\2\n\3\2\2"+
-    "\2\2\f\3\2\2\2\2\16\3\2\2\2\2\20\3\2\2\2\2\22\3\2\2\2\2\24\3\2\2\2\2\26"+
-    "\3\2\2\2\2\30\3\2\2\2\2\32\3\2\2\2\2\34\3\2\2\2\2\36\3\2\2\2\2 \3\2\2"+
-    "\2\2\"\3\2\2\2\2$\3\2\2\2\2&\3\2\2\2\2(\3\2\2\2\2*\3\2\2\2\2,\3\2\2\2"+
-    "\2.\3\2\2\2\2\60\3\2\2\2\2\62\3\2\2\2\2\64\3\2\2\2\2\66\3\2\2\2\28\3\2"+
-    "\2\2\2:\3\2\2\2\2<\3\2\2\2\2>\3\2\2\2\2@\3\2\2\2\2B\3\2\2\2\2D\3\2\2\2"+
-    "\2F\3\2\2\2\2H\3\2\2\2\2J\3\2\2\2\2L\3\2\2\2\2N\3\2\2\2\2P\3\2\2\2\2R"+
-    "\3\2\2\2\2T\3\2\2\2\2V\3\2\2\2\2X\3\2\2\2\2Z\3\2\2\2\2\\\3\2\2\2\2^\3"+
-    "\2\2\2\2`\3\2\2\2\2b\3\2\2\2\2d\3\2\2\2\2f\3\2\2\2\2h\3\2\2\2\2j\3\2\2"+
-    "\2\2l\3\2\2\2\2n\3\2\2\2\2p\3\2\2\2\2r\3\2\2\2\2t\3\2\2\2\2v\3\2\2\2\2"+
-    "x\3\2\2\2\2z\3\2\2\2\2|\3\2\2\2\2~\3\2\2\2\2\u0080\3\2\2\2\2\u0082\3\2"+
-    "\2\2\2\u0084\3\2\2\2\2\u0086\3\2\2\2\2\u0088\3\2\2\2\2\u008a\3\2\2\2\2"+
-    "\u008c\3\2\2\2\2\u008e\3\2\2\2\2\u0090\3\2\2\2\2\u0092\3\2\2\2\3\u0094"+
-    "\3\2\2\2\3\u0096\3\2\2\2\4\u0099\3\2\2\2\6\u00b4\3\2\2\2\b\u00b8\3\2\2"+
-    "\2\n\u00ba\3\2\2\2\f\u00bc\3\2\2\2\16\u00be\3\2\2\2\20\u00c0\3\2\2\2\22"+
-    "\u00c2\3\2\2\2\24\u00c4\3\2\2\2\26\u00c8\3\2\2\2\30\u00ca\3\2\2\2\32\u00cc"+
-    "\3\2\2\2\34\u00cf\3\2\2\2\36\u00d4\3\2\2\2 \u00da\3\2\2\2\"\u00dd\3\2"+
-    "\2\2$\u00e1\3\2\2\2&\u00ea\3\2\2\2(\u00f0\3\2\2\2*\u00f7\3\2\2\2,\u00fb"+
-    "\3\2\2\2.\u00ff\3\2\2\2\60\u0105\3\2\2\2\62\u010b\3\2\2\2\64\u010d\3\2"+
-    "\2\2\66\u010f\3\2\2\28\u0111\3\2\2\2:\u0113\3\2\2\2<\u0115\3\2\2\2>\u0117"+
-    "\3\2\2\2@\u0119\3\2\2\2B\u011c\3\2\2\2D\u011f\3\2\2\2F\u0123\3\2\2\2H"+
-    "\u0125\3\2\2\2J\u0128\3\2\2\2L\u012a\3\2\2\2N\u012d\3\2\2\2P\u0130\3\2"+
-    "\2\2R\u0134\3\2\2\2T\u0137\3\2\2\2V\u013b\3\2\2\2X\u013d\3\2\2\2Z\u013f"+
-    "\3\2\2\2\\\u0141\3\2\2\2^\u0144\3\2\2\2`\u0147\3\2\2\2b\u0149\3\2\2\2"+
-    "d\u014b\3\2\2\2f\u014e\3\2\2\2h\u0151\3\2\2\2j\u0153\3\2\2\2l\u0156\3"+
-    "\2\2\2n\u0159\3\2\2\2p\u015c\3\2\2\2r\u015f\3\2\2\2t\u0162\3\2\2\2v\u0165"+
-    "\3\2\2\2x\u0168\3\2\2\2z\u016b\3\2\2\2|\u016f\3\2\2\2~\u0173\3\2\2\2\u0080"+
-    "\u0178\3\2\2\2\u0082\u0181\3\2\2\2\u0084\u0193\3\2\2\2\u0086\u01a0\3\2"+
-    "\2\2\u0088\u01b7\3\2\2\2\u008a\u01c4\3\2\2\2\u008c\u01c8\3\2\2\2\u008e"+
-    "\u01cd\3\2\2\2\u0090\u01d3\3\2\2\2\u0092\u01d8\3\2\2\2\u0094\u01e7\3\2"+
-    "\2\2\u0096\u01eb\3\2\2\2\u0098\u009a\t\2\2\2\u0099\u0098\3\2\2\2\u009a"+
-    "\u009b\3\2\2\2\u009b\u0099\3\2\2\2\u009b\u009c\3\2\2\2\u009c\u009d\3\2"+
-    "\2\2\u009d\u009e\b\2\2\2\u009e\5\3\2\2\2\u009f\u00a0\7\61\2\2\u00a0\u00a1"+
-    "\7\61\2\2\u00a1\u00a5\3\2\2\2\u00a2\u00a4\13\2\2\2\u00a3\u00a2\3\2\2\2"+
-    "\u00a4\u00a7\3\2\2\2\u00a5\u00a6\3\2\2\2\u00a5\u00a3\3\2\2\2\u00a6\u00a8"+
-    "\3\2\2\2\u00a7\u00a5\3\2\2\2\u00a8\u00b5\t\3\2\2\u00a9\u00aa\7\61\2\2"+
-    "\u00aa\u00ab\7,\2\2\u00ab\u00af\3\2\2\2\u00ac\u00ae\13\2\2\2\u00ad\u00ac"+
-    "\3\2\2\2\u00ae\u00b1\3\2\2\2\u00af\u00b0\3\2\2\2\u00af\u00ad\3\2\2\2\u00b0"+
-    "\u00b2\3\2\2\2\u00b1\u00af\3\2\2\2\u00b2\u00b3\7,\2\2\u00b3\u00b5\7\61"+
-    "\2\2\u00b4\u009f\3\2\2\2\u00b4\u00a9\3\2\2\2\u00b5\u00b6\3\2\2\2\u00b6"+
-    "\u00b7\b\3\2\2\u00b7\7\3\2\2\2\u00b8\u00b9\7}\2\2\u00b9\t\3\2\2\2\u00ba"+
-    "\u00bb\7\177\2\2\u00bb\13\3\2\2\2\u00bc\u00bd\7]\2\2\u00bd\r\3\2\2\2\u00be"+
-    "\u00bf\7_\2\2\u00bf\17\3\2\2\2\u00c0\u00c1\7*\2\2\u00c1\21\3\2\2\2\u00c2"+
-    "\u00c3\7+\2\2\u00c3\23\3\2\2\2\u00c4\u00c5\7\60\2\2\u00c5\u00c6\3\2\2"+
-    "\2\u00c6\u00c7\b\n\3\2\u00c7\25\3\2\2\2\u00c8\u00c9\7.\2\2\u00c9\27\3"+
-    "\2\2\2\u00ca\u00cb\7=\2\2\u00cb\31\3\2\2\2\u00cc\u00cd\7k\2\2\u00cd\u00ce"+
-    "\7h\2\2\u00ce\33\3\2\2\2\u00cf\u00d0\7g\2\2\u00d0\u00d1\7n\2\2\u00d1\u00d2"+
-    "\7u\2\2\u00d2\u00d3\7g\2\2\u00d3\35\3\2\2\2\u00d4\u00d5\7y\2\2\u00d5\u00d6"+
-    "\7j\2\2\u00d6\u00d7\7k\2\2\u00d7\u00d8\7n\2\2\u00d8\u00d9\7g\2\2\u00d9"+
-    "\37\3\2\2\2\u00da\u00db\7f\2\2\u00db\u00dc\7q\2\2\u00dc!\3\2\2\2\u00dd"+
-    "\u00de\7h\2\2\u00de\u00df\7q\2\2\u00df\u00e0\7t\2\2\u00e0#\3\2\2\2\u00e1"+
-    "\u00e2\7e\2\2\u00e2\u00e3\7q\2\2\u00e3\u00e4\7p\2\2\u00e4\u00e5\7v\2\2"+
-    "\u00e5\u00e6\7k\2\2\u00e6\u00e7\7p\2\2\u00e7\u00e8\7w\2\2\u00e8\u00e9"+
-    "\7g\2\2\u00e9%\3\2\2\2\u00ea\u00eb\7d\2\2\u00eb\u00ec\7t\2\2\u00ec\u00ed"+
-    "\7g\2\2\u00ed\u00ee\7c\2\2\u00ee\u00ef\7m\2\2\u00ef\'\3\2\2\2\u00f0\u00f1"+
-    "\7t\2\2\u00f1\u00f2\7g\2\2\u00f2\u00f3\7v\2\2\u00f3\u00f4\7w\2\2\u00f4"+
-    "\u00f5\7t\2\2\u00f5\u00f6\7p\2\2\u00f6)\3\2\2\2\u00f7\u00f8\7p\2\2\u00f8"+
-    "\u00f9\7g\2\2\u00f9\u00fa\7y\2\2\u00fa+\3\2\2\2\u00fb\u00fc\7v\2\2\u00fc"+
-    "\u00fd\7t\2\2\u00fd\u00fe\7{\2\2\u00fe-\3\2\2\2\u00ff\u0100\7e\2\2\u0100"+
-    "\u0101\7c\2\2\u0101\u0102\7v\2\2\u0102\u0103\7e\2\2\u0103\u0104\7j\2\2"+
-    "\u0104/\3\2\2\2\u0105\u0106\7v\2\2\u0106\u0107\7j\2\2\u0107\u0108\7t\2"+
-    "\2\u0108\u0109\7q\2\2\u0109\u010a\7y\2\2\u010a\61\3\2\2\2\u010b\u010c"+
-    "\7#\2\2\u010c\63\3\2\2\2\u010d\u010e\7\u0080\2\2\u010e\65\3\2\2\2\u010f"+
-    "\u0110\7,\2\2\u0110\67\3\2\2\2\u0111\u0112\7\61\2\2\u01129\3\2\2\2\u0113"+
-    "\u0114\7\'\2\2\u0114;\3\2\2\2\u0115\u0116\7-\2\2\u0116=\3\2\2\2\u0117"+
-    "\u0118\7/\2\2\u0118?\3\2\2\2\u0119\u011a\7>\2\2\u011a\u011b\7>\2\2\u011b"+
-    "A\3\2\2\2\u011c\u011d\7@\2\2\u011d\u011e\7@\2\2\u011eC\3\2\2\2\u011f\u0120"+
-    "\7@\2\2\u0120\u0121\7@\2\2\u0121\u0122\7@\2\2\u0122E\3\2\2\2\u0123\u0124"+
-    "\7>\2\2\u0124G\3\2\2\2\u0125\u0126\7>\2\2\u0126\u0127\7?\2\2\u0127I\3"+
-    "\2\2\2\u0128\u0129\7@\2\2\u0129K\3\2\2\2\u012a\u012b\7@\2\2\u012b\u012c"+
-    "\7?\2\2\u012cM\3\2\2\2\u012d\u012e\7?\2\2\u012e\u012f\7?\2\2\u012fO\3"+
-    "\2\2\2\u0130\u0131\7?\2\2\u0131\u0132\7?\2\2\u0132\u0133\7?\2\2\u0133"+
-    "Q\3\2\2\2\u0134\u0135\7#\2\2\u0135\u0136\7?\2\2\u0136S\3\2\2\2\u0137\u0138"+
-    "\7#\2\2\u0138\u0139\7?\2\2\u0139\u013a\7?\2\2\u013aU\3\2\2\2\u013b\u013c"+
-    "\7(\2\2\u013cW\3\2\2\2\u013d\u013e\7`\2\2\u013eY\3\2\2\2\u013f\u0140\7"+
-    "~\2\2\u0140[\3\2\2\2\u0141\u0142\7(\2\2\u0142\u0143\7(\2\2\u0143]\3\2"+
-    "\2\2\u0144\u0145\7~\2\2\u0145\u0146\7~\2\2\u0146_\3\2\2\2\u0147\u0148"+
-    "\7A\2\2\u0148a\3\2\2\2\u0149\u014a\7<\2\2\u014ac\3\2\2\2\u014b\u014c\7"+
-    "-\2\2\u014c\u014d\7-\2\2\u014de\3\2\2\2\u014e\u014f\7/\2\2\u014f\u0150"+
-    "\7/\2\2\u0150g\3\2\2\2\u0151\u0152\7?\2\2\u0152i\3\2\2\2\u0153\u0154\7"+
-    "-\2\2\u0154\u0155\7?\2\2\u0155k\3\2\2\2\u0156\u0157\7/\2\2\u0157\u0158"+
-    "\7?\2\2\u0158m\3\2\2\2\u0159\u015a\7,\2\2\u015a\u015b\7?\2\2\u015bo\3"+
-    "\2\2\2\u015c\u015d\7\61\2\2\u015d\u015e\7?\2\2\u015eq\3\2\2\2\u015f\u0160"+
-    "\7\'\2\2\u0160\u0161\7?\2\2\u0161s\3\2\2\2\u0162\u0163\7(\2\2\u0163\u0164"+
-    "\7?\2\2\u0164u\3\2\2\2\u0165\u0166\7`\2\2\u0166\u0167\7?\2\2\u0167w\3"+
-    "\2\2\2\u0168\u0169\7~\2\2\u0169\u016a\7?\2\2\u016ay\3\2\2\2\u016b\u016c"+
-    "\7>\2\2\u016c\u016d\7>\2\2\u016d\u016e\7?\2\2\u016e{\3\2\2\2\u016f\u0170"+
-    "\7@\2\2\u0170\u0171\7@\2\2\u0171\u0172\7?\2\2\u0172}\3\2\2\2\u0173\u0174"+
-    "\7@\2\2\u0174\u0175\7@\2\2\u0175\u0176\7@\2\2\u0176\u0177\7?\2\2\u0177"+
-    "\177\3\2\2\2\u0178\u017a\7\62\2\2\u0179\u017b\t\4\2\2\u017a\u0179\3\2"+
-    "\2\2\u017b\u017c\3\2\2\2\u017c\u017a\3\2\2\2\u017c\u017d\3\2\2\2\u017d"+
-    "\u017f\3\2\2\2\u017e\u0180\t\5\2\2\u017f\u017e\3\2\2\2\u017f\u0180\3\2"+
-    "\2\2\u0180\u0081\3\2\2\2\u0181\u0182\7\62\2\2\u0182\u0184\t\6\2\2\u0183"+
-    "\u0185\t\7\2\2\u0184\u0183\3\2\2\2\u0185\u0186\3\2\2\2\u0186\u0184\3\2"+
-    "\2\2\u0186\u0187\3\2\2\2\u0187\u0189\3\2\2\2\u0188\u018a\t\5\2\2\u0189"+
-    "\u0188\3\2\2\2\u0189\u018a\3\2\2\2\u018a\u0083\3\2\2\2\u018b\u0194\7\62"+
-    "\2\2\u018c\u0190\t\b\2\2\u018d\u018f\t\t\2\2\u018e\u018d\3\2\2\2\u018f"+
-    "\u0192\3\2\2\2\u0190\u018e\3\2\2\2\u0190\u0191\3\2\2\2\u0191\u0194\3\2"+
-    "\2\2\u0192\u0190\3\2\2\2\u0193\u018b\3\2\2\2\u0193\u018c\3\2\2\2\u0194"+
-    "\u0196\3\2\2\2\u0195\u0197\t\n\2\2\u0196\u0195\3\2\2\2\u0196\u0197\3\2"+
-    "\2\2\u0197\u0085\3\2\2\2\u0198\u01a1\7\62\2\2\u0199\u019d\t\b\2\2\u019a"+
-    "\u019c\t\t\2\2\u019b\u019a\3\2\2\2\u019c\u019f\3\2\2\2\u019d\u019b\3\2"+
-    "\2\2\u019d\u019e\3\2\2\2\u019e\u01a1\3\2\2\2\u019f\u019d\3\2\2\2\u01a0"+
-    "\u0198\3\2\2\2\u01a0\u0199\3\2\2\2\u01a1\u01a2\3\2\2\2\u01a2\u01a6\5\24"+
-    "\n\2\u01a3\u01a5\t\t\2\2\u01a4\u01a3\3\2\2\2\u01a5\u01a8\3\2\2\2\u01a6"+
-    "\u01a4\3\2\2\2\u01a6\u01a7\3\2\2\2\u01a7\u01b2\3\2\2\2\u01a8\u01a6\3\2"+
-    "\2\2\u01a9\u01ab\t\13\2\2\u01aa\u01ac\t\f\2\2\u01ab\u01aa\3\2\2\2\u01ab"+
-    "\u01ac\3\2\2\2\u01ac\u01ae\3\2\2\2\u01ad\u01af\t\t\2\2\u01ae\u01ad\3\2"+
-    "\2\2\u01af\u01b0\3\2\2\2\u01b0\u01ae\3\2\2\2\u01b0\u01b1\3\2\2\2\u01b1"+
-    "\u01b3\3\2\2\2\u01b2\u01a9\3\2\2\2\u01b2\u01b3\3\2\2\2\u01b3\u01b5\3\2"+
-    "\2\2\u01b4\u01b6\t\r\2\2\u01b5\u01b4\3\2\2\2\u01b5\u01b6\3\2\2\2\u01b6"+
-    "\u0087\3\2\2\2\u01b7\u01bf\7$\2\2\u01b8\u01b9\7^\2\2\u01b9\u01be\7$\2"+
-    "\2\u01ba\u01bb\7^\2\2\u01bb\u01be\7^\2\2\u01bc\u01be\n\16\2\2\u01bd\u01b8"+
-    "\3\2\2\2\u01bd\u01ba\3\2\2\2\u01bd\u01bc\3\2\2\2\u01be\u01c1\3\2\2\2\u01bf"+
-    "\u01c0\3\2\2\2\u01bf\u01bd\3\2\2\2\u01c0\u01c2\3\2\2\2\u01c1\u01bf\3\2"+
-    "\2\2\u01c2\u01c3\7$\2\2\u01c3\u0089\3\2\2\2\u01c4\u01c5\7)\2\2\u01c5\u01c6"+
-    "\13\2\2\2\u01c6\u01c7\7)\2\2\u01c7\u008b\3\2\2\2\u01c8\u01c9\7v\2\2\u01c9"+
-    "\u01ca\7t\2\2\u01ca\u01cb\7w\2\2\u01cb\u01cc\7g\2\2\u01cc\u008d\3\2\2"+
-    "\2\u01cd\u01ce\7h\2\2\u01ce\u01cf\7c\2\2\u01cf\u01d0\7n\2\2\u01d0\u01d1"+
-    "\7u\2\2\u01d1\u01d2\7g\2\2\u01d2\u008f\3\2\2\2\u01d3\u01d4\7p\2\2\u01d4"+
-    "\u01d5\7w\2\2\u01d5\u01d6\7n\2\2\u01d6\u01d7\7n\2\2\u01d7\u0091\3\2\2"+
-    "\2\u01d8\u01dc\t\17\2\2\u01d9\u01db\t\20\2\2\u01da\u01d9\3\2\2\2\u01db"+
-    "\u01de\3\2\2\2\u01dc\u01da\3\2\2\2\u01dc\u01dd\3\2\2\2\u01dd\u0093\3\2"+
-    "\2\2\u01de\u01dc\3\2\2\2\u01df\u01e8\7\62\2\2\u01e0\u01e4\t\b\2\2\u01e1"+
-    "\u01e3\t\t\2\2\u01e2\u01e1\3\2\2\2\u01e3\u01e6\3\2\2\2\u01e4\u01e2\3\2"+
-    "\2\2\u01e4\u01e5\3\2\2\2\u01e5\u01e8\3\2\2\2\u01e6\u01e4\3\2\2\2\u01e7"+
-    "\u01df\3\2\2\2\u01e7\u01e0\3\2\2\2\u01e8\u01e9\3\2\2\2\u01e9\u01ea\bJ"+
-    "\4\2\u01ea\u0095\3\2\2\2\u01eb\u01ef\t\17\2\2\u01ec\u01ee\t\20\2\2\u01ed"+
-    "\u01ec\3\2\2\2\u01ee\u01f1\3\2\2\2\u01ef\u01ed\3\2\2\2\u01ef\u01f0\3\2"+
-    "\2\2\u01f0\u01f2\3\2\2\2\u01f1\u01ef\3\2\2\2\u01f2\u01f3\bK\4\2\u01f3"+
-    "\u0097\3\2\2\2\34\2\3\u009b\u00a5\u00af\u00b4\u017c\u017f\u0186\u0189"+
-    "\u0190\u0193\u0196\u019d\u01a0\u01a6\u01ab\u01b0\u01b2\u01b5\u01bd\u01bf"+
-    "\u01dc\u01e4\u01e7\u01ef\5\b\2\2\4\3\2\4\2\2";
+    "I\tI\4J\tJ\3\2\6\2\u0098\n\2\r\2\16\2\u0099\3\2\3\2\3\3\3\3\3\3\3\3\7"+
+    "\3\u00a2\n\3\f\3\16\3\u00a5\13\3\3\3\3\3\3\3\3\3\3\3\7\3\u00ac\n\3\f\3"+
+    "\16\3\u00af\13\3\3\3\3\3\5\3\u00b3\n\3\3\3\3\3\3\4\3\4\3\5\3\5\3\6\3\6"+
+    "\3\7\3\7\3\b\3\b\3\t\3\t\3\n\3\n\3\n\3\n\3\13\3\13\3\f\3\f\3\r\3\r\3\r"+
+    "\3\16\3\16\3\16\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3\17\3\20\3\20\3\20"+
+    "\3\21\3\21\3\21\3\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\23"+
+    "\3\23\3\23\3\23\3\23\3\23\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\25\3\25"+
+    "\3\25\3\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\27\3\27\3\27\3\30\3\30"+
+    "\3\30\3\30\3\30\3\30\3\31\3\31\3\32\3\32\3\33\3\33\3\34\3\34\3\35\3\35"+
+    "\3\36\3\36\3\37\3\37\3 \3 \3 \3!\3!\3!\3\"\3\"\3\"\3\"\3#\3#\3$\3$\3$"+
+    "\3%\3%\3&\3&\3&\3\'\3\'\3\'\3(\3(\3(\3(\3)\3)\3)\3*\3*\3*\3*\3+\3+\3,"+
+    "\3,\3-\3-\3.\3.\3.\3/\3/\3/\3\60\3\60\3\61\3\61\3\62\3\62\3\62\3\63\3"+
+    "\63\3\63\3\64\3\64\3\65\3\65\3\65\3\66\3\66\3\66\3\67\3\67\3\67\38\38"+
+    "\38\39\39\39\3:\3:\3:\3;\3;\3;\3<\3<\3<\3=\3=\3=\3=\3>\3>\3>\3>\3?\3?"+
+    "\3?\3?\3?\3@\3@\6@\u0179\n@\r@\16@\u017a\3@\5@\u017e\n@\3A\3A\3A\6A\u0183"+
+    "\nA\rA\16A\u0184\3A\5A\u0188\nA\3B\3B\3B\7B\u018d\nB\fB\16B\u0190\13B"+
+    "\5B\u0192\nB\3B\5B\u0195\nB\3C\3C\3C\7C\u019a\nC\fC\16C\u019d\13C\5C\u019f"+
+    "\nC\3C\3C\7C\u01a3\nC\fC\16C\u01a6\13C\3C\3C\5C\u01aa\nC\3C\6C\u01ad\n"+
+    "C\rC\16C\u01ae\5C\u01b1\nC\3C\5C\u01b4\nC\3D\3D\3D\3D\3D\3D\7D\u01bc\n"+
+    "D\fD\16D\u01bf\13D\3D\3D\3D\3D\3D\3D\3D\7D\u01c8\nD\fD\16D\u01cb\13D\3"+
+    "D\5D\u01ce\nD\3E\3E\3E\3E\3E\3F\3F\3F\3F\3F\3F\3G\3G\3G\3G\3G\3H\3H\7"+
+    "H\u01e2\nH\fH\16H\u01e5\13H\3I\3I\3I\7I\u01ea\nI\fI\16I\u01ed\13I\5I\u01ef"+
+    "\nI\3I\3I\3J\3J\7J\u01f5\nJ\fJ\16J\u01f8\13J\3J\3J\6\u00a3\u00ad\u01bd"+
+    "\u01c9\2K\4\3\6\4\b\5\n\6\f\7\16\b\20\t\22\n\24\13\26\f\30\r\32\16\34"+
+    "\17\36\20 \21\"\22$\23&\24(\25*\26,\27.\30\60\31\62\32\64\33\66\348\35"+
+    ":\36<\37> @!B\"D#F$H%J&L\'N(P)R*T+V,X-Z.\\/^\60`\61b\62d\63f\64h\65j\66"+
+    "l\67n8p9r:t;v<x=z>|?~@\u0080A\u0082B\u0084C\u0086D\u0088E\u008aF\u008c"+
+    "G\u008eH\u0090I\u0092J\u0094K\4\2\3\21\5\2\13\f\17\17\"\"\4\2\f\f\17\17"+
+    "\3\2\629\4\2NNnn\4\2ZZzz\5\2\62;CHch\3\2\63;\3\2\62;\b\2FFHHNNffhhnn\4"+
+    "\2GGgg\4\2--//\4\2HHhh\4\2$$^^\5\2C\\aac|\6\2\62;C\\aac|\u0216\2\4\3\2"+
+    "\2\2\2\6\3\2\2\2\2\b\3\2\2\2\2\n\3\2\2\2\2\f\3\2\2\2\2\16\3\2\2\2\2\20"+
+    "\3\2\2\2\2\22\3\2\2\2\2\24\3\2\2\2\2\26\3\2\2\2\2\30\3\2\2\2\2\32\3\2"+
+    "\2\2\2\34\3\2\2\2\2\36\3\2\2\2\2 \3\2\2\2\2\"\3\2\2\2\2$\3\2\2\2\2&\3"+
+    "\2\2\2\2(\3\2\2\2\2*\3\2\2\2\2,\3\2\2\2\2.\3\2\2\2\2\60\3\2\2\2\2\62\3"+
+    "\2\2\2\2\64\3\2\2\2\2\66\3\2\2\2\28\3\2\2\2\2:\3\2\2\2\2<\3\2\2\2\2>\3"+
+    "\2\2\2\2@\3\2\2\2\2B\3\2\2\2\2D\3\2\2\2\2F\3\2\2\2\2H\3\2\2\2\2J\3\2\2"+
+    "\2\2L\3\2\2\2\2N\3\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2T\3\2\2\2\2V\3\2\2\2\2"+
+    "X\3\2\2\2\2Z\3\2\2\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3\2\2\2\2b\3\2\2\2\2d\3"+
+    "\2\2\2\2f\3\2\2\2\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2\2\2n\3\2\2\2\2p\3\2\2"+
+    "\2\2r\3\2\2\2\2t\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2z\3\2\2\2\2|\3\2\2\2\2"+
+    "~\3\2\2\2\2\u0080\3\2\2\2\2\u0082\3\2\2\2\2\u0084\3\2\2\2\2\u0086\3\2"+
+    "\2\2\2\u0088\3\2\2\2\2\u008a\3\2\2\2\2\u008c\3\2\2\2\2\u008e\3\2\2\2\2"+
+    "\u0090\3\2\2\2\3\u0092\3\2\2\2\3\u0094\3\2\2\2\4\u0097\3\2\2\2\6\u00b2"+
+    "\3\2\2\2\b\u00b6\3\2\2\2\n\u00b8\3\2\2\2\f\u00ba\3\2\2\2\16\u00bc\3\2"+
+    "\2\2\20\u00be\3\2\2\2\22\u00c0\3\2\2\2\24\u00c2\3\2\2\2\26\u00c6\3\2\2"+
+    "\2\30\u00c8\3\2\2\2\32\u00ca\3\2\2\2\34\u00cd\3\2\2\2\36\u00d2\3\2\2\2"+
+    " \u00d8\3\2\2\2\"\u00db\3\2\2\2$\u00df\3\2\2\2&\u00e8\3\2\2\2(\u00ee\3"+
+    "\2\2\2*\u00f5\3\2\2\2,\u00f9\3\2\2\2.\u00fd\3\2\2\2\60\u0103\3\2\2\2\62"+
+    "\u0109\3\2\2\2\64\u010b\3\2\2\2\66\u010d\3\2\2\28\u010f\3\2\2\2:\u0111"+
+    "\3\2\2\2<\u0113\3\2\2\2>\u0115\3\2\2\2@\u0117\3\2\2\2B\u011a\3\2\2\2D"+
+    "\u011d\3\2\2\2F\u0121\3\2\2\2H\u0123\3\2\2\2J\u0126\3\2\2\2L\u0128\3\2"+
+    "\2\2N\u012b\3\2\2\2P\u012e\3\2\2\2R\u0132\3\2\2\2T\u0135\3\2\2\2V\u0139"+
+    "\3\2\2\2X\u013b\3\2\2\2Z\u013d\3\2\2\2\\\u013f\3\2\2\2^\u0142\3\2\2\2"+
+    "`\u0145\3\2\2\2b\u0147\3\2\2\2d\u0149\3\2\2\2f\u014c\3\2\2\2h\u014f\3"+
+    "\2\2\2j\u0151\3\2\2\2l\u0154\3\2\2\2n\u0157\3\2\2\2p\u015a\3\2\2\2r\u015d"+
+    "\3\2\2\2t\u0160\3\2\2\2v\u0163\3\2\2\2x\u0166\3\2\2\2z\u0169\3\2\2\2|"+
+    "\u016d\3\2\2\2~\u0171\3\2\2\2\u0080\u0176\3\2\2\2\u0082\u017f\3\2\2\2"+
+    "\u0084\u0191\3\2\2\2\u0086\u019e\3\2\2\2\u0088\u01cd\3\2\2\2\u008a\u01cf"+
+    "\3\2\2\2\u008c\u01d4\3\2\2\2\u008e\u01da\3\2\2\2\u0090\u01df\3\2\2\2\u0092"+
+    "\u01ee\3\2\2\2\u0094\u01f2\3\2\2\2\u0096\u0098\t\2\2\2\u0097\u0096\3\2"+
+    "\2\2\u0098\u0099\3\2\2\2\u0099\u0097\3\2\2\2\u0099\u009a\3\2\2\2\u009a"+
+    "\u009b\3\2\2\2\u009b\u009c\b\2\2\2\u009c\5\3\2\2\2\u009d\u009e\7\61\2"+
+    "\2\u009e\u009f\7\61\2\2\u009f\u00a3\3\2\2\2\u00a0\u00a2\13\2\2\2\u00a1"+
+    "\u00a0\3\2\2\2\u00a2\u00a5\3\2\2\2\u00a3\u00a4\3\2\2\2\u00a3\u00a1\3\2"+
+    "\2\2\u00a4\u00a6\3\2\2\2\u00a5\u00a3\3\2\2\2\u00a6\u00b3\t\3\2\2\u00a7"+
+    "\u00a8\7\61\2\2\u00a8\u00a9\7,\2\2\u00a9\u00ad\3\2\2\2\u00aa\u00ac\13"+
+    "\2\2\2\u00ab\u00aa\3\2\2\2\u00ac\u00af\3\2\2\2\u00ad\u00ae\3\2\2\2\u00ad"+
+    "\u00ab\3\2\2\2\u00ae\u00b0\3\2\2\2\u00af\u00ad\3\2\2\2\u00b0\u00b1\7,"+
+    "\2\2\u00b1\u00b3\7\61\2\2\u00b2\u009d\3\2\2\2\u00b2\u00a7\3\2\2\2\u00b3"+
+    "\u00b4\3\2\2\2\u00b4\u00b5\b\3\2\2\u00b5\7\3\2\2\2\u00b6\u00b7\7}\2\2"+
+    "\u00b7\t\3\2\2\2\u00b8\u00b9\7\177\2\2\u00b9\13\3\2\2\2\u00ba\u00bb\7"+
+    "]\2\2\u00bb\r\3\2\2\2\u00bc\u00bd\7_\2\2\u00bd\17\3\2\2\2\u00be\u00bf"+
+    "\7*\2\2\u00bf\21\3\2\2\2\u00c0\u00c1\7+\2\2\u00c1\23\3\2\2\2\u00c2\u00c3"+
+    "\7\60\2\2\u00c3\u00c4\3\2\2\2\u00c4\u00c5\b\n\3\2\u00c5\25\3\2\2\2\u00c6"+
+    "\u00c7\7.\2\2\u00c7\27\3\2\2\2\u00c8\u00c9\7=\2\2\u00c9\31\3\2\2\2\u00ca"+
+    "\u00cb\7k\2\2\u00cb\u00cc\7h\2\2\u00cc\33\3\2\2\2\u00cd\u00ce\7g\2\2\u00ce"+
+    "\u00cf\7n\2\2\u00cf\u00d0\7u\2\2\u00d0\u00d1\7g\2\2\u00d1\35\3\2\2\2\u00d2"+
+    "\u00d3\7y\2\2\u00d3\u00d4\7j\2\2\u00d4\u00d5\7k\2\2\u00d5\u00d6\7n\2\2"+
+    "\u00d6\u00d7\7g\2\2\u00d7\37\3\2\2\2\u00d8\u00d9\7f\2\2\u00d9\u00da\7"+
+    "q\2\2\u00da!\3\2\2\2\u00db\u00dc\7h\2\2\u00dc\u00dd\7q\2\2\u00dd\u00de"+
+    "\7t\2\2\u00de#\3\2\2\2\u00df\u00e0\7e\2\2\u00e0\u00e1\7q\2\2\u00e1\u00e2"+
+    "\7p\2\2\u00e2\u00e3\7v\2\2\u00e3\u00e4\7k\2\2\u00e4\u00e5\7p\2\2\u00e5"+
+    "\u00e6\7w\2\2\u00e6\u00e7\7g\2\2\u00e7%\3\2\2\2\u00e8\u00e9\7d\2\2\u00e9"+
+    "\u00ea\7t\2\2\u00ea\u00eb\7g\2\2\u00eb\u00ec\7c\2\2\u00ec\u00ed\7m\2\2"+
+    "\u00ed\'\3\2\2\2\u00ee\u00ef\7t\2\2\u00ef\u00f0\7g\2\2\u00f0\u00f1\7v"+
+    "\2\2\u00f1\u00f2\7w\2\2\u00f2\u00f3\7t\2\2\u00f3\u00f4\7p\2\2\u00f4)\3"+
+    "\2\2\2\u00f5\u00f6\7p\2\2\u00f6\u00f7\7g\2\2\u00f7\u00f8\7y\2\2\u00f8"+
+    "+\3\2\2\2\u00f9\u00fa\7v\2\2\u00fa\u00fb\7t\2\2\u00fb\u00fc\7{\2\2\u00fc"+
+    "-\3\2\2\2\u00fd\u00fe\7e\2\2\u00fe\u00ff\7c\2\2\u00ff\u0100\7v\2\2\u0100"+
+    "\u0101\7e\2\2\u0101\u0102\7j\2\2\u0102/\3\2\2\2\u0103\u0104\7v\2\2\u0104"+
+    "\u0105\7j\2\2\u0105\u0106\7t\2\2\u0106\u0107\7q\2\2\u0107\u0108\7y\2\2"+
+    "\u0108\61\3\2\2\2\u0109\u010a\7#\2\2\u010a\63\3\2\2\2\u010b\u010c\7\u0080"+
+    "\2\2\u010c\65\3\2\2\2\u010d\u010e\7,\2\2\u010e\67\3\2\2\2\u010f\u0110"+
+    "\7\61\2\2\u01109\3\2\2\2\u0111\u0112\7\'\2\2\u0112;\3\2\2\2\u0113\u0114"+
+    "\7-\2\2\u0114=\3\2\2\2\u0115\u0116\7/\2\2\u0116?\3\2\2\2\u0117\u0118\7"+
+    ">\2\2\u0118\u0119\7>\2\2\u0119A\3\2\2\2\u011a\u011b\7@\2\2\u011b\u011c"+
+    "\7@\2\2\u011cC\3\2\2\2\u011d\u011e\7@\2\2\u011e\u011f\7@\2\2\u011f\u0120"+
+    "\7@\2\2\u0120E\3\2\2\2\u0121\u0122\7>\2\2\u0122G\3\2\2\2\u0123\u0124\7"+
+    ">\2\2\u0124\u0125\7?\2\2\u0125I\3\2\2\2\u0126\u0127\7@\2\2\u0127K\3\2"+
+    "\2\2\u0128\u0129\7@\2\2\u0129\u012a\7?\2\2\u012aM\3\2\2\2\u012b\u012c"+
+    "\7?\2\2\u012c\u012d\7?\2\2\u012dO\3\2\2\2\u012e\u012f\7?\2\2\u012f\u0130"+
+    "\7?\2\2\u0130\u0131\7?\2\2\u0131Q\3\2\2\2\u0132\u0133\7#\2\2\u0133\u0134"+
+    "\7?\2\2\u0134S\3\2\2\2\u0135\u0136\7#\2\2\u0136\u0137\7?\2\2\u0137\u0138"+
+    "\7?\2\2\u0138U\3\2\2\2\u0139\u013a\7(\2\2\u013aW\3\2\2\2\u013b\u013c\7"+
+    "`\2\2\u013cY\3\2\2\2\u013d\u013e\7~\2\2\u013e[\3\2\2\2\u013f\u0140\7("+
+    "\2\2\u0140\u0141\7(\2\2\u0141]\3\2\2\2\u0142\u0143\7~\2\2\u0143\u0144"+
+    "\7~\2\2\u0144_\3\2\2\2\u0145\u0146\7A\2\2\u0146a\3\2\2\2\u0147\u0148\7"+
+    "<\2\2\u0148c\3\2\2\2\u0149\u014a\7-\2\2\u014a\u014b\7-\2\2\u014be\3\2"+
+    "\2\2\u014c\u014d\7/\2\2\u014d\u014e\7/\2\2\u014eg\3\2\2\2\u014f\u0150"+
+    "\7?\2\2\u0150i\3\2\2\2\u0151\u0152\7-\2\2\u0152\u0153\7?\2\2\u0153k\3"+
+    "\2\2\2\u0154\u0155\7/\2\2\u0155\u0156\7?\2\2\u0156m\3\2\2\2\u0157\u0158"+
+    "\7,\2\2\u0158\u0159\7?\2\2\u0159o\3\2\2\2\u015a\u015b\7\61\2\2\u015b\u015c"+
+    "\7?\2\2\u015cq\3\2\2\2\u015d\u015e\7\'\2\2\u015e\u015f\7?\2\2\u015fs\3"+
+    "\2\2\2\u0160\u0161\7(\2\2\u0161\u0162\7?\2\2\u0162u\3\2\2\2\u0163\u0164"+
+    "\7`\2\2\u0164\u0165\7?\2\2\u0165w\3\2\2\2\u0166\u0167\7~\2\2\u0167\u0168"+
+    "\7?\2\2\u0168y\3\2\2\2\u0169\u016a\7>\2\2\u016a\u016b\7>\2\2\u016b\u016c"+
+    "\7?\2\2\u016c{\3\2\2\2\u016d\u016e\7@\2\2\u016e\u016f\7@\2\2\u016f\u0170"+
+    "\7?\2\2\u0170}\3\2\2\2\u0171\u0172\7@\2\2\u0172\u0173\7@\2\2\u0173\u0174"+
+    "\7@\2\2\u0174\u0175\7?\2\2\u0175\177\3\2\2\2\u0176\u0178\7\62\2\2\u0177"+
+    "\u0179\t\4\2\2\u0178\u0177\3\2\2\2\u0179\u017a\3\2\2\2\u017a\u0178\3\2"+
+    "\2\2\u017a\u017b\3\2\2\2\u017b\u017d\3\2\2\2\u017c\u017e\t\5\2\2\u017d"+
+    "\u017c\3\2\2\2\u017d\u017e\3\2\2\2\u017e\u0081\3\2\2\2\u017f\u0180\7\62"+
+    "\2\2\u0180\u0182\t\6\2\2\u0181\u0183\t\7\2\2\u0182\u0181\3\2\2\2\u0183"+
+    "\u0184\3\2\2\2\u0184\u0182\3\2\2\2\u0184\u0185\3\2\2\2\u0185\u0187\3\2"+
+    "\2\2\u0186\u0188\t\5\2\2\u0187\u0186\3\2\2\2\u0187\u0188\3\2\2\2\u0188"+
+    "\u0083\3\2\2\2\u0189\u0192\7\62\2\2\u018a\u018e\t\b\2\2\u018b\u018d\t"+
+    "\t\2\2\u018c\u018b\3\2\2\2\u018d\u0190\3\2\2\2\u018e\u018c\3\2\2\2\u018e"+
+    "\u018f\3\2\2\2\u018f\u0192\3\2\2\2\u0190\u018e\3\2\2\2\u0191\u0189\3\2"+
+    "\2\2\u0191\u018a\3\2\2\2\u0192\u0194\3\2\2\2\u0193\u0195\t\n\2\2\u0194"+
+    "\u0193\3\2\2\2\u0194\u0195\3\2\2\2\u0195\u0085\3\2\2\2\u0196\u019f\7\62"+
+    "\2\2\u0197\u019b\t\b\2\2\u0198\u019a\t\t\2\2\u0199\u0198\3\2\2\2\u019a"+
+    "\u019d\3\2\2\2\u019b\u0199\3\2\2\2\u019b\u019c\3\2\2\2\u019c\u019f\3\2"+
+    "\2\2\u019d\u019b\3\2\2\2\u019e\u0196\3\2\2\2\u019e\u0197\3\2\2\2\u019f"+
+    "\u01a0\3\2\2\2\u01a0\u01a4\5\24\n\2\u01a1\u01a3\t\t\2\2\u01a2\u01a1\3"+
+    "\2\2\2\u01a3\u01a6\3\2\2\2\u01a4\u01a2\3\2\2\2\u01a4\u01a5\3\2\2\2\u01a5"+
+    "\u01b0\3\2\2\2\u01a6\u01a4\3\2\2\2\u01a7\u01a9\t\13\2\2\u01a8\u01aa\t"+
+    "\f\2\2\u01a9\u01a8\3\2\2\2\u01a9\u01aa\3\2\2\2\u01aa\u01ac\3\2\2\2\u01ab"+
+    "\u01ad\t\t\2\2\u01ac\u01ab\3\2\2\2\u01ad\u01ae\3\2\2\2\u01ae\u01ac\3\2"+
+    "\2\2\u01ae\u01af\3\2\2\2\u01af\u01b1\3\2\2\2\u01b0\u01a7\3\2\2\2\u01b0"+
+    "\u01b1\3\2\2\2\u01b1\u01b3\3\2\2\2\u01b2\u01b4\t\r\2\2\u01b3\u01b2\3\2"+
+    "\2\2\u01b3\u01b4\3\2\2\2\u01b4\u0087\3\2\2\2\u01b5\u01bd\7$\2\2\u01b6"+
+    "\u01b7\7^\2\2\u01b7\u01bc\7$\2\2\u01b8\u01b9\7^\2\2\u01b9\u01bc\7^\2\2"+
+    "\u01ba\u01bc\n\16\2\2\u01bb\u01b6\3\2\2\2\u01bb\u01b8\3\2\2\2\u01bb\u01ba"+
+    "\3\2\2\2\u01bc\u01bf\3\2\2\2\u01bd\u01be\3\2\2\2\u01bd\u01bb\3\2\2\2\u01be"+
+    "\u01c0\3\2\2\2\u01bf\u01bd\3\2\2\2\u01c0\u01ce\7$\2\2\u01c1\u01c9\7)\2"+
+    "\2\u01c2\u01c3\7^\2\2\u01c3\u01c8\7)\2\2\u01c4\u01c5\7^\2\2\u01c5\u01c8"+
+    "\7^\2\2\u01c6\u01c8\n\16\2\2\u01c7\u01c2\3\2\2\2\u01c7\u01c4\3\2\2\2\u01c7"+
+    "\u01c6\3\2\2\2\u01c8\u01cb\3\2\2\2\u01c9\u01ca\3\2\2\2\u01c9\u01c7\3\2"+
+    "\2\2\u01ca\u01cc\3\2\2\2\u01cb\u01c9\3\2\2\2\u01cc\u01ce\7)\2\2\u01cd"+
+    "\u01b5\3\2\2\2\u01cd\u01c1\3\2\2\2\u01ce\u0089\3\2\2\2\u01cf\u01d0\7v"+
+    "\2\2\u01d0\u01d1\7t\2\2\u01d1\u01d2\7w\2\2\u01d2\u01d3\7g\2\2\u01d3\u008b"+
+    "\3\2\2\2\u01d4\u01d5\7h\2\2\u01d5\u01d6\7c\2\2\u01d6\u01d7\7n\2\2\u01d7"+
+    "\u01d8\7u\2\2\u01d8\u01d9\7g\2\2\u01d9\u008d\3\2\2\2\u01da\u01db\7p\2"+
+    "\2\u01db\u01dc\7w\2\2\u01dc\u01dd\7n\2\2\u01dd\u01de\7n\2\2\u01de\u008f"+
+    "\3\2\2\2\u01df\u01e3\t\17\2\2\u01e0\u01e2\t\20\2\2\u01e1\u01e0\3\2\2\2"+
+    "\u01e2\u01e5\3\2\2\2\u01e3\u01e1\3\2\2\2\u01e3\u01e4\3\2\2\2\u01e4\u0091"+
+    "\3\2\2\2\u01e5\u01e3\3\2\2\2\u01e6\u01ef\7\62\2\2\u01e7\u01eb\t\b\2\2"+
+    "\u01e8\u01ea\t\t\2\2\u01e9\u01e8\3\2\2\2\u01ea\u01ed\3\2\2\2\u01eb\u01e9"+
+    "\3\2\2\2\u01eb\u01ec\3\2\2\2\u01ec\u01ef\3\2\2\2\u01ed\u01eb\3\2\2\2\u01ee"+
+    "\u01e6\3\2\2\2\u01ee\u01e7\3\2\2\2\u01ef\u01f0\3\2\2\2\u01f0\u01f1\bI"+
+    "\4\2\u01f1\u0093\3\2\2\2\u01f2\u01f6\t\17\2\2\u01f3\u01f5\t\20\2\2\u01f4"+
+    "\u01f3\3\2\2\2\u01f5\u01f8\3\2\2\2\u01f6\u01f4\3\2\2\2\u01f6\u01f7\3\2"+
+    "\2\2\u01f7\u01f9\3\2\2\2\u01f8\u01f6\3\2\2\2\u01f9\u01fa\bJ\4\2\u01fa"+
+    "\u0095\3\2\2\2\37\2\3\u0099\u00a3\u00ad\u00b2\u017a\u017d\u0184\u0187"+
+    "\u018e\u0191\u0194\u019b\u019e\u01a4\u01a9\u01ae\u01b0\u01b3\u01bb\u01bd"+
+    "\u01c7\u01c9\u01cd\u01e3\u01eb\u01ee\u01f6\5\b\2\2\4\3\2\4\2\2";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessParser.java
index c4499fca638..b92c9d75198 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessParser.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessParser.java
@@ -25,7 +25,7 @@ class PainlessParser extends Parser {
     BWOR=44, BOOLAND=45, BOOLOR=46, COND=47, COLON=48, INCR=49, DECR=50, ASSIGN=51, 
     AADD=52, ASUB=53, AMUL=54, ADIV=55, AREM=56, AAND=57, AXOR=58, AOR=59, 
     ALSH=60, ARSH=61, AUSH=62, OCTAL=63, HEX=64, INTEGER=65, DECIMAL=66, STRING=67, 
-    CHAR=68, TRUE=69, FALSE=70, NULL=71, ID=72, EXTINTEGER=73, EXTID=74;
+    TRUE=68, FALSE=69, NULL=70, ID=71, EXTINTEGER=72, EXTID=73;
   public static final int
     RULE_source = 0, RULE_statement = 1, RULE_block = 2, RULE_empty = 3, RULE_emptyscope = 4, 
     RULE_initializer = 5, RULE_afterthought = 6, RULE_declaration = 7, RULE_decltype = 8, 
@@ -50,7 +50,7 @@ class PainlessParser extends Parser {
     "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", "'^'", "'|'", "'&&'", 
     "'||'", "'?'", "':'", "'++'", "'--'", "'='", "'+='", "'-='", "'*='", "'/='", 
     "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", "'>>>='", null, null, 
-    null, null, null, null, "'true'", "'false'", "'null'"
+    null, null, null, "'true'", "'false'", "'null'"
   };
   private static final String[] _SYMBOLIC_NAMES = {
     null, "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", 
@@ -60,8 +60,8 @@ class PainlessParser extends Parser {
     "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "BWXOR", "BWOR", "BOOLAND", 
     "BOOLOR", "COND", "COLON", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", 
     "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", 
-    "HEX", "INTEGER", "DECIMAL", "STRING", "CHAR", "TRUE", "FALSE", "NULL", 
-    "ID", "EXTINTEGER", "EXTID"
+    "HEX", "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "ID", 
+    "EXTINTEGER", "EXTID"
   };
   public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
 
@@ -151,7 +151,7 @@ class PainlessParser extends Parser {
         setState(55); 
         _errHandler.sync(this);
         _la = _input.LA(1);
-      } while ( (((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << IF) | (1L << WHILE) | (1L << DO) | (1L << FOR) | (1L << CONTINUE) | (1L << BREAK) | (1L << RETURN) | (1L << NEW) | (1L << TRY) | (1L << THROW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (CHAR - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0) );
+      } while ( (((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << IF) | (1L << WHILE) | (1L << DO) | (1L << FOR) | (1L << CONTINUE) | (1L << BREAK) | (1L << RETURN) | (1L << NEW) | (1L << TRY) | (1L << THROW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0) );
       setState(57);
       match(EOF);
       }
@@ -458,7 +458,7 @@ class PainlessParser extends Parser {
         match(LP);
         setState(88);
         _la = _input.LA(1);
-        if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (CHAR - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0)) {
+        if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0)) {
           {
           setState(87);
           initializer();
@@ -469,7 +469,7 @@ class PainlessParser extends Parser {
         match(SEMICOLON);
         setState(92);
         _la = _input.LA(1);
-        if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (CHAR - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0)) {
+        if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0)) {
           {
           setState(91);
           expression(0);
@@ -480,7 +480,7 @@ class PainlessParser extends Parser {
         match(SEMICOLON);
         setState(96);
         _la = _input.LA(1);
-        if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (CHAR - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0)) {
+        if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0)) {
           {
           setState(95);
           afterthought();
@@ -720,7 +720,7 @@ class PainlessParser extends Parser {
           setState(142); 
           _errHandler.sync(this);
           _la = _input.LA(1);
-        } while ( (((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << IF) | (1L << WHILE) | (1L << DO) | (1L << FOR) | (1L << CONTINUE) | (1L << BREAK) | (1L << RETURN) | (1L << NEW) | (1L << TRY) | (1L << THROW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (CHAR - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0) );
+        } while ( (((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << IF) | (1L << WHILE) | (1L << DO) | (1L << FOR) | (1L << CONTINUE) | (1L << BREAK) | (1L << RETURN) | (1L << NEW) | (1L << TRY) | (1L << THROW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0) );
         setState(144);
         match(RBRACK);
         }
@@ -747,7 +747,6 @@ class PainlessParser extends Parser {
       case INTEGER:
       case DECIMAL:
       case STRING:
-      case CHAR:
       case TRUE:
       case FALSE:
       case NULL:
@@ -1541,15 +1540,6 @@ class PainlessParser extends Parser {
       else return visitor.visitChildren(this);
     }
   }
-  public static class CharContext extends ExpressionContext {
-    public TerminalNode CHAR() { return getToken(PainlessParser.CHAR, 0); }
-    public CharContext(ExpressionContext ctx) { copyFrom(ctx); }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitChar(this);
-      else return visitor.visitChildren(this);
-    }
-  }
   public static class TrueContext extends ExpressionContext {
     public TerminalNode TRUE() { return getToken(PainlessParser.TRUE, 0); }
     public TrueContext(ExpressionContext ctx) { copyFrom(ctx); }
@@ -1576,7 +1566,7 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(237);
+      setState(236);
       switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) {
       case 1:
         {
@@ -1657,74 +1647,65 @@ class PainlessParser extends Parser {
         break;
       case 6:
         {
-        _localctx = new CharContext(_localctx);
+        _localctx = new TrueContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
         setState(226);
-        match(CHAR);
+        match(TRUE);
         }
         break;
       case 7:
         {
-        _localctx = new TrueContext(_localctx);
+        _localctx = new FalseContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
         setState(227);
-        match(TRUE);
+        match(FALSE);
         }
         break;
       case 8:
         {
-        _localctx = new FalseContext(_localctx);
+        _localctx = new NullContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
         setState(228);
-        match(FALSE);
+        match(NULL);
         }
         break;
       case 9:
         {
-        _localctx = new NullContext(_localctx);
+        _localctx = new PostincContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
         setState(229);
-        match(NULL);
+        extstart();
+        setState(230);
+        increment();
         }
         break;
       case 10:
         {
-        _localctx = new PostincContext(_localctx);
+        _localctx = new PreincContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(230);
-        extstart();
-        setState(231);
+        setState(232);
         increment();
+        setState(233);
+        extstart();
         }
         break;
       case 11:
-        {
-        _localctx = new PreincContext(_localctx);
-        _ctx = _localctx;
-        _prevctx = _localctx;
-        setState(233);
-        increment();
-        setState(234);
-        extstart();
-        }
-        break;
-      case 12:
         {
         _localctx = new ExternalContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(236);
+        setState(235);
         extstart();
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(277);
+      setState(276);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,28,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -1732,22 +1713,22 @@ class PainlessParser extends Parser {
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(275);
+          setState(274);
           switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) {
           case 1:
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(239);
+            setState(238);
             if (!(precpred(_ctx, 12))) throw new FailedPredicateException(this, "precpred(_ctx, 12)");
-            setState(240);
+            setState(239);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << MUL) | (1L << DIV) | (1L << REM))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(241);
+            setState(240);
             expression(13);
             }
             break;
@@ -1755,16 +1736,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(242);
+            setState(241);
             if (!(precpred(_ctx, 11))) throw new FailedPredicateException(this, "precpred(_ctx, 11)");
-            setState(243);
+            setState(242);
             _la = _input.LA(1);
             if ( !(_la==ADD || _la==SUB) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(244);
+            setState(243);
             expression(12);
             }
             break;
@@ -1772,16 +1753,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(245);
+            setState(244);
             if (!(precpred(_ctx, 10))) throw new FailedPredicateException(this, "precpred(_ctx, 10)");
-            setState(246);
+            setState(245);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LSH) | (1L << RSH) | (1L << USH))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(247);
+            setState(246);
             expression(11);
             }
             break;
@@ -1789,16 +1770,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(248);
+            setState(247);
             if (!(precpred(_ctx, 9))) throw new FailedPredicateException(this, "precpred(_ctx, 9)");
-            setState(249);
+            setState(248);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LT) | (1L << LTE) | (1L << GT) | (1L << GTE))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(250);
+            setState(249);
             expression(10);
             }
             break;
@@ -1806,16 +1787,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(251);
+            setState(250);
             if (!(precpred(_ctx, 8))) throw new FailedPredicateException(this, "precpred(_ctx, 8)");
-            setState(252);
+            setState(251);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << EQ) | (1L << EQR) | (1L << NE) | (1L << NER))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(253);
+            setState(252);
             expression(9);
             }
             break;
@@ -1823,11 +1804,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(254);
+            setState(253);
             if (!(precpred(_ctx, 7))) throw new FailedPredicateException(this, "precpred(_ctx, 7)");
-            setState(255);
+            setState(254);
             match(BWAND);
-            setState(256);
+            setState(255);
             expression(8);
             }
             break;
@@ -1835,11 +1816,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(257);
+            setState(256);
             if (!(precpred(_ctx, 6))) throw new FailedPredicateException(this, "precpred(_ctx, 6)");
-            setState(258);
+            setState(257);
             match(BWXOR);
-            setState(259);
+            setState(258);
             expression(7);
             }
             break;
@@ -1847,11 +1828,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(260);
+            setState(259);
             if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)");
-            setState(261);
+            setState(260);
             match(BWOR);
-            setState(262);
+            setState(261);
             expression(6);
             }
             break;
@@ -1859,11 +1840,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(263);
+            setState(262);
             if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)");
-            setState(264);
+            setState(263);
             match(BOOLAND);
-            setState(265);
+            setState(264);
             expression(5);
             }
             break;
@@ -1871,11 +1852,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(266);
+            setState(265);
             if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)");
-            setState(267);
+            setState(266);
             match(BOOLOR);
-            setState(268);
+            setState(267);
             expression(4);
             }
             break;
@@ -1883,22 +1864,22 @@ class PainlessParser extends Parser {
             {
             _localctx = new ConditionalContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(269);
+            setState(268);
             if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
-            setState(270);
+            setState(269);
             match(COND);
-            setState(271);
+            setState(270);
             expression(0);
-            setState(272);
+            setState(271);
             match(COLON);
-            setState(273);
+            setState(272);
             expression(2);
             }
             break;
           }
           } 
         }
-        setState(279);
+        setState(278);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,28,_ctx);
       }
@@ -1946,40 +1927,40 @@ class PainlessParser extends Parser {
     ExtstartContext _localctx = new ExtstartContext(_ctx, getState());
     enterRule(_localctx, 28, RULE_extstart);
     try {
-      setState(285);
+      setState(284);
       switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(280);
+        setState(279);
         extprec();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(281);
+        setState(280);
         extcast();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(282);
+        setState(281);
         extvar();
         }
         break;
       case 4:
         enterOuterAlt(_localctx, 4);
         {
-        setState(283);
+        setState(282);
         extnew();
         }
         break;
       case 5:
         enterOuterAlt(_localctx, 5);
         {
-        setState(284);
+        setState(283);
         extstring();
         }
         break;
@@ -2037,54 +2018,54 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(287);
+      setState(286);
       match(LP);
-      setState(293);
+      setState(292);
       switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
       case 1:
         {
-        setState(288);
+        setState(287);
         extprec();
         }
         break;
       case 2:
         {
-        setState(289);
+        setState(288);
         extcast();
         }
         break;
       case 3:
         {
-        setState(290);
+        setState(289);
         extvar();
         }
         break;
       case 4:
         {
-        setState(291);
+        setState(290);
         extnew();
         }
         break;
       case 5:
         {
-        setState(292);
+        setState(291);
         extstring();
         }
         break;
       }
-      setState(295);
+      setState(294);
       match(RP);
-      setState(298);
+      setState(297);
       switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) {
       case 1:
         {
-        setState(296);
+        setState(295);
         extdot();
         }
         break;
       case 2:
         {
-        setState(297);
+        setState(296);
         extbrace();
         }
         break;
@@ -2140,41 +2121,41 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(300);
+      setState(299);
       match(LP);
-      setState(301);
+      setState(300);
       decltype();
-      setState(302);
+      setState(301);
       match(RP);
-      setState(308);
+      setState(307);
       switch ( getInterpreter().adaptivePredict(_input,32,_ctx) ) {
       case 1:
         {
-        setState(303);
+        setState(302);
         extprec();
         }
         break;
       case 2:
         {
-        setState(304);
+        setState(303);
         extcast();
         }
         break;
       case 3:
         {
-        setState(305);
+        setState(304);
         extvar();
         }
         break;
       case 4:
         {
-        setState(306);
+        setState(305);
         extnew();
         }
         break;
       case 5:
         {
-        setState(307);
+        setState(306);
         extstring();
         }
         break;
@@ -2221,23 +2202,23 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(310);
+      setState(309);
       match(LBRACE);
-      setState(311);
+      setState(310);
       expression(0);
-      setState(312);
+      setState(311);
       match(RBRACE);
-      setState(315);
+      setState(314);
       switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) {
       case 1:
         {
-        setState(313);
+        setState(312);
         extdot();
         }
         break;
       case 2:
         {
-        setState(314);
+        setState(313);
         extbrace();
         }
         break;
@@ -2280,19 +2261,19 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(317);
+      setState(316);
       match(DOT);
-      setState(320);
+      setState(319);
       switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) {
       case 1:
         {
-        setState(318);
+        setState(317);
         extcall();
         }
         break;
       case 2:
         {
-        setState(319);
+        setState(318);
         extfield();
         }
         break;
@@ -2338,21 +2319,21 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(322);
+      setState(321);
       match(EXTID);
-      setState(323);
+      setState(322);
       arguments();
-      setState(326);
+      setState(325);
       switch ( getInterpreter().adaptivePredict(_input,35,_ctx) ) {
       case 1:
         {
-        setState(324);
+        setState(323);
         extdot();
         }
         break;
       case 2:
         {
-        setState(325);
+        setState(324);
         extbrace();
         }
         break;
@@ -2397,19 +2378,19 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(328);
+      setState(327);
       identifier();
-      setState(331);
+      setState(330);
       switch ( getInterpreter().adaptivePredict(_input,36,_ctx) ) {
       case 1:
         {
-        setState(329);
+        setState(328);
         extdot();
         }
         break;
       case 2:
         {
-        setState(330);
+        setState(329);
         extbrace();
         }
         break;
@@ -2454,24 +2435,24 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(333);
+      setState(332);
       _la = _input.LA(1);
       if ( !(_la==EXTINTEGER || _la==EXTID) ) {
       _errHandler.recoverInline(this);
       } else {
         consume();
       }
-      setState(336);
+      setState(335);
       switch ( getInterpreter().adaptivePredict(_input,37,_ctx) ) {
       case 1:
         {
-        setState(334);
+        setState(333);
         extdot();
         }
         break;
       case 2:
         {
-        setState(335);
+        setState(334);
         extbrace();
         }
         break;
@@ -2532,22 +2513,22 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(338);
+      setState(337);
       match(NEW);
-      setState(339);
+      setState(338);
       identifier();
-      setState(355);
+      setState(354);
       switch (_input.LA(1)) {
       case LP:
         {
         {
-        setState(340);
+        setState(339);
         arguments();
-        setState(342);
+        setState(341);
         switch ( getInterpreter().adaptivePredict(_input,38,_ctx) ) {
         case 1:
           {
-          setState(341);
+          setState(340);
           extdot();
           }
           break;
@@ -2558,7 +2539,7 @@ class PainlessParser extends Parser {
       case LBRACE:
         {
         {
-        setState(348); 
+        setState(347); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -2566,11 +2547,11 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(344);
+            setState(343);
             match(LBRACE);
-            setState(345);
+            setState(344);
             expression(0);
-            setState(346);
+            setState(345);
             match(RBRACE);
             }
             }
@@ -2578,15 +2559,15 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(350); 
+          setState(349); 
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,39,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
-        setState(353);
+        setState(352);
         switch ( getInterpreter().adaptivePredict(_input,40,_ctx) ) {
         case 1:
           {
-          setState(352);
+          setState(351);
           extdot();
           }
           break;
@@ -2635,19 +2616,19 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(357);
+      setState(356);
       match(STRING);
-      setState(360);
+      setState(359);
       switch ( getInterpreter().adaptivePredict(_input,42,_ctx) ) {
       case 1:
         {
-        setState(358);
+        setState(357);
         extdot();
         }
         break;
       case 2:
         {
-        setState(359);
+        setState(358);
         extbrace();
         }
         break;
@@ -2697,34 +2678,34 @@ class PainlessParser extends Parser {
       enterOuterAlt(_localctx, 1);
       {
       {
-      setState(362);
+      setState(361);
       match(LP);
-      setState(371);
+      setState(370);
       _la = _input.LA(1);
-      if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (CHAR - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0)) {
+      if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0)) {
         {
-        setState(363);
+        setState(362);
         expression(0);
-        setState(368);
+        setState(367);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(364);
+          setState(363);
           match(COMMA);
-          setState(365);
+          setState(364);
           expression(0);
           }
           }
-          setState(370);
+          setState(369);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
       }
 
-      setState(373);
+      setState(372);
       match(RP);
       }
       }
@@ -2761,7 +2742,7 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(375);
+      setState(374);
       _la = _input.LA(1);
       if ( !(_la==INCR || _la==DECR) ) {
       _errHandler.recoverInline(this);
@@ -2817,7 +2798,7 @@ class PainlessParser extends Parser {
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3L\u017c\4\2\t\2\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3K\u017b\4\2\t\2\4"+
     "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
     "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
     "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+
@@ -2834,40 +2815,40 @@ class PainlessParser extends Parser {
     "\f\3\r\3\r\5\r\u00c7\n\r\3\16\3\16\3\16\3\16\7\16\u00cd\n\16\f\16\16\16"+
     "\u00d0\13\16\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3"+
     "\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3"+
-    "\17\3\17\3\17\3\17\3\17\5\17\u00f0\n\17\3\17\3\17\3\17\3\17\3\17\3\17"+
+    "\17\3\17\3\17\3\17\5\17\u00ef\n\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
     "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
     "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
-    "\3\17\3\17\7\17\u0116\n\17\f\17\16\17\u0119\13\17\3\20\3\20\3\20\3\20"+
-    "\3\20\5\20\u0120\n\20\3\21\3\21\3\21\3\21\3\21\3\21\5\21\u0128\n\21\3"+
-    "\21\3\21\3\21\5\21\u012d\n\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22"+
-    "\5\22\u0137\n\22\3\23\3\23\3\23\3\23\3\23\5\23\u013e\n\23\3\24\3\24\3"+
-    "\24\5\24\u0143\n\24\3\25\3\25\3\25\3\25\5\25\u0149\n\25\3\26\3\26\3\26"+
-    "\5\26\u014e\n\26\3\27\3\27\3\27\5\27\u0153\n\27\3\30\3\30\3\30\3\30\5"+
-    "\30\u0159\n\30\3\30\3\30\3\30\3\30\6\30\u015f\n\30\r\30\16\30\u0160\3"+
-    "\30\5\30\u0164\n\30\5\30\u0166\n\30\3\31\3\31\3\31\5\31\u016b\n\31\3\32"+
-    "\3\32\3\32\3\32\7\32\u0171\n\32\f\32\16\32\u0174\13\32\5\32\u0176\n\32"+
-    "\3\32\3\32\3\33\3\33\3\33\2\3\34\34\2\4\6\b\n\f\16\20\22\24\26\30\32\34"+
-    "\36 \"$&(*,.\60\62\64\2\f\4\2\32\33\37 \3\2\65@\3\2AD\3\2\34\36\3\2\37"+
-    " \3\2!#\3\2$\'\3\2(+\3\2KL\3\2\63\64\u01b9\2\67\3\2\2\2\4\u008a\3\2\2"+
-    "\2\6\u0095\3\2\2\2\b\u0099\3\2\2\2\n\u009b\3\2\2\2\f\u00a0\3\2\2\2\16"+
-    "\u00a2\3\2\2\2\20\u00a4\3\2\2\2\22\u00ad\3\2\2\2\24\u00b5\3\2\2\2\26\u00ba"+
-    "\3\2\2\2\30\u00c4\3\2\2\2\32\u00c8\3\2\2\2\34\u00ef\3\2\2\2\36\u011f\3"+
-    "\2\2\2 \u0121\3\2\2\2\"\u012e\3\2\2\2$\u0138\3\2\2\2&\u013f\3\2\2\2(\u0144"+
-    "\3\2\2\2*\u014a\3\2\2\2,\u014f\3\2\2\2.\u0154\3\2\2\2\60\u0167\3\2\2\2"+
-    "\62\u016c\3\2\2\2\64\u0179\3\2\2\2\668\5\4\3\2\67\66\3\2\2\289\3\2\2\2"+
-    "9\67\3\2\2\29:\3\2\2\2:;\3\2\2\2;<\7\2\2\3<\3\3\2\2\2=>\7\16\2\2>?\7\t"+
-    "\2\2?@\5\34\17\2@A\7\n\2\2AD\5\6\4\2BC\7\17\2\2CE\5\6\4\2DB\3\2\2\2DE"+
-    "\3\2\2\2E\u008b\3\2\2\2FG\7\20\2\2GH\7\t\2\2HI\5\34\17\2IL\7\n\2\2JM\5"+
-    "\6\4\2KM\5\b\5\2LJ\3\2\2\2LK\3\2\2\2M\u008b\3\2\2\2NO\7\21\2\2OP\5\6\4"+
-    "\2PQ\7\20\2\2QR\7\t\2\2RS\5\34\17\2SU\7\n\2\2TV\7\r\2\2UT\3\2\2\2UV\3"+
-    "\2\2\2V\u008b\3\2\2\2WX\7\22\2\2XZ\7\t\2\2Y[\5\f\7\2ZY\3\2\2\2Z[\3\2\2"+
-    "\2[\\\3\2\2\2\\^\7\r\2\2]_\5\34\17\2^]\3\2\2\2^_\3\2\2\2_`\3\2\2\2`b\7"+
-    "\r\2\2ac\5\16\b\2ba\3\2\2\2bc\3\2\2\2cd\3\2\2\2dg\7\n\2\2eh\5\6\4\2fh"+
-    "\5\b\5\2ge\3\2\2\2gf\3\2\2\2h\u008b\3\2\2\2ik\5\20\t\2jl\7\r\2\2kj\3\2"+
-    "\2\2kl\3\2\2\2l\u008b\3\2\2\2mo\7\23\2\2np\7\r\2\2on\3\2\2\2op\3\2\2\2"+
-    "p\u008b\3\2\2\2qs\7\24\2\2rt\7\r\2\2sr\3\2\2\2st\3\2\2\2t\u008b\3\2\2"+
-    "\2uv\7\25\2\2vx\5\34\17\2wy\7\r\2\2xw\3\2\2\2xy\3\2\2\2y\u008b\3\2\2\2"+
-    "z{\7\27\2\2{}\5\6\4\2|~\5\26\f\2}|\3\2\2\2~\177\3\2\2\2\177}\3\2\2\2\177"+
+    "\3\17\7\17\u0115\n\17\f\17\16\17\u0118\13\17\3\20\3\20\3\20\3\20\3\20"+
+    "\5\20\u011f\n\20\3\21\3\21\3\21\3\21\3\21\3\21\5\21\u0127\n\21\3\21\3"+
+    "\21\3\21\5\21\u012c\n\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\5\22"+
+    "\u0136\n\22\3\23\3\23\3\23\3\23\3\23\5\23\u013d\n\23\3\24\3\24\3\24\5"+
+    "\24\u0142\n\24\3\25\3\25\3\25\3\25\5\25\u0148\n\25\3\26\3\26\3\26\5\26"+
+    "\u014d\n\26\3\27\3\27\3\27\5\27\u0152\n\27\3\30\3\30\3\30\3\30\5\30\u0158"+
+    "\n\30\3\30\3\30\3\30\3\30\6\30\u015e\n\30\r\30\16\30\u015f\3\30\5\30\u0163"+
+    "\n\30\5\30\u0165\n\30\3\31\3\31\3\31\5\31\u016a\n\31\3\32\3\32\3\32\3"+
+    "\32\7\32\u0170\n\32\f\32\16\32\u0173\13\32\5\32\u0175\n\32\3\32\3\32\3"+
+    "\33\3\33\3\33\2\3\34\34\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*"+
+    ",.\60\62\64\2\f\4\2\32\33\37 \3\2\65@\3\2AD\3\2\34\36\3\2\37 \3\2!#\3"+
+    "\2$\'\3\2(+\3\2JK\3\2\63\64\u01b7\2\67\3\2\2\2\4\u008a\3\2\2\2\6\u0095"+
+    "\3\2\2\2\b\u0099\3\2\2\2\n\u009b\3\2\2\2\f\u00a0\3\2\2\2\16\u00a2\3\2"+
+    "\2\2\20\u00a4\3\2\2\2\22\u00ad\3\2\2\2\24\u00b5\3\2\2\2\26\u00ba\3\2\2"+
+    "\2\30\u00c4\3\2\2\2\32\u00c8\3\2\2\2\34\u00ee\3\2\2\2\36\u011e\3\2\2\2"+
+    " \u0120\3\2\2\2\"\u012d\3\2\2\2$\u0137\3\2\2\2&\u013e\3\2\2\2(\u0143\3"+
+    "\2\2\2*\u0149\3\2\2\2,\u014e\3\2\2\2.\u0153\3\2\2\2\60\u0166\3\2\2\2\62"+
+    "\u016b\3\2\2\2\64\u0178\3\2\2\2\668\5\4\3\2\67\66\3\2\2\289\3\2\2\29\67"+
+    "\3\2\2\29:\3\2\2\2:;\3\2\2\2;<\7\2\2\3<\3\3\2\2\2=>\7\16\2\2>?\7\t\2\2"+
+    "?@\5\34\17\2@A\7\n\2\2AD\5\6\4\2BC\7\17\2\2CE\5\6\4\2DB\3\2\2\2DE\3\2"+
+    "\2\2E\u008b\3\2\2\2FG\7\20\2\2GH\7\t\2\2HI\5\34\17\2IL\7\n\2\2JM\5\6\4"+
+    "\2KM\5\b\5\2LJ\3\2\2\2LK\3\2\2\2M\u008b\3\2\2\2NO\7\21\2\2OP\5\6\4\2P"+
+    "Q\7\20\2\2QR\7\t\2\2RS\5\34\17\2SU\7\n\2\2TV\7\r\2\2UT\3\2\2\2UV\3\2\2"+
+    "\2V\u008b\3\2\2\2WX\7\22\2\2XZ\7\t\2\2Y[\5\f\7\2ZY\3\2\2\2Z[\3\2\2\2["+
+    "\\\3\2\2\2\\^\7\r\2\2]_\5\34\17\2^]\3\2\2\2^_\3\2\2\2_`\3\2\2\2`b\7\r"+
+    "\2\2ac\5\16\b\2ba\3\2\2\2bc\3\2\2\2cd\3\2\2\2dg\7\n\2\2eh\5\6\4\2fh\5"+
+    "\b\5\2ge\3\2\2\2gf\3\2\2\2h\u008b\3\2\2\2ik\5\20\t\2jl\7\r\2\2kj\3\2\2"+
+    "\2kl\3\2\2\2l\u008b\3\2\2\2mo\7\23\2\2np\7\r\2\2on\3\2\2\2op\3\2\2\2p"+
+    "\u008b\3\2\2\2qs\7\24\2\2rt\7\r\2\2sr\3\2\2\2st\3\2\2\2t\u008b\3\2\2\2"+
+    "uv\7\25\2\2vx\5\34\17\2wy\7\r\2\2xw\3\2\2\2xy\3\2\2\2y\u008b\3\2\2\2z"+
+    "{\7\27\2\2{}\5\6\4\2|~\5\26\f\2}|\3\2\2\2~\177\3\2\2\2\177}\3\2\2\2\177"+
     "\u0080\3\2\2\2\u0080\u008b\3\2\2\2\u0081\u0082\7\31\2\2\u0082\u0084\5"+
     "\34\17\2\u0083\u0085\7\r\2\2\u0084\u0083\3\2\2\2\u0084\u0085\3\2\2\2\u0085"+
     "\u008b\3\2\2\2\u0086\u0088\5\34\17\2\u0087\u0089\7\r\2\2\u0088\u0087\3"+
@@ -2892,77 +2873,77 @@ class PainlessParser extends Parser {
     "\2\u00b9\25\3\2\2\2\u00ba\u00bb\7\30\2\2\u00bb\u00bc\7\t\2\2\u00bc\u00bd"+
     "\5\30\r\2\u00bd\u00be\5\30\r\2\u00be\u00bf\3\2\2\2\u00bf\u00c2\7\n\2\2"+
     "\u00c0\u00c3\5\6\4\2\u00c1\u00c3\5\n\6\2\u00c2\u00c0\3\2\2\2\u00c2\u00c1"+
-    "\3\2\2\2\u00c3\27\3\2\2\2\u00c4\u00c6\7J\2\2\u00c5\u00c7\5\32\16\2\u00c6"+
+    "\3\2\2\2\u00c3\27\3\2\2\2\u00c4\u00c6\7I\2\2\u00c5\u00c7\5\32\16\2\u00c6"+
     "\u00c5\3\2\2\2\u00c6\u00c7\3\2\2\2\u00c7\31\3\2\2\2\u00c8\u00c9\7$\2\2"+
     "\u00c9\u00ce\5\30\r\2\u00ca\u00cb\7\f\2\2\u00cb\u00cd\5\30\r\2\u00cc\u00ca"+
     "\3\2\2\2\u00cd\u00d0\3\2\2\2\u00ce\u00cc\3\2\2\2\u00ce\u00cf\3\2\2\2\u00cf"+
     "\u00d1\3\2\2\2\u00d0\u00ce\3\2\2\2\u00d1\u00d2\7&\2\2\u00d2\33\3\2\2\2"+
-    "\u00d3\u00d4\b\17\1\2\u00d4\u00d5\t\2\2\2\u00d5\u00f0\5\34\17\20\u00d6"+
+    "\u00d3\u00d4\b\17\1\2\u00d4\u00d5\t\2\2\2\u00d5\u00ef\5\34\17\20\u00d6"+
     "\u00d7\7\t\2\2\u00d7\u00d8\5\22\n\2\u00d8\u00d9\7\n\2\2\u00d9\u00da\5"+
-    "\34\17\17\u00da\u00f0\3\2\2\2\u00db\u00dc\5\36\20\2\u00dc\u00dd\t\3\2"+
-    "\2\u00dd\u00de\5\34\17\3\u00de\u00f0\3\2\2\2\u00df\u00e0\7\t\2\2\u00e0"+
-    "\u00e1\5\34\17\2\u00e1\u00e2\7\n\2\2\u00e2\u00f0\3\2\2\2\u00e3\u00f0\t"+
-    "\4\2\2\u00e4\u00f0\7F\2\2\u00e5\u00f0\7G\2\2\u00e6\u00f0\7H\2\2\u00e7"+
-    "\u00f0\7I\2\2\u00e8\u00e9\5\36\20\2\u00e9\u00ea\5\64\33\2\u00ea\u00f0"+
-    "\3\2\2\2\u00eb\u00ec\5\64\33\2\u00ec\u00ed\5\36\20\2\u00ed\u00f0\3\2\2"+
-    "\2\u00ee\u00f0\5\36\20\2\u00ef\u00d3\3\2\2\2\u00ef\u00d6\3\2\2\2\u00ef"+
-    "\u00db\3\2\2\2\u00ef\u00df\3\2\2\2\u00ef\u00e3\3\2\2\2\u00ef\u00e4\3\2"+
-    "\2\2\u00ef\u00e5\3\2\2\2\u00ef\u00e6\3\2\2\2\u00ef\u00e7\3\2\2\2\u00ef"+
-    "\u00e8\3\2\2\2\u00ef\u00eb\3\2\2\2\u00ef\u00ee\3\2\2\2\u00f0\u0117\3\2"+
-    "\2\2\u00f1\u00f2\f\16\2\2\u00f2\u00f3\t\5\2\2\u00f3\u0116\5\34\17\17\u00f4"+
-    "\u00f5\f\r\2\2\u00f5\u00f6\t\6\2\2\u00f6\u0116\5\34\17\16\u00f7\u00f8"+
-    "\f\f\2\2\u00f8\u00f9\t\7\2\2\u00f9\u0116\5\34\17\r\u00fa\u00fb\f\13\2"+
-    "\2\u00fb\u00fc\t\b\2\2\u00fc\u0116\5\34\17\f\u00fd\u00fe\f\n\2\2\u00fe"+
-    "\u00ff\t\t\2\2\u00ff\u0116\5\34\17\13\u0100\u0101\f\t\2\2\u0101\u0102"+
-    "\7,\2\2\u0102\u0116\5\34\17\n\u0103\u0104\f\b\2\2\u0104\u0105\7-\2\2\u0105"+
-    "\u0116\5\34\17\t\u0106\u0107\f\7\2\2\u0107\u0108\7.\2\2\u0108\u0116\5"+
-    "\34\17\b\u0109\u010a\f\6\2\2\u010a\u010b\7/\2\2\u010b\u0116\5\34\17\7"+
-    "\u010c\u010d\f\5\2\2\u010d\u010e\7\60\2\2\u010e\u0116\5\34\17\6\u010f"+
-    "\u0110\f\4\2\2\u0110\u0111\7\61\2\2\u0111\u0112\5\34\17\2\u0112\u0113"+
-    "\7\62\2\2\u0113\u0114\5\34\17\4\u0114\u0116\3\2\2\2\u0115\u00f1\3\2\2"+
-    "\2\u0115\u00f4\3\2\2\2\u0115\u00f7\3\2\2\2\u0115\u00fa\3\2\2\2\u0115\u00fd"+
-    "\3\2\2\2\u0115\u0100\3\2\2\2\u0115\u0103\3\2\2\2\u0115\u0106\3\2\2\2\u0115"+
-    "\u0109\3\2\2\2\u0115\u010c\3\2\2\2\u0115\u010f\3\2\2\2\u0116\u0119\3\2"+
-    "\2\2\u0117\u0115\3\2\2\2\u0117\u0118\3\2\2\2\u0118\35\3\2\2\2\u0119\u0117"+
-    "\3\2\2\2\u011a\u0120\5 \21\2\u011b\u0120\5\"\22\2\u011c\u0120\5*\26\2"+
-    "\u011d\u0120\5.\30\2\u011e\u0120\5\60\31\2\u011f\u011a\3\2\2\2\u011f\u011b"+
-    "\3\2\2\2\u011f\u011c\3\2\2\2\u011f\u011d\3\2\2\2\u011f\u011e\3\2\2\2\u0120"+
-    "\37\3\2\2\2\u0121\u0127\7\t\2\2\u0122\u0128\5 \21\2\u0123\u0128\5\"\22"+
-    "\2\u0124\u0128\5*\26\2\u0125\u0128\5.\30\2\u0126\u0128\5\60\31\2\u0127"+
-    "\u0122\3\2\2\2\u0127\u0123\3\2\2\2\u0127\u0124\3\2\2\2\u0127\u0125\3\2"+
-    "\2\2\u0127\u0126\3\2\2\2\u0128\u0129\3\2\2\2\u0129\u012c\7\n\2\2\u012a"+
-    "\u012d\5&\24\2\u012b\u012d\5$\23\2\u012c\u012a\3\2\2\2\u012c\u012b\3\2"+
-    "\2\2\u012c\u012d\3\2\2\2\u012d!\3\2\2\2\u012e\u012f\7\t\2\2\u012f\u0130"+
-    "\5\22\n\2\u0130\u0136\7\n\2\2\u0131\u0137\5 \21\2\u0132\u0137\5\"\22\2"+
-    "\u0133\u0137\5*\26\2\u0134\u0137\5.\30\2\u0135\u0137\5\60\31\2\u0136\u0131"+
-    "\3\2\2\2\u0136\u0132\3\2\2\2\u0136\u0133\3\2\2\2\u0136\u0134\3\2\2\2\u0136"+
-    "\u0135\3\2\2\2\u0137#\3\2\2\2\u0138\u0139\7\7\2\2\u0139\u013a\5\34\17"+
-    "\2\u013a\u013d\7\b\2\2\u013b\u013e\5&\24\2\u013c\u013e\5$\23\2\u013d\u013b"+
-    "\3\2\2\2\u013d\u013c\3\2\2\2\u013d\u013e\3\2\2\2\u013e%\3\2\2\2\u013f"+
-    "\u0142\7\13\2\2\u0140\u0143\5(\25\2\u0141\u0143\5,\27\2\u0142\u0140\3"+
-    "\2\2\2\u0142\u0141\3\2\2\2\u0143\'\3\2\2\2\u0144\u0145\7L\2\2\u0145\u0148"+
-    "\5\62\32\2\u0146\u0149\5&\24\2\u0147\u0149\5$\23\2\u0148\u0146\3\2\2\2"+
-    "\u0148\u0147\3\2\2\2\u0148\u0149\3\2\2\2\u0149)\3\2\2\2\u014a\u014d\5"+
-    "\30\r\2\u014b\u014e\5&\24\2\u014c\u014e\5$\23\2\u014d\u014b\3\2\2\2\u014d"+
-    "\u014c\3\2\2\2\u014d\u014e\3\2\2\2\u014e+\3\2\2\2\u014f\u0152\t\n\2\2"+
-    "\u0150\u0153\5&\24\2\u0151\u0153\5$\23\2\u0152\u0150\3\2\2\2\u0152\u0151"+
-    "\3\2\2\2\u0152\u0153\3\2\2\2\u0153-\3\2\2\2\u0154\u0155\7\26\2\2\u0155"+
-    "\u0165\5\30\r\2\u0156\u0158\5\62\32\2\u0157\u0159\5&\24\2\u0158\u0157"+
-    "\3\2\2\2\u0158\u0159\3\2\2\2\u0159\u0166\3\2\2\2\u015a\u015b\7\7\2\2\u015b"+
-    "\u015c\5\34\17\2\u015c\u015d\7\b\2\2\u015d\u015f\3\2\2\2\u015e\u015a\3"+
-    "\2\2\2\u015f\u0160\3\2\2\2\u0160\u015e\3\2\2\2\u0160\u0161\3\2\2\2\u0161"+
-    "\u0163\3\2\2\2\u0162\u0164\5&\24\2\u0163\u0162\3\2\2\2\u0163\u0164\3\2"+
-    "\2\2\u0164\u0166\3\2\2\2\u0165\u0156\3\2\2\2\u0165\u015e\3\2\2\2\u0166"+
-    "/\3\2\2\2\u0167\u016a\7E\2\2\u0168\u016b\5&\24\2\u0169\u016b\5$\23\2\u016a"+
-    "\u0168\3\2\2\2\u016a\u0169\3\2\2\2\u016a\u016b\3\2\2\2\u016b\61\3\2\2"+
-    "\2\u016c\u0175\7\t\2\2\u016d\u0172\5\34\17\2\u016e\u016f\7\f\2\2\u016f"+
-    "\u0171\5\34\17\2\u0170\u016e\3\2\2\2\u0171\u0174\3\2\2\2\u0172\u0170\3"+
-    "\2\2\2\u0172\u0173\3\2\2\2\u0173\u0176\3\2\2\2\u0174\u0172\3\2\2\2\u0175"+
-    "\u016d\3\2\2\2\u0175\u0176\3\2\2\2\u0176\u0177\3\2\2\2\u0177\u0178\7\n"+
-    "\2\2\u0178\63\3\2\2\2\u0179\u017a\t\13\2\2\u017a\65\3\2\2\2/9DLUZ^bgk"+
-    "osx\177\u0084\u0088\u008a\u0090\u0095\u0099\u00a0\u00aa\u00b2\u00b8\u00c2"+
-    "\u00c6\u00ce\u00ef\u0115\u0117\u011f\u0127\u012c\u0136\u013d\u0142\u0148"+
-    "\u014d\u0152\u0158\u0160\u0163\u0165\u016a\u0172\u0175";
+    "\34\17\17\u00da\u00ef\3\2\2\2\u00db\u00dc\5\36\20\2\u00dc\u00dd\t\3\2"+
+    "\2\u00dd\u00de\5\34\17\3\u00de\u00ef\3\2\2\2\u00df\u00e0\7\t\2\2\u00e0"+
+    "\u00e1\5\34\17\2\u00e1\u00e2\7\n\2\2\u00e2\u00ef\3\2\2\2\u00e3\u00ef\t"+
+    "\4\2\2\u00e4\u00ef\7F\2\2\u00e5\u00ef\7G\2\2\u00e6\u00ef\7H\2\2\u00e7"+
+    "\u00e8\5\36\20\2\u00e8\u00e9\5\64\33\2\u00e9\u00ef\3\2\2\2\u00ea\u00eb"+
+    "\5\64\33\2\u00eb\u00ec\5\36\20\2\u00ec\u00ef\3\2\2\2\u00ed\u00ef\5\36"+
+    "\20\2\u00ee\u00d3\3\2\2\2\u00ee\u00d6\3\2\2\2\u00ee\u00db\3\2\2\2\u00ee"+
+    "\u00df\3\2\2\2\u00ee\u00e3\3\2\2\2\u00ee\u00e4\3\2\2\2\u00ee\u00e5\3\2"+
+    "\2\2\u00ee\u00e6\3\2\2\2\u00ee\u00e7\3\2\2\2\u00ee\u00ea\3\2\2\2\u00ee"+
+    "\u00ed\3\2\2\2\u00ef\u0116\3\2\2\2\u00f0\u00f1\f\16\2\2\u00f1\u00f2\t"+
+    "\5\2\2\u00f2\u0115\5\34\17\17\u00f3\u00f4\f\r\2\2\u00f4\u00f5\t\6\2\2"+
+    "\u00f5\u0115\5\34\17\16\u00f6\u00f7\f\f\2\2\u00f7\u00f8\t\7\2\2\u00f8"+
+    "\u0115\5\34\17\r\u00f9\u00fa\f\13\2\2\u00fa\u00fb\t\b\2\2\u00fb\u0115"+
+    "\5\34\17\f\u00fc\u00fd\f\n\2\2\u00fd\u00fe\t\t\2\2\u00fe\u0115\5\34\17"+
+    "\13\u00ff\u0100\f\t\2\2\u0100\u0101\7,\2\2\u0101\u0115\5\34\17\n\u0102"+
+    "\u0103\f\b\2\2\u0103\u0104\7-\2\2\u0104\u0115\5\34\17\t\u0105\u0106\f"+
+    "\7\2\2\u0106\u0107\7.\2\2\u0107\u0115\5\34\17\b\u0108\u0109\f\6\2\2\u0109"+
+    "\u010a\7/\2\2\u010a\u0115\5\34\17\7\u010b\u010c\f\5\2\2\u010c\u010d\7"+
+    "\60\2\2\u010d\u0115\5\34\17\6\u010e\u010f\f\4\2\2\u010f\u0110\7\61\2\2"+
+    "\u0110\u0111\5\34\17\2\u0111\u0112\7\62\2\2\u0112\u0113\5\34\17\4\u0113"+
+    "\u0115\3\2\2\2\u0114\u00f0\3\2\2\2\u0114\u00f3\3\2\2\2\u0114\u00f6\3\2"+
+    "\2\2\u0114\u00f9\3\2\2\2\u0114\u00fc\3\2\2\2\u0114\u00ff\3\2\2\2\u0114"+
+    "\u0102\3\2\2\2\u0114\u0105\3\2\2\2\u0114\u0108\3\2\2\2\u0114\u010b\3\2"+
+    "\2\2\u0114\u010e\3\2\2\2\u0115\u0118\3\2\2\2\u0116\u0114\3\2\2\2\u0116"+
+    "\u0117\3\2\2\2\u0117\35\3\2\2\2\u0118\u0116\3\2\2\2\u0119\u011f\5 \21"+
+    "\2\u011a\u011f\5\"\22\2\u011b\u011f\5*\26\2\u011c\u011f\5.\30\2\u011d"+
+    "\u011f\5\60\31\2\u011e\u0119\3\2\2\2\u011e\u011a\3\2\2\2\u011e\u011b\3"+
+    "\2\2\2\u011e\u011c\3\2\2\2\u011e\u011d\3\2\2\2\u011f\37\3\2\2\2\u0120"+
+    "\u0126\7\t\2\2\u0121\u0127\5 \21\2\u0122\u0127\5\"\22\2\u0123\u0127\5"+
+    "*\26\2\u0124\u0127\5.\30\2\u0125\u0127\5\60\31\2\u0126\u0121\3\2\2\2\u0126"+
+    "\u0122\3\2\2\2\u0126\u0123\3\2\2\2\u0126\u0124\3\2\2\2\u0126\u0125\3\2"+
+    "\2\2\u0127\u0128\3\2\2\2\u0128\u012b\7\n\2\2\u0129\u012c\5&\24\2\u012a"+
+    "\u012c\5$\23\2\u012b\u0129\3\2\2\2\u012b\u012a\3\2\2\2\u012b\u012c\3\2"+
+    "\2\2\u012c!\3\2\2\2\u012d\u012e\7\t\2\2\u012e\u012f\5\22\n\2\u012f\u0135"+
+    "\7\n\2\2\u0130\u0136\5 \21\2\u0131\u0136\5\"\22\2\u0132\u0136\5*\26\2"+
+    "\u0133\u0136\5.\30\2\u0134\u0136\5\60\31\2\u0135\u0130\3\2\2\2\u0135\u0131"+
+    "\3\2\2\2\u0135\u0132\3\2\2\2\u0135\u0133\3\2\2\2\u0135\u0134\3\2\2\2\u0136"+
+    "#\3\2\2\2\u0137\u0138\7\7\2\2\u0138\u0139\5\34\17\2\u0139\u013c\7\b\2"+
+    "\2\u013a\u013d\5&\24\2\u013b\u013d\5$\23\2\u013c\u013a\3\2\2\2\u013c\u013b"+
+    "\3\2\2\2\u013c\u013d\3\2\2\2\u013d%\3\2\2\2\u013e\u0141\7\13\2\2\u013f"+
+    "\u0142\5(\25\2\u0140\u0142\5,\27\2\u0141\u013f\3\2\2\2\u0141\u0140\3\2"+
+    "\2\2\u0142\'\3\2\2\2\u0143\u0144\7K\2\2\u0144\u0147\5\62\32\2\u0145\u0148"+
+    "\5&\24\2\u0146\u0148\5$\23\2\u0147\u0145\3\2\2\2\u0147\u0146\3\2\2\2\u0147"+
+    "\u0148\3\2\2\2\u0148)\3\2\2\2\u0149\u014c\5\30\r\2\u014a\u014d\5&\24\2"+
+    "\u014b\u014d\5$\23\2\u014c\u014a\3\2\2\2\u014c\u014b\3\2\2\2\u014c\u014d"+
+    "\3\2\2\2\u014d+\3\2\2\2\u014e\u0151\t\n\2\2\u014f\u0152\5&\24\2\u0150"+
+    "\u0152\5$\23\2\u0151\u014f\3\2\2\2\u0151\u0150\3\2\2\2\u0151\u0152\3\2"+
+    "\2\2\u0152-\3\2\2\2\u0153\u0154\7\26\2\2\u0154\u0164\5\30\r\2\u0155\u0157"+
+    "\5\62\32\2\u0156\u0158\5&\24\2\u0157\u0156\3\2\2\2\u0157\u0158\3\2\2\2"+
+    "\u0158\u0165\3\2\2\2\u0159\u015a\7\7\2\2\u015a\u015b\5\34\17\2\u015b\u015c"+
+    "\7\b\2\2\u015c\u015e\3\2\2\2\u015d\u0159\3\2\2\2\u015e\u015f\3\2\2\2\u015f"+
+    "\u015d\3\2\2\2\u015f\u0160\3\2\2\2\u0160\u0162\3\2\2\2\u0161\u0163\5&"+
+    "\24\2\u0162\u0161\3\2\2\2\u0162\u0163\3\2\2\2\u0163\u0165\3\2\2\2\u0164"+
+    "\u0155\3\2\2\2\u0164\u015d\3\2\2\2\u0165/\3\2\2\2\u0166\u0169\7E\2\2\u0167"+
+    "\u016a\5&\24\2\u0168\u016a\5$\23\2\u0169\u0167\3\2\2\2\u0169\u0168\3\2"+
+    "\2\2\u0169\u016a\3\2\2\2\u016a\61\3\2\2\2\u016b\u0174\7\t\2\2\u016c\u0171"+
+    "\5\34\17\2\u016d\u016e\7\f\2\2\u016e\u0170\5\34\17\2\u016f\u016d\3\2\2"+
+    "\2\u0170\u0173\3\2\2\2\u0171\u016f\3\2\2\2\u0171\u0172\3\2\2\2\u0172\u0175"+
+    "\3\2\2\2\u0173\u0171\3\2\2\2\u0174\u016c\3\2\2\2\u0174\u0175\3\2\2\2\u0175"+
+    "\u0176\3\2\2\2\u0176\u0177\7\n\2\2\u0177\63\3\2\2\2\u0178\u0179\t\13\2"+
+    "\2\u0179\65\3\2\2\2/9DLUZ^bgkosx\177\u0084\u0088\u008a\u0090\u0095\u0099"+
+    "\u00a0\u00aa\u00b2\u00b8\u00c2\u00c6\u00ce\u00ee\u0114\u0116\u011e\u0126"+
+    "\u012b\u0135\u013c\u0141\u0147\u014c\u0151\u0157\u015f\u0162\u0164\u0169"+
+    "\u0171\u0174";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessParserBaseVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessParserBaseVisitor.java
index f1cc222edd9..824b83fe952 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessParserBaseVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessParserBaseVisitor.java
@@ -277,13 +277,6 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitBinary(PainlessParser.BinaryContext ctx) { return visitChildren(ctx); }
-  /**
-   * {@inheritDoc}
-   *
-   * <p>The default implementation returns the result of calling
-   * {@link #visitChildren} on {@code ctx}.</p>
-   */
-  @Override public T visitChar(PainlessParser.CharContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessParserVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessParserVisitor.java
index 3873d6f1e08..915328a3f05 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessParserVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessParserVisitor.java
@@ -265,13 +265,6 @@ interface PainlessParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitBinary(PainlessParser.BinaryContext ctx);
-  /**
-   * Visit a parse tree produced by the {@code char}
-   * labeled alternative in {@link PainlessParser#expression}.
-   * @param ctx the parse tree
-   * @return the visitor result
-   */
-  T visitChar(PainlessParser.CharContext ctx);
   /**
    * Visit a parse tree produced by the {@code true}
    * labeled alternative in {@link PainlessParser#expression}.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java
index 45c507b42ba..ee5a5f8cd1c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java
@@ -248,6 +248,10 @@ public class Utility {
         return (double)value;
     }
 
+    public static String charToString(final char value) {
+        return String.valueOf(value);
+    }
+
     public static boolean CharacterToboolean(final Character value) {
         return value != 0;
     }
@@ -304,6 +308,10 @@ public class Utility {
         return value == null ? null : (double)value;
     }
 
+    public static String CharacterToString(final Character value) {
+        return value == null ? null : value.toString();
+    }
+
     public static boolean intToboolean(final int value) {
         return value != 0;
     }
@@ -448,6 +456,26 @@ public class Utility {
         return (char)value.doubleValue();
     }
 
+    public static char StringTochar(final String value) {
+        if (value.length() != 1) {
+            throw new ClassCastException("Cannot cast [String] with length greater than one to [char].");
+        }
+
+        return value.charAt(0);
+    }
+
+    public static Character StringToCharacter(final String value) {
+        if (value == null) {
+            return null;
+        }
+
+        if (value.length() != 1) {
+            throw new ClassCastException("Cannot cast [String] with length greater than one to [Character].");
+        }
+
+        return value.charAt(0);
+    }
+
     // although divide by zero is guaranteed, the special overflow case is not caught.
     // its not needed for remainder because it is not possible there.
     // see https://docs.oracle.com/javase/specs/jls/se8/html/jls-15.html#jls-15.17.2
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
index 3cc6a2aa22b..4490bf119b4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
@@ -27,7 +27,6 @@ import org.elasticsearch.painless.PainlessParser.BinaryContext;
 import org.elasticsearch.painless.PainlessParser.BoolContext;
 import org.elasticsearch.painless.PainlessParser.BreakContext;
 import org.elasticsearch.painless.PainlessParser.CastContext;
-import org.elasticsearch.painless.PainlessParser.CharContext;
 import org.elasticsearch.painless.PainlessParser.CompContext;
 import org.elasticsearch.painless.PainlessParser.ConditionalContext;
 import org.elasticsearch.painless.PainlessParser.ContinueContext;
@@ -354,13 +353,6 @@ class Writer extends PainlessParserBaseVisitor<Void> {
         return null;
     }
 
-    @Override
-    public Void visitChar(final CharContext ctx) {
-        expression.processChar(ctx);
-
-        return null;
-    }
-
     @Override
     public Void visitTrue(final TrueContext ctx) {
         expression.processTrue(ctx);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExpression.java
index c850031efa0..7d10a275690 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExpression.java
@@ -26,7 +26,6 @@ import org.elasticsearch.painless.PainlessParser.AssignmentContext;
 import org.elasticsearch.painless.PainlessParser.BinaryContext;
 import org.elasticsearch.painless.PainlessParser.BoolContext;
 import org.elasticsearch.painless.PainlessParser.CastContext;
-import org.elasticsearch.painless.PainlessParser.CharContext;
 import org.elasticsearch.painless.PainlessParser.CompContext;
 import org.elasticsearch.painless.PainlessParser.ConditionalContext;
 import org.elasticsearch.painless.PainlessParser.ExpressionContext;
@@ -104,20 +103,6 @@ class WriterExpression {
         utility.checkWriteBranch(ctx);
     }
 
-    void processChar(final CharContext ctx) {
-        final ExpressionMetadata charemd = metadata.getExpressionMetadata(ctx);
-        final Object postConst = charemd.postConst;
-
-        if (postConst == null) {
-            utility.writeNumeric(ctx, (int)(char)charemd.preConst);
-            caster.checkWriteCast(charemd);
-        } else {
-            utility.writeConstant(ctx, postConst);
-        }
-
-        utility.checkWriteBranch(ctx);
-    }
-
     void processTrue(final TrueContext ctx) {
         final ExpressionMetadata trueemd = metadata.getExpressionMetadata(ctx);
         final Object postConst = trueemd.postConst;
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java
index ee3607242ca..d2a24ef9533 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java
@@ -38,7 +38,7 @@ public class BasicExpressionTests extends ScriptTestCase {
     }
 
     public void testReturnConstantChar() {
-        assertEquals('x', exec("return 'x';"));
+        assertEquals('x', exec("return (char)'x';"));
     }
 
     public void testConstantCharTruncation() {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java
index 607da9f0f32..8043d9da915 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java
@@ -131,51 +131,51 @@ public class EqualsTests extends ScriptTestCase {
     }
 
     public void testBranchEquals() {
-        assertEquals(0, exec("Character a = 'a'; Character b = 'b'; if (a == b) return 1; else return 0;"));
-        assertEquals(1, exec("Character a = 'a'; Character b = 'a'; if (a == b) return 1; else return 0;"));
+        assertEquals(0, exec("Character a = (char)'a'; Character b = (char)'b'; if (a == b) return 1; else return 0;"));
+        assertEquals(1, exec("Character a = (char)'a'; Character b = (char)'a'; if (a == b) return 1; else return 0;"));
         assertEquals(0, exec("Integer a = new Integer(1); Integer b = 1; if (a === b) return 1; else return 0;"));
-        assertEquals(0, exec("Character a = 'a'; Character b = new Character('a'); if (a === b) return 1; else return 0;"));
-        assertEquals(1, exec("Character a = 'a'; Object b = a; if (a === b) return 1; else return 0;"));
+        assertEquals(0, exec("Character a = (char)'a'; Character b = new Character((char)'a'); if (a === b) return 1; else return 0;"));
+        assertEquals(1, exec("Character a = (char)'a'; Object b = a; if (a === b) return 1; else return 0;"));
         assertEquals(1, exec("Integer a = 1; Number b = a; Number c = a; if (c === b) return 1; else return 0;"));
-        assertEquals(0, exec("Integer a = 1; Character b = 'a'; if (a === (Object)b) return 1; else return 0;"));
+        assertEquals(0, exec("Integer a = 1; Character b = (char)'a'; if (a === (Object)b) return 1; else return 0;"));
     }
 
     public void testBranchNotEquals() {
-        assertEquals(1, exec("Character a = 'a'; Character b = 'b'; if (a != b) return 1; else return 0;"));
-        assertEquals(0, exec("Character a = 'a'; Character b = 'a'; if (a != b) return 1; else return 0;"));
+        assertEquals(1, exec("Character a = (char)'a'; Character b = (char)'b'; if (a != b) return 1; else return 0;"));
+        assertEquals(0, exec("Character a = (char)'a'; Character b = (char)'a'; if (a != b) return 1; else return 0;"));
         assertEquals(1, exec("Integer a = new Integer(1); Integer b = 1; if (a !== b) return 1; else return 0;"));
-        assertEquals(1, exec("Character a = 'a'; Character b = new Character('a'); if (a !== b) return 1; else return 0;"));
-        assertEquals(0, exec("Character a = 'a'; Object b = a; if (a !== b) return 1; else return 0;"));
+        assertEquals(1, exec("Character a = (char)'a'; Character b = new Character((char)'a'); if (a !== b) return 1; else return 0;"));
+        assertEquals(0, exec("Character a = (char)'a'; Object b = a; if (a !== b) return 1; else return 0;"));
         assertEquals(0, exec("Integer a = 1; Number b = a; Number c = a; if (c !== b) return 1; else return 0;"));
-        assertEquals(1, exec("Integer a = 1; Character b = 'a'; if (a !== (Object)b) return 1; else return 0;"));
+        assertEquals(1, exec("Integer a = 1; Character b = (char)'a'; if (a !== (Object)b) return 1; else return 0;"));
     }
 
     public void testRightHandNull() {
-        assertEquals(false, exec("Character a = 'a'; return a == null;"));
-        assertEquals(false, exec("Character a = 'a'; return a === null;"));
-        assertEquals(true, exec("Character a = 'a'; return a != null;"));
-        assertEquals(true, exec("Character a = 'a'; return a !== null;"));
+        assertEquals(false, exec("Character a = (char)'a'; return a == null;"));
+        assertEquals(false, exec("Character a = (char)'a'; return a === null;"));
+        assertEquals(true, exec("Character a = (char)'a'; return a != null;"));
+        assertEquals(true, exec("Character a = (char)'a'; return a !== null;"));
         assertEquals(true, exec("Character a = null; return a == null;"));
         assertEquals(false, exec("Character a = null; return a != null;"));
-        assertEquals(false, exec("Character a = 'a'; Character b = null; return a == b;"));
+        assertEquals(false, exec("Character a = (char)'a'; Character b = null; return a == b;"));
         assertEquals(true, exec("Character a = null; Character b = null; return a === b;"));
-        assertEquals(true, exec("Character a = 'a'; Character b = null; return a != b;"));
+        assertEquals(true, exec("Character a = (char)'a'; Character b = null; return a != b;"));
         assertEquals(false, exec("Character a = null; Character b = null; return a !== b;"));
         assertEquals(false, exec("Integer x = null; double y = 2.0; return x == y;"));
         assertEquals(true, exec("Integer x = null; Short y = null; return x == y;"));
     }
 
     public void testLeftHandNull() {
-        assertEquals(false, exec("Character a = 'a'; return null == a;"));
-        assertEquals(false, exec("Character a = 'a'; return null === a;"));
-        assertEquals(true, exec("Character a = 'a'; return null != a;"));
-        assertEquals(true, exec("Character a = 'a'; return null !== a;"));
+        assertEquals(false, exec("Character a = (char)'a'; return null == a;"));
+        assertEquals(false, exec("Character a = (char)'a'; return null === a;"));
+        assertEquals(true, exec("Character a = (char)'a'; return null != a;"));
+        assertEquals(true, exec("Character a = (char)'a'; return null !== a;"));
         assertEquals(true, exec("Character a = null; return null == a;"));
         assertEquals(false, exec("Character a = null; return null != a;"));
-        assertEquals(false, exec("Character a = null; Character b = 'a'; return a == b;"));
+        assertEquals(false, exec("Character a = null; Character b = (char)'a'; return a == b;"));
         assertEquals(true, exec("Character a = null; Character b = null; return a == b;"));
         assertEquals(true, exec("Character a = null; Character b = null; return b === a;"));
-        assertEquals(true, exec("Character a = null; Character b = 'a'; return a != b;"));
+        assertEquals(true, exec("Character a = null; Character b = (char)'a'; return a != b;"));
         assertEquals(false, exec("Character a = null; Character b = null; return b != a;"));
         assertEquals(false, exec("Character a = null; Character b = null; return b !== a;"));
         assertEquals(false, exec("Integer x = null; double y = 2.0; return y == x;"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
index 7d489332f92..f619d44ad81 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
@@ -41,6 +41,26 @@ public class StringTests extends ScriptTestCase {
         assertEquals("cat" + 2.0, exec("String s = \"cat\"; return s + 2.0;"));
         // String
         assertEquals("cat" + "cat", exec("String s = \"cat\"; return s + s;"));
+
+        // boolean
+        assertEquals("cat" + true, exec("String s = 'cat'; return s + true;"));
+        // byte
+        assertEquals("cat" + (byte)3, exec("String s = 'cat'; return s + (byte)3;"));
+        // short
+        assertEquals("cat" + (short)3, exec("String s = 'cat'; return s + (short)3;"));
+        // char
+        assertEquals("cat" + 't', exec("String s = 'cat'; return s + 't';"));
+        assertEquals("cat" + (char)40, exec("String s = 'cat'; return s + (char)40;"));
+        // int
+        assertEquals("cat" + 2, exec("String s = 'cat'; return s + 2;"));
+        // long
+        assertEquals("cat" + 2L, exec("String s = 'cat'; return s + 2L;"));
+        // float
+        assertEquals("cat" + 2F, exec("String s = 'cat'; return s + 2F;"));
+        // double
+        assertEquals("cat" + 2.0, exec("String s = 'cat'; return s + 2.0;"));
+        // String
+        assertEquals("cat" + "cat", exec("String s = 'cat'; return s + s;"));
     }
 
     public void testStringAPI() {
@@ -71,5 +91,99 @@ public class StringTests extends ScriptTestCase {
         assertEquals("e", exec("return \"abcde\".substring(4, 5);"));
         assertEquals(97, ((char[])exec("return \"a\".toCharArray();"))[0]);
         assertEquals("a", exec("return \" a \".trim();"));
+
+        assertEquals("", exec("return new String();"));
+        assertEquals('x', exec("String s = 'x'; return s.charAt(0);"));
+        assertEquals(120, exec("String s = 'x'; return s.codePointAt(0);"));
+        assertEquals(0, exec("String s = 'x'; return s.compareTo('x');"));
+        assertEquals("xx", exec("String s = 'x'; return s.concat('x');"));
+        assertEquals(true, exec("String s = 'xy'; return s.endsWith('y');"));
+        assertEquals(2, exec("String t = 'abcde'; return t.indexOf('cd', 1);"));
+        assertEquals(false, exec("String t = 'abcde'; return t.isEmpty();"));
+        assertEquals(5, exec("String t = 'abcde'; return t.length();"));
+        assertEquals("cdcde", exec("String t = 'abcde'; return t.replace('ab', 'cd');"));
+        assertEquals(false, exec("String s = 'xy'; return s.startsWith('y');"));
+        assertEquals("e", exec("String t = 'abcde'; return t.substring(4, 5);"));
+        assertEquals(97, ((char[])exec("String s = 'a'; return s.toCharArray();"))[0]);
+        assertEquals("a", exec("String s = ' a '; return s.trim();"));
+        assertEquals('x', exec("return 'x'.charAt(0);"));
+        assertEquals(120, exec("return 'x'.codePointAt(0);"));
+        assertEquals(0, exec("return 'x'.compareTo('x');"));
+        assertEquals("xx", exec("return 'x'.concat('x');"));
+        assertEquals(true, exec("return 'xy'.endsWith('y');"));
+        assertEquals(2, exec("return 'abcde'.indexOf('cd', 1);"));
+        assertEquals(false, exec("return 'abcde'.isEmpty();"));
+        assertEquals(5, exec("return 'abcde'.length();"));
+        assertEquals("cdcde", exec("return 'abcde'.replace('ab', 'cd');"));
+        assertEquals(false, exec("return 'xy'.startsWith('y');"));
+        assertEquals("e", exec("return 'abcde'.substring(4, 5);"));
+        assertEquals(97, ((char[])exec("return 'a'.toCharArray();"))[0]);
+        assertEquals("a", exec("return ' a '.trim();"));
+    }
+
+    public void testStringAndCharacter() {
+        assertEquals('c', exec("return (char)\"c\""));
+        assertEquals('c', exec("return (char)'c'"));
+        assertEquals("c", exec("return (String)(char)\"c\""));
+        assertEquals("c", exec("return (String)(char)'c'"));
+
+        assertEquals('c', exec("String s = \"c\" (char)s"));
+        assertEquals('c', exec("String s = 'c' (char)s"));
+
+        try {
+            assertEquals("cc", exec("return (String)(char)\"cc\""));
+        } catch (final IllegalArgumentException ise) {
+            ise.getMessage().contains("Cannot cast constant from [String] to [char].");
+        }
+
+        try {
+            assertEquals("cc", exec("return (String)(char)'cc'"));
+        } catch (final IllegalArgumentException ise) {
+            ise.getMessage().contains("Cannot cast constant from [String] to [char].");
+        }
+
+        try {
+            assertEquals('c', exec("String s = \"cc\" (char)s"));
+        } catch (final ClassCastException cce) {
+            cce.getMessage().contains("Cannot cast [String] with length greater than one to [char].");
+        }
+
+        try {
+            assertEquals('c', exec("String s = 'cc' (char)s"));
+        } catch (final ClassCastException cce) {
+            cce.getMessage().contains("Cannot cast [String] with length greater than one to [char].");
+        }
+
+        assertEquals('c', exec("return (Character)\"c\""));
+        assertEquals('c', exec("return (Character)'c'"));
+        assertEquals("c", exec("return (String)(Character)\"c\""));
+        assertEquals("c", exec("return (String)(Character)'c'"));
+
+        assertEquals('c', exec("String s = \"c\" (Character)s"));
+        assertEquals('c', exec("String s = 'c' (Character)s"));
+
+        try {
+            assertEquals("cc", exec("return (String)(Character)\"cc\""));
+        } catch (final ClassCastException ise) {
+            ise.getMessage().contains("Cannot cast [String] with length greater than one to [Character].");
+        }
+
+        try {
+            assertEquals("cc", exec("return (String)(Character)'cc'"));
+        } catch (final ClassCastException ise) {
+            ise.getMessage().contains("Cannot cast [String] with length greater than one to [Character].");
+        }
+
+        try {
+            assertEquals('c', exec("String s = \"cc\" (Character)s"));
+        } catch (final ClassCastException cce) {
+            cce.getMessage().contains("Cannot cast [String] with length greater than one to [Character].");
+        }
+
+        try {
+            assertEquals('c', exec("String s = 'cc' (Character)s"));
+        } catch (final ClassCastException cce) {
+            cce.getMessage().contains("Cannot cast [String] with length greater than one to [Character].");
+        }
     }
 }
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/20_scriptfield.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/20_scriptfield.yaml
index 66cff5df792..d53306b5d47 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/20_scriptfield.yaml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/20_scriptfield.yaml
@@ -28,7 +28,7 @@ setup:
                 script_fields:
                     bar:
                         script: 
-                            inline: "input.doc.foo.0 + input.x;"
+                            inline: "input.doc['foo'].0 + input.x;"
                             lang: painless
                             params:
                                 x: "bbb"
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml
index da28a1f4201..2dd1a6004ff 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml
@@ -29,12 +29,12 @@
                 query:
                     script:
                         script:
-                            inline: "input.doc.num1.0 > 1;"
+                            inline: "input.doc['num1'].0 > 1;"
                             lang: painless
                 script_fields:
                     sNum1:
                         script: 
-                            inline: "input.doc.num1.0;"
+                            inline: "input.doc['num1'].0;"
                             lang: painless
                 sort:
                     num1:
@@ -51,7 +51,7 @@
                 query:
                     script:
                         script:
-                            inline: "input.doc.num1.0 > input.param1;"
+                            inline: "input.doc['num1'].0 > input.param1;"
                             lang: painless
                             params:
                                 param1: 1
@@ -59,7 +59,7 @@
                 script_fields:
                     sNum1:
                         script:
-                            inline: "return input.doc.num1.0;"
+                            inline: "return input.doc['num1'].0;"
                             lang: painless
                 sort:
                     num1:
@@ -76,7 +76,7 @@
                 query:
                     script:
                         script:
-                            inline: "input.doc.num1.0 > input.param1;"
+                            inline: "input.doc['num1'].0 > input.param1;"
                             lang: painless
                             params:
                                 param1: -1
@@ -84,7 +84,7 @@
                 script_fields:
                     sNum1:
                         script: 
-                            inline: "input.doc.num1.0;"
+                            inline: "input.doc['num1'].0;"
                             lang: painless
                 sort:
                     num1:

From 4b1c116461f91fd365eefc697fcaad6f27786321 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 29 Apr 2016 10:42:03 -0400
Subject: [PATCH 0084/1311] Generate and run tests from the docs

Adds infrastructure so `gradle :docs:check` will extract tests from
snippets in the documentation and execute the tests. This is included
in `gradle check` so it should happen on CI and during a normal build.

By default each `// AUTOSENSE` snippet creates a unique REST test. These
tests are executed in a random order and the cluster is wiped between
each one. If multiple snippets chain together into a test you can annotate
all snippets after the first with `// TEST[continued]` to have the
generated tests for both snippets joined.

Snippets marked as `// TESTRESPONSE` are checked against the response
of the last action.

See docs/README.asciidoc for lots more.

Closes #12583. That issue is about catching bugs in the docs during build.
This catches *some* bugs in the docs during build which is a good start.
---
 .../gradle/RestTestsFromSnippetsTask.groovy   | 214 +++++++++++++
 .../elasticsearch/gradle/SnippetsTask.groovy  | 299 ++++++++++++++++++
 .../index/mapper/MapperService.java           |   2 +-
 docs/README.asciidoc                          |  45 ++-
 docs/build.gradle                             |  84 +++++
 .../analyzers/pattern-analyzer.asciidoc       |  23 +-
 docs/reference/cluster/tasks.asciidoc         |  21 +-
 docs/reference/docs/index_.asciidoc           |  99 ++++--
 docs/reference/docs/reindex.asciidoc          |  83 +++--
 docs/reference/docs/update-by-query.asciidoc  |  64 +++-
 .../index-modules/allocation/delayed.asciidoc |   7 +-
 .../allocation/filtering.asciidoc             |   5 +
 .../allocation/prioritization.asciidoc        |   1 +
 docs/reference/indices/analyze.asciidoc       |   3 +-
 docs/reference/indices/flush.asciidoc         |  32 +-
 docs/reference/indices/put-mapping.asciidoc   |  22 +-
 docs/reference/indices/templates.asciidoc     |   2 +-
 docs/reference/ingest.asciidoc                |   7 +-
 docs/reference/ingest/ingest-node.asciidoc    |  47 +--
 docs/reference/mapping.asciidoc               |   1 +
 .../mapping/dynamic-mapping.asciidoc          |   5 +-
 .../mapping/dynamic/templates.asciidoc        |  16 +-
 .../mapping/fields/all-field.asciidoc         |   7 +-
 .../mapping/fields/field-names-field.asciidoc |  16 +-
 .../mapping/fields/parent-field.asciidoc      |  16 +-
 .../mapping/fields/routing-field.asciidoc     |  28 +-
 .../mapping/fields/timestamp-field.asciidoc   |   1 +
 .../mapping/fields/type-field.asciidoc        |  29 +-
 docs/reference/mapping/params.asciidoc        |   7 -
 .../mapping/params/analyzer.asciidoc          |  15 +-
 docs/reference/mapping/params/boost.asciidoc  |  21 +-
 docs/reference/mapping/params/coerce.asciidoc |  13 +-
 .../reference/mapping/params/copy-to.asciidoc |   6 +-
 .../reference/mapping/params/dynamic.asciidoc |  21 +-
 .../mapping/params/fielddata.asciidoc         |  13 +-
 .../mapping/params/ignore-malformed.asciidoc  |  12 +-
 .../mapping/params/include-in-all.asciidoc    |   2 +-
 .../mapping/params/multi-fields.asciidoc      |   9 +-
 docs/reference/mapping/params/norms.asciidoc  |   3 +-
 .../params/position-increment-gap.asciidoc    |  10 +-
 .../mapping/params/properties.asciidoc        |   3 +-
 .../mapping/params/search-analyzer.asciidoc   |   4 +-
 docs/reference/mapping/params/store.asciidoc  |   5 +-
 docs/reference/mapping/types/date.asciidoc    |   3 +-
 docs/reference/mapping/types/nested.asciidoc  |   3 +-
 .../migration/migrate_5_0/mapping.asciidoc    |   3 +-
 .../cluster/allocation_filtering.asciidoc     |   4 +-
 .../modules/cluster/disk_allocator.asciidoc   |   3 +-
 docs/reference/modules/node.asciidoc          |   2 +-
 .../modules/scripting/painless.asciidoc       |  37 ++-
 .../modules/scripting/using.asciidoc          |  27 +-
 docs/reference/query-dsl/bool-query.asciidoc  |  60 ++--
 .../query-dsl/function-score-query.asciidoc   |   3 +-
 docs/reference/query-dsl/term-query.asciidoc  |  11 +-
 docs/reference/setup/cluster_restart.asciidoc |   8 +-
 docs/reference/setup/rolling_upgrade.asciidoc |   8 +-
 .../smoketest/SmokeTestDocsIT.java            |  53 ++++
 settings.gradle                               |   1 +
 .../test/rest/RestTestExecutionContext.java   |   2 +-
 .../org/elasticsearch/test/rest/Stash.java    |  27 +-
 .../test/rest/client/RestClient.java          |  16 +
 .../test/rest/json/JsonPath.java              |   6 +-
 .../rest/parser/RestTestSectionParser.java    |  28 +-
 .../parser/RestTestSuiteParseContext.java     |  30 +-
 .../rest/section/ResponseBodyAssertion.java   | 167 ++++++++++
 65 files changed, 1385 insertions(+), 440 deletions(-)
 create mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/RestTestsFromSnippetsTask.groovy
 create mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/SnippetsTask.groovy
 create mode 100644 docs/build.gradle
 create mode 100644 docs/src/test/java/org/elasticsearch/smoketest/SmokeTestDocsIT.java
 create mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/section/ResponseBodyAssertion.java

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/RestTestsFromSnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/RestTestsFromSnippetsTask.groovy
new file mode 100644
index 00000000000..c4201a01f26
--- /dev/null
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/RestTestsFromSnippetsTask.groovy
@@ -0,0 +1,214 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle
+
+import org.elasticsearch.gradle.SnippetsTask.Snippet
+import org.gradle.api.InvalidUserDataException
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.OutputDirectory
+
+import java.nio.file.Files
+import java.nio.file.Path
+import java.util.regex.Matcher
+
+/**
+ * Generates REST tests for each snippet marked // TEST.
+ */
+public class RestTestsFromSnippetsTask extends SnippetsTask {
+    @Input
+    Map<String, String> setups = new HashMap()
+
+    /**
+     * Root directory of the tests being generated. To make rest tests happy
+     * we generate them in a testRoot() which is contained in this directory.
+     */
+    @OutputDirectory
+    File testRoot = project.file('build/rest')
+
+    public RestTestsFromSnippetsTask() {
+        project.afterEvaluate {
+            // Wait to set this so testRoot can be customized
+            project.sourceSets.test.output.dir(testRoot, builtBy: this)
+        }
+        TestBuilder builder = new TestBuilder()
+        doFirst { outputRoot().delete() }
+        perSnippet builder.&handleSnippet
+        doLast builder.&finishLastTest
+    }
+
+    /**
+     * Root directory containing all the files generated by this task. It is
+     * contained withing testRoot.
+     */
+    File outputRoot() {
+        return new File(testRoot, '/rest-api-spec/test')
+    }
+
+    private class TestBuilder {
+        private static final String SYNTAX = {
+            String method = /(?<method>GET|PUT|POST|HEAD|OPTIONS|DELETE)/
+            String pathAndQuery = /(?<pathAndQuery>[^\n]+)/
+            String badBody = /GET|PUT|POST|HEAD|OPTIONS|DELETE|#/
+            String body = /(?<body>(?:\n(?!$badBody)[^\n]+)+)/
+            String nonComment = /$method\s+$pathAndQuery$body?/
+            String comment = /(?<comment>#.+)/
+            /(?:$comment|$nonComment)\n+/
+        }()
+
+        /**
+         * The file in which we saw the last snippet that made a test.
+         */
+        Path lastDocsPath
+
+        /**
+         * The file we're building.
+         */
+        PrintWriter current
+
+        /**
+         * Called each time a snippet is encountered. Tracks the snippets and
+         * calls buildTest to actually build the test.
+         */
+        void handleSnippet(Snippet snippet) {
+            if (snippet.testSetup) {
+                setup(snippet)
+                return
+            }
+            if (snippet.testResponse) {
+                response(snippet)
+                return
+            }
+            if (snippet.test || snippet.autoSense) {
+                test(snippet)
+                return
+            }
+            // Must be an unmarked snippet....
+        }
+
+        private void test(Snippet test) {
+            setupCurrent(test)
+
+            if (false == test.continued) {
+                current.println('---')
+                current.println("\"$test.start\":")
+            }
+            if (test.skipTest) {
+                current.println("  - skip:")
+                current.println("      features: always_skip")
+                current.println("      reason: $test.skipTest")
+            }
+            if (test.setup != null) {
+                String setup = setups[test.setup]
+                if (setup == null) {
+                    throw new InvalidUserDataException("Couldn't find setup "
+                        + "for $test")
+                }
+                current.println(setup)
+            }
+
+            body(test)
+        }
+
+        private void response(Snippet response) {
+            current.println("  - response_body: |")
+            response.contents.eachLine { current.println("      $it") }
+        }
+
+        void emitDo(String method, String pathAndQuery,
+                String body, String catchPart) {
+            def (String path, String query) = pathAndQuery.tokenize('?')
+            current.println("  - do:")
+            if (catchPart != null) {
+                current.println("      catch: $catchPart")
+            }
+            current.println("      raw:")
+            current.println("        method: $method")
+            current.println("        path: \"$path\"")
+            if (query != null) {
+                for (String param: query.tokenize('&')) {
+                    def (String name, String value) = param.tokenize('=')
+                    current.println("        $name: \"$value\"")
+                }
+            }
+            if (body != null) {
+                // Throw out the leading newline we get from parsing the body
+                body = body.substring(1)
+                current.println("        body: |")
+                body.eachLine { current.println("          $it") }
+            }
+        }
+
+        private void setup(Snippet setup) {
+            if (lastDocsPath == setup.path) {
+                throw new InvalidUserDataException("$setup: wasn't first")
+            }
+            setupCurrent(setup)
+            current.println('---')
+            current.println("setup:")
+            body(setup)
+        }
+
+        private void body(Snippet snippet) {
+            parse("$snippet", snippet.contents, SYNTAX) { matcher, last ->
+                if (matcher.group("comment") != null) {
+                    // Comment
+                    return
+                }
+                String method = matcher.group("method")
+                String pathAndQuery = matcher.group("pathAndQuery")
+                String body = matcher.group("body")
+                String catchPart = last ? snippet.catchPart : null
+                if (pathAndQuery.startsWith('/')) {
+                    // Why not do some light linting while we're here?
+                    throw new InvalidUserDataException(
+                        "Path shouldn't start with a '/': $snippet\n"
+                        + snippet.contents)
+                }
+                emitDo(method, pathAndQuery, body, catchPart)
+            }
+        }
+
+        private PrintWriter setupCurrent(Snippet test) {
+            if (lastDocsPath == test.path) {
+                return
+            }
+            finishLastTest()
+            lastDocsPath = test.path
+
+            // Make the destination file:
+            // Shift the path into the destination directory tree
+            Path dest = outputRoot().toPath().resolve(test.path)
+            // Replace the extension
+            String fileName = dest.getName(dest.nameCount - 1)
+            dest = dest.parent.resolve(fileName.replace('.asciidoc', '.yaml'))
+
+            // Now setup the writer
+            Files.createDirectories(dest.parent)
+            current = dest.newPrintWriter()
+        }
+
+        void finishLastTest() {
+            if (current != null) {
+                current.close()
+                current = null
+            }
+        }
+    }
+}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/SnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/SnippetsTask.groovy
new file mode 100644
index 00000000000..73160cde7f1
--- /dev/null
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/SnippetsTask.groovy
@@ -0,0 +1,299 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle
+
+import org.gradle.api.DefaultTask
+import org.gradle.api.InvalidUserDataException
+import org.gradle.api.file.ConfigurableFileTree
+import org.gradle.api.tasks.InputFiles
+import org.gradle.api.tasks.TaskAction
+
+import java.nio.file.Path
+import java.util.regex.Matcher
+
+/**
+ * A task which will run a closure on each snippet in the documentation.
+ */
+public class SnippetsTask extends DefaultTask {
+    private static final String SCHAR = /(?:\\\/|[^\/])/
+    private static final String SUBSTITUTION = /s\/($SCHAR+)\/($SCHAR*)\//
+    private static final String CATCH = /catch:\s*((?:\/[^\/]+\/)|[^ \]]+)/
+    private static final String SKIP = /skip:([^\]]+)/
+    private static final String SETUP = /setup:([^ \]]+)/
+    private static final String TEST_SYNTAX =
+        /(?:$CATCH|$SUBSTITUTION|$SKIP|(continued)|$SETUP) ?/
+
+    /**
+     * Action to take on each snippet. Called with a single parameter, an
+     * instance of Snippet.
+     */
+    Closure perSnippet
+
+    /**
+     * The docs to scan. Defaults to every file in the directory exception the
+     * build.gradle file because that is appropriate for Elasticsearch's docs
+     * directory.
+     */
+    @InputFiles
+    ConfigurableFileTree docs = project.fileTree(project.projectDir) {
+        // No snippets in the build file
+        exclude 'build.gradle'
+    }
+
+    @TaskAction
+    public void executeTask() {
+        /*
+         * Walks each line of each file, building snippets as it encounters
+         * the lines that make up the snippet.
+         */
+        for (File file: docs) {
+            String lastLanguage
+            int lastLanguageLine
+            Snippet snippet = null
+            StringBuilder contents = null
+            List substitutions = null
+            Closure emit = {
+                snippet.contents = contents.toString()
+                contents = null
+                if (substitutions != null) {
+                    substitutions.each { String pattern, String subst ->
+                        /*
+                         * $body is really common but it looks like a
+                         * backreference so we just escape it here to make the
+                         * tests cleaner.
+                         */
+                        subst = subst.replace('$body', '\\$body')
+                        // \n is a new line....
+                        subst = subst.replace('\\n', '\n')
+                        snippet.contents = snippet.contents.replaceAll(
+                            pattern, subst)
+                    }
+                    substitutions = null
+                }
+                perSnippet(snippet)
+                snippet = null
+            }
+            file.eachLine('UTF-8') { String line, int lineNumber ->
+                Matcher matcher
+                if (line ==~ /-{4,}\s*/) { // Four dashes looks like a snippet
+                    if (snippet == null) {
+                        Path path = docs.dir.toPath().relativize(file.toPath())
+                        snippet = new Snippet(path: path, start: lineNumber)
+                        if (lastLanguageLine == lineNumber - 1) {
+                            snippet.language = lastLanguage
+                        }
+                    } else {
+                        snippet.end = lineNumber
+                    }
+                    return
+                }
+                matcher = line =~ /\[source,(\w+)]\s*/
+                if (matcher.matches()) {
+                    lastLanguage = matcher.group(1)
+                    lastLanguageLine = lineNumber
+                    return
+                }
+                if (line ==~ /\/\/ AUTOSENSE\s*/) {
+                    if (snippet == null) {
+                        throw new InvalidUserDataException("AUTOSENSE not " +
+                            "paired with a snippet at $file:$lineNumber")
+                    }
+                    snippet.autoSense = true
+                    return
+                }
+                matcher = line =~ /\/\/ TEST(\[(.+)\])?\s*/
+                if (matcher.matches()) {
+                    if (snippet == null) {
+                        throw new InvalidUserDataException("TEST not " +
+                            "paired with a snippet at $file:$lineNumber")
+                    }
+                    snippet.test = true
+                    if (matcher.group(2) != null) {
+                        String loc = "$file:$lineNumber"
+                        parse(loc, matcher.group(2), TEST_SYNTAX) {
+                            if (it.group(1) != null) {
+                                snippet.catchPart = it.group(1)
+                                return
+                            }
+                            if (it.group(2) != null) {
+                                if (substitutions == null) {
+                                    substitutions = []
+                                }
+                                substitutions.add([it.group(2), it.group(3)])
+                                return
+                            }
+                            if (it.group(4) != null) {
+                                snippet.skipTest = it.group(4)
+                                return
+                            }
+                            if (it.group(5) != null) {
+                                snippet.continued = true
+                                return
+                            }
+                            if (it.group(6) != null) {
+                                snippet.setup = it.group(6)
+                                return
+                            }
+                            throw new InvalidUserDataException(
+                                    "Invalid test marker: $line")
+                        }
+                    }
+                    return
+                }
+                matcher = line =~ /\/\/ TESTRESPONSE(\[(.+)\])?\s*/
+                if (matcher.matches()) {
+                    if (snippet == null) {
+                        throw new InvalidUserDataException("TESTRESPONSE not " +
+                            "paired with a snippet at $file:$lineNumber")
+                    }
+                    snippet.testResponse = true
+                    if (matcher.group(2) != null) {
+                        substitutions = []
+                        String loc = "$file:$lineNumber"
+                        parse(loc, matcher.group(2), /$SUBSTITUTION ?/) {
+                            substitutions.add([it.group(1), it.group(2)])
+                        }
+                    }
+                    return
+                }
+                if (line ==~ /\/\/ TESTSETUP\s*/) {
+                    snippet.testSetup = true
+                    return
+                }
+                if (snippet == null) {
+                    // Outside
+                    return
+                }
+                if (snippet.end == Snippet.NOT_FINISHED) {
+                    // Inside
+                    if (contents == null) {
+                        contents = new StringBuilder()
+                    }
+                    // We don't need the annotations
+                    line = line.replaceAll(/<\d+>/, '')
+                    // Nor any trailing spaces
+                    line = line.replaceAll(/\s+$/, '')
+                    contents.append(line).append('\n')
+                    return
+                }
+                // Just finished
+                emit()
+            }
+            if (snippet != null) emit()
+        }
+    }
+
+    static class Snippet {
+        static final int NOT_FINISHED = -1
+
+        /**
+         * Path to the file containing this snippet. Relative to docs.dir of the
+         * SnippetsTask that created it.
+         */
+        Path path
+        int start
+        int end = NOT_FINISHED
+        String contents
+
+        boolean autoSense = false
+        boolean test = false
+        boolean testResponse = false
+        boolean testSetup = false
+        String skipTest = null
+        boolean continued = false
+        String language = null
+        String catchPart = null
+        String setup = null
+
+        @Override
+        public String toString() {
+            String result = "$path[$start:$end]"
+            if (language != null) {
+                result += "($language)"
+            }
+            if (autoSense) {
+                result += '// AUTOSENSE'
+            }
+            if (test) {
+                result += '// TEST'
+                if (catchPart) {
+                    result += "[catch: $catchPart]"
+                }
+                if (skipTest) {
+                    result += "[skip=$skipTest]"
+                }
+                if (continued) {
+                    result += '[continued]'
+                }
+                if (setup) {
+                    result += "[setup:$setup]"
+                }
+            }
+            if (testResponse) {
+                result += '// TESTRESPONSE'
+            }
+            if (testSetup) {
+                result += '// TESTSETUP'
+            }
+            return result
+        }
+    }
+
+    /**
+     * Repeatedly match the pattern to the string, calling the closure with the
+     * matchers each time there is a match. If there are characters that don't
+     * match then blow up. If the closure takes two parameters then the second
+     * one is "is this the last match?".
+     */
+    protected parse(String location, String s, String pattern, Closure c) {
+        if (s == null) {
+            return // Silly null, only real stuff gets to match!
+        }
+        Matcher m = s =~ pattern
+        int offset = 0
+        Closure extraContent = { message ->
+            StringBuilder cutOut = new StringBuilder()
+            cutOut.append(s[offset - 6..offset - 1])
+            cutOut.append('*')
+            cutOut.append(s[offset..Math.min(offset + 5, s.length() - 1)])
+            String cutOutNoNl = cutOut.toString().replace('\n', '\\n')
+            throw new InvalidUserDataException("$location: Extra content "
+                + "$message ('$cutOutNoNl') matching [$pattern]: $s")
+        }
+        while (m.find()) {
+            if (m.start() != offset) {
+                extraContent("between [$offset] and [${m.start()}]")
+            }
+            offset = m.end()
+            if (c.maximumNumberOfParameters == 1) {
+                c(m)
+            } else {
+                c(m, offset == s.length())
+            }
+        }
+        if (offset == 0) {
+            throw new InvalidUserDataException("$location: Didn't match "
+                + "$pattern: $s")
+        }
+        if (offset != s.length()) {
+            extraContent("after [$offset]")
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
index b1c8b03fe37..0635a532a58 100755
--- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
@@ -92,7 +92,7 @@ public class MapperService extends AbstractIndexComponent {
             Setting.longSetting("index.mapping.depth.limit", 20L, 1, Property.Dynamic, Property.IndexScope);
     public static final boolean INDEX_MAPPER_DYNAMIC_DEFAULT = true;
     public static final Setting<Boolean> INDEX_MAPPER_DYNAMIC_SETTING =
-        Setting.boolSetting("index.mapper.dynamic", INDEX_MAPPER_DYNAMIC_DEFAULT, Property.IndexScope);
+        Setting.boolSetting("index.mapper.dynamic", INDEX_MAPPER_DYNAMIC_DEFAULT, Property.Dynamic, Property.IndexScope);
     private static ObjectHashSet<String> META_FIELDS = ObjectHashSet.from(
             "_uid", "_id", "_type", "_all", "_parent", "_routing", "_index",
             "_size", "_timestamp", "_ttl"
diff --git a/docs/README.asciidoc b/docs/README.asciidoc
index 3146d76f1eb..7317ce71a82 100644
--- a/docs/README.asciidoc
+++ b/docs/README.asciidoc
@@ -1,4 +1,47 @@
-The Elasticsearch docs are in AsciiDoc format and can be built using the Elasticsearch documentation build process
+The Elasticsearch docs are in AsciiDoc format and can be built using the
+Elasticsearch documentation build process.
 
 See: https://github.com/elastic/docs
 
+Snippets marked with `// AUTOSENSE` are automatically annotated with "VIEW IN
+SENSE" in the documentation and are automatically tested by the command
+`gradle :docs:check`. By default `// AUTOSENSE` snippet runs as its own isolated
+test. You can manipulate the test execution in the following ways:
+
+* `// TEST`: Explicitly marks a snippet as a test. Snippets marked this way
+are tests even if they don't have `// AUTOSENSE`.
+  * `// TEST[s/foo/bar/]`: Replace `foo` with `bar` in the test. This should be
+  used sparingly because it makes the test "lie". Sometimes, though, you can use
+  it to make the tests more clear.
+  * `// TEST[catch:foo]`: Used to expect errors in the requests. Replace `foo`
+  with `request` to expect a 400 error, for example. If the snippet contains
+  multiple requests then only the last request will expect the error.
+  * `// TEST[continued]`: Continue the test started in the last snippet. Between
+  tests the nodes are cleaned: indexes are removed, etc. This will prevent that.
+  This is really useful when you have text and snippets that work together to
+  tell the story of some use case because it merges the snippets (and thus the
+  use case) into one big test.
+  * `// TEST[skip:reason]`: Skip this test. Replace `reason` with the actual
+  reason to skip the test. Snippets without `// TEST` or `// AUTOSENSE` aren't
+  considered tests anyway but this is useful for explicitly documenting the
+  reason why the test shouldn't be run.
+  * `// TEST[setup:name]`: Run some setup code before running the snippet. This
+  is useful for creating and populating indexes used in the snippet. The setup
+  code is defined in `docs/build.gradle`.
+* `// TESTRESPONSE`: Matches this snippet against the body of the response of
+  the last test. If the response is JSON then order is ignored. With
+  `// TEST[continued]` you can make tests that contain multiple command snippets
+  and multiple response snippets.
+  * `// TESTRESPONSE[s/foo/bar/]`: Substitutions. See `// TEST[s/foo/bar]`.
+* `// TESTSETUP`: Marks this snippet as the "setup" for all other snippets in
+  this file. This is a somewhat natural way of structuring documentation. You
+  say "this is the data we use to explain this feature" then you add the
+  snippet that you mark `// TESTSETUP` and then every snippet will turn into
+  a test that runs the setup snippet first. See the "painless" docs for a file
+  that puts this to good use. This is fairly similar to `// TEST[setup:name]`
+  but rather than the setup defined in `docs/build.gradle` the setup is defined
+  right in the documentation file.
+
+Any place you can use json you can use elements like `$body.path.to.thing`
+which is replaced on the fly with the contents of the thing at `path.to.thing`
+in the last response.
diff --git a/docs/build.gradle b/docs/build.gradle
new file mode 100644
index 00000000000..be4b10b1010
--- /dev/null
+++ b/docs/build.gradle
@@ -0,0 +1,84 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.elasticsearch.gradle.SnippetsTask
+import org.elasticsearch.gradle.SnippetsTask.Snippet
+import org.elasticsearch.gradle.RestTestsFromSnippetsTask
+
+apply plugin: 'elasticsearch.rest-test'
+
+task listSnippets(type: SnippetsTask) {
+  group 'Docs'
+  description 'List each snippet'
+  perSnippet { println(it) }
+}
+
+task listAutoSenseCandidates(type: SnippetsTask) {
+  group 'Docs'
+  description 'List snippets that probably should be marked // AUTOSENSE'
+  perSnippet {
+    if (
+             it.autoSense    // Already marked, nothing to do
+          || it.testResponse // Only commands are autosense
+        ) {
+      return
+    }
+    List<String> languages = [
+      'js', 'json', // These languages should almost always be marked autosense
+      'sh', 'shell', // These are often curl commands that should be converted
+    ]
+    if (false == languages.contains(it.language)) {
+      return
+    }
+    println(it)
+  }
+}
+
+task buildRestTests(type: RestTestsFromSnippetsTask) {
+  docs = fileTree(project.projectDir) {
+    // No snippets in here!
+    exclude 'build.gradle'
+    // Remove plugins because they aren't installed during this test. Yet?
+    exclude 'plugins'
+    // This file simply doesn't pass yet. We should figure out how to fix it.
+    exclude 'reference/modules/snapshots.asciidoc'
+  }
+  Closure setupTwitter = { String name, int count ->
+    setups[name] = '''
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |'''
+    for (int i = 0; i < count; i++) {
+      setups[name] += """
+          {"index":{}}
+          {"msg": "some message with the number $i", "date": $i}"""
+    }
+  }
+  setupTwitter('twitter', 5)
+  setupTwitter('big_twitter', 120)
+}
+
+integTest {
+  cluster {
+    setting 'script.inline', 'true'
+  }
+}
diff --git a/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc b/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc
index 873e6e3102f..b012d3a9673 100644
--- a/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc
@@ -33,9 +33,7 @@ before running each example.
 
 [source,js]
 --------------------------------------------------
-DELETE test
-
-PUT /test
+PUT test
 {
   "settings": {
     "analysis": {
@@ -49,8 +47,9 @@ PUT /test
   }
 }
 
-GET /test/_analyze?analyzer=whitespace&text=foo,bar baz
+GET _cluster/health?wait_for_status=yellow
 
+GET test/_analyze?analyzer=whitespace&text=foo,bar baz
 # "foo,bar", "baz"
 --------------------------------------------------
 // AUTOSENSE
@@ -60,9 +59,7 @@ GET /test/_analyze?analyzer=whitespace&text=foo,bar baz
 
 [source,js]
 --------------------------------------------------
-DELETE test
-
-PUT /test
+PUT test
 {
   "settings": {
     "analysis": {
@@ -76,10 +73,12 @@ PUT /test
   }
 }
 
-GET /test/_analyze?analyzer=nonword&text=foo,bar baz
+GET _cluster/health?wait_for_status=yellow
+
+GET test/_analyze?analyzer=nonword&text=foo,bar baz
 # "foo,bar baz" becomes "foo", "bar", "baz"
 
-GET /test/_analyze?analyzer=nonword&text=type_1-type_4
+GET test/_analyze?analyzer=nonword&text=type_1-type_4
 # "type_1","type_4"
 --------------------------------------------------
 // AUTOSENSE
@@ -90,9 +89,7 @@ GET /test/_analyze?analyzer=nonword&text=type_1-type_4
 
 [source,js]
 --------------------------------------------------
-DELETE test
-
-PUT /test?pretty=1
+PUT test?pretty=1
 {
   "settings": {
     "analysis": {
@@ -106,7 +103,7 @@ PUT /test?pretty=1
   }
 }
 
-GET /test/_analyze?analyzer=camel&text=MooseX::FTPClass2_beta
+GET test/_analyze?analyzer=camel&text=MooseX::FTPClass2_beta
 # "moose","x","ftp","class","2","beta"
 --------------------------------------------------
 // AUTOSENSE
diff --git a/docs/reference/cluster/tasks.asciidoc b/docs/reference/cluster/tasks.asciidoc
index 66d29f2bcfd..0ac888c953f 100644
--- a/docs/reference/cluster/tasks.asciidoc
+++ b/docs/reference/cluster/tasks.asciidoc
@@ -11,9 +11,9 @@ executing on one or more nodes in the cluster.
 
 [source,js]
 --------------------------------------------------
-GET /_tasks <1>
-GET /_tasks?nodes=nodeId1,nodeId2 <2>
-GET /_tasks?nodes=nodeId1,nodeId2&actions=cluster:* <3>
+GET _tasks <1>
+GET _tasks?nodes=nodeId1,nodeId2 <2>
+GET _tasks?nodes=nodeId1,nodeId2&actions=cluster:* <3>
 --------------------------------------------------
 // AUTOSENSE
 
@@ -56,7 +56,6 @@ The result will look similar to the following:
     }
   }
 }
-
 --------------------------------------------------
 
 It is also possible to retrieve information for a particular task, or all children of a particular
@@ -64,8 +63,8 @@ tasks using the following two commands:
 
 [source,js]
 --------------------------------------------------
-GET /_tasks/taskId1
-GET /_tasks?parent_task_id=parentTaskId1
+GET _tasks/taskId:1
+GET _tasks?parent_task_id=parentTaskId:1
 --------------------------------------------------
 // AUTOSENSE
 
@@ -74,7 +73,7 @@ block for 10 seconds or until the task with id `oTUltX4IQMOUUVeiohTt8A:12345` is
 
 [source,js]
 --------------------------------------------------
-GET /_tasks/oTUltX4IQMOUUVeiohTt8A:12345?wait_for_completion=true&timeout=10s
+GET _tasks/oTUltX4IQMOUUVeiohTt8A:12345?wait_for_completion=true&timeout=10s
 --------------------------------------------------
 // AUTOSENSE
 
@@ -83,7 +82,7 @@ as the standard list tasks command.
 
 [source,js]
 --------------------------------------------------
-GET /_cat/tasks
+GET _cat/tasks
 --------------------------------------------------
 // AUTOSENSE
 
@@ -94,7 +93,7 @@ If a long-running task supports cancellation, it can be cancelled by the followi
 
 [source,js]
 --------------------------------------------------
-POST /_tasks/taskId1/_cancel
+POST _tasks/taskId:1/_cancel
 --------------------------------------------------
 // AUTOSENSE
 
@@ -104,7 +103,7 @@ nodes `nodeId1` and `nodeId2`.
 
 [source,js]
 --------------------------------------------------
-POST /_tasks/_cancel?node_id=nodeId1,nodeId2&actions=*reindex
+POST _tasks/_cancel?node_id=nodeId1,nodeId2&actions=*reindex
 --------------------------------------------------
 // AUTOSENSE
 
@@ -117,6 +116,6 @@ The following command will change the grouping to parent tasks:
 
 [source,js]
 --------------------------------------------------
-GET /_tasks?group_by=parents
+GET _tasks?group_by=parents
 --------------------------------------------------
 // AUTOSENSE
diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc
index 27ac85b9595..0acb148e82a 100644
--- a/docs/reference/docs/index_.asciidoc
+++ b/docs/reference/docs/index_.asciidoc
@@ -7,12 +7,14 @@ into the "twitter" index, under a type called "tweet" with an id of 1:
 
 [source,js]
 --------------------------------------------------
-$ curl -XPUT 'http://localhost:9200/twitter/tweet/1' -d '{
+PUT twitter/tweet/1
+{
     "user" : "kimchy",
     "post_date" : "2009-11-15T14:12:12",
     "message" : "trying out Elasticsearch"
-}'
+}
 --------------------------------------------------
+// AUTOSENSE
 
 The result of the above index operation is:
 
@@ -20,9 +22,9 @@ The result of the above index operation is:
 --------------------------------------------------
 {
     "_shards" : {
-        "total" : 10,
+        "total" : 2,
         "failed" : 0,
-        "successful" : 10
+        "successful" : 2
     },
     "_index" : "twitter",
     "_type" : "tweet",
@@ -31,6 +33,7 @@ The result of the above index operation is:
     "created" : true
 }
 --------------------------------------------------
+// TESTRESPONSE[s/"successful" : 2/"successful" : 1/]
 
 The `_shards` header provides information about the replication process of the index operation.
 
@@ -90,10 +93,13 @@ meantime (when reading in order to update, it is recommended to set
 
 [source,js]
 --------------------------------------------------
-curl -XPUT 'localhost:9200/twitter/tweet/1?version=2' -d '{
+PUT twitter/tweet/1?version=2
+{
     "message" : "elasticsearch now has versioning support, double cool!"
-}'
+}
 --------------------------------------------------
+// AUTOSENSE
+// TEST[catch: conflict]
 
 *NOTE:* versioning is completely real time, and is not affected by the
 near real time aspects of search operations. If no version is provided,
@@ -160,23 +166,27 @@ Here is an example of using the `op_type` parameter:
 
 [source,js]
 --------------------------------------------------
-$ curl -XPUT 'http://localhost:9200/twitter/tweet/1?op_type=create' -d '{
+PUT twitter/tweet/1?op_type=create
+{
     "user" : "kimchy",
     "post_date" : "2009-11-15T14:12:12",
     "message" : "trying out Elasticsearch"
-}'
+}
 --------------------------------------------------
+// AUTOSENSE
 
 Another option to specify `create` is to use the following uri:
 
 [source,js]
 --------------------------------------------------
-$ curl -XPUT 'http://localhost:9200/twitter/tweet/1/_create' -d '{
+PUT twitter/tweet/1/_create
+{
     "user" : "kimchy",
     "post_date" : "2009-11-15T14:12:12",
     "message" : "trying out Elasticsearch"
-}'
+}
 --------------------------------------------------
+// AUTOSENSE
 
 [float]
 === Automatic ID Generation
@@ -188,18 +198,25 @@ will automatically be set to `create`. Here is an example (note the
 
 [source,js]
 --------------------------------------------------
-$ curl -XPOST 'http://localhost:9200/twitter/tweet/' -d '{
+POST twitter/tweet/
+{
     "user" : "kimchy",
     "post_date" : "2009-11-15T14:12:12",
     "message" : "trying out Elasticsearch"
-}'
+}
 --------------------------------------------------
+// AUTOSENSE
 
 The result of the above index operation is:
 
 [source,js]
 --------------------------------------------------
 {
+    "_shards" : {
+        "total" : 2,
+        "failed" : 0,
+        "successful" : 2
+    },
     "_index" : "twitter",
     "_type" : "tweet",
     "_id" : "6a8ca01c-7896-48e9-81cc-9f70661fcb32",
@@ -207,6 +224,7 @@ The result of the above index operation is:
     "created" : true
 }
 --------------------------------------------------
+// TESTRESPONSE[s/6a8ca01c-7896-48e9-81cc-9f70661fcb32/$body._id/ s/"successful" : 2/"successful" : 1/]
 
 [float]
 [[index-routing]]
@@ -219,12 +237,14 @@ on a per-operation basis using the `routing` parameter. For example:
 
 [source,js]
 --------------------------------------------------
-$ curl -XPOST 'http://localhost:9200/twitter/tweet?routing=kimchy' -d '{
+POST twitter/tweet?routing=kimchy
+{
     "user" : "kimchy",
     "post_date" : "2009-11-15T14:12:12",
     "message" : "trying out Elasticsearch"
-}'
+}
 --------------------------------------------------
+// AUTOSENSE
 
 In the example above, the "tweet" document is routed to a shard based on
 the `routing` parameter provided: "kimchy".
@@ -245,10 +265,24 @@ For example:
 
 [source,js]
 --------------------------------------------------
-$ curl -XPUT localhost:9200/blogs/blog_tag/1122?parent=1111 -d '{
+PUT blogs
+{
+  "mappings": {
+    "tag_parent": {},
+    "blog_tag": {
+      "_parent": {
+        "type": "tag_parent"
+      }
+    }
+  }
+}
+
+PUT blogs/blog_tag/1122?parent=1111
+{
     "tag" : "something"
-}'
+}
 --------------------------------------------------
+// AUTOSENSE
 
 When indexing a child document, the routing value is automatically set
 to be the same as its parent, unless the routing value is explicitly
@@ -266,11 +300,13 @@ parameter. For example:
 
 [source,js]
 --------------------------------------------------
-$ curl -XPUT localhost:9200/twitter/tweet/1?timestamp=2009-11-15T14%3A12%3A12 -d '{
+PUT twitter/tweet/1?timestamp=2009-11-15T14:12:12
+{
     "user" : "kimchy",
     "message" : "trying out Elasticsearch"
-}'
+}
 --------------------------------------------------
+// AUTOSENSE
 
 If the `timestamp` value is not provided externally or in the `_source`,
 the `timestamp` will be automatically set to the date the document was
@@ -295,28 +331,23 @@ as shown in the following examples:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT 'http://localhost:9200/twitter/tweet/1?ttl=86400000' -d '{
+PUT twitter/tweet/1?ttl=86400000ms
+{
     "user": "kimchy",
     "message": "Trying out elasticsearch, so far so good?"
-}'
+}
 --------------------------------------------------
+// AUTOSENSE
 
 [source,js]
 --------------------------------------------------
-curl -XPUT 'http://localhost:9200/twitter/tweet/1?ttl=1d' -d '{
+PUT twitter/tweet/1?ttl=1d
+{
     "user": "kimchy",
     "message": "Trying out elasticsearch, so far so good?"
-}'
---------------------------------------------------
-
-[source,js]
---------------------------------------------------
-curl -XPUT 'http://localhost:9200/twitter/tweet/1' -d '{
-    "_ttl": "1d",
-    "user": "kimchy",
-    "message": "Trying out elasticsearch, so far so good?"
-}'
+}
 --------------------------------------------------
+// AUTOSENSE
 
 More information can be found on the
 <<mapping-ttl-field,_ttl mapping page>>.
@@ -392,9 +423,11 @@ to 5 minutes:
 
 [source,js]
 --------------------------------------------------
-$ curl -XPUT 'http://localhost:9200/twitter/tweet/1?timeout=5m' -d '{
+PUT twitter/tweet/1?timeout=5m
+{
     "user" : "kimchy",
     "post_date" : "2009-11-15T14:12:12",
     "message" : "trying out Elasticsearch"
-}'
+}
 --------------------------------------------------
+// AUTOSENSE
diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index 8742e52c5c8..513281c07e3 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -8,7 +8,7 @@ This will copy documents from the `twitter` index into the `new_twitter` index:
 
 [source,js]
 --------------------------------------------------
-POST /_reindex
+POST _reindex
 {
   "source": {
     "index": "twitter"
@@ -19,20 +19,29 @@ POST /_reindex
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[setup:big_twitter]
 
 That will return something like this:
 
 [source,js]
 --------------------------------------------------
 {
-  "took" : 639,
-  "updated": 112,
-  "batches": 130,
+  "took" : 147,
+  "timed_out": false,
+  "created": 120,
+  "updated": 0,
+  "batches": 2,
   "version_conflicts": 0,
-  "failures" : [ ],
-  "created": 12344
+  "noops": 0,
+  "retries": 0,
+  "throttled_millis": 0,
+  "requests_per_second": "unlimited",
+  "throttled_until_millis": 0,
+  "total": 120,
+  "failures" : [ ]
 }
 --------------------------------------------------
+// TESTRESPONSE[s/"took" : 147/"took" : "$body.took"/]
 
 Just like <<docs-update-by-query,`_update_by_query`>>, `_reindex` gets a
 snapshot of the source index but its target must be a **different** index so
@@ -44,7 +53,7 @@ the same type and id:
 
 [source,js]
 --------------------------------------------------
-POST /_reindex
+POST _reindex
 {
   "source": {
     "index": "twitter"
@@ -56,6 +65,7 @@ POST /_reindex
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[setup:twitter]
 
 Setting `version_type` to `external` will cause Elasticsearch to preserve the
 `version` from the source, create any documents that are missing, and update
@@ -64,7 +74,7 @@ in the source index:
 
 [source,js]
 --------------------------------------------------
-POST /_reindex
+POST _reindex
 {
   "source": {
     "index": "twitter"
@@ -76,6 +86,7 @@ POST /_reindex
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[setup:twitter]
 
 Settings `op_type` to `create` will cause `_reindex` to only create missing
 documents in the target index. All existing documents will cause a version
@@ -83,7 +94,7 @@ conflict:
 
 [source,js]
 --------------------------------------------------
-POST /_reindex
+POST _reindex
 {
   "source": {
     "index": "twitter"
@@ -95,13 +106,14 @@ POST /_reindex
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[setup:twitter]
 
 By default version conflicts abort the `_reindex` process but you can just
 count them by settings `"conflicts": "proceed"` in the request body:
 
 [source,js]
 --------------------------------------------------
-POST /_reindex
+POST _reindex
 {
   "conflicts": "proceed",
   "source": {
@@ -114,13 +126,14 @@ POST /_reindex
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[setup:twitter]
 
 You can limit the documents by adding a type to the `source` or by adding a
 query. This will only copy ++tweet++&apos;s made by `kimchy` into `new_twitter`:
 
 [source,js]
 --------------------------------------------------
-POST /_reindex
+POST _reindex
 {
   "source": {
     "index": "twitter",
@@ -137,6 +150,7 @@ POST /_reindex
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[setup:twitter]
 
 `index` and `type` in `source` can both be lists, allowing you to copy from
 lots of sources in one request. This will copy documents from the `tweet` and
@@ -148,7 +162,7 @@ which document will survive because the iteration order isn't well defined.
 
 [source,js]
 --------------------------------------------------
-POST /_reindex
+POST _reindex
 {
   "source": {
     "index": ["twitter", "blog"],
@@ -160,6 +174,7 @@ POST /_reindex
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[s/^/PUT twitter\nPUT blog\n/]
 
 It's also possible to limit the number of processed documents by setting
 `size`. This will only copy a single document from `twitter` to
@@ -167,7 +182,7 @@ It's also possible to limit the number of processed documents by setting
 
 [source,js]
 --------------------------------------------------
-POST /_reindex
+POST _reindex
 {
   "size": 1,
   "source": {
@@ -179,6 +194,7 @@ POST /_reindex
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[setup:twitter]
 
 If you want a particular set of documents from the twitter index you'll
 need to sort. Sorting makes the scroll less efficient but in some contexts
@@ -187,7 +203,7 @@ This will copy 10000 documents from `twitter` into `new_twitter`:
 
 [source,js]
 --------------------------------------------------
-POST /_reindex
+POST _reindex
 {
   "size": 10000,
   "source": {
@@ -200,6 +216,7 @@ POST /_reindex
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[setup:twitter]
 
 Like `_update_by_query`, `_reindex` supports a script that modifies the
 document. Unlike `_update_by_query`, the script is allowed to modify the
@@ -207,21 +224,22 @@ document's metadata. This example bumps the version of the source document:
 
 [source,js]
 --------------------------------------------------
-POST /_reindex
+POST _reindex
 {
   "source": {
-    "index": "twitter",
+    "index": "twitter"
   },
   "dest": {
     "index": "new_twitter",
     "version_type": "external"
-  }
+  },
   "script": {
-    "internal": "if (ctx._source.foo == 'bar') {ctx._version++; ctx._source.remove('foo')}"
+    "script": "if (ctx._source.foo == 'bar') {ctx._version++; ctx._source.remove('foo')}"
   }
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[setup:twitter]
 
 Think of the possibilities! Just be careful! With great power.... You can
 change:
@@ -264,7 +282,7 @@ routing set to `cat`.
 
 [source,js]
 --------------------------------------------------
-POST /_reindex
+POST _reindex
 {
   "source": {
     "index": "source",
@@ -281,13 +299,14 @@ POST /_reindex
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[s/^/PUT source\n/]
 
 By default `_reindex` uses scroll batches of 100. You can change the
 batch size with the `size` field in the `source` element:
 
 [source,js]
 --------------------------------------------------
-POST /_reindex
+POST _reindex
 {
   "source": {
     "index": "source",
@@ -300,13 +319,14 @@ POST /_reindex
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[s/^/PUT source\n/]
 
 Reindex can also use the <<ingest>> feature by specifying a
 `pipeline` like this:
 
 [source,js]
 --------------------------------------------------
-POST /_reindex
+POST _reindex
 {
   "source": {
     "index": "source"
@@ -318,6 +338,7 @@ POST /_reindex
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[s/^/PUT source\n/]
 
 [float]
 === URL Parameters
@@ -414,7 +435,7 @@ While Reindex is running you can fetch their status using the
 
 [source,js]
 --------------------------------------------------
-GET /_tasks/?pretty&detailed=true&actions=*reindex
+GET _tasks/?pretty&detailed=true&actions=*reindex
 --------------------------------------------------
 // AUTOSENSE
 
@@ -473,7 +494,7 @@ Any Reindex can be canceled using the <<tasks,Task Cancel API>>:
 
 [source,js]
 --------------------------------------------------
-POST /_tasks/{task_id}/_cancel
+POST _tasks/taskid:1/_cancel
 --------------------------------------------------
 // AUTOSENSE
 
@@ -492,7 +513,7 @@ the `_rethrottle` API:
 
 [source,js]
 --------------------------------------------------
-POST /_reindex/{task_id}/_rethrottle?requests_per_second=unlimited
+POST _reindex/taskid:1/_rethrottle?requests_per_second=unlimited
 --------------------------------------------------
 // AUTOSENSE
 
@@ -540,6 +561,7 @@ POST _reindex?pretty
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[continued]
 
 Now you can get the new document:
 
@@ -548,15 +570,24 @@ Now you can get the new document:
 GET test2/test/1?pretty
 --------------------------------------------------
 // AUTOSENSE
+// TEST[continued]
 
 and it'll look like:
 
 [source,js]
 --------------------------------------------------
 {
-  "text": "words words",
-  "tag": "foo"
+  "found": true,
+  "_id": "1",
+  "_index": "test2",
+  "_type": "test",
+  "_version": 1,
+  "_source": {
+    "text": "words words",
+    "tag": "foo"
+  }
 }
 --------------------------------------------------
+// TESTRESPONSE
 
 Or you can search by `tag` or whatever you want.
diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc
index e307d53e8f4..cb0c806f3a4 100644
--- a/docs/reference/docs/update-by-query.asciidoc
+++ b/docs/reference/docs/update-by-query.asciidoc
@@ -10,22 +10,31 @@ mapping change. Here is the API:
 
 [source,js]
 --------------------------------------------------
-POST /twitter/_update_by_query?conflicts=proceed
+POST twitter/_update_by_query?conflicts=proceed
 --------------------------------------------------
 // AUTOSENSE
+// TEST[setup:big_twitter]
 
 That will return something like this:
 
 [source,js]
 --------------------------------------------------
 {
-  "took" : 639,
-  "updated": 1235,
-  "batches": 13,
-  "version_conflicts": 2,
+  "took" : 147,
+  "timed_out": false,
+  "updated": 120,
+  "batches": 2,
+  "version_conflicts": 0,
+  "noops": 0,
+  "retries": 0,
+  "throttled_millis": 0,
+  "requests_per_second": "unlimited",
+  "throttled_until_millis": 0,
+  "total": 120,
   "failures" : [ ]
 }
 --------------------------------------------------
+// TESTRESPONSE[s/"took" : 147/"took" : "$body.took"/]
 
 `_update_by_query` gets a snapshot of the index when it starts and indexes what
 it finds using `internal` versioning. That means that you'll get a version
@@ -53,9 +62,10 @@ will only update `tweet`s from the `twitter` index:
 
 [source,js]
 --------------------------------------------------
-POST /twitter/tweet/_update_by_query?conflicts=proceed
+POST twitter/tweet/_update_by_query?conflicts=proceed
 --------------------------------------------------
 // AUTOSENSE
+// TEST[setup:twitter]
 
 You can also limit `_update_by_query` using the
 <<query-dsl,Query DSL>>. This will update all documents from the
@@ -63,7 +73,7 @@ You can also limit `_update_by_query` using the
 
 [source,js]
 --------------------------------------------------
-POST /twitter/_update_by_query?conflicts=proceed
+POST twitter/_update_by_query?conflicts=proceed
 {
   "query": { <1>
     "term": {
@@ -73,6 +83,7 @@ POST /twitter/_update_by_query?conflicts=proceed
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[setup:twitter]
 
 <1> The query must be passed as a value to the `query` key, in the same
 way as the <<search-search,Search API>>. You can also use the `q`
@@ -86,7 +97,7 @@ will increment the `likes` field on all of kimchy's tweets:
 
 [source,js]
 --------------------------------------------------
-POST /twitter/_update_by_query
+POST twitter/_update_by_query
 {
   "script": {
     "inline": "ctx._source.likes++"
@@ -99,6 +110,7 @@ POST /twitter/_update_by_query
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[setup:twitter]
 
 Just as in <<docs-update,Update API>> you can set `ctx.op = "noop"` if
 your script decides that it doesn't have to make any changes. That will cause
@@ -119,36 +131,50 @@ types at once, just like the search API:
 
 [source,js]
 --------------------------------------------------
-POST /twitter,blog/tweet,post/_update_by_query
+POST twitter,blog/tweet,post/_update_by_query
 --------------------------------------------------
 // AUTOSENSE
+// TEST[s/^/PUT twitter\nPUT blog\n/]
 
 If you provide `routing` then the routing is copied to the scroll query,
 limiting the process to the shards that match that routing value:
 
 [source,js]
 --------------------------------------------------
-POST /twitter/_update_by_query?routing=1
+POST twitter/_update_by_query?routing=1
 --------------------------------------------------
 // AUTOSENSE
+// TEST[setup:twitter]
 
 By default `_update_by_query` uses scroll batches of 100. You can change the
 batch size with the `scroll_size` URL parameter:
 
 [source,js]
 --------------------------------------------------
-POST /twitter/_update_by_query?scroll_size=1000
+POST twitter/_update_by_query?scroll_size=1000
 --------------------------------------------------
 // AUTOSENSE
+// TEST[setup:twitter]
 
 `_update_by_query` can also use the <<ingest>> feature by
 specifying a `pipeline` like this:
 
 [source,js]
 --------------------------------------------------
-POST /twitter/_update_by_query?pipeline=some_ingest_pipeline
+PUT _ingest/pipeline/set-foo
+{
+  "description" : "sets foo",
+  "processors" : [ {
+      "set" : {
+        "field": "foo",
+        "value": "bar"
+      }
+  } ]
+}
+POST twitter/_update_by_query?pipeline=set-foo
 --------------------------------------------------
 // AUTOSENSE
+// TEST[setup:twitter]
 
 [float]
 === URL Parameters
@@ -240,7 +266,7 @@ While Update By Query is running you can fetch their status using the
 
 [source,js]
 --------------------------------------------------
-POST /_tasks/?pretty&detailed=true&action=*byquery
+GET _tasks/?pretty&detailed=true&action=*byquery
 --------------------------------------------------
 // AUTOSENSE
 
@@ -299,7 +325,7 @@ Any Update By Query can be canceled using the <<tasks,Task Cancel API>>:
 
 [source,js]
 --------------------------------------------------
-POST /_tasks/{task_id}/_cancel
+POST _tasks/taskid:1/_cancel
 --------------------------------------------------
 // AUTOSENSE
 
@@ -318,7 +344,7 @@ using the `_rethrottle` API:
 
 [source,js]
 --------------------------------------------------
-POST /_update_by_query/{task_id}/_rethrottle?requests_per_second=unlimited
+POST _update_by_query/taskid:1/_rethrottle?requests_per_second=unlimited
 --------------------------------------------------
 // AUTOSENSE
 
@@ -356,12 +382,12 @@ POST test/test?refresh
 {
   "text": "words words",
   "flag": "bar"
-}'
+}
 POST test/test?refresh
 {
   "text": "words words",
   "flag": "foo"
-}'
+}
 PUT test/_mapping/test   <2>
 {
   "properties": {
@@ -391,6 +417,7 @@ POST test/_search?filter_path=hits.total
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[continued]
 
 [source,js]
 --------------------------------------------------
@@ -400,6 +427,7 @@ POST test/_search?filter_path=hits.total
   }
 }
 --------------------------------------------------
+// TESTRESPONSE
 
 But you can issue an `_update_by_query` request to pick up the new mapping:
 
@@ -416,6 +444,7 @@ POST test/_search?filter_path=hits.total
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[continued]
 
 [source,js]
 --------------------------------------------------
@@ -425,5 +454,6 @@ POST test/_search?filter_path=hits.total
   }
 }
 --------------------------------------------------
+// TESTRESPONSE
 
 You can do the exact same thing when adding a field to a multifield.
diff --git a/docs/reference/index-modules/allocation/delayed.asciidoc b/docs/reference/index-modules/allocation/delayed.asciidoc
index baaa3cb944d..6bedba95586 100644
--- a/docs/reference/index-modules/allocation/delayed.asciidoc
+++ b/docs/reference/index-modules/allocation/delayed.asciidoc
@@ -38,7 +38,7 @@ This setting can be updated on a live index (or on all indices):
 
 [source,js]
 ------------------------------
-PUT /_all/_settings
+PUT _all/_settings
 {
   "settings": {
     "index.unassigned.node_left.delayed_timeout": "5m"
@@ -46,6 +46,7 @@ PUT /_all/_settings
 }
 ------------------------------
 // AUTOSENSE
+// TEST[s/^/PUT test\n/]
 
 With delayed allocation enabled, the above scenario changes to look like this:
 
@@ -82,6 +83,7 @@ can be viewed with the <<cluster-health,cluster health API>>:
 ------------------------------
 GET _cluster/health <1>
 ------------------------------
+// AUTOSENSE
 <1> This request will return a `delayed_unassigned_shards` value.
 
 ==== Removing a node permanently
@@ -92,7 +94,7 @@ the missing shards immediately, just update the timeout to zero:
 
 [source,js]
 ------------------------------
-PUT /_all/_settings
+PUT _all/_settings
 {
   "settings": {
     "index.unassigned.node_left.delayed_timeout": "0"
@@ -100,5 +102,6 @@ PUT /_all/_settings
 }
 ------------------------------
 // AUTOSENSE
+// TEST[s/^/PUT test\n/]
 
 You can reset the timeout as soon as the missing shards have started to recover.
diff --git a/docs/reference/index-modules/allocation/filtering.asciidoc b/docs/reference/index-modules/allocation/filtering.asciidoc
index 29ce2f535a3..f5b73a9f78e 100644
--- a/docs/reference/index-modules/allocation/filtering.asciidoc
+++ b/docs/reference/index-modules/allocation/filtering.asciidoc
@@ -23,6 +23,7 @@ These metadata attributes can be used with the
 group of nodes.  For instance, we can move the index `test` to either `big` or
 `medium` nodes as follows:
 
+
 [source,js]
 ------------------------
 PUT test/_settings
@@ -31,6 +32,7 @@ PUT test/_settings
 }
 ------------------------
 // AUTOSENSE
+// TEST[s/^/PUT test\n/]
 
 Alternatively, we can move the index `test` away from the `small` nodes with
 an `exclude` rule:
@@ -43,6 +45,7 @@ PUT test/_settings
 }
 ------------------------
 // AUTOSENSE
+// TEST[s/^/PUT test\n/]
 
 Multiple rules can be specified, in which case all conditions must be
 satisfied.  For instance, we could move the index `test` to `big` nodes in
@@ -57,6 +60,7 @@ PUT test/_settings
 }
 ------------------------
 // AUTOSENSE
+// TEST[s/^/PUT test\n/]
 
 NOTE: If some conditions cannot be satisfied then shards will not be moved.
 
@@ -97,3 +101,4 @@ PUT test/_settings
 }
 ------------------------
 // AUTOSENSE
+// TEST[skip:indexes don't assign]
diff --git a/docs/reference/index-modules/allocation/prioritization.asciidoc b/docs/reference/index-modules/allocation/prioritization.asciidoc
index b3307e90b79..149e43fd134 100644
--- a/docs/reference/index-modules/allocation/prioritization.asciidoc
+++ b/docs/reference/index-modules/allocation/prioritization.asciidoc
@@ -53,3 +53,4 @@ PUT index_4/_settings
 }
 ------------------------------
 // AUTOSENSE
+// TEST[continued]
diff --git a/docs/reference/indices/analyze.asciidoc b/docs/reference/indices/analyze.asciidoc
index 47d73b7d3dc..3597ccc51e5 100644
--- a/docs/reference/indices/analyze.asciidoc
+++ b/docs/reference/indices/analyze.asciidoc
@@ -112,7 +112,7 @@ experimental[The format of the additional detail information is experimental and
 
 [source,js]
 --------------------------------------------------
-GET test/_analyze
+GET _analyze
 {
   "tokenizer" : "standard",
   "token_filter" : ["snowball"],
@@ -172,4 +172,3 @@ The request returns the following result:
 }
 --------------------------------------------------
 <1> Output only "keyword" attribute, since specify "attributes" in the request.
-
diff --git a/docs/reference/indices/flush.asciidoc b/docs/reference/indices/flush.asciidoc
index b02fb15135b..0279ee3abcf 100644
--- a/docs/reference/indices/flush.asciidoc
+++ b/docs/reference/indices/flush.asciidoc
@@ -10,9 +10,10 @@ trigger flush operations as required in order to clear memory.
 
 [source,js]
 --------------------------------------------------
-POST /twitter/_flush
+POST twitter/_flush
 --------------------------------------------------
 // AUTOSENSE
+// TEST[setup:twitter]
 
 [float]
 [[flush-parameters]]
@@ -23,7 +24,7 @@ The flush API accepts the following request parameters:
 [horizontal]
 `wait_if_ongoing`::  If set to `true` the flush operation will block until the
 flush can be executed if another flush operation is already executing.
-The default is `false` and will cause an exception to be thrown on 
+The default is `false` and will cause an exception to be thrown on
 the shard level if another flush operation is already running.
 
 `force`:: Whether a flush should be forced even if it is not necessarily needed ie.
@@ -40,11 +41,12 @@ or even on `_all` the indices.
 
 [source,js]
 --------------------------------------------------
-POST /kimchy,elasticsearch/_flush
+POST kimchy,elasticsearch/_flush
 
-POST /_flush
+POST _flush
 --------------------------------------------------
 // AUTOSENSE
+// TEST[s/^/PUT kimchy\nPUT elasticsearch\n/]
 
 [[indices-synced-flush]]
 === Synced Flush
@@ -72,9 +74,10 @@ the <<indices-stats,indices stats>> API:
 
 [source,sh]
 --------------------------------------------------
-GET /twitter/_stats/commit?level=shards
+GET twitter/_stats/commit?level=shards
 --------------------------------------------------
 // AUTOSENSE
+// TEST[s/^/PUT twitter\n/]
 
 
 which returns something similar to:
@@ -136,9 +139,10 @@ NOTE: It is harmless to request a synced flush while there is ongoing indexing.
 
 [source,sh]
 --------------------------------------------------
-POST /twitter/_flush/synced
+POST twitter/_flush/synced
 --------------------------------------------------
 // AUTOSENSE
+// TEST[setup:twitter]
 
 The response contains details about how many shards were successfully sync-flushed and information about any failure.
 
@@ -149,18 +153,18 @@ sync-flushed:
 --------------------------------------------------
 {
    "_shards": {
-      "total": 4,
-      "successful": 4,
+      "total": 10,
+      "successful": 10,
       "failed": 0
    },
    "twitter": {
-      "total": 4,
-      "successful": 4,
+      "total": 10,
+      "successful": 10,
       "failed": 0
    }
 }
 --------------------------------------------------
-
+// TESTRESPONSE[s/"successful": 10/"successful": 5/]
 
 Here is what it looks like when one shard group failed due to pending operations:
 
@@ -230,8 +234,8 @@ or even on `_all` the indices.
 
 [source,js]
 --------------------------------------------------
-POST /kimchy,elasticsearch/_flush/synced
+POST kimchy,elasticsearch/_flush/synced
 
-POST /_flush/synced
+POST _flush/synced
 --------------------------------------------------
-// AUTOSENSE
\ No newline at end of file
+// AUTOSENSE
diff --git a/docs/reference/indices/put-mapping.asciidoc b/docs/reference/indices/put-mapping.asciidoc
index cc94a08f626..70a5aa8bef0 100644
--- a/docs/reference/indices/put-mapping.asciidoc
+++ b/docs/reference/indices/put-mapping.asciidoc
@@ -135,9 +135,9 @@ exists in more than one type will throw an exception, unless you specify the
 across all fields with the same name in the same index.
 
 TIP: The only parameters which are exempt from this rule -- they can be set to
-different values on each field -- can be found in  <<field-conflicts>>.
+different values on each field -- can be found in <<field-conflicts>>.
 
-For example:
+For example, this fails:
 
 [source,js]
 -----------------------------------
@@ -173,8 +173,17 @@ PUT my_index/_mapping/type_one <2>
     }
   }
 }
+-----------------------------------
+// AUTOSENSE
+// TEST[catch:request]
+<1> Create an index with two types, both of which contain a `text` field which have the same mapping.
+<2> Trying to update the `search_analyzer` just for `type_one` throws an exception like `"Merge failed with failures..."`.
 
-PUT my_index/_mapping/type_one?update_all_types <3>
+But this then running this succeeds:
+
+[source,js]
+-----------------------------------
+PUT my_index/_mapping/type_one?update_all_types <1>
 {
   "properties": {
     "text": {
@@ -186,8 +195,5 @@ PUT my_index/_mapping/type_one?update_all_types <3>
 }
 -----------------------------------
 // AUTOSENSE
-<1> Create an index with two types, both of which contain a `text` field which have the same mapping.
-<2> Trying to update the `search_analyzer` just for `type_one` throws an exception like `"Merge failed with failures..."`.
-<3> Adding the `update_all_types` parameter updates the `text` field in `type_one` and `type_two`.
-
-
+// TEST[continued]
+<1> Adding the `update_all_types` parameter updates the `text` field in `type_one` and `type_two`.
diff --git a/docs/reference/indices/templates.asciidoc b/docs/reference/indices/templates.asciidoc
index 9e97b0c5240..86bc77c52e4 100644
--- a/docs/reference/indices/templates.asciidoc
+++ b/docs/reference/indices/templates.asciidoc
@@ -13,7 +13,7 @@ For example:
 
 [source,js]
 --------------------------------------------------
-PUT /_template/template_1
+PUT _template/template_1
 {
   "template": "te*",
   "settings": {
diff --git a/docs/reference/ingest.asciidoc b/docs/reference/ingest.asciidoc
index c565f3b2047..767b928d205 100644
--- a/docs/reference/ingest.asciidoc
+++ b/docs/reference/ingest.asciidoc
@@ -25,15 +25,16 @@ tell the ingest node which pipeline to use. For example:
 
 [source,js]
 --------------------------------------------------
-PUT /my-index/my-type/my-id?pipeline=my_pipeline_id
+PUT my-index/my-type/my-id?pipeline=my_pipeline_id
 {
-  ...
+  "foo": "bar"
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[catch:request]
 
 See <<ingest-apis,Ingest APIs>> for more information about creating, adding, and deleting pipelines.
 
 --
 
-include::ingest/ingest-node.asciidoc[]
\ No newline at end of file
+include::ingest/ingest-node.asciidoc[]
diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc
index 7831ee86290..631e4bb40f7 100644
--- a/docs/reference/ingest/ingest-node.asciidoc
+++ b/docs/reference/ingest/ingest-node.asciidoc
@@ -41,10 +41,11 @@ PUT _ingest/pipeline/my-pipeline-id
   "description" : "describe pipeline",
   "processors" : [
     {
-      "simple" : {
-        // settings
+      "set" : {
+        "field": "foo",
+        "value": "bar"
       }
-    },
+    }
     // other processors
   ]
 }
@@ -64,28 +65,31 @@ The get pipeline API returns pipelines based on ID. This API always returns a lo
 GET _ingest/pipeline/my-pipeline-id
 --------------------------------------------------
 // AUTOSENSE
+// TEST[continued]
 
 Example response:
 
 [source,js]
 --------------------------------------------------
 {
-   "my-pipeline-id": {
-      "_source" : {
-        "description": "describe pipeline",
-        "processors": [
-          {
-            "simple" : {
-              // settings
-            }
-          },
-          // other processors
-        ]
-      },
-      "_version" : 0
-   }
+  "pipelines": [ {
+    "id": "my-pipeline-id",
+    "config": {
+      "description": "describe pipeline",
+      "processors": [
+        {
+          "set" : {
+            "field": "foo",
+            "value": "bar"
+          }
+        }
+        // other processors
+      ]
+    }
+  } ]
 }
 --------------------------------------------------
+// TESTRESPONSE
 
 For each returned pipeline, the source and the version are returned.
 The version is useful for knowing which version of the pipeline the node has.
@@ -101,6 +105,7 @@ The delete pipeline API deletes pipelines by ID.
 DELETE _ingest/pipeline/my-pipeline-id
 --------------------------------------------------
 // AUTOSENSE
+// TEST[continued]
 
 [[simulate-pipeline-api]]
 === Simulate Pipeline API
@@ -674,9 +679,9 @@ Specifying `boolean` will set the field to true if its string value is equal to
 false if its string value is equal to `false` (ignore case), or it will throw an exception otherwise.
 
 Specifying `auto` will attempt to convert the string-valued `field` into the closest non-string type.
-For example, a field whose value is `"true"` will be converted to its respective boolean type: `true`. And 
-a value of `"242.15"` will "automatically" be converted to `242.15` of type `float`. If a provided field cannot 
-be appropriately converted, the Convert Processor will still process successfully and leave the field value as-is. In 
+For example, a field whose value is `"true"` will be converted to its respective boolean type: `true`. And
+a value of `"242.15"` will "automatically" be converted to `242.15` of type `float`. If a provided field cannot
+be appropriately converted, the Convert Processor will still process successfully and leave the field value as-is. In
 such a case, `target_field` will still be updated with the unconverted field value.
 
 [[convert-options]]
@@ -1320,5 +1325,3 @@ Converts a string to its uppercase equivalent.
   }
 }
 --------------------------------------------------
-
-
diff --git a/docs/reference/mapping.asciidoc b/docs/reference/mapping.asciidoc
index 8ead0436978..70de0a2735b 100644
--- a/docs/reference/mapping.asciidoc
+++ b/docs/reference/mapping.asciidoc
@@ -140,6 +140,7 @@ PUT my_index <1>
       }
     },
     "blogpost": { <2>
+      "_all":       { "enabled": false  }, <3>
       "properties": { <4>
         "title":    { "type": "text"  }, <5>
         "body":     { "type": "text"  }, <5>
diff --git a/docs/reference/mapping/dynamic-mapping.asciidoc b/docs/reference/mapping/dynamic-mapping.asciidoc
index beb7d4360d4..81aa5858db3 100644
--- a/docs/reference/mapping/dynamic-mapping.asciidoc
+++ b/docs/reference/mapping/dynamic-mapping.asciidoc
@@ -46,12 +46,14 @@ setting to `false`, either by setting the default value in the
 
 [source,js]
 --------------------------------------------------
-PUT /_settings <1>
+PUT data/_settings <1>
 {
   "index.mapper.dynamic":false
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[continued]
+
 <1> Disable automatic type creation for all indices.
 
 Regardless of the value of this setting, types can still be added explicitly
@@ -64,4 +66,3 @@ include::dynamic/default-mapping.asciidoc[]
 include::dynamic/field-mapping.asciidoc[]
 
 include::dynamic/templates.asciidoc[]
-
diff --git a/docs/reference/mapping/dynamic/templates.asciidoc b/docs/reference/mapping/dynamic/templates.asciidoc
index 315ed05d759..6396abe8522 100644
--- a/docs/reference/mapping/dynamic/templates.asciidoc
+++ b/docs/reference/mapping/dynamic/templates.asciidoc
@@ -347,7 +347,7 @@ PUT my_index
 --------------------------------------------------
 
 The sub `keyword` field appears in this template to be consistent with the
-default rules of dynamic mappings. Of course if you do not need them because 
+default rules of dynamic mappings. Of course if you do not need them because
 you don't need to perform exact search or aggregate on this field, you could
 remove it as described in the previous section.
 
@@ -406,14 +406,12 @@ new indices, you could create the following index template:
 --------------------------------------------------
 PUT _template/disable_all_field
 {
-  "disable_all_field": {
-    "order": 0,
-    "template": "*", <1>
-    "mappings": {
-      "_default_": { <2>
-        "_all": { <3>
-          "enabled": false
-        }
+  "order": 0,
+  "template": "*", <1>
+  "mappings": {
+    "_default_": { <2>
+      "_all": { <3>
+        "enabled": false
       }
     }
   }
diff --git a/docs/reference/mapping/fields/all-field.asciidoc b/docs/reference/mapping/fields/all-field.asciidoc
index 6c5f073aee8..58f0e783af4 100644
--- a/docs/reference/mapping/fields/all-field.asciidoc
+++ b/docs/reference/mapping/fields/all-field.asciidoc
@@ -116,6 +116,7 @@ PUT my_index
 }
 --------------------------------
 // AUTOSENSE
+// TEST[s/\.\.\.//]
 
 <1> The `_all` field in `type_1` is enabled.
 <2> The `_all` field in `type_2` is completely disabled.
@@ -143,7 +144,7 @@ PUT my_index
   },
   "settings": {
     "index.query.default_field": "content" <2>
-  },
+  }
 }
 --------------------------------
 // AUTOSENSE
@@ -337,7 +338,7 @@ GET _search
   "highlight": {
     "fields": {
       "*_name": { <2>
-        "require_field_match": "false"  <3>
+        "require_field_match": false  <3>
       }
     }
   }
@@ -348,5 +349,3 @@ GET _search
 <1> The query inspects the `_all` field to find matching documents.
 <2> Highlighting is performed on the two name fields, which are available from the `_source`.
 <3> The query wasn't run against the name fields, so set `require_field_match` to `false`.
-
-
diff --git a/docs/reference/mapping/fields/field-names-field.asciidoc b/docs/reference/mapping/fields/field-names-field.asciidoc
index bafc3e3f7d9..ebb228fca94 100644
--- a/docs/reference/mapping/fields/field-names-field.asciidoc
+++ b/docs/reference/mapping/fields/field-names-field.asciidoc
@@ -6,8 +6,7 @@ contains any value other than `null`.  This field is used by the
 <<query-dsl-exists-query,`exists`>> query to find documents that
 either have or don't have any non-+null+ value for a particular field.
 
-The value of the `_field_name` field is accessible in queries, aggregations, and
-scripts:
+The value of the `_field_name` field is accessible in queries and scripts:
 
 [source,js]
 --------------------------
@@ -30,17 +29,9 @@ GET my_index/_search
       "_field_names": [ "title" ] <1>
     }
   },
-  "aggs": {
-    "Field names": {
-      "terms": {
-        "field": "_field_names", <2>
-        "size": 10
-      }
-    }
-  },
   "script_fields": {
     "Field names": {
-      "script": "doc['_field_names']" <3>
+      "script": "doc['_field_names']" <2>
     }
   }
 }
@@ -49,5 +40,4 @@ GET my_index/_search
 // AUTOSENSE
 
 <1> Querying on the `_field_names` field (also see the <<query-dsl-exists-query,`exists`>> query)
-<2> Aggregating on the `_field_names` field
-<3> Accessing the `_field_names` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
+<2> Accessing the `_field_names` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
diff --git a/docs/reference/mapping/fields/parent-field.asciidoc b/docs/reference/mapping/fields/parent-field.asciidoc
index fb066580044..c15a7bcd3b9 100644
--- a/docs/reference/mapping/fields/parent-field.asciidoc
+++ b/docs/reference/mapping/fields/parent-field.asciidoc
@@ -59,8 +59,8 @@ See the <<query-dsl-has-child-query,`has_child`>> and
 the <<search-aggregations-bucket-children-aggregation,`children`>> aggregation,
 and <<parent-child-inner-hits,inner hits>> for more information.
 
-The value of the `_parent` field is accessible in queries, aggregations, scripts,
-and when sorting:
+The value of the `_parent` field is accessible in queries, aggregations,
+and scripts:
 
 [source,js]
 --------------------------
@@ -79,13 +79,6 @@ GET my_index/_search
       }
     }
   },
-  "sort": [
-    {
-      "_parent": { <3>
-        "order": "desc"
-      }
-    }
-  ],
   "script_fields": {
     "parent": {
       "script": "doc['_parent']" <4>
@@ -94,11 +87,11 @@ GET my_index/_search
 }
 --------------------------
 // AUTOSENSE
+// TEST[continued]
 
 <1> Querying on the `_parent` field (also see the <<query-dsl-has-parent-query,`has_parent` query>> and the <<query-dsl-has-child-query,`has_child` query>>)
 <2> Aggregating on the `_parent` field (also see the <<search-aggregations-bucket-children-aggregation,`children`>> aggregation)
-<3> Sorting on the `_parent` field
-<4> Accessing the `_parent` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
+<3> Accessing the `_parent` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
 
 
 ==== Parent-child restrictions
@@ -158,4 +151,3 @@ GET _stats/fielddata?human&fields=_parent
 GET _nodes/stats/indices/fielddata?human&fields=_parent
 --------------------------------------------------
 // AUTOSENSE
-
diff --git a/docs/reference/mapping/fields/routing-field.asciidoc b/docs/reference/mapping/fields/routing-field.asciidoc
index deb46178ccf..496d8dcf56e 100644
--- a/docs/reference/mapping/fields/routing-field.asciidoc
+++ b/docs/reference/mapping/fields/routing-field.asciidoc
@@ -22,14 +22,14 @@ PUT my_index/my_type/1?routing=user1 <1>
 GET my_index/my_type/1?routing=user1 <2>
 ------------------------------
 // AUTOSENSE
+// TESTSETUP
 
 <1> This document uses `user1` as its routing value, instead of its ID.
 <2> The same `routing` value needs to be provided when
     <<docs-get,getting>>, <<docs-delete,deleting>>, or <<docs-update,updating>>
     the document.
 
-The value of the `_routing` field is accessible in queries, aggregations, scripts,
-and when sorting:
+The value of the `_routing` field is accessible in queries and scripts:
 
 [source,js]
 --------------------------
@@ -40,21 +40,6 @@ GET my_index/_search
       "_routing": [ "user1" ] <1>
     }
   },
-  "aggs": {
-    "Routing values": {
-      "terms": {
-        "field": "_routing", <2>
-        "size": 10
-      }
-    }
-  },
-  "sort": [
-    {
-      "_routing": { <3>
-        "order": "desc"
-      }
-    }
-  ],
   "script_fields": {
     "Routing value": {
       "script": "doc['_routing']" <4>
@@ -65,9 +50,7 @@ GET my_index/_search
 // AUTOSENSE
 
 <1> Querying on the `_routing` field (also see the <<query-dsl-ids-query,`ids` query>>)
-<2> Aggregating on the `_routing` field
-<3> Sorting on the `_routing` field
-<4> Accessing the `_routing` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
+<2> Accessing the `_routing` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
 
 
 ==== Searching with custom routing
@@ -104,7 +87,7 @@ custom `routing` value required for all CRUD operations:
 
 [source,js]
 ------------------------------
-PUT my_index
+PUT my_index2
 {
   "mappings": {
     "my_type": {
@@ -115,12 +98,13 @@ PUT my_index
   }
 }
 
-PUT my_index/my_type/1 <2>
+PUT my_index2/my_type/1 <2>
 {
   "text": "No routing value provided"
 }
 ------------------------------
 // AUTOSENSE
+// TEST[catch:request]
 <1> Routing is required for `my_type` documents.
 <2> This index request throws a `routing_missing_exception`.
 
diff --git a/docs/reference/mapping/fields/timestamp-field.asciidoc b/docs/reference/mapping/fields/timestamp-field.asciidoc
index 3f4bf8a8134..c6b3887f889 100644
--- a/docs/reference/mapping/fields/timestamp-field.asciidoc
+++ b/docs/reference/mapping/fields/timestamp-field.asciidoc
@@ -89,6 +89,7 @@ GET my_index/_search
 }
 --------------------------
 // AUTOSENSE
+// TEST[continued]
 
 <1> Querying on the `_timestamp` field
 <2> Aggregating on the `_timestamp` field
diff --git a/docs/reference/mapping/fields/type-field.asciidoc b/docs/reference/mapping/fields/type-field.asciidoc
index c8f3817bcbf..8a569d22f53 100644
--- a/docs/reference/mapping/fields/type-field.asciidoc
+++ b/docs/reference/mapping/fields/type-field.asciidoc
@@ -1,12 +1,11 @@
-[[mapping-type-field]]
+\[[mapping-type-field]]
 === `_type` field
 
 Each document indexed is associated with a <<mapping-type-field,`_type`>> (see
 <<mapping-type>>) and an <<mapping-id-field,`_id`>>.  The `_type` field is
 indexed in order to make searching by type name fast.
 
-The value of the `_type` field is accessible in queries, aggregations,
-scripts, and when sorting:
+The value of the `_type` field is accessible in queries and scripts:
 
 [source,js]
 --------------------------
@@ -21,31 +20,16 @@ PUT my_index/type_2/2
   "text": "Document with type 2"
 }
 
-GET my_index/_search/type_*
+GET my_index/type_*/_search
 {
   "query": {
     "terms": {
       "_type": [ "type_1", "type_2" ] <1>
     }
   },
-  "aggs": {
-    "types": {
-      "terms": {
-        "field": "_type", <2>
-        "size": 10
-      }
-    }
-  },
-  "sort": [
-    {
-      "_type": { <3>
-        "order": "desc"
-      }
-    }
-  ],
   "script_fields": {
     "type": {
-      "script": "doc['_type']" <4>
+      "script": "doc['_type']" <2>
     }
   }
 }
@@ -54,7 +38,4 @@ GET my_index/_search/type_*
 // AUTOSENSE
 
 <1> Querying on the `_type` field
-<2> Aggregating on the `_type` field
-<3> Sorting on the `_type` field
-<4> Accessing the `_type` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
-
+<2> Accessing the `_type` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
diff --git a/docs/reference/mapping/params.asciidoc b/docs/reference/mapping/params.asciidoc
index 4071c383ba1..d0591a7479f 100644
--- a/docs/reference/mapping/params.asciidoc
+++ b/docs/reference/mapping/params.asciidoc
@@ -91,10 +91,3 @@ include::params/similarity.asciidoc[]
 include::params/store.asciidoc[]
 
 include::params/term-vector.asciidoc[]
-
-
-[source,js]
---------------------------------------------------
---------------------------------------------------
-// AUTOSENSE
-
diff --git a/docs/reference/mapping/params/analyzer.asciidoc b/docs/reference/mapping/params/analyzer.asciidoc
index 2a452465e2d..5f564f9a668 100644
--- a/docs/reference/mapping/params/analyzer.asciidoc
+++ b/docs/reference/mapping/params/analyzer.asciidoc
@@ -80,7 +80,7 @@ GET my_index/_analyze?field=text.english <4>
 [[search-quote-analyzer]]
 ==== `search_quote_analyzer`
 
-The `search_quote_analyzer` setting allows you to specify an analyzer for phrases, this is particularly useful when dealing with disabling 
+The `search_quote_analyzer` setting allows you to specify an analyzer for phrases, this is particularly useful when dealing with disabling
 stop words for phrase queries.
 
 To disable stop words for phrases a field utilising three analyzer settings will be required:
@@ -91,7 +91,7 @@ To disable stop words for phrases a field utilising three analyzer settings will
 
 [source,js]
 --------------------------------------------------
-PUT /my_index
+PUT my_index
 {
    "settings":{
       "analysis":{
@@ -128,7 +128,6 @@ PUT /my_index
                "analyzer":"my_analyzer", <3>
                "search_analyzer":"my_stop_analyzer", <4>
                "search_quote_analyzer":"my_analyzer" <5>
-              }
             }
          }
       }
@@ -162,10 +161,10 @@ GET my_index/my_type/_search
 <2> `my_stop_analyzer` analyzer which removes stop words
 <3> `analyzer` setting that points to the `my_analyzer` analyzer which will be used at index time
 <4> `search_analyzer` setting that points to the `my_stop_analyzer` and removes stop words for non-phrase queries
-<5> `search_quote_analyzer` setting that points to the `my_analyzer` analyzer and ensures that stop words are not removed from phrase queries  
+<5> `search_quote_analyzer` setting that points to the `my_analyzer` analyzer and ensures that stop words are not removed from phrase queries
 <6> Since the query is wrapped in quotes it is detected as a phrase query therefore the `search_quote_analyzer` kicks in and ensures the stop words
-are not removed from the query. The `my_analyzer` analyzer will then return the following tokens [`the`, `quick`, `brown`, `fox`] which will match one 
-of the documents. Meanwhile term queries will be analyzed with the `my_stop_analyzer` analyzer which will filter out stop words. So a search for either 
-`The quick brown fox` or `A quick brown fox` will return both documents since both documents contain the following tokens [`quick`, `brown`, `fox`]. 
-Without the `search_quote_analyzer` it would not be possible to do exact matches for phrase queries as the stop words from phrase queries would be 
+are not removed from the query. The `my_analyzer` analyzer will then return the following tokens [`the`, `quick`, `brown`, `fox`] which will match one
+of the documents. Meanwhile term queries will be analyzed with the `my_stop_analyzer` analyzer which will filter out stop words. So a search for either
+`The quick brown fox` or `A quick brown fox` will return both documents since both documents contain the following tokens [`quick`, `brown`, `fox`].
+Without the `search_quote_analyzer` it would not be possible to do exact matches for phrase queries as the stop words from phrase queries would be
 removed resulting in both documents matching.
diff --git a/docs/reference/mapping/params/boost.asciidoc b/docs/reference/mapping/params/boost.asciidoc
index add6f806844..a3379d581c7 100644
--- a/docs/reference/mapping/params/boost.asciidoc
+++ b/docs/reference/mapping/params/boost.asciidoc
@@ -34,24 +34,31 @@ You can achieve the same effect by using the boost parameter directly in the que
 
 [source,js]
 --------------------------------------------------
+POST _search
 {
-    "match" : {
-        "title": {
-            "query": "quick brown fox"
+    "query": {
+        "match" : {
+            "title": {
+                "query": "quick brown fox"
+            }
         }
     }
 }
 --------------------------------------------------
+// AUTOSENSE
 
 is equivalent to:
 
 [source,js]
 --------------------------------------------------
+POST _search
 {
-    "match" : {
-        "title": {
-            "query": "quick brown fox",
-            "boost": 2
+    "query": {
+        "match" : {
+            "title": {
+                "query": "quick brown fox",
+                "boost": 2
+            }
         }
     }
 }
diff --git a/docs/reference/mapping/params/coerce.asciidoc b/docs/reference/mapping/params/coerce.asciidoc
index 0121c307230..9d55a8b80c8 100644
--- a/docs/reference/mapping/params/coerce.asciidoc
+++ b/docs/reference/mapping/params/coerce.asciidoc
@@ -45,6 +45,7 @@ PUT my_index/my_type/2
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[catch:request]
 <1> The `number_one` field will contain the integer `10`.
 <2> This document will be rejected because coercion is disabled.
 
@@ -69,11 +70,11 @@ PUT my_index
     "my_type": {
       "properties": {
         "number_one": {
-          "type": "integer"
-        },
-        "number_two": {
           "type": "integer",
           "coerce": true
+        },
+        "number_two": {
+          "type": "integer"
         }
       }
     }
@@ -87,6 +88,6 @@ PUT my_index/my_type/2
 { "number_two": "10" } <2>
 --------------------------------------------------
 // AUTOSENSE
-<1> This document will be rejected because the `number_one` field inherits the index-level coercion setting.
-<2> The `number_two` field overrides the index level setting to enable coercion.
-
+// TEST[catch:request]
+<1> The `number_one` field overrides the index level setting to enable coercion.
+<2> This document will be rejected because the `number_two` field inherits the index-level coercion setting.
diff --git a/docs/reference/mapping/params/copy-to.asciidoc b/docs/reference/mapping/params/copy-to.asciidoc
index 863bf1996cd..a6288bcbc48 100644
--- a/docs/reference/mapping/params/copy-to.asciidoc
+++ b/docs/reference/mapping/params/copy-to.asciidoc
@@ -9,7 +9,7 @@ the `full_name` field as follows:
 
 [source,js]
 --------------------------------------------------
-PUT /my_index
+PUT my_index
 {
   "mappings": {
     "my_type": {
@@ -30,13 +30,13 @@ PUT /my_index
   }
 }
 
-PUT /my_index/my_type/1
+PUT my_index/my_type/1
 {
   "first_name": "John",
   "last_name": "Smith"
 }
 
-GET /my_index/_search
+GET my_index/_search
 {
   "query": {
     "match": {
diff --git a/docs/reference/mapping/params/dynamic.asciidoc b/docs/reference/mapping/params/dynamic.asciidoc
index 72bbd369d7f..16bb89fbf6b 100644
--- a/docs/reference/mapping/params/dynamic.asciidoc
+++ b/docs/reference/mapping/params/dynamic.asciidoc
@@ -7,9 +7,7 @@ containing the new field.  For instance:
 
 [source,js]
 --------------------------------------------------
-DELETE my_index <1>
-
-PUT my_index/my_type/1 <2>
+PUT my_index/my_type/1 <1>
 {
   "username": "johnsmith",
   "name": {
@@ -18,9 +16,9 @@ PUT my_index/my_type/1 <2>
   }
 }
 
-GET my_index/_mapping <3>
+GET my_index/_mapping <2>
 
-PUT my_index/my_type/2 <4>
+PUT my_index/my_type/2 <3>
 {
   "username": "marywhite",
   "email": "mary@white.com",
@@ -31,16 +29,15 @@ PUT my_index/my_type/2 <4>
   }
 }
 
-GET my_index/_mapping <5>
+GET my_index/_mapping <4>
 --------------------------------------------------
 // AUTOSENSE
-<1> First delete the index, in case it already exists.
-<2> This document introduces the string field `username`, the object field
+<1> This document introduces the string field `username`, the object field
     `name`, and two string fields under the `name` object which can be
     referred to as `name.first` and `name.last`.
-<3> Check the mapping to verify the above.
-<4> This document adds two string fields: `email` and `name.middle`.
-<5> Check the mapping to verify the changes.
+<2> Check the mapping to verify the above.
+<3> This document adds two string fields: `email` and `name.middle`.
+<4> Check the mapping to verify the changes.
 
 The details of how new fields are detected and added to the mapping is explained in <<dynamic-mapping>>.
 
@@ -88,5 +85,3 @@ PUT my_index
 TIP: The `dynamic` setting is allowed to have different settings for fields of
 the same name in the same index.  Its value can be updated on existing fields
 using the <<indices-put-mapping,PUT mapping API>>.
-
-
diff --git a/docs/reference/mapping/params/fielddata.asciidoc b/docs/reference/mapping/params/fielddata.asciidoc
index e67b47a831f..e20a82bfd5c 100644
--- a/docs/reference/mapping/params/fielddata.asciidoc
+++ b/docs/reference/mapping/params/fielddata.asciidoc
@@ -100,14 +100,11 @@ PUT my_index
       "properties": {
         "tag": {
           "type": "text",
-          "fielddata": {
-            "filter": {
-              "frequency": {
-                "min": 0.001,
-                "max": 0.1,
-                "min_segment_size": 500
-              }
-            }
+          "fielddata": true,
+          "fielddata_frequency_filter": {
+            "min": 0.001,
+            "max": 0.1,
+            "min_segment_size": 500
           }
         }
       }
diff --git a/docs/reference/mapping/params/ignore-malformed.asciidoc b/docs/reference/mapping/params/ignore-malformed.asciidoc
index b5e5ee99b2c..00b57cb32e9 100644
--- a/docs/reference/mapping/params/ignore-malformed.asciidoc
+++ b/docs/reference/mapping/params/ignore-malformed.asciidoc
@@ -20,11 +20,11 @@ PUT my_index
     "my_type": {
       "properties": {
         "number_one": {
-          "type": "integer"
-        },
-        "number_two": {
           "type": "integer",
           "ignore_malformed": true
+        },
+        "number_two": {
+          "type": "integer"
         }
       }
     }
@@ -44,8 +44,9 @@ PUT my_index/my_type/2
 }
 --------------------------------------------------
 // AUTOSENSE
-<1> This document will be rejected because `number_one` does not allow malformed values.
-<2> This document will have the `text` field indexed, but not the `number_two` field.
+// TEST[catch:request]
+<1> This document will have the `text` field indexed, but not the `number_one` field.
+<2> This document will be rejected because `number_two` does not allow malformed values.
 
 TIP: The `ignore_malformed` setting is allowed to have different settings for
 fields of the same name in the same index.  Its value can be updated on
@@ -84,4 +85,3 @@ PUT my_index
 
 <1> The `number_one` field inherits the index-level setting.
 <2> The `number_two` field overrides the index-level setting to turn off `ignore_malformed`.
-
diff --git a/docs/reference/mapping/params/include-in-all.asciidoc b/docs/reference/mapping/params/include-in-all.asciidoc
index 18c454ae70c..bd2b5b8a179 100644
--- a/docs/reference/mapping/params/include-in-all.asciidoc
+++ b/docs/reference/mapping/params/include-in-all.asciidoc
@@ -15,7 +15,7 @@ PUT my_index
       "properties": {
         "title": { <1>
           "type": "text"
-        }
+        },
         "content": { <1>
           "type": "text"
         },
diff --git a/docs/reference/mapping/params/multi-fields.asciidoc b/docs/reference/mapping/params/multi-fields.asciidoc
index 8ca2809c7f2..bee9dae1763 100644
--- a/docs/reference/mapping/params/multi-fields.asciidoc
+++ b/docs/reference/mapping/params/multi-fields.asciidoc
@@ -8,7 +8,7 @@ search, and as a `keyword` field for sorting or aggregations:
 
 [source,js]
 --------------------------------------------------
-PUT /my_index
+PUT my_index
 {
   "mappings": {
     "my_type": {
@@ -26,17 +26,17 @@ PUT /my_index
   }
 }
 
-PUT /my_index/my_type/1
+PUT my_index/my_type/1
 {
   "city": "New York"
 }
 
-PUT /my_index/my_type/2
+PUT my_index/my_type/2
 {
   "city": "York"
 }
 
-GET /my_index/_search
+GET my_index/_search
 {
   "query": {
     "match": {
@@ -132,4 +132,3 @@ stemmed field allows a query for `foxes` to also match the document containing
 just `fox`.  This allows us to match as many documents as possible.  By also
 querying the unstemmed `text` field, we improve the relevance score of the
 document which matches `foxes` exactly.
-
diff --git a/docs/reference/mapping/params/norms.asciidoc b/docs/reference/mapping/params/norms.asciidoc
index f6e42219a1f..9f78e0d62d8 100644
--- a/docs/reference/mapping/params/norms.asciidoc
+++ b/docs/reference/mapping/params/norms.asciidoc
@@ -31,11 +31,10 @@ PUT my_index/_mapping/my_type
 }
 ------------
 // AUTOSENSE
+// TEST[s/^/PUT my_index\n/]
 
 NOTE: Norms will not be removed instantly, but will be removed as old segments
 are merged into new segments as you continue indexing new documents. Any score
 computation on a field that has had norms removed might return inconsistent
 results since some documents won't have norms anymore while other documents
 might still have norms.
-
-
diff --git a/docs/reference/mapping/params/position-increment-gap.asciidoc b/docs/reference/mapping/params/position-increment-gap.asciidoc
index d2cf1360080..871b6275a1c 100644
--- a/docs/reference/mapping/params/position-increment-gap.asciidoc
+++ b/docs/reference/mapping/params/position-increment-gap.asciidoc
@@ -13,12 +13,12 @@ For example:
 
 [source,js]
 --------------------------------------------------
-PUT /my_index/groups/1
+PUT my_index/groups/1
 {
     "names": [ "John Abraham", "Lincoln Smith"]
 }
 
-GET /my_index/groups/_search
+GET my_index/groups/_search
 {
     "query": {
         "match_phrase": {
@@ -29,7 +29,7 @@ GET /my_index/groups/_search
     }
 }
 
-GET /my_index/groups/_search
+GET my_index/groups/_search
 {
     "query": {
         "match_phrase": {
@@ -65,12 +65,12 @@ PUT my_index
   }
 }
 
-PUT /my_index/groups/1
+PUT my_index/groups/1
 {
     "names": [ "John Abraham", "Lincoln Smith"]
 }
 
-GET /my_index/groups/_search
+GET my_index/groups/_search
 {
     "query": {
         "match_phrase": {
diff --git a/docs/reference/mapping/params/properties.asciidoc b/docs/reference/mapping/params/properties.asciidoc
index a4f5277649a..d9e3a2db75d 100644
--- a/docs/reference/mapping/params/properties.asciidoc
+++ b/docs/reference/mapping/params/properties.asciidoc
@@ -99,7 +99,6 @@ GET my_index/_search
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[continued]
 
 IMPORTANT: The full path to the inner field must be specified.
-
-
diff --git a/docs/reference/mapping/params/search-analyzer.asciidoc b/docs/reference/mapping/params/search-analyzer.asciidoc
index 5a732ee3aac..3e51aad14af 100644
--- a/docs/reference/mapping/params/search-analyzer.asciidoc
+++ b/docs/reference/mapping/params/search-analyzer.asciidoc
@@ -14,7 +14,7 @@ this can be overridden with the `search_analyzer` setting:
 
 [source,js]
 --------------------------------------------------
-PUT /my_index
+PUT my_index
 {
   "settings": {
     "analysis": {
@@ -81,5 +81,3 @@ type] for a full explanation of this example.
 TIP: The `search_analyzer` setting must have the same setting for fields of
 the same name in the same index.  Its value can be updated on existing fields
 using the <<indices-put-mapping,PUT mapping API>>.
-
-
diff --git a/docs/reference/mapping/params/store.asciidoc b/docs/reference/mapping/params/store.asciidoc
index 9f10b25724c..9c30217c146 100644
--- a/docs/reference/mapping/params/store.asciidoc
+++ b/docs/reference/mapping/params/store.asciidoc
@@ -18,7 +18,7 @@ to extract those fields from a large `_source` field:
 
 [source,js]
 --------------------------------------------------
-PUT /my_index
+PUT my_index
 {
   "mappings": {
     "my_type": {
@@ -39,7 +39,7 @@ PUT /my_index
   }
 }
 
-PUT /my_index/my_type/1
+PUT my_index/my_type/1
 {
   "title":   "Some short title",
   "date":    "2015-01-01",
@@ -70,4 +70,3 @@ field instead.
 
 Another situation where it can make sense to make a field stored is for those
 that don't appear in the `_source` field (such as <<copy-to,`copy_to` fields>>).
-
diff --git a/docs/reference/mapping/types/date.asciidoc b/docs/reference/mapping/types/date.asciidoc
index 85e86e6a6a1..3756717b0d1 100644
--- a/docs/reference/mapping/types/date.asciidoc
+++ b/docs/reference/mapping/types/date.asciidoc
@@ -40,7 +40,7 @@ PUT my_index/my_type/1
 { "date": "2015-01-01" } <2>
 
 PUT my_index/my_type/2
-{ "date": "2015-01-01T12:10:30Z" } <3>
+{ "date": "2015-01-01T12:10:30Z" } <3>
 
 PUT my_index/my_type/3
 { "date": 1420070400001 } <4>
@@ -134,4 +134,3 @@ The following parameters are accepted by `date` fields:
     Whether the field value should be stored and retrievable separately from
     the <<mapping-source-field,`_source`>> field. Accepts `true` or `false`
     (default).
-
diff --git a/docs/reference/mapping/types/nested.asciidoc b/docs/reference/mapping/types/nested.asciidoc
index ed0bb47e9d4..90b54fafb04 100644
--- a/docs/reference/mapping/types/nested.asciidoc
+++ b/docs/reference/mapping/types/nested.asciidoc
@@ -62,6 +62,7 @@ GET my_index/_search
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[continued]
 
 ==== Using `nested` fields for arrays of objects
 
@@ -137,7 +138,7 @@ GET my_index/_search
             "user.first": {}
           }
         }
-
+      }
     }
   }
 }
diff --git a/docs/reference/migration/migrate_5_0/mapping.asciidoc b/docs/reference/migration/migrate_5_0/mapping.asciidoc
index 4d1e2653356..2da16a62309 100644
--- a/docs/reference/migration/migrate_5_0/mapping.asciidoc
+++ b/docs/reference/migration/migrate_5_0/mapping.asciidoc
@@ -53,7 +53,7 @@ you could map it both as a number and a `keyword` using <<multi-fields>>:
 
 [source,js]
 --------------------------------------------------
-PUT /my_index
+PUT my_index
 {
   "mappings": {
     "my_type": {
@@ -149,4 +149,3 @@ Per-field boosts on the `_all` are now compressed into a single byte instead
 of the 4 bytes used previously. While this will make the index much more
 space-efficient, it also means that index time boosts will be less accurately
 encoded.
-
diff --git a/docs/reference/modules/cluster/allocation_filtering.asciidoc b/docs/reference/modules/cluster/allocation_filtering.asciidoc
index 051c88ac9cd..fbdeb64909f 100644
--- a/docs/reference/modules/cluster/allocation_filtering.asciidoc
+++ b/docs/reference/modules/cluster/allocation_filtering.asciidoc
@@ -14,7 +14,7 @@ For instance, we could decommission a node using its IP address as follows:
 
 [source,js]
 --------------------------------------------------
-PUT /_cluster/settings
+PUT _cluster/settings
 {
   "transient" : {
     "cluster.routing.allocation.exclude._ip" : "10.0.0.1"
@@ -67,4 +67,4 @@ PUT _cluster/settings
 }
 ------------------------
 // AUTOSENSE
-
+// TEST[skip:indexes don't assign]
diff --git a/docs/reference/modules/cluster/disk_allocator.asciidoc b/docs/reference/modules/cluster/disk_allocator.asciidoc
index 9baf8a379fb..fdd8fc5036b 100644
--- a/docs/reference/modules/cluster/disk_allocator.asciidoc
+++ b/docs/reference/modules/cluster/disk_allocator.asciidoc
@@ -56,7 +56,7 @@ the cluster every minute:
 
 [source,js]
 --------------------------------------------------
-PUT /_cluster/settings
+PUT _cluster/settings
 {
   "transient": {
     "cluster.routing.allocation.disk.watermark.low": "80%",
@@ -73,4 +73,3 @@ data paths, one with 50b out of 100b free (50% used) and another with
 40b out of 50b free (80% used) it would see the node's disk usage as 90b
 out of 150b). In 2.0.0, the minimum and maximum disk usages are tracked
 separately.
-
diff --git a/docs/reference/modules/node.asciidoc b/docs/reference/modules/node.asciidoc
index 5d7e5a10482..2cb59c6de3a 100644
--- a/docs/reference/modules/node.asciidoc
+++ b/docs/reference/modules/node.asciidoc
@@ -158,6 +158,7 @@ PUT _cluster/settings
 }
 ----------------------------
 // AUTOSENSE
+// TEST[catch:/cannot set discovery.zen.minimum_master_nodes to more than the current master nodes/]
 
 TIP: An advantage of splitting the master and data roles between dedicated
 nodes is that you can have just three master-eligible nodes and set
@@ -299,4 +300,3 @@ same data directory. This can lead to unexpected data loss.
 More node settings can be found in <<modules,Modules>>.  Of particular note are
 the <<cluster.name,`cluster.name`>>, the <<node.name,`node.name`>> and the
 <<modules-network,network settings>>.
-
diff --git a/docs/reference/modules/scripting/painless.asciidoc b/docs/reference/modules/scripting/painless.asciidoc
index b96e1f099de..98cb052d7a3 100644
--- a/docs/reference/modules/scripting/painless.asciidoc
+++ b/docs/reference/modules/scripting/painless.asciidoc
@@ -40,11 +40,9 @@ to `painless`.
 
 To illustrate how Painless works, let's load some hockey stats into an Elasticsearch index:
 
-[source,sh]
+[source,js]
 ----------------------------------------------------------------
-DELETE /hockey-stats
-
-PUT /hockey-stats/player/_bulk
+PUT hockey/player/_bulk?refresh
 {"index":{"_id":1}}
 {"first":"johnny","last":"gaudreau","goals":[9,27,1],"assists":[17,46,0],"gp":[26,82,1]}
 {"index":{"_id":2}}
@@ -69,6 +67,7 @@ PUT /hockey-stats/player/_bulk
 {"first":"joe","last":"colborne","goals":[3,18,13],"assists":[6,20,24],"gp":[26,67,82]}
 ----------------------------------------------------------------
 // AUTOSENSE
+// TESTSETUP
 
 [float]
 === Accessing Doc Values from Painless
@@ -77,9 +76,9 @@ All Painless scripts take in a `Map<String,def>` of values called `input`.  Docu
 
 For example, the following script calculates a player's total goals. This example uses a strongly typed `int` and a `for` loop.
 
-[source,sh]
+[source,js]
 ----------------------------------------------------------------
-GET /hockey-stats/_search
+GET hockey/_search
 {
   "query": {
     "function_score": {
@@ -97,9 +96,9 @@ GET /hockey-stats/_search
 
 Alternatively, you could do the same thing using a script field instead of a function score:
 
-[source,sh]
+[source,js]
 ----------------------------------------------------------------
-GET /hockey-stats/_search
+GET hockey/_search
 {
   "query": {
     "match_all": {}
@@ -120,16 +119,16 @@ You must always specify the index of the field value you want, even if there's o
 All fields in Elasticsearch are multi-valued and Painless does not provide a `.value` shortcut. The following example uses a Painless script to sort the players by their combined first and last names. The names are accessed using
 `input.doc['first'].0` and `input.doc['last'].0`.
 
-[source,sh]
+[source,js]
 ----------------------------------------------------------------
-GET /hockey-stats/_search
+GET hockey/_search
 {
   "query": {
     "match_all": {}
   },
   "sort": {
     "_script": {
-      "type": "keyword",
+      "type": "string",
       "order": "asc",
       "script": {
         "lang": "painless",
@@ -148,9 +147,9 @@ You can also easily update fields. You access the original source for a field as
 
 First, let's look at the source data for a player by submitting the following request:
 
-[source,sh]
+[source,js]
 ----------------------------------------------------------------
-GET /hockey-stats/_search
+GET hockey/_search
 {
   "fields": [
     "_id",
@@ -167,9 +166,9 @@ GET /hockey-stats/_search
 
 To change player 1's last name to `hockey`, simply set `input.ctx._source.last` to the new value:
 
-[source,sh]
+[source,js]
 ----------------------------------------------------------------
-POST /hockey-stats/player/1/_update
+POST hockey/player/1/_update
 {
   "script": {
     "lang": "painless",
@@ -185,9 +184,9 @@ POST /hockey-stats/player/1/_update
 You can also add fields to a document. For example, this script adds a new field that contains
 the player's nickname,  _hockey_.
 
-[source,sh]
+[source,js]
 ----------------------------------------------------------------
-POST /hockey-stats/player/1/_update
+POST hockey/player/1/_update
 {
   "script": {
     "lang": "painless",
@@ -208,9 +207,9 @@ If you explicitly specify types, the compiler doesn't have to perform type looku
 improve performance. For example, the following script performs the same first name, last name sort we showed before,
 but it's fully type-safe.
 
-[source,sh]
+[source,js]
 ----------------------------------------------------------------
-GET /hockey-stats/_search
+GET hockey/_search
 {
   "query": {
     "match_all": {}
diff --git a/docs/reference/modules/scripting/using.asciidoc b/docs/reference/modules/scripting/using.asciidoc
index 71eba9386e3..81636f6189e 100644
--- a/docs/reference/modules/scripting/using.asciidoc
+++ b/docs/reference/modules/scripting/using.asciidoc
@@ -186,35 +186,27 @@ state:
 
 [source,js]
 -----------------------------------
-POST /_scripts/groovy/calculate-score
+POST _scripts/groovy/calculate-score
 {
   "script": "log(_score * 2) + my_modifier"
 }
 -----------------------------------
 // AUTOSENSE
 
-
 This same script can be retrieved with:
 
 [source,js]
 -----------------------------------
-GET /_scripts/groovy/calculate-score
------------------------------------
-// AUTOSENSE
-
-or deleted with:
-
-[source,js]
------------------------------------
-DELETE /_scripts/groovy/calculate-score
+GET _scripts/groovy/calculate-score
 -----------------------------------
 // AUTOSENSE
+// TEST[continued]
 
 Stored scripts can be used by specifying the `lang` and `id` parameters as follows:
 
 [source,js]
 --------------------------------------------------
-GET my_index/_search
+GET _search
 {
   "query": {
     "script": {
@@ -229,10 +221,19 @@ GET my_index/_search
   }
 }
 --------------------------------------------------
+// AUTOSENSE
+// TEST[continued]
 
+And deleted with:
+
+[source,js]
+-----------------------------------
+DELETE _scripts/groovy/calculate-score
+-----------------------------------
+// AUTOSENSE
+// TEST[continued]
 
 NOTE: The size of stored scripts is limited to 65,535 bytes. This can be
 changed by setting `script.max_size_in_bytes` setting to increase that soft
 limit, but if scripts are really large then alternatives like
 <<modules-scripting-native,native>> scripts should be considered instead.
-
diff --git a/docs/reference/query-dsl/bool-query.asciidoc b/docs/reference/query-dsl/bool-query.asciidoc
index d3895826c68..ab39eb49611 100644
--- a/docs/reference/query-dsl/bool-query.asciidoc
+++ b/docs/reference/query-dsl/bool-query.asciidoc
@@ -39,47 +39,47 @@ on the fraction of all query terms that a document contains. See Lucene
 `BooleanQuery` for more details.
 
 The `bool` query takes a _more-matches-is-better_ approach, so the score from
-each matching `must` or `should` clause will be added together to provide the 
+each matching `must` or `should` clause will be added together to provide the
 final `_score` for each document.
 
 [source,js]
 --------------------------------------------------
+POST _search
 {
+  "query": {
     "bool" : {
-        "must" : {
-            "term" : { "user" : "kimchy" }
-        },
-        "filter": {
-            "term" : { "tag" : "tech" }
-        },
-        "must_not" : {
-            "range" : {
-                "age" : { "from" : 10, "to" : 20 }
-            }
-        },
-        "should" : [
-            {
-                "term" : { "tag" : "wow" }
-            },
-            {
-                "term" : { "tag" : "elasticsearch" }
-            }
-        ],
-        "minimum_should_match" : 1,
-        "boost" : 1.0
+      "must" : {
+        "term" : { "user" : "kimchy" }
+      },
+      "filter": {
+        "term" : { "tag" : "tech" }
+      },
+      "must_not" : {
+        "range" : {
+          "age" : { "from" : 10, "to" : 20 }
+        }
+      },
+      "should" : [
+        { "term" : { "tag" : "wow" } },
+        { "term" : { "tag" : "elasticsearch" } }
+      ],
+      "minimum_should_match" : 1,
+      "boost" : 1.0
     }
+  }
 }
 --------------------------------------------------
+// AUTOSENSE
 
-==== Scoring with `bool.filter` 
+==== Scoring with `bool.filter`
 
 Queries specified under the `filter` element have no effect on scoring --
 scores are returned as `0`.  Scores are only affected by the query that has
-been specified.  For instance, all three of the following queries return 
-all documents where the `status` field contains the term `active`. 
+been specified.  For instance, all three of the following queries return
+all documents where the `status` field contains the term `active`.
 
 This first query assigns a score of `0` to all documents, as no scoring
-query has been specified: 
+query has been specified:
 
 [source,json]
 ---------------------------------
@@ -98,7 +98,7 @@ GET _search
 ---------------------------------
 // AUTOSENSE
 
-This `bool` query has a `match_all` query, which assigns a score of `1.0` to 
+This `bool` query has a `match_all` query, which assigns a score of `1.0` to
 all documents.
 
 [source,json]
@@ -107,7 +107,7 @@ GET _search
 {
   "query": {
     "bool": {
-      "query": {
+      "must": {
         "match_all": {}
       },
       "filter": {
@@ -121,9 +121,9 @@ GET _search
 ---------------------------------
 // AUTOSENSE
 
-This `constant_score` query behaves in exactly the same way as the second example above.  
+This `constant_score` query behaves in exactly the same way as the second example above.
 The `constant_score` query assigns a score of `1.0` to all documents matched
-by the filter. 
+by the filter.
 
 [source,json]
 ---------------------------------
diff --git a/docs/reference/query-dsl/function-score-query.asciidoc b/docs/reference/query-dsl/function-score-query.asciidoc
index 402eeb44550..3f9943d90d0 100644
--- a/docs/reference/query-dsl/function-score-query.asciidoc
+++ b/docs/reference/query-dsl/function-score-query.asciidoc
@@ -459,7 +459,7 @@ the request would look like this:
 
 [source,js]
 --------------------------------------------------
-GET /hotels/_search/
+GET _search
 {
   "query": {
     "function_score": {
@@ -546,4 +546,3 @@ Only numeric, date, and geo-point fields are supported.
 
 If the numeric field is missing in the document, the function will
 return 1.
-
diff --git a/docs/reference/query-dsl/term-query.asciidoc b/docs/reference/query-dsl/term-query.asciidoc
index 801abf65af8..b27885ac1ed 100644
--- a/docs/reference/query-dsl/term-query.asciidoc
+++ b/docs/reference/query-dsl/term-query.asciidoc
@@ -6,10 +6,14 @@ in the inverted index.  For instance:
 
 [source,js]
 --------------------------------------------------
+POST _search
 {
+  "query": {
     "term" : { "user" : "Kimchy" } <1>
+  }
 }
 --------------------------------------------------
+// AUTOSENSE
 <1> Finds documents which contain the exact term `Kimchy` in the inverted index
     of the `user` field.
 
@@ -18,7 +22,7 @@ relevance score than another query, for instance:
 
 [source,js]
 --------------------------------------------------
-GET /_search
+GET _search
 {
   "query": {
     "bool": {
@@ -41,6 +45,7 @@ GET /_search
   }
 }
 --------------------------------------------------
+// AUTOSENSE
 
 <1> The `urgent` query clause has a boost of `2.0`, meaning it is twice as important
     as the query clause for `normal`.
@@ -113,7 +118,6 @@ Now, compare the results for the `term` query and the `match` query:
 
 [source,js]
 --------------------------------------------------
-
 GET my_index/my_type/_search
 {
   "query": {
@@ -151,6 +155,7 @@ GET my_index/my_type/_search
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[continued]
 
 <1> This query matches because the `exact_value` field contains the exact
     term `Quick Foxes!`.
@@ -161,5 +166,3 @@ GET my_index/my_type/_search
 <4> This `match` query on the `full_text` field first analyzes the query string,
     then looks for documents containing `quick` or `foxes` or both.
 **************************************************
-
-
diff --git a/docs/reference/setup/cluster_restart.asciidoc b/docs/reference/setup/cluster_restart.asciidoc
index 5d93633972b..1dbb6e7616d 100644
--- a/docs/reference/setup/cluster_restart.asciidoc
+++ b/docs/reference/setup/cluster_restart.asciidoc
@@ -17,7 +17,7 @@ before shutting down a node:
 
 [source,js]
 --------------------------------------------------
-PUT /_cluster/settings
+PUT _cluster/settings
 {
   "persistent": {
     "cluster.routing.allocation.enable": "none"
@@ -25,6 +25,7 @@ PUT /_cluster/settings
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[skip:indexes don't assign]
 
 ==== Step 2: Perform a synced flush
 
@@ -33,7 +34,7 @@ Shard recovery will be much faster if you stop indexing and issue a
 
 [source,sh]
 --------------------------------------------------
-POST /_flush/synced
+POST _flush/synced
 --------------------------------------------------
 // AUTOSENSE
 
@@ -96,7 +97,7 @@ reenable shard allocation:
 
 [source,js]
 ------------------------------------------------------
-PUT /_cluster/settings
+PUT _cluster/settings
 {
   "persistent": {
     "cluster.routing.allocation.enable": "all"
@@ -123,4 +124,3 @@ GET _cat/recovery
 
 Once the `status` column in the `_cat/health` output has reached `green`, all
 primary and replica shards have been successfully allocated.
-
diff --git a/docs/reference/setup/rolling_upgrade.asciidoc b/docs/reference/setup/rolling_upgrade.asciidoc
index 07c3060adf6..8c098a1ac04 100644
--- a/docs/reference/setup/rolling_upgrade.asciidoc
+++ b/docs/reference/setup/rolling_upgrade.asciidoc
@@ -21,7 +21,7 @@ allocation before shutting down a node:
 
 [source,js]
 --------------------------------------------------
-PUT /_cluster/settings
+PUT _cluster/settings
 {
   "transient": {
     "cluster.routing.allocation.enable": "none"
@@ -29,6 +29,7 @@ PUT /_cluster/settings
 }
 --------------------------------------------------
 // AUTOSENSE
+// TEST[skip:indexes don't assign]
 
 ==== Step 2: Stop non-essential indexing and perform a synced flush (Optional)
 
@@ -38,7 +39,7 @@ will be much faster if you temporarily stop non-essential indexing and issue a
 
 [source,js]
 --------------------------------------------------
-POST /_flush/synced
+POST _flush/synced
 --------------------------------------------------
 // AUTOSENSE
 
@@ -111,7 +112,7 @@ the node:
 
 [source,js]
 --------------------------------------------------
-PUT /_cluster/settings
+PUT _cluster/settings
 {
   "transient": {
     "cluster.routing.allocation.enable": "all"
@@ -172,4 +173,3 @@ recovery has completed.
 
 When the cluster is stable and the node has recovered, repeat the above steps
 for all remaining nodes.
-
diff --git a/docs/src/test/java/org/elasticsearch/smoketest/SmokeTestDocsIT.java b/docs/src/test/java/org/elasticsearch/smoketest/SmokeTestDocsIT.java
new file mode 100644
index 00000000000..452bea4a647
--- /dev/null
+++ b/docs/src/test/java/org/elasticsearch/smoketest/SmokeTestDocsIT.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.smoketest;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.RestTestCandidate;
+import org.elasticsearch.test.rest.parser.RestTestParseException;
+
+import java.io.IOException;
+import java.util.List;
+
+public class SmokeTestDocsIT extends ESRestTestCase {
+
+    public SmokeTestDocsIT(@Name("yaml") RestTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws IOException, RestTestParseException {
+        return ESRestTestCase.createParameters(0, 1);
+    }
+
+    @Override
+    protected void afterIfFailed(List<Throwable> errors) {
+        super.afterIfFailed(errors);
+        String name = getTestName().split("=")[1];
+        name = name.substring(0, name.length() - 1);
+        name = name.replaceAll("/([^/]+)$", ".asciidoc:$1");
+        logger.error("This failing test was generated by documentation starting at {}. It may include many snippets. "
+                + "See docs/README.asciidoc for an explanation of test generation.", name);
+    }
+}
+
diff --git a/settings.gradle b/settings.gradle
index d03cac653ee..1fc74a0ef2e 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -3,6 +3,7 @@ rootProject.name = 'elasticsearch'
 List projects = [
   'rest-api-spec',
   'core',
+  'docs',
   'distribution:integ-test-zip',
   'distribution:zip',
   'distribution:tar',
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java
index 83860b18bd9..4a7f83953de 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java
@@ -78,7 +78,7 @@ public class RestTestExecutionContext implements Closeable {
         try {
             response = callApiInternal(apiName, requestParams, body, headers);
             //we always stash the last response body
-            stash.stashValue("body", response.getBody());
+            stash.stashResponse(response);
             return response;
         } catch(RestException e) {
             response = e.restResponse();
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java b/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
index 855cbb4d974..9eae902c3e3 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
@@ -19,16 +19,17 @@
 
 package org.elasticsearch.test.rest;
 
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import org.elasticsearch.test.rest.client.RestResponse;
 
 /**
  * Allows to cache the last obtained test response and or part of it within variables
@@ -41,6 +42,7 @@ public class Stash implements ToXContent {
     public static final Stash EMPTY = new Stash();
 
     private final Map<String, Object> stash = new HashMap<>();
+    private RestResponse response;
 
     /**
      * Allows to saved a specific field in the stash as key-value pair
@@ -53,6 +55,12 @@ public class Stash implements ToXContent {
         }
     }
 
+    public void stashResponse(RestResponse response) throws IOException {
+        // TODO we can almost certainly save time by lazily evaluating the body
+        stashValue("body", response.getBody());
+        this.response = response;
+    }
+
     /**
      * Clears the previously stashed values
      */
@@ -78,7 +86,10 @@ public class Stash implements ToXContent {
      * The stash contains fields eventually extracted from previous responses that can be reused
      * as arguments for following requests (e.g. scroll_id)
      */
-    public Object unstashValue(String value) {
+    public Object unstashValue(String value) throws IOException {
+        if (value.startsWith("$body.")) {
+            return response.evaluate(value.substring("$body".length()), this);
+        }
         Object stashedValue = stash.get(value.substring(1));
         if (stashedValue == null) {
             throw new IllegalArgumentException("stashed value not found for key [" + value + "]");
@@ -89,14 +100,14 @@ public class Stash implements ToXContent {
     /**
      * Recursively unstashes map values if needed
      */
-    public Map<String, Object> unstashMap(Map<String, Object> map) {
+    public Map<String, Object> unstashMap(Map<String, Object> map) throws IOException {
         Map<String, Object> copy = new HashMap<>(map);
         unstashObject(copy);
         return copy;
     }
 
     @SuppressWarnings("unchecked")
-    private void unstashObject(Object obj) {
+    private void unstashObject(Object obj) throws IOException {
         if (obj instanceof List) {
             List list = (List) obj;
             for (int i = 0; i < list.size(); i++) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestClient.java
index e798fd8c8ab..5fb6e199b17 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestClient.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestClient.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.test.rest.client;
 
 import com.carrotsearch.randomizedtesting.RandomizedTest;
+
 import org.apache.http.config.Registry;
 import org.apache.http.config.RegistryBuilder;
 import org.apache.http.conn.socket.ConnectionSocketFactory;
@@ -61,6 +62,8 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
+import static java.util.Objects.requireNonNull;
+
 /**
  * REST client used to test the elasticsearch REST layer
  * Holds the {@link RestSpec} used to translate api calls into REST calls
@@ -186,6 +189,19 @@ public class RestClient implements Closeable {
     }
 
     private HttpRequestBuilder callApiBuilder(String apiName, Map<String, String> params, String body) {
+        if ("raw".equals(apiName)) {
+            // Raw requests are bit simpler....
+            HttpRequestBuilder httpRequestBuilder = httpRequestBuilder();
+            httpRequestBuilder.method(requireNonNull(params.remove("method"), "Method must be set to use raw request"));
+            httpRequestBuilder.path("/"+ requireNonNull(params.remove("path"), "Path must be set to use raw request"));
+            httpRequestBuilder.body(body);
+
+            // And everything else is a url parameter!
+            for (Map.Entry<String, String> entry : params.entrySet()) {
+                httpRequestBuilder.addParam(entry.getKey(), entry.getValue());
+            }
+            return httpRequestBuilder;
+        }
 
         //create doesn't exist in the spec but is supported in the clients (index with op_type=create)
         boolean indexCreateApi = "create".equals(apiName);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/json/JsonPath.java b/test/framework/src/main/java/org/elasticsearch/test/rest/json/JsonPath.java
index c3dbd583430..b338d76d985 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/json/JsonPath.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/json/JsonPath.java
@@ -49,14 +49,14 @@ public class JsonPath {
     /**
      * Returns the object corresponding to the provided path if present, null otherwise
      */
-    public Object evaluate(String path) {
+    public Object evaluate(String path) throws IOException {
         return evaluate(path, Stash.EMPTY);
     }
 
     /**
      * Returns the object corresponding to the provided path if present, null otherwise
      */
-    public Object evaluate(String path, Stash stash) {
+    public Object evaluate(String path, Stash stash) throws IOException {
         String[] parts = parsePath(path);
         Object object = jsonMap;
         for (String part : parts) {
@@ -69,7 +69,7 @@ public class JsonPath {
     }
 
     @SuppressWarnings("unchecked")
-    private Object evaluate(String key, Object object, Stash stash) {
+    private Object evaluate(String key, Object object, Stash stash) throws IOException {
         if (stash.isStashedValue(key)) {
             key = stash.unstashValue(key).toString();
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSectionParser.java
index 07636151119..e1d2011e231 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSectionParser.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSectionParser.java
@@ -33,19 +33,23 @@ public class RestTestSectionParser implements RestTestFragmentParser<TestSection
         XContentParser parser = parseContext.parser();
         parseContext.advanceToFieldName();
         TestSection testSection = new TestSection(parser.currentName());
-        parser.nextToken();
-        testSection.setSkipSection(parseContext.parseSkipSection());
-
-        while ( parser.currentToken() != XContentParser.Token.END_ARRAY) {
-            parseContext.advanceToFieldName();
-            testSection.addExecutableSection(parseContext.parseExecutableSection());
+        try {
+            parser.nextToken();
+            testSection.setSkipSection(parseContext.parseSkipSection());
+    
+            while ( parser.currentToken() != XContentParser.Token.END_ARRAY) {
+                parseContext.advanceToFieldName();
+                testSection.addExecutableSection(parseContext.parseExecutableSection());
+            }
+    
+            parser.nextToken();
+            assert parser.currentToken() == XContentParser.Token.END_OBJECT;
+            parser.nextToken();
+    
+            return testSection;
+        } catch (Exception e) {
+            throw new RestTestParseException("Error parsing test named [" + testSection.getName() + "]", e);
         }
-
-        parser.nextToken();
-        assert parser.currentToken() == XContentParser.Token.END_OBJECT;
-        parser.nextToken();
-
-        return testSection;
     }
 
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java
index b99bf3475da..0a0c2722020 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java
@@ -18,18 +18,20 @@
  */
 package org.elasticsearch.test.rest.parser;
 
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.test.rest.section.DoSection;
-import org.elasticsearch.test.rest.section.ExecutableSection;
-import org.elasticsearch.test.rest.section.SetupSection;
-import org.elasticsearch.test.rest.section.SkipSection;
-import org.elasticsearch.test.rest.section.TestSection;
-
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.rest.section.DoSection;
+import org.elasticsearch.test.rest.section.ExecutableSection;
+import org.elasticsearch.test.rest.section.ResponseBodyAssertion;
+import org.elasticsearch.test.rest.section.SetupSection;
+import org.elasticsearch.test.rest.section.SkipSection;
+import org.elasticsearch.test.rest.section.TestSection;
+
 /**
  * Context shared across the whole tests parse phase.
  * Provides shared parse methods and holds information needed to parse the test sections (e.g. es version)
@@ -52,6 +54,7 @@ public class RestTestSuiteParseContext {
         EXECUTABLE_SECTIONS_PARSERS.put("lt", new LessThanParser());
         EXECUTABLE_SECTIONS_PARSERS.put("lte", new LessThanOrEqualToParser());
         EXECUTABLE_SECTIONS_PARSERS.put("length", new LengthParser());
+        EXECUTABLE_SECTIONS_PARSERS.put("response_body", ResponseBodyAssertion.PARSER);
     }
 
     private final String api;
@@ -114,9 +117,14 @@ public class RestTestSuiteParseContext {
         if (execSectionParser == null) {
             throw new RestTestParseException("no parser found for executable section [" + section + "]");
         }
-        ExecutableSection executableSection = execSectionParser.parse(this);
-        parser.nextToken();
-        return executableSection;
+        XContentLocation location = parser.getTokenLocation();
+        try {
+            ExecutableSection executableSection = execSectionParser.parse(this);
+            parser.nextToken();
+            return executableSection;
+        } catch (Exception e) {
+            throw new IOException("Error parsing section starting at ["+ location + "]", e);
+        }
     }
 
     public DoSection parseDoSection() throws IOException, RestTestParseException {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/ResponseBodyAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/ResponseBodyAssertion.java
new file mode 100644
index 00000000000..4a878a77323
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/ResponseBodyAssertion.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.test.rest.section;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.test.rest.parser.RestTestFragmentParser;
+import org.elasticsearch.test.rest.parser.RestTestParseException;
+import org.elasticsearch.test.rest.parser.RestTestSuiteParseContext;
+
+/**
+ * Checks that the response body matches some text. 
+ */
+public class ResponseBodyAssertion extends Assertion {
+    public static final RestTestFragmentParser<ResponseBodyAssertion> PARSER = new RestTestFragmentParser<ResponseBodyAssertion>() {
+        @Override
+        public ResponseBodyAssertion parse(RestTestSuiteParseContext parseContext) throws IOException, RestTestParseException {
+            try (XContentParser parser = JsonXContent.jsonXContent.createParser(parseContext.parseField())) {
+                return new ResponseBodyAssertion("$body", parser.map());
+            }
+        }
+    };
+
+    private ResponseBodyAssertion(String field, Map<String, Object> expectedValue) {
+        super(field, expectedValue);
+    }
+
+    @Override
+    protected void doAssert(Object actualValue, Object expectedValue) {
+        if (false == expectedValue.equals(actualValue)) {
+            @SuppressWarnings("unchecked")
+            Map<String, Object> actual = (Map<String, Object>) actualValue;
+            @SuppressWarnings("unchecked")
+            Map<String, Object> expected = (Map<String, Object>) expectedValue;
+            FailureMessage message = new FailureMessage();
+            message.compareMaps(actual, expected);
+            throw new AssertionError(message.message);
+        }
+    }
+
+    private class FailureMessage {
+        private final StringBuilder message = new StringBuilder("body didn't match the expected value:\n");
+        private int indent = 0;
+
+        private void compareMaps(Map<String, Object> actual, Map<String, Object> expected) {
+            actual = new TreeMap<>(actual);
+            expected = new TreeMap<>(expected);
+            for (Map.Entry<String, Object> expectedEntry : expected.entrySet()) {
+                compare(expectedEntry.getKey(), expectedEntry.getValue(), actual.remove(expectedEntry.getKey()));
+            }
+            for (Map.Entry<String, Object> unmatchedEntry : actual.entrySet()) {
+                field(unmatchedEntry.getKey(), "unexpected but found [" + unmatchedEntry.getValue() + "]");
+            }
+        }
+
+        private void compareLists(List<Object> actual, List<Object> expected) {
+            int i = 0;
+            while (i < actual.size() && i < expected.size()) {
+                compare(i, actual.get(i), expected.get(i));
+            }
+            if (actual.size() == expected.size()) {
+                return;
+            }
+            indent();
+            if (actual.size() < expected.size()) {
+                message.append("expected [").append(expected.size() - i).append("] more entries\n");
+                return;
+            }
+            message.append("received [").append(actual.size() - i).append("] more entries than expected\n");
+        }
+
+        private void compare(Object field, Object expected, @Nullable Object actual) {
+            if (expected instanceof Map) {
+                if (actual == null) {
+                    field(field, "expected map but not found");
+                    return;
+                }
+                if (false == actual instanceof Map) {
+                    field(field, "expected map but found [" + actual + "]");
+                    return;
+                }
+                @SuppressWarnings("unchecked")
+                Map<String, Object> expectedMap = (Map<String, Object>) expected;
+                @SuppressWarnings("unchecked")
+                Map<String, Object> actualMap = (Map<String, Object>) actual;
+                if (expectedMap.isEmpty() && actualMap.isEmpty()) {
+                    field(field, "same [empty map]");
+                }
+                field(field, null);
+                indent += 1;
+                compareMaps(expectedMap, actualMap);
+                indent -= 1;
+                return;
+            }
+            if (expected instanceof List) {
+                if (actual == null) {
+                    field(field, "expected list but not found");
+                    return;
+                }
+                if (false == actual instanceof List) {
+                    field(field, "expected list but found [" + actual + "]");
+                    return;
+                }
+                @SuppressWarnings("unchecked")
+                List<Object> expectedList = (List<Object>) expected;
+                @SuppressWarnings("unchecked")
+                List<Object> actualList = (List<Object>) actual;
+                if (expectedList.isEmpty() && actualList.isEmpty()) {
+                    field(field, "same [empty list]");
+                    return;
+                }
+                field(field, null);
+                indent += 1;
+                compareLists(expectedList, actualList);
+                indent -= 1;
+                return;
+            }
+            if (actual == null) {
+                field(field, "expected [" + expected + "] but not found");
+                return;
+            }
+            if (expected.equals(actual)) {
+                field(field, "same [" + expected + "]");
+                return;
+            }
+            field(field, "expected [" + expected + "] but was [" + actual + "]");
+        }
+
+        private void indent() {
+            for (int i = 0; i < indent; i++) {
+                message.append("  ");
+            }
+        }
+
+        private void field(Object name, String info) {
+            indent();
+            message.append(String.format(Locale.ROOT, "%30s: ", name));
+            if (info != null) {
+                message.append(info);
+            }
+            message.append('\n');
+        }
+    }
+}

From 3912761572f42765365851541cb1f9a9c3eabd55 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 5 May 2016 15:59:21 -0400
Subject: [PATCH 0085/1311] [docs] Add wait_until_yellow to fix build failure

The snippet in the docs creates and index and uses it with the
_analyze api. The trouble is that if the index hasn't been created
fully the _analyze API will fail. This adds a
GET _cluster/health?wait_for_status=yellow
which fixes the issue.

While this does make the docs more cluttered, it also makes the snippets
actually runnable.

Closes #18165
---
 docs/reference/analysis/analyzers/pattern-analyzer.asciidoc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc b/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc
index b012d3a9673..2aaa0d6bb16 100644
--- a/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc
@@ -103,6 +103,8 @@ PUT test?pretty=1
   }
 }
 
+GET _cluster/health?wait_for_status=yellow
+
 GET test/_analyze?analyzer=camel&text=MooseX::FTPClass2_beta
 # "moose","x","ftp","class","2","beta"
 --------------------------------------------------

From c4090a184170875dfce4780a0a31060e969c162a Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Thu, 5 May 2016 16:43:20 -0400
Subject: [PATCH 0086/1311] Remove the Snapshot class in favor of using
 SnapshotInfo

o/e/snapshots/Snapshot and o/e/snapshots/SnapshotInfo contain the same
fields and represent the same information.  Snapshot was used to
maintain snapshot information to the snapshot repository, while
SnapshotInfo was used to represent the snapshot information as presented
through the REST layer.  This removes the Snapshot class and combines
all uses into the SnapshotInfo class.

Closes #18167
---
 .../resources/checkstyle_suppressions.xml     |   1 -
 .../create/CreateSnapshotResponse.java        |   6 +-
 .../snapshots/get/GetSnapshotsResponse.java   |   4 +-
 .../get/TransportGetSnapshotsAction.java      |  17 +-
 .../TransportSnapshotsStatusAction.java       |   4 +-
 .../repositories/Repository.java              |   8 +-
 .../blobstore/BlobStoreRepository.java        |  22 +-
 .../snapshots/RestoreService.java             |   6 +-
 .../org/elasticsearch/snapshots/Snapshot.java | 358 -----------------
 .../elasticsearch/snapshots/SnapshotInfo.java | 376 ++++++++++++------
 .../snapshots/SnapshotsService.java           |  22 +-
 11 files changed, 305 insertions(+), 519 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/snapshots/Snapshot.java

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 152405fcffc..0473f58cf73 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -778,7 +778,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]phrase[/\\]WordScorer.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]term[/\\]TermSuggestParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]RestoreService.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]SnapshotInfo.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]SnapshotShardFailure.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]SnapshotShardsService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]snapshots[/\\]SnapshotsService.java" checks="LineLength" />
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
index 0be07c703f1..0a7a8a9ce80 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
@@ -57,13 +57,13 @@ public class CreateSnapshotResponse extends ActionResponse implements ToXContent
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        snapshotInfo = SnapshotInfo.readOptionalSnapshotInfo(in);
+        snapshotInfo = in.readOptionalWriteable(SnapshotInfo::new);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeOptionalStreamable(snapshotInfo);
+        out.writeOptionalWriteable(snapshotInfo);
     }
 
     /**
@@ -90,7 +90,7 @@ public class CreateSnapshotResponse extends ActionResponse implements ToXContent
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         if (snapshotInfo != null) {
             builder.field(Fields.SNAPSHOT);
-            snapshotInfo.toXContent(builder, params);
+            snapshotInfo.toExternalXContent(builder, params);
         } else {
             builder.field(Fields.ACCEPTED, true);
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
index 65b0e4faa4a..a5db19684b2 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
@@ -60,7 +60,7 @@ public class GetSnapshotsResponse extends ActionResponse implements ToXContent {
         int size = in.readVInt();
         List<SnapshotInfo> builder = new ArrayList<>();
         for (int i = 0; i < size; i++) {
-            builder.add(SnapshotInfo.readSnapshotInfo(in));
+            builder.add(new SnapshotInfo(in));
         }
         snapshots = Collections.unmodifiableList(builder);
     }
@@ -82,7 +82,7 @@ public class GetSnapshotsResponse extends ActionResponse implements ToXContent {
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
         builder.startArray(Fields.SNAPSHOTS);
         for (SnapshotInfo snapshotInfo : snapshots) {
-            snapshotInfo.toXContent(builder, params);
+            snapshotInfo.toExternalXContent(builder, params);
         }
         builder.endArray();
         return builder;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
index 0198102a200..833b1a62289 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
@@ -31,7 +31,6 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -77,18 +76,12 @@ public class TransportGetSnapshotsAction extends TransportMasterNodeAction<GetSn
         try {
             List<SnapshotInfo> snapshotInfoBuilder = new ArrayList<>();
             if (isAllSnapshots(request.snapshots())) {
-                List<Snapshot> snapshots = snapshotsService.snapshots(request.repository(), request.ignoreUnavailable());
-                for (Snapshot snapshot : snapshots) {
-                    snapshotInfoBuilder.add(new SnapshotInfo(snapshot));
-                }
+                snapshotInfoBuilder.addAll(snapshotsService.snapshots(request.repository(), request.ignoreUnavailable()));
             } else if (isCurrentSnapshots(request.snapshots())) {
-                List<Snapshot> snapshots = snapshotsService.currentSnapshots(request.repository());
-                for (Snapshot snapshot : snapshots) {
-                    snapshotInfoBuilder.add(new SnapshotInfo(snapshot));
-                }
+                snapshotInfoBuilder.addAll(snapshotsService.currentSnapshots(request.repository()));
             } else {
                 Set<String> snapshotsToGet = new LinkedHashSet<>(); // to keep insertion order
-                List<Snapshot> snapshots = null;
+                List<SnapshotInfo> snapshots = null;
                 for (String snapshotOrPattern : request.snapshots()) {
                     if (Regex.isSimpleMatchPattern(snapshotOrPattern) == false) {
                         snapshotsToGet.add(snapshotOrPattern);
@@ -96,7 +89,7 @@ public class TransportGetSnapshotsAction extends TransportMasterNodeAction<GetSn
                         if (snapshots == null) { // lazily load snapshots
                             snapshots = snapshotsService.snapshots(request.repository(), request.ignoreUnavailable());
                         }
-                        for (Snapshot snapshot : snapshots) {
+                        for (SnapshotInfo snapshot : snapshots) {
                             if (Regex.simpleMatch(snapshotOrPattern, snapshot.name())) {
                                 snapshotsToGet.add(snapshot.name());
                             }
@@ -105,7 +98,7 @@ public class TransportGetSnapshotsAction extends TransportMasterNodeAction<GetSn
                 }
                 for (String snapshot : snapshotsToGet) {
                     SnapshotId snapshotId = new SnapshotId(request.repository(), snapshot);
-                    snapshotInfoBuilder.add(new SnapshotInfo(snapshotsService.snapshot(snapshotId)));
+                    snapshotInfoBuilder.add(snapshotsService.snapshot(snapshotId));
                 }
             }
             listener.onResponse(new GetSnapshotsResponse(Collections.unmodifiableList(snapshotInfoBuilder)));
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
index efa156eaa0c..ee60a919da6 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
@@ -36,7 +36,7 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
-import org.elasticsearch.snapshots.Snapshot;
+import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -202,7 +202,7 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction<Sn
                         // This is a snapshot the is currently running - skipping
                         continue;
                     }
-                    Snapshot snapshot = snapshotsService.snapshot(snapshotId);
+                    SnapshotInfo snapshot = snapshotsService.snapshot(snapshotId);
                     List<SnapshotIndexShardStatus> shardStatusBuilder = new ArrayList<>();
                     if (snapshot.state().completed()) {
                         Map<ShardId, IndexShardSnapshotStatus> shardStatues = snapshotsService.snapshotShards(snapshotId);
diff --git a/core/src/main/java/org/elasticsearch/repositories/Repository.java b/core/src/main/java/org/elasticsearch/repositories/Repository.java
index 294b36df491..f3c44f29226 100644
--- a/core/src/main/java/org/elasticsearch/repositories/Repository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/Repository.java
@@ -24,7 +24,7 @@ import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
-import org.elasticsearch.snapshots.Snapshot;
+import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotShardFailure;
 
 import java.io.IOException;
@@ -54,7 +54,7 @@ public interface Repository extends LifecycleComponent<Repository> {
      * @param snapshotId snapshot ID
      * @return information about snapshot
      */
-    Snapshot readSnapshot(SnapshotId snapshotId);
+    SnapshotInfo readSnapshot(SnapshotId snapshotId);
 
     /**
      * Returns global metadata associate with the snapshot.
@@ -65,7 +65,7 @@ public interface Repository extends LifecycleComponent<Repository> {
      * @param indices    list of indices
      * @return information about snapshot
      */
-    MetaData readSnapshotMetaData(SnapshotId snapshotId, Snapshot snapshot, List<String> indices) throws IOException;
+    MetaData readSnapshotMetaData(SnapshotId snapshotId, SnapshotInfo snapshot, List<String> indices) throws IOException;
 
     /**
      * Returns the list of snapshots currently stored in the repository
@@ -94,7 +94,7 @@ public interface Repository extends LifecycleComponent<Repository> {
      * @param shardFailures list of shard failures
      * @return snapshot description
      */
-    Snapshot finalizeSnapshot(SnapshotId snapshotId, List<String> indices, long startTime, String failure, int totalShards, List<SnapshotShardFailure> shardFailures);
+    SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, List<String> indices, long startTime, String failure, int totalShards, List<SnapshotShardFailure> shardFailures);
 
     /**
      * Deletes snapshot
diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index ef0cab5c156..121df3e5832 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -57,9 +57,9 @@ import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.RepositorySettings;
 import org.elasticsearch.repositories.RepositoryVerificationException;
 import org.elasticsearch.snapshots.InvalidSnapshotNameException;
-import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotCreationException;
 import org.elasticsearch.snapshots.SnapshotException;
+import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotMissingException;
 import org.elasticsearch.snapshots.SnapshotShardFailure;
 
@@ -165,9 +165,9 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
 
     private LegacyBlobStoreFormat<IndexMetaData> indexMetaDataLegacyFormat;
 
-    private ChecksumBlobStoreFormat<Snapshot> snapshotFormat;
+    private ChecksumBlobStoreFormat<SnapshotInfo> snapshotFormat;
 
-    private LegacyBlobStoreFormat<Snapshot> snapshotLegacyFormat;
+    private LegacyBlobStoreFormat<SnapshotInfo> snapshotLegacyFormat;
 
     private final boolean readOnly;
 
@@ -202,8 +202,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
         indexMetaDataFormat = new ChecksumBlobStoreFormat<>(INDEX_METADATA_CODEC, METADATA_NAME_FORMAT, IndexMetaData.PROTO, parseFieldMatcher, isCompress());
         indexMetaDataLegacyFormat = new LegacyBlobStoreFormat<>(LEGACY_SNAPSHOT_NAME_FORMAT, IndexMetaData.PROTO, parseFieldMatcher);
 
-        snapshotFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_CODEC, SNAPSHOT_NAME_FORMAT, Snapshot.PROTO, parseFieldMatcher, isCompress());
-        snapshotLegacyFormat = new LegacyBlobStoreFormat<>(LEGACY_SNAPSHOT_NAME_FORMAT, Snapshot.PROTO, parseFieldMatcher);
+        snapshotFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_CODEC, SNAPSHOT_NAME_FORMAT, SnapshotInfo.PROTO, parseFieldMatcher, isCompress());
+        snapshotLegacyFormat = new LegacyBlobStoreFormat<>(LEGACY_SNAPSHOT_NAME_FORMAT, SnapshotInfo.PROTO, parseFieldMatcher);
     }
 
     /**
@@ -294,7 +294,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
             throw new RepositoryException(this.repositoryName, "cannot delete snapshot from a readonly repository");
         }
         List<String> indices = Collections.emptyList();
-        Snapshot snapshot = null;
+        SnapshotInfo snapshot = null;
         try {
             snapshot = readSnapshot(snapshotId);
             indices = snapshot.indices();
@@ -368,9 +368,9 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
      * {@inheritDoc}
      */
     @Override
-    public Snapshot finalizeSnapshot(SnapshotId snapshotId, List<String> indices, long startTime, String failure, int totalShards, List<SnapshotShardFailure> shardFailures) {
+    public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, List<String> indices, long startTime, String failure, int totalShards, List<SnapshotShardFailure> shardFailures) {
         try {
-            Snapshot blobStoreSnapshot = new Snapshot(snapshotId.getSnapshot(), indices, startTime, failure, System.currentTimeMillis(), totalShards, shardFailures);
+            SnapshotInfo blobStoreSnapshot = new SnapshotInfo(snapshotId.getSnapshot(), indices, startTime, failure, System.currentTimeMillis(), totalShards, shardFailures);
             snapshotFormat.write(blobStoreSnapshot, snapshotsBlobContainer, snapshotId.getSnapshot());
             List<SnapshotId> snapshotIds = snapshots();
             if (!snapshotIds.contains(snapshotId)) {
@@ -425,7 +425,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
      * {@inheritDoc}
      */
     @Override
-    public MetaData readSnapshotMetaData(SnapshotId snapshotId, Snapshot snapshot, List<String> indices) throws IOException {
+    public MetaData readSnapshotMetaData(SnapshotId snapshotId, SnapshotInfo snapshot, List<String> indices) throws IOException {
         return readSnapshotMetaData(snapshotId, snapshot.version(), indices, false);
     }
 
@@ -433,7 +433,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
      * {@inheritDoc}
      */
     @Override
-    public Snapshot readSnapshot(SnapshotId snapshotId) {
+    public SnapshotInfo readSnapshot(SnapshotId snapshotId) {
         try {
             return snapshotFormat.read(snapshotsBlobContainer, snapshotId.getSnapshot());
         } catch (FileNotFoundException | NoSuchFileException ex) {
@@ -520,7 +520,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
     /**
      * Returns appropriate snapshot format based on the provided version of the snapshot
      */
-    private BlobStoreFormat<Snapshot> snapshotFormat(Version version) {
+    private BlobStoreFormat<SnapshotInfo> snapshotFormat(Version version) {
         if(legacyMetaData(version)) {
             return snapshotLegacyFormat;
         } else {
diff --git a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
index 8419b7c2667..5338f927005 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
@@ -190,7 +190,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
             // Read snapshot info and metadata from the repository
             Repository repository = repositoriesService.repository(request.repository());
             final SnapshotId snapshotId = new SnapshotId(request.repository(), request.name());
-            final Snapshot snapshot = repository.readSnapshot(snapshotId);
+            final SnapshotInfo snapshot = repository.readSnapshot(snapshotId);
             List<String> filteredIndices = SnapshotUtils.filterIndices(snapshot.indices(), request.indices(), request.indicesOptions());
             MetaData metaDataIn = repository.readSnapshotMetaData(snapshotId, snapshot, filteredIndices);
 
@@ -708,7 +708,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
      * @param snapshotId snapshot id
      * @param snapshot   snapshot metadata
      */
-    private void validateSnapshotRestorable(SnapshotId snapshotId, Snapshot snapshot) {
+    private void validateSnapshotRestorable(SnapshotId snapshotId, SnapshotInfo snapshot) {
         if (!snapshot.state().restorable()) {
             throw new SnapshotRestoreException(snapshotId, "unsupported snapshot state [" + snapshot.state() + "]");
         }
@@ -765,7 +765,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
                     UPDATE_RESTORE_ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME);
     }
 
-    private boolean failed(Snapshot snapshot, String index) {
+    private boolean failed(SnapshotInfo snapshot, String index) {
         for (SnapshotShardFailure failure : snapshot.shardFailures()) {
             if (index.equals(failure.index())) {
                 return true;
diff --git a/core/src/main/java/org/elasticsearch/snapshots/Snapshot.java b/core/src/main/java/org/elasticsearch/snapshots/Snapshot.java
deleted file mode 100644
index 13ec659b629..00000000000
--- a/core/src/main/java/org/elasticsearch/snapshots/Snapshot.java
+++ /dev/null
@@ -1,358 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.snapshots;
-
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.Version;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.xcontent.FromXContentBuilder;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * Represent information about snapshot
- */
-public class Snapshot implements Comparable<Snapshot>, ToXContent, FromXContentBuilder<Snapshot> {
-
-    private final String name;
-
-    private final Version version;
-
-    private final SnapshotState state;
-
-    private final String reason;
-
-    private final List<String> indices;
-
-    private final long startTime;
-
-    private final long endTime;
-
-    private final int totalShard;
-
-    private final int successfulShards;
-
-    private final List<SnapshotShardFailure> shardFailures;
-
-    private final static List<SnapshotShardFailure> NO_FAILURES = Collections.emptyList();
-
-    public final static Snapshot PROTO = new Snapshot();
-
-    private Snapshot(String name, List<String> indices, SnapshotState state, String reason, Version version, long startTime, long endTime,
-                              int totalShard, int successfulShards, List<SnapshotShardFailure> shardFailures) {
-        assert name != null;
-        assert indices != null;
-        assert state != null;
-        assert shardFailures != null;
-        this.name = name;
-        this.indices = indices;
-        this.state = state;
-        this.reason = reason;
-        this.version = version;
-        this.startTime = startTime;
-        this.endTime = endTime;
-        this.totalShard = totalShard;
-        this.successfulShards = successfulShards;
-        this.shardFailures = shardFailures;
-    }
-
-
-    public Snapshot(String name, List<String> indices, long startTime) {
-        this(name, indices, SnapshotState.IN_PROGRESS, null, Version.CURRENT, startTime, 0L, 0, 0, NO_FAILURES);
-    }
-
-    public Snapshot(String name, List<String> indices, long startTime, String reason, long endTime,
-                             int totalShard, List<SnapshotShardFailure> shardFailures) {
-        this(name, indices, snapshotState(reason, shardFailures), reason, Version.CURRENT,
-                startTime, endTime, totalShard, totalShard - shardFailures.size(), shardFailures);
-    }
-
-    /**
-     * Special constructor for the prototype object
-     */
-    private Snapshot() {
-        this("", Collections.emptyList(), 0);
-    }
-
-    private static SnapshotState snapshotState(String reason, List<SnapshotShardFailure> shardFailures) {
-        if (reason == null) {
-            if (shardFailures.isEmpty()) {
-                return SnapshotState.SUCCESS;
-            } else {
-                return SnapshotState.PARTIAL;
-            }
-        } else {
-            return SnapshotState.FAILED;
-        }
-    }
-
-    /**
-     * Returns snapshot name
-     *
-     * @return snapshot name
-     */
-    public String name() {
-        return name;
-    }
-
-    /**
-     * Returns current snapshot state
-     *
-     * @return snapshot state
-     */
-    public SnapshotState state() {
-        return state;
-    }
-
-    /**
-     * Returns reason for complete snapshot failure
-     *
-     * @return snapshot failure reason
-     */
-    public String reason() {
-        return reason;
-    }
-
-    /**
-     * Returns version of Elasticsearch that was used to create this snapshot
-     *
-     * @return Elasticsearch version
-     */
-    public Version version() {
-        return version;
-    }
-
-    /**
-     * Returns indices that were included into this snapshot
-     *
-     * @return list of indices
-     */
-    public List<String> indices() {
-        return indices;
-    }
-
-    /**
-     * Returns time when snapshot started
-     *
-     * @return snapshot start time
-     */
-    public long startTime() {
-        return startTime;
-    }
-
-    /**
-     * Returns time when snapshot ended
-     * <p>
-     * Can be 0L if snapshot is still running
-     *
-     * @return snapshot end time
-     */
-    public long endTime() {
-        return endTime;
-    }
-
-    /**
-     * Returns total number of shards that were snapshotted
-     *
-     * @return number of shards
-     */
-    public int totalShard() {
-        return totalShard;
-    }
-
-    /**
-     * Returns total number of shards that were successfully snapshotted
-     *
-     * @return number of successful shards
-     */
-    public int successfulShards() {
-        return successfulShards;
-    }
-
-    /**
-     * Returns shard failures
-     */
-    public List<SnapshotShardFailure> shardFailures() {
-        return shardFailures;
-    }
-
-    /**
-     * Compares two snapshots by their start time
-     *
-     * @param o other snapshot
-     * @return the value {@code 0} if snapshots were created at the same time;
-     * a value less than {@code 0} if this snapshot was created before snapshot {@code o}; and
-     * a value greater than {@code 0} if this snapshot was created after snapshot {@code o};
-     */
-    @Override
-    public int compareTo(Snapshot o) {
-        return Long.compare(startTime, o.startTime);
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-
-        Snapshot that = (Snapshot) o;
-
-        if (startTime != that.startTime) return false;
-        if (!name.equals(that.name)) return false;
-
-        return true;
-    }
-
-    @Override
-    public int hashCode() {
-        int result = name.hashCode();
-        result = 31 * result + Long.hashCode(startTime);
-        return result;
-    }
-
-    @Override
-    public Snapshot fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
-        return fromXContent(parser);
-    }
-
-    static final class Fields {
-        static final String SNAPSHOT = "snapshot";
-        static final String NAME = "name";
-        static final String VERSION_ID = "version_id";
-        static final String INDICES = "indices";
-        static final String STATE = "state";
-        static final String REASON = "reason";
-        static final String START_TIME = "start_time";
-        static final String END_TIME = "end_time";
-        static final String TOTAL_SHARDS = "total_shards";
-        static final String SUCCESSFUL_SHARDS = "successful_shards";
-        static final String FAILURES = "failures";
-    }
-
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
-        builder.startObject(Fields.SNAPSHOT);
-        builder.field(Fields.NAME, name);
-        builder.field(Fields.VERSION_ID, version.id);
-        builder.startArray(Fields.INDICES);
-        for (String index : indices) {
-            builder.value(index);
-        }
-        builder.endArray();
-        builder.field(Fields.STATE, state);
-        if (reason != null) {
-            builder.field(Fields.REASON, reason);
-        }
-        builder.field(Fields.START_TIME, startTime);
-        builder.field(Fields.END_TIME, endTime);
-        builder.field(Fields.TOTAL_SHARDS, totalShard);
-        builder.field(Fields.SUCCESSFUL_SHARDS, successfulShards);
-        builder.startArray(Fields.FAILURES);
-        for (SnapshotShardFailure shardFailure : shardFailures) {
-            builder.startObject();
-            shardFailure.toXContent(builder, params);
-            builder.endObject();
-        }
-        builder.endArray();
-        builder.endObject();
-        return builder;
-    }
-
-
-    public static Snapshot fromXContent(XContentParser parser) throws IOException {
-        String name = null;
-        Version version = Version.CURRENT;
-        SnapshotState state = SnapshotState.IN_PROGRESS;
-        String reason = null;
-        List<String> indices = Collections.emptyList();
-        long startTime = 0;
-        long endTime = 0;
-        int totalShard = 0;
-        int successfulShards = 0;
-        List<SnapshotShardFailure> shardFailures = NO_FAILURES;
-        if (parser.currentToken() == null) { // fresh parser? move to the first token
-            parser.nextToken();
-        }
-        if (parser.currentToken() == XContentParser.Token.START_OBJECT) {  // on a start object move to next token
-            parser.nextToken();
-        }
-        XContentParser.Token token;
-        if ((token = parser.nextToken()) == XContentParser.Token.START_OBJECT) {
-            String currentFieldName = parser.currentName();
-            if ("snapshot".equals(currentFieldName)) {
-                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                    if (token == XContentParser.Token.FIELD_NAME) {
-                        currentFieldName = parser.currentName();
-                        token = parser.nextToken();
-                        if (token.isValue()) {
-                            if ("name".equals(currentFieldName)) {
-                                name = parser.text();
-                            } else if ("state".equals(currentFieldName)) {
-                                state = SnapshotState.valueOf(parser.text());
-                            } else if ("reason".equals(currentFieldName)) {
-                                reason = parser.text();
-                            } else if ("start_time".equals(currentFieldName)) {
-                                startTime = parser.longValue();
-                            } else if ("end_time".equals(currentFieldName)) {
-                                endTime = parser.longValue();
-                            } else if ("total_shards".equals(currentFieldName)) {
-                                totalShard = parser.intValue();
-                            } else if ("successful_shards".equals(currentFieldName)) {
-                                successfulShards = parser.intValue();
-                            } else if ("version_id".equals(currentFieldName)) {
-                                version = Version.fromId(parser.intValue());
-                            }
-                        } else if (token == XContentParser.Token.START_ARRAY) {
-                            if ("indices".equals(currentFieldName)) {
-                                ArrayList<String> indicesArray = new ArrayList<>();
-                                while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
-                                    indicesArray.add(parser.text());
-                                }
-                                indices = Collections.unmodifiableList(indicesArray);
-                            } else if ("failures".equals(currentFieldName)) {
-                                ArrayList<SnapshotShardFailure> shardFailureArrayList = new ArrayList<>();
-                                while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
-                                    shardFailureArrayList.add(SnapshotShardFailure.fromXContent(parser));
-                                }
-                                shardFailures = Collections.unmodifiableList(shardFailureArrayList);
-                            } else {
-                                // It was probably created by newer version - ignoring
-                                parser.skipChildren();
-                            }
-                        } else if (token == XContentParser.Token.START_OBJECT) {
-                            // It was probably created by newer version - ignoring
-                            parser.skipChildren();
-                        }
-                    }
-                }
-            }
-        } else {
-            throw new ElasticsearchParseException("unexpected token  [" + token + "]");
-        }
-        return new Snapshot(name, indices, state, reason, version, startTime, endTime, totalShard, successfulShards, shardFailures);
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
index 354094404ae..871e765cfd0 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
@@ -18,15 +18,19 @@
  */
 package org.elasticsearch.snapshots;
 
+import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ShardOperationFailedException;
+import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
 import org.elasticsearch.common.joda.Joda;
+import org.elasticsearch.common.xcontent.FromXContentBuilder;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
@@ -35,52 +39,109 @@ import java.util.Collections;
 import java.util.List;
 
 /**
- * Information about snapshot
+ * Information about a snapshot
  */
-public class SnapshotInfo implements ToXContent, Streamable {
+public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent, FromXContentBuilder<SnapshotInfo>, Writeable {
 
+    public static final SnapshotInfo PROTO = new SnapshotInfo("", Collections.emptyList(), 0);
     private static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("strictDateOptionalTime");
+    private static final String SNAPSHOT = "snapshot";
+    private static final String INDICES = "indices";
+    private static final String STATE = "state";
+    private static final String REASON = "reason";
+    private static final String START_TIME = "start_time";
+    private static final String START_TIME_IN_MILLIS = "start_time_in_millis";
+    private static final String END_TIME = "end_time";
+    private static final String END_TIME_IN_MILLIS = "end_time_in_millis";
+    private static final String DURATION = "duration";
+    private static final String DURATION_IN_MILLIS = "duration_in_millis";
+    private static final String FAILURES = "failures";
+    private static final String SHARDS = "shards";
+    private static final String TOTAL = "total";
+    private static final String FAILED = "failed";
+    private static final String SUCCESSFUL = "successful";
+    private static final String VERSION_ID = "version_id";
+    private static final String VERSION = "version";
+    private static final String NAME = "name";
+    private static final String TOTAL_SHARDS = "total_shards";
+    private static final String SUCCESSFUL_SHARDS = "successful_shards";
 
-    private String name;
+    private final String name;
 
-    private SnapshotState state;
+    private final SnapshotState state;
 
-    private String reason;
+    private final String reason;
 
-    private List<String> indices;
+    private final List<String> indices;
 
-    private long startTime;
+    private final long startTime;
 
-    private long endTime;
+    private final long endTime;
 
-    private int totalShards;
+    private final int totalShards;
 
-    private int successfulShards;
+    private final int successfulShards;
 
-    private Version version;
+    private final Version version;
 
-    private List<SnapshotShardFailure> shardFailures;
+    private final List<SnapshotShardFailure> shardFailures;
 
-    SnapshotInfo() {
+    public SnapshotInfo(String name, List<String> indices, long startTime) {
+        this(name, indices, SnapshotState.IN_PROGRESS, null, Version.CURRENT, startTime, 0L, 0, 0, Collections.emptyList());
+    }
 
+    public SnapshotInfo(String name, List<String> indices, long startTime, String reason, long endTime,
+                        int totalShards, List<SnapshotShardFailure> shardFailures) {
+        this(name, indices, snapshotState(reason, shardFailures), reason, Version.CURRENT,
+             startTime, endTime, totalShards, totalShards - shardFailures.size(), shardFailures);
+    }
+
+    private SnapshotInfo(String name, List<String> indices, SnapshotState state, String reason, Version version, long startTime,
+                         long endTime, int totalShards, int successfulShards, List<SnapshotShardFailure> shardFailures) {
+        assert name != null;
+        assert indices != null;
+        assert state != null;
+        assert shardFailures != null;
+        this.name = name;
+        this.indices = indices;
+        this.state = state;
+        this.reason = reason;
+        this.version = version;
+        this.startTime = startTime;
+        this.endTime = endTime;
+        this.totalShards = totalShards;
+        this.successfulShards = successfulShards;
+        this.shardFailures = shardFailures;
     }
 
     /**
-     * Creates a new snapshot information from a {@link Snapshot}
-     *
-     * @param snapshot snapshot information returned by repository
+     * Constructs snapshot information from stream input
      */
-    public SnapshotInfo(Snapshot snapshot) {
-        name = snapshot.name();
-        state = snapshot.state();
-        reason = snapshot.reason();
-        indices = snapshot.indices();
-        startTime = snapshot.startTime();
-        endTime = snapshot.endTime();
-        totalShards = snapshot.totalShard();
-        successfulShards = snapshot.successfulShards();
-        shardFailures = snapshot.shardFailures();
-        version = snapshot.version();
+    public SnapshotInfo(final StreamInput in) throws IOException {
+        name = in.readString();
+        int size = in.readVInt();
+        List<String> indicesListBuilder = new ArrayList<>();
+        for (int i = 0; i < size; i++) {
+            indicesListBuilder.add(in.readString());
+        }
+        indices = Collections.unmodifiableList(indicesListBuilder);
+        state = SnapshotState.fromValue(in.readByte());
+        reason = in.readOptionalString();
+        startTime = in.readVLong();
+        endTime = in.readVLong();
+        totalShards = in.readVInt();
+        successfulShards = in.readVInt();
+        size = in.readVInt();
+        if (size > 0) {
+            List<SnapshotShardFailure> failureBuilder = new ArrayList<>();
+            for (int i = 0; i < size; i++) {
+                failureBuilder.add(SnapshotShardFailure.readSnapshotShardFailure(in));
+            }
+            shardFailures = Collections.unmodifiableList(failureBuilder);
+        } else {
+            shardFailures = Collections.emptyList();
+        }
+        version = Version.readVersion(in);
     }
 
     /**
@@ -184,6 +245,39 @@ public class SnapshotInfo implements ToXContent, Streamable {
         return version;
     }
 
+    /**
+     * Compares two snapshots by their start time
+     *
+     * @param o other snapshot
+     * @return the value {@code 0} if snapshots were created at the same time;
+     * a value less than {@code 0} if this snapshot was created before snapshot {@code o}; and
+     * a value greater than {@code 0} if this snapshot was created after snapshot {@code o};
+     */
+    @Override
+    public int compareTo(final SnapshotInfo o) {
+        return Long.compare(startTime, o.startTime);
+    }
+
+    @Override
+    public boolean equals(final Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+
+        final SnapshotInfo that = (SnapshotInfo) o;
+        return startTime == that.startTime && name.equals(that.name);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = name.hashCode();
+        result = 31 * result + Long.hashCode(startTime);
+        return result;
+    }
+
     /**
      * Returns snapshot REST status
      */
@@ -194,98 +288,166 @@ public class SnapshotInfo implements ToXContent, Streamable {
         if (shardFailures.size() == 0) {
             return RestStatus.OK;
         }
-        return RestStatus.status(successfulShards, totalShards, shardFailures.toArray(new ShardOperationFailedException[shardFailures.size()]));
-    }
-
-    static final class Fields {
-        static final String INDICES = "indices";
-        static final String STATE = "state";
-        static final String REASON = "reason";
-        static final String START_TIME = "start_time";
-        static final String START_TIME_IN_MILLIS = "start_time_in_millis";
-        static final String END_TIME = "end_time";
-        static final String END_TIME_IN_MILLIS = "end_time_in_millis";
-        static final String DURATION = "duration";
-        static final String DURATION_IN_MILLIS = "duration_in_millis";
-        static final String FAILURES = "failures";
-        static final String SHARDS = "shards";
-        static final String TOTAL = "total";
-        static final String FAILED = "failed";
-        static final String SUCCESSFUL = "successful";
-        static final String VERSION_ID = "version_id";
-        static final String VERSION = "version";
+        return RestStatus.status(successfulShards, totalShards,
+                                 shardFailures.toArray(new ShardOperationFailedException[shardFailures.size()]));
     }
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        builder.field("snapshot", name);
-        builder.field(Fields.VERSION_ID, version.id);
-        builder.field(Fields.VERSION, version.toString());
-        builder.startArray(Fields.INDICES);
+    public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
+        builder.startObject(SNAPSHOT);
+        builder.field(NAME, name);
+        builder.field(VERSION_ID, version.id);
+        builder.startArray(INDICES);
         for (String index : indices) {
             builder.value(index);
         }
         builder.endArray();
-        builder.field(Fields.STATE, state);
+        builder.field(STATE, state);
         if (reason != null) {
-            builder.field(Fields.REASON, reason);
+            builder.field(REASON, reason);
         }
-        if (startTime != 0) {
-            builder.field(Fields.START_TIME, DATE_TIME_FORMATTER.printer().print(startTime));
-            builder.field(Fields.START_TIME_IN_MILLIS, startTime);
-        }
-        if (endTime != 0) {
-            builder.field(Fields.END_TIME, DATE_TIME_FORMATTER.printer().print(endTime));
-            builder.field(Fields.END_TIME_IN_MILLIS, endTime);
-            builder.timeValueField(Fields.DURATION_IN_MILLIS, Fields.DURATION, endTime - startTime);
-        }
-        builder.startArray(Fields.FAILURES);
+        builder.field(START_TIME, startTime);
+        builder.field(END_TIME, endTime);
+        builder.field(TOTAL_SHARDS, totalShards);
+        builder.field(SUCCESSFUL_SHARDS, successfulShards);
+        builder.startArray(FAILURES);
         for (SnapshotShardFailure shardFailure : shardFailures) {
             builder.startObject();
             shardFailure.toXContent(builder, params);
             builder.endObject();
         }
         builder.endArray();
-        builder.startObject(Fields.SHARDS);
-        builder.field(Fields.TOTAL, totalShards);
-        builder.field(Fields.FAILED, failedShards());
-        builder.field(Fields.SUCCESSFUL, successfulShards);
+        builder.endObject();
+        return builder;
+    }
+
+    /**
+     * Produces the external X-content that is delivered through the REST layer.
+     */
+    public XContentBuilder toExternalXContent(final XContentBuilder builder, final ToXContent.Params params) throws IOException {
+        builder.startObject();
+        builder.field(SNAPSHOT, name);
+        builder.field(VERSION_ID, version.id);
+        builder.field(VERSION, version.toString());
+        builder.startArray(INDICES);
+        for (String index : indices) {
+            builder.value(index);
+        }
+        builder.endArray();
+        builder.field(STATE, state);
+        if (reason != null) {
+            builder.field(REASON, reason);
+        }
+        if (startTime != 0) {
+            builder.field(START_TIME, DATE_TIME_FORMATTER.printer().print(startTime));
+            builder.field(START_TIME_IN_MILLIS, startTime);
+        }
+        if (endTime != 0) {
+            builder.field(END_TIME, DATE_TIME_FORMATTER.printer().print(endTime));
+            builder.field(END_TIME_IN_MILLIS, endTime);
+            builder.timeValueField(DURATION_IN_MILLIS, DURATION, endTime - startTime);
+        }
+        builder.startArray(FAILURES);
+        for (SnapshotShardFailure shardFailure : shardFailures) {
+            builder.startObject();
+            shardFailure.toXContent(builder, params);
+            builder.endObject();
+        }
+        builder.endArray();
+        builder.startObject(SHARDS);
+        builder.field(TOTAL, totalShards);
+        builder.field(FAILED, failedShards());
+        builder.field(SUCCESSFUL, successfulShards);
         builder.endObject();
         builder.endObject();
         return builder;
     }
 
     @Override
-    public void readFrom(StreamInput in) throws IOException {
-        name = in.readString();
-        int size = in.readVInt();
-        List<String> indicesListBuilder = new ArrayList<>();
-        for (int i = 0; i < size; i++) {
-            indicesListBuilder.add(in.readString());
+    public SnapshotInfo fromXContent(final XContentParser parser, final ParseFieldMatcher matcher) throws IOException {
+        return fromXContent(parser);
+    }
+
+    /**
+     * This method creates a SnapshotInfo from internal x-content.  It does not
+     * handle x-content written with the external version as external x-content
+     * is only for display purposes and does not need to be parsed.
+     */
+    public static SnapshotInfo fromXContent(final XContentParser parser) throws IOException {
+        String name = null;
+        Version version = Version.CURRENT;
+        SnapshotState state = SnapshotState.IN_PROGRESS;
+        String reason = null;
+        List<String> indices = Collections.emptyList();
+        long startTime = 0;
+        long endTime = 0;
+        int totalShard = 0;
+        int successfulShards = 0;
+        List<SnapshotShardFailure> shardFailures = Collections.emptyList();
+        if (parser.currentToken() == null) { // fresh parser? move to the first token
+            parser.nextToken();
         }
-        indices = Collections.unmodifiableList(indicesListBuilder);
-        state = SnapshotState.fromValue(in.readByte());
-        reason = in.readOptionalString();
-        startTime = in.readVLong();
-        endTime = in.readVLong();
-        totalShards = in.readVInt();
-        successfulShards = in.readVInt();
-        size = in.readVInt();
-        if (size > 0) {
-            List<SnapshotShardFailure> failureBuilder = new ArrayList<>();
-            for (int i = 0; i < size; i++) {
-                failureBuilder.add(SnapshotShardFailure.readSnapshotShardFailure(in));
+        if (parser.currentToken() == XContentParser.Token.START_OBJECT) {  // on a start object move to next token
+            parser.nextToken();
+        }
+        XContentParser.Token token;
+        if ((token = parser.nextToken()) == XContentParser.Token.START_OBJECT) {
+            String currentFieldName = parser.currentName();
+            if (SNAPSHOT.equals(currentFieldName)) {
+                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                    if (token == XContentParser.Token.FIELD_NAME) {
+                        currentFieldName = parser.currentName();
+                        token = parser.nextToken();
+                        if (token.isValue()) {
+                            if (NAME.equals(currentFieldName)) {
+                                name = parser.text();
+                            } else if (STATE.equals(currentFieldName)) {
+                                state = SnapshotState.valueOf(parser.text());
+                            } else if (REASON.equals(currentFieldName)) {
+                                reason = parser.text();
+                            } else if (START_TIME.equals(currentFieldName)) {
+                                startTime = parser.longValue();
+                            } else if (END_TIME.equals(currentFieldName)) {
+                                endTime = parser.longValue();
+                            } else if (TOTAL_SHARDS.equals(currentFieldName)) {
+                                totalShard = parser.intValue();
+                            } else if (SUCCESSFUL_SHARDS.equals(currentFieldName)) {
+                                successfulShards = parser.intValue();
+                            } else if (VERSION_ID.equals(currentFieldName)) {
+                                version = Version.fromId(parser.intValue());
+                            }
+                        } else if (token == XContentParser.Token.START_ARRAY) {
+                            if (INDICES.equals(currentFieldName)) {
+                                ArrayList<String> indicesArray = new ArrayList<>();
+                                while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
+                                    indicesArray.add(parser.text());
+                                }
+                                indices = Collections.unmodifiableList(indicesArray);
+                            } else if (FAILURES.equals(currentFieldName)) {
+                                ArrayList<SnapshotShardFailure> shardFailureArrayList = new ArrayList<>();
+                                while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
+                                    shardFailureArrayList.add(SnapshotShardFailure.fromXContent(parser));
+                                }
+                                shardFailures = Collections.unmodifiableList(shardFailureArrayList);
+                            } else {
+                                // It was probably created by newer version - ignoring
+                                parser.skipChildren();
+                            }
+                        } else if (token == XContentParser.Token.START_OBJECT) {
+                            // It was probably created by newer version - ignoring
+                            parser.skipChildren();
+                        }
+                    }
+                }
             }
-            shardFailures = Collections.unmodifiableList(failureBuilder);
         } else {
-            shardFailures = Collections.emptyList();
+            throw new ElasticsearchParseException("unexpected token  [" + token + "]");
         }
-        version = Version.readVersion(in);
+        return new SnapshotInfo(name, indices, state, reason, version, startTime, endTime, totalShard, successfulShards, shardFailures);
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
+    public void writeTo(final StreamOutput out) throws IOException {
         out.writeString(name);
         out.writeVInt(indices.size());
         for (String index : indices) {
@@ -304,26 +466,16 @@ public class SnapshotInfo implements ToXContent, Streamable {
         Version.writeVersion(version, out);
     }
 
-    /**
-     * Reads snapshot information from stream input
-     *
-     * @param in stream input
-     * @return deserialized snapshot info
-     */
-    public static SnapshotInfo readSnapshotInfo(StreamInput in) throws IOException {
-        SnapshotInfo snapshotInfo = new SnapshotInfo();
-        snapshotInfo.readFrom(in);
-        return snapshotInfo;
-    }
-
-    /**
-     * Reads optional snapshot information from stream input
-     *
-     * @param in stream input
-     * @return deserialized snapshot info or null
-     */
-    public static SnapshotInfo readOptionalSnapshotInfo(StreamInput in) throws IOException {
-        return in.readOptionalStreamable(SnapshotInfo::new);
+    private static SnapshotState snapshotState(final String reason, final List<SnapshotShardFailure> shardFailures) {
+        if (reason == null) {
+            if (shardFailures.isEmpty()) {
+                return SnapshotState.SUCCESS;
+            } else {
+                return SnapshotState.PARTIAL;
+            }
+        } else {
+            return SnapshotState.FAILED;
+        }
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
index 186a1965a96..8e6681893c9 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
@@ -126,7 +126,7 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
      * @return snapshot
      * @throws SnapshotMissingException if snapshot is not found
      */
-    public Snapshot snapshot(SnapshotId snapshotId) {
+    public SnapshotInfo snapshot(SnapshotId snapshotId) {
         validate(snapshotId);
         List<SnapshotsInProgress.Entry> entries = currentSnapshots(snapshotId.getRepository(), new String[]{snapshotId.getSnapshot()});
         if (!entries.isEmpty()) {
@@ -141,8 +141,8 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
      * @param repositoryName repository name
      * @return list of snapshots
      */
-    public List<Snapshot> snapshots(String repositoryName, boolean ignoreUnavailable) {
-        Set<Snapshot> snapshotSet = new HashSet<>();
+    public List<SnapshotInfo> snapshots(String repositoryName, boolean ignoreUnavailable) {
+        Set<SnapshotInfo> snapshotSet = new HashSet<>();
         List<SnapshotsInProgress.Entry> entries = currentSnapshots(repositoryName, null);
         for (SnapshotsInProgress.Entry entry : entries) {
             snapshotSet.add(inProgressSnapshot(entry));
@@ -161,7 +161,7 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
             }
         }
 
-        ArrayList<Snapshot> snapshotList = new ArrayList<>(snapshotSet);
+        ArrayList<SnapshotInfo> snapshotList = new ArrayList<>(snapshotSet);
         CollectionUtil.timSort(snapshotList);
         return Collections.unmodifiableList(snapshotList);
     }
@@ -172,8 +172,8 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
      * @param repositoryName repository name
      * @return list of snapshots
      */
-    public List<Snapshot> currentSnapshots(String repositoryName) {
-        List<Snapshot> snapshotList = new ArrayList<>();
+    public List<SnapshotInfo> currentSnapshots(String repositoryName) {
+        List<SnapshotInfo> snapshotList = new ArrayList<>();
         List<SnapshotsInProgress.Entry> entries = currentSnapshots(repositoryName, null);
         for (SnapshotsInProgress.Entry entry : entries) {
             snapshotList.add(inProgressSnapshot(entry));
@@ -408,8 +408,8 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
         }
     }
 
-    private Snapshot inProgressSnapshot(SnapshotsInProgress.Entry entry) {
-        return new Snapshot(entry.snapshotId().getSnapshot(), entry.indices(), entry.startTime());
+    private SnapshotInfo inProgressSnapshot(SnapshotsInProgress.Entry entry) {
+        return new SnapshotInfo(entry.snapshotId().getSnapshot(), entry.indices(), entry.startTime());
     }
 
     /**
@@ -482,7 +482,7 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
         Map<ShardId, IndexShardSnapshotStatus> shardStatus = new HashMap<>();
         Repository repository = repositoriesService.repository(snapshotId.getRepository());
         IndexShardRepository indexShardRepository = repositoriesService.indexShardRepository(snapshotId.getRepository());
-        Snapshot snapshot = repository.readSnapshot(snapshotId);
+        SnapshotInfo snapshot = repository.readSnapshot(snapshotId);
         MetaData metaData = repository.readSnapshotMetaData(snapshotId, snapshot, snapshot.indices());
         for (String index : snapshot.indices()) {
             IndexMetaData indexMetaData = metaData.indices().get(index);
@@ -800,8 +800,8 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
                             shardFailures.add(new SnapshotShardFailure(status.nodeId(), shardId, status.reason()));
                         }
                     }
-                    Snapshot snapshot = repository.finalizeSnapshot(snapshotId, entry.indices(), entry.startTime(), failure, entry.shards().size(), Collections.unmodifiableList(shardFailures));
-                    removeSnapshotFromClusterState(snapshotId, new SnapshotInfo(snapshot), null);
+                    SnapshotInfo snapshot = repository.finalizeSnapshot(snapshotId, entry.indices(), entry.startTime(), failure, entry.shards().size(), Collections.unmodifiableList(shardFailures));
+                    removeSnapshotFromClusterState(snapshotId, snapshot, null);
                 } catch (Throwable t) {
                     logger.warn("[{}] failed to finalize snapshot", t, snapshotId);
                     removeSnapshotFromClusterState(snapshotId, null, t);

From 8ff699e41a59845a5a80b66240e18c989d4e1dbe Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Thu, 5 May 2016 14:21:14 -0700
Subject: [PATCH 0087/1311] Packaging: Make rpm not include parent dirs

closes #18162
---
 distribution/rpm/build.gradle | 1 +
 1 file changed, 1 insertion(+)

diff --git a/distribution/rpm/build.gradle b/distribution/rpm/build.gradle
index 0d9f658f488..72ed58e52c3 100644
--- a/distribution/rpm/build.gradle
+++ b/distribution/rpm/build.gradle
@@ -40,6 +40,7 @@ task buildRpm(type: Rpm) {
   vendor 'Elasticsearch'
   dirMode 0755
   fileMode 0644
+  addParentDirs false
   // TODO ospackage doesn't support icon but we used to have one
 }
 

From e3ce6c9048d95e6e19b02ccb209f17d28543170c Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Thu, 5 May 2016 18:31:48 -0400
Subject: [PATCH 0088/1311] Painless: add fielddata accessors
 (.value/.values/.distance()/etc)

This gives better coverage and consistency with the scripting APIs, by
whitelisting the primary search scripting API classes and using them instead
of only Map and List methods.

For example, accessing fields can now be done with `.value` instead of `.0`
because `getValue()` is whitelisted. For now, access to a document's fields in
this way (loads) are fast-pathed in the code, to avoid dynamic overhead.

Access to geo fields and geo distance functions is now supported.

TODO: date support (e.g. whitelist ReadableDateTime methods as a start)
TODO: improve docs (like expressions and groovy have for document's fields)
TODO: remove fast-path hack

Closes #18169

Squashed commit of the following:

commit ec9f24b2424891a7429bb4c0a03f9868cba0a213
Author: Robert Muir <rmuir@apache.org>
Date:   Thu May 5 17:59:37 2016 -0400

    cutover to <Def> instead of <Object> here

commit 9edb1550438acd209733bc36f0d2e0aecf190ecb
Author: Robert Muir <rmuir@apache.org>
Date:   Thu May 5 17:03:02 2016 -0400

    add fast-path for docvalues field loads

commit f8e38c0932fccc0cfa217516130ad61522e59fe5
Author: Robert Muir <rmuir@apache.org>
Date:   Thu May 5 16:47:31 2016 -0400

    Painless: add fielddata accessors (.value/.values/.distance()/etc)
---
 .../elasticsearch/common/geo/GeoPoint.java    |  2 +-
 .../modules/scripting/painless.asciidoc       |  9 +-
 .../java/org/elasticsearch/painless/Def.java  | 22 ++++-
 .../elasticsearch/painless/Definition.java    | 96 +++++++++++++++++++
 .../test/plan_a/20_scriptfield.yaml           |  2 +-
 .../rest-api-spec/test/plan_a/30_search.yaml  | 12 +--
 6 files changed, 126 insertions(+), 17 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java b/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java
index 7713157422e..5d1250a5148 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java
@@ -38,7 +38,7 @@ public final class GeoPoint {
     }
 
     /**
-     * Create a new Geopointform a string. This String must either be a geohash
+     * Create a new Geopoint from a string. This String must either be a geohash
      * or a lat-lon tuple.
      *
      * @param value String to create the point from
diff --git a/docs/reference/modules/scripting/painless.asciidoc b/docs/reference/modules/scripting/painless.asciidoc
index 98cb052d7a3..b8c4c2cc81f 100644
--- a/docs/reference/modules/scripting/painless.asciidoc
+++ b/docs/reference/modules/scripting/painless.asciidoc
@@ -115,9 +115,8 @@ GET hockey/_search
 ----------------------------------------------------------------
 // AUTOSENSE
 
-You must always specify the index of the field value you want, even if there's only a single item in the field.
-All fields in Elasticsearch are multi-valued and Painless does not provide a `.value` shortcut. The following example uses a Painless script to sort the players by their combined first and last names. The names are accessed using
-`input.doc['first'].0` and `input.doc['last'].0`.
+The following example uses a Painless script to sort the players by their combined first and last names. The names are accessed using
+`input.doc['first'].value` and `input.doc['last'].value`.
 
 [source,js]
 ----------------------------------------------------------------
@@ -132,7 +131,7 @@ GET hockey/_search
       "order": "asc",
       "script": {
         "lang": "painless",
-        "inline": "input.doc['first'].0 + ' ' + input.doc['last'].0"
+        "inline": "input.doc['first'].value + ' ' + input.doc['last'].value"
       }
     }
   }
@@ -218,7 +217,7 @@ GET hockey/_search
     "full_name_dynamic": {
       "script": {
         "lang": "painless",
-        "inline": "def first = input.doc['first'].0; def last = input.doc['last'].0; return first + ' ' + last;"
+        "inline": "def first = input.doc['first'].value; def last = input.doc['last'].value; return first + ' ' + last;"
       }
     },
     "full_name_static": {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 4e170c93e49..136746026fa 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.painless;
 
+import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Definition.Field;
 import org.elasticsearch.painless.Definition.Method;
@@ -119,9 +120,22 @@ public class Def {
 
     @SuppressWarnings("rawtypes")
     public static Object fieldLoad(final Object owner, final String name, final Definition definition) {
-        if (owner.getClass().isArray() && "length".equals(name)) {
+        final Class<?> clazz = owner.getClass();
+        if (clazz.isArray() && "length".equals(name)) {
             return Array.getLength(owner);
         } else {
+            // TODO: remove this fast-path, once we speed up dynamics some more
+            if ("value".equals(name) && owner instanceof ScriptDocValues) {
+                if (clazz == ScriptDocValues.Doubles.class) {
+                    return ((ScriptDocValues.Doubles)owner).getValue();
+                } else if (clazz == ScriptDocValues.Longs.class) {
+                    return ((ScriptDocValues.Longs)owner).getValue();
+                } else if (clazz == ScriptDocValues.Strings.class) {
+                    return ((ScriptDocValues.Strings)owner).getValue();
+                } else if (clazz == ScriptDocValues.GeoPoints.class) {
+                    return ((ScriptDocValues.GeoPoints)owner).getValue();
+                }
+            }
             final Field field = getField(owner, name, definition);
             MethodHandle handle;
 
@@ -143,7 +157,7 @@ public class Def {
                     }
                 } else {
                     throw new IllegalArgumentException("Unable to find dynamic field [" + name + "] " +
-                            "for class [" + owner.getClass().getCanonicalName() + "].");
+                            "for class [" + clazz.getCanonicalName() + "].");
                 }
             } else {
                 handle = field.getter;
@@ -151,13 +165,13 @@ public class Def {
 
             if (handle == null) {
                 throw new IllegalArgumentException(
-                        "Unable to read from field [" + name + "] with owner class [" + owner.getClass() + "].");
+                        "Unable to read from field [" + name + "] with owner class [" + clazz + "].");
             } else {
                 try {
                     return handle.invoke(owner);
                 } catch (final Throwable throwable) {
                     throw new IllegalArgumentException("Error loading value from " +
-                            "field [" + name + "] with owner class [" + owner.getClass() + "].", throwable);
+                            "field [" + name + "] with owner class [" + clazz + "].", throwable);
                 }
             }
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 450f2aa6fa4..5a0e0c1e636 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -33,6 +33,9 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.index.fielddata.ScriptDocValues;
+
 class Definition {
     enum Sort {
         VOID(       void.class      , 0 , true  , false , false , false ),
@@ -393,6 +396,14 @@ class Definition {
     final Type iargexcepType;
     final Type istateexceptType;
     final Type nfexcepType;
+    
+    // docvalues accessors
+    final Type geoPointType;
+    final Type stringsType;
+    // TODO: add ReadableDateTime? or don't expose the joda stuff?
+    final Type longsType;
+    final Type doublesType;
+    final Type geoPointsType;
 
     public Definition() {
         structs = new HashMap<>();
@@ -471,6 +482,12 @@ class Definition {
         istateexceptType = getType("IllegalStateException");
         nfexcepType = getType("NumberFormatException");
 
+        geoPointType = getType("GeoPoint");
+        stringsType = getType("Strings");
+        longsType = getType("Longs");
+        doublesType = getType("Doubles");
+        geoPointsType = getType("GeoPoints");
+
         addDefaultElements();
         copyDefaultStructs();
         addDefaultTransforms();
@@ -564,6 +581,12 @@ class Definition {
         iargexcepType = definition.iargexcepType;
         istateexceptType = definition.istateexceptType;
         nfexcepType = definition.nfexcepType;
+        
+        geoPointType = definition.geoPointType;
+        stringsType = definition.stringsType;
+        longsType = definition.longsType;
+        doublesType = definition.doublesType;
+        geoPointsType = definition.geoPointsType;
     }
 
     private void addDefaultStructs() {
@@ -634,6 +657,12 @@ class Definition {
         addStruct( "IllegalArgumentException" , IllegalArgumentException.class);
         addStruct( "IllegalStateException"    , IllegalStateException.class);
         addStruct( "NumberFormatException"    , NumberFormatException.class);
+        
+        addStruct( "GeoPoint"  , GeoPoint.class);
+        addStruct( "Strings"   , ScriptDocValues.Strings.class);
+        addStruct( "Longs"     , ScriptDocValues.Longs.class);
+        addStruct( "Doubles"   , ScriptDocValues.Doubles.class);
+        addStruct( "GeoPoints" , ScriptDocValues.GeoPoints.class);
     }
 
     private void addDefaultClasses() {
@@ -670,6 +699,12 @@ class Definition {
         addClass("HashMap");
 
         addClass("Exception");
+        
+        addClass("GeoPoint");
+        addClass("Strings");
+        addClass("Longs");
+        addClass("Doubles");
+        addClass("GeoPoints");
     }
 
     private void addDefaultElements() {
@@ -1032,6 +1067,61 @@ class Definition {
         addConstructor("IllegalStateException", "new", new Type[] {stringType}, null);
 
         addConstructor("NumberFormatException", "new", new Type[] {stringType}, null);
+        
+        addMethod("GeoPoint", "getLat", null, false, doubleType, new Type[] {}, null, null);
+        addMethod("GeoPoint", "getLon", null, false, doubleType, new Type[] {}, null, null);
+        addMethod("Strings", "getValue", null, false, stringType, new Type[] {}, null, null);
+        addMethod("Strings", "getValues", null, false, slistType, new Type[] {}, null, null);
+        addMethod("Longs", "getValue", null, false, longType, new Type[] {}, null, null);
+        addMethod("Longs", "getValues", null, false, olistType, new Type[] {}, null, null);
+        // TODO: add better date support for Longs here? (carefully?)
+        addMethod("Doubles", "getValue", null, false, doubleType, new Type[] {}, null, null);
+        addMethod("Doubles", "getValues", null, false, olistType, new Type[] {}, null, null);
+        addMethod("GeoPoints", "getValue", null, false, geoPointType, new Type[] {}, null, null);
+        addMethod("GeoPoints", "getValues", null, false, olistType, new Type[] {}, null, null);
+        addMethod("GeoPoints", "getLat", null, false, doubleType, new Type[] {}, null, null);
+        addMethod("GeoPoints", "getLon", null, false, doubleType, new Type[] {}, null, null);
+        addMethod("GeoPoints", "getLats", null, false, getType(doubleType.struct, 1), new Type[] {}, null, null);
+        addMethod("GeoPoints", "getLons", null, false, getType(doubleType.struct, 1), new Type[] {}, null, null);
+        // geo distance functions... so many...
+        addMethod("GeoPoints", "factorDistance", null, false, doubleType, 
+                  new Type[] { doubleType, doubleType }, null, null);
+        addMethod("GeoPoints", "factorDistanceWithDefault", null, false, doubleType, 
+                  new Type[] { doubleType, doubleType, doubleType }, null, null);
+        addMethod("GeoPoints", "factorDistance02", null, false, doubleType, 
+                  new Type[] { doubleType, doubleType }, null, null);
+        addMethod("GeoPoints", "factorDistance13", null, false, doubleType, 
+                  new Type[] { doubleType, doubleType }, null, null);
+        addMethod("GeoPoints", "arcDistance", null, false, doubleType, 
+                  new Type[] { doubleType, doubleType }, null, null);
+        addMethod("GeoPoints", "arcDistanceWithDefault", null, false, doubleType, 
+                  new Type[] { doubleType, doubleType, doubleType }, null, null);
+        addMethod("GeoPoints", "arcDistanceInKm", null, false, doubleType, 
+                  new Type[] { doubleType, doubleType }, null, null);
+        addMethod("GeoPoints", "arcDistanceInKmWithDefault", null, false, doubleType, 
+                  new Type[] { doubleType, doubleType, doubleType }, null, null);
+        addMethod("GeoPoints", "arcDistanceInMiles", null, false, doubleType, 
+                  new Type[] { doubleType, doubleType }, null, null);
+        addMethod("GeoPoints", "arcDistanceInMilesWithDefault", null, false, doubleType, 
+                  new Type[] { doubleType, doubleType, doubleType }, null, null);
+        addMethod("GeoPoints", "distance", null, false, doubleType, 
+                  new Type[] { doubleType, doubleType }, null, null);
+        addMethod("GeoPoints", "distanceWithDefault", null, false, doubleType, 
+                  new Type[] { doubleType, doubleType, doubleType }, null, null);
+        addMethod("GeoPoints", "distanceInKm", null, false, doubleType, 
+                  new Type[] { doubleType, doubleType }, null, null);
+        addMethod("GeoPoints", "distanceInKmWithDefault", null, false, doubleType, 
+                  new Type[] { doubleType, doubleType, doubleType }, null, null);
+        addMethod("GeoPoints", "distanceInMiles", null, false, doubleType, 
+                  new Type[] { doubleType, doubleType }, null, null);
+        addMethod("GeoPoints", "distanceInMilesWithDefault", null, false, doubleType, 
+                  new Type[] { doubleType, doubleType, doubleType }, null, null);
+        addMethod("GeoPoints", "geohashDistance", null, false, doubleType, 
+                  new Type[] { stringType }, null, null);
+        addMethod("GeoPoints", "geohashDistanceInKm", null, false, doubleType, 
+                  new Type[] { stringType }, null, null);
+        addMethod("GeoPoints", "geohashDistanceInMiles", null, false, doubleType, 
+                  new Type[] { stringType }, null, null);
     }
 
     private void copyDefaultStructs() {
@@ -1079,6 +1169,12 @@ class Definition {
         copyStruct("IllegalArgumentException", "Exception", "Object");
         copyStruct("IllegalStateException", "Exception", "Object");
         copyStruct("NumberFormatException", "Exception", "Object");
+        
+        copyStruct("GeoPoint", "Object");
+        copyStruct("Strings", "List<String>", "Collection<String>", "Object");
+        copyStruct("Longs", "List", "Collection", "Object");
+        copyStruct("Doubles", "List", "Collection", "Object");
+        copyStruct("GeoPoints", "List", "Collection", "Object");
     }
 
     private void addDefaultTransforms() {
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/20_scriptfield.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/20_scriptfield.yaml
index d53306b5d47..a1087f17d4e 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/20_scriptfield.yaml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/20_scriptfield.yaml
@@ -28,7 +28,7 @@ setup:
                 script_fields:
                     bar:
                         script: 
-                            inline: "input.doc['foo'].0 + input.x;"
+                            inline: "input.doc['foo'].value + input.x;"
                             lang: painless
                             params:
                                 x: "bbb"
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml
index 2dd1a6004ff..4a1ec86a267 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml
@@ -29,12 +29,12 @@
                 query:
                     script:
                         script:
-                            inline: "input.doc['num1'].0 > 1;"
+                            inline: "input.doc['num1'].value > 1;"
                             lang: painless
                 script_fields:
                     sNum1:
                         script: 
-                            inline: "input.doc['num1'].0;"
+                            inline: "input.doc['num1'].value;"
                             lang: painless
                 sort:
                     num1:
@@ -51,7 +51,7 @@
                 query:
                     script:
                         script:
-                            inline: "input.doc['num1'].0 > input.param1;"
+                            inline: "input.doc['num1'].value > input.param1;"
                             lang: painless
                             params:
                                 param1: 1
@@ -59,7 +59,7 @@
                 script_fields:
                     sNum1:
                         script:
-                            inline: "return input.doc['num1'].0;"
+                            inline: "return input.doc['num1'].value;"
                             lang: painless
                 sort:
                     num1:
@@ -76,7 +76,7 @@
                 query:
                     script:
                         script:
-                            inline: "input.doc['num1'].0 > input.param1;"
+                            inline: "input.doc['num1'].value > input.param1;"
                             lang: painless
                             params:
                                 param1: -1
@@ -84,7 +84,7 @@
                 script_fields:
                     sNum1:
                         script: 
-                            inline: "input.doc['num1'].0;"
+                            inline: "input.doc['num1'].value;"
                             lang: painless
                 sort:
                     num1:

From f3b2ab822d3812c6853ca031f128325711d4d71d Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 5 May 2016 19:03:23 -0400
Subject: [PATCH 0089/1311] Another wait_for_yellow to the docs

All in service of the snippets passing consistently.
---
 docs/reference/mapping/params/analyzer.asciidoc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/reference/mapping/params/analyzer.asciidoc b/docs/reference/mapping/params/analyzer.asciidoc
index 5f564f9a668..89d83f2b369 100644
--- a/docs/reference/mapping/params/analyzer.asciidoc
+++ b/docs/reference/mapping/params/analyzer.asciidoc
@@ -60,6 +60,8 @@ PUT my_index
   }
 }
 
+GET _cluster/health?wait_for_status=yellow
+
 GET my_index/_analyze?field=text <3>
 {
   "text": "The quick Brown Foxes."

From 2d2b232a7d7ddafc830b4e7dbf3c0398da4a50eb Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Thu, 5 May 2016 16:08:51 -0700
Subject: [PATCH 0090/1311] Tests: Delay starting progress loggers for vagrant
 until test is running

This was broken recently as part of making the vagrant tasks extend
LoggedExec. This change fixes the progress logger to not be started
until we start seeing output from vagrant.
---
 .../vagrant/TapLoggerOutputStream.groovy      | 136 +++++++++---------
 .../vagrant/VagrantLoggerOutputStream.groovy  |  47 ++----
 2 files changed, 84 insertions(+), 99 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.groovy
index 6b87ceeaf96..3f980c57a49 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.groovy
@@ -19,9 +19,11 @@
 package org.elasticsearch.gradle.vagrant
 
 import com.carrotsearch.gradle.junit4.LoggingOutputStream
+import groovy.transform.PackageScope
 import org.gradle.api.GradleScriptException
 import org.gradle.api.logging.Logger
 import org.gradle.logging.ProgressLogger
+import org.gradle.logging.ProgressLoggerFactory
 
 import java.util.regex.Matcher
 
@@ -35,73 +37,77 @@ import java.util.regex.Matcher
  * There is a Tap4j project but we can't use it because it wants to parse the
  * entire TAP stream at once and won't parse it stream-wise.
  */
-class TapLoggerOutputStream extends LoggingOutputStream {
-  ProgressLogger progressLogger
-  Logger logger
-  int testsCompleted = 0
-  int testsFailed = 0
-  int testsSkipped = 0
-  Integer testCount
-  String countsFormat
+public class TapLoggerOutputStream extends LoggingOutputStream {
+    private final ProgressLogger progressLogger
+    private boolean isStarted = false
+    private final Logger logger
+    private int testsCompleted = 0
+    private int testsFailed = 0
+    private int testsSkipped = 0
+    private Integer testCount
+    private String countsFormat
 
-  TapLoggerOutputStream(Map args) {
-    logger = args.logger
-    progressLogger = args.factory.newOperation(VagrantLoggerOutputStream)
-    progressLogger.setDescription("TAP output for `$args.command`")
-    progressLogger.started()
-    progressLogger.progress("Starting `$args.command`...")
-  }
-
-  void flush() {
-    if (end == start) return
-    line(new String(buffer, start, end - start))
-    start = end
-  }
-
-  void line(String line) {
-    // System.out.print "===> $line\n"
-    if (testCount == null) {
-      try {
-        testCount = line.split('\\.').last().toInteger()
-        def length = (testCount as String).length()
-        countsFormat = "%0${length}d"
-        countsFormat = "[$countsFormat|$countsFormat|$countsFormat/$countsFormat]"
-        return
-      } catch (Exception e) {
-        throw new GradleScriptException(
-          'Error parsing first line of TAP stream!!', e)
-      }
-    }
-    Matcher m = line =~ /(?<status>ok|not ok) \d+(?<skip> # skip (?<skipReason>\(.+\))?)? \[(?<suite>.+)\] (?<test>.+)/
-    if (!m.matches()) {
-      /* These might be failure report lines or comments or whatever. Its hard
-        to tell and it doesn't matter. */
-      logger.warn(line)
-      return
-    }
-    boolean skipped = m.group('skip') != null
-    boolean success = !skipped && m.group('status') == 'ok'
-    String skipReason = m.group('skipReason')
-    String suiteName = m.group('suite')
-    String testName = m.group('test')
-
-    String status
-    if (skipped) {
-      status = "SKIPPED"
-      testsSkipped++
-    } else if (success) {
-      status = "     OK"
-      testsCompleted++
-    } else {
-      status = " FAILED"
-      testsFailed++
+    TapLoggerOutputStream(Map args) {
+        logger = args.logger
+        progressLogger = args.factory.newOperation(VagrantLoggerOutputStream)
+        progressLogger.setDescription("TAP output for `${args.command}`")
     }
 
-    String counts = sprintf(countsFormat,
-      [testsCompleted, testsFailed, testsSkipped, testCount])
-    progressLogger.progress("Tests $counts, $status [$suiteName] $testName")
-    if (!success) {
-      logger.warn(line)
+    @Override
+    public void flush() {
+        if (isStarted == false) {
+            progressLogger.started()
+            isStarted = true
+        }
+        if (end == start) return
+        line(new String(buffer, start, end - start))
+        start = end
+    }
+
+    void line(String line) {
+        // System.out.print "===> $line\n"
+        if (testCount == null) {
+            try {
+                testCount = line.split('\\.').last().toInteger()
+                def length = (testCount as String).length()
+                countsFormat = "%0${length}d"
+                countsFormat = "[$countsFormat|$countsFormat|$countsFormat/$countsFormat]"
+                return
+            } catch (Exception e) {
+                throw new GradleScriptException(
+                        'Error parsing first line of TAP stream!!', e)
+            }
+        }
+        Matcher m = line =~ /(?<status>ok|not ok) \d+(?<skip> # skip (?<skipReason>\(.+\))?)? \[(?<suite>.+)\] (?<test>.+)/
+        if (!m.matches()) {
+            /* These might be failure report lines or comments or whatever. Its hard
+              to tell and it doesn't matter. */
+            logger.warn(line)
+            return
+        }
+        boolean skipped = m.group('skip') != null
+        boolean success = !skipped && m.group('status') == 'ok'
+        String skipReason = m.group('skipReason')
+        String suiteName = m.group('suite')
+        String testName = m.group('test')
+
+        String status
+        if (skipped) {
+            status = "SKIPPED"
+            testsSkipped++
+        } else if (success) {
+            status = "     OK"
+            testsCompleted++
+        } else {
+            status = " FAILED"
+            testsFailed++
+        }
+
+        String counts = sprintf(countsFormat,
+                [testsCompleted, testsFailed, testsSkipped, testCount])
+        progressLogger.progress("Tests $counts, $status [$suiteName] $testName")
+        if (!success) {
+            logger.warn(line)
+        }
     }
-  }
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.groovy
index 22008fa1a0e..331a638b5ca 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.groovy
@@ -19,6 +19,7 @@
 package org.elasticsearch.gradle.vagrant
 
 import com.carrotsearch.gradle.junit4.LoggingOutputStream
+import org.gradle.api.logging.Logger
 import org.gradle.logging.ProgressLogger
 import org.gradle.logging.ProgressLoggerFactory
 
@@ -46,31 +47,31 @@ import org.gradle.logging.ProgressLoggerFactory
 public class VagrantLoggerOutputStream extends LoggingOutputStream {
     private static final String HEADING_PREFIX = '==> '
 
-    ProgressLoggerFactory progressLoggerFactory
-
-
-    private ProgressLogger progressLogger
-    String squashedPrefix
-    String lastLine = ''
-    boolean inProgressReport = false
-    String heading = ''
+    private final ProgressLogger progressLogger
+    private boolean isStarted = false
+    private String squashedPrefix
+    private String lastLine = ''
+    private boolean inProgressReport = false
+    private String heading = ''
 
     VagrantLoggerOutputStream(Map args) {
         progressLogger = args.factory.newOperation(VagrantLoggerOutputStream)
         progressLogger.setDescription("Vagrant output for `$args.command`")
-        progressLogger.started()
-        progressLogger.progress("Starting `$args.command`...")
         squashedPrefix = args.squashedPrefix
     }
 
-    void flush() {
+    @Override
+    public void flush() {
+        if (isStarted == false) {
+            progressLogger.started()
+            isStarted = true
+        }
         if (end == start) return
         line(new String(buffer, start, end - start))
         start = end
     }
 
     void line(String line) {
-        // debugPrintLine(line) // Uncomment me to log every incoming line
         if (line.startsWith('\r\u001b')) {
             /* We don't want to try to be a full terminal emulator but we want to
               keep the escape sequences from leaking and catch _some_ of the
@@ -97,28 +98,6 @@ public class VagrantLoggerOutputStream extends LoggingOutputStream {
         } else {
             return
         }
-        // debugLogLine(line) // Uncomment me to log every line we add to the logger
         progressLogger.progress(line)
     }
-
-    void debugPrintLine(line) {
-        System.out.print '----------> '
-        for (int i = start; i < end; i++) {
-            switch (buffer[i] as char) {
-                case ' '..'~':
-                    System.out.print buffer[i] as char
-                    break
-                default:
-                    System.out.print '%'
-                    System.out.print Integer.toHexString(buffer[i])
-            }
-        }
-        System.out.print '\n'
-    }
-
-    void debugLogLine(line) {
-        System.out.print '>>>>>>>>>>> '
-        System.out.print line
-        System.out.print '\n'
-    }
 }

From f1fb6a37c0e348ef0898782d58acf989f2b5c9be Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 5 May 2016 19:15:01 -0400
Subject: [PATCH 0091/1311] Fix error message on ES_MAX_MEM set

This commit fixes a typo in the error message when the now unsupported
environment variable ES_MAX_MEM is set on Windows.
---
 distribution/src/main/resources/bin/elasticsearch.bat | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distribution/src/main/resources/bin/elasticsearch.bat b/distribution/src/main/resources/bin/elasticsearch.bat
index 3a9d561e2e3..9f1d871d0a2 100644
--- a/distribution/src/main/resources/bin/elasticsearch.bat
+++ b/distribution/src/main/resources/bin/elasticsearch.bat
@@ -19,7 +19,7 @@ if %bad_env_var% == 1 (
     echo Error: encountered environment variables that are no longer supported
     echo Use jvm.options or ES_JAVA_OPTS to configure the JVM
     if not "%ES_MIN_MEM%" == "" echo ES_MIN_MEM=%ES_MIN_MEM%: set -Xms%ES_MIN_MEM% in jvm.options or add "-Xms%ES_MIN_MEM%" to ES_JAVA_OPTS
-    if not "%ES_MAX_MEM%" == "" echo ES_MAX_MEM=%ES_MAX_MEM%: set -Xms%ES_MAX_MEM% in jvm.options or add "-Xmx%ES_MAX_MEM%" to ES_JAVA_OPTS
+    if not "%ES_MAX_MEM%" == "" echo ES_MAX_MEM=%ES_MAX_MEM%: set -Xmx%ES_MAX_MEM% in jvm.options or add "-Xmx%ES_MAX_MEM%" to ES_JAVA_OPTS
     if not "%ES_HEAP_SIZE%" == "" echo ES_HEAP_SIZE=%ES_HEAP_SIZE%: set -Xms%ES_HEAP_SIZE% and -Xmx%ES_HEAP_SIZE% in jvm.options or add "-Xms%ES_HEAP_SIZE% -Xmx%ES_HEAP_SIZE%" to ES_JAVA_OPTS
     if not "%ES_HEAP_NEWSIZE%" == "" echo ES_HEAP_NEWSIZE=%ES_HEAP_NEWSIZE%: set -Xmn%ES_HEAP_NEWSIZE% in jvm.options or add "-Xmn%ES_HEAP_SIZE%" to ES_JAVA_OPTS
     if not "%ES_DIRECT_SIZE%" == "" echo ES_DIRECT_SIZE=%ES_DIRECT_SIZE%: set -XX:MaxDirectMemorySize=%ES_DIRECT_SIZE% in jvm.options or add "-XX:MaxDirectMemorySize=%ES_DIRECT_SIZE%" to ES_JAVA_OPTS

From 443976d03c8dc9a9527b25f62e735922e842848b Mon Sep 17 00:00:00 2001
From: Radovan Ondas <ondas.radovan@gmail.com>
Date: Fri, 6 May 2016 01:17:31 +0200
Subject: [PATCH 0092/1311] Fix typo in message for variable setup ES_MAX_MEM

Small typo fix in startup script.

Relates #18168
---
 distribution/src/main/resources/bin/elasticsearch | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distribution/src/main/resources/bin/elasticsearch b/distribution/src/main/resources/bin/elasticsearch
index 2f62223cfe9..0101a108d4b 100755
--- a/distribution/src/main/resources/bin/elasticsearch
+++ b/distribution/src/main/resources/bin/elasticsearch
@@ -72,7 +72,7 @@ if test -n "$ES_MIN_MEM" ||
     echo "Error: encountered environment variables that are no longer supported"
     echo "Use jvm.options or ES_JAVA_OPTS to configure the JVM"
     unsupported_environment_variable "$ES_MIN_MEM" ES_MIN_MEM "set -Xms$ES_MIN_MEM in jvm.options or add \"-Xms$ES_MIN_MEM\" to ES_JAVA_OPTS"
-    unsupported_environment_variable "$ES_MAX_MEM" ES_MAX_MEM "set -Xms$ES_MAX_MEM in jvm.options or add \"-Xms$ES_MAX_MEM\" to ES_JAVA_OPTS"
+    unsupported_environment_variable "$ES_MAX_MEM" ES_MAX_MEM "set -Xmx$ES_MAX_MEM in jvm.options or add \"-Xmx$ES_MAX_MEM\" to ES_JAVA_OPTS"
     unsupported_environment_variable "$ES_HEAP_SIZE" ES_HEAP_SIZE "set -Xms$ES_HEAP_SIZE and -Xmx$ES_HEAP_SIZE in jvm.options or add \"-Xms$ES_HEAP_SIZE -Xmx$ES_HEAP_SIZE\" to ES_JAVA_OPTS"
     unsupported_environment_variable "$ES_HEAP_NEWSIZE" ES_HEAP_NEWSIZE "set -Xmn$ES_HEAP_NEWSIZE in jvm.options or add \"-Xmn$ES_HEAP_SIZE\" to ES_JAVA_OPTS"
     unsupported_environment_variable "$ES_DIRECT_SIZE" ES_DIRECT_SIZE "set -XX:MaxDirectMemorySize=$ES_DIRECT_SIZE in jvm.options or add \"-XX:MaxDirectMemorySize=$ES_DIRECT_SIZE\" to ES_JAVA_OPTS"

From 1199cd8e2a1343ba74f208c96d6798ab0da4bdfd Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 5 May 2016 20:16:57 -0400
Subject: [PATCH 0093/1311] Mark IHBT#testFromAndToXContent as awaits fix

This commit marks InnerHitsBuilderTests#testFromAndToXContent as
awaiting a fix.
---
 .../java/org/elasticsearch/index/query/InnerHitBuilderTests.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
index d2cbec890e8..4bc95a1c3ce 100644
--- a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
@@ -87,6 +87,7 @@ public class InnerHitBuilderTests extends ESTestCase {
         }
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/18166")
     public void testFromAndToXContent() throws Exception {
         for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
             InnerHitBuilder innerHit = randomInnerHits(true, false);

From e16af604bf74442625d84b0b09b9cd8da67c336d Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Thu, 5 May 2016 17:53:01 -0700
Subject: [PATCH 0094/1311] Build: Add pom generation to assemble task

In preparation for a unified release process, we need to be able to
generate the pom files independently of trying to actually publish. This
change adds back the maven-publish plugin just for that purpose. The
nexus plugin still exists for now, so that we do not break snapshots,
but that can be removed at a later time once snapshots are happenign
through the unified tools. Note I also changed the dir jars are written
into so that all our artifacts are under build/distributions.
---
 build.gradle                                  |  20 +++
 buildSrc/build.gradle                         |   1 +
 .../elasticsearch/gradle/BuildPlugin.groovy   | 116 +++++++++++-------
 .../gradle/plugin/PluginBuildPlugin.groovy    |  32 ++++-
 core/build.gradle                             |  12 ++
 distribution/build.gradle                     |  13 ++
 distribution/integ-test-zip/build.gradle      |   8 ++
 distribution/tar/build.gradle                 |   1 +
 distribution/zip/build.gradle                 |   8 ++
 test/build.gradle                             |   2 +
 10 files changed, 171 insertions(+), 42 deletions(-)

diff --git a/build.gradle b/build.gradle
index ad1f2456dea..459c8c8ae2e 100644
--- a/build.gradle
+++ b/build.gradle
@@ -28,6 +28,26 @@ subprojects {
   group = 'org.elasticsearch'
   version = org.elasticsearch.gradle.VersionProperties.elasticsearch
 
+  // we only use maven publish to add tasks for pom generation
+  plugins.withType(MavenPublishPlugin).whenPluginAdded {
+    publishing {
+      publications {
+        // add license information to generated poms
+        all {
+          pom.withXml { XmlProvider xml ->
+            Node node = xml.asNode()
+            node.appendNode('inceptionYear', '2009')
+
+            Node license = node.appendNode('licenses').appendNode('license')
+            license.appendNode('name', 'The Apache Software License, Version 2.0')
+            license.appendNode('url', 'http://www.apache.org/licenses/LICENSE-2.0.txt')
+            license.appendNode('distribution', 'repo')
+          }
+        }
+      }
+    }
+  }
+
   plugins.withType(NexusPlugin).whenPluginAdded {
     modifyPom {
       project {
diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
index e36451311e7..1b0d3c52127 100644
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -69,6 +69,7 @@ dependencies {
     transitive = false
   }
   compile 'com.netflix.nebula:gradle-extra-configurations-plugin:3.0.3'
+  compile 'com.netflix.nebula:nebula-publishing-plugin:4.4.4'
   compile 'com.netflix.nebula:gradle-info-plugin:3.0.3'
   compile 'org.eclipse.jgit:org.eclipse.jgit:3.2.0.201312181205-r'
   compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index ab2ba5abfef..de5dbb73af6 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -33,6 +33,9 @@ import org.gradle.api.artifacts.ProjectDependency
 import org.gradle.api.artifacts.ResolvedArtifact
 import org.gradle.api.artifacts.dsl.RepositoryHandler
 import org.gradle.api.artifacts.maven.MavenPom
+import org.gradle.api.publish.maven.MavenPublication
+import org.gradle.api.publish.maven.plugins.MavenPublishPlugin
+import org.gradle.api.publish.maven.tasks.GenerateMavenPom
 import org.gradle.api.tasks.bundling.Jar
 import org.gradle.api.tasks.compile.JavaCompile
 import org.gradle.internal.jvm.Jvm
@@ -54,7 +57,7 @@ class BuildPlugin implements Plugin<Project> {
         project.pluginManager.apply('java')
         project.pluginManager.apply('carrotsearch.randomized-testing')
         // these plugins add lots of info to our jars
-        configureJarManifest(project) // jar config must be added before info broker
+        configureJars(project) // jar config must be added before info broker
         project.pluginManager.apply('nebula.info-broker')
         project.pluginManager.apply('nebula.info-basic')
         project.pluginManager.apply('nebula.info-java')
@@ -68,6 +71,7 @@ class BuildPlugin implements Plugin<Project> {
         configureConfigurations(project)
         project.ext.versions = VersionProperties.versions
         configureCompile(project)
+        configurePomGeneration(project)
 
         configureTest(project)
         configurePrecommit(project)
@@ -266,44 +270,7 @@ class BuildPlugin implements Plugin<Project> {
 
         // add exclusions to the pom directly, for each of the transitive deps of this project's deps
         project.modifyPom { MavenPom pom ->
-            pom.withXml { XmlProvider xml ->
-                // first find if we have dependencies at all, and grab the node
-                NodeList depsNodes = xml.asNode().get('dependencies')
-                if (depsNodes.isEmpty()) {
-                    return
-                }
-
-                // check each dependency for any transitive deps
-                for (Node depNode : depsNodes.get(0).children()) {
-                    String groupId = depNode.get('groupId').get(0).text()
-                    String artifactId = depNode.get('artifactId').get(0).text()
-                    String version = depNode.get('version').get(0).text()
-
-                    // collect the transitive deps now that we know what this dependency is
-                    String depConfig = transitiveDepConfigName(groupId, artifactId, version)
-                    Configuration configuration = project.configurations.findByName(depConfig)
-                    if (configuration == null) {
-                        continue // we did not make this dep non-transitive
-                    }
-                    Set<ResolvedArtifact> artifacts = configuration.resolvedConfiguration.resolvedArtifacts
-                    if (artifacts.size() <= 1) {
-                        // this dep has no transitive deps (or the only artifact is itself)
-                        continue
-                    }
-
-                    // we now know we have something to exclude, so add the exclusion elements
-                    Node exclusions = depNode.appendNode('exclusions')
-                    for (ResolvedArtifact transitiveArtifact : artifacts) {
-                        ModuleVersionIdentifier transitiveDep = transitiveArtifact.moduleVersion.id
-                        if (transitiveDep.group == groupId && transitiveDep.name == artifactId) {
-                            continue; // don't exclude the dependency itself!
-                        }
-                        Node exclusion = exclusions.appendNode('exclusion')
-                        exclusion.appendNode('groupId', transitiveDep.group)
-                        exclusion.appendNode('artifactId', transitiveDep.name)
-                    }
-                }
-            }
+            pom.withXml(removeTransitiveDependencies(project))
         }
     }
 
@@ -332,6 +299,70 @@ class BuildPlugin implements Plugin<Project> {
         }
     }
 
+    /** Returns a closure which can be used with a MavenPom for removing transitive dependencies. */
+    private static Closure removeTransitiveDependencies(Project project) {
+        // TODO: remove this when enforcing gradle 2.13+, it now properly handles exclusions
+        return { XmlProvider xml ->
+            // first find if we have dependencies at all, and grab the node
+            NodeList depsNodes = xml.asNode().get('dependencies')
+            if (depsNodes.isEmpty()) {
+                return
+            }
+
+            // check each dependency for any transitive deps
+            for (Node depNode : depsNodes.get(0).children()) {
+                String groupId = depNode.get('groupId').get(0).text()
+                String artifactId = depNode.get('artifactId').get(0).text()
+                String version = depNode.get('version').get(0).text()
+
+                // collect the transitive deps now that we know what this dependency is
+                String depConfig = transitiveDepConfigName(groupId, artifactId, version)
+                Configuration configuration = project.configurations.findByName(depConfig)
+                if (configuration == null) {
+                    continue // we did not make this dep non-transitive
+                }
+                Set<ResolvedArtifact> artifacts = configuration.resolvedConfiguration.resolvedArtifacts
+                if (artifacts.size() <= 1) {
+                    // this dep has no transitive deps (or the only artifact is itself)
+                    continue
+                }
+
+                // we now know we have something to exclude, so add the exclusion elements
+                Node exclusions = depNode.appendNode('exclusions')
+                for (ResolvedArtifact transitiveArtifact : artifacts) {
+                    ModuleVersionIdentifier transitiveDep = transitiveArtifact.moduleVersion.id
+                    if (transitiveDep.group == groupId && transitiveDep.name == artifactId) {
+                        continue; // don't exclude the dependency itself!
+                    }
+                    Node exclusion = exclusions.appendNode('exclusion')
+                    exclusion.appendNode('groupId', transitiveDep.group)
+                    exclusion.appendNode('artifactId', transitiveDep.name)
+                }
+            }
+        }
+    }
+
+    /**Configuration generation of maven poms. */
+    private static void configurePomGeneration(Project project) {
+        project.plugins.withType(MavenPublishPlugin.class).whenPluginAdded {
+            project.publishing {
+                publications {
+                    all { MavenPublication publication -> // we only deal with maven
+                        // add exclusions to the pom directly, for each of the transitive deps of this project's deps
+                        publication.pom.withXml(removeTransitiveDependencies(project))
+                    }
+                }
+            }
+
+            project.tasks.withType(GenerateMavenPom.class) { GenerateMavenPom t ->
+                // place the pom next to the jar it is for
+                t.destination = new File(project.buildDir, "distributions/${project.archivesBaseName}-${project.version}.pom")
+                // build poms with assemble
+                project.assemble.dependsOn(t)
+            }
+        }
+    }
+
     /** Adds compiler settings to the project */
     static void configureCompile(Project project) {
         project.ext.compactProfile = 'compact3'
@@ -364,9 +395,12 @@ class BuildPlugin implements Plugin<Project> {
         }
     }
 
-    /** Adds additional manifest info to jars */
-    static void configureJarManifest(Project project) {
+    /** Adds additional manifest info to jars, and adds source and javadoc jars */
+    static void configureJars(Project project) {
         project.tasks.withType(Jar) { Jar jarTask ->
+            // we put all our distributable files under distributions
+            jarTask.destinationDir = new File(project.buildDir, 'distributions')
+            // fixup the jar manifest
             jarTask.doFirst {
                 boolean isSnapshot = VersionProperties.elasticsearch.endsWith("-SNAPSHOT");
                 String version = VersionProperties.elasticsearch;
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
index b04f959e068..36770ab35f7 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
@@ -18,11 +18,12 @@
  */
 package org.elasticsearch.gradle.plugin
 
+import nebula.plugin.publishing.maven.MavenManifestPlugin
+import nebula.plugin.publishing.maven.MavenScmPlugin
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.test.RestIntegTestTask
 import org.elasticsearch.gradle.test.RunTask
 import org.gradle.api.Project
-import org.gradle.api.artifacts.Dependency
 import org.gradle.api.tasks.SourceSet
 import org.gradle.api.tasks.bundling.Zip
 
@@ -50,6 +51,7 @@ public class PluginBuildPlugin extends BuildPlugin {
             } else {
                 project.integTest.clusterConfig.plugin(name, project.bundlePlugin.outputs.files)
                 project.tasks.run.clusterConfig.plugin(name, project.bundlePlugin.outputs.files)
+                configurePomGeneration(project)
             }
 
             project.namingConventions {
@@ -125,4 +127,32 @@ public class PluginBuildPlugin extends BuildPlugin {
         project.configurations.getByName('default').extendsFrom = []
         project.artifacts.add('default', bundle)
     }
+
+    /**
+     * Adds the plugin jar and zip as publications.
+     */
+    private static void configurePomGeneration(Project project) {
+        project.plugins.apply(MavenScmPlugin.class)
+        project.plugins.apply(MavenManifestPlugin.class)
+
+        project.publishing {
+            publications {
+                nebula {
+                    artifact project.bundlePlugin
+                    pom.withXml {
+                        // overwrite the name/description in the pom nebula set up
+                        Node root = asNode()
+                        for (Node node : root.children()) {
+                            if (node.name() == 'name') {
+                                node.setValue(project.pluginProperties.extension.name)
+                            } else if (node.name() == 'description') {
+                                node.setValue(project.pluginProperties.extension.description)
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+    }
 }
diff --git a/core/build.gradle b/core/build.gradle
index ab3754e72ff..e12a80fcbf7 100644
--- a/core/build.gradle
+++ b/core/build.gradle
@@ -24,6 +24,18 @@ import org.elasticsearch.gradle.BuildPlugin
 apply plugin: 'elasticsearch.build'
 apply plugin: 'com.bmuschko.nexus'
 apply plugin: 'nebula.optional-base'
+apply plugin: 'nebula.maven-base-publish'
+apply plugin: 'nebula.maven-scm'
+//apply plugin: 'nebula.source-jar'
+//apply plugin: 'nebula.javadoc-jar'
+
+publishing {
+  publications {
+    nebula {
+      artifactId 'elasticsearch'
+    }
+  }
+}
 
 archivesBaseName = 'elasticsearch'
 
diff --git a/distribution/build.gradle b/distribution/build.gradle
index 09050db2159..bb4cc167f10 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -158,6 +158,19 @@ subprojects {
       MavenFilteringHack.filter(it, expansions)
     }
   }
+
+   /*****************************************************************************
+   *                           Publishing setup                                *
+   *****************************************************************************/
+  apply plugin: 'nebula.maven-base-publish'
+  apply plugin: 'nebula.maven-scm'
+  publishing {
+    publications {
+      nebula {
+        artifactId 'elasticsearch'
+      }
+    }
+  }
 }
 
 /*****************************************************************************
diff --git a/distribution/integ-test-zip/build.gradle b/distribution/integ-test-zip/build.gradle
index 23191ff03a4..67f99aa884a 100644
--- a/distribution/integ-test-zip/build.gradle
+++ b/distribution/integ-test-zip/build.gradle
@@ -27,5 +27,13 @@ artifacts {
   archives buildZip
 }
 
+publishing {
+  publications {
+    nebula {
+      artifact buildZip
+    }
+  }
+}
+
 integTest.dependsOn buildZip
 
diff --git a/distribution/tar/build.gradle b/distribution/tar/build.gradle
index 9edba6c11a2..72425659056 100644
--- a/distribution/tar/build.gradle
+++ b/distribution/tar/build.gradle
@@ -33,3 +33,4 @@ artifacts {
     project.signArchives.singleSignature.type = 'tar.gz.asc'
   }
 }
+
diff --git a/distribution/zip/build.gradle b/distribution/zip/build.gradle
index 23191ff03a4..67f99aa884a 100644
--- a/distribution/zip/build.gradle
+++ b/distribution/zip/build.gradle
@@ -27,5 +27,13 @@ artifacts {
   archives buildZip
 }
 
+publishing {
+  publications {
+    nebula {
+      artifact buildZip
+    }
+  }
+}
+
 integTest.dependsOn buildZip
 
diff --git a/test/build.gradle b/test/build.gradle
index 7feb332b717..a80ca59978c 100644
--- a/test/build.gradle
+++ b/test/build.gradle
@@ -25,6 +25,8 @@ subprojects {
 
   group = 'org.elasticsearch.test'
   apply plugin: 'elasticsearch.build'
+  apply plugin: 'nebula.maven-base-publish'
+  apply plugin: 'nebula.maven-scm'
 
   
   // the main files are actually test files, so use the appropriate forbidden api sigs

From 7d8708716e0f92a4ea5073b76a308cd226266d36 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Wed, 4 May 2016 15:18:13 +0200
Subject: [PATCH 0095/1311] QueryBuilder does not need generics. #18133

QueryBuilder has generics, but those are never used: all call sites use
`QueryBuilder<?>`. Only `AbstractQueryBuilder` needs generics so that the base
class can contain a default implementation for setters that returns `this`.
---
 .../query/ShardValidateQueryRequest.java      |  4 +-
 .../validate/query/ValidateQueryRequest.java  |  6 +-
 .../action/explain/ExplainRequest.java        |  6 +-
 .../percolate/PercolateSourceBuilder.java     |  4 +-
 .../percolate/TransportPercolateAction.java   |  2 +-
 .../action/search/SearchRequestBuilder.java   |  4 +-
 .../cluster/metadata/AliasValidator.java      |  2 +-
 .../percolator/PercolatorFieldMapper.java     |  4 +-
 .../index/query/AbstractQueryBuilder.java     | 18 +++---
 .../index/query/BoolQueryBuilder.java         | 64 +++++++++----------
 .../index/query/BoostingQueryBuilder.java     |  8 +--
 .../query/ConstantScoreQueryBuilder.java      | 12 ++--
 .../index/query/DisMaxQueryBuilder.java       | 16 ++---
 .../query/FieldMaskingSpanQueryBuilder.java   |  8 +--
 .../index/query/FuzzyQueryBuilder.java        |  2 +-
 .../index/query/GeoShapeQueryBuilder.java     |  2 +-
 .../index/query/HasChildQueryBuilder.java     | 14 ++--
 .../index/query/HasParentQueryBuilder.java    | 14 ++--
 .../index/query/IndicesQueryBuilder.java      | 26 ++++----
 .../index/query/InnerHitBuilder.java          |  6 +-
 .../index/query/MoreLikeThisQueryBuilder.java |  3 +-
 .../index/query/MultiTermQueryBuilder.java    |  2 +-
 .../index/query/NestedQueryBuilder.java       |  6 +-
 .../index/query/PercolateQueryBuilder.java    |  2 +-
 .../index/query/PrefixQueryBuilder.java       |  2 +-
 .../index/query/QueryBuilder.java             | 16 +++--
 .../index/query/QueryParseContext.java        |  8 +--
 .../index/query/QueryParser.java              |  2 +-
 .../index/query/RangeQueryBuilder.java        |  4 +-
 .../index/query/RegexpQueryBuilder.java       |  2 +-
 .../query/SpanContainingQueryBuilder.java     | 24 +++----
 .../index/query/SpanFirstQueryBuilder.java    | 10 +--
 .../query/SpanMultiTermQueryBuilder.java      | 10 +--
 .../index/query/SpanNearQueryBuilder.java     | 16 ++---
 .../index/query/SpanNotQueryBuilder.java      | 12 ++--
 .../index/query/SpanOrQueryBuilder.java       | 16 ++---
 .../index/query/SpanQueryBuilder.java         |  2 +-
 .../index/query/SpanTermQueryBuilder.java     |  2 +-
 .../index/query/SpanWithinQueryBuilder.java   | 12 ++--
 .../index/query/TemplateQueryBuilder.java     |  4 +-
 .../index/query/TermsQueryBuilder.java        |  2 +-
 .../index/query/WildcardQueryBuilder.java     |  2 +-
 .../index/query/WrapperQueryBuilder.java      |  4 +-
 .../FunctionScoreQueryBuilder.java            | 26 ++++----
 .../query/RestValidateQueryAction.java        |  2 +-
 .../rest/action/cat/RestCountAction.java      |  2 +-
 .../rest/action/count/RestCountAction.java    |  2 +-
 .../action/explain/RestExplainAction.java     |  2 +-
 .../rest/action/search/RestSearchAction.java  |  2 +-
 .../rest/action/support/RestActions.java      |  4 +-
 .../elasticsearch/search/SearchModule.java    |  2 +-
 .../aggregations/AggregationBuilders.java     |  4 +-
 .../filter/FilterAggregatorBuilder.java       |  6 +-
 .../filter/FilterAggregatorFactory.java       |  2 +-
 .../bucket/filters/FiltersAggregator.java     |  6 +-
 .../filters/FiltersAggregatorBuilder.java     | 10 +--
 .../SignificantTermsAggregatorBuilder.java    |  6 +-
 .../SignificantTermsAggregatorFactory.java    |  4 +-
 .../significant/SignificantTermsParser.java   |  4 +-
 .../search/builder/SearchSourceBuilder.java   | 18 +++---
 .../highlight/AbstractHighlighterBuilder.java |  6 +-
 .../search/rescore/QueryRescorerBuilder.java  | 10 +--
 .../search/rescore/RescoreBuilder.java        |  2 +-
 .../search/sort/FieldSortBuilder.java         |  8 +--
 .../search/sort/GeoDistanceSortBuilder.java   |  6 +-
 .../search/sort/ScriptSortBuilder.java        |  8 +--
 .../search/sort/SortBuilder.java              |  2 +-
 .../index/query/AbstractQueryTestCase.java    | 34 +++++-----
 .../index/query/BoolQueryBuilderTests.java    | 18 +++---
 .../query/BoostingQueryBuilderTests.java      |  2 +-
 .../query/ConstantScoreQueryBuilderTests.java |  2 +-
 .../index/query/DisMaxQueryBuilderTests.java  |  4 +-
 .../GeoBoundingBoxQueryBuilderTests.java      |  2 +-
 .../query/GeoShapeQueryBuilderTests.java      |  2 +-
 .../query/HasChildQueryBuilderTests.java      |  4 +-
 .../index/query/IndicesQueryBuilderTests.java |  2 +-
 .../index/query/InnerHitBuilderTests.java     |  2 +-
 .../index/query/NestedQueryBuilderTests.java  |  2 +-
 .../query/PercolateQueryBuilderTests.java     |  2 +-
 .../query/QueryStringQueryBuilderTests.java   |  2 +-
 .../index/query/RandomQueryBuilder.java       |  4 +-
 .../index/query/RangeQueryBuilderTests.java   |  8 +--
 .../query/SpanMultiTermQueryBuilderTests.java |  2 +-
 .../query/SpanNearQueryBuilderTests.java      |  2 +-
 .../index/query/SpanOrQueryBuilderTests.java  |  4 +-
 .../query/TemplateQueryBuilderTests.java      | 12 ++--
 .../index/query/TermsQueryBuilderTests.java   |  2 +-
 .../index/query/WrapperQueryBuilderTests.java |  8 +--
 .../FunctionScoreQueryBuilderTests.java       | 16 ++---
 .../search/aggregations/bucket/FilterIT.java  |  4 +-
 .../search/aggregations/bucket/FiltersIT.java |  6 +-
 .../aggregations/metrics/FiltersTests.java    |  2 +-
 .../functionscore/DecayFunctionScoreIT.java   |  2 +-
 .../search/functionscore/QueryRescorerIT.java |  2 +-
 .../rescore/QueryRescoreBuilderTests.java     |  2 +-
 .../search/sort/AbstractSortTestCase.java     |  2 +-
 .../migration/migrate_5_0/java.asciidoc       |  8 +--
 .../messy/tests/MinDocCountTests.java         |  2 +-
 .../AbstractBulkByScrollRequestBuilder.java   |  2 +-
 .../reindex/ReindexParentChildTests.java      |  6 +-
 .../deletebyquery/DeleteByQueryRequest.java   |  6 +-
 .../DeleteByQueryRequestBuilder.java          |  2 +-
 .../RestDeleteByQueryAction.java              |  2 +-
 103 files changed, 359 insertions(+), 354 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java
index 0aec5777974..831ef6e1060 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ShardValidateQueryRequest.java
@@ -34,7 +34,7 @@ import java.io.IOException;
  */
 public class ShardValidateQueryRequest extends BroadcastShardRequest {
 
-    private QueryBuilder<?> query;
+    private QueryBuilder query;
     private String[] types = Strings.EMPTY_ARRAY;
     private boolean explain;
     private boolean rewrite;
@@ -57,7 +57,7 @@ public class ShardValidateQueryRequest extends BroadcastShardRequest {
         this.nowInMillis = request.nowInMillis;
     }
 
-    public QueryBuilder<?> query() {
+    public QueryBuilder query() {
         return query;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java
index 603da6bfe9d..41ef37ad621 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/ValidateQueryRequest.java
@@ -39,7 +39,7 @@ import java.util.Arrays;
  */
 public class ValidateQueryRequest extends BroadcastRequest<ValidateQueryRequest> {
 
-    private QueryBuilder<?> query = new MatchAllQueryBuilder();
+    private QueryBuilder query = new MatchAllQueryBuilder();
 
     private boolean explain;
     private boolean rewrite;
@@ -73,11 +73,11 @@ public class ValidateQueryRequest extends BroadcastRequest<ValidateQueryRequest>
     /**
      * The query to validate.
      */
-    public QueryBuilder<?> query() {
+    public QueryBuilder query() {
         return query;
     }
 
-    public ValidateQueryRequest query(QueryBuilder<?> query) {
+    public ValidateQueryRequest query(QueryBuilder query) {
         this.query = query;
         return this;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java b/core/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java
index 6e1b6e82730..611d57345ba 100644
--- a/core/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/explain/ExplainRequest.java
@@ -39,7 +39,7 @@ public class ExplainRequest extends SingleShardRequest<ExplainRequest> {
     private String id;
     private String routing;
     private String preference;
-    private QueryBuilder<?> query;
+    private QueryBuilder query;
     private String[] fields;
     private FetchSourceContext fetchSourceContext;
 
@@ -100,11 +100,11 @@ public class ExplainRequest extends SingleShardRequest<ExplainRequest> {
         return this;
     }
 
-    public QueryBuilder<?> query() {
+    public QueryBuilder query() {
         return query;
     }
 
-    public ExplainRequest query(QueryBuilder<?> query) {
+    public ExplainRequest query(QueryBuilder query) {
         this.query = query;
         return this;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java b/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java
index 5a5924f7883..a6ee99a476c 100644
--- a/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java
@@ -48,7 +48,7 @@ import java.util.Map;
 public class PercolateSourceBuilder extends ToXContentToBytes {
 
     private DocBuilder docBuilder;
-    private QueryBuilder<?> queryBuilder;
+    private QueryBuilder queryBuilder;
     private Integer size;
     private List<SortBuilder<?>> sorts;
     private Boolean trackScores;
@@ -68,7 +68,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes {
      * Sets a query to reduce the number of percolate queries to be evaluated and score the queries that match based
      * on this query.
      */
-    public PercolateSourceBuilder setQueryBuilder(QueryBuilder<?> queryBuilder) {
+    public PercolateSourceBuilder setQueryBuilder(QueryBuilder queryBuilder) {
         this.queryBuilder = queryBuilder;
         return this;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/TransportPercolateAction.java b/core/src/main/java/org/elasticsearch/action/percolate/TransportPercolateAction.java
index 5c8b20b1f92..b80589df936 100644
--- a/core/src/main/java/org/elasticsearch/action/percolate/TransportPercolateAction.java
+++ b/core/src/main/java/org/elasticsearch/action/percolate/TransportPercolateAction.java
@@ -203,7 +203,7 @@ public class TransportPercolateAction extends HandledTransportAction<PercolateRe
         if (querySource != null) {
             try (XContentParser parser = XContentHelper.createParser(querySource)) {
                 QueryParseContext queryParseContext = new QueryParseContext(queryRegistry, parser, parseFieldMatcher);
-                QueryBuilder<?> queryBuilder = queryParseContext.parseInnerQueryBuilder();
+                QueryBuilder queryBuilder = queryParseContext.parseInnerQueryBuilder();
                 BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
                 boolQueryBuilder.must(queryBuilder);
                 boolQueryBuilder.filter(percolateQueryBuilder);
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
index fc28addc111..9830f7be203 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
@@ -166,7 +166,7 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
      *
      * @see org.elasticsearch.index.query.QueryBuilders
      */
-    public SearchRequestBuilder setQuery(QueryBuilder<?> queryBuilder) {
+    public SearchRequestBuilder setQuery(QueryBuilder queryBuilder) {
         sourceBuilder().query(queryBuilder);
         return this;
     }
@@ -175,7 +175,7 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
      * Sets a filter that will be executed after the query has been executed and only has affect on the search hits
      * (not aggregations). This filter is always executed as last filtering mechanism.
      */
-    public SearchRequestBuilder setPostFilter(QueryBuilder<?> postFilter) {
+    public SearchRequestBuilder setPostFilter(QueryBuilder postFilter) {
         sourceBuilder().postFilter(postFilter);
         return this;
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java
index 64da04e6511..735916504da 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java
@@ -141,7 +141,7 @@ public class AliasValidator extends AbstractComponent {
 
     private static void validateAliasFilter(XContentParser parser, QueryShardContext queryShardContext) throws IOException {
         QueryParseContext queryParseContext = queryShardContext.newParseContext(parser);
-        QueryBuilder<?> queryBuilder = QueryBuilder.rewriteQuery(queryParseContext.parseInnerQueryBuilder(), queryShardContext);
+        QueryBuilder queryBuilder = QueryBuilder.rewriteQuery(queryParseContext.parseInnerQueryBuilder(), queryShardContext);
         queryBuilder.toFilter(queryShardContext);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
index 10c7e46e353..58f9f6c695c 100644
--- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
@@ -186,7 +186,7 @@ public class PercolatorFieldMapper extends FieldMapper {
         }
 
         XContentParser parser = context.parser();
-        QueryBuilder<?> queryBuilder = parseQueryBuilder(queryShardContext.newParseContext(parser), parser.getTokenLocation());
+        QueryBuilder queryBuilder = parseQueryBuilder(queryShardContext.newParseContext(parser), parser.getTokenLocation());
         // Fetching of terms, shapes and indexed scripts happen during this rewrite:
         queryBuilder = queryBuilder.rewrite(queryShardContext);
 
@@ -206,7 +206,7 @@ public class PercolatorFieldMapper extends FieldMapper {
         return toQuery(context, mapUnmappedFieldsAsString, parseQueryBuilder(context.newParseContext(parser), parser.getTokenLocation()));
     }
 
-    static Query toQuery(QueryShardContext context, boolean mapUnmappedFieldsAsString, QueryBuilder<?> queryBuilder) throws IOException {
+    static Query toQuery(QueryShardContext context, boolean mapUnmappedFieldsAsString, QueryBuilder queryBuilder) throws IOException {
         // This means that fields in the query need to exist in the mapping prior to registering this query
         // The reason that this is required, is that if a field doesn't exist then the query assumes defaults, which may be undesired.
         //
diff --git a/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java
index 6e82e7059d8..e883166f900 100644
--- a/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java
@@ -43,7 +43,7 @@ import java.util.Objects;
  * Base class for all classes producing lucene queries.
  * Supports conversion to BytesReference and creation of lucene Query objects.
  */
-public abstract class AbstractQueryBuilder<QB extends AbstractQueryBuilder<QB>> extends ToXContentToBytes implements QueryBuilder<QB> {
+public abstract class AbstractQueryBuilder<QB extends AbstractQueryBuilder<QB>> extends ToXContentToBytes implements QueryBuilder {
 
     /** Default for boost to apply to resulting Lucene query. Defaults to 1.0*/
     public static final float DEFAULT_BOOST = 1.0f;
@@ -221,10 +221,10 @@ public abstract class AbstractQueryBuilder<QB extends AbstractQueryBuilder<QB>>
      * their {@link QueryBuilder#toQuery(QueryShardContext)} method are not added to the
      * resulting collection.
      */
-    protected static Collection<Query> toQueries(Collection<QueryBuilder<?>> queryBuilders, QueryShardContext context) throws QueryShardException,
+    protected static Collection<Query> toQueries(Collection<QueryBuilder> queryBuilders, QueryShardContext context) throws QueryShardException,
             IOException {
         List<Query> queries = new ArrayList<>(queryBuilders.size());
-        for (QueryBuilder<?> queryBuilder : queryBuilders) {
+        for (QueryBuilder queryBuilder : queryBuilders) {
             Query query = queryBuilder.toQuery(context);
             if (query != null) {
                 queries.add(query);
@@ -241,13 +241,13 @@ public abstract class AbstractQueryBuilder<QB extends AbstractQueryBuilder<QB>>
 
     protected final static void writeQueries(StreamOutput out, List<? extends QueryBuilder> queries) throws IOException {
         out.writeVInt(queries.size());
-        for (QueryBuilder<?> query : queries) {
+        for (QueryBuilder query : queries) {
             out.writeNamedWriteable(query);
         }
     }
 
-    protected final static List<QueryBuilder<?>> readQueries(StreamInput in) throws IOException {
-        List<QueryBuilder<?>> queries = new ArrayList<>();
+    protected final static List<QueryBuilder> readQueries(StreamInput in) throws IOException {
+        List<QueryBuilder> queries = new ArrayList<>();
         int size = in.readVInt();
         for (int i = 0; i < size; i++) {
             queries.add(in.readNamedWriteable(QueryBuilder.class));
@@ -256,8 +256,8 @@ public abstract class AbstractQueryBuilder<QB extends AbstractQueryBuilder<QB>>
     }
 
     @Override
-    public final QueryBuilder<?> rewrite(QueryRewriteContext queryShardContext) throws IOException {
-        QueryBuilder<?> rewritten = doRewrite(queryShardContext);
+    public final QueryBuilder rewrite(QueryRewriteContext queryShardContext) throws IOException {
+        QueryBuilder rewritten = doRewrite(queryShardContext);
         if (rewritten == this) {
             return rewritten;
         }
@@ -270,7 +270,7 @@ public abstract class AbstractQueryBuilder<QB extends AbstractQueryBuilder<QB>>
         return rewritten;
     }
 
-    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryShardContext) throws IOException {
+    protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOException {
         return this;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
index c6de1213a8d..e5aa774addc 100644
--- a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
@@ -62,13 +62,13 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
     private static final ParseField MINIMUM_NUMBER_SHOULD_MATCH = new ParseField("minimum_number_should_match");
     private static final ParseField ADJUST_PURE_NEGATIVE = new ParseField("adjust_pure_negative");
 
-    private final List<QueryBuilder<?>> mustClauses = new ArrayList<>();
+    private final List<QueryBuilder> mustClauses = new ArrayList<>();
 
-    private final List<QueryBuilder<?>> mustNotClauses = new ArrayList<>();
+    private final List<QueryBuilder> mustNotClauses = new ArrayList<>();
 
-    private final List<QueryBuilder<?>> filterClauses = new ArrayList<>();
+    private final List<QueryBuilder> filterClauses = new ArrayList<>();
 
-    private final List<QueryBuilder<?>> shouldClauses = new ArrayList<>();
+    private final List<QueryBuilder> shouldClauses = new ArrayList<>();
 
     private boolean disableCoord = DISABLE_COORD_DEFAULT;
 
@@ -111,7 +111,7 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
      * Adds a query that <b>must</b> appear in the matching documents and will
      * contribute to scoring. No <tt>null</tt> value allowed.
      */
-    public BoolQueryBuilder must(QueryBuilder<?> queryBuilder) {
+    public BoolQueryBuilder must(QueryBuilder queryBuilder) {
         if (queryBuilder == null) {
             throw new IllegalArgumentException("inner bool query clause cannot be null");
         }
@@ -122,7 +122,7 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
     /**
      * Gets the queries that <b>must</b> appear in the matching documents.
      */
-    public List<QueryBuilder<?>> must() {
+    public List<QueryBuilder> must() {
         return this.mustClauses;
     }
 
@@ -130,7 +130,7 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
      * Adds a query that <b>must</b> appear in the matching documents but will
      * not contribute to scoring. No <tt>null</tt> value allowed.
      */
-    public BoolQueryBuilder filter(QueryBuilder<?> queryBuilder) {
+    public BoolQueryBuilder filter(QueryBuilder queryBuilder) {
         if (queryBuilder == null) {
             throw new IllegalArgumentException("inner bool query clause cannot be null");
         }
@@ -141,7 +141,7 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
     /**
      * Gets the queries that <b>must</b> appear in the matching documents but don't contribute to scoring
      */
-    public List<QueryBuilder<?>> filter() {
+    public List<QueryBuilder> filter() {
         return this.filterClauses;
     }
 
@@ -149,7 +149,7 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
      * Adds a query that <b>must not</b> appear in the matching documents.
      * No <tt>null</tt> value allowed.
      */
-    public BoolQueryBuilder mustNot(QueryBuilder<?> queryBuilder) {
+    public BoolQueryBuilder mustNot(QueryBuilder queryBuilder) {
         if (queryBuilder == null) {
             throw new IllegalArgumentException("inner bool query clause cannot be null");
         }
@@ -160,7 +160,7 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
     /**
      * Gets the queries that <b>must not</b> appear in the matching documents.
      */
-    public List<QueryBuilder<?>> mustNot() {
+    public List<QueryBuilder> mustNot() {
         return this.mustNotClauses;
     }
 
@@ -171,7 +171,7 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
      *
      * @see #minimumNumberShouldMatch(int)
      */
-    public BoolQueryBuilder should(QueryBuilder<?> queryBuilder) {
+    public BoolQueryBuilder should(QueryBuilder queryBuilder) {
         if (queryBuilder == null) {
             throw new IllegalArgumentException("inner bool query clause cannot be null");
         }
@@ -185,7 +185,7 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
      * @see #should(QueryBuilder)
      *  @see #minimumNumberShouldMatch(int)
      */
-    public List<QueryBuilder<?>> should() {
+    public List<QueryBuilder> should() {
         return this.shouldClauses;
     }
 
@@ -288,13 +288,13 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
         builder.endObject();
     }
 
-    private static void doXArrayContent(String field, List<QueryBuilder<?>> clauses, XContentBuilder builder, Params params)
+    private static void doXArrayContent(String field, List<QueryBuilder> clauses, XContentBuilder builder, Params params)
             throws IOException {
         if (clauses.isEmpty()) {
             return;
         }
         builder.startArray(field);
-        for (QueryBuilder<?> clause : clauses) {
+        for (QueryBuilder clause : clauses) {
             clause.toXContent(builder, params);
         }
         builder.endArray();
@@ -308,15 +308,15 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
         String minimumShouldMatch = null;
 
-        final List<QueryBuilder<?>> mustClauses = new ArrayList<>();
-        final List<QueryBuilder<?>> mustNotClauses = new ArrayList<>();
-        final List<QueryBuilder<?>> shouldClauses = new ArrayList<>();
-        final List<QueryBuilder<?>> filterClauses = new ArrayList<>();
+        final List<QueryBuilder> mustClauses = new ArrayList<>();
+        final List<QueryBuilder> mustNotClauses = new ArrayList<>();
+        final List<QueryBuilder> shouldClauses = new ArrayList<>();
+        final List<QueryBuilder> filterClauses = new ArrayList<>();
         String queryName = null;
 
         String currentFieldName = null;
         XContentParser.Token token;
-        QueryBuilder<?> query;
+        QueryBuilder query;
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
@@ -387,16 +387,16 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
             }
         }
         BoolQueryBuilder boolQuery = new BoolQueryBuilder();
-        for (QueryBuilder<?> queryBuilder : mustClauses) {
+        for (QueryBuilder queryBuilder : mustClauses) {
             boolQuery.must(queryBuilder);
         }
-        for (QueryBuilder<?> queryBuilder : mustNotClauses) {
+        for (QueryBuilder queryBuilder : mustNotClauses) {
             boolQuery.mustNot(queryBuilder);
         }
-        for (QueryBuilder<?> queryBuilder : shouldClauses) {
+        for (QueryBuilder queryBuilder : shouldClauses) {
             boolQuery.should(queryBuilder);
         }
-        for (QueryBuilder<?> queryBuilder : filterClauses) {
+        for (QueryBuilder queryBuilder : filterClauses) {
             boolQuery.filter(queryBuilder);
         }
         boolQuery.boost(boost);
@@ -436,8 +436,8 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
     }
 
     private static void addBooleanClauses(QueryShardContext context, BooleanQuery.Builder booleanQueryBuilder,
-                                          List<QueryBuilder<?>> clauses, Occur occurs) throws IOException {
-        for (QueryBuilder<?> query : clauses) {
+                                          List<QueryBuilder> clauses, Occur occurs) throws IOException {
+        for (QueryBuilder query : clauses) {
             Query luceneQuery = null;
             switch (occurs) {
                 case MUST:
@@ -473,7 +473,7 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
     }
 
     @Override
-    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+    protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
         BoolQueryBuilder newBuilder = new BoolQueryBuilder();
         boolean changed = false;
         final int clauses = mustClauses.size() + mustNotClauses.size() + filterClauses.size() + shouldClauses.size();
@@ -498,20 +498,20 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
 
     @Override
     protected void extractInnerHitBuilders(Map<String, InnerHitBuilder> innerHits) {
-        List<QueryBuilder<?>> clauses = new ArrayList<>(filter());
+        List<QueryBuilder> clauses = new ArrayList<>(filter());
         clauses.addAll(must());
         clauses.addAll(should());
         // no need to include must_not (since there will be no hits for it)
-        for (QueryBuilder<?> clause : clauses) {
+        for (QueryBuilder clause : clauses) {
             InnerHitBuilder.extractInnerHits(clause, innerHits);
         }
     }
 
-    private static boolean rewriteClauses(QueryRewriteContext queryRewriteContext, List<QueryBuilder<?>> builders,
-                                          Consumer<QueryBuilder<?>> consumer) throws IOException {
+    private static boolean rewriteClauses(QueryRewriteContext queryRewriteContext, List<QueryBuilder> builders,
+                                          Consumer<QueryBuilder> consumer) throws IOException {
         boolean changed = false;
-        for (QueryBuilder<?> builder : builders) {
-            QueryBuilder<?> result = builder.rewrite(queryRewriteContext);
+        for (QueryBuilder builder : builders) {
+            QueryBuilder result = builder.rewrite(queryRewriteContext);
             if (result != builder) {
                 changed = true;
             }
diff --git a/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java
index fa439bc71d8..496cb7ec8a7 100644
--- a/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java
@@ -53,9 +53,9 @@ public class BoostingQueryBuilder extends AbstractQueryBuilder<BoostingQueryBuil
     private static final ParseField NEGATIVE_FIELD = new ParseField("negative");
     private static final ParseField NEGATIVE_BOOST_FIELD = new ParseField("negative_boost");
 
-    private final QueryBuilder<?> positiveQuery;
+    private final QueryBuilder positiveQuery;
 
-    private final QueryBuilder<?> negativeQuery;
+    private final QueryBuilder negativeQuery;
 
     private float negativeBoost = -1;
 
@@ -66,7 +66,7 @@ public class BoostingQueryBuilder extends AbstractQueryBuilder<BoostingQueryBuil
      * @param positiveQuery the positive query for this boosting query.
      * @param negativeQuery the negative query for this boosting query.
      */
-    public BoostingQueryBuilder(QueryBuilder<?> positiveQuery, QueryBuilder<?> negativeQuery) {
+    public BoostingQueryBuilder(QueryBuilder positiveQuery, QueryBuilder negativeQuery) {
         if (positiveQuery == null) {
             throw new IllegalArgumentException("inner clause [positive] cannot be null.");
         }
@@ -226,7 +226,7 @@ public class BoostingQueryBuilder extends AbstractQueryBuilder<BoostingQueryBuil
     }
 
     @Override
-    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+    protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
         QueryBuilder positiveQuery = this.positiveQuery.rewrite(queryRewriteContext);
         QueryBuilder negativeQuery = this.negativeQuery.rewrite(queryRewriteContext);
         if (positiveQuery != this.positiveQuery || negativeQuery != this.negativeQuery) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java
index 8ae2a088f2f..306a4568576 100644
--- a/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java
@@ -43,7 +43,7 @@ public class ConstantScoreQueryBuilder extends AbstractQueryBuilder<ConstantScor
 
     private static final ParseField INNER_QUERY_FIELD = new ParseField("filter", "query");
 
-    private final QueryBuilder<?> filterBuilder;
+    private final QueryBuilder filterBuilder;
 
     /**
      * A query that wraps another query and simply returns a constant score equal to the
@@ -51,7 +51,7 @@ public class ConstantScoreQueryBuilder extends AbstractQueryBuilder<ConstantScor
      *
      * @param filterBuilder The query to wrap in a constant score query
      */
-    public ConstantScoreQueryBuilder(QueryBuilder<?> filterBuilder) {
+    public ConstantScoreQueryBuilder(QueryBuilder filterBuilder) {
         if (filterBuilder == null) {
             throw new IllegalArgumentException("inner clause [filter] cannot be null.");
         }
@@ -74,7 +74,7 @@ public class ConstantScoreQueryBuilder extends AbstractQueryBuilder<ConstantScor
     /**
      * @return the query that was wrapped in this constant score query
      */
-    public QueryBuilder<?> innerQuery() {
+    public QueryBuilder innerQuery() {
         return this.filterBuilder;
     }
 
@@ -90,7 +90,7 @@ public class ConstantScoreQueryBuilder extends AbstractQueryBuilder<ConstantScor
     public static ConstantScoreQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
-        QueryBuilder<?> query = null;
+        QueryBuilder query = null;
         boolean queryFound = false;
         String queryName = null;
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
@@ -163,8 +163,8 @@ public class ConstantScoreQueryBuilder extends AbstractQueryBuilder<ConstantScor
     }
 
     @Override
-    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
-        QueryBuilder<?> rewrite = filterBuilder.rewrite(queryRewriteContext);
+    protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+        QueryBuilder rewrite = filterBuilder.rewrite(queryRewriteContext);
         if (rewrite != filterBuilder) {
             return new ConstantScoreQueryBuilder(rewrite);
         }
diff --git a/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java
index 04662fbe863..f38b2c09b16 100644
--- a/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java
@@ -50,7 +50,7 @@ public class DisMaxQueryBuilder extends AbstractQueryBuilder<DisMaxQueryBuilder>
     private static final ParseField TIE_BREAKER_FIELD = new ParseField("tie_breaker");
     private static final ParseField QUERIES_FIELD = new ParseField("queries");
 
-    private final List<QueryBuilder<?>> queries = new ArrayList<>();
+    private final List<QueryBuilder> queries = new ArrayList<>();
 
     private float tieBreaker = DEFAULT_TIE_BREAKER;
 
@@ -75,7 +75,7 @@ public class DisMaxQueryBuilder extends AbstractQueryBuilder<DisMaxQueryBuilder>
     /**
      * Add a sub-query to this disjunction.
      */
-    public DisMaxQueryBuilder add(QueryBuilder<?> queryBuilder) {
+    public DisMaxQueryBuilder add(QueryBuilder queryBuilder) {
         if (queryBuilder == null) {
             throw new IllegalArgumentException("inner dismax query clause cannot be null");
         }
@@ -86,7 +86,7 @@ public class DisMaxQueryBuilder extends AbstractQueryBuilder<DisMaxQueryBuilder>
     /**
      * @return an immutable list copy of the current sub-queries of this disjunction
      */
-    public List<QueryBuilder<?>> innerQueries() {
+    public List<QueryBuilder> innerQueries() {
         return this.queries;
     }
 
@@ -114,7 +114,7 @@ public class DisMaxQueryBuilder extends AbstractQueryBuilder<DisMaxQueryBuilder>
         builder.startObject(NAME);
         builder.field(TIE_BREAKER_FIELD.getPreferredName(), tieBreaker);
         builder.startArray(QUERIES_FIELD.getPreferredName());
-        for (QueryBuilder<?> queryBuilder : queries) {
+        for (QueryBuilder queryBuilder : queries) {
             queryBuilder.toXContent(builder, params);
         }
         builder.endArray();
@@ -128,7 +128,7 @@ public class DisMaxQueryBuilder extends AbstractQueryBuilder<DisMaxQueryBuilder>
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
         float tieBreaker = DisMaxQueryBuilder.DEFAULT_TIE_BREAKER;
 
-        final List<QueryBuilder<?>> queries = new ArrayList<>();
+        final List<QueryBuilder> queries = new ArrayList<>();
         boolean queriesFound = false;
         String queryName = null;
 
@@ -140,7 +140,7 @@ public class DisMaxQueryBuilder extends AbstractQueryBuilder<DisMaxQueryBuilder>
             } else if (token == XContentParser.Token.START_OBJECT) {
                 if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERIES_FIELD)) {
                     queriesFound = true;
-                    QueryBuilder<?> query = parseContext.parseInnerQueryBuilder();
+                    QueryBuilder query = parseContext.parseInnerQueryBuilder();
                     queries.add(query);
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[dis_max] query does not support [" + currentFieldName + "]");
@@ -149,7 +149,7 @@ public class DisMaxQueryBuilder extends AbstractQueryBuilder<DisMaxQueryBuilder>
                 if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERIES_FIELD)) {
                     queriesFound = true;
                     while (token != XContentParser.Token.END_ARRAY) {
-                        QueryBuilder<?> query = parseContext.parseInnerQueryBuilder();
+                        QueryBuilder query = parseContext.parseInnerQueryBuilder();
                         queries.add(query);
                         token = parser.nextToken();
                     }
@@ -177,7 +177,7 @@ public class DisMaxQueryBuilder extends AbstractQueryBuilder<DisMaxQueryBuilder>
         disMaxQuery.tieBreaker(tieBreaker);
         disMaxQuery.queryName(queryName);
         disMaxQuery.boost(boost);
-        for (QueryBuilder<?> query : queries) {
+        for (QueryBuilder query : queries) {
             disMaxQuery.add(query);
         }
         return disMaxQuery;
diff --git a/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java
index e44f2868040..0382f353cb3 100644
--- a/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java
@@ -35,7 +35,7 @@ import java.io.IOException;
 import java.util.Objects;
 
 public class FieldMaskingSpanQueryBuilder extends AbstractQueryBuilder<FieldMaskingSpanQueryBuilder>
-        implements SpanQueryBuilder<FieldMaskingSpanQueryBuilder>{
+        implements SpanQueryBuilder {
 
     public static final String NAME = "field_masking_span";
     public static final ParseField QUERY_NAME_FIELD = new ParseField(NAME);
@@ -43,7 +43,7 @@ public class FieldMaskingSpanQueryBuilder extends AbstractQueryBuilder<FieldMask
     private static final ParseField FIELD_FIELD = new ParseField("field");
     private static final ParseField QUERY_FIELD = new ParseField("query");
 
-    private final SpanQueryBuilder<?> queryBuilder;
+    private final SpanQueryBuilder queryBuilder;
 
     private final String fieldName;
 
@@ -53,7 +53,7 @@ public class FieldMaskingSpanQueryBuilder extends AbstractQueryBuilder<FieldMask
      * @param queryBuilder inner {@link SpanQueryBuilder}
      * @param fieldName the field name
      */
-    public FieldMaskingSpanQueryBuilder(SpanQueryBuilder<?> queryBuilder, String fieldName) {
+    public FieldMaskingSpanQueryBuilder(SpanQueryBuilder queryBuilder, String fieldName) {
         if (Strings.isEmpty(fieldName)) {
             throw new IllegalArgumentException("field name is null or empty");
         }
@@ -69,7 +69,7 @@ public class FieldMaskingSpanQueryBuilder extends AbstractQueryBuilder<FieldMask
      */
     public FieldMaskingSpanQueryBuilder(StreamInput in) throws IOException {
         super(in);
-        queryBuilder = (SpanQueryBuilder<?>) in.readNamedWriteable(QueryBuilder.class);
+        queryBuilder = (SpanQueryBuilder) in.readNamedWriteable(QueryBuilder.class);
         fieldName = in.readString();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryBuilder.java
index 7604ef94eb6..70b4f5eb7f5 100644
--- a/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryBuilder.java
@@ -45,7 +45,7 @@ import java.util.Objects;
  * a match query with the fuzziness parameter for strings or range queries for numeric and date fields.
  */
 @Deprecated
-public class FuzzyQueryBuilder extends AbstractQueryBuilder<FuzzyQueryBuilder> implements MultiTermQueryBuilder<FuzzyQueryBuilder> {
+public class FuzzyQueryBuilder extends AbstractQueryBuilder<FuzzyQueryBuilder> implements MultiTermQueryBuilder {
 
     public static final String NAME = "fuzzy";
     public static final ParseField QUERY_NAME_FIELD = new ParseField(NAME);
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
index d3c47d5bc70..8b3c8a62486 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
@@ -587,7 +587,7 @@ public class GeoShapeQueryBuilder extends AbstractQueryBuilder<GeoShapeQueryBuil
     }
 
     @Override
-    protected QueryBuilder<GeoShapeQueryBuilder> doRewrite(QueryRewriteContext queryShardContext) throws IOException {
+    protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOException {
         if (this.shape == null) {
             GetRequest getRequest = new GetRequest(indexedShapeIndex, indexedShapeType, indexedShapeId);
             ShapeBuilder shape = fetch(queryShardContext.getClient(), getRequest, indexedShapePath);
diff --git a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
index 61466ab5ea5..990b5a35fd9 100644
--- a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
@@ -77,7 +77,7 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
     private static final ParseField INNER_HITS_FIELD = new ParseField("inner_hits");
     private static final ParseField IGNORE_UNMAPPED_FIELD = new ParseField("ignore_unmapped");
 
-    private final QueryBuilder<?> query;
+    private final QueryBuilder query;
     private final String type;
     private final ScoreMode scoreMode;
     private InnerHitBuilder innerHitBuilder;
@@ -85,11 +85,11 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
     private int maxChildren = DEFAULT_MAX_CHILDREN;
     private boolean ignoreUnmapped = false;
 
-    public HasChildQueryBuilder(String type, QueryBuilder<?> query, ScoreMode scoreMode) {
+    public HasChildQueryBuilder(String type, QueryBuilder query, ScoreMode scoreMode) {
         this(type, query, DEFAULT_MIN_CHILDREN, DEFAULT_MAX_CHILDREN, scoreMode, null);
     }
 
-    private HasChildQueryBuilder(String type, QueryBuilder<?> query, int minChildren, int maxChildren, ScoreMode scoreMode,
+    private HasChildQueryBuilder(String type, QueryBuilder query, int minChildren, int maxChildren, ScoreMode scoreMode,
                                 InnerHitBuilder innerHitBuilder) {
         this.type = requireValue(type, "[" + NAME + "] requires 'type' field");
         this.query = requireValue(query, "[" + NAME + "] requires 'query' field");
@@ -158,7 +158,7 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
     /**
      * Returns the children query to execute.
      */
-    public QueryBuilder<?> query() {
+    public QueryBuilder query() {
         return query;
     }
 
@@ -238,7 +238,7 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
         InnerHitBuilder innerHitBuilder = null;
         String currentFieldName = null;
         XContentParser.Token token;
-        QueryBuilder<?> iqb = null;
+        QueryBuilder iqb = null;
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
@@ -467,8 +467,8 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
     }
 
     @Override
-    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
-        QueryBuilder<?> rewrite = query.rewrite(queryRewriteContext);
+    protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+        QueryBuilder rewrite = query.rewrite(queryRewriteContext);
         if (rewrite != query) {
             return new HasChildQueryBuilder(type, rewrite, minChildren, minChildren, scoreMode, innerHitBuilder);
         }
diff --git a/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java
index dc3b4d9c66a..96356c276e7 100644
--- a/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java
@@ -60,17 +60,17 @@ public class HasParentQueryBuilder extends AbstractQueryBuilder<HasParentQueryBu
     private static final ParseField INNER_HITS_FIELD = new ParseField("inner_hits");
     private static final ParseField IGNORE_UNMAPPED_FIELD = new ParseField("ignore_unmapped");
 
-    private final QueryBuilder<?> query;
+    private final QueryBuilder query;
     private final String type;
     private final boolean score;
     private InnerHitBuilder innerHit;
     private boolean ignoreUnmapped = false;
 
-    public HasParentQueryBuilder(String type, QueryBuilder<?> query, boolean score) {
+    public HasParentQueryBuilder(String type, QueryBuilder query, boolean score) {
         this(type, query, score, null);
     }
 
-    private HasParentQueryBuilder(String type, QueryBuilder<?> query, boolean score, InnerHitBuilder innerHit) {
+    private HasParentQueryBuilder(String type, QueryBuilder query, boolean score, InnerHitBuilder innerHit) {
         this.type = requireValue(type, "[" + NAME + "] requires 'type' field");
         this.query = requireValue(query, "[" + NAME + "] requires 'query' field");
         this.score = score;
@@ -101,7 +101,7 @@ public class HasParentQueryBuilder extends AbstractQueryBuilder<HasParentQueryBu
     /**
      * Returns the query to execute.
      */
-    public QueryBuilder<?> query() {
+    public QueryBuilder query() {
         return query;
     }
 
@@ -238,7 +238,7 @@ public class HasParentQueryBuilder extends AbstractQueryBuilder<HasParentQueryBu
 
         String currentFieldName = null;
         XContentParser.Token token;
-        QueryBuilder<?> iqb = null;
+        QueryBuilder iqb = null;
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
@@ -306,8 +306,8 @@ public class HasParentQueryBuilder extends AbstractQueryBuilder<HasParentQueryBu
     }
 
     @Override
-    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryShardContext) throws IOException {
-        QueryBuilder<?> rewrite = query.rewrite(queryShardContext);
+    protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOException {
+        QueryBuilder rewrite = query.rewrite(queryShardContext);
         if (rewrite != query) {
             return new HasParentQueryBuilder(type, rewrite, score, innerHit);
         }
diff --git a/core/src/main/java/org/elasticsearch/index/query/IndicesQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/IndicesQueryBuilder.java
index 56fdeb779fa..7cfdf1baa1e 100644
--- a/core/src/main/java/org/elasticsearch/index/query/IndicesQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/IndicesQueryBuilder.java
@@ -55,17 +55,17 @@ public class IndicesQueryBuilder extends AbstractQueryBuilder<IndicesQueryBuilde
 
     private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(IndicesQueryBuilder.class));
 
-    private final QueryBuilder<?> innerQuery;
+    private final QueryBuilder innerQuery;
 
     private final String[] indices;
 
-    private QueryBuilder<?> noMatchQuery = defaultNoMatchQuery();
+    private QueryBuilder noMatchQuery = defaultNoMatchQuery();
 
     /**
      * @deprecated instead search on the `_index` field
      */
     @Deprecated
-    public IndicesQueryBuilder(QueryBuilder<?> innerQuery, String... indices) {
+    public IndicesQueryBuilder(QueryBuilder innerQuery, String... indices) {
         DEPRECATION_LOGGER.deprecated("{} query is deprecated. Instead search on the '_index' field", NAME);
         if (innerQuery == null) {
             throw new IllegalArgumentException("inner query cannot be null");
@@ -94,7 +94,7 @@ public class IndicesQueryBuilder extends AbstractQueryBuilder<IndicesQueryBuilde
         out.writeNamedWriteable(noMatchQuery);
     }
 
-    public QueryBuilder<?> innerQuery() {
+    public QueryBuilder innerQuery() {
         return this.innerQuery;
     }
 
@@ -105,7 +105,7 @@ public class IndicesQueryBuilder extends AbstractQueryBuilder<IndicesQueryBuilde
     /**
      * Sets the query to use when it executes on an index that does not match the indices provided.
      */
-    public IndicesQueryBuilder noMatchQuery(QueryBuilder<?> noMatchQuery) {
+    public IndicesQueryBuilder noMatchQuery(QueryBuilder noMatchQuery) {
         if (noMatchQuery == null) {
             throw new IllegalArgumentException("noMatch query cannot be null");
         }
@@ -121,11 +121,11 @@ public class IndicesQueryBuilder extends AbstractQueryBuilder<IndicesQueryBuilde
         return this;
     }
 
-    public QueryBuilder<?> noMatchQuery() {
+    public QueryBuilder noMatchQuery() {
         return this.noMatchQuery;
     }
 
-    private static QueryBuilder<?> defaultNoMatchQuery() {
+    private static QueryBuilder defaultNoMatchQuery() {
         return QueryBuilders.matchAllQuery();
     }
 
@@ -144,9 +144,9 @@ public class IndicesQueryBuilder extends AbstractQueryBuilder<IndicesQueryBuilde
     public static IndicesQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException, ParsingException {
         XContentParser parser = parseContext.parser();
 
-        QueryBuilder<?> innerQuery = null;
+        QueryBuilder innerQuery = null;
         Collection<String> indices = new ArrayList<>();
-        QueryBuilder<?> noMatchQuery = defaultNoMatchQuery();
+        QueryBuilder noMatchQuery = defaultNoMatchQuery();
 
         String queryName = null;
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
@@ -209,7 +209,7 @@ public class IndicesQueryBuilder extends AbstractQueryBuilder<IndicesQueryBuilde
                 .queryName(queryName);
     }
 
-    static QueryBuilder<?> parseNoMatchQuery(String type) {
+    static QueryBuilder parseNoMatchQuery(String type) {
         if ("all".equals(type)) {
             return QueryBuilders.matchAllQuery();
         } else if ("none".equals(type)) {
@@ -244,9 +244,9 @@ public class IndicesQueryBuilder extends AbstractQueryBuilder<IndicesQueryBuilde
     }
 
     @Override
-    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryShardContext) throws IOException {
-        QueryBuilder<?> newInnnerQuery = innerQuery.rewrite(queryShardContext);
-        QueryBuilder<?> newNoMatchQuery = noMatchQuery.rewrite(queryShardContext);
+    protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOException {
+        QueryBuilder newInnnerQuery = innerQuery.rewrite(queryShardContext);
+        QueryBuilder newNoMatchQuery = noMatchQuery.rewrite(queryShardContext);
         if (newInnnerQuery != innerQuery || newNoMatchQuery != noMatchQuery) {
             return new IndicesQueryBuilder(innerQuery, indices).noMatchQuery(noMatchQuery);
         }
diff --git a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
index 61663713c8e..c07c75532f1 100644
--- a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
@@ -131,7 +131,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
     private boolean trackScores;
 
     private List<String> fieldNames;
-    private QueryBuilder<?> query = new MatchAllQueryBuilder();
+    private QueryBuilder query = new MatchAllQueryBuilder();
     private List<SortBuilder<?>> sorts;
     private List<String> fieldDataFields;
     private Set<ScriptField> scriptFields;
@@ -411,7 +411,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         return this;
     }
 
-    QueryBuilder<?> getQuery() {
+    QueryBuilder getQuery() {
         return query;
     }
 
@@ -632,7 +632,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         return PARSER.parse(context.parser(), new InnerHitBuilder(), context);
     }
 
-    public static void extractInnerHits(QueryBuilder<?> query, Map<String, InnerHitBuilder> innerHitBuilders) {
+    public static void extractInnerHits(QueryBuilder query, Map<String, InnerHitBuilder> innerHitBuilders) {
         if (query instanceof AbstractQueryBuilder) {
             ((AbstractQueryBuilder) query).extractInnerHitBuilders(innerHitBuilders);
         } else {
diff --git a/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
index c71db8c22a5..66f623cbbb3 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
@@ -57,7 +57,6 @@ import org.elasticsearch.index.mapper.core.KeywordFieldMapper.KeywordFieldType;
 import org.elasticsearch.index.mapper.core.StringFieldMapper.StringFieldType;
 import org.elasticsearch.index.mapper.core.TextFieldMapper.TextFieldType;
 import org.elasticsearch.index.mapper.internal.UidFieldMapper;
-import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -1197,7 +1196,7 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ
     }
 
     @Override
-    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+    protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
         // TODO this needs heavy cleanups before we can rewrite it
         return this;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/query/MultiTermQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MultiTermQueryBuilder.java
index 0e946d628a1..be9abfc5e44 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MultiTermQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MultiTermQueryBuilder.java
@@ -18,6 +18,6 @@
  */
 package org.elasticsearch.index.query;
 
-public interface MultiTermQueryBuilder<QB extends MultiTermQueryBuilder<QB>> extends QueryBuilder<QB> {
+public interface MultiTermQueryBuilder extends QueryBuilder {
 
 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
index bba0365d0ae..5d74b540116 100644
--- a/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
@@ -57,7 +57,7 @@ public class NestedQueryBuilder extends AbstractQueryBuilder<NestedQueryBuilder>
 
     private final String path;
     private final ScoreMode scoreMode;
-    private final QueryBuilder<?> query;
+    private final QueryBuilder query;
     private InnerHitBuilder innerHitBuilder;
     private boolean ignoreUnmapped = DEFAULT_IGNORE_UNMAPPED;
 
@@ -161,7 +161,7 @@ public class NestedQueryBuilder extends AbstractQueryBuilder<NestedQueryBuilder>
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
         ScoreMode scoreMode = ScoreMode.Avg;
         String queryName = null;
-        QueryBuilder<?> query = null;
+        QueryBuilder query = null;
         String path = null;
         String currentFieldName = null;
         InnerHitBuilder innerHitBuilder = null;
@@ -259,7 +259,7 @@ public class NestedQueryBuilder extends AbstractQueryBuilder<NestedQueryBuilder>
     }
 
     @Override
-    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+    protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
         QueryBuilder rewrite = query.rewrite(queryRewriteContext);
         if (rewrite != query) {
             return new NestedQueryBuilder(path, rewrite, scoreMode, innerHitBuilder);
diff --git a/core/src/main/java/org/elasticsearch/index/query/PercolateQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/PercolateQueryBuilder.java
index 257460964d3..0e4b999dc33 100644
--- a/core/src/main/java/org/elasticsearch/index/query/PercolateQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/PercolateQueryBuilder.java
@@ -327,7 +327,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
     }
 
     @Override
-    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryShardContext) throws IOException {
+    protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOException {
         if (document != null) {
             return this;
         }
diff --git a/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java
index ffb4179aa3f..44677d1840d 100644
--- a/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java
@@ -40,7 +40,7 @@ import java.util.Objects;
 /**
  * A Query that matches documents containing terms with a specified prefix.
  */
-public class PrefixQueryBuilder extends AbstractQueryBuilder<PrefixQueryBuilder> implements MultiTermQueryBuilder<PrefixQueryBuilder> {
+public class PrefixQueryBuilder extends AbstractQueryBuilder<PrefixQueryBuilder> implements MultiTermQueryBuilder {
 
     public static final String NAME = "prefix";
     public static final ParseField QUERY_NAME_FIELD = new ParseField(NAME);
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/QueryBuilder.java
index dcedd0f1d5e..197af655d54 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryBuilder.java
@@ -25,7 +25,7 @@ import org.elasticsearch.common.xcontent.ToXContent;
 
 import java.io.IOException;
 
-public interface QueryBuilder<QB extends QueryBuilder<QB>> extends NamedWriteable, ToXContent {
+public interface QueryBuilder extends NamedWriteable, ToXContent {
 
     /**
      * Converts this QueryBuilder to a lucene {@link Query}.
@@ -49,8 +49,11 @@ public interface QueryBuilder<QB extends QueryBuilder<QB>> extends NamedWriteabl
 
     /**
      * Sets the arbitrary name to be assigned to the query (see named queries).
+     * Implementers should return the concrete type of the
+     * {@link QueryBuilder} so that calls can be chained. This is done
+     * automatically when extending {@link AbstractQueryBuilder}.
      */
-    QB queryName(String queryName);
+    QueryBuilder queryName(String queryName);
 
     /**
      * Returns the arbitrary name assigned to the query (see named queries).
@@ -65,8 +68,11 @@ public interface QueryBuilder<QB extends QueryBuilder<QB>> extends NamedWriteabl
     /**
      * Sets the boost for this query.  Documents matching this query will (in addition to the normal
      * weightings) have their score multiplied by the boost provided.
+     * Implementers should return the concrete type of the
+     * {@link QueryBuilder} so that calls can be chained. This is done
+     * automatically when extending {@link AbstractQueryBuilder}.
      */
-    QB boost(float boost);
+    QueryBuilder boost(float boost);
 
     /**
      * Returns the name that identifies uniquely the query
@@ -77,7 +83,7 @@ public interface QueryBuilder<QB extends QueryBuilder<QB>> extends NamedWriteabl
      * Rewrites this query builder into its primitive form. By default this method return the builder itself. If the builder
      * did not change the identity reference must be returned otherwise the builder will be rewritten infinitely.
      */
-    default QueryBuilder<?> rewrite(QueryRewriteContext queryShardContext) throws IOException {
+    default QueryBuilder rewrite(QueryRewriteContext queryShardContext) throws IOException {
         return this;
     }
 
@@ -87,7 +93,7 @@ public interface QueryBuilder<QB extends QueryBuilder<QB>> extends NamedWriteabl
      * rewrites the query until it doesn't change anymore.
      * @throws IOException if an {@link IOException} occurs
      */
-    static QueryBuilder<?> rewriteQuery(QueryBuilder<?> original, QueryRewriteContext context) throws IOException {
+    static QueryBuilder rewriteQuery(QueryBuilder original, QueryRewriteContext context) throws IOException {
         QueryBuilder builder = original;
         for (QueryBuilder rewrittenBuilder = builder.rewrite(context); rewrittenBuilder != builder;
              rewrittenBuilder = builder.rewrite(context)) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
index 1199330ee4f..62662914fd8 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
@@ -55,9 +55,9 @@ public class QueryParseContext implements ParseFieldMatcherSupplier {
     /**
      * Parses a top level query including the query element that wraps it
      */
-    public QueryBuilder<?> parseTopLevelQueryBuilder() {
+    public QueryBuilder parseTopLevelQueryBuilder() {
         try {
-            QueryBuilder<?> queryBuilder = null;
+            QueryBuilder queryBuilder = null;
             for (XContentParser.Token token = parser.nextToken(); token != XContentParser.Token.END_OBJECT; token = parser.nextToken()) {
                 if (token == XContentParser.Token.FIELD_NAME) {
                     String fieldName = parser.currentName();
@@ -82,7 +82,7 @@ public class QueryParseContext implements ParseFieldMatcherSupplier {
     /**
      * Parses a query excluding the query element that wraps it
      */
-    public QueryBuilder<?> parseInnerQueryBuilder() throws IOException {
+    public QueryBuilder parseInnerQueryBuilder() throws IOException {
         // move to START object
         XContentParser.Token token;
         if (parser.currentToken() != XContentParser.Token.START_OBJECT) {
@@ -105,7 +105,7 @@ public class QueryParseContext implements ParseFieldMatcherSupplier {
         if (token != XContentParser.Token.START_OBJECT && token != XContentParser.Token.START_ARRAY) {
             throw new ParsingException(parser.getTokenLocation(), "[_na] query malformed, no field after start_object");
         }
-        QueryBuilder<?> result = indicesQueriesRegistry.lookup(queryName, parseFieldMatcher, parser.getTokenLocation()).fromXContent(this);
+        QueryBuilder result = indicesQueriesRegistry.lookup(queryName, parseFieldMatcher, parser.getTokenLocation()).fromXContent(this);
         if (parser.currentToken() == XContentParser.Token.END_OBJECT || parser.currentToken() == XContentParser.Token.END_ARRAY) {
             // if we are at END_OBJECT, move to the next one...
             parser.nextToken();
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryParser.java b/core/src/main/java/org/elasticsearch/index/query/QueryParser.java
index 6f27b8e0647..069dc86cf8a 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryParser.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryParser.java
@@ -25,7 +25,7 @@ import java.io.IOException;
  * Defines a query parser that is able to parse {@link QueryBuilder}s from {@link org.elasticsearch.common.xcontent.XContent}.
  */
 @FunctionalInterface
-public interface QueryParser<QB extends QueryBuilder<QB>> {
+public interface QueryParser<QB extends QueryBuilder> {
     /**
      * Creates a new {@link QueryBuilder} from the query held by the {@link QueryParseContext}
      * in {@link org.elasticsearch.common.xcontent.XContent} format
diff --git a/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
index 5890e2bdffe..241c38475ab 100644
--- a/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
@@ -46,7 +46,7 @@ import java.util.Objects;
 /**
  * A Query that matches documents within an range of terms.
  */
-public class RangeQueryBuilder extends AbstractQueryBuilder<RangeQueryBuilder> implements MultiTermQueryBuilder<RangeQueryBuilder> {
+public class RangeQueryBuilder extends AbstractQueryBuilder<RangeQueryBuilder> implements MultiTermQueryBuilder {
     public static final String NAME = "range";
     public static final ParseField QUERY_NAME_FIELD = new ParseField(NAME);
 
@@ -410,7 +410,7 @@ public class RangeQueryBuilder extends AbstractQueryBuilder<RangeQueryBuilder> i
     }
 
     @Override
-    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+    protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
         final MappedFieldType.Relation relation = getRelation(queryRewriteContext);
         switch (relation) {
         case DISJOINT:
diff --git a/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java
index c895830d5a5..703b2463b11 100644
--- a/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java
@@ -41,7 +41,7 @@ import java.util.Objects;
 /**
  * A Query that does fuzzy matching for a specific value.
  */
-public class RegexpQueryBuilder extends AbstractQueryBuilder<RegexpQueryBuilder> implements MultiTermQueryBuilder<RegexpQueryBuilder> {
+public class RegexpQueryBuilder extends AbstractQueryBuilder<RegexpQueryBuilder> implements MultiTermQueryBuilder {
 
     public static final String NAME = "regexp";
     public static final ParseField QUERY_NAME_FIELD = new ParseField(NAME);
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java
index d923b219897..ae4297e431d 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java
@@ -36,7 +36,7 @@ import java.util.Objects;
  * Builder for {@link org.apache.lucene.search.spans.SpanContainingQuery}.
  */
 public class SpanContainingQueryBuilder extends AbstractQueryBuilder<SpanContainingQueryBuilder>
-        implements SpanQueryBuilder<SpanContainingQueryBuilder> {
+        implements SpanQueryBuilder {
 
     public static final String NAME = "span_containing";
     public static final ParseField QUERY_NAME_FIELD = new ParseField(NAME);
@@ -44,14 +44,14 @@ public class SpanContainingQueryBuilder extends AbstractQueryBuilder<SpanContain
     private static final ParseField BIG_FIELD = new ParseField("big");
     private static final ParseField LITTLE_FIELD = new ParseField("little");
 
-    private final SpanQueryBuilder<?> big;
-    private final SpanQueryBuilder<?> little;
+    private final SpanQueryBuilder big;
+    private final SpanQueryBuilder little;
 
     /**
      * @param big the big clause, it must enclose {@code little} for a match.
      * @param little the little clause, it must be contained within {@code big} for a match.
      */
-    public SpanContainingQueryBuilder(SpanQueryBuilder<?> big, SpanQueryBuilder<?> little) {
+    public SpanContainingQueryBuilder(SpanQueryBuilder big, SpanQueryBuilder little) {
         if (big == null) {
             throw new IllegalArgumentException("inner clause [big] cannot be null.");
         }
@@ -67,8 +67,8 @@ public class SpanContainingQueryBuilder extends AbstractQueryBuilder<SpanContain
      */
     public SpanContainingQueryBuilder(StreamInput in) throws IOException {
         super(in);
-        big = (SpanQueryBuilder<?>) in.readNamedWriteable(QueryBuilder.class);
-        little = (SpanQueryBuilder<?>) in.readNamedWriteable(QueryBuilder.class);
+        big = (SpanQueryBuilder) in.readNamedWriteable(QueryBuilder.class);
+        little = (SpanQueryBuilder) in.readNamedWriteable(QueryBuilder.class);
     }
 
     @Override
@@ -106,8 +106,8 @@ public class SpanContainingQueryBuilder extends AbstractQueryBuilder<SpanContain
         XContentParser parser = parseContext.parser();
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
         String queryName = null;
-        SpanQueryBuilder<?> big = null;
-        SpanQueryBuilder<?> little = null;
+        SpanQueryBuilder big = null;
+        SpanQueryBuilder little = null;
 
         String currentFieldName = null;
         XContentParser.Token token;
@@ -117,16 +117,16 @@ public class SpanContainingQueryBuilder extends AbstractQueryBuilder<SpanContain
             } else if (token == XContentParser.Token.START_OBJECT) {
                 if (parseContext.getParseFieldMatcher().match(currentFieldName, BIG_FIELD)) {
                     QueryBuilder query = parseContext.parseInnerQueryBuilder();
-                    if (!(query instanceof SpanQueryBuilder<?>)) {
+                    if (!(query instanceof SpanQueryBuilder)) {
                         throw new ParsingException(parser.getTokenLocation(), "span_containing [big] must be of type span query");
                     }
-                    big = (SpanQueryBuilder<?>) query;
+                    big = (SpanQueryBuilder) query;
                 } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LITTLE_FIELD)) {
                     QueryBuilder query = parseContext.parseInnerQueryBuilder();
-                    if (!(query instanceof SpanQueryBuilder<?>)) {
+                    if (!(query instanceof SpanQueryBuilder)) {
                         throw new ParsingException(parser.getTokenLocation(), "span_containing [little] must be of type span query");
                     }
-                    little = (SpanQueryBuilder<?>) query;
+                    little = (SpanQueryBuilder) query;
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "[span_containing] query does not support [" + currentFieldName + "]");
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java
index 58e80af977f..009f8a39652 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java
@@ -32,7 +32,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import java.io.IOException;
 import java.util.Objects;
 
-public class SpanFirstQueryBuilder extends AbstractQueryBuilder<SpanFirstQueryBuilder> implements SpanQueryBuilder<SpanFirstQueryBuilder>{
+public class SpanFirstQueryBuilder extends AbstractQueryBuilder<SpanFirstQueryBuilder> implements SpanQueryBuilder {
 
     public static final String NAME = "span_first";
     public static final ParseField QUERY_NAME_FIELD = new ParseField(NAME);
@@ -40,7 +40,7 @@ public class SpanFirstQueryBuilder extends AbstractQueryBuilder<SpanFirstQueryBu
     private static final ParseField MATCH_FIELD = new ParseField("match");
     private static final ParseField END_FIELD = new ParseField("end");
 
-    private final SpanQueryBuilder<?> matchBuilder;
+    private final SpanQueryBuilder matchBuilder;
 
     private final int end;
 
@@ -51,7 +51,7 @@ public class SpanFirstQueryBuilder extends AbstractQueryBuilder<SpanFirstQueryBu
      * @param end maximum end position of the match, needs to be positive
      * @throws IllegalArgumentException for negative <code>end</code> positions
      */
-    public SpanFirstQueryBuilder(SpanQueryBuilder<?> matchBuilder, int end) {
+    public SpanFirstQueryBuilder(SpanQueryBuilder matchBuilder, int end) {
         if (matchBuilder == null) {
             throw new IllegalArgumentException("inner span query cannot be null");
         }
@@ -67,7 +67,7 @@ public class SpanFirstQueryBuilder extends AbstractQueryBuilder<SpanFirstQueryBu
      */
     public SpanFirstQueryBuilder(StreamInput in) throws IOException {
         super(in);
-        matchBuilder = (SpanQueryBuilder<?>) in.readNamedWriteable(QueryBuilder.class);
+        matchBuilder = (SpanQueryBuilder) in.readNamedWriteable(QueryBuilder.class);
         end = in.readInt();
     }
 
@@ -80,7 +80,7 @@ public class SpanFirstQueryBuilder extends AbstractQueryBuilder<SpanFirstQueryBu
     /**
      * @return the inner {@link SpanQueryBuilder} defined in this query
      */
-    public SpanQueryBuilder<?> innerQuery() {
+    public SpanQueryBuilder innerQuery() {
         return this.matchBuilder;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java
index 46f639f5f70..86418903f70 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java
@@ -39,16 +39,16 @@ import java.util.Objects;
  * as a {@link SpanQueryBuilder} so it can be nested.
  */
 public class SpanMultiTermQueryBuilder extends AbstractQueryBuilder<SpanMultiTermQueryBuilder>
-        implements SpanQueryBuilder<SpanMultiTermQueryBuilder> {
+        implements SpanQueryBuilder {
 
     public static final String NAME = "span_multi";
     public static final ParseField QUERY_NAME_FIELD = new ParseField(NAME);
 
     private static final ParseField MATCH_FIELD = new ParseField("match");
 
-    private final MultiTermQueryBuilder<?> multiTermQueryBuilder;
+    private final MultiTermQueryBuilder multiTermQueryBuilder;
 
-    public SpanMultiTermQueryBuilder(MultiTermQueryBuilder<?> multiTermQueryBuilder) {
+    public SpanMultiTermQueryBuilder(MultiTermQueryBuilder multiTermQueryBuilder) {
         if (multiTermQueryBuilder == null) {
             throw new IllegalArgumentException("inner multi term query cannot be null");
         }
@@ -60,7 +60,7 @@ public class SpanMultiTermQueryBuilder extends AbstractQueryBuilder<SpanMultiTer
      */
     public SpanMultiTermQueryBuilder(StreamInput in) throws IOException {
         super(in);
-        multiTermQueryBuilder = (MultiTermQueryBuilder<?>) in.readNamedWriteable(QueryBuilder.class);
+        multiTermQueryBuilder = (MultiTermQueryBuilder) in.readNamedWriteable(QueryBuilder.class);
     }
 
     @Override
@@ -68,7 +68,7 @@ public class SpanMultiTermQueryBuilder extends AbstractQueryBuilder<SpanMultiTer
         out.writeNamedWriteable(multiTermQueryBuilder);
     }
 
-    public MultiTermQueryBuilder<?> innerQuery() {
+    public MultiTermQueryBuilder innerQuery() {
         return this.multiTermQueryBuilder;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
index 1bd8e85b902..a503b708633 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
@@ -39,7 +39,7 @@ import java.util.Objects;
  * of intervening unmatched positions, as well as whether matches are required to be in-order.
  * The span near query maps to Lucene {@link SpanNearQuery}.
  */
-public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuilder> implements SpanQueryBuilder<SpanNearQueryBuilder> {
+public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuilder> implements SpanQueryBuilder {
 
     public static final String NAME = "span_near";
     public static final ParseField QUERY_NAME_FIELD = new ParseField(NAME);
@@ -52,7 +52,7 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
     private static final ParseField CLAUSES_FIELD = new ParseField("clauses");
     private static final ParseField IN_ORDER_FIELD = new ParseField("in_order");
 
-    private final List<SpanQueryBuilder<?>> clauses = new ArrayList<>();
+    private final List<SpanQueryBuilder> clauses = new ArrayList<>();
 
     private final int slop;
 
@@ -62,7 +62,7 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
      * @param initialClause an initial span query clause
      * @param slop controls the maximum number of intervening unmatched positions permitted
      */
-    public SpanNearQueryBuilder(SpanQueryBuilder<?> initialClause, int slop) {
+    public SpanNearQueryBuilder(SpanQueryBuilder initialClause, int slop) {
         if (initialClause == null) {
             throw new IllegalArgumentException("query must include at least one clause");
         }
@@ -75,8 +75,8 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
      */
     public SpanNearQueryBuilder(StreamInput in) throws IOException {
         super(in);
-        for (QueryBuilder<?> clause : readQueries(in)) {
-            this.clauses.add((SpanQueryBuilder<?>) clause);
+        for (QueryBuilder clause : readQueries(in)) {
+            this.clauses.add((SpanQueryBuilder) clause);
         }
         slop = in.readVInt();
         inOrder = in.readBoolean();
@@ -96,7 +96,7 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
         return this.slop;
     }
 
-    public SpanNearQueryBuilder clause(SpanQueryBuilder<?> clause) {
+    public SpanNearQueryBuilder clause(SpanQueryBuilder clause) {
         if (clause == null) {
             throw new IllegalArgumentException("query clauses cannot be null");
         }
@@ -107,7 +107,7 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
     /**
      * @return the {@link SpanQueryBuilder} clauses that were set for this query
      */
-    public List<SpanQueryBuilder<?>> clauses() {
+    public List<SpanQueryBuilder> clauses() {
         return this.clauses;
     }
 
@@ -132,7 +132,7 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
     protected void doXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(NAME);
         builder.startArray(CLAUSES_FIELD.getPreferredName());
-        for (SpanQueryBuilder<?> clause : clauses) {
+        for (SpanQueryBuilder clause : clauses) {
             clause.toXContent(builder, params);
         }
         builder.endArray();
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java
index aa2388dd89a..02ce431de1e 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java
@@ -32,7 +32,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import java.io.IOException;
 import java.util.Objects;
 
-public class SpanNotQueryBuilder extends AbstractQueryBuilder<SpanNotQueryBuilder> implements SpanQueryBuilder<SpanNotQueryBuilder> {
+public class SpanNotQueryBuilder extends AbstractQueryBuilder<SpanNotQueryBuilder> implements SpanQueryBuilder {
 
     public static final String NAME = "span_not";
     public static final ParseField QUERY_NAME_FIELD = new ParseField(NAME);
@@ -48,9 +48,9 @@ public class SpanNotQueryBuilder extends AbstractQueryBuilder<SpanNotQueryBuilde
     private static final ParseField EXCLUDE_FIELD = new ParseField("exclude");
     private static final ParseField INCLUDE_FIELD = new ParseField("include");
 
-    private final SpanQueryBuilder<?> include;
+    private final SpanQueryBuilder include;
 
-    private final SpanQueryBuilder<?> exclude;
+    private final SpanQueryBuilder exclude;
 
     private int pre = DEFAULT_PRE;
 
@@ -62,7 +62,7 @@ public class SpanNotQueryBuilder extends AbstractQueryBuilder<SpanNotQueryBuilde
      * @param include the span query whose matches are filtered
      * @param exclude the span query whose matches must not overlap
      */
-    public SpanNotQueryBuilder(SpanQueryBuilder<?> include, SpanQueryBuilder<?> exclude) {
+    public SpanNotQueryBuilder(SpanQueryBuilder include, SpanQueryBuilder exclude) {
         if (include == null) {
             throw new IllegalArgumentException("inner clause [include] cannot be null.");
         }
@@ -78,8 +78,8 @@ public class SpanNotQueryBuilder extends AbstractQueryBuilder<SpanNotQueryBuilde
      */
     public SpanNotQueryBuilder(StreamInput in) throws IOException {
         super(in);
-        include = (SpanQueryBuilder<?>) in.readNamedWriteable(QueryBuilder.class);
-        exclude = (SpanQueryBuilder<?>) in.readNamedWriteable(QueryBuilder.class);
+        include = (SpanQueryBuilder) in.readNamedWriteable(QueryBuilder.class);
+        exclude = (SpanQueryBuilder) in.readNamedWriteable(QueryBuilder.class);
         pre = in.readVInt();
         post = in.readVInt();
     }
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java
index 71b6b74b828..2a67a84be81 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java
@@ -37,16 +37,16 @@ import java.util.Objects;
 /**
  * Span query that matches the union of its clauses. Maps to {@link SpanOrQuery}.
  */
-public class SpanOrQueryBuilder extends AbstractQueryBuilder<SpanOrQueryBuilder> implements SpanQueryBuilder<SpanOrQueryBuilder> {
+public class SpanOrQueryBuilder extends AbstractQueryBuilder<SpanOrQueryBuilder> implements SpanQueryBuilder {
 
     public static final String NAME = "span_or";
     public static final ParseField QUERY_NAME_FIELD = new ParseField(NAME);
 
     private static final ParseField CLAUSES_FIELD = new ParseField("clauses");
 
-    private final List<SpanQueryBuilder<?>> clauses = new ArrayList<>();
+    private final List<SpanQueryBuilder> clauses = new ArrayList<>();
 
-    public SpanOrQueryBuilder(SpanQueryBuilder<?> initialClause) {
+    public SpanOrQueryBuilder(SpanQueryBuilder initialClause) {
         if (initialClause == null) {
             throw new IllegalArgumentException("query must include at least one clause");
         }
@@ -58,8 +58,8 @@ public class SpanOrQueryBuilder extends AbstractQueryBuilder<SpanOrQueryBuilder>
      */
     public SpanOrQueryBuilder(StreamInput in) throws IOException {
         super(in);
-        for (QueryBuilder<?> clause: readQueries(in)) {
-            clauses.add((SpanQueryBuilder<?>) clause);
+        for (QueryBuilder clause: readQueries(in)) {
+            clauses.add((SpanQueryBuilder) clause);
         }
     }
 
@@ -68,7 +68,7 @@ public class SpanOrQueryBuilder extends AbstractQueryBuilder<SpanOrQueryBuilder>
         writeQueries(out, clauses);
     }
 
-    public SpanOrQueryBuilder clause(SpanQueryBuilder<?> clause) {
+    public SpanOrQueryBuilder clause(SpanQueryBuilder clause) {
         if (clause == null) {
             throw new IllegalArgumentException("inner bool query clause cannot be null");
         }
@@ -79,7 +79,7 @@ public class SpanOrQueryBuilder extends AbstractQueryBuilder<SpanOrQueryBuilder>
     /**
      * @return the {@link SpanQueryBuilder} clauses that were set for this query
      */
-    public List<SpanQueryBuilder<?>> clauses() {
+    public List<SpanQueryBuilder> clauses() {
         return this.clauses;
     }
 
@@ -87,7 +87,7 @@ public class SpanOrQueryBuilder extends AbstractQueryBuilder<SpanOrQueryBuilder>
     protected void doXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(NAME);
         builder.startArray(CLAUSES_FIELD.getPreferredName());
-        for (SpanQueryBuilder<?> clause : clauses) {
+        for (SpanQueryBuilder clause : clauses) {
             clause.toXContent(builder, params);
         }
         builder.endArray();
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanQueryBuilder.java
index 90a75a5af1b..fec1cac2696 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanQueryBuilder.java
@@ -22,6 +22,6 @@ package org.elasticsearch.index.query;
 /**
  * Marker interface for a specific type of {@link QueryBuilder} that allows to build span queries
  */
-public interface SpanQueryBuilder<QB extends SpanQueryBuilder<QB>> extends QueryBuilder<QB> {
+public interface SpanQueryBuilder extends QueryBuilder {
 
 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java
index a4bafc5001c..3bb374ff276 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java
@@ -36,7 +36,7 @@ import java.io.IOException;
  * A Span Query that matches documents containing a term.
  * @see SpanTermQuery
  */
-public class SpanTermQueryBuilder extends BaseTermQueryBuilder<SpanTermQueryBuilder> implements SpanQueryBuilder<SpanTermQueryBuilder> {
+public class SpanTermQueryBuilder extends BaseTermQueryBuilder<SpanTermQueryBuilder> implements SpanQueryBuilder {
 
     public static final String NAME = "span_term";
     public static final ParseField QUERY_NAME_FIELD = new ParseField(NAME);
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java
index cf430881ab9..eaedc80bab4 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java
@@ -36,7 +36,7 @@ import java.util.Objects;
  * Builder for {@link org.apache.lucene.search.spans.SpanWithinQuery}.
  */
 public class SpanWithinQueryBuilder extends AbstractQueryBuilder<SpanWithinQueryBuilder>
-        implements SpanQueryBuilder<SpanWithinQueryBuilder> {
+        implements SpanQueryBuilder {
 
     public static final String NAME = "span_within";
     public static final ParseField QUERY_NAME_FIELD = new ParseField(NAME);
@@ -44,15 +44,15 @@ public class SpanWithinQueryBuilder extends AbstractQueryBuilder<SpanWithinQuery
     private static final ParseField BIG_FIELD = new ParseField("big");
     private static final ParseField LITTLE_FIELD = new ParseField("little");
 
-    private final SpanQueryBuilder<?> big;
-    private final SpanQueryBuilder<?> little;
+    private final SpanQueryBuilder big;
+    private final SpanQueryBuilder little;
 
     /**
      * Query that returns spans from <code>little</code> that are contained in a spans from <code>big</code>.
      * @param big clause that must enclose {@code little} for a match.
      * @param little the little clause, it must be contained within {@code big} for a match.
      */
-    public SpanWithinQueryBuilder(SpanQueryBuilder<?> big, SpanQueryBuilder<?> little) {
+    public SpanWithinQueryBuilder(SpanQueryBuilder big, SpanQueryBuilder little) {
         if (big == null) {
             throw new IllegalArgumentException("inner clause [big] cannot be null.");
         }
@@ -68,8 +68,8 @@ public class SpanWithinQueryBuilder extends AbstractQueryBuilder<SpanWithinQuery
      */
     public SpanWithinQueryBuilder(StreamInput in) throws IOException {
         super(in);
-        big = (SpanQueryBuilder<?>) in.readNamedWriteable(QueryBuilder.class);
-        little = (SpanQueryBuilder<?>) in.readNamedWriteable(QueryBuilder.class);
+        big = (SpanQueryBuilder) in.readNamedWriteable(QueryBuilder.class);
+        little = (SpanQueryBuilder) in.readNamedWriteable(QueryBuilder.class);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java
index 80b3f2b1404..1a4d05374b3 100644
--- a/core/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java
@@ -173,13 +173,13 @@ public class TemplateQueryBuilder extends AbstractQueryBuilder<TemplateQueryBuil
     }
 
     @Override
-    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+    protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
         ExecutableScript executable = queryRewriteContext.getScriptService().executable(template,
             ScriptContext.Standard.SEARCH, Collections.emptyMap(), queryRewriteContext.getClusterState());
         BytesReference querySource = (BytesReference) executable.run();
         try (XContentParser qSourceParser = XContentFactory.xContent(querySource).createParser(querySource)) {
             final QueryParseContext queryParseContext = queryRewriteContext.newParseContext(qSourceParser);
-            final QueryBuilder<?> queryBuilder = queryParseContext.parseInnerQueryBuilder();
+            final QueryBuilder queryBuilder = queryParseContext.parseInnerQueryBuilder();
             if (boost() != DEFAULT_BOOST || queryName() != null) {
                 final BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
                 boolQueryBuilder.must(queryBuilder);
diff --git a/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
index 084f2e766d4..5b0650eb7e4 100644
--- a/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
@@ -381,7 +381,7 @@ public class TermsQueryBuilder extends AbstractQueryBuilder<TermsQueryBuilder> {
     }
 
     @Override
-    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+    protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
         if (this.termsLookup != null) {
             TermsLookup termsLookup = new TermsLookup(this.termsLookup);
             if (termsLookup.index() == null) { // TODO this should go away?
diff --git a/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java
index bf6921e9452..ef49472b8cc 100644
--- a/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java
@@ -46,7 +46,7 @@ import java.util.Objects;
  * <tt>?</tt>.
  */
 public class WildcardQueryBuilder extends AbstractQueryBuilder<WildcardQueryBuilder>
-        implements MultiTermQueryBuilder<WildcardQueryBuilder> {
+        implements MultiTermQueryBuilder {
 
     public static final String NAME = "wildcard";
     public static final ParseField QUERY_NAME_FIELD = new ParseField(NAME);
diff --git a/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java
index 4af7081f308..4037666393d 100644
--- a/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java
@@ -160,11 +160,11 @@ public class WrapperQueryBuilder extends AbstractQueryBuilder<WrapperQueryBuilde
     }
 
     @Override
-    protected QueryBuilder<?> doRewrite(QueryRewriteContext context) throws IOException {
+    protected QueryBuilder doRewrite(QueryRewriteContext context) throws IOException {
         try (XContentParser qSourceParser = XContentFactory.xContent(source).createParser(source)) {
             QueryParseContext parseContext = context.newParseContext(qSourceParser);
 
-            final QueryBuilder<?> queryBuilder = parseContext.parseInnerQueryBuilder();
+            final QueryBuilder queryBuilder = parseContext.parseInnerQueryBuilder();
             if (boost() != DEFAULT_BOOST || queryName() != null) {
                 final BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
                 boolQueryBuilder.must(queryBuilder);
diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java
index 0dffa6d714a..98f878d1c2e 100644
--- a/core/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java
@@ -75,7 +75,7 @@ public class FunctionScoreQueryBuilder extends AbstractQueryBuilder<FunctionScor
     public static final CombineFunction DEFAULT_BOOST_MODE = CombineFunction.MULTIPLY;
     public static final FiltersFunctionScoreQuery.ScoreMode DEFAULT_SCORE_MODE = FiltersFunctionScoreQuery.ScoreMode.MULTIPLY;
 
-    private final QueryBuilder<?> query;
+    private final QueryBuilder query;
 
     private float maxBoost = FunctionScoreQuery.DEFAULT_MAX_BOOST;
 
@@ -92,7 +92,7 @@ public class FunctionScoreQueryBuilder extends AbstractQueryBuilder<FunctionScor
      *
      * @param query the query that needs to be custom scored
      */
-    public FunctionScoreQueryBuilder(QueryBuilder<?> query) {
+    public FunctionScoreQueryBuilder(QueryBuilder query) {
         this(query, new FilterFunctionBuilder[0]);
     }
 
@@ -120,7 +120,7 @@ public class FunctionScoreQueryBuilder extends AbstractQueryBuilder<FunctionScor
      * @param query the query to custom score
      * @param scoreFunctionBuilder score function that is executed
      */
-    public FunctionScoreQueryBuilder(QueryBuilder<?> query, ScoreFunctionBuilder<?> scoreFunctionBuilder) {
+    public FunctionScoreQueryBuilder(QueryBuilder query, ScoreFunctionBuilder<?> scoreFunctionBuilder) {
         this(query, new FilterFunctionBuilder[]{new FilterFunctionBuilder(scoreFunctionBuilder)});
     }
 
@@ -130,7 +130,7 @@ public class FunctionScoreQueryBuilder extends AbstractQueryBuilder<FunctionScor
      * @param query the query that defines which documents the function_score query will be executed on.
      * @param filterFunctionBuilders the filters and functions
      */
-    public FunctionScoreQueryBuilder(QueryBuilder<?> query, FilterFunctionBuilder[] filterFunctionBuilders) {
+    public FunctionScoreQueryBuilder(QueryBuilder query, FilterFunctionBuilder[] filterFunctionBuilders) {
         if (query == null) {
             throw new IllegalArgumentException("function_score: query must not be null");
         }
@@ -172,7 +172,7 @@ public class FunctionScoreQueryBuilder extends AbstractQueryBuilder<FunctionScor
     /**
      * Returns the query that defines which documents the function_score query will be executed on.
      */
-    public QueryBuilder<?> query() {
+    public QueryBuilder query() {
         return this.query;
     }
 
@@ -334,14 +334,14 @@ public class FunctionScoreQueryBuilder extends AbstractQueryBuilder<FunctionScor
      * that match the given filter.
      */
     public static class FilterFunctionBuilder implements ToXContent, Writeable {
-        private final QueryBuilder<?> filter;
+        private final QueryBuilder filter;
         private final ScoreFunctionBuilder<?> scoreFunction;
 
         public FilterFunctionBuilder(ScoreFunctionBuilder<?> scoreFunctionBuilder) {
             this(new MatchAllQueryBuilder(), scoreFunctionBuilder);
         }
 
-        public FilterFunctionBuilder(QueryBuilder<?> filter, ScoreFunctionBuilder<?> scoreFunction) {
+        public FilterFunctionBuilder(QueryBuilder filter, ScoreFunctionBuilder<?> scoreFunction) {
             if (filter == null) {
                 throw new IllegalArgumentException("function_score: filter must not be null");
             }
@@ -366,7 +366,7 @@ public class FunctionScoreQueryBuilder extends AbstractQueryBuilder<FunctionScor
             out.writeNamedWriteable(scoreFunction);
         }
 
-        public QueryBuilder<?> getFilter() {
+        public QueryBuilder getFilter() {
             return filter;
         }
 
@@ -403,7 +403,7 @@ public class FunctionScoreQueryBuilder extends AbstractQueryBuilder<FunctionScor
         }
 
         public FilterFunctionBuilder rewrite(QueryRewriteContext context) throws IOException {
-            QueryBuilder<?> rewrite = filter.rewrite(context);
+            QueryBuilder rewrite = filter.rewrite(context);
             if (rewrite != filter) {
                 return new FilterFunctionBuilder(rewrite, scoreFunction);
             }
@@ -412,8 +412,8 @@ public class FunctionScoreQueryBuilder extends AbstractQueryBuilder<FunctionScor
     }
 
     @Override
-    protected QueryBuilder<?> doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
-        QueryBuilder<?> queryBuilder = this.query.rewrite(queryRewriteContext);
+    protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+        QueryBuilder queryBuilder = this.query.rewrite(queryRewriteContext);
         FilterFunctionBuilder[] rewrittenBuilders = new FilterFunctionBuilder[this.filterFunctionBuilders.length];
         boolean rewritten = false;
         for (int i = 0; i < rewrittenBuilders.length; i++) {
@@ -442,7 +442,7 @@ public class FunctionScoreQueryBuilder extends AbstractQueryBuilder<FunctionScor
                                                          QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
-        QueryBuilder<?> query = null;
+        QueryBuilder query = null;
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
         String queryName = null;
 
@@ -571,7 +571,7 @@ public class FunctionScoreQueryBuilder extends AbstractQueryBuilder<FunctionScor
         XContentParser.Token token;
         XContentParser parser = parseContext.parser();
         while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-            QueryBuilder<?> filter = null;
+            QueryBuilder filter = null;
             ScoreFunctionBuilder<?> scoreFunction = null;
             Float functionWeight = null;
             if (token != XContentParser.Token.START_OBJECT) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/query/RestValidateQueryAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/query/RestValidateQueryAction.java
index 4ff7234b2ef..6ac71708e32 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/query/RestValidateQueryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/query/RestValidateQueryAction.java
@@ -82,7 +82,7 @@ public class RestValidateQueryAction extends BaseRestHandler {
                 return;
             }
         } else {
-            QueryBuilder<?> queryBuilder = RestActions.urlParamsToQueryBuilder(request);
+            QueryBuilder queryBuilder = RestActions.urlParamsToQueryBuilder(request);
             if (queryBuilder != null) {
                 validateQueryRequest.query(queryBuilder);
             }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java
index 6fd64430c20..46e8fadd05a 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java
@@ -68,7 +68,7 @@ public class RestCountAction extends AbstractCatAction {
         if (source != null) {
             searchSourceBuilder.query(RestActions.getQueryContent(new BytesArray(source), indicesQueriesRegistry, parseFieldMatcher));
         } else {
-            QueryBuilder<?> queryBuilder = RestActions.urlParamsToQueryBuilder(request);
+            QueryBuilder queryBuilder = RestActions.urlParamsToQueryBuilder(request);
             if (queryBuilder != null) {
                 searchSourceBuilder.query(queryBuilder);
             }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/count/RestCountAction.java b/core/src/main/java/org/elasticsearch/rest/action/count/RestCountAction.java
index c423f7a8537..71e5832071c 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/count/RestCountAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/count/RestCountAction.java
@@ -74,7 +74,7 @@ public class RestCountAction extends BaseRestHandler {
             BytesReference restContent = RestActions.getRestContent(request);
             searchSourceBuilder.query(RestActions.getQueryContent(restContent, indicesQueriesRegistry, parseFieldMatcher));
         } else {
-            QueryBuilder<?> queryBuilder = RestActions.urlParamsToQueryBuilder(request);
+            QueryBuilder queryBuilder = RestActions.urlParamsToQueryBuilder(request);
             if (queryBuilder != null) {
                 searchSourceBuilder.query(queryBuilder);
             }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/explain/RestExplainAction.java b/core/src/main/java/org/elasticsearch/rest/action/explain/RestExplainAction.java
index a1fdc13437c..47f5e7da19b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/explain/RestExplainAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/explain/RestExplainAction.java
@@ -74,7 +74,7 @@ public class RestExplainAction extends BaseRestHandler {
             BytesReference restContent = RestActions.getRestContent(request);
             explainRequest.query(RestActions.getQueryContent(restContent, indicesQueriesRegistry, parseFieldMatcher));
         } else if (queryString != null) {
-            QueryBuilder<?> query = RestActions.urlParamsToQueryBuilder(request);
+            QueryBuilder query = RestActions.urlParamsToQueryBuilder(request);
             explainRequest.query(query);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
index 5a65c8df116..8d6b003779b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
@@ -162,7 +162,7 @@ public class RestSearchAction extends BaseRestHandler {
      * values that are not overridden by the rest request.
      */
     private static void parseSearchSource(final SearchSourceBuilder searchSourceBuilder, RestRequest request) {
-        QueryBuilder<?> queryBuilder = RestActions.urlParamsToQueryBuilder(request);
+        QueryBuilder queryBuilder = RestActions.urlParamsToQueryBuilder(request);
         if (queryBuilder != null) {
             searchSourceBuilder.query(queryBuilder);
         }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/support/RestActions.java b/core/src/main/java/org/elasticsearch/rest/action/support/RestActions.java
index 9ab523ba1e6..652cb8c61e9 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/support/RestActions.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/support/RestActions.java
@@ -93,7 +93,7 @@ public class RestActions {
         builder.endObject();
     }
 
-    public static QueryBuilder<?> urlParamsToQueryBuilder(RestRequest request) {
+    public static QueryBuilder urlParamsToQueryBuilder(RestRequest request) {
         String queryString = request.param("q");
         if (queryString == null) {
             return null;
@@ -130,7 +130,7 @@ public class RestActions {
         return content;
     }
 
-    public static QueryBuilder<?> getQueryContent(BytesReference source, IndicesQueriesRegistry indicesQueriesRegistry, ParseFieldMatcher parseFieldMatcher) {
+    public static QueryBuilder getQueryContent(BytesReference source, IndicesQueriesRegistry indicesQueriesRegistry, ParseFieldMatcher parseFieldMatcher) {
         try (XContentParser requestParser = XContentFactory.xContent(source).createParser(source)) {
             QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, requestParser, parseFieldMatcher);
             return context.parseTopLevelQueryBuilder();
diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java
index 3429b1ba18b..e0a6ccd09b4 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -359,7 +359,7 @@ public class SearchModule extends AbstractModule {
      *        is the name by under which the reader is registered. So it is the name that the query should use as its
      *        {@link NamedWriteable#getWriteableName()} too.
      */
-    public <QB extends QueryBuilder<QB>> void registerQuery(Writeable.Reader<QB> reader, QueryParser<QB> queryParser,
+    public <QB extends QueryBuilder> void registerQuery(Writeable.Reader<QB> reader, QueryParser<QB> queryParser,
                                                                          ParseField queryName) {
         queryParserRegistry.register(queryParser, queryName);
         namedWriteableRegistry.register(QueryBuilder.class, queryName.getPreferredName(), reader);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java
index 1da7c28cf09..23fa8007d5b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java
@@ -142,7 +142,7 @@ public class AggregationBuilders {
     /**
      * Create a new {@link Filter} aggregation with the given name.
      */
-    public static FilterAggregatorBuilder filter(String name, QueryBuilder<?> filter) {
+    public static FilterAggregatorBuilder filter(String name, QueryBuilder filter) {
         return new FilterAggregatorBuilder(name, filter);
     }
 
@@ -156,7 +156,7 @@ public class AggregationBuilders {
     /**
      * Create a new {@link Filters} aggregation with the given name.
      */
-    public static FiltersAggregatorBuilder filters(String name, QueryBuilder<?>... filters) {
+    public static FiltersAggregatorBuilder filters(String name, QueryBuilder... filters) {
         return new FiltersAggregatorBuilder(name, filters);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorBuilder.java
index 1a0c6decdf9..9024e902d7d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorBuilder.java
@@ -41,7 +41,7 @@ public class FilterAggregatorBuilder extends AggregatorBuilder<FilterAggregatorB
     public static final String NAME = InternalFilter.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
-    private final QueryBuilder<?> filter;
+    private final QueryBuilder filter;
 
     /**
      * @param name
@@ -51,7 +51,7 @@ public class FilterAggregatorBuilder extends AggregatorBuilder<FilterAggregatorB
      *            filter will fall into the bucket defined by this
      *            {@link Filter} aggregation.
      */
-    public FilterAggregatorBuilder(String name, QueryBuilder<?> filter) {
+    public FilterAggregatorBuilder(String name, QueryBuilder filter) {
         super(name, InternalFilter.TYPE);
         if (filter == null) {
             throw new IllegalArgumentException("[filter] must not be null: [" + name + "]");
@@ -92,7 +92,7 @@ public class FilterAggregatorBuilder extends AggregatorBuilder<FilterAggregatorB
 
     public static FilterAggregatorBuilder parse(String aggregationName, QueryParseContext context)
             throws IOException {
-        QueryBuilder<?> filter = context.parseInnerQueryBuilder();
+        QueryBuilder filter = context.parseInnerQueryBuilder();
 
         if (filter == null) {
             throw new ParsingException(null, "filter cannot be null in filter aggregation [{}]", aggregationName);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorFactory.java
index 13b1e62b338..212494ef48b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorFactory.java
@@ -38,7 +38,7 @@ public class FilterAggregatorFactory extends AggregatorFactory<FilterAggregatorF
 
     private final Weight weight;
 
-    public FilterAggregatorFactory(String name, Type type, QueryBuilder<?> filterBuilder, AggregationContext context,
+    public FilterAggregatorFactory(String name, Type type, QueryBuilder filterBuilder, AggregationContext context,
             AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
         super(name, type, context, parent, subFactoriesBuilder, metaData);
         IndexSearcher contextSearcher = context.searchContext().searcher();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregator.java
index ca071d7cfc9..a5ce89c4666 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregator.java
@@ -59,9 +59,9 @@ public class FiltersAggregator extends BucketsAggregator {
 
     public static class KeyedFilter implements Writeable, ToXContent {
         private final String key;
-        private final QueryBuilder<?> filter;
+        private final QueryBuilder filter;
 
-        public KeyedFilter(String key, QueryBuilder<?> filter) {
+        public KeyedFilter(String key, QueryBuilder filter) {
             if (key == null) {
                 throw new IllegalArgumentException("[key] must not be null");
             }
@@ -94,7 +94,7 @@ public class FiltersAggregator extends BucketsAggregator {
             return key;
         }
 
-        public QueryBuilder<?> filter() {
+        public QueryBuilder filter() {
             return filter;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregatorBuilder.java
index ca9f7f6c7c3..52a1b949b4b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregatorBuilder.java
@@ -80,7 +80,7 @@ public class FiltersAggregatorBuilder extends AggregatorBuilder<FiltersAggregato
      * @param filters
      *            the filters to use with this aggregation
      */
-    public FiltersAggregatorBuilder(String name, QueryBuilder<?>... filters) {
+    public FiltersAggregatorBuilder(String name, QueryBuilder... filters) {
         super(name, InternalFilters.TYPE);
         List<KeyedFilter> keyedFilters = new ArrayList<>(filters.length);
         for (int i = 0; i < filters.length; i++) {
@@ -204,7 +204,7 @@ public class FiltersAggregatorBuilder extends AggregatorBuilder<FiltersAggregato
         XContentParser parser = context.parser();
 
         List<FiltersAggregator.KeyedFilter> keyedFilters = null;
-        List<QueryBuilder<?>> nonKeyedFilters = null;
+        List<QueryBuilder> nonKeyedFilters = null;
 
         XContentParser.Token token = null;
         String currentFieldName = null;
@@ -235,7 +235,7 @@ public class FiltersAggregatorBuilder extends AggregatorBuilder<FiltersAggregato
                         if (token == XContentParser.Token.FIELD_NAME) {
                             key = parser.currentName();
                         } else {
-                            QueryBuilder<?> filter = context.parseInnerQueryBuilder();
+                            QueryBuilder filter = context.parseInnerQueryBuilder();
                             keyedFilters.add(new FiltersAggregator.KeyedFilter(key, filter == null ? matchAllQuery() : filter));
                         }
                     }
@@ -247,7 +247,7 @@ public class FiltersAggregatorBuilder extends AggregatorBuilder<FiltersAggregato
                 if (context.getParseFieldMatcher().match(currentFieldName, FILTERS_FIELD)) {
                     nonKeyedFilters = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                        QueryBuilder<?> filter = context.parseInnerQueryBuilder();
+                        QueryBuilder filter = context.parseInnerQueryBuilder();
                         nonKeyedFilters.add(filter == null ? QueryBuilders.matchAllQuery() : filter);
                     }
                 } else {
@@ -270,7 +270,7 @@ public class FiltersAggregatorBuilder extends AggregatorBuilder<FiltersAggregato
                     keyedFilters.toArray(new FiltersAggregator.KeyedFilter[keyedFilters.size()]));
         } else {
             factory = new FiltersAggregatorBuilder(aggregationName,
-                    nonKeyedFilters.toArray(new QueryBuilder<?>[nonKeyedFilters.size()]));
+                    nonKeyedFilters.toArray(new QueryBuilder[nonKeyedFilters.size()]));
         }
         if (otherBucket != null) {
             factory.otherBucket(otherBucket);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorBuilder.java
index 373d757a22c..55b85bef0d9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorBuilder.java
@@ -58,7 +58,7 @@ public class SignificantTermsAggregatorBuilder extends ValuesSourceAggregatorBui
 
     private IncludeExclude includeExclude = null;
     private String executionHint = null;
-    private QueryBuilder<?> filterBuilder = null;
+    private QueryBuilder filterBuilder = null;
     private TermsAggregator.BucketCountThresholds bucketCountThresholds = new BucketCountThresholds(DEFAULT_BUCKET_COUNT_THRESHOLDS);
     private SignificanceHeuristic significanceHeuristic = DEFAULT_SIGNIFICANCE_HEURISTIC;
 
@@ -176,7 +176,7 @@ public class SignificantTermsAggregatorBuilder extends ValuesSourceAggregatorBui
         return executionHint;
     }
 
-    public SignificantTermsAggregatorBuilder backgroundFilter(QueryBuilder<?> backgroundFilter) {
+    public SignificantTermsAggregatorBuilder backgroundFilter(QueryBuilder backgroundFilter) {
         if (backgroundFilter == null) {
             throw new IllegalArgumentException("[backgroundFilter] must not be null: [" + name + "]");
         }
@@ -184,7 +184,7 @@ public class SignificantTermsAggregatorBuilder extends ValuesSourceAggregatorBui
         return this;
     }
 
-    public QueryBuilder<?> backgroundFilter() {
+    public QueryBuilder backgroundFilter() {
         return filterBuilder;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
index 0f0a37d909a..ce3f83079ea 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
@@ -65,12 +65,12 @@ public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFac
     private MappedFieldType fieldType;
     private FilterableTermsEnum termsEnum;
     private int numberOfAggregatorsCreated;
-    private final QueryBuilder<?> filterBuilder;
+    private final QueryBuilder filterBuilder;
     private final TermsAggregator.BucketCountThresholds bucketCountThresholds;
     private final SignificanceHeuristic significanceHeuristic;
 
     public SignificantTermsAggregatorFactory(String name, Type type, ValuesSourceConfig<ValuesSource> config, IncludeExclude includeExclude,
-            String executionHint, QueryBuilder<?> filterBuilder, TermsAggregator.BucketCountThresholds bucketCountThresholds,
+            String executionHint, QueryBuilder filterBuilder, TermsAggregator.BucketCountThresholds bucketCountThresholds,
             SignificanceHeuristic significanceHeuristic, AggregationContext context, AggregatorFactory<?> parent,
             AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
         super(name, type, config, context, parent, subFactoriesBuilder, metaData);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParser.java
index b6549c6dc29..60805bea692 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParser.java
@@ -66,7 +66,7 @@ public class SignificantTermsParser extends AbstractTermsParser {
         if (incExc != null) {
             factory.includeExclude(incExc);
         }
-        QueryBuilder<?> backgroundFilter = (QueryBuilder<?>) otherOptions.get(SignificantTermsAggregatorBuilder.BACKGROUND_FILTER);
+        QueryBuilder backgroundFilter = (QueryBuilder) otherOptions.get(SignificantTermsAggregatorBuilder.BACKGROUND_FILTER);
         if (backgroundFilter != null) {
             factory.backgroundFilter(backgroundFilter);
         }
@@ -89,7 +89,7 @@ public class SignificantTermsParser extends AbstractTermsParser {
                 return true;
             } else if (parseFieldMatcher.match(currentFieldName, SignificantTermsAggregatorBuilder.BACKGROUND_FILTER)) {
                 QueryParseContext queryParseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-                QueryBuilder<?> filter = queryParseContext.parseInnerQueryBuilder();
+                QueryBuilder filter = queryParseContext.parseInnerQueryBuilder();
                 otherOptions.put(SignificantTermsAggregatorBuilder.BACKGROUND_FILTER, filter);
                 return true;
             }
diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index b656df4e154..61f4acb81aa 100644
--- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -120,9 +120,9 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         return new HighlightBuilder();
     }
 
-    private QueryBuilder<?> queryBuilder;
+    private QueryBuilder queryBuilder;
 
-    private QueryBuilder<?> postQueryBuilder;
+    private QueryBuilder postQueryBuilder;
 
     private int from = -1;
 
@@ -371,7 +371,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
      *
      * @see org.elasticsearch.index.query.QueryBuilders
      */
-    public SearchSourceBuilder query(QueryBuilder<?> query) {
+    public SearchSourceBuilder query(QueryBuilder query) {
         this.queryBuilder = query;
         return this;
     }
@@ -379,7 +379,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
     /**
      * Gets the query for this request
      */
-    public QueryBuilder<?> query() {
+    public QueryBuilder query() {
         return queryBuilder;
     }
 
@@ -388,7 +388,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
      * only has affect on the search hits (not aggregations). This filter is
      * always executed as last filtering mechanism.
      */
-    public SearchSourceBuilder postFilter(QueryBuilder<?> postFilter) {
+    public SearchSourceBuilder postFilter(QueryBuilder postFilter) {
         this.postQueryBuilder = postFilter;
         return this;
     }
@@ -396,7 +396,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
     /**
      * Gets the post filter for this request
      */
-    public QueryBuilder<?> postFilter() {
+    public QueryBuilder postFilter() {
         return postQueryBuilder;
     }
 
@@ -910,11 +910,11 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
      */
     public SearchSourceBuilder rewrite(QueryShardContext context) throws IOException {
         assert (this.equals(shallowCopy(queryBuilder, postQueryBuilder)));
-        QueryBuilder<?> queryBuilder = null;
+        QueryBuilder queryBuilder = null;
         if (this.queryBuilder != null) {
             queryBuilder = this.queryBuilder.rewrite(context);
         }
-        QueryBuilder<?> postQueryBuilder = null;
+        QueryBuilder postQueryBuilder = null;
         if (this.postQueryBuilder != null) {
             postQueryBuilder = this.postQueryBuilder.rewrite(context);
         }
@@ -925,7 +925,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         return this;
     }
 
-    private SearchSourceBuilder shallowCopy(QueryBuilder<?> queryBuilder, QueryBuilder<?> postQueryBuilder) {
+    private SearchSourceBuilder shallowCopy(QueryBuilder queryBuilder, QueryBuilder postQueryBuilder) {
             SearchSourceBuilder rewrittenBuilder = new SearchSourceBuilder();
             rewrittenBuilder.aggregations = aggregations;
             rewrittenBuilder.explain = explain;
diff --git a/core/src/main/java/org/elasticsearch/search/highlight/AbstractHighlighterBuilder.java b/core/src/main/java/org/elasticsearch/search/highlight/AbstractHighlighterBuilder.java
index e19dd33efc8..557567fe354 100644
--- a/core/src/main/java/org/elasticsearch/search/highlight/AbstractHighlighterBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/highlight/AbstractHighlighterBuilder.java
@@ -80,7 +80,7 @@ public abstract class AbstractHighlighterBuilder<HB extends AbstractHighlighterB
 
     protected String fragmenter;
 
-    protected QueryBuilder<?> highlightQuery;
+    protected QueryBuilder highlightQuery;
 
     protected Order order;
 
@@ -275,7 +275,7 @@ public abstract class AbstractHighlighterBuilder<HB extends AbstractHighlighterB
      * Sets a query to be used for highlighting instead of the search query.
      */
     @SuppressWarnings("unchecked")
-    public HB highlightQuery(QueryBuilder<?> highlightQuery) {
+    public HB highlightQuery(QueryBuilder highlightQuery) {
         this.highlightQuery = highlightQuery;
         return (HB) this;
     }
@@ -283,7 +283,7 @@ public abstract class AbstractHighlighterBuilder<HB extends AbstractHighlighterB
     /**
      * @return the value set by {@link #highlightQuery(QueryBuilder)}
      */
-    public QueryBuilder<?> highlightQuery() {
+    public QueryBuilder highlightQuery() {
         return this.highlightQuery;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java
index 2d4778012d3..e91468c805f 100644
--- a/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java
@@ -41,7 +41,7 @@ public class QueryRescorerBuilder extends RescoreBuilder<QueryRescorerBuilder> {
     public static final float DEFAULT_RESCORE_QUERYWEIGHT = 1.0f;
     public static final float DEFAULT_QUERYWEIGHT = 1.0f;
     public static final QueryRescoreMode DEFAULT_SCORE_MODE = QueryRescoreMode.Total;
-    private final QueryBuilder<?> queryBuilder;
+    private final QueryBuilder queryBuilder;
     private float rescoreQueryWeight = DEFAULT_RESCORE_QUERYWEIGHT;
     private float queryWeight = DEFAULT_QUERYWEIGHT;
     private QueryRescoreMode scoreMode = DEFAULT_SCORE_MODE;
@@ -70,7 +70,7 @@ public class QueryRescorerBuilder extends RescoreBuilder<QueryRescorerBuilder> {
      * Creates a new {@link QueryRescorerBuilder} instance
      * @param builder the query builder to build the rescore query from
      */
-    public QueryRescorerBuilder(QueryBuilder<?> builder) {
+    public QueryRescorerBuilder(QueryBuilder builder) {
         this.queryBuilder = builder;
     }
 
@@ -96,7 +96,7 @@ public class QueryRescorerBuilder extends RescoreBuilder<QueryRescorerBuilder> {
     /**
      * @return the query used for this rescore query
      */
-    public QueryBuilder<?> getRescoreQuery() {
+    public QueryBuilder getRescoreQuery() {
         return this.queryBuilder;
     }
 
@@ -209,12 +209,12 @@ public class QueryRescorerBuilder extends RescoreBuilder<QueryRescorerBuilder> {
      */
     private static class InnerBuilder {
 
-        private QueryBuilder<?> queryBuilder;
+        private QueryBuilder queryBuilder;
         private float rescoreQueryWeight = DEFAULT_RESCORE_QUERYWEIGHT;
         private float queryWeight = DEFAULT_QUERYWEIGHT;
         private QueryRescoreMode scoreMode = DEFAULT_SCORE_MODE;
 
-        void setQueryBuilder(QueryBuilder<?> builder) {
+        void setQueryBuilder(QueryBuilder builder) {
             this.queryBuilder = builder;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/search/rescore/RescoreBuilder.java b/core/src/main/java/org/elasticsearch/search/rescore/RescoreBuilder.java
index e29e7dcd6e9..16c9c9ba8c7 100644
--- a/core/src/main/java/org/elasticsearch/search/rescore/RescoreBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/rescore/RescoreBuilder.java
@@ -125,7 +125,7 @@ public abstract class RescoreBuilder<RB extends RescoreBuilder<RB>> extends ToXC
 
     public abstract QueryRescoreContext build(QueryShardContext context) throws IOException;
 
-    public static QueryRescorerBuilder queryRescorer(QueryBuilder<?> queryBuilder) {
+    public static QueryRescorerBuilder queryRescorer(QueryBuilder queryBuilder) {
         return new QueryRescorerBuilder(queryBuilder);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
index 8d89322e5cd..70876f0acea 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
@@ -66,7 +66,7 @@ public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {
 
     private SortMode sortMode;
 
-    private QueryBuilder<?> nestedFilter;
+    private QueryBuilder nestedFilter;
 
     private String nestedPath;
 
@@ -189,7 +189,7 @@ public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {
      * TODO should the above getters and setters be deprecated/ changed in
      * favour of real getters and setters?
      */
-    public FieldSortBuilder setNestedFilter(QueryBuilder<?> nestedFilter) {
+    public FieldSortBuilder setNestedFilter(QueryBuilder nestedFilter) {
         this.nestedFilter = nestedFilter;
         return this;
     }
@@ -198,7 +198,7 @@ public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {
      * Returns the nested filter that the nested objects should match with in
      * order to be taken into account for sorting.
      */
-    public QueryBuilder<?> getNestedFilter() {
+    public QueryBuilder getNestedFilter() {
         return this.nestedFilter;
     }
 
@@ -324,7 +324,7 @@ public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {
     public static FieldSortBuilder fromXContent(QueryParseContext context, String fieldName) throws IOException {
         XContentParser parser = context.parser();
 
-        QueryBuilder<?> nestedFilter = null;
+        QueryBuilder nestedFilter = null;
         String nestedPath = null;
         Object missing = null;
         SortOrder order = null;
diff --git a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
index 2e08e38a96a..491a37be8b6 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
@@ -292,7 +292,7 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
      * Sets the nested filter that the nested objects should match with in order to be taken into account
      * for sorting.
      */
-    public GeoDistanceSortBuilder setNestedFilter(QueryBuilder<?> nestedFilter) {
+    public GeoDistanceSortBuilder setNestedFilter(QueryBuilder nestedFilter) {
         this.nestedFilter = nestedFilter;
         return this;
     }
@@ -301,7 +301,7 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
      * Returns the nested filter that the nested objects should match with in order to be taken into account
      * for sorting.
      **/
-    public QueryBuilder<?> getNestedFilter() {
+    public QueryBuilder getNestedFilter() {
         return this.nestedFilter;
     }
 
@@ -406,7 +406,7 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
         GeoDistance geoDistance = GeoDistance.DEFAULT;
         SortOrder order = SortOrder.ASC;
         SortMode sortMode = null;
-        QueryBuilder<?> nestedFilter = null;
+        QueryBuilder nestedFilter = null;
         String nestedPath = null;
 
         boolean coerce = GeoValidationMethod.DEFAULT_LENIENT_PARSING;
diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
index 8a713e4992b..c574fbcb7df 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
@@ -80,7 +80,7 @@ public class ScriptSortBuilder extends SortBuilder<ScriptSortBuilder> {
 
     private SortMode sortMode;
 
-    private QueryBuilder<?> nestedFilter;
+    private QueryBuilder nestedFilter;
 
     private String nestedPath;
 
@@ -170,7 +170,7 @@ public class ScriptSortBuilder extends SortBuilder<ScriptSortBuilder> {
      * Sets the nested filter that the nested objects should match with in order to be taken into account
      * for sorting.
      */
-    public ScriptSortBuilder setNestedFilter(QueryBuilder<?> nestedFilter) {
+    public ScriptSortBuilder setNestedFilter(QueryBuilder nestedFilter) {
         this.nestedFilter = nestedFilter;
         return this;
     }
@@ -178,7 +178,7 @@ public class ScriptSortBuilder extends SortBuilder<ScriptSortBuilder> {
     /**
      * Gets the nested filter.
      */
-    public QueryBuilder<?> getNestedFilter() {
+    public QueryBuilder getNestedFilter() {
         return this.nestedFilter;
     }
 
@@ -236,7 +236,7 @@ public class ScriptSortBuilder extends SortBuilder<ScriptSortBuilder> {
         ScriptSortType type = null;
         SortMode sortMode = null;
         SortOrder order = null;
-        QueryBuilder<?> nestedFilter = null;
+        QueryBuilder nestedFilter = null;
         String nestedPath = null;
         Map<String, Object> params = new HashMap<>();
 
diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java
index 805d6e90c47..8cd3fb82413 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java
@@ -169,7 +169,7 @@ public abstract class SortBuilder<T extends SortBuilder<T>> extends ToXContentTo
         return Optional.empty();
     }
 
-    protected static Nested resolveNested(QueryShardContext context, String nestedPath, QueryBuilder<?> nestedFilter) throws IOException {
+    protected static Nested resolveNested(QueryShardContext context, String nestedPath, QueryBuilder nestedFilter) throws IOException {
         Nested nested = null;
         if (nestedPath != null) {
             BitSetProducer rootDocumentsFilter = context.bitsetFilter(Queries.newNonNestedFilter());
diff --git a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
index fac11ab1f78..c2a5fe5aacd 100644
--- a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
@@ -433,7 +433,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
         return Collections.emptySet();
     }
 
-    protected static XContentBuilder toXContent(QueryBuilder<?> query, XContentType contentType) throws IOException {
+    protected static XContentBuilder toXContent(QueryBuilder query, XContentType contentType) throws IOException {
         XContentBuilder builder = XContentFactory.contentBuilder(contentType);
         if (randomBoolean()) {
             builder.prettyPrint();
@@ -499,12 +499,12 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
     /**
      * Parses the query provided as string argument and compares it with the expected result provided as argument as a {@link QueryBuilder}
      */
-    protected final void assertParsedQuery(String queryAsString, QueryBuilder<?> expectedQuery) throws IOException {
+    protected final void assertParsedQuery(String queryAsString, QueryBuilder expectedQuery) throws IOException {
         assertParsedQuery(queryAsString, expectedQuery, ParseFieldMatcher.STRICT);
     }
 
-    protected final void assertParsedQuery(String queryAsString, QueryBuilder<?> expectedQuery, ParseFieldMatcher matcher) throws IOException {
-        QueryBuilder<?> newQuery = parseQuery(queryAsString, matcher);
+    protected final void assertParsedQuery(String queryAsString, QueryBuilder expectedQuery, ParseFieldMatcher matcher) throws IOException {
+        QueryBuilder newQuery = parseQuery(queryAsString, matcher);
         assertNotSame(newQuery, expectedQuery);
         assertEquals(expectedQuery, newQuery);
         assertEquals(expectedQuery.hashCode(), newQuery.hashCode());
@@ -513,38 +513,38 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
     /**
      * Parses the query provided as bytes argument and compares it with the expected result provided as argument as a {@link QueryBuilder}
      */
-    protected final void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder<?> expectedQuery) throws IOException {
+    protected final void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery) throws IOException {
         assertParsedQuery(queryAsBytes, expectedQuery, ParseFieldMatcher.STRICT);
     }
 
-    protected final void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder<?> expectedQuery, ParseFieldMatcher matcher) throws IOException {
-        QueryBuilder<?> newQuery = parseQuery(queryAsBytes, matcher);
+    protected final void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery, ParseFieldMatcher matcher) throws IOException {
+        QueryBuilder newQuery = parseQuery(queryAsBytes, matcher);
         assertNotSame(newQuery, expectedQuery);
         assertEquals(expectedQuery, newQuery);
         assertEquals(expectedQuery.hashCode(), newQuery.hashCode());
     }
 
-    protected final QueryBuilder<?> parseQuery(String queryAsString) throws IOException {
+    protected final QueryBuilder parseQuery(String queryAsString) throws IOException {
         return parseQuery(queryAsString, ParseFieldMatcher.STRICT);
     }
 
-    protected final QueryBuilder<?> parseQuery(String queryAsString, ParseFieldMatcher matcher) throws IOException {
+    protected final QueryBuilder parseQuery(String queryAsString, ParseFieldMatcher matcher) throws IOException {
         XContentParser parser = XContentFactory.xContent(queryAsString).createParser(queryAsString);
         return parseQuery(parser, matcher);
     }
 
-    protected final QueryBuilder<?> parseQuery(BytesReference queryAsBytes) throws IOException {
+    protected final QueryBuilder parseQuery(BytesReference queryAsBytes) throws IOException {
         return parseQuery(queryAsBytes, ParseFieldMatcher.STRICT);
     }
 
-    protected final QueryBuilder<?> parseQuery(BytesReference queryAsBytes, ParseFieldMatcher matcher) throws IOException {
+    protected final QueryBuilder parseQuery(BytesReference queryAsBytes, ParseFieldMatcher matcher) throws IOException {
         XContentParser parser = XContentFactory.xContent(queryAsBytes).createParser(queryAsBytes);
         return parseQuery(parser, matcher);
     }
 
-    private QueryBuilder<?> parseQuery(XContentParser parser, ParseFieldMatcher matcher) throws IOException {
+    private QueryBuilder parseQuery(XContentParser parser, ParseFieldMatcher matcher) throws IOException {
         QueryParseContext context = createParseContext(parser, matcher);
-        QueryBuilder<?> parseInnerQueryBuilder = context.parseInnerQueryBuilder();
+        QueryBuilder parseInnerQueryBuilder = context.parseInnerQueryBuilder();
         assertNull(parser.nextToken());
         return parseInnerQueryBuilder;
     }
@@ -602,8 +602,8 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
         }
     }
 
-    private QueryBuilder<?> rewriteQuery(QB queryBuilder, QueryRewriteContext rewriteContext) throws IOException {
-        QueryBuilder<?> rewritten = QueryBuilder.rewriteQuery(queryBuilder, rewriteContext);
+    private QueryBuilder rewriteQuery(QB queryBuilder, QueryRewriteContext rewriteContext) throws IOException {
+        QueryBuilder rewritten = QueryBuilder.rewriteQuery(queryBuilder, rewriteContext);
         // extra safety to fail fast - serialize the rewritten version to ensure it's serializable.
         assertSerialization(rewritten);
         return rewritten;
@@ -686,7 +686,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             output.writeNamedWriteable(testQuery);
             try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
-                QueryBuilder<?> deserializedQuery = in.readNamedWriteable(QueryBuilder.class);
+                QueryBuilder deserializedQuery = in.readNamedWriteable(QueryBuilder.class);
                 assertEquals(testQuery, deserializedQuery);
                 assertEquals(testQuery.hashCode(), deserializedQuery.hashCode());
                 assertNotSame(testQuery, deserializedQuery);
@@ -963,7 +963,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
      * <li> By now the roundtrip check for the json should be happy.
      * </ul>
      **/
-    public static void checkGeneratedJson(String expected, QueryBuilder<?> source) throws IOException {
+    public static void checkGeneratedJson(String expected, QueryBuilder source) throws IOException {
         // now assert that we actually generate the same JSON
         XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
         source.toXContent(builder, ToXContent.EMPTY_PARAMS);
diff --git a/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
index b6b97f6fbf2..a49d410fdd6 100644
--- a/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
@@ -113,9 +113,9 @@ public class BoolQueryBuilderTests extends AbstractQueryTestCase<BoolQueryBuilde
         }
     }
 
-    private static List<BooleanClause> getBooleanClauses(List<QueryBuilder<?>> queryBuilders, BooleanClause.Occur occur, QueryShardContext context) throws IOException {
+    private static List<BooleanClause> getBooleanClauses(List<QueryBuilder> queryBuilders, BooleanClause.Occur occur, QueryShardContext context) throws IOException {
         List<BooleanClause> clauses = new ArrayList<>();
-        for (QueryBuilder<?> query : queryBuilders) {
+        for (QueryBuilder query : queryBuilders) {
             Query innerQuery = query.toQuery(context);
             if (innerQuery != null) {
                 clauses.add(new BooleanClause(innerQuery, occur));
@@ -132,22 +132,22 @@ public class BoolQueryBuilderTests extends AbstractQueryTestCase<BoolQueryBuilde
         String contentString = "{\n" +
                 "    \"bool\" : {\n";
         if (tempQueryBuilder.must().size() > 0) {
-            QueryBuilder<?> must = tempQueryBuilder.must().get(0);
+            QueryBuilder must = tempQueryBuilder.must().get(0);
             contentString += "\"must\": " + must.toString() + ",";
             expectedQuery.must(must);
         }
         if (tempQueryBuilder.mustNot().size() > 0) {
-            QueryBuilder<?> mustNot = tempQueryBuilder.mustNot().get(0);
+            QueryBuilder mustNot = tempQueryBuilder.mustNot().get(0);
             contentString += (randomBoolean() ? "\"must_not\": " : "\"mustNot\": ") + mustNot.toString() + ",";
             expectedQuery.mustNot(mustNot);
         }
         if (tempQueryBuilder.should().size() > 0) {
-            QueryBuilder<?> should = tempQueryBuilder.should().get(0);
+            QueryBuilder should = tempQueryBuilder.should().get(0);
             contentString += "\"should\": " + should.toString() + ",";
             expectedQuery.should(should);
         }
         if (tempQueryBuilder.filter().size() > 0) {
-            QueryBuilder<?> filter = tempQueryBuilder.filter().get(0);
+            QueryBuilder filter = tempQueryBuilder.filter().get(0);
             contentString += "\"filter\": " + filter.toString() + ",";
             expectedQuery.filter(filter);
         }
@@ -366,7 +366,7 @@ public class BoolQueryBuilderTests extends AbstractQueryTestCase<BoolQueryBuilde
         if (mustRewrite == false && randomBoolean()) {
             boolQueryBuilder.must(new TermsQueryBuilder("foo", "no_rewrite"));
         }
-        QueryBuilder<?> rewritten = boolQueryBuilder.rewrite(createShardContext());
+        QueryBuilder rewritten = boolQueryBuilder.rewrite(createShardContext());
         if (mustRewrite == false && boolQueryBuilder.must().isEmpty()) {
             // if it's empty we rewrite to match all
             assertEquals(rewritten, new MatchAllQueryBuilder());
@@ -398,14 +398,14 @@ public class BoolQueryBuilderTests extends AbstractQueryTestCase<BoolQueryBuilde
     public void testRewriteMultipleTimes() throws IOException {
         BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
         boolQueryBuilder.must(new WrapperQueryBuilder(new WrapperQueryBuilder(new MatchAllQueryBuilder().toString()).toString()));
-        QueryBuilder<?> rewritten = boolQueryBuilder.rewrite(createShardContext());
+        QueryBuilder rewritten = boolQueryBuilder.rewrite(createShardContext());
         BoolQueryBuilder expected = new BoolQueryBuilder();
         expected.must(new WrapperQueryBuilder(new MatchAllQueryBuilder().toString()));
         assertEquals(expected, rewritten);
 
         expected = new BoolQueryBuilder();
         expected.must(new MatchAllQueryBuilder());
-        QueryBuilder<?> rewrittenAgain = rewritten.rewrite(createShardContext());
+        QueryBuilder rewrittenAgain = rewritten.rewrite(createShardContext());
         assertEquals(rewrittenAgain, expected);
         assertEquals(QueryBuilder.rewriteQuery(boolQueryBuilder, createShardContext()), expected);
     }
diff --git a/core/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java
index 27c888aa9dd..ecdadeca923 100644
--- a/core/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java
@@ -108,7 +108,7 @@ public class BoostingQueryBuilderTests extends AbstractQueryTestCase<BoostingQue
         QueryBuilder positive = randomBoolean() ? new MatchAllQueryBuilder() : new WrapperQueryBuilder(new TermQueryBuilder("pos", "bar").toString());
         QueryBuilder negative = randomBoolean() ? new MatchAllQueryBuilder() : new WrapperQueryBuilder(new TermQueryBuilder("neg", "bar").toString());
         BoostingQueryBuilder qb = new BoostingQueryBuilder(positive, negative);
-        QueryBuilder<?> rewrite = qb.rewrite(createShardContext());
+        QueryBuilder rewrite = qb.rewrite(createShardContext());
         if (positive instanceof MatchAllQueryBuilder && negative instanceof MatchAllQueryBuilder) {
             assertSame(rewrite, qb);
         } else {
diff --git a/core/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java
index 90b692e1db2..8d2b30c9ba8 100644
--- a/core/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java
@@ -96,7 +96,7 @@ public class ConstantScoreQueryBuilderTests extends AbstractQueryTestCase<Consta
     }
 
     public void testIllegalArguments() {
-        expectThrows(IllegalArgumentException.class, () -> new ConstantScoreQueryBuilder((QueryBuilder<?>) null));
+        expectThrows(IllegalArgumentException.class, () -> new ConstantScoreQueryBuilder((QueryBuilder) null));
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
index 810cca0b240..b77abd280a7 100644
--- a/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
@@ -74,7 +74,7 @@ public class DisMaxQueryBuilderTests extends AbstractQueryTestCase<DisMaxQueryBu
     @Override
     protected Map<String, DisMaxQueryBuilder> getAlternateVersions() {
         Map<String, DisMaxQueryBuilder> alternateVersions = new HashMap<>();
-        QueryBuilder<?> innerQuery = createTestQueryBuilder().innerQueries().get(0);
+        QueryBuilder innerQuery = createTestQueryBuilder().innerQueries().get(0);
         DisMaxQueryBuilder expectedQuery = new DisMaxQueryBuilder();
         expectedQuery.add(innerQuery);
         String contentString = "{\n" +
@@ -101,7 +101,7 @@ public class DisMaxQueryBuilderTests extends AbstractQueryTestCase<DisMaxQueryBu
      */
     public void testInnerQueryReturnsNull() throws IOException {
         String queryString = "{ \"" + ConstantScoreQueryBuilder.NAME + "\" : { \"filter\" : { } } }";
-        QueryBuilder<?> innerQueryBuilder = parseQuery(queryString);
+        QueryBuilder innerQueryBuilder = parseQuery(queryString);
         DisMaxQueryBuilder disMaxBuilder = new DisMaxQueryBuilder().add(innerQueryBuilder);
         assertNull(disMaxBuilder.toQuery(createShardContext()));
     }
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java
index 8ba48839aba..4c6b26d6848 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java
@@ -494,7 +494,7 @@ public class GeoBoundingBoxQueryBuilderTests extends AbstractQueryTestCase<GeoBo
                         "    \"boost\" : 1.0\n" +
                         "  }\n" +
                         "}";
-        QueryBuilder<?> parsedGeoBboxShortcut = parseQuery(json, ParseFieldMatcher.EMPTY);
+        QueryBuilder parsedGeoBboxShortcut = parseQuery(json, ParseFieldMatcher.EMPTY);
         assertThat(parsedGeoBboxShortcut, equalTo(parsed));
 
         try {
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
index 08f988c2a0b..aaf804e7703 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
@@ -261,7 +261,7 @@ public class GeoShapeQueryBuilderTests extends AbstractQueryTestCase<GeoShapeQue
         } catch (UnsupportedOperationException e) {
             assertEquals("query must be rewritten first", e.getMessage());
         }
-        QueryBuilder<?> rewrite = sqb.rewrite(createShardContext());
+        QueryBuilder rewrite = sqb.rewrite(createShardContext());
         GeoShapeQueryBuilder geoShapeQueryBuilder = new GeoShapeQueryBuilder(GEO_SHAPE_FIELD_NAME, indexedShapeToReturn);
         geoShapeQueryBuilder.strategy(sqb.strategy());
         geoShapeQueryBuilder.relation(sqb.relation());
diff --git a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java
index 36a135a9fb1..35945f21209 100644
--- a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java
@@ -115,7 +115,7 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase<HasChildQue
 
     @Override
     protected void doAssertLuceneQuery(HasChildQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
-        QueryBuilder<?> innerQueryBuilder = queryBuilder.query();
+        QueryBuilder innerQueryBuilder = queryBuilder.query();
         if (innerQueryBuilder instanceof EmptyQueryBuilder) {
             assertNull(query);
         } else {
@@ -149,7 +149,7 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase<HasChildQue
     }
 
     public void testIllegalValues() {
-        QueryBuilder<?> query = RandomQueryBuilder.createQuery(random());
+        QueryBuilder query = RandomQueryBuilder.createQuery(random());
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
                 () -> QueryBuilders.hasChildQuery(null, query, ScoreMode.None));
         assertEquals("[has_child] requires 'type' field", e.getMessage());
diff --git a/core/src/test/java/org/elasticsearch/index/query/IndicesQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/IndicesQueryBuilderTests.java
index 2cccb3f613a..c234016c465 100644
--- a/core/src/test/java/org/elasticsearch/index/query/IndicesQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/IndicesQueryBuilderTests.java
@@ -71,7 +71,7 @@ public class IndicesQueryBuilderTests extends AbstractQueryTestCase<IndicesQuery
         expectThrows(IllegalArgumentException.class, () -> new IndicesQueryBuilder(new MatchAllQueryBuilder(), new String[0]));
 
         IndicesQueryBuilder indicesQueryBuilder = new IndicesQueryBuilder(new MatchAllQueryBuilder(), "index");
-        expectThrows(IllegalArgumentException.class, () -> indicesQueryBuilder.noMatchQuery((QueryBuilder<?>) null));
+        expectThrows(IllegalArgumentException.class, () -> indicesQueryBuilder.noMatchQuery((QueryBuilder) null));
         expectThrows(IllegalArgumentException.class, () -> indicesQueryBuilder.noMatchQuery((String) null));
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
index 4bc95a1c3ce..c7106a7ccf6 100644
--- a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
@@ -247,7 +247,7 @@ public class InnerHitBuilderTests extends ESTestCase {
         }
 
         if (includeQueryTypeOrPath) {
-            QueryBuilder<?> query = new MatchQueryBuilder(randomAsciiOfLengthBetween(1, 16), randomAsciiOfLengthBetween(1, 16));
+            QueryBuilder query = new MatchQueryBuilder(randomAsciiOfLengthBetween(1, 16), randomAsciiOfLengthBetween(1, 16));
             if (randomBoolean()) {
                 return new InnerHitBuilder(innerHits, randomAsciiOfLength(8), query);
             } else {
diff --git a/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java
index b35201f47a1..39c204ec8ae 100644
--- a/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java
@@ -114,7 +114,7 @@ public class NestedQueryBuilderTests extends AbstractQueryTestCase<NestedQueryBu
     }
 
     public void testValidate() {
-        QueryBuilder<?> innerQuery = RandomQueryBuilder.createQuery(random());
+        QueryBuilder innerQuery = RandomQueryBuilder.createQuery(random());
         IllegalArgumentException e =
                 expectThrows(IllegalArgumentException.class, () -> QueryBuilders.nestedQuery(null, innerQuery, ScoreMode.Avg));
         assertThat(e.getMessage(), equalTo("[nested] requires 'path' field"));
diff --git a/core/src/test/java/org/elasticsearch/index/query/PercolateQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/PercolateQueryBuilderTests.java
index 59d79378caa..9dc0bdbd133 100644
--- a/core/src/test/java/org/elasticsearch/index/query/PercolateQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/PercolateQueryBuilderTests.java
@@ -142,7 +142,7 @@ public class PercolateQueryBuilderTests extends AbstractQueryTestCase<PercolateQ
         PercolateQueryBuilder pqb = doCreateTestQueryBuilder(true);
         IllegalStateException e = expectThrows(IllegalStateException.class, () -> pqb.toQuery(createShardContext()));
         assertThat(e.getMessage(), equalTo("query builder must be rewritten first"));
-        QueryBuilder<?> rewrite = pqb.rewrite(createShardContext());
+        QueryBuilder rewrite = pqb.rewrite(createShardContext());
         PercolateQueryBuilder geoShapeQueryBuilder = new PercolateQueryBuilder(pqb.getField(), pqb.getDocumentType(), documentSource);
         assertEquals(geoShapeQueryBuilder, rewrite);
     }
diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
index a763b257afb..4de7dae8c0b 100644
--- a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
@@ -413,7 +413,7 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
                 "        \"query\":\"" + DATE_FIELD_NAME + ":[2012 TO 2014]\"\n" +
                 "    }\n" +
                 "}";
-        QueryBuilder<?> queryBuilder = parseQuery(queryAsString);
+        QueryBuilder queryBuilder = parseQuery(queryAsString);
         assertThat(queryBuilder, instanceOf(QueryStringQueryBuilder.class));
         QueryStringQueryBuilder queryStringQueryBuilder = (QueryStringQueryBuilder) queryBuilder;
         assertThat(queryStringQueryBuilder.timeZone(), equalTo(DateTimeZone.forID("Europe/Paris")));
diff --git a/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java b/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java
index f99a202eb88..23ef8045cc1 100644
--- a/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java
+++ b/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java
@@ -36,7 +36,7 @@ public class RandomQueryBuilder {
      * @param r random seed
      * @return a random {@link QueryBuilder}
      */
-    public static QueryBuilder<?> createQuery(Random r) {
+    public static QueryBuilder createQuery(Random r) {
         switch (RandomInts.randomIntBetween(r, 0, 4)) {
             case 0:
                 return new MatchAllQueryBuilderTests().createTestQueryBuilder();
@@ -61,7 +61,7 @@ public class RandomQueryBuilder {
     public static MultiTermQueryBuilder createMultiTermQuery(Random r) {
         // for now, only use String Rangequeries for MultiTerm test, numeric and date makes little sense
         // see issue #12123 for discussion
-        MultiTermQueryBuilder<?> multiTermQueryBuilder;
+        MultiTermQueryBuilder multiTermQueryBuilder;
         switch(RandomInts.randomIntBetween(r, 0, 3)) {
             case 0:
                 RangeQueryBuilder stringRangeQuery = new RangeQueryBuilder(AbstractQueryTestCase.STRING_FIELD_NAME);
diff --git a/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
index d242e2ecf8f..c3db90f1f6d 100644
--- a/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
@@ -464,7 +464,7 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase<RangeQueryBuil
         query.from(queryFromValue);
         query.to(queryToValue);
         QueryShardContext queryShardContext = createShardContext();
-        QueryBuilder<?> rewritten = query.rewrite(queryShardContext);
+        QueryBuilder rewritten = query.rewrite(queryShardContext);
         assertThat(rewritten, instanceOf(RangeQueryBuilder.class));
         RangeQueryBuilder rewrittenRange = (RangeQueryBuilder) rewritten;
         assertThat(rewrittenRange.fieldName(), equalTo(fieldName));
@@ -485,7 +485,7 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase<RangeQueryBuil
         query.from(queryFromValue);
         query.to(queryToValue);
         QueryShardContext queryShardContext = createShardContext();
-        QueryBuilder<?> rewritten = query.rewrite(queryShardContext);
+        QueryBuilder rewritten = query.rewrite(queryShardContext);
         assertThat(rewritten, instanceOf(MatchNoneQueryBuilder.class));
     }
 
@@ -502,7 +502,7 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase<RangeQueryBuil
         query.from(queryFromValue);
         query.to(queryToValue);
         QueryShardContext queryShardContext = createShardContext();
-        QueryBuilder<?> rewritten = query.rewrite(queryShardContext);
+        QueryBuilder rewritten = query.rewrite(queryShardContext);
         assertThat(rewritten, sameInstance(query));
     }
 
@@ -515,7 +515,7 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase<RangeQueryBuil
             }
         };
         QueryShardContext queryShardContext = createShardContext();
-        QueryBuilder<?> rewritten = query.rewrite(queryShardContext);
+        QueryBuilder rewritten = query.rewrite(queryShardContext);
         assertThat(rewritten, sameInstance(query));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java
index cb35b85d3d9..e9ac69001ef 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java
@@ -59,7 +59,7 @@ public class SpanMultiTermQueryBuilderTests extends AbstractQueryTestCase<SpanMu
     }
 
     public void testIllegalArgument() {
-        expectThrows(IllegalArgumentException.class, () -> new SpanMultiTermQueryBuilder((MultiTermQueryBuilder<?>) null));
+        expectThrows(IllegalArgumentException.class, () -> new SpanMultiTermQueryBuilder((MultiTermQueryBuilder) null));
     }
 
     /**
diff --git a/core/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java
index 11ebd431030..202623d1980 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java
@@ -50,7 +50,7 @@ public class SpanNearQueryBuilderTests extends AbstractQueryTestCase<SpanNearQue
         assertThat(spanNearQuery.getSlop(), equalTo(queryBuilder.slop()));
         assertThat(spanNearQuery.isInOrder(), equalTo(queryBuilder.inOrder()));
         assertThat(spanNearQuery.getClauses().length, equalTo(queryBuilder.clauses().size()));
-        Iterator<SpanQueryBuilder<?>> spanQueryBuilderIterator = queryBuilder.clauses().iterator();
+        Iterator<SpanQueryBuilder> spanQueryBuilderIterator = queryBuilder.clauses().iterator();
         for (SpanQuery spanQuery : spanNearQuery.getClauses()) {
             assertThat(spanQuery, equalTo(spanQueryBuilderIterator.next().toQuery(context)));
         }
diff --git a/core/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java
index b7533fe4605..a9c4ec7d7dc 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java
@@ -45,14 +45,14 @@ public class SpanOrQueryBuilderTests extends AbstractQueryTestCase<SpanOrQueryBu
         assertThat(query, instanceOf(SpanOrQuery.class));
         SpanOrQuery spanOrQuery = (SpanOrQuery) query;
         assertThat(spanOrQuery.getClauses().length, equalTo(queryBuilder.clauses().size()));
-        Iterator<SpanQueryBuilder<?>> spanQueryBuilderIterator = queryBuilder.clauses().iterator();
+        Iterator<SpanQueryBuilder> spanQueryBuilderIterator = queryBuilder.clauses().iterator();
         for (SpanQuery spanQuery : spanOrQuery.getClauses()) {
             assertThat(spanQuery, equalTo(spanQueryBuilderIterator.next().toQuery(context)));
         }
     }
 
     public void testIllegalArguments() {
-        expectThrows(IllegalArgumentException.class, () -> new SpanOrQueryBuilder((SpanQueryBuilder<?>) null));
+        expectThrows(IllegalArgumentException.class, () -> new SpanOrQueryBuilder((SpanQueryBuilder) null));
 
         try {
             SpanOrQueryBuilder spanOrBuilder = new SpanOrQueryBuilder(new SpanTermQueryBuilder("field", "value"));
diff --git a/core/src/test/java/org/elasticsearch/index/query/TemplateQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/TemplateQueryBuilderTests.java
index a6dc42a266e..ee6621bef5b 100644
--- a/core/src/test/java/org/elasticsearch/index/query/TemplateQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/TemplateQueryBuilderTests.java
@@ -40,7 +40,7 @@ public class TemplateQueryBuilderTests extends AbstractQueryTestCase<TemplateQue
     /**
      * The query type all template tests will be based on.
      */
-    private static QueryBuilder<?> templateBase;
+    private static QueryBuilder templateBase;
 
     @BeforeClass
     public static void setupClass() {
@@ -102,7 +102,7 @@ public class TemplateQueryBuilderTests extends AbstractQueryTestCase<TemplateQue
         String query = "{\"template\": {\"query\": \"{\\\"match_{{template}}\\\": {}}\\\"\",\"params\" : {\"template\" : \"all\"}}}";
         Map<String, Object> params = new HashMap<>();
         params.put("template", "all");
-        QueryBuilder<?> expectedBuilder = new TemplateQueryBuilder(new Template(expectedTemplateString, ScriptType.INLINE, null, null,
+        QueryBuilder expectedBuilder = new TemplateQueryBuilder(new Template(expectedTemplateString, ScriptType.INLINE, null, null,
                 params));
         assertParsedQuery(query, expectedBuilder);
     }
@@ -112,7 +112,7 @@ public class TemplateQueryBuilderTests extends AbstractQueryTestCase<TemplateQue
         String query = "{\"template\": {\"query\": {\"match_{{template}}\": {}},\"params\" : {\"template\" : \"all\"}}}";
         Map<String, Object> params = new HashMap<>();
         params.put("template", "all");
-        QueryBuilder<?> expectedBuilder = new TemplateQueryBuilder(new Template(expectedTemplateString, ScriptType.INLINE, null,
+        QueryBuilder expectedBuilder = new TemplateQueryBuilder(new Template(expectedTemplateString, ScriptType.INLINE, null,
                 XContentType.JSON, params));
         assertParsedQuery(query, expectedBuilder);
     }
@@ -120,7 +120,7 @@ public class TemplateQueryBuilderTests extends AbstractQueryTestCase<TemplateQue
     @Override
     public void testMustRewrite() throws IOException {
         String query = "{ \"match_all\" : {}}";
-        QueryBuilder<?> builder = new TemplateQueryBuilder(new Template(query, ScriptType.INLINE, "mockscript",
+        QueryBuilder builder = new TemplateQueryBuilder(new Template(query, ScriptType.INLINE, "mockscript",
         XContentType.JSON, Collections.emptyMap()));
         try {
             builder.toQuery(createShardContext());
@@ -133,7 +133,7 @@ public class TemplateQueryBuilderTests extends AbstractQueryTestCase<TemplateQue
 
     public void testRewriteWithInnerName() throws IOException {
         final String query = "{ \"match_all\" : {\"_name\" : \"foobar\"}}";
-        QueryBuilder<?> builder = new TemplateQueryBuilder(new Template(query, ScriptType.INLINE, "mockscript",
+        QueryBuilder builder = new TemplateQueryBuilder(new Template(query, ScriptType.INLINE, "mockscript",
             XContentType.JSON, Collections.emptyMap()));
         assertEquals(new MatchAllQueryBuilder().queryName("foobar"), builder.rewrite(createShardContext()));
 
@@ -145,7 +145,7 @@ public class TemplateQueryBuilderTests extends AbstractQueryTestCase<TemplateQue
 
     public void testRewriteWithInnerBoost() throws IOException {
         final TermQueryBuilder query = new TermQueryBuilder("foo", "bar").boost(2);
-        QueryBuilder<?> builder = new TemplateQueryBuilder(new Template(query.toString(), ScriptType.INLINE, "mockscript",
+        QueryBuilder builder = new TemplateQueryBuilder(new Template(query.toString(), ScriptType.INLINE, "mockscript",
             XContentType.JSON, Collections.emptyMap()));
         assertEquals(query, builder.rewrite(createShardContext()));
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
index a16f4e2f2ea..0b4faebe18d 100644
--- a/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
@@ -291,7 +291,7 @@ public class TermsQueryBuilderTests extends AbstractQueryTestCase<TermsQueryBuil
                         "    \"boost\" : 1.0\n" +
                         "  }\n" +
                         "}";
-        QueryBuilder<?> inShortcutParsed = parseQuery(json, ParseFieldMatcher.EMPTY);
+        QueryBuilder inShortcutParsed = parseQuery(json, ParseFieldMatcher.EMPTY);
         assertThat(inShortcutParsed, equalTo(parsed));
 
         try {
diff --git a/core/src/test/java/org/elasticsearch/index/query/WrapperQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/WrapperQueryBuilderTests.java
index 749cef61c61..e2a0d551919 100644
--- a/core/src/test/java/org/elasticsearch/index/query/WrapperQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/WrapperQueryBuilderTests.java
@@ -54,7 +54,7 @@ public class WrapperQueryBuilderTests extends AbstractQueryTestCase<WrapperQuery
 
     @Override
     protected void doAssertLuceneQuery(WrapperQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
-        QueryBuilder<?> innerQuery = queryBuilder.rewrite(createShardContext());
+        QueryBuilder innerQuery = queryBuilder.rewrite(createShardContext());
         Query expected = rewrite(innerQuery.toQuery(context));
         assertEquals(rewrite(query), expected);
     }
@@ -138,12 +138,12 @@ public class WrapperQueryBuilderTests extends AbstractQueryTestCase<WrapperQuery
         } catch (UnsupportedOperationException e) {
             assertEquals("this query must be rewritten first", e.getMessage());
         }
-        QueryBuilder<?> rewrite = qb.rewrite(createShardContext());
+        QueryBuilder rewrite = qb.rewrite(createShardContext());
         assertEquals(tqb, rewrite);
     }
 
     public void testRewriteWithInnerName() throws IOException {
-        QueryBuilder<?> builder = new WrapperQueryBuilder("{ \"match_all\" : {\"_name\" : \"foobar\"}}");
+        QueryBuilder builder = new WrapperQueryBuilder("{ \"match_all\" : {\"_name\" : \"foobar\"}}");
         QueryShardContext shardContext = createShardContext();
         assertEquals(new MatchAllQueryBuilder().queryName("foobar"), builder.rewrite(shardContext));
         builder = new WrapperQueryBuilder("{ \"match_all\" : {\"_name\" : \"foobar\"}}").queryName("outer");
@@ -153,7 +153,7 @@ public class WrapperQueryBuilderTests extends AbstractQueryTestCase<WrapperQuery
 
     public void testRewriteWithInnerBoost() throws IOException {
         final TermQueryBuilder query = new TermQueryBuilder("foo", "bar").boost(2);
-        QueryBuilder<?> builder = new WrapperQueryBuilder(query.toString());
+        QueryBuilder builder = new WrapperQueryBuilder(query.toString());
         QueryShardContext shardContext = createShardContext();
         assertEquals(query, builder.rewrite(shardContext));
         builder = new WrapperQueryBuilder(query.toString()).boost(3);
diff --git a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
index 540066ba58b..0cbd80dc76c 100644
--- a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
@@ -240,7 +240,7 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase<Functi
     }
 
     public void testIllegalArguments() {
-        expectThrows(IllegalArgumentException.class, () -> new FunctionScoreQueryBuilder((QueryBuilder<?>) null));
+        expectThrows(IllegalArgumentException.class, () -> new FunctionScoreQueryBuilder((QueryBuilder) null));
         expectThrows(IllegalArgumentException.class, () -> new FunctionScoreQueryBuilder((ScoreFunctionBuilder<?>) null));
         expectThrows(IllegalArgumentException.class, () -> new FunctionScoreQueryBuilder((FilterFunctionBuilder[]) null));
         expectThrows(IllegalArgumentException.class, () -> new FunctionScoreQueryBuilder(null, randomFunction(123)));
@@ -301,7 +301,7 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase<Functi
             "    }\n" +
             "}";
 
-        QueryBuilder<?> queryBuilder = parseQuery(functionScoreQuery);
+        QueryBuilder queryBuilder = parseQuery(functionScoreQuery);
         /*
          * given that we copy part of the decay functions as bytes, we test that fromXContent and toXContent both work no matter what the
          * initial format was
@@ -343,7 +343,7 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase<Functi
             assertThat(functionScoreQueryBuilder.maxBoost(), equalTo(10f));
 
             if (i < XContentType.values().length) {
-                queryBuilder = parseQuery(((AbstractQueryBuilder<?>) queryBuilder).buildAsBytes(XContentType.values()[i]));
+                queryBuilder = parseQuery(((AbstractQueryBuilder) queryBuilder).buildAsBytes(XContentType.values()[i]));
             }
         }
     }
@@ -369,7 +369,7 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase<Functi
             "    }\n" +
             "}";
 
-        QueryBuilder<?> queryBuilder = parseQuery(functionScoreQuery);
+        QueryBuilder queryBuilder = parseQuery(functionScoreQuery);
         /*
          * given that we copy part of the decay functions as bytes, we test that fromXContent and toXContent both work no matter what the
          * initial format was
@@ -395,7 +395,7 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase<Functi
             assertThat(functionScoreQueryBuilder.maxBoost(), equalTo(10f));
 
             if (i < XContentType.values().length) {
-                queryBuilder = parseQuery(((AbstractQueryBuilder<?>) queryBuilder).buildAsBytes(XContentType.values()[i]));
+                queryBuilder = parseQuery(((AbstractQueryBuilder) queryBuilder).buildAsBytes(XContentType.values()[i]));
             }
         }
     }
@@ -476,7 +476,7 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase<Functi
             .endArray()
             .endObject()
             .endObject().string();
-        QueryBuilder<?> query = parseQuery(queryString);
+        QueryBuilder query = parseQuery(queryString);
         assertThat(query, instanceOf(FunctionScoreQueryBuilder.class));
         FunctionScoreQueryBuilder functionScoreQueryBuilder = (FunctionScoreQueryBuilder) query;
         assertThat(functionScoreQueryBuilder.filterFunctionBuilders()[0].getScoreFunction(),
@@ -618,9 +618,9 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase<Functi
     }
 
     public void testRewriteWithFunction() throws IOException {
-        QueryBuilder<?> firstFunction = new WrapperQueryBuilder(new TermQueryBuilder("tq", "1").toString());
+        QueryBuilder firstFunction = new WrapperQueryBuilder(new TermQueryBuilder("tq", "1").toString());
         TermQueryBuilder secondFunction = new TermQueryBuilder("tq", "2");
-        QueryBuilder<?> queryBuilder = randomBoolean() ? new WrapperQueryBuilder(new TermQueryBuilder("foo", "bar").toString())
+        QueryBuilder queryBuilder = randomBoolean() ? new WrapperQueryBuilder(new TermQueryBuilder("foo", "bar").toString())
                 : new TermQueryBuilder("foo", "bar");
         FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(queryBuilder,
                 new FunctionScoreQueryBuilder.FilterFunctionBuilder[] {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java
index b309ed25217..9859936b410 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java
@@ -109,7 +109,7 @@ public class FilterIT extends ESIntegTestCase {
     // See NullPointer issue when filters are empty:
     // https://github.com/elastic/elasticsearch/issues/8438
     public void testEmptyFilterDeclarations() throws Exception {
-        QueryBuilder<?> emptyFilter = new BoolQueryBuilder();
+        QueryBuilder emptyFilter = new BoolQueryBuilder();
         SearchResponse response = client().prepareSearch("idx").addAggregation(filter("tag1", emptyFilter)).execute().actionGet();
 
         assertSearchResponse(response);
@@ -120,7 +120,7 @@ public class FilterIT extends ESIntegTestCase {
     }
 
     public void testEmptyFilter() throws Exception {
-        QueryBuilder<?> emptyFilter = new EmptyQueryBuilder();
+        QueryBuilder emptyFilter = new EmptyQueryBuilder();
         SearchResponse response = client().prepareSearch("idx").addAggregation(filter("tag1", emptyFilter)).execute().actionGet();
 
         assertSearchResponse(response);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java
index 2f076a34eac..fd4db82f2db 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java
@@ -138,7 +138,7 @@ public class FiltersIT extends ESIntegTestCase {
     // See NullPointer issue when filters are empty:
     // https://github.com/elastic/elasticsearch/issues/8438
     public void testEmptyFilterDeclarations() throws Exception {
-        QueryBuilder<?> emptyFilter = new BoolQueryBuilder();
+        QueryBuilder emptyFilter = new BoolQueryBuilder();
         SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(filters("tags", randomOrder(new KeyedFilter("all", emptyFilter),
                         new KeyedFilter("tag1", termQuery("tag", "tag1")))))
@@ -207,7 +207,7 @@ public class FiltersIT extends ESIntegTestCase {
     }
 
     public void testEmptyFilter() throws Exception {
-        QueryBuilder<?> emptyFilter = new EmptyQueryBuilder();
+        QueryBuilder emptyFilter = new EmptyQueryBuilder();
         SearchResponse response = client().prepareSearch("idx").addAggregation(filters("tag1", emptyFilter)).execute().actionGet();
 
         assertSearchResponse(response);
@@ -219,7 +219,7 @@ public class FiltersIT extends ESIntegTestCase {
     }
 
     public void testEmptyKeyedFilter() throws Exception {
-        QueryBuilder<?> emptyFilter = new EmptyQueryBuilder();
+        QueryBuilder emptyFilter = new EmptyQueryBuilder();
         SearchResponse response = client().prepareSearch("idx").addAggregation(filters("tag1", new KeyedFilter("foo", emptyFilter)))
                 .execute().actionGet();
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FiltersTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FiltersTests.java
index 43b76f74ca3..cd2dae53327 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FiltersTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FiltersTests.java
@@ -42,7 +42,7 @@ public class FiltersTests extends BaseAggregationTestCase<FiltersAggregatorBuild
             }
             factory = new FiltersAggregatorBuilder(randomAsciiOfLengthBetween(1, 20), filters);
         } else {
-            QueryBuilder<?>[] filters = new QueryBuilder<?>[size];
+            QueryBuilder[] filters = new QueryBuilder[size];
             for (int i = 0; i < size; i++) {
                 filters[i] = QueryBuilders.termQuery(randomAsciiOfLengthBetween(5, 20), randomAsciiOfLengthBetween(5, 20));
             }
diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java
index 23860883186..83e62072683 100644
--- a/core/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java
+++ b/core/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java
@@ -78,7 +78,7 @@ public class DecayFunctionScoreIT extends ESIntegTestCase {
         return pluginList(InternalSettingsPlugin.class); // uses index.version.created
     }
 
-    private final QueryBuilder<?> baseQuery = constantScoreQuery(termQuery("test", "value"));
+    private final QueryBuilder baseQuery = constantScoreQuery(termQuery("test", "value"));
 
     public void testDistanceScoreGeoLinGaussExp() throws Exception {
         assertAcked(prepareCreate("test").addMapping(
diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java
index f6ef6175530..14d0fc959c3 100644
--- a/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java
+++ b/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java
@@ -583,7 +583,7 @@ public class QueryRescorerIT extends ESIntegTestCase {
                 String[] intToEnglish = new String[] { English.intToEnglish(i), English.intToEnglish(i + 1), English.intToEnglish(i + 2),
                         English.intToEnglish(i + 3) };
 
-                QueryBuilder<?> query = boolQuery().disableCoord(true)
+                QueryBuilder query = boolQuery().disableCoord(true)
                         .should(functionScoreQuery(termQuery("field1", intToEnglish[0]), weightFactorFunction(2.0f)).boostMode(REPLACE))
                         .should(functionScoreQuery(termQuery("field1", intToEnglish[1]), weightFactorFunction(3.0f)).boostMode(REPLACE))
                         .should(functionScoreQuery(termQuery("field1", intToEnglish[2]), weightFactorFunction(5.0f)).boostMode(REPLACE))
diff --git a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
index fde0b08974f..fb179dc7c16 100644
--- a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
@@ -314,7 +314,7 @@ public class QueryRescoreBuilderTests extends ESTestCase {
      * create random shape that is put under test
      */
     public static QueryRescorerBuilder randomRescoreBuilder() {
-        QueryBuilder<MatchAllQueryBuilder> queryBuilder = new MatchAllQueryBuilder().boost(randomFloat())
+        QueryBuilder queryBuilder = new MatchAllQueryBuilder().boost(randomFloat())
                 .queryName(randomAsciiOfLength(20));
         org.elasticsearch.search.rescore.QueryRescorerBuilder rescorer = new
                 org.elasticsearch.search.rescore.QueryRescorerBuilder(queryBuilder);
diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
index 76d5eba12f9..2978c5dae7a 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
@@ -256,7 +256,7 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST
         return doubleFieldType;
     }
 
-    protected static QueryBuilder<?> randomNestedFilter() {
+    protected static QueryBuilder randomNestedFilter() {
         int id = randomIntBetween(0, 2);
         switch(id) {
             case 0: return (new MatchAllQueryBuilder()).boost(randomFloat());
diff --git a/docs/reference/migration/migrate_5_0/java.asciidoc b/docs/reference/migration/migrate_5_0/java.asciidoc
index d2bdf5528ac..004d803dc40 100644
--- a/docs/reference/migration/migrate_5_0/java.asciidoc
+++ b/docs/reference/migration/migrate_5_0/java.asciidoc
@@ -279,20 +279,20 @@ requests can now be validated at call time which results in much clearer errors.
 ==== ValidateQueryRequest
 
 `source(QuerySourceBuilder)`, `source(Map)`, `source(XContentBuilder)`, `source(String)`, `source(byte[])`, `source(byte[], int, int)`,
-`source(BytesReference)` and `source()` have been removed in favor of using `query(QueryBuilder<?>)` and `query()`
+`source(BytesReference)` and `source()` have been removed in favor of using `query(QueryBuilder)` and `query()`
 
 ==== ValidateQueryRequestBuilder
 
-`setSource()` methods have been removed in favor of using `setQuery(QueryBuilder<?>)`
+`setSource()` methods have been removed in favor of using `setQuery(QueryBuilder)`
 
 ==== ExplainRequest
 
 `source(QuerySourceBuilder)`, `source(Map)`, `source(BytesReference)` and `source()` have been removed in favor of using
-`query(QueryBuilder<?>)` and `query()`
+`query(QueryBuilder)` and `query()`
 
 ==== ExplainRequestBuilder
 
-The `setQuery(BytesReference)` method have been removed in favor of using `setQuery(QueryBuilder<?>)`
+The `setQuery(BytesReference)` method have been removed in favor of using `setQuery(QueryBuilder)`
 
 === ClusterStatsResponse
 
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java
index e00ac3320d1..1c17c1966e5 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java
@@ -68,7 +68,7 @@ public class MinDocCountTests extends AbstractTermsTestCase {
         return Collections.singleton(GroovyPlugin.class);
     }
 
-    private static final QueryBuilder<?> QUERY = QueryBuilders.termQuery("match", true);
+    private static final QueryBuilder QUERY = QueryBuilders.termQuery("match", true);
 
     private static int cardinality;
 
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
index e93a7ba7e42..c158a7669d2 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
@@ -62,7 +62,7 @@ public abstract class AbstractBulkByScrollRequestBuilder<
      * Set the query that will filter the source. Just a convenience method for
      * easy chaining.
      */
-    public Self filter(QueryBuilder<?> filter) {
+    public Self filter(QueryBuilder filter) {
         source.setQuery(filter);
         return self();
     }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParentChildTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParentChildTests.java
index c2f0b5625d8..45bacbcf13a 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParentChildTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParentChildTests.java
@@ -32,9 +32,9 @@ import static org.hamcrest.Matchers.equalTo;
  * Index-by-search tests for parent/child.
  */
 public class ReindexParentChildTests extends ReindexTestCase {
-    QueryBuilder<?> findsCountry;
-    QueryBuilder<?> findsCity;
-    QueryBuilder<?> findsNeighborhood;
+    QueryBuilder findsCountry;
+    QueryBuilder findsCity;
+    QueryBuilder findsNeighborhood;
 
     public void testParentChild() throws Exception {
         createParentChildIndex("source");
diff --git a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java b/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java
index 145ef690dc3..682fec46c3b 100644
--- a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java
+++ b/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java
@@ -69,7 +69,7 @@ public class DeleteByQueryRequest extends ActionRequest<DeleteByQueryRequest> im
 
     private String[] types = Strings.EMPTY_ARRAY;
 
-    private QueryBuilder<?> query;
+    private QueryBuilder query;
 
     private String routing;
 
@@ -132,11 +132,11 @@ public class DeleteByQueryRequest extends ActionRequest<DeleteByQueryRequest> im
         return this;
     }
 
-    public QueryBuilder<?> query() {
+    public QueryBuilder query() {
         return query;
     }
 
-    public DeleteByQueryRequest query(QueryBuilder<?> queryBuilder) {
+    public DeleteByQueryRequest query(QueryBuilder queryBuilder) {
         this.query = queryBuilder;
         return this;
     }
diff --git a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequestBuilder.java b/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequestBuilder.java
index 7560e1e8b11..dc5ba3a15fb 100644
--- a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequestBuilder.java
+++ b/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequestBuilder.java
@@ -55,7 +55,7 @@ public class DeleteByQueryRequestBuilder extends ActionRequestBuilder<DeleteByQu
      *
      * @see org.elasticsearch.index.query.QueryBuilders
      */
-    public DeleteByQueryRequestBuilder setQuery(QueryBuilder<?> queryBuilder) {
+    public DeleteByQueryRequestBuilder setQuery(QueryBuilder queryBuilder) {
         request.query(queryBuilder);
         return this;
     }
diff --git a/plugins/delete-by-query/src/main/java/org/elasticsearch/rest/action/deletebyquery/RestDeleteByQueryAction.java b/plugins/delete-by-query/src/main/java/org/elasticsearch/rest/action/deletebyquery/RestDeleteByQueryAction.java
index a7146c2a768..2b537d1cf8a 100644
--- a/plugins/delete-by-query/src/main/java/org/elasticsearch/rest/action/deletebyquery/RestDeleteByQueryAction.java
+++ b/plugins/delete-by-query/src/main/java/org/elasticsearch/rest/action/deletebyquery/RestDeleteByQueryAction.java
@@ -66,7 +66,7 @@ public class RestDeleteByQueryAction extends BaseRestHandler {
         if (RestActions.hasBodyContent(request)) {
             delete.query(RestActions.getQueryContent(RestActions.getRestContent(request), indicesQueriesRegistry, parseFieldMatcher));
         } else {
-            QueryBuilder<?> queryBuilder = RestActions.urlParamsToQueryBuilder(request);
+            QueryBuilder queryBuilder = RestActions.urlParamsToQueryBuilder(request);
             if (queryBuilder != null) {
                 delete.query(queryBuilder);
             }

From c55df195c5a11b92bad9316ef8e6bfdc5a39cbe3 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Fri, 6 May 2016 09:25:58 +0200
Subject: [PATCH 0096/1311] Fixed bad asciidoc

---
 docs/reference/mapping/fields/parent-field.asciidoc  | 2 +-
 docs/reference/mapping/fields/routing-field.asciidoc | 2 +-
 docs/reference/mapping/fields/type-field.asciidoc    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/reference/mapping/fields/parent-field.asciidoc b/docs/reference/mapping/fields/parent-field.asciidoc
index c15a7bcd3b9..78ce1ebfd00 100644
--- a/docs/reference/mapping/fields/parent-field.asciidoc
+++ b/docs/reference/mapping/fields/parent-field.asciidoc
@@ -81,7 +81,7 @@ GET my_index/_search
   },
   "script_fields": {
     "parent": {
-      "script": "doc['_parent']" <4>
+      "script": "doc['_parent']" <3>
     }
   }
 }
diff --git a/docs/reference/mapping/fields/routing-field.asciidoc b/docs/reference/mapping/fields/routing-field.asciidoc
index 496d8dcf56e..c90a52aab37 100644
--- a/docs/reference/mapping/fields/routing-field.asciidoc
+++ b/docs/reference/mapping/fields/routing-field.asciidoc
@@ -42,7 +42,7 @@ GET my_index/_search
   },
   "script_fields": {
     "Routing value": {
-      "script": "doc['_routing']" <4>
+      "script": "doc['_routing']" <2>
     }
   }
 }
diff --git a/docs/reference/mapping/fields/type-field.asciidoc b/docs/reference/mapping/fields/type-field.asciidoc
index 8a569d22f53..a1d48129f5d 100644
--- a/docs/reference/mapping/fields/type-field.asciidoc
+++ b/docs/reference/mapping/fields/type-field.asciidoc
@@ -1,4 +1,4 @@
-\[[mapping-type-field]]
+[[mapping-type-field]]
 === `_type` field
 
 Each document indexed is associated with a <<mapping-type-field,`_type`>> (see

From de8354dd7fa93dbcc71f5dbf154079f92b2b0bf0 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Mon, 25 Apr 2016 17:45:33 +0200
Subject: [PATCH 0097/1311] Allow binary sort values. #17959

The `ip` field uses a binary representation internally. This breaks when
rendering sort values in search responses since elasticsearch tries to write a
binary byte[] as an utf8 json string. This commit extends the `DocValueFormat`
API in order to give fields a chance to choose how to render values.

Closes #6077
---
 .../index/query/InnerHitBuilder.java          |  4 +-
 .../elasticsearch/search/DocValueFormat.java  | 58 +++++++++++++--
 .../elasticsearch/search/SearchService.java   |  3 +-
 .../metrics/tophits/TopHitsAggregator.java    | 16 ++---
 .../tophits/TopHitsAggregatorFactory.java     |  4 +-
 .../controller/SearchPhaseController.java     |  2 +-
 .../fetch/innerhits/InnerHitsContext.java     |  4 +-
 .../innerhits/InnerHitsFetchSubPhase.java     |  4 +-
 .../search/internal/DefaultSearchContext.java |  8 +--
 .../internal/FilteredSearchContext.java       |  6 +-
 .../search/internal/InternalSearchHit.java    | 26 ++-----
 .../search/internal/SearchContext.java        |  5 +-
 .../search/internal/SubSearchContext.java     |  9 ++-
 .../search/query/QueryPhase.java              | 19 +++--
 .../search/query/QuerySearchResult.java       | 34 ++++++++-
 .../search/rescore/RescorePhase.java          |  2 +-
 .../searchafter/SearchAfterBuilder.java       | 26 +++----
 .../search/sort/FieldSortBuilder.java         | 12 ++--
 .../search/sort/GeoDistanceSortBuilder.java   |  5 +-
 .../search/sort/ScoreSortBuilder.java         |  9 ++-
 .../search/sort/ScriptSortBuilder.java        |  5 +-
 .../search/sort/SortAndFormats.java           | 38 ++++++++++
 .../search/sort/SortBuilder.java              | 16 +++--
 .../search/sort/SortFieldAndFormat.java       | 36 ++++++++++
 .../common/network/NetworkAddressTests.java   |  1 +
 .../query/HasChildQueryBuilderTests.java      |  4 +-
 .../query/HasParentQueryBuilderTests.java     |  4 +-
 .../index/query/NestedQueryBuilderTests.java  |  4 +-
 .../search/DocValueFormatTests.java           | 64 +++++++++++++++++
 .../search/searchafter/SearchAfterIT.java     |  7 +-
 .../search/sort/AbstractSortTestCase.java     |  7 +-
 .../search/sort/FieldSortBuilderTests.java    |  4 +-
 .../search/sort/FieldSortIT.java              | 72 ++++++++++++-------
 .../search/sort/GeoDistanceSortBuilderIT.java |  2 +-
 .../sort/GeoDistanceSortBuilderTests.java     |  3 +-
 .../search/sort/ScoreSortBuilderTests.java    |  3 +-
 .../search/sort/ScriptSortBuilderTests.java   |  3 +-
 .../migration/migrate_5_0/java.asciidoc       |  4 ++
 .../elasticsearch/test/TestSearchContext.java | 12 +---
 39 files changed, 402 insertions(+), 143 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/search/sort/SortAndFormats.java
 create mode 100644 core/src/main/java/org/elasticsearch/search/sort/SortFieldAndFormat.java

diff --git a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
index c07c75532f1..b62b5a18a1d 100644
--- a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.index.query;
 
-import org.apache.lucene.search.Sort;
 import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
@@ -41,6 +40,7 @@ import org.elasticsearch.search.fetch.innerhits.InnerHitsContext;
 import org.elasticsearch.search.fetch.source.FetchSourceContext;
 import org.elasticsearch.search.highlight.HighlightBuilder;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.sort.SortBuilder;
 
 import java.io.IOException;
@@ -512,7 +512,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
             innerHitsContext.fetchSourceContext(fetchSourceContext);
         }
         if (sorts != null) {
-            Optional<Sort> optionalSort = SortBuilder.buildSort(sorts, context);
+            Optional<SortAndFormats> optionalSort = SortBuilder.buildSort(sorts, context);
             if (optionalSort.isPresent()) {
                 innerHitsContext.sort(optionalSort.get());
             }
diff --git a/core/src/main/java/org/elasticsearch/search/DocValueFormat.java b/core/src/main/java/org/elasticsearch/search/DocValueFormat.java
index 78f8460f1cc..c7c993b82b6 100644
--- a/core/src/main/java/org/elasticsearch/search/DocValueFormat.java
+++ b/core/src/main/java/org/elasticsearch/search/DocValueFormat.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.search;
 
 import org.apache.lucene.document.InetAddressPoint;
-import org.apache.lucene.index.Term;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.geo.GeoHashUtils;
 import org.elasticsearch.common.io.stream.NamedWriteable;
@@ -29,8 +28,8 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.joda.DateMathParser;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
 import org.elasticsearch.common.joda.Joda;
+import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.network.NetworkAddress;
-import org.elasticsearch.index.mapper.ip.IpFieldMapper;
 import org.elasticsearch.index.mapper.ip.LegacyIpFieldMapper;
 import org.joda.time.DateTimeZone;
 
@@ -48,16 +47,33 @@ import java.util.concurrent.Callable;
 /** A formatter for values as returned by the fielddata/doc-values APIs. */
 public interface DocValueFormat extends NamedWriteable {
 
+    /** Format a long value. This is used by terms and histogram aggregations
+     *  to format keys for fields that use longs as a doc value representation
+     *  such as the {@code long} and {@code date} fields. */
     String format(long value);
 
+    /** Format a double value. This is used by terms and stats aggregations
+     *  to format keys for fields that use numbers as a doc value representation
+     *  such as the {@code long}, {@code double} or {@code date} fields. */
     String format(double value);
 
+    /** Format a double value. This is used by terms aggregations to format
+     *  keys for fields that use binary doc value representations such as the
+     *  {@code keyword} and {@code ip} fields. */
     String format(BytesRef value);
 
+    /** Parse a value that was formatted with {@link #format(long)} back to the
+     *  original long value. */
     long parseLong(String value, boolean roundUp, Callable<Long> now);
 
+    /** Parse a value that was formatted with {@link #format(double)} back to
+     *  the original double value. */
     double parseDouble(String value, boolean roundUp, Callable<Long> now);
 
+    /** Parse a value that was formatted with {@link #format(BytesRef)} back
+     *  to the original BytesRef. */
+    BytesRef parseBytesRef(String value);
+
     public static final DocValueFormat RAW = new DocValueFormat() {
 
         @Override
@@ -81,7 +97,7 @@ public interface DocValueFormat extends NamedWriteable {
 
         @Override
         public String format(BytesRef value) {
-            return Term.toString(value);
+            return value.utf8ToString();
         }
 
         @Override
@@ -99,6 +115,10 @@ public interface DocValueFormat extends NamedWriteable {
         public double parseDouble(String value, boolean roundUp, Callable<Long> now) {
             return Double.parseDouble(value);
         }
+
+        public BytesRef parseBytesRef(String value) {
+            return new BytesRef(value);
+        }
     };
 
     public static final class DateTime implements DocValueFormat {
@@ -154,6 +174,11 @@ public interface DocValueFormat extends NamedWriteable {
         public double parseDouble(String value, boolean roundUp, Callable<Long> now) {
             return parseLong(value, roundUp, now);
         }
+
+        @Override
+        public BytesRef parseBytesRef(String value) {
+            throw new UnsupportedOperationException();
+        }
     }
 
     public static final DocValueFormat GEOHASH = new DocValueFormat() {
@@ -191,6 +216,11 @@ public interface DocValueFormat extends NamedWriteable {
         public double parseDouble(String value, boolean roundUp, Callable<Long> now) {
             throw new UnsupportedOperationException();
         }
+
+        @Override
+        public BytesRef parseBytesRef(String value) {
+            throw new UnsupportedOperationException();
+        }
     };
 
     public static final DocValueFormat BOOLEAN = new DocValueFormat() {
@@ -221,13 +251,24 @@ public interface DocValueFormat extends NamedWriteable {
 
         @Override
         public long parseLong(String value, boolean roundUp, Callable<Long> now) {
-            throw new UnsupportedOperationException();
+            switch (value) {
+            case "false":
+                return 0;
+            case "true":
+                return 1;
+            }
+            throw new IllegalArgumentException("Cannot parse boolean [" + value + "], expected either [true] or [false]");
         }
 
         @Override
         public double parseDouble(String value, boolean roundUp, Callable<Long> now) {
             throw new UnsupportedOperationException();
         }
+
+        @Override
+        public BytesRef parseBytesRef(String value) {
+            throw new UnsupportedOperationException();
+        }
     };
 
     public static final DocValueFormat IP = new DocValueFormat() {
@@ -268,6 +309,11 @@ public interface DocValueFormat extends NamedWriteable {
         public double parseDouble(String value, boolean roundUp, Callable<Long> now) {
             return parseLong(value, roundUp, now);
         }
+
+        @Override
+        public BytesRef parseBytesRef(String value) {
+            return new BytesRef(InetAddressPoint.encode(InetAddresses.forString(value)));
+        }
     };
 
     public static final class Decimal implements DocValueFormat {
@@ -344,5 +390,9 @@ public interface DocValueFormat extends NamedWriteable {
             return n.doubleValue();
         }
 
+        @Override
+        public BytesRef parseBytesRef(String value) {
+            throw new UnsupportedOperationException();
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java
index 24746431949..636e4338063 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchService.java
@@ -104,6 +104,7 @@ import org.elasticsearch.search.query.QuerySearchResultProvider;
 import org.elasticsearch.search.query.ScrollQuerySearchResult;
 import org.elasticsearch.search.rescore.RescoreBuilder;
 import org.elasticsearch.search.searchafter.SearchAfterBuilder;
+import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.search.suggest.Suggesters;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -698,7 +699,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
         }
         if (source.sorts() != null) {
             try {
-                Optional<Sort> optionalSort = SortBuilder.buildSort(source.sorts(), context.getQueryShardContext());
+                Optional<SortAndFormats> optionalSort = SortBuilder.buildSort(source.sorts(), context.getQueryShardContext());
                 if (optionalSort.isPresent()) {
                     context.sort(optionalSort.get());
                 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregator.java
index 8e50dc60ea8..07292f1d29f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregator.java
@@ -24,7 +24,6 @@ import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.LeafCollector;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TopDocsCollector;
 import org.apache.lucene.search.TopFieldCollector;
@@ -45,6 +44,7 @@ import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.internal.InternalSearchHit;
 import org.elasticsearch.search.internal.InternalSearchHits;
 import org.elasticsearch.search.internal.SubSearchContext;
+import org.elasticsearch.search.sort.SortAndFormats;
 
 import java.io.IOException;
 import java.util.List;
@@ -78,9 +78,9 @@ public class TopHitsAggregator extends MetricsAggregator {
 
     @Override
     public boolean needsScores() {
-        Sort sort = subSearchContext.sort();
+        SortAndFormats sort = subSearchContext.sort();
         if (sort != null) {
-            return sort.needsScores() || subSearchContext.trackScores();
+            return sort.sort.needsScores() || subSearchContext.trackScores();
         } else {
             // sort by score
             return true;
@@ -112,12 +112,12 @@ public class TopHitsAggregator extends MetricsAggregator {
             public void collect(int docId, long bucket) throws IOException {
                 TopDocsAndLeafCollector collectors = topDocsCollectors.get(bucket);
                 if (collectors == null) {
-                    Sort sort = subSearchContext.sort();
+                    SortAndFormats sort = subSearchContext.sort();
                     int topN = subSearchContext.from() + subSearchContext.size();
                     // In the QueryPhase we don't need this protection, because it is build into the IndexSearcher,
                     // but here we create collectors ourselves and we need prevent OOM because of crazy an offset and size.
                     topN = Math.min(topN, subSearchContext.searcher().getIndexReader().maxDoc());
-                    TopDocsCollector<?> topLevelCollector = sort != null ? TopFieldCollector.create(sort, topN, true, subSearchContext.trackScores(), subSearchContext.trackScores()) : TopScoreDocCollector.create(topN);
+                    TopDocsCollector<?> topLevelCollector = sort != null ? TopFieldCollector.create(sort.sort, topN, true, subSearchContext.trackScores(), subSearchContext.trackScores()) : TopScoreDocCollector.create(topN);
                     collectors = new TopDocsAndLeafCollector(topLevelCollector);
                     collectors.leafCollector = collectors.topLevelCollector.getLeafCollector(ctx);
                     collectors.leafCollector.setScorer(scorer);
@@ -137,7 +137,7 @@ public class TopHitsAggregator extends MetricsAggregator {
         } else {
             final TopDocs topDocs = topDocsCollector.topLevelCollector.topDocs();
 
-            subSearchContext.queryResult().topDocs(topDocs);
+            subSearchContext.queryResult().topDocs(topDocs, subSearchContext.sort() == null ? null : subSearchContext.sort().formats);
             int[] docIdsToLoad = new int[topDocs.scoreDocs.length];
             for (int i = 0; i < topDocs.scoreDocs.length; i++) {
                 docIdsToLoad[i] = topDocs.scoreDocs[i].doc;
@@ -153,7 +153,7 @@ public class TopHitsAggregator extends MetricsAggregator {
                 searchHitFields.score(scoreDoc.score);
                 if (scoreDoc instanceof FieldDoc) {
                     FieldDoc fieldDoc = (FieldDoc) scoreDoc;
-                    searchHitFields.sortValues(fieldDoc.fields);
+                    searchHitFields.sortValues(fieldDoc.fields, subSearchContext.sort().formats);
                 }
             }
             topHits = new InternalTopHits(name, subSearchContext.from(), subSearchContext.size(), topDocs, fetchResult.hits(), pipelineAggregators(),
@@ -166,7 +166,7 @@ public class TopHitsAggregator extends MetricsAggregator {
     public InternalTopHits buildEmptyAggregation() {
         TopDocs topDocs;
         if (subSearchContext.sort() != null) {
-            topDocs = new TopFieldDocs(0, new FieldDoc[0], subSearchContext.sort().getSort(), Float.NaN);
+            topDocs = new TopFieldDocs(0, new FieldDoc[0], subSearchContext.sort().sort.getSort(), Float.NaN);
         } else {
             topDocs = Lucene.EMPTY_TOP_DOCS;
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java
index 478f03c7eac..ac001222301 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.search.aggregations.metrics.tophits;
 
-import org.apache.lucene.search.Sort;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.aggregations.Aggregator;
@@ -35,6 +34,7 @@ import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase;
 import org.elasticsearch.search.fetch.source.FetchSourceContext;
 import org.elasticsearch.search.highlight.HighlightBuilder;
 import org.elasticsearch.search.internal.SubSearchContext;
+import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.sort.SortBuilder;
 
 import java.io.IOException;
@@ -87,7 +87,7 @@ public class TopHitsAggregatorFactory extends AggregatorFactory<TopHitsAggregato
         subSearchContext.from(from);
         subSearchContext.size(size);
         if (sorts != null) {
-            Optional<Sort> optionalSort = SortBuilder.buildSort(sorts, subSearchContext.getQueryShardContext());
+            Optional<SortAndFormats> optionalSort = SortBuilder.buildSort(sorts, subSearchContext.getQueryShardContext());
             if (optionalSort.isPresent()) {
                 subSearchContext.sort(optionalSort.get());
             }
diff --git a/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java b/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java
index 1c48be1b959..a8ecc7a508c 100644
--- a/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java
+++ b/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java
@@ -362,7 +362,7 @@ public class SearchPhaseController extends AbstractComponent {
 
                     if (sorted) {
                         FieldDoc fieldDoc = (FieldDoc) shardDoc;
-                        searchHit.sortValues(fieldDoc.fields);
+                        searchHit.sortValues(fieldDoc.fields, firstResult.sortValueFormats());
                         if (sortScoreIndex != -1) {
                             searchHit.score(((Number) fieldDoc.fields[sortScoreIndex]).floatValue());
                         }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java
index 56059fc0d01..31921457207 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java
@@ -142,7 +142,7 @@ public final class InnerHitsContext {
                 TopDocsCollector topDocsCollector;
                 if (sort() != null) {
                     try {
-                        topDocsCollector = TopFieldCollector.create(sort(), topN, true, trackScores(), trackScores());
+                        topDocsCollector = TopFieldCollector.create(sort().sort, topN, true, trackScores(), trackScores());
                     } catch (IOException e) {
                         throw ExceptionsHelper.convertToElastic(e);
                     }
@@ -317,7 +317,7 @@ public final class InnerHitsContext {
                 int topN = Math.min(from() + size(), context.searcher().getIndexReader().maxDoc());
                 TopDocsCollector topDocsCollector;
                 if (sort() != null) {
-                    topDocsCollector = TopFieldCollector.create(sort(), topN, true, trackScores(), trackScores());
+                    topDocsCollector = TopFieldCollector.create(sort().sort, topN, true, trackScores(), trackScores());
                 } else {
                     topDocsCollector = TopScoreDocCollector.create(topN);
                 }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java
index ea7edcc3dd4..82c3755cdc9 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java
@@ -73,7 +73,7 @@ public class InnerHitsFetchSubPhase implements FetchSubPhase {
             } catch (IOException e) {
                 throw ExceptionsHelper.convertToElastic(e);
             }
-            innerHits.queryResult().topDocs(topDocs);
+            innerHits.queryResult().topDocs(topDocs, innerHits.sort() == null ? null : innerHits.sort().formats);
             int[] docIdsToLoad = new int[topDocs.scoreDocs.length];
             for (int i = 0; i < topDocs.scoreDocs.length; i++) {
                 docIdsToLoad[i] = topDocs.scoreDocs[i].doc;
@@ -89,7 +89,7 @@ public class InnerHitsFetchSubPhase implements FetchSubPhase {
                 searchHitFields.score(scoreDoc.score);
                 if (scoreDoc instanceof FieldDoc) {
                     FieldDoc fieldDoc = (FieldDoc) scoreDoc;
-                    searchHitFields.sortValues(fieldDoc.fields);
+                    searchHitFields.sortValues(fieldDoc.fields, innerHits.sort().formats);
                 }
             }
             results.put(entry.getKey(), fetchResult.hits());
diff --git a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
index 435e809a893..96319303420 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
@@ -25,7 +25,6 @@ import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Counter;
@@ -71,6 +70,7 @@ import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.query.QueryPhaseExecutionException;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.rescore.RescoreSearchContext;
+import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
 
 import java.io.IOException;
@@ -114,7 +114,7 @@ public class DefaultSearchContext extends SearchContext {
     private FetchSourceContext fetchSourceContext;
     private int from = -1;
     private int size = -1;
-    private Sort sort;
+    private SortAndFormats sort;
     private Float minimumScore;
     private boolean trackScores = false; // when sorting, track scores as well...
     private FieldDoc searchAfter;
@@ -532,13 +532,13 @@ public class DefaultSearchContext extends SearchContext {
     }
 
     @Override
-    public SearchContext sort(Sort sort) {
+    public SearchContext sort(SortAndFormats sort) {
         this.sort = sort;
         return this;
     }
 
     @Override
-    public Sort sort() {
+    public SortAndFormats sort() {
         return this.sort;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
index 283e1dd60e8..8009d0b5fe4 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.internal;
 import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Sort;
 import org.apache.lucene.util.Counter;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cache.recycler.PageCacheRecycler;
@@ -55,6 +54,7 @@ import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.rescore.RescoreSearchContext;
+import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
 
 import java.util.List;
@@ -306,12 +306,12 @@ public abstract class FilteredSearchContext extends SearchContext {
     }
 
     @Override
-    public SearchContext sort(Sort sort) {
+    public SearchContext sort(SortAndFormats sort) {
         return in.sort(sort);
     }
 
     @Override
-    public Sort sort() {
+    public SortAndFormats sort() {
         return in.sort();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
index 628f409974e..f67ed51b226 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
@@ -24,7 +24,6 @@ import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressorFactory;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -34,6 +33,7 @@ import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.search.SearchHits;
@@ -44,6 +44,7 @@ import org.elasticsearch.search.lookup.SourceLookup;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -326,21 +327,13 @@ public class InternalSearchHit implements SearchHit {
         this.highlightFields = highlightFields;
     }
 
-    public void sortValues(Object[] sortValues) {
-        // LUCENE 4 UPGRADE: There must be a better way
-        // we want to convert to a Text object here, and not BytesRef
-
-        // Don't write into sortValues! Otherwise the fields in FieldDoc is modified, which may be used in other places. (SearchContext#lastEmitedDoc)
-        Object[] sortValuesCopy = new Object[sortValues.length];
-        System.arraycopy(sortValues, 0, sortValuesCopy, 0, sortValues.length);
-        if (sortValues != null) {
-            for (int i = 0; i < sortValues.length; i++) {
-                if (sortValues[i] instanceof BytesRef) {
-                    sortValuesCopy[i] = new Text(new BytesArray((BytesRef) sortValues[i]));
-                }
+    public void sortValues(Object[] sortValues, DocValueFormat[] sortValueFormats) {
+        this.sortValues = Arrays.copyOf(sortValues, sortValues.length);
+        for (int i = 0; i < sortValues.length; ++i) {
+            if (this.sortValues[i] instanceof BytesRef) {
+                this.sortValues[i] = sortValueFormats[i].format((BytesRef) sortValues[i]);
             }
         }
-        this.sortValues = sortValuesCopy;
     }
 
     @Override
@@ -618,8 +611,6 @@ public class InternalSearchHit implements SearchHit {
                     sortValues[i] = in.readShort();
                 } else if (type == 8) {
                     sortValues[i] = in.readBoolean();
-                } else if (type == 9) {
-                    sortValues[i] = in.readText();
                 } else {
                     throw new IOException("Can't match type [" + type + "]");
                 }
@@ -726,9 +717,6 @@ public class InternalSearchHit implements SearchHit {
                     } else if (type == Boolean.class) {
                         out.writeByte((byte) 8);
                         out.writeBoolean((Boolean) sortValue);
-                    } else if (sortValue instanceof Text) {
-                        out.writeByte((byte) 9);
-                        out.writeText((Text) sortValue);
                     } else {
                         throw new IOException("Can't handle sort field value of type [" + type + "]");
                     }
diff --git a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
index 8b55c764bbb..550a5f76caf 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
@@ -59,6 +59,7 @@ import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.rescore.RescoreSearchContext;
+import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
 
 import java.util.ArrayList;
@@ -244,9 +245,9 @@ public abstract class SearchContext implements Releasable {
 
     public abstract Float minimumScore();
 
-    public abstract SearchContext sort(Sort sort);
+    public abstract SearchContext sort(SortAndFormats sort);
 
-    public abstract Sort sort();
+    public abstract SortAndFormats sort();
 
     public abstract SearchContext trackScores(boolean trackScores);
 
diff --git a/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
index be2c3798430..6cacf86d65f 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
@@ -19,19 +19,18 @@
 package org.elasticsearch.search.internal;
 
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Sort;
 import org.apache.lucene.util.Counter;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.search.aggregations.SearchContextAggregations;
 import org.elasticsearch.search.fetch.FetchSearchResult;
-import org.elasticsearch.search.fetch.innerhits.InnerHitsContext;
 import org.elasticsearch.search.fetch.script.ScriptFieldsContext;
 import org.elasticsearch.search.fetch.source.FetchSourceContext;
 import org.elasticsearch.search.highlight.SearchContextHighlight;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.rescore.RescoreSearchContext;
+import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
 
 import java.util.ArrayList;
@@ -48,7 +47,7 @@ public class SubSearchContext extends FilteredSearchContext {
 
     private int from;
     private int size = DEFAULT_SIZE;
-    private Sort sort;
+    private SortAndFormats sort;
     private ParsedQuery parsedQuery;
     private Query query;
 
@@ -172,13 +171,13 @@ public class SubSearchContext extends FilteredSearchContext {
     }
 
     @Override
-    public SearchContext sort(Sort sort) {
+    public SearchContext sort(SortAndFormats sort) {
         this.sort = sort;
         return this;
     }
 
     @Override
-    public Sort sort() {
+    public SortAndFormats sort() {
         return sort;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index 62210655a00..a29831f5a04 100644
--- a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -46,6 +46,7 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.MinimumScoreCollector;
 import org.elasticsearch.common.lucene.search.FilteredCollector;
+import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.SearchPhase;
 import org.elasticsearch.search.SearchService;
@@ -58,6 +59,7 @@ import org.elasticsearch.search.profile.ProfileShardResult;
 import org.elasticsearch.search.profile.Profiler;
 import org.elasticsearch.search.rescore.RescorePhase;
 import org.elasticsearch.search.rescore.RescoreSearchContext;
+import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.sort.TrackScoresParseElement;
 import org.elasticsearch.search.suggest.SuggestPhase;
 
@@ -119,7 +121,9 @@ public class QueryPhase implements SearchPhase {
         if (searchContext.hasOnlySuggest()) {
             suggestPhase.execute(searchContext);
             // TODO: fix this once we can fetch docs for suggestions
-            searchContext.queryResult().topDocs(new TopDocs(0, Lucene.EMPTY_SCORE_DOCS, 0));
+            searchContext.queryResult().topDocs(
+                    new TopDocs(0, Lucene.EMPTY_SCORE_DOCS, 0),
+                    new DocValueFormat[0]);
             return;
         }
         // Pre-process aggregations as late as possible. In the case of a DFS_Q_T_F
@@ -141,15 +145,15 @@ public class QueryPhase implements SearchPhase {
         }
     }
 
-    private static boolean returnsDocsInOrder(Query query, Sort sort) {
-        if (sort == null || Sort.RELEVANCE.equals(sort)) {
+    private static boolean returnsDocsInOrder(Query query, SortAndFormats sf) {
+        if (sf == null || Sort.RELEVANCE.equals(sf.sort)) {
             // sort by score
             // queries that return constant scores will return docs in index
             // order since Lucene tie-breaks on the doc id
             return query.getClass() == ConstantScoreQuery.class
                     || query.getClass() == MatchAllDocsQuery.class;
         } else {
-            return Sort.INDEXORDER.equals(sort);
+            return Sort.INDEXORDER.equals(sf.sort);
         }
     }
 
@@ -176,6 +180,7 @@ public class QueryPhase implements SearchPhase {
 
             Collector collector;
             Callable<TopDocs> topDocsCallable;
+            DocValueFormat[] sortValueFormats = new DocValueFormat[0];
 
             assert query == searcher.rewrite(query); // already rewritten
 
@@ -229,8 +234,10 @@ public class QueryPhase implements SearchPhase {
                 }
                 assert numDocs > 0;
                 if (searchContext.sort() != null) {
-                    topDocsCollector = TopFieldCollector.create(searchContext.sort(), numDocs,
+                    SortAndFormats sf = searchContext.sort();
+                    topDocsCollector = TopFieldCollector.create(sf.sort, numDocs,
                             (FieldDoc) after, true, searchContext.trackScores(), searchContext.trackScores());
+                    sortValueFormats = sf.formats;
                 } else {
                     rescore = !searchContext.rescore().isEmpty();
                     for (RescoreSearchContext rescoreContext : searchContext.rescore()) {
@@ -402,7 +409,7 @@ public class QueryPhase implements SearchPhase {
                 queryResult.terminatedEarly(false);
             }
 
-            queryResult.topDocs(topDocsCallable.call());
+            queryResult.topDocs(topDocsCallable.call(), sortValueFormats);
 
             if (searchContext.getProfilers() != null) {
                 List<ProfileShardResult> shardResults = Profiler.buildShardResults(searchContext.getProfilers().getProfilers());
diff --git a/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
index 2b82633ebfd..1408ebe8359 100644
--- a/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
+++ b/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
@@ -19,12 +19,14 @@
 
 package org.elasticsearch.search.query;
 
+import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.TopDocs;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.InternalAggregations;
@@ -51,6 +53,7 @@ public class QuerySearchResult extends QuerySearchResultProvider {
     private int from;
     private int size;
     private TopDocs topDocs;
+    private DocValueFormat[] sortValueFormats;
     private InternalAggregations aggregations;
     private List<SiblingPipelineAggregator> pipelineAggregators;
     private Suggest suggest;
@@ -112,8 +115,20 @@ public class QuerySearchResult extends QuerySearchResultProvider {
         return topDocs;
     }
 
-    public void topDocs(TopDocs topDocs) {
+    public void topDocs(TopDocs topDocs, DocValueFormat[] sortValueFormats) {
         this.topDocs = topDocs;
+        if (topDocs.scoreDocs.length > 0 && topDocs.scoreDocs[0] instanceof FieldDoc) {
+            int numFields = ((FieldDoc) topDocs.scoreDocs[0]).fields.length;
+            if (numFields != sortValueFormats.length) {
+                throw new IllegalArgumentException("The number of sort fields does not match: "
+                        + numFields + " != " + sortValueFormats.length);
+            }
+        }
+        this.sortValueFormats = sortValueFormats;
+    }
+
+    public DocValueFormat[] sortValueFormats() {
+        return sortValueFormats;
     }
 
     public Aggregations aggregations() {
@@ -192,6 +207,15 @@ public class QuerySearchResult extends QuerySearchResultProvider {
 //        shardTarget = readSearchShardTarget(in);
         from = in.readVInt();
         size = in.readVInt();
+        int numSortFieldsPlus1 = in.readVInt();
+        if (numSortFieldsPlus1 == 0) {
+            sortValueFormats = null;
+        } else {
+            sortValueFormats = new DocValueFormat[numSortFieldsPlus1 - 1];
+            for (int i = 0; i < sortValueFormats.length; ++i) {
+                sortValueFormats[i] = in.readNamedWriteable(DocValueFormat.class);
+            }
+        }
         topDocs = readTopDocs(in);
         if (in.readBoolean()) {
             aggregations = InternalAggregations.readAggregations(in);
@@ -233,6 +257,14 @@ public class QuerySearchResult extends QuerySearchResultProvider {
 //        shardTarget.writeTo(out);
         out.writeVInt(from);
         out.writeVInt(size);
+        if (sortValueFormats == null) {
+            out.writeVInt(0);
+        } else {
+            out.writeVInt(1 + sortValueFormats.length);
+            for (int i = 0; i < sortValueFormats.length; ++i) {
+                out.writeNamedWriteable(sortValueFormats[i]);
+            }
+        }
         writeTopDocs(out, topDocs);
         if (aggregations == null) {
             out.writeBoolean(false);
diff --git a/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java b/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java
index 732a80bad20..b82ed941e1c 100644
--- a/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java
+++ b/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java
@@ -61,7 +61,7 @@ public class RescorePhase extends AbstractComponent implements SearchPhase {
             for (RescoreSearchContext ctx : context.rescore()) {
                 topDocs = ctx.rescorer().rescore(topDocs, context, ctx);
             }
-            context.queryResult().topDocs(topDocs);
+            context.queryResult().topDocs(topDocs, context.queryResult().sortValueFormats());
         } catch (IOException e) {
             throw new ElasticsearchException("Rescore Phase Failed", e);
         }
diff --git a/core/src/main/java/org/elasticsearch/search/searchafter/SearchAfterBuilder.java b/core/src/main/java/org/elasticsearch/search/searchafter/SearchAfterBuilder.java
index 6d72efdf368..6ed4b0db5bc 100644
--- a/core/src/main/java/org/elasticsearch/search/searchafter/SearchAfterBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/searchafter/SearchAfterBuilder.java
@@ -20,9 +20,7 @@
 package org.elasticsearch.search.searchafter;
 
 import org.apache.lucene.search.FieldDoc;
-import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParseFieldMatcher;
@@ -36,6 +34,8 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.sort.SortAndFormats;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -104,21 +104,23 @@ public class SearchAfterBuilder implements ToXContent, Writeable {
         return Arrays.copyOf(sortValues, sortValues.length);
     }
 
-    public static FieldDoc buildFieldDoc(Sort sort, Object[] values) {
-        if (sort == null || sort.getSort() == null || sort.getSort().length == 0) {
+    public static FieldDoc buildFieldDoc(SortAndFormats sort, Object[] values) {
+        if (sort == null || sort.sort.getSort() == null || sort.sort.getSort().length == 0) {
             throw new IllegalArgumentException("Sort must contain at least one field.");
         }
 
-        SortField[] sortFields = sort.getSort();
+        SortField[] sortFields = sort.sort.getSort();
         if (sortFields.length != values.length) {
             throw new IllegalArgumentException(
-                    SEARCH_AFTER.getPreferredName() + " has " + values.length + " value(s) but sort has " + sort.getSort().length + ".");
+                    SEARCH_AFTER.getPreferredName() + " has " + values.length + " value(s) but sort has "
+                            + sort.sort.getSort().length + ".");
         }
         Object[] fieldValues = new Object[sortFields.length];
         for (int i = 0; i < sortFields.length; i++) {
             SortField sortField = sortFields[i];
+            DocValueFormat format = sort.formats[i];
             if (values[i] != null) {
-                fieldValues[i] = convertValueFromSortField(values[i], sortField);
+                fieldValues[i] = convertValueFromSortField(values[i], sortField, format);
             } else {
                 fieldValues[i] = null;
             }
@@ -130,15 +132,15 @@ public class SearchAfterBuilder implements ToXContent, Writeable {
         return new FieldDoc(Integer.MAX_VALUE, 0, fieldValues);
     }
 
-    private static Object convertValueFromSortField(Object value, SortField sortField) {
+    private static Object convertValueFromSortField(Object value, SortField sortField, DocValueFormat format) {
         if (sortField.getComparatorSource() instanceof IndexFieldData.XFieldComparatorSource) {
             IndexFieldData.XFieldComparatorSource cmpSource = (IndexFieldData.XFieldComparatorSource) sortField.getComparatorSource();
-            return convertValueFromSortType(sortField.getField(), cmpSource.reducedType(), value);
+            return convertValueFromSortType(sortField.getField(), cmpSource.reducedType(), value, format);
         }
-        return convertValueFromSortType(sortField.getField(), sortField.getType(), value);
+        return convertValueFromSortType(sortField.getField(), sortField.getType(), value, format);
     }
 
-    private static Object convertValueFromSortType(String fieldName, SortField.Type sortType, Object value) {
+    private static Object convertValueFromSortType(String fieldName, SortField.Type sortType, Object value, DocValueFormat format) {
         try {
             switch (sortType) {
                 case DOC:
@@ -179,7 +181,7 @@ public class SearchAfterBuilder implements ToXContent, Writeable {
 
                 case STRING_VAL:
                 case STRING:
-                    return new BytesRef(value.toString());
+                    return format.parseBytesRef(value.toString());
 
                 default:
                     throw new IllegalArgumentException("Comparator type [" + sortType.name() + "] for field [" + fieldName
diff --git a/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
index 70876f0acea..892673f890a 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
@@ -34,6 +34,7 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.MultiValueMode;
 
 import java.io.IOException;
@@ -55,8 +56,10 @@ public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {
      * special field name to sort by index order
      */
     public static final String DOC_FIELD_NAME = "_doc";
-    private static final SortField SORT_DOC = new SortField(null, SortField.Type.DOC);
-    private static final SortField SORT_DOC_REVERSE = new SortField(null, SortField.Type.DOC, true);
+    private static final SortFieldAndFormat SORT_DOC = new SortFieldAndFormat(
+            new SortField(null, SortField.Type.DOC), DocValueFormat.RAW);
+    private static final SortFieldAndFormat SORT_DOC_REVERSE = new SortFieldAndFormat(
+            new SortField(null, SortField.Type.DOC, true), DocValueFormat.RAW);
 
     private final String fieldName;
 
@@ -246,7 +249,7 @@ public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {
     }
 
     @Override
-    public SortField build(QueryShardContext context) throws IOException {
+    public SortFieldAndFormat build(QueryShardContext context) throws IOException {
         if (DOC_FIELD_NAME.equals(fieldName)) {
             if (order == SortOrder.DESC) {
                 return SORT_DOC_REVERSE;
@@ -281,7 +284,8 @@ public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {
             }
             IndexFieldData.XFieldComparatorSource fieldComparatorSource = fieldData
                     .comparatorSource(missing, localSortMode, nested);
-            return new SortField(fieldType.name(), fieldComparatorSource, reverse);
+            SortField field = new SortField(fieldType.name(), fieldComparatorSource, reverse);
+            return new SortFieldAndFormat(field, fieldType.docValueFormat(null, null));
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
index 491a37be8b6..dce9a7ec3fe 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
@@ -51,6 +51,7 @@ import org.elasticsearch.index.query.GeoValidationMethod;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.MultiValueMode;
 
 import java.io.IOException;
@@ -504,7 +505,7 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
     }
 
     @Override
-    public SortField build(QueryShardContext context) throws IOException {
+    public SortFieldAndFormat build(QueryShardContext context) throws IOException {
         final boolean indexCreatedBeforeV2_0 = context.indexVersionCreated().before(Version.V_2_0_0);
         // validation was not available prior to 2.x, so to support bwc percolation queries we only ignore_malformed on 2.x created indexes
         List<GeoPoint> localPoints = new ArrayList<GeoPoint>();
@@ -585,7 +586,7 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
 
         };
 
-        return new SortField(fieldName, geoDistanceComparatorSource, reverse);
+        return new SortFieldAndFormat(new SortField(fieldName, geoDistanceComparatorSource, reverse), DocValueFormat.RAW);
     }
 
     static void parseGeoPoints(XContentParser parser, List<GeoPoint> geoPoints) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java
index 8267429f77c..5b9b139e495 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java
@@ -29,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.search.DocValueFormat;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -40,8 +41,10 @@ public class ScoreSortBuilder extends SortBuilder<ScoreSortBuilder> {
 
     public static final String NAME = "_score";
     public static final ParseField ORDER_FIELD = new ParseField("order");
-    private static final SortField SORT_SCORE = new SortField(null, SortField.Type.SCORE);
-    private static final SortField SORT_SCORE_REVERSE = new SortField(null, SortField.Type.SCORE, true);
+    private static final SortFieldAndFormat SORT_SCORE = new SortFieldAndFormat(
+            new SortField(null, SortField.Type.SCORE), DocValueFormat.RAW);
+    private static final SortFieldAndFormat SORT_SCORE_REVERSE = new SortFieldAndFormat(
+            new SortField(null, SortField.Type.SCORE, true), DocValueFormat.RAW);
 
     /**
      * Build a ScoreSortBuilder default to descending sort order.
@@ -106,7 +109,7 @@ public class ScoreSortBuilder extends SortBuilder<ScoreSortBuilder> {
     }
 
     @Override
-    public SortField build(QueryShardContext context) {
+    public SortFieldAndFormat build(QueryShardContext context) {
         if (order == SortOrder.DESC) {
             return SORT_SCORE;
         } else {
diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
index c574fbcb7df..eeb418c0a97 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
@@ -52,6 +52,7 @@ import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptParameterParser;
 import org.elasticsearch.script.ScriptParameterParser.ScriptParameterValue;
 import org.elasticsearch.script.SearchScript;
+import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.MultiValueMode;
 
 import java.io.IOException;
@@ -302,7 +303,7 @@ public class ScriptSortBuilder extends SortBuilder<ScriptSortBuilder> {
 
 
     @Override
-    public SortField build(QueryShardContext context) throws IOException {
+    public SortFieldAndFormat build(QueryShardContext context) throws IOException {
         final SearchScript searchScript = context.getScriptService().search(
                 context.lookup(), script, ScriptContext.Standard.SEARCH, Collections.emptyMap(), context.getClusterState());
 
@@ -366,7 +367,7 @@ public class ScriptSortBuilder extends SortBuilder<ScriptSortBuilder> {
             throw new QueryShardException(context, "custom script sort type [" + type + "] not supported");
         }
 
-        return new SortField("_script", fieldComparatorSource, reverse);
+        return new SortFieldAndFormat(new SortField("_script", fieldComparatorSource, reverse), DocValueFormat.RAW);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortAndFormats.java b/core/src/main/java/org/elasticsearch/search/sort/SortAndFormats.java
new file mode 100644
index 00000000000..21a9d112fea
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/sort/SortAndFormats.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.sort;
+
+import org.apache.lucene.search.Sort;
+import org.elasticsearch.search.DocValueFormat;
+
+public final class SortAndFormats {
+
+    public final Sort sort;
+    public final DocValueFormat[] formats;
+
+    public SortAndFormats(Sort sort, DocValueFormat[] formats) {
+        if (sort.getSort().length != formats.length) {
+            throw new IllegalArgumentException("Number of sort field mismatch: "
+                    + sort.getSort().length + " != " + formats.length);
+        }
+        this.sort = sort;
+        this.formats = formats;
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java
index 8cd3fb82413..7fb0baf6548 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java
@@ -34,6 +34,7 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.search.DocValueFormat;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -65,9 +66,9 @@ public abstract class SortBuilder<T extends SortBuilder<T>> extends ToXContentTo
     }
 
     /**
-     * Create a @link {@link SortField} from this builder.
+     * Create a @link {@link SortFieldAndFormat} from this builder.
      */
-    protected abstract SortField build(QueryShardContext context) throws IOException;
+    protected abstract SortFieldAndFormat build(QueryShardContext context) throws IOException;
 
     /**
      * Set the order of sorting.
@@ -143,10 +144,13 @@ public abstract class SortBuilder<T extends SortBuilder<T>> extends ToXContentTo
         }
     }
 
-    public static Optional<Sort> buildSort(List<SortBuilder<?>> sortBuilders, QueryShardContext context) throws IOException {
+    public static Optional<SortAndFormats> buildSort(List<SortBuilder<?>> sortBuilders, QueryShardContext context) throws IOException {
         List<SortField> sortFields = new ArrayList<>(sortBuilders.size());
+        List<DocValueFormat> sortFormats = new ArrayList<>(sortBuilders.size());
         for (SortBuilder<?> builder : sortBuilders) {
-            sortFields.add(builder.build(context));
+            SortFieldAndFormat sf = builder.build(context);
+            sortFields.add(sf.field);
+            sortFormats.add(sf.format);
         }
         if (!sortFields.isEmpty()) {
             // optimize if we just sort on score non reversed, we don't really
@@ -163,7 +167,9 @@ public abstract class SortBuilder<T extends SortBuilder<T>> extends ToXContentTo
                 }
             }
             if (sort) {
-                return Optional.of(new Sort(sortFields.toArray(new SortField[sortFields.size()])));
+                return Optional.of(new SortAndFormats(
+                        new Sort(sortFields.toArray(new SortField[sortFields.size()])),
+                        sortFormats.toArray(new DocValueFormat[sortFormats.size()])));
             }
         }
         return Optional.empty();
diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortFieldAndFormat.java b/core/src/main/java/org/elasticsearch/search/sort/SortFieldAndFormat.java
new file mode 100644
index 00000000000..f9756b79068
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/sort/SortFieldAndFormat.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.sort;
+
+import java.util.Objects;
+
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.search.DocValueFormat;
+
+public final class SortFieldAndFormat {
+
+    public final SortField field;
+    public final DocValueFormat format;
+
+    public SortFieldAndFormat(SortField field, DocValueFormat format) {
+        this.field = Objects.requireNonNull(field);
+        this.format = Objects.requireNonNull(format);
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/common/network/NetworkAddressTests.java b/core/src/test/java/org/elasticsearch/common/network/NetworkAddressTests.java
index d62e6ac7dba..4f08eee9b77 100644
--- a/core/src/test/java/org/elasticsearch/common/network/NetworkAddressTests.java
+++ b/core/src/test/java/org/elasticsearch/common/network/NetworkAddressTests.java
@@ -100,4 +100,5 @@ public class NetworkAddressTests extends ESTestCase {
         byte bytes[] = InetAddress.getByName(address).getAddress();
         return Inet6Address.getByAddress(hostname, bytes, scopeid);
     }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java
index 35945f21209..2df15592f45 100644
--- a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java
@@ -140,8 +140,8 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase<HasChildQue
                 InnerHitsContext.BaseInnerHits innerHits =
                         searchContext.innerHits().getInnerHits().get(queryBuilder.innerHit().getName());
                 assertEquals(innerHits.size(), queryBuilder.innerHit().getSize());
-                assertEquals(innerHits.sort().getSort().length, 1);
-                assertEquals(innerHits.sort().getSort()[0].getField(), STRING_FIELD_NAME_2);
+                assertEquals(innerHits.sort().sort.getSort().length, 1);
+                assertEquals(innerHits.sort().sort.getSort()[0].getField(), STRING_FIELD_NAME_2);
             } else {
                 assertThat(searchContext.innerHits().getInnerHits().size(), equalTo(0));
             }
diff --git a/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java
index 59f2d9c85b7..d1b0c6ab222 100644
--- a/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java
@@ -121,8 +121,8 @@ public class HasParentQueryBuilderTests extends AbstractQueryTestCase<HasParentQ
                 InnerHitsContext.BaseInnerHits innerHits = searchContext.innerHits()
                         .getInnerHits().get(queryBuilder.innerHit().getName());
                 assertEquals(innerHits.size(), queryBuilder.innerHit().getSize());
-                assertEquals(innerHits.sort().getSort().length, 1);
-                assertEquals(innerHits.sort().getSort()[0].getField(), STRING_FIELD_NAME_2);
+                assertEquals(innerHits.sort().sort.getSort().length, 1);
+                assertEquals(innerHits.sort().sort.getSort()[0].getField(), STRING_FIELD_NAME_2);
             } else {
                 assertThat(searchContext.innerHits().getInnerHits().size(), equalTo(0));
             }
diff --git a/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java
index 39c204ec8ae..23d595107b4 100644
--- a/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java
@@ -105,8 +105,8 @@ public class NestedQueryBuilderTests extends AbstractQueryTestCase<NestedQueryBu
                 assertTrue(searchContext.innerHits().getInnerHits().containsKey(queryBuilder.innerHit().getName()));
                 InnerHitsContext.BaseInnerHits innerHits = searchContext.innerHits().getInnerHits().get(queryBuilder.innerHit().getName());
                 assertEquals(innerHits.size(), queryBuilder.innerHit().getSize());
-                assertEquals(innerHits.sort().getSort().length, 1);
-                assertEquals(innerHits.sort().getSort()[0].getField(), INT_FIELD_NAME);
+                assertEquals(innerHits.sort().sort.getSort().length, 1);
+                assertEquals(innerHits.sort().sort.getSort()[0].getField(), INT_FIELD_NAME);
             } else {
                 assertThat(searchContext.innerHits().getInnerHits().size(), equalTo(0));
             }
diff --git a/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java b/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java
index a936a315b1e..2ca255ea1a3 100644
--- a/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java
+++ b/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java
@@ -19,11 +19,14 @@
 
 package org.elasticsearch.search;
 
+import org.apache.lucene.document.InetAddressPoint;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.joda.Joda;
+import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.test.ESTestCase;
 import org.joda.time.DateTimeZone;
 
@@ -76,4 +79,65 @@ public class DocValueFormatTests extends ESTestCase {
         assertSame(DocValueFormat.RAW, in.readNamedWriteable(DocValueFormat.class));
     }
 
+    public void testRawFormat() {
+        assertEquals("0", DocValueFormat.RAW.format(0));
+        assertEquals("-1", DocValueFormat.RAW.format(-1));
+        assertEquals("1", DocValueFormat.RAW.format(1));
+
+        assertEquals("0.0", DocValueFormat.RAW.format(0d));
+        assertEquals("0.5", DocValueFormat.RAW.format(.5d));
+        assertEquals("-1.0", DocValueFormat.RAW.format(-1d));
+
+        assertEquals("abc", DocValueFormat.RAW.format(new BytesRef("abc")));
+    }
+
+    public void testBooleanFormat() {
+        assertEquals("false", DocValueFormat.BOOLEAN.format(0));
+        assertEquals("true", DocValueFormat.BOOLEAN.format(1));
+    }
+
+    public void testIpFormat() {
+        assertEquals("192.168.1.7",
+                DocValueFormat.IP.format(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("192.168.1.7")))));
+        assertEquals("::1",
+                DocValueFormat.IP.format(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1")))));
+    }
+
+    public void testRawParse() {
+        assertEquals(-1L, DocValueFormat.RAW.parseLong("-1", randomBoolean(), null));
+        assertEquals(1L, DocValueFormat.RAW.parseLong("1", randomBoolean(), null));
+        // not checking exception messages as they could depend on the JVM
+        expectThrows(IllegalArgumentException.class, () -> DocValueFormat.RAW.parseLong("", randomBoolean(), null));
+        expectThrows(IllegalArgumentException.class, () -> DocValueFormat.RAW.parseLong("abc", randomBoolean(), null));
+
+        assertEquals(-1d, DocValueFormat.RAW.parseDouble("-1", randomBoolean(), null), 0d);
+        assertEquals(1d, DocValueFormat.RAW.parseDouble("1", randomBoolean(), null), 0d);
+        assertEquals(.5, DocValueFormat.RAW.parseDouble("0.5", randomBoolean(), null), 0d);
+        // not checking exception messages as they could depend on the JVM
+        expectThrows(IllegalArgumentException.class, () -> DocValueFormat.RAW.parseLong("", randomBoolean(), null));
+        expectThrows(IllegalArgumentException.class, () -> DocValueFormat.RAW.parseLong("abc", randomBoolean(), null));
+
+        assertEquals(new BytesRef("abc"), DocValueFormat.RAW.parseBytesRef("abc"));
+    }
+
+    public void testBooleanParse() {
+        assertEquals(0L, DocValueFormat.BOOLEAN.parseLong("false", randomBoolean(), null));
+        assertEquals(1L, DocValueFormat.BOOLEAN.parseLong("true", randomBoolean(), null));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> DocValueFormat.BOOLEAN.parseLong("", randomBoolean(), null));
+        assertEquals("Cannot parse boolean [], expected either [true] or [false]", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class,
+                () -> DocValueFormat.BOOLEAN.parseLong("0", randomBoolean(), null));
+        assertEquals("Cannot parse boolean [0], expected either [true] or [false]", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class,
+                () -> DocValueFormat.BOOLEAN.parseLong("False", randomBoolean(), null));
+        assertEquals("Cannot parse boolean [False], expected either [true] or [false]", e.getMessage());
+    }
+
+    public void testIPParse() {
+        assertEquals(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("192.168.1.7"))),
+                DocValueFormat.IP.parseBytesRef("192.168.1.7"));
+        assertEquals(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1"))),
+                DocValueFormat.IP.parseBytesRef("::1"));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterIT.java b/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterIT.java
index 13c99944959..08c661a130b 100644
--- a/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterIT.java
+++ b/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterIT.java
@@ -25,7 +25,6 @@ import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.SearchContextException;
 import org.elasticsearch.search.SearchHit;
@@ -189,11 +188,11 @@ public class SearchAfterIT extends ESIntegTestCase {
                         values.add(randomDouble());
                         break;
                     case 6:
-                        values.add(new Text(randomAsciiOfLengthBetween(5, 20)));
+                        values.add(randomAsciiOfLengthBetween(5, 20));
                         break;
                 }
             }
-            values.add(new Text(UUIDs.randomBase64UUID()));
+            values.add(UUIDs.randomBase64UUID());
             documents.add(values);
         }
         int reqSize = randomInt(NUM_DOCS-1);
@@ -296,7 +295,7 @@ public class SearchAfterIT extends ESIntegTestCase {
             } else if (type == Boolean.class) {
                 mappings.add("field" + Integer.toString(i));
                 mappings.add("type=boolean");
-            } else if (types.get(i) instanceof Text) {
+            } else if (types.get(i) instanceof String) {
                 mappings.add("field" + Integer.toString(i));
                 mappings.add("type=keyword");
             } else {
diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
index 2978c5dae7a..01a8a07d5bb 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
@@ -62,6 +62,7 @@ import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptServiceTests.TestEngineService;
 import org.elasticsearch.script.ScriptSettings;
+import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
@@ -163,12 +164,12 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST
         QueryShardContext mockShardContext = createMockShardContext();
         for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
             T sortBuilder = createTestItem();
-            SortField sortField = sortBuilder.build(mockShardContext);
-            sortFieldAssertions(sortBuilder, sortField);
+            SortFieldAndFormat sortField = sortBuilder.build(mockShardContext);
+            sortFieldAssertions(sortBuilder, sortField.field, sortField.format);
         }
     }
 
-    protected abstract void sortFieldAssertions(T builder, SortField sortField) throws IOException;
+    protected abstract void sortFieldAssertions(T builder, SortField sortField, DocValueFormat format) throws IOException;
 
     /**
      * Test serialization and deserialization of the test sort.
diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
index 4b6eb82304a..baaf3ac5d3c 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.DocValueFormat;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -110,7 +111,7 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
     }
 
     @Override
-    protected void sortFieldAssertions(FieldSortBuilder builder, SortField sortField) throws IOException {
+    protected void sortFieldAssertions(FieldSortBuilder builder, SortField sortField, DocValueFormat format) throws IOException {
         SortField.Type expectedType;
         if (builder.getFieldName().equals(FieldSortBuilder.DOC_FIELD_NAME)) {
             expectedType = SortField.Type.DOC;
@@ -122,6 +123,7 @@ public class FieldSortBuilderTests extends AbstractSortTestCase<FieldSortBuilder
         if (expectedType == SortField.Type.CUSTOM) {
             assertEquals(builder.getFieldName(), sortField.getField());
         }
+        assertEquals(DocValueFormat.RAW, format);
     }
 
     public void testReverseOptionFails() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
index 97c592deb9b..188ae22471a 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
@@ -29,7 +29,6 @@ import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.mapper.Uid;
@@ -76,7 +75,6 @@ import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
 public class FieldSortIT extends ESIntegTestCase {
@@ -1214,13 +1212,13 @@ public class FieldSortIT extends ESIntegTestCase {
         assertThat(searchResponse.getHits().hits().length, equalTo(3));
 
         assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(3)));
-        assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("!4"));
+        assertThat(searchResponse.getHits().getAt(0).sortValues()[0], equalTo("!4"));
 
         assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
-        assertThat(((Text) searchResponse.getHits().getAt(1).sortValues()[0]).string(), equalTo("01"));
+        assertThat(searchResponse.getHits().getAt(1).sortValues()[0], equalTo("01"));
 
         assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(2)));
-        assertThat(((Text) searchResponse.getHits().getAt(2).sortValues()[0]).string(), equalTo("07"));
+        assertThat(searchResponse.getHits().getAt(2).sortValues()[0], equalTo("07"));
 
         searchResponse = client().prepareSearch()
                 .setQuery(matchAllQuery())
@@ -1232,13 +1230,13 @@ public class FieldSortIT extends ESIntegTestCase {
         assertThat(searchResponse.getHits().hits().length, equalTo(3));
 
         assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2)));
-        assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("20"));
+        assertThat(searchResponse.getHits().getAt(0).sortValues()[0], equalTo("20"));
 
         assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
-        assertThat(((Text) searchResponse.getHits().getAt(1).sortValues()[0]).string(), equalTo("10"));
+        assertThat(searchResponse.getHits().getAt(1).sortValues()[0], equalTo("10"));
 
         assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3)));
-        assertThat(((Text) searchResponse.getHits().getAt(2).sortValues()[0]).string(), equalTo("03"));
+        assertThat(searchResponse.getHits().getAt(2).sortValues()[0], equalTo("03"));
     }
 
     public void testSortOnRareField() throws IOException {
@@ -1262,7 +1260,7 @@ public class FieldSortIT extends ESIntegTestCase {
 
 
         assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(1)));
-        assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("10"));
+        assertThat(searchResponse.getHits().getAt(0).sortValues()[0], equalTo("10"));
 
         client().prepareIndex("test", "type1", Integer.toString(2)).setSource(jsonBuilder().startObject()
                 .array("string_values", "11", "15", "20", "07")
@@ -1283,10 +1281,10 @@ public class FieldSortIT extends ESIntegTestCase {
         assertThat(searchResponse.getHits().hits().length, equalTo(2));
 
         assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2)));
-        assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("20"));
+        assertThat(searchResponse.getHits().getAt(0).sortValues()[0], equalTo("20"));
 
         assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
-        assertThat(((Text) searchResponse.getHits().getAt(1).sortValues()[0]).string(), equalTo("10"));
+        assertThat(searchResponse.getHits().getAt(1).sortValues()[0], equalTo("10"));
 
 
         client().prepareIndex("test", "type1", Integer.toString(3)).setSource(jsonBuilder().startObject()
@@ -1308,13 +1306,13 @@ public class FieldSortIT extends ESIntegTestCase {
         assertThat(searchResponse.getHits().hits().length, equalTo(3));
 
         assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2)));
-        assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("20"));
+        assertThat(searchResponse.getHits().getAt(0).sortValues()[0], equalTo("20"));
 
         assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
-        assertThat(((Text) searchResponse.getHits().getAt(1).sortValues()[0]).string(), equalTo("10"));
+        assertThat(searchResponse.getHits().getAt(1).sortValues()[0], equalTo("10"));
 
         assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3)));
-        assertThat(((Text) searchResponse.getHits().getAt(2).sortValues()[0]).string(), equalTo("03"));
+        assertThat(searchResponse.getHits().getAt(2).sortValues()[0], equalTo("03"));
 
         for (int i = 0; i < 15; i++) {
             client().prepareIndex("test", "type1", Integer.toString(300 + i)).setSource(jsonBuilder().startObject()
@@ -1332,13 +1330,13 @@ public class FieldSortIT extends ESIntegTestCase {
         assertThat(searchResponse.getHits().hits().length, equalTo(3));
 
         assertThat(searchResponse.getHits().getAt(0).id(), equalTo(Integer.toString(2)));
-        assertThat(((Text) searchResponse.getHits().getAt(0).sortValues()[0]).string(), equalTo("20"));
+        assertThat(searchResponse.getHits().getAt(0).sortValues()[0], equalTo("20"));
 
         assertThat(searchResponse.getHits().getAt(1).id(), equalTo(Integer.toString(1)));
-        assertThat(((Text) searchResponse.getHits().getAt(1).sortValues()[0]).string(), equalTo("10"));
+        assertThat(searchResponse.getHits().getAt(1).sortValues()[0], equalTo("10"));
 
         assertThat(searchResponse.getHits().getAt(2).id(), equalTo(Integer.toString(3)));
-        assertThat(((Text) searchResponse.getHits().getAt(2).sortValues()[0]).string(), equalTo("03"));
+        assertThat(searchResponse.getHits().getAt(2).sortValues()[0], equalTo("03"));
     }
 
     public void testSortMetaField() throws Exception {
@@ -1448,10 +1446,7 @@ public class FieldSortIT extends ESIntegTestCase {
         SearchHit[] hits = searchResponse.getHits().hits();
         for (int i = 0; i < hits.length; ++i) {
             assertThat(hits[i].getSortValues().length, is(1));
-            Object o = hits[i].getSortValues()[0];
-            assertThat(o, notNullValue());
-            Text text = (Text) o;
-            assertThat(text.string(), is("bar"));
+            assertThat(hits[i].getSortValues()[0], is("bar"));
         }
 
 
@@ -1464,10 +1459,7 @@ public class FieldSortIT extends ESIntegTestCase {
         hits = searchResponse.getHits().hits();
         for (int i = 0; i < hits.length; ++i) {
             assertThat(hits[i].getSortValues().length, is(1));
-            Object o = hits[i].getSortValues()[0];
-            assertThat(o, notNullValue());
-            Text text = (Text) o;
-            assertThat(text.string(), is("bar bar"));
+            assertThat(hits[i].getSortValues()[0], is("bar bar"));
         }
     }
 
@@ -1506,4 +1498,34 @@ public class FieldSortIT extends ESIntegTestCase {
         }
     }
 
+    public void testCustomFormat() throws Exception {
+        // Use an ip field, which uses different internal/external
+        // representations of values, to make sure values are both correctly
+        // rendered and parsed (search_after)
+        assertAcked(prepareCreate("test")
+                .addMapping("type", "ip", "type=ip"));
+        indexRandom(true,
+                client().prepareIndex("test", "type", "1").setSource("ip", "192.168.1.7"),
+                client().prepareIndex("test", "type", "2").setSource("ip", "2001:db8::ff00:42:8329"));
+
+        SearchResponse response = client().prepareSearch("test")
+                .addSort(SortBuilders.fieldSort("ip"))
+                .get();
+        assertSearchResponse(response);
+        assertEquals(2, response.getHits().totalHits());
+        assertArrayEquals(new String[] {"192.168.1.7"},
+                response.getHits().getAt(0).getSortValues());
+        assertArrayEquals(new String[] {"2001:db8::ff00:42:8329"},
+                response.getHits().getAt(1).getSortValues());
+
+        response = client().prepareSearch("test")
+                .addSort(SortBuilders.fieldSort("ip"))
+                .searchAfter(new Object[] {"192.168.1.7"})
+                .get();
+        assertSearchResponse(response);
+        assertEquals(2, response.getHits().totalHits());
+        assertEquals(1, response.getHits().hits().length);
+        assertArrayEquals(new String[] {"2001:db8::ff00:42:8329"},
+                response.getHits().getAt(0).getSortValues());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java
index 5229c905bf8..d40158129dc 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java
@@ -350,7 +350,7 @@ public class GeoDistanceSortBuilderIT extends ESIntegTestCase {
                 .addSort(fieldSort("str_field2").order(SortOrder.DESC).unmappedType("keyword")).get();
 
         assertSortValues(resp,
-                new Object[] {new Text("bcd"), null},
+                new Object[] {"bcd", null},
                 new Object[] {null, null});
 
         resp = client().prepareSearch("test1", "test2")
diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
index 3c94f69eba5..2051233cc95 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
@@ -36,6 +36,7 @@ import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
 import org.elasticsearch.index.query.GeoValidationMethod;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.test.geo.RandomGeoGenerator;
 
 import java.io.IOException;
@@ -177,7 +178,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
     }
 
     @Override
-    protected void sortFieldAssertions(GeoDistanceSortBuilder builder, SortField sortField) throws IOException {
+    protected void sortFieldAssertions(GeoDistanceSortBuilder builder, SortField sortField, DocValueFormat format) throws IOException {
         assertEquals(SortField.Type.CUSTOM, sortField.getType());
         assertEquals(builder.order() == SortOrder.ASC ? false : true, sortField.getReverse());
         assertEquals(builder.fieldName(), sortField.getField());
diff --git a/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java
index 605573e9b71..94e0054b397 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/ScoreSortBuilderTests.java
@@ -26,6 +26,7 @@ import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.DocValueFormat;
 import org.junit.Rule;
 import org.junit.rules.ExpectedException;
 
@@ -98,7 +99,7 @@ public class ScoreSortBuilderTests extends AbstractSortTestCase<ScoreSortBuilder
     }
 
     @Override
-    protected void sortFieldAssertions(ScoreSortBuilder builder, SortField sortField) {
+    protected void sortFieldAssertions(ScoreSortBuilder builder, SortField sortField, DocValueFormat format) {
         assertEquals(SortField.Type.SCORE, sortField.getType());
         assertEquals(builder.order() == SortOrder.DESC ? false : true, sortField.getReverse());
     }
diff --git a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
index 6491c36bf6d..cc615a22f17 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java
@@ -28,6 +28,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
 import org.junit.Rule;
 import org.junit.rules.ExpectedException;
@@ -124,7 +125,7 @@ public class ScriptSortBuilderTests extends AbstractSortTestCase<ScriptSortBuild
     }
 
     @Override
-    protected void sortFieldAssertions(ScriptSortBuilder builder, SortField sortField) throws IOException {
+    protected void sortFieldAssertions(ScriptSortBuilder builder, SortField sortField, DocValueFormat format) throws IOException {
         assertEquals(SortField.Type.CUSTOM, sortField.getType());
         assertEquals(builder.order() == SortOrder.ASC ? false : true, sortField.getReverse());
     }
diff --git a/docs/reference/migration/migrate_5_0/java.asciidoc b/docs/reference/migration/migrate_5_0/java.asciidoc
index 004d803dc40..cdc471d19f2 100644
--- a/docs/reference/migration/migrate_5_0/java.asciidoc
+++ b/docs/reference/migration/migrate_5_0/java.asciidoc
@@ -270,6 +270,10 @@ at call time which results in much clearer errors.
 
 All `extraSource` methods have been removed.
 
+==== SearchResponse
+
+Sort values for `string` fields are now return as `java.lang.String` objects rather than `org.elasticsearch.common.text.Text`.
+
 ==== AggregationBuilder
 
 All methods which take an `XContentBuilder`, `BytesReference` `Map<String, Object>` or `bytes[]` have been removed in favor of providing the
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
index 60a660ead42..5cac904df35 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
@@ -18,16 +18,13 @@
  */
 package org.elasticsearch.test;
 
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Sort;
 import org.apache.lucene.util.Counter;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cache.recycler.PageCacheRecycler;
@@ -65,13 +62,10 @@ import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.rescore.RescoreSearchContext;
+import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
 import org.elasticsearch.threadpool.ThreadPool;
 
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
 public class TestSearchContext extends SearchContext {
 
     final PageCacheRecycler pageCacheRecycler;
@@ -365,12 +359,12 @@ public class TestSearchContext extends SearchContext {
     }
 
     @Override
-    public SearchContext sort(Sort sort) {
+    public SearchContext sort(SortAndFormats sort) {
         return null;
     }
 
     @Override
-    public Sort sort() {
+    public SortAndFormats sort() {
         return null;
     }
 

From b91df36a627be0f1c78d61b68fe017f2976b8f6e Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Mon, 2 May 2016 17:17:00 +0200
Subject: [PATCH 0098/1311] Fix and test handling of `null_value`. #18090

This was mostly untested and had some bugs.

Closes #18085
---
 .../index/mapper/core/DateFieldMapper.java    | 10 ++-
 .../index/mapper/core/NumberFieldMapper.java  | 37 ++++++++++-
 .../index/mapper/ip/IpFieldMapper.java        |  6 +-
 .../mapper/core/DateFieldMapperTests.java     | 51 ++++++++++++++++
 .../mapper/core/KeywordFieldMapperTests.java  | 18 +++++-
 .../mapper/core/NumberFieldMapperTests.java   | 61 +++++++++++++++++++
 .../mapper/core/NumberFieldTypeTests.java     | 31 ++++++++++
 .../index/mapper/ip/IpFieldMapperTests.java   | 52 ++++++++++++++++
 8 files changed, 258 insertions(+), 8 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
index 81977fe6caf..89e2d8409d5 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
@@ -36,7 +36,6 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.joda.DateMathParser;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
 import org.elasticsearch.common.joda.Joda;
-import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.util.LocaleUtils;
@@ -152,7 +151,7 @@ public class DateFieldMapper extends FieldMapper implements AllFieldMapper.Inclu
                     if (propNode == null) {
                         throw new MapperParsingException("Property [null_value] cannot be null.");
                     }
-                    builder.nullValue(InetAddresses.forString(propNode.toString()));
+                    builder.nullValue(propNode.toString());
                     iterator.remove();
                 } else if (propName.equals("ignore_malformed")) {
                     builder.ignoreMalformed(TypeParsers.nodeBooleanValue("ignore_malformed", propNode, parserContext));
@@ -561,7 +560,7 @@ public class DateFieldMapper extends FieldMapper implements AllFieldMapper.Inclu
                 dateAsString = dateAsObject.toString();
             }
         } else {
-            dateAsString = context.parser().text();
+            dateAsString = context.parser().textOrNull();
         }
 
         if (dateAsString == null) {
@@ -615,6 +614,11 @@ public class DateFieldMapper extends FieldMapper implements AllFieldMapper.Inclu
         if (includeDefaults || ignoreMalformed.explicit()) {
             builder.field("ignore_malformed", ignoreMalformed.value());
         }
+
+        if (includeDefaults || fieldType().nullValue() != null) {
+            builder.field("null_value", fieldType().nullValueAsString());
+        }
+
         if (includeInAll != null) {
             builder.field("include_in_all", includeInAll);
         } else if (includeDefaults) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
index 2c3a9a5c83e..f1088652988 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
@@ -366,8 +366,15 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
         BYTE("byte", NumericType.BYTE) {
             @Override
             Byte parse(Object value) {
-                if (value instanceof Byte) {
-                    return (Byte) value;
+                if (value instanceof Number) {
+                    double doubleValue = ((Number) value).doubleValue();
+                    if (doubleValue < Byte.MIN_VALUE || doubleValue > Byte.MAX_VALUE) {
+                        throw new IllegalArgumentException("Value [" + value + "] is out of range for a byte");
+                    }
+                    if (doubleValue % 1 != 0) {
+                        throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
+                    }
+                    return ((Number) value).byteValue();
                 }
                 if (value instanceof BytesRef) {
                     value = ((BytesRef) value).utf8ToString();
@@ -426,6 +433,13 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
             @Override
             Short parse(Object value) {
                 if (value instanceof Number) {
+                    double doubleValue = ((Number) value).doubleValue();
+                    if (doubleValue < Short.MIN_VALUE || doubleValue > Short.MAX_VALUE) {
+                        throw new IllegalArgumentException("Value [" + value + "] is out of range for a short");
+                    }
+                    if (doubleValue % 1 != 0) {
+                        throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
+                    }
                     return ((Number) value).shortValue();
                 }
                 if (value instanceof BytesRef) {
@@ -485,6 +499,13 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
             @Override
             Integer parse(Object value) {
                 if (value instanceof Number) {
+                    double doubleValue = ((Number) value).doubleValue();
+                    if (doubleValue < Integer.MIN_VALUE || doubleValue > Integer.MAX_VALUE) {
+                        throw new IllegalArgumentException("Value [" + value + "] is out of range for an integer");
+                    }
+                    if (doubleValue % 1 != 0) {
+                        throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
+                    }
                     return ((Number) value).intValue();
                 }
                 if (value instanceof BytesRef) {
@@ -581,6 +602,13 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
             @Override
             Long parse(Object value) {
                 if (value instanceof Number) {
+                    double doubleValue = ((Number) value).doubleValue();
+                    if (doubleValue < Long.MIN_VALUE || doubleValue > Long.MAX_VALUE) {
+                        throw new IllegalArgumentException("Value [" + value + "] is out of range for a long");
+                    }
+                    if (doubleValue % 1 != 0) {
+                        throw new IllegalArgumentException("Value [" + value + "] has a decimal part");
+                    }
                     return ((Number) value).longValue();
                 }
                 if (value instanceof BytesRef) {
@@ -944,6 +972,11 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
         if (includeDefaults || coerce.explicit()) {
             builder.field("coerce", coerce.value());
         }
+
+        if (includeDefaults || fieldType().nullValue() != null) {
+            builder.field("null_value", fieldType().nullValue());
+        }
+
         if (includeInAll != null) {
             builder.field("include_in_all", includeInAll);
         } else if (includeDefaults) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
index 6cb3cbe46c8..5f5a3d85fe1 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
@@ -339,7 +339,7 @@ public class IpFieldMapper extends FieldMapper implements AllFieldMapper.Include
         if (context.externalValueSet()) {
             addressAsObject = context.externalValue();
         } else {
-            addressAsObject = context.parser().text();
+            addressAsObject = context.parser().textOrNull();
         }
 
         if (addressAsObject == null) {
@@ -395,6 +395,10 @@ public class IpFieldMapper extends FieldMapper implements AllFieldMapper.Include
     protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
         super.doXContentBody(builder, includeDefaults, params);
 
+        if (includeDefaults || fieldType().nullValue() != null) {
+            builder.field("null_value", InetAddresses.toAddrString((InetAddress) fieldType().nullValue()));
+        }
+
         if (includeDefaults || ignoreMalformed.explicit()) {
             builder.field("ignore_malformed", ignoreMalformed.value());
         }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/DateFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/DateFieldMapperTests.java
index 9f09e3e2e13..ee19d094a3f 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/core/DateFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/core/DateFieldMapperTests.java
@@ -251,4 +251,55 @@ public class DateFieldMapperTests extends ESSingleNodeTestCase {
                 .endObject()
                 .bytes());
     }
+
+    public void testNullValue() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("field")
+                            .field("type", "date")
+                        .endObject()
+                    .endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .nullField("field")
+                .endObject()
+                .bytes());
+        assertArrayEquals(new IndexableField[0], doc.rootDoc().getFields("field"));
+
+        mapping = XContentFactory.jsonBuilder().startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("field")
+                            .field("type", "date")
+                            .field("null_value", "2016-03-11")
+                        .endObject()
+                    .endObject()
+                .endObject().endObject().string();
+
+        mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .nullField("field")
+                .endObject()
+                .bytes());
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        assertEquals(2, fields.length);
+        IndexableField pointField = fields[0];
+        assertEquals(1, pointField.fieldType().pointDimensionCount());
+        assertEquals(8, pointField.fieldType().pointNumBytes());
+        assertFalse(pointField.fieldType().stored());
+        assertEquals(1457654400000L, pointField.numericValue().longValue());
+        IndexableField dvField = fields[1];
+        assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
+        assertEquals(1457654400000L, dvField.numericValue().longValue());
+        assertFalse(dvField.fieldType().stored());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/KeywordFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/KeywordFieldMapperTests.java
index 28867ed1f73..8b9c9134fa3 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/core/KeywordFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/core/KeywordFieldMapperTests.java
@@ -126,14 +126,28 @@ public class KeywordFieldMapperTests extends ESSingleNodeTestCase {
 
     public void testNullValue() throws IOException {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("properties").startObject("field").field("type", "keyword").field("null_value", "uri").endObject().endObject()
+                .startObject("properties").startObject("field").field("type", "keyword").endObject().endObject()
                 .endObject().endObject().string();
 
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
-
         assertEquals(mapping, mapper.mappingSource().toString());
 
         ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .nullField("field")
+                .endObject()
+                .bytes());
+        assertArrayEquals(new IndexableField[0], doc.rootDoc().getFields("field"));
+
+        mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field").field("type", "keyword").field("null_value", "uri").endObject().endObject()
+                .endObject().endObject().string();
+
+        mapper = parser.parse("type", new CompressedXContent(mapping));
+
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
                 .startObject()
                 .endObject()
                 .bytes());
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/NumberFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/NumberFieldMapperTests.java
index 0f0f5a33213..45cc09fff0e 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/core/NumberFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/core/NumberFieldMapperTests.java
@@ -316,4 +316,65 @@ public class NumberFieldMapperTests extends ESSingleNodeTestCase {
             assertThat(e.getMessage(), containsString("Mapping definition for [foo] has unsupported parameters:  [norms"));
         }
     }
+
+    public void testNullValue() throws IOException {
+        for (String type : TYPES) {
+            doTestNullValue(type);
+        }
+    }
+
+    private void doTestNullValue(String type) throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("field")
+                            .field("type", type)
+                        .endObject()
+                    .endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .nullField("field")
+                .endObject()
+                .bytes());
+        assertArrayEquals(new IndexableField[0], doc.rootDoc().getFields("field"));
+
+        Object missing;
+        if (Arrays.asList("float", "double").contains(type)) {
+            missing = 123d;
+        } else {
+            missing = 123L;
+        }
+        mapping = XContentFactory.jsonBuilder().startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("field")
+                            .field("type", type)
+                            .field("null_value", missing)
+                        .endObject()
+                    .endObject()
+                .endObject().endObject().string();
+
+        mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .nullField("field")
+                .endObject()
+                .bytes());
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        assertEquals(2, fields.length);
+        IndexableField pointField = fields[0];
+        assertEquals(1, pointField.fieldType().pointDimensionCount());
+        assertFalse(pointField.fieldType().stored());
+        assertEquals(123, pointField.numericValue().doubleValue(), 0d);
+        IndexableField dvField = fields[1];
+        assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
+        assertFalse(dvField.fieldType().stored());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/NumberFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/NumberFieldTypeTests.java
index 25591591e42..22dec5bdde5 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/core/NumberFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/core/NumberFieldTypeTests.java
@@ -75,4 +75,35 @@ public class NumberFieldTypeTests extends FieldTypeTestCase {
                 () -> ft.rangeQuery("1", "3", true, true));
         assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
     }
+
+    public void testConversions() {
+        assertEquals((byte) 3, NumberType.BYTE.parse(3d));
+        assertEquals((short) 3, NumberType.SHORT.parse(3d));
+        assertEquals(3, NumberType.INTEGER.parse(3d));
+        assertEquals(3L, NumberType.LONG.parse(3d));
+        assertEquals(3f, NumberType.FLOAT.parse(3d));
+        assertEquals(3d, NumberType.DOUBLE.parse(3d));
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> NumberType.BYTE.parse(3.5));
+        assertEquals("Value [3.5] has a decimal part", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class, () -> NumberType.SHORT.parse(3.5));
+        assertEquals("Value [3.5] has a decimal part", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class, () -> NumberType.INTEGER.parse(3.5));
+        assertEquals("Value [3.5] has a decimal part", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class, () -> NumberType.LONG.parse(3.5));
+        assertEquals("Value [3.5] has a decimal part", e.getMessage());
+        assertEquals(3.5f, NumberType.FLOAT.parse(3.5));
+        assertEquals(3.5d, NumberType.DOUBLE.parse(3.5));
+
+        e = expectThrows(IllegalArgumentException.class, () -> NumberType.BYTE.parse(128));
+        assertEquals("Value [128] is out of range for a byte", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class, () -> NumberType.SHORT.parse(65536));
+        assertEquals("Value [65536] is out of range for a short", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class, () -> NumberType.INTEGER.parse(2147483648L));
+        assertEquals("Value [2147483648] is out of range for an integer", e.getMessage());
+        e = expectThrows(IllegalArgumentException.class, () -> NumberType.LONG.parse(10000000000000000000d));
+        assertEquals("Value [1.0E19] is out of range for a long", e.getMessage());
+        assertEquals(1.1f, NumberType.FLOAT.parse(1.1)); // accuracy loss is expected
+        assertEquals(1.1d, NumberType.DOUBLE.parse(1.1));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ip/IpFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ip/IpFieldMapperTests.java
index 3bb96cce31e..25979c5a632 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/ip/IpFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ip/IpFieldMapperTests.java
@@ -36,6 +36,7 @@ import org.junit.Before;
 
 import static org.hamcrest.Matchers.containsString;
 
+import java.io.IOException;
 import java.net.InetAddress;
 
 public class IpFieldMapperTests extends ESSingleNodeTestCase {
@@ -217,4 +218,55 @@ public class IpFieldMapperTests extends ESSingleNodeTestCase {
         fields = doc.rootDoc().getFields("_all");
         assertEquals(0, fields.length);
     }
+
+    public void testNullValue() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("field")
+                            .field("type", "ip")
+                        .endObject()
+                    .endObject()
+                .endObject().endObject().string();
+
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .nullField("field")
+                .endObject()
+                .bytes());
+        assertArrayEquals(new IndexableField[0], doc.rootDoc().getFields("field"));
+
+        mapping = XContentFactory.jsonBuilder().startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("field")
+                            .field("type", "ip")
+                            .field("null_value", "::1")
+                        .endObject()
+                    .endObject()
+                .endObject().endObject().string();
+
+        mapper = parser.parse("type", new CompressedXContent(mapping));
+        assertEquals(mapping, mapper.mappingSource().toString());
+
+        doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject()
+                .nullField("field")
+                .endObject()
+                .bytes());
+        IndexableField[] fields = doc.rootDoc().getFields("field");
+        assertEquals(2, fields.length);
+        IndexableField pointField = fields[0];
+        assertEquals(1, pointField.fieldType().pointDimensionCount());
+        assertEquals(16, pointField.fieldType().pointNumBytes());
+        assertFalse(pointField.fieldType().stored());
+        assertEquals(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1"))), pointField.binaryValue());
+        IndexableField dvField = fields[1];
+        assertEquals(DocValuesType.SORTED_SET, dvField.fieldType().docValuesType());
+        assertEquals(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1"))), dvField.binaryValue());
+        assertFalse(dvField.fieldType().stored());
+    }
 }

From 4ddf916aabaf6e60e68f0cf59ef5dbc3c17cffed Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Fri, 6 May 2016 10:40:19 +0200
Subject: [PATCH 0099/1311] Removed scripting docs for docs[field].multiValued

Closes #18164
---
 docs/reference/modules/scripting/groovy.asciidoc | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/docs/reference/modules/scripting/groovy.asciidoc b/docs/reference/modules/scripting/groovy.asciidoc
index 60b64e0131a..0c7f4676d2c 100644
--- a/docs/reference/modules/scripting/groovy.asciidoc
+++ b/docs/reference/modules/scripting/groovy.asciidoc
@@ -31,9 +31,6 @@ on the underlying field type):
 `doc['field_name'].empty`::
     A boolean indicating if the field has no values within the doc.
 
-`doc['field_name'].multiValued`::
-    A boolean indicating that the field has several values within the corpus.
-
 `doc['field_name'].lat`::
     The latitude of a geo point type, or `null`.
 

From 7d14728960f4b0678f9970b0aff4d4f7fbca8459 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Mon, 2 May 2016 19:30:47 +0200
Subject: [PATCH 0100/1311] Add xContent shuffling to some more tests

This adds some random shuffling of xContent to some more test cases.

Relates to #5831
---
 .../common/xcontent/XContentBuilder.java      |  4 ++++
 .../common/xcontent/XContentGenerator.java    |  3 ++-
 .../xcontent/json/JsonXContentGenerator.java  |  5 +++++
 .../AbstractShapeBuilderTestCase.java         |  4 +---
 .../ConstructingObjectParserTests.java        |  3 +--
 .../index/query/AbstractQueryTestCase.java    |  6 +++---
 .../index/query/InnerHitBuilderTests.java     |  3 +--
 .../query/PercolateQueryBuilderTests.java     |  7 ++-----
 .../snapshots/blobstore/FileInfoTests.java    |  3 +--
 .../ingest/IngestMetadataTests.java           |  3 +--
 .../script/ScriptMetaDataTests.java           |  4 +---
 .../aggregations/BaseAggregationTestCase.java |  2 +-
 .../BasePipelineAggregationTestCase.java      |  2 +-
 .../highlight/HighlightBuilderTests.java      |  3 +--
 .../rescore/QueryRescoreBuilderTests.java     |  4 +---
 .../searchafter/SearchAfterBuilderTests.java  |  5 ++---
 .../search/sort/AbstractSortTestCase.java     |  2 +-
 .../AbstractSuggestionBuilderTestCase.java    |  7 ++-----
 .../CompletionSuggesterBuilderTests.java      |  7 ++-----
 .../phrase/DirectCandidateGeneratorTests.java |  4 +---
 .../phrase/SmoothingModelTestCase.java        |  2 +-
 .../org/elasticsearch/test/ESTestCase.java    | 20 +++++++++++--------
 .../test/test/ESTestCaseTests.java            |  4 +---
 23 files changed, 48 insertions(+), 59 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
index 4367b15fd52..3fa182e3727 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
@@ -113,6 +113,10 @@ public final class XContentBuilder implements BytesStream, Releasable {
         return this;
     }
 
+    public boolean isPrettyPrint() {
+        return generator.isPrettyPrint();
+    }
+
     public XContentBuilder lfAtEnd() {
         generator.usePrintLineFeedAtEnd();
         return this;
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java
index f23ae441989..a2cceae8367 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.common.xcontent;
 
 import org.elasticsearch.common.bytes.BytesReference;
-
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
@@ -34,6 +33,8 @@ public interface XContentGenerator extends Closeable {
 
     void usePrettyPrint();
 
+    boolean isPrettyPrint();
+
     void usePrintLineFeedAtEnd();
 
     void writeStartArray() throws IOException;
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java
index 4148148e507..e74ab01bd6a 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java
@@ -104,6 +104,11 @@ public class JsonXContentGenerator implements XContentGenerator {
         prettyPrint = true;
     }
 
+    @Override
+    public boolean isPrettyPrint() {
+        return this.prettyPrint;
+    }
+
     @Override
     public void usePrintLineFeedAtEnd() {
         writeLineFeedAtEnd = true;
diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java b/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java
index 79b890ab401..9cbd4bb769d 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java
@@ -34,8 +34,6 @@ import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
-import java.util.Collections;
-
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 
@@ -81,7 +79,7 @@ public abstract class AbstractShapeBuilderTestCase<SB extends ShapeBuilder> exte
                 contentBuilder.prettyPrint();
             }
             XContentBuilder builder = testShape.toXContent(contentBuilder, ToXContent.EMPTY_PARAMS);
-            XContentBuilder shuffled = shuffleXContent(builder, Collections.emptySet());
+            XContentBuilder shuffled = shuffleXContent(builder);
             XContentParser shapeParser = XContentHelper.createParser(shuffled.bytes());
             shapeParser.nextToken();
             ShapeBuilder parsedShape = ShapeBuilder.parse(shapeParser);
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java
index ff23a822832..e04fdab12aa 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java
@@ -28,7 +28,6 @@ import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 
-import static java.util.Collections.emptySet;
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -48,7 +47,7 @@ public class ConstructingObjectParserTests extends ESTestCase {
         expected.setD(randomBoolean());
         XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
         expected.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        builder = shuffleXContent(builder, emptySet());
+        builder = shuffleXContent(builder);
         BytesReference bytes = builder.bytes();
         XContentParser parser = XContentFactory.xContent(bytes).createParser(bytes);
         try {
diff --git a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
index c2a5fe5aacd..078809b43f5 100644
--- a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
@@ -426,11 +426,11 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
     }
 
     /**
-     * Subclasses can override this method and return a set of fields which should be protected from
+     * Subclasses can override this method and return an array of fieldnames which should be protected from
      * recursive random shuffling in the {@link #testFromXContent()} test case
      */
-    protected Set<String> shuffleProtectedFields() {
-        return Collections.emptySet();
+    protected String[] shuffleProtectedFields() {
+        return new String[0];
     }
 
     protected static XContentBuilder toXContent(QueryBuilder query, XContentType contentType) throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
index c7106a7ccf6..676671f066e 100644
--- a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
@@ -24,7 +24,6 @@ import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.sameInstance;
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -93,7 +92,7 @@ public class InnerHitBuilderTests extends ESTestCase {
             InnerHitBuilder innerHit = randomInnerHits(true, false);
             XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
             innerHit.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            XContentBuilder shuffled = shuffleXContent(builder, Collections.emptySet());
+            XContentBuilder shuffled = shuffleXContent(builder);
             if (randomBoolean()) {
                 shuffled.prettyPrint();
             }
diff --git a/core/src/test/java/org/elasticsearch/index/query/PercolateQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/PercolateQueryBuilderTests.java
index 9dc0bdbd133..690b2c03a2f 100644
--- a/core/src/test/java/org/elasticsearch/index/query/PercolateQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/PercolateQueryBuilderTests.java
@@ -42,15 +42,12 @@ import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.util.Collections;
-import java.util.Set;
-
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
 public class PercolateQueryBuilderTests extends AbstractQueryTestCase<PercolateQueryBuilder> {
 
-    private static final Set<String> SHUFFLE_PROTECTED_FIELDS =
-            Collections.singleton(PercolateQueryBuilder.DOCUMENT_FIELD.getPreferredName());
+    private static final String[] SHUFFLE_PROTECTED_FIELDS = new String[] { PercolateQueryBuilder.DOCUMENT_FIELD.getPreferredName()};
 
     private static String queryField;
     private static String docType;
@@ -105,7 +102,7 @@ public class PercolateQueryBuilderTests extends AbstractQueryTestCase<PercolateQ
      * compare when check for equality of the original and the shuffled builder
      */
     @Override
-    protected Set<String> shuffleProtectedFields() {
+    protected String[] shuffleProtectedFields() {
         return SHUFFLE_PROTECTED_FIELDS;
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java b/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java
index 1a31df45575..67c431135a0 100644
--- a/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java
+++ b/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java
@@ -32,7 +32,6 @@ import org.elasticsearch.index.store.StoreFileMetaData;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
-
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
@@ -53,7 +52,7 @@ public class FileInfoTests extends ESTestCase {
             BlobStoreIndexShardSnapshot.FileInfo info = new BlobStoreIndexShardSnapshot.FileInfo("_foobar", meta, size);
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint();
             BlobStoreIndexShardSnapshot.FileInfo.toXContent(info, builder, ToXContent.EMPTY_PARAMS);
-            byte[] xcontent = builder.bytes().toBytes();
+            byte[] xcontent = shuffleXContent(builder).bytes().toBytes();
 
             final BlobStoreIndexShardSnapshot.FileInfo parsedInfo;
             try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(xcontent)) {
diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java b/core/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java
index 38c1684b7ff..7924d069f2e 100644
--- a/core/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/IngestMetadataTests.java
@@ -31,7 +31,6 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -57,7 +56,7 @@ public class IngestMetadataTests extends ESTestCase {
         builder.startObject();
         ingestMetadata.toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder.endObject();
-        XContentBuilder shuffled = shuffleXContent(builder, Collections.emptySet());
+        XContentBuilder shuffled = shuffleXContent(builder);
         final XContentParser parser = XContentFactory.xContent(shuffled.bytes()).createParser(shuffled.bytes());
         MetaData.Custom custom = ingestMetadata.fromXContent(parser);
         assertTrue(custom instanceof IngestMetadata);
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java b/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java
index 86c3fe561d6..c934de2dd72 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptMetaDataTests.java
@@ -34,8 +34,6 @@ import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
 
 
 public class ScriptMetaDataTests extends ESTestCase {
@@ -79,7 +77,7 @@ public class ScriptMetaDataTests extends ESTestCase {
         xContentBuilder.startObject();
         expected.toXContent(xContentBuilder, new ToXContent.MapParams(Collections.emptyMap()));
         xContentBuilder.endObject();
-        xContentBuilder = shuffleXContent(xContentBuilder, Collections.emptySet());
+        xContentBuilder = shuffleXContent(xContentBuilder);
 
         XContentParser parser = XContentHelper.createParser(xContentBuilder.bytes());
         parser.nextToken();
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
index 329b90058e4..4af87c982e1 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
@@ -223,7 +223,7 @@ public abstract class BaseAggregationTestCase<AB extends AggregatorBuilder<AB>>
             builder.prettyPrint();
         }
         factoriesBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        XContentBuilder shuffled = shuffleXContent(builder, Collections.emptySet());
+        XContentBuilder shuffled = shuffleXContent(builder);
         XContentParser parser = XContentFactory.xContent(shuffled.bytes()).createParser(shuffled.bytes());
         QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
         assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
index daee1782f4b..2acbd919484 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
@@ -225,7 +225,7 @@ public abstract class BasePipelineAggregationTestCase<AF extends PipelineAggrega
             builder.prettyPrint();
         }
         factoriesBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        XContentBuilder shuffled = shuffleXContent(builder, Collections.emptySet());
+        XContentBuilder shuffled = shuffleXContent(builder);
         XContentParser parser = XContentFactory.xContent(shuffled.bytes()).createParser(shuffled.bytes());
         QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
         String contentString = factoriesBuilder.toString();
diff --git a/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java b/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java
index 5e706d1f509..d5c6bf98835 100644
--- a/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java
@@ -59,7 +59,6 @@ import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -150,7 +149,7 @@ public class HighlightBuilderTests extends ESTestCase {
                 builder.prettyPrint();
             }
             highlightBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            XContentBuilder shuffled = shuffleXContent(builder, Collections.emptySet());
+            XContentBuilder shuffled = shuffleXContent(builder);
 
             XContentParser parser = XContentHelper.createParser(shuffled.bytes());
             QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.EMPTY);
diff --git a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
index fb179dc7c16..d0f2c0492da 100644
--- a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
@@ -54,8 +54,6 @@ import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
-import java.util.Collections;
-
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 
@@ -137,7 +135,7 @@ public class QueryRescoreBuilderTests extends ESTestCase {
                 builder.prettyPrint();
             }
             rescoreBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            XContentBuilder shuffled = shuffleXContent(builder, Collections.emptySet());
+            XContentBuilder shuffled = shuffleXContent(builder);
 
 
             XContentParser parser = XContentHelper.createParser(shuffled.bytes());
diff --git a/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java b/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java
index 14bf8eac7e6..3c675926328 100644
--- a/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java
@@ -41,7 +41,6 @@ import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
-
 import static org.hamcrest.Matchers.equalTo;
 
 public class SearchAfterBuilderTests extends ESTestCase {
@@ -220,8 +219,8 @@ public class SearchAfterBuilderTests extends ESTestCase {
             builder.startObject();
             searchAfterBuilder.innerToXContent(builder);
             builder.endObject();
-            XContentParser parser = XContentHelper.createParser(builder.bytes());
-            QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
+            XContentParser parser = XContentHelper.createParser(shuffleXContent(builder).bytes());
+            new QueryParseContext(indicesQueriesRegistry, parser, ParseFieldMatcher.STRICT);
             parser.nextToken();
             parser.nextToken();
             parser.nextToken();
diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
index 01a8a07d5bb..3f2bd6e9a56 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
@@ -137,7 +137,7 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST
                 builder.prettyPrint();
             }
             testItem.toXContent(builder, ToXContent.EMPTY_PARAMS);
-            XContentBuilder shuffled = shuffleXContent(builder, Collections.emptySet());
+            XContentBuilder shuffled = shuffleXContent(builder);
             XContentParser itemParser = XContentHelper.createParser(shuffled.bytes());
             itemParser.nextToken();
 
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
index ed1bc323557..1f4030f487c 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
@@ -39,9 +39,6 @@ import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
-import java.util.Collections;
-import java.util.Set;
-
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 
@@ -173,8 +170,8 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
      * Subclasses can override this method and return a set of fields which should be protected from
      * recursive random shuffling in the {@link #testFromXContent()} test case
      */
-    protected Set<String> shuffleProtectedFields() {
-        return Collections.emptySet();
+    protected String[] shuffleProtectedFields() {
+        return new String[0];
     }
 
     private SB mutate(SB firstBuilder) throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggesterBuilderTests.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggesterBuilderTests.java
index af728647212..04412d47065 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggesterBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/CompletionSuggesterBuilderTests.java
@@ -37,14 +37,11 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Set;
-
 import static org.hamcrest.Matchers.containsString;
 
 public class CompletionSuggesterBuilderTests extends AbstractSuggestionBuilderTestCase<CompletionSuggestionBuilder> {
 
-    private static final Set<String> SHUFFLE_PROTECTED_FIELDS =
-            Collections.singleton(CompletionSuggestionBuilder.CONTEXTS_FIELD.getPreferredName());
+    private static final String[] SHUFFLE_PROTECTED_FIELDS = new String[] {CompletionSuggestionBuilder.CONTEXTS_FIELD.getPreferredName()};
 
     @Override
     protected CompletionSuggestionBuilder randomSuggestionBuilder() {
@@ -113,7 +110,7 @@ public class CompletionSuggesterBuilderTests extends AbstractSuggestionBuilderTe
      * the equals() test will fail because their {@link BytesReference} representation isn't the same
      */
     @Override
-    protected Set<String> shuffleProtectedFields() {
+    protected String[] shuffleProtectedFields() {
         return SHUFFLE_PROTECTED_FIELDS;
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java
index a3a5ea01c33..3fd3850b98a 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java
@@ -35,7 +35,6 @@ import org.elasticsearch.search.suggest.phrase.PhraseSuggestionContext.DirectCan
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
-
 import static org.hamcrest.Matchers.equalTo;
 
 public class DirectCandidateGeneratorTests extends ESTestCase{
@@ -117,8 +116,7 @@ public class DirectCandidateGeneratorTests extends ESTestCase{
                 builder.prettyPrint();
             }
             generator.toXContent(builder, ToXContent.EMPTY_PARAMS);
-
-            XContentParser parser = XContentHelper.createParser(builder.bytes());
+            XContentParser parser = XContentHelper.createParser(shuffleXContent(builder).bytes());
             QueryParseContext context = new QueryParseContext(mockRegistry, parser, ParseFieldMatcher.STRICT);
             parser.nextToken();
             DirectCandidateGeneratorBuilder secondGenerator = DirectCandidateGeneratorBuilder.fromXContent(context);
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java
index 1731ddd7c4f..f167eefa43d 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java
@@ -100,7 +100,7 @@ public abstract class SmoothingModelTestCase extends ESTestCase {
         contentBuilder.startObject();
         testModel.innerToXContent(contentBuilder, ToXContent.EMPTY_PARAMS);
         contentBuilder.endObject();
-        XContentParser parser = XContentHelper.createParser(contentBuilder.bytes());
+        XContentParser parser = XContentHelper.createParser(shuffleXContent(contentBuilder).bytes());
         QueryParseContext context = new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.STRICT);
         parser.nextToken();  // go to start token, real parsing would do that in the outer element parser
         SmoothingModel parsedModel = fromXContent(context);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 01f0d5b8151..3dbbf25e202 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -414,7 +414,7 @@ public abstract class ESTestCase extends LuceneTestCase {
         if (input != null) {
             return randomValueOtherThanMany(input::equals, randomSupplier);
         }
-        
+
         return(randomSupplier.get());
     }
 
@@ -634,25 +634,29 @@ public abstract class ESTestCase extends LuceneTestCase {
      * recursive shuffling behavior can be made by passing in the names of fields which
      * internally should stay untouched.
      */
-    public static XContentBuilder shuffleXContent(XContentBuilder builder, Set<String> exceptFieldNames) throws IOException {
+    public static XContentBuilder shuffleXContent(XContentBuilder builder, String... exceptFieldNames) throws IOException {
         BytesReference bytes = builder.bytes();
         XContentParser parser = XContentFactory.xContent(bytes).createParser(bytes);
         // use ordered maps for reproducibility
-        Map<String, Object> shuffledMap = shuffleMap(parser.mapOrdered(), exceptFieldNames);
-        XContentBuilder jsonBuilder = XContentFactory.contentBuilder(builder.contentType());
-        return jsonBuilder.map(shuffledMap);
+        Map<String, Object> shuffledMap = shuffleMap(parser.mapOrdered(), new HashSet<>(Arrays.asList(exceptFieldNames)));
+        XContentBuilder xContentBuilder = XContentFactory.contentBuilder(builder.contentType());
+        if (builder.isPrettyPrint()) {
+            xContentBuilder.prettyPrint();
+        }
+        return xContentBuilder.map(shuffledMap);
     }
 
-    private static Map<String, Object> shuffleMap(Map<String, Object> map, Set<String> exceptFieldNames) {
+    private static Map<String, Object> shuffleMap(Map<String, Object> map, Set<String> exceptFields) {
         List<String> keys = new ArrayList<>(map.keySet());
+
         // even though we shuffle later, we need this to make tests reproduce on different jvms
         Collections.sort(keys);
         Map<String, Object> targetMap = new TreeMap<>();
         Collections.shuffle(keys, random());
         for (String key : keys) {
             Object value = map.get(key);
-            if (value instanceof Map && exceptFieldNames.contains(key) == false) {
-                targetMap.put(key, shuffleMap((Map) value, exceptFieldNames));
+            if (value instanceof Map && exceptFields.contains(key) == false) {
+                targetMap.put(key, shuffleMap((Map) value, exceptFields));
             } else {
                 targetMap.put(key, value);
             }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/ESTestCaseTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/ESTestCaseTests.java
index 714354cc24c..005cd643480 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/test/ESTestCaseTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/test/ESTestCaseTests.java
@@ -29,11 +29,9 @@ import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.hamcrest.Matchers.greaterThan;
@@ -71,7 +69,7 @@ public class ESTestCaseTests extends ESTestCase {
         Map<String, Object> randomStringObjectMap = randomStringObjectMap(5);
         XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
         builder.map(randomStringObjectMap);
-        XContentBuilder shuffleXContent = shuffleXContent(builder, Collections.emptySet());
+        XContentBuilder shuffleXContent = shuffleXContent(builder);
         XContentParser parser = XContentFactory.xContent(shuffleXContent.bytes()).createParser(shuffleXContent.bytes());
         Map<String, Object> resultMap = parser.map();
         assertEquals("both maps should contain the same mappings", randomStringObjectMap, resultMap);

From e88ac116335d9217d5b9b13ca6bb2575c6d52611 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Fri, 6 May 2016 09:34:57 +0200
Subject: [PATCH 0101/1311] Add back Version.V_5_0_0. #18176

This was lost whene releasing alpha2 since the version constant got renamed.
---
 buildSrc/version.properties                       | 2 +-
 core/src/main/java/org/elasticsearch/Version.java | 6 +++++-
 qa/backwards-5.0/build.gradle                     | 2 +-
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index 0f6a09327d6..fee8404080a 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,4 +1,4 @@
-elasticsearch     = 5.0.0-alpha2
+elasticsearch     = 5.0.0
 lucene            = 6.0.0
 
 # optional dependencies
diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java
index 56d245ddc51..0e869d06149 100644
--- a/core/src/main/java/org/elasticsearch/Version.java
+++ b/core/src/main/java/org/elasticsearch/Version.java
@@ -73,7 +73,9 @@ public class Version {
     public static final Version V_5_0_0_alpha1 = new Version(V_5_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_6_0_0);
     public static final int V_5_0_0_alpha2_ID = 5000002;
     public static final Version V_5_0_0_alpha2 = new Version(V_5_0_0_alpha2_ID, org.apache.lucene.util.Version.LUCENE_6_0_0);
-    public static final Version CURRENT = V_5_0_0_alpha2;
+    public static final int V_5_0_0_ID = 5000099;
+    public static final Version V_5_0_0 = new Version(V_5_0_0_ID, org.apache.lucene.util.Version.LUCENE_6_0_0);
+    public static final Version CURRENT = V_5_0_0;
 
     static {
         assert CURRENT.luceneVersion.equals(org.apache.lucene.util.Version.LATEST) : "Version must be upgraded to ["
@@ -86,6 +88,8 @@ public class Version {
 
     public static Version fromId(int id) {
         switch (id) {
+            case V_5_0_0_ID:
+                return V_5_0_0;
             case V_5_0_0_alpha2_ID:
                 return V_5_0_0_alpha2;
             case V_5_0_0_alpha1_ID:
diff --git a/qa/backwards-5.0/build.gradle b/qa/backwards-5.0/build.gradle
index 164f0e53d52..93d361c989c 100644
--- a/qa/backwards-5.0/build.gradle
+++ b/qa/backwards-5.0/build.gradle
@@ -18,6 +18,6 @@ integTest {
   cluster {
     numNodes = 2
     numBwcNodes = 1
-    bwcVersion = "5.0.0-alpha2-SNAPSHOT" // this is the same as the current version until we released the first RC
+    bwcVersion = "5.0.0-SNAPSHOT" // this is the same as the current version until we released the first RC
   }
 }

From 93567a2f1b3c93f7ede97da6fa6fb6a2657aaa5e Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Fri, 6 May 2016 12:51:11 +0200
Subject: [PATCH 0102/1311] Remove StringBuilder reuse for uid creation. #18181

This would be better handled by escape analysis.
---
 .../index/mapper/ParseContext.java            | 23 -------------------
 .../org/elasticsearch/index/mapper/Uid.java   | 13 +----------
 .../mapper/internal/ParentFieldMapper.java    |  3 +--
 .../index/mapper/internal/UidFieldMapper.java |  3 +--
 4 files changed, 3 insertions(+), 39 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java b/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
index 931fbebede2..33476b48cc3 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
@@ -326,11 +326,6 @@ public abstract class ParseContext {
             return in.externalValue();
         }
 
-        @Override
-        public StringBuilder stringBuilder() {
-            return in.stringBuilder();
-        }
-
         @Override
         public void addDynamicMapper(Mapper update) {
             in.addDynamicMapper(update);
@@ -366,8 +361,6 @@ public abstract class ParseContext {
 
         private Field uid, version;
 
-        private StringBuilder stringBuilder = new StringBuilder();
-
         private AllEntries allEntries = new AllEntries();
 
         private List<Mapper> dynamicMappers = new ArrayList<>();
@@ -526,16 +519,6 @@ public abstract class ParseContext {
             return this.allEntries;
         }
 
-        /**
-         * A string builder that can be used to construct complex names for example.
-         * Its better to reuse the.
-         */
-        @Override
-        public StringBuilder stringBuilder() {
-            stringBuilder.setLength(0);
-            return this.stringBuilder;
-        }
-
         @Override
         public void addDynamicMapper(Mapper mapper) {
             dynamicMappers.add(mapper);
@@ -736,12 +719,6 @@ public abstract class ParseContext {
         return clazz.cast(externalValue());
     }
 
-    /**
-     * A string builder that can be used to construct complex names for example.
-     * Its better to reuse the.
-     */
-    public abstract StringBuilder stringBuilder();
-
     /**
      * Add a new mapper dynamically created while parsing.
      */
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/Uid.java b/core/src/main/java/org/elasticsearch/index/mapper/Uid.java
index 414a00d8eed..70acbcd82c3 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/Uid.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/Uid.java
@@ -123,13 +123,6 @@ public final class Uid {
         return ref;
     }
 
-    public static BytesRef createUidAsBytes(BytesRef type, BytesRef id, BytesRefBuilder spare) {
-        spare.copyBytes(type);
-        spare.append(DELIMITER_BYTES);
-        spare.append(id);
-        return spare.get();
-    }
-
     public static BytesRef[] createUidsForTypesAndId(Collection<String> types, Object id) {
         return createUidsForTypesAndIds(types, Collections.singletonList(id));
     }
@@ -149,11 +142,7 @@ public final class Uid {
     }
 
     public static String createUid(String type, String id) {
-        return createUid(new StringBuilder(), type, id);
-    }
-
-    public static String createUid(StringBuilder sb, String type, String id) {
-        return sb.append(type).append(DELIMITER).append(id).toString();
+        return type + DELIMITER + id;
     }
 
     public static boolean hasDelimiter(BytesRef uid) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
index 9473297de6f..2bf4f75f417 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
@@ -31,7 +31,6 @@ import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.loader.SettingsLoader;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -264,7 +263,7 @@ public class ParentFieldMapper extends MetadataFieldMapper {
                     }
                     // we did not add it in the parsing phase, add it now
                     fields.add(new SortedDocValuesField(fieldType.name(), new BytesRef(parentId)));
-                } else if (parentId != null && !parsedParentId.equals(Uid.createUid(context.stringBuilder(), parentType, parentId))) {
+                } else if (parentId != null && !parsedParentId.equals(Uid.createUid(parentType, parentId))) {
                     throw new MapperParsingException("Parent id mismatch, document value is [" + Uid.createUid(parsedParentId).id() + "], while external value is [" + parentId + "]");
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
index e150a8cd10f..b22369ef812 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
@@ -23,7 +23,6 @@ import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.Term;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
@@ -177,7 +176,7 @@ public class UidFieldMapper extends MetadataFieldMapper {
 
     @Override
     protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
-        Field uid = new Field(NAME, Uid.createUid(context.stringBuilder(), context.type(), context.id()), Defaults.FIELD_TYPE);
+        Field uid = new Field(NAME, Uid.createUid(context.type(), context.id()), Defaults.FIELD_TYPE);
         context.uid(uid);
         fields.add(uid);
         if (fieldType().hasDocValues()) {

From d3c5f865be75577a96c4014c07d104ab0dcf60e3 Mon Sep 17 00:00:00 2001
From: Britta Weber <britta.weber@elasticsearch.com>
Date: Fri, 6 May 2016 13:20:28 +0200
Subject: [PATCH 0103/1311] Exclude all but string fields from highlighting if
 wildcards are used in fieldname

We should prevent highlighting if a field is anything but a text or keyword field.
However, someone might implement a custom field type that has text and still want to
highlight on that. We cannot know in advance if the highlighter will be able to
highlight such a field and so we do the following:
If the field is only highlighted because the field matches a wildcard we assume
it was a mistake and do not process it.
If the field was explicitly given we assume that whoever issued the query knew
what they were doing and try to highlight anyway.

closes #17537
---
 .../search/highlight/HighlightPhase.java      |  16 ++
 .../externalvalues/ExternalMapperPlugin.java  |   3 +-
 .../ExternalValuesMapperIntegrationIT.java    |  50 +++++
 .../externalvalues/FakeStringFieldMapper.java | 193 ++++++++++++++++++
 .../search/highlight/HighlighterSearchIT.java |  30 ++-
 .../search/request/highlighting.asciidoc      |   4 +-
 6 files changed, 293 insertions(+), 3 deletions(-)
 create mode 100755 core/src/test/java/org/elasticsearch/index/mapper/externalvalues/FakeStringFieldMapper.java

diff --git a/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java b/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java
index b6f41135025..591aaab01d1 100644
--- a/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java
@@ -26,6 +26,8 @@ import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.core.StringFieldMapper;
+import org.elasticsearch.index.mapper.core.TextFieldMapper;
 import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
 import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.fetch.FetchSubPhase;
@@ -102,6 +104,20 @@ public class HighlightPhase extends AbstractComponent implements FetchSubPhase {
                     continue;
                 }
 
+                // We should prevent highlighting if a field is anything but a text or keyword field.
+                // However, someone might implement a custom field type that has text and still want to
+                // highlight on that. We cannot know in advance if the highlighter will be able to
+                // highlight such a field and so we do the following:
+                // If the field is only highlighted because the field matches a wildcard we assume
+                // it was a mistake and do not process it.
+                // If the field was explicitly given we assume that whoever issued the query knew
+                // what they were doing and try to highlight anyway.
+                if (fieldNameContainsWildcards) {
+                    if (fieldMapper.fieldType().typeName().equals(TextFieldMapper.CONTENT_TYPE) == false && fieldMapper.fieldType()
+                        .typeName().equals(StringFieldMapper.CONTENT_TYPE) == false) {
+                        continue;
+                    }
+                }
                 String highlighterType = field.fieldOptions().highlighterType();
                 if (highlighterType == null) {
                     for(String highlighterCandidate : STANDARD_HIGHLIGHTERS_BY_PRECEDENCE) {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java
index 863e0c25fb0..87daaa58769 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java
@@ -43,6 +43,7 @@ public class ExternalMapperPlugin extends Plugin {
         indicesModule.registerMapper(EXTERNAL, new ExternalMapper.TypeParser(EXTERNAL, "foo"));
         indicesModule.registerMapper(EXTERNAL_BIS, new ExternalMapper.TypeParser(EXTERNAL_BIS, "bar"));
         indicesModule.registerMapper(EXTERNAL_UPPER, new ExternalMapper.TypeParser(EXTERNAL_UPPER, "FOO BAR"));
+        indicesModule.registerMapper(FakeStringFieldMapper.CONTENT_TYPE, new FakeStringFieldMapper.TypeParser());
     }
 
-}
\ No newline at end of file
+}
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalValuesMapperIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalValuesMapperIntegrationIT.java
index 171245841a0..350cbc43f9a 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalValuesMapperIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalValuesMapperIntegrationIT.java
@@ -25,10 +25,12 @@ import org.elasticsearch.common.geo.builders.ShapeBuilders;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.highlight.HighlightBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.Collection;
 
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.equalTo;
 
 public class ExternalValuesMapperIntegrationIT extends ESIntegTestCase {
@@ -37,6 +39,54 @@ public class ExternalValuesMapperIntegrationIT extends ESIntegTestCase {
         return pluginList(ExternalMapperPlugin.class);
     }
 
+    public void testHighlightingOnCustomString() throws Exception {
+        prepareCreate("test-idx").addMapping("type",
+            XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties")
+                .startObject("field").field("type", FakeStringFieldMapper.CONTENT_TYPE).endObject()
+                .endObject()
+                .endObject().endObject()).execute().get();
+        ensureYellow("test-idx");
+
+        index("test-idx", "type", "1", XContentFactory.jsonBuilder()
+            .startObject()
+            .field("field", "Every day is exactly the same")
+            .endObject());
+        refresh();
+
+        SearchResponse response;
+        // test if the highlighting is excluded when we use wildcards
+        response = client().prepareSearch("test-idx")
+            .setQuery(QueryBuilders.matchQuery("field", "exactly the same"))
+            .highlighter(new HighlightBuilder().field("*"))
+            .execute().actionGet();
+        assertSearchResponse(response);
+        assertThat(response.getHits().getTotalHits(), equalTo(1L));
+        assertThat(response.getHits().getAt(0).getHighlightFields().size(), equalTo(0));
+
+        // make sure it is not excluded when we explicitly provide the fieldname
+        response = client().prepareSearch("test-idx")
+            .setQuery(QueryBuilders.matchQuery("field", "exactly the same"))
+            .highlighter(new HighlightBuilder().field("field"))
+            .execute().actionGet();
+        assertSearchResponse(response);
+        assertThat(response.getHits().getTotalHits(), equalTo(1L));
+        assertThat(response.getHits().getAt(0).getHighlightFields().size(), equalTo(1));
+        assertThat(response.getHits().getAt(0).getHighlightFields().get("field").fragments()[0].string(), equalTo("Every day is " +
+            "<em>exactly</em> <em>the</em> <em>same</em>"));
+
+        // make sure it is not excluded when we explicitly provide the fieldname and a wildcard
+        response = client().prepareSearch("test-idx")
+            .setQuery(QueryBuilders.matchQuery("field", "exactly the same"))
+            .highlighter(new HighlightBuilder().field("*").field("field"))
+            .execute().actionGet();
+        assertSearchResponse(response);
+        assertThat(response.getHits().getTotalHits(), equalTo(1L));
+        assertThat(response.getHits().getAt(0).getHighlightFields().size(), equalTo(1));
+        assertThat(response.getHits().getAt(0).getHighlightFields().get("field").fragments()[0].string(), equalTo("Every day is " +
+            "<em>exactly</em> <em>the</em> <em>same</em>"));
+    }
+
     public void testExternalValues() throws Exception {
         prepareCreate("test-idx").addMapping("type",
                 XContentFactory.jsonBuilder().startObject().startObject("type")
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/FakeStringFieldMapper.java b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/FakeStringFieldMapper.java
new file mode 100755
index 00000000000..e0c1243f82f
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/FakeStringFieldMapper.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper.externalvalues;
+
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.MultiTermQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.RegexpQuery;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.core.StringFieldMapper;
+import org.elasticsearch.index.query.QueryShardContext;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.index.mapper.core.TypeParsers.parseTextField;
+
+// Like a String mapper but with very few options. We just use it to test if highlighting on a custom string mapped field works as expected.
+public class FakeStringFieldMapper extends FieldMapper {
+
+    public static final String CONTENT_TYPE = "fake_string";
+
+    public static class Defaults {
+
+        public static final MappedFieldType FIELD_TYPE = new FakeStringFieldType();
+
+        static {
+            FIELD_TYPE.freeze();
+        }
+    }
+
+    public static class Builder extends FieldMapper.Builder<Builder, FakeStringFieldMapper> {
+
+        public Builder(String name) {
+            super(name, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
+            builder = this;
+        }
+
+        @Override
+        public FakeStringFieldType fieldType() {
+            return (FakeStringFieldType) super.fieldType();
+        }
+
+        @Override
+        public FakeStringFieldMapper build(BuilderContext context) {
+            setupFieldType(context);
+            return new FakeStringFieldMapper(
+                name, fieldType(), defaultFieldType,
+                context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
+        }
+    }
+
+    public static class TypeParser implements Mapper.TypeParser {
+
+        public TypeParser() {
+        }
+
+        @Override
+        public Mapper.Builder parse(String fieldName, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+            FakeStringFieldMapper.Builder builder = new FakeStringFieldMapper.Builder(fieldName);
+            parseTextField(builder, fieldName, node, parserContext);
+            return builder;
+        }
+    }
+
+    public static final class FakeStringFieldType extends MappedFieldType {
+
+
+        public FakeStringFieldType() {
+        }
+
+        protected FakeStringFieldType(FakeStringFieldType ref) {
+            super(ref);
+        }
+
+        public FakeStringFieldType clone() {
+            return new FakeStringFieldType(this);
+        }
+
+        @Override
+        public String typeName() {
+            return CONTENT_TYPE;
+        }
+
+        @Override
+        public Query nullValueQuery() {
+            if (nullValue() == null) {
+                return null;
+            }
+            return termQuery(nullValue(), null);
+        }
+
+        @Override
+        public Query regexpQuery(String value, int flags, int maxDeterminizedStates, @Nullable MultiTermQuery.RewriteMethod method,
+                                 @Nullable QueryShardContext context) {
+            RegexpQuery query = new RegexpQuery(new Term(name(), indexedValueForSearch(value)), flags, maxDeterminizedStates);
+            if (method != null) {
+                query.setRewriteMethod(method);
+            }
+            return query;
+        }
+    }
+
+    protected FakeStringFieldMapper(String simpleName, FakeStringFieldType fieldType, MappedFieldType defaultFieldType,
+                                    Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
+        super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
+    }
+
+    @Override
+    protected StringFieldMapper clone() {
+        return (StringFieldMapper) super.clone();
+    }
+
+    @Override
+    protected boolean customBoost() {
+        return true;
+    }
+
+    @Override
+    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
+        StringFieldMapper.ValueAndBoost valueAndBoost = parseCreateFieldForString(context, fieldType().boost());
+        if (valueAndBoost.value() == null) {
+            return;
+        }
+        if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
+            Field field = new Field(fieldType().name(), valueAndBoost.value(), fieldType());
+            fields.add(field);
+        }
+        if (fieldType().hasDocValues()) {
+            fields.add(new SortedSetDocValuesField(fieldType().name(), new BytesRef(valueAndBoost.value())));
+        }
+    }
+
+    public static StringFieldMapper.ValueAndBoost parseCreateFieldForString(ParseContext context, float defaultBoost) throws IOException {
+        if (context.externalValueSet()) {
+            return new StringFieldMapper.ValueAndBoost(context.externalValue().toString(), defaultBoost);
+        }
+        XContentParser parser = context.parser();
+        return new StringFieldMapper.ValueAndBoost(parser.textOrNull(), defaultBoost);
+    }
+
+    @Override
+    protected String contentType() {
+        return CONTENT_TYPE;
+    }
+
+    @Override
+    protected void doMerge(Mapper mergeWith, boolean updateAllTypes) {
+        super.doMerge(mergeWith, updateAllTypes);
+    }
+
+    @Override
+    public FakeStringFieldType fieldType() {
+        return (FakeStringFieldType) super.fieldType();
+    }
+
+    @Override
+    protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
+        super.doXContentBody(builder, includeDefaults, params);
+        doXContentAnalyzers(builder, includeDefaults);
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java b/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java
index b56588e1759..95be99ac3b6 100644
--- a/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.search.highlight;
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
@@ -2542,4 +2541,33 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         response = search.setQuery(boostingQuery(phrase, terms).boost(1).negativeBoost(1/boost)).get();
         assertHighlight(response, 0, "field1", 0, 1, highlightedMatcher);
     }
+
+    public void testGeoFieldHighlighting() throws IOException {
+        // check that we do not get an exception for geo_point fields in case someone tries to highlight
+        // it accidential with a wildcard
+        // see https://github.com/elastic/elasticsearch/issues/17537
+        XContentBuilder mappings = jsonBuilder();
+        mappings.startObject();
+        mappings.startObject("type")
+            .startObject("properties")
+            .startObject("geo_point")
+            .field("type", "geo_point")
+            .endObject()
+            .endObject()
+            .endObject();
+        mappings.endObject();
+        assertAcked(prepareCreate("test")
+            .addMapping("type", mappings));
+        ensureYellow();
+
+        client().prepareIndex("test", "type", "1")
+            .setSource(jsonBuilder().startObject().field("geo_point", "60.12,100.34").endObject())
+            .get();
+        refresh();
+        SearchResponse search = client().prepareSearch().setSource(
+            new SearchSourceBuilder().query(QueryBuilders.geoBoundingBoxQuery("geo_point").setCorners(61.10078883158897, -170.15625,
+                -64.92354174306496, 118.47656249999999)).highlighter(new HighlightBuilder().field("*"))).get();
+        assertNoFailures(search);
+        assertThat(search.getHits().totalHits(), equalTo(1L));
+    }
 }
diff --git a/docs/reference/search/request/highlighting.asciidoc b/docs/reference/search/request/highlighting.asciidoc
index bce9f13ab26..bfb5ffae7b3 100644
--- a/docs/reference/search/request/highlighting.asciidoc
+++ b/docs/reference/search/request/highlighting.asciidoc
@@ -35,7 +35,9 @@ be used for highlighting if it mapped to have `store` set to `true`.
 ==================================
 
 The field name supports wildcard notation. For example, using `comment_*`
-will cause all fields that match the expression to be highlighted.
+will cause all <<text,text>> or <<keyword,keyword>> fields that match the expression to be highlighted.
+Note that all other fields will not be highlighted. If you use a custom mapper and want to
+highlight on a field anyway, you have to provide the field name explicitly.
 
 [[plain-highlighter]]
 ==== Plain highlighter

From 7b69e4ef43cbe0a2607df95dbd5924391c390856 Mon Sep 17 00:00:00 2001
From: Britta Weber <britta.weber@elasticsearch.com>
Date: Fri, 6 May 2016 14:29:19 +0200
Subject: [PATCH 0104/1311] keyword fields should also be highlighted

---
 .../search/highlight/HighlightPhase.java      |  5 ++--
 .../search/highlight/HighlighterSearchIT.java | 30 +++++++++++++++++++
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java b/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java
index 591aaab01d1..1c4a9edca2a 100644
--- a/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java
@@ -26,6 +26,7 @@ import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.FieldMapper;
+import org.elasticsearch.index.mapper.core.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.core.StringFieldMapper;
 import org.elasticsearch.index.mapper.core.TextFieldMapper;
 import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
@@ -113,8 +114,8 @@ public class HighlightPhase extends AbstractComponent implements FetchSubPhase {
                 // If the field was explicitly given we assume that whoever issued the query knew
                 // what they were doing and try to highlight anyway.
                 if (fieldNameContainsWildcards) {
-                    if (fieldMapper.fieldType().typeName().equals(TextFieldMapper.CONTENT_TYPE) == false && fieldMapper.fieldType()
-                        .typeName().equals(StringFieldMapper.CONTENT_TYPE) == false) {
+                    if (fieldMapper.fieldType().typeName().equals(TextFieldMapper.CONTENT_TYPE) == false && fieldMapper.fieldType().typeName().equals
+                        (KeywordFieldMapper.CONTENT_TYPE) == false) {
                         continue;
                     }
                 }
diff --git a/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java b/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java
index 95be99ac3b6..8b80c7657cf 100644
--- a/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java
@@ -2570,4 +2570,34 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         assertNoFailures(search);
         assertThat(search.getHits().totalHits(), equalTo(1L));
     }
+
+    public void testKeywordFieldHighlighting() throws IOException {
+        // check that we do not get an exception for geo_point fields in case someone tries to highlight
+        // it accidential with a wildcard
+        // see https://github.com/elastic/elasticsearch/issues/17537
+        XContentBuilder mappings = jsonBuilder();
+        mappings.startObject();
+        mappings.startObject("type")
+            .startObject("properties")
+            .startObject("keyword_field")
+            .field("type", "keyword")
+            .endObject()
+            .endObject()
+            .endObject();
+        mappings.endObject();
+        assertAcked(prepareCreate("test")
+            .addMapping("type", mappings));
+        ensureYellow();
+
+        client().prepareIndex("test", "type", "1")
+            .setSource(jsonBuilder().startObject().field("keyword_field", "some text").endObject())
+            .get();
+        refresh();
+        SearchResponse search = client().prepareSearch().setSource(
+            new SearchSourceBuilder().query(QueryBuilders.matchQuery("keyword_field", "some text")).highlighter(new HighlightBuilder().field("*")))
+            .get();
+        assertNoFailures(search);
+        assertThat(search.getHits().totalHits(), equalTo(1L));
+        assertThat(search.getHits().getAt(0).getHighlightFields().get("keyword_field").getFragments()[0].string(), equalTo("<em>some text</em>"));
+    }
 }

From b12a42351ebe6f5b90d4a22110e4c6e2d8608378 Mon Sep 17 00:00:00 2001
From: Alexander Reelsen <alexander@reelsen.net>
Date: Fri, 6 May 2016 15:00:41 +0200
Subject: [PATCH 0105/1311] Pipeline Stats: Fix concurrent modification
 exception (#18177)

Due to trying to modify a map while iterating it, a concurrent modification
in the pipeline stats could be thrown. This uses an iterator to prevent this.

Closes #18126
---
 .../ingest/PipelineExecutionService.java      |  7 ++++--
 .../ingest/PipelineExecutionServiceTests.java | 22 +++++++++++++++++--
 2 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java b/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java
index 94c79db30a0..8674e805974 100644
--- a/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java
+++ b/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java
@@ -35,6 +35,7 @@ import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
@@ -124,9 +125,11 @@ public class PipelineExecutionService implements ClusterStateListener {
     void updatePipelineStats(IngestMetadata ingestMetadata) {
         boolean changed = false;
         Map<String, StatsHolder> newStatsPerPipeline = new HashMap<>(statsHolderPerPipeline);
-        for (String pipeline : newStatsPerPipeline.keySet()) {
+        Iterator<String> iterator = newStatsPerPipeline.keySet().iterator();
+        while (iterator.hasNext()) {
+            String pipeline = iterator.next();
             if (ingestMetadata.getPipelines().containsKey(pipeline) == false) {
-                newStatsPerPipeline.remove(pipeline);
+                iterator.remove();
                 changed = true;
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
index b84ba928be4..3c0de328c8c 100644
--- a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
@@ -46,11 +46,13 @@ import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 
 import static org.hamcrest.Matchers.equalTo;
-import static org.mockito.Matchers.anyBoolean;
-import static org.mockito.Matchers.eq;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.not;
 import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Matchers.argThat;
+import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
@@ -380,6 +382,22 @@ public class PipelineExecutionServiceTests extends ESTestCase {
         assertThat(ingestStats.getTotalStats().getIngestCount(), equalTo(2L));
     }
 
+    // issue: https://github.com/elastic/elasticsearch/issues/18126
+    public void testUpdatingStatsWhenRemovingPipelineWorks() throws Exception {
+        Map<String, PipelineConfiguration> configurationMap = new HashMap<>();
+        configurationMap.put("_id1", new PipelineConfiguration("_id1", new BytesArray("{}")));
+        configurationMap.put("_id2", new PipelineConfiguration("_id2", new BytesArray("{}")));
+        executionService.updatePipelineStats(new IngestMetadata(configurationMap));
+        assertThat(executionService.stats().getStatsPerPipeline(), hasKey("_id1"));
+        assertThat(executionService.stats().getStatsPerPipeline(), hasKey("_id2"));
+
+        configurationMap = new HashMap<>();
+        configurationMap.put("_id3", new PipelineConfiguration("_id3", new BytesArray("{}")));
+        executionService.updatePipelineStats(new IngestMetadata(configurationMap));
+        assertThat(executionService.stats().getStatsPerPipeline(), not(hasKey("_id1")));
+        assertThat(executionService.stats().getStatsPerPipeline(), not(hasKey("_id2")));
+    }
+
     private IngestDocument eqID(String index, String type, String id, Map<String, Object> source) {
         return argThat(new IngestDocumentMatcher(index, type, id, source));
     }

From b6698c3145b78c5f996ea52432abd12447866218 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 5 May 2016 18:07:54 -0400
Subject: [PATCH 0106/1311] Random script fields can't overlap

This causes round tripping through xcontent to fail.

Closes #18166
---
 .../elasticsearch/index/query/InnerHitBuilderTests.java   | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
index 676671f066e..d4ba6ca9062 100644
--- a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
@@ -86,7 +86,6 @@ public class InnerHitBuilderTests extends ESTestCase {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/18166")
     public void testFromAndToXContent() throws Exception {
         for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) {
             InnerHitBuilder innerHit = randomInnerHits(true, false);
@@ -221,7 +220,12 @@ public class InnerHitBuilderTests extends ESTestCase {
         innerHits.setTrackScores(randomBoolean());
         innerHits.setFieldNames(randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16)));
         innerHits.setFieldDataFields(randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16)));
-        innerHits.setScriptFields(new HashSet<>(randomListStuff(16, InnerHitBuilderTests::randomScript)));
+        // Random script fields deduped on their field name.
+        Map<String, SearchSourceBuilder.ScriptField> scriptFields = new HashMap<>();
+        for (SearchSourceBuilder.ScriptField field: randomListStuff(16, InnerHitBuilderTests::randomScript)) {
+            scriptFields.put(field.fieldName(), field);
+        }
+        innerHits.setScriptFields(new HashSet<>(scriptFields.values()));
         FetchSourceContext randomFetchSourceContext;
         if (randomBoolean()) {
             randomFetchSourceContext = new FetchSourceContext(randomBoolean());

From e839dad978ed66ecec2876432fb5688df83d8aa5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Thu, 5 May 2016 20:00:22 +0200
Subject: [PATCH 0107/1311] Remove unused chechstyle_supressions line length
 checks

The files removed are below the 140 character line length limit.
---
 .../resources/checkstyle_suppressions.xml     | 91 -------------------
 1 file changed, 91 deletions(-)

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 0473f58cf73..bebe07612b0 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -21,7 +21,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ActionRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ReplicationResponse.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]health[/\\]ClusterHealthRequestBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]health[/\\]ClusterHealthResponse.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]health[/\\]TransportClusterHealthAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]node[/\\]hotthreads[/\\]NodesHotThreadsRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]node[/\\]hotthreads[/\\]TransportNodesHotThreadsAction.java" checks="LineLength" />
@@ -61,7 +60,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]snapshots[/\\]status[/\\]TransportSnapshotsStatusAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]state[/\\]ClusterStateRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]state[/\\]TransportClusterStateAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]stats[/\\]ClusterStatsIndices.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]stats[/\\]ClusterStatsNodeResponse.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]stats[/\\]ClusterStatsRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]stats[/\\]TransportClusterStatsAction.java" checks="LineLength" />
@@ -178,21 +176,17 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]IngestActionFilter.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]IngestProxyActionFilter.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]PutPipelineTransportAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]SimulateExecutionService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]SimulatePipelineRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]SimulatePipelineRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]SimulatePipelineTransportAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]percolate[/\\]MultiPercolateRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]percolate[/\\]MultiPercolateRequestBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]percolate[/\\]PercolateRequestBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]percolate[/\\]PercolateResponse.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]percolate[/\\]PercolateShardResponse.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]percolate[/\\]TransportMultiPercolateAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]percolate[/\\]TransportPercolateAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]percolate[/\\]TransportShardMultiPercolateAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]MultiSearchRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]SearchPhaseExecutionException.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]SearchRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]SearchResponse.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]ShardSearchFailure.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]TransportClearScrollAction.java" checks="LineLength" />
@@ -201,10 +195,8 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]ActionFilter.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]AutoCreateIndex.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]DelegatingActionListener.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]DestructiveOperations.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]HandledTransportAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]IndicesOptions.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]ThreadedActionListener.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]ToXContentToBytes.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]broadcast[/\\]BroadcastOperationRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]broadcast[/\\]BroadcastRequest.java" checks="LineLength" />
@@ -216,13 +208,11 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]master[/\\]MasterNodeOperationRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]master[/\\]MasterNodeReadOperationRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]master[/\\]TransportMasterNodeAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]master[/\\]TransportMasterNodeReadAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]master[/\\]info[/\\]ClusterInfoRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]master[/\\]info[/\\]ClusterInfoRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]master[/\\]info[/\\]TransportClusterInfoAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]nodes[/\\]NodesOperationRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]nodes[/\\]TransportNodesAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]replication[/\\]ReplicationRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]replication[/\\]ReplicationRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]replication[/\\]TransportBroadcastReplicationAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]single[/\\]instance[/\\]InstanceShardOperationRequestBuilder.java" checks="LineLength" />
@@ -251,7 +241,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]JarHell.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]Seccomp.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]Security.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cache[/\\]recycler[/\\]PageCacheRecycler.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]ElasticsearchClient.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]FilterClient.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]node[/\\]NodeClient.java" checks="LineLength" />
@@ -267,7 +256,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]InternalClusterInfoService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]LocalNodeMasterListener.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]SnapshotsInProgress.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]action[/\\]index[/\\]MappingUpdatedAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]action[/\\]index[/\\]NodeIndexDeletedAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]action[/\\]index[/\\]NodeMappingRefreshAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]action[/\\]shard[/\\]ShardStateAction.java" checks="LineLength" />
@@ -375,8 +363,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]publish[/\\]PendingClusterStatesQueue.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]publish[/\\]PublishClusterStateAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]env[/\\]ESFileStore.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]env[/\\]Environment.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]env[/\\]NodeEnvironment.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]AsyncShardFetch.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]DanglingIndicesState.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]GatewayAllocator.java" checks="LineLength" />
@@ -392,21 +378,16 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexSettings.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexingSlowLog.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]MergePolicyConfig.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]MergeSchedulerConfig.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]NodeServicesProvider.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]SearchSlowLog.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]AnalysisRegistry.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]AnalysisService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]CommonGramsTokenFilterFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]CustomAnalyzerProvider.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]EdgeNGramTokenizerFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]NumericDoubleAnalyzer.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]ShingleTokenFilterFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]StemmerOverrideTokenFilterFactory.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]StopTokenFilterFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]compound[/\\]HyphenationCompoundWordTokenFilterFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]cache[/\\]bitset[/\\]BitsetFilterCache.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]cache[/\\]request[/\\]ShardRequestCache.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]codec[/\\]PerFieldMappingPostingFormatCodec.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]engine[/\\]ElasticsearchConcurrentMergeScheduler.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]engine[/\\]Engine.java" checks="LineLength" />
@@ -420,16 +401,13 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]fieldcomparator[/\\]FloatValuesComparatorSource.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]fieldcomparator[/\\]LongValuesComparatorSource.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]ordinals[/\\]GlobalOrdinalsBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]ordinals[/\\]GlobalOrdinalsIndexFieldData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]ordinals[/\\]InternalGlobalOrdinalsIndexFieldData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]ordinals[/\\]MultiOrdinals.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]ordinals[/\\]OrdinalsBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]ordinals[/\\]SinglePackedOrdinals.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]plain[/\\]AbstractAtomicParentChildFieldData.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]plain[/\\]AbstractIndexGeoPointFieldData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]plain[/\\]AbstractIndexOrdinalsFieldData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]plain[/\\]BinaryDVIndexFieldData.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]plain[/\\]GeoPointArrayIndexFieldData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]plain[/\\]PagedBytesIndexFieldData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]plain[/\\]ParentChildIndexFieldData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]plain[/\\]SortedNumericDVIndexFieldData.java" checks="LineLength" />
@@ -481,9 +459,7 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]percolator[/\\]PercolatorQueriesRegistry.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]AbstractQueryBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]MatchQueryParser.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]MoreLikeThisQueryBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]QueryBuilders.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]QueryShardContext.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]QueryValidationException.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]support[/\\]InnerHitsQueryParserHelper.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]support[/\\]QueryParsers.java" checks="LineLength" />
@@ -500,7 +476,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]ShardStateMetaData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]StoreRecovery.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]shard[/\\]TranslogRecoveryPerformer.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]similarity[/\\]SimilarityService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]snapshots[/\\]blobstore[/\\]BlobStoreIndexShardRepository.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]snapshots[/\\]blobstore[/\\]BlobStoreIndexShardSnapshots.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]store[/\\]IndexStore.java" checks="LineLength" />
@@ -530,28 +505,21 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]RecoveryFailedException.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]RecoverySettings.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]RecoverySource.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]RecoverySourceHandler.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]RecoveryState.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]recovery[/\\]StartRecoveryRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]store[/\\]IndicesStore.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]store[/\\]TransportNodesListShardStoreMetaData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]ttl[/\\]IndicesTTLService.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]IngestMetadata.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]PipelineExecutionService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]PipelineStore.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]CompoundProcessor.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]IngestDocument.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]Pipeline.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]ConvertProcessor.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]fs[/\\]FsService.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]DeadlockAnalyzer.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]GcNames.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]HotThreads.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]JvmGcMonitorService.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]JvmService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]JvmStats.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]os[/\\]OsService.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]process[/\\]ProcessService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]node[/\\]Node.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]node[/\\]internal[/\\]InternalSettingsPreparer.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolatorQuery.java" checks="LineLength" />
@@ -569,13 +537,11 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]fs[/\\]FsRepository.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]uri[/\\]URLIndexShardRepository.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]uri[/\\]URLRepository.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]BaseRestHandler.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]BytesRestResponse.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]RestController.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]admin[/\\]cluster[/\\]health[/\\]RestClusterHealthAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]admin[/\\]cluster[/\\]node[/\\]info[/\\]RestNodesInfoAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]admin[/\\]cluster[/\\]node[/\\]stats[/\\]RestNodesStatsAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]admin[/\\]cluster[/\\]reroute[/\\]RestClusterRerouteAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]admin[/\\]cluster[/\\]settings[/\\]RestClusterGetSettingsAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]admin[/\\]cluster[/\\]settings[/\\]RestClusterUpdateSettingsAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]admin[/\\]cluster[/\\]state[/\\]RestClusterStateAction.java" checks="LineLength" />
@@ -596,20 +562,16 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]bulk[/\\]RestBulkAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestCountAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestIndicesAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestNodeAttrsAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestNodesAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestPendingClusterTasksAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestShardsAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestThreadPoolAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]get[/\\]RestMultiGetAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]index[/\\]RestIndexAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]main[/\\]RestMainAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]percolate[/\\]RestPercolateAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]script[/\\]RestDeleteIndexedScriptAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]script[/\\]RestPutIndexedScriptAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]search[/\\]RestClearScrollAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]search[/\\]RestMultiSearchAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]search[/\\]RestSearchAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]search[/\\]RestSearchScrollAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]suggest[/\\]RestSuggestAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]support[/\\]RestActions.java" checks="LineLength" />
@@ -626,7 +588,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]MultiValueMode.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]SearchService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]AggregatorFactories.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]AggregatorFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]InternalAggregation.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]InternalMultiBucketAggregation.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]ValuesSourceAggregationBuilder.java" checks="LineLength" />
@@ -639,10 +600,8 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]filters[/\\]FiltersParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]filters[/\\]InternalFilters.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]geogrid[/\\]GeoHashGridAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]geogrid[/\\]GeoHashGridParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]global[/\\]GlobalAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]global[/\\]InternalGlobal.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]histogram[/\\]DateHistogramParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]histogram[/\\]HistogramAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]missing[/\\]InternalMissing.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]missing[/\\]MissingAggregator.java" checks="LineLength" />
@@ -651,10 +610,7 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]nested[/\\]ReverseNestedAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]range[/\\]InternalRange.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]range[/\\]RangeAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]range[/\\]date[/\\]DateRangeParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]range[/\\]date[/\\]InternalDateRange.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]range[/\\]geodistance[/\\]GeoDistanceParser.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]range[/\\]geodistance[/\\]InternalGeoDistance.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]range[/\\]ipv4[/\\]InternalIPv4Range.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]sampler[/\\]DiversifiedBytesHashSamplerAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]sampler[/\\]DiversifiedMapSamplerAggregator.java" checks="LineLength" />
@@ -665,15 +621,11 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]GlobalOrdinalsSignificantTermsAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]InternalSignificantTerms.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]SignificantLongTerms.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]SignificantLongTermsAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]SignificantStringTerms.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]SignificantStringTermsAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]SignificantTermsAggregatorFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]SignificantTermsParametersParser.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]SignificantTermsParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]UnmappedSignificantTerms.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]heuristics[/\\]GND.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]heuristics[/\\]JLHScore.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]heuristics[/\\]NXYSignificanceHeuristic.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]heuristics[/\\]PercentageScore.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]heuristics[/\\]ScriptHeuristic.java" checks="LineLength" />
@@ -691,37 +643,23 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]TermsAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]TermsAggregatorFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]TermsParametersParser.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]TermsParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]UnmappedTerms.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]support[/\\]IncludeExclude.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]ValuesSourceMetricsAggregationBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]cardinality[/\\]CardinalityAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]cardinality[/\\]CardinalityAggregatorFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]cardinality[/\\]HyperLogLogPlusPlus.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]geobounds[/\\]GeoBoundsAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]geobounds[/\\]InternalGeoBounds.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]percentiles[/\\]AbstractPercentilesParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]percentiles[/\\]tdigest[/\\]AbstractTDigestPercentilesAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]percentiles[/\\]tdigest[/\\]TDigestPercentileRanksAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]percentiles[/\\]tdigest[/\\]TDigestPercentilesAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]scripted[/\\]InternalScriptedMetric.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]scripted[/\\]ScriptedMetricAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]stats[/\\]StatsAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]stats[/\\]extended[/\\]ExtendedStatsAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]stats[/\\]extended[/\\]ExtendedStatsParser.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]stats[/\\]extended[/\\]InternalExtendedStats.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]tophits[/\\]TopHitsAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]pipeline[/\\]BucketHelpers.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]pipeline[/\\]bucketmetrics[/\\]BucketMetricsParser.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]pipeline[/\\]bucketmetrics[/\\]avg[/\\]AvgBucketPipelineAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]pipeline[/\\]bucketscript[/\\]BucketScriptPipelineAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]pipeline[/\\]cumulativesum[/\\]CumulativeSumPipelineAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]pipeline[/\\]derivative[/\\]DerivativePipelineAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]pipeline[/\\]derivative[/\\]InternalDerivative.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]pipeline[/\\]having[/\\]BucketSelectorPipelineAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]support[/\\]AggregationContext.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]support[/\\]AggregationPath.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]support[/\\]GeoPointParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]support[/\\]ValuesSourceParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]support[/\\]format[/\\]ValueFormat.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]support[/\\]format[/\\]ValueParser.java" checks="LineLength" />
@@ -736,10 +674,7 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]fetch[/\\]FetchSubPhaseParseElement.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]fetch[/\\]explain[/\\]ExplainFetchSubPhase.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]fetch[/\\]fielddata[/\\]FieldDataFieldsParseElement.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]fetch[/\\]innerhits[/\\]InnerHitsContext.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]fetch[/\\]innerhits[/\\]InnerHitsFetchSubPhase.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]fetch[/\\]innerhits[/\\]InnerHitsParseElement.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]fetch[/\\]script[/\\]ScriptFieldsParseElement.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]fetch[/\\]source[/\\]FetchSourceContext.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]highlight[/\\]FastVectorHighlighter.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]highlight[/\\]HighlightPhase.java" checks="LineLength" />
@@ -764,7 +699,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]sort[/\\]GeoDistanceSortParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]sort[/\\]ScriptSortParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]sort[/\\]SortParseElement.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]SuggestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]SuggestContextParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]SuggestUtils.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]suggest[/\\]completion[/\\]CompletionSuggestParser.java" checks="LineLength" />
@@ -787,7 +721,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ESExceptionTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]NamingConventionTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]VersionTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ListenerActionIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]RejectionActionIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]HotThreadsIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]health[/\\]ClusterHealthResponsesTests.java" checks="LineLength" />
@@ -804,14 +737,12 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]shards[/\\]IndicesShardStoreRequestIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]shards[/\\]IndicesShardStoreResponseTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]template[/\\]put[/\\]MetaDataIndexTemplateServiceTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]upgrade[/\\]UpgradeIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]bulk[/\\]BulkProcessorIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]bulk[/\\]BulkRequestTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]bulk[/\\]RetryTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]get[/\\]MultiGetShardRequestTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]BulkRequestModifierTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]IngestProxyActionFilterTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]SimulateDocumentSimpleResultTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]SimulateExecutionServiceTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]SimulatePipelineRequestParsingTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]SimulatePipelineResponseTests.java" checks="LineLength" />
@@ -843,7 +774,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterHealthIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterInfoServiceIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterModuleTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterServiceIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterStateDiffIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterStateTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]DiskUsageTests.java" checks="LineLength" />
@@ -864,7 +794,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]allocation[/\\]SimpleAllocationIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]health[/\\]ClusterIndexHealthTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]health[/\\]ClusterStateHealthTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]health[/\\]RoutingTableGenerator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]AutoExpandReplicasTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]DateMathExpressionResolverTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]HumanReadableIndexSettingsTests.java" checks="LineLength" />
@@ -931,7 +860,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]geo[/\\]ShapeBuilderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]hash[/\\]MessageDigestsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]inject[/\\]ModuleTestCase.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]io[/\\]stream[/\\]BytesStreamsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]lucene[/\\]index[/\\]FreqTermsEnumTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]lucene[/\\]uid[/\\]VersionsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]network[/\\]CidrsTests.java" checks="LineLength" />
@@ -947,14 +875,11 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]xcontent[/\\]builder[/\\]XContentBuilderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]xcontent[/\\]cbor[/\\]JsonVsCborTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]xcontent[/\\]smile[/\\]JsonVsSmileTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]xcontent[/\\]support[/\\]filtering[/\\]AbstractFilteringJsonGeneratorTestCase.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]xcontent[/\\]support[/\\]filtering[/\\]FilterPathGeneratorFilteringTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]consistencylevel[/\\]WriteConsistencyLevelIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]deps[/\\]joda[/\\]SimpleJodaTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]deps[/\\]lucene[/\\]VectorHighlighterTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]BlockingClusterStatePublishResponseHandlerTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]DiscoveryWithServiceDisruptionsIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]ZenFaultDetectionTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]ZenUnicastDiscoveryIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]NodeJoinControllerTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]ZenDiscoveryUnitTests.java" checks="LineLength" />
@@ -1006,11 +931,8 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]BinaryDVFieldDataTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]DuelFieldDataTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]FieldDataCacheTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]FilterFieldDataTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]IndexFieldDataServiceTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]PagedBytesStringFieldDataTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]ParentChildFieldDataTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]fielddata[/\\]SortedSetDVStringFieldDataTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentFieldMapperTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DynamicMappingTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]FieldTypeTestCase.java" checks="LineLength" />
@@ -1074,7 +996,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]MultiMatchQueryBuilderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]RandomQueryBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]RangeQueryBuilderTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]ScoreModeTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]SpanMultiTermQueryBuilderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]SpanNotQueryBuilderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]plugin[/\\]CustomQueryParserIT.java" checks="LineLength" />
@@ -1105,7 +1026,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]IndicesLifecycleListenerIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]IndicesLifecycleListenerSingleNodeTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]IndicesOptionsIntegrationIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]analysis[/\\]PreBuiltAnalyzerIntegrationIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]analyze[/\\]AnalyzeActionIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]analyze[/\\]HunspellServiceIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]exists[/\\]indices[/\\]IndicesExistsIT.java" checks="LineLength" />
@@ -1138,7 +1058,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]PipelineExecutionServiceTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]PipelineStoreTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]CompoundProcessorTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]IngestDocumentTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]PipelineFactoryTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]ValueSourceTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]AbstractStringProcessorTestCase.java" checks="LineLength" />
@@ -1167,7 +1086,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]BytesRestResponseTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]CorsRegexDefaultIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]CorsRegexIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]NoOpClient.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]RestControllerTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]util[/\\]RestUtilsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]routing[/\\]AliasResolveRoutingIT.java" checks="LineLength" />
@@ -1279,13 +1197,11 @@
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]BulkTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]DoubleTermsTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]EquivalenceTests.java" checks="LineLength" />
-  <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]GeoDistanceTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]HDRPercentileRanksTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]HDRPercentilesTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]HistogramTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]IPv4RangeTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]IndexLookupTests.java" checks="LineLength" />
-  <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]IndexedScriptTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]IndicesRequestTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]LongTermsTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]MinDocCountTests.java" checks="LineLength" />
@@ -1325,15 +1241,12 @@
   <suppress files="plugins[/\\]delete-by-query[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]deletebyquery[/\\]IndexDeleteByQueryResponseTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]delete-by-query[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]deletebyquery[/\\]TransportDeleteByQueryActionTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]delete-by-query[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugin[/\\]deletebyquery[/\\]DeleteByQueryTests.java" checks="LineLength" />
-  <suppress files="plugins[/\\]discovery-azure[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]management[/\\]AzureComputeService.java" checks="LineLength" />
   <suppress files="plugins[/\\]discovery-azure[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]AbstractAzureTestCase.java" checks="LineLength" />
   <suppress files="plugins[/\\]discovery-azure[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]azure[/\\]AzureMinimumMasterNodesTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]discovery-azure[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]azure[/\\]AzureSimpleTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]discovery-azure[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]azure[/\\]AzureTwoStartedNodesTests.java" checks="LineLength" />
-  <suppress files="plugins[/\\]discovery-ec2[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]aws[/\\]AwsEc2ServiceImpl.java" checks="LineLength" />
   <suppress files="plugins[/\\]discovery-ec2[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]aws[/\\]AbstractAwsTestCase.java" checks="LineLength" />
   <suppress files="plugins[/\\]discovery-ec2[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]ec2[/\\]AmazonEC2Mock.java" checks="LineLength" />
-  <suppress files="plugins[/\\]discovery-gce[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]gce[/\\]GceUnicastHostsProvider.java" checks="LineLength" />
   <suppress files="plugins[/\\]discovery-gce[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]gce[/\\]GceNetworkTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]ingest-geoip[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]geoip[/\\]GeoIpProcessor.java" checks="LineLength" />
   <suppress files="plugins[/\\]ingest-geoip[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]geoip[/\\]GeoIpProcessorFactoryTests.java" checks="LineLength" />
@@ -1365,7 +1278,6 @@
   <suppress files="plugins[/\\]mapper-size[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]size[/\\]SizeMappingTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]repository-azure[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]blobstore[/\\]AzureBlobContainer.java" checks="LineLength" />
   <suppress files="plugins[/\\]repository-azure[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]blobstore[/\\]AzureBlobStore.java" checks="LineLength" />
-  <suppress files="plugins[/\\]repository-azure[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]storage[/\\]AzureStorageService.java" checks="LineLength" />
   <suppress files="plugins[/\\]repository-azure[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]storage[/\\]AzureStorageServiceImpl.java" checks="LineLength" />
   <suppress files="plugins[/\\]repository-azure[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]storage[/\\]AzureStorageSettings.java" checks="LineLength" />
   <suppress files="plugins[/\\]repository-azure[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]azure[/\\]AzureRepository.java" checks="LineLength" />
@@ -1398,7 +1310,6 @@
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]TestShardRouting.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]cli[/\\]CliToolTestCase.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]MockBigArrays.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]MockSearchService.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]script[/\\]NativeSignificanceScoreScriptWithParams.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]BackgroundIndexer.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]CompositeTestCluster.java" checks="LineLength" />
@@ -1422,7 +1333,6 @@
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]disruption[/\\]SlowClusterStateProcessing.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]engine[/\\]AssertingSearcher.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]engine[/\\]MockEngineSupport.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]engine[/\\]MockInternalEngine.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]hamcrest[/\\]ElasticsearchAssertions.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]junit[/\\]listeners[/\\]LoggingListener.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]ESRestTestCase.java" checks="LineLength" />
@@ -1451,7 +1361,6 @@
   <suppress files="test[/\\]framework[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]test[/\\]JsonPathTests.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]test[/\\]RestTestParserTests.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]test[/\\]InternalTestClusterTests.java" checks="LineLength" />
-
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]cli[/\\]CliTool.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]admin[/\\]indices[/\\]settings[/\\]RestGetSettingsAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]tribe[/\\]TribeService.java" checks="LineLength" />

From e90d00ffce2eb21469f8a931728ba776e72cc804 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 6 May 2016 09:17:18 -0400
Subject: [PATCH 0108/1311] Remove handshake from transport client

This commit removes handshaking from the transport client. This
handshaking is not needed because of the existence of the liveness
check.

Relates #18174
---
 .../TransportClientNodesService.java          |  4 +--
 .../zen/ping/unicast/UnicastZenPing.java      |  2 +-
 .../transport/TransportService.java           | 27 +++++++++++++++----
 .../transport/FailAndRetryMockTransport.java  |  6 +----
 .../AbstractSimpleTransportTestCase.java      |  5 ++--
 .../NettyTransportServiceHandshakeTests.java  |  8 +++---
 6 files changed, 32 insertions(+), 20 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
index 68ed7c927ac..c3379c9ceaf 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
@@ -360,7 +360,7 @@ public class TransportClientNodesService extends AbstractComponent {
                     try {
                         // its a listed node, light connect to it...
                         logger.trace("connecting to listed node (light) [{}]", listedNode);
-                        transportService.connectToNodeLight(listedNode, pingTimeout, !ignoreClusterName);
+                        transportService.connectToNodeLight(listedNode);
                     } catch (Throwable e) {
                         logger.debug("failed to connect to node [{}], removed from nodes list", e, listedNode);
                         newFilteredNodes.add(listedNode);
@@ -435,7 +435,7 @@ public class TransportClientNodesService extends AbstractComponent {
                                     } else {
                                         // its a listed node, light connect to it...
                                         logger.trace("connecting to listed node (light) [{}]", listedNode);
-                                        transportService.connectToNodeLight(listedNode, pingTimeout, !ignoreClusterName);
+                                        transportService.connectToNodeLight(listedNode);
                                     }
                                 } catch (Exception e) {
                                     logger.debug("failed to connect to node [{}], ignoring...", e, listedNode);
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
index cc37504360c..d200ca5b07b 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
@@ -402,7 +402,7 @@ public class UnicastZenPing extends AbstractLifecycleComponent<ZenPing> implemen
                             // connect to the node, see if we manage to do it, if not, bail
                             if (!nodeFoundByAddress) {
                                 logger.trace("[{}] connecting (light) to {}", sendPingsHandler.id(), finalNodeToSend);
-                                transportService.connectToNodeLight(finalNodeToSend, timeout.getMillis());
+                                transportService.connectToNodeLightAndHandshake(finalNodeToSend, timeout.getMillis());
                             } else {
                                 logger.trace("[{}] connecting to {}", sendPingsHandler.id(), finalNodeToSend);
                                 transportService.connectToNode(finalNodeToSend);
diff --git a/core/src/main/java/org/elasticsearch/transport/TransportService.java b/core/src/main/java/org/elasticsearch/transport/TransportService.java
index 89cc68debfd..820cdfc130a 100644
--- a/core/src/main/java/org/elasticsearch/transport/TransportService.java
+++ b/core/src/main/java/org/elasticsearch/transport/TransportService.java
@@ -277,6 +277,18 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
         transport.connectToNode(node);
     }
 
+    /**
+     * Lightly connect to the specified node
+     *
+     * @param node the node to connect to
+     */
+    public void connectToNodeLight(final DiscoveryNode node) {
+        if (node.equals(localNode)) {
+            return;
+        }
+        transport.connectToNodeLight(node);
+    }
+
     /**
      * Lightly connect to the specified node, and handshake cluster
      * name and version
@@ -287,8 +299,10 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
      * @throws ConnectTransportException if the connection or the
      *                                   handshake failed
      */
-    public DiscoveryNode connectToNodeLight(final DiscoveryNode node, final long handshakeTimeout) throws ConnectTransportException {
-        return connectToNodeLight(node, handshakeTimeout, true);
+    public DiscoveryNode connectToNodeLightAndHandshake(
+            final DiscoveryNode node,
+            final long handshakeTimeout) throws ConnectTransportException {
+        return connectToNodeLightAndHandshake(node, handshakeTimeout, true);
     }
 
     /**
@@ -305,7 +319,10 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
      * @throws ConnectTransportException if the connection or the
      *                                   handshake failed
      */
-    public DiscoveryNode connectToNodeLight(final DiscoveryNode node, final long handshakeTimeout, final boolean checkClusterName) {
+    public DiscoveryNode connectToNodeLightAndHandshake(
+            final DiscoveryNode node,
+            final long handshakeTimeout,
+            final boolean checkClusterName) {
         if (node.equals(localNode)) {
             return localNode;
         }
@@ -353,7 +370,7 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
                 localNode != null ? localNode.getVersion().minimumCompatibilityVersion() : Version.CURRENT.minimumCompatibilityVersion());
     }
 
-    public static class HandshakeRequest extends TransportRequest {
+    static class HandshakeRequest extends TransportRequest {
 
         public static final HandshakeRequest INSTANCE = new HandshakeRequest();
 
@@ -362,7 +379,7 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
 
     }
 
-    public static class HandshakeResponse extends TransportResponse {
+    static class HandshakeResponse extends TransportResponse {
         private DiscoveryNode discoveryNode;
         private ClusterName clusterName;
         private Version version;
diff --git a/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java b/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java
index 0aaec63fef0..50fbbd4fcb8 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java
@@ -74,11 +74,7 @@ abstract class FailAndRetryMockTransport<Response extends TransportResponse> imp
         //we make sure that nodes get added to the connected ones when calling addTransportAddress, by returning proper nodes info
         if (connectMode) {
             TransportResponseHandler transportResponseHandler = transportServiceAdapter.onResponseReceived(requestId);
-            if (action.equals(TransportLivenessAction.NAME)) {
-                transportResponseHandler.handleResponse(new LivenessResponse(clusterName, node));
-            } else {
-                transportResponseHandler.handleResponse(new TransportService.HandshakeResponse(node, clusterName, Version.CURRENT));
-            }
+            transportResponseHandler.handleResponse(new LivenessResponse(ClusterName.DEFAULT, node));
             return;
         }
 
diff --git a/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
index 45966606d31..f285a1db52e 100644
--- a/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
+++ b/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
@@ -22,7 +22,6 @@ package org.elasticsearch.transport;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -1179,7 +1178,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         }
 
         try {
-            serviceB.connectToNodeLight(nodeA, 100);
+            serviceB.connectToNodeLightAndHandshake(nodeA, 100);
             fail("exception should be thrown");
         } catch (ConnectTransportException e) {
             // all is well
@@ -1239,7 +1238,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         }
 
         try {
-            serviceB.connectToNodeLight(nodeA, 100);
+            serviceB.connectToNodeLightAndHandshake(nodeA, 100);
             fail("exception should be thrown");
         } catch (ConnectTransportException e) {
             // all is well
diff --git a/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java b/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java
index b376a55af73..2b0ff6e0c1a 100644
--- a/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java
@@ -109,7 +109,7 @@ public class NettyTransportServiceHandshakeTests extends ESTestCase {
                         test);
 
         DiscoveryNode connectedNode =
-                handleA.transportService.connectToNodeLight(
+                handleA.transportService.connectToNodeLightAndHandshake(
                         new DiscoveryNode(
                                 "",
                                 handleB.discoveryNode.getAddress(),
@@ -131,7 +131,7 @@ public class NettyTransportServiceHandshakeTests extends ESTestCase {
         NetworkHandle handleB = startServices("TS_B", settings, Version.CURRENT, new ClusterName("b"));
 
         try {
-            handleA.transportService.connectToNodeLight(
+            handleA.transportService.connectToNodeLightAndHandshake(
                     new DiscoveryNode(
                             "",
                             handleB.discoveryNode.getAddress(),
@@ -154,7 +154,7 @@ public class NettyTransportServiceHandshakeTests extends ESTestCase {
                 startServices("TS_B", settings, VersionUtils.getPreviousVersion(Version.CURRENT.minimumCompatibilityVersion()), test);
 
         try {
-            handleA.transportService.connectToNodeLight(
+            handleA.transportService.connectToNodeLightAndHandshake(
                     new DiscoveryNode(
                             "",
                             handleB.discoveryNode.getAddress(),
@@ -180,7 +180,7 @@ public class NettyTransportServiceHandshakeTests extends ESTestCase {
                         new ClusterName("b")
                 );
 
-        DiscoveryNode connectedNode = handleA.transportService.connectToNodeLight(
+        DiscoveryNode connectedNode = handleA.transportService.connectToNodeLightAndHandshake(
                 new DiscoveryNode(
                         "",
                         handleB.discoveryNode.getAddress(),

From cb40b986d15317e7afdc2872eccf25b6d0f05fef Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 6 May 2016 09:09:06 -0400
Subject: [PATCH 0109/1311] Allow leading `/` in AUTOSENSE path

Relates to #18160
---
 .../elasticsearch/gradle/RestTestsFromSnippetsTask.groovy   | 6 ++----
 docs/reference/mapping/params/analyzer.asciidoc             | 2 +-
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/RestTestsFromSnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/RestTestsFromSnippetsTask.groovy
index c4201a01f26..1ce8adf1ba4 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/RestTestsFromSnippetsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/RestTestsFromSnippetsTask.groovy
@@ -176,10 +176,8 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
                 String body = matcher.group("body")
                 String catchPart = last ? snippet.catchPart : null
                 if (pathAndQuery.startsWith('/')) {
-                    // Why not do some light linting while we're here?
-                    throw new InvalidUserDataException(
-                        "Path shouldn't start with a '/': $snippet\n"
-                        + snippet.contents)
+                    // Leading '/'s break the generated paths
+                    pathAndQuery = pathAndQuery.substring(1)
                 }
                 emitDo(method, pathAndQuery, body, catchPart)
             }
diff --git a/docs/reference/mapping/params/analyzer.asciidoc b/docs/reference/mapping/params/analyzer.asciidoc
index 89d83f2b369..b2138794546 100644
--- a/docs/reference/mapping/params/analyzer.asciidoc
+++ b/docs/reference/mapping/params/analyzer.asciidoc
@@ -41,7 +41,7 @@ in the field mapping, as follows:
 
 [source,js]
 --------------------------------------------------
-PUT my_index
+PUT /my_index
 {
   "mappings": {
     "my_type": {

From d52537dc7bcb6331dd18b920f04d661cf688a66f Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 6 May 2016 09:54:26 -0400
Subject: [PATCH 0110/1311] Add semicolon query string parameter delimiter

This commit adds support for the semicolon character as a valid query
string parameter delimiter.

Relates #18186
---
 .../elasticsearch/rest/support/RestUtils.java |  2 +-
 .../rest/util/RestUtilsTests.java             | 28 +++++++++++++------
 2 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/rest/support/RestUtils.java b/core/src/main/java/org/elasticsearch/rest/support/RestUtils.java
index 7a976fe1f1d..ef7a7ac0edd 100644
--- a/core/src/main/java/org/elasticsearch/rest/support/RestUtils.java
+++ b/core/src/main/java/org/elasticsearch/rest/support/RestUtils.java
@@ -73,7 +73,7 @@ public class RestUtils {
                     name = decodeComponent(s.substring(pos, i));
                 }
                 pos = i + 1;
-            } else if (c == '&') {
+            } else if (c == '&' || c == ';') {
                 if (name == null && pos != i) {
                     // We haven't seen an `=' so far but moved forward.
                     // Must be a param of the form '&a&' so add it with
diff --git a/core/src/test/java/org/elasticsearch/rest/util/RestUtilsTests.java b/core/src/test/java/org/elasticsearch/rest/util/RestUtilsTests.java
index 065b99ea5ae..a2f9e42fc3c 100644
--- a/core/src/test/java/org/elasticsearch/rest/util/RestUtilsTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/util/RestUtilsTests.java
@@ -36,6 +36,10 @@ import static org.hamcrest.Matchers.nullValue;
  */
 public class RestUtilsTests extends ESTestCase {
 
+    static char randomDelimiter() {
+        return randomBoolean() ? '&' : ';';
+    }
+
     public void testDecodeQueryString() {
         Map<String, String> params = new HashMap<>();
 
@@ -45,7 +49,7 @@ public class RestUtilsTests extends ESTestCase {
         assertThat(params.get("test"), equalTo("value"));
 
         params.clear();
-        uri = "something?test=value&test1=value1";
+        uri = String.format(Locale.ROOT, "something?test=value%ctest1=value1", randomDelimiter());
         RestUtils.decodeQueryString(uri, uri.indexOf('?') + 1, params);
         assertThat(params.size(), equalTo(2));
         assertThat(params.get("test"), equalTo("value"));
@@ -70,12 +74,12 @@ public class RestUtilsTests extends ESTestCase {
         assertThat(params.size(), equalTo(0));
 
         params.clear();
-        uri = "something?&";
+        uri = String.format(Locale.ROOT, "something?%c", randomDelimiter());
         RestUtils.decodeQueryString(uri, uri.indexOf('?') + 1, params);
         assertThat(params.size(), equalTo(0));
 
         params.clear();
-        uri = "something?p=v&&p1=v1";
+        uri = String.format(Locale.ROOT, "something?p=v%c%cp1=v1", randomDelimiter(), randomDelimiter());
         RestUtils.decodeQueryString(uri, uri.indexOf('?') + 1, params);
         assertThat(params.size(), equalTo(2));
         assertThat(params.get("p"), equalTo("v"));
@@ -87,7 +91,7 @@ public class RestUtilsTests extends ESTestCase {
         assertThat(params.size(), equalTo(0));
 
         params.clear();
-        uri = "something?&=";
+        uri = String.format(Locale.ROOT, "something?%c=", randomDelimiter());
         RestUtils.decodeQueryString(uri, uri.indexOf('?') + 1, params);
         assertThat(params.size(), equalTo(0));
 
@@ -98,14 +102,14 @@ public class RestUtilsTests extends ESTestCase {
         assertThat(params.get("a"), equalTo(""));
 
         params.clear();
-        uri = "something?p=v&a";
+        uri = String.format(Locale.ROOT, "something?p=v%ca", randomDelimiter());
         RestUtils.decodeQueryString(uri, uri.indexOf('?') + 1, params);
         assertThat(params.size(), equalTo(2));
         assertThat(params.get("a"), equalTo(""));
         assertThat(params.get("p"), equalTo("v"));
 
         params.clear();
-        uri = "something?p=v&a&p1=v1";
+        uri = String.format(Locale.ROOT, "something?p=v%ca%cp1=v1", randomDelimiter(), randomDelimiter());
         RestUtils.decodeQueryString(uri, uri.indexOf('?') + 1, params);
         assertThat(params.size(), equalTo(3));
         assertThat(params.get("a"), equalTo(""));
@@ -113,7 +117,7 @@ public class RestUtilsTests extends ESTestCase {
         assertThat(params.get("p1"), equalTo("v1"));
 
         params.clear();
-        uri = "something?p=v&a&b&p1=v1";
+        uri = String.format(Locale.ROOT, "something?p=v%ca%cb%cp1=v1", randomDelimiter(), randomDelimiter(), randomDelimiter());
         RestUtils.decodeQueryString(uri, uri.indexOf('?') + 1, params);
         assertThat(params.size(), equalTo(4));
         assertThat(params.get("a"), equalTo(""));
@@ -139,9 +143,15 @@ public class RestUtilsTests extends ESTestCase {
         Map<String, String> params = new HashMap<>();
 
         // This is a valid URL
-        String uri = "example.com/:@-._~!$&'()*+,=;:@-._~!$&'()*+,=:@-._~!$&'()*+,==?/?:@-._~!$'()*+,;=/?:@-._~!$'()*+,;==#/?:@-._~!$&'()*+,;=";
+        String uri = String.format(
+                Locale.ROOT,
+                "example.com/:@-._~!$%c'()*+,=;:@-._~!$%c'()*+,=:@-._~!$%c'()*+,==?/?:@-._~!$'()*+,=/?:@-._~!$'()*+,==#/?:@-._~!$%c'()*+,;=",
+                randomDelimiter(),
+                randomDelimiter(),
+                randomDelimiter(),
+                randomDelimiter());
         RestUtils.decodeQueryString(uri, uri.indexOf('?') + 1, params);
-        assertThat(params.get("/?:@-._~!$'()* ,;"), equalTo("/?:@-._~!$'()* ,;=="));
+        assertThat(params.get("/?:@-._~!$'()* ,"), equalTo("/?:@-._~!$'()* ,=="));
         assertThat(params.size(), equalTo(1));
     }
 

From 473be0137398f18d40660622fcfaf77756dd93eb Mon Sep 17 00:00:00 2001
From: Alexander Reelsen <alexander@reelsen.net>
Date: Fri, 6 May 2016 16:05:19 +0200
Subject: [PATCH 0111/1311] Documentation: Switch to https for debian
 repository

---
 docs/reference/setup/install/deb.asciidoc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/reference/setup/install/deb.asciidoc b/docs/reference/setup/install/deb.asciidoc
index b327dc4a85f..9c7a043bdd5 100644
--- a/docs/reference/setup/install/deb.asciidoc
+++ b/docs/reference/setup/install/deb.asciidoc
@@ -33,7 +33,7 @@ Save the repository definition to  +/etc/apt/sources.list.d/elasticsearch-{major
 
 ["source","sh",subs="attributes,callouts"]
 --------------------------------------------------
-echo "deb http://packages.elastic.co/elasticsearch/{major-version}/debian stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch-{major-version}.list
+echo "deb https://packages.elastic.co/elasticsearch/{major-version}/debian stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch-{major-version}.list
 --------------------------------------------------
 
 [WARNING]
@@ -63,7 +63,7 @@ If two entries exist for the same Elasticsearch repository, you will see an erro
 
 ["literal",subs="attributes,callouts"]
 
-Duplicate sources.list entry http://packages.elastic.co/elasticsearch/{major-version}/debian/ ...`
+Duplicate sources.list entry https://packages.elastic.co/elasticsearch/{major-version}/debian/ ...`
 
 Examine +/etc/apt/sources.list.d/elasticsearch-{major-version}.list+ for the duplicate entry or locate the duplicate entry amongst the files in `/etc/apt/sources.list.d/` and the `/etc/apt/sources.list` file.
 ==================================================
@@ -188,4 +188,4 @@ locations for a Debian-based system:
 
 |=======================================================================
 
-include::next-steps.asciidoc[]
\ No newline at end of file
+include::next-steps.asciidoc[]

From f270ed26c36a381a32778444ec83e797bfda504f Mon Sep 17 00:00:00 2001
From: Britta Weber <britta.weber@elasticsearch.com>
Date: Fri, 6 May 2016 16:26:43 +0200
Subject: [PATCH 0112/1311] fix highlighing for old version indices with string
 fields

---
 .../search/highlight/HighlightPhase.java      |  5 ++-
 .../search/highlight/HighlighterSearchIT.java | 44 +++++++++++++++++--
 2 files changed, 44 insertions(+), 5 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java b/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java
index 1c4a9edca2a..92011fd77e7 100644
--- a/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java
@@ -114,8 +114,9 @@ public class HighlightPhase extends AbstractComponent implements FetchSubPhase {
                 // If the field was explicitly given we assume that whoever issued the query knew
                 // what they were doing and try to highlight anyway.
                 if (fieldNameContainsWildcards) {
-                    if (fieldMapper.fieldType().typeName().equals(TextFieldMapper.CONTENT_TYPE) == false && fieldMapper.fieldType().typeName().equals
-                        (KeywordFieldMapper.CONTENT_TYPE) == false) {
+                    if (fieldMapper.fieldType().typeName().equals(TextFieldMapper.CONTENT_TYPE) == false &&
+                        fieldMapper.fieldType().typeName().equals(KeywordFieldMapper.CONTENT_TYPE) == false &&
+                        fieldMapper.fieldType().typeName().equals(StringFieldMapper.CONTENT_TYPE) == false) {
                         continue;
                     }
                 }
diff --git a/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java b/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java
index 8b80c7657cf..869182cb51a 100644
--- a/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java
@@ -19,9 +19,11 @@
 package org.elasticsearch.search.highlight;
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.Settings.Builder;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -35,15 +37,18 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.search.MatchQuery;
 import org.elasticsearch.index.search.MatchQuery.Type;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.highlight.HighlightBuilder.Field;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
 import org.hamcrest.Matcher;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -84,6 +89,12 @@ import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.startsWith;
 
 public class HighlighterSearchIT extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return pluginList(InternalSettingsPlugin.class);
+    }
+
     public void testHighlightingWithWildcardName() throws IOException {
         // test the kibana case with * as fieldname that will try highlight all fields including meta fields
         XContentBuilder mappings = jsonBuilder();
@@ -2572,9 +2583,7 @@ public class HighlighterSearchIT extends ESIntegTestCase {
     }
 
     public void testKeywordFieldHighlighting() throws IOException {
-        // check that we do not get an exception for geo_point fields in case someone tries to highlight
-        // it accidential with a wildcard
-        // see https://github.com/elastic/elasticsearch/issues/17537
+        // check that keyword highlighting works
         XContentBuilder mappings = jsonBuilder();
         mappings.startObject();
         mappings.startObject("type")
@@ -2600,4 +2609,33 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         assertThat(search.getHits().totalHits(), equalTo(1L));
         assertThat(search.getHits().getAt(0).getHighlightFields().get("keyword_field").getFragments()[0].string(), equalTo("<em>some text</em>"));
     }
+
+    public void testStringFieldHighlighting() throws IOException {
+        // check that string field highlighting on old indexes works
+        XContentBuilder mappings = jsonBuilder();
+        mappings.startObject();
+        mappings.startObject("type")
+            .startObject("properties")
+            .startObject("string_field")
+            .field("type", "string")
+            .endObject()
+            .endObject()
+            .endObject();
+        mappings.endObject();
+        assertAcked(prepareCreate("test")
+            .addMapping("type", mappings)
+        .setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_2)));
+        ensureYellow();
+
+        client().prepareIndex("test", "type", "1")
+            .setSource(jsonBuilder().startObject().field("string_field", "some text").endObject())
+            .get();
+        refresh();
+        SearchResponse search = client().prepareSearch().setSource(
+            new SearchSourceBuilder().query(QueryBuilders.matchQuery("string_field", "some text")).highlighter(new HighlightBuilder().field("*")))
+            .get();
+        assertNoFailures(search);
+        assertThat(search.getHits().totalHits(), equalTo(1L));
+        assertThat(search.getHits().getAt(0).getHighlightFields().get("string_field").getFragments()[0].string(), equalTo("<em>some</em> <em>text</em>"));
+    }
 }

From ddebbb9536fadf5f08b99073f2ce551f31f44140 Mon Sep 17 00:00:00 2001
From: Britta Weber <britta.weber@elasticsearch.com>
Date: Fri, 6 May 2016 16:50:34 +0200
Subject: [PATCH 0113/1311] add string to documentation

---
 docs/reference/search/request/highlighting.asciidoc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/reference/search/request/highlighting.asciidoc b/docs/reference/search/request/highlighting.asciidoc
index bfb5ffae7b3..53d6e23fa80 100644
--- a/docs/reference/search/request/highlighting.asciidoc
+++ b/docs/reference/search/request/highlighting.asciidoc
@@ -35,7 +35,8 @@ be used for highlighting if it mapped to have `store` set to `true`.
 ==================================
 
 The field name supports wildcard notation. For example, using `comment_*`
-will cause all <<text,text>> or <<keyword,keyword>> fields that match the expression to be highlighted.
+will cause all <<text,text>> and <<keyword,keyword>> fields (and <<string,string>>
+from versions before 5.0) that match the expression to be highlighted.
 Note that all other fields will not be highlighted. If you use a custom mapper and want to
 highlight on a field anyway, you have to provide the field name explicitly.
 

From 0ff5652fffb56853fb98a63dda8d63160695c34e Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Fri, 6 May 2016 16:59:53 +0200
Subject: [PATCH 0114/1311] Add node name to Cat Recovery

closes #8041
---
 .../rest/action/cat/RestRecoveryAction.java   |  4 +++
 docs/reference/cat/recovery.asciidoc          | 12 +++----
 .../test/cat.recovery/10_basic.yaml           | 33 ++++++++++++++++++-
 3 files changed, 42 insertions(+), 7 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java
index 7c555c9b357..4764462d958 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java
@@ -88,7 +88,9 @@ public class RestRecoveryAction extends AbstractCatAction {
                 .addCell("type", "alias:ty;desc:recovery type")
                 .addCell("stage", "alias:st;desc:recovery stage")
                 .addCell("source_host", "alias:shost;desc:source host")
+                .addCell("source_node", "alias:snode;desc:source node name")
                 .addCell("target_host", "alias:thost;desc:target host")
+                .addCell("target_node", "alias:tnode;desc:target node name")
                 .addCell("repository", "alias:rep;desc:repository")
                 .addCell("snapshot", "alias:snap;desc:snapshot")
                 .addCell("files", "alias:f;desc:number of files to recover")
@@ -149,7 +151,9 @@ public class RestRecoveryAction extends AbstractCatAction {
                 t.addCell(state.getType().toString().toLowerCase(Locale.ROOT));
                 t.addCell(state.getStage().toString().toLowerCase(Locale.ROOT));
                 t.addCell(state.getSourceNode() == null ? "n/a" : state.getSourceNode().getHostName());
+                t.addCell(state.getSourceNode() == null ? "n/a" : state.getSourceNode().getName());
                 t.addCell(state.getTargetNode().getHostName());
+                t.addCell(state.getTargetNode().getName());
                 t.addCell(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshotId().getRepository());
                 t.addCell(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshotId().getSnapshot());
                 t.addCell(state.getIndex().totalRecoverFiles());
diff --git a/docs/reference/cat/recovery.asciidoc b/docs/reference/cat/recovery.asciidoc
index b9a16b2913d..6fe748096d1 100644
--- a/docs/reference/cat/recovery.asciidoc
+++ b/docs/reference/cat/recovery.asciidoc
@@ -15,12 +15,12 @@ are no shards in transit from one node to another:
 [source,sh]
 ----------------------------------------------------------------------------
 > curl -XGET 'localhost:9200/_cat/recovery?v'
-index shard time type  stage source_host target_host repository snapshot files files_percent bytes bytes_percent total_files total_bytes translog translog_percent total_translog
-index 0     87ms store done  127.0.0.1   127.0.0.1   n/a        n/a      0     0.0%          0     0.0%          0           0           0        100.0%           0
-index 1     97ms store done  127.0.0.1   127.0.0.1   n/a        n/a      0     0.0%          0     0.0%          0           0           0        100.0%           0
-index 2     93ms store done  127.0.0.1   127.0.0.1   n/a        n/a      0     0.0%          0     0.0%          0           0           0        100.0%           0
-index 3     90ms store done  127.0.0.1   127.0.0.1   n/a        n/a      0     0.0%          0     0.0%          0           0           0        100.0%           0
-index 4     9ms  store done  127.0.0.1   127.0.0.1   n/a        n/a      0     0.0%          0     0.0%          0           0           0        100.0%           0
+index shard time type  stage source_host source_node target_host target_node repository snapshot files files_percent bytes bytes_percent total_files total_bytes translog translog_percent total_translog
+index 0     87ms store done  127.0.0.1        Athena      127.0.0.1        Athena      n/a        n/a      0     0.0%          0     0.0%          0           0           0        100.0%           0
+index 1     97ms store done  127.0.0.1        Athena      127.0.0.1        Athena      n/a        n/a      0     0.0%          0     0.0%          0           0           0        100.0%           0
+index 2     93ms store done  127.0.0.1        Athena      127.0.0.1        Athena      n/a        n/a      0     0.0%          0     0.0%          0           0           0        100.0%           0
+index 3     90ms store done  127.0.0.1        Athena      127.0.0.1        Athena      n/a        n/a      0     0.0%          0     0.0%          0           0           0        100.0%           0
+index 4     9ms  store done  127.0.0.1        Athena      127.0.0.1        Athena      n/a        n/a      0     0.0%          0     0.0%          0           0           0        100.0%           0
 ---------------------------------------------------------------------------
 
 In the above case, the source and target nodes are the same because the recovery
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yaml
index effc4c20313..fc596dd5792 100755
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.recovery/10_basic.yaml
@@ -19,7 +19,8 @@
        cluster.health:
          wait_for_status: yellow
   - do:
-      cat.recovery: {}
+      cat.recovery:
+          h: i,s,t,ty,st,shost,thost,rep,snap,f,fr,fp,tf,b,br,bp,tb,to,tor,top
 
   - match:
       $body: |
@@ -48,3 +49,33 @@
                 \n
               )+
               $/
+
+  - do:
+      cat.recovery:
+          h: shard,source_node,bytes
+
+  - match:
+      $body: |
+              /^
+              (
+                \d          \s+                                 # shard
+                ((\S+\s?){1,10})\s+                             # source_node
+                \d+                                             # bytes
+                \n
+              )+
+              $/
+
+  - do:
+      cat.recovery:
+          h: shard,target_node,bytes
+
+  - match:
+      $body: |
+              /^
+              (
+                \d          \s+                                 # shard
+                ((\S+\s?){1,10})\s+                             # target_node
+                \d+                                             # bytes
+                \n
+              )+
+              $/

From 1d5b9f1ce65567b5d4ff195fe27090d0b72bfee1 Mon Sep 17 00:00:00 2001
From: Chris Earle <chris.earle@elastic.co>
Date: Fri, 6 May 2016 13:17:43 -0400
Subject: [PATCH 0115/1311] Test was not updated with #18187

---
 .../action/cat/RestRecoveryActionTests.java   | 61 ++++++++++---------
 1 file changed, 33 insertions(+), 28 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java
index 4e8ea3b3eb0..978d226da40 100644
--- a/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java
@@ -127,26 +127,29 @@ public class RestRecoveryActionTests extends ESTestCase {
         assertNotNull(table);
 
         List<Table.Cell> headers = table.getHeaders();
+
         assertThat(headers.get(0).value, equalTo("index"));
         assertThat(headers.get(1).value, equalTo("shard"));
         assertThat(headers.get(2).value, equalTo("time"));
         assertThat(headers.get(3).value, equalTo("type"));
         assertThat(headers.get(4).value, equalTo("stage"));
         assertThat(headers.get(5).value, equalTo("source_host"));
-        assertThat(headers.get(6).value, equalTo("target_host"));
-        assertThat(headers.get(7).value, equalTo("repository"));
-        assertThat(headers.get(8).value, equalTo("snapshot"));
-        assertThat(headers.get(9).value, equalTo("files"));
-        assertThat(headers.get(10).value, equalTo("files_recovered"));
-        assertThat(headers.get(11).value, equalTo("files_percent"));
-        assertThat(headers.get(12).value, equalTo("files_total"));
-        assertThat(headers.get(13).value, equalTo("bytes"));
-        assertThat(headers.get(14).value, equalTo("bytes_recovered"));
-        assertThat(headers.get(15).value, equalTo("bytes_percent"));
-        assertThat(headers.get(16).value, equalTo("bytes_total"));
-        assertThat(headers.get(17).value, equalTo("translog_ops"));
-        assertThat(headers.get(18).value, equalTo("translog_ops_recovered"));
-        assertThat(headers.get(19).value, equalTo("translog_ops_percent"));
+        assertThat(headers.get(6).value, equalTo("source_node"));
+        assertThat(headers.get(7).value, equalTo("target_host"));
+        assertThat(headers.get(8).value, equalTo("target_node"));
+        assertThat(headers.get(9).value, equalTo("repository"));
+        assertThat(headers.get(10).value, equalTo("snapshot"));
+        assertThat(headers.get(11).value, equalTo("files"));
+        assertThat(headers.get(12).value, equalTo("files_recovered"));
+        assertThat(headers.get(13).value, equalTo("files_percent"));
+        assertThat(headers.get(14).value, equalTo("files_total"));
+        assertThat(headers.get(15).value, equalTo("bytes"));
+        assertThat(headers.get(16).value, equalTo("bytes_recovered"));
+        assertThat(headers.get(17).value, equalTo("bytes_percent"));
+        assertThat(headers.get(18).value, equalTo("bytes_total"));
+        assertThat(headers.get(19).value, equalTo("translog_ops"));
+        assertThat(headers.get(20).value, equalTo("translog_ops_recovered"));
+        assertThat(headers.get(21).value, equalTo("translog_ops_percent"));
 
         assertThat(table.getRows().size(), equalTo(successfulShards));
         for (int i = 0; i < successfulShards; i++) {
@@ -158,24 +161,26 @@ public class RestRecoveryActionTests extends ESTestCase {
             assertThat(cells.get(3).value, equalTo(state.getType().name().toLowerCase(Locale.ROOT)));
             assertThat(cells.get(4).value, equalTo(state.getStage().name().toLowerCase(Locale.ROOT)));
             assertThat(cells.get(5).value, equalTo(state.getSourceNode() == null ? "n/a" : state.getSourceNode().getHostName()));
-            assertThat(cells.get(6).value, equalTo(state.getTargetNode().getHostName()));
+            assertThat(cells.get(6).value, equalTo(state.getSourceNode() == null ? "n/a" : state.getSourceNode().getName()));
+            assertThat(cells.get(7).value, equalTo(state.getTargetNode().getHostName()));
+            assertThat(cells.get(8).value, equalTo(state.getTargetNode().getName()));
             assertThat(
-                    cells.get(7).value,
+                    cells.get(9).value,
                     equalTo(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshotId().getRepository()));
             assertThat(
-                    cells.get(8).value,
+                    cells.get(10).value,
                     equalTo(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshotId().getSnapshot()));
-            assertThat(cells.get(9).value, equalTo(state.getIndex().totalRecoverFiles()));
-            assertThat(cells.get(10).value, equalTo(state.getIndex().recoveredFileCount()));
-            assertThat(cells.get(11).value, equalTo(percent(state.getIndex().recoveredFilesPercent())));
-            assertThat(cells.get(12).value, equalTo(state.getIndex().totalFileCount()));
-            assertThat(cells.get(13).value, equalTo(state.getIndex().totalRecoverBytes()));
-            assertThat(cells.get(14).value, equalTo(state.getIndex().recoveredBytes()));
-            assertThat(cells.get(15).value, equalTo(percent(state.getIndex().recoveredBytesPercent())));
-            assertThat(cells.get(16).value, equalTo(state.getIndex().totalBytes()));
-            assertThat(cells.get(17).value, equalTo(state.getTranslog().totalOperations()));
-            assertThat(cells.get(18).value, equalTo(state.getTranslog().recoveredOperations()));
-            assertThat(cells.get(19).value, equalTo(percent(state.getTranslog().recoveredPercent())));
+            assertThat(cells.get(11).value, equalTo(state.getIndex().totalRecoverFiles()));
+            assertThat(cells.get(12).value, equalTo(state.getIndex().recoveredFileCount()));
+            assertThat(cells.get(13).value, equalTo(percent(state.getIndex().recoveredFilesPercent())));
+            assertThat(cells.get(14).value, equalTo(state.getIndex().totalFileCount()));
+            assertThat(cells.get(15).value, equalTo(state.getIndex().totalRecoverBytes()));
+            assertThat(cells.get(16).value, equalTo(state.getIndex().recoveredBytes()));
+            assertThat(cells.get(17).value, equalTo(percent(state.getIndex().recoveredBytesPercent())));
+            assertThat(cells.get(18).value, equalTo(state.getIndex().totalBytes()));
+            assertThat(cells.get(19).value, equalTo(state.getTranslog().totalOperations()));
+            assertThat(cells.get(20).value, equalTo(state.getTranslog().recoveredOperations()));
+            assertThat(cells.get(21).value, equalTo(percent(state.getTranslog().recoveredPercent())));
         }
     }
 

From 0eaa831f481ff06a4bd1d201b10f9f9df1b98efd Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 6 May 2016 13:24:54 -0400
Subject: [PATCH 0116/1311] Preserve config files from RPM install

This commit modifies the packaging for the RPM package so that edits to
config files will not get lost during removal and upgrade.

Relates #18188
---
 distribution/build.gradle                     | 13 ++++--
 .../src/main/packaging/scripts/postrm         |  7 +++
 distribution/src/main/packaging/scripts/prerm |  6 +++
 .../packaging/scripts/40_rpm_package.bats     | 45 ++++++++++++++++---
 4 files changed, 61 insertions(+), 10 deletions(-)

diff --git a/distribution/build.gradle b/distribution/build.gradle
index 09050db2159..d1d79d2db74 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -322,12 +322,13 @@ configure(subprojects.findAll { ['deb', 'rpm'].contains(it.name) }) {
     configurationFile '/etc/elasticsearch/elasticsearch.yml'
     configurationFile '/etc/elasticsearch/jvm.options'
     configurationFile '/etc/elasticsearch/logging.yml'
-    into('/etc') {
-      from "${packagingFiles}/etc"
+    into('/etc/elasticsearch') {
       fileMode 0750
       permissionGroup 'elasticsearch'
       includeEmptyDirs true
       createDirectoryEntry true
+      fileType CONFIG | NOREPLACE
+      from "${packagingFiles}/etc/elasticsearch"
     }
 
     into('/usr/lib/tmpfiles.d') {
@@ -335,21 +336,25 @@ configure(subprojects.findAll { ['deb', 'rpm'].contains(it.name) }) {
     }
     configurationFile '/usr/lib/systemd/system/elasticsearch.service'
     into('/usr/lib/systemd/system') {
+      fileType CONFIG | NOREPLACE
       from "${packagingFiles}/systemd/elasticsearch.service"
     }
     into('/usr/lib/sysctl.d') {
+      fileType CONFIG | NOREPLACE
       from "${packagingFiles}/systemd/sysctl/elasticsearch.conf"
     }
     configurationFile '/etc/init.d/elasticsearch'
     into('/etc/init.d') {
-      from "${packagingFiles}/init.d/elasticsearch"
       fileMode 0755
+      fileType CONFIG | NOREPLACE
+      from "${packagingFiles}/init.d/elasticsearch"
     }
     configurationFile project.expansions['path.env']
     into(new File(project.expansions['path.env']).getParent()) {
-      from "${project.packagingFiles}/env/elasticsearch"
       fileMode 0644
       dirMode 0755
+      fileType CONFIG | NOREPLACE
+      from "${project.packagingFiles}/env/elasticsearch"
     }
 
     /**
diff --git a/distribution/src/main/packaging/scripts/postrm b/distribution/src/main/packaging/scripts/postrm
index d4104845249..2fa42678d96 100644
--- a/distribution/src/main/packaging/scripts/postrm
+++ b/distribution/src/main/packaging/scripts/postrm
@@ -55,6 +55,7 @@ LOG_DIR="/var/log/elasticsearch"
 PLUGINS_DIR="/usr/share/elasticsearch/plugins"
 PID_DIR="/var/run/elasticsearch"
 DATA_DIR="/var/lib/elasticsearch"
+CONF_DIR="/etc/elasticsearch"
 
 # Source the default env file
 if [ "$SOURCE_ENV_FILE" = "true" ]; then
@@ -102,6 +103,12 @@ if [ "$REMOVE_DIRS" = "true" ]; then
     if [ -d "$DATA_DIR" ]; then
         rmdir --ignore-fail-on-non-empty "$DATA_DIR"
     fi
+
+    # delete the conf directory if and only if empty
+    if [ -d "$CONF_DIR" ]; then
+        rmdir --ignore-fail-on-non-empty "$CONF_DIR"
+    fi
+
 fi
 
 if [ "$REMOVE_USER_AND_GROUP" = "true" ]; then
diff --git a/distribution/src/main/packaging/scripts/prerm b/distribution/src/main/packaging/scripts/prerm
index 07f39759aee..79523629f77 100644
--- a/distribution/src/main/packaging/scripts/prerm
+++ b/distribution/src/main/packaging/scripts/prerm
@@ -64,4 +64,10 @@ if [ "$STOP_REQUIRED" = "true" ]; then
     echo " OK"
 fi
 
+SCRIPTS_DIR="/etc/elasticsearch/scripts"
+# delete the scripts directory if and only if empty
+if [ -d "$SCRIPTS_DIR" ]; then
+    rmdir --ignore-fail-on-non-empty "$SCRIPTS_DIR"
+fi
+
 ${scripts.footer}
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats b/qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats
index 064439cdd5a..0d3170cafeb 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats
+++ b/qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats
@@ -116,7 +116,7 @@ setup() {
 
     assert_file_not_exist "/etc/elasticsearch"
     assert_file_not_exist "/etc/elasticsearch/elasticsearch.yml"
-    assert_file_not_exist  "/etc/elasticsearch/jvm.options"
+    assert_file_not_exist "/etc/elasticsearch/jvm.options"
     assert_file_not_exist "/etc/elasticsearch/logging.yml"
 
     assert_file_not_exist "/etc/init.d/elasticsearch"
@@ -125,7 +125,6 @@ setup() {
     assert_file_not_exist "/etc/sysconfig/elasticsearch"
 }
 
-
 @test "[RPM] reinstall package" {
     rpm -i elasticsearch-$(cat version).rpm
 }
@@ -134,12 +133,46 @@ setup() {
     rpm -qe 'elasticsearch'
 }
 
-@test "[RPM] verify package reinstallation" {
-    verify_package_installation
+@test "[RPM] reremove package" {
+    echo "# ping" >> "/etc/elasticsearch/elasticsearch.yml"
+    echo "# ping" >> "/etc/elasticsearch/jvm.options"
+    echo "# ping" >> "/etc/elasticsearch/logging.yml"
+    echo "# ping" >> "/etc/elasticsearch/scripts/script"
+    rpm -e 'elasticsearch'
 }
 
-@test "[RPM] reremove package" {
-    rpm -e 'elasticsearch'
+@test "[RPM] verify preservation" {
+    # The removal must disable the service
+    # see prerm file
+    if is_systemd; then
+        run systemctl is-enabled elasticsearch.service
+        [ "$status" -eq 1 ]
+    fi
+
+    # Those directories are deleted when removing the package
+    # see postrm file
+    assert_file_not_exist "/var/log/elasticsearch"
+    assert_file_not_exist "/usr/share/elasticsearch/plugins"
+    assert_file_not_exist "/var/run/elasticsearch"
+
+    assert_file_not_exist "/etc/elasticsearch/elasticsearch.yml"
+    assert_file_exist "/etc/elasticsearch/elasticsearch.yml.rpmsave"
+    assert_file_not_exist "/etc/elasticsearch/jvm.options"
+    assert_file_exist "/etc/elasticsearch/jvm.options.rpmsave"
+    assert_file_not_exist "/etc/elasticsearch/logging.yml"
+    assert_file_exist "/etc/elasticsearch/logging.yml.rpmsave"
+    assert_file_exist "/etc/elasticsearch/scripts.rpmsave"
+    assert_file_exist "/etc/elasticsearch/scripts.rpmsave/script"
+
+    assert_file_not_exist "/etc/init.d/elasticsearch"
+    assert_file_not_exist "/usr/lib/systemd/system/elasticsearch.service"
+
+    assert_file_not_exist "/etc/sysconfig/elasticsearch"
+}
+
+@test "[RPM] finalize package removal" {
+    # cleanup
+    rm -rf /etc/elasticsearch
 }
 
 @test "[RPM] package has been removed again" {

From 5be79ed02ceb9fbe6c867a888ada2af8d53fd8d2 Mon Sep 17 00:00:00 2001
From: Chris Earle <chris.earle@elastic.co>
Date: Mon, 25 Apr 2016 15:55:25 -0400
Subject: [PATCH 0117/1311] Add Failure Details to every NodesResponse

Most of the current implementations of BaseNodesResponse (plural Nodes) ignore FailedNodeExceptions.

- This adds a helper function to do the grouping to TransportNodesAction
- Requires a non-null array of FailedNodeExceptions within the BaseNodesResponse constructor
- Reads/writes the array to output
- Also adds StreamInput and StreamOutput methods for generically reading and writing arrays
---
 .../elasticsearch/ElasticsearchException.java |   4 +-
 .../hotthreads/NodesHotThreadsResponse.java   |  22 +--
 .../TransportNodesHotThreadsAction.java       |  22 +--
 .../cluster/node/info/NodesInfoResponse.java  |  26 +--
 .../node/info/TransportNodesInfoAction.java   |  25 ++-
 .../node/stats/NodesStatsResponse.java        |  26 +--
 .../node/stats/TransportNodesStatsAction.java |  26 ++-
 .../status/TransportNodesSnapshotsStatus.java |  75 ++------
 .../cluster/stats/ClusterStatsIndices.java    |  79 +--------
 .../cluster/stats/ClusterStatsNodes.java      | 161 +-----------------
 .../cluster/stats/ClusterStatsResponse.java   |  77 +++------
 .../stats/TransportClusterStatsAction.java    |  19 +--
 .../TransportIndicesShardStoresAction.java    |   9 +-
 .../ingest/PutPipelineTransportAction.java    |   2 +-
 .../support/nodes/BaseNodesResponse.java      |  85 ++++++---
 .../support/nodes/TransportNodesAction.java   |  68 +++++++-
 .../cluster/InternalClusterInfoService.java   |   2 +-
 .../cluster/health/ClusterHealthStatus.java   |  24 ++-
 .../common/io/stream/StreamInput.java         |  24 +++
 .../common/io/stream/StreamOutput.java        |  14 +-
 .../gateway/AsyncShardFetch.java              |  18 +-
 .../org/elasticsearch/gateway/Gateway.java    |   6 +-
 .../gateway/GatewayAllocator.java             |   2 +-
 .../TransportNodesListGatewayMetaState.java   |  59 ++-----
 ...ransportNodesListGatewayStartedShards.java |  60 ++-----
 .../TransportNodesListShardStoreMetaData.java |  61 ++-----
 .../hotthreads/RestNodesHotThreadsAction.java |   2 +-
 .../node/info/RestNodesInfoAction.java        |  18 +-
 .../node/stats/RestNodesStatsAction.java      |   5 +-
 .../cluster/stats/RestClusterStatsAction.java |   5 +-
 .../rest/action/support/RestActions.java      | 156 +++++++++++++++--
 .../action/admin/HotThreadsIT.java            |   2 +-
 .../ClusterAllocationExplainIT.java           |   4 +-
 .../node/tasks/CancellableTasksTests.java     |   4 +-
 .../node/tasks/TaskManagerTestCase.java       |  40 ++---
 .../cluster/node/tasks/TestTaskPlugin.java    |  40 ++---
 .../nodes/TransportNodesActionTests.java      |  85 ++++++---
 .../ClusterStateBackwardsCompatIT.java        |   4 +-
 .../elasticsearch/cluster/DiskUsageTests.java |  10 +-
 .../ack/AckClusterUpdateSettingsIT.java       |   4 +-
 .../allocation/decider/MockDiskUsagesIT.java  |   2 +-
 .../common/io/stream/BytesStreamsTests.java   |  54 +++++-
 .../discovery/zen/ZenDiscoveryIT.java         |   4 +-
 .../gateway/AsyncShardFetchTests.java         |   7 +-
 .../gateway/RecoveryFromGatewayIT.java        |  10 +-
 .../index/store/CorruptedFileIT.java          |   8 +-
 .../index/store/CorruptedTranslogIT.java      |   2 +-
 .../index/suggest/stats/SuggestStatsIT.java   |   3 +-
 .../breaker/CircuitBreakerServiceIT.java      |   4 +-
 .../indices/recovery/IndexRecoveryIT.java     |   8 +-
 .../indices/stats/IndexStatsIT.java           |  24 +--
 .../nodesinfo/SimpleNodesInfoIT.java          |  14 +-
 .../threadpool/SimpleThreadPoolIT.java        |   4 +-
 .../netty/NettyTransportPublishAddressIT.java |   2 +-
 .../messy/tests/SearchStatsTests.java         |   4 +-
 .../AbstractAzureComputeServiceTestCase.java  |   2 +-
 .../MockInternalClusterInfoService.java       |   4 +-
 .../elasticsearch/test/ESIntegTestCase.java   |   4 +-
 .../org/elasticsearch/test/ExternalNode.java  |   6 +-
 .../test/ExternalTestCluster.java             |   6 +-
 60 files changed, 726 insertions(+), 821 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/ElasticsearchException.java b/core/src/main/java/org/elasticsearch/ElasticsearchException.java
index 3332bfed0c3..b242811b7be 100644
--- a/core/src/main/java/org/elasticsearch/ElasticsearchException.java
+++ b/core/src/main/java/org/elasticsearch/ElasticsearchException.java
@@ -24,6 +24,7 @@ import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -47,7 +48,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_UUID_NA_VAL
 /**
  * A base class for all elasticsearch exceptions.
  */
-public class ElasticsearchException extends RuntimeException implements ToXContent {
+public class ElasticsearchException extends RuntimeException implements ToXContent, Writeable {
 
     public static final String REST_EXCEPTION_SKIP_CAUSE = "rest.exception.cause.skip";
     public static final String REST_EXCEPTION_SKIP_STACK_TRACE = "rest.exception.stacktrace.skip";
@@ -235,6 +236,7 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
         }
     }
 
+    @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(this.getMessage());
         out.writeThrowable(this.getCause());
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsResponse.java
index 22d4795fc95..3136f2b6826 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsResponse.java
@@ -19,12 +19,14 @@
 
 package org.elasticsearch.action.admin.cluster.node.hotthreads;
 
+import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 
 import java.io.IOException;
+import java.util.List;
 
 /**
  */
@@ -33,26 +35,18 @@ public class NodesHotThreadsResponse extends BaseNodesResponse<NodeHotThreads> {
     NodesHotThreadsResponse() {
     }
 
-    public NodesHotThreadsResponse(ClusterName clusterName, NodeHotThreads[] nodes) {
-        super(clusterName, nodes);
+    public NodesHotThreadsResponse(ClusterName clusterName, List<NodeHotThreads> nodes, List<FailedNodeException> failures) {
+        super(clusterName, nodes, failures);
     }
 
     @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        nodes = new NodeHotThreads[in.readVInt()];
-        for (int i = 0; i < nodes.length; i++) {
-            nodes[i] = NodeHotThreads.readNodeHotThreads(in);
-        }
+    protected List<NodeHotThreads> readNodesFrom(StreamInput in) throws IOException {
+        return in.readList(NodeHotThreads::readNodeHotThreads);
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeVInt(nodes.length);
-        for (NodeHotThreads node : nodes) {
-            node.writeTo(out);
-        }
+    protected void writeNodesTo(StreamOutput out, List<NodeHotThreads> nodes) throws IOException {
+        out.writeStreamableList(nodes);
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java
index d53f651da45..7198851fd28 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.action.admin.cluster.node.hotthreads;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.nodes.BaseNodeRequest;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
@@ -35,33 +36,28 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.atomic.AtomicReferenceArray;
 
 /**
  *
  */
-public class TransportNodesHotThreadsAction extends TransportNodesAction<NodesHotThreadsRequest, NodesHotThreadsResponse, TransportNodesHotThreadsAction.NodeRequest, NodeHotThreads> {
+public class TransportNodesHotThreadsAction extends TransportNodesAction<NodesHotThreadsRequest,
+                                                                         NodesHotThreadsResponse,
+                                                                         TransportNodesHotThreadsAction.NodeRequest,
+                                                                         NodeHotThreads> {
 
     @Inject
     public TransportNodesHotThreadsAction(Settings settings, ClusterName clusterName, ThreadPool threadPool,
                                           ClusterService clusterService, TransportService transportService,
                                           ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
         super(settings, NodesHotThreadsAction.NAME, clusterName, threadPool, clusterService, transportService, actionFilters,
-                indexNameExpressionResolver, NodesHotThreadsRequest::new, NodeRequest::new, ThreadPool.Names.GENERIC);
+              indexNameExpressionResolver, NodesHotThreadsRequest::new, NodeRequest::new, ThreadPool.Names.GENERIC, NodeHotThreads.class);
     }
 
     @Override
-    protected NodesHotThreadsResponse newResponse(NodesHotThreadsRequest request, AtomicReferenceArray responses) {
-        final List<NodeHotThreads> nodes = new ArrayList<>();
-        for (int i = 0; i < responses.length(); i++) {
-            Object resp = responses.get(i);
-            if (resp instanceof NodeHotThreads) {
-                nodes.add((NodeHotThreads) resp);
-            }
-        }
-        return new NodesHotThreadsResponse(clusterName, nodes.toArray(new NodeHotThreads[nodes.size()]));
+    protected NodesHotThreadsResponse newResponse(NodesHotThreadsRequest request,
+                                                  List<NodeHotThreads> responses, List<FailedNodeException> failures) {
+        return new NodesHotThreadsResponse(clusterName, responses, failures);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java
index fdb15db8ffc..d5a43eb030e 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.admin.cluster.node.info;
 
+import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -30,6 +31,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -40,34 +42,24 @@ public class NodesInfoResponse extends BaseNodesResponse<NodeInfo> implements To
     public NodesInfoResponse() {
     }
 
-    public NodesInfoResponse(ClusterName clusterName, NodeInfo[] nodes) {
-        super(clusterName, nodes);
+    public NodesInfoResponse(ClusterName clusterName, List<NodeInfo> nodes, List<FailedNodeException> failures) {
+        super(clusterName, nodes, failures);
     }
 
     @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        nodes = new NodeInfo[in.readVInt()];
-        for (int i = 0; i < nodes.length; i++) {
-            nodes[i] = NodeInfo.readNodeInfo(in);
-        }
+    protected List<NodeInfo> readNodesFrom(StreamInput in) throws IOException {
+        return in.readList(NodeInfo::readNodeInfo);
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeVInt(nodes.length);
-        for (NodeInfo node : nodes) {
-            node.writeTo(out);
-        }
+    protected void writeNodesTo(StreamOutput out, List<NodeInfo> nodes) throws IOException {
+        out.writeStreamableList(nodes);
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field("cluster_name", getClusterName().value());
-
         builder.startObject("nodes");
-        for (NodeInfo nodeInfo : this) {
+        for (NodeInfo nodeInfo : getNodes()) {
             builder.startObject(nodeInfo.getNode().getId());
 
             builder.field("name", nodeInfo.getNode().getName());
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
index f52729faa4f..f68e2d65903 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.admin.cluster.node.info;
 
+import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.nodes.BaseNodeRequest;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
@@ -34,36 +35,32 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.atomic.AtomicReferenceArray;
 
 /**
  *
  */
-public class TransportNodesInfoAction extends TransportNodesAction<NodesInfoRequest, NodesInfoResponse, TransportNodesInfoAction.NodeInfoRequest, NodeInfo> {
+public class TransportNodesInfoAction extends TransportNodesAction<NodesInfoRequest,
+                                                                   NodesInfoResponse,
+                                                                   TransportNodesInfoAction.NodeInfoRequest,
+                                                                   NodeInfo> {
 
     private final NodeService nodeService;
 
     @Inject
     public TransportNodesInfoAction(Settings settings, ClusterName clusterName, ThreadPool threadPool,
                                     ClusterService clusterService, TransportService transportService,
-                                    NodeService nodeService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+                                    NodeService nodeService, ActionFilters actionFilters,
+                                    IndexNameExpressionResolver indexNameExpressionResolver) {
         super(settings, NodesInfoAction.NAME, clusterName, threadPool, clusterService, transportService, actionFilters,
-                indexNameExpressionResolver, NodesInfoRequest::new, NodeInfoRequest::new, ThreadPool.Names.MANAGEMENT);
+              indexNameExpressionResolver, NodesInfoRequest::new, NodeInfoRequest::new, ThreadPool.Names.MANAGEMENT, NodeInfo.class);
         this.nodeService = nodeService;
     }
 
     @Override
-    protected NodesInfoResponse newResponse(NodesInfoRequest nodesInfoRequest, AtomicReferenceArray responses) {
-        final List<NodeInfo> nodesInfos = new ArrayList<>();
-        for (int i = 0; i < responses.length(); i++) {
-            Object resp = responses.get(i);
-            if (resp instanceof NodeInfo) {
-                nodesInfos.add((NodeInfo) resp);
-            }
-        }
-        return new NodesInfoResponse(clusterName, nodesInfos.toArray(new NodeInfo[nodesInfos.size()]));
+    protected NodesInfoResponse newResponse(NodesInfoRequest nodesInfoRequest,
+                                            List<NodeInfo> responses, List<FailedNodeException> failures) {
+        return new NodesInfoResponse(clusterName, responses, failures);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsResponse.java
index af28c1fb5d5..1a9023ab93c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsResponse.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.admin.cluster.node.stats;
 
+import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -28,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 
 import java.io.IOException;
+import java.util.List;
 
 /**
  *
@@ -37,34 +39,24 @@ public class NodesStatsResponse extends BaseNodesResponse<NodeStats> implements
     NodesStatsResponse() {
     }
 
-    public NodesStatsResponse(ClusterName clusterName, NodeStats[] nodes) {
-        super(clusterName, nodes);
+    public NodesStatsResponse(ClusterName clusterName, List<NodeStats> nodes, List<FailedNodeException> failures) {
+        super(clusterName, nodes, failures);
     }
 
     @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        nodes = new NodeStats[in.readVInt()];
-        for (int i = 0; i < nodes.length; i++) {
-            nodes[i] = NodeStats.readNodeStats(in);
-        }
+    protected List<NodeStats> readNodesFrom(StreamInput in) throws IOException {
+        return in.readList(NodeStats::readNodeStats);
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeVInt(nodes.length);
-        for (NodeStats node : nodes) {
-            node.writeTo(out);
-        }
+    protected void writeNodesTo(StreamOutput out, List<NodeStats> nodes) throws IOException {
+        out.writeStreamableList(nodes);
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field("cluster_name", getClusterName().value());
-
         builder.startObject("nodes");
-        for (NodeStats nodeStats : this) {
+        for (NodeStats nodeStats : getNodes()) {
             builder.startObject(nodeStats.getNode().getId());
             builder.field("timestamp", nodeStats.getTimestamp());
             nodeStats.toXContent(builder, params);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
index 8ba3d00558b..d61e3f1acce 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.admin.cluster.node.stats;
 
+import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.nodes.BaseNodeRequest;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
@@ -34,36 +35,31 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.atomic.AtomicReferenceArray;
 
 /**
  *
  */
-public class TransportNodesStatsAction extends TransportNodesAction<NodesStatsRequest, NodesStatsResponse, TransportNodesStatsAction.NodeStatsRequest, NodeStats> {
+public class TransportNodesStatsAction extends TransportNodesAction<NodesStatsRequest,
+                                                                    NodesStatsResponse,
+                                                                    TransportNodesStatsAction.NodeStatsRequest,
+                                                                    NodeStats> {
 
     private final NodeService nodeService;
 
     @Inject
     public TransportNodesStatsAction(Settings settings, ClusterName clusterName, ThreadPool threadPool,
                                      ClusterService clusterService, TransportService transportService,
-                                     NodeService nodeService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, NodesStatsAction.NAME, clusterName, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
-                NodesStatsRequest::new, NodeStatsRequest::new, ThreadPool.Names.MANAGEMENT);
+                                     NodeService nodeService, ActionFilters actionFilters,
+                                     IndexNameExpressionResolver indexNameExpressionResolver) {
+        super(settings, NodesStatsAction.NAME, clusterName, threadPool, clusterService, transportService, actionFilters,
+              indexNameExpressionResolver, NodesStatsRequest::new, NodeStatsRequest::new, ThreadPool.Names.MANAGEMENT, NodeStats.class);
         this.nodeService = nodeService;
     }
 
     @Override
-    protected NodesStatsResponse newResponse(NodesStatsRequest nodesInfoRequest, AtomicReferenceArray responses) {
-        final List<NodeStats> nodeStats = new ArrayList<>();
-        for (int i = 0; i < responses.length(); i++) {
-            Object resp = responses.get(i);
-            if (resp instanceof NodeStats) {
-                nodeStats.add((NodeStats) resp);
-            }
-        }
-        return new NodesStatsResponse(clusterName, nodeStats.toArray(new NodeStats[nodeStats.size()]));
+    protected NodesStatsResponse newResponse(NodesStatsRequest request, List<NodeStats> responses, List<FailedNodeException> failures) {
+        return new NodesStatsResponse(clusterName, responses, failures);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java
index fb6310a45bf..bc139389460 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java
@@ -43,18 +43,20 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
-import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicReferenceArray;
 
 import static java.util.Collections.unmodifiableMap;
 
 /**
  * Transport client that collects snapshot shard statuses from data nodes
  */
-public class TransportNodesSnapshotsStatus extends TransportNodesAction<TransportNodesSnapshotsStatus.Request, TransportNodesSnapshotsStatus.NodesSnapshotStatus, TransportNodesSnapshotsStatus.NodeRequest, TransportNodesSnapshotsStatus.NodeSnapshotStatus> {
+public class TransportNodesSnapshotsStatus extends TransportNodesAction<TransportNodesSnapshotsStatus.Request,
+                                                                        TransportNodesSnapshotsStatus.NodesSnapshotStatus,
+                                                                        TransportNodesSnapshotsStatus.NodeRequest,
+                                                                        TransportNodesSnapshotsStatus.NodeSnapshotStatus> {
 
     public static final String ACTION_NAME = SnapshotsStatusAction.NAME + "[nodes]";
 
@@ -66,7 +68,7 @@ public class TransportNodesSnapshotsStatus extends TransportNodesAction<Transpor
                                          SnapshotShardsService snapshotShardsService, ActionFilters actionFilters,
                                          IndexNameExpressionResolver indexNameExpressionResolver) {
         super(settings, ACTION_NAME, clusterName, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
-                Request::new, NodeRequest::new, ThreadPool.Names.GENERIC);
+              Request::new, NodeRequest::new, ThreadPool.Names.GENERIC, NodeSnapshotStatus.class);
         this.snapshotShardsService = snapshotShardsService;
     }
 
@@ -86,21 +88,8 @@ public class TransportNodesSnapshotsStatus extends TransportNodesAction<Transpor
     }
 
     @Override
-    protected NodesSnapshotStatus newResponse(Request request, AtomicReferenceArray responses) {
-        final List<NodeSnapshotStatus> nodesList = new ArrayList<>();
-        final List<FailedNodeException> failures = new ArrayList<>();
-        for (int i = 0; i < responses.length(); i++) {
-            Object resp = responses.get(i);
-            if (resp instanceof NodeSnapshotStatus) { // will also filter out null response for unallocated ones
-                nodesList.add((NodeSnapshotStatus) resp);
-            } else if (resp instanceof FailedNodeException) {
-                failures.add((FailedNodeException) resp);
-            } else {
-                logger.warn("unknown response type [{}], expected NodeSnapshotStatus or FailedNodeException", resp);
-            }
-        }
-        return new NodesSnapshotStatus(clusterName, nodesList.toArray(new NodeSnapshotStatus[nodesList.size()]),
-                failures.toArray(new FailedNodeException[failures.size()]));
+    protected NodesSnapshotStatus newResponse(Request request, List<NodeSnapshotStatus> responses, List<FailedNodeException> failures) {
+        return new NodesSnapshotStatus(clusterName, responses, failures);
     }
 
     @Override
@@ -169,75 +158,47 @@ public class TransportNodesSnapshotsStatus extends TransportNodesAction<Transpor
 
     public static class NodesSnapshotStatus extends BaseNodesResponse<NodeSnapshotStatus> {
 
-        private FailedNodeException[] failures;
-
         NodesSnapshotStatus() {
         }
 
-        public NodesSnapshotStatus(ClusterName clusterName, NodeSnapshotStatus[] nodes, FailedNodeException[] failures) {
-            super(clusterName, nodes);
-            this.failures = failures;
+        public NodesSnapshotStatus(ClusterName clusterName, List<NodeSnapshotStatus> nodes, List<FailedNodeException> failures) {
+            super(clusterName, nodes, failures);
         }
 
         @Override
-        public FailedNodeException[] failures() {
-            return failures;
+        protected List<NodeSnapshotStatus> readNodesFrom(StreamInput in) throws IOException {
+            return in.readStreamableList(NodeSnapshotStatus::new);
         }
 
         @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            nodes = new NodeSnapshotStatus[in.readVInt()];
-            for (int i = 0; i < nodes.length; i++) {
-                nodes[i] = new NodeSnapshotStatus();
-                nodes[i].readFrom(in);
-            }
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeVInt(nodes.length);
-            for (NodeSnapshotStatus response : nodes) {
-                response.writeTo(out);
-            }
+        protected void writeNodesTo(StreamOutput out, List<NodeSnapshotStatus> nodes) throws IOException {
+            out.writeStreamableList(nodes);
         }
     }
 
 
     public static class NodeRequest extends BaseNodeRequest {
 
-        private SnapshotId[] snapshotIds;
+        private List<SnapshotId> snapshotIds;
 
         public NodeRequest() {
         }
 
         NodeRequest(String nodeId, TransportNodesSnapshotsStatus.Request request) {
             super(nodeId);
-            snapshotIds = request.snapshotIds;
+            snapshotIds = Arrays.asList(request.snapshotIds);
         }
 
         @Override
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
-            int n = in.readVInt();
-            snapshotIds = new SnapshotId[n];
-            for (int i = 0; i < n; i++) {
-                snapshotIds[i] = SnapshotId.readSnapshotId(in);
-            }
+            snapshotIds = in.readList(SnapshotId::readSnapshotId);
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            if (snapshotIds != null) {
-                out.writeVInt(snapshotIds.length);
-                for (int i = 0; i < snapshotIds.length; i++) {
-                    snapshotIds[i].writeTo(out);
-                }
-            } else {
-                out.writeVInt(0);
-            }
+            out.writeStreamableList(snapshotIds);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java
index b23b6467288..8c0c427beea 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java
@@ -22,9 +22,6 @@ package org.elasticsearch.action.admin.cluster.stats;
 import com.carrotsearch.hppc.ObjectObjectHashMap;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.cache.query.QueryCacheStats;
@@ -36,8 +33,9 @@ import org.elasticsearch.index.store.StoreStats;
 import org.elasticsearch.search.suggest.completion.CompletionStats;
 
 import java.io.IOException;
+import java.util.List;
 
-public class ClusterStatsIndices implements ToXContent, Streamable {
+public class ClusterStatsIndices implements ToXContent {
 
     private int indexCount;
     private ShardStats shards;
@@ -49,10 +47,7 @@ public class ClusterStatsIndices implements ToXContent, Streamable {
     private SegmentsStats segments;
     private PercolatorQueryCacheStats percolatorCache;
 
-    private ClusterStatsIndices() {
-    }
-
-    public ClusterStatsIndices(ClusterStatsNodeResponse[] nodeResponses) {
+    public ClusterStatsIndices(List<ClusterStatsNodeResponse> nodeResponses) {
         ObjectObjectHashMap<String, ShardStats> countsPerIndex = new ObjectObjectHashMap<>();
 
         this.docs = new DocsStats();
@@ -131,38 +126,6 @@ public class ClusterStatsIndices implements ToXContent, Streamable {
         return percolatorCache;
     }
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        indexCount = in.readVInt();
-        shards = ShardStats.readShardStats(in);
-        docs = DocsStats.readDocStats(in);
-        store = StoreStats.readStoreStats(in);
-        fieldData = FieldDataStats.readFieldDataStats(in);
-        queryCache = QueryCacheStats.readQueryCacheStats(in);
-        completion = CompletionStats.readCompletionStats(in);
-        segments = SegmentsStats.readSegmentsStats(in);
-        percolatorCache = PercolatorQueryCacheStats.readPercolateStats(in);
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(indexCount);
-        shards.writeTo(out);
-        docs.writeTo(out);
-        store.writeTo(out);
-        fieldData.writeTo(out);
-        queryCache.writeTo(out);
-        completion.writeTo(out);
-        segments.writeTo(out);
-        percolatorCache.writeTo(out);
-    }
-
-    public static ClusterStatsIndices readIndicesStats(StreamInput in) throws IOException {
-        ClusterStatsIndices indicesStats = new ClusterStatsIndices();
-        indicesStats.readFrom(in);
-        return indicesStats;
-    }
-
     static final class Fields {
         static final String COUNT = "count";
     }
@@ -181,7 +144,7 @@ public class ClusterStatsIndices implements ToXContent, Streamable {
         return builder;
     }
 
-    public static class ShardStats implements ToXContent, Streamable {
+    public static class ShardStats implements ToXContent {
 
         int indices;
         int total;
@@ -326,40 +289,6 @@ public class ClusterStatsIndices implements ToXContent, Streamable {
             }
         }
 
-        public static ShardStats readShardStats(StreamInput in) throws IOException {
-            ShardStats c = new ShardStats();
-            c.readFrom(in);
-            return c;
-        }
-
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            indices = in.readVInt();
-            total = in.readVInt();
-            primaries = in.readVInt();
-            minIndexShards = in.readVInt();
-            maxIndexShards = in.readVInt();
-            minIndexPrimaryShards = in.readVInt();
-            maxIndexPrimaryShards = in.readVInt();
-            minIndexReplication = in.readDouble();
-            totalIndexReplication = in.readDouble();
-            maxIndexReplication = in.readDouble();
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeVInt(indices);
-            out.writeVInt(total);
-            out.writeVInt(primaries);
-            out.writeVInt(minIndexShards);
-            out.writeVInt(maxIndexShards);
-            out.writeVInt(minIndexPrimaryShards);
-            out.writeVInt(maxIndexPrimaryShards);
-            out.writeDouble(minIndexReplication);
-            out.writeDouble(totalIndexReplication);
-            out.writeDouble(maxIndexReplication);
-        }
-
         static final class Fields {
             static final String SHARDS = "shards";
             static final String TOTAL = "total";
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java
index 05b7753ef3a..017b4481240 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java
@@ -26,9 +26,6 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -48,7 +45,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-public class ClusterStatsNodes implements ToXContent, Writeable {
+public class ClusterStatsNodes implements ToXContent {
 
     private final Counts counts;
     private final Set<Version> versions;
@@ -58,33 +55,12 @@ public class ClusterStatsNodes implements ToXContent, Writeable {
     private final FsInfo.Path fs;
     private final Set<PluginInfo> plugins;
 
-    ClusterStatsNodes(StreamInput in) throws IOException {
-        this.counts = new Counts(in);
-
-        int size = in.readVInt();
-        this.versions = new HashSet<>(size);
-        for (int i = 0; i < size; i++) {
-            this.versions.add(Version.readVersion(in));
-        }
-
-        this.os = new OsStats(in);
-        this.process = new ProcessStats(in);
-        this.jvm = new JvmStats(in);
-        this.fs = new FsInfo.Path(in);
-
-        size = in.readVInt();
-        this.plugins = new HashSet<>(size);
-        for (int i = 0; i < size; i++) {
-            this.plugins.add(PluginInfo.readFromStream(in));
-        }
-    }
-
-    ClusterStatsNodes(ClusterStatsNodeResponse[] nodeResponses) {
+    ClusterStatsNodes(List<ClusterStatsNodeResponse> nodeResponses) {
         this.versions = new HashSet<>();
         this.fs = new FsInfo.Path();
         this.plugins = new HashSet<>();
 
-        Set<InetAddress> seenAddresses = new HashSet<>(nodeResponses.length);
+        Set<InetAddress> seenAddresses = new HashSet<>(nodeResponses.size());
         List<NodeInfo> nodeInfos = new ArrayList<>();
         List<NodeStats> nodeStats = new ArrayList<>();
         for (ClusterStatsNodeResponse nodeResponse : nodeResponses) {
@@ -140,21 +116,6 @@ public class ClusterStatsNodes implements ToXContent, Writeable {
         return plugins;
     }
 
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        counts.writeTo(out);
-        out.writeVInt(versions.size());
-        for (Version v : versions) Version.writeVersion(v, out);
-        os.writeTo(out);
-        process.writeTo(out);
-        jvm.writeTo(out);
-        fs.writeTo(out);
-        out.writeVInt(plugins.size());
-        for (PluginInfo p : plugins) {
-            p.writeTo(out);
-        }
-    }
-
     static final class Fields {
         static final String COUNT = "count";
         static final String VERSIONS = "versions";
@@ -200,18 +161,12 @@ public class ClusterStatsNodes implements ToXContent, Writeable {
         return builder;
     }
 
-    public static class Counts implements Writeable, ToXContent {
+    public static class Counts implements ToXContent {
         static final String COORDINATING_ONLY = "coordinating_only";
 
         private final int total;
         private final Map<String, Integer> roles;
 
-        @SuppressWarnings("unchecked")
-        private Counts(StreamInput in) throws IOException {
-            this.total = in.readVInt();
-            this.roles = (Map<String, Integer>)in.readGenericValue();
-        }
-
         private Counts(List<NodeInfo> nodeInfos) {
             this.roles = new HashMap<>();
             for (DiscoveryNode.Role role : DiscoveryNode.Role.values()) {
@@ -243,12 +198,6 @@ public class ClusterStatsNodes implements ToXContent, Writeable {
             return roles;
         }
 
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeVInt(total);
-            out.writeGenericValue(roles);
-        }
-
         static final class Fields {
             static final String TOTAL = "total";
         }
@@ -263,7 +212,7 @@ public class ClusterStatsNodes implements ToXContent, Writeable {
         }
     }
 
-    public static class OsStats implements ToXContent, Writeable {
+    public static class OsStats implements ToXContent {
         final int availableProcessors;
         final int allocatedProcessors;
         final ObjectIntHashMap<String> names;
@@ -287,30 +236,6 @@ public class ClusterStatsNodes implements ToXContent, Writeable {
             this.allocatedProcessors = allocatedProcessors;
         }
 
-        /**
-         * Read from a stream.
-         */
-        private OsStats(StreamInput in) throws IOException {
-            this.availableProcessors = in.readVInt();
-            this.allocatedProcessors = in.readVInt();
-            int size = in.readVInt();
-            this.names = new ObjectIntHashMap<>();
-            for (int i = 0; i < size; i++) {
-                names.addTo(in.readString(), in.readVInt());
-            }
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeVInt(availableProcessors);
-            out.writeVInt(allocatedProcessors);
-            out.writeVInt(names.size());
-            for (ObjectIntCursor<String> name : names) {
-                out.writeString(name.key);
-                out.writeVInt(name.value);
-            }
-        }
-
         public int getAvailableProcessors() {
             return availableProcessors;
         }
@@ -343,7 +268,7 @@ public class ClusterStatsNodes implements ToXContent, Writeable {
         }
     }
 
-    public static class ProcessStats implements ToXContent, Writeable {
+    public static class ProcessStats implements ToXContent {
 
         final int count;
         final int cpuPercent;
@@ -384,27 +309,6 @@ public class ClusterStatsNodes implements ToXContent, Writeable {
             this.maxOpenFileDescriptors = maxOpenFileDescriptors;
         }
 
-        /**
-         * Read from a stream.
-         */
-        private ProcessStats(StreamInput in) throws IOException {
-            this.count = in.readVInt();
-            this.cpuPercent = in.readVInt();
-            this.totalOpenFileDescriptors = in.readVLong();
-            this.minOpenFileDescriptors = in.readLong();
-            this.maxOpenFileDescriptors = in.readLong();
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeVInt(count);
-            out.writeVInt(cpuPercent);
-            out.writeVLong(totalOpenFileDescriptors);
-            out.writeLong(minOpenFileDescriptors);
-            out.writeLong(maxOpenFileDescriptors);
-        }
-
-
         /**
          * Cpu usage in percentages - 100 is 1 core.
          */
@@ -456,7 +360,7 @@ public class ClusterStatsNodes implements ToXContent, Writeable {
         }
     }
 
-    public static class JvmStats implements Writeable, ToXContent {
+    public static class JvmStats implements ToXContent {
 
         private final ObjectIntHashMap<JvmVersion> versions;
         private final long threads;
@@ -497,34 +401,6 @@ public class ClusterStatsNodes implements ToXContent, Writeable {
             this.heapMax = heapMax;
         }
 
-        /**
-         * Read from a stream.
-         */
-        private JvmStats(StreamInput in) throws IOException {
-            int size = in.readVInt();
-            this.versions = new ObjectIntHashMap<>(size);
-            for (int i = 0; i < size; i++) {
-                this.versions.addTo(new JvmVersion(in), in.readVInt());
-            }
-            this.threads = in.readVLong();
-            this.maxUptime = in.readVLong();
-            this.heapUsed = in.readVLong();
-            this.heapMax = in.readVLong();
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeVInt(versions.size());
-            for (ObjectIntCursor<JvmVersion> v : versions) {
-                v.key.writeTo(out);
-                out.writeVInt(v.value);
-            }
-            out.writeVLong(threads);
-            out.writeVLong(maxUptime);
-            out.writeVLong(heapUsed);
-            out.writeVLong(heapMax);
-        }
-
         public ObjectIntHashMap<JvmVersion> getVersions() {
             return versions;
         }
@@ -598,7 +474,7 @@ public class ClusterStatsNodes implements ToXContent, Writeable {
         }
     }
 
-    public static class JvmVersion implements Writeable {
+    public static class JvmVersion {
         String version;
         String vmName;
         String vmVersion;
@@ -611,27 +487,6 @@ public class ClusterStatsNodes implements ToXContent, Writeable {
             vmVendor = jvmInfo.getVmVendor();
         }
 
-        /**
-         * Read from a stream.
-         */
-        JvmVersion(StreamInput in) throws IOException {
-            version = in.readString();
-            vmName = in.readString();
-            vmVersion = in.readString();
-            vmVendor = in.readString();
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeString(version);
-            out.writeString(vmName);
-            out.writeString(vmVersion);
-            out.writeString(vmVendor);
-        }
-
-        JvmVersion() {
-        }
-
         @Override
         public boolean equals(Object o) {
             if (this == o) {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java
index c272e6d6fbe..efc72d104f8 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.admin.cluster.stats;
 
+import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
@@ -29,9 +30,8 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 
 import java.io.IOException;
-import java.util.Iterator;
+import java.util.List;
 import java.util.Locale;
-import java.util.Map;
 
 /**
  *
@@ -48,8 +48,9 @@ public class ClusterStatsResponse extends BaseNodesResponse<ClusterStatsNodeResp
     ClusterStatsResponse() {
     }
 
-    public ClusterStatsResponse(long timestamp, ClusterName clusterName, String clusterUUID, ClusterStatsNodeResponse[] nodes) {
-        super(clusterName, null);
+    public ClusterStatsResponse(long timestamp, ClusterName clusterName, String clusterUUID,
+                                List<ClusterStatsNodeResponse> nodes, List<FailedNodeException> failures) {
+        super(clusterName, nodes, failures);
         this.timestamp = timestamp;
         this.clusterUUID = clusterUUID;
         nodesStats = new ClusterStatsNodes(nodes);
@@ -79,77 +80,53 @@ public class ClusterStatsResponse extends BaseNodesResponse<ClusterStatsNodeResp
         return indicesStats;
     }
 
-    @Override
-    public ClusterStatsNodeResponse[] getNodes() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public Map<String, ClusterStatsNodeResponse> getNodesMap() {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public ClusterStatsNodeResponse getAt(int position) {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public Iterator<ClusterStatsNodeResponse> iterator() {
-        throw new UnsupportedOperationException();
-    }
-
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         timestamp = in.readVLong();
-        status = null;
-        if (in.readBoolean()) {
-            // it may be that the master switched on us while doing the operation. In this case the status may be null.
-            status = ClusterHealthStatus.fromValue(in.readByte());
-        }
         clusterUUID = in.readString();
-        nodesStats = new ClusterStatsNodes(in);
-        indicesStats = ClusterStatsIndices.readIndicesStats(in);
+        // it may be that the master switched on us while doing the operation. In this case the status may be null.
+        status = in.readOptionalWriteable(ClusterHealthStatus::readFrom);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeVLong(timestamp);
-        if (status == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            out.writeByte(status.value());
-        }
         out.writeString(clusterUUID);
-        nodesStats.writeTo(out);
-        indicesStats.writeTo(out);
+        out.writeOptionalWriteable(status);
     }
 
-    static final class Fields {
-        static final String NODES = "nodes";
-        static final String INDICES = "indices";
-        static final String UUID = "uuid";
-        static final String CLUSTER_NAME = "cluster_name";
-        static final String STATUS = "status";
+    @Override
+    protected List<ClusterStatsNodeResponse> readNodesFrom(StreamInput in) throws IOException {
+        List<ClusterStatsNodeResponse> nodes = in.readList(ClusterStatsNodeResponse::readNodeResponse);
+
+        // built from nodes rather than from the stream directly
+        nodesStats = new ClusterStatsNodes(nodes);
+        indicesStats = new ClusterStatsIndices(nodes);
+
+        return nodes;
+    }
+
+    @Override
+    protected void writeNodesTo(StreamOutput out, List<ClusterStatsNodeResponse> nodes) throws IOException {
+        // nodeStats and indicesStats are rebuilt from nodes
+        out.writeStreamableList(nodes);
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field("timestamp", getTimestamp());
-        builder.field(Fields.CLUSTER_NAME, getClusterName().value());
         if (params.paramAsBoolean("output_uuid", false)) {
-            builder.field(Fields.UUID, clusterUUID);
+            builder.field("uuid", clusterUUID);
         }
         if (status != null) {
-            builder.field(Fields.STATUS, status.name().toLowerCase(Locale.ROOT));
+            builder.field("status", status.name().toLowerCase(Locale.ROOT));
         }
-        builder.startObject(Fields.INDICES);
+        builder.startObject("indices");
         indicesStats.toXContent(builder, params);
         builder.endObject();
-        builder.startObject(Fields.NODES);
+        builder.startObject("nodes");
         nodesStats.toXContent(builder, params);
         builder.endObject();
         return builder;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
index 4a0eb33c0b5..bae7b20694d 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.admin.cluster.stats;
 
+import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
@@ -46,7 +47,6 @@ import org.elasticsearch.transport.TransportService;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.atomic.AtomicReferenceArray;
 
 /**
  *
@@ -68,22 +68,17 @@ public class TransportClusterStatsAction extends TransportNodesAction<ClusterSta
                                        NodeService nodeService, IndicesService indicesService,
                                        ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
         super(settings, ClusterStatsAction.NAME, clusterName, threadPool, clusterService, transportService, actionFilters,
-                indexNameExpressionResolver, ClusterStatsRequest::new, ClusterStatsNodeRequest::new, ThreadPool.Names.MANAGEMENT);
+              indexNameExpressionResolver, ClusterStatsRequest::new, ClusterStatsNodeRequest::new, ThreadPool.Names.MANAGEMENT,
+              ClusterStatsNodeResponse.class);
         this.nodeService = nodeService;
         this.indicesService = indicesService;
     }
 
     @Override
-    protected ClusterStatsResponse newResponse(ClusterStatsRequest clusterStatsRequest, AtomicReferenceArray responses) {
-        final List<ClusterStatsNodeResponse> nodeStats = new ArrayList<>(responses.length());
-        for (int i = 0; i < responses.length(); i++) {
-            Object resp = responses.get(i);
-            if (resp instanceof ClusterStatsNodeResponse) {
-                nodeStats.add((ClusterStatsNodeResponse) resp);
-            }
-        }
-        return new ClusterStatsResponse(System.currentTimeMillis(), clusterName,
-                clusterService.state().metaData().clusterUUID(), nodeStats.toArray(new ClusterStatsNodeResponse[nodeStats.size()]));
+    protected ClusterStatsResponse newResponse(ClusterStatsRequest request,
+                                               List<ClusterStatsNodeResponse> responses, List<FailedNodeException> failures) {
+        return new ClusterStatsResponse(System.currentTimeMillis(), clusterName, clusterService.state().metaData().clusterUUID(),
+                                        responses, failures);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
index 5dbac12f694..250e4123bba 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java
@@ -53,6 +53,7 @@ import org.elasticsearch.transport.TransportService;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Queue;
 import java.util.Set;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -152,7 +153,7 @@ public class TransportIndicesShardStoresAction extends TransportMasterNodeReadAc
             }
 
             @Override
-            protected synchronized void processAsyncFetch(ShardId shardId, NodeGatewayStartedShards[] responses, FailedNodeException[] failures) {
+            protected synchronized void processAsyncFetch(ShardId shardId, List<NodeGatewayStartedShards> responses, List<FailedNodeException> failures) {
                 fetchResponses.add(new Response(shardId, responses, failures));
                 if (expectedOps.countDown()) {
                     finish();
@@ -220,10 +221,10 @@ public class TransportIndicesShardStoresAction extends TransportMasterNodeReadAc
 
             public class Response {
                 private final ShardId shardId;
-                private final NodeGatewayStartedShards[] responses;
-                private final FailedNodeException[] failures;
+                private final List<NodeGatewayStartedShards> responses;
+                private final List<FailedNodeException> failures;
 
-                public Response(ShardId shardId, NodeGatewayStartedShards[] responses, FailedNodeException[] failures) {
+                public Response(ShardId shardId, List<NodeGatewayStartedShards> responses, List<FailedNodeException> failures) {
                     this.shardId = shardId;
                     this.responses = responses;
                     this.failures = failures;
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
index e1a34413e2c..bea5a2c8bc3 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
@@ -80,7 +80,7 @@ public class PutPipelineTransportAction extends TransportMasterNodeAction<PutPip
             public void onResponse(NodesInfoResponse nodeInfos) {
                 try {
                     Map<DiscoveryNode, IngestInfo> ingestInfos = new HashMap<>();
-                    for (NodeInfo nodeInfo : nodeInfos) {
+                    for (NodeInfo nodeInfo : nodeInfos.getNodes()) {
                         ingestInfos.put(nodeInfo.getNode(), nodeInfo.getIngest());
                     }
                     pipelineStore.put(clusterService, ingestInfos, request, listener);
diff --git a/core/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesResponse.java b/core/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesResponse.java
index 01401bc7c6e..a49864154db 100644
--- a/core/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesResponse.java
@@ -22,61 +22,77 @@ package org.elasticsearch.action.support.nodes;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 
 import java.io.IOException;
 import java.util.HashMap;
-import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 /**
  *
  */
-public abstract class BaseNodesResponse<TNodeResponse extends BaseNodeResponse> extends ActionResponse implements Iterable<TNodeResponse> {
+public abstract class BaseNodesResponse<TNodeResponse extends BaseNodeResponse> extends ActionResponse {
 
     private ClusterName clusterName;
-    protected TNodeResponse[] nodes;
+    private List<FailedNodeException> failures;
+    private List<TNodeResponse> nodes;
     private Map<String, TNodeResponse> nodesMap;
 
     protected BaseNodesResponse() {
     }
 
-    protected BaseNodesResponse(ClusterName clusterName, TNodeResponse[] nodes) {
-        this.clusterName = clusterName;
-        this.nodes = nodes;
+    protected BaseNodesResponse(ClusterName clusterName, List<TNodeResponse> nodes, List<FailedNodeException> failures) {
+        this.clusterName = Objects.requireNonNull(clusterName);
+        this.failures = Objects.requireNonNull(failures);
+        this.nodes = Objects.requireNonNull(nodes);
     }
 
     /**
-     * The failed nodes, if set to be captured.
+     * Get the {@link ClusterName} associated with all of the nodes.
+     *
+     * @return Never {@code null}.
      */
-    @Nullable
-    public FailedNodeException[] failures() {
-        return null;
-    }
-
     public ClusterName getClusterName() {
-        return this.clusterName;
+        return clusterName;
     }
 
-    public String getClusterNameAsString() {
-        return this.clusterName.value();
+    /**
+     * Get the failed node exceptions.
+     *
+     * @return Never {@code null}. Can be empty.
+     */
+    public List<FailedNodeException> failures() {
+        return failures;
     }
 
-    public TNodeResponse[] getNodes() {
+    /**
+     * Determine if there are any node failures in {@link #failures}.
+     *
+     * @return {@code true} if {@link #failures} contains at least 1 {@link FailedNodeException}.
+     */
+    public boolean hasFailures() {
+        return failures.isEmpty() == false;
+    }
+
+    /**
+     * Get the <em>successful</em> node responses.
+     *
+     * @return Never {@code null}. Can be empty.
+     * @see #hasFailures()
+     */
+    public List<TNodeResponse> getNodes() {
         return nodes;
     }
 
-    public TNodeResponse getAt(int position) {
-        return nodes[position];
-    }
-
-    @Override
-    public Iterator<TNodeResponse> iterator() {
-        return getNodesMap().values().iterator();
-    }
-
+    /**
+     * Lazily build and get a map of Node ID to node response.
+     *
+     * @return Never {@code null}. Can be empty.
+     * @see #getNodes()
+     */
     public Map<String, TNodeResponse> getNodesMap() {
         if (nodesMap == null) {
             nodesMap = new HashMap<>();
@@ -91,11 +107,28 @@ public abstract class BaseNodesResponse<TNodeResponse extends BaseNodeResponse>
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         clusterName = ClusterName.readClusterName(in);
+        nodes = readNodesFrom(in);
+        failures = in.readList(FailedNodeException::new);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         clusterName.writeTo(out);
+        writeNodesTo(out, nodes);
+        out.writeList(failures);
     }
+
+    /**
+     * Read the {@link #nodes} from the stream.
+     *
+     * @return Never {@code null}.
+     */
+    protected abstract List<TNodeResponse> readNodesFrom(StreamInput in) throws IOException;
+
+    /**
+     * Write the {@link #nodes} to the stream.
+     */
+    protected abstract void writeNodesTo(StreamOutput out, List<TNodeResponse> nodes) throws IOException;
+
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
index c996d530dce..2767bc80bf3 100644
--- a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
@@ -43,6 +43,9 @@ import org.elasticsearch.transport.TransportRequestHandler;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReferenceArray;
 import java.util.function.Supplier;
@@ -50,22 +53,30 @@ import java.util.function.Supplier;
 /**
  *
  */
-public abstract class TransportNodesAction<NodesRequest extends BaseNodesRequest<NodesRequest>, NodesResponse extends BaseNodesResponse, NodeRequest extends BaseNodeRequest, NodeResponse extends BaseNodeResponse> extends HandledTransportAction<NodesRequest, NodesResponse> {
+public abstract class TransportNodesAction<NodesRequest extends BaseNodesRequest<NodesRequest>,
+                                           NodesResponse extends BaseNodesResponse,
+                                           NodeRequest extends BaseNodeRequest,
+                                           NodeResponse extends BaseNodeResponse>
+    extends HandledTransportAction<NodesRequest, NodesResponse> {
 
     protected final ClusterName clusterName;
     protected final ClusterService clusterService;
     protected final TransportService transportService;
+    protected final Class<NodeResponse> nodeResponseClass;
 
     final String transportNodeAction;
 
     protected TransportNodesAction(Settings settings, String actionName, ClusterName clusterName, ThreadPool threadPool,
                                    ClusterService clusterService, TransportService transportService, ActionFilters actionFilters,
-                                   IndexNameExpressionResolver indexNameExpressionResolver, Supplier<NodesRequest> request, Supplier<NodeRequest> nodeRequest,
-                                   String nodeExecutor) {
+                                   IndexNameExpressionResolver indexNameExpressionResolver,
+                                   Supplier<NodesRequest> request, Supplier<NodeRequest> nodeRequest,
+                                   String nodeExecutor,
+                                   Class<NodeResponse> nodeResponseClass) {
         super(settings, actionName, threadPool, transportService, actionFilters, indexNameExpressionResolver, request);
-        this.clusterName = clusterName;
-        this.clusterService = clusterService;
-        this.transportService = transportService;
+        this.clusterName = Objects.requireNonNull(clusterName);
+        this.clusterService = Objects.requireNonNull(clusterService);
+        this.transportService = Objects.requireNonNull(transportService);
+        this.nodeResponseClass = Objects.requireNonNull(nodeResponseClass);
 
         this.transportNodeAction = actionName + "[n]";
 
@@ -87,7 +98,46 @@ public abstract class TransportNodesAction<NodesRequest extends BaseNodesRequest
         return false;
     }
 
-    protected abstract NodesResponse newResponse(NodesRequest request, AtomicReferenceArray nodesResponses);
+    /**
+     * Map the responses into {@code nodeResponseClass} responses and {@link FailedNodeException}s.
+     *
+     * @param request The associated request.
+     * @param nodesResponses All node-level responses
+     * @return Never {@code null}.
+     * @throws NullPointerException if {@code nodesResponses} is {@code null}
+     * @see #newResponse(BaseNodesRequest, List, List)
+     */
+    protected NodesResponse newResponse(NodesRequest request, AtomicReferenceArray nodesResponses) {
+        final List<NodeResponse> responses = new ArrayList<>();
+        final List<FailedNodeException> failures = new ArrayList<>();
+
+        for (int i = 0; i < nodesResponses.length(); ++i) {
+            Object response = nodesResponses.get(i);
+
+            if (nodeResponseClass.isInstance(response)) {
+                responses.add(nodeResponseClass.cast(response));
+            } else if (response instanceof FailedNodeException) {
+                failures.add((FailedNodeException)response);
+            } else {
+                logger.warn("ignoring unexpected response [{}] of type [{}], expected [{}] or [{}]",
+                            response, response != null ? response.getClass().getName() : null,
+                            nodeResponseClass.getSimpleName(), FailedNodeException.class.getSimpleName());
+            }
+        }
+
+        return newResponse(request, responses, failures);
+    }
+
+    /**
+     * Create a new {@link NodesResponse} (multi-node response).
+     *
+     * @param request The associated request.
+     * @param responses All successful node-level responses.
+     * @param failures All node-level failures.
+     * @return Never {@code null}.
+     * @throws NullPointerException if any parameter is {@code null}.
+     */
+    protected abstract NodesResponse newResponse(NodesRequest request, List<NodeResponse> responses, List<FailedNodeException> failures);
 
     protected abstract NodeRequest newNodeRequest(String nodeId, NodesRequest request);
 
@@ -165,7 +215,8 @@ public abstract class TransportNodesAction<NodesRequest extends BaseNodesRequest
                             taskManager.registerChildTask(task, node.getId());
                         }
 
-                        transportService.sendRequest(node, transportNodeAction, nodeRequest, builder.build(), new BaseTransportResponseHandler<NodeResponse>() {
+                        transportService.sendRequest(node, transportNodeAction, nodeRequest, builder.build(),
+                                                     new BaseTransportResponseHandler<NodeResponse>() {
                             @Override
                             public NodeResponse newInstance() {
                                 return newNodeResponse();
@@ -238,4 +289,5 @@ public abstract class TransportNodesAction<NodesRequest extends BaseNodesRequest
         }
 
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
index b1362538cb9..3101e700327 100644
--- a/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
@@ -404,7 +404,7 @@ public class InternalClusterInfoService extends AbstractComponent implements Clu
         }
     }
 
-    static void fillDiskUsagePerNode(ESLogger logger, NodeStats[] nodeStatsArray,
+    static void fillDiskUsagePerNode(ESLogger logger, List<NodeStats> nodeStatsArray,
             ImmutableOpenMap.Builder<String, DiskUsage> newLeastAvaiableUsages,
             ImmutableOpenMap.Builder<String, DiskUsage> newMostAvaiableUsages) {
         for (NodeStats nodeStats : nodeStatsArray) {
diff --git a/core/src/main/java/org/elasticsearch/cluster/health/ClusterHealthStatus.java b/core/src/main/java/org/elasticsearch/cluster/health/ClusterHealthStatus.java
index 6d3e136eb1a..a261d28f537 100644
--- a/core/src/main/java/org/elasticsearch/cluster/health/ClusterHealthStatus.java
+++ b/core/src/main/java/org/elasticsearch/cluster/health/ClusterHealthStatus.java
@@ -20,10 +20,16 @@
 package org.elasticsearch.cluster.health;
 
 
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+
 /**
  *
  */
-public enum ClusterHealthStatus {
+public enum ClusterHealthStatus implements Writeable {
     GREEN((byte) 0),
     YELLOW((byte) 1),
     RED((byte) 2);
@@ -38,7 +44,21 @@ public enum ClusterHealthStatus {
         return value;
     }
 
-    public static ClusterHealthStatus fromValue(byte value) {
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeByte(value);
+    }
+
+    /**
+     * Read from a stream.
+     *
+     * @throws IllegalArgumentException if the value is unrecognized
+     */
+    public static ClusterHealthStatus readFrom(StreamInput in) throws IOException {
+        return fromValue(in.readByte());
+    }
+
+    public static ClusterHealthStatus fromValue(byte value) throws IOException {
         switch (value) {
             case 0:
                 return GREEN;
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
index 8df43f687f6..ec56103e7a0 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
@@ -736,6 +736,29 @@ public abstract class StreamInput extends InputStream {
         return null;
     }
 
+    /**
+     * Read a {@link List} of {@link Streamable} objects, using the {@code constructor} to instantiate each instance.
+     * <p>
+     * This is expected to take the form:
+     * <code>
+     * List&lt;MyStreamableClass&gt; list = in.readStreamList(MyStreamableClass::new);
+     * </code>
+     *
+     * @param constructor Streamable instance creator
+     * @return Never {@code null}.
+     * @throws IOException if any step fails
+     */
+    public <T extends Streamable> List<T> readStreamableList(Supplier<T> constructor) throws IOException {
+        int count = readVInt();
+        List<T> builder = new ArrayList<>(count);
+        for (int i=0; i<count; i++) {
+            T instance = constructor.get();
+            instance.readFrom(this);
+            builder.add(instance);
+        }
+        return builder;
+    }
+
     /**
      * Reads a list of objects
      */
@@ -762,4 +785,5 @@ public abstract class StreamInput extends InputStream {
     public static StreamInput wrap(byte[] bytes, int offset, int length) {
         return new InputStreamStreamInput(new ByteArrayInputStream(bytes, offset, length));
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
index 5b774d82cea..eb93c9af628 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
@@ -727,12 +727,22 @@ public abstract class StreamOutput extends OutputStream {
         }
     }
 
+    /**
+     * Writes a list of {@link Streamable} objects
+     */
+    public void writeStreamableList(List<? extends Streamable> list) throws IOException {
+        writeVInt(list.size());
+        for (Streamable obj: list) {
+            obj.writeTo(this);
+        }
+    }
+
     /**
      * Writes a list of {@link Writeable} objects
      */
-    public <T extends Writeable> void writeList(List<T> list) throws IOException {
+    public void writeList(List<? extends Writeable> list) throws IOException {
         writeVInt(list.size());
-        for (T obj: list) {
+        for (Writeable obj: list) {
             obj.writeTo(this);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java b/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java
index cd6268d04ec..b74507a4acc 100644
--- a/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java
+++ b/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java
@@ -35,9 +35,11 @@ import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.transport.ReceiveTimeoutTransportException;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -57,24 +59,24 @@ public abstract class AsyncShardFetch<T extends BaseNodeResponse> implements Rel
     /**
      * An action that lists the relevant shard data that needs to be fetched.
      */
-    public interface List<NodesResponse extends BaseNodesResponse<NodeResponse>, NodeResponse extends BaseNodeResponse> {
+    public interface Lister<NodesResponse extends BaseNodesResponse<NodeResponse>, NodeResponse extends BaseNodeResponse> {
         void list(ShardId shardId, String[] nodesIds, ActionListener<NodesResponse> listener);
     }
 
     protected final ESLogger logger;
     protected final String type;
     private final ShardId shardId;
-    private final List<BaseNodesResponse<T>, T> action;
+    private final Lister<BaseNodesResponse<T>, T> action;
     private final Map<String, NodeEntry<T>> cache = new HashMap<>();
     private final Set<String> nodesToIgnore = new HashSet<>();
     private boolean closed;
 
     @SuppressWarnings("unchecked")
-    protected AsyncShardFetch(ESLogger logger, String type, ShardId shardId, List<? extends BaseNodesResponse<T>, T> action) {
+    protected AsyncShardFetch(ESLogger logger, String type, ShardId shardId, Lister<? extends BaseNodesResponse<T>, T> action) {
         this.logger = logger;
         this.type = type;
         this.shardId = shardId;
-        this.action = (List<BaseNodesResponse<T>, T>) action;
+        this.action = (Lister<BaseNodesResponse<T>, T>) action;
     }
 
     @Override
@@ -167,7 +169,7 @@ public abstract class AsyncShardFetch<T extends BaseNodeResponse> implements Rel
      * the shard (response + failures), issuing a reroute at the end of it to make sure there will be another round
      * of allocations taking this new data into account.
      */
-    protected synchronized void processAsyncFetch(ShardId shardId, T[] responses, FailedNodeException[] failures) {
+    protected synchronized void processAsyncFetch(ShardId shardId, List<T> responses, List<FailedNodeException> failures) {
         if (closed) {
             // we are closed, no need to process this async fetch at all
             logger.trace("{} ignoring fetched [{}] results, already closed", shardId, type);
@@ -276,9 +278,9 @@ public abstract class AsyncShardFetch<T extends BaseNodeResponse> implements Rel
 
             @Override
             public void onFailure(Throwable e) {
-                FailedNodeException[] failures = new FailedNodeException[nodesIds.length];
-                for (int i = 0; i < failures.length; i++) {
-                    failures[i] = new FailedNodeException(nodesIds[i], "total failure in fetching", e);
+                List<FailedNodeException> failures = new ArrayList<>(nodesIds.length);
+                for (String nodeId : nodesIds) {
+                    failures.add(new FailedNodeException(nodeId, "total failure in fetching", e));
                 }
                 processAsyncFetch(shardId, null, failures);
             }
diff --git a/core/src/main/java/org/elasticsearch/gateway/Gateway.java b/core/src/main/java/org/elasticsearch/gateway/Gateway.java
index 4da789b43fc..0e6b1959279 100644
--- a/core/src/main/java/org/elasticsearch/gateway/Gateway.java
+++ b/core/src/main/java/org/elasticsearch/gateway/Gateway.java
@@ -82,7 +82,7 @@ public class Gateway extends AbstractComponent implements ClusterStateListener {
         int requiredAllocation = Math.max(1, minimumMasterNodesProvider.get());
 
 
-        if (nodesState.failures().length > 0) {
+        if (nodesState.hasFailures()) {
             for (FailedNodeException failedNodeException : nodesState.failures()) {
                 logger.warn("failed to fetch state from node", failedNodeException);
             }
@@ -91,7 +91,7 @@ public class Gateway extends AbstractComponent implements ClusterStateListener {
         ObjectFloatHashMap<Index> indices = new ObjectFloatHashMap<>();
         MetaData electedGlobalState = null;
         int found = 0;
-        for (TransportNodesListGatewayMetaState.NodeGatewayMetaState nodeState : nodesState) {
+        for (TransportNodesListGatewayMetaState.NodeGatewayMetaState nodeState : nodesState.getNodes()) {
             if (nodeState.metaData() == null) {
                 continue;
             }
@@ -119,7 +119,7 @@ public class Gateway extends AbstractComponent implements ClusterStateListener {
                 Index index = (Index) keys[i];
                 IndexMetaData electedIndexMetaData = null;
                 int indexMetaDataCount = 0;
-                for (TransportNodesListGatewayMetaState.NodeGatewayMetaState nodeState : nodesState) {
+                for (TransportNodesListGatewayMetaState.NodeGatewayMetaState nodeState : nodesState.getNodes()) {
                     if (nodeState.metaData() == null) {
                         continue;
                     }
diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java b/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
index e76e8085e86..15cd0e2bf63 100644
--- a/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
+++ b/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
@@ -125,7 +125,7 @@ public class GatewayAllocator extends AbstractComponent {
 
     class InternalAsyncFetch<T extends BaseNodeResponse> extends AsyncShardFetch<T> {
 
-        public InternalAsyncFetch(ESLogger logger, String type, ShardId shardId, List<? extends BaseNodesResponse<T>, T> action) {
+        public InternalAsyncFetch(ESLogger logger, String type, ShardId shardId, Lister<? extends BaseNodesResponse<T>, T> action) {
             super(logger, type, shardId, action);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java
index 0fd1fd35809..2d0b894939d 100644
--- a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java
+++ b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java
@@ -43,14 +43,15 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.atomic.AtomicReferenceArray;
 
 /**
  *
  */
-public class TransportNodesListGatewayMetaState extends TransportNodesAction<TransportNodesListGatewayMetaState.Request, TransportNodesListGatewayMetaState.NodesGatewayMetaState, TransportNodesListGatewayMetaState.NodeRequest, TransportNodesListGatewayMetaState.NodeGatewayMetaState> {
+public class TransportNodesListGatewayMetaState extends TransportNodesAction<TransportNodesListGatewayMetaState.Request,
+                                                                             TransportNodesListGatewayMetaState.NodesGatewayMetaState,
+                                                                             TransportNodesListGatewayMetaState.NodeRequest,
+                                                                             TransportNodesListGatewayMetaState.NodeGatewayMetaState> {
 
     public static final String ACTION_NAME = "internal:gateway/local/meta_state";
 
@@ -61,7 +62,7 @@ public class TransportNodesListGatewayMetaState extends TransportNodesAction<Tra
                                               ClusterService clusterService, TransportService transportService,
                                               ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
         super(settings, ACTION_NAME, clusterName, threadPool, clusterService, transportService, actionFilters,
-                indexNameExpressionResolver, Request::new, NodeRequest::new, ThreadPool.Names.GENERIC);
+              indexNameExpressionResolver, Request::new, NodeRequest::new, ThreadPool.Names.GENERIC, NodeGatewayMetaState.class);
     }
 
     TransportNodesListGatewayMetaState init(GatewayMetaState metaState) {
@@ -80,7 +81,7 @@ public class TransportNodesListGatewayMetaState extends TransportNodesAction<Tra
 
     @Override
     protected NodeRequest newNodeRequest(String nodeId, Request request) {
-        return new NodeRequest(nodeId, request);
+        return new NodeRequest(nodeId);
     }
 
     @Override
@@ -89,21 +90,8 @@ public class TransportNodesListGatewayMetaState extends TransportNodesAction<Tra
     }
 
     @Override
-    protected NodesGatewayMetaState newResponse(Request request, AtomicReferenceArray responses) {
-        final List<NodeGatewayMetaState> nodesList = new ArrayList<>();
-        final List<FailedNodeException> failures = new ArrayList<>();
-        for (int i = 0; i < responses.length(); i++) {
-            Object resp = responses.get(i);
-            if (resp instanceof NodeGatewayMetaState) { // will also filter out null response for unallocated ones
-                nodesList.add((NodeGatewayMetaState) resp);
-            } else if (resp instanceof FailedNodeException) {
-                failures.add((FailedNodeException) resp);
-            } else {
-                logger.warn("unknown response type [{}], expected NodeLocalGatewayMetaState or FailedNodeException", resp);
-            }
-        }
-        return new NodesGatewayMetaState(clusterName, nodesList.toArray(new NodeGatewayMetaState[nodesList.size()]),
-                failures.toArray(new FailedNodeException[failures.size()]));
+    protected NodesGatewayMetaState newResponse(Request request, List<NodeGatewayMetaState> responses, List<FailedNodeException> failures) {
+        return new NodesGatewayMetaState(clusterName, responses, failures);
     }
 
     @Override
@@ -142,47 +130,30 @@ public class TransportNodesListGatewayMetaState extends TransportNodesAction<Tra
 
     public static class NodesGatewayMetaState extends BaseNodesResponse<NodeGatewayMetaState> {
 
-        private FailedNodeException[] failures;
-
         NodesGatewayMetaState() {
         }
 
-        public NodesGatewayMetaState(ClusterName clusterName, NodeGatewayMetaState[] nodes, FailedNodeException[] failures) {
-            super(clusterName, nodes);
-            this.failures = failures;
-        }
-
-        public FailedNodeException[] failures() {
-            return failures;
+        public NodesGatewayMetaState(ClusterName clusterName, List<NodeGatewayMetaState> nodes, List<FailedNodeException> failures) {
+            super(clusterName, nodes, failures);
         }
 
         @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            nodes = new NodeGatewayMetaState[in.readVInt()];
-            for (int i = 0; i < nodes.length; i++) {
-                nodes[i] = new NodeGatewayMetaState();
-                nodes[i].readFrom(in);
-            }
+        protected List<NodeGatewayMetaState> readNodesFrom(StreamInput in) throws IOException {
+            return in.readStreamableList(NodeGatewayMetaState::new);
         }
 
         @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeVInt(nodes.length);
-            for (NodeGatewayMetaState response : nodes) {
-                response.writeTo(out);
-            }
+        protected void writeNodesTo(StreamOutput out, List<NodeGatewayMetaState> nodes) throws IOException {
+            out.writeStreamableList(nodes);
         }
     }
 
-
     public static class NodeRequest extends BaseNodeRequest {
 
         public NodeRequest() {
         }
 
-        NodeRequest(String nodeId, TransportNodesListGatewayMetaState.Request request) {
+        NodeRequest(String nodeId) {
             super(nodeId);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
index bdeb6d1660f..675c0088082 100644
--- a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
+++ b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
@@ -48,9 +48,7 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.atomic.AtomicReferenceArray;
 
 /**
  * This transport action is used to fetch the shard version from each node during primary allocation in {@link GatewayAllocator}.
@@ -63,7 +61,7 @@ public class TransportNodesListGatewayStartedShards extends
         TransportNodesListGatewayStartedShards.NodeRequest,
         TransportNodesListGatewayStartedShards.NodeGatewayStartedShards>
     implements
-    AsyncShardFetch.List<TransportNodesListGatewayStartedShards.NodesGatewayStartedShards,
+    AsyncShardFetch.Lister<TransportNodesListGatewayStartedShards.NodesGatewayStartedShards,
         TransportNodesListGatewayStartedShards.NodeGatewayStartedShards> {
 
     public static final String ACTION_NAME = "internal:gateway/local/started_shards";
@@ -77,7 +75,8 @@ public class TransportNodesListGatewayStartedShards extends
                                                   IndexNameExpressionResolver indexNameExpressionResolver,
                                                   NodeEnvironment env) {
         super(settings, ACTION_NAME, clusterName, threadPool, clusterService, transportService, actionFilters,
-            indexNameExpressionResolver, Request::new, NodeRequest::new, ThreadPool.Names.FETCH_SHARD_STARTED);
+              indexNameExpressionResolver, Request::new, NodeRequest::new, ThreadPool.Names.FETCH_SHARD_STARTED,
+              NodeGatewayStartedShards.class);
         this.nodeEnv = env;
     }
 
@@ -110,23 +109,9 @@ public class TransportNodesListGatewayStartedShards extends
     }
 
     @Override
-    protected NodesGatewayStartedShards newResponse(Request request, AtomicReferenceArray responses) {
-        final List<NodeGatewayStartedShards> nodesList = new ArrayList<>();
-        final List<FailedNodeException> failures = new ArrayList<>();
-        for (int i = 0; i < responses.length(); i++) {
-            Object resp = responses.get(i);
-            if (resp instanceof NodeGatewayStartedShards) { // will also filter out null response for unallocated ones
-                nodesList.add((NodeGatewayStartedShards) resp);
-            } else if (resp instanceof FailedNodeException) {
-                failures.add((FailedNodeException) resp);
-            } else {
-                logger.warn("unknown response type [{}], expected NodeLocalGatewayStartedShards or FailedNodeException",
-                    resp);
-            }
-        }
-        return new NodesGatewayStartedShards(clusterName,
-            nodesList.toArray(new NodeGatewayStartedShards[nodesList.size()]),
-            failures.toArray(new FailedNodeException[failures.size()]));
+    protected NodesGatewayStartedShards newResponse(Request request,
+                                                    List<NodeGatewayStartedShards> responses, List<FailedNodeException> failures) {
+        return new NodesGatewayStartedShards(clusterName, responses, failures);
     }
 
     @Override
@@ -217,36 +202,19 @@ public class TransportNodesListGatewayStartedShards extends
 
     public static class NodesGatewayStartedShards extends BaseNodesResponse<NodeGatewayStartedShards> {
 
-        private FailedNodeException[] failures;
-
-        public NodesGatewayStartedShards(ClusterName clusterName, NodeGatewayStartedShards[] nodes,
-                                         FailedNodeException[] failures) {
-            super(clusterName, nodes);
-            this.failures = failures;
+        public NodesGatewayStartedShards(ClusterName clusterName, List<NodeGatewayStartedShards> nodes,
+                                         List<FailedNodeException> failures) {
+            super(clusterName, nodes, failures);
         }
 
         @Override
-        public FailedNodeException[] failures() {
-            return failures;
+        protected List<NodeGatewayStartedShards> readNodesFrom(StreamInput in) throws IOException {
+            return in.readStreamableList(NodeGatewayStartedShards::new);
         }
 
         @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            nodes = new NodeGatewayStartedShards[in.readVInt()];
-            for (int i = 0; i < nodes.length; i++) {
-                nodes[i] = new NodeGatewayStartedShards();
-                nodes[i].readFrom(in);
-            }
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeVInt(nodes.length);
-            for (NodeGatewayStartedShards response : nodes) {
-                response.writeTo(out);
-            }
+        protected void writeNodesTo(StreamOutput out, List<NodeGatewayStartedShards> nodes) throws IOException {
+            out.writeStreamableList(nodes);
         }
     }
 
@@ -258,7 +226,7 @@ public class TransportNodesListGatewayStartedShards extends
         public NodeRequest() {
         }
 
-        NodeRequest(String nodeId, Request request) {
+        public NodeRequest(String nodeId, Request request) {
             super(nodeId);
             this.shardId = request.shardId();
         }
diff --git a/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java b/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
index e4a1709db55..92c9bd3b575 100644
--- a/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
+++ b/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
@@ -54,18 +54,20 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReferenceArray;
 
 /**
  *
  */
-public class TransportNodesListShardStoreMetaData extends TransportNodesAction<TransportNodesListShardStoreMetaData.Request, TransportNodesListShardStoreMetaData.NodesStoreFilesMetaData, TransportNodesListShardStoreMetaData.NodeRequest, TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData>
-        implements AsyncShardFetch.List<TransportNodesListShardStoreMetaData.NodesStoreFilesMetaData, TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> {
+public class TransportNodesListShardStoreMetaData extends TransportNodesAction<TransportNodesListShardStoreMetaData.Request,
+                                                                               TransportNodesListShardStoreMetaData.NodesStoreFilesMetaData,
+                                                                               TransportNodesListShardStoreMetaData.NodeRequest,
+                                                                               TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData>
+    implements AsyncShardFetch.Lister<TransportNodesListShardStoreMetaData.NodesStoreFilesMetaData,
+                                      TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> {
 
     public static final String ACTION_NAME = "internal:cluster/nodes/indices/shard/store";
 
@@ -74,11 +76,12 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
     private final NodeEnvironment nodeEnv;
 
     @Inject
-    public TransportNodesListShardStoreMetaData(Settings settings, ClusterName clusterName, ThreadPool threadPool, ClusterService clusterService, TransportService transportService,
+    public TransportNodesListShardStoreMetaData(Settings settings, ClusterName clusterName, ThreadPool threadPool,
+                                                ClusterService clusterService, TransportService transportService,
                                                 IndicesService indicesService, NodeEnvironment nodeEnv, ActionFilters actionFilters,
                                                 IndexNameExpressionResolver indexNameExpressionResolver) {
         super(settings, ACTION_NAME, clusterName, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
-                Request::new, NodeRequest::new, ThreadPool.Names.FETCH_SHARD_STORE);
+              Request::new, NodeRequest::new, ThreadPool.Names.FETCH_SHARD_STORE, NodeStoreFilesMetaData.class);
         this.indicesService = indicesService;
         this.nodeEnv = nodeEnv;
     }
@@ -106,21 +109,9 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
     }
 
     @Override
-    protected NodesStoreFilesMetaData newResponse(Request request, AtomicReferenceArray responses) {
-        final List<NodeStoreFilesMetaData> nodeStoreFilesMetaDatas = new ArrayList<>();
-        final List<FailedNodeException> failures = new ArrayList<>();
-        for (int i = 0; i < responses.length(); i++) {
-            Object resp = responses.get(i);
-            if (resp instanceof NodeStoreFilesMetaData) { // will also filter out null response for unallocated ones
-                nodeStoreFilesMetaDatas.add((NodeStoreFilesMetaData) resp);
-            } else if (resp instanceof FailedNodeException) {
-                failures.add((FailedNodeException) resp);
-            } else {
-                logger.warn("unknown response type [{}], expected NodeStoreFilesMetaData or FailedNodeException", resp);
-            }
-        }
-        return new NodesStoreFilesMetaData(clusterName, nodeStoreFilesMetaDatas.toArray(new NodeStoreFilesMetaData[nodeStoreFilesMetaDatas.size()]),
-                failures.toArray(new FailedNodeException[failures.size()]));
+    protected NodesStoreFilesMetaData newResponse(Request request,
+                                                  List<NodeStoreFilesMetaData> responses, List<FailedNodeException> failures) {
+        return new NodesStoreFilesMetaData(clusterName, responses, failures);
     }
 
     @Override
@@ -293,37 +284,21 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
 
     public static class NodesStoreFilesMetaData extends BaseNodesResponse<NodeStoreFilesMetaData> {
 
-        private FailedNodeException[] failures;
-
         NodesStoreFilesMetaData() {
         }
 
-        public NodesStoreFilesMetaData(ClusterName clusterName, NodeStoreFilesMetaData[] nodes, FailedNodeException[] failures) {
-            super(clusterName, nodes);
-            this.failures = failures;
+        public NodesStoreFilesMetaData(ClusterName clusterName, List<NodeStoreFilesMetaData> nodes, List<FailedNodeException> failures) {
+            super(clusterName, nodes, failures);
         }
 
         @Override
-        public FailedNodeException[] failures() {
-            return failures;
+        protected List<NodeStoreFilesMetaData> readNodesFrom(StreamInput in) throws IOException {
+            return in.readList(NodeStoreFilesMetaData::readListShardStoreNodeOperationResponse);
         }
 
         @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            nodes = new NodeStoreFilesMetaData[in.readVInt()];
-            for (int i = 0; i < nodes.length; i++) {
-                nodes[i] = NodeStoreFilesMetaData.readListShardStoreNodeOperationResponse(in);
-            }
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeVInt(nodes.length);
-            for (NodeStoreFilesMetaData response : nodes) {
-                response.writeTo(out);
-            }
+        protected void writeNodesTo(StreamOutput out, List<NodeStoreFilesMetaData> nodes) throws IOException {
+            out.writeStreamableList(nodes);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/hotthreads/RestNodesHotThreadsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/hotthreads/RestNodesHotThreadsAction.java
index 53bec14f967..19bee19509a 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/hotthreads/RestNodesHotThreadsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/hotthreads/RestNodesHotThreadsAction.java
@@ -69,7 +69,7 @@ public class RestNodesHotThreadsAction extends BaseRestHandler {
             @Override
             public RestResponse buildResponse(NodesHotThreadsResponse response) throws Exception {
                 StringBuilder sb = new StringBuilder();
-                for (NodeHotThreads node : response) {
+                for (NodeHotThreads node : response.getNodes()) {
                     sb.append("::: ").append(node.getNode().toString()).append("\n");
                     Strings.spaceify(3, node.getHotThreads(), sb);
                     sb.append('\n');
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java
index bd6637cb788..65b7715385f 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java
@@ -20,22 +20,17 @@
 package org.elasticsearch.rest.action.admin.cluster.node.info;
 
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
-import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestResponse;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.rest.action.support.RestBuilderListener;
+import org.elasticsearch.rest.action.support.RestActions.NodesResponseRestListener;
 
 import java.util.Set;
 
@@ -106,15 +101,6 @@ public class RestNodesInfoAction extends BaseRestHandler {
 
         settingsFilter.addFilterSettingParams(request);
 
-        client.admin().cluster().nodesInfo(nodesInfoRequest, new RestBuilderListener<NodesInfoResponse>(channel) {
-
-            @Override
-            public RestResponse buildResponse(NodesInfoResponse response, XContentBuilder builder) throws Exception {
-                builder.startObject();
-                response.toXContent(builder, request);
-                builder.endObject();
-                return new BytesRestResponse(RestStatus.OK, builder);
-            }
-        });
+        client.admin().cluster().nodesInfo(nodesInfoRequest, new NodesResponseRestListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java
index 1e2aece1646..47cce5283a9 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.rest.action.admin.cluster.node.stats;
 
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest;
-import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
 import org.elasticsearch.client.Client;
@@ -31,7 +30,7 @@ import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.support.RestToXContentListener;
+import org.elasticsearch.rest.action.support.RestActions.NodesResponseRestListener;
 
 import java.util.Set;
 
@@ -114,6 +113,6 @@ public class RestNodesStatsAction extends BaseRestHandler {
             nodesStatsRequest.indices().includeSegmentFileSizes(true);
         }
 
-        client.admin().cluster().nodesStats(nodesStatsRequest, new RestToXContentListener<>(channel));
+        client.admin().cluster().nodesStats(nodesStatsRequest, new NodesResponseRestListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/stats/RestClusterStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/stats/RestClusterStatsAction.java
index a02dcfccb98..29cc6377494 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/stats/RestClusterStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/stats/RestClusterStatsAction.java
@@ -27,8 +27,7 @@ import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.support.RestToXContentListener;
-
+import org.elasticsearch.rest.action.support.RestActions.NodesResponseRestListener;
 
 /**
  *
@@ -46,6 +45,6 @@ public class RestClusterStatsAction extends BaseRestHandler {
     public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
         ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest().nodesIds(request.paramAsStringArray("nodeId", null));
         clusterStatsRequest.timeout(request.param("timeout"));
-        client.admin().cluster().clusterStats(clusterStatsRequest, new RestToXContentListener<>(channel));
+        client.admin().cluster().clusterStats(clusterStatsRequest, new NodesResponseRestListener<>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/support/RestActions.java b/core/src/main/java/org/elasticsearch/rest/action/support/RestActions.java
index 652cb8c61e9..51d5089ec6f 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/support/RestActions.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/support/RestActions.java
@@ -21,13 +21,17 @@ package org.elasticsearch.rest.action.support;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
+import org.elasticsearch.action.support.nodes.BaseNodeResponse;
+import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.ToXContent.Params;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -38,9 +42,14 @@ import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryStringQueryBuilder;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
+import java.util.List;
 
 /**
  *
@@ -63,25 +72,21 @@ public class RestActions {
         return (version == Versions.MATCH_ANY) ? defaultVersion : version;
     }
 
-    static final class Fields {
-        static final String _SHARDS = "_shards";
-        static final String TOTAL = "total";
-        static final String SUCCESSFUL = "successful";
-        static final String FAILED = "failed";
-        static final String FAILURES = "failures";
+    public static void buildBroadcastShardsHeader(XContentBuilder builder, Params params, BroadcastResponse response) throws IOException {
+        buildBroadcastShardsHeader(builder, params,
+                                   response.getTotalShards(), response.getSuccessfulShards(), response.getFailedShards(),
+                                   response.getShardFailures());
     }
 
-    public static void buildBroadcastShardsHeader(XContentBuilder builder, ToXContent.Params params, BroadcastResponse response) throws IOException {
-        buildBroadcastShardsHeader(builder, params, response.getTotalShards(), response.getSuccessfulShards(), response.getFailedShards(), response.getShardFailures());
-    }
-
-    public static void buildBroadcastShardsHeader(XContentBuilder builder, ToXContent.Params params, int total, int successful, int failed, ShardOperationFailedException[] shardFailures) throws IOException {
-        builder.startObject(Fields._SHARDS);
-        builder.field(Fields.TOTAL, total);
-        builder.field(Fields.SUCCESSFUL, successful);
-        builder.field(Fields.FAILED, failed);
+    public static void buildBroadcastShardsHeader(XContentBuilder builder, Params params,
+                                                  int total, int successful, int failed,
+                                                  ShardOperationFailedException[] shardFailures) throws IOException {
+        builder.startObject("_shards");
+        builder.field("total", total);
+        builder.field("successful", successful);
+        builder.field("failed", failed);
         if (shardFailures != null && shardFailures.length > 0) {
-            builder.startArray(Fields.FAILURES);
+            builder.startArray("failures");
             final boolean group = params.paramAsBoolean("group_shard_failures", true); // we group by default
             for (ShardOperationFailedException shardFailure : group ? ExceptionsHelper.groupBy(shardFailures) : shardFailures) {
                 builder.startObject();
@@ -92,6 +97,94 @@ public class RestActions {
         }
         builder.endObject();
     }
+    /**
+     * Create the XContent header for any {@link BaseNodesResponse}.
+     *
+     * @param builder XContent builder.
+     * @param params XContent parameters.
+     * @param response The response containing individual, node-level responses.
+     * @see #buildNodesHeader(XContentBuilder, Params, int, int, int, List)
+     */
+    public static <NodeResponse extends BaseNodeResponse> void buildNodesHeader(final XContentBuilder builder, final Params params,
+                                                                                final BaseNodesResponse<NodeResponse> response)
+            throws IOException {
+        final int successful = response.getNodes().size();
+        final int failed = response.failures().size();
+
+        buildNodesHeader(builder, params, successful + failed, successful, failed, response.failures());
+    }
+
+    /**
+     * Create the XContent header for any {@link BaseNodesResponse}. This looks like:
+     * <code>
+     * "_nodes" : {
+     *   "total" : 3,
+     *   "successful" : 1,
+     *   "failed" : 2,
+     *   "failures" : [ { ... }, { ... } ]
+     * }
+     * </code>
+     * Prefer the overload that properly invokes this method to calling this directly.
+     *
+     * @param builder XContent builder.
+     * @param params XContent parameters.
+     * @param total The total number of nodes touched.
+     * @param successful The successful number of responses received.
+     * @param failed The number of failures (effectively {@code total - successful}).
+     * @param failures The failure exceptions related to {@code failed}.
+     * @see #buildNodesHeader(XContentBuilder, Params, BaseNodesResponse)
+     */
+    public static void buildNodesHeader(final XContentBuilder builder, final Params params,
+                                        final int total, final int successful, final int failed,
+                                        final List<FailedNodeException> failures) throws IOException {
+        builder.startObject("_nodes");
+        builder.field("total", total);
+        builder.field("successful", successful);
+        builder.field("failed", failed);
+
+        if (failures.isEmpty() == false) {
+            builder.startArray("failures");
+            for (FailedNodeException failure : failures) {
+                builder.startObject();
+                failure.toXContent(builder, params);
+                builder.endObject();
+            }
+            builder.endArray();
+        }
+
+        builder.endObject();
+    }
+
+    /**
+     * Automatically transform the {@link ToXContent}-compatible, nodes-level {@code response} into a a {@link BytesRestResponse}.
+     * <p>
+     * This looks like:
+     * <code>
+     * {
+     *   "_nodes" : { ... },
+     *   "cluster_name" : "...",
+     *   ...
+     * }
+     * </code>
+     *
+     * @param builder XContent builder.
+     * @param params XContent parameters.
+     * @param response The nodes-level (plural) response.
+     * @return Never {@code null}.
+     * @throws IOException if building the response causes an issue
+     */
+    public static <NodesResponse extends BaseNodesResponse & ToXContent> BytesRestResponse nodesResponse(final XContentBuilder builder,
+                                                                                                         final Params params,
+                                                                                                         final NodesResponse response)
+            throws IOException {
+        builder.startObject();
+        RestActions.buildNodesHeader(builder, params, response);
+        builder.field("cluster_name", response.getClusterName().value());
+        response.toXContent(builder, params);
+        builder.endObject();
+
+        return new BytesRestResponse(RestStatus.OK, builder);
+    }
 
     public static QueryBuilder urlParamsToQueryBuilder(RestRequest request) {
         String queryString = request.param("q");
@@ -130,7 +223,8 @@ public class RestActions {
         return content;
     }
 
-    public static QueryBuilder getQueryContent(BytesReference source, IndicesQueriesRegistry indicesQueriesRegistry, ParseFieldMatcher parseFieldMatcher) {
+    public static QueryBuilder getQueryContent(BytesReference source, IndicesQueriesRegistry indicesQueriesRegistry,
+                                               ParseFieldMatcher parseFieldMatcher) {
         try (XContentParser requestParser = XContentFactory.xContent(source).createParser(source)) {
             QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, requestParser, parseFieldMatcher);
             return context.parseTopLevelQueryBuilder();
@@ -158,4 +252,32 @@ public class RestActions {
     public static boolean hasBodyContent(final RestRequest request) {
         return request.hasContent() || request.hasParam("source");
     }
+
+    /**
+     * {@code NodesResponseRestBuilderListener} automatically translates any {@link BaseNodesResponse} (multi-node) response that is
+     * {@link ToXContent}-compatible into a {@link RestResponse} with the necessary header info (e.g., "cluster_name").
+     * <p>
+     * This is meant to avoid a slew of anonymous classes doing (or worse):
+     * <code>
+     * client.admin().cluster().request(nodesRequest, new RestBuilderListener&lt;NodesResponse&gt;(channel) {
+     *     public RestResponse buildResponse(NodesResponse response, XContentBuilder builder) throws Exception {
+     *         return RestActions.nodesResponse(builder, ToXContent.EMPTY_PARAMS, response);
+     *     }
+     * });
+     * </code>
+     */
+    public static class NodesResponseRestListener<NodesResponse extends BaseNodesResponse & ToXContent>
+        extends RestBuilderListener<NodesResponse> {
+
+        public NodesResponseRestListener(RestChannel channel) {
+            super(channel);
+        }
+
+        @Override
+        public RestResponse buildResponse(NodesResponse response, XContentBuilder builder) throws Exception {
+            return RestActions.nodesResponse(builder, ToXContent.EMPTY_PARAMS, response);
+        }
+
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/HotThreadsIT.java b/core/src/test/java/org/elasticsearch/action/admin/HotThreadsIT.java
index 47823307ffd..480c103df81 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/HotThreadsIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/HotThreadsIT.java
@@ -86,7 +86,7 @@ public class HotThreadsIT extends ESIntegTestCase {
                         assertThat(nodeHotThreads, notNullValue());
                         Map<String, NodeHotThreads> nodesMap = nodeHotThreads.getNodesMap();
                         assertThat(nodesMap.size(), equalTo(cluster().size()));
-                        for (NodeHotThreads ht : nodeHotThreads) {
+                        for (NodeHotThreads ht : nodeHotThreads.getNodes()) {
                             assertNotNull(ht.getHotThreads());
                             //logger.info(ht.getHotThreads());
                         }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
index 8168d1a8819..628e26bc4cf 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
@@ -28,7 +28,6 @@ import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.ESSingleNodeTestCase;
 
 import java.util.HashMap;
 import java.util.List;
@@ -37,7 +36,6 @@ import java.util.Map;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 /**
  * Tests for the cluster allocation explanation
@@ -54,7 +52,7 @@ public final class ClusterAllocationExplainIT extends ESIntegTestCase {
             @Override
             public void run() {
                 NodesStatsResponse resp = client().admin().cluster().prepareNodesStats().get();
-                assertThat(resp.getNodes().length, equalTo(3));
+                assertThat(resp.getNodes().size(), equalTo(3));
             }
         });
 
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
index d96531b2f6c..9af2bb07417 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
@@ -255,12 +255,12 @@ public class CancellableTasksTests extends TaskManagerTestCase {
             // Make sure that the request was successful
             assertNull(throwableReference.get());
             assertNotNull(responseReference.get());
-            assertEquals(nodesCount, responseReference.get().getNodes().length);
+            assertEquals(nodesCount, responseReference.get().getNodes().size());
             assertEquals(0, responseReference.get().failureCount());
         } else {
             // We canceled the request, in this case it should have fail, but we should get partial response
             assertNull(throwableReference.get());
-            assertEquals(nodesCount, responseReference.get().failureCount() + responseReference.get().getNodes().length);
+            assertEquals(nodesCount, responseReference.get().failureCount() + responseReference.get().getNodes().size());
             // and we should have at least as many failures as the number of blocked operations
             // (we might have cancelled some non-blocked operations before they even started and that's ok)
             assertThat(responseReference.get().failureCount(), greaterThanOrEqualTo(blockedNodesCount));
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
index 9c0e2bfcafd..ffc19aa0dcd 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
@@ -50,11 +50,9 @@ import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReferenceArray;
 import java.util.function.Supplier;
 
 import static java.util.Collections.emptyMap;
@@ -88,7 +86,6 @@ public abstract class TaskManagerTestCase extends ESTestCase {
         testNodes = new TestNode[nodesCount];
         for (int i = 0; i < testNodes.length; i++) {
             testNodes[i] = new TestNode("node" + i, threadPool, settings);
-            ;
         }
     }
 
@@ -113,27 +110,22 @@ public abstract class TaskManagerTestCase extends ESTestCase {
 
     static class NodesResponse extends BaseNodesResponse<NodeResponse> {
 
-        private int failureCount;
-
-        protected NodesResponse(ClusterName clusterName, NodeResponse[] nodes, int failureCount) {
-            super(clusterName, nodes);
-            this.failureCount = failureCount;
+        protected NodesResponse(ClusterName clusterName, List<NodeResponse> nodes, List<FailedNodeException> failures) {
+            super(clusterName, nodes, failures);
         }
 
         @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            failureCount = in.readVInt();
+        protected List<NodeResponse> readNodesFrom(StreamInput in) throws IOException {
+            return in.readStreamableList(NodeResponse::new);
         }
 
         @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeVInt(failureCount);
+        protected void writeNodesTo(StreamOutput out, List<NodeResponse> nodes) throws IOException {
+            out.writeStreamableList(nodes);
         }
 
         public int failureCount() {
-            return failureCount;
+            return failures().size();
         }
     }
 
@@ -148,24 +140,12 @@ public abstract class TaskManagerTestCase extends ESTestCase {
                                 Supplier<NodeRequest> nodeRequest) {
             super(settings, actionName, clusterName, threadPool, clusterService, transportService,
                     new ActionFilters(new HashSet<>()), new IndexNameExpressionResolver(Settings.EMPTY),
-                    request, nodeRequest, ThreadPool.Names.GENERIC);
+                    request, nodeRequest, ThreadPool.Names.GENERIC, NodeResponse.class);
         }
 
         @Override
-        protected NodesResponse newResponse(NodesRequest request, AtomicReferenceArray responses) {
-            final List<NodeResponse> nodesList = new ArrayList<>();
-            int failureCount = 0;
-            for (int i = 0; i < responses.length(); i++) {
-                Object resp = responses.get(i);
-                if (resp instanceof NodeResponse) { // will also filter out null response for unallocated ones
-                    nodesList.add((NodeResponse) resp);
-                } else if (resp instanceof FailedNodeException) {
-                    failureCount++;
-                } else {
-                    logger.warn("unknown response type [{}], expected NodeLocalGatewayMetaState or FailedNodeException", resp);
-                }
-            }
-            return new NodesResponse(clusterName, nodesList.toArray(new NodeResponse[nodesList.size()]), failureCount);
+        protected NodesResponse newResponse(NodesRequest request, List<NodeResponse> responses, List<FailedNodeException> failures) {
+            return new NodesResponse(clusterName, responses, failures);
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
index 7f1d6c8b835..ec05ff9c984 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
@@ -56,7 +56,6 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
-import java.util.concurrent.atomic.AtomicReferenceArray;
 
 import static org.elasticsearch.test.ESTestCase.awaitBusy;
 
@@ -111,31 +110,26 @@ public class TestTaskPlugin extends Plugin {
 
     public static class NodesResponse extends BaseNodesResponse<NodeResponse> {
 
-        private int failureCount;
-
         NodesResponse() {
 
         }
 
-        public NodesResponse(ClusterName clusterName, NodeResponse[] nodes, int failureCount) {
-            super(clusterName, nodes);
-            this.failureCount = failureCount;
+        public NodesResponse(ClusterName clusterName, List<NodeResponse> nodes, List<FailedNodeException> failures) {
+            super(clusterName, nodes, failures);
         }
 
         @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            failureCount = in.readVInt();
+        protected List<NodeResponse> readNodesFrom(StreamInput in) throws IOException {
+            return in.readStreamableList(NodeResponse::new);
         }
 
         @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeVInt(failureCount);
+        protected void writeNodesTo(StreamOutput out, List<NodeResponse> nodes) throws IOException {
+            out.writeStreamableList(nodes);
         }
 
         public int failureCount() {
-            return failureCount;
+            return failures().size();
         }
     }
 
@@ -219,25 +213,13 @@ public class TestTaskPlugin extends Plugin {
         public TransportTestTaskAction(Settings settings, ClusterName clusterName, ThreadPool threadPool,
                                        ClusterService clusterService, TransportService transportService) {
             super(settings, TestTaskAction.NAME, clusterName, threadPool, clusterService, transportService,
-                new ActionFilters(new HashSet<>()), new IndexNameExpressionResolver(Settings.EMPTY),
-                NodesRequest::new, NodeRequest::new, ThreadPool.Names.GENERIC);
+                  new ActionFilters(new HashSet<>()), new IndexNameExpressionResolver(Settings.EMPTY),
+                  NodesRequest::new, NodeRequest::new, ThreadPool.Names.GENERIC, NodeResponse.class);
         }
 
         @Override
-        protected NodesResponse newResponse(NodesRequest request, AtomicReferenceArray responses) {
-            final List<NodeResponse> nodesList = new ArrayList<>();
-            int failureCount = 0;
-            for (int i = 0; i < responses.length(); i++) {
-                Object resp = responses.get(i);
-                if (resp instanceof NodeResponse) { // will also filter out null response for unallocated ones
-                    nodesList.add((NodeResponse) resp);
-                } else if (resp instanceof FailedNodeException) {
-                    failureCount++;
-                } else {
-                    logger.warn("unknown response type [{}], expected NodeLocalGatewayMetaState or FailedNodeException", resp);
-                }
-            }
-            return new NodesResponse(clusterName, nodesList.toArray(new NodeResponse[nodesList.size()]), failureCount);
+        protected NodesResponse newResponse(NodesRequest request, List<NodeResponse> responses, List<FailedNodeException> failures) {
+            return new NodesResponse(clusterName, responses, failures);
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
index 6a7f7ac3398..a3335d87fd6 100644
--- a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.action.support.nodes;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeActionTests;
@@ -28,6 +29,8 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.test.ESTestCase;
@@ -39,6 +42,7 @@ import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -53,6 +57,7 @@ import java.util.function.Supplier;
 
 import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
 import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
+import static org.mockito.Mockito.mock;
 
 public class TransportNodesActionTests extends ESTestCase {
 
@@ -92,6 +97,42 @@ public class TransportNodesActionTests extends ESTestCase {
         assertEquals(clusterService.state().nodes().resolveNodesIds(finalNodesIds).length, capturedRequests.size());
     }
 
+    public void testNewResponseNullArray() {
+        expectThrows(NullPointerException.class, () -> action.newResponse(new TestNodesRequest(), null));
+    }
+
+    public void testNewResponse() {
+        TestNodesRequest request = new TestNodesRequest();
+        List<TestNodeResponse> expectedNodeResponses = mockList(TestNodeResponse.class, randomIntBetween(0, 2));
+        expectedNodeResponses.add(new TestNodeResponse());
+        List<BaseNodeResponse> nodeResponses = new ArrayList<>(expectedNodeResponses);
+        // This should be ignored:
+        nodeResponses.add(new OtherNodeResponse());
+        List<FailedNodeException> failures = mockList(FailedNodeException.class, randomIntBetween(0, 2));
+
+        List<Object> allResponses = new ArrayList<>(expectedNodeResponses);
+        allResponses.addAll(failures);
+
+        Collections.shuffle(allResponses, random());
+
+        AtomicReferenceArray<?> atomicArray = new AtomicReferenceArray<>(allResponses.toArray());
+
+        TestNodesResponse response = action.newResponse(request, atomicArray);
+
+        assertSame(request, response.request);
+        // note: I shuffled the overall list, so it's not possible to guarantee that it's in the right order
+        assertTrue(expectedNodeResponses.containsAll(response.getNodes()));
+        assertTrue(failures.containsAll(response.failures()));
+    }
+
+    private <T> List<T> mockList(Class<T> clazz, int size) {
+        List<T> failures = new ArrayList<>(size);
+        for (int i = 0; i < size; ++i) {
+            failures.add(mock(clazz));
+        }
+        return failures;
+    }
+
     private enum NodeSelector {
         LOCAL("_local"), ELECTED_MASTER("_master"), MASTER_ELIGIBLE("master:true"), DATA("data:true"), CUSTOM_ATTRIBUTE("attr:value");
 
@@ -165,26 +206,20 @@ public class TransportNodesActionTests extends ESTestCase {
         return new DiscoveryNode(node, node, DummyTransportAddress.INSTANCE, attributes, roles, Version.CURRENT);
     }
 
-    private static class TestTransportNodesAction extends TransportNodesAction<TestNodesRequest, TestNodesResponse, TestNodeRequest,
-                TestNodeResponse> {
+    private static class TestTransportNodesAction
+        extends TransportNodesAction<TestNodesRequest, TestNodesResponse, TestNodeRequest, TestNodeResponse> {
 
         TestTransportNodesAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService
                 transportService, ActionFilters actionFilters, Supplier<TestNodesRequest> request,
                                  Supplier<TestNodeRequest> nodeRequest, String nodeExecutor) {
             super(settings, "indices:admin/test", CLUSTER_NAME, threadPool, clusterService, transportService, actionFilters,
-                    null, request, nodeRequest, nodeExecutor);
+                    null, request, nodeRequest, nodeExecutor, TestNodeResponse.class);
         }
 
         @Override
-        protected TestNodesResponse newResponse(TestNodesRequest request, AtomicReferenceArray nodesResponses) {
-            final List<TestNodeResponse> nodeResponses = new ArrayList<>();
-            for (int i = 0; i < nodesResponses.length(); i++) {
-                Object resp = nodesResponses.get(i);
-                if (resp instanceof TestNodeResponse) {
-                    nodeResponses.add((TestNodeResponse) resp);
-                }
-            }
-            return new TestNodesResponse(nodeResponses);
+        protected TestNodesResponse newResponse(TestNodesRequest request,
+                                                List<TestNodeResponse> responses, List<FailedNodeException> failures) {
+            return new TestNodesResponse(request, responses, failures);
         }
 
         @Override
@@ -216,16 +251,28 @@ public class TransportNodesActionTests extends ESTestCase {
 
     private static class TestNodesResponse extends BaseNodesResponse<TestNodeResponse> {
 
-        private final List<TestNodeResponse> nodeResponses;
+        private final TestNodesRequest request;
 
-        TestNodesResponse(List<TestNodeResponse> nodeResponses) {
-            this.nodeResponses = nodeResponses;
+        TestNodesResponse(TestNodesRequest request, List<TestNodeResponse> nodeResponses, List<FailedNodeException> failures) {
+            super(CLUSTER_NAME, nodeResponses, failures);
+            this.request = request;
+        }
+
+        @Override
+        protected List<TestNodeResponse> readNodesFrom(StreamInput in) throws IOException {
+            return in.readStreamableList(TestNodeResponse::new);
+        }
+
+        @Override
+        protected void writeNodesTo(StreamOutput out, List<TestNodeResponse> nodes) throws IOException {
+            out.writeStreamableList(nodes);
         }
     }
 
-    private static class TestNodeRequest extends BaseNodeRequest {
-    }
+    private static class TestNodeRequest extends BaseNodeRequest { }
+
+    private static class TestNodeResponse extends BaseNodeResponse { }
+
+    private static class OtherNodeResponse extends BaseNodeResponse { }
 
-    private static class TestNodeResponse extends BaseNodeResponse {
-    }
 }
diff --git a/core/src/test/java/org/elasticsearch/bwcompat/ClusterStateBackwardsCompatIT.java b/core/src/test/java/org/elasticsearch/bwcompat/ClusterStateBackwardsCompatIT.java
index 81d894e3349..a4427befea2 100644
--- a/core/src/test/java/org/elasticsearch/bwcompat/ClusterStateBackwardsCompatIT.java
+++ b/core/src/test/java/org/elasticsearch/bwcompat/ClusterStateBackwardsCompatIT.java
@@ -44,7 +44,7 @@ public class ClusterStateBackwardsCompatIT extends ESBackcompatTestCase {
         createIndex("test");
 
         // connect to each node with a custom TransportClient, issue a ClusterStateRequest to test serialization
-        for (NodeInfo n : clusterNodes()) {
+        for (NodeInfo n : clusterNodes().getNodes()) {
             try (TransportClient tc = newTransportClient()) {
                 tc.addTransportAddress(n.getNode().getAddress());
                 ClusterStateResponse response = tc.admin().cluster().prepareState().execute().actionGet();
@@ -68,7 +68,7 @@ public class ClusterStateBackwardsCompatIT extends ESBackcompatTestCase {
             try {
                 enableIndexBlock("test-blocks", block.getKey());
 
-                for (NodeInfo n : clusterNodes()) {
+                for (NodeInfo n : clusterNodes().getNodes()) {
                     try (TransportClient tc = newTransportClient()) {
                         tc.addTransportAddress(n.getNode().getAddress());
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
index 1826ced24ea..31c9c21b8e7 100644
--- a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
@@ -41,6 +41,8 @@ import org.elasticsearch.monitor.fs.FsInfo;
 import org.elasticsearch.test.ESTestCase;
 
 import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
@@ -196,14 +198,14 @@ public class DiskUsageTests extends ESTestCase {
                 new FsInfo.Path("/least", "/dev/sda", 100, 90, 70),
                 new FsInfo.Path("/most", "/dev/sda", 100, 90, 80),
         };
-        NodeStats[] nodeStats = new NodeStats[] {
+        List<NodeStats> nodeStats = Arrays.asList(
                 new NodeStats(new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0,
                         null,null,null,null,null,new FsInfo(0, node1FSInfo), null,null,null,null,null, null),
                 new NodeStats(new DiscoveryNode("node_2", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0,
                         null,null,null,null,null, new FsInfo(0, node2FSInfo), null,null,null,null,null, null),
                 new NodeStats(new DiscoveryNode("node_3", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0,
                         null,null,null,null,null, new FsInfo(0, node3FSInfo), null,null,null,null,null, null)
-        };
+        );
         InternalClusterInfoService.fillDiskUsagePerNode(logger, nodeStats, newLeastAvaiableUsages, newMostAvaiableUsages);
         DiskUsage leastNode_1 = newLeastAvaiableUsages.get("node_1");
         DiskUsage mostNode_1 = newMostAvaiableUsages.get("node_1");
@@ -237,14 +239,14 @@ public class DiskUsageTests extends ESTestCase {
                 new FsInfo.Path("/most", "/dev/sda", 100, 90, 70),
                 new FsInfo.Path("/least", "/dev/sda", 10, -8, 0),
         };
-        NodeStats[] nodeStats = new NodeStats[] {
+        List<NodeStats> nodeStats = Arrays.asList(
                 new NodeStats(new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0,
                         null,null,null,null,null,new FsInfo(0, node1FSInfo), null,null,null,null,null, null),
                 new NodeStats(new DiscoveryNode("node_2", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0,
                         null,null,null,null,null, new FsInfo(0, node2FSInfo), null,null,null,null,null, null),
                 new NodeStats(new DiscoveryNode("node_3", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0,
                         null,null,null,null,null, new FsInfo(0, node3FSInfo), null,null,null,null,null, null)
-        };
+        );
         InternalClusterInfoService.fillDiskUsagePerNode(logger, nodeStats, newLeastAvailableUsages, newMostAvailableUsages);
         DiskUsage leastNode_1 = newLeastAvailableUsages.get("node_1");
         DiskUsage mostNode_1 = newMostAvailableUsages.get("node_1");
diff --git a/core/src/test/java/org/elasticsearch/cluster/ack/AckClusterUpdateSettingsIT.java b/core/src/test/java/org/elasticsearch/cluster/ack/AckClusterUpdateSettingsIT.java
index 8e2dea63d6a..8d13f04240e 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ack/AckClusterUpdateSettingsIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ack/AckClusterUpdateSettingsIT.java
@@ -81,7 +81,7 @@ public class AckClusterUpdateSettingsIT extends ESIntegTestCase {
 
         NodesInfoResponse nodesInfo = client().admin().cluster().prepareNodesInfo().get();
         String excludedNodeId = null;
-        for (NodeInfo nodeInfo : nodesInfo) {
+        for (NodeInfo nodeInfo : nodesInfo.getNodes()) {
             if (nodeInfo.getNode().isDataNode()) {
                 excludedNodeId = nodeInfo.getNode().getId();
                 break;
@@ -124,7 +124,7 @@ public class AckClusterUpdateSettingsIT extends ESIntegTestCase {
 
         NodesInfoResponse nodesInfo = client().admin().cluster().prepareNodesInfo().get();
         String excludedNodeId = null;
-        for (NodeInfo nodeInfo : nodesInfo) {
+        for (NodeInfo nodeInfo : nodesInfo.getNodes()) {
             if (nodeInfo.getNode().isDataNode()) {
                 excludedNodeId = nodeInfo.getNode().getId();
                 break;
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java
index 81a28adee2d..ef54c6fd637 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/MockDiskUsagesIT.java
@@ -59,7 +59,7 @@ public class MockDiskUsagesIT extends ESIntegTestCase {
             @Override
             public void run() {
                 NodesStatsResponse resp = client().admin().cluster().prepareNodesStats().get();
-                assertThat(resp.getNodes().length, equalTo(3));
+                assertThat(resp.getNodes().size(), equalTo(3));
             }
         });
 
diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
index 381120fdc0f..9fcbb708156 100644
--- a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
@@ -22,20 +22,20 @@ package org.elasticsearch.common.io.stream;
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.geo.GeoPoint;
-import org.elasticsearch.common.joda.FormatDateTimeFormatter;
-import org.elasticsearch.common.joda.Joda;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.test.ESTestCase;
 import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Objects;
 
 import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.startsWith;
 
@@ -431,6 +431,33 @@ public class BytesStreamsTests extends ESTestCase {
                 endsWith(" claims to have a different name [intentionally-broken] than it was read from [test-named-writeable]."));
     }
 
+    public void testWriteStreamableList() throws IOException {
+        final int size = randomIntBetween(0, 5);
+        final List<TestStreamable> expected = new ArrayList<>(size);
+
+        for (int i = 0; i < size; ++i) {
+            expected.add(new TestStreamable(randomBoolean()));
+        }
+
+        final BytesStreamOutput out = new BytesStreamOutput();
+        out.writeStreamableList(expected);
+
+        final StreamInput in = StreamInput.wrap(out.bytes().toBytes());
+
+        List<TestStreamable> loaded = in.readStreamableList(TestStreamable::new);
+
+        assertThat(loaded, hasSize(expected.size()));
+
+        for (int i = 0; i < expected.size(); ++i) {
+            assertEquals(expected.get(i).value, loaded.get(i).value);
+        }
+
+        assertEquals(0, in.available());
+
+        in.close();
+        out.close();
+    }
+
     private static abstract class BaseNamedWriteable implements NamedWriteable {
 
     }
@@ -544,4 +571,25 @@ public class BytesStreamsTests extends ESTestCase {
             assertEquals(point, geoPoint);
         }
     }
+
+    private static class TestStreamable implements Streamable {
+
+        private boolean value;
+
+        public TestStreamable() { }
+
+        public TestStreamable(boolean value) {
+            this.value = value;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            value = in.readBoolean();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeBoolean(value);
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java
index 2de9d669511..d12a0ff55a3 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java
@@ -349,9 +349,9 @@ public class ZenDiscoveryIT extends ESIntegTestCase {
 
         logger.info("--> request node discovery stats");
         NodesStatsResponse statsResponse = client().admin().cluster().prepareNodesStats().clear().setDiscovery(true).get();
-        assertThat(statsResponse.getNodes().length, equalTo(1));
+        assertThat(statsResponse.getNodes().size(), equalTo(1));
 
-        DiscoveryStats stats = statsResponse.getNodes()[0].getDiscoveryStats();
+        DiscoveryStats stats = statsResponse.getNodes().get(0).getDiscoveryStats();
         assertThat(stats.getQueueStats(), notNullValue());
         assertThat(stats.getQueueStats().getTotal(), equalTo(0));
         assertThat(stats.getQueueStats().getCommitted(), equalTo(0));
diff --git a/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java b/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
index 3ab15baf2e0..b710aa50ee0 100644
--- a/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
@@ -284,10 +284,11 @@ public class AsyncShardFetchTests extends ESTestCase {
                             assert entry != null;
                             entry.executeLatch.await();
                             if (entry.failure != null) {
-                                processAsyncFetch(shardId, null, new FailedNodeException[]{new FailedNodeException(nodeId,
-                                        "unexpected", entry.failure)});
+                                processAsyncFetch(shardId, null, Collections.singletonList(new FailedNodeException(nodeId,
+                                                                                                                   "unexpected",
+                                                                                                                   entry.failure)));
                             } else {
-                                processAsyncFetch(shardId, new Response[]{entry.response}, null);
+                                processAsyncFetch(shardId, Collections.singletonList(entry.response), null);
                             }
                         } catch (Throwable e) {
                             logger.error("unexpected failure", e);
diff --git a/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java b/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java
index 702e83e7d55..aaa29ad1970 100644
--- a/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java
+++ b/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java
@@ -63,9 +63,9 @@ import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
@@ -568,12 +568,12 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
             .execute(new TransportNodesListGatewayStartedShards.Request(shardId, new String[]{node.getId()}))
             .get();
 
-        assertThat(response.getNodes(), arrayWithSize(1));
-        assertThat(response.getNodes()[0].allocationId(), notNullValue());
+        assertThat(response.getNodes(), hasSize(1));
+        assertThat(response.getNodes().get(0).allocationId(), notNullValue());
         if (corrupt) {
-            assertThat(response.getNodes()[0].storeException(), notNullValue());
+            assertThat(response.getNodes().get(0).storeException(), notNullValue());
         } else {
-            assertThat(response.getNodes()[0].storeException(), nullValue());
+            assertThat(response.getNodes().get(0).storeException(), nullValue());
         }
 
         // start another node so cluster consistency checks won't time out due to the lack of state
diff --git a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
index 367ecf76768..df8736b44ef 100644
--- a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
+++ b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
@@ -578,7 +578,7 @@ public class CorruptedFileIT extends ESIntegTestCase {
             }
             assertTrue(shardRouting.assignedToNode());
             NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(shardRouting.currentNodeId()).setFs(true).get();
-            NodeStats nodeStats = nodeStatses.getNodes()[0];
+            NodeStats nodeStats = nodeStatses.getNodes().get(0);
             List<Path> files = new ArrayList<>();
             filesToNodes.put(nodeStats.getNode().getName(), files);
             for (FsInfo.Path info : nodeStats.getFs()) {
@@ -615,7 +615,7 @@ public class CorruptedFileIT extends ESIntegTestCase {
         String nodeId = shardRouting.currentNodeId();
         NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(nodeId).setFs(true).get();
         Set<Path> files = new TreeSet<>(); // treeset makes sure iteration order is deterministic
-        for (FsInfo.Path info : nodeStatses.getNodes()[0].getFs()) {
+        for (FsInfo.Path info : nodeStatses.getNodes().get(0).getFs()) {
             String path = info.getPath();
             Path file = PathUtils.get(path).resolve("indices").resolve(test.getUUID()).resolve(Integer.toString(shardRouting.getId())).resolve("index");
             if (Files.exists(file)) { // multi data path might only have one path in use
@@ -678,9 +678,9 @@ public class CorruptedFileIT extends ESIntegTestCase {
         NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(routing.currentNodeId()).setFs(true).get();
         ClusterState state = client().admin().cluster().prepareState().get().getState();
         final Index test = state.metaData().index("test").getIndex();
-        assertThat(routing.toString(), nodeStatses.getNodes().length, equalTo(1));
+        assertThat(routing.toString(), nodeStatses.getNodes().size(), equalTo(1));
         List<Path> files = new ArrayList<>();
-        for (FsInfo.Path info : nodeStatses.getNodes()[0].getFs()) {
+        for (FsInfo.Path info : nodeStatses.getNodes().get(0).getFs()) {
             String path = info.getPath();
             Path file = PathUtils.get(path).resolve("indices/" + test.getUUID() + "/" + Integer.toString(routing.getId()) + "/index");
             if (Files.exists(file)) { // multi data path might only have one path in use
diff --git a/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java b/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
index 94ee490c729..018bce16566 100644
--- a/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
+++ b/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
@@ -121,7 +121,7 @@ public class CorruptedTranslogIT extends ESIntegTestCase {
         String nodeId = shardRouting.currentNodeId();
         NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(nodeId).setFs(true).get();
         Set<Path> files = new TreeSet<>(); // treeset makes sure iteration order is deterministic
-        for (FsInfo.Path fsPath : nodeStatses.getNodes()[0].getFs()) {
+        for (FsInfo.Path fsPath : nodeStatses.getNodes().get(0).getFs()) {
             String path = fsPath.getPath();
             final String relativeDataLocationPath =  "indices/"+ test.getUUID() +"/" + Integer.toString(shardRouting.getId()) + "/translog";
             Path file = PathUtils.get(path).resolve(relativeDataLocationPath);
diff --git a/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java b/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java
index 0d850632393..cb0397990ab 100644
--- a/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java
+++ b/core/src/test/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java
@@ -119,10 +119,9 @@ public class SuggestStatsIT extends ESIntegTestCase {
         assertThat(suggest.getSuggestTimeInMillis(), lessThanOrEqualTo(totalShards * (endTime - startTime)));
 
         NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().execute().actionGet();
-        NodeStats[] nodes = nodeStats.getNodes();
         Set<String> nodeIdsWithIndex = nodeIdsWithIndex("test1", "test2");
         int num = 0;
-        for (NodeStats stat : nodes) {
+        for (NodeStats stat : nodeStats.getNodes()) {
             SearchStats.Stats suggestStats = stat.getIndices().getSearch().getTotal();
             logger.info("evaluating {}", stat.getNode());
             if (nodeIdsWithIndex.contains(stat.getNode().getId())) {
diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
index 2f039f6c472..351742d9712 100644
--- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
@@ -101,7 +101,7 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase {
     /** Returns true if any of the nodes used a noop breaker */
     private boolean noopBreakerUsed() {
         NodesStatsResponse stats = client().admin().cluster().prepareNodesStats().setBreaker(true).get();
-        for (NodeStats nodeStats : stats) {
+        for (NodeStats nodeStats : stats.getNodes()) {
             if (nodeStats.getBreaker().getStats(CircuitBreaker.REQUEST).getLimit() == NoopCircuitBreaker.LIMIT) {
                 return true;
             }
@@ -230,7 +230,7 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase {
 
         // We need the request limit beforehand, just from a single node because the limit should always be the same
         long beforeReqLimit = client.admin().cluster().prepareNodesStats().setBreaker(true).get()
-                .getNodes()[0].getBreaker().getStats(CircuitBreaker.REQUEST).getLimit();
+                .getNodes().get(0).getBreaker().getStats(CircuitBreaker.REQUEST).getLimit();
 
         Settings resetSettings = Settings.builder()
                 .put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "10b")
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
index 17b1b8e7d70..77cffc20ae1 100644
--- a/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
@@ -69,10 +69,10 @@ import java.util.concurrent.ExecutionException;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
@@ -315,7 +315,7 @@ public class IndexRecoveryIT extends ESIntegTestCase {
             @Override
             public void run() {
                 NodesStatsResponse statsResponse = client().admin().cluster().prepareNodesStats().clear().setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)).get();
-                assertThat(statsResponse.getNodes(), arrayWithSize(2));
+                assertThat(statsResponse.getNodes(), hasSize(2));
                 for (NodeStats nodeStats : statsResponse.getNodes()) {
                     final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
                     if (nodeStats.getNode().getName().equals(nodeA)) {
@@ -344,7 +344,7 @@ public class IndexRecoveryIT extends ESIntegTestCase {
         validateIndexRecoveryState(recoveryStates.get(0).getIndex());
 
         statsResponse = client().admin().cluster().prepareNodesStats().clear().setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)).get();
-        assertThat(statsResponse.getNodes(), arrayWithSize(2));
+        assertThat(statsResponse.getNodes(), hasSize(2));
         for (NodeStats nodeStats : statsResponse.getNodes()) {
             final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
             assertThat(recoveryStats.currentAsSource(), equalTo(0));
@@ -363,7 +363,7 @@ public class IndexRecoveryIT extends ESIntegTestCase {
         ensureGreen();
 
         statsResponse = client().admin().cluster().prepareNodesStats().clear().setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)).get();
-        assertThat(statsResponse.getNodes(), arrayWithSize(2));
+        assertThat(statsResponse.getNodes(), hasSize(2));
         for (NodeStats nodeStats : statsResponse.getNodes()) {
             final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
             assertThat(recoveryStats.currentAsSource(), equalTo(0));
diff --git a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
index b1513908874..4716e7dba78 100644
--- a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
@@ -104,7 +104,7 @@ public class IndexStatsIT extends ESIntegTestCase {
         client().admin().indices().prepareRefresh().execute().actionGet();
 
         NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
-        assertThat(nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes()[1].getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L));
+        assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L));
         IndicesStatsResponse indicesStats = client().admin().indices().prepareStats("test").clear().setFieldData(true).execute().actionGet();
         assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0L));
 
@@ -113,7 +113,7 @@ public class IndexStatsIT extends ESIntegTestCase {
         client().prepareSearch().addSort("field", SortOrder.ASC).execute().actionGet();
 
         nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
-        assertThat(nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes()[1].getIndices().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
+        assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
         indicesStats = client().admin().indices().prepareStats("test").clear().setFieldData(true).execute().actionGet();
         assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
 
@@ -123,9 +123,9 @@ public class IndexStatsIT extends ESIntegTestCase {
 
         // now check the per field stats
         nodesStats = client().admin().cluster().prepareNodesStats().setIndices(new CommonStatsFlags().set(CommonStatsFlags.Flag.FieldData, true).fieldDataFields("*")).execute().actionGet();
-        assertThat(nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes()[1].getIndices().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
-        assertThat(nodesStats.getNodes()[0].getIndices().getFieldData().getFields().get("field") + nodesStats.getNodes()[1].getIndices().getFieldData().getFields().get("field"), greaterThan(0L));
-        assertThat(nodesStats.getNodes()[0].getIndices().getFieldData().getFields().get("field") + nodesStats.getNodes()[1].getIndices().getFieldData().getFields().get("field"), lessThan(nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes()[1].getIndices().getFieldData().getMemorySizeInBytes()));
+        assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
+        assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getFields().get("field") + nodesStats.getNodes().get(1).getIndices().getFieldData().getFields().get("field"), greaterThan(0L));
+        assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getFields().get("field") + nodesStats.getNodes().get(1).getIndices().getFieldData().getFields().get("field"), lessThan(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes()));
 
         indicesStats = client().admin().indices().prepareStats("test").clear().setFieldData(true).setFieldDataFields("*").execute().actionGet();
         assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
@@ -134,7 +134,7 @@ public class IndexStatsIT extends ESIntegTestCase {
 
         client().admin().indices().prepareClearCache().setFieldDataCache(true).execute().actionGet();
         nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
-        assertThat(nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes()[1].getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L));
+        assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L));
         indicesStats = client().admin().indices().prepareStats("test").clear().setFieldData(true).execute().actionGet();
         assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0L));
 
@@ -152,8 +152,8 @@ public class IndexStatsIT extends ESIntegTestCase {
 
         NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true)
                 .execute().actionGet();
-        assertThat(nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes()[1].getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L));
-        assertThat(nodesStats.getNodes()[0].getIndices().getQueryCache().getMemorySizeInBytes() + nodesStats.getNodes()[1].getIndices().getQueryCache().getMemorySizeInBytes(), equalTo(0L));
+        assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L));
+        assertThat(nodesStats.getNodes().get(0).getIndices().getQueryCache().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getQueryCache().getMemorySizeInBytes(), equalTo(0L));
 
         IndicesStatsResponse indicesStats = client().admin().indices().prepareStats("test")
                 .clear().setFieldData(true).setQueryCache(true)
@@ -173,8 +173,8 @@ public class IndexStatsIT extends ESIntegTestCase {
 
         nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true)
                 .execute().actionGet();
-        assertThat(nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes()[1].getIndices().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
-        assertThat(nodesStats.getNodes()[0].getIndices().getQueryCache().getMemorySizeInBytes() + nodesStats.getNodes()[1].getIndices().getQueryCache().getMemorySizeInBytes(), greaterThan(0L));
+        assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
+        assertThat(nodesStats.getNodes().get(0).getIndices().getQueryCache().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getQueryCache().getMemorySizeInBytes(), greaterThan(0L));
 
         indicesStats = client().admin().indices().prepareStats("test")
                 .clear().setFieldData(true).setQueryCache(true)
@@ -186,8 +186,8 @@ public class IndexStatsIT extends ESIntegTestCase {
         Thread.sleep(100); // Make sure the filter cache entries have been removed...
         nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true)
                 .execute().actionGet();
-        assertThat(nodesStats.getNodes()[0].getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes()[1].getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L));
-        assertThat(nodesStats.getNodes()[0].getIndices().getQueryCache().getMemorySizeInBytes() + nodesStats.getNodes()[1].getIndices().getQueryCache().getMemorySizeInBytes(), equalTo(0L));
+        assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L));
+        assertThat(nodesStats.getNodes().get(0).getIndices().getQueryCache().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getQueryCache().getMemorySizeInBytes(), equalTo(0L));
 
         indicesStats = client().admin().indices().prepareStats("test")
                 .clear().setFieldData(true).setQueryCache(true)
diff --git a/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java b/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java
index c38aeb967ef..b7dcf2872e2 100644
--- a/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java
+++ b/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java
@@ -54,29 +54,29 @@ public class SimpleNodesInfoIT extends ESIntegTestCase {
         logger.info("--> started nodes: {} and {}", server1NodeId, server2NodeId);
 
         NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().execute().actionGet();
-        assertThat(response.getNodes().length, is(2));
+        assertThat(response.getNodes().size(), is(2));
         assertThat(response.getNodesMap().get(server1NodeId), notNullValue());
         assertThat(response.getNodesMap().get(server2NodeId), notNullValue());
 
         response = client().admin().cluster().nodesInfo(nodesInfoRequest()).actionGet();
-        assertThat(response.getNodes().length, is(2));
+        assertThat(response.getNodes().size(), is(2));
         assertThat(response.getNodesMap().get(server1NodeId), notNullValue());
         assertThat(response.getNodesMap().get(server2NodeId), notNullValue());
 
         response = client().admin().cluster().nodesInfo(nodesInfoRequest(server1NodeId)).actionGet();
-        assertThat(response.getNodes().length, is(1));
+        assertThat(response.getNodes().size(), is(1));
         assertThat(response.getNodesMap().get(server1NodeId), notNullValue());
 
         response = client().admin().cluster().nodesInfo(nodesInfoRequest(server1NodeId)).actionGet();
-        assertThat(response.getNodes().length, is(1));
+        assertThat(response.getNodes().size(), is(1));
         assertThat(response.getNodesMap().get(server1NodeId), notNullValue());
 
         response = client().admin().cluster().nodesInfo(nodesInfoRequest(server2NodeId)).actionGet();
-        assertThat(response.getNodes().length, is(1));
+        assertThat(response.getNodes().size(), is(1));
         assertThat(response.getNodesMap().get(server2NodeId), notNullValue());
 
         response = client().admin().cluster().nodesInfo(nodesInfoRequest(server2NodeId)).actionGet();
-        assertThat(response.getNodes().length, is(1));
+        assertThat(response.getNodes().size(), is(1));
         assertThat(response.getNodesMap().get(server2NodeId), notNullValue());
     }
 
@@ -99,7 +99,7 @@ public class SimpleNodesInfoIT extends ESIntegTestCase {
 
         NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().execute().actionGet();
 
-        assertThat(response.getNodes().length, is(2));
+        assertThat(response.getNodes().size(), is(2));
         assertThat(response.getNodesMap().get(server1NodeId), notNullValue());
         assertThat(response.getNodesMap().get(server2NodeId), notNullValue());
 
diff --git a/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java b/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java
index 95a931d1d54..baf017bc7b9 100644
--- a/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java
+++ b/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java
@@ -168,8 +168,8 @@ public class SimpleThreadPoolIT extends ESIntegTestCase {
 
         // Check that node info is correct
         NodesInfoResponse nodesInfoResponse = client().admin().cluster().prepareNodesInfo().all().execute().actionGet();
-        for (int i = 0; i < 2; i++) {
-            NodeInfo nodeInfo = nodesInfoResponse.getNodes()[i];
+        assertEquals(2, nodesInfoResponse.getNodes().size());
+        for (NodeInfo nodeInfo : nodesInfoResponse.getNodes()) {
             boolean found = false;
             for (ThreadPool.Info info : nodeInfo.getThreadPool()) {
                 if (info.getName().equals(Names.SEARCH)) {
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportPublishAddressIT.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportPublishAddressIT.java
index 0fceda31664..75faa8c49b4 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportPublishAddressIT.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportPublishAddressIT.java
@@ -67,7 +67,7 @@ public class NettyTransportPublishAddressIT extends ESIntegTestCase {
 
         logger.info("--> checking if boundAddress matching publishAddress has same port");
         NodesInfoResponse nodesInfoResponse = client().admin().cluster().prepareNodesInfo().get();
-        for (NodeInfo nodeInfo : nodesInfoResponse) {
+        for (NodeInfo nodeInfo : nodesInfoResponse.getNodes()) {
             BoundTransportAddress boundTransportAddress = nodeInfo.getTransport().getAddress();
             if (nodeInfo.getNode().getName().equals(ipv4OnlyNode)) {
                 assertThat(boundTransportAddress.boundAddresses().length, equalTo(1));
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchStatsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchStatsTests.java
index 95a2691d1c4..52b2f5af797 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchStatsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchStatsTests.java
@@ -130,10 +130,10 @@ public class SearchStatsTests extends ESIntegTestCase {
         assertThat(indicesStats.getTotal().getSearch().getGroupStats().get("group1").getFetchCount(), greaterThan(0L));
         assertThat(indicesStats.getTotal().getSearch().getGroupStats().get("group1").getFetchTimeInMillis(), greaterThan(0L));
         NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().execute().actionGet();
-        NodeStats[] nodes = nodeStats.getNodes();
+
         Set<String> nodeIdsWithIndex = nodeIdsWithIndex("test1", "test2");
         int num = 0;
-        for (NodeStats stat : nodes) {
+        for (NodeStats stat : nodeStats.getNodes()) {
             Stats total = stat.getIndices().getSearch().getTotal();
             if (nodeIdsWithIndex.contains(stat.getNode().getId())) {
                 assertThat(total.getQueryCount(), greaterThan(0L));
diff --git a/plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureComputeServiceTestCase.java b/plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureComputeServiceTestCase.java
index 3f78b1f6683..0c57ec3f16e 100644
--- a/plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureComputeServiceTestCase.java
+++ b/plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureComputeServiceTestCase.java
@@ -65,6 +65,6 @@ public abstract class AbstractAzureComputeServiceTestCase extends ESIntegTestCas
         NodesInfoResponse nodeInfos = client().admin().cluster().prepareNodesInfo().execute().actionGet();
         assertNotNull(nodeInfos);
         assertNotNull(nodeInfos.getNodes());
-        assertEquals(expected, nodeInfos.getNodes().length);
+        assertEquals(expected, nodeInfos.getNodes().size());
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
index d66b1f9fc9e..ef8857475b3 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
@@ -38,6 +38,8 @@ import org.elasticsearch.monitor.fs.FsInfo;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.threadpool.ThreadPool;
 
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.concurrent.CountDownLatch;
 
 import static java.util.Collections.emptyMap;
@@ -107,7 +109,7 @@ public class MockInternalClusterInfoService extends InternalClusterInfoService {
 
     @Override
     public CountDownLatch updateNodeStats(final ActionListener<NodesStatsResponse> listener) {
-        NodesStatsResponse response = new NodesStatsResponse(clusterName, stats);
+        NodesStatsResponse response = new NodesStatsResponse(clusterName, Arrays.asList(stats), Collections.emptyList());
         listener.onResponse(response);
         return new CountDownLatch(0);
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index a31ef76272e..247a340d8c4 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -2046,8 +2046,8 @@ public abstract class ESIntegTestCase extends ESTestCase {
 
     protected HttpRequestBuilder httpClient(CloseableHttpClient httpClient) {
         final NodesInfoResponse nodeInfos = client().admin().cluster().prepareNodesInfo().get();
-        final NodeInfo[] nodes = nodeInfos.getNodes();
-        assertTrue(nodes.length > 0);
+        final List<NodeInfo> nodes = nodeInfos.getNodes();
+        assertFalse(nodeInfos.hasFailures());
         TransportAddress publishAddress = randomFrom(nodes).getHttp().address().publishAddress();
         assertEquals(1, publishAddress.uniqueAddressTypeId());
         InetSocketAddress address = ((InetSocketTransportAddress) publishAddress).address();
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java b/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java
index 296f34637dd..4625aa77e25 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java
@@ -154,8 +154,7 @@ final class ExternalNode implements Closeable {
     static boolean waitForNode(final Client client, final String name) throws InterruptedException {
         return ESTestCase.awaitBusy(() -> {
             final NodesInfoResponse nodeInfos = client.admin().cluster().prepareNodesInfo().get();
-            final NodeInfo[] nodes = nodeInfos.getNodes();
-            for (NodeInfo info : nodes) {
+            for (NodeInfo info : nodeInfos.getNodes()) {
                 if (name.equals(info.getNode().getName())) {
                     return true;
                 }
@@ -166,8 +165,7 @@ final class ExternalNode implements Closeable {
 
     static NodeInfo nodeInfo(final Client client, final String nodeName) {
         final NodesInfoResponse nodeInfos = client.admin().cluster().prepareNodesInfo().get();
-        final NodeInfo[] nodes = nodeInfos.getNodes();
-        for (NodeInfo info : nodes) {
+        for (NodeInfo info : nodeInfos.getNodes()) {
             if (nodeName.equals(info.getNode().getName())) {
                 return info;
             }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java
index 17c3e3d0805..5372c319dae 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java
@@ -87,12 +87,12 @@ public final class ExternalTestCluster extends TestCluster {
         try {
             client.addTransportAddresses(transportAddresses);
             NodesInfoResponse nodeInfos = client.admin().cluster().prepareNodesInfo().clear().setSettings(true).setHttp(true).get();
-            httpAddresses = new InetSocketAddress[nodeInfos.getNodes().length];
+            httpAddresses = new InetSocketAddress[nodeInfos.getNodes().size()];
             this.clusterName = nodeInfos.getClusterName().value();
             int dataNodes = 0;
             int masterAndDataNodes = 0;
-            for (int i = 0; i < nodeInfos.getNodes().length; i++) {
-                NodeInfo nodeInfo = nodeInfos.getNodes()[i];
+            for (int i = 0; i < nodeInfos.getNodes().size(); i++) {
+                NodeInfo nodeInfo = nodeInfos.getNodes().get(i);
                 httpAddresses[i] = ((InetSocketTransportAddress) nodeInfo.getHttp().address().publishAddress()).address();
                 if (DiscoveryNode.isDataNode(nodeInfo.getSettings())) {
                     dataNodes++;

From 2c3d3d91cb1aa99656a345ba6481ef1d8a644ace Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Fri, 6 May 2016 13:09:48 -0700
Subject: [PATCH 0118/1311] Remove unnecessary nebula source or javadoc plugins

---
 core/build.gradle | 2 --
 1 file changed, 2 deletions(-)

diff --git a/core/build.gradle b/core/build.gradle
index e12a80fcbf7..a549f3260fa 100644
--- a/core/build.gradle
+++ b/core/build.gradle
@@ -26,8 +26,6 @@ apply plugin: 'com.bmuschko.nexus'
 apply plugin: 'nebula.optional-base'
 apply plugin: 'nebula.maven-base-publish'
 apply plugin: 'nebula.maven-scm'
-//apply plugin: 'nebula.source-jar'
-//apply plugin: 'nebula.javadoc-jar'
 
 publishing {
   publications {

From d0d2d2be8c2cbd94949273b4bc848aa0430c10a4 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 6 May 2016 16:14:43 -0400
Subject: [PATCH 0119/1311] Fix exception message in lifecycle

This commit fixes the exception messages for lifecycles when stopping in
illegal states.

Relates #18189
---
 .../elasticsearch/common/component/Lifecycle.java   | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/component/Lifecycle.java b/core/src/main/java/org/elasticsearch/common/component/Lifecycle.java
index 479496dd9b6..4f0ef4c6887 100644
--- a/core/src/main/java/org/elasticsearch/common/component/Lifecycle.java
+++ b/core/src/main/java/org/elasticsearch/common/component/Lifecycle.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.common.component;
 
-
 /**
  * Lifecycle state. Allows the following transitions:
  * <ul>
@@ -56,7 +55,7 @@ package org.elasticsearch.common.component;
  */
 public class Lifecycle {
 
-    public static enum State {
+    public enum State {
         INITIALIZED,
         STOPPED,
         STARTED,
@@ -141,9 +140,9 @@ public class Lifecycle {
             return false;
         }
         if (localState == State.CLOSED) {
-            throw new IllegalStateException("Can't move to started state when closed");
+            throw new IllegalStateException("Can't move to stopped state when closed");
         }
-        throw new IllegalStateException("Can't move to started with unknown state");
+        throw new IllegalStateException("Can't move to stopped with unknown state");
     }
 
     public boolean moveToStopped() throws IllegalStateException {
@@ -156,9 +155,9 @@ public class Lifecycle {
             return false;
         }
         if (localState == State.CLOSED) {
-            throw new IllegalStateException("Can't move to started state when closed");
+            throw new IllegalStateException("Can't move to stopped state when closed");
         }
-        throw new IllegalStateException("Can't move to started with unknown state");
+        throw new IllegalStateException("Can't move to stopped with unknown state");
     }
 
     public boolean canMoveToClosed() throws IllegalStateException {
@@ -172,7 +171,6 @@ public class Lifecycle {
         return true;
     }
 
-
     public boolean moveToClosed() throws IllegalStateException {
         State localState = state;
         if (localState == State.CLOSED) {
@@ -189,4 +187,5 @@ public class Lifecycle {
     public String toString() {
         return state.toString();
     }
+
 }

From c1afcb543e70942dc0196216bb50d7e23af7d541 Mon Sep 17 00:00:00 2001
From: Tal Levy <JubBoy333@gmail.com>
Date: Fri, 6 May 2016 13:34:22 -0700
Subject: [PATCH 0120/1311] add check for non-existent pipelines provided to
 simulate requests (#18190)

fixes #18139
---
 .../ingest/SimulatePipelineRequest.java       |  3 ++
 .../SimulatePipelineRequestParsingTests.java  | 30 +++++++++++++++----
 2 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
index 0a8cbdfe0c1..14092ad62cb 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
@@ -132,6 +132,9 @@ public class SimulatePipelineRequest extends ActionRequest<SimulatePipelineReque
             throw new IllegalArgumentException("param [pipeline] is null");
         }
         Pipeline pipeline = pipelineStore.get(pipelineId);
+        if (pipeline == null) {
+            throw new IllegalArgumentException("pipeline [" + pipelineId + "] does not exist");
+        }
         List<IngestDocument> ingestDocumentList = parseDocs(config);
         return new Parsed(pipeline, ingestDocumentList, verbose);
     }
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
index de0a28fd7e1..b7cd5ef0b92 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
@@ -39,6 +39,7 @@ import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.action.ingest.SimulatePipelineRequest.Fields;
+import static org.elasticsearch.action.ingest.SimulatePipelineRequest.SIMULATED_PIPELINE_ID;
 import static org.elasticsearch.ingest.core.IngestDocument.MetaData.ID;
 import static org.elasticsearch.ingest.core.IngestDocument.MetaData.INDEX;
 import static org.elasticsearch.ingest.core.IngestDocument.MetaData.TYPE;
@@ -55,12 +56,12 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
     public void init() throws IOException {
         TestProcessor processor = new TestProcessor(ingestDocument -> {});
         CompoundProcessor pipelineCompoundProcessor = new CompoundProcessor(processor);
-        Pipeline pipeline = new Pipeline(SimulatePipelineRequest.SIMULATED_PIPELINE_ID, null, pipelineCompoundProcessor);
+        Pipeline pipeline = new Pipeline(SIMULATED_PIPELINE_ID, null, pipelineCompoundProcessor);
         ProcessorsRegistry.Builder processorRegistryBuilder = new ProcessorsRegistry.Builder();
         processorRegistryBuilder.registerProcessor("mock_processor", ((templateService, registry) -> mock(Processor.Factory.class)));
         ProcessorsRegistry processorRegistry = processorRegistryBuilder.build(TestTemplateService.instance());
         store = mock(PipelineStore.class);
-        when(store.get(SimulatePipelineRequest.SIMULATED_PIPELINE_ID)).thenReturn(pipeline);
+        when(store.get(SIMULATED_PIPELINE_ID)).thenReturn(pipeline);
         when(store.getProcessorRegistry()).thenReturn(processorRegistry);
     }
 
@@ -91,7 +92,7 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
             expectedDocs.add(expectedDoc);
         }
 
-        SimulatePipelineRequest.Parsed actualRequest = SimulatePipelineRequest.parseWithPipelineId(SimulatePipelineRequest.SIMULATED_PIPELINE_ID, requestContent, false, store);
+        SimulatePipelineRequest.Parsed actualRequest = SimulatePipelineRequest.parseWithPipelineId(SIMULATED_PIPELINE_ID, requestContent, false, store);
         assertThat(actualRequest.isVerbose(), equalTo(false));
         assertThat(actualRequest.getDocuments().size(), equalTo(numDocs));
         Iterator<Map<String, Object>> expectedDocsIterator = expectedDocs.iterator();
@@ -104,7 +105,7 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
             assertThat(ingestDocument.getSourceAndMetadata(), equalTo(expectedDocument.get(Fields.SOURCE)));
         }
 
-        assertThat(actualRequest.getPipeline().getId(), equalTo(SimulatePipelineRequest.SIMULATED_PIPELINE_ID));
+        assertThat(actualRequest.getPipeline().getId(), equalTo(SIMULATED_PIPELINE_ID));
         assertThat(actualRequest.getPipeline().getDescription(), nullValue());
         assertThat(actualRequest.getPipeline().getProcessors().size(), equalTo(1));
     }
@@ -177,8 +178,27 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
             assertThat(ingestDocument.getSourceAndMetadata(), equalTo(expectedDocument.get(Fields.SOURCE)));
         }
 
-        assertThat(actualRequest.getPipeline().getId(), equalTo(SimulatePipelineRequest.SIMULATED_PIPELINE_ID));
+        assertThat(actualRequest.getPipeline().getId(), equalTo(SIMULATED_PIPELINE_ID));
         assertThat(actualRequest.getPipeline().getDescription(), nullValue());
         assertThat(actualRequest.getPipeline().getProcessors().size(), equalTo(numProcessors));
     }
+
+    public void testNullPipelineId() {
+        Map<String, Object> requestContent = new HashMap<>();
+        List<Map<String, Object>> docs = new ArrayList<>();
+        requestContent.put(Fields.DOCS, docs);
+        Exception e = expectThrows(IllegalArgumentException.class,
+            () -> SimulatePipelineRequest.parseWithPipelineId(null, requestContent, false, store));
+        assertThat(e.getMessage(), equalTo("param [pipeline] is null"));
+    }
+
+    public void testNonExistentPipelineId() {
+        String pipelineId = randomAsciiOfLengthBetween(1, 10);
+        Map<String, Object> requestContent = new HashMap<>();
+        List<Map<String, Object>> docs = new ArrayList<>();
+        requestContent.put(Fields.DOCS, docs);
+        Exception e = expectThrows(IllegalArgumentException.class,
+            () -> SimulatePipelineRequest.parseWithPipelineId(pipelineId, requestContent, false, store));
+        assertThat(e.getMessage(), equalTo("pipeline [" + pipelineId + "] does not exist"));
+    }
 }

From c740feb3a75ac4991d299b94de1e40981b274a40 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Fri, 6 May 2016 17:07:24 -0700
Subject: [PATCH 0121/1311] Mappings: Fix ip mapper to correctly serialize a
 null null_value

We recently added correct serialization for null values, but the helper
method used does not allow null. This fixes serialization to handle the
null.
---
 .../index/mapper/ip/IpFieldMapper.java        |  6 +++++-
 .../index/mapper/ip/IpFieldMapperTests.java   | 20 +++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
index 5f5a3d85fe1..0d9810bd405 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
@@ -396,7 +396,11 @@ public class IpFieldMapper extends FieldMapper implements AllFieldMapper.Include
         super.doXContentBody(builder, includeDefaults, params);
 
         if (includeDefaults || fieldType().nullValue() != null) {
-            builder.field("null_value", InetAddresses.toAddrString((InetAddress) fieldType().nullValue()));
+            Object nullValue = fieldType().nullValue();
+            if (nullValue != null) {
+                nullValue = InetAddresses.toAddrString((InetAddress) nullValue);
+            }
+            builder.field("null_value", nullValue);
         }
 
         if (includeDefaults || ignoreMalformed.explicit()) {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ip/IpFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ip/IpFieldMapperTests.java
index 25979c5a632..d99b37fc667 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/ip/IpFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ip/IpFieldMapperTests.java
@@ -25,6 +25,8 @@ import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.network.InetAddresses;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.DocumentMapper;
@@ -269,4 +271,22 @@ public class IpFieldMapperTests extends ESSingleNodeTestCase {
         assertEquals(new BytesRef(InetAddressPoint.encode(InetAddresses.forString("::1"))), dvField.binaryValue());
         assertFalse(dvField.fieldType().stored());
     }
+
+    public void testSerializeDefaults() throws Exception {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties").startObject("field").field("type", "ip").endObject().endObject()
+            .endObject().endObject().string();
+
+        DocumentMapper docMapper = parser.parse("type", new CompressedXContent(mapping));
+        IpFieldMapper mapper = (IpFieldMapper)docMapper.root().getMapper("field");
+        XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+        mapper.doXContentBody(builder, true, ToXContent.EMPTY_PARAMS);
+        String got = builder.endObject().string();
+
+        // it would be nice to check the entire serialized default mapper, but there are
+        // a whole lot of bogus settings right now it picks up from calling super.doXContentBody...
+        assertTrue(got, got.contains("\"null_value\":null"));
+        assertTrue(got, got.contains("\"ignore_malformed\":false"));
+        assertTrue(got, got.contains("\"include_in_all\":false"));
+    }
 }

From a78cdcdbc812361fa959baca485a61d14fcedef4 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Fri, 6 May 2016 22:22:38 -0700
Subject: [PATCH 0122/1311] Build: Add fake project to include buildSrc as
 normal project

This is a follow up to #18173 and includes adding pom generation to the
fake build-tools project, which is really just buildSrc, but builds
during normal builds.
---
 buildSrc/.gitignore                           |   1 +
 buildSrc/build.gradle                         | 179 +++++++++++-------
 .../elasticsearch/gradle/BuildPlugin.groovy   |   2 +-
 settings.gradle                               |   3 +
 4 files changed, 115 insertions(+), 70 deletions(-)
 create mode 100644 buildSrc/.gitignore

diff --git a/buildSrc/.gitignore b/buildSrc/.gitignore
new file mode 100644
index 00000000000..bfdaf60b97e
--- /dev/null
+++ b/buildSrc/.gitignore
@@ -0,0 +1 @@
+build-bootstrap/
diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
index 1b0d3c52127..be83d525f5f 100644
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -1,5 +1,3 @@
-import java.nio.file.Files
-
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -19,25 +17,21 @@ import java.nio.file.Files
  * under the License.
  */
 
-// we must use buildscript + apply so that an external plugin
-// can apply this file, since the plugins directive is not
-// supported through file includes
-buildscript {
-  repositories {
-    jcenter()
-  }
-  dependencies {
-    classpath 'com.bmuschko:gradle-nexus-plugin:2.3.1'
-  }
-}
+import java.nio.file.Files
+
 apply plugin: 'groovy'
-apply plugin: 'com.bmuschko.nexus'
-// TODO: move common IDE configuration to a common file to include
-apply plugin: 'idea'
-apply plugin: 'eclipse'
 
 group = 'org.elasticsearch.gradle'
-archivesBaseName = 'build-tools'
+
+if (project == rootProject) {
+  // change the build dir used during build init, so that doing a clean
+  // won't wipe out the buildscript jar
+  buildDir = 'build-bootstrap'
+}
+
+/*****************************************************************************
+ *         Propagating version.properties to the rest of the build           *
+ *****************************************************************************/
 
 Properties props = new Properties()
 props.load(project.file('version.properties').newDataInputStream())
@@ -51,33 +45,6 @@ if (snapshot) {
   props.put("elasticsearch", version);
 }
 
-
-repositories {
-  mavenCentral()
-  maven {
-    name 'sonatype-snapshots'
-    url "https://oss.sonatype.org/content/repositories/snapshots/"
-  }
-  jcenter()
-}
-
-dependencies {
-  compile gradleApi()
-  compile localGroovy()
-  compile "com.carrotsearch.randomizedtesting:junit4-ant:${props.getProperty('randomizedrunner')}"
-  compile("junit:junit:${props.getProperty('junit')}") {
-    transitive = false
-  }
-  compile 'com.netflix.nebula:gradle-extra-configurations-plugin:3.0.3'
-  compile 'com.netflix.nebula:nebula-publishing-plugin:4.4.4'
-  compile 'com.netflix.nebula:gradle-info-plugin:3.0.3'
-  compile 'org.eclipse.jgit:org.eclipse.jgit:3.2.0.201312181205-r'
-  compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
-  compile 'de.thetaphi:forbiddenapis:2.0'
-  compile 'com.bmuschko:gradle-nexus-plugin:2.3.1'
-  compile 'org.apache.rat:apache-rat:0.11'
-}
-
 File tempPropertiesFile = new File(project.buildDir, "version.properties")
 task writeVersionProperties {
   inputs.properties(props)
@@ -97,31 +64,105 @@ processResources {
   from tempPropertiesFile
 }
 
-extraArchive {
-  javadoc = false
-  tests = false
+/*****************************************************************************
+ *                    Dependencies used by the entire build                  *
+ *****************************************************************************/
+
+repositories {
+  jcenter()
 }
 
-idea {
-  module {
-    inheritOutputDirs = false
-    outputDir = file('build-idea/classes/main')
-    testOutputDir = file('build-idea/classes/test')
+dependencies {
+  compile gradleApi()
+  compile localGroovy()
+  compile "com.carrotsearch.randomizedtesting:junit4-ant:${props.getProperty('randomizedrunner')}"
+  compile("junit:junit:${props.getProperty('junit')}") {
+    transitive = false
+  }
+  compile 'com.netflix.nebula:gradle-extra-configurations-plugin:3.0.3'
+  compile 'com.netflix.nebula:nebula-publishing-plugin:4.4.4'
+  compile 'com.netflix.nebula:gradle-info-plugin:3.0.3'
+  compile 'org.eclipse.jgit:org.eclipse.jgit:3.2.0.201312181205-r'
+  compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
+  compile 'de.thetaphi:forbiddenapis:2.0'
+  compile 'com.bmuschko:gradle-nexus-plugin:2.3.1'
+  compile 'org.apache.rat:apache-rat:0.11'
+}
+
+
+/*****************************************************************************
+ *                    Bootstrap repositories and IDE setup                   *
+ *****************************************************************************/
+// this will only happen when buildSrc is built on its own during build init
+if (project == rootProject) {
+
+  // TODO: move common IDE configuration to a common file to include
+  apply plugin: 'idea'
+  apply plugin: 'eclipse'
+
+  repositories {
+    mavenCentral()
+    maven {
+      name 'sonatype-snapshots'
+      url "https://oss.sonatype.org/content/repositories/snapshots/"
+    }
+  }
+
+  idea {
+    module {
+      inheritOutputDirs = false
+      outputDir = file('build-idea/classes/main')
+      testOutputDir = file('build-idea/classes/test')
+    }
+  }
+
+  eclipse {
+    classpath {
+      defaultOutputDir = file('build-eclipse')
+    }
+  }
+
+  task copyEclipseSettings(type: Copy) {
+    from project.file('src/main/resources/eclipse.settings')
+    into '.settings'
+  }
+  // otherwise .settings is not nuked entirely
+  tasks.cleanEclipse {
+    delete '.settings'
+  }
+  tasks.eclipse.dependsOn(cleanEclipse, copyEclipseSettings)
+}
+
+/*****************************************************************************
+ *                           Normal project checks                           *
+ *****************************************************************************/
+
+// this happens when included as a normal project in the build, which we do
+// to enforce precommit checks like forbidden apis, as well as setup publishing
+if (project != rootProject) {
+  apply plugin: 'elasticsearch.build'
+  apply plugin: 'nebula.maven-base-publish'
+  apply plugin: 'nebula.maven-scm'
+
+  // groovydoc succeeds, but has some weird internal exception...
+  groovydoc.enabled = false
+
+  // build-tools is not ready for primetime with these...
+  dependencyLicenses.enabled = false
+  forbiddenApisMain.enabled = false
+  jarHell.enabled = false
+  loggerUsageCheck.enabled = false
+  thirdPartyAudit.enabled = false
+
+  // test for elasticsearch.build tries to run with ES...
+  test.enabled = false
+
+  // TODO: re-enable once randomizedtesting gradle code is published and removed from here
+  licenseHeaders.enabled = false
+
+  forbiddenPatterns {
+    exclude '**/*.wav'
+    // the file that actually defines nocommit
+    exclude '**/ForbiddenPatternsTask.groovy'
   }
 }
-
-eclipse {
-  classpath {
-    defaultOutputDir = file('build-eclipse')
-  }
-}
-
-task copyEclipseSettings(type: Copy) {
-  from project.file('src/main/resources/eclipse.settings')
-  into '.settings'
-}
-// otherwise .settings is not nuked entirely
-tasks.cleanEclipse {
-  delete '.settings'
-}
-tasks.eclipse.dependsOn(cleanEclipse, copyEclipseSettings)
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index de5dbb73af6..2cfd98f276f 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -232,7 +232,7 @@ class BuildPlugin implements Plugin<Project> {
      */
     static void configureConfigurations(Project project) {
         // we are not shipping these jars, we act like dumb consumers of these things
-        if (project.path.startsWith(':test:fixtures')) {
+        if (project.path.startsWith(':test:fixtures') || project.path == ':build-tools') {
             return
         }
         // fail on any conflicting dependency versions
diff --git a/settings.gradle b/settings.gradle
index 1fc74a0ef2e..3a8b0f66210 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -1,6 +1,7 @@
 rootProject.name = 'elasticsearch'
 
 List projects = [
+  'build-tools',
   'rest-api-spec',
   'core',
   'docs',
@@ -60,6 +61,8 @@ if (isEclipse) {
 
 include projects.toArray(new String[0])
 
+project(':build-tools').projectDir = new File(rootProject.projectDir, 'buildSrc')
+
 if (isEclipse) {
   project(":core").projectDir = new File(rootProject.projectDir, 'core/src/main')
   project(":core").buildFileName = 'eclipse-build.gradle'

From 4ae5c0d93da2e2a9ae5af020e22baad440eab3a4 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Sat, 7 May 2016 11:06:06 -0400
Subject: [PATCH 0123/1311] Add link to field data docs from field data fields

This commit adds a link to the field data docs from the field data
fields docs for completeness.

Closes #17672
---
 docs/reference/search/request/fielddata-fields.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/search/request/fielddata-fields.asciidoc b/docs/reference/search/request/fielddata-fields.asciidoc
index 3a2d507c74e..aaaa606980e 100644
--- a/docs/reference/search/request/fielddata-fields.asciidoc
+++ b/docs/reference/search/request/fielddata-fields.asciidoc
@@ -1,7 +1,7 @@
 [[search-request-fielddata-fields]]
 === Field Data Fields
 
-Allows to return the field data representation of a field for each hit, for
+Allows to return the <<fielddata,field data>> representation of a field for each hit, for
 example:
 
 [source,js]

From b352a90454085fde41cfb82d220f39513e6d83bb Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Sat, 7 May 2016 17:16:31 +0200
Subject: [PATCH 0124/1311] Correct docs for dynamic mapping of fields

Floating point numbers are added as `float`, and Strings are added as `text` with `keyword sub-field
---
 docs/reference/mapping/dynamic/field-mapping.asciidoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/reference/mapping/dynamic/field-mapping.asciidoc b/docs/reference/mapping/dynamic/field-mapping.asciidoc
index d80b52440c6..a1ce6683502 100644
--- a/docs/reference/mapping/dynamic/field-mapping.asciidoc
+++ b/docs/reference/mapping/dynamic/field-mapping.asciidoc
@@ -14,7 +14,7 @@ determine which datatype the field should have:
 
 `null`::                            No field is added.
 `true` or `false`::                 <<boolean,`boolean`>> field
-floating{nbsp}point{nbsp}number::   <<number,`double`>> field
+floating{nbsp}point{nbsp}number::   <<number,`float`>> field
 integer::                           <<number,`long`>> field
 object::                            <<object,`object`>> field
 array::                             Depends on the first non-`null` value in the array.
@@ -22,7 +22,7 @@ string::                            Either a <<date,`date`>> field
                                         (if the value passes <<date-detection,date detection>>),
                                     a <<number,`double`>> or <<number,`long`>> field
                                         (if the value passes <<numeric-detection,numeric detection>>)
-                                    or an <<text,`text`>> field.
+                                    or a <<text,`text`>> field, with a <<keyword,`keyword`>> sub-field.
 
 These are the only <<mapping-types,field datatypes>> that are dynamically
 detected.  All other datatypes must be mapped explicitly.

From 000dd62318712e2327eaa68fea6205c0e86a5bec Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@elastic.co>
Date: Mon, 9 May 2016 11:11:59 +0200
Subject: [PATCH 0125/1311] Determine content length eagerly in HttpServer

With this commit we eagerly evaluate content length in HttpServer
and also pass the same value to ResourceHandlingHttpChannel. With
this change it easier to reason about the content length that is
freed leaving no doubt that it must be identical to the reserved
amount.
---
 .../main/java/org/elasticsearch/http/HttpServer.java  | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/http/HttpServer.java b/core/src/main/java/org/elasticsearch/http/HttpServer.java
index 45abad0fb81..33a1c1304f9 100644
--- a/core/src/main/java/org/elasticsearch/http/HttpServer.java
+++ b/core/src/main/java/org/elasticsearch/http/HttpServer.java
@@ -106,9 +106,10 @@ public class HttpServer extends AbstractLifecycleComponent<HttpServer> implement
         }
         RestChannel responseChannel = channel;
         try {
-            inFlightRequestsBreaker(circuitBreakerService).addEstimateBytesAndMaybeBreak(request.content().length(), "<http_request>");
+            int contentLength = request.content().length();
+            inFlightRequestsBreaker(circuitBreakerService).addEstimateBytesAndMaybeBreak(contentLength, "<http_request>");
             // iff we could reserve bytes for the request we need to send the response also over this channel
-            responseChannel = new ResourceHandlingHttpChannel(channel, circuitBreakerService);
+            responseChannel = new ResourceHandlingHttpChannel(channel, circuitBreakerService, contentLength);
             restController.dispatchRequest(request, responseChannel, threadContext);
         } catch (Throwable t) {
             restController.sendErrorResponse(request, responseChannel, t);
@@ -135,11 +136,13 @@ public class HttpServer extends AbstractLifecycleComponent<HttpServer> implement
     private static final class ResourceHandlingHttpChannel implements RestChannel {
         private final RestChannel delegate;
         private final CircuitBreakerService circuitBreakerService;
+        private final int contentLength;
         private final AtomicBoolean closed = new AtomicBoolean();
 
-        public ResourceHandlingHttpChannel(RestChannel delegate, CircuitBreakerService circuitBreakerService) {
+        public ResourceHandlingHttpChannel(RestChannel delegate, CircuitBreakerService circuitBreakerService, int contentLength) {
             this.delegate = delegate;
             this.circuitBreakerService = circuitBreakerService;
+            this.contentLength = contentLength;
         }
 
         @Override
@@ -183,7 +186,7 @@ public class HttpServer extends AbstractLifecycleComponent<HttpServer> implement
             if (closed.compareAndSet(false, true) == false) {
                 throw new IllegalStateException("Channel is already closed");
             }
-            inFlightRequestsBreaker(circuitBreakerService).addWithoutBreaking(-request().content().length());
+            inFlightRequestsBreaker(circuitBreakerService).addWithoutBreaking(-contentLength);
         }
 
     }

From 3a46812a335a56c56106aabd98029854ee2b54b3 Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@elastic.co>
Date: Mon, 9 May 2016 11:16:49 +0200
Subject: [PATCH 0126/1311] Free bytes reserved on request breaker

With this commit we free all bytes reserved on the request
circuit breaker.

Closes #18144
---
 .../org/elasticsearch/http/HttpServer.java    |  5 +-
 .../elasticsearch/rest/BytesRestResponse.java |  4 --
 .../elasticsearch/rest/RestController.java    |  6 ++-
 .../RestClusterAllocationExplainAction.java   |  6 ++-
 .../alias/head/RestAliasesExistAction.java    |  8 +--
 .../indices/RestIndicesExistsAction.java      |  5 +-
 .../exists/types/RestTypesExistsAction.java   |  5 +-
 .../head/RestHeadIndexTemplateAction.java     |  5 +-
 .../rest/action/get/RestHeadAction.java       |  5 +-
 .../rest/action/main/RestMainAction.java      |  3 +-
 .../elasticsearch/http/HttpServerTests.java   |  4 +-
 .../rest/action/main/RestMainActionIT.java    | 52 +++++++++++++++++++
 12 files changed, 84 insertions(+), 24 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java

diff --git a/core/src/main/java/org/elasticsearch/http/HttpServer.java b/core/src/main/java/org/elasticsearch/http/HttpServer.java
index 45abad0fb81..e96b0a85beb 100644
--- a/core/src/main/java/org/elasticsearch/http/HttpServer.java
+++ b/core/src/main/java/org/elasticsearch/http/HttpServer.java
@@ -21,6 +21,7 @@ package org.elasticsearch.http;
 
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
@@ -125,10 +126,10 @@ public class HttpServer extends AbstractLifecycleComponent<HttpServer> implement
                     channel.sendResponse(restResponse);
                 }
             } catch (IOException e) {
-                channel.sendResponse(new BytesRestResponse(INTERNAL_SERVER_ERROR));
+                channel.sendResponse(new BytesRestResponse(INTERNAL_SERVER_ERROR, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
             }
         } else {
-            channel.sendResponse(new BytesRestResponse(FORBIDDEN));
+            channel.sendResponse(new BytesRestResponse(FORBIDDEN, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java b/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java
index 52f624849fc..2194732445d 100644
--- a/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java
+++ b/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java
@@ -40,10 +40,6 @@ public class BytesRestResponse extends RestResponse {
     private final BytesReference content;
     private final String contentType;
 
-    public BytesRestResponse(RestStatus status) {
-        this(status, TEXT_CONTENT_TYPE, BytesArray.EMPTY);
-    }
-
     /**
      * Creates a new response based on {@link XContentBuilder}.
      */
diff --git a/core/src/main/java/org/elasticsearch/rest/RestController.java b/core/src/main/java/org/elasticsearch/rest/RestController.java
index 6da1a929f7c..aa567af09f7 100644
--- a/core/src/main/java/org/elasticsearch/rest/RestController.java
+++ b/core/src/main/java/org/elasticsearch/rest/RestController.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.rest;
 
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.path.PathTrie;
@@ -215,9 +216,10 @@ public class RestController extends AbstractLifecycleComponent<RestController> {
         } else {
             if (request.method() == RestRequest.Method.OPTIONS) {
                 // when we have OPTIONS request, simply send OK by default (with the Access Control Origin header which gets automatically added)
-                channel.sendResponse(new BytesRestResponse(OK));
+                channel.sendResponse(new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
             } else {
-                channel.sendResponse(new BytesRestResponse(BAD_REQUEST, "No handler found for uri [" + request.uri() + "] and method [" + request.method() + "]"));
+                final String msg = "No handler found for uri [" + request.uri() + "] and method [" + request.method() + "]";
+                channel.sendResponse(new BytesRestResponse(BAD_REQUEST, msg));
             }
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/allocation/RestClusterAllocationExplainAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/allocation/RestClusterAllocationExplainAction.java
index 06ba2a9be87..0785151e8d7 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/allocation/RestClusterAllocationExplainAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/allocation/RestClusterAllocationExplainAction.java
@@ -24,6 +24,7 @@ import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplai
 import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.Requests;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -67,7 +68,8 @@ public class RestClusterAllocationExplainAction extends BaseRestHandler {
                 req = ClusterAllocationExplainRequest.parse(parser);
             } catch (IOException e) {
                 logger.debug("failed to parse allocation explain request", e);
-                channel.sendResponse(new BytesRestResponse(ExceptionsHelper.status(e)));
+                channel.sendResponse(
+                    new BytesRestResponse(ExceptionsHelper.status(e), BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
                 return;
             }
         }
@@ -83,7 +85,7 @@ public class RestClusterAllocationExplainAction extends BaseRestHandler {
             });
         } catch (Exception e) {
             logger.error("failed to explain allocation", e);
-            channel.sendResponse(new BytesRestResponse(ExceptionsHelper.status(e)));
+            channel.sendResponse(new BytesRestResponse(ExceptionsHelper.status(e), BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/head/RestAliasesExistAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/head/RestAliasesExistAction.java
index 15ea664245d..65f8fd363ea 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/head/RestAliasesExistAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/head/RestAliasesExistAction.java
@@ -26,6 +26,7 @@ import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -65,9 +66,9 @@ public class RestAliasesExistAction extends BaseRestHandler {
             public void onResponse(AliasesExistResponse response) {
                 try {
                     if (response.isExists()) {
-                        channel.sendResponse(new BytesRestResponse(OK));
+                        channel.sendResponse(new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
                     } else {
-                        channel.sendResponse(new BytesRestResponse(NOT_FOUND));
+                        channel.sendResponse(new BytesRestResponse(NOT_FOUND, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
                     }
                 } catch (Throwable e) {
                     onFailure(e);
@@ -77,7 +78,8 @@ public class RestAliasesExistAction extends BaseRestHandler {
             @Override
             public void onFailure(Throwable e) {
                 try {
-                    channel.sendResponse(new BytesRestResponse(ExceptionsHelper.status(e)));
+                    channel.sendResponse(
+                        new BytesRestResponse(ExceptionsHelper.status(e), BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
                 } catch (Exception e1) {
                     logger.error("Failed to send failure response", e1);
                 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/exists/indices/RestIndicesExistsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/exists/indices/RestIndicesExistsAction.java
index 72dea18abd9..0e240352a73 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/exists/indices/RestIndicesExistsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/exists/indices/RestIndicesExistsAction.java
@@ -24,6 +24,7 @@ import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRespon
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -58,9 +59,9 @@ public class RestIndicesExistsAction extends BaseRestHandler {
             @Override
             public RestResponse buildResponse(IndicesExistsResponse response) {
                 if (response.isExists()) {
-                    return new BytesRestResponse(OK);
+                    return new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
                 } else {
-                    return new BytesRestResponse(NOT_FOUND);
+                    return new BytesRestResponse(NOT_FOUND, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
                 }
             }
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/exists/types/RestTypesExistsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/exists/types/RestTypesExistsAction.java
index dd206dcb63a..7a55be7e3af 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/exists/types/RestTypesExistsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/exists/types/RestTypesExistsAction.java
@@ -23,6 +23,7 @@ import org.elasticsearch.action.admin.indices.exists.types.TypesExistsResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -59,9 +60,9 @@ public class RestTypesExistsAction extends BaseRestHandler {
             @Override
             public RestResponse buildResponse(TypesExistsResponse response) throws Exception {
                 if (response.isExists()) {
-                    return new BytesRestResponse(OK);
+                    return new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
                 } else {
-                    return new BytesRestResponse(NOT_FOUND);
+                    return new BytesRestResponse(NOT_FOUND, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
                 }
             }
         });
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/template/head/RestHeadIndexTemplateAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/template/head/RestHeadIndexTemplateAction.java
index 648d083e763..2a40de984b9 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/template/head/RestHeadIndexTemplateAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/template/head/RestHeadIndexTemplateAction.java
@@ -21,6 +21,7 @@ package org.elasticsearch.rest.action.admin.indices.template.head;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -57,9 +58,9 @@ public class RestHeadIndexTemplateAction extends BaseRestHandler {
             public RestResponse buildResponse(GetIndexTemplatesResponse getIndexTemplatesResponse) {
                 boolean templateExists = getIndexTemplatesResponse.getIndexTemplates().size() > 0;
                 if (templateExists) {
-                    return new BytesRestResponse(OK);
+                    return new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
                 } else {
-                    return new BytesRestResponse(NOT_FOUND);
+                    return new BytesRestResponse(NOT_FOUND, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
                 }
             }
         });
diff --git a/core/src/main/java/org/elasticsearch/rest/action/get/RestHeadAction.java b/core/src/main/java/org/elasticsearch/rest/action/get/RestHeadAction.java
index 9c98c98012c..747e62ea381 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/get/RestHeadAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/get/RestHeadAction.java
@@ -23,6 +23,7 @@ import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -66,9 +67,9 @@ public class RestHeadAction extends BaseRestHandler {
             @Override
             public RestResponse buildResponse(GetResponse response) {
                 if (!response.isExists()) {
-                    return new BytesRestResponse(NOT_FOUND);
+                    return new BytesRestResponse(NOT_FOUND, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
                 } else {
-                    return new BytesRestResponse(OK);
+                    return new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
                 }
             }
         });
diff --git a/core/src/main/java/org/elasticsearch/rest/action/main/RestMainAction.java b/core/src/main/java/org/elasticsearch/rest/action/main/RestMainAction.java
index b9e57b4965b..e57bb790e5e 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/main/RestMainAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/main/RestMainAction.java
@@ -25,7 +25,6 @@ import org.elasticsearch.action.main.MainResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
@@ -66,7 +65,7 @@ public class RestMainAction extends BaseRestHandler {
     static BytesRestResponse convertMainResponse(MainResponse response, RestRequest request, XContentBuilder builder) throws IOException {
         RestStatus status = response.isAvailable() ? RestStatus.OK : RestStatus.SERVICE_UNAVAILABLE;
         if (request.method() == RestRequest.Method.HEAD) {
-            return new BytesRestResponse(status);
+            return new BytesRestResponse(status, builder);
         }
 
         // Default to pretty printing, but allow ?pretty=false to disable
diff --git a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
index 28fc315d3e1..9f002afd2dc 100644
--- a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
+++ b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
@@ -22,6 +22,7 @@ import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.bytes.ByteBufferBytesReference;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -66,7 +67,8 @@ public class HttpServerTests extends ESTestCase {
         HttpServerTransport httpServerTransport = new TestHttpServerTransport();
         RestController restController = new RestController(settings);
         restController.registerHandler(RestRequest.Method.GET, "/",
-            (request, channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.OK)));
+            (request, channel) -> channel.sendResponse(
+                new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)));
         restController.registerHandler(RestRequest.Method.GET, "/error", (request, channel) -> {
                 throw new IllegalArgumentException("test error");
             });
diff --git a/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java b/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java
new file mode 100644
index 00000000000..0ad40f84cf7
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.rest.action.main;
+
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.rest.client.http.HttpResponse;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
+public class RestMainActionIT extends ESIntegTestCase {
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder()
+            .put(super.nodeSettings(nodeOrdinal))
+            .put(NetworkModule.HTTP_ENABLED.getKey(), true)
+            .build();
+    }
+
+    public void testHeadRequest() throws IOException {
+        final HttpResponse response = httpClient().method("HEAD").path("/").execute();
+        assertThat(response.getStatusCode(), equalTo(200));
+        assertThat(response.getBody(), nullValue());
+    }
+
+    public void testGetRequest() throws IOException {
+        final HttpResponse response = httpClient().path("/").execute();
+        assertThat(response.getStatusCode(), equalTo(200));
+        assertThat(response.getBody(), containsString("cluster_name"));
+    }
+}

From 25289344118e0611e0ee99040ce0bbbbf2925a28 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 9 May 2016 09:23:23 -0400
Subject: [PATCH 0127/1311] CONSOLE is the new AUTOSENSE

This makes the test generation support both while we move from
`// AUTOSENSE` to `// CONSOLE`.

Will bother #18160
---
 .../main/groovy/org/elasticsearch/gradle/SnippetsTask.groovy   | 3 ++-
 docs/reference/docs/index_.asciidoc                            | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/SnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/SnippetsTask.groovy
index 73160cde7f1..a2e6502690f 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/SnippetsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/SnippetsTask.groovy
@@ -110,7 +110,8 @@ public class SnippetsTask extends DefaultTask {
                     lastLanguageLine = lineNumber
                     return
                 }
-                if (line ==~ /\/\/ AUTOSENSE\s*/) {
+                if (line ==~ /\/\/ AUTOSENSE\s*/
+                        || line ==~ /\/\/ CONSOLE\s*/) {
                     if (snippet == null) {
                         throw new InvalidUserDataException("AUTOSENSE not " +
                             "paired with a snippet at $file:$lineNumber")
diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc
index 0acb148e82a..93d04982d6f 100644
--- a/docs/reference/docs/index_.asciidoc
+++ b/docs/reference/docs/index_.asciidoc
@@ -14,7 +14,7 @@ PUT twitter/tweet/1
     "message" : "trying out Elasticsearch"
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The result of the above index operation is:
 

From 3f594089c2f091c8fd9cc809b44e2f56072d371e Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Mon, 9 May 2016 15:42:23 +0200
Subject: [PATCH 0128/1311] Renamed all AUTOSENSE snippets to CONSOLE (#18210)

---
 docs/README.asciidoc                          |  8 ++--
 docs/build.gradle                             |  2 +-
 docs/plugins/analysis-icu.asciidoc            | 16 ++++----
 docs/plugins/analysis-kuromoji.asciidoc       | 14 +++----
 docs/plugins/analysis-phonetic.asciidoc       |  2 +-
 docs/plugins/delete-by-query.asciidoc         |  4 +-
 docs/plugins/lang-javascript.asciidoc         |  6 +--
 docs/plugins/lang-python.asciidoc             |  6 +--
 docs/plugins/mapper-attachments.asciidoc      | 22 +++++------
 docs/plugins/mapper-murmur3.asciidoc          |  4 +-
 docs/plugins/mapper-size.asciidoc             |  4 +-
 docs/plugins/repository-azure.asciidoc        |  2 +-
 docs/plugins/repository-s3.asciidoc           |  2 +-
 docs/plugins/store-smb.asciidoc               |  2 +-
 .../analyzers/pattern-analyzer.asciidoc       |  6 +--
 docs/reference/cluster/tasks.asciidoc         | 14 +++----
 docs/reference/docs/index_.asciidoc           | 20 +++++-----
 docs/reference/docs/reindex.asciidoc          | 38 +++++++++----------
 docs/reference/docs/update-by-query.asciidoc  | 28 +++++++-------
 .../index-modules/allocation/delayed.asciidoc |  6 +--
 .../allocation/filtering.asciidoc             |  8 ++--
 .../allocation/prioritization.asciidoc        |  4 +-
 docs/reference/indices/analyze.asciidoc       |  2 +-
 docs/reference/indices/flush.asciidoc         | 10 ++---
 docs/reference/indices/put-mapping.asciidoc   |  8 ++--
 docs/reference/indices/templates.asciidoc     |  2 +-
 docs/reference/ingest.asciidoc                |  2 +-
 docs/reference/ingest/ingest-node.asciidoc    | 10 ++---
 docs/reference/mapping.asciidoc               |  2 +-
 .../mapping/dynamic-mapping.asciidoc          |  4 +-
 .../mapping/dynamic/default-mapping.asciidoc  |  4 +-
 .../mapping/dynamic/field-mapping.asciidoc    |  8 ++--
 .../mapping/dynamic/templates.asciidoc        | 10 ++---
 .../mapping/fields/all-field.asciidoc         | 16 ++++----
 .../mapping/fields/field-names-field.asciidoc |  2 +-
 .../mapping/fields/id-field.asciidoc          |  2 +-
 .../mapping/fields/index-field.asciidoc       |  2 +-
 .../mapping/fields/meta-field.asciidoc        |  2 +-
 .../mapping/fields/parent-field.asciidoc      |  8 ++--
 .../mapping/fields/routing-field.asciidoc     |  8 ++--
 .../mapping/fields/source-field.asciidoc      |  4 +-
 .../mapping/fields/timestamp-field.asciidoc   |  4 +-
 .../mapping/fields/ttl-field.asciidoc         |  4 +-
 .../mapping/fields/type-field.asciidoc        |  2 +-
 .../mapping/fields/uid-field.asciidoc         |  2 +-
 .../mapping/params/analyzer.asciidoc          |  4 +-
 docs/reference/mapping/params/boost.asciidoc  |  6 +--
 docs/reference/mapping/params/coerce.asciidoc |  4 +-
 .../reference/mapping/params/copy-to.asciidoc |  2 +-
 .../mapping/params/doc-values.asciidoc        |  2 +-
 .../reference/mapping/params/dynamic.asciidoc |  4 +-
 .../reference/mapping/params/enabled.asciidoc |  4 +-
 .../mapping/params/fielddata.asciidoc         |  2 +-
 docs/reference/mapping/params/format.asciidoc |  2 +-
 .../mapping/params/geohash-precision.asciidoc |  2 +-
 .../mapping/params/geohash-prefix.asciidoc    |  2 +-
 .../reference/mapping/params/geohash.asciidoc |  2 +-
 .../mapping/params/ignore-above.asciidoc      |  2 +-
 .../mapping/params/ignore-malformed.asciidoc  |  4 +-
 .../mapping/params/include-in-all.asciidoc    |  4 +-
 .../mapping/params/index-options.asciidoc     |  2 +-
 .../reference/mapping/params/lat-lon.asciidoc |  2 +-
 .../mapping/params/multi-fields.asciidoc      |  4 +-
 docs/reference/mapping/params/norms.asciidoc  |  2 +-
 .../mapping/params/null-value.asciidoc        |  2 +-
 .../params/position-increment-gap.asciidoc    |  4 +-
 .../mapping/params/properties.asciidoc        |  4 +-
 .../mapping/params/search-analyzer.asciidoc   |  2 +-
 .../mapping/params/similarity.asciidoc        |  2 +-
 docs/reference/mapping/params/store.asciidoc  |  2 +-
 .../mapping/params/term-vector.asciidoc       |  2 +-
 docs/reference/mapping/types/array.asciidoc   |  2 +-
 docs/reference/mapping/types/boolean.asciidoc |  4 +-
 docs/reference/mapping/types/date.asciidoc    |  4 +-
 .../mapping/types/geo-point.asciidoc          |  2 +-
 docs/reference/mapping/types/ip.asciidoc      |  2 +-
 docs/reference/mapping/types/keyword.asciidoc |  2 +-
 docs/reference/mapping/types/nested.asciidoc  |  6 +--
 docs/reference/mapping/types/numeric.asciidoc |  2 +-
 docs/reference/mapping/types/object.asciidoc  |  4 +-
 docs/reference/mapping/types/text.asciidoc    |  2 +-
 .../mapping/types/token-count.asciidoc        |  2 +-
 .../migration/migrate_5_0/mapping.asciidoc    |  2 +-
 .../cluster/allocation_filtering.asciidoc     |  4 +-
 .../modules/cluster/disk_allocator.asciidoc   |  2 +-
 docs/reference/modules/node.asciidoc          |  2 +-
 .../modules/scripting/fields.asciidoc         |  6 +--
 .../modules/scripting/painless.asciidoc       | 16 ++++----
 .../modules/scripting/using.asciidoc          | 10 ++---
 docs/reference/modules/snapshots.asciidoc     | 38 +++++++++----------
 docs/reference/query-dsl/bool-query.asciidoc  |  8 ++--
 .../query-dsl/function-score-query.asciidoc   |  2 +-
 docs/reference/query-dsl/term-query.asciidoc  |  8 ++--
 docs/reference/setup/cluster_restart.asciidoc | 10 ++---
 docs/reference/setup/rolling_upgrade.asciidoc | 12 +++---
 95 files changed, 281 insertions(+), 281 deletions(-)

diff --git a/docs/README.asciidoc b/docs/README.asciidoc
index 7317ce71a82..da07009d13d 100644
--- a/docs/README.asciidoc
+++ b/docs/README.asciidoc
@@ -3,13 +3,13 @@ Elasticsearch documentation build process.
 
 See: https://github.com/elastic/docs
 
-Snippets marked with `// AUTOSENSE` are automatically annotated with "VIEW IN
+Snippets marked with `// CONSOLE` are automatically annotated with "VIEW IN
 SENSE" in the documentation and are automatically tested by the command
-`gradle :docs:check`. By default `// AUTOSENSE` snippet runs as its own isolated
+`gradle :docs:check`. By default `// CONSOLE` snippet runs as its own isolated
 test. You can manipulate the test execution in the following ways:
 
 * `// TEST`: Explicitly marks a snippet as a test. Snippets marked this way
-are tests even if they don't have `// AUTOSENSE`.
+are tests even if they don't have `// CONSOLE`.
   * `// TEST[s/foo/bar/]`: Replace `foo` with `bar` in the test. This should be
   used sparingly because it makes the test "lie". Sometimes, though, you can use
   it to make the tests more clear.
@@ -22,7 +22,7 @@ are tests even if they don't have `// AUTOSENSE`.
   tell the story of some use case because it merges the snippets (and thus the
   use case) into one big test.
   * `// TEST[skip:reason]`: Skip this test. Replace `reason` with the actual
-  reason to skip the test. Snippets without `// TEST` or `// AUTOSENSE` aren't
+  reason to skip the test. Snippets without `// TEST` or `// CONSOLE` aren't
   considered tests anyway but this is useful for explicitly documenting the
   reason why the test shouldn't be run.
   * `// TEST[setup:name]`: Run some setup code before running the snippet. This
diff --git a/docs/build.gradle b/docs/build.gradle
index be4b10b1010..3858ec1f9fb 100644
--- a/docs/build.gradle
+++ b/docs/build.gradle
@@ -31,7 +31,7 @@ task listSnippets(type: SnippetsTask) {
 
 task listAutoSenseCandidates(type: SnippetsTask) {
   group 'Docs'
-  description 'List snippets that probably should be marked // AUTOSENSE'
+  description 'List snippets that probably should be marked // CONSOLE'
   perSnippet {
     if (
              it.autoSense    // Already marked, nothing to do
diff --git a/docs/plugins/analysis-icu.asciidoc b/docs/plugins/analysis-icu.asciidoc
index 03d0dc0296c..19b4fdb07cd 100644
--- a/docs/plugins/analysis-icu.asciidoc
+++ b/docs/plugins/analysis-icu.asciidoc
@@ -81,7 +81,7 @@ PUT icu_sample
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> Uses the default `nfkc_cf` normalization.
 <2> Uses the customized `nfd_normalizer` token filter, which is set to use `nfc` normalization with decomposition.
@@ -113,7 +113,7 @@ PUT icu_sample
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 ===== Rules customization
 
@@ -163,7 +163,7 @@ PUT icu_sample
 
 POST icu_sample/_analyze?analyzer=my_analyzer&text=Elasticsearch. Wow!
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The above `analyze` request returns the following:
 
@@ -230,7 +230,7 @@ PUT icu_sample
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> Uses the default `nfkc_cf` normalization.
 <2> Uses the customized `nfc_normalizer` token filter, which is set to use `nfc` normalization.
@@ -264,7 +264,7 @@ PUT icu_sample
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The ICU folding token filter already does Unicode normalization, so there is
 no need to use Normalize character or token filter as well.
@@ -305,7 +305,7 @@ PUT icu_sample
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 [[analysis-icu-collation]]
 ==== ICU Collation Token Filter
@@ -370,7 +370,7 @@ GET _search <3>
 }
 
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> The `name` field uses the `standard` analyzer, and so support full text queries.
 <2> The `name.sort` field uses the `keyword` analyzer to preserve the name as
@@ -494,7 +494,7 @@ GET icu_sample/_analyze?analyzer=latin
 }
 
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> This transforms transliterates characters to Latin, and separates accents
     from their base characters, removes the accents, and then puts the
diff --git a/docs/plugins/analysis-kuromoji.asciidoc b/docs/plugins/analysis-kuromoji.asciidoc
index ff42287d03f..2d679de07f4 100644
--- a/docs/plugins/analysis-kuromoji.asciidoc
+++ b/docs/plugins/analysis-kuromoji.asciidoc
@@ -174,7 +174,7 @@ PUT kuromoji_sample
 
 POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=�京スカイツリー
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The above `analyze` request returns the following:
 
@@ -226,7 +226,7 @@ PUT kuromoji_sample
 
 POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=飲�
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 [source,text]
 --------------------------------------------------
@@ -285,7 +285,7 @@ PUT kuromoji_sample
 POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=寿�������
 
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 [source,text]
 --------------------------------------------------
@@ -359,7 +359,7 @@ POST kuromoji_sample/_analyze?analyzer=katakana_analyzer&text=寿� <1>
 POST kuromoji_sample/_analyze?analyzer=romaji_analyzer&text=寿� <2>
 
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> Returns `スシ`.
 <2> Returns `sushi`.
@@ -410,7 +410,7 @@ POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=コピー <1>
 POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=サー�ー <2>
 
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> Returns `コピー`.
 <2> Return `サー�`.
@@ -456,7 +456,7 @@ PUT kuromoji_sample
 
 POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=ストップ�消�る
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The above request returns:
 
@@ -503,7 +503,7 @@ PUT kuromoji_sample
 POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=一〇〇〇
 
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 [source,text]
 --------------------------------------------------
diff --git a/docs/plugins/analysis-phonetic.asciidoc b/docs/plugins/analysis-phonetic.asciidoc
index f4a9684519d..a0055ef5d69 100644
--- a/docs/plugins/analysis-phonetic.asciidoc
+++ b/docs/plugins/analysis-phonetic.asciidoc
@@ -81,7 +81,7 @@ PUT phonetic_sample
 
 POST phonetic_sample/_analyze?analyzer=my_analyzer&text=Joe Bloggs <1>
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> Returns: `J`, `joe`, `BLKS`, `bloggs`
 
diff --git a/docs/plugins/delete-by-query.asciidoc b/docs/plugins/delete-by-query.asciidoc
index 643dbfed8b3..aebc0dd7433 100644
--- a/docs/plugins/delete-by-query.asciidoc
+++ b/docs/plugins/delete-by-query.asciidoc
@@ -58,7 +58,7 @@ a parameter:
 --------------------------------------------------
 DELETE /twitter/tweet/_query?q=user:kimchy
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 or using the {ref}/query-dsl.html[Query DSL] defined within the request body:
 
@@ -73,7 +73,7 @@ DELETE /twitter/tweet/_query
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> The query must be passed as a value to the `query` key, in the same way as
 the {ref}/search-search.html[search api].
diff --git a/docs/plugins/lang-javascript.asciidoc b/docs/plugins/lang-javascript.asciidoc
index 856e4bf4f03..3877fb38d67 100644
--- a/docs/plugins/lang-javascript.asciidoc
+++ b/docs/plugins/lang-javascript.asciidoc
@@ -82,7 +82,7 @@ GET test/_search
   }
 }
 ----
-// AUTOSENSE
+// CONSOLE
 
 [[lang-javascript-stored]]
 [float]
@@ -131,7 +131,7 @@ GET test/_search
 }
 
 ----
-// AUTOSENSE
+// CONSOLE
 
 <1> We store the script under the id `my_script`.
 <2> The function score query retrieves the script with id `my_script`.
@@ -187,7 +187,7 @@ GET test/_search
 }
 
 ----
-// AUTOSENSE
+// CONSOLE
 
 <1> The function score query retrieves the script with filename `my_script.javascript`.
 
diff --git a/docs/plugins/lang-python.asciidoc b/docs/plugins/lang-python.asciidoc
index 49494828b13..070ed3eb6a9 100644
--- a/docs/plugins/lang-python.asciidoc
+++ b/docs/plugins/lang-python.asciidoc
@@ -81,7 +81,7 @@ GET test/_search
   }
 }
 ----
-// AUTOSENSE
+// CONSOLE
 
 [[lang-python-stored]]
 [float]
@@ -130,7 +130,7 @@ GET test/_search
 }
 
 ----
-// AUTOSENSE
+// CONSOLE
 
 <1> We store the script under the id `my_script`.
 <2> The function score query retrieves the script with id `my_script`.
@@ -186,7 +186,7 @@ GET test/_search
 }
 
 ----
-// AUTOSENSE
+// CONSOLE
 
 <1> The function score query retrieves the script with filename `my_script.py`.
 
diff --git a/docs/plugins/mapper-attachments.asciidoc b/docs/plugins/mapper-attachments.asciidoc
index 1da99ee3531..70d760e00b8 100644
--- a/docs/plugins/mapper-attachments.asciidoc
+++ b/docs/plugins/mapper-attachments.asciidoc
@@ -51,7 +51,7 @@ POST /trying-out-mapper-attachments
         "cv": { "type": "attachment" }
 }}}}
 --------------------------
-// AUTOSENSE
+// CONSOLE
 
 Index a new document populated with a `base64`-encoded attachment:
 
@@ -62,7 +62,7 @@ POST /trying-out-mapper-attachments/person/1
   "cv": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0="
 }
 --------------------------
-// AUTOSENSE
+// CONSOLE
 
 Search for the document using words in the attachment:
 
@@ -75,7 +75,7 @@ POST /trying-out-mapper-attachments/person/_search
       "query": "ipsum"
 }}}
 --------------------------
-// AUTOSENSE
+// CONSOLE
 
 If you get a hit for your indexed document, the plugin should be installed and working.
 
@@ -96,7 +96,7 @@ PUT /test/person/_mapping
     }
 }
 --------------------------
-// AUTOSENSE
+// CONSOLE
 
 In this case, the JSON to index can be:
 
@@ -107,7 +107,7 @@ PUT /test/person/1
     "my_attachment" : "... base64 encoded attachment ..."
 }
 --------------------------
-// AUTOSENSE
+// CONSOLE
 
 Or it is possible to use more elaborated JSON if content type, resource name or language need to be set explicitly:
 
@@ -123,7 +123,7 @@ PUT /test/person/1
     }
 }
 --------------------------
-// AUTOSENSE
+// CONSOLE
 
 The `attachment` type not only indexes the content of the doc in `content` sub field, but also automatically adds meta
 data on the attachment as well (when available).
@@ -167,7 +167,7 @@ PUT /test/person/_mapping
     }
 }
 --------------------------
-// AUTOSENSE
+// CONSOLE
 
 In the above example, the actual content indexed is mapped under `fields` name `content`, and we decide not to index it, so
 it will only be available in the `_all` field. The other fields map to their respective metadata names, but there is no
@@ -201,7 +201,7 @@ PUT /test/person/_mapping
   }
 }
 --------------------------
-// AUTOSENSE
+// CONSOLE
 
 In this example, the extracted content will be copy as well to `copy` field.
 
@@ -244,7 +244,7 @@ GET /test/person/_search
   }
 }
 --------------------------
-// AUTOSENSE
+// CONSOLE
 
 Will give you:
 
@@ -296,7 +296,7 @@ PUT /test/person/1
     }
 }
 --------------------------
-// AUTOSENSE
+// CONSOLE
 
 [[mapper-attachments-error-handling]]
 ==== Metadata parsing error handling
@@ -372,7 +372,7 @@ GET /test/person/_search
   }
 }
 --------------------------
-// AUTOSENSE
+// CONSOLE
 
 It gives back:
 
diff --git a/docs/plugins/mapper-murmur3.asciidoc b/docs/plugins/mapper-murmur3.asciidoc
index 63fb47b294b..28b7a2387ef 100644
--- a/docs/plugins/mapper-murmur3.asciidoc
+++ b/docs/plugins/mapper-murmur3.asciidoc
@@ -58,7 +58,7 @@ PUT my_index
   }
 }
 --------------------------
-// AUTOSENSE
+// CONSOLE
 
 Such a mapping would allow to refer to `my_field.hash` in order to get hashes
 of the values of the `my_field` field. This is only useful in order to run
@@ -88,7 +88,7 @@ GET my_index/_search
   }
 }
 --------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> Counting unique values on the `my_field.hash` field
 
diff --git a/docs/plugins/mapper-size.asciidoc b/docs/plugins/mapper-size.asciidoc
index 708c8bbd8f7..4111a15ef6c 100644
--- a/docs/plugins/mapper-size.asciidoc
+++ b/docs/plugins/mapper-size.asciidoc
@@ -50,7 +50,7 @@ PUT my_index
   }
 }
 --------------------------
-// AUTOSENSE
+// CONSOLE
 
 The value of the `_size` field is accessible in queries, aggregations, scripts,
 and when sorting:
@@ -99,7 +99,7 @@ GET my_index/_search
   }
 }
 --------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> Querying on the `_size` field
 <2> Aggregating on the `_size` field
diff --git a/docs/plugins/repository-azure.asciidoc b/docs/plugins/repository-azure.asciidoc
index 0ad1106e8c2..dd2ae068eb6 100644
--- a/docs/plugins/repository-azure.asciidoc
+++ b/docs/plugins/repository-azure.asciidoc
@@ -167,7 +167,7 @@ PUT _snapshot/my_backup4
     }
 }
 ----
-// AUTOSENSE
+// CONSOLE
 
 Example using Java:
 
diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc
index 8780b3a710e..61ae79180dd 100644
--- a/docs/plugins/repository-s3.asciidoc
+++ b/docs/plugins/repository-s3.asciidoc
@@ -148,7 +148,7 @@ PUT _snapshot/my_s3_repository
   }
 }
 ----
-// AUTOSENSE
+// CONSOLE
 
 The following settings are supported:
 
diff --git a/docs/plugins/store-smb.asciidoc b/docs/plugins/store-smb.asciidoc
index 1c0b5f4ec18..cf0094173d1 100644
--- a/docs/plugins/store-smb.asciidoc
+++ b/docs/plugins/store-smb.asciidoc
@@ -77,4 +77,4 @@ PUT my_index
    }
 }
 ----
-// AUTOSENSE
+// CONSOLE
diff --git a/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc b/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc
index 2aaa0d6bb16..453967a46d7 100644
--- a/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc
@@ -52,7 +52,7 @@ GET _cluster/health?wait_for_status=yellow
 GET test/_analyze?analyzer=whitespace&text=foo,bar baz
 # "foo,bar", "baz"
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 [float]
 ===== Non-word character tokenizer
@@ -81,7 +81,7 @@ GET test/_analyze?analyzer=nonword&text=foo,bar baz
 GET test/_analyze?analyzer=nonword&text=type_1-type_4
 # "type_1","type_4"
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 
 [float]
@@ -108,7 +108,7 @@ GET _cluster/health?wait_for_status=yellow
 GET test/_analyze?analyzer=camel&text=MooseX::FTPClass2_beta
 # "moose","x","ftp","class","2","beta"
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The regex above is easier to understand as:
 
diff --git a/docs/reference/cluster/tasks.asciidoc b/docs/reference/cluster/tasks.asciidoc
index 0ac888c953f..b53d8de93a1 100644
--- a/docs/reference/cluster/tasks.asciidoc
+++ b/docs/reference/cluster/tasks.asciidoc
@@ -15,7 +15,7 @@ GET _tasks <1>
 GET _tasks?nodes=nodeId1,nodeId2 <2>
 GET _tasks?nodes=nodeId1,nodeId2&actions=cluster:* <3>
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> Retrieves all tasks currently running on all nodes in the cluster.
 <2> Retrieves all tasks running on nodes `nodeId1` and `nodeId2`.  See <<cluster-nodes>> for more info about how to select individual nodes.
@@ -66,7 +66,7 @@ tasks using the following two commands:
 GET _tasks/taskId:1
 GET _tasks?parent_task_id=parentTaskId:1
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The task API can be also used to wait for completion of a particular task. The following call will
 block for 10 seconds or until the task with id `oTUltX4IQMOUUVeiohTt8A:12345` is completed.
@@ -75,7 +75,7 @@ block for 10 seconds or until the task with id `oTUltX4IQMOUUVeiohTt8A:12345` is
 --------------------------------------------------
 GET _tasks/oTUltX4IQMOUUVeiohTt8A:12345?wait_for_completion=true&timeout=10s
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Tasks can be also listed using _cat version of the list tasks command, which accepts the same arguments
 as the standard list tasks command.
@@ -84,7 +84,7 @@ as the standard list tasks command.
 --------------------------------------------------
 GET _cat/tasks
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 [float]
 === Task Cancellation
@@ -95,7 +95,7 @@ If a long-running task supports cancellation, it can be cancelled by the followi
 --------------------------------------------------
 POST _tasks/taskId:1/_cancel
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The task cancellation command supports the same task selection parameters as the list tasks command, so multiple tasks
 can be cancelled at the same time. For example, the following command will cancel all reindex tasks running on the
@@ -105,7 +105,7 @@ nodes `nodeId1` and `nodeId2`.
 --------------------------------------------------
 POST _tasks/_cancel?node_id=nodeId1,nodeId2&actions=*reindex
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 
 [float]
@@ -118,4 +118,4 @@ The following command will change the grouping to parent tasks:
 --------------------------------------------------
 GET _tasks?group_by=parents
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc
index 93d04982d6f..eb1e45d6160 100644
--- a/docs/reference/docs/index_.asciidoc
+++ b/docs/reference/docs/index_.asciidoc
@@ -98,7 +98,7 @@ PUT twitter/tweet/1?version=2
     "message" : "elasticsearch now has versioning support, double cool!"
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[catch: conflict]
 
 *NOTE:* versioning is completely real time, and is not affected by the
@@ -173,7 +173,7 @@ PUT twitter/tweet/1?op_type=create
     "message" : "trying out Elasticsearch"
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Another option to specify `create` is to use the following uri:
 
@@ -186,7 +186,7 @@ PUT twitter/tweet/1/_create
     "message" : "trying out Elasticsearch"
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 [float]
 === Automatic ID Generation
@@ -205,7 +205,7 @@ POST twitter/tweet/
     "message" : "trying out Elasticsearch"
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The result of the above index operation is:
 
@@ -244,7 +244,7 @@ POST twitter/tweet?routing=kimchy
     "message" : "trying out Elasticsearch"
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 In the example above, the "tweet" document is routed to a shard based on
 the `routing` parameter provided: "kimchy".
@@ -282,7 +282,7 @@ PUT blogs/blog_tag/1122?parent=1111
     "tag" : "something"
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 When indexing a child document, the routing value is automatically set
 to be the same as its parent, unless the routing value is explicitly
@@ -306,7 +306,7 @@ PUT twitter/tweet/1?timestamp=2009-11-15T14:12:12
     "message" : "trying out Elasticsearch"
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 If the `timestamp` value is not provided externally or in the `_source`,
 the `timestamp` will be automatically set to the date the document was
@@ -337,7 +337,7 @@ PUT twitter/tweet/1?ttl=86400000ms
     "message": "Trying out elasticsearch, so far so good?"
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 [source,js]
 --------------------------------------------------
@@ -347,7 +347,7 @@ PUT twitter/tweet/1?ttl=1d
     "message": "Trying out elasticsearch, so far so good?"
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 More information can be found on the
 <<mapping-ttl-field,_ttl mapping page>>.
@@ -430,4 +430,4 @@ PUT twitter/tweet/1?timeout=5m
     "message" : "trying out Elasticsearch"
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index 513281c07e3..00c5d34304e 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -18,7 +18,7 @@ POST _reindex
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[setup:big_twitter]
 
 That will return something like this:
@@ -64,7 +64,7 @@ POST _reindex
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[setup:twitter]
 
 Setting `version_type` to `external` will cause Elasticsearch to preserve the
@@ -85,7 +85,7 @@ POST _reindex
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[setup:twitter]
 
 Settings `op_type` to `create` will cause `_reindex` to only create missing
@@ -105,7 +105,7 @@ POST _reindex
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[setup:twitter]
 
 By default version conflicts abort the `_reindex` process but you can just
@@ -125,7 +125,7 @@ POST _reindex
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[setup:twitter]
 
 You can limit the documents by adding a type to the `source` or by adding a
@@ -149,7 +149,7 @@ POST _reindex
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[setup:twitter]
 
 `index` and `type` in `source` can both be lists, allowing you to copy from
@@ -173,7 +173,7 @@ POST _reindex
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[s/^/PUT twitter\nPUT blog\n/]
 
 It's also possible to limit the number of processed documents by setting
@@ -193,7 +193,7 @@ POST _reindex
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[setup:twitter]
 
 If you want a particular set of documents from the twitter index you'll
@@ -215,7 +215,7 @@ POST _reindex
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[setup:twitter]
 
 Like `_update_by_query`, `_reindex` supports a script that modifies the
@@ -238,7 +238,7 @@ POST _reindex
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[setup:twitter]
 
 Think of the possibilities! Just be careful! With great power.... You can
@@ -298,7 +298,7 @@ POST _reindex
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[s/^/PUT source\n/]
 
 By default `_reindex` uses scroll batches of 100. You can change the
@@ -318,7 +318,7 @@ POST _reindex
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[s/^/PUT source\n/]
 
 Reindex can also use the <<ingest>> feature by specifying a
@@ -337,7 +337,7 @@ POST _reindex
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[s/^/PUT source\n/]
 
 [float]
@@ -437,7 +437,7 @@ While Reindex is running you can fetch their status using the
 --------------------------------------------------
 GET _tasks/?pretty&detailed=true&actions=*reindex
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The responses looks like:
 
@@ -496,7 +496,7 @@ Any Reindex can be canceled using the <<tasks,Task Cancel API>>:
 --------------------------------------------------
 POST _tasks/taskid:1/_cancel
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The `task_id` can be found using the tasks API above.
 
@@ -515,7 +515,7 @@ the `_rethrottle` API:
 --------------------------------------------------
 POST _reindex/taskid:1/_rethrottle?requests_per_second=unlimited
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The `task_id` can be found using the tasks API above.
 
@@ -540,7 +540,7 @@ POST test/test/1?refresh&pretty
   "flag": "foo"
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 But you don't like the name `flag` and want to replace it with `tag`.
 `_reindex` can create the other index for you:
@@ -560,7 +560,7 @@ POST _reindex?pretty
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[continued]
 
 Now you can get the new document:
@@ -569,7 +569,7 @@ Now you can get the new document:
 --------------------------------------------------
 GET test2/test/1?pretty
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[continued]
 
 and it'll look like:
diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc
index cb0c806f3a4..2de592f59a1 100644
--- a/docs/reference/docs/update-by-query.asciidoc
+++ b/docs/reference/docs/update-by-query.asciidoc
@@ -12,7 +12,7 @@ mapping change. Here is the API:
 --------------------------------------------------
 POST twitter/_update_by_query?conflicts=proceed
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[setup:big_twitter]
 
 That will return something like this:
@@ -64,7 +64,7 @@ will only update `tweet`s from the `twitter` index:
 --------------------------------------------------
 POST twitter/tweet/_update_by_query?conflicts=proceed
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[setup:twitter]
 
 You can also limit `_update_by_query` using the
@@ -82,7 +82,7 @@ POST twitter/_update_by_query?conflicts=proceed
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[setup:twitter]
 
 <1> The query must be passed as a value to the `query` key, in the same
@@ -109,7 +109,7 @@ POST twitter/_update_by_query
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[setup:twitter]
 
 Just as in <<docs-update,Update API>> you can set `ctx.op = "noop"` if
@@ -133,7 +133,7 @@ types at once, just like the search API:
 --------------------------------------------------
 POST twitter,blog/tweet,post/_update_by_query
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[s/^/PUT twitter\nPUT blog\n/]
 
 If you provide `routing` then the routing is copied to the scroll query,
@@ -143,7 +143,7 @@ limiting the process to the shards that match that routing value:
 --------------------------------------------------
 POST twitter/_update_by_query?routing=1
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[setup:twitter]
 
 By default `_update_by_query` uses scroll batches of 100. You can change the
@@ -153,7 +153,7 @@ batch size with the `scroll_size` URL parameter:
 --------------------------------------------------
 POST twitter/_update_by_query?scroll_size=1000
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[setup:twitter]
 
 `_update_by_query` can also use the <<ingest>> feature by
@@ -173,7 +173,7 @@ PUT _ingest/pipeline/set-foo
 }
 POST twitter/_update_by_query?pipeline=set-foo
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[setup:twitter]
 
 [float]
@@ -268,7 +268,7 @@ While Update By Query is running you can fetch their status using the
 --------------------------------------------------
 GET _tasks/?pretty&detailed=true&action=*byquery
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The responses looks like:
 
@@ -327,7 +327,7 @@ Any Update By Query can be canceled using the <<tasks,Task Cancel API>>:
 --------------------------------------------------
 POST _tasks/taskid:1/_cancel
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The `task_id` can be found using the tasks API above.
 
@@ -346,7 +346,7 @@ using the `_rethrottle` API:
 --------------------------------------------------
 POST _update_by_query/taskid:1/_rethrottle?requests_per_second=unlimited
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The `task_id` can be found using the tasks API above.
 
@@ -396,7 +396,7 @@ PUT test/_mapping/test   <2>
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> This means that new fields won't be indexed, just stored in `_source`.
 
@@ -416,7 +416,7 @@ POST test/_search?filter_path=hits.total
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[continued]
 
 [source,js]
@@ -443,7 +443,7 @@ POST test/_search?filter_path=hits.total
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[continued]
 
 [source,js]
diff --git a/docs/reference/index-modules/allocation/delayed.asciidoc b/docs/reference/index-modules/allocation/delayed.asciidoc
index 6bedba95586..907d927fd68 100644
--- a/docs/reference/index-modules/allocation/delayed.asciidoc
+++ b/docs/reference/index-modules/allocation/delayed.asciidoc
@@ -45,7 +45,7 @@ PUT _all/_settings
   }
 }
 ------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[s/^/PUT test\n/]
 
 With delayed allocation enabled, the above scenario changes to look like this:
@@ -83,7 +83,7 @@ can be viewed with the <<cluster-health,cluster health API>>:
 ------------------------------
 GET _cluster/health <1>
 ------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> This request will return a `delayed_unassigned_shards` value.
 
 ==== Removing a node permanently
@@ -101,7 +101,7 @@ PUT _all/_settings
   }
 }
 ------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[s/^/PUT test\n/]
 
 You can reset the timeout as soon as the missing shards have started to recover.
diff --git a/docs/reference/index-modules/allocation/filtering.asciidoc b/docs/reference/index-modules/allocation/filtering.asciidoc
index f5b73a9f78e..be45cd2a1ac 100644
--- a/docs/reference/index-modules/allocation/filtering.asciidoc
+++ b/docs/reference/index-modules/allocation/filtering.asciidoc
@@ -31,7 +31,7 @@ PUT test/_settings
   "index.routing.allocation.include.size": "big,medium"
 }
 ------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[s/^/PUT test\n/]
 
 Alternatively, we can move the index `test` away from the `small` nodes with
@@ -44,7 +44,7 @@ PUT test/_settings
   "index.routing.allocation.exclude.size": "small"
 }
 ------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[s/^/PUT test\n/]
 
 Multiple rules can be specified, in which case all conditions must be
@@ -59,7 +59,7 @@ PUT test/_settings
   "index.routing.allocation.include.rack": "rack1"
 }
 ------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[s/^/PUT test\n/]
 
 NOTE: If some conditions cannot be satisfied then shards will not be moved.
@@ -100,5 +100,5 @@ PUT test/_settings
   "index.routing.allocation.include._ip": "192.168.2.*"
 }
 ------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[skip:indexes don't assign]
diff --git a/docs/reference/index-modules/allocation/prioritization.asciidoc b/docs/reference/index-modules/allocation/prioritization.asciidoc
index 149e43fd134..eecc98abcb0 100644
--- a/docs/reference/index-modules/allocation/prioritization.asciidoc
+++ b/docs/reference/index-modules/allocation/prioritization.asciidoc
@@ -33,7 +33,7 @@ PUT index_4
   }
 }
 ------------------------------
-// AUTOSENSE
+// CONSOLE
 
 In the above example:
 
@@ -52,5 +52,5 @@ PUT index_4/_settings
   "index.priority": 1
 }
 ------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[continued]
diff --git a/docs/reference/indices/analyze.asciidoc b/docs/reference/indices/analyze.asciidoc
index 3597ccc51e5..d9016c9e8ec 100644
--- a/docs/reference/indices/analyze.asciidoc
+++ b/docs/reference/indices/analyze.asciidoc
@@ -121,7 +121,7 @@ GET _analyze
   "attributes" : ["keyword"] <1>
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> Set "keyword" to output "keyword" attribute only
 
 coming[2.0.0, body based parameters were added in 2.0.0]
diff --git a/docs/reference/indices/flush.asciidoc b/docs/reference/indices/flush.asciidoc
index 0279ee3abcf..acf2b7c1a6a 100644
--- a/docs/reference/indices/flush.asciidoc
+++ b/docs/reference/indices/flush.asciidoc
@@ -12,7 +12,7 @@ trigger flush operations as required in order to clear memory.
 --------------------------------------------------
 POST twitter/_flush
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[setup:twitter]
 
 [float]
@@ -45,7 +45,7 @@ POST kimchy,elasticsearch/_flush
 
 POST _flush
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[s/^/PUT kimchy\nPUT elasticsearch\n/]
 
 [[indices-synced-flush]]
@@ -76,7 +76,7 @@ the <<indices-stats,indices stats>> API:
 --------------------------------------------------
 GET twitter/_stats/commit?level=shards
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[s/^/PUT twitter\n/]
 
 
@@ -141,7 +141,7 @@ NOTE: It is harmless to request a synced flush while there is ongoing indexing.
 --------------------------------------------------
 POST twitter/_flush/synced
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[setup:twitter]
 
 The response contains details about how many shards were successfully sync-flushed and information about any failure.
@@ -238,4 +238,4 @@ POST kimchy,elasticsearch/_flush/synced
 
 POST _flush/synced
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
diff --git a/docs/reference/indices/put-mapping.asciidoc b/docs/reference/indices/put-mapping.asciidoc
index 70a5aa8bef0..cb1d956e39f 100644
--- a/docs/reference/indices/put-mapping.asciidoc
+++ b/docs/reference/indices/put-mapping.asciidoc
@@ -37,7 +37,7 @@ PUT twitter/_mapping/tweet <3>
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> <<indices-create-index,Creates an index>> called `twitter` with the `message` field in the `tweet` <<mapping-type,mapping type>>.
 <2> Uses the PUT mapping API to add a new mapping type called `user`.
 <3> Uses the PUT mapping API to add a new field called `user_name` to the `tweet` mapping type.
@@ -115,7 +115,7 @@ PUT my_index/_mapping/user
   }
 }
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> Create an index with a `first` field under the `name` <<object>> field, and a `user_id` field.
 <2> Add a `last` field under the `name` object field.
 <3> Update the `ignore_above` setting from its default of 0.
@@ -174,7 +174,7 @@ PUT my_index/_mapping/type_one <2>
   }
 }
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[catch:request]
 <1> Create an index with two types, both of which contain a `text` field which have the same mapping.
 <2> Trying to update the `search_analyzer` just for `type_one` throws an exception like `"Merge failed with failures..."`.
@@ -194,6 +194,6 @@ PUT my_index/_mapping/type_one?update_all_types <1>
   }
 }
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[continued]
 <1> Adding the `update_all_types` parameter updates the `text` field in `type_one` and `type_two`.
diff --git a/docs/reference/indices/templates.asciidoc b/docs/reference/indices/templates.asciidoc
index 86bc77c52e4..754a93e3096 100644
--- a/docs/reference/indices/templates.asciidoc
+++ b/docs/reference/indices/templates.asciidoc
@@ -37,7 +37,7 @@ PUT _template/template_1
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Defines a template named template_1, with a template pattern of `te*`.
 The settings and mappings will be applied to any index name that matches
diff --git a/docs/reference/ingest.asciidoc b/docs/reference/ingest.asciidoc
index 767b928d205..7150bd32739 100644
--- a/docs/reference/ingest.asciidoc
+++ b/docs/reference/ingest.asciidoc
@@ -30,7 +30,7 @@ PUT my-index/my-type/my-id?pipeline=my_pipeline_id
   "foo": "bar"
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[catch:request]
 
 See <<ingest-apis,Ingest APIs>> for more information about creating, adding, and deleting pipelines.
diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc
index 631e4bb40f7..ec6253f2d93 100644
--- a/docs/reference/ingest/ingest-node.asciidoc
+++ b/docs/reference/ingest/ingest-node.asciidoc
@@ -50,7 +50,7 @@ PUT _ingest/pipeline/my-pipeline-id
   ]
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 NOTE: The put pipeline API also instructs all ingest nodes to reload their in-memory representation of pipelines, so that
       pipeline changes take effect immediately.
@@ -64,7 +64,7 @@ The get pipeline API returns pipelines based on ID. This API always returns a lo
 --------------------------------------------------
 GET _ingest/pipeline/my-pipeline-id
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[continued]
 
 Example response:
@@ -104,7 +104,7 @@ The delete pipeline API deletes pipelines by ID.
 --------------------------------------------------
 DELETE _ingest/pipeline/my-pipeline-id
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[continued]
 
 [[simulate-pipeline-api]]
@@ -189,7 +189,7 @@ POST _ingest/pipeline/_simulate
   ]
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Response:
 
@@ -288,7 +288,7 @@ POST _ingest/pipeline/_simulate?verbose
   ]
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Response:
 
diff --git a/docs/reference/mapping.asciidoc b/docs/reference/mapping.asciidoc
index 70de0a2735b..5507ba97ff6 100644
--- a/docs/reference/mapping.asciidoc
+++ b/docs/reference/mapping.asciidoc
@@ -156,7 +156,7 @@ PUT my_index <1>
   }
 }
 ---------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> Create an index called `my_index`.
 <2> Add mapping types called `user` and `blogpost`.
 <3> Disable the `_all` <<mapping-fields,meta field>> for the `user` mapping type.
diff --git a/docs/reference/mapping/dynamic-mapping.asciidoc b/docs/reference/mapping/dynamic-mapping.asciidoc
index 81aa5858db3..2370684ff44 100644
--- a/docs/reference/mapping/dynamic-mapping.asciidoc
+++ b/docs/reference/mapping/dynamic-mapping.asciidoc
@@ -12,7 +12,7 @@ type, and fields will spring to life automatically:
 PUT data/counters/1 <1>
 { "count": 5 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> Creates the `data` index, the `counters` mapping type, and a field
     called `count` with datatype `long`.
 
@@ -51,7 +51,7 @@ PUT data/_settings <1>
   "index.mapper.dynamic":false
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[continued]
 
 <1> Disable automatic type creation for all indices.
diff --git a/docs/reference/mapping/dynamic/default-mapping.asciidoc b/docs/reference/mapping/dynamic/default-mapping.asciidoc
index bef90301f0c..268ae4f29c7 100644
--- a/docs/reference/mapping/dynamic/default-mapping.asciidoc
+++ b/docs/reference/mapping/dynamic/default-mapping.asciidoc
@@ -27,7 +27,7 @@ PUT my_index
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The `_default_` mapping defaults the <<mapping-all-field,`_all`>> field to disabled.
 <2> The `user` type inherits the settings from `_default_`.
 <3> The `blogpost` type overrides the defaults and enables the <<mapping-all-field,`_all`>> field.
@@ -74,7 +74,7 @@ PUT _template/logging
 PUT logs-2015.10.01/event/1
 { "message": "error:16" }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The `logging` template will match any indices beginning with `logs-`.
 <2> Matching indices will be created with a single primary shard.
 <3> The `_all` field will be disabled by default for new type mappings.
diff --git a/docs/reference/mapping/dynamic/field-mapping.asciidoc b/docs/reference/mapping/dynamic/field-mapping.asciidoc
index a1ce6683502..020c5d24552 100644
--- a/docs/reference/mapping/dynamic/field-mapping.asciidoc
+++ b/docs/reference/mapping/dynamic/field-mapping.asciidoc
@@ -68,7 +68,7 @@ PUT my_index/my_type/1
 
 GET my_index/_mapping <1>
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The `create_date` field has been added as a <<date,`date`>>
     field with the <<mapping-date-format,`format`>>: +
     `"yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z"`.
@@ -93,7 +93,7 @@ PUT my_index/my_type/1 <1>
   "create": "2015/09/02"
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> The `create_date` field has been added as a <<text,`text`>> field.
 
@@ -118,7 +118,7 @@ PUT my_index/my_type/1
   "create_date": "09/25/2015"
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 
 [[numeric-detection]]
@@ -147,7 +147,7 @@ PUT my_index/my_type/1
   "my_integer": "1" <2>
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The `my_float` field is added as a <<number,`double`>> field.
 <2> The `my_integer` field is added as a <<number,`long`>> field.
 
diff --git a/docs/reference/mapping/dynamic/templates.asciidoc b/docs/reference/mapping/dynamic/templates.asciidoc
index 6396abe8522..21b0eadf683 100644
--- a/docs/reference/mapping/dynamic/templates.asciidoc
+++ b/docs/reference/mapping/dynamic/templates.asciidoc
@@ -96,7 +96,7 @@ PUT my_index/my_type/1
 }
 
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The `my_integer` field is mapped as an `integer`.
 <2> The `my_string` field is mapped as a `text`, with a `keyword` <<multi-fields,multi field>>.
 
@@ -140,7 +140,7 @@ PUT my_index/my_type/1
   "long_text": "foo" <2>
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The `long_num` field is mapped as a `long`.
 <2> The `long_text` field uses the default `string` mapping.
 
@@ -198,7 +198,7 @@ PUT my_index/my_type/1
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 [[template-variables]]
 ==== `{name}` and `{dynamic_type}`
@@ -245,7 +245,7 @@ PUT my_index/my_type/1
   "count":   5 <2>
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The `english` field is mapped as a `string` field with the `english` analyzer.
 <2> The `count` field is mapped as a `long` field with `doc_values` disabled
 
@@ -417,7 +417,7 @@ PUT _template/disable_all_field
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> Applies the mappings to an `index` which matches the pattern `*`, in other
     words, all new indices.
 <2> Defines the `_default_` type mapping types within the index.
diff --git a/docs/reference/mapping/fields/all-field.asciidoc b/docs/reference/mapping/fields/all-field.asciidoc
index 58f0e783af4..70fac4ed068 100644
--- a/docs/reference/mapping/fields/all-field.asciidoc
+++ b/docs/reference/mapping/fields/all-field.asciidoc
@@ -28,7 +28,7 @@ GET my_index/_search
   }
 }
 --------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The `_all` field will contain the terms: [ `"john"`, `"smith"`, `"1970"`, `"10"`, `"24"` ]
 
 [NOTE]
@@ -77,7 +77,7 @@ GET _search
   }
 }
 --------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The same goes for the `?q=` parameter in <<search-uri-request, URI search
 requests>> (which is rewritten to a `query_string` query internally):
@@ -115,7 +115,7 @@ PUT my_index
   }
 }
 --------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[s/\.\.\.//]
 
 <1> The `_all` field in `type_1` is enabled.
@@ -147,7 +147,7 @@ PUT my_index
   }
 }
 --------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> The `_all` field is disabled for the `my_type` type.
 <2> The `query_string` query will default to querying the `content` field in this index.
@@ -184,7 +184,7 @@ PUT myindex
   }
 }
 --------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> When querying the `_all` field, words that originated in the
     `title` field are twice as relevant as words that originated in
@@ -241,7 +241,7 @@ GET myindex/_search
   }
 }
 --------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> The `first_name` and `last_name` values are copied to the `full_name` field.
 
@@ -296,7 +296,7 @@ GET _search
   }
 }
 --------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Of course, storing the `_all` field will use significantly more disk space
 and, because it is a combination of other fields, it may result in odd
@@ -344,7 +344,7 @@ GET _search
   }
 }
 --------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> The query inspects the `_all` field to find matching documents.
 <2> Highlighting is performed on the two name fields, which are available from the `_source`.
diff --git a/docs/reference/mapping/fields/field-names-field.asciidoc b/docs/reference/mapping/fields/field-names-field.asciidoc
index ebb228fca94..a56a9081e88 100644
--- a/docs/reference/mapping/fields/field-names-field.asciidoc
+++ b/docs/reference/mapping/fields/field-names-field.asciidoc
@@ -37,7 +37,7 @@ GET my_index/_search
 }
 
 --------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> Querying on the `_field_names` field (also see the <<query-dsl-exists-query,`exists`>> query)
 <2> Accessing the `_field_names` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
diff --git a/docs/reference/mapping/fields/id-field.asciidoc b/docs/reference/mapping/fields/id-field.asciidoc
index 4ee470e4dbd..a852f4f4770 100644
--- a/docs/reference/mapping/fields/id-field.asciidoc
+++ b/docs/reference/mapping/fields/id-field.asciidoc
@@ -38,7 +38,7 @@ GET my_index/_search
   }
 }
 --------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> Querying on the `_id` field (also see the <<query-dsl-ids-query,`ids` query>>)
 <2> Accessing the `_id` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
diff --git a/docs/reference/mapping/fields/index-field.asciidoc b/docs/reference/mapping/fields/index-field.asciidoc
index bc506db1836..c31b6f1d0f7 100644
--- a/docs/reference/mapping/fields/index-field.asciidoc
+++ b/docs/reference/mapping/fields/index-field.asciidoc
@@ -55,7 +55,7 @@ GET index_1,index_2/_search
   }
 }
 --------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> Querying on the `_index` field
 <2> Aggregating on the `_index` field
diff --git a/docs/reference/mapping/fields/meta-field.asciidoc b/docs/reference/mapping/fields/meta-field.asciidoc
index 4bcab15b4c0..befaf4e8df0 100644
--- a/docs/reference/mapping/fields/meta-field.asciidoc
+++ b/docs/reference/mapping/fields/meta-field.asciidoc
@@ -22,7 +22,7 @@ PUT my_index
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> This `_meta` info can be retrieved with the
     <<indices-get-mapping,GET mapping>> API.
 
diff --git a/docs/reference/mapping/fields/parent-field.asciidoc b/docs/reference/mapping/fields/parent-field.asciidoc
index 78ce1ebfd00..9dd8646172d 100644
--- a/docs/reference/mapping/fields/parent-field.asciidoc
+++ b/docs/reference/mapping/fields/parent-field.asciidoc
@@ -47,7 +47,7 @@ GET my_index/my_parent/_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The `my_parent` type is parent to the `my_child` type.
 <2> Index a parent document.
 <3> Index two child documents, specifying the parent document's ID.
@@ -86,7 +86,7 @@ GET my_index/_search
   }
 }
 --------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[continued]
 
 <1> Querying on the `_parent` field (also see the <<query-dsl-has-parent-query,`has_parent` query>> and the <<query-dsl-has-child-query,`has_child` query>>)
@@ -138,7 +138,7 @@ PUT my_index
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The amount of heap used by global ordinals can be checked as follows:
 
@@ -150,4 +150,4 @@ GET _stats/fielddata?human&fields=_parent
 # Per-node per-index
 GET _nodes/stats/indices/fielddata?human&fields=_parent
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
diff --git a/docs/reference/mapping/fields/routing-field.asciidoc b/docs/reference/mapping/fields/routing-field.asciidoc
index c90a52aab37..49f269511dc 100644
--- a/docs/reference/mapping/fields/routing-field.asciidoc
+++ b/docs/reference/mapping/fields/routing-field.asciidoc
@@ -21,7 +21,7 @@ PUT my_index/my_type/1?routing=user1 <1>
 
 GET my_index/my_type/1?routing=user1 <2>
 ------------------------------
-// AUTOSENSE
+// CONSOLE
 // TESTSETUP
 
 <1> This document uses `user1` as its routing value, instead of its ID.
@@ -47,7 +47,7 @@ GET my_index/_search
   }
 }
 --------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> Querying on the `_routing` field (also see the <<query-dsl-ids-query,`ids` query>>)
 <2> Accessing the `_routing` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
@@ -70,7 +70,7 @@ GET my_index/_search?routing=user1,user2 <1>
   }
 }
 ------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> This search request will only be executed on the shards associated with the `user1` and `user2` routing values.
 
@@ -103,7 +103,7 @@ PUT my_index2/my_type/1 <2>
   "text": "No routing value provided"
 }
 ------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[catch:request]
 <1> Routing is required for `my_type` documents.
 <2> This index request throws a `routing_missing_exception`.
diff --git a/docs/reference/mapping/fields/source-field.asciidoc b/docs/reference/mapping/fields/source-field.asciidoc
index 9c509593b61..db005f6eb37 100644
--- a/docs/reference/mapping/fields/source-field.asciidoc
+++ b/docs/reference/mapping/fields/source-field.asciidoc
@@ -24,7 +24,7 @@ PUT tweets
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 [WARNING]
 .Think before disabling the `_source` field
@@ -130,7 +130,7 @@ GET logs/event/_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> These fields will be removed from the stored `_source` field.
 <2> We can still search on this field, even though it is not in the stored `_source`.
diff --git a/docs/reference/mapping/fields/timestamp-field.asciidoc b/docs/reference/mapping/fields/timestamp-field.asciidoc
index c6b3887f889..104b07cfee1 100644
--- a/docs/reference/mapping/fields/timestamp-field.asciidoc
+++ b/docs/reference/mapping/fields/timestamp-field.asciidoc
@@ -30,7 +30,7 @@ PUT my_index/my_type/3 <4>
 { "text": "Autogenerated timestamp set to now()" }
 
 ------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> Enable the `_timestamp` field with default settings.
 <2> Set the timestamp manually with a formatted date.
@@ -88,7 +88,7 @@ GET my_index/_search
   }
 }
 --------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[continued]
 
 <1> Querying on the `_timestamp` field
diff --git a/docs/reference/mapping/fields/ttl-field.asciidoc b/docs/reference/mapping/fields/ttl-field.asciidoc
index 1fd258d1ea1..26760d5aca8 100644
--- a/docs/reference/mapping/fields/ttl-field.asciidoc
+++ b/docs/reference/mapping/fields/ttl-field.asciidoc
@@ -44,7 +44,7 @@ PUT my_index/my_type/2 <2>
   "text": "Will not expire"
 }
 -------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> This document will expire 10 minutes after being indexed.
 <2> This document has no TTL set and will not expire.
 
@@ -80,7 +80,7 @@ PUT my_index/my_type/2 <2>
   "text": "Will expire in 5 minutes"
 }
 -------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> This document will expire 10 minutes after being indexed.
 <2> This document has no TTL set and so will expire after the default 5 minutes.
 
diff --git a/docs/reference/mapping/fields/type-field.asciidoc b/docs/reference/mapping/fields/type-field.asciidoc
index a1d48129f5d..ae283e44569 100644
--- a/docs/reference/mapping/fields/type-field.asciidoc
+++ b/docs/reference/mapping/fields/type-field.asciidoc
@@ -35,7 +35,7 @@ GET my_index/type_*/_search
 }
 
 --------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> Querying on the `_type` field
 <2> Accessing the `_type` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
diff --git a/docs/reference/mapping/fields/uid-field.asciidoc b/docs/reference/mapping/fields/uid-field.asciidoc
index cab5f4f8216..43d5751d426 100644
--- a/docs/reference/mapping/fields/uid-field.asciidoc
+++ b/docs/reference/mapping/fields/uid-field.asciidoc
@@ -50,7 +50,7 @@ GET my_index/_search
   }
 }
 --------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> Querying on the `_uid` field (also see the <<query-dsl-ids-query,`ids` query>>)
 <2> Aggregating on the `_uid` field
diff --git a/docs/reference/mapping/params/analyzer.asciidoc b/docs/reference/mapping/params/analyzer.asciidoc
index b2138794546..7fc6c801e16 100644
--- a/docs/reference/mapping/params/analyzer.asciidoc
+++ b/docs/reference/mapping/params/analyzer.asciidoc
@@ -72,7 +72,7 @@ GET my_index/_analyze?field=text.english <4>
   "text": "The quick Brown Foxes."
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The `text` field uses the default `standard` analyzer`.
 <2> The `text.english` <<multi-fields,multi-field>> uses the `english` analyzer, which removes stop words and applies stemming.
 <3> This returns the tokens: [ `the`, `quick`, `brown`, `foxes` ].
@@ -136,7 +136,7 @@ PUT my_index
    }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 [source,js]
 --------------------------------------------------
diff --git a/docs/reference/mapping/params/boost.asciidoc b/docs/reference/mapping/params/boost.asciidoc
index a3379d581c7..e6890f948be 100644
--- a/docs/reference/mapping/params/boost.asciidoc
+++ b/docs/reference/mapping/params/boost.asciidoc
@@ -23,7 +23,7 @@ PUT my_index
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> Matches on the `title` field will have twice the weight as those on the
     `content` field, which has the default `boost` of `1.0`.
@@ -45,7 +45,7 @@ POST _search
     }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 is equivalent to:
 
@@ -63,7 +63,7 @@ POST _search
     }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 
 The boost is also applied when it is copied with the
diff --git a/docs/reference/mapping/params/coerce.asciidoc b/docs/reference/mapping/params/coerce.asciidoc
index 9d55a8b80c8..dacdabaafc0 100644
--- a/docs/reference/mapping/params/coerce.asciidoc
+++ b/docs/reference/mapping/params/coerce.asciidoc
@@ -44,7 +44,7 @@ PUT my_index/my_type/2
   "number_two": "10" <2>
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[catch:request]
 <1> The `number_one` field will contain the integer `10`.
 <2> This document will be rejected because coercion is disabled.
@@ -87,7 +87,7 @@ PUT my_index/my_type/1
 PUT my_index/my_type/2
 { "number_two": "10" } <2>
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[catch:request]
 <1> The `number_one` field overrides the index level setting to enable coercion.
 <2> This document will be rejected because the `number_two` field inherits the index-level coercion setting.
diff --git a/docs/reference/mapping/params/copy-to.asciidoc b/docs/reference/mapping/params/copy-to.asciidoc
index a6288bcbc48..bc22c923821 100644
--- a/docs/reference/mapping/params/copy-to.asciidoc
+++ b/docs/reference/mapping/params/copy-to.asciidoc
@@ -49,7 +49,7 @@ GET my_index/_search
 }
 
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1>  The values of the `first_name` and `last_name` fields are copied to the
      `full_name` field.
 
diff --git a/docs/reference/mapping/params/doc-values.asciidoc b/docs/reference/mapping/params/doc-values.asciidoc
index 4ded2212de1..d9d6bbc9afc 100644
--- a/docs/reference/mapping/params/doc-values.asciidoc
+++ b/docs/reference/mapping/params/doc-values.asciidoc
@@ -40,7 +40,7 @@ PUT my_index
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The `status_code` field has `doc_values` enabled by default.
 <2> The `session_id` has `doc_values` disabled, but can still be queried.
 
diff --git a/docs/reference/mapping/params/dynamic.asciidoc b/docs/reference/mapping/params/dynamic.asciidoc
index 16bb89fbf6b..985e5a63df3 100644
--- a/docs/reference/mapping/params/dynamic.asciidoc
+++ b/docs/reference/mapping/params/dynamic.asciidoc
@@ -31,7 +31,7 @@ PUT my_index/my_type/2 <3>
 
 GET my_index/_mapping <4>
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> This document introduces the string field `username`, the object field
     `name`, and two string fields under the `name` object which can be
     referred to as `name.first` and `name.last`.
@@ -77,7 +77,7 @@ PUT my_index
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> Dynamic mapping is disabled at the type level, so no new top-level fields will be added dynamically.
 <2> The `user` object inherits the type-level setting.
 <3> The `user.social_networks` object enables dynamic mapping, so new fields may be added to this inner object.
diff --git a/docs/reference/mapping/params/enabled.asciidoc b/docs/reference/mapping/params/enabled.asciidoc
index 7bffcfddf2e..9d7dc05b382 100644
--- a/docs/reference/mapping/params/enabled.asciidoc
+++ b/docs/reference/mapping/params/enabled.asciidoc
@@ -52,7 +52,7 @@ PUT my_index/session/session_2
   "last_updated": "2015-12-06T18:22:13"
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The `session_data` field is disabled.
 <2> Any arbitrary data can be passed to the `session_data` field as it will be entirely ignored.
 <3> The `session_data` will also ignore values that are not JSON objects.
@@ -87,7 +87,7 @@ GET my_index/session/session_1 <2>
 
 GET my_index/_mapping <3>
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The entire `session` mapping type is disabled.
 <2> The document can be retrieved.
 <3> Checking the mapping reveals that no fields have been added.
diff --git a/docs/reference/mapping/params/fielddata.asciidoc b/docs/reference/mapping/params/fielddata.asciidoc
index e20a82bfd5c..92151c56d49 100644
--- a/docs/reference/mapping/params/fielddata.asciidoc
+++ b/docs/reference/mapping/params/fielddata.asciidoc
@@ -112,4 +112,4 @@ PUT my_index
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
diff --git a/docs/reference/mapping/params/format.asciidoc b/docs/reference/mapping/params/format.asciidoc
index 106b5b0f10d..38cb0ea624f 100644
--- a/docs/reference/mapping/params/format.asciidoc
+++ b/docs/reference/mapping/params/format.asciidoc
@@ -25,7 +25,7 @@ PUT my_index
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Many APIs which support date values also support <<date-math,date math>>
 expressions, such as `now-1m/d` -- the current time, minus one month, rounded
diff --git a/docs/reference/mapping/params/geohash-precision.asciidoc b/docs/reference/mapping/params/geohash-precision.asciidoc
index c571a70a9de..6e4e859d03f 100644
--- a/docs/reference/mapping/params/geohash-precision.asciidoc
+++ b/docs/reference/mapping/params/geohash-precision.asciidoc
@@ -56,5 +56,5 @@ GET my_index/_search?fielddata_fields=location.geohash
 }
 
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> A `geohash_precision` of 6 equates to geohash cells of approximately 1.26km x 0.6km
diff --git a/docs/reference/mapping/params/geohash-prefix.asciidoc b/docs/reference/mapping/params/geohash-prefix.asciidoc
index da4708f89b0..56f0b42ae8d 100644
--- a/docs/reference/mapping/params/geohash-prefix.asciidoc
+++ b/docs/reference/mapping/params/geohash-prefix.asciidoc
@@ -60,5 +60,5 @@ GET my_index/_search?fielddata_fields=location.geohash
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
diff --git a/docs/reference/mapping/params/geohash.asciidoc b/docs/reference/mapping/params/geohash.asciidoc
index a1319c071a3..d362d373437 100644
--- a/docs/reference/mapping/params/geohash.asciidoc
+++ b/docs/reference/mapping/params/geohash.asciidoc
@@ -53,7 +53,7 @@ GET my_index/_search?fielddata_fields=location.geohash <2>
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> A `location.geohash` field will be indexed for each geo-point.
 <2> The geohash can be retrieved with <<doc-values,`doc_values`>>.
 <3> A <<query-dsl-prefix-query,`prefix`>> query can find all geohashes which start with a particular prefix.
diff --git a/docs/reference/mapping/params/ignore-above.asciidoc b/docs/reference/mapping/params/ignore-above.asciidoc
index a9fa5377144..6a24ca626d9 100644
--- a/docs/reference/mapping/params/ignore-above.asciidoc
+++ b/docs/reference/mapping/params/ignore-above.asciidoc
@@ -40,7 +40,7 @@ GET _search <4>
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> This field will ignore any string longer than 20 characters.
 <2> This document is indexed successfully.
 <3> This document will be indexed, but without indexing the `message` field.
diff --git a/docs/reference/mapping/params/ignore-malformed.asciidoc b/docs/reference/mapping/params/ignore-malformed.asciidoc
index 00b57cb32e9..916b01b33c1 100644
--- a/docs/reference/mapping/params/ignore-malformed.asciidoc
+++ b/docs/reference/mapping/params/ignore-malformed.asciidoc
@@ -43,7 +43,7 @@ PUT my_index/my_type/2
   "number_two": "foo" <2>
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[catch:request]
 <1> This document will have the `text` field indexed, but not the `number_one` field.
 <2> This document will be rejected because `number_two` does not allow malformed values.
@@ -81,7 +81,7 @@ PUT my_index
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> The `number_one` field inherits the index-level setting.
 <2> The `number_two` field overrides the index-level setting to turn off `ignore_malformed`.
diff --git a/docs/reference/mapping/params/include-in-all.asciidoc b/docs/reference/mapping/params/include-in-all.asciidoc
index bd2b5b8a179..ade2db80d2c 100644
--- a/docs/reference/mapping/params/include-in-all.asciidoc
+++ b/docs/reference/mapping/params/include-in-all.asciidoc
@@ -28,7 +28,7 @@ PUT my_index
   }
 }
 --------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> The `title` and `content` fields will be included in the `_all` field.
 <2> The `date` field will not be included in the `_all` field.
@@ -69,7 +69,7 @@ PUT my_index
   }
 }
 --------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> All fields in `my_type` are excluded from `_all`.
 <2> The `author.first_name` and `author.last_name` fields are included in `_all`.
diff --git a/docs/reference/mapping/params/index-options.asciidoc b/docs/reference/mapping/params/index-options.asciidoc
index 9f327adb969..bda174b5234 100644
--- a/docs/reference/mapping/params/index-options.asciidoc
+++ b/docs/reference/mapping/params/index-options.asciidoc
@@ -66,5 +66,5 @@ GET my_index/_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The `text` field will use the postings highlighter by default because `offsets` are indexed.
diff --git a/docs/reference/mapping/params/lat-lon.asciidoc b/docs/reference/mapping/params/lat-lon.asciidoc
index 0a5cce48437..c610d8a1771 100644
--- a/docs/reference/mapping/params/lat-lon.asciidoc
+++ b/docs/reference/mapping/params/lat-lon.asciidoc
@@ -51,7 +51,7 @@ GET my_index/_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> Setting `lat_lon` to true indexes the geo-point in the `location.lat` and `location.lon` fields.
 <2> The `indexed` option tells the geo-distance query to use the inverted index instead of the in-memory calculation.
 
diff --git a/docs/reference/mapping/params/multi-fields.asciidoc b/docs/reference/mapping/params/multi-fields.asciidoc
index bee9dae1763..80a92a5d83e 100644
--- a/docs/reference/mapping/params/multi-fields.asciidoc
+++ b/docs/reference/mapping/params/multi-fields.asciidoc
@@ -55,7 +55,7 @@ GET my_index/_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The `city.raw` field is a `keyword` version of the `city` field.
 <2> The `city` field can be used for full text search.
 <3> The `city.raw` field can be used for sorting and aggregations
@@ -115,7 +115,7 @@ GET my_index/_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> The `text` field uses the `standard` analyzer.
 <2> The `text.english` field uses the `english` analyzer.
diff --git a/docs/reference/mapping/params/norms.asciidoc b/docs/reference/mapping/params/norms.asciidoc
index 9f78e0d62d8..0d259c0bc7c 100644
--- a/docs/reference/mapping/params/norms.asciidoc
+++ b/docs/reference/mapping/params/norms.asciidoc
@@ -30,7 +30,7 @@ PUT my_index/_mapping/my_type
   }
 }
 ------------
-// AUTOSENSE
+// CONSOLE
 // TEST[s/^/PUT my_index\n/]
 
 NOTE: Norms will not be removed instantly, but will be removed as old segments
diff --git a/docs/reference/mapping/params/null-value.asciidoc b/docs/reference/mapping/params/null-value.asciidoc
index b77a2e4da69..ee1ea0c3257 100644
--- a/docs/reference/mapping/params/null-value.asciidoc
+++ b/docs/reference/mapping/params/null-value.asciidoc
@@ -43,7 +43,7 @@ GET my_index/_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> Replace explicit `null` values with the term `NULL`.
 <2> An empty array does not contain an explicit `null`, and so won't be replaced with the `null_value`.
 <3> A query for `NULL` returns document 1, but not document 2.
diff --git a/docs/reference/mapping/params/position-increment-gap.asciidoc b/docs/reference/mapping/params/position-increment-gap.asciidoc
index 871b6275a1c..ea73e730759 100644
--- a/docs/reference/mapping/params/position-increment-gap.asciidoc
+++ b/docs/reference/mapping/params/position-increment-gap.asciidoc
@@ -41,7 +41,7 @@ GET my_index/groups/_search
     }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> This phrase query doesn't match our document which is totally expected.
 <2> This phrase query matches our document, even though `Abraham` and `Lincoln`
     are in separate strings, because `slop` > `position_increment_gap`.
@@ -79,7 +79,7 @@ GET my_index/groups/_search
     }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The first term in the next array element will be 0 terms apart from the
     last term in the previous array element.
 <2> The phrase query matches our document which is weird, but its what we asked
diff --git a/docs/reference/mapping/params/properties.asciidoc b/docs/reference/mapping/params/properties.asciidoc
index d9e3a2db75d..9efe1f8e82c 100644
--- a/docs/reference/mapping/params/properties.asciidoc
+++ b/docs/reference/mapping/params/properties.asciidoc
@@ -57,7 +57,7 @@ PUT my_index/my_type/1 <4>
   ]
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> Properties under the `my_type` mapping type.
 <2> Properties under the `manager` object field.
 <3> Properties under the `employees` nested field.
@@ -98,7 +98,7 @@ GET my_index/_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[continued]
 
 IMPORTANT: The full path to the inner field must be specified.
diff --git a/docs/reference/mapping/params/search-analyzer.asciidoc b/docs/reference/mapping/params/search-analyzer.asciidoc
index 3e51aad14af..d31a997430d 100644
--- a/docs/reference/mapping/params/search-analyzer.asciidoc
+++ b/docs/reference/mapping/params/search-analyzer.asciidoc
@@ -68,7 +68,7 @@ GET my_index/_search
 }
 
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> Analysis settings to define the custom `autocomplete` analyzer.
 <2> The `text` field uses the `autocomplete` analyzer at index time, but the `standard` analyzer at search time.
diff --git a/docs/reference/mapping/params/similarity.asciidoc b/docs/reference/mapping/params/similarity.asciidoc
index 731c557e730..527e961960b 100644
--- a/docs/reference/mapping/params/similarity.asciidoc
+++ b/docs/reference/mapping/params/similarity.asciidoc
@@ -48,7 +48,7 @@ PUT my_index
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The `default_field` uses the `classic` similarity (ie TF/IDF).
 <2> The `bm25_field` uses the `BM25` similarity.
 
diff --git a/docs/reference/mapping/params/store.asciidoc b/docs/reference/mapping/params/store.asciidoc
index 9c30217c146..d34d1016546 100644
--- a/docs/reference/mapping/params/store.asciidoc
+++ b/docs/reference/mapping/params/store.asciidoc
@@ -51,7 +51,7 @@ GET my_index/_search
   "fields": [ "title", "date" ] <2>
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The `title` and `date` fields are stored.
 <2> This request will retrieve the values of the `title` and `date` fields.
 
diff --git a/docs/reference/mapping/params/term-vector.asciidoc b/docs/reference/mapping/params/term-vector.asciidoc
index 136ab084149..121ab39a6df 100644
--- a/docs/reference/mapping/params/term-vector.asciidoc
+++ b/docs/reference/mapping/params/term-vector.asciidoc
@@ -62,7 +62,7 @@ GET my_index/_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The fast vector highlighter will be used by default for the `text` field
     because term vectors are enabled.
 
diff --git a/docs/reference/mapping/types/array.asciidoc b/docs/reference/mapping/types/array.asciidoc
index 2422e48e3d1..a1bf9f1eb65 100644
--- a/docs/reference/mapping/types/array.asciidoc
+++ b/docs/reference/mapping/types/array.asciidoc
@@ -75,7 +75,7 @@ GET my_index/_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The `tags` field is dynamically added as a `string` field.
 <2> The `lists` field is dynamically added as an `object` field.
 <3> The second document contains no arrays, but can be indexed into the same fields.
diff --git a/docs/reference/mapping/types/boolean.asciidoc b/docs/reference/mapping/types/boolean.asciidoc
index 65f03942fc2..914646a7220 100644
--- a/docs/reference/mapping/types/boolean.asciidoc
+++ b/docs/reference/mapping/types/boolean.asciidoc
@@ -44,7 +44,7 @@ GET my_index/_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> Indexing a document with a JSON `true`.
 <2> Querying for the document with `1`, which is interpreted as `true`.
 
@@ -81,7 +81,7 @@ GET my_index/_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> Inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work.
 
 [[boolean-params]]
diff --git a/docs/reference/mapping/types/date.asciidoc b/docs/reference/mapping/types/date.asciidoc
index 3756717b0d1..0f3c8a7862c 100644
--- a/docs/reference/mapping/types/date.asciidoc
+++ b/docs/reference/mapping/types/date.asciidoc
@@ -50,7 +50,7 @@ GET my_index/_search
   "sort": { "date": "asc"} <5>
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The `date` field uses the default `format`.
 <2> This document uses a plain date.
 <3> This document includes a time.
@@ -81,7 +81,7 @@ PUT my_index
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 [[date-params]]
 ==== Parameters for `date` fields
diff --git a/docs/reference/mapping/types/geo-point.asciidoc b/docs/reference/mapping/types/geo-point.asciidoc
index 45c5e65addb..909d9444527 100644
--- a/docs/reference/mapping/types/geo-point.asciidoc
+++ b/docs/reference/mapping/types/geo-point.asciidoc
@@ -74,7 +74,7 @@ GET my_index/_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> Geo-point expressed as an object, with `lat` and `lon` keys.
 <2> Geo-point expressed as a string with the format: `"lat,lon"`.
 <3> Geo-point expressed as a geohash.
diff --git a/docs/reference/mapping/types/ip.asciidoc b/docs/reference/mapping/types/ip.asciidoc
index 624bf63660f..4bcfe553c5c 100644
--- a/docs/reference/mapping/types/ip.asciidoc
+++ b/docs/reference/mapping/types/ip.asciidoc
@@ -33,7 +33,7 @@ GET my_index/_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 
 [[ip-params]]
diff --git a/docs/reference/mapping/types/keyword.asciidoc b/docs/reference/mapping/types/keyword.asciidoc
index 48fbf3cd75b..2ae1008c01b 100644
--- a/docs/reference/mapping/types/keyword.asciidoc
+++ b/docs/reference/mapping/types/keyword.asciidoc
@@ -28,7 +28,7 @@ PUT my_index
   }
 }
 --------------------------------
-// AUTOSENSE
+// CONSOLE
 
 [[keyword-params]]
 ==== Parameters for keyword fields
diff --git a/docs/reference/mapping/types/nested.asciidoc b/docs/reference/mapping/types/nested.asciidoc
index 90b54fafb04..8850763cdc8 100644
--- a/docs/reference/mapping/types/nested.asciidoc
+++ b/docs/reference/mapping/types/nested.asciidoc
@@ -29,7 +29,7 @@ PUT my_index/my_type/1
   ]
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The `user` field is dynamically added as a field of type `object`.
 
 would be transformed internally into a document that looks more like this:
@@ -61,7 +61,7 @@ GET my_index/_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[continued]
 
 ==== Using `nested` fields for arrays of objects
@@ -143,7 +143,7 @@ GET my_index/_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The `user` field is mapped as type `nested` instead of type `object`.
 <2> This query doesn't match because `Alice` and `Smith` are not in the same nested object.
 <3> This query matches because `Alice` and `White` are in the same nested object.
diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc
index 25d14386b8c..6982047db34 100644
--- a/docs/reference/mapping/types/numeric.asciidoc
+++ b/docs/reference/mapping/types/numeric.asciidoc
@@ -31,7 +31,7 @@ PUT my_index
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 [[number-params]]
 ==== Parameters for numeric fields
diff --git a/docs/reference/mapping/types/object.asciidoc b/docs/reference/mapping/types/object.asciidoc
index 6d35e6aee97..202c05512fe 100644
--- a/docs/reference/mapping/types/object.asciidoc
+++ b/docs/reference/mapping/types/object.asciidoc
@@ -18,7 +18,7 @@ PUT my_index/my_type/1
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The outer document is also a JSON object.
 <2> It contains an inner object called `manager`.
 <3> Which in turn contains an inner object called `name`.
@@ -64,7 +64,7 @@ PUT my_index
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The mapping type is a type of object, and has a `properties` field.
 <2> The `manager` field is an inner `object` field.
 <3> The `manager.name` field is an inner `object` field within the `manager` field.
diff --git a/docs/reference/mapping/types/text.asciidoc b/docs/reference/mapping/types/text.asciidoc
index 4a5be8287e8..7e51f4d6724 100644
--- a/docs/reference/mapping/types/text.asciidoc
+++ b/docs/reference/mapping/types/text.asciidoc
@@ -30,7 +30,7 @@ PUT my_index
   }
 }
 --------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Sometimes it is useful to have both a full text (`text`) and a keyword
 (`keyword`) version of the same field: one for full text search and the
diff --git a/docs/reference/mapping/types/token-count.asciidoc b/docs/reference/mapping/types/token-count.asciidoc
index baa6d409ead..c472c7c7770 100644
--- a/docs/reference/mapping/types/token-count.asciidoc
+++ b/docs/reference/mapping/types/token-count.asciidoc
@@ -43,7 +43,7 @@ GET my_index/_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The `name` field is an analyzed string field which uses the default `standard` analyzer.
 <2> The `name.length` field is a `token_count` <<multi-fields,multi-field>> which will index the number of tokens in the `name` field.
 <3> This query matches only the document containing `Rachel Alice Williams`, as it contains three tokens.
diff --git a/docs/reference/migration/migrate_5_0/mapping.asciidoc b/docs/reference/migration/migrate_5_0/mapping.asciidoc
index 2da16a62309..233ef84ae09 100644
--- a/docs/reference/migration/migrate_5_0/mapping.asciidoc
+++ b/docs/reference/migration/migrate_5_0/mapping.asciidoc
@@ -71,7 +71,7 @@ PUT my_index
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Also the `precision_step` parameter is now irrelevant and will be rejected on
 indices that are created on or after 5.0.
diff --git a/docs/reference/modules/cluster/allocation_filtering.asciidoc b/docs/reference/modules/cluster/allocation_filtering.asciidoc
index fbdeb64909f..533a3e83c6c 100644
--- a/docs/reference/modules/cluster/allocation_filtering.asciidoc
+++ b/docs/reference/modules/cluster/allocation_filtering.asciidoc
@@ -21,7 +21,7 @@ PUT _cluster/settings
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 NOTE: Shards will only be relocated if it is possible to do so without
 breaking another routing constraint, such as never allocating a primary and
@@ -66,5 +66,5 @@ PUT _cluster/settings
   }
 }
 ------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[skip:indexes don't assign]
diff --git a/docs/reference/modules/cluster/disk_allocator.asciidoc b/docs/reference/modules/cluster/disk_allocator.asciidoc
index fdd8fc5036b..b6289e37c73 100644
--- a/docs/reference/modules/cluster/disk_allocator.asciidoc
+++ b/docs/reference/modules/cluster/disk_allocator.asciidoc
@@ -65,7 +65,7 @@ PUT _cluster/settings
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 NOTE: Prior to 2.0.0, when using multiple data paths, the disk threshold
 decider only factored in the usage across all data paths (if you had two
diff --git a/docs/reference/modules/node.asciidoc b/docs/reference/modules/node.asciidoc
index 2cb59c6de3a..124d68f1d6d 100644
--- a/docs/reference/modules/node.asciidoc
+++ b/docs/reference/modules/node.asciidoc
@@ -157,7 +157,7 @@ PUT _cluster/settings
   }
 }
 ----------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[catch:/cannot set discovery.zen.minimum_master_nodes to more than the current master nodes/]
 
 TIP: An advantage of splitting the master and data roles between dedicated
diff --git a/docs/reference/modules/scripting/fields.asciidoc b/docs/reference/modules/scripting/fields.asciidoc
index 9ddf8267db6..35e1488fc28 100644
--- a/docs/reference/modules/scripting/fields.asciidoc
+++ b/docs/reference/modules/scripting/fields.asciidoc
@@ -79,7 +79,7 @@ GET my_index/_search
   }
 }
 -------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 
 [float]
@@ -113,7 +113,7 @@ GET my_index/_search
   }
 }
 -------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Doc-values can only return "simple" field values like numbers, dates, geo-
 points, terms, etc, or arrays of these values if the field is multi-valued.
@@ -211,7 +211,7 @@ GET my_index/_search
   }
 }
 -------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The `title` field is not stored and so cannot be used with the `_fields[]` syntax.
 <2> The `title` field can still be accessed from the `_source`.
 
diff --git a/docs/reference/modules/scripting/painless.asciidoc b/docs/reference/modules/scripting/painless.asciidoc
index b8c4c2cc81f..4f7d36f38f5 100644
--- a/docs/reference/modules/scripting/painless.asciidoc
+++ b/docs/reference/modules/scripting/painless.asciidoc
@@ -66,7 +66,7 @@ PUT hockey/player/_bulk?refresh
 {"index":{"_id":11}}
 {"first":"joe","last":"colborne","goals":[3,18,13],"assists":[6,20,24],"gp":[26,67,82]}
 ----------------------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TESTSETUP
 
 [float]
@@ -92,7 +92,7 @@ GET hockey/_search
   }
 }
 ----------------------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Alternatively, you could do the same thing using a script field instead of a function score:
 
@@ -113,7 +113,7 @@ GET hockey/_search
   }
 }
 ----------------------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The following example uses a Painless script to sort the players by their combined first and last names. The names are accessed using
 `input.doc['first'].value` and `input.doc['last'].value`.
@@ -137,7 +137,7 @@ GET hockey/_search
   }
 }
 ----------------------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 [float]
 === Updating Fields with Painless
@@ -161,7 +161,7 @@ GET hockey/_search
   }
 }
 ----------------------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 To change player 1's last name to `hockey`, simply set `input.ctx._source.last` to the new value:
 
@@ -178,7 +178,7 @@ POST hockey/player/1/_update
   }
 }
 ----------------------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 You can also add fields to a document. For example, this script adds a new field that contains
 the player's nickname,  _hockey_.
@@ -197,7 +197,7 @@ POST hockey/player/1/_update
   }
 }
 ----------------------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 [float]
 === Writing Type-Safe Scripts to Improve Performance
@@ -229,7 +229,7 @@ GET hockey/_search
   }
 }
 ----------------------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 [[painless-api]]
 [float]
diff --git a/docs/reference/modules/scripting/using.asciidoc b/docs/reference/modules/scripting/using.asciidoc
index 81636f6189e..856311fac24 100644
--- a/docs/reference/modules/scripting/using.asciidoc
+++ b/docs/reference/modules/scripting/using.asciidoc
@@ -41,7 +41,7 @@ GET my_index/_search
   }
 }
 -------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 
 [float]
@@ -191,7 +191,7 @@ POST _scripts/groovy/calculate-score
   "script": "log(_score * 2) + my_modifier"
 }
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 
 This same script can be retrieved with:
 
@@ -199,7 +199,7 @@ This same script can be retrieved with:
 -----------------------------------
 GET _scripts/groovy/calculate-score
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[continued]
 
 Stored scripts can be used by specifying the `lang` and `id` parameters as follows:
@@ -221,7 +221,7 @@ GET _search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[continued]
 
 And deleted with:
@@ -230,7 +230,7 @@ And deleted with:
 -----------------------------------
 DELETE _scripts/groovy/calculate-score
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[continued]
 
 NOTE: The size of stored scripts is limited to 65,535 bytes. This can be
diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc
index 76042d6b516..6c5245ce646 100644
--- a/docs/reference/modules/snapshots.asciidoc
+++ b/docs/reference/modules/snapshots.asciidoc
@@ -28,7 +28,7 @@ Once a repository is registered, its information can be obtained using the follo
 -----------------------------------
 GET /_snapshot/my_backup
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 
 which returns:
 
@@ -180,7 +180,7 @@ PUT /_snapshot/s3_repository?verify=false
   }
 }
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The verification process can also be executed manually by running the following command:
 
@@ -188,7 +188,7 @@ The verification process can also be executed manually by running the following
 -----------------------------------
 POST /_snapshot/s3_repository/_verify
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 
 It returns a list of nodes where repository was successfully verified or an error message if verification process failed.
 
@@ -203,7 +203,7 @@ command:
 -----------------------------------
 PUT /_snapshot/my_backup/snapshot_1?wait_for_completion=true
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The `wait_for_completion` parameter specifies whether or not the request should return immediately after snapshot
 initialization (default) or wait for snapshot completion. During snapshot initialization, information about all
@@ -222,7 +222,7 @@ PUT /_snapshot/my_backup/snapshot_1
   "include_global_state": false
 }
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The list of indices that should be included into the snapshot can be specified using the `indices` parameter that
 supports <<search-multi-index-type,multi index syntax>>. The snapshot request also supports the
@@ -258,7 +258,7 @@ Once a snapshot is created information about this snapshot can be obtained using
 -----------------------------------
 GET /_snapshot/my_backup/snapshot_1
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Similar as for repositories, information about multiple snapshots can be queried in one go, supporting wildcards as well:
 
@@ -266,7 +266,7 @@ Similar as for repositories, information about multiple snapshots can be queried
 -----------------------------------
 GET /_snapshot/my_backup/snapshot_*,some_other_snapshot
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 
 All snapshots currently stored in the repository can be listed using the following command:
 
@@ -274,7 +274,7 @@ All snapshots currently stored in the repository can be listed using the followi
 -----------------------------------
 GET /_snapshot/my_backup/_all
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The command fails if some of the snapshots are unavailable. The boolean parameter `ignore_unvailable` can be used to
 return all snapshots that are currently available.
@@ -292,7 +292,7 @@ A snapshot can be deleted from the repository using the following command:
 -----------------------------------
 DELETE /_snapshot/my_backup/snapshot_1
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 
 When a snapshot is deleted from a repository, Elasticsearch deletes all files that are associated with the deleted
 snapshot and not used by any other snapshots. If the deleted snapshot operation is executed while the snapshot is being
@@ -306,7 +306,7 @@ A repository can be deleted using the following command:
 -----------------------------------
 DELETE /_snapshot/my_backup
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 
 When a repository is deleted, Elasticsearch only removes the reference to the location where the repository is storing
 the snapshots. The snapshots themselves are left untouched and in place.
@@ -320,7 +320,7 @@ A snapshot can be restored using the following command:
 -----------------------------------
 POST /_snapshot/my_backup/snapshot_1/_restore
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 
 By default, all indices in the snapshot as well as cluster state are restored. It's possible to select indices that
 should be restored as well as prevent global cluster state from being restored by using `indices` and
@@ -341,7 +341,7 @@ POST /_snapshot/my_backup/snapshot_1/_restore
   "rename_replacement": "restored_index_$1"
 }
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The restore operation can be performed on a functioning cluster. However, an existing index can be only restored if it's
 <<indices-open-close,closed>> and has the same number of shards as the index in the snapshot.
@@ -378,7 +378,7 @@ POST /_snapshot/my_backup/snapshot_1/_restore
   ]
 }
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Please note, that some settings such as `index.number_of_shards` cannot be changed during restore operation.
 
@@ -413,7 +413,7 @@ A list of currently running snapshots with their detailed status information can
 -----------------------------------
 GET /_snapshot/_status
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 
 In this format, the command will return information about all currently running snapshots. By specifying a repository name, it's possible
 to limit the results to a particular repository:
@@ -422,7 +422,7 @@ to limit the results to a particular repository:
 -----------------------------------
 GET /_snapshot/my_backup/_status
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 
 If both repository name and snapshot id are specified, this command will return detailed status information for the given snapshot even
 if it's not currently running:
@@ -431,7 +431,7 @@ if it's not currently running:
 -----------------------------------
 GET /_snapshot/my_backup/snapshot_1/_status
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Multiple ids are also supported:
 
@@ -439,7 +439,7 @@ Multiple ids are also supported:
 -----------------------------------
 GET /_snapshot/my_backup/snapshot_1,snapshot_2/_status
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 
 [float]
 === Monitoring snapshot/restore progress
@@ -454,7 +454,7 @@ The snapshot operation can be also monitored by periodic calls to the snapshot i
 -----------------------------------
 GET /_snapshot/my_backup/snapshot_1
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Please note that snapshot info operation uses the same resources and thread pool as the snapshot operation. So,
 executing a snapshot info operation while large shards are being snapshotted can cause the snapshot info operation to wait
@@ -466,7 +466,7 @@ To get more immediate and complete information about snapshots the snapshot stat
 -----------------------------------
 GET /_snapshot/my_backup/snapshot_1/_status
 -----------------------------------
-// AUTOSENSE
+// CONSOLE
 
 While snapshot info method returns only basic information about the snapshot in progress, the snapshot status returns
 complete breakdown of the current state for each shard participating in the snapshot.
diff --git a/docs/reference/query-dsl/bool-query.asciidoc b/docs/reference/query-dsl/bool-query.asciidoc
index ab39eb49611..8fcd66129cd 100644
--- a/docs/reference/query-dsl/bool-query.asciidoc
+++ b/docs/reference/query-dsl/bool-query.asciidoc
@@ -69,7 +69,7 @@ POST _search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 ==== Scoring with `bool.filter`
 
@@ -96,7 +96,7 @@ GET _search
   }
 }
 ---------------------------------
-// AUTOSENSE
+// CONSOLE
 
 This `bool` query has a `match_all` query, which assigns a score of `1.0` to
 all documents.
@@ -119,7 +119,7 @@ GET _search
   }
 }
 ---------------------------------
-// AUTOSENSE
+// CONSOLE
 
 This `constant_score` query behaves in exactly the same way as the second example above.
 The `constant_score` query assigns a score of `1.0` to all documents matched
@@ -140,7 +140,7 @@ GET _search
   }
 }
 ---------------------------------
-// AUTOSENSE
+// CONSOLE
 
 ==== Using named queries to see which clauses matched
 
diff --git a/docs/reference/query-dsl/function-score-query.asciidoc b/docs/reference/query-dsl/function-score-query.asciidoc
index 3f9943d90d0..5f92d8c172c 100644
--- a/docs/reference/query-dsl/function-score-query.asciidoc
+++ b/docs/reference/query-dsl/function-score-query.asciidoc
@@ -491,7 +491,7 @@ GET _search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Next, we show how the computed score looks like for each of the three
 possible decay functions.
diff --git a/docs/reference/query-dsl/term-query.asciidoc b/docs/reference/query-dsl/term-query.asciidoc
index b27885ac1ed..bdb62b4d698 100644
--- a/docs/reference/query-dsl/term-query.asciidoc
+++ b/docs/reference/query-dsl/term-query.asciidoc
@@ -13,7 +13,7 @@ POST _search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> Finds documents which contain the exact term `Kimchy` in the inverted index
     of the `user` field.
 
@@ -45,7 +45,7 @@ GET _search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> The `urgent` query clause has a boost of `2.0`, meaning it is twice as important
     as the query clause for `normal`.
@@ -107,7 +107,7 @@ PUT my_index/my_type/1
   "exact_value": "Quick Foxes!"  <4>
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> The `full_text` field is of type `text` and will be analyzed.
 <2> The `exact_value` field is of type `keyword` and will NOT be analyzed.
@@ -154,7 +154,7 @@ GET my_index/my_type/_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[continued]
 
 <1> This query matches because the `exact_value` field contains the exact
diff --git a/docs/reference/setup/cluster_restart.asciidoc b/docs/reference/setup/cluster_restart.asciidoc
index 1dbb6e7616d..7f9f8f89afa 100644
--- a/docs/reference/setup/cluster_restart.asciidoc
+++ b/docs/reference/setup/cluster_restart.asciidoc
@@ -24,7 +24,7 @@ PUT _cluster/settings
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[skip:indexes don't assign]
 
 ==== Step 2: Perform a synced flush
@@ -36,7 +36,7 @@ Shard recovery will be much faster if you stop indexing and issue a
 --------------------------------------------------
 POST _flush/synced
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 A synced flush request is a ``best effort'' operation. It will fail if there
 are any pending indexing operations, but it is safe to reissue the request
@@ -72,7 +72,7 @@ GET _cat/health
 
 GET _cat/nodes
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Use these APIs to check that all nodes have successfully joined the cluster.
 
@@ -104,7 +104,7 @@ PUT _cluster/settings
   }
 }
 ------------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The cluster will now start allocating replica shards to all data nodes. At this
 point it is safe to resume indexing and searching, but your cluster will
@@ -120,7 +120,7 @@ GET _cat/health
 
 GET _cat/recovery
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Once the `status` column in the `_cat/health` output has reached `green`, all
 primary and replica shards have been successfully allocated.
diff --git a/docs/reference/setup/rolling_upgrade.asciidoc b/docs/reference/setup/rolling_upgrade.asciidoc
index 8c098a1ac04..5ba578ce768 100644
--- a/docs/reference/setup/rolling_upgrade.asciidoc
+++ b/docs/reference/setup/rolling_upgrade.asciidoc
@@ -28,7 +28,7 @@ PUT _cluster/settings
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[skip:indexes don't assign]
 
 ==== Step 2: Stop non-essential indexing and perform a synced flush (Optional)
@@ -41,7 +41,7 @@ will be much faster if you temporarily stop non-essential indexing and issue a
 --------------------------------------------------
 POST _flush/synced
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 A synced flush request is a ``best effort'' operation. It will fail if there
 are any pending indexing operations, but it is safe to reissue the request
@@ -103,7 +103,7 @@ the log file or by checking the output of this request:
 --------------------------------------------------
 GET _cat/nodes
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 ==== Step 6: Reenable shard allocation
 
@@ -119,7 +119,7 @@ PUT _cluster/settings
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 ==== Step 7: Wait for the node to recover
 
@@ -131,7 +131,7 @@ request:
 --------------------------------------------------
 GET _cat/health
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Wait for the `status` column to move from `yellow` to `green`.  Status `green`
 means that all primary and replica shards have been allocated.
@@ -164,7 +164,7 @@ recover.  The recovery status of individual shards can be monitored with the
 --------------------------------------------------
 GET _cat/recovery
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 If you stopped indexing, then it is safe to resume indexing as soon as
 recovery has completed.

From ef2e3a8c39dde33d7df2a19729df716eb233f2b8 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 9 May 2016 09:50:35 -0400
Subject: [PATCH 0129/1311] Rest tests: More defense around stashing body

Integration tests failed:
https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+master+multijob-intake/483/console

We'll see if the rest tests were hiding some other failure.
---
 .../src/main/java/org/elasticsearch/test/rest/Stash.java    | 3 +++
 .../org/elasticsearch/test/rest/client/RestResponse.java    | 6 +++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java b/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
index 9eae902c3e3..52b531e5cd1 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
@@ -88,6 +88,9 @@ public class Stash implements ToXContent {
      */
     public Object unstashValue(String value) throws IOException {
         if (value.startsWith("$body.")) {
+            if (response == null) {
+                return null;
+            }
             return response.evaluate(value.substring("$body".length()), this);
         }
         Object stashedValue = stash.get(value.substring(1));
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestResponse.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestResponse.java
index cdcd6a21a52..e0b2f3ab729 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestResponse.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestResponse.java
@@ -51,7 +51,11 @@ public class RestResponse {
      */
     public Object getBody() throws IOException {
         if (isJson()) {
-            return parsedResponse().evaluate("");
+            JsonPath parsedResponse = parsedResponse();
+            if (parsedResponse == null) {
+                return null;
+            }
+            return parsedResponse.evaluate("");
         }
         return response.getBody();
     }

From 8c52e8814b005d6d8d3fd48f6c10b25c9e27819c Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Mon, 9 May 2016 14:20:17 +0200
Subject: [PATCH 0130/1311] Remove ReindexResponse in favor of
 BulkIndexByScrollResponse

---
 .../AbstractAsyncBulkIndexByScrollAction.java | 18 +++--
 .../AbstractBaseReindexRestHandler.java       | 15 +++-
 .../index/reindex/BulkByScrollTask.java       | 42 +++++++----
 .../reindex/BulkIndexByScrollResponse.java    | 15 +++-
 ...kIndexByScrollResponseContentListener.java | 28 +++++--
 .../index/reindex/ReindexAction.java          |  6 +-
 .../index/reindex/ReindexRequestBuilder.java  |  6 +-
 .../index/reindex/ReindexResponse.java        | 75 -------------------
 .../index/reindex/RestReindexAction.java      |  4 +-
 .../reindex/RestUpdateByQueryAction.java      |  6 +-
 .../index/reindex/TransportReindexAction.java | 19 ++---
 .../reindex/TransportUpdateByQueryAction.java | 13 +---
 ...tAsyncBulkIndexByScrollActionTestCase.java |  2 +-
 .../index/reindex/ReindexCancelTests.java     |  2 +-
 .../index/reindex/ReindexFailureTests.java    |  6 +-
 .../index/reindex/ReindexMetadataTests.java   |  2 +-
 .../index/reindex/ReindexScriptTests.java     |  4 +-
 .../index/reindex/ReindexTestCase.java        |  4 +-
 .../index/reindex/ReindexVersioningTests.java |  6 +-
 .../index/reindex/RoundTripTests.java         |  6 +-
 .../reindex/UpdateByQueryWithScriptTests.java |  2 +-
 21 files changed, 121 insertions(+), 160 deletions(-)
 delete mode 100644 modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexResponse.java

diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java
index 1932f2ef5be..df9245346b3 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java
@@ -20,12 +20,15 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.client.ParentTaskAssigningClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.mapper.internal.IdFieldMapper;
 import org.elasticsearch.index.mapper.internal.IndexFieldMapper;
 import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
@@ -44,6 +47,7 @@ import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
@@ -53,17 +57,15 @@ import static java.util.Collections.emptyMap;
  * Abstract base for scrolling across a search and executing bulk indexes on all
  * results.
  */
-public abstract class AbstractAsyncBulkIndexByScrollAction<
-                Request extends AbstractBulkIndexByScrollRequest<Request>,
-                Response extends BulkIndexByScrollResponse>
-        extends AbstractAsyncBulkByScrollAction<Request, Response> {
+public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends AbstractBulkIndexByScrollRequest<Request>>
+        extends AbstractAsyncBulkByScrollAction<Request, BulkIndexByScrollResponse> {
 
     private final ScriptService scriptService;
     private final CompiledScript script;
 
     public AbstractAsyncBulkIndexByScrollAction(BulkByScrollTask task, ESLogger logger, ScriptService scriptService, ClusterState state,
             ParentTaskAssigningClient client, ThreadPool threadPool, Request mainRequest, SearchRequest firstSearchRequest,
-            ActionListener<Response> listener) {
+            ActionListener<BulkIndexByScrollResponse> listener) {
         super(task, logger, client, threadPool, mainRequest, firstSearchRequest, listener);
         this.scriptService = scriptService;
         if (mainRequest.getScript() == null) {
@@ -73,6 +75,12 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<
         }
     }
 
+    @Override
+    protected BulkIndexByScrollResponse buildResponse(TimeValue took, List<BulkItemResponse.Failure> indexingFailures,
+                                     List<ShardSearchFailure> searchFailures, boolean timedOut) {
+        return new BulkIndexByScrollResponse(took, task.getStatus(), indexingFailures, searchFailures, timedOut);
+    }
+
     /**
      * Build the IndexRequest for a single search hit. This shouldn't handle
      * metadata or the script. That will be handled by copyMetadata and
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
index 665ad37f1d8..d98735d3fb4 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
@@ -37,11 +37,12 @@ import org.elasticsearch.tasks.LoggingTaskListener;
 import org.elasticsearch.tasks.Task;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 
 public abstract class AbstractBaseReindexRestHandler<
                 Request extends AbstractBulkByScrollRequest<Request>,
-                Response extends BulkIndexByScrollResponse,
-                TA extends TransportAction<Request, Response>
+                TA extends TransportAction<Request, BulkIndexByScrollResponse>
             > extends BaseRestHandler {
 
     /**
@@ -87,14 +88,20 @@ public abstract class AbstractBaseReindexRestHandler<
         this.action = action;
     }
 
-    protected void execute(RestRequest request, Request internalRequest, RestChannel channel) throws IOException {
+    protected void execute(RestRequest request, Request internalRequest, RestChannel channel,
+                           boolean includeCreated, boolean includeUpdated, boolean includeDeleted) throws IOException {
         Float requestsPerSecond = parseRequestsPerSecond(request);
         if (requestsPerSecond != null) {
             internalRequest.setRequestsPerSecond(requestsPerSecond);
         }
 
         if (request.paramAsBoolean("wait_for_completion", true)) {
-            action.execute(internalRequest, new BulkIndexByScrollResponseContentListener<Response>(channel));
+            Map<String, String> params = new HashMap<>();
+            params.put(BulkByScrollTask.Status.INCLUDE_CREATED, Boolean.toString(includeCreated));
+            params.put(BulkByScrollTask.Status.INCLUDE_UPDATED, Boolean.toString(includeUpdated));
+            params.put(BulkByScrollTask.Status.INCLUDE_DELETED, Boolean.toString(includeDeleted));
+
+            action.execute(internalRequest, new BulkIndexByScrollResponseContentListener<>(channel, params));
             return;
         }
         /*
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
index 39a254bf757..9f4c8163b88 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
@@ -108,6 +108,24 @@ public class BulkByScrollTask extends CancellableTask {
     public static class Status implements Task.Status {
         public static final String NAME = "bulk-by-scroll";
 
+        /**
+         * XContent param name to indicate if "created" count must be included
+         * in the response.
+         */
+        public static final String INCLUDE_CREATED = "include_created";
+
+        /**
+         * XContent param name to indicate if "updated" count must be included
+         * in the response.
+         */
+        public static final String INCLUDE_UPDATED = "include_updated";
+
+        /**
+         * XContent param name to indicate if "deleted" count must be included
+         * in the response.
+         */
+        public static final String INCLUDE_DELETED = "include_deleted";
+
         private final long total;
         private final long updated;
         private final long created;
@@ -171,18 +189,20 @@ public class BulkByScrollTask extends CancellableTask {
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
-            innerXContent(builder, params, true, true);
+            innerXContent(builder, params);
             return builder.endObject();
         }
 
-        public XContentBuilder innerXContent(XContentBuilder builder, Params params, boolean includeCreated, boolean includeDeleted)
+        public XContentBuilder innerXContent(XContentBuilder builder, Params params)
                 throws IOException {
             builder.field("total", total);
-            builder.field("updated", updated);
-            if (includeCreated) {
+            if (params.paramAsBoolean(INCLUDE_UPDATED, true)) {
+                builder.field("updated", updated);
+            }
+            if (params.paramAsBoolean(INCLUDE_CREATED, true)) {
                 builder.field("created", created);
             }
-            if (includeDeleted) {
+            if (params.paramAsBoolean(INCLUDE_DELETED, true)) {
                 builder.field("deleted", deleted);
             }
             builder.field("batches", batches);
@@ -202,18 +222,14 @@ public class BulkByScrollTask extends CancellableTask {
         public String toString() {
             StringBuilder builder = new StringBuilder();
             builder.append("BulkIndexByScrollResponse[");
-            innerToString(builder, true, true);
+            innerToString(builder);
             return builder.append(']').toString();
         }
 
-        public void innerToString(StringBuilder builder, boolean includeCreated, boolean includeDeleted) {
+        public void innerToString(StringBuilder builder) {
             builder.append("updated=").append(updated);
-            if (includeCreated) {
-                builder.append(",created=").append(created);
-            }
-            if (includeDeleted) {
-                builder.append(",deleted=").append(deleted);
-            }
+            builder.append(",created=").append(created);
+            builder.append(",deleted=").append(deleted);
             builder.append(",batches=").append(batches);
             builder.append(",versionConflicts=").append(versionConflicts);
             builder.append(",noops=").append(noops);
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java
index e537d7788a8..35d45f3d5d4 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java
@@ -51,7 +51,7 @@ public class BulkIndexByScrollResponse extends ActionResponse implements ToXCont
     }
 
     public BulkIndexByScrollResponse(TimeValue took, BulkByScrollTask.Status status, List<Failure> indexingFailures,
-            List<ShardSearchFailure> searchFailures, boolean timedOut) {
+                                     List<ShardSearchFailure> searchFailures, boolean timedOut) {
         this.took = took;
         this.status = requireNonNull(status, "Null status not supported");
         this.indexingFailures = indexingFailures;
@@ -67,6 +67,14 @@ public class BulkIndexByScrollResponse extends ActionResponse implements ToXCont
         return status;
     }
 
+    public long getCreated() {
+        return status.getCreated();
+    }
+
+    public long getDeleted() {
+        return status.getDeleted();
+    }
+
     public long getUpdated() {
         return status.getUpdated();
     }
@@ -152,7 +160,7 @@ public class BulkIndexByScrollResponse extends ActionResponse implements ToXCont
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field("took", took.millis());
         builder.field("timed_out", timedOut);
-        status.innerXContent(builder, params, false, false);
+        status.innerXContent(builder, params);
         builder.startArray("failures");
         for (Failure failure: indexingFailures) {
             builder.startObject();
@@ -173,7 +181,8 @@ public class BulkIndexByScrollResponse extends ActionResponse implements ToXCont
         StringBuilder builder = new StringBuilder();
         builder.append("BulkIndexByScrollResponse[");
         builder.append("took=").append(took).append(',');
-        status.innerToString(builder, false, false);
+        builder.append("timed_out=").append(timedOut).append(',');
+        status.innerToString(builder);
         builder.append(",indexing_failures=").append(getIndexingFailures().subList(0, min(3, getIndexingFailures().size())));
         builder.append(",search_failures=").append(getSearchFailures().subList(0, min(3, getSearchFailures().size())));
         return builder.append(']').toString();
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java
index 24612aa14de..72bf6957e12 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java
@@ -22,21 +22,37 @@ package org.elasticsearch.index.reindex;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
 import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.rest.action.support.RestToXContentListener;
+import org.elasticsearch.rest.action.support.RestBuilderListener;
+
+import java.util.Map;
 
 /**
- * Just like RestToXContentListener but will return higher than 200 status if
- * there are any failures.
+ * RestBuilderListener that returns higher than 200 status if there are any failures and allows to set XContent.Params.
  */
-public class BulkIndexByScrollResponseContentListener<R extends BulkIndexByScrollResponse> extends RestToXContentListener<R> {
-    public BulkIndexByScrollResponseContentListener(RestChannel channel) {
+public class BulkIndexByScrollResponseContentListener<R extends BulkIndexByScrollResponse> extends RestBuilderListener<R> {
+
+    private final Map<String, String> params;
+
+    public BulkIndexByScrollResponseContentListener(RestChannel channel, Map<String, String> params) {
         super(channel);
+        this.params = params;
     }
 
     @Override
-    protected RestStatus getStatus(R response) {
+    public RestResponse buildResponse(R response, XContentBuilder builder) throws Exception {
+        builder.startObject();
+        response.toXContent(builder, new ToXContent.DelegatingMapParams(params, channel.request()));
+        builder.endObject();
+        return new BytesRestResponse(getStatus(response), builder);
+    }
+
+    private RestStatus getStatus(R response) {
         /*
          * Return the highest numbered rest status under the assumption that higher numbered statuses are "more error" and thus more
          * interesting to the user.
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexAction.java
index 9b2c51bcf8d..aa16a859c1d 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.index.reindex;
 import org.elasticsearch.action.Action;
 import org.elasticsearch.client.ElasticsearchClient;
 
-public class ReindexAction extends Action<ReindexRequest, ReindexResponse, ReindexRequestBuilder> {
+public class ReindexAction extends Action<ReindexRequest, BulkIndexByScrollResponse, ReindexRequestBuilder> {
     public static final ReindexAction INSTANCE = new ReindexAction();
     public static final String NAME = "indices:data/write/reindex";
 
@@ -36,7 +36,7 @@ public class ReindexAction extends Action<ReindexRequest, ReindexResponse, Reind
     }
 
     @Override
-    public ReindexResponse newResponse() {
-        return new ReindexResponse();
+    public BulkIndexByScrollResponse newResponse() {
+        return new BulkIndexByScrollResponse();
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java
index 96330e56f99..6b5cee46c97 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java
@@ -27,17 +27,17 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 
 public class ReindexRequestBuilder extends
-        AbstractBulkIndexByScrollRequestBuilder<ReindexRequest, ReindexResponse, ReindexRequestBuilder> {
+        AbstractBulkIndexByScrollRequestBuilder<ReindexRequest, BulkIndexByScrollResponse, ReindexRequestBuilder> {
     private final IndexRequestBuilder destination;
 
     public ReindexRequestBuilder(ElasticsearchClient client,
-            Action<ReindexRequest, ReindexResponse, ReindexRequestBuilder> action) {
+            Action<ReindexRequest, BulkIndexByScrollResponse, ReindexRequestBuilder> action) {
         this(client, action, new SearchRequestBuilder(client, SearchAction.INSTANCE),
                 new IndexRequestBuilder(client, IndexAction.INSTANCE));
     }
 
     private ReindexRequestBuilder(ElasticsearchClient client,
-            Action<ReindexRequest, ReindexResponse, ReindexRequestBuilder> action,
+            Action<ReindexRequest, BulkIndexByScrollResponse, ReindexRequestBuilder> action,
             SearchRequestBuilder search, IndexRequestBuilder destination) {
         super(client, action, search, new ReindexRequest(search.request(), destination.request()));
         this.destination = destination;
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexResponse.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexResponse.java
deleted file mode 100644
index 0cd4c9cbc18..00000000000
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexResponse.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.reindex;
-
-import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
-import org.elasticsearch.action.search.ShardSearchFailure;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.reindex.BulkByScrollTask.Status;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Response for the ReindexAction.
- */
-public class ReindexResponse extends BulkIndexByScrollResponse {
-    public ReindexResponse() {
-    }
-
-    public ReindexResponse(TimeValue took, Status status, List<Failure> indexingFailures, List<ShardSearchFailure> searchFailures,
-            boolean timedOut) {
-        super(took, status, indexingFailures, searchFailures, timedOut);
-    }
-
-    public long getCreated() {
-        return getStatus().getCreated();
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field("took", getTook().millis());
-        builder.field("timed_out", isTimedOut());
-        getStatus().innerXContent(builder, params, true, false);
-        builder.startArray("failures");
-        for (Failure failure: getIndexingFailures()) {
-            builder.startObject();
-            failure.toXContent(builder, params);
-            builder.endObject();
-        }
-        for (ShardSearchFailure failure: getSearchFailures()) {
-            builder.startObject();
-            failure.toXContent(builder, params);
-            builder.endObject();
-        }
-        builder.endArray();
-        return builder;
-    }
-
-    @Override
-    public String toString() {
-        StringBuilder builder = new StringBuilder();
-        builder.append("ReindexResponse[");
-        builder.append("took=").append(getTook()).append(',');
-        getStatus().innerToString(builder, true, false);
-        return builder.append(']').toString();
-    }
-}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
index a3869080789..d61980fb8ce 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
@@ -58,7 +58,7 @@ import static org.elasticsearch.rest.RestStatus.BAD_REQUEST;
 /**
  * Expose IndexBySearchRequest over rest.
  */
-public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexRequest, ReindexResponse, TransportReindexAction> {
+public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexRequest, TransportReindexAction> {
     private static final ObjectParser<ReindexRequest, ReindexParseContext> PARSER = new ObjectParser<>("reindex");
     static {
         ObjectParser.Parser<SearchRequest, ReindexParseContext> sourceParser = (parser, search, context) -> {
@@ -130,7 +130,7 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexReq
         }
         parseCommon(internalRequest, request);
 
-        execute(request, internalRequest, channel);
+        execute(request, internalRequest, channel, true, true, false);
     }
 
     private void badRequest(RestChannel channel, String message) {
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
index 8da8b2c904f..9f76be3f0ff 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
@@ -46,8 +46,8 @@ import static org.elasticsearch.index.reindex.AbstractBulkByScrollRequest.SIZE_A
 import static org.elasticsearch.index.reindex.RestReindexAction.parseCommon;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
-public class RestUpdateByQueryAction extends
-        AbstractBaseReindexRestHandler<UpdateByQueryRequest, BulkIndexByScrollResponse, TransportUpdateByQueryAction> {
+public class RestUpdateByQueryAction extends AbstractBaseReindexRestHandler<UpdateByQueryRequest, TransportUpdateByQueryAction> {
+
     @Inject
     public RestUpdateByQueryAction(Settings settings, RestController controller, Client client,
             IndicesQueriesRegistry indicesQueriesRegistry, AggregatorParsers aggParsers, Suggesters suggesters,
@@ -113,6 +113,6 @@ public class RestUpdateByQueryAction extends
             internalRequest.getSearchRequest().source().timeout(request.paramAsTime("search_timeout", null));
         }
 
-        execute(request, internalRequest, channel);
+        execute(request, internalRequest, channel, false, true, false);
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
index 7054fdbc4e6..0f07cc560c8 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
@@ -21,10 +21,8 @@ package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.AutoCreateIndex;
 import org.elasticsearch.action.support.HandledTransportAction;
@@ -46,13 +44,12 @@ import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-import java.util.List;
 import java.util.Objects;
 
 import static java.util.Objects.requireNonNull;
 import static org.elasticsearch.index.VersionType.INTERNAL;
 
-public class TransportReindexAction extends HandledTransportAction<ReindexRequest, ReindexResponse> {
+public class TransportReindexAction extends HandledTransportAction<ReindexRequest, BulkIndexByScrollResponse> {
     private final ClusterService clusterService;
     private final ScriptService scriptService;
     private final AutoCreateIndex autoCreateIndex;
@@ -71,7 +68,7 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
     }
 
     @Override
-    protected void doExecute(Task task, ReindexRequest request, ActionListener<ReindexResponse> listener) {
+    protected void doExecute(Task task, ReindexRequest request, ActionListener<BulkIndexByScrollResponse> listener) {
         ClusterState state = clusterService.state();
         validateAgainstAliases(request.getSearchRequest(), request.getDestination(), indexNameExpressionResolver, autoCreateIndex, state);
         ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task);
@@ -79,7 +76,7 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
     }
 
     @Override
-    protected void doExecute(ReindexRequest request, ActionListener<ReindexResponse> listener) {
+    protected void doExecute(ReindexRequest request, ActionListener<BulkIndexByScrollResponse> listener) {
         throw new UnsupportedOperationException("task required");
     }
 
@@ -116,10 +113,10 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
      * but this makes no attempt to do any of them so it can be as simple
      * possible.
      */
-    static class AsyncIndexBySearchAction extends AbstractAsyncBulkIndexByScrollAction<ReindexRequest, ReindexResponse> {
+    static class AsyncIndexBySearchAction extends AbstractAsyncBulkIndexByScrollAction<ReindexRequest> {
         public AsyncIndexBySearchAction(BulkByScrollTask task, ESLogger logger, ScriptService scriptService,
                 ParentTaskAssigningClient client, ClusterState state, ThreadPool threadPool, ReindexRequest request,
-                ActionListener<ReindexResponse> listener) {
+                ActionListener<BulkIndexByScrollResponse> listener) {
             super(task, logger, scriptService, state, client, threadPool, request, request.getSearchRequest(), listener);
         }
 
@@ -193,12 +190,6 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
             }
         }
 
-        @Override
-        protected ReindexResponse buildResponse(TimeValue took, List<Failure> indexingFailures, List<ShardSearchFailure> searchFailures,
-                boolean timedOut) {
-            return new ReindexResponse(took, task.getStatus(), indexingFailures, searchFailures, timedOut);
-        }
-
         /*
          * Methods below here handle script updating the index request. They try
          * to be pretty liberal with regards to types because script are often
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
index 38d3e3d55c3..4d702be4e9d 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
@@ -20,9 +20,7 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
 import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.Client;
@@ -33,7 +31,6 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.mapper.internal.IdFieldMapper;
 import org.elasticsearch.index.mapper.internal.IndexFieldMapper;
@@ -48,8 +45,6 @@ import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-import java.util.List;
-
 public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateByQueryRequest, BulkIndexByScrollResponse> {
     private final Client client;
     private final ScriptService scriptService;
@@ -82,7 +77,7 @@ public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateB
     /**
      * Simple implementation of update-by-query using scrolling and bulk.
      */
-    static class AsyncIndexBySearchAction extends AbstractAsyncBulkIndexByScrollAction<UpdateByQueryRequest, BulkIndexByScrollResponse> {
+    static class AsyncIndexBySearchAction extends AbstractAsyncBulkIndexByScrollAction<UpdateByQueryRequest> {
         public AsyncIndexBySearchAction(BulkByScrollTask task, ESLogger logger, ScriptService scriptService,
                 ParentTaskAssigningClient client, ThreadPool threadPool, ClusterState clusterState, UpdateByQueryRequest request,
                 ActionListener<BulkIndexByScrollResponse> listener) {
@@ -102,12 +97,6 @@ public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateB
             return index;
         }
 
-        @Override
-        protected BulkIndexByScrollResponse buildResponse(TimeValue took, List<Failure> indexingFailures,
-                List<ShardSearchFailure> searchFailures, boolean timedOut) {
-            return new BulkIndexByScrollResponse(took, task.getStatus(), indexingFailures, searchFailures, timedOut);
-        }
-
         @Override
         protected void scriptChangedIndex(IndexRequest index, Object to) {
             throw new IllegalArgumentException("Modifying [" + IndexFieldMapper.NAME + "] not allowed");
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionTestCase.java
index 703dbaa6dbe..ab5f8f0d748 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionTestCase.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionTestCase.java
@@ -46,7 +46,7 @@ public abstract class AbstractAsyncBulkIndexByScrollActionTestCase<
         threadPool.shutdown();
     }
 
-    protected abstract AbstractAsyncBulkIndexByScrollAction<Request, Response> action();
+    protected abstract AbstractAsyncBulkIndexByScrollAction<Request> action();
 
     protected abstract Request request();
 
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexCancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexCancelTests.java
index 44f5a3296cb..1412331736d 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexCancelTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexCancelTests.java
@@ -33,7 +33,7 @@ import static org.hamcrest.Matchers.equalTo;
  */
 public class ReindexCancelTests extends ReindexTestCase {
     public void testCancel() throws Exception {
-        ReindexResponse response = CancelTestUtils.testCancel(this, reindex().destination("dest", "test"), ReindexAction.NAME);
+        BulkIndexByScrollResponse response = CancelTestUtils.testCancel(this, reindex().destination("dest", "test"), ReindexAction.NAME);
 
         assertThat(response, reindexResponseMatcher().created(1).reasonCancelled(equalTo("by user request")));
         refresh("dest");
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java
index 196fa6c7e57..74de297e43a 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java
@@ -57,7 +57,7 @@ public class ReindexFailureTests extends ReindexTestCase {
          */
         copy.source().setSize(1);
 
-        ReindexResponse response = copy.get();
+        BulkIndexByScrollResponse response = copy.get();
         assertThat(response, reindexResponseMatcher()
                 .batches(1)
                 .failures(both(greaterThan(0)).and(lessThanOrEqualTo(maximumNumberOfShards()))));
@@ -77,7 +77,7 @@ public class ReindexFailureTests extends ReindexTestCase {
         // CREATE will cause the conflict to prevent the write.
         copy.destination().setOpType(CREATE);
 
-        ReindexResponse response = copy.get();
+        BulkIndexByScrollResponse response = copy.get();
         assertThat(response, reindexResponseMatcher().batches(1).versionConflicts(1).failures(1).created(99));
         for (Failure failure: response.getIndexingFailures()) {
             assertThat(failure.getMessage(), containsString("VersionConflictEngineException[[test]["));
@@ -99,7 +99,7 @@ public class ReindexFailureTests extends ReindexTestCase {
             indexDocs(100);
             ReindexRequestBuilder copy = reindex().source("source").destination("dest");
             copy.source().setSize(10);
-            Future<ReindexResponse> response = copy.execute();
+            Future<BulkIndexByScrollResponse> response = copy.execute();
             client().admin().indices().prepareDelete("source").get();
 
             try {
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java
index 186dc76804f..e82dd12a407 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java
@@ -26,7 +26,7 @@ import org.elasticsearch.index.mapper.internal.RoutingFieldMapper;
 /**
  * Index-by-search test for ttl, timestamp, and routing.
  */
-public class ReindexMetadataTests extends AbstractAsyncBulkIndexbyScrollActionMetadataTestCase<ReindexRequest, ReindexResponse> {
+public class ReindexMetadataTests extends AbstractAsyncBulkIndexbyScrollActionMetadataTestCase<ReindexRequest, BulkIndexByScrollResponse> {
     public void testRoutingCopiedByDefault() throws Exception {
         IndexRequest index = new IndexRequest();
         action().copyMetadata(index, doc(RoutingFieldMapper.NAME, "foo"));
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java
index a21a16deb0f..b805dbd2772 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java
@@ -30,7 +30,7 @@ import static org.hamcrest.Matchers.containsString;
 /**
  * Tests index-by-search with a script modifying the documents.
  */
-public class ReindexScriptTests extends AbstractAsyncBulkIndexByScrollActionScriptTestCase<ReindexRequest, ReindexResponse> {
+public class ReindexScriptTests extends AbstractAsyncBulkIndexByScrollActionScriptTestCase<ReindexRequest, BulkIndexByScrollResponse> {
     public void testSetIndex() throws Exception {
         Object dest = randomFrom(new Object[] {234, 234L, "pancake"});
         IndexRequest index = applyScript((Map<String, Object> ctx) -> ctx.put("_index", dest));
@@ -133,7 +133,7 @@ public class ReindexScriptTests extends AbstractAsyncBulkIndexByScrollActionScri
     }
 
     @Override
-    protected AbstractAsyncBulkIndexByScrollAction<ReindexRequest, ReindexResponse> action() {
+    protected AbstractAsyncBulkIndexByScrollAction<ReindexRequest> action() {
         return new TransportReindexAction.AsyncIndexBySearchAction(task, logger, null, null, null, threadPool, request(), listener());
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java
index d5426723b85..00956ac5982 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java
@@ -58,7 +58,7 @@ public abstract class ReindexTestCase extends ESIntegTestCase {
     }
 
     protected static class IndexBySearchResponseMatcher
-            extends AbstractBulkIndexByScrollResponseMatcher<ReindexResponse, IndexBySearchResponseMatcher> {
+            extends AbstractBulkIndexByScrollResponseMatcher<BulkIndexByScrollResponse, IndexBySearchResponseMatcher> {
         private Matcher<Long> createdMatcher = equalTo(0L);
 
         public IndexBySearchResponseMatcher created(Matcher<Long> updatedMatcher) {
@@ -71,7 +71,7 @@ public abstract class ReindexTestCase extends ESIntegTestCase {
         }
 
         @Override
-        protected boolean matchesSafely(ReindexResponse item) {
+        protected boolean matchesSafely(BulkIndexByScrollResponse item) {
             return super.matchesSafely(item) && createdMatcher.matches(item.getCreated());
         }
 
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexVersioningTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexVersioningTests.java
index 0ce5b8740f7..82c0e0af25f 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexVersioningTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexVersioningTests.java
@@ -88,7 +88,7 @@ public class ReindexVersioningTests extends ReindexTestCase {
     /**
      * Perform a reindex with EXTERNAL versioning which has "refresh" semantics.
      */
-    private ReindexResponse reindexExternal() {
+    private BulkIndexByScrollResponse reindexExternal() {
         ReindexRequestBuilder reindex =  reindex().source("source").destination("dest").abortOnVersionConflict(false);
         reindex.destination().setVersionType(EXTERNAL);
         return reindex.get();
@@ -97,7 +97,7 @@ public class ReindexVersioningTests extends ReindexTestCase {
     /**
      * Perform a reindex with INTERNAL versioning which has "overwrite" semantics.
      */
-    private ReindexResponse reindexInternal() {
+    private BulkIndexByScrollResponse reindexInternal() {
         ReindexRequestBuilder reindex =  reindex().source("source").destination("dest").abortOnVersionConflict(false);
         reindex.destination().setVersionType(INTERNAL);
         return reindex.get();
@@ -106,7 +106,7 @@ public class ReindexVersioningTests extends ReindexTestCase {
     /**
      * Perform a reindex with CREATE OpType which has "create" semantics.
      */
-    private ReindexResponse reindexCreate() {
+    private BulkIndexByScrollResponse reindexCreate() {
         ReindexRequestBuilder reindex =  reindex().source("source").destination("dest").abortOnVersionConflict(false);
         reindex.destination().setOpType(CREATE);
         return reindex.get();
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
index cbbe77e6479..55a665cfa7e 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
@@ -106,9 +106,9 @@ public class RoundTripTests extends ESTestCase {
     }
 
     public void testReindexResponse() throws IOException {
-        ReindexResponse response = new ReindexResponse(timeValueMillis(randomPositiveLong()), randomStatus(), randomIndexingFailures(),
-                randomSearchFailures(), randomBoolean());
-        ReindexResponse tripped = new ReindexResponse();
+        BulkIndexByScrollResponse response = new BulkIndexByScrollResponse(timeValueMillis(randomPositiveLong()), randomStatus(),
+                randomIndexingFailures(), randomSearchFailures(), randomBoolean());
+        BulkIndexByScrollResponse tripped = new BulkIndexByScrollResponse();
         roundTrip(response, tripped);
         assertResponseEquals(response, tripped);
     }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java
index 7a75759bfda..62f405cb0cd 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java
@@ -49,7 +49,7 @@ public class UpdateByQueryWithScriptTests
     }
 
     @Override
-    protected AbstractAsyncBulkIndexByScrollAction<UpdateByQueryRequest, BulkIndexByScrollResponse> action() {
+    protected AbstractAsyncBulkIndexByScrollAction<UpdateByQueryRequest> action() {
         return new TransportUpdateByQueryAction.AsyncIndexBySearchAction(task, logger, null, null, threadPool, null, request(), listener());
     }
 }

From 3f3fa594062b3748cf9de6488345eafdbb55dfc4 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 9 May 2016 12:25:52 -0400
Subject: [PATCH 0131/1311] Account for rpm behavior difference on directories

This commit modifies the packaging tests to account for the fact that
rpm behaves differently with respect to preserving directories marked as
"CONFIG | NOREPLACE" on older versions versus newer versions. Older
versions will leave the directory as-is while newer versions will append
the suffix ".rpmsave" to the directory name.

Relates #18216
---
 .../test/resources/packaging/scripts/40_rpm_package.bats    | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats b/qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats
index 0d3170cafeb..0555f55cba6 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats
+++ b/qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats
@@ -161,8 +161,10 @@ setup() {
     assert_file_exist "/etc/elasticsearch/jvm.options.rpmsave"
     assert_file_not_exist "/etc/elasticsearch/logging.yml"
     assert_file_exist "/etc/elasticsearch/logging.yml.rpmsave"
-    assert_file_exist "/etc/elasticsearch/scripts.rpmsave"
-    assert_file_exist "/etc/elasticsearch/scripts.rpmsave/script"
+    # older versions of rpm behave differently and preserve the
+    # directory but do not append the ".rpmsave" suffix
+    test -e "/etc/elasticsearch/scripts" || test -e "/etc/elasticsearch/scripts.rpmsave"
+    test -e "/etc/elasticsearch/scripts/script" || test -e "/etc/elasticsearch/scripts.rpmsave/script"
 
     assert_file_not_exist "/etc/init.d/elasticsearch"
     assert_file_not_exist "/usr/lib/systemd/system/elasticsearch.service"

From b7d02fbd1e19b3f3114a1162c92e1583b7a9d832 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 9 May 2016 10:54:40 -0400
Subject: [PATCH 0132/1311] Improve logging of raw rest actions on failure

Log the method and the path.
---
 .../java/org/elasticsearch/test/rest/section/DoSection.java | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/DoSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/DoSection.java
index cdbf65f187d..7b392a4043e 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/DoSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/DoSection.java
@@ -129,7 +129,11 @@ public class DoSection implements ExecutableSection {
     }
 
     private String formatStatusCodeMessage(RestResponse restResponse, String expected) {
-        return "expected [" + expected + "] status code but api [" + apiCallSection.getApi() + "] returned ["
+        String api = apiCallSection.getApi();
+        if ("raw".equals(api)) {
+            api += "[method=" + apiCallSection.getParams().get("method") + " path=" + apiCallSection.getParams().get("path") + "]";
+        }
+        return "expected [" + expected + "] status code but api [" + api + "] returned ["
                 + restResponse.getStatusCode() + " " + restResponse.getReasonPhrase() + "] [" + restResponse.getBodyAsString() + "]";
     }
 

From 7d1fd17172f5f8cb7ed381de1ff23dfb9f1856a5 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 9 May 2016 13:06:18 -0400
Subject: [PATCH 0133/1311] Remove plugin script parsing of system properties

The plugin script parses command-line options looking for Java system
properties and extracts these arguments to pass to the java command when
starting the JVM. Since elasticsearch-plugin allows arbitrary user
arguments to the JVM via ES_JAVA_OPTS, this parsing is unnecessary. This
commit removes this unnecessary

Relates #18207
---
 dev-tools/prepare_release_candidate.py        |  8 ++++----
 .../main/resources/bin/elasticsearch-plugin   | 19 +------------------
 docs/plugins/plugin-script.asciidoc           |  4 ++--
 .../migration/migrate_5_0/plugins.asciidoc    |  6 ++++++
 .../scripts/module_and_plugin_test_cases.bash |  2 +-
 5 files changed, 14 insertions(+), 25 deletions(-)

diff --git a/dev-tools/prepare_release_candidate.py b/dev-tools/prepare_release_candidate.py
index 450106a3bc9..be5660c60ae 100644
--- a/dev-tools/prepare_release_candidate.py
+++ b/dev-tools/prepare_release_candidate.py
@@ -60,10 +60,10 @@ Plugins can be installed as follows:
 
 The same goes for the x-plugins:
 
-    bin/elasticsearch-plugin -Des.plugins.staging=true install license
-    bin/elasticsearch-plugin -Des.plugins.staging=true install marvel-agent
-    bin/elasticsearch-plugin -Des.plugins.staging=true install shield
-    bin/elasticsearch-plugin -Des.plugins.staging=true install watcher
+    ES_JAVA_OPTS="-Des.plugins.staging=true" bin/elasticsearch-plugin install license
+    ES_JAVA_OPTS="-Des.plugins.staging=true" bin/elasticsearch-plugin install marvel-agent
+    ES_JAVA_OPTS="-Des.plugins.staging=true" bin/elasticsearch-plugin install shield
+    ES_JAVA_OPTS="-Des.plugins.staging=true" bin/elasticsearch-plugin install watcher
 
 To install the deb from an APT repo:
 
diff --git a/distribution/src/main/resources/bin/elasticsearch-plugin b/distribution/src/main/resources/bin/elasticsearch-plugin
index eec162dab94..b337d9d0273 100755
--- a/distribution/src/main/resources/bin/elasticsearch-plugin
+++ b/distribution/src/main/resources/bin/elasticsearch-plugin
@@ -77,23 +77,6 @@ if [ ! -x "$JAVA" ]; then
     exit 1
 fi
 
-# real getopt cannot be used because we need to hand options over to the PluginManager
-while [ $# -gt 0 ]; do
-  case $1 in
-    -D*=*)
-      properties="$properties \"$1\""
-      ;;
-    -D*)
-      var=$1
-      shift
-      properties="$properties \"$var\"=\"$1\""
-      ;;
-    *)
-      args="$args \"$1\""
-  esac
-  shift
-done
-
 # check if properties already has a config file or config dir
 if [ -e "$CONF_DIR" ]; then
   case "$properties" in
@@ -110,4 +93,4 @@ fi
 HOSTNAME=`hostname | cut -d. -f1`
 export HOSTNAME
 
-eval "\"$JAVA\"" "$ES_JAVA_OPTS" -Delasticsearch -Des.path.home="\"$ES_HOME\"" $properties -cp "\"$ES_HOME/lib/*\"" org.elasticsearch.plugins.PluginCli $args
+eval "\"$JAVA\"" "$ES_JAVA_OPTS" -Delasticsearch -Des.path.home="\"$ES_HOME\"" $properties -cp "\"$ES_HOME/lib/*\"" org.elasticsearch.plugins.PluginCli "$@"
diff --git a/docs/plugins/plugin-script.asciidoc b/docs/plugins/plugin-script.asciidoc
index d51b31726a0..7cb7f396608 100644
--- a/docs/plugins/plugin-script.asciidoc
+++ b/docs/plugins/plugin-script.asciidoc
@@ -77,7 +77,7 @@ to a local Java truststore and pass the location to the script as follows:
 
 [source,shell]
 -----------------------------------
-sudo bin/elasticsearch-plugin -Djavax.net.ssl.trustStore=/path/to/trustStore.jks install https://....
+sudo ES_JAVA_OPTS="-Djavax.net.ssl.trustStore=/path/to/trustStore.jks" bin/elasticsearch-plugin install https://....
 -----------------------------------
 
 [[listing-removing]]
@@ -169,7 +169,7 @@ options can be set on the command line:
 
 [source,shell]
 -----------------------------------
-sudo bin/elasticsearch-plugin install mobz/elasticsearch-head -DproxyHost=host_name -DproxyPort=port_number
+sudo ES_JAVA_OPTS="-DproxyHost=host_name -DproxyPort=port_number" bin/elasticsearch-plugin install mobz/elasticsearch-head
 -----------------------------------
 
 On Windows, they need to be added to the `ES_JAVA_OPTS` environment variable:
diff --git a/docs/reference/migration/migrate_5_0/plugins.asciidoc b/docs/reference/migration/migrate_5_0/plugins.asciidoc
index 46014e3bcd4..1ed45b9b596 100644
--- a/docs/reference/migration/migrate_5_0/plugins.asciidoc
+++ b/docs/reference/migration/migrate_5_0/plugins.asciidoc
@@ -101,3 +101,9 @@ Cloud GCE plugin has been renamed to {plugins}/discovery-gce.html[Discovery GCE
 Mapper attachments has been deprecated. Users should use now the {plugins}/ingest-attachment.html[`ingest-attachment`]
 plugin.
 
+==== Passing of Java System Properties
+
+Previously, Java system properties could be passed to the plugin
+command by passing `-D` style arguments directly to the plugin script.
+This is no longer permitted and such system properties must be passed
+via ES_JAVA_OPTS.
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
index 85d829718e2..bd5cd499015 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
@@ -438,7 +438,7 @@ fi
     remove_jvm_example
 
     local relativePath=${1:-$(readlink -m jvm-example-*.zip)}
-    sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/elasticsearch-plugin" install "file://$relativePath" -Des.logger.level=DEBUG > /tmp/plugin-cli-output
+    sudo -E -u $ESPLUGIN_COMMAND_USER ES_JAVA_OPTS="-Des.logger.level=DEBUG" "$ESHOME/bin/elasticsearch-plugin" install "file://$relativePath" > /tmp/plugin-cli-output
     local loglines=$(cat /tmp/plugin-cli-output | wc -l)
     if [ "$GROUP" == "TAR PLUGINS" ]; then
         [ "$loglines" -gt "3" ] || {

From 6094704246f63c9be19dea9fc07ea853cfbabe17 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Mon, 9 May 2016 10:17:47 -0700
Subject: [PATCH 0134/1311] Remove IDE settings as they are now pulled in
 through the virtual build-tools project

---
 buildSrc/build.gradle | 30 +-----------------------------
 1 file changed, 1 insertion(+), 29 deletions(-)

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
index be83d525f5f..b286124bfee 100644
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -91,15 +91,11 @@ dependencies {
 
 
 /*****************************************************************************
- *                    Bootstrap repositories and IDE setup                   *
+ *                         Bootstrap repositories                            *
  *****************************************************************************/
 // this will only happen when buildSrc is built on its own during build init
 if (project == rootProject) {
 
-  // TODO: move common IDE configuration to a common file to include
-  apply plugin: 'idea'
-  apply plugin: 'eclipse'
-
   repositories {
     mavenCentral()
     maven {
@@ -107,30 +103,6 @@ if (project == rootProject) {
       url "https://oss.sonatype.org/content/repositories/snapshots/"
     }
   }
-
-  idea {
-    module {
-      inheritOutputDirs = false
-      outputDir = file('build-idea/classes/main')
-      testOutputDir = file('build-idea/classes/test')
-    }
-  }
-
-  eclipse {
-    classpath {
-      defaultOutputDir = file('build-eclipse')
-    }
-  }
-
-  task copyEclipseSettings(type: Copy) {
-    from project.file('src/main/resources/eclipse.settings')
-    into '.settings'
-  }
-  // otherwise .settings is not nuked entirely
-  tasks.cleanEclipse {
-    delete '.settings'
-  }
-  tasks.eclipse.dependsOn(cleanEclipse, copyEclipseSettings)
 }
 
 /*****************************************************************************

From aef78ceb13deaf5a15c15f3b58ed46ec63d86fb5 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Mon, 9 May 2016 16:42:02 +0200
Subject: [PATCH 0135/1311] Do not return fieldstats information for fields
 that exist in the mapping but not in the index.

---
 .../TransportFieldStatsTransportAction.java   |  6 ++-
 .../index/mapper/MappedFieldType.java         |  2 +-
 .../index/mapper/core/DateFieldMapper.java    |  2 +-
 .../mapper/core/LegacyByteFieldMapper.java    |  2 +-
 .../mapper/core/LegacyDateFieldMapper.java    |  2 +-
 .../mapper/core/LegacyDoubleFieldMapper.java  |  2 +-
 .../mapper/core/LegacyFloatFieldMapper.java   |  2 +-
 .../mapper/core/LegacyIntegerFieldMapper.java |  2 +-
 .../mapper/core/LegacyLongFieldMapper.java    |  3 +-
 .../mapper/core/LegacyShortFieldMapper.java   |  2 +-
 .../index/mapper/core/NumberFieldMapper.java  |  8 +--
 .../index/mapper/ip/IpFieldMapper.java        |  2 +-
 .../index/mapper/ip/LegacyIpFieldMapper.java  |  2 +-
 .../fieldstats/FieldStatsIntegrationIT.java   | 49 +++++++++++++------
 .../fieldstats/FieldStatsTests.java           | 40 +++++----------
 .../test/field_stats/10_basics.yaml           |  2 +-
 16 files changed, 66 insertions(+), 62 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java b/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java
index fbff98bbf4c..4d2d1db161d 100644
--- a/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java
@@ -117,11 +117,13 @@ public class TransportFieldStatsTransportAction extends
                     if (existing != null) {
                         if (existing.getType() != entry.getValue().getType()) {
                             if (conflicts.containsKey(entry.getKey()) == false) {
+                                FieldStats[] fields = new FieldStats[] {entry.getValue(), existing};
+                                Arrays.sort(fields, (o1, o2) -> Byte.compare(o1.getType(), o2.getType()));
                                 conflicts.put(entry.getKey(),
                                     "Field [" + entry.getKey() + "] of type [" +
-                                        FieldStats.typeName(entry.getValue().getType()) +
+                                        FieldStats.typeName(fields[0].getType()) +
                                         "] conflicts with existing field of type [" +
-                                        FieldStats.typeName(existing.getType()) +
+                                        FieldStats.typeName(fields[1].getType()) +
                                         "] in other index.");
                             }
                         } else {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
index 1a63a4dfb27..3f8dc0e142c 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
@@ -404,7 +404,7 @@ public abstract class MappedFieldType extends FieldType {
         int maxDoc = reader.maxDoc();
         Terms terms = MultiFields.getTerms(reader, name());
         if (terms == null) {
-            return new FieldStats.Text(maxDoc, isSearchable(), isAggregatable());
+            return null;
         }
         FieldStats stats = new FieldStats.Text(maxDoc, terms.getDocCount(),
             terms.getSumDocFreq(), terms.getSumTotalTermFreq(),
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
index 89e2d8409d5..8b744ce97da 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
@@ -399,7 +399,7 @@ public class DateFieldMapper extends FieldMapper implements AllFieldMapper.Inclu
             String field = name();
             long size = XPointValues.size(reader, field);
             if (size == 0) {
-                return new FieldStats.Date(reader.maxDoc(), isSearchable(), isAggregatable(), dateTimeFormatter());
+                return null;
             }
             int docCount = XPointValues.getDocCount(reader, field);
             byte[] min = XPointValues.getMinPackedValue(reader, field);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyByteFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyByteFieldMapper.java
index 5c6712a31a1..216fd7d359f 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyByteFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyByteFieldMapper.java
@@ -174,7 +174,7 @@ public class LegacyByteFieldMapper extends LegacyNumberFieldMapper {
             int maxDoc = reader.maxDoc();
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return new FieldStats.Long(maxDoc, isSearchable(), isAggregatable());
+                return null;
             }
             long minValue = LegacyNumericUtils.getMinInt(terms);
             long maxValue = LegacyNumericUtils.getMaxInt(terms);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDateFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDateFieldMapper.java
index 10a6b4a2ecd..13d1cd23ffd 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDateFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDateFieldMapper.java
@@ -379,7 +379,7 @@ public class LegacyDateFieldMapper extends LegacyNumberFieldMapper {
             int maxDoc = reader.maxDoc();
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return new FieldStats.Date(maxDoc, isSearchable(), isAggregatable(), dateTimeFormatter());
+                return null;
             }
             long minValue = LegacyNumericUtils.getMinLong(terms);
             long maxValue = LegacyNumericUtils.getMaxLong(terms);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDoubleFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDoubleFieldMapper.java
index 3a3c506a3ef..50135c0a66d 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDoubleFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDoubleFieldMapper.java
@@ -185,7 +185,7 @@ public class LegacyDoubleFieldMapper extends LegacyNumberFieldMapper {
             int maxDoc = reader.maxDoc();
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return new FieldStats.Double(maxDoc, isSearchable(), isAggregatable());
+                return null;
             }
             double minValue = NumericUtils.sortableLongToDouble(LegacyNumericUtils.getMinLong(terms));
             double maxValue = NumericUtils.sortableLongToDouble(LegacyNumericUtils.getMaxLong(terms));
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyFloatFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyFloatFieldMapper.java
index a5dbc98fa7d..5e45cf4359d 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyFloatFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyFloatFieldMapper.java
@@ -170,7 +170,7 @@ public class LegacyFloatFieldMapper extends LegacyNumberFieldMapper {
             int maxDoc = reader.maxDoc();
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return new FieldStats.Double(maxDoc, isSearchable(), isAggregatable());
+                return null;
             }
             float minValue = NumericUtils.sortableIntToFloat(LegacyNumericUtils.getMinInt(terms));
             float maxValue = NumericUtils.sortableIntToFloat(LegacyNumericUtils.getMaxInt(terms));
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyIntegerFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyIntegerFieldMapper.java
index b2b95950436..be4d8cad261 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyIntegerFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyIntegerFieldMapper.java
@@ -174,7 +174,7 @@ public class LegacyIntegerFieldMapper extends LegacyNumberFieldMapper {
             int maxDoc = reader.maxDoc();
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return new FieldStats.Long(maxDoc, isSearchable(), isAggregatable());
+                return null;
             }
             long minValue = LegacyNumericUtils.getMinInt(terms);
             long maxValue = LegacyNumericUtils.getMaxInt(terms);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyLongFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyLongFieldMapper.java
index b8fb3c7fcef..1601e009824 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyLongFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyLongFieldMapper.java
@@ -173,8 +173,7 @@ public class LegacyLongFieldMapper extends LegacyNumberFieldMapper {
             int maxDoc = reader.maxDoc();
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return new FieldStats.Long(
-                    maxDoc, isSearchable(), isAggregatable());
+                return null;
             }
             long minValue = LegacyNumericUtils.getMinLong(terms);
             long maxValue = LegacyNumericUtils.getMaxLong(terms);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyShortFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyShortFieldMapper.java
index ff85e30be2e..32c1955fc50 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyShortFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyShortFieldMapper.java
@@ -178,7 +178,7 @@ public class LegacyShortFieldMapper extends LegacyNumberFieldMapper {
             int maxDoc = reader.maxDoc();
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return new FieldStats.Long(maxDoc, isSearchable(), isAggregatable());
+                return null;
             }
             long minValue = LegacyNumericUtils.getMinInt(terms);
             long maxValue = LegacyNumericUtils.getMaxInt(terms);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
index f1088652988..4717140277b 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
@@ -262,7 +262,7 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
                                     boolean isSearchable, boolean isAggregatable) throws IOException {
                 long size = XPointValues.size(reader, fieldName);
                 if (size == 0) {
-                    return new FieldStats.Double(reader.maxDoc(), isSearchable, isAggregatable);
+                    return null;
                 }
                 int docCount = XPointValues.getDocCount(reader, fieldName);
                 byte[] min = XPointValues.getMinPackedValue(reader, fieldName);
@@ -353,7 +353,7 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
                                     boolean isSearchable, boolean isAggregatable) throws IOException {
                 long size = XPointValues.size(reader, fieldName);
                 if (size == 0) {
-                    return new FieldStats.Double(reader.maxDoc(), isSearchable, isAggregatable);
+                    return null;
                 }
                 int docCount = XPointValues.getDocCount(reader, fieldName);
                 byte[] min = XPointValues.getMinPackedValue(reader, fieldName);
@@ -588,7 +588,7 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
                                   boolean isSearchable, boolean isAggregatable) throws IOException {
                 long size = XPointValues.size(reader, fieldName);
                 if (size == 0) {
-                    return new FieldStats.Long(reader.maxDoc(), isSearchable, isAggregatable);
+                    return null;
                 }
                 int docCount = XPointValues.getDocCount(reader, fieldName);
                 byte[] min = XPointValues.getMinPackedValue(reader, fieldName);
@@ -691,7 +691,7 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
                                   boolean isSearchable, boolean isAggregatable) throws IOException {
                 long size = XPointValues.size(reader, fieldName);
                 if (size == 0) {
-                    return new FieldStats.Long(reader.maxDoc(), isSearchable, isAggregatable);
+                    return null;
                 }
                 int docCount = XPointValues.getDocCount(reader, fieldName);
                 byte[] min = XPointValues.getMinPackedValue(reader, fieldName);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
index 0d9810bd405..8435bdeb681 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
@@ -232,7 +232,7 @@ public class IpFieldMapper extends FieldMapper implements AllFieldMapper.Include
             String field = name();
             long size = XPointValues.size(reader, field);
             if (size == 0) {
-                return new FieldStats.Ip(reader.maxDoc(), isSearchable(), isAggregatable());
+                return null;
             }
             int docCount = XPointValues.getDocCount(reader, field);
             byte[] min = XPointValues.getMinPackedValue(reader, field);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpFieldMapper.java
index 583ad3cee2f..aef6869ffc3 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpFieldMapper.java
@@ -252,7 +252,7 @@ public class LegacyIpFieldMapper extends LegacyNumberFieldMapper {
             int maxDoc = reader.maxDoc();
             Terms terms = org.apache.lucene.index.MultiFields.getTerms(reader, name());
             if (terms == null) {
-                return new FieldStats.Ip(maxDoc, isSearchable(), isAggregatable());
+                return null;
             }
             long minValue = LegacyNumericUtils.getMinLong(terms);
             long maxValue = LegacyNumericUtils.getMaxLong(terms);
diff --git a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java
index faa9489e033..d04bc28936c 100644
--- a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java
@@ -246,7 +246,7 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
         assertThat(response.getIndicesMergedFieldStats().get("_all").size(), equalTo(0));
         assertThat(response.getConflicts().size(), equalTo(1));
         assertThat(response.getConflicts().get("value"),
-            equalTo("Field [value] of type [text] conflicts with existing field of type [whole-number] " +
+            equalTo("Field [value] of type [whole-number] conflicts with existing field of type [text] " +
                 "in other index."));
 
         response = client().prepareFieldStats().setFields("value").setLevel("indices").get();
@@ -260,7 +260,6 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
             equalTo(new BytesRef("b")));
     }
 
-    @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/LUCENE-7257")
     public void testIncompatibleFieldTypesMultipleFields() {
         assertAcked(prepareCreate("test1").addMapping(
                 "test", "value", "type=long", "value2", "type=long"
@@ -284,7 +283,7 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
         assertThat(response.getIndicesMergedFieldStats().get("_all").get("value2").getMaxValue(), equalTo(1L));
         assertThat(response.getConflicts().size(), equalTo(1));
         assertThat(response.getConflicts().get("value"),
-            equalTo("Field [value] of type [text] conflicts with existing field of type [whole-number] " +
+            equalTo("Field [value] of type [whole-number] conflicts with existing field of type [text] " +
                 "in other index."));
 
         response = client().prepareFieldStats().setFields("value", "value2").setLevel("indices").get();
@@ -418,19 +417,32 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
 
     public void testWildcardFields() throws Exception {
         assertAcked(prepareCreate("test1").addMapping(
-            "test", "foo", "type=long", "foobar", "type=text", "barfoo", "type=long"
+            "test", "foo", "type=long", "foobar", "type=long", "barfoo", "type=long"
         ));
         assertAcked(prepareCreate("test2").addMapping(
-            "test", "foobar", "type=text", "barfoo", "type=long"
+            "test", "foobar", "type=long", "barfoo", "type=long"
         ));
         ensureGreen("test1", "test2");
         FieldStatsResponse response = client().prepareFieldStats()
             .setFields("foo*")
             .get();
         assertAllSuccessful(response);
+        assertThat(response.getAllFieldStats().size(), equalTo(0));
+
+        indexRange("test1", "foo", -100, 0);
+        indexRange("test2", "foo", -10, 100);
+        indexRange("test1", "foobar", -10, 100);
+        indexRange("test2", "foobar", -100, 0);
+
+        response = client().prepareFieldStats()
+            .setFields("foo*")
+            .get();
+        assertAllSuccessful(response);
         assertThat(response.getAllFieldStats().size(), equalTo(2));
-        assertThat(response.getAllFieldStats().get("foo").getMinValue(), nullValue());
-        assertThat(response.getAllFieldStats().get("foobar").getMaxValue(), nullValue());
+        assertThat(response.getAllFieldStats().get("foo").getMinValue(), equalTo(-100L));
+        assertThat(response.getAllFieldStats().get("foo").getMaxValue(), equalTo(100L));
+        assertThat(response.getAllFieldStats().get("foobar").getMinValue(), equalTo(-100L));
+        assertThat(response.getAllFieldStats().get("foobar").getMaxValue(), equalTo(100L));
 
         response = client().prepareFieldStats()
             .setFields("foo*")
@@ -439,21 +451,26 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
         assertAllSuccessful(response);
         assertThat(response.getIndicesMergedFieldStats().size(), equalTo(2));
         assertThat(response.getIndicesMergedFieldStats().get("test1").size(), equalTo(2));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("foo").getMinValue(), nullValue());
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("foo").getMaxValue(), nullValue());
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("foobar").getMinValue(), nullValue());
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("foobar").getMaxValue(), nullValue());
-        assertThat(response.getIndicesMergedFieldStats().get("test2").size(), equalTo(1));
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("foobar").getMinValue(), nullValue());
-        assertThat(response.getIndicesMergedFieldStats().get("test2").get("foobar").getMaxValue(), nullValue());
+        assertThat(response.getIndicesMergedFieldStats().get("test1").get("foo").getMinValue(), equalTo(-100L));
+        assertThat(response.getIndicesMergedFieldStats().get("test1").get("foo").getMaxValue(), equalTo(0L));
+        assertThat(response.getIndicesMergedFieldStats().get("test1").get("foobar").getMinValue(), equalTo(-10L));
+        assertThat(response.getIndicesMergedFieldStats().get("test1").get("foobar").getMaxValue(), equalTo(100L));
+        assertThat(response.getIndicesMergedFieldStats().get("test2").size(), equalTo(2));
+        assertThat(response.getIndicesMergedFieldStats().get("test2").get("foobar").getMinValue(), equalTo(-100L));
+        assertThat(response.getIndicesMergedFieldStats().get("test2").get("foobar").getMaxValue(), equalTo(0L));
+        assertThat(response.getIndicesMergedFieldStats().get("test2").get("foo").getMinValue(), equalTo(-10L));
+        assertThat(response.getIndicesMergedFieldStats().get("test2").get("foo").getMaxValue(), equalTo(100L));
     }
 
     private void indexRange(String index, long from, long to) throws Exception {
+        indexRange(index, "value", from, to);
+    }
+
+    private void indexRange(String index, String field, long from, long to) throws Exception {
         List<IndexRequestBuilder> requests = new ArrayList<>();
         for (long value = from; value <= to; value++) {
-            requests.add(client().prepareIndex(index, "test").setSource("value", value));
+            requests.add(client().prepareIndex(index, "test").setSource(field, value));
         }
         indexRandom(true, false, requests);
     }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java
index 01d129990cb..57f0f001a83 100644
--- a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java
@@ -452,37 +452,23 @@ public class FieldStatsTests extends ESSingleNodeTestCase {
     public void testEmptyIndex() {
         createIndex("test1", Settings.EMPTY, "type", "value", "type=date");
         FieldStatsResponse response = client().prepareFieldStats()
-                .setFields("value")
-                .setLevel("indices")
-                .get();
+            .setFields("*")
+            .setLevel("indices")
+            .get();
         assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").size(), equalTo(1));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMaxDoc(), equalTo(0L));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getDocCount(), equalTo(0L));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getSumDocFreq(), equalTo(0L));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getSumTotalTermFreq(), equalTo(0L));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").isSearchable(), equalTo(true));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").isAggregatable(), equalTo(true));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValue(), equalTo(null));
-        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMaxValue(), equalTo(null));
-
-        response = client().prepareFieldStats()
-                .setFields("value")
-                .setIndexContraints(new IndexConstraint("value", MIN, GTE, "1998-01-01T00:00:00.000Z"))
-                .setLevel("indices")
-                .get();
-        assertThat(response.getIndicesMergedFieldStats().size(), equalTo(0));
+        assertThat(response.getIndicesMergedFieldStats().get("test1").size(), equalTo(0));
     }
 
-    public void testMetaFieldsSearchable() {
-        createIndex("test1", Settings.EMPTY, "type", "value", "type=date");
+    public void testMetaFieldsNotIndexed() {
+        createIndex("test", Settings.EMPTY);
+        client().prepareIndex("test", "type").setSource().get();
+        client().admin().indices().prepareRefresh().get();
+
         FieldStatsResponse response = client().prepareFieldStats()
-            .setFields("_id", "_index")
+            .setFields("_id", "_type")
             .get();
-        assertThat(response.getAllFieldStats().size(), equalTo(2));
-        assertThat(response.getAllFieldStats().get("_id").isSearchable(), equalTo(true));
-        assertThat(response.getAllFieldStats().get("_index").isSearchable(), equalTo(true));
-        assertThat(response.getAllFieldStats().get("_id").isAggregatable(), equalTo(false));
-        assertThat(response.getAllFieldStats().get("_index").isAggregatable(), equalTo(true));
+        assertThat(response.getAllFieldStats().size(), equalTo(1));
+        assertThat(response.getAllFieldStats().get("_type").isSearchable(), equalTo(true));
+        // assertThat(response.getAllFieldStats().get("_type").isAggregatable(), equalTo(true));
     }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml
index fd8621f7594..eb1d1c6f758 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml
@@ -164,5 +164,5 @@ setup:
   - match: { indices._all.fields.number.min_value_as_string: "123" }
   - match: { indices._all.fields.number.max_value: 456 }
   - match: { indices._all.fields.number.max_value_as_string: "456" }
-  - match: { conflicts.bar: "Field [bar] of type [text] conflicts with existing field of type [whole-number] in other index." }
+  - match: { conflicts.bar: "Field [bar] of type [whole-number] conflicts with existing field of type [text] in other index." }
   - is_false: indices._all.fields.bar

From ddc531e729d403247d6d53512a7b004ebceb6799 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 5 May 2016 16:46:40 -0400
Subject: [PATCH 0136/1311] Build a plugin for testing docs

This makes it much easier to apply to other projects.

Fixes to doc tests infrastructure:
* Fix comparing lists. Was totally broken.
* Fix order of actual vs expected parameters.
* Allow multiple `// TESTRESPONSE` lines with substitutions to join
into one big list of subtitutions. This makes lets the docs look
tidier.
* Exclude build from snippet scanning
* Allow subclasses of ESRestTestCase access to the admin execution context
---
 .../gradle/doc/DocsTestPlugin.groovy          | 65 +++++++++++++++++
 .../RestTestsFromSnippetsTask.groovy          |  4 +-
 .../gradle/{ => doc}/SnippetsTask.groovy      |  8 ++-
 .../elasticsearch.docs-test.properties        | 20 ++++++
 docs/build.gradle                             | 70 ++++++-------------
 .../test/rest/ESRestTestCase.java             |  4 ++
 .../rest/section/ResponseBodyAssertion.java   | 13 ++--
 7 files changed, 125 insertions(+), 59 deletions(-)
 create mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy
 rename buildSrc/src/main/groovy/org/elasticsearch/gradle/{ => doc}/RestTestsFromSnippetsTask.groovy (98%)
 rename buildSrc/src/main/groovy/org/elasticsearch/gradle/{ => doc}/SnippetsTask.groovy (97%)
 create mode 100644 buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.docs-test.properties

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy
new file mode 100644
index 00000000000..3c3ad15a45b
--- /dev/null
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy
@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.doc
+
+import org.elasticsearch.gradle.test.RestTestPlugin
+import org.gradle.api.Project
+import org.gradle.api.Task
+
+/**
+ * Sets up tests for documentation.
+ */
+public class DocsTestPlugin extends RestTestPlugin {
+
+    @Override
+    public void apply(Project project) {
+        super.apply(project)
+        Task listSnippets = project.tasks.create('listSnippets', SnippetsTask)
+        listSnippets.group 'Docs'
+        listSnippets.description 'List each snippet'
+        listSnippets.perSnippet { println(it.toString()) }
+
+        Task listConsoleCandidates = project.tasks.create(
+                'listConsoleCandidates', SnippetsTask)
+        listConsoleCandidates.group 'Docs'
+        listConsoleCandidates.description
+                'List snippets that probably should be marked // CONSOLE'
+        listConsoleCandidates.perSnippet {
+            if (
+                       it.autoSense    // Already marked, nothing to do
+                    || it.testResponse // It is a response
+                ) {
+                return
+            }
+            List<String> languages = [
+                // These languages should almost always be marked autosense
+                'js', 'json',
+                // These are often curl commands that should be converted but
+                // are probably false positives
+                'sh', 'shell',
+            ]
+            if (false == languages.contains(it.language)) {
+                return
+            }
+            println(it.toString())
+        }
+
+        project.tasks.create('buildRestTests', RestTestsFromSnippetsTask)
+    }
+}
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/RestTestsFromSnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
similarity index 98%
rename from buildSrc/src/main/groovy/org/elasticsearch/gradle/RestTestsFromSnippetsTask.groovy
rename to buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
index 1ce8adf1ba4..2696f9ce661 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/RestTestsFromSnippetsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
@@ -17,9 +17,9 @@
  * under the License.
  */
 
-package org.elasticsearch.gradle
+package org.elasticsearch.gradle.doc
 
-import org.elasticsearch.gradle.SnippetsTask.Snippet
+import org.elasticsearch.gradle.doc.SnippetsTask.Snippet
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.OutputDirectory
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/SnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy
similarity index 97%
rename from buildSrc/src/main/groovy/org/elasticsearch/gradle/SnippetsTask.groovy
rename to buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy
index a2e6502690f..954505e65f2 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/SnippetsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.gradle
+package org.elasticsearch.gradle.doc
 
 import org.gradle.api.DefaultTask
 import org.gradle.api.InvalidUserDataException
@@ -55,6 +55,8 @@ public class SnippetsTask extends DefaultTask {
     ConfigurableFileTree docs = project.fileTree(project.projectDir) {
         // No snippets in the build file
         exclude 'build.gradle'
+        // That is where the snippets go, not where they come from!
+        exclude 'build'
     }
 
     @TaskAction
@@ -166,7 +168,9 @@ public class SnippetsTask extends DefaultTask {
                     }
                     snippet.testResponse = true
                     if (matcher.group(2) != null) {
-                        substitutions = []
+                        if (substitutions == null) {
+                            substitutions = []
+                        }
                         String loc = "$file:$lineNumber"
                         parse(loc, matcher.group(2), /$SUBSTITUTION ?/) {
                             substitutions.add([it.group(1), it.group(2)])
diff --git a/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.docs-test.properties b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.docs-test.properties
new file mode 100644
index 00000000000..fb264ff4fd0
--- /dev/null
+++ b/buildSrc/src/main/resources/META-INF/gradle-plugins/elasticsearch.docs-test.properties
@@ -0,0 +1,20 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+implementation-class=org.elasticsearch.gradle.doc.DocsTestPlugin
diff --git a/docs/build.gradle b/docs/build.gradle
index 3858ec1f9fb..f80ef1e1fd0 100644
--- a/docs/build.gradle
+++ b/docs/build.gradle
@@ -17,50 +17,27 @@
  * under the License.
  */
 
-import org.elasticsearch.gradle.SnippetsTask
-import org.elasticsearch.gradle.SnippetsTask.Snippet
-import org.elasticsearch.gradle.RestTestsFromSnippetsTask
+apply plugin: 'elasticsearch.docs-test'
 
-apply plugin: 'elasticsearch.rest-test'
-
-task listSnippets(type: SnippetsTask) {
-  group 'Docs'
-  description 'List each snippet'
-  perSnippet { println(it) }
-}
-
-task listAutoSenseCandidates(type: SnippetsTask) {
-  group 'Docs'
-  description 'List snippets that probably should be marked // CONSOLE'
-  perSnippet {
-    if (
-             it.autoSense    // Already marked, nothing to do
-          || it.testResponse // Only commands are autosense
-        ) {
-      return
-    }
-    List<String> languages = [
-      'js', 'json', // These languages should almost always be marked autosense
-      'sh', 'shell', // These are often curl commands that should be converted
-    ]
-    if (false == languages.contains(it.language)) {
-      return
-    }
-    println(it)
+integTest {
+  cluster {
+    setting 'script.inline', 'true'
   }
 }
 
-task buildRestTests(type: RestTestsFromSnippetsTask) {
-  docs = fileTree(project.projectDir) {
-    // No snippets in here!
-    exclude 'build.gradle'
-    // Remove plugins because they aren't installed during this test. Yet?
-    exclude 'plugins'
-    // This file simply doesn't pass yet. We should figure out how to fix it.
-    exclude 'reference/modules/snapshots.asciidoc'
-  }
-  Closure setupTwitter = { String name, int count ->
-    setups[name] = '''
+buildRestTests.docs = fileTree(projectDir) {
+  // No snippets in here!
+  exclude 'build.gradle'
+  // That is where the snippets go, not where they come from!
+  exclude 'build'
+  // Remove plugins because they aren't installed during this test. Yet?
+  exclude 'plugins'
+  // This file simply doesn't pass yet. We should figure out how to fix it.
+  exclude 'reference/modules/snapshots.asciidoc'
+}
+
+Closure setupTwitter = { String name, int count ->
+  buildRestTests.setups[name] = '''
   - do:
       bulk:
         index: twitter
@@ -68,17 +45,10 @@ task buildRestTests(type: RestTestsFromSnippetsTask) {
         refresh: true
         body: |'''
     for (int i = 0; i < count; i++) {
-      setups[name] += """
+      buildRestTests.setups[name] += """
           {"index":{}}
           {"msg": "some message with the number $i", "date": $i}"""
     }
   }
-  setupTwitter('twitter', 5)
-  setupTwitter('big_twitter', 120)
-}
-
-integTest {
-  cluster {
-    setting 'script.inline', 'true'
-  }
-}
+setupTwitter('twitter', 5)
+setupTwitter('big_twitter', 120)
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index fbc518b136d..ab2c1d2afbe 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -249,6 +249,10 @@ public abstract class ESRestTestCase extends ESTestCase {
         adminExecutionContext = new RestTestExecutionContext(restSpec);
     }
 
+    protected RestTestExecutionContext getAdminExecutionContext() {
+        return adminExecutionContext;
+    }
+
     private static void validateSpec(RestSpec restSpec) {
         boolean validateSpec = RandomizedTest.systemPropertyAsBoolean(REST_TESTS_VALIDATE_SPEC, true);
         if (validateSpec) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/ResponseBodyAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/ResponseBodyAssertion.java
index 4a878a77323..3ead65a2111 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/ResponseBodyAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/ResponseBodyAssertion.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 import java.util.TreeMap;
 
 import org.elasticsearch.common.Nullable;
@@ -69,7 +70,7 @@ public class ResponseBodyAssertion extends Assertion {
             actual = new TreeMap<>(actual);
             expected = new TreeMap<>(expected);
             for (Map.Entry<String, Object> expectedEntry : expected.entrySet()) {
-                compare(expectedEntry.getKey(), expectedEntry.getValue(), actual.remove(expectedEntry.getKey()));
+                compare(expectedEntry.getKey(), actual.remove(expectedEntry.getKey()), expectedEntry.getValue());
             }
             for (Map.Entry<String, Object> unmatchedEntry : actual.entrySet()) {
                 field(unmatchedEntry.getKey(), "unexpected but found [" + unmatchedEntry.getValue() + "]");
@@ -80,6 +81,7 @@ public class ResponseBodyAssertion extends Assertion {
             int i = 0;
             while (i < actual.size() && i < expected.size()) {
                 compare(i, actual.get(i), expected.get(i));
+                i++;
             }
             if (actual.size() == expected.size()) {
                 return;
@@ -92,7 +94,7 @@ public class ResponseBodyAssertion extends Assertion {
             message.append("received [").append(actual.size() - i).append("] more entries than expected\n");
         }
 
-        private void compare(Object field, Object expected, @Nullable Object actual) {
+        private void compare(Object field, @Nullable Object actual, Object expected) {
             if (expected instanceof Map) {
                 if (actual == null) {
                     field(field, "expected map but not found");
@@ -108,10 +110,11 @@ public class ResponseBodyAssertion extends Assertion {
                 Map<String, Object> actualMap = (Map<String, Object>) actual;
                 if (expectedMap.isEmpty() && actualMap.isEmpty()) {
                     field(field, "same [empty map]");
+                    return;
                 }
                 field(field, null);
                 indent += 1;
-                compareMaps(expectedMap, actualMap);
+                compareMaps(actualMap, expectedMap);
                 indent -= 1;
                 return;
             }
@@ -134,7 +137,7 @@ public class ResponseBodyAssertion extends Assertion {
                 }
                 field(field, null);
                 indent += 1;
-                compareLists(expectedList, actualList);
+                compareLists(actualList, expectedList);
                 indent -= 1;
                 return;
             }
@@ -142,7 +145,7 @@ public class ResponseBodyAssertion extends Assertion {
                 field(field, "expected [" + expected + "] but not found");
                 return;
             }
-            if (expected.equals(actual)) {
+            if (Objects.equals(expected, actual)) {
                 field(field, "same [" + expected + "]");
                 return;
             }

From 0970c59608a3f868633f8414e68a017327f9faed Mon Sep 17 00:00:00 2001
From: Johnny Lim <izeye@naver.com>
Date: Tue, 10 May 2016 05:56:02 +0900
Subject: [PATCH 0137/1311] Use Strings.toString() for toString() in
 SearchResponse and GetResponse (#18102)

---
 .../org/elasticsearch/action/get/GetResponse.java    |  6 ++++++
 .../elasticsearch/action/search/SearchResponse.java  | 12 ++----------
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/get/GetResponse.java b/core/src/main/java/org/elasticsearch/action/get/GetResponse.java
index c3042470daa..5741984d35f 100644
--- a/core/src/main/java/org/elasticsearch/action/get/GetResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/get/GetResponse.java
@@ -21,6 +21,7 @@ package org.elasticsearch.action.get;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -168,4 +169,9 @@ public class GetResponse extends ActionResponse implements Iterable<GetField>, T
         super.writeTo(out);
         getResult.writeTo(out);
     }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this, true);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java
index be61301bbd6..ab8395c72a1 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java
@@ -21,12 +21,12 @@ package org.elasticsearch.action.search;
 
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.StatusToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.support.RestActions;
 import org.elasticsearch.search.SearchHits;
@@ -232,14 +232,6 @@ public class SearchResponse extends ActionResponse implements StatusToXContent {
 
     @Override
     public String toString() {
-        try {
-            XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
-            builder.startObject();
-            toXContent(builder, EMPTY_PARAMS);
-            builder.endObject();
-            return builder.string();
-        } catch (IOException e) {
-            return "{ \"error\" : \"" + e.getMessage() + "\"}";
-        }
+        return Strings.toString(this, true);
     }
 }

From 5b6f9abf8249a93506b7ddfd2b4f01c50ed5b402 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 9 May 2016 16:25:46 -0400
Subject: [PATCH 0138/1311] Docs tests: allow any number of leading spaces

0 too!
---
 .../org/elasticsearch/gradle/doc/SnippetsTask.groovy   | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy
index 954505e65f2..6282ac23690 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy
@@ -112,8 +112,8 @@ public class SnippetsTask extends DefaultTask {
                     lastLanguageLine = lineNumber
                     return
                 }
-                if (line ==~ /\/\/ AUTOSENSE\s*/
-                        || line ==~ /\/\/ CONSOLE\s*/) {
+                if (line ==~ /\/\/\s*AUTOSENSE\s*/
+                        || line ==~ /\/\/\s*CONSOLE\s*/) {
                     if (snippet == null) {
                         throw new InvalidUserDataException("AUTOSENSE not " +
                             "paired with a snippet at $file:$lineNumber")
@@ -121,7 +121,7 @@ public class SnippetsTask extends DefaultTask {
                     snippet.autoSense = true
                     return
                 }
-                matcher = line =~ /\/\/ TEST(\[(.+)\])?\s*/
+                matcher = line =~ /\/\/\s*TEST(\[(.+)\])?\s*/
                 if (matcher.matches()) {
                     if (snippet == null) {
                         throw new InvalidUserDataException("TEST not " +
@@ -160,7 +160,7 @@ public class SnippetsTask extends DefaultTask {
                     }
                     return
                 }
-                matcher = line =~ /\/\/ TESTRESPONSE(\[(.+)\])?\s*/
+                matcher = line =~ /\/\/\s*TESTRESPONSE(\[(.+)\])?\s*/
                 if (matcher.matches()) {
                     if (snippet == null) {
                         throw new InvalidUserDataException("TESTRESPONSE not " +
@@ -178,7 +178,7 @@ public class SnippetsTask extends DefaultTask {
                     }
                     return
                 }
-                if (line ==~ /\/\/ TESTSETUP\s*/) {
+                if (line ==~ /\/\/\s*TESTSETUP\s*/) {
                     snippet.testSetup = true
                     return
                 }

From e54c24db02f024a6806461d098547e630b7834b2 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 9 May 2016 17:12:53 -0400
Subject: [PATCH 0139/1311] Docs tests: rename autoSense to console

Since `// AUTOSENSE` has been replaced by `// CONSOLE` we should use
the new name for the variable name.
---
 .../org/elasticsearch/gradle/doc/DocsTestPlugin.groovy |  4 ++--
 .../gradle/doc/RestTestsFromSnippetsTask.groovy        |  2 +-
 .../org/elasticsearch/gradle/doc/SnippetsTask.groovy   | 10 +++++-----
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy
index 3c3ad15a45b..3b1ec3c5d87 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy
@@ -42,13 +42,13 @@ public class DocsTestPlugin extends RestTestPlugin {
                 'List snippets that probably should be marked // CONSOLE'
         listConsoleCandidates.perSnippet {
             if (
-                       it.autoSense    // Already marked, nothing to do
+                       it.console      // Already marked, nothing to do
                     || it.testResponse // It is a response
                 ) {
                 return
             }
             List<String> languages = [
-                // These languages should almost always be marked autosense
+                // These languages should almost always be marked console
                 'js', 'json',
                 // These are often curl commands that should be converted but
                 // are probably false positives
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
index 2696f9ce661..21ae89256d2 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
@@ -95,7 +95,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
                 response(snippet)
                 return
             }
-            if (snippet.test || snippet.autoSense) {
+            if (snippet.test || snippet.console) {
                 test(snippet)
                 return
             }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy
index 6282ac23690..fee0c15d383 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy
@@ -115,10 +115,10 @@ public class SnippetsTask extends DefaultTask {
                 if (line ==~ /\/\/\s*AUTOSENSE\s*/
                         || line ==~ /\/\/\s*CONSOLE\s*/) {
                     if (snippet == null) {
-                        throw new InvalidUserDataException("AUTOSENSE not " +
+                        throw new InvalidUserDataException("CONSOLE not " +
                             "paired with a snippet at $file:$lineNumber")
                     }
-                    snippet.autoSense = true
+                    snippet.console = true
                     return
                 }
                 matcher = line =~ /\/\/\s*TEST(\[(.+)\])?\s*/
@@ -217,7 +217,7 @@ public class SnippetsTask extends DefaultTask {
         int end = NOT_FINISHED
         String contents
 
-        boolean autoSense = false
+        boolean console = false
         boolean test = false
         boolean testResponse = false
         boolean testSetup = false
@@ -233,8 +233,8 @@ public class SnippetsTask extends DefaultTask {
             if (language != null) {
                 result += "($language)"
             }
-            if (autoSense) {
-                result += '// AUTOSENSE'
+            if (console) {
+                result += '// CONSOLE'
             }
             if (test) {
                 result += '// TEST'

From fdae97a2b507c1a99ec22e3d57574b20106bd670 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 9 May 2016 17:28:15 -0400
Subject: [PATCH 0140/1311] Fail build when docs contain // AUTOSENSE

It is deprecated and should be replaced with // CONSOLE.
---
 .../org/elasticsearch/gradle/doc/SnippetsTask.groovy      | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy
index fee0c15d383..afd91858e9d 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy
@@ -112,8 +112,12 @@ public class SnippetsTask extends DefaultTask {
                     lastLanguageLine = lineNumber
                     return
                 }
-                if (line ==~ /\/\/\s*AUTOSENSE\s*/
-                        || line ==~ /\/\/\s*CONSOLE\s*/) {
+                if (line ==~ /\/\/\s*AUTOSENSE\s*/) {
+                    throw new InvalidUserDataException("AUTOSENSE has been " +
+                            "replaced by CONSOLE. Use that instead at " +
+                            "$file:$lineNumber")
+                }
+                if (line ==~ /\/\/\s*CONSOLE\s*/) {
                     if (snippet == null) {
                         throw new InvalidUserDataException("CONSOLE not " +
                             "paired with a snippet at $file:$lineNumber")

From eed5b0be4f60fb64f914f201e1a0e8c6d09b0b3e Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 9 May 2016 19:02:54 -0400
Subject: [PATCH 0141/1311] Increaes timeout on Netty handshake tests

This commit increases the timeout on the Netty handshake tests because
the previous value could cause timeout exceptions on slow machines.
---
 .../NettyTransportServiceHandshakeTests.java          | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java b/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java
index 2b0ff6e0c1a..e1f9bf75874 100644
--- a/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java
@@ -45,7 +45,9 @@ import static java.util.Collections.emptySet;
 import static org.hamcrest.Matchers.containsString;
 
 public class NettyTransportServiceHandshakeTests extends ESTestCase {
+
     private static ThreadPool threadPool;
+    private static final long timeout = Long.MAX_VALUE;
 
     @BeforeClass
     public static void startThreadPool() {
@@ -116,7 +118,7 @@ public class NettyTransportServiceHandshakeTests extends ESTestCase {
                                 emptyMap(),
                                 emptySet(),
                                 Version.CURRENT.minimumCompatibilityVersion()),
-                        100);
+                        timeout);
         assertNotNull(connectedNode);
 
         // the name and version should be updated
@@ -138,7 +140,7 @@ public class NettyTransportServiceHandshakeTests extends ESTestCase {
                             emptyMap(),
                             emptySet(),
                             Version.CURRENT.minimumCompatibilityVersion()),
-                    100);
+                    timeout);
             fail("expected handshake to fail from mismatched cluster names");
         } catch (ConnectTransportException e) {
             assertThat(e.getMessage(), containsString("handshake failed, mismatched cluster name [Cluster [b]]"));
@@ -161,7 +163,7 @@ public class NettyTransportServiceHandshakeTests extends ESTestCase {
                             emptyMap(),
                             emptySet(),
                             Version.CURRENT.minimumCompatibilityVersion()),
-                    100);
+                    timeout);
             fail("expected handshake to fail from incompatible versions");
         } catch (ConnectTransportException e) {
             assertThat(e.getMessage(), containsString("handshake failed, incompatible version"));
@@ -187,7 +189,7 @@ public class NettyTransportServiceHandshakeTests extends ESTestCase {
                         emptyMap(),
                         emptySet(),
                         Version.CURRENT.minimumCompatibilityVersion()),
-                100,
+                timeout,
                 false);
         assertNotNull(connectedNode);
         assertEquals(connectedNode.getName(), "TS_B");
@@ -203,4 +205,5 @@ public class NettyTransportServiceHandshakeTests extends ESTestCase {
             this.discoveryNode = discoveryNode;
         }
     }
+
 }

From 5a7edf992c58e2e9ffbeeee399d46fe0245007ac Mon Sep 17 00:00:00 2001
From: Zack Dever <zackdever@gmail.com>
Date: Mon, 9 May 2016 18:17:36 -0700
Subject: [PATCH 0142/1311] Add missing comma in JSON data in curl example

Relates #18227
---
 docs/reference/indices/create-index.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/indices/create-index.asciidoc b/docs/reference/indices/create-index.asciidoc
index e225b2f15e8..6a45f8a0259 100644
--- a/docs/reference/indices/create-index.asciidoc
+++ b/docs/reference/indices/create-index.asciidoc
@@ -17,7 +17,7 @@ associated with it.
 $ curl -XPUT 'http://localhost:9200/twitter/' -d '{
     "settings" : {
         "index" : {
-            "number_of_shards" : 3 <1>
+            "number_of_shards" : 3, <1>
             "number_of_replicas" : 2 <2>
         }
     }

From ba2fe156e814cfa7adefd1eabbe6503b89366f5e Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 9 May 2016 21:44:32 -0400
Subject: [PATCH 0143/1311] Switch over dynamic method calls, loads and stores
 to invokedynamic. Remove performance hack for accessing a document's fields,
 its not needed. Add support for accessing is-getter methods like
 List.isEmpty() as .empty

Closes #18201
---
 .../modules/scripting/painless.asciidoc       |  32 -
 .../painless/AnalyzerUtility.java             |   5 +-
 .../org/elasticsearch/painless/Compiler.java  |  11 +-
 .../java/org/elasticsearch/painless/Def.java  | 567 ++++++++----------
 .../elasticsearch/painless/Definition.java    | 113 +++-
 .../painless/DynamicCallSite.java             | 151 +++++
 .../elasticsearch/painless/PainlessError.java |   1 +
 .../painless/PainlessScriptEngineService.java |  14 +-
 .../painless/WriterConstants.java             |  22 +-
 .../painless/WriterExternal.java              |  84 +--
 .../painless/WriterStatement.java             |   1 -
 .../elasticsearch/painless/BasicAPITests.java |  30 +
 .../painless/BasicStatementTests.java         |   1 +
 .../painless/DynamicCallSiteTests.java        |  97 +++
 .../elasticsearch/painless/FieldTests.java    | 109 ----
 .../painless/NoSemiColonTests.java            |   1 +
 .../painless/WhenThingsGoWrongTests.java      | 150 +++--
 .../rest-api-spec/test/plan_a/15_update.yaml  |  59 ++
 .../rest-api-spec/test/plan_a/16_update2.yaml |  54 ++
 .../test/plan_a/25_script_upsert.yaml         |  63 ++
 .../rest-api-spec/test/plan_a/30_search.yaml  | 269 +++++++++
 21 files changed, 1182 insertions(+), 652 deletions(-)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/DynamicCallSite.java
 create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/DynamicCallSiteTests.java
 delete mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/FieldTests.java
 create mode 100644 modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/15_update.yaml
 create mode 100644 modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/16_update2.yaml
 create mode 100644 modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/25_script_upsert.yaml

diff --git a/docs/reference/modules/scripting/painless.asciidoc b/docs/reference/modules/scripting/painless.asciidoc
index 4f7d36f38f5..b5300751caf 100644
--- a/docs/reference/modules/scripting/painless.asciidoc
+++ b/docs/reference/modules/scripting/painless.asciidoc
@@ -199,38 +199,6 @@ POST hockey/player/1/_update
 ----------------------------------------------------------------
 // CONSOLE
 
-[float]
-=== Writing Type-Safe Scripts to Improve Performance
-
-If you explicitly specify types, the compiler doesn't have to perform type lookups at runtime, which can significantly
-improve performance. For example, the following script performs the same first name, last name sort we showed before,
-but it's fully type-safe.
-
-[source,js]
-----------------------------------------------------------------
-GET hockey/_search
-{
-  "query": {
-    "match_all": {}
-  },
-  "script_fields": {
-    "full_name_dynamic": {
-      "script": {
-        "lang": "painless",
-        "inline": "def first = input.doc['first'].value; def last = input.doc['last'].value; return first + ' ' + last;"
-      }
-    },
-    "full_name_static": {
-      "script": {
-        "lang": "painless",
-        "inline": "String first = (String)((List)((Map)input.get('doc')).get('first')).get(0); String last = (String)((List)((Map)input.get('doc')).get('last')).get(0); return first + ' ' + last;"
-      }
-    }
-  }
-}
-----------------------------------------------------------------
-// CONSOLE
-
 [[painless-api]]
 [float]
 == Painless API
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerUtility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerUtility.java
index af40c58a7d7..0b345431ac5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerUtility.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerUtility.java
@@ -22,7 +22,6 @@ package org.elasticsearch.painless;
 import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.tree.ParseTree;
 import org.elasticsearch.painless.Definition.Type;
-import org.elasticsearch.painless.Metadata.ExtNodeMetadata;
 import org.elasticsearch.painless.PainlessParser.ExpressionContext;
 import org.elasticsearch.painless.PainlessParser.IdentifierContext;
 import org.elasticsearch.painless.PainlessParser.PrecedenceContext;
@@ -109,15 +108,13 @@ class AnalyzerUtility {
         return source;
     }
 
-    private final Metadata metadata;
     private final Definition definition;
 
     private final Deque<Integer> scopes = new ArrayDeque<>();
     private final Deque<Variable> variables = new ArrayDeque<>();
 
     AnalyzerUtility(final Metadata metadata) {
-        this.metadata = metadata;
-        definition = metadata.definition;
+        this.definition = metadata.definition;
     }
 
     void incrementScope() {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
index e3300fade78..4cba3cb7923 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
@@ -42,12 +42,6 @@ final class Compiler {
      */
     static int MAXIMUM_SOURCE_LENGTH = 16384;
 
-    /**
-     * The default language API to be used with Painless.  The second construction is used
-     * to finalize all the variables, so there is no mistake of modification afterwards.
-     */
-    private static Definition DEFAULT_DEFINITION = new Definition(new Definition());
-
     /**
      * Define the class with lowest privileges.
      */
@@ -95,15 +89,14 @@ final class Compiler {
      * @param settings The CompilerSettings to be used during the compilation.
      * @return An {@link Executable} Painless script.
      */
-    static Executable compile(final Loader loader, final String name, final String source,
-                              final Definition custom, final CompilerSettings settings) {
+    static Executable compile(final Loader loader, final String name, final String source, final CompilerSettings settings) {
         if (source.length() > MAXIMUM_SOURCE_LENGTH) {
             throw new IllegalArgumentException("Scripts may be no longer than " + MAXIMUM_SOURCE_LENGTH +
                 " characters.  The passed in script is " + source.length() + " characters.  Consider using a" +
                 " plugin if a script longer than this length is a requirement.");
         }
 
-        final Definition definition = custom != null ? new Definition(custom) : DEFAULT_DEFINITION;
+        final Definition definition = Definition.INSTANCE;
         final ParserRuleContext root = createParseTree(source);
         final Metadata metadata = new Metadata(definition, source, root, settings);
         Analyzer.analyze(metadata);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 136746026fa..3623368c21e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -19,371 +19,302 @@
 
 package org.elasticsearch.painless;
 
-import org.elasticsearch.index.fielddata.ScriptDocValues;
-import org.elasticsearch.painless.Definition.Cast;
-import org.elasticsearch.painless.Definition.Field;
 import org.elasticsearch.painless.Definition.Method;
-import org.elasticsearch.painless.Definition.Struct;
-import org.elasticsearch.painless.Definition.Transform;
-import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Definition.RuntimeClass;
 
 import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
+import java.lang.invoke.MethodHandles.Lookup;
 import java.lang.reflect.Array;
 import java.util.List;
 import java.util.Map;
 
+/**
+ * Support for dynamic type (def).
+ * <p>
+ * Dynamic types can invoke methods, load/store fields, and be passed as parameters to operators without 
+ * compile-time type information. 
+ * <p>
+ * Dynamic methods, loads, and stores involve locating the appropriate field or method depending
+ * on the receiver's class. For these, we emit an {@code invokedynamic} instruction that, for each new 
+ * type encountered will query a corresponding {@code lookupXXX} method to retrieve the appropriate method.
+ * In most cases, the {@code lookupXXX} methods here will only be called once for a given call site, because 
+ * caching ({@link DynamicCallSite}) generally works: usually all objects at any call site will be consistently 
+ * the same type (or just a few types).  In extreme cases, if there is type explosion, they may be called every 
+ * single time, but simplicity is still more valuable than performance in this code.
+ * <p>
+ * Dynamic array loads and stores and operator functions (e.g. {@code +}) are called directly
+ * with {@code invokestatic}. Because these features cannot be overloaded in painless, they are hardcoded 
+ * decision trees based on the only types that are possible. This keeps overhead low, and seems to be as fast
+ * on average as the more adaptive methodhandle caching. 
+ */
 public class Def {
-    public static Object methodCall(final Object owner, final String name, final Definition definition,
-                                    final Object[] arguments, final boolean[] typesafe) {
-        final Method method = getMethod(owner, name, definition);
 
-        if (method == null) {
-            throw new IllegalArgumentException("Unable to find dynamic method [" + name + "] " +
-                    "for class [" + owner.getClass().getCanonicalName() + "].");
-        }
-
-        final MethodHandle handle = method.handle;
-        final List<Type> types = method.arguments;
-        final Object[] parameters = new Object[arguments.length + 1];
-
-        parameters[0] = owner;
-
-        if (types.size() != arguments.length) {
-            throw new IllegalArgumentException("When dynamically calling [" + name + "] from class " +
-                    "[" + owner.getClass() + "] expected [" + types.size() + "] arguments," +
-                    " but found [" + arguments.length + "].");
-        }
-
-        try {
-            for (int count = 0; count < arguments.length; ++count) {
-                if (typesafe[count]) {
-                    parameters[count + 1] = arguments[count];
-                } else {
-                    final Transform transform = getTransform(arguments[count].getClass(), types.get(count).clazz, definition);
-                    parameters[count + 1] = transform == null ? arguments[count] : transform.method.handle.invoke(arguments[count]);
-                }
-            }
-
-            return handle.invokeWithArguments(parameters);
-        } catch (Throwable throwable) {
-            throw new IllegalArgumentException("Error invoking method [" + name + "] " +
-                    "with owner class [" + owner.getClass().getCanonicalName() + "].", throwable);
-        }
-    }
-
-    @SuppressWarnings({ "unchecked", "rawtypes" })
-    public static void fieldStore(final Object owner, Object value, final String name,
-                                  final Definition definition, final boolean typesafe) {
-        final Field field = getField(owner, name, definition);
-        MethodHandle handle = null;
-
-        if (field == null) {
-            final String set = "set" + Character.toUpperCase(name.charAt(0)) + name.substring(1);
-            final Method method = getMethod(owner, set, definition);
-
-            if (method != null) {
-                handle = method.handle;
-            }
-        } else {
-            handle = field.setter;
-        }
-
-        if (handle != null) {
-            try {
-                if (!typesafe) {
-                    final Transform transform = getTransform(value.getClass(), handle.type().parameterType(1), definition);
-
-                    if (transform != null) {
-                        value = transform.method.handle.invoke(value);
-                    }
-                }
-
-                handle.invoke(owner, value);
-            } catch (Throwable throwable) {
-                throw new IllegalArgumentException("Error storing value [" + value + "] " +
-                        "in field [" + name + "] with owner class [" + owner.getClass() + "].", throwable);
-            }
-        } else if (owner instanceof Map) {
-            ((Map)owner).put(name, value);
-        } else if (owner instanceof List) {
-            try {
-                final int index = Integer.parseInt(name);
-                ((List)owner).set(index, value);
-            } catch (NumberFormatException exception) {
-                throw new IllegalArgumentException( "Illegal list shortcut value [" + name + "].");
-            }
-        } else {
-            throw new IllegalArgumentException("Unable to find dynamic field [" + name + "] " +
-                    "for class [" + owner.getClass().getCanonicalName() + "].");
-        }
-    }
-
-    @SuppressWarnings("rawtypes")
-    public static Object fieldLoad(final Object owner, final String name, final Definition definition) {
-        final Class<?> clazz = owner.getClass();
-        if (clazz.isArray() && "length".equals(name)) {
-            return Array.getLength(owner);
-        } else {
-            // TODO: remove this fast-path, once we speed up dynamics some more
-            if ("value".equals(name) && owner instanceof ScriptDocValues) {
-                if (clazz == ScriptDocValues.Doubles.class) {
-                    return ((ScriptDocValues.Doubles)owner).getValue();
-                } else if (clazz == ScriptDocValues.Longs.class) {
-                    return ((ScriptDocValues.Longs)owner).getValue();
-                } else if (clazz == ScriptDocValues.Strings.class) {
-                    return ((ScriptDocValues.Strings)owner).getValue();
-                } else if (clazz == ScriptDocValues.GeoPoints.class) {
-                    return ((ScriptDocValues.GeoPoints)owner).getValue();
-                }
-            }
-            final Field field = getField(owner, name, definition);
-            MethodHandle handle;
-
-            if (field == null) {
-                final String get = "get" + Character.toUpperCase(name.charAt(0)) + name.substring(1);
-                final Method method = getMethod(owner, get, definition);
-
-                if (method != null) {
-                    handle = method.handle;
-                } else if (owner instanceof Map) {
-                    return ((Map)owner).get(name);
-                } else if (owner instanceof List) {
-                    try {
-                        final int index = Integer.parseInt(name);
-
-                        return ((List)owner).get(index);
-                    } catch (NumberFormatException exception) {
-                        throw new IllegalArgumentException( "Illegal list shortcut value [" + name + "].");
-                    }
-                } else {
-                    throw new IllegalArgumentException("Unable to find dynamic field [" + name + "] " +
-                            "for class [" + clazz.getCanonicalName() + "].");
-                }
-            } else {
-                handle = field.getter;
-            }
-
-            if (handle == null) {
-                throw new IllegalArgumentException(
-                        "Unable to read from field [" + name + "] with owner class [" + clazz + "].");
-            } else {
-                try {
-                    return handle.invoke(owner);
-                } catch (final Throwable throwable) {
-                    throw new IllegalArgumentException("Error loading value from " +
-                            "field [" + name + "] with owner class [" + clazz + "].", throwable);
-                }
-            }
-        }
-    }
-
-    @SuppressWarnings({ "unchecked", "rawtypes" })
-    public static void arrayStore(final Object array, Object index, Object value, final Definition definition,
-                                  final boolean indexsafe, final boolean valuesafe) {
-        if (array instanceof Map) {
-            ((Map)array).put(index, value);
-        } else {
-            try {
-                if (!indexsafe) {
-                    final Transform transform = getTransform(index.getClass(), Integer.class, definition);
-
-                    if (transform != null) {
-                        index = transform.method.handle.invoke(index);
-                    }
-                }
-            } catch (final Throwable throwable) {
-                throw new IllegalArgumentException(
-                        "Error storing value [" + value + "] in list using index [" + index + "].", throwable);
-            }
-
-            if (array.getClass().isArray()) {
-                try {
-                    if (!valuesafe) {
-                        final Transform transform = getTransform(value.getClass(), array.getClass().getComponentType(), definition);
-
-                        if (transform != null) {
-                            value = transform.method.handle.invoke(value);
-                        }
-                    }
-
-                    Array.set(array, (int)index, value);
-                } catch (final Throwable throwable) {
-                    throw new IllegalArgumentException("Error storing value [" + value + "] " +
-                            "in array class [" + array.getClass().getCanonicalName() + "].", throwable);
-                }
-            } else if (array instanceof List) {
-                ((List)array).set((int)index, value);
-            } else {
-                throw new IllegalArgumentException("Attempting to address a non-array type " +
-                        "[" + array.getClass().getCanonicalName() + "] as an array.");
-            }
-        }
-    }
-
-    @SuppressWarnings("rawtypes")
-    public static Object arrayLoad(final Object array, Object index,
-                                   final Definition definition, final boolean indexsafe) {
-        if (array instanceof Map) {
-            return ((Map)array).get(index);
-        } else {
-            try {
-                if (!indexsafe) {
-                    final Transform transform = getTransform(index.getClass(), Integer.class, definition);
-
-                    if (transform != null) {
-                        index = transform.method.handle.invoke(index);
-                    }
-                }
-            } catch (final Throwable throwable) {
-                throw new IllegalArgumentException(
-                        "Error loading value using index [" + index + "].", throwable);
-            }
-
-            if (array.getClass().isArray()) {
-                try {
-                    return Array.get(array, (int)index);
-                } catch (final Throwable throwable) {
-                    throw new IllegalArgumentException("Error loading value from " +
-                            "array class [" + array.getClass().getCanonicalName() + "].", throwable);
-                }
-            } else if (array instanceof List) {
-                return ((List)array).get((int)index);
-            } else {
-                throw new IllegalArgumentException("Attempting to address a non-array type " +
-                        "[" + array.getClass().getCanonicalName() + "] as an array.");
-            }
-        }
-    }
-
-    /** Method lookup for owner.name(), returns null if no matching method was found */ 
-    private static Method getMethod(final Object owner, final String name, final Definition definition) {
-        Class<?> clazz = owner.getClass();
-
-        while (clazz != null) {
-            Struct struct = definition.classes.get(clazz);
+    /** 
+     * Looks up handle for a dynamic method call.
+     * <p>
+     * A dynamic method call for variable {@code x} of type {@code def} looks like:
+     * {@code x.method(args...)}
+     * <p>
+     * This method traverses {@code recieverClass}'s class hierarchy (including interfaces) 
+     * until it finds a matching whitelisted method. If one is not found, it throws an exception. 
+     * Otherwise it returns a handle to the matching method.
+     * <p>
+     * @param receiverClass Class of the object to invoke the method on.
+     * @param name Name of the method.
+     * @param definition Whitelist to check.
+     * @return pointer to matching method to invoke. never returns null.
+     * @throws IllegalArgumentException if no matching whitelisted method was found.
+     */
+    static MethodHandle lookupMethod(Class<?> receiverClass, String name, Definition definition) {
+        // check whitelist for matching method
+        for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
+            RuntimeClass struct = definition.runtimeMap.get(clazz);
 
             if (struct != null) {
                 Method method = struct.methods.get(name);
-
                 if (method != null) {
-                    return method;
+                    return method.handle;
                 }
             }
 
-            for (final Class<?> iface : clazz.getInterfaces()) {
-                struct = definition.classes.get(iface);
+            for (Class<?> iface : clazz.getInterfaces()) {
+                struct = definition.runtimeMap.get(iface);
 
                 if (struct != null) {
                     Method method = struct.methods.get(name);
-
                     if (method != null) {
-                        return method;
+                        return method.handle;
                     }
                 }
             }
-
-            clazz = clazz.getSuperclass();
         }
 
-        return null;
+        // no matching methods in whitelist found
+        throw new IllegalArgumentException("Unable to find dynamic method [" + name + "] " +
+                                           "for class [" + receiverClass.getCanonicalName() + "].");
     }
 
-    /** Field lookup for owner.name, returns null if no matching field was found */ 
-    private static Field getField(final Object owner, final String name, final Definition definition) {
-        Class<?> clazz = owner.getClass();
-
-        while (clazz != null) {
-            Struct struct = definition.classes.get(clazz);
+    /** pointer to Array.getLength(Object) */
+    private static final MethodHandle ARRAY_LENGTH;
+    /** pointer to Map.get(Object) */
+    private static final MethodHandle MAP_GET;
+    /** pointer to Map.put(Object,Object) */
+    private static final MethodHandle MAP_PUT;
+    /** pointer to List.get(int) */
+    private static final MethodHandle LIST_GET;
+    /** pointer to List.set(int,Object) */
+    private static final MethodHandle LIST_SET;
+    static {
+        Lookup lookup = MethodHandles.publicLookup();
+        try {
+            // TODO: maybe specialize handles for different array types. this may be slower, but simple :)
+            ARRAY_LENGTH = lookup.findStatic(Array.class, "getLength",
+                                             MethodType.methodType(int.class, Object.class));
+            MAP_GET      = lookup.findVirtual(Map.class, "get",
+                                             MethodType.methodType(Object.class, Object.class));
+            MAP_PUT      = lookup.findVirtual(Map.class, "put",
+                                             MethodType.methodType(Object.class, Object.class, Object.class));
+            LIST_GET     = lookup.findVirtual(List.class, "get",
+                                             MethodType.methodType(Object.class, int.class));
+            LIST_SET     = lookup.findVirtual(List.class, "set",
+                                             MethodType.methodType(Object.class, int.class, Object.class));
+        } catch (ReflectiveOperationException e) {
+            throw new AssertionError(e);
+        }
+    }
+    
+    /** 
+     * Looks up handle for a dynamic field getter (field load)
+     * <p>
+     * A dynamic field load for variable {@code x} of type {@code def} looks like:
+     * {@code y = x.field}
+     * <p>
+     * The following field loads are allowed:
+     * <ul>
+     *   <li>Whitelisted {@code field} from receiver's class or any superclasses.
+     *   <li>Whitelisted method named {@code getField()} from receiver's class/superclasses/interfaces.
+     *   <li>Whitelisted method named {@code isField()} from receiver's class/superclasses/interfaces.
+     *   <li>The {@code length} field of an array.
+     *   <li>The value corresponding to a map key named {@code field} when the receiver is a Map.
+     *   <li>The value in a list at element {@code field} (integer) when the receiver is a List.
+     * </ul>
+     * <p>
+     * This method traverses {@code recieverClass}'s class hierarchy (including interfaces) 
+     * until it finds a matching whitelisted getter. If one is not found, it throws an exception. 
+     * Otherwise it returns a handle to the matching getter.
+     * <p>
+     * @param receiverClass Class of the object to retrieve the field from.
+     * @param name Name of the field.
+     * @param definition Whitelist to check.
+     * @return pointer to matching field. never returns null.
+     * @throws IllegalArgumentException if no matching whitelisted field was found.
+     */
+    static MethodHandle lookupGetter(Class<?> receiverClass, String name, Definition definition) {
+        // first try whitelist
+        for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
+            RuntimeClass struct = definition.runtimeMap.get(clazz);
 
             if (struct != null) {
-                Field field = struct.members.get(name);
-
-                if (field != null) {
-                    return field;
+                MethodHandle handle = struct.getters.get(name);
+                if (handle != null) {
+                    return handle;
                 }
             }
 
             for (final Class<?> iface : clazz.getInterfaces()) {
-                struct = definition.classes.get(iface);
+                struct = definition.runtimeMap.get(iface);
 
                 if (struct != null) {
-                    Field field = struct.members.get(name);
-
-                    if (field != null) {
-                        return field;
+                    MethodHandle handle = struct.getters.get(name);
+                    if (handle != null) {
+                        return handle;
                     }
                 }
             }
-
-            clazz = clazz.getSuperclass();
         }
+        // special case: arrays, maps, and lists
+        if (receiverClass.isArray() && "length".equals(name)) {
+            // arrays expose .length as a read-only getter
+            return ARRAY_LENGTH;
+        } else if (Map.class.isAssignableFrom(receiverClass)) {
+            // maps allow access like mymap.key
+            // wire 'key' as a parameter, its a constant in painless
+            return MethodHandles.insertArguments(MAP_GET, 1, name);
+        } else if (List.class.isAssignableFrom(receiverClass)) {
+            // lists allow access like mylist.0
+            // wire '0' (index) as a parameter, its a constant. this also avoids
+            // parsing the same integer millions of times!
+            try {
+                int index = Integer.parseInt(name);
+                return MethodHandles.insertArguments(LIST_GET, 1, index);            
+            } catch (NumberFormatException exception) {
+                throw new IllegalArgumentException( "Illegal list shortcut value [" + name + "].");
+            }
+        }
+        
+        throw new IllegalArgumentException("Unable to find dynamic field [" + name + "] " +
+                                           "for class [" + receiverClass.getCanonicalName() + "].");
+    }
+    
+    /** 
+     * Looks up handle for a dynamic field setter (field store)
+     * <p>
+     * A dynamic field store for variable {@code x} of type {@code def} looks like:
+     * {@code x.field = y}
+     * <p>
+     * The following field stores are allowed:
+     * <ul>
+     *   <li>Whitelisted {@code field} from receiver's class or any superclasses.
+     *   <li>Whitelisted method named {@code setField()} from receiver's class/superclasses/interfaces.
+     *   <li>The value corresponding to a map key named {@code field} when the receiver is a Map.
+     *   <li>The value in a list at element {@code field} (integer) when the receiver is a List.
+     * </ul>
+     * <p>
+     * This method traverses {@code recieverClass}'s class hierarchy (including interfaces) 
+     * until it finds a matching whitelisted setter. If one is not found, it throws an exception. 
+     * Otherwise it returns a handle to the matching setter.
+     * <p>
+     * @param receiverClass Class of the object to retrieve the field from.
+     * @param name Name of the field.
+     * @param definition Whitelist to check.
+     * @return pointer to matching field. never returns null.
+     * @throws IllegalArgumentException if no matching whitelisted field was found.
+     */
+    static MethodHandle lookupSetter(Class<?> receiverClass, String name, Definition definition) {
+        // first try whitelist
+        for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
+            RuntimeClass struct = definition.runtimeMap.get(clazz);
 
-        return null;
+            if (struct != null) {
+                MethodHandle handle = struct.setters.get(name);
+                if (handle != null) {
+                    return handle;
+                }
+            }
+
+            for (final Class<?> iface : clazz.getInterfaces()) {
+                struct = definition.runtimeMap.get(iface);
+
+                if (struct != null) {
+                    MethodHandle handle = struct.setters.get(name);
+                    if (handle != null) {
+                        return handle;
+                    }
+                }
+            }
+        }
+        // special case: maps, and lists
+        if (Map.class.isAssignableFrom(receiverClass)) {
+            // maps allow access like mymap.key
+            // wire 'key' as a parameter, its a constant in painless
+            return MethodHandles.insertArguments(MAP_PUT, 1, name);
+        } else if (List.class.isAssignableFrom(receiverClass)) {
+            // lists allow access like mylist.0
+            // wire '0' (index) as a parameter, its a constant. this also avoids
+            // parsing the same integer millions of times!
+            try {
+                int index = Integer.parseInt(name);
+                return MethodHandles.insertArguments(LIST_SET, 1, index);            
+            } catch (NumberFormatException exception) {
+                throw new IllegalArgumentException( "Illegal list shortcut value [" + name + "].");
+            }
+        }
+        
+        throw new IllegalArgumentException("Unable to find dynamic field [" + name + "] " +
+                                           "for class [" + receiverClass.getCanonicalName() + "].");
     }
 
-    public static Transform getTransform(Class<?> fromClass, Class<?> toClass, final Definition definition) {
-        Struct fromStruct = null;
-        Struct toStruct = null;
+    // NOTE: below methods are not cached, instead invoked directly because they are performant.
 
-        if (fromClass.equals(toClass)) {
-            return null;
-        }
-
-        while (fromClass != null) {
-            fromStruct = definition.classes.get(fromClass);
-
-            if (fromStruct != null) {
-                break;
+    /**
+     * Performs an actual array store.
+     * @param array array object
+     * @param index map key, array index (integer), or list index (integer)
+     * @param value value to store in the array.
+     */
+    @SuppressWarnings({ "unchecked", "rawtypes" })
+    public static void arrayStore(final Object array, Object index, Object value) {
+        if (array instanceof Map) {
+            ((Map)array).put(index, value);
+        } else if (array.getClass().isArray()) {
+            try {
+                Array.set(array, (int)index, value);
+            } catch (final Throwable throwable) {
+                throw new IllegalArgumentException("Error storing value [" + value + "] " +
+                                                   "in array class [" + array.getClass().getCanonicalName() + "].", throwable);
             }
-
-            for (final Class<?> iface : fromClass.getInterfaces()) {
-                fromStruct = definition.classes.get(iface);
-
-                if (fromStruct != null) {
-                    break;
-                }
-            }
-
-            if (fromStruct != null) {
-                break;
-            }
-
-            fromClass = fromClass.getSuperclass();
+        } else if (array instanceof List) {
+            ((List)array).set((int)index, value);
+        } else {
+            throw new IllegalArgumentException("Attempting to address a non-array type " +
+                                               "[" + array.getClass().getCanonicalName() + "] as an array.");
         }
-
-        if (fromStruct != null) {
-            while (toClass != null) {
-                toStruct = definition.classes.get(toClass);
-
-                if (toStruct != null) {
-                    break;
-                }
-
-                for (final Class<?> iface : toClass.getInterfaces()) {
-                    toStruct = definition.classes.get(iface);
-
-                    if (toStruct != null) {
-                        break;
-                    }
-                }
-
-                if (toStruct != null) {
-                    break;
-                }
-
-                toClass = toClass.getSuperclass();
+    }
+    
+    /**
+     * Performs an actual array load.
+     * @param array array object
+     * @param index map key, array index (integer), or list index (integer)
+     */
+    @SuppressWarnings("rawtypes")
+    public static Object arrayLoad(final Object array, Object index) {
+        if (array instanceof Map) {
+            return ((Map)array).get(index);
+        } else if (array.getClass().isArray()) {
+            try {
+                return Array.get(array, (int)index);
+            } catch (final Throwable throwable) {
+                throw new IllegalArgumentException("Error loading value from " +
+                                                   "array class [" + array.getClass().getCanonicalName() + "].", throwable);
             }
+        } else if (array instanceof List) {
+            return ((List)array).get((int)index);
+        } else {
+            throw new IllegalArgumentException("Attempting to address a non-array type " +
+                                               "[" + array.getClass().getCanonicalName() + "] as an array.");
         }
-
-        if (toStruct != null) {
-            final Type fromType = definition.getType(fromStruct.name);
-            final Type toType = definition.getType(toStruct.name);
-            final Cast cast = new Cast(fromType, toType);
-
-            return definition.transforms.get(cast);
-        }
-
-        return null;
     }
 
     public static Object not(final Object unary) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 5a0e0c1e636..f418edd2beb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -21,7 +21,6 @@ package org.elasticsearch.painless;
 
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
-import java.lang.invoke.MethodType;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -37,6 +36,12 @@ import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 
 class Definition {
+    /**
+     * The default language API to be used with Painless.  The second construction is used
+     * to finalize all the variables, so there is no mistake of modification afterwards.
+     */
+    static Definition INSTANCE = new Definition(new Definition());
+
     enum Sort {
         VOID(       void.class      , 0 , true  , false , false , false ),
         BOOL(       boolean.class   , 1 , true  , true  , false , true  ),
@@ -323,11 +328,24 @@ class Definition {
             this.downcast = downcast;
         }
     }
+    
+    static class RuntimeClass {
+        final Map<String, Method> methods;
+        final Map<String, MethodHandle> getters;
+        final Map<String, MethodHandle> setters;
+        
+        private RuntimeClass(Map<String, Method> methods, Map<String, MethodHandle> getters, Map<String, MethodHandle> setters) {
+            this.methods = methods;
+            this.getters = getters;
+            this.setters = setters;
+        }
+    }
 
     final Map<String, Struct> structs;
     final Map<Class<?>, Struct> classes;
     final Map<Cast, Transform> transforms;
     final Map<Pair, Type> bounds;
+    final Map<Class<?>, RuntimeClass> runtimeMap; 
 
     final Type voidType;
     final Type booleanType;
@@ -405,11 +423,12 @@ class Definition {
     final Type doublesType;
     final Type geoPointsType;
 
-    public Definition() {
+    private Definition() {
         structs = new HashMap<>();
         classes = new HashMap<>();
         transforms = new HashMap<>();
         bounds = new HashMap<>();
+        runtimeMap = new HashMap<>();
 
         addDefaultStructs();
         addDefaultClasses();
@@ -492,9 +511,64 @@ class Definition {
         copyDefaultStructs();
         addDefaultTransforms();
         addDefaultBounds();
+        computeRuntimeClasses();
+    }
+    
+    // precompute a more efficient structure for dynamic method/field access:
+    void computeRuntimeClasses() {
+        this.runtimeMap.clear();
+        for (Class<?> clazz : classes.keySet()) {
+            runtimeMap.put(clazz, computeRuntimeClass(clazz));
+        }
+    }
+    
+    RuntimeClass computeRuntimeClass(Class<?> clazz) {
+        Struct struct = classes.get(clazz);
+        Map<String, Method> methods = struct.methods;
+        Map<String, MethodHandle> getters = new HashMap<>();
+        Map<String, MethodHandle> setters = new HashMap<>();
+        // add all members
+        for (Map.Entry<String,Field> member : struct.members.entrySet()) {
+            getters.put(member.getKey(), member.getValue().getter);
+            setters.put(member.getKey(), member.getValue().setter);
+        }
+        // add all getters/setters
+        for (Map.Entry<String,Method> method : methods.entrySet()) {
+            String name = method.getKey();
+            Method m = method.getValue();
+            
+            if (m.arguments.size() == 0 &&
+                name.startsWith("get") &&
+                name.length() > 3 &&
+                Character.isUpperCase(name.charAt(3))) {
+              StringBuilder newName = new StringBuilder();
+              newName.append(Character.toLowerCase(name.charAt(3)));
+              newName.append(name.substring(4));
+              getters.putIfAbsent(newName.toString(), m.handle);
+            } else if (m.arguments.size() == 0 &&
+                       name.startsWith("is") &&
+                       name.length() > 2 && 
+                       Character.isUpperCase(name.charAt(2))) {
+              StringBuilder newName = new StringBuilder();
+              newName.append(Character.toLowerCase(name.charAt(2)));
+              newName.append(name.substring(3));
+              getters.putIfAbsent(newName.toString(), m.handle);
+            }
+            
+            if (m.arguments.size() == 1 &&
+                name.startsWith("set") &&
+                name.length() > 3 &&
+                Character.isUpperCase(name.charAt(3))) {
+              StringBuilder newName = new StringBuilder();
+              newName.append(Character.toLowerCase(name.charAt(3)));
+              newName.append(name.substring(4));
+              setters.putIfAbsent(newName.toString(), m.handle);
+            }
+        }
+        return new RuntimeClass(methods, getters, setters);
     }
 
-    Definition(final Definition definition) {
+    private Definition(final Definition definition) {
         final Map<String, Struct> structs = new HashMap<>();
 
         for (final Struct struct : definition.structs.values()) {
@@ -513,6 +587,7 @@ class Definition {
 
         transforms = Collections.unmodifiableMap(definition.transforms);
         bounds = Collections.unmodifiableMap(definition.bounds);
+        this.runtimeMap = Collections.unmodifiableMap(definition.runtimeMap);
 
         voidType = definition.voidType;
         booleanType = definition.booleanType;
@@ -1815,14 +1890,8 @@ class Definition {
         MethodHandle handle;
 
         try {
-            if (statik) {
-                handle = MethodHandles.publicLookup().in(owner.clazz).findStatic(
-                    owner.clazz, alias == null ? name : alias, MethodType.methodType(rtn.clazz, classes));
-            } else {
-                handle = MethodHandles.publicLookup().in(owner.clazz).findVirtual(
-                    owner.clazz, alias == null ? name : alias, MethodType.methodType(rtn.clazz, classes));
-            }
-        } catch (NoSuchMethodException | IllegalAccessException exception) {
+            handle = MethodHandles.publicLookup().in(owner.clazz).unreflect(reflect);
+        } catch (IllegalAccessException exception) {
             throw new IllegalArgumentException("Method [" + (alias == null ? name : alias) + "]" +
                 " not found for class [" + owner.clazz.getName() + "]" +
                 " with arguments " + Arrays.toString(classes) + ".");
@@ -1907,12 +1976,10 @@ class Definition {
 
         try {
             if (!statik) {
-                getter = MethodHandles.publicLookup().in(owner.clazz).findGetter(
-                    owner.clazz, alias == null ? name : alias, type.clazz);
-                setter = MethodHandles.publicLookup().in(owner.clazz).findSetter(
-                    owner.clazz, alias == null ? name : alias, type.clazz);
+                getter = MethodHandles.publicLookup().unreflectGetter(reflect);
+                setter = MethodHandles.publicLookup().unreflectSetter(reflect);
             }
-        } catch (NoSuchFieldException | IllegalAccessException exception) {
+        } catch (IllegalAccessException exception) {
             throw new IllegalArgumentException("Getter/Setter [" + (alias == null ? name : alias) + "]" +
                 " not found for class [" + owner.clazz.getName() + "].");
         }
@@ -1982,10 +2049,8 @@ class Definition {
                     }
 
                     try {
-                        handle = MethodHandles.publicLookup().in(owner.clazz).findVirtual(
-                            owner.clazz, method.method.getName(),
-                            MethodType.methodType(method.reflect.getReturnType(), method.reflect.getParameterTypes()));
-                    } catch (NoSuchMethodException | IllegalAccessException exception) {
+                        handle = MethodHandles.publicLookup().in(owner.clazz).unreflect(reflect);
+                    } catch (IllegalAccessException exception) {
                         throw new IllegalArgumentException("Method [" + method.method.getName() + "] not found for" +
                             " class [" + owner.clazz.getName() + "] with arguments " +
                             Arrays.toString(method.reflect.getParameterTypes()) + ".");
@@ -2010,11 +2075,9 @@ class Definition {
                     }
 
                     try {
-                        getter = MethodHandles.publicLookup().in(owner.clazz).findGetter(
-                            owner.clazz, field.name, field.type.clazz);
-                        setter = MethodHandles.publicLookup().in(owner.clazz).findSetter(
-                            owner.clazz, field.name, field.type.clazz);
-                    } catch (NoSuchFieldException | IllegalAccessException exception) {
+                        getter = MethodHandles.publicLookup().unreflectGetter(reflect);
+                        setter = MethodHandles.publicLookup().unreflectSetter(reflect);
+                    } catch (IllegalAccessException exception) {
                         throw new IllegalArgumentException("Getter/Setter [" + field.name + "]" +
                             " not found for class [" + owner.clazz.getName() + "].");
                     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DynamicCallSite.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DynamicCallSite.java
new file mode 100644
index 00000000000..b0870c60219
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DynamicCallSite.java
@@ -0,0 +1,151 @@
+package org.elasticsearch.painless;
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.lang.invoke.CallSite;
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodHandles.Lookup;
+import java.lang.invoke.MethodType;
+import java.lang.invoke.MutableCallSite;
+
+/**
+ * Painless invokedynamic call site.
+ * <p>
+ * Has 3 flavors (passed as static bootstrap parameters): dynamic method call,
+ * dynamic field load (getter), and dynamic field store (setter).
+ * <p>
+ * When a new type is encountered at the call site, we lookup from the appropriate
+ * whitelist, and cache with a guard. If we encounter too many types, we stop caching.
+ * <p>
+ * Based on the cascaded inlining cache from the JSR 292 cookbook 
+ * (https://code.google.com/archive/p/jsr292-cookbook/, BSD license)
+ */
+// NOTE: this class must be public, because generated painless classes are in a different package,
+// and it needs to be accessible by that code.
+public final class DynamicCallSite {
+    // NOTE: these must be primitive types, see https://docs.oracle.com/javase/specs/jvms/se7/html/jvms-6.html#jvms-6.5.invokedynamic
+    /** static bootstrap parameter indicating a dynamic method call, e.g. foo.bar(...) */
+    static final int METHOD_CALL = 0;
+    /** static bootstrap parameter indicating a dynamic load (getter), e.g. baz = foo.bar */
+    static final int LOAD = 1;
+    /** static bootstrap parameter indicating a dynamic store (setter), e.g. foo.bar = baz */
+    static final int STORE = 2;
+    
+    static class InliningCacheCallSite extends MutableCallSite {
+        /** maximum number of types before we go megamorphic */
+        static final int MAX_DEPTH = 5;
+        
+        final Lookup lookup;
+        final String name;
+        final int flavor;
+        int depth;
+        
+        InliningCacheCallSite(Lookup lookup, String name, MethodType type, int flavor) {
+            super(type);
+            this.lookup = lookup;
+            this.name = name;
+            this.flavor = flavor;
+        }
+    }
+    
+    /** 
+     * invokeDynamic bootstrap method
+     * <p>
+     * In addition to ordinary parameters, we also take a static parameter {@code flavor} which
+     * tells us what type of dynamic call it is (and which part of whitelist to look at).
+     * <p>
+     * see https://docs.oracle.com/javase/specs/jvms/se7/html/jvms-6.html#jvms-6.5.invokedynamic
+     */
+    public static CallSite bootstrap(Lookup lookup, String name, MethodType type, int flavor) {
+        InliningCacheCallSite callSite = new InliningCacheCallSite(lookup, name, type, flavor);
+        
+        MethodHandle fallback = FALLBACK.bindTo(callSite);
+        fallback = fallback.asCollector(Object[].class, type.parameterCount());
+        fallback = fallback.asType(type);
+        
+        callSite.setTarget(fallback);
+        return callSite;
+    }
+    
+    /** 
+     * guard method for inline caching: checks the receiver's class is the same
+     * as the cached class
+     */
+    static boolean checkClass(Class<?> clazz, Object receiver) {
+        return receiver.getClass() == clazz;
+    }
+    
+    /**
+     * Does a slow lookup against the whitelist.
+     */
+    private static MethodHandle lookup(int flavor, Class<?> clazz, String name) {
+        switch(flavor) {
+            case METHOD_CALL: 
+                return Def.lookupMethod(clazz, name, Definition.INSTANCE);
+            case LOAD: 
+                return Def.lookupGetter(clazz, name, Definition.INSTANCE);
+            case STORE:
+                return Def.lookupSetter(clazz, name, Definition.INSTANCE);
+            default: throw new AssertionError();
+        }
+    }
+    
+    /**
+     * Called when a new type is encountered (or, when we have encountered more than {@code MAX_DEPTH}
+     * types at this call site and given up on caching). 
+     */
+    static Object fallback(InliningCacheCallSite callSite, Object[] args) throws Throwable {
+        MethodType type = callSite.type();
+        Object receiver = args[0];
+        Class<?> receiverClass = receiver.getClass();
+        MethodHandle target = lookup(callSite.flavor, receiverClass, callSite.name);
+        target = target.asType(type);
+        
+        if (callSite.depth >= InliningCacheCallSite.MAX_DEPTH) {
+            // revert to a vtable call
+            callSite.setTarget(target);
+            return target.invokeWithArguments(args);
+        }
+        
+        MethodHandle test = CHECK_CLASS.bindTo(receiverClass);
+        test = test.asType(test.type().changeParameterType(0, type.parameterType(0)));
+        
+        MethodHandle guard = MethodHandles.guardWithTest(test, target, callSite.getTarget());
+        callSite.depth++;
+        
+        callSite.setTarget(guard);
+        return target.invokeWithArguments(args);
+    }
+    
+    private static final MethodHandle CHECK_CLASS;
+    private static final MethodHandle FALLBACK;
+    static {
+        Lookup lookup = MethodHandles.lookup();
+        try {
+            CHECK_CLASS = lookup.findStatic(DynamicCallSite.class, "checkClass",
+                                            MethodType.methodType(boolean.class, Class.class, Object.class));
+            FALLBACK = lookup.findStatic(DynamicCallSite.class, "fallback",
+                                         MethodType.methodType(Object.class, InliningCacheCallSite.class, Object[].class));
+        } catch (ReflectiveOperationException e) {
+            throw new AssertionError(e);
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessError.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessError.java
index 0d6fa915caf..68eac7a260e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessError.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessError.java
@@ -26,6 +26,7 @@ package org.elasticsearch.painless;
  * something hazardous.  The alternative was extending {@link Throwable}, but that seemed worse than using
  * an {@link Error} in this case.
  */
+@SuppressWarnings("serial")
 public class PainlessError extends Error {
     /**
      * Constructor.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
index 34ebddd9fb0..7be3f9832be 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
@@ -88,18 +88,6 @@ public class PainlessScriptEngineService extends AbstractComponent implements Sc
         });
     }
 
-    /**
-     * Used only for testing.
-     */
-    private Definition definition = null;
-
-    /**
-     * Used only for testing.
-     */
-    void setDefinition(final Definition definition) {
-        this.definition = definition;
-    }
-
     /**
      * Constructor.
      * @param settings The settings to initialize the engine with.
@@ -189,7 +177,7 @@ public class PainlessScriptEngineService extends AbstractComponent implements Sc
         return AccessController.doPrivileged(new PrivilegedAction<Executable>() {
             @Override
             public Executable run() {
-                return Compiler.compile(loader, "unknown", script, definition, compilerSettings);
+                return Compiler.compile(loader, "unknown", script, compilerSettings);
             }
         }, COMPILATION_CONTEXT);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index 33fea094058..a1a594b354e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -20,9 +20,13 @@
 package org.elasticsearch.painless;
 
 import org.elasticsearch.script.ScoreAccessor;
+import org.objectweb.asm.Handle;
+import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.Type;
 import org.objectweb.asm.commons.Method;
 
+import java.lang.invoke.CallSite;
+import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodType;
 import java.util.Map;
 
@@ -40,22 +44,24 @@ class WriterConstants {
 
     final static Type DEFINITION_TYPE = Type.getType(Definition.class);
 
+    final static Type OBJECT_TYPE = Type.getType(Object.class);
+
     final static Type MAP_TYPE  = Type.getType(Map.class);
     final static Method MAP_GET = getAsmMethod(Object.class, "get", Object.class);
 
     final static Type SCORE_ACCESSOR_TYPE    = Type.getType(ScoreAccessor.class);
     final static Method SCORE_ACCESSOR_FLOAT = getAsmMethod(float.class, "floatValue");
 
-    final static Method DEF_METHOD_CALL = getAsmMethod(
-        Object.class, "methodCall", Object.class, String.class, Definition.class, Object[].class, boolean[].class);
+    /** dynamic callsite bootstrap signature */
+    final static MethodType DEF_BOOTSTRAP_TYPE = MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, 
+                                                                       String.class, MethodType.class, int.class);
+    final static Handle DEF_BOOTSTRAP_HANDLE = new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(DynamicCallSite.class),
+                                                          "bootstrap", WriterConstants.DEF_BOOTSTRAP_TYPE.toMethodDescriptorString());
+
     final static Method DEF_ARRAY_STORE = getAsmMethod(
-        void.class, "arrayStore", Object.class, Object.class, Object.class, Definition.class, boolean.class, boolean.class);
+        void.class, "arrayStore", Object.class, Object.class, Object.class);
     final static Method DEF_ARRAY_LOAD = getAsmMethod(
-        Object.class, "arrayLoad", Object.class, Object.class, Definition.class, boolean.class);
-    final static Method DEF_FIELD_STORE = getAsmMethod(
-        void.class, "fieldStore", Object.class, Object.class, String.class, Definition.class, boolean.class);
-    final static Method DEF_FIELD_LOAD = getAsmMethod(
-        Object.class, "fieldLoad", Object.class, String.class, Definition.class);
+        Object.class, "arrayLoad", Object.class, Object.class);
 
     final static Method DEF_NOT_CALL = getAsmMethod(Object.class, "not", Object.class);
     final static Method DEF_NEG_CALL = getAsmMethod(Object.class, "neg", Object.class);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java
index c89e856f858..ab05e1b6e56 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java
@@ -49,13 +49,8 @@ import static org.elasticsearch.painless.PainlessParser.DIV;
 import static org.elasticsearch.painless.PainlessParser.MUL;
 import static org.elasticsearch.painless.PainlessParser.REM;
 import static org.elasticsearch.painless.PainlessParser.SUB;
-import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE;
-import static org.elasticsearch.painless.WriterConstants.DEFINITION_TYPE;
 import static org.elasticsearch.painless.WriterConstants.DEF_ARRAY_LOAD;
 import static org.elasticsearch.painless.WriterConstants.DEF_ARRAY_STORE;
-import static org.elasticsearch.painless.WriterConstants.DEF_FIELD_LOAD;
-import static org.elasticsearch.painless.WriterConstants.DEF_FIELD_STORE;
-import static org.elasticsearch.painless.WriterConstants.DEF_METHOD_CALL;
 import static org.elasticsearch.painless.WriterConstants.TOBYTEEXACT_INT;
 import static org.elasticsearch.painless.WriterConstants.TOBYTEEXACT_LONG;
 import static org.elasticsearch.painless.WriterConstants.TOBYTEWOOVERFLOW_DOUBLE;
@@ -473,20 +468,11 @@ class WriterExternal {
 
     private void writeLoadStoreField(final ParserRuleContext source, final boolean store, final String name) {
         if (store) {
-            final ExtNodeMetadata sourceemd = metadata.getExtNodeMetadata(source);
-            final ExternalMetadata parentemd = metadata.getExternalMetadata(sourceemd.parent);
-            final ExpressionMetadata expremd = metadata.getExpressionMetadata(parentemd.storeExpr);
-
-            execute.push(name);
-            execute.loadThis();
-            execute.getField(CLASS_TYPE, "definition", DEFINITION_TYPE);
-            execute.push(parentemd.token == 0 && expremd.typesafe);
-            execute.invokeStatic(definition.defobjType.type, DEF_FIELD_STORE);
+            execute.visitInvokeDynamicInsn(name, "(Ljava/lang/Object;Ljava/lang/Object;)V", 
+                                           WriterConstants.DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.STORE });
         } else {
-            execute.push(name);
-            execute.loadThis();
-            execute.getField(CLASS_TYPE, "definition", DEFINITION_TYPE);
-            execute.invokeStatic(definition.defobjType.type, DEF_FIELD_LOAD);
+            execute.visitInvokeDynamicInsn(name, "(Ljava/lang/Object;)Ljava/lang/Object;", 
+                                           WriterConstants.DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.LOAD });
         }
     }
 
@@ -496,23 +482,9 @@ class WriterExternal {
         }
 
         if (type.sort == Sort.DEF) {
-            final ExtbraceContext bracectx = (ExtbraceContext)source;
-            final ExpressionMetadata expremd0 = metadata.getExpressionMetadata(bracectx.expression());
-
             if (store) {
-                final ExtNodeMetadata braceenmd = metadata.getExtNodeMetadata(bracectx);
-                final ExternalMetadata parentemd = metadata.getExternalMetadata(braceenmd.parent);
-                final ExpressionMetadata expremd1 = metadata.getExpressionMetadata(parentemd.storeExpr);
-
-                execute.loadThis();
-                execute.getField(CLASS_TYPE, "definition", DEFINITION_TYPE);
-                execute.push(expremd0.typesafe);
-                execute.push(parentemd.token == 0 && expremd1.typesafe);
                 execute.invokeStatic(definition.defobjType.type, DEF_ARRAY_STORE);
             } else {
-                execute.loadThis();
-                execute.getField(CLASS_TYPE, "definition", DEFINITION_TYPE);
-                execute.push(expremd0.typesafe);
                 execute.invokeStatic(definition.defobjType.type, DEF_ARRAY_LOAD);
             }
         } else {
@@ -729,31 +701,29 @@ class WriterExternal {
                 execute.checkCast(target.rtn.type);
             }
         } else {
-            execute.push((String)sourceenmd.target);
-            execute.loadThis();
-            execute.getField(CLASS_TYPE, "definition", DEFINITION_TYPE);
-
-            execute.push(arguments.size());
-            execute.newArray(definition.defType.type);
-
-            for (int argument = 0; argument < arguments.size(); ++argument) {
-                execute.dup();
-                execute.push(argument);
-                writer.visit(arguments.get(argument));
-                execute.arrayStore(definition.defType.type);
-            }
-
-            execute.push(arguments.size());
-            execute.newArray(definition.booleanType.type);
-
-            for (int argument = 0; argument < arguments.size(); ++argument) {
-                execute.dup();
-                execute.push(argument);
-                execute.push(metadata.getExpressionMetadata(arguments.get(argument)).typesafe);
-                execute.arrayStore(definition.booleanType.type);
-            }
-
-            execute.invokeStatic(definition.defobjType.type, DEF_METHOD_CALL);
+            writeDynamicCallExternal(source);
         }
     }
+    
+    private void writeDynamicCallExternal(final ExtcallContext source) {
+        final ExtNodeMetadata sourceenmd = metadata.getExtNodeMetadata(source);
+        final List<ExpressionContext> arguments = source.arguments().expression();
+        
+        StringBuilder signature = new StringBuilder();
+        signature.append('(');
+        // first parameter is the receiver, we never know its type: always Object
+        signature.append(WriterConstants.OBJECT_TYPE.getDescriptor());
+
+        // TODO: remove our explicit conversions and feed more type information for args/return value,
+        // it can avoid some unnecessary boxing etc.
+        for (int i = 0; i < arguments.size(); i++) {
+            signature.append(WriterConstants.OBJECT_TYPE.getDescriptor());
+            writer.visit(arguments.get(i));
+        }
+        signature.append(')');
+        // return value
+        signature.append(WriterConstants.OBJECT_TYPE.getDescriptor());
+        execute.visitInvokeDynamicInsn((String)sourceenmd.target, signature.toString(), 
+                                       WriterConstants.DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.METHOD_CALL });
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterStatement.java
index d7a964882a4..4cd698a74cd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterStatement.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterStatement.java
@@ -26,7 +26,6 @@ import org.elasticsearch.painless.PainlessParser.AfterthoughtContext;
 import org.elasticsearch.painless.PainlessParser.BlockContext;
 import org.elasticsearch.painless.PainlessParser.DeclContext;
 import org.elasticsearch.painless.PainlessParser.DeclarationContext;
-import org.elasticsearch.painless.PainlessParser.DecltypeContext;
 import org.elasticsearch.painless.PainlessParser.DeclvarContext;
 import org.elasticsearch.painless.PainlessParser.DoContext;
 import org.elasticsearch.painless.PainlessParser.EmptyscopeContext;
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
index bcfec2343d1..46fc6768c2f 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
@@ -49,4 +49,34 @@ public class BasicAPITests extends ScriptTestCase {
         assertEquals(3, exec("Map x = new HashMap(); x.put(2, 2); x.put(3, 3); x.put(-2, -2); Iterator y = x.values().iterator(); " +
             "int total = 0; while (y.hasNext()) total += (int)y.next(); return total;"));
     }
+    
+    /** Test loads and stores with a map */
+    public void testMapLoadStore() {
+        assertEquals(5, exec("def x = new HashMap(); x.abc = 5; return x.abc;"));
+        assertEquals(5, exec("def x = new HashMap(); x['abc'] = 5; return x['abc'];"));
+    }
+    
+    /** Test loads and stores with a list */
+    public void testListLoadStore() {
+        assertEquals(5, exec("def x = new ArrayList(); x.add(3); x.0 = 5; return x.0;"));
+        assertEquals(5, exec("def x = new ArrayList(); x.add(3); x[0] = 5; return x[0];"));
+    }
+    
+    /** Test loads and stores with a list */
+    public void testArrayLoadStore() {
+        assertEquals(5, exec("def x = new int[5]; return x.length"));
+        assertEquals(5, exec("def x = new int[4]; x[0] = 5; return x[0];"));
+    }
+    
+    /** Test shortcut for getters with isXXXX */
+    public void testListEmpty() {
+        assertEquals(true, exec("def x = new ArrayList(); return x.empty;"));
+        assertEquals(true, exec("def x = new HashMap(); return x.empty;"));
+    }
+    
+    /** Test list method invocation */
+    public void testListGet() {
+        assertEquals(5, exec("def x = new ArrayList(); x.add(5); return x.get(0);"));
+        assertEquals(5, exec("def x = new ArrayList(); x.add(5); def index = 0; return x.get(index);"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
index ebdb9021b7b..f0022e6bcf1 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
@@ -168,6 +168,7 @@ public class BasicStatementTests extends ScriptTestCase {
         assertEquals(4, exec("int x = 0, y = 0; while (x < 10) { ++x; if (x == 5) break; ++y; } return y;"));
     }
 
+    @SuppressWarnings("rawtypes")
     public void testReturnStatement() {
         assertEquals(10, exec("return 10;"));
         assertEquals(5, exec("int x = 5; return x;"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DynamicCallSiteTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DynamicCallSiteTests.java
new file mode 100644
index 00000000000..de5202236db
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DynamicCallSiteTests.java
@@ -0,0 +1,97 @@
+package org.elasticsearch.painless;
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.lang.invoke.CallSite;
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
+
+import org.elasticsearch.test.ESTestCase;
+
+public class DynamicCallSiteTests extends ESTestCase {
+    
+    /** calls toString() on integers, twice */
+    public void testOneType() throws Throwable {
+        CallSite site = DynamicCallSite.bootstrap(MethodHandles.publicLookup(), 
+                                                  "toString", 
+                                                  MethodType.methodType(String.class, Object.class), 
+                                                  DynamicCallSite.METHOD_CALL);
+        MethodHandle handle = site.dynamicInvoker();
+        assertDepthEquals(site, 0);
+
+        // invoke with integer, needs lookup
+        assertEquals("5", handle.invoke(Integer.valueOf(5)));
+        assertDepthEquals(site, 1);
+
+        // invoked with integer again: should be cached
+        assertEquals("6", handle.invoke(Integer.valueOf(6)));
+        assertDepthEquals(site, 1);
+    }
+    
+    public void testTwoTypes() throws Throwable {
+        CallSite site = DynamicCallSite.bootstrap(MethodHandles.publicLookup(), 
+                                                  "toString", 
+                                                  MethodType.methodType(String.class, Object.class), 
+                                                  DynamicCallSite.METHOD_CALL);
+        MethodHandle handle = site.dynamicInvoker();
+        assertDepthEquals(site, 0);
+
+        assertEquals("5", handle.invoke(Integer.valueOf(5)));
+        assertDepthEquals(site, 1);
+        assertEquals("1.5", handle.invoke(Float.valueOf(1.5f)));
+        assertDepthEquals(site, 2);
+
+        // both these should be cached
+        assertEquals("6", handle.invoke(Integer.valueOf(6)));
+        assertDepthEquals(site, 2);
+        assertEquals("2.5", handle.invoke(Float.valueOf(2.5f)));
+        assertDepthEquals(site, 2);
+    }
+    
+    public void testTooManyTypes() throws Throwable {
+        // if this changes, test must be rewritten
+        assertEquals(5, DynamicCallSite.InliningCacheCallSite.MAX_DEPTH);
+        CallSite site = DynamicCallSite.bootstrap(MethodHandles.publicLookup(), 
+                                                  "toString", 
+                                                  MethodType.methodType(String.class, Object.class), 
+                                                  DynamicCallSite.METHOD_CALL);
+        MethodHandle handle = site.dynamicInvoker();
+        assertDepthEquals(site, 0);
+
+        assertEquals("5", handle.invoke(Integer.valueOf(5)));
+        assertDepthEquals(site, 1);
+        assertEquals("1.5", handle.invoke(Float.valueOf(1.5f)));
+        assertDepthEquals(site, 2);
+        assertEquals("6", handle.invoke(Long.valueOf(6)));
+        assertDepthEquals(site, 3);
+        assertEquals("3.2", handle.invoke(Double.valueOf(3.2d)));
+        assertDepthEquals(site, 4);
+        assertEquals("foo", handle.invoke(new String("foo")));
+        assertDepthEquals(site, 5);
+        assertEquals("c", handle.invoke(Character.valueOf('c')));
+        assertDepthEquals(site, 5);
+    }
+    
+    static void assertDepthEquals(CallSite site, int expected) {
+        DynamicCallSite.InliningCacheCallSite dsite = (DynamicCallSite.InliningCacheCallSite) site;
+        assertEquals(expected, dsite.depth);
+    }
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FieldTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FieldTests.java
deleted file mode 100644
index 04d0b1a64a1..00000000000
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FieldTests.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless;
-
-import org.junit.Before;
-
-public class FieldTests extends ScriptTestCase {
-    public static class FieldClass {
-        public boolean z = false;
-        public byte b = 0;
-        public short s = 1;
-        public char c = 'c';
-        public int i = 2;
-        public int si = -1;
-        public long j = 3L;
-        public float f = 4.0f;
-        public double d = 5.0;
-        public String t = "s";
-        public Object l = new Object();
-
-        public float test(float a, float b) {
-            return Math.min(a, b);
-        }
-
-        public int getSi() {
-            return si;
-        }
-
-        public void setSi(final int si) {
-            this.si = si;
-        }
-    }
-
-    public static class FieldDefinition extends Definition {
-        FieldDefinition() {
-            super();
-
-            addStruct("FieldClass", FieldClass.class);
-            addConstructor("FieldClass", "new", new Type[] {}, null);
-            addField("FieldClass", "z", null, false, booleanType, null);
-            addField("FieldClass", "b", null, false, byteType, null);
-            addField("FieldClass", "s", null, false, shortType, null);
-            addField("FieldClass", "c", null, false, charType, null);
-            addField("FieldClass", "i", null, false, intType, null);
-            addField("FieldClass", "j", null, false, longType, null);
-            addField("FieldClass", "f", null, false, floatType, null);
-            addField("FieldClass", "d", null, false, doubleType, null);
-            addField("FieldClass", "t", null, false, stringType, null);
-            addField("FieldClass", "l", null, false, objectType, null);
-            addClass("FieldClass");
-            addMethod("FieldClass", "getSi", null, false, intType, new Type[] {}, null, null);
-            addMethod("FieldClass", "setSi", null, false, voidType, new Type[] {intType}, null, null);
-            addMethod("FieldClass", "test", null, false, floatType, new Type[] {floatType, floatType}, null, null);
-        }
-    }
-
-    @Before
-    public void setDefinition() {
-        scriptEngine.setDefinition(new FieldDefinition());
-    }
-
-    public void testIntField() {
-        assertEquals("s5t42", exec("def fc = new FieldClass() return fc.t += 2 + fc.j + \"t\" + 4 + (3 - 1)"));
-        assertEquals(2.0f, exec("def fc = new FieldClass(); def l = new Double(3) Byte b = new Byte((byte)2) return fc.test(l, b)"));
-        assertEquals(4, exec("def fc = new FieldClass() fc.i = 4 return fc.i"));
-        assertEquals(5,
-                exec("FieldClass fc0 = new FieldClass() FieldClass fc1 = new FieldClass() fc0.i = 7 - fc0.i fc1.i = fc0.i return fc1.i"));
-        assertEquals(8, exec("def fc0 = new FieldClass() def fc1 = new FieldClass() fc0.i += fc1.i fc0.i += fc0.i return fc0.i"));
-    }
-
-    public void testExplicitShortcut() {
-        assertEquals(5, exec("FieldClass fc = new FieldClass() fc.setSi(5) return fc.si"));
-        assertEquals(-1, exec("FieldClass fc = new FieldClass() def x = fc.getSi() x"));
-        assertEquals(5, exec("FieldClass fc = new FieldClass() fc.si = 5 return fc.si"));
-        assertEquals(0, exec("FieldClass fc = new FieldClass() fc.si++ return fc.si"));
-        assertEquals(-1, exec("FieldClass fc = new FieldClass() def x = fc.si++ return x"));
-        assertEquals(0, exec("FieldClass fc = new FieldClass() def x = ++fc.si return x"));
-        assertEquals(-2, exec("FieldClass fc = new FieldClass() fc.si *= 2 fc.si"));
-        assertEquals("-1test", exec("FieldClass fc = new FieldClass() fc.si + \"test\""));
-    }
-
-    public void testImplicitShortcut() {
-        assertEquals(5, exec("def fc = new FieldClass() fc.setSi(5) return fc.si"));
-        assertEquals(-1, exec("def fc = new FieldClass() def x = fc.getSi() x"));
-        assertEquals(5, exec("def fc = new FieldClass() fc.si = 5 return fc.si"));
-        assertEquals(0, exec("def fc = new FieldClass() fc.si++ return fc.si"));
-        assertEquals(-1, exec("def fc = new FieldClass() def x = fc.si++ return x"));
-        assertEquals(0, exec("def fc = new FieldClass() def x = ++fc.si return x"));
-        assertEquals(-2, exec("def fc = new FieldClass() fc.si *= 2 fc.si"));
-        assertEquals("-1test", exec("def fc = new FieldClass() fc.si + \"test\""));
-    }
-}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NoSemiColonTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NoSemiColonTests.java
index e9c399e1eff..5ca98c2e575 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NoSemiColonTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NoSemiColonTests.java
@@ -168,6 +168,7 @@ public class NoSemiColonTests extends ScriptTestCase {
         assertEquals(4, exec("int x = 0, y = 0 while (x < 10) { ++x if (x == 5) break ++y } return y"));
     }
 
+    @SuppressWarnings("rawtypes")
     public void testReturnStatement() {
         assertEquals(10, exec("return 10"));
         assertEquals(5, exec("int x = 5 return x"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
index 088d7cf2fde..3f79ae1974b 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
@@ -24,126 +24,124 @@ import java.util.Collections;
 
 public class WhenThingsGoWrongTests extends ScriptTestCase {
     public void testNullPointer() {
-        try {
+        expectThrows(NullPointerException.class, () -> {
             exec("int x = (int) ((Map) input).get(\"missing\"); return x;");
-            fail("should have hit npe");
-        } catch (NullPointerException expected) {}
+        });
     }
 
     public void testInvalidShift() {
-        try {
+        expectThrows(ClassCastException.class, () -> {
             exec("float x = 15F; x <<= 2; return x;");
-            fail("should have hit cce");
-        } catch (ClassCastException expected) {}
+        });
 
-        try {
+        expectThrows(ClassCastException.class, () -> {
             exec("double x = 15F; x <<= 2; return x;");
-            fail("should have hit cce");
-        } catch (ClassCastException expected) {}
+        });
     }
 
     public void testBogusParameter() {
-        try {
+        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
             exec("return 5;", null, Collections.singletonMap("bogusParameterKey", "bogusParameterValue"));
-            fail("should have hit IAE");
-        } catch (IllegalArgumentException expected) {
-            assertTrue(expected.getMessage().contains("Unrecognized compile-time parameter"));
-        }
+        });
+        assertTrue(expected.getMessage().contains("Unrecognized compile-time parameter"));
     }
 
     public void testInfiniteLoops() {
-        try {
+        PainlessError expected = expectThrows(PainlessError.class, () -> {
             exec("boolean x = true; while (x) {}");
-            fail("should have hit PainlessError");
-        } catch (PainlessError expected) {
-            assertTrue(expected.getMessage().contains(
-                "The maximum number of statements that can be executed in a loop has been reached."));
-        }
+        });
+        assertTrue(expected.getMessage().contains(
+                   "The maximum number of statements that can be executed in a loop has been reached."));
 
-        try {
+        expected = expectThrows(PainlessError.class, () -> {
             exec("while (true) {int y = 5}");
-            fail("should have hit PainlessError");
-        } catch (PainlessError expected) {
-            assertTrue(expected.getMessage().contains(
-                "The maximum number of statements that can be executed in a loop has been reached."));
-        }
+        });
+        assertTrue(expected.getMessage().contains(
+                   "The maximum number of statements that can be executed in a loop has been reached."));
 
-        try {
+        expected = expectThrows(PainlessError.class, () -> {
             exec("while (true) { boolean x = true; while (x) {} }");
-            fail("should have hit PainlessError");
-        } catch (PainlessError expected) {
-            assertTrue(expected.getMessage().contains(
-                "The maximum number of statements that can be executed in a loop has been reached."));
-        }
+        });
+        assertTrue(expected.getMessage().contains(
+                   "The maximum number of statements that can be executed in a loop has been reached."));
 
-        try {
+        expected = expectThrows(PainlessError.class, () -> {
             exec("while (true) { boolean x = false; while (x) {} }");
             fail("should have hit PainlessError");
-        } catch (PainlessError expected) {
-            assertTrue(expected.getMessage().contains(
-                "The maximum number of statements that can be executed in a loop has been reached."));
-        }
+        });
+        assertTrue(expected.getMessage().contains(
+                   "The maximum number of statements that can be executed in a loop has been reached."));
 
-        try {
+        expected = expectThrows(PainlessError.class, () -> {
             exec("boolean x = true; for (;x;) {}");
             fail("should have hit PainlessError");
-        } catch (PainlessError expected) {
-            assertTrue(expected.getMessage().contains(
-                "The maximum number of statements that can be executed in a loop has been reached."));
-        }
+        });
+        assertTrue(expected.getMessage().contains(
+                   "The maximum number of statements that can be executed in a loop has been reached."));
 
-        try {
+        expected = expectThrows(PainlessError.class, () -> {
             exec("for (;;) {int x = 5}");
             fail("should have hit PainlessError");
-        } catch (PainlessError expected) {
-            assertTrue(expected.getMessage().contains(
-                "The maximum number of statements that can be executed in a loop has been reached."));
-        }
+        });
+        assertTrue(expected.getMessage().contains(
+                   "The maximum number of statements that can be executed in a loop has been reached."));
 
-        try {
+        expected = expectThrows(PainlessError.class, () -> {
             exec("def x = true; do {int y = 5;} while (x)");
             fail("should have hit PainlessError");
-        } catch (PainlessError expected) {
-            assertTrue(expected.getMessage().contains(
-                "The maximum number of statements that can be executed in a loop has been reached."));
-        }
+        });
+        assertTrue(expected.getMessage().contains(
+                   "The maximum number of statements that can be executed in a loop has been reached."));
 
-        try {
+        RuntimeException parseException = expectThrows(RuntimeException.class, () -> {
             exec("try { int x } catch (PainlessError error) {}");
             fail("should have hit ParseException");
-        } catch (RuntimeException expected) {
-            assertTrue(expected.getMessage().contains(
-                "Invalid type [PainlessError]."));
-        }
-
+        });
+        assertTrue(parseException.getMessage().contains("Invalid type [PainlessError]."));
     }
 
     public void testLoopLimits() {
+        // right below limit: ok
         exec("for (int x = 0; x < 9999; ++x) {}");
 
-        try {
+        PainlessError expected = expectThrows(PainlessError.class, () -> {
             exec("for (int x = 0; x < 10000; ++x) {}");
-            fail("should have hit PainlessError");
-        } catch (PainlessError expected) {
-            assertTrue(expected.getMessage().contains(
-                "The maximum number of statements that can be executed in a loop has been reached."));
-        }
+        });
+        assertTrue(expected.getMessage().contains(
+                   "The maximum number of statements that can be executed in a loop has been reached."));
     }
 
     public void testSourceLimits() {
-        char[] chars = new char[Compiler.MAXIMUM_SOURCE_LENGTH + 1];
-        Arrays.fill(chars, '0');
+        final char[] tooManyChars = new char[Compiler.MAXIMUM_SOURCE_LENGTH + 1];
+        Arrays.fill(tooManyChars, '0');
 
-        try {
-            exec(new String(chars));
-            fail("should have hit IllegalArgumentException");
-        } catch (IllegalArgumentException expected) {
-            assertTrue(expected.getMessage().contains("Scripts may be no longer than"));
-        }
+        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+            exec(new String(tooManyChars));
+        });
+        assertTrue(expected.getMessage().contains("Scripts may be no longer than"));
 
-        chars = new char[Compiler.MAXIMUM_SOURCE_LENGTH];
-        Arrays.fill(chars, '0');
-
-        assertEquals(0, exec(new String(chars)));
+        final char[] exactlyAtLimit = new char[Compiler.MAXIMUM_SOURCE_LENGTH];
+        Arrays.fill(exactlyAtLimit, '0');
+        // ok
+        assertEquals(0, exec(new String(exactlyAtLimit)));
+    }
+    
+    public void testIllegalDynamicMethod() {
+        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+            exec("def x = 'test'; return x.getClass().toString()");
+        });
+        assertTrue(expected.getMessage().contains("Unable to find dynamic method"));
+    }
+    
+    public void testDynamicNPE() {
+        expectThrows(NullPointerException.class, () -> {
+            exec("def x = null; return x.toString()");
+        });
+    }
+    
+    public void testDynamicWrongArgs() {
+        expectThrows(ClassCastException.class, () -> {
+            exec("def x = new ArrayList(); return x.get('bogus');");
+        });
     }
 }
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/15_update.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/15_update.yaml
new file mode 100644
index 00000000000..0cd8b52a2bb
--- /dev/null
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/15_update.yaml
@@ -0,0 +1,59 @@
+---
+"Update Script":
+
+  - do:
+      index:
+          index:  test_1
+          type:   test
+          id:     1
+          body:
+              foo:    bar
+              count:  1
+
+  - do:
+      update:
+          index:  test_1
+          type:   test
+          id:     1
+          script: "1"
+          body:
+            lang:   painless
+            script: "input.ctx._source.foo = input.bar"
+            params: { bar: 'xxx' }
+
+  - match: { _index:   test_1 }
+  - match: { _type:    test   }
+  - match: { _id:      "1"    }
+  - match: { _version: 2      }
+
+  - do:
+      get:
+          index:  test_1
+          type:   test
+          id:     1
+
+  - match: { _source.foo:        xxx }
+  - match: { _source.count:      1   }
+
+  - do:
+      update:
+          index:  test_1
+          type:   test
+          id:     1
+          lang:   painless
+          script: "input.ctx._source.foo = 'yyy'"
+
+  - match: { _index:   test_1 }
+  - match: { _type:    test   }
+  - match: { _id:      "1"    }
+  - match: { _version: 3      }
+
+  - do:
+      get:
+          index:  test_1
+          type:   test
+          id:     1
+
+  - match: { _source.foo:        yyy }
+  - match: { _source.count:      1   }
+  
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/16_update2.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/16_update2.yaml
new file mode 100644
index 00000000000..bef3250621b
--- /dev/null
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/16_update2.yaml
@@ -0,0 +1,54 @@
+---
+"Indexed script":
+
+  - do:
+      put_script:
+        id: "1"
+        lang: "painless"
+        body: { "script":  "_score * input.doc[\"myParent.weight\"].value" }
+  - match: { acknowledged: true }
+
+  - do:
+     get_script:
+       id: "1"
+       lang: "painless"
+  - match: { found: true }
+  - match: { lang: painless }
+  - match: { _id: "1" }
+  - match: { "script":  "_score * input.doc[\"myParent.weight\"].value" }
+
+  - do:
+     catch: missing
+     get_script:
+       id: "2"
+       lang: "painless"
+  - match: { found: false }
+  - match: { lang: painless }
+  - match: { _id: "2" }
+  - is_false: script
+
+  - do:
+     delete_script:
+       id: "1"
+       lang: "painless"
+  - match: { acknowledged: true }
+
+  - do:
+     catch: missing
+     delete_script:
+       id: "non_existing"
+       lang: "painless"
+
+  - do:
+      catch: request
+      put_script:
+        id: "1"
+        lang: "painless"
+        body: { "script":  "_score * foo bar + input.doc[\"myParent.weight\"].value" }
+
+  - do:
+      catch: /Unable.to.parse.*/
+      put_script:
+        id: "1"
+        lang: "painless"
+        body: { "script":  "_score * foo bar + input.doc[\"myParent.weight\"].value" }
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/25_script_upsert.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/25_script_upsert.yaml
new file mode 100644
index 00000000000..6a0fef06e37
--- /dev/null
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/25_script_upsert.yaml
@@ -0,0 +1,63 @@
+---
+"Script upsert":
+
+  - do:
+      update:
+          index:    test_1
+          type:     test
+          id:       1
+          body:
+            script: "input.ctx._source.foo = input.bar"
+            lang: "painless"
+            params: { bar: 'xxx' }
+            upsert: { foo: baz }
+
+  - do:
+      get:
+          index:  test_1
+          type:   test
+          id:     1
+
+  - match:  { _source.foo: baz }
+
+
+  - do:
+      update:
+          index:    test_1
+          type:     test
+          id:       1
+          body:
+            script: "input.ctx._source.foo = input.bar"
+            lang: "painless"
+            params: { bar: 'xxx' }
+            upsert: { foo: baz }
+
+  - do:
+      get:
+          index:  test_1
+          type:   test
+          id:     1
+
+  - match:  { _source.foo: xxx }
+
+  - do:
+      update:
+          index:    test_1
+          type:     test
+          id:       2
+          body:
+            script: "input.ctx._source.foo = input.bar"
+            lang: "painless"
+            params: { bar: 'xxx' }
+            upsert: { foo: baz }
+            scripted_upsert: true
+
+  - do:
+      get:
+          index:  test_1
+          type:   test
+          id:     2
+
+  - match:  { _source.foo: xxx }
+
+
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml
index 4a1ec86a267..db39e6a31b9 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml
@@ -94,3 +94,272 @@
     - match: { hits.hits.0.fields.sNum1.0: 1.0 }
     - match: { hits.hits.1.fields.sNum1.0: 2.0 }
     - match: { hits.hits.2.fields.sNum1.0: 3.0 }
+
+---
+
+"Custom Script Boost":
+    - do:
+        index:
+            index: test
+            type: test
+            id: 1
+            body: { "test": "value beck", "num1": 1.0 }
+    - do:
+        index:
+            index: test
+            type: test
+            id: 2
+            body: { "test": "value beck", "num1": 2.0 }
+    - do:
+        indices.refresh: {}
+
+    - do:
+        index: test
+        search:
+            body:
+                query:
+                    function_score:
+                        query:
+                            term:
+                                test: value
+                        "functions": [{
+                            "script_score": {
+                                "script": {
+                                    "lang": "painless",
+                                    "inline": "input.doc['num1'].value"
+                                }
+                            }
+                        }]
+
+    - match: { hits.total: 2 }
+    - match: { hits.hits.0._id: "2" }
+    - match: { hits.hits.1._id: "1" }
+
+    - do:
+        index: test
+        search:
+            body:
+                query:
+                    function_score:
+                        query:
+                            term:
+                                test: value
+                        "functions": [{
+                            "script_score": {
+                                "script": {
+                                    "lang": "painless",
+                                    "inline": "-input.doc['num1'].value"
+                                }
+                            }
+                        }]
+
+    - match: { hits.total: 2 }
+    - match: { hits.hits.0._id: "1" }
+    - match: { hits.hits.1._id: "2" }
+
+    - do:
+        index: test
+        search:
+            body:
+                query:
+                    function_score:
+                        query:
+                            term:
+                                test: value
+                        "functions": [{
+                            "script_score": {
+                                "script": {
+                                    "lang": "painless",
+                                    "inline": "Math.pow(input.doc['num1'].value, 2)"
+                                }
+                            }
+                        }]
+
+    - match: { hits.total: 2 }
+    - match: { hits.hits.0._id: "2" }
+    - match: { hits.hits.1._id: "1" }
+
+    - do:
+        index: test
+        search:
+            body:
+                query:
+                    function_score:
+                        query:
+                            term:
+                                test: value
+                        "functions": [{
+                            "script_score": {
+                                "script": {
+                                    "lang": "painless",
+                                    "inline": "Math.max(input.doc['num1'].value, 1)"
+                                }
+                            }
+                        }]
+
+    - match: { hits.total: 2 }
+    - match: { hits.hits.0._id: "2" }
+    - match: { hits.hits.1._id: "1" }
+
+    - do:
+        index: test
+        search:
+            body:
+                query:
+                    function_score:
+                        query:
+                            term:
+                                test: value
+                        "functions": [{
+                            "script_score": {
+                                "script": {
+                                    "lang": "painless",
+                                    "inline": "input.doc['num1'].value * _score"
+                                }
+                            }
+                        }]
+
+    - match: { hits.total: 2 }
+    - match: { hits.hits.0._id: "2" }
+    - match: { hits.hits.1._id: "1" }
+
+    - do:
+        index: test
+        search:
+            body:
+                query:
+                    function_score:
+                        query:
+                            term:
+                                test: value
+                        "functions": [{
+                            "script_score": {
+                                "script": {
+                                    "lang": "painless",
+                                    "inline": "input.param1 * input.param2 * _score",
+                                    "params": {
+                                        "param1": 2,
+                                        "param2": 2
+
+                                    }
+                                }
+                            }
+                        }]
+
+    - match: { hits.total: 2 }
+
+---
+
+"Scores Nested":
+    - do:
+        index:
+            index: test
+            type: test
+            id: 1
+            body: { "dummy_field": 1 }
+    - do:
+        indices.refresh: {}
+
+    - do:
+        index: test
+        search:
+            body:
+                query:
+                    function_score:
+                        query:
+                            function_score:
+                                "functions": [
+                                    {
+                                        "script_score": {
+                                            "script": {
+                                                "lang": "painless",
+                                                "inline": "1"
+                                            }
+                                        }
+                                    }, {
+                                        "script_score": {
+                                            "script": {
+                                                "lang": "painless",
+                                                "inline": "_score"
+                                            }
+                                        }
+                                    }
+                                ]
+                        "functions": [{
+                            "script_score": {
+                                "script": {
+                                    "lang": "painless",
+                                    "inline": "_score"
+                                }
+                            }
+                        }]
+
+    - match: { hits.total: 1 }
+    - match: { hits.hits.0._score: 1.0 }
+
+
+---
+
+"Scores With Agg":
+    - do:
+        index:
+            index: test
+            type: test
+            id: 1
+            body: { "dummy_field": 1 }
+    - do:
+        indices.refresh: {}
+
+
+    - do:
+        index: test
+        search:
+            body:
+                query:
+                    function_score:
+                        "functions": [{
+                            "script_score": {
+                                "script": {
+                                    "lang": "painless",
+                                    "inline": "_score"
+                                }
+                            }
+                        }]
+                aggs:
+                    score_agg:
+                        terms:
+                            script:
+                                lang: painless
+                                inline: "_score"
+
+    - match: { hits.total: 1 }
+    - match: { hits.hits.0._score: 1.0 }
+    - match: { aggregations.score_agg.buckets.0.key: "1.0" }
+    - match: { aggregations.score_agg.buckets.0.doc_count: 1 }
+
+---
+
+"Use List Size In Scripts":
+    - do:
+        index:
+            index: test
+            type: test
+            id: 1
+            body: { "f": 42 }
+    - do:
+        indices.refresh: {}
+
+
+    - do:
+        index: test
+        search:
+            body:
+                script_fields:
+                    foobar:
+                        script: 
+                            inline: "input.doc['f'].values.size()"
+                            lang: painless
+
+
+    - match: { hits.total: 1 }
+    - match: { hits.hits.0.fields.foobar.0: 1 }

From 5d8f684319cb0a5d025d6fc7d6990fcd0edd07d6 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Fri, 6 May 2016 12:29:07 +0200
Subject: [PATCH 0144/1311] Mapping cleanups. #18180

This removes dead/duplicate code and makes the `_index` field not configurable.
(Configuration used to jus be ignored, now we would throw an exception if any
is provided.)
---
 .../action/index/TransportIndexAction.java    |   4 +-
 .../index/mapper/ContentPath.java             |   4 -
 .../index/mapper/DocumentMapper.java          |   2 +-
 .../index/mapper/DocumentParser.java          |  45 ++---
 .../index/mapper/ParseContext.java            | 162 ++----------------
 .../index/mapper/ParsedDocument.java          |  19 +-
 .../index/mapper/SourceToParse.java           |  56 ++----
 .../org/elasticsearch/index/mapper/Uid.java   |   9 -
 .../index/mapper/internal/IdFieldMapper.java  |  45 +----
 .../mapper/internal/IndexFieldMapper.java     |  68 ++------
 .../mapper/internal/ParentFieldMapper.java    |   4 +-
 .../mapper/internal/RoutingFieldMapper.java   |  10 +-
 .../mapper/internal/SourceFieldMapper.java    |   2 +-
 .../index/mapper/internal/TTLFieldMapper.java |   4 +-
 .../mapper/internal/TypeFieldMapper.java      |   4 +-
 .../index/mapper/internal/UidFieldMapper.java |  35 +---
 .../percolator/PercolatorFieldMapper.java     |   2 +-
 .../index/query/PercolateQueryBuilder.java    |   5 +-
 .../elasticsearch/index/shard/IndexShard.java |   3 +-
 .../shard/TranslogRecoveryPerformer.java      |   2 +-
 .../index/termvectors/TermVectorsService.java |   2 +-
 .../index/IndexingSlowLogTests.java           |   6 +-
 .../index/engine/InternalEngineTests.java     |   2 +-
 .../index/engine/ShadowEngineTests.java       |   2 +-
 .../index/mapper/DocumentParserTests.java     |  11 +-
 .../index/mapper/DynamicMappingTests.java     |   7 +-
 .../index/mapper/id/IdMappingTests.java       |  21 +--
 .../mapper/index/IndexTypeMapperTests.java    |  38 +++-
 .../mapper/parent/ParentMappingTests.java     |   9 +-
 .../routing/RoutingTypeMapperTests.java       |   4 +-
 .../timestamp/TimestampMappingTests.java      |   4 +-
 .../index/mapper/ttl/TTLMappingTests.java     |   5 +-
 .../index/shard/IndexShardTests.java          |   2 +-
 .../index/mapper/size/SizeFieldMapper.java    |   5 +-
 .../index/mapper/size/SizeMappingTests.java   |   9 +-
 35 files changed, 158 insertions(+), 454 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java b/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
index 3915e23c2ed..10e18c82b86 100644
--- a/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
@@ -169,7 +169,7 @@ public class TransportIndexAction extends TransportReplicationAction<IndexReques
      */
     public static Engine.Index executeIndexRequestOnReplica(IndexRequest request, IndexShard indexShard) {
         final ShardId shardId = indexShard.shardId();
-        SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.REPLICA, request.source()).index(shardId.getIndexName()).type(request.type()).id(request.id())
+        SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.REPLICA, shardId.getIndexName(), request.type(), request.id(), request.source())
                 .routing(request.routing()).parent(request.parent()).timestamp(request.timestamp()).ttl(request.ttl());
 
         final Engine.Index operation = indexShard.prepareIndexOnReplica(sourceToParse, request.version(), request.versionType());
@@ -183,7 +183,7 @@ public class TransportIndexAction extends TransportReplicationAction<IndexReques
 
     /** Utility method to prepare an index operation on primary shards */
     public static Engine.Index prepareIndexOperationOnPrimary(IndexRequest request, IndexShard indexShard) {
-        SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.PRIMARY, request.source()).index(request.index()).type(request.type()).id(request.id())
+        SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.PRIMARY, request.index(), request.type(), request.id(), request.source())
             .routing(request.routing()).parent(request.parent()).timestamp(request.timestamp()).ttl(request.ttl());
         return indexShard.prepareIndexOnPrimary(sourceToParse, request.version(), request.versionType());
     }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ContentPath.java b/core/src/main/java/org/elasticsearch/index/mapper/ContentPath.java
index 54c6ef20e3e..6ab9019e849 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ContentPath.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ContentPath.java
@@ -42,10 +42,6 @@ public final class ContentPath {
     public ContentPath(int offset) {
         this.sb = new StringBuilder();
         this.offset = offset;
-        reset();
-    }
-
-    public void reset() {
         this.index = 0;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
index 3eb1e84e65a..9201844e743 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
@@ -279,7 +279,7 @@ public class DocumentMapper implements ToXContent {
     }
 
     public ParsedDocument parse(String index, String type, String id, BytesReference source) throws MapperParsingException {
-        return parse(SourceToParse.source(source).index(index).type(type).id(id));
+        return parse(SourceToParse.source(index, type, id, source));
     }
 
     public ParsedDocument parse(SourceToParse source) throws MapperParsingException {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
index a666ffc0ed5..197944aaacb 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
@@ -50,6 +50,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Objects;
 
 /** A parser for documents, given mappings from a DocumentMapper */
 final class DocumentParser {
@@ -67,23 +68,16 @@ final class DocumentParser {
     final ParsedDocument parseDocument(SourceToParse source) throws MapperParsingException {
         validateType(source);
 
-        source.type(docMapper.type());
         final Mapping mapping = docMapper.mapping();
-        final ParseContext.InternalParseContext context = new ParseContext.InternalParseContext(indexSettings.getSettings(), docMapperParser, docMapper, new ContentPath(0));
-        XContentParser parser = null;
-        try {
-            parser = parser(source);
-            context.reset(parser, new ParseContext.Document(), source);
+        final ParseContext.InternalParseContext context;
+        try (XContentParser parser = XContentHelper.createParser(source.source())) {
+            context = new ParseContext.InternalParseContext(indexSettings.getSettings(),
+                    docMapperParser, docMapper, source, parser);
             validateStart(parser);
             internalParseDocument(mapping, context, parser);
             validateEnd(source, parser);
         } catch (Throwable t) {
             throw wrapInMapperParsingException(source, t);
-        } finally {
-            // only close the parser when its not provided externally
-            if (internalParser(source, parser)) {
-                parser.close();
-            }
         }
         String remainingPath = context.path().pathAsText("");
         if (remainingPath.isEmpty() == false) {
@@ -120,19 +114,11 @@ final class DocumentParser {
             throw new IllegalArgumentException("It is forbidden to index into the default mapping [" + MapperService.DEFAULT_MAPPING + "]");
         }
 
-        if (source.type() != null && !source.type().equals(docMapper.type())) {
+        if (Objects.equals(source.type(), docMapper.type()) == false) {
             throw new MapperParsingException("Type mismatch, provide type [" + source.type() + "] but mapper is of type [" + docMapper.type() + "]");
         }
     }
 
-    private static XContentParser parser(SourceToParse source) throws IOException {
-        return source.parser() == null ? XContentHelper.createParser(source.source()) : source.parser();
-    }
-
-    private static boolean internalParser(SourceToParse source, XContentParser parser) {
-        return source.parser() == null && parser != null;
-    }
-
     private static void validateStart(XContentParser parser) throws IOException {
         // will result in START_OBJECT
         XContentParser.Token token = parser.nextToken();
@@ -143,13 +129,11 @@ final class DocumentParser {
 
     private static void validateEnd(SourceToParse source, XContentParser parser) throws IOException {
         XContentParser.Token token;// only check for end of tokens if we created the parser here
-        if (internalParser(source, parser)) {
-            // try to parse the next token, this should be null if the object is ended properly
-            // but will throw a JSON exception if the extra tokens is not valid JSON (this will be handled by the catch)
-            token = parser.nextToken();
-            if (token != null) {
-                throw new IllegalArgumentException("Malformed content, found extra data after parsing: " + token);
-            }
+        // try to parse the next token, this should be null if the object is ended properly
+        // but will throw a JSON exception if the extra tokens is not valid JSON (this will be handled by the catch)
+        token = parser.nextToken();
+        if (token != null) {
+            throw new IllegalArgumentException("Malformed content, found extra data after parsing: " + token);
         }
     }
 
@@ -175,15 +159,14 @@ final class DocumentParser {
 
     private static ParsedDocument parsedDocument(SourceToParse source, ParseContext.InternalParseContext context, Mapping update) {
         return new ParsedDocument(
-            context.uid(),
             context.version(),
-            context.id(),
-            context.type(),
+            context.sourceToParse().id(),
+            context.sourceToParse().type(),
             source.routing(),
             source.timestamp(),
             source.ttl(),
             context.docs(),
-            context.source(),
+            context.sourceToParse().source(),
             update
         ).parent(source.parent());
     }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java b/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
index 33476b48cc3..27b983060a3 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
@@ -22,17 +22,11 @@ package org.elasticsearch.index.mapper;
 import com.carrotsearch.hppc.ObjectObjectHashMap;
 import com.carrotsearch.hppc.ObjectObjectMap;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.document.LegacyIntField;
-import org.apache.lucene.document.LegacyLongField;
-import org.apache.lucene.document.LegacyFloatField;
-import org.apache.lucene.document.LegacyDoubleField;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.all.AllEntries;
-import org.elasticsearch.common.lucene.all.AllField;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.AnalysisService;
@@ -133,8 +127,6 @@ public abstract class ParseContext {
          * Returns an array of values of the field specified as the method parameter.
          * This method returns an empty array when there are no
          * matching fields.  It never returns null.
-         * For {@link org.apache.lucene.document.LegacyIntField}, {@link org.apache.lucene.document.LegacyLongField}, {@link
-         * org.apache.lucene.document.LegacyFloatField} and {@link org.apache.lucene.document.LegacyDoubleField} it returns the string value of the number.
          * If you want the actual numeric field instances back, use {@link #getFields}.
          * @param name the name of the field
          * @return a <code>String[]</code> of field values
@@ -201,36 +193,16 @@ public abstract class ParseContext {
             return in.isWithinMultiFields();
         }
 
-        @Override
-        public String index() {
-            return in.index();
-        }
-
         @Override
         public Settings indexSettings() {
             return in.indexSettings();
         }
 
-        @Override
-        public String type() {
-            return in.type();
-        }
-
         @Override
         public SourceToParse sourceToParse() {
             return in.sourceToParse();
         }
 
-        @Override
-        public BytesReference source() {
-            return in.source();
-        }
-
-        @Override
-        public void source(BytesReference source) {
-            in.source(source);
-        }
-
         @Override
         public ContentPath path() {
             return in.path();
@@ -257,7 +229,7 @@ public abstract class ParseContext {
         }
 
         @Override
-        public void addDoc(Document doc) {
+        protected void addDoc(Document doc) {
             in.addDoc(doc);
         }
 
@@ -281,26 +253,6 @@ public abstract class ParseContext {
             return in.mapperService();
         }
 
-        @Override
-        public String id() {
-            return in.id();
-        }
-
-        @Override
-        public void id(String id) {
-            in.id(id);
-        }
-
-        @Override
-        public Field uid() {
-            return in.uid();
-        }
-
-        @Override
-        public void uid(Field uid) {
-            in.uid(uid);
-        }
-
         @Override
         public Field version() {
             return in.version();
@@ -345,48 +297,37 @@ public abstract class ParseContext {
 
         private final ContentPath path;
 
-        private XContentParser parser;
+        private final XContentParser parser;
 
         private Document document;
 
-        private List<Document> documents = new ArrayList<>();
+        private final List<Document> documents;
 
         @Nullable
         private final Settings indexSettings;
 
-        private SourceToParse sourceToParse;
-        private BytesReference source;
+        private final SourceToParse sourceToParse;
 
-        private String id;
+        private Field version;
 
-        private Field uid, version;
+        private StringBuilder stringBuilder = new StringBuilder();
 
-        private AllEntries allEntries = new AllEntries();
+        private final AllEntries allEntries;
 
-        private List<Mapper> dynamicMappers = new ArrayList<>();
+        private final List<Mapper> dynamicMappers;
 
-        public InternalParseContext(@Nullable Settings indexSettings, DocumentMapperParser docMapperParser, DocumentMapper docMapper, ContentPath path) {
+        public InternalParseContext(@Nullable Settings indexSettings, DocumentMapperParser docMapperParser, DocumentMapper docMapper,
+                SourceToParse source, XContentParser parser) {
             this.indexSettings = indexSettings;
             this.docMapper = docMapper;
             this.docMapperParser = docMapperParser;
-            this.path = path;
-        }
-
-        public void reset(XContentParser parser, Document document, SourceToParse source) {
+            this.path = new ContentPath(0);
             this.parser = parser;
-            this.document = document;
-            if (document != null) {
-                this.documents = new ArrayList<>();
-                this.documents.add(document);
-            } else {
-                this.documents = null;
-            }
-            this.uid = null;
+            this.document = new Document();
+            this.documents = new ArrayList<>();
+            this.documents.add(document);
             this.version = null;
-            this.id = null;
             this.sourceToParse = source;
-            this.source = source == null ? null : sourceToParse.source();
-            this.path.reset();
             this.allEntries = new AllEntries();
             this.dynamicMappers = new ArrayList<>();
         }
@@ -396,38 +337,17 @@ public abstract class ParseContext {
             return this.docMapperParser;
         }
 
-        @Override
-        public String index() {
-            return sourceToParse.index();
-        }
-
         @Override
         @Nullable
         public Settings indexSettings() {
             return this.indexSettings;
         }
 
-        @Override
-        public String type() {
-            return sourceToParse.type();
-        }
-
         @Override
         public SourceToParse sourceToParse() {
             return this.sourceToParse;
         }
 
-        @Override
-        public BytesReference source() {
-            return source;
-        }
-
-        // only should be used by SourceFieldMapper to update with a compressed source
-        @Override
-        public void source(BytesReference source) {
-            this.source = source;
-        }
-
         @Override
         public ContentPath path() {
             return this.path;
@@ -454,7 +374,7 @@ public abstract class ParseContext {
         }
 
         @Override
-        public void addDoc(Document doc) {
+        protected void addDoc(Document doc) {
             this.documents.add(doc);
         }
 
@@ -478,32 +398,6 @@ public abstract class ParseContext {
             return docMapperParser.mapperService;
         }
 
-        @Override
-        public String id() {
-            return id;
-        }
-
-        /**
-         * Really, just the id mapper should set this.
-         */
-        @Override
-        public void id(String id) {
-            this.id = id;
-        }
-
-        @Override
-        public Field uid() {
-            return this.uid;
-        }
-
-        /**
-         * Really, just the uid mapper should set this.
-         */
-        @Override
-        public void uid(Field uid) {
-            this.uid = uid;
-        }
-
         @Override
         public Field version() {
             return this.version;
@@ -597,21 +491,11 @@ public abstract class ParseContext {
         return false;
     }
 
-    public abstract String index();
-
     @Nullable
     public abstract Settings indexSettings();
 
-    public abstract String type();
-
     public abstract SourceToParse sourceToParse();
 
-    @Nullable
-    public abstract BytesReference source();
-
-    // only should be used by SourceFieldMapper to update with a compressed source
-    public abstract void source(BytesReference source);
-
     public abstract ContentPath path();
 
     public abstract XContentParser parser();
@@ -622,7 +506,7 @@ public abstract class ParseContext {
 
     public abstract Document doc();
 
-    public abstract void addDoc(Document doc);
+    protected abstract void addDoc(Document doc);
 
     public abstract RootObjectMapper root();
 
@@ -632,20 +516,6 @@ public abstract class ParseContext {
 
     public abstract MapperService mapperService();
 
-    public abstract String id();
-
-    /**
-     * Really, just the id mapper should set this.
-     */
-    public abstract void id(String id);
-
-    public abstract Field uid();
-
-    /**
-     * Really, just the uid mapper should set this.
-     */
-    public abstract void uid(Field uid);
-
     public abstract Field version();
 
     public abstract void version(Field version);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java b/core/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java
index aa35e699b2d..81ca392a50e 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java
@@ -30,11 +30,9 @@ import java.util.List;
  */
 public class ParsedDocument {
 
-    private final Field uid, version;
+    private final Field version;
 
-    private final String id;
-
-    private final String type;
+    private final String uid, id, type;
 
     private final String routing;
 
@@ -50,11 +48,11 @@ public class ParsedDocument {
 
     private String parent;
 
-    public ParsedDocument(Field uid, Field version, String id, String type, String routing, long timestamp, long ttl, List<Document> documents, BytesReference source, Mapping dynamicMappingsUpdate) {
-        this.uid = uid;
+    public ParsedDocument(Field version, String id, String type, String routing, long timestamp, long ttl, List<Document> documents, BytesReference source, Mapping dynamicMappingsUpdate) {
         this.version = version;
         this.id = id;
         this.type = type;
+        this.uid = Uid.createUid(type, id);
         this.routing = routing;
         this.timestamp = timestamp;
         this.ttl = ttl;
@@ -62,15 +60,14 @@ public class ParsedDocument {
         this.source = source;
         this.dynamicMappingsUpdate = dynamicMappingsUpdate;
     }
-
-    public Field uid() {
-        return this.uid;
-    }
-
     public Field version() {
         return version;
     }
 
+    public String uid() {
+        return uid;
+    }
+
     public String id() {
         return this.id;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java b/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java
index 6094caa319a..173b14eefef 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java
@@ -19,38 +19,33 @@
 
 package org.elasticsearch.index.mapper;
 
+import java.util.Objects;
+
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.XContentParser;
 
 /**
  *
  */
 public class SourceToParse {
 
-    public static SourceToParse source(XContentParser parser) {
-        return new SourceToParse(Origin.PRIMARY, parser);
+    public static SourceToParse source(String index, String type, String id, BytesReference source) {
+        return source(Origin.PRIMARY, index, type, id, source);
     }
 
-    public static SourceToParse source(BytesReference source) {
-        return new SourceToParse(Origin.PRIMARY, source);
-    }
-
-    public static SourceToParse source(Origin origin, BytesReference source) {
-        return new SourceToParse(origin, source);
+    public static SourceToParse source(Origin origin, String index, String type, String id, BytesReference source) {
+        return new SourceToParse(origin, index, type, id, source);
     }
 
     private final Origin origin;
 
     private final BytesReference source;
 
-    private final XContentParser parser;
+    private final String index;
 
-    private String index;
+    private final String type;
 
-    private String type;
-
-    private String id;
+    private final String id;
 
     private String routing;
 
@@ -60,28 +55,20 @@ public class SourceToParse {
 
     private long ttl;
 
-    private SourceToParse(Origin origin, XContentParser parser) {
-        this.origin = origin;
-        this.parser = parser;
-        this.source = null;
-    }
-
-    private SourceToParse(Origin origin, BytesReference source) {
-        this.origin = origin;
+    private SourceToParse(Origin origin, String index, String type, String id, BytesReference source) {
+        this.origin = Objects.requireNonNull(origin);
+        this.index = Objects.requireNonNull(index);
+        this.type = Objects.requireNonNull(type);
+        this.id = Objects.requireNonNull(id);
         // we always convert back to byte array, since we store it and Field only supports bytes..
         // so, we might as well do it here, and improve the performance of working with direct byte arrays
         this.source = source.toBytesArray();
-        this.parser = null;
     }
 
     public Origin origin() {
         return origin;
     }
 
-    public XContentParser parser() {
-        return this.parser;
-    }
-
     public BytesReference source() {
         return this.source;
     }
@@ -90,29 +77,14 @@ public class SourceToParse {
         return this.index;
     }
 
-    public SourceToParse index(String index) {
-        this.index = index;
-        return this;
-    }
-
     public String type() {
         return this.type;
     }
 
-    public SourceToParse type(String type) {
-        this.type = type;
-        return this;
-    }
-
     public String id() {
         return this.id;
     }
 
-    public SourceToParse id(String id) {
-        this.id = id;
-        return this;
-    }
-
     public String parent() {
         return this.parentId;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/Uid.java b/core/src/main/java/org/elasticsearch/index/mapper/Uid.java
index 70acbcd82c3..831f4575f3d 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/Uid.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/Uid.java
@@ -95,15 +95,6 @@ public final class Uid {
         return new Uid(uid.substring(0, delimiterIndex), uid.substring(delimiterIndex + 1));
     }
 
-    public static BytesRef[] createUids(List<? extends DocumentRequest> requests) {
-        BytesRef[] uids = new BytesRef[requests.size()];
-        int idx = 0;
-        for (DocumentRequest item : requests) {
-            uids[idx++] = createUidAsBytes(item.type(), item.id());
-        }
-        return uids;
-    }
-
     public static BytesRef createUidAsBytes(String type, String id) {
         return createUidAsBytes(new BytesRef(type), new BytesRef(id));
     }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
index a419ad11ebf..064bc5fb69d 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.index.mapper.internal;
 
-import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.Term;
@@ -37,7 +36,6 @@ import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.iterable.Iterables;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
@@ -52,7 +50,9 @@ import java.util.List;
 import java.util.Map;
 
 /**
- *
+ * A mapper for the _id field. It does nothing since _id is neither indexed nor
+ * stored, but we need to keep it so that its FieldType can be used to generate
+ * queries.
  */
 public class IdFieldMapper extends MetadataFieldMapper {
 
@@ -84,12 +84,6 @@ public class IdFieldMapper extends MetadataFieldMapper {
             indexName = Defaults.NAME;
         }
 
-        // if we are indexed we use DOCS
-        @Override
-        protected IndexOptions getDefaultIndexOption() {
-            return IndexOptions.DOCS;
-        }
-
         @Override
         public IdFieldMapper build(BuilderContext context) {
             setupFieldType(context);
@@ -203,40 +197,13 @@ public class IdFieldMapper extends MetadataFieldMapper {
     }
 
     @Override
-    public void preParse(ParseContext context) throws IOException {
-        if (context.sourceToParse().id() != null) {
-            context.id(context.sourceToParse().id());
-            super.parse(context);
-        }
-    }
+    public void preParse(ParseContext context) throws IOException {}
 
     @Override
-    public void postParse(ParseContext context) throws IOException {
-        if (context.id() == null) {
-            throw new MapperParsingException("No id found while parsing the content source");
-        }
-        // it either get built in the preParse phase, or get parsed...
-    }
+    public void postParse(ParseContext context) throws IOException {}
 
     @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
-        XContentParser parser = context.parser();
-        if (parser.currentName() != null && parser.currentName().equals(Defaults.NAME) && parser.currentToken().isValue()) {
-            // we are in the parse Phase
-            String id = parser.text();
-            if (context.id() != null && !context.id().equals(id)) {
-                throw new MapperParsingException("Provided id [" + context.id() + "] does not match the content one [" + id + "]");
-            }
-            context.id(id);
-        } // else we are in the pre/post parse phase
-
-        if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
-            fields.add(new Field(fieldType().name(), context.id(), fieldType()));
-        }
-        if (fieldType().hasDocValues()) {
-            fields.add(new BinaryDocValuesField(fieldType().name(), new BytesRef(context.id())));
-        }
-    }
+    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {}
 
     @Override
     protected String contentType() {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
index b402f45bfdd..a24597f7dff 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
@@ -19,11 +19,11 @@
 
 package org.elasticsearch.index.mapper.internal;
 
-import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.search.Queries;
@@ -58,7 +58,7 @@ public class IndexFieldMapper extends MetadataFieldMapper {
         public static final MappedFieldType FIELD_TYPE = new IndexFieldType();
 
         static {
-            FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
+            FIELD_TYPE.setIndexOptions(IndexOptions.NONE);
             FIELD_TYPE.setTokenized(false);
             FIELD_TYPE.setStored(false);
             FIELD_TYPE.setOmitNorms(true);
@@ -67,35 +67,28 @@ public class IndexFieldMapper extends MetadataFieldMapper {
             FIELD_TYPE.setName(NAME);
             FIELD_TYPE.freeze();
         }
-
-        public static final EnabledAttributeMapper ENABLED_STATE = EnabledAttributeMapper.UNSET_DISABLED;
     }
 
     public static class Builder extends MetadataFieldMapper.Builder<Builder, IndexFieldMapper> {
 
-        private EnabledAttributeMapper enabledState = EnabledAttributeMapper.UNSET_DISABLED;
-
         public Builder(MappedFieldType existing) {
             super(Defaults.NAME, existing == null ? Defaults.FIELD_TYPE : existing, Defaults.FIELD_TYPE);
             indexName = Defaults.NAME;
         }
 
-        public Builder enabled(EnabledAttributeMapper enabledState) {
-            this.enabledState = enabledState;
-            return this;
-        }
-
         @Override
         public IndexFieldMapper build(BuilderContext context) {
             setupFieldType(context);
-            fieldType.setHasDocValues(false);
-            return new IndexFieldMapper(fieldType, enabledState, context.indexSettings());
+            return new IndexFieldMapper(fieldType, context.indexSettings());
         }
     }
 
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
-        public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+        public MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+            if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0)) {
+                throw new MapperParsingException(NAME + " is not configurable");
+            }
             return new Builder(parserContext.mapperService().fullName(NAME));
         }
 
@@ -179,43 +172,22 @@ public class IndexFieldMapper extends MetadataFieldMapper {
         }
     }
 
-    private EnabledAttributeMapper enabledState;
-
     private IndexFieldMapper(Settings indexSettings, MappedFieldType existing) {
-        this(existing == null ? Defaults.FIELD_TYPE.clone() : existing, Defaults.ENABLED_STATE, indexSettings);
+        this(existing == null ? Defaults.FIELD_TYPE.clone() : existing, indexSettings);
     }
 
-    private IndexFieldMapper(MappedFieldType fieldType, EnabledAttributeMapper enabledState, Settings indexSettings) {
+    private IndexFieldMapper(MappedFieldType fieldType, Settings indexSettings) {
         super(NAME, fieldType, Defaults.FIELD_TYPE, indexSettings);
-        this.enabledState = enabledState;
-    }
-
-    public boolean enabled() {
-        return this.enabledState.enabled;
     }
 
     @Override
-    public void preParse(ParseContext context) throws IOException {
-        // we pre parse it and not in parse, since its not part of the root object
-        super.parse(context);
-    }
+    public void preParse(ParseContext context) throws IOException {}
 
     @Override
-    public void postParse(ParseContext context) throws IOException {
-    }
+    public void postParse(ParseContext context) throws IOException {}
 
     @Override
-    public Mapper parse(ParseContext context) throws IOException {
-        return null;
-    }
-
-    @Override
-    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
-        if (!enabledState.enabled) {
-            return;
-        }
-        fields.add(new Field(fieldType().name(), context.index(), fieldType()));
-    }
+    protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {}
 
     @Override
     protected String contentType() {
@@ -224,26 +196,12 @@ public class IndexFieldMapper extends MetadataFieldMapper {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        boolean includeDefaults = params.paramAsBoolean("include_defaults", false);
-
-        // if all defaults, no need to write it at all
-        if (includeDefaults == false && enabledState == Defaults.ENABLED_STATE) {
-            return builder;
-        }
-        builder.startObject(CONTENT_TYPE);
-        if (includeDefaults || enabledState != Defaults.ENABLED_STATE) {
-            builder.field("enabled", enabledState.enabled);
-        }
-        builder.endObject();
         return builder;
     }
 
     @Override
     protected void doMerge(Mapper mergeWith, boolean updateAllTypes) {
-        IndexFieldMapper indexFieldMapperMergeWith = (IndexFieldMapper) mergeWith;
-        if (indexFieldMapperMergeWith.enabledState != enabledState && !indexFieldMapperMergeWith.enabledState.unset()) {
-            this.enabledState = indexFieldMapperMergeWith.enabledState;
-        }
+        // nothing to do
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
index 2bf4f75f417..b98eb6720bc 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
@@ -238,9 +238,9 @@ public class ParentFieldMapper extends MetadataFieldMapper {
 
     @Override
     protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
-        boolean parent = context.docMapper().isParent(context.type());
+        boolean parent = context.docMapper().isParent(context.sourceToParse().type());
         if (parent) {
-            fields.add(new SortedDocValuesField(parentJoinField.fieldType().name(), new BytesRef(context.id())));
+            fields.add(new SortedDocValuesField(parentJoinField.fieldType().name(), new BytesRef(context.sourceToParse().id())));
         }
 
         if (!active()) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java
index 7777d90d055..68641ebc29a 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java
@@ -165,12 +165,10 @@ public class RoutingFieldMapper extends MetadataFieldMapper {
 
     @Override
     protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
-        if (context.sourceToParse().routing() != null) {
-            String routing = context.sourceToParse().routing();
-            if (routing != null) {
-                if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
-                    fields.add(new Field(fieldType().name(), routing, fieldType()));
-                }
+        String routing = context.sourceToParse().routing();
+        if (routing != null) {
+            if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
+                fields.add(new Field(fieldType().name(), routing, fieldType()));
             }
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
index ae9ef44f90d..a05b3a451c7 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
@@ -229,7 +229,7 @@ public class SourceFieldMapper extends MetadataFieldMapper {
         if (!fieldType().stored()) {
             return;
         }
-        BytesReference source = context.source();
+        BytesReference source = context.sourceToParse().source();
         // Percolate and tv APIs may not set the source and that is ok, because these APIs will not index any data
         if (source == null) {
             return;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java
index 3418d33a976..33d194a7892 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java
@@ -21,7 +21,6 @@ package org.elasticsearch.index.mapper.internal;
 
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -223,7 +222,8 @@ public class TTLFieldMapper extends MetadataFieldMapper {
                 long now = System.currentTimeMillis();
                 // there is not point indexing already expired doc
                 if (context.sourceToParse().origin() == SourceToParse.Origin.PRIMARY && now >= expire) {
-                    throw new AlreadyExpiredException(context.index(), context.type(), context.id(), timestamp, ttl, now);
+                    throw new AlreadyExpiredException(context.sourceToParse().index(),
+                            context.sourceToParse().type(), context.sourceToParse().id(), timestamp, ttl, now);
                 }
                 // the expiration timestamp (timestamp + ttl) is set as field
                 fields.add(new LegacyLongFieldMapper.CustomLongNumericField(expire, fieldType()));
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
index c548155e158..6e2dbbcc7d0 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
@@ -208,9 +208,9 @@ public class TypeFieldMapper extends MetadataFieldMapper {
         if (fieldType().indexOptions() == IndexOptions.NONE && !fieldType().stored()) {
             return;
         }
-        fields.add(new Field(fieldType().name(), context.type(), fieldType()));
+        fields.add(new Field(fieldType().name(), context.sourceToParse().type(), fieldType()));
         if (fieldType().hasDocValues()) {
-            fields.add(new SortedSetDocValuesField(fieldType().name(), new BytesRef(context.type())));
+            fields.add(new SortedSetDocValuesField(fieldType().name(), new BytesRef(context.sourceToParse().type())));
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
index b22369ef812..fc19201154c 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
@@ -22,7 +22,6 @@ package org.elasticsearch.index.mapper.internal;
 import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
@@ -34,7 +33,6 @@ import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.ParseContext;
-import org.elasticsearch.index.mapper.ParseContext.Document;
 import org.elasticsearch.index.mapper.core.TextFieldMapper;
 import org.elasticsearch.index.mapper.Uid;
 
@@ -138,46 +136,21 @@ public class UidFieldMapper extends MetadataFieldMapper {
 
     @Override
     public void preParse(ParseContext context) throws IOException {
-        // if we have the id provided, fill it, and parse now
-        if (context.sourceToParse().id() != null) {
-            context.id(context.sourceToParse().id());
-            super.parse(context);
-        }
+        super.parse(context);
     }
 
     @Override
-    public void postParse(ParseContext context) throws IOException {
-        if (context.id() == null) {
-            throw new MapperParsingException("No id found while parsing the content source");
-        }
-        // if we did not have the id as part of the sourceToParse, then we need to parse it here
-        // it would have been filled in the _id parse phase
-        if (context.sourceToParse().id() == null) {
-            super.parse(context);
-            // since we did not have the uid in the pre phase, we did not add it automatically to the nested docs
-            // as they were created we need to make sure we add it to all the nested docs...
-            if (context.docs().size() > 1) {
-                final IndexableField uidField = context.rootDoc().getField(UidFieldMapper.NAME);
-                assert uidField != null;
-                // we need to go over the docs and add it...
-                for (int i = 1; i < context.docs().size(); i++) {
-                    final Document doc = context.docs().get(i);
-                    doc.add(new Field(UidFieldMapper.NAME, uidField.stringValue(), Defaults.NESTED_FIELD_TYPE));
-                }
-            }
-        }
-    }
+    public void postParse(ParseContext context) throws IOException {}
 
     @Override
     public Mapper parse(ParseContext context) throws IOException {
-        // nothing to do here, we either do it in post parse, or in pre parse.
+        // nothing to do here, we do everything in preParse
         return null;
     }
 
     @Override
     protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
-        Field uid = new Field(NAME, Uid.createUid(context.type(), context.id()), Defaults.FIELD_TYPE);
-        context.uid(uid);
+        Field uid = new Field(NAME, Uid.createUid(context.sourceToParse().type(), context.sourceToParse().id()), Defaults.FIELD_TYPE);
         fields.add(uid);
         if (fieldType().hasDocValues()) {
             fields.add(new BinaryDocValuesField(NAME, new BytesRef(uid.stringValue())));
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
index 58f9f6c695c..c50085857ff 100644
--- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
@@ -172,7 +172,7 @@ public class PercolatorFieldMapper extends FieldMapper {
     @Override
     public Mapper parse(ParseContext context) throws IOException {
         QueryShardContext queryShardContext = new QueryShardContext(this.queryShardContext);
-        DocumentMapper documentMapper = queryShardContext.getMapperService().documentMapper(context.type());
+        DocumentMapper documentMapper = queryShardContext.getMapperService().documentMapper(context.sourceToParse().type());
         for (FieldMapper fieldMapper : documentMapper.mappers()) {
             if (fieldMapper instanceof PercolatorFieldMapper) {
                 PercolatorFieldType fieldType = (PercolatorFieldType) fieldMapper.fieldType();
diff --git a/core/src/main/java/org/elasticsearch/index/query/PercolateQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/PercolateQueryBuilder.java
index 0e4b999dc33..8341459b155 100644
--- a/core/src/main/java/org/elasticsearch/index/query/PercolateQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/PercolateQueryBuilder.java
@@ -362,10 +362,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
         DocumentMapperForType docMapperForType = mapperService.documentMapperWithAutoCreate(documentType);
         DocumentMapper docMapper = docMapperForType.getDocumentMapper();
 
-        ParsedDocument doc = docMapper.parse(source(document)
-            .index(context.index().getName())
-            .id("_temp_id")
-            .type(documentType));
+        ParsedDocument doc = docMapper.parse(source(context.index().getName(), documentType, "_temp_id", document));
 
         FieldNameAnalyzer fieldNameAnalyzer = (FieldNameAnalyzer) docMapper.mappers().indexAnalyzer();
         // Need to this custom impl because FieldNameAnalyzer is strict and the percolator sometimes isn't when
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index c1610e9bb1f..59ea1a29bcf 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -85,7 +85,6 @@ import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.merge.MergeStats;
-import org.elasticsearch.index.percolator.PercolatorFieldMapper;
 import org.elasticsearch.index.recovery.RecoveryStats;
 import org.elasticsearch.index.refresh.RefreshStats;
 import org.elasticsearch.index.search.stats.SearchStats;
@@ -478,7 +477,7 @@ public class IndexShard extends AbstractIndexShardComponent {
             doc.addDynamicMappingsUpdate(docMapper.getMapping());
         }
         MappedFieldType uidFieldType = docMapper.getDocumentMapper().uidMapper().fieldType();
-        Query uidQuery = uidFieldType.termQuery(doc.uid().stringValue(), null);
+        Query uidQuery = uidFieldType.termQuery(doc.uid(), null);
         Term uid = MappedFieldType.extractTerm(uidQuery);
         return new Engine.Index(uid, doc, version, versionType, origin, startTime);
     }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java
index 68aa7dfa865..f3903fb6f9d 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java
@@ -151,7 +151,7 @@ public class TranslogRecoveryPerformer {
             switch (operation.opType()) {
                 case INDEX:
                     Translog.Index index = (Translog.Index) operation;
-                    Engine.Index engineIndex = IndexShard.prepareIndex(docMapper(index.type()), source(index.source()).type(index.type()).id(index.id())
+                    Engine.Index engineIndex = IndexShard.prepareIndex(docMapper(index.type()), source(shardId.getIndexName(), index.type(), index.id(), index.source())
                                     .routing(index.routing()).parent(index.parent()).timestamp(index.timestamp()).ttl(index.ttl()),
                             index.version(), index.versionType().versionTypeForReplicationAndRecovery(), Engine.Operation.Origin.RECOVERY);
                     maybeAddMappingUpdate(engineIndex.type(), engineIndex.parsedDoc().dynamicMappingsUpdate(), engineIndex.id(), allowMappingUpdates);
diff --git a/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
index 2465c7718a0..e4534e59d80 100644
--- a/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
+++ b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
@@ -290,7 +290,7 @@ public class TermVectorsService  {
     private static ParsedDocument parseDocument(IndexShard indexShard, String index, String type, BytesReference doc) throws Throwable {
         MapperService mapperService = indexShard.mapperService();
         DocumentMapperForType docMapper = mapperService.documentMapperWithAutoCreate(type);
-        ParsedDocument parsedDocument = docMapper.getDocumentMapper().parse(source(doc).index(index).type(type).id("_id_for_tv_api"));
+        ParsedDocument parsedDocument = docMapper.getDocumentMapper().parse(source(index, type, "_id_for_tv_api", doc));
         if (docMapper.getMapping() != null) {
             parsedDocument.addDynamicMappingsUpdate(docMapper.getMapping());
         }
diff --git a/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java b/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
index 0c202ee71be..a076224eae5 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
@@ -19,9 +19,7 @@
 
 package org.elasticsearch.index;
 
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.LegacyIntField;
-import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.NumericDocValuesField;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -41,7 +39,7 @@ import static org.hamcrest.Matchers.startsWith;
 public class IndexingSlowLogTests extends ESTestCase {
     public void testSlowLogParsedDocumentPrinterSourceToLog() throws IOException {
         BytesReference source = JsonXContent.contentBuilder().startObject().field("foo", "bar").endObject().bytes();
-        ParsedDocument pd = new ParsedDocument(new StringField("uid", "test:id", Store.YES), new LegacyIntField("version", 1, Store.YES), "id",
+        ParsedDocument pd = new ParsedDocument(new NumericDocValuesField("version", 1), "id",
                 "test", null, 0, -1, null, source, null);
         Index index = new Index("foo", "123");
         // Turning off document logging doesn't log source[]
diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index afdcb42ae83..0e20c22e119 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -229,7 +229,7 @@ public class InternalEngineTests extends ESTestCase {
         Field versionField = new NumericDocValuesField("_version", 0);
         document.add(uidField);
         document.add(versionField);
-        return new ParsedDocument(uidField, versionField, id, type, routing, timestamp, ttl, Arrays.asList(document), source, mappingUpdate);
+        return new ParsedDocument(versionField, id, type, routing, timestamp, ttl, Arrays.asList(document), source, mappingUpdate);
     }
 
     protected Store createStore() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
index 6695fb471f6..8a5dcea5d6d 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
@@ -172,7 +172,7 @@ public class ShadowEngineTests extends ESTestCase {
         document.add(uidField);
         document.add(versionField);
         document.add(new LongPoint("point_field", 42)); // so that points report memory/disk usage
-        return new ParsedDocument(uidField, versionField, id, type, routing, timestamp, ttl, Arrays.asList(document), source, mappingsUpdate);
+        return new ParsedDocument(versionField, id, type, routing, timestamp, ttl, Arrays.asList(document), source, mappingsUpdate);
     }
 
     protected Store createStore(Path p) throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
index 0e53fcb8d7f..b9e5d6d3cf8 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
@@ -171,13 +171,12 @@ public class DocumentParserTests extends ESSingleNodeTestCase {
 
     // creates an object mapper, which is about 100x harder than it should be....
     ObjectMapper createObjectMapper(MapperService mapperService, String name) throws Exception {
-        String[] nameParts = name.split("\\.");
-        ContentPath path = new ContentPath();
-        for (int i = 0; i < nameParts.length - 1; ++i) {
-            path.add(nameParts[i]);
-        }
         ParseContext context = new ParseContext.InternalParseContext(Settings.EMPTY,
-            mapperService.documentMapperParser(), mapperService.documentMapper("type"), path);
+            mapperService.documentMapperParser(), mapperService.documentMapper("type"), null, null);
+        String[] nameParts = name.split("\\.");
+        for (int i = 0; i < nameParts.length - 1; ++i) {
+            context.path().add(nameParts[i]);
+        }
         Mapper.Builder builder = new ObjectMapper.Builder(nameParts[nameParts.length - 1]).enabled(true);
         Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings(), context.path());
         return (ObjectMapper)builder.build(builderContext);
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
index 5ea5630d463..af96bc503c0 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
@@ -206,9 +206,8 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
 
     private Mapper parse(DocumentMapper mapper, DocumentMapperParser parser, XContentBuilder builder) throws Exception {
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
-        ParseContext.InternalParseContext ctx = new ParseContext.InternalParseContext(settings, parser, mapper, new ContentPath(0));
-        SourceToParse source = SourceToParse.source(builder.bytes());
-        ctx.reset(XContentHelper.createParser(source.source()), new ParseContext.Document(), source);
+        SourceToParse source = SourceToParse.source("test", mapper.type(), "some_id", builder.bytes());
+        ParseContext.InternalParseContext ctx = new ParseContext.InternalParseContext(settings, parser, mapper, source, XContentHelper.createParser(source.source()));
         assertEquals(XContentParser.Token.START_OBJECT, ctx.parser().nextToken());
         ctx.parser().nextToken();
         DocumentParser.parseObjectOrNested(ctx, mapper.root(), true);
@@ -558,7 +557,7 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
         XContentBuilder json = XContentFactory.jsonBuilder().startObject()
                     .field("field", "foo")
                 .endObject();
-        SourceToParse source = SourceToParse.source(json.bytes()).id("1");
+        SourceToParse source = SourceToParse.source("test", "type1", "1", json.bytes());
         DocumentMapper mapper = indexService.mapperService().documentMapper("type1");
         assertNull(mapper.mappers().getMapper("field.raw"));
         ParsedDocument parsed = mapper.parse(source);
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/id/IdMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/id/IdMappingTests.java
index da56d346ed5..602275d6559 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/id/IdMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/id/IdMappingTests.java
@@ -19,23 +19,16 @@
 
 package org.elasticsearch.index.mapper.id;
 
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceToParse;
-import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.mapper.internal.IdFieldMapper;
 import org.elasticsearch.index.mapper.internal.UidFieldMapper;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
-import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -53,16 +46,6 @@ public class IdMappingTests extends ESSingleNodeTestCase {
 
         assertThat(doc.rootDoc().get(UidFieldMapper.NAME), notNullValue());
         assertThat(doc.rootDoc().get(IdFieldMapper.NAME), nullValue());
-
-        try {
-            docMapper.parse("test", "type", null, XContentFactory.jsonBuilder()
-                    .startObject()
-                    .endObject()
-                    .bytes());
-            fail("expect missing id");
-        } catch (MapperParsingException e) {
-            assertTrue(e.getMessage().contains("No id found"));
-        }
     }
 
     public void testIncludeInObjectNotAllowed() throws Exception {
@@ -70,8 +53,8 @@ public class IdMappingTests extends ESSingleNodeTestCase {
         DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
         try {
-            docMapper.parse(SourceToParse.source(XContentFactory.jsonBuilder()
-                .startObject().field("_id", "1").endObject().bytes()).type("type"));
+            docMapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject().field("_id", "1").endObject().bytes()));
             fail("Expected failure to parse metadata field");
         } catch (MapperParsingException e) {
             assertTrue(e.getMessage(), e.getMessage().contains("Field [_id] is a metadata field and cannot be added inside a document"));
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/index/IndexTypeMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/index/IndexTypeMapperTests.java
index b0476ae7ada..1460672e489 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/index/IndexTypeMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/index/IndexTypeMapperTests.java
@@ -19,24 +19,36 @@
 
 package org.elasticsearch.index.mapper.index;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.DocumentMapperParser;
+import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.index.mapper.internal.IndexFieldMapper;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
 
+import java.io.IOException;
+import java.util.Collection;
+
 public class IndexTypeMapperTests extends ESSingleNodeTestCase {
 
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(InternalSettingsPlugin.class);
+    }
+
     public void testDefaultDisabledIndexMapper() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .endObject().endObject().string();
         DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
-        IndexFieldMapper indexMapper = docMapper.metadataMapper(IndexFieldMapper.class);
-        assertThat(indexMapper.enabled(), equalTo(false));
 
         ParsedDocument doc = docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
                 .startObject()
@@ -47,4 +59,24 @@ public class IndexTypeMapperTests extends ESSingleNodeTestCase {
         assertThat(doc.rootDoc().get("_index"), nullValue());
         assertThat(doc.rootDoc().get("field"), equalTo("value"));
     }
+
+    public void testIndexNotConfigurable() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("_index").endObject()
+                .endObject().endObject().string();
+        DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
+        MapperParsingException e = expectThrows(MapperParsingException.class,
+                () -> parser.parse("type", new CompressedXContent(mapping)));
+        assertEquals("_index is not configurable", e.getMessage());
+    }
+
+    public void testBwCompatIndexNotConfigurable() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("_index").endObject()
+                .endObject().endObject().string();
+        DocumentMapperParser parser = createIndex("test",
+                Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0).build())
+                .mapperService().documentMapperParser();
+        parser.parse("type", new CompressedXContent(mapping)); // no exception
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/parent/ParentMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/parent/ParentMappingTests.java
index 41dd8d957b0..abc25cc9371 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/parent/ParentMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/parent/ParentMappingTests.java
@@ -24,7 +24,6 @@ import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceToParse;
-import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
 public class ParentMappingTests extends ESSingleNodeTestCase {
@@ -35,8 +34,8 @@ public class ParentMappingTests extends ESSingleNodeTestCase {
         DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
         try {
-            docMapper.parse(SourceToParse.source(XContentFactory.jsonBuilder()
-                .startObject().field("_parent", "1122").endObject().bytes()).type("type").id("1"));
+            docMapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
+                .startObject().field("_parent", "1122").endObject().bytes()));
             fail("Expected failure to parse metadata field");
         } catch (MapperParsingException e) {
             assertTrue(e.getMessage(), e.getMessage().contains("Field [_parent] is a metadata field and cannot be added inside a document"));
@@ -49,11 +48,11 @@ public class ParentMappingTests extends ESSingleNodeTestCase {
                 .endObject().endObject().string();
         DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
-        ParsedDocument doc = docMapper.parse(SourceToParse.source(XContentFactory.jsonBuilder()
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
                 .startObject()
                 .field("x_field", "x_value")
                 .endObject()
-                .bytes()).type("type").id("1").parent("1122"));
+                .bytes()).parent("1122"));
 
         assertEquals("1122", doc.rootDoc().getBinaryValue("_parent#p_type").utf8ToString());
     }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/routing/RoutingTypeMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/routing/RoutingTypeMapperTests.java
index a658948f022..0440019310b 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/routing/RoutingTypeMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/routing/RoutingTypeMapperTests.java
@@ -36,11 +36,11 @@ public class RoutingTypeMapperTests extends ESSingleNodeTestCase {
                 .endObject().endObject().string();
         DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
-        ParsedDocument doc = docMapper.parse(SourceToParse.source(XContentFactory.jsonBuilder()
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
             .startObject()
             .field("field", "value")
             .endObject()
-            .bytes()).type("type").id("1").routing("routing_value"));
+            .bytes()).routing("routing_value"));
 
         assertThat(doc.rootDoc().get("_routing"), equalTo("routing_value"));
         assertThat(doc.rootDoc().get("field"), equalTo("value"));
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
index b562a9f19b4..a0794325bdf 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
@@ -76,7 +76,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                 .field("field", "value")
                 .endObject()
                 .bytes();
-        ParsedDocument doc = docMapper.parse(SourceToParse.source(source).type("type").id("1").timestamp(1));
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source).timestamp(1));
 
         assertThat(doc.rootDoc().getField("_timestamp"), equalTo(null));
     }
@@ -91,7 +91,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                 .field("field", "value")
                 .endObject()
                 .bytes();
-        ParsedDocument doc = docMapper.parse(SourceToParse.source(source).type("type").id("1").timestamp(1));
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source).timestamp(1));
 
         assertThat(doc.rootDoc().getField("_timestamp").fieldType().stored(), equalTo(true));
         assertNotSame(IndexOptions.NONE, doc.rootDoc().getField("_timestamp").fieldType().indexOptions());
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java
index 6776ed3a1cc..2d28b699944 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.index.mapper.ttl;
 
 import org.apache.lucene.index.IndexOptions;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
@@ -52,7 +51,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
                 .field("field", "value")
                 .endObject()
                 .bytes();
-        ParsedDocument doc = docMapper.parse(SourceToParse.source(source).type("type").id("1").ttl(Long.MAX_VALUE));
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source).ttl(Long.MAX_VALUE));
 
         assertThat(doc.rootDoc().getField("_ttl"), equalTo(null));
     }
@@ -67,7 +66,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
                 .field("field", "value")
                 .endObject()
                 .bytes();
-        ParsedDocument doc = docMapper.parse(SourceToParse.source(source).type("type").id("1").ttl(Long.MAX_VALUE));
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source).ttl(Long.MAX_VALUE));
 
         assertThat(doc.rootDoc().getField("_ttl").fieldType().stored(), equalTo(true));
         assertNotSame(IndexOptions.NONE, doc.rootDoc().getField("_ttl").fieldType().indexOptions());
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index f21973d19db..39aaf072954 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -674,7 +674,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         Field versionField = new NumericDocValuesField("_version", 0);
         document.add(uidField);
         document.add(versionField);
-        return new ParsedDocument(uidField, versionField, id, type, routing, timestamp, ttl, Arrays.asList(document), source, mappingUpdate);
+        return new ParsedDocument(versionField, id, type, routing, timestamp, ttl, Arrays.asList(document), source, mappingUpdate);
     }
 
     public void testIndexingOperationsListeners() throws IOException {
diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java
index 7d9ad0793a8..9c07c7b3eb3 100644
--- a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java
+++ b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java
@@ -156,10 +156,7 @@ public class SizeFieldMapper extends MetadataFieldMapper {
         if (!enabledState.enabled) {
             return;
         }
-        if (context.source() == null) {
-            return;
-        }
-        final int value = context.source().length();
+        final int value = context.sourceToParse().source().length();
         if (Version.indexCreated(context.indexSettings()).before(Version.V_5_0_0_alpha2)) {
             fields.add(new LegacyIntegerFieldMapper.CustomIntegerNumericField(value, fieldType()));
         } else {
diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java
index a55ab2e50bb..352d84e975a 100644
--- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java
+++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java
@@ -29,14 +29,11 @@ import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.indices.IndicesModule;
-import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.junit.Before;
 
 
-import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
 import org.apache.lucene.index.IndexableField;
@@ -67,7 +64,7 @@ public class SizeMappingTests extends ESSingleNodeTestCase {
                 .field("field", "value")
                 .endObject()
                 .bytes();
-        ParsedDocument doc = docMapper.parse(SourceToParse.source(source).type("type").id("1"));
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source));
 
         boolean stored = false;
         boolean points = false;
@@ -90,7 +87,7 @@ public class SizeMappingTests extends ESSingleNodeTestCase {
                 .field("field", "value")
                 .endObject()
                 .bytes();
-        ParsedDocument doc = docMapper.parse(SourceToParse.source(source).type("type").id("1"));
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source));
 
         assertThat(doc.rootDoc().getField("_size"), nullValue());
     }
@@ -105,7 +102,7 @@ public class SizeMappingTests extends ESSingleNodeTestCase {
                 .field("field", "value")
                 .endObject()
                 .bytes();
-        ParsedDocument doc = docMapper.parse(SourceToParse.source(source).type("type").id("1"));
+        ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source));
 
         assertThat(doc.rootDoc().getField("_size"), nullValue());
     }

From 68e7ac4166322f7cb1b7aa5ec67976c11f71aa93 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Mon, 9 May 2016 18:09:09 +0200
Subject: [PATCH 0145/1311] Remove old backward compatibility layer for version
 lookups. #18215

The current code tries to handle the case that document versions are either
missing or stored in payloads rather than doc values. Since none of the 2.x
releases allowed this, we can remove this logic.
---
 .../uid/PerThreadIDAndVersionLookup.java      |  51 ++----
 .../common/lucene/uid/Versions.java           |   9 +-
 .../org/elasticsearch/index/VersionType.java  |  20 +--
 .../index/shard/ElasticsearchMergePolicy.java |  61 +------
 .../common/lucene/uid/VersionsTests.java      | 154 ------------------
 .../elasticsearch/index/VersionTypeTests.java |  12 --
 6 files changed, 17 insertions(+), 290 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDAndVersionLookup.java b/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDAndVersionLookup.java
index 60b8e2086de..3a87fbb523b 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDAndVersionLookup.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDAndVersionLookup.java
@@ -29,7 +29,6 @@ import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.Numbers;
 import org.elasticsearch.common.lucene.uid.Versions.DocIdAndVersion;
 import org.elasticsearch.index.mapper.internal.UidFieldMapper;
 import org.elasticsearch.index.mapper.internal.VersionFieldMapper;
@@ -52,12 +51,8 @@ final class PerThreadIDAndVersionLookup {
     private final TermsEnum termsEnum;
     /** _version data */
     private final NumericDocValues versions;
-    /** Only true when versions are indexed as payloads instead of docvalues */
-    private final boolean hasPayloads;
     /** Reused for iteration (when the term exists) */
     private PostingsEnum docsEnum;
-    /** Only used for back compat, to lookup a version from payload */
-    private PostingsEnum posEnum;
 
     /**
      * Initialize lookup for the provided segment
@@ -65,63 +60,37 @@ final class PerThreadIDAndVersionLookup {
     public PerThreadIDAndVersionLookup(LeafReader reader) throws IOException {
         TermsEnum termsEnum = null;
         NumericDocValues versions = null;
-        boolean hasPayloads = false;
 
         Fields fields = reader.fields();
         if (fields != null) {
             Terms terms = fields.terms(UidFieldMapper.NAME);
             if (terms != null) {
-                hasPayloads = terms.hasPayloads();
                 termsEnum = terms.iterator();
                 assert termsEnum != null;
                 versions = reader.getNumericDocValues(VersionFieldMapper.NAME);
+                assert versions != null;
             }
         }
 
         this.versions = versions;
         this.termsEnum = termsEnum;
-        this.hasPayloads = hasPayloads;
     }
 
     /** Return null if id is not found. */
     public DocIdAndVersion lookup(BytesRef id, Bits liveDocs, LeafReaderContext context) throws IOException {
         if (termsEnum.seekExact(id)) {
-            if (versions != null || hasPayloads == false) {
-                // Use NDV to retrieve the version, in which case we only need PostingsEnum:
-
-                // there may be more than one matching docID, in the case of nested docs, so we want the last one:
-                docsEnum = termsEnum.postings(docsEnum, 0);
-                int docID = DocIdSetIterator.NO_MORE_DOCS;
-                for (int d = docsEnum.nextDoc(); d != DocIdSetIterator.NO_MORE_DOCS; d = docsEnum.nextDoc()) {
-                    if (liveDocs != null && liveDocs.get(d) == false) {
-                        continue;
-                    }
-                    docID = d;
-                }
-
-                if (docID != DocIdSetIterator.NO_MORE_DOCS) {
-                    if (versions != null) {
-                        return new DocIdAndVersion(docID, versions.get(docID), context);
-                    } else {
-                        // _uid found, but no doc values and no payloads
-                        return new DocIdAndVersion(docID, Versions.NOT_SET, context);
-                    }
-                }
-            }
-
-            // ... but used to be stored as payloads; in this case we must use PostingsEnum
-            posEnum = termsEnum.postings(posEnum, PostingsEnum.PAYLOADS);
-            assert posEnum != null; // terms has payloads
-            for (int d = posEnum.nextDoc(); d != DocIdSetIterator.NO_MORE_DOCS; d = posEnum.nextDoc()) {
+            // there may be more than one matching docID, in the case of nested docs, so we want the last one:
+            docsEnum = termsEnum.postings(docsEnum, 0);
+            int docID = DocIdSetIterator.NO_MORE_DOCS;
+            for (int d = docsEnum.nextDoc(); d != DocIdSetIterator.NO_MORE_DOCS; d = docsEnum.nextDoc()) {
                 if (liveDocs != null && liveDocs.get(d) == false) {
                     continue;
                 }
-                posEnum.nextPosition();
-                final BytesRef payload = posEnum.getPayload();
-                if (payload != null && payload.length == 8) {
-                    // TODO: does this break the nested docs case?  we are not returning the last matching docID here?
-                    return new DocIdAndVersion(d, Numbers.bytesToLong(payload), context);
-                }
+                docID = d;
+            }
+
+            if (docID != DocIdSetIterator.NO_MORE_DOCS) {
+                return new DocIdAndVersion(docID, versions.get(docID), context);
             }
         }
 
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/uid/Versions.java b/core/src/main/java/org/elasticsearch/common/lucene/uid/Versions.java
index a0cf923c5f1..3da8d800d8a 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/uid/Versions.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/uid/Versions.java
@@ -41,11 +41,7 @@ public class Versions {
     /** indicates that the current document was not found in lucene and in the version map */
     public static final long NOT_FOUND = -1L;
 
-    /**
-     * used when the document is old and doesn't contain any version information in the index
-     * see {@link PerThreadIDAndVersionLookup#lookup}
-     */
-    public static final long NOT_SET = -2L;
+    // -2 was used for docs that can be found in the index but do not have a version
 
     /**
      * used to indicate that the write operation should be executed if the document is currently deleted
@@ -114,7 +110,7 @@ public class Versions {
     /**
      * Load the internal doc ID and version for the uid from the reader, returning<ul>
      * <li>null if the uid wasn't found,
-     * <li>a doc ID and a version otherwise, the version being potentially set to {@link #NOT_SET} if the uid has no associated version
+     * <li>a doc ID and a version otherwise
      * </ul>
      */
     public static DocIdAndVersion loadDocIdAndVersion(IndexReader reader, Term term) throws IOException {
@@ -140,7 +136,6 @@ public class Versions {
     /**
      * Load the version for the uid from the reader, returning<ul>
      * <li>{@link #NOT_FOUND} if no matching doc exists,
-     * <li>{@link #NOT_SET} if no version is available,
      * <li>the version associated with the provided uid otherwise
      * </ul>
      */
diff --git a/core/src/main/java/org/elasticsearch/index/VersionType.java b/core/src/main/java/org/elasticsearch/index/VersionType.java
index 1c88a703ba6..3d0448d16a8 100644
--- a/core/src/main/java/org/elasticsearch/index/VersionType.java
+++ b/core/src/main/java/org/elasticsearch/index/VersionType.java
@@ -54,9 +54,6 @@ public enum VersionType implements Writeable {
         }
 
         private boolean isVersionConflict(long currentVersion, long expectedVersion, boolean deleted) {
-            if (currentVersion == Versions.NOT_SET) {
-                return false;
-            }
             if (expectedVersion == Versions.MATCH_ANY) {
                 return false;
             }
@@ -71,7 +68,7 @@ public enum VersionType implements Writeable {
 
         @Override
         public long updateVersion(long currentVersion, long expectedVersion) {
-            return (currentVersion == Versions.NOT_SET || currentVersion == Versions.NOT_FOUND) ? 1 : currentVersion + 1;
+            return currentVersion == Versions.NOT_FOUND ? 1 : currentVersion + 1;
         }
 
         @Override
@@ -95,9 +92,6 @@ public enum VersionType implements Writeable {
     EXTERNAL((byte) 1) {
         @Override
         public boolean isVersionConflictForWrites(long currentVersion, long expectedVersion, boolean deleted) {
-            if (currentVersion == Versions.NOT_SET) {
-                return false;
-            }
             if (currentVersion == Versions.NOT_FOUND) {
                 return false;
             }
@@ -117,9 +111,6 @@ public enum VersionType implements Writeable {
 
         @Override
         public boolean isVersionConflictForReads(long currentVersion, long expectedVersion) {
-            if (currentVersion == Versions.NOT_SET) {
-                return false;
-            }
             if (expectedVersion == Versions.MATCH_ANY) {
                 return false;
             }
@@ -156,9 +147,6 @@ public enum VersionType implements Writeable {
     EXTERNAL_GTE((byte) 2) {
         @Override
         public boolean isVersionConflictForWrites(long currentVersion, long expectedVersion, boolean deleted) {
-            if (currentVersion == Versions.NOT_SET) {
-                return false;
-            }
             if (currentVersion == Versions.NOT_FOUND) {
                 return false;
             }
@@ -178,9 +166,6 @@ public enum VersionType implements Writeable {
 
         @Override
         public boolean isVersionConflictForReads(long currentVersion, long expectedVersion) {
-            if (currentVersion == Versions.NOT_SET) {
-                return false;
-            }
             if (expectedVersion == Versions.MATCH_ANY) {
                 return false;
             }
@@ -220,9 +205,6 @@ public enum VersionType implements Writeable {
     FORCE((byte) 3) {
         @Override
         public boolean isVersionConflictForWrites(long currentVersion, long expectedVersion, boolean deleted) {
-            if (currentVersion == Versions.NOT_SET) {
-                return false;
-            }
             if (currentVersion == Versions.NOT_FOUND) {
                 return false;
             }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/ElasticsearchMergePolicy.java b/core/src/main/java/org/elasticsearch/index/shard/ElasticsearchMergePolicy.java
index adae6caf452..3cc4ea11526 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/ElasticsearchMergePolicy.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/ElasticsearchMergePolicy.java
@@ -19,21 +19,17 @@
 
 package org.elasticsearch.index.shard;
 
-import org.apache.lucene.index.CodecReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.MergePolicy;
 import org.apache.lucene.index.MergeTrigger;
 import org.apache.lucene.index.SegmentCommitInfo;
 import org.apache.lucene.index.SegmentInfos;
-import org.apache.lucene.store.Directory;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.List;
 import java.util.Map;
 
 /**
@@ -67,59 +63,10 @@ public final class ElasticsearchMergePolicy extends MergePolicy {
         this.delegate = delegate;
     }
 
-    /** Return an "upgraded" view of the reader. */
-    static CodecReader filter(CodecReader reader) throws IOException {
-        // TODO: remove 0.90.x/1.x freqs/prox/payloads from _uid?
-        // the previous code never did this, so some indexes carry around trash.
-        return reader;
-    }
-
-    static class IndexUpgraderOneMerge extends OneMerge {
-
-        public IndexUpgraderOneMerge(List<SegmentCommitInfo> segments) {
-            super(segments);
-        }
-
-        @Override
-        public List<CodecReader> getMergeReaders() throws IOException {
-            final List<CodecReader> newReaders = new ArrayList<>();
-            for (CodecReader reader : super.getMergeReaders()) {
-                newReaders.add(filter(reader));
-            }
-            return newReaders;
-        }
-
-    }
-
-    static class IndexUpgraderMergeSpecification extends MergeSpecification {
-
-        @Override
-        public void add(OneMerge merge) {
-            super.add(new IndexUpgraderOneMerge(merge.segments));
-        }
-
-        @Override
-        public String segString(Directory dir) {
-            return "IndexUpgraderMergeSpec[" + super.segString(dir) + "]";
-        }
-
-    }
-
-    static MergeSpecification upgradedMergeSpecification(MergeSpecification spec) {
-        if (spec == null) {
-            return null;
-        }
-        MergeSpecification upgradedSpec = new IndexUpgraderMergeSpecification();
-        for (OneMerge merge : spec.merges) {
-            upgradedSpec.add(merge);
-        }
-        return upgradedSpec;
-    }
-
     @Override
     public MergeSpecification findMerges(MergeTrigger mergeTrigger,
         SegmentInfos segmentInfos, IndexWriter writer) throws IOException {
-        return upgradedMergeSpecification(delegate.findMerges(mergeTrigger, segmentInfos, writer));
+        return delegate.findMerges(mergeTrigger, segmentInfos, writer);
     }
 
     private boolean shouldUpgrade(SegmentCommitInfo info) {
@@ -148,7 +95,7 @@ public final class ElasticsearchMergePolicy extends MergePolicy {
         throws IOException {
 
         if (upgradeInProgress) {
-            MergeSpecification spec = new IndexUpgraderMergeSpecification();
+            MergeSpecification spec = new MergeSpecification();
             for (SegmentCommitInfo info : segmentInfos) {
 
                 if (shouldUpgrade(info)) {
@@ -183,13 +130,13 @@ public final class ElasticsearchMergePolicy extends MergePolicy {
             // has a chance to decide what to do (e.g. collapse the segments to satisfy maxSegmentCount)
         }
 
-        return upgradedMergeSpecification(delegate.findForcedMerges(segmentInfos, maxSegmentCount, segmentsToMerge, writer));
+        return delegate.findForcedMerges(segmentInfos, maxSegmentCount, segmentsToMerge, writer);
     }
 
     @Override
     public MergeSpecification findForcedDeletesMerges(SegmentInfos segmentInfos, IndexWriter writer)
         throws IOException {
-        return upgradedMergeSpecification(delegate.findForcedDeletesMerges(segmentInfos, writer));
+        return delegate.findForcedDeletesMerges(segmentInfos, writer);
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java b/core/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java
index 7f405ea0531..b59b61b27f3 100644
--- a/core/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java
@@ -18,45 +18,27 @@
  */
 package org.elasticsearch.common.lucene.uid;
 
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.SlowCompositeReaderWrapper;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.Numbers;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.index.mapper.internal.UidFieldMapper;
 import org.elasticsearch.index.mapper.internal.VersionFieldMapper;
-import org.elasticsearch.index.shard.ElasticsearchMergePolicy;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.MatcherAssert;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
 public class VersionsTests extends ESTestCase {
@@ -83,13 +65,6 @@ public class VersionsTests extends ESTestCase {
 
         Document doc = new Document();
         doc.add(new Field(UidFieldMapper.NAME, "1", UidFieldMapper.Defaults.FIELD_TYPE));
-        writer.addDocument(doc);
-        directoryReader = reopen(directoryReader);
-        assertThat(Versions.loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), equalTo(Versions.NOT_SET));
-        assertThat(Versions.loadDocIdAndVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")).version, equalTo(Versions.NOT_SET));
-
-        doc = new Document();
-        doc.add(new Field(UidFieldMapper.NAME, "1", UidFieldMapper.Defaults.FIELD_TYPE));
         doc.add(new NumericDocValuesField(VersionFieldMapper.NAME, 1));
         writer.updateDocument(new Term(UidFieldMapper.NAME, "1"), doc);
         directoryReader = reopen(directoryReader);
@@ -166,135 +141,6 @@ public class VersionsTests extends ESTestCase {
         dir.close();
     }
 
-    public void testBackwardCompatibility() throws IOException {
-        Directory dir = newDirectory();
-        IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
-
-        DirectoryReader directoryReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "_na_", 1));
-        MatcherAssert.assertThat(Versions.loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), equalTo(Versions.NOT_FOUND));
-
-        Document doc = new Document();
-        UidField uidAndVersion = new UidField("1", 1L);
-        doc.add(uidAndVersion);
-        writer.addDocument(doc);
-
-        uidAndVersion.uid = "2";
-        uidAndVersion.version = 2;
-        writer.addDocument(doc);
-        writer.commit();
-
-        directoryReader = reopen(directoryReader);
-        assertThat(Versions.loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "1")), equalTo(1L));
-        assertThat(Versions.loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "2")), equalTo(2L));
-        assertThat(Versions.loadVersion(directoryReader, new Term(UidFieldMapper.NAME, "3")), equalTo(Versions.NOT_FOUND));
-        directoryReader.close();
-        writer.close();
-        dir.close();
-    }
-
-    // This is how versions used to be encoded
-    private static class UidField extends Field {
-        private static final FieldType FIELD_TYPE = new FieldType();
-        static {
-            FIELD_TYPE.setTokenized(true);
-            FIELD_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
-            FIELD_TYPE.setStored(true);
-            FIELD_TYPE.freeze();
-        }
-        String uid;
-        long version;
-        UidField(String uid, long version) {
-            super(UidFieldMapper.NAME, uid, FIELD_TYPE);
-            this.uid = uid;
-            this.version = version;
-        }
-        @Override
-        public TokenStream tokenStream(Analyzer analyzer, TokenStream reuse) {
-            return new TokenStream() {
-                boolean finished = true;
-                final CharTermAttribute term = addAttribute(CharTermAttribute.class);
-                final PayloadAttribute payload = addAttribute(PayloadAttribute.class);
-                @Override
-                public boolean incrementToken() throws IOException {
-                    if (finished) {
-                        return false;
-                    }
-                    term.setEmpty().append(uid);
-                    payload.setPayload(new BytesRef(Numbers.longToBytes(version)));
-                    finished = true;
-                    return true;
-                }
-                @Override
-                public void reset() throws IOException {
-                    finished = false;
-                }
-            };
-        }
-    }
-
-    public void testMergingOldIndices() throws Exception {
-        final IndexWriterConfig iwConf = new IndexWriterConfig(new KeywordAnalyzer());
-        iwConf.setMergePolicy(new ElasticsearchMergePolicy(iwConf.getMergePolicy()));
-        final Directory dir = newDirectory();
-        final IndexWriter iw = new IndexWriter(dir, iwConf);
-
-        // 1st segment, no _version
-        Document document = new Document();
-        // Add a dummy field (enough to trigger #3237)
-        document.add(new StringField("a", "b", Store.NO));
-        StringField uid = new StringField(UidFieldMapper.NAME, "1", Store.YES);
-        document.add(uid);
-        iw.addDocument(document);
-        uid.setStringValue("2");
-        iw.addDocument(document);
-        iw.commit();
-
-        // 2nd segment, old layout
-        document = new Document();
-        UidField uidAndVersion = new UidField("3", 3L);
-        document.add(uidAndVersion);
-        iw.addDocument(document);
-        uidAndVersion.uid = "4";
-        uidAndVersion.version = 4L;
-        iw.addDocument(document);
-        iw.commit();
-
-        // 3rd segment new layout
-        document = new Document();
-        uid.setStringValue("5");
-        Field version = new NumericDocValuesField(VersionFieldMapper.NAME, 5L);
-        document.add(uid);
-        document.add(version);
-        iw.addDocument(document);
-        uid.setStringValue("6");
-        version.setLongValue(6L);
-        iw.addDocument(document);
-        iw.commit();
-
-        Map<String, Long> expectedVersions = new HashMap<>();
-        expectedVersions.put("1", 0L);
-        expectedVersions.put("2", 0L);
-        expectedVersions.put("3", 0L);
-        expectedVersions.put("4", 4L);
-        expectedVersions.put("5", 5L);
-        expectedVersions.put("6", 6L);
-
-        // Force merge and check versions
-        iw.forceMerge(1, true);
-        final LeafReader ir = SlowCompositeReaderWrapper.wrap(ElasticsearchDirectoryReader.wrap(DirectoryReader.open(iw.getDirectory()), new ShardId("foo", "_na_", 1)));
-        final NumericDocValues versions = ir.getNumericDocValues(VersionFieldMapper.NAME);
-        assertThat(versions, notNullValue());
-        for (int i = 0; i < ir.maxDoc(); ++i) {
-            final String uidValue = ir.document(i).get(UidFieldMapper.NAME);
-            final long expectedVersion = expectedVersions.get(uidValue);
-            assertThat(versions.get(i), equalTo(expectedVersion));
-        }
-
-        iw.close();
-        ir.close();
-        dir.close();
-    }
-
     /** Test that version map cache works, is evicted on close, etc */
     public void testCache() throws Exception {
         int size = Versions.lookupStates.size();
diff --git a/core/src/test/java/org/elasticsearch/index/VersionTypeTests.java b/core/src/test/java/org/elasticsearch/index/VersionTypeTests.java
index 837d998dfec..4c79ce1b495 100644
--- a/core/src/test/java/org/elasticsearch/index/VersionTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/VersionTypeTests.java
@@ -28,11 +28,6 @@ public class VersionTypeTests extends ESTestCase {
     public void testInternalVersionConflict() throws Exception {
         assertFalse(VersionType.INTERNAL.isVersionConflictForWrites(10, Versions.MATCH_ANY, randomBoolean()));
         assertFalse(VersionType.INTERNAL.isVersionConflictForReads(10, Versions.MATCH_ANY));
-        // if we don't have a version in the index we accept everything
-        assertFalse(VersionType.INTERNAL.isVersionConflictForWrites(Versions.NOT_SET, 10, randomBoolean()));
-        assertFalse(VersionType.INTERNAL.isVersionConflictForReads(Versions.NOT_SET, 10));
-        assertFalse(VersionType.INTERNAL.isVersionConflictForWrites(Versions.NOT_SET, Versions.MATCH_ANY, randomBoolean()));
-        assertFalse(VersionType.INTERNAL.isVersionConflictForReads(Versions.NOT_SET, Versions.MATCH_ANY));
 
         // if we didn't find a version (but the index does support it), we don't like it unless MATCH_ANY
         assertTrue(VersionType.INTERNAL.isVersionConflictForWrites(Versions.NOT_FOUND, 10, randomBoolean()));
@@ -99,7 +94,6 @@ public class VersionTypeTests extends ESTestCase {
 
     public void testExternalVersionConflict() throws Exception {
         assertFalse(VersionType.EXTERNAL.isVersionConflictForWrites(Versions.NOT_FOUND, 10, randomBoolean()));
-        assertFalse(VersionType.EXTERNAL.isVersionConflictForWrites(Versions.NOT_SET, 10, randomBoolean()));
         // MATCH_ANY must throw an exception in the case of external version, as the version must be set! it used as the new value
         assertTrue(VersionType.EXTERNAL.isVersionConflictForWrites(10, Versions.MATCH_ANY, randomBoolean()));
 
@@ -135,7 +129,6 @@ public class VersionTypeTests extends ESTestCase {
 
     public void testExternalGTEVersionConflict() throws Exception {
         assertFalse(VersionType.EXTERNAL_GTE.isVersionConflictForWrites(Versions.NOT_FOUND, 10, randomBoolean()));
-        assertFalse(VersionType.EXTERNAL_GTE.isVersionConflictForWrites(Versions.NOT_SET, 10, randomBoolean()));
         // MATCH_ANY must throw an exception in the case of external version, as the version must be set! it used as the new value
         assertTrue(VersionType.EXTERNAL_GTE.isVersionConflictForWrites(10, Versions.MATCH_ANY, randomBoolean()));
 
@@ -162,7 +155,6 @@ public class VersionTypeTests extends ESTestCase {
 
     public void testForceVersionConflict() throws Exception {
         assertFalse(VersionType.FORCE.isVersionConflictForWrites(Versions.NOT_FOUND, 10, randomBoolean()));
-        assertFalse(VersionType.FORCE.isVersionConflictForWrites(Versions.NOT_SET, 10, randomBoolean()));
 
         // MATCH_ANY must throw an exception in the case of force version, as the version must be set! it used as the new value
         try {
@@ -192,22 +184,18 @@ public class VersionTypeTests extends ESTestCase {
     }
 
     public void testUpdateVersion() {
-        assertThat(VersionType.INTERNAL.updateVersion(Versions.NOT_SET, 10), equalTo(1L));
         assertThat(VersionType.INTERNAL.updateVersion(Versions.NOT_FOUND, 10), equalTo(1L));
         assertThat(VersionType.INTERNAL.updateVersion(1, 1), equalTo(2L));
         assertThat(VersionType.INTERNAL.updateVersion(2, Versions.MATCH_ANY), equalTo(3L));
 
 
-        assertThat(VersionType.EXTERNAL.updateVersion(Versions.NOT_SET, 10), equalTo(10L));
         assertThat(VersionType.EXTERNAL.updateVersion(Versions.NOT_FOUND, 10), equalTo(10L));
         assertThat(VersionType.EXTERNAL.updateVersion(1, 10), equalTo(10L));
 
-        assertThat(VersionType.EXTERNAL_GTE.updateVersion(Versions.NOT_SET, 10), equalTo(10L));
         assertThat(VersionType.EXTERNAL_GTE.updateVersion(Versions.NOT_FOUND, 10), equalTo(10L));
         assertThat(VersionType.EXTERNAL_GTE.updateVersion(1, 10), equalTo(10L));
         assertThat(VersionType.EXTERNAL_GTE.updateVersion(10, 10), equalTo(10L));
 
-        assertThat(VersionType.FORCE.updateVersion(Versions.NOT_SET, 10), equalTo(10L));
         assertThat(VersionType.FORCE.updateVersion(Versions.NOT_FOUND, 10), equalTo(10L));
         assertThat(VersionType.FORCE.updateVersion(11, 10), equalTo(10L));
 

From f481492af3a5c9fa4bc451a481903dd2a074aedf Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Mon, 9 May 2016 18:51:47 +0200
Subject: [PATCH 0146/1311] Remove FieldMapper.Builder.indexName. #18219

The ability to configure index names that are different from the full name was
removed in 2.0.
---
 .../java/org/elasticsearch/index/mapper/FieldMapper.java    | 6 ------
 .../elasticsearch/index/mapper/internal/AllFieldMapper.java | 1 -
 .../index/mapper/internal/FieldNamesFieldMapper.java        | 1 -
 .../elasticsearch/index/mapper/internal/IdFieldMapper.java  | 1 -
 .../index/mapper/internal/IndexFieldMapper.java             | 1 -
 .../index/mapper/internal/TypeFieldMapper.java              | 1 -
 .../elasticsearch/index/mapper/internal/UidFieldMapper.java | 1 -
 .../elasticsearch/mapper/attachments/AttachmentMapper.java  | 1 -
 8 files changed, 13 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
index 8077a732142..c9f5e416f8a 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
@@ -58,7 +58,6 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
         protected final MappedFieldType defaultFieldType;
         private final IndexOptions defaultOptions;
         protected boolean omitNormsSet = false;
-        protected String indexName;
         protected Boolean includeInAll;
         protected boolean indexOptionsSet = false;
         protected boolean docValuesSet = false;
@@ -169,11 +168,6 @@ public abstract class FieldMapper extends Mapper implements Cloneable {
             return builder;
         }
 
-        public T indexName(String indexName) {
-            this.indexName = indexName;
-            return builder;
-        }
-
         public T indexAnalyzer(NamedAnalyzer indexAnalyzer) {
             this.fieldType.setIndexAnalyzer(indexAnalyzer);
             return builder;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
index 4be6024f4ed..8a384e94e16 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
@@ -102,7 +102,6 @@ public class AllFieldMapper extends MetadataFieldMapper {
         public Builder(MappedFieldType existing) {
             super(Defaults.NAME, existing == null ? Defaults.FIELD_TYPE : existing, Defaults.FIELD_TYPE);
             builder = this;
-            indexName = Defaults.INDEX_NAME;
         }
 
         public Builder enabled(EnabledAttributeMapper enabled) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java
index a0e04b55a13..c2db784e058 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java
@@ -78,7 +78,6 @@ public class FieldNamesFieldMapper extends MetadataFieldMapper {
 
         public Builder(MappedFieldType existing) {
             super(Defaults.NAME, existing == null ? Defaults.FIELD_TYPE : existing, Defaults.FIELD_TYPE);
-            indexName = Defaults.NAME;
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
index 064bc5fb69d..ba7c377bde8 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
@@ -81,7 +81,6 @@ public class IdFieldMapper extends MetadataFieldMapper {
 
         public Builder(MappedFieldType existing) {
             super(Defaults.NAME, existing == null ? Defaults.FIELD_TYPE : existing, Defaults.FIELD_TYPE);
-            indexName = Defaults.NAME;
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
index a24597f7dff..3189aa9b582 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
@@ -73,7 +73,6 @@ public class IndexFieldMapper extends MetadataFieldMapper {
 
         public Builder(MappedFieldType existing) {
             super(Defaults.NAME, existing == null ? Defaults.FIELD_TYPE : existing, Defaults.FIELD_TYPE);
-            indexName = Defaults.NAME;
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
index 6e2dbbcc7d0..548e20771c5 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
@@ -77,7 +77,6 @@ public class TypeFieldMapper extends MetadataFieldMapper {
 
         public Builder(MappedFieldType existing) {
             super(Defaults.NAME, existing == null ? Defaults.FIELD_TYPE : existing, Defaults.FIELD_TYPE);
-            indexName = Defaults.NAME;
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
index fc19201154c..354de024b92 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
@@ -75,7 +75,6 @@ public class UidFieldMapper extends MetadataFieldMapper {
 
         public Builder(MappedFieldType existing) {
             super(Defaults.NAME, existing == null ? Defaults.FIELD_TYPE : existing, Defaults.FIELD_TYPE);
-            indexName = Defaults.NAME;
         }
 
         @Override
diff --git a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java
index 4878dc0f2cd..bb3018195fa 100644
--- a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java
+++ b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java
@@ -206,7 +206,6 @@ public class AttachmentMapper extends FieldMapper {
                 if (contentBuilder instanceof FieldMapper.Builder == false) {
                     throw new IllegalStateException("content field for attachment must be a field mapper");
                 }
-                ((FieldMapper.Builder<?, ?>)contentBuilder).indexName(name);
                 contentBuilder.name = name + "." + FieldNames.CONTENT;
                 contentMapper = (FieldMapper) contentBuilder.build(context);
                 context.path().add(name);

From b6556e275a2e6b1e11a911e8d2af27f56f5cc25d Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Mon, 9 May 2016 18:41:36 +0200
Subject: [PATCH 0147/1311] Remove code duplication in FieldsVisitor. #18218

It currently has twice the same method, once with a MapperService instance and
once with a DocumentMapper. This commits only keeps the former.
---
 .../index/fieldvisitor/FieldsVisitor.java     | 38 +------------------
 .../index/get/ShardGetService.java            | 21 +++-------
 .../lucene/StoredNumericValuesTests.java      |  7 +++-
 3 files changed, 13 insertions(+), 53 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java b/core/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java
index 9d7ed36bfd7..f40b54ebb69 100644
--- a/core/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java
+++ b/core/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java
@@ -23,8 +23,6 @@ import org.apache.lucene.index.StoredFieldVisitor;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.Uid;
@@ -84,19 +82,11 @@ public class FieldsVisitor extends StoredFieldVisitor {
     }
 
     public void postProcess(MapperService mapperService) {
-        if (uid != null) {
-            DocumentMapper documentMapper = mapperService.documentMapper(uid.type());
-            if (documentMapper != null) {
-                // we can derive the exact type for the mapping
-                postProcess(documentMapper);
-                return;
-            }
-        }
-        // can't derive exact mapping type
         for (Map.Entry<String, List<Object>> entry : fields().entrySet()) {
             MappedFieldType fieldType = mapperService.fullName(entry.getKey());
             if (fieldType == null) {
-                continue;
+                throw new IllegalStateException("Field [" + entry.getKey()
+                    + "] exists in the index but not in mappings");
             }
             List<Object> fieldValues = entry.getValue();
             for (int i = 0; i < fieldValues.size(); i++) {
@@ -105,30 +95,6 @@ public class FieldsVisitor extends StoredFieldVisitor {
         }
     }
 
-    public void postProcess(DocumentMapper documentMapper) {
-        for (Map.Entry<String, List<Object>> entry : fields().entrySet()) {
-            String indexName = entry.getKey();
-            FieldMapper fieldMapper = documentMapper.mappers().getMapper(indexName);
-            if (fieldMapper == null) {
-                // it's possible index name doesn't match field name (legacy feature)
-                for (FieldMapper mapper : documentMapper.mappers()) {
-                    if (mapper.fieldType().name().equals(indexName)) {
-                        fieldMapper = mapper;
-                        break;
-                    }
-                }
-                if (fieldMapper == null) {
-                    // no index name or full name found, so skip
-                    continue;
-                }
-            }
-            List<Object> fieldValues = entry.getValue();
-            for (int i = 0; i < fieldValues.size(); i++) {
-                fieldValues.set(i, fieldMapper.fieldType().valueForSearch(fieldValues.get(i)));
-            }
-        }
-    }
-
     @Override
     public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
         if (SourceFieldMapper.NAME.equals(fieldInfo.name)) {
diff --git a/core/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/core/src/main/java/org/elasticsearch/index/get/ShardGetService.java
index 709d86e2b22..10c2e636e9a 100644
--- a/core/src/main/java/org/elasticsearch/index/get/ShardGetService.java
+++ b/core/src/main/java/org/elasticsearch/index/get/ShardGetService.java
@@ -118,13 +118,8 @@ public final class ShardGetService extends AbstractIndexShardComponent {
         currentMetric.inc();
         try {
             long now = System.nanoTime();
-            DocumentMapper docMapper = mapperService.documentMapper(type);
-            if (docMapper == null) {
-                missingMetric.inc(System.nanoTime() - now);
-                return new GetResult(shardId.getIndexName(), type, id, -1, false, null, null);
-            }
             fetchSourceContext = normalizeFetchSourceContent(fetchSourceContext, fields);
-            GetResult getResult = innerGetLoadFromStoredFields(type, id, fields, fetchSourceContext, engineGetResult, docMapper, ignoreErrorsOnGeneratedFields);
+            GetResult getResult = innerGetLoadFromStoredFields(type, id, fields, fetchSourceContext, engineGetResult, mapperService, ignoreErrorsOnGeneratedFields);
             if (getResult.isExists()) {
                 existsMetric.inc(System.nanoTime() - now);
             } else {
@@ -185,16 +180,10 @@ public final class ShardGetService extends AbstractIndexShardComponent {
             }
         }
 
-        DocumentMapper docMapper = mapperService.documentMapper(type);
-        if (docMapper == null) {
-            get.release();
-            return new GetResult(shardId.getIndexName(), type, id, -1, false, null, null);
-        }
-
         try {
             // break between having loaded it from translog (so we only have _source), and having a document to load
             if (get.docIdAndVersion() != null) {
-                return innerGetLoadFromStoredFields(type, id, gFields, fetchSourceContext, get, docMapper, ignoreErrorsOnGeneratedFields);
+                return innerGetLoadFromStoredFields(type, id, gFields, fetchSourceContext, get, mapperService, ignoreErrorsOnGeneratedFields);
             } else {
                 Translog.Source source = get.source();
 
@@ -205,6 +194,7 @@ public final class ShardGetService extends AbstractIndexShardComponent {
                 Set<String> neededFields = new HashSet<>();
                 // add meta fields
                 neededFields.add(RoutingFieldMapper.NAME);
+                DocumentMapper docMapper = mapperService.documentMapper(type);
                 if (docMapper.parentFieldMapper().active()) {
                     neededFields.add(ParentFieldMapper.NAME);
                 }
@@ -326,7 +316,7 @@ public final class ShardGetService extends AbstractIndexShardComponent {
         }
     }
 
-    private GetResult innerGetLoadFromStoredFields(String type, String id, String[] gFields, FetchSourceContext fetchSourceContext, Engine.GetResult get, DocumentMapper docMapper, boolean ignoreErrorsOnGeneratedFields) {
+    private GetResult innerGetLoadFromStoredFields(String type, String id, String[] gFields, FetchSourceContext fetchSourceContext, Engine.GetResult get, MapperService mapperService, boolean ignoreErrorsOnGeneratedFields) {
         Map<String, GetField> fields = null;
         BytesReference source = null;
         Versions.DocIdAndVersion docIdAndVersion = get.docIdAndVersion();
@@ -340,7 +330,7 @@ public final class ShardGetService extends AbstractIndexShardComponent {
             source = fieldVisitor.source();
 
             if (!fieldVisitor.fields().isEmpty()) {
-                fieldVisitor.postProcess(docMapper);
+                fieldVisitor.postProcess(mapperService);
                 fields = new HashMap<>(fieldVisitor.fields().size());
                 for (Map.Entry<String, List<Object>> entry : fieldVisitor.fields().entrySet()) {
                     fields.put(entry.getKey(), new GetField(entry.getKey(), entry.getValue()));
@@ -348,6 +338,7 @@ public final class ShardGetService extends AbstractIndexShardComponent {
             }
         }
 
+        DocumentMapper docMapper = mapperService.documentMapper(type);
         if (docMapper.parentFieldMapper().active()) {
             String parentId = ParentFieldSubFetchPhase.getParentId(docMapper.parentFieldMapper(), docIdAndVersion.context.reader(), docIdAndVersion.docId);
             if (fields == null) {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/lucene/StoredNumericValuesTests.java b/core/src/test/java/org/elasticsearch/index/mapper/lucene/StoredNumericValuesTests.java
index a4d66a388a6..7b2bfc78a93 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/lucene/StoredNumericValuesTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/lucene/StoredNumericValuesTests.java
@@ -29,6 +29,8 @@ import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor;
 import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
@@ -61,7 +63,8 @@ public class StoredNumericValuesTests extends ESSingleNodeTestCase {
                     .endObject()
                 .endObject()
                 .string();
-        DocumentMapper mapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        MapperService mapperService = createIndex("test").mapperService();
+        DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, false);
 
         ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
                 .startObject()
@@ -86,7 +89,7 @@ public class StoredNumericValuesTests extends ESSingleNodeTestCase {
         CustomFieldsVisitor fieldsVisitor = new CustomFieldsVisitor(
                 Collections.emptySet(), Collections.singletonList("field*"), false);
         searcher.doc(0, fieldsVisitor);
-        fieldsVisitor.postProcess(mapper);
+        fieldsVisitor.postProcess(mapperService);
         assertThat(fieldsVisitor.fields().size(), equalTo(10));
         assertThat(fieldsVisitor.fields().get("field1").size(), equalTo(1));
         assertThat(fieldsVisitor.fields().get("field1").get(0), equalTo((byte) 1));

From 7bf8e6719ceb15625773c1de00bdfb8e3f1eeb6f Mon Sep 17 00:00:00 2001
From: jasonquick <jasonquick@users.noreply.github.com>
Date: Tue, 10 May 2016 03:39:42 -0400
Subject: [PATCH 0148/1311] Fix minor typo (#18222)

---
 docs/reference/search/search.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/search/search.asciidoc b/docs/reference/search/search.asciidoc
index 30f156050d9..40118e76931 100644
--- a/docs/reference/search/search.asciidoc
+++ b/docs/reference/search/search.asciidoc
@@ -1,7 +1,7 @@
 [[search-search]]
 == Search
 
-The search API allows to execute a search query and get back search hits
+The search API allows you to execute a search query and get back search hits
 that match the query. The query can either be provided using a simple
 <<search-uri-request,query string as a parameter>>, or using a
 <<search-request-body,request body>>.

From 17225d9ac1f4e13b07ba252f914cb71cc2d267a7 Mon Sep 17 00:00:00 2001
From: Colin Goodheart-Smithe <colings86@users.noreply.github.com>
Date: Tue, 10 May 2016 09:57:54 +0100
Subject: [PATCH 0149/1311] Removes the now obsolete SearchParseElement
 implementations

All implementations of SearchParseElement have been removed since they are no longer used now that parsing is done on the coordinating node. The SearchParseElement and FetchSubPhaseParseElement classes are not removed as currently they are needed for plugins that add a custom fetch sub phase. These will be removed in a follow up PR that will allow fetch sub phase plugins to register a parser in a different way.
---
 .../search/stats/StatsGroupsParseElement.java |  47 --------
 .../elasticsearch/search/SearchModule.java    |   9 +-
 .../org/elasticsearch/search/SearchPhase.java |   5 +-
 .../elasticsearch/search/SearchService.java   |   2 -
 .../AggregationBinaryParseElement.java        |  45 --------
 .../aggregations/AggregationParseElement.java |  72 ------------
 .../search/aggregations/AggregationPhase.java |  21 +---
 .../elasticsearch/search/dfs/DfsPhase.java    |   9 --
 .../search/fetch/FetchPhase.java              |   1 -
 .../search/fetch/FetchSubPhase.java           |   5 +-
 .../search/fetch/FieldsParseElement.java      |  50 --------
 .../fetch/explain/ExplainFetchSubPhase.java   |   9 --
 .../fetch/explain/ExplainParseElement.java    |  38 -------
 .../FieldDataFieldsFetchSubPhase.java         |  12 --
 .../FieldDataFieldsParseElement.java          |  58 ----------
 .../innerhits/InnerHitsFetchSubPhase.java     |   3 -
 .../script/ScriptFieldsFetchSubPhase.java     |  16 ---
 .../script/ScriptFieldsParseElement.java      | 107 ------------------
 .../fetch/source/FetchSourceParseElement.java |  97 ----------------
 .../fetch/source/FetchSourceSubPhase.java     |  10 --
 .../fetch/version/VersionFetchSubPhase.java   |  10 --
 .../fetch/version/VersionParseElement.java    |  37 ------
 .../search/query/FromParseElement.java        |  44 -------
 .../query/IndicesBoostParseElement.java       |  54 ---------
 .../search/query/MinScoreParseElement.java    |  38 -------
 .../search/query/PostFilterParseElement.java  |  40 -------
 .../search/query/QueryParseElement.java       |  37 ------
 .../search/query/QueryPhase.java              |  28 -----
 .../search/query/SizeParseElement.java        |  44 -------
 .../query/TerminateAfterParseElement.java     |  42 -------
 .../search/query/TimeoutParseElement.java     |  41 -------
 .../search/sort/TrackScoresParseElement.java  |  37 ------
 .../search/fetch/FetchSubPhasePluginIT.java   |   3 +-
 .../search/fetch/FieldDataFieldsTests.java    |  80 -------------
 34 files changed, 13 insertions(+), 1138 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/index/search/stats/StatsGroupsParseElement.java
 delete mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/AggregationBinaryParseElement.java
 delete mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/AggregationParseElement.java
 delete mode 100644 core/src/main/java/org/elasticsearch/search/fetch/FieldsParseElement.java
 delete mode 100644 core/src/main/java/org/elasticsearch/search/fetch/explain/ExplainParseElement.java
 delete mode 100644 core/src/main/java/org/elasticsearch/search/fetch/fielddata/FieldDataFieldsParseElement.java
 delete mode 100644 core/src/main/java/org/elasticsearch/search/fetch/script/ScriptFieldsParseElement.java
 delete mode 100644 core/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceParseElement.java
 delete mode 100644 core/src/main/java/org/elasticsearch/search/fetch/version/VersionParseElement.java
 delete mode 100644 core/src/main/java/org/elasticsearch/search/query/FromParseElement.java
 delete mode 100644 core/src/main/java/org/elasticsearch/search/query/IndicesBoostParseElement.java
 delete mode 100644 core/src/main/java/org/elasticsearch/search/query/MinScoreParseElement.java
 delete mode 100644 core/src/main/java/org/elasticsearch/search/query/PostFilterParseElement.java
 delete mode 100644 core/src/main/java/org/elasticsearch/search/query/QueryParseElement.java
 delete mode 100644 core/src/main/java/org/elasticsearch/search/query/SizeParseElement.java
 delete mode 100644 core/src/main/java/org/elasticsearch/search/query/TerminateAfterParseElement.java
 delete mode 100644 core/src/main/java/org/elasticsearch/search/query/TimeoutParseElement.java
 delete mode 100644 core/src/main/java/org/elasticsearch/search/sort/TrackScoresParseElement.java
 delete mode 100644 core/src/test/java/org/elasticsearch/search/fetch/FieldDataFieldsTests.java

diff --git a/core/src/main/java/org/elasticsearch/index/search/stats/StatsGroupsParseElement.java b/core/src/main/java/org/elasticsearch/index/search/stats/StatsGroupsParseElement.java
deleted file mode 100644
index f4dcebd9d54..00000000000
--- a/core/src/main/java/org/elasticsearch/index/search/stats/StatsGroupsParseElement.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.search.stats;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.SearchParseElement;
-import org.elasticsearch.search.internal.SearchContext;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-/**
- */
-public class StatsGroupsParseElement implements SearchParseElement {
-
-    @Override
-    public void parse(XContentParser parser, SearchContext context) throws Exception {
-        XContentParser.Token token = parser.currentToken();
-        if (token.isValue()) {
-            context.groupStats(Collections.singletonList(parser.text()));
-        } else if (token == XContentParser.Token.START_ARRAY) {
-            List<String> groupStats = new ArrayList<>(4);
-            while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                groupStats.add(parser.text());
-            }
-            context.groupStats(groupStats);
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java
index e0a6ccd09b4..6fe4913d074 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -95,8 +95,6 @@ import org.elasticsearch.index.query.functionscore.ScriptScoreFunctionBuilder;
 import org.elasticsearch.index.query.functionscore.WeightBuilder;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.action.SearchTransportService;
-import org.elasticsearch.search.aggregations.AggregationBinaryParseElement;
-import org.elasticsearch.search.aggregations.AggregationParseElement;
 import org.elasticsearch.search.aggregations.AggregationPhase;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorBuilder;
@@ -375,7 +373,7 @@ public class SearchModule extends AbstractModule {
 
     /**
      * Register a {@link SignificanceHeuristic}.
-     * 
+     *
      * @param heuristicName the name(s) at which the heuristic is parsed and streamed. The {@link ParseField#getPreferredName()} is the name
      *        under which it is streamed. All names work for the parser.
      * @param reader reads the heuristic from a stream
@@ -557,11 +555,8 @@ public class SearchModule extends AbstractModule {
         registerPipelineAggregation(SerialDiffPipelineAggregatorBuilder::new, SerialDiffPipelineAggregatorBuilder::parse,
                 SerialDiffPipelineAggregatorBuilder.AGGREGATION_NAME_FIELD);
 
-        AggregationParseElement aggParseElement = new AggregationParseElement(aggregatorParsers, queryParserRegistry);
-        AggregationBinaryParseElement aggBinaryParseElement = new AggregationBinaryParseElement(aggregatorParsers, queryParserRegistry);
-        AggregationPhase aggPhase = new AggregationPhase(aggParseElement, aggBinaryParseElement);
+        AggregationPhase aggPhase = new AggregationPhase();
         bind(AggregatorParsers.class).toInstance(aggregatorParsers);
-        bind(AggregationParseElement.class).toInstance(aggParseElement);
         bind(AggregationPhase.class).toInstance(aggPhase);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/SearchPhase.java b/core/src/main/java/org/elasticsearch/search/SearchPhase.java
index 85a75b9738d..48c041f12f5 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchPhase.java
@@ -21,6 +21,7 @@ package org.elasticsearch.search;
 
 import org.elasticsearch.search.internal.SearchContext;
 
+import java.util.Collections;
 import java.util.Map;
 
 /**
@@ -28,7 +29,9 @@ import java.util.Map;
  */
 public interface SearchPhase {
 
-    Map<String, ? extends SearchParseElement> parseElements();
+    default Map<String, ? extends SearchParseElement> parseElements() {
+        return Collections.emptyMap();
+    }
 
     /**
      * Performs pre processing of the search context before the execute.
diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java
index 636e4338063..4cf440f0a95 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchService.java
@@ -63,7 +63,6 @@ import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.InnerHitBuilder;
-import org.elasticsearch.index.search.stats.StatsGroupsParseElement;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.SearchOperationListener;
@@ -186,7 +185,6 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
         elementParsers.putAll(dfsPhase.parseElements());
         elementParsers.putAll(queryPhase.parseElements());
         elementParsers.putAll(fetchPhase.parseElements());
-        elementParsers.put("stats", new StatsGroupsParseElement());
         this.elementParsers = unmodifiableMap(elementParsers);
 
         this.keepAliveReaper = threadPool.scheduleWithFixedDelay(new Reaper(), keepAliveInterval);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBinaryParseElement.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBinaryParseElement.java
deleted file mode 100644
index d1fd5b137aa..00000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBinaryParseElement.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations;
-
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.search.internal.SearchContext;
-
-/**
- *
- */
-public class AggregationBinaryParseElement extends AggregationParseElement {
-
-    @Inject
-    public AggregationBinaryParseElement(AggregatorParsers aggregatorParsers, IndicesQueriesRegistry queriesRegistry) {
-        super(aggregatorParsers, queriesRegistry);
-    }
-
-    @Override
-    public void parse(XContentParser parser, SearchContext context) throws Exception {
-        byte[] aggSource = parser.binaryValue();
-        try (XContentParser aSourceParser = XContentFactory.xContent(aggSource).createParser(aggSource)) {
-            aSourceParser.nextToken(); // move past the first START_OBJECT
-            super.parse(aSourceParser, context);
-        }
-    }
-}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationParseElement.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationParseElement.java
deleted file mode 100644
index 2c94fc45ffc..00000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationParseElement.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations;
-
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.search.SearchParseElement;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
-import org.elasticsearch.search.internal.SearchContext;
-
-/**
- * The search parse element that is responsible for parsing the get part of the request.
- *
- * For example (in bold):
- * <pre>
- *      curl -XGET 'localhost:9200/_search?search_type=count' -d '{
- *          query: {
- *              match_all : {}
- *          },
- *          addAggregation : {
- *              avg_price: {
- *                  avg : { field : price }
- *              },
- *              categories: {
- *                  terms : { field : category, size : 12 },
- *                  addAggregation: {
- *                      avg_price : { avg : { field : price }}
- *                  }
- *              }
- *          }
- *      }'
- * </pre>
- */
-public class AggregationParseElement implements SearchParseElement {
-
-    private final AggregatorParsers aggregatorParsers;
-    private IndicesQueriesRegistry queriesRegistry;
-
-    @Inject
-    public AggregationParseElement(AggregatorParsers aggregatorParsers, IndicesQueriesRegistry queriesRegistry) {
-        this.aggregatorParsers = aggregatorParsers;
-        this.queriesRegistry = queriesRegistry;
-    }
-
-    @Override
-    public void parse(XContentParser parser, SearchContext context) throws Exception {
-        QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, context.parseFieldMatcher());
-        AggregatorFactories.Builder builders = aggregatorParsers.parseAggregators(parseContext);
-        AggregationContext aggContext = new AggregationContext(context);
-        AggregatorFactories factories = builders.build(aggContext, null);
-        factories.validate();
-        context.aggregations(new SearchContextAggregations(factories));
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
index 4c17a23c8fa..4953a39d4b5 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
@@ -24,7 +24,6 @@ import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.SearchPhase;
 import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -38,33 +37,15 @@ import org.elasticsearch.search.query.QueryPhaseExecutionException;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
-
-import static java.util.Collections.unmodifiableMap;
 
 /**
  *
  */
 public class AggregationPhase implements SearchPhase {
-    private final Map<String, SearchParseElement> parseElements;
 
     @Inject
-    public AggregationPhase(AggregationParseElement parseElement, AggregationBinaryParseElement binaryParseElement) {
-        Map<String, SearchParseElement> parseElements = new HashMap<>();
-        parseElements.put("aggregations", parseElement);
-        parseElements.put("aggs", parseElement);
-        parseElements.put("aggregations_binary", binaryParseElement);
-        parseElements.put("aggregationsBinary", binaryParseElement);
-        parseElements.put("aggs_binary", binaryParseElement);
-        parseElements.put("aggsBinary", binaryParseElement);
-        this.parseElements = unmodifiableMap(parseElements);
-    }
-
-    @Override
-    public Map<String, ? extends SearchParseElement> parseElements() {
-        return parseElements;
+    public AggregationPhase() {
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java b/core/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java
index fab8323d04d..de06655f414 100644
--- a/core/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/dfs/DfsPhase.java
@@ -28,7 +28,6 @@ import org.apache.lucene.index.TermContext;
 import org.apache.lucene.search.CollectionStatistics;
 import org.apache.lucene.search.TermStatistics;
 import org.elasticsearch.common.collect.HppcMaps;
-import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.SearchPhase;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.rescore.RescoreSearchContext;
@@ -36,20 +35,12 @@ import org.elasticsearch.search.rescore.RescoreSearchContext;
 import java.util.AbstractSet;
 import java.util.Collection;
 import java.util.Iterator;
-import java.util.Map;
-
-import static java.util.Collections.emptyMap;
 
 /**
  *
  */
 public class DfsPhase implements SearchPhase {
 
-    @Override
-    public Map<String, ? extends SearchParseElement> parseElements() {
-        return emptyMap();
-    }
-
     @Override
     public void preProcess(SearchContext context) {
     }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
index 5019bab9128..32326aa1579 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
@@ -83,7 +83,6 @@ public class FetchPhase implements SearchPhase {
     @Override
     public Map<String, ? extends SearchParseElement> parseElements() {
         Map<String, SearchParseElement> parseElements = new HashMap<>();
-        parseElements.put("fields", new FieldsParseElement());
         for (FetchSubPhase fetchSubPhase : fetchSubPhases) {
             parseElements.putAll(fetchSubPhase.parseElements());
         }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java
index b6a25d9db11..58d6ff36a2f 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java
@@ -27,6 +27,7 @@ import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.internal.InternalSearchHit;
 import org.elasticsearch.search.internal.SearchContext;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -101,7 +102,9 @@ public interface FetchSubPhase {
 
     }
 
-    Map<String, ? extends SearchParseElement> parseElements();
+    default Map<String, ? extends SearchParseElement> parseElements() {
+        return Collections.emptyMap();
+    }
 
     boolean hitExecutionNeeded(SearchContext context);
 
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FieldsParseElement.java b/core/src/main/java/org/elasticsearch/search/fetch/FieldsParseElement.java
deleted file mode 100644
index 74a37c7e8be..00000000000
--- a/core/src/main/java/org/elasticsearch/search/fetch/FieldsParseElement.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.fetch;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.SearchParseElement;
-import org.elasticsearch.search.internal.SearchContext;
-
-/**
- *
- */
-public class FieldsParseElement implements SearchParseElement {
-
-    @Override
-    public void parse(XContentParser parser, SearchContext context) throws Exception {
-        XContentParser.Token token = parser.currentToken();
-        if (token == XContentParser.Token.START_ARRAY) {
-            boolean added = false;
-            while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                String name = parser.text();
-                added = true;
-                context.fieldNames().add(name);
-            }
-            if (!added) {
-                context.emptyFieldNames();
-            }
-        } else if (token == XContentParser.Token.VALUE_STRING) {
-            String name = parser.text();
-            context.fieldNames().add(name);
-
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/explain/ExplainFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/explain/ExplainFetchSubPhase.java
index 42eecb58820..0d8b4ef3271 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/explain/ExplainFetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/explain/ExplainFetchSubPhase.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.search.fetch.explain;
 
 import org.apache.lucene.search.Explanation;
-import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.fetch.FetchPhaseExecutionException;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.internal.InternalSearchHit;
@@ -27,20 +26,12 @@ import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.rescore.RescoreSearchContext;
 
 import java.io.IOException;
-import java.util.Map;
-
-import static java.util.Collections.singletonMap;
 
 /**
  *
  */
 public class ExplainFetchSubPhase implements FetchSubPhase {
 
-    @Override
-    public Map<String, ? extends SearchParseElement> parseElements() {
-        return singletonMap("explain", new ExplainParseElement());
-    }
-
     @Override
     public boolean hitsExecutionNeeded(SearchContext context) {
         return false;
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/explain/ExplainParseElement.java b/core/src/main/java/org/elasticsearch/search/fetch/explain/ExplainParseElement.java
deleted file mode 100644
index 7d44a9d0f01..00000000000
--- a/core/src/main/java/org/elasticsearch/search/fetch/explain/ExplainParseElement.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.fetch.explain;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.SearchParseElement;
-import org.elasticsearch.search.internal.SearchContext;
-
-/**
- *
- */
-public class ExplainParseElement implements SearchParseElement {
-
-    @Override
-    public void parse(XContentParser parser, SearchContext context) throws Exception {
-        XContentParser.Token token = parser.currentToken();
-        if (token.isValue()) {
-            context.explain(parser.booleanValue());
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/fielddata/FieldDataFieldsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/fielddata/FieldDataFieldsFetchSubPhase.java
index 16b1311809c..f8034719a1d 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/fielddata/FieldDataFieldsFetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/fielddata/FieldDataFieldsFetchSubPhase.java
@@ -23,7 +23,6 @@ import org.elasticsearch.index.fielddata.AtomicFieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.search.SearchHitField;
-import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.internal.InternalSearchHit;
 import org.elasticsearch.search.internal.InternalSearchHitField;
@@ -31,9 +30,6 @@ import org.elasticsearch.search.internal.SearchContext;
 
 import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.Map;
-
-import static java.util.Collections.unmodifiableMap;
 
 /**
  * Query sub phase which pulls data from field data (using the cache if
@@ -61,14 +57,6 @@ public class FieldDataFieldsFetchSubPhase implements FetchSubPhase {
     public FieldDataFieldsFetchSubPhase() {
     }
 
-    @Override
-    public Map<String, ? extends SearchParseElement> parseElements() {
-        Map<String, SearchParseElement> parseElements = new HashMap<>();
-        parseElements.put("fielddata_fields", new FieldDataFieldsParseElement());
-        parseElements.put("fielddataFields", new FieldDataFieldsParseElement());
-        return unmodifiableMap(parseElements);
-    }
-
     @Override
     public boolean hitsExecutionNeeded(SearchContext context) {
         return false;
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/fielddata/FieldDataFieldsParseElement.java b/core/src/main/java/org/elasticsearch/search/fetch/fielddata/FieldDataFieldsParseElement.java
deleted file mode 100644
index 4f746c47ec6..00000000000
--- a/core/src/main/java/org/elasticsearch/search/fetch/fielddata/FieldDataFieldsParseElement.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.fetch.fielddata;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.fetch.FetchSubPhase;
-import org.elasticsearch.search.fetch.FetchSubPhaseParseElement;
-import org.elasticsearch.search.internal.SearchContext;
-
-/**
- * Parses field name values from the {@code fielddata_fields} parameter in a
- * search request.
- * <pre>
- * {
- *   "query": {...},
- *   "fielddata_fields" : ["field1", "field2"]
- * }
- * </pre>
- */
-public class FieldDataFieldsParseElement extends FetchSubPhaseParseElement<FieldDataFieldsContext> {
-
-    @Override
-    protected void innerParse(XContentParser parser, FieldDataFieldsContext fieldDataFieldsContext, SearchContext searchContext) throws Exception {
-        XContentParser.Token token = parser.currentToken();
-        if (token == XContentParser.Token.START_ARRAY) {
-            while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
-                String fieldName = parser.text();
-                fieldDataFieldsContext.add(new FieldDataFieldsContext.FieldDataField(fieldName));
-            }
-        } else if (token == XContentParser.Token.VALUE_STRING) {
-            String fieldName = parser.text();
-            fieldDataFieldsContext.add(new FieldDataFieldsContext.FieldDataField(fieldName));
-        } else {
-            throw new IllegalStateException("Expected either a VALUE_STRING or an START_ARRAY but got " + token);
-        }
-    }
-
-    @Override
-    protected FetchSubPhase.ContextFactory getContextFactory() {
-        return FieldDataFieldsFetchSubPhase.CONTEXT_FACTORY;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java
index 82c3755cdc9..993d1ccbb5b 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java
@@ -28,9 +28,6 @@ import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.fetch.FetchSubPhase;
-import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsParseElement;
-import org.elasticsearch.search.fetch.script.ScriptFieldsParseElement;
-import org.elasticsearch.search.fetch.source.FetchSourceParseElement;
 import org.elasticsearch.search.internal.InternalSearchHit;
 import org.elasticsearch.search.internal.InternalSearchHits;
 import org.elasticsearch.search.internal.SearchContext;
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/script/ScriptFieldsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/script/ScriptFieldsFetchSubPhase.java
index 8abcdfb54d6..540ce9aa66b 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/script/ScriptFieldsFetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/script/ScriptFieldsFetchSubPhase.java
@@ -20,7 +20,6 @@ package org.elasticsearch.search.fetch.script;
 
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.search.SearchHitField;
-import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.internal.InternalSearchHit;
 import org.elasticsearch.search.internal.InternalSearchHitField;
@@ -32,26 +31,11 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
-
-import static java.util.Collections.unmodifiableMap;
 
 /**
  *
  */
 public class ScriptFieldsFetchSubPhase implements FetchSubPhase {
-    private static final Map<String, SearchParseElement> PARSE_ELEMENTS;
-    static {
-        Map<String, SearchParseElement> parseElements = new HashMap<>();
-        parseElements.put("script_fields", new ScriptFieldsParseElement());
-        parseElements.put("scriptFields", new ScriptFieldsParseElement());
-        PARSE_ELEMENTS = unmodifiableMap(parseElements);
-    }
-
-    @Override
-    public Map<String, ? extends SearchParseElement> parseElements() {
-        return PARSE_ELEMENTS;
-    }
 
     @Override
     public boolean hitsExecutionNeeded(SearchContext context) {
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/script/ScriptFieldsParseElement.java b/core/src/main/java/org/elasticsearch/search/fetch/script/ScriptFieldsParseElement.java
deleted file mode 100644
index 408bf166e15..00000000000
--- a/core/src/main/java/org/elasticsearch/search/fetch/script/ScriptFieldsParseElement.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.fetch.script;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.script.Script;
-import org.elasticsearch.script.Script.ScriptField;
-import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.ScriptParameterParser;
-import org.elasticsearch.script.ScriptParameterParser.ScriptParameterValue;
-import org.elasticsearch.script.SearchScript;
-import org.elasticsearch.search.SearchParseElement;
-import org.elasticsearch.search.SearchParseException;
-import org.elasticsearch.search.internal.SearchContext;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * <pre>
- * "script_fields" : {
- *  "test1" : {
- *      "script" : "doc['field_name'].value"
- *  },
- *  "test2" : {
- *      "script" : "..."
- *  }
- * }
- * </pre>
- *
- *
- */
-public class ScriptFieldsParseElement implements SearchParseElement {
-
-    @Override
-    public void parse(XContentParser parser, SearchContext context) throws Exception {
-        XContentParser.Token token;
-        String currentFieldName = null;
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                currentFieldName = parser.currentName();
-            } else if (token == XContentParser.Token.START_OBJECT) {
-                String fieldName = currentFieldName;
-                ScriptParameterParser scriptParameterParser = new ScriptParameterParser();
-                Script script = null;
-                Map<String, Object> params = null;
-                boolean ignoreException = false;
-                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                    if (token == XContentParser.Token.FIELD_NAME) {
-                        currentFieldName = parser.currentName();
-                    } else if (token == XContentParser.Token.START_OBJECT) {
-                        if (context.parseFieldMatcher().match(currentFieldName, ScriptField.SCRIPT)) {
-                            script = Script.parse(parser, context.parseFieldMatcher());
-                        } else if ("params".equals(currentFieldName)) {
-                            params = parser.map();
-                        }
-                    } else if (token.isValue()) {
-                        if ("ignore_failure".equals(currentFieldName)) {
-                            ignoreException = parser.booleanValue();
-                        } else {
-                            scriptParameterParser.token(currentFieldName, token, parser, context.parseFieldMatcher());
-                        }
-                    }
-                }
-
-                if (script == null) { // Didn't find anything using the new API so try using the old one instead
-                    ScriptParameterValue scriptValue = scriptParameterParser.getDefaultScriptParameterValue();
-                    if (scriptValue != null) {
-                        if (params == null) {
-                            params = new HashMap<>();
-                        }
-                        script = new Script(scriptValue.script(), scriptValue.scriptType(), scriptParameterParser.lang(), params);
-                    }
-                } else if (params != null) {
-                    throw new SearchParseException(context, "script params must be specified inside script object",
-                            parser.getTokenLocation());
-                }
-
-                if (script == null) {
-                    throw new SearchParseException(context, "must specify a script in script fields", parser.getTokenLocation());
-                }
-
-                SearchScript searchScript = context.scriptService().search(context.lookup(), script, ScriptContext.Standard.SEARCH,
-                        Collections.emptyMap(), context.getQueryShardContext().getClusterState());
-                context.scriptFields().add(new ScriptFieldsContext.ScriptField(fieldName, searchScript, ignoreException));
-            }
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceParseElement.java b/core/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceParseElement.java
deleted file mode 100644
index 3eb212fb250..00000000000
--- a/core/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceParseElement.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.fetch.source;
-
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.SearchParseElement;
-import org.elasticsearch.search.internal.SearchContext;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * <pre>
- * "source" : true/false
- * "source" : "field"
- * "source" : [ "include", "include" ]
- * "source" : {
- *     "include" : ["obj"]
- *     "exclude" : ["obj"]
- * }
- * </pre>
- */
-public class FetchSourceParseElement implements SearchParseElement {
-
-    @Override
-    public void parse(XContentParser parser, SearchContext context) throws Exception {
-        context.fetchSourceContext(parse(parser));
-    }
-
-    public FetchSourceContext parse(XContentParser parser) throws IOException {
-        XContentParser.Token token;
-
-        List<String> includes = null, excludes = null;
-        String currentFieldName = null;
-        token = parser.currentToken(); // we get it on the value
-        if (parser.isBooleanValue()) {
-            return new FetchSourceContext(parser.booleanValue());
-        } else if (token == XContentParser.Token.VALUE_STRING) {
-            return new FetchSourceContext(new String[]{parser.text()});
-        } else if (token == XContentParser.Token.START_ARRAY) {
-            includes = new ArrayList<>();
-            while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                includes.add(parser.text());
-            }
-        } else if (token == XContentParser.Token.START_OBJECT) {
-
-            List<String> currentList = null;
-
-            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    currentFieldName = parser.currentName();
-                    if ("includes".equals(currentFieldName) || "include".equals(currentFieldName)) {
-                        currentList = includes != null ? includes : (includes = new ArrayList<>(2));
-                    } else if ("excludes".equals(currentFieldName) || "exclude".equals(currentFieldName)) {
-                        currentList = excludes != null ? excludes : (excludes = new ArrayList<>(2));
-                    } else {
-                        throw new ElasticsearchParseException("source definition may not contain [{}]", parser.text());
-                    }
-                } else if (token == XContentParser.Token.START_ARRAY) {
-                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                        currentList.add(parser.text());
-                    }
-                } else if (token.isValue()) {
-                    currentList.add(parser.text());
-                } else {
-                    throw new ElasticsearchParseException("unexpected token while parsing source settings");
-                }
-            }
-        } else {
-            throw new ElasticsearchParseException("source element value can be of type [{}]", token.name());
-        }
-
-        return new FetchSourceContext(
-                includes == null ? Strings.EMPTY_ARRAY : includes.toArray(new String[includes.size()]),
-                excludes == null ? Strings.EMPTY_ARRAY : excludes.toArray(new String[excludes.size()]));
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceSubPhase.java
index 1ed47383bd0..30220d12694 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceSubPhase.java
@@ -22,26 +22,16 @@ package org.elasticsearch.search.fetch.source;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.internal.InternalSearchHit;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.lookup.SourceLookup;
 
 import java.io.IOException;
-import java.util.Map;
-
-import static java.util.Collections.singletonMap;
 
 /**
  */
 public class FetchSourceSubPhase implements FetchSubPhase {
-    private static final Map<String, SearchParseElement> PARSE_ELEMENTS = singletonMap("_source", new FetchSourceParseElement());
-
-    @Override
-    public Map<String, ? extends SearchParseElement> parseElements() {
-        return PARSE_ELEMENTS;
-    }
 
     @Override
     public boolean hitsExecutionNeeded(SearchContext context) {
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/version/VersionFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/version/VersionFetchSubPhase.java
index ec36b7895c7..c5511352ec2 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/version/VersionFetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/version/VersionFetchSubPhase.java
@@ -24,26 +24,16 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.mapper.internal.UidFieldMapper;
-import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.internal.InternalSearchHit;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
-import java.util.Map;
-
-import static java.util.Collections.singletonMap;
 
 /**
  *
  */
 public class VersionFetchSubPhase implements FetchSubPhase {
-    private static final Map<String, ? extends SearchParseElement> PARSE_ELEMENTS = singletonMap("version", new VersionParseElement());
-
-    @Override
-    public Map<String, ? extends SearchParseElement> parseElements() {
-        return PARSE_ELEMENTS;
-    }
 
     @Override
     public boolean hitsExecutionNeeded(SearchContext context) {
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/version/VersionParseElement.java b/core/src/main/java/org/elasticsearch/search/fetch/version/VersionParseElement.java
deleted file mode 100644
index bbcf967e1e1..00000000000
--- a/core/src/main/java/org/elasticsearch/search/fetch/version/VersionParseElement.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.fetch.version;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.SearchParseElement;
-import org.elasticsearch.search.internal.SearchContext;
-
-/**
- *
- */
-public class VersionParseElement implements SearchParseElement {
-
-    @Override
-    public void parse(XContentParser parser, SearchContext context) throws Exception {
-        XContentParser.Token token = parser.currentToken();
-        if (token.isValue()) {
-            context.version(parser.booleanValue());
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/query/FromParseElement.java b/core/src/main/java/org/elasticsearch/search/query/FromParseElement.java
deleted file mode 100644
index 21063a93d35..00000000000
--- a/core/src/main/java/org/elasticsearch/search/query/FromParseElement.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.query;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.SearchParseElement;
-import org.elasticsearch.search.SearchParseException;
-import org.elasticsearch.search.internal.SearchContext;
-
-/**
- *
- */
-public class FromParseElement implements SearchParseElement {
-
-    @Override
-    public void parse(XContentParser parser, SearchContext context) throws Exception {
-        XContentParser.Token token = parser.currentToken();
-        if (token.isValue()) {
-            int from = parser.intValue();
-            if (from < 0) {
-                throw new SearchParseException(context, "from is set to [" + from + "] and is expected to be higher or equal to 0",
-                        parser.getTokenLocation());
-            }
-            context.from(from);
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/query/IndicesBoostParseElement.java b/core/src/main/java/org/elasticsearch/search/query/IndicesBoostParseElement.java
deleted file mode 100644
index eeb70ebe805..00000000000
--- a/core/src/main/java/org/elasticsearch/search/query/IndicesBoostParseElement.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.query;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.SearchParseElement;
-import org.elasticsearch.search.internal.SearchContext;
-
-/**
- * <pre>
- * {
- *    indicesBoost : {
- *         "index1" : 1.4,
- *         "index2" : 1.5
- *    }
- * }
- * </pre>
- *
- *
- */
-public class IndicesBoostParseElement implements SearchParseElement {
-
-    @Override
-    public void parse(XContentParser parser, SearchContext context) throws Exception {
-        XContentParser.Token token;
-        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-            if (token == XContentParser.Token.FIELD_NAME) {
-                String indexName = parser.currentName();
-                if (indexName.equals(context.shardTarget().index())) {
-                    parser.nextToken(); // move to the value
-                    // we found our query boost
-                    context.queryBoost(parser.floatValue());
-                }
-            }
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/query/MinScoreParseElement.java b/core/src/main/java/org/elasticsearch/search/query/MinScoreParseElement.java
deleted file mode 100644
index 49048717cdc..00000000000
--- a/core/src/main/java/org/elasticsearch/search/query/MinScoreParseElement.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.query;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.SearchParseElement;
-import org.elasticsearch.search.internal.SearchContext;
-
-/**
- *
- */
-public class MinScoreParseElement implements SearchParseElement {
-
-    @Override
-    public void parse(XContentParser parser, SearchContext context) throws Exception {
-        XContentParser.Token token = parser.currentToken();
-        if (token.isValue()) {
-            context.minimumScore(parser.floatValue());
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/query/PostFilterParseElement.java b/core/src/main/java/org/elasticsearch/search/query/PostFilterParseElement.java
deleted file mode 100644
index 1b9fee22d76..00000000000
--- a/core/src/main/java/org/elasticsearch/search/query/PostFilterParseElement.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.query;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.ParsedQuery;
-import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.search.SearchParseElement;
-import org.elasticsearch.search.internal.SearchContext;
-
-/**
- *
- */
-public class PostFilterParseElement implements SearchParseElement {
-
-    @Override
-    public void parse(XContentParser parser, SearchContext context) throws Exception {
-        QueryShardContext shardContext = context.getQueryShardContext();
-        ParsedQuery postFilter = shardContext.toFilter(shardContext.newParseContext(parser).parseInnerQueryBuilder());
-        if (postFilter != null) {
-            context.parsedPostFilter(postFilter);
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryParseElement.java b/core/src/main/java/org/elasticsearch/search/query/QueryParseElement.java
deleted file mode 100644
index cfa4ea21747..00000000000
--- a/core/src/main/java/org/elasticsearch/search/query/QueryParseElement.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.query;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.search.SearchParseElement;
-import org.elasticsearch.search.internal.SearchContext;
-
-/**
- *
- */
-public class QueryParseElement implements SearchParseElement {
-
-    @Override
-    public void parse(XContentParser parser, SearchContext context) throws Exception {
-        QueryShardContext queryShardContext = context.getQueryShardContext();
-        context.parsedQuery(queryShardContext.toQuery(queryShardContext.newParseContext(parser).parseInnerQueryBuilder()));
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index a29831f5a04..1ae80337d76 100644
--- a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -47,7 +47,6 @@ import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.MinimumScoreCollector;
 import org.elasticsearch.common.lucene.search.FilteredCollector;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.SearchPhase;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.aggregations.AggregationPhase;
@@ -60,19 +59,14 @@ import org.elasticsearch.search.profile.Profiler;
 import org.elasticsearch.search.rescore.RescorePhase;
 import org.elasticsearch.search.rescore.RescoreSearchContext;
 import org.elasticsearch.search.sort.SortAndFormats;
-import org.elasticsearch.search.sort.TrackScoresParseElement;
 import org.elasticsearch.search.suggest.SuggestPhase;
 
 import java.util.AbstractList;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.concurrent.Callable;
 
-import static java.util.Collections.unmodifiableMap;
-
 /**
  *
  */
@@ -89,28 +83,6 @@ public class QueryPhase implements SearchPhase {
         this.rescorePhase = rescorePhase;
     }
 
-    @Override
-    public Map<String, ? extends SearchParseElement> parseElements() {
-        Map<String, SearchParseElement> parseElements = new HashMap<>();
-        parseElements.put("from", new FromParseElement());
-        parseElements.put("size", new SizeParseElement());
-        parseElements.put("indices_boost", new IndicesBoostParseElement());
-        parseElements.put("indicesBoost", new IndicesBoostParseElement());
-        parseElements.put("query", new QueryParseElement());
-        parseElements.put("post_filter", new PostFilterParseElement());
-        parseElements.put("postFilter", new PostFilterParseElement());
-        parseElements.put("trackScores", new TrackScoresParseElement());
-        parseElements.put("track_scores", new TrackScoresParseElement());
-        parseElements.put("min_score", new MinScoreParseElement());
-        parseElements.put("minScore", new MinScoreParseElement());
-        parseElements.put("timeout", new TimeoutParseElement());
-        parseElements.put("terminate_after", new TerminateAfterParseElement());
-        parseElements.putAll(aggregationPhase.parseElements());
-        parseElements.putAll(suggestPhase.parseElements());
-        parseElements.putAll(rescorePhase.parseElements());
-        return unmodifiableMap(parseElements);
-    }
-
     @Override
     public void preProcess(SearchContext context) {
         context.preProcess();
diff --git a/core/src/main/java/org/elasticsearch/search/query/SizeParseElement.java b/core/src/main/java/org/elasticsearch/search/query/SizeParseElement.java
deleted file mode 100644
index 5560ec939c4..00000000000
--- a/core/src/main/java/org/elasticsearch/search/query/SizeParseElement.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.query;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.SearchParseElement;
-import org.elasticsearch.search.SearchParseException;
-import org.elasticsearch.search.internal.SearchContext;
-
-/**
- *
- */
-public class SizeParseElement implements SearchParseElement {
-
-    @Override
-    public void parse(XContentParser parser, SearchContext context) throws Exception {
-        XContentParser.Token token = parser.currentToken();
-        if (token.isValue()) {
-            int size = parser.intValue();
-            if (size < 0) {
-                throw new SearchParseException(context, "size is set to [" + size + "] and is expected to be higher or equal to 0",
-                        parser.getTokenLocation());
-            }
-            context.size(size);
-        }
-    }
-}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/query/TerminateAfterParseElement.java b/core/src/main/java/org/elasticsearch/search/query/TerminateAfterParseElement.java
deleted file mode 100644
index fc4b64fe9ac..00000000000
--- a/core/src/main/java/org/elasticsearch/search/query/TerminateAfterParseElement.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.query;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.SearchParseElement;
-import org.elasticsearch.search.internal.SearchContext;
-
-/**
- * Parser element for 'terminate_after'
- */
-public class TerminateAfterParseElement implements SearchParseElement {
-
-    @Override
-    public void parse(XContentParser parser, SearchContext context) throws Exception {
-        XContentParser.Token token = parser.currentToken();
-        if (token == XContentParser.Token.VALUE_NUMBER) {
-            int terminateAfterCount = parser.intValue();
-            if (terminateAfterCount <= 0) {
-                throw new IllegalArgumentException("terminateAfter must be > 0");
-            }
-            context.terminateAfter(parser.intValue());
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/query/TimeoutParseElement.java b/core/src/main/java/org/elasticsearch/search/query/TimeoutParseElement.java
deleted file mode 100644
index d57f5362a25..00000000000
--- a/core/src/main/java/org/elasticsearch/search/query/TimeoutParseElement.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.query;
-
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.SearchParseElement;
-import org.elasticsearch.search.internal.SearchContext;
-
-/**
- */
-public class TimeoutParseElement implements SearchParseElement {
-
-    @Override
-    public void parse(XContentParser parser, SearchContext context) throws Exception {
-        XContentParser.Token token = parser.currentToken();
-        if (token == XContentParser.Token.VALUE_NUMBER) {
-            context.timeoutInMillis(parser.longValue());
-        } else {
-            context.timeoutInMillis(TimeValue.parseTimeValue(parser.text(), null, "timeout").millis());
-        }
-    }
-}
-
diff --git a/core/src/main/java/org/elasticsearch/search/sort/TrackScoresParseElement.java b/core/src/main/java/org/elasticsearch/search/sort/TrackScoresParseElement.java
deleted file mode 100644
index a2eaf54e691..00000000000
--- a/core/src/main/java/org/elasticsearch/search/sort/TrackScoresParseElement.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.sort;
-
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.SearchParseElement;
-import org.elasticsearch.search.internal.SearchContext;
-
-/**
- *
- */
-public class TrackScoresParseElement implements SearchParseElement {
-
-    @Override
-    public void parse(XContentParser parser, SearchContext context) throws Exception {
-        XContentParser.Token token = parser.currentToken();
-        if (token.isValue()) {
-            context.trackScores(parser.booleanValue());
-        }
-    }
-}
\ No newline at end of file
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
index 071f7375383..ea76dbf7d6c 100644
--- a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
+++ b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
@@ -182,7 +182,8 @@ public class FetchSubPhasePluginIT extends ESIntegTestCase {
     public static class TermVectorsFetchParseElement extends FetchSubPhaseParseElement<TermVectorsFetchContext> {
 
         @Override
-        protected void innerParse(XContentParser parser, TermVectorsFetchContext termVectorsFetchContext, SearchContext searchContext) throws Exception {
+        protected void innerParse(XContentParser parser, TermVectorsFetchContext termVectorsFetchContext, SearchContext searchContext)
+                throws Exception {
             XContentParser.Token token = parser.currentToken();
             if (token == XContentParser.Token.VALUE_STRING) {
                 String fieldName = parser.text();
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/FieldDataFieldsTests.java b/core/src/test/java/org/elasticsearch/search/fetch/FieldDataFieldsTests.java
deleted file mode 100644
index ba7f70c63a7..00000000000
--- a/core/src/test/java/org/elasticsearch/search/fetch/FieldDataFieldsTests.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.fetch;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsParseElement;
-import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.TestSearchContext;
-
-import static org.hamcrest.Matchers.containsString;
-
-public class FieldDataFieldsTests extends ESTestCase {
-
-    public void testValidFieldDataFieldString() throws Exception {
-        FieldDataFieldsParseElement parseElement = new FieldDataFieldsParseElement();
-
-        BytesArray data = new BytesArray(new BytesRef("{\"fielddata_fields\": \"foobar\"}"));
-        XContentParser parser = XContentFactory.xContent(data).createParser(data);
-        parser.nextToken();
-        parser.nextToken();
-        parser.nextToken();
-        SearchContext context = new TestSearchContext(null);
-        parseElement.parse(parser, context);
-    }
-
-    public void testValidFieldDataFieldArray() throws Exception {
-        FieldDataFieldsParseElement parseElement = new FieldDataFieldsParseElement();
-
-        BytesArray data = new BytesArray(new BytesRef("{\"fielddata_fields\": [ \"foo\", \"bar\", \"baz\"]}}"));
-        XContentParser parser = XContentFactory.xContent(data).createParser(data);
-        parser.nextToken();
-        parser.nextToken();
-        parser.nextToken();
-        SearchContext context = new TestSearchContext(null);
-        parseElement.parse(parser, context);
-    }
-
-    public void testInvalidFieldDataField() throws Exception {
-        FieldDataFieldsParseElement parseElement = new FieldDataFieldsParseElement();
-
-        BytesArray data;
-        if (randomBoolean()) {
-            data = new BytesArray(new BytesRef("{\"fielddata_fields\": {}}}"));
-        } else {
-            data = new BytesArray(new BytesRef("{\"fielddata_fields\": 1.0}}"));
-        }
-        XContentParser parser = XContentFactory.xContent(data).createParser(data);
-        parser.nextToken();
-        parser.nextToken();
-        parser.nextToken();
-        SearchContext context = new TestSearchContext(null);
-        try {
-            parseElement.parse(parser, context);
-            fail("Expected IllegalStateException");
-        } catch (IllegalStateException e) {
-            assertThat(e.getMessage(), containsString("Expected either a VALUE_STRING or an START_ARRAY but got "));
-        }
-    }
-}

From 3aeb66885bacbc175fc51c93e303173d2a654a74 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 10 May 2016 05:44:16 -0400
Subject: [PATCH 0150/1311] painless: use better typing for dynamic method
 calls

---
 .../org/elasticsearch/painless/WriterExternal.java    | 11 +++++++----
 .../painless/WhenThingsGoWrongTests.java              |  3 ++-
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java
index ab05e1b6e56..1ead39c467b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless;
 
 import org.antlr.v4.runtime.ParserRuleContext;
+import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Definition.Constructor;
 import org.elasticsearch.painless.Definition.Field;
 import org.elasticsearch.painless.Definition.Method;
@@ -714,14 +715,16 @@ class WriterExternal {
         // first parameter is the receiver, we never know its type: always Object
         signature.append(WriterConstants.OBJECT_TYPE.getDescriptor());
 
-        // TODO: remove our explicit conversions and feed more type information for args/return value,
-        // it can avoid some unnecessary boxing etc.
         for (int i = 0; i < arguments.size(); i++) {
-            signature.append(WriterConstants.OBJECT_TYPE.getDescriptor());
+            ExpressionMetadata arg = metadata.getExpressionMetadata(arguments.get(i));
+            // disable any implicit casts/conversion for arguments, let invokeDynamic take care
+            arg.to = arg.from;
+            arg.cast = new Cast(arg.from, arg.from);
+            signature.append(arg.from.type.getDescriptor());
             writer.visit(arguments.get(i));
         }
         signature.append(')');
-        // return value
+        // return value: currently always Object. making this better may be tricky...
         signature.append(WriterConstants.OBJECT_TYPE.getDescriptor());
         execute.visitInvokeDynamicInsn((String)sourceenmd.target, signature.toString(), 
                                        WriterConstants.DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.METHOD_CALL });
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
index 3f79ae1974b..975e2301877 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.painless;
 
+import java.lang.invoke.WrongMethodTypeException;
 import java.util.Arrays;
 import java.util.Collections;
 
@@ -140,7 +141,7 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
     }
     
     public void testDynamicWrongArgs() {
-        expectThrows(ClassCastException.class, () -> {
+        expectThrows(WrongMethodTypeException.class, () -> {
             exec("def x = new ArrayList(); return x.get('bogus');");
         });
     }

From e486560ea85134c62504aa9df5a3fa485060717b Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Mon, 9 May 2016 16:13:07 +0200
Subject: [PATCH 0151/1311] Add Autosense annotation for query dsl testing

this adds the autosense annotation to a couple of query dsl
docs files and fixes the snippets to work in the tests along
the way.
---
 docs/reference/query-dsl/bool-query.asciidoc  |   6 +-
 .../query-dsl/constant-score-query.asciidoc   |   4 +
 .../query-dsl/function-score-query.asciidoc   |  68 ++++++----
 .../query-dsl/geo-distance-query.asciidoc     |  56 +++++++-
 .../query-dsl/match-all-query.asciidoc        |  21 ++-
 docs/reference/query-dsl/mlt-query.asciidoc   |  16 ++-
 .../query-dsl/multi-match-query.asciidoc      | 120 +++++++++++++-----
 .../reference/query-dsl/prefix-query.asciidoc |  15 ++-
 .../query-dsl/query_filter_context.asciidoc   |   3 +-
 docs/reference/query-dsl/range-query.asciidoc |  43 +++++--
 .../simple-query-string-query.asciidoc        |  12 ++
 .../query-dsl/span-first-query.asciidoc       |   4 +
 .../query-dsl/span-or-query.asciidoc          |   4 +
 .../query-dsl/template-query.asciidoc         |  15 ++-
 docs/reference/query-dsl/type-query.asciidoc  |   4 +
 15 files changed, 304 insertions(+), 87 deletions(-)

diff --git a/docs/reference/query-dsl/bool-query.asciidoc b/docs/reference/query-dsl/bool-query.asciidoc
index 8fcd66129cd..2a289910c43 100644
--- a/docs/reference/query-dsl/bool-query.asciidoc
+++ b/docs/reference/query-dsl/bool-query.asciidoc
@@ -81,7 +81,7 @@ all documents where the `status` field contains the term `active`.
 This first query assigns a score of `0` to all documents, as no scoring
 query has been specified:
 
-[source,json]
+[source,js]
 ---------------------------------
 GET _search
 {
@@ -101,7 +101,7 @@ GET _search
 This `bool` query has a `match_all` query, which assigns a score of `1.0` to
 all documents.
 
-[source,json]
+[source,js]
 ---------------------------------
 GET _search
 {
@@ -125,7 +125,7 @@ This `constant_score` query behaves in exactly the same way as the second exampl
 The `constant_score` query assigns a score of `1.0` to all documents matched
 by the filter.
 
-[source,json]
+[source,js]
 ---------------------------------
 GET _search
 {
diff --git a/docs/reference/query-dsl/constant-score-query.asciidoc b/docs/reference/query-dsl/constant-score-query.asciidoc
index 8e76ac13ff5..0928e9a5718 100644
--- a/docs/reference/query-dsl/constant-score-query.asciidoc
+++ b/docs/reference/query-dsl/constant-score-query.asciidoc
@@ -7,12 +7,16 @@ filter. Maps to Lucene `ConstantScoreQuery`.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
+  "query": {
     "constant_score" : {
         "filter" : {
             "term" : { "user" : "kimchy"}
         },
         "boost" : 1.2
     }
+  }
 }
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/query-dsl/function-score-query.asciidoc b/docs/reference/query-dsl/function-score-query.asciidoc
index 5f92d8c172c..e48a396e6c8 100644
--- a/docs/reference/query-dsl/function-score-query.asciidoc
+++ b/docs/reference/query-dsl/function-score-query.asciidoc
@@ -14,13 +14,20 @@ by the query.
 
 [source,js]
 --------------------------------------------------
-"function_score": {
-    "query": {},
-    "boost": "boost for the whole query",
-    "FUNCTION": {}, <1>
-    "boost_mode":"(multiply|replace|...)"
+GET /_search
+{
+  "query": {
+    "function_score": {
+        "query": {},
+        "boost": "5",
+        "random_score": {}, <1>
+        "boost_mode":"multiply"
+    }
+  }
 }
 --------------------------------------------------
+// CONSOLE
+
 <1> See <<score-functions>> for a list of supported functions.
 
 Furthermore, several functions can be combined. In this case one can
@@ -29,30 +36,35 @@ given filtering query
 
 [source,js]
 --------------------------------------------------
-"function_score": {
-    "query": {},
-    "boost": "boost for the whole query",
-    "functions": [
-        {
-            "filter": {},
-            "FUNCTION": {}, <1>
-            "weight": number
-        },
-        {
-            "FUNCTION": {} <1>
-        },
-        {
-            "filter": {},
-            "weight": number
-        }
-    ],
-    "max_boost": number,
-    "score_mode": "(multiply|max|...)",
-    "boost_mode": "(multiply|replace|...)",
-    "min_score" : number
+GET /_search
+{
+  "query": {
+    "function_score": {
+      "query": {},
+      "boost": "5", <1>
+      "functions": [
+          {
+              "filter": {},
+              "random_score": {}, <2>
+              "weight": 23
+          },
+          {
+              "filter": {},
+              "weight": 42 
+          }
+      ],
+      "max_boost": 42,
+      "score_mode": "max",
+      "boost_mode": "multiply",
+      "min_score" : 42
+    }
+  }
 }
 --------------------------------------------------
-<1> See <<score-functions>> for a list of supported functions.
+// CONSOLE
+
+<1> Boost for the whole query.
+<2> See <<score-functions>> for a list of supported functions.
 
 NOTE: The scores produced by the filtering query of each function do not matter.
 
@@ -459,7 +471,7 @@ the request would look like this:
 
 [source,js]
 --------------------------------------------------
-GET _search
+GET /_search
 {
   "query": {
     "function_score": {
diff --git a/docs/reference/query-dsl/geo-distance-query.asciidoc b/docs/reference/query-dsl/geo-distance-query.asciidoc
index c29391e5b57..228ab804d7a 100644
--- a/docs/reference/query-dsl/geo-distance-query.asciidoc
+++ b/docs/reference/query-dsl/geo-distance-query.asciidoc
@@ -2,10 +2,35 @@
 === Geo Distance Query
 
 Filters documents that include only hits that exists within a specific
-distance from a geo point. Assuming the following indexed json:
+distance from a geo point. Assuming the following mapping:
 
 [source,js]
 --------------------------------------------------
+PUT /my_locations
+{
+  "mappings": {
+        "location": {
+            "properties": {
+                "pin": {
+                    "properties": {
+                        "location": {
+                            "type": "geo_point"
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+
+... and indexed document:
+
+
+[source,js]
+--------------------------------------------------
+PUT /my_locations/location/1
 {
     "pin" : {
         "location" : {
@@ -15,13 +40,18 @@ distance from a geo point. Assuming the following indexed json:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
 
 Then the following simple query can be executed with a `geo_distance`
 filter:
 
 [source,js]
 --------------------------------------------------
+GET /my_locations/location/_search
 {
+  "query": {
     "bool" : {
         "must" : {
             "match_all" : {}
@@ -36,8 +66,11 @@ filter:
             }
         }
     }
+  }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 [float]
 ==== Accepted Formats
@@ -50,7 +83,9 @@ representation of the geo point, the filter can accept it as well:
 
 [source,js]
 --------------------------------------------------
+GET /my_locations/location/_search
 {
+  "query": {
     "bool" : {
         "must" : {
             "match_all" : {}
@@ -65,8 +100,11 @@ representation of the geo point, the filter can accept it as well:
             }
         }
     }
+  }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 [float]
 ===== Lat Lon As Array
@@ -76,7 +114,9 @@ conform with http://geojson.org/[GeoJSON].
 
 [source,js]
 --------------------------------------------------
+GET /my_locations/location/_search
 {
+  "query": {
     "bool" : {
         "must" : {
             "match_all" : {}
@@ -88,8 +128,12 @@ conform with http://geojson.org/[GeoJSON].
             }
         }
     }
+  }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
 
 [float]
 ===== Lat Lon As String
@@ -98,7 +142,9 @@ Format in `lat,lon`.
 
 [source,js]
 --------------------------------------------------
+GET /my_locations/location/_search
 {
+  "query": {
     "bool" : {
         "must" : {
             "match_all" : {}
@@ -110,15 +156,20 @@ Format in `lat,lon`.
             }
         }
     }
+  }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 [float]
 ===== Geohash
 
 [source,js]
 --------------------------------------------------
+GET /my_locations/location/_search
 {
+  "query": {
     "bool" : {
         "must" : {
             "match_all" : {}
@@ -130,8 +181,11 @@ Format in `lat,lon`.
             }
         }
     }
+  }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 [float]
 ==== Options
diff --git a/docs/reference/query-dsl/match-all-query.asciidoc b/docs/reference/query-dsl/match-all-query.asciidoc
index d46b08f9e55..d4064d053f6 100644
--- a/docs/reference/query-dsl/match-all-query.asciidoc
+++ b/docs/reference/query-dsl/match-all-query.asciidoc
@@ -6,15 +6,25 @@ of `1.0`.
 
 [source,js]
 --------------------------------------------------
-{ "match_all": {} }
+GET /_search
+{ "query": {
+    "match_all": {}
+  }
+}
 --------------------------------------------------
+// AUTOSENSE
 
 The `_score` can be changed with the `boost` parameter:
 
 [source,js]
 --------------------------------------------------
-{ "match_all": { "boost" : 1.2 }}
+GET /_search
+{ "query": {
+    "match_all": { "boost" : 1.2 }
+  }
+}
 --------------------------------------------------
+// AUTOSENSE
 
 [[query-dsl-match-none-query]]
 [float]
@@ -24,5 +34,10 @@ This is the inverse of the `match_all` query, which matches no documents.
 
 [source,js]
 --------------------------------------------------
-{ "match_none": {} }
+GET /_search
+{ "query": {
+    "match_none": {} 
+  }
+}
 --------------------------------------------------
+// AUTOSENSE
diff --git a/docs/reference/query-dsl/mlt-query.asciidoc b/docs/reference/query-dsl/mlt-query.asciidoc
index dc525c2d311..795bf51584d 100644
--- a/docs/reference/query-dsl/mlt-query.asciidoc
+++ b/docs/reference/query-dsl/mlt-query.asciidoc
@@ -15,15 +15,19 @@ fields, limiting the number of selected terms to 12.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
+  "query": {
     "more_like_this" : {
         "fields" : ["title", "description"],
         "like" : "Once upon a time",
         "min_term_freq" : 1,
         "max_query_terms" : 12
     }
+  }
 }
 --------------------------------------------------
+// AUTOSENSE
 
 A more complicated use case consists of mixing texts with documents already
 existing in the index. In this case, the syntax to specify a document is
@@ -31,7 +35,9 @@ similar to the one used in the <<docs-multi-get,Multi GET API>>.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
+  "query": {
     "more_like_this" : {
         "fields" : ["title", "description"],
         "like" : [
@@ -50,8 +56,10 @@ similar to the one used in the <<docs-multi-get,Multi GET API>>.
         "min_term_freq" : 1,
         "max_query_terms" : 12
     }
+  }
 }
 --------------------------------------------------
+// AUTOSENSE
 
 Finally, users can mix some texts, a chosen set of documents but also provide
 documents not necessarily present in the index. To provide documents not
@@ -59,7 +67,9 @@ present in the index, the syntax is similar to <<docs-termvectors-artificial-doc
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
+  "query": {
     "more_like_this" : {
         "fields" : ["name.first", "name.last"],
         "like" : [
@@ -83,8 +93,10 @@ present in the index, the syntax is similar to <<docs-termvectors-artificial-doc
         "min_term_freq" : 1,
         "max_query_terms" : 12
     }
+  }
 }
 --------------------------------------------------
+// AUTOSENSE
 
 ==== How it Works
 
@@ -111,7 +123,8 @@ default, but there will be no speed up on analysis for these fields.
 
 [source,js]
 --------------------------------------------------
-curl -s -XPUT 'http://localhost:9200/imdb/' -d '{
+PUT /imdb
+{
   "mappings": {
     "movies": {
       "properties": {
@@ -137,6 +150,7 @@ curl -s -XPUT 'http://localhost:9200/imdb/' -d '{
   }
 }
 --------------------------------------------------
+// AUTOSENSE
 
 ==== Parameters
 
diff --git a/docs/reference/query-dsl/multi-match-query.asciidoc b/docs/reference/query-dsl/multi-match-query.asciidoc
index fe1a5f6b974..02f38ebf3a1 100644
--- a/docs/reference/query-dsl/multi-match-query.asciidoc
+++ b/docs/reference/query-dsl/multi-match-query.asciidoc
@@ -6,13 +6,17 @@ to allow multi-field queries:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "multi_match" : {
-    "query":    "this is a test", <1>
-    "fields": [ "subject", "message" ] <2>
+  "query": {
+    "multi_match" : {
+      "query":    "this is a test", <1>
+      "fields": [ "subject", "message" ] <2>
+    }
   }
 }
 --------------------------------------------------
+// AUTOSENSE
 <1> The query string.
 <2> The fields to be queried.
 
@@ -23,26 +27,35 @@ Fields can be specified with wildcards, eg:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "multi_match" : {
-    "query":    "Will Smith",
-    "fields": [ "title", "*_name" ] <1>
+  "query": {
+    "multi_match" : {
+      "query":    "Will Smith",
+      "fields": [ "title", "*_name" ] <1>
+    }
   }
 }
 --------------------------------------------------
+// AUTOSENSE
 <1> Query the `title`, `first_name` and `last_name` fields.
 
 Individual fields can be boosted with the caret (`^`) notation:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "multi_match" : {
-    "query" : "this is a test",
-    "fields" : [ "subject^3", "message" ] <1>
+  "query": {
+    "multi_match" : {
+      "query" : "this is a test",
+      "fields" : [ "subject^3", "message" ] <1>
+    }
   }
 }
 --------------------------------------------------
+// AUTOSENSE
+
 <1> The `subject` field is three times as important as the `message` field.
 
 [[multi-match-types]]
@@ -82,30 +95,38 @@ find the single best matching field.  For instance, this query:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
+  "query": {
   "multi_match" : {
     "query":      "brown fox",
     "type":       "best_fields",
     "fields":     [ "subject", "message" ],
     "tie_breaker": 0.3
   }
+  }
 }
 --------------------------------------------------
+// AUTOSENSE
 
 would be executed as:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "dis_max": {
-    "queries": [
-      { "match": { "subject": "brown fox" }},
-      { "match": { "message": "brown fox" }}
-    ],
-    "tie_breaker": 0.3
+  "query": {
+    "dis_max": {
+      "queries": [
+        { "match": { "subject": "brown fox" }},
+        { "match": { "message": "brown fox" }}
+      ],
+      "tie_breaker": 0.3
+    }
   }
 }
 --------------------------------------------------
+// AUTOSENSE
 
 Normally the `best_fields` type uses the score of the *single* best matching
 field, but if `tie_breaker` is specified, then it calculates the score as
@@ -132,15 +153,20 @@ Take this query for example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "multi_match" : {
-    "query":      "Will Smith",
-    "type":       "best_fields",
-    "fields":     [ "first_name", "last_name" ],
-    "operator":   "and" <1>
+  "query": {
+    "multi_match" : {
+      "query":      "Will Smith",
+      "type":       "best_fields",
+      "fields":     [ "first_name", "last_name" ],
+      "operator":   "and" <1>
+    }
   }
 }
 --------------------------------------------------
+// AUTOSENSE
+
 <1> All terms must be present.
 
 This query is executed as:
@@ -170,29 +196,37 @@ This query:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "multi_match" : {
-    "query":      "quick brown fox",
-    "type":       "most_fields",
-    "fields":     [ "title", "title.original", "title.shingles" ]
+  "query": {
+    "multi_match" : {
+      "query":      "quick brown fox",
+      "type":       "most_fields",
+      "fields":     [ "title", "title.original", "title.shingles" ]
+    }
   }
 }
 --------------------------------------------------
+// AUTOSENSE
 
 would be executed as:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "bool": {
-    "should": [
-      { "match": { "title":          "quick brown fox" }},
-      { "match": { "title.original": "quick brown fox" }},
-      { "match": { "title.shingles": "quick brown fox" }}
-    ]
+  "query": {
+    "bool": {
+      "should": [
+        { "match": { "title":          "quick brown fox" }},
+        { "match": { "title.original": "quick brown fox" }},
+        { "match": { "title.shingles": "quick brown fox" }}
+      ]
+    }
   }
 }
 --------------------------------------------------
+// AUTOSENSE
 
 The score from each `match` clause is added together, then divided by the
 number of `match` clauses.
@@ -212,28 +246,36 @@ but they use a `match_phrase` or `match_phrase_prefix` query instead of a
 This query:
 [source,js]
 --------------------------------------------------
+GET /_search
 {
+  "query": {
   "multi_match" : {
     "query":      "quick brown f",
     "type":       "phrase_prefix",
     "fields":     [ "subject", "message" ]
   }
+  }
 }
 --------------------------------------------------
+// AUTOSENSE
 
 would be executed as:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
+  "query": {
   "dis_max": {
     "queries": [
       { "match_phrase_prefix": { "subject": "quick brown f" }},
       { "match_phrase_prefix": { "message": "quick brown f" }}
     ]
   }
+  }
 }
 --------------------------------------------------
+// AUTOSENSE
 
 Also, accepts `analyzer`, `boost`, `slop` and `zero_terms_query`  as explained
 in <<query-dsl-match-query>>.  Type `phrase_prefix` additionally accepts
@@ -281,15 +323,19 @@ A query like:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
+  "query": {
   "multi_match" : {
     "query":      "Will Smith",
     "type":       "cross_fields",
     "fields":     [ "first_name", "last_name" ],
     "operator":   "and"
   }
+  }
 }
 --------------------------------------------------
+// AUTOSENSE
 
 is executed as:
 
@@ -337,7 +383,9 @@ both use an `edge_ngram` analyzer, this query:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
+  "query": {
   "multi_match" : {
     "query":      "Jon",
     "type":       "cross_fields",
@@ -346,8 +394,10 @@ both use an `edge_ngram` analyzer, this query:
         "last",  "last.edge"
     ]
   }
+  }
 }
 --------------------------------------------------
+// AUTOSENSE
 
 would be executed as:
 
@@ -372,7 +422,9 @@ parameter to just one of them:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
+  "query": {
     "bool": {
         "should": [
             {
@@ -392,8 +444,11 @@ parameter to just one of them:
             }
         ]
     }
+  }
 }
 --------------------------------------------------
+// AUTOSENSE 
+
 <1> Either `will` or `smith` must be present in either of the `first`
     or `last` fields
 
@@ -402,15 +457,20 @@ parameter in the query.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
+  "query": {
   "multi_match" : {
     "query":      "Jon",
     "type":       "cross_fields",
     "analyzer":   "standard", <1>
     "fields":     [ "first", "last", "*.edge" ]
   }
+  }
 }
 --------------------------------------------------
+// AUTOSENSE
+
 <1> Use the `standard` analyzer for all fields.
 
 which will be executed as:
diff --git a/docs/reference/query-dsl/prefix-query.asciidoc b/docs/reference/query-dsl/prefix-query.asciidoc
index cf26e850ad8..543cccfeef5 100644
--- a/docs/reference/query-dsl/prefix-query.asciidoc
+++ b/docs/reference/query-dsl/prefix-query.asciidoc
@@ -8,28 +8,37 @@ that starts with `ki`:
 
 [source,js]
 --------------------------------------------------
-{
+GET /_search
+{ "query": {
     "prefix" : { "user" : "ki" }
+  }
 }
 --------------------------------------------------
+// AUTOSENSE
 
 A boost can also be associated with the query:
 
 [source,js]
 --------------------------------------------------
-{
+GET /_search
+{ "query": {
     "prefix" : { "user" :  { "value" : "ki", "boost" : 2.0 } }
+  }
 }
 --------------------------------------------------
+// AUTOSENSE
 
 Or :
 
 [source,js]
 --------------------------------------------------
-{
+GET /_search
+{ "query": {
     "prefix" : { "user" :  { "prefix" : "ki", "boost" : 2.0 } }
+  }
 }
 --------------------------------------------------
+// AUTOSENSE
 
 This multi term query allows you to control how it gets rewritten using the
 <<query-dsl-multi-term-rewrite,rewrite>>
diff --git a/docs/reference/query-dsl/query_filter_context.asciidoc b/docs/reference/query-dsl/query_filter_context.asciidoc
index 6fadc36aa73..7ef0d7b0f4f 100644
--- a/docs/reference/query-dsl/query_filter_context.asciidoc
+++ b/docs/reference/query-dsl/query_filter_context.asciidoc
@@ -47,7 +47,7 @@ conditions are met:
 
 [source,js]
 ------------------------------------
-GET _search
+GET /_search
 {
   "query": { <1>
     "bool": { <2>
@@ -63,6 +63,7 @@ GET _search
   }
 }
 ------------------------------------
+// AUTOSENSE
 <1> The `query` parameter indicates query context.
 <2> The `bool` and two `match` clauses are used in query context,
     which means that they are used to score how well each document
diff --git a/docs/reference/query-dsl/range-query.asciidoc b/docs/reference/query-dsl/range-query.asciidoc
index eaf1dca8181..939c300f1a0 100644
--- a/docs/reference/query-dsl/range-query.asciidoc
+++ b/docs/reference/query-dsl/range-query.asciidoc
@@ -9,16 +9,20 @@ a `NumericRangeQuery`. The following example returns all documents where
 
 [source,js]
 --------------------------------------------------
+GET _search
 {
-    "range" : {
-        "age" : {
-            "gte" : 10,
-            "lte" : 20,
-            "boost" : 2.0
+    "query": {
+        "range" : {
+            "age" : {
+                "gte" : 10,
+                "lte" : 20,
+                "boost" : 2.0
+            }
         }
     }
 }
 --------------------------------------------------
+// AUTOSENSE
 
 The `range` query accepts the following parameters:
 
@@ -38,15 +42,19 @@ specified using <<date-math>>:
 
 [source,js]
 --------------------------------------------------
+GET _search
 {
-    "range" : {
-        "date" : {
-            "gte" : "now-1d/d",
-            "lt" :  "now/d"
+    "query": {
+        "range" : {
+            "date" : {
+                "gte" : "now-1d/d",
+                "lt" :  "now/d"
+            }
         }
     }
 }
 --------------------------------------------------
+// AUTOSENSE
 
 ===== Date math and rounding
 
@@ -86,7 +94,9 @@ passing the `format` parameter to the `range` query:
 
 [source,js]
 --------------------------------------------------
+GET _search
 {
+    "query": 
     "range" : {
         "born" : {
             "gte": "01/01/2012",
@@ -105,15 +115,20 @@ accepts it), or it can be specified as the `time_zone` parameter:
 
 [source,js]
 --------------------------------------------------
+GET _search
 {
-    "range" : {
-        "timestamp" : {
-            "gte": "2015-01-01 00:00:00", <1>
-            "lte": "now", <2>
-            "time_zone": "+01:00"
+    "query":
+    {
+        "range" : {
+            "timestamp" : {
+                "gte": "2015-01-01 00:00:00", <1>
+                "lte": "now", <2>
+                "time_zone": "+01:00"
+            }
         }
     }
 }
 --------------------------------------------------
+// AUTOSENSE
 <1> This date will be converted to `2014-12-31T23:00:00 UTC`.
 <2> `now` is not affected by the `time_zone` parameter (dates must be stored as UTC).
diff --git a/docs/reference/query-dsl/simple-query-string-query.asciidoc b/docs/reference/query-dsl/simple-query-string-query.asciidoc
index 572c0661fa4..72093a53f6e 100644
--- a/docs/reference/query-dsl/simple-query-string-query.asciidoc
+++ b/docs/reference/query-dsl/simple-query-string-query.asciidoc
@@ -8,15 +8,19 @@ an example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
+  "query": {
     "simple_query_string" : {
         "query": "\"fried eggs\" +(eggplant | potato) -frittata",
         "analyzer": "snowball",
         "fields": ["body^5","_all"],
         "default_operator": "and"
     }
+  }
 }
 --------------------------------------------------
+// CONSOLE
 
 The `simple_query_string` top level parameters include:
 
@@ -94,13 +98,17 @@ introduced fields included). For example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
+  "query": {
     "simple_query_string" : {
         "fields" : ["content", "name.*^5"],
         "query" : "foo bar baz"
     }
+  }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== Flags
@@ -110,13 +118,17 @@ should be enabled. It is specified as a `|`-delimited string with the
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
+  "query": {
     "simple_query_string" : {
         "query" : "foo | bar + baz*",
         "flags" : "OR|AND|PREFIX"
     }
+  }
 }
 --------------------------------------------------
+// CONSOLE
 
 The available flags are: `ALL`, `NONE`, `AND`, `OR`, `NOT`, `PREFIX`, `PHRASE`,
 `PRECEDENCE`, `ESCAPE`, `WHITESPACE`, `FUZZY`, `NEAR`, and `SLOP`.
diff --git a/docs/reference/query-dsl/span-first-query.asciidoc b/docs/reference/query-dsl/span-first-query.asciidoc
index 74fe7ff88ba..36a6a2edd9f 100644
--- a/docs/reference/query-dsl/span-first-query.asciidoc
+++ b/docs/reference/query-dsl/span-first-query.asciidoc
@@ -6,15 +6,19 @@ to Lucene `SpanFirstQuery`. Here is an example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
+  "query": {
     "span_first" : {
         "match" : {
             "span_term" : { "user" : "kimchy" }
         },
         "end" : 3
     }
+  }
 }    
 --------------------------------------------------
+// AUTOSENSE
 
 The `match` clause can be any other span type query. The `end` controls
 the maximum end position permitted in a match.
diff --git a/docs/reference/query-dsl/span-or-query.asciidoc b/docs/reference/query-dsl/span-or-query.asciidoc
index 72a4ce8724b..487dcffb6b4 100644
--- a/docs/reference/query-dsl/span-or-query.asciidoc
+++ b/docs/reference/query-dsl/span-or-query.asciidoc
@@ -6,7 +6,9 @@ Matches the union of its span clauses. The span or query maps to Lucene
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
+  "query": {
     "span_or" : {
         "clauses" : [
             { "span_term" : { "field" : "value1" } },
@@ -14,7 +16,9 @@ Matches the union of its span clauses. The span or query maps to Lucene
             { "span_term" : { "field" : "value3" } }
         ]
     }
+  }
 }
 --------------------------------------------------
+// CONSOLE
 
 The `clauses` element is a list of one or more other span type queries.
diff --git a/docs/reference/query-dsl/template-query.asciidoc b/docs/reference/query-dsl/template-query.asciidoc
index 35365e6f9b2..ed2432a3479 100644
--- a/docs/reference/query-dsl/template-query.asciidoc
+++ b/docs/reference/query-dsl/template-query.asciidoc
@@ -19,8 +19,8 @@ GET /_search
         }
     }
 }
-
 ------------------------------------------
+// AUTOSENSE
 
 The above request is translated into:
 
@@ -34,8 +34,8 @@ GET /_search
         }
     }
 }
-
 ------------------------------------------
+// AUTOSENSE
 
 Alternatively passing the template as an escaped string works as well:
 
@@ -53,6 +53,8 @@ GET /_search
     }
 }
 ------------------------------------------
+// AUTOSENSE
+
 <1> New line characters (`\n`) should be escaped as `\\n` or removed,
     and quotes (`"`) should be escaped as `\\"`.
 
@@ -77,6 +79,8 @@ GET /_search
     }
 }
 ------------------------------------------
+// AUTOSENSE
+
 <1> Name of the query template in `config/scripts/`, i.e., `my_template.mustache`.
 
 Alternatively, you can register a query template in the cluster state with:
@@ -85,9 +89,10 @@ Alternatively, you can register a query template in the cluster state with:
 ------------------------------------------
 PUT /_search/template/my_template
 {
-    "template": { "match": { "text": "{{query_string}}" }},
+    "template": { "match": { "text": "{{query_string}}" }}
 }
 ------------------------------------------
+// AUTOSENSE
 
 and refer to it in the `template` query with the `id` parameter:
 
@@ -106,9 +111,13 @@ GET /_search
     }
 }
 ------------------------------------------
+// AUTOSENSE
+// TEST[continued]
+
 <1> Name of the query template in `config/scripts/`, i.e., `my_template.mustache`.
 
 
 There is also a dedicated `template` endpoint, allows you to template an entire search request.
 Please see <<search-template>> for more details.
 
+
diff --git a/docs/reference/query-dsl/type-query.asciidoc b/docs/reference/query-dsl/type-query.asciidoc
index d3ef1a30fb1..94aaa2d2b65 100644
--- a/docs/reference/query-dsl/type-query.asciidoc
+++ b/docs/reference/query-dsl/type-query.asciidoc
@@ -5,9 +5,13 @@ Filters documents matching the provided document / mapping type.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
+  "query": {
     "type" : {
         "value" : "my_type"
     }
+  }
 }    
 --------------------------------------------------
+// AUTOSENSE

From 02cf429e539687256ce90a7f33f66b45d536b0ac Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Tue, 10 May 2016 12:18:21 +0200
Subject: [PATCH 0152/1311] Update full-text-queries.asciidoc

Updated the full text query intro to add `match_phrase` and `match_phrase_prefix`
---
 docs/reference/query-dsl/full-text-queries.asciidoc | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/reference/query-dsl/full-text-queries.asciidoc b/docs/reference/query-dsl/full-text-queries.asciidoc
index 987e8785096..ba3924669d8 100644
--- a/docs/reference/query-dsl/full-text-queries.asciidoc
+++ b/docs/reference/query-dsl/full-text-queries.asciidoc
@@ -13,6 +13,14 @@ The queries in this group are:
     The standard query for performing full text queries, including fuzzy matching
     and phrase or proximity queries.
 
+<<query-dsl-match-query-phrase,`match_phrase` query>>::
+
+    Like the `match` query but used for matching exact phrases or word proximity matches.
+    
+<<query-dsl-match-query-phrase-prefix,`match_phrase_prefix` query>>::
+
+    The poor man's _search-as-you-type_.  Like the `match_phrase` query, but does a wildcard search on the final word.
+    
 <<query-dsl-multi-match-query,`multi_match` query>>::
 
     The multi-field version of the `match` query.

From 8972f39a9ca95a8b738c11d44c5a70f133dd2c6d Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@elastic.co>
Date: Tue, 10 May 2016 12:24:49 +0200
Subject: [PATCH 0153/1311] Reenable
 CircuitBreakerServiceIT#testParentChecking()

---
 .../breaker/CircuitBreakerServiceIT.java      | 28 +++++++++++--------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
index 351742d9712..539d75fdc15 100644
--- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
@@ -62,8 +62,10 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.startsWith;
 
 /**
  * Integration tests for InternalCircuitBreakerService
@@ -84,6 +86,8 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase {
                 .put(HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(),
                         HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getDefaultRaw(null))
                 .put(HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey(), 1.0)
+                .put(HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(),
+                    HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getDefaultRaw(null))
                 .build();
         assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(resetSettings));
     }
@@ -210,7 +214,6 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase {
      * Test that a breaker correctly redistributes to a different breaker, in
      * this case, the fielddata breaker borrows space from the request breaker
      */
-    @AwaitsFix(bugUrl = "way too unstable request size. Needs a proper and more stable fix.")
     public void testParentChecking() throws Exception {
         if (noopBreakerUsed()) {
             logger.info("--> noop breakers used, skipping test");
@@ -228,10 +231,6 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase {
         }
         indexRandom(true, reqs);
 
-        // We need the request limit beforehand, just from a single node because the limit should always be the same
-        long beforeReqLimit = client.admin().cluster().prepareNodesStats().setBreaker(true).get()
-                .getNodes().get(0).getBreaker().getStats(CircuitBreaker.REQUEST).getLimit();
-
         Settings resetSettings = Settings.builder()
                 .put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "10b")
                 .put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey(), 1.0)
@@ -252,11 +251,11 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase {
                 RestStatus.INTERNAL_SERVER_ERROR,
                 containsString("Data too large, data for [test] would be larger than limit of [10/10b]"));
 
+        reset();
+
         // Adjust settings so the parent breaker will fail, but neither the fielddata breaker nor the node request breaker will fail
-        // There is no "one size fits all" breaker size as internal request size will vary based on doc count.
-        int parentBreakerSize = docCount * 3;
         resetSettings = Settings.builder()
-                .put(HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), parentBreakerSize + "b")
+                .put(HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "500b")
                 .put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "90%")
                 .put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey(), 1.0)
                 .build();
@@ -267,9 +266,16 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase {
             client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC).get();
             fail("should have thrown an exception");
         } catch (Exception e) {
-            String errMsg = "[parent] Data too large, data for [test] would be larger than limit of [" + parentBreakerSize;
-            assertThat("Exception: [" + e.toString() + "] should contain a CircuitBreakingException",
-                    e.toString(), containsString(errMsg));
+            final Throwable cause = ExceptionsHelper.unwrap(e, CircuitBreakingException.class);
+            assertNotNull("CircuitBreakingException is not the cause of " + e, cause);
+            String errMsg = "would be larger than limit of [500/500b]]";
+            assertThat("Exception: [" + cause.toString() + "] should contain a CircuitBreakingException",
+                cause.toString(), startsWith("CircuitBreakingException[[parent] Data too large"));
+            assertThat("Exception: [" + cause.toString() + "] should contain a CircuitBreakingException",
+                cause.toString(), endsWith(errMsg));
+        } finally {
+            // reset before teardown as it requires properly set up breakers
+            reset();
         }
     }
 

From 2e613f49b34c73a74ad9649f781dac19da9132bb Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Tue, 10 May 2016 09:53:48 +0200
Subject: [PATCH 0154/1311] painless: Array load/store and length with
 invokedynamic

---
 .../java/org/elasticsearch/painless/Def.java  | 160 +++++++++++-------
 .../painless/DynamicCallSite.java             |   8 +
 .../painless/WriterConstants.java             |  12 +-
 .../painless/WriterExternal.java              |  12 +-
 .../elasticsearch/painless/ArrayTests.java    |  62 +++++++
 .../elasticsearch/painless/BasicAPITests.java |   6 -
 6 files changed, 179 insertions(+), 81 deletions(-)
 create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 3623368c21e..5b4dcc01bdc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -26,9 +26,12 @@ import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodType;
 import java.lang.invoke.MethodHandles.Lookup;
-import java.lang.reflect.Array;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * Support for dynamic type (def).
@@ -36,18 +39,13 @@ import java.util.Map;
  * Dynamic types can invoke methods, load/store fields, and be passed as parameters to operators without 
  * compile-time type information. 
  * <p>
- * Dynamic methods, loads, and stores involve locating the appropriate field or method depending
- * on the receiver's class. For these, we emit an {@code invokedynamic} instruction that, for each new 
- * type encountered will query a corresponding {@code lookupXXX} method to retrieve the appropriate method.
- * In most cases, the {@code lookupXXX} methods here will only be called once for a given call site, because 
+ * Dynamic methods, loads, stores, and array/list/map load/stores involve locating the appropriate field
+ * or method depending on the receiver's class. For these, we emit an {@code invokedynamic} instruction that,
+ * for each new  type encountered will query a corresponding {@code lookupXXX} method to retrieve the appropriate
+ * method. In most cases, the {@code lookupXXX} methods here will only be called once for a given call site, because 
  * caching ({@link DynamicCallSite}) generally works: usually all objects at any call site will be consistently 
  * the same type (or just a few types).  In extreme cases, if there is type explosion, they may be called every 
  * single time, but simplicity is still more valuable than performance in this code.
- * <p>
- * Dynamic array loads and stores and operator functions (e.g. {@code +}) are called directly
- * with {@code invokestatic}. Because these features cannot be overloaded in painless, they are hardcoded 
- * decision trees based on the only types that are possible. This keeps overhead low, and seems to be as fast
- * on average as the more adaptive methodhandle caching. 
  */
 public class Def {
 
@@ -96,8 +94,6 @@ public class Def {
                                            "for class [" + receiverClass.getCanonicalName() + "].");
     }
 
-    /** pointer to Array.getLength(Object) */
-    private static final MethodHandle ARRAY_LENGTH;
     /** pointer to Map.get(Object) */
     private static final MethodHandle MAP_GET;
     /** pointer to Map.put(Object,Object) */
@@ -109,9 +105,6 @@ public class Def {
     static {
         Lookup lookup = MethodHandles.publicLookup();
         try {
-            // TODO: maybe specialize handles for different array types. this may be slower, but simple :)
-            ARRAY_LENGTH = lookup.findStatic(Array.class, "getLength",
-                                             MethodType.methodType(int.class, Object.class));
             MAP_GET      = lookup.findVirtual(Map.class, "get",
                                              MethodType.methodType(Object.class, Object.class));
             MAP_PUT      = lookup.findVirtual(Map.class, "put",
@@ -125,6 +118,54 @@ public class Def {
         }
     }
     
+    // TODO: Once Java has a factory for those in java.lang.invoke.MethodHandles, use it:
+
+    /** Helper class for isolating MethodHandles and methods to get the length of arrays
+     * (to emulate a "arraystore" byteoode using MethodHandles).
+     * This should really be a method in {@link MethodHandles} class!
+     */
+    private static final class ArrayLengthHelper {
+      private ArrayLengthHelper() {}
+    
+      private static final Lookup PRIV_LOOKUP = MethodHandles.lookup();
+      private static final Map<Class<?>,MethodHandle> ARRAY_TYPE_MH_MAPPING = Collections.unmodifiableMap(
+        Stream.of(boolean[].class, byte[].class, short[].class, int[].class, long[].class,
+        char[].class, float[].class, double[].class, Object[].class)
+          .collect(Collectors.toMap(Function.identity(), type -> {
+            try {
+              return PRIV_LOOKUP.findStatic(PRIV_LOOKUP.lookupClass(), "getArrayLength", MethodType.methodType(int.class, type));
+            } catch (ReflectiveOperationException e) {
+              throw new AssertionError(e);
+            }
+          }))
+      );
+      private static final MethodHandle OBJECT_ARRAY_MH = ARRAY_TYPE_MH_MAPPING.get(Object[].class);
+     
+      static int getArrayLength(boolean[] array) { return array.length; }
+      static int getArrayLength(byte[] array) { return array.length; }
+      static int getArrayLength(short[] array) { return array.length; }
+      static int getArrayLength(int[] array) { return array.length; }
+      static int getArrayLength(long[] array) { return array.length; }
+      static int getArrayLength(char[] array) { return array.length; }
+      static int getArrayLength(float[] array) { return array.length; }
+      static int getArrayLength(double[] array) { return array.length; }
+      static int getArrayLength(Object[] array) { return array.length; }
+      
+      public static MethodHandle arrayLengthGetter(Class<?> arrayType) {
+        if (!arrayType.isArray()) {
+          throw new IllegalArgumentException("type must be an array");
+        }
+        return (ARRAY_TYPE_MH_MAPPING.containsKey(arrayType)) ?
+            ARRAY_TYPE_MH_MAPPING.get(arrayType) :
+            OBJECT_ARRAY_MH.asType(OBJECT_ARRAY_MH.type().changeParameterType(0, arrayType));
+      }
+    }
+    
+    /** Returns an array length getter MethodHandle for the given array type */
+    public static MethodHandle arrayLengthGetter(Class<?> arrayType) {
+      return ArrayLengthHelper.arrayLengthGetter(arrayType);
+    }
+    
     /** 
      * Looks up handle for a dynamic field getter (field load)
      * <p>
@@ -177,7 +218,7 @@ public class Def {
         // special case: arrays, maps, and lists
         if (receiverClass.isArray() && "length".equals(name)) {
             // arrays expose .length as a read-only getter
-            return ARRAY_LENGTH;
+            return arrayLengthGetter(receiverClass);
         } else if (Map.class.isAssignableFrom(receiverClass)) {
             // maps allow access like mymap.key
             // wire 'key' as a parameter, its a constant in painless
@@ -266,56 +307,45 @@ public class Def {
                                            "for class [" + receiverClass.getCanonicalName() + "].");
     }
 
-    // NOTE: below methods are not cached, instead invoked directly because they are performant.
+    /**
+     * Returns a method handle to do an array store.
+     * @param receiverClass Class of the array to store the value in
+     * @return a MethodHandle that accepts the receiver as first argument, the index as second argument,
+     *   and the value to set as 3rd argument. Return value is undefined and should be ignored.
+     */
+    static MethodHandle lookupArrayStore(Class<?> receiverClass) {
+        if (receiverClass.isArray()) {
+            return MethodHandles.arrayElementSetter(receiverClass);
+        } else if (Map.class.isAssignableFrom(receiverClass)) {
+            // maps allow access like mymap[key]
+            return MAP_PUT;
+        } else if (List.class.isAssignableFrom(receiverClass)) {
+            return LIST_SET;
+        }
+        throw new IllegalArgumentException("Attempting to address a non-array type " +
+                                           "[" + receiverClass.getCanonicalName() + "] as an array.");
+    }
+   
+    /**
+     * Returns a method handle to do an array load.
+     * @param receiverClass Class of the array to load the value from
+     * @return a MethodHandle that accepts the receiver as first argument, the index as second argument.
+     *   It returns the loaded value.
+     */
+    static MethodHandle lookupArrayLoad(Class<?> receiverClass) {
+        if (receiverClass.isArray()) {
+            return MethodHandles.arrayElementGetter(receiverClass);
+        } else if (Map.class.isAssignableFrom(receiverClass)) {
+            // maps allow access like mymap[key]
+            return MAP_GET;
+        } else if (List.class.isAssignableFrom(receiverClass)) {
+            return LIST_GET;
+        }
+        throw new IllegalArgumentException("Attempting to address a non-array type " +
+                                           "[" + receiverClass.getCanonicalName() + "] as an array.");
+    }
 
-    /**
-     * Performs an actual array store.
-     * @param array array object
-     * @param index map key, array index (integer), or list index (integer)
-     * @param value value to store in the array.
-     */
-    @SuppressWarnings({ "unchecked", "rawtypes" })
-    public static void arrayStore(final Object array, Object index, Object value) {
-        if (array instanceof Map) {
-            ((Map)array).put(index, value);
-        } else if (array.getClass().isArray()) {
-            try {
-                Array.set(array, (int)index, value);
-            } catch (final Throwable throwable) {
-                throw new IllegalArgumentException("Error storing value [" + value + "] " +
-                                                   "in array class [" + array.getClass().getCanonicalName() + "].", throwable);
-            }
-        } else if (array instanceof List) {
-            ((List)array).set((int)index, value);
-        } else {
-            throw new IllegalArgumentException("Attempting to address a non-array type " +
-                                               "[" + array.getClass().getCanonicalName() + "] as an array.");
-        }
-    }
-    
-    /**
-     * Performs an actual array load.
-     * @param array array object
-     * @param index map key, array index (integer), or list index (integer)
-     */
-    @SuppressWarnings("rawtypes")
-    public static Object arrayLoad(final Object array, Object index) {
-        if (array instanceof Map) {
-            return ((Map)array).get(index);
-        } else if (array.getClass().isArray()) {
-            try {
-                return Array.get(array, (int)index);
-            } catch (final Throwable throwable) {
-                throw new IllegalArgumentException("Error loading value from " +
-                                                   "array class [" + array.getClass().getCanonicalName() + "].", throwable);
-            }
-        } else if (array instanceof List) {
-            return ((List)array).get((int)index);
-        } else {
-            throw new IllegalArgumentException("Attempting to address a non-array type " +
-                                               "[" + array.getClass().getCanonicalName() + "] as an array.");
-        }
-    }
+    // NOTE: below methods are not cached, instead invoked directly because they are performant.
 
     public static Object not(final Object unary) {
         if (unary instanceof Double || unary instanceof Float || unary instanceof Long) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DynamicCallSite.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DynamicCallSite.java
index b0870c60219..8222465c0ad 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DynamicCallSite.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DynamicCallSite.java
@@ -48,6 +48,10 @@ public final class DynamicCallSite {
     static final int LOAD = 1;
     /** static bootstrap parameter indicating a dynamic store (setter), e.g. foo.bar = baz */
     static final int STORE = 2;
+    /** static bootstrap parameter indicating a dynamic array load, e.g. baz = foo[bar] */
+    static final int ARRAY_LOAD = 3;
+    /** static bootstrap parameter indicating a dynamic array store, e.g. foo[bar] = baz */
+    static final int ARRAY_STORE = 4;
     
     static class InliningCacheCallSite extends MutableCallSite {
         /** maximum number of types before we go megamorphic */
@@ -104,6 +108,10 @@ public final class DynamicCallSite {
                 return Def.lookupGetter(clazz, name, Definition.INSTANCE);
             case STORE:
                 return Def.lookupSetter(clazz, name, Definition.INSTANCE);
+            case ARRAY_LOAD: 
+                return Def.lookupArrayLoad(clazz);
+            case ARRAY_STORE:
+                return Def.lookupArrayStore(clazz);
             default: throw new AssertionError();
         }
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index a1a594b354e..115f118adf6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -58,10 +58,14 @@ class WriterConstants {
     final static Handle DEF_BOOTSTRAP_HANDLE = new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(DynamicCallSite.class),
                                                           "bootstrap", WriterConstants.DEF_BOOTSTRAP_TYPE.toMethodDescriptorString());
 
-    final static Method DEF_ARRAY_STORE = getAsmMethod(
-        void.class, "arrayStore", Object.class, Object.class, Object.class);
-    final static Method DEF_ARRAY_LOAD = getAsmMethod(
-        Object.class, "arrayLoad", Object.class, Object.class);
+    final static String DEF_DYNAMIC_LOAD_FIELD_DESC = MethodType.methodType(Object.class, Object.class)
+        .toMethodDescriptorString();
+    final static String DEF_DYNAMIC_STORE_FIELD_DESC = MethodType.methodType(void.class, Object.class, Object.class)
+        .toMethodDescriptorString();
+    final static String DEF_DYNAMIC_ARRAY_LOAD_DESC = MethodType.methodType(Object.class, Object.class, Object.class)
+        .toMethodDescriptorString();
+    final static String DEF_DYNAMIC_ARRAY_STORE_DESC = MethodType.methodType(void.class, Object.class, Object.class, Object.class)
+        .toMethodDescriptorString();
 
     final static Method DEF_NOT_CALL = getAsmMethod(Object.class, "not", Object.class);
     final static Method DEF_NEG_CALL = getAsmMethod(Object.class, "neg", Object.class);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java
index ab05e1b6e56..6f3800a27ac 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java
@@ -49,8 +49,6 @@ import static org.elasticsearch.painless.PainlessParser.DIV;
 import static org.elasticsearch.painless.PainlessParser.MUL;
 import static org.elasticsearch.painless.PainlessParser.REM;
 import static org.elasticsearch.painless.PainlessParser.SUB;
-import static org.elasticsearch.painless.WriterConstants.DEF_ARRAY_LOAD;
-import static org.elasticsearch.painless.WriterConstants.DEF_ARRAY_STORE;
 import static org.elasticsearch.painless.WriterConstants.TOBYTEEXACT_INT;
 import static org.elasticsearch.painless.WriterConstants.TOBYTEEXACT_LONG;
 import static org.elasticsearch.painless.WriterConstants.TOBYTEWOOVERFLOW_DOUBLE;
@@ -468,10 +466,10 @@ class WriterExternal {
 
     private void writeLoadStoreField(final ParserRuleContext source, final boolean store, final String name) {
         if (store) {
-            execute.visitInvokeDynamicInsn(name, "(Ljava/lang/Object;Ljava/lang/Object;)V", 
+            execute.visitInvokeDynamicInsn(name, WriterConstants.DEF_DYNAMIC_STORE_FIELD_DESC, 
                                            WriterConstants.DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.STORE });
         } else {
-            execute.visitInvokeDynamicInsn(name, "(Ljava/lang/Object;)Ljava/lang/Object;", 
+            execute.visitInvokeDynamicInsn(name, WriterConstants.DEF_DYNAMIC_LOAD_FIELD_DESC, 
                                            WriterConstants.DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.LOAD });
         }
     }
@@ -483,9 +481,11 @@ class WriterExternal {
 
         if (type.sort == Sort.DEF) {
             if (store) {
-                execute.invokeStatic(definition.defobjType.type, DEF_ARRAY_STORE);
+                execute.visitInvokeDynamicInsn("arrayStore", WriterConstants.DEF_DYNAMIC_ARRAY_STORE_DESC, 
+                                           WriterConstants.DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.ARRAY_STORE });
             } else {
-                execute.invokeStatic(definition.defobjType.type, DEF_ARRAY_LOAD);
+                execute.visitInvokeDynamicInsn("arrayLoad", WriterConstants.DEF_DYNAMIC_ARRAY_LOAD_DESC, 
+                                           WriterConstants.DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.ARRAY_LOAD });
             }
         } else {
             if (store) {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
new file mode 100644
index 00000000000..51ee659e501
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+/** Tests for or operator across all types */
+public class ArrayTests extends ScriptTestCase {
+
+    public void testArrayLengthHelper() throws Throwable {
+        assertArrayLength(2, new int[2]);
+        assertArrayLength(3, new long[3]);
+        assertArrayLength(4, new byte[4]);
+        assertArrayLength(5, new float[5]);
+        assertArrayLength(6, new double[6]);
+        assertArrayLength(7, new char[7]);
+        assertArrayLength(8, new short[8]);
+        assertArrayLength(9, new Object[9]);
+        assertArrayLength(10, new Integer[10]);
+        assertArrayLength(11, new String[11][2]);
+    }
+    
+    private void assertArrayLength(int length, Object array) throws Throwable {
+        assertEquals(length, (int) Def.arrayLengthGetter(array.getClass()).invoke(array));
+    }
+
+    public void testArrayLoadStoreInt() {
+        assertEquals(5, exec("def x = new int[5]; return x.length"));
+        assertEquals(5, exec("def x = new int[4]; x[0] = 5; return x[0];"));
+    }
+    
+    public void testArrayLoadStoreString() {
+        assertEquals(5, exec("def x = new String[5]; return x.length"));
+        assertEquals("foobar", exec("def x = new String[4]; x[0] = 'foobar'; return x[0];"));
+    }
+    
+    public void testArrayLoadStoreDef() {
+        assertEquals(5, exec("def x = new def[5]; return x.length"));
+        assertEquals(5, exec("def x = new def[4]; x[0] = 5; return x[0];"));
+    }
+    
+    public void testForLoop() {
+        assertEquals(999*1000/2, exec("def a = new int[1000]; for (int x = 0; x < a.length; x++) { a[x] = x; } "+
+            "int total = 0; for (int x = 0; x < a.length; x++) { total += a[x]; } return total;"));
+    }
+    
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
index 46fc6768c2f..2a14dad375a 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
@@ -62,12 +62,6 @@ public class BasicAPITests extends ScriptTestCase {
         assertEquals(5, exec("def x = new ArrayList(); x.add(3); x[0] = 5; return x[0];"));
     }
     
-    /** Test loads and stores with a list */
-    public void testArrayLoadStore() {
-        assertEquals(5, exec("def x = new int[5]; return x.length"));
-        assertEquals(5, exec("def x = new int[4]; x[0] = 5; return x[0];"));
-    }
-    
     /** Test shortcut for getters with isXXXX */
     public void testListEmpty() {
         assertEquals(true, exec("def x = new ArrayList(); return x.empty;"));

From a865090cf33e88ceef39ea164c37b07b884343ae Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Tue, 10 May 2016 12:42:17 +0200
Subject: [PATCH 0155/1311] CONSOLE is the new AUTOSENSE

---
 .../query-dsl/constant-score-query.asciidoc   |  14 +-
 .../query-dsl/function-score-query.asciidoc   | 102 ++++++------
 .../query-dsl/geo-distance-query.asciidoc     | 110 ++++++-------
 .../query-dsl/match-all-query.asciidoc        |  27 ++--
 docs/reference/query-dsl/mlt-query.asciidoc   | 146 +++++++++---------
 .../query-dsl/multi-match-query.asciidoc      | 132 ++++++++--------
 .../reference/query-dsl/prefix-query.asciidoc |   6 +-
 .../query-dsl/query_filter_context.asciidoc   |   2 +-
 docs/reference/query-dsl/range-query.asciidoc |  10 +-
 .../simple-query-string-query.asciidoc        |  20 +--
 .../query-dsl/span-first-query.asciidoc       |  16 +-
 .../query-dsl/span-or-query.asciidoc          |  16 +-
 .../query-dsl/template-query.asciidoc         |  12 +-
 docs/reference/query-dsl/type-query.asciidoc  |  10 +-
 14 files changed, 313 insertions(+), 310 deletions(-)

diff --git a/docs/reference/query-dsl/constant-score-query.asciidoc b/docs/reference/query-dsl/constant-score-query.asciidoc
index 0928e9a5718..5002efcdabe 100644
--- a/docs/reference/query-dsl/constant-score-query.asciidoc
+++ b/docs/reference/query-dsl/constant-score-query.asciidoc
@@ -9,14 +9,14 @@ filter. Maps to Lucene `ConstantScoreQuery`.
 --------------------------------------------------
 GET /_search
 {
-  "query": {
-    "constant_score" : {
-        "filter" : {
-            "term" : { "user" : "kimchy"}
-        },
-        "boost" : 1.2
+    "query": {
+        "constant_score" : {
+            "filter" : {
+                "term" : { "user" : "kimchy"}
+            },
+                "boost" : 1.2
+        }
     }
-  }
 }
 --------------------------------------------------
 // CONSOLE
diff --git a/docs/reference/query-dsl/function-score-query.asciidoc b/docs/reference/query-dsl/function-score-query.asciidoc
index e48a396e6c8..e7e4b8f5877 100644
--- a/docs/reference/query-dsl/function-score-query.asciidoc
+++ b/docs/reference/query-dsl/function-score-query.asciidoc
@@ -16,14 +16,14 @@ by the query.
 --------------------------------------------------
 GET /_search
 {
-  "query": {
-    "function_score": {
-        "query": {},
-        "boost": "5",
-        "random_score": {}, <1>
-        "boost_mode":"multiply"
+    "query": {
+        "function_score": {
+            "query": {},
+            "boost": "5",
+            "random_score": {}, <1>
+            "boost_mode":"multiply"
+        }
     }
-  }
 }
 --------------------------------------------------
 // CONSOLE
@@ -38,27 +38,27 @@ given filtering query
 --------------------------------------------------
 GET /_search
 {
-  "query": {
-    "function_score": {
-      "query": {},
-      "boost": "5", <1>
-      "functions": [
-          {
-              "filter": {},
-              "random_score": {}, <2>
-              "weight": 23
-          },
-          {
-              "filter": {},
-              "weight": 42 
-          }
-      ],
-      "max_boost": 42,
-      "score_mode": "max",
-      "boost_mode": "multiply",
-      "min_score" : 42
+    "query": {
+        "function_score": {
+          "query": {},
+          "boost": "5", <1>
+          "functions": [
+              {
+                  "filter": {},
+                  "random_score": {}, <2>
+                  "weight": 23
+              },
+              {
+                  "filter": {},
+                  "weight": 42 
+              }
+          ],
+          "max_boost": 42,
+          "score_mode": "max",
+          "boost_mode": "multiply",
+          "min_score" : 42
+        }
     }
-  }
 }
 --------------------------------------------------
 // CONSOLE
@@ -473,34 +473,34 @@ the request would look like this:
 --------------------------------------------------
 GET /_search
 {
-  "query": {
-    "function_score": {
-      "functions": [
-        {
-          "gauss": {
-            "price": {
-              "origin": "0",
-              "scale": "20"
+    "query": {
+        "function_score": {
+          "functions": [
+            {
+              "gauss": {
+                "price": {
+                  "origin": "0",
+                  "scale": "20"
+                }
+              }
+            },
+            {
+              "gauss": {
+                "location": {
+                  "origin": "11, 12",
+                  "scale": "2km"
+                }
+              }
             }
-          }
-        },
-        {
-          "gauss": {
-            "location": {
-              "origin": "11, 12",
-              "scale": "2km"
+          ],
+          "query": {
+            "match": {
+              "properties": "balcony"
             }
-          }
+          },
+          "score_mode": "multiply"
         }
-      ],
-      "query": {
-        "match": {
-          "properties": "balcony"
-        }
-      },
-      "score_mode": "multiply"
     }
-  }
 }
 --------------------------------------------------
 // CONSOLE
diff --git a/docs/reference/query-dsl/geo-distance-query.asciidoc b/docs/reference/query-dsl/geo-distance-query.asciidoc
index 228ab804d7a..2abb296c546 100644
--- a/docs/reference/query-dsl/geo-distance-query.asciidoc
+++ b/docs/reference/query-dsl/geo-distance-query.asciidoc
@@ -8,7 +8,7 @@ distance from a geo point. Assuming the following mapping:
 --------------------------------------------------
 PUT /my_locations
 {
-  "mappings": {
+    "mappings": {
         "location": {
             "properties": {
                 "pin": {
@@ -51,22 +51,22 @@ filter:
 --------------------------------------------------
 GET /my_locations/location/_search
 {
-  "query": {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_distance" : {
-                "distance" : "200km",
-                "pin.location" : {
-                    "lat" : 40,
-                    "lon" : -70
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_distance" : {
+                    "distance" : "200km",
+                    "pin.location" : {
+                        "lat" : 40,
+                        "lon" : -70
+                    }
                 }
             }
         }
     }
-  }
 }
 --------------------------------------------------
 // CONSOLE
@@ -85,22 +85,22 @@ representation of the geo point, the filter can accept it as well:
 --------------------------------------------------
 GET /my_locations/location/_search
 {
-  "query": {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_distance" : {
-                "distance" : "12km",
-                "pin.location" : {
-                    "lat" : 40,
-                    "lon" : -70
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_distance" : {
+                    "distance" : "12km",
+                    "pin.location" : {
+                        "lat" : 40,
+                        "lon" : -70
+                    }
                 }
             }
         }
     }
-  }
 }
 --------------------------------------------------
 // CONSOLE
@@ -116,19 +116,19 @@ conform with http://geojson.org/[GeoJSON].
 --------------------------------------------------
 GET /my_locations/location/_search
 {
-  "query": {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_distance" : {
-                "distance" : "12km",
-                "pin.location" : [-70, 40]
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_distance" : {
+                    "distance" : "12km",
+                    "pin.location" : [-70, 40]
+                }
             }
         }
     }
-  }
 }
 --------------------------------------------------
 // CONSOLE
@@ -144,19 +144,19 @@ Format in `lat,lon`.
 --------------------------------------------------
 GET /my_locations/location/_search
 {
-  "query": {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_distance" : {
-                "distance" : "12km",
-                "pin.location" : "40,-70"
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_distance" : {
+                    "distance" : "12km",
+                    "pin.location" : "40,-70"
+                }
             }
         }
     }
-  }
 }
 --------------------------------------------------
 // CONSOLE
@@ -169,19 +169,19 @@ GET /my_locations/location/_search
 --------------------------------------------------
 GET /my_locations/location/_search
 {
-  "query": {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_distance" : {
-                "distance" : "12km",
-                "pin.location" : "drm3btev3e86"
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_distance" : {
+                    "distance" : "12km",
+                    "pin.location" : "drm3btev3e86"
+                }
             }
         }
     }
-  }
 }
 --------------------------------------------------
 // CONSOLE
diff --git a/docs/reference/query-dsl/match-all-query.asciidoc b/docs/reference/query-dsl/match-all-query.asciidoc
index d4064d053f6..6e448828676 100644
--- a/docs/reference/query-dsl/match-all-query.asciidoc
+++ b/docs/reference/query-dsl/match-all-query.asciidoc
@@ -7,24 +7,26 @@ of `1.0`.
 [source,js]
 --------------------------------------------------
 GET /_search
-{ "query": {
-    "match_all": {}
-  }
+{ 
+    "query": {
+        "match_all": {}
+    }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The `_score` can be changed with the `boost` parameter:
 
 [source,js]
 --------------------------------------------------
 GET /_search
-{ "query": {
-    "match_all": { "boost" : 1.2 }
-  }
+{
+    "query": {
+        "match_all": { "boost" : 1.2 }
+    }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 [[query-dsl-match-none-query]]
 [float]
@@ -35,9 +37,10 @@ This is the inverse of the `match_all` query, which matches no documents.
 [source,js]
 --------------------------------------------------
 GET /_search
-{ "query": {
-    "match_none": {} 
-  }
+{
+    "query": {
+        "match_none": {} 
+    }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
diff --git a/docs/reference/query-dsl/mlt-query.asciidoc b/docs/reference/query-dsl/mlt-query.asciidoc
index 795bf51584d..b132b49f234 100644
--- a/docs/reference/query-dsl/mlt-query.asciidoc
+++ b/docs/reference/query-dsl/mlt-query.asciidoc
@@ -17,17 +17,17 @@ fields, limiting the number of selected terms to 12.
 --------------------------------------------------
 GET /_search
 {
-  "query": {
-    "more_like_this" : {
-        "fields" : ["title", "description"],
-        "like" : "Once upon a time",
-        "min_term_freq" : 1,
-        "max_query_terms" : 12
+    "query": {
+        "more_like_this" : {
+            "fields" : ["title", "description"],
+            "like" : "Once upon a time",
+            "min_term_freq" : 1,
+            "max_query_terms" : 12
+        }
     }
-  }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 A more complicated use case consists of mixing texts with documents already
 existing in the index. In this case, the syntax to specify a document is
@@ -37,29 +37,29 @@ similar to the one used in the <<docs-multi-get,Multi GET API>>.
 --------------------------------------------------
 GET /_search
 {
-  "query": {
-    "more_like_this" : {
-        "fields" : ["title", "description"],
-        "like" : [
-        {
-            "_index" : "imdb",
-            "_type" : "movies",
-            "_id" : "1"
-        },
-        {
-            "_index" : "imdb",
-            "_type" : "movies",
-            "_id" : "2"
-        },
-        "and potentially some more text here as well"
-        ],
-        "min_term_freq" : 1,
-        "max_query_terms" : 12
+    "query": {
+        "more_like_this" : {
+            "fields" : ["title", "description"],
+            "like" : [
+            {
+                "_index" : "imdb",
+                "_type" : "movies",
+                "_id" : "1"
+            },
+            {
+                "_index" : "imdb",
+                "_type" : "movies",
+                "_id" : "2"
+            },
+            "and potentially some more text here as well"
+            ],
+            "min_term_freq" : 1,
+            "max_query_terms" : 12
+        }
     }
-  }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Finally, users can mix some texts, a chosen set of documents but also provide
 documents not necessarily present in the index. To provide documents not
@@ -69,34 +69,34 @@ present in the index, the syntax is similar to <<docs-termvectors-artificial-doc
 --------------------------------------------------
 GET /_search
 {
-  "query": {
-    "more_like_this" : {
-        "fields" : ["name.first", "name.last"],
-        "like" : [
-        {
-            "_index" : "marvel",
-            "_type" : "quotes",
-            "doc" : {
-                "name": {
-                    "first": "Ben",
-                    "last": "Grimm"
-                },
-                "tweet": "You got no idea what I'd... what I'd give to be invisible."
-              }
-        },
-        {
-            "_index" : "marvel",
-            "_type" : "quotes",
-            "_id" : "2"
+    "query": {
+        "more_like_this" : {
+            "fields" : ["name.first", "name.last"],
+            "like" : [
+            {
+                "_index" : "marvel",
+                "_type" : "quotes",
+                "doc" : {
+                    "name": {
+                        "first": "Ben",
+                        "last": "Grimm"
+                    },
+                    "tweet": "You got no idea what I'd... what I'd give to be invisible."
+                  }
+            },
+            {
+                "_index" : "marvel",
+                "_type" : "quotes",
+                "_id" : "2"
+            }
+            ],
+            "min_term_freq" : 1,
+            "max_query_terms" : 12
         }
-        ],
-        "min_term_freq" : 1,
-        "max_query_terms" : 12
     }
-  }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 ==== How it Works
 
@@ -125,32 +125,32 @@ default, but there will be no speed up on analysis for these fields.
 --------------------------------------------------
 PUT /imdb
 {
-  "mappings": {
-    "movies": {
-      "properties": {
-        "title": {
-          "type": "text",
-          "term_vector": "yes"
-         },
-         "description": {
-          "type": "text"
-        },
-        "tags": {
-          "type": "text",
-          "fields" : {
-            "raw": {
-              "type" : "text",
-              "analyzer": "keyword",
-              "term_vector" : "yes"
+    "mappings": {
+        "movies": {
+            "properties": {
+                "title": {
+                    "type": "text",
+                    "term_vector": "yes"
+                },
+                "description": {
+                    "type": "text"
+                },
+                "tags": {
+                    "type": "text",
+                    "fields" : {
+                        "raw": {
+                            "type" : "text",
+                            "analyzer": "keyword",
+                            "term_vector" : "yes"
+                        }
+                    }
+                }
             }
-          }
         }
-      }
     }
-  }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 ==== Parameters
 
diff --git a/docs/reference/query-dsl/multi-match-query.asciidoc b/docs/reference/query-dsl/multi-match-query.asciidoc
index 02f38ebf3a1..3720716a16f 100644
--- a/docs/reference/query-dsl/multi-match-query.asciidoc
+++ b/docs/reference/query-dsl/multi-match-query.asciidoc
@@ -16,7 +16,7 @@ GET /_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The query string.
 <2> The fields to be queried.
 
@@ -37,7 +37,7 @@ GET /_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> Query the `title`, `first_name` and `last_name` fields.
 
 Individual fields can be boosted with the caret (`^`) notation:
@@ -54,7 +54,7 @@ GET /_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> The `subject` field is three times as important as the `message` field.
 
@@ -98,16 +98,16 @@ find the single best matching field.  For instance, this query:
 GET /_search
 {
   "query": {
-  "multi_match" : {
-    "query":      "brown fox",
-    "type":       "best_fields",
-    "fields":     [ "subject", "message" ],
-    "tie_breaker": 0.3
-  }
+    "multi_match" : {
+      "query":      "brown fox",
+      "type":       "best_fields",
+      "fields":     [ "subject", "message" ],
+      "tie_breaker": 0.3
+    }
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 would be executed as:
 
@@ -126,7 +126,7 @@ GET /_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Normally the `best_fields` type uses the score of the *single* best matching
 field, but if `tie_breaker` is specified, then it calculates the score as
@@ -165,7 +165,7 @@ GET /_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> All terms must be present.
 
@@ -207,7 +207,7 @@ GET /_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 would be executed as:
 
@@ -226,7 +226,7 @@ GET /_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The score from each `match` clause is added together, then divided by the
 number of `match` clauses.
@@ -249,15 +249,15 @@ This query:
 GET /_search
 {
   "query": {
-  "multi_match" : {
-    "query":      "quick brown f",
-    "type":       "phrase_prefix",
-    "fields":     [ "subject", "message" ]
-  }
+    "multi_match" : {
+      "query":      "quick brown f",
+      "type":       "phrase_prefix",
+      "fields":     [ "subject", "message" ]
+    }
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 would be executed as:
 
@@ -266,16 +266,16 @@ would be executed as:
 GET /_search
 {
   "query": {
-  "dis_max": {
-    "queries": [
-      { "match_phrase_prefix": { "subject": "quick brown f" }},
-      { "match_phrase_prefix": { "message": "quick brown f" }}
-    ]
-  }
+    "dis_max": {
+      "queries": [
+        { "match_phrase_prefix": { "subject": "quick brown f" }},
+        { "match_phrase_prefix": { "message": "quick brown f" }}
+      ]
+    }
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Also, accepts `analyzer`, `boost`, `slop` and `zero_terms_query`  as explained
 in <<query-dsl-match-query>>.  Type `phrase_prefix` additionally accepts
@@ -326,16 +326,16 @@ A query like:
 GET /_search
 {
   "query": {
-  "multi_match" : {
-    "query":      "Will Smith",
-    "type":       "cross_fields",
-    "fields":     [ "first_name", "last_name" ],
-    "operator":   "and"
-  }
+    "multi_match" : {
+      "query":      "Will Smith",
+      "type":       "cross_fields",
+      "fields":     [ "first_name", "last_name" ],
+      "operator":   "and"
+    }
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 is executed as:
 
@@ -386,18 +386,18 @@ both use an `edge_ngram` analyzer, this query:
 GET /_search
 {
   "query": {
-  "multi_match" : {
-    "query":      "Jon",
-    "type":       "cross_fields",
-    "fields":     [
+    "multi_match" : {
+      "query":      "Jon",
+      "type":       "cross_fields",
+      "fields":     [
         "first", "first.edge",
         "last",  "last.edge"
-    ]
-  }
+      ]
+    }
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 would be executed as:
 
@@ -426,28 +426,28 @@ GET /_search
 {
   "query": {
     "bool": {
-        "should": [
-            {
-              "multi_match" : {
-                "query":      "Will Smith",
-                "type":       "cross_fields",
-                "fields":     [ "first", "last" ],
-                "minimum_should_match": "50%" <1>
-              }
-            },
-            {
-              "multi_match" : {
-                "query":      "Will Smith",
-                "type":       "cross_fields",
-                "fields":     [ "*.edge" ]
-              }
-            }
-        ]
+      "should": [
+        {
+          "multi_match" : {
+            "query":      "Will Smith",
+            "type":       "cross_fields",
+            "fields":     [ "first", "last" ],
+            "minimum_should_match": "50%" <1>
+          }
+        },
+        {
+          "multi_match" : {
+            "query":      "Will Smith",
+            "type":       "cross_fields",
+            "fields":     [ "*.edge" ]
+          }
+        }
+      ]
     }
   }
 }
 --------------------------------------------------
-// AUTOSENSE 
+// CONSOLE 
 
 <1> Either `will` or `smith` must be present in either of the `first`
     or `last` fields
@@ -460,16 +460,16 @@ parameter in the query.
 GET /_search
 {
   "query": {
-  "multi_match" : {
-    "query":      "Jon",
-    "type":       "cross_fields",
-    "analyzer":   "standard", <1>
-    "fields":     [ "first", "last", "*.edge" ]
-  }
+   "multi_match" : {
+      "query":      "Jon",
+      "type":       "cross_fields",
+      "analyzer":   "standard", <1>
+      "fields":     [ "first", "last", "*.edge" ]
+    }
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> Use the `standard` analyzer for all fields.
 
diff --git a/docs/reference/query-dsl/prefix-query.asciidoc b/docs/reference/query-dsl/prefix-query.asciidoc
index 543cccfeef5..d2b75d10e5f 100644
--- a/docs/reference/query-dsl/prefix-query.asciidoc
+++ b/docs/reference/query-dsl/prefix-query.asciidoc
@@ -14,7 +14,7 @@ GET /_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 A boost can also be associated with the query:
 
@@ -26,7 +26,7 @@ GET /_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Or :
 
@@ -38,7 +38,7 @@ GET /_search
   }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 This multi term query allows you to control how it gets rewritten using the
 <<query-dsl-multi-term-rewrite,rewrite>>
diff --git a/docs/reference/query-dsl/query_filter_context.asciidoc b/docs/reference/query-dsl/query_filter_context.asciidoc
index 7ef0d7b0f4f..79f8c4bd960 100644
--- a/docs/reference/query-dsl/query_filter_context.asciidoc
+++ b/docs/reference/query-dsl/query_filter_context.asciidoc
@@ -63,7 +63,7 @@ GET /_search
   }
 }
 ------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> The `query` parameter indicates query context.
 <2> The `bool` and two `match` clauses are used in query context,
     which means that they are used to score how well each document
diff --git a/docs/reference/query-dsl/range-query.asciidoc b/docs/reference/query-dsl/range-query.asciidoc
index 939c300f1a0..67d76953f03 100644
--- a/docs/reference/query-dsl/range-query.asciidoc
+++ b/docs/reference/query-dsl/range-query.asciidoc
@@ -22,7 +22,7 @@ GET _search
     }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE 
 
 The `range` query accepts the following parameters:
 
@@ -54,7 +54,7 @@ GET _search
     }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 ===== Date math and rounding
 
@@ -106,6 +106,7 @@ GET _search
     }
 }
 --------------------------------------------------
+// CONSOLE 
 
 ===== Time zone in range queries
 
@@ -117,8 +118,7 @@ accepts it), or it can be specified as the `time_zone` parameter:
 --------------------------------------------------
 GET _search
 {
-    "query":
-    {
+    "query": {
         "range" : {
             "timestamp" : {
                 "gte": "2015-01-01 00:00:00", <1>
@@ -129,6 +129,6 @@ GET _search
     }
 }
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 <1> This date will be converted to `2014-12-31T23:00:00 UTC`.
 <2> `now` is not affected by the `time_zone` parameter (dates must be stored as UTC).
diff --git a/docs/reference/query-dsl/simple-query-string-query.asciidoc b/docs/reference/query-dsl/simple-query-string-query.asciidoc
index 72093a53f6e..796f2517fea 100644
--- a/docs/reference/query-dsl/simple-query-string-query.asciidoc
+++ b/docs/reference/query-dsl/simple-query-string-query.asciidoc
@@ -100,12 +100,12 @@ introduced fields included). For example:
 --------------------------------------------------
 GET /_search
 {
-  "query": {
-    "simple_query_string" : {
-        "fields" : ["content", "name.*^5"],
-        "query" : "foo bar baz"
+    "query": {
+        "simple_query_string" : {
+            "fields" : ["content", "name.*^5"],
+            "query" : "foo bar baz"
+        }
     }
-  }
 }
 --------------------------------------------------
 // CONSOLE
@@ -120,12 +120,12 @@ should be enabled. It is specified as a `|`-delimited string with the
 --------------------------------------------------
 GET /_search
 {
-  "query": {
-    "simple_query_string" : {
-        "query" : "foo | bar + baz*",
-        "flags" : "OR|AND|PREFIX"
+    "query": {
+        "simple_query_string" : {
+            "query" : "foo | bar + baz*",
+            "flags" : "OR|AND|PREFIX"
+        }
     }
-  }
 }
 --------------------------------------------------
 // CONSOLE
diff --git a/docs/reference/query-dsl/span-first-query.asciidoc b/docs/reference/query-dsl/span-first-query.asciidoc
index 36a6a2edd9f..dba7932661d 100644
--- a/docs/reference/query-dsl/span-first-query.asciidoc
+++ b/docs/reference/query-dsl/span-first-query.asciidoc
@@ -8,17 +8,17 @@ to Lucene `SpanFirstQuery`. Here is an example:
 --------------------------------------------------
 GET /_search
 {
-  "query": {
-    "span_first" : {
-        "match" : {
-            "span_term" : { "user" : "kimchy" }
-        },
-        "end" : 3
+    "query": {
+        "span_first" : {
+            "match" : {
+                "span_term" : { "user" : "kimchy" }
+            },
+            "end" : 3
+        }
     }
-  }
 }    
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The `match` clause can be any other span type query. The `end` controls
 the maximum end position permitted in a match.
diff --git a/docs/reference/query-dsl/span-or-query.asciidoc b/docs/reference/query-dsl/span-or-query.asciidoc
index 487dcffb6b4..470935d6f5c 100644
--- a/docs/reference/query-dsl/span-or-query.asciidoc
+++ b/docs/reference/query-dsl/span-or-query.asciidoc
@@ -8,15 +8,15 @@ Matches the union of its span clauses. The span or query maps to Lucene
 --------------------------------------------------
 GET /_search
 {
-  "query": {
-    "span_or" : {
-        "clauses" : [
-            { "span_term" : { "field" : "value1" } },
-            { "span_term" : { "field" : "value2" } },
-            { "span_term" : { "field" : "value3" } }
-        ]
+    "query": {
+        "span_or" : {
+            "clauses" : [
+                { "span_term" : { "field" : "value1" } },
+                { "span_term" : { "field" : "value2" } },
+                { "span_term" : { "field" : "value3" } }
+            ]
+        }
     }
-  }
 }
 --------------------------------------------------
 // CONSOLE
diff --git a/docs/reference/query-dsl/template-query.asciidoc b/docs/reference/query-dsl/template-query.asciidoc
index ed2432a3479..f66dbe56d3e 100644
--- a/docs/reference/query-dsl/template-query.asciidoc
+++ b/docs/reference/query-dsl/template-query.asciidoc
@@ -20,7 +20,7 @@ GET /_search
     }
 }
 ------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 The above request is translated into:
 
@@ -35,7 +35,7 @@ GET /_search
     }
 }
 ------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 Alternatively passing the template as an escaped string works as well:
 
@@ -53,7 +53,7 @@ GET /_search
     }
 }
 ------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> New line characters (`\n`) should be escaped as `\\n` or removed,
     and quotes (`"`) should be escaped as `\\"`.
@@ -79,7 +79,7 @@ GET /_search
     }
 }
 ------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 <1> Name of the query template in `config/scripts/`, i.e., `my_template.mustache`.
 
@@ -92,7 +92,7 @@ PUT /_search/template/my_template
     "template": { "match": { "text": "{{query_string}}" }}
 }
 ------------------------------------------
-// AUTOSENSE
+// CONSOLE
 
 and refer to it in the `template` query with the `id` parameter:
 
@@ -111,7 +111,7 @@ GET /_search
     }
 }
 ------------------------------------------
-// AUTOSENSE
+// CONSOLE
 // TEST[continued]
 
 <1> Name of the query template in `config/scripts/`, i.e., `my_template.mustache`.
diff --git a/docs/reference/query-dsl/type-query.asciidoc b/docs/reference/query-dsl/type-query.asciidoc
index 94aaa2d2b65..05e909bc366 100644
--- a/docs/reference/query-dsl/type-query.asciidoc
+++ b/docs/reference/query-dsl/type-query.asciidoc
@@ -7,11 +7,11 @@ Filters documents matching the provided document / mapping type.
 --------------------------------------------------
 GET /_search
 {
-  "query": {
-    "type" : {
-        "value" : "my_type"
+    "query": {
+        "type" : {
+            "value" : "my_type"
+        }
     }
-  }
 }    
 --------------------------------------------------
-// AUTOSENSE
+// CONSOLE

From 754a677ff503e6d1d4f8542891da41721a1c6441 Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Tue, 10 May 2016 12:46:00 +0200
Subject: [PATCH 0156/1311] Use TESTSETUP instead of TEST[continued]

---
 .../query-dsl/geo-distance-query.asciidoc       | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

diff --git a/docs/reference/query-dsl/geo-distance-query.asciidoc b/docs/reference/query-dsl/geo-distance-query.asciidoc
index 2abb296c546..5e2f0ce43c7 100644
--- a/docs/reference/query-dsl/geo-distance-query.asciidoc
+++ b/docs/reference/query-dsl/geo-distance-query.asciidoc
@@ -2,7 +2,8 @@
 === Geo Distance Query
 
 Filters documents that include only hits that exists within a specific
-distance from a geo point. Assuming the following mapping:
+distance from a geo point. Assuming the following mapping and indexed
+document:
 
 [source,js]
 --------------------------------------------------
@@ -22,14 +23,7 @@ PUT /my_locations
         }
     }
 }
---------------------------------------------------
-// CONSOLE
 
-... and indexed document:
-
-
-[source,js]
---------------------------------------------------
 PUT /my_locations/location/1
 {
     "pin" : {
@@ -41,7 +35,7 @@ PUT /my_locations/location/1
 }
 --------------------------------------------------
 // CONSOLE
-// TEST[continued]
+// TESTSETUP
 
 
 Then the following simple query can be executed with a `geo_distance`
@@ -70,7 +64,6 @@ GET /my_locations/location/_search
 }
 --------------------------------------------------
 // CONSOLE
-// TEST[continued]
 
 [float]
 ==== Accepted Formats
@@ -104,7 +97,6 @@ GET /my_locations/location/_search
 }
 --------------------------------------------------
 // CONSOLE
-// TEST[continued]
 
 [float]
 ===== Lat Lon As Array
@@ -132,7 +124,6 @@ GET /my_locations/location/_search
 }
 --------------------------------------------------
 // CONSOLE
-// TEST[continued]
 
 
 [float]
@@ -160,7 +151,6 @@ GET /my_locations/location/_search
 }
 --------------------------------------------------
 // CONSOLE
-// TEST[continued]
 
 [float]
 ===== Geohash
@@ -185,7 +175,6 @@ GET /my_locations/location/_search
 }
 --------------------------------------------------
 // CONSOLE
-// TEST[continued]
 
 [float]
 ==== Options

From ab4367c07ea1745aee801204572a95097a9650e2 Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Tue, 10 May 2016 12:59:44 +0200
Subject: [PATCH 0157/1311] Add CONSOLE to span queries.

... and range, and terms...
---
 docs/reference/query-dsl/range-query.asciidoc | 13 +++++----
 .../query-dsl/span-containing-query.asciidoc  | 28 +++++++++++--------
 .../query-dsl/span-multi-term-query.asciidoc  | 20 +++++++++----
 .../query-dsl/span-near-query.asciidoc        | 20 +++++++------
 .../query-dsl/span-not-query.asciidoc         | 28 +++++++++++--------
 .../query-dsl/span-term-query.asciidoc        | 18 ++++++++++--
 .../query-dsl/span-within-query.asciidoc      | 28 +++++++++++--------
 docs/reference/query-dsl/terms-query.asciidoc | 10 +++++--
 8 files changed, 103 insertions(+), 62 deletions(-)

diff --git a/docs/reference/query-dsl/range-query.asciidoc b/docs/reference/query-dsl/range-query.asciidoc
index 67d76953f03..a0005ff3ff2 100644
--- a/docs/reference/query-dsl/range-query.asciidoc
+++ b/docs/reference/query-dsl/range-query.asciidoc
@@ -96,12 +96,13 @@ passing the `format` parameter to the `range` query:
 --------------------------------------------------
 GET _search
 {
-    "query": 
-    "range" : {
-        "born" : {
-            "gte": "01/01/2012",
-            "lte": "2013",
-            "format": "dd/MM/yyyy||yyyy"
+    "query": {
+        "range" : {
+            "born" : {
+                "gte": "01/01/2012",
+                "lte": "2013",
+                "format": "dd/MM/yyyy||yyyy"
+            }
         }
     }
 }
diff --git a/docs/reference/query-dsl/span-containing-query.asciidoc b/docs/reference/query-dsl/span-containing-query.asciidoc
index 965bf855b6f..638c6999233 100644
--- a/docs/reference/query-dsl/span-containing-query.asciidoc
+++ b/docs/reference/query-dsl/span-containing-query.asciidoc
@@ -6,24 +6,28 @@ query maps to Lucene `SpanContainingQuery`. Here is an example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "span_containing" : {
-        "little" : {
-            "span_term" : { "field1" : "foo" }
-        },
-        "big" : {
-            "span_near" : { 
-                "clauses" : [
-                    { "span_term" : { "field1" : "bar" } },
-                    { "span_term" : { "field1" : "baz" } }
-                ],
-                "slop" : 5,
-                "in_order" : true
+    "query": {
+        "span_containing" : {
+            "little" : {
+                "span_term" : { "field1" : "foo" }
+            },
+            "big" : {
+                "span_near" : { 
+                    "clauses" : [
+                        { "span_term" : { "field1" : "bar" } },
+                        { "span_term" : { "field1" : "baz" } }
+                    ],
+                    "slop" : 5,
+                    "in_order" : true
+                }
             }
         }
     }
 }
 --------------------------------------------------
+//  CONSOLE
 
 The `big` and `little` clauses can be any span type query. Matching
 spans from `big` that contain matches from `little` are returned.
diff --git a/docs/reference/query-dsl/span-multi-term-query.asciidoc b/docs/reference/query-dsl/span-multi-term-query.asciidoc
index af3da5cf7dd..3d1672753d2 100644
--- a/docs/reference/query-dsl/span-multi-term-query.asciidoc
+++ b/docs/reference/query-dsl/span-multi-term-query.asciidoc
@@ -7,24 +7,32 @@ it can be nested. Example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "span_multi":{
-        "match":{
-            "prefix" : { "user" :  { "value" : "ki" } }
+    "query": {
+        "span_multi":{
+            "match":{
+                "prefix" : { "user" :  { "value" : "ki" } }
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 A boost can also be associated with the query:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "span_multi":{
-        "match":{
-            "prefix" : { "user" :  { "value" : "ki", "boost" : 1.08 } }
+    "query": {
+        "span_multi":{
+            "match":{
+                "prefix" : { "user" :  { "value" : "ki", "boost" : 1.08 } }
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/query-dsl/span-near-query.asciidoc b/docs/reference/query-dsl/span-near-query.asciidoc
index 258bc3b51ad..e69be783e3d 100644
--- a/docs/reference/query-dsl/span-near-query.asciidoc
+++ b/docs/reference/query-dsl/span-near-query.asciidoc
@@ -8,18 +8,22 @@ matches are required to be in-order. The span near query maps to Lucene
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "span_near" : {
-        "clauses" : [
-            { "span_term" : { "field" : "value1" } },
-            { "span_term" : { "field" : "value2" } },
-            { "span_term" : { "field" : "value3" } }
-        ],
-        "slop" : 12,
-        "in_order" : false
+    "query": {
+        "span_near" : {
+            "clauses" : [
+                { "span_term" : { "field" : "value1" } },
+                { "span_term" : { "field" : "value2" } },
+                { "span_term" : { "field" : "value3" } }
+            ],
+            "slop" : 12,
+            "in_order" : false
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 The `clauses` element is a list of one or more other span type queries
 and the `slop` controls the maximum number of intervening unmatched
diff --git a/docs/reference/query-dsl/span-not-query.asciidoc b/docs/reference/query-dsl/span-not-query.asciidoc
index 73186985b0d..5a648bd4b0e 100644
--- a/docs/reference/query-dsl/span-not-query.asciidoc
+++ b/docs/reference/query-dsl/span-not-query.asciidoc
@@ -6,24 +6,28 @@ query maps to Lucene `SpanNotQuery`. Here is an example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "span_not" : {
-        "include" : {
-            "span_term" : { "field1" : "hoya" }
-        },
-        "exclude" : {
-            "span_near" : { 
-                "clauses" : [
-                    { "span_term" : { "field1" : "la" } },
-                    { "span_term" : { "field1" : "hoya" } }
-                ],
-                "slop" : 0,
-                "in_order" : true
+    "query": {
+        "span_not" : {
+            "include" : {
+                "span_term" : { "field1" : "hoya" }
+            },
+            "exclude" : {
+                "span_near" : { 
+                    "clauses" : [
+                        { "span_term" : { "field1" : "la" } },
+                        { "span_term" : { "field1" : "hoya" } }
+                    ],
+                    "slop" : 0,
+                    "in_order" : true
+                }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 The `include` and `exclude` clauses can be any span type query. The
 `include` clause is the span query whose matches are filtered, and the
diff --git a/docs/reference/query-dsl/span-term-query.asciidoc b/docs/reference/query-dsl/span-term-query.asciidoc
index 9de86d48684..1b12a3c35f7 100644
--- a/docs/reference/query-dsl/span-term-query.asciidoc
+++ b/docs/reference/query-dsl/span-term-query.asciidoc
@@ -6,25 +6,37 @@ Matches spans containing a term. The span term query maps to Lucene
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "span_term" : { "user" : "kimchy" }
+    "query": {
+        "span_term" : { "user" : "kimchy" }
+    }
 }    
 --------------------------------------------------
+// CONSOLE
 
 A boost can also be associated with the query:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "span_term" : { "user" : { "value" : "kimchy", "boost" : 2.0 } }
+    "query": {
+       "span_term" : { "user" : { "value" : "kimchy", "boost" : 2.0 } }
+    }
 }    
 --------------------------------------------------
+// CONSOLE
 
 Or :
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "span_term" : { "user" : { "term" : "kimchy", "boost" : 2.0 } }
+    "query": {
+        "span_term" : { "user" : { "term" : "kimchy", "boost" : 2.0 } }
+    }
 }    
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/query-dsl/span-within-query.asciidoc b/docs/reference/query-dsl/span-within-query.asciidoc
index dc5c4bbfdfd..b70835c4134 100644
--- a/docs/reference/query-dsl/span-within-query.asciidoc
+++ b/docs/reference/query-dsl/span-within-query.asciidoc
@@ -6,24 +6,28 @@ query maps to Lucene `SpanWithinQuery`. Here is an example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "span_within" : {
-        "little" : {
-            "span_term" : { "field1" : "foo" }
-        },
-        "big" : {
-            "span_near" : { 
-                "clauses" : [
-                    { "span_term" : { "field1" : "bar" } },
-                    { "span_term" : { "field1" : "baz" } }
-                ],
-                "slop" : 5,
-                "in_order" : true
+    "query": {
+        "span_within" : {
+            "little" : {
+                "span_term" : { "field1" : "foo" }
+            },
+            "big" : {
+                "span_near" : { 
+                    "clauses" : [
+                        { "span_term" : { "field1" : "bar" } },
+                        { "span_term" : { "field1" : "baz" } }
+                    ],
+                    "slop" : 5,
+                    "in_order" : true
+                }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 The `big` and `little` clauses can be any span type query. Matching
 spans from `little` that are enclosed within `big` are returned.
diff --git a/docs/reference/query-dsl/terms-query.asciidoc b/docs/reference/query-dsl/terms-query.asciidoc
index 08df1e6fece..7382928e7f8 100644
--- a/docs/reference/query-dsl/terms-query.asciidoc
+++ b/docs/reference/query-dsl/terms-query.asciidoc
@@ -6,14 +6,18 @@ Filters documents that have fields that match any of the provided terms
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "constant_score" : {
-        "filter" : {
-            "terms" : { "user" : ["kimchy", "elasticsearch"]}
+    "query": {
+        "constant_score" : {
+            "filter" : {
+                "terms" : { "user" : ["kimchy", "elasticsearch"]}
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 The `terms` query is also aliased with `in` as the filter name for
 simpler usage deprecated[5.0.0,use `terms` instead].

From 93d1b385a4373eb59284ba4afeffd7f37ffc7416 Mon Sep 17 00:00:00 2001
From: Alexander Kazakov <alexshadow007@gmail.com>
Date: Wed, 6 Apr 2016 17:54:17 +0300
Subject: [PATCH 0158/1311] Improving parsing of sigma param for Extended Stats
 Bucket Aggregation  #17499

---
 .../bucketmetrics/BucketMetricsParser.java    | 45 ++++++++--------
 .../AvgBucketPipelineAggregatorBuilder.java   |  4 +-
 .../MaxBucketPipelineAggregatorBuilder.java   |  4 +-
 .../MinBucketPipelineAggregatorBuilder.java   |  4 +-
 ...ntilesBucketPipelineAggregatorBuilder.java | 52 +++++++++----------
 .../StatsBucketPipelineAggregatorBuilder.java |  4 +-
 .../extended/ExtendedStatsBucketParser.java   | 33 ++++++------
 .../SumBucketPipelineAggregatorBuilder.java   |  4 +-
 .../ExtendedStatsBucketTests.java             | 23 ++++++++
 .../bucketmetrics/PercentilesBucketTests.java | 23 ++++++++
 10 files changed, 121 insertions(+), 75 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsParser.java
index dbd0a5d128c..8eddd83b7e2 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsParser.java
@@ -27,7 +27,6 @@ import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
-import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -53,7 +52,7 @@ public abstract class BucketMetricsParser implements PipelineAggregator.Parser {
         String[] bucketsPaths = null;
         String format = null;
         GapPolicy gapPolicy = null;
-        Map<String, Object> leftover = new HashMap<>(5);
+        Map<String, Object> params = new HashMap<>(5);
 
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {
@@ -66,7 +65,7 @@ public abstract class BucketMetricsParser implements PipelineAggregator.Parser {
                 } else if (context.getParseFieldMatcher().match(currentFieldName, GAP_POLICY)) {
                     gapPolicy = GapPolicy.parse(context, parser.text(), parser.getTokenLocation());
                 } else {
-                    leftover.put(currentFieldName, parser.text());
+                    parseToken(pipelineAggregatorName, parser, context, currentFieldName, token, params);
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
                 if (context.getParseFieldMatcher().match(currentFieldName, BUCKETS_PATH)) {
@@ -77,10 +76,10 @@ public abstract class BucketMetricsParser implements PipelineAggregator.Parser {
                     }
                     bucketsPaths = paths.toArray(new String[paths.size()]);
                 } else {
-                    leftover.put(currentFieldName, parser.list());
+                    parseToken(pipelineAggregatorName, parser, context, currentFieldName, token, params);
                 }
             } else {
-                leftover.put(currentFieldName, parser.objectText());
+                parseToken(pipelineAggregatorName, parser, context, currentFieldName, token, params);
             }
         }
 
@@ -89,30 +88,32 @@ public abstract class BucketMetricsParser implements PipelineAggregator.Parser {
                     "Missing required field [" + BUCKETS_PATH.getPreferredName() + "] for aggregation [" + pipelineAggregatorName + "]");
         }
 
-        BucketMetricsPipelineAggregatorBuilder<?> factory = null;
-        try {
-            factory = buildFactory(pipelineAggregatorName, bucketsPaths[0], leftover);
-            if (format != null) {
-                factory.format(format);
-            }
-            if (gapPolicy != null) {
-                factory.gapPolicy(gapPolicy);
-            }
-        } catch (ParseException exception) {
-            throw new ParsingException(parser.getTokenLocation(),
-                    "Could not parse settings for aggregation [" + pipelineAggregatorName + "].", exception);
+        BucketMetricsPipelineAggregatorBuilder<?> factory  = buildFactory(pipelineAggregatorName, bucketsPaths[0], params);
+        if (format != null) {
+            factory.format(format);
+        }
+        if (gapPolicy != null) {
+            factory.gapPolicy(gapPolicy);
         }
 
-        if (leftover.size() > 0) {
-            throw new ParsingException(parser.getTokenLocation(),
-                    "Unexpected tokens " + leftover.keySet() + " in [" + pipelineAggregatorName + "].");
-        }
         assert(factory != null);
 
         return factory;
     }
 
     protected abstract BucketMetricsPipelineAggregatorBuilder<?> buildFactory(String pipelineAggregatorName, String bucketsPaths,
-            Map<String, Object> unparsedParams) throws ParseException;
+                                                                              Map<String, Object> params);
 
+    protected boolean token(XContentParser parser, QueryParseContext context, String field,
+                            XContentParser.Token token, Map<String, Object> params) throws IOException {
+        return false;
+    }
+
+    private void parseToken(String aggregationName, XContentParser parser, QueryParseContext context, String currentFieldName,
+                       XContentParser.Token currentToken, Map<String, Object> params) throws IOException {
+        if (token(parser, context, currentFieldName, currentToken, params) == false) {
+            throw new ParsingException(parser.getTokenLocation(),
+                "Unexpected token " + currentToken + " [" + currentFieldName + "] in [" + aggregationName + "]");
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/avg/AvgBucketPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/avg/AvgBucketPipelineAggregatorBuilder.java
index 3435c9768b0..248186b24bc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/avg/AvgBucketPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/avg/AvgBucketPipelineAggregatorBuilder.java
@@ -75,7 +75,7 @@ public class AvgBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
     public static final PipelineAggregator.Parser PARSER = new BucketMetricsParser() {
         @Override
         protected AvgBucketPipelineAggregatorBuilder buildFactory(String pipelineAggregatorName,
-                String bucketsPath, Map<String, Object> unparsedParams) {
+                String bucketsPath, Map<String, Object> params) {
             return new AvgBucketPipelineAggregatorBuilder(pipelineAggregatorName, bucketsPath);
         }
     };
@@ -94,4 +94,4 @@ public class AvgBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/max/MaxBucketPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/max/MaxBucketPipelineAggregatorBuilder.java
index 56ae7d76302..1ea6dcb59ff 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/max/MaxBucketPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/max/MaxBucketPipelineAggregatorBuilder.java
@@ -75,7 +75,7 @@ public class MaxBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
     public static final PipelineAggregator.Parser PARSER = new BucketMetricsParser() {
         @Override
         protected MaxBucketPipelineAggregatorBuilder buildFactory(String pipelineAggregatorName,
-                String bucketsPath, Map<String, Object> unparsedParams) {
+                String bucketsPath, Map<String, Object> params) {
             return new MaxBucketPipelineAggregatorBuilder(pipelineAggregatorName, bucketsPath);
         }
     };
@@ -94,4 +94,4 @@ public class MaxBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/min/MinBucketPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/min/MinBucketPipelineAggregatorBuilder.java
index de4df9fac70..1681cc1b7b2 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/min/MinBucketPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/min/MinBucketPipelineAggregatorBuilder.java
@@ -75,7 +75,7 @@ public class MinBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
     public static final PipelineAggregator.Parser PARSER = new BucketMetricsParser() {
         @Override
         protected MinBucketPipelineAggregatorBuilder buildFactory(String pipelineAggregatorName,
-                String bucketsPath, Map<String, Object> unparsedParams) {
+                String bucketsPath, Map<String, Object> params) {
             return new MinBucketPipelineAggregatorBuilder(pipelineAggregatorName, bucketsPath);
         }
     };
@@ -94,4 +94,4 @@ public class MinBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregatorBuilder.java
index 515e032da4f..8f3ee4d9eea 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregatorBuilder.java
@@ -19,10 +19,13 @@
 
 package org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile;
 
+import com.carrotsearch.hppc.DoubleArrayList;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
@@ -117,41 +120,36 @@ public class PercentilesBucketPipelineAggregatorBuilder
     }
 
     public static final PipelineAggregator.Parser PARSER = new BucketMetricsParser() {
+
         @Override
         protected PercentilesBucketPipelineAggregatorBuilder buildFactory(String pipelineAggregatorName,
-                String bucketsPath, Map<String, Object> unparsedParams) throws ParseException {
-            double[] percents = null;
-            int counter = 0;
-            Object percentParam = unparsedParams.get(PERCENTS_FIELD.getPreferredName());
-
-            if (percentParam != null) {
-                if (percentParam instanceof List) {
-                    percents = new double[((List<?>) percentParam).size()];
-                    for (Object p : (List<?>) percentParam) {
-                        if (p instanceof Double) {
-                            percents[counter] = (Double) p;
-                            counter += 1;
-                        } else {
-                            throw new ParseException(
-                                    "Parameter [" + PERCENTS_FIELD.getPreferredName() + "] must be an array of doubles, type `"
-                                            + percentParam.getClass().getSimpleName() + "` provided instead",
-                                    0);
-                        }
-                    }
-                    unparsedParams.remove(PERCENTS_FIELD.getPreferredName());
-                } else {
-                    throw new ParseException("Parameter [" + PERCENTS_FIELD.getPreferredName() + "] must be an array of doubles, type `"
-                            + percentParam.getClass().getSimpleName() + "` provided instead", 0);
-                }
-            }
+                                                                          String bucketsPath, Map<String, Object> params) {
 
             PercentilesBucketPipelineAggregatorBuilder factory = new
-                    PercentilesBucketPipelineAggregatorBuilder(pipelineAggregatorName, bucketsPath);
+                PercentilesBucketPipelineAggregatorBuilder(pipelineAggregatorName, bucketsPath);
+
+            double[] percents = (double[]) params.get(PERCENTS_FIELD.getPreferredName());
             if (percents != null) {
                 factory.percents(percents);
             }
+
             return factory;
         }
+
+        @Override
+        protected boolean token(XContentParser parser, QueryParseContext context, String field,
+                                XContentParser.Token token, Map<String, Object> params) throws IOException {
+            if (context.getParseFieldMatcher().match(field, PERCENTS_FIELD) && token == XContentParser.Token.START_ARRAY) {
+                DoubleArrayList percents = new DoubleArrayList(10);
+                while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
+                    percents.add(parser.doubleValue());
+                }
+                params.put(PERCENTS_FIELD.getPreferredName(), percents.toArray());
+                return true;
+            }
+            return false;
+        }
+
     };
 
     @Override
@@ -169,4 +167,4 @@ public class PercentilesBucketPipelineAggregatorBuilder
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregatorBuilder.java
index f56e28d988f..24ca14bff28 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregatorBuilder.java
@@ -77,7 +77,7 @@ public class StatsBucketPipelineAggregatorBuilder extends BucketMetricsPipelineA
     public static final PipelineAggregator.Parser PARSER = new BucketMetricsParser() {
         @Override
         protected StatsBucketPipelineAggregatorBuilder buildFactory(String pipelineAggregatorName,
-                String bucketsPath, Map<String, Object> unparsedParams) {
+                String bucketsPath, Map<String, Object> params) {
             return new StatsBucketPipelineAggregatorBuilder(pipelineAggregatorName, bucketsPath);
         }
     };
@@ -96,4 +96,4 @@ public class StatsBucketPipelineAggregatorBuilder extends BucketMetricsPipelineA
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketParser.java
index 698af21387f..5ab1a6afae0 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketParser.java
@@ -20,9 +20,11 @@
 package org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended;
 
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsParser;
 
-import java.text.ParseException;
+import java.io.IOException;
 import java.util.Map;
 
 public class ExtendedStatsBucketParser extends BucketMetricsParser {
@@ -30,25 +32,24 @@ public class ExtendedStatsBucketParser extends BucketMetricsParser {
 
     @Override
     protected ExtendedStatsBucketPipelineAggregatorBuilder buildFactory(String pipelineAggregatorName,
-            String bucketsPath, Map<String, Object> unparsedParams) throws ParseException {
-
-        Double sigma = null;
-        Object param = unparsedParams.get(SIGMA.getPreferredName());
-
-        if (param != null) {
-            if (param instanceof Double) {
-                sigma = (Double) param;
-                unparsedParams.remove(SIGMA.getPreferredName());
-            } else {
-                throw new ParseException("Parameter [" + SIGMA.getPreferredName() + "] must be a Double, type `"
-                        + param.getClass().getSimpleName() + "` provided instead", 0);
-            }
-        }
+            String bucketsPath, Map<String, Object> params) {
         ExtendedStatsBucketPipelineAggregatorBuilder factory =
-                new ExtendedStatsBucketPipelineAggregatorBuilder(pipelineAggregatorName, bucketsPath);
+            new ExtendedStatsBucketPipelineAggregatorBuilder(pipelineAggregatorName, bucketsPath);
+        Double sigma = (Double) params.get(SIGMA.getPreferredName());
         if (sigma != null) {
             factory.sigma(sigma);
         }
+
         return factory;
     }
+
+    @Override
+    protected boolean token(XContentParser parser, QueryParseContext context, String field,
+                            XContentParser.Token token, Map<String, Object> params) throws IOException {
+        if (context.getParseFieldMatcher().match(field, SIGMA) && token == XContentParser.Token.VALUE_NUMBER) {
+            params.put(SIGMA.getPreferredName(), parser.doubleValue());
+            return true;
+        }
+        return false;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/sum/SumBucketPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/sum/SumBucketPipelineAggregatorBuilder.java
index 0243de20916..3fb2322dedc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/sum/SumBucketPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/sum/SumBucketPipelineAggregatorBuilder.java
@@ -75,7 +75,7 @@ public class SumBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
     public static final PipelineAggregator.Parser PARSER = new BucketMetricsParser() {
         @Override
         protected SumBucketPipelineAggregatorBuilder buildFactory(String pipelineAggregatorName,
-                String bucketsPath, Map<String, Object> unparsedParams) {
+                String bucketsPath, Map<String, Object> params) {
             return new SumBucketPipelineAggregatorBuilder(pipelineAggregatorName, bucketsPath);
         }
     };
@@ -94,4 +94,4 @@ public class SumBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java
index f0e320d4e80..65e8156f71b 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java
@@ -19,8 +19,14 @@
 
 package org.elasticsearch.search.aggregations.pipeline.bucketmetrics;
 
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregatorBuilder;
 
+import static org.hamcrest.Matchers.equalTo;
+
 public class ExtendedStatsBucketTests extends AbstractBucketMetricsTestCase<ExtendedStatsBucketPipelineAggregatorBuilder> {
 
     @Override
@@ -32,5 +38,22 @@ public class ExtendedStatsBucketTests extends AbstractBucketMetricsTestCase<Exte
         return factory;
     }
 
+    public void testSigmaFromInt() throws Exception {
+        String content = XContentFactory.jsonBuilder()
+            .startObject()
+                .field("sigma", 5)
+                .field("buckets_path", "test")
+            .endObject()
+            .string();
 
+        XContentParser parser = XContentFactory.xContent(content).createParser(content);
+        QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
+        parser.nextToken(); // skip object start
+
+        ExtendedStatsBucketPipelineAggregatorBuilder builder = (ExtendedStatsBucketPipelineAggregatorBuilder) aggParsers
+            .pipelineParser(ExtendedStatsBucketPipelineAggregator.TYPE.name(), parseFieldMatcher)
+            .parse("test", parseContext);
+
+        assertThat(builder.sigma(), equalTo(5.0));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/PercentilesBucketTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/PercentilesBucketTests.java
index 0ee57c309a1..236f122bdd0 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/PercentilesBucketTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/PercentilesBucketTests.java
@@ -19,8 +19,14 @@
 
 package org.elasticsearch.search.aggregations.pipeline.bucketmetrics;
 
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.PercentilesBucketPipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.PercentilesBucketPipelineAggregatorBuilder;
 
+import static org.hamcrest.Matchers.equalTo;
+
 public class PercentilesBucketTests extends AbstractBucketMetricsTestCase<PercentilesBucketPipelineAggregatorBuilder> {
 
     @Override
@@ -37,5 +43,22 @@ public class PercentilesBucketTests extends AbstractBucketMetricsTestCase<Percen
         return factory;
     }
 
+    public void testPercentsFromMixedArray() throws Exception {
+        String content = XContentFactory.jsonBuilder()
+            .startObject()
+                .field("buckets_path", "test")
+                .array("percents", 0, 20.0, 50, 75.99)
+            .endObject()
+            .string();
 
+        XContentParser parser = XContentFactory.xContent(content).createParser(content);
+        QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
+        parser.nextToken(); // skip object start
+
+        PercentilesBucketPipelineAggregatorBuilder builder = (PercentilesBucketPipelineAggregatorBuilder) aggParsers
+            .pipelineParser(PercentilesBucketPipelineAggregator.TYPE.name(), parseFieldMatcher)
+            .parse("test", parseContext);
+
+        assertThat(builder.percents(), equalTo(new double[]{0.0, 20.0, 50.0, 75.99}));
+    }
 }

From 243c9e77fa44d352e218ca05cb431fb38baa9aea Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Tue, 10 May 2016 15:01:49 +0200
Subject: [PATCH 0159/1311] painless: Add synthetic length property as alias to
 Lists, so they can really be used like arrays

---
 .../main/java/org/elasticsearch/painless/Definition.java  | 3 +++
 .../java/org/elasticsearch/painless/BasicAPITests.java    | 8 ++++++++
 2 files changed, 11 insertions(+)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index f418edd2beb..a3ee43b00a5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -1061,12 +1061,14 @@ class Definition {
         addMethod("List", "set", null, false, objectType, new Type[] {intType, objectType}, defType, new Type[] {intType, defType});
         addMethod("List", "get", null, false, objectType, new Type[] {intType}, defType, null);
         addMethod("List", "remove", null, false, objectType, new Type[] {intType}, defType, null);
+        addMethod("List", "getLength", "size", false, intType, new Type[] {}, null, null);
 
         addConstructor("ArrayList", "new", new Type[] {}, null);
 
         addMethod("List<Object>", "set", null, false, objectType, new Type[] {intType, objectType}, null, null);
         addMethod("List<Object>", "get", null, false, objectType, new Type[] {intType}, null, null);
         addMethod("List<Object>", "remove", null, false, objectType, new Type[] {intType}, null, null);
+        addMethod("List<Object>", "getLength", "size", false, intType, new Type[] {}, null, null);
 
         addConstructor("ArrayList<Object>", "new", new Type[] {}, null);
 
@@ -1074,6 +1076,7 @@ class Definition {
             new Type[] {intType, stringType});
         addMethod("List<String>", "get", null, false, objectType, new Type[] {intType}, stringType, null);
         addMethod("List<String>", "remove", null, false, objectType, new Type[] {intType}, stringType, null);
+        addMethod("List<String>", "getLength", "size", false, intType, new Type[] {}, null, null);
 
         addConstructor("ArrayList<String>", "new", new Type[] {}, null);
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
index 2a14dad375a..b1df5276feb 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
@@ -73,4 +73,12 @@ public class BasicAPITests extends ScriptTestCase {
         assertEquals(5, exec("def x = new ArrayList(); x.add(5); return x.get(0);"));
         assertEquals(5, exec("def x = new ArrayList(); x.add(5); def index = 0; return x.get(index);"));
     }
+    
+    public void testListAsArray() {
+        assertEquals(1, exec("def x = new ArrayList(); x.add(5); return x.length"));
+        assertEquals(5, exec("def x = new ArrayList(); x.add(5); return x[0]"));
+        assertEquals(1, exec("List x = new ArrayList(); x.add('Hallo'); return x.length"));
+        assertEquals(1, exec("List<String> x = new ArrayList<String>(); x.add('Hallo'); return x.length"));
+        assertEquals(1, exec("List<Object> x = new ArrayList<Object>(); x.add('Hallo'); return x.length"));
+    }
 }

From 667a091205a915fa4b0c5d986909ed9b21e77b8e Mon Sep 17 00:00:00 2001
From: Alexander Kazakov <alexshadow007@gmail.com>
Date: Tue, 10 May 2016 16:41:21 +0300
Subject: [PATCH 0160/1311] Add note about cat field data API changes into
 migration doc

---
 docs/reference/migration/migrate_5_0/cat.asciidoc | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/docs/reference/migration/migrate_5_0/cat.asciidoc b/docs/reference/migration/migrate_5_0/cat.asciidoc
index 5a38845e051..6294b233103 100644
--- a/docs/reference/migration/migrate_5_0/cat.asciidoc
+++ b/docs/reference/migration/migrate_5_0/cat.asciidoc
@@ -38,3 +38,10 @@ and `i` for ingest. A node with no explicit roles will be a coordinating
 only node and marked with `-`. A node can have multiple roles. The
 master column has been adapted to return only whether a node is the
 current master (`*`) or not (`-`).
+
+==== Changes to cat field data API
+
+The cat field data endpoint adds a row per field instead of a column per field.
+
+The `total` field has been removed from the field data API. Total field data usage per node
+can be got by cat nodes API.

From 557fc8b43720d222fdee9219ad71f9b08ca7b90e Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 10 May 2016 10:08:58 -0400
Subject: [PATCH 0161/1311] Further simplifications of plugin script

In 7d1fd17172f5f8cb7ed381de1ff23dfb9f1856a5 the parsing of command-line
properties in the plugin script was removed. That commit missed
additional parsing of the properties for es.default.path.conf. This
commit removes that parsing and also replaces the use of eval with exec.

Relates #18239
---
 .../src/main/resources/bin/elasticsearch-plugin    | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/distribution/src/main/resources/bin/elasticsearch-plugin b/distribution/src/main/resources/bin/elasticsearch-plugin
index b337d9d0273..b1ba48d44f1 100755
--- a/distribution/src/main/resources/bin/elasticsearch-plugin
+++ b/distribution/src/main/resources/bin/elasticsearch-plugin
@@ -77,20 +77,10 @@ if [ ! -x "$JAVA" ]; then
     exit 1
 fi
 
-# check if properties already has a config file or config dir
-if [ -e "$CONF_DIR" ]; then
-  case "$properties" in
-    *-Des.default.path.conf=*|*-Des.path.conf=*)
-    ;;
-    *)
-      properties="$properties -Des.default.path.conf=\"$CONF_DIR\""
-    ;;
-  esac
-fi
-
 # full hostname passed through cut for portability on systems that do not support hostname -s
 # export on separate line for shells that do not support combining definition and export
 HOSTNAME=`hostname | cut -d. -f1`
 export HOSTNAME
 
-eval "\"$JAVA\"" "$ES_JAVA_OPTS" -Delasticsearch -Des.path.home="\"$ES_HOME\"" $properties -cp "\"$ES_HOME/lib/*\"" org.elasticsearch.plugins.PluginCli "$@"
+exec "$JAVA" $ES_JAVA_OPTS -Delasticsearch -Des.path.home="$ES_HOME" -Des.default.path.conf="$CONF_DIR" \
+  -cp "$ES_HOME/lib/*" org.elasticsearch.plugins.PluginCli "$@"

From a45e1cc75026500705de0e51f660fd2857beca40 Mon Sep 17 00:00:00 2001
From: Lee Hinman <lee@writequit.org>
Date: Tue, 10 May 2016 08:00:36 -0600
Subject: [PATCH 0162/1311] Add test for NullPointerException in SQS when
 analyzing text produces null query

---
 .../search/query/SimpleQueryStringIT.java     | 27 +++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
index ffaebecf89a..687e8e52186 100644
--- a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
@@ -370,4 +370,31 @@ public class SimpleQueryStringIT extends ESIntegTestCase {
         assertHitCount(searchResponse, 2L);
         assertSearchHits(searchResponse, "1", "2");
     }
+
+    public void testEmptySimpleQueryStringWithAnalysis() throws Exception {
+        // https://github.com/elastic/elasticsearch/issues/18202
+        String mapping = XContentFactory.jsonBuilder()
+                .startObject()
+                .startObject("type1")
+                .startObject("properties")
+                .startObject("body")
+                .field("type", "string")
+                .field("analyzer", "stop")
+                .endObject()
+                .endObject()
+                .endObject()
+                .endObject().string();
+
+        CreateIndexRequestBuilder mappingRequest = client().admin().indices()
+                .prepareCreate("test1")
+                .addMapping("type1", mapping);
+        mappingRequest.execute().actionGet();
+        indexRandom(true, client().prepareIndex("test1", "type1", "1").setSource("body", "Some Text"));
+        refresh();
+
+        SearchResponse searchResponse = client().prepareSearch()
+                .setQuery(simpleQueryStringQuery("the*").analyzeWildcard(true).field("body")).get();
+        assertNoFailures(searchResponse);
+        assertHitCount(searchResponse, 0l);
+    }
 }

From 0660386976d068f71e37c53ddc2fa7037f23e498 Mon Sep 17 00:00:00 2001
From: Gabriel Moskovicz <gmoskovicz@gmail.com>
Date: Tue, 10 May 2016 12:23:17 -0300
Subject: [PATCH 0163/1311] Add plugin information for Verbose mode

Relates #18051
---
 .../plugins/ListPluginsCommand.java           |  2 +
 .../plugins/ListPluginsCommandTests.java      | 86 +++++++++++++++++--
 2 files changed, 79 insertions(+), 9 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java b/core/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java
index 953e698a4c2..b513be99dcf 100644
--- a/core/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java
@@ -51,6 +51,8 @@ class ListPluginsCommand extends Command {
         try (DirectoryStream<Path> stream = Files.newDirectoryStream(env.pluginsFile())) {
             for (Path plugin : stream) {
                 terminal.println(plugin.getFileName().toString());
+                PluginInfo info = PluginInfo.readFromProperties(env.pluginsFile().resolve(plugin.toAbsolutePath()));
+                terminal.println(Terminal.Verbosity.VERBOSE, info.toString());
             }
         }
     }
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java
index cbdd031dea1..f26857e19af 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java
@@ -19,9 +19,13 @@
 
 package org.elasticsearch.plugins;
 
+
 import java.io.IOException;
 import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.stream.Collectors;
 
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.cli.ExitCodes;
@@ -29,6 +33,7 @@ import org.elasticsearch.cli.MockTerminal;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.Version;
 
 @LuceneTestCase.SuppressFileSystems("*")
 public class ListPluginsCommandTests extends ESTestCase {
@@ -43,12 +48,30 @@ public class ListPluginsCommandTests extends ESTestCase {
     }
 
     static MockTerminal listPlugins(Environment env) throws Exception {
+        return listPlugins(env, new String[0]);
+    }
+    
+    static MockTerminal listPlugins(Environment env, String[] args) throws Exception {
         MockTerminal terminal = new MockTerminal();
-        String[] args = {};
         int status = new ListPluginsCommand(env).main(args, terminal);
         assertEquals(ExitCodes.OK, status);
         return terminal;
     }
+    
+    static String buildMultiline(String... args){
+        return Arrays.asList(args).stream().collect(Collectors.joining("\n", "", "\n"));
+    }
+    
+    static void buildFakePlugin(Environment env, String description, String name, String classname) throws IOException {
+        PluginTestUtil.writeProperties(env.pluginsFile().resolve(name),
+                "description", description,
+                "name", name,
+                "version", "1.0",
+                "elasticsearch.version", Version.CURRENT.toString(),
+                "java.version", System.getProperty("java.specification.version"),
+                "classname", classname);
+    }
+
 
     public void testPluginsDirMissing() throws Exception {
         Environment env = createEnv();
@@ -56,7 +79,7 @@ public class ListPluginsCommandTests extends ESTestCase {
         IOException e = expectThrows(IOException.class, () -> {
            listPlugins(env);
         });
-        assertTrue(e.getMessage(), e.getMessage().contains("Plugins directory missing"));
+        assertEquals(e.getMessage(), "Plugins directory missing: " + env.pluginsFile());
     }
 
     public void testNoPlugins() throws Exception {
@@ -66,18 +89,63 @@ public class ListPluginsCommandTests extends ESTestCase {
 
     public void testOnePlugin() throws Exception {
         Environment env = createEnv();
-        Files.createDirectory(env.pluginsFile().resolve("fake"));
+        buildFakePlugin(env, "fake desc", "fake", "org.fake");
         MockTerminal terminal = listPlugins(env);
-        assertTrue(terminal.getOutput(), terminal.getOutput().contains("fake"));
+        assertEquals(terminal.getOutput(), buildMultiline("fake"));
     }
 
     public void testTwoPlugins() throws Exception {
         Environment env = createEnv();
-        Files.createDirectory(env.pluginsFile().resolve("fake1"));
-        Files.createDirectory(env.pluginsFile().resolve("fake2"));
+        buildFakePlugin(env, "fake desc", "fake1", "org.fake");
+        buildFakePlugin(env, "fake desc 2", "fake2", "org.fake");
         MockTerminal terminal = listPlugins(env);
-        String output = terminal.getOutput();
-        assertTrue(output, output.contains("fake1"));
-        assertTrue(output, output.contains("fake2"));
+        assertEquals(terminal.getOutput(), buildMultiline("fake1", "fake2"));
     }
+    
+    public void testPluginWithVerbose() throws Exception {
+        Environment env = createEnv();
+        buildFakePlugin(env, "fake desc", "fake_plugin", "org.fake");
+        String[] params = { "-v" };
+        MockTerminal terminal = listPlugins(env, params);
+        assertEquals(terminal.getOutput(), buildMultiline("Plugins directory: " + env.pluginsFile(), "fake_plugin",
+                "- Plugin information:", "Name: fake_plugin", "Description: fake desc", "Version: 1.0", " * Classname: org.fake"));
+    }
+    
+    public void testPluginWithVerboseMultiplePlugins() throws Exception {
+        Environment env = createEnv();
+        buildFakePlugin(env, "fake desc 1", "fake_plugin1", "org.fake");
+        buildFakePlugin(env, "fake desc 2", "fake_plugin2", "org.fake2");
+        String[] params = { "-v" };
+        MockTerminal terminal = listPlugins(env, params);
+        assertEquals(terminal.getOutput(), buildMultiline("Plugins directory: " + env.pluginsFile(),
+                "fake_plugin1", "- Plugin information:", "Name: fake_plugin1", "Description: fake desc 1", "Version: 1.0",
+                " * Classname: org.fake", "fake_plugin2", "- Plugin information:", "Name: fake_plugin2",
+                "Description: fake desc 2", "Version: 1.0", " * Classname: org.fake2"));
+    }
+        
+    public void testPluginWithoutVerboseMultiplePlugins() throws Exception {
+        Environment env = createEnv();
+        buildFakePlugin(env, "fake desc 1", "fake_plugin1", "org.fake");
+        buildFakePlugin(env, "fake desc 2", "fake_plugin2", "org.fake2");
+        MockTerminal terminal = listPlugins(env, new String[0]);
+        String output = terminal.getOutput();
+        assertEquals(output, buildMultiline("fake_plugin1", "fake_plugin2"));
+    }
+    
+    public void testPluginWithoutDescriptorFile() throws Exception{
+        Environment env = createEnv();
+        Files.createDirectories(env.pluginsFile().resolve("fake1"));
+        NoSuchFileException e = expectThrows(NoSuchFileException.class, () -> listPlugins(env));
+        assertEquals(e.getFile(), env.pluginsFile().resolve("fake1").resolve(PluginInfo.ES_PLUGIN_PROPERTIES).toString());
+    }
+    
+    public void testPluginWithWrongDescriptorFile() throws Exception{
+        Environment env = createEnv();        
+        PluginTestUtil.writeProperties(env.pluginsFile().resolve("fake1"),
+                "description", "fake desc");
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> listPlugins(env));
+        assertEquals(e.getMessage(), "Property [name] is missing in [" +
+                env.pluginsFile().resolve("fake1").resolve(PluginInfo.ES_PLUGIN_PROPERTIES).toString() + "]");
+    }
+    
 }

From 546aed8390952c2802a3fb13026f42f6fa9c5e6b Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 10 May 2016 11:23:24 -0400
Subject: [PATCH 0164/1311] hack

---
 .../main/java/org/elasticsearch/painless/NeedsScore.java | 5 +++++
 .../src/main/java/org/elasticsearch/painless/Writer.java | 9 ++++++++-
 .../java/org/elasticsearch/painless/WriterConstants.java | 2 ++
 3 files changed, 15 insertions(+), 1 deletion(-)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/NeedsScore.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/NeedsScore.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/NeedsScore.java
new file mode 100644
index 00000000000..055229ca10b
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/NeedsScore.java
@@ -0,0 +1,5 @@
+package org.elasticsearch.painless;
+
+/** Marker interface that a generated {@link Executable} uses the {@code _score} value */
+public interface NeedsScore {
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
index 4490bf119b4..4e96a71019e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
@@ -135,7 +135,14 @@ class Writer extends PainlessParserBaseVisitor<Void> {
         final String base = BASE_CLASS_TYPE.getInternalName();
         final String name = CLASS_TYPE.getInternalName();
 
-        writer.visit(version, access, name, null, base, null);
+        // apply marker interface NeedsScore if we use the score!
+        final String interfaces[];
+        if (metadata.scoreValueUsed) {
+          interfaces = new String[] { WriterConstants.NEEDS_SCORE_TYPE.getInternalName() };
+        } else {
+          interfaces = null;
+        }
+        writer.visit(version, access, name, null, base, interfaces);
         writer.visitSource(source, null);
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index a1a594b354e..7ed10c75ab2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -43,6 +43,8 @@ class WriterConstants {
     final static Type PAINLESS_ERROR_TYPE = Type.getType(PainlessError.class);
 
     final static Type DEFINITION_TYPE = Type.getType(Definition.class);
+    
+    final static Type NEEDS_SCORE_TYPE = Type.getType(NeedsScore.class);
 
     final static Type OBJECT_TYPE = Type.getType(Object.class);
 

From ad5ce598dbee921a4eea3271f75982b6987a7a42 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Tue, 10 May 2016 17:25:21 +0200
Subject: [PATCH 0165/1311] Use uppercase 'L' for long literal

---
 .../org/elasticsearch/search/query/SimpleQueryStringIT.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
index 687e8e52186..9502a818315 100644
--- a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java
@@ -395,6 +395,6 @@ public class SimpleQueryStringIT extends ESIntegTestCase {
         SearchResponse searchResponse = client().prepareSearch()
                 .setQuery(simpleQueryStringQuery("the*").analyzeWildcard(true).field("body")).get();
         assertNoFailures(searchResponse);
-        assertHitCount(searchResponse, 0l);
+        assertHitCount(searchResponse, 0L);
     }
 }

From eb1b2cf11128737b20d0b4c885002fe0ff51e8ef Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 10 May 2016 11:34:33 -0400
Subject: [PATCH 0166/1311] add test and cleanup

---
 .../elasticsearch/painless/NeedsScore.java    | 18 ++++++
 .../painless/PainlessScriptEngineService.java |  3 +-
 .../org/elasticsearch/painless/Writer.java    |  4 +-
 .../painless/NeedsScoreTests.java             | 61 +++++++++++++++++++
 4 files changed, 82 insertions(+), 4 deletions(-)
 create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/NeedsScore.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/NeedsScore.java
index 055229ca10b..180b8c9d8e2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/NeedsScore.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/NeedsScore.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
 package org.elasticsearch.painless;
 
 /** Marker interface that a generated {@link Executable} uses the {@code _score} value */
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
index 7be3f9832be..6f3eb4cfbab 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
@@ -215,11 +215,10 @@ public class PainlessScriptEngineService extends AbstractComponent implements Sc
 
             /**
              * Whether or not the score is needed.
-             * @return Always true as it's assumed score is needed.
              */
             @Override
             public boolean needsScores() {
-                return true;
+                return compiledScript.compiled() instanceof NeedsScore;
             }
         };
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
index 4e96a71019e..36cfb91ac4d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
@@ -138,9 +138,9 @@ class Writer extends PainlessParserBaseVisitor<Void> {
         // apply marker interface NeedsScore if we use the score!
         final String interfaces[];
         if (metadata.scoreValueUsed) {
-          interfaces = new String[] { WriterConstants.NEEDS_SCORE_TYPE.getInternalName() };
+            interfaces = new String[] { WriterConstants.NEEDS_SCORE_TYPE.getInternalName() };
         } else {
-          interfaces = null;
+            interfaces = null;
         }
         writer.visit(version, access, name, null, base, interfaces);
         writer.visitSource(source, null);
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java
new file mode 100644
index 00000000000..561e9202c1e
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.script.CompiledScript;
+import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.script.SearchScript;
+import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+import java.util.Collections;
+
+/**
+ * Test that needsScores() is reported correctly depending on whether _score is used
+ */
+// TODO: can we test this better? this is a port of the ExpressionsTests method.
+public class NeedsScoreTests extends ESSingleNodeTestCase {
+
+    public void testNeedsScores() {
+        IndexService index = createIndex("test", Settings.EMPTY, "type", "d", "type=double");
+
+        PainlessScriptEngineService service = new PainlessScriptEngineService(Settings.EMPTY);
+        SearchLookup lookup = new SearchLookup(index.mapperService(), index.fieldData(), null);
+
+        Object compiled = service.compile("1.2", Collections.emptyMap());
+        SearchScript ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled), lookup, Collections.<String, Object>emptyMap());
+        assertFalse(ss.needsScores());
+
+        compiled = service.compile("input.doc['d'].value", Collections.emptyMap());
+        ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled), lookup, Collections.<String, Object>emptyMap());
+        assertFalse(ss.needsScores());
+
+        compiled = service.compile("1/_score", Collections.emptyMap());
+        ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled), lookup, Collections.<String, Object>emptyMap());
+        assertTrue(ss.needsScores());
+
+        compiled = service.compile("input.doc['d'].value * _score", Collections.emptyMap());
+        ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled), lookup, Collections.<String, Object>emptyMap());
+        assertTrue(ss.needsScores());
+    }
+
+}

From 81c0b7bfa9eaa4cbd67175f5a6e4d408ed12ea87 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 10 May 2016 11:55:27 -0400
Subject: [PATCH 0167/1311] Log exception when join validation fails

Today when join validation fails, we log a warning but do not log the
exception that led to the join validation failing. This commit modifies
this so that we do log this exception.
---
 .../main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
index 607f8d34db1..596e40abfba 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
@@ -832,7 +832,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
             try {
                 membership.sendValidateJoinRequestBlocking(node, state, joinTimeout);
             } catch (Throwable e) {
-                logger.warn("failed to validate incoming join request from node [{}]", node);
+                logger.warn("failed to validate incoming join request from node [{}]", e, node);
                 callback.onFailure(new IllegalStateException("failure when sending a validation request to node", e));
                 return;
             }

From 1c2d3b10ef8fc6f22bb512e5153d7d5099b26d35 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 10 May 2016 12:31:02 -0400
Subject: [PATCH 0168/1311] some cleanups suggested by @uschindler and break
 some long lines

---
 .../org/elasticsearch/painless/DynamicCallSite.java  |  5 +++--
 .../main/java/org/elasticsearch/painless/Writer.java |  3 +--
 .../org/elasticsearch/painless/WriterConstants.java  |  1 -
 .../org/elasticsearch/painless/NeedsScoreTests.java  | 12 ++++++++----
 4 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DynamicCallSite.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DynamicCallSite.java
index 8222465c0ad..116f52836a4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DynamicCallSite.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DynamicCallSite.java
@@ -29,8 +29,9 @@ import java.lang.invoke.MutableCallSite;
 /**
  * Painless invokedynamic call site.
  * <p>
- * Has 3 flavors (passed as static bootstrap parameters): dynamic method call,
- * dynamic field load (getter), and dynamic field store (setter).
+ * Has 5 flavors (passed as static bootstrap parameters): dynamic method call,
+ * dynamic field load (getter), and dynamic field store (setter), dynamic array load,
+ * and dynamic array store.
  * <p>
  * When a new type is encountered at the call site, we lookup from the appropriate
  * whitelist, and cache with a guard. If we encounter too many types, we stop caching.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
index 36cfb91ac4d..2c60d3ad113 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
@@ -84,7 +84,6 @@ import static org.elasticsearch.painless.WriterConstants.MAP_GET;
 import static org.elasticsearch.painless.WriterConstants.MAP_TYPE;
 import static org.elasticsearch.painless.WriterConstants.SCORE_ACCESSOR_FLOAT;
 import static org.elasticsearch.painless.WriterConstants.SCORE_ACCESSOR_TYPE;
-import static org.elasticsearch.painless.WriterConstants.SIGNATURE;
 
 class Writer extends PainlessParserBaseVisitor<Void> {
     static byte[] write(Metadata metadata) {
@@ -116,7 +115,7 @@ class Writer extends PainlessParserBaseVisitor<Void> {
         writeBegin();
         writeConstructor();
 
-        execute = new GeneratorAdapter(Opcodes.ACC_PUBLIC, EXECUTE, SIGNATURE, null, writer);
+        execute = new GeneratorAdapter(Opcodes.ACC_PUBLIC, EXECUTE, null, null, writer);
 
         final WriterUtility utility = new WriterUtility(metadata, execute);
         final WriterCaster caster = new WriterCaster(execute);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index 42c709da969..a7cc7346229 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -38,7 +38,6 @@ class WriterConstants {
 
     final static Method CONSTRUCTOR = getAsmMethod(void.class, "<init>", Definition.class, String.class, String.class);
     final static Method EXECUTE     = getAsmMethod(Object.class, "execute", Map.class);
-    final static String SIGNATURE   = "(Ljava/util/Map<Ljava/lang/String;Ljava/lang/Object;>;)Ljava/lang/Object;";
 
     final static Type PAINLESS_ERROR_TYPE = Type.getType(PainlessError.class);
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java
index 561e9202c1e..8861514e9bc 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java
@@ -42,19 +42,23 @@ public class NeedsScoreTests extends ESSingleNodeTestCase {
         SearchLookup lookup = new SearchLookup(index.mapperService(), index.fieldData(), null);
 
         Object compiled = service.compile("1.2", Collections.emptyMap());
-        SearchScript ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled), lookup, Collections.<String, Object>emptyMap());
+        SearchScript ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled), 
+                                         lookup, Collections.<String, Object>emptyMap());
         assertFalse(ss.needsScores());
 
         compiled = service.compile("input.doc['d'].value", Collections.emptyMap());
-        ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled), lookup, Collections.<String, Object>emptyMap());
+        ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled), 
+                            lookup, Collections.<String, Object>emptyMap());
         assertFalse(ss.needsScores());
 
         compiled = service.compile("1/_score", Collections.emptyMap());
-        ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled), lookup, Collections.<String, Object>emptyMap());
+        ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled), 
+                            lookup, Collections.<String, Object>emptyMap());
         assertTrue(ss.needsScores());
 
         compiled = service.compile("input.doc['d'].value * _score", Collections.emptyMap());
-        ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled), lookup, Collections.<String, Object>emptyMap());
+        ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled), 
+                            lookup, Collections.<String, Object>emptyMap());
         assertTrue(ss.needsScores());
     }
 

From 77534205403897818e09fa7f52e9ab5a81953e76 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Tue, 10 May 2016 19:11:04 +0200
Subject: [PATCH 0169/1311] Make ShardRouting and UnassignedInfo immutable
 (#17821)

This makes defensive copying of ShardRouting objects obsolete whenever we do a reroute and trashes less objects.
---
 .../shards/ClusterSearchShardsGroup.java      |  19 +-
 .../TransportClusterSearchShardsAction.java   |   6 +-
 .../admin/indices/segments/ShardSegments.java |   6 +-
 .../admin/indices/stats/ShardStats.java       |   4 +-
 .../upgrade/get/ShardUpgradeStatus.java       |   6 +-
 .../node/TransportBroadcastByNodeAction.java  |   2 +-
 .../action/shard/ShardStateAction.java        |   6 +-
 .../cluster/routing/AllocationId.java         |   4 +-
 .../IllegalShardRoutingStateException.java    |   2 +-
 .../cluster/routing/IndexRoutingTable.java    |  36 +-
 .../routing/IndexShardRoutingTable.java       |   5 +-
 .../cluster/routing/RoutingNode.java          |  62 ++--
 .../cluster/routing/RoutingNodes.java         | 245 +++++++++-----
 .../cluster/routing/ShardRouting.java         | 308 ++++++------------
 .../cluster/routing/UnassignedInfo.java       |  27 +-
 .../routing/allocation/AllocationService.java |  64 ++--
 .../allocator/BalancedShardsAllocator.java    |  48 ++-
 .../AbstractAllocateAllocationCommand.java    |  10 +-
 ...AllocateEmptyPrimaryAllocationCommand.java |  19 +-
 .../command/CancelAllocationCommand.java      |   2 +-
 .../cluster/IndicesClusterStateService.java   |  25 +-
 .../flush/ShardsSyncedFlushResult.java        |   2 +-
 .../ClusterAllocationExplanationTests.java    |   6 +-
 .../TransportReplicationActionTests.java      |   5 +-
 .../cluster/ClusterStateDiffIT.java           |   7 +-
 .../elasticsearch/cluster/DiskUsageTests.java |  25 +-
 .../org/elasticsearch/cluster/ack/AckIT.java  |   3 +-
 ...rdFailedClusterStateTaskExecutorTests.java |   7 +-
 .../action/shard/ShardStateActionTests.java   |   2 +-
 .../cluster/allocation/ClusterRerouteIT.java  |  20 +-
 .../cluster/routing/AllocationIdTests.java    |  43 +--
 .../routing/RandomShardRoutingMutator.java    |  11 +-
 .../cluster/routing/ShardRoutingHelper.java   |  36 +-
 .../cluster/routing/ShardRoutingTests.java    | 126 +------
 .../cluster/routing/UnassignedInfoTests.java  |  37 ++-
 .../allocation/AddIncrementallyTests.java     |   1 +
 .../allocation/AllocationCommandsTests.java   |   9 +-
 .../ClusterRebalanceRoutingTests.java         |   6 +-
 .../allocation/DeadNodesAllocationTests.java  |  40 +--
 .../ExpectedShardSizeAllocationTests.java     |  12 +-
 .../allocation/FailedShardsRoutingTests.java  |  40 +--
 .../allocation/FilterRoutingTests.java        |   1 +
 .../allocation/StartedShardsRoutingTests.java |  21 +-
 .../allocation/ThrottlingAllocationTests.java |   6 +-
 .../decider/DiskThresholdDeciderTests.java    |   8 +-
 .../DiskThresholdDeciderUnitTests.java        |  68 ++--
 .../gateway/ReplicaShardAllocatorTests.java   |   8 +-
 .../index/shard/IndexShardTests.java          |  69 ++--
 .../IndexingMemoryControllerTests.java        |  13 +-
 ...dicesLifecycleListenerSingleNodeTests.java |   9 +-
 .../cluster/routing/TestShardRouting.java     |  35 +-
 51 files changed, 741 insertions(+), 841 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java
index 1b329d17289..ccb4d32465e 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsGroup.java
@@ -26,6 +26,7 @@ import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
 
 import java.io.IOException;
 
@@ -33,16 +34,14 @@ import java.io.IOException;
  */
 public class ClusterSearchShardsGroup implements Streamable, ToXContent {
 
-    private Index index;
-    private int shardId;
+    private ShardId shardId;
     ShardRouting[] shards;
 
     ClusterSearchShardsGroup() {
 
     }
 
-    public ClusterSearchShardsGroup(Index index, int shardId, ShardRouting[] shards) {
-        this.index = index;
+    public ClusterSearchShardsGroup(ShardId shardId, ShardRouting[] shards) {
         this.shardId = shardId;
         this.shards = shards;
     }
@@ -54,11 +53,11 @@ public class ClusterSearchShardsGroup implements Streamable, ToXContent {
     }
 
     public String getIndex() {
-        return index.getName();
+        return shardId.getIndexName();
     }
 
     public int getShardId() {
-        return shardId;
+        return shardId.id();
     }
 
     public ShardRouting[] getShards() {
@@ -67,18 +66,16 @@ public class ClusterSearchShardsGroup implements Streamable, ToXContent {
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
-        index = new Index(in);
-        shardId = in.readVInt();
+        shardId = ShardId.readShardId(in);
         shards = new ShardRouting[in.readVInt()];
         for (int i = 0; i < shards.length; i++) {
-            shards[i] = ShardRouting.readShardRoutingEntry(in, index, shardId);
+            shards[i] = new ShardRouting(shardId, in);
         }
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        index.writeTo(out);
-        out.writeVInt(shardId);
+        shardId.writeTo(out);
         out.writeVInt(shards.length);
         for (ShardRouting shardRouting : shards) {
             shardRouting.writeToThin(out);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java
index 8b26fd6c04f..2f9a6e7dede 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java
@@ -34,6 +34,7 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -78,8 +79,7 @@ public class TransportClusterSearchShardsAction extends TransportMasterNodeReadA
         ClusterSearchShardsGroup[] groupResponses = new ClusterSearchShardsGroup[groupShardsIterator.size()];
         int currentGroup = 0;
         for (ShardIterator shardIt : groupShardsIterator) {
-            Index index = shardIt.shardId().getIndex();
-            int shardId = shardIt.shardId().getId();
+            ShardId shardId = shardIt.shardId();
             ShardRouting[] shardRoutings = new ShardRouting[shardIt.size()];
             int currentShard = 0;
             shardIt.reset();
@@ -87,7 +87,7 @@ public class TransportClusterSearchShardsAction extends TransportMasterNodeReadA
                 shardRoutings[currentShard++] = shard;
                 nodeIds.add(shard.currentNodeId());
             }
-            groupResponses[currentGroup++] = new ClusterSearchShardsGroup(index, shardId, shardRoutings);
+            groupResponses[currentGroup++] = new ClusterSearchShardsGroup(shardId, shardRoutings);
         }
         DiscoveryNode[] nodes = new DiscoveryNode[nodeIds.size()];
         int currentNode = 0;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/segments/ShardSegments.java b/core/src/main/java/org/elasticsearch/action/admin/indices/segments/ShardSegments.java
index 3e39cfd561f..1052740248d 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/segments/ShardSegments.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/segments/ShardSegments.java
@@ -31,8 +31,6 @@ import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
-import static org.elasticsearch.cluster.routing.ShardRouting.readShardRoutingEntry;
-
 public class ShardSegments implements Streamable, Iterable<Segment> {
 
     private ShardRouting shardRouting;
@@ -88,7 +86,7 @@ public class ShardSegments implements Streamable, Iterable<Segment> {
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
-        shardRouting = readShardRoutingEntry(in);
+        shardRouting = new ShardRouting(in);
         int size = in.readVInt();
         if (size == 0) {
             segments = Collections.emptyList();
@@ -108,4 +106,4 @@ public class ShardSegments implements Streamable, Iterable<Segment> {
             segment.writeTo(out);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java
index 50b3157bfab..6640defc1c0 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java
@@ -31,8 +31,6 @@ import org.elasticsearch.index.shard.ShardPath;
 
 import java.io.IOException;
 
-import static org.elasticsearch.cluster.routing.ShardRouting.readShardRoutingEntry;
-
 /**
  */
 public class ShardStats implements Streamable, ToXContent {
@@ -91,7 +89,7 @@ public class ShardStats implements Streamable, ToXContent {
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
-        shardRouting = readShardRoutingEntry(in);
+        shardRouting = new ShardRouting(in);
         commonStats = CommonStats.readCommonStats(in);
         commitStats = CommitStats.readOptionalCommitStatsFrom(in);
         statePath = in.readString();
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/get/ShardUpgradeStatus.java b/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/get/ShardUpgradeStatus.java
index 1f4aa897bfa..527b21343e9 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/get/ShardUpgradeStatus.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/get/ShardUpgradeStatus.java
@@ -26,8 +26,6 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 
 import java.io.IOException;
 
-import static org.elasticsearch.cluster.routing.ShardRouting.readShardRoutingEntry;
-
 public class ShardUpgradeStatus extends BroadcastShardResponse {
 
     private ShardRouting shardRouting;
@@ -75,7 +73,7 @@ public class ShardUpgradeStatus extends BroadcastShardResponse {
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        shardRouting = readShardRoutingEntry(in);
+        shardRouting = new ShardRouting(in);
         totalBytes = in.readLong();
         toUpgradeBytes = in.readLong();
         toUpgradeBytesAncient = in.readLong();
@@ -89,4 +87,4 @@ public class ShardUpgradeStatus extends BroadcastShardResponse {
         out.writeLong(toUpgradeBytes);
         out.writeLong(toUpgradeBytesAncient);
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java b/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
index b594b6a7a84..29863419a4e 100644
--- a/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
@@ -475,7 +475,7 @@ public abstract class TransportBroadcastByNodeAction<Request extends BroadcastRe
             int size = in.readVInt();
             shards = new ArrayList<>(size);
             for (int i = 0; i < size; i++) {
-                shards.add(ShardRouting.readShardRoutingEntry(in));
+                shards.add(new ShardRouting(in));
             }
             nodeId = in.readString();
         }
diff --git a/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
index 5a44a645467..eb43322cdd0 100644
--- a/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
+++ b/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
@@ -69,8 +69,6 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.cluster.routing.ShardRouting.readShardRoutingEntry;
-
 public class ShardStateAction extends AbstractComponent {
 
     public static final String SHARD_STARTED_ACTION_NAME = "internal:cluster/shard/started";
@@ -418,8 +416,8 @@ public class ShardStateAction extends AbstractComponent {
         @Override
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
-            shardRouting = readShardRoutingEntry(in);
-            sourceShardRouting = readShardRoutingEntry(in);
+            shardRouting = new ShardRouting(in);
+            sourceShardRouting = new ShardRouting(in);
             message = in.readString();
             failure = in.readThrowable();
         }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/AllocationId.java b/core/src/main/java/org/elasticsearch/cluster/routing/AllocationId.java
index 100752289f7..cb0fb487693 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/AllocationId.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/AllocationId.java
@@ -26,6 +26,7 @@ import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -43,7 +44,7 @@ import java.util.Objects;
  * relocationId. Once relocation is done, the new allocation id is set to the relocationId. This is similar
  * behavior to how ShardRouting#currentNodeId is used.
  */
-public class AllocationId implements ToXContent {
+public class AllocationId implements ToXContent, Writeable {
     private static final String ID_KEY = "id";
     private static final String RELOCATION_ID_KEY = "relocation_id";
 
@@ -81,6 +82,7 @@ public class AllocationId implements ToXContent {
         this.relocationId = in.readOptionalString();
     }
 
+    @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(this.id);
         out.writeOptionalString(this.relocationId);
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IllegalShardRoutingStateException.java b/core/src/main/java/org/elasticsearch/cluster/routing/IllegalShardRoutingStateException.java
index af1d4195075..67e7b8b016b 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/IllegalShardRoutingStateException.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/IllegalShardRoutingStateException.java
@@ -42,7 +42,7 @@ public class IllegalShardRoutingStateException extends RoutingException {
 
     public IllegalShardRoutingStateException(StreamInput in) throws IOException {
         super(in);
-        shard = ShardRouting.readShardRoutingEntry(in);
+        shard = new ShardRouting(in);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
index 6111f317c76..1d38dcbdb7e 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
@@ -76,7 +76,6 @@ public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> imple
         List<ShardRouting> allActiveShards = new ArrayList<>();
         for (IntObjectCursor<IndexShardRoutingTable> cursor : shards) {
             for (ShardRouting shardRouting : cursor.value) {
-                shardRouting.freeze();
                 if (shardRouting.active()) {
                     allActiveShards.add(shardRouting);
                 }
@@ -395,17 +394,18 @@ public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> imple
             if (!shards.isEmpty()) {
                 throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created");
             }
-            for (int shardId = 0; shardId < indexMetaData.getNumberOfShards(); shardId++) {
-                IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(new ShardId(index, shardId));
+            for (int shardNumber = 0; shardNumber < indexMetaData.getNumberOfShards(); shardNumber++) {
+                ShardId shardId = new ShardId(index, shardNumber);
+                IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId);
                 for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) {
-                    if (asNew && ignoreShards.contains(shardId)) {
+                    if (asNew && ignoreShards.contains(shardNumber)) {
                         // This shards wasn't completely snapshotted - restore it as new shard
-                        indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, null, i == 0, unassignedInfo));
+                        indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, null, i == 0, unassignedInfo));
                     } else {
-                        indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, i == 0 ? restoreSource : null, i == 0, unassignedInfo));
+                        indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, i == 0 ? restoreSource : null, i == 0, unassignedInfo));
                     }
                 }
-                shards.put(shardId, indexShardRoutingBuilder.build());
+                shards.put(shardNumber, indexShardRoutingBuilder.build());
             }
             return this;
         }
@@ -418,22 +418,24 @@ public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> imple
             if (!shards.isEmpty()) {
                 throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created");
             }
-            for (int shardId = 0; shardId < indexMetaData.getNumberOfShards(); shardId++) {
-                IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(new ShardId(index, shardId));
+            for (int shardNumber = 0; shardNumber < indexMetaData.getNumberOfShards(); shardNumber++) {
+                ShardId shardId = new ShardId(index, shardNumber);
+                IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId);
                 for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) {
-                    indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, null, i == 0, unassignedInfo));
+                    indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, null, i == 0, unassignedInfo));
                 }
-                shards.put(shardId, indexShardRoutingBuilder.build());
+                shards.put(shardNumber, indexShardRoutingBuilder.build());
             }
             return this;
         }
 
         public Builder addReplica() {
             for (IntCursor cursor : shards.keys()) {
-                int shardId = cursor.value;
+                int shardNumber = cursor.value;
+                ShardId shardId = new ShardId(index, shardNumber);
                 // version 0, will get updated when reroute will happen
-                ShardRouting shard = ShardRouting.newUnassigned(index, shardId, null, false, new UnassignedInfo(UnassignedInfo.Reason.REPLICA_ADDED, null));
-                shards.put(shardId,
+                ShardRouting shard = ShardRouting.newUnassigned(shardId, null, false, new UnassignedInfo(UnassignedInfo.Reason.REPLICA_ADDED, null));
+                shards.put(shardNumber,
                         new IndexShardRoutingTable.Builder(shards.get(shard.id())).addShard(shard).build()
                 );
             }
@@ -451,7 +453,7 @@ public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> imple
                 // re-add all the current ones
                 IndexShardRoutingTable.Builder builder = new IndexShardRoutingTable.Builder(indexShard.shardId());
                 for (ShardRouting shardRouting : indexShard) {
-                    builder.addShard(new ShardRouting(shardRouting));
+                    builder.addShard(shardRouting);
                 }
                 // first check if there is one that is not assigned to a node, and remove it
                 boolean removed = false;
@@ -487,9 +489,9 @@ public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> imple
         public Builder addShard(IndexShardRoutingTable refData, ShardRouting shard) {
             IndexShardRoutingTable indexShard = shards.get(shard.id());
             if (indexShard == null) {
-                indexShard = new IndexShardRoutingTable.Builder(refData.shardId()).addShard(new ShardRouting(shard)).build();
+                indexShard = new IndexShardRoutingTable.Builder(refData.shardId()).addShard(shard).build();
             } else {
-                indexShard = new IndexShardRoutingTable.Builder(indexShard).addShard(new ShardRouting(shard)).build();
+                indexShard = new IndexShardRoutingTable.Builder(indexShard).addShard(shard).build();
             }
             shards.put(indexShard.shardId().id(), indexShard);
             return this;
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
index d504a74dc3c..53b094bc34b 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
@@ -590,11 +590,12 @@ public class IndexShardRoutingTable implements Iterable<ShardRouting> {
 
         public static IndexShardRoutingTable readFromThin(StreamInput in, Index index) throws IOException {
             int iShardId = in.readVInt();
-            Builder builder = new Builder(new ShardId(index, iShardId));
+            ShardId shardId = new ShardId(index, iShardId);
+            Builder builder = new Builder(shardId);
 
             int size = in.readVInt();
             for (int i = 0; i < size; i++) {
-                ShardRouting shard = ShardRouting.readShardRoutingEntry(in, index, iShardId);
+                ShardRouting shard = new ShardRouting(shardId, in);
                 builder.addShard(shard);
             }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
index 77ae7b41d91..8e099eb14f6 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
@@ -20,12 +20,15 @@
 package org.elasticsearch.cluster.routing;
 
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.index.shard.ShardId;
 
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 
 /**
  * A {@link RoutingNode} represents a cluster node associated with a single {@link DiscoveryNode} including all shards
@@ -37,25 +40,37 @@ public class RoutingNode implements Iterable<ShardRouting> {
 
     private final DiscoveryNode node;
 
-    private final List<ShardRouting> shards;
+    private final LinkedHashMap<ShardId, ShardRouting> shards; // LinkedHashMap to preserve order
 
-    public RoutingNode(String nodeId, DiscoveryNode node) {
-        this(nodeId, node, new ArrayList<ShardRouting>());
+    public RoutingNode(String nodeId, DiscoveryNode node, ShardRouting... shards) {
+        this(nodeId, node, buildShardRoutingMap(shards));
     }
 
-    public RoutingNode(String nodeId, DiscoveryNode node, List<ShardRouting> shards) {
+    RoutingNode(String nodeId, DiscoveryNode node, LinkedHashMap<ShardId, ShardRouting> shards) {
         this.nodeId = nodeId;
         this.node = node;
         this.shards = shards;
     }
 
+    private static LinkedHashMap<ShardId, ShardRouting> buildShardRoutingMap(ShardRouting... shardRoutings) {
+        final LinkedHashMap<ShardId, ShardRouting> shards = new LinkedHashMap<>();
+        for (ShardRouting shardRouting : shardRoutings) {
+            ShardRouting previousValue = shards.put(shardRouting.shardId(), shardRouting);
+            if (previousValue != null) {
+                throw new IllegalArgumentException("Cannot have two different shards with same shard id " + shardRouting.shardId() +
+                    " on same node ");
+            }
+        }
+        return shards;
+    }
+
     @Override
     public Iterator<ShardRouting> iterator() {
-        return Collections.unmodifiableCollection(shards).iterator();
+        return Collections.unmodifiableCollection(shards.values()).iterator();
     }
 
     Iterator<ShardRouting> mutableIterator() {
-        return shards.iterator();
+        return shards.values().iterator();
     }
 
     /**
@@ -67,6 +82,10 @@ public class RoutingNode implements Iterable<ShardRouting> {
         return this.node;
     }
 
+    public @Nullable ShardRouting getByShardId(ShardId id) {
+        return shards.get(id);
+    }
+
     /**
      * Get the id of this node
      * @return id of the node
@@ -84,13 +103,20 @@ public class RoutingNode implements Iterable<ShardRouting> {
      * @param shard Shard to crate on this Node
      */
     void add(ShardRouting shard) {
-        // TODO use Set with ShardIds for faster lookup.
-        for (ShardRouting shardRouting : shards) {
-            if (shardRouting.isSameShard(shard)) {
-                throw new IllegalStateException("Trying to add a shard [" + shard.shardId().getIndex().getName() + "][" + shard.shardId().id() + "] to a node [" + nodeId + "] where it already exists");
-            }
+        if (shards.containsKey(shard.shardId())) {
+            throw new IllegalStateException("Trying to add a shard " + shard.shardId() + " to a node [" + nodeId + "] where it already exists");
         }
-        shards.add(shard);
+        shards.put(shard.shardId(), shard);
+    }
+
+    void update(ShardRouting oldShard, ShardRouting newShard) {
+        if (shards.containsKey(oldShard.shardId()) == false) {
+            // Shard was already removed by routing nodes iterator
+            // TODO: change caller logic in RoutingNodes so that this check can go away
+            return;
+        }
+        ShardRouting previousValue = shards.put(newShard.shardId(), newShard);
+        assert previousValue == oldShard : "expected shard " + previousValue + " but was " + oldShard;
     }
 
     /**
@@ -166,7 +192,7 @@ public class RoutingNode implements Iterable<ShardRouting> {
     public String prettyPrint() {
         StringBuilder sb = new StringBuilder();
         sb.append("-----node_id[").append(nodeId).append("][" + (node == null ? "X" : "V") + "]\n");
-        for (ShardRouting entry : shards) {
+        for (ShardRouting entry : shards.values()) {
             sb.append("--------").append(entry.shortSummary()).append('\n');
         }
         return sb.toString();
@@ -188,12 +214,8 @@ public class RoutingNode implements Iterable<ShardRouting> {
         return sb.toString();
     }
 
-    public ShardRouting get(int i) {
-        return shards.get(i) ;
-    }
-
-    public Collection<ShardRouting> copyShards() {
-        return new ArrayList<>(shards);
+    public List<ShardRouting> copyShards() {
+        return new ArrayList<>(shards.values());
     }
 
     public boolean isEmpty() {
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
index 173f42ce241..6004d2af94a 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
@@ -29,16 +29,22 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 
+import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.ListIterator;
 import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Queue;
 import java.util.Set;
 import java.util.function.Predicate;
 
@@ -71,7 +77,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
     private int relocatingShards = 0;
 
     private final Map<String, ObjectIntHashMap<String>> nodesPerAttributeNames = new HashMap<>();
-    private final Map<String, Recoveries> recoveryiesPerNode = new HashMap<>();
+    private final Map<String, Recoveries> recoveriesPerNode = new HashMap<>();
 
     public RoutingNodes(ClusterState clusterState) {
         this(clusterState, true);
@@ -84,10 +90,10 @@ public class RoutingNodes implements Iterable<RoutingNode> {
         this.routingTable = clusterState.routingTable();
         this.customs = clusterState.customs();
 
-        Map<String, List<ShardRouting>> nodesToShards = new HashMap<>();
+        Map<String, LinkedHashMap<ShardId, ShardRouting>> nodesToShards = new HashMap<>();
         // fill in the nodeToShards with the "live" nodes
         for (ObjectCursor<DiscoveryNode> cursor : clusterState.nodes().getDataNodes().values()) {
-            nodesToShards.put(cursor.value.getId(), new ArrayList<>());
+            nodesToShards.put(cursor.value.getId(), new LinkedHashMap<>()); // LinkedHashMap to preserve order
         }
 
         // fill in the inverse of node -> shards allocated
@@ -101,21 +107,25 @@ public class RoutingNodes implements Iterable<RoutingNode> {
                     // by the ShardId, as this is common for primary and replicas.
                     // A replica Set might have one (and not more) replicas with the state of RELOCATING.
                     if (shard.assignedToNode()) {
-                        List<ShardRouting> entries = nodesToShards.computeIfAbsent(shard.currentNodeId(), k -> new ArrayList<>());
-                        final ShardRouting sr = getRouting(shard, readOnly);
-                        entries.add(sr);
-                        assignedShardsAdd(sr);
+                        Map<ShardId, ShardRouting> entries = nodesToShards.computeIfAbsent(shard.currentNodeId(),
+                            k -> new LinkedHashMap<>()); // LinkedHashMap to preserve order
+                        ShardRouting previousValue = entries.put(shard.shardId(), shard);
+                        if (previousValue != null) {
+                            throw new IllegalArgumentException("Cannot have two different shards with same shard id on same node");
+                        }
+                        assignedShardsAdd(shard);
                         if (shard.relocating()) {
                             relocatingShards++;
-                            entries = nodesToShards.computeIfAbsent(shard.relocatingNodeId(), k -> new ArrayList<>());
+                            entries = nodesToShards.computeIfAbsent(shard.relocatingNodeId(),
+                                k -> new LinkedHashMap<>()); // LinkedHashMap to preserve order
                             // add the counterpart shard with relocatingNodeId reflecting the source from which
                             // it's relocating from.
                             ShardRouting targetShardRouting = shard.buildTargetRelocatingShard();
                             addInitialRecovery(targetShardRouting);
-                            if (readOnly) {
-                                targetShardRouting.freeze();
+                            previousValue = entries.put(targetShardRouting.shardId(), targetShardRouting);
+                            if (previousValue != null) {
+                                throw new IllegalArgumentException("Cannot have two different shards with same shard id on same node");
                             }
-                            entries.add(targetShardRouting);
                             assignedShardsAdd(targetShardRouting);
                         } else if (shard.active() == false) { // shards that are initializing without being relocated
                             if (shard.primary()) {
@@ -125,14 +135,12 @@ public class RoutingNodes implements Iterable<RoutingNode> {
                             addInitialRecovery(shard);
                         }
                     } else {
-                        final ShardRouting sr = getRouting(shard, readOnly);
-                        assignedShardsAdd(sr);
-                        unassignedShards.add(sr);
+                        unassignedShards.add(shard);
                     }
                 }
             }
         }
-        for (Map.Entry<String, List<ShardRouting>> entry : nodesToShards.entrySet()) {
+        for (Map.Entry<String, LinkedHashMap<ShardId, ShardRouting>> entry : nodesToShards.entrySet()) {
             String nodeId = entry.getKey();
             this.nodesToShards.put(nodeId, new RoutingNode(nodeId, clusterState.nodes().get(nodeId), entry.getValue()));
         }
@@ -153,7 +161,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
     private void addRecovery(final ShardRouting routing, final boolean increment, final boolean initializing) {
         final int howMany = increment ? 1 : -1;
         assert routing.initializing() : "routing must be initializing: " + routing;
-        Recoveries.getOrAdd(recoveryiesPerNode, routing.currentNodeId()).addIncoming(howMany);
+        Recoveries.getOrAdd(recoveriesPerNode, routing.currentNodeId()).addIncoming(howMany);
         final String sourceNodeId;
         if (routing.relocatingNodeId() != null) { // this is a relocation-target
             sourceNodeId = routing.relocatingNodeId();
@@ -165,8 +173,8 @@ public class RoutingNodes implements Iterable<RoutingNode> {
                     }
                 }
                 // we transfer the recoveries to the relocated primary
-                recoveryiesPerNode.get(sourceNodeId).addOutgoing(-numRecoveringReplicas);
-                recoveryiesPerNode.get(routing.currentNodeId()).addOutgoing(numRecoveringReplicas);
+                recoveriesPerNode.get(sourceNodeId).addOutgoing(-numRecoveringReplicas);
+                recoveriesPerNode.get(routing.currentNodeId()).addOutgoing(numRecoveringReplicas);
             }
         } else if (routing.primary() == false) { // primary without relocationID is initial recovery
             ShardRouting primary = findPrimary(routing);
@@ -180,16 +188,16 @@ public class RoutingNodes implements Iterable<RoutingNode> {
             sourceNodeId = null;
         }
         if (sourceNodeId != null) {
-            Recoveries.getOrAdd(recoveryiesPerNode, sourceNodeId).addOutgoing(howMany);
+            Recoveries.getOrAdd(recoveriesPerNode, sourceNodeId).addOutgoing(howMany);
         }
     }
 
     public int getIncomingRecoveries(String nodeId) {
-        return recoveryiesPerNode.getOrDefault(nodeId, Recoveries.EMPTY).getIncoming();
+        return recoveriesPerNode.getOrDefault(nodeId, Recoveries.EMPTY).getIncoming();
     }
 
     public int getOutgoingRecoveries(String nodeId) {
-        return recoveryiesPerNode.getOrDefault(nodeId, Recoveries.EMPTY).getOutgoing();
+        return recoveriesPerNode.getOrDefault(nodeId, Recoveries.EMPTY).getOutgoing();
     }
 
     private ShardRouting findPrimary(ShardRouting routing) {
@@ -211,15 +219,6 @@ public class RoutingNodes implements Iterable<RoutingNode> {
         return primary;
     }
 
-    private static ShardRouting getRouting(ShardRouting src, boolean readOnly) {
-        if (readOnly) {
-            src.freeze(); // we just freeze and reuse this instance if we are read only
-        } else {
-            src = new ShardRouting(src);
-        }
-        return src;
-    }
-
     @Override
     public Iterator<RoutingNode> iterator() {
         return Collections.unmodifiableCollection(nodesToShards.values()).iterator();
@@ -423,40 +422,46 @@ public class RoutingNodes implements Iterable<RoutingNode> {
      * Moves a shard from unassigned to initialize state
      *
      * @param existingAllocationId allocation id to use. If null, a fresh allocation id is generated.
+     * @return                     the initialized shard
      */
-    public void initialize(ShardRouting shard, String nodeId, @Nullable String existingAllocationId, long expectedSize) {
+    public ShardRouting initialize(ShardRouting shard, String nodeId, @Nullable String existingAllocationId, long expectedSize) {
         ensureMutable();
-        assert shard.unassigned() : shard;
-        shard.initialize(nodeId, existingAllocationId, expectedSize);
-        node(nodeId).add(shard);
+        assert shard.unassigned() : "expected an unassigned shard " + shard;
+        ShardRouting initializedShard = shard.initialize(nodeId, existingAllocationId, expectedSize);
+        node(nodeId).add(initializedShard);
         inactiveShardCount++;
-        if (shard.primary()) {
+        if (initializedShard.primary()) {
             inactivePrimaryCount++;
         }
-        addRecovery(shard);
-        assignedShardsAdd(shard);
+        addRecovery(initializedShard);
+        assignedShardsAdd(initializedShard);
+        return initializedShard;
     }
 
     /**
      * Relocate a shard to another node, adding the target initializing
-     * shard as well as assigning it. And returning the target initializing
-     * shard.
+     * shard as well as assigning it.
+     *
+     * @return pair of source relocating and target initializing shards.
      */
-    public ShardRouting relocate(ShardRouting shard, String nodeId, long expectedShardSize) {
+    public Tuple<ShardRouting,ShardRouting> relocate(ShardRouting shard, String nodeId, long expectedShardSize) {
         ensureMutable();
         relocatingShards++;
-        shard.relocate(nodeId, expectedShardSize);
-        ShardRouting target = shard.buildTargetRelocatingShard();
+        ShardRouting source = shard.relocate(nodeId, expectedShardSize);
+        ShardRouting target = source.buildTargetRelocatingShard();
+        updateAssigned(shard, source);
         node(target.currentNodeId()).add(target);
         assignedShardsAdd(target);
         addRecovery(target);
-        return target;
+        return Tuple.tuple(source, target);
     }
 
     /**
      * Mark a shard as started and adjusts internal statistics.
+     *
+     * @return the started shard
      */
-    public void started(ShardRouting shard) {
+    public ShardRouting started(ShardRouting shard) {
         ensureMutable();
         assert !shard.active() : "expected an initializing shard " + shard;
         if (shard.relocatingNodeId() == null) {
@@ -467,40 +472,39 @@ public class RoutingNodes implements Iterable<RoutingNode> {
             }
         }
         removeRecovery(shard);
-        shard.moveToStarted();
+        ShardRouting startedShard = shard.moveToStarted();
+        updateAssigned(shard, startedShard);
+        return startedShard;
     }
 
 
 
     /**
      * Cancels a relocation of a shard that shard must relocating.
+     *
+     * @return the shard after cancelling relocation
      */
-    public void cancelRelocation(ShardRouting shard) {
+    public ShardRouting cancelRelocation(ShardRouting shard) {
         ensureMutable();
         relocatingShards--;
-        shard.cancelRelocation();
+        ShardRouting cancelledShard = shard.cancelRelocation();
+        updateAssigned(shard, cancelledShard);
+        return cancelledShard;
     }
 
     /**
-     * swaps the status of a shard, making replicas primary and vice versa.
+     * moves the assigned replica shard to primary.
      *
-     * @param shards the shard to have its primary status swapped.
+     * @param replicaShard the replica shard to be promoted to primary
+     * @return             the resulting primary shard
      */
-    public void swapPrimaryFlag(ShardRouting... shards) {
+    public ShardRouting promoteAssignedReplicaShardToPrimary(ShardRouting replicaShard) {
         ensureMutable();
-        for (ShardRouting shard : shards) {
-            if (shard.primary()) {
-                shard.moveFromPrimary();
-                if (shard.unassigned()) {
-                    unassignedShards.primaries--;
-                }
-            } else {
-                shard.moveToPrimary();
-                if (shard.unassigned()) {
-                    unassignedShards.primaries++;
-                }
-            }
-        }
+        assert replicaShard.unassigned() == false : "unassigned shard cannot be promoted to primary: " + replicaShard;
+        assert replicaShard.primary() == false : "primary shard cannot be promoted to primary: " + replicaShard;
+        ShardRouting primaryShard = replicaShard.moveToPrimary();
+        updateAssigned(replicaShard, primaryShard);
+        return primaryShard;
     }
 
     private static final List<ShardRouting> EMPTY = Collections.emptyList();
@@ -523,7 +527,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
                 inactivePrimaryCount--;
             }
         } else if (shard.relocating()) {
-            cancelRelocation(shard);
+            shard = cancelRelocation(shard);
         }
         assignedShardsRemove(shard);
         if (shard.initializing()) {
@@ -532,12 +536,9 @@ public class RoutingNodes implements Iterable<RoutingNode> {
     }
 
     private void assignedShardsAdd(ShardRouting shard) {
-        if (shard.unassigned()) {
-            // no unassigned
-            return;
-        }
+        assert shard.unassigned() == false : "unassigned shard " + shard + " cannot be added to list of assigned shards";
         List<ShardRouting> shards = assignedShards.computeIfAbsent(shard.shardId(), k -> new ArrayList<>());
-        assert assertInstanceNotInList(shard, shards);
+        assert assertInstanceNotInList(shard, shards) : "shard " + shard + " cannot appear twice in list of assigned shards";
         shards.add(shard);
     }
 
@@ -560,8 +561,8 @@ public class RoutingNodes implements Iterable<RoutingNode> {
                     return;
                 }
             }
-            assert false : "Illegal state";
         }
+        assert false : "No shard found to remove";
     }
 
     public boolean isKnown(DiscoveryNode node) {
@@ -586,15 +587,30 @@ public class RoutingNodes implements Iterable<RoutingNode> {
         return nodesToShards.values().toArray(new RoutingNode[nodesToShards.size()]);
     }
 
-    public void reinitShadowPrimary(ShardRouting candidate) {
+    public ShardRouting reinitShadowPrimary(ShardRouting candidate) {
         ensureMutable();
         if (candidate.relocating()) {
             cancelRelocation(candidate);
         }
-        candidate.reinitializeShard();
+        ShardRouting reinitializedShard = candidate.reinitializeShard();
+        updateAssigned(candidate, reinitializedShard);
         inactivePrimaryCount++;
         inactiveShardCount++;
+        return reinitializedShard;
+    }
 
+    private void updateAssigned(ShardRouting oldShard, ShardRouting newShard) {
+        assert oldShard.shardId().equals(newShard.shardId()) :
+            "can only update " + oldShard + " by shard with same shard id but was " + newShard;
+        assert oldShard.unassigned() == false && newShard.unassigned() == false :
+            "only assigned shards can be updated in list of assigned shards (prev: " + oldShard + ", new: " + newShard + ")";
+        assert oldShard.currentNodeId().equals(newShard.currentNodeId()) : "shard to update " + oldShard +
+            " can only update " + oldShard + " by shard assigned to same node but was " + newShard;
+        node(oldShard.currentNodeId()).update(oldShard, newShard);
+        List<ShardRouting> shardsWithMatchingShardId = assignedShards.computeIfAbsent(oldShard.shardId(), k -> new ArrayList<>());
+        int previousShardIndex = shardsWithMatchingShardId.indexOf(oldShard);
+        assert previousShardIndex >= 0 : "shard to update " + oldShard + " does not exist in list of assigned shards";
+        shardsWithMatchingShardId.set(previousShardIndex, newShard);
     }
 
     /**
@@ -683,11 +699,11 @@ public class RoutingNodes implements Iterable<RoutingNode> {
 
         public class UnassignedIterator implements Iterator<ShardRouting> {
 
-            private final Iterator<ShardRouting> iterator;
+            private final ListIterator<ShardRouting> iterator;
             private ShardRouting current;
 
             public UnassignedIterator() {
-                this.iterator = unassigned.iterator();
+                this.iterator = unassigned.listIterator();
             }
 
             @Override
@@ -705,9 +721,9 @@ public class RoutingNodes implements Iterable<RoutingNode> {
              *
              * @param existingAllocationId allocation id to use. If null, a fresh allocation id is generated.
              */
-            public void initialize(String nodeId, @Nullable String existingAllocationId, long expectedShardSize) {
+            public ShardRouting initialize(String nodeId, @Nullable String existingAllocationId, long expectedShardSize) {
                 innerRemove();
-                nodes.initialize(new ShardRouting(current), nodeId, existingAllocationId, expectedShardSize);
+                return nodes.initialize(current, nodeId, existingAllocationId, expectedShardSize);
             }
 
             /**
@@ -721,6 +737,35 @@ public class RoutingNodes implements Iterable<RoutingNode> {
                 ignoreShard(current);
             }
 
+            private void updateShardRouting(ShardRouting shardRouting) {
+                current = shardRouting;
+                iterator.set(shardRouting);
+            }
+
+            /**
+             * updates the unassigned info on the current unassigned shard
+             *
+             * @param  unassignedInfo the new unassigned info to use
+             * @return the shard with unassigned info updated
+             */
+            public ShardRouting updateUnassignedInfo(UnassignedInfo unassignedInfo) {
+                ShardRouting updatedShardRouting = current.updateUnassignedInfo(unassignedInfo);
+                updateShardRouting(updatedShardRouting);
+                return updatedShardRouting;
+            }
+
+            /**
+             * marks the current primary shard as replica
+             *
+             * @return the shard with primary status swapped
+             */
+            public ShardRouting demotePrimaryToReplicaShard() {
+                assert current.primary() : "non-primary shard " + current + " cannot be demoted";
+                updateShardRouting(current.moveFromPrimary());
+                primaries--;
+                return current;
+            }
+
             /**
              * Unsupported operation, just there for the interface. Use {@link #removeAndIgnore()} or
              * {@link #initialize(String, String, long)}.
@@ -847,7 +892,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
             }
         }
 
-        for (Map.Entry<String, Recoveries> recoveries : routingNodes.recoveryiesPerNode.entrySet()) {
+        for (Map.Entry<String, Recoveries> recoveries : routingNodes.recoveriesPerNode.entrySet()) {
             String node = recoveries.getKey();
             final Recoveries value = recoveries.getValue();
             int incoming = 0;
@@ -962,14 +1007,14 @@ public class RoutingNodes implements Iterable<RoutingNode> {
             return iterable.iterator();
         }
 
-        public void moveToUnassigned(UnassignedInfo unassignedInfo) {
+        public ShardRouting moveToUnassigned(UnassignedInfo unassignedInfo) {
             ensureMutable();
             if (isRemoved() == false) {
                 remove();
             }
-            ShardRouting unassigned = new ShardRouting(shard); // protective copy of the mutable shard
-            unassigned.moveToUnassigned(unassignedInfo);
-            unassigned().add(unassigned);
+            ShardRouting unassigned = shard.moveToUnassigned(unassignedInfo);
+            unassignedShards.add(unassigned);
+            return unassigned;
         }
 
         public ShardRouting current() {
@@ -983,6 +1028,44 @@ public class RoutingNodes implements Iterable<RoutingNode> {
         }
     }
 
+    /**
+     * Creates an iterator over shards interleaving between nodes: The iterator returns the first shard from
+     * the first node, then the first shard of the second node, etc. until one shard from each node has been returned.
+     * The iterator then resumes on the first node by returning the second shard and continues until all shards from
+     * all the nodes have been returned.
+     */
+    public Iterator<ShardRouting> nodeInterleavedShardIterator() {
+        final Queue<Iterator<ShardRouting>> queue = new ArrayDeque<>();
+        for (Map.Entry<String, RoutingNode> entry : nodesToShards.entrySet()) {
+            queue.add(entry.getValue().copyShards().iterator());
+        }
+        return new Iterator<ShardRouting>() {
+            public boolean hasNext() {
+                while (!queue.isEmpty()) {
+                    if (queue.peek().hasNext()) {
+                        return true;
+                    }
+                    queue.poll();
+                }
+                return false;
+            }
+
+            public ShardRouting next() {
+                if (hasNext() == false) {
+                    throw new NoSuchElementException();
+                }
+                Iterator<ShardRouting> iter = queue.poll();
+                ShardRouting result = iter.next();
+                queue.offer(iter);
+                return result;
+            }
+
+            public void remove() {
+                throw new UnsupportedOperationException();
+            }
+        };
+    }
+
     private static final class Recoveries {
         private static final Recoveries EMPTY = new Recoveries();
         private int incoming = 0;
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
index 269a238e710..ce31483f47f 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
@@ -21,10 +21,11 @@ package org.elasticsearch.cluster.routing;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.Index;
@@ -38,43 +39,31 @@ import java.util.List;
  * {@link ShardRouting} immutably encapsulates information about shard
  * routings like id, state, version, etc.
  */
-public final class ShardRouting implements Streamable, ToXContent {
+public final class ShardRouting implements Writeable, ToXContent {
 
     /**
      * Used if shard size is not available
      */
     public static final long UNAVAILABLE_EXPECTED_SHARD_SIZE = -1;
 
-    private Index index;
-    private int shardId;
-    private String currentNodeId;
-    private String relocatingNodeId;
-    private boolean primary;
-    private ShardRoutingState state;
-    private RestoreSource restoreSource;
-    private UnassignedInfo unassignedInfo;
-    private AllocationId allocationId;
+    private final ShardId shardId;
+    private final String currentNodeId;
+    private final String relocatingNodeId;
+    private final boolean primary;
+    private final ShardRoutingState state;
+    private final RestoreSource restoreSource;
+    private final UnassignedInfo unassignedInfo;
+    private final AllocationId allocationId;
     private final transient List<ShardRouting> asList;
-    private transient ShardId shardIdentifier;
-    private boolean frozen = false;
-    private long expectedShardSize = UNAVAILABLE_EXPECTED_SHARD_SIZE;
-
-    private ShardRouting() {
-        this.asList = Collections.singletonList(this);
-    }
-
-    public ShardRouting(ShardRouting copy) {
-        this(copy.index(), copy.id(), copy.currentNodeId(), copy.relocatingNodeId(), copy.restoreSource(), copy.primary(), copy.state(), copy.unassignedInfo(), copy.allocationId(), true, copy.getExpectedShardSize());
-    }
+    private final long expectedShardSize;
 
     /**
      * A constructor to internally create shard routing instances, note, the internal flag should only be set to true
      * by either this class or tests. Visible for testing.
      */
-    ShardRouting(Index index, int shardId, String currentNodeId,
+    ShardRouting(ShardId shardId, String currentNodeId,
                  String relocatingNodeId, RestoreSource restoreSource, boolean primary, ShardRoutingState state,
-                 UnassignedInfo unassignedInfo, AllocationId allocationId, boolean internal, long expectedShardSize) {
-        this.index = index;
+                 UnassignedInfo unassignedInfo, AllocationId allocationId, long expectedShardSize) {
         this.shardId = shardId;
         this.currentNodeId = currentNodeId;
         this.relocatingNodeId = relocatingNodeId;
@@ -88,38 +77,31 @@ public final class ShardRouting implements Streamable, ToXContent {
         assert expectedShardSize == UNAVAILABLE_EXPECTED_SHARD_SIZE || state == ShardRoutingState.INITIALIZING || state == ShardRoutingState.RELOCATING : expectedShardSize + " state: " + state;
         assert expectedShardSize >= 0 || state != ShardRoutingState.INITIALIZING || state != ShardRoutingState.RELOCATING : expectedShardSize + " state: " + state;
         assert !(state == ShardRoutingState.UNASSIGNED && unassignedInfo == null) : "unassigned shard must be created with meta";
-        if (!internal) {
-            assert state == ShardRoutingState.UNASSIGNED;
-            assert currentNodeId == null;
-            assert relocatingNodeId == null;
-            assert allocationId == null;
-        }
-
     }
 
     /**
      * Creates a new unassigned shard.
      */
-    public static ShardRouting newUnassigned(Index index, int shardId, RestoreSource restoreSource, boolean primary, UnassignedInfo unassignedInfo) {
-        return new ShardRouting(index, shardId, null, null, restoreSource, primary, ShardRoutingState.UNASSIGNED, unassignedInfo, null, true, UNAVAILABLE_EXPECTED_SHARD_SIZE);
+    public static ShardRouting newUnassigned(ShardId shardId, RestoreSource restoreSource, boolean primary, UnassignedInfo unassignedInfo) {
+        return new ShardRouting(shardId, null, null, restoreSource, primary, ShardRoutingState.UNASSIGNED, unassignedInfo, null, UNAVAILABLE_EXPECTED_SHARD_SIZE);
     }
 
     public Index index() {
-        return this.index;
+        return shardId.getIndex();
     }
 
     /**
      * The index name.
      */
     public String getIndexName() {
-        return index().getName();
+        return shardId.getIndexName();
     }
 
     /**
      * The shard id.
      */
     public int id() {
-        return this.shardId;
+        return shardId.id();
     }
 
     /**
@@ -201,8 +183,8 @@ public final class ShardRouting implements Streamable, ToXContent {
      */
     public ShardRouting buildTargetRelocatingShard() {
         assert relocating();
-        return new ShardRouting(index, shardId, relocatingNodeId, currentNodeId, restoreSource, primary, ShardRoutingState.INITIALIZING, unassignedInfo,
-            AllocationId.newTargetRelocation(allocationId), true, expectedShardSize);
+        return new ShardRouting(shardId, relocatingNodeId, currentNodeId, restoreSource, primary, ShardRoutingState.INITIALIZING, unassignedInfo,
+            AllocationId.newTargetRelocation(allocationId), expectedShardSize);
     }
 
     /**
@@ -247,11 +229,7 @@ public final class ShardRouting implements Streamable, ToXContent {
      * The shard id.
      */
     public ShardId shardId() {
-        if (shardIdentifier != null) {
-            return shardIdentifier;
-        }
-        shardIdentifier = new ShardId(index, shardId);
-        return shardIdentifier;
+        return shardId;
     }
 
     public boolean allocatedPostIndexCreate(IndexMetaData indexMetaData) {
@@ -278,57 +256,30 @@ public final class ShardRouting implements Streamable, ToXContent {
      * A shard iterator with just this shard in it.
      */
     public ShardIterator shardsIt() {
-        return new PlainShardIterator(shardId(), asList);
+        return new PlainShardIterator(shardId, asList);
     }
 
-    public static ShardRouting readShardRoutingEntry(StreamInput in) throws IOException {
-        ShardRouting entry = new ShardRouting();
-        entry.readFrom(in);
-        return entry;
-    }
-
-    public static ShardRouting readShardRoutingEntry(StreamInput in, Index index, int shardId) throws IOException {
-        ShardRouting entry = new ShardRouting();
-        entry.readFrom(in, index, shardId);
-        return entry;
-    }
-
-    public void readFrom(StreamInput in, Index index, int shardId) throws IOException {
-        this.index = index;
+    public ShardRouting(ShardId shardId, StreamInput in) throws IOException {
         this.shardId = shardId;
-        readFromThin(in);
-    }
-
-    public void readFromThin(StreamInput in) throws IOException {
-        if (in.readBoolean()) {
-            currentNodeId = in.readString();
-        }
-
-        if (in.readBoolean()) {
-            relocatingNodeId = in.readString();
-        }
-
+        currentNodeId = in.readOptionalString();
+        relocatingNodeId = in.readOptionalString();
         primary = in.readBoolean();
         state = ShardRoutingState.fromValue(in.readByte());
-
         restoreSource = RestoreSource.readOptionalRestoreSource(in);
-        if (in.readBoolean()) {
-            unassignedInfo = new UnassignedInfo(in);
-        }
-        if (in.readBoolean()) {
-            allocationId = new AllocationId(in);
-        }
-        if (relocating() || initializing()) {
-            expectedShardSize = in.readLong();
+        unassignedInfo = in.readOptionalWriteable(UnassignedInfo::new);
+        allocationId = in.readOptionalWriteable(AllocationId::new);
+        final long shardSize;
+        if (state == ShardRoutingState.RELOCATING || state == ShardRoutingState.INITIALIZING) {
+            shardSize = in.readLong();
         } else {
-            expectedShardSize = UNAVAILABLE_EXPECTED_SHARD_SIZE;
+            shardSize = UNAVAILABLE_EXPECTED_SHARD_SIZE;
         }
-        freeze();
+        expectedShardSize = shardSize;
+        asList = Collections.singletonList(this);
     }
 
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        readFrom(in, new Index(in), in.readVInt());
+    public ShardRouting(StreamInput in) throws IOException {
+        this(ShardId.readShardId(in), in);
     }
 
     /**
@@ -338,74 +289,37 @@ public final class ShardRouting implements Streamable, ToXContent {
      * @throws IOException if something happens during write
      */
     public void writeToThin(StreamOutput out) throws IOException {
-        if (currentNodeId != null) {
-            out.writeBoolean(true);
-            out.writeString(currentNodeId);
-        } else {
-            out.writeBoolean(false);
-        }
-
-        if (relocatingNodeId != null) {
-            out.writeBoolean(true);
-            out.writeString(relocatingNodeId);
-        } else {
-            out.writeBoolean(false);
-        }
-
+        out.writeOptionalString(currentNodeId);
+        out.writeOptionalString(relocatingNodeId);
         out.writeBoolean(primary);
         out.writeByte(state.value());
-
-        if (restoreSource != null) {
-            out.writeBoolean(true);
-            restoreSource.writeTo(out);
-        } else {
-            out.writeBoolean(false);
-        }
-        if (unassignedInfo != null) {
-            out.writeBoolean(true);
-            unassignedInfo.writeTo(out);
-        } else {
-            out.writeBoolean(false);
-        }
-        if (allocationId != null) {
-            out.writeBoolean(true);
-            allocationId.writeTo(out);
-        } else {
-            out.writeBoolean(false);
-        }
-        if (relocating() || initializing()) {
+        out.writeOptionalStreamable(restoreSource);
+        out.writeOptionalWriteable(unassignedInfo);
+        out.writeOptionalWriteable(allocationId);
+        if (state == ShardRoutingState.RELOCATING || state == ShardRoutingState.INITIALIZING) {
             out.writeLong(expectedShardSize);
         }
-
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        index.writeTo(out);
-        out.writeVInt(shardId);
+        shardId.writeTo(out);
         writeToThin(out);
     }
 
-    public void updateUnassignedInfo(UnassignedInfo unassignedInfo) {
-        ensureNotFrozen();
+    public ShardRouting updateUnassignedInfo(UnassignedInfo unassignedInfo) {
         assert this.unassignedInfo != null : "can only update unassign info if they are already set";
-        this.unassignedInfo = unassignedInfo;
+        return new ShardRouting(shardId, currentNodeId, relocatingNodeId, restoreSource, primary, state,
+            unassignedInfo, allocationId, expectedShardSize);
     }
 
-    // package private mutators start here
-
     /**
      * Moves the shard to unassigned state.
      */
-    void moveToUnassigned(UnassignedInfo unassignedInfo) {
-        ensureNotFrozen();
+    public ShardRouting moveToUnassigned(UnassignedInfo unassignedInfo) {
         assert state != ShardRoutingState.UNASSIGNED : this;
-        state = ShardRoutingState.UNASSIGNED;
-        currentNodeId = null;
-        relocatingNodeId = null;
-        this.unassignedInfo = unassignedInfo;
-        allocationId = null;
-        expectedShardSize = UNAVAILABLE_EXPECTED_SHARD_SIZE;
+        return new ShardRouting(shardId, null, null, restoreSource, primary, ShardRoutingState.UNASSIGNED,
+            unassignedInfo, null, UNAVAILABLE_EXPECTED_SHARD_SIZE);
     }
 
     /**
@@ -413,18 +327,17 @@ public final class ShardRouting implements Streamable, ToXContent {
      *
      * @param existingAllocationId allocation id to use. If null, a fresh allocation id is generated.
      */
-    void initialize(String nodeId, @Nullable String existingAllocationId, long expectedShardSize) {
-        ensureNotFrozen();
+    public ShardRouting initialize(String nodeId, @Nullable String existingAllocationId, long expectedShardSize) {
         assert state == ShardRoutingState.UNASSIGNED : this;
         assert relocatingNodeId == null : this;
-        state = ShardRoutingState.INITIALIZING;
-        currentNodeId = nodeId;
+        final AllocationId allocationId;
         if (existingAllocationId == null) {
             allocationId = AllocationId.newInitializing();
         } else {
             allocationId = AllocationId.newInitializing(existingAllocationId);
         }
-        this.expectedShardSize = expectedShardSize;
+        return new ShardRouting(shardId, nodeId, null, restoreSource, primary, ShardRoutingState.INITIALIZING,
+            unassignedInfo, allocationId, expectedShardSize);
     }
 
     /**
@@ -432,39 +345,31 @@ public final class ShardRouting implements Streamable, ToXContent {
      *
      * @param relocatingNodeId id of the node to relocate the shard
      */
-    void relocate(String relocatingNodeId, long expectedShardSize) {
-        ensureNotFrozen();
+    public ShardRouting relocate(String relocatingNodeId, long expectedShardSize) {
         assert state == ShardRoutingState.STARTED : "current shard has to be started in order to be relocated " + this;
-        state = ShardRoutingState.RELOCATING;
-        this.relocatingNodeId = relocatingNodeId;
-        this.allocationId = AllocationId.newRelocation(allocationId);
-        this.expectedShardSize = expectedShardSize;
+        return new ShardRouting(shardId, currentNodeId, relocatingNodeId, restoreSource, primary, ShardRoutingState.RELOCATING,
+            null, AllocationId.newRelocation(allocationId), expectedShardSize);
     }
 
     /**
      * Cancel relocation of a shard. The shards state must be set
      * to <code>RELOCATING</code>.
      */
-    void cancelRelocation() {
-        ensureNotFrozen();
+    public ShardRouting cancelRelocation() {
         assert state == ShardRoutingState.RELOCATING : this;
         assert assignedToNode() : this;
         assert relocatingNodeId != null : this;
-        expectedShardSize = UNAVAILABLE_EXPECTED_SHARD_SIZE;
-        state = ShardRoutingState.STARTED;
-        relocatingNodeId = null;
-        allocationId = AllocationId.cancelRelocation(allocationId);
+        return new ShardRouting(shardId, currentNodeId, null, restoreSource, primary, ShardRoutingState.STARTED,
+            null, AllocationId.cancelRelocation(allocationId), UNAVAILABLE_EXPECTED_SHARD_SIZE);
     }
 
     /**
      * Moves the shard from started to initializing
      */
-    void reinitializeShard() {
-        ensureNotFrozen();
+    public ShardRouting reinitializeShard() {
         assert state == ShardRoutingState.STARTED;
-        state = ShardRoutingState.INITIALIZING;
-        allocationId = AllocationId.newInitializing();
-        this.unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.REINITIALIZED, null);
+        return new ShardRouting(shardId, currentNodeId, null, restoreSource, primary, ShardRoutingState.INITIALIZING,
+            new UnassignedInfo(UnassignedInfo.Reason.REINITIALIZED, null), AllocationId.newInitializing(), UNAVAILABLE_EXPECTED_SHARD_SIZE);
     }
 
     /**
@@ -472,46 +377,46 @@ public final class ShardRouting implements Streamable, ToXContent {
      * <code>INITIALIZING</code> or <code>RELOCATING</code>. Any relocation will be
      * canceled.
      */
-    void moveToStarted() {
-        ensureNotFrozen();
+    public ShardRouting moveToStarted() {
         assert state == ShardRoutingState.INITIALIZING : "expected an initializing shard " + this;
-        relocatingNodeId = null;
-        restoreSource = null;
-        unassignedInfo = null; // we keep the unassigned data until the shard is started
+        AllocationId allocationId = this.allocationId;
         if (allocationId.getRelocationId() != null) {
             // relocation target
             allocationId = AllocationId.finishRelocation(allocationId);
         }
-        expectedShardSize = UNAVAILABLE_EXPECTED_SHARD_SIZE;
-        state = ShardRoutingState.STARTED;
+        return new ShardRouting(shardId, currentNodeId, null, restoreSource, primary, ShardRoutingState.STARTED, null, allocationId,
+            UNAVAILABLE_EXPECTED_SHARD_SIZE);
     }
 
     /**
      * Make the shard primary unless it's not Primary
-     * //TODO: doc exception
+     *
+     * @throws IllegalShardRoutingStateException if shard is already a primary
      */
-    void moveToPrimary() {
-        ensureNotFrozen();
+    public ShardRouting moveToPrimary() {
         if (primary) {
             throw new IllegalShardRoutingStateException(this, "Already primary, can't move to primary");
         }
-        primary = true;
+        return new ShardRouting(shardId, currentNodeId, relocatingNodeId, restoreSource, true, state, unassignedInfo, allocationId,
+            expectedShardSize);
     }
 
     /**
      * Set the primary shard to non-primary
+     *
+     * @throws IllegalShardRoutingStateException if shard is already a replica
      */
-    void moveFromPrimary() {
-        ensureNotFrozen();
+    public ShardRouting moveFromPrimary() {
         if (!primary) {
             throw new IllegalShardRoutingStateException(this, "Not primary, can't move to replica");
         }
-        primary = false;
+        return new ShardRouting(shardId, currentNodeId, relocatingNodeId, restoreSource, false, state, unassignedInfo, allocationId,
+            expectedShardSize);
     }
 
     /** returns true if this routing has the same shardId as another */
     public boolean isSameShard(ShardRouting other) {
-        return index.equals(other.index) && shardId == other.shardId;
+        return getIndexName().equals(other.getIndexName()) && id() == other.id();
     }
 
     /**
@@ -592,15 +497,12 @@ public final class ShardRouting implements Streamable, ToXContent {
         if (primary != other.primary) {
             return false;
         }
-        if (shardId != other.shardId) {
+        if (shardId != null ? !shardId.equals(other.shardId) : other.shardId != null) {
             return false;
         }
         if (currentNodeId != null ? !currentNodeId.equals(other.currentNodeId) : other.currentNodeId != null) {
             return false;
         }
-        if (index != null ? !index.equals(other.index) : other.index != null) {
-            return false;
-        }
         if (relocatingNodeId != null ? !relocatingNodeId.equals(other.relocatingNodeId) : other.relocatingNodeId != null) {
             return false;
         }
@@ -631,27 +533,27 @@ public final class ShardRouting implements Streamable, ToXContent {
         return equalsIgnoringMetaData(that);
     }
 
-    private boolean usePreComputedHashCode = false;
-    private int hashCode = 0;
+    /**
+     * Cache hash code in same same way as {@link String#hashCode()}) using racy single-check idiom
+     * as it is mainly used in single-threaded code ({@link BalancedShardsAllocator}).
+     */
+    private int hashCode; // default to 0
 
     @Override
     public int hashCode() {
-        if (frozen && usePreComputedHashCode) {
-            return hashCode;
+        int h = hashCode;
+        if (h == 0) {
+            h = shardId.hashCode();
+            h = 31 * h + (currentNodeId != null ? currentNodeId.hashCode() : 0);
+            h = 31 * h + (relocatingNodeId != null ? relocatingNodeId.hashCode() : 0);
+            h = 31 * h + (primary ? 1 : 0);
+            h = 31 * h + (state != null ? state.hashCode() : 0);
+            h = 31 * h + (restoreSource != null ? restoreSource.hashCode() : 0);
+            h = 31 * h + (allocationId != null ? allocationId.hashCode() : 0);
+            h = 31 * h + (unassignedInfo != null ? unassignedInfo.hashCode() : 0);
+            hashCode = h;
         }
-        int result = index != null ? index.hashCode() : 0;
-        result = 31 * result + shardId;
-        result = 31 * result + (currentNodeId != null ? currentNodeId.hashCode() : 0);
-        result = 31 * result + (relocatingNodeId != null ? relocatingNodeId.hashCode() : 0);
-        result = 31 * result + (primary ? 1 : 0);
-        result = 31 * result + (state != null ? state.hashCode() : 0);
-        result = 31 * result + (restoreSource != null ? restoreSource.hashCode() : 0);
-        result = 31 * result + (allocationId != null ? allocationId.hashCode() : 0);
-        result = 31 * result + (unassignedInfo != null ? unassignedInfo.hashCode() : 0);
-        if (frozen) {
-            usePreComputedHashCode = true;
-        }
-        return hashCode = result;
+        return h;
     }
 
     @Override
@@ -664,7 +566,7 @@ public final class ShardRouting implements Streamable, ToXContent {
      */
     public String shortSummary() {
         StringBuilder sb = new StringBuilder();
-        sb.append('[').append(index).append(']').append('[').append(shardId).append(']');
+        sb.append('[').append(shardId.getIndexName()).append(']').append('[').append(shardId.getId()).append(']');
         sb.append(", node[").append(currentNodeId).append("], ");
         if (relocatingNodeId != null) {
             sb.append("relocating [").append(relocatingNodeId).append("], ");
@@ -697,8 +599,8 @@ public final class ShardRouting implements Streamable, ToXContent {
             .field("primary", primary())
             .field("node", currentNodeId())
             .field("relocating_node", relocatingNodeId())
-            .field("shard", shardId().id())
-            .field("index", shardId().getIndex().getName());
+            .field("shard", id())
+            .field("index", getIndexName());
         if (expectedShardSize != UNAVAILABLE_EXPECTED_SHARD_SIZE) {
             builder.field("expected_shard_size_in_bytes", expectedShardSize);
         }
@@ -716,20 +618,6 @@ public final class ShardRouting implements Streamable, ToXContent {
         return builder.endObject();
     }
 
-    private void ensureNotFrozen() {
-        if (frozen) {
-            throw new IllegalStateException("ShardRouting can't be modified anymore - already frozen");
-        }
-    }
-
-    void freeze() {
-        frozen = true;
-    }
-
-    boolean isFrozen() {
-        return frozen;
-    }
-
     /**
      * Returns the expected shard size for {@link ShardRoutingState#RELOCATING} and {@link ShardRoutingState#INITIALIZING}
      * shards. If it's size is not available {@value #UNAVAILABLE_EXPECTED_SHARD_SIZE} will be returned.
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java b/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
index 92f42e2bc4f..2670363364d 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
@@ -40,7 +40,7 @@ import java.io.IOException;
 /**
  * Holds additional information as to why the shard is in unassigned state.
  */
-public class UnassignedInfo implements ToXContent, Writeable {
+public final class UnassignedInfo implements ToXContent, Writeable {
 
     public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("dateOptionalTime");
     private static final TimeValue DEFAULT_DELAYED_NODE_LEFT_TIMEOUT = TimeValue.timeValueMinutes(1);
@@ -109,7 +109,7 @@ public class UnassignedInfo implements ToXContent, Writeable {
     private final Reason reason;
     private final long unassignedTimeMillis; // used for display and log messages, in milliseconds
     private final long unassignedTimeNanos; // in nanoseconds, used to calculate delay for delayed shard allocation
-    private volatile long lastComputedLeftDelayNanos = 0L; // how long to delay shard allocation, not serialized (always positive, 0 means no delay)
+    private final long lastComputedLeftDelayNanos; // how long to delay shard allocation, not serialized (always positive, 0 means no delay)
     private final String message;
     private final Throwable failure;
 
@@ -134,17 +134,28 @@ public class UnassignedInfo implements ToXContent, Writeable {
         this.reason = reason;
         this.unassignedTimeMillis = unassignedTimeMillis;
         this.unassignedTimeNanos = unassignedTimeNanos;
+        this.lastComputedLeftDelayNanos = 0L;
         this.message = message;
         this.failure = failure;
         assert !(message == null && failure != null) : "provide a message if a failure exception is provided";
     }
 
+    public UnassignedInfo(UnassignedInfo unassignedInfo, long newComputedLeftDelayNanos) {
+        this.reason = unassignedInfo.reason;
+        this.unassignedTimeMillis = unassignedInfo.unassignedTimeMillis;
+        this.unassignedTimeNanos = unassignedInfo.unassignedTimeNanos;
+        this.lastComputedLeftDelayNanos = newComputedLeftDelayNanos;
+        this.message = unassignedInfo.message;
+        this.failure = unassignedInfo.failure;
+    }
+
     public UnassignedInfo(StreamInput in) throws IOException {
         this.reason = Reason.values()[(int) in.readByte()];
         this.unassignedTimeMillis = in.readLong();
         // As System.nanoTime() cannot be compared across different JVMs, reset it to now.
         // This means that in master failover situations, elapsed delay time is forgotten.
         this.unassignedTimeNanos = System.nanoTime();
+        this.lastComputedLeftDelayNanos = 0L;
         this.message = in.readOptionalString();
         this.failure = in.readThrowable();
     }
@@ -247,14 +258,16 @@ public class UnassignedInfo implements ToXContent, Writeable {
     }
 
     /**
-     * Updates delay left based on current time (in nanoseconds) and index/node settings.
+     * Creates new UnassignedInfo object if delay needs updating.
      *
-     * @return updated delay in nanoseconds
+     * @return new Unassigned with updated delay, or this if no change in delay
      */
-    public long updateDelay(final long nanoTimeNow, final Settings settings, final Settings indexSettings) {
+    public UnassignedInfo updateDelay(final long nanoTimeNow, final Settings settings, final Settings indexSettings) {
         final long newComputedLeftDelayNanos = getRemainingDelay(nanoTimeNow, settings, indexSettings);
-        lastComputedLeftDelayNanos = newComputedLeftDelayNanos;
-        return newComputedLeftDelayNanos;
+        if (lastComputedLeftDelayNanos == newComputedLeftDelayNanos) {
+            return this;
+        }
+        return new UnassignedInfo(this, newComputedLeftDelayNanos);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
index ed6de16c11f..608477e55fe 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
@@ -353,10 +353,17 @@ public class AllocationService extends AbstractComponent {
 
     // public for testing
     public static void updateLeftDelayOfUnassignedShards(RoutingAllocation allocation, Settings settings) {
-        for (ShardRouting shardRouting : allocation.routingNodes().unassigned()) {
-            final MetaData metaData = allocation.metaData();
+        final RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator = allocation.routingNodes().unassigned().iterator();
+        final MetaData metaData = allocation.metaData();
+        while (unassignedIterator.hasNext()) {
+            ShardRouting shardRouting = unassignedIterator.next();
             final IndexMetaData indexMetaData = metaData.getIndexSafe(shardRouting.index());
-            shardRouting.unassignedInfo().updateDelay(allocation.getCurrentNanoTime(), settings, indexMetaData.getSettings());
+            UnassignedInfo previousUnassignedInfo = shardRouting.unassignedInfo();
+            UnassignedInfo updatedUnassignedInfo = previousUnassignedInfo.updateDelay(allocation.getCurrentNanoTime(), settings,
+                indexMetaData.getSettings());
+            if (updatedUnassignedInfo != previousUnassignedInfo) { // reference equality!
+                unassignedIterator.updateUnassignedInfo(updatedUnassignedInfo);
+            }
         }
     }
 
@@ -369,29 +376,32 @@ public class AllocationService extends AbstractComponent {
         }
         // now, go over and elect a new primary if possible, not, from this code block on, if one is elected,
         // routingNodes.hasUnassignedPrimaries() will potentially be false
-        for (ShardRouting shardEntry : routingNodes.unassigned()) {
+        final RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator = routingNodes.unassigned().iterator();
+        while (unassignedIterator.hasNext()) {
+            ShardRouting shardEntry = unassignedIterator.next();
             if (shardEntry.primary()) {
                 // remove dangling replicas that are initializing for primary shards
                 changed |= failReplicasForUnassignedPrimary(allocation, shardEntry);
                 ShardRouting candidate = allocation.routingNodes().activeReplica(shardEntry);
                 if (candidate != null) {
-                    routingNodes.swapPrimaryFlag(shardEntry, candidate);
-                    if (candidate.relocatingNodeId() != null) {
+                    shardEntry = unassignedIterator.demotePrimaryToReplicaShard();
+                    ShardRouting primarySwappedCandidate = routingNodes.promoteAssignedReplicaShardToPrimary(candidate);
+                    if (primarySwappedCandidate.relocatingNodeId() != null) {
                         changed = true;
                         // its also relocating, make sure to move the other routing to primary
-                        RoutingNode node = routingNodes.node(candidate.relocatingNodeId());
+                        RoutingNode node = routingNodes.node(primarySwappedCandidate.relocatingNodeId());
                         if (node != null) {
                             for (ShardRouting shardRouting : node) {
-                                if (shardRouting.shardId().equals(candidate.shardId()) && !shardRouting.primary()) {
-                                    routingNodes.swapPrimaryFlag(shardRouting);
+                                if (shardRouting.shardId().equals(primarySwappedCandidate.shardId()) && !shardRouting.primary()) {
+                                    routingNodes.promoteAssignedReplicaShardToPrimary(shardRouting);
                                     break;
                                 }
                             }
                         }
                     }
-                    IndexMetaData index = allocation.metaData().getIndexSafe(candidate.index());
+                    IndexMetaData index = allocation.metaData().getIndexSafe(primarySwappedCandidate.index());
                     if (IndexMetaData.isIndexUsingShadowReplicas(index.getSettings())) {
-                        routingNodes.reinitShadowPrimary(candidate);
+                        routingNodes.reinitShadowPrimary(primarySwappedCandidate);
                         changed = true;
                     }
                 }
@@ -466,24 +476,26 @@ public class AllocationService extends AbstractComponent {
             }
 
 
-            RoutingNodes.RoutingNodeIterator currentRoutingNode = routingNodes.routingNodeIter(startedShard.currentNodeId());
+            RoutingNode currentRoutingNode = routingNodes.node(startedShard.currentNodeId());
             if (currentRoutingNode == null) {
                 logger.debug("{} failed to find shard in order to start it [failed to find node], ignoring (routing: {})", startedShard.shardId(), startedShard);
                 continue;
             }
 
-            for (ShardRouting shard : currentRoutingNode) {
-                if (shard.isSameAllocation(startedShard)) {
-                    if (shard.active()) {
-                        logger.trace("{} shard is already started, ignoring (routing: {})", startedShard.shardId(), startedShard);
-                    } else {
-                        dirty = true;
-                        // override started shard with the latest copy. Capture it now , before starting the shard destroys it...
-                        startedShard = new ShardRouting(shard);
-                        routingNodes.started(shard);
-                        logger.trace("{} marked shard as started (routing: {})", startedShard.shardId(), startedShard);
-                    }
-                    break;
+            ShardRouting matchingShard = currentRoutingNode.getByShardId(startedShard.shardId());
+            if (matchingShard == null) {
+                logger.debug("{} failed to find shard in order to start it [failed to find shard], ignoring (routing: {})", startedShard.shardId(), startedShard);
+            } else if (matchingShard.isSameAllocation(startedShard) == false) {
+                logger.debug("{} failed to find shard with matching allocation id in order to start it [failed to find matching shard], ignoring (routing: {}, matched shard routing: {})", startedShard.shardId(), startedShard, matchingShard);
+            } else {
+                if (matchingShard.active()) {
+                    logger.trace("{} shard is already started, ignoring (routing: {})", startedShard.shardId(), startedShard);
+                } else {
+                    dirty = true;
+                    // override started shard with the latest copy.
+                    startedShard = matchingShard;
+                    routingNodes.started(matchingShard);
+                    logger.trace("{} marked shard as started (routing: {})", startedShard.shardId(), startedShard);
                 }
             }
 
@@ -545,8 +557,8 @@ public class AllocationService extends AbstractComponent {
             // fail replicas first otherwise we move RoutingNodes into an inconsistent state
             failReplicasForUnassignedPrimary(allocation, failedShard);
         }
-        // replace incoming instance to make sure we work on the latest one. Copy it to maintain information during modifications.
-        failedShard = new ShardRouting(matchedNode.current());
+        // replace incoming instance to make sure we work on the latest one
+        failedShard = matchedNode.current();
 
         // remove the current copy of the shard
         matchedNode.remove();
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
index 296e4929ec5..7fd3a03a23e 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
@@ -32,6 +32,7 @@ import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.ESLogger;
@@ -512,28 +513,19 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
             // shard movements, the goal of this iteration order is to achieve a fairer movement of shards from the nodes that are
             // offloading the shards.
             boolean changed = false;
-            int index = 0;
-            boolean found = true;
             final NodeSorter sorter = newNodeSorter();
-            while (found) {
-                found = false;
-                for (RoutingNode routingNode : allocation.routingNodes()) {
-                    if (index >= routingNode.size()) {
-                        continue;
-                    }
-                    found = true;
-                    ShardRouting shardRouting = routingNode.get(index);
-                    // we can only move started shards...
-                    if (shardRouting.started()) {
-                        final ModelNode sourceNode = nodes.get(shardRouting.currentNodeId());
-                        assert sourceNode != null && sourceNode.containsShard(shardRouting);
-                        Decision decision = allocation.deciders().canRemain(shardRouting, routingNode, allocation);
-                        if (decision.type() == Decision.Type.NO) {
-                            changed |= moveShard(sorter, shardRouting, sourceNode, routingNode);
-                        }
+            for (Iterator<ShardRouting> it = allocation.routingNodes().nodeInterleavedShardIterator(); it.hasNext(); ) {
+                ShardRouting shardRouting = it.next();
+                // we can only move started shards...
+                if (shardRouting.started()) {
+                    final ModelNode sourceNode = nodes.get(shardRouting.currentNodeId());
+                    assert sourceNode != null && sourceNode.containsShard(shardRouting);
+                    RoutingNode routingNode = sourceNode.getRoutingNode();
+                    Decision decision = allocation.deciders().canRemain(shardRouting, routingNode, allocation);
+                    if (decision.type() == Decision.Type.NO) {
+                        changed |= moveShard(sorter, shardRouting, sourceNode, routingNode);
                     }
                 }
-                index++;
             }
 
             return changed;
@@ -560,8 +552,8 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
                     Decision allocationDecision = allocation.deciders().canAllocate(shardRouting, target, allocation);
                     if (allocationDecision.type() == Type.YES) { // TODO maybe we can respect throttling here too?
                         sourceNode.removeShard(shardRouting);
-                        ShardRouting targetRelocatingShard = routingNodes.relocate(shardRouting, target.nodeId(), allocation.clusterInfo().getShardSize(shardRouting, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE));
-                        currentNode.addShard(targetRelocatingShard);
+                        Tuple<ShardRouting, ShardRouting> relocatingShards = routingNodes.relocate(shardRouting, target.nodeId(), allocation.clusterInfo().getShardSize(shardRouting, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE));
+                        currentNode.addShard(relocatingShards.v2());
                         if (logger.isTraceEnabled()) {
                             logger.trace("Moved shard [{}] to node [{}]", shardRouting, routingNode.node());
                         }
@@ -729,15 +721,17 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
                     }
                     assert decision != null && minNode != null || decision == null && minNode == null;
                     if (minNode != null) {
-                        minNode.addShard(shard);
+                        long shardSize = allocation.clusterInfo().getShardSize(shard, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
                         if (decision.type() == Type.YES) {
                             if (logger.isTraceEnabled()) {
                                 logger.trace("Assigned shard [{}] to [{}]", shard, minNode.getNodeId());
                             }
-                            routingNodes.initialize(shard, minNode.getNodeId(), null, allocation.clusterInfo().getShardSize(shard, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE));
+                            shard = routingNodes.initialize(shard, minNode.getNodeId(), null, shardSize);
+                            minNode.addShard(shard);
                             changed = true;
                             continue; // don't add to ignoreUnassigned
                         } else {
+                            minNode.addShard(shard.initialize(minNode.getNodeId(), null, shardSize));
                             final RoutingNode node = minNode.getRoutingNode();
                             if (deciders.canAllocate(node, allocation).type() != Type.YES) {
                                 if (logger.isTraceEnabled()) {
@@ -810,15 +804,19 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
                 if (candidate != null) {
                     /* allocate on the model even if not throttled */
                     maxNode.removeShard(candidate);
-                    minNode.addShard(candidate);
+                    long shardSize = allocation.clusterInfo().getShardSize(candidate, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
+
                     if (decision.type() == Type.YES) { /* only allocate on the cluster if we are not throttled */
                         if (logger.isTraceEnabled()) {
                             logger.trace("Relocate shard [{}] from node [{}] to node [{}]", candidate, maxNode.getNodeId(),
                                     minNode.getNodeId());
                         }
                         /* now allocate on the cluster */
-                        routingNodes.relocate(candidate, minNode.getNodeId(), allocation.clusterInfo().getShardSize(candidate, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE));
+                        minNode.addShard(routingNodes.relocate(candidate, minNode.getNodeId(), shardSize).v1());
                         return true;
+                    } else {
+                        assert decision.type() == Type.THROTTLE;
+                        minNode.addShard(candidate.relocate(minNode.getNodeId(), shardSize));
                     }
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java
index 50e71c5094a..20918159619 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java
@@ -23,6 +23,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.RerouteExplanation;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
@@ -38,6 +39,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.function.Consumer;
+import java.util.function.Function;
 
 /**
  * Abstract base class for allocating an unassigned shard to a node
@@ -196,17 +198,17 @@ public abstract class AbstractAllocateAllocationCommand implements AllocationCom
      * @param routingNodes the routing nodes
      * @param routingNode the node to initialize it to
      * @param shardRouting the shard routing that is to be matched in unassigned shards
-     * @param shardRoutingChanges changes to apply for shard routing in unassigned shards before initialization
+     * @param unassignedInfo unassigned info to override
      */
     protected void initializeUnassignedShard(RoutingAllocation allocation, RoutingNodes routingNodes, RoutingNode routingNode,
-                                             ShardRouting shardRouting, @Nullable Consumer<ShardRouting> shardRoutingChanges) {
+                                             ShardRouting shardRouting, @Nullable UnassignedInfo unassignedInfo) {
         for (RoutingNodes.UnassignedShards.UnassignedIterator it = routingNodes.unassigned().iterator(); it.hasNext(); ) {
             ShardRouting unassigned = it.next();
             if (!unassigned.equalsIgnoringMetaData(shardRouting)) {
                 continue;
             }
-            if (shardRoutingChanges != null) {
-                shardRoutingChanges.accept(unassigned);
+            if (unassignedInfo != null) {
+                unassigned = it.updateUnassignedInfo(unassignedInfo);
             }
             it.initialize(routingNode.nodeId(), null, allocation.clusterInfo().getShardSize(unassigned, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE));
             return;
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java
index 74e3a18efbc..d4191292cfc 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java
@@ -120,15 +120,16 @@ public class AllocateEmptyPrimaryAllocationCommand extends BasePrimaryAllocation
                 "allocating an empty primary for [" + index + "][" + shardId + "] can result in data loss. Please confirm by setting the accept_data_loss parameter to true");
         }
 
-        initializeUnassignedShard(allocation, routingNodes, routingNode, shardRouting,
-            shr -> {
-                if (shr.unassignedInfo().getReason() != UnassignedInfo.Reason.INDEX_CREATED) {
-                    // we need to move the unassigned info back to treat it as if it was index creation
-                    shr.updateUnassignedInfo(new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED,
-                        "force empty allocation from previous reason " + shardRouting.unassignedInfo().getReason() + ", " + shardRouting.unassignedInfo().getMessage(),
-                        shardRouting.unassignedInfo().getFailure(), System.nanoTime(), System.currentTimeMillis()));
-                }
-            });
+        UnassignedInfo unassignedInfoToUpdate = null;
+        if (shardRouting.unassignedInfo().getReason() != UnassignedInfo.Reason.INDEX_CREATED) {
+            // we need to move the unassigned info back to treat it as if it was index creation
+            unassignedInfoToUpdate = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED,
+                "force empty allocation from previous reason " + shardRouting.unassignedInfo().getReason() + ", " + shardRouting.unassignedInfo().getMessage(),
+                shardRouting.unassignedInfo().getFailure(), System.nanoTime(), System.currentTimeMillis());
+        }
+
+        initializeUnassignedShard(allocation, routingNodes, routingNode, shardRouting, unassignedInfoToUpdate);
+
         return new RerouteExplanation(this, allocation.decision(Decision.YES, name() + " (allocation command)", "ignore deciders"));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java
index 3b2f0fbd55b..60b0842b273 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java
@@ -155,7 +155,7 @@ public class CancelAllocationCommand implements AllocationCommand {
                         throw new IllegalArgumentException("[cancel_allocation] can't cancel " + shardId + " on node " +
                                 discoNode + ", shard is primary and initializing its state");
                     }
-                    it.moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.REROUTE_CANCELLED, null));
+                    shardRouting = it.moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.REROUTE_CANCELLED, null));
                     // now, go and find the shard that is initializing on the target node, and cancel it as well...
                     RoutingNodes.RoutingNodeIterator initializingNode = allocation.routingNodes().routingNodeIter(shardRouting.relocatingNodeId());
                     if (initializingNode != null) {
diff --git a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
index 74a71c44735..4487a36c0ae 100644
--- a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
+++ b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
@@ -30,11 +30,9 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RestoreSource;
 import org.elasticsearch.cluster.routing.RoutingNode;
-import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -194,27 +192,16 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
     }
 
     private void cleanFailedShards(final ClusterChangedEvent event) {
-        RoutingNodes.RoutingNodeIterator routingNode = event.state().getRoutingNodes().routingNodeIter(event.state().nodes().getLocalNodeId());
+        RoutingNode routingNode = event.state().getRoutingNodes().node(event.state().nodes().getLocalNodeId());
         if (routingNode == null) {
             failedShards.clear();
             return;
         }
-        RoutingTable routingTable = event.state().routingTable();
         for (Iterator<Map.Entry<ShardId, ShardRouting>> iterator = failedShards.entrySet().iterator(); iterator.hasNext(); ) {
             Map.Entry<ShardId, ShardRouting> entry = iterator.next();
-            ShardId failedShardId = entry.getKey();
             ShardRouting failedShardRouting = entry.getValue();
-            IndexRoutingTable indexRoutingTable = routingTable.index(failedShardId.getIndex());
-            if (indexRoutingTable == null) {
-                iterator.remove();
-                continue;
-            }
-            IndexShardRoutingTable shardRoutingTable = indexRoutingTable.shard(failedShardId.id());
-            if (shardRoutingTable == null) {
-                iterator.remove();
-                continue;
-            }
-            if (shardRoutingTable.assignedShards().stream().noneMatch(shr -> shr.isSameAllocation(failedShardRouting))) {
+            ShardRouting matchedShardRouting = routingNode.getByShardId(failedShardRouting.shardId());
+            if (matchedShardRouting == null || matchedShardRouting.isSameAllocation(failedShardRouting) == false) {
                 iterator.remove();
             }
         }
@@ -279,7 +266,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
     }
 
     private void applyDeletedShards(final ClusterChangedEvent event) {
-        RoutingNodes.RoutingNodeIterator routingNode = event.state().getRoutingNodes().routingNodeIter(event.state().nodes().getLocalNodeId());
+        RoutingNode routingNode = event.state().getRoutingNodes().node(event.state().nodes().getLocalNodeId());
         if (routingNode == null) {
             return;
         }
@@ -382,7 +369,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
 
     private void applyNewIndices(final ClusterChangedEvent event) {
         // we only create indices for shards that are allocated
-        RoutingNodes.RoutingNodeIterator routingNode = event.state().getRoutingNodes().routingNodeIter(event.state().nodes().getLocalNodeId());
+        RoutingNode routingNode = event.state().getRoutingNodes().node(event.state().nodes().getLocalNodeId());
         if (routingNode == null) {
             return;
         }
@@ -479,7 +466,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
         }
 
         RoutingTable routingTable = event.state().routingTable();
-        RoutingNodes.RoutingNodeIterator routingNode = event.state().getRoutingNodes().routingNodeIter(event.state().nodes().getLocalNodeId());
+        RoutingNode routingNode = event.state().getRoutingNodes().node(event.state().nodes().getLocalNodeId());
 
         if (routingNode == null) {
             failedShards.clear();
diff --git a/core/src/main/java/org/elasticsearch/indices/flush/ShardsSyncedFlushResult.java b/core/src/main/java/org/elasticsearch/indices/flush/ShardsSyncedFlushResult.java
index 220ce9120e9..04564943ca7 100644
--- a/core/src/main/java/org/elasticsearch/indices/flush/ShardsSyncedFlushResult.java
+++ b/core/src/main/java/org/elasticsearch/indices/flush/ShardsSyncedFlushResult.java
@@ -141,7 +141,7 @@ public class ShardsSyncedFlushResult implements Streamable {
         int numResponses = in.readInt();
         shardResponses = new HashMap<>();
         for (int i = 0; i < numResponses; i++) {
-            ShardRouting shardRouting = ShardRouting.readShardRoutingEntry(in);
+            ShardRouting shardRouting = new ShardRouting(in);
             SyncedFlushService.ShardSyncedFlushResponse response = SyncedFlushService.ShardSyncedFlushResponse.readSyncedFlushResponse(in);
             shardResponses.put(shardRouting, response);
         }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
index ab39e6dabf7..1f6df76c82d 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
@@ -59,9 +59,9 @@ import static java.util.Collections.emptySet;
 public final class ClusterAllocationExplanationTests extends ESTestCase {
 
     private Index i = new Index("foo", "uuid");
-    private ShardRouting primaryShard = ShardRouting.newUnassigned(i, 0, null, true,
+    private ShardRouting primaryShard = ShardRouting.newUnassigned(new ShardId(i, 0), null, true,
             new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
-    private ShardRouting replicaShard = ShardRouting.newUnassigned(i, 0, null, false,
+    private ShardRouting replicaShard = ShardRouting.newUnassigned(new ShardId(i, 0), null, false,
             new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
     private IndexMetaData indexMetaData = IndexMetaData.builder("foo")
             .settings(Settings.builder()
@@ -114,7 +114,7 @@ public final class ClusterAllocationExplanationTests extends ESTestCase {
         Float nodeWeight = randomFloat();
         Set<String> activeAllocationIds = new HashSet<>();
         activeAllocationIds.add("eggplant");
-        ShardRouting primaryStartedShard = ShardRouting.newUnassigned(i, 0, null, true,
+        ShardRouting primaryStartedShard = ShardRouting.newUnassigned(new ShardId(i, 0), null, true,
                 new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, "foo"));
         assertTrue(primaryStartedShard.allocatedPostIndexCreate(indexMetaData));
 
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
index e9a92ff25b9..d88960ea9ac 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
@@ -43,6 +43,7 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -503,8 +504,7 @@ public class TransportReplicationActionTests extends ESTestCase {
         // check that at unknown node fails
         PlainActionFuture<TransportResponse.Empty> listener = new PlainActionFuture<>();
         proxy.performOn(
-            TestShardRouting.newShardRouting(shardId.getIndex(), shardId.id(), "NOT THERE", false,
-                randomFrom(ShardRoutingState.values())),
+            TestShardRouting.newShardRouting(shardId, "NOT THERE", false, randomFrom(ShardRoutingState.values())),
             new Request(), listener);
         assertTrue(listener.isDone());
         assertListenerThrows("non existent node should throw a NoNodeAvailableException", listener, NoNodeAvailableException.class);
@@ -593,7 +593,6 @@ public class TransportReplicationActionTests extends ESTestCase {
         primaryPhase.messageReceived(new Request(shardId), createTransportChannel(new PlainActionFuture<>()), null);
     }
 
-
     public void testCounterOnPrimary() throws Exception {
         final String index = "test";
         final ShardId shardId = new ShardId(index, "_na_", 0);
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
index b6b4af3fd44..65f451ac85c 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
@@ -245,7 +245,7 @@ public class ClusterStateDiffIT extends ESIntegTestCase {
                 }
                 indexShard.addShard(
                         TestShardRouting.newShardRouting(index, i, randomFrom(nodeIds), null, null, j == 0,
-                                ShardRoutingState.fromValue((byte) randomIntBetween(2, 4)), unassignedInfo));
+                                ShardRoutingState.fromValue((byte) randomIntBetween(2, 3)), unassignedInfo));
             }
             builder.addIndexShard(indexShard.build());
         }
@@ -259,9 +259,8 @@ public class ClusterStateDiffIT extends ESIntegTestCase {
         IndexRoutingTable.Builder builder = IndexRoutingTable.builder(original.getIndex());
         for (ObjectCursor<IndexShardRoutingTable> indexShardRoutingTable :  original.shards().values()) {
             for (ShardRouting shardRouting : indexShardRoutingTable.value.shards()) {
-                final ShardRouting newShardRouting = new ShardRouting(shardRouting);
-                randomChange(newShardRouting, nodes);
-                builder.addShard(indexShardRoutingTable.value, newShardRouting);
+                final ShardRouting updatedShardRouting = randomChange(shardRouting, nodes);
+                builder.addShard(indexShardRoutingTable.value, updatedShardRouting);
             }
         }
         return builder.build();
diff --git a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
index 31c9c21b8e7..d7738b1abcb 100644
--- a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
@@ -35,6 +35,7 @@ import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardPath;
 import org.elasticsearch.index.store.StoreStats;
 import org.elasticsearch.monitor.fs.FsInfo;
@@ -102,15 +103,15 @@ public class DiskUsageTests extends ESTestCase {
 
     public void testFillShardLevelInfo() {
         final Index index = new Index("test", "0xdeadbeef");
-        ShardRouting test_0 = ShardRouting.newUnassigned(index, 0, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
-        ShardRoutingHelper.initialize(test_0, "node1");
-        ShardRoutingHelper.moveToStarted(test_0);
+        ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        test_0 = ShardRoutingHelper.initialize(test_0, "node1");
+        test_0 = ShardRoutingHelper.moveToStarted(test_0);
         Path test0Path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve("0");
         CommonStats commonStats0 = new CommonStats();
         commonStats0.store = new StoreStats(100, 1);
-        ShardRouting test_1 = ShardRouting.newUnassigned(index, 1, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
-        ShardRoutingHelper.initialize(test_1, "node2");
-        ShardRoutingHelper.moveToStarted(test_1);
+        ShardRouting test_1 = ShardRouting.newUnassigned(new ShardId(index, 1), null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        test_1 = ShardRoutingHelper.initialize(test_1, "node2");
+        test_1 = ShardRoutingHelper.moveToStarted(test_1);
         Path test1Path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve("1");
         CommonStats commonStats1 = new CommonStats();
         commonStats1.store = new StoreStats(1000, 1);
@@ -137,16 +138,16 @@ public class DiskUsageTests extends ESTestCase {
 
     public void testFillShardsWithShadowIndices() {
         final Index index = new Index("non-shadow", "0xcafe0000");
-        ShardRouting s0 = ShardRouting.newUnassigned(index, 0, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
-        ShardRoutingHelper.initialize(s0, "node1");
-        ShardRoutingHelper.moveToStarted(s0);
+        ShardRouting s0 = ShardRouting.newUnassigned(new ShardId(index, 0), null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        s0 = ShardRoutingHelper.initialize(s0, "node1");
+        s0 = ShardRoutingHelper.moveToStarted(s0);
         Path i0Path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve("0");
         CommonStats commonStats0 = new CommonStats();
         commonStats0.store = new StoreStats(100, 1);
         final Index index2 = new Index("shadow", "0xcafe0001");
-        ShardRouting s1 = ShardRouting.newUnassigned(index2, 0, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
-        ShardRoutingHelper.initialize(s1, "node2");
-        ShardRoutingHelper.moveToStarted(s1);
+        ShardRouting s1 = ShardRouting.newUnassigned(new ShardId(index2, 0), null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        s1 = ShardRoutingHelper.initialize(s1, "node2");
+        s1 = ShardRoutingHelper.moveToStarted(s1);
         Path i1Path = createTempDir().resolve("indices").resolve(index2.getUUID()).resolve("0");
         CommonStats commonStats1 = new CommonStats();
         commonStats1.store = new StoreStats(1000, 1);
diff --git a/core/src/test/java/org/elasticsearch/cluster/ack/AckIT.java b/core/src/test/java/org/elasticsearch/cluster/ack/AckIT.java
index fdf4dfaaae8..0a0732252b7 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ack/AckIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ack/AckIT.java
@@ -36,6 +36,7 @@ import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
+import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.index.Index;
@@ -189,7 +190,7 @@ public class AckIT extends ESIntegTestCase {
             if (routingNode.node().isDataNode()) {
                 if (fromNodeId == null && routingNode.numberOfOwningShards() > 0) {
                     fromNodeId = routingNode.nodeId();
-                    shardToBeMoved = routingNode.get(randomInt(routingNode.size() - 1));
+                    shardToBeMoved = routingNode.copyShards().get(randomInt(routingNode.size() - 1));
                 } else {
                     toNodeId = routingNode.nodeId();
                 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java
index 317bb6cfc47..426a574e2cd 100644
--- a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java
@@ -43,6 +43,7 @@ import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESAllocationTestCase;
 import org.junit.Before;
 
@@ -200,7 +201,7 @@ public class ShardFailedClusterStateTaskExecutorTests extends ESAllocationTestCa
         for (ShardStateAction.ShardRoutingEntry existingShard : existingShards) {
             ShardRouting sr = existingShard.getShardRouting();
             ShardRouting nonExistentShardRouting =
-                TestShardRouting.newShardRouting(sr.index(), sr.id(), sr.currentNodeId(), sr.relocatingNodeId(), sr.restoreSource(), sr.primary(), sr.state());
+                TestShardRouting.newShardRouting(sr.shardId(), sr.currentNodeId(), sr.relocatingNodeId(), sr.restoreSource(), sr.primary(), sr.state());
             shardsWithMismatchedAllocationIds.add(new ShardStateAction.ShardRoutingEntry(nonExistentShardRouting, nonExistentShardRouting, existingShard.message, existingShard.failure));
         }
 
@@ -211,7 +212,7 @@ public class ShardFailedClusterStateTaskExecutorTests extends ESAllocationTestCa
     }
 
     private ShardRouting nonExistentShardRouting(Index index, List<String> nodeIds, boolean primary) {
-        return TestShardRouting.newShardRouting(index, 0, randomFrom(nodeIds), primary, randomFrom(ShardRoutingState.INITIALIZING, ShardRoutingState.RELOCATING, ShardRoutingState.STARTED));
+        return TestShardRouting.newShardRouting(new ShardId(index, 0), randomFrom(nodeIds), primary, randomFrom(ShardRoutingState.INITIALIZING, ShardRoutingState.RELOCATING, ShardRoutingState.STARTED));
     }
 
     private static void assertTasksSuccessful(
@@ -304,7 +305,7 @@ public class ShardFailedClusterStateTaskExecutorTests extends ESAllocationTestCa
             return randomSubsetOf(1, shards.toArray(new ShardRouting[0])).get(0);
         } else {
             return
-                    TestShardRouting.newShardRouting(shardRouting.index(), shardRouting.id(), DiscoveryNodeService.generateNodeId(Settings.EMPTY), randomBoolean(), randomFrom(ShardRoutingState.values()));
+                    TestShardRouting.newShardRouting(shardRouting.shardId(), DiscoveryNodeService.generateNodeId(Settings.EMPTY), randomBoolean(), randomFrom(ShardRoutingState.values()));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
index 6085bf92c32..fabee819a11 100644
--- a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
@@ -338,7 +338,7 @@ public class ShardStateActionTests extends ESTestCase {
         AtomicReference<Throwable> failure = new AtomicReference<>();
         CountDownLatch latch = new CountDownLatch(1);
 
-        ShardRouting sourceFailedShard = TestShardRouting.newShardRouting(failedShard.index(), failedShard.id(), nodeId, randomBoolean(), randomFrom(ShardRoutingState.values()));
+        ShardRouting sourceFailedShard = TestShardRouting.newShardRouting(failedShard.shardId(), nodeId, randomBoolean(), randomFrom(ShardRoutingState.values()));
         shardStateAction.shardFailed(failedShard, sourceFailedShard, "test", getSimulatedFailure(), new ShardStateAction.Listener() {
             @Override
             public void onSuccess() {
diff --git a/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java b/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java
index a6bafb912a2..0d43580c9bc 100644
--- a/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java
@@ -104,7 +104,7 @@ public class ClusterRerouteIT extends ESIntegTestCase {
                 .setDryRun(true)
                 .execute().actionGet().getState();
         assertThat(state.getRoutingNodes().unassigned().size(), equalTo(1));
-        assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_1).getId()).get(0).state(), equalTo(ShardRoutingState.INITIALIZING));
+        assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_1).getId()).iterator().next().state(), equalTo(ShardRoutingState.INITIALIZING));
 
         logger.info("--> get the state, verify nothing changed because of the dry run");
         state = client().admin().cluster().prepareState().execute().actionGet().getState();
@@ -116,7 +116,7 @@ public class ClusterRerouteIT extends ESIntegTestCase {
                 .add(new AllocateEmptyPrimaryAllocationCommand("test", 0, node_1, true))
                 .execute().actionGet().getState();
         assertThat(state.getRoutingNodes().unassigned().size(), equalTo(1));
-        assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_1).getId()).get(0).state(), equalTo(ShardRoutingState.INITIALIZING));
+        assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_1).getId()).iterator().next().state(), equalTo(ShardRoutingState.INITIALIZING));
 
         ClusterHealthResponse healthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForYellowStatus().execute().actionGet();
         assertThat(healthResponse.isTimedOut(), equalTo(false));
@@ -124,7 +124,7 @@ public class ClusterRerouteIT extends ESIntegTestCase {
         logger.info("--> get the state, verify shard 1 primary allocated");
         state = client().admin().cluster().prepareState().execute().actionGet().getState();
         assertThat(state.getRoutingNodes().unassigned().size(), equalTo(1));
-        assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_1).getId()).get(0).state(), equalTo(ShardRoutingState.STARTED));
+        assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_1).getId()).iterator().next().state(), equalTo(ShardRoutingState.STARTED));
 
         logger.info("--> move shard 1 primary from node1 to node2");
         state = client().admin().cluster().prepareReroute()
@@ -132,8 +132,8 @@ public class ClusterRerouteIT extends ESIntegTestCase {
                 .add(new MoveAllocationCommand("test", 0, node_1, node_2))
                 .execute().actionGet().getState();
 
-        assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_1).getId()).get(0).state(), equalTo(ShardRoutingState.RELOCATING));
-        assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_2).getId()).get(0).state(), equalTo(ShardRoutingState.INITIALIZING));
+        assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_1).getId()).iterator().next().state(), equalTo(ShardRoutingState.RELOCATING));
+        assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_2).getId()).iterator().next().state(), equalTo(ShardRoutingState.INITIALIZING));
 
 
         healthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForYellowStatus().setWaitForRelocatingShards(0).execute().actionGet();
@@ -142,7 +142,7 @@ public class ClusterRerouteIT extends ESIntegTestCase {
         logger.info("--> get the state, verify shard 1 primary moved from node1 to node2");
         state = client().admin().cluster().prepareState().execute().actionGet().getState();
         assertThat(state.getRoutingNodes().unassigned().size(), equalTo(1));
-        assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_2).getId()).get(0).state(), equalTo(ShardRoutingState.STARTED));
+        assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_2).getId()).iterator().next().state(), equalTo(ShardRoutingState.STARTED));
     }
 
     public void testRerouteWithAllocateLocalGateway_disableAllocationSettings() throws Exception {
@@ -215,7 +215,7 @@ public class ClusterRerouteIT extends ESIntegTestCase {
                 .add(new AllocateEmptyPrimaryAllocationCommand("test", 0, node_1, true))
                 .execute().actionGet().getState();
         assertThat(state.getRoutingNodes().unassigned().size(), equalTo(1));
-        assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_1).getId()).get(0).state(), equalTo(ShardRoutingState.INITIALIZING));
+        assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_1).getId()).iterator().next().state(), equalTo(ShardRoutingState.INITIALIZING));
 
         healthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForYellowStatus().execute().actionGet();
         assertThat(healthResponse.isTimedOut(), equalTo(false));
@@ -223,7 +223,7 @@ public class ClusterRerouteIT extends ESIntegTestCase {
         logger.info("--> get the state, verify shard 1 primary allocated");
         state = client().admin().cluster().prepareState().execute().actionGet().getState();
         assertThat(state.getRoutingNodes().unassigned().size(), equalTo(1));
-        assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_1).getId()).get(0).state(), equalTo(ShardRoutingState.STARTED));
+        assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_1).getId()).iterator().next().state(), equalTo(ShardRoutingState.STARTED));
 
         client().prepareIndex("test", "type", "1").setSource("field", "value").setRefresh(true).execute().actionGet();
         final Index index = resolveIndex("test");
@@ -250,7 +250,7 @@ public class ClusterRerouteIT extends ESIntegTestCase {
                 .add(new AllocateEmptyPrimaryAllocationCommand("test", 0, node_1, true))
                 .execute().actionGet().getState();
         assertThat(state.getRoutingNodes().unassigned().size(), equalTo(1));
-        assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_1).getId()).get(0).state(), equalTo(ShardRoutingState.INITIALIZING));
+        assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_1).getId()).iterator().next().state(), equalTo(ShardRoutingState.INITIALIZING));
 
         healthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForYellowStatus().execute().actionGet();
         assertThat(healthResponse.isTimedOut(), equalTo(false));
@@ -258,7 +258,7 @@ public class ClusterRerouteIT extends ESIntegTestCase {
         logger.info("--> get the state, verify shard 1 primary allocated");
         state = client().admin().cluster().prepareState().execute().actionGet().getState();
         assertThat(state.getRoutingNodes().unassigned().size(), equalTo(1));
-        assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_1).getId()).get(0).state(), equalTo(ShardRoutingState.STARTED));
+        assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_1).getId()).iterator().next().state(), equalTo(ShardRoutingState.STARTED));
 
     }
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java
index 53a0faf0705..2e368f322ee 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -38,18 +39,18 @@ import static org.hamcrest.Matchers.nullValue;
 public class AllocationIdTests extends ESTestCase {
     public void testShardToStarted() {
         logger.info("-- create unassigned shard");
-        ShardRouting shard = ShardRouting.newUnassigned(new Index("test","_na_"), 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
+        ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
         assertThat(shard.allocationId(), nullValue());
 
         logger.info("-- initialize the shard");
-        shard.initialize("node1", null, -1);
+        shard = shard.initialize("node1", null, -1);
         AllocationId allocationId = shard.allocationId();
         assertThat(allocationId, notNullValue());
         assertThat(allocationId.getId(), notNullValue());
         assertThat(allocationId.getRelocationId(), nullValue());
 
         logger.info("-- start the shard");
-        shard.moveToStarted();
+        shard = shard.moveToStarted();
         assertThat(shard.allocationId().getId(), equalTo(allocationId.getId()));
         allocationId = shard.allocationId();
         assertThat(allocationId.getId(), notNullValue());
@@ -58,13 +59,13 @@ public class AllocationIdTests extends ESTestCase {
 
     public void testSuccessfulRelocation() {
         logger.info("-- build started shard");
-        ShardRouting shard = ShardRouting.newUnassigned(new Index("test","_na_"), 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
-        shard.initialize("node1", null, -1);
-        shard.moveToStarted();
+        ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
+        shard = shard.initialize("node1", null, -1);
+        shard = shard.moveToStarted();
 
         AllocationId allocationId = shard.allocationId();
         logger.info("-- relocate the shard");
-        shard.relocate("node2", -1);
+        shard = shard.relocate("node2", -1);
         assertThat(shard.allocationId(), not(equalTo(allocationId)));
         assertThat(shard.allocationId().getId(), equalTo(allocationId.getId()));
         assertThat(shard.allocationId().getRelocationId(), notNullValue());
@@ -74,51 +75,51 @@ public class AllocationIdTests extends ESTestCase {
         assertThat(target.allocationId().getRelocationId(), equalTo(shard.allocationId().getId()));
 
         logger.info("-- finalize the relocation");
-        target.moveToStarted();
+        target = target.moveToStarted();
         assertThat(target.allocationId().getId(), equalTo(shard.allocationId().getRelocationId()));
         assertThat(target.allocationId().getRelocationId(), nullValue());
     }
 
     public void testCancelRelocation() {
         logger.info("-- build started shard");
-        ShardRouting shard = ShardRouting.newUnassigned(new Index("test","_na_"), 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
-        shard.initialize("node1", null, -1);
-        shard.moveToStarted();
+        ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
+        shard = shard.initialize("node1", null, -1);
+        shard = shard.moveToStarted();
 
         AllocationId allocationId = shard.allocationId();
         logger.info("-- relocate the shard");
-        shard.relocate("node2", -1);
+        shard = shard.relocate("node2", -1);
         assertThat(shard.allocationId(), not(equalTo(allocationId)));
         assertThat(shard.allocationId().getId(), equalTo(allocationId.getId()));
         assertThat(shard.allocationId().getRelocationId(), notNullValue());
         allocationId = shard.allocationId();
 
         logger.info("-- cancel relocation");
-        shard.cancelRelocation();
+        shard = shard.cancelRelocation();
         assertThat(shard.allocationId().getId(), equalTo(allocationId.getId()));
         assertThat(shard.allocationId().getRelocationId(), nullValue());
     }
 
     public void testMoveToUnassigned() {
         logger.info("-- build started shard");
-        ShardRouting shard = ShardRouting.newUnassigned(new Index("test","_na_"), 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
-        shard.initialize("node1", null, -1);
-        shard.moveToStarted();
+        ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
+        shard = shard.initialize("node1", null, -1);
+        shard = shard.moveToStarted();
 
         logger.info("-- move to unassigned");
-        shard.moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, null));
+        shard = shard.moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, null));
         assertThat(shard.allocationId(), nullValue());
     }
 
     public void testReinitializing() {
         logger.info("-- build started shard");
-        ShardRouting shard = ShardRouting.newUnassigned(new Index("test","_na_"), 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
-        shard.initialize("node1", null, -1);
-        shard.moveToStarted();
+        ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
+        shard = shard.initialize("node1", null, -1);
+        shard = shard.moveToStarted();
         AllocationId allocationId = shard.allocationId();
 
         logger.info("-- reinitializing shard");
-        shard.reinitializeShard();
+        shard = shard.reinitializeShard();
         assertThat(shard.allocationId().getId(), notNullValue());
         assertThat(shard.allocationId().getRelocationId(), nullValue());
         assertThat(shard.allocationId().getId(), not(equalTo(allocationId.getId())));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RandomShardRoutingMutator.java b/core/src/test/java/org/elasticsearch/cluster/routing/RandomShardRoutingMutator.java
index 2b581a9d82e..5d24b3d1e17 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/RandomShardRoutingMutator.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/RandomShardRoutingMutator.java
@@ -31,26 +31,27 @@ public final class RandomShardRoutingMutator {
 
     }
 
-    public static void randomChange(ShardRouting shardRouting, String[] nodes) {
+    public static ShardRouting randomChange(ShardRouting shardRouting, String[] nodes) {
         switch (randomInt(2)) {
             case 0:
                 if (shardRouting.unassigned() == false) {
-                    shardRouting.moveToUnassigned(new UnassignedInfo(randomReason(), randomAsciiOfLength(10)));
+                    shardRouting = shardRouting.moveToUnassigned(new UnassignedInfo(randomReason(), randomAsciiOfLength(10)));
                 } else if (shardRouting.unassignedInfo() != null) {
-                    shardRouting.updateUnassignedInfo(new UnassignedInfo(randomReason(), randomAsciiOfLength(10)));
+                    shardRouting = shardRouting.updateUnassignedInfo(new UnassignedInfo(randomReason(), randomAsciiOfLength(10)));
                 }
                 break;
             case 1:
                 if (shardRouting.unassigned()) {
-                    shardRouting.initialize(randomFrom(nodes), null, -1);
+                    shardRouting = shardRouting.initialize(randomFrom(nodes), null, -1);
                 }
                 break;
             case 2:
                 if (shardRouting.initializing()) {
-                    shardRouting.moveToStarted();
+                    shardRouting = shardRouting.moveToStarted();
                 }
                 break;
         }
+        return shardRouting;
     }
 
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java
index 7299cbdf590..81d5c45c39b 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java
@@ -24,40 +24,40 @@ package org.elasticsearch.cluster.routing;
  */
 public class ShardRoutingHelper {
 
-    public static void relocate(ShardRouting routing, String nodeId) {
-        relocate(routing, nodeId, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
+    public static ShardRouting relocate(ShardRouting routing, String nodeId) {
+        return relocate(routing, nodeId, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
     }
 
-    public static void relocate(ShardRouting routing, String nodeId, long expectedByteSize) {
-        routing.relocate(nodeId, expectedByteSize);
+    public static ShardRouting relocate(ShardRouting routing, String nodeId, long expectedByteSize) {
+        return routing.relocate(nodeId, expectedByteSize);
     }
 
-    public static void moveToStarted(ShardRouting routing) {
-        routing.moveToStarted();
+    public static ShardRouting moveToStarted(ShardRouting routing) {
+        return routing.moveToStarted();
     }
 
-    public static void initialize(ShardRouting routing, String nodeId) {
-        initialize(routing, nodeId, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
+    public static ShardRouting initialize(ShardRouting routing, String nodeId) {
+        return initialize(routing, nodeId, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
     }
 
-    public static void initialize(ShardRouting routing, String nodeId, long expectedSize) {
-        routing.initialize(nodeId, null, expectedSize);
+    public static ShardRouting initialize(ShardRouting routing, String nodeId, long expectedSize) {
+        return routing.initialize(nodeId, null, expectedSize);
     }
 
-    public static void reinit(ShardRouting routing) {
-        routing.reinitializeShard();
+    public static ShardRouting reinit(ShardRouting routing) {
+        return routing.reinitializeShard();
     }
 
-    public static void reinit(ShardRouting routing, UnassignedInfo.Reason reason) {
-        routing.reinitializeShard();
-        routing.updateUnassignedInfo(new UnassignedInfo(reason, "test_reinit"));
+    public static ShardRouting reinit(ShardRouting routing, UnassignedInfo.Reason reason) {
+        return routing.reinitializeShard().updateUnassignedInfo(new UnassignedInfo(reason, "test_reinit"));
     }
 
-    public static void moveToUnassigned(ShardRouting routing, UnassignedInfo info) {
-        routing.moveToUnassigned(info);
+    public static ShardRouting moveToUnassigned(ShardRouting routing, UnassignedInfo info) {
+        return routing.moveToUnassigned(info);
     }
 
     public static ShardRouting newWithRestoreSource(ShardRouting routing, RestoreSource restoreSource) {
-        return new ShardRouting(routing.index(), routing.shardId().id(), routing.currentNodeId(), routing.relocatingNodeId(), restoreSource, routing.primary(), routing.state(), routing.unassignedInfo(), routing.allocationId(), true, routing.getExpectedShardSize());
+        return new ShardRouting(routing.shardId(), routing.currentNodeId(), routing.relocatingNodeId(), restoreSource,
+            routing.primary(), routing.state(), routing.unassignedInfo(), routing.allocationId(), routing.getExpectedShardSize());
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java
index 4e70761b169..31f48a68654 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java
@@ -30,34 +30,17 @@ import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 
-public class ShardRoutingTests extends ESTestCase {
+import static org.hamcrest.Matchers.equalTo;
 
-    public void testFrozenAfterRead() throws IOException {
-        ShardRouting routing = TestShardRouting.newShardRouting("foo", 1, "node_1", null, null, false, ShardRoutingState.INITIALIZING);
-        routing.moveToPrimary();
-        assertTrue(routing.primary());
-        routing.moveFromPrimary();
-        assertFalse(routing.primary());
-        BytesStreamOutput out = new BytesStreamOutput();
-        routing.writeTo(out);
-        ShardRouting newRouting = ShardRouting.readShardRoutingEntry(StreamInput.wrap(out.bytes()));
-        try {
-            newRouting.moveToPrimary();
-            fail("must be frozen");
-        } catch (IllegalStateException ex) {
-            // expected
-        }
-    }
+public class ShardRoutingTests extends ESTestCase {
 
     public void testIsSameAllocation() {
         ShardRouting unassignedShard0 = TestShardRouting.newShardRouting("test", 0, null, false, ShardRoutingState.UNASSIGNED);
         ShardRouting unassignedShard1 = TestShardRouting.newShardRouting("test", 1, null, false, ShardRoutingState.UNASSIGNED);
         ShardRouting initializingShard0 = TestShardRouting.newShardRouting("test", 0, "1", randomBoolean(), ShardRoutingState.INITIALIZING);
         ShardRouting initializingShard1 = TestShardRouting.newShardRouting("test", 1, "1", randomBoolean(), ShardRoutingState.INITIALIZING);
-        ShardRouting startedShard0 = new ShardRouting(initializingShard0);
-        startedShard0.moveToStarted();
-        ShardRouting startedShard1 = new ShardRouting(initializingShard1);
-        startedShard1.moveToStarted();
+        ShardRouting startedShard0 = initializingShard0.moveToStarted();
+        ShardRouting startedShard1 = initializingShard1.moveToStarted();
 
         // test identity
         assertTrue(initializingShard0.isSameAllocation(initializingShard0));
@@ -98,23 +81,18 @@ public class ShardRoutingTests extends ESTestCase {
         ShardRouting unassignedShard0 = TestShardRouting.newShardRouting("test", 0, null, false, ShardRoutingState.UNASSIGNED);
         ShardRouting initializingShard0 = TestShardRouting.newShardRouting("test", 0, "node1", randomBoolean(), ShardRoutingState.INITIALIZING);
         ShardRouting initializingShard1 = TestShardRouting.newShardRouting("test", 1, "node1", randomBoolean(), ShardRoutingState.INITIALIZING);
-        ShardRouting startedShard0 = new ShardRouting(initializingShard0);
+        assertFalse(initializingShard0.isRelocationTarget());
+        ShardRouting startedShard0 = initializingShard0.moveToStarted();
         assertFalse(startedShard0.isRelocationTarget());
-        startedShard0.moveToStarted();
-        assertFalse(startedShard0.isRelocationTarget());
-        ShardRouting startedShard1 = new ShardRouting(initializingShard1);
+        assertFalse(initializingShard1.isRelocationTarget());
+        ShardRouting startedShard1 = initializingShard1.moveToStarted();
         assertFalse(startedShard1.isRelocationTarget());
-        startedShard1.moveToStarted();
-        assertFalse(startedShard1.isRelocationTarget());
-        ShardRouting sourceShard0a = new ShardRouting(startedShard0);
-        sourceShard0a.relocate("node2", -1);
+        ShardRouting sourceShard0a = startedShard0.relocate("node2", -1);
         assertFalse(sourceShard0a.isRelocationTarget());
         ShardRouting targetShard0a = sourceShard0a.buildTargetRelocatingShard();
         assertTrue(targetShard0a.isRelocationTarget());
-        ShardRouting sourceShard0b = new ShardRouting(startedShard0);
-        sourceShard0b.relocate("node2", -1);
-        ShardRouting sourceShard1 = new ShardRouting(startedShard1);
-        sourceShard1.relocate("node2", -1);
+        ShardRouting sourceShard0b = startedShard0.relocate("node2", -1);
+        ShardRouting sourceShard1 = startedShard1.relocate("node2", -1);
 
         // test true scenarios
         assertTrue(targetShard0a.isRelocationTargetOf(sourceShard0a));
@@ -148,14 +126,8 @@ public class ShardRoutingTests extends ESTestCase {
     public void testEqualsIgnoringVersion() {
         ShardRouting routing = randomShardRouting("test", 0);
 
-        ShardRouting otherRouting = new ShardRouting(routing);
+        ShardRouting otherRouting = routing;
 
-        assertTrue("expected equality\nthis  " + routing + ",\nother " + otherRouting, routing.equalsIgnoringMetaData(otherRouting));
-        otherRouting = new ShardRouting(routing);
-        assertTrue("expected equality\nthis  " + routing + ",\nother " + otherRouting, routing.equalsIgnoringMetaData(otherRouting));
-
-
-        otherRouting = new ShardRouting(routing);
         Integer[] changeIds = new Integer[]{0, 1, 2, 3, 4, 5, 6};
         for (int changeId : randomSubsetOf(randomIntBetween(1, changeIds.length), changeIds)) {
             switch (changeId) {
@@ -222,87 +194,22 @@ public class ShardRoutingTests extends ESTestCase {
         }
     }
 
-    public void testFrozenOnRoutingTable() {
-        MetaData metaData = MetaData.builder()
-                .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(1))
-                .build();
-
-        RoutingTable routingTable = RoutingTable.builder()
-                .addAsNew(metaData.index("test"))
-                .build();
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
-        for (ShardRouting routing : clusterState.routingTable().allShards()) {
-            assertTrue(routing.isFrozen());
-            try {
-                routing.moveToPrimary();
-                fail("must be frozen");
-            } catch (IllegalStateException ex) {
-                // expected
-            }
-            try {
-                routing.moveToStarted();
-                fail("must be frozen");
-            } catch (IllegalStateException ex) {
-                // expected
-            }
-
-            try {
-                routing.moveFromPrimary();
-                fail("must be frozen");
-            } catch (IllegalStateException ex) {
-                // expected
-            }
-
-            try {
-                routing.initialize("boom", null, -1);
-                fail("must be frozen");
-            } catch (IllegalStateException ex) {
-                // expected
-            }
-            try {
-                routing.cancelRelocation();
-                fail("must be frozen");
-            } catch (IllegalStateException ex) {
-                // expected
-            }
-            try {
-                routing.moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.REPLICA_ADDED, "foobar"));
-                fail("must be frozen");
-            } catch (IllegalStateException ex) {
-                // expected
-            }
-
-            try {
-                routing.relocate("foobar", -1);
-                fail("must be frozen");
-            } catch (IllegalStateException ex) {
-                // expected
-            }
-            try {
-                routing.reinitializeShard();
-                fail("must be frozen");
-            } catch (IllegalStateException ex) {
-                // expected
-            }
-        }
-    }
-
     public void testExpectedSize() throws IOException {
         final int iters = randomIntBetween(10, 100);
         for (int i = 0; i < iters; i++) {
             ShardRouting routing = randomShardRouting("test", 0);
             long byteSize = randomIntBetween(0, Integer.MAX_VALUE);
             if (routing.unassigned()) {
-                ShardRoutingHelper.initialize(routing, "foo", byteSize);
+                routing = ShardRoutingHelper.initialize(routing, "foo", byteSize);
             } else if (routing.started()) {
-                ShardRoutingHelper.relocate(routing, "foo", byteSize);
+                routing = ShardRoutingHelper.relocate(routing, "foo", byteSize);
             } else {
                 byteSize = -1;
             }
             if (randomBoolean()) {
                 BytesStreamOutput out = new BytesStreamOutput();
                 routing.writeTo(out);
-                routing = ShardRouting.readShardRoutingEntry(StreamInput.wrap(out.bytes()));
+                routing = new ShardRouting(StreamInput.wrap(out.bytes()));
             }
             if (routing.initializing() || routing.relocating()) {
                 assertEquals(routing.toString(), byteSize, routing.getExpectedShardSize());
@@ -310,8 +217,7 @@ public class ShardRoutingTests extends ESTestCase {
                     assertTrue(routing.toString(), routing.toString().contains("expected_shard_size[" + byteSize + "]"));
                 }
                 if (routing.initializing()) {
-                    routing = new ShardRouting(routing);
-                    routing.moveToStarted();
+                    routing = routing.moveToStarted();
                     assertEquals(-1, routing.getExpectedShardSize());
                     assertFalse(routing.toString(), routing.toString().contains("expected_shard_size[" + byteSize + "]"));
                 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
index 5c91c80f298..708f8ca5079 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
@@ -185,14 +185,13 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
      */
     public void testStateTransitionMetaHandling() {
         ShardRouting shard = TestShardRouting.newShardRouting("test", 1, null, null, null, true, ShardRoutingState.UNASSIGNED, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
-        ShardRouting mutable = new ShardRouting(shard);
-        assertThat(mutable.unassignedInfo(), notNullValue());
-        mutable.initialize("test_node", null, -1);
-        assertThat(mutable.state(), equalTo(ShardRoutingState.INITIALIZING));
-        assertThat(mutable.unassignedInfo(), notNullValue());
-        mutable.moveToStarted();
-        assertThat(mutable.state(), equalTo(ShardRoutingState.STARTED));
-        assertThat(mutable.unassignedInfo(), nullValue());
+        assertThat(shard.unassignedInfo(), notNullValue());
+        shard = shard.initialize("test_node", null, -1);
+        assertThat(shard.state(), equalTo(ShardRoutingState.INITIALIZING));
+        assertThat(shard.unassignedInfo(), notNullValue());
+        shard = shard.moveToStarted();
+        assertThat(shard.state(), equalTo(ShardRoutingState.STARTED));
+        assertThat(shard.unassignedInfo(), nullValue());
     }
 
     /**
@@ -259,9 +258,10 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
      * Verifies that delayed allocation calculation are correct.
      */
     public void testUnassignedDelayedOnlyOnNodeLeft() throws Exception {
-        final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, null);
-        long delay = unassignedInfo.updateDelay(unassignedInfo.getUnassignedTimeInNanos() + 1, // add 1 tick delay
+        UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, null);
+        unassignedInfo = unassignedInfo.updateDelay(unassignedInfo.getUnassignedTimeInNanos() + 1, // add 1 tick delay
                 Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "10h").build(), Settings.EMPTY);
+        long delay = unassignedInfo.getLastComputedLeftDelayNanos();
         long cachedDelay = unassignedInfo.getLastComputedLeftDelayNanos();
         assertThat(delay, equalTo(cachedDelay));
         assertThat(delay, equalTo(TimeValue.timeValueHours(10).nanos() - 1));
@@ -274,8 +274,9 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
         EnumSet<UnassignedInfo.Reason> reasons = EnumSet.allOf(UnassignedInfo.Reason.class);
         reasons.remove(UnassignedInfo.Reason.NODE_LEFT);
         UnassignedInfo unassignedInfo = new UnassignedInfo(RandomPicks.randomFrom(random(), reasons), null);
-        long delay = unassignedInfo.updateDelay(unassignedInfo.getUnassignedTimeInNanos() + 1, // add 1 tick delay
+        unassignedInfo = unassignedInfo.updateDelay(unassignedInfo.getUnassignedTimeInNanos() + 1, // add 1 tick delay
                 Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "10h").build(), Settings.EMPTY);
+        long delay = unassignedInfo.getLastComputedLeftDelayNanos();
         assertThat(delay, equalTo(0L));
         delay = unassignedInfo.getLastComputedLeftDelayNanos();
         assertThat(delay, equalTo(0L));
@@ -286,20 +287,24 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
      */
     public void testLeftDelayCalculation() throws Exception {
         final long baseTime = System.nanoTime();
-        final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, "test", null, baseTime, System.currentTimeMillis());
+        UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, "test", null, baseTime, System.currentTimeMillis());
         final long totalDelayNanos = TimeValue.timeValueMillis(10).nanos();
         final Settings settings = Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), TimeValue.timeValueNanos(totalDelayNanos)).build();
-        long delay = unassignedInfo.updateDelay(baseTime, settings, Settings.EMPTY);
+        unassignedInfo = unassignedInfo.updateDelay(baseTime, settings, Settings.EMPTY);
+        long delay = unassignedInfo.getLastComputedLeftDelayNanos();
         assertThat(delay, equalTo(totalDelayNanos));
         assertThat(delay, equalTo(unassignedInfo.getLastComputedLeftDelayNanos()));
         long delta1 = randomIntBetween(1, (int) (totalDelayNanos - 1));
-        delay = unassignedInfo.updateDelay(baseTime + delta1, settings, Settings.EMPTY);
+        unassignedInfo = unassignedInfo.updateDelay(baseTime + delta1, settings, Settings.EMPTY);
+        delay = unassignedInfo.getLastComputedLeftDelayNanos();
         assertThat(delay, equalTo(totalDelayNanos - delta1));
         assertThat(delay, equalTo(unassignedInfo.getLastComputedLeftDelayNanos()));
-        delay = unassignedInfo.updateDelay(baseTime + totalDelayNanos, settings, Settings.EMPTY);
+        unassignedInfo = unassignedInfo.updateDelay(baseTime + totalDelayNanos, settings, Settings.EMPTY);
+        delay = unassignedInfo.getLastComputedLeftDelayNanos();
         assertThat(delay, equalTo(0L));
         assertThat(delay, equalTo(unassignedInfo.getLastComputedLeftDelayNanos()));
-        delay = unassignedInfo.updateDelay(baseTime + totalDelayNanos + randomIntBetween(1, 20), settings, Settings.EMPTY);
+        unassignedInfo = unassignedInfo.updateDelay(baseTime + totalDelayNanos + randomIntBetween(1, 20), settings, Settings.EMPTY);
+        delay = unassignedInfo.getLastComputedLeftDelayNanos();
         assertThat(delay, equalTo(0L));
         assertThat(delay, equalTo(unassignedInfo.getLastComputedLeftDelayNanos()));
     }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java
index cfc0f506d09..fe4937eea7e 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java
@@ -120,6 +120,7 @@ public class AddIncrementallyTests extends ESAllocationTestCase {
         assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(INITIALIZING).size(), Matchers.equalTo(0));
 
         RoutingTable prev = routingTable;
+        logger.error(clusterState.prettyPrint());
         routingTable = service.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING)).routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
         routingNodes = clusterState.getRoutingNodes();
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
index 2cd688a661c..7aa8576ece3 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
@@ -97,15 +97,15 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
         rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", 0, existingNodeId, toNodeId)));
         assertThat(rerouteResult.changed(), equalTo(true));
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
-        assertThat(clusterState.getRoutingNodes().node(existingNodeId).get(0).state(), equalTo(ShardRoutingState.RELOCATING));
-        assertThat(clusterState.getRoutingNodes().node(toNodeId).get(0).state(), equalTo(ShardRoutingState.INITIALIZING));
+        assertThat(clusterState.getRoutingNodes().node(existingNodeId).iterator().next().state(), equalTo(ShardRoutingState.RELOCATING));
+        assertThat(clusterState.getRoutingNodes().node(toNodeId).iterator().next().state(), equalTo(ShardRoutingState.INITIALIZING));
 
         logger.info("finish moving the shard");
         rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
 
         assertThat(clusterState.getRoutingNodes().node(existingNodeId).isEmpty(), equalTo(true));
-        assertThat(clusterState.getRoutingNodes().node(toNodeId).get(0).state(), equalTo(ShardRoutingState.STARTED));
+        assertThat(clusterState.getRoutingNodes().node(toNodeId).iterator().next().state(), equalTo(ShardRoutingState.STARTED));
     }
 
     private AbstractAllocateAllocationCommand randomAllocateCommand(String index, int shardId, String node) {
@@ -386,7 +386,8 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
         rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new CancelAllocationCommand("test", 0, "node1", true)));
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
         assertThat(rerouteResult.changed(), equalTo(true));
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).get(0).primary(), equalTo(true));
+        logger.error(clusterState.prettyPrint());
+        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).iterator().next().primary(), equalTo(true));
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(0));
         assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(0));
     }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java
index 88fad86794e..4e99d69ef89 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java
@@ -128,7 +128,7 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase {
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(routingNodes.node("node3").size(), equalTo(1));
-        assertThat(routingNodes.node("node3").get(0).shardId().getIndex().getName(), equalTo("test1"));
+        assertThat(routingNodes.node("node3").iterator().next().shardId().getIndex().getName(), equalTo("test1"));
     }
 
 
@@ -233,7 +233,7 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase {
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(routingNodes.node("node3").size(), equalTo(1));
-        assertThat(routingNodes.node("node3").get(0).shardId().getIndex().getName(), equalTo("test1"));
+        assertThat(routingNodes.node("node3").iterator().next().shardId().getIndex().getName(), equalTo("test1"));
     }
 
     public void testClusterPrimariesActive2() {
@@ -440,7 +440,7 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase {
         routingNodes = clusterState.getRoutingNodes();
 
         assertThat(routingNodes.node("node3").size(), equalTo(1));
-        assertThat(routingNodes.node("node3").get(0).shardId().getIndex().getName(), anyOf(equalTo("test1"), equalTo("test2")));
+        assertThat(routingNodes.node("node3").iterator().next().shardId().getIndex().getName(), anyOf(equalTo("test1"), equalTo("test2")));
     }
 
     public void testClusterAllActive2() {
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java
index f6539a167ff..11b78d2ae6a 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java
@@ -76,9 +76,9 @@ public class DeadNodesAllocationTests extends ESAllocationTestCase {
 
         logger.info("--> verifying all is allocated");
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node1").get(0).state(), equalTo(STARTED));
+        assertThat(clusterState.getRoutingNodes().node("node1").iterator().next().state(), equalTo(STARTED));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node2").get(0).state(), equalTo(STARTED));
+        assertThat(clusterState.getRoutingNodes().node("node2").iterator().next().state(), equalTo(STARTED));
 
         logger.info("--> fail node with primary");
         String nodeIdToFail = clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId();
@@ -90,8 +90,8 @@ public class DeadNodesAllocationTests extends ESAllocationTestCase {
         rerouteResult = allocation.reroute(clusterState, "reroute");
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
 
-        assertThat(clusterState.getRoutingNodes().node(nodeIdRemaining).get(0).primary(), equalTo(true));
-        assertThat(clusterState.getRoutingNodes().node(nodeIdRemaining).get(0).state(), equalTo(STARTED));
+        assertThat(clusterState.getRoutingNodes().node(nodeIdRemaining).iterator().next().primary(), equalTo(true));
+        assertThat(clusterState.getRoutingNodes().node(nodeIdRemaining).iterator().next().state(), equalTo(STARTED));
     }
 
     public void testDeadNodeWhileRelocatingOnToNode() {
@@ -127,9 +127,9 @@ public class DeadNodesAllocationTests extends ESAllocationTestCase {
 
         logger.info("--> verifying all is allocated");
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node1").get(0).state(), equalTo(STARTED));
+        assertThat(clusterState.getRoutingNodes().node("node1").iterator().next().state(), equalTo(STARTED));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node2").get(0).state(), equalTo(STARTED));
+        assertThat(clusterState.getRoutingNodes().node("node2").iterator().next().state(), equalTo(STARTED));
 
         logger.info("--> adding additional node");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
@@ -139,9 +139,9 @@ public class DeadNodesAllocationTests extends ESAllocationTestCase {
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
 
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node1").get(0).state(), equalTo(STARTED));
+        assertThat(clusterState.getRoutingNodes().node("node1").iterator().next().state(), equalTo(STARTED));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node2").get(0).state(), equalTo(STARTED));
+        assertThat(clusterState.getRoutingNodes().node("node2").iterator().next().state(), equalTo(STARTED));
         assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(0));
 
         String origPrimaryNodeId = clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId();
@@ -153,8 +153,8 @@ public class DeadNodesAllocationTests extends ESAllocationTestCase {
         );
         assertThat(rerouteResult.changed(), equalTo(true));
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
-        assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).get(0).state(), equalTo(RELOCATING));
-        assertThat(clusterState.getRoutingNodes().node("node3").get(0).state(), equalTo(INITIALIZING));
+        assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).iterator().next().state(), equalTo(RELOCATING));
+        assertThat(clusterState.getRoutingNodes().node("node3").iterator().next().state(), equalTo(INITIALIZING));
 
         logger.info("--> fail primary shard recovering instance on node3 being initialized by killing node3");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -164,8 +164,8 @@ public class DeadNodesAllocationTests extends ESAllocationTestCase {
         rerouteResult = allocation.reroute(clusterState, "reroute");
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
 
-        assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).get(0).state(), equalTo(STARTED));
-        assertThat(clusterState.getRoutingNodes().node(origReplicaNodeId).get(0).state(), equalTo(STARTED));
+        assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).iterator().next().state(), equalTo(STARTED));
+        assertThat(clusterState.getRoutingNodes().node(origReplicaNodeId).iterator().next().state(), equalTo(STARTED));
     }
 
     public void testDeadNodeWhileRelocatingOnFromNode() {
@@ -201,9 +201,9 @@ public class DeadNodesAllocationTests extends ESAllocationTestCase {
 
         logger.info("--> verifying all is allocated");
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node1").get(0).state(), equalTo(STARTED));
+        assertThat(clusterState.getRoutingNodes().node("node1").iterator().next().state(), equalTo(STARTED));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node2").get(0).state(), equalTo(STARTED));
+        assertThat(clusterState.getRoutingNodes().node("node2").iterator().next().state(), equalTo(STARTED));
 
         logger.info("--> adding additional node");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
@@ -213,9 +213,9 @@ public class DeadNodesAllocationTests extends ESAllocationTestCase {
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
 
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node1").get(0).state(), equalTo(STARTED));
+        assertThat(clusterState.getRoutingNodes().node("node1").iterator().next().state(), equalTo(STARTED));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node2").get(0).state(), equalTo(STARTED));
+        assertThat(clusterState.getRoutingNodes().node("node2").iterator().next().state(), equalTo(STARTED));
         assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(0));
 
         String origPrimaryNodeId = clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId();
@@ -227,8 +227,8 @@ public class DeadNodesAllocationTests extends ESAllocationTestCase {
         );
         assertThat(rerouteResult.changed(), equalTo(true));
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
-        assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).get(0).state(), equalTo(RELOCATING));
-        assertThat(clusterState.getRoutingNodes().node("node3").get(0).state(), equalTo(INITIALIZING));
+        assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).iterator().next().state(), equalTo(RELOCATING));
+        assertThat(clusterState.getRoutingNodes().node("node3").iterator().next().state(), equalTo(INITIALIZING));
 
         logger.info("--> fail primary shard recovering instance on 'origPrimaryNodeId' being relocated");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -238,7 +238,7 @@ public class DeadNodesAllocationTests extends ESAllocationTestCase {
         rerouteResult = allocation.reroute(clusterState, "reroute");
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
 
-        assertThat(clusterState.getRoutingNodes().node(origReplicaNodeId).get(0).state(), equalTo(STARTED));
-        assertThat(clusterState.getRoutingNodes().node("node3").get(0).state(), equalTo(INITIALIZING));
+        assertThat(clusterState.getRoutingNodes().node(origReplicaNodeId).iterator().next().state(), equalTo(STARTED));
+        assertThat(clusterState.getRoutingNodes().node("node3").iterator().next().state(), equalTo(INITIALIZING));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java
index 4b720036b9a..bfa27a36d8b 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java
@@ -152,18 +152,18 @@ public class ExpectedShardSizeAllocationTests extends ESAllocationTestCase {
         rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", 0, existingNodeId, toNodeId)));
         assertThat(rerouteResult.changed(), equalTo(true));
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
-        assertEquals(clusterState.getRoutingNodes().node(existingNodeId).get(0).state(), ShardRoutingState.RELOCATING);
-        assertEquals(clusterState.getRoutingNodes().node(toNodeId).get(0).state(),ShardRoutingState.INITIALIZING);
+        assertEquals(clusterState.getRoutingNodes().node(existingNodeId).iterator().next().state(), ShardRoutingState.RELOCATING);
+        assertEquals(clusterState.getRoutingNodes().node(toNodeId).iterator().next().state(),ShardRoutingState.INITIALIZING);
 
-        assertEquals(clusterState.getRoutingNodes().node(existingNodeId).get(0).getExpectedShardSize(), byteSize);
-        assertEquals(clusterState.getRoutingNodes().node(toNodeId).get(0).getExpectedShardSize(), byteSize);
+        assertEquals(clusterState.getRoutingNodes().node(existingNodeId).iterator().next().getExpectedShardSize(), byteSize);
+        assertEquals(clusterState.getRoutingNodes().node(toNodeId).iterator().next().getExpectedShardSize(), byteSize);
 
         logger.info("finish moving the shard");
         rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
 
         assertThat(clusterState.getRoutingNodes().node(existingNodeId).isEmpty(), equalTo(true));
-        assertThat(clusterState.getRoutingNodes().node(toNodeId).get(0).state(), equalTo(ShardRoutingState.STARTED));
-        assertEquals(clusterState.getRoutingNodes().node(toNodeId).get(0).getExpectedShardSize(), -1);
+        assertThat(clusterState.getRoutingNodes().node(toNodeId).iterator().next().state(), equalTo(ShardRoutingState.STARTED));
+        assertEquals(clusterState.getRoutingNodes().node(toNodeId).iterator().next().getExpectedShardSize(), -1);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java
index 37a46fe4c10..58e2397b043 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java
@@ -87,9 +87,9 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase {
 
         logger.info("--> verifying all is allocated");
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node1").get(0).state(), equalTo(STARTED));
+        assertThat(clusterState.getRoutingNodes().node("node1").iterator().next().state(), equalTo(STARTED));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node2").get(0).state(), equalTo(STARTED));
+        assertThat(clusterState.getRoutingNodes().node("node2").iterator().next().state(), equalTo(STARTED));
 
         logger.info("--> adding additional node");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
@@ -99,9 +99,9 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase {
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
 
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node1").get(0).state(), equalTo(STARTED));
+        assertThat(clusterState.getRoutingNodes().node("node1").iterator().next().state(), equalTo(STARTED));
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
-        assertThat(clusterState.getRoutingNodes().node("node2").get(0).state(), equalTo(STARTED));
+        assertThat(clusterState.getRoutingNodes().node("node2").iterator().next().state(), equalTo(STARTED));
         assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(0));
 
         String origPrimaryNodeId = clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId();
@@ -113,14 +113,14 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase {
         );
         assertThat(rerouteResult.changed(), equalTo(true));
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
-        assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).get(0).state(), equalTo(RELOCATING));
-        assertThat(clusterState.getRoutingNodes().node("node3").get(0).state(), equalTo(INITIALIZING));
+        assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).iterator().next().state(), equalTo(RELOCATING));
+        assertThat(clusterState.getRoutingNodes().node("node3").iterator().next().state(), equalTo(INITIALIZING));
 
         logger.info("--> fail primary shard recovering instance on node3 being initialized");
-        rerouteResult = allocation.applyFailedShard(clusterState, new ShardRouting(clusterState.getRoutingNodes().node("node3").get(0)));
+        rerouteResult = allocation.applyFailedShard(clusterState, clusterState.getRoutingNodes().node("node3").iterator().next());
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
 
-        assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).get(0).state(), equalTo(STARTED));
+        assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).iterator().next().state(), equalTo(STARTED));
         assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(0));
 
         logger.info("--> moving primary shard to node3");
@@ -129,15 +129,15 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase {
         );
         assertThat(rerouteResult.changed(), equalTo(true));
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
-        assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).get(0).state(), equalTo(RELOCATING));
-        assertThat(clusterState.getRoutingNodes().node("node3").get(0).state(), equalTo(INITIALIZING));
+        assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).iterator().next().state(), equalTo(RELOCATING));
+        assertThat(clusterState.getRoutingNodes().node("node3").iterator().next().state(), equalTo(INITIALIZING));
 
         logger.info("--> fail primary shard recovering instance on node1 being relocated");
-        rerouteResult = allocation.applyFailedShard(clusterState, new ShardRouting(clusterState.getRoutingNodes().node(origPrimaryNodeId).get(0)));
+        rerouteResult = allocation.applyFailedShard(clusterState, clusterState.getRoutingNodes().node(origPrimaryNodeId).iterator().next());
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
 
         // check promotion of replica to primary
-        assertThat(clusterState.getRoutingNodes().node(origReplicaNodeId).get(0).state(), equalTo(STARTED));
+        assertThat(clusterState.getRoutingNodes().node(origReplicaNodeId).iterator().next().state(), equalTo(STARTED));
         assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), equalTo(origReplicaNodeId));
         assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().get(0).currentNodeId(), anyOf(equalTo(origPrimaryNodeId), equalTo("node3")));
     }
@@ -204,7 +204,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase {
         }
 
         logger.info("fail the primary shard, will have no place to be rerouted to (single node), so stays unassigned");
-        ShardRouting shardToFail = new ShardRouting(routingTable.index("test").shard(0).primaryShard());
+        ShardRouting shardToFail = routingTable.index("test").shard(0).primaryShard();
         prevRoutingTable = routingTable;
         routingTable = strategy.applyFailedShard(clusterState, shardToFail).routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
@@ -260,7 +260,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase {
 
         logger.info("fail the first shard, will have no place to be rerouted to (single node), so stays unassigned");
         prevRoutingTable = routingTable;
-        ShardRouting firstShard = clusterState.getRoutingNodes().node("node1").get(0);
+        ShardRouting firstShard = clusterState.getRoutingNodes().node("node1").iterator().next();
         routingTable = strategy.applyFailedShard(clusterState, firstShard).routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
 
@@ -320,8 +320,8 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase {
         for (int i = 0; i < shardsToFail; i++) {
             String n = "node" + Integer.toString(randomInt(numberOfReplicas));
             logger.info("failing shard on node [{}]", n);
-            ShardRouting shardToFail = routingNodes.node(n).get(0);
-            failedShards.add(new FailedRerouteAllocation.FailedShard(new ShardRouting(shardToFail), null, null));
+            ShardRouting shardToFail = routingNodes.node(n).iterator().next();
+            failedShards.add(new FailedRerouteAllocation.FailedShard(shardToFail, null, null));
         }
 
         routingTable = strategy.applyFailedShards(clusterState, failedShards).routingTable();
@@ -373,7 +373,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase {
 
         logger.info("fail the first shard, will start INITIALIZING on the second node");
         prevRoutingTable = routingTable;
-        final ShardRouting firstShard = clusterState.getRoutingNodes().node(nodeHoldingPrimary).get(0);
+        final ShardRouting firstShard = clusterState.getRoutingNodes().node(nodeHoldingPrimary).iterator().next();
         routingTable = strategy.applyFailedShard(clusterState, firstShard).routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
         assertThat(prevRoutingTable != routingTable, equalTo(true));
@@ -472,10 +472,10 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase {
 
 
         logger.info("Fail the shards on node 3");
-        ShardRouting shardToFail = routingNodes.node("node3").get(0);
+        ShardRouting shardToFail = routingNodes.node("node3").iterator().next();
         routingNodes = clusterState.getRoutingNodes();
         prevRoutingTable = routingTable;
-        routingTable = strategy.applyFailedShard(clusterState, new ShardRouting(shardToFail)).routingTable();
+        routingTable = strategy.applyFailedShard(clusterState, shardToFail).routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
         routingNodes = clusterState.getRoutingNodes();
 
@@ -487,7 +487,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase {
         assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), lessThan(3));
         assertThat(routingNodes.node("node3").numberOfShardsWithState(INITIALIZING), equalTo(1));
         // make sure the failedShard is not INITIALIZING again on node3
-        assertThat(routingNodes.node("node3").get(0).shardId(), not(equalTo(shardToFail.shardId())));
+        assertThat(routingNodes.node("node3").iterator().next().shardId(), not(equalTo(shardToFail.shardId())));
     }
 
     public void testFailAllReplicasInitializingOnPrimaryFail() {
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java
index c84fe546951..5a555621bec 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java
@@ -153,6 +153,7 @@ public class FilterRoutingTests extends ESAllocationTestCase {
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2));
         assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(2));
+        assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(2));
 
         logger.info("--> finish relocation");
         routingTable = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)).routingTable();
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java
index efbcb774e63..5d01caaa899 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java
@@ -32,6 +32,7 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESAllocationTestCase;
 
 import java.util.Arrays;
@@ -53,9 +54,9 @@ public class StartedShardsRoutingTests extends ESAllocationTestCase {
                 .nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")))
                 .metaData(MetaData.builder().put(indexMetaData, false));
 
-        final ShardRouting initShard = TestShardRouting.newShardRouting(index, 0, "node1", true, ShardRoutingState.INITIALIZING);
-        final ShardRouting startedShard = TestShardRouting.newShardRouting(index, 1, "node2", true, ShardRoutingState.STARTED);
-        final ShardRouting relocatingShard = TestShardRouting.newShardRouting(index, 2, "node1", "node2", true, ShardRoutingState.RELOCATING);
+        final ShardRouting initShard = TestShardRouting.newShardRouting(new ShardId(index, 0), "node1", true, ShardRoutingState.INITIALIZING);
+        final ShardRouting startedShard = TestShardRouting.newShardRouting(new ShardId(index, 1), "node2", true, ShardRoutingState.STARTED);
+        final ShardRouting relocatingShard = TestShardRouting.newShardRouting(new ShardId(index, 2), "node1", "node2", true, ShardRoutingState.RELOCATING);
         stateBuilder.routingTable(RoutingTable.builder().add(IndexRoutingTable.builder(index)
                 .addIndexShard(new IndexShardRoutingTable.Builder(initShard.shardId()).addShard(initShard).build())
                 .addIndexShard(new IndexShardRoutingTable.Builder(startedShard.shardId()).addShard(startedShard).build())
@@ -66,7 +67,7 @@ public class StartedShardsRoutingTests extends ESAllocationTestCase {
         logger.info("--> test starting of shard");
 
         RoutingAllocation.Result result = allocation.applyStartedShards(state, Arrays.asList(
-                TestShardRouting.newShardRouting(initShard.index(), initShard.id(), initShard.currentNodeId(), initShard.relocatingNodeId(), initShard.primary(),
+                TestShardRouting.newShardRouting(initShard.shardId(), initShard.currentNodeId(), initShard.relocatingNodeId(), initShard.primary(),
                         ShardRoutingState.INITIALIZING, initShard.allocationId())), false);
         assertTrue("failed to start " + initShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed());
         assertTrue(initShard + "isn't started \ncurrent routing table:" + result.routingTable().prettyPrint(),
@@ -76,12 +77,12 @@ public class StartedShardsRoutingTests extends ESAllocationTestCase {
         logger.info("--> testing shard variants that shouldn't match the initializing shard");
 
         result = allocation.applyStartedShards(state, Arrays.asList(
-                TestShardRouting.newShardRouting(initShard.index(), initShard.id(), initShard.currentNodeId(), initShard.relocatingNodeId(), initShard.primary(),
+                TestShardRouting.newShardRouting(initShard.shardId(), initShard.currentNodeId(), initShard.relocatingNodeId(), initShard.primary(),
                         ShardRoutingState.INITIALIZING)), false);
         assertFalse("wrong allocation id flag shouldn't start shard " + initShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed());
 
         result = allocation.applyStartedShards(state, Arrays.asList(
-                TestShardRouting.newShardRouting(initShard.index(), initShard.id(), "some_node", initShard.currentNodeId(), initShard.primary(),
+                TestShardRouting.newShardRouting(initShard.shardId(), "some_node", initShard.currentNodeId(), initShard.primary(),
                         ShardRoutingState.INITIALIZING, AllocationId.newTargetRelocation(AllocationId.newRelocation(initShard.allocationId())))), false);
         assertFalse("relocating shard from node shouldn't start shard " + initShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed());
 
@@ -90,14 +91,14 @@ public class StartedShardsRoutingTests extends ESAllocationTestCase {
         logger.info("--> testing double starting");
 
         result = allocation.applyStartedShards(state, Arrays.asList(
-                TestShardRouting.newShardRouting(startedShard.index(), startedShard.id(), startedShard.currentNodeId(), startedShard.relocatingNodeId(), startedShard.primary(),
+                TestShardRouting.newShardRouting(startedShard.shardId(), startedShard.currentNodeId(), startedShard.relocatingNodeId(), startedShard.primary(),
                         ShardRoutingState.INITIALIZING, startedShard.allocationId())), false);
         assertFalse("duplicate starting of the same shard should be ignored \ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed());
 
         logger.info("--> testing starting of relocating shards");
         final AllocationId targetAllocationId = AllocationId.newTargetRelocation(relocatingShard.allocationId());
         result = allocation.applyStartedShards(state, Arrays.asList(
-                TestShardRouting.newShardRouting(relocatingShard.index(), relocatingShard.id(), relocatingShard.relocatingNodeId(), relocatingShard.currentNodeId(), relocatingShard.primary(),
+                TestShardRouting.newShardRouting(relocatingShard.shardId(), relocatingShard.relocatingNodeId(), relocatingShard.currentNodeId(), relocatingShard.primary(),
                         ShardRoutingState.INITIALIZING, targetAllocationId)), false);
 
         assertTrue("failed to start " + relocatingShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed());
@@ -109,12 +110,12 @@ public class StartedShardsRoutingTests extends ESAllocationTestCase {
         logger.info("--> testing shard variants that shouldn't match the initializing relocating shard");
 
         result = allocation.applyStartedShards(state, Arrays.asList(
-                TestShardRouting.newShardRouting(relocatingShard.index(), relocatingShard.id(), relocatingShard.relocatingNodeId(), relocatingShard.currentNodeId(), relocatingShard.primary(),
+                TestShardRouting.newShardRouting(relocatingShard.shardId(), relocatingShard.relocatingNodeId(), relocatingShard.currentNodeId(), relocatingShard.primary(),
                         ShardRoutingState.INITIALIZING)));
         assertFalse("wrong allocation id shouldn't start shard" + relocatingShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed());
 
         result = allocation.applyStartedShards(state, Arrays.asList(
-                TestShardRouting.newShardRouting(relocatingShard.index(), relocatingShard.id(), relocatingShard.relocatingNodeId(), relocatingShard.currentNodeId(), relocatingShard.primary(),
+                TestShardRouting.newShardRouting(relocatingShard.shardId(), relocatingShard.relocatingNodeId(), relocatingShard.currentNodeId(), relocatingShard.primary(),
                         ShardRoutingState.INITIALIZING, relocatingShard.allocationId())), false);
         assertFalse("wrong allocation id shouldn't start shard even if relocatingId==shard.id" + relocatingShard + "\ncurrent routing table:" + result.routingTable().prettyPrint(), result.changed());
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java
index 4749e5666d8..28d916e20c1 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java
@@ -283,7 +283,7 @@ public class ThrottlingAllocationTests extends ESAllocationTestCase {
         assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node2"), 0);
         assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node3"), 0);
 
-        RoutingAllocation.Result reroute = strategy.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", clusterState.getRoutingNodes().node("node1").get(0).shardId().id(), "node1", "node2")));
+        RoutingAllocation.Result reroute = strategy.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", clusterState.getRoutingNodes().node("node1").iterator().next().shardId().id(), "node1", "node2")));
         assertEquals(reroute.explanations().explanations().size(), 1);
         assertEquals(reroute.explanations().explanations().get(0).decisions().type(), Decision.Type.YES);
         routingTable = reroute.routingTable();
@@ -296,7 +296,7 @@ public class ThrottlingAllocationTests extends ESAllocationTestCase {
         assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node3"), 0);
 
         // outgoing throttles
-        reroute = strategy.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", clusterState.getRoutingNodes().node("node3").get(0).shardId().id(), "node3", "node1")), true);
+        reroute = strategy.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", clusterState.getRoutingNodes().node("node3").iterator().next().shardId().id(), "node3", "node1")), true);
         assertEquals(reroute.explanations().explanations().size(), 1);
         assertEquals(reroute.explanations().explanations().get(0).decisions().type(), Decision.Type.THROTTLE);
         assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node1"), 0);
@@ -311,7 +311,7 @@ public class ThrottlingAllocationTests extends ESAllocationTestCase {
         assertThat(routingTable.shardsWithState(UNASSIGNED).size(), equalTo(0));
 
         // incoming throttles
-        reroute = strategy.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", clusterState.getRoutingNodes().node("node3").get(0).shardId().id(), "node3", "node2")), true);
+        reroute = strategy.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", clusterState.getRoutingNodes().node("node3").iterator().next().shardId().id(), "node3", "node2")), true);
         assertEquals(reroute.explanations().explanations().size(), 1);
         assertEquals(reroute.explanations().explanations().get(0).decisions().type(), Decision.Type.THROTTLE);
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
index 1d1a88e1db9..a58c9c3f58b 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
@@ -862,7 +862,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
         // Two shards consuming each 80% of disk space while 70% is allowed, so shard 0 isn't allowed here
         ShardRouting firstRouting = TestShardRouting.newShardRouting("test", 0, "node1", null, null, true, ShardRoutingState.STARTED);
         ShardRouting secondRouting = TestShardRouting.newShardRouting("test", 1, "node1", null, null, true, ShardRoutingState.STARTED);
-        RoutingNode firstRoutingNode = new RoutingNode("node1", discoveryNode1, Arrays.asList(firstRouting, secondRouting));
+        RoutingNode firstRoutingNode = new RoutingNode("node1", discoveryNode1, firstRouting, secondRouting);
         RoutingTable.Builder builder = RoutingTable.builder().add(
                 IndexRoutingTable.builder(firstRouting.index())
                         .addIndexShard(new IndexShardRoutingTable.Builder(firstRouting.shardId())
@@ -883,7 +883,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
         // Two shards consuming each 80% of disk space while 70% is allowed, but one is relocating, so shard 0 can stay
         firstRouting = TestShardRouting.newShardRouting("test", 0, "node1", null, null, true, ShardRoutingState.STARTED);
         secondRouting = TestShardRouting.newShardRouting("test", 1, "node1", "node2", null, true, ShardRoutingState.RELOCATING);
-        firstRoutingNode = new RoutingNode("node1", discoveryNode1, Arrays.asList(firstRouting, secondRouting));
+        firstRoutingNode = new RoutingNode("node1", discoveryNode1, firstRouting, secondRouting);
         builder = RoutingTable.builder().add(
                 IndexRoutingTable.builder(firstRouting.index())
                         .addIndexShard(new IndexShardRoutingTable.Builder(firstRouting.shardId())
@@ -977,7 +977,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
         // Two shards consumes 80% of disk space in data node, but we have only one data node, shards should remain.
         ShardRouting firstRouting = TestShardRouting.newShardRouting("test", 0, "node2", null, null, true, ShardRoutingState.STARTED);
         ShardRouting secondRouting = TestShardRouting.newShardRouting("test", 1, "node2", null, null, true, ShardRoutingState.STARTED);
-        RoutingNode firstRoutingNode = new RoutingNode("node2", discoveryNode2, Arrays.asList(firstRouting, secondRouting));
+        RoutingNode firstRoutingNode = new RoutingNode("node2", discoveryNode2, firstRouting, secondRouting);
 
         RoutingTable.Builder builder = RoutingTable.builder().add(
                 IndexRoutingTable.builder(firstRouting.index())
@@ -1037,7 +1037,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
 
         firstRouting = TestShardRouting.newShardRouting("test", 0, "node2", null, null, true, ShardRoutingState.STARTED);
         secondRouting = TestShardRouting.newShardRouting("test", 1, "node2", "node3", null, true, ShardRoutingState.RELOCATING);
-        firstRoutingNode = new RoutingNode("node2", discoveryNode2, Arrays.asList(firstRouting, secondRouting));
+        firstRoutingNode = new RoutingNode("node2", discoveryNode2, firstRouting, secondRouting);
         builder = RoutingTable.builder().add(
                 IndexRoutingTable.builder(firstRouting.index())
                         .addIndexShard(new IndexShardRoutingTable.Builder(firstRouting.shardId())
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
index 15b1463e4d0..3b1799e4b06 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
@@ -43,6 +43,7 @@ import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Arrays;
@@ -106,7 +107,7 @@ public class DiskThresholdDeciderUnitTests extends ESTestCase {
 
         final Index index = metaData.index("test").getIndex();
 
-        ShardRouting test_0 = ShardRouting.newUnassigned(index, 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         DiscoveryNode node_0 = new DiscoveryNode("node_0", DummyTransportAddress.INSTANCE, Collections.emptyMap(),
                 new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
         DiscoveryNode node_1 = new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, Collections.emptyMap(),
@@ -156,24 +157,24 @@ public class DiskThresholdDeciderUnitTests extends ESTestCase {
                 .build();
         final IndexMetaData indexMetaData = metaData.index("test");
 
-        ShardRouting test_0 = ShardRouting.newUnassigned(indexMetaData.getIndex(), 0, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
-        ShardRoutingHelper.initialize(test_0, node_0.getId());
-        ShardRoutingHelper.moveToStarted(test_0);
+        ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 0), null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        test_0 = ShardRoutingHelper.initialize(test_0, node_0.getId());
+        test_0 = ShardRoutingHelper.moveToStarted(test_0);
         shardRoutingMap.put(test_0, "/node0/least");
 
-        ShardRouting test_1 = ShardRouting.newUnassigned(indexMetaData.getIndex(), 1, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
-        ShardRoutingHelper.initialize(test_1, node_1.getId());
-        ShardRoutingHelper.moveToStarted(test_1);
+        ShardRouting test_1 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 1), null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        test_1 = ShardRoutingHelper.initialize(test_1, node_1.getId());
+        test_1 = ShardRoutingHelper.moveToStarted(test_1);
         shardRoutingMap.put(test_1, "/node1/least");
 
-        ShardRouting test_2 = ShardRouting.newUnassigned(indexMetaData.getIndex(), 2, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
-        ShardRoutingHelper.initialize(test_2, node_1.getId());
-        ShardRoutingHelper.moveToStarted(test_2);
+        ShardRouting test_2 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 2), null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        test_2 = ShardRoutingHelper.initialize(test_2, node_1.getId());
+        test_2 = ShardRoutingHelper.moveToStarted(test_2);
         shardRoutingMap.put(test_2, "/node1/most");
 
-        ShardRouting test_3 = ShardRouting.newUnassigned(indexMetaData.getIndex(), 3, null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
-        ShardRoutingHelper.initialize(test_3, node_1.getId());
-        ShardRoutingHelper.moveToStarted(test_3);
+        ShardRouting test_3 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 3), null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        test_3 = ShardRoutingHelper.initialize(test_3, node_1.getId());
+        test_3 = ShardRoutingHelper.moveToStarted(test_3);
         // Intentionally not in the shardRoutingMap. We want to test what happens when we don't know where it is.
 
         RoutingTable routingTable = RoutingTable.builder()
@@ -233,45 +234,44 @@ public class DiskThresholdDeciderUnitTests extends ESTestCase {
         shardSizes.put("[other][0][p]", 10000L);
         ClusterInfo info = new DevNullClusterInfo(ImmutableOpenMap.of(), ImmutableOpenMap.of(), shardSizes.build());
         final Index index = new Index("test", "_na_");
-        ShardRouting test_0 = ShardRouting.newUnassigned(index, 0, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
-        ShardRoutingHelper.initialize(test_0, "node1");
-        ShardRoutingHelper.moveToStarted(test_0);
-        ShardRoutingHelper.relocate(test_0, "node2");
+        ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        test_0 = ShardRoutingHelper.initialize(test_0, "node1");
+        test_0 = ShardRoutingHelper.moveToStarted(test_0);
+        test_0 = ShardRoutingHelper.relocate(test_0, "node2");
 
-        ShardRouting test_1 = ShardRouting.newUnassigned(index, 1, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
-        ShardRoutingHelper.initialize(test_1, "node2");
-        ShardRoutingHelper.moveToStarted(test_1);
-        ShardRoutingHelper.relocate(test_1, "node1");
+        ShardRouting test_1 = ShardRouting.newUnassigned(new ShardId(index, 1), null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        test_1 = ShardRoutingHelper.initialize(test_1, "node2");
+        test_1 = ShardRoutingHelper.moveToStarted(test_1);
+        test_1 = ShardRoutingHelper.relocate(test_1, "node1");
 
-        ShardRouting test_2 = ShardRouting.newUnassigned(index, 2, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
-        ShardRoutingHelper.initialize(test_2, "node1");
-        ShardRoutingHelper.moveToStarted(test_2);
+        ShardRouting test_2 = ShardRouting.newUnassigned(new ShardId(index, 2), null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        test_2 = ShardRoutingHelper.initialize(test_2, "node1");
+        test_2 = ShardRoutingHelper.moveToStarted(test_2);
 
         assertEquals(1000L, DiskThresholdDecider.getShardSize(test_2, info));
         assertEquals(100L, DiskThresholdDecider.getShardSize(test_1, info));
         assertEquals(10L, DiskThresholdDecider.getShardSize(test_0, info));
 
         RoutingNode node = new RoutingNode("node1", new DiscoveryNode("node1", new LocalTransportAddress("test"),
-                emptyMap(), emptySet(), Version.CURRENT), Arrays.asList(test_0, test_1.buildTargetRelocatingShard(), test_2));
+                emptyMap(), emptySet(), Version.CURRENT), test_0, test_1.buildTargetRelocatingShard(), test_2);
         assertEquals(100L, DiskThresholdDecider.sizeOfRelocatingShards(node, info, false, "/dev/null"));
         assertEquals(90L, DiskThresholdDecider.sizeOfRelocatingShards(node, info, true, "/dev/null"));
         assertEquals(0L, DiskThresholdDecider.sizeOfRelocatingShards(node, info, true, "/dev/some/other/dev"));
         assertEquals(0L, DiskThresholdDecider.sizeOfRelocatingShards(node, info, true, "/dev/some/other/dev"));
 
-        ShardRouting test_3 = ShardRouting.newUnassigned(index, 3, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
-        ShardRoutingHelper.initialize(test_3, "node1");
-        ShardRoutingHelper.moveToStarted(test_3);
+        ShardRouting test_3 = ShardRouting.newUnassigned(new ShardId(index, 3), null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        test_3 = ShardRoutingHelper.initialize(test_3, "node1");
+        test_3 = ShardRoutingHelper.moveToStarted(test_3);
         assertEquals(0L, DiskThresholdDecider.getShardSize(test_3, info));
 
 
-        ShardRouting other_0 = ShardRouting.newUnassigned(new Index("other", "_NA_"), 0, null, randomBoolean(), new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
-        ShardRoutingHelper.initialize(other_0, "node2");
-        ShardRoutingHelper.moveToStarted(other_0);
-        ShardRoutingHelper.relocate(other_0, "node1");
-
+        ShardRouting other_0 = ShardRouting.newUnassigned(new ShardId("other", "_NA_", 0), null, randomBoolean(), new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        other_0 = ShardRoutingHelper.initialize(other_0, "node2");
+        other_0 = ShardRoutingHelper.moveToStarted(other_0);
+        other_0 = ShardRoutingHelper.relocate(other_0, "node1");
 
         node = new RoutingNode("node1", new DiscoveryNode("node1", new LocalTransportAddress("test"),
-                emptyMap(), emptySet(), Version.CURRENT), Arrays.asList(test_0, test_1.buildTargetRelocatingShard(), test_2, other_0.buildTargetRelocatingShard()));
+                emptyMap(), emptySet(), Version.CURRENT), test_0, test_1.buildTargetRelocatingShard(), test_2, other_0.buildTargetRelocatingShard());
         if (other_0.primary()) {
             assertEquals(10100L, DiskThresholdDecider.sizeOfRelocatingShards(node, info, false, "/dev/null"));
             assertEquals(10090L, DiskThresholdDecider.sizeOfRelocatingShards(node, info, true, "/dev/null"));
diff --git a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
index bb1bc01780c..96f81e2ab73 100644
--- a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
@@ -284,7 +284,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
     }
 
     private RoutingAllocation onePrimaryOnNode1And1Replica(AllocationDeciders deciders, Settings settings, UnassignedInfo.Reason reason) {
-        ShardRouting primaryShard = TestShardRouting.newShardRouting(shardId.getIndex(), shardId.getId(), node1.getId(), true, ShardRoutingState.STARTED);
+        ShardRouting primaryShard = TestShardRouting.newShardRouting(shardId, node1.getId(), true, ShardRoutingState.STARTED);
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder(shardId.getIndexName()).settings(settings(Version.CURRENT).put(settings))
                     .numberOfShards(1).numberOfReplicas(1)
@@ -294,7 +294,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
                 .add(IndexRoutingTable.builder(shardId.getIndex())
                                 .addIndexShard(new IndexShardRoutingTable.Builder(shardId)
                                         .addShard(primaryShard)
-                                        .addShard(ShardRouting.newUnassigned(shardId.getIndex(), shardId.getId(), null, false, new UnassignedInfo(reason, null)))
+                                        .addShard(ShardRouting.newUnassigned(shardId, null, false, new UnassignedInfo(reason, null)))
                                         .build())
                 )
                 .build();
@@ -306,7 +306,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
     }
 
     private RoutingAllocation onePrimaryOnNode1And1ReplicaRecovering(AllocationDeciders deciders) {
-        ShardRouting primaryShard = TestShardRouting.newShardRouting(shardId.getIndex(), shardId.getId(), node1.getId(), true, ShardRoutingState.STARTED);
+        ShardRouting primaryShard = TestShardRouting.newShardRouting(shardId, node1.getId(), true, ShardRoutingState.STARTED);
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder(shardId.getIndexName()).settings(settings(Version.CURRENT))
                     .numberOfShards(1).numberOfReplicas(1)
@@ -316,7 +316,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
                 .add(IndexRoutingTable.builder(shardId.getIndex())
                                 .addIndexShard(new IndexShardRoutingTable.Builder(shardId)
                                         .addShard(primaryShard)
-                                        .addShard(TestShardRouting.newShardRouting(shardId.getIndex(), shardId.getId(), node2.getId(), null, null, false, ShardRoutingState.INITIALIZING, new UnassignedInfo(UnassignedInfo.Reason.CLUSTER_RECOVERED, null)))
+                                        .addShard(TestShardRouting.newShardRouting(shardId, node2.getId(), null, null, false, ShardRoutingState.INITIALIZING, new UnassignedInfo(UnassignedInfo.Reason.CLUSTER_RECOVERED, null)))
                                         .build())
                 )
                 .build();
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 39aaf072954..551b6389b84 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -207,7 +207,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndexShard shard = test.getShardOrNull(0);
         ShardStateMetaData shardStateMetaData = load(logger, env.availableShardPaths(shard.shardId));
         assertEquals(getShardStateMetadata(shard), shardStateMetaData);
-        ShardRouting routing = new ShardRouting(shard.shardRouting);
+        ShardRouting routing = shard.shardRouting;
         shard.updateRoutingEntry(routing, true);
 
         shardStateMetaData = load(logger, env.availableShardPaths(shard.shardId));
@@ -215,8 +215,8 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         assertEquals(shardStateMetaData, new ShardStateMetaData(routing.primary(), shard.indexSettings().getUUID(), routing.allocationId()));
 
         // check that we don't write shard state metadata if persist == false
-        ShardRouting updatedRouting = new ShardRouting(shard.shardRouting);
-        TestShardRouting.relocate(updatedRouting, "some node", 42L);
+        ShardRouting updatedRouting = shard.shardRouting;
+        updatedRouting = TestShardRouting.relocate(shard.shardRouting, "some node", 42L);
         shard.updateRoutingEntry(updatedRouting, false);
         shardStateMetaData = load(logger, env.availableShardPaths(shard.shardId));
         assertFalse("shard state persisted despite of persist=false", shardStateMetaData.equals(getShardStateMetadata(shard)));
@@ -224,7 +224,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
 
         // check that we write shard state metadata if persist == true
         shard.updateRoutingEntry(routing, false); // move back state in IndexShard
-        routing = new ShardRouting(updatedRouting);
+        routing = updatedRouting;
         shard.updateRoutingEntry(routing, true);
         shardStateMetaData = load(logger, env.availableShardPaths(shard.shardId));
         assertEquals(shardStateMetaData, getShardStateMetadata(shard));
@@ -321,16 +321,16 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         final ShardRouting newPrimaryShardRouting;
         if (randomBoolean()) {
             // relocation target
-            newPrimaryShardRouting = TestShardRouting.newShardRouting(temp.index(), temp.id(), temp.currentNodeId(), "other node",
+            newPrimaryShardRouting = TestShardRouting.newShardRouting(temp.shardId(), temp.currentNodeId(), "other node",
                 true, ShardRoutingState.INITIALIZING, AllocationId.newRelocation(temp.allocationId()));
         } else if (randomBoolean()) {
             // simulate promotion
-            ShardRouting newReplicaShardRouting = TestShardRouting.newShardRouting(temp.index(), temp.id(), temp.currentNodeId(), null,
+            ShardRouting newReplicaShardRouting = TestShardRouting.newShardRouting(temp.shardId(), temp.currentNodeId(), null,
                 false, ShardRoutingState.STARTED, temp.allocationId());
             indexShard.updateRoutingEntry(newReplicaShardRouting, false);
             primaryTerm = primaryTerm + 1;
             indexShard.updatePrimaryTerm(primaryTerm);
-            newPrimaryShardRouting = TestShardRouting.newShardRouting(temp.index(), temp.id(), temp.currentNodeId(), null,
+            newPrimaryShardRouting = TestShardRouting.newShardRouting(temp.shardId(), temp.currentNodeId(), null,
                 true, ShardRoutingState.STARTED, temp.allocationId());
         } else {
             newPrimaryShardRouting = temp;
@@ -369,7 +369,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         switch (randomInt(2)) {
             case 0:
                 // started replica
-                newShardRouting = TestShardRouting.newShardRouting(temp.index(), temp.id(), temp.currentNodeId(), null,
+                newShardRouting = TestShardRouting.newShardRouting(temp.shardId(), temp.currentNodeId(), null,
                     false, ShardRoutingState.STARTED, AllocationId.newRelocation(temp.allocationId()));
 
                 indexShard.updateRoutingEntry(newShardRouting, false);
@@ -377,7 +377,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
             case 1:
                 // initializing replica / primary
                 final boolean relocating = randomBoolean();
-                newShardRouting = TestShardRouting.newShardRouting(temp.index(), temp.id(), temp.currentNodeId(),
+                newShardRouting = TestShardRouting.newShardRouting(temp.shardId(), temp.currentNodeId(),
                     relocating ? "sourceNode" : null,
                     relocating ? randomBoolean() : false,
                     ShardRoutingState.INITIALIZING,
@@ -386,7 +386,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
                 break;
             case 2:
                 // relocation source
-                newShardRouting = TestShardRouting.newShardRouting(temp.index(), temp.id(), temp.currentNodeId(), "otherNode",
+                newShardRouting = TestShardRouting.newShardRouting(temp.shardId(), temp.currentNodeId(), "otherNode",
                     false, ShardRoutingState.RELOCATING, AllocationId.newRelocation(temp.allocationId()));
                 indexShard.updateRoutingEntry(newShardRouting, false);
                 indexShard.relocated("test");
@@ -968,9 +968,9 @@ public class IndexShardTests extends ESSingleNodeTestCase {
             client().admin().indices().prepareFlush().get();
             translogOps = 0;
         }
-        ShardRouting routing = new ShardRouting(shard.routingEntry());
+        ShardRouting routing = shard.routingEntry();
         test.removeShard(0, "b/c simon says so");
-        ShardRoutingHelper.reinit(routing);
+        routing = ShardRoutingHelper.reinit(routing);
         IndexShard newShard = test.createShard(routing);
         newShard.updateRoutingEntry(routing, false);
         DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
@@ -980,8 +980,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         assertEquals(translogOps, newShard.recoveryState().getTranslog().totalOperations());
         assertEquals(translogOps, newShard.recoveryState().getTranslog().totalOperationsOnStart());
         assertEquals(100.0f, newShard.recoveryState().getTranslog().recoveredPercent(), 0.01f);
-        routing = new ShardRouting(routing);
-        ShardRoutingHelper.moveToStarted(routing);
+        routing = ShardRoutingHelper.moveToStarted(routing);
         newShard.updateRoutingEntry(routing, true);
         SearchResponse response = client().prepareSearch().get();
         assertHitCount(response, 1);
@@ -997,9 +996,9 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         if (randomBoolean()) {
             client().admin().indices().prepareFlush().get();
         }
-        ShardRouting routing = new ShardRouting(shard.routingEntry());
+        ShardRouting routing = shard.routingEntry();
         test.removeShard(0, "b/c simon says so");
-        ShardRoutingHelper.reinit(routing, UnassignedInfo.Reason.INDEX_CREATED);
+        routing = ShardRoutingHelper.reinit(routing, UnassignedInfo.Reason.INDEX_CREATED);
         IndexShard newShard = test.createShard(routing);
         newShard.updateRoutingEntry(routing, false);
         DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
@@ -1010,8 +1009,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         assertEquals(0, newShard.recoveryState().getTranslog().totalOperations());
         assertEquals(0, newShard.recoveryState().getTranslog().totalOperationsOnStart());
         assertEquals(100.0f, newShard.recoveryState().getTranslog().recoveredPercent(), 0.01f);
-        routing = new ShardRouting(routing);
-        ShardRoutingHelper.moveToStarted(routing);
+        routing = ShardRoutingHelper.moveToStarted(routing);
         newShard.updateRoutingEntry(routing, true);
         SearchResponse response = client().prepareSearch().get();
         assertHitCount(response, 0);
@@ -1030,13 +1028,13 @@ public class IndexShardTests extends ESSingleNodeTestCase {
             client().admin().indices().prepareFlush().get();
         }
         final ShardRouting origRouting = shard.routingEntry();
-        ShardRouting routing = new ShardRouting(origRouting);
+        ShardRouting routing = origRouting;
         Store store = shard.store();
         store.incRef();
         test.removeShard(0, "b/c simon says so");
         cleanLuceneIndex(store.directory());
         store.decRef();
-        ShardRoutingHelper.reinit(routing);
+        routing = ShardRoutingHelper.reinit(routing);
         IndexShard newShard = test.createShard(routing);
         newShard.updateRoutingEntry(routing, false);
         newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode));
@@ -1047,8 +1045,8 @@ public class IndexShardTests extends ESSingleNodeTestCase {
             assertTrue(ex.getMessage().contains("failed to fetch index version after copying it over"));
         }
 
-        ShardRoutingHelper.moveToUnassigned(routing, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "because I say so"));
-        ShardRoutingHelper.initialize(routing, origRouting.currentNodeId());
+        routing = ShardRoutingHelper.moveToUnassigned(routing, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "because I say so"));
+        routing = ShardRoutingHelper.initialize(routing, origRouting.currentNodeId());
         assertTrue("it's already recovering, we should ignore new ones", newShard.ignoreRecoveryAttempt());
         try {
             newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode));
@@ -1062,8 +1060,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode));
         assertTrue("recover even if there is nothing to recover", newShard.recoverFromStore(localNode));
 
-        routing = new ShardRouting(routing);
-        ShardRoutingHelper.moveToStarted(routing);
+        routing = ShardRoutingHelper.moveToStarted(routing);
         newShard.updateRoutingEntry(routing, true);
         SearchResponse response = client().prepareSearch().get();
         assertHitCount(response, 0);
@@ -1084,14 +1081,12 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         final IndexShard shard = test.getShardOrNull(0);
         ShardRouting origRouting = shard.routingEntry();
         assertThat(shard.state(), equalTo(IndexShardState.STARTED));
-        ShardRouting inRecoveryRouting = new ShardRouting(origRouting);
-        ShardRoutingHelper.relocate(inRecoveryRouting, "some_node");
+        ShardRouting inRecoveryRouting = ShardRoutingHelper.relocate(origRouting, "some_node");
         shard.updateRoutingEntry(inRecoveryRouting, true);
         shard.relocated("simulate mark as relocated");
         assertThat(shard.state(), equalTo(IndexShardState.RELOCATED));
-        ShardRouting failedRecoveryRouting = new ShardRouting(origRouting);
         try {
-            shard.updateRoutingEntry(failedRecoveryRouting, true);
+            shard.updateRoutingEntry(origRouting, true);
             fail("Expected IndexShardRelocatedException");
         } catch (IndexShardRelocatedException expected) {
         }
@@ -1112,8 +1107,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         assertSearchHits(client().prepareSearch("test_target").get(), "1");
         client().admin().indices().prepareFlush("test").get(); // only flush test
         final ShardRouting origRouting = test_target.getShardOrNull(0).routingEntry();
-        ShardRouting routing = new ShardRouting(origRouting);
-        ShardRoutingHelper.reinit(routing);
+        ShardRouting routing = ShardRoutingHelper.reinit(origRouting);
         routing = ShardRoutingHelper.newWithRestoreSource(routing, new RestoreSource(new SnapshotId("foo", "bar"), Version.CURRENT, "test"));
         test_target.removeShard(0, "just do it man!");
         final IndexShard test_target_shard = test_target.createShard(routing);
@@ -1153,8 +1147,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
             }
         }, localNode));
 
-        routing = new ShardRouting(routing);
-        ShardRoutingHelper.moveToStarted(routing);
+        routing = ShardRoutingHelper.moveToStarted(routing);
         test_target_shard.updateRoutingEntry(routing, true);
         assertHitCount(client().prepareSearch("test_target").get(), 1);
         assertSearchHits(client().prepareSearch("test_target").get(), "0");
@@ -1407,14 +1400,12 @@ public class IndexShardTests extends ESSingleNodeTestCase {
     }
 
     public  static final IndexShard recoverShard(IndexShard newShard, ShardRouting oldRouting) throws IOException {
-        ShardRouting routing = new ShardRouting(oldRouting);
-        ShardRoutingHelper.reinit(routing);
+        ShardRouting routing = ShardRoutingHelper.reinit(oldRouting);
         newShard.updateRoutingEntry(routing, false);
         DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
         newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode));
         assertTrue(newShard.recoverFromStore(localNode));
-        routing = new ShardRouting(routing);
-        ShardRoutingHelper.moveToStarted(routing);
+        routing = ShardRoutingHelper.moveToStarted(routing);
         newShard.updateRoutingEntry(routing, true);
         return newShard;
     }
@@ -1441,7 +1432,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         IndexService test = indicesService.indexService(resolveIndex("testindexfortranslogsync"));
         IndexShard shard = test.getShardOrNull(0);
-        ShardRouting routing = new ShardRouting(shard.routingEntry());
+        ShardRouting routing = shard.routingEntry();
         test.removeShard(0, "b/c britta says so");
         IndexShard newShard = test.createShard(routing);
         DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
@@ -1468,7 +1459,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         IndexService test = indicesService.indexService(resolveIndex("index"));
         IndexShard shard = test.getShardOrNull(0);
-        ShardRouting routing = new ShardRouting(shard.routingEntry());
+        ShardRouting routing = shard.routingEntry();
         test.removeShard(0, "b/c britta says so");
         IndexShard newShard = test.createShard(routing);
         newShard.shardRouting = routing;
@@ -1497,7 +1488,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         IndexService test = indicesService.indexService(resolveIndex("index"));
         IndexShard shard = test.getShardOrNull(0);
-        ShardRouting routing = new ShardRouting(shard.routingEntry());
+        ShardRouting routing = shard.routingEntry();
         test.removeShard(0, "b/c britta says so");
         IndexShard newShard = test.createShard(routing);
         newShard.shardRouting = routing;
diff --git a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
index 759288f3dc4..efd42f7f260 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
@@ -18,11 +18,8 @@
  */
 package org.elasticsearch.indices;
 
-import org.apache.lucene.index.DirectoryReader;
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
-import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingHelper;
@@ -30,13 +27,11 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.shard.IndexSearcherWrapper;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardTests;
-import org.elasticsearch.index.shard.IndexingOperationListener;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -46,7 +41,6 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -454,8 +448,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase {
         shardRef.set(newShard);
         try {
             assertEquals(0, imc.availableShards().size());
-            ShardRouting routing = new ShardRouting(shard.routingEntry());
-            ShardRoutingHelper.reinit(routing);
+            ShardRouting routing = ShardRoutingHelper.reinit(shard.routingEntry());
             newShard.updateRoutingEntry(routing, false);
             DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
             newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode));
@@ -463,9 +456,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase {
             assertEquals(1, imc.availableShards().size());
             assertTrue(newShard.recoverFromStore(localNode));
             assertTrue("we should have flushed in IMC at least once but did: " + flushes.get(), flushes.get() >= 1);
-            routing = new ShardRouting(routing);
-            ShardRoutingHelper.moveToStarted(routing);
-            newShard.updateRoutingEntry(routing, true);
+            newShard.updateRoutingEntry(ShardRoutingHelper.moveToStarted(routing), true);
         } finally {
             newShard.close("simon says", false);
         }
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java
index 8763db6b28b..bbb1e8bf2bf 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java
@@ -97,18 +97,17 @@ public class IndicesLifecycleListenerSingleNodeTests extends ESSingleNodeTestCas
             NodeServicesProvider nodeServicesProvider = getInstanceFromNode(NodeServicesProvider.class);
             IndexService index = indicesService.createIndex(nodeServicesProvider, metaData, Arrays.asList(countingListener));
             idx = index.index();
-            ShardRouting newRouting = new ShardRouting(shardRouting);
+            ShardRouting newRouting = shardRouting;
             String nodeId = newRouting.currentNodeId();
-            ShardRoutingHelper.moveToUnassigned(newRouting, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "boom"));
-            ShardRoutingHelper.initialize(newRouting, nodeId);
+            newRouting = ShardRoutingHelper.moveToUnassigned(newRouting, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "boom"));
+            newRouting = ShardRoutingHelper.initialize(newRouting, nodeId);
             IndexShard shard = index.createShard(newRouting);
             shard.updateRoutingEntry(newRouting, true);
             final DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE,
                     emptyMap(), emptySet(), Version.CURRENT);
             shard.markAsRecovering("store", new RecoveryState(shard.shardId(), newRouting.primary(), RecoveryState.Type.SNAPSHOT, newRouting.restoreSource(), localNode));
             shard.recoverFromStore(localNode);
-            newRouting = new ShardRouting(newRouting);
-            ShardRoutingHelper.moveToStarted(newRouting);
+            newRouting = ShardRoutingHelper.moveToStarted(newRouting);
             shard.updateRoutingEntry(newRouting, true);
         } finally {
             indicesService.deleteIndex(idx, "simon says");
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java b/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java
index d15773db545..ca38a575acd 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java
@@ -21,6 +21,7 @@ package org.elasticsearch.cluster.routing;
 
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
 
 /**
@@ -30,51 +31,51 @@ import org.elasticsearch.test.ESTestCase;
 public class TestShardRouting {
 
     public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, boolean primary, ShardRoutingState state) {
-        return newShardRouting(new Index(index, IndexMetaData.INDEX_UUID_NA_VALUE), shardId, currentNodeId, primary, state);
+        return newShardRouting(new ShardId(index, IndexMetaData.INDEX_UUID_NA_VALUE, shardId), currentNodeId, primary, state);
     }
 
-    public static ShardRouting newShardRouting(Index index, int shardId, String currentNodeId, boolean primary, ShardRoutingState state) {
-        return new ShardRouting(index, shardId, currentNodeId, null, null, primary, state, buildUnassignedInfo(state), buildAllocationId(state), true, -1);
+    public static ShardRouting newShardRouting(ShardId shardId, String currentNodeId, boolean primary, ShardRoutingState state) {
+        return new ShardRouting(shardId, currentNodeId, null, null, primary, state, buildUnassignedInfo(state), buildAllocationId(state), -1);
     }
 
     public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, String relocatingNodeId, boolean primary, ShardRoutingState state) {
-        return newShardRouting(new Index(index, IndexMetaData.INDEX_UUID_NA_VALUE), shardId, currentNodeId, relocatingNodeId, primary, state);
+        return newShardRouting(new ShardId(index, IndexMetaData.INDEX_UUID_NA_VALUE, shardId), currentNodeId, relocatingNodeId, primary, state);
     }
 
-    public static ShardRouting newShardRouting(Index index, int shardId, String currentNodeId, String relocatingNodeId, boolean primary, ShardRoutingState state) {
-        return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, null, primary, state, buildUnassignedInfo(state), buildAllocationId(state), true, -1);
+    public static ShardRouting newShardRouting(ShardId shardId, String currentNodeId, String relocatingNodeId, boolean primary, ShardRoutingState state) {
+        return new ShardRouting(shardId, currentNodeId, relocatingNodeId, null, primary, state, buildUnassignedInfo(state), buildAllocationId(state), -1);
     }
 
     public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, String relocatingNodeId, boolean primary, ShardRoutingState state, AllocationId allocationId) {
-        return newShardRouting(new Index(index, IndexMetaData.INDEX_UUID_NA_VALUE), shardId, currentNodeId, relocatingNodeId, primary, state, allocationId);
+        return newShardRouting(new ShardId(index, IndexMetaData.INDEX_UUID_NA_VALUE, shardId), currentNodeId, relocatingNodeId, primary, state, allocationId);
     }
 
-    public static ShardRouting newShardRouting(Index index, int shardId, String currentNodeId, String relocatingNodeId, boolean primary, ShardRoutingState state, AllocationId allocationId) {
-        return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, null, primary, state, buildUnassignedInfo(state), allocationId, true, -1);
+    public static ShardRouting newShardRouting(ShardId shardId, String currentNodeId, String relocatingNodeId, boolean primary, ShardRoutingState state, AllocationId allocationId) {
+        return new ShardRouting(shardId, currentNodeId, relocatingNodeId, null, primary, state, buildUnassignedInfo(state), allocationId, -1);
     }
 
     public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId, String relocatingNodeId, RestoreSource restoreSource, boolean primary, ShardRoutingState state) {
-        return newShardRouting(new Index(index, IndexMetaData.INDEX_UUID_NA_VALUE), shardId, currentNodeId, relocatingNodeId, restoreSource, primary, state);
+        return newShardRouting(new ShardId(index, IndexMetaData.INDEX_UUID_NA_VALUE, shardId), currentNodeId, relocatingNodeId, restoreSource, primary, state);
     }
 
-    public static ShardRouting newShardRouting(Index index, int shardId, String currentNodeId, String relocatingNodeId, RestoreSource restoreSource, boolean primary, ShardRoutingState state) {
-        return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, restoreSource, primary, state, buildUnassignedInfo(state), buildAllocationId(state), true, -1);
+    public static ShardRouting newShardRouting(ShardId shardId, String currentNodeId, String relocatingNodeId, RestoreSource restoreSource, boolean primary, ShardRoutingState state) {
+        return new ShardRouting(shardId, currentNodeId, relocatingNodeId, restoreSource, primary, state, buildUnassignedInfo(state), buildAllocationId(state), -1);
     }
 
     public static ShardRouting newShardRouting(String index, int shardId, String currentNodeId,
                                                String relocatingNodeId, RestoreSource restoreSource, boolean primary, ShardRoutingState state,
                                                UnassignedInfo unassignedInfo) {
-        return newShardRouting(new Index(index, IndexMetaData.INDEX_UUID_NA_VALUE), shardId, currentNodeId, relocatingNodeId, restoreSource, primary, state, unassignedInfo);
+        return newShardRouting(new ShardId(index, IndexMetaData.INDEX_UUID_NA_VALUE, shardId), currentNodeId, relocatingNodeId, restoreSource, primary, state, unassignedInfo);
     }
 
-    public static ShardRouting newShardRouting(Index index, int shardId, String currentNodeId,
+    public static ShardRouting newShardRouting(ShardId shardId, String currentNodeId,
                                                String relocatingNodeId, RestoreSource restoreSource, boolean primary, ShardRoutingState state,
                                                UnassignedInfo unassignedInfo) {
-        return new ShardRouting(index, shardId, currentNodeId, relocatingNodeId, restoreSource, primary, state, unassignedInfo, buildAllocationId(state), true, -1);
+        return new ShardRouting(shardId, currentNodeId, relocatingNodeId, restoreSource, primary, state, unassignedInfo, buildAllocationId(state), -1);
     }
 
-    public static void relocate(ShardRouting shardRouting, String relocatingNodeId, long expectedShardSize) {
-        shardRouting.relocate(relocatingNodeId, expectedShardSize);
+    public static ShardRouting relocate(ShardRouting shardRouting, String relocatingNodeId, long expectedShardSize) {
+        return shardRouting.relocate(relocatingNodeId, expectedShardSize);
     }
 
     private static AllocationId buildAllocationId(ShardRoutingState state) {

From a0ffe6ea89770787783c7aefc0f8c8990198cb80 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Tue, 10 May 2016 19:40:19 +0200
Subject: [PATCH 0170/1311] =?UTF-8?q?[TEST]=C2=A0Ensure=20creation=20of=20?=
 =?UTF-8?q?valid=20routing=20table?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

An additional sanity check introduced by #17821 makes some tests fail. This check verifies that
only one shard with same shard id is allocated to a node. This commit fixes a bug in
ClusterStateCreationUtils which would construct a cluster state that allocated two shards with same
id to the same node.
---
 .../action/support/replication/ClusterStateCreationUtils.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
index c498bf6ef5b..dc40fda3f8e 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
@@ -107,6 +107,7 @@ public class ClusterStateCreationUtils {
                 Set<String> unassignedNodesExecludingPrimary = new HashSet<>(unassignedNodes);
                 unassignedNodesExecludingPrimary.remove(newNode(0).getId());
                 primaryNode = selectAndRemove(unassignedNodesExecludingPrimary);
+                unassignedNodes.remove(primaryNode);
             }
             if (primaryState == ShardRoutingState.RELOCATING) {
                 relocatingNode = selectAndRemove(unassignedNodes);

From ef6a7631cfa29e5f3819c23b83b8194aacd36476 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 10 May 2016 15:06:51 -0400
Subject: [PATCH 0171/1311] Switch scripts to use bash

This commit switches the command-line scripts to use bash instead of sh
so that we can take advantage of features that bash provides like
arrays.

Relates #18251
---
 distribution/src/main/resources/bin/elasticsearch        | 2 +-
 distribution/src/main/resources/bin/elasticsearch-plugin | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/distribution/src/main/resources/bin/elasticsearch b/distribution/src/main/resources/bin/elasticsearch
index 0101a108d4b..b41304f70d0 100755
--- a/distribution/src/main/resources/bin/elasticsearch
+++ b/distribution/src/main/resources/bin/elasticsearch
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # CONTROLLING STARTUP:
 #
diff --git a/distribution/src/main/resources/bin/elasticsearch-plugin b/distribution/src/main/resources/bin/elasticsearch-plugin
index b1ba48d44f1..e609591bc80 100755
--- a/distribution/src/main/resources/bin/elasticsearch-plugin
+++ b/distribution/src/main/resources/bin/elasticsearch-plugin
@@ -1,5 +1,4 @@
-#!/bin/sh
-
+#!/bin/bash
 
 CDPATH=""
 SCRIPT="$0"

From 1dfbe8b0f13fae3c3dbe71f02b752b4d3a4cfeb0 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 10 May 2016 15:32:23 -0400
Subject: [PATCH 0172/1311] Fix handling of default path conf in plugin script

In the plugin script we set the setting default.path.conf to control the
path to the configuration file. But the default directory might not
exist in which case we can run into exceptions. This commit restores a
guard against this directory not existing.
---
 distribution/src/main/resources/bin/elasticsearch-plugin | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/distribution/src/main/resources/bin/elasticsearch-plugin b/distribution/src/main/resources/bin/elasticsearch-plugin
index e609591bc80..8a3b6676a98 100755
--- a/distribution/src/main/resources/bin/elasticsearch-plugin
+++ b/distribution/src/main/resources/bin/elasticsearch-plugin
@@ -81,5 +81,10 @@ fi
 HOSTNAME=`hostname | cut -d. -f1`
 export HOSTNAME
 
-exec "$JAVA" $ES_JAVA_OPTS -Delasticsearch -Des.path.home="$ES_HOME" -Des.default.path.conf="$CONF_DIR" \
-  -cp "$ES_HOME/lib/*" org.elasticsearch.plugins.PluginCli "$@"
+declare -a properties=(-Delasticsearch -Des.path.home="$ES_HOME")
+
+if [ -e "$CONF_DIR" ]; then
+  properties=("${properties[@]}" -Des.default.path.conf="$CONF_DIR")
+fi
+
+exec "$JAVA" $ES_JAVA_OPTS "${properties[@]}" -cp "$ES_HOME/lib/*" org.elasticsearch.plugins.PluginCli "$@"

From 531dcbf20ae1b7cb69f01e512d5ca79714b19047 Mon Sep 17 00:00:00 2001
From: John Barker <jebarker@gmail.com>
Date: Tue, 10 May 2016 16:19:46 -0400
Subject: [PATCH 0173/1311] Add TAG_SETTING to list of allowed tags for the ec2
 discovery plugin.

I am unable to set ec2 discovery tags because this setting was
accidentally omitted from the register settings list in
Ec2DiscoveryPlugin.java. I get this:

java.lang.IllegalArgumentException: unknown setting [discovery.ec2.tag.project]
---
 .../elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
index b4c293ec736..7582e138c61 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
@@ -134,6 +134,7 @@ public class Ec2DiscoveryPlugin extends Plugin {
         settingsModule.registerSetting(AwsEc2Service.DISCOVERY_EC2.GROUPS_SETTING);
         settingsModule.registerSetting(AwsEc2Service.DISCOVERY_EC2.AVAILABILITY_ZONES_SETTING);
         settingsModule.registerSetting(AwsEc2Service.DISCOVERY_EC2.NODE_CACHE_TIME_SETTING);
+        settingsModule.registerSetting(AwsEc2Service.DISCOVERY_EC2.TAG_SETTING);
     }
 
     /**

From a63285aaf997b27de7c78dfa1fa873ce3ac9b838 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 10 May 2016 20:53:08 -0400
Subject: [PATCH 0174/1311] painless: retrieve _score directly from Scorer

---
 .../org/elasticsearch/painless/Analyzer.java  |  1 +
 .../elasticsearch/painless/Executable.java    |  4 +++-
 .../org/elasticsearch/painless/Metadata.java  |  6 +++++
 .../elasticsearch/painless/ScriptImpl.java    | 11 ++++++---
 .../org/elasticsearch/painless/Writer.java    | 24 ++++---------------
 .../painless/WriterConstants.java             | 10 ++++----
 6 files changed, 27 insertions(+), 29 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
index f411db00802..092338804f9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
@@ -94,6 +94,7 @@ class Analyzer extends PainlessParserBaseVisitor<Void> {
         utility.incrementScope();
         utility.addVariable(null, "#this", definition.execType);
         metadata.inputValueSlot = utility.addVariable(null, "input", definition.smapType).slot;
+        metadata.scorerValueSlot = utility.addVariable(null, "#scorer", definition.objectType).slot;
         metadata.loopCounterSlot = utility.addVariable(null, "#loop", definition.intType).slot;
         metadata.scoreValueSlot = utility.addVariable(null, "_score", definition.floatType).slot;
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java
index 6d9a7108fa9..ba284cc455b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.painless;
 
+import org.apache.lucene.search.Scorer;
+
 import java.util.Map;
 
 public abstract class Executable {
@@ -46,5 +48,5 @@ public abstract class Executable {
         return definition;
     }
 
-    public abstract Object execute(Map<String, Object> input);
+    public abstract Object execute(Map<String, Object> input, Scorer scorer);
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java
index bb5c48f7de1..ce46acd6379 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java
@@ -410,6 +410,12 @@ class Metadata {
      */
     int inputValueSlot = -1;
 
+    /**
+     * Used to determine what slot the Scorer variable is stored in.  This is used in the {@link Writer} to load
+     * _score from it, if _score will be accessed by the script.
+     */
+    int scorerValueSlot = -1;
+
     /**
      * Used to determine what slot the loopCounter variable is stored in.  This is used n the {@link Writer} whenever
      * the loop variable is accessed.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
index 34fc2cdd90a..93f9f625b64 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
@@ -22,7 +22,6 @@ package org.elasticsearch.painless;
 import org.apache.lucene.search.Scorer;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.LeafSearchScript;
-import org.elasticsearch.script.ScoreAccessor;
 import org.elasticsearch.search.lookup.LeafSearchLookup;
 
 import java.util.HashMap;
@@ -48,6 +47,12 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
      */
     private final LeafSearchLookup lookup;
 
+    /**
+     * Current scorer being used
+     * @see #setScorer(Scorer)
+     */
+    private Scorer scorer;
+
     /**
      * Creates a ScriptImpl for the a previously compiled Painless script.
      * @param executable The previously compiled Painless script.
@@ -84,7 +89,7 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
      */
     @Override
     public Object run() {
-        return executable.execute(variables);
+        return executable.execute(variables, scorer);
     }
 
     /**
@@ -120,7 +125,7 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
      */
     @Override
     public void setScorer(final Scorer scorer) {
-        variables.put("#score", new ScoreAccessor(scorer));
+        this.scorer = scorer;
     }
 
     /**
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
index 2c60d3ad113..6a57c40b817 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
@@ -72,7 +72,6 @@ import org.elasticsearch.painless.PainlessParser.TryContext;
 import org.elasticsearch.painless.PainlessParser.UnaryContext;
 import org.elasticsearch.painless.PainlessParser.WhileContext;
 import org.objectweb.asm.ClassWriter;
-import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.commons.GeneratorAdapter;
 
@@ -80,10 +79,6 @@ import static org.elasticsearch.painless.WriterConstants.BASE_CLASS_TYPE;
 import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE;
 import static org.elasticsearch.painless.WriterConstants.CONSTRUCTOR;
 import static org.elasticsearch.painless.WriterConstants.EXECUTE;
-import static org.elasticsearch.painless.WriterConstants.MAP_GET;
-import static org.elasticsearch.painless.WriterConstants.MAP_TYPE;
-import static org.elasticsearch.painless.WriterConstants.SCORE_ACCESSOR_FLOAT;
-import static org.elasticsearch.painless.WriterConstants.SCORE_ACCESSOR_TYPE;
 
 class Writer extends PainlessParserBaseVisitor<Void> {
     static byte[] write(Metadata metadata) {
@@ -155,22 +150,11 @@ class Writer extends PainlessParserBaseVisitor<Void> {
     }
 
     private void writeExecute() {
-        final Label fals = new Label();
-        final Label end = new Label();
-
         if (metadata.scoreValueUsed) {
-            execute.visitVarInsn(Opcodes.ALOAD, metadata.inputValueSlot);
-            execute.push("#score");
-            execute.invokeInterface(MAP_TYPE, MAP_GET);
-            execute.dup();
-            execute.ifNull(fals);
-            execute.checkCast(SCORE_ACCESSOR_TYPE);
-            execute.invokeVirtual(SCORE_ACCESSOR_TYPE, SCORE_ACCESSOR_FLOAT);
-            execute.goTo(end);
-            execute.mark(fals);
-            execute.pop();
-            execute.push(0F);
-            execute.mark(end);
+            // if the _score value is used, we do this once:
+            //   float _score = scorer.score();
+            execute.visitVarInsn(Opcodes.ALOAD, metadata.scorerValueSlot);
+            execute.invokeVirtual(WriterConstants.SCORER_TYPE, WriterConstants.SCORER_SCORE);
             execute.visitVarInsn(Opcodes.FSTORE, metadata.scoreValueSlot);
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index a7cc7346229..5c8e3604671 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.painless;
 
-import org.elasticsearch.script.ScoreAccessor;
+import org.apache.lucene.search.Scorer;
 import org.objectweb.asm.Handle;
 import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.Type;
@@ -37,7 +37,7 @@ class WriterConstants {
     final static Type CLASS_TYPE        = Type.getType("L" + CLASS_NAME.replace(".", "/") + ";");
 
     final static Method CONSTRUCTOR = getAsmMethod(void.class, "<init>", Definition.class, String.class, String.class);
-    final static Method EXECUTE     = getAsmMethod(Object.class, "execute", Map.class);
+    final static Method EXECUTE     = getAsmMethod(Object.class, "execute", Map.class, Scorer.class);
 
     final static Type PAINLESS_ERROR_TYPE = Type.getType(PainlessError.class);
 
@@ -47,12 +47,12 @@ class WriterConstants {
 
     final static Type OBJECT_TYPE = Type.getType(Object.class);
 
+    final static Type SCORER_TYPE = Type.getType(Scorer.class);
+    final static Method SCORER_SCORE = getAsmMethod(float.class, "score");
+
     final static Type MAP_TYPE  = Type.getType(Map.class);
     final static Method MAP_GET = getAsmMethod(Object.class, "get", Object.class);
 
-    final static Type SCORE_ACCESSOR_TYPE    = Type.getType(ScoreAccessor.class);
-    final static Method SCORE_ACCESSOR_FLOAT = getAsmMethod(float.class, "floatValue");
-
     /** dynamic callsite bootstrap signature */
     final static MethodType DEF_BOOTSTRAP_TYPE = MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, 
                                                                        String.class, MethodType.class, int.class);

From 2bf585e642f4895abc5f5966f3664645891d1586 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 10 May 2016 21:17:09 -0400
Subject: [PATCH 0175/1311] Require /bin/bash in packaging

This commit adds a hard requirement to the RPM and Debian packages for
/bin/bash to be present, and adds a note regarding this to the migration
docs.

Relates #18259
---
 distribution/build.gradle                               | 2 ++
 docs/reference/migration/migrate_5_0/packaging.asciidoc | 5 +++++
 2 files changed, 7 insertions(+)

diff --git a/distribution/build.gradle b/distribution/build.gradle
index 2fab35229be..5cd1e761b83 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -317,6 +317,8 @@ configure(subprojects.findAll { ['deb', 'rpm'].contains(it.name) }) {
     preUninstall file("${scripts}/prerm")
     postUninstall file("${scripts}/postrm")
 
+    requires('/bin/bash')
+
     into '/usr/share/elasticsearch'
     fileMode 0644
     dirMode 0755
diff --git a/docs/reference/migration/migrate_5_0/packaging.asciidoc b/docs/reference/migration/migrate_5_0/packaging.asciidoc
index c714ef163e1..5911b964b6b 100644
--- a/docs/reference/migration/migrate_5_0/packaging.asciidoc
+++ b/docs/reference/migration/migrate_5_0/packaging.asciidoc
@@ -37,4 +37,9 @@ from the tar or zip distributions, and /etc/elasticsearch/jvm.options if install
 from the Debian or RPM packages. You can specify an alternative location by setting
 the environment variable `ES_JVM_OPTIONS` to the path to the file.
 
+==== /bin/bash is now required
 
+Previously, the scripts used to start Elasticsearch and run plugin
+commands only required a Bourne-compatible shell. Starting in
+Elasticsearch 5.0.0, the bash shell is now required and `/bin/bash` is a
+hard-dependency for the RPM and Debian packages.

From 5f0cc79562506c84fa091019dad26d71b57098c1 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 10 May 2016 22:06:37 -0400
Subject: [PATCH 0176/1311] Sort plugins in list plugins command

This commit modifies the list plugins command to produce deterministic
output by sorting the plugins by comparing paths.

Relates #18260
---
 .../plugins/ListPluginsCommand.java            | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java b/core/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java
index b513be99dcf..c03e70ad4da 100644
--- a/core/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java
@@ -23,6 +23,9 @@ import java.io.IOException;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 
 import joptsimple.OptionSet;
 import org.elasticsearch.cli.Command;
@@ -48,12 +51,17 @@ class ListPluginsCommand extends Command {
         }
 
         terminal.println(Terminal.Verbosity.VERBOSE, "Plugins directory: " + env.pluginsFile());
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(env.pluginsFile())) {
-            for (Path plugin : stream) {
-                terminal.println(plugin.getFileName().toString());
-                PluginInfo info = PluginInfo.readFromProperties(env.pluginsFile().resolve(plugin.toAbsolutePath()));
-                terminal.println(Terminal.Verbosity.VERBOSE, info.toString());
+        final List<Path> plugins = new ArrayList<>();
+        try (DirectoryStream<Path> paths = Files.newDirectoryStream(env.pluginsFile())) {
+            for (Path plugin : paths) {
+                plugins.add(plugin);
             }
         }
+        Collections.sort(plugins);
+        for (final Path plugin : plugins) {
+            terminal.println(plugin.getFileName().toString());
+            PluginInfo info = PluginInfo.readFromProperties(env.pluginsFile().resolve(plugin.toAbsolutePath()));
+            terminal.println(Terminal.Verbosity.VERBOSE, info.toString());
+        }
     }
 }

From abb015bcb95a474e35a686f340033b5254d1b05e Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 10 May 2016 22:55:06 -0400
Subject: [PATCH 0177/1311] add tests for reserved words. ban stores against
 _score

---
 .../painless/AnalyzerExternal.java            |  5 +++
 .../painless/ReservedWordTests.java           | 40 +++++++++++++++++++
 2 files changed, 45 insertions(+)
 create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java
index 32ab8f43c9e..5edd4e09c4e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java
@@ -448,7 +448,12 @@ class AnalyzerExternal {
                 throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Unknown variable [" + id + "].");
             }
 
+            // special cases: reserved words
             if ("_score".equals(id)) {
+                // read-only: don't allow stores
+                if (parentemd.storeExpr != null) {
+                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Variable [" + id + "] is read-only.");
+                }
                 metadata.scoreValueUsed = true;
             }
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java
new file mode 100644
index 00000000000..c68a158207b
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+/** Tests for special reserved words such as _score */
+public class ReservedWordTests extends ScriptTestCase {
+    
+    /** check that we can't declare a variable of _score, its really reserved! */
+    public void testScoreVar() {
+        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+            exec("int _score = 5; return _score;");
+        });
+        assertTrue(expected.getMessage().contains("Variable name [_score] already defined"));
+    }
+    
+    /** check that we can't write to _score, its read-only! */
+    public void testScoreStore() {
+        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+            exec("_score = 5; return _score;");
+        });
+        assertTrue(expected.getMessage().contains("Variable [_score] is read-only"));
+    }
+}

From 389ef462c4fad06b7b7362bde7fdd6c543b8eb79 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 10 May 2016 23:43:57 -0400
Subject: [PATCH 0178/1311] special access to doc

---
 .../org/elasticsearch/painless/Analyzer.java  | 13 ++++++++++
 .../painless/AnalyzerExternal.java            |  6 ++++-
 .../elasticsearch/painless/Executable.java    |  3 ++-
 .../org/elasticsearch/painless/Metadata.java  |  6 +++++
 .../elasticsearch/painless/ScriptImpl.java    | 11 ++++++++-
 .../painless/WriterConstants.java             |  3 ++-
 .../painless/NeedsScoreTests.java             |  4 ++--
 .../rest-api-spec/test/plan_a/16_update2.yaml |  8 +++----
 .../test/plan_a/20_scriptfield.yaml           |  2 +-
 .../rest-api-spec/test/plan_a/30_search.yaml  | 24 +++++++++----------
 10 files changed, 57 insertions(+), 23 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
index 092338804f9..9803cd36ef8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
@@ -93,9 +93,22 @@ class Analyzer extends PainlessParserBaseVisitor<Void> {
 
         utility.incrementScope();
         utility.addVariable(null, "#this", definition.execType);
+        //
+        // reserved words parameters.
+        //
+        // input map of variables passed to the script. TODO: rename to 'params' since that will be its use
         metadata.inputValueSlot = utility.addVariable(null, "input", definition.smapType).slot;
+        // scorer parameter passed to the script. internal use only.
         metadata.scorerValueSlot = utility.addVariable(null, "#scorer", definition.objectType).slot;
+        // doc parameter passed to the script.
+        // TODO: currently working as a def type, should be smapType...
+        metadata.docValueSlot = utility.addVariable(null, "doc", definition.defType).slot;
+        //
+        // reserved words implemented as local variables
+        //
+        // loop counter to catch runaway scripts. internal use only.
         metadata.loopCounterSlot = utility.addVariable(null, "#loop", definition.intType).slot;
+        // document's score as a read-only float.
         metadata.scoreValueSlot = utility.addVariable(null, "_score", definition.floatType).slot;
 
         metadata.createStatementMetadata(metadata.root);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java
index 5edd4e09c4e..176a19d7dc2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java
@@ -449,11 +449,15 @@ class AnalyzerExternal {
             }
 
             // special cases: reserved words
-            if ("_score".equals(id)) {
+            if ("_score".equals(id) || "doc".equals(id)) {
                 // read-only: don't allow stores
                 if (parentemd.storeExpr != null) {
                     throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Variable [" + id + "] is read-only.");
                 }
+            }
+
+            // track if the _score value is ever used, we will invoke Scorer.score() only once if so.
+            if ("_score".equals(id)) {
                 metadata.scoreValueUsed = true;
             }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java
index ba284cc455b..7878c72da89 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless;
 
 import org.apache.lucene.search.Scorer;
+import org.elasticsearch.search.lookup.LeafDocLookup;
 
 import java.util.Map;
 
@@ -48,5 +49,5 @@ public abstract class Executable {
         return definition;
     }
 
-    public abstract Object execute(Map<String, Object> input, Scorer scorer);
+    public abstract Object execute(Map<String, Object> input, Scorer scorer, LeafDocLookup doc);
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java
index ce46acd6379..85657360ef0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java
@@ -433,6 +433,12 @@ class Metadata {
      * variable slots at the completion of analysis if _score is not used.
      */
     boolean scoreValueUsed = false;
+    
+    /**
+     * Used to determine what slot the doc variable is stored in.  This is used in the {@link Writer} whenever
+     * the doc variable is accessed.
+     */
+    int docValueSlot = -1;
 
     /**
      * Maps the relevant ANTLR node to its metadata.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
index 93f9f625b64..3bcab8799e6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
@@ -22,6 +22,7 @@ package org.elasticsearch.painless;
 import org.apache.lucene.search.Scorer;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.LeafSearchScript;
+import org.elasticsearch.search.lookup.LeafDocLookup;
 import org.elasticsearch.search.lookup.LeafSearchLookup;
 
 import java.util.HashMap;
@@ -46,6 +47,11 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
      * The lookup is used to access search field values at run-time.
      */
     private final LeafSearchLookup lookup;
+    
+    /**
+     * the 'doc' object accessed by the script, if available.
+     */
+    private final LeafDocLookup doc;
 
     /**
      * Current scorer being used
@@ -70,6 +76,9 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
 
         if (lookup != null) {
             variables.putAll(lookup.asMap());
+            doc = lookup.doc();
+        } else {
+            doc = null;
         }
     }
 
@@ -89,7 +98,7 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
      */
     @Override
     public Object run() {
-        return executable.execute(variables, scorer);
+        return executable.execute(variables, scorer, doc);
     }
 
     /**
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index 5c8e3604671..ec6e4e35b2b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless;
 
 import org.apache.lucene.search.Scorer;
+import org.elasticsearch.search.lookup.LeafDocLookup;
 import org.objectweb.asm.Handle;
 import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.Type;
@@ -37,7 +38,7 @@ class WriterConstants {
     final static Type CLASS_TYPE        = Type.getType("L" + CLASS_NAME.replace(".", "/") + ";");
 
     final static Method CONSTRUCTOR = getAsmMethod(void.class, "<init>", Definition.class, String.class, String.class);
-    final static Method EXECUTE     = getAsmMethod(Object.class, "execute", Map.class, Scorer.class);
+    final static Method EXECUTE     = getAsmMethod(Object.class, "execute", Map.class, Scorer.class, LeafDocLookup.class);
 
     final static Type PAINLESS_ERROR_TYPE = Type.getType(PainlessError.class);
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java
index 8861514e9bc..e3aefda4766 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java
@@ -46,7 +46,7 @@ public class NeedsScoreTests extends ESSingleNodeTestCase {
                                          lookup, Collections.<String, Object>emptyMap());
         assertFalse(ss.needsScores());
 
-        compiled = service.compile("input.doc['d'].value", Collections.emptyMap());
+        compiled = service.compile("doc['d'].value", Collections.emptyMap());
         ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled), 
                             lookup, Collections.<String, Object>emptyMap());
         assertFalse(ss.needsScores());
@@ -56,7 +56,7 @@ public class NeedsScoreTests extends ESSingleNodeTestCase {
                             lookup, Collections.<String, Object>emptyMap());
         assertTrue(ss.needsScores());
 
-        compiled = service.compile("input.doc['d'].value * _score", Collections.emptyMap());
+        compiled = service.compile("doc['d'].value * _score", Collections.emptyMap());
         ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled), 
                             lookup, Collections.<String, Object>emptyMap());
         assertTrue(ss.needsScores());
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/16_update2.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/16_update2.yaml
index bef3250621b..c7eed81f84b 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/16_update2.yaml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/16_update2.yaml
@@ -5,7 +5,7 @@
       put_script:
         id: "1"
         lang: "painless"
-        body: { "script":  "_score * input.doc[\"myParent.weight\"].value" }
+        body: { "script":  "_score * doc[\"myParent.weight\"].value" }
   - match: { acknowledged: true }
 
   - do:
@@ -15,7 +15,7 @@
   - match: { found: true }
   - match: { lang: painless }
   - match: { _id: "1" }
-  - match: { "script":  "_score * input.doc[\"myParent.weight\"].value" }
+  - match: { "script":  "_score * doc[\"myParent.weight\"].value" }
 
   - do:
      catch: missing
@@ -44,11 +44,11 @@
       put_script:
         id: "1"
         lang: "painless"
-        body: { "script":  "_score * foo bar + input.doc[\"myParent.weight\"].value" }
+        body: { "script":  "_score * foo bar + doc[\"myParent.weight\"].value" }
 
   - do:
       catch: /Unable.to.parse.*/
       put_script:
         id: "1"
         lang: "painless"
-        body: { "script":  "_score * foo bar + input.doc[\"myParent.weight\"].value" }
+        body: { "script":  "_score * foo bar + doc[\"myParent.weight\"].value" }
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/20_scriptfield.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/20_scriptfield.yaml
index a1087f17d4e..f85bbef121f 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/20_scriptfield.yaml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/20_scriptfield.yaml
@@ -28,7 +28,7 @@ setup:
                 script_fields:
                     bar:
                         script: 
-                            inline: "input.doc['foo'].value + input.x;"
+                            inline: "doc['foo'].value + input.x;"
                             lang: painless
                             params:
                                 x: "bbb"
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml
index db39e6a31b9..73d267a3fe5 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml
@@ -29,12 +29,12 @@
                 query:
                     script:
                         script:
-                            inline: "input.doc['num1'].value > 1;"
+                            inline: "doc['num1'].value > 1;"
                             lang: painless
                 script_fields:
                     sNum1:
                         script: 
-                            inline: "input.doc['num1'].value;"
+                            inline: "doc['num1'].value;"
                             lang: painless
                 sort:
                     num1:
@@ -51,7 +51,7 @@
                 query:
                     script:
                         script:
-                            inline: "input.doc['num1'].value > input.param1;"
+                            inline: "doc['num1'].value > input.param1;"
                             lang: painless
                             params:
                                 param1: 1
@@ -59,7 +59,7 @@
                 script_fields:
                     sNum1:
                         script:
-                            inline: "return input.doc['num1'].value;"
+                            inline: "return doc['num1'].value;"
                             lang: painless
                 sort:
                     num1:
@@ -76,7 +76,7 @@
                 query:
                     script:
                         script:
-                            inline: "input.doc['num1'].value > input.param1;"
+                            inline: "doc['num1'].value > input.param1;"
                             lang: painless
                             params:
                                 param1: -1
@@ -84,7 +84,7 @@
                 script_fields:
                     sNum1:
                         script: 
-                            inline: "input.doc['num1'].value;"
+                            inline: "doc['num1'].value;"
                             lang: painless
                 sort:
                     num1:
@@ -126,7 +126,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "painless",
-                                    "inline": "input.doc['num1'].value"
+                                    "inline": "doc['num1'].value"
                                 }
                             }
                         }]
@@ -148,7 +148,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "painless",
-                                    "inline": "-input.doc['num1'].value"
+                                    "inline": "-doc['num1'].value"
                                 }
                             }
                         }]
@@ -170,7 +170,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "painless",
-                                    "inline": "Math.pow(input.doc['num1'].value, 2)"
+                                    "inline": "Math.pow(doc['num1'].value, 2)"
                                 }
                             }
                         }]
@@ -192,7 +192,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "painless",
-                                    "inline": "Math.max(input.doc['num1'].value, 1)"
+                                    "inline": "Math.max(doc['num1'].value, 1)"
                                 }
                             }
                         }]
@@ -214,7 +214,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "painless",
-                                    "inline": "input.doc['num1'].value * _score"
+                                    "inline": "doc['num1'].value * _score"
                                 }
                             }
                         }]
@@ -357,7 +357,7 @@
                 script_fields:
                     foobar:
                         script: 
-                            inline: "input.doc['f'].values.size()"
+                            inline: "doc['f'].values.size()"
                             lang: painless
 
 

From 632bb4c720570fad4dcd7da44b7521bc6fa6290b Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Wed, 11 May 2016 00:17:19 -0400
Subject: [PATCH 0179/1311] add reserved word tests for 'doc', too

---
 .../painless/ReservedWordTests.java              | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java
index c68a158207b..64296812bda 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java
@@ -37,4 +37,20 @@ public class ReservedWordTests extends ScriptTestCase {
         });
         assertTrue(expected.getMessage().contains("Variable [_score] is read-only"));
     }
+    
+    /** check that we can't declare a variable of doc, its really reserved! */
+    public void testDocVar() {
+        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+            exec("int doc = 5; return doc;");
+        });
+        assertTrue(expected.getMessage().contains("Variable name [doc] already defined"));
+    }
+    
+    /** check that we can't write to _score, its read-only! */
+    public void testDocStore() {
+        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+            exec("doc = 5; return doc;");
+        });
+        assertTrue(expected.getMessage().contains("Variable [doc] is read-only"));
+    }
 }

From d8b373cac604faa8cceb5e4af3945e5b75c372fc Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Tue, 10 May 2016 23:01:43 -0700
Subject: [PATCH 0180/1311] Fixed an issue related to bracket shortcuts.

---
 .../org/elasticsearch/painless/Analyzer.java  |  2 +-
 .../painless/AnalyzerExternal.java            | 18 ++++----
 .../painless/WriterExternal.java              | 46 ++++++++-----------
 .../painless/NeedsScoreTests.java             |  8 ++--
 4 files changed, 33 insertions(+), 41 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
index 9803cd36ef8..eb9ac2979c7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
@@ -102,7 +102,7 @@ class Analyzer extends PainlessParserBaseVisitor<Void> {
         metadata.scorerValueSlot = utility.addVariable(null, "#scorer", definition.objectType).slot;
         // doc parameter passed to the script.
         // TODO: currently working as a def type, should be smapType...
-        metadata.docValueSlot = utility.addVariable(null, "doc", definition.defType).slot;
+        metadata.docValueSlot = utility.addVariable(null, "doc", definition.smapType).slot;
         //
         // reserved words implemented as local variables
         //
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java
index 176a19d7dc2..f907138f752 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java
@@ -238,14 +238,6 @@ class AnalyzerExternal {
                 definition.getType(parentemd.current.struct, parentemd.current.type.getDimensions() - 1);
             analyzeLoadStoreExternal(ctx);
             parentemd.current = braceenmd.type;
-
-            if (dotctx != null) {
-                metadata.createExtNodeMetadata(parent, dotctx);
-                analyzer.visit(dotctx);
-            } else if (bracectx != null) {
-                metadata.createExtNodeMetadata(parent, bracectx);
-                analyzer.visit(bracectx);
-            }
         } else {
             final boolean store = braceenmd.last && parentemd.storeExpr != null;
             final boolean get = parentemd.read || parentemd.token > 0 || !braceenmd.last;
@@ -319,6 +311,14 @@ class AnalyzerExternal {
             throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
                 "Attempting to address a non-array type [" + parentemd.current.name + "] as an array.");
         }
+
+        if (dotctx != null) {
+            metadata.createExtNodeMetadata(parent, dotctx);
+            analyzer.visit(dotctx);
+        } else if (bracectx != null) {
+            metadata.createExtNodeMetadata(parent, bracectx);
+            analyzer.visit(bracectx);
+        }
     }
 
     void processExtdot(final ExtdotContext ctx) {
@@ -449,7 +449,7 @@ class AnalyzerExternal {
             }
 
             // special cases: reserved words
-            if ("_score".equals(id) || "doc".equals(id)) {
+            if (varenmd.last && ("_score".equals(id) || "doc".equals(id))) {
                 // read-only: don't allow stores
                 if (parentemd.storeExpr != null) {
                     throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Variable [" + id + "] is read-only.");
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java
index cd10a5ac3a6..f25780ab496 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java
@@ -306,18 +306,14 @@ class WriterExternal {
             final boolean cat = utility.containsStrings(parentemd.storeExpr);
 
             if (cat) {
-                if (field || name || shortcut) {
-                    execute.dupX1();
-                } else if (array) {
-                    execute.dup2X1();
+                if (maplist && constant != null) {
+                    utility.writeConstant(source, constant);
                 }
 
-                if (maplist) {
-                    if (constant != null) {
-                        utility.writeConstant(source, constant);
-                    }
-
-                    execute.dupX2();
+                if (field || name || (shortcut && !maplist)) {
+                    execute.dupX1();
+                } else if (array || maplist) {
+                    execute.dup2X1();
                 }
 
                 writeLoadStoreInstruction(source, false, variable, field, name, array, shortcut);
@@ -340,18 +336,14 @@ class WriterExternal {
             } else if (parentemd.token > 0) {
                 final int token = parentemd.token;
 
-                if (field || name || shortcut) {
-                    execute.dup();
-                } else if (array) {
-                    execute.dup2();
+                if (maplist && constant != null) {
+                    utility.writeConstant(source, constant);
                 }
 
-                if (maplist) {
-                    if (constant != null) {
-                        utility.writeConstant(source, constant);
-                    }
-
-                    execute.dupX1();
+                if (field || name || (shortcut && !maplist)) {
+                    execute.dup();
+                } else if (array || maplist) {
+                    execute.dup2();
                 }
 
                 writeLoadStoreInstruction(source, false, variable, field, name, array, shortcut);
@@ -467,10 +459,10 @@ class WriterExternal {
 
     private void writeLoadStoreField(final ParserRuleContext source, final boolean store, final String name) {
         if (store) {
-            execute.visitInvokeDynamicInsn(name, WriterConstants.DEF_DYNAMIC_STORE_FIELD_DESC, 
+            execute.visitInvokeDynamicInsn(name, WriterConstants.DEF_DYNAMIC_STORE_FIELD_DESC,
                                            WriterConstants.DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.STORE });
         } else {
-            execute.visitInvokeDynamicInsn(name, WriterConstants.DEF_DYNAMIC_LOAD_FIELD_DESC, 
+            execute.visitInvokeDynamicInsn(name, WriterConstants.DEF_DYNAMIC_LOAD_FIELD_DESC,
                                            WriterConstants.DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.LOAD });
         }
     }
@@ -482,10 +474,10 @@ class WriterExternal {
 
         if (type.sort == Sort.DEF) {
             if (store) {
-                execute.visitInvokeDynamicInsn("arrayStore", WriterConstants.DEF_DYNAMIC_ARRAY_STORE_DESC, 
+                execute.visitInvokeDynamicInsn("arrayStore", WriterConstants.DEF_DYNAMIC_ARRAY_STORE_DESC,
                                            WriterConstants.DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.ARRAY_STORE });
             } else {
-                execute.visitInvokeDynamicInsn("arrayLoad", WriterConstants.DEF_DYNAMIC_ARRAY_LOAD_DESC, 
+                execute.visitInvokeDynamicInsn("arrayLoad", WriterConstants.DEF_DYNAMIC_ARRAY_LOAD_DESC,
                                            WriterConstants.DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.ARRAY_LOAD });
             }
         } else {
@@ -705,11 +697,11 @@ class WriterExternal {
             writeDynamicCallExternal(source);
         }
     }
-    
+
     private void writeDynamicCallExternal(final ExtcallContext source) {
         final ExtNodeMetadata sourceenmd = metadata.getExtNodeMetadata(source);
         final List<ExpressionContext> arguments = source.arguments().expression();
-        
+
         StringBuilder signature = new StringBuilder();
         signature.append('(');
         // first parameter is the receiver, we never know its type: always Object
@@ -726,7 +718,7 @@ class WriterExternal {
         signature.append(')');
         // return value: currently always Object. making this better may be tricky...
         signature.append(WriterConstants.OBJECT_TYPE.getDescriptor());
-        execute.visitInvokeDynamicInsn((String)sourceenmd.target, signature.toString(), 
+        execute.visitInvokeDynamicInsn((String)sourceenmd.target, signature.toString(),
                                        WriterConstants.DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.METHOD_CALL });
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java
index e3aefda4766..2a0dfc17242 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java
@@ -42,22 +42,22 @@ public class NeedsScoreTests extends ESSingleNodeTestCase {
         SearchLookup lookup = new SearchLookup(index.mapperService(), index.fieldData(), null);
 
         Object compiled = service.compile("1.2", Collections.emptyMap());
-        SearchScript ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled), 
+        SearchScript ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled),
                                          lookup, Collections.<String, Object>emptyMap());
         assertFalse(ss.needsScores());
 
         compiled = service.compile("doc['d'].value", Collections.emptyMap());
-        ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled), 
+        ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled),
                             lookup, Collections.<String, Object>emptyMap());
         assertFalse(ss.needsScores());
 
         compiled = service.compile("1/_score", Collections.emptyMap());
-        ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled), 
+        ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled),
                             lookup, Collections.<String, Object>emptyMap());
         assertTrue(ss.needsScores());
 
         compiled = service.compile("doc['d'].value * _score", Collections.emptyMap());
-        ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled), 
+        ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled),
                             lookup, Collections.<String, Object>emptyMap());
         assertTrue(ss.needsScores());
     }

From 7689a1af28d9dd8127843b0b59beba062a6f9146 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Wed, 11 May 2016 02:29:35 -0400
Subject: [PATCH 0181/1311] Add 'ctx' keyword to painless.

---
 .../org/elasticsearch/painless/Analyzer.java  |  4 ++-
 .../painless/AnalyzerExternal.java            | 18 +++++++------
 .../org/elasticsearch/painless/Metadata.java  | 16 ++++++++++--
 .../org/elasticsearch/painless/Writer.java    |  9 +++++++
 .../painless/WriterExternal.java              |  4 ---
 .../painless/WriterStatement.java             |  4 ---
 .../painless/ReservedWordTests.java           | 26 ++++++++++++++++++-
 .../rest-api-spec/test/plan_a/15_update.yaml  |  4 +--
 .../test/plan_a/25_script_upsert.yaml         |  6 ++---
 9 files changed, 66 insertions(+), 25 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
index eb9ac2979c7..f374bccb54c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
@@ -101,7 +101,7 @@ class Analyzer extends PainlessParserBaseVisitor<Void> {
         // scorer parameter passed to the script. internal use only.
         metadata.scorerValueSlot = utility.addVariable(null, "#scorer", definition.objectType).slot;
         // doc parameter passed to the script.
-        // TODO: currently working as a def type, should be smapType...
+        // TODO: currently working as a Map<String,Def>, we can do better?
         metadata.docValueSlot = utility.addVariable(null, "doc", definition.smapType).slot;
         //
         // reserved words implemented as local variables
@@ -110,6 +110,8 @@ class Analyzer extends PainlessParserBaseVisitor<Void> {
         metadata.loopCounterSlot = utility.addVariable(null, "#loop", definition.intType).slot;
         // document's score as a read-only float.
         metadata.scoreValueSlot = utility.addVariable(null, "_score", definition.floatType).slot;
+        // ctx map set by executable scripts as a read-only map.
+        metadata.ctxValueSlot = utility.addVariable(null, "ctx", definition.smapType).slot;
 
         metadata.createStatementMetadata(metadata.root);
         visit(metadata.root);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java
index f907138f752..e8e8cb88fef 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java
@@ -449,16 +449,18 @@ class AnalyzerExternal {
             }
 
             // special cases: reserved words
-            if (varenmd.last && ("_score".equals(id) || "doc".equals(id))) {
-                // read-only: don't allow stores
-                if (parentemd.storeExpr != null) {
+            if ("_score".equals(id) || "doc".equals(id) || "ctx".equals(id)) {
+                // read-only: don't allow stores to ourself
+                if (varenmd.last && parentemd.storeExpr != null) {
                     throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Variable [" + id + "] is read-only.");
                 }
-            }
-
-            // track if the _score value is ever used, we will invoke Scorer.score() only once if so.
-            if ("_score".equals(id)) {
-                metadata.scoreValueUsed = true;
+                if ("_score".equals(id)) {
+                    // track if the _score value is ever used, we will invoke Scorer.score() only once if so.
+                    metadata.scoreValueUsed = true;
+                } else if ("ctx".equals(id)) {
+                    // track if ctx value is ever used, we will invoke Map.get() only once if so.
+                    metadata.ctxValueUsed = true;
+                }
             }
 
             varenmd.target = variable.slot;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java
index 85657360ef0..8692ea4a0a1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java
@@ -429,8 +429,8 @@ class Metadata {
     int scoreValueSlot = -1;
 
     /**
-     * Used to determine if the _score variable is actually used.  This is used in the {@link Analyzer} to update
-     * variable slots at the completion of analysis if _score is not used.
+     * Used to determine if the _score variable is actually used.  This is used to know if we should call
+     * Scorer.score() once and cache into a local variable, and expose NeedsScore interface (to allow query caching)
      */
     boolean scoreValueUsed = false;
     
@@ -439,6 +439,18 @@ class Metadata {
      * the doc variable is accessed.
      */
     int docValueSlot = -1;
+    
+    /**
+     * Used to determine what slot the ctx variable is stored in.  This is used in the {@link Writer} whenever
+     * the ctx variable is accessed.
+     */
+    int ctxValueSlot = -1;
+    
+    /**
+     * Used to determine if the ctx variable is actually used.  This is used to determine if we should call
+     * Map.get once and store into a local variable on startup.
+     */
+    boolean ctxValueUsed = false;
 
     /**
      * Maps the relevant ANTLR node to its metadata.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
index 6a57c40b817..20b609fa8c7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
@@ -157,6 +157,15 @@ class Writer extends PainlessParserBaseVisitor<Void> {
             execute.invokeVirtual(WriterConstants.SCORER_TYPE, WriterConstants.SCORER_SCORE);
             execute.visitVarInsn(Opcodes.FSTORE, metadata.scoreValueSlot);
         }
+        
+        if (metadata.ctxValueUsed) {
+            // if the _ctx value is used, we do this once:
+            //   Map<String,Object> ctx = input.get("ctx");
+            execute.visitVarInsn(Opcodes.ALOAD, metadata.inputValueSlot);
+            execute.push("ctx");
+            execute.invokeInterface(WriterConstants.MAP_TYPE, WriterConstants.MAP_GET);
+            execute.visitVarInsn(Opcodes.ASTORE, metadata.ctxValueSlot);
+        }
 
         execute.push(settings.getMaxLoopCounter());
         execute.visitVarInsn(Opcodes.ISTORE, metadata.loopCounterSlot);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java
index f25780ab496..2ce231412c1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java
@@ -422,10 +422,6 @@ class WriterExternal {
             throw new IllegalStateException(WriterUtility.error(source) + "Cannot load/store void type.");
         }
 
-        if (!metadata.scoreValueUsed && slot > metadata.scoreValueSlot) {
-            --slot;
-        }
-
         if (store) {
             execute.visitVarInsn(type.type.getOpcode(Opcodes.ISTORE), slot);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterStatement.java
index 4cd698a74cd..fc8b5ba71e4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterStatement.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterStatement.java
@@ -328,10 +328,6 @@ class WriterStatement {
         final Sort sort = declvaremd.to.sort;
         int slot = (int)declvaremd.postConst;
 
-        if (!metadata.scoreValueUsed && slot > metadata.scoreValueSlot) {
-            --slot;
-        }
-
         final ExpressionContext exprctx = ctx.expression();
         final boolean initialize = exprctx == null;
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java
index 64296812bda..46e08d60c2d 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java
@@ -19,6 +19,9 @@
 
 package org.elasticsearch.painless;
 
+import java.util.Collections;
+import java.util.HashMap;
+
 /** Tests for special reserved words such as _score */
 public class ReservedWordTests extends ScriptTestCase {
     
@@ -46,11 +49,32 @@ public class ReservedWordTests extends ScriptTestCase {
         assertTrue(expected.getMessage().contains("Variable name [doc] already defined"));
     }
     
-    /** check that we can't write to _score, its read-only! */
+    /** check that we can't write to doc, its read-only! */
     public void testDocStore() {
         IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
             exec("doc = 5; return doc;");
         });
         assertTrue(expected.getMessage().contains("Variable [doc] is read-only"));
     }
+    
+    /** check that we can't declare a variable of ctx, its really reserved! */
+    public void testCtxVar() {
+        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+            exec("int ctx = 5; return ctx;");
+        });
+        assertTrue(expected.getMessage().contains("Variable name [ctx] already defined"));
+    }
+    
+    /** check that we can't write to ctx, its read-only! */
+    public void testCtxStore() {
+        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+            exec("ctx = 5; return ctx;");
+        });
+        assertTrue(expected.getMessage().contains("Variable [ctx] is read-only"));
+    }
+    
+    /** check that we can modify its contents though */
+    public void testCtxStoreMap() {
+        assertEquals(5, exec("ctx.foo = 5; return ctx.foo;", Collections.singletonMap("ctx", new HashMap<String,Object>())));
+    }
 }
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/15_update.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/15_update.yaml
index 0cd8b52a2bb..39cee4e82a3 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/15_update.yaml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/15_update.yaml
@@ -18,7 +18,7 @@
           script: "1"
           body:
             lang:   painless
-            script: "input.ctx._source.foo = input.bar"
+            script: "ctx._source.foo = input.bar"
             params: { bar: 'xxx' }
 
   - match: { _index:   test_1 }
@@ -41,7 +41,7 @@
           type:   test
           id:     1
           lang:   painless
-          script: "input.ctx._source.foo = 'yyy'"
+          script: "ctx._source.foo = 'yyy'"
 
   - match: { _index:   test_1 }
   - match: { _type:    test   }
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/25_script_upsert.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/25_script_upsert.yaml
index 6a0fef06e37..d4a447e98a7 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/25_script_upsert.yaml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/25_script_upsert.yaml
@@ -7,7 +7,7 @@
           type:     test
           id:       1
           body:
-            script: "input.ctx._source.foo = input.bar"
+            script: "ctx._source.foo = input.bar"
             lang: "painless"
             params: { bar: 'xxx' }
             upsert: { foo: baz }
@@ -27,7 +27,7 @@
           type:     test
           id:       1
           body:
-            script: "input.ctx._source.foo = input.bar"
+            script: "ctx._source.foo = input.bar"
             lang: "painless"
             params: { bar: 'xxx' }
             upsert: { foo: baz }
@@ -46,7 +46,7 @@
           type:     test
           id:       2
           body:
-            script: "input.ctx._source.foo = input.bar"
+            script: "ctx._source.foo = input.bar"
             lang: "painless"
             params: { bar: 'xxx' }
             upsert: { foo: baz }

From 6d288dec11e05fcc9f63cd76b6abbe9ab3d65d15 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Tue, 10 May 2016 16:40:40 +0200
Subject: [PATCH 0182/1311] Clean up tests in Reindex module

---
 ... => BulkIndexByScrollResponseMatcher.java} | 62 +++++++++++--------
 .../index/reindex/ReindexBasicTests.java      | 12 ++--
 .../index/reindex/ReindexCancelTests.java     |  2 +-
 .../index/reindex/ReindexFailureTests.java    |  4 +-
 .../reindex/ReindexParentChildTests.java      |  8 +--
 .../index/reindex/ReindexTestCase.java        | 50 +--------------
 .../index/reindex/ReindexVersioningTests.java | 18 +++---
 .../reindex/UpdateByQueryBasicTests.java      | 10 ++-
 .../reindex/UpdateByQueryCancelTests.java     |  2 +-
 .../UpdateByQueryWhileModifyingTests.java     |  2 +-
 10 files changed, 66 insertions(+), 104 deletions(-)
 rename modules/reindex/src/test/java/org/elasticsearch/index/reindex/{AbstractBulkIndexByScrollResponseMatcher.java => BulkIndexByScrollResponseMatcher.java} (62%)

diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollResponseMatcher.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseMatcher.java
similarity index 62%
rename from modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollResponseMatcher.java
rename to modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseMatcher.java
index 0faf7d54549..b34dfc4a595 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollResponseMatcher.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseMatcher.java
@@ -26,10 +26,9 @@ import org.hamcrest.TypeSafeMatcher;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
 
-public abstract class AbstractBulkIndexByScrollResponseMatcher<
-                Response extends BulkIndexByScrollResponse,
-                Self extends AbstractBulkIndexByScrollResponseMatcher<Response, Self>>
-        extends TypeSafeMatcher<Response> {
+public class BulkIndexByScrollResponseMatcher extends TypeSafeMatcher<BulkIndexByScrollResponse> {
+
+    private Matcher<Long> createdMatcher = equalTo(0L);
     private Matcher<Long> updatedMatcher = equalTo(0L);
     /**
      * Matches for number of batches. Optional.
@@ -39,14 +38,21 @@ public abstract class AbstractBulkIndexByScrollResponseMatcher<
     private Matcher<Integer> failuresMatcher = equalTo(0);
     private Matcher<String> reasonCancelledMatcher = nullValue(String.class);
 
-    protected abstract Self self();
-
-    public Self updated(Matcher<Long> updatedMatcher) {
-        this.updatedMatcher = updatedMatcher;
-        return self();
+    public BulkIndexByScrollResponseMatcher created(Matcher<Long> createdMatcher) {
+        this.createdMatcher = createdMatcher;
+        return this;
     }
 
-    public Self updated(long updated) {
+    public BulkIndexByScrollResponseMatcher created(long created) {
+        return created(equalTo(created));
+    }
+
+    public BulkIndexByScrollResponseMatcher updated(Matcher<Long> updatedMatcher) {
+        this.updatedMatcher = updatedMatcher;
+        return this;
+    }
+
+    public BulkIndexByScrollResponseMatcher updated(long updated) {
         return updated(equalTo(updated));
     }
 
@@ -55,26 +61,26 @@ public abstract class AbstractBulkIndexByScrollResponseMatcher<
      * integer because we usually don't care about how many batches the job
      * takes.
      */
-    public Self batches(Matcher<Integer> batchesMatcher) {
+    public BulkIndexByScrollResponseMatcher batches(Matcher<Integer> batchesMatcher) {
         this.batchesMatcher = batchesMatcher;
-        return self();
+        return this;
     }
 
-    public Self batches(int batches) {
+    public BulkIndexByScrollResponseMatcher batches(int batches) {
         return batches(equalTo(batches));
     }
 
-    public Self batches(int total, int batchSize) {
+    public BulkIndexByScrollResponseMatcher batches(int total, int batchSize) {
         // Round up
         return batches((total + batchSize - 1) / batchSize);
     }
 
-    public Self versionConflicts(Matcher<Long> versionConflictsMatcher) {
+    public BulkIndexByScrollResponseMatcher versionConflicts(Matcher<Long> versionConflictsMatcher) {
         this.versionConflictsMatcher = versionConflictsMatcher;
-        return self();
+        return this;
     }
 
-    public Self versionConflicts(long versionConflicts) {
+    public BulkIndexByScrollResponseMatcher versionConflicts(long versionConflicts) {
         return versionConflicts(equalTo(versionConflicts));
     }
 
@@ -83,35 +89,37 @@ public abstract class AbstractBulkIndexByScrollResponseMatcher<
      * matching do it by hand. The type signatures required to match the
      * actual failures list here just don't work.
      */
-    public Self failures(Matcher<Integer> failuresMatcher) {
+    public BulkIndexByScrollResponseMatcher failures(Matcher<Integer> failuresMatcher) {
         this.failuresMatcher = failuresMatcher;
-        return self();
+        return this;
     }
 
     /**
      * Set the expected size of the failures list.
      */
-    public Self failures(int failures) {
+    public BulkIndexByScrollResponseMatcher failures(int failures) {
         return failures(equalTo(failures));
     }
 
-    public Self reasonCancelled(Matcher<String> reasonCancelledMatcher) {
+    public BulkIndexByScrollResponseMatcher reasonCancelled(Matcher<String> reasonCancelledMatcher) {
         this.reasonCancelledMatcher = reasonCancelledMatcher;
-        return self();
+        return this;
     }
 
     @Override
-    protected boolean matchesSafely(Response item) {
+    protected boolean matchesSafely(BulkIndexByScrollResponse item) {
         return updatedMatcher.matches(item.getUpdated()) &&
-            (batchesMatcher == null || batchesMatcher.matches(item.getBatches())) &&
-            versionConflictsMatcher.matches(item.getVersionConflicts()) &&
-            failuresMatcher.matches(item.getIndexingFailures().size()) &&
-            reasonCancelledMatcher.matches(item.getReasonCancelled());
+                createdMatcher.matches(item.getCreated()) &&
+                (batchesMatcher == null || batchesMatcher.matches(item.getBatches())) &&
+                versionConflictsMatcher.matches(item.getVersionConflicts()) &&
+                failuresMatcher.matches(item.getIndexingFailures().size()) &&
+                reasonCancelledMatcher.matches(item.getReasonCancelled());
     }
 
     @Override
     public void describeTo(Description description) {
         description.appendText("indexed matches ").appendDescriptionOf(updatedMatcher);
+        description.appendText(" and created matches ").appendDescriptionOf(createdMatcher);
         if (batchesMatcher != null) {
             description.appendText(" and batches matches ").appendDescriptionOf(batchesMatcher);
         }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java
index 1ff7d6d4492..5742a37f11e 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexBasicTests.java
@@ -37,22 +37,22 @@ public class ReindexBasicTests extends ReindexTestCase {
 
         // Copy all the docs
         ReindexRequestBuilder copy = reindex().source("source").destination("dest", "all").refresh(true);
-        assertThat(copy.get(), reindexResponseMatcher().created(4));
+        assertThat(copy.get(), matcher().created(4));
         assertHitCount(client().prepareSearch("dest").setTypes("all").setSize(0).get(), 4);
 
         // Now none of them
         copy = reindex().source("source").destination("all", "none").filter(termQuery("foo", "no_match")).refresh(true);
-        assertThat(copy.get(), reindexResponseMatcher().created(0));
+        assertThat(copy.get(), matcher().created(0));
         assertHitCount(client().prepareSearch("dest").setTypes("none").setSize(0).get(), 0);
 
         // Now half of them
         copy = reindex().source("source").destination("dest", "half").filter(termQuery("foo", "a")).refresh(true);
-        assertThat(copy.get(), reindexResponseMatcher().created(2));
+        assertThat(copy.get(), matcher().created(2));
         assertHitCount(client().prepareSearch("dest").setTypes("half").setSize(0).get(), 2);
 
         // Limit with size
         copy = reindex().source("source").destination("dest", "size_one").size(1).refresh(true);
-        assertThat(copy.get(), reindexResponseMatcher().created(1));
+        assertThat(copy.get(), matcher().created(1));
         assertHitCount(client().prepareSearch("dest").setTypes("size_one").setSize(0).get(), 1);
     }
 
@@ -70,7 +70,7 @@ public class ReindexBasicTests extends ReindexTestCase {
         ReindexRequestBuilder copy = reindex().source("source").destination("dest", "all").refresh(true);
         // Use a small batch size so we have to use more than one batch
         copy.source().setSize(5);
-        assertThat(copy.get(), reindexResponseMatcher().created(max).batches(max, 5));
+        assertThat(copy.get(), matcher().created(max).batches(max, 5));
         assertHitCount(client().prepareSearch("dest").setTypes("all").setSize(0).get(), max);
 
         // Copy some of the docs
@@ -79,7 +79,7 @@ public class ReindexBasicTests extends ReindexTestCase {
         // Use a small batch size so we have to use more than one batch
         copy.source().setSize(5);
         copy.size(half); // The real "size" of the request.
-        assertThat(copy.get(), reindexResponseMatcher().created(half).batches(half, 5));
+        assertThat(copy.get(), matcher().created(half).batches(half, 5));
         assertHitCount(client().prepareSearch("dest").setTypes("half").setSize(0).get(), half);
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexCancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexCancelTests.java
index 1412331736d..f9dde6045a6 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexCancelTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexCancelTests.java
@@ -35,7 +35,7 @@ public class ReindexCancelTests extends ReindexTestCase {
     public void testCancel() throws Exception {
         BulkIndexByScrollResponse response = CancelTestUtils.testCancel(this, reindex().destination("dest", "test"), ReindexAction.NAME);
 
-        assertThat(response, reindexResponseMatcher().created(1).reasonCancelled(equalTo("by user request")));
+        assertThat(response, matcher().created(1).reasonCancelled(equalTo("by user request")));
         refresh("dest");
         assertHitCount(client().prepareSearch("dest").setSize(0).get(), 1);
     }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java
index 74de297e43a..ed60b3e7ddf 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java
@@ -58,7 +58,7 @@ public class ReindexFailureTests extends ReindexTestCase {
         copy.source().setSize(1);
 
         BulkIndexByScrollResponse response = copy.get();
-        assertThat(response, reindexResponseMatcher()
+        assertThat(response, matcher()
                 .batches(1)
                 .failures(both(greaterThan(0)).and(lessThanOrEqualTo(maximumNumberOfShards()))));
         for (Failure failure: response.getIndexingFailures()) {
@@ -78,7 +78,7 @@ public class ReindexFailureTests extends ReindexTestCase {
         copy.destination().setOpType(CREATE);
 
         BulkIndexByScrollResponse response = copy.get();
-        assertThat(response, reindexResponseMatcher().batches(1).versionConflicts(1).failures(1).created(99));
+        assertThat(response, matcher().batches(1).versionConflicts(1).failures(1).created(99));
         for (Failure failure: response.getIndexingFailures()) {
             assertThat(failure.getMessage(), containsString("VersionConflictEngineException[[test]["));
         }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParentChildTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParentChildTests.java
index 45bacbcf13a..58770e10a1c 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParentChildTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexParentChildTests.java
@@ -43,18 +43,18 @@ public class ReindexParentChildTests extends ReindexTestCase {
 
         // Copy parent to the new index
         ReindexRequestBuilder copy = reindex().source("source").destination("dest").filter(findsCountry).refresh(true);
-        assertThat(copy.get(), reindexResponseMatcher().created(1));
+        assertThat(copy.get(), matcher().created(1));
 
         // Copy the child to a new index
         copy = reindex().source("source").destination("dest").filter(findsCity).refresh(true);
-        assertThat(copy.get(), reindexResponseMatcher().created(1));
+        assertThat(copy.get(), matcher().created(1));
 
         // Make sure parent/child is intact on that index
         assertSearchHits(client().prepareSearch("dest").setQuery(findsCity).get(), "pittsburgh");
 
         // Copy the grandchild to a new index
         copy = reindex().source("source").destination("dest").filter(findsNeighborhood).refresh(true);
-        assertThat(copy.get(), reindexResponseMatcher().created(1));
+        assertThat(copy.get(), matcher().created(1));
 
         // Make sure parent/child is intact on that index
         assertSearchHits(client().prepareSearch("dest").setQuery(findsNeighborhood).get(),
@@ -63,7 +63,7 @@ public class ReindexParentChildTests extends ReindexTestCase {
         // Copy the parent/child/grandchild structure all at once to a third index
         createParentChildIndex("dest_all_at_once");
         copy = reindex().source("source").destination("dest_all_at_once").refresh(true);
-        assertThat(copy.get(), reindexResponseMatcher().created(3));
+        assertThat(copy.get(), matcher().created(3));
 
         // Make sure parent/child/grandchild is intact there too
         assertSearchHits(client().prepareSearch("dest_all_at_once").setQuery(findsNeighborhood).get(),
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java
index 00956ac5982..33c72baa7cb 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java
@@ -22,16 +22,14 @@ package org.elasticsearch.index.reindex;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.hamcrest.Description;
-import org.hamcrest.Matcher;
 
 import java.util.Collection;
 
 import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
-import static org.hamcrest.Matchers.equalTo;
 
 @ClusterScope(scope = SUITE, transportClientRatio = 0)
 public abstract class ReindexTestCase extends ESIntegTestCase {
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return pluginList(ReindexPlugin.class);
@@ -41,57 +39,15 @@ public abstract class ReindexTestCase extends ESIntegTestCase {
         return ReindexAction.INSTANCE.newRequestBuilder(client());
     }
 
-    protected IndexBySearchResponseMatcher reindexResponseMatcher() {
-        return new IndexBySearchResponseMatcher();
-    }
-
     protected UpdateByQueryRequestBuilder updateByQuery() {
         return UpdateByQueryAction.INSTANCE.newRequestBuilder(client());
     }
 
-    protected BulkIndexbyScrollResponseMatcher updateByQueryResponseMatcher() {
-        return new BulkIndexbyScrollResponseMatcher();
-    }
-
     protected RethrottleRequestBuilder rethrottle() {
         return RethrottleAction.INSTANCE.newRequestBuilder(client());
     }
 
-    protected static class IndexBySearchResponseMatcher
-            extends AbstractBulkIndexByScrollResponseMatcher<BulkIndexByScrollResponse, IndexBySearchResponseMatcher> {
-        private Matcher<Long> createdMatcher = equalTo(0L);
-
-        public IndexBySearchResponseMatcher created(Matcher<Long> updatedMatcher) {
-            this.createdMatcher = updatedMatcher;
-            return this;
-        }
-
-        public IndexBySearchResponseMatcher created(long created) {
-            return created(equalTo(created));
-        }
-
-        @Override
-        protected boolean matchesSafely(BulkIndexByScrollResponse item) {
-            return super.matchesSafely(item) && createdMatcher.matches(item.getCreated());
-        }
-
-        @Override
-        public void describeTo(Description description) {
-            super.describeTo(description);
-            description.appendText(" and created matches ").appendDescriptionOf(createdMatcher);
-        }
-
-        @Override
-        protected IndexBySearchResponseMatcher self() {
-            return this;
-        }
-    }
-
-    protected static class BulkIndexbyScrollResponseMatcher
-            extends AbstractBulkIndexByScrollResponseMatcher<BulkIndexByScrollResponse, BulkIndexbyScrollResponseMatcher> {
-        @Override
-        protected BulkIndexbyScrollResponseMatcher self() {
-            return this;
-        }
+    protected static BulkIndexByScrollResponseMatcher matcher() {
+        return new BulkIndexByScrollResponseMatcher();
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexVersioningTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexVersioningTests.java
index 82c0e0af25f..2988fcb5ca6 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexVersioningTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexVersioningTests.java
@@ -33,55 +33,55 @@ public class ReindexVersioningTests extends ReindexTestCase {
 
     public void testExternalVersioningCreatesWhenAbsentAndSetsVersion() throws Exception {
         setupSourceAbsent();
-        assertThat(reindexExternal(), reindexResponseMatcher().created(1));
+        assertThat(reindexExternal(), matcher().created(1));
         assertDest("source", SOURCE_VERSION);
     }
 
     public void testExternalVersioningUpdatesOnOlderAndSetsVersion() throws Exception {
         setupDestOlder();
-        assertThat(reindexExternal(), reindexResponseMatcher().updated(1));
+        assertThat(reindexExternal(), matcher().updated(1));
         assertDest("source", SOURCE_VERSION);
     }
 
     public void testExternalVersioningVersionConflictsOnNewer() throws Exception {
         setupDestNewer();
-        assertThat(reindexExternal(), reindexResponseMatcher().versionConflicts(1));
+        assertThat(reindexExternal(), matcher().versionConflicts(1));
         assertDest("dest", NEWER_VERSION);
     }
 
     public void testInternalVersioningCreatesWhenAbsent() throws Exception {
         setupSourceAbsent();
-        assertThat(reindexInternal(), reindexResponseMatcher().created(1));
+        assertThat(reindexInternal(), matcher().created(1));
         assertDest("source", 1);
     }
 
     public void testInternalVersioningUpdatesOnOlder() throws Exception {
         setupDestOlder();
-        assertThat(reindexInternal(), reindexResponseMatcher().updated(1));
+        assertThat(reindexInternal(), matcher().updated(1));
         assertDest("source", OLDER_VERSION + 1);
     }
 
     public void testInternalVersioningUpdatesOnNewer() throws Exception {
         setupDestNewer();
-        assertThat(reindexInternal(), reindexResponseMatcher().updated(1));
+        assertThat(reindexInternal(), matcher().updated(1));
         assertDest("source", NEWER_VERSION + 1);
     }
 
     public void testCreateCreatesWhenAbsent() throws Exception {
         setupSourceAbsent();
-        assertThat(reindexCreate(), reindexResponseMatcher().created(1));
+        assertThat(reindexCreate(), matcher().created(1));
         assertDest("source", 1);
     }
 
     public void testCreateVersionConflictsOnOlder() throws Exception {
         setupDestOlder();
-        assertThat(reindexCreate(), reindexResponseMatcher().versionConflicts(1));
+        assertThat(reindexCreate(), matcher().versionConflicts(1));
         assertDest("dest", OLDER_VERSION);
     }
 
     public void testCreateVersionConflictsOnNewer() throws Exception {
         setupDestNewer();
-        assertThat(reindexCreate(), reindexResponseMatcher().versionConflicts(1));
+        assertThat(reindexCreate(), matcher().versionConflicts(1));
         assertDest("dest", NEWER_VERSION);
     }
 
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java
index 3d70d71a9bd..773e693a9bf 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryBasicTests.java
@@ -35,19 +35,17 @@ public class UpdateByQueryBasicTests extends ReindexTestCase {
         assertEquals(1, client().prepareGet("test", "test", "4").get().getVersion());
 
         // Reindex all the docs
-        assertThat(updateByQuery().source("test").refresh(true).get(), updateByQueryResponseMatcher().updated(4));
+        assertThat(updateByQuery().source("test").refresh(true).get(), matcher().updated(4));
         assertEquals(2, client().prepareGet("test", "test", "1").get().getVersion());
         assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion());
 
         // Now none of them
-        assertThat(updateByQuery().source("test").filter(termQuery("foo", "no_match")).refresh(true).get(),
-                updateByQueryResponseMatcher().updated(0));
+        assertThat(updateByQuery().source("test").filter(termQuery("foo", "no_match")).refresh(true).get(), matcher().updated(0));
         assertEquals(2, client().prepareGet("test", "test", "1").get().getVersion());
         assertEquals(2, client().prepareGet("test", "test", "4").get().getVersion());
 
         // Now half of them
-        assertThat(updateByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).get(),
-                updateByQueryResponseMatcher().updated(2));
+        assertThat(updateByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).get(), matcher().updated(2));
         assertEquals(3, client().prepareGet("test", "test", "1").get().getVersion());
         assertEquals(3, client().prepareGet("test", "test", "2").get().getVersion());
         assertEquals(2, client().prepareGet("test", "test", "3").get().getVersion());
@@ -56,7 +54,7 @@ public class UpdateByQueryBasicTests extends ReindexTestCase {
         // Limit with size
         UpdateByQueryRequestBuilder request = updateByQuery().source("test").size(3).refresh(true);
         request.source().addSort("foo.keyword", SortOrder.ASC);
-        assertThat(request.get(), updateByQueryResponseMatcher().updated(3));
+        assertThat(request.get(), matcher().updated(3));
         // Only the first three documents are updated because of sort
         assertEquals(4, client().prepareGet("test", "test", "1").get().getVersion());
         assertEquals(4, client().prepareGet("test", "test", "2").get().getVersion());
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryCancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryCancelTests.java
index dd63f90322e..4cb859c0017 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryCancelTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryCancelTests.java
@@ -36,7 +36,7 @@ public class UpdateByQueryCancelTests extends ReindexTestCase {
     public void testCancel() throws Exception {
         BulkIndexByScrollResponse response = CancelTestUtils.testCancel(this, updateByQuery(), UpdateByQueryAction.NAME);
 
-        assertThat(response, updateByQueryResponseMatcher().updated(1).reasonCancelled(equalTo("by user request")));
+        assertThat(response, matcher().updated(1).reasonCancelled(equalTo("by user request")));
         refresh("source");
         assertHitCount(client().prepareSearch("source").setSize(0).setQuery(matchQuery("giraffes", "giraffes")).get(), 1);
     }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java
index c36cbe974dc..7be073e32af 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java
@@ -49,7 +49,7 @@ public class UpdateByQueryWhileModifyingTests extends ReindexTestCase {
             while (keepUpdating.get()) {
                 try {
                     BulkIndexByScrollResponse response = updateByQuery().source("test").refresh(true).abortOnVersionConflict(false).get();
-                    assertThat(response, updateByQueryResponseMatcher().updated(either(equalTo(0L)).or(equalTo(1L)))
+                    assertThat(response, matcher().updated(either(equalTo(0L)).or(equalTo(1L)))
                             .versionConflicts(either(equalTo(0L)).or(equalTo(1L))));
                 } catch (Throwable t) {
                     failure.set(t);

From ae01a4f7f9f559b31ea955ced96621b8d5132215 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Wed, 11 May 2016 11:58:49 +0200
Subject: [PATCH 0183/1311] Increased logging level for
 testDelayedAllocationChangeWithSettingTo100ms

---
 .../org/elasticsearch/cluster/routing/DelayedAllocationIT.java  | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java
index 169e68f4637..2da75ee1115 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java
@@ -26,6 +26,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalTestCluster;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.util.Collections;
 import java.util.List;
@@ -107,6 +108,7 @@ public class DelayedAllocationIT extends ESIntegTestCase {
      * allocation to a very small value, it kicks the allocation of the unassigned shard
      * even though the node it was hosted on will not come back.
      */
+    @TestLogging("_root:DEBUG,cluster.routing:TRACE")
     public void testDelayedAllocationChangeWithSettingTo100ms() throws Exception {
         internalCluster().startNodesAsync(3).get();
         prepareCreate("test").setSettings(Settings.builder()

From 97a41ee9733a0311a4716d35a72016bbc904782d Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Wed, 11 May 2016 14:17:56 +0200
Subject: [PATCH 0184/1311] First pass at improving analyzer docs (#18269)

* Docs: First pass at improving analyzer docs

I've rewritten the intro to analyzers plus the docs
for all analyzers to provide working examples.

I've also removed:

* analyzer aliases (see #18244)
* analyzer versions (see #18267)
* snowball analyzer (see #8690)

Next steps will be tokenizers, token filters, char filters

* Fixed two typos
---
 docs/reference/analysis.asciidoc              | 155 ++++++++-----
 docs/reference/analysis/analyzers.asciidoc    | 105 ++++-----
 .../analysis/analyzers/configuring.asciidoc   |  66 ++++++
 .../analyzers/custom-analyzer.asciidoc        | 210 ++++++++++++++----
 .../analyzers/fingerprint-analyzer.asciidoc   | 130 ++++++++---
 .../analyzers/keyword-analyzer.asciidoc       |  39 +++-
 .../analysis/analyzers/lang-analyzer.asciidoc |  16 +-
 .../analyzers/pattern-analyzer.asciidoc       | 165 +++++++++-----
 .../analyzers/simple-analyzer.asciidoc        |  38 +++-
 .../analyzers/snowball-analyzer.asciidoc      |  64 ------
 .../analyzers/standard-analyzer.asciidoc      | 121 ++++++++--
 .../analysis/analyzers/stop-analyzer.asciidoc | 105 +++++++--
 .../analyzers/whitespace-analyzer.asciidoc    |  38 +++-
 docs/reference/analysis/anatomy.asciidoc      |  60 +++++
 docs/reference/analysis/testing.asciidoc      |  91 ++++++++
 .../params/position-increment-gap.asciidoc    |   4 +-
 16 files changed, 1045 insertions(+), 362 deletions(-)
 create mode 100644 docs/reference/analysis/analyzers/configuring.asciidoc
 delete mode 100644 docs/reference/analysis/analyzers/snowball-analyzer.asciidoc
 create mode 100644 docs/reference/analysis/anatomy.asciidoc
 create mode 100644 docs/reference/analysis/testing.asciidoc

diff --git a/docs/reference/analysis.asciidoc b/docs/reference/analysis.asciidoc
index 8461b5c010f..42b1ef65d24 100644
--- a/docs/reference/analysis.asciidoc
+++ b/docs/reference/analysis.asciidoc
@@ -3,68 +3,113 @@
 
 [partintro]
 --
-The index analysis module acts as a configurable registry of Analyzers
-that can be used in order to both break indexed (analyzed) fields when a
-document is indexed and process query strings. It maps to the Lucene
-`Analyzer`.
 
-
-Analyzers are composed of a single <<analysis-tokenizers,Tokenizer>> 
-and zero or more <<analysis-tokenfilters,TokenFilters>>. The tokenizer may 
-be preceded by one or more <<analysis-charfilters,CharFilters>>. The
-analysis module allows one to register `TokenFilters`, `Tokenizers` and
-`Analyzers` under logical names that can then be referenced either in
-mapping definitions or in certain APIs. The Analysis module
-automatically registers (*if not explicitly defined*) built in
-analyzers, token filters, and tokenizers.
-
-Here is a sample configuration:
-
-[source,js]
---------------------------------------------------
-index :
-    analysis :
-        analyzer : 
-            standard : 
-                type : standard
-                stopwords : [stop1, stop2]
-            myAnalyzer1 :
-                type : standard
-                stopwords : [stop1, stop2, stop3]
-                max_token_length : 500
-            # configure a custom analyzer which is 
-            # exactly like the default standard analyzer
-            myAnalyzer2 :
-                tokenizer : standard
-                filter : [standard, lowercase, stop]
-        tokenizer :
-            myTokenizer1 :
-                type : standard
-                max_token_length : 900
-            myTokenizer2 :
-                type : keyword
-                buffer_size : 512
-        filter :
-            myTokenFilter1 :
-                type : stop
-                stopwords : [stop1, stop2, stop3, stop4]
-            myTokenFilter2 :
-                type : length
-                min : 0
-                max : 2000
---------------------------------------------------
+_Analysis_ is the process of converting text, like the body of any email, into
+_tokens_ or _terms_ which are added to the inverted index for searching.
+Analysis is performed by an <<analysis-analyzers,_analyzer_>> which can be
+either a built-in analyzer or a <<analysis-custom-analyzer,`custom`>> analyzer
+defined per index.
 
 [float]
-[[backwards-compatibility]]
-=== Backwards compatibility
+== Index time analysis
 
-All analyzers, tokenizers, and token filters can be configured with a
-`version` parameter to control which Lucene version behavior they should
-use. Possible values are: `3.0` - `3.6`, `4.0` - `4.3` (the highest
-version number is the default option).
+For instance at index time, the built-in <<english-analyzer,`english`>> _analyzer_ would
+convert this sentence:
+
+[source,text]
+------
+"The QUICK brown foxes jumped over the lazy dog!"
+------
+
+into these terms, which would be added to the inverted index.
+
+[source,text]
+------
+[ quick, brown, fox, jump, over, lazi, dog ]
+------
+
+[float]
+=== Specifying an index time analyzer
+
+Each <<text,`text`>> field in a mapping can specify its own
+<<analyzer,`analyzer`>>:
+
+[source,js]
+-------------------------
+PUT my_index
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "title": {
+          "type":     "text",
+          "analyzer": "standard"
+        }
+      }
+    }
+  }
+}
+-------------------------
+// CONSOLE
+
+At index time, if no `analyzer` has been specified, it looks for an analyzer
+in the index settings called `default`.  Failing that, it defaults to using
+the <<analysis-standard-analyzer,`standard` analyzer>>.
+
+
+[float]
+== Search time analysis
+
+This same analysis process is applied to the query string at search time in
+<<full-text-queries,full text queries>> like the
+<<query-dsl-match-query,`match` query>>
+to convert the text in the query string into terms of the same form as those
+that are stored in the inverted index.
+
+For instance, a user might search for:
+
+[source,text]
+------
+"a quick fox"
+------
+
+which would be analysed by the same `english` analyzer into the following terms:
+
+[source,text]
+------
+[ quick, fox ]
+------
+
+Even though the exact words used in the query string don't appear in the
+original text (`quick` vs `QUICK`, `fox` vs `foxes`), because we have applied
+the same analyzer to both the text and the query string, the terms from the
+query string exactly match the terms from the text in the inverted index,
+which means that this query would match our example document.
+
+[float]
+=== Specifying a search time analyzer
+
+Usually the same analyzer should be used both at
+index time and at search time, and <<full-text-queries,full text queries>>
+like the  <<query-dsl-match-query,`match` query>> will use the mapping to look
+up the analyzer to use for each field.
+
+The analyzer to use to search a particular field is determined by
+looking for:
+
+* An `analyzer` specified in the query itself.
+* The <<search-analyzer,`search_analyzer`>> mapping parameter.
+* The <<analyzer,`analyzer`>> mapping parameter.
+* An analyzer in the index settings called `default_search`.
+* An analyzer in the index settings called `default`.
+* The `standard` analyzer.
 
 --
 
+include::analysis/anatomy.asciidoc[]
+
+include::analysis/testing.asciidoc[]
+
 include::analysis/analyzers.asciidoc[]
 
 include::analysis/tokenizers.asciidoc[]
diff --git a/docs/reference/analysis/analyzers.asciidoc b/docs/reference/analysis/analyzers.asciidoc
index e062a179b96..c10bca1da39 100644
--- a/docs/reference/analysis/analyzers.asciidoc
+++ b/docs/reference/analysis/analyzers.asciidoc
@@ -1,67 +1,60 @@
 [[analysis-analyzers]]
 == Analyzers
 
-Analyzers are composed of a single <<analysis-tokenizers,Tokenizer>> 
-and zero or more <<analysis-tokenfilters,TokenFilters>>. The tokenizer may 
-be preceded by one or more <<analysis-charfilters,CharFilters>>.
-The analysis module allows you to register `Analyzers` under logical
-names which can then be referenced either in mapping definitions or in
-certain APIs. 
+Elasticsearch ships with a wide range of built-in analyzers, which can be used
+in any index without further configuration:
 
-Elasticsearch comes with a number of prebuilt analyzers which are
-ready to use.  Alternatively, you can combine the built in
-character filters, tokenizers and token filters to create 
-<<analysis-custom-analyzer,custom analyzers>>.
+<<analysis-standard-analyzer,Standard Analyzer>>::
+
+The `standard` analyzer divides text into terms on word boundaries, as defined
+by the Unicode Text Segmentation algorithm. It removes most punctuation,
+lowercases terms, and supports removing stop words.
+
+<<analysis-simple-analyzer,Simple Analyzer>>::
+
+The `simple` analyzer divides text into terms whenever it encounters a
+character which is not a letter.  It lowercases all terms.
+
+<<analysis-whitespace-analyzer,Whitespace Analyzer>>::
+
+The `whitespace` analyzer divides text into terms whenever it encounters any
+whitespace character.  It does not lowercase terms.
+
+<<analysis-stop-analyzer,Stop Analyzer>>::
+
+The `stop` analyzer is like the `simple` analyzer, but also supports removal
+of stop words.
+
+<<analysis-keyword-analyzer,Keyword Analyzer>>::
+
+The `keyword` analyzer is a ``noop'' analyzer that accepts whatever text it is
+given and outputs the exact same text as a single term.
+
+<<analysis-pattern-analyzer,Pattern Analyzer>>::
+
+The `pattern` analyzer uses a regular expression to split the text into terms.
+It supports lower-casing and stop words.
+
+<<analysis-lang-analyzer,Language Analyzers>>::
+
+Elasticsearch provides many language-specific analyzers like `english` or
+`french`.
+
+<<analysis-fingerprint-analyzer,Fingerprint Analyzer>>::
+
+The `fingerprint` analyzer is a specialist analyzer which creates a
+fingerprint which can be used for duplicate detection.
 
 [float]
-[[default-analyzers]]
-=== Default Analyzers
+=== Custom analyzers
 
-An analyzer is registered under a logical name. It can then be
-referenced from mapping definitions or certain APIs. When none are
-defined, defaults are used. There is an option to define which analyzers
-will be used by default when none can be derived.
-
-The `default` logical name allows one to configure an analyzer that will
-be used both for indexing and for searching APIs. The `default_search`
-logical name can be used to configure a default analyzer that will be
-used just when searching (the `default` analyzer would still be used for
-indexing).
-
-For instance, the following settings could be used to perform exact matching
-only by default:
-
-[source,js]
---------------------------------------------------
-index :
-  analysis :
-    analyzer :
-      default :
-        tokenizer : keyword
---------------------------------------------------
-
-[float]
-[[aliasing-analyzers]]
-=== Aliasing Analyzers
-
-Analyzers can be aliased to have several registered lookup names
-associated with them. For example, the following will allow 
-the `standard` analyzer to also be referenced with `alias1`
-and `alias2` values.
+If you do not find an analyzer suitable for your needs, you can create a
+<<analysis-custom-analyzer,`custom`>> analyzer which combines the appropriate
+<<analysis-charfilters, character filters>>,
+<<analysis-tokenizers,tokenizer>>, and <<analysis-tokenfilters,token filters>>.
 
 
-[source,js]
---------------------------------------------------
-index :
-  analysis :
-    analyzer :
-      standard :
-        alias: [alias1, alias2]
-        type : standard
-        stopwords : [test1, test2, test3]
---------------------------------------------------
-
-Below is a list of the built in analyzers.
+include::analyzers/configuring.asciidoc[]
 
 include::analyzers/standard-analyzer.asciidoc[]
 
@@ -77,8 +70,6 @@ include::analyzers/pattern-analyzer.asciidoc[]
 
 include::analyzers/lang-analyzer.asciidoc[]
 
-include::analyzers/snowball-analyzer.asciidoc[]
-
 include::analyzers/fingerprint-analyzer.asciidoc[]
 
 include::analyzers/custom-analyzer.asciidoc[]
diff --git a/docs/reference/analysis/analyzers/configuring.asciidoc b/docs/reference/analysis/analyzers/configuring.asciidoc
new file mode 100644
index 00000000000..c93d800afb9
--- /dev/null
+++ b/docs/reference/analysis/analyzers/configuring.asciidoc
@@ -0,0 +1,66 @@
+[[configuring-analyzers]]
+=== Configuring built-in analyzers
+
+The built-in analyzers can be used directly without any configuration.  Some
+of them, however, support configuration options to alter their behaviour.  For
+instance, the <<analysis-standard-analyzer,`standard` analyzer>> can be configured
+to support a list of stop words:
+
+[source,js]
+--------------------------------
+PUT my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "std_english": { <1>
+          "type":      "standard",
+          "stopwords": "_english_"
+        }
+      }
+    }
+  },
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "my_text": {
+          "type":     "text",
+          "analyzer": "standard", <2>
+          "fields": {
+            "english": {
+              "type":     "text",
+              "analyzer": "std_english" <3>
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+GET _cluster/health?wait_for_status=yellow
+
+POST my_index/_analyze
+{
+  "field": "my_text", <2>
+  "text": "The old brown cow"
+}
+
+POST my_index/_analyze
+{
+  "field": "my_text.english", <3>
+  "text": "The old brown cow"
+}
+
+--------------------------------
+// CONSOLE
+
+<1> We define the `std_english` analyzer to be based on the `standard`
+    analyzer, but configured to remove the pre-defined list of English stopwords.
+<2> The `my_text` field uses the `standard` analyzer directly, without
+    any configuration.  No stop words will be removed from this field.
+    The resulting terms are: `[ the, old, brown, cow ]`
+<3> The `my_text.english` field uses the `std_english` analyzer, so
+    English stop words will be removed.  The resulting terms are:
+    `[ old, brown, cow ]`
+
diff --git a/docs/reference/analysis/analyzers/custom-analyzer.asciidoc b/docs/reference/analysis/analyzers/custom-analyzer.asciidoc
index 46111e32a17..eccd16c23be 100644
--- a/docs/reference/analysis/analyzers/custom-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/custom-analyzer.asciidoc
@@ -1,59 +1,179 @@
 [[analysis-custom-analyzer]]
 === Custom Analyzer
 
-An analyzer of type `custom` that allows to combine a `Tokenizer` with
-zero or more `Token Filters`, and zero or more `Char Filters`. The
-custom analyzer accepts a logical/registered name of the tokenizer to
-use, and a list of logical/registered names of token filters.
-The name of the custom analyzer must not start with "_".
+When the built-in analyzers do not fulfill your needs, you can create a
+`custom` analyzer which uses the appropriate combination of:
 
-The following are settings that can be set for a `custom` analyzer type:
+* zero or more <<analysis-charfilters, character filters>>
+* a <<analysis-tokenizers,tokenizer>>
+* zero or more <<analysis-tokenfilters,token filters>>.
 
-[cols="<,<",options="header",]
-|=======================================================================
-|Setting |Description
-|`tokenizer` |The logical / registered name of the tokenizer to use.
+[float]
+=== Configuration
 
-|`filter` |An optional list of logical / registered name of token
-filters.
+The `custom` analyzer accepts the following parameters:
 
-|`char_filter` |An optional list of logical / registered name of char
-filters.
+[horizontal]
+`tokenizer`::
+
+    A built-in or customised <<analysis-tokenizers,tokenizer>>.
+    (Required)
+
+`char_filter`::
+
+    An optional array of built-in or customised
+    <<analysis-charfilters, character filters>>.
+
+`filter`::
+
+    An optional array of built-in or customised
+    <<analysis-tokenfilters, token filters>>.
+
+`position_increment_gap`::
+
+    When indexing an array of text values, Elasticsearch inserts a fake "gap"
+    between the last term of one value and the first term of the next value to
+    ensure that a phrase query doesn't match two terms from different array
+    elements.  Defaults to `100`. See <<position-increment-gap>> for more.
+
+[float]
+=== Example configuration
+
+Here is an example that combines the following:
+
+Character Filter::
+* <<analysis-htmlstrip-charfilter,HTML Strip Character Filter>>
+
+Tokenizer::
+* <<analysis-standard-tokenizer,Standard Tokenizer>>
+
+Token Filters::
+* <<analysis-lowercase-tokenfilter,Lowercase Token Filter>>
+* <<analysis-asciifolding-tokenfilter,ASCII-Folding Token Filter>>
+
+[source,js]
+--------------------------------
+PUT my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_custom_analyzer": {
+          "type":      "custom",
+          "tokenizer": "standard",
+          "char_filter": [
+            "html_strip"
+          ],
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ]
+        }
+      }
+    }
+  }
+}
+
+GET _cluster/health?wait_for_status=yellow
+
+POST my_index/_analyze
+{
+  "analyzer": "my_custom_analyzer",
+  "text": "Is this <b>déjà vu</b>?"
+}
+--------------------------------
+// CONSOLE
+
+The above example produces the following terms:
+
+[source,text]
+---------------------------
+[ is, this, deja, vu ]
+---------------------------
+
+The previous example used tokenizer, token filters, and character filters with
+their default configurations, but it is possible to create configured versions
+of each and to use them in a custom analyzer.
+
+Here is a more complicated example that combines the following:
+
+Character Filter::
+* <<analysis-mapping-charfilter,Mapping Character Filter>>, configured to replace `:)` with `_happy_` and `:(` with `_sad_`
+
+Tokenizer::
+*  <<analysis-pattern-tokenizer,Pattern Tokenizer>>, configured to split on punctuation characters
+
+Token Filters::
+* <<analysis-lowercase-tokenfilter,Lowercase Token Filter>>
+* <<analysis-stop-tokenfilter,Stop Token Filter>>, configured to use the pre-defined list of English stop words
 
-|`position_increment_gap` |An optional number of positions to increment
-between each field value of a field using this analyzer. Defaults to 100.
-100 was chosen because it prevents phrase queries with reasonably large
-slops (less than 100) from matching terms across field values.
-|=======================================================================
 
 Here is an example:
 
 [source,js]
 --------------------------------------------------
-index :
-    analysis :
-        analyzer :
-            myAnalyzer2 :
-                type : custom
-                tokenizer : myTokenizer1
-                filter : [myTokenFilter1, myTokenFilter2]
-                char_filter : [my_html]
-                position_increment_gap: 256
-        tokenizer :
-            myTokenizer1 :
-                type : standard
-                max_token_length : 900
-        filter :
-            myTokenFilter1 :
-                type : stop
-                stopwords : [stop1, stop2, stop3, stop4]
-            myTokenFilter2 :
-                type : length
-                min : 0
-                max : 2000
-        char_filter :
-              my_html :
-                type : html_strip
-                escaped_tags : [xxx, yyy]
-                read_ahead : 1024
+PUT my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_custom_analyzer": {
+          "type": "custom",
+
+          "char_filter": [
+            "emoticons" <1>
+          ],
+
+          "tokenizer": "punctuation", <1>
+
+          "filter": [
+            "lowercase",
+            "english_stop" <1>
+          ]
+        }
+      },
+      "tokenizer": {
+        "punctuation": { <1>
+          "type": "pattern",
+          "pattern": "[ .,!?]"
+        }
+      },
+      "char_filter": {
+        "emoticons": { <1>
+          "type": "mapping",
+          "mappings": [
+            ":) => _happy_",
+            ":( => _sad_"
+          ]
+        }
+      },
+      "filter": {
+        "english_stop": { <1>
+          "type": "stop",
+          "stopwords": "_english_"
+        }
+      }
+    }
+  }
+}
+
+GET _cluster/health?wait_for_status=yellow
+
+POST my_index/_analyze
+{
+  "analyzer": "my_custom_analyzer",
+  "text":     "I'm a :) person, and you?"
+}
 --------------------------------------------------
+
+<1> The `emoticon` character filter, `punctuation` tokenizer and
+    `english_stop` token filter are custom implementations which are defined
+    in the same index settings.
+
+The above example produces the following terms:
+
+[source,text]
+---------------------------
+[ i'm, _happy_, person, you ]
+---------------------------
+
diff --git a/docs/reference/analysis/analyzers/fingerprint-analyzer.asciidoc b/docs/reference/analysis/analyzers/fingerprint-analyzer.asciidoc
index e0850077555..a66495acdbe 100644
--- a/docs/reference/analysis/analyzers/fingerprint-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/fingerprint-analyzer.asciidoc
@@ -5,37 +5,115 @@ The `fingerprint` analyzer implements a
 https://github.com/OpenRefine/OpenRefine/wiki/Clustering-In-Depth#fingerprint[fingerprinting algorithm]
 which is used by the OpenRefine project to assist in clustering.
 
-The `fingerprint` analyzer is composed of a <<analysis-standard-tokenizer>>, and four
-token filters (in this order): <<analysis-lowercase-tokenfilter>>, <<analysis-stop-tokenfilter>>,
-<<analysis-fingerprint-tokenfilter>> and <<analysis-asciifolding-tokenfilter>>.
+Input text is lowercased, normalized to remove extended characters, sorted,
+deduplicated and concatenated into a single token.  If a stopword list is
+configured, stop words will also be removed.
 
-Input text is lowercased, normalized to remove extended characters, sorted, deduplicated and
-concatenated into a single token.  If a stopword list is configured, stop words will
-also be removed. For example, the sentence:
+[float]
+=== Definition
 
-____
-"Yes yes, Gödel said this sentence is consistent and."
-____
+It consists of:
 
-will be transformed into the token: `"and consistent godel is said sentence this yes"`
+Tokenizer::
+* <<analysis-standard-tokenizer,Standard Tokenizer>>
+
+Token Filters::
+* <<analysis-lowercase-tokenfilter,Lower Case Token Filter>>
+* <<analysis-stop-tokenfilter,Stop Token Filter>> (disabled by default)
+* <<analysis-fingerprint-tokenfilter>>
+* <<analysis-asciifolding-tokenfilter>>
+
+[float]
+=== Example output
+
+[source,js]
+---------------------------
+POST _analyze
+{
+  "analyzer": "fingerprint",
+  "text": "Yes yes, Gödel said this sentence is consistent and."
+}
+---------------------------
+// CONSOLE
+
+The above sentence would produce the following single term:
+
+[source,text]
+---------------------------
+[ and consistent godel is said sentence this yes ]
+---------------------------
+
+[float]
+=== Configuration
+
+The `fingerprint` analyzer accepts the following parameters:
+
+[horizontal]
+`separator`::
+
+    The character to use to concate the terms.  Defaults to a space.
+
+`max_output_size`::
+
+    The maximum token size to emit.  Defaults to `255`. Tokens larger than
+    this size will be discarded.
+
+`preserve_original`::
+
+    If `true`, emits two tokens: one with ASCII-folding of terms that contain
+    extended characters (if any) and one with the original characters.
+    Defaults to `false`.
+
+`stopwords`::
+
+    A pre-defined stop words list like `_english_` or an array  containing a
+    list of stop words.  Defaults to `_none_`.
+
+`stopwords_path`::
+
+    The path to a file containing stop words.
+
+See the <<analysis-stop-tokenfilter,Stop Token Filter>> for more information
+about stop word configuration.
 
 
-Notice how the words are all lowercased, the umlaut in "gödel" has been normalized to "godel",
-punctuation has been removed, and "yes" has been de-duplicated.
+[float]
+=== Example configuration
 
-The `fingerprint` analyzer has these configurable settings
+In this example, we configure the `fingerprint` analyzer to use the
+pre-defined list of English stop words, and to emit a second token in
+the presence of non-ASCII characters:
 
-[cols="<,<",options="header",]
-|=======================================================================
-|Setting |Description
-|`separator` | The character that separates the tokens after concatenation.
-Defaults to a space.
-|`max_output_size` | The maximum token size to emit. Defaults to `255`. See <<analysis-fingerprint-tokenfilter-max-size>>
-|`preserve_original`| If true, emits both the original and folded version of
- tokens that contain extended characters.  Defaults to `false`
-|`stopwords` | A list of stop words to use. Defaults to an empty list (`_none_`).
-|`stopwords_path` | A path (either relative to `config` location, or absolute) to a stopwords
-                        file configuration. Each stop word should be in its own "line" (separated
-                        by a line break). The file must be UTF-8 encoded.
-|=======================================================================
+[source,js]
+----------------------------
+PUT my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_fingerprint_analyzer": {
+          "type": "fingerprint",
+          "stopwords": "_english_",
+          "preserve_original": true
+        }
+      }
+    }
+  }
+}
 
+GET _cluster/health?wait_for_status=yellow
+
+POST my_index/_analyze
+{
+  "analyzer": "my_fingerprint_analyzer",
+  "text": "Yes yes, Gödel said this sentence is consistent and."
+}
+----------------------------
+// CONSOLE
+
+The above example produces the following two terms:
+
+[source,text]
+---------------------------
+[ consistent godel said sentence yes, consistent gödel said sentence yes ]
+---------------------------
diff --git a/docs/reference/analysis/analyzers/keyword-analyzer.asciidoc b/docs/reference/analysis/analyzers/keyword-analyzer.asciidoc
index 815037596cf..a0c1b1b0a6a 100644
--- a/docs/reference/analysis/analyzers/keyword-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/keyword-analyzer.asciidoc
@@ -1,7 +1,38 @@
 [[analysis-keyword-analyzer]]
 === Keyword Analyzer
 
-An analyzer of type `keyword` that "tokenizes" an entire stream as a
-single token. This is useful for data like zip codes, ids and so on.
-Note, when using mapping definitions, it might make more sense to simply
-map the field as a <<keyword,`keyword`>>.
+The `keyword` analyzer is a ``noop'' analyzer which returns the entire input
+string as a single token.
+
+[float]
+=== Definition
+
+It consists of:
+
+Tokenizer::
+* <<analysis-keyword-tokenizer,Keyword Tokenizer>>
+
+[float]
+=== Example output
+
+[source,js]
+---------------------------
+POST _analyze
+{
+  "analyzer": "keyword",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+---------------------------
+// CONSOLE
+
+The above sentence would produce the following single term:
+
+[source,text]
+---------------------------
+[ The 2 QUICK Brown-Foxes jumped over the lazy dog's bone. ]
+---------------------------
+
+[float]
+=== Configuration
+
+The `keyword` analyzer is not configurable.
diff --git a/docs/reference/analysis/analyzers/lang-analyzer.asciidoc b/docs/reference/analysis/analyzers/lang-analyzer.asciidoc
index 1ac3a7af244..983a719e4f4 100644
--- a/docs/reference/analysis/analyzers/lang-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/lang-analyzer.asciidoc
@@ -303,8 +303,8 @@ The `catalan` analyzer could be reimplemented as a `custom` analyzer as follows:
     "analysis": {
       "filter": {
         "catalan_elision": {
-        "type":         "elision",
-            "articles": [ "d", "l", "m", "n", "s", "t"]
+          "type":       "elision",
+          "articles":   [ "d", "l", "m", "n", "s", "t"]
         },
         "catalan_stop": {
           "type":       "stop",
@@ -623,10 +623,10 @@ The `french` analyzer could be reimplemented as a `custom` analyzer as follows:
         "french_elision": {
           "type":         "elision",
           "articles_case": true,
-            "articles": [ 
+          "articles": [
               "l", "m", "t", "qu", "n", "s",
-              "j", "d", "c", "jusqu", "quoiqu", 
-              "lorsqu", "puisqu" 
+              "j", "d", "c", "jusqu", "quoiqu",
+              "lorsqu", "puisqu"
             ]
         },
         "french_stop": {
@@ -1000,13 +1000,13 @@ The `italian` analyzer could be reimplemented as a `custom` analyzer as follows:
     "analysis": {
       "filter": {
         "italian_elision": {
-        "type":         "elision",
-            "articles": [
+          "type": "elision",
+          "articles": [
                 "c", "l", "all", "dall", "dell",
                 "nell", "sull", "coll", "pell",
                 "gl", "agl", "dagl", "degl", "negl",
                 "sugl", "un", "m", "t", "s", "v", "d"
-            ]
+          ]
         },
         "italian_stop": {
           "type":       "stop",
diff --git a/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc b/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc
index 453967a46d7..6a4ca274416 100644
--- a/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc
@@ -1,46 +1,96 @@
 [[analysis-pattern-analyzer]]
 === Pattern Analyzer
 
-An analyzer of type `pattern` that can flexibly separate text into terms
-via a regular expression. Accepts the following settings:
+The `pattern` analyzer uses a regular expression to split the text into terms.
+The regular expression should match the *token separators*  not the tokens
+themselves. The regular expression defaults to `\W+` (or all non-word characters).
 
-The following are settings that can be set for a `pattern` analyzer
-type:
+[float]
+=== Definition
+
+It consists of:
+
+Tokenizer::
+* <<analysis-pattern-tokenizer,Pattern Tokenizer>>
+
+Token Filters::
+*  <<analysis-lowercase-tokenfilter,Lower Case Token Filter>>
+*  <<analysis-stop-tokenfilter,Stop Token Filter>> (disabled by default)
+
+[float]
+=== Example output
+
+[source,js]
+---------------------------
+POST _analyze
+{
+  "analyzer": "pattern",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+---------------------------
+// CONSOLE
+
+The above sentence would produce the following terms:
+
+[source,text]
+---------------------------
+[ the, 2, quick, brown, foxes, jumped, over, the, lazy, dog, s, bone ]
+---------------------------
+
+[float]
+=== Configuration
+
+The `pattern` analyzer accepts the following parameters:
 
 [horizontal]
-`lowercase`::   Should terms be lowercased or not. Defaults to `true`.
-`pattern`::     The regular expression pattern, defaults to `\W+`.
-`flags`::       The regular expression flags.
-`stopwords`::   A list of stopwords to initialize the stop filter with.
-                Defaults to an 'empty' stopword list Check
-                <<analysis-stop-analyzer,Stop Analyzer>> for more details.
+`pattern`::
 
-*IMPORTANT*: The regular expression should match the *token separators*,
-not the tokens themselves.
+    A http://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html[Java regular expression], defaults to `\W+`.
+
+`flags`::
+
+    Java regular expression http://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html#field.summary[flags].
+    lags should be pipe-separated, eg `"CASE_INSENSITIVE|COMMENTS"`.
+
+`lowercase`::
+
+    Should terms be lowercased or not. Defaults to `true`.
+
+`max_token_length`::
+
+    The maximum token length. If a token is seen that exceeds this length then
+    it is split at `max_token_length` intervals. Defaults to `255`.
+
+`stopwords`::
+
+    A pre-defined stop words list like `_english_` or an array  containing a
+    list of stop words.  Defaults to `_none_`.
+
+`stopwords_path`::
+
+    The path to a file containing stop words.
+
+See the <<analysis-stop-tokenfilter,Stop Token Filter>> for more information
+about stop word configuration.
 
-Flags should be pipe-separated, eg `"CASE_INSENSITIVE|COMMENTS"`. Check
-http://download.oracle.com/javase/6/docs/api/java/util/regex/Pattern.html#field_summary[Java
-Pattern API] for more details about `flags` options.
 
 [float]
-==== Pattern Analyzer Examples
+=== Example configuration
 
-In order to try out these examples, you should delete the `test` index
-before running each example.
-
-[float]
-===== Whitespace tokenizer
+In this example, we configure the `pattern` analyzer to split email addresses
+on non-word characters or on underscores (`\W|_`), and to lower-case the result:
 
 [source,js]
---------------------------------------------------
-PUT test
+----------------------------
+PUT my_index
 {
   "settings": {
     "analysis": {
       "analyzer": {
-        "whitespace": {
-          "type": "pattern",
-          "pattern": "\\s+"
+        "my_email_analyzer": {
+          "type":      "pattern",
+          "pattern":   "\\W|_", <1>
+          "lowercase": true
         }
       }
     }
@@ -49,47 +99,32 @@ PUT test
 
 GET _cluster/health?wait_for_status=yellow
 
-GET test/_analyze?analyzer=whitespace&text=foo,bar baz
-# "foo,bar", "baz"
---------------------------------------------------
-// CONSOLE
-
-[float]
-===== Non-word character tokenizer
-
-[source,js]
---------------------------------------------------
-PUT test
+POST my_index/_analyze
 {
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "nonword": {
-          "type": "pattern",
-          "pattern": "[^\\w]+" <1>
-        }
-      }
-    }
-  }
+  "analyzer": "my_email_analyzer",
+  "text": "John_Smith@foo-bar.com"
 }
-
-GET _cluster/health?wait_for_status=yellow
-
-GET test/_analyze?analyzer=nonword&text=foo,bar baz
-# "foo,bar baz" becomes "foo", "bar", "baz"
-
-GET test/_analyze?analyzer=nonword&text=type_1-type_4
-# "type_1","type_4"
---------------------------------------------------
+----------------------------
 // CONSOLE
 
+<1> The backslashes in the pattern need to be escaped when specifying the
+    pattern as a JSON string.
+
+The above example produces the following terms:
+
+[source,text]
+---------------------------
+[ john, smith, foo, bar, com ]
+---------------------------
 
 [float]
-===== CamelCase tokenizer
+==== CamelCase tokenizer
+
+The following more complicated example splits CamelCase text into tokens:
 
 [source,js]
 --------------------------------------------------
-PUT test?pretty=1
+PUT my_index
 {
   "settings": {
     "analysis": {
@@ -105,11 +140,21 @@ PUT test?pretty=1
 
 GET _cluster/health?wait_for_status=yellow
 
-GET test/_analyze?analyzer=camel&text=MooseX::FTPClass2_beta
-# "moose","x","ftp","class","2","beta"
+GET my_index/_analyze
+{
+  "analyzer": "camel",
+  "text": "MooseX::FTPClass2_beta"
+}
 --------------------------------------------------
 // CONSOLE
 
+The above example produces the following terms:
+
+[source,text]
+---------------------------
+[ moose, x, ftp, class, 2, beta ]
+---------------------------
+
 The regex above is easier to understand as:
 
 [source,js]
diff --git a/docs/reference/analysis/analyzers/simple-analyzer.asciidoc b/docs/reference/analysis/analyzers/simple-analyzer.asciidoc
index 9d7a7c30423..4c932bb5d3e 100644
--- a/docs/reference/analysis/analyzers/simple-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/simple-analyzer.asciidoc
@@ -1,6 +1,38 @@
 [[analysis-simple-analyzer]]
 === Simple Analyzer
 
-An analyzer of type `simple` that is built using a
-<<analysis-lowercase-tokenizer,Lower
-Case Tokenizer>>.
+The `simple` analyzer breaks text into terms whenever it encounters a
+character which is not a letter. All terms are lower cased.
+
+[float]
+=== Definition
+
+It consists of:
+
+Tokenizer::
+* <<analysis-lowercase-tokenizer,Lower Case Tokenizer>>
+
+[float]
+=== Example output
+
+[source,js]
+---------------------------
+POST _analyze
+{
+  "analyzer": "simple",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+---------------------------
+// CONSOLE
+
+The above sentence would produce the following terms:
+
+[source,text]
+---------------------------
+[ the, quick, brown, foxes, jumped, over, the, lazy, dog, s, bone ]
+---------------------------
+
+[float]
+=== Configuration
+
+The `simple` analyzer is not configurable.
diff --git a/docs/reference/analysis/analyzers/snowball-analyzer.asciidoc b/docs/reference/analysis/analyzers/snowball-analyzer.asciidoc
deleted file mode 100644
index 1f489bd0cf5..00000000000
--- a/docs/reference/analysis/analyzers/snowball-analyzer.asciidoc
+++ /dev/null
@@ -1,64 +0,0 @@
-[[analysis-snowball-analyzer]]
-=== Snowball Analyzer
-
-An analyzer of type `snowball` that uses the
-<<analysis-standard-tokenizer,standard
-tokenizer>>, with
-<<analysis-standard-tokenfilter,standard
-filter>>,
-<<analysis-lowercase-tokenfilter,lowercase
-filter>>,
-<<analysis-stop-tokenfilter,stop
-filter>>, and
-<<analysis-snowball-tokenfilter,snowball
-filter>>.
-
-The Snowball Analyzer is a stemming analyzer from Lucene that is
-originally based on the snowball project from
-http://snowballstem.org[snowballstem.org].
-
-Sample usage:
-
-[source,js]
---------------------------------------------------
-{
-    "index" : {
-        "analysis" : {
-            "analyzer" : {
-                "my_analyzer" : {
-                    "type" : "snowball",
-                    "language" : "English"
-                }
-            }
-        }
-    }
-}
---------------------------------------------------
-
-The `language` parameter can have the same values as the
-<<analysis-snowball-tokenfilter,snowball
-filter>> and defaults to `English`. Note that not all the language
-analyzers have a default set of stopwords provided.
-
-The `stopwords` parameter can be used to provide stopwords for the
-languages that have no defaults, or to simply replace the default set
-with your custom list. Check <<analysis-stop-analyzer,Stop Analyzer>>
-for more details. A default set of stopwords for many of these
-languages is available from for instance
-https://github.com/apache/lucene-solr/tree/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/[here]
-and
-https://github.com/apache/lucene-solr/tree/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball[here.]
-
-A sample configuration (in YAML format) specifying Swedish with
-stopwords:
-
-[source,js]
---------------------------------------------------
-index :
-    analysis :
-        analyzer :
-           my_analyzer:
-                type: snowball
-                language: Swedish
-                stopwords: "och,det,att,i,en,jag,hon,som,han,på,den,med,var,sig,för,så,till,är,men,ett,om,hade,de,av,icke,mig,du,henne,då,sin,nu,har,inte,hans,honom,skulle,hennes,där,min,man,ej,vid,kunde,något,från,ut,när,efter,upp,vi,dem,vara,vad,över,än,dig,kan,sina,här,ha,mot,alla,under,någon,allt,mycket,sedan,ju,denna,själv,detta,åt,utan,varit,hur,ingen,mitt,ni,bli,blev,oss,din,dessa,några,deras,blir,mina,samma,vilken,er,sådan,vår,blivit,dess,inom,mellan,sådant,varför,varje,vilka,ditt,vem,vilket,sitta,sådana,vart,dina,vars,vårt,våra,ert,era,vilkas"
---------------------------------------------------
diff --git a/docs/reference/analysis/analyzers/standard-analyzer.asciidoc b/docs/reference/analysis/analyzers/standard-analyzer.asciidoc
index 58fe89d947b..72292e1d40d 100644
--- a/docs/reference/analysis/analyzers/standard-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/standard-analyzer.asciidoc
@@ -1,26 +1,107 @@
 [[analysis-standard-analyzer]]
 === Standard Analyzer
 
-An analyzer of type `standard` is built using the
-<<analysis-standard-tokenizer,Standard
-Tokenizer>> with the
-<<analysis-standard-tokenfilter,Standard
-Token Filter>>,
-<<analysis-lowercase-tokenfilter,Lower
-Case Token Filter>>, and
-<<analysis-stop-tokenfilter,Stop
-Token Filter>>.
+The `standard` analyzer is the default analyzer which is used if none is
+specified. It provides grammar based tokenization (based on the Unicode Text
+Segmentation algorithm, as specified in
+http://unicode.org/reports/tr29/[Unicode Standard Annex #29]) and works well
+for most languages.
 
-The following are settings that can be set for a `standard` analyzer
-type:
+[float]
+=== Definition
 
-[cols="<,<",options="header",]
-|=======================================================================
-|Setting |Description
-|`stopwords` |A list of stopwords to initialize the stop filter with.
-Defaults to an 'empty' stopword list Check
-<<analysis-stop-analyzer,Stop Analyzer>> for more details.
-|`max_token_length` |The maximum token length. If a token is seen that exceeds
-this length then it is split at `max_token_length` intervals. Defaults to `255`.
-|=======================================================================
+It consists of:
+
+Tokenizer::
+* <<analysis-standard-tokenizer,Standard Tokenizer>>
+
+Token Filters::
+* <<analysis-standard-tokenfilter,Standard Token Filter>>
+* <<analysis-lowercase-tokenfilter,Lower Case Token Filter>>
+* <<analysis-stop-tokenfilter,Stop Token Filter>> (disabled by default)
+
+[float]
+=== Example output
+
+[source,js]
+---------------------------
+POST _analyze
+{
+  "analyzer": "standard",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+---------------------------
+// CONSOLE
+
+The above sentence would produce the following terms:
+
+[source,text]
+---------------------------
+[ the, 2, quick, brown, foxes, jumped, over, the, lazy, dog's, bone ]
+---------------------------
+
+[float]
+=== Configuration
+
+The `standard` analyzer accepts the following parameters:
+
+[horizontal]
+`max_token_length`::
+
+    The maximum token length. If a token is seen that exceeds this length then
+    it is split at `max_token_length` intervals. Defaults to `255`.
+
+`stopwords`::
+
+    A pre-defined stop words list like `_english_` or an array  containing a
+    list of stop words.  Defaults to `_none_`.
+
+`stopwords_path`::
+
+    The path to a file containing stop words.
+
+See the <<analysis-stop-tokenfilter,Stop Token Filter>> for more information
+about stop word configuration.
+
+
+[float]
+=== Example configuration
+
+In this example, we configure the `standard` analyzer to have a
+`max_token_length` of 5 (for demonstration purposes), and to use the
+pre-defined list of English stop words:
+
+[source,js]
+----------------------------
+PUT my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_english_analyzer": {
+          "type": "standard",
+          "max_token_length": 5,
+          "stopwords": "_english_"
+        }
+      }
+    }
+  }
+}
+
+GET _cluster/health?wait_for_status=yellow
+
+POST my_index/_analyze
+{
+  "analyzer": "my_english_analyzer",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+----------------------------
+// CONSOLE
+
+The above example produces the following terms:
+
+[source,text]
+---------------------------
+[ 2, quick, brown, foxes, jumpe, d, over, lazy, dog's, bone ]
+---------------------------
 
diff --git a/docs/reference/analysis/analyzers/stop-analyzer.asciidoc b/docs/reference/analysis/analyzers/stop-analyzer.asciidoc
index 9a19772795f..ada9022a287 100644
--- a/docs/reference/analysis/analyzers/stop-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/stop-analyzer.asciidoc
@@ -1,22 +1,97 @@
 [[analysis-stop-analyzer]]
 === Stop Analyzer
 
-An analyzer of type `stop` that is built using a
-<<analysis-lowercase-tokenizer,Lower
-Case Tokenizer>>, with
-<<analysis-stop-tokenfilter,Stop
-Token Filter>>.
+The `stop` analyzer is the same as the <<analysis-simple-analyzer,`simple` analyzer>>
+but adds support for removing stop words.  It defaults to using the
+`_english_` stop words.
 
-The following are settings that can be set for a `stop` analyzer type:
+[float]
+=== Definition
 
-[cols="<,<",options="header",]
-|=======================================================================
-|Setting |Description
-|`stopwords` |A list of stopwords to initialize the stop filter with.
-Defaults to the english stop words.
-|`stopwords_path` |A path (either relative to `config` location, or
-absolute) to a stopwords file configuration.
-|=======================================================================
+It consists of:
+
+Tokenizer::
+* <<analysis-lowercase-tokenizer,Lower Case Tokenizer>>
+
+Token filters::
+* <<analysis-stop-tokenfilter,Stop Token Filter>>
+
+[float]
+=== Example output
+
+[source,js]
+---------------------------
+POST _analyze
+{
+  "analyzer": "stop",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+---------------------------
+// CONSOLE
+
+The above sentence would produce the following terms:
+
+[source,text]
+---------------------------
+[ quick, brown, foxes, jumped, over, lazy, dog, s, bone ]
+---------------------------
+
+[float]
+=== Configuration
+
+The `stop` analyzer accepts the following parameters:
+
+[horizontal]
+`stopwords`::
+
+    A pre-defined stop words list like `_english_` or an array  containing a
+    list of stop words.  Defaults to `_english_`.
+
+`stopwords_path`::
+
+    The path to a file containing stop words.
+
+
+See the <<analysis-stop-tokenfilter,Stop Token Filter>> for more information
+about stop word configuration.
+
+[float]
+=== Example configuration
+
+In this example, we configure the `stop` analyzer to use a specified list of
+words as stop words:
+
+[source,js]
+----------------------------
+PUT my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_stop_analyzer": {
+          "type": "stop",
+          "stopwords": ["the", "over"]
+        }
+      }
+    }
+  }
+}
+
+GET _cluster/health?wait_for_status=yellow
+
+POST my_index/_analyze
+{
+  "analyzer": "my_stop_analyzer",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+----------------------------
+// CONSOLE
+
+The above example produces the following terms:
+
+[source,text]
+---------------------------
+[ quick, brown, foxes, jumped, lazy, dog, s, bone ]
+---------------------------
 
-Use `stopwords: _none_` to explicitly specify an 'empty' stopword list.
 
diff --git a/docs/reference/analysis/analyzers/whitespace-analyzer.asciidoc b/docs/reference/analysis/analyzers/whitespace-analyzer.asciidoc
index 20956867544..0dce8db1c99 100644
--- a/docs/reference/analysis/analyzers/whitespace-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/whitespace-analyzer.asciidoc
@@ -1,6 +1,38 @@
 [[analysis-whitespace-analyzer]]
 === Whitespace Analyzer
 
-An analyzer of type `whitespace` that is built using a
-<<analysis-whitespace-tokenizer,Whitespace
-Tokenizer>>.
+The `whitespace` analyzer breaks text into terms whenever it encounters a
+whitespace character.
+
+[float]
+=== Definition
+
+It consists of:
+
+Tokenizer::
+* <<analysis-whitespace-tokenizer,Whitespace Tokenizer>>
+
+[float]
+=== Example output
+
+[source,js]
+---------------------------
+POST _analyze
+{
+  "analyzer": "whitespace",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+---------------------------
+// CONSOLE
+
+The above sentence would produce the following terms:
+
+[source,text]
+---------------------------
+[ The, 2, QUICK, Brown-Foxes, jumped, over, the, lazy, dog's, bone. ]
+---------------------------
+
+[float]
+=== Configuration
+
+The `whitespace` analyzer is not configurable.
diff --git a/docs/reference/analysis/anatomy.asciidoc b/docs/reference/analysis/anatomy.asciidoc
new file mode 100644
index 00000000000..87739f80f60
--- /dev/null
+++ b/docs/reference/analysis/anatomy.asciidoc
@@ -0,0 +1,60 @@
+[[analyzer-anatomy]]
+== Anatomy of an analyzer
+
+An _analyzer_  -- whether built-in or custom -- is just a package which
+contains three lower-level building blocks: _character filters_,
+_tokenizers_, and _token filters_.
+
+The built-in <<analysis-analyzers,analyzers>> pre-package these building
+blocks into analyzers suitable for different languages and types of text.
+Elasticsearch also exposes the individual building blocks so that they can be
+combined to define new <<analysis-custom-analyzer,`custom`>> analyzers.
+
+[float]
+=== Character filters
+
+A _character filter_ receives the original text as a stream of characters and
+can transform the stream by adding, removing, or changing characters.  For
+instance, a character filter could be used to convert Arabic numerals
+(٠‎١٢٣٤٥٦٧٨‎٩‎) into their Latin equivalents (0123456789), or to strip HTML
+elements like `<b>` from the stream.
+
+An analyzer may have *zero or more* <<analysis-charfilters,character filters>>,
+which are applied in order.
+
+[float]
+=== Tokenizer
+
+A _tokenizer_  receives a stream of characters, breaks it up into individual
+_tokens_ (usually individual words), and outputs a stream of _tokens_. For
+instance, a <<analysis-whitespace-tokenizer,`whitespace`>> tokenizer breaks
+text into tokens whenever it sees any whitespace.  It would convert the text
+`"Quick brown fox!"` into the terms `[Quick, brown, fox!]`.
+
+The tokenizer is also responsible for recording the order or _position_ of
+each term and the start and end _character offsets_ of the original word which
+the term represents.
+
+An analyzer must have *exactly one* <<analysis-tokenizers,tokenizer>>.
+
+
+[float]
+=== Token filters
+
+A _token filter_ receives the token stream and may add, remove, or change
+tokens.  For example, a <<analysis-lowercase-tokenfilter,`lowercase`>> token
+filter converts all tokens to lowercase, a
+<<analysis-stop-tokenfilter,`stop`>> token filter removes common words
+(_stop words_) like `the` from the token stream, and a
+<<analysis-synonym-tokenfilter,`synonym`>> token filter introduces synonyms
+into the token stream.
+
+Token filters are not allowed to change the position or character offsets of
+each token.
+
+An analyzer may have *zero or more* <<analysis-tokenfilters,token filters>>,
+which are applied in order.
+
+
+
+
diff --git a/docs/reference/analysis/testing.asciidoc b/docs/reference/analysis/testing.asciidoc
new file mode 100644
index 00000000000..a8c735f5027
--- /dev/null
+++ b/docs/reference/analysis/testing.asciidoc
@@ -0,0 +1,91 @@
+== Testing analyzers
+
+The <<indices-analyze,`analyze` API>> is an invaluable tool for viewing the
+terms produced by an analyzer. A built-in analyzer (or combination of built-in
+tokenizer, token filters, and character filters) can be specified inline in
+the request:
+
+[source,js]
+-------------------------------------
+POST _analyze
+{
+  "analyzer": "whitespace",
+  "text":     "The quick brown fox."
+}
+
+POST _analyze
+{
+  "tokenizer": "standard",
+  "filter":  [ "lowercase", "asciifolding" ],
+  "text":      "Is this déja vu?"
+}
+-------------------------------------
+// CONSOLE
+
+
+
+.Positions and character offsets
+*********************************************************
+
+As can be seen from the output of the `analyze` API, analyzers not only
+convert words into terms, they also record the order or relative _positions_
+of each term (used for phrase queries or word proximity queries), and the
+start and end _character offsets_ of each term in the original text (used for
+highlighting search snippets).
+
+*********************************************************
+
+
+Alternatively, a <<analysis-custom-analyzer,`custom` analyzer>> can be
+referred to when running the `analyze` API on a specific index:
+
+[source,js]
+-------------------------------------
+PUT my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "std_folded": { <1>
+          "type": "custom",
+          "tokenizer": "standard",
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ]
+        }
+      }
+    }
+  },
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "my_text": {
+          "type": "text",
+          "analyzer": "std_folded" <2>
+        }
+      }
+    }
+  }
+}
+
+GET my_index/_analyze <3>
+{
+  "analyzer": "std_folded", <4>
+  "text":     "Is this déjà vu?"
+}
+
+GET my_index/_analyze <3>
+{
+  "field": "my_text", <5>
+  "text":  "Is this déjà vu?"
+}
+-------------------------------------
+// CONSOLE
+
+<1> Define a `custom` analyzer called `std_folded`.
+<2> The field `my_text` uses the `std_folded` analyzer.
+<3> To refer to this analyzer, the `analyze` API must specify the index name.
+<4> Refer to the analyzer by name.
+<5> Refer to the analyzer used by field `my_text`.
+
diff --git a/docs/reference/mapping/params/position-increment-gap.asciidoc b/docs/reference/mapping/params/position-increment-gap.asciidoc
index ea73e730759..d9d37e2108f 100644
--- a/docs/reference/mapping/params/position-increment-gap.asciidoc
+++ b/docs/reference/mapping/params/position-increment-gap.asciidoc
@@ -1,10 +1,10 @@
 [[position-increment-gap]]
 === `position_increment_gap`
 
-<<mapping-index,Analyzed>> string fields take term <<index-options,positions>>
+<<mapping-index,Analyzed>> text fields take term <<index-options,positions>>
 into account, in order to be able to support
 <<query-dsl-match-query-phrase,proximity or phrase queries>>.
-When indexing string fields with multiple values a "fake" gap is added between
+When indexing text fields with multiple values a "fake" gap is added between
 the values to prevent most phrase queries from matching across the values. The
 size of this gap is configured using `position_increment_gap` and defaults to
 `100`.

From 85f1ab44d947253911c0c36e41bf48ff2da88659 Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Wed, 11 May 2016 14:37:19 +0200
Subject: [PATCH 0185/1311] Convert rest of query-dsl docs to be run in tests

---
 .../query-dsl/boosting-query.asciidoc         |  28 ++-
 .../query-dsl/common-terms-query.asciidoc     | 226 ++++++++++--------
 .../query-dsl/dis-max-query.asciidoc          |  26 +-
 .../reference/query-dsl/exists-query.asciidoc |  20 +-
 docs/reference/query-dsl/fuzzy-query.asciidoc |  49 ++--
 .../query-dsl/geo-bounding-box-query.asciidoc | 222 ++++++++++-------
 .../geo-distance-range-query.asciidoc         |  28 ++-
 .../query-dsl/geo-polygon-query.asciidoc      | 105 ++++----
 .../query-dsl/geo-shape-query.asciidoc        |  40 ++--
 .../query-dsl/geohash-cell-query.asciidoc     |  33 ++-
 .../query-dsl/has-child-query.asciidoc        |  56 +++--
 .../query-dsl/has-parent-query.asciidoc       |  34 ++-
 docs/reference/query-dsl/ids-query.asciidoc   |  10 +-
 .../query-dsl/indices-query.asciidoc          |  18 +-
 .../match-phrase-prefix-query.asciidoc        |  22 +-
 .../query-dsl/match-phrase-query.asciidoc     |  20 +-
 docs/reference/query-dsl/match-query.asciidoc |  47 ++--
 .../reference/query-dsl/nested-query.asciidoc |  33 ++-
 .../query-dsl/parent-id-query.asciidoc        |  19 +-
 .../query-dsl/percolate-query.asciidoc        |  82 ++++---
 .../query-dsl/query-string-query.asciidoc     |  64 +++--
 .../reference/query-dsl/regexp-query.asciidoc |  47 ++--
 .../reference/query-dsl/script-query.asciidoc |  40 ++--
 docs/reference/query-dsl/terms-query.asciidoc |  44 ++--
 .../query-dsl/wildcard-query.asciidoc         |  18 +-
 25 files changed, 814 insertions(+), 517 deletions(-)

diff --git a/docs/reference/query-dsl/boosting-query.asciidoc b/docs/reference/query-dsl/boosting-query.asciidoc
index 969b3bbedfe..fdeace68a49 100644
--- a/docs/reference/query-dsl/boosting-query.asciidoc
+++ b/docs/reference/query-dsl/boosting-query.asciidoc
@@ -8,19 +8,23 @@ overall score.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "boosting" : {
-        "positive" : {
-            "term" : {
-                "field1" : "value1"
-            }
-        },
-        "negative" : {
-            "term" : {
-                "field2" : "value2"
-            }
-        },
-        "negative_boost" : 0.2
+    "query": {
+        "boosting" : {
+            "positive" : {
+                "term" : {
+                    "field1" : "value1"
+                }
+            },
+                "negative" : {
+                    "term" : {
+                        "field2" : "value2"
+                    }
+                },
+                "negative_boost" : 0.2
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/query-dsl/common-terms-query.asciidoc b/docs/reference/query-dsl/common-terms-query.asciidoc
index a956c33c1ee..9c1dde02831 100644
--- a/docs/reference/query-dsl/common-terms-query.asciidoc
+++ b/docs/reference/query-dsl/common-terms-query.asciidoc
@@ -70,15 +70,19 @@ In this example, words that have a document frequency greater than 0.1%
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "common": {
-    "body": {
-      "query": "this is bonsai cool",
-      "cutoff_frequency": 0.001
+    "query": {
+        "common": {
+            "body": {
+                "query": "this is bonsai cool",
+                    "cutoff_frequency": 0.001
+            }
+        }
     }
-  }
 }
 --------------------------------------------------
+// CONSOLE
 
 The number of terms which should match can be controlled with the
 <<query-dsl-minimum-should-match,`minimum_should_match`>>
@@ -90,36 +94,44 @@ all terms required:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "common": {
-    "body": {
-      "query": "nelly the elephant as a cartoon",
-      "cutoff_frequency": 0.001,
-      "low_freq_operator": "and"
+    "query": {
+        "common": {
+            "body": {
+                "query": "nelly the elephant as a cartoon",
+                    "cutoff_frequency": 0.001,
+                    "low_freq_operator": "and"
+            }
+        }
     }
-  }
 }
 --------------------------------------------------
+// CONSOLE
 
 which is roughly equivalent to:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "bool": {
-    "must": [
-      { "term": { "body": "nelly"}},
-      { "term": { "body": "elephant"}},
-      { "term": { "body": "cartoon"}}
-    ],
-    "should": [
-      { "term": { "body": "the"}}
-      { "term": { "body": "as"}}
-      { "term": { "body": "a"}}
-    ]
-  }
+    "query": {
+        "bool": {
+            "must": [
+            { "term": { "body": "nelly"}},
+            { "term": { "body": "elephant"}},
+            { "term": { "body": "cartoon"}}
+            ],
+            "should": [
+            { "term": { "body": "the"}},
+            { "term": { "body": "as"}},
+            { "term": { "body": "a"}}
+            ]
+        }
+    }
 }
 --------------------------------------------------
+// CONSOLE
 
 Alternatively use
 <<query-dsl-minimum-should-match,`minimum_should_match`>>
@@ -128,41 +140,49 @@ must be present, for instance:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "common": {
-    "body": {
-      "query": "nelly the elephant as a cartoon",
-      "cutoff_frequency": 0.001,
-      "minimum_should_match": 2
+    "query": {
+        "common": {
+            "body": {
+                "query": "nelly the elephant as a cartoon",
+                "cutoff_frequency": 0.001,
+                "minimum_should_match": 2
+            }
+        }
     }
-  }
 }
 --------------------------------------------------
+// CONSOLE
 
 which is roughly equivalent to:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "bool": {
-    "must": {
-      "bool": {
-        "should": [
-          { "term": { "body": "nelly"}},
-          { "term": { "body": "elephant"}},
-          { "term": { "body": "cartoon"}}
-        ],
-        "minimum_should_match": 2
-      }
-    },
-    "should": [
-      { "term": { "body": "the"}}
-      { "term": { "body": "as"}}
-      { "term": { "body": "a"}}
-    ]
-  }
+    "query": {
+        "bool": {
+            "must": {
+                "bool": {
+                    "should": [
+                    { "term": { "body": "nelly"}},
+                    { "term": { "body": "elephant"}},
+                    { "term": { "body": "cartoon"}}
+                    ],
+                    "minimum_should_match": 2
+                }
+            },
+                "should": [
+                    { "term": { "body": "the"}},
+                    { "term": { "body": "as"}},
+                    { "term": { "body": "a"}}
+                    ]
+        }
+    }
 }
 --------------------------------------------------
+// CONSOLE
 
 minimum_should_match
 
@@ -174,50 +194,58 @@ additional parameters (note the change in structure):
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "common": {
-    "body": {
-      "query": "nelly the elephant not as a cartoon",
-      "cutoff_frequency": 0.001,
-      "minimum_should_match": {
-          "low_freq" : 2,
-          "high_freq" : 3
-       }
+    "query": {
+        "common": {
+            "body": {
+                "query": "nelly the elephant not as a cartoon",
+                    "cutoff_frequency": 0.001,
+                    "minimum_should_match": {
+                        "low_freq" : 2,
+                        "high_freq" : 3
+                    }
+            }
+        }
     }
-  }
 }
 --------------------------------------------------
+// CONSOLE
 
 which is roughly equivalent to:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "bool": {
-    "must": {
-      "bool": {
-        "should": [
-          { "term": { "body": "nelly"}},
-          { "term": { "body": "elephant"}},
-          { "term": { "body": "cartoon"}}
-        ],
-        "minimum_should_match": 2
-      }
-    },
-    "should": {
-      "bool": {
-        "should": [
-          { "term": { "body": "the"}},
-          { "term": { "body": "not"}},
-          { "term": { "body": "as"}},
-          { "term": { "body": "a"}}
-        ],
-        "minimum_should_match": 3
-      }
+    "query": {
+        "bool": {
+            "must": {
+                "bool": {
+                    "should": [
+                    { "term": { "body": "nelly"}},
+                    { "term": { "body": "elephant"}},
+                    { "term": { "body": "cartoon"}}
+                    ],
+                    "minimum_should_match": 2
+                }
+            },
+                "should": {
+                    "bool": {
+                        "should": [
+                        { "term": { "body": "the"}},
+                        { "term": { "body": "not"}},
+                        { "term": { "body": "as"}},
+                        { "term": { "body": "a"}}
+                        ],
+                        "minimum_should_match": 3
+                    }
+                }
+        }
     }
-  }
 }
 --------------------------------------------------
+// CONSOLE
 
 In this case it means the high frequency terms have only an impact on
 relevance when there are at least three of them. But the most
@@ -227,36 +255,44 @@ for high frequency terms is when there are only high frequency terms:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "common": {
-    "body": {
-      "query": "how not to be",
-      "cutoff_frequency": 0.001,
-      "minimum_should_match": {
-          "low_freq" : 2,
-          "high_freq" : 3
-       }
+    "query": {
+        "common": {
+            "body": {
+                "query": "how not to be",
+                    "cutoff_frequency": 0.001,
+                    "minimum_should_match": {
+                        "low_freq" : 2,
+                        "high_freq" : 3
+                    }
+            }
+        }
     }
-  }
 }
 --------------------------------------------------
+// CONSOLE
 
 which is roughly equivalent to:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "bool": {
-    "should": [
-      { "term": { "body": "how"}},
-      { "term": { "body": "not"}},
-      { "term": { "body": "to"}},
-      { "term": { "body": "be"}}
-    ],
-    "minimum_should_match": "3<50%"
-  }
+    "query": {
+        "bool": {
+            "should": [
+            { "term": { "body": "how"}},
+            { "term": { "body": "not"}},
+            { "term": { "body": "to"}},
+            { "term": { "body": "be"}}
+            ],
+            "minimum_should_match": "3<50%"
+        }
+    }
 }
 --------------------------------------------------
+// CONSOLE
 
 The high frequency generated query is then slightly less restrictive
 than with an `AND`.
diff --git a/docs/reference/query-dsl/dis-max-query.asciidoc b/docs/reference/query-dsl/dis-max-query.asciidoc
index 2938c8db8ea..2f82f2294b3 100644
--- a/docs/reference/query-dsl/dis-max-query.asciidoc
+++ b/docs/reference/query-dsl/dis-max-query.asciidoc
@@ -27,18 +27,22 @@ This query maps to Lucene `DisjunctionMaxQuery`.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "dis_max" : {
-        "tie_breaker" : 0.7,
-        "boost" : 1.2,
-        "queries" : [
-            {
-                "term" : { "age" : 34 }
-            },
-            {
-                "term" : { "age" : 35 }
-            }
-        ]
+    "query": {
+        "dis_max" : {
+            "tie_breaker" : 0.7,
+                "boost" : 1.2,
+                "queries" : [
+                {
+                    "term" : { "age" : 34 }
+                },
+                {
+                    "term" : { "age" : 35 }
+                }
+                ]
+        }
     }
 }    
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/query-dsl/exists-query.asciidoc b/docs/reference/query-dsl/exists-query.asciidoc
index b484d47f4b6..4971219366f 100644
--- a/docs/reference/query-dsl/exists-query.asciidoc
+++ b/docs/reference/query-dsl/exists-query.asciidoc
@@ -5,10 +5,14 @@ Returns documents that have at least one non-`null` value in the original field:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "exists" : { "field" : "user" }
+    "query": {
+        "exists" : { "field" : "user" }
+    }
 }
 --------------------------------------------------
+// CONSOLE
 
 For instance, these documents would all match the above query:
 
@@ -77,14 +81,20 @@ clause as follows:
 
 [source,js]
 --------------------------------------------------
-"bool": {
-    "must_not": {
-        "exists": {
-            "field": "user"
+GET /_search
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "exists": {
+                    "field": "user"
+                }
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 This query returns documents that have no value in the user field.
 
diff --git a/docs/reference/query-dsl/fuzzy-query.asciidoc b/docs/reference/query-dsl/fuzzy-query.asciidoc
index c647bab2cbb..e950b43aa97 100644
--- a/docs/reference/query-dsl/fuzzy-query.asciidoc
+++ b/docs/reference/query-dsl/fuzzy-query.asciidoc
@@ -17,27 +17,35 @@ Here is a simple example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "fuzzy" : { "user" : "ki" }
+    "query": {
+       "fuzzy" : { "user" : "ki" }
+    }
 }
 --------------------------------------------------
+// CONSOLE
 
 Or with more advanced settings:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "fuzzy" : {
-        "user" : {
-            "value" :         "ki",
-            "boost" :         1.0,
-            "fuzziness" :     2,
-            "prefix_length" : 0,
-            "max_expansions": 100
+    "query": {
+        "fuzzy" : {
+            "user" : {
+                "value" :         "ki",
+                    "boost" :         1.0,
+                    "fuzziness" :     2,
+                    "prefix_length" : 0,
+                    "max_expansions": 100
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ===== Parameters
@@ -75,29 +83,36 @@ For example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "fuzzy" : {
-        "price" : {
-            "value" : 12,
-            "fuzziness" : 2
+    "query": {
+        "fuzzy" : {
+            "price" : {
+                "value" : 12,
+                    "fuzziness" : 2
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Will result in a range query between 10 and 14. Date fields support
 <<time-units,time values>>, eg:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "fuzzy" : {
-        "created" : {
-            "value" : "2010-02-05T12:05:07",
-            "fuzziness" : "1d"
+    "query": {
+        "fuzzy" : {
+            "created" : {
+                "value" : "2010-02-05T12:05:07",
+                    "fuzziness" : "1d"
+            }
         }
     }
 }
 --------------------------------------------------
-
+// CONSOLE
 See <<fuzziness>> for more details about accepted values.
diff --git a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
index 00120a98fc9..dfe4ba25b39 100644
--- a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
+++ b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
@@ -6,6 +6,24 @@ bounding box. Assuming the following indexed document:
 
 [source,js]
 --------------------------------------------------
+PUT /my_locations
+{
+    "mappings": {
+        "location": {
+            "properties": {
+                "pin": {
+                    "properties": {
+                        "location": {
+                            "type": "geo_point"
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+PUT /my_locations/location/1
 {
     "pin" : {
         "location" : {
@@ -15,34 +33,40 @@ bounding box. Assuming the following indexed document:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TESTSETUP
 
 Then the following simple query can be executed with a
 `geo_bounding_box` filter:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_bounding_box" : {
-                "pin.location" : {
-                    "top_left" : {
-                        "lat" : 40.73,
-                        "lon" : -74.1
-                    },
-                    "bottom_right" : {
-                        "lat" : 40.01,
-                        "lon" : -71.12
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+                "filter" : {
+                    "geo_bounding_box" : {
+                        "pin.location" : {
+                            "top_left" : {
+                                "lat" : 40.73,
+                                "lon" : -74.1
+                            },
+                            "bottom_right" : {
+                                "lat" : 40.01,
+                                "lon" : -71.12
+                            }
+                        }
                     }
                 }
-            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== Query Options
@@ -75,21 +99,24 @@ representation of the geo point, the filter can accept it as well:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_bounding_box" : {
-                "pin.location" : {
-                    "top_left" : {
-                        "lat" : 40.73,
-                        "lon" : -74.1
-                    },
-                    "bottom_right" : {
-                        "lat" : 40.01,
-                        "lon" : -71.12
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_bounding_box" : {
+                    "pin.location" : {
+                        "top_left" : {
+                            "lat" : 40.73,
+                            "lon" : -74.1
+                        },
+                        "bottom_right" : {
+                            "lat" : 40.01,
+                            "lon" : -71.12
+                        }
                     }
                 }
             }
@@ -97,6 +124,7 @@ representation of the geo point, the filter can accept it as well:
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ===== Lat Lon As Array
@@ -106,22 +134,26 @@ conform with http://geojson.org/[GeoJSON].
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_bounding_box" : {
-                "pin.location" : {
-                    "top_left" : [-74.1, 40.73],
-                    "bottom_right" : [-71.12, 40.01]
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+                "filter" : {
+                    "geo_bounding_box" : {
+                        "pin.location" : {
+                            "top_left" : [-74.1, 40.73],
+                            "bottom_right" : [-71.12, 40.01]
+                        }
+                    }
                 }
-            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ===== Lat Lon As String
@@ -130,44 +162,52 @@ Format in `lat,lon`.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_bounding_box" : {
-                "pin.location" : {
-                    "top_left" : "40.73, -74.1",
-                    "bottom_right" : "40.01, -71.12"
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+                "filter" : {
+                    "geo_bounding_box" : {
+                        "pin.location" : {
+                            "top_left" : "40.73, -74.1",
+                            "bottom_right" : "40.01, -71.12"
+                        }
+                    }
                 }
-            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ===== Geohash
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_bounding_box" : {
-                "pin.location" : {
-                    "top_left" : "dr5r9ydj2y73",
-                    "bottom_right" : "drj7teegpus6"
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+                "filter" : {
+                    "geo_bounding_box" : {
+                        "pin.location" : {
+                            "top_left" : "dr5r9ydj2y73",
+                            "bottom_right" : "drj7teegpus6"
+                        }
+                    }
                 }
-            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== Vertices
@@ -181,24 +221,28 @@ values separately.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_bounding_box" : {
-                "pin.location" : {
-                    "top" : 40.73,
-                    "left" : -74.1,
-                    "bottom" : 40.01,
-                    "right" : -71.12
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+                "filter" : {
+                    "geo_bounding_box" : {
+                        "pin.location" : {
+                            "top" : 40.73,
+                            "left" : -74.1,
+                            "bottom" : 40.01,
+                            "right" : -71.12
+                        }
+                    }
                 }
-            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 
 [float]
@@ -227,29 +271,33 @@ are not supported. Here is an example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_bounding_box" : {
-                "pin.location" : {
-                    "top_left" : {
-                        "lat" : 40.73,
-                        "lon" : -74.1
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_bounding_box" : {
+                    "pin.location" : {
+                        "top_left" : {
+                            "lat" : 40.73,
+                            "lon" : -74.1
+                        },
+                        "bottom_right" : {
+                            "lat" : 40.10,
+                            "lon" : -71.12
+                        }
                     },
-                    "bottom_right" : {
-                        "lat" : 40.10,
-                        "lon" : -71.12
-                    }
-                },
-                "type" : "indexed"
+                    "type" : "indexed"
+                }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== Ignore Unmapped
diff --git a/docs/reference/query-dsl/geo-distance-range-query.asciidoc b/docs/reference/query-dsl/geo-distance-range-query.asciidoc
index abf557d68c7..4236e2b370c 100644
--- a/docs/reference/query-dsl/geo-distance-range-query.asciidoc
+++ b/docs/reference/query-dsl/geo-distance-range-query.asciidoc
@@ -5,24 +5,28 @@ Filters documents that exists within a range from a specific point:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_distance_range" : {
-                "from" : "200km",
-                "to" : "400km",
-                "pin.location" : {
-                    "lat" : 40,
-                    "lon" : -70
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+                "filter" : {
+                    "geo_distance_range" : {
+                        "from" : "200km",
+                        "to" : "400km",
+                        "pin.location" : {
+                            "lat" : 40,
+                            "lon" : -70
+                        }
+                    }
                 }
-            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Supports the same point location parameter and query options as the
 <<query-dsl-geo-distance-query,geo_distance>>
diff --git a/docs/reference/query-dsl/geo-polygon-query.asciidoc b/docs/reference/query-dsl/geo-polygon-query.asciidoc
index 5717c5dc924..9a0800c7550 100644
--- a/docs/reference/query-dsl/geo-polygon-query.asciidoc
+++ b/docs/reference/query-dsl/geo-polygon-query.asciidoc
@@ -6,25 +6,26 @@ points. Here is an example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "query" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_polygon" : {
-                "person.location" : {
-                    "points" : [
+    "query": {
+        "bool" : {
+            "query" : {
+                "geo_polygon" : {
+                    "person.location" : {
+                        "points" : [
                         {"lat" : 40, "lon" : -70},
                         {"lat" : 30, "lon" : -80},
                         {"lat" : 20, "lon" : -90}
-                    ]
+                        ]
+                    }
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== Query Options
@@ -53,25 +54,29 @@ conform with http://geojson.org/[GeoJSON].
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_polygon" : {
-                "person.location" : {
-                    "points" : [
-                        [-70, 40],
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_polygon" : {
+                    "person.location" : {
+                        "points" : [
+                            [-70, 40],
                         [-80, 30],
-                        [-90, 20]
-                    ]
+                            [-90, 20]
+                        ]
+                    }
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ===== Lat Lon as String
@@ -80,50 +85,58 @@ Format in `lat,lon`.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_polygon" : {
-                "person.location" : {
-                    "points" : [
-                        "40, -70",
-                        "30, -80",
-                        "20, -90"
-                    ]
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+                "filter" : {
+                    "geo_polygon" : {
+                        "person.location" : {
+                            "points" : [
+                                "40, -70",
+                            "30, -80",
+                            "20, -90"
+                            ]
+                        }
+                    }
                 }
-            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ===== Geohash
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_polygon" : {
-                "person.location" : {
-                    "points" : [
-                        "drn5x1g8cu2y",
-                        "30, -80",
-                        "20, -90"
-                    ]
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+                "filter" : {
+                    "geo_polygon" : {
+                        "person.location" : {
+                            "points" : [
+                                "drn5x1g8cu2y",
+                            "30, -80",
+                            "20, -90"
+                            ]
+                        }
+                    }
                 }
-            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== geo_point Type
diff --git a/docs/reference/query-dsl/geo-shape-query.asciidoc b/docs/reference/query-dsl/geo-shape-query.asciidoc
index a892080dda4..12203061336 100644
--- a/docs/reference/query-dsl/geo-shape-query.asciidoc
+++ b/docs/reference/query-dsl/geo-shape-query.asciidoc
@@ -26,10 +26,10 @@ Given a document that looks like this:
 --------------------------------------------------
 {
     "name": "Wind & Wetter, Berlin, Germany",
-    "location": {
-        "type": "Point",
-        "coordinates": [13.400544, 52.530286]
-    }
+        "location": {
+            "type": "Point",
+            "coordinates": [13.400544, 52.530286]
+        }
 }
 --------------------------------------------------
 
@@ -38,6 +38,7 @@ The following query will find the point using the Elasticsearch's
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "query":{
         "bool": {
@@ -59,6 +60,7 @@ The following query will find the point using the Elasticsearch's
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 ==== Pre-Indexed Shape
 
@@ -81,26 +83,30 @@ shape:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool": {
-        "must": {
-            "match_all": {}
-        },
-        "filter": {
-            "geo_shape": {
-                "location": {
-                    "indexed_shape": {
-                        "id": "DEU",
-                        "type": "countries",
-                        "index": "shapes",
-                        "path": "location"
+    "query": {
+        "bool": {
+            "must": {
+                "match_all": {}
+            },
+                "filter": {
+                    "geo_shape": {
+                        "location": {
+                            "indexed_shape": {
+                                "id": "DEU",
+                                "type": "countries",
+                                "index": "shapes",
+                                "path": "location"
+                            }
+                        }
                     }
                 }
-            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 ==== Spatial Relations
 
diff --git a/docs/reference/query-dsl/geohash-cell-query.asciidoc b/docs/reference/query-dsl/geohash-cell-query.asciidoc
index 2dd701fd958..3728a36de5b 100644
--- a/docs/reference/query-dsl/geohash-cell-query.asciidoc
+++ b/docs/reference/query-dsl/geohash-cell-query.asciidoc
@@ -13,6 +13,7 @@ setting the `geohash_prefix` option:
 
 [source,js]
 --------------------------------------------------
+PUT /my_index
 {
     "mappings" : {
         "location": {
@@ -28,6 +29,8 @@ setting the `geohash_prefix` option:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TESTSETUP
 
 The geohash cell can defined by all formats of `geo_points`. If such a cell is
 defined by a latitude and longitude pair the size of the cell needs to be
@@ -42,24 +45,28 @@ next to the given cell.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geohash_cell": {
-                "pin": {
-                    "lat": 13.4080,
-                    "lon": 52.5186
-                },
-                "precision": 3,
-                "neighbors": true
-            }
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+                "filter" : {
+                    "geohash_cell": {
+                        "pin": {
+                            "lat": 13.4080,
+                            "lon": 52.5186
+                        },
+                        "precision": 3,
+                        "neighbors": true
+                    }
+                }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== Ignore Unmapped
diff --git a/docs/reference/query-dsl/has-child-query.asciidoc b/docs/reference/query-dsl/has-child-query.asciidoc
index bfadc33c06c..b863a91074c 100644
--- a/docs/reference/query-dsl/has-child-query.asciidoc
+++ b/docs/reference/query-dsl/has-child-query.asciidoc
@@ -7,17 +7,21 @@ an example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "has_child" : {
-        "type" : "blog_tag",
-        "query" : {
-            "term" : {
-                "tag" : "something"
-            }
+    "query": {
+        "has_child" : {
+            "type" : "blog_tag",
+                "query" : {
+                    "term" : {
+                        "tag" : "something"
+                    }
+                }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== Scoring capabilities
@@ -32,18 +36,22 @@ inside the `has_child` query:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "has_child" : {
-        "type" : "blog_tag",
-        "score_mode" : "min",
-        "query" : {
-            "term" : {
-                "tag" : "something"
-            }
+    "query": {
+        "has_child" : {
+            "type" : "blog_tag",
+                "score_mode" : "min",
+                "query" : {
+                    "term" : {
+                        "tag" : "something"
+                    }
+                }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== Min/Max Children
@@ -54,20 +62,24 @@ a match:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "has_child" : {
-        "type" : "blog_tag",
-        "score_mode" : "min",
-        "min_children": 2, <1>
-        "max_children": 10, <1>
-        "query" : {
-            "term" : {
-                "tag" : "something"
-            }
+    "query": {
+        "has_child" : {
+            "type" : "blog_tag",
+                "score_mode" : "min",
+                "min_children": 2, <1>
+                    "max_children": 10, <1>
+                    "query" : {
+                        "term" : {
+                            "tag" : "something"
+                        }
+                    }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 <1> Both `min_children` and `max_children` are optional.
 
 The  `min_children` and `max_children` parameters can be combined with
diff --git a/docs/reference/query-dsl/has-parent-query.asciidoc b/docs/reference/query-dsl/has-parent-query.asciidoc
index 9896be6a0a5..202bcaac432 100644
--- a/docs/reference/query-dsl/has-parent-query.asciidoc
+++ b/docs/reference/query-dsl/has-parent-query.asciidoc
@@ -9,17 +9,21 @@ in the same manner as the `has_child` query.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "has_parent" : {
-        "parent_type" : "blog",
-        "query" : {
-            "term" : {
-                "tag" : "something"
-            }
+    "query": {
+        "has_parent" : {
+            "parent_type" : "blog",
+                "query" : {
+                    "term" : {
+                        "tag" : "something"
+                    }
+                }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== Scoring capabilities
@@ -34,18 +38,22 @@ matching parent document. The score mode can be specified with the
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "has_parent" : {
-        "parent_type" : "blog",
-        "score" : true,
-        "query" : {
-            "term" : {
-                "tag" : "something"
-            }
+    "query": {
+        "has_parent" : {
+            "parent_type" : "blog",
+                "score" : true,
+                "query" : {
+                    "term" : {
+                        "tag" : "something"
+                    }
+                }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== Ignore Unmapped
diff --git a/docs/reference/query-dsl/ids-query.asciidoc b/docs/reference/query-dsl/ids-query.asciidoc
index 7d08243a78f..acf5722fe72 100644
--- a/docs/reference/query-dsl/ids-query.asciidoc
+++ b/docs/reference/query-dsl/ids-query.asciidoc
@@ -6,13 +6,17 @@ uses the <<mapping-uid-field,_uid>> field.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "ids" : {
-        "type" : "my_type",
-        "values" : ["1", "4", "100"]
+    "query": {
+        "ids" : {
+            "type" : "my_type",
+                "values" : ["1", "4", "100"]
+        }
     }
 }    
 --------------------------------------------------
+// CONSOLE
 
 The `type` is optional and can be omitted, and can also accept an array
 of values. If no type is specified, all types defined in the index mapping are tried.
diff --git a/docs/reference/query-dsl/indices-query.asciidoc b/docs/reference/query-dsl/indices-query.asciidoc
index e3b604b7a39..552f5927fe1 100644
--- a/docs/reference/query-dsl/indices-query.asciidoc
+++ b/docs/reference/query-dsl/indices-query.asciidoc
@@ -9,18 +9,22 @@ on the list, the alternative `no_match_query` is executed.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "indices" : {
-        "indices" : ["index1", "index2"],
-        "query" : {
-            "term" : { "tag" : "wow" }
-        },
-        "no_match_query" : {
-            "term" : { "tag" : "kow" }
+    "query": {
+        "indices" : {
+            "indices" : ["index1", "index2"],
+                "query" : {
+                    "term" : { "tag" : "wow" }
+                },
+                "no_match_query" : {
+                    "term" : { "tag" : "kow" }
+                }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE 
 
 You can use the `index` field to provide a single index.
 
diff --git a/docs/reference/query-dsl/match-phrase-prefix-query.asciidoc b/docs/reference/query-dsl/match-phrase-prefix-query.asciidoc
index b715bf8bd6d..e7e0f618652 100644
--- a/docs/reference/query-dsl/match-phrase-prefix-query.asciidoc
+++ b/docs/reference/query-dsl/match-phrase-prefix-query.asciidoc
@@ -6,12 +6,16 @@ allows for prefix matches on the last term in the text. For example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "match_phrase_prefix" : {
-        "message" : "quick brown f"
+    "query": {
+        "match_phrase_prefix" : {
+            "message" : "quick brown f"
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 It accepts the same parameters as the phrase type. In addition, it also
 accepts a `max_expansions` parameter (default `50`) that can control to how
@@ -21,15 +25,19 @@ example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "match_phrase_prefix" : {
-        "message" : {
-            "query" : "quick brown f",
-            "max_expansions" : 10
+    "query": {
+        "match_phrase_prefix" : {
+            "message" : {
+                "query" : "quick brown f",
+                "max_expansions" : 10
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [IMPORTANT]
 ===================================================
@@ -53,4 +61,4 @@ For better solutions for _search-as-you-type_ see the
 <<search-suggesters-completion,completion suggester>> and
 {guide}/_index_time_search_as_you_type.html[Index-Time Search-as-You-Type].
 
-===================================================
\ No newline at end of file
+===================================================
diff --git a/docs/reference/query-dsl/match-phrase-query.asciidoc b/docs/reference/query-dsl/match-phrase-query.asciidoc
index 105866be12a..717cb94312e 100644
--- a/docs/reference/query-dsl/match-phrase-query.asciidoc
+++ b/docs/reference/query-dsl/match-phrase-query.asciidoc
@@ -6,12 +6,16 @@ out of the analyzed text. For example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "match_phrase" : {
-        "message" : "this is a test"
+    "query": {
+        "match_phrase" : {
+            "message" : "this is a test"
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 A phrase query matches terms up to a configurable `slop`
 (which defaults to 0) in any order. Transposed terms have a slop of 2.
@@ -22,12 +26,16 @@ definition, or the default search analyzer, for example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "match_phrase" : {
-        "message" : {
-            "query" : "this is a test",
-            "analyzer" : "my_analyzer"
+    "query": {
+        "match_phrase" : {
+            "message" : {
+                "query" : "this is a test",
+                    "analyzer" : "my_analyzer"
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/query-dsl/match-query.asciidoc b/docs/reference/query-dsl/match-query.asciidoc
index 90936f024f6..8a301081445 100644
--- a/docs/reference/query-dsl/match-query.asciidoc
+++ b/docs/reference/query-dsl/match-query.asciidoc
@@ -7,12 +7,16 @@ them, and constructs a query. For example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "match" : {
-        "message" : "this is a test"
+    "query": {
+        "match" : {
+            "message" : "this is a test"
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Note, `message` is the name of a field, you can substitute the name of
 any field (including `_all`) instead.
@@ -54,15 +58,19 @@ change in structure, `message` is the field name):
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "match" : {
-        "message" : {
-            "query" : "this is a test",
-            "operator" : "and"
+    "query": {
+        "match" : {
+            "message" : {
+                "query" : "this is a test",
+                    "operator" : "and"
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [[query-dsl-match-query-zero]]
 ===== Zero terms query
@@ -73,16 +81,20 @@ change that the `zero_terms_query` option can be used, which accepts
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "match" : {
-        "message" : {
-            "query" : "to be or not to be",
-            "operator" : "and",
-            "zero_terms_query": "all"
+    "query": {
+        "match" : {
+            "message" : {
+                "query" : "to be or not to be",
+                    "operator" : "and",
+                    "zero_terms_query": "all"
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [[query-dsl-match-query-cutoff]]
 ===== Cutoff frequency
@@ -110,16 +122,19 @@ Here is an example showing a query composed of stopwords exclusively:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "match" : {
-        "message" : {
-            "query" : "to be or not to be",
-            "cutoff_frequency" : 0.001
+    "query": {
+        "match" : {
+            "message" : {
+                "query" : "to be or not to be",
+                    "cutoff_frequency" : 0.001
+            }
         }
     }
 }
 --------------------------------------------------
-
+// CONSOLE
 
 IMPORTANT: The `cutoff_frequency` option operates on a per-shard-level. This means
 that when trying it out on test indexes with low document numbers you
diff --git a/docs/reference/query-dsl/nested-query.asciidoc b/docs/reference/query-dsl/nested-query.asciidoc
index 0b861509c0e..17ee20b983e 100644
--- a/docs/reference/query-dsl/nested-query.asciidoc
+++ b/docs/reference/query-dsl/nested-query.asciidoc
@@ -10,6 +10,7 @@ will work with:
 
 [source,js]
 --------------------------------------------------
+PUT /my_index
 {
     "type1" : {
         "properties" : {
@@ -20,30 +21,36 @@ will work with:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TESTSETUP
 
 And here is a sample nested query usage:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "nested" : {
-        "path" : "obj1",
-        "score_mode" : "avg",
-        "query" : {
-            "bool" : {
-                "must" : [
-                    {
-                        "match" : {"obj1.name" : "blue"}
-                    },
-                    {
-                        "range" : {"obj1.count" : {"gt" : 5}}
+    "query": {
+        "nested" : {
+            "path" : "obj1",
+                "score_mode" : "avg",
+                "query" : {
+                    "bool" : {
+                        "must" : [
+                        {
+                            "match" : {"obj1.name" : "blue"}
+                        },
+                        {
+                            "range" : {"obj1.count" : {"gt" : 5}}
+                        }
+                        ]
                     }
-                ]
-            }
+                }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 The query `path` points to the nested object path, and the `query`
 includes the query that will run on the nested docs matching the
diff --git a/docs/reference/query-dsl/parent-id-query.asciidoc b/docs/reference/query-dsl/parent-id-query.asciidoc
index 1aef5b1948f..321adeecb5e 100644
--- a/docs/reference/query-dsl/parent-id-query.asciidoc
+++ b/docs/reference/query-dsl/parent-id-query.asciidoc
@@ -7,6 +7,7 @@ The `parent_id` query can be used to find child documents which belong to a part
 
 [source,js]
 --------------------------------------------------
+PUT /my_index
 {
     "parent_id" : {
         "type" : "blog_tag",
@@ -14,6 +15,8 @@ The `parent_id` query can be used to find child documents which belong to a part
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TESTSETUP
 
 The above is functionally equivalent to using the following
 <<query-dsl-has-parent-query, `has_parent`>> query, but performs
@@ -21,17 +24,21 @@ better as it does not need to do a join:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "has_parent": {
-    "type": "blog",
     "query": {
-      "term": {
-        "_id": "1"
-      }
+        "has_parent": {
+            "type": "blog",
+                "query": {
+                    "term": {
+                        "_id": "1"
+                    }
+                }
+        }
     }
-  }
 }
 --------------------------------------------------
+// CONSOLE
 
 ==== Parameters
 
diff --git a/docs/reference/query-dsl/percolate-query.asciidoc b/docs/reference/query-dsl/percolate-query.asciidoc
index 267647c9dbb..8c0e4dc1feb 100644
--- a/docs/reference/query-dsl/percolate-query.asciidoc
+++ b/docs/reference/query-dsl/percolate-query.asciidoc
@@ -13,26 +13,27 @@ Create an index with two mappings:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT "http://localhost:9200/my-index" -d'
+PUT /my-index
 {
-  "mappings": {
-    "doctype": {
-      "properties": {
-        "message": {
-          "type": "string"
-        }
-      }
-    },
-    "queries": {
-      "properties": {
-        "query": {
-          "type": "percolator"
-        }
-      }
+    "mappings": {
+        "doctype": {
+            "properties": {
+                "message": {
+                    "type": "string"
+                }
+            }
+        },
+            "queries": {
+                "properties": {
+                    "query": {
+                        "type": "percolator"
+                    }
+                }
+            }
     }
-  }
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
 The `doctype` mapping is the mapping used to preprocess
 the document defined in the `percolator` query before it
@@ -50,20 +51,24 @@ Register a query in the percolator:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT 'localhost:9200/my-index/queries/1' -d '{
+PUT /my-index/queries/1
+{
     "query" : {
         "match" : {
             "message" : "bonsai tree"
         }
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 Match a document to the registered percolator queries:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/my-index/_search' -d '{
+GET /my-index/_search
+{
     "query" : {
         "percolate" : {
             "field" : "query",
@@ -73,8 +78,10 @@ curl -XGET 'localhost:9200/my-index/_search' -d '{
             }
         }
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 The above request will yield the following response:
 
@@ -151,12 +158,13 @@ Index the document we want to percolate:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT "http://localhost:9200/my-index/message/1" -d'
+PUT /my-index/message/1
 {
   "message" : "A new bonsai tree in the office"
-}'
+}
 --------------------------------------------------
-
+// CONSOLE
+// TEST[continued]
 Index response:
 
 [source,js]
@@ -179,7 +187,7 @@ Percolating an existing document, using the index response as basis to build to
 
 [source,js]
 --------------------------------------------------
-curl -XGET "http://localhost:9200/my-index/_search" -d'
+GET /my-index/_search
 {
     "query" : {
         "percolate" : {
@@ -191,8 +199,10 @@ curl -XGET "http://localhost:9200/my-index/_search" -d'
             "version" : 1 <1>
         }
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 <1> The version is optional, but useful in certain cases. We can then ensure that we are try to percolate
 the document we just have indexed. A change may be made after we have indexed, and if that is the
@@ -216,35 +226,39 @@ Save a query:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT "http://localhost:9200/my-index/queries/1" -d'
+PUT /my-index/queries/1
 {
     "query" : {
         "match" : {
             "message" : "brown fox"
         }
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 Save another query:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT "http://localhost:9200/my-index/queries/2" -d'
+PUT /my-index/queries/2
 {
     "query" : {
         "match" : {
             "message" : "lazy dog"
         }
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 Execute a search request with the `percolate` query and highlighting enabled:
 
 [source,js]
 --------------------------------------------------
-curl -XGET "http://localhost:9200/my-index/_search" -d'
+GET /my-index/_search
 {
     "query" : {
         "percolate" : {
@@ -260,8 +274,10 @@ curl -XGET "http://localhost:9200/my-index/_search" -d'
         "message": {}
       }
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 This will yield the following response.
 
@@ -336,4 +352,4 @@ that are registered to the index that the search request is targeted for, are go
 in-memory index. This happens on each shard the search request needs to execute.
 
 By using `routing` or additional queries the amount of percolator queries that need to be executed can be reduced and thus
-the time the search API needs to run can be decreased.
\ No newline at end of file
+the time the search API needs to run can be decreased.
diff --git a/docs/reference/query-dsl/query-string-query.asciidoc b/docs/reference/query-dsl/query-string-query.asciidoc
index 9ad68ad4f73..c1f5c06f65c 100644
--- a/docs/reference/query-dsl/query-string-query.asciidoc
+++ b/docs/reference/query-dsl/query-string-query.asciidoc
@@ -6,13 +6,17 @@ an example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query_string" : {
-        "default_field" : "content",
-        "query" : "this AND that OR thus"
+    "query": {
+        "query_string" : {
+            "default_field" : "content",
+                "query" : "this AND that OR thus"
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 The `query_string` top level parameters include:
 
@@ -113,25 +117,33 @@ For example, the following query
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query_string" : {
-        "fields" : ["content", "name"],
-        "query" : "this AND that"
+    "query": {
+        "query_string" : {
+            "fields" : ["content", "name"],
+                "query" : "this AND that"
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 matches the same words as
 
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query_string": {
-      "query": "(content:this OR name:this) AND (content:that OR name:that)"
+    "query": {
+        "query_string": {
+            "query": "(content:this OR name:this) AND (content:that OR name:that)"
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Since several queries are generated from the individual search terms,
 combining them can be automatically done using either a `dis_max` query or a
@@ -140,14 +152,18 @@ notation):
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query_string" : {
-        "fields" : ["content", "name^5"],
-        "query" : "this AND that OR thus",
-        "use_dis_max" : true
+    "query": {
+        "query_string" : {
+            "fields" : ["content", "name^5"],
+            "query" : "this AND that OR thus",
+            "use_dis_max" : true
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Simple wildcard can also be used to search "within" specific inner
 elements of the document. For example, if we have a `city` object with
@@ -156,14 +172,18 @@ search on all "city" fields:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query_string" : {
-        "fields" : ["city.*"],
-        "query" : "this AND that OR thus",
-        "use_dis_max" : true
+    "query": {
+        "query_string" : {
+            "fields" : ["city.*"],
+                "query" : "this AND that OR thus",
+                "use_dis_max" : true
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Another option is to provide the wildcard fields search in the query
 string itself (properly escaping the `*` sign), for example:
@@ -188,13 +208,17 @@ introduced fields included). For example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query_string" : {
-        "fields" : ["content", "name.*^5"],
-        "query" : "this AND that OR thus",
-        "use_dis_max" : true
+    "query": {
+        "query_string" : {
+            "fields" : ["content", "name.*^5"],
+            "query" : "this AND that OR thus",
+            "use_dis_max" : true
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 include::query-string-syntax.asciidoc[]
diff --git a/docs/reference/query-dsl/regexp-query.asciidoc b/docs/reference/query-dsl/regexp-query.asciidoc
index 692caf43480..55a1c3469c6 100644
--- a/docs/reference/query-dsl/regexp-query.asciidoc
+++ b/docs/reference/query-dsl/regexp-query.asciidoc
@@ -15,40 +15,52 @@ matchers like `.*?+` will mostly lower performance.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "regexp":{
-        "name.first": "s.*y"
+    "query": {
+        "regexp":{
+            "name.first": "s.*y"
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Boosting is also supported
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "regexp":{
-        "name.first":{
-            "value":"s.*y",
-            "boost":1.2
+    "query": {
+        "regexp":{
+            "name.first":{
+                "value":"s.*y",
+                    "boost":1.2
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 You can also use special flags
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "regexp":{
-        "name.first": {
-            "value": "s.*y",
-            "flags" : "INTERSECTION|COMPLEMENT|EMPTY"
+    "query": {
+        "regexp":{
+            "name.first": {
+                "value": "s.*y",
+                    "flags" : "INTERSECTION|COMPLEMENT|EMPTY"
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Possible flags are `ALL` (default), `ANYSTRING`, `COMPLEMENT`,
 `EMPTY`, `INTERSECTION`, `INTERVAL`, or `NONE`. Please check the
@@ -64,16 +76,19 @@ this limit to allow more complex regular expressions to execute.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "regexp":{
-        "name.first": {
-            "value": "s.*y",
-            "flags" : "INTERSECTION|COMPLEMENT|EMPTY",
-	    "max_determinized_states": 20000
+    "query": {
+        "regexp":{
+            "name.first": {
+                "value": "s.*y",
+                "flags" : "INTERSECTION|COMPLEMENT|EMPTY",
+                "max_determinized_states": 20000
+            }
         }
     }
 }
 --------------------------------------------------
-
+// CONSOLE
 
 include::regexp-syntax.asciidoc[]
diff --git a/docs/reference/query-dsl/script-query.asciidoc b/docs/reference/query-dsl/script-query.asciidoc
index 223460f723d..ee06a1b64bd 100644
--- a/docs/reference/query-dsl/script-query.asciidoc
+++ b/docs/reference/query-dsl/script-query.asciidoc
@@ -7,17 +7,20 @@ context, for example:
 
 [source,js]
 ----------------------------------------------
-"bool" : {
-    "must" : {
-        ...
-    }, 
-    "filter" : {
-        "script" : {
-            "script" : "doc['num1'].value > 1"
+GET /_search
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "script" : {
+                    "script" : "doc['num1'].value > 1"
+                }
+            }
         }
     }
 }
 ----------------------------------------------
+// CONSOLE
 
 [float]
 ==== Custom Parameters
@@ -28,20 +31,23 @@ to use the ability to pass parameters to the script itself, for example:
 
 [source,js]
 ----------------------------------------------
-"bool" : {
-    "must" : {
-        ...
-    }, 
-    "filter" : {
-        "script" : {
-            "script" : {
-                "inline" : "doc['num1'].value > param1"
-                "params" : {
-                    "param1" : 5
+GET /_search
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "script" : {
+                    "script" : {
+                        "inline" : "doc['num1'].value > param1",
+                        "params" : {
+                            "param1" : 5
+                        }
+                    }
                 }
             }
         }
     }
 }
 ----------------------------------------------
+// CONSOLE
 
diff --git a/docs/reference/query-dsl/terms-query.asciidoc b/docs/reference/query-dsl/terms-query.asciidoc
index 7382928e7f8..a840292d242 100644
--- a/docs/reference/query-dsl/terms-query.asciidoc
+++ b/docs/reference/query-dsl/terms-query.asciidoc
@@ -67,33 +67,37 @@ possible, reducing the need for networking.
 
 [float]
 ===== Terms lookup twitter example
+At first we index the information for user with id 2, specifically, its
+followers, than index a tweet from user with id 1. Finally we search on
+all the tweets that match the followers of user 2.
 
 [source,js]
 --------------------------------------------------
-# index the information for user with id 2, specifically, its followers
-curl -XPUT localhost:9200/users/user/2 -d '{
-   "followers" : ["1", "3"]
-}'
+PUT /users/user/2
+{
+    "followers" : ["1", "3"]
+}
 
-# index a tweet, from user with id 1
-curl -XPUT localhost:9200/tweets/tweet/1 -d '{
-   "user" : "1"
-}'
+PUT /tweets/tweet/1
+{
+    "user" : "1"
+}
 
-# search on all the tweets that match the followers of user 2
-curl -XGET localhost:9200/tweets/_search -d '{
-  "query" : {
-    "terms" : {
-      "user" : {
-        "index" : "users",
-        "type" : "user",
-        "id" : "2",
-        "path" : "followers"
-      }
+GET /tweets/_search
+{
+    "query" : {
+        "terms" : {
+            "user" : {
+                "index" : "users",
+                    "type" : "user",
+                    "id" : "2",
+                    "path" : "followers"
+            }
+        }
     }
-  }
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
 The structure of the external terms document can also include array of
 inner objects, for example:
diff --git a/docs/reference/query-dsl/wildcard-query.asciidoc b/docs/reference/query-dsl/wildcard-query.asciidoc
index d72dbec2481..ad82c029d8c 100644
--- a/docs/reference/query-dsl/wildcard-query.asciidoc
+++ b/docs/reference/query-dsl/wildcard-query.asciidoc
@@ -11,28 +11,40 @@ query maps to Lucene `WildcardQuery`.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "wildcard" : { "user" : "ki*y" }
+    "query": {
+        "wildcard" : { "user" : "ki*y" }
+    }
 }
 --------------------------------------------------
+// CONSOLE
 
 A boost can also be associated with the query:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "wildcard" : { "user" : { "value" : "ki*y", "boost" : 2.0 } }
+    "query": {
+        "wildcard" : { "user" : { "value" : "ki*y", "boost" : 2.0 } }
+    }
 }
 --------------------------------------------------
+// CONSOLE
 
 Or :
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "wildcard" : { "user" : { "wildcard" : "ki*y", "boost" : 2.0 } }
+    "query": {
+        "wildcard" : { "user" : { "wildcard" : "ki*y", "boost" : 2.0 } }
+    }
 }
 --------------------------------------------------
+// CONSOLE
 
 This multi term query allows to control how it gets rewritten using the
 <<query-dsl-multi-term-rewrite,rewrite>>

From d0edd13f7bd8f87053faa94891c6a5a775b4d689 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 11 May 2016 10:37:23 -0400
Subject: [PATCH 0186/1311] Log setting key not setting object in IMC

This commit modifies two logging statements in the
IndexingMemoryController to log the key for the setting
indices.memory.index_buffer_size instead of the object.

Relates #18191
---
 .../org/elasticsearch/indices/IndexingMemoryController.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java b/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
index 87035044297..b47172bb0ae 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
@@ -314,7 +314,7 @@ public class IndexingMemoryController extends AbstractComponent implements Index
 
             if (logger.isTraceEnabled()) {
                 logger.trace("total indexing heap bytes used [{}] vs {} [{}], currently writing bytes [{}]",
-                             new ByteSizeValue(totalBytesUsed), INDEX_BUFFER_SIZE_SETTING, indexingBuffer, new ByteSizeValue(totalBytesWriting));
+                             new ByteSizeValue(totalBytesUsed), INDEX_BUFFER_SIZE_SETTING.getKey(), indexingBuffer, new ByteSizeValue(totalBytesWriting));
             }
 
             // If we are using more than 50% of our budget across both indexing buffer and bytes we are still moving to disk, then we now
@@ -354,7 +354,7 @@ public class IndexingMemoryController extends AbstractComponent implements Index
                 }
 
                 logger.debug("now write some indexing buffers: total indexing heap bytes used [{}] vs {} [{}], currently writing bytes [{}], [{}] shards with non-zero indexing buffer",
-                             new ByteSizeValue(totalBytesUsed), INDEX_BUFFER_SIZE_SETTING, indexingBuffer, new ByteSizeValue(totalBytesWriting), queue.size());
+                             new ByteSizeValue(totalBytesUsed), INDEX_BUFFER_SIZE_SETTING.getKey(), indexingBuffer, new ByteSizeValue(totalBytesWriting), queue.size());
 
                 while (totalBytesUsed > indexingBuffer.bytes() && queue.isEmpty() == false) {
                     ShardAndBytesUsed largest = queue.poll();

From e4edee5d9a7ae6e03e3062a3f141d22fc8ad0bd1 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 11 May 2016 10:48:08 -0400
Subject: [PATCH 0187/1311] Equal default min and max heap settings

Today we encourage users to set their minimum and maximum heap settings
equal to each other to prevent the heap from resizing. Yet, the default
heap settings do not start Elasticsearch in this fashion. This commit
addresses this discrepancy by setting the default min heap to '512m' and
the default max heap to the default min heap.

Relates #16334
---
 distribution/build.gradle | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/distribution/build.gradle b/distribution/build.gradle
index 5cd1e761b83..adbce86ba24 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -438,6 +438,7 @@ task run(type: RunTask) {}
  * </dl>
  */
 Map<String, String> expansionsForDistribution(distributionType) {
+  String heapSize = '512m'
   String footer = "# Built for ${project.name}-${project.version} " +
       "(${distributionType})"
   Map<String, Object> expansions = [
@@ -457,8 +458,8 @@ Map<String, String> expansionsForDistribution(distributionType) {
       'def': '',
     ],
 
-    'heap.min': '256m',
-    'heap.max': '1g',
+    'heap.min': "${heapSize}",
+    'heap.max': "${heapSize}",
 
     'stopping.timeout': [
       'rpm': 86400,

From 866a5459f023e4fa2775eb9158d03342e6221fd0 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Wed, 27 Apr 2016 20:59:28 +0200
Subject: [PATCH 0188/1311] Make significant terms work on fields that are
 indexed with points. #18031

It will keep using the caching terms enum for keyword/text fields and falls back
to IndexSearcher.count for fields that do not use the inverted index for
searching (such as numbers and ip addresses). Note that this probably means that
significant terms aggregations on these fields will be less efficient than they
used to be. It should be ok under a sampler aggregation though.

This moves tests back to the state they were in before numbers started using
points, and also adds a new test that significant terms aggs fail if a field is
not indexed.

In the long term, we might want to follow the approach that Robert initially
proposed that consists in collecting all documents from the background filter in
order to compute frequencies using doc values. This would also mean that
significant terms aggregations do not require fields to be indexed anymore.
---
 .../lucene/index/FilterableTermsEnum.java     |  14 ---
 ...balOrdinalsSignificantTermsAggregator.java |   2 +-
 .../SignificantLongTermsAggregator.java       |   2 +-
 .../SignificantStringTermsAggregator.java     |   2 +-
 .../SignificantTermsAggregatorFactory.java    | 107 +++++++++---------
 .../bucket/SignificantTermsIT.java            |  44 +++++--
 .../SignificantTermsSignificanceScoreIT.java  |  15 ++-
 .../bucket/TermsShardMinDocCountIT.java       |   2 +-
 .../migrate_5_0/aggregations.asciidoc         |  12 +-
 9 files changed, 111 insertions(+), 89 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/lucene/index/FilterableTermsEnum.java b/core/src/main/java/org/elasticsearch/common/lucene/index/FilterableTermsEnum.java
index 971cbdafffe..ded912cf9e8 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/index/FilterableTermsEnum.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/index/FilterableTermsEnum.java
@@ -66,18 +66,12 @@ public class FilterableTermsEnum extends TermsEnum {
     protected long currentTotalTermFreq = 0;
     protected BytesRef current;
     protected final int docsEnumFlag;
-    protected int numDocs;
 
     public FilterableTermsEnum(IndexReader reader, String field, int docsEnumFlag, @Nullable Query filter) throws IOException {
         if ((docsEnumFlag != PostingsEnum.FREQS) && (docsEnumFlag != PostingsEnum.NONE)) {
             throw new IllegalArgumentException("invalid docsEnumFlag of " + docsEnumFlag);
         }
         this.docsEnumFlag = docsEnumFlag;
-        if (filter == null) {
-            // Important - need to use the doc count that includes deleted docs
-            // or we have this issue: https://github.com/elastic/elasticsearch/issues/7951
-            numDocs = reader.maxDoc();
-        }
         List<LeafReaderContext> leaves = reader.leaves();
         List<Holder> enums = new ArrayList<>(leaves.size());
         final Weight weight;
@@ -118,20 +112,12 @@ public class FilterableTermsEnum extends TermsEnum {
                 }
 
                 bits = BitSet.of(docs, context.reader().maxDoc());
-
-                // Count how many docs are in our filtered set
-                // TODO make this lazy-loaded only for those that need it?
-                numDocs += bits.cardinality();
             }
             enums.add(new Holder(termsEnum, bits));
         }
         this.enums = enums.toArray(new Holder[enums.size()]);
     }
 
-    public int getNumDocs() {
-        return numDocs;
-    }
-
     @Override
     public BytesRef term() throws IOException {
         return current;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java
index 342b07b0007..2372886672e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java
@@ -91,7 +91,7 @@ public class GlobalOrdinalsSignificantTermsAggregator extends GlobalOrdinalsStri
         } else {
             size = (int) Math.min(maxBucketOrd(), bucketCountThresholds.getShardSize());
         }
-        long supersetSize = termsAggFactory.prepareBackground(context);
+        long supersetSize = termsAggFactory.getSupersetNumDocs();
         long subsetSize = numCollectedDocs;
 
         BucketSignificancePriorityQueue ordered = new BucketSignificancePriorityQueue(size);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsAggregator.java
index 6c3ce5423c8..91a585812d1 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTermsAggregator.java
@@ -79,7 +79,7 @@ public class SignificantLongTermsAggregator extends LongTermsAggregator {
 
         final int size = (int) Math.min(bucketOrds.size(), bucketCountThresholds.getShardSize());
 
-        long supersetSize = termsAggFactory.prepareBackground(context);
+        long supersetSize = termsAggFactory.getSupersetNumDocs();
         long subsetSize = numCollectedDocs;
 
         BucketSignificancePriorityQueue ordered = new BucketSignificancePriorityQueue(size);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsAggregator.java
index 320f5165b97..c1d4e2225ea 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsAggregator.java
@@ -78,7 +78,7 @@ public class SignificantStringTermsAggregator extends StringTermsAggregator {
         assert owningBucketOrdinal == 0;
 
         final int size = (int) Math.min(bucketOrds.size(), bucketCountThresholds.getShardSize());
-        long supersetSize = termsAggFactory.prepareBackground(context);
+        long supersetSize = termsAggFactory.getSupersetNumDocs();
         long subsetSize = numCollectedDocs;
 
         BucketSignificancePriorityQueue ordered = new BucketSignificancePriorityQueue(size);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
index ce3f83079ea..13126029b8e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
@@ -19,12 +19,15 @@
 
 package org.elasticsearch.search.aggregations.bucket.significant;
 
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.PostingsEnum;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.search.Query;
+import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.lease.Releasable;
@@ -65,7 +68,8 @@ public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFac
     private MappedFieldType fieldType;
     private FilterableTermsEnum termsEnum;
     private int numberOfAggregatorsCreated;
-    private final QueryBuilder filterBuilder;
+    private final Query filter;
+    private final int supersetNumDocs;
     private final TermsAggregator.BucketCountThresholds bucketCountThresholds;
     private final SignificanceHeuristic significanceHeuristic;
 
@@ -76,7 +80,15 @@ public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFac
         super(name, type, config, context, parent, subFactoriesBuilder, metaData);
         this.includeExclude = includeExclude;
         this.executionHint = executionHint;
-        this.filterBuilder = filterBuilder;
+        this.filter = filterBuilder == null
+                ? null
+                : filterBuilder.toQuery(context.searchContext().getQueryShardContext());
+        IndexSearcher searcher = context.searchContext().searcher();
+        this.supersetNumDocs = filter == null
+                // Important - need to use the doc count that includes deleted docs
+                // or we have this issue: https://github.com/elastic/elasticsearch/issues/7951
+                ? searcher.getIndexReader().maxDoc()
+                : searcher.count(filter);
         this.bucketCountThresholds = bucketCountThresholds;
         this.significanceHeuristic = significanceHeuristic;
         this.significanceHeuristic.initialize(context.searchContext());
@@ -92,65 +104,56 @@ public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFac
     }
 
     /**
-     * Creates the TermsEnum (if not already created) and must be called before
-     * any calls to getBackgroundFrequency
-     *
-     * @param context
-     *            The aggregation context
-     * @return The number of documents in the index (after an optional filter
-     *         might have been applied)
+     * Get the number of docs in the superset.
      */
-    public long prepareBackground(AggregationContext context) {
+    public long getSupersetNumDocs() {
+        return supersetNumDocs;
+    }
+
+    private FilterableTermsEnum getTermsEnum(String field) throws IOException {
         if (termsEnum != null) {
-            // already prepared - return
-            return termsEnum.getNumDocs();
+            return termsEnum;
         }
-        SearchContext searchContext = context.searchContext();
-        IndexReader reader = searchContext.searcher().getIndexReader();
-        Query filter = null;
-        try {
-            if (filterBuilder != null) {
-                filter = filterBuilder.toFilter(context.searchContext().getQueryShardContext());
-            }
-        } catch (IOException e) {
-            throw new ElasticsearchException("failed to create filter: " + filterBuilder.toString(), e);
+        IndexReader reader = context.searchContext().searcher().getIndexReader();
+        if (numberOfAggregatorsCreated > 1) {
+            termsEnum = new FreqTermsEnum(reader, field, true, false, filter, context.bigArrays());
+        } else {
+            termsEnum = new FilterableTermsEnum(reader, indexedFieldName, PostingsEnum.NONE, filter);
         }
-        try {
-            if (numberOfAggregatorsCreated == 1) {
-                // Setup a termsEnum for sole use by one aggregator
-                termsEnum = new FilterableTermsEnum(reader, indexedFieldName, PostingsEnum.NONE, filter);
+        return termsEnum;
+    }
+
+    private long getBackgroundFrequency(String value) throws IOException {
+        Query query = fieldType.termQuery(value, context.searchContext().getQueryShardContext());
+        if (query instanceof TermQuery) {
+            // for types that use the inverted index, we prefer using a caching terms
+            // enum that will do a better job at reusing index inputs
+            Term term = ((TermQuery) query).getTerm();
+            FilterableTermsEnum termsEnum = getTermsEnum(term.field());
+            if (termsEnum.seekExact(term.bytes())) {
+                return termsEnum.docFreq();
             } else {
-                // When we have > 1 agg we have possibility of duplicate term
-                // frequency lookups
-                // and so use a TermsEnum that caches results of all term
-                // lookups
-                termsEnum = new FreqTermsEnum(reader, indexedFieldName, true, false, filter, searchContext.bigArrays());
+                return 0;
             }
-        } catch (IOException e) {
-            throw new ElasticsearchException("failed to build terms enumeration", e);
         }
-        return termsEnum.getNumDocs();
+        // otherwise do it the naive way
+        if (filter != null) {
+            query = new BooleanQuery.Builder()
+                    .add(query, Occur.FILTER)
+                    .add(filter, Occur.FILTER)
+                    .build();
+        }
+        return context.searchContext().searcher().count(query);
     }
 
-    public long getBackgroundFrequency(BytesRef termBytes) {
-        assert termsEnum != null; // having failed to find a field in the index
-                                  // we don't expect any calls for frequencies
-        long result = 0;
-        try {
-            if (termsEnum.seekExact(termBytes)) {
-                result = termsEnum.docFreq();
-            }
-        } catch (IOException e) {
-            throw new ElasticsearchException("IOException loading background document frequency info", e);
-        }
-        return result;
+    public long getBackgroundFrequency(BytesRef termBytes) throws IOException {
+        String value = config.format().format(termBytes);
+        return getBackgroundFrequency(value);
     }
 
-    public long getBackgroundFrequency(long value) {
-        Query query = fieldType.termQuery(value, null);
-        Term term = MappedFieldType.extractTerm(query);
-        BytesRef indexedVal = term.bytes();
-        return getBackgroundFrequency(indexedVal);
+    public long getBackgroundFrequency(long termNum) throws IOException {
+        String value = config.format().format(termNum);
+        return getBackgroundFrequency(value);
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsIT.java
index f3ca8aa0b7b..587e44b21b2 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsIT.java
@@ -19,8 +19,10 @@
 package org.elasticsearch.search.aggregations.bucket;
 
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.TermQueryBuilder;
@@ -47,6 +49,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.significantTerms;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
@@ -69,15 +72,15 @@ public class SignificantTermsIT extends ESIntegTestCase {
                 .build();
     }
 
-    public static final String MUSIC_CATEGORY="1";
-    public static final String OTHER_CATEGORY="2";
-    public static final String SNOWBOARDING_CATEGORY="3";
+    public static final int MUSIC_CATEGORY=1;
+    public static final int OTHER_CATEGORY=2;
+    public static final int SNOWBOARDING_CATEGORY=3;
 
     @Override
     public void setupSuiteScopeCluster() throws Exception {
         assertAcked(prepareCreate("test").setSettings(SETTING_NUMBER_OF_SHARDS, 5, SETTING_NUMBER_OF_REPLICAS, 0).addMapping("fact",
                 "_routing", "required=true", "routing_id", "type=keyword", "fact_category",
-                "type=keyword,index=true", "description", "type=text,fielddata=true"));
+                "type=integer", "description", "type=text,fielddata=true"));
         createIndex("idx_unmapped");
 
         ensureGreen();
@@ -110,6 +113,15 @@ public class SignificantTermsIT extends ESIntegTestCase {
                     .setSource("fact_category", parts[1], "description", parts[2]).get();
         }
         client().admin().indices().refresh(new RefreshRequest("test")).get();
+
+        assertAcked(prepareCreate("test_not_indexed")
+                .setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .addMapping("type",
+                        "my_keyword", "type=keyword,index=false",
+                        "my_long", "type=long,index=false"));
+        indexRandom(true,
+                client().prepareIndex("test_not_indexed", "type", "1").setSource(
+                        "my_keyword", "foo", "my_long", 42));
     }
 
     public void testStructuredAnalysis() throws Exception {
@@ -123,12 +135,12 @@ public class SignificantTermsIT extends ESIntegTestCase {
                 .actionGet();
         assertSearchResponse(response);
         SignificantTerms topTerms = response.getAggregations().get("mySignificantTerms");
-        String topCategory = (String) topTerms.getBuckets().iterator().next().getKey();
-        assertTrue(topCategory.equals(SNOWBOARDING_CATEGORY));
+        Number topCategory = (Number) topTerms.getBuckets().iterator().next().getKey();
+        assertTrue(topCategory.equals(Long.valueOf(SNOWBOARDING_CATEGORY)));
     }
 
     public void testStructuredAnalysisWithIncludeExclude() throws Exception {
-        String[] excludeTerms = { MUSIC_CATEGORY };
+        long[] excludeTerms = { MUSIC_CATEGORY };
         SearchResponse response = client().prepareSearch("test")
                 .setSearchType(SearchType.QUERY_AND_FETCH)
                 .setQuery(new TermQueryBuilder("_all", "paul"))
@@ -139,8 +151,8 @@ public class SignificantTermsIT extends ESIntegTestCase {
                 .actionGet();
         assertSearchResponse(response);
         SignificantTerms topTerms = response.getAggregations().get("mySignificantTerms");
-        String topCategory = topTerms.getBuckets().iterator().next().getKeyAsString();
-        assertTrue(topCategory.equals(OTHER_CATEGORY));
+        Number topCategory = (Number) topTerms.getBuckets().iterator().next().getKey();
+        assertTrue(topCategory.equals(Long.valueOf(OTHER_CATEGORY)));
     }
 
     public void testIncludeExclude() throws Exception {
@@ -422,4 +434,18 @@ public class SignificantTermsIT extends ESIntegTestCase {
         SignificantTerms topTerms = response.getAggregations().get("mySignificantTerms");
         checkExpectedStringTermsFound(topTerms);
     }
+
+    public void testFailIfFieldNotIndexed() {
+        SearchPhaseExecutionException e = expectThrows(SearchPhaseExecutionException.class,
+                () -> client().prepareSearch("test_not_indexed").addAggregation(
+                        significantTerms("mySignificantTerms").field("my_keyword")).get());
+        assertThat(e.toString(),
+                containsString("Cannot search on field [my_keyword] since it is not indexed."));
+
+        e = expectThrows(SearchPhaseExecutionException.class,
+                () -> client().prepareSearch("test_not_indexed").addAggregation(
+                        significantTerms("mySignificantTerms").field("my_long")).get());
+        assertThat(e.toString(),
+                containsString("Cannot search on field [my_long] since it is not indexed."));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
index 67e6d209ad5..0467992002f 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
@@ -97,7 +97,7 @@ public class SignificantTermsSignificanceScoreIT extends ESIntegTestCase {
     }
 
     public void testPlugin() throws Exception {
-        String type = randomBoolean() ? "text" : "keyword";
+        String type = randomBoolean() ? "text" : "long";
         String settings = "{\"index.number_of_shards\": 1, \"index.number_of_replicas\": 0}";
         SharedSignificantTermsTestMethods.index01Docs(type, settings, this);
         SearchResponse response = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE)
@@ -250,7 +250,7 @@ public class SignificantTermsSignificanceScoreIT extends ESIntegTestCase {
     }
 
     public void testXContentResponse() throws Exception {
-        String type = randomBoolean() ? "text" : "keyword";
+        String type = false || randomBoolean() ? "text" : "long";
         String settings = "{\"index.number_of_shards\": 1, \"index.number_of_replicas\": 0}";
         SharedSignificantTermsTestMethods.index01Docs(type, settings, this);
         SearchResponse response = client().prepareSearch(INDEX_NAME).setTypes(DOC_TYPE)
@@ -272,7 +272,12 @@ public class SignificantTermsSignificanceScoreIT extends ESIntegTestCase {
 
         XContentBuilder responseBuilder = XContentFactory.jsonBuilder();
         classes.toXContent(responseBuilder, null);
-        String result = "\"class\"{\"doc_count_error_upper_bound\":0,\"sum_other_doc_count\":0,\"buckets\":[{\"key\":\"0\",\"doc_count\":4,\"sig_terms\":{\"doc_count\":4,\"buckets\":[{\"key\":\"0\",\"doc_count\":4,\"score\":0.39999999999999997,\"bg_count\":5}]}},{\"key\":\"1\",\"doc_count\":3,\"sig_terms\":{\"doc_count\":3,\"buckets\":[{\"key\":\"1\",\"doc_count\":3,\"score\":0.75,\"bg_count\":4}]}}]}";
+        String result = null;
+        if (type.equals("long")) {
+            result = "\"class\"{\"doc_count_error_upper_bound\":0,\"sum_other_doc_count\":0,\"buckets\":[{\"key\":\"0\",\"doc_count\":4,\"sig_terms\":{\"doc_count\":4,\"buckets\":[{\"key\":0,\"doc_count\":4,\"score\":0.39999999999999997,\"bg_count\":5}]}},{\"key\":\"1\",\"doc_count\":3,\"sig_terms\":{\"doc_count\":3,\"buckets\":[{\"key\":1,\"doc_count\":3,\"score\":0.75,\"bg_count\":4}]}}]}";
+        } else {
+            result = "\"class\"{\"doc_count_error_upper_bound\":0,\"sum_other_doc_count\":0,\"buckets\":[{\"key\":\"0\",\"doc_count\":4,\"sig_terms\":{\"doc_count\":4,\"buckets\":[{\"key\":\"0\",\"doc_count\":4,\"score\":0.39999999999999997,\"bg_count\":5}]}},{\"key\":\"1\",\"doc_count\":3,\"sig_terms\":{\"doc_count\":3,\"buckets\":[{\"key\":\"1\",\"doc_count\":3,\"score\":0.75,\"bg_count\":4}]}}]}";
+        }
         assertThat(responseBuilder.string(), equalTo(result));
 
     }
@@ -321,7 +326,7 @@ public class SignificantTermsSignificanceScoreIT extends ESIntegTestCase {
     }
 
     public void testBackgroundVsSeparateSet() throws Exception {
-        String type = randomBoolean() ? "text" : "keyword";
+        String type = randomBoolean() ? "text" : "long";
         String settings = "{\"index.number_of_shards\": 1, \"index.number_of_replicas\": 0}";
         SharedSignificantTermsTestMethods.index01Docs(type, settings, this);
         testBackgroundVsSeparateSet(new MutualInformation(true, true), new MutualInformation(true, false));
@@ -448,7 +453,7 @@ public class SignificantTermsSignificanceScoreIT extends ESIntegTestCase {
     }
 
     public void testScriptScore() throws ExecutionException, InterruptedException, IOException {
-        indexRandomFrequencies01(randomBoolean() ? "text" : "keyword");
+        indexRandomFrequencies01(randomBoolean() ? "text" : "long");
         ScriptHeuristic scriptHeuristic = getScriptSignificanceHeuristic();
         ensureYellow();
         SearchResponse response = client().prepareSearch(INDEX_NAME)
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java
index 408a9137eb8..dad55a20828 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsShardMinDocCountIT.java
@@ -53,7 +53,7 @@ public class TermsShardMinDocCountIT extends ESIntegTestCase {
     public void testShardMinDocCountSignificantTermsTest() throws Exception {
         String textMappings;
         if (randomBoolean()) {
-            textMappings = "type=keyword";
+            textMappings = "type=long";
         } else {
             textMappings = "type=text,fielddata=true";
         }
diff --git a/docs/reference/migration/migrate_5_0/aggregations.asciidoc b/docs/reference/migration/migrate_5_0/aggregations.asciidoc
index 32a8bbd35d2..60c1060f159 100644
--- a/docs/reference/migration/migrate_5_0/aggregations.asciidoc
+++ b/docs/reference/migration/migrate_5_0/aggregations.asciidoc
@@ -5,9 +5,11 @@
 
 Numeric fields have been refactored to use a different data structure that
 performs better for range queries. However, since this data structure does
-not record document frequencies, numeric fields can no longer be used for
-significant terms aggregations. It is recommended to use <<keyword,`keyword`>>
-fields instead, either directly or through a <<multi-fields,multi-field>>
-if the numeric representation is still needed for sorting, range queries or
-numeric aggregations like
+not record document frequencies, numeric fields need to fall back to running
+queries in order to estimate the number of matching documents in the
+background set, which may incur a performance degradation.
+
+It is recommended to use <<keyword,`keyword`>> fields instead, either directly
+or through a <<multi-fields,multi-field>> if the numeric representation is
+still needed for sorting, range queries or numeric aggregations like
 <<search-aggregations-metrics-stats-aggregation,`stats` aggregations>>.

From d790c7663657c1474189a297ab97f30b7625fe9d Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Wed, 11 May 2016 11:19:18 -0400
Subject: [PATCH 0189/1311] painless: _score as double, not float

---
 .../src/main/java/org/elasticsearch/painless/Analyzer.java | 4 ++--
 .../src/main/java/org/elasticsearch/painless/Writer.java   | 7 ++++---
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
index f374bccb54c..07dc7df19db 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
@@ -108,8 +108,8 @@ class Analyzer extends PainlessParserBaseVisitor<Void> {
         //
         // loop counter to catch runaway scripts. internal use only.
         metadata.loopCounterSlot = utility.addVariable(null, "#loop", definition.intType).slot;
-        // document's score as a read-only float.
-        metadata.scoreValueSlot = utility.addVariable(null, "_score", definition.floatType).slot;
+        // document's score as a read-only double.
+        metadata.scoreValueSlot = utility.addVariable(null, "_score", definition.doubleType).slot;
         // ctx map set by executable scripts as a read-only map.
         metadata.ctxValueSlot = utility.addVariable(null, "ctx", definition.smapType).slot;
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
index 20b609fa8c7..9b3b6378dea 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
@@ -152,15 +152,16 @@ class Writer extends PainlessParserBaseVisitor<Void> {
     private void writeExecute() {
         if (metadata.scoreValueUsed) {
             // if the _score value is used, we do this once:
-            //   float _score = scorer.score();
+            //   final double _score = scorer.score();
             execute.visitVarInsn(Opcodes.ALOAD, metadata.scorerValueSlot);
             execute.invokeVirtual(WriterConstants.SCORER_TYPE, WriterConstants.SCORER_SCORE);
-            execute.visitVarInsn(Opcodes.FSTORE, metadata.scoreValueSlot);
+            execute.visitInsn(Opcodes.F2D);
+            execute.visitVarInsn(Opcodes.DSTORE, metadata.scoreValueSlot);
         }
         
         if (metadata.ctxValueUsed) {
             // if the _ctx value is used, we do this once:
-            //   Map<String,Object> ctx = input.get("ctx");
+            //   final Map<String,Object> ctx = input.get("ctx");
             execute.visitVarInsn(Opcodes.ALOAD, metadata.inputValueSlot);
             execute.push("ctx");
             execute.invokeInterface(WriterConstants.MAP_TYPE, WriterConstants.MAP_GET);

From 2764b47d22382ec751372bbf2611a9cedf22bae7 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Wed, 11 May 2016 09:32:30 -0700
Subject: [PATCH 0190/1311] Give long priority over float in Def math calcs. 
 Promotion of types still works correclty.

---
 .../java/org/elasticsearch/painless/Def.java  | 192 ++++++++++--------
 1 file changed, 106 insertions(+), 86 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 5b4dcc01bdc..4eb71a7e988 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -36,27 +36,27 @@ import java.util.stream.Stream;
 /**
  * Support for dynamic type (def).
  * <p>
- * Dynamic types can invoke methods, load/store fields, and be passed as parameters to operators without 
- * compile-time type information. 
+ * Dynamic types can invoke methods, load/store fields, and be passed as parameters to operators without
+ * compile-time type information.
  * <p>
  * Dynamic methods, loads, stores, and array/list/map load/stores involve locating the appropriate field
  * or method depending on the receiver's class. For these, we emit an {@code invokedynamic} instruction that,
  * for each new  type encountered will query a corresponding {@code lookupXXX} method to retrieve the appropriate
- * method. In most cases, the {@code lookupXXX} methods here will only be called once for a given call site, because 
- * caching ({@link DynamicCallSite}) generally works: usually all objects at any call site will be consistently 
- * the same type (or just a few types).  In extreme cases, if there is type explosion, they may be called every 
+ * method. In most cases, the {@code lookupXXX} methods here will only be called once for a given call site, because
+ * caching ({@link DynamicCallSite}) generally works: usually all objects at any call site will be consistently
+ * the same type (or just a few types).  In extreme cases, if there is type explosion, they may be called every
  * single time, but simplicity is still more valuable than performance in this code.
  */
 public class Def {
 
-    /** 
+    /**
      * Looks up handle for a dynamic method call.
      * <p>
      * A dynamic method call for variable {@code x} of type {@code def} looks like:
      * {@code x.method(args...)}
      * <p>
-     * This method traverses {@code recieverClass}'s class hierarchy (including interfaces) 
-     * until it finds a matching whitelisted method. If one is not found, it throws an exception. 
+     * This method traverses {@code recieverClass}'s class hierarchy (including interfaces)
+     * until it finds a matching whitelisted method. If one is not found, it throws an exception.
      * Otherwise it returns a handle to the matching method.
      * <p>
      * @param receiverClass Class of the object to invoke the method on.
@@ -117,7 +117,7 @@ public class Def {
             throw new AssertionError(e);
         }
     }
-    
+
     // TODO: Once Java has a factory for those in java.lang.invoke.MethodHandles, use it:
 
     /** Helper class for isolating MethodHandles and methods to get the length of arrays
@@ -126,7 +126,7 @@ public class Def {
      */
     private static final class ArrayLengthHelper {
       private ArrayLengthHelper() {}
-    
+
       private static final Lookup PRIV_LOOKUP = MethodHandles.lookup();
       private static final Map<Class<?>,MethodHandle> ARRAY_TYPE_MH_MAPPING = Collections.unmodifiableMap(
         Stream.of(boolean[].class, byte[].class, short[].class, int[].class, long[].class,
@@ -140,7 +140,7 @@ public class Def {
           }))
       );
       private static final MethodHandle OBJECT_ARRAY_MH = ARRAY_TYPE_MH_MAPPING.get(Object[].class);
-     
+
       static int getArrayLength(boolean[] array) { return array.length; }
       static int getArrayLength(byte[] array) { return array.length; }
       static int getArrayLength(short[] array) { return array.length; }
@@ -150,7 +150,7 @@ public class Def {
       static int getArrayLength(float[] array) { return array.length; }
       static int getArrayLength(double[] array) { return array.length; }
       static int getArrayLength(Object[] array) { return array.length; }
-      
+
       public static MethodHandle arrayLengthGetter(Class<?> arrayType) {
         if (!arrayType.isArray()) {
           throw new IllegalArgumentException("type must be an array");
@@ -160,13 +160,13 @@ public class Def {
             OBJECT_ARRAY_MH.asType(OBJECT_ARRAY_MH.type().changeParameterType(0, arrayType));
       }
     }
-    
+
     /** Returns an array length getter MethodHandle for the given array type */
     public static MethodHandle arrayLengthGetter(Class<?> arrayType) {
       return ArrayLengthHelper.arrayLengthGetter(arrayType);
     }
-    
-    /** 
+
+    /**
      * Looks up handle for a dynamic field getter (field load)
      * <p>
      * A dynamic field load for variable {@code x} of type {@code def} looks like:
@@ -182,8 +182,8 @@ public class Def {
      *   <li>The value in a list at element {@code field} (integer) when the receiver is a List.
      * </ul>
      * <p>
-     * This method traverses {@code recieverClass}'s class hierarchy (including interfaces) 
-     * until it finds a matching whitelisted getter. If one is not found, it throws an exception. 
+     * This method traverses {@code recieverClass}'s class hierarchy (including interfaces)
+     * until it finds a matching whitelisted getter. If one is not found, it throws an exception.
      * Otherwise it returns a handle to the matching getter.
      * <p>
      * @param receiverClass Class of the object to retrieve the field from.
@@ -229,17 +229,17 @@ public class Def {
             // parsing the same integer millions of times!
             try {
                 int index = Integer.parseInt(name);
-                return MethodHandles.insertArguments(LIST_GET, 1, index);            
+                return MethodHandles.insertArguments(LIST_GET, 1, index);
             } catch (NumberFormatException exception) {
                 throw new IllegalArgumentException( "Illegal list shortcut value [" + name + "].");
             }
         }
-        
+
         throw new IllegalArgumentException("Unable to find dynamic field [" + name + "] " +
                                            "for class [" + receiverClass.getCanonicalName() + "].");
     }
-    
-    /** 
+
+    /**
      * Looks up handle for a dynamic field setter (field store)
      * <p>
      * A dynamic field store for variable {@code x} of type {@code def} looks like:
@@ -253,8 +253,8 @@ public class Def {
      *   <li>The value in a list at element {@code field} (integer) when the receiver is a List.
      * </ul>
      * <p>
-     * This method traverses {@code recieverClass}'s class hierarchy (including interfaces) 
-     * until it finds a matching whitelisted setter. If one is not found, it throws an exception. 
+     * This method traverses {@code recieverClass}'s class hierarchy (including interfaces)
+     * until it finds a matching whitelisted setter. If one is not found, it throws an exception.
      * Otherwise it returns a handle to the matching setter.
      * <p>
      * @param receiverClass Class of the object to retrieve the field from.
@@ -297,12 +297,12 @@ public class Def {
             // parsing the same integer millions of times!
             try {
                 int index = Integer.parseInt(name);
-                return MethodHandles.insertArguments(LIST_SET, 1, index);            
+                return MethodHandles.insertArguments(LIST_SET, 1, index);
             } catch (NumberFormatException exception) {
                 throw new IllegalArgumentException( "Illegal list shortcut value [" + name + "].");
             }
         }
-        
+
         throw new IllegalArgumentException("Unable to find dynamic field [" + name + "] " +
                                            "for class [" + receiverClass.getCanonicalName() + "].");
     }
@@ -325,7 +325,7 @@ public class Def {
         throw new IllegalArgumentException("Attempting to address a non-array type " +
                                            "[" + receiverClass.getCanonicalName() + "] as an array.");
     }
-   
+
     /**
      * Returns a method handle to do an array load.
      * @param receiverClass Class of the array to load the value from
@@ -382,6 +382,8 @@ public class Def {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double) {
                     return ((Number)left).doubleValue() * ((Number)right).doubleValue();
+                } else if (left instanceof Long && right instanceof Long) {
+                    return ((Number)left).longValue() * ((Number)right).longValue();
                 } else if (left instanceof Float || right instanceof Float) {
                     return ((Number)left).floatValue() * ((Number)right).floatValue();
                 } else if (left instanceof Long || right instanceof Long) {
@@ -392,10 +394,10 @@ public class Def {
             } else if (right instanceof Character) {
                 if (left instanceof Double) {
                     return ((Number)left).doubleValue() * (char)right;
-                } else if (left instanceof Float) {
-                    return ((Number)left).floatValue() * (char)right;
                 } else if (left instanceof Long) {
                     return ((Number)left).longValue() * (char)right;
+                } else if (left instanceof Float) {
+                    return ((Number)left).floatValue() * (char)right;
                 } else {
                     return ((Number)left).intValue() * (char)right;
                 }
@@ -404,10 +406,10 @@ public class Def {
             if (right instanceof Number) {
                 if (right instanceof Double) {
                     return (char)left * ((Number)right).doubleValue();
-                } else if (right instanceof Float) {
-                    return (char)left * ((Number)right).floatValue();
                 } else if (right instanceof Long) {
                     return (char)left * ((Number)right).longValue();
+                } else if (right instanceof Float) {
+                    return (char)left * ((Number)right).floatValue();
                 } else {
                     return (char)left * ((Number)right).intValue();
                 }
@@ -425,6 +427,8 @@ public class Def {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double) {
                     return ((Number)left).doubleValue() / ((Number)right).doubleValue();
+                } else if (left instanceof Long && right instanceof Long) {
+                    return ((Number)left).longValue() / ((Number)right).longValue();
                 } else if (left instanceof Float || right instanceof Float) {
                     return ((Number)left).floatValue() / ((Number)right).floatValue();
                 } else if (left instanceof Long || right instanceof Long) {
@@ -435,10 +439,10 @@ public class Def {
             } else if (right instanceof Character) {
                 if (left instanceof Double) {
                     return ((Number)left).doubleValue() / (char)right;
-                } else if (left instanceof Float) {
-                    return ((Number)left).floatValue() / (char)right;
                 } else if (left instanceof Long) {
                     return ((Number)left).longValue() / (char)right;
+                } else if (left instanceof Float) {
+                    return ((Number)left).floatValue() / (char)right;
                 } else {
                     return ((Number)left).intValue() / (char)right;
                 }
@@ -447,10 +451,10 @@ public class Def {
             if (right instanceof Number) {
                 if (right instanceof Double) {
                     return (char)left / ((Number)right).doubleValue();
-                } else if (right instanceof Float) {
-                    return (char)left / ((Number)right).floatValue();
                 } else if (right instanceof Long) {
                     return (char)left / ((Number)right).longValue();
+                } else if (right instanceof Float) {
+                    return (char)left / ((Number)right).floatValue();
                 } else {
                     return (char)left / ((Number)right).intValue();
                 }
@@ -468,6 +472,8 @@ public class Def {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double) {
                     return ((Number)left).doubleValue() % ((Number)right).doubleValue();
+                } else if (left instanceof Long && right instanceof Long) {
+                    return ((Number)left).longValue() % ((Number)right).longValue();
                 } else if (left instanceof Float || right instanceof Float) {
                     return ((Number)left).floatValue() % ((Number)right).floatValue();
                 } else if (left instanceof Long || right instanceof Long) {
@@ -478,10 +484,10 @@ public class Def {
             } else if (right instanceof Character) {
                 if (left instanceof Double) {
                     return ((Number)left).doubleValue() % (char)right;
-                } else if (left instanceof Float) {
-                    return ((Number)left).floatValue() % (char)right;
                 } else if (left instanceof Long) {
                     return ((Number)left).longValue() % (char)right;
+                } else if (left instanceof Float) {
+                    return ((Number)left).floatValue() % (char)right;
                 } else {
                     return ((Number)left).intValue() % (char)right;
                 }
@@ -490,10 +496,10 @@ public class Def {
             if (right instanceof Number) {
                 if (right instanceof Double) {
                     return (char)left % ((Number)right).doubleValue();
-                } else if (right instanceof Float) {
-                    return (char)left % ((Number)right).floatValue();
                 } else if (right instanceof Long) {
                     return (char)left % ((Number)right).longValue();
+                } else if (right instanceof Float) {
+                    return (char)left % ((Number)right).floatValue();
                 } else {
                     return (char)left % ((Number)right).intValue();
                 }
@@ -513,6 +519,8 @@ public class Def {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double) {
                     return ((Number)left).doubleValue() + ((Number)right).doubleValue();
+                } else if (left instanceof Long && right instanceof Long) {
+                    return ((Number)left).longValue() + ((Number)right).longValue();
                 } else if (left instanceof Float || right instanceof Float) {
                     return ((Number)left).floatValue() + ((Number)right).floatValue();
                 } else if (left instanceof Long || right instanceof Long) {
@@ -523,10 +531,10 @@ public class Def {
             } else if (right instanceof Character) {
                 if (left instanceof Double) {
                     return ((Number)left).doubleValue() + (char)right;
-                } else if (left instanceof Float) {
-                    return ((Number)left).floatValue() + (char)right;
                 } else if (left instanceof Long) {
                     return ((Number)left).longValue() + (char)right;
+                } else if (left instanceof Float) {
+                    return ((Number)left).floatValue() + (char)right;
                 } else {
                     return ((Number)left).intValue() + (char)right;
                 }
@@ -535,10 +543,10 @@ public class Def {
             if (right instanceof Number) {
                 if (right instanceof Double) {
                     return (char)left + ((Number)right).doubleValue();
-                } else if (right instanceof Float) {
-                    return (char)left + ((Number)right).floatValue();
                 } else if (right instanceof Long) {
                     return (char)left + ((Number)right).longValue();
+                } else if (right instanceof Float) {
+                    return (char)left + ((Number)right).floatValue();
                 } else {
                     return (char)left + ((Number)right).intValue();
                 }
@@ -556,6 +564,8 @@ public class Def {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double) {
                     return ((Number)left).doubleValue() - ((Number)right).doubleValue();
+                } else if (left instanceof Long && right instanceof Long) {
+                    return ((Number)left).longValue() - ((Number)right).longValue();
                 } else if (left instanceof Float || right instanceof Float) {
                     return ((Number)left).floatValue() - ((Number)right).floatValue();
                 } else if (left instanceof Long || right instanceof Long) {
@@ -566,10 +576,10 @@ public class Def {
             } else if (right instanceof Character) {
                 if (left instanceof Double) {
                     return ((Number)left).doubleValue() - (char)right;
-                } else if (left instanceof Float) {
-                    return ((Number)left).floatValue() - (char)right;
                 } else if (left instanceof Long) {
                     return ((Number)left).longValue() - (char)right;
+                } else if (left instanceof Float) {
+                    return ((Number)left).floatValue() - (char)right;
                 } else {
                     return ((Number)left).intValue() - (char)right;
                 }
@@ -578,10 +588,10 @@ public class Def {
             if (right instanceof Number) {
                 if (right instanceof Double) {
                     return (char)left - ((Number)right).doubleValue();
-                } else if (right instanceof Float) {
-                    return (char)left - ((Number)right).floatValue();
                 } else if (right instanceof Long) {
                     return (char)left - ((Number)right).longValue();
+                } else if (right instanceof Float) {
+                    return (char)left - ((Number)right).floatValue();
                 } else {
                     return (char)left - ((Number)right).intValue();
                 }
@@ -598,14 +608,14 @@ public class Def {
         if (left instanceof Number) {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double ||
-                        left instanceof Float || right instanceof Float ||
-                        left instanceof Long || right instanceof Long) {
+                    left instanceof Long || right instanceof Long ||
+                    left instanceof Float || right instanceof Float) {
                     return ((Number)left).longValue() << ((Number)right).longValue();
                 } else {
                     return ((Number)left).intValue() << ((Number)right).intValue();
                 }
             } else if (right instanceof Character) {
-                if (left instanceof Double || left instanceof Float || left instanceof Long) {
+                if (left instanceof Double ||  left instanceof Long || left instanceof Float) {
                     return ((Number)left).longValue() << (char)right;
                 } else {
                     return ((Number)left).intValue() << (char)right;
@@ -613,7 +623,7 @@ public class Def {
             }
         } else if (left instanceof Character) {
             if (right instanceof Number) {
-                if (right instanceof Double || right instanceof Float || right instanceof Long) {
+                if (right instanceof Double || right instanceof Long || right instanceof Float) {
                     return (long)(char)left << ((Number)right).longValue();
                 } else {
                     return (char)left << ((Number)right).intValue();
@@ -631,14 +641,14 @@ public class Def {
         if (left instanceof Number) {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double ||
-                        left instanceof Float || right instanceof Float ||
-                        left instanceof Long || right instanceof Long) {
+                    left instanceof Long || right instanceof Long ||
+                    left instanceof Float || right instanceof Float) {
                     return ((Number)left).longValue() >> ((Number)right).longValue();
                 } else {
                     return ((Number)left).intValue() >> ((Number)right).intValue();
                 }
             } else if (right instanceof Character) {
-                if (left instanceof Double || left instanceof Float || left instanceof Long) {
+                if (left instanceof Double || left instanceof Long || left instanceof Float) {
                     return ((Number)left).longValue() >> (char)right;
                 } else {
                     return ((Number)left).intValue() >> (char)right;
@@ -646,7 +656,7 @@ public class Def {
             }
         } else if (left instanceof Character) {
             if (right instanceof Number) {
-                if (right instanceof Double || right instanceof Float || right instanceof Long) {
+                if (right instanceof Double || right instanceof Long || right instanceof Float) {
                     return (long)(char)left >> ((Number)right).longValue();
                 } else {
                     return (char)left >> ((Number)right).intValue();
@@ -664,14 +674,14 @@ public class Def {
         if (left instanceof Number) {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double ||
-                        left instanceof Float || right instanceof Float ||
-                        left instanceof Long || right instanceof Long) {
+                    left instanceof Long || right instanceof Long ||
+                    left instanceof Float || right instanceof Float) {
                     return ((Number)left).longValue() >>> ((Number)right).longValue();
                 } else {
                     return ((Number)left).intValue() >>> ((Number)right).intValue();
                 }
             } else if (right instanceof Character) {
-                if (left instanceof Double || left instanceof Float || left instanceof Long) {
+                if (left instanceof Double || left instanceof Long || left instanceof Float) {
                     return ((Number)left).longValue() >>> (char)right;
                 } else {
                     return ((Number)left).intValue() >>> (char)right;
@@ -679,7 +689,7 @@ public class Def {
             }
         } else if (left instanceof Character) {
             if (right instanceof Number) {
-                if (right instanceof Double || right instanceof Float || right instanceof Long) {
+                if (right instanceof Double || right instanceof Long || right instanceof Float) {
                     return (long)(char)left >>> ((Number)right).longValue();
                 } else {
                     return (char)left >>> ((Number)right).intValue();
@@ -699,14 +709,14 @@ public class Def {
         } else if (left instanceof Number) {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double ||
-                        left instanceof Float || right instanceof Float ||
-                        left instanceof Long || right instanceof Long) {
+                    left instanceof Long || right instanceof Long ||
+                    left instanceof Float || right instanceof Float) {
                     return ((Number)left).longValue() & ((Number)right).longValue();
                 } else {
                     return ((Number)left).intValue() & ((Number)right).intValue();
                 }
             } else if (right instanceof Character) {
-                if (left instanceof Double || left instanceof Float || left instanceof Long) {
+                if (left instanceof Double || left instanceof Long || left instanceof Float) {
                     return ((Number)left).longValue() & (char)right;
                 } else {
                     return ((Number)left).intValue() & (char)right;
@@ -714,7 +724,7 @@ public class Def {
             }
         } else if (left instanceof Character) {
             if (right instanceof Number) {
-                if (right instanceof Double || right instanceof Float || right instanceof Long) {
+                if (right instanceof Double || right instanceof Long || right instanceof Float) {
                     return (char)left & ((Number)right).longValue();
                 } else {
                     return (char)left & ((Number)right).intValue();
@@ -734,14 +744,14 @@ public class Def {
         } else if (left instanceof Number) {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double ||
-                        left instanceof Float || right instanceof Float ||
-                        left instanceof Long || right instanceof Long) {
+                    left instanceof Long || right instanceof Long ||
+                    left instanceof Float || right instanceof Float) {
                     return ((Number)left).longValue() ^ ((Number)right).longValue();
                 } else {
                     return ((Number)left).intValue() ^ ((Number)right).intValue();
                 }
             } else if (right instanceof Character) {
-                if (left instanceof Double || left instanceof Float || left instanceof Long) {
+                if (left instanceof Double || left instanceof Long || left instanceof Float) {
                     return ((Number)left).longValue() ^ (char)right;
                 } else {
                     return ((Number)left).intValue() ^ (char)right;
@@ -749,7 +759,7 @@ public class Def {
             }
         } else if (left instanceof Character) {
             if (right instanceof Number) {
-                if (right instanceof Double || right instanceof Float || right instanceof Long) {
+                if (right instanceof Double || right instanceof Long || right instanceof Float) {
                     return (char)left ^ ((Number)right).longValue();
                 } else {
                     return (char)left ^ ((Number)right).intValue();
@@ -769,14 +779,14 @@ public class Def {
         } else if (left instanceof Number) {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double ||
-                        left instanceof Float || right instanceof Float ||
-                        left instanceof Long || right instanceof Long) {
+                    left instanceof Long || right instanceof Long ||
+                    left instanceof Float || right instanceof Float) {
                     return ((Number)left).longValue() | ((Number)right).longValue();
                 } else {
                     return ((Number)left).intValue() | ((Number)right).intValue();
                 }
             } else if (right instanceof Character) {
-                if (left instanceof Double || left instanceof Float || left instanceof Long) {
+                if (left instanceof Double || left instanceof Long || left instanceof Float) {
                     return ((Number)left).longValue() | (char)right;
                 } else {
                     return ((Number)left).intValue() | (char)right;
@@ -784,7 +794,7 @@ public class Def {
             }
         } else if (left instanceof Character) {
             if (right instanceof Number) {
-                if (right instanceof Double || right instanceof Float || right instanceof Long) {
+                if (right instanceof Double || right instanceof Long || right instanceof Float) {
                     return (char)left | ((Number)right).longValue();
                 } else {
                     return (char)left | ((Number)right).intValue();
@@ -812,6 +822,8 @@ public class Def {
                 } else if (left instanceof Character) {
                     return (char)left == ((Number)right).doubleValue();
                 }
+            } else if (left instanceof Long && right instanceof Long) {
+                return (long)left == (long)right;
             } else if (left instanceof Float) {
                 if (right instanceof Number) {
                     return (float)left == ((Number)right).floatValue();
@@ -859,6 +871,8 @@ public class Def {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double) {
                     return ((Number)left).doubleValue() < ((Number)right).doubleValue();
+                } else if (left instanceof Long && right instanceof Long) {
+                    return ((Number)left).longValue() < ((Number)right).longValue();
                 } else if (left instanceof Float || right instanceof Float) {
                     return ((Number)left).floatValue() < ((Number)right).floatValue();
                 } else if (left instanceof Long || right instanceof Long) {
@@ -869,10 +883,10 @@ public class Def {
             } else if (right instanceof Character) {
                 if (left instanceof Double) {
                     return ((Number)left).doubleValue() < (char)right;
-                } else if (left instanceof Float) {
-                    return ((Number)left).floatValue() < (char)right;
                 } else if (left instanceof Long) {
                     return ((Number)left).longValue() < (char)right;
+                } else if (left instanceof Float) {
+                    return ((Number)left).floatValue() < (char)right;
                 } else {
                     return ((Number)left).intValue() < (char)right;
                 }
@@ -881,10 +895,10 @@ public class Def {
             if (right instanceof Number) {
                 if (right instanceof Double) {
                     return (char)left < ((Number)right).doubleValue();
-                } else if (right instanceof Float) {
-                    return (char)left < ((Number)right).floatValue();
                 } else if (right instanceof Long) {
                     return (char)left < ((Number)right).longValue();
+                } else if (right instanceof Float) {
+                    return (char)left < ((Number)right).floatValue();
                 } else {
                     return (char)left < ((Number)right).intValue();
                 }
@@ -902,6 +916,8 @@ public class Def {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double) {
                     return ((Number)left).doubleValue() <= ((Number)right).doubleValue();
+                } else if (left instanceof Long && right instanceof Long) {
+                    return ((Number)left).longValue() <= ((Number)right).longValue();
                 } else if (left instanceof Float || right instanceof Float) {
                     return ((Number)left).floatValue() <= ((Number)right).floatValue();
                 } else if (left instanceof Long || right instanceof Long) {
@@ -912,10 +928,10 @@ public class Def {
             } else if (right instanceof Character) {
                 if (left instanceof Double) {
                     return ((Number)left).doubleValue() <= (char)right;
-                } else if (left instanceof Float) {
-                    return ((Number)left).floatValue() <= (char)right;
                 } else if (left instanceof Long) {
                     return ((Number)left).longValue() <= (char)right;
+                } else if (left instanceof Float) {
+                    return ((Number)left).floatValue() <= (char)right;
                 } else {
                     return ((Number)left).intValue() <= (char)right;
                 }
@@ -924,10 +940,10 @@ public class Def {
             if (right instanceof Number) {
                 if (right instanceof Double) {
                     return (char)left <= ((Number)right).doubleValue();
-                } else if (right instanceof Float) {
-                    return (char)left <= ((Number)right).floatValue();
                 } else if (right instanceof Long) {
                     return (char)left <= ((Number)right).longValue();
+                } else if (right instanceof Float) {
+                    return (char)left <= ((Number)right).floatValue();
                 } else {
                     return (char)left <= ((Number)right).intValue();
                 }
@@ -945,6 +961,8 @@ public class Def {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double) {
                     return ((Number)left).doubleValue() > ((Number)right).doubleValue();
+                } else if (left instanceof Long && right instanceof Long) {
+                    return ((Number)left).longValue() > ((Number)right).longValue();
                 } else if (left instanceof Float || right instanceof Float) {
                     return ((Number)left).floatValue() > ((Number)right).floatValue();
                 } else if (left instanceof Long || right instanceof Long) {
@@ -955,10 +973,10 @@ public class Def {
             } else if (right instanceof Character) {
                 if (left instanceof Double) {
                     return ((Number)left).doubleValue() > (char)right;
-                } else if (left instanceof Float) {
-                    return ((Number)left).floatValue() > (char)right;
                 } else if (left instanceof Long) {
                     return ((Number)left).longValue() > (char)right;
+                } else if (left instanceof Float) {
+                    return ((Number)left).floatValue() > (char)right;
                 } else {
                     return ((Number)left).intValue() > (char)right;
                 }
@@ -967,10 +985,10 @@ public class Def {
             if (right instanceof Number) {
                 if (right instanceof Double) {
                     return (char)left > ((Number)right).doubleValue();
-                } else if (right instanceof Float) {
-                    return (char)left > ((Number)right).floatValue();
                 } else if (right instanceof Long) {
                     return (char)left > ((Number)right).longValue();
+                } else if (right instanceof Float) {
+                    return (char)left > ((Number)right).floatValue();
                 } else {
                     return (char)left > ((Number)right).intValue();
                 }
@@ -988,6 +1006,8 @@ public class Def {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double) {
                     return ((Number)left).doubleValue() >= ((Number)right).doubleValue();
+                } else if (left instanceof Long && right instanceof Long) {
+                    return ((Number)left).longValue() >= ((Number)right).longValue();
                 } else if (left instanceof Float || right instanceof Float) {
                     return ((Number)left).floatValue() >= ((Number)right).floatValue();
                 } else if (left instanceof Long || right instanceof Long) {
@@ -998,10 +1018,10 @@ public class Def {
             } else if (right instanceof Character) {
                 if (left instanceof Double) {
                     return ((Number)left).doubleValue() >= (char)right;
-                } else if (left instanceof Float) {
-                    return ((Number)left).floatValue() >= (char)right;
                 } else if (left instanceof Long) {
                     return ((Number)left).longValue() >= (char)right;
+                } else if (left instanceof Float) {
+                    return ((Number)left).floatValue() >= (char)right;
                 } else {
                     return ((Number)left).intValue() >= (char)right;
                 }
@@ -1010,10 +1030,10 @@ public class Def {
             if (right instanceof Number) {
                 if (right instanceof Double) {
                     return (char)left >= ((Number)right).doubleValue();
-                } else if (right instanceof Float) {
-                    return (char)left >= ((Number)right).floatValue();
                 } else if (right instanceof Long) {
                     return (char)left >= ((Number)right).longValue();
+                } else if (right instanceof Float) {
+                    return (char)left >= ((Number)right).floatValue();
                 } else {
                     return (char)left >= ((Number)right).intValue();
                 }

From ce4af4be424fd180e4af835fc5419f67202af9c4 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Wed, 11 May 2016 18:37:50 +0200
Subject: [PATCH 0191/1311] Remove dead code.

---
 .../index/mapper/internal/IdFieldMapper.java  | 13 ------------
 .../mapper/internal/TypeFieldMapper.java      | 13 ------------
 .../index/mapper/internal/UidFieldMapper.java | 13 ------------
 .../mapper/internal/VersionFieldMapper.java   | 12 -----------
 .../cardinality/CardinalityAggregator.java    | 20 -------------------
 5 files changed, 71 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
index ba7c377bde8..5f38a4c6a0b 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
@@ -77,19 +77,6 @@ public class IdFieldMapper extends MetadataFieldMapper {
         }
     }
 
-    public static class Builder extends MetadataFieldMapper.Builder<Builder, IdFieldMapper> {
-
-        public Builder(MappedFieldType existing) {
-            super(Defaults.NAME, existing == null ? Defaults.FIELD_TYPE : existing, Defaults.FIELD_TYPE);
-        }
-
-        @Override
-        public IdFieldMapper build(BuilderContext context) {
-            setupFieldType(context);
-            return new IdFieldMapper(fieldType, context.indexSettings());
-        }
-    }
-
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
         public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
index 548e20771c5..0dbdca87b60 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
@@ -73,19 +73,6 @@ public class TypeFieldMapper extends MetadataFieldMapper {
         }
     }
 
-    public static class Builder extends MetadataFieldMapper.Builder<Builder, TypeFieldMapper> {
-
-        public Builder(MappedFieldType existing) {
-            super(Defaults.NAME, existing == null ? Defaults.FIELD_TYPE : existing, Defaults.FIELD_TYPE);
-        }
-
-        @Override
-        public TypeFieldMapper build(BuilderContext context) {
-            fieldType.setName(buildFullName(context));
-            return new TypeFieldMapper(fieldType, context.indexSettings());
-        }
-    }
-
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
         public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
index 354de024b92..133c018cc6a 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
@@ -71,19 +71,6 @@ public class UidFieldMapper extends MetadataFieldMapper {
         }
     }
 
-    public static class Builder extends MetadataFieldMapper.Builder<Builder, UidFieldMapper> {
-
-        public Builder(MappedFieldType existing) {
-            super(Defaults.NAME, existing == null ? Defaults.FIELD_TYPE : existing, Defaults.FIELD_TYPE);
-        }
-
-        @Override
-        public UidFieldMapper build(BuilderContext context) {
-            setupFieldType(context);
-            return new UidFieldMapper(fieldType, defaultFieldType, context.indexSettings());
-        }
-    }
-
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
         public MetadataFieldMapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java
index fbc75940b43..1206a058eac 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java
@@ -54,18 +54,6 @@ public class VersionFieldMapper extends MetadataFieldMapper {
         }
     }
 
-    public static class Builder extends MetadataFieldMapper.Builder<Builder, VersionFieldMapper> {
-
-        public Builder() {
-            super(Defaults.NAME, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE);
-        }
-
-        @Override
-        public VersionFieldMapper build(BuilderContext context) {
-            return new VersionFieldMapper(context.indexSettings());
-        }
-    }
-
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
         public MetadataFieldMapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java
index bb297812277..b5ed80a0022 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java
@@ -302,26 +302,6 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue
 
         public abstract long valueAt(int index);
 
-        /**
-         * Return a {@link MurmurHash3Values} instance that returns each value as its hash.
-         */
-        public static MurmurHash3Values cast(final SortedNumericDocValues values) {
-            return new MurmurHash3Values() {
-                @Override
-                public void setDocument(int docId) {
-                    values.setDocument(docId);
-                }
-                @Override
-                public int count() {
-                    return values.count();
-                }
-                @Override
-                public long valueAt(int index) {
-                    return values.valueAt(index);
-                }
-            };
-        }
-
         /**
          * Return a {@link MurmurHash3Values} instance that computes hashes on the fly for each double value.
          */

From 5189eb41c7a3a862316849f4571d4e5b21ae41fe Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Wed, 11 May 2016 12:54:41 -0400
Subject: [PATCH 0192/1311] Dangling indices are not imported if a tombstone
 for the same index (same name and UUID) exists in the cluster state.  This
 resolves a situation where if an index data folder was copied into a node's
 data directory while the node is running and that index had a tombstone in
 the cluster state, the index would still get imported.

Closes #18250
Closes #18249
---
 .../resources/checkstyle_suppressions.xml     |  1 -
 .../cluster/metadata/IndexGraveyard.java      | 12 +++++++
 .../gateway/DanglingIndicesState.java         | 36 +++++++++++--------
 .../elasticsearch/indices/IndicesService.java |  3 +-
 .../cluster/metadata/IndexGraveyardTests.java | 17 +++++++++
 .../gateway/DanglingIndicesStateTests.java    | 18 ++++++++++
 6 files changed, 71 insertions(+), 16 deletions(-)

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index bebe07612b0..1f5e928c391 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -364,7 +364,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]publish[/\\]PublishClusterStateAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]env[/\\]ESFileStore.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]AsyncShardFetch.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]DanglingIndicesState.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]GatewayAllocator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]GatewayMetaState.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]GatewayService.java" checks="LineLength" />
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
index 07bb941cc86..7831862784e 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
@@ -123,6 +123,18 @@ final public class IndexGraveyard implements MetaData.Custom {
         return tombstones;
     }
 
+    /**
+     * Returns true if the graveyard contains a tombstone for the given index.
+     */
+    public boolean containsIndex(final Index index) {
+        for (Tombstone tombstone : tombstones) {
+            if (tombstone.getIndex().equals(index)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     @Override
     public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
         builder.startArray(TOMBSTONES_FIELD.getPreferredName());
diff --git a/core/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java b/core/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java
index b4d8eeae532..2c652fc1481 100644
--- a/core/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java
+++ b/core/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.gateway;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
+import org.elasticsearch.cluster.metadata.IndexGraveyard;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.component.AbstractComponent;
@@ -68,7 +69,7 @@ public class DanglingIndicesState extends AbstractComponent {
      * Process dangling indices based on the provided meta data, handling cleanup, finding
      * new dangling indices, and allocating outstanding ones.
      */
-    public void processDanglingIndices(MetaData metaData) {
+    public void processDanglingIndices(final MetaData metaData) {
         if (nodeEnv.hasNodeFile() == false) {
             return;
         }
@@ -107,7 +108,7 @@ public class DanglingIndicesState extends AbstractComponent {
      * Finds (@{link #findNewAndAddDanglingIndices}) and adds the new dangling indices
      * to the currently tracked dangling indices.
      */
-    void findNewAndAddDanglingIndices(MetaData metaData) {
+    void findNewAndAddDanglingIndices(final MetaData metaData) {
         danglingIndices.putAll(findNewDanglingIndices(metaData));
     }
 
@@ -116,7 +117,7 @@ public class DanglingIndicesState extends AbstractComponent {
      * that have state on disk, but are not part of the provided meta data, or not detected
      * as dangled already.
      */
-    Map<Index, IndexMetaData> findNewDanglingIndices(MetaData metaData) {
+    Map<Index, IndexMetaData> findNewDanglingIndices(final MetaData metaData) {
         final Set<String> excludeIndexPathIds = new HashSet<>(metaData.indices().size() + danglingIndices.size());
         for (ObjectCursor<IndexMetaData> cursor : metaData.indices().values()) {
             excludeIndexPathIds.add(cursor.value.getIndex().getUUID());
@@ -125,13 +126,18 @@ public class DanglingIndicesState extends AbstractComponent {
         try {
             final List<IndexMetaData> indexMetaDataList = metaStateService.loadIndicesStates(excludeIndexPathIds::contains);
             Map<Index, IndexMetaData> newIndices = new HashMap<>(indexMetaDataList.size());
+            final IndexGraveyard graveyard = metaData.indexGraveyard();
             for (IndexMetaData indexMetaData : indexMetaDataList) {
                 if (metaData.hasIndex(indexMetaData.getIndex().getName())) {
                     logger.warn("[{}] can not be imported as a dangling index, as index with same name already exists in cluster metadata",
                         indexMetaData.getIndex());
+                } else if (graveyard.containsIndex(indexMetaData.getIndex())) {
+                    logger.warn("[{}] can not be imported as a dangling index, as an index with the same name and UUID exist in the " +
+                                "index tombstones.  This situation is likely caused by copying over the data directory for an index " +
+                                "that was previously deleted.", indexMetaData.getIndex());
                 } else {
-                    logger.info("[{}] dangling index, exists on local file system, but not in cluster metadata, auto import to cluster state",
-                        indexMetaData.getIndex());
+                    logger.info("[{}] dangling index exists on local file system, but not in cluster metadata, " +
+                                "auto import to cluster state", indexMetaData.getIndex());
                     newIndices.put(indexMetaData.getIndex(), indexMetaData);
                 }
             }
@@ -151,17 +157,19 @@ public class DanglingIndicesState extends AbstractComponent {
             return;
         }
         try {
-            allocateDangledIndices.allocateDangled(Collections.unmodifiableCollection(new ArrayList<>(danglingIndices.values())), new LocalAllocateDangledIndices.Listener() {
-                @Override
-                public void onResponse(LocalAllocateDangledIndices.AllocateDangledResponse response) {
-                    logger.trace("allocated dangled");
-                }
+            allocateDangledIndices.allocateDangled(Collections.unmodifiableCollection(new ArrayList<>(danglingIndices.values())),
+                new LocalAllocateDangledIndices.Listener() {
+                    @Override
+                    public void onResponse(LocalAllocateDangledIndices.AllocateDangledResponse response) {
+                        logger.trace("allocated dangled");
+                    }
 
-                @Override
-                public void onFailure(Throwable e) {
-                    logger.info("failed to send allocated dangled", e);
+                    @Override
+                    public void onFailure(Throwable e) {
+                        logger.info("failed to send allocated dangled", e);
+                    }
                 }
-            });
+            );
         } catch (Throwable e) {
             logger.warn("failed to send allocate dangled", e);
         }
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index 61aaf551610..9375a11c242 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -527,7 +527,8 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
             try {
                 if (clusterState.metaData().hasIndex(indexName)) {
                     final IndexMetaData index = clusterState.metaData().index(indexName);
-                    throw new IllegalStateException("Can't delete unassigned index store for [" + indexName + "] - it's still part of the cluster state [" + index.getIndexUUID() + "] [" + metaData.getIndexUUID() + "]");
+                    throw new IllegalStateException("Can't delete unassigned index store for [" + indexName + "] - it's still part of " +
+                                                    "the cluster state [" + index.getIndexUUID() + "] [" + metaData.getIndexUUID() + "]");
                 }
                 deleteIndexStore(reason, metaData, clusterState);
             } catch (IOException e) {
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java
index 3f2e6de9256..aec701052fb 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java
@@ -130,6 +130,23 @@ public class IndexGraveyardTests extends ESTestCase {
         assertThat(diff.getRemovedCount(), equalTo(removals.size()));
     }
 
+    public void testContains() {
+        List<Index> indices = new ArrayList<>();
+        final int numIndices = randomIntBetween(1, 5);
+        for (int i = 0; i < numIndices; i++) {
+            indices.add(new Index("idx-" + i, UUIDs.randomBase64UUID()));
+        }
+        final IndexGraveyard.Builder graveyard = IndexGraveyard.builder();
+        for (final Index index : indices) {
+            graveyard.addTombstone(index);
+        }
+        final IndexGraveyard indexGraveyard = graveyard.build();
+        for (final Index index : indices) {
+            assertTrue(indexGraveyard.containsIndex(index));
+        }
+        assertFalse(indexGraveyard.containsIndex(new Index(randomAsciiOfLength(6), UUIDs.randomBase64UUID())));
+    }
+
     public static IndexGraveyard createRandom() {
         final IndexGraveyard.Builder graveyard = IndexGraveyard.builder();
         final int numTombstones = randomIntBetween(0, 4);
diff --git a/core/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java b/core/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java
index 51536375dca..74df2f33fc0 100644
--- a/core/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/DanglingIndicesStateTests.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.gateway;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexGraveyard;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.settings.Settings;
@@ -37,6 +38,7 @@ import static org.hamcrest.Matchers.equalTo;
 /**
  */
 public class DanglingIndicesStateTests extends ESTestCase {
+
     private static Settings indexSettings = Settings.builder()
             .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
             .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
@@ -139,4 +141,20 @@ public class DanglingIndicesStateTests extends ESTestCase {
             assertTrue(danglingState.getDanglingIndices().isEmpty());
         }
     }
+
+    public void testDanglingIndicesNotImportedWhenTombstonePresent() throws Exception {
+        try (NodeEnvironment env = newNodeEnvironment()) {
+            MetaStateService metaStateService = new MetaStateService(Settings.EMPTY, env);
+            DanglingIndicesState danglingState = new DanglingIndicesState(Settings.EMPTY, env, metaStateService, null);
+
+            final Settings.Builder settings = Settings.builder().put(indexSettings).put(IndexMetaData.SETTING_INDEX_UUID, "test1UUID");
+            IndexMetaData dangledIndex = IndexMetaData.builder("test1").settings(settings).build();
+            metaStateService.writeIndex("test_write", dangledIndex);
+
+            final IndexGraveyard graveyard = IndexGraveyard.builder().addTombstone(dangledIndex.getIndex()).build();
+            final MetaData metaData = MetaData.builder().indexGraveyard(graveyard).build();
+            assertThat(danglingState.findNewDanglingIndices(metaData).size(), equalTo(0));
+
+        }
+    }
 }

From 20af482dd7fab78662228837fc0bc71c8843c062 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Wed, 11 May 2016 10:22:40 -0700
Subject: [PATCH 0193/1311] Removed extra branches.  Only check for long first
 where we already know the second type.

---
 .../java/org/elasticsearch/painless/Def.java  | 20 -------------------
 1 file changed, 20 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 4eb71a7e988..e50addfb479 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -382,8 +382,6 @@ public class Def {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double) {
                     return ((Number)left).doubleValue() * ((Number)right).doubleValue();
-                } else if (left instanceof Long && right instanceof Long) {
-                    return ((Number)left).longValue() * ((Number)right).longValue();
                 } else if (left instanceof Float || right instanceof Float) {
                     return ((Number)left).floatValue() * ((Number)right).floatValue();
                 } else if (left instanceof Long || right instanceof Long) {
@@ -427,8 +425,6 @@ public class Def {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double) {
                     return ((Number)left).doubleValue() / ((Number)right).doubleValue();
-                } else if (left instanceof Long && right instanceof Long) {
-                    return ((Number)left).longValue() / ((Number)right).longValue();
                 } else if (left instanceof Float || right instanceof Float) {
                     return ((Number)left).floatValue() / ((Number)right).floatValue();
                 } else if (left instanceof Long || right instanceof Long) {
@@ -472,8 +468,6 @@ public class Def {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double) {
                     return ((Number)left).doubleValue() % ((Number)right).doubleValue();
-                } else if (left instanceof Long && right instanceof Long) {
-                    return ((Number)left).longValue() % ((Number)right).longValue();
                 } else if (left instanceof Float || right instanceof Float) {
                     return ((Number)left).floatValue() % ((Number)right).floatValue();
                 } else if (left instanceof Long || right instanceof Long) {
@@ -519,8 +513,6 @@ public class Def {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double) {
                     return ((Number)left).doubleValue() + ((Number)right).doubleValue();
-                } else if (left instanceof Long && right instanceof Long) {
-                    return ((Number)left).longValue() + ((Number)right).longValue();
                 } else if (left instanceof Float || right instanceof Float) {
                     return ((Number)left).floatValue() + ((Number)right).floatValue();
                 } else if (left instanceof Long || right instanceof Long) {
@@ -564,8 +556,6 @@ public class Def {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double) {
                     return ((Number)left).doubleValue() - ((Number)right).doubleValue();
-                } else if (left instanceof Long && right instanceof Long) {
-                    return ((Number)left).longValue() - ((Number)right).longValue();
                 } else if (left instanceof Float || right instanceof Float) {
                     return ((Number)left).floatValue() - ((Number)right).floatValue();
                 } else if (left instanceof Long || right instanceof Long) {
@@ -822,8 +812,6 @@ public class Def {
                 } else if (left instanceof Character) {
                     return (char)left == ((Number)right).doubleValue();
                 }
-            } else if (left instanceof Long && right instanceof Long) {
-                return (long)left == (long)right;
             } else if (left instanceof Float) {
                 if (right instanceof Number) {
                     return (float)left == ((Number)right).floatValue();
@@ -871,8 +859,6 @@ public class Def {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double) {
                     return ((Number)left).doubleValue() < ((Number)right).doubleValue();
-                } else if (left instanceof Long && right instanceof Long) {
-                    return ((Number)left).longValue() < ((Number)right).longValue();
                 } else if (left instanceof Float || right instanceof Float) {
                     return ((Number)left).floatValue() < ((Number)right).floatValue();
                 } else if (left instanceof Long || right instanceof Long) {
@@ -916,8 +902,6 @@ public class Def {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double) {
                     return ((Number)left).doubleValue() <= ((Number)right).doubleValue();
-                } else if (left instanceof Long && right instanceof Long) {
-                    return ((Number)left).longValue() <= ((Number)right).longValue();
                 } else if (left instanceof Float || right instanceof Float) {
                     return ((Number)left).floatValue() <= ((Number)right).floatValue();
                 } else if (left instanceof Long || right instanceof Long) {
@@ -961,8 +945,6 @@ public class Def {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double) {
                     return ((Number)left).doubleValue() > ((Number)right).doubleValue();
-                } else if (left instanceof Long && right instanceof Long) {
-                    return ((Number)left).longValue() > ((Number)right).longValue();
                 } else if (left instanceof Float || right instanceof Float) {
                     return ((Number)left).floatValue() > ((Number)right).floatValue();
                 } else if (left instanceof Long || right instanceof Long) {
@@ -1006,8 +988,6 @@ public class Def {
             if (right instanceof Number) {
                 if (left instanceof Double || right instanceof Double) {
                     return ((Number)left).doubleValue() >= ((Number)right).doubleValue();
-                } else if (left instanceof Long && right instanceof Long) {
-                    return ((Number)left).longValue() >= ((Number)right).longValue();
                 } else if (left instanceof Float || right instanceof Float) {
                     return ((Number)left).floatValue() >= ((Number)right).floatValue();
                 } else if (left instanceof Long || right instanceof Long) {

From fced8dac7264a8211e1c66d65cd43f1c174dff5f Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Wed, 11 May 2016 15:13:43 -0400
Subject: [PATCH 0194/1311] When CORS is enabled, permit requests from the same
 origin as the request host, as the request is not a cross origin.

Relates #18256
---
 .../http/netty/cors/CorsHandler.java          | 23 +++++++-
 .../http/netty/NettyHttpChannelTests.java     | 52 +++++++++++++++----
 2 files changed, 63 insertions(+), 12 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/http/netty/cors/CorsHandler.java b/core/src/main/java/org/elasticsearch/http/netty/cors/CorsHandler.java
index b04d9013c4f..7ecc9b1fd5d 100644
--- a/core/src/main/java/org/elasticsearch/http/netty/cors/CorsHandler.java
+++ b/core/src/main/java/org/elasticsearch/http/netty/cors/CorsHandler.java
@@ -31,6 +31,7 @@ import org.jboss.netty.handler.codec.http.HttpMethod;
 import org.jboss.netty.handler.codec.http.HttpRequest;
 import org.jboss.netty.handler.codec.http.HttpResponse;
 
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_ALLOW_CREDENTIALS;
@@ -38,6 +39,7 @@ import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTRO
 import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_ALLOW_METHODS;
 import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN;
 import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_MAX_AGE;
+import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.HOST;
 import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.ORIGIN;
 import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.USER_AGENT;
 import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.VARY;
@@ -55,8 +57,9 @@ import static org.jboss.netty.handler.codec.http.HttpResponseStatus.OK;
 public class CorsHandler extends SimpleChannelUpstreamHandler {
 
     public static final String ANY_ORIGIN = "*";
-    private final CorsConfig config;
+    private static Pattern SCHEME_PATTERN = Pattern.compile("^https?://");
 
+    private final CorsConfig config;
     private HttpRequest request;
 
     /**
@@ -96,7 +99,7 @@ public class CorsHandler extends SimpleChannelUpstreamHandler {
             final String originHeaderVal;
             if (config.isAnyOriginSupported()) {
                 originHeaderVal = ANY_ORIGIN;
-            } else if (config.isOriginAllowed(originHeader)) {
+            } else if (config.isOriginAllowed(originHeader) || isSameOrigin(originHeader, request.headers().get(HOST))) {
                 originHeaderVal = originHeader;
             } else {
                 originHeaderVal = null;
@@ -129,6 +132,17 @@ public class CorsHandler extends SimpleChannelUpstreamHandler {
             .addListener(ChannelFutureListener.CLOSE);
     }
 
+    private static boolean isSameOrigin(final String origin, final String host) {
+        if (Strings.isNullOrEmpty(host) == false) {
+            // strip protocol from origin
+            final String originDomain = SCHEME_PATTERN.matcher(origin).replaceFirst("");
+            if (host.equals(originDomain)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     /**
      * This is a non CORS specification feature which enables the setting of preflight
      * response headers that might be required by intermediaries.
@@ -179,6 +193,11 @@ public class CorsHandler extends SimpleChannelUpstreamHandler {
             return true;
         }
 
+        // if the origin is the same as the host of the request, then allow
+        if (isSameOrigin(origin, request.headers().get(HOST))) {
+            return true;
+        }
+
         return config.isOriginAllowed(origin);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java
index ce9051ad189..f809ec85280 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java
@@ -63,8 +63,6 @@ import static org.hamcrest.Matchers.nullValue;
 
 public class NettyHttpChannelTests extends ESTestCase {
 
-    private static final String ORIGIN = "remote-host";
-
     private NetworkService networkService;
     private ThreadPool threadPool;
     private MockBigArrays bigArrays;
@@ -93,34 +91,67 @@ public class NettyHttpChannelTests extends ESTestCase {
         Settings settings = Settings.builder()
                 .put(HttpTransportSettings.SETTING_CORS_ENABLED.getKey(), true)
                 .build();
-        HttpResponse response = execRequestWithCors(settings, ORIGIN);
+        HttpResponse response = execRequestWithCors(settings, "remote-host", "request-host");
         // inspect response and validate
         assertThat(response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN), nullValue());
     }
 
     public void testCorsEnabledWithAllowOrigins() {
-        final String originValue = ORIGIN;
+        final String originValue = "remote-host";
         // create a http transport with CORS enabled and allow origin configured
         Settings settings = Settings.builder()
                 .put(SETTING_CORS_ENABLED.getKey(), true)
                 .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), originValue)
                 .build();
-        HttpResponse response = execRequestWithCors(settings, originValue);
+        HttpResponse response = execRequestWithCors(settings, originValue, "request-host");
         // inspect response and validate
         assertThat(response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
         String allowedOrigins = response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN);
         assertThat(allowedOrigins, is(originValue));
     }
 
+    public void testCorsAllowOriginWithSameHost() {
+        String originValue = "remote-host";
+        String host = "remote-host";
+        // create a http transport with CORS enabled
+        Settings settings = Settings.builder()
+                                .put(SETTING_CORS_ENABLED.getKey(), true)
+                                .build();
+        HttpResponse response = execRequestWithCors(settings, originValue, host);
+        // inspect response and validate
+        assertThat(response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
+        String allowedOrigins = response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN);
+        assertThat(allowedOrigins, is(originValue));
+
+        originValue = "http://" + originValue;
+        response = execRequestWithCors(settings, originValue, host);
+        assertThat(response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
+        allowedOrigins = response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN);
+        assertThat(allowedOrigins, is(originValue));
+
+        originValue = originValue + ":5555";
+        host = host + ":5555";
+        response = execRequestWithCors(settings, originValue, host);
+        assertThat(response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
+        allowedOrigins = response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN);
+        assertThat(allowedOrigins, is(originValue));
+
+        originValue = originValue.replace("http", "https");
+        response = execRequestWithCors(settings, originValue, host);
+        assertThat(response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
+        allowedOrigins = response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN);
+        assertThat(allowedOrigins, is(originValue));
+    }
+
     public void testThatStringLiteralWorksOnMatch() {
-        final String originValue = ORIGIN;
+        final String originValue = "remote-host";
         Settings settings = Settings.builder()
                                 .put(SETTING_CORS_ENABLED.getKey(), true)
                                 .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), originValue)
                                 .put(SETTING_CORS_ALLOW_METHODS.getKey(), "get, options, post")
                                 .put(SETTING_CORS_ALLOW_CREDENTIALS.getKey(), true)
                                 .build();
-        HttpResponse response = execRequestWithCors(settings, originValue);
+        HttpResponse response = execRequestWithCors(settings, originValue, "request-host");
         // inspect response and validate
         assertThat(response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
         String allowedOrigins = response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN);
@@ -134,7 +165,7 @@ public class NettyHttpChannelTests extends ESTestCase {
                                 .put(SETTING_CORS_ENABLED.getKey(), true)
                                 .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), originValue)
                                 .build();
-        HttpResponse response = execRequestWithCors(settings, originValue);
+        HttpResponse response = execRequestWithCors(settings, originValue, "request-host");
         // inspect response and validate
         assertThat(response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN), notNullValue());
         String allowedOrigins = response.headers().get(HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN);
@@ -169,12 +200,13 @@ public class NettyHttpChannelTests extends ESTestCase {
         assertThat(response.headers().get(HttpHeaders.Names.CONTENT_TYPE), equalTo(resp.contentType()));
     }
 
-    private HttpResponse execRequestWithCors(final Settings settings, final String originValue) {
+    private HttpResponse execRequestWithCors(final Settings settings, final String originValue, final String host) {
         // construct request and send it over the transport layer
         httpServerTransport = new NettyHttpServerTransport(settings, networkService, bigArrays, threadPool);
         HttpRequest httpRequest = new TestHttpRequest();
-        httpRequest.headers().add(HttpHeaders.Names.ORIGIN, ORIGIN);
+        httpRequest.headers().add(HttpHeaders.Names.ORIGIN, originValue);
         httpRequest.headers().add(HttpHeaders.Names.USER_AGENT, "Mozilla fake");
+        httpRequest.headers().add(HttpHeaders.Names.HOST, host);
         WriteCapturingChannel writeCapturingChannel = new WriteCapturingChannel();
         NettyHttpRequest request = new NettyHttpRequest(httpRequest, writeCapturingChannel);
 

From 189341da101adf75e542a118021ef618857b59db Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Wed, 11 May 2016 13:06:56 -0400
Subject: [PATCH 0195/1311] CORS handling triggered whether User-Agent is a
 browser or not

This commit ensures that if CORS is enabled, then Origin headers are
checked regardless of whether the request came from a browser or not.
In the past, we only proceeded with CORS checks if the User-Agent was a
browser.
---
 .../http/netty/cors/CorsHandler.java           | 18 +++++++-----------
 .../elasticsearch/rest/support/RestUtils.java  | 11 -----------
 .../http/netty/NettyHttpChannelTests.java      |  2 --
 3 files changed, 7 insertions(+), 24 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/http/netty/cors/CorsHandler.java b/core/src/main/java/org/elasticsearch/http/netty/cors/CorsHandler.java
index 7ecc9b1fd5d..5d42de780b3 100644
--- a/core/src/main/java/org/elasticsearch/http/netty/cors/CorsHandler.java
+++ b/core/src/main/java/org/elasticsearch/http/netty/cors/CorsHandler.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.http.netty.cors;
 
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.rest.support.RestUtils;
 import org.jboss.netty.channel.ChannelFutureListener;
 import org.jboss.netty.channel.ChannelHandlerContext;
 import org.jboss.netty.channel.MessageEvent;
@@ -41,7 +40,6 @@ import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTRO
 import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.ACCESS_CONTROL_MAX_AGE;
 import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.HOST;
 import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.ORIGIN;
-import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.USER_AGENT;
 import static org.jboss.netty.handler.codec.http.HttpHeaders.Names.VARY;
 import static org.jboss.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN;
 import static org.jboss.netty.handler.codec.http.HttpResponseStatus.OK;
@@ -76,15 +74,13 @@ public class CorsHandler extends SimpleChannelUpstreamHandler {
     public void messageReceived(final ChannelHandlerContext ctx, final MessageEvent e) throws Exception {
         if (config.isCorsSupportEnabled() && e.getMessage() instanceof HttpRequest) {
             request = (HttpRequest) e.getMessage();
-            if (RestUtils.isBrowser(request.headers().get(USER_AGENT))) {
-                if (isPreflightRequest(request)) {
-                    handlePreflight(ctx, request);
-                    return;
-                }
-                if (config.isShortCircuit() && !validateOrigin()) {
-                    forbidden(ctx, request);
-                    return;
-                }
+            if (isPreflightRequest(request)) {
+                handlePreflight(ctx, request);
+                return;
+            }
+            if (config.isShortCircuit() && !validateOrigin()) {
+                forbidden(ctx, request);
+                return;
             }
         }
         super.messageReceived(ctx, e);
diff --git a/core/src/main/java/org/elasticsearch/rest/support/RestUtils.java b/core/src/main/java/org/elasticsearch/rest/support/RestUtils.java
index ef7a7ac0edd..7b8bf780719 100644
--- a/core/src/main/java/org/elasticsearch/rest/support/RestUtils.java
+++ b/core/src/main/java/org/elasticsearch/rest/support/RestUtils.java
@@ -41,17 +41,6 @@ public class RestUtils {
         }
     };
 
-    public static boolean isBrowser(@Nullable String userAgent) {
-        if (userAgent == null) {
-            return false;
-        }
-        // chrome, safari, firefox, ie
-        if (userAgent.startsWith("Mozilla")) {
-            return true;
-        }
-        return false;
-    }
-
     public static void decodeQueryString(String s, int fromIndex, Map<String, String> params) {
         if (fromIndex < 0) {
             return;
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java
index f809ec85280..0602e963ed8 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java
@@ -178,7 +178,6 @@ public class NettyHttpChannelTests extends ESTestCase {
         httpServerTransport = new NettyHttpServerTransport(settings, networkService, bigArrays, threadPool);
         HttpRequest httpRequest = new TestHttpRequest();
         httpRequest.headers().add(HttpHeaders.Names.ORIGIN, "remote");
-        httpRequest.headers().add(HttpHeaders.Names.USER_AGENT, "Mozilla fake");
         WriteCapturingChannel writeCapturingChannel = new WriteCapturingChannel();
         NettyHttpRequest request = new NettyHttpRequest(httpRequest, writeCapturingChannel);
 
@@ -205,7 +204,6 @@ public class NettyHttpChannelTests extends ESTestCase {
         httpServerTransport = new NettyHttpServerTransport(settings, networkService, bigArrays, threadPool);
         HttpRequest httpRequest = new TestHttpRequest();
         httpRequest.headers().add(HttpHeaders.Names.ORIGIN, originValue);
-        httpRequest.headers().add(HttpHeaders.Names.USER_AGENT, "Mozilla fake");
         httpRequest.headers().add(HttpHeaders.Names.HOST, host);
         WriteCapturingChannel writeCapturingChannel = new WriteCapturingChannel();
         NettyHttpRequest request = new NettyHttpRequest(httpRequest, writeCapturingChannel);

From 6b4e47bf96e2d3a2253a9509b796be122d897160 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Wed, 11 May 2016 15:39:00 -0400
Subject: [PATCH 0196/1311] this makes aggregations per-document _value fast
 (bypass hash put, hash get, etc) for painless.

but i have no clue how to test it, it seems this feature never worked via REST?

Should we drop the feature instead?
---
 .../elasticsearch/script/ExecutableScript.java   |  8 ++++++++
 .../elasticsearch/script/LeafSearchScript.java   | 12 ++++++++++++
 .../aggregations/support/ValuesSource.java       |  6 +++---
 .../expression/ExpressionSearchScript.java       | 11 +++++++----
 .../org/elasticsearch/painless/Analyzer.java     |  2 ++
 .../elasticsearch/painless/AnalyzerExternal.java |  2 +-
 .../org/elasticsearch/painless/Executable.java   |  2 +-
 .../org/elasticsearch/painless/Metadata.java     |  5 +++++
 .../org/elasticsearch/painless/ScriptImpl.java   | 13 ++++++++++++-
 .../elasticsearch/painless/WriterConstants.java  |  2 +-
 .../painless/ReservedWordTests.java              | 16 ++++++++++++++++
 11 files changed, 68 insertions(+), 11 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/script/ExecutableScript.java b/core/src/main/java/org/elasticsearch/script/ExecutableScript.java
index bdf93acc4de..70f42def216 100644
--- a/core/src/main/java/org/elasticsearch/script/ExecutableScript.java
+++ b/core/src/main/java/org/elasticsearch/script/ExecutableScript.java
@@ -24,6 +24,14 @@ package org.elasticsearch.script;
  */
 public interface ExecutableScript {
 
+    /**
+     * Sets a runtime script parameter.
+     * <p>
+     * Note that this method may be slow, involving put() and get() calls
+     * to a hashmap or similar.
+     * @param name parameter name
+     * @param value parameter value
+     */
     void setNextVar(String name, Object value);
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/script/LeafSearchScript.java b/core/src/main/java/org/elasticsearch/script/LeafSearchScript.java
index d0b82c2d17b..cff0e42e5cf 100644
--- a/core/src/main/java/org/elasticsearch/script/LeafSearchScript.java
+++ b/core/src/main/java/org/elasticsearch/script/LeafSearchScript.java
@@ -31,6 +31,18 @@ public interface LeafSearchScript extends ScorerAware, ExecutableScript {
     void setDocument(int doc);
 
     void setSource(Map<String, Object> source);
+    
+    /**
+     * Sets per-document aggregation {@code _value}.
+     * <p>
+     * The default implementation just calls {@code setNextVar("_value", value)} but
+     * some engines might want to handle this differently for better performance.
+     * <p>
+     * @param value per-document value, typically a String, Long, or Double
+     */
+    default void setNextAggregationValue(Object value) {
+        setNextVar("_value", value);
+    }
 
     float runAsFloat();
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java
index b03bc8d6833..70262096cb0 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java
@@ -329,7 +329,7 @@ public abstract class ValuesSource {
                     resize(longValues.count());
                     script.setDocument(doc);
                     for (int i = 0; i < count(); ++i) {
-                        script.setNextVar("_value", longValues.valueAt(i));
+                        script.setNextAggregationValue(longValues.valueAt(i));
                         values[i] = script.runAsLong();
                     }
                     sort();
@@ -357,7 +357,7 @@ public abstract class ValuesSource {
                     resize(doubleValues.count());
                     script.setDocument(doc);
                     for (int i = 0; i < count(); ++i) {
-                        script.setNextVar("_value", doubleValues.valueAt(i));
+                        script.setNextAggregationValue(doubleValues.valueAt(i));
                         values[i] = script.runAsDouble();
                     }
                     sort();
@@ -474,7 +474,7 @@ public abstract class ValuesSource {
                 grow();
                 for (int i = 0; i < count; ++i) {
                     final BytesRef value = bytesValues.valueAt(i);
-                    script.setNextVar("_value", value.utf8ToString());
+                    script.setNextAggregationValue(value.utf8ToString());
                     values[i].copyChars(script.run().toString());
                 }
                 sort();
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java
index 3944090cef2..cb99c879058 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java
@@ -111,10 +111,7 @@ class ExpressionSearchScript implements SearchScript {
             }
 
             @Override
-            public void setNextVar(String name, Object value) {
-                // this should only be used for the special "_value" variable used in aggregations
-                assert(name.equals("_value"));
-
+            public void setNextAggregationValue(Object value) {
                 // _value isn't used in script if specialValue == null
                 if (specialValue != null) {
                     if (value instanceof Number) {
@@ -124,6 +121,12 @@ class ExpressionSearchScript implements SearchScript {
                     }
                 }
             }
+
+            @Override
+            public void setNextVar(String name, Object value) {
+                // other per-document variables aren't supported yet, even if they are numbers
+                // but we shouldn't encourage this anyway.
+            }
         };
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
index 07dc7df19db..8c911809388 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
@@ -103,6 +103,8 @@ class Analyzer extends PainlessParserBaseVisitor<Void> {
         // doc parameter passed to the script.
         // TODO: currently working as a Map<String,Def>, we can do better?
         metadata.docValueSlot = utility.addVariable(null, "doc", definition.smapType).slot;
+        // aggregation _value parameter passed to the script
+        metadata.aggregationValueSlot = utility.addVariable(null, "_value", definition.defType).slot;
         //
         // reserved words implemented as local variables
         //
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java
index e8e8cb88fef..53b7ab6fd96 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java
@@ -449,7 +449,7 @@ class AnalyzerExternal {
             }
 
             // special cases: reserved words
-            if ("_score".equals(id) || "doc".equals(id) || "ctx".equals(id)) {
+            if ("_score".equals(id) || "_value".equals(id) || "doc".equals(id) || "ctx".equals(id)) {
                 // read-only: don't allow stores to ourself
                 if (varenmd.last && parentemd.storeExpr != null) {
                     throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Variable [" + id + "] is read-only.");
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java
index 7878c72da89..fd060b329cd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java
@@ -49,5 +49,5 @@ public abstract class Executable {
         return definition;
     }
 
-    public abstract Object execute(Map<String, Object> input, Scorer scorer, LeafDocLookup doc);
+    public abstract Object execute(Map<String, Object> input, Scorer scorer, LeafDocLookup doc, Object value);
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java
index 8692ea4a0a1..a8d8d35f2f5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java
@@ -415,6 +415,11 @@ class Metadata {
      * _score from it, if _score will be accessed by the script.
      */
     int scorerValueSlot = -1;
+    
+    /**
+     * Used to determine what slot the _value variable is scored in. 
+     */
+    int aggregationValueSlot = -1;
 
     /**
      * Used to determine what slot the loopCounter variable is stored in.  This is used n the {@link Writer} whenever
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
index 3bcab8799e6..c500ed712fa 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
@@ -59,6 +59,12 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
      */
     private Scorer scorer;
 
+    /**
+     * Current _value for aggregation
+     * @see #setNextAggregationValue(Object)
+     */
+    private Object aggregationValue;
+
     /**
      * Creates a ScriptImpl for the a previously compiled Painless script.
      * @param executable The previously compiled Painless script.
@@ -91,6 +97,11 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
     public void setNextVar(final String name, final Object value) {
         variables.put(name, value);
     }
+    
+    @Override
+    public void setNextAggregationValue(Object value) {
+        this.aggregationValue = value;
+    }
 
     /**
      * Run the script.
@@ -98,7 +109,7 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
      */
     @Override
     public Object run() {
-        return executable.execute(variables, scorer, doc);
+        return executable.execute(variables, scorer, doc, aggregationValue);
     }
 
     /**
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index ec6e4e35b2b..e4effe85bc5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -38,7 +38,7 @@ class WriterConstants {
     final static Type CLASS_TYPE        = Type.getType("L" + CLASS_NAME.replace(".", "/") + ";");
 
     final static Method CONSTRUCTOR = getAsmMethod(void.class, "<init>", Definition.class, String.class, String.class);
-    final static Method EXECUTE     = getAsmMethod(Object.class, "execute", Map.class, Scorer.class, LeafDocLookup.class);
+    final static Method EXECUTE     = getAsmMethod(Object.class, "execute", Map.class, Scorer.class, LeafDocLookup.class, Object.class);
 
     final static Type PAINLESS_ERROR_TYPE = Type.getType(PainlessError.class);
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java
index 46e08d60c2d..dd3c2c67fdb 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java
@@ -77,4 +77,20 @@ public class ReservedWordTests extends ScriptTestCase {
     public void testCtxStoreMap() {
         assertEquals(5, exec("ctx.foo = 5; return ctx.foo;", Collections.singletonMap("ctx", new HashMap<String,Object>())));
     }
+    
+    /** check that we can't declare a variable of _value, its really reserved! */
+    public void testAggregationValueVar() {
+        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+            exec("int _value = 5; return _value;");
+        });
+        assertTrue(expected.getMessage().contains("Variable name [_value] already defined"));
+    }
+    
+    /** check that we can't write to _value, its read-only! */
+    public void testAggregationValueStore() {
+        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+            exec("_value = 5; return _value;");
+        });
+        assertTrue(expected.getMessage().contains("Variable [_value] is read-only"));
+    }
 }

From c5532d3df09a79f5f38e8b3982c7506a1cc64d26 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Wed, 11 May 2016 16:07:08 -0400
Subject: [PATCH 0197/1311] add a rest test for this that seems to work, fix
 the documentation. thanks @s1monw

---
 .../bucket/terms-aggregation.asciidoc         | 10 +++++--
 .../rest-api-spec/test/plan_a/30_search.yaml  | 30 +++++++++++++++++++
 2 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
index 14dec039435..1d9a65a7428 100644
--- a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
@@ -443,7 +443,10 @@ Generating the terms using a script:
     "aggs" : {
         "genders" : {
             "terms" : {
-                "script" : "doc['gender'].value"
+                "script" : { 
+                    "inline": "doc['gender'].value"
+                    "lang": "painless"
+                }
             }
         }
     }
@@ -482,7 +485,10 @@ TIP: for indexed scripts replace the `file` parameter with an `id` parameter.
         "genders" : {
             "terms" : {
                 "field" : "gender",
-                "script" : "'Gender: ' +_value"
+                "script" : { 
+                    "inline" : "'Gender: ' +_value"
+                    "lang" : "painless" 
+                }
             }
         }
     }
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml
index 73d267a3fe5..cd9ed7511d3 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml
@@ -363,3 +363,33 @@
 
     - match: { hits.total: 1 }
     - match: { hits.hits.0.fields.foobar.0: 1 }
+
+---
+
+"Agg _value":
+    - do:
+        index:
+            index: test
+            type: test
+            id: 1
+            body: { "dummy_field": 1 }
+    - do:
+        indices.refresh: {}
+
+
+    - do:
+        index: test
+        search:
+            body:
+                aggs:
+                    value_agg:
+                        terms:
+                            field: dummy_field
+                            script:
+                                lang: painless
+                                inline: "_value + 1"
+
+    - match: { hits.total: 1 }
+    - match: { hits.hits.0._score: 1.0 }
+    - match: { aggregations.value_agg.buckets.0.key: 2 }
+    - match: { aggregations.value_agg.buckets.0.doc_count: 1 }

From 8adfd60079121ad76ffd4df5520207bc5e14bbff Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 11 May 2016 20:05:09 -0400
Subject: [PATCH 0198/1311] Fix Debian package bash requires clause

This commit fixes the Debian package requires clause for bash. For the
RPM it is okay to specify the binary, but for the Debian package the
package name must be specified.
---
 distribution/build.gradle | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/distribution/build.gradle b/distribution/build.gradle
index adbce86ba24..44949cd0604 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -317,7 +317,11 @@ configure(subprojects.findAll { ['deb', 'rpm'].contains(it.name) }) {
     preUninstall file("${scripts}/prerm")
     postUninstall file("${scripts}/postrm")
 
-    requires('/bin/bash')
+    if (project.name == 'rpm') {
+      requires('/bin/bash')
+    } else if (project.name == 'deb') {
+      requires('bash')
+    }
 
     into '/usr/share/elasticsearch'
     fileMode 0644

From 25dd64250b837487d086b81fc8c9823d4c00523c Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Wed, 11 May 2016 21:32:10 -0400
Subject: [PATCH 0199/1311] painless: remove input, support params instead

---
 .../modules/scripting/painless.asciidoc        | 18 +++++++++---------
 .../org/elasticsearch/painless/Analyzer.java   |  6 +++---
 .../org/elasticsearch/painless/Executable.java |  2 +-
 .../org/elasticsearch/painless/Metadata.java   |  2 +-
 .../org/elasticsearch/painless/Writer.java     |  2 +-
 .../painless/BasicExpressionTests.java         |  6 +++---
 .../painless/ScriptEngineTests.java            | 16 ++++++++--------
 .../painless/WhenThingsGoWrongTests.java       |  2 +-
 .../rest-api-spec/test/plan_a/15_update.yaml   |  2 +-
 .../rest-api-spec/test/plan_a/16_update2.yaml  |  8 ++++----
 .../test/plan_a/20_scriptfield.yaml            |  2 +-
 .../test/plan_a/25_script_upsert.yaml          |  6 +++---
 .../rest-api-spec/test/plan_a/30_search.yaml   |  6 +++---
 13 files changed, 39 insertions(+), 39 deletions(-)

diff --git a/docs/reference/modules/scripting/painless.asciidoc b/docs/reference/modules/scripting/painless.asciidoc
index b5300751caf..156705bfe78 100644
--- a/docs/reference/modules/scripting/painless.asciidoc
+++ b/docs/reference/modules/scripting/painless.asciidoc
@@ -72,7 +72,7 @@ PUT hockey/player/_bulk?refresh
 [float]
 === Accessing Doc Values from Painless
 
-All Painless scripts take in a `Map<String,def>` of values called `input`.  Document values can be accessed through another `Map<String,def>` within the `input` variable.
+Document values can be accessed from a `Map<String,def>` named `doc`.
 
 For example, the following script calculates a player's total goals. This example uses a strongly typed `int` and a `for` loop.
 
@@ -85,7 +85,7 @@ GET hockey/_search
       "script_score": {
         "script": {
           "lang": "painless",
-          "inline": "int total = 0; for (int i = 0; i < input.doc['goals'].size(); ++i) { total += input.doc['goals'][i]; } return total;"
+          "inline": "int total = 0; for (int i = 0; i < doc['goals'].length; ++i) { total += doc['goals'][i]; } return total;"
         }
       }
     }
@@ -107,7 +107,7 @@ GET hockey/_search
     "total_goals": {
       "script": {
         "lang": "painless",
-        "inline": "int total = 0; for (int i = 0; i < input.doc['goals'].size(); ++i) { total += input.doc['goals'][i]; } return total;"
+        "inline": "int total = 0; for (int i = 0; i < doc['goals'].length; ++i) { total += doc['goals'][i]; } return total;"
       }
     }
   }
@@ -116,7 +116,7 @@ GET hockey/_search
 // CONSOLE
 
 The following example uses a Painless script to sort the players by their combined first and last names. The names are accessed using
-`input.doc['first'].value` and `input.doc['last'].value`.
+`doc['first'].value` and `doc['last'].value`.
 
 [source,js]
 ----------------------------------------------------------------
@@ -131,7 +131,7 @@ GET hockey/_search
       "order": "asc",
       "script": {
         "lang": "painless",
-        "inline": "input.doc['first'].value + ' ' + input.doc['last'].value"
+        "inline": "doc['first'].value + ' ' + doc['last'].value"
       }
     }
   }
@@ -142,7 +142,7 @@ GET hockey/_search
 [float]
 === Updating Fields with Painless
 
-You can also easily update fields. You access the original source for a field as `input.ctx._source.<field-name>`.
+You can also easily update fields. You access the original source for a field as `ctx._source.<field-name>`.
 
 First, let's look at the source data for a player by submitting the following request:
 
@@ -163,7 +163,7 @@ GET hockey/_search
 ----------------------------------------------------------------
 // CONSOLE
 
-To change player 1's last name to `hockey`, simply set `input.ctx._source.last` to the new value:
+To change player 1's last name to `hockey`, simply set `ctx._source.last` to the new value:
 
 [source,js]
 ----------------------------------------------------------------
@@ -171,7 +171,7 @@ POST hockey/player/1/_update
 {
   "script": {
     "lang": "painless",
-    "inline": "input.ctx._source.last = input.last",
+    "inline": "ctx._source.last = params.last",
     "params": {
       "last": "hockey"
     }
@@ -189,7 +189,7 @@ POST hockey/player/1/_update
 {
   "script": {
     "lang": "painless",
-    "inline": "input.ctx._source.last = input.last input.ctx._source.nick = input.nick",
+    "inline": "ctx._source.last = params.last ctx._source.nick = params.nick",
     "params": {
       "last": "gaudreau",
       "nick": "hockey"
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
index 8c911809388..398e46f403d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
@@ -94,10 +94,10 @@ class Analyzer extends PainlessParserBaseVisitor<Void> {
         utility.incrementScope();
         utility.addVariable(null, "#this", definition.execType);
         //
-        // reserved words parameters.
+        // reserved words.
         //
-        // input map of variables passed to the script. TODO: rename to 'params' since that will be its use
-        metadata.inputValueSlot = utility.addVariable(null, "input", definition.smapType).slot;
+        // input map of parameters passed to the script.
+        metadata.paramsValueSlot = utility.addVariable(null, "params", definition.smapType).slot;
         // scorer parameter passed to the script. internal use only.
         metadata.scorerValueSlot = utility.addVariable(null, "#scorer", definition.objectType).slot;
         // doc parameter passed to the script.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java
index fd060b329cd..199fe5c11ba 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java
@@ -49,5 +49,5 @@ public abstract class Executable {
         return definition;
     }
 
-    public abstract Object execute(Map<String, Object> input, Scorer scorer, LeafDocLookup doc, Object value);
+    public abstract Object execute(Map<String, Object> params, Scorer scorer, LeafDocLookup doc, Object value);
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java
index a8d8d35f2f5..0a326b6393f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java
@@ -408,7 +408,7 @@ class Metadata {
      * Used to determine what slot the input variable is stored in.  This is used in the {@link Writer} whenever
      * the input variable is accessed.
      */
-    int inputValueSlot = -1;
+    int paramsValueSlot = -1;
 
     /**
      * Used to determine what slot the Scorer variable is stored in.  This is used in the {@link Writer} to load
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
index 9b3b6378dea..d17047933b8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
@@ -162,7 +162,7 @@ class Writer extends PainlessParserBaseVisitor<Void> {
         if (metadata.ctxValueUsed) {
             // if the _ctx value is used, we do this once:
             //   final Map<String,Object> ctx = input.get("ctx");
-            execute.visitVarInsn(Opcodes.ALOAD, metadata.inputValueSlot);
+            execute.visitVarInsn(Opcodes.ALOAD, metadata.paramsValueSlot);
             execute.push("ctx");
             execute.invokeInterface(WriterConstants.MAP_TYPE, WriterConstants.MAP_GET);
             execute.visitVarInsn(Opcodes.ASTORE, metadata.ctxValueSlot);
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java
index d2a24ef9533..2fb676bf299 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java
@@ -100,11 +100,11 @@ public class BasicExpressionTests extends ScriptTestCase {
      */
     public void testBoxing() {
         // return
-        assertEquals(4, exec("return input.get(\"x\");", Collections.singletonMap("x", 4)));
+        assertEquals(4, exec("return params.get(\"x\");", Collections.singletonMap("x", 4)));
         // assignment
-        assertEquals(4, exec("int y = (Integer)input.get(\"x\"); return y;", Collections.singletonMap("x", 4)));
+        assertEquals(4, exec("int y = (Integer)params.get(\"x\"); return y;", Collections.singletonMap("x", 4)));
         // comparison
-        assertEquals(true, exec("return 5 > (Integer)input.get(\"x\");", Collections.singletonMap("x", 4)));
+        assertEquals(true, exec("return 5 > (Integer)params.get(\"x\");", Collections.singletonMap("x", 4)));
     }
 
     public void testBool() {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptEngineTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptEngineTests.java
index 768de29e948..4871a72776f 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptEngineTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptEngineTests.java
@@ -46,12 +46,12 @@ public class ScriptEngineTests extends ScriptTestCase {
         obj1.put("l", Arrays.asList("2", "1"));
         vars.put("obj1", obj1);
 
-        Object value = exec("return input.get(\"obj1\");", vars);
+        Object value = exec("return params['obj1'];", vars);
         obj1 = (Map<String, Object>)value;
         assertEquals("value1", obj1.get("prop1"));
         assertEquals("value2", ((Map<String, Object>) obj1.get("obj2")).get("prop2"));
 
-        value = exec("return ((List)((Map<String, Object>)input.get(\"obj1\")).get(\"l\")).get(0);", vars);
+        value = exec("return params.obj1.l.0;", vars);
         assertEquals("2", value);
     }
 
@@ -65,15 +65,15 @@ public class ScriptEngineTests extends ScriptTestCase {
         obj1.put("obj2", obj2);
         vars.put("l", Arrays.asList("1", "2", "3", obj1));
 
-        assertEquals(4, exec("return ((List)input.get(\"l\")).size();", vars));
-        assertEquals("1", exec("return ((List)input.get(\"l\")).get(0);", vars));
+        assertEquals(4, exec("return params.l.size();", vars));
+        assertEquals("1", exec("return params.l.0;", vars));
 
-        Object value = exec("return ((List)input.get(\"l\")).get(3);", vars);
+        Object value = exec("return params.l.3;", vars);
         obj1 = (Map<String, Object>)value;
         assertEquals("value1", obj1.get("prop1"));
         assertEquals("value2", ((Map<String, Object>)obj1.get("obj2")).get("prop2"));
 
-        assertEquals("value1", exec("return ((Map<String, Object>)((List)input.get(\"l\")).get(3)).get(\"prop1\");", vars));
+        assertEquals("value1", exec("return params.l.3.prop1;", vars));
     }
 
     public void testChangingVarsCrossExecution1() {
@@ -82,7 +82,7 @@ public class ScriptEngineTests extends ScriptTestCase {
         vars.put("ctx", ctx);
 
         Object compiledScript = scriptEngine.compile(
-                "return ((Map<String, Object>)input.get(\"ctx\")).get(\"value\");", Collections.emptyMap());
+                "return ctx.value;", Collections.emptyMap());
         ExecutableScript script = scriptEngine.executable(new CompiledScript(ScriptService.ScriptType.INLINE,
                 "testChangingVarsCrossExecution1", "painless", compiledScript), vars);
 
@@ -97,7 +97,7 @@ public class ScriptEngineTests extends ScriptTestCase {
 
     public void testChangingVarsCrossExecution2() {
         Map<String, Object> vars = new HashMap<>();
-        Object compiledScript = scriptEngine.compile("return input.get(\"value\");", Collections.emptyMap());
+        Object compiledScript = scriptEngine.compile("return params['value'];", Collections.emptyMap());
 
         ExecutableScript script = scriptEngine.executable(new CompiledScript(ScriptService.ScriptType.INLINE,
                 "testChangingVarsCrossExecution2", "painless", compiledScript), vars);
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
index 975e2301877..2f80d1d1c39 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
@@ -26,7 +26,7 @@ import java.util.Collections;
 public class WhenThingsGoWrongTests extends ScriptTestCase {
     public void testNullPointer() {
         expectThrows(NullPointerException.class, () -> {
-            exec("int x = (int) ((Map) input).get(\"missing\"); return x;");
+            exec("int x = params['missing']; return x;");
         });
     }
 
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/15_update.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/15_update.yaml
index 39cee4e82a3..555cbaee0cb 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/15_update.yaml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/15_update.yaml
@@ -18,7 +18,7 @@
           script: "1"
           body:
             lang:   painless
-            script: "ctx._source.foo = input.bar"
+            script: "ctx._source.foo = params.bar"
             params: { bar: 'xxx' }
 
   - match: { _index:   test_1 }
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/16_update2.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/16_update2.yaml
index c7eed81f84b..8a29e8e786b 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/16_update2.yaml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/16_update2.yaml
@@ -5,7 +5,7 @@
       put_script:
         id: "1"
         lang: "painless"
-        body: { "script":  "_score * doc[\"myParent.weight\"].value" }
+        body: { "script":  "_score * doc['myParent.weight'].value" }
   - match: { acknowledged: true }
 
   - do:
@@ -15,7 +15,7 @@
   - match: { found: true }
   - match: { lang: painless }
   - match: { _id: "1" }
-  - match: { "script":  "_score * doc[\"myParent.weight\"].value" }
+  - match: { "script":  "_score * doc['myParent.weight'].value" }
 
   - do:
      catch: missing
@@ -44,11 +44,11 @@
       put_script:
         id: "1"
         lang: "painless"
-        body: { "script":  "_score * foo bar + doc[\"myParent.weight\"].value" }
+        body: { "script":  "_score * foo bar + doc['myParent.weight'].value" }
 
   - do:
       catch: /Unable.to.parse.*/
       put_script:
         id: "1"
         lang: "painless"
-        body: { "script":  "_score * foo bar + doc[\"myParent.weight\"].value" }
+        body: { "script":  "_score * foo bar + doc['myParent.weight'].value" }
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/20_scriptfield.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/20_scriptfield.yaml
index f85bbef121f..b92012959d1 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/20_scriptfield.yaml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/20_scriptfield.yaml
@@ -28,7 +28,7 @@ setup:
                 script_fields:
                     bar:
                         script: 
-                            inline: "doc['foo'].value + input.x;"
+                            inline: "doc['foo'].value + params.x;"
                             lang: painless
                             params:
                                 x: "bbb"
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/25_script_upsert.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/25_script_upsert.yaml
index d4a447e98a7..66dee6bcc3a 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/25_script_upsert.yaml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/25_script_upsert.yaml
@@ -7,7 +7,7 @@
           type:     test
           id:       1
           body:
-            script: "ctx._source.foo = input.bar"
+            script: "ctx._source.foo = params.bar"
             lang: "painless"
             params: { bar: 'xxx' }
             upsert: { foo: baz }
@@ -27,7 +27,7 @@
           type:     test
           id:       1
           body:
-            script: "ctx._source.foo = input.bar"
+            script: "ctx._source.foo = params.bar"
             lang: "painless"
             params: { bar: 'xxx' }
             upsert: { foo: baz }
@@ -46,7 +46,7 @@
           type:     test
           id:       2
           body:
-            script: "ctx._source.foo = input.bar"
+            script: "ctx._source.foo = params.bar"
             lang: "painless"
             params: { bar: 'xxx' }
             upsert: { foo: baz }
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml
index cd9ed7511d3..f1051ba7106 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/30_search.yaml
@@ -51,7 +51,7 @@
                 query:
                     script:
                         script:
-                            inline: "doc['num1'].value > input.param1;"
+                            inline: "doc['num1'].value > params.param1;"
                             lang: painless
                             params:
                                 param1: 1
@@ -76,7 +76,7 @@
                 query:
                     script:
                         script:
-                            inline: "doc['num1'].value > input.param1;"
+                            inline: "doc['num1'].value > params.param1;"
                             lang: painless
                             params:
                                 param1: -1
@@ -236,7 +236,7 @@
                             "script_score": {
                                 "script": {
                                     "lang": "painless",
-                                    "inline": "input.param1 * input.param2 * _score",
+                                    "inline": "params.param1 * params.param2 * _score",
                                     "params": {
                                         "param1": 2,
                                         "param2": 2

From 930a215b5c959b658471354f507ca8219ec9585d Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Wed, 11 May 2016 23:56:19 -0400
Subject: [PATCH 0200/1311] add debugger

---
 modules/lang-painless/build.gradle            |  1 +
 .../org/elasticsearch/painless/Compiler.java  | 30 ++++-------
 .../org/elasticsearch/painless/Debugger.java  | 53 +++++++++++++++++++
 3 files changed, 65 insertions(+), 19 deletions(-)
 create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java

diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle
index bf0694a0b73..9f617c4e436 100644
--- a/modules/lang-painless/build.gradle
+++ b/modules/lang-painless/build.gradle
@@ -29,6 +29,7 @@ dependencies {
   compile 'org.ow2.asm:asm:5.0.4'
   compile 'org.ow2.asm:asm-commons:5.0.4'
   compile 'org.ow2.asm:asm-tree:5.0.4'
+  testCompile 'org.ow2.asm:asm-util:5.0.4'
 }
 
 dependencyLicenses {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
index 4cba3cb7923..6a21b7cc877 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
@@ -90,20 +90,23 @@ final class Compiler {
      * @return An {@link Executable} Painless script.
      */
     static Executable compile(final Loader loader, final String name, final String source, final CompilerSettings settings) {
+        final byte[] bytes = compile(name, source, Definition.INSTANCE, settings);
+        return createExecutable(loader, Definition.INSTANCE, name, source, bytes);
+    }
+
+    /**
+     * Compiles the script to bytecode
+     */
+    static byte[] compile(String name, String source, Definition definition, CompilerSettings settings) {
         if (source.length() > MAXIMUM_SOURCE_LENGTH) {
             throw new IllegalArgumentException("Scripts may be no longer than " + MAXIMUM_SOURCE_LENGTH +
-                " characters.  The passed in script is " + source.length() + " characters.  Consider using a" +
-                " plugin if a script longer than this length is a requirement.");
+                    " characters.  The passed in script is " + source.length() + " characters.  Consider using a" +
+                    " plugin if a script longer than this length is a requirement.");
         }
-
-        final Definition definition = Definition.INSTANCE;
         final ParserRuleContext root = createParseTree(source);
         final Metadata metadata = new Metadata(definition, source, root, settings);
         Analyzer.analyze(metadata);
-        final byte[] bytes = Writer.write(metadata);
-        final Executable executable = createExecutable(loader, definition, name, source, bytes);
-
-        return executable;
+        return Writer.write(metadata);
     }
 
     /**
@@ -140,17 +143,6 @@ final class Compiler {
     private static Executable createExecutable(final Loader loader, final Definition definition,
                                                final String name, final String source, final byte[] bytes) {
         try {
-            // Used for debugging.  Uncomment this code and add -Dtests.security.manager=false when running to save
-            // the generated Java class files.  The javap tool can then be used to inspect the generated byte code.
-
-            // try {
-            //    FileOutputStream f = new FileOutputStream(new File("<path>"), false);
-            //    f.write(bytes);
-            //    f.close();
-            // } catch (Exception e) {
-            //    throw new RuntimeException(e);
-            // }
-
             final Class<? extends Executable> clazz = loader.define(WriterConstants.CLASS_NAME, bytes);
             final java.lang.reflect.Constructor<? extends Executable> constructor =
                     clazz.getConstructor(Definition.class, String.class, String.class);
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
new file mode 100644
index 00000000000..949dd57ec5a
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.objectweb.asm.ClassReader;
+import org.objectweb.asm.util.TraceClassVisitor;
+
+import java.io.ByteArrayOutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.StandardCharsets;
+
+/** quick and dirty tools for debugging */
+final class Debugger {
+
+    /** compiles source to bytecode, and returns debugging output */
+    static String toString(String source) {
+        return toString(source, new CompilerSettings());
+    }
+
+    /** compiles to bytecode, and returns debugging output */
+    static String toString(String source, CompilerSettings settings) {
+        byte[] bytes = Compiler.compile("debugger", source, Definition.INSTANCE, settings);
+        ByteArrayOutputStream output = new ByteArrayOutputStream();
+        PrintWriter outputWriter = new PrintWriter(new OutputStreamWriter(output, StandardCharsets.UTF_8));
+        ClassReader reader = new ClassReader(bytes);
+        reader.accept(new TraceClassVisitor(outputWriter), 0);
+        outputWriter.flush();
+        try {
+            return output.toString("UTF-8");
+        } catch (UnsupportedEncodingException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

From 9fbbd73de1309f570dc058b00d79de9f5ae31e30 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Thu, 12 May 2016 09:05:27 +0200
Subject: [PATCH 0201/1311] Use proper encoding to write docs test snippets

---
 .../elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
index 21ae89256d2..1a519fe7c2b 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
@@ -199,7 +199,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
 
             // Now setup the writer
             Files.createDirectories(dest.parent)
-            current = dest.newPrintWriter()
+            current = dest.newPrintWriter('UTF-8')
         }
 
         void finishLastTest() {

From 28164ccf2e2d05c40f7a7f9bea4d3e9c9bae191e Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Thu, 12 May 2016 00:46:48 -0700
Subject: [PATCH 0202/1311] Decouple the ANTLR AST from Painless.

Closes #18286
---
 .../resources/checkstyle_suppressions.xml     |    4 +-
 modules/lang-painless/ant.xml                 |   12 +-
 .../src/main/antlr/PainlessLexer.g4           |    2 +-
 .../src/main/antlr/PainlessParser.g4          |   44 +-
 .../org/elasticsearch/painless/Analyzer.java  |  456 +-----
 .../painless/AnalyzerCaster.java              |  449 ++++--
 .../painless/AnalyzerExpression.java          |  856 ------------
 .../painless/AnalyzerExternal.java            |  813 -----------
 .../painless/AnalyzerPromoter.java            |  281 ----
 .../painless/AnalyzerStatement.java           |  591 --------
 .../painless/AnalyzerUtility.java             |  191 ---
 .../org/elasticsearch/painless/Compiler.java  |   72 +-
 .../painless/CompilerSettings.java            |   14 +-
 .../java/org/elasticsearch/painless/Def.java  |  269 ++--
 .../elasticsearch/painless/Definition.java    |  985 +++++--------
 .../painless/DynamicCallSite.java             |   54 +-
 .../elasticsearch/painless/Executable.java    |   15 +-
 .../org/elasticsearch/painless/Metadata.java  |  605 --------
 .../elasticsearch/painless/NeedsScore.java    |    4 +-
 .../org/elasticsearch/painless/Operation.java |   61 +
 .../elasticsearch/painless/PainlessError.java |    1 +
 .../painless/PainlessPlugin.java              |    3 +
 .../painless/PainlessScriptEngineService.java |   11 +-
 .../elasticsearch/painless/ScriptImpl.java    |    9 +-
 .../org/elasticsearch/painless/Utility.java   |    9 +-
 .../org/elasticsearch/painless/Variables.java |  219 +++
 .../org/elasticsearch/painless/Writer.java    |  503 +------
 .../elasticsearch/painless/WriterCaster.java  |   86 --
 .../painless/WriterConstants.java             |  208 +--
 .../painless/WriterExpression.java            |  669 ---------
 .../painless/WriterExternal.java              |  720 ----------
 .../painless/WriterStatement.java             |  390 ------
 .../elasticsearch/painless/WriterUtility.java |  551 ++++----
 .../{ => antlr}/ErrorHandlingLexer.java       |   21 +-
 .../painless/{ => antlr}/PainlessLexer.java   |   24 +-
 .../painless/{ => antlr}/PainlessParser.java  | 1242 ++++++++---------
 .../PainlessParserBaseVisitor.java            |   43 +-
 .../{ => antlr}/PainlessParserVisitor.java    |   62 +-
 .../{ => antlr}/ParserErrorStrategy.java      |   29 +-
 .../elasticsearch/painless/antlr/Walker.java  |  790 +++++++++++
 .../painless/node/AExpression.java            |  173 +++
 .../elasticsearch/painless/node/ALink.java    |  121 ++
 .../elasticsearch/painless/node/ANode.java    |   39 +
 .../painless/node/AStatement.java             |  125 ++
 .../elasticsearch/painless/node/EBinary.java  |  528 +++++++
 .../elasticsearch/painless/node/EBool.java    |  139 ++
 .../elasticsearch/painless/node/EBoolean.java |   47 +
 .../elasticsearch/painless/node/ECast.java    |   60 +
 .../elasticsearch/painless/node/EChain.java   |  310 ++++
 .../elasticsearch/painless/node/EComp.java    |  528 +++++++
 .../painless/node/EConditional.java           |   94 ++
 .../painless/node/EConstant.java              |   97 ++
 .../elasticsearch/painless/node/EDecimal.java |   63 +
 .../painless/node/EExplicit.java              |   70 +
 .../elasticsearch/painless/node/ENull.java    |   56 +
 .../elasticsearch/painless/node/ENumeric.java |  102 ++
 .../elasticsearch/painless/node/EUnary.java   |  234 ++++
 .../painless/node/LArrayLength.java           |   71 +
 .../elasticsearch/painless/node/LBrace.java   |   98 ++
 .../elasticsearch/painless/node/LCall.java    |  120 ++
 .../elasticsearch/painless/node/LCast.java    |   78 ++
 .../painless/node/LDefArray.java              |   74 +
 .../elasticsearch/painless/node/LDefCall.java |   94 ++
 .../painless/node/LDefField.java              |   67 +
 .../elasticsearch/painless/node/LField.java   |  136 ++
 .../painless/node/LListShortcut.java          |  105 ++
 .../painless/node/LMapShortcut.java           |  104 ++
 .../painless/node/LNewArray.java              |   98 ++
 .../elasticsearch/painless/node/LNewObj.java  |  118 ++
 .../painless/node/LShortcut.java              |  105 ++
 .../elasticsearch/painless/node/LString.java  |   67 +
 .../painless/node/LVariable.java              |   90 ++
 .../elasticsearch/painless/node/SBlock.java   |   75 +
 .../elasticsearch/painless/node/SBreak.java   |   52 +
 .../painless/node/SContinue.java              |   55 +
 .../painless/node/SDeclBlock.java             |   58 +
 .../painless/node/SDeclaration.java           |   86 ++
 .../org/elasticsearch/painless/node/SDo.java  |  105 ++
 .../painless/node/SExpression.java            |   72 +
 .../org/elasticsearch/painless/node/SFor.java |  181 +++
 .../elasticsearch/painless/node/SIfElse.java  |  111 ++
 .../elasticsearch/painless/node/SReturn.java  |   58 +
 .../elasticsearch/painless/node/SSource.java  |   76 +
 .../elasticsearch/painless/node/SThrow.java   |   57 +
 .../elasticsearch/painless/node/STrap.java    |   98 ++
 .../org/elasticsearch/painless/node/STry.java |  114 ++
 .../elasticsearch/painless/node/SWhile.java   |  121 ++
 .../painless/node/package-info.java           |  130 ++
 .../painless/BinaryOperatorTests.java         |    7 +-
 .../org/elasticsearch/painless/Debugger.java  |   16 +-
 .../org/elasticsearch/painless/DefTests.java  |   72 +-
 .../painless/ReservedWordTests.java           |   26 +-
 .../elasticsearch/painless/StringTests.java   |   28 +-
 .../painless/WhenThingsGoWrongTests.java      |    8 +-
 94 files changed, 8930 insertions(+), 8255 deletions(-)
 delete mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExpression.java
 delete mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java
 delete mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerPromoter.java
 delete mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerStatement.java
 delete mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerUtility.java
 delete mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
 delete mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterCaster.java
 delete mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExpression.java
 delete mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java
 delete mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterStatement.java
 rename modules/lang-painless/src/main/java/org/elasticsearch/painless/{ => antlr}/ErrorHandlingLexer.java (67%)
 rename modules/lang-painless/src/main/java/org/elasticsearch/painless/{ => antlr}/PainlessLexer.java (96%)
 rename modules/lang-painless/src/main/java/org/elasticsearch/painless/{ => antlr}/PainlessParser.java (76%)
 rename modules/lang-painless/src/main/java/org/elasticsearch/painless/{ => antlr}/PainlessParserBaseVisitor.java (91%)
 rename modules/lang-painless/src/main/java/org/elasticsearch/painless/{ => antlr}/PainlessParserVisitor.java (86%)
 rename modules/lang-painless/src/main/java/org/elasticsearch/painless/{ => antlr}/ParserErrorStrategy.java (69%)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 1f5e928c391..1c052597e55 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -7,8 +7,8 @@
   <!-- On Windows, Checkstyle matches files using \ path separator -->
 
   <!-- These files are generated by ANTLR so its silly to hold them to our rules. -->
-  <suppress files="org[/\\]elasticsearch[/\\]painless[/\\]PainlessLexer\.java" checks="." />
-  <suppress files="org[/\\]elasticsearch[/\\]painless[/\\]PainlessParser(|BaseVisitor|Visitor)\.java" checks="." />
+  <suppress files="org[/\\]elasticsearch[/\\]painless[/\\]antlr[/\\]PainlessLexer\.java" checks="." />
+  <suppress files="org[/\\]elasticsearch[/\\]painless[/\\]antlr[/\\]PainlessParser(|BaseVisitor|Visitor)\.java" checks="." />
 
   <!-- Hopefully temporary suppression of LineLength on files that don't pass it. We should remove these when we the
     files start to pass. -->
diff --git a/modules/lang-painless/ant.xml b/modules/lang-painless/ant.xml
index fa53d714257..1d2bdb678e8 100644
--- a/modules/lang-painless/ant.xml
+++ b/modules/lang-painless/ant.xml
@@ -49,7 +49,7 @@
         <include name="@{grammar}ParserVisitor.java" />
         <include name="@{grammar}ParserBaseVisitor.java" />
       </patternset>
-      <property name="output.path" location="src/main/java/org/elasticsearch/painless"/>
+      <property name="output.path" location="src/main/java/org/elasticsearch/painless/antlr"/>
       <!-- delete parser and lexer so files will be generated -->
       <delete dir="${output.path}">
         <patternset refid="grammar.@{grammar}.patternset"/>
@@ -63,7 +63,7 @@
       <local name="grammar.path"/>
       <local name="output.path"/>
       <property name="grammar.path" location="src/main/antlr"/>
-      <property name="output.path" location="src/main/java/org/elasticsearch/painless"/>
+      <property name="output.path" location="src/main/java/org/elasticsearch/painless/antlr"/>
       <!-- invoke ANTLR4 -->
       <java classname="org.antlr.v4.Tool" fork="true" failonerror="true" classpathref="regenerate.classpath" taskname="antlr">
         <sysproperty key="file.encoding" value="UTF-8"/>
@@ -71,7 +71,7 @@
         <sysproperty key="user.country" value="US"/>
         <sysproperty key="user.variant" value=""/>
         <arg value="-package"/>
-        <arg value="org.elasticsearch.painless"/>
+        <arg value="org.elasticsearch.painless.antlr"/>
         <arg value="-o"/>
         <arg path="${output.path}"/>
         <arg path="${grammar.path}/@{grammar}Lexer.g4"/>
@@ -85,7 +85,7 @@
       <local name="grammar.path"/>
       <local name="output.path"/>
       <property name="grammar.path" location="src/main/antlr"/>
-      <property name="output.path" location="src/main/java/org/elasticsearch/painless"/>
+      <property name="output.path" location="src/main/java/org/elasticsearch/painless/antlr"/>
       <!-- invoke ANTLR4 -->
       <java classname="org.antlr.v4.Tool" fork="true" failonerror="true" classpathref="regenerate.classpath" taskname="antlr">
         <sysproperty key="file.encoding" value="UTF-8"/>
@@ -93,7 +93,7 @@
         <sysproperty key="user.country" value="US"/>
         <sysproperty key="user.variant" value=""/>
         <arg value="-package"/>
-        <arg value="org.elasticsearch.painless"/>
+        <arg value="org.elasticsearch.painless.antlr"/>
         <arg value="-no-listener"/>
         <arg value="-visitor"/>
         <!-- <arg value="-Xlog"/> -->
@@ -110,7 +110,7 @@
       <local name="grammar.path"/>
       <local name="output.path"/>
       <property name="grammar.path" location="src/main/antlr"/>
-      <property name="output.path" location="src/main/java/org/elasticsearch/painless"/>
+      <property name="output.path" location="src/main/java/org/elasticsearch/painless/antlr"/>
       <patternset id="grammar.@{grammar}.patternset">
         <include name="@{grammar}Lexer.java" />
         <include name="@{grammar}Parser.java" />
diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.g4 b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
index eab0670f973..97c79845043 100644
--- a/modules/lang-painless/src/main/antlr/PainlessLexer.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
@@ -63,7 +63,7 @@ EQR:     '===';
 NE:      '!=';
 NER:     '!==';
 BWAND:   '&';
-BWXOR:   '^';
+XOR:     '^';
 BWOR:    '|';
 BOOLAND: '&&';
 BOOLOR:  '||';
diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index 1ecd72528e0..e8aa3592c59 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -92,9 +92,9 @@ expression
     |               TRUE                                                # true
     |               FALSE                                               # false
     |               NULL                                                # null
-    | <assoc=right> extstart increment                                  # postinc
-    | <assoc=right> increment extstart                                  # preinc
-    |               extstart                                            # external
+    | <assoc=right> chain ( INCR | DECR )                               # postinc
+    | <assoc=right> ( INCR | DECR ) chain                               # preinc
+    |               chain                                               # read
     | <assoc=right> ( BOOLNOT | BWNOT | ADD | SUB ) expression          # unary
     | <assoc=right> LP decltype RP expression                           # cast
     |               expression ( MUL | DIV | REM ) expression           # binary
@@ -103,39 +103,35 @@ expression
     |               expression ( LT | LTE | GT | GTE ) expression       # comp
     |               expression ( EQ | EQR | NE | NER ) expression       # comp
     |               expression BWAND expression                         # binary
-    |               expression BWXOR expression                         # binary
+    |               expression XOR expression                           # binary
     |               expression BWOR expression                          # binary
     |               expression BOOLAND expression                       # bool
     |               expression BOOLOR expression                        # bool
     | <assoc=right> expression COND expression COLON expression         # conditional
-    | <assoc=right> extstart ( ASSIGN | AADD | ASUB | AMUL | ADIV
+    | <assoc=right> chain ( ASSIGN | AADD | ASUB | AMUL | ADIV
                                       | AREM | AAND | AXOR | AOR
                                       | ALSH | ARSH | AUSH ) expression # assignment
     ;
 
-extstart
-    : extprec
-    | extcast
-    | extvar
-    | extnew
-    | extstring
+chain
+    : linkprec
+    | linkcast
+    | linkvar
+    | linknew
+    | linkstring
     ;
 
-extprec:   LP ( extprec | extcast | extvar | extnew | extstring ) RP ( extdot | extbrace )?;
-extcast:   LP decltype RP ( extprec | extcast | extvar | extnew | extstring );
-extbrace:  LBRACE expression RBRACE ( extdot | extbrace )?;
-extdot:    DOT ( extcall | extfield );
-extcall:   EXTID arguments ( extdot | extbrace )?;
-extvar:    identifier ( extdot | extbrace )?;
-extfield:  ( EXTID | EXTINTEGER ) ( extdot | extbrace )?;
-extnew:    NEW identifier ( ( arguments extdot? ) | ( ( LBRACE expression RBRACE )+ extdot? ) );
-extstring: STRING (extdot | extbrace )?;
+linkprec:   LP ( linkprec | linkcast | linkvar | linknew | linkstring ) RP ( linkdot | linkbrace )?;
+linkcast:   LP decltype RP ( linkprec | linkcast | linkvar | linknew | linkstring );
+linkbrace:  LBRACE expression RBRACE ( linkdot | linkbrace )?;
+linkdot:    DOT ( linkcall | linkfield );
+linkcall:   EXTID arguments ( linkdot | linkbrace )?;
+linkvar:    identifier ( linkdot | linkbrace )?;
+linkfield:  ( EXTID | EXTINTEGER ) ( linkdot | linkbrace )?;
+linknew:    NEW identifier ( ( arguments linkdot? ) | ( ( LBRACE expression RBRACE )+ linkdot? ) );
+linkstring: STRING (linkdot | linkbrace )?;
 
 arguments
     : ( LP ( expression ( COMMA expression )* )? RP )
     ;
 
-increment
-    : INCR
-    | DECR
-    ;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
index 398e46f403d..f215e39ad01 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
@@ -19,450 +19,20 @@
 
 package org.elasticsearch.painless;
 
-import org.elasticsearch.painless.PainlessParser.AfterthoughtContext;
-import org.elasticsearch.painless.PainlessParser.ArgumentsContext;
-import org.elasticsearch.painless.PainlessParser.AssignmentContext;
-import org.elasticsearch.painless.PainlessParser.BinaryContext;
-import org.elasticsearch.painless.PainlessParser.BoolContext;
-import org.elasticsearch.painless.PainlessParser.BreakContext;
-import org.elasticsearch.painless.PainlessParser.CastContext;
-import org.elasticsearch.painless.PainlessParser.CompContext;
-import org.elasticsearch.painless.PainlessParser.ConditionalContext;
-import org.elasticsearch.painless.PainlessParser.ContinueContext;
-import org.elasticsearch.painless.PainlessParser.DeclContext;
-import org.elasticsearch.painless.PainlessParser.DeclarationContext;
-import org.elasticsearch.painless.PainlessParser.DecltypeContext;
-import org.elasticsearch.painless.PainlessParser.DeclvarContext;
-import org.elasticsearch.painless.PainlessParser.DoContext;
-import org.elasticsearch.painless.PainlessParser.EmptyContext;
-import org.elasticsearch.painless.PainlessParser.EmptyscopeContext;
-import org.elasticsearch.painless.PainlessParser.ExprContext;
-import org.elasticsearch.painless.PainlessParser.ExtbraceContext;
-import org.elasticsearch.painless.PainlessParser.ExtcallContext;
-import org.elasticsearch.painless.PainlessParser.ExtcastContext;
-import org.elasticsearch.painless.PainlessParser.ExtdotContext;
-import org.elasticsearch.painless.PainlessParser.ExternalContext;
-import org.elasticsearch.painless.PainlessParser.ExtfieldContext;
-import org.elasticsearch.painless.PainlessParser.ExtnewContext;
-import org.elasticsearch.painless.PainlessParser.ExtprecContext;
-import org.elasticsearch.painless.PainlessParser.ExtstartContext;
-import org.elasticsearch.painless.PainlessParser.ExtstringContext;
-import org.elasticsearch.painless.PainlessParser.ExtvarContext;
-import org.elasticsearch.painless.PainlessParser.FalseContext;
-import org.elasticsearch.painless.PainlessParser.ForContext;
-import org.elasticsearch.painless.PainlessParser.GenericContext;
-import org.elasticsearch.painless.PainlessParser.IdentifierContext;
-import org.elasticsearch.painless.PainlessParser.IfContext;
-import org.elasticsearch.painless.PainlessParser.IncrementContext;
-import org.elasticsearch.painless.PainlessParser.InitializerContext;
-import org.elasticsearch.painless.PainlessParser.MultipleContext;
-import org.elasticsearch.painless.PainlessParser.NullContext;
-import org.elasticsearch.painless.PainlessParser.NumericContext;
-import org.elasticsearch.painless.PainlessParser.PostincContext;
-import org.elasticsearch.painless.PainlessParser.PrecedenceContext;
-import org.elasticsearch.painless.PainlessParser.PreincContext;
-import org.elasticsearch.painless.PainlessParser.ReturnContext;
-import org.elasticsearch.painless.PainlessParser.SingleContext;
-import org.elasticsearch.painless.PainlessParser.SourceContext;
-import org.elasticsearch.painless.PainlessParser.ThrowContext;
-import org.elasticsearch.painless.PainlessParser.TrapContext;
-import org.elasticsearch.painless.PainlessParser.TrueContext;
-import org.elasticsearch.painless.PainlessParser.TryContext;
-import org.elasticsearch.painless.PainlessParser.UnaryContext;
-import org.elasticsearch.painless.PainlessParser.WhileContext;
+import org.elasticsearch.painless.Variables.Reserved;
+import org.elasticsearch.painless.node.SSource;
 
-class Analyzer extends PainlessParserBaseVisitor<Void> {
-    static void analyze(final Metadata metadata) {
-        new Analyzer(metadata);
+/**
+ * Runs the analysis phase of compilation using the Painless AST.
+ */
+final class Analyzer {
+    static Variables analyze(final CompilerSettings settings, final Definition definition,
+                                    final Reserved shortcut, final SSource root) {
+        final Variables variables = new Variables(settings, definition, shortcut);
+        root.analyze(settings, definition, variables);
+
+        return variables;
     }
 
-    private final AnalyzerStatement statement;
-    private final AnalyzerExpression expression;
-    private final AnalyzerExternal external;
-
-    private Analyzer(final Metadata metadata) {
-        final Definition definition = metadata.definition;
-
-        final AnalyzerUtility utility = new AnalyzerUtility(metadata);
-        final AnalyzerCaster caster = new AnalyzerCaster(definition);
-        final AnalyzerPromoter promoter = new AnalyzerPromoter(definition);
-
-        statement = new AnalyzerStatement(metadata, this, utility, caster);
-        expression = new AnalyzerExpression(metadata, this, caster, promoter);
-        external = new AnalyzerExternal(metadata, this, utility, caster, promoter);
-
-        utility.incrementScope();
-        utility.addVariable(null, "#this", definition.execType);
-        //
-        // reserved words.
-        //
-        // input map of parameters passed to the script.
-        metadata.paramsValueSlot = utility.addVariable(null, "params", definition.smapType).slot;
-        // scorer parameter passed to the script. internal use only.
-        metadata.scorerValueSlot = utility.addVariable(null, "#scorer", definition.objectType).slot;
-        // doc parameter passed to the script.
-        // TODO: currently working as a Map<String,Def>, we can do better?
-        metadata.docValueSlot = utility.addVariable(null, "doc", definition.smapType).slot;
-        // aggregation _value parameter passed to the script
-        metadata.aggregationValueSlot = utility.addVariable(null, "_value", definition.defType).slot;
-        //
-        // reserved words implemented as local variables
-        //
-        // loop counter to catch runaway scripts. internal use only.
-        metadata.loopCounterSlot = utility.addVariable(null, "#loop", definition.intType).slot;
-        // document's score as a read-only double.
-        metadata.scoreValueSlot = utility.addVariable(null, "_score", definition.doubleType).slot;
-        // ctx map set by executable scripts as a read-only map.
-        metadata.ctxValueSlot = utility.addVariable(null, "ctx", definition.smapType).slot;
-
-        metadata.createStatementMetadata(metadata.root);
-        visit(metadata.root);
-
-        utility.decrementScope();
-    }
-
-    @Override
-    public Void visitSource(final SourceContext ctx) {
-        statement.processSource(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitIf(final IfContext ctx) {
-        statement.processIf(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitWhile(final WhileContext ctx) {
-        statement.processWhile(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitDo(final DoContext ctx) {
-        statement.processDo(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitFor(final ForContext ctx) {
-        statement.processFor(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitDecl(final DeclContext ctx) {
-        statement.processDecl(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitContinue(final ContinueContext ctx) {
-        statement.processContinue(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitBreak(final BreakContext ctx) {
-        statement.processBreak(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitReturn(final ReturnContext ctx) {
-        statement.processReturn(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitTry(final TryContext ctx) {
-        statement.processTry(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitThrow(final ThrowContext ctx) {
-        statement.processThrow(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExpr(final ExprContext ctx) {
-        statement.processExpr(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitMultiple(final MultipleContext ctx) {
-        statement.processMultiple(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitSingle(final SingleContext ctx) {
-        statement.processSingle(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitEmpty(final EmptyContext ctx) {
-        throw new UnsupportedOperationException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-    }
-
-    @Override
-    public Void visitEmptyscope(final EmptyscopeContext ctx) {
-        throw new UnsupportedOperationException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-    }
-
-    @Override
-    public Void visitInitializer(final InitializerContext ctx) {
-        statement.processInitializer(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitAfterthought(final AfterthoughtContext ctx) {
-        statement.processAfterthought(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitDeclaration(final DeclarationContext ctx) {
-        statement.processDeclaration(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitDecltype(final DecltypeContext ctx) {
-        statement.processDecltype(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitDeclvar(final DeclvarContext ctx) {
-        statement.processDeclvar(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitTrap(final TrapContext ctx) {
-        statement.processTrap(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitIdentifier(IdentifierContext ctx) {
-        throw new UnsupportedOperationException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-    }
-
-    @Override
-    public Void visitGeneric(GenericContext ctx) {
-        throw new UnsupportedOperationException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-    }
-
-    @Override
-    public Void visitPrecedence(final PrecedenceContext ctx) {
-        throw new UnsupportedOperationException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-    }
-
-    @Override
-    public Void visitNumeric(final NumericContext ctx) {
-        expression.processNumeric(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitTrue(final TrueContext ctx) {
-        expression.processTrue(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitFalse(final FalseContext ctx) {
-        expression.processFalse(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitNull(final NullContext ctx) {
-        expression.processNull(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExternal(final ExternalContext ctx) {
-        expression.processExternal(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitPostinc(final PostincContext ctx) {
-        expression.processPostinc(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitPreinc(final PreincContext ctx) {
-        expression.processPreinc(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitUnary(final UnaryContext ctx) {
-        expression.processUnary(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitCast(final CastContext ctx) {
-        expression.processCast(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitBinary(final BinaryContext ctx) {
-        expression.processBinary(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitComp(final CompContext ctx) {
-        expression.processComp(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitBool(final BoolContext ctx) {
-        expression.processBool(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitConditional(final ConditionalContext ctx) {
-        expression.processConditional(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitAssignment(final AssignmentContext ctx) {
-        expression.processAssignment(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtstart(final ExtstartContext ctx) {
-        external.processExtstart(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtprec(final ExtprecContext ctx) {
-        external.processExtprec(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtcast(final ExtcastContext ctx) {
-        external.processExtcast(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtbrace(final ExtbraceContext ctx) {
-        external.processExtbrace(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtdot(final ExtdotContext ctx) {
-        external.processExtdot(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtcall(final ExtcallContext ctx) {
-        external.processExtcall(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtvar(final ExtvarContext ctx) {
-        external.processExtvar(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtfield(final ExtfieldContext ctx) {
-        external.processExtfield(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtnew(final ExtnewContext ctx) {
-        external.processExtnew(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtstring(final ExtstringContext ctx) {
-        external.processExtstring(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitArguments(final ArgumentsContext ctx) {
-        throw new UnsupportedOperationException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-    }
-
-    @Override
-    public Void visitIncrement(final IncrementContext ctx) {
-        expression.processIncrement(ctx);
-
-        return null;
-    }
+    private Analyzer() {}
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
index dddb3491f64..5042608d063 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
@@ -19,46 +19,30 @@
 
 package org.elasticsearch.painless;
 
-import org.antlr.v4.runtime.ParserRuleContext;
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Transform;
 import org.elasticsearch.painless.Definition.Type;
-import org.elasticsearch.painless.Metadata.ExpressionMetadata;
 
-class AnalyzerCaster {
-    private final Definition definition;
+import java.lang.reflect.InvocationTargetException;
 
-    AnalyzerCaster(final Definition definition) {
-        this.definition = definition;
-    }
+/**
+ * Used during the analysis phase to collect legal type casts and promotions
+ * for type-checking and later to write necessary casts in the bytecode.
+ */
+public final class AnalyzerCaster {
 
-    void markCast(final ExpressionMetadata emd) {
-        if (emd.from == null) {
-            throw new IllegalStateException(AnalyzerUtility.error(emd.source) + "From cast type should never be null.");
+    public static Cast getLegalCast(final Definition definition,
+                                    final String location, final Type actual, final Type expected, final boolean explicit) {
+        final Cast cast = new Cast(actual, expected);
+
+        if (actual.equals(expected)) {
+            return null;
         }
 
-        if (emd.to != null) {
-            emd.cast = getLegalCast(emd.source, emd.from, emd.to, emd.explicit || !emd.typesafe);
-
-            if (emd.preConst != null && emd.to.sort.constant) {
-                emd.postConst = constCast(emd.source, emd.preConst, emd.cast);
-            }
-        } else {
-            throw new IllegalStateException(AnalyzerUtility.error(emd.source) + "To cast type should never be null.");
-        }
-    }
-
-    Cast getLegalCast(final ParserRuleContext source, final Type from, final Type to, final boolean explicit) {
-        final Cast cast = new Cast(from, to);
-
-        if (from.equals(to)) {
-            return cast;
-        }
-
-        if (from.sort == Sort.DEF && to.sort != Sort.VOID || from.sort != Sort.VOID && to.sort == Sort.DEF) {
-            final Transform transform = definition.transforms.get(cast);
+        if (actual.sort == Sort.DEF && expected.sort != Sort.VOID || actual.sort != Sort.VOID && expected.sort == Sort.DEF) {
+            final Transform transform = definition.transformsMap.get(cast);
 
             if (transform != null) {
                 return transform;
@@ -67,17 +51,17 @@ class AnalyzerCaster {
             return cast;
         }
 
-        switch (from.sort) {
+        switch (actual.sort) {
             case BOOL:
-                switch (to.sort) {
+                switch (expected.sort) {
                     case OBJECT:
                     case BOOL_OBJ:
-                        return checkTransform(source, cast);
+                        return checkTransform(definition, location, cast);
                 }
 
                 break;
             case BYTE:
-                switch (to.sort) {
+                switch (expected.sort) {
                     case SHORT:
                     case INT:
                     case LONG:
@@ -97,17 +81,17 @@ class AnalyzerCaster {
                     case LONG_OBJ:
                     case FLOAT_OBJ:
                     case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
+                        return checkTransform(definition, location, cast);
                     case CHAR_OBJ:
                         if (explicit)
-                            return checkTransform(source, cast);
+                            return checkTransform(definition, location, cast);
 
                         break;
                 }
 
                 break;
             case SHORT:
-                switch (to.sort) {
+                switch (expected.sort) {
                     case INT:
                     case LONG:
                     case FLOAT:
@@ -126,18 +110,18 @@ class AnalyzerCaster {
                     case LONG_OBJ:
                     case FLOAT_OBJ:
                     case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
+                        return checkTransform(definition, location, cast);
                     case BYTE_OBJ:
                     case CHAR_OBJ:
                         if (explicit)
-                            return checkTransform(source, cast);
+                            return checkTransform(definition, location, cast);
 
                         break;
                 }
 
                 break;
             case CHAR:
-                switch (to.sort) {
+                switch (expected.sort) {
                     case INT:
                     case LONG:
                     case FLOAT:
@@ -156,19 +140,19 @@ class AnalyzerCaster {
                     case LONG_OBJ:
                     case FLOAT_OBJ:
                     case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
+                        return checkTransform(definition, location, cast);
                     case BYTE_OBJ:
                     case SHORT_OBJ:
                     case STRING:
                         if (explicit)
-                            return checkTransform(source, cast);
+                            return checkTransform(definition, location, cast);
 
                         break;
                 }
 
                 break;
             case INT:
-                switch (to.sort) {
+                switch (expected.sort) {
                     case LONG:
                     case FLOAT:
                     case DOUBLE:
@@ -186,19 +170,19 @@ class AnalyzerCaster {
                     case LONG_OBJ:
                     case FLOAT_OBJ:
                     case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
+                        return checkTransform(definition, location, cast);
                     case BYTE_OBJ:
                     case SHORT_OBJ:
                     case CHAR_OBJ:
                         if (explicit)
-                            return checkTransform(source, cast);
+                            return checkTransform(definition, location, cast);
 
                         break;
                 }
 
                 break;
             case LONG:
-                switch (to.sort) {
+                switch (expected.sort) {
                     case FLOAT:
                     case DOUBLE:
                         return cast;
@@ -215,20 +199,20 @@ class AnalyzerCaster {
                     case LONG_OBJ:
                     case FLOAT_OBJ:
                     case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
+                        return checkTransform(definition, location, cast);
                     case BYTE_OBJ:
                     case SHORT_OBJ:
                     case CHAR_OBJ:
                     case INT_OBJ:
                         if (explicit)
-                            return checkTransform(source, cast);
+                            return checkTransform(definition, location, cast);
 
                         break;
                 }
 
                 break;
             case FLOAT:
-                switch (to.sort) {
+                switch (expected.sort) {
                     case DOUBLE:
                         return cast;
                     case BYTE:
@@ -244,21 +228,21 @@ class AnalyzerCaster {
                     case NUMBER:
                     case FLOAT_OBJ:
                     case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
+                        return checkTransform(definition, location, cast);
                     case BYTE_OBJ:
                     case SHORT_OBJ:
                     case CHAR_OBJ:
                     case INT_OBJ:
                     case LONG_OBJ:
                         if (explicit)
-                            return checkTransform(source, cast);
+                            return checkTransform(definition, location, cast);
 
                         break;
                 }
 
                 break;
             case DOUBLE:
-                switch (to.sort) {
+                switch (expected.sort) {
                     case BYTE:
                     case SHORT:
                     case CHAR:
@@ -272,7 +256,7 @@ class AnalyzerCaster {
                     case OBJECT:
                     case NUMBER:
                     case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
+                        return checkTransform(definition, location, cast);
                     case BYTE_OBJ:
                     case SHORT_OBJ:
                     case CHAR_OBJ:
@@ -280,7 +264,7 @@ class AnalyzerCaster {
                     case LONG_OBJ:
                     case FLOAT_OBJ:
                         if (explicit)
-                            return checkTransform(source, cast);
+                            return checkTransform(definition, location, cast);
 
                         break;
                 }
@@ -288,7 +272,7 @@ class AnalyzerCaster {
                 break;
             case OBJECT:
             case NUMBER:
-                switch (to.sort) {
+                switch (expected.sort) {
                     case BYTE:
                     case SHORT:
                     case CHAR:
@@ -297,21 +281,21 @@ class AnalyzerCaster {
                     case FLOAT:
                     case DOUBLE:
                         if (explicit)
-                            return checkTransform(source, cast);
+                            return checkTransform(definition, location, cast);
 
                         break;
                 }
 
                 break;
             case BOOL_OBJ:
-                switch (to.sort) {
+                switch (expected.sort) {
                     case BOOL:
-                        return checkTransform(source, cast);
+                        return checkTransform(definition, location, cast);
                 }
 
                 break;
             case BYTE_OBJ:
-                switch (to.sort) {
+                switch (expected.sort) {
                     case BYTE:
                     case SHORT:
                     case INT:
@@ -323,18 +307,18 @@ class AnalyzerCaster {
                     case LONG_OBJ:
                     case FLOAT_OBJ:
                     case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
+                        return checkTransform(definition, location, cast);
                     case CHAR:
                     case CHAR_OBJ:
                         if (explicit)
-                            return checkTransform(source, cast);
+                            return checkTransform(definition, location, cast);
 
                         break;
                 }
 
                 break;
             case SHORT_OBJ:
-                switch (to.sort) {
+                switch (expected.sort) {
                     case SHORT:
                     case INT:
                     case LONG:
@@ -344,20 +328,20 @@ class AnalyzerCaster {
                     case LONG_OBJ:
                     case FLOAT_OBJ:
                     case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
+                        return checkTransform(definition, location, cast);
                     case BYTE:
                     case CHAR:
                     case BYTE_OBJ:
                     case CHAR_OBJ:
                         if (explicit)
-                            return checkTransform(source, cast);
+                            return checkTransform(definition, location, cast);
 
                         break;
                 }
 
                 break;
             case CHAR_OBJ:
-                switch (to.sort) {
+                switch (expected.sort) {
                     case CHAR:
                     case INT:
                     case LONG:
@@ -367,21 +351,21 @@ class AnalyzerCaster {
                     case LONG_OBJ:
                     case FLOAT_OBJ:
                     case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
+                        return checkTransform(definition, location, cast);
                     case BYTE:
                     case SHORT:
                     case BYTE_OBJ:
                     case SHORT_OBJ:
                     case STRING:
                         if (explicit)
-                            return checkTransform(source, cast);
+                            return checkTransform(definition, location, cast);
 
                         break;
                 }
 
                 break;
             case INT_OBJ:
-                switch (to.sort) {
+                switch (expected.sort) {
                     case INT:
                     case LONG:
                     case FLOAT:
@@ -389,7 +373,7 @@ class AnalyzerCaster {
                     case LONG_OBJ:
                     case FLOAT_OBJ:
                     case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
+                        return checkTransform(definition, location, cast);
                     case BYTE:
                     case SHORT:
                     case CHAR:
@@ -397,20 +381,20 @@ class AnalyzerCaster {
                     case SHORT_OBJ:
                     case CHAR_OBJ:
                         if (explicit)
-                            return checkTransform(source, cast);
+                            return checkTransform(definition, location, cast);
 
                         break;
                 }
 
                 break;
             case LONG_OBJ:
-                switch (to.sort) {
+                switch (expected.sort) {
                     case LONG:
                     case FLOAT:
                     case DOUBLE:
                     case FLOAT_OBJ:
                     case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
+                        return checkTransform(definition, location, cast);
                     case BYTE:
                     case SHORT:
                     case CHAR:
@@ -420,18 +404,18 @@ class AnalyzerCaster {
                     case CHAR_OBJ:
                     case INT_OBJ:
                         if (explicit)
-                            return checkTransform(source, cast);
+                            return checkTransform(definition, location, cast);
 
                         break;
                 }
 
                 break;
             case FLOAT_OBJ:
-                switch (to.sort) {
+                switch (expected.sort) {
                     case FLOAT:
                     case DOUBLE:
                     case DOUBLE_OBJ:
-                        return checkTransform(source, cast);
+                        return checkTransform(definition, location, cast);
                     case BYTE:
                     case SHORT:
                     case CHAR:
@@ -443,16 +427,16 @@ class AnalyzerCaster {
                     case INT_OBJ:
                     case LONG_OBJ:
                         if (explicit)
-                            return checkTransform(source, cast);
+                            return checkTransform(definition, location, cast);
 
                         break;
                 }
 
                 break;
             case DOUBLE_OBJ:
-                switch (to.sort) {
+                switch (expected.sort) {
                     case DOUBLE:
-                        return checkTransform(source, cast);
+                        return checkTransform(definition, location, cast);
                     case BYTE:
                     case SHORT:
                     case CHAR:
@@ -466,59 +450,57 @@ class AnalyzerCaster {
                     case LONG_OBJ:
                     case FLOAT_OBJ:
                         if (explicit)
-                            return checkTransform(source, cast);
+                            return checkTransform(definition, location, cast);
 
                         break;
                 }
 
                 break;
             case STRING:
-                switch (to.sort) {
+                switch (expected.sort) {
                     case CHAR:
                     case CHAR_OBJ:
                         if (explicit)
-                            return checkTransform(source, cast);
+                            return checkTransform(definition, location, cast);
 
                         break;
                 }
         }
 
         try {
-            from.clazz.asSubclass(to.clazz);
+            actual.clazz.asSubclass(expected.clazz);
 
             return cast;
         } catch (final ClassCastException cce0) {
             try {
                 if (explicit) {
-                    to.clazz.asSubclass(from.clazz);
+                    expected.clazz.asSubclass(actual.clazz);
 
                     return cast;
                 } else {
                     throw new ClassCastException(
-                        AnalyzerUtility.error(source) + "Cannot cast from [" + from.name + "] to [" + to.name + "].");
+                        "Error" + location + ": Cannot cast from [" + actual.name + "] to [" + expected.name + "].");
                 }
             } catch (final ClassCastException cce1) {
-                throw new ClassCastException(
-                    AnalyzerUtility.error(source) + "Cannot cast from [" + from.name + "] to [" + to.name + "].");
+                throw new ClassCastException("Error" + location + ": Cannot cast from [" + actual.name + "] to [" + expected.name + "].");
             }
         }
     }
 
-    private Transform checkTransform(final ParserRuleContext source, final Cast cast) {
-        final Transform transform = definition.transforms.get(cast);
+    private static Transform checkTransform(final Definition definition, final String location, final Cast cast) {
+        final Transform transform = definition.transformsMap.get(cast);
 
         if (transform == null) {
-            throw new ClassCastException(
-                AnalyzerUtility.error(source) + "Cannot cast from [" + cast.from.name + "] to [" + cast.to.name + "].");
+            throw new ClassCastException("Error" + location + ": Cannot cast from [" + cast.from.name + "] to [" + cast.to.name + "].");
         }
 
         return transform;
     }
 
-    private Object constCast(final ParserRuleContext source, final Object constant, final Cast cast) {
+    public static Object constCast(final String location, final Object constant, final Cast cast) {
         if (cast instanceof Transform) {
             final Transform transform = (Transform)cast;
-            return invokeTransform(source, transform, constant);
+            return invokeTransform(location, transform, constant);
         } else {
             final Sort fsort = cast.from.sort;
             final Sort tsort = cast.to.sort;
@@ -543,17 +525,17 @@ class AnalyzerCaster {
                     case FLOAT:  return number.floatValue();
                     case DOUBLE: return number.doubleValue();
                     default:
-                        throw new IllegalStateException(AnalyzerUtility.error(source) + "Expected numeric type for cast.");
+                        throw new IllegalStateException("Error" + location + ": Cannot cast from " +
+                            "[" + cast.from.clazz.getCanonicalName() + "] to [" + cast.to.clazz.getCanonicalName() + "].");
                 }
             } else {
-                throw new IllegalStateException(AnalyzerUtility.error(source) + "No valid constant cast from " +
-                    "[" + cast.from.clazz.getCanonicalName() + "] to " +
-                    "[" + cast.to.clazz.getCanonicalName() + "].");
+                throw new IllegalStateException("Error" + location + ": Cannot cast from " +
+                    "[" + cast.from.clazz.getCanonicalName() + "] to [" + cast.to.clazz.getCanonicalName() + "].");
             }
         }
     }
 
-    private Object invokeTransform(final ParserRuleContext source, final Transform transform, final Object object) {
+    private static Object invokeTransform(final String location, final Transform transform, final Object object) {
         final Method method = transform.method;
         final java.lang.reflect.Method jmethod = method.reflect;
         final int modifiers = jmethod.getModifiers();
@@ -564,11 +546,274 @@ class AnalyzerCaster {
             } else {
                 return jmethod.invoke(object);
             }
-        } catch (IllegalAccessException | IllegalArgumentException |
-            java.lang.reflect.InvocationTargetException | NullPointerException |
-            ExceptionInInitializerError exception) {
-            throw new IllegalArgumentException(AnalyzerUtility.error(source) +
-                "Cannot cast constant from [" + transform.from.name + "] to [" + transform.to.name + "].");
+        } catch (final IllegalAccessException | IllegalArgumentException |
+            InvocationTargetException | NullPointerException | ExceptionInInitializerError exception) {
+            throw new ClassCastException(
+                "Error" + location + ": Cannot cast from [" + transform.from.name + "] to [" + transform.to.name + "].");
         }
     }
+
+    public static Type promoteNumeric(final Definition definition, final Type from, final boolean decimal, final boolean primitive) {
+        final Sort sort = from.sort;
+
+        if (sort == Sort.DEF) {
+            return definition.defType;
+        } else if ((sort == Sort.DOUBLE || sort == Sort.DOUBLE_OBJ) && decimal) {
+            return primitive ? definition.doubleType : definition.doubleobjType;
+        } else if ((sort == Sort.FLOAT || sort == Sort.FLOAT_OBJ) && decimal) {
+            return primitive ? definition.floatType : definition.floatobjType;
+        } else if (sort == Sort.LONG || sort == Sort.LONG_OBJ) {
+            return primitive ? definition.longType : definition.longobjType;
+        } else if (sort == Sort.INT   || sort == Sort.INT_OBJ   ||
+                   sort == Sort.CHAR  || sort == Sort.CHAR_OBJ  ||
+                   sort == Sort.SHORT || sort == Sort.SHORT_OBJ ||
+                   sort == Sort.BYTE  || sort == Sort.BYTE_OBJ) {
+            return primitive ? definition.intType : definition.intobjType;
+        }
+
+        return null;
+    }
+
+    public static Type promoteNumeric(final Definition definition,
+                                      final Type from0, final Type from1, final boolean decimal, final boolean primitive) {
+        final Sort sort0 = from0.sort;
+        final Sort sort1 = from1.sort;
+
+        if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
+            return definition.defType;
+        }
+
+        if (decimal) {
+            if (sort0 == Sort.DOUBLE || sort0 == Sort.DOUBLE_OBJ ||
+                sort1 == Sort.DOUBLE || sort1 == Sort.DOUBLE_OBJ) {
+                return primitive ? definition.doubleType : definition.doubleobjType;
+            } else if (sort0 == Sort.FLOAT || sort0 == Sort.FLOAT_OBJ || sort1 == Sort.FLOAT || sort1 == Sort.FLOAT_OBJ) {
+                return primitive ? definition.floatType : definition.floatobjType;
+            }
+        }
+
+        if (sort0 == Sort.LONG || sort0 == Sort.LONG_OBJ ||
+            sort1 == Sort.LONG || sort1 == Sort.LONG_OBJ) {
+            return primitive ? definition.longType : definition.longobjType;
+        } else if (sort0 == Sort.INT   || sort0 == Sort.INT_OBJ   ||
+                   sort1 == Sort.INT   || sort1 == Sort.INT_OBJ   ||
+                   sort0 == Sort.CHAR  || sort0 == Sort.CHAR_OBJ  ||
+                   sort1 == Sort.CHAR  || sort1 == Sort.CHAR_OBJ  ||
+                   sort0 == Sort.SHORT || sort0 == Sort.SHORT_OBJ ||
+                   sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ ||
+                   sort0 == Sort.BYTE  || sort0 == Sort.BYTE_OBJ  ||
+                   sort1 == Sort.BYTE  || sort1 == Sort.BYTE_OBJ) {
+            return primitive ? definition.intType : definition.intobjType;
+        }
+
+        return null;
+    }
+
+    public static Type promoteAdd(final Definition definition, final Type from0, final Type from1) {
+        final Sort sort0 = from0.sort;
+        final Sort sort1 = from1.sort;
+
+        if (sort0 == Sort.STRING || sort1 == Sort.STRING) {
+            return definition.stringType;
+        }
+
+        return promoteNumeric(definition, from0, from1, true, true);
+    }
+
+    public static Type promoteXor(final Definition definition, final Type from0, final Type from1) {
+        final Sort sort0 = from0.sort;
+        final Sort sort1 = from1.sort;
+
+        if (sort0.bool || sort1.bool) {
+            return definition.booleanType;
+        }
+
+        return promoteNumeric(definition, from0, from1, false, true);
+    }
+
+    public static Type promoteEquality(final Definition definition, final Type from0, final Type from1) {
+        final Sort sort0 = from0.sort;
+        final Sort sort1 = from1.sort;
+
+        if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
+            return definition.defType;
+        }
+
+        final boolean primitive = sort0.primitive && sort1.primitive;
+
+        if (sort0.bool && sort1.bool) {
+            return primitive ? definition.booleanType : definition.booleanobjType;
+        }
+
+        if (sort0.numeric && sort1.numeric) {
+            return promoteNumeric(definition, from0, from1, true, primitive);
+        }
+
+        return definition.objectType;
+    }
+
+    public static Type promoteReference(final Definition definition, final Type from0, final Type from1) {
+        final Sort sort0 = from0.sort;
+        final Sort sort1 = from1.sort;
+
+        if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
+            return definition.defType;
+        }
+
+        if (sort0.primitive && sort1.primitive) {
+            if (sort0.bool && sort1.bool) {
+                return definition.booleanType;
+            }
+
+            if (sort0.numeric && sort1.numeric) {
+                return promoteNumeric(definition, from0, from1, true, true);
+            }
+        }
+
+        return definition.objectType;
+    }
+
+    public static Type promoteConditional(final Definition definition,
+                                          final Type from0, final Type from1, final Object const0, final Object const1) {
+        if (from0.equals(from1)) {
+            return from0;
+        }
+
+        final Sort sort0 = from0.sort;
+        final Sort sort1 = from1.sort;
+
+        if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
+            return definition.defType;
+        }
+
+        final boolean primitive = sort0.primitive && sort1.primitive;
+
+        if (sort0.bool && sort1.bool) {
+            return primitive ? definition.booleanType : definition.booleanobjType;
+        }
+
+        if (sort0.numeric && sort1.numeric) {
+            if (sort0 == Sort.DOUBLE || sort0 == Sort.DOUBLE_OBJ || sort1 == Sort.DOUBLE || sort1 == Sort.DOUBLE_OBJ) {
+                return primitive ? definition.doubleType : definition.doubleobjType;
+            } else if (sort0 == Sort.FLOAT || sort0 == Sort.FLOAT_OBJ || sort1 == Sort.FLOAT || sort1 == Sort.FLOAT_OBJ) {
+                return primitive ? definition.floatType : definition.floatobjType;
+            } else if (sort0 == Sort.LONG || sort0 == Sort.LONG_OBJ || sort1 == Sort.LONG || sort1 == Sort.LONG_OBJ) {
+                return sort0.primitive && sort1.primitive ? definition.longType : definition.longobjType;
+            } else {
+                if (sort0 == Sort.BYTE || sort0 == Sort.BYTE_OBJ) {
+                    if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
+                        return primitive ? definition.byteType : definition.byteobjType;
+                    } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
+                        if (const1 != null) {
+                            final short constant = (short)const1;
+
+                            if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
+                                return primitive ? definition.byteType : definition.byteobjType;
+                            }
+                        }
+
+                        return primitive ? definition.shortType : definition.shortobjType;
+                    } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
+                        return primitive ? definition.intType : definition.intobjType;
+                    } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
+                        if (const1 != null) {
+                            final int constant = (int)const1;
+
+                            if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
+                                return primitive ? definition.byteType : definition.byteobjType;
+                            }
+                        }
+
+                        return primitive ? definition.intType : definition.intobjType;
+                    }
+                } else if (sort0 == Sort.SHORT || sort0 == Sort.SHORT_OBJ) {
+                    if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
+                        if (const0 != null) {
+                            final short constant = (short)const0;
+
+                            if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
+                                return primitive ? definition.byteType : definition.byteobjType;
+                            }
+                        }
+
+                        return primitive ? definition.shortType : definition.shortobjType;
+                    } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
+                        return primitive ? definition.shortType : definition.shortobjType;
+                    } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
+                        return primitive ? definition.intType : definition.intobjType;
+                    } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
+                        if (const1 != null) {
+                            final int constant = (int)const1;
+
+                            if (constant <= Short.MAX_VALUE && constant >= Short.MIN_VALUE) {
+                                return primitive ? definition.shortType : definition.shortobjType;
+                            }
+                        }
+
+                        return primitive ? definition.intType : definition.intobjType;
+                    }
+                } else if (sort0 == Sort.CHAR || sort0 == Sort.CHAR_OBJ) {
+                    if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
+                        return primitive ? definition.intType : definition.intobjType;
+                    } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
+                        return primitive ? definition.intType : definition.intobjType;
+                    } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
+                        return primitive ? definition.charType : definition.charobjType;
+                    } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
+                        if (const1 != null) {
+                            final int constant = (int)const1;
+
+                            if (constant <= Character.MAX_VALUE && constant >= Character.MIN_VALUE) {
+                                return primitive ? definition.byteType : definition.byteobjType;
+                            }
+                        }
+
+                        return primitive ? definition.intType : definition.intobjType;
+                    }
+                } else if (sort0 == Sort.INT || sort0 == Sort.INT_OBJ) {
+                    if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
+                        if (const0 != null) {
+                            final int constant = (int)const0;
+
+                            if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
+                                return primitive ? definition.byteType : definition.byteobjType;
+                            }
+                        }
+
+                        return primitive ? definition.intType : definition.intobjType;
+                    } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
+                        if (const0 != null) {
+                            final int constant = (int)const0;
+
+                            if (constant <= Short.MAX_VALUE && constant >= Short.MIN_VALUE) {
+                                return primitive ? definition.byteType : definition.byteobjType;
+                            }
+                        }
+
+                        return primitive ? definition.intType : definition.intobjType;
+                    } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
+                        if (const0 != null) {
+                            final int constant = (int)const0;
+
+                            if (constant <= Character.MAX_VALUE && constant >= Character.MIN_VALUE) {
+                                return primitive ? definition.byteType : definition.byteobjType;
+                            }
+                        }
+
+                        return primitive ? definition.intType : definition.intobjType;
+                    } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
+                        return primitive ? definition.intType : definition.intobjType;
+                    }
+                }
+            }
+        }
+
+        // TODO: In the rare case we still haven't reached a correct promotion we need
+        //       to calculate the highest upper bound for the two types and return that.
+        //       However, for now we just return objectType that may require an extra cast.
+
+        return definition.objectType;
+    }
+
+    private AnalyzerCaster() {}
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExpression.java
deleted file mode 100644
index 3db42f88696..00000000000
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExpression.java
+++ /dev/null
@@ -1,856 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless;
-
-import org.elasticsearch.painless.Definition.Sort;
-import org.elasticsearch.painless.Definition.Type;
-import org.elasticsearch.painless.Metadata.ExpressionMetadata;
-import org.elasticsearch.painless.Metadata.ExternalMetadata;
-import org.elasticsearch.painless.PainlessParser.AssignmentContext;
-import org.elasticsearch.painless.PainlessParser.BinaryContext;
-import org.elasticsearch.painless.PainlessParser.BoolContext;
-import org.elasticsearch.painless.PainlessParser.CastContext;
-import org.elasticsearch.painless.PainlessParser.CompContext;
-import org.elasticsearch.painless.PainlessParser.ConditionalContext;
-import org.elasticsearch.painless.PainlessParser.DecltypeContext;
-import org.elasticsearch.painless.PainlessParser.ExpressionContext;
-import org.elasticsearch.painless.PainlessParser.ExternalContext;
-import org.elasticsearch.painless.PainlessParser.ExtstartContext;
-import org.elasticsearch.painless.PainlessParser.FalseContext;
-import org.elasticsearch.painless.PainlessParser.IncrementContext;
-import org.elasticsearch.painless.PainlessParser.NullContext;
-import org.elasticsearch.painless.PainlessParser.NumericContext;
-import org.elasticsearch.painless.PainlessParser.PostincContext;
-import org.elasticsearch.painless.PainlessParser.PreincContext;
-import org.elasticsearch.painless.PainlessParser.TrueContext;
-import org.elasticsearch.painless.PainlessParser.UnaryContext;
-
-import static org.elasticsearch.painless.PainlessParser.ADD;
-import static org.elasticsearch.painless.PainlessParser.BWAND;
-import static org.elasticsearch.painless.PainlessParser.BWOR;
-import static org.elasticsearch.painless.PainlessParser.BWXOR;
-import static org.elasticsearch.painless.PainlessParser.DIV;
-import static org.elasticsearch.painless.PainlessParser.LSH;
-import static org.elasticsearch.painless.PainlessParser.MUL;
-import static org.elasticsearch.painless.PainlessParser.REM;
-import static org.elasticsearch.painless.PainlessParser.RSH;
-import static org.elasticsearch.painless.PainlessParser.SUB;
-import static org.elasticsearch.painless.PainlessParser.USH;
-
-class AnalyzerExpression {
-    private final Metadata metadata;
-    private final Definition definition;
-    private final CompilerSettings settings;
-
-    private final Analyzer analyzer;
-    private final AnalyzerCaster caster;
-    private final AnalyzerPromoter promoter;
-
-    AnalyzerExpression(final Metadata metadata, final Analyzer analyzer,
-                       final AnalyzerCaster caster, final AnalyzerPromoter promoter) {
-        this.metadata = metadata;
-        this.definition = metadata.definition;
-        this.settings = metadata.settings;
-
-        this.analyzer = analyzer;
-        this.caster = caster;
-        this.promoter = promoter;
-    }
-
-    void processNumeric(final NumericContext ctx) {
-        final ExpressionMetadata numericemd = metadata.getExpressionMetadata(ctx);
-        final boolean negate = ctx.parent instanceof UnaryContext && ((UnaryContext)ctx.parent).SUB() != null;
-
-        if (ctx.DECIMAL() != null) {
-            final String svalue = (negate ? "-" : "") + ctx.DECIMAL().getText();
-
-            if (svalue.endsWith("f") || svalue.endsWith("F")) {
-                try {
-                    numericemd.from = definition.floatType;
-                    numericemd.preConst = Float.parseFloat(svalue.substring(0, svalue.length() - 1));
-                } catch (NumberFormatException exception) {
-                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Invalid float constant [" + svalue + "].");
-                }
-            } else {
-                try {
-                    numericemd.from = definition.doubleType;
-                    numericemd.preConst = Double.parseDouble(svalue);
-                } catch (NumberFormatException exception) {
-                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Invalid double constant [" + svalue + "].");
-                }
-            }
-        } else {
-            String svalue = negate ? "-" : "";
-            int radix;
-
-            if (ctx.OCTAL() != null) {
-                svalue += ctx.OCTAL().getText();
-                radix = 8;
-            } else if (ctx.INTEGER() != null) {
-                svalue += ctx.INTEGER().getText();
-                radix = 10;
-            } else if (ctx.HEX() != null) {
-                svalue += ctx.HEX().getText();
-                radix = 16;
-            } else {
-                throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-            }
-
-            if (svalue.endsWith("d") || svalue.endsWith("D")) {
-                try {
-                    numericemd.from = definition.doubleType;
-                    numericemd.preConst = Double.parseDouble(svalue.substring(0, svalue.length() - 1));
-                } catch (NumberFormatException exception) {
-                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Invalid float constant [" + svalue + "].");
-                }
-            } else if (svalue.endsWith("f") || svalue.endsWith("F")) {
-                try {
-                    numericemd.from = definition.floatType;
-                    numericemd.preConst = Float.parseFloat(svalue.substring(0, svalue.length() - 1));
-                } catch (NumberFormatException exception) {
-                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Invalid float constant [" + svalue + "].");
-                }
-            } else if (svalue.endsWith("l") || svalue.endsWith("L")) {
-                try {
-                    numericemd.from = definition.longType;
-                    numericemd.preConst = Long.parseLong(svalue.substring(0, svalue.length() - 1), radix);
-                } catch (NumberFormatException exception) {
-                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Invalid long constant [" + svalue + "].");
-                }
-            } else {
-                try {
-                    final Type type = numericemd.to;
-                    final Sort sort = type == null ? Sort.INT : type.sort;
-                    final int value = Integer.parseInt(svalue, radix);
-
-                    if (sort == Sort.BYTE && value >= Byte.MIN_VALUE && value <= Byte.MAX_VALUE) {
-                        numericemd.from = definition.byteType;
-                        numericemd.preConst = (byte)value;
-                    } else if (sort == Sort.CHAR && value >= Character.MIN_VALUE && value <= Character.MAX_VALUE) {
-                        numericemd.from = definition.charType;
-                        numericemd.preConst = (char)value;
-                    } else if (sort == Sort.SHORT && value >= Short.MIN_VALUE && value <= Short.MAX_VALUE) {
-                        numericemd.from = definition.shortType;
-                        numericemd.preConst = (short)value;
-                    } else {
-                        numericemd.from = definition.intType;
-                        numericemd.preConst = value;
-                    }
-                } catch (NumberFormatException exception) {
-                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Invalid int constant [" + svalue + "].");
-                }
-            }
-        }
-    }
-
-    void processTrue(final TrueContext ctx) {
-        final ExpressionMetadata trueemd = metadata.getExpressionMetadata(ctx);
-
-        if (ctx.TRUE() == null) {
-            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-        }
-
-        trueemd.preConst = true;
-        trueemd.from = definition.booleanType;
-    }
-
-    void processFalse(final FalseContext ctx) {
-        final ExpressionMetadata falseemd = metadata.getExpressionMetadata(ctx);
-
-        if (ctx.FALSE() == null) {
-            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-        }
-
-        falseemd.preConst = false;
-        falseemd.from = definition.booleanType;
-    }
-
-    void processNull(final NullContext ctx) {
-        final ExpressionMetadata nullemd = metadata.getExpressionMetadata(ctx);
-
-        if (ctx.NULL() == null) {
-            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-        }
-
-        nullemd.isNull = true;
-
-        if (nullemd.to != null) {
-            if (nullemd.to.sort.primitive) {
-                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
-                    "Cannot cast null to a primitive type [" + nullemd.to.name + "].");
-            }
-
-            nullemd.from = nullemd.to;
-        } else {
-            nullemd.from = definition.objectType;
-        }
-    }
-
-    void processExternal(final ExternalContext ctx) {
-        final ExpressionMetadata extemd = metadata.getExpressionMetadata(ctx);
-
-        final ExtstartContext extstartctx = ctx.extstart();
-        final ExternalMetadata extstartemd = metadata.createExternalMetadata(extstartctx);
-        extstartemd.read = extemd.read;
-        analyzer.visit(extstartctx);
-
-        extemd.statement = extstartemd.statement;
-        extemd.preConst = extstartemd.constant;
-        extemd.from = extstartemd.current;
-        extemd.typesafe = extstartemd.current.sort != Sort.DEF;
-    }
-
-    void processPostinc(final PostincContext ctx) {
-        final ExpressionMetadata postincemd = metadata.getExpressionMetadata(ctx);
-
-        final ExtstartContext extstartctx = ctx.extstart();
-        final ExternalMetadata extstartemd = metadata.createExternalMetadata(extstartctx);
-        extstartemd.read = postincemd.read;
-        extstartemd.storeExpr = ctx.increment();
-        extstartemd.token = ADD;
-        extstartemd.post = true;
-        analyzer.visit(extstartctx);
-
-        postincemd.statement = true;
-        postincemd.from = extstartemd.read ? extstartemd.current : definition.voidType;
-        postincemd.typesafe = extstartemd.current.sort != Sort.DEF;
-    }
-
-    void processPreinc(final PreincContext ctx) {
-        final ExpressionMetadata preincemd = metadata.getExpressionMetadata(ctx);
-
-        final ExtstartContext extstartctx = ctx.extstart();
-        final ExternalMetadata extstartemd = metadata.createExternalMetadata(extstartctx);
-        extstartemd.read = preincemd.read;
-        extstartemd.storeExpr = ctx.increment();
-        extstartemd.token = ADD;
-        extstartemd.pre = true;
-        analyzer.visit(extstartctx);
-
-        preincemd.statement = true;
-        preincemd.from = extstartemd.read ? extstartemd.current : definition.voidType;
-        preincemd.typesafe = extstartemd.current.sort != Sort.DEF;
-    }
-
-    void processUnary(final UnaryContext ctx) {
-        final ExpressionMetadata unaryemd = metadata.getExpressionMetadata(ctx);
-
-        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
-        final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-
-        if (ctx.BOOLNOT() != null) {
-            expremd.to = definition.booleanType;
-            analyzer.visit(exprctx);
-            caster.markCast(expremd);
-
-            if (expremd.postConst != null) {
-                unaryemd.preConst = !(boolean)expremd.postConst;
-            }
-
-            unaryemd.from = definition.booleanType;
-        } else if (ctx.BWNOT() != null || ctx.ADD() != null || ctx.SUB() != null) {
-            analyzer.visit(exprctx);
-
-            final Type promote = promoter.promoteNumeric(expremd.from, ctx.BWNOT() == null, true);
-
-            if (promote == null) {
-                throw new ClassCastException(AnalyzerUtility.error(ctx) + "Cannot apply [" + ctx.getChild(0).getText() + "] " +
-                    "operation to type [" + expremd.from.name + "].");
-            }
-
-            expremd.to = promote;
-            caster.markCast(expremd);
-
-            if (expremd.postConst != null) {
-                final Sort sort = promote.sort;
-
-                if (ctx.BWNOT() != null) {
-                    if (sort == Sort.INT) {
-                        unaryemd.preConst = ~(int)expremd.postConst;
-                    } else if (sort == Sort.LONG) {
-                        unaryemd.preConst = ~(long)expremd.postConst;
-                    } else {
-                        throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-                    }
-                } else if (ctx.SUB() != null) {
-                    if (exprctx instanceof NumericContext) {
-                        unaryemd.preConst = expremd.postConst;
-                    } else {
-                        if (sort == Sort.INT) {
-                            if (settings.getNumericOverflow()) {
-                                unaryemd.preConst = -(int)expremd.postConst;
-                            } else {
-                                unaryemd.preConst = Math.negateExact((int)expremd.postConst);
-                            }
-                        } else if (sort == Sort.LONG) {
-                            if (settings.getNumericOverflow()) {
-                                unaryemd.preConst = -(long)expremd.postConst;
-                            } else {
-                                unaryemd.preConst = Math.negateExact((long)expremd.postConst);
-                            }
-                        } else if (sort == Sort.FLOAT) {
-                            unaryemd.preConst = -(float)expremd.postConst;
-                        } else if (sort == Sort.DOUBLE) {
-                            unaryemd.preConst = -(double)expremd.postConst;
-                        } else {
-                            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-                        }
-                    }
-                } else if (ctx.ADD() != null) {
-                    if (sort == Sort.INT) {
-                        unaryemd.preConst = +(int)expremd.postConst;
-                    } else if (sort == Sort.LONG) {
-                        unaryemd.preConst = +(long)expremd.postConst;
-                    } else if (sort == Sort.FLOAT) {
-                        unaryemd.preConst = +(float)expremd.postConst;
-                    } else if (sort == Sort.DOUBLE) {
-                        unaryemd.preConst = +(double)expremd.postConst;
-                    } else {
-                        throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-                    }
-                } else {
-                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-                }
-            }
-
-            unaryemd.from = promote;
-            unaryemd.typesafe = expremd.typesafe;
-        } else {
-            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-        }
-    }
-
-    void processCast(final CastContext ctx) {
-        final ExpressionMetadata castemd = metadata.getExpressionMetadata(ctx);
-
-        final DecltypeContext decltypectx = ctx.decltype();
-        final ExpressionMetadata decltypemd = metadata.createExpressionMetadata(decltypectx);
-        analyzer.visit(decltypectx);
-
-        final Type type = decltypemd.from;
-        castemd.from = type;
-
-        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
-        final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-        expremd.to = type;
-        expremd.explicit = true;
-        analyzer.visit(exprctx);
-        caster.markCast(expremd);
-
-        if (expremd.postConst != null) {
-            castemd.preConst = expremd.postConst;
-        }
-
-        castemd.typesafe = expremd.typesafe && castemd.from.sort != Sort.DEF;
-    }
-
-    void processBinary(final BinaryContext ctx) {
-        final ExpressionMetadata binaryemd = metadata.getExpressionMetadata(ctx);
-
-        final ExpressionContext exprctx0 = AnalyzerUtility.updateExpressionTree(ctx.expression(0));
-        final ExpressionMetadata expremd0 = metadata.createExpressionMetadata(exprctx0);
-        analyzer.visit(exprctx0);
-
-        final ExpressionContext exprctx1 = AnalyzerUtility.updateExpressionTree(ctx.expression(1));
-        final ExpressionMetadata expremd1 = metadata.createExpressionMetadata(exprctx1);
-        analyzer.visit(exprctx1);
-
-        final boolean decimal = ctx.MUL() != null || ctx.DIV() != null || ctx.REM() != null || ctx.SUB() != null;
-        final boolean add = ctx.ADD() != null;
-        final boolean xor = ctx.BWXOR() != null;
-        final Type promote = add ? promoter.promoteAdd(expremd0.from, expremd1.from) :
-            xor ? promoter.promoteXor(expremd0.from, expremd1.from) :
-                promoter.promoteNumeric(expremd0.from, expremd1.from, decimal, true);
-
-        if (promote == null) {
-            throw new ClassCastException(AnalyzerUtility.error(ctx) + "Cannot apply [" + ctx.getChild(1).getText() + "] " +
-                "operation to types [" + expremd0.from.name + "] and [" + expremd1.from.name + "].");
-        }
-
-        final Sort sort = promote.sort;
-        expremd0.to = add && sort == Sort.STRING ? expremd0.from : promote;
-        expremd1.to = add && sort == Sort.STRING ? expremd1.from : promote;
-        caster.markCast(expremd0);
-        caster.markCast(expremd1);
-
-        if (expremd0.postConst != null && expremd1.postConst != null) {
-            if (ctx.MUL() != null) {
-                if (sort == Sort.INT) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (int)expremd0.postConst * (int)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Math.multiplyExact((int)expremd0.postConst, (int)expremd1.postConst);
-                    }
-                } else if (sort == Sort.LONG) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (long)expremd0.postConst * (long)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Math.multiplyExact((long)expremd0.postConst, (long)expremd1.postConst);
-                    }
-                } else if (sort == Sort.FLOAT) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (float)expremd0.postConst * (float)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.multiplyWithoutOverflow((float)expremd0.postConst, (float)expremd1.postConst);
-                    }
-                } else if (sort == Sort.DOUBLE) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (double)expremd0.postConst * (double)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.multiplyWithoutOverflow((double)expremd0.postConst, (double)expremd1.postConst);
-                    }
-                } else {
-                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-                }
-            } else if (ctx.DIV() != null) {
-                if (sort == Sort.INT) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (int)expremd0.postConst / (int)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.divideWithoutOverflow((int)expremd0.postConst, (int)expremd1.postConst);
-                    }
-                } else if (sort == Sort.LONG) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (long)expremd0.postConst / (long)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.divideWithoutOverflow((long)expremd0.postConst, (long)expremd1.postConst);
-                    }
-                } else if (sort == Sort.FLOAT) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (float)expremd0.postConst / (float)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.divideWithoutOverflow((float)expremd0.postConst, (float)expremd1.postConst);
-                    }
-                } else if (sort == Sort.DOUBLE) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (double)expremd0.postConst / (double)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.divideWithoutOverflow((double)expremd0.postConst, (double)expremd1.postConst);
-                    }
-                } else {
-                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-                }
-            } else if (ctx.REM() != null) {
-                if (sort == Sort.INT) {
-                    binaryemd.preConst = (int)expremd0.postConst % (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    binaryemd.preConst = (long)expremd0.postConst % (long)expremd1.postConst;
-                } else if (sort == Sort.FLOAT) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (float)expremd0.postConst % (float)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.remainderWithoutOverflow((float)expremd0.postConst, (float)expremd1.postConst);
-                    }
-                } else if (sort == Sort.DOUBLE) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (double)expremd0.postConst % (double)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.remainderWithoutOverflow((double)expremd0.postConst, (double)expremd1.postConst);
-                    }
-                } else {
-                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-                }
-            } else if (ctx.ADD() != null) {
-                if (sort == Sort.INT) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (int)expremd0.postConst + (int)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Math.addExact((int)expremd0.postConst, (int)expremd1.postConst);
-                    }
-                } else if (sort == Sort.LONG) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (long)expremd0.postConst + (long)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Math.addExact((long)expremd0.postConst, (long)expremd1.postConst);
-                    }
-                } else if (sort == Sort.FLOAT) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (float)expremd0.postConst + (float)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.addWithoutOverflow((float)expremd0.postConst, (float)expremd1.postConst);
-                    }
-                } else if (sort == Sort.DOUBLE) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (double)expremd0.postConst + (double)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.addWithoutOverflow((double)expremd0.postConst, (double)expremd1.postConst);
-                    }
-                } else if (sort == Sort.STRING) {
-                    binaryemd.preConst = "" + expremd0.postConst + expremd1.postConst;
-                } else {
-                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-                }
-            } else if (ctx.SUB() != null) {
-                if (sort == Sort.INT) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (int)expremd0.postConst - (int)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Math.subtractExact((int)expremd0.postConst, (int)expremd1.postConst);
-                    }
-                } else if (sort == Sort.LONG) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (long)expremd0.postConst - (long)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Math.subtractExact((long)expremd0.postConst, (long)expremd1.postConst);
-                    }
-                } else if (sort == Sort.FLOAT) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (float)expremd0.postConst - (float)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.subtractWithoutOverflow((float)expremd0.postConst, (float)expremd1.postConst);
-                    }
-                } else if (sort == Sort.DOUBLE) {
-                    if (settings.getNumericOverflow()) {
-                        binaryemd.preConst = (double)expremd0.postConst - (double)expremd1.postConst;
-                    } else {
-                        binaryemd.preConst = Utility.subtractWithoutOverflow((double)expremd0.postConst, (double)expremd1.postConst);
-                    }
-                } else {
-                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-                }
-            } else if (ctx.LSH() != null) {
-                if (sort == Sort.INT) {
-                    binaryemd.preConst = (int)expremd0.postConst << (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    binaryemd.preConst = (long)expremd0.postConst << (long)expremd1.postConst;
-                } else {
-                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-                }
-            } else if (ctx.RSH() != null) {
-                if (sort == Sort.INT) {
-                    binaryemd.preConst = (int)expremd0.postConst >> (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    binaryemd.preConst = (long)expremd0.postConst >> (long)expremd1.postConst;
-                } else {
-                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-                }
-            } else if (ctx.USH() != null) {
-                if (sort == Sort.INT) {
-                    binaryemd.preConst = (int)expremd0.postConst >>> (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    binaryemd.preConst = (long)expremd0.postConst >>> (long)expremd1.postConst;
-                } else {
-                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-                }
-            } else if (ctx.BWAND() != null) {
-                if (sort == Sort.INT) {
-                    binaryemd.preConst = (int)expremd0.postConst & (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    binaryemd.preConst = (long)expremd0.postConst & (long)expremd1.postConst;
-                } else {
-                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-                }
-            } else if (ctx.BWXOR() != null) {
-                if (sort == Sort.BOOL) {
-                    binaryemd.preConst = (boolean)expremd0.postConst ^ (boolean)expremd1.postConst;
-                } else if (sort == Sort.INT) {
-                    binaryemd.preConst = (int)expremd0.postConst ^ (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    binaryemd.preConst = (long)expremd0.postConst ^ (long)expremd1.postConst;
-                } else {
-                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-                }
-            } else if (ctx.BWOR() != null) {
-                if (sort == Sort.INT) {
-                    binaryemd.preConst = (int)expremd0.postConst | (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    binaryemd.preConst = (long)expremd0.postConst | (long)expremd1.postConst;
-                } else {
-                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-                }
-            } else {
-                throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-            }
-        }
-
-        binaryemd.from = promote;
-        binaryemd.typesafe = expremd0.typesafe && expremd1.typesafe;
-    }
-
-    void processComp(final CompContext ctx) {
-        final ExpressionMetadata compemd = metadata.getExpressionMetadata(ctx);
-        final boolean equality = ctx.EQ() != null || ctx.NE() != null;
-        final boolean reference = ctx.EQR() != null || ctx.NER() != null;
-
-        final ExpressionContext exprctx0 = AnalyzerUtility.updateExpressionTree(ctx.expression(0));
-        final ExpressionMetadata expremd0 = metadata.createExpressionMetadata(exprctx0);
-        analyzer.visit(exprctx0);
-
-        final ExpressionContext exprctx1 = AnalyzerUtility.updateExpressionTree(ctx.expression(1));
-        final ExpressionMetadata expremd1 = metadata.createExpressionMetadata(exprctx1);
-        analyzer.visit(exprctx1);
-
-        if (expremd0.isNull && expremd1.isNull) {
-            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Unnecessary comparison of null constants.");
-        }
-
-        final Type promote = equality ? promoter.promoteEquality(expremd0.from, expremd1.from) :
-            reference ? promoter.promoteReference(expremd0.from, expremd1.from) :
-                promoter.promoteNumeric(expremd0.from, expremd1.from, true, true);
-
-        if (promote == null) {
-            throw new ClassCastException(AnalyzerUtility.error(ctx) + "Cannot apply [" + ctx.getChild(1).getText() + "] " +
-                "operation to types [" + expremd0.from.name + "] and [" + expremd1.from.name + "].");
-        }
-
-        expremd0.to = promote;
-        expremd1.to = promote;
-        caster.markCast(expremd0);
-        caster.markCast(expremd1);
-
-        if (expremd0.postConst != null && expremd1.postConst != null) {
-            final Sort sort = promote.sort;
-
-            if (ctx.EQ() != null || ctx.EQR() != null) {
-                if (sort == Sort.BOOL) {
-                    compemd.preConst = (boolean)expremd0.postConst == (boolean)expremd1.postConst;
-                } else if (sort == Sort.INT) {
-                    compemd.preConst = (int)expremd0.postConst == (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    compemd.preConst = (long)expremd0.postConst == (long)expremd1.postConst;
-                } else if (sort == Sort.FLOAT) {
-                    compemd.preConst = (float)expremd0.postConst == (float)expremd1.postConst;
-                } else if (sort == Sort.DOUBLE) {
-                    compemd.preConst = (double)expremd0.postConst == (double)expremd1.postConst;
-                } else {
-                    if (ctx.EQ() != null && !expremd0.isNull && !expremd1.isNull) {
-                        compemd.preConst = expremd0.postConst.equals(expremd1.postConst);
-                    } else if (ctx.EQR() != null) {
-                        compemd.preConst = expremd0.postConst == expremd1.postConst;
-                    } else {
-                        throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-                    }
-                }
-            } else if (ctx.NE() != null || ctx.NER() != null) {
-                if (sort == Sort.BOOL) {
-                    compemd.preConst = (boolean)expremd0.postConst != (boolean)expremd1.postConst;
-                } else if (sort == Sort.INT) {
-                    compemd.preConst = (int)expremd0.postConst != (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    compemd.preConst = (long)expremd0.postConst != (long)expremd1.postConst;
-                } else if (sort == Sort.FLOAT) {
-                    compemd.preConst = (float)expremd0.postConst != (float)expremd1.postConst;
-                } else if (sort == Sort.DOUBLE) {
-                    compemd.preConst = (double)expremd0.postConst != (double)expremd1.postConst;
-                } else {
-                    if (ctx.NE() != null && !expremd0.isNull && !expremd1.isNull) {
-                        compemd.preConst = expremd0.postConst.equals(expremd1.postConst);
-                    } else if (ctx.NER() != null) {
-                        compemd.preConst = expremd0.postConst == expremd1.postConst;
-                    } else {
-                        throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-                    }
-                }
-            } else if (ctx.GTE() != null) {
-                if (sort == Sort.INT) {
-                    compemd.preConst = (int)expremd0.postConst >= (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    compemd.preConst = (long)expremd0.postConst >= (long)expremd1.postConst;
-                } else if (sort == Sort.FLOAT) {
-                    compemd.preConst = (float)expremd0.postConst >= (float)expremd1.postConst;
-                } else if (sort == Sort.DOUBLE) {
-                    compemd.preConst = (double)expremd0.postConst >= (double)expremd1.postConst;
-                } else {
-                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-                }
-            } else if (ctx.GT() != null) {
-                if (sort == Sort.INT) {
-                    compemd.preConst = (int)expremd0.postConst > (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    compemd.preConst = (long)expremd0.postConst > (long)expremd1.postConst;
-                } else if (sort == Sort.FLOAT) {
-                    compemd.preConst = (float)expremd0.postConst > (float)expremd1.postConst;
-                } else if (sort == Sort.DOUBLE) {
-                    compemd.preConst = (double)expremd0.postConst > (double)expremd1.postConst;
-                } else {
-                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-                }
-            } else if (ctx.LTE() != null) {
-                if (sort == Sort.INT) {
-                    compemd.preConst = (int)expremd0.postConst <= (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    compemd.preConst = (long)expremd0.postConst <= (long)expremd1.postConst;
-                } else if (sort == Sort.FLOAT) {
-                    compemd.preConst = (float)expremd0.postConst <= (float)expremd1.postConst;
-                } else if (sort == Sort.DOUBLE) {
-                    compemd.preConst = (double)expremd0.postConst <= (double)expremd1.postConst;
-                } else {
-                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-                }
-            } else if (ctx.LT() != null) {
-                if (sort == Sort.INT) {
-                    compemd.preConst = (int)expremd0.postConst < (int)expremd1.postConst;
-                } else if (sort == Sort.LONG) {
-                    compemd.preConst = (long)expremd0.postConst < (long)expremd1.postConst;
-                } else if (sort == Sort.FLOAT) {
-                    compemd.preConst = (float)expremd0.postConst < (float)expremd1.postConst;
-                } else if (sort == Sort.DOUBLE) {
-                    compemd.preConst = (double)expremd0.postConst < (double)expremd1.postConst;
-                } else {
-                    throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-                }
-            } else {
-                throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-            }
-        }
-
-        compemd.from = definition.booleanType;
-        compemd.typesafe = expremd0.typesafe && expremd1.typesafe;
-    }
-
-    void processBool(final BoolContext ctx) {
-        final ExpressionMetadata boolemd = metadata.getExpressionMetadata(ctx);
-
-        final ExpressionContext exprctx0 = AnalyzerUtility.updateExpressionTree(ctx.expression(0));
-        final ExpressionMetadata expremd0 = metadata.createExpressionMetadata(exprctx0);
-        expremd0.to = definition.booleanType;
-        analyzer.visit(exprctx0);
-        caster.markCast(expremd0);
-
-        final ExpressionContext exprctx1 = AnalyzerUtility.updateExpressionTree(ctx.expression(1));
-        final ExpressionMetadata expremd1 = metadata.createExpressionMetadata(exprctx1);
-        expremd1.to = definition.booleanType;
-        analyzer.visit(exprctx1);
-        caster.markCast(expremd1);
-
-        if (expremd0.postConst != null && expremd1.postConst != null) {
-            if (ctx.BOOLAND() != null) {
-                boolemd.preConst = (boolean)expremd0.postConst && (boolean)expremd1.postConst;
-            } else if (ctx.BOOLOR() != null) {
-                boolemd.preConst = (boolean)expremd0.postConst || (boolean)expremd1.postConst;
-            } else {
-                throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-            }
-        }
-
-        boolemd.from = definition.booleanType;
-        boolemd.typesafe = expremd0.typesafe && expremd1.typesafe;
-    }
-
-    void processConditional(final ConditionalContext ctx) {
-        final ExpressionMetadata condemd = metadata.getExpressionMetadata(ctx);
-
-        final ExpressionContext exprctx0 = AnalyzerUtility.updateExpressionTree(ctx.expression(0));
-        final ExpressionMetadata expremd0 = metadata.createExpressionMetadata(exprctx0);
-        expremd0.to = definition.booleanType;
-        analyzer.visit(exprctx0);
-        caster.markCast(expremd0);
-
-        if (expremd0.postConst != null) {
-            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Unnecessary conditional statement.");
-        }
-
-        final ExpressionContext exprctx1 = AnalyzerUtility.updateExpressionTree(ctx.expression(1));
-        final ExpressionMetadata expremd1 = metadata.createExpressionMetadata(exprctx1);
-        expremd1.to = condemd.to;
-        expremd1.explicit = condemd.explicit;
-        analyzer.visit(exprctx1);
-
-        final ExpressionContext exprctx2 = AnalyzerUtility.updateExpressionTree(ctx.expression(2));
-        final ExpressionMetadata expremd2 = metadata.createExpressionMetadata(exprctx2);
-        expremd2.to = condemd.to;
-        expremd2.explicit = condemd.explicit;
-        analyzer.visit(exprctx2);
-
-        if (condemd.to == null) {
-            final Type promote = promoter.promoteConditional(expremd1.from, expremd2.from, expremd1.preConst, expremd2.preConst);
-
-            expremd1.to = promote;
-            expremd2.to = promote;
-            condemd.from = promote;
-        } else {
-            condemd.from = condemd.to;
-        }
-
-        caster.markCast(expremd1);
-        caster.markCast(expremd2);
-
-        condemd.typesafe = expremd0.typesafe && expremd1.typesafe;
-    }
-
-    void processAssignment(final AssignmentContext ctx) {
-        final ExpressionMetadata assignemd = metadata.getExpressionMetadata(ctx);
-
-        final ExtstartContext extstartctx = ctx.extstart();
-        final ExternalMetadata extstartemd = metadata.createExternalMetadata(extstartctx);
-
-        extstartemd.read = assignemd.read;
-        extstartemd.storeExpr = AnalyzerUtility.updateExpressionTree(ctx.expression());
-
-        if (ctx.AMUL() != null) {
-            extstartemd.token = MUL;
-        } else if (ctx.ADIV() != null) {
-            extstartemd.token = DIV;
-        } else if (ctx.AREM() != null) {
-            extstartemd.token = REM;
-        } else if (ctx.AADD() != null) {
-            extstartemd.token = ADD;
-        } else if (ctx.ASUB() != null) {
-            extstartemd.token = SUB;
-        } else if (ctx.ALSH() != null) {
-            extstartemd.token = LSH;
-        } else if (ctx.AUSH() != null) {
-            extstartemd.token = USH;
-        } else if (ctx.ARSH() != null) {
-            extstartemd.token = RSH;
-        } else if (ctx.AAND() != null) {
-            extstartemd.token = BWAND;
-        } else if (ctx.AXOR() != null) {
-            extstartemd.token = BWXOR;
-        } else if (ctx.AOR() != null) {
-            extstartemd.token = BWOR;
-        }
-
-        analyzer.visit(extstartctx);
-
-        assignemd.statement = true;
-        assignemd.from = extstartemd.read ? extstartemd.current : definition.voidType;
-        assignemd.typesafe = extstartemd.current.sort != Sort.DEF;
-    }
-
-    void processIncrement(final IncrementContext ctx) {
-        final ExpressionMetadata incremd = metadata.getExpressionMetadata(ctx);
-        final Sort sort = incremd.to == null ? null : incremd.to.sort;
-        final boolean positive = ctx.INCR() != null;
-
-        if (incremd.to == null) {
-            incremd.preConst = positive ? 1 : -1;
-            incremd.from = definition.intType;
-        } else {
-            switch (sort) {
-                case LONG:
-                    incremd.preConst = positive ? 1L : -1L;
-                    incremd.from = definition.longType;
-                    break;
-                case FLOAT:
-                    incremd.preConst = positive ? 1.0F : -1.0F;
-                    incremd.from = definition.floatType;
-                    break;
-                case DOUBLE:
-                    incremd.preConst = positive ? 1.0 : -1.0;
-                    incremd.from = definition.doubleType;
-                    break;
-                default:
-                    incremd.preConst = positive ? 1 : -1;
-                    incremd.from = definition.intType;
-            }
-        }
-    }
-}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java
deleted file mode 100644
index 53b7ab6fd96..00000000000
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerExternal.java
+++ /dev/null
@@ -1,813 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless;
-
-import org.antlr.v4.runtime.ParserRuleContext;
-import org.elasticsearch.painless.AnalyzerUtility.Variable;
-import org.elasticsearch.painless.Definition.Constructor;
-import org.elasticsearch.painless.Definition.Field;
-import org.elasticsearch.painless.Definition.Method;
-import org.elasticsearch.painless.Definition.Sort;
-import org.elasticsearch.painless.Definition.Struct;
-import org.elasticsearch.painless.Definition.Type;
-import org.elasticsearch.painless.Metadata.ExpressionMetadata;
-import org.elasticsearch.painless.Metadata.ExtNodeMetadata;
-import org.elasticsearch.painless.Metadata.ExternalMetadata;
-import org.elasticsearch.painless.PainlessParser.DecltypeContext;
-import org.elasticsearch.painless.PainlessParser.ExpressionContext;
-import org.elasticsearch.painless.PainlessParser.ExtbraceContext;
-import org.elasticsearch.painless.PainlessParser.ExtcallContext;
-import org.elasticsearch.painless.PainlessParser.ExtcastContext;
-import org.elasticsearch.painless.PainlessParser.ExtdotContext;
-import org.elasticsearch.painless.PainlessParser.ExtfieldContext;
-import org.elasticsearch.painless.PainlessParser.ExtnewContext;
-import org.elasticsearch.painless.PainlessParser.ExtprecContext;
-import org.elasticsearch.painless.PainlessParser.ExtstartContext;
-import org.elasticsearch.painless.PainlessParser.ExtstringContext;
-import org.elasticsearch.painless.PainlessParser.ExtvarContext;
-import org.elasticsearch.painless.PainlessParser.IdentifierContext;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import static org.elasticsearch.painless.PainlessParser.ADD;
-import static org.elasticsearch.painless.PainlessParser.BWAND;
-import static org.elasticsearch.painless.PainlessParser.BWOR;
-import static org.elasticsearch.painless.PainlessParser.BWXOR;
-import static org.elasticsearch.painless.PainlessParser.DIV;
-import static org.elasticsearch.painless.PainlessParser.MUL;
-import static org.elasticsearch.painless.PainlessParser.REM;
-import static org.elasticsearch.painless.PainlessParser.SUB;
-
-class AnalyzerExternal {
-    private final Metadata metadata;
-    private final Definition definition;
-
-    private final Analyzer analyzer;
-    private final AnalyzerUtility utility;
-    private final AnalyzerCaster caster;
-    private final AnalyzerPromoter promoter;
-
-    AnalyzerExternal(final Metadata metadata, final Analyzer analyzer, final AnalyzerUtility utility,
-                     final AnalyzerCaster caster, final AnalyzerPromoter promoter) {
-        this.metadata = metadata;
-        this.definition = metadata.definition;
-
-        this.analyzer = analyzer;
-        this.utility = utility;
-        this.caster = caster;
-        this.promoter = promoter;
-    }
-
-    void processExtstart(final ExtstartContext ctx) {
-        final ExtprecContext precctx = ctx.extprec();
-        final ExtcastContext castctx = ctx.extcast();
-        final ExtvarContext varctx = ctx.extvar();
-        final ExtnewContext newctx = ctx.extnew();
-        final ExtstringContext stringctx = ctx.extstring();
-
-        if (precctx != null) {
-            metadata.createExtNodeMetadata(ctx, precctx);
-            analyzer.visit(precctx);
-        } else if (castctx != null) {
-            metadata.createExtNodeMetadata(ctx, castctx);
-            analyzer.visit(castctx);
-        } else if (varctx != null) {
-            metadata.createExtNodeMetadata(ctx, varctx);
-            analyzer.visit(varctx);
-        } else if (newctx != null) {
-            metadata.createExtNodeMetadata(ctx, newctx);
-            analyzer.visit(newctx);
-        } else if (stringctx != null) {
-            metadata.createExtNodeMetadata(ctx, stringctx);
-            analyzer.visit(stringctx);
-        } else {
-            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-        }
-    }
-
-    void processExtprec(final ExtprecContext ctx) {
-        final ExtNodeMetadata precenmd = metadata.getExtNodeMetadata(ctx);
-        final ParserRuleContext parent = precenmd.parent;
-        final ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
-
-        final ExtprecContext precctx = ctx.extprec();
-        final ExtcastContext castctx = ctx.extcast();
-        final ExtvarContext varctx = ctx.extvar();
-        final ExtnewContext newctx = ctx.extnew();
-        final ExtstringContext stringctx = ctx.extstring();
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        if (dotctx != null || bracectx != null) {
-            ++parentemd.scope;
-        }
-
-        if (precctx != null) {
-            metadata.createExtNodeMetadata(parent, precctx);
-            analyzer.visit(precctx);
-        } else if (castctx != null) {
-            metadata.createExtNodeMetadata(parent, castctx);
-            analyzer.visit(castctx);
-        } else if (varctx != null) {
-            metadata.createExtNodeMetadata(parent, varctx);
-            analyzer.visit(varctx);
-        } else if (newctx != null) {
-            metadata.createExtNodeMetadata(parent, newctx);
-            analyzer.visit(newctx);
-        } else if (stringctx != null) {
-            metadata.createExtNodeMetadata(ctx, stringctx);
-            analyzer.visit(stringctx);
-        } else {
-            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-        }
-
-        parentemd.statement = false;
-
-        if (dotctx != null) {
-            --parentemd.scope;
-
-            metadata.createExtNodeMetadata(parent, dotctx);
-            analyzer.visit(dotctx);
-        } else if (bracectx != null) {
-            --parentemd.scope;
-
-            metadata.createExtNodeMetadata(parent, bracectx);
-            analyzer.visit(bracectx);
-        }
-    }
-
-    void processExtcast(final ExtcastContext ctx) {
-        final ExtNodeMetadata castenmd = metadata.getExtNodeMetadata(ctx);
-        final ParserRuleContext parent = castenmd.parent;
-        final ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
-
-        final ExtprecContext precctx = ctx.extprec();
-        final ExtcastContext castctx = ctx.extcast();
-        final ExtvarContext varctx = ctx.extvar();
-        final ExtnewContext newctx = ctx.extnew();
-        final ExtstringContext stringctx = ctx.extstring();
-
-        if (precctx != null) {
-            metadata.createExtNodeMetadata(parent, precctx);
-            analyzer.visit(precctx);
-        } else if (castctx != null) {
-            metadata.createExtNodeMetadata(parent, castctx);
-            analyzer.visit(castctx);
-        } else if (varctx != null) {
-            metadata.createExtNodeMetadata(parent, varctx);
-            analyzer.visit(varctx);
-        } else if (newctx != null) {
-            metadata.createExtNodeMetadata(parent, newctx);
-            analyzer.visit(newctx);
-        } else if (stringctx != null) {
-            metadata.createExtNodeMetadata(ctx, stringctx);
-            analyzer.visit(stringctx);
-        } else {
-            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-        }
-
-        final DecltypeContext declctx = ctx.decltype();
-        final ExpressionMetadata declemd = metadata.createExpressionMetadata(declctx);
-        analyzer.visit(declctx);
-
-        castenmd.castTo = caster.getLegalCast(ctx, parentemd.current, declemd.from, true);
-        castenmd.type = declemd.from;
-        parentemd.current = declemd.from;
-        parentemd.statement = false;
-    }
-
-    void processExtbrace(final ExtbraceContext ctx) {
-        final ExtNodeMetadata braceenmd = metadata.getExtNodeMetadata(ctx);
-        final ParserRuleContext parent = braceenmd.parent;
-        final ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
-
-        final boolean array = parentemd.current.sort == Sort.ARRAY;
-        final boolean def = parentemd.current.sort == Sort.DEF;
-        boolean map = false;
-        boolean list = false;
-
-        try {
-            parentemd.current.clazz.asSubclass(Map.class);
-            map = true;
-        } catch (final ClassCastException exception) {
-            // Do nothing.
-        }
-
-        try {
-            parentemd.current.clazz.asSubclass(List.class);
-            list = true;
-        } catch (final ClassCastException exception) {
-            // Do nothing.
-        }
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        braceenmd.last = parentemd.scope == 0 && dotctx == null && bracectx == null;
-
-        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
-        final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-
-        if (array || def) {
-            expremd.to = array ? definition.intType : definition.objectType;
-            analyzer.visit(exprctx);
-            caster.markCast(expremd);
-
-            braceenmd.target = "#brace";
-            braceenmd.type = def ? definition.defType :
-                definition.getType(parentemd.current.struct, parentemd.current.type.getDimensions() - 1);
-            analyzeLoadStoreExternal(ctx);
-            parentemd.current = braceenmd.type;
-        } else {
-            final boolean store = braceenmd.last && parentemd.storeExpr != null;
-            final boolean get = parentemd.read || parentemd.token > 0 || !braceenmd.last;
-            final boolean set = braceenmd.last && store;
-
-            Method getter;
-            Method setter;
-            Type valuetype;
-            Type settype;
-
-            if (map) {
-                getter = parentemd.current.struct.methods.get("get");
-                setter = parentemd.current.struct.methods.get("put");
-
-                if (getter != null && (getter.rtn.sort == Sort.VOID || getter.arguments.size() != 1)) {
-                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
-                        "Illegal map get shortcut for type [" + parentemd.current.name + "].");
-                }
-
-                if (setter != null && setter.arguments.size() != 2) {
-                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
-                        "Illegal map set shortcut for type [" + parentemd.current.name + "].");
-                }
-
-                if (getter != null && setter != null && (!getter.arguments.get(0).equals(setter.arguments.get(0))
-                    || !getter.rtn.equals(setter.arguments.get(1)))) {
-                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Shortcut argument types must match.");
-                }
-
-                valuetype = setter != null ? setter.arguments.get(0) : getter != null ? getter.arguments.get(0) : null;
-                settype = setter == null ? null : setter.arguments.get(1);
-            } else if (list) {
-                getter = parentemd.current.struct.methods.get("get");
-                setter = parentemd.current.struct.methods.get("set");
-
-                if (getter != null && (getter.rtn.sort == Sort.VOID || getter.arguments.size() != 1 ||
-                    getter.arguments.get(0).sort != Sort.INT)) {
-                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
-                        "Illegal list get shortcut for type [" + parentemd.current.name + "].");
-                }
-
-                if (setter != null && (setter.arguments.size() != 2 || setter.arguments.get(0).sort != Sort.INT)) {
-                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
-                        "Illegal list set shortcut for type [" + parentemd.current.name + "].");
-                }
-
-                if (getter != null && setter != null && (!getter.arguments.get(0).equals(setter.arguments.get(0))
-                    || !getter.rtn.equals(setter.arguments.get(1)))) {
-                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Shortcut argument types must match.");
-                }
-
-                valuetype = definition.intType;
-                settype = setter == null ? null : setter.arguments.get(1);
-            } else {
-                throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-            }
-
-            if ((get || set) && (!get || getter != null) && (!set || setter != null)) {
-                expremd.to = valuetype;
-                analyzer.visit(exprctx);
-                caster.markCast(expremd);
-
-                braceenmd.target = new Object[] {getter, setter, true, null};
-                braceenmd.type = get ? getter.rtn : settype;
-                analyzeLoadStoreExternal(ctx);
-                parentemd.current = get ? getter.rtn : setter.rtn;
-            }
-        }
-
-        if (braceenmd.target == null) {
-            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
-                "Attempting to address a non-array type [" + parentemd.current.name + "] as an array.");
-        }
-
-        if (dotctx != null) {
-            metadata.createExtNodeMetadata(parent, dotctx);
-            analyzer.visit(dotctx);
-        } else if (bracectx != null) {
-            metadata.createExtNodeMetadata(parent, bracectx);
-            analyzer.visit(bracectx);
-        }
-    }
-
-    void processExtdot(final ExtdotContext ctx) {
-        final ExtNodeMetadata dotemnd = metadata.getExtNodeMetadata(ctx);
-        final ParserRuleContext parent = dotemnd.parent;
-
-        final ExtcallContext callctx = ctx.extcall();
-        final ExtfieldContext fieldctx = ctx.extfield();
-
-        if (callctx != null) {
-            metadata.createExtNodeMetadata(parent, callctx);
-            analyzer.visit(callctx);
-        } else if (fieldctx != null) {
-            metadata.createExtNodeMetadata(parent, fieldctx);
-            analyzer.visit(fieldctx);
-        }
-    }
-
-    void processExtcall(final ExtcallContext ctx) {
-        final ExtNodeMetadata callenmd = metadata.getExtNodeMetadata(ctx);
-        final ParserRuleContext parent = callenmd.parent;
-        final ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        callenmd.last = parentemd.scope == 0 && dotctx == null && bracectx == null;
-
-        final String name = ctx.EXTID().getText();
-
-        if (parentemd.current.sort == Sort.ARRAY) {
-            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Unexpected call [" + name + "] on an array.");
-        } else if (callenmd.last && parentemd.storeExpr != null) {
-            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Cannot assign a value to a call [" + name + "].");
-        }
-
-        final Struct struct = parentemd.current.struct;
-        final List<ExpressionContext> arguments = ctx.arguments().expression();
-        final int size = arguments.size();
-        Type[] types;
-
-        final Method method = parentemd.statik ? struct.functions.get(name) : struct.methods.get(name);
-        final boolean def = parentemd.current.sort == Sort.DEF;
-
-        if (method == null && !def) {
-            throw new IllegalArgumentException(
-                AnalyzerUtility.error(ctx) + "Unknown call [" + name + "] on type [" + struct.name + "].");
-        } else if (method != null) {
-            types = new Type[method.arguments.size()];
-            method.arguments.toArray(types);
-
-            callenmd.target = method;
-            callenmd.type = method.rtn;
-            parentemd.statement = !parentemd.read && callenmd.last;
-            parentemd.current = method.rtn;
-
-            if (size != types.length) {
-                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "When calling [" + name + "] on type " +
-                    "[" + struct.name + "] expected [" + types.length + "] arguments," +
-                    " but found [" + arguments.size() + "].");
-            }
-        } else {
-            types = new Type[arguments.size()];
-            Arrays.fill(types, definition.defType);
-
-            callenmd.target = name;
-            callenmd.type = definition.defType;
-            parentemd.statement = !parentemd.read && callenmd.last;
-            parentemd.current = callenmd.type;
-        }
-
-        for (int argument = 0; argument < size; ++argument) {
-            final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(arguments.get(argument));
-            final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-            expremd.to = types[argument];
-            analyzer.visit(exprctx);
-            caster.markCast(expremd);
-        }
-
-        parentemd.statik = false;
-
-        if (dotctx != null) {
-            metadata.createExtNodeMetadata(parent, dotctx);
-            analyzer.visit(dotctx);
-        } else if (bracectx != null) {
-            metadata.createExtNodeMetadata(parent, bracectx);
-            analyzer.visit(bracectx);
-        }
-    }
-
-    void processExtvar(final ExtvarContext ctx) {
-        final ExtNodeMetadata varenmd = metadata.getExtNodeMetadata(ctx);
-        final ParserRuleContext parent = varenmd.parent;
-        final ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
-
-        final IdentifierContext idctx = ctx.identifier();
-        final String id = idctx.getText();
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        final boolean type = utility.isValidType(idctx, false);
-
-        if (type) {
-            if (parentemd.current != null || dotctx == null || bracectx != null) {
-                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Unexpected static type [" + id + "].");
-            }
-
-            varenmd.type = definition.getType(id);
-            parentemd.current = varenmd.type;
-            parentemd.statik = true;
-
-            metadata.createExtNodeMetadata(parent, dotctx);
-            analyzer.visit(dotctx);
-        } else {
-            utility.isValidIdentifier(idctx, true);
-
-            if (parentemd.current != null) {
-                throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected variable [" + id + "] load.");
-            }
-
-            varenmd.last = parentemd.scope == 0 && dotctx == null && bracectx == null;
-
-            final Variable variable = utility.getVariable(id);
-
-            if (variable == null) {
-                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Unknown variable [" + id + "].");
-            }
-
-            // special cases: reserved words
-            if ("_score".equals(id) || "_value".equals(id) || "doc".equals(id) || "ctx".equals(id)) {
-                // read-only: don't allow stores to ourself
-                if (varenmd.last && parentemd.storeExpr != null) {
-                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Variable [" + id + "] is read-only.");
-                }
-                if ("_score".equals(id)) {
-                    // track if the _score value is ever used, we will invoke Scorer.score() only once if so.
-                    metadata.scoreValueUsed = true;
-                } else if ("ctx".equals(id)) {
-                    // track if ctx value is ever used, we will invoke Map.get() only once if so.
-                    metadata.ctxValueUsed = true;
-                }
-            }
-
-            varenmd.target = variable.slot;
-            varenmd.type = variable.type;
-            analyzeLoadStoreExternal(ctx);
-            parentemd.current = varenmd.type;
-
-            if (dotctx != null) {
-                metadata.createExtNodeMetadata(parent, dotctx);
-                analyzer.visit(dotctx);
-            } else if (bracectx != null) {
-                metadata.createExtNodeMetadata(parent, bracectx);
-                analyzer.visit(bracectx);
-            }
-        }
-    }
-
-    void processExtfield(final ExtfieldContext ctx) {
-        final ExtNodeMetadata memberenmd = metadata.getExtNodeMetadata(ctx);
-        final ParserRuleContext parent = memberenmd.parent;
-        final ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
-
-        if (ctx.EXTID() == null && ctx.EXTINTEGER() == null) {
-            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-        }
-
-        final String value = ctx.EXTID() == null ? ctx.EXTINTEGER().getText() : ctx.EXTID().getText();
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        memberenmd.last = parentemd.scope == 0 && dotctx == null && bracectx == null;
-        final boolean store = memberenmd.last && parentemd.storeExpr != null;
-
-        if (parentemd.current == null) {
-            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected field [" + value + "] load.");
-        }
-
-        if (parentemd.current.sort == Sort.ARRAY) {
-            if ("length".equals(value)) {
-                if (!parentemd.read) {
-                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Must read array field [length].");
-                } else if (store) {
-                    throw new IllegalArgumentException(
-                        AnalyzerUtility.error(ctx) + "Cannot write to read-only array field [length].");
-                }
-
-                memberenmd.target = "#length";
-                memberenmd.type = definition.intType;
-                parentemd.current = definition.intType;
-            } else {
-                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Unexpected array field [" + value + "].");
-            }
-        } else if (parentemd.current.sort == Sort.DEF) {
-            memberenmd.target = value;
-            memberenmd.type = definition.defType;
-            analyzeLoadStoreExternal(ctx);
-            parentemd.current = memberenmd.type;
-        } else {
-            final Struct struct = parentemd.current.struct;
-            final Field field = parentemd.statik ? struct.statics.get(value) : struct.members.get(value);
-
-            if (field != null) {
-                if (store && java.lang.reflect.Modifier.isFinal(field.reflect.getModifiers())) {
-                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Cannot write to read-only" +
-                        " field [" + value + "] for type [" + struct.name + "].");
-                }
-
-                memberenmd.target = field;
-                memberenmd.type = field.type;
-                analyzeLoadStoreExternal(ctx);
-                parentemd.current = memberenmd.type;
-            } else {
-                final boolean get = parentemd.read || parentemd.token > 0 || !memberenmd.last;
-                final boolean set = memberenmd.last && store;
-
-                Method getter = struct.methods.get("get" + Character.toUpperCase(value.charAt(0)) + value.substring(1));
-                Method setter = struct.methods.get("set" + Character.toUpperCase(value.charAt(0)) + value.substring(1));
-                Object constant = null;
-
-                if (getter != null && (getter.rtn.sort == Sort.VOID || !getter.arguments.isEmpty())) {
-                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
-                        "Illegal get shortcut on field [" + value + "] for type [" + struct.name + "].");
-                }
-
-                if (setter != null && (setter.rtn.sort != Sort.VOID || setter.arguments.size() != 1)) {
-                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
-                        "Illegal set shortcut on field [" + value + "] for type [" + struct.name + "].");
-                }
-
-                Type settype = setter == null ? null : setter.arguments.get(0);
-
-                if (getter == null && setter == null) {
-                    if (ctx.EXTID() != null) {
-                        try {
-                            parentemd.current.clazz.asSubclass(Map.class);
-
-                            getter = parentemd.current.struct.methods.get("get");
-                            setter = parentemd.current.struct.methods.get("put");
-
-                            if (getter != null && (getter.rtn.sort == Sort.VOID || getter.arguments.size() != 1 ||
-                                getter.arguments.get(0).sort != Sort.STRING)) {
-                                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
-                                    "Illegal map get shortcut [" + value + "] for type [" + struct.name + "].");
-                            }
-
-                            if (setter != null && (setter.arguments.size() != 2 ||
-                                setter.arguments.get(0).sort != Sort.STRING)) {
-                                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
-                                    "Illegal map set shortcut [" + value + "] for type [" + struct.name + "].");
-                            }
-
-                            if (getter != null && setter != null && !getter.rtn.equals(setter.arguments.get(1))) {
-                                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Shortcut argument types must match.");
-                            }
-
-                            settype = setter == null ? null : setter.arguments.get(1);
-                            constant = value;
-                        } catch (ClassCastException exception) {
-                            //Do nothing.
-                        }
-                    } else if (ctx.EXTINTEGER() != null) {
-                        try {
-                            parentemd.current.clazz.asSubclass(List.class);
-
-                            getter = parentemd.current.struct.methods.get("get");
-                            setter = parentemd.current.struct.methods.get("set");
-
-                            if (getter != null && (getter.rtn.sort == Sort.VOID || getter.arguments.size() != 1 ||
-                                getter.arguments.get(0).sort != Sort.INT)) {
-                                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
-                                    "Illegal list get shortcut [" + value + "] for type [" + struct.name + "].");
-                            }
-
-                            if (setter != null && (setter.arguments.size() != 2 || setter.arguments.get(0).sort != Sort.INT)) {
-                                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
-                                    "Illegal list set shortcut [" + value + "] for type [" + struct.name + "].");
-                            }
-
-                            if (getter != null && setter != null && !getter.rtn.equals(setter.arguments.get(1))) {
-                                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Shortcut argument types must match.");
-                            }
-
-                            settype = setter == null ? null : setter.arguments.get(1);
-
-                            try {
-                                constant = Integer.parseInt(value);
-                            } catch (NumberFormatException exception) {
-                                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) +
-                                    "Illegal list shortcut value [" + value + "].");
-                            }
-                        } catch (ClassCastException exception) {
-                            //Do nothing.
-                        }
-                    } else {
-                        throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-                    }
-                }
-
-                if ((get || set) && (!get || getter != null) && (!set || setter != null)) {
-                    memberenmd.target = new Object[] {getter, setter, constant != null, constant};
-                    memberenmd.type = get ? getter.rtn : settype;
-                    analyzeLoadStoreExternal(ctx);
-                    parentemd.current = get ? getter.rtn : setter.rtn;
-                }
-            }
-
-            if (memberenmd.target == null) {
-                throw new IllegalArgumentException(
-                    AnalyzerUtility.error(ctx) + "Unknown field [" + value + "] for type [" + struct.name + "].");
-            }
-        }
-
-        parentemd.statik = false;
-
-        if (dotctx != null) {
-            metadata.createExtNodeMetadata(parent, dotctx);
-            analyzer.visit(dotctx);
-        } else if (bracectx != null) {
-            metadata.createExtNodeMetadata(parent, bracectx);
-            analyzer.visit(bracectx);
-        }
-    }
-
-    void processExtnew(final ExtnewContext ctx) {
-        final ExtNodeMetadata newenmd = metadata.getExtNodeMetadata(ctx);
-        final ParserRuleContext parent = newenmd.parent;
-        final ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
-
-        final ExtdotContext dotctx = ctx.extdot();
-        newenmd.last = parentemd.scope == 0 && dotctx == null;
-
-        final IdentifierContext idctx = ctx.identifier();
-        final String type = idctx.getText();
-        utility.isValidType(idctx, true);
-
-        if (parentemd.current != null) {
-            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Unexpected new call.");
-        } else if (newenmd.last && parentemd.storeExpr != null) {
-            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Cannot assign a value to a new call.");
-        }
-
-        final Struct struct = definition.structs.get(type);
-
-        final boolean newclass = ctx.arguments() != null;
-        final boolean newarray = !ctx.expression().isEmpty();
-
-        final List<ExpressionContext> arguments = newclass ? ctx.arguments().expression() : ctx.expression();
-        final int size = arguments.size();
-
-        Type[] types;
-
-        if (newarray) {
-            if (!parentemd.read) {
-                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "A newly created array must be assigned.");
-            }
-
-            types = new Type[size];
-            Arrays.fill(types, definition.intType);
-
-            newenmd.target = "#makearray";
-
-            if (size > 1) {
-                newenmd.type = definition.getType(struct, size);
-                parentemd.current = newenmd.type;
-            } else if (size == 1) {
-                newenmd.type = definition.getType(struct, 0);
-                parentemd.current = definition.getType(struct, 1);
-            } else {
-                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "A newly created array cannot have zero dimensions.");
-            }
-        } else if (newclass) {
-            final Constructor constructor = struct.constructors.get("new");
-
-            if (constructor != null) {
-                types = new Type[constructor.arguments.size()];
-                constructor.arguments.toArray(types);
-
-                newenmd.target = constructor;
-                newenmd.type = definition.getType(struct, 0);
-                parentemd.statement = !parentemd.read && newenmd.last;
-                parentemd.current = newenmd.type;
-            } else {
-                throw new IllegalArgumentException(
-                    AnalyzerUtility.error(ctx) + "Unknown new call on type [" + struct.name + "].");
-            }
-        } else {
-            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Unknown state.");
-        }
-
-        if (size != types.length) {
-            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "When calling constructor on type " +
-                "[" + struct.name + "] expected [" + types.length + "] arguments," +
-                " but found [" + arguments.size() + "].");
-        }
-
-        for (int argument = 0; argument < size; ++argument) {
-            final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(arguments.get(argument));
-            final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-            expremd.to = types[argument];
-            analyzer.visit(exprctx);
-            caster.markCast(expremd);
-        }
-
-        if (dotctx != null) {
-            metadata.createExtNodeMetadata(parent, dotctx);
-            analyzer.visit(dotctx);
-        }
-    }
-
-    void processExtstring(final ExtstringContext ctx) {
-        final ExtNodeMetadata memberenmd = metadata.getExtNodeMetadata(ctx);
-        final ParserRuleContext parent = memberenmd.parent;
-        final ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
-
-        final String string = ctx.STRING().getText().substring(1, ctx.STRING().getText().length() - 1);
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        memberenmd.last = parentemd.scope == 0 && dotctx == null && bracectx == null;
-        final boolean store = memberenmd.last && parentemd.storeExpr != null;
-
-        if (parentemd.current != null) {
-            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected String constant [" + string + "].");
-        }
-
-        if (!parentemd.read) {
-            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Must read String constant [" + string + "].");
-        } else if (store) {
-            throw new IllegalArgumentException(
-                AnalyzerUtility.error(ctx) + "Cannot write to read-only String constant [" + string + "].");
-        }
-
-        memberenmd.target = string;
-        memberenmd.type = definition.stringType;
-        parentemd.current = definition.stringType;
-
-        if (memberenmd.last) {
-            parentemd.constant = string;
-        }
-
-        if (dotctx != null) {
-            metadata.createExtNodeMetadata(parent, dotctx);
-            analyzer.visit(dotctx);
-        } else if (bracectx != null) {
-            metadata.createExtNodeMetadata(parent, bracectx);
-            analyzer.visit(bracectx);
-        }
-    }
-
-    private void analyzeLoadStoreExternal(final ParserRuleContext source) {
-        final ExtNodeMetadata extenmd = metadata.getExtNodeMetadata(source);
-        final ParserRuleContext parent = extenmd.parent;
-        final ExternalMetadata parentemd = metadata.getExternalMetadata(parent);
-
-        if (extenmd.last && parentemd.storeExpr != null) {
-            final ParserRuleContext store = parentemd.storeExpr;
-            final ExpressionMetadata storeemd = metadata.createExpressionMetadata(parentemd.storeExpr);
-            final int token = parentemd.token;
-
-            if (token > 0) {
-                analyzer.visit(store);
-
-                final boolean add = token == ADD;
-                final boolean xor = token == BWAND || token == BWXOR || token == BWOR;
-                final boolean decimal = token == MUL || token == DIV || token == REM || token == SUB;
-
-                extenmd.promote = add ? promoter.promoteAdd(extenmd.type, storeemd.from) :
-                    xor ? promoter.promoteXor(extenmd.type, storeemd.from) :
-                        promoter.promoteNumeric(extenmd.type, storeemd.from, decimal, true);
-
-                if (extenmd.promote == null) {
-                    throw new IllegalArgumentException("Cannot apply compound assignment to " +
-                        "types [" + extenmd.type.name + "] and [" + storeemd.from.name + "].");
-                }
-
-                extenmd.castFrom = caster.getLegalCast(source, extenmd.type, extenmd.promote, false);
-                extenmd.castTo = caster.getLegalCast(source, extenmd.promote, extenmd.type, true);
-
-                storeemd.to = add && extenmd.promote.sort == Sort.STRING ? storeemd.from : extenmd.promote;
-                caster.markCast(storeemd);
-            } else {
-                storeemd.to = extenmd.type;
-                analyzer.visit(store);
-                caster.markCast(storeemd);
-            }
-        }
-    }
-}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerPromoter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerPromoter.java
deleted file mode 100644
index ff77fb06d93..00000000000
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerPromoter.java
+++ /dev/null
@@ -1,281 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless;
-
-import org.elasticsearch.painless.Definition.Pair;
-import org.elasticsearch.painless.Definition.Sort;
-import org.elasticsearch.painless.Definition.Type;
-
-class AnalyzerPromoter {
-    private final Definition definition;
-
-    AnalyzerPromoter(final Definition definition) {
-        this.definition = definition;
-    }
-
-    Type promoteNumeric(final Type from, final boolean decimal, final boolean primitive) {
-        final Sort sort = from.sort;
-
-        if (sort == Sort.DEF) {
-            return definition.defType;
-        } else if ((sort == Sort.DOUBLE || sort == Sort.DOUBLE_OBJ || sort == Sort.NUMBER) && decimal) {
-            return primitive ? definition.doubleType : definition.doubleobjType;
-        } else if ((sort == Sort.FLOAT || sort == Sort.FLOAT_OBJ) && decimal) {
-            return primitive ? definition.floatType : definition.floatobjType;
-        } else if (sort == Sort.LONG || sort == Sort.LONG_OBJ || sort == Sort.NUMBER) {
-            return primitive ? definition.longType : definition.longobjType;
-        } else if (sort.numeric) {
-            return primitive ? definition.intType : definition.intobjType;
-        }
-
-        return null;
-    }
-
-    Type promoteNumeric(final Type from0, final Type from1, final boolean decimal, final boolean primitive) {
-        final Sort sort0 = from0.sort;
-        final Sort sort1 = from1.sort;
-
-        if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return definition.defType;
-        }
-
-        if (decimal) {
-            if (sort0 == Sort.DOUBLE || sort0 == Sort.DOUBLE_OBJ || sort0 == Sort.NUMBER ||
-                sort1 == Sort.DOUBLE || sort1 == Sort.DOUBLE_OBJ || sort1 == Sort.NUMBER) {
-                return primitive ? definition.doubleType : definition.doubleobjType;
-            } else if (sort0 == Sort.FLOAT || sort0 == Sort.FLOAT_OBJ || sort1 == Sort.FLOAT || sort1 == Sort.FLOAT_OBJ) {
-                return primitive ? definition.floatType : definition.floatobjType;
-            }
-        }
-
-        if (sort0 == Sort.LONG || sort0 == Sort.LONG_OBJ || sort0 == Sort.NUMBER ||
-            sort1 == Sort.LONG || sort1 == Sort.LONG_OBJ || sort1 == Sort.NUMBER) {
-            return primitive ? definition.longType : definition.longobjType;
-        } else if (sort0.numeric && sort1.numeric) {
-            return primitive ? definition.intType : definition.intobjType;
-        }
-
-        return null;
-    }
-
-    Type promoteAdd(final Type from0, final Type from1) {
-        final Sort sort0 = from0.sort;
-        final Sort sort1 = from1.sort;
-
-        if (sort0 == Sort.STRING || sort1 == Sort.STRING) {
-            return definition.stringType;
-        }
-
-        return promoteNumeric(from0, from1, true, true);
-    }
-
-    Type promoteXor(final Type from0, final Type from1) {
-        final Sort sort0 = from0.sort;
-        final Sort sort1 = from1.sort;
-
-        if (sort0.bool || sort1.bool) {
-            return definition.booleanType;
-        }
-
-        return promoteNumeric(from0, from1, false, true);
-    }
-
-    Type promoteEquality(final Type from0, final Type from1) {
-        final Sort sort0 = from0.sort;
-        final Sort sort1 = from1.sort;
-
-        if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return definition.defType;
-        }
-
-        final boolean primitive = sort0.primitive && sort1.primitive;
-
-        if (sort0.bool && sort1.bool) {
-            return primitive ? definition.booleanType : definition.booleanobjType;
-        }
-
-        if (sort0.numeric && sort1.numeric) {
-            return promoteNumeric(from0, from1, true, primitive);
-        }
-
-        return definition.objectType;
-    }
-
-    Type promoteReference(final Type from0, final Type from1) {
-        final Sort sort0 = from0.sort;
-        final Sort sort1 = from1.sort;
-
-        if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return definition.defType;
-        }
-
-        if (sort0.primitive && sort1.primitive) {
-            if (sort0.bool && sort1.bool) {
-                return definition.booleanType;
-            }
-
-            if (sort0.numeric && sort1.numeric) {
-                return promoteNumeric(from0, from1, true, true);
-            }
-        }
-
-        return definition.objectType;
-    }
-
-    Type promoteConditional(final Type from0, final Type from1, final Object const0, final Object const1) {
-        if (from0.equals(from1)) {
-            return from0;
-        }
-
-        final Sort sort0 = from0.sort;
-        final Sort sort1 = from1.sort;
-
-        if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return definition.defType;
-        }
-
-        final boolean primitive = sort0.primitive && sort1.primitive;
-
-        if (sort0.bool && sort1.bool) {
-            return primitive ? definition.booleanType : definition.booleanobjType;
-        }
-
-        if (sort0.numeric && sort1.numeric) {
-            if (sort0 == Sort.DOUBLE || sort0 == Sort.DOUBLE_OBJ || sort1 == Sort.DOUBLE || sort1 == Sort.DOUBLE_OBJ) {
-                return primitive ? definition.doubleType : definition.doubleobjType;
-            } else if (sort0 == Sort.FLOAT || sort0 == Sort.FLOAT_OBJ || sort1 == Sort.FLOAT || sort1 == Sort.FLOAT_OBJ) {
-                return primitive ? definition.floatType : definition.floatobjType;
-            } else if (sort0 == Sort.LONG || sort0 == Sort.LONG_OBJ || sort1 == Sort.LONG || sort1 == Sort.LONG_OBJ) {
-                return sort0.primitive && sort1.primitive ? definition.longType : definition.longobjType;
-            } else {
-                if (sort0 == Sort.BYTE || sort0 == Sort.BYTE_OBJ) {
-                    if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
-                        return primitive ? definition.byteType : definition.byteobjType;
-                    } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
-                        if (const1 != null) {
-                            final short constant = (short)const1;
-
-                            if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? definition.byteType : definition.byteobjType;
-                            }
-                        }
-
-                        return primitive ? definition.shortType : definition.shortobjType;
-                    } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
-                        return primitive ? definition.intType : definition.intobjType;
-                    } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
-                        if (const1 != null) {
-                            final int constant = (int)const1;
-
-                            if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? definition.byteType : definition.byteobjType;
-                            }
-                        }
-
-                        return primitive ? definition.intType : definition.intobjType;
-                    }
-                } else if (sort0 == Sort.SHORT || sort0 == Sort.SHORT_OBJ) {
-                    if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
-                        if (const0 != null) {
-                            final short constant = (short)const0;
-
-                            if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? definition.byteType : definition.byteobjType;
-                            }
-                        }
-
-                        return primitive ? definition.shortType : definition.shortobjType;
-                    } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
-                        return primitive ? definition.shortType : definition.shortobjType;
-                    } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
-                        return primitive ? definition.intType : definition.intobjType;
-                    } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
-                        if (const1 != null) {
-                            final int constant = (int)const1;
-
-                            if (constant <= Short.MAX_VALUE && constant >= Short.MIN_VALUE) {
-                                return primitive ? definition.shortType : definition.shortobjType;
-                            }
-                        }
-
-                        return primitive ? definition.intType : definition.intobjType;
-                    }
-                } else if (sort0 == Sort.CHAR || sort0 == Sort.CHAR_OBJ) {
-                    if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
-                        return primitive ? definition.intType : definition.intobjType;
-                    } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
-                        return primitive ? definition.intType : definition.intobjType;
-                    } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
-                        return primitive ? definition.charType : definition.charobjType;
-                    } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
-                        if (const1 != null) {
-                            final int constant = (int)const1;
-
-                            if (constant <= Character.MAX_VALUE && constant >= Character.MIN_VALUE) {
-                                return primitive ? definition.byteType : definition.byteobjType;
-                            }
-                        }
-
-                        return primitive ? definition.intType : definition.intobjType;
-                    }
-                } else if (sort0 == Sort.INT || sort0 == Sort.INT_OBJ) {
-                    if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
-                        if (const0 != null) {
-                            final int constant = (int)const0;
-
-                            if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? definition.byteType : definition.byteobjType;
-                            }
-                        }
-
-                        return primitive ? definition.intType : definition.intobjType;
-                    } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
-                        if (const0 != null) {
-                            final int constant = (int)const0;
-
-                            if (constant <= Short.MAX_VALUE && constant >= Short.MIN_VALUE) {
-                                return primitive ? definition.byteType : definition.byteobjType;
-                            }
-                        }
-
-                        return primitive ? definition.intType : definition.intobjType;
-                    } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
-                        if (const0 != null) {
-                            final int constant = (int)const0;
-
-                            if (constant <= Character.MAX_VALUE && constant >= Character.MIN_VALUE) {
-                                return primitive ? definition.byteType : definition.byteobjType;
-                            }
-                        }
-
-                        return primitive ? definition.intType : definition.intobjType;
-                    } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
-                        return primitive ? definition.intType : definition.intobjType;
-                    }
-                }
-            }
-        }
-
-        final Pair pair = new Pair(from0, from1);
-        final Type bound = definition.bounds.get(pair);
-
-        return bound == null ? definition.objectType : bound;
-    }
-}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerStatement.java
deleted file mode 100644
index 3221cd608f3..00000000000
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerStatement.java
+++ /dev/null
@@ -1,591 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless;
-
-import org.elasticsearch.painless.Definition.Sort;
-import org.elasticsearch.painless.Metadata.ExpressionMetadata;
-import org.elasticsearch.painless.Metadata.StatementMetadata;
-import org.elasticsearch.painless.PainlessParser.AfterthoughtContext;
-import org.elasticsearch.painless.PainlessParser.BlockContext;
-import org.elasticsearch.painless.PainlessParser.BreakContext;
-import org.elasticsearch.painless.PainlessParser.ContinueContext;
-import org.elasticsearch.painless.PainlessParser.DeclContext;
-import org.elasticsearch.painless.PainlessParser.DeclarationContext;
-import org.elasticsearch.painless.PainlessParser.DecltypeContext;
-import org.elasticsearch.painless.PainlessParser.DeclvarContext;
-import org.elasticsearch.painless.PainlessParser.DoContext;
-import org.elasticsearch.painless.PainlessParser.ExprContext;
-import org.elasticsearch.painless.PainlessParser.ExpressionContext;
-import org.elasticsearch.painless.PainlessParser.ForContext;
-import org.elasticsearch.painless.PainlessParser.IdentifierContext;
-import org.elasticsearch.painless.PainlessParser.IfContext;
-import org.elasticsearch.painless.PainlessParser.InitializerContext;
-import org.elasticsearch.painless.PainlessParser.MultipleContext;
-import org.elasticsearch.painless.PainlessParser.ReturnContext;
-import org.elasticsearch.painless.PainlessParser.SingleContext;
-import org.elasticsearch.painless.PainlessParser.SourceContext;
-import org.elasticsearch.painless.PainlessParser.StatementContext;
-import org.elasticsearch.painless.PainlessParser.ThrowContext;
-import org.elasticsearch.painless.PainlessParser.TrapContext;
-import org.elasticsearch.painless.PainlessParser.TryContext;
-import org.elasticsearch.painless.PainlessParser.WhileContext;
-
-import java.util.List;
-
-class AnalyzerStatement {
-    private final Metadata metadata;
-    private final Definition definition;
-
-    private final Analyzer analyzer;
-    private final AnalyzerUtility utility;
-    private final AnalyzerCaster caster;
-
-    AnalyzerStatement(final Metadata metadata, final Analyzer analyzer,
-                      final AnalyzerUtility utility, final AnalyzerCaster caster) {
-        this.metadata = metadata;
-        this.definition = metadata.definition;
-
-        this.analyzer = analyzer;
-        this.utility = utility;
-        this.caster = caster;
-    }
-
-    void processSource(final SourceContext ctx) {
-        final StatementMetadata sourcesmd = metadata.getStatementMetadata(ctx);
-        final List<StatementContext> statectxs = ctx.statement();
-        final StatementContext lastctx = statectxs.get(statectxs.size() - 1);
-
-        utility.incrementScope();
-
-        for (final StatementContext statectx : statectxs) {
-            if (sourcesmd.allLast) {
-                throw new IllegalArgumentException(AnalyzerUtility.error(statectx) +
-                    "Statement will never be executed because all prior paths escape.");
-            }
-
-            final StatementMetadata statesmd = metadata.createStatementMetadata(statectx);
-            statesmd.lastSource = statectx == lastctx;
-            analyzer.visit(statectx);
-
-            sourcesmd.methodEscape = statesmd.methodEscape;
-            sourcesmd.allLast = statesmd.allLast;
-        }
-
-        utility.decrementScope();
-    }
-
-    void processIf(final IfContext ctx) {
-        final StatementMetadata ifsmd = metadata.getStatementMetadata(ctx);
-
-        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
-        final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-        expremd.to = definition.booleanType;
-        analyzer.visit(exprctx);
-        caster.markCast(expremd);
-
-        if (expremd.postConst != null) {
-            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "If statement is not necessary.");
-        }
-
-        final BlockContext blockctx0 = ctx.block(0);
-        final StatementMetadata blocksmd0 = metadata.createStatementMetadata(blockctx0);
-        blocksmd0.lastSource = ifsmd.lastSource;
-        blocksmd0.inLoop = ifsmd.inLoop;
-        blocksmd0.lastLoop = ifsmd.lastLoop;
-        utility.incrementScope();
-        analyzer.visit(blockctx0);
-        utility.decrementScope();
-
-        ifsmd.anyContinue = blocksmd0.anyContinue;
-        ifsmd.anyBreak = blocksmd0.anyBreak;
-
-        ifsmd.count = blocksmd0.count;
-
-        if (ctx.ELSE() != null) {
-            final BlockContext blockctx1 = ctx.block(1);
-            final StatementMetadata blocksmd1 = metadata.createStatementMetadata(blockctx1);
-            blocksmd1.lastSource = ifsmd.lastSource;
-            utility.incrementScope();
-            analyzer.visit(blockctx1);
-            utility.decrementScope();
-
-            ifsmd.methodEscape = blocksmd0.methodEscape && blocksmd1.methodEscape;
-            ifsmd.loopEscape = blocksmd0.loopEscape && blocksmd1.loopEscape;
-            ifsmd.allLast = blocksmd0.allLast && blocksmd1.allLast;
-            ifsmd.anyContinue |= blocksmd1.anyContinue;
-            ifsmd.anyBreak |= blocksmd1.anyBreak;
-
-            ifsmd.count = Math.max(ifsmd.count, blocksmd1.count);
-        }
-    }
-
-    void processWhile(final WhileContext ctx) {
-        final StatementMetadata whilesmd = metadata.getStatementMetadata(ctx);
-
-        utility.incrementScope();
-
-        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
-        final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-        expremd.to = definition.booleanType;
-        analyzer.visit(exprctx);
-        caster.markCast(expremd);
-
-        boolean continuous = false;
-
-        if (expremd.postConst != null) {
-            continuous = (boolean)expremd.postConst;
-
-            if (!continuous) {
-                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "The loop will never be executed.");
-            }
-
-            if (ctx.empty() != null) {
-                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "The loop will never exit.");
-            }
-        }
-
-        final BlockContext blockctx = ctx.block();
-
-        if (blockctx != null) {
-            final StatementMetadata blocksmd = metadata.createStatementMetadata(blockctx);
-            blocksmd.beginLoop = true;
-            blocksmd.inLoop = true;
-            analyzer.visit(blockctx);
-
-            if (blocksmd.loopEscape && !blocksmd.anyContinue) {
-                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "All paths escape so the loop is not necessary.");
-            }
-
-            if (continuous && !blocksmd.anyBreak) {
-                whilesmd.methodEscape = true;
-                whilesmd.allLast = true;
-            }
-        }
-
-        whilesmd.count = 1;
-
-        utility.decrementScope();
-    }
-
-    void processDo(final DoContext ctx) {
-        final StatementMetadata dosmd = metadata.getStatementMetadata(ctx);
-
-        utility.incrementScope();
-
-        final BlockContext blockctx = ctx.block();
-        final StatementMetadata blocksmd = metadata.createStatementMetadata(blockctx);
-        blocksmd.beginLoop = true;
-        blocksmd.inLoop = true;
-        analyzer.visit(blockctx);
-
-        if (blocksmd.loopEscape && !blocksmd.anyContinue) {
-            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "All paths escape so the loop is not necessary.");
-        }
-
-        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
-        final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-        expremd.to = definition.booleanType;
-        analyzer.visit(exprctx);
-        caster.markCast(expremd);
-
-        if (expremd.postConst != null) {
-            final boolean continuous = (boolean)expremd.postConst;
-
-            if (!continuous) {
-                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "All paths escape so the loop is not necessary.");
-            }
-
-            if (!blocksmd.anyBreak) {
-                dosmd.methodEscape = true;
-                dosmd.allLast = true;
-            }
-        }
-
-        dosmd.count = 1;
-
-        utility.decrementScope();
-    }
-
-    void processFor(final ForContext ctx) {
-        final StatementMetadata forsmd = metadata.getStatementMetadata(ctx);
-        boolean continuous = false;
-
-        utility.incrementScope();
-
-        final InitializerContext initctx = ctx.initializer();
-
-        if (initctx != null) {
-            metadata.createStatementMetadata(initctx);
-            analyzer.visit(initctx);
-        }
-
-        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
-
-        if (exprctx != null) {
-            final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-            expremd.to = definition.booleanType;
-            analyzer.visit(exprctx);
-            caster.markCast(expremd);
-
-            if (expremd.postConst != null) {
-                continuous = (boolean)expremd.postConst;
-
-                if (!continuous) {
-                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "The loop will never be executed.");
-                }
-
-                if (ctx.empty() != null) {
-                    throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "The loop is continuous.");
-                }
-            }
-        } else {
-            continuous = true;
-        }
-
-        final AfterthoughtContext atctx = ctx.afterthought();
-
-        if (atctx != null) {
-            metadata.createStatementMetadata(atctx);
-            analyzer.visit(atctx);
-        }
-
-        final BlockContext blockctx = ctx.block();
-
-        if (blockctx != null) {
-            final StatementMetadata blocksmd = metadata.createStatementMetadata(blockctx);
-            blocksmd.beginLoop = true;
-            blocksmd.inLoop = true;
-            analyzer.visit(blockctx);
-
-            if (blocksmd.loopEscape && !blocksmd.anyContinue) {
-                throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "All paths escape so the loop is not necessary.");
-            }
-
-            if (continuous && !blocksmd.anyBreak) {
-                forsmd.methodEscape = true;
-                forsmd.allLast = true;
-            }
-        }
-
-        forsmd.count = 1;
-
-        utility.decrementScope();
-    }
-
-    void processDecl(final DeclContext ctx) {
-        final StatementMetadata declsmd = metadata.getStatementMetadata(ctx);
-
-        final DeclarationContext declctx = ctx.declaration();
-        metadata.createStatementMetadata(declctx);
-        analyzer.visit(declctx);
-
-        declsmd.count = 1;
-    }
-
-    void processContinue(final ContinueContext ctx) {
-        final StatementMetadata continuesmd = metadata.getStatementMetadata(ctx);
-
-        if (!continuesmd.inLoop) {
-            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Cannot have a continue statement outside of a loop.");
-        }
-
-        if (continuesmd.lastLoop) {
-            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Unnecessary continue statement at the end of a loop.");
-        }
-
-        continuesmd.allLast = true;
-        continuesmd.anyContinue = true;
-
-        continuesmd.count = 1;
-    }
-
-    void processBreak(final BreakContext ctx) {
-        final StatementMetadata breaksmd = metadata.getStatementMetadata(ctx);
-
-        if (!breaksmd.inLoop) {
-            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Cannot have a break statement outside of a loop.");
-        }
-
-        breaksmd.loopEscape = true;
-        breaksmd.allLast = true;
-        breaksmd.anyBreak = true;
-
-        breaksmd.count = 1;
-    }
-
-    void processReturn(final ReturnContext ctx) {
-        final StatementMetadata returnsmd = metadata.getStatementMetadata(ctx);
-
-        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
-        final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-        expremd.to = definition.objectType;
-        analyzer.visit(exprctx);
-        caster.markCast(expremd);
-
-        returnsmd.methodEscape = true;
-        returnsmd.loopEscape = true;
-        returnsmd.allLast = true;
-
-        returnsmd.count = 1;
-    }
-
-    void processTry(final TryContext ctx) {
-        final StatementMetadata trysmd = metadata.getStatementMetadata(ctx);
-
-        final BlockContext blockctx = ctx.block();
-        final StatementMetadata blocksmd = metadata.createStatementMetadata(blockctx);
-        blocksmd.lastSource = trysmd.lastSource;
-        blocksmd.inLoop = trysmd.inLoop;
-        blocksmd.lastLoop = trysmd.lastLoop;
-        utility.incrementScope();
-        analyzer.visit(blockctx);
-        utility.decrementScope();
-
-        trysmd.methodEscape = blocksmd.methodEscape;
-        trysmd.loopEscape = blocksmd.loopEscape;
-        trysmd.allLast = blocksmd.allLast;
-        trysmd.anyContinue = blocksmd.anyContinue;
-        trysmd.anyBreak = blocksmd.anyBreak;
-
-        int trapcount = 0;
-
-        for (final TrapContext trapctx : ctx.trap()) {
-            final StatementMetadata trapsmd = metadata.createStatementMetadata(trapctx);
-            trapsmd.lastSource = trysmd.lastSource;
-            trapsmd.inLoop = trysmd.inLoop;
-            trapsmd.lastLoop = trysmd.lastLoop;
-            utility.incrementScope();
-            analyzer.visit(trapctx);
-            utility.decrementScope();
-
-            trysmd.methodEscape &= trapsmd.methodEscape;
-            trysmd.loopEscape &= trapsmd.loopEscape;
-            trysmd.allLast &= trapsmd.allLast;
-            trysmd.anyContinue |= trapsmd.anyContinue;
-            trysmd.anyBreak |= trapsmd.anyBreak;
-
-            trapcount = Math.max(trapcount, trapsmd.count);
-        }
-
-        trysmd.count = blocksmd.count + trapcount;
-    }
-
-    void processThrow(final ThrowContext ctx) {
-        final StatementMetadata throwsmd = metadata.getStatementMetadata(ctx);
-
-        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
-        final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-        expremd.to = definition.exceptionType;
-        analyzer.visit(exprctx);
-        caster.markCast(expremd);
-
-        throwsmd.methodEscape = true;
-        throwsmd.loopEscape = true;
-        throwsmd.allLast = true;
-
-        throwsmd.count = 1;
-    }
-
-    void processExpr(final ExprContext ctx) {
-        final StatementMetadata exprsmd = metadata.getStatementMetadata(ctx);
-        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
-        final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-        expremd.read = exprsmd.lastSource;
-        analyzer.visit(exprctx);
-
-        if (!expremd.statement && !exprsmd.lastSource) {
-            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Not a statement.");
-        }
-
-        final boolean rtn = exprsmd.lastSource && expremd.from.sort != Sort.VOID;
-        exprsmd.methodEscape = rtn;
-        exprsmd.loopEscape = rtn;
-        exprsmd.allLast = rtn;
-        expremd.to = rtn ? definition.objectType : expremd.from;
-        caster.markCast(expremd);
-
-        exprsmd.count = 1;
-    }
-
-    void processMultiple(final MultipleContext ctx) {
-        final StatementMetadata multiplesmd = metadata.getStatementMetadata(ctx);
-        final List<StatementContext> statectxs = ctx.statement();
-        final StatementContext lastctx = statectxs.get(statectxs.size() - 1);
-
-        for (StatementContext statectx : statectxs) {
-            if (multiplesmd.allLast) {
-                throw new IllegalArgumentException(AnalyzerUtility.error(statectx) +
-                    "Statement will never be executed because all prior paths escape.");
-            }
-
-            final StatementMetadata statesmd = metadata.createStatementMetadata(statectx);
-            statesmd.lastSource = multiplesmd.lastSource && statectx == lastctx;
-            statesmd.inLoop = multiplesmd.inLoop;
-            statesmd.lastLoop = (multiplesmd.beginLoop || multiplesmd.lastLoop) && statectx == lastctx;
-            analyzer.visit(statectx);
-
-            multiplesmd.methodEscape = statesmd.methodEscape;
-            multiplesmd.loopEscape = statesmd.loopEscape;
-            multiplesmd.allLast = statesmd.allLast;
-            multiplesmd.anyContinue |= statesmd.anyContinue;
-            multiplesmd.anyBreak |= statesmd.anyBreak;
-
-            multiplesmd.count += statesmd.count;
-        }
-    }
-
-    void processSingle(final SingleContext ctx) {
-        final StatementMetadata singlesmd = metadata.getStatementMetadata(ctx);
-
-        final StatementContext statectx = ctx.statement();
-        final StatementMetadata statesmd = metadata.createStatementMetadata(statectx);
-        statesmd.lastSource = singlesmd.lastSource;
-        statesmd.inLoop = singlesmd.inLoop;
-        statesmd.lastLoop = singlesmd.beginLoop || singlesmd.lastLoop;
-        analyzer.visit(statectx);
-
-        singlesmd.methodEscape = statesmd.methodEscape;
-        singlesmd.loopEscape = statesmd.loopEscape;
-        singlesmd.allLast = statesmd.allLast;
-        singlesmd.anyContinue = statesmd.anyContinue;
-        singlesmd.anyBreak = statesmd.anyBreak;
-
-        singlesmd.count = statesmd.count;
-    }
-
-    void processInitializer(InitializerContext ctx) {
-        final DeclarationContext declctx = ctx.declaration();
-        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
-
-        if (declctx != null) {
-            metadata.createStatementMetadata(declctx);
-            analyzer.visit(declctx);
-        } else if (exprctx != null) {
-            final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-            expremd.read = false;
-            analyzer.visit(exprctx);
-
-            expremd.to = expremd.from;
-            caster.markCast(expremd);
-
-            if (!expremd.statement) {
-                throw new IllegalArgumentException(AnalyzerUtility.error(exprctx) +
-                    "The initializer of a for loop must be a statement.");
-            }
-        } else {
-            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-        }
-    }
-
-    void processAfterthought(AfterthoughtContext ctx) {
-        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
-
-        if (exprctx != null) {
-            final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-            expremd.read = false;
-            analyzer.visit(exprctx);
-
-            expremd.to = expremd.from;
-            caster.markCast(expremd);
-
-            if (!expremd.statement) {
-                throw new IllegalArgumentException(AnalyzerUtility.error(exprctx) +
-                    "The afterthought of a for loop must be a statement.");
-            }
-        }
-    }
-
-    void processDeclaration(final DeclarationContext ctx) {
-        final DecltypeContext decltypectx = ctx.decltype();
-        final ExpressionMetadata decltypeemd = metadata.createExpressionMetadata(decltypectx);
-        analyzer.visit(decltypectx);
-
-        for (final DeclvarContext declvarctx : ctx.declvar()) {
-            final ExpressionMetadata declvaremd = metadata.createExpressionMetadata(declvarctx);
-            declvaremd.to = decltypeemd.from;
-            analyzer.visit(declvarctx);
-        }
-    }
-
-    void processDecltype(final DecltypeContext ctx) {
-        final ExpressionMetadata decltypeemd = metadata.getExpressionMetadata(ctx);
-        final IdentifierContext idctx = ctx.identifier();
-        final String type = ctx.getText();
-
-        utility.isValidType(idctx, true);
-        decltypeemd.from = definition.getType(type);
-    }
-
-    void processDeclvar(final DeclvarContext ctx) {
-        final ExpressionMetadata declvaremd = metadata.getExpressionMetadata(ctx);
-        final IdentifierContext idctx = ctx.identifier();
-        final String identifier = idctx.getText();
-
-        utility.isValidIdentifier(idctx, true);
-        declvaremd.postConst = utility.addVariable(ctx, identifier, declvaremd.to).slot;
-
-        final ExpressionContext exprctx = AnalyzerUtility.updateExpressionTree(ctx.expression());
-
-        if (exprctx != null) {
-            final ExpressionMetadata expremd = metadata.createExpressionMetadata(exprctx);
-            expremd.to = declvaremd.to;
-            analyzer.visit(exprctx);
-            caster.markCast(expremd);
-        }
-    }
-
-    void processTrap(final TrapContext ctx) {
-        final StatementMetadata trapsmd = metadata.getStatementMetadata(ctx);
-
-        final IdentifierContext idctx0 = ctx.identifier(0);
-        final String type = idctx0.getText();
-        utility.isValidType(idctx0, true);
-        trapsmd.exception = definition.getType(type);
-
-        try {
-            trapsmd.exception.clazz.asSubclass(Exception.class);
-        } catch (final ClassCastException exception) {
-            throw new IllegalArgumentException(AnalyzerUtility.error(ctx) + "Invalid exception type [" + trapsmd.exception.name + "].");
-        }
-
-        final IdentifierContext idctx1 = ctx.identifier(1);
-        final String identifier = idctx1.getText();
-        utility.isValidIdentifier(idctx1, true);
-        trapsmd.slot = utility.addVariable(ctx, identifier, trapsmd.exception).slot;
-
-        final BlockContext blockctx = ctx.block();
-
-        if (blockctx != null) {
-            final StatementMetadata blocksmd = metadata.createStatementMetadata(blockctx);
-            blocksmd.lastSource = trapsmd.lastSource;
-            blocksmd.inLoop = trapsmd.inLoop;
-            blocksmd.lastLoop = trapsmd.lastLoop;
-            analyzer.visit(blockctx);
-
-            trapsmd.methodEscape = blocksmd.methodEscape;
-            trapsmd.loopEscape = blocksmd.loopEscape;
-            trapsmd.allLast = blocksmd.allLast;
-            trapsmd.anyContinue = blocksmd.anyContinue;
-            trapsmd.anyBreak = blocksmd.anyBreak;
-        } else if (ctx.emptyscope() == null) {
-            throw new IllegalStateException(AnalyzerUtility.error(ctx) + "Unexpected state.");
-        }
-    }
-}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerUtility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerUtility.java
deleted file mode 100644
index 0b345431ac5..00000000000
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerUtility.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless;
-
-import org.antlr.v4.runtime.ParserRuleContext;
-import org.antlr.v4.runtime.tree.ParseTree;
-import org.elasticsearch.painless.Definition.Type;
-import org.elasticsearch.painless.PainlessParser.ExpressionContext;
-import org.elasticsearch.painless.PainlessParser.IdentifierContext;
-import org.elasticsearch.painless.PainlessParser.PrecedenceContext;
-
-import java.util.ArrayDeque;
-import java.util.Deque;
-import java.util.Iterator;
-
-class AnalyzerUtility {
-    static class Variable {
-        final String name;
-        final Type type;
-        final int slot;
-
-        private Variable(final String name, final Type type, final int slot) {
-            this.name = name;
-            this.type = type;
-            this.slot = slot;
-        }
-    }
-
-    /**
-     * A utility method to output consistent error messages.
-     * @param ctx The ANTLR node the error occurred in.
-     * @return The error message with tacked on line number and character position.
-     */
-    static String error(final ParserRuleContext ctx) {
-        return "Analyzer Error [" + ctx.getStart().getLine() + ":" + ctx.getStart().getCharPositionInLine() + "]: ";
-    }
-
-    /**
-     * A utility method to output consistent error messages for invalid types.
-     * @param ctx The ANTLR node the error occurred in.
-     * @param type The invalid type.
-     * @return The error message with tacked on line number and character position.
-     */
-    static String typeError(final ParserRuleContext ctx, final String type) {
-        return error(ctx) + "Invalid type [" + type + "].";
-    }
-
-    /**
-     * A utility method to output consistent error messages for invalid identifiers.
-     * @param ctx The ANTLR node the error occurred in.
-     * @param identifier The invalid identifier.
-     * @return The error message with tacked on line number and character position.
-     */
-    static String identifierError(final ParserRuleContext ctx, final String identifier) {
-        return error(ctx) + "Invalid identifier [" + identifier + "].";
-    }
-
-    /**
-     * The ANTLR parse tree is modified in one single case; a parent node needs to check a child node to see if it's
-     * a precedence node, and if so, it must be removed from the tree permanently.  Once the ANTLR tree is built,
-     * precedence nodes are no longer necessary to maintain the correct ordering of the tree, so they only
-     * add a level of indirection where complicated decisions about metadata passing would have to be made.  This
-     * method removes the need for those decisions.
-     * @param source The child ANTLR node to check for precedence.
-     * @return The updated child ANTLR node.
-     */
-    static ExpressionContext updateExpressionTree(ExpressionContext source) {
-        // Check to see if the ANTLR node is a precedence node.
-        if (source instanceof PainlessParser.PrecedenceContext) {
-            final ParserRuleContext parent = source.getParent();
-            int index = 0;
-
-            // Mark the index of the source node within the list of child nodes from the parent.
-            for (final ParseTree child : parent.children) {
-                if (child == source) {
-                    break;
-                }
-
-                ++index;
-            }
-
-            // If there are multiple precedence nodes in a row, remove them all.
-            while (source instanceof PrecedenceContext) {
-                source = ((PrecedenceContext)source).expression();
-            }
-
-            // Update the parent node with the child of the precedence node.
-            parent.children.set(index, source);
-        }
-
-        return source;
-    }
-
-    private final Definition definition;
-
-    private final Deque<Integer> scopes = new ArrayDeque<>();
-    private final Deque<Variable> variables = new ArrayDeque<>();
-
-    AnalyzerUtility(final Metadata metadata) {
-        this.definition = metadata.definition;
-    }
-
-    void incrementScope() {
-        scopes.push(0);
-    }
-
-    void decrementScope() {
-        int remove = scopes.pop();
-
-        while (remove > 0) {
-            variables.pop();
-            --remove;
-        }
-    }
-
-    Variable getVariable(final String name) {
-        final Iterator<Variable> itr = variables.iterator();
-
-        while (itr.hasNext()) {
-            final Variable variable = itr.next();
-
-            if (variable.name.equals(name)) {
-                return variable;
-            }
-        }
-
-        return null;
-    }
-
-    Variable addVariable(final ParserRuleContext source, final String name, final Type type) {
-        if (getVariable(name) != null) {
-            if (source == null) {
-                throw new IllegalArgumentException("Argument name [" + name + "] already defined within the scope.");
-            } else {
-                throw new IllegalArgumentException(error(source) + "Variable name [" + name + "] already defined within the scope.");
-            }
-        }
-
-        final Variable previous = variables.peekFirst();
-        int slot = 0;
-
-        if (previous != null) {
-            slot += previous.slot + previous.type.type.getSize();
-        }
-
-        final Variable variable = new Variable(name, type, slot);
-        variables.push(variable);
-
-        final int update = scopes.pop() + 1;
-        scopes.push(update);
-
-        return variable;
-    }
-
-    boolean isValidType(final IdentifierContext idctx, final boolean error) {
-        boolean valid = definition.structs.containsKey(idctx.getText());
-
-        if (!valid && error) {
-            throw new IllegalArgumentException(typeError(idctx, idctx.getText()));
-        }
-
-        return valid;
-    }
-
-    boolean isValidIdentifier(final IdentifierContext idctx, final boolean error) {
-        boolean valid = !definition.structs.containsKey(idctx.getText()) && idctx.generic() == null;
-
-        if (!valid && error) {
-            throw new IllegalArgumentException(identifierError(idctx, idctx.getText()));
-        }
-
-        return valid;
-    }
-}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
index 6a21b7cc877..ef75e904931 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
@@ -19,10 +19,10 @@
 
 package org.elasticsearch.painless;
 
-import org.antlr.v4.runtime.ANTLRInputStream;
-import org.antlr.v4.runtime.CommonTokenStream;
-import org.antlr.v4.runtime.ParserRuleContext;
 import org.elasticsearch.bootstrap.BootstrapInfo;
+import org.elasticsearch.painless.Variables.Reserved;
+import org.elasticsearch.painless.antlr.Walker;
+import org.elasticsearch.painless.node.SSource;
 
 import java.net.MalformedURLException;
 import java.net.URL;
@@ -30,13 +30,16 @@ import java.security.CodeSource;
 import java.security.SecureClassLoader;
 import java.security.cert.Certificate;
 
+import static org.elasticsearch.painless.WriterConstants.CLASS_NAME;
+
 /**
- * The Compiler is the entry point for generating a Painless script.  The compiler will generate an ANTLR
- * parse tree based on the source code that is passed in.  Two passes will then be run over the parse tree,
+ * The Compiler is the entry point for generating a Painless script.  The compiler will receive a Painless
+ * tree based on the type of input passed in (currently only ANTLR).  Two passes will then be run over the tree,
  * one for analysis using the {@link Analyzer} and another to generate the actual byte code using ASM in
  * the {@link Writer}.
  */
 final class Compiler {
+
     /**
      * The maximum number of characters allowed in the script source.
      */
@@ -54,7 +57,7 @@ final class Compiler {
         try {
             // Setup the code privileges.
             CODESOURCE = new CodeSource(new URL("file:" + BootstrapInfo.UNTRUSTED_CODEBASE), (Certificate[]) null);
-        } catch (MalformedURLException impossible) {
+        } catch (final MalformedURLException impossible) {
             throw new RuntimeException(impossible);
         }
     }
@@ -62,7 +65,7 @@ final class Compiler {
     /**
      * A secure class loader used to define Painless scripts.
      */
-    static class Loader extends SecureClassLoader {
+    static final class Loader extends SecureClassLoader {
         /**
          * @param parent The parent ClassLoader.
          */
@@ -90,64 +93,45 @@ final class Compiler {
      * @return An {@link Executable} Painless script.
      */
     static Executable compile(final Loader loader, final String name, final String source, final CompilerSettings settings) {
-        final byte[] bytes = compile(name, source, Definition.INSTANCE, settings);
-        return createExecutable(loader, Definition.INSTANCE, name, source, bytes);
+        byte[] bytes = compile(source, settings);
+
+        return createExecutable(loader, name, source, bytes);
     }
 
     /**
-     * Compiles the script to bytecode
+     * Runs the two-pass compiler to generate a Painless script.  (Used by the debugger.)
+     * @param source The source code for the script.
+     * @param settings The CompilerSettings to be used during the compilation.
+     * @return The bytes for compilation.
      */
-    static byte[] compile(String name, String source, Definition definition, CompilerSettings settings) {
+    static byte[] compile(final String source, final CompilerSettings settings) {
         if (source.length() > MAXIMUM_SOURCE_LENGTH) {
             throw new IllegalArgumentException("Scripts may be no longer than " + MAXIMUM_SOURCE_LENGTH +
-                    " characters.  The passed in script is " + source.length() + " characters.  Consider using a" +
-                    " plugin if a script longer than this length is a requirement.");
+                " characters.  The passed in script is " + source.length() + " characters.  Consider using a" +
+                " plugin if a script longer than this length is a requirement.");
         }
-        final ParserRuleContext root = createParseTree(source);
-        final Metadata metadata = new Metadata(definition, source, root, settings);
-        Analyzer.analyze(metadata);
-        return Writer.write(metadata);
-    }
 
-    /**
-     * Generates the ANTLR tree from the given source code.  Several methods below, are used
-     * to ensure that the first error generated by ANTLR will cause the compilation to fail rather than
-     * use ANTLR's recovery strategies that may be potentially dangerous.
-     * @param source The source code for the script.
-     * @return The root node for the ANTLR parse tree.
-     */
-    private static ParserRuleContext createParseTree(final String source) {
-        final ANTLRInputStream stream = new ANTLRInputStream(source);
-        final ErrorHandlingLexer lexer = new ErrorHandlingLexer(stream);
-        final PainlessParser parser = new PainlessParser(new CommonTokenStream(lexer));
-        final ParserErrorStrategy strategy = new ParserErrorStrategy();
+        final Reserved reserved = new Reserved();
+        final SSource root = Walker.buildPainlessTree(source, reserved);
+        final Variables variables = Analyzer.analyze(settings, Definition.INSTANCE, reserved, root);
 
-        lexer.removeErrorListeners();
-        parser.removeErrorListeners();
-        parser.setErrorHandler(strategy);
-
-        ParserRuleContext root = parser.source();
-
-        return root;
+        return Writer.write(settings, Definition.INSTANCE, source, variables, root);
     }
 
     /**
      * Generates an {@link Executable} that can run a Painless script.
      * @param loader The {@link Loader} to define the script's class file.
-     * @param definition The Painless API.
      * @param name The name of the script.
      * @param source The source text of the script.
      * @param bytes The ASM generated byte code to define the class with.
      * @return A Painless {@link Executable} script.
      */
-    private static Executable createExecutable(final Loader loader, final Definition definition,
-                                               final String name, final String source, final byte[] bytes) {
+    private static Executable createExecutable(final Loader loader, final String name, final String source, final byte[] bytes) {
         try {
-            final Class<? extends Executable> clazz = loader.define(WriterConstants.CLASS_NAME, bytes);
-            final java.lang.reflect.Constructor<? extends Executable> constructor =
-                    clazz.getConstructor(Definition.class, String.class, String.class);
+            final Class<? extends Executable> clazz = loader.define(CLASS_NAME, bytes);
+            final java.lang.reflect.Constructor<? extends Executable> constructor = clazz.getConstructor(String.class, String.class);
 
-            return constructor.newInstance(definition, name, source);
+            return constructor.newInstance(name, source);
         } catch (final Exception exception) { // Catch everything to let the user know this is something caused internally.
             throw new IllegalStateException(
                     "An internal error occurred attempting to define the script [" + name + "].", exception);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java
index 79f7c212ef3..9ec26bf345a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java
@@ -22,7 +22,8 @@ package org.elasticsearch.painless;
 /**
  * Settings to use when compiling a script.
  */
-final class CompilerSettings {
+public final class CompilerSettings {
+
     /**
      * Constant to be used when specifying numeric overflow when compiling a script.
      */
@@ -52,7 +53,7 @@ final class CompilerSettings {
      * signs, and overflow for floating point types can result in infinite or
      * {@code NaN} values.
      */
-    public boolean getNumericOverflow() {
+    public final boolean getNumericOverflow() {
         return numericOverflow;
     }
 
@@ -60,15 +61,16 @@ final class CompilerSettings {
      * Set {@code true} for numerics to overflow, false to deliver exceptions.
      * @see #getNumericOverflow
      */
-    public void setNumericOverflow(boolean allow) {
+    public final void setNumericOverflow(boolean allow) {
         this.numericOverflow = allow;
     }
 
     /**
      * Returns the value for the cumulative total number of statements that can be made in all loops
-     * in a script before an exception is thrown.  This attempts to prevent infinite loops.
+     * in a script before an exception is thrown.  This attempts to prevent infinite loops.  Note if
+     * the counter is set to 0, no loop counter will be written.
      */
-    public int getMaxLoopCounter() {
+    public final int getMaxLoopCounter() {
         return maxLoopCounter;
     }
 
@@ -76,7 +78,7 @@ final class CompilerSettings {
      * Set the cumulative total number of statements that can be made in all loops.
      * @see #getMaxLoopCounter
      */
-    public void setMaxLoopCounter(int max) {
+    public final void setMaxLoopCounter(int max) {
         this.maxLoopCounter = max;
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index e50addfb479..580363bc574 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -24,8 +24,8 @@ import org.elasticsearch.painless.Definition.RuntimeClass;
 
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
-import java.lang.invoke.MethodType;
 import java.lang.invoke.MethodHandles.Lookup;
+import java.lang.invoke.MethodType;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -41,13 +41,85 @@ import java.util.stream.Stream;
  * <p>
  * Dynamic methods, loads, stores, and array/list/map load/stores involve locating the appropriate field
  * or method depending on the receiver's class. For these, we emit an {@code invokedynamic} instruction that,
- * for each new  type encountered will query a corresponding {@code lookupXXX} method to retrieve the appropriate
+ * for each new type encountered will query a corresponding {@code lookupXXX} method to retrieve the appropriate
  * method. In most cases, the {@code lookupXXX} methods here will only be called once for a given call site, because
  * caching ({@link DynamicCallSite}) generally works: usually all objects at any call site will be consistently
  * the same type (or just a few types).  In extreme cases, if there is type explosion, they may be called every
  * single time, but simplicity is still more valuable than performance in this code.
  */
-public class Def {
+public final class Def {
+
+    // TODO: Once Java has a factory for those in java.lang.invoke.MethodHandles, use it:
+
+    /** Helper class for isolating MethodHandles and methods to get the length of arrays
+     * (to emulate a "arraystore" byteoode using MethodHandles).
+     * This should really be a method in {@link MethodHandles} class!
+     */
+    private static final class ArrayLengthHelper {
+        private static final Lookup PRIV_LOOKUP = MethodHandles.lookup();
+
+        private static final Map<Class<?>,MethodHandle> ARRAY_TYPE_MH_MAPPING = Collections.unmodifiableMap(
+            Stream.of(boolean[].class, byte[].class, short[].class, int[].class, long[].class,
+                char[].class, float[].class, double[].class, Object[].class)
+                .collect(Collectors.toMap(Function.identity(), type -> {
+                    try {
+                        return PRIV_LOOKUP.findStatic(PRIV_LOOKUP.lookupClass(), "getArrayLength", MethodType.methodType(int.class, type));
+                    } catch (ReflectiveOperationException e) {
+                        throw new AssertionError(e);
+                    }
+                }))
+        );
+
+        private static final MethodHandle OBJECT_ARRAY_MH = ARRAY_TYPE_MH_MAPPING.get(Object[].class);
+
+        static int getArrayLength(final boolean[] array) { return array.length; }
+        static int getArrayLength(final byte[] array)    { return array.length; }
+        static int getArrayLength(final short[] array)   { return array.length; }
+        static int getArrayLength(final int[] array)     { return array.length; }
+        static int getArrayLength(final long[] array)    { return array.length; }
+        static int getArrayLength(final char[] array)    { return array.length; }
+        static int getArrayLength(final float[] array)   { return array.length; }
+        static int getArrayLength(final double[] array)  { return array.length; }
+        static int getArrayLength(final Object[] array)  { return array.length; }
+
+        static MethodHandle arrayLengthGetter(Class<?> arrayType) {
+            if (!arrayType.isArray()) {
+                throw new IllegalArgumentException("type must be an array");
+            }
+            return (ARRAY_TYPE_MH_MAPPING.containsKey(arrayType)) ?
+                ARRAY_TYPE_MH_MAPPING.get(arrayType) :
+                OBJECT_ARRAY_MH.asType(OBJECT_ARRAY_MH.type().changeParameterType(0, arrayType));
+        }
+
+        private ArrayLengthHelper() {}
+    }
+
+    /** pointer to Map.get(Object) */
+    private static final MethodHandle MAP_GET;
+    /** pointer to Map.put(Object,Object) */
+    private static final MethodHandle MAP_PUT;
+    /** pointer to List.get(int) */
+    private static final MethodHandle LIST_GET;
+    /** pointer to List.set(int,Object) */
+    private static final MethodHandle LIST_SET;
+
+    static {
+        final Lookup lookup = MethodHandles.publicLookup();
+
+        try {
+            MAP_GET  = lookup.findVirtual(Map.class , "get", MethodType.methodType(Object.class, Object.class));
+            MAP_PUT  = lookup.findVirtual(Map.class , "put", MethodType.methodType(Object.class, Object.class, Object.class));
+            LIST_GET = lookup.findVirtual(List.class, "get", MethodType.methodType(Object.class, int.class));
+            LIST_SET = lookup.findVirtual(List.class, "set", MethodType.methodType(Object.class, int.class, Object.class));
+        } catch (final ReflectiveOperationException roe) {
+            throw new AssertionError(roe);
+        }
+    }
+
+    /** Returns an array length getter MethodHandle for the given array type */
+    static MethodHandle arrayLengthGetter(Class<?> arrayType) {
+        return ArrayLengthHelper.arrayLengthGetter(arrayType);
+    }
 
     /**
      * Looks up handle for a dynamic method call.
@@ -65,7 +137,7 @@ public class Def {
      * @return pointer to matching method to invoke. never returns null.
      * @throws IllegalArgumentException if no matching whitelisted method was found.
      */
-    static MethodHandle lookupMethod(Class<?> receiverClass, String name, Definition definition) {
+     static MethodHandle lookupMethod(Class<?> receiverClass, String name, Definition definition) {
         // check whitelist for matching method
         for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
             RuntimeClass struct = definition.runtimeMap.get(clazz);
@@ -77,7 +149,7 @@ public class Def {
                 }
             }
 
-            for (Class<?> iface : clazz.getInterfaces()) {
+            for (final Class<?> iface : clazz.getInterfaces()) {
                 struct = definition.runtimeMap.get(iface);
 
                 if (struct != null) {
@@ -94,78 +166,6 @@ public class Def {
                                            "for class [" + receiverClass.getCanonicalName() + "].");
     }
 
-    /** pointer to Map.get(Object) */
-    private static final MethodHandle MAP_GET;
-    /** pointer to Map.put(Object,Object) */
-    private static final MethodHandle MAP_PUT;
-    /** pointer to List.get(int) */
-    private static final MethodHandle LIST_GET;
-    /** pointer to List.set(int,Object) */
-    private static final MethodHandle LIST_SET;
-    static {
-        Lookup lookup = MethodHandles.publicLookup();
-        try {
-            MAP_GET      = lookup.findVirtual(Map.class, "get",
-                                             MethodType.methodType(Object.class, Object.class));
-            MAP_PUT      = lookup.findVirtual(Map.class, "put",
-                                             MethodType.methodType(Object.class, Object.class, Object.class));
-            LIST_GET     = lookup.findVirtual(List.class, "get",
-                                             MethodType.methodType(Object.class, int.class));
-            LIST_SET     = lookup.findVirtual(List.class, "set",
-                                             MethodType.methodType(Object.class, int.class, Object.class));
-        } catch (ReflectiveOperationException e) {
-            throw new AssertionError(e);
-        }
-    }
-
-    // TODO: Once Java has a factory for those in java.lang.invoke.MethodHandles, use it:
-
-    /** Helper class for isolating MethodHandles and methods to get the length of arrays
-     * (to emulate a "arraystore" byteoode using MethodHandles).
-     * This should really be a method in {@link MethodHandles} class!
-     */
-    private static final class ArrayLengthHelper {
-      private ArrayLengthHelper() {}
-
-      private static final Lookup PRIV_LOOKUP = MethodHandles.lookup();
-      private static final Map<Class<?>,MethodHandle> ARRAY_TYPE_MH_MAPPING = Collections.unmodifiableMap(
-        Stream.of(boolean[].class, byte[].class, short[].class, int[].class, long[].class,
-        char[].class, float[].class, double[].class, Object[].class)
-          .collect(Collectors.toMap(Function.identity(), type -> {
-            try {
-              return PRIV_LOOKUP.findStatic(PRIV_LOOKUP.lookupClass(), "getArrayLength", MethodType.methodType(int.class, type));
-            } catch (ReflectiveOperationException e) {
-              throw new AssertionError(e);
-            }
-          }))
-      );
-      private static final MethodHandle OBJECT_ARRAY_MH = ARRAY_TYPE_MH_MAPPING.get(Object[].class);
-
-      static int getArrayLength(boolean[] array) { return array.length; }
-      static int getArrayLength(byte[] array) { return array.length; }
-      static int getArrayLength(short[] array) { return array.length; }
-      static int getArrayLength(int[] array) { return array.length; }
-      static int getArrayLength(long[] array) { return array.length; }
-      static int getArrayLength(char[] array) { return array.length; }
-      static int getArrayLength(float[] array) { return array.length; }
-      static int getArrayLength(double[] array) { return array.length; }
-      static int getArrayLength(Object[] array) { return array.length; }
-
-      public static MethodHandle arrayLengthGetter(Class<?> arrayType) {
-        if (!arrayType.isArray()) {
-          throw new IllegalArgumentException("type must be an array");
-        }
-        return (ARRAY_TYPE_MH_MAPPING.containsKey(arrayType)) ?
-            ARRAY_TYPE_MH_MAPPING.get(arrayType) :
-            OBJECT_ARRAY_MH.asType(OBJECT_ARRAY_MH.type().changeParameterType(0, arrayType));
-      }
-    }
-
-    /** Returns an array length getter MethodHandle for the given array type */
-    public static MethodHandle arrayLengthGetter(Class<?> arrayType) {
-      return ArrayLengthHelper.arrayLengthGetter(arrayType);
-    }
-
     /**
      * Looks up handle for a dynamic field getter (field load)
      * <p>
@@ -298,7 +298,7 @@ public class Def {
             try {
                 int index = Integer.parseInt(name);
                 return MethodHandles.insertArguments(LIST_SET, 1, index);
-            } catch (NumberFormatException exception) {
+            } catch (final NumberFormatException exception) {
                 throw new IllegalArgumentException( "Illegal list shortcut value [" + name + "].");
             }
         }
@@ -345,10 +345,12 @@ public class Def {
                                            "[" + receiverClass.getCanonicalName() + "] as an array.");
     }
 
-    // NOTE: below methods are not cached, instead invoked directly because they are performant.
+    // NOTE: Below methods are not cached, instead invoked directly because they are performant.
+    //       We also check for Long values first when possible since the type is more
+    //       likely to be a Long than a Float.
 
     public static Object not(final Object unary) {
-        if (unary instanceof Double || unary instanceof Float || unary instanceof Long) {
+        if (unary instanceof Double || unary instanceof Long || unary instanceof Float) {
             return ~((Number)unary).longValue();
         } else if (unary instanceof Number) {
             return ~((Number)unary).intValue();
@@ -594,103 +596,40 @@ public class Def {
                 "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
     }
 
-    public static Object lsh(final Object left, final Object right) {
-        if (left instanceof Number) {
-            if (right instanceof Number) {
-                if (left instanceof Double || right instanceof Double ||
-                    left instanceof Long || right instanceof Long ||
-                    left instanceof Float || right instanceof Float) {
-                    return ((Number)left).longValue() << ((Number)right).longValue();
-                } else {
-                    return ((Number)left).intValue() << ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                if (left instanceof Double ||  left instanceof Long || left instanceof Float) {
-                    return ((Number)left).longValue() << (char)right;
-                } else {
-                    return ((Number)left).intValue() << (char)right;
-                }
-            }
+    public static Object lsh(final Object left, final int right) {
+        if (left instanceof Double || left instanceof Long || left instanceof Float) {
+            return ((Number)left).longValue() << right;
+        } else if (left instanceof Number) {
+            return ((Number)left).intValue() << right;
         } else if (left instanceof Character) {
-            if (right instanceof Number) {
-                if (right instanceof Double || right instanceof Long || right instanceof Float) {
-                    return (long)(char)left << ((Number)right).longValue();
-                } else {
-                    return (char)left << ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                return (char)left << (char)right;
-            }
+            return (char)left << right;
         }
 
-        throw new ClassCastException("Cannot apply [<<] operation to types " +
-                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
+        throw new ClassCastException("Cannot apply [<<] operation to types [" + left.getClass().getCanonicalName() + "] and [int].");
     }
 
-    public static Object rsh(final Object left, final Object right) {
-        if (left instanceof Number) {
-            if (right instanceof Number) {
-                if (left instanceof Double || right instanceof Double ||
-                    left instanceof Long || right instanceof Long ||
-                    left instanceof Float || right instanceof Float) {
-                    return ((Number)left).longValue() >> ((Number)right).longValue();
-                } else {
-                    return ((Number)left).intValue() >> ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                if (left instanceof Double || left instanceof Long || left instanceof Float) {
-                    return ((Number)left).longValue() >> (char)right;
-                } else {
-                    return ((Number)left).intValue() >> (char)right;
-                }
-            }
+    public static Object rsh(final Object left, final int right) {
+        if (left instanceof Double || left instanceof Long || left instanceof Float) {
+            return ((Number)left).longValue() >> right;
+        } else if (left instanceof Number) {
+            return ((Number)left).intValue() >> right;
         } else if (left instanceof Character) {
-            if (right instanceof Number) {
-                if (right instanceof Double || right instanceof Long || right instanceof Float) {
-                    return (long)(char)left >> ((Number)right).longValue();
-                } else {
-                    return (char)left >> ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                return (char)left >> (char)right;
-            }
+            return (char)left >> right;
         }
 
-        throw new ClassCastException("Cannot apply [>>] operation to types " +
-                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
+        throw new ClassCastException("Cannot apply [>>] operation to types [" + left.getClass().getCanonicalName() + "] and [int].");
     }
 
-    public static Object ush(final Object left, final Object right) {
-        if (left instanceof Number) {
-            if (right instanceof Number) {
-                if (left instanceof Double || right instanceof Double ||
-                    left instanceof Long || right instanceof Long ||
-                    left instanceof Float || right instanceof Float) {
-                    return ((Number)left).longValue() >>> ((Number)right).longValue();
-                } else {
-                    return ((Number)left).intValue() >>> ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                if (left instanceof Double || left instanceof Long || left instanceof Float) {
-                    return ((Number)left).longValue() >>> (char)right;
-                } else {
-                    return ((Number)left).intValue() >>> (char)right;
-                }
-            }
+    public static Object ush(final Object left, final int right) {
+        if (left instanceof Double || left instanceof Long || left instanceof Float) {
+            return ((Number)left).longValue() >>> right;
+        } else if (left instanceof Number) {
+            return ((Number)left).intValue() >>> right;
         } else if (left instanceof Character) {
-            if (right instanceof Number) {
-                if (right instanceof Double || right instanceof Long || right instanceof Float) {
-                    return (long)(char)left >>> ((Number)right).longValue();
-                } else {
-                    return (char)left >>> ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                return (char)left >>> (char)right;
-            }
+            return (char)left >>> right;
         }
 
-        throw new ClassCastException("Cannot apply [>>>] operation to types " +
-                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
+        throw new ClassCastException("Cannot apply [>>>] operation to types [" + left.getClass().getCanonicalName() + "] and [int].");
     }
 
     public static Object and(final Object left, final Object right) {
@@ -1026,6 +965,8 @@ public class Def {
                 "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
     }
 
+    // Conversion methods for Def to primitive types.
+
     public static boolean DefToboolean(final Object value) {
         if (value instanceof Boolean) {
             return (boolean)value;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index a3ee43b00a5..e2f312be7b4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -19,6 +19,9 @@
 
 package org.elasticsearch.painless;
 
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.index.fielddata.ScriptDocValues;
+
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
@@ -32,17 +35,19 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import org.elasticsearch.common.geo.GeoPoint;
-import org.elasticsearch.index.fielddata.ScriptDocValues;
+/**
+ * The entire API for Painless.  Also used as a whitelist for checking for legal
+ * methods and fields during at both compile-time and runtime.
+ */
+public final class Definition {
 
-class Definition {
     /**
      * The default language API to be used with Painless.  The second construction is used
      * to finalize all the variables, so there is no mistake of modification afterwards.
      */
     static Definition INSTANCE = new Definition(new Definition());
 
-    enum Sort {
+    public enum Sort {
         VOID(       void.class      , 0 , true  , false , false , false ),
         BOOL(       boolean.class   , 1 , true  , true  , false , true  ),
         BYTE(       byte.class      , 1 , true  , false , true  , true  ),
@@ -63,19 +68,19 @@ class Definition {
         FLOAT_OBJ(  Float.class     , 1 , false , false , true  , false ),
         DOUBLE_OBJ( Double.class    , 1 , false , false , true  , false ),
 
-        NUMBER(     Number.class    , 1 , false , false , true  , false ),
+        NUMBER(     Number.class    , 1 , false , false , false , false ),
         STRING(     String.class    , 1 , false , false , false , true  ),
 
         OBJECT(     null            , 1 , false , false , false , false ),
         DEF(        null            , 1 , false , false , false , false ),
         ARRAY(      null            , 1 , false , false , false , false );
 
-        final Class<?> clazz;
-        final int size;
-        final boolean primitive;
-        final boolean bool;
-        final boolean numeric;
-        final boolean constant;
+        public final Class<?> clazz;
+        public final int size;
+        public final boolean primitive;
+        public final boolean bool;
+        public final boolean numeric;
+        public final boolean constant;
 
         Sort(final Class<?> clazz, final int size, final boolean primitive,
              final boolean bool, final boolean numeric, final boolean constant) {
@@ -88,16 +93,18 @@ class Definition {
         }
     }
 
-    static class Type {
-        final String name;
-        final Struct struct;
-        final Class<?> clazz;
-        final org.objectweb.asm.Type type;
-        final Sort sort;
+    public static final class Type {
+        public final String name;
+        public final int dimensions;
+        public final Struct struct;
+        public final Class<?> clazz;
+        public final org.objectweb.asm.Type type;
+        public final Sort sort;
 
-        private Type(final String name, final Struct struct, final Class<?> clazz,
-                     final org.objectweb.asm.Type type, final Sort sort) {
+        private Type(final String name, final int dimensions, final Struct struct,
+                     final Class<?> clazz, final org.objectweb.asm.Type type, final Sort sort) {
             this.name = name;
+            this.dimensions = dimensions;
             this.struct = struct;
             this.clazz = clazz;
             this.type = type;
@@ -128,12 +135,12 @@ class Definition {
         }
     }
 
-    static class Constructor {
-        final String name;
-        final Struct owner;
-        final List<Type> arguments;
-        final org.objectweb.asm.commons.Method method;
-        final java.lang.reflect.Constructor<?> reflect;
+    public static final class Constructor {
+        public final String name;
+        public final Struct owner;
+        public final List<Type> arguments;
+        public final org.objectweb.asm.commons.Method method;
+        public final java.lang.reflect.Constructor<?> reflect;
 
         private Constructor(final String name, final Struct owner, final List<Type> arguments,
                             final org.objectweb.asm.commons.Method method, final java.lang.reflect.Constructor<?> reflect) {
@@ -145,14 +152,14 @@ class Definition {
         }
     }
 
-    static class Method {
-        final String name;
-        final Struct owner;
-        final Type rtn;
-        final List<Type> arguments;
-        final org.objectweb.asm.commons.Method method;
-        final java.lang.reflect.Method reflect;
-        final MethodHandle handle;
+    public static class Method {
+        public final String name;
+        public final Struct owner;
+        public final Type rtn;
+        public final List<Type> arguments;
+        public final org.objectweb.asm.commons.Method method;
+        public final java.lang.reflect.Method reflect;
+        public final MethodHandle handle;
 
         private Method(final String name, final Struct owner, final Type rtn, final List<Type> arguments,
                        final org.objectweb.asm.commons.Method method, final java.lang.reflect.Method reflect,
@@ -167,14 +174,14 @@ class Definition {
         }
     }
 
-    static class Field {
-        final String name;
-        final Struct owner;
-        final Type generic;
-        final Type type;
-        final java.lang.reflect.Field reflect;
-        final MethodHandle getter;
-        final MethodHandle setter;
+    public static final class Field {
+        public final String name;
+        public final Struct owner;
+        public final Type generic;
+        public final Type type;
+        public final java.lang.reflect.Field reflect;
+        public final MethodHandle getter;
+        public final MethodHandle setter;
 
         private Field(final String name, final Struct owner, final Type generic, final Type type,
                       final java.lang.reflect.Field reflect, final MethodHandle getter, final MethodHandle setter) {
@@ -188,17 +195,17 @@ class Definition {
         }
     }
 
-    static class Struct {
-        final String name;
-        final Class<?> clazz;
-        final org.objectweb.asm.Type type;
+    public static final class Struct {
+        public final String name;
+        public final Class<?> clazz;
+        public final org.objectweb.asm.Type type;
 
-        final Map<String, Constructor> constructors;
-        final Map<String, Method> functions;
-        final Map<String, Method> methods;
+        public final Map<String, Constructor> constructors;
+        public final Map<String, Method> functions;
+        public final Map<String, Method> methods;
 
-        final Map<String, Field> statics;
-        final Map<String, Field> members;
+        public final Map<String, Field> statics;
+        public final Map<String, Field> members;
 
         private Struct(final String name, final Class<?> clazz, final org.objectweb.asm.Type type) {
             this.name = name;
@@ -247,44 +254,11 @@ class Definition {
         }
     }
 
-    static class Pair {
-        final Type type0;
-        final Type type1;
+    public static class Cast {
+        public final Type from;
+        public final Type to;
 
-        Pair(final Type type0, final Type type1) {
-            this.type0 = type0;
-            this.type1 = type1;
-        }
-
-        @Override
-        public boolean equals(final Object object) {
-            if (this == object) {
-                return true;
-            }
-
-            if (object == null || getClass() != object.getClass()) {
-                return false;
-            }
-
-            final Pair pair = (Pair)object;
-
-            return type0.equals(pair.type0) && type1.equals(pair.type1);
-        }
-
-        @Override
-        public int hashCode() {
-            int result = type0.hashCode();
-            result = 31 * result + type1.hashCode();
-
-            return result;
-        }
-    }
-
-    static class Cast {
-        final Type from;
-        final Type to;
-
-        Cast(final Type from, final Type to) {
+        public Cast(final Type from, final Type to) {
             this.from = from;
             this.to = to;
         }
@@ -313,13 +287,13 @@ class Definition {
         }
     }
 
-    static class Transform extends Cast {
-        final Cast cast;
-        final Method method;
-        final Type upcast;
-        final Type downcast;
+    public static final class Transform extends Cast {
+        public final Cast cast;
+        public final Method method;
+        public final Type upcast;
+        public final Type downcast;
 
-        private Transform(final Cast cast, Method method, final Type upcast, final Type downcast) {
+        public Transform(final Cast cast, Method method, final Type upcast, final Type downcast) {
             super(cast.from, cast.to);
 
             this.cast = cast;
@@ -328,110 +302,106 @@ class Definition {
             this.downcast = downcast;
         }
     }
-    
-    static class RuntimeClass {
-        final Map<String, Method> methods;
-        final Map<String, MethodHandle> getters;
-        final Map<String, MethodHandle> setters;
-        
-        private RuntimeClass(Map<String, Method> methods, Map<String, MethodHandle> getters, Map<String, MethodHandle> setters) {
+
+    public static final class RuntimeClass {
+        public final Map<String, Method> methods;
+        public final Map<String, MethodHandle> getters;
+        public final Map<String, MethodHandle> setters;
+
+        private RuntimeClass(final Map<String, Method> methods,
+                             final Map<String, MethodHandle> getters, final Map<String, MethodHandle> setters) {
             this.methods = methods;
             this.getters = getters;
             this.setters = setters;
         }
     }
 
-    final Map<String, Struct> structs;
-    final Map<Class<?>, Struct> classes;
-    final Map<Cast, Transform> transforms;
-    final Map<Pair, Type> bounds;
-    final Map<Class<?>, RuntimeClass> runtimeMap; 
+    public final Map<String, Struct> structsMap;
+    public final Map<Cast, Transform> transformsMap;
+    public final Map<Class<?>, RuntimeClass> runtimeMap;
 
-    final Type voidType;
-    final Type booleanType;
-    final Type byteType;
-    final Type shortType;
-    final Type charType;
-    final Type intType;
-    final Type longType;
-    final Type floatType;
-    final Type doubleType;
+    public final Type voidType;
+    public final Type booleanType;
+    public final Type byteType;
+    public final Type shortType;
+    public final Type charType;
+    public final Type intType;
+    public final Type longType;
+    public final Type floatType;
+    public final Type doubleType;
 
-    final Type voidobjType;
-    final Type booleanobjType;
-    final Type byteobjType;
-    final Type shortobjType;
-    final Type charobjType;
-    final Type intobjType;
-    final Type longobjType;
-    final Type floatobjType;
-    final Type doubleobjType;
+    public final Type voidobjType;
+    public final Type booleanobjType;
+    public final Type byteobjType;
+    public final Type shortobjType;
+    public final Type charobjType;
+    public final Type intobjType;
+    public final Type longobjType;
+    public final Type floatobjType;
+    public final Type doubleobjType;
 
-    final Type objectType;
-    final Type defType;
-    final Type numberType;
-    final Type charseqType;
-    final Type stringType;
-    final Type mathType;
-    final Type utilityType;
-    final Type defobjType;
+    public final Type objectType;
+    public final Type defType;
+    public final Type numberType;
+    public final Type charseqType;
+    public final Type stringType;
+    public final Type mathType;
+    public final Type utilityType;
+    public final Type defobjType;
 
-    final Type itrType;
-    final Type oitrType;
-    final Type sitrType;
+    public final Type itrType;
+    public final Type oitrType;
+    public final Type sitrType;
 
-    final Type collectionType;
-    final Type ocollectionType;
-    final Type scollectionType;
+    public final Type collectionType;
+    public final Type ocollectionType;
+    public final Type scollectionType;
 
-    final Type listType;
-    final Type arraylistType;
-    final Type olistType;
-    final Type oarraylistType;
-    final Type slistType;
-    final Type sarraylistType;
+    public final Type listType;
+    public final Type arraylistType;
+    public final Type olistType;
+    public final Type oarraylistType;
+    public final Type slistType;
+    public final Type sarraylistType;
 
-    final Type setType;
-    final Type hashsetType;
-    final Type osetType;
-    final Type ohashsetType;
-    final Type ssetType;
-    final Type shashsetType;
+    public final Type setType;
+    public final Type hashsetType;
+    public final Type osetType;
+    public final Type ohashsetType;
+    public final Type ssetType;
+    public final Type shashsetType;
 
-    final Type mapType;
-    final Type hashmapType;
-    final Type oomapType;
-    final Type oohashmapType;
-    final Type smapType;
-    final Type shashmapType;
-    final Type somapType;
-    final Type sohashmapType;
+    public final Type mapType;
+    public final Type hashmapType;
+    public final Type oomapType;
+    public final Type oohashmapType;
+    public final Type smapType;
+    public final Type shashmapType;
+    public final Type somapType;
+    public final Type sohashmapType;
 
-    final Type execType;
+    public final Type execType;
+
+    public final Type exceptionType;
+    public final Type arithexcepType;
+    public final Type iargexcepType;
+    public final Type istateexceptType;
+    public final Type nfexcepType;
 
-    final Type exceptionType;
-    final Type arithexcepType;
-    final Type iargexcepType;
-    final Type istateexceptType;
-    final Type nfexcepType;
-    
     // docvalues accessors
-    final Type geoPointType;
-    final Type stringsType;
+    public final Type geoPointType;
+    public final Type stringsType;
     // TODO: add ReadableDateTime? or don't expose the joda stuff?
-    final Type longsType;
-    final Type doublesType;
-    final Type geoPointsType;
+    public final Type longsType;
+    public final Type doublesType;
+    public final Type geoPointsType;
 
     private Definition() {
-        structs = new HashMap<>();
-        classes = new HashMap<>();
-        transforms = new HashMap<>();
-        bounds = new HashMap<>();
+        structsMap = new HashMap<>();
+        transformsMap = new HashMap<>();
         runtimeMap = new HashMap<>();
 
-        addDefaultStructs();
-        addDefaultClasses();
+        addStructs();
 
         voidType = getType("void");
         booleanType = getType("boolean");
@@ -507,164 +477,99 @@ class Definition {
         doublesType = getType("Doubles");
         geoPointsType = getType("GeoPoints");
 
-        addDefaultElements();
-        copyDefaultStructs();
-        addDefaultTransforms();
-        addDefaultBounds();
-        computeRuntimeClasses();
-    }
-    
-    // precompute a more efficient structure for dynamic method/field access:
-    void computeRuntimeClasses() {
-        this.runtimeMap.clear();
-        for (Class<?> clazz : classes.keySet()) {
-            runtimeMap.put(clazz, computeRuntimeClass(clazz));
-        }
-    }
-    
-    RuntimeClass computeRuntimeClass(Class<?> clazz) {
-        Struct struct = classes.get(clazz);
-        Map<String, Method> methods = struct.methods;
-        Map<String, MethodHandle> getters = new HashMap<>();
-        Map<String, MethodHandle> setters = new HashMap<>();
-        // add all members
-        for (Map.Entry<String,Field> member : struct.members.entrySet()) {
-            getters.put(member.getKey(), member.getValue().getter);
-            setters.put(member.getKey(), member.getValue().setter);
-        }
-        // add all getters/setters
-        for (Map.Entry<String,Method> method : methods.entrySet()) {
-            String name = method.getKey();
-            Method m = method.getValue();
-            
-            if (m.arguments.size() == 0 &&
-                name.startsWith("get") &&
-                name.length() > 3 &&
-                Character.isUpperCase(name.charAt(3))) {
-              StringBuilder newName = new StringBuilder();
-              newName.append(Character.toLowerCase(name.charAt(3)));
-              newName.append(name.substring(4));
-              getters.putIfAbsent(newName.toString(), m.handle);
-            } else if (m.arguments.size() == 0 &&
-                       name.startsWith("is") &&
-                       name.length() > 2 && 
-                       Character.isUpperCase(name.charAt(2))) {
-              StringBuilder newName = new StringBuilder();
-              newName.append(Character.toLowerCase(name.charAt(2)));
-              newName.append(name.substring(3));
-              getters.putIfAbsent(newName.toString(), m.handle);
-            }
-            
-            if (m.arguments.size() == 1 &&
-                name.startsWith("set") &&
-                name.length() > 3 &&
-                Character.isUpperCase(name.charAt(3))) {
-              StringBuilder newName = new StringBuilder();
-              newName.append(Character.toLowerCase(name.charAt(3)));
-              newName.append(name.substring(4));
-              setters.putIfAbsent(newName.toString(), m.handle);
-            }
-        }
-        return new RuntimeClass(methods, getters, setters);
+        addElements();
+        copyStructs();
+        addTransforms();
+        addRuntimeClasses();
     }
 
     private Definition(final Definition definition) {
         final Map<String, Struct> structs = new HashMap<>();
 
-        for (final Struct struct : definition.structs.values()) {
+        for (final Struct struct : definition.structsMap.values()) {
             structs.put(struct.name, new Struct(struct));
         }
 
-        this.structs = Collections.unmodifiableMap(structs);
-
-        final Map<Class<?>, Struct> classes = new HashMap<>();
-
-        for (final Struct struct : definition.classes.values()) {
-            classes.put(struct.clazz, this.structs.get(struct.name));
-        }
-
-        this.classes = Collections.unmodifiableMap(classes);
-
-        transforms = Collections.unmodifiableMap(definition.transforms);
-        bounds = Collections.unmodifiableMap(definition.bounds);
+        this.structsMap = Collections.unmodifiableMap(structs);
+        this.transformsMap = Collections.unmodifiableMap(definition.transformsMap);
         this.runtimeMap = Collections.unmodifiableMap(definition.runtimeMap);
 
-        voidType = definition.voidType;
-        booleanType = definition.booleanType;
-        byteType = definition.byteType;
-        shortType = definition.shortType;
-        charType = definition.charType;
-        intType = definition.intType;
-        longType = definition.longType;
-        floatType = definition.floatType;
-        doubleType = definition.doubleType;
+        this.voidType = definition.voidType;
+        this.booleanType = definition.booleanType;
+        this.byteType = definition.byteType;
+        this.shortType = definition.shortType;
+        this.charType = definition.charType;
+        this.intType = definition.intType;
+        this.longType = definition.longType;
+        this.floatType = definition.floatType;
+        this.doubleType = definition.doubleType;
 
-        voidobjType = definition.voidobjType;
-        booleanobjType = definition.booleanobjType;
-        byteobjType = definition.byteobjType;
-        shortobjType = definition.shortobjType;
-        charobjType = definition.charobjType;
-        intobjType = definition.intobjType;
-        longobjType = definition.longobjType;
-        floatobjType = definition.floatobjType;
-        doubleobjType = definition.doubleobjType;
+        this.voidobjType = definition.voidobjType;
+        this.booleanobjType = definition.booleanobjType;
+        this.byteobjType = definition.byteobjType;
+        this.shortobjType = definition.shortobjType;
+        this.charobjType = definition.charobjType;
+        this.intobjType = definition.intobjType;
+        this.longobjType = definition.longobjType;
+        this.floatobjType = definition.floatobjType;
+        this.doubleobjType = definition.doubleobjType;
 
-        objectType = definition.objectType;
-        defType = definition.defType;
-        numberType = definition.numberType;
-        charseqType = definition.charseqType;
-        stringType = definition.stringType;
-        mathType = definition.mathType;
-        utilityType = definition.utilityType;
-        defobjType = definition.defobjType;
+        this.objectType = definition.objectType;
+        this.defType = definition.defType;
+        this.numberType = definition.numberType;
+        this.charseqType = definition.charseqType;
+        this.stringType = definition.stringType;
+        this.mathType = definition.mathType;
+        this.utilityType = definition.utilityType;
+        this.defobjType = definition.defobjType;
 
-        itrType = definition.itrType;
-        oitrType = definition.oitrType;
-        sitrType = definition.sitrType;
+        this.itrType = definition.itrType;
+        this.oitrType = definition.oitrType;
+        this.sitrType = definition.sitrType;
 
-        collectionType = definition.collectionType;
-        ocollectionType = definition.ocollectionType;
-        scollectionType = definition.scollectionType;
+        this.collectionType = definition.collectionType;
+        this.ocollectionType = definition.ocollectionType;
+        this.scollectionType = definition.scollectionType;
 
-        listType = definition.listType;
-        arraylistType = definition.arraylistType;
-        olistType = definition.olistType;
-        oarraylistType = definition.oarraylistType;
-        slistType = definition.slistType;
-        sarraylistType = definition.sarraylistType;
+        this.listType = definition.listType;
+        this.arraylistType = definition.arraylistType;
+        this.olistType = definition.olistType;
+        this.oarraylistType = definition.oarraylistType;
+        this.slistType = definition.slistType;
+        this.sarraylistType = definition.sarraylistType;
 
-        setType = definition.setType;
-        hashsetType = definition.hashsetType;
-        osetType = definition.osetType;
-        ohashsetType = definition.ohashsetType;
-        ssetType = definition.ssetType;
-        shashsetType = definition.shashsetType;
+        this.setType = definition.setType;
+        this.hashsetType = definition.hashsetType;
+        this.osetType = definition.osetType;
+        this.ohashsetType = definition.ohashsetType;
+        this.ssetType = definition.ssetType;
+        this.shashsetType = definition.shashsetType;
 
-        mapType = definition.mapType;
-        hashmapType = definition.hashmapType;
-        oomapType = definition.oomapType;
-        oohashmapType = definition.oohashmapType;
-        smapType = definition.smapType;
-        shashmapType = definition.shashmapType;
-        somapType = definition.somapType;
-        sohashmapType = definition.sohashmapType;
+        this.mapType = definition.mapType;
+        this.hashmapType = definition.hashmapType;
+        this.oomapType = definition.oomapType;
+        this.oohashmapType = definition.oohashmapType;
+        this.smapType = definition.smapType;
+        this.shashmapType = definition.shashmapType;
+        this.somapType = definition.somapType;
+        this.sohashmapType = definition.sohashmapType;
 
-        execType = definition.execType;
+        this.execType = definition.execType;
 
-        exceptionType = definition.exceptionType;
-        arithexcepType = definition.arithexcepType;
-        iargexcepType = definition.iargexcepType;
-        istateexceptType = definition.istateexceptType;
-        nfexcepType = definition.nfexcepType;
-        
-        geoPointType = definition.geoPointType;
-        stringsType = definition.stringsType;
-        longsType = definition.longsType;
-        doublesType = definition.doublesType;
-        geoPointsType = definition.geoPointsType;
+        this.exceptionType = definition.exceptionType;
+        this.arithexcepType = definition.arithexcepType;
+        this.iargexcepType = definition.iargexcepType;
+        this.istateexceptType = definition.istateexceptType;
+        this.nfexcepType = definition.nfexcepType;
+
+        this.geoPointType = definition.geoPointType;
+        this.stringsType = definition.stringsType;
+        this.longsType = definition.longsType;
+        this.doublesType = definition.doublesType;
+        this.geoPointsType = definition.geoPointsType;
     }
 
-    private void addDefaultStructs() {
+    private void addStructs() {
         addStruct( "void"    , void.class    );
         addStruct( "boolean" , boolean.class );
         addStruct( "byte"    , byte.class    );
@@ -732,7 +637,7 @@ class Definition {
         addStruct( "IllegalArgumentException" , IllegalArgumentException.class);
         addStruct( "IllegalStateException"    , IllegalStateException.class);
         addStruct( "NumberFormatException"    , NumberFormatException.class);
-        
+
         addStruct( "GeoPoint"  , GeoPoint.class);
         addStruct( "Strings"   , ScriptDocValues.Strings.class);
         addStruct( "Longs"     , ScriptDocValues.Longs.class);
@@ -740,49 +645,7 @@ class Definition {
         addStruct( "GeoPoints" , ScriptDocValues.GeoPoints.class);
     }
 
-    private void addDefaultClasses() {
-        addClass("boolean");
-        addClass("byte");
-        addClass("short");
-        addClass("char");
-        addClass("int");
-        addClass("long");
-        addClass("float");
-        addClass("double");
-
-        addClass("Boolean");
-        addClass("Byte");
-        addClass("Short");
-        addClass("Character");
-        addClass("Integer");
-        addClass("Long");
-        addClass("Float");
-        addClass("Double");
-
-        addClass("Object");
-        addClass("Number");
-        addClass("CharSequence");
-        addClass("String");
-
-        addClass("Iterator");
-        addClass("Collection");
-        addClass("List");
-        addClass("ArrayList");
-        addClass("Set");
-        addClass("HashSet");
-        addClass("Map");
-        addClass("HashMap");
-
-        addClass("Exception");
-        
-        addClass("GeoPoint");
-        addClass("Strings");
-        addClass("Longs");
-        addClass("Doubles");
-        addClass("GeoPoints");
-    }
-
-    private void addDefaultElements() {
+    private void addElements() {
         addMethod("Object", "toString", null, false, stringType, new Type[] {}, null, null);
         addMethod("Object", "equals", null, false, booleanType, new Type[] {objectType}, null, null);
         addMethod("Object", "hashCode", null, false, intType, new Type[] {}, null, null);
@@ -1145,7 +1008,7 @@ class Definition {
         addConstructor("IllegalStateException", "new", new Type[] {stringType}, null);
 
         addConstructor("NumberFormatException", "new", new Type[] {stringType}, null);
-        
+
         addMethod("GeoPoint", "getLat", null, false, doubleType, new Type[] {}, null, null);
         addMethod("GeoPoint", "getLon", null, false, doubleType, new Type[] {}, null, null);
         addMethod("Strings", "getValue", null, false, stringType, new Type[] {}, null, null);
@@ -1162,47 +1025,47 @@ class Definition {
         addMethod("GeoPoints", "getLats", null, false, getType(doubleType.struct, 1), new Type[] {}, null, null);
         addMethod("GeoPoints", "getLons", null, false, getType(doubleType.struct, 1), new Type[] {}, null, null);
         // geo distance functions... so many...
-        addMethod("GeoPoints", "factorDistance", null, false, doubleType, 
+        addMethod("GeoPoints", "factorDistance", null, false, doubleType,
                   new Type[] { doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "factorDistanceWithDefault", null, false, doubleType, 
+        addMethod("GeoPoints", "factorDistanceWithDefault", null, false, doubleType,
                   new Type[] { doubleType, doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "factorDistance02", null, false, doubleType, 
+        addMethod("GeoPoints", "factorDistance02", null, false, doubleType,
                   new Type[] { doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "factorDistance13", null, false, doubleType, 
+        addMethod("GeoPoints", "factorDistance13", null, false, doubleType,
                   new Type[] { doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "arcDistance", null, false, doubleType, 
+        addMethod("GeoPoints", "arcDistance", null, false, doubleType,
                   new Type[] { doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "arcDistanceWithDefault", null, false, doubleType, 
+        addMethod("GeoPoints", "arcDistanceWithDefault", null, false, doubleType,
                   new Type[] { doubleType, doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "arcDistanceInKm", null, false, doubleType, 
+        addMethod("GeoPoints", "arcDistanceInKm", null, false, doubleType,
                   new Type[] { doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "arcDistanceInKmWithDefault", null, false, doubleType, 
+        addMethod("GeoPoints", "arcDistanceInKmWithDefault", null, false, doubleType,
                   new Type[] { doubleType, doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "arcDistanceInMiles", null, false, doubleType, 
+        addMethod("GeoPoints", "arcDistanceInMiles", null, false, doubleType,
                   new Type[] { doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "arcDistanceInMilesWithDefault", null, false, doubleType, 
+        addMethod("GeoPoints", "arcDistanceInMilesWithDefault", null, false, doubleType,
                   new Type[] { doubleType, doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "distance", null, false, doubleType, 
+        addMethod("GeoPoints", "distance", null, false, doubleType,
                   new Type[] { doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "distanceWithDefault", null, false, doubleType, 
+        addMethod("GeoPoints", "distanceWithDefault", null, false, doubleType,
                   new Type[] { doubleType, doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "distanceInKm", null, false, doubleType, 
+        addMethod("GeoPoints", "distanceInKm", null, false, doubleType,
                   new Type[] { doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "distanceInKmWithDefault", null, false, doubleType, 
+        addMethod("GeoPoints", "distanceInKmWithDefault", null, false, doubleType,
                   new Type[] { doubleType, doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "distanceInMiles", null, false, doubleType, 
+        addMethod("GeoPoints", "distanceInMiles", null, false, doubleType,
                   new Type[] { doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "distanceInMilesWithDefault", null, false, doubleType, 
+        addMethod("GeoPoints", "distanceInMilesWithDefault", null, false, doubleType,
                   new Type[] { doubleType, doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "geohashDistance", null, false, doubleType, 
+        addMethod("GeoPoints", "geohashDistance", null, false, doubleType,
                   new Type[] { stringType }, null, null);
-        addMethod("GeoPoints", "geohashDistanceInKm", null, false, doubleType, 
+        addMethod("GeoPoints", "geohashDistanceInKm", null, false, doubleType,
                   new Type[] { stringType }, null, null);
-        addMethod("GeoPoints", "geohashDistanceInMiles", null, false, doubleType, 
+        addMethod("GeoPoints", "geohashDistanceInMiles", null, false, doubleType,
                   new Type[] { stringType }, null, null);
     }
 
-    private void copyDefaultStructs() {
+    private void copyStructs() {
         copyStruct("Void", "Object");
         copyStruct("Boolean", "Object");
         copyStruct("Byte", "Number", "Object");
@@ -1247,7 +1110,7 @@ class Definition {
         copyStruct("IllegalArgumentException", "Exception", "Object");
         copyStruct("IllegalStateException", "Exception", "Object");
         copyStruct("NumberFormatException", "Exception", "Object");
-        
+
         copyStruct("GeoPoint", "Object");
         copyStruct("Strings", "List<String>", "Collection<String>", "Object");
         copyStruct("Longs", "List", "Collection", "Object");
@@ -1255,7 +1118,7 @@ class Definition {
         copyStruct("GeoPoints", "List", "Collection", "Object");
     }
 
-    private void addDefaultTransforms() {
+    private void addTransforms() {
         addTransform(booleanType, byteType, "Utility", "booleanTobyte", true);
         addTransform(booleanType, shortType, "Utility", "booleanToshort", true);
         addTransform(booleanType, charType, "Utility", "booleanTochar", true);
@@ -1530,217 +1393,64 @@ class Definition {
         addTransform(stringType, charobjType, "Utility", "StringToCharacter", true);
     }
 
-    private void addDefaultBounds() {
-        addBound(byteobjType, numberType, numberType);
+    private void addRuntimeClasses() {
+        addRuntimeClass(booleanType.struct);
+        addRuntimeClass(byteType.struct);
+        addRuntimeClass(shortType.struct);
+        addRuntimeClass(charType.struct);
+        addRuntimeClass(intType.struct);
+        addRuntimeClass(longType.struct);
+        addRuntimeClass(floatType.struct);
+        addRuntimeClass(doubleType.struct);
 
-        addBound(shortobjType, numberType, numberType);
-        addBound(shortobjType, byteobjType, numberType);
+        addRuntimeClass(booleanobjType.struct);
+        addRuntimeClass(byteobjType.struct);
+        addRuntimeClass(shortobjType.struct);
+        addRuntimeClass(charobjType.struct);
+        addRuntimeClass(intobjType.struct);
+        addRuntimeClass(longobjType.struct);
+        addRuntimeClass(floatobjType.struct);
+        addRuntimeClass(doubleobjType.struct);
 
-        addBound(intobjType, numberType, numberType);
-        addBound(intobjType, byteobjType, numberType);
-        addBound(intobjType, shortobjType, numberType);
+        addRuntimeClass(objectType.struct);
+        addRuntimeClass(numberType.struct);
+        addRuntimeClass(charseqType.struct);
+        addRuntimeClass(stringType.struct);
 
-        addBound(longobjType, numberType, numberType);
-        addBound(longobjType, byteobjType, numberType);
-        addBound(longobjType, shortobjType, numberType);
-        addBound(longobjType, intobjType, numberType);
+        addRuntimeClass(oitrType.struct);
+        addRuntimeClass(ocollectionType.struct);
+        addRuntimeClass(olistType.struct);
+        addRuntimeClass(oarraylistType.struct);
+        addRuntimeClass(osetType.struct);
+        addRuntimeClass(ohashsetType.struct);
+        addRuntimeClass(oomapType.struct);
+        addRuntimeClass(oohashmapType.struct);
 
-        addBound(floatobjType, numberType, numberType);
-        addBound(floatobjType, byteobjType, numberType);
-        addBound(floatobjType, shortobjType, numberType);
-        addBound(floatobjType, intobjType, numberType);
-        addBound(floatobjType, longobjType, numberType);
+        addRuntimeClass(exceptionType.struct);
 
-        addBound(doubleobjType, numberType, numberType);
-        addBound(doubleobjType, byteobjType, numberType);
-        addBound(doubleobjType, shortobjType, numberType);
-        addBound(doubleobjType, intobjType, numberType);
-        addBound(doubleobjType, longobjType, numberType);
-        addBound(doubleobjType, floatobjType, numberType);
-
-        addBound(stringType, charseqType, charseqType);
-
-        addBound(oitrType, itrType, itrType);
-        addBound(oitrType, sitrType, itrType);
-        addBound(sitrType, itrType, itrType);
-
-        addBound(ocollectionType, collectionType, collectionType);
-        addBound(scollectionType, collectionType, collectionType);
-        addBound(scollectionType, ocollectionType, ocollectionType);
-        addBound(listType, collectionType, collectionType);
-        addBound(listType, ocollectionType, collectionType);
-        addBound(listType, scollectionType, collectionType);
-        addBound(arraylistType, collectionType, collectionType);
-        addBound(arraylistType, ocollectionType, collectionType);
-        addBound(arraylistType, scollectionType, collectionType);
-        addBound(arraylistType, listType, listType);
-        addBound(olistType, collectionType, collectionType);
-        addBound(olistType, ocollectionType, ocollectionType);
-        addBound(olistType, scollectionType, ocollectionType);
-        addBound(olistType, listType, listType);
-        addBound(olistType, arraylistType, listType);
-        addBound(oarraylistType, collectionType, collectionType);
-        addBound(oarraylistType, ocollectionType, ocollectionType);
-        addBound(oarraylistType, scollectionType, ocollectionType);
-        addBound(oarraylistType, listType, listType);
-        addBound(oarraylistType, arraylistType, arraylistType);
-        addBound(oarraylistType, olistType, olistType);
-        addBound(slistType, collectionType, collectionType);
-        addBound(slistType, ocollectionType, ocollectionType);
-        addBound(slistType, scollectionType, scollectionType);
-        addBound(slistType, listType, listType);
-        addBound(slistType, arraylistType, listType);
-        addBound(slistType, olistType, olistType);
-        addBound(slistType, oarraylistType, olistType);
-        addBound(sarraylistType, collectionType, collectionType);
-        addBound(sarraylistType, ocollectionType, ocollectionType);
-        addBound(sarraylistType, scollectionType, scollectionType);
-        addBound(sarraylistType, listType, listType);
-        addBound(sarraylistType, arraylistType, arraylistType);
-        addBound(sarraylistType, olistType, olistType);
-        addBound(sarraylistType, oarraylistType, oarraylistType);
-        addBound(sarraylistType, slistType, slistType);
-        addBound(setType, collectionType, collectionType);
-        addBound(setType, ocollectionType, collectionType);
-        addBound(setType, scollectionType, collectionType);
-        addBound(setType, listType, collectionType);
-        addBound(setType, arraylistType, collectionType);
-        addBound(setType, olistType, collectionType);
-        addBound(setType, oarraylistType, collectionType);
-        addBound(setType, slistType, collectionType);
-        addBound(setType, sarraylistType, collectionType);
-        addBound(hashsetType, collectionType, collectionType);
-        addBound(hashsetType, ocollectionType, collectionType);
-        addBound(hashsetType, scollectionType, collectionType);
-        addBound(hashsetType, listType, collectionType);
-        addBound(hashsetType, arraylistType, collectionType);
-        addBound(hashsetType, olistType, collectionType);
-        addBound(hashsetType, oarraylistType, collectionType);
-        addBound(hashsetType, slistType, collectionType);
-        addBound(hashsetType, sarraylistType, collectionType);
-        addBound(hashsetType, setType, setType);
-        addBound(osetType, collectionType, collectionType);
-        addBound(osetType, ocollectionType, ocollectionType);
-        addBound(osetType, scollectionType, ocollectionType);
-        addBound(osetType, listType, collectionType);
-        addBound(osetType, arraylistType, collectionType);
-        addBound(osetType, olistType, ocollectionType);
-        addBound(osetType, oarraylistType, ocollectionType);
-        addBound(osetType, slistType, ocollectionType);
-        addBound(osetType, sarraylistType, ocollectionType);
-        addBound(osetType, setType, setType);
-        addBound(osetType, hashsetType, setType);
-        addBound(ohashsetType, collectionType, collectionType);
-        addBound(ohashsetType, ocollectionType, ocollectionType);
-        addBound(ohashsetType, scollectionType, ocollectionType);
-        addBound(ohashsetType, listType, collectionType);
-        addBound(ohashsetType, arraylistType, collectionType);
-        addBound(ohashsetType, olistType, ocollectionType);
-        addBound(ohashsetType, oarraylistType, ocollectionType);
-        addBound(ohashsetType, slistType, ocollectionType);
-        addBound(ohashsetType, sarraylistType, ocollectionType);
-        addBound(ohashsetType, setType, setType);
-        addBound(ohashsetType, hashsetType, hashsetType);
-        addBound(ohashsetType, osetType, osetType);
-        addBound(ssetType, collectionType, collectionType);
-        addBound(ssetType, ocollectionType, ocollectionType);
-        addBound(ssetType, scollectionType, scollectionType);
-        addBound(ssetType, listType, collectionType);
-        addBound(ssetType, arraylistType, collectionType);
-        addBound(ssetType, olistType, ocollectionType);
-        addBound(ssetType, oarraylistType, ocollectionType);
-        addBound(ssetType, slistType, scollectionType);
-        addBound(ssetType, sarraylistType, scollectionType);
-        addBound(ssetType, setType, setType);
-        addBound(ssetType, hashsetType, setType);
-        addBound(ssetType, osetType, osetType);
-        addBound(ssetType, ohashsetType, osetType);
-        addBound(shashsetType, collectionType, collectionType);
-        addBound(shashsetType, ocollectionType, ocollectionType);
-        addBound(shashsetType, scollectionType, scollectionType);
-        addBound(shashsetType, listType, collectionType);
-        addBound(shashsetType, arraylistType, collectionType);
-        addBound(shashsetType, olistType, ocollectionType);
-        addBound(shashsetType, oarraylistType, ocollectionType);
-        addBound(shashsetType, slistType, scollectionType);
-        addBound(shashsetType, sarraylistType, scollectionType);
-        addBound(shashsetType, setType, setType);
-        addBound(shashsetType, hashsetType, hashsetType);
-        addBound(shashsetType, osetType, osetType);
-        addBound(shashsetType, ohashsetType, hashsetType);
-        addBound(shashsetType, ssetType, ssetType);
-
-        addBound(hashmapType, mapType, mapType);
-        addBound(oomapType, mapType, mapType);
-        addBound(oomapType, hashmapType, mapType);
-        addBound(oohashmapType, mapType, mapType);
-        addBound(oohashmapType, hashmapType, hashmapType);
-        addBound(oohashmapType, oomapType, oomapType);
-        addBound(smapType, mapType, mapType);
-        addBound(smapType, hashmapType, mapType);
-        addBound(smapType, oomapType, oomapType);
-        addBound(smapType, oohashmapType, oomapType);
-        addBound(shashmapType, mapType, mapType);
-        addBound(shashmapType, hashmapType, hashmapType);
-        addBound(shashmapType, oomapType, oomapType);
-        addBound(shashmapType, oohashmapType, oohashmapType);
-        addBound(shashmapType, smapType, smapType);
-        addBound(somapType, mapType, mapType);
-        addBound(somapType, hashmapType, mapType);
-        addBound(somapType, oomapType, oomapType);
-        addBound(somapType, oohashmapType, oomapType);
-        addBound(somapType, smapType, smapType);
-        addBound(somapType, shashmapType, smapType);
-        addBound(sohashmapType, mapType, mapType);
-        addBound(sohashmapType, hashmapType, hashmapType);
-        addBound(sohashmapType, oomapType, oomapType);
-        addBound(sohashmapType, oohashmapType, oohashmapType);
-        addBound(sohashmapType, smapType, smapType);
-        addBound(sohashmapType, shashmapType, shashmapType);
-        addBound(sohashmapType, somapType, somapType);
-
-        addBound(arithexcepType, exceptionType, exceptionType);
-        addBound(iargexcepType, exceptionType, exceptionType);
-        addBound(istateexceptType, exceptionType, exceptionType);
-        addBound(nfexcepType, exceptionType, exceptionType);
-        addBound(arithexcepType, iargexcepType, exceptionType);
-        addBound(arithexcepType, istateexceptType, exceptionType);
-        addBound(arithexcepType, nfexcepType, exceptionType);
-        addBound(iargexcepType, istateexceptType, exceptionType);
-        addBound(iargexcepType, nfexcepType, exceptionType);
-        addBound(istateexceptType, nfexcepType, exceptionType);
+        addRuntimeClass(geoPointType.struct);
+        addRuntimeClass(stringsType.struct);
+        addRuntimeClass(longsType.struct);
+        addRuntimeClass(doublesType.struct);
+        addRuntimeClass(geoPointsType.struct);
     }
 
-    public final void addStruct(final String name, final Class<?> clazz) {
+    private final void addStruct(final String name, final Class<?> clazz) {
         if (!name.matches("^[_a-zA-Z][<>,_a-zA-Z0-9]*$")) {
             throw new IllegalArgumentException("Invalid struct name [" + name + "].");
         }
 
-        if (structs.containsKey(name)) {
+        if (structsMap.containsKey(name)) {
             throw new IllegalArgumentException("Duplicate struct name [" + name + "].");
         }
 
         final Struct struct = new Struct(name, clazz, org.objectweb.asm.Type.getType(clazz));
 
-        structs.put(name, struct);
+        structsMap.put(name, struct);
     }
 
-    public final void addClass(final String name) {
-        final Struct struct = structs.get(name);
-
-        if (struct == null) {
-            throw new IllegalArgumentException("Struct [" + name + "] is not defined.");
-        }
-
-        if (classes.containsKey(struct.clazz)) {
-            throw new IllegalArgumentException("Duplicate struct class [" + struct.clazz + "] when defining dynamic.");
-        }
-
-        classes.put(struct.clazz, struct);
-    }
-
-    public final void addConstructor(final String struct, final String name, final Type[] args, final Type[] genargs) {
-        final Struct owner = structs.get(struct);
+    private final void addConstructor(final String struct, final String name, final Type[] args, final Type[] genargs) {
+        final Struct owner = structsMap.get(struct);
 
         if (owner == null) {
             throw new IllegalArgumentException(
@@ -1773,7 +1483,7 @@ class Definition {
             if (genargs != null) {
                 try {
                     genargs[count].clazz.asSubclass(args[count].clazz);
-                } catch (ClassCastException exception) {
+                } catch (final ClassCastException exception) {
                     throw new ClassCastException("Generic argument [" + genargs[count].name + "]" +
                         " is not a sub class of [" + args[count].name + "] in the constructor" +
                         " [" + name + " ] from the struct [" + owner.name + "].");
@@ -1787,7 +1497,7 @@ class Definition {
 
         try {
             reflect = owner.clazz.getConstructor(classes);
-        } catch (NoSuchMethodException exception) {
+        } catch (final NoSuchMethodException exception) {
             throw new IllegalArgumentException("Constructor [" + name + "] not found for class" +
                 " [" + owner.clazz.getName() + "] with arguments " + Arrays.toString(classes) + ".");
         }
@@ -1799,9 +1509,9 @@ class Definition {
         owner.constructors.put(name, constructor);
     }
 
-    public final void addMethod(final String struct, final String name, final String alias, final boolean statik,
+    private final void addMethod(final String struct, final String name, final String alias, final boolean statik,
                                 final Type rtn, final Type[] args, final Type genrtn, final Type[] genargs) {
-        final Struct owner = structs.get(struct);
+        final Struct owner = structsMap.get(struct);
 
         if (owner == null) {
             throw new IllegalArgumentException("Owner struct [" + struct + "] not defined" +
@@ -1842,7 +1552,7 @@ class Definition {
         if (genrtn != null) {
             try {
                 genrtn.clazz.asSubclass(rtn.clazz);
-            } catch (ClassCastException exception) {
+            } catch (final ClassCastException exception) {
                 throw new ClassCastException("Generic return [" + genrtn.clazz.getCanonicalName() + "]" +
                     " is not a sub class of [" + rtn.clazz.getCanonicalName() + "] in the method" +
                     " [" + name + " ] from the struct [" + owner.name + "].");
@@ -1861,7 +1571,7 @@ class Definition {
             if (genargs != null) {
                 try {
                     genargs[count].clazz.asSubclass(args[count].clazz);
-                } catch (ClassCastException exception) {
+                } catch (final ClassCastException exception) {
                     throw new ClassCastException("Generic argument [" + genargs[count].name + "] is not a sub class" +
                         " of [" + args[count].name + "] in the " + (statik ? "function" : "method") +
                         " [" + name + " ] from the struct [" + owner.name + "].");
@@ -1875,7 +1585,7 @@ class Definition {
 
         try {
             reflect = owner.clazz.getMethod(alias == null ? name : alias, classes);
-        } catch (NoSuchMethodException exception) {
+        } catch (final NoSuchMethodException exception) {
             throw new IllegalArgumentException((statik ? "Function" : "Method") +
                 " [" + (alias == null ? name : alias) + "] not found for class [" + owner.clazz.getName() + "]" +
                 " with arguments " + Arrays.toString(classes) + ".");
@@ -1894,7 +1604,7 @@ class Definition {
 
         try {
             handle = MethodHandles.publicLookup().in(owner.clazz).unreflect(reflect);
-        } catch (IllegalAccessException exception) {
+        } catch (final IllegalAccessException exception) {
             throw new IllegalArgumentException("Method [" + (alias == null ? name : alias) + "]" +
                 " not found for class [" + owner.clazz.getName() + "]" +
                 " with arguments " + Arrays.toString(classes) + ".");
@@ -1921,9 +1631,9 @@ class Definition {
         }
     }
 
-    public final void addField(final String struct, final String name, final String alias,
+    private final void addField(final String struct, final String name, final String alias,
                                final boolean statik, final Type type, final Type generic) {
-        final Struct owner = structs.get(struct);
+        final Struct owner = structsMap.get(struct);
 
         if (owner == null) {
             throw new IllegalArgumentException("Owner struct [" + struct + "] not defined for " +
@@ -1958,7 +1668,7 @@ class Definition {
         if (generic != null) {
             try {
                 generic.clazz.asSubclass(type.clazz);
-            } catch (ClassCastException exception) {
+            } catch (final ClassCastException exception) {
                 throw new ClassCastException("Generic type [" + generic.clazz.getCanonicalName() + "]" +
                     " is not a sub class of [" + type.clazz.getCanonicalName() + "] for the field" +
                     " [" + name + " ] from the struct [" + owner.name + "].");
@@ -1969,7 +1679,7 @@ class Definition {
 
         try {
             reflect = owner.clazz.getField(alias == null ? name : alias);
-        } catch (NoSuchFieldException exception) {
+        } catch (final NoSuchFieldException exception) {
             throw new IllegalArgumentException("Field [" + (alias == null ? name : alias) + "]" +
                 " not found for class [" + owner.clazz.getName() + "].");
         }
@@ -1982,7 +1692,7 @@ class Definition {
                 getter = MethodHandles.publicLookup().unreflectGetter(reflect);
                 setter = MethodHandles.publicLookup().unreflectSetter(reflect);
             }
-        } catch (IllegalAccessException exception) {
+        } catch (final IllegalAccessException exception) {
             throw new IllegalArgumentException("Getter/Setter [" + (alias == null ? name : alias) + "]" +
                 " not found for class [" + owner.clazz.getName() + "].");
         }
@@ -2011,15 +1721,15 @@ class Definition {
         }
     }
 
-    public final void copyStruct(final String struct, final String... children) {
-        final Struct owner = structs.get(struct);
+    private final void copyStruct(final String struct, final String... children) {
+        final Struct owner = structsMap.get(struct);
 
         if (owner == null) {
             throw new IllegalArgumentException("Owner struct [" + struct + "] not defined for copy.");
         }
 
         for (int count = 0; count < children.length; ++count) {
-            final Struct child = structs.get(children[count]);
+            final Struct child = structsMap.get(children[count]);
 
             if (struct == null) {
                 throw new IllegalArgumentException("Child struct [" + children[count] + "]" +
@@ -2028,7 +1738,7 @@ class Definition {
 
             try {
                 owner.clazz.asSubclass(child.clazz);
-            } catch (ClassCastException exception) {
+            } catch (final ClassCastException exception) {
                 throw new ClassCastException("Child struct [" + child.name + "]" +
                     " is not a super type of owner struct [" + owner.name + "] in copy.");
             }
@@ -2045,7 +1755,7 @@ class Definition {
 
                     try {
                         reflect = clazz.getMethod(method.method.getName(), method.reflect.getParameterTypes());
-                    } catch (NoSuchMethodException exception) {
+                    } catch (final NoSuchMethodException exception) {
                         throw new IllegalArgumentException("Method [" + method.method.getName() + "] not found for" +
                             " class [" + owner.clazz.getName() + "] with arguments " +
                             Arrays.toString(method.reflect.getParameterTypes()) + ".");
@@ -2053,7 +1763,7 @@ class Definition {
 
                     try {
                         handle = MethodHandles.publicLookup().in(owner.clazz).unreflect(reflect);
-                    } catch (IllegalAccessException exception) {
+                    } catch (final IllegalAccessException exception) {
                         throw new IllegalArgumentException("Method [" + method.method.getName() + "] not found for" +
                             " class [" + owner.clazz.getName() + "] with arguments " +
                             Arrays.toString(method.reflect.getParameterTypes()) + ".");
@@ -2072,7 +1782,7 @@ class Definition {
 
                     try {
                         reflect = owner.clazz.getField(field.reflect.getName());
-                    } catch (NoSuchFieldException exception) {
+                    } catch (final NoSuchFieldException exception) {
                         throw new IllegalArgumentException("Field [" + field.reflect.getName() + "]" +
                             " not found for class [" + owner.clazz.getName() + "].");
                     }
@@ -2080,7 +1790,7 @@ class Definition {
                     try {
                         getter = MethodHandles.publicLookup().unreflectGetter(reflect);
                         setter = MethodHandles.publicLookup().unreflectSetter(reflect);
-                    } catch (IllegalAccessException exception) {
+                    } catch (final IllegalAccessException exception) {
                         throw new IllegalArgumentException("Getter/Setter [" + field.name + "]" +
                             " not found for class [" + owner.clazz.getName() + "].");
                     }
@@ -2092,9 +1802,9 @@ class Definition {
         }
     }
 
-    public final void addTransform(final Type from, final Type to, final String struct,
+    private final void addTransform(final Type from, final Type to, final String struct,
                                    final String name, final boolean statik) {
-        final Struct owner = structs.get(struct);
+        final Struct owner = structsMap.get(struct);
 
         if (owner == null) {
             throw new IllegalArgumentException("Owner struct [" + struct + "] not defined for" +
@@ -2108,7 +1818,7 @@ class Definition {
 
         final Cast cast = new Cast(from, to);
 
-        if (transforms.containsKey(cast)) {
+        if (transformsMap.containsKey(cast)) {
             throw new IllegalArgumentException("Transform with owner struct [" + owner.name + "]" +
                 " and cast type from [" + from.name + "] to cast type to [" + to.name + "] already defined.");
         }
@@ -2136,11 +1846,11 @@ class Definition {
 
             try {
                 from.clazz.asSubclass(argument.clazz);
-            } catch (ClassCastException cce0) {
+            } catch (final ClassCastException cce0) {
                 try {
                     argument.clazz.asSubclass(from.clazz);
                     upcast = argument;
-                } catch (ClassCastException cce1) {
+                } catch (final ClassCastException cce1) {
                     throw new ClassCastException("Transform with owner struct [" + owner.name + "]" +
                         " and cast type from [" + from.name + "] to cast type to [" + to.name + "] using" +
                         " function [" + name + "] cannot cast from type to the function input argument type.");
@@ -2151,11 +1861,11 @@ class Definition {
 
             try {
                 rtn.clazz.asSubclass(to.clazz);
-            } catch (ClassCastException cce0) {
+            } catch (final ClassCastException cce0) {
                 try {
                     to.clazz.asSubclass(rtn.clazz);
                     downcast = to;
-                } catch (ClassCastException cce1) {
+                } catch (final ClassCastException cce1) {
                     throw new ClassCastException("Transform with owner struct [" + owner.name + "]" +
                         " and cast type from [" + from.name + "] to cast type to [" + to.name + "] using" +
                         " function [" + name + "] cannot cast to type to the function return argument type.");
@@ -2178,11 +1888,11 @@ class Definition {
 
             try {
                 from.clazz.asSubclass(owner.clazz);
-            } catch (ClassCastException cce0) {
+            } catch (final ClassCastException cce0) {
                 try {
                     owner.clazz.asSubclass(from.clazz);
                     upcast = getType(owner.name);
-                } catch (ClassCastException cce1) {
+                } catch (final ClassCastException cce1) {
                     throw new ClassCastException("Transform with owner struct [" + owner.name + "]" +
                         " and cast type from [" + from.name + "] to cast type to [" + to.name + "] using" +
                         " method [" + name + "] cannot cast from type to the method input argument type.");
@@ -2193,11 +1903,11 @@ class Definition {
 
             try {
                 rtn.clazz.asSubclass(to.clazz);
-            } catch (ClassCastException cce0) {
+            } catch (final ClassCastException cce0) {
                 try {
                     to.clazz.asSubclass(rtn.clazz);
                     downcast = to;
-                } catch (ClassCastException cce1) {
+                } catch (final ClassCastException cce1) {
                     throw new ClassCastException("Transform with owner struct [" + owner.name + "]" +
                         " and cast type from [" + from.name + "] to cast type to [" + to.name + "]" +
                         " using method [" + name + "] cannot cast to type to the method return argument type.");
@@ -2206,31 +1916,64 @@ class Definition {
         }
 
         final Transform transform = new Transform(cast, method, upcast, downcast);
-        transforms.put(cast, transform);
+        transformsMap.put(cast, transform);
     }
 
-    public final void addBound(final Type type0, final Type type1, final Type bound) {
-        final Pair pair0 = new Pair(type0, type1);
-        final Pair pair1 = new Pair(type1, type0);
+    /**
+     * Precomputes a more efficient structure for dynamic method/field access.
+     */
+    private void addRuntimeClass(final Struct struct) {
+        final Map<String, Method> methods = struct.methods;
+        final Map<String, MethodHandle> getters = new HashMap<>();
+        final Map<String, MethodHandle> setters = new HashMap<>();
 
-        if (bounds.containsKey(pair0)) {
-            throw new IllegalArgumentException(
-                "Bound already defined for types [" + type0.name + "] and [" + type1.name + "].");
+        // add all members
+        for (final Map.Entry<String, Field> member : struct.members.entrySet()) {
+            getters.put(member.getKey(), member.getValue().getter);
+            setters.put(member.getKey(), member.getValue().setter);
         }
 
-        if (bounds.containsKey(pair1)) {
-            throw new IllegalArgumentException(
-                "Bound already defined for types [" + type1.name + "] and [" + type0.name + "].");
+        // add all getters/setters
+        for (final Map.Entry<String, Method> method : methods.entrySet()) {
+            final String name = method.getKey();
+            final Method m = method.getValue();
+
+            if (m.arguments.size() == 0 &&
+                name.startsWith("get") &&
+                name.length() > 3 &&
+                Character.isUpperCase(name.charAt(3))) {
+                final StringBuilder newName = new StringBuilder();
+                newName.append(Character.toLowerCase(name.charAt(3)));
+                newName.append(name.substring(4));
+                getters.putIfAbsent(newName.toString(), m.handle);
+            } else if (m.arguments.size() == 0 &&
+                name.startsWith("is") &&
+                name.length() > 2 &&
+                Character.isUpperCase(name.charAt(2))) {
+                final StringBuilder newName = new StringBuilder();
+                newName.append(Character.toLowerCase(name.charAt(2)));
+                newName.append(name.substring(3));
+                getters.putIfAbsent(newName.toString(), m.handle);
+            }
+
+            if (m.arguments.size() == 1 &&
+                name.startsWith("set") &&
+                name.length() > 3 &&
+                Character.isUpperCase(name.charAt(3))) {
+                final StringBuilder newName = new StringBuilder();
+                newName.append(Character.toLowerCase(name.charAt(3)));
+                newName.append(name.substring(4));
+                setters.putIfAbsent(newName.toString(), m.handle);
+            }
         }
 
-        bounds.put(pair0, bound);
-        bounds.put(pair1, bound);
+        runtimeMap.put(struct.clazz, new RuntimeClass(methods, getters, setters));
     }
 
-    Type getType(final String name) {
+    public final Type getType(final String name) {
         final int dimensions = getDimensions(name);
         final String structstr = dimensions == 0 ? name : name.substring(0, name.indexOf('['));
-        final Struct struct = structs.get(structstr);
+        final Struct struct = structsMap.get(structstr);
 
         if (struct == null) {
             throw new IllegalArgumentException("The struct with name [" + name + "] has not been defined.");
@@ -2239,7 +1982,7 @@ class Definition {
         return getType(struct, dimensions);
     }
 
-    Type getType(final Struct struct, final int dimensions) {
+    public final Type getType(final Struct struct, final int dimensions) {
         String name = struct.name;
         org.objectweb.asm.Type type = struct.type;
         Class<?> clazz = struct.clazz;
@@ -2261,7 +2004,7 @@ class Definition {
 
             try {
                 clazz = Class.forName(type.getInternalName().replace('/', '.'));
-            } catch (ClassNotFoundException exception) {
+            } catch (final ClassNotFoundException exception) {
                 throw new IllegalArgumentException("The class [" + type.getInternalName() + "]" +
                     " could not be found to create type [" + name + "].");
             }
@@ -2285,7 +2028,7 @@ class Definition {
             }
         }
 
-        return new Type(name, struct, clazz, type, sort);
+        return new Type(name, dimensions, struct, clazz, type, sort);
     }
 
     private int getDimensions(final String name) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DynamicCallSite.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DynamicCallSite.java
index 116f52836a4..235bfedf8f7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DynamicCallSite.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DynamicCallSite.java
@@ -36,33 +36,34 @@ import java.lang.invoke.MutableCallSite;
  * When a new type is encountered at the call site, we lookup from the appropriate
  * whitelist, and cache with a guard. If we encounter too many types, we stop caching.
  * <p>
- * Based on the cascaded inlining cache from the JSR 292 cookbook 
+ * Based on the cascaded inlining cache from the JSR 292 cookbook
  * (https://code.google.com/archive/p/jsr292-cookbook/, BSD license)
  */
 // NOTE: this class must be public, because generated painless classes are in a different package,
 // and it needs to be accessible by that code.
 public final class DynamicCallSite {
+
     // NOTE: these must be primitive types, see https://docs.oracle.com/javase/specs/jvms/se7/html/jvms-6.html#jvms-6.5.invokedynamic
     /** static bootstrap parameter indicating a dynamic method call, e.g. foo.bar(...) */
-    static final int METHOD_CALL = 0;
+    public static final int METHOD_CALL = 0;
     /** static bootstrap parameter indicating a dynamic load (getter), e.g. baz = foo.bar */
-    static final int LOAD = 1;
+    public static final int LOAD = 1;
     /** static bootstrap parameter indicating a dynamic store (setter), e.g. foo.bar = baz */
-    static final int STORE = 2;
+    public static final int STORE = 2;
     /** static bootstrap parameter indicating a dynamic array load, e.g. baz = foo[bar] */
-    static final int ARRAY_LOAD = 3;
+    public static final int ARRAY_LOAD = 3;
     /** static bootstrap parameter indicating a dynamic array store, e.g. foo[bar] = baz */
-    static final int ARRAY_STORE = 4;
-    
+    public static final int ARRAY_STORE = 4;
+
     static class InliningCacheCallSite extends MutableCallSite {
         /** maximum number of types before we go megamorphic */
         static final int MAX_DEPTH = 5;
-        
+
         final Lookup lookup;
         final String name;
         final int flavor;
         int depth;
-        
+
         InliningCacheCallSite(Lookup lookup, String name, MethodType type, int flavor) {
             super(type);
             this.lookup = lookup;
@@ -70,8 +71,8 @@ public final class DynamicCallSite {
             this.flavor = flavor;
         }
     }
-    
-    /** 
+
+    /**
      * invokeDynamic bootstrap method
      * <p>
      * In addition to ordinary parameters, we also take a static parameter {@code flavor} which
@@ -81,45 +82,45 @@ public final class DynamicCallSite {
      */
     public static CallSite bootstrap(Lookup lookup, String name, MethodType type, int flavor) {
         InliningCacheCallSite callSite = new InliningCacheCallSite(lookup, name, type, flavor);
-        
+
         MethodHandle fallback = FALLBACK.bindTo(callSite);
         fallback = fallback.asCollector(Object[].class, type.parameterCount());
         fallback = fallback.asType(type);
-        
+
         callSite.setTarget(fallback);
         return callSite;
     }
-    
-    /** 
+
+    /**
      * guard method for inline caching: checks the receiver's class is the same
      * as the cached class
      */
     static boolean checkClass(Class<?> clazz, Object receiver) {
         return receiver.getClass() == clazz;
     }
-    
+
     /**
      * Does a slow lookup against the whitelist.
      */
     private static MethodHandle lookup(int flavor, Class<?> clazz, String name) {
         switch(flavor) {
-            case METHOD_CALL: 
+            case METHOD_CALL:
                 return Def.lookupMethod(clazz, name, Definition.INSTANCE);
-            case LOAD: 
+            case LOAD:
                 return Def.lookupGetter(clazz, name, Definition.INSTANCE);
             case STORE:
                 return Def.lookupSetter(clazz, name, Definition.INSTANCE);
-            case ARRAY_LOAD: 
+            case ARRAY_LOAD:
                 return Def.lookupArrayLoad(clazz);
             case ARRAY_STORE:
                 return Def.lookupArrayStore(clazz);
             default: throw new AssertionError();
         }
     }
-    
+
     /**
      * Called when a new type is encountered (or, when we have encountered more than {@code MAX_DEPTH}
-     * types at this call site and given up on caching). 
+     * types at this call site and given up on caching).
      */
     static Object fallback(InliningCacheCallSite callSite, Object[] args) throws Throwable {
         MethodType type = callSite.type();
@@ -127,25 +128,26 @@ public final class DynamicCallSite {
         Class<?> receiverClass = receiver.getClass();
         MethodHandle target = lookup(callSite.flavor, receiverClass, callSite.name);
         target = target.asType(type);
-        
+
         if (callSite.depth >= InliningCacheCallSite.MAX_DEPTH) {
             // revert to a vtable call
             callSite.setTarget(target);
             return target.invokeWithArguments(args);
         }
-        
+
         MethodHandle test = CHECK_CLASS.bindTo(receiverClass);
         test = test.asType(test.type().changeParameterType(0, type.parameterType(0)));
-        
+
         MethodHandle guard = MethodHandles.guardWithTest(test, target, callSite.getTarget());
         callSite.depth++;
-        
+
         callSite.setTarget(guard);
         return target.invokeWithArguments(args);
     }
-    
+
     private static final MethodHandle CHECK_CLASS;
     private static final MethodHandle FALLBACK;
+
     static {
         Lookup lookup = MethodHandles.lookup();
         try {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java
index 199fe5c11ba..7681ae8232a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java
@@ -24,15 +24,15 @@ import org.elasticsearch.search.lookup.LeafDocLookup;
 
 import java.util.Map;
 
+/**
+ * The superclass used to build all Painless scripts on top of.
+ */
 public abstract class Executable {
-    protected final Definition definition;
 
     private final String name;
     private final String source;
 
-    public Executable(final Definition definition, final String name, final String source) {
-        this.definition = definition;
-
+    public Executable(final String name, final String source) {
         this.name = name;
         this.source = source;
     }
@@ -45,9 +45,6 @@ public abstract class Executable {
         return source;
     }
 
-    public Definition getDefinition() {
-        return definition;
-    }
-
-    public abstract Object execute(Map<String, Object> params, Scorer scorer, LeafDocLookup doc, Object value);
+    public abstract Object execute(
+        final Map<String, Object> params, final Scorer scorer, final LeafDocLookup doc, final Object value);
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java
deleted file mode 100644
index 0a326b6393f..00000000000
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Metadata.java
+++ /dev/null
@@ -1,605 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless;
-
-import org.antlr.v4.runtime.ParserRuleContext;
-import org.elasticsearch.painless.Definition.Cast;
-import org.elasticsearch.painless.Definition.Type;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Metadata is a wrapper for all the data that is collected by the {@link Analyzer}.  Each node in the ANTLR parse tree
- * will have one of the types of metadata to store information used either in a different node by the analyzer
- * or by the {@link Writer} during byte code generation.  Metadata also contains several objects passed into the
- * {@link Analyzer} and {@link Writer} used during compilation including the {@link Definition}, the source code,
- * the root of the ANTLR parse tree, and the {@link CompilerSettings}.
- */
-class Metadata {
-    /**
-     * StatementMetadata is used to store metadata mostly about
-     * control flow for ANTLR nodes related to if/else, do, while, for, etc.
-     */
-    static class StatementMetadata {
-        /**
-         * The source variable is the ANTLR node used to generate this metadata.
-         */
-        final ParserRuleContext source;
-
-        /**
-         * The lastSource variable will be set to true when the final statement from the root ANTLR node is about
-         * to be visited.  This is used to determine whether or not the auto-return feature is allowed to be used,
-         * and if a null return value needs to be generated automatically since a return value is always required.
-         */
-        boolean lastSource = false;
-
-        /**
-         * The beginLoop variable will be set to true whenever a loop node is initially visited including inner
-         * loops.  This will not be propagated down the parse tree afterwards, though.  This is used to determine
-         * whether or not inLoop should be set further down the tree.  Note that inLoop alone is not enough
-         * information to determine whether we are in the last statement of a loop because we may inside of
-         * multiple loops, so this variable is necessary.
-         */
-        boolean beginLoop = false;
-
-        /**
-         * The inLoop variable is set to true when inside a loop.  This will be propagated down the parse tree.  This
-         * is used to determine whether or not continue and break statements are legal.
-         */
-        boolean inLoop = false;
-
-        /**
-         * The lastLoop variable is set to true when the final statement of a loop is reached.  This will be
-         * propagated down the parse tree until another loop is reached and then will not be propagated further for
-         * the current loop.  This is used to determine whether or not a continue statement is superfluous.
-         */
-        boolean lastLoop = false;
-
-        /**
-         * The methodEscape variable is set to true when a statement would cause the method to potentially exit.  This
-         * includes return, throw, and continuous loop statements.  Note that a catch statement may possibly
-         * reset this to false after a throw statement.  This will be propagated up the tree as far as necessary.
-         * This is used by the {@link Writer} to ensure that superfluous statements aren't unnecessarily written
-         * into generated bytecode.
-         */
-        boolean methodEscape = false;
-
-        /**
-         * The loopEscape variable is set to true when a loop is going to be exited.  This may be caused by a number of
-         * different statements including continue, break, return, etc.  This will only be propagated as far as the
-         * loop node.  This is used to ensure that in certain case an infinite loop will be caught at
-         * compile-time rather than run-time.
-         */
-        boolean loopEscape = false;
-
-        /**
-         * The allLast variable is set whenever a final statement in a block is reached. This includes the end of loop,
-         * if, else, etc.  This will be only propagated to the top of the block statement ANTLR node.
-         * This is used to ensure that there are no unreachable statements within the script.
-         */
-        boolean allLast = false;
-
-        /**
-         * The anyContinue will be set to true when a continue statement is visited.  This will be propagated to the
-         * loop node it's within.  This is used to ensure that in certain case an infinite loop will be caught at
-         * compile-time rather than run-time.
-         */
-        boolean anyContinue = false;
-
-        /**
-         * The anyBreak will be set to true when a break statement is visited.  This will be propagated to the
-         * loop node it's within.  This is used to in conjunction with methodEscape to ensure there are no unreachable
-         * statements within the script.
-         */
-        boolean anyBreak = false;
-
-        /**
-         * The count variable is used as a rudimentary count of statements within a loop.  This will be used in
-         * the {@link Writer} to keep a count of statements that have been executed at run-time to ensure that a loop
-         * will exit if it runs too long.
-         */
-        int count = 0;
-
-        /**
-         * The exception variable is used to store the exception type when a throw node is visited.  This is used by
-         * the {@link Writer} to write the correct type of exception in the generated byte code.
-         */
-        Type exception = null;
-
-        /**
-         * The slot variable is used to store the place on the stack of where a thrown exception will be stored to.
-         * This is used by the {@link Writer}.
-         */
-        int slot = -1;
-
-        /**
-         * Constructor.
-         * @param source The associated ANTLR node.
-         */
-        private StatementMetadata(final ParserRuleContext source) {
-            this.source = source;
-        }
-    }
-
-    /**
-     * ExpressionMetadata is used to store metadata mostly about constants and casting
-     * for ANTLR nodes related to mathematical operations.
-     */
-    static class ExpressionMetadata {
-        /**
-         * The source variable is the ANTLR node used to generate this metadata.
-         */
-        final ParserRuleContext source;
-
-        /**
-         * The read variable is used to determine whether or not the value of an expression will be read from.
-         * This is set to false when the expression is the left-hand side of an assignment that is not chained or
-         * when a method call is made alone.  This will propagate down the tree as far as necessary.
-         * The {@link Writer} uses this to determine when a value may need to be popped from the stack
-         * such as when a method call returns a value that is never read.
-         */
-        boolean read = true;
-
-        /**
-         * The statement variable is set true when an expression is a complete meaning that there is some sort
-         * of effect on a variable or a method call is made.  This will propagate up the tree as far as necessary.
-         * This prevents statements that have no effect on the output of a script from being executed.
-         */
-        boolean statement = false;
-
-        /**
-         * The preConst variable is set to a non-null value when a constant statement is made in a script.  This is
-         * used to track the constant value prior to any casts being made on an ANTLR node.
-         */
-        Object preConst = null;
-
-        /**
-         * The postConst variable is set to a non-null value when a cast is made on a node where a preConst variable
-         * has already been set when the cast would leave the constant as a non-object value except in the case of a
-         * String.  This will be propagated up the tree and used to simplify constants when possible such as making
-         * the value of 2*2 be 4 in the * node, so that the {@link Writer} only has to push a 4 onto the stack.
-         */
-        Object postConst = null;
-
-        /**
-         * The isNull variable is set to true when a null constant statement is made in the script.  This allows the
-         * {@link Writer} to potentially shortcut certain comparison operations.
-         */
-        boolean isNull = false;
-
-        /**
-         * The to variable is used to track what an ANTLR node's value should be cast to.  This is set on every ANTLR
-         * node in the tree, and used by the {@link Writer} to make a run-time cast if necessary in the byte code.
-         * This is also used by the {@link Analyzer} to determine if a cast is legal.
-         */
-        Type to = null;
-
-        /**
-         * The from variable is used to track what an ANTLR node's value should be cast from.  This is set on every
-         * ANTLR node in the tree independent of other nodes.  This is used by the {@link Analyzer} to determine if a
-         * cast is legal.
-         */
-        Type from = null;
-
-        /**
-         * The explicit variable is set to true when a cast is explicitly made in the script.  This tracks whether
-         * or not a cast is a legal up cast.
-         */
-        boolean explicit = false;
-
-        /**
-         * The typesafe variable is set to true when a dynamic type is used as part of an expression.  This propagates
-         * up the tree to the top of the expression.  This allows for implicit up casts throughout the expression and
-         * is used by the {@link Analyzer}.
-         */
-        boolean typesafe = true;
-
-        /**
-         * This is set to the combination of the to and from variables at the end of each node visit in the
-         * {@link Analyzer}.  This is set on every ANTLR node in the tree independent of other nodes, and is
-         * used by {@link Writer} to make a run-time cast if necessary in the byte code.
-         */
-        Cast cast = null;
-
-        /**
-         * Constructor.
-         * @param source The associated ANTLR node.
-         */
-        private ExpressionMetadata(final ParserRuleContext source) {
-            this.source = source;
-        }
-    }
-
-    /**
-     * ExternalMetadata is used to store metadata about the overall state of a variable/method chain such as
-     * '(int)x.get(3)' where each piece of that chain is broken into it's indiviual pieces and stored in
-     * {@link ExtNodeMetadata}.
-     */
-    static class ExternalMetadata {
-        /**
-         * The source variable is the ANTLR node used to generate this metadata.
-         */
-        final ParserRuleContext source;
-
-        /**
-         * The read variable is set to true when the value of a variable/method chain is going to be read from.
-         * This is used by the {@link Analyzer} to determine if this variable/method chain will be in a standalone
-         * statement.
-         */
-        boolean read = false;
-
-        /**
-         * The storeExpr variable is set to the right-hand side of an assignment in the variable/method chain if
-         * necessary.  This is used by the {@link Analyzer} to set the proper metadata for a read versus a write,
-         * and is used by the {@link Writer} to determine if a bytecode operation should be a load or a store.
-         */
-        ParserRuleContext storeExpr = null;
-
-        /**
-         * The token variable is set to a constant value of the operator type (+, -, etc.) when a compound assignment
-         * is being visited.  This is also used by the increment and decrement operators.  This is used by both the
-         * {@link Analyzer} and {@link Writer} to correctly handle the compound assignment.
-         */
-        int token = 0;
-
-        /**
-         * The pre variable is set to true when pre-increment or pre-decrement is visited.  This is used by both the
-         * {@link Analyzer} and {@link Writer} to correctly handle any reads of the variable/method chain that are
-         * necessary.
-         */
-        boolean pre = false;
-
-        /**
-         * The post variable is set to true when post-increment or post-decrement is visited. This is used by both the
-         * {@link Analyzer} and {@link Writer} to correctly handle any reads of the variable/method chain that are
-         * necessary.
-         */
-        boolean post = false;
-
-        /**
-         * The scope variable is incremented and decremented when a precedence node is visited as part of a
-         * variable/method chain.  This is used by the {@link Analyzer} to determine when the final piece of the
-         * variable/method chain has been reached.
-         */
-        int scope = 0;
-
-        /**
-         * The current variable is set to whatever the current type is within the visited node of the variable/method
-         * chain.  This changes as the nodes for the variable/method are walked through.  This is used by the
-         * {@link Analyzer} to make decisions about whether or not a cast is legal, and what methods are available
-         * for that specific type.
-         */
-        Type current = null;
-
-        /**
-         * The statik variable is set to true when a variable/method chain begins with static type.  This is used by
-         * the {@link Analyzer} to determine what methods/members are available for that specific type.
-         */
-        boolean statik = false;
-
-        /**
-         * The statement variable is set to true when a variable/method chain can be standalone statement.  This is
-         * used by the {@link Analyzer} to error out if there a variable/method chain that is not a statement.
-         */
-        boolean statement = false;
-
-        /**
-         * The constant variable is set when a String constant is part of the variable/method chain.  String is a
-         * special case because methods/members need to be able to be called on a String constant, so this can't be
-         * only as part of {@link ExpressionMetadata}.  This is used by the {@link Writer} to write out the String
-         * constant in the byte code.
-         */
-        Object constant = null;
-
-        /**
-         * Constructor.
-         * @param source The associated ANTLR node.
-         */
-        private ExternalMetadata(final ParserRuleContext source) {
-            this.source = source;
-        }
-    }
-
-    static class ExtNodeMetadata {
-        /**
-         * The parent variable is top-level ANTLR node of the variable/method chain.  This is used to retrieve the
-         * ExternalMetadata for the variable/method chain this ExtNodeMetadata is a piece of.
-         */
-        final ParserRuleContext parent;
-
-        /**
-         * The source variable is the ANTLR node used to generate this metadata.
-         */
-        final ParserRuleContext source;
-
-        /**
-         * The target variable is set to a value based on the type of ANTLR node that is visited.  This is used by
-         * {@link Writer} to determine whether a cast, store, load, or method call should be written in byte code
-         * depending on what the target variable is.
-         */
-        Object target = null;
-
-        /**
-         * The last variable is set to true when the last ANTLR node of the variable/method chain is visted.  This is
-         * used by the {@link Writer} in conjuction with the storeExpr variable to determine whether or not a store
-         * needs to be written as opposed to a load.
-         */
-        boolean last = false;
-
-        /**
-         * The type variable is set to the type that a visited node ends with.  This is used by both the
-         * {@link Analyzer} and {@link Writer} to make decisions about compound assignments, String constants, and
-         * shortcuts.
-         */
-        Type type = null;
-
-        /**
-         * The promote variable is set to the type of a promotion within a compound assignment.  Compound assignments
-         * may require promotion between the left-hand side variable and right-hand side value.  This is used by the
-         * {@link Writer} to make the correct decision about the byte code operation.
-         */
-        Type promote = null;
-
-        /**
-         * The castFrom variable is set during a compound assignment.  This is used by the {@link Writer} to
-         * cast the values to the promoted type during a compound assignment.
-         */
-        Cast castFrom = null;
-
-        /**
-         * The castTo variable is set during an explicit cast in a variable/method chain or during a compound
-         * assignment.  This is used by the {@link Writer} to either do an explicit cast, or cast the values
-         * from the promoted type back to the original type during a compound assignment.
-         */
-        Cast castTo = null;
-
-        /**
-         * Constructor.
-         * @param parent The top-level ANTLR node for the variable/method chain.
-         * @param source The associated ANTLR node.
-         */
-        private ExtNodeMetadata(final ParserRuleContext parent, final ParserRuleContext source) {
-            this.parent = parent;
-            this.source = source;
-        }
-    }
-
-    /**
-     * Acts as both the Painless API and white-list for what types and methods are allowed.
-     */
-    final Definition definition;
-
-    /**
-     * The original text of the input script.  This is used to write out the source code into
-     * the byte code file for debugging purposes.
-     */
-    final String source;
-
-    /**
-     * Toot node of the ANTLR tree for the Painless script.
-     */
-    final ParserRuleContext root;
-
-    /**
-     * Used to determine certain compile-time constraints such as whether or not numeric overflow is allowed
-     * and how many statements are allowed before a loop will throw an exception.
-     */
-    final CompilerSettings settings;
-
-    /**
-     * Used to determine what slot the input variable is stored in.  This is used in the {@link Writer} whenever
-     * the input variable is accessed.
-     */
-    int paramsValueSlot = -1;
-
-    /**
-     * Used to determine what slot the Scorer variable is stored in.  This is used in the {@link Writer} to load
-     * _score from it, if _score will be accessed by the script.
-     */
-    int scorerValueSlot = -1;
-    
-    /**
-     * Used to determine what slot the _value variable is scored in. 
-     */
-    int aggregationValueSlot = -1;
-
-    /**
-     * Used to determine what slot the loopCounter variable is stored in.  This is used n the {@link Writer} whenever
-     * the loop variable is accessed.
-     */
-    int loopCounterSlot = -1;
-
-    /**
-     * Used to determine what slot the _score variable is stored in.  This is used in the {@link Writer} whenever
-     * the score variable is accessed.
-     */
-    int scoreValueSlot = -1;
-
-    /**
-     * Used to determine if the _score variable is actually used.  This is used to know if we should call
-     * Scorer.score() once and cache into a local variable, and expose NeedsScore interface (to allow query caching)
-     */
-    boolean scoreValueUsed = false;
-    
-    /**
-     * Used to determine what slot the doc variable is stored in.  This is used in the {@link Writer} whenever
-     * the doc variable is accessed.
-     */
-    int docValueSlot = -1;
-    
-    /**
-     * Used to determine what slot the ctx variable is stored in.  This is used in the {@link Writer} whenever
-     * the ctx variable is accessed.
-     */
-    int ctxValueSlot = -1;
-    
-    /**
-     * Used to determine if the ctx variable is actually used.  This is used to determine if we should call
-     * Map.get once and store into a local variable on startup.
-     */
-    boolean ctxValueUsed = false;
-
-    /**
-     * Maps the relevant ANTLR node to its metadata.
-     */
-    private final Map<ParserRuleContext, StatementMetadata> statementMetadata = new HashMap<>();
-
-    /**
-     * Maps the relevant ANTLR node to its metadata.
-     */
-    private final Map<ParserRuleContext, ExpressionMetadata> expressionMetadata = new HashMap<>();
-
-    /**
-     * Maps the relevant ANTLR node to its metadata.
-     */
-    private final Map<ParserRuleContext, ExternalMetadata> externalMetadata = new HashMap<>();
-
-    /**
-     * Maps the relevant ANTLR node to its metadata.
-     */
-    private final Map<ParserRuleContext, ExtNodeMetadata> extNodeMetadata = new HashMap<>();
-
-    /**
-     * Constructor.
-     * @param definition The Painless definition.
-     * @param source The source text for the script.
-     * @param root The root ANTLR node.
-     * @param settings The compile-time settings.
-     */
-    Metadata(final Definition definition, final String source, final ParserRuleContext root, final CompilerSettings settings) {
-        this.definition = definition;
-        this.source = source;
-        this.root = root;
-        this.settings = settings;
-    }
-
-    /**
-     * Creates a new StatementMetadata and stores it in the statementMetadata map.
-     * @param source The ANTLR node for this metadata.
-     * @return The new StatementMetadata.
-     */
-    StatementMetadata createStatementMetadata(final ParserRuleContext source) {
-        final StatementMetadata sourcesmd = new StatementMetadata(source);
-        statementMetadata.put(source, sourcesmd);
-
-        return sourcesmd;
-    }
-
-    /**
-     * Retrieves StatementMetadata from the statementMetadata map.
-     * @param source The ANTLR node for this metadata.
-     * @return The retrieved StatementMetadata.
-     */
-    StatementMetadata getStatementMetadata(final ParserRuleContext source) {
-        final StatementMetadata sourcesmd = statementMetadata.get(source);
-
-        if (sourcesmd == null) {
-            throw new IllegalStateException("Statement metadata does not exist at" +
-                " the parse node with text [" + source.getText() + "].");
-        }
-
-        return sourcesmd;
-    }
-
-    /**
-     * Creates a new ExpressionMetadata and stores it in the expressionMetadata map.
-     * @param source The ANTLR node for this metadata.
-     * @return The new ExpressionMetadata.
-     */
-    ExpressionMetadata createExpressionMetadata(ParserRuleContext source) {
-        final ExpressionMetadata sourceemd = new ExpressionMetadata(source);
-        expressionMetadata.put(source, sourceemd);
-
-        return sourceemd;
-    }
-
-    /**
-     * Retrieves ExpressionMetadata from the expressionMetadata map.
-     * @param source The ANTLR node for this metadata.
-     * @return The retrieved ExpressionMetadata.
-     */
-    ExpressionMetadata getExpressionMetadata(final ParserRuleContext source) {
-        final ExpressionMetadata sourceemd = expressionMetadata.get(source);
-
-        if (sourceemd == null) {
-            throw new IllegalStateException("Expression metadata does not exist at" +
-                " the parse node with text [" + source.getText() + "].");
-        }
-
-        return sourceemd;
-    }
-
-    /**
-     * Creates a new ExternalMetadata and stores it in the externalMetadata map.
-     * @param source The ANTLR node for this metadata.
-     * @return The new ExternalMetadata.
-     */
-    ExternalMetadata createExternalMetadata(final ParserRuleContext source) {
-        final ExternalMetadata sourceemd = new ExternalMetadata(source);
-        externalMetadata.put(source, sourceemd);
-
-        return sourceemd;
-    }
-
-    /**
-     * Retrieves ExternalMetadata from the externalMetadata map.
-     * @param source The ANTLR node for this metadata.
-     * @return The retrieved ExternalMetadata.
-     */
-    ExternalMetadata getExternalMetadata(final ParserRuleContext source) {
-        final ExternalMetadata sourceemd = externalMetadata.get(source);
-
-        if (sourceemd == null) {
-            throw new IllegalStateException("External metadata does not exist at" +
-                " the parse node with text [" + source.getText() + "].");
-        }
-
-        return sourceemd;
-    }
-
-    /**
-     * Creates a new ExtNodeMetadata and stores it in the extNodeMetadata map.
-     * @param source The ANTLR node for this metadata.
-     * @return The new ExtNodeMetadata.
-     */
-    ExtNodeMetadata createExtNodeMetadata(final ParserRuleContext parent, final ParserRuleContext source) {
-        final ExtNodeMetadata sourceemd = new ExtNodeMetadata(parent, source);
-        extNodeMetadata.put(source, sourceemd);
-
-        return sourceemd;
-    }
-
-    /**
-     * Retrieves ExtNodeMetadata from the extNodeMetadata map.
-     * @param source The ANTLR node for this metadata.
-     * @return The retrieved ExtNodeMetadata.
-     */
-    ExtNodeMetadata getExtNodeMetadata(final ParserRuleContext source) {
-        final ExtNodeMetadata sourceemd = extNodeMetadata.get(source);
-
-        if (sourceemd == null) {
-            throw new IllegalStateException("External metadata does not exist at" +
-                " the parse node with text [" + source.getText() + "].");
-        }
-
-        return sourceemd;
-    }
-}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/NeedsScore.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/NeedsScore.java
index 180b8c9d8e2..6812e1019b2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/NeedsScore.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/NeedsScore.java
@@ -18,6 +18,8 @@
  */
 package org.elasticsearch.painless;
 
-/** Marker interface that a generated {@link Executable} uses the {@code _score} value */
+/**
+ * Marker interface that a generated {@link Executable} uses the {@code _score} value
+ */
 public interface NeedsScore {
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java
new file mode 100644
index 00000000000..9cafd1d33d7
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+/**
+ * Provides a way to represent operations independently of ASM, to keep ASM
+ * contained to only the writing phase of compilation.  Note there are also
+ * a few extra operations not in ASM that are used internally by the
+ * Painless tree.
+ */
+public enum Operation {
+
+    MUL     ( "+"   ),
+    DIV     ( "/"   ),
+    REM     ( "%"   ),
+    ADD     ( "+"   ),
+    SUB     ( "-"   ),
+    LSH     ( "<<"  ),
+    RSH     ( ">>"  ),
+    USH     ( ">>>" ),
+    BWNOT   ( "~"   ),
+    BWAND   ( "&"   ),
+    XOR     ( "^"   ),
+    BWOR    ( "|"   ),
+    NOT     ( "!"   ),
+    AND     ( "&&"  ),
+    OR      ( "||"  ),
+    LT      ( "<"   ),
+    LTE     ( "<="  ),
+    GT      ( ">"   ),
+    GTE     ( ">="  ),
+    EQ      ( "=="  ),
+    EQR     ( "===" ),
+    NE      ( "!="  ),
+    NER     ( "!==" ),
+    INCR    ( "++"  ),
+    DECR    ( "--"  );
+
+    public final String symbol;
+
+    Operation(final String symbol) {
+        this.symbol = symbol;
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessError.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessError.java
index 68eac7a260e..f2709272667 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessError.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessError.java
@@ -28,6 +28,7 @@ package org.elasticsearch.painless;
  */
 @SuppressWarnings("serial")
 public class PainlessError extends Error {
+
     /**
      * Constructor.
      * @param message The error message.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
index a2e72e52bbb..8ab6195ed18 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
@@ -23,6 +23,9 @@ import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptModule;
 
+/**
+ * Registers Painless as a plugin.
+ */
 public final class PainlessPlugin extends Plugin {
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
index 6f3eb4cfbab..c219d747c0b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
@@ -24,6 +24,7 @@ import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.painless.Compiler.Loader;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.LeafSearchScript;
@@ -45,7 +46,7 @@ import java.util.Map;
 /**
  * Implementation of a ScriptEngine for the Painless language.
  */
-public class PainlessScriptEngineService extends AbstractComponent implements ScriptEngineService {
+public final class PainlessScriptEngineService extends AbstractComponent implements ScriptEngineService {
 
     /**
      * Standard name of the Painless language.
@@ -60,7 +61,7 @@ public class PainlessScriptEngineService extends AbstractComponent implements Sc
     /**
      * Standard extension of the Painless language.
      */
-    public static final String EXTENSION = "pain";
+    public static final String EXTENSION = "painless";
 
     /**
      * Standard list of extensions for the Painless language.  (There is only one.)
@@ -166,10 +167,10 @@ public class PainlessScriptEngineService extends AbstractComponent implements Sc
         }
 
         // Create our loader (which loads compiled code with no permissions).
-        final Compiler.Loader loader = AccessController.doPrivileged(new PrivilegedAction<Compiler.Loader>() {
+        final Loader loader = AccessController.doPrivileged(new PrivilegedAction<Loader>() {
             @Override
-            public Compiler.Loader run() {
-                return new Compiler.Loader(getClass().getClassLoader());
+            public Loader run() {
+                return new Loader(getClass().getClassLoader());
             }
         });
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
index c500ed712fa..23be345ed1c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
@@ -33,6 +33,7 @@ import java.util.Map;
  * to run a previously compiled Painless script.
  */
 final class ScriptImpl implements ExecutableScript, LeafSearchScript {
+
     /**
      * The Painless Executable script that can be run.
      */
@@ -47,7 +48,7 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
      * The lookup is used to access search field values at run-time.
      */
     private final LeafSearchLookup lookup;
-    
+
     /**
      * the 'doc' object accessed by the script, if available.
      */
@@ -97,7 +98,11 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
     public void setNextVar(final String name, final Object value) {
         variables.put(name, value);
     }
-    
+
+    /**
+     * Set the next aggregation value.
+     * @param value Per-document value, typically a String, Long, or Double.
+     */
     @Override
     public void setNextAggregationValue(Object value) {
         this.aggregationValue = value;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java
index ee5a5f8cd1c..32641649827 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java
@@ -19,7 +19,12 @@
 
 package org.elasticsearch.painless;
 
+/**
+ * A set of methods for non-native boxing and non-native
+ * exact math operations used at both compile-time and runtime.
+ */
 public class Utility {
+
     public static boolean NumberToboolean(final Number value) {
         return value.longValue() != 0;
     }
@@ -818,11 +823,11 @@ public class Utility {
     }
 
     public static boolean checkEquals(final Object left, final Object right) {
-        if (left != null && right != null) {
+        if (left != null) {
             return left.equals(right);
         }
 
-        return left == null && right == null;
+        return right == null || right.equals(null);
     }
 
     private Utility() {}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
new file mode 100644
index 00000000000..adf930b017c
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.elasticsearch.painless.Definition.Type;
+
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.Iterator;
+
+/**
+ * Tracks variables across compilation phases.
+ */
+public final class Variables {
+
+    /**
+     * Tracks reserved variables.  Must be given to any source of input
+     * prior to beginning the analysis phase so that reserved variables
+     * are known ahead of time to assign appropriate slots without
+     * being wasteful.
+     */
+    public static final class Reserved {
+        public static final String THIS   = "#this";
+        public static final String PARAMS = "params";
+        public static final String SCORER = "#scorer";
+        public static final String DOC    = "doc";
+        public static final String VALUE  = "_value";
+        public static final String SCORE  = "_score";
+        public static final String CTX    = "ctx";
+        public static final String LOOP   = "#loop";
+
+        boolean score = false;
+        boolean ctx = false;
+        boolean loop = false;
+
+        public void markReserved(final String name) {
+            if (SCORE.equals(name)) {
+                score = true;
+            } else if (CTX.equals(name)) {
+                ctx = true;
+            }
+        }
+
+        public boolean isReserved(final String name) {
+            return name.equals(THIS) || name.equals(PARAMS) || name.equals(SCORER) || name.equals(DOC) ||
+                name.equals(VALUE) || name.equals(SCORE) || name.equals(CTX) || name.equals(LOOP);
+         }
+
+        public void usesLoop() {
+            loop = true;
+        }
+    }
+
+    public static final class Variable {
+        public final String location;
+        public final String name;
+        public final Type type;
+        public final int slot;
+        public final boolean readonly;
+
+        public boolean read = false;
+
+        private Variable(final String location, final String name, final Type type, final int slot, final boolean readonly) {
+            this.location = location;
+            this.name = name;
+            this.type = type;
+            this.slot = slot;
+            this.readonly = readonly;
+        }
+    }
+
+    private final Definition definition;
+    final Reserved reserved;
+
+    private final Deque<Integer> scopes = new ArrayDeque<>();
+    private final Deque<Variable> variables = new ArrayDeque<>();
+
+    public Variables(final CompilerSettings settings, final Definition definition, final Reserved reserved) {
+        this.definition = definition;
+        this.reserved = reserved;
+
+        incrementScope();
+
+        // Method variables.
+
+        // This reference.  Internal use only.
+        addVariable("[" + Reserved.THIS + "]"  , definition.execType.name, Reserved.THIS  , true, true);
+
+        // Input map of variables passed to the script.  TODO: Rename to 'params' since that will be its use.
+        addVariable("[" + Reserved.PARAMS + "]", definition.smapType.name, Reserved.PARAMS, true, true);
+
+        // Scorer parameter passed to the script.  Internal use only.
+        addVariable("[" + Reserved.SCORER + "]", definition.defType.name , Reserved.SCORER, true, true);
+
+        // Doc parameter passed to the script. TODO: Currently working as a Map<String,Def>, we can do better?
+        addVariable("[" + Reserved.DOC + "]"   , definition.smapType.name, Reserved.DOC   , true, true);
+
+        // Aggregation _value parameter passed to the script.
+        addVariable("[" + Reserved.VALUE + "]" , definition.defType.name , Reserved.VALUE , true, true);
+
+        // Shortcut variables.
+
+        // Document's score as a read-only double.
+        if (reserved.score) {
+            addVariable("[" + Reserved.SCORE + "]", definition.doubleType.name, Reserved.SCORE, true, true);
+        }
+
+        // The ctx map set by executable scripts as a read-only map.
+        if (reserved.ctx) {
+            addVariable("[" + Reserved.CTX + "]", definition.smapType.name, Reserved.CTX, true, true);
+        }
+
+        // Loop counter to catch infinite loops.  Internal use only.
+        if (reserved.loop && settings.getMaxLoopCounter() > 0) {
+            addVariable("[" + Reserved.LOOP + "]", definition.intType.name, Reserved.LOOP, true, true);
+        }
+    }
+
+    public void incrementScope() {
+        scopes.push(0);
+    }
+
+    public void decrementScope() {
+        int remove = scopes.pop();
+
+        while (remove > 0) {
+            final Variable variable = variables.pop();
+
+            if (variable.read) {
+                throw new IllegalArgumentException("Error [" + variable.location + "]: Variable [" + variable.name + "] never used.");
+            }
+
+            --remove;
+        }
+    }
+
+    public Variable getVariable(final String location, final String name) {
+        final Iterator<Variable> itr = variables.iterator();
+
+        while (itr.hasNext()) {
+            final Variable variable = itr.next();
+
+            if (variable.name.equals(name)) {
+                return variable;
+            }
+        }
+
+        if (location != null) {
+            throw new IllegalArgumentException("Error " + location + ": Variable [" + name + "] not defined.");
+        }
+
+        return null;
+    }
+
+    public Variable addVariable(final String location, final String typestr, final String name,
+                                final boolean readonly, final boolean reserved) {
+        if (!reserved && this.reserved.isReserved(name)) {
+            throw new IllegalArgumentException("Error " + location + ": Variable name [" + name + "] is reserved.");
+        }
+
+        if (getVariable(null, name) != null) {
+            throw new IllegalArgumentException("Error " + location + ": Variable name [" + name + "] already defined.");
+        }
+
+        final Type type;
+
+        try {
+            type = definition.getType(typestr);
+        } catch (final IllegalArgumentException exception) {
+            throw new IllegalArgumentException("Error " + location + ": Not a type [" + typestr + "].");
+        }
+
+        boolean legal = !name.contains("<");
+
+        try {
+            definition.getType(name);
+            legal = false;
+        } catch (final IllegalArgumentException exception) {
+            // Do nothing.
+        }
+
+        if (!legal) {
+            throw new IllegalArgumentException("Error " + location + ": Variable name [" + name + "] cannot be a type.");
+        }
+
+        final Variable previous = variables.peekFirst();
+        int slot = 0;
+
+        if (previous != null) {
+            slot = previous.slot + previous.type.type.getSize();
+        }
+
+        final Variable variable = new Variable(location, name, type, slot, readonly);
+        variables.push(variable);
+
+        final int update = scopes.pop() + 1;
+        scopes.push(update);
+
+        return variable;
+    }
+}
+
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
index d17047933b8..ab85cf27398 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
@@ -19,58 +19,9 @@
 
 package org.elasticsearch.painless;
 
-import org.antlr.v4.runtime.tree.ParseTree;
-import org.elasticsearch.painless.PainlessParser.AfterthoughtContext;
-import org.elasticsearch.painless.PainlessParser.ArgumentsContext;
-import org.elasticsearch.painless.PainlessParser.AssignmentContext;
-import org.elasticsearch.painless.PainlessParser.BinaryContext;
-import org.elasticsearch.painless.PainlessParser.BoolContext;
-import org.elasticsearch.painless.PainlessParser.BreakContext;
-import org.elasticsearch.painless.PainlessParser.CastContext;
-import org.elasticsearch.painless.PainlessParser.CompContext;
-import org.elasticsearch.painless.PainlessParser.ConditionalContext;
-import org.elasticsearch.painless.PainlessParser.ContinueContext;
-import org.elasticsearch.painless.PainlessParser.DeclContext;
-import org.elasticsearch.painless.PainlessParser.DeclarationContext;
-import org.elasticsearch.painless.PainlessParser.DecltypeContext;
-import org.elasticsearch.painless.PainlessParser.DeclvarContext;
-import org.elasticsearch.painless.PainlessParser.DoContext;
-import org.elasticsearch.painless.PainlessParser.EmptyContext;
-import org.elasticsearch.painless.PainlessParser.EmptyscopeContext;
-import org.elasticsearch.painless.PainlessParser.ExprContext;
-import org.elasticsearch.painless.PainlessParser.ExtbraceContext;
-import org.elasticsearch.painless.PainlessParser.ExtcallContext;
-import org.elasticsearch.painless.PainlessParser.ExtcastContext;
-import org.elasticsearch.painless.PainlessParser.ExtdotContext;
-import org.elasticsearch.painless.PainlessParser.ExternalContext;
-import org.elasticsearch.painless.PainlessParser.ExtfieldContext;
-import org.elasticsearch.painless.PainlessParser.ExtnewContext;
-import org.elasticsearch.painless.PainlessParser.ExtprecContext;
-import org.elasticsearch.painless.PainlessParser.ExtstartContext;
-import org.elasticsearch.painless.PainlessParser.ExtstringContext;
-import org.elasticsearch.painless.PainlessParser.ExtvarContext;
-import org.elasticsearch.painless.PainlessParser.FalseContext;
-import org.elasticsearch.painless.PainlessParser.ForContext;
-import org.elasticsearch.painless.PainlessParser.GenericContext;
-import org.elasticsearch.painless.PainlessParser.IdentifierContext;
-import org.elasticsearch.painless.PainlessParser.IfContext;
-import org.elasticsearch.painless.PainlessParser.IncrementContext;
-import org.elasticsearch.painless.PainlessParser.InitializerContext;
-import org.elasticsearch.painless.PainlessParser.MultipleContext;
-import org.elasticsearch.painless.PainlessParser.NullContext;
-import org.elasticsearch.painless.PainlessParser.NumericContext;
-import org.elasticsearch.painless.PainlessParser.PostincContext;
-import org.elasticsearch.painless.PainlessParser.PrecedenceContext;
-import org.elasticsearch.painless.PainlessParser.PreincContext;
-import org.elasticsearch.painless.PainlessParser.ReturnContext;
-import org.elasticsearch.painless.PainlessParser.SingleContext;
-import org.elasticsearch.painless.PainlessParser.SourceContext;
-import org.elasticsearch.painless.PainlessParser.ThrowContext;
-import org.elasticsearch.painless.PainlessParser.TrapContext;
-import org.elasticsearch.painless.PainlessParser.TrueContext;
-import org.elasticsearch.painless.PainlessParser.TryContext;
-import org.elasticsearch.painless.PainlessParser.UnaryContext;
-import org.elasticsearch.painless.PainlessParser.WhileContext;
+import org.elasticsearch.painless.Variables.Reserved;
+import org.elasticsearch.painless.Variables.Variable;
+import org.elasticsearch.painless.node.SSource;
 import org.objectweb.asm.ClassWriter;
 import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.commons.GeneratorAdapter;
@@ -79,45 +30,44 @@ import static org.elasticsearch.painless.WriterConstants.BASE_CLASS_TYPE;
 import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE;
 import static org.elasticsearch.painless.WriterConstants.CONSTRUCTOR;
 import static org.elasticsearch.painless.WriterConstants.EXECUTE;
+import static org.elasticsearch.painless.WriterConstants.MAP_GET;
+import static org.elasticsearch.painless.WriterConstants.MAP_TYPE;
 
-class Writer extends PainlessParserBaseVisitor<Void> {
-    static byte[] write(Metadata metadata) {
-        final Writer writer = new Writer(metadata);
+/**
+ * Runs the writing phase of compilation using the Painless AST.
+ */
+final class Writer {
+
+    static byte[] write(final CompilerSettings settings, final Definition definition,
+                               final String source, final Variables variables, final SSource root) {
+        final Writer writer = new Writer(settings, definition, source, variables, root);
 
         return writer.getBytes();
     }
 
-    private final Metadata metadata;
-    private final ParseTree root;
-    private final String source;
     private final CompilerSettings settings;
+    private final Definition definition;
+    private final String source;
+    private final Variables variables;
+    private final SSource root;
 
     private final ClassWriter writer;
-    private final GeneratorAdapter execute;
+    private final GeneratorAdapter adapter;
 
-    private final WriterStatement statement;
-    private final WriterExpression expression;
-    private final WriterExternal external;
-
-    private Writer(final Metadata metadata) {
-        this.metadata = metadata;
-        root = metadata.root;
-        source = metadata.source;
-        settings = metadata.settings;
+    private Writer(final CompilerSettings settings, final Definition definition,
+                     final String source, final Variables variables, final SSource root) {
+        this.settings = settings;
+        this.definition = definition;
+        this.source = source;
+        this.variables = variables;
+        this.root = root;
 
         writer = new ClassWriter(ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS);
 
         writeBegin();
         writeConstructor();
 
-        execute = new GeneratorAdapter(Opcodes.ACC_PUBLIC, EXECUTE, null, null, writer);
-
-        final WriterUtility utility = new WriterUtility(metadata, execute);
-        final WriterCaster caster = new WriterCaster(execute);
-
-        statement = new WriterStatement(metadata, execute, this, utility);
-        expression = new WriterExpression(metadata, execute, this, utility, caster);
-        external = new WriterExternal(metadata, execute, this, utility, caster);
+        adapter = new GeneratorAdapter(Opcodes.ACC_PUBLIC, EXECUTE, null, null, writer);
 
         writeExecute();
         writeEnd();
@@ -130,12 +80,9 @@ class Writer extends PainlessParserBaseVisitor<Void> {
         final String name = CLASS_TYPE.getInternalName();
 
         // apply marker interface NeedsScore if we use the score!
-        final String interfaces[];
-        if (metadata.scoreValueUsed) {
-            interfaces = new String[] { WriterConstants.NEEDS_SCORE_TYPE.getInternalName() };
-        } else {
-            interfaces = null;
-        }
+        final String interfaces[] = variables.reserved.score ?
+            new String[] { WriterConstants.NEEDS_SCORE_TYPE.getInternalName() } : null;
+
         writer.visit(version, access, name, null, base, interfaces);
         writer.visitSource(source, null);
     }
@@ -150,29 +97,43 @@ class Writer extends PainlessParserBaseVisitor<Void> {
     }
 
     private void writeExecute() {
-        if (metadata.scoreValueUsed) {
+        if (variables.reserved.score) {
             // if the _score value is used, we do this once:
-            //   final double _score = scorer.score();
-            execute.visitVarInsn(Opcodes.ALOAD, metadata.scorerValueSlot);
-            execute.invokeVirtual(WriterConstants.SCORER_TYPE, WriterConstants.SCORER_SCORE);
-            execute.visitInsn(Opcodes.F2D);
-            execute.visitVarInsn(Opcodes.DSTORE, metadata.scoreValueSlot);
+            // final double _score = scorer.score();
+            final Variable scorer = variables.getVariable(null, Reserved.SCORER);
+            final Variable score = variables.getVariable(null, Reserved.SCORE);
+
+            adapter.visitVarInsn(Opcodes.ALOAD, scorer.slot);
+            adapter.invokeVirtual(WriterConstants.SCORER_TYPE, WriterConstants.SCORER_SCORE);
+            adapter.visitInsn(Opcodes.F2D);
+            adapter.visitVarInsn(Opcodes.DSTORE, score.slot);
         }
-        
-        if (metadata.ctxValueUsed) {
+
+        if (variables.reserved.ctx) {
             // if the _ctx value is used, we do this once:
-            //   final Map<String,Object> ctx = input.get("ctx");
-            execute.visitVarInsn(Opcodes.ALOAD, metadata.paramsValueSlot);
-            execute.push("ctx");
-            execute.invokeInterface(WriterConstants.MAP_TYPE, WriterConstants.MAP_GET);
-            execute.visitVarInsn(Opcodes.ASTORE, metadata.ctxValueSlot);
+            // final Map<String,Object> ctx = input.get("ctx");
+
+            final Variable input = variables.getVariable(null, Reserved.PARAMS);
+            final Variable ctx = variables.getVariable(null, Reserved.CTX);
+
+            adapter.visitVarInsn(Opcodes.ALOAD, input.slot);
+            adapter.push(Reserved.CTX);
+            adapter.invokeInterface(MAP_TYPE, MAP_GET);
+            adapter.visitVarInsn(Opcodes.ASTORE, ctx.slot);
         }
 
-        execute.push(settings.getMaxLoopCounter());
-        execute.visitVarInsn(Opcodes.ISTORE, metadata.loopCounterSlot);
+        if (variables.reserved.loop) {
+            // if there is infinite loop protection, we do this once:
+            // int #loop = settings.getMaxLoopCounter()
 
-        visit(root);
-        execute.endMethod();
+            final Variable loop = variables.getVariable(null, Reserved.LOOP);
+
+            adapter.push(settings.getMaxLoopCounter());
+            adapter.visitVarInsn(Opcodes.ISTORE, loop.slot);
+        }
+
+        root.write(settings, definition, adapter);
+        adapter.endMethod();
     }
 
     private void writeEnd() {
@@ -182,348 +143,4 @@ class Writer extends PainlessParserBaseVisitor<Void> {
     private byte[] getBytes() {
         return writer.toByteArray();
     }
-
-    @Override
-    public Void visitSource(final SourceContext ctx) {
-        statement.processSource(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitIf(final IfContext ctx) {
-        statement.processIf(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitWhile(final WhileContext ctx) {
-        statement.processWhile(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitDo(final DoContext ctx) {
-        statement.processDo(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitFor(final ForContext ctx) {
-        statement.processFor(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitDecl(final DeclContext ctx) {
-        statement.processDecl(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitContinue(final ContinueContext ctx) {
-        statement.processContinue();
-
-        return null;
-    }
-
-    @Override
-    public Void visitBreak(final BreakContext ctx) {
-        statement.processBreak();
-
-        return null;
-    }
-
-    @Override
-    public Void visitReturn(final ReturnContext ctx) {
-        statement.processReturn(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitTry(final TryContext ctx) {
-        statement.processTry(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitThrow(final ThrowContext ctx) {
-        statement.processThrow(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExpr(final ExprContext ctx) {
-        statement.processExpr(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitMultiple(final MultipleContext ctx) {
-        statement.processMultiple(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitSingle(final SingleContext ctx) {
-        statement.processSingle(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitEmpty(final EmptyContext ctx) {
-        throw new UnsupportedOperationException(WriterUtility.error(ctx) + "Unexpected state.");
-    }
-
-    @Override
-    public Void visitEmptyscope(final EmptyscopeContext ctx) {
-        throw new UnsupportedOperationException(WriterUtility.error(ctx) + "Unexpected state.");
-    }
-
-    @Override
-    public Void visitInitializer(final InitializerContext ctx) {
-        statement.processInitializer(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitAfterthought(final AfterthoughtContext ctx) {
-        statement.processAfterthought(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitDeclaration(DeclarationContext ctx) {
-        statement.processDeclaration(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitDecltype(final DecltypeContext ctx) {
-        throw new UnsupportedOperationException(WriterUtility.error(ctx) + "Unexpected state.");
-    }
-
-    @Override
-    public Void visitDeclvar(final DeclvarContext ctx) {
-        statement.processDeclvar(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitTrap(final TrapContext ctx) {
-        statement.processTrap(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitIdentifier(IdentifierContext ctx) {
-        throw new UnsupportedOperationException(WriterUtility.error(ctx) + "Unexpected state.");
-    }
-
-    @Override
-    public Void visitGeneric(GenericContext ctx) {
-        throw new UnsupportedOperationException(WriterUtility.error(ctx) + "Unexpected state.");
-    }
-
-    @Override
-    public Void visitPrecedence(final PrecedenceContext ctx) {
-        throw new UnsupportedOperationException(WriterUtility.error(ctx) + "Unexpected state.");
-    }
-
-    @Override
-    public Void visitNumeric(final NumericContext ctx) {
-        expression.processNumeric(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitTrue(final TrueContext ctx) {
-        expression.processTrue(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitFalse(final FalseContext ctx) {
-        expression.processFalse(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitNull(final NullContext ctx) {
-        expression.processNull(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExternal(final ExternalContext ctx) {
-        expression.processExternal(ctx);
-
-        return null;
-    }
-
-
-    @Override
-    public Void visitPostinc(final PostincContext ctx) {
-        expression.processPostinc(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitPreinc(final PreincContext ctx) {
-        expression.processPreinc(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitUnary(final UnaryContext ctx) {
-        expression.processUnary(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitCast(final CastContext ctx) {
-        expression.processCast(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitBinary(final BinaryContext ctx) {
-        expression.processBinary(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitComp(final CompContext ctx) {
-        expression.processComp(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitBool(final BoolContext ctx) {
-        expression.processBool(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitConditional(final ConditionalContext ctx) {
-        expression.processConditional(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitAssignment(final AssignmentContext ctx) {
-        expression.processAssignment(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtstart(final ExtstartContext ctx) {
-        external.processExtstart(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtprec(final ExtprecContext ctx) {
-        external.processExtprec(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtcast(final ExtcastContext ctx) {
-        external.processExtcast(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtbrace(final ExtbraceContext ctx) {
-        external.processExtbrace(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtdot(final ExtdotContext ctx) {
-        external.processExtdot(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtcall(final ExtcallContext ctx) {
-        external.processExtcall(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtvar(final ExtvarContext ctx) {
-        external.processExtvar(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtfield(final ExtfieldContext ctx) {
-        external.processExtfield(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtnew(final ExtnewContext ctx) {
-        external.processExtnew(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitExtstring(final ExtstringContext ctx) {
-        external.processExtstring(ctx);
-
-        return null;
-    }
-
-    @Override
-    public Void visitArguments(final ArgumentsContext ctx) {
-        throw new UnsupportedOperationException(WriterUtility.error(ctx) + "Unexpected state.");
-    }
-
-    @Override
-    public Void visitIncrement(final IncrementContext ctx) {
-        expression.processIncrement(ctx);
-
-        return null;
-    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterCaster.java
deleted file mode 100644
index c55dff2f549..00000000000
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterCaster.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless;
-
-import org.antlr.v4.runtime.ParserRuleContext;
-import org.elasticsearch.painless.Definition.Cast;
-import org.elasticsearch.painless.Definition.Transform;
-import org.elasticsearch.painless.Definition.Type;
-import org.elasticsearch.painless.Metadata.ExpressionMetadata;
-import org.objectweb.asm.commons.GeneratorAdapter;
-
-class WriterCaster {
-    private final GeneratorAdapter execute;
-
-    WriterCaster(final GeneratorAdapter execute) {
-        this.execute = execute;
-    }
-
-    void checkWriteCast(final ExpressionMetadata sort) {
-        checkWriteCast(sort.source, sort.cast);
-    }
-
-    void checkWriteCast(final ParserRuleContext source, final Cast cast) {
-        if (cast instanceof Transform) {
-            writeTransform((Transform)cast);
-        } else if (cast != null) {
-            writeCast(cast);
-        } else {
-            throw new IllegalStateException(WriterUtility.error(source) + "Unexpected cast object.");
-        }
-    }
-
-    private void writeCast(final Cast cast) {
-        final Type from = cast.from;
-        final Type to = cast.to;
-
-        if (from.equals(to)) {
-            return;
-        }
-
-        if (from.sort.numeric && from.sort.primitive && to.sort.numeric && to.sort.primitive) {
-            execute.cast(from.type, to.type);
-        } else {
-            try {
-                from.clazz.asSubclass(to.clazz);
-            } catch (ClassCastException exception) {
-                execute.checkCast(to.type);
-            }
-        }
-    }
-
-    private void writeTransform(final Transform transform) {
-        if (transform.upcast != null) {
-            execute.checkCast(transform.upcast.type);
-        }
-
-        if (java.lang.reflect.Modifier.isStatic(transform.method.reflect.getModifiers())) {
-            execute.invokeStatic(transform.method.owner.type, transform.method.method);
-        } else if (java.lang.reflect.Modifier.isInterface(transform.method.owner.clazz.getModifiers())) {
-            execute.invokeInterface(transform.method.owner.type, transform.method.method);
-        } else {
-            execute.invokeVirtual(transform.method.owner.type, transform.method.method);
-        }
-
-        if (transform.downcast != null) {
-            execute.checkCast(transform.downcast.type);
-        }
-    }
-}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index e4effe85bc5..b13787cece5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -31,116 +31,130 @@ import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodType;
 import java.util.Map;
 
-class WriterConstants {
-    final static String BASE_CLASS_NAME = Executable.class.getName();
-    final static String CLASS_NAME      = BASE_CLASS_NAME + "$CompiledPainlessExecutable";
-    final static Type BASE_CLASS_TYPE   = Type.getType(Executable.class);
-    final static Type CLASS_TYPE        = Type.getType("L" + CLASS_NAME.replace(".", "/") + ";");
+/**
+ * General pool of constants used during the writing phase of compilation.
+ */
+public final class WriterConstants {
 
-    final static Method CONSTRUCTOR = getAsmMethod(void.class, "<init>", Definition.class, String.class, String.class);
-    final static Method EXECUTE     = getAsmMethod(Object.class, "execute", Map.class, Scorer.class, LeafDocLookup.class, Object.class);
+    public final static String BASE_CLASS_NAME = Executable.class.getName();
+    public final static String CLASS_NAME      = BASE_CLASS_NAME + "$CompiledPainlessExecutable";
+    public final static Type BASE_CLASS_TYPE   = Type.getType(Executable.class);
+    public final static Type CLASS_TYPE        = Type.getType("L" + CLASS_NAME.replace(".", "/") + ";");
 
-    final static Type PAINLESS_ERROR_TYPE = Type.getType(PainlessError.class);
+    public final static Method CONSTRUCTOR = getAsmMethod(void.class, "<init>", String.class, String.class);
+    public final static Method EXECUTE     =
+        getAsmMethod(Object.class, "execute", Map.class, Scorer.class, LeafDocLookup.class, Object.class);
 
-    final static Type DEFINITION_TYPE = Type.getType(Definition.class);
-    
-    final static Type NEEDS_SCORE_TYPE = Type.getType(NeedsScore.class);
+    public final static Type PAINLESS_ERROR_TYPE = Type.getType(PainlessError.class);
 
-    final static Type OBJECT_TYPE = Type.getType(Object.class);
+    public final static Type NEEDS_SCORE_TYPE = Type.getType(NeedsScore.class);
+    public final static Type SCORER_TYPE = Type.getType(Scorer.class);
+    public final static Method SCORER_SCORE = getAsmMethod(float.class, "score");
 
-    final static Type SCORER_TYPE = Type.getType(Scorer.class);
-    final static Method SCORER_SCORE = getAsmMethod(float.class, "score");
-
-    final static Type MAP_TYPE  = Type.getType(Map.class);
-    final static Method MAP_GET = getAsmMethod(Object.class, "get", Object.class);
+    public final static Type MAP_TYPE  = Type.getType(Map.class);
+    public final static Method MAP_GET = getAsmMethod(Object.class, "get", Object.class);
 
     /** dynamic callsite bootstrap signature */
-    final static MethodType DEF_BOOTSTRAP_TYPE = MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, 
-                                                                       String.class, MethodType.class, int.class);
-    final static Handle DEF_BOOTSTRAP_HANDLE = new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(DynamicCallSite.class),
-                                                          "bootstrap", WriterConstants.DEF_BOOTSTRAP_TYPE.toMethodDescriptorString());
+    public final static MethodType DEF_BOOTSTRAP_TYPE =
+        MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, int.class);
+    public final static Handle DEF_BOOTSTRAP_HANDLE =
+        new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(DynamicCallSite.class),
+            "bootstrap", DEF_BOOTSTRAP_TYPE.toMethodDescriptorString());
 
-    final static String DEF_DYNAMIC_LOAD_FIELD_DESC = MethodType.methodType(Object.class, Object.class)
-        .toMethodDescriptorString();
-    final static String DEF_DYNAMIC_STORE_FIELD_DESC = MethodType.methodType(void.class, Object.class, Object.class)
-        .toMethodDescriptorString();
-    final static String DEF_DYNAMIC_ARRAY_LOAD_DESC = MethodType.methodType(Object.class, Object.class, Object.class)
-        .toMethodDescriptorString();
-    final static String DEF_DYNAMIC_ARRAY_STORE_DESC = MethodType.methodType(void.class, Object.class, Object.class, Object.class)
-        .toMethodDescriptorString();
+    public final static String DEF_DYNAMIC_LOAD_FIELD_DESC =
+        MethodType.methodType(Object.class, Object.class).toMethodDescriptorString();
+    public final static String DEF_DYNAMIC_STORE_FIELD_DESC =
+        MethodType.methodType(void.class, Object.class, Object.class).toMethodDescriptorString();
+    public final static String DEF_DYNAMIC_ARRAY_LOAD_DESC =
+        MethodType.methodType(Object.class, Object.class, Object.class).toMethodDescriptorString();
+    public final static String DEF_DYNAMIC_ARRAY_STORE_DESC =
+        MethodType.methodType(void.class, Object.class, Object.class, Object.class).toMethodDescriptorString();
 
-    final static Method DEF_NOT_CALL = getAsmMethod(Object.class, "not", Object.class);
-    final static Method DEF_NEG_CALL = getAsmMethod(Object.class, "neg", Object.class);
-    final static Method DEF_MUL_CALL = getAsmMethod(Object.class, "mul", Object.class, Object.class);
-    final static Method DEF_DIV_CALL = getAsmMethod(Object.class, "div", Object.class, Object.class);
-    final static Method DEF_REM_CALL = getAsmMethod(Object.class, "rem", Object.class, Object.class);
-    final static Method DEF_ADD_CALL = getAsmMethod(Object.class, "add", Object.class, Object.class);
-    final static Method DEF_SUB_CALL = getAsmMethod(Object.class, "sub", Object.class, Object.class);
-    final static Method DEF_LSH_CALL = getAsmMethod(Object.class, "lsh", Object.class, Object.class);
-    final static Method DEF_RSH_CALL = getAsmMethod(Object.class, "rsh", Object.class, Object.class);
-    final static Method DEF_USH_CALL = getAsmMethod(Object.class, "ush", Object.class, Object.class);
-    final static Method DEF_AND_CALL = getAsmMethod(Object.class, "and", Object.class, Object.class);
-    final static Method DEF_XOR_CALL = getAsmMethod(Object.class, "xor", Object.class, Object.class);
-    final static Method DEF_OR_CALL  = getAsmMethod(Object.class, "or" , Object.class, Object.class);
-    final static Method DEF_EQ_CALL  = getAsmMethod(boolean.class, "eq" , Object.class, Object.class);
-    final static Method DEF_LT_CALL  = getAsmMethod(boolean.class, "lt" , Object.class, Object.class);
-    final static Method DEF_LTE_CALL = getAsmMethod(boolean.class, "lte", Object.class, Object.class);
-    final static Method DEF_GT_CALL  = getAsmMethod(boolean.class, "gt" , Object.class, Object.class);
-    final static Method DEF_GTE_CALL = getAsmMethod(boolean.class, "gte", Object.class, Object.class);
+    public final static Method DEF_NOT_CALL = getAsmMethod(Object.class, "not", Object.class);
+    public final static Method DEF_NEG_CALL = getAsmMethod(Object.class, "neg", Object.class);
+    public final static Method DEF_MUL_CALL = getAsmMethod(Object.class, "mul", Object.class, Object.class);
+    public final static Method DEF_DIV_CALL = getAsmMethod(Object.class, "div", Object.class, Object.class);
+    public final static Method DEF_REM_CALL = getAsmMethod(Object.class, "rem", Object.class, Object.class);
+    public final static Method DEF_ADD_CALL = getAsmMethod(Object.class, "add", Object.class, Object.class);
+    public final static Method DEF_SUB_CALL = getAsmMethod(Object.class, "sub", Object.class, Object.class);
+    public final static Method DEF_LSH_CALL = getAsmMethod(Object.class, "lsh", Object.class, int.class);
+    public final static Method DEF_RSH_CALL = getAsmMethod(Object.class, "rsh", Object.class, int.class);
+    public final static Method DEF_USH_CALL = getAsmMethod(Object.class, "ush", Object.class, int.class);
+    public final static Method DEF_AND_CALL = getAsmMethod(Object.class, "and", Object.class, Object.class);
+    public final static Method DEF_XOR_CALL = getAsmMethod(Object.class, "xor", Object.class, Object.class);
+    public final static Method DEF_OR_CALL  = getAsmMethod(Object.class, "or" , Object.class, Object.class);
+    public final static Method DEF_EQ_CALL  = getAsmMethod(boolean.class, "eq" , Object.class, Object.class);
+    public final static Method DEF_LT_CALL  = getAsmMethod(boolean.class, "lt" , Object.class, Object.class);
+    public final static Method DEF_LTE_CALL = getAsmMethod(boolean.class, "lte", Object.class, Object.class);
+    public final static Method DEF_GT_CALL  = getAsmMethod(boolean.class, "gt" , Object.class, Object.class);
+    public final static Method DEF_GTE_CALL = getAsmMethod(boolean.class, "gte", Object.class, Object.class);
 
-    final static Type STRINGBUILDER_TYPE = Type.getType(StringBuilder.class);
+    public final static Type STRINGBUILDER_TYPE = Type.getType(StringBuilder.class);
 
-    final static Method STRINGBUILDER_CONSTRUCTOR    = getAsmMethod(void.class, "<init>");
-    final static Method STRINGBUILDER_APPEND_BOOLEAN = getAsmMethod(StringBuilder.class, "append", boolean.class);
-    final static Method STRINGBUILDER_APPEND_CHAR    = getAsmMethod(StringBuilder.class, "append", char.class);
-    final static Method STRINGBUILDER_APPEND_INT     = getAsmMethod(StringBuilder.class, "append", int.class);
-    final static Method STRINGBUILDER_APPEND_LONG    = getAsmMethod(StringBuilder.class, "append", long.class);
-    final static Method STRINGBUILDER_APPEND_FLOAT   = getAsmMethod(StringBuilder.class, "append", float.class);
-    final static Method STRINGBUILDER_APPEND_DOUBLE  = getAsmMethod(StringBuilder.class, "append", double.class);
-    final static Method STRINGBUILDER_APPEND_STRING  = getAsmMethod(StringBuilder.class, "append", String.class);
-    final static Method STRINGBUILDER_APPEND_OBJECT  = getAsmMethod(StringBuilder.class, "append", Object.class);
-    final static Method STRINGBUILDER_TOSTRING       = getAsmMethod(String.class, "toString");
+    public final static Method STRINGBUILDER_CONSTRUCTOR    = getAsmMethod(void.class, "<init>");
+    public final static Method STRINGBUILDER_APPEND_BOOLEAN = getAsmMethod(StringBuilder.class, "append", boolean.class);
+    public final static Method STRINGBUILDER_APPEND_CHAR    = getAsmMethod(StringBuilder.class, "append", char.class);
+    public final static Method STRINGBUILDER_APPEND_INT     = getAsmMethod(StringBuilder.class, "append", int.class);
+    public final static Method STRINGBUILDER_APPEND_LONG    = getAsmMethod(StringBuilder.class, "append", long.class);
+    public final static Method STRINGBUILDER_APPEND_FLOAT   = getAsmMethod(StringBuilder.class, "append", float.class);
+    public final static Method STRINGBUILDER_APPEND_DOUBLE  = getAsmMethod(StringBuilder.class, "append", double.class);
+    public final static Method STRINGBUILDER_APPEND_STRING  = getAsmMethod(StringBuilder.class, "append", String.class);
+    public final static Method STRINGBUILDER_APPEND_OBJECT  = getAsmMethod(StringBuilder.class, "append", Object.class);
+    public final static Method STRINGBUILDER_TOSTRING       = getAsmMethod(String.class, "toString");
 
-    final static Method TOINTEXACT_LONG  = getAsmMethod(int.class,  "toIntExact",    long.class);
-    final static Method NEGATEEXACT_INT  = getAsmMethod(int.class,  "negateExact",   int.class);
-    final static Method NEGATEEXACT_LONG = getAsmMethod(long.class, "negateExact",   long.class);
-    final static Method MULEXACT_INT     = getAsmMethod(int.class,  "multiplyExact", int.class,  int.class);
-    final static Method MULEXACT_LONG    = getAsmMethod(long.class, "multiplyExact", long.class, long.class);
-    final static Method ADDEXACT_INT     = getAsmMethod(int.class,  "addExact",      int.class,  int.class);
-    final static Method ADDEXACT_LONG    = getAsmMethod(long.class, "addExact",      long.class, long.class);
-    final static Method SUBEXACT_INT     = getAsmMethod(int.class,  "subtractExact", int.class,  int.class);
-    final static Method SUBEXACT_LONG    = getAsmMethod(long.class, "subtractExact", long.class, long.class);
+    public final static Method TOINTEXACT_LONG  = getAsmMethod(int.class,  "toIntExact",    long.class);
+    public final static Method NEGATEEXACT_INT  = getAsmMethod(int.class,  "negateExact",   int.class);
+    public final static Method NEGATEEXACT_LONG = getAsmMethod(long.class, "negateExact",   long.class);
+    public final static Method MULEXACT_INT     = getAsmMethod(int.class,  "multiplyExact", int.class,  int.class);
+    public final static Method MULEXACT_LONG    = getAsmMethod(long.class, "multiplyExact", long.class, long.class);
+    public final static Method ADDEXACT_INT     = getAsmMethod(int.class,  "addExact",      int.class,  int.class);
+    public final static Method ADDEXACT_LONG    = getAsmMethod(long.class, "addExact",      long.class, long.class);
+    public final static Method SUBEXACT_INT     = getAsmMethod(int.class,  "subtractExact", int.class,  int.class);
+    public final static Method SUBEXACT_LONG    = getAsmMethod(long.class, "subtractExact", long.class, long.class);
 
-    final static Method CHECKEQUALS              = getAsmMethod(boolean.class, "checkEquals",              Object.class, Object.class);
-    final static Method TOBYTEEXACT_INT          = getAsmMethod(byte.class,    "toByteExact",              int.class);
-    final static Method TOBYTEEXACT_LONG         = getAsmMethod(byte.class,    "toByteExact",              long.class);
-    final static Method TOBYTEWOOVERFLOW_FLOAT   = getAsmMethod(byte.class,    "toByteWithoutOverflow",    float.class);
-    final static Method TOBYTEWOOVERFLOW_DOUBLE  = getAsmMethod(byte.class,    "toByteWithoutOverflow",    double.class);
-    final static Method TOSHORTEXACT_INT         = getAsmMethod(short.class,   "toShortExact",             int.class);
-    final static Method TOSHORTEXACT_LONG        = getAsmMethod(short.class,   "toShortExact",             long.class);
-    final static Method TOSHORTWOOVERFLOW_FLOAT  = getAsmMethod(short.class,   "toShortWithoutOverflow",   float.class);
-    final static Method TOSHORTWOOVERFLOW_DOUBLE = getAsmMethod(short.class,   "toShortWihtoutOverflow",   double.class);
-    final static Method TOCHAREXACT_INT          = getAsmMethod(char.class,    "toCharExact",              int.class);
-    final static Method TOCHAREXACT_LONG         = getAsmMethod(char.class,    "toCharExact",              long.class);
-    final static Method TOCHARWOOVERFLOW_FLOAT   = getAsmMethod(char.class,    "toCharWithoutOverflow",    float.class);
-    final static Method TOCHARWOOVERFLOW_DOUBLE  = getAsmMethod(char.class,    "toCharWithoutOverflow",    double.class);
-    final static Method TOINTWOOVERFLOW_FLOAT    = getAsmMethod(int.class,     "toIntWithoutOverflow",     float.class);
-    final static Method TOINTWOOVERFLOW_DOUBLE   = getAsmMethod(int.class,     "toIntWithoutOverflow",     double.class);
-    final static Method TOLONGWOOVERFLOW_FLOAT   = getAsmMethod(long.class,    "toLongWithoutOverflow",    float.class);
-    final static Method TOLONGWOOVERFLOW_DOUBLE  = getAsmMethod(long.class,    "toLongWithoutOverflow",    double.class);
-    final static Method TOFLOATWOOVERFLOW_DOUBLE = getAsmMethod(float.class ,  "toFloatWihtoutOverflow",   double.class);
-    final static Method MULWOOVERLOW_FLOAT       = getAsmMethod(float.class,   "multiplyWithoutOverflow",  float.class,  float.class);
-    final static Method MULWOOVERLOW_DOUBLE      = getAsmMethod(double.class,  "multiplyWithoutOverflow",  double.class, double.class);
-    final static Method DIVWOOVERLOW_INT         = getAsmMethod(int.class,     "divideWithoutOverflow",    int.class,    int.class);
-    final static Method DIVWOOVERLOW_LONG        = getAsmMethod(long.class,    "divideWithoutOverflow",    long.class,   long.class);
-    final static Method DIVWOOVERLOW_FLOAT       = getAsmMethod(float.class,   "divideWithoutOverflow",    float.class,  float.class);
-    final static Method DIVWOOVERLOW_DOUBLE      = getAsmMethod(double.class,  "divideWithoutOverflow",    double.class, double.class);
-    final static Method REMWOOVERLOW_FLOAT       = getAsmMethod(float.class,   "remainderWithoutOverflow", float.class,  float.class);
-    final static Method REMWOOVERLOW_DOUBLE      = getAsmMethod(double.class,  "remainderWithoutOverflow", double.class, double.class);
-    final static Method ADDWOOVERLOW_FLOAT       = getAsmMethod(float.class,   "addWithoutOverflow",       float.class,  float.class);
-    final static Method ADDWOOVERLOW_DOUBLE      = getAsmMethod(double.class,  "addWithoutOverflow",       double.class, double.class);
-    final static Method SUBWOOVERLOW_FLOAT       = getAsmMethod(float.class,   "subtractWithoutOverflow",  float.class,  float.class);
-    final static Method SUBWOOVERLOW_DOUBLE      = getAsmMethod(double.class,  "subtractWithoutOverflow",  double.class, double.class);
+    public final static Method CHECKEQUALS              =
+        getAsmMethod(boolean.class, "checkEquals",              Object.class, Object.class);
+    public final static Method TOBYTEEXACT_INT          = getAsmMethod(byte.class,    "toByteExact",              int.class);
+    public final static Method TOBYTEEXACT_LONG         = getAsmMethod(byte.class,    "toByteExact",              long.class);
+    public final static Method TOBYTEWOOVERFLOW_FLOAT   = getAsmMethod(byte.class,    "toByteWithoutOverflow",    float.class);
+    public final static Method TOBYTEWOOVERFLOW_DOUBLE  = getAsmMethod(byte.class,    "toByteWithoutOverflow",    double.class);
+    public final static Method TOSHORTEXACT_INT         = getAsmMethod(short.class,   "toShortExact",             int.class);
+    public final static Method TOSHORTEXACT_LONG        = getAsmMethod(short.class,   "toShortExact",             long.class);
+    public final static Method TOSHORTWOOVERFLOW_FLOAT  = getAsmMethod(short.class,   "toShortWithoutOverflow",   float.class);
+    public final static Method TOSHORTWOOVERFLOW_DOUBLE = getAsmMethod(short.class,   "toShortWihtoutOverflow",   double.class);
+    public final static Method TOCHAREXACT_INT          = getAsmMethod(char.class,    "toCharExact",              int.class);
+    public final static Method TOCHAREXACT_LONG         = getAsmMethod(char.class,    "toCharExact",              long.class);
+    public final static Method TOCHARWOOVERFLOW_FLOAT   = getAsmMethod(char.class,    "toCharWithoutOverflow",    float.class);
+    public final static Method TOCHARWOOVERFLOW_DOUBLE  = getAsmMethod(char.class,    "toCharWithoutOverflow",    double.class);
+    public final static Method TOINTWOOVERFLOW_FLOAT    = getAsmMethod(int.class,     "toIntWithoutOverflow",     float.class);
+    public final static Method TOINTWOOVERFLOW_DOUBLE   = getAsmMethod(int.class,     "toIntWithoutOverflow",     double.class);
+    public final static Method TOLONGWOOVERFLOW_FLOAT   = getAsmMethod(long.class,    "toLongWithoutOverflow",    float.class);
+    public final static Method TOLONGWOOVERFLOW_DOUBLE  = getAsmMethod(long.class,    "toLongWithoutOverflow",    double.class);
+    public final static Method TOFLOATWOOVERFLOW_DOUBLE = getAsmMethod(float.class ,  "toFloatWihtoutOverflow",   double.class);
+    public final static Method MULWOOVERLOW_FLOAT       =
+        getAsmMethod(float.class,   "multiplyWithoutOverflow",  float.class,  float.class);
+    public final static Method MULWOOVERLOW_DOUBLE      =
+        getAsmMethod(double.class,  "multiplyWithoutOverflow",  double.class, double.class);
+    public final static Method DIVWOOVERLOW_INT         =
+        getAsmMethod(int.class,     "divideWithoutOverflow",    int.class,    int.class);
+    public final static Method DIVWOOVERLOW_LONG        =
+        getAsmMethod(long.class,    "divideWithoutOverflow",    long.class,   long.class);
+    public final static Method DIVWOOVERLOW_FLOAT       =
+        getAsmMethod(float.class,   "divideWithoutOverflow",    float.class,  float.class);
+    public final static Method DIVWOOVERLOW_DOUBLE      =
+        getAsmMethod(double.class,  "divideWithoutOverflow",    double.class, double.class);
+    public final static Method REMWOOVERLOW_FLOAT       =
+        getAsmMethod(float.class,   "remainderWithoutOverflow", float.class,  float.class);
+    public final static Method REMWOOVERLOW_DOUBLE      =
+        getAsmMethod(double.class,  "remainderWithoutOverflow", double.class, double.class);
+    public final static Method ADDWOOVERLOW_FLOAT       =
+        getAsmMethod(float.class,   "addWithoutOverflow",       float.class,  float.class);
+    public final static Method ADDWOOVERLOW_DOUBLE      =
+        getAsmMethod(double.class,  "addWithoutOverflow",       double.class, double.class);
+    public final static Method SUBWOOVERLOW_FLOAT       =
+        getAsmMethod(float.class,   "subtractWithoutOverflow",  float.class,  float.class);
+    public final static Method SUBWOOVERLOW_DOUBLE      =
+        getAsmMethod(double.class,  "subtractWithoutOverflow",  double.class, double.class);
 
     private static Method getAsmMethod(final Class<?> rtype, final String name, final Class<?>... ptypes) {
         return new Method(name, MethodType.methodType(rtype, ptypes).toMethodDescriptorString());
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExpression.java
deleted file mode 100644
index 7d10a275690..00000000000
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExpression.java
+++ /dev/null
@@ -1,669 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless;
-
-import org.elasticsearch.painless.Definition.Sort;
-import org.elasticsearch.painless.Definition.Type;
-import org.elasticsearch.painless.Metadata.ExpressionMetadata;
-import org.elasticsearch.painless.PainlessParser.AssignmentContext;
-import org.elasticsearch.painless.PainlessParser.BinaryContext;
-import org.elasticsearch.painless.PainlessParser.BoolContext;
-import org.elasticsearch.painless.PainlessParser.CastContext;
-import org.elasticsearch.painless.PainlessParser.CompContext;
-import org.elasticsearch.painless.PainlessParser.ConditionalContext;
-import org.elasticsearch.painless.PainlessParser.ExpressionContext;
-import org.elasticsearch.painless.PainlessParser.ExternalContext;
-import org.elasticsearch.painless.PainlessParser.FalseContext;
-import org.elasticsearch.painless.PainlessParser.IncrementContext;
-import org.elasticsearch.painless.PainlessParser.NullContext;
-import org.elasticsearch.painless.PainlessParser.NumericContext;
-import org.elasticsearch.painless.PainlessParser.PostincContext;
-import org.elasticsearch.painless.PainlessParser.PreincContext;
-import org.elasticsearch.painless.PainlessParser.TrueContext;
-import org.elasticsearch.painless.PainlessParser.UnaryContext;
-import org.elasticsearch.painless.WriterUtility.Branch;
-import org.objectweb.asm.Label;
-import org.objectweb.asm.Opcodes;
-import org.objectweb.asm.commons.GeneratorAdapter;
-
-import static org.elasticsearch.painless.PainlessParser.ADD;
-import static org.elasticsearch.painless.PainlessParser.BWAND;
-import static org.elasticsearch.painless.PainlessParser.BWOR;
-import static org.elasticsearch.painless.PainlessParser.BWXOR;
-import static org.elasticsearch.painless.PainlessParser.DIV;
-import static org.elasticsearch.painless.PainlessParser.LSH;
-import static org.elasticsearch.painless.PainlessParser.MUL;
-import static org.elasticsearch.painless.PainlessParser.REM;
-import static org.elasticsearch.painless.PainlessParser.RSH;
-import static org.elasticsearch.painless.PainlessParser.SUB;
-import static org.elasticsearch.painless.PainlessParser.USH;
-import static org.elasticsearch.painless.WriterConstants.CHECKEQUALS;
-import static org.elasticsearch.painless.WriterConstants.DEF_EQ_CALL;
-import static org.elasticsearch.painless.WriterConstants.DEF_GTE_CALL;
-import static org.elasticsearch.painless.WriterConstants.DEF_GT_CALL;
-import static org.elasticsearch.painless.WriterConstants.DEF_LTE_CALL;
-import static org.elasticsearch.painless.WriterConstants.DEF_LT_CALL;
-import static org.elasticsearch.painless.WriterConstants.DEF_NEG_CALL;
-import static org.elasticsearch.painless.WriterConstants.DEF_NOT_CALL;
-import static org.elasticsearch.painless.WriterConstants.NEGATEEXACT_INT;
-import static org.elasticsearch.painless.WriterConstants.NEGATEEXACT_LONG;
-
-class WriterExpression {
-    private final Metadata metadata;
-    private final Definition definition;
-    private final CompilerSettings settings;
-
-    private final GeneratorAdapter execute;
-
-    private final Writer writer;
-    private final WriterUtility utility;
-    private final WriterCaster caster;
-
-    WriterExpression(final Metadata metadata, final GeneratorAdapter execute, final Writer writer,
-                     final WriterUtility utility, final WriterCaster caster) {
-        this.metadata = metadata;
-        definition = metadata.definition;
-        settings = metadata.settings;
-
-        this.execute = execute;
-
-        this.writer = writer;
-        this.utility = utility;
-        this.caster = caster;
-    }
-
-    void processNumeric(final NumericContext ctx) {
-        final ExpressionMetadata numericemd = metadata.getExpressionMetadata(ctx);
-        final Object postConst = numericemd.postConst;
-
-        if (postConst == null) {
-            utility.writeNumeric(ctx, numericemd.preConst);
-            caster.checkWriteCast(numericemd);
-        } else {
-            utility.writeConstant(ctx, postConst);
-        }
-
-        utility.checkWriteBranch(ctx);
-    }
-
-    void processTrue(final TrueContext ctx) {
-        final ExpressionMetadata trueemd = metadata.getExpressionMetadata(ctx);
-        final Object postConst = trueemd.postConst;
-        final Branch branch = utility.getBranch(ctx);
-
-        if (branch == null) {
-            if (postConst == null) {
-                utility.writeBoolean(ctx, true);
-                caster.checkWriteCast(trueemd);
-            } else {
-                utility.writeConstant(ctx, postConst);
-            }
-        } else if (branch.tru != null) {
-            execute.goTo(branch.tru);
-        }
-    }
-
-    void processFalse(final FalseContext ctx) {
-        final ExpressionMetadata falseemd = metadata.getExpressionMetadata(ctx);
-        final Object postConst = falseemd.postConst;
-        final Branch branch = utility.getBranch(ctx);
-
-        if (branch == null) {
-            if (postConst == null) {
-                utility.writeBoolean(ctx, false);
-                caster.checkWriteCast(falseemd);
-            } else {
-                utility.writeConstant(ctx, postConst);
-            }
-        } else if (branch.fals != null) {
-            execute.goTo(branch.fals);
-        }
-    }
-
-    void processNull(final NullContext ctx) {
-        final ExpressionMetadata nullemd = metadata.getExpressionMetadata(ctx);
-
-        execute.visitInsn(Opcodes.ACONST_NULL);
-        caster.checkWriteCast(nullemd);
-        utility.checkWriteBranch(ctx);
-    }
-
-    void processExternal(final ExternalContext ctx) {
-        final ExpressionMetadata expremd = metadata.getExpressionMetadata(ctx);
-        writer.visit(ctx.extstart());
-        caster.checkWriteCast(expremd);
-        utility.checkWriteBranch(ctx);
-    }
-
-
-    void processPostinc(final PostincContext ctx) {
-        final ExpressionMetadata expremd = metadata.getExpressionMetadata(ctx);
-        writer.visit(ctx.extstart());
-        caster.checkWriteCast(expremd);
-        utility.checkWriteBranch(ctx);
-    }
-
-    void processPreinc(final PreincContext ctx) {
-        final ExpressionMetadata expremd = metadata.getExpressionMetadata(ctx);
-        writer.visit(ctx.extstart());
-        caster.checkWriteCast(expremd);
-        utility.checkWriteBranch(ctx);
-    }
-
-    void processUnary(final UnaryContext ctx) {
-        final ExpressionMetadata unaryemd = metadata.getExpressionMetadata(ctx);
-        final Object postConst = unaryemd.postConst;
-        final Object preConst = unaryemd.preConst;
-        final Branch branch = utility.getBranch(ctx);
-
-        if (postConst != null) {
-            if (ctx.BOOLNOT() != null) {
-                if (branch == null) {
-                    utility.writeConstant(ctx, postConst);
-                } else {
-                    if ((boolean)postConst && branch.tru != null) {
-                        execute.goTo(branch.tru);
-                    } else if (!(boolean)postConst && branch.fals != null) {
-                        execute.goTo(branch.fals);
-                    }
-                }
-            } else {
-                utility.writeConstant(ctx, postConst);
-                utility.checkWriteBranch(ctx);
-            }
-        } else if (preConst != null) {
-            if (branch == null) {
-                utility.writeConstant(ctx, preConst);
-                caster.checkWriteCast(unaryemd);
-            } else {
-                throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
-            }
-        } else {
-            final ExpressionContext exprctx = ctx.expression();
-
-            if (ctx.BOOLNOT() != null) {
-                final Branch local = utility.markBranch(ctx, exprctx);
-
-                if (branch == null) {
-                    local.fals = new Label();
-                    final Label aend = new Label();
-
-                    writer.visit(exprctx);
-
-                    execute.push(false);
-                    execute.goTo(aend);
-                    execute.mark(local.fals);
-                    execute.push(true);
-                    execute.mark(aend);
-
-                    caster.checkWriteCast(unaryemd);
-                } else {
-                    local.tru = branch.fals;
-                    local.fals = branch.tru;
-
-                    writer.visit(exprctx);
-                }
-            } else {
-                final org.objectweb.asm.Type type = unaryemd.from.type;
-                final Sort sort = unaryemd.from.sort;
-
-                writer.visit(exprctx);
-
-                if (ctx.BWNOT() != null) {
-                    if (sort == Sort.DEF) {
-                        execute.invokeStatic(definition.defobjType.type, DEF_NOT_CALL);
-                    } else {
-                        if (sort == Sort.INT) {
-                            utility.writeConstant(ctx, -1);
-                        } else if (sort == Sort.LONG) {
-                            utility.writeConstant(ctx, -1L);
-                        } else {
-                            throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
-                        }
-
-                        execute.math(GeneratorAdapter.XOR, type);
-                    }
-                } else if (ctx.SUB() != null) {
-                    if (sort == Sort.DEF) {
-                        execute.invokeStatic(definition.defobjType.type, DEF_NEG_CALL);
-                    } else {
-                        if (settings.getNumericOverflow()) {
-                            execute.math(GeneratorAdapter.NEG, type);
-                        } else {
-                            if (sort == Sort.INT) {
-                                execute.invokeStatic(definition.mathType.type, NEGATEEXACT_INT);
-                            } else if (sort == Sort.LONG) {
-                                execute.invokeStatic(definition.mathType.type, NEGATEEXACT_LONG);
-                            } else {
-                                throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
-                            }
-                        }
-                    }
-                } else if (ctx.ADD() == null) {
-                    throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
-                }
-
-                caster.checkWriteCast(unaryemd);
-                utility.checkWriteBranch(ctx);
-            }
-        }
-    }
-
-    void processCast(final CastContext ctx) {
-        final ExpressionMetadata castemd = metadata.getExpressionMetadata(ctx);
-        final Object postConst = castemd.postConst;
-
-        if (postConst == null) {
-            writer.visit(ctx.expression());
-            caster.checkWriteCast(castemd);
-        } else {
-            utility.writeConstant(ctx, postConst);
-        }
-
-        utility.checkWriteBranch(ctx);
-    }
-
-    void processBinary(final BinaryContext ctx) {
-        final ExpressionMetadata binaryemd = metadata.getExpressionMetadata(ctx);
-        final Object postConst = binaryemd.postConst;
-        final Object preConst = binaryemd.preConst;
-        final Branch branch = utility.getBranch(ctx);
-
-        if (postConst != null) {
-            utility.writeConstant(ctx, postConst);
-        } else if (preConst != null) {
-            if (branch == null) {
-                utility.writeConstant(ctx, preConst);
-                caster.checkWriteCast(binaryemd);
-            } else {
-                throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
-            }
-        } else if (binaryemd.from.sort == Sort.STRING) {
-            final boolean marked = utility.containsStrings(ctx);
-
-            if (!marked) {
-                utility.writeNewStrings();
-            }
-
-            final ExpressionContext exprctx0 = ctx.expression(0);
-            final ExpressionMetadata expremd0 = metadata.getExpressionMetadata(exprctx0);
-            utility.addStrings(exprctx0);
-            writer.visit(exprctx0);
-
-            if (utility.containsStrings(exprctx0)) {
-                utility.writeAppendStrings(expremd0.from.sort);
-                utility.removeStrings(exprctx0);
-            }
-
-            final ExpressionContext exprctx1 = ctx.expression(1);
-            final ExpressionMetadata expremd1 = metadata.getExpressionMetadata(exprctx1);
-            utility.addStrings(exprctx1);
-            writer.visit(exprctx1);
-
-            if (utility.containsStrings(exprctx1)) {
-                utility.writeAppendStrings(expremd1.from.sort);
-                utility.removeStrings(exprctx1);
-            }
-
-            if (marked) {
-                utility.removeStrings(ctx);
-            } else {
-                utility.writeToStrings();
-            }
-
-            caster.checkWriteCast(binaryemd);
-        } else {
-            final ExpressionContext exprctx0 = ctx.expression(0);
-            final ExpressionContext exprctx1 = ctx.expression(1);
-
-            writer.visit(exprctx0);
-            writer.visit(exprctx1);
-
-            final Type type = binaryemd.from;
-
-            if      (ctx.MUL()   != null) utility.writeBinaryInstruction(ctx, type, MUL);
-            else if (ctx.DIV()   != null) utility.writeBinaryInstruction(ctx, type, DIV);
-            else if (ctx.REM()   != null) utility.writeBinaryInstruction(ctx, type, REM);
-            else if (ctx.ADD()   != null) utility.writeBinaryInstruction(ctx, type, ADD);
-            else if (ctx.SUB()   != null) utility.writeBinaryInstruction(ctx, type, SUB);
-            else if (ctx.LSH()   != null) utility.writeBinaryInstruction(ctx, type, LSH);
-            else if (ctx.USH()   != null) utility.writeBinaryInstruction(ctx, type, USH);
-            else if (ctx.RSH()   != null) utility.writeBinaryInstruction(ctx, type, RSH);
-            else if (ctx.BWAND() != null) utility.writeBinaryInstruction(ctx, type, BWAND);
-            else if (ctx.BWXOR() != null) utility.writeBinaryInstruction(ctx, type, BWXOR);
-            else if (ctx.BWOR()  != null) utility.writeBinaryInstruction(ctx, type, BWOR);
-            else {
-                throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
-            }
-
-            caster.checkWriteCast(binaryemd);
-        }
-
-        utility.checkWriteBranch(ctx);
-    }
-
-    void processComp(final CompContext ctx) {
-        final ExpressionMetadata compemd = metadata.getExpressionMetadata(ctx);
-        final Object postConst = compemd.postConst;
-        final Object preConst = compemd.preConst;
-        final Branch branch = utility.getBranch(ctx);
-
-        if (postConst != null) {
-            if (branch == null) {
-                utility.writeConstant(ctx, postConst);
-            } else {
-                if ((boolean)postConst && branch.tru != null) {
-                    execute.mark(branch.tru);
-                } else if (!(boolean)postConst && branch.fals != null) {
-                    execute.mark(branch.fals);
-                }
-            }
-        } else if (preConst != null) {
-            if (branch == null) {
-                utility.writeConstant(ctx, preConst);
-                caster.checkWriteCast(compemd);
-            } else {
-                throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
-            }
-        } else {
-            final ExpressionContext exprctx0 = ctx.expression(0);
-            final ExpressionMetadata expremd0 = metadata.getExpressionMetadata(exprctx0);
-
-            final ExpressionContext exprctx1 = ctx.expression(1);
-            final ExpressionMetadata expremd1 = metadata.getExpressionMetadata(exprctx1);
-            final org.objectweb.asm.Type type = expremd1.to.type;
-            final Sort sort1 = expremd1.to.sort;
-
-            writer.visit(exprctx0);
-
-            if (!expremd1.isNull) {
-                writer.visit(exprctx1);
-            }
-
-            final boolean tru = branch != null && branch.tru != null;
-            final boolean fals = branch != null && branch.fals != null;
-            final Label jump = tru ? branch.tru : fals ? branch.fals : new Label();
-            final Label end = new Label();
-
-            final boolean eq = (ctx.EQ() != null || ctx.EQR() != null) && (tru || !fals) ||
-                (ctx.NE() != null || ctx.NER() != null) && fals;
-            final boolean ne = (ctx.NE() != null || ctx.NER() != null) && (tru || !fals) ||
-                (ctx.EQ() != null || ctx.EQR() != null) && fals;
-            final boolean lt  = ctx.LT()  != null && (tru || !fals) || ctx.GTE() != null && fals;
-            final boolean lte = ctx.LTE() != null && (tru || !fals) || ctx.GT()  != null && fals;
-            final boolean gt  = ctx.GT()  != null && (tru || !fals) || ctx.LTE() != null && fals;
-            final boolean gte = ctx.GTE() != null && (tru || !fals) || ctx.LT()  != null && fals;
-
-            boolean writejump = true;
-
-            switch (sort1) {
-                case VOID:
-                case BYTE:
-                case SHORT:
-                case CHAR:
-                    throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
-                case BOOL:
-                    if      (eq) execute.ifZCmp(GeneratorAdapter.EQ, jump);
-                    else if (ne) execute.ifZCmp(GeneratorAdapter.NE, jump);
-                    else {
-                        throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
-                    }
-
-                    break;
-                case INT:
-                case LONG:
-                case FLOAT:
-                case DOUBLE:
-                    if      (eq)  execute.ifCmp(type, GeneratorAdapter.EQ, jump);
-                    else if (ne)  execute.ifCmp(type, GeneratorAdapter.NE, jump);
-                    else if (lt)  execute.ifCmp(type, GeneratorAdapter.LT, jump);
-                    else if (lte) execute.ifCmp(type, GeneratorAdapter.LE, jump);
-                    else if (gt)  execute.ifCmp(type, GeneratorAdapter.GT, jump);
-                    else if (gte) execute.ifCmp(type, GeneratorAdapter.GE, jump);
-                    else {
-                        throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
-                    }
-
-                    break;
-                case DEF:
-                    if (eq) {
-                        if (expremd1.isNull) {
-                            execute.ifNull(jump);
-                        } else if (!expremd0.isNull && ctx.EQ() != null) {
-                            execute.invokeStatic(definition.defobjType.type, DEF_EQ_CALL);
-                        } else {
-                            execute.ifCmp(type, GeneratorAdapter.EQ, jump);
-                        }
-                    } else if (ne) {
-                        if (expremd1.isNull) {
-                            execute.ifNonNull(jump);
-                        } else if (!expremd0.isNull && ctx.NE() != null) {
-                            execute.invokeStatic(definition.defobjType.type, DEF_EQ_CALL);
-                            execute.ifZCmp(GeneratorAdapter.EQ, jump);
-                        } else {
-                            execute.ifCmp(type, GeneratorAdapter.NE, jump);
-                        }
-                    } else if (lt) {
-                        execute.invokeStatic(definition.defobjType.type, DEF_LT_CALL);
-                    } else if (lte) {
-                        execute.invokeStatic(definition.defobjType.type, DEF_LTE_CALL);
-                    } else if (gt) {
-                        execute.invokeStatic(definition.defobjType.type, DEF_GT_CALL);
-                    } else if (gte) {
-                        execute.invokeStatic(definition.defobjType.type, DEF_GTE_CALL);
-                    } else {
-                        throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
-                    }
-
-                    writejump = expremd1.isNull || ne || ctx.EQR() != null;
-
-                    if (branch != null && !writejump) {
-                        execute.ifZCmp(GeneratorAdapter.NE, jump);
-                    }
-
-                    break;
-                default:
-                    if (eq) {
-                        if (expremd1.isNull) {
-                            execute.ifNull(jump);
-                        } else if (ctx.EQ() != null) {
-                            execute.invokeStatic(definition.utilityType.type, CHECKEQUALS);
-
-                            if (branch != null) {
-                                execute.ifZCmp(GeneratorAdapter.NE, jump);
-                            }
-
-                            writejump = false;
-                        } else {
-                            execute.ifCmp(type, GeneratorAdapter.EQ, jump);
-                        }
-                    } else if (ne) {
-                        if (expremd1.isNull) {
-                            execute.ifNonNull(jump);
-                        } else if (ctx.NE() != null) {
-                            execute.invokeStatic(definition.utilityType.type, CHECKEQUALS);
-                            execute.ifZCmp(GeneratorAdapter.EQ, jump);
-                        } else {
-                            execute.ifCmp(type, GeneratorAdapter.NE, jump);
-                        }
-                    } else {
-                        throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
-                    }
-            }
-
-            if (branch == null) {
-                if (writejump) {
-                    execute.push(false);
-                    execute.goTo(end);
-                    execute.mark(jump);
-                    execute.push(true);
-                    execute.mark(end);
-                }
-
-                caster.checkWriteCast(compemd);
-            }
-        }
-    }
-
-    void processBool(final BoolContext ctx) {
-        final ExpressionMetadata boolemd = metadata.getExpressionMetadata(ctx);
-        final Object postConst = boolemd.postConst;
-        final Object preConst = boolemd.preConst;
-        final Branch branch = utility.getBranch(ctx);
-
-        if (postConst != null) {
-            if (branch == null) {
-                utility.writeConstant(ctx, postConst);
-            } else {
-                if ((boolean)postConst && branch.tru != null) {
-                    execute.mark(branch.tru);
-                } else if (!(boolean)postConst && branch.fals != null) {
-                    execute.mark(branch.fals);
-                }
-            }
-        } else if (preConst != null) {
-            if (branch == null) {
-                utility.writeConstant(ctx, preConst);
-                caster.checkWriteCast(boolemd);
-            } else {
-                throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
-            }
-        } else {
-            final ExpressionContext exprctx0 = ctx.expression(0);
-            final ExpressionContext exprctx1 = ctx.expression(1);
-
-            if (branch == null) {
-                if (ctx.BOOLAND() != null) {
-                    final Branch local = utility.markBranch(ctx, exprctx0, exprctx1);
-                    local.fals = new Label();
-                    final Label end = new Label();
-
-                    writer.visit(exprctx0);
-                    writer.visit(exprctx1);
-
-                    execute.push(true);
-                    execute.goTo(end);
-                    execute.mark(local.fals);
-                    execute.push(false);
-                    execute.mark(end);
-                } else if (ctx.BOOLOR() != null) {
-                    final Branch branch0 = utility.markBranch(ctx, exprctx0);
-                    branch0.tru = new Label();
-                    final Branch branch1 = utility.markBranch(ctx, exprctx1);
-                    branch1.fals = new Label();
-                    final Label aend = new Label();
-
-                    writer.visit(exprctx0);
-                    writer.visit(exprctx1);
-
-                    execute.mark(branch0.tru);
-                    execute.push(true);
-                    execute.goTo(aend);
-                    execute.mark(branch1.fals);
-                    execute.push(false);
-                    execute.mark(aend);
-                } else {
-                    throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
-                }
-
-                caster.checkWriteCast(boolemd);
-            } else {
-                if (ctx.BOOLAND() != null) {
-                    final Branch branch0 = utility.markBranch(ctx, exprctx0);
-                    branch0.fals = branch.fals == null ? new Label() : branch.fals;
-                    final Branch branch1 = utility.markBranch(ctx, exprctx1);
-                    branch1.tru = branch.tru;
-                    branch1.fals = branch.fals;
-
-                    writer.visit(exprctx0);
-                    writer.visit(exprctx1);
-
-                    if (branch.fals == null) {
-                        execute.mark(branch0.fals);
-                    }
-                } else if (ctx.BOOLOR() != null) {
-                    final Branch branch0 = utility.markBranch(ctx, exprctx0);
-                    branch0.tru = branch.tru == null ? new Label() : branch.tru;
-                    final Branch branch1 = utility.markBranch(ctx, exprctx1);
-                    branch1.tru = branch.tru;
-                    branch1.fals = branch.fals;
-
-                    writer.visit(exprctx0);
-                    writer.visit(exprctx1);
-
-                    if (branch.tru == null) {
-                        execute.mark(branch0.tru);
-                    }
-                } else {
-                    throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
-                }
-            }
-        }
-    }
-
-    void processConditional(final ConditionalContext ctx) {
-        final ExpressionMetadata condemd = metadata.getExpressionMetadata(ctx);
-        final Branch branch = utility.getBranch(ctx);
-
-        final ExpressionContext expr0 = ctx.expression(0);
-        final ExpressionContext expr1 = ctx.expression(1);
-        final ExpressionContext expr2 = ctx.expression(2);
-
-        final Branch local = utility.markBranch(ctx, expr0);
-        local.fals = new Label();
-        local.end = new Label();
-
-        if (branch != null) {
-            utility.copyBranch(branch, expr1, expr2);
-        }
-
-        writer.visit(expr0);
-        writer.visit(expr1);
-        execute.goTo(local.end);
-        execute.mark(local.fals);
-        writer.visit(expr2);
-        execute.mark(local.end);
-
-        if (branch == null) {
-            caster.checkWriteCast(condemd);
-        }
-    }
-
-    void processAssignment(final AssignmentContext ctx) {
-        final ExpressionMetadata expremd = metadata.getExpressionMetadata(ctx);
-        writer.visit(ctx.extstart());
-        caster.checkWriteCast(expremd);
-        utility.checkWriteBranch(ctx);
-    }
-
-    void processIncrement(final IncrementContext ctx) {
-        final ExpressionMetadata incremd = metadata.getExpressionMetadata(ctx);
-        final Object postConst = incremd.postConst;
-
-        if (postConst == null) {
-            utility.writeNumeric(ctx, incremd.preConst);
-            caster.checkWriteCast(incremd);
-        } else {
-            utility.writeConstant(ctx, postConst);
-        }
-
-        utility.checkWriteBranch(ctx);
-    }
-}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java
deleted file mode 100644
index 2ce231412c1..00000000000
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterExternal.java
+++ /dev/null
@@ -1,720 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless;
-
-import org.antlr.v4.runtime.ParserRuleContext;
-import org.elasticsearch.painless.Definition.Cast;
-import org.elasticsearch.painless.Definition.Constructor;
-import org.elasticsearch.painless.Definition.Field;
-import org.elasticsearch.painless.Definition.Method;
-import org.elasticsearch.painless.Definition.Sort;
-import org.elasticsearch.painless.Definition.Type;
-import org.elasticsearch.painless.Metadata.ExpressionMetadata;
-import org.elasticsearch.painless.Metadata.ExtNodeMetadata;
-import org.elasticsearch.painless.Metadata.ExternalMetadata;
-import org.elasticsearch.painless.PainlessParser.ExpressionContext;
-import org.elasticsearch.painless.PainlessParser.ExtbraceContext;
-import org.elasticsearch.painless.PainlessParser.ExtcallContext;
-import org.elasticsearch.painless.PainlessParser.ExtcastContext;
-import org.elasticsearch.painless.PainlessParser.ExtdotContext;
-import org.elasticsearch.painless.PainlessParser.ExtfieldContext;
-import org.elasticsearch.painless.PainlessParser.ExtnewContext;
-import org.elasticsearch.painless.PainlessParser.ExtprecContext;
-import org.elasticsearch.painless.PainlessParser.ExtstartContext;
-import org.elasticsearch.painless.PainlessParser.ExtstringContext;
-import org.elasticsearch.painless.PainlessParser.ExtvarContext;
-import org.objectweb.asm.Opcodes;
-import org.objectweb.asm.commons.GeneratorAdapter;
-
-import java.util.List;
-
-import static org.elasticsearch.painless.PainlessParser.ADD;
-import static org.elasticsearch.painless.PainlessParser.DIV;
-import static org.elasticsearch.painless.PainlessParser.MUL;
-import static org.elasticsearch.painless.PainlessParser.REM;
-import static org.elasticsearch.painless.PainlessParser.SUB;
-import static org.elasticsearch.painless.WriterConstants.TOBYTEEXACT_INT;
-import static org.elasticsearch.painless.WriterConstants.TOBYTEEXACT_LONG;
-import static org.elasticsearch.painless.WriterConstants.TOBYTEWOOVERFLOW_DOUBLE;
-import static org.elasticsearch.painless.WriterConstants.TOBYTEWOOVERFLOW_FLOAT;
-import static org.elasticsearch.painless.WriterConstants.TOCHAREXACT_INT;
-import static org.elasticsearch.painless.WriterConstants.TOCHAREXACT_LONG;
-import static org.elasticsearch.painless.WriterConstants.TOCHARWOOVERFLOW_DOUBLE;
-import static org.elasticsearch.painless.WriterConstants.TOCHARWOOVERFLOW_FLOAT;
-import static org.elasticsearch.painless.WriterConstants.TOFLOATWOOVERFLOW_DOUBLE;
-import static org.elasticsearch.painless.WriterConstants.TOINTEXACT_LONG;
-import static org.elasticsearch.painless.WriterConstants.TOINTWOOVERFLOW_DOUBLE;
-import static org.elasticsearch.painless.WriterConstants.TOINTWOOVERFLOW_FLOAT;
-import static org.elasticsearch.painless.WriterConstants.TOLONGWOOVERFLOW_DOUBLE;
-import static org.elasticsearch.painless.WriterConstants.TOLONGWOOVERFLOW_FLOAT;
-import static org.elasticsearch.painless.WriterConstants.TOSHORTEXACT_INT;
-import static org.elasticsearch.painless.WriterConstants.TOSHORTEXACT_LONG;
-import static org.elasticsearch.painless.WriterConstants.TOSHORTWOOVERFLOW_DOUBLE;
-import static org.elasticsearch.painless.WriterConstants.TOSHORTWOOVERFLOW_FLOAT;
-
-class WriterExternal {
-    private final Metadata metadata;
-    private final Definition definition;
-    private final CompilerSettings settings;
-
-    private final GeneratorAdapter execute;
-
-    private final Writer writer;
-    private final WriterUtility utility;
-    private final WriterCaster caster;
-
-    WriterExternal(final Metadata metadata, final GeneratorAdapter execute, final Writer writer,
-                   final WriterUtility utility, final WriterCaster caster) {
-        this.metadata = metadata;
-        definition = metadata.definition;
-        settings = metadata.settings;
-
-        this.execute = execute;
-
-        this.writer = writer;
-        this.utility = utility;
-        this.caster = caster;
-    }
-
-    void processExtstart(final ExtstartContext ctx) {
-        final ExternalMetadata startemd = metadata.getExternalMetadata(ctx);
-
-        if (startemd.token == ADD) {
-            final ExpressionMetadata storeemd = metadata.getExpressionMetadata(startemd.storeExpr);
-
-            if (startemd.current.sort == Sort.STRING || storeemd.from.sort == Sort.STRING) {
-                utility.writeNewStrings();
-                utility.addStrings(startemd.storeExpr);
-            }
-        }
-
-        final ExtprecContext precctx = ctx.extprec();
-        final ExtcastContext castctx = ctx.extcast();
-        final ExtvarContext varctx = ctx.extvar();
-        final ExtnewContext newctx = ctx.extnew();
-        final ExtstringContext stringctx = ctx.extstring();
-
-        if (precctx != null) {
-            writer.visit(precctx);
-        } else if (castctx != null) {
-            writer.visit(castctx);
-        } else if (varctx != null) {
-            writer.visit(varctx);
-        } else if (newctx != null) {
-            writer.visit(newctx);
-        } else if (stringctx != null) {
-            writer.visit(stringctx);
-        } else {
-            throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
-        }
-    }
-
-    void processExtprec(final ExtprecContext ctx) {
-        final ExtprecContext precctx = ctx.extprec();
-        final ExtcastContext castctx = ctx.extcast();
-        final ExtvarContext varctx = ctx.extvar();
-        final ExtnewContext newctx = ctx.extnew();
-        final ExtstringContext stringctx = ctx.extstring();
-
-        if (precctx != null) {
-            writer.visit(precctx);
-        } else if (castctx != null) {
-            writer.visit(castctx);
-        } else if (varctx != null) {
-            writer.visit(varctx);
-        } else if (newctx != null) {
-            writer.visit(newctx);
-        } else if (stringctx != null) {
-            writer.visit(stringctx);
-        } else {
-            throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
-        }
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        if (dotctx != null) {
-            writer.visit(dotctx);
-        } else if (bracectx != null) {
-            writer.visit(bracectx);
-        }
-    }
-
-    void processExtcast(final ExtcastContext ctx) {
-        ExtNodeMetadata castenmd = metadata.getExtNodeMetadata(ctx);
-
-        final ExtprecContext precctx = ctx.extprec();
-        final ExtcastContext castctx = ctx.extcast();
-        final ExtvarContext varctx = ctx.extvar();
-        final ExtnewContext newctx = ctx.extnew();
-        final ExtstringContext stringctx = ctx.extstring();
-
-        if (precctx != null) {
-            writer.visit(precctx);
-        } else if (castctx != null) {
-            writer.visit(castctx);
-        } else if (varctx != null) {
-            writer.visit(varctx);
-        } else if (newctx != null) {
-            writer.visit(newctx);
-        } else if (stringctx != null) {
-            writer.visit(stringctx);
-        } else {
-            throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
-        }
-
-        caster.checkWriteCast(ctx, castenmd.castTo);
-    }
-
-    void processExtbrace(final ExtbraceContext ctx) {
-        final ExpressionContext exprctx = ctx.expression();
-
-        writer.visit(exprctx);
-        writeLoadStoreExternal(ctx);
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        if (dotctx != null) {
-            writer.visit(dotctx);
-        } else if (bracectx != null) {
-            writer.visit(bracectx);
-        }
-    }
-
-    void processExtdot(final ExtdotContext ctx) {
-        final ExtcallContext callctx = ctx.extcall();
-        final ExtfieldContext fieldctx = ctx.extfield();
-
-        if (callctx != null) {
-            writer.visit(callctx);
-        } else if (fieldctx != null) {
-            writer.visit(fieldctx);
-        }
-    }
-
-    void processExtcall(final ExtcallContext ctx) {
-        writeCallExternal(ctx);
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        if (dotctx != null) {
-            writer.visit(dotctx);
-        } else if (bracectx != null) {
-            writer.visit(bracectx);
-        }
-    }
-
-    void processExtvar(final ExtvarContext ctx) {
-        writeLoadStoreExternal(ctx);
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        if (dotctx != null) {
-            writer.visit(dotctx);
-        } else if (bracectx != null) {
-            writer.visit(bracectx);
-        }
-    }
-
-    void processExtfield(final ExtfieldContext ctx) {
-        writeLoadStoreExternal(ctx);
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        if (dotctx != null) {
-            writer.visit(dotctx);
-        } else if (bracectx != null) {
-            writer.visit(bracectx);
-        }
-    }
-
-    void processExtnew(final ExtnewContext ctx) {
-        writeNewExternal(ctx);
-
-        final ExtdotContext dotctx = ctx.extdot();
-
-        if (dotctx != null) {
-            writer.visit(dotctx);
-        }
-    }
-
-    void processExtstring(final ExtstringContext ctx) {
-        final ExtNodeMetadata stringenmd = metadata.getExtNodeMetadata(ctx);
-
-        utility.writeConstant(ctx, stringenmd.target);
-
-        final ExtdotContext dotctx = ctx.extdot();
-        final ExtbraceContext bracectx = ctx.extbrace();
-
-        if (dotctx != null) {
-            writer.visit(dotctx);
-        } else if (bracectx != null) {
-            writer.visit(bracectx);
-        }
-    }
-
-    private void writeLoadStoreExternal(final ParserRuleContext source) {
-        final ExtNodeMetadata sourceenmd = metadata.getExtNodeMetadata(source);
-        final ExternalMetadata parentemd = metadata.getExternalMetadata(sourceenmd.parent);
-
-        if (sourceenmd.target == null) {
-            return;
-        }
-
-        final boolean length = "#length".equals(sourceenmd.target);
-        final boolean array = "#brace".equals(sourceenmd.target);
-        final boolean name = sourceenmd.target instanceof String && !length && !array;
-        final boolean variable = sourceenmd.target instanceof Integer;
-        final boolean field = sourceenmd.target instanceof Field;
-        final boolean shortcut = sourceenmd.target instanceof Object[];
-
-        if (!length && !variable && !field && !array && !name && !shortcut) {
-            throw new IllegalStateException(WriterUtility.error(source) + "Target not found for load/store.");
-        }
-
-        final boolean maplist = shortcut && (boolean)((Object[])sourceenmd.target)[2];
-        final Object constant = shortcut ? ((Object[])sourceenmd.target)[3] : null;
-
-        final boolean x1 = field || name || (shortcut && !maplist);
-        final boolean x2 = array || (shortcut && maplist);
-
-        if (length) {
-            execute.arrayLength();
-        } else if (sourceenmd.last && parentemd.storeExpr != null) {
-            final ExpressionMetadata expremd = metadata.getExpressionMetadata(parentemd.storeExpr);
-            final boolean cat = utility.containsStrings(parentemd.storeExpr);
-
-            if (cat) {
-                if (maplist && constant != null) {
-                    utility.writeConstant(source, constant);
-                }
-
-                if (field || name || (shortcut && !maplist)) {
-                    execute.dupX1();
-                } else if (array || maplist) {
-                    execute.dup2X1();
-                }
-
-                writeLoadStoreInstruction(source, false, variable, field, name, array, shortcut);
-                utility.writeAppendStrings(sourceenmd.type.sort);
-                writer.visit(parentemd.storeExpr);
-
-                if (utility.containsStrings(parentemd.storeExpr)) {
-                    utility.writeAppendStrings(expremd.to.sort);
-                    utility.removeStrings(parentemd.storeExpr);
-                }
-
-                utility.writeToStrings();
-                caster.checkWriteCast(source, sourceenmd.castTo);
-
-                if (parentemd.read) {
-                    utility.writeDup(sourceenmd.type.sort.size, x1, x2);
-                }
-
-                writeLoadStoreInstruction(source, true, variable, field, name, array, shortcut);
-            } else if (parentemd.token > 0) {
-                final int token = parentemd.token;
-
-                if (maplist && constant != null) {
-                    utility.writeConstant(source, constant);
-                }
-
-                if (field || name || (shortcut && !maplist)) {
-                    execute.dup();
-                } else if (array || maplist) {
-                    execute.dup2();
-                }
-
-                writeLoadStoreInstruction(source, false, variable, field, name, array, shortcut);
-
-                if (parentemd.read && parentemd.post) {
-                    utility.writeDup(sourceenmd.type.sort.size, x1, x2);
-                }
-
-                caster.checkWriteCast(source, sourceenmd.castFrom);
-                writer.visit(parentemd.storeExpr);
-
-                utility.writeBinaryInstruction(source, sourceenmd.promote, token);
-
-                boolean exact = false;
-
-                if (!settings.getNumericOverflow() && expremd.typesafe && sourceenmd.type.sort != Sort.DEF &&
-                    (token == MUL || token == DIV || token == REM || token == ADD || token == SUB)) {
-                    exact = writeExactInstruction(sourceenmd.type.sort, sourceenmd.promote.sort);
-                }
-
-                if (!exact) {
-                    caster.checkWriteCast(source, sourceenmd.castTo);
-                }
-
-                if (parentemd.read && !parentemd.post) {
-                    utility.writeDup(sourceenmd.type.sort.size, x1, x2);
-                }
-
-                writeLoadStoreInstruction(source, true, variable, field, name, array, shortcut);
-            } else {
-                if (constant != null) {
-                    utility.writeConstant(source, constant);
-                }
-
-                writer.visit(parentemd.storeExpr);
-
-                if (parentemd.read) {
-                    utility.writeDup(sourceenmd.type.sort.size, x1, x2);
-                }
-
-                writeLoadStoreInstruction(source, true, variable, field, name, array, shortcut);
-            }
-        } else {
-            if (constant != null) {
-                utility.writeConstant(source, constant);
-            }
-
-            writeLoadStoreInstruction(source, false, variable, field, name, array, shortcut);
-        }
-    }
-
-    private void writeLoadStoreInstruction(final ParserRuleContext source,
-                                           final boolean store, final boolean variable,
-                                           final boolean field, final boolean name,
-                                           final boolean array, final boolean shortcut) {
-        final ExtNodeMetadata sourceemd = metadata.getExtNodeMetadata(source);
-
-        if (variable) {
-            writeLoadStoreVariable(source, store, sourceemd.type, (int)sourceemd.target);
-        } else if (field) {
-            writeLoadStoreField(store, (Field)sourceemd.target);
-        } else if (name) {
-            writeLoadStoreField(source, store, (String)sourceemd.target);
-        } else if (array) {
-            writeLoadStoreArray(source, store, sourceemd.type);
-        } else if (shortcut) {
-            Object[] targets = (Object[])sourceemd.target;
-            writeLoadStoreShortcut(store, (Method)targets[0], (Method)targets[1]);
-        } else {
-            throw new IllegalStateException(WriterUtility.error(source) + "Load/Store requires a variable, field, or array.");
-        }
-    }
-
-    private void writeLoadStoreVariable(final ParserRuleContext source, final boolean store, final Type type, int slot) {
-        if (type.sort == Sort.VOID) {
-            throw new IllegalStateException(WriterUtility.error(source) + "Cannot load/store void type.");
-        }
-
-        if (store) {
-            execute.visitVarInsn(type.type.getOpcode(Opcodes.ISTORE), slot);
-        } else {
-            execute.visitVarInsn(type.type.getOpcode(Opcodes.ILOAD), slot);
-        }
-    }
-
-    private void writeLoadStoreField(final boolean store, final Field field) {
-        if (java.lang.reflect.Modifier.isStatic(field.reflect.getModifiers())) {
-            if (store) {
-                execute.putStatic(field.owner.type, field.reflect.getName(), field.type.type);
-            } else {
-                execute.getStatic(field.owner.type, field.reflect.getName(), field.type.type);
-
-                if (!field.generic.clazz.equals(field.type.clazz)) {
-                    execute.checkCast(field.generic.type);
-                }
-            }
-        } else {
-            if (store) {
-                execute.putField(field.owner.type, field.reflect.getName(), field.type.type);
-            } else {
-                execute.getField(field.owner.type, field.reflect.getName(), field.type.type);
-
-                if (!field.generic.clazz.equals(field.type.clazz)) {
-                    execute.checkCast(field.generic.type);
-                }
-            }
-        }
-    }
-
-    private void writeLoadStoreField(final ParserRuleContext source, final boolean store, final String name) {
-        if (store) {
-            execute.visitInvokeDynamicInsn(name, WriterConstants.DEF_DYNAMIC_STORE_FIELD_DESC,
-                                           WriterConstants.DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.STORE });
-        } else {
-            execute.visitInvokeDynamicInsn(name, WriterConstants.DEF_DYNAMIC_LOAD_FIELD_DESC,
-                                           WriterConstants.DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.LOAD });
-        }
-    }
-
-    private void writeLoadStoreArray(final ParserRuleContext source, final boolean store, final Type type) {
-        if (type.sort == Sort.VOID) {
-            throw new IllegalStateException(WriterUtility.error(source) + "Cannot load/store void type.");
-        }
-
-        if (type.sort == Sort.DEF) {
-            if (store) {
-                execute.visitInvokeDynamicInsn("arrayStore", WriterConstants.DEF_DYNAMIC_ARRAY_STORE_DESC,
-                                           WriterConstants.DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.ARRAY_STORE });
-            } else {
-                execute.visitInvokeDynamicInsn("arrayLoad", WriterConstants.DEF_DYNAMIC_ARRAY_LOAD_DESC,
-                                           WriterConstants.DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.ARRAY_LOAD });
-            }
-        } else {
-            if (store) {
-                execute.arrayStore(type.type);
-            } else {
-                execute.arrayLoad(type.type);
-            }
-        }
-    }
-
-    private void writeLoadStoreShortcut(final boolean store, final Method getter, final Method setter) {
-        final Method method = store ? setter : getter;
-
-        if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
-            execute.invokeInterface(method.owner.type, method.method);
-        } else {
-            execute.invokeVirtual(method.owner.type, method.method);
-        }
-
-        if (store) {
-            utility.writePop(method.rtn.type.getSize());
-        } else if (!method.rtn.clazz.equals(method.handle.type().returnType())) {
-            execute.checkCast(method.rtn.type);
-        }
-    }
-
-    /**
-     * Called for any compound assignment (including increment/decrement instructions).
-     * We have to be stricter than writeBinary, and do overflow checks against the original type's size
-     * instead of the promoted type's size, since the result will be implicitly cast back.
-     *
-     * @return This will be true if an instruction is written, false otherwise.
-     */
-    private boolean writeExactInstruction(final Sort osort, final Sort psort) {
-        if (psort == Sort.DOUBLE) {
-            if (osort == Sort.FLOAT) {
-                execute.invokeStatic(definition.utilityType.type, TOFLOATWOOVERFLOW_DOUBLE);
-            } else if (osort == Sort.FLOAT_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOFLOATWOOVERFLOW_DOUBLE);
-                execute.checkCast(definition.floatobjType.type);
-            } else if (osort == Sort.LONG) {
-                execute.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_DOUBLE);
-            } else if (osort == Sort.LONG_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_DOUBLE);
-                execute.checkCast(definition.longobjType.type);
-            } else if (osort == Sort.INT) {
-                execute.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_DOUBLE);
-            } else if (osort == Sort.INT_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_DOUBLE);
-                execute.checkCast(definition.intobjType.type);
-            } else if (osort == Sort.CHAR) {
-                execute.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_DOUBLE);
-            } else if (osort == Sort.CHAR_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_DOUBLE);
-                execute.checkCast(definition.charobjType.type);
-            } else if (osort == Sort.SHORT) {
-                execute.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_DOUBLE);
-            } else if (osort == Sort.SHORT_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_DOUBLE);
-                execute.checkCast(definition.shortobjType.type);
-            } else if (osort == Sort.BYTE) {
-                execute.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_DOUBLE);
-            } else if (osort == Sort.BYTE_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_DOUBLE);
-                execute.checkCast(definition.byteobjType.type);
-            } else {
-                return false;
-            }
-        } else if (psort == Sort.FLOAT) {
-            if (osort == Sort.LONG) {
-                execute.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_FLOAT);
-            } else if (osort == Sort.LONG_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_FLOAT);
-                execute.checkCast(definition.longobjType.type);
-            } else if (osort == Sort.INT) {
-                execute.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_FLOAT);
-            } else if (osort == Sort.INT_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_FLOAT);
-                execute.checkCast(definition.intobjType.type);
-            } else if (osort == Sort.CHAR) {
-                execute.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_FLOAT);
-            } else if (osort == Sort.CHAR_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_FLOAT);
-                execute.checkCast(definition.charobjType.type);
-            } else if (osort == Sort.SHORT) {
-                execute.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_FLOAT);
-            } else if (osort == Sort.SHORT_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_FLOAT);
-                execute.checkCast(definition.shortobjType.type);
-            } else if (osort == Sort.BYTE) {
-                execute.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_FLOAT);
-            } else if (osort == Sort.BYTE_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_FLOAT);
-                execute.checkCast(definition.byteobjType.type);
-            } else {
-                return false;
-            }
-        } else if (psort == Sort.LONG) {
-            if (osort == Sort.INT) {
-                execute.invokeStatic(definition.mathType.type, TOINTEXACT_LONG);
-            } else if (osort == Sort.INT_OBJ) {
-                execute.invokeStatic(definition.mathType.type, TOINTEXACT_LONG);
-                execute.checkCast(definition.intobjType.type);
-            } else if (osort == Sort.CHAR) {
-                execute.invokeStatic(definition.utilityType.type, TOCHAREXACT_LONG);
-            } else if (osort == Sort.CHAR_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOCHAREXACT_LONG);
-                execute.checkCast(definition.charobjType.type);
-            } else if (osort == Sort.SHORT) {
-                execute.invokeStatic(definition.utilityType.type, TOSHORTEXACT_LONG);
-            } else if (osort == Sort.SHORT_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOSHORTEXACT_LONG);
-                execute.checkCast(definition.shortobjType.type);
-            } else if (osort == Sort.BYTE) {
-                execute.invokeStatic(definition.utilityType.type, TOBYTEEXACT_LONG);
-            } else if (osort == Sort.BYTE_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOBYTEEXACT_LONG);
-                execute.checkCast(definition.byteobjType.type);
-            } else {
-                return false;
-            }
-        } else if (psort == Sort.INT) {
-            if (osort == Sort.CHAR) {
-                execute.invokeStatic(definition.utilityType.type, TOCHAREXACT_INT);
-            } else if (osort == Sort.CHAR_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOCHAREXACT_INT);
-                execute.checkCast(definition.charobjType.type);
-            } else if (osort == Sort.SHORT) {
-                execute.invokeStatic(definition.utilityType.type, TOSHORTEXACT_INT);
-            } else if (osort == Sort.SHORT_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOSHORTEXACT_INT);
-                execute.checkCast(definition.shortobjType.type);
-            } else if (osort == Sort.BYTE) {
-                execute.invokeStatic(definition.utilityType.type, TOBYTEEXACT_INT);
-            } else if (osort == Sort.BYTE_OBJ) {
-                execute.invokeStatic(definition.utilityType.type, TOBYTEEXACT_INT);
-                execute.checkCast(definition.byteobjType.type);
-            } else {
-                return false;
-            }
-        } else {
-            return false;
-        }
-
-        return true;
-    }
-
-    private void writeNewExternal(final ExtnewContext source) {
-        final ExtNodeMetadata sourceenmd = metadata.getExtNodeMetadata(source);
-        final ExternalMetadata parentemd = metadata.getExternalMetadata(sourceenmd.parent);
-
-        final boolean makearray = "#makearray".equals(sourceenmd.target);
-        final boolean constructor = sourceenmd.target instanceof Constructor;
-
-        if (!makearray && !constructor) {
-            throw new IllegalStateException(WriterUtility.error(source) + "Target not found for new call.");
-        }
-
-        if (makearray) {
-            for (final ExpressionContext exprctx : source.expression()) {
-                writer.visit(exprctx);
-            }
-
-            if (sourceenmd.type.sort == Sort.ARRAY) {
-                execute.visitMultiANewArrayInsn(sourceenmd.type.type.getDescriptor(), sourceenmd.type.type.getDimensions());
-            } else {
-                execute.newArray(sourceenmd.type.type);
-            }
-        } else {
-            execute.newInstance(sourceenmd.type.type);
-
-            if (parentemd.read) {
-                execute.dup();
-            }
-
-            for (final ExpressionContext exprctx : source.arguments().expression()) {
-                writer.visit(exprctx);
-            }
-
-            final Constructor target = (Constructor)sourceenmd.target;
-            execute.invokeConstructor(target.owner.type, target.method);
-        }
-    }
-
-    private void writeCallExternal(final ExtcallContext source) {
-        final ExtNodeMetadata sourceenmd = metadata.getExtNodeMetadata(source);
-
-        final boolean method = sourceenmd.target instanceof Method;
-        final boolean def = sourceenmd.target instanceof String;
-
-        if (!method && !def) {
-            throw new IllegalStateException(WriterUtility.error(source) + "Target not found for call.");
-        }
-
-        final List<ExpressionContext> arguments = source.arguments().expression();
-
-        if (method) {
-            for (final ExpressionContext exprctx : arguments) {
-                writer.visit(exprctx);
-            }
-
-            final Method target = (Method)sourceenmd.target;
-
-            if (java.lang.reflect.Modifier.isStatic(target.reflect.getModifiers())) {
-                execute.invokeStatic(target.owner.type, target.method);
-            } else if (java.lang.reflect.Modifier.isInterface(target.owner.clazz.getModifiers())) {
-                execute.invokeInterface(target.owner.type, target.method);
-            } else {
-                execute.invokeVirtual(target.owner.type, target.method);
-            }
-
-            if (!target.rtn.clazz.equals(target.handle.type().returnType())) {
-                execute.checkCast(target.rtn.type);
-            }
-        } else {
-            writeDynamicCallExternal(source);
-        }
-    }
-
-    private void writeDynamicCallExternal(final ExtcallContext source) {
-        final ExtNodeMetadata sourceenmd = metadata.getExtNodeMetadata(source);
-        final List<ExpressionContext> arguments = source.arguments().expression();
-
-        StringBuilder signature = new StringBuilder();
-        signature.append('(');
-        // first parameter is the receiver, we never know its type: always Object
-        signature.append(WriterConstants.OBJECT_TYPE.getDescriptor());
-
-        for (int i = 0; i < arguments.size(); i++) {
-            ExpressionMetadata arg = metadata.getExpressionMetadata(arguments.get(i));
-            // disable any implicit casts/conversion for arguments, let invokeDynamic take care
-            arg.to = arg.from;
-            arg.cast = new Cast(arg.from, arg.from);
-            signature.append(arg.from.type.getDescriptor());
-            writer.visit(arguments.get(i));
-        }
-        signature.append(')');
-        // return value: currently always Object. making this better may be tricky...
-        signature.append(WriterConstants.OBJECT_TYPE.getDescriptor());
-        execute.visitInvokeDynamicInsn((String)sourceenmd.target, signature.toString(),
-                                       WriterConstants.DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.METHOD_CALL });
-    }
-}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterStatement.java
deleted file mode 100644
index fc8b5ba71e4..00000000000
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterStatement.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless;
-
-import org.elasticsearch.painless.Definition.Sort;
-import org.elasticsearch.painless.Metadata.ExpressionMetadata;
-import org.elasticsearch.painless.Metadata.StatementMetadata;
-import org.elasticsearch.painless.PainlessParser.AfterthoughtContext;
-import org.elasticsearch.painless.PainlessParser.BlockContext;
-import org.elasticsearch.painless.PainlessParser.DeclContext;
-import org.elasticsearch.painless.PainlessParser.DeclarationContext;
-import org.elasticsearch.painless.PainlessParser.DeclvarContext;
-import org.elasticsearch.painless.PainlessParser.DoContext;
-import org.elasticsearch.painless.PainlessParser.EmptyscopeContext;
-import org.elasticsearch.painless.PainlessParser.ExprContext;
-import org.elasticsearch.painless.PainlessParser.ExpressionContext;
-import org.elasticsearch.painless.PainlessParser.ForContext;
-import org.elasticsearch.painless.PainlessParser.IfContext;
-import org.elasticsearch.painless.PainlessParser.InitializerContext;
-import org.elasticsearch.painless.PainlessParser.MultipleContext;
-import org.elasticsearch.painless.PainlessParser.ReturnContext;
-import org.elasticsearch.painless.PainlessParser.SingleContext;
-import org.elasticsearch.painless.PainlessParser.SourceContext;
-import org.elasticsearch.painless.PainlessParser.StatementContext;
-import org.elasticsearch.painless.PainlessParser.ThrowContext;
-import org.elasticsearch.painless.PainlessParser.TrapContext;
-import org.elasticsearch.painless.PainlessParser.TryContext;
-import org.elasticsearch.painless.PainlessParser.WhileContext;
-import org.elasticsearch.painless.WriterUtility.Branch;
-import org.objectweb.asm.Label;
-import org.objectweb.asm.Opcodes;
-import org.objectweb.asm.commons.GeneratorAdapter;
-
-import static org.elasticsearch.painless.WriterConstants.PAINLESS_ERROR_TYPE;
-
-class WriterStatement {
-    private final Metadata metadata;
-
-    private final GeneratorAdapter execute;
-
-    private final Writer writer;
-    private final WriterUtility utility;
-
-    WriterStatement(final Metadata metadata, final GeneratorAdapter execute,
-                    final Writer writer, final WriterUtility utility) {
-        this.metadata = metadata;
-
-        this.execute = execute;
-
-        this.writer = writer;
-        this.utility = utility;
-    }
-
-    void processSource(final SourceContext ctx) {
-        final StatementMetadata sourcesmd = metadata.getStatementMetadata(ctx);
-
-        for (final StatementContext sctx : ctx.statement()) {
-            writer.visit(sctx);
-        }
-
-        if (!sourcesmd.methodEscape) {
-            execute.visitInsn(Opcodes.ACONST_NULL);
-            execute.returnValue();
-        }
-    }
-
-    void processIf(final IfContext ctx) {
-        final ExpressionContext exprctx = ctx.expression();
-        final boolean els = ctx.ELSE() != null;
-        final Branch branch = utility.markBranch(ctx, exprctx);
-        branch.end = new Label();
-        branch.fals = els ? new Label() : branch.end;
-
-        writer.visit(exprctx);
-
-        final BlockContext blockctx0 = ctx.block(0);
-        final StatementMetadata blockmd0 = metadata.getStatementMetadata(blockctx0);
-        writer.visit(blockctx0);
-
-        if (els) {
-            if (!blockmd0.allLast) {
-                execute.goTo(branch.end);
-            }
-
-            execute.mark(branch.fals);
-            writer.visit(ctx.block(1));
-        }
-
-        execute.mark(branch.end);
-    }
-
-    void processWhile(final WhileContext ctx) {
-        final ExpressionContext exprctx = ctx.expression();
-        final Branch branch = utility.markBranch(ctx, exprctx);
-        branch.begin = new Label();
-        branch.end = new Label();
-        branch.fals = branch.end;
-
-        utility.pushJump(branch);
-        execute.mark(branch.begin);
-        writer.visit(exprctx);
-
-        final BlockContext blockctx = ctx.block();
-        boolean allLast = false;
-
-        if (blockctx != null) {
-            final StatementMetadata blocksmd = metadata.getStatementMetadata(blockctx);
-            allLast = blocksmd.allLast;
-            writeLoopCounter(blocksmd.count > 0 ? blocksmd.count : 1);
-            writer.visit(blockctx);
-        } else if (ctx.empty() != null) {
-            writeLoopCounter(1);
-        } else {
-            throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
-        }
-
-        if (!allLast) {
-            execute.goTo(branch.begin);
-        }
-
-        execute.mark(branch.end);
-        utility.popJump();
-    }
-
-    void processDo(final DoContext ctx) {
-        final ExpressionContext exprctx = ctx.expression();
-        final Branch branch = utility.markBranch(ctx, exprctx);
-        Label start = new Label();
-        branch.begin = new Label();
-        branch.end = new Label();
-        branch.fals = branch.end;
-
-        final BlockContext blockctx = ctx.block();
-        final StatementMetadata blocksmd = metadata.getStatementMetadata(blockctx);
-
-        utility.pushJump(branch);
-        execute.mark(start);
-        writer.visit(blockctx);
-        execute.mark(branch.begin);
-        writer.visit(exprctx);
-        writeLoopCounter(blocksmd.count > 0 ? blocksmd.count : 1);
-        execute.goTo(start);
-        execute.mark(branch.end);
-        utility.popJump();
-    }
-
-    void processFor(final ForContext ctx) {
-        final ExpressionContext exprctx = ctx.expression();
-        final AfterthoughtContext atctx = ctx.afterthought();
-        final Branch branch = utility.markBranch(ctx, exprctx);
-        final Label start = new Label();
-        branch.begin = atctx == null ? start : new Label();
-        branch.end = new Label();
-        branch.fals = branch.end;
-
-        utility.pushJump(branch);
-
-        if (ctx.initializer() != null) {
-            writer.visit(ctx.initializer());
-        }
-
-        execute.mark(start);
-
-        if (exprctx != null) {
-            writer.visit(exprctx);
-        }
-
-        final BlockContext blockctx = ctx.block();
-        boolean allLast = false;
-
-        if (blockctx != null) {
-            StatementMetadata blocksmd = metadata.getStatementMetadata(blockctx);
-            allLast = blocksmd.allLast;
-
-            int count = blocksmd.count > 0 ? blocksmd.count : 1;
-
-            if (atctx != null) {
-                ++count;
-            }
-
-            writeLoopCounter(count);
-            writer.visit(blockctx);
-        } else if (ctx.empty() != null) {
-            writeLoopCounter(1);
-        } else {
-            throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
-        }
-
-        if (atctx != null) {
-            execute.mark(branch.begin);
-            writer.visit(atctx);
-        }
-
-        if (atctx != null || !allLast) {
-            execute.goTo(start);
-        }
-
-        execute.mark(branch.end);
-        utility.popJump();
-    }
-
-    void processDecl(final DeclContext ctx) {
-        writer.visit(ctx.declaration());
-    }
-
-    void processContinue() {
-        final Branch jump = utility.peekJump();
-        execute.goTo(jump.begin);
-    }
-
-    void processBreak() {
-        final Branch jump = utility.peekJump();
-        execute.goTo(jump.end);
-    }
-
-    void processReturn(final ReturnContext ctx) {
-        writer.visit(ctx.expression());
-        execute.returnValue();
-    }
-
-    void processTry(final TryContext ctx) {
-        final TrapContext[] trapctxs = new TrapContext[ctx.trap().size()];
-        ctx.trap().toArray(trapctxs);
-        final Branch branch = utility.markBranch(ctx, trapctxs);
-
-        Label end = new Label();
-        branch.begin = new Label();
-        branch.end = new Label();
-        branch.tru = trapctxs.length > 1 ? end : null;
-
-        execute.mark(branch.begin);
-
-        final BlockContext blockctx = ctx.block();
-        final StatementMetadata blocksmd = metadata.getStatementMetadata(blockctx);
-        writer.visit(blockctx);
-
-        if (!blocksmd.allLast) {
-            execute.goTo(end);
-        }
-
-        execute.mark(branch.end);
-
-        for (final TrapContext trapctx : trapctxs) {
-            writer.visit(trapctx);
-        }
-
-        if (!blocksmd.allLast || trapctxs.length > 1) {
-            execute.mark(end);
-        }
-    }
-
-    void processThrow(final ThrowContext ctx) {
-        writer.visit(ctx.expression());
-        execute.throwException();
-    }
-
-    void processExpr(final ExprContext ctx) {
-        final StatementMetadata exprsmd = metadata.getStatementMetadata(ctx);
-        final ExpressionContext exprctx = ctx.expression();
-        final ExpressionMetadata expremd = metadata.getExpressionMetadata(exprctx);
-        writer.visit(exprctx);
-
-        if (exprsmd.methodEscape) {
-            execute.returnValue();
-        } else {
-            utility.writePop(expremd.to.type.getSize());
-        }
-    }
-
-    void processMultiple(final MultipleContext ctx) {
-        for (final StatementContext sctx : ctx.statement()) {
-            writer.visit(sctx);
-        }
-    }
-
-    void processSingle(final SingleContext ctx) {
-        writer.visit(ctx.statement());
-    }
-
-    void processInitializer(InitializerContext ctx) {
-        final DeclarationContext declctx = ctx.declaration();
-        final ExpressionContext exprctx = ctx.expression();
-
-        if (declctx != null) {
-            writer.visit(declctx);
-        } else if (exprctx != null) {
-            final ExpressionMetadata expremd = metadata.getExpressionMetadata(exprctx);
-            writer.visit(exprctx);
-            utility.writePop(expremd.to.type.getSize());
-        } else {
-            throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
-        }
-    }
-
-    void processAfterthought(AfterthoughtContext ctx) {
-        final ExpressionContext exprctx = ctx.expression();
-        final ExpressionMetadata expremd = metadata.getExpressionMetadata(exprctx);
-        writer.visit(ctx.expression());
-        utility.writePop(expremd.to.type.getSize());
-    }
-
-    void processDeclaration(DeclarationContext ctx) {
-        for (final DeclvarContext declctx : ctx.declvar()) {
-            writer.visit(declctx);
-        }
-    }
-
-    void processDeclvar(final DeclvarContext ctx) {
-        final ExpressionMetadata declvaremd = metadata.getExpressionMetadata(ctx);
-        final org.objectweb.asm.Type type = declvaremd.to.type;
-        final Sort sort = declvaremd.to.sort;
-        int slot = (int)declvaremd.postConst;
-
-        final ExpressionContext exprctx = ctx.expression();
-        final boolean initialize = exprctx == null;
-
-        if (!initialize) {
-            writer.visit(exprctx);
-        }
-
-        switch (sort) {
-            case VOID:   throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
-            case BOOL:
-            case BYTE:
-            case SHORT:
-            case CHAR:
-            case INT:    if (initialize) execute.push(0);    break;
-            case LONG:   if (initialize) execute.push(0L);   break;
-            case FLOAT:  if (initialize) execute.push(0.0F); break;
-            case DOUBLE: if (initialize) execute.push(0.0);  break;
-            default:     if (initialize) execute.visitInsn(Opcodes.ACONST_NULL);
-        }
-
-        execute.visitVarInsn(type.getOpcode(Opcodes.ISTORE), slot);
-    }
-
-    void processTrap(final TrapContext ctx) {
-        final StatementMetadata trapsmd = metadata.getStatementMetadata(ctx);
-
-        final Branch branch = utility.getBranch(ctx);
-        final Label jump = new Label();
-
-        final BlockContext blockctx = ctx.block();
-        final EmptyscopeContext emptyctx = ctx.emptyscope();
-
-        execute.mark(jump);
-        execute.visitVarInsn(trapsmd.exception.type.getOpcode(Opcodes.ISTORE), trapsmd.slot);
-
-        if (blockctx != null) {
-            writer.visit(ctx.block());
-        } else if (emptyctx == null) {
-            throw new IllegalStateException(WriterUtility.error(ctx) + "Unexpected state.");
-        }
-
-        execute.visitTryCatchBlock(branch.begin, branch.end, jump, trapsmd.exception.type.getInternalName());
-
-        if (branch.tru != null && !trapsmd.allLast) {
-            execute.goTo(branch.tru);
-        }
-    }
-
-    private void writeLoopCounter(final int count) {
-        final Label end = new Label();
-
-        execute.iinc(metadata.loopCounterSlot, -count);
-        execute.visitVarInsn(Opcodes.ILOAD, metadata.loopCounterSlot);
-        execute.push(0);
-        execute.ifICmp(GeneratorAdapter.GT, end);
-        execute.throwException(PAINLESS_ERROR_TYPE,
-            "The maximum number of statements that can be executed in a loop has been reached.");
-        execute.mark(end);
-    }
-}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterUtility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterUtility.java
index d7e4f4ce42c..bb0b303764d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterUtility.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterUtility.java
@@ -19,31 +19,14 @@
 
 package org.elasticsearch.painless;
 
-import org.antlr.v4.runtime.ParserRuleContext;
+import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Definition.Transform;
 import org.elasticsearch.painless.Definition.Type;
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.commons.GeneratorAdapter;
 
-import java.util.ArrayDeque;
-import java.util.Deque;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import static org.elasticsearch.painless.PainlessParser.ADD;
-import static org.elasticsearch.painless.PainlessParser.BWAND;
-import static org.elasticsearch.painless.PainlessParser.BWOR;
-import static org.elasticsearch.painless.PainlessParser.BWXOR;
-import static org.elasticsearch.painless.PainlessParser.DIV;
-import static org.elasticsearch.painless.PainlessParser.LSH;
-import static org.elasticsearch.painless.PainlessParser.MUL;
-import static org.elasticsearch.painless.PainlessParser.REM;
-import static org.elasticsearch.painless.PainlessParser.RSH;
-import static org.elasticsearch.painless.PainlessParser.SUB;
-import static org.elasticsearch.painless.PainlessParser.USH;
 import static org.elasticsearch.painless.WriterConstants.ADDEXACT_INT;
 import static org.elasticsearch.painless.WriterConstants.ADDEXACT_LONG;
 import static org.elasticsearch.painless.WriterConstants.ADDWOOVERLOW_DOUBLE;
@@ -67,6 +50,7 @@ import static org.elasticsearch.painless.WriterConstants.MULEXACT_INT;
 import static org.elasticsearch.painless.WriterConstants.MULEXACT_LONG;
 import static org.elasticsearch.painless.WriterConstants.MULWOOVERLOW_DOUBLE;
 import static org.elasticsearch.painless.WriterConstants.MULWOOVERLOW_FLOAT;
+import static org.elasticsearch.painless.WriterConstants.PAINLESS_ERROR_TYPE;
 import static org.elasticsearch.painless.WriterConstants.REMWOOVERLOW_DOUBLE;
 import static org.elasticsearch.painless.WriterConstants.REMWOOVERLOW_FLOAT;
 import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_BOOLEAN;
@@ -84,304 +68,369 @@ import static org.elasticsearch.painless.WriterConstants.SUBEXACT_INT;
 import static org.elasticsearch.painless.WriterConstants.SUBEXACT_LONG;
 import static org.elasticsearch.painless.WriterConstants.SUBWOOVERLOW_DOUBLE;
 import static org.elasticsearch.painless.WriterConstants.SUBWOOVERLOW_FLOAT;
+import static org.elasticsearch.painless.WriterConstants.TOBYTEEXACT_INT;
+import static org.elasticsearch.painless.WriterConstants.TOBYTEEXACT_LONG;
+import static org.elasticsearch.painless.WriterConstants.TOBYTEWOOVERFLOW_DOUBLE;
+import static org.elasticsearch.painless.WriterConstants.TOBYTEWOOVERFLOW_FLOAT;
+import static org.elasticsearch.painless.WriterConstants.TOCHAREXACT_INT;
+import static org.elasticsearch.painless.WriterConstants.TOCHAREXACT_LONG;
+import static org.elasticsearch.painless.WriterConstants.TOCHARWOOVERFLOW_DOUBLE;
+import static org.elasticsearch.painless.WriterConstants.TOCHARWOOVERFLOW_FLOAT;
+import static org.elasticsearch.painless.WriterConstants.TOFLOATWOOVERFLOW_DOUBLE;
+import static org.elasticsearch.painless.WriterConstants.TOINTEXACT_LONG;
+import static org.elasticsearch.painless.WriterConstants.TOINTWOOVERFLOW_DOUBLE;
+import static org.elasticsearch.painless.WriterConstants.TOINTWOOVERFLOW_FLOAT;
+import static org.elasticsearch.painless.WriterConstants.TOLONGWOOVERFLOW_DOUBLE;
+import static org.elasticsearch.painless.WriterConstants.TOLONGWOOVERFLOW_FLOAT;
+import static org.elasticsearch.painless.WriterConstants.TOSHORTEXACT_INT;
+import static org.elasticsearch.painless.WriterConstants.TOSHORTEXACT_LONG;
+import static org.elasticsearch.painless.WriterConstants.TOSHORTWOOVERFLOW_DOUBLE;
+import static org.elasticsearch.painless.WriterConstants.TOSHORTWOOVERFLOW_FLOAT;
 
-class WriterUtility {
-    static class Branch {
-        final ParserRuleContext source;
+/**
+ * Set of methods used during the writing phase of compilation
+ * shared by the nodes of the Painless tree.
+ */
+public final class WriterUtility {
 
-        Label begin = null;
-        Label end = null;
-        Label tru = null;
-        Label fals = null;
+    public static void writeLoopCounter(final GeneratorAdapter adapter, final int slot, final int count) {
+        if (slot > -1) {
+            final Label end = new Label();
 
-        private Branch(final ParserRuleContext source) {
-            this.source = source;
+            adapter.iinc(slot, -count);
+            adapter.visitVarInsn(Opcodes.ILOAD, slot);
+            adapter.push(0);
+            adapter.ifICmp(GeneratorAdapter.GT, end);
+            adapter.throwException(PAINLESS_ERROR_TYPE,
+                "The maximum number of statements that can be executed in a loop has been reached.");
+            adapter.mark(end);
         }
     }
 
-    /**
-     * A utility method to output consistent error messages.
-     * @param ctx The ANTLR node the error occurred in.
-     * @return The error message with tacked on line number and character position.
-     */
-    static String error(final ParserRuleContext ctx) {
-        return "Writer Error [" + ctx.getStart().getLine() + ":" + ctx.getStart().getCharPositionInLine() + "]: ";
-    }
+    public static void writeCast(final GeneratorAdapter adapter, final Cast cast) {
+        if (cast instanceof Transform) {
+            final Transform transform = (Transform)cast;
 
-    private final Definition definition;
-    private final CompilerSettings settings;
-
-    private final GeneratorAdapter execute;
-
-    private final Map<ParserRuleContext, Branch> branches = new HashMap<>();
-    private final Deque<Branch> jumps = new ArrayDeque<>();
-    private final Set<ParserRuleContext> strings = new HashSet<>();
-
-    WriterUtility(final Metadata metadata, final GeneratorAdapter execute) {
-        definition = metadata.definition;
-        settings = metadata.settings;
-
-        this.execute = execute;
-    }
-
-    Branch markBranch(final ParserRuleContext source, final ParserRuleContext... nodes) {
-        final Branch branch = new Branch(source);
-
-        for (final ParserRuleContext node : nodes) {
-            branches.put(node, branch);
-        }
-
-        return branch;
-    }
-
-    void copyBranch(final Branch branch, final ParserRuleContext... nodes) {
-        for (final ParserRuleContext node : nodes) {
-            branches.put(node, branch);
-        }
-    }
-
-    Branch getBranch(final ParserRuleContext source) {
-        return branches.get(source);
-    }
-
-    void checkWriteBranch(final ParserRuleContext source) {
-        final Branch branch = getBranch(source);
-
-        if (branch != null) {
-            if (branch.tru != null) {
-                execute.visitJumpInsn(Opcodes.IFNE, branch.tru);
-            } else if (branch.fals != null) {
-                execute.visitJumpInsn(Opcodes.IFEQ, branch.fals);
+            if (transform.upcast != null) {
+                adapter.checkCast(transform.upcast.type);
             }
-        }
-    }
 
-    void pushJump(final Branch branch) {
-        jumps.push(branch);
-    }
-
-    Branch peekJump() {
-        return jumps.peek();
-    }
-
-    void popJump() {
-        jumps.pop();
-    }
-
-    void addStrings(final ParserRuleContext source) {
-        strings.add(source);
-    }
-
-    boolean containsStrings(final ParserRuleContext source) {
-        return strings.contains(source);
-    }
-
-    void removeStrings(final ParserRuleContext source) {
-        strings.remove(source);
-    }
-
-    void writeDup(final int size, final boolean x1, final boolean x2) {
-        if (size == 1) {
-            if (x2) {
-                execute.dupX2();
-            } else if (x1) {
-                execute.dupX1();
+            if (java.lang.reflect.Modifier.isStatic(transform.method.reflect.getModifiers())) {
+                adapter.invokeStatic(transform.method.owner.type, transform.method.method);
+            } else if (java.lang.reflect.Modifier.isInterface(transform.method.owner.clazz.getModifiers())) {
+                adapter.invokeInterface(transform.method.owner.type, transform.method.method);
             } else {
-                execute.dup();
+                adapter.invokeVirtual(transform.method.owner.type, transform.method.method);
             }
-        } else if (size == 2) {
-            if (x2) {
-                execute.dup2X2();
-            } else if (x1) {
-                execute.dup2X1();
+
+            if (transform.downcast != null) {
+                adapter.checkCast(transform.downcast.type);
+            }
+        } else if (cast != null) {
+            final Type from = cast.from;
+            final Type to = cast.to;
+
+            if (from.equals(to)) {
+                return;
+            }
+
+            if (from.sort.numeric && from.sort.primitive && to.sort.numeric && to.sort.primitive) {
+                adapter.cast(from.type, to.type);
             } else {
-                execute.dup2();
+                try {
+                    from.clazz.asSubclass(to.clazz);
+                } catch (ClassCastException exception) {
+                    adapter.checkCast(to.type);
+                }
             }
         }
     }
 
-    void writePop(final int size) {
-        if (size == 1) {
-            execute.pop();
-        } else if (size == 2) {
-            execute.pop2();
+    public static void writeBranch(final GeneratorAdapter adapter, final Label tru, final Label fals) {
+        if (tru != null) {
+            adapter.visitJumpInsn(Opcodes.IFNE, tru);
+        } else if (fals != null) {
+            adapter.visitJumpInsn(Opcodes.IFEQ, fals);
         }
     }
 
-    void writeConstant(final ParserRuleContext source, final Object constant) {
-        if (constant instanceof Number) {
-            writeNumeric(source, constant);
-        } else if (constant instanceof Character) {
-            writeNumeric(source, (int)(char)constant);
-        } else if (constant instanceof String) {
-            writeString(source, constant);
-        } else if (constant instanceof Boolean) {
-            writeBoolean(source, constant);
-        } else if (constant != null) {
-            throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
-        }
+    public static void writeNewStrings(final GeneratorAdapter adapter) {
+        adapter.newInstance(STRINGBUILDER_TYPE);
+        adapter.dup();
+        adapter.invokeConstructor(STRINGBUILDER_TYPE, STRINGBUILDER_CONSTRUCTOR);
     }
 
-    void writeNumeric(final ParserRuleContext source, final Object numeric) {
-        if (numeric instanceof Double) {
-            execute.push((double)numeric);
-        } else if (numeric instanceof Float) {
-            execute.push((float)numeric);
-        } else if (numeric instanceof Long) {
-            execute.push((long)numeric);
-        } else if (numeric instanceof Number) {
-            execute.push(((Number)numeric).intValue());
-        } else {
-            throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
-        }
-    }
-
-    void writeString(final ParserRuleContext source, final Object string) {
-        if (string instanceof String) {
-            execute.push((String)string);
-        } else {
-            throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
-        }
-    }
-
-    void writeBoolean(final ParserRuleContext source, final Object bool) {
-        if (bool instanceof Boolean) {
-            execute.push((boolean)bool);
-        } else {
-            throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
-        }
-    }
-
-    void writeNewStrings() {
-        execute.newInstance(STRINGBUILDER_TYPE);
-        execute.dup();
-        execute.invokeConstructor(STRINGBUILDER_TYPE, STRINGBUILDER_CONSTRUCTOR);
-    }
-
-    void writeAppendStrings(final Sort sort) {
+    public static void writeAppendStrings(final GeneratorAdapter adapter, final Sort sort) {
         switch (sort) {
-            case BOOL:   execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_BOOLEAN); break;
-            case CHAR:   execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_CHAR);    break;
+            case BOOL:   adapter.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_BOOLEAN); break;
+            case CHAR:   adapter.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_CHAR);    break;
             case BYTE:
             case SHORT:
-            case INT:    execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_INT);     break;
-            case LONG:   execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_LONG);    break;
-            case FLOAT:  execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_FLOAT);   break;
-            case DOUBLE: execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_DOUBLE);  break;
-            case STRING: execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_STRING);  break;
-            default:     execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_OBJECT);
+            case INT:    adapter.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_INT);     break;
+            case LONG:   adapter.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_LONG);    break;
+            case FLOAT:  adapter.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_FLOAT);   break;
+            case DOUBLE: adapter.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_DOUBLE);  break;
+            case STRING: adapter.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_STRING);  break;
+            default:     adapter.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_OBJECT);
         }
     }
 
-    void writeToStrings() {
-        execute.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_TOSTRING);
+    public static void writeToStrings(final GeneratorAdapter adapter) {
+        adapter.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_TOSTRING);
     }
 
-    void writeBinaryInstruction(final ParserRuleContext source, final Type type, final int token) {
+    public static void writeBinaryInstruction(final CompilerSettings settings, final Definition definition,
+                                              final GeneratorAdapter adapter, final String location,
+                                              final Type type, final Operation operation) {
         final Sort sort = type.sort;
-        final boolean exact = !settings.getNumericOverflow() &&
+        boolean exact = !settings.getNumericOverflow() &&
             ((sort == Sort.INT || sort == Sort.LONG) &&
-                (token == MUL || token == DIV || token == ADD || token == SUB) ||
+                (operation == Operation.MUL || operation == Operation.DIV ||
+                    operation == Operation.ADD || operation == Operation.SUB) ||
                 (sort == Sort.FLOAT || sort == Sort.DOUBLE) &&
-                    (token == MUL || token == DIV || token == REM || token == ADD || token == SUB));
-
-        // If it's a 64-bit shift, fix-up the last argument to truncate to 32-bits.
-        // Note that unlike java, this means we still do binary promotion of shifts,
-        // but it keeps things simple, and this check works because we promote shifts.
-        if (sort == Sort.LONG && (token == LSH || token == USH || token == RSH)) {
-            execute.cast(org.objectweb.asm.Type.LONG_TYPE, org.objectweb.asm.Type.INT_TYPE);
-        }
+                    (operation == Operation.MUL || operation == Operation.DIV || operation == Operation.REM ||
+                        operation == Operation.ADD || operation == Operation.SUB));
 
         if (exact) {
             switch (sort) {
                 case INT:
-                    switch (token) {
-                        case MUL: execute.invokeStatic(definition.mathType.type,    MULEXACT_INT);     break;
-                        case DIV: execute.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_INT); break;
-                        case ADD: execute.invokeStatic(definition.mathType.type,    ADDEXACT_INT);     break;
-                        case SUB: execute.invokeStatic(definition.mathType.type,    SUBEXACT_INT);     break;
-                        default:
-                            throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
+                    switch (operation) {
+                        case MUL: adapter.invokeStatic(definition.mathType.type,    MULEXACT_INT);     break;
+                        case DIV: adapter.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_INT); break;
+                        case ADD: adapter.invokeStatic(definition.mathType.type,    ADDEXACT_INT);     break;
+                        case SUB: adapter.invokeStatic(definition.mathType.type,    SUBEXACT_INT);     break;
                     }
 
                     break;
                 case LONG:
-                    switch (token) {
-                        case MUL: execute.invokeStatic(definition.mathType.type,    MULEXACT_LONG);     break;
-                        case DIV: execute.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_LONG); break;
-                        case ADD: execute.invokeStatic(definition.mathType.type,    ADDEXACT_LONG);     break;
-                        case SUB: execute.invokeStatic(definition.mathType.type,    SUBEXACT_LONG);     break;
-                        default:
-                            throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
+                    switch (operation) {
+                        case MUL: adapter.invokeStatic(definition.mathType.type,    MULEXACT_LONG);     break;
+                        case DIV: adapter.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_LONG); break;
+                        case ADD: adapter.invokeStatic(definition.mathType.type,    ADDEXACT_LONG);     break;
+                        case SUB: adapter.invokeStatic(definition.mathType.type,    SUBEXACT_LONG);     break;
                     }
 
                     break;
                 case FLOAT:
-                    switch (token) {
-                        case MUL: execute.invokeStatic(definition.utilityType.type, MULWOOVERLOW_FLOAT); break;
-                        case DIV: execute.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_FLOAT); break;
-                        case REM: execute.invokeStatic(definition.utilityType.type, REMWOOVERLOW_FLOAT); break;
-                        case ADD: execute.invokeStatic(definition.utilityType.type, ADDWOOVERLOW_FLOAT); break;
-                        case SUB: execute.invokeStatic(definition.utilityType.type, SUBWOOVERLOW_FLOAT); break;
+                    switch (operation) {
+                        case MUL: adapter.invokeStatic(definition.utilityType.type, MULWOOVERLOW_FLOAT); break;
+                        case DIV: adapter.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_FLOAT); break;
+                        case REM: adapter.invokeStatic(definition.utilityType.type, REMWOOVERLOW_FLOAT); break;
+                        case ADD: adapter.invokeStatic(definition.utilityType.type, ADDWOOVERLOW_FLOAT); break;
+                        case SUB: adapter.invokeStatic(definition.utilityType.type, SUBWOOVERLOW_FLOAT); break;
                         default:
-                            throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
+                            throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
                     }
 
                     break;
                 case DOUBLE:
-                    switch (token) {
-                        case MUL: execute.invokeStatic(definition.utilityType.type, MULWOOVERLOW_DOUBLE); break;
-                        case DIV: execute.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_DOUBLE); break;
-                        case REM: execute.invokeStatic(definition.utilityType.type, REMWOOVERLOW_DOUBLE); break;
-                        case ADD: execute.invokeStatic(definition.utilityType.type, ADDWOOVERLOW_DOUBLE); break;
-                        case SUB: execute.invokeStatic(definition.utilityType.type, SUBWOOVERLOW_DOUBLE); break;
+                    switch (operation) {
+                        case MUL: adapter.invokeStatic(definition.utilityType.type, MULWOOVERLOW_DOUBLE); break;
+                        case DIV: adapter.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_DOUBLE); break;
+                        case REM: adapter.invokeStatic(definition.utilityType.type, REMWOOVERLOW_DOUBLE); break;
+                        case ADD: adapter.invokeStatic(definition.utilityType.type, ADDWOOVERLOW_DOUBLE); break;
+                        case SUB: adapter.invokeStatic(definition.utilityType.type, SUBWOOVERLOW_DOUBLE); break;
                         default:
-                            throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
+                            throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
                     }
 
                     break;
                 default:
-                    throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
+                    throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
             }
         } else {
             if ((sort == Sort.FLOAT || sort == Sort.DOUBLE) &&
-                (token == LSH || token == USH || token == RSH || token == BWAND || token == BWXOR || token == BWOR)) {
-                throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
+                (operation == Operation.LSH || operation == Operation.USH ||
+                    operation == Operation.RSH || operation == Operation.BWAND ||
+                    operation == Operation.XOR || operation == Operation.BWOR)) {
+                throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
             }
 
             if (sort == Sort.DEF) {
-                switch (token) {
-                    case MUL:   execute.invokeStatic(definition.defobjType.type, DEF_MUL_CALL); break;
-                    case DIV:   execute.invokeStatic(definition.defobjType.type, DEF_DIV_CALL); break;
-                    case REM:   execute.invokeStatic(definition.defobjType.type, DEF_REM_CALL); break;
-                    case ADD:   execute.invokeStatic(definition.defobjType.type, DEF_ADD_CALL); break;
-                    case SUB:   execute.invokeStatic(definition.defobjType.type, DEF_SUB_CALL); break;
-                    case LSH:   execute.invokeStatic(definition.defobjType.type, DEF_LSH_CALL); break;
-                    case USH:   execute.invokeStatic(definition.defobjType.type, DEF_RSH_CALL); break;
-                    case RSH:   execute.invokeStatic(definition.defobjType.type, DEF_USH_CALL); break;
-                    case BWAND: execute.invokeStatic(definition.defobjType.type, DEF_AND_CALL); break;
-                    case BWXOR: execute.invokeStatic(definition.defobjType.type, DEF_XOR_CALL); break;
-                    case BWOR:  execute.invokeStatic(definition.defobjType.type, DEF_OR_CALL);  break;
+                switch (operation) {
+                    case MUL:   adapter.invokeStatic(definition.defobjType.type, DEF_MUL_CALL); break;
+                    case DIV:   adapter.invokeStatic(definition.defobjType.type, DEF_DIV_CALL); break;
+                    case REM:   adapter.invokeStatic(definition.defobjType.type, DEF_REM_CALL); break;
+                    case ADD:   adapter.invokeStatic(definition.defobjType.type, DEF_ADD_CALL); break;
+                    case SUB:   adapter.invokeStatic(definition.defobjType.type, DEF_SUB_CALL); break;
+                    case LSH:   adapter.invokeStatic(definition.defobjType.type, DEF_LSH_CALL); break;
+                    case USH:   adapter.invokeStatic(definition.defobjType.type, DEF_RSH_CALL); break;
+                    case RSH:   adapter.invokeStatic(definition.defobjType.type, DEF_USH_CALL); break;
+                    case BWAND: adapter.invokeStatic(definition.defobjType.type, DEF_AND_CALL); break;
+                    case XOR:   adapter.invokeStatic(definition.defobjType.type, DEF_XOR_CALL); break;
+                    case BWOR:  adapter.invokeStatic(definition.defobjType.type, DEF_OR_CALL);  break;
                     default:
-                        throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
+                        throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
                 }
             } else {
-                switch (token) {
-                    case MUL:   execute.math(GeneratorAdapter.MUL,  type.type); break;
-                    case DIV:   execute.math(GeneratorAdapter.DIV,  type.type); break;
-                    case REM:   execute.math(GeneratorAdapter.REM,  type.type); break;
-                    case ADD:   execute.math(GeneratorAdapter.ADD,  type.type); break;
-                    case SUB:   execute.math(GeneratorAdapter.SUB,  type.type); break;
-                    case LSH:   execute.math(GeneratorAdapter.SHL,  type.type); break;
-                    case USH:   execute.math(GeneratorAdapter.USHR, type.type); break;
-                    case RSH:   execute.math(GeneratorAdapter.SHR,  type.type); break;
-                    case BWAND: execute.math(GeneratorAdapter.AND,  type.type); break;
-                    case BWXOR: execute.math(GeneratorAdapter.XOR,  type.type); break;
-                    case BWOR:  execute.math(GeneratorAdapter.OR,   type.type); break;
+                switch (operation) {
+                    case MUL:   adapter.math(GeneratorAdapter.MUL,  type.type); break;
+                    case DIV:   adapter.math(GeneratorAdapter.DIV,  type.type); break;
+                    case REM:   adapter.math(GeneratorAdapter.REM,  type.type); break;
+                    case ADD:   adapter.math(GeneratorAdapter.ADD,  type.type); break;
+                    case SUB:   adapter.math(GeneratorAdapter.SUB,  type.type); break;
+                    case LSH:   adapter.math(GeneratorAdapter.SHL,  type.type); break;
+                    case USH:   adapter.math(GeneratorAdapter.USHR, type.type); break;
+                    case RSH:   adapter.math(GeneratorAdapter.SHR,  type.type); break;
+                    case BWAND: adapter.math(GeneratorAdapter.AND,  type.type); break;
+                    case XOR:   adapter.math(GeneratorAdapter.XOR,  type.type); break;
+                    case BWOR:  adapter.math(GeneratorAdapter.OR,   type.type); break;
                     default:
-                        throw new IllegalStateException(WriterUtility.error(source) + "Unexpected state.");
+                        throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
                 }
             }
         }
     }
+
+    /**
+     * Called for any compound assignment (including increment/decrement instructions).
+     * We have to be stricter than writeBinary and do overflow checks against the original type's size
+     * instead of the promoted type's size, since the result will be implicitly cast back.
+     *
+     * @return This will be true if an instruction is written, false otherwise.
+     */
+    public static boolean writeExactInstruction(
+        final Definition definition, final GeneratorAdapter adapter, final Sort fsort, final Sort tsort) {
+        if (fsort == Sort.DOUBLE) {
+            if (tsort == Sort.FLOAT) {
+                adapter.invokeStatic(definition.utilityType.type, TOFLOATWOOVERFLOW_DOUBLE);
+            } else if (tsort == Sort.FLOAT_OBJ) {
+                adapter.invokeStatic(definition.utilityType.type, TOFLOATWOOVERFLOW_DOUBLE);
+                adapter.checkCast(definition.floatobjType.type);
+            } else if (tsort == Sort.LONG) {
+                adapter.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_DOUBLE);
+            } else if (tsort == Sort.LONG_OBJ) {
+                adapter.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_DOUBLE);
+                adapter.checkCast(definition.longobjType.type);
+            } else if (tsort == Sort.INT) {
+                adapter.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_DOUBLE);
+            } else if (tsort == Sort.INT_OBJ) {
+                adapter.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_DOUBLE);
+                adapter.checkCast(definition.intobjType.type);
+            } else if (tsort == Sort.CHAR) {
+                adapter.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_DOUBLE);
+            } else if (tsort == Sort.CHAR_OBJ) {
+                adapter.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_DOUBLE);
+                adapter.checkCast(definition.charobjType.type);
+            } else if (tsort == Sort.SHORT) {
+                adapter.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_DOUBLE);
+            } else if (tsort == Sort.SHORT_OBJ) {
+                adapter.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_DOUBLE);
+                adapter.checkCast(definition.shortobjType.type);
+            } else if (tsort == Sort.BYTE) {
+                adapter.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_DOUBLE);
+            } else if (tsort == Sort.BYTE_OBJ) {
+                adapter.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_DOUBLE);
+                adapter.checkCast(definition.byteobjType.type);
+            } else {
+                return false;
+            }
+        } else if (fsort == Sort.FLOAT) {
+            if (tsort == Sort.LONG) {
+                adapter.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_FLOAT);
+            } else if (tsort == Sort.LONG_OBJ) {
+                adapter.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_FLOAT);
+                adapter.checkCast(definition.longobjType.type);
+            } else if (tsort == Sort.INT) {
+                adapter.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_FLOAT);
+            } else if (tsort == Sort.INT_OBJ) {
+                adapter.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_FLOAT);
+                adapter.checkCast(definition.intobjType.type);
+            } else if (tsort == Sort.CHAR) {
+                adapter.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_FLOAT);
+            } else if (tsort == Sort.CHAR_OBJ) {
+                adapter.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_FLOAT);
+                adapter.checkCast(definition.charobjType.type);
+            } else if (tsort == Sort.SHORT) {
+                adapter.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_FLOAT);
+            } else if (tsort == Sort.SHORT_OBJ) {
+                adapter.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_FLOAT);
+                adapter.checkCast(definition.shortobjType.type);
+            } else if (tsort == Sort.BYTE) {
+                adapter.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_FLOAT);
+            } else if (tsort == Sort.BYTE_OBJ) {
+                adapter.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_FLOAT);
+                adapter.checkCast(definition.byteobjType.type);
+            } else {
+                return false;
+            }
+        } else if (fsort == Sort.LONG) {
+            if (tsort == Sort.INT) {
+                adapter.invokeStatic(definition.mathType.type, TOINTEXACT_LONG);
+            } else if (tsort == Sort.INT_OBJ) {
+                adapter.invokeStatic(definition.mathType.type, TOINTEXACT_LONG);
+                adapter.checkCast(definition.intobjType.type);
+            } else if (tsort == Sort.CHAR) {
+                adapter.invokeStatic(definition.utilityType.type, TOCHAREXACT_LONG);
+            } else if (tsort == Sort.CHAR_OBJ) {
+                adapter.invokeStatic(definition.utilityType.type, TOCHAREXACT_LONG);
+                adapter.checkCast(definition.charobjType.type);
+            } else if (tsort == Sort.SHORT) {
+                adapter.invokeStatic(definition.utilityType.type, TOSHORTEXACT_LONG);
+            } else if (tsort == Sort.SHORT_OBJ) {
+                adapter.invokeStatic(definition.utilityType.type, TOSHORTEXACT_LONG);
+                adapter.checkCast(definition.shortobjType.type);
+            } else if (tsort == Sort.BYTE) {
+                adapter.invokeStatic(definition.utilityType.type, TOBYTEEXACT_LONG);
+            } else if (tsort == Sort.BYTE_OBJ) {
+                adapter.invokeStatic(definition.utilityType.type, TOBYTEEXACT_LONG);
+                adapter.checkCast(definition.byteobjType.type);
+            } else {
+                return false;
+            }
+        } else if (fsort == Sort.INT) {
+            if (tsort == Sort.CHAR) {
+                adapter.invokeStatic(definition.utilityType.type, TOCHAREXACT_INT);
+            } else if (tsort == Sort.CHAR_OBJ) {
+                adapter.invokeStatic(definition.utilityType.type, TOCHAREXACT_INT);
+                adapter.checkCast(definition.charobjType.type);
+            } else if (tsort == Sort.SHORT) {
+                adapter.invokeStatic(definition.utilityType.type, TOSHORTEXACT_INT);
+            } else if (tsort == Sort.SHORT_OBJ) {
+                adapter.invokeStatic(definition.utilityType.type, TOSHORTEXACT_INT);
+                adapter.checkCast(definition.shortobjType.type);
+            } else if (tsort == Sort.BYTE) {
+                adapter.invokeStatic(definition.utilityType.type, TOBYTEEXACT_INT);
+            } else if (tsort == Sort.BYTE_OBJ) {
+                adapter.invokeStatic(definition.utilityType.type, TOBYTEEXACT_INT);
+                adapter.checkCast(definition.byteobjType.type);
+            } else {
+                return false;
+            }
+        } else {
+            return false;
+        }
+
+        return true;
+    }
+
+    public static void writeDup(final GeneratorAdapter adapter, final int size, final int xsize) {
+        if (size == 1) {
+            if (xsize == 2) {
+                adapter.dupX2();
+            } else if (xsize == 1) {
+                adapter.dupX1();
+            } else {
+                adapter.dup();
+            }
+        } else if (size == 2) {
+            if (xsize == 2) {
+                adapter.dup2X2();
+            } else if (xsize == 1) {
+                adapter.dup2X1();
+            } else {
+                adapter.dup2();
+            }
+        }
+    }
+
+    public static void writePop(final GeneratorAdapter adapter, final int size) {
+        if (size == 1) {
+            adapter.pop();
+        } else if (size == 2) {
+            adapter.pop2();
+        }
+    }
+
+    private WriterUtility() {}
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ErrorHandlingLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ErrorHandlingLexer.java
similarity index 67%
rename from modules/lang-painless/src/main/java/org/elasticsearch/painless/ErrorHandlingLexer.java
rename to modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ErrorHandlingLexer.java
index e42de385d75..25069a7e9d3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ErrorHandlingLexer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ErrorHandlingLexer.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.painless;
+package org.elasticsearch.painless.antlr;
 
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.LexerNoViableAltException;
@@ -25,21 +25,26 @@ import org.antlr.v4.runtime.misc.Interval;
 
 import java.text.ParseException;
 
-class ErrorHandlingLexer extends PainlessLexer {
-    public ErrorHandlingLexer(CharStream charStream) {
+/**
+ * A lexer that will override the default error behavior to fail on the first error.
+ */
+final class ErrorHandlingLexer extends PainlessLexer {
+
+    ErrorHandlingLexer(final CharStream charStream) {
         super(charStream);
     }
 
     @Override
-    public void recover(LexerNoViableAltException lnvae) {
-        CharStream charStream = lnvae.getInputStream();
-        int startIndex = lnvae.getStartIndex();
-        String text = charStream.getText(Interval.of(startIndex, charStream.index()));
+    public void recover(final LexerNoViableAltException lnvae) {
+        final CharStream charStream = lnvae.getInputStream();
+        final int startIndex = lnvae.getStartIndex();
+        final String text = charStream.getText(Interval.of(startIndex, charStream.index()));
 
-        ParseException parseException = new ParseException("Error [" + _tokenStartLine + ":" +
+        final ParseException parseException = new ParseException("Error [" + _tokenStartLine + ":" +
                 _tokenStartCharPositionInLine + "]: unexpected character [" +
                 getErrorDisplay(text) + "].",  _tokenStartCharIndex);
         parseException.initCause(lnvae);
+
         throw new RuntimeException(parseException);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
similarity index 96%
rename from modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessLexer.java
rename to modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
index 6951c67605a..8cb9a605868 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessLexer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
@@ -1,5 +1,5 @@
 // ANTLR GENERATED CODE: DO NOT EDIT
-package org.elasticsearch.painless;
+package org.elasticsearch.painless.antlr;
 import org.antlr.v4.runtime.Lexer;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.Token;
@@ -21,7 +21,7 @@ class PainlessLexer extends Lexer {
     COMMA=10, SEMICOLON=11, IF=12, ELSE=13, WHILE=14, DO=15, FOR=16, CONTINUE=17, 
     BREAK=18, RETURN=19, NEW=20, TRY=21, CATCH=22, THROW=23, BOOLNOT=24, BWNOT=25, 
     MUL=26, DIV=27, REM=28, ADD=29, SUB=30, LSH=31, RSH=32, USH=33, LT=34, 
-    LTE=35, GT=36, GTE=37, EQ=38, EQR=39, NE=40, NER=41, BWAND=42, BWXOR=43, 
+    LTE=35, GT=36, GTE=37, EQ=38, EQR=39, NE=40, NER=41, BWAND=42, XOR=43, 
     BWOR=44, BOOLAND=45, BOOLOR=46, COND=47, COLON=48, INCR=49, DECR=50, ASSIGN=51, 
     AADD=52, ASUB=53, AMUL=54, ADIV=55, AREM=56, AAND=57, AXOR=58, AOR=59, 
     ALSH=60, ARSH=61, AUSH=62, OCTAL=63, HEX=64, INTEGER=65, DECIMAL=66, STRING=67, 
@@ -36,11 +36,11 @@ class PainlessLexer extends Lexer {
     "COMMA", "SEMICOLON", "IF", "ELSE", "WHILE", "DO", "FOR", "CONTINUE", 
     "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "BOOLNOT", "BWNOT", 
     "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", "LTE", "GT", 
-    "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "BWXOR", "BWOR", "BOOLAND", 
-    "BOOLOR", "COND", "COLON", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", 
-    "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", 
-    "HEX", "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "ID", 
-    "EXTINTEGER", "EXTID"
+    "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", "BOOLAND", "BOOLOR", 
+    "COND", "COLON", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", 
+    "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", 
+    "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "ID", "EXTINTEGER", 
+    "EXTID"
   };
 
   private static final String[] _LITERAL_NAMES = {
@@ -58,11 +58,11 @@ class PainlessLexer extends Lexer {
     "DOT", "COMMA", "SEMICOLON", "IF", "ELSE", "WHILE", "DO", "FOR", "CONTINUE", 
     "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "BOOLNOT", "BWNOT", 
     "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", "LTE", "GT", 
-    "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "BWXOR", "BWOR", "BOOLAND", 
-    "BOOLOR", "COND", "COLON", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", 
-    "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", 
-    "HEX", "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "ID", 
-    "EXTINTEGER", "EXTID"
+    "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", "BOOLAND", "BOOLOR", 
+    "COND", "COLON", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", 
+    "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", 
+    "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "ID", "EXTINTEGER", 
+    "EXTID"
   };
   public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
similarity index 76%
rename from modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessParser.java
rename to modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
index b92c9d75198..9d7ae5c628d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessParser.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
@@ -1,5 +1,5 @@
 // ANTLR GENERATED CODE: DO NOT EDIT
-package org.elasticsearch.painless;
+package org.elasticsearch.painless.antlr;
 import org.antlr.v4.runtime.atn.*;
 import org.antlr.v4.runtime.dfa.DFA;
 import org.antlr.v4.runtime.*;
@@ -21,7 +21,7 @@ class PainlessParser extends Parser {
     COMMA=10, SEMICOLON=11, IF=12, ELSE=13, WHILE=14, DO=15, FOR=16, CONTINUE=17, 
     BREAK=18, RETURN=19, NEW=20, TRY=21, CATCH=22, THROW=23, BOOLNOT=24, BWNOT=25, 
     MUL=26, DIV=27, REM=28, ADD=29, SUB=30, LSH=31, RSH=32, USH=33, LT=34, 
-    LTE=35, GT=36, GTE=37, EQ=38, EQR=39, NE=40, NER=41, BWAND=42, BWXOR=43, 
+    LTE=35, GT=36, GTE=37, EQ=38, EQR=39, NE=40, NER=41, BWAND=42, XOR=43, 
     BWOR=44, BOOLAND=45, BOOLOR=46, COND=47, COLON=48, INCR=49, DECR=50, ASSIGN=51, 
     AADD=52, ASUB=53, AMUL=54, ADIV=55, AREM=56, AAND=57, AXOR=58, AOR=59, 
     ALSH=60, ARSH=61, AUSH=62, OCTAL=63, HEX=64, INTEGER=65, DECIMAL=66, STRING=67, 
@@ -30,16 +30,15 @@ class PainlessParser extends Parser {
     RULE_source = 0, RULE_statement = 1, RULE_block = 2, RULE_empty = 3, RULE_emptyscope = 4, 
     RULE_initializer = 5, RULE_afterthought = 6, RULE_declaration = 7, RULE_decltype = 8, 
     RULE_declvar = 9, RULE_trap = 10, RULE_identifier = 11, RULE_generic = 12, 
-    RULE_expression = 13, RULE_extstart = 14, RULE_extprec = 15, RULE_extcast = 16, 
-    RULE_extbrace = 17, RULE_extdot = 18, RULE_extcall = 19, RULE_extvar = 20, 
-    RULE_extfield = 21, RULE_extnew = 22, RULE_extstring = 23, RULE_arguments = 24, 
-    RULE_increment = 25;
+    RULE_expression = 13, RULE_chain = 14, RULE_linkprec = 15, RULE_linkcast = 16, 
+    RULE_linkbrace = 17, RULE_linkdot = 18, RULE_linkcall = 19, RULE_linkvar = 20, 
+    RULE_linkfield = 21, RULE_linknew = 22, RULE_linkstring = 23, RULE_arguments = 24;
   public static final String[] ruleNames = {
     "source", "statement", "block", "empty", "emptyscope", "initializer", 
     "afterthought", "declaration", "decltype", "declvar", "trap", "identifier", 
-    "generic", "expression", "extstart", "extprec", "extcast", "extbrace", 
-    "extdot", "extcall", "extvar", "extfield", "extnew", "extstring", "arguments", 
-    "increment"
+    "generic", "expression", "chain", "linkprec", "linkcast", "linkbrace", 
+    "linkdot", "linkcall", "linkvar", "linkfield", "linknew", "linkstring", 
+    "arguments"
   };
 
   private static final String[] _LITERAL_NAMES = {
@@ -57,11 +56,11 @@ class PainlessParser extends Parser {
     "DOT", "COMMA", "SEMICOLON", "IF", "ELSE", "WHILE", "DO", "FOR", "CONTINUE", 
     "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "BOOLNOT", "BWNOT", 
     "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", "LTE", "GT", 
-    "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "BWXOR", "BWOR", "BOOLAND", 
-    "BOOLOR", "COND", "COLON", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", 
-    "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", 
-    "HEX", "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "ID", 
-    "EXTINTEGER", "EXTID"
+    "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", "BOOLAND", "BOOLOR", 
+    "COND", "COLON", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", 
+    "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", 
+    "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "ID", "EXTINTEGER", 
+    "EXTID"
   };
   public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
 
@@ -138,21 +137,21 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(53); 
+      setState(51); 
       _errHandler.sync(this);
       _la = _input.LA(1);
       do {
         {
         {
-        setState(52);
+        setState(50);
         statement();
         }
         }
-        setState(55); 
+        setState(53); 
         _errHandler.sync(this);
         _la = _input.LA(1);
       } while ( (((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << IF) | (1L << WHILE) | (1L << DO) | (1L << FOR) | (1L << CONTINUE) | (1L << BREAK) | (1L << RETURN) | (1L << NEW) | (1L << TRY) | (1L << THROW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0) );
-      setState(57);
+      setState(55);
       match(EOF);
       }
     }
@@ -363,29 +362,29 @@ class PainlessParser extends Parser {
     int _la;
     try {
       int _alt;
-      setState(136);
+      setState(134);
       switch ( getInterpreter().adaptivePredict(_input,15,_ctx) ) {
       case 1:
         _localctx = new IfContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(59);
+        setState(57);
         match(IF);
-        setState(60);
+        setState(58);
         match(LP);
-        setState(61);
+        setState(59);
         expression(0);
-        setState(62);
+        setState(60);
         match(RP);
-        setState(63);
+        setState(61);
         block();
-        setState(66);
+        setState(64);
         switch ( getInterpreter().adaptivePredict(_input,1,_ctx) ) {
         case 1:
           {
-          setState(64);
+          setState(62);
           match(ELSE);
-          setState(65);
+          setState(63);
           block();
           }
           break;
@@ -396,25 +395,25 @@ class PainlessParser extends Parser {
         _localctx = new WhileContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(68);
+        setState(66);
         match(WHILE);
-        setState(69);
+        setState(67);
         match(LP);
-        setState(70);
+        setState(68);
         expression(0);
-        setState(71);
+        setState(69);
         match(RP);
-        setState(74);
+        setState(72);
         switch ( getInterpreter().adaptivePredict(_input,2,_ctx) ) {
         case 1:
           {
-          setState(72);
+          setState(70);
           block();
           }
           break;
         case 2:
           {
-          setState(73);
+          setState(71);
           empty();
           }
           break;
@@ -425,23 +424,23 @@ class PainlessParser extends Parser {
         _localctx = new DoContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(76);
+        setState(74);
         match(DO);
-        setState(77);
+        setState(75);
         block();
-        setState(78);
+        setState(76);
         match(WHILE);
-        setState(79);
+        setState(77);
         match(LP);
-        setState(80);
+        setState(78);
         expression(0);
-        setState(81);
+        setState(79);
         match(RP);
-        setState(83);
+        setState(81);
         _la = _input.LA(1);
         if (_la==SEMICOLON) {
           {
-          setState(82);
+          setState(80);
           match(SEMICOLON);
           }
         }
@@ -452,54 +451,54 @@ class PainlessParser extends Parser {
         _localctx = new ForContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(85);
+        setState(83);
         match(FOR);
-        setState(86);
+        setState(84);
         match(LP);
-        setState(88);
+        setState(86);
         _la = _input.LA(1);
         if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0)) {
           {
-          setState(87);
+          setState(85);
           initializer();
           }
         }
 
-        setState(90);
+        setState(88);
         match(SEMICOLON);
-        setState(92);
+        setState(90);
         _la = _input.LA(1);
         if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0)) {
           {
-          setState(91);
+          setState(89);
           expression(0);
           }
         }
 
-        setState(94);
+        setState(92);
         match(SEMICOLON);
-        setState(96);
+        setState(94);
         _la = _input.LA(1);
         if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0)) {
           {
-          setState(95);
+          setState(93);
           afterthought();
           }
         }
 
-        setState(98);
+        setState(96);
         match(RP);
-        setState(101);
+        setState(99);
         switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) {
         case 1:
           {
-          setState(99);
+          setState(97);
           block();
           }
           break;
         case 2:
           {
-          setState(100);
+          setState(98);
           empty();
           }
           break;
@@ -510,13 +509,13 @@ class PainlessParser extends Parser {
         _localctx = new DeclContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(103);
+        setState(101);
         declaration();
-        setState(105);
+        setState(103);
         _la = _input.LA(1);
         if (_la==SEMICOLON) {
           {
-          setState(104);
+          setState(102);
           match(SEMICOLON);
           }
         }
@@ -527,13 +526,13 @@ class PainlessParser extends Parser {
         _localctx = new ContinueContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(107);
+        setState(105);
         match(CONTINUE);
-        setState(109);
+        setState(107);
         _la = _input.LA(1);
         if (_la==SEMICOLON) {
           {
-          setState(108);
+          setState(106);
           match(SEMICOLON);
           }
         }
@@ -544,13 +543,13 @@ class PainlessParser extends Parser {
         _localctx = new BreakContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(111);
+        setState(109);
         match(BREAK);
-        setState(113);
+        setState(111);
         _la = _input.LA(1);
         if (_la==SEMICOLON) {
           {
-          setState(112);
+          setState(110);
           match(SEMICOLON);
           }
         }
@@ -561,15 +560,15 @@ class PainlessParser extends Parser {
         _localctx = new ReturnContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(115);
+        setState(113);
         match(RETURN);
-        setState(116);
+        setState(114);
         expression(0);
-        setState(118);
+        setState(116);
         _la = _input.LA(1);
         if (_la==SEMICOLON) {
           {
-          setState(117);
+          setState(115);
           match(SEMICOLON);
           }
         }
@@ -580,11 +579,11 @@ class PainlessParser extends Parser {
         _localctx = new TryContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(120);
+        setState(118);
         match(TRY);
-        setState(121);
+        setState(119);
         block();
-        setState(123); 
+        setState(121); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -592,7 +591,7 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(122);
+            setState(120);
             trap();
             }
             }
@@ -600,7 +599,7 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(125); 
+          setState(123); 
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,12,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
@@ -610,15 +609,15 @@ class PainlessParser extends Parser {
         _localctx = new ThrowContext(_localctx);
         enterOuterAlt(_localctx, 10);
         {
-        setState(127);
+        setState(125);
         match(THROW);
-        setState(128);
+        setState(126);
         expression(0);
-        setState(130);
+        setState(128);
         _la = _input.LA(1);
         if (_la==SEMICOLON) {
           {
-          setState(129);
+          setState(127);
           match(SEMICOLON);
           }
         }
@@ -629,13 +628,13 @@ class PainlessParser extends Parser {
         _localctx = new ExprContext(_localctx);
         enterOuterAlt(_localctx, 11);
         {
-        setState(132);
+        setState(130);
         expression(0);
-        setState(134);
+        setState(132);
         _la = _input.LA(1);
         if (_la==SEMICOLON) {
           {
-          setState(133);
+          setState(131);
           match(SEMICOLON);
           }
         }
@@ -699,29 +698,29 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 4, RULE_block);
     int _la;
     try {
-      setState(147);
+      setState(145);
       switch (_input.LA(1)) {
       case LBRACK:
         _localctx = new MultipleContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(138);
+        setState(136);
         match(LBRACK);
-        setState(140); 
+        setState(138); 
         _errHandler.sync(this);
         _la = _input.LA(1);
         do {
           {
           {
-          setState(139);
+          setState(137);
           statement();
           }
           }
-          setState(142); 
+          setState(140); 
           _errHandler.sync(this);
           _la = _input.LA(1);
         } while ( (((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << IF) | (1L << WHILE) | (1L << DO) | (1L << FOR) | (1L << CONTINUE) | (1L << BREAK) | (1L << RETURN) | (1L << NEW) | (1L << TRY) | (1L << THROW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0) );
-        setState(144);
+        setState(142);
         match(RBRACK);
         }
         break;
@@ -754,7 +753,7 @@ class PainlessParser extends Parser {
         _localctx = new SingleContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(146);
+        setState(144);
         statement();
         }
         break;
@@ -793,19 +792,19 @@ class PainlessParser extends Parser {
     EmptyContext _localctx = new EmptyContext(_ctx, getState());
     enterRule(_localctx, 6, RULE_empty);
     try {
-      setState(151);
+      setState(149);
       switch (_input.LA(1)) {
       case LBRACK:
         enterOuterAlt(_localctx, 1);
         {
-        setState(149);
+        setState(147);
         emptyscope();
         }
         break;
       case SEMICOLON:
         enterOuterAlt(_localctx, 2);
         {
-        setState(150);
+        setState(148);
         match(SEMICOLON);
         }
         break;
@@ -844,9 +843,9 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(153);
+      setState(151);
       match(LBRACK);
-      setState(154);
+      setState(152);
       match(RBRACK);
       }
     }
@@ -883,19 +882,19 @@ class PainlessParser extends Parser {
     InitializerContext _localctx = new InitializerContext(_ctx, getState());
     enterRule(_localctx, 10, RULE_initializer);
     try {
-      setState(158);
+      setState(156);
       switch ( getInterpreter().adaptivePredict(_input,19,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(156);
+        setState(154);
         declaration();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(157);
+        setState(155);
         expression(0);
         }
         break;
@@ -933,7 +932,7 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(160);
+      setState(158);
       expression(0);
       }
     }
@@ -980,23 +979,23 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(162);
+      setState(160);
       decltype();
-      setState(163);
+      setState(161);
       declvar();
-      setState(168);
+      setState(166);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==COMMA) {
         {
         {
-        setState(164);
+        setState(162);
         match(COMMA);
-        setState(165);
+        setState(163);
         declvar();
         }
         }
-        setState(170);
+        setState(168);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
@@ -1043,21 +1042,21 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(171);
+      setState(169);
       identifier();
-      setState(176);
+      setState(174);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==LBRACE) {
         {
         {
-        setState(172);
+        setState(170);
         match(LBRACE);
-        setState(173);
+        setState(171);
         match(RBRACE);
         }
         }
-        setState(178);
+        setState(176);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
@@ -1100,15 +1099,15 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(179);
+      setState(177);
       identifier();
-      setState(182);
+      setState(180);
       _la = _input.LA(1);
       if (_la==ASSIGN) {
         {
-        setState(180);
+        setState(178);
         match(ASSIGN);
-        setState(181);
+        setState(179);
         expression(0);
         }
       }
@@ -1159,29 +1158,29 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(184);
+      setState(182);
       match(CATCH);
-      setState(185);
+      setState(183);
       match(LP);
       {
-      setState(186);
+      setState(184);
       identifier();
-      setState(187);
+      setState(185);
       identifier();
       }
-      setState(189);
+      setState(187);
       match(RP);
-      setState(192);
+      setState(190);
       switch ( getInterpreter().adaptivePredict(_input,23,_ctx) ) {
       case 1:
         {
-        setState(190);
+        setState(188);
         block();
         }
         break;
       case 2:
         {
-        setState(191);
+        setState(189);
         emptyscope();
         }
         break;
@@ -1221,13 +1220,13 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(194);
+      setState(192);
       match(ID);
-      setState(196);
+      setState(194);
       switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) {
       case 1:
         {
-        setState(195);
+        setState(193);
         generic();
         }
         break;
@@ -1276,27 +1275,27 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(198);
+      setState(196);
       match(LT);
-      setState(199);
+      setState(197);
       identifier();
-      setState(204);
+      setState(202);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==COMMA) {
         {
         {
-        setState(200);
+        setState(198);
         match(COMMA);
-        setState(201);
+        setState(199);
         identifier();
         }
         }
-        setState(206);
+        setState(204);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
-      setState(207);
+      setState(205);
       match(GT);
       }
     }
@@ -1344,6 +1343,17 @@ class PainlessParser extends Parser {
       else return visitor.visitChildren(this);
     }
   }
+  public static class ReadContext extends ExpressionContext {
+    public ChainContext chain() {
+      return getRuleContext(ChainContext.class,0);
+    }
+    public ReadContext(ExpressionContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitRead(this);
+      else return visitor.visitChildren(this);
+    }
+  }
   public static class BoolContext extends ExpressionContext {
     public List<ExpressionContext> expression() {
       return getRuleContexts(ExpressionContext.class);
@@ -1377,8 +1387,8 @@ class PainlessParser extends Parser {
     }
   }
   public static class AssignmentContext extends ExpressionContext {
-    public ExtstartContext extstart() {
-      return getRuleContext(ExtstartContext.class,0);
+    public ChainContext chain() {
+      return getRuleContext(ChainContext.class,0);
     }
     public ExpressionContext expression() {
       return getRuleContext(ExpressionContext.class,0);
@@ -1452,12 +1462,11 @@ class PainlessParser extends Parser {
     }
   }
   public static class PreincContext extends ExpressionContext {
-    public IncrementContext increment() {
-      return getRuleContext(IncrementContext.class,0);
-    }
-    public ExtstartContext extstart() {
-      return getRuleContext(ExtstartContext.class,0);
+    public ChainContext chain() {
+      return getRuleContext(ChainContext.class,0);
     }
+    public TerminalNode INCR() { return getToken(PainlessParser.INCR, 0); }
+    public TerminalNode DECR() { return getToken(PainlessParser.DECR, 0); }
     public PreincContext(ExpressionContext ctx) { copyFrom(ctx); }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
@@ -1466,12 +1475,11 @@ class PainlessParser extends Parser {
     }
   }
   public static class PostincContext extends ExpressionContext {
-    public ExtstartContext extstart() {
-      return getRuleContext(ExtstartContext.class,0);
-    }
-    public IncrementContext increment() {
-      return getRuleContext(IncrementContext.class,0);
+    public ChainContext chain() {
+      return getRuleContext(ChainContext.class,0);
     }
+    public TerminalNode INCR() { return getToken(PainlessParser.INCR, 0); }
+    public TerminalNode DECR() { return getToken(PainlessParser.DECR, 0); }
     public PostincContext(ExpressionContext ctx) { copyFrom(ctx); }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
@@ -1495,17 +1503,6 @@ class PainlessParser extends Parser {
       else return visitor.visitChildren(this);
     }
   }
-  public static class ExternalContext extends ExpressionContext {
-    public ExtstartContext extstart() {
-      return getRuleContext(ExtstartContext.class,0);
-    }
-    public ExternalContext(ExpressionContext ctx) { copyFrom(ctx); }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitExternal(this);
-      else return visitor.visitChildren(this);
-    }
-  }
   public static class NullContext extends ExpressionContext {
     public TerminalNode NULL() { return getToken(PainlessParser.NULL, 0); }
     public NullContext(ExpressionContext ctx) { copyFrom(ctx); }
@@ -1531,7 +1528,7 @@ class PainlessParser extends Parser {
     public TerminalNode RSH() { return getToken(PainlessParser.RSH, 0); }
     public TerminalNode USH() { return getToken(PainlessParser.USH, 0); }
     public TerminalNode BWAND() { return getToken(PainlessParser.BWAND, 0); }
-    public TerminalNode BWXOR() { return getToken(PainlessParser.BWXOR, 0); }
+    public TerminalNode XOR() { return getToken(PainlessParser.XOR, 0); }
     public TerminalNode BWOR() { return getToken(PainlessParser.BWOR, 0); }
     public BinaryContext(ExpressionContext ctx) { copyFrom(ctx); }
     @Override
@@ -1566,7 +1563,7 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(236);
+      setState(233);
       switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) {
       case 1:
         {
@@ -1574,14 +1571,14 @@ class PainlessParser extends Parser {
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(210);
+        setState(208);
         _la = _input.LA(1);
         if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(211);
+        setState(209);
         expression(14);
         }
         break;
@@ -1590,13 +1587,13 @@ class PainlessParser extends Parser {
         _localctx = new CastContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(212);
+        setState(210);
         match(LP);
-        setState(213);
+        setState(211);
         decltype();
-        setState(214);
+        setState(212);
         match(RP);
-        setState(215);
+        setState(213);
         expression(13);
         }
         break;
@@ -1605,16 +1602,16 @@ class PainlessParser extends Parser {
         _localctx = new AssignmentContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(217);
-        extstart();
-        setState(218);
+        setState(215);
+        chain();
+        setState(216);
         _la = _input.LA(1);
         if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ASSIGN) | (1L << AADD) | (1L << ASUB) | (1L << AMUL) | (1L << ADIV) | (1L << AREM) | (1L << AAND) | (1L << AXOR) | (1L << AOR) | (1L << ALSH) | (1L << ARSH) | (1L << AUSH))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(219);
+        setState(217);
         expression(1);
         }
         break;
@@ -1623,11 +1620,11 @@ class PainlessParser extends Parser {
         _localctx = new PrecedenceContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(221);
+        setState(219);
         match(LP);
-        setState(222);
+        setState(220);
         expression(0);
-        setState(223);
+        setState(221);
         match(RP);
         }
         break;
@@ -1636,7 +1633,7 @@ class PainlessParser extends Parser {
         _localctx = new NumericContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(225);
+        setState(223);
         _la = _input.LA(1);
         if ( !(((((_la - 63)) & ~0x3f) == 0 && ((1L << (_la - 63)) & ((1L << (OCTAL - 63)) | (1L << (HEX - 63)) | (1L << (INTEGER - 63)) | (1L << (DECIMAL - 63)))) != 0)) ) {
         _errHandler.recoverInline(this);
@@ -1650,7 +1647,7 @@ class PainlessParser extends Parser {
         _localctx = new TrueContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(226);
+        setState(224);
         match(TRUE);
         }
         break;
@@ -1659,7 +1656,7 @@ class PainlessParser extends Parser {
         _localctx = new FalseContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(227);
+        setState(225);
         match(FALSE);
         }
         break;
@@ -1668,7 +1665,7 @@ class PainlessParser extends Parser {
         _localctx = new NullContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(228);
+        setState(226);
         match(NULL);
         }
         break;
@@ -1677,10 +1674,15 @@ class PainlessParser extends Parser {
         _localctx = new PostincContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(229);
-        extstart();
-        setState(230);
-        increment();
+        setState(227);
+        chain();
+        setState(228);
+        _la = _input.LA(1);
+        if ( !(_la==INCR || _la==DECR) ) {
+        _errHandler.recoverInline(this);
+        } else {
+          consume();
+        }
         }
         break;
       case 10:
@@ -1688,24 +1690,29 @@ class PainlessParser extends Parser {
         _localctx = new PreincContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(232);
-        increment();
-        setState(233);
-        extstart();
+        setState(230);
+        _la = _input.LA(1);
+        if ( !(_la==INCR || _la==DECR) ) {
+        _errHandler.recoverInline(this);
+        } else {
+          consume();
+        }
+        setState(231);
+        chain();
         }
         break;
       case 11:
         {
-        _localctx = new ExternalContext(_localctx);
+        _localctx = new ReadContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(235);
-        extstart();
+        setState(232);
+        chain();
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(276);
+      setState(273);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,28,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -1713,22 +1720,22 @@ class PainlessParser extends Parser {
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(274);
+          setState(271);
           switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) {
           case 1:
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(238);
+            setState(235);
             if (!(precpred(_ctx, 12))) throw new FailedPredicateException(this, "precpred(_ctx, 12)");
-            setState(239);
+            setState(236);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << MUL) | (1L << DIV) | (1L << REM))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(240);
+            setState(237);
             expression(13);
             }
             break;
@@ -1736,16 +1743,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(241);
+            setState(238);
             if (!(precpred(_ctx, 11))) throw new FailedPredicateException(this, "precpred(_ctx, 11)");
-            setState(242);
+            setState(239);
             _la = _input.LA(1);
             if ( !(_la==ADD || _la==SUB) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(243);
+            setState(240);
             expression(12);
             }
             break;
@@ -1753,16 +1760,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(244);
+            setState(241);
             if (!(precpred(_ctx, 10))) throw new FailedPredicateException(this, "precpred(_ctx, 10)");
-            setState(245);
+            setState(242);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LSH) | (1L << RSH) | (1L << USH))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(246);
+            setState(243);
             expression(11);
             }
             break;
@@ -1770,16 +1777,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(247);
+            setState(244);
             if (!(precpred(_ctx, 9))) throw new FailedPredicateException(this, "precpred(_ctx, 9)");
-            setState(248);
+            setState(245);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LT) | (1L << LTE) | (1L << GT) | (1L << GTE))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(249);
+            setState(246);
             expression(10);
             }
             break;
@@ -1787,16 +1794,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(250);
+            setState(247);
             if (!(precpred(_ctx, 8))) throw new FailedPredicateException(this, "precpred(_ctx, 8)");
-            setState(251);
+            setState(248);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << EQ) | (1L << EQR) | (1L << NE) | (1L << NER))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(252);
+            setState(249);
             expression(9);
             }
             break;
@@ -1804,11 +1811,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(253);
+            setState(250);
             if (!(precpred(_ctx, 7))) throw new FailedPredicateException(this, "precpred(_ctx, 7)");
-            setState(254);
+            setState(251);
             match(BWAND);
-            setState(255);
+            setState(252);
             expression(8);
             }
             break;
@@ -1816,11 +1823,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(256);
+            setState(253);
             if (!(precpred(_ctx, 6))) throw new FailedPredicateException(this, "precpred(_ctx, 6)");
-            setState(257);
-            match(BWXOR);
-            setState(258);
+            setState(254);
+            match(XOR);
+            setState(255);
             expression(7);
             }
             break;
@@ -1828,11 +1835,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(259);
+            setState(256);
             if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)");
-            setState(260);
+            setState(257);
             match(BWOR);
-            setState(261);
+            setState(258);
             expression(6);
             }
             break;
@@ -1840,11 +1847,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(262);
+            setState(259);
             if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)");
-            setState(263);
+            setState(260);
             match(BOOLAND);
-            setState(264);
+            setState(261);
             expression(5);
             }
             break;
@@ -1852,11 +1859,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(265);
+            setState(262);
             if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)");
-            setState(266);
+            setState(263);
             match(BOOLOR);
-            setState(267);
+            setState(264);
             expression(4);
             }
             break;
@@ -1864,22 +1871,22 @@ class PainlessParser extends Parser {
             {
             _localctx = new ConditionalContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(268);
+            setState(265);
             if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
-            setState(269);
+            setState(266);
             match(COND);
-            setState(270);
+            setState(267);
             expression(0);
-            setState(271);
+            setState(268);
             match(COLON);
-            setState(272);
+            setState(269);
             expression(2);
             }
             break;
           }
           } 
         }
-        setState(278);
+        setState(275);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,28,_ctx);
       }
@@ -1896,72 +1903,72 @@ class PainlessParser extends Parser {
     return _localctx;
   }
 
-  public static class ExtstartContext extends ParserRuleContext {
-    public ExtprecContext extprec() {
-      return getRuleContext(ExtprecContext.class,0);
+  public static class ChainContext extends ParserRuleContext {
+    public LinkprecContext linkprec() {
+      return getRuleContext(LinkprecContext.class,0);
     }
-    public ExtcastContext extcast() {
-      return getRuleContext(ExtcastContext.class,0);
+    public LinkcastContext linkcast() {
+      return getRuleContext(LinkcastContext.class,0);
     }
-    public ExtvarContext extvar() {
-      return getRuleContext(ExtvarContext.class,0);
+    public LinkvarContext linkvar() {
+      return getRuleContext(LinkvarContext.class,0);
     }
-    public ExtnewContext extnew() {
-      return getRuleContext(ExtnewContext.class,0);
+    public LinknewContext linknew() {
+      return getRuleContext(LinknewContext.class,0);
     }
-    public ExtstringContext extstring() {
-      return getRuleContext(ExtstringContext.class,0);
+    public LinkstringContext linkstring() {
+      return getRuleContext(LinkstringContext.class,0);
     }
-    public ExtstartContext(ParserRuleContext parent, int invokingState) {
+    public ChainContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
     }
-    @Override public int getRuleIndex() { return RULE_extstart; }
+    @Override public int getRuleIndex() { return RULE_chain; }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitExtstart(this);
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitChain(this);
       else return visitor.visitChildren(this);
     }
   }
 
-  public final ExtstartContext extstart() throws RecognitionException {
-    ExtstartContext _localctx = new ExtstartContext(_ctx, getState());
-    enterRule(_localctx, 28, RULE_extstart);
+  public final ChainContext chain() throws RecognitionException {
+    ChainContext _localctx = new ChainContext(_ctx, getState());
+    enterRule(_localctx, 28, RULE_chain);
     try {
-      setState(284);
+      setState(281);
       switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(279);
-        extprec();
+        setState(276);
+        linkprec();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(280);
-        extcast();
+        setState(277);
+        linkcast();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(281);
-        extvar();
+        setState(278);
+        linkvar();
         }
         break;
       case 4:
         enterOuterAlt(_localctx, 4);
         {
-        setState(282);
-        extnew();
+        setState(279);
+        linknew();
         }
         break;
       case 5:
         enterOuterAlt(_localctx, 5);
         {
-        setState(283);
-        extstring();
+        setState(280);
+        linkstring();
         }
         break;
       }
@@ -1977,96 +1984,96 @@ class PainlessParser extends Parser {
     return _localctx;
   }
 
-  public static class ExtprecContext extends ParserRuleContext {
+  public static class LinkprecContext extends ParserRuleContext {
     public TerminalNode LP() { return getToken(PainlessParser.LP, 0); }
     public TerminalNode RP() { return getToken(PainlessParser.RP, 0); }
-    public ExtprecContext extprec() {
-      return getRuleContext(ExtprecContext.class,0);
+    public LinkprecContext linkprec() {
+      return getRuleContext(LinkprecContext.class,0);
     }
-    public ExtcastContext extcast() {
-      return getRuleContext(ExtcastContext.class,0);
+    public LinkcastContext linkcast() {
+      return getRuleContext(LinkcastContext.class,0);
     }
-    public ExtvarContext extvar() {
-      return getRuleContext(ExtvarContext.class,0);
+    public LinkvarContext linkvar() {
+      return getRuleContext(LinkvarContext.class,0);
     }
-    public ExtnewContext extnew() {
-      return getRuleContext(ExtnewContext.class,0);
+    public LinknewContext linknew() {
+      return getRuleContext(LinknewContext.class,0);
     }
-    public ExtstringContext extstring() {
-      return getRuleContext(ExtstringContext.class,0);
+    public LinkstringContext linkstring() {
+      return getRuleContext(LinkstringContext.class,0);
     }
-    public ExtdotContext extdot() {
-      return getRuleContext(ExtdotContext.class,0);
+    public LinkdotContext linkdot() {
+      return getRuleContext(LinkdotContext.class,0);
     }
-    public ExtbraceContext extbrace() {
-      return getRuleContext(ExtbraceContext.class,0);
+    public LinkbraceContext linkbrace() {
+      return getRuleContext(LinkbraceContext.class,0);
     }
-    public ExtprecContext(ParserRuleContext parent, int invokingState) {
+    public LinkprecContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
     }
-    @Override public int getRuleIndex() { return RULE_extprec; }
+    @Override public int getRuleIndex() { return RULE_linkprec; }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitExtprec(this);
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitLinkprec(this);
       else return visitor.visitChildren(this);
     }
   }
 
-  public final ExtprecContext extprec() throws RecognitionException {
-    ExtprecContext _localctx = new ExtprecContext(_ctx, getState());
-    enterRule(_localctx, 30, RULE_extprec);
+  public final LinkprecContext linkprec() throws RecognitionException {
+    LinkprecContext _localctx = new LinkprecContext(_ctx, getState());
+    enterRule(_localctx, 30, RULE_linkprec);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(286);
+      setState(283);
       match(LP);
-      setState(292);
+      setState(289);
       switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
       case 1:
         {
-        setState(287);
-        extprec();
+        setState(284);
+        linkprec();
         }
         break;
       case 2:
         {
-        setState(288);
-        extcast();
+        setState(285);
+        linkcast();
         }
         break;
       case 3:
         {
-        setState(289);
-        extvar();
+        setState(286);
+        linkvar();
         }
         break;
       case 4:
         {
-        setState(290);
-        extnew();
+        setState(287);
+        linknew();
         }
         break;
       case 5:
         {
-        setState(291);
-        extstring();
+        setState(288);
+        linkstring();
         }
         break;
       }
-      setState(294);
+      setState(291);
       match(RP);
-      setState(297);
+      setState(294);
       switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) {
       case 1:
         {
-        setState(295);
-        extdot();
+        setState(292);
+        linkdot();
         }
         break;
       case 2:
         {
-        setState(296);
-        extbrace();
+        setState(293);
+        linkbrace();
         }
         break;
       }
@@ -2083,80 +2090,80 @@ class PainlessParser extends Parser {
     return _localctx;
   }
 
-  public static class ExtcastContext extends ParserRuleContext {
+  public static class LinkcastContext extends ParserRuleContext {
     public TerminalNode LP() { return getToken(PainlessParser.LP, 0); }
     public DecltypeContext decltype() {
       return getRuleContext(DecltypeContext.class,0);
     }
     public TerminalNode RP() { return getToken(PainlessParser.RP, 0); }
-    public ExtprecContext extprec() {
-      return getRuleContext(ExtprecContext.class,0);
+    public LinkprecContext linkprec() {
+      return getRuleContext(LinkprecContext.class,0);
     }
-    public ExtcastContext extcast() {
-      return getRuleContext(ExtcastContext.class,0);
+    public LinkcastContext linkcast() {
+      return getRuleContext(LinkcastContext.class,0);
     }
-    public ExtvarContext extvar() {
-      return getRuleContext(ExtvarContext.class,0);
+    public LinkvarContext linkvar() {
+      return getRuleContext(LinkvarContext.class,0);
     }
-    public ExtnewContext extnew() {
-      return getRuleContext(ExtnewContext.class,0);
+    public LinknewContext linknew() {
+      return getRuleContext(LinknewContext.class,0);
     }
-    public ExtstringContext extstring() {
-      return getRuleContext(ExtstringContext.class,0);
+    public LinkstringContext linkstring() {
+      return getRuleContext(LinkstringContext.class,0);
     }
-    public ExtcastContext(ParserRuleContext parent, int invokingState) {
+    public LinkcastContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
     }
-    @Override public int getRuleIndex() { return RULE_extcast; }
+    @Override public int getRuleIndex() { return RULE_linkcast; }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitExtcast(this);
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitLinkcast(this);
       else return visitor.visitChildren(this);
     }
   }
 
-  public final ExtcastContext extcast() throws RecognitionException {
-    ExtcastContext _localctx = new ExtcastContext(_ctx, getState());
-    enterRule(_localctx, 32, RULE_extcast);
+  public final LinkcastContext linkcast() throws RecognitionException {
+    LinkcastContext _localctx = new LinkcastContext(_ctx, getState());
+    enterRule(_localctx, 32, RULE_linkcast);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(299);
+      setState(296);
       match(LP);
-      setState(300);
+      setState(297);
       decltype();
-      setState(301);
+      setState(298);
       match(RP);
-      setState(307);
+      setState(304);
       switch ( getInterpreter().adaptivePredict(_input,32,_ctx) ) {
       case 1:
         {
-        setState(302);
-        extprec();
+        setState(299);
+        linkprec();
         }
         break;
       case 2:
         {
-        setState(303);
-        extcast();
+        setState(300);
+        linkcast();
         }
         break;
       case 3:
         {
-        setState(304);
-        extvar();
+        setState(301);
+        linkvar();
         }
         break;
       case 4:
         {
-        setState(305);
-        extnew();
+        setState(302);
+        linknew();
         }
         break;
       case 5:
         {
-        setState(306);
-        extstring();
+        setState(303);
+        linkstring();
         }
         break;
       }
@@ -2173,53 +2180,53 @@ class PainlessParser extends Parser {
     return _localctx;
   }
 
-  public static class ExtbraceContext extends ParserRuleContext {
+  public static class LinkbraceContext extends ParserRuleContext {
     public TerminalNode LBRACE() { return getToken(PainlessParser.LBRACE, 0); }
     public ExpressionContext expression() {
       return getRuleContext(ExpressionContext.class,0);
     }
     public TerminalNode RBRACE() { return getToken(PainlessParser.RBRACE, 0); }
-    public ExtdotContext extdot() {
-      return getRuleContext(ExtdotContext.class,0);
+    public LinkdotContext linkdot() {
+      return getRuleContext(LinkdotContext.class,0);
     }
-    public ExtbraceContext extbrace() {
-      return getRuleContext(ExtbraceContext.class,0);
+    public LinkbraceContext linkbrace() {
+      return getRuleContext(LinkbraceContext.class,0);
     }
-    public ExtbraceContext(ParserRuleContext parent, int invokingState) {
+    public LinkbraceContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
     }
-    @Override public int getRuleIndex() { return RULE_extbrace; }
+    @Override public int getRuleIndex() { return RULE_linkbrace; }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitExtbrace(this);
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitLinkbrace(this);
       else return visitor.visitChildren(this);
     }
   }
 
-  public final ExtbraceContext extbrace() throws RecognitionException {
-    ExtbraceContext _localctx = new ExtbraceContext(_ctx, getState());
-    enterRule(_localctx, 34, RULE_extbrace);
+  public final LinkbraceContext linkbrace() throws RecognitionException {
+    LinkbraceContext _localctx = new LinkbraceContext(_ctx, getState());
+    enterRule(_localctx, 34, RULE_linkbrace);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(309);
+      setState(306);
       match(LBRACE);
-      setState(310);
+      setState(307);
       expression(0);
-      setState(311);
+      setState(308);
       match(RBRACE);
-      setState(314);
+      setState(311);
       switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) {
       case 1:
         {
-        setState(312);
-        extdot();
+        setState(309);
+        linkdot();
         }
         break;
       case 2:
         {
-        setState(313);
-        extbrace();
+        setState(310);
+        linkbrace();
         }
         break;
       }
@@ -2236,45 +2243,45 @@ class PainlessParser extends Parser {
     return _localctx;
   }
 
-  public static class ExtdotContext extends ParserRuleContext {
+  public static class LinkdotContext extends ParserRuleContext {
     public TerminalNode DOT() { return getToken(PainlessParser.DOT, 0); }
-    public ExtcallContext extcall() {
-      return getRuleContext(ExtcallContext.class,0);
+    public LinkcallContext linkcall() {
+      return getRuleContext(LinkcallContext.class,0);
     }
-    public ExtfieldContext extfield() {
-      return getRuleContext(ExtfieldContext.class,0);
+    public LinkfieldContext linkfield() {
+      return getRuleContext(LinkfieldContext.class,0);
     }
-    public ExtdotContext(ParserRuleContext parent, int invokingState) {
+    public LinkdotContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
     }
-    @Override public int getRuleIndex() { return RULE_extdot; }
+    @Override public int getRuleIndex() { return RULE_linkdot; }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitExtdot(this);
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitLinkdot(this);
       else return visitor.visitChildren(this);
     }
   }
 
-  public final ExtdotContext extdot() throws RecognitionException {
-    ExtdotContext _localctx = new ExtdotContext(_ctx, getState());
-    enterRule(_localctx, 36, RULE_extdot);
+  public final LinkdotContext linkdot() throws RecognitionException {
+    LinkdotContext _localctx = new LinkdotContext(_ctx, getState());
+    enterRule(_localctx, 36, RULE_linkdot);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(316);
+      setState(313);
       match(DOT);
-      setState(319);
+      setState(316);
       switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) {
       case 1:
         {
-        setState(317);
-        extcall();
+        setState(314);
+        linkcall();
         }
         break;
       case 2:
         {
-        setState(318);
-        extfield();
+        setState(315);
+        linkfield();
         }
         break;
       }
@@ -2291,50 +2298,50 @@ class PainlessParser extends Parser {
     return _localctx;
   }
 
-  public static class ExtcallContext extends ParserRuleContext {
+  public static class LinkcallContext extends ParserRuleContext {
     public TerminalNode EXTID() { return getToken(PainlessParser.EXTID, 0); }
     public ArgumentsContext arguments() {
       return getRuleContext(ArgumentsContext.class,0);
     }
-    public ExtdotContext extdot() {
-      return getRuleContext(ExtdotContext.class,0);
+    public LinkdotContext linkdot() {
+      return getRuleContext(LinkdotContext.class,0);
     }
-    public ExtbraceContext extbrace() {
-      return getRuleContext(ExtbraceContext.class,0);
+    public LinkbraceContext linkbrace() {
+      return getRuleContext(LinkbraceContext.class,0);
     }
-    public ExtcallContext(ParserRuleContext parent, int invokingState) {
+    public LinkcallContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
     }
-    @Override public int getRuleIndex() { return RULE_extcall; }
+    @Override public int getRuleIndex() { return RULE_linkcall; }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitExtcall(this);
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitLinkcall(this);
       else return visitor.visitChildren(this);
     }
   }
 
-  public final ExtcallContext extcall() throws RecognitionException {
-    ExtcallContext _localctx = new ExtcallContext(_ctx, getState());
-    enterRule(_localctx, 38, RULE_extcall);
+  public final LinkcallContext linkcall() throws RecognitionException {
+    LinkcallContext _localctx = new LinkcallContext(_ctx, getState());
+    enterRule(_localctx, 38, RULE_linkcall);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(321);
+      setState(318);
       match(EXTID);
-      setState(322);
+      setState(319);
       arguments();
-      setState(325);
+      setState(322);
       switch ( getInterpreter().adaptivePredict(_input,35,_ctx) ) {
       case 1:
         {
-        setState(323);
-        extdot();
+        setState(320);
+        linkdot();
         }
         break;
       case 2:
         {
-        setState(324);
-        extbrace();
+        setState(321);
+        linkbrace();
         }
         break;
       }
@@ -2351,47 +2358,47 @@ class PainlessParser extends Parser {
     return _localctx;
   }
 
-  public static class ExtvarContext extends ParserRuleContext {
+  public static class LinkvarContext extends ParserRuleContext {
     public IdentifierContext identifier() {
       return getRuleContext(IdentifierContext.class,0);
     }
-    public ExtdotContext extdot() {
-      return getRuleContext(ExtdotContext.class,0);
+    public LinkdotContext linkdot() {
+      return getRuleContext(LinkdotContext.class,0);
     }
-    public ExtbraceContext extbrace() {
-      return getRuleContext(ExtbraceContext.class,0);
+    public LinkbraceContext linkbrace() {
+      return getRuleContext(LinkbraceContext.class,0);
     }
-    public ExtvarContext(ParserRuleContext parent, int invokingState) {
+    public LinkvarContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
     }
-    @Override public int getRuleIndex() { return RULE_extvar; }
+    @Override public int getRuleIndex() { return RULE_linkvar; }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitExtvar(this);
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitLinkvar(this);
       else return visitor.visitChildren(this);
     }
   }
 
-  public final ExtvarContext extvar() throws RecognitionException {
-    ExtvarContext _localctx = new ExtvarContext(_ctx, getState());
-    enterRule(_localctx, 40, RULE_extvar);
+  public final LinkvarContext linkvar() throws RecognitionException {
+    LinkvarContext _localctx = new LinkvarContext(_ctx, getState());
+    enterRule(_localctx, 40, RULE_linkvar);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(327);
+      setState(324);
       identifier();
-      setState(330);
+      setState(327);
       switch ( getInterpreter().adaptivePredict(_input,36,_ctx) ) {
       case 1:
         {
-        setState(328);
-        extdot();
+        setState(325);
+        linkdot();
         }
         break;
       case 2:
         {
-        setState(329);
-        extbrace();
+        setState(326);
+        linkbrace();
         }
         break;
       }
@@ -2408,52 +2415,52 @@ class PainlessParser extends Parser {
     return _localctx;
   }
 
-  public static class ExtfieldContext extends ParserRuleContext {
+  public static class LinkfieldContext extends ParserRuleContext {
     public TerminalNode EXTID() { return getToken(PainlessParser.EXTID, 0); }
     public TerminalNode EXTINTEGER() { return getToken(PainlessParser.EXTINTEGER, 0); }
-    public ExtdotContext extdot() {
-      return getRuleContext(ExtdotContext.class,0);
+    public LinkdotContext linkdot() {
+      return getRuleContext(LinkdotContext.class,0);
     }
-    public ExtbraceContext extbrace() {
-      return getRuleContext(ExtbraceContext.class,0);
+    public LinkbraceContext linkbrace() {
+      return getRuleContext(LinkbraceContext.class,0);
     }
-    public ExtfieldContext(ParserRuleContext parent, int invokingState) {
+    public LinkfieldContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
     }
-    @Override public int getRuleIndex() { return RULE_extfield; }
+    @Override public int getRuleIndex() { return RULE_linkfield; }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitExtfield(this);
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitLinkfield(this);
       else return visitor.visitChildren(this);
     }
   }
 
-  public final ExtfieldContext extfield() throws RecognitionException {
-    ExtfieldContext _localctx = new ExtfieldContext(_ctx, getState());
-    enterRule(_localctx, 42, RULE_extfield);
+  public final LinkfieldContext linkfield() throws RecognitionException {
+    LinkfieldContext _localctx = new LinkfieldContext(_ctx, getState());
+    enterRule(_localctx, 42, RULE_linkfield);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(332);
+      setState(329);
       _la = _input.LA(1);
       if ( !(_la==EXTINTEGER || _la==EXTID) ) {
       _errHandler.recoverInline(this);
       } else {
         consume();
       }
-      setState(335);
+      setState(332);
       switch ( getInterpreter().adaptivePredict(_input,37,_ctx) ) {
       case 1:
         {
-        setState(333);
-        extdot();
+        setState(330);
+        linkdot();
         }
         break;
       case 2:
         {
-        setState(334);
-        extbrace();
+        setState(331);
+        linkbrace();
         }
         break;
       }
@@ -2470,7 +2477,7 @@ class PainlessParser extends Parser {
     return _localctx;
   }
 
-  public static class ExtnewContext extends ParserRuleContext {
+  public static class LinknewContext extends ParserRuleContext {
     public TerminalNode NEW() { return getToken(PainlessParser.NEW, 0); }
     public IdentifierContext identifier() {
       return getRuleContext(IdentifierContext.class,0);
@@ -2478,8 +2485,8 @@ class PainlessParser extends Parser {
     public ArgumentsContext arguments() {
       return getRuleContext(ArgumentsContext.class,0);
     }
-    public ExtdotContext extdot() {
-      return getRuleContext(ExtdotContext.class,0);
+    public LinkdotContext linkdot() {
+      return getRuleContext(LinkdotContext.class,0);
     }
     public List<TerminalNode> LBRACE() { return getTokens(PainlessParser.LBRACE); }
     public TerminalNode LBRACE(int i) {
@@ -2495,41 +2502,41 @@ class PainlessParser extends Parser {
     public TerminalNode RBRACE(int i) {
       return getToken(PainlessParser.RBRACE, i);
     }
-    public ExtnewContext(ParserRuleContext parent, int invokingState) {
+    public LinknewContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
     }
-    @Override public int getRuleIndex() { return RULE_extnew; }
+    @Override public int getRuleIndex() { return RULE_linknew; }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitExtnew(this);
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitLinknew(this);
       else return visitor.visitChildren(this);
     }
   }
 
-  public final ExtnewContext extnew() throws RecognitionException {
-    ExtnewContext _localctx = new ExtnewContext(_ctx, getState());
-    enterRule(_localctx, 44, RULE_extnew);
+  public final LinknewContext linknew() throws RecognitionException {
+    LinknewContext _localctx = new LinknewContext(_ctx, getState());
+    enterRule(_localctx, 44, RULE_linknew);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(337);
+      setState(334);
       match(NEW);
-      setState(338);
+      setState(335);
       identifier();
-      setState(354);
+      setState(351);
       switch (_input.LA(1)) {
       case LP:
         {
         {
-        setState(339);
+        setState(336);
         arguments();
-        setState(341);
+        setState(338);
         switch ( getInterpreter().adaptivePredict(_input,38,_ctx) ) {
         case 1:
           {
-          setState(340);
-          extdot();
+          setState(337);
+          linkdot();
           }
           break;
         }
@@ -2539,7 +2546,7 @@ class PainlessParser extends Parser {
       case LBRACE:
         {
         {
-        setState(347); 
+        setState(344); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -2547,11 +2554,11 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(343);
+            setState(340);
             match(LBRACE);
-            setState(344);
+            setState(341);
             expression(0);
-            setState(345);
+            setState(342);
             match(RBRACE);
             }
             }
@@ -2559,16 +2566,16 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(349); 
+          setState(346); 
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,39,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
-        setState(352);
+        setState(349);
         switch ( getInterpreter().adaptivePredict(_input,40,_ctx) ) {
         case 1:
           {
-          setState(351);
-          extdot();
+          setState(348);
+          linkdot();
           }
           break;
         }
@@ -2591,45 +2598,45 @@ class PainlessParser extends Parser {
     return _localctx;
   }
 
-  public static class ExtstringContext extends ParserRuleContext {
+  public static class LinkstringContext extends ParserRuleContext {
     public TerminalNode STRING() { return getToken(PainlessParser.STRING, 0); }
-    public ExtdotContext extdot() {
-      return getRuleContext(ExtdotContext.class,0);
+    public LinkdotContext linkdot() {
+      return getRuleContext(LinkdotContext.class,0);
     }
-    public ExtbraceContext extbrace() {
-      return getRuleContext(ExtbraceContext.class,0);
+    public LinkbraceContext linkbrace() {
+      return getRuleContext(LinkbraceContext.class,0);
     }
-    public ExtstringContext(ParserRuleContext parent, int invokingState) {
+    public LinkstringContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
     }
-    @Override public int getRuleIndex() { return RULE_extstring; }
+    @Override public int getRuleIndex() { return RULE_linkstring; }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitExtstring(this);
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitLinkstring(this);
       else return visitor.visitChildren(this);
     }
   }
 
-  public final ExtstringContext extstring() throws RecognitionException {
-    ExtstringContext _localctx = new ExtstringContext(_ctx, getState());
-    enterRule(_localctx, 46, RULE_extstring);
+  public final LinkstringContext linkstring() throws RecognitionException {
+    LinkstringContext _localctx = new LinkstringContext(_ctx, getState());
+    enterRule(_localctx, 46, RULE_linkstring);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(356);
+      setState(353);
       match(STRING);
-      setState(359);
+      setState(356);
       switch ( getInterpreter().adaptivePredict(_input,42,_ctx) ) {
       case 1:
         {
-        setState(357);
-        extdot();
+        setState(354);
+        linkdot();
         }
         break;
       case 2:
         {
-        setState(358);
-        extbrace();
+        setState(355);
+        linkbrace();
         }
         break;
       }
@@ -2678,34 +2685,34 @@ class PainlessParser extends Parser {
       enterOuterAlt(_localctx, 1);
       {
       {
-      setState(361);
+      setState(358);
       match(LP);
-      setState(370);
+      setState(367);
       _la = _input.LA(1);
       if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0)) {
         {
-        setState(362);
+        setState(359);
         expression(0);
-        setState(367);
+        setState(364);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(363);
+          setState(360);
           match(COMMA);
-          setState(364);
+          setState(361);
           expression(0);
           }
           }
-          setState(369);
+          setState(366);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
       }
 
-      setState(372);
+      setState(369);
       match(RP);
       }
       }
@@ -2721,47 +2728,6 @@ class PainlessParser extends Parser {
     return _localctx;
   }
 
-  public static class IncrementContext extends ParserRuleContext {
-    public TerminalNode INCR() { return getToken(PainlessParser.INCR, 0); }
-    public TerminalNode DECR() { return getToken(PainlessParser.DECR, 0); }
-    public IncrementContext(ParserRuleContext parent, int invokingState) {
-      super(parent, invokingState);
-    }
-    @Override public int getRuleIndex() { return RULE_increment; }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitIncrement(this);
-      else return visitor.visitChildren(this);
-    }
-  }
-
-  public final IncrementContext increment() throws RecognitionException {
-    IncrementContext _localctx = new IncrementContext(_ctx, getState());
-    enterRule(_localctx, 50, RULE_increment);
-    int _la;
-    try {
-      enterOuterAlt(_localctx, 1);
-      {
-      setState(374);
-      _la = _input.LA(1);
-      if ( !(_la==INCR || _la==DECR) ) {
-      _errHandler.recoverInline(this);
-      } else {
-        consume();
-      }
-      }
-    }
-    catch (RecognitionException re) {
-      _localctx.exception = re;
-      _errHandler.reportError(this, re);
-      _errHandler.recover(this, re);
-    }
-    finally {
-      exitRule();
-    }
-    return _localctx;
-  }
-
   public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) {
     switch (ruleIndex) {
     case 13:
@@ -2798,152 +2764,150 @@ class PainlessParser extends Parser {
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3K\u017b\4\2\t\2\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3K\u0176\4\2\t\2\4"+
     "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
     "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
     "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+
-    "\4\32\t\32\4\33\t\33\3\2\6\28\n\2\r\2\16\29\3\2\3\2\3\3\3\3\3\3\3\3\3"+
-    "\3\3\3\3\3\5\3E\n\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3M\n\3\3\3\3\3\3\3\3\3\3"+
-    "\3\3\3\3\3\5\3V\n\3\3\3\3\3\3\3\5\3[\n\3\3\3\3\3\5\3_\n\3\3\3\3\3\5\3"+
-    "c\n\3\3\3\3\3\3\3\5\3h\n\3\3\3\3\3\5\3l\n\3\3\3\3\3\5\3p\n\3\3\3\3\3\5"+
-    "\3t\n\3\3\3\3\3\3\3\5\3y\n\3\3\3\3\3\3\3\6\3~\n\3\r\3\16\3\177\3\3\3\3"+
-    "\3\3\5\3\u0085\n\3\3\3\3\3\5\3\u0089\n\3\5\3\u008b\n\3\3\4\3\4\6\4\u008f"+
-    "\n\4\r\4\16\4\u0090\3\4\3\4\3\4\5\4\u0096\n\4\3\5\3\5\5\5\u009a\n\5\3"+
-    "\6\3\6\3\6\3\7\3\7\5\7\u00a1\n\7\3\b\3\b\3\t\3\t\3\t\3\t\7\t\u00a9\n\t"+
-    "\f\t\16\t\u00ac\13\t\3\n\3\n\3\n\7\n\u00b1\n\n\f\n\16\n\u00b4\13\n\3\13"+
-    "\3\13\3\13\5\13\u00b9\n\13\3\f\3\f\3\f\3\f\3\f\3\f\3\f\3\f\5\f\u00c3\n"+
-    "\f\3\r\3\r\5\r\u00c7\n\r\3\16\3\16\3\16\3\16\7\16\u00cd\n\16\f\16\16\16"+
-    "\u00d0\13\16\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3"+
-    "\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3"+
-    "\17\3\17\3\17\3\17\5\17\u00ef\n\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
+    "\4\32\t\32\3\2\6\2\66\n\2\r\2\16\2\67\3\2\3\2\3\3\3\3\3\3\3\3\3\3\3\3"+
+    "\3\3\5\3C\n\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3K\n\3\3\3\3\3\3\3\3\3\3\3\3\3"+
+    "\3\3\5\3T\n\3\3\3\3\3\3\3\5\3Y\n\3\3\3\3\3\5\3]\n\3\3\3\3\3\5\3a\n\3\3"+
+    "\3\3\3\3\3\5\3f\n\3\3\3\3\3\5\3j\n\3\3\3\3\3\5\3n\n\3\3\3\3\3\5\3r\n\3"+
+    "\3\3\3\3\3\3\5\3w\n\3\3\3\3\3\3\3\6\3|\n\3\r\3\16\3}\3\3\3\3\3\3\5\3\u0083"+
+    "\n\3\3\3\3\3\5\3\u0087\n\3\5\3\u0089\n\3\3\4\3\4\6\4\u008d\n\4\r\4\16"+
+    "\4\u008e\3\4\3\4\3\4\5\4\u0094\n\4\3\5\3\5\5\5\u0098\n\5\3\6\3\6\3\6\3"+
+    "\7\3\7\5\7\u009f\n\7\3\b\3\b\3\t\3\t\3\t\3\t\7\t\u00a7\n\t\f\t\16\t\u00aa"+
+    "\13\t\3\n\3\n\3\n\7\n\u00af\n\n\f\n\16\n\u00b2\13\n\3\13\3\13\3\13\5\13"+
+    "\u00b7\n\13\3\f\3\f\3\f\3\f\3\f\3\f\3\f\3\f\5\f\u00c1\n\f\3\r\3\r\5\r"+
+    "\u00c5\n\r\3\16\3\16\3\16\3\16\7\16\u00cb\n\16\f\16\16\16\u00ce\13\16"+
+    "\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
     "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
+    "\5\17\u00ec\n\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
     "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
-    "\3\17\7\17\u0115\n\17\f\17\16\17\u0118\13\17\3\20\3\20\3\20\3\20\3\20"+
-    "\5\20\u011f\n\20\3\21\3\21\3\21\3\21\3\21\3\21\5\21\u0127\n\21\3\21\3"+
-    "\21\3\21\5\21\u012c\n\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\5\22"+
-    "\u0136\n\22\3\23\3\23\3\23\3\23\3\23\5\23\u013d\n\23\3\24\3\24\3\24\5"+
-    "\24\u0142\n\24\3\25\3\25\3\25\3\25\5\25\u0148\n\25\3\26\3\26\3\26\5\26"+
-    "\u014d\n\26\3\27\3\27\3\27\5\27\u0152\n\27\3\30\3\30\3\30\3\30\5\30\u0158"+
-    "\n\30\3\30\3\30\3\30\3\30\6\30\u015e\n\30\r\30\16\30\u015f\3\30\5\30\u0163"+
-    "\n\30\5\30\u0165\n\30\3\31\3\31\3\31\5\31\u016a\n\31\3\32\3\32\3\32\3"+
-    "\32\7\32\u0170\n\32\f\32\16\32\u0173\13\32\5\32\u0175\n\32\3\32\3\32\3"+
-    "\33\3\33\3\33\2\3\34\34\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*"+
-    ",.\60\62\64\2\f\4\2\32\33\37 \3\2\65@\3\2AD\3\2\34\36\3\2\37 \3\2!#\3"+
-    "\2$\'\3\2(+\3\2JK\3\2\63\64\u01b7\2\67\3\2\2\2\4\u008a\3\2\2\2\6\u0095"+
-    "\3\2\2\2\b\u0099\3\2\2\2\n\u009b\3\2\2\2\f\u00a0\3\2\2\2\16\u00a2\3\2"+
-    "\2\2\20\u00a4\3\2\2\2\22\u00ad\3\2\2\2\24\u00b5\3\2\2\2\26\u00ba\3\2\2"+
-    "\2\30\u00c4\3\2\2\2\32\u00c8\3\2\2\2\34\u00ee\3\2\2\2\36\u011e\3\2\2\2"+
-    " \u0120\3\2\2\2\"\u012d\3\2\2\2$\u0137\3\2\2\2&\u013e\3\2\2\2(\u0143\3"+
-    "\2\2\2*\u0149\3\2\2\2,\u014e\3\2\2\2.\u0153\3\2\2\2\60\u0166\3\2\2\2\62"+
-    "\u016b\3\2\2\2\64\u0178\3\2\2\2\668\5\4\3\2\67\66\3\2\2\289\3\2\2\29\67"+
-    "\3\2\2\29:\3\2\2\2:;\3\2\2\2;<\7\2\2\3<\3\3\2\2\2=>\7\16\2\2>?\7\t\2\2"+
-    "?@\5\34\17\2@A\7\n\2\2AD\5\6\4\2BC\7\17\2\2CE\5\6\4\2DB\3\2\2\2DE\3\2"+
-    "\2\2E\u008b\3\2\2\2FG\7\20\2\2GH\7\t\2\2HI\5\34\17\2IL\7\n\2\2JM\5\6\4"+
-    "\2KM\5\b\5\2LJ\3\2\2\2LK\3\2\2\2M\u008b\3\2\2\2NO\7\21\2\2OP\5\6\4\2P"+
-    "Q\7\20\2\2QR\7\t\2\2RS\5\34\17\2SU\7\n\2\2TV\7\r\2\2UT\3\2\2\2UV\3\2\2"+
-    "\2V\u008b\3\2\2\2WX\7\22\2\2XZ\7\t\2\2Y[\5\f\7\2ZY\3\2\2\2Z[\3\2\2\2["+
-    "\\\3\2\2\2\\^\7\r\2\2]_\5\34\17\2^]\3\2\2\2^_\3\2\2\2_`\3\2\2\2`b\7\r"+
-    "\2\2ac\5\16\b\2ba\3\2\2\2bc\3\2\2\2cd\3\2\2\2dg\7\n\2\2eh\5\6\4\2fh\5"+
-    "\b\5\2ge\3\2\2\2gf\3\2\2\2h\u008b\3\2\2\2ik\5\20\t\2jl\7\r\2\2kj\3\2\2"+
-    "\2kl\3\2\2\2l\u008b\3\2\2\2mo\7\23\2\2np\7\r\2\2on\3\2\2\2op\3\2\2\2p"+
-    "\u008b\3\2\2\2qs\7\24\2\2rt\7\r\2\2sr\3\2\2\2st\3\2\2\2t\u008b\3\2\2\2"+
-    "uv\7\25\2\2vx\5\34\17\2wy\7\r\2\2xw\3\2\2\2xy\3\2\2\2y\u008b\3\2\2\2z"+
-    "{\7\27\2\2{}\5\6\4\2|~\5\26\f\2}|\3\2\2\2~\177\3\2\2\2\177}\3\2\2\2\177"+
-    "\u0080\3\2\2\2\u0080\u008b\3\2\2\2\u0081\u0082\7\31\2\2\u0082\u0084\5"+
-    "\34\17\2\u0083\u0085\7\r\2\2\u0084\u0083\3\2\2\2\u0084\u0085\3\2\2\2\u0085"+
-    "\u008b\3\2\2\2\u0086\u0088\5\34\17\2\u0087\u0089\7\r\2\2\u0088\u0087\3"+
-    "\2\2\2\u0088\u0089\3\2\2\2\u0089\u008b\3\2\2\2\u008a=\3\2\2\2\u008aF\3"+
-    "\2\2\2\u008aN\3\2\2\2\u008aW\3\2\2\2\u008ai\3\2\2\2\u008am\3\2\2\2\u008a"+
-    "q\3\2\2\2\u008au\3\2\2\2\u008az\3\2\2\2\u008a\u0081\3\2\2\2\u008a\u0086"+
-    "\3\2\2\2\u008b\5\3\2\2\2\u008c\u008e\7\5\2\2\u008d\u008f\5\4\3\2\u008e"+
-    "\u008d\3\2\2\2\u008f\u0090\3\2\2\2\u0090\u008e\3\2\2\2\u0090\u0091\3\2"+
-    "\2\2\u0091\u0092\3\2\2\2\u0092\u0093\7\6\2\2\u0093\u0096\3\2\2\2\u0094"+
-    "\u0096\5\4\3\2\u0095\u008c\3\2\2\2\u0095\u0094\3\2\2\2\u0096\7\3\2\2\2"+
-    "\u0097\u009a\5\n\6\2\u0098\u009a\7\r\2\2\u0099\u0097\3\2\2\2\u0099\u0098"+
-    "\3\2\2\2\u009a\t\3\2\2\2\u009b\u009c\7\5\2\2\u009c\u009d\7\6\2\2\u009d"+
-    "\13\3\2\2\2\u009e\u00a1\5\20\t\2\u009f\u00a1\5\34\17\2\u00a0\u009e\3\2"+
-    "\2\2\u00a0\u009f\3\2\2\2\u00a1\r\3\2\2\2\u00a2\u00a3\5\34\17\2\u00a3\17"+
-    "\3\2\2\2\u00a4\u00a5\5\22\n\2\u00a5\u00aa\5\24\13\2\u00a6\u00a7\7\f\2"+
-    "\2\u00a7\u00a9\5\24\13\2\u00a8\u00a6\3\2\2\2\u00a9\u00ac\3\2\2\2\u00aa"+
-    "\u00a8\3\2\2\2\u00aa\u00ab\3\2\2\2\u00ab\21\3\2\2\2\u00ac\u00aa\3\2\2"+
-    "\2\u00ad\u00b2\5\30\r\2\u00ae\u00af\7\7\2\2\u00af\u00b1\7\b\2\2\u00b0"+
-    "\u00ae\3\2\2\2\u00b1\u00b4\3\2\2\2\u00b2\u00b0\3\2\2\2\u00b2\u00b3\3\2"+
-    "\2\2\u00b3\23\3\2\2\2\u00b4\u00b2\3\2\2\2\u00b5\u00b8\5\30\r\2\u00b6\u00b7"+
-    "\7\65\2\2\u00b7\u00b9\5\34\17\2\u00b8\u00b6\3\2\2\2\u00b8\u00b9\3\2\2"+
-    "\2\u00b9\25\3\2\2\2\u00ba\u00bb\7\30\2\2\u00bb\u00bc\7\t\2\2\u00bc\u00bd"+
-    "\5\30\r\2\u00bd\u00be\5\30\r\2\u00be\u00bf\3\2\2\2\u00bf\u00c2\7\n\2\2"+
-    "\u00c0\u00c3\5\6\4\2\u00c1\u00c3\5\n\6\2\u00c2\u00c0\3\2\2\2\u00c2\u00c1"+
-    "\3\2\2\2\u00c3\27\3\2\2\2\u00c4\u00c6\7I\2\2\u00c5\u00c7\5\32\16\2\u00c6"+
-    "\u00c5\3\2\2\2\u00c6\u00c7\3\2\2\2\u00c7\31\3\2\2\2\u00c8\u00c9\7$\2\2"+
-    "\u00c9\u00ce\5\30\r\2\u00ca\u00cb\7\f\2\2\u00cb\u00cd\5\30\r\2\u00cc\u00ca"+
-    "\3\2\2\2\u00cd\u00d0\3\2\2\2\u00ce\u00cc\3\2\2\2\u00ce\u00cf\3\2\2\2\u00cf"+
-    "\u00d1\3\2\2\2\u00d0\u00ce\3\2\2\2\u00d1\u00d2\7&\2\2\u00d2\33\3\2\2\2"+
-    "\u00d3\u00d4\b\17\1\2\u00d4\u00d5\t\2\2\2\u00d5\u00ef\5\34\17\20\u00d6"+
-    "\u00d7\7\t\2\2\u00d7\u00d8\5\22\n\2\u00d8\u00d9\7\n\2\2\u00d9\u00da\5"+
-    "\34\17\17\u00da\u00ef\3\2\2\2\u00db\u00dc\5\36\20\2\u00dc\u00dd\t\3\2"+
-    "\2\u00dd\u00de\5\34\17\3\u00de\u00ef\3\2\2\2\u00df\u00e0\7\t\2\2\u00e0"+
-    "\u00e1\5\34\17\2\u00e1\u00e2\7\n\2\2\u00e2\u00ef\3\2\2\2\u00e3\u00ef\t"+
-    "\4\2\2\u00e4\u00ef\7F\2\2\u00e5\u00ef\7G\2\2\u00e6\u00ef\7H\2\2\u00e7"+
-    "\u00e8\5\36\20\2\u00e8\u00e9\5\64\33\2\u00e9\u00ef\3\2\2\2\u00ea\u00eb"+
-    "\5\64\33\2\u00eb\u00ec\5\36\20\2\u00ec\u00ef\3\2\2\2\u00ed\u00ef\5\36"+
-    "\20\2\u00ee\u00d3\3\2\2\2\u00ee\u00d6\3\2\2\2\u00ee\u00db\3\2\2\2\u00ee"+
-    "\u00df\3\2\2\2\u00ee\u00e3\3\2\2\2\u00ee\u00e4\3\2\2\2\u00ee\u00e5\3\2"+
-    "\2\2\u00ee\u00e6\3\2\2\2\u00ee\u00e7\3\2\2\2\u00ee\u00ea\3\2\2\2\u00ee"+
-    "\u00ed\3\2\2\2\u00ef\u0116\3\2\2\2\u00f0\u00f1\f\16\2\2\u00f1\u00f2\t"+
-    "\5\2\2\u00f2\u0115\5\34\17\17\u00f3\u00f4\f\r\2\2\u00f4\u00f5\t\6\2\2"+
-    "\u00f5\u0115\5\34\17\16\u00f6\u00f7\f\f\2\2\u00f7\u00f8\t\7\2\2\u00f8"+
-    "\u0115\5\34\17\r\u00f9\u00fa\f\13\2\2\u00fa\u00fb\t\b\2\2\u00fb\u0115"+
-    "\5\34\17\f\u00fc\u00fd\f\n\2\2\u00fd\u00fe\t\t\2\2\u00fe\u0115\5\34\17"+
-    "\13\u00ff\u0100\f\t\2\2\u0100\u0101\7,\2\2\u0101\u0115\5\34\17\n\u0102"+
-    "\u0103\f\b\2\2\u0103\u0104\7-\2\2\u0104\u0115\5\34\17\t\u0105\u0106\f"+
-    "\7\2\2\u0106\u0107\7.\2\2\u0107\u0115\5\34\17\b\u0108\u0109\f\6\2\2\u0109"+
-    "\u010a\7/\2\2\u010a\u0115\5\34\17\7\u010b\u010c\f\5\2\2\u010c\u010d\7"+
-    "\60\2\2\u010d\u0115\5\34\17\6\u010e\u010f\f\4\2\2\u010f\u0110\7\61\2\2"+
-    "\u0110\u0111\5\34\17\2\u0111\u0112\7\62\2\2\u0112\u0113\5\34\17\4\u0113"+
-    "\u0115\3\2\2\2\u0114\u00f0\3\2\2\2\u0114\u00f3\3\2\2\2\u0114\u00f6\3\2"+
-    "\2\2\u0114\u00f9\3\2\2\2\u0114\u00fc\3\2\2\2\u0114\u00ff\3\2\2\2\u0114"+
-    "\u0102\3\2\2\2\u0114\u0105\3\2\2\2\u0114\u0108\3\2\2\2\u0114\u010b\3\2"+
-    "\2\2\u0114\u010e\3\2\2\2\u0115\u0118\3\2\2\2\u0116\u0114\3\2\2\2\u0116"+
-    "\u0117\3\2\2\2\u0117\35\3\2\2\2\u0118\u0116\3\2\2\2\u0119\u011f\5 \21"+
-    "\2\u011a\u011f\5\"\22\2\u011b\u011f\5*\26\2\u011c\u011f\5.\30\2\u011d"+
-    "\u011f\5\60\31\2\u011e\u0119\3\2\2\2\u011e\u011a\3\2\2\2\u011e\u011b\3"+
-    "\2\2\2\u011e\u011c\3\2\2\2\u011e\u011d\3\2\2\2\u011f\37\3\2\2\2\u0120"+
-    "\u0126\7\t\2\2\u0121\u0127\5 \21\2\u0122\u0127\5\"\22\2\u0123\u0127\5"+
-    "*\26\2\u0124\u0127\5.\30\2\u0125\u0127\5\60\31\2\u0126\u0121\3\2\2\2\u0126"+
-    "\u0122\3\2\2\2\u0126\u0123\3\2\2\2\u0126\u0124\3\2\2\2\u0126\u0125\3\2"+
-    "\2\2\u0127\u0128\3\2\2\2\u0128\u012b\7\n\2\2\u0129\u012c\5&\24\2\u012a"+
-    "\u012c\5$\23\2\u012b\u0129\3\2\2\2\u012b\u012a\3\2\2\2\u012b\u012c\3\2"+
-    "\2\2\u012c!\3\2\2\2\u012d\u012e\7\t\2\2\u012e\u012f\5\22\n\2\u012f\u0135"+
-    "\7\n\2\2\u0130\u0136\5 \21\2\u0131\u0136\5\"\22\2\u0132\u0136\5*\26\2"+
-    "\u0133\u0136\5.\30\2\u0134\u0136\5\60\31\2\u0135\u0130\3\2\2\2\u0135\u0131"+
-    "\3\2\2\2\u0135\u0132\3\2\2\2\u0135\u0133\3\2\2\2\u0135\u0134\3\2\2\2\u0136"+
-    "#\3\2\2\2\u0137\u0138\7\7\2\2\u0138\u0139\5\34\17\2\u0139\u013c\7\b\2"+
-    "\2\u013a\u013d\5&\24\2\u013b\u013d\5$\23\2\u013c\u013a\3\2\2\2\u013c\u013b"+
-    "\3\2\2\2\u013c\u013d\3\2\2\2\u013d%\3\2\2\2\u013e\u0141\7\13\2\2\u013f"+
-    "\u0142\5(\25\2\u0140\u0142\5,\27\2\u0141\u013f\3\2\2\2\u0141\u0140\3\2"+
-    "\2\2\u0142\'\3\2\2\2\u0143\u0144\7K\2\2\u0144\u0147\5\62\32\2\u0145\u0148"+
-    "\5&\24\2\u0146\u0148\5$\23\2\u0147\u0145\3\2\2\2\u0147\u0146\3\2\2\2\u0147"+
-    "\u0148\3\2\2\2\u0148)\3\2\2\2\u0149\u014c\5\30\r\2\u014a\u014d\5&\24\2"+
-    "\u014b\u014d\5$\23\2\u014c\u014a\3\2\2\2\u014c\u014b\3\2\2\2\u014c\u014d"+
-    "\3\2\2\2\u014d+\3\2\2\2\u014e\u0151\t\n\2\2\u014f\u0152\5&\24\2\u0150"+
-    "\u0152\5$\23\2\u0151\u014f\3\2\2\2\u0151\u0150\3\2\2\2\u0151\u0152\3\2"+
-    "\2\2\u0152-\3\2\2\2\u0153\u0154\7\26\2\2\u0154\u0164\5\30\r\2\u0155\u0157"+
-    "\5\62\32\2\u0156\u0158\5&\24\2\u0157\u0156\3\2\2\2\u0157\u0158\3\2\2\2"+
-    "\u0158\u0165\3\2\2\2\u0159\u015a\7\7\2\2\u015a\u015b\5\34\17\2\u015b\u015c"+
-    "\7\b\2\2\u015c\u015e\3\2\2\2\u015d\u0159\3\2\2\2\u015e\u015f\3\2\2\2\u015f"+
-    "\u015d\3\2\2\2\u015f\u0160\3\2\2\2\u0160\u0162\3\2\2\2\u0161\u0163\5&"+
-    "\24\2\u0162\u0161\3\2\2\2\u0162\u0163\3\2\2\2\u0163\u0165\3\2\2\2\u0164"+
-    "\u0155\3\2\2\2\u0164\u015d\3\2\2\2\u0165/\3\2\2\2\u0166\u0169\7E\2\2\u0167"+
-    "\u016a\5&\24\2\u0168\u016a\5$\23\2\u0169\u0167\3\2\2\2\u0169\u0168\3\2"+
-    "\2\2\u0169\u016a\3\2\2\2\u016a\61\3\2\2\2\u016b\u0174\7\t\2\2\u016c\u0171"+
-    "\5\34\17\2\u016d\u016e\7\f\2\2\u016e\u0170\5\34\17\2\u016f\u016d\3\2\2"+
-    "\2\u0170\u0173\3\2\2\2\u0171\u016f\3\2\2\2\u0171\u0172\3\2\2\2\u0172\u0175"+
-    "\3\2\2\2\u0173\u0171\3\2\2\2\u0174\u016c\3\2\2\2\u0174\u0175\3\2\2\2\u0175"+
-    "\u0176\3\2\2\2\u0176\u0177\7\n\2\2\u0177\63\3\2\2\2\u0178\u0179\t\13\2"+
-    "\2\u0179\65\3\2\2\2/9DLUZ^bgkosx\177\u0084\u0088\u008a\u0090\u0095\u0099"+
-    "\u00a0\u00aa\u00b2\u00b8\u00c2\u00c6\u00ce\u00ee\u0114\u0116\u011e\u0126"+
-    "\u012b\u0135\u013c\u0141\u0147\u014c\u0151\u0157\u015f\u0162\u0164\u0169"+
-    "\u0171\u0174";
+    "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\7\17\u0112\n\17"+
+    "\f\17\16\17\u0115\13\17\3\20\3\20\3\20\3\20\3\20\5\20\u011c\n\20\3\21"+
+    "\3\21\3\21\3\21\3\21\3\21\5\21\u0124\n\21\3\21\3\21\3\21\5\21\u0129\n"+
+    "\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\5\22\u0133\n\22\3\23\3\23"+
+    "\3\23\3\23\3\23\5\23\u013a\n\23\3\24\3\24\3\24\5\24\u013f\n\24\3\25\3"+
+    "\25\3\25\3\25\5\25\u0145\n\25\3\26\3\26\3\26\5\26\u014a\n\26\3\27\3\27"+
+    "\3\27\5\27\u014f\n\27\3\30\3\30\3\30\3\30\5\30\u0155\n\30\3\30\3\30\3"+
+    "\30\3\30\6\30\u015b\n\30\r\30\16\30\u015c\3\30\5\30\u0160\n\30\5\30\u0162"+
+    "\n\30\3\31\3\31\3\31\5\31\u0167\n\31\3\32\3\32\3\32\3\32\7\32\u016d\n"+
+    "\32\f\32\16\32\u0170\13\32\5\32\u0172\n\32\3\32\3\32\3\32\2\3\34\33\2"+
+    "\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,.\60\62\2\f\4\2\32\33\37"+
+    " \3\2\65@\3\2AD\3\2\63\64\3\2\34\36\3\2\37 \3\2!#\3\2$\'\3\2(+\3\2JK\u01b3"+
+    "\2\65\3\2\2\2\4\u0088\3\2\2\2\6\u0093\3\2\2\2\b\u0097\3\2\2\2\n\u0099"+
+    "\3\2\2\2\f\u009e\3\2\2\2\16\u00a0\3\2\2\2\20\u00a2\3\2\2\2\22\u00ab\3"+
+    "\2\2\2\24\u00b3\3\2\2\2\26\u00b8\3\2\2\2\30\u00c2\3\2\2\2\32\u00c6\3\2"+
+    "\2\2\34\u00eb\3\2\2\2\36\u011b\3\2\2\2 \u011d\3\2\2\2\"\u012a\3\2\2\2"+
+    "$\u0134\3\2\2\2&\u013b\3\2\2\2(\u0140\3\2\2\2*\u0146\3\2\2\2,\u014b\3"+
+    "\2\2\2.\u0150\3\2\2\2\60\u0163\3\2\2\2\62\u0168\3\2\2\2\64\66\5\4\3\2"+
+    "\65\64\3\2\2\2\66\67\3\2\2\2\67\65\3\2\2\2\678\3\2\2\289\3\2\2\29:\7\2"+
+    "\2\3:\3\3\2\2\2;<\7\16\2\2<=\7\t\2\2=>\5\34\17\2>?\7\n\2\2?B\5\6\4\2@"+
+    "A\7\17\2\2AC\5\6\4\2B@\3\2\2\2BC\3\2\2\2C\u0089\3\2\2\2DE\7\20\2\2EF\7"+
+    "\t\2\2FG\5\34\17\2GJ\7\n\2\2HK\5\6\4\2IK\5\b\5\2JH\3\2\2\2JI\3\2\2\2K"+
+    "\u0089\3\2\2\2LM\7\21\2\2MN\5\6\4\2NO\7\20\2\2OP\7\t\2\2PQ\5\34\17\2Q"+
+    "S\7\n\2\2RT\7\r\2\2SR\3\2\2\2ST\3\2\2\2T\u0089\3\2\2\2UV\7\22\2\2VX\7"+
+    "\t\2\2WY\5\f\7\2XW\3\2\2\2XY\3\2\2\2YZ\3\2\2\2Z\\\7\r\2\2[]\5\34\17\2"+
+    "\\[\3\2\2\2\\]\3\2\2\2]^\3\2\2\2^`\7\r\2\2_a\5\16\b\2`_\3\2\2\2`a\3\2"+
+    "\2\2ab\3\2\2\2be\7\n\2\2cf\5\6\4\2df\5\b\5\2ec\3\2\2\2ed\3\2\2\2f\u0089"+
+    "\3\2\2\2gi\5\20\t\2hj\7\r\2\2ih\3\2\2\2ij\3\2\2\2j\u0089\3\2\2\2km\7\23"+
+    "\2\2ln\7\r\2\2ml\3\2\2\2mn\3\2\2\2n\u0089\3\2\2\2oq\7\24\2\2pr\7\r\2\2"+
+    "qp\3\2\2\2qr\3\2\2\2r\u0089\3\2\2\2st\7\25\2\2tv\5\34\17\2uw\7\r\2\2v"+
+    "u\3\2\2\2vw\3\2\2\2w\u0089\3\2\2\2xy\7\27\2\2y{\5\6\4\2z|\5\26\f\2{z\3"+
+    "\2\2\2|}\3\2\2\2}{\3\2\2\2}~\3\2\2\2~\u0089\3\2\2\2\177\u0080\7\31\2\2"+
+    "\u0080\u0082\5\34\17\2\u0081\u0083\7\r\2\2\u0082\u0081\3\2\2\2\u0082\u0083"+
+    "\3\2\2\2\u0083\u0089\3\2\2\2\u0084\u0086\5\34\17\2\u0085\u0087\7\r\2\2"+
+    "\u0086\u0085\3\2\2\2\u0086\u0087\3\2\2\2\u0087\u0089\3\2\2\2\u0088;\3"+
+    "\2\2\2\u0088D\3\2\2\2\u0088L\3\2\2\2\u0088U\3\2\2\2\u0088g\3\2\2\2\u0088"+
+    "k\3\2\2\2\u0088o\3\2\2\2\u0088s\3\2\2\2\u0088x\3\2\2\2\u0088\177\3\2\2"+
+    "\2\u0088\u0084\3\2\2\2\u0089\5\3\2\2\2\u008a\u008c\7\5\2\2\u008b\u008d"+
+    "\5\4\3\2\u008c\u008b\3\2\2\2\u008d\u008e\3\2\2\2\u008e\u008c\3\2\2\2\u008e"+
+    "\u008f\3\2\2\2\u008f\u0090\3\2\2\2\u0090\u0091\7\6\2\2\u0091\u0094\3\2"+
+    "\2\2\u0092\u0094\5\4\3\2\u0093\u008a\3\2\2\2\u0093\u0092\3\2\2\2\u0094"+
+    "\7\3\2\2\2\u0095\u0098\5\n\6\2\u0096\u0098\7\r\2\2\u0097\u0095\3\2\2\2"+
+    "\u0097\u0096\3\2\2\2\u0098\t\3\2\2\2\u0099\u009a\7\5\2\2\u009a\u009b\7"+
+    "\6\2\2\u009b\13\3\2\2\2\u009c\u009f\5\20\t\2\u009d\u009f\5\34\17\2\u009e"+
+    "\u009c\3\2\2\2\u009e\u009d\3\2\2\2\u009f\r\3\2\2\2\u00a0\u00a1\5\34\17"+
+    "\2\u00a1\17\3\2\2\2\u00a2\u00a3\5\22\n\2\u00a3\u00a8\5\24\13\2\u00a4\u00a5"+
+    "\7\f\2\2\u00a5\u00a7\5\24\13\2\u00a6\u00a4\3\2\2\2\u00a7\u00aa\3\2\2\2"+
+    "\u00a8\u00a6\3\2\2\2\u00a8\u00a9\3\2\2\2\u00a9\21\3\2\2\2\u00aa\u00a8"+
+    "\3\2\2\2\u00ab\u00b0\5\30\r\2\u00ac\u00ad\7\7\2\2\u00ad\u00af\7\b\2\2"+
+    "\u00ae\u00ac\3\2\2\2\u00af\u00b2\3\2\2\2\u00b0\u00ae\3\2\2\2\u00b0\u00b1"+
+    "\3\2\2\2\u00b1\23\3\2\2\2\u00b2\u00b0\3\2\2\2\u00b3\u00b6\5\30\r\2\u00b4"+
+    "\u00b5\7\65\2\2\u00b5\u00b7\5\34\17\2\u00b6\u00b4\3\2\2\2\u00b6\u00b7"+
+    "\3\2\2\2\u00b7\25\3\2\2\2\u00b8\u00b9\7\30\2\2\u00b9\u00ba\7\t\2\2\u00ba"+
+    "\u00bb\5\30\r\2\u00bb\u00bc\5\30\r\2\u00bc\u00bd\3\2\2\2\u00bd\u00c0\7"+
+    "\n\2\2\u00be\u00c1\5\6\4\2\u00bf\u00c1\5\n\6\2\u00c0\u00be\3\2\2\2\u00c0"+
+    "\u00bf\3\2\2\2\u00c1\27\3\2\2\2\u00c2\u00c4\7I\2\2\u00c3\u00c5\5\32\16"+
+    "\2\u00c4\u00c3\3\2\2\2\u00c4\u00c5\3\2\2\2\u00c5\31\3\2\2\2\u00c6\u00c7"+
+    "\7$\2\2\u00c7\u00cc\5\30\r\2\u00c8\u00c9\7\f\2\2\u00c9\u00cb\5\30\r\2"+
+    "\u00ca\u00c8\3\2\2\2\u00cb\u00ce\3\2\2\2\u00cc\u00ca\3\2\2\2\u00cc\u00cd"+
+    "\3\2\2\2\u00cd\u00cf\3\2\2\2\u00ce\u00cc\3\2\2\2\u00cf\u00d0\7&\2\2\u00d0"+
+    "\33\3\2\2\2\u00d1\u00d2\b\17\1\2\u00d2\u00d3\t\2\2\2\u00d3\u00ec\5\34"+
+    "\17\20\u00d4\u00d5\7\t\2\2\u00d5\u00d6\5\22\n\2\u00d6\u00d7\7\n\2\2\u00d7"+
+    "\u00d8\5\34\17\17\u00d8\u00ec\3\2\2\2\u00d9\u00da\5\36\20\2\u00da\u00db"+
+    "\t\3\2\2\u00db\u00dc\5\34\17\3\u00dc\u00ec\3\2\2\2\u00dd\u00de\7\t\2\2"+
+    "\u00de\u00df\5\34\17\2\u00df\u00e0\7\n\2\2\u00e0\u00ec\3\2\2\2\u00e1\u00ec"+
+    "\t\4\2\2\u00e2\u00ec\7F\2\2\u00e3\u00ec\7G\2\2\u00e4\u00ec\7H\2\2\u00e5"+
+    "\u00e6\5\36\20\2\u00e6\u00e7\t\5\2\2\u00e7\u00ec\3\2\2\2\u00e8\u00e9\t"+
+    "\5\2\2\u00e9\u00ec\5\36\20\2\u00ea\u00ec\5\36\20\2\u00eb\u00d1\3\2\2\2"+
+    "\u00eb\u00d4\3\2\2\2\u00eb\u00d9\3\2\2\2\u00eb\u00dd\3\2\2\2\u00eb\u00e1"+
+    "\3\2\2\2\u00eb\u00e2\3\2\2\2\u00eb\u00e3\3\2\2\2\u00eb\u00e4\3\2\2\2\u00eb"+
+    "\u00e5\3\2\2\2\u00eb\u00e8\3\2\2\2\u00eb\u00ea\3\2\2\2\u00ec\u0113\3\2"+
+    "\2\2\u00ed\u00ee\f\16\2\2\u00ee\u00ef\t\6\2\2\u00ef\u0112\5\34\17\17\u00f0"+
+    "\u00f1\f\r\2\2\u00f1\u00f2\t\7\2\2\u00f2\u0112\5\34\17\16\u00f3\u00f4"+
+    "\f\f\2\2\u00f4\u00f5\t\b\2\2\u00f5\u0112\5\34\17\r\u00f6\u00f7\f\13\2"+
+    "\2\u00f7\u00f8\t\t\2\2\u00f8\u0112\5\34\17\f\u00f9\u00fa\f\n\2\2\u00fa"+
+    "\u00fb\t\n\2\2\u00fb\u0112\5\34\17\13\u00fc\u00fd\f\t\2\2\u00fd\u00fe"+
+    "\7,\2\2\u00fe\u0112\5\34\17\n\u00ff\u0100\f\b\2\2\u0100\u0101\7-\2\2\u0101"+
+    "\u0112\5\34\17\t\u0102\u0103\f\7\2\2\u0103\u0104\7.\2\2\u0104\u0112\5"+
+    "\34\17\b\u0105\u0106\f\6\2\2\u0106\u0107\7/\2\2\u0107\u0112\5\34\17\7"+
+    "\u0108\u0109\f\5\2\2\u0109\u010a\7\60\2\2\u010a\u0112\5\34\17\6\u010b"+
+    "\u010c\f\4\2\2\u010c\u010d\7\61\2\2\u010d\u010e\5\34\17\2\u010e\u010f"+
+    "\7\62\2\2\u010f\u0110\5\34\17\4\u0110\u0112\3\2\2\2\u0111\u00ed\3\2\2"+
+    "\2\u0111\u00f0\3\2\2\2\u0111\u00f3\3\2\2\2\u0111\u00f6\3\2\2\2\u0111\u00f9"+
+    "\3\2\2\2\u0111\u00fc\3\2\2\2\u0111\u00ff\3\2\2\2\u0111\u0102\3\2\2\2\u0111"+
+    "\u0105\3\2\2\2\u0111\u0108\3\2\2\2\u0111\u010b\3\2\2\2\u0112\u0115\3\2"+
+    "\2\2\u0113\u0111\3\2\2\2\u0113\u0114\3\2\2\2\u0114\35\3\2\2\2\u0115\u0113"+
+    "\3\2\2\2\u0116\u011c\5 \21\2\u0117\u011c\5\"\22\2\u0118\u011c\5*\26\2"+
+    "\u0119\u011c\5.\30\2\u011a\u011c\5\60\31\2\u011b\u0116\3\2\2\2\u011b\u0117"+
+    "\3\2\2\2\u011b\u0118\3\2\2\2\u011b\u0119\3\2\2\2\u011b\u011a\3\2\2\2\u011c"+
+    "\37\3\2\2\2\u011d\u0123\7\t\2\2\u011e\u0124\5 \21\2\u011f\u0124\5\"\22"+
+    "\2\u0120\u0124\5*\26\2\u0121\u0124\5.\30\2\u0122\u0124\5\60\31\2\u0123"+
+    "\u011e\3\2\2\2\u0123\u011f\3\2\2\2\u0123\u0120\3\2\2\2\u0123\u0121\3\2"+
+    "\2\2\u0123\u0122\3\2\2\2\u0124\u0125\3\2\2\2\u0125\u0128\7\n\2\2\u0126"+
+    "\u0129\5&\24\2\u0127\u0129\5$\23\2\u0128\u0126\3\2\2\2\u0128\u0127\3\2"+
+    "\2\2\u0128\u0129\3\2\2\2\u0129!\3\2\2\2\u012a\u012b\7\t\2\2\u012b\u012c"+
+    "\5\22\n\2\u012c\u0132\7\n\2\2\u012d\u0133\5 \21\2\u012e\u0133\5\"\22\2"+
+    "\u012f\u0133\5*\26\2\u0130\u0133\5.\30\2\u0131\u0133\5\60\31\2\u0132\u012d"+
+    "\3\2\2\2\u0132\u012e\3\2\2\2\u0132\u012f\3\2\2\2\u0132\u0130\3\2\2\2\u0132"+
+    "\u0131\3\2\2\2\u0133#\3\2\2\2\u0134\u0135\7\7\2\2\u0135\u0136\5\34\17"+
+    "\2\u0136\u0139\7\b\2\2\u0137\u013a\5&\24\2\u0138\u013a\5$\23\2\u0139\u0137"+
+    "\3\2\2\2\u0139\u0138\3\2\2\2\u0139\u013a\3\2\2\2\u013a%\3\2\2\2\u013b"+
+    "\u013e\7\13\2\2\u013c\u013f\5(\25\2\u013d\u013f\5,\27\2\u013e\u013c\3"+
+    "\2\2\2\u013e\u013d\3\2\2\2\u013f\'\3\2\2\2\u0140\u0141\7K\2\2\u0141\u0144"+
+    "\5\62\32\2\u0142\u0145\5&\24\2\u0143\u0145\5$\23\2\u0144\u0142\3\2\2\2"+
+    "\u0144\u0143\3\2\2\2\u0144\u0145\3\2\2\2\u0145)\3\2\2\2\u0146\u0149\5"+
+    "\30\r\2\u0147\u014a\5&\24\2\u0148\u014a\5$\23\2\u0149\u0147\3\2\2\2\u0149"+
+    "\u0148\3\2\2\2\u0149\u014a\3\2\2\2\u014a+\3\2\2\2\u014b\u014e\t\13\2\2"+
+    "\u014c\u014f\5&\24\2\u014d\u014f\5$\23\2\u014e\u014c\3\2\2\2\u014e\u014d"+
+    "\3\2\2\2\u014e\u014f\3\2\2\2\u014f-\3\2\2\2\u0150\u0151\7\26\2\2\u0151"+
+    "\u0161\5\30\r\2\u0152\u0154\5\62\32\2\u0153\u0155\5&\24\2\u0154\u0153"+
+    "\3\2\2\2\u0154\u0155\3\2\2\2\u0155\u0162\3\2\2\2\u0156\u0157\7\7\2\2\u0157"+
+    "\u0158\5\34\17\2\u0158\u0159\7\b\2\2\u0159\u015b\3\2\2\2\u015a\u0156\3"+
+    "\2\2\2\u015b\u015c\3\2\2\2\u015c\u015a\3\2\2\2\u015c\u015d\3\2\2\2\u015d"+
+    "\u015f\3\2\2\2\u015e\u0160\5&\24\2\u015f\u015e\3\2\2\2\u015f\u0160\3\2"+
+    "\2\2\u0160\u0162\3\2\2\2\u0161\u0152\3\2\2\2\u0161\u015a\3\2\2\2\u0162"+
+    "/\3\2\2\2\u0163\u0166\7E\2\2\u0164\u0167\5&\24\2\u0165\u0167\5$\23\2\u0166"+
+    "\u0164\3\2\2\2\u0166\u0165\3\2\2\2\u0166\u0167\3\2\2\2\u0167\61\3\2\2"+
+    "\2\u0168\u0171\7\t\2\2\u0169\u016e\5\34\17\2\u016a\u016b\7\f\2\2\u016b"+
+    "\u016d\5\34\17\2\u016c\u016a\3\2\2\2\u016d\u0170\3\2\2\2\u016e\u016c\3"+
+    "\2\2\2\u016e\u016f\3\2\2\2\u016f\u0172\3\2\2\2\u0170\u016e\3\2\2\2\u0171"+
+    "\u0169\3\2\2\2\u0171\u0172\3\2\2\2\u0172\u0173\3\2\2\2\u0173\u0174\7\n"+
+    "\2\2\u0174\63\3\2\2\2/\67BJSX\\`eimqv}\u0082\u0086\u0088\u008e\u0093\u0097"+
+    "\u009e\u00a8\u00b0\u00b6\u00c0\u00c4\u00cc\u00eb\u0111\u0113\u011b\u0123"+
+    "\u0128\u0132\u0139\u013e\u0144\u0149\u014e\u0154\u015c\u015f\u0161\u0166"+
+    "\u016e\u0171";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessParserBaseVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
similarity index 91%
rename from modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessParserBaseVisitor.java
rename to modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
index 824b83fe952..dad6e419e25 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessParserBaseVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
@@ -1,5 +1,5 @@
 // ANTLR GENERATED CODE: DO NOT EDIT
-package org.elasticsearch.painless;
+package org.elasticsearch.painless.antlr;
 import org.antlr.v4.runtime.tree.AbstractParseTreeVisitor;
 
 /**
@@ -186,6 +186,13 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitComp(PainlessParser.CompContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitRead(PainlessParser.ReadContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
@@ -256,13 +263,6 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitCast(PainlessParser.CastContext ctx) { return visitChildren(ctx); }
-  /**
-   * {@inheritDoc}
-   *
-   * <p>The default implementation returns the result of calling
-   * {@link #visitChildren} on {@code ctx}.</p>
-   */
-  @Override public T visitExternal(PainlessParser.ExternalContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
@@ -290,70 +290,70 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitExtstart(PainlessParser.ExtstartContext ctx) { return visitChildren(ctx); }
+  @Override public T visitChain(PainlessParser.ChainContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitExtprec(PainlessParser.ExtprecContext ctx) { return visitChildren(ctx); }
+  @Override public T visitLinkprec(PainlessParser.LinkprecContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitExtcast(PainlessParser.ExtcastContext ctx) { return visitChildren(ctx); }
+  @Override public T visitLinkcast(PainlessParser.LinkcastContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitExtbrace(PainlessParser.ExtbraceContext ctx) { return visitChildren(ctx); }
+  @Override public T visitLinkbrace(PainlessParser.LinkbraceContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitExtdot(PainlessParser.ExtdotContext ctx) { return visitChildren(ctx); }
+  @Override public T visitLinkdot(PainlessParser.LinkdotContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitExtcall(PainlessParser.ExtcallContext ctx) { return visitChildren(ctx); }
+  @Override public T visitLinkcall(PainlessParser.LinkcallContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitExtvar(PainlessParser.ExtvarContext ctx) { return visitChildren(ctx); }
+  @Override public T visitLinkvar(PainlessParser.LinkvarContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitExtfield(PainlessParser.ExtfieldContext ctx) { return visitChildren(ctx); }
+  @Override public T visitLinkfield(PainlessParser.LinkfieldContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitExtnew(PainlessParser.ExtnewContext ctx) { return visitChildren(ctx); }
+  @Override public T visitLinknew(PainlessParser.LinknewContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitExtstring(PainlessParser.ExtstringContext ctx) { return visitChildren(ctx); }
+  @Override public T visitLinkstring(PainlessParser.LinkstringContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
@@ -361,11 +361,4 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitArguments(PainlessParser.ArgumentsContext ctx) { return visitChildren(ctx); }
-  /**
-   * {@inheritDoc}
-   *
-   * <p>The default implementation returns the result of calling
-   * {@link #visitChildren} on {@code ctx}.</p>
-   */
-  @Override public T visitIncrement(PainlessParser.IncrementContext ctx) { return visitChildren(ctx); }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessParserVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
similarity index 86%
rename from modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessParserVisitor.java
rename to modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
index 915328a3f05..7a3b594ab15 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessParserVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
@@ -1,5 +1,5 @@
 // ANTLR GENERATED CODE: DO NOT EDIT
-package org.elasticsearch.painless;
+package org.elasticsearch.painless.antlr;
 import org.antlr.v4.runtime.tree.ParseTreeVisitor;
 
 /**
@@ -174,6 +174,13 @@ interface PainlessParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitComp(PainlessParser.CompContext ctx);
+  /**
+   * Visit a parse tree produced by the {@code read}
+   * labeled alternative in {@link PainlessParser#expression}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitRead(PainlessParser.ReadContext ctx);
   /**
    * Visit a parse tree produced by the {@code bool}
    * labeled alternative in {@link PainlessParser#expression}.
@@ -244,13 +251,6 @@ interface PainlessParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitCast(PainlessParser.CastContext ctx);
-  /**
-   * Visit a parse tree produced by the {@code external}
-   * labeled alternative in {@link PainlessParser#expression}.
-   * @param ctx the parse tree
-   * @return the visitor result
-   */
-  T visitExternal(PainlessParser.ExternalContext ctx);
   /**
    * Visit a parse tree produced by the {@code null}
    * labeled alternative in {@link PainlessParser#expression}.
@@ -273,75 +273,69 @@ interface PainlessParserVisitor<T> extends ParseTreeVisitor<T> {
    */
   T visitTrue(PainlessParser.TrueContext ctx);
   /**
-   * Visit a parse tree produced by {@link PainlessParser#extstart}.
+   * Visit a parse tree produced by {@link PainlessParser#chain}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitExtstart(PainlessParser.ExtstartContext ctx);
+  T visitChain(PainlessParser.ChainContext ctx);
   /**
-   * Visit a parse tree produced by {@link PainlessParser#extprec}.
+   * Visit a parse tree produced by {@link PainlessParser#linkprec}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitExtprec(PainlessParser.ExtprecContext ctx);
+  T visitLinkprec(PainlessParser.LinkprecContext ctx);
   /**
-   * Visit a parse tree produced by {@link PainlessParser#extcast}.
+   * Visit a parse tree produced by {@link PainlessParser#linkcast}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitExtcast(PainlessParser.ExtcastContext ctx);
+  T visitLinkcast(PainlessParser.LinkcastContext ctx);
   /**
-   * Visit a parse tree produced by {@link PainlessParser#extbrace}.
+   * Visit a parse tree produced by {@link PainlessParser#linkbrace}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitExtbrace(PainlessParser.ExtbraceContext ctx);
+  T visitLinkbrace(PainlessParser.LinkbraceContext ctx);
   /**
-   * Visit a parse tree produced by {@link PainlessParser#extdot}.
+   * Visit a parse tree produced by {@link PainlessParser#linkdot}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitExtdot(PainlessParser.ExtdotContext ctx);
+  T visitLinkdot(PainlessParser.LinkdotContext ctx);
   /**
-   * Visit a parse tree produced by {@link PainlessParser#extcall}.
+   * Visit a parse tree produced by {@link PainlessParser#linkcall}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitExtcall(PainlessParser.ExtcallContext ctx);
+  T visitLinkcall(PainlessParser.LinkcallContext ctx);
   /**
-   * Visit a parse tree produced by {@link PainlessParser#extvar}.
+   * Visit a parse tree produced by {@link PainlessParser#linkvar}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitExtvar(PainlessParser.ExtvarContext ctx);
+  T visitLinkvar(PainlessParser.LinkvarContext ctx);
   /**
-   * Visit a parse tree produced by {@link PainlessParser#extfield}.
+   * Visit a parse tree produced by {@link PainlessParser#linkfield}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitExtfield(PainlessParser.ExtfieldContext ctx);
+  T visitLinkfield(PainlessParser.LinkfieldContext ctx);
   /**
-   * Visit a parse tree produced by {@link PainlessParser#extnew}.
+   * Visit a parse tree produced by {@link PainlessParser#linknew}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitExtnew(PainlessParser.ExtnewContext ctx);
+  T visitLinknew(PainlessParser.LinknewContext ctx);
   /**
-   * Visit a parse tree produced by {@link PainlessParser#extstring}.
+   * Visit a parse tree produced by {@link PainlessParser#linkstring}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitExtstring(PainlessParser.ExtstringContext ctx);
+  T visitLinkstring(PainlessParser.LinkstringContext ctx);
   /**
    * Visit a parse tree produced by {@link PainlessParser#arguments}.
    * @param ctx the parse tree
    * @return the visitor result
    */
   T visitArguments(PainlessParser.ArgumentsContext ctx);
-  /**
-   * Visit a parse tree produced by {@link PainlessParser#increment}.
-   * @param ctx the parse tree
-   * @return the visitor result
-   */
-  T visitIncrement(PainlessParser.IncrementContext ctx);
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ParserErrorStrategy.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ParserErrorStrategy.java
similarity index 69%
rename from modules/lang-painless/src/main/java/org/elasticsearch/painless/ParserErrorStrategy.java
rename to modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ParserErrorStrategy.java
index 0bffa9535f6..40f617a660c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ParserErrorStrategy.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ParserErrorStrategy.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.painless;
+package org.elasticsearch.painless.antlr;
 
 import org.antlr.v4.runtime.DefaultErrorStrategy;
 import org.antlr.v4.runtime.InputMismatchException;
@@ -28,10 +28,14 @@ import org.antlr.v4.runtime.Token;
 
 import java.text.ParseException;
 
-class ParserErrorStrategy extends DefaultErrorStrategy {
+/**
+ * An error strategy that will override the default error behavior to fail on the first parser error.
+ */
+final class ParserErrorStrategy extends DefaultErrorStrategy {
+
     @Override
-    public void recover(Parser recognizer, RecognitionException re) {
-        Token token = re.getOffendingToken();
+    public void recover(final Parser recognizer, final RecognitionException re) {
+        final Token token = re.getOffendingToken();
         String message;
 
         if (token == null) {
@@ -52,23 +56,24 @@ class ParserErrorStrategy extends DefaultErrorStrategy {
                     " unexpected token near [" + getTokenErrorDisplay(token) + "].";
         }
 
-        ParseException parseException = new ParseException(message, token == null ? -1 : token.getStartIndex());
+        final ParseException parseException = new ParseException(message, token == null ? -1 : token.getStartIndex());
         parseException.initCause(re);
 
         throw new RuntimeException(parseException);
     }
 
     @Override
-    public Token recoverInline(Parser recognizer) throws RecognitionException {
-        Token token = recognizer.getCurrentToken();
-        String message = "Error[" + token.getLine() + ":" + token.getCharPositionInLine() + "]:" +
-                " unexpected token [" + getTokenErrorDisplay(token) + "]" +
-                " was expecting one of [" + recognizer.getExpectedTokens().toString(recognizer.getVocabulary()) + "].";
-        ParseException parseException = new ParseException(message, token.getStartIndex());
+    public Token recoverInline(final Parser recognizer) throws RecognitionException {
+        final Token token = recognizer.getCurrentToken();
+        final String message = "Error[" + token.getLine() + ":" + token.getCharPositionInLine() + "]:" +
+            " unexpected token [" + getTokenErrorDisplay(token) + "]" +
+            " was expecting one of [" + recognizer.getExpectedTokens().toString(recognizer.getVocabulary()) + "].";
+        final ParseException parseException = new ParseException(message, token.getStartIndex());
+
         throw new RuntimeException(parseException);
     }
 
     @Override
-    public void sync(Parser recognizer) {
+    public void sync(final Parser recognizer) {
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
new file mode 100644
index 00000000000..71ada2fdf9b
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -0,0 +1,790 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.antlr;
+
+import org.antlr.v4.runtime.ANTLRInputStream;
+import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.elasticsearch.painless.Operation;
+import org.elasticsearch.painless.Variables.Reserved;
+import org.elasticsearch.painless.antlr.PainlessParser.AfterthoughtContext;
+import org.elasticsearch.painless.antlr.PainlessParser.ArgumentsContext;
+import org.elasticsearch.painless.antlr.PainlessParser.AssignmentContext;
+import org.elasticsearch.painless.antlr.PainlessParser.BinaryContext;
+import org.elasticsearch.painless.antlr.PainlessParser.BreakContext;
+import org.elasticsearch.painless.antlr.PainlessParser.CastContext;
+import org.elasticsearch.painless.antlr.PainlessParser.ConditionalContext;
+import org.elasticsearch.painless.antlr.PainlessParser.ContinueContext;
+import org.elasticsearch.painless.antlr.PainlessParser.DeclContext;
+import org.elasticsearch.painless.antlr.PainlessParser.DeclarationContext;
+import org.elasticsearch.painless.antlr.PainlessParser.DecltypeContext;
+import org.elasticsearch.painless.antlr.PainlessParser.DeclvarContext;
+import org.elasticsearch.painless.antlr.PainlessParser.DoContext;
+import org.elasticsearch.painless.antlr.PainlessParser.EmptyContext;
+import org.elasticsearch.painless.antlr.PainlessParser.EmptyscopeContext;
+import org.elasticsearch.painless.antlr.PainlessParser.ExprContext;
+import org.elasticsearch.painless.antlr.PainlessParser.ExpressionContext;
+import org.elasticsearch.painless.antlr.PainlessParser.LinkbraceContext;
+import org.elasticsearch.painless.antlr.PainlessParser.LinkcallContext;
+import org.elasticsearch.painless.antlr.PainlessParser.LinkcastContext;
+import org.elasticsearch.painless.antlr.PainlessParser.LinkdotContext;
+import org.elasticsearch.painless.antlr.PainlessParser.ReadContext;
+import org.elasticsearch.painless.antlr.PainlessParser.LinkfieldContext;
+import org.elasticsearch.painless.antlr.PainlessParser.LinknewContext;
+import org.elasticsearch.painless.antlr.PainlessParser.LinkprecContext;
+import org.elasticsearch.painless.antlr.PainlessParser.ChainContext;
+import org.elasticsearch.painless.antlr.PainlessParser.LinkstringContext;
+import org.elasticsearch.painless.antlr.PainlessParser.LinkvarContext;
+import org.elasticsearch.painless.antlr.PainlessParser.FalseContext;
+import org.elasticsearch.painless.antlr.PainlessParser.ForContext;
+import org.elasticsearch.painless.antlr.PainlessParser.GenericContext;
+import org.elasticsearch.painless.antlr.PainlessParser.IdentifierContext;
+import org.elasticsearch.painless.antlr.PainlessParser.IfContext;
+import org.elasticsearch.painless.antlr.PainlessParser.InitializerContext;
+import org.elasticsearch.painless.antlr.PainlessParser.MultipleContext;
+import org.elasticsearch.painless.antlr.PainlessParser.NullContext;
+import org.elasticsearch.painless.antlr.PainlessParser.NumericContext;
+import org.elasticsearch.painless.antlr.PainlessParser.PostincContext;
+import org.elasticsearch.painless.antlr.PainlessParser.PrecedenceContext;
+import org.elasticsearch.painless.antlr.PainlessParser.PreincContext;
+import org.elasticsearch.painless.antlr.PainlessParser.ReturnContext;
+import org.elasticsearch.painless.antlr.PainlessParser.SingleContext;
+import org.elasticsearch.painless.antlr.PainlessParser.SourceContext;
+import org.elasticsearch.painless.antlr.PainlessParser.StatementContext;
+import org.elasticsearch.painless.antlr.PainlessParser.ThrowContext;
+import org.elasticsearch.painless.antlr.PainlessParser.TrapContext;
+import org.elasticsearch.painless.antlr.PainlessParser.TrueContext;
+import org.elasticsearch.painless.antlr.PainlessParser.TryContext;
+import org.elasticsearch.painless.antlr.PainlessParser.UnaryContext;
+import org.elasticsearch.painless.antlr.PainlessParser.WhileContext;
+import org.elasticsearch.painless.node.AExpression;
+import org.elasticsearch.painless.node.ALink;
+import org.elasticsearch.painless.node.ANode;
+import org.elasticsearch.painless.node.AStatement;
+import org.elasticsearch.painless.node.EBinary;
+import org.elasticsearch.painless.node.EBool;
+import org.elasticsearch.painless.node.EBoolean;
+import org.elasticsearch.painless.node.EExplicit;
+import org.elasticsearch.painless.node.EChain;
+import org.elasticsearch.painless.node.EComp;
+import org.elasticsearch.painless.node.EConditional;
+import org.elasticsearch.painless.node.EDecimal;
+import org.elasticsearch.painless.node.ENull;
+import org.elasticsearch.painless.node.ENumeric;
+import org.elasticsearch.painless.node.EUnary;
+import org.elasticsearch.painless.node.LBrace;
+import org.elasticsearch.painless.node.LCall;
+import org.elasticsearch.painless.node.LCast;
+import org.elasticsearch.painless.node.LField;
+import org.elasticsearch.painless.node.LNewArray;
+import org.elasticsearch.painless.node.LNewObj;
+import org.elasticsearch.painless.node.LString;
+import org.elasticsearch.painless.node.LVariable;
+import org.elasticsearch.painless.node.SBlock;
+import org.elasticsearch.painless.node.SBreak;
+import org.elasticsearch.painless.node.SContinue;
+import org.elasticsearch.painless.node.SDeclBlock;
+import org.elasticsearch.painless.node.SDeclaration;
+import org.elasticsearch.painless.node.SDo;
+import org.elasticsearch.painless.node.SExpression;
+import org.elasticsearch.painless.node.SFor;
+import org.elasticsearch.painless.node.SIfElse;
+import org.elasticsearch.painless.node.SReturn;
+import org.elasticsearch.painless.node.SSource;
+import org.elasticsearch.painless.node.SThrow;
+import org.elasticsearch.painless.node.STrap;
+import org.elasticsearch.painless.node.STry;
+import org.elasticsearch.painless.node.SWhile;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Converts the ANTLR tree to a Painless tree.
+ */
+public final class Walker extends PainlessParserBaseVisitor<ANode> {
+
+    public static SSource buildPainlessTree(final String source, final Reserved reserved) {
+        return new Walker(source, reserved).source;
+    }
+
+    private final Reserved reserved;
+    private final SSource source;
+
+    private Walker(final String source, final Reserved reserved) {
+        this.reserved = reserved;
+        this.source = (SSource)visit(buildAntlrTree(source));
+    }
+
+    private SourceContext buildAntlrTree(final String source) {
+        final ANTLRInputStream stream = new ANTLRInputStream(source);
+        final PainlessLexer lexer = new ErrorHandlingLexer(stream);
+        final PainlessParser parser = new PainlessParser(new CommonTokenStream(lexer));
+        final ParserErrorStrategy strategy = new ParserErrorStrategy();
+
+        lexer.removeErrorListeners();
+        parser.removeErrorListeners();
+        parser.setErrorHandler(strategy);
+
+        return parser.source();
+    }
+
+    private String location(final ParserRuleContext ctx) {
+        return "[ " + ctx.getStart().getLine() + " : " + ctx.getStart().getCharPositionInLine() + " ]";
+    }
+
+    @Override
+    public ANode visitSource(final SourceContext ctx) {
+        final List<AStatement> statements = new ArrayList<>();
+
+        for (final StatementContext statement : ctx.statement()) {
+            statements.add((AStatement)visit(statement));
+        }
+
+        return new SSource(location(ctx), statements);
+    }
+
+    @Override
+    public ANode visitIf(final IfContext ctx) {
+        final AExpression condition = (AExpression)visit(ctx.expression());
+        final AStatement ifblock = (AStatement)visit(ctx.block(0));
+        final AStatement elseblock = ctx.block(1) == null ? null : (AStatement)visit(ctx.block(1));
+
+        return new SIfElse(location(ctx), condition, ifblock, elseblock);
+    }
+
+    @Override
+    public ANode visitWhile(final WhileContext ctx) {
+        final AExpression condition = (AExpression)visit(ctx.expression());
+        final AStatement block = ctx.block() == null ? null : (AStatement)visit(ctx.block());
+
+        reserved.usesLoop();
+
+        return new SWhile(location(ctx), condition, block);
+    }
+
+    @Override
+    public ANode visitDo(final DoContext ctx) {
+        final AStatement block = ctx.block() == null ? null : (AStatement)visit(ctx.block());
+        final AExpression condition = (AExpression)visit(ctx.expression());
+
+        reserved.usesLoop();
+
+        return new SDo(location(ctx), block, condition);
+    }
+
+    @Override
+    public ANode visitFor(final ForContext ctx) {
+        final ANode intializer = ctx.initializer() == null ? null : visit(ctx.initializer());
+        final AExpression condition = ctx.expression() == null ? null : (AExpression)visit(ctx.expression());
+        final AExpression afterthought = ctx.afterthought() == null ? null : (AExpression)visit(ctx.afterthought());
+        final AStatement block = ctx.block() == null ? null : (AStatement)visit(ctx.block());
+
+        reserved.usesLoop();
+
+        return new SFor(location(ctx), intializer, condition, afterthought, block);
+    }
+
+    @Override
+    public ANode visitDecl(final DeclContext ctx) {
+        return visit(ctx.declaration());
+    }
+
+    @Override
+    public ANode visitContinue(final ContinueContext ctx) {
+        return new SContinue(location(ctx));
+    }
+
+    @Override
+    public ANode visitBreak(final BreakContext ctx) {
+        return new SBreak(location(ctx));
+    }
+
+    @Override
+    public ANode visitReturn(final ReturnContext ctx) {
+        final AExpression expression = (AExpression)visit(ctx.expression());
+
+        return new SReturn(location(ctx), expression);
+    }
+
+    @Override
+    public ANode visitTry(final TryContext ctx) {
+        final AStatement block = (AStatement)visit(ctx.block());
+        final List<STrap> traps = new ArrayList<>();
+
+        for (final TrapContext trap : ctx.trap()) {
+            traps.add((STrap)visit(trap));
+        }
+
+        return new STry(location(ctx), block, traps);
+    }
+
+    @Override
+    public ANode visitThrow(final ThrowContext ctx) {
+        final AExpression expression = (AExpression)visit(ctx.expression());
+
+        return new SThrow(location(ctx), expression);
+    }
+
+    @Override
+    public ANode visitExpr(final ExprContext ctx) {
+        final AExpression expression = (AExpression)visit(ctx.expression());
+
+        return new SExpression(location(ctx), expression);
+    }
+
+    @Override
+    public ANode visitMultiple(final MultipleContext ctx) {
+        final List<AStatement> statements = new ArrayList<>();
+
+        for (final StatementContext statement : ctx.statement()) {
+            statements.add((AStatement)visit(statement));
+        }
+
+        return new SBlock(location(ctx), statements);
+    }
+
+    @Override
+    public ANode visitSingle(final SingleContext ctx) {
+        final List<AStatement> statements = new ArrayList<>();
+        statements.add((AStatement)visit(ctx.statement()));
+
+        return new SBlock(location(ctx), statements);
+    }
+
+    @Override
+    public ANode visitEmpty(final EmptyContext ctx) {
+        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    }
+
+    @Override
+    public ANode visitEmptyscope(final EmptyscopeContext ctx) {
+        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    }
+
+    @Override
+    public ANode visitInitializer(final InitializerContext ctx) {
+        if (ctx.declaration() != null) {
+            return visit(ctx.declaration());
+        } else if (ctx.expression() != null) {
+            return visit(ctx.expression());
+        }
+
+        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    }
+
+    @Override
+    public ANode visitAfterthought(final AfterthoughtContext ctx) {
+        return visit(ctx.expression());
+    }
+
+    @Override
+    public ANode visitDeclaration(final DeclarationContext ctx) {
+        final String type = ctx.decltype().getText();
+        final List<SDeclaration> declarations = new ArrayList<>();
+
+        for (final DeclvarContext declvar : ctx.declvar()) {
+            final String name = declvar.identifier().getText();
+            final AExpression expression = declvar.expression() == null ? null : (AExpression)visit(declvar.expression());
+            declarations.add(new SDeclaration(location(ctx), type, name, expression));
+        }
+
+        return new SDeclBlock(location(ctx), declarations);
+    }
+
+    @Override
+    public ANode visitDecltype(final DecltypeContext ctx) {
+        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    }
+
+    @Override
+    public ANode visitDeclvar(final DeclvarContext ctx) {
+        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    }
+
+    @Override
+    public ANode visitTrap(final TrapContext ctx) {
+        final String type = ctx.identifier(0).getText();
+        final String name = ctx.identifier(1).getText();
+        final AStatement block = ctx.block() == null ? null : (AStatement)visit(ctx.block());
+
+        return new STrap(location(ctx), type, name, block);
+    }
+
+    @Override
+    public ANode visitIdentifier(final IdentifierContext ctx) {
+        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    }
+
+    @Override
+    public ANode visitGeneric(final GenericContext ctx) {
+        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    }
+
+    @Override
+    public ANode visitPrecedence(final PrecedenceContext ctx) {
+        return visit(ctx.expression());
+    }
+
+    @Override
+    public ANode visitNumeric(final NumericContext ctx) {
+        final boolean negate = ctx.parent instanceof UnaryContext && ((UnaryContext)ctx.parent).SUB() != null;
+
+        if (ctx.DECIMAL() != null) {
+            return new EDecimal(location(ctx), (negate ? "-" : "") + ctx.DECIMAL().getText());
+        } else if (ctx.HEX() != null) {
+            return new ENumeric(location(ctx), (negate ? "-" : "") + ctx.HEX().getText().substring(2), 16);
+        } else if (ctx.INTEGER() != null) {
+            return new ENumeric(location(ctx), (negate ? "-" : "") + ctx.INTEGER().getText(), 10);
+        } else if (ctx.OCTAL() != null) {
+            return new ENumeric(location(ctx), (negate ? "-" : "") + ctx.OCTAL().getText().substring(1), 8);
+        } else {
+            throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+        }
+    }
+
+    @Override
+    public ANode visitTrue(final TrueContext ctx) {
+        return new EBoolean(location(ctx), true);
+    }
+
+    @Override
+    public ANode visitFalse(FalseContext ctx) {
+        return new EBoolean(location(ctx), false);
+    }
+
+    @Override
+    public ANode visitNull(final NullContext ctx) {
+        return new ENull(location(ctx));
+    }
+
+    @Override
+    public ANode visitPostinc(final PostincContext ctx) {
+        final List<ALink> links = new ArrayList<>();
+        final Operation operation;
+
+        visitChain(ctx.chain(), links);
+
+        if (ctx.INCR() != null) {
+            operation = Operation.INCR;
+        } else if (ctx.DECR() != null) {
+            operation = Operation.DECR;
+        } else {
+            throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+        }
+
+        return new EChain(location(ctx), links, false, true, operation, null);
+    }
+
+    @Override
+    public ANode visitPreinc(final PreincContext ctx) {
+        final List<ALink> links = new ArrayList<>();
+        final Operation operation;
+
+        visitChain(ctx.chain(), links);
+
+        if (ctx.INCR() != null) {
+            operation = Operation.INCR;
+        } else if (ctx.DECR() != null) {
+            operation = Operation.DECR;
+        } else {
+            throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+        }
+
+        return new EChain(location(ctx), links, true, false, operation, null);
+    }
+
+    @Override
+    public ANode visitRead(final ReadContext ctx) {
+        final List<ALink> links = new ArrayList<>();
+
+        visitChain(ctx.chain(), links);
+
+        return new EChain(location(ctx), links, false, false, null, null);
+    }
+
+    @Override
+    public ANode visitUnary(final UnaryContext ctx) {
+        if (ctx.SUB() != null && ctx.expression() instanceof NumericContext) {
+            return visit(ctx.expression());
+        } else {
+            final Operation operation;
+
+            if (ctx.BOOLNOT() != null) {
+                operation = Operation.NOT;
+            } else if (ctx.BWNOT() != null) {
+                operation = Operation.BWNOT;
+            } else if (ctx.ADD() != null) {
+                operation = Operation.ADD;
+            } else if (ctx.SUB() != null) {
+                operation = Operation.SUB;
+            } else {
+                throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+            }
+
+            return new EUnary(location(ctx), operation, (AExpression)visit(ctx.expression()));
+        }
+    }
+
+    @Override
+    public ANode visitCast(final CastContext ctx) {
+        return new EExplicit(location(ctx), ctx.decltype().getText(), (AExpression)visit(ctx.expression()));
+    }
+
+    @Override
+    public ANode visitBinary(final BinaryContext ctx) {
+        final AExpression left = (AExpression)visit(ctx.expression(0));
+        final AExpression right = (AExpression)visit(ctx.expression(1));
+        final Operation operation;
+
+        if (ctx.MUL() != null) {
+            operation = Operation.MUL;
+        } else if (ctx.DIV() != null) {
+            operation = Operation.DIV;
+        } else if (ctx.REM() != null) {
+            operation = Operation.REM;
+        } else if (ctx.ADD() != null) {
+            operation = Operation.ADD;
+        } else if (ctx.SUB() != null) {
+            operation = Operation.SUB;
+        } else if (ctx.LSH() != null) {
+            operation = Operation.LSH;
+        } else if (ctx.RSH() != null) {
+            operation = Operation.RSH;
+        } else if (ctx.USH() != null) {
+            operation = Operation.USH;
+        } else if (ctx.BWAND() != null) {
+            operation = Operation.BWAND;
+        } else if (ctx.XOR() != null) {
+            operation = Operation.XOR;
+        } else if (ctx.BWOR() != null) {
+            operation = Operation.BWOR;
+        } else {
+            throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+        }
+
+        return new EBinary(location(ctx), operation, left, right);
+    }
+
+    @Override
+    public ANode visitComp(PainlessParser.CompContext ctx) {
+        final AExpression left = (AExpression)visit(ctx.expression(0));
+        final AExpression right = (AExpression)visit(ctx.expression(1));
+        final Operation operation;
+
+        if (ctx.LT() != null) {
+            operation = Operation.LT;
+        } else if (ctx.LTE() != null) {
+            operation = Operation.LTE;
+        } else if (ctx.GT() != null) {
+            operation = Operation.GT;
+        } else if (ctx.GTE() != null) {
+            operation = Operation.GTE;
+        } else if (ctx.EQ() != null) {
+            operation = Operation.EQ;
+        } else if (ctx.EQR() != null) {
+            operation = Operation.EQR;
+        } else if (ctx.NE() != null) {
+            operation = Operation.NE;
+        } else if (ctx.NER() != null) {
+            operation = Operation.NER;
+        } else {
+            throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+        }
+
+        return new EComp(location(ctx), operation, left, right);
+    }
+
+    @Override
+    public ANode visitBool(PainlessParser.BoolContext ctx) {
+        final AExpression left = (AExpression)visit(ctx.expression(0));
+        final AExpression right = (AExpression)visit(ctx.expression(1));
+        final Operation operation;
+
+        if (ctx.BOOLAND() != null) {
+            operation = Operation.AND;
+        } else if (ctx.BOOLOR() != null) {
+            operation = Operation.OR;
+        } else {
+            throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+        }
+
+        return new EBool(location(ctx), operation, left, right);
+    }
+
+
+    @Override
+    public ANode visitConditional(final ConditionalContext ctx) {
+        final AExpression condition = (AExpression)visit(ctx.expression(0));
+        final AExpression left = (AExpression)visit(ctx.expression(1));
+        final AExpression right = (AExpression)visit(ctx.expression(2));
+
+        return new EConditional(location(ctx), condition, left, right);
+    }
+
+    @Override
+    public ANode visitAssignment(final AssignmentContext ctx) {
+        final List<ALink> links = new ArrayList<>();
+        final Operation operation;
+
+        visitChain(ctx.chain(), links);
+
+        if (ctx.AMUL() != null) {
+            operation = Operation.MUL;
+        } else if (ctx.ADIV() != null) {
+            operation = Operation.DIV;
+        } else if (ctx.AREM() != null) {
+            operation = Operation.REM;
+        } else if (ctx.AADD() != null) {
+            operation = Operation.ADD;
+        } else if (ctx.ASUB() != null) {
+            operation = Operation.SUB;
+        } else if (ctx.ALSH() != null) {
+            operation = Operation.LSH;
+        } else if (ctx.ARSH() != null) {
+            operation = Operation.RSH;
+        } else if (ctx.AUSH() != null) {
+            operation = Operation.USH;
+        } else if (ctx.AAND() != null) {
+            operation = Operation.BWAND;
+        } else if (ctx.AXOR() != null) {
+            operation = Operation.XOR;
+        } else if (ctx.AOR() != null) {
+            operation = Operation.BWOR;
+        } else {
+            operation = null;
+        }
+
+        return new EChain(location(ctx), links, false, false, operation, (AExpression)visit(ctx.expression()));
+    }
+
+    private void visitChain(final ChainContext ctx, final List<ALink> links) {
+        if (ctx.linkprec() != null) {
+            visitLinkprec(ctx.linkprec(), links);
+        } else if (ctx.linkcast() != null) {
+            visitLinkcast(ctx.linkcast(), links);
+        } else if (ctx.linkvar() != null) {
+            visitLinkvar(ctx.linkvar(), links);
+        } else if (ctx.linknew() != null) {
+            visitLinknew(ctx.linknew(), links);
+        } else if (ctx.linkstring() != null) {
+            visitLinkstring(ctx.linkstring(), links);
+        } else {
+            throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+        }
+    }
+
+    @Override
+    public ANode visitChain(final ChainContext ctx) {
+        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    }
+
+    private void visitLinkprec(final LinkprecContext ctx, final List<ALink> links) {
+        if (ctx.linkprec() != null) {
+            visitLinkprec(ctx.linkprec(), links);
+        } else if (ctx.linkcast() != null) {
+            visitLinkcast(ctx.linkcast(), links);
+        } else if (ctx.linkvar() != null) {
+            visitLinkvar(ctx.linkvar(), links);
+        } else if (ctx.linknew() != null) {
+            visitLinknew(ctx.linknew(), links);
+        } else if (ctx.linkstring() != null) {
+            visitLinkstring(ctx.linkstring(), links);
+        } else {
+            throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+        }
+
+        if (ctx.linkbrace() != null) {
+            visitLinkbrace(ctx.linkbrace(), links);
+        } else if (ctx.linkdot() != null) {
+            visitLinkdot(ctx.linkdot(), links);
+        }
+    }
+
+    @Override
+    public ANode visitLinkprec(final LinkprecContext ctx) {
+        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    }
+
+    private void visitLinkcast(final LinkcastContext ctx, final List<ALink> links) {
+        if (ctx.linkprec() != null) {
+            visitLinkprec(ctx.linkprec(), links);
+        } else if (ctx.linkcast() != null) {
+            visitLinkcast(ctx.linkcast(), links);
+        } else if (ctx.linkvar() != null) {
+            visitLinkvar(ctx.linkvar(), links);
+        } else if (ctx.linknew() != null) {
+            visitLinknew(ctx.linknew(), links);
+        } else if (ctx.linkstring() != null) {
+            visitLinkstring(ctx.linkstring(), links);
+        } else {
+            throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+        }
+
+        links.add(new LCast(location(ctx), ctx.decltype().getText()));
+    }
+
+    @Override
+    public ANode visitLinkcast(final LinkcastContext ctx) {
+        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    }
+
+    private void visitLinkbrace(final LinkbraceContext ctx, final List<ALink> links) {
+        links.add(new LBrace(location(ctx), (AExpression)visit(ctx.expression())));
+
+        if (ctx.linkbrace() != null) {
+            visitLinkbrace(ctx.linkbrace(), links);
+        } else if (ctx.linkdot() != null) {
+            visitLinkdot(ctx.linkdot(), links);
+        }
+    }
+
+    @Override
+    public ANode visitLinkbrace(final LinkbraceContext ctx) {
+        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    }
+
+    private void visitLinkdot(final LinkdotContext ctx, final List<ALink> links) {
+        if (ctx.linkcall() != null) {
+            visitLinkcall(ctx.linkcall(), links);
+        } else if (ctx.linkfield() != null) {
+            visitLinkfield(ctx.linkfield(), links);
+        }
+    }
+
+    @Override
+    public ANode visitLinkdot(final LinkdotContext ctx) {
+        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    }
+
+    private void visitLinkcall(final LinkcallContext ctx, final List<ALink> links) {
+        final List<AExpression> arguments = new ArrayList<>();
+
+        for (final ExpressionContext expression : ctx.arguments().expression()) {
+            arguments.add((AExpression)visit(expression));
+        }
+
+        links.add(new LCall(location(ctx), ctx.EXTID().getText(), arguments));
+
+        if (ctx.linkbrace() != null) {
+            visitLinkbrace(ctx.linkbrace(), links);
+        } else if (ctx.linkdot() != null) {
+            visitLinkdot(ctx.linkdot(), links);
+        }
+    }
+
+    @Override
+    public ANode visitLinkcall(final LinkcallContext ctx) {
+        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    }
+
+    private void visitLinkvar(final LinkvarContext ctx, final List<ALink> links) {
+        final String name = ctx.identifier().getText();
+
+        reserved.markReserved(name);
+
+        links.add(new LVariable(location(ctx), name));
+
+        if (ctx.linkbrace() != null) {
+            visitLinkbrace(ctx.linkbrace(), links);
+        } else if (ctx.linkdot() != null) {
+            visitLinkdot(ctx.linkdot(), links);
+        }
+    }
+
+    @Override
+    public ANode visitLinkvar(final LinkvarContext ctx) {
+        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    }
+
+    private void visitLinkfield(final LinkfieldContext ctx, final List<ALink> links) {
+        final String value;
+
+        if (ctx.EXTID() != null) {
+            value = ctx.EXTID().getText();
+        } else if (ctx.EXTINTEGER() != null) {
+            value = ctx.EXTINTEGER().getText();
+        } else {
+            throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+        }
+
+        links.add(new LField(location(ctx), value));
+
+        if (ctx.linkbrace() != null) {
+            visitLinkbrace(ctx.linkbrace(), links);
+        } else if (ctx.linkdot() != null) {
+            visitLinkdot(ctx.linkdot(), links);
+        }
+    }
+
+    @Override
+    public ANode visitLinkfield(final LinkfieldContext ctx) {
+        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    }
+
+    private void visitLinknew(final LinknewContext ctx, final List<ALink> links) {
+        final List<AExpression> arguments = new ArrayList<>();
+
+        if (ctx.arguments() != null) {
+            for (final ExpressionContext expression : ctx.arguments().expression()) {
+                arguments.add((AExpression)visit(expression));
+            }
+
+            links.add(new LNewObj(location(ctx), ctx.identifier().getText(), arguments));
+        } else if (ctx.expression().size() > 0) {
+            for (final ExpressionContext expression : ctx.expression()) {
+                arguments.add((AExpression)visit(expression));
+            }
+
+            links.add(new LNewArray(location(ctx), ctx.identifier().getText(), arguments));
+        } else {
+            throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+        }
+
+        if (ctx.linkdot() != null) {
+            visitLinkdot(ctx.linkdot(), links);
+        }
+    }
+
+    @Override
+    public ANode visitLinknew(final LinknewContext ctx) {
+        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    }
+
+    private void visitLinkstring(final LinkstringContext ctx, final List<ALink> links) {
+        links.add(new LString(location(ctx), ctx.STRING().getText().substring(1, ctx.STRING().getText().length() - 1)));
+
+        if (ctx.linkbrace() != null) {
+            visitLinkbrace(ctx.linkbrace(), links);
+        } else if (ctx.linkdot() != null) {
+            visitLinkdot(ctx.linkdot(), links);
+        }
+    }
+
+    @Override
+    public ANode visitLinkstring(final LinkstringContext ctx) {
+        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    }
+
+    @Override
+    public ANode visitArguments(final ArgumentsContext ctx) {
+        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
new file mode 100644
index 00000000000..185968c0a09
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Cast;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.AnalyzerCaster;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * The superclass for all E* (expression) nodes.
+ */
+public abstract class AExpression extends ANode {
+
+    /**
+     * Set to false when an expression will not be read from such as
+     * a basic assignment.  Note this variable is always set by the parent
+     * as input.
+     */
+    protected boolean read = true;
+
+    /**
+     * Set to true when an expression can be considered a stand alone
+     * statement.  Used to prevent extraneous bytecode. This is always
+     * set by the node as output.
+     */
+    protected boolean statement = false;
+
+    /**
+     * Set to the expected type this node needs to be.  Note this variable
+     * is always set by the parent as input and should never be read from.
+     */
+    protected Type expected = null;
+
+    /**
+     * Set to the actual type this node is.  Note this variable is always
+     * set by the node as output and should only be read from outside of the
+     * node itself.  <b>Also, actual can always be read after a cast is
+     * called on this node to get the type of the node after the cast.</b>
+     */
+    protected Type actual = null;
+
+    /**
+     * Set by {@link EExplicit} if a cast made on an expression node should be
+     * explicit.
+     */
+    protected boolean explicit = false;
+
+    /**
+     * Set to the value of the constant this expression node represents if
+     * and only if the node represents a constant.  If this is not null
+     * this node will be replaced by an {@link EConstant} during casting
+     * if it's not already one.
+     */
+    protected Object constant = null;
+
+    /**
+     * Set to true by {@link ENull} to represent a null value.
+     */
+    protected boolean isNull = false;
+
+    /**
+     * If an expression represents a branch statement, represents the jump should
+     * the expression evaluate to a true value.  It should always be the case that only
+     * one of tru and fals are non-null or both are null.  Only used during the writing phase.
+     */
+    protected Label tru = null;
+
+    /**
+     * If an expression represents a branch statement, represents the jump should
+     * the expression evaluate to a false value.  It should always be the case that only
+     * one of tru and fals are non-null or both are null.  Only used during the writing phase.
+     */
+    protected Label fals = null;
+
+    public AExpression(final String location) {
+        super(location);
+    }
+
+    /**
+     * Checks for errors and collects data for the writing phase.
+     */
+    abstract void analyze(final CompilerSettings settings, final Definition definition, final Variables variables);
+
+    /**
+     * Writes ASM based on the data collected during the analysis phase.
+     */
+    abstract void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter);
+
+    /**
+     * Inserts {@link ECast} nodes into the tree for implicit casts.  Also replaces
+     * nodes with the constant variable set to a non-null value with {@link EConstant}.
+     * @return The new child node for the parent node calling this method.
+     */
+    AExpression cast(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        final Cast cast = AnalyzerCaster.getLegalCast(definition, location, actual, expected, explicit);
+
+        if (cast == null) {
+            if (constant == null || this instanceof EConstant) {
+                return this;
+            } else {
+                final EConstant econstant = new EConstant(location, constant);
+                econstant.analyze(settings, definition, variables);
+
+                if (!expected.equals(econstant.actual)) {
+                    throw new IllegalStateException(error("Illegal tree structure."));
+                }
+
+                return econstant;
+            }
+        } else {
+            if (constant == null) {
+                final ECast ecast = new ECast(location, this, cast);
+                ecast.statement = statement;
+                ecast.actual = expected;
+                ecast.isNull = isNull;
+
+                return ecast;
+            } else {
+                if (expected.sort.constant) {
+                    constant = AnalyzerCaster.constCast(location, constant, cast);
+
+                    final EConstant econstant = new EConstant(location, constant);
+                    econstant.analyze(settings, definition, variables);
+
+                    if (!expected.equals(econstant.actual)) {
+                        throw new IllegalStateException(error("Illegal tree structure."));
+                    }
+
+                    return econstant;
+                } else if (this instanceof EConstant) {
+                    final ECast ecast = new ECast(location, this, cast);
+                    ecast.actual = expected;
+
+                    return ecast;
+                } else {
+                    final EConstant econstant = new EConstant(location, constant);
+                    econstant.analyze(settings, definition, variables);
+
+                    if (!actual.equals(econstant.actual)) {
+                        throw new IllegalStateException(error("Illegal tree structure."));
+                    }
+
+                    final ECast ecast = new ECast(location, econstant, cast);
+                    ecast.actual = expected;
+
+                    return ecast;
+                }
+            }
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
new file mode 100644
index 00000000000..f5cccfedc7d
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * The superclass for all L* (link) nodes.
+ */
+public abstract class ALink extends ANode {
+
+    /**
+     * Size is set to a value based on this link's size on the stack.  This is
+     * used during the writing phase to dup stack values from this link as
+     * necessary during certain store operations.
+     */
+    final int size;
+
+    /**
+     * Set to false only if the link is not going to be read from.
+     */
+    boolean load = true;
+
+    /**
+     * Set to true only if the link is going to be written to and
+     * is the final link in a chain.
+     */
+    boolean store = false;
+
+    /**
+     * Set to true if this link represents a statik type to be accessed.
+     */
+    boolean statik = false;
+
+    /**
+     * Set by the parent chain to type of the previous link or null if
+     * there was no previous link.
+     */
+    Type before = null;
+
+    /**
+     * Set by the link to be the type after the link has been loaded/stored.
+     */
+    Type after = null;
+
+    /**
+     * Set to true if this link could be a stand-alone statement.
+     */
+    boolean statement = false;
+
+    /**
+     * Used by {@link LString} to set the value of the String constant.  Also
+     * used by shortcuts to represent a constant key.
+     */
+    String string = null;
+
+    ALink(final String location, final int size) {
+        super(location);
+
+        this.size = size;
+    }
+
+    /**
+     * Checks for errors and collects data for the writing phase.
+     * @return Possibly returns a different {@link ALink} node if a type is
+     * def or a shortcut is used. Otherwise, returns itself.  This will be
+     * updated into the {@link EChain} node's list of links.
+     */
+    abstract ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables);
+
+    /**
+     * Write values before a load/store occurs such as an array index.
+     */
+    abstract void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter);
+
+    /**
+     * Write a load for the specific link type.
+     */
+    abstract void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter);
+
+    /**
+     * Write a store for the specific link type.
+     */
+    abstract void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter);
+
+    /**
+     * Used to copy link data from one to another during analysis in the case of replacement.
+     */
+    final ALink copy(final ALink link) {
+        load       = link.load;
+        store      = link.store;
+        statik     = link.statik;
+        before     = link.before;
+        after      = link.after;
+        statement  = link.statement;
+        string     = link.string;
+
+        return this;
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
new file mode 100644
index 00000000000..8faaaff51b3
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+/**
+ * The superclass for all other nodes.
+ */
+public abstract class ANode {
+
+    /**
+     * The location in the original source to be printed in error messages.
+     */
+    final String location;
+
+    ANode(final String location) {
+        this.location = location;
+    }
+
+    public String error(final String message) {
+        return "Error " + location  + ": " + message;
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
new file mode 100644
index 00000000000..d6830adc9bd
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * The superclass for all S* (statement) nodes.
+ */
+public abstract class AStatement extends ANode {
+
+    /**
+     * Set to true when the final statement in an {@link SSource} is reached.
+     * Used to determine whether or not an auto-return is necessary.
+     */
+    boolean lastSource = false;
+
+    /**
+     * Set to true when a loop begins.  Used by {@link SBlock} to help determine
+     * when the final statement of a loop is reached.
+     */
+    boolean beginLoop = false;
+
+    /**
+     * Set to true when inside a loop.  Used by {@link SBreak} and {@link SContinue}
+     * to determine if a break/continue statement is legal.
+     */
+    boolean inLoop = false;
+
+    /**
+     * Set to true when on the last statement of a loop.  Used by {@link SContinue}
+     * to prevent extraneous continue statements.
+     */
+    boolean lastLoop = false;
+
+    /**
+     * Set to true if a statement would cause the method to exit.  Used to
+     * determine whether or not an auto-return is necessary.
+     */
+    boolean methodEscape = false;
+
+    /**
+     * Set to true if a statement would cause a loop to exit.  Used to
+     * prevent unreachable statements.
+     */
+    boolean loopEscape = false;
+
+    /**
+     * Set to true if all current paths escape from the current {@link SBlock}.
+     * Used during the analysis phase to prevent unreachable statements and
+     * the writing phase to prevent extraneous bytecode gotos from being written.
+     */
+    boolean allEscape = false;
+
+    /**
+     * Set to true if any continue statement occurs in a loop.  Used to prevent
+     * unnecessary infinite loops.
+     */
+    boolean anyContinue = false;
+
+    /**
+     * Set to true if any break statement occurs in a loop.  Used to prevent
+     * extraneous loops.
+     */
+    boolean anyBreak = false;
+
+    /**
+     * Set to the loop counter variable slot as a shortcut if loop statements
+     * are being counted.
+     */
+    int loopCounterSlot = -1;
+
+    /**
+     * Set to the approximate number of statements in a loop block to prevent
+     * infinite loops during runtime.
+     */
+    int statementCount = 0;
+
+    /**
+     * Set to the beginning of a loop so a continue statement knows where to
+     * jump to.  Only used during the writing phase.
+     */
+    Label continu = null;
+
+    /**
+     * Set to the beginning of a loop so a break statement knows where to
+     * jump to.  Only used during the writing phase.
+     */
+    Label brake = null;
+
+    AStatement(final String location) {
+        super(location);
+    }
+
+    /**
+     * Checks for errors and collects data for the writing phase.
+     */
+    abstract void analyze(final CompilerSettings settings, final Definition definition, final Variables variables);
+
+    /**
+     * Writes ASM based on the data collected during the analysis phase.
+     */
+    abstract void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter);
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
new file mode 100644
index 00000000000..a3dd45e75c4
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -0,0 +1,528 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.AnalyzerCaster;
+import org.elasticsearch.painless.Operation;
+import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.WriterUtility;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents a binary math expression.
+ */
+public final class EBinary extends AExpression {
+
+    final Operation operation;
+    AExpression left;
+    AExpression right;
+
+    boolean cat = false;
+
+    public EBinary(final String location, final Operation operation, final AExpression left, final AExpression right) {
+        super(location);
+
+        this.operation = operation;
+        this.left = left;
+        this.right = right;
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        if (operation == Operation.MUL) {
+            analyzeMul(settings, definition, variables);
+        } else if (operation == Operation.DIV) {
+            analyzeDiv(settings, definition, variables);
+        } else if (operation == Operation.REM) {
+            analyzeRem(settings, definition, variables);
+        } else if (operation == Operation.ADD) {
+            analyzeAdd(settings, definition, variables);
+        } else if (operation == Operation.SUB) {
+            analyzeSub(settings, definition, variables);
+        } else if (operation == Operation.LSH) {
+            analyzeLSH(settings, definition, variables);
+        } else if (operation == Operation.RSH) {
+            analyzeRSH(settings, definition, variables);
+        } else if (operation == Operation.USH) {
+            analyzeUSH(settings, definition, variables);
+        } else if (operation == Operation.BWAND) {
+            analyzeBWAnd(settings, definition, variables);
+        } else if (operation == Operation.XOR) {
+            analyzeXor(settings, definition, variables);
+        } else if (operation == Operation.BWOR) {
+            analyzeBWOr(settings, definition, variables);
+        } else {
+            throw new IllegalStateException(error("Illegal tree structure."));
+        }
+    }
+
+    private void analyzeMul(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        left.analyze(settings, definition, variables);
+        right.analyze(settings, definition, variables);
+
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+
+        if (promote == null) {
+            throw new ClassCastException(error("Cannot apply multiply [*] to types " +
+                "[" + left.actual.name + "] and [" + right.actual.name + "]."));
+        }
+
+        left.expected = promote;
+        right.expected = promote;
+
+        left = left.cast(settings, definition, variables);
+        right = right.cast(settings, definition, variables);
+
+        if (left.constant != null && right.constant != null) {
+            final boolean overflow = settings.getNumericOverflow();
+            final Sort sort = promote.sort;
+
+            if (sort == Sort.INT) {
+                constant = overflow ? (int)left.constant * (int)right.constant :
+                    Math.multiplyExact((int)left.constant, (int)right.constant);
+            } else if (sort == Sort.LONG) {
+                constant = overflow ? (long)left.constant * (long)right.constant :
+                    Math.multiplyExact((long)left.constant, (long)right.constant);
+            } else if (sort == Sort.FLOAT) {
+                constant = overflow ? (float)left.constant * (float)right.constant :
+                    org.elasticsearch.painless.Utility.multiplyWithoutOverflow((float)left.constant, (float)right.constant);
+            } else if (sort == Sort.DOUBLE) {
+                constant = overflow ? (double)left.constant * (double)right.constant :
+                    org.elasticsearch.painless.Utility.multiplyWithoutOverflow((double)left.constant, (double)right.constant);
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+
+        actual = promote;
+    }
+
+    private void analyzeDiv(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        left.analyze(settings, definition, variables);
+        right.analyze(settings, definition, variables);
+
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+
+        if (promote == null) {
+            throw new ClassCastException(error("Cannot apply divide [/] to types " +
+                "[" + left.actual.name + "] and [" + right.actual.name + "]."));
+        }
+
+        left.expected = promote;
+        right.expected = promote;
+
+        left = left.cast(settings, definition, variables);
+        right = right.cast(settings, definition, variables);
+
+        if (left.constant != null && right.constant != null) {
+            final boolean overflow = settings.getNumericOverflow();
+            final Sort sort = promote.sort;
+
+            if (sort == Sort.INT) {
+                constant = overflow ? (int)left.constant / (int)right.constant :
+                    org.elasticsearch.painless.Utility.divideWithoutOverflow((int)left.constant, (int)right.constant);
+            } else if (sort == Sort.LONG) {
+                constant = overflow ? (long)left.constant / (long)right.constant :
+                    org.elasticsearch.painless.Utility.divideWithoutOverflow((long)left.constant, (long)right.constant);
+            } else if (sort == Sort.FLOAT) {
+                constant = overflow ? (float)left.constant / (float)right.constant :
+                    org.elasticsearch.painless.Utility.divideWithoutOverflow((float)left.constant, (float)right.constant);
+            } else if (sort == Sort.DOUBLE) {
+                constant = overflow ? (double)left.constant / (double)right.constant :
+                    org.elasticsearch.painless.Utility.divideWithoutOverflow((double)left.constant, (double)right.constant);
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+
+        actual = promote;
+    }
+
+    private void analyzeRem(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        left.analyze(settings, definition, variables);
+        right.analyze(settings, definition, variables);
+
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+
+        if (promote == null) {
+            throw new ClassCastException(error("Cannot apply remainder [%] to types " +
+                "[" + left.actual.name + "] and [" + right.actual.name + "]."));
+        }
+
+        left.expected = promote;
+        right.expected = promote;
+
+        left = left.cast(settings, definition, variables);
+        right = right.cast(settings, definition, variables);
+
+        if (left.constant != null && right.constant != null) {
+            final boolean overflow = settings.getNumericOverflow();
+            final Sort sort = promote.sort;
+
+            if (sort == Sort.INT) {
+                constant = (int)left.constant % (int)right.constant;
+            } else if (sort == Sort.LONG) {
+                constant = (long)left.constant % (long)right.constant;
+            } else if (sort == Sort.FLOAT) {
+                constant = overflow ? (float)left.constant % (float)right.constant :
+                    org.elasticsearch.painless.Utility.remainderWithoutOverflow((float)left.constant, (float)right.constant);
+            } else if (sort == Sort.DOUBLE) {
+                constant = overflow ? (double)left.constant % (double)right.constant :
+                    org.elasticsearch.painless.Utility.remainderWithoutOverflow((double)left.constant, (double)right.constant);
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+
+        actual = promote;
+    }
+
+    private void analyzeAdd(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        left.analyze(settings, definition, variables);
+        right.analyze(settings, definition, variables);
+
+        final Type promote = AnalyzerCaster.promoteAdd(definition, left.actual, right.actual);
+
+        if (promote == null) {
+            throw new ClassCastException(error("Cannot apply add [+] to types " +
+                "[" + left.actual.name + "] and [" + right.actual.name + "]."));
+        }
+
+        final Sort sort = promote.sort;
+
+        if (sort == Sort.STRING) {
+            left.expected = left.actual;
+
+            if (left instanceof EBinary && ((EBinary)left).operation == Operation.ADD && left.actual.sort == Sort.STRING) {
+                ((EBinary)left).cat = true;
+            }
+
+            right.expected = right.actual;
+
+            if (right instanceof EBinary && ((EBinary)right).operation == Operation.ADD && right.actual.sort == Sort.STRING) {
+                ((EBinary)right).cat = true;
+            }
+        } else {
+            left.expected = promote;
+            right.expected = promote;
+        }
+
+        left = left.cast(settings, definition, variables);
+        right = right.cast(settings, definition, variables);
+
+        if (left.constant != null && right.constant != null) {
+            final boolean overflow = settings.getNumericOverflow();
+
+            if (sort == Sort.INT) {
+                constant = overflow ? (int)left.constant + (int)right.constant :
+                    Math.addExact((int)left.constant, (int)right.constant);
+            } else if (sort == Sort.LONG) {
+                constant = overflow ? (long)left.constant + (long)right.constant :
+                    Math.addExact((long)left.constant, (long)right.constant);
+            } else if (sort == Sort.FLOAT) {
+                constant = overflow ? (float)left.constant + (float)right.constant :
+                    org.elasticsearch.painless.Utility.addWithoutOverflow((float)left.constant, (float)right.constant);
+            } else if (sort == Sort.DOUBLE) {
+                constant = overflow ? (double)left.constant + (double)right.constant :
+                    org.elasticsearch.painless.Utility.addWithoutOverflow((double)left.constant, (double)right.constant);
+            } else if (sort == Sort.STRING) {
+                constant = "" + left.constant + right.constant;
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+
+        actual = promote;
+    }
+
+    private void analyzeSub(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        left.analyze(settings, definition, variables);
+        right.analyze(settings, definition, variables);
+
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+
+        if (promote == null) {
+            throw new ClassCastException(error("Cannot apply subtract [-] to types " +
+                "[" + left.actual.name + "] and [" + right.actual.name + "]."));
+        }
+
+        left.expected = promote;
+        right.expected = promote;
+
+        left = left.cast(settings, definition, variables);
+        right = right.cast(settings, definition, variables);
+
+        if (left.constant != null && right.constant != null) {
+            final boolean overflow = settings.getNumericOverflow();
+            final Sort sort = promote.sort;
+
+            if (sort == Sort.INT) {
+                constant = overflow ? (int)left.constant - (int)right.constant :
+                    Math.subtractExact((int)left.constant, (int)right.constant);
+            } else if (sort == Sort.LONG) {
+                constant = overflow ? (long)left.constant - (long)right.constant :
+                    Math.subtractExact((long)left.constant, (long)right.constant);
+            } else if (sort == Sort.FLOAT) {
+                constant = overflow ? (float)left.constant - (float)right.constant :
+                    org.elasticsearch.painless.Utility.subtractWithoutOverflow((float)left.constant, (float)right.constant);
+            } else if (sort == Sort.DOUBLE) {
+                constant = overflow ? (double)left.constant - (double)right.constant :
+                    org.elasticsearch.painless.Utility.subtractWithoutOverflow((double)left.constant, (double)right.constant);
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+
+        actual = promote;
+    }
+
+    private void analyzeLSH(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        left.analyze(settings, definition, variables);
+        right.analyze(settings, definition, variables);
+
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, false, true);
+
+        if (promote == null) {
+            throw new ClassCastException(error("Cannot apply left shift [<<] to types " +
+                "[" + left.actual.name + "] and [" + right.actual.name + "]."));
+        }
+
+        left.expected = promote;
+        right.expected = definition.intType;
+        right.explicit = true;
+
+        left = left.cast(settings, definition, variables);
+        right = right.cast(settings, definition, variables);
+
+        if (left.constant != null && right.constant != null) {
+            final Sort sort = promote.sort;
+
+            if (sort == Sort.INT) {
+                constant = (int)left.constant << (int)right.constant;
+            } else if (sort == Sort.LONG) {
+                constant = (long)left.constant << (int)right.constant;
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+
+        actual = promote;
+    }
+
+    private void analyzeRSH(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        left.analyze(settings, definition, variables);
+        right.analyze(settings, definition, variables);
+
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, false, true);
+
+        if (promote == null) {
+            throw new ClassCastException(error("Cannot apply right shift [>>] to types " +
+                "[" + left.actual.name + "] and [" + right.actual.name + "]."));
+        }
+
+        left.expected = promote;
+        right.expected = definition.intType;
+        right.explicit = true;
+
+        left = left.cast(settings, definition, variables);
+        right = right.cast(settings, definition, variables);
+
+        if (left.constant != null && right.constant != null) {
+            final Sort sort = promote.sort;
+
+            if (sort == Sort.INT) {
+                constant = (int)left.constant >> (int)right.constant;
+            } else if (sort == Sort.LONG) {
+                constant = (long)left.constant >> (int)right.constant;
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+
+        actual = promote;
+    }
+
+    private void analyzeUSH(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        left.analyze(settings, definition, variables);
+        right.analyze(settings, definition, variables);
+
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, false, true);
+
+        if (promote == null) {
+            throw new ClassCastException(error("Cannot apply unsigned shift [>>>] to types " +
+                "[" + left.actual.name + "] and [" + right.actual.name + "]."));
+        }
+
+        left.expected = promote;
+        right.expected = definition.intType;
+        right.explicit = true;
+
+        left = left.cast(settings, definition, variables);
+        right = right.cast(settings, definition, variables);
+
+        if (left.constant != null && right.constant != null) {
+            final Sort sort = promote.sort;
+
+            if (sort == Sort.INT) {
+                constant = (int)left.constant >>> (int)right.constant;
+            } else if (sort == Sort.LONG) {
+                constant = (long)left.constant >>> (int)right.constant;
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+
+        actual = promote;
+    }
+
+    private void analyzeBWAnd(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        left.analyze(settings, definition, variables);
+        right.analyze(settings, definition, variables);
+
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, false, true);
+
+        if (promote == null) {
+            throw new ClassCastException(error("Cannot apply and [&] to types " +
+                "[" + left.actual.name + "] and [" + right.actual.name + "]."));
+        }
+
+        left.expected = promote;
+        right.expected = promote;
+
+        left = left.cast(settings, definition, variables);
+        right = right.cast(settings, definition, variables);
+
+        if (left.constant != null && right.constant != null) {
+            final Sort sort = promote.sort;
+
+            if (sort == Sort.INT) {
+                constant = (int)left.constant & (int)right.constant;
+            } else if (sort == Sort.LONG) {
+                constant = (long)left.constant & (long)right.constant;
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+
+        actual = promote;
+    }
+
+    private void analyzeXor(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        left.analyze(settings, definition, variables);
+        right.analyze(settings, definition, variables);
+
+        final Type promote = AnalyzerCaster.promoteXor(definition, left.actual, right.actual);
+
+        if (promote == null) {
+            throw new ClassCastException(error("Cannot apply xor [^] to types " +
+                "[" + left.actual.name + "] and [" + right.actual.name + "]."));
+        }
+
+        left.expected = promote;
+        right.expected = promote;
+
+        left = left.cast(settings, definition, variables);
+        right = right.cast(settings, definition, variables);
+
+        if (left.constant != null && right.constant != null) {
+            final Sort sort = promote.sort;
+
+            if (sort == Sort.BOOL) {
+                constant = (boolean)left.constant ^ (boolean)right.constant;
+            } else if (sort == Sort.INT) {
+                constant = (int)left.constant ^ (int)right.constant;
+            } else if (sort == Sort.LONG) {
+                constant = (long)left.constant ^ (long)right.constant;
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+
+        actual = promote;
+    }
+
+    private void analyzeBWOr(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        left.analyze(settings, definition, variables);
+        right.analyze(settings, definition, variables);
+
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, false, true);
+
+        if (promote == null) {
+            throw new ClassCastException(error("Cannot apply or [|] to types " +
+                "[" + left.actual.name + "] and [" + right.actual.name + "]."));
+        }
+
+        left.expected = promote;
+        right.expected = promote;
+
+        left = left.cast(settings, definition, variables);
+        right = right.cast(settings, definition, variables);
+
+        if (left.constant != null && right.constant != null) {
+            final Sort sort = promote.sort;
+
+            if (sort == Sort.INT) {
+                constant = (int)left.constant | (int)right.constant;
+            } else if (sort == Sort.LONG) {
+                constant = (long)left.constant | (long)right.constant;
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+
+        actual = promote;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        if (actual.sort == Sort.STRING && operation == Operation.ADD) {
+            if (!cat) {
+                WriterUtility.writeNewStrings(adapter);
+            }
+
+            left.write(settings, definition, adapter);
+
+            if (!(left instanceof EBinary) || ((EBinary)left).operation != Operation.ADD || left.actual.sort != Sort.STRING) {
+                WriterUtility.writeAppendStrings(adapter, left.actual.sort);
+            }
+
+            right.write(settings, definition, adapter);
+
+            if (!(right instanceof EBinary) || ((EBinary)right).operation != Operation.ADD || right.actual.sort != Sort.STRING) {
+                WriterUtility.writeAppendStrings(adapter, right.actual.sort);
+            }
+
+            if (!cat) {
+                WriterUtility.writeToStrings(adapter);
+            }
+        } else {
+            left.write(settings, definition, adapter);
+            right.write(settings, definition, adapter);
+
+            WriterUtility.writeBinaryInstruction(settings, definition, adapter, location, actual, operation);
+        }
+
+        WriterUtility.writeBranch(adapter, tru, fals);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
new file mode 100644
index 00000000000..650c4cfc97e
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Operation;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents a boolean expression.
+ */
+public final class EBool extends AExpression {
+
+    final Operation operation;
+    AExpression left;
+    AExpression right;
+
+    public EBool(final String location, final Operation operation, final AExpression left, final AExpression right) {
+        super(location);
+
+        this.operation = operation;
+        this.left = left;
+        this.right = right;
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        left.expected = definition.booleanType;
+        left.analyze(settings, definition, variables);
+        left = left.cast(settings, definition, variables);
+
+        right.expected = definition.booleanType;
+        right.analyze(settings, definition, variables);
+        right = right.cast(settings, definition, variables);
+
+        if (left.constant != null && right.constant != null) {
+            if (operation == Operation.AND) {
+                constant = (boolean)left.constant && (boolean)right.constant;
+            } else if (operation == Operation.OR) {
+                constant = (boolean)left.constant || (boolean)right.constant;
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+
+        actual = definition.booleanType;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        if (tru != null || fals != null) {
+            if (operation == Operation.AND) {
+                final Label localfals = fals == null ? new Label() : fals;
+
+                left.fals = localfals;
+                right.tru = tru;
+                right.fals = fals;
+
+                left.write(settings, definition, adapter);
+                right.write(settings, definition, adapter);
+
+                if (fals == null) {
+                    adapter.mark(localfals);
+                }
+            } else if (operation == Operation.OR) {
+                final Label localtru = tru == null ? new Label() : tru;
+
+                left.tru = localtru;
+                right.tru = tru;
+                right.fals = fals;
+
+                left.write(settings, definition, adapter);
+                right.write(settings, definition, adapter);
+
+                if (tru == null) {
+                    adapter.mark(localtru);
+                }
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        } else {
+            if (operation == Operation.AND) {
+                final Label localfals = new Label();
+                final Label end = new Label();
+
+                left.fals = localfals;
+                right.fals = localfals;
+
+                left.write(settings, definition, adapter);
+                right.write(settings, definition, adapter);
+
+                adapter.push(true);
+                adapter.goTo(end);
+                adapter.mark(localfals);
+                adapter.push(false);
+                adapter.mark(end);
+            } else if (operation == Operation.OR) {
+                final Label localtru = new Label();
+                final Label localfals = new Label();
+                final Label end = new Label();
+
+                left.tru = localtru;
+                right.fals = localfals;
+
+                left.write(settings, definition, adapter);
+                right.write(settings, definition, adapter);
+
+                adapter.mark(localtru);
+                adapter.push(true);
+                adapter.goTo(end);
+                adapter.mark(localfals);
+                adapter.push(false);
+                adapter.mark(end);
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
new file mode 100644
index 00000000000..84bf38ea459
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents a boolean constant.
+ */
+public final class EBoolean extends AExpression {
+
+    public EBoolean(final String location, final boolean constant) {
+        super(location);
+
+        this.constant = constant;
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        actual = definition.booleanType;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        throw new IllegalArgumentException(error("Illegal tree structure."));
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
new file mode 100644
index 00000000000..77d39b92081
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Cast;
+import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.WriterUtility;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents an implicit cast in most cases, though it will replace
+ * explicit casts in the tree for simplicity.  (Internal only.)
+ */
+final class ECast extends AExpression {
+
+    final String type;
+    AExpression child;
+
+    Cast cast = null;
+
+    ECast(final String location, final AExpression child, final Cast cast) {
+        super(location);
+
+        this.type = null;
+        this.child = child;
+
+        this.cast = cast;
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        throw new IllegalStateException(error("Illegal tree structure."));
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        child.write(settings, definition, adapter);
+        WriterUtility.writeCast(adapter, cast);
+        WriterUtility.writeBranch(adapter, tru, fals);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
new file mode 100644
index 00000000000..af3ba4fc422
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -0,0 +1,310 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Cast;
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.AnalyzerCaster;
+import org.elasticsearch.painless.Operation;
+import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.WriterUtility;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+import java.util.List;
+
+/**
+ * Represents the entirety of a variable/method chain for read/write operations.
+ */
+public final class EChain extends AExpression {
+
+    final List<ALink> links;
+    final boolean pre;
+    final boolean post;
+    Operation operation;
+    AExpression expression;
+
+    boolean cat = false;
+    Type promote = null;
+    boolean exact = false;
+    Cast there = null;
+    Cast back = null;
+
+    public EChain(final String location, final List<ALink> links,
+                  final boolean pre, final boolean post, final Operation operation, final AExpression expression) {
+        super(location);
+
+        this.links = links;
+        this.pre = pre;
+        this.post = post;
+        this.operation = operation;
+        this.expression = expression;
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        analyzeLinks(settings, definition, variables);
+        analyzeIncrDecr();
+
+        if (operation != null) {
+            analyzeCompound(settings, definition, variables);
+        } else if (expression != null) {
+            analyzeWrite(settings, definition, variables);
+        } else {
+            analyzeRead();
+        }
+    }
+
+    private void analyzeLinks(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        ALink previous = null;
+        int index = 0;
+
+        while (index < links.size()) {
+            final ALink current = links.get(index);
+
+            if (previous != null) {
+                current.before = previous.after;
+
+                if (index == 1) {
+                    current.statik = previous.statik;
+                }
+            }
+
+            if (index == links.size() - 1) {
+                current.load = read;
+                current.store = expression != null || pre || post;
+            }
+
+            final ALink analyzed = current.analyze(settings, definition, variables);
+
+            if (analyzed == null) {
+                links.remove(index);
+            } else {
+                if (analyzed != current) {
+                    links.set(index, analyzed);
+                }
+
+                previous = analyzed;
+                ++index;
+            }
+        }
+
+        if (links.get(0).statik) {
+            links.remove(0);
+        }
+    }
+
+    private void analyzeIncrDecr() {
+        final ALink last = links.get(links.size() - 1);
+
+        if (pre && post) {
+            throw new IllegalStateException(error("Illegal tree structure."));
+        } else if (pre || post) {
+            if (expression != null) {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+
+            final Sort sort = last.after.sort;
+
+            if (operation == Operation.INCR) {
+                if (sort == Sort.DOUBLE) {
+                    expression = new EConstant(location, 1D);
+                } else if (sort == Sort.FLOAT) {
+                    expression = new EConstant(location, 1F);
+                } else if (sort == Sort.LONG) {
+                    expression = new EConstant(location, 1L);
+                } else {
+                    expression = new EConstant(location, 1);
+                }
+
+                operation = Operation.ADD;
+            } else if (operation == Operation.DECR) {
+                if (sort == Sort.DOUBLE) {
+                    expression = new EConstant(location, 1D);
+                } else if (sort == Sort.FLOAT) {
+                    expression = new EConstant(location, 1F);
+                } else if (sort == Sort.LONG) {
+                    expression = new EConstant(location, 1L);
+                } else {
+                    expression = new EConstant(location, 1);
+                }
+
+                operation = Operation.SUB;
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+    }
+
+    private void analyzeCompound(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        final ALink last = links.get(links.size() - 1);
+
+        expression.analyze(settings, definition, variables);
+
+        if (operation == Operation.MUL) {
+            promote = AnalyzerCaster.promoteNumeric(definition, last.after, expression.actual, true, true);
+        } else if (operation == Operation.DIV) {
+            promote = AnalyzerCaster.promoteNumeric(definition, last.after, expression.actual, true, true);
+        } else if (operation == Operation.REM) {
+            promote = AnalyzerCaster.promoteNumeric(definition, last.after, expression.actual, true, true);
+        } else if (operation == Operation.ADD) {
+            promote = AnalyzerCaster.promoteAdd(definition, last.after, expression.actual);
+        } else if (operation == Operation.SUB) {
+            promote = AnalyzerCaster.promoteNumeric(definition, last.after, expression.actual, true, true);
+        } else if (operation == Operation.LSH) {
+            promote = AnalyzerCaster.promoteNumeric(definition, last.after, false, true);
+        } else if (operation == Operation.RSH) {
+            promote = AnalyzerCaster.promoteNumeric(definition, last.after, false, true);
+        } else if (operation == Operation.USH) {
+            promote = AnalyzerCaster.promoteNumeric(definition, last.after, false, true);
+        } else if (operation == Operation.BWAND) {
+            promote = AnalyzerCaster.promoteXor(definition, last.after, expression.actual);
+        } else if (operation == Operation.XOR) {
+            promote = AnalyzerCaster.promoteXor(definition, last.after, expression.actual);
+        } else if (operation == Operation.BWOR) {
+            promote = AnalyzerCaster.promoteXor(definition, last.after, expression.actual);
+        } else {
+            throw new IllegalStateException(error("Illegal tree structure."));
+        }
+
+        if (promote == null) {
+            throw new ClassCastException("Cannot apply compound assignment " +
+                "[" + operation.symbol + "=] to types [" + last.after + "] and [" + expression.actual + "].");
+        }
+
+        cat = operation == Operation.ADD && promote.sort == Sort.STRING;
+
+        if (cat) {
+            if (expression instanceof EBinary && ((EBinary)expression).operation == Operation.ADD &&
+                expression.actual.sort == Sort.STRING) {
+                ((EBinary)expression).cat = true;
+            }
+
+            expression.expected = expression.actual;
+        } else if (operation == Operation.LSH || operation == Operation.RSH || operation == Operation.USH) {
+            expression.expected = definition.intType;
+            expression.explicit = true;
+        } else {
+            expression.expected = promote;
+        }
+
+        expression = expression.cast(settings, definition, variables);
+
+        exact = !settings.getNumericOverflow() &&
+            (operation == Operation.MUL || operation == Operation.DIV || operation == Operation.REM ||
+                operation == Operation.ADD || operation == Operation.SUB);
+        there = AnalyzerCaster.getLegalCast(definition, location, last.after, promote, false);
+        back = AnalyzerCaster.getLegalCast(definition, location, promote, last.after, true);
+
+        statement = true;
+        actual = read ? last.after : definition.voidType;
+    }
+
+    private void analyzeWrite(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        final ALink last = links.get(links.size() - 1);
+
+        expression.expected = last.after;
+        expression.analyze(settings, definition, variables);
+        expression = expression.cast(settings, definition, variables);
+
+        statement = true;
+        actual = read ? last.after : definition.voidType;
+    }
+
+    private void analyzeRead() {
+        final ALink last = links.get(links.size() - 1);
+
+        constant = last.string;
+        statement = last.statement;
+        actual = last.after;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        if (cat) {
+            WriterUtility.writeNewStrings(adapter);
+        }
+
+        final ALink last = links.get(links.size() - 1);
+
+        for (final ALink link : links) {
+            link.write(settings, definition, adapter);
+
+            if (link == last && link.store) {
+                if (cat) {
+                    WriterUtility.writeDup(adapter, link.size, 1);
+                    link.load(settings, definition, adapter);
+                    WriterUtility.writeAppendStrings(adapter, link.after.sort);
+
+                    expression.write(settings, definition, adapter);
+
+                    if (!(expression instanceof EBinary) ||
+                        ((EBinary)expression).operation != Operation.ADD || expression.actual.sort != Sort.STRING) {
+                        WriterUtility.writeAppendStrings(adapter, expression.actual.sort);
+                    }
+
+                    WriterUtility.writeToStrings(adapter);
+                    WriterUtility.writeCast(adapter, back);
+
+                    if (link.load) {
+                        WriterUtility.writeDup(adapter, link.after.sort.size, link.size);
+                    }
+
+                    link.store(settings, definition, adapter);
+                } else if (operation != null) {
+                    WriterUtility.writeDup(adapter, link.size, 0);
+                    link.load(settings, definition, adapter);
+
+                    if (link.load && post) {
+                        WriterUtility.writeDup(adapter, link.after.sort.size, link.size);
+                    }
+
+                    WriterUtility.writeCast(adapter, there);
+                    expression.write(settings, definition, adapter);
+                    WriterUtility.writeBinaryInstruction(settings, definition, adapter, location, promote, operation);
+
+                    if (!exact || !WriterUtility.writeExactInstruction(definition, adapter, promote.sort, link.after.sort)) {
+                        WriterUtility.writeCast(adapter, back);
+                    }
+
+                    if (link.load && !post) {
+                        WriterUtility.writeDup(adapter, link.after.sort.size, link.size);
+                    }
+
+                    link.store(settings, definition, adapter);
+                } else {
+                    expression.write(settings, definition, adapter);
+
+                    if (link.load) {
+                        WriterUtility.writeDup(adapter, link.after.sort.size, link.size);
+                    }
+
+                    link.store(settings, definition, adapter);
+                }
+            } else {
+                link.load(settings, definition, adapter);
+            }
+        }
+
+        WriterUtility.writeBranch(adapter, tru, fals);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
new file mode 100644
index 00000000000..0fd14237ef5
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -0,0 +1,528 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.AnalyzerCaster;
+import org.elasticsearch.painless.Operation;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+import static org.elasticsearch.painless.WriterConstants.CHECKEQUALS;
+import static org.elasticsearch.painless.WriterConstants.DEF_EQ_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_GTE_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_GT_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_LTE_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_LT_CALL;
+
+/**
+ * Represents a comparison expression.
+ */
+public final class EComp extends AExpression {
+
+    final Operation operation;
+    AExpression left;
+    AExpression right;
+
+    public EComp(final String location, final Operation operation, final AExpression left, final AExpression right) {
+        super(location);
+
+        this.operation = operation;
+        this.left = left;
+        this.right = right;
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        if (operation == Operation.EQ) {
+            analyzeEq(settings, definition, variables);
+        } else if (operation == Operation.EQR) {
+            analyzeEqR(settings, definition, variables);
+        } else if (operation == Operation.NE) {
+            analyzeNE(settings, definition, variables);
+        } else if (operation == Operation.NER) {
+            analyzeNER(settings, definition, variables);
+        } else if (operation == Operation.GTE) {
+            analyzeGTE(settings, definition, variables);
+        } else if (operation == Operation.GT) {
+            analyzeGT(settings, definition, variables);
+        } else if (operation == Operation.LTE) {
+            analyzeLTE(settings, definition, variables);
+        } else if (operation == Operation.LT) {
+            analyzeLT(settings, definition, variables);
+        } else {
+            throw new IllegalStateException(error("Illegal tree structure."));
+        }
+    }
+
+    private void analyzeEq(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        left.analyze(settings, definition, variables);
+        right.analyze(settings, definition, variables);
+
+        final Type promote = AnalyzerCaster.promoteEquality(definition, left.actual, right.actual);
+
+        if (promote == null) {
+            throw new ClassCastException(error("Cannot apply equals [==] to types " +
+                "[" + left.actual.name + "] and [" + right.actual.name + "]."));
+        }
+
+        left.expected = promote;
+        right.expected = promote;
+
+        left = left.cast(settings, definition, variables);
+        right = right.cast(settings, definition, variables);
+
+        if (left.isNull && right.isNull) {
+            throw new IllegalArgumentException(error("Extraneous comparison of null constants."));
+        }
+
+        if ((left.constant != null || left.isNull) && (right.constant != null || right.isNull)) {
+            final Sort sort = promote.sort;
+
+            if (sort == Sort.BOOL) {
+                constant = (boolean)left.constant == (boolean)right.constant;
+            } else if (sort == Sort.INT) {
+                constant = (int)left.constant == (int)right.constant;
+            } else if (sort == Sort.LONG) {
+                constant = (long)left.constant == (long)right.constant;
+            } else if (sort == Sort.FLOAT) {
+                constant = (float)left.constant == (float)right.constant;
+            } else if (sort == Sort.DOUBLE) {
+                constant = (double)left.constant == (double)right.constant;
+            } else if (!left.isNull) {
+                constant = left.constant.equals(right.constant);
+            } else if (!right.isNull) {
+                constant = right.constant.equals(null);
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+
+        actual = definition.booleanType;
+    }
+
+    private void analyzeEqR(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        left.analyze(settings, definition, variables);
+        right.analyze(settings, definition, variables);
+
+        final Type promote = AnalyzerCaster.promoteReference(definition, left.actual, right.actual);
+
+        if (promote == null) {
+            throw new ClassCastException(error("Cannot apply reference equals [===] to types " +
+                "[" + left.actual.name + "] and [" + right.actual.name + "]."));
+        }
+
+        left.expected = promote;
+        right.expected = promote;
+
+        left = left.cast(settings, definition, variables);
+        right = right.cast(settings, definition, variables);
+
+        if (left.isNull && right.isNull) {
+            throw new IllegalArgumentException(error("Extraneous comparison of null constants."));
+        }
+
+        if ((left.constant != null || left.isNull) && (right.constant != null || right.isNull)) {
+            final Sort sort = promote.sort;
+
+            if (sort == Sort.BOOL) {
+                constant = (boolean)left.constant == (boolean)right.constant;
+            } else if (sort == Sort.INT) {
+                constant = (int)left.constant == (int)right.constant;
+            } else if (sort == Sort.LONG) {
+                constant = (long)left.constant == (long)right.constant;
+            } else if (sort == Sort.FLOAT) {
+                constant = (float)left.constant == (float)right.constant;
+            } else if (sort == Sort.DOUBLE) {
+                constant = (double)left.constant == (double)right.constant;
+            } else {
+                constant = left.constant == right.constant;
+            }
+        }
+
+        actual = definition.booleanType;
+    }
+
+    private void analyzeNE(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        left.analyze(settings, definition, variables);
+        right.analyze(settings, definition, variables);
+
+        final Type promote = AnalyzerCaster.promoteEquality(definition, left.actual, right.actual);
+
+        if (promote == null) {
+            throw new ClassCastException(error("Cannot apply not equals [!=] to types " +
+                "[" + left.actual.name + "] and [" + right.actual.name + "]."));
+        }
+
+        left.expected = promote;
+        right.expected = promote;
+
+        left = left.cast(settings, definition, variables);
+        right = right.cast(settings, definition, variables);
+
+        if (left.isNull && right.isNull) {
+            throw new IllegalArgumentException(error("Extraneous comparison of null constants."));
+        }
+
+        if ((left.constant != null || left.isNull) && (right.constant != null || right.isNull)) {
+            final Sort sort = promote.sort;
+
+            if (sort == Sort.BOOL) {
+                constant = (boolean)left.constant != (boolean)right.constant;
+            } else if (sort == Sort.INT) {
+                constant = (int)left.constant != (int)right.constant;
+            } else if (sort == Sort.LONG) {
+                constant = (long)left.constant != (long)right.constant;
+            } else if (sort == Sort.FLOAT) {
+                constant = (float)left.constant != (float)right.constant;
+            } else if (sort == Sort.DOUBLE) {
+                constant = (double)left.constant != (double)right.constant;
+            } else if (!left.isNull) {
+                constant = !left.constant.equals(right.constant);
+            } else if (!right.isNull) {
+                constant = !right.constant.equals(null);
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+
+        actual = definition.booleanType;
+    }
+
+    private void analyzeNER(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        left.analyze(settings, definition, variables);
+        right.analyze(settings, definition, variables);
+
+        final Type promote = AnalyzerCaster.promoteReference(definition, left.actual, right.actual);
+
+        if (promote == null) {
+            throw new ClassCastException(error("Cannot apply reference not equals [!==] to types " +
+                "[" + left.actual.name + "] and [" + right.actual.name + "]."));
+        }
+
+        left.expected = promote;
+        right.expected = promote;
+
+        left = left.cast(settings, definition, variables);
+        right = right.cast(settings, definition, variables);
+
+        if (left.isNull && right.isNull) {
+            throw new IllegalArgumentException(error("Extraneous comparison of null constants."));
+        }
+
+        if ((left.constant != null || left.isNull) && (right.constant != null || right.isNull)) {
+            final Sort sort = promote.sort;
+
+            if (sort == Sort.BOOL) {
+                constant = (boolean)left.constant != (boolean)right.constant;
+            } else if (sort == Sort.INT) {
+                constant = (int)left.constant != (int)right.constant;
+            } else if (sort == Sort.LONG) {
+                constant = (long)left.constant != (long)right.constant;
+            } else if (sort == Sort.FLOAT) {
+                constant = (float)left.constant != (float)right.constant;
+            } else if (sort == Sort.DOUBLE) {
+                constant = (double)left.constant != (double)right.constant;
+            } else {
+                constant = left.constant != right.constant;
+            }
+        }
+
+        actual = definition.booleanType;
+    }
+
+    private void analyzeGTE(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        left.analyze(settings, definition, variables);
+        right.analyze(settings, definition, variables);
+
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+
+        if (promote == null) {
+            throw new ClassCastException(error("Cannot apply greater than or equals [>=] to types " +
+                "[" + left.actual.name + "] and [" + right.actual.name + "]."));
+        }
+
+        left.expected = promote;
+        right.expected = promote;
+
+        left = left.cast(settings, definition, variables);
+        right = right.cast(settings, definition, variables);
+
+        if (left.constant != null && right.constant != null) {
+            final Sort sort = promote.sort;
+
+            if (sort == Sort.INT) {
+                constant = (int)left.constant >= (int)right.constant;
+            } else if (sort == Sort.LONG) {
+                constant = (long)left.constant >= (long)right.constant;
+            } else if (sort == Sort.FLOAT) {
+                constant = (float)left.constant >= (float)right.constant;
+            } else if (sort == Sort.DOUBLE) {
+                constant = (double)left.constant >= (double)right.constant;
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+
+        actual = definition.booleanType;
+    }
+
+    private void analyzeGT(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        left.analyze(settings, definition, variables);
+        right.analyze(settings, definition, variables);
+
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+
+        if (promote == null) {
+            throw new ClassCastException(error("Cannot apply greater than [>] to types " +
+                "[" + left.actual.name + "] and [" + right.actual.name + "]."));
+        }
+
+        left.expected = promote;
+        right.expected = promote;
+
+        left = left.cast(settings, definition, variables);
+        right = right.cast(settings, definition, variables);
+
+        if (left.constant != null && right.constant != null) {
+            final Sort sort = promote.sort;
+
+            if (sort == Sort.INT) {
+                constant = (int)left.constant > (int)right.constant;
+            } else if (sort == Sort.LONG) {
+                constant = (long)left.constant > (long)right.constant;
+            } else if (sort == Sort.FLOAT) {
+                constant = (float)left.constant > (float)right.constant;
+            } else if (sort == Sort.DOUBLE) {
+                constant = (double)left.constant > (double)right.constant;
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+
+        actual = definition.booleanType;
+    }
+
+    private void analyzeLTE(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        left.analyze(settings, definition, variables);
+        right.analyze(settings, definition, variables);
+
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+
+        if (promote == null) {
+            throw new ClassCastException(error("Cannot apply less than or equals [<=] to types " +
+                "[" + left.actual.name + "] and [" + right.actual.name + "]."));
+        }
+
+        left.expected = promote;
+        right.expected = promote;
+
+        left = left.cast(settings, definition, variables);
+        right = right.cast(settings, definition, variables);
+
+        if (left.constant != null && right.constant != null) {
+            final Sort sort = promote.sort;
+
+            if (sort == Sort.INT) {
+                constant = (int)left.constant <= (int)right.constant;
+            } else if (sort == Sort.LONG) {
+                constant = (long)left.constant <= (long)right.constant;
+            } else if (sort == Sort.FLOAT) {
+                constant = (float)left.constant <= (float)right.constant;
+            } else if (sort == Sort.DOUBLE) {
+                constant = (double)left.constant <= (double)right.constant;
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+
+        actual = definition.booleanType;
+    }
+
+    private void analyzeLT(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        left.analyze(settings, definition, variables);
+        right.analyze(settings, definition, variables);
+
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+
+        if (promote == null) {
+            throw new ClassCastException(error("Cannot apply less than [>=] to types " +
+                "[" + left.actual.name + "] and [" + right.actual.name + "]."));
+        }
+
+        left.expected = promote;
+        right.expected = promote;
+
+        left = left.cast(settings, definition, variables);
+        right = right.cast(settings, definition, variables);
+
+        if (left.constant != null && right.constant != null) {
+            final Sort sort = promote.sort;
+
+            if (sort == Sort.INT) {
+                constant = (int)left.constant < (int)right.constant;
+            } else if (sort == Sort.LONG) {
+                constant = (long)left.constant < (long)right.constant;
+            } else if (sort == Sort.FLOAT) {
+                constant = (float)left.constant < (float)right.constant;
+            } else if (sort == Sort.DOUBLE) {
+                constant = (double)left.constant < (double)right.constant;
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+
+        actual = definition.booleanType;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        final boolean branch = tru != null || fals != null;
+        final org.objectweb.asm.Type rtype = right.actual.type;
+        final Sort rsort = right.actual.sort;
+
+        left.write(settings, definition, adapter);
+
+        if (!right.isNull) {
+            right.write(settings, definition, adapter);
+        }
+
+        final Label jump = tru != null ? tru : fals != null ? fals : new Label();
+        final Label end = new Label();
+
+        final boolean eq = (operation == Operation.EQ || operation == Operation.EQR) && (tru != null || fals == null) ||
+            (operation == Operation.NE || operation == Operation.NER) && fals != null;
+        final boolean ne = (operation == Operation.NE || operation == Operation.NER) && (tru != null || fals == null) ||
+            (operation == Operation.EQ || operation == Operation.EQR) && fals != null;
+        final boolean lt  = operation == Operation.LT  && (tru != null || fals == null) || operation == Operation.GTE && fals != null;
+        final boolean lte = operation == Operation.LTE && (tru != null || fals == null) || operation == Operation.GT  && fals != null;
+        final boolean gt  = operation == Operation.GT  && (tru != null || fals == null) || operation == Operation.LTE && fals != null;
+        final boolean gte = operation == Operation.GTE && (tru != null || fals == null) || operation == Operation.LT  && fals != null;
+
+        boolean writejump = true;
+
+        switch (rsort) {
+            case VOID:
+            case BYTE:
+            case SHORT:
+            case CHAR:
+                throw new IllegalStateException(error("Illegal tree structure."));
+            case BOOL:
+                if      (eq) adapter.ifZCmp(GeneratorAdapter.EQ, jump);
+                else if (ne) adapter.ifZCmp(GeneratorAdapter.NE, jump);
+                else {
+                    throw new IllegalStateException(error("Illegal tree structure."));
+                }
+
+                break;
+            case INT:
+            case LONG:
+            case FLOAT:
+            case DOUBLE:
+                if      (eq)  adapter.ifCmp(rtype, GeneratorAdapter.EQ, jump);
+                else if (ne)  adapter.ifCmp(rtype, GeneratorAdapter.NE, jump);
+                else if (lt)  adapter.ifCmp(rtype, GeneratorAdapter.LT, jump);
+                else if (lte) adapter.ifCmp(rtype, GeneratorAdapter.LE, jump);
+                else if (gt)  adapter.ifCmp(rtype, GeneratorAdapter.GT, jump);
+                else if (gte) adapter.ifCmp(rtype, GeneratorAdapter.GE, jump);
+                else {
+                    throw new IllegalStateException(error("Illegal tree structure."));
+                }
+
+                break;
+            case DEF:
+                if (eq) {
+                    if (right.isNull) {
+                        adapter.ifNull(jump);
+                    } else if (!left.isNull && operation == Operation.EQ) {
+                        adapter.invokeStatic(definition.defobjType.type, DEF_EQ_CALL);
+                    } else {
+                        adapter.ifCmp(rtype, GeneratorAdapter.EQ, jump);
+                    }
+                } else if (ne) {
+                    if (right.isNull) {
+                        adapter.ifNonNull(jump);
+                    } else if (!left.isNull && operation == Operation.NE) {
+                        adapter.invokeStatic(definition.defobjType.type, DEF_EQ_CALL);
+                        adapter.ifZCmp(GeneratorAdapter.EQ, jump);
+                    } else {
+                        adapter.ifCmp(rtype, GeneratorAdapter.NE, jump);
+                    }
+                } else if (lt) {
+                    adapter.invokeStatic(definition.defobjType.type, DEF_LT_CALL);
+                } else if (lte) {
+                    adapter.invokeStatic(definition.defobjType.type, DEF_LTE_CALL);
+                } else if (gt) {
+                    adapter.invokeStatic(definition.defobjType.type, DEF_GT_CALL);
+                } else if (gte) {
+                    adapter.invokeStatic(definition.defobjType.type, DEF_GTE_CALL);
+                } else {
+                    throw new IllegalStateException(error("Illegal tree structure."));
+                }
+
+                writejump = left.isNull || ne || operation == Operation.EQR;
+
+                if (branch && !writejump) {
+                    adapter.ifZCmp(GeneratorAdapter.NE, jump);
+                }
+
+                break;
+            default:
+                if (eq) {
+                    if (right.isNull) {
+                        adapter.ifNull(jump);
+                    } else if (operation == Operation.EQ) {
+                        adapter.invokeStatic(definition.utilityType.type, CHECKEQUALS);
+
+                        if (branch) {
+                            adapter.ifZCmp(GeneratorAdapter.NE, jump);
+                        }
+
+                        writejump = false;
+                    } else {
+                        adapter.ifCmp(rtype, GeneratorAdapter.EQ, jump);
+                    }
+                } else if (ne) {
+                    if (right.isNull) {
+                        adapter.ifNonNull(jump);
+                    } else if (operation == Operation.NE) {
+                        adapter.invokeStatic(definition.utilityType.type, CHECKEQUALS);
+                        adapter.ifZCmp(GeneratorAdapter.EQ, jump);
+                    } else {
+                        adapter.ifCmp(rtype, GeneratorAdapter.NE, jump);
+                    }
+                } else {
+                    throw new IllegalStateException(error("Illegal tree structure."));
+                }
+        }
+
+        if (!branch && writejump) {
+            adapter.push(false);
+            adapter.goTo(end);
+            adapter.mark(jump);
+            adapter.push(true);
+            adapter.mark(end);
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
new file mode 100644
index 00000000000..0006e5f8b97
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.AnalyzerCaster;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Respresents a conditional expression.
+ */
+public final class EConditional extends AExpression {
+
+    AExpression condition;
+    AExpression left;
+    AExpression right;
+
+    public EConditional(final String location, final AExpression condition, final AExpression left, final AExpression right) {
+        super(location);
+
+        this.condition = condition;
+        this.left = left;
+        this.right = right;
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        condition.expected = definition.booleanType;
+        condition.analyze(settings, definition, variables);
+        condition = condition.cast(settings, definition, variables);
+
+        if (condition.constant != null) {
+            throw new IllegalArgumentException(error("Extraneous conditional statement."));
+        }
+
+        left.expected = expected;
+        left.explicit = explicit;
+        right.expected = expected;
+        right.explicit = explicit;
+        actual = expected;
+
+        left.analyze(settings, definition, variables);
+        right.analyze(settings, definition, variables);
+
+        if (expected == null) {
+            final Type promote = AnalyzerCaster.promoteConditional(definition, left.actual, right.actual, left.constant, right.constant);
+
+            left.expected = promote;
+            right.expected = promote;
+            actual = promote;
+        }
+
+        left = left.cast(settings, definition, variables);
+        right = right.cast(settings, definition, variables);
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        final Label localfals = new Label();
+        final Label end = new Label();
+
+        condition.fals = localfals;
+        left.tru = right.tru = tru;
+        left.fals = right.fals = fals;
+
+        condition.write(settings, definition, adapter);
+        left.write(settings, definition, adapter);
+        adapter.goTo(end);
+        adapter.mark(localfals);
+        right.write(settings, definition, adapter);
+        adapter.mark(end);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
new file mode 100644
index 00000000000..5b1e17142d9
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.WriterUtility;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Respresents a constant.  Note this replaces any other expression
+ * node with a constant value set during a cast.  (Internal only.)
+ */
+final class EConstant extends AExpression {
+
+    EConstant(final String location, final Object constant) {
+        super(location);
+
+        this.constant = constant;
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        if (constant instanceof String) {
+            actual = definition.stringType;
+        } else if (constant instanceof Double) {
+            actual = definition.doubleType;
+        } else if (constant instanceof Float) {
+            actual = definition.floatType;
+        } else if (constant instanceof Long) {
+            actual = definition.longType;
+        } else if (constant instanceof Integer) {
+            actual = definition.intType;
+        } else if (constant instanceof Character) {
+            actual = definition.charType;
+        } else if (constant instanceof Short) {
+            actual = definition.shortType;
+        } else if (constant instanceof Byte) {
+            actual = definition.byteType;
+        } else if (constant instanceof Boolean) {
+            actual = definition.booleanType;
+        } else {
+            throw new IllegalStateException(error("Illegal tree structure."));
+        }
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        final Sort sort = actual.sort;
+
+        switch (sort) {
+            case STRING: adapter.push((String)constant);  break;
+            case DOUBLE: adapter.push((double)constant);  break;
+            case FLOAT:  adapter.push((float)constant);   break;
+            case LONG:   adapter.push((long)constant);    break;
+            case INT:    adapter.push((int)constant);     break;
+            case CHAR:   adapter.push((char)constant);    break;
+            case SHORT:  adapter.push((short)constant);   break;
+            case BYTE:   adapter.push((byte)constant);    break;
+            case BOOL:
+                if (tru != null && (boolean)constant) {
+                    adapter.goTo(tru);
+                } else if (fals != null && !(boolean)constant) {
+                    adapter.goTo(fals);
+                } else if (tru == null && fals == null) {
+                    adapter.push((boolean)constant);
+                }
+
+                break;
+            default:
+                throw new IllegalStateException(error("Illegal tree structure."));
+        }
+
+        if (sort != Sort.BOOL) {
+            WriterUtility.writeBranch(adapter, tru, fals);
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
new file mode 100644
index 00000000000..c2fc010aa51
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Respresents a decimal constant.
+ */
+public final class EDecimal extends AExpression {
+
+    final String value;
+
+    public EDecimal(final String location, final String value) {
+        super(location);
+
+        this.value = value;
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        if (value.endsWith("f") || value.endsWith("F")) {
+            try {
+                constant = Float.parseFloat(value.substring(0, value.length() - 1));
+                actual = definition.floatType;
+            } catch (final NumberFormatException exception) {
+                throw new IllegalArgumentException(error("Invalid float constant [" + value + "]."));
+            }
+        } else {
+            try {
+                constant = Double.parseDouble(value);
+                actual = definition.doubleType;
+            } catch (final NumberFormatException exception) {
+                throw new IllegalArgumentException(error("Invalid double constant [" + value + "]."));
+            }
+        }
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        throw new IllegalArgumentException(error("Illegal tree structure."));
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
new file mode 100644
index 00000000000..0a3a39df7af
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Cast;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents an explicit cast.
+ */
+public final class EExplicit extends AExpression {
+
+    final String type;
+    AExpression child;
+
+    Cast cast = null;
+
+    public EExplicit(final String location, final String type, final AExpression child) {
+        super(location);
+
+        this.type = type;
+        this.child = child;
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        try {
+            actual = definition.getType(this.type);
+        } catch (final IllegalArgumentException exception) {
+            throw new IllegalArgumentException(error("Not a type [" + this.type + "]."));
+        }
+
+        child.expected = actual;
+        child.explicit = true;
+        child.analyze(settings, definition, variables);
+        child = child.cast(settings, definition, variables);
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        throw new IllegalArgumentException(error("Illegal tree structure."));
+    }
+
+    AExpression cast(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        child.expected = expected;
+        child.explicit = explicit;
+
+        return child.cast(settings, definition, variables);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
new file mode 100644
index 00000000000..189b12d9f97
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents a null constant.
+ */
+public final class ENull extends AExpression {
+
+    public ENull(final String location) {
+        super(location);
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        isNull = true;
+
+        if (expected != null) {
+            if (expected.sort.primitive) {
+                throw new IllegalArgumentException(error("Cannot cast null to a primitive type [" + expected.name + "]."));
+            }
+
+            actual = expected;
+        } else {
+            actual = definition.objectType;
+        }
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        adapter.visitInsn(Opcodes.ACONST_NULL);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
new file mode 100644
index 00000000000..32260483b64
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Respresents a non-decimal numeric constant.
+ */
+public final class ENumeric extends AExpression {
+
+    final String value;
+    int radix;
+
+    public ENumeric(final String location, final String value, final int radix) {
+        super(location);
+
+        this.value = value;
+        this.radix = radix;
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        if (value.endsWith("d") || value.endsWith("D")) {
+            if (radix != 10) {
+                throw new IllegalStateException(error("Invalid tree structure."));
+            }
+
+            try {
+                constant = Double.parseDouble(value.substring(0, value.length() - 1));
+                actual = definition.doubleType;
+            } catch (final NumberFormatException exception) {
+                throw new IllegalArgumentException(error("Invalid double constant [" + value + "]."));
+            }
+        } else if (value.endsWith("f") || value.endsWith("F")) {
+            if (radix != 10) {
+                throw new IllegalStateException(error("Invalid tree structure."));
+            }
+
+            try {
+                constant = Float.parseFloat(value.substring(0, value.length() - 1));
+                actual = definition.floatType;
+            } catch (final NumberFormatException exception) {
+                throw new IllegalArgumentException(error("Invalid float constant [" + value + "]."));
+            }
+        } else if (value.endsWith("l") || value.endsWith("L")) {
+            try {
+                constant = Long.parseLong(value.substring(0, value.length() - 1), radix);
+                actual = definition.longType;
+            } catch (final NumberFormatException exception) {
+                throw new IllegalArgumentException(error("Invalid long constant [" + value + "]."));
+            }
+        } else {
+            try {
+                final Sort sort = expected == null ? Sort.INT : expected.sort;
+                final int integer = Integer.parseInt(value, radix);
+
+                if (sort == Sort.BYTE && integer >= Byte.MIN_VALUE && integer <= Byte.MAX_VALUE) {
+                    constant = (byte)integer;
+                    actual = definition.byteType;
+                } else if (sort == Sort.CHAR && integer >= Character.MIN_VALUE && integer <= Character.MAX_VALUE) {
+                    constant = (char)integer;
+                    actual = definition.charType;
+                } else if (sort == Sort.SHORT && integer >= Short.MIN_VALUE && integer <= Short.MAX_VALUE) {
+                    constant = (short)integer;
+                    actual = definition.shortType;
+                } else {
+                    constant = integer;
+                    actual = definition.intType;
+                }
+            } catch (final NumberFormatException exception) {
+                throw new IllegalArgumentException(error("Invalid int constant [" + value + "]."));
+            }
+        }
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        throw new IllegalArgumentException(error("Illegal tree structure."));
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
new file mode 100644
index 00000000000..7f6b52be89c
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.AnalyzerCaster;
+import org.elasticsearch.painless.Operation;
+import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.WriterUtility;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+import static org.elasticsearch.painless.WriterConstants.DEF_NEG_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_NOT_CALL;
+import static org.elasticsearch.painless.WriterConstants.NEGATEEXACT_INT;
+import static org.elasticsearch.painless.WriterConstants.NEGATEEXACT_LONG;
+
+/**
+ * Represents a unary math expression.
+ */
+public final class EUnary extends AExpression {
+
+    Operation operation;
+    AExpression child;
+
+    public EUnary(final String location, final Operation operation, final AExpression child) {
+        super(location);
+
+        this.operation = operation;
+        this.child = child;
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        if (operation == Operation.NOT) {
+            analyzeNot(settings, definition, variables);
+        } else if (operation == Operation.BWNOT) {
+            analyzeBWNot(settings, definition, variables);
+        } else if (operation == Operation.ADD) {
+            analyzerAdd(settings, definition, variables);
+        } else if (operation == Operation.SUB) {
+            analyzerSub(settings, definition, variables);
+        } else {
+            throw new IllegalStateException(error("Illegal tree structure."));
+        }
+    }
+
+    void analyzeNot(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        child.expected = definition.booleanType;
+        child.analyze(settings, definition, variables);
+        child = child.cast(settings, definition, variables);
+
+        if (child.constant != null) {
+            constant = !(boolean)child.constant;
+        }
+
+        actual = definition.booleanType;
+    }
+
+    void analyzeBWNot(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        child.analyze(settings, definition, variables);
+
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, child.actual, false, true);
+
+        if (promote == null) {
+            throw new ClassCastException(error("Cannot apply not [~] to type [" + child.actual.name + "]."));
+        }
+
+        child.expected = promote;
+        child = child.cast(settings, definition, variables);
+
+        if (child.constant != null) {
+            final Sort sort = promote.sort;
+
+            if (sort == Sort.INT) {
+                constant = ~(int)child.constant;
+            } else if (sort == Sort.LONG) {
+                constant = ~(long)child.constant;
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+
+        actual = promote;
+    }
+
+    void analyzerAdd(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        child.analyze(settings, definition, variables);
+
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, child.actual, true, true);
+
+        if (promote == null) {
+            throw new ClassCastException(error("Cannot apply positive [+] to type [" + child.actual.name + "]."));
+        }
+
+        child.expected = promote;
+        child = child.cast(settings, definition, variables);
+
+        if (child.constant != null) {
+            final Sort sort = promote.sort;
+
+            if (sort == Sort.INT) {
+                constant = +(int)child.constant;
+            } else if (sort == Sort.LONG) {
+                constant = +(long)child.constant;
+            } else if (sort == Sort.FLOAT) {
+                constant = +(float)child.constant;
+            } else if (sort == Sort.DOUBLE) {
+                constant = +(double)child.constant;
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+
+        actual = promote;
+    }
+
+    void analyzerSub(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        child.analyze(settings, definition, variables);
+
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, child.actual, true, true);
+
+        if (promote == null) {
+            throw new ClassCastException(error("Cannot apply negative [-] to type [" + child.actual.name + "]."));
+        }
+
+        child.expected = promote;
+        child = child.cast(settings, definition, variables);
+
+        if (child.constant != null) {
+            final boolean overflow = settings.getNumericOverflow();
+            final Sort sort = promote.sort;
+
+
+            if (sort == Sort.INT) {
+                constant = overflow ? -(int)child.constant : Math.negateExact((int)child.constant);
+            } else if (sort == Sort.LONG) {
+                constant = overflow ? -(long)child.constant : Math.negateExact((long)child.constant);
+            } else if (sort == Sort.FLOAT) {
+                constant = -(float)child.constant;
+            } else if (sort == Sort.DOUBLE) {
+                constant = -(double)child.constant;
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+
+        actual = promote;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        if (operation == Operation.NOT) {
+            if (tru == null && fals == null) {
+                final Label localfals = new Label();
+                final Label end = new Label();
+
+                child.fals = localfals;
+                child.write(settings, definition, adapter);
+
+                adapter.push(false);
+                adapter.goTo(end);
+                adapter.mark(localfals);
+                adapter.push(true);
+                adapter.mark(end);
+            } else {
+                child.tru = fals;
+                child.fals = tru;
+                child.write(settings, definition, adapter);
+            }
+        } else {
+            final org.objectweb.asm.Type type = actual.type;
+            final Sort sort = actual.sort;
+
+            child.write(settings, definition, adapter);
+
+            if (operation == Operation.BWNOT) {
+                if (sort == Sort.DEF) {
+                    adapter.invokeStatic(definition.defobjType.type, DEF_NOT_CALL);
+                } else {
+                    if (sort == Sort.INT) {
+                        adapter.push(-1);
+                    } else if (sort == Sort.LONG) {
+                        adapter.push(-1L);
+                    } else {
+                        throw new IllegalStateException(error("Illegal tree structure."));
+                    }
+
+                    adapter.math(GeneratorAdapter.XOR, type);
+                }
+            } else if (operation == Operation.SUB) {
+                if (sort == Sort.DEF) {
+                    adapter.invokeStatic(definition.defobjType.type, DEF_NEG_CALL);
+                } else {
+                    if (settings.getNumericOverflow()) {
+                        adapter.math(GeneratorAdapter.NEG, type);
+                    } else {
+                        if (sort == Sort.INT) {
+                            adapter.invokeStatic(definition.mathType.type, NEGATEEXACT_INT);
+                        } else if (sort == Sort.LONG) {
+                            adapter.invokeStatic(definition.mathType.type, NEGATEEXACT_LONG);
+                        } else {
+                            throw new IllegalStateException(error("Illegal tree structure."));
+                        }
+                    }
+                }
+            } else if (operation != Operation.ADD) {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+
+            WriterUtility.writeBranch(adapter, tru, fals);
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
new file mode 100644
index 00000000000..b9cd0437992
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents an array length field load.
+ */
+public final class LArrayLength extends ALink {
+
+    final String value;
+
+    LArrayLength(final String location, final String value) {
+        super(location, -1);
+
+        this.value = value;
+    }
+
+    @Override
+    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        if ("length".equals(value)) {
+            if (!load) {
+                throw new IllegalArgumentException(error("Must read array field [length]."));
+            } else if (store) {
+                throw new IllegalArgumentException(error("Cannot write to read-only array field [length]."));
+            }
+
+            after = definition.intType;
+        } else {
+            throw new IllegalArgumentException(error("Illegal field access [" + value + "]."));
+        }
+
+        return this;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        // Do nothing.
+    }
+
+    @Override
+    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        adapter.arrayLength();
+    }
+
+    @Override
+    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        throw new IllegalStateException(error("Illegal tree structure."));
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
new file mode 100644
index 00000000000..ff7c7fabc16
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Represents an array load/store or defers to possible shortcuts.
+ */
+public final class LBrace extends ALink {
+
+    AExpression index;
+
+    public LBrace(final String location, final AExpression index) {
+        super(location, 2);
+
+        this.index = index;
+    }
+
+    @Override
+    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        if (before == null) {
+            throw new IllegalStateException(error("Illegal tree structure."));
+        }
+
+        final Sort sort = before.sort;
+
+        if (sort == Sort.ARRAY) {
+            index.expected = definition.intType;
+            index.analyze(settings, definition, variables);
+            index = index.cast(settings, definition, variables);
+
+            after = definition.getType(before.struct, before.dimensions - 1);
+
+            return this;
+        } else if (sort == Sort.DEF) {
+            return new LDefArray(location, index).copy(this).analyze(settings, definition, variables);
+        } else {
+            try {
+                before.clazz.asSubclass(Map.class);
+
+                return new LMapShortcut(location, index).copy(this).analyze(settings, definition, variables);
+            } catch (final ClassCastException exception) {
+                // Do nothing.
+            }
+
+            try {
+                before.clazz.asSubclass(List.class);
+
+                return new LListShortcut(location, index).copy(this).analyze(settings, definition, variables);
+            } catch (final ClassCastException exception) {
+                // Do nothing.
+            }
+        }
+
+        throw new IllegalArgumentException(error("Illegal array access on type [" + before.name + "]."));
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        index.write(settings, definition, adapter);
+    }
+
+    @Override
+    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        adapter.arrayLoad(after.type);
+    }
+
+    @Override
+    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        adapter.arrayStore(after.type);
+    }
+
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
new file mode 100644
index 00000000000..0d957a8b551
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Method;
+import org.elasticsearch.painless.Definition.Struct;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+import java.util.List;
+
+/**
+ * Represents a method call or deferes to a def call.
+ */
+public final class LCall extends ALink {
+
+    final String name;
+    final List<AExpression> arguments;
+
+    Method method = null;
+
+    public LCall(final String location, final String name, final List<AExpression> arguments) {
+        super(location, -1);
+
+        this.name = name;
+        this.arguments = arguments;
+    }
+
+    @Override
+    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        if (before == null) {
+            throw new IllegalStateException(error("Illegal tree structure."));
+        } else if (before.sort == Definition.Sort.ARRAY) {
+            throw new IllegalArgumentException(error("Illegal call [" + name + "] on array type."));
+        } else if (store) {
+            throw new IllegalArgumentException(error("Cannot assign a value to a call [" + name + "]."));
+        }
+
+        final Struct struct = before.struct;
+        method = statik ? struct.functions.get(name) : struct.methods.get(name);
+
+        if (method != null) {
+            final Definition.Type[] types = new Definition.Type[method.arguments.size()];
+            method.arguments.toArray(types);
+
+            if (method.arguments.size() != arguments.size()) {
+                throw new IllegalArgumentException(error("When calling [" + name + "] on type [" + struct.name + "]" +
+                    " expected [" + method.arguments.size() + "] arguments, but found [" + arguments.size() + "]."));
+            }
+
+            for (int argument = 0; argument < arguments.size(); ++argument) {
+                final AExpression expression = arguments.get(argument);
+
+                expression.expected = types[argument];
+                expression.analyze(settings, definition, variables);
+                arguments.set(argument, expression.cast(settings, definition, variables));
+            }
+
+            statement = true;
+            after = method.rtn;
+
+            return this;
+        } else if (before.sort == Definition.Sort.DEF) {
+            final ALink link = new LDefCall(location, name, arguments);
+            link.copy(this);
+
+            return link.analyze(settings, definition, variables);
+        }
+
+        throw new IllegalArgumentException(error("Unknown call [" + name + "] on type [" + struct.name + "]."));
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        // Do nothing.
+    }
+
+    @Override
+    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        for (final AExpression argument : arguments) {
+            argument.write(settings, definition, adapter);
+        }
+
+        if (java.lang.reflect.Modifier.isStatic(method.reflect.getModifiers())) {
+            adapter.invokeStatic(method.owner.type, method.method);
+        } else if (java.lang.reflect.Modifier.isInterface(method.owner.clazz.getModifiers())) {
+            adapter.invokeInterface(method.owner.type, method.method);
+        } else {
+            adapter.invokeVirtual(method.owner.type, method.method);
+        }
+
+        if (!method.rtn.clazz.equals(method.handle.type().returnType())) {
+            adapter.checkCast(method.rtn.type);
+        }
+    }
+
+    @Override
+    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        throw new IllegalStateException(error("Illegal tree structure."));
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
new file mode 100644
index 00000000000..52c23be993e
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Cast;
+import org.elasticsearch.painless.AnalyzerCaster;
+import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.WriterUtility;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents a cast made in a variable/method chain.
+ */
+public final class LCast extends ALink {
+
+    final String type;
+
+    Cast cast = null;
+
+    public LCast(final String location, final String type) {
+        super(location, -1);
+
+        this.type = type;
+    }
+
+    @Override
+    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        if (before == null) {
+            throw new IllegalStateException(error("Illegal tree structure."));
+        } else if (store) {
+            throw new IllegalArgumentException(error("Cannot assign a value to a cast."));
+        }
+
+        try {
+            after = definition.getType(type);
+        } catch (final IllegalArgumentException exception) {
+            throw new IllegalArgumentException(error("Not a type [" + type + "]."));
+        }
+
+        cast = AnalyzerCaster.getLegalCast(definition, location, before, after, true);
+
+        return cast != null ? this : null;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        WriterUtility.writeCast(adapter, cast);
+    }
+
+    @Override
+    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        // Do nothing.
+    }
+
+    @Override
+    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        throw new IllegalStateException(error("Illegal tree structure."));
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
new file mode 100644
index 00000000000..6e0b60a9227
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.DynamicCallSite;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
+import static org.elasticsearch.painless.WriterConstants.DEF_DYNAMIC_ARRAY_LOAD_DESC;
+import static org.elasticsearch.painless.WriterConstants.DEF_DYNAMIC_ARRAY_STORE_DESC;
+
+/**
+ * Represents an array load/store or shortcut on a def type.  (Internal only.)
+ */
+final class LDefArray extends ALink {
+
+    AExpression index;
+
+    LDefArray(final String location, final AExpression index) {
+        super(location, 0);
+
+        this.index = index;
+    }
+
+
+    @Override
+    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        index.expected = definition.objectType;
+        index.analyze(settings, definition, variables);
+        index = index.cast(settings, definition, variables);
+
+        after = definition.defType;
+
+        return this;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        index.write(settings, definition, adapter);
+    }
+
+    @Override
+    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        adapter.visitInvokeDynamicInsn(
+            "arrayLoad", DEF_DYNAMIC_ARRAY_LOAD_DESC, DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.ARRAY_LOAD });
+
+    }
+
+    @Override
+    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        adapter.visitInvokeDynamicInsn(
+            "arrayStore", DEF_DYNAMIC_ARRAY_STORE_DESC, DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.ARRAY_STORE });
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
new file mode 100644
index 00000000000..cda802e0138
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.DynamicCallSite;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+import java.util.List;
+
+import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
+
+/**
+ * Represents a method call made on a def type. (Internal only.)
+ */
+final class LDefCall extends ALink {
+
+    final String name;
+    final List<AExpression> arguments;
+
+    LDefCall(final String location, final String name, final List<AExpression> arguments) {
+        super(location, -1);
+
+        this.name = name;
+        this.arguments = arguments;
+    }
+
+    @Override
+    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        for (int argument = 0; argument < arguments.size(); ++argument) {
+            final AExpression expression = arguments.get(argument);
+
+            expression.analyze(settings, definition, variables);
+            expression.expected = expression.actual;
+            arguments.set(argument, expression.cast(settings, definition, variables));
+        }
+
+        statement = true;
+        after = definition.defType;
+
+        return this;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        // Do nothing.
+    }
+
+    @Override
+    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        final StringBuilder signature = new StringBuilder();
+
+        signature.append('(');
+        // first parameter is the receiver, we never know its type: always Object
+        signature.append(definition.defType.type.getDescriptor());
+
+        // TODO: remove our explicit conversions and feed more type information for return value,
+        // it can avoid some unnecessary boxing etc.
+        for (final AExpression argument : arguments) {
+            signature.append(argument.actual.type.getDescriptor());
+            argument.write(settings, definition, adapter);
+        }
+
+        signature.append(')');
+        // return value
+        signature.append(definition.defType.type.getDescriptor());
+
+        adapter.visitInvokeDynamicInsn(name, signature.toString(), DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.METHOD_CALL });
+    }
+
+    @Override
+    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        throw new IllegalStateException(error("Illegal tree structure."));
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
new file mode 100644
index 00000000000..f198e2999bd
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.DynamicCallSite;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
+import static org.elasticsearch.painless.WriterConstants.DEF_DYNAMIC_LOAD_FIELD_DESC;
+import static org.elasticsearch.painless.WriterConstants.DEF_DYNAMIC_STORE_FIELD_DESC;
+
+/**
+ * Represents a field load/store or shortcut on a def type.  (Internal only.)
+ */
+final class LDefField extends ALink {
+
+    final String value;
+
+    LDefField(final String location, final String value) {
+        super(location, 1);
+
+        this.value = value;
+    }
+
+
+    @Override
+    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        after = definition.defType;
+
+        return this;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        // Do nothing.
+    }
+
+    @Override
+    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        adapter.visitInvokeDynamicInsn(value, DEF_DYNAMIC_LOAD_FIELD_DESC, DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.LOAD });
+    }
+
+    @Override
+    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        adapter.visitInvokeDynamicInsn(value, DEF_DYNAMIC_STORE_FIELD_DESC, DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.STORE });
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
new file mode 100644
index 00000000000..577b4123152
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Field;
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Definition.Struct;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Represents a field load/store or defers to a possible shortcuts.
+ */
+public final class LField extends ALink {
+
+    final String value;
+
+    Field field;
+
+    public LField(final String location, final String value) {
+        super(location, 1);
+
+        this.value = value;
+    }
+
+    @Override
+    ALink analyze(CompilerSettings settings, Definition definition, Variables variables) {
+        if (before == null) {
+            throw new IllegalStateException(error("Illegal tree structure."));
+        }
+
+        final Sort sort = before.sort;
+
+        if (sort == Sort.ARRAY) {
+            return new LArrayLength(location, value).copy(this).analyze(settings, definition, variables);
+        } else if (sort == Sort.DEF) {
+            return new LDefField(location, value).copy(this).analyze(settings, definition, variables);
+        }
+
+        final Struct struct = before.struct;
+        field = statik ? struct.statics.get(value) : struct.members.get(value);
+
+        if (field != null) {
+            if (store && java.lang.reflect.Modifier.isFinal(field.reflect.getModifiers())) {
+                throw new IllegalArgumentException(error(
+                    "Cannot write to read-only field [" + value + "] for type [" + struct.name + "]."));
+            }
+
+            after = field.type;
+
+            return this;
+        } else {
+            final boolean shortcut =
+                struct.methods.containsKey("get" + Character.toUpperCase(value.charAt(0)) + value.substring(1)) ||
+                struct.methods.containsKey("set" + Character.toUpperCase(value.charAt(0)) + value.substring(1));
+
+            if (shortcut) {
+                return new LShortcut(location, value).copy(this).analyze(settings, definition, variables);
+            } else {
+                final EConstant index = new EConstant(location, value);
+                index.analyze(settings, definition, variables);
+
+                try {
+                    before.clazz.asSubclass(Map.class);
+
+                    return new LMapShortcut(location, index).copy(this).analyze(settings, definition, variables);
+                } catch (final ClassCastException exception) {
+                    // Do nothing.
+                }
+
+                try {
+                    before.clazz.asSubclass(List.class);
+
+                    return new LListShortcut(location, index).copy(this).analyze(settings, definition, variables);
+                } catch (final ClassCastException exception) {
+                    // Do nothing.
+                }
+            }
+        }
+
+        throw new IllegalArgumentException(error("Unknown field [" + value + "] for type [" + struct.name + "]."));
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        // Do nothing.
+    }
+
+    @Override
+    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        if (java.lang.reflect.Modifier.isStatic(field.reflect.getModifiers())) {
+            adapter.getStatic(field.owner.type, field.reflect.getName(), field.type.type);
+
+            if (!field.generic.clazz.equals(field.type.clazz)) {
+                adapter.checkCast(field.generic.type);
+            }
+        } else {
+            adapter.getField(field.owner.type, field.reflect.getName(), field.type.type);
+
+            if (!field.generic.clazz.equals(field.type.clazz)) {
+                adapter.checkCast(field.generic.type);
+            }
+        }
+    }
+
+    @Override
+    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        if (java.lang.reflect.Modifier.isStatic(field.reflect.getModifiers())) {
+            adapter.putStatic(field.owner.type, field.reflect.getName(), field.type.type);
+        } else {
+            adapter.putField(field.owner.type, field.reflect.getName(), field.type.type);
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
new file mode 100644
index 00000000000..05231f81b89
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Method;
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.WriterUtility;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents a list load/store shortcut.  (Internal only.)
+ */
+final class LListShortcut extends ALink {
+
+    AExpression index;
+    Method getter;
+    Method setter;
+
+    LListShortcut(final String location, final AExpression index) {
+        super(location, 2);
+
+        this.index = index;
+    }
+
+    @Override
+    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        getter = before.struct.methods.get("get");
+        setter = before.struct.methods.get("set");
+
+        if (getter != null && (getter.rtn.sort == Sort.VOID || getter.arguments.size() != 1 ||
+            getter.arguments.get(0).sort != Sort.INT)) {
+            throw new IllegalArgumentException(error("Illegal list get shortcut for type [" + before.name + "]."));
+        }
+
+        if (setter != null && (setter.arguments.size() != 2 || setter.arguments.get(0).sort != Sort.INT)) {
+            throw new IllegalArgumentException(error("Illegal list set shortcut for type [" + before.name + "]."));
+        }
+
+        if (getter != null && setter != null && (!getter.arguments.get(0).equals(setter.arguments.get(0))
+            || !getter.rtn.equals(setter.arguments.get(1)))) {
+            throw new IllegalArgumentException(error("Shortcut argument types must match."));
+        }
+
+        if ((load || store) && (!load || getter != null) && (!store || setter != null)) {
+            index.expected = definition.intType;
+            index.analyze(settings, definition, variables);
+            index = index.cast(settings, definition, variables);
+
+            after = setter != null ? setter.arguments.get(1) : getter.rtn;
+        } else {
+            throw new IllegalArgumentException(error("Illegal list shortcut for type [" + before.name + "]."));
+        }
+
+        return this;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        index.write(settings, definition, adapter);
+    }
+
+    @Override
+    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
+            adapter.invokeInterface(getter.owner.type, getter.method);
+        } else {
+            adapter.invokeVirtual(getter.owner.type, getter.method);
+        }
+
+        if (!getter.rtn.clazz.equals(getter.handle.type().returnType())) {
+            adapter.checkCast(getter.rtn.type);
+        }
+    }
+
+    @Override
+    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
+            adapter.invokeInterface(setter.owner.type, setter.method);
+        } else {
+            adapter.invokeVirtual(setter.owner.type, setter.method);
+        }
+
+        WriterUtility.writePop(adapter, setter.rtn.sort.size);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
new file mode 100644
index 00000000000..871950ef592
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Method;
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.WriterUtility;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents a map load/store shortcut. (Internal only.)
+ */
+final class LMapShortcut extends ALink {
+
+    AExpression index;
+    Method getter;
+    Method setter;
+
+    LMapShortcut(final String location, final AExpression index) {
+        super(location, 2);
+
+        this.index = index;
+    }
+
+    @Override
+    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        getter = before.struct.methods.get("get");
+        setter = before.struct.methods.get("put");
+
+        if (getter != null && (getter.rtn.sort == Sort.VOID || getter.arguments.size() != 1)) {
+            throw new IllegalArgumentException(error("Illegal map get shortcut for type [" + before.name + "]."));
+        }
+
+        if (setter != null && setter.arguments.size() != 2) {
+            throw new IllegalArgumentException(error("Illegal map set shortcut for type [" + before.name + "]."));
+        }
+
+        if (getter != null && setter != null &&
+            (!getter.arguments.get(0).equals(setter.arguments.get(0)) || !getter.rtn.equals(setter.arguments.get(1)))) {
+            throw new IllegalArgumentException(error("Shortcut argument types must match."));
+        }
+
+        if ((load || store) && (!load || getter != null) && (!store || setter != null)) {
+            index.expected = setter != null ? setter.arguments.get(0) : getter.arguments.get(0);
+            index.analyze(settings, definition, variables);
+            index = index.cast(settings, definition, variables);
+
+            after = setter != null ? setter.arguments.get(1) : getter.rtn;
+        } else {
+            throw new IllegalArgumentException(error("Illegal map shortcut for type [" + before.name + "]."));
+        }
+
+        return this;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        index.write(settings, definition, adapter);
+    }
+
+    @Override
+    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
+            adapter.invokeInterface(getter.owner.type, getter.method);
+        } else {
+            adapter.invokeVirtual(getter.owner.type, getter.method);
+        }
+
+        if (!getter.rtn.clazz.equals(getter.handle.type().returnType())) {
+            adapter.checkCast(getter.rtn.type);
+        }
+    }
+
+    @Override
+    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
+            adapter.invokeInterface(setter.owner.type, setter.method);
+        } else {
+            adapter.invokeVirtual(setter.owner.type, setter.method);
+        }
+
+        WriterUtility.writePop(adapter, setter.rtn.sort.size);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
new file mode 100644
index 00000000000..beca6057022
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+import java.util.List;
+
+/**
+ * Represents an array instantiation.
+ */
+public final class LNewArray extends ALink {
+
+    final String type;
+    final List<AExpression> arguments;
+
+    public LNewArray(final String location, final String type, final List<AExpression> arguments) {
+        super(location, -1);
+
+        this.type = type;
+        this.arguments = arguments;
+    }
+
+    @Override
+    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        if (before != null) {
+            throw new IllegalStateException(error("Illegal tree structure."));
+        } else if (store) {
+            throw new IllegalArgumentException(error("Cannot assign a value to a new array."));
+        } else if (!load) {
+            throw new IllegalArgumentException(error("A newly created array must be assigned."));
+        }
+
+        final Type type;
+
+        try {
+            type = definition.getType(this.type);
+        } catch (final IllegalArgumentException exception) {
+            throw new IllegalArgumentException(error("Not a type [" + this.type + "]."));
+        }
+
+        for (int argument = 0; argument < arguments.size(); ++argument) {
+            final AExpression expression = arguments.get(argument);
+
+            expression.expected = definition.intType;
+            expression.analyze(settings, definition, variables);
+            arguments.set(argument, expression.cast(settings, definition, variables));
+        }
+
+        after = definition.getType(type.struct, arguments.size());
+
+        return this;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        // Do nothing.
+    }
+
+    @Override
+    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        for (final AExpression argument : arguments) {
+            argument.write(settings, definition, adapter);
+        }
+
+        if (arguments.size() > 1) {
+            adapter.visitMultiANewArrayInsn(after.type.getDescriptor(), after.type.getDimensions());
+        } else {
+            adapter.newArray(definition.getType(after.struct, 0).type);
+        }
+    }
+
+    @Override
+    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        throw new IllegalStateException(error("Illegal tree structure."));
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
new file mode 100644
index 00000000000..c460571f208
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Constructor;
+import org.elasticsearch.painless.Definition.Struct;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+import java.util.List;
+
+/**
+ * Respresents and object instantiation.
+ */
+public final class LNewObj extends ALink {
+
+    final String type;
+    final List<AExpression> arguments;
+
+    Constructor constructor;
+
+    public LNewObj(final String location, final String type, final List<AExpression> arguments) {
+        super(location, -1);
+
+        this.type = type;
+        this.arguments = arguments;
+    }
+
+    @Override
+    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        if (before != null) {
+            throw new IllegalStateException(error("Illegal tree structure"));
+        } else if (store) {
+            throw new IllegalArgumentException(error("Cannot assign a value to a new call."));
+        }
+
+        final Type type;
+
+        try {
+            type = definition.getType(this.type);
+        } catch (final IllegalArgumentException exception) {
+            throw new IllegalArgumentException(error("Not a type [" + this.type + "]."));
+        }
+
+        final Struct struct = type.struct;
+        constructor = struct.constructors.get("new");
+
+        if (constructor != null) {
+            final Type[] types = new Type[constructor.arguments.size()];
+            constructor.arguments.toArray(types);
+
+            if (constructor.arguments.size() != arguments.size()) {
+                throw new IllegalArgumentException(error("When calling constructor on type [" + struct.name + "]" +
+                    " expected [" + constructor.arguments.size() + "] arguments, but found [" + arguments.size() + "]."));
+            }
+
+            for (int argument = 0; argument < arguments.size(); ++argument) {
+                final AExpression expression = arguments.get(argument);
+
+                expression.expected = types[argument];
+                expression.analyze(settings, definition, variables);
+                arguments.set(argument, expression.cast(settings, definition, variables));
+            }
+
+            statement = true;
+            after = type;
+        } else {
+            throw new IllegalArgumentException(error("Unknown new call on type [" + struct.name + "]."));
+        }
+
+        return this;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        // Do nothing.
+    }
+
+    @Override
+    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        adapter.newInstance(after.type);
+
+        if (load) {
+            adapter.dup();
+        }
+
+        for (final AExpression argument : arguments) {
+            argument.write(settings, definition, adapter);
+        }
+
+        adapter.invokeConstructor(constructor.owner.type, constructor.method);
+    }
+
+    @Override
+    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        throw new IllegalStateException(error("Illegal tree structure."));
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
new file mode 100644
index 00000000000..df190896940
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Method;
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Definition.Struct;
+import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.WriterUtility;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents a field load/store shortcut.  (Internal only.)
+ */
+final class LShortcut extends ALink {
+
+    final String value;
+
+    Method getter = null;
+    Method setter = null;
+
+    LShortcut(final String location, final String value) {
+        super(location, 1);
+
+        this.value = value;
+    }
+
+    @Override
+    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        final Struct struct = before.struct;
+
+        getter = struct.methods.get("get" + Character.toUpperCase(value.charAt(0)) + value.substring(1));
+        setter = struct.methods.get("set" + Character.toUpperCase(value.charAt(0)) + value.substring(1));
+
+        if (getter != null && (getter.rtn.sort == Sort.VOID || !getter.arguments.isEmpty())) {
+            throw new IllegalArgumentException(error(
+                "Illegal get shortcut on field [" + value + "] for type [" + struct.name + "]."));
+        }
+
+        if (setter != null && (setter.rtn.sort != Sort.VOID || setter.arguments.size() != 1)) {
+            throw new IllegalArgumentException(error(
+                "Illegal set shortcut on field [" + value + "] for type [" + struct.name + "]."));
+        }
+
+        if (getter != null && setter != null && setter.arguments.get(0) != getter.rtn) {
+            throw new IllegalArgumentException(error("Shortcut argument types must match."));
+        }
+
+        if ((getter != null || setter != null) && (!load || getter != null) && (!store || setter != null)) {
+            after = setter != null ? setter.arguments.get(0) : getter.rtn;
+        } else {
+            throw new IllegalArgumentException(error("Illegal shortcut on field [" + value + "] for type [" + struct.name + "]."));
+        }
+
+        return this;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        // Do nothing.
+    }
+
+    @Override
+    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
+            adapter.invokeInterface(getter.owner.type, getter.method);
+        } else {
+            adapter.invokeVirtual(getter.owner.type, getter.method);
+        }
+
+        if (!getter.rtn.clazz.equals(getter.handle.type().returnType())) {
+            adapter.checkCast(getter.rtn.type);
+        }
+    }
+
+    @Override
+    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
+            adapter.invokeInterface(setter.owner.type, setter.method);
+        } else {
+            adapter.invokeVirtual(setter.owner.type, setter.method);
+        }
+
+        WriterUtility.writePop(adapter, setter.rtn.sort.size);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
new file mode 100644
index 00000000000..70c554b327a
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents a string constant.
+ */
+public final class LString extends ALink {
+
+    public LString(final String location, final String string) {
+        super(location, -1);
+
+        this.string = string;
+    }
+
+    @Override
+    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        if (before != null) {
+            throw new IllegalStateException("Illegal tree structure.");
+        } else if (store) {
+            throw new IllegalArgumentException(error("Cannot write to read-only String constant [" + string + "]."));
+        } else if (!load) {
+            throw new IllegalArgumentException(error("Must read String constant [" + string + "]."));
+        }
+
+        after = definition.stringType;
+
+        return this;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        // Do nothing.
+    }
+
+    @Override
+    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        adapter.push(string);
+    }
+
+    @Override
+    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        throw new IllegalStateException(error("Illegal tree structure."));
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
new file mode 100644
index 00000000000..73ead63b961
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Variables.Variable;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents a variable load/store.
+ */
+public final class LVariable extends ALink {
+
+    final String name;
+
+    int slot;
+
+    public LVariable(final String location, final String name) {
+        super(location, 0);
+
+        this.name = name;
+    }
+
+    @Override
+    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        if (before != null) {
+            throw new IllegalStateException(error("Illegal tree structure."));
+        }
+
+        Type type = null;
+
+        try {
+            type = definition.getType(name);
+        } catch (final IllegalArgumentException exception) {
+            // Do nothing.
+        }
+
+        if (type != null) {
+            statik = true;
+            after = type;
+        } else {
+            final Variable variable = variables.getVariable(location, name);
+
+            if (store && variable.readonly) {
+                throw new IllegalArgumentException(error("Variable [" + variable.name + "] is read-only."));
+            }
+
+            slot = variable.slot;
+            after = variable.type;
+        }
+
+        return this;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        // Do nothing.
+    }
+
+    @Override
+    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        adapter.visitVarInsn(after.type.getOpcode(Opcodes.ILOAD), slot);
+    }
+
+    @Override
+    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        adapter.visitVarInsn(after.type.getOpcode(Opcodes.ISTORE), slot);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
new file mode 100644
index 00000000000..f24a603ad61
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Represents a set of statements as a branch of control-flow.
+ */
+public final class SBlock extends AStatement {
+
+    final List<AStatement> statements;
+
+    public SBlock(final String location, final List<AStatement> statements) {
+        super(location);
+
+        this.statements = Collections.unmodifiableList(statements);
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        final AStatement last = statements.get(statements.size() - 1);
+
+        for (final AStatement statement : statements) {
+            if (allEscape) {
+                throw new IllegalArgumentException(error("Unreachable statement."));
+            }
+
+            statement.inLoop = inLoop;
+            statement.lastSource = lastSource && statement == last;
+            statement.lastLoop = (beginLoop || lastLoop) && statement == last;
+
+            statement.analyze(settings, definition, variables);
+
+            methodEscape = statement.methodEscape;
+            loopEscape = statement.loopEscape;
+            allEscape = statement.allEscape;
+            anyContinue |= statement.anyContinue;
+            anyBreak |= statement.anyBreak;
+            statementCount += statement.statementCount;
+        }
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        for (final AStatement statement : statements) {
+            statement.continu = continu;
+            statement.brake = brake;
+            statement.write(settings, definition, adapter);
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
new file mode 100644
index 00000000000..23fd8d0603c
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents a break statement.
+ */
+public final class SBreak extends AStatement {
+
+    public SBreak(final String location) {
+        super(location);
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        if (!inLoop) {
+            throw new IllegalArgumentException(error("Break statement outside of a loop."));
+        }
+
+        loopEscape = true;
+        allEscape = true;
+        anyBreak = true;
+        statementCount = 1;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        adapter.goTo(brake);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
new file mode 100644
index 00000000000..3ed7fa8905a
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents a continue statement.
+ */
+public final class SContinue extends AStatement {
+
+    public SContinue(final String location) {
+        super(location);
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        if (!inLoop) {
+            throw new IllegalArgumentException(error("Continue statement outside of a loop."));
+        }
+
+        if (lastLoop) {
+            throw new IllegalArgumentException(error("Extraneous continue statement."));
+        }
+
+        allEscape = true;
+        anyContinue = true;
+        statementCount = 1;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        adapter.goTo(continu);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
new file mode 100644
index 00000000000..875d4857c45
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Represents a series of declarations.
+ */
+public final class SDeclBlock extends AStatement {
+
+    final List<SDeclaration> declarations;
+
+    public SDeclBlock(final String location, final List<SDeclaration> declarations) {
+        super(location);
+
+        this.declarations = Collections.unmodifiableList(declarations);
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        for (final SDeclaration declaration : declarations) {
+            declaration.analyze(settings, definition, variables);
+        }
+
+        statementCount = declarations.size();
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        for (final SDeclaration declaration : declarations) {
+            declaration.write(settings, definition, adapter);
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
new file mode 100644
index 00000000000..cdcdce7acc8
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Variables.Variable;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents a single variable declaration.
+ */
+public final class SDeclaration extends AStatement {
+
+    final String type;
+    final String name;
+    AExpression expression;
+
+    Variable variable;
+
+    public SDeclaration(final String location, final String type, final String name, final AExpression expression) {
+        super(location);
+
+        this.type = type;
+        this.name = name;
+        this.expression = expression;
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        variable = variables.addVariable(location, type, name, false, false);
+
+        if (expression != null) {
+            expression.expected = variable.type;
+            expression.analyze(settings, definition, variables);
+            expression = expression.cast(settings, definition, variables);
+        }
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        final org.objectweb.asm.Type type = variable.type.type;
+        final Sort sort = variable.type.sort;
+
+        final boolean initialize = expression == null;
+
+        if (!initialize) {
+            expression.write(settings, definition, adapter);
+        }
+
+        switch (sort) {
+            case VOID:   throw new IllegalStateException(error("Illegal tree structure."));
+            case BOOL:
+            case BYTE:
+            case SHORT:
+            case CHAR:
+            case INT:    if (initialize) adapter.push(0);    break;
+            case LONG:   if (initialize) adapter.push(0L);   break;
+            case FLOAT:  if (initialize) adapter.push(0.0F); break;
+            case DOUBLE: if (initialize) adapter.push(0.0);  break;
+            default:     if (initialize) adapter.visitInsn(Opcodes.ACONST_NULL);
+        }
+
+        adapter.visitVarInsn(type.getOpcode(Opcodes.ISTORE), variable.slot);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
new file mode 100644
index 00000000000..41b2acd5fcb
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.WriterUtility;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents a do-while loop.
+ */
+public final class SDo extends AStatement {
+
+    final AStatement block;
+    AExpression condition;
+
+    public SDo(final String location, final AStatement block, final AExpression condition) {
+        super(location);
+
+        this.condition = condition;
+        this.block = block;
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        variables.incrementScope();
+
+        block.beginLoop = true;
+        block.inLoop = true;
+
+        block.analyze(settings, definition, variables);
+
+        if (block.loopEscape && !block.anyContinue) {
+            throw new IllegalArgumentException(error("Extraneous do while loop."));
+        }
+
+        condition.expected = definition.booleanType;
+        condition.analyze(settings, definition, variables);
+        condition = condition.cast(settings, definition, variables);
+
+        if (condition.constant != null) {
+            final boolean continuous = (boolean)condition.constant;
+
+            if (!continuous) {
+                throw new IllegalArgumentException(error("Extraneous do while loop."));
+            }
+
+            if (!block.anyBreak) {
+                methodEscape = true;
+                allEscape = true;
+            }
+        }
+
+        statementCount = 1;
+
+        if (settings.getMaxLoopCounter() > 0) {
+            loopCounterSlot = variables.getVariable(location, "#loop").slot;
+        }
+
+        variables.decrementScope();
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        final Label start = new Label();
+        final Label begin = new Label();
+        final Label end = new Label();
+
+        adapter.mark(start);
+
+        block.continu = begin;
+        block.brake = end;
+        block.write(settings, definition, adapter);
+
+        adapter.mark(begin);
+
+        condition.fals = end;
+        condition.write(settings, definition, adapter);
+
+        WriterUtility.writeLoopCounter(adapter, loopCounterSlot, Math.max(1, block.statementCount));
+
+        adapter.goTo(start);
+        adapter.mark(end);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
new file mode 100644
index 00000000000..b9fd106e590
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.WriterUtility;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents the top-level node for an expression as a statement.
+ */
+public final class SExpression extends AStatement {
+
+    AExpression expression;
+
+    public SExpression(final String location, final AExpression expression) {
+        super(location);
+
+        this.expression = expression;
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        expression.read = lastSource;
+        expression.analyze(settings, definition, variables);
+
+        if (!lastSource && !expression.statement) {
+            throw new IllegalArgumentException(error("Not a statement."));
+        }
+
+        final boolean rtn = lastSource && expression.actual.sort != Sort.VOID;
+
+        expression.expected = rtn ? definition.objectType : expression.actual;
+        expression = expression.cast(settings, definition, variables);
+
+        methodEscape = rtn;
+        loopEscape = rtn;
+        allEscape = rtn;
+        statementCount = 1;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        expression.write(settings, definition, adapter);
+
+        if (methodEscape) {
+            adapter.returnValue();
+        } else {
+            WriterUtility.writePop(adapter, expression.expected.sort.size);
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
new file mode 100644
index 00000000000..0c26db7c63b
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.WriterUtility;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents a for loop.
+ */
+public final class SFor extends AStatement {
+
+    ANode initializer;
+    AExpression condition;
+    AExpression afterthought;
+    final AStatement block;
+
+    public SFor(final String location,
+                final ANode initializer, final AExpression condition, final AExpression afterthought, final AStatement block) {
+        super(location);
+
+        this.initializer = initializer;
+        this.condition = condition;
+        this.afterthought = afterthought;
+        this.block = block;
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        variables.incrementScope();
+
+        boolean continuous = false;
+
+        if (initializer != null) {
+            if (initializer instanceof SDeclBlock) {
+                ((SDeclBlock)initializer).analyze(settings, definition, variables);
+            } else if (initializer instanceof AExpression) {
+                final AExpression initializer = (AExpression)this.initializer;
+
+                initializer.read = false;
+                initializer.analyze(settings, definition, variables);
+
+                if (!initializer.statement) {
+                    throw new IllegalArgumentException(initializer.error("Not a statement."));
+                }
+            } else {
+                throw new IllegalStateException(error("Illegal tree structure."));
+            }
+        }
+
+        if (condition != null) {
+
+            condition.expected = definition.booleanType;
+            condition.analyze(settings, definition, variables);
+            condition = condition.cast(settings, definition, variables);
+
+            if (condition.constant != null) {
+                continuous = (boolean)condition.constant;
+
+                if (!continuous) {
+                    throw new IllegalArgumentException(error("Extraneous for loop."));
+                }
+
+                if (block == null) {
+                    throw new IllegalArgumentException(error("For loop has no escape."));
+                }
+            }
+        } else {
+            continuous = true;
+        }
+
+        if (afterthought != null) {
+            afterthought.read = false;
+            afterthought.analyze(settings, definition, variables);
+
+            if (!afterthought.statement) {
+                throw new IllegalArgumentException(afterthought.error("Not a statement."));
+            }
+        }
+
+        int count = 1;
+
+        if (block != null) {
+            block.beginLoop = true;
+            block.inLoop = true;
+
+            block.analyze(settings, definition, variables);
+
+            if (block.loopEscape && !block.anyContinue) {
+                throw new IllegalArgumentException(error("Extraneous for loop."));
+            }
+
+            if (continuous && !block.anyBreak) {
+                methodEscape = true;
+                allEscape = true;
+            }
+
+            block.statementCount = Math.max(count, block.statementCount);
+        }
+
+        statementCount = 1;
+
+        if (settings.getMaxLoopCounter() > 0) {
+            loopCounterSlot = variables.getVariable(location, "#loop").slot;
+        }
+
+        variables.decrementScope();
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        final Label start = new Label();
+        final Label begin = afterthought == null ? start : new Label();
+        final Label end = new Label();
+
+        if (initializer instanceof SDeclBlock) {
+            ((SDeclBlock)initializer).write(settings, definition, adapter);
+        } else if (initializer instanceof AExpression) {
+            AExpression initializer = (AExpression)this.initializer;
+
+            initializer.write(settings, definition, adapter);
+            WriterUtility.writePop(adapter, initializer.expected.sort.size);
+        }
+
+        adapter.mark(start);
+
+        if (condition != null) {
+            condition.fals = end;
+            condition.write(settings, definition, adapter);
+        }
+
+        boolean allEscape = false;
+
+        if (block != null) {
+            allEscape = block.allEscape;
+
+            int statementCount = Math.max(1, block.statementCount);
+
+            if (afterthought != null) {
+                ++statementCount;
+            }
+
+            WriterUtility.writeLoopCounter(adapter, loopCounterSlot, statementCount);
+            block.write(settings, definition, adapter);
+        } else {
+            WriterUtility.writeLoopCounter(adapter, loopCounterSlot, 1);
+        }
+
+        if (afterthought != null) {
+            adapter.mark(begin);
+            afterthought.write(settings, definition, adapter);
+        }
+
+        if (afterthought != null || !allEscape) {
+            adapter.goTo(start);
+        }
+
+        adapter.mark(end);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
new file mode 100644
index 00000000000..747730b1c53
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents an if/else block.
+ */
+public final class SIfElse extends AStatement {
+
+    AExpression condition;
+    final AStatement ifblock;
+    final AStatement elseblock;
+
+    public SIfElse(final String location, final AExpression condition, final AStatement ifblock, final AStatement elseblock) {
+        super(location);
+
+        this.condition = condition;
+        this.ifblock = ifblock;
+        this.elseblock = elseblock;
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        condition.expected = definition.booleanType;
+        condition.analyze(settings, definition, variables);
+        condition = condition.cast(settings, definition, variables);
+
+        if (condition.constant != null) {
+            throw new IllegalArgumentException(error("Extraneous if statement."));
+        }
+
+        ifblock.lastSource = lastSource;
+        ifblock.inLoop = inLoop;
+        ifblock.lastLoop = lastLoop;
+
+        variables.incrementScope();
+        ifblock.analyze(settings, definition, variables);
+        variables.decrementScope();
+
+        anyContinue = ifblock.anyContinue;
+        anyBreak = ifblock.anyBreak;
+        statementCount = ifblock.statementCount;
+
+        if (elseblock != null) {
+            elseblock.lastSource = lastSource;
+            elseblock.inLoop = inLoop;
+            elseblock.lastLoop = lastLoop;
+
+            variables.incrementScope();
+            elseblock.analyze(settings, definition, variables);
+            variables.decrementScope();
+
+            methodEscape = ifblock.methodEscape && elseblock.methodEscape;
+            loopEscape = ifblock.loopEscape && elseblock.loopEscape;
+            allEscape = ifblock.allEscape && elseblock.allEscape;
+            anyContinue |= elseblock.anyContinue;
+            anyBreak |= elseblock.anyBreak;
+            statementCount = Math.max(ifblock.statementCount, elseblock.statementCount);
+        }
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        final Label end = new Label();
+        final Label fals = elseblock != null ? new Label() : end;
+
+        condition.fals = fals;
+        condition.write(settings, definition, adapter);
+
+        ifblock.continu = continu;
+        ifblock.brake = brake;
+        ifblock.write(settings, definition, adapter);
+
+        if (elseblock != null) {
+            if (!ifblock.allEscape) {
+                adapter.goTo(end);
+            }
+
+            adapter.mark(fals);
+
+            elseblock.continu = continu;
+            elseblock.brake = brake;
+            elseblock.write(settings, definition, adapter);
+        }
+
+        adapter.mark(end);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
new file mode 100644
index 00000000000..c59156b1c4a
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents a return statement.
+ */
+public final class SReturn extends AStatement {
+
+    AExpression expression;
+
+    public SReturn(final String location, final AExpression expression) {
+        super(location);
+
+        this.expression = expression;
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        expression.expected = definition.objectType;
+        expression.analyze(settings, definition, variables);
+        expression = expression.cast(settings, definition, variables);
+
+        methodEscape = true;
+        loopEscape = true;
+        allEscape = true;
+
+        statementCount = 1;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        expression.write(settings, definition, adapter);
+        adapter.returnValue();
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
new file mode 100644
index 00000000000..c00fc0c7fc4
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * The root of all Painless trees.  Contains a series of statements.
+ */
+public final class SSource extends AStatement {
+
+    final List<AStatement> statements;
+
+    public SSource(final String location, final List<AStatement> statements) {
+        super(location);
+
+        this.statements = Collections.unmodifiableList(statements);
+    }
+
+    @Override
+    public void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        variables.incrementScope();
+
+        final AStatement last = statements.get(statements.size() - 1);
+
+        for (final AStatement statement : statements) {
+            if (allEscape) {
+                throw new IllegalArgumentException(error("Unreachable statement."));
+            }
+
+            statement.lastSource = statement == last;
+            statement.analyze(settings, definition, variables);
+
+            methodEscape = statement.methodEscape;
+            allEscape = statement.allEscape;
+        }
+
+        variables.decrementScope();
+    }
+
+    @Override
+    public void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        for (final AStatement statement : statements) {
+            statement.write(settings, definition, adapter);
+        }
+
+        if (!methodEscape) {
+            adapter.visitInsn(Opcodes.ACONST_NULL);
+            adapter.returnValue();
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
new file mode 100644
index 00000000000..613b75e6332
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents a throw statement.
+ */
+public final class SThrow extends AStatement {
+
+    AExpression expression;
+
+    public SThrow(final String location, final AExpression expression) {
+        super(location);
+
+        this.expression = expression;
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        expression.expected = definition.exceptionType;
+        expression.analyze(settings, definition, variables);
+        expression = expression.cast(settings, definition, variables);
+
+        methodEscape = true;
+        loopEscape = true;
+        allEscape = true;
+        statementCount = 1;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        expression.write(settings, definition, adapter);
+        adapter.throwException();
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java
new file mode 100644
index 00000000000..8608344ecd1
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Variables.Variable;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents a catch block as part of a try-catch block.
+ */
+public final class STrap extends AStatement {
+
+    final String type;
+    final String name;
+    final AStatement block;
+
+    Variable variable;
+
+    Label begin;
+    Label end;
+    Label exception;
+
+    public STrap(final String location, final String type, final String name, final AStatement block) {
+        super(location);
+
+        this.type = type;
+        this.name = name;
+        this.block = block;
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        variable = variables.addVariable(location, type, name, true, false);
+
+        try {
+            variable.type.clazz.asSubclass(Exception.class);
+        } catch (final ClassCastException cce) {
+            throw new ClassCastException(error("Not an exception type [" + variable.type.name + "]."));
+        }
+
+        if (block != null) {
+            block.lastSource = lastSource;
+            block.inLoop = inLoop;
+            block.lastLoop = lastLoop;
+
+            block.analyze(settings, definition, variables);
+
+            methodEscape = block.methodEscape;
+            loopEscape = block.loopEscape;
+            allEscape = block.allEscape;
+            anyContinue = block.anyContinue;
+            anyBreak = block.anyBreak;
+            statementCount = block.statementCount;
+        }
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        final Label jump = new Label();
+
+        adapter.mark(jump);
+        adapter.visitVarInsn(variable.type.type.getOpcode(Opcodes.ISTORE), variable.slot);
+
+        if (block != null) {
+            block.continu = continu;
+            block.brake = brake;
+            block.write(settings, definition, adapter);
+        }
+
+        adapter.visitTryCatchBlock(begin, end, jump, variable.type.type.getInternalName());
+
+        if (exception != null && !block.allEscape) {
+            adapter.goTo(exception);
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
new file mode 100644
index 00000000000..39aa68a0686
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Represents the try block as part of a try-catch block.
+ */
+public final class STry extends AStatement {
+
+    final AStatement block;
+    final List<STrap> traps;
+
+    public STry(final String location, final AStatement block, final List<STrap> traps) {
+        super(location);
+
+        this.block = block;
+        this.traps = Collections.unmodifiableList(traps);
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        block.lastSource = lastSource;
+        block.inLoop = inLoop;
+        block.lastLoop = lastLoop;
+
+        variables.incrementScope();
+        block.analyze(settings, definition, variables);
+        variables.decrementScope();
+
+        methodEscape = block.methodEscape;
+        loopEscape = block.loopEscape;
+        allEscape = block.allEscape;
+        anyContinue = block.anyContinue;
+        anyBreak = block.anyBreak;
+
+        int statementCount = 0;
+
+        for (final STrap trap : traps) {
+            trap.lastSource = lastSource;
+            trap.inLoop = inLoop;
+            trap.lastLoop = lastLoop;
+
+            variables.incrementScope();
+            trap.analyze(settings, definition, variables);
+            variables.decrementScope();
+
+            methodEscape &= trap.methodEscape;
+            loopEscape &= trap.loopEscape;
+            allEscape &= trap.allEscape;
+            anyContinue |= trap.anyContinue;
+            anyBreak |= trap.anyBreak;
+
+            statementCount = Math.max(statementCount, trap.statementCount);
+        }
+
+        this.statementCount = block.statementCount + statementCount;
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        final Label begin = new Label();
+        final Label end = new Label();
+        final Label exception = new Label();
+
+        adapter.mark(begin);
+
+        block.continu = continu;
+        block.brake = brake;
+        block.write(settings, definition, adapter);
+
+        if (!block.allEscape) {
+            adapter.goTo(exception);
+        }
+
+        adapter.mark(end);
+
+        for (final STrap trap : traps) {
+            trap.begin = begin;
+            trap.end = end;
+            trap.exception = traps.size() > 1 ? exception : null;
+            trap.write(settings, definition, adapter);
+        }
+
+        if (!block.allEscape || traps.size() > 1) {
+            adapter.mark(exception);
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
new file mode 100644
index 00000000000..493ddfe0e13
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.WriterUtility;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
+/**
+ * Represents a while loop.
+ */
+public final class SWhile extends AStatement {
+
+    AExpression condition;
+    final AStatement block;
+
+    public SWhile(final String location, final AExpression condition, final AStatement block) {
+        super(location);
+
+        this.condition = condition;
+        this.block = block;
+    }
+
+    @Override
+    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+        variables.incrementScope();
+
+        condition.expected = definition.booleanType;
+        condition.analyze(settings, definition, variables);
+        condition = condition.cast(settings, definition, variables);
+
+        boolean continuous = false;
+
+        if (condition.constant != null) {
+            continuous = (boolean)condition.constant;
+
+            if (!continuous) {
+                throw new IllegalArgumentException(error("Extraneous while loop."));
+            }
+
+            if (block == null) {
+                throw new IllegalArgumentException(error("While loop has no escape."));
+            }
+        }
+
+        int count = 1;
+
+        if (block != null) {
+            block.beginLoop = true;
+            block.inLoop = true;
+
+            block.analyze(settings, definition, variables);
+
+            if (block.loopEscape && !block.anyContinue) {
+                throw new IllegalArgumentException(error("Extranous while loop."));
+            }
+
+            if (continuous && !block.anyBreak) {
+                methodEscape = true;
+                allEscape = true;
+            }
+
+            block.statementCount = Math.max(count, block.statementCount);
+        }
+
+        statementCount = 1;
+
+        if (settings.getMaxLoopCounter() > 0) {
+            loopCounterSlot = variables.getVariable(location, "#loop").slot;
+        }
+
+        variables.decrementScope();
+    }
+
+    @Override
+    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        final Label begin = new Label();
+        final Label end = new Label();
+
+        adapter.mark(begin);
+
+        condition.fals = end;
+        condition.write(settings, definition, adapter);
+
+        if (block != null) {
+            WriterUtility.writeLoopCounter(adapter, loopCounterSlot, Math.max(1, block.statementCount));
+
+            block.continu = begin;
+            block.brake = end;
+            block.write(settings, definition, adapter);
+        } else {
+            WriterUtility.writeLoopCounter(adapter, loopCounterSlot, 1);
+        }
+
+        if (block == null || !block.allEscape) {
+            adapter.goTo(begin);
+        }
+
+        adapter.mark(end);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
new file mode 100644
index 00000000000..0a53e909bf8
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * A painless tree is composed of the node classes found in this package.
+ * <p>
+ * The following are the types of nodes:
+ * A* (abstract) - These are the abstract nodes that are the superclasses for the other types.
+ * S* (statement) - These are nodes that represent a statement in Painless.  These are the highest level nodes.
+ * E* (expression) - These are nodess that represent an expression in Painless.  These are the middle level nodes.
+ * L* (link) - These are nodes that respresent a piece of a variable/method chain.  The are the lowest level nodes.
+ * <p>
+ * The following is a brief description of each node:
+ * {@link org.elasticsearch.painless.node.AExpression} - The superclass for all E* (expression) nodes.
+ * {@link org.elasticsearch.painless.node.ALink} - The superclass for all L* (link) nodes.
+ * {@link org.elasticsearch.painless.node.ANode} - The superclass for all other nodes.
+ * {@link org.elasticsearch.painless.node.AStatement} - The superclass for all S* (statement) nodes.
+ * {@link org.elasticsearch.painless.node.EBinary} - Represents a binary math expression.
+ * {@link org.elasticsearch.painless.node.EBool} - Represents a boolean expression.
+ * {@link org.elasticsearch.painless.node.EBoolean} - Represents a boolean constant.
+ * {@link org.elasticsearch.painless.node.ECast} - Represents an implicit cast in most cases.  (Internal only.)
+ * {@link org.elasticsearch.painless.node.EChain} - Represents the entirety of a variable/method chain for read/write operations.
+ * {@link org.elasticsearch.painless.node.EComp} - Represents a comparison expression.
+ * {@link org.elasticsearch.painless.node.EConditional} - Respresents a conditional expression.
+ * {@link org.elasticsearch.painless.node.EConstant} - Respresents a constant.  (Internal only.)
+ * {@link org.elasticsearch.painless.node.EDecimal} - Respresents a decimal constant.
+ * {@link org.elasticsearch.painless.node.EExplicit} - Represents an explicit cast.
+ * {@link org.elasticsearch.painless.node.ENull} - Represents a null constant.
+ * {@link org.elasticsearch.painless.node.ENumeric} - Respresents a non-decimal numeric constant.
+ * {@link org.elasticsearch.painless.node.EUnary} - Represents a unary math expression.
+ * {@link org.elasticsearch.painless.node.LArrayLength} - Represents an array length field load.
+ * {@link org.elasticsearch.painless.node.LBrace} - Represents an array load/store or defers to possible shortcuts.
+ * {@link org.elasticsearch.painless.node.LCall} - Represents a method call or deferes to a def call.
+ * {@link org.elasticsearch.painless.node.LCast} - Represents a cast made in a variable/method chain.
+ * {@link org.elasticsearch.painless.node.LDefArray} - Represents an array load/store or shortcut on a def type.  (Internal only.)
+ * {@link org.elasticsearch.painless.node.LDefCall} - Represents a method call made on a def type. (Internal only.)
+ * {@link org.elasticsearch.painless.node.LDefField} - Represents a field load/store or shortcut on a def type.  (Internal only.)
+ * {@link org.elasticsearch.painless.node.LField} - Represents a field load/store or defers to a possible shortcuts.
+ * {@link org.elasticsearch.painless.node.LListShortcut} - Represents a list load/store shortcut.  (Internal only.)
+ * {@link org.elasticsearch.painless.node.LMapShortcut} - Represents a map load/store shortcut. (Internal only.)
+ * {@link org.elasticsearch.painless.node.LNewArray} - Represents an array instantiation.
+ * {@link org.elasticsearch.painless.node.LNewObj} - Respresents and object instantiation.
+ * {@link org.elasticsearch.painless.node.LShortcut} - Represents a field load/store shortcut.  (Internal only.)
+ * {@link org.elasticsearch.painless.node.LString} - Represents a string constant.
+ * {@link org.elasticsearch.painless.node.LVariable} - Represents a variable load/store.
+ * {@link org.elasticsearch.painless.node.SBlock} - Represents a set of statements as a branch of control-flow.
+ * {@link org.elasticsearch.painless.node.SBreak} - Represents a break statement.
+ * {@link org.elasticsearch.painless.node.SContinue} - Represents a continue statement.
+ * {@link org.elasticsearch.painless.node.SDeclaration} - Represents a single variable declaration.
+ * {@link org.elasticsearch.painless.node.SDeclBlock} - Represents a series of declarations.
+ * {@link org.elasticsearch.painless.node.SDo} - Represents a do-while loop.
+ * {@link org.elasticsearch.painless.node.SExpression} - Represents the top-level node for an expression as a statement.
+ * {@link org.elasticsearch.painless.node.SFor} - Represents a for loop.
+ * {@link org.elasticsearch.painless.node.SIfElse} - Represents an if/else block.
+ * {@link org.elasticsearch.painless.node.SReturn} - Represents a return statement.
+ * {@link org.elasticsearch.painless.node.SSource} - The root of all Painless trees.  Contains a series of statements.
+ * {@link org.elasticsearch.painless.node.SThrow} - Represents a throw statement.
+ * {@link org.elasticsearch.painless.node.STrap} - Represents a catch block as part of a try-catch block.
+ * {@link org.elasticsearch.painless.node.STry} - Represents the try block as part of a try-catch block.
+ * {@link org.elasticsearch.painless.node.SWhile} - Represents a while loop.
+ * <p>
+ * Note that internal nodes are generated during the analysis phase by modifying the tree on-the-fly
+ * for clarity of development and convenience during the writing phase.
+ * <p>
+ * All Painless trees must start with an SSource node at the root.  Each node has a constructor that requires
+ * all of its values and children be passed in at the time of instantiation.  This means that Painless trees
+ * are build bottom-up; however, this helps enforce tree structure to be correct and fits naturally with a
+ * standard recurvise-descent parser.
+ * <p>
+ * Generally, statement nodes have member data that evaluate legal control-flow during the analysis phase.
+ * The typical order for statement nodes is for each node to call analyze on it's children during the analysis phase
+ * and write on it's children during the writing phase.  No modifications are made to the structure of statement nodes.
+ * <p>
+ * Generally, expression nodes have member data that evaluate static types.  The typical order for an expression node
+ * during the analysis phase looks like the following:
+ * {@code
+ * For known expected types:
+ *
+ * expression.child.expected = expectedType      // set the known expected type
+ *
+ * expression.child.analyze(...)                 // analyze the child node to set the child's actual type
+ *
+ * expression.child = expression.child.cast(...) // add an implicit cast node if the child node's
+ *                                               // actual type is not the expected type and set the
+ *                                               // expression's child to the implicit cast node
+ *
+ * For unknown expected types that need promotion:
+ *
+ * expression.child.analyze(...)                 // analyze the child node to set the child's actual type
+ *
+ * Type promote = Caster.promote(...)            // get the promotion type for the child based on
+ *                                               // the current operation and child's actual type
+ *
+ * expression.child.expected = promote           // set the expected type to the promotion type
+ *
+ * expression.child = expression.child.cast(...) // add an implicit cast node if the child node's
+ *                                               // actual type is not the expected type and set the
+ *                                               // expression's child to the implicit cast node
+ * }
+ * Expression nodes just call each child during the writing phase.
+ * <p>
+ * Generally, link nodes have member data that help keep track of items necessary to do a
+ * load/store on a variable/field/method.  Analysis of link nodes happens in a chain node
+ * where each link node will be analysed with the chain node acting as a bridge to pass the
+ * previous link's after type to the next link's before type.  Upon analysis completion, a link
+ * will return either itself or another link node depending on if a shortcut or def type was found.
+ * Cast nodes as links will return null and be removed from the chain node if the cast is
+ * unnecessary.  Link nodes have three methods for writing -- write, load, and store.  The write
+ * method is always once called before a load/store to give links a chance to write any values
+ * such as array indices before the load/store happens.  Load is called to read a link node, and
+ * store is called to write a link node.  Note that store will only ever be called on the final
+ * link node in a chain, all previous links will be considered loads.
+ */
+package org.elasticsearch.painless.node;
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BinaryOperatorTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BinaryOperatorTests.java
index 5971cca0151..54a36234fd8 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BinaryOperatorTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BinaryOperatorTests.java
@@ -55,13 +55,12 @@ public class BinaryOperatorTests extends ScriptTestCase {
     }
 
     public void testLongShiftsConst() {
-        // note: we always promote the results of shifts too (unlike java)
         assertEquals(1L << 2, exec("return 1L << 2;"));
-        assertEquals(1L << 2L, exec("return 1 << 2L;"));
-        assertEquals(4L >> 2L, exec("return 4 >> 2L;"));
+        assertEquals(1 << 2L, exec("return 1 << 2L;"));
+        assertEquals(4 >> 2L, exec("return 4 >> 2L;"));
         assertEquals(4L >> 2, exec("return 4L >> 2;"));
         assertEquals(-1L >>> 29, exec("return -1L >>> 29;"));
-        assertEquals(-1L >>> 29L, exec("return -1 >>> 29L;"));
+        assertEquals(-1 >>> 29L, exec("return -1 >>> 29L;"));
     }
 
     public void testMixedTypes() {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
index 949dd57ec5a..d29e5a33a04 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
@@ -32,21 +32,23 @@ import java.nio.charset.StandardCharsets;
 final class Debugger {
 
     /** compiles source to bytecode, and returns debugging output */
-    static String toString(String source) {
+    static String toString(final String source) {
         return toString(source, new CompilerSettings());
     }
 
     /** compiles to bytecode, and returns debugging output */
-    static String toString(String source, CompilerSettings settings) {
-        byte[] bytes = Compiler.compile("debugger", source, Definition.INSTANCE, settings);
-        ByteArrayOutputStream output = new ByteArrayOutputStream();
-        PrintWriter outputWriter = new PrintWriter(new OutputStreamWriter(output, StandardCharsets.UTF_8));
-        ClassReader reader = new ClassReader(bytes);
+    static String toString(final String source, final CompilerSettings settings) {
+        final byte[] bytes = Compiler.compile(source, settings);
+        final ByteArrayOutputStream output = new ByteArrayOutputStream();
+        final PrintWriter outputWriter = new PrintWriter(new OutputStreamWriter(output, StandardCharsets.UTF_8));
+        final ClassReader reader = new ClassReader(bytes);
+
         reader.accept(new TraceClassVisitor(outputWriter), 0);
         outputWriter.flush();
+
         try {
             return output.toString("UTF-8");
-        } catch (UnsupportedEncodingException e) {
+        } catch (final UnsupportedEncodingException e) {
             throw new RuntimeException(e);
         }
     }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefTests.java
index 5a3cea711a5..1d3906f248f 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefTests.java
@@ -401,26 +401,26 @@ public class DefTests extends ScriptTestCase {
         assertEquals(2L, exec("def x = (float)1 def y = (int)1 return x << y"));
         assertEquals(2L, exec("def x = (double)1 def y = (int)1 return x << y"));
 
-        assertEquals(2L, exec("def x = (byte)1 def y = (long)1 return x << y"));
-        assertEquals(2L, exec("def x = (short)1 def y = (long)1 return x << y"));
-        assertEquals(2L, exec("def x = (char)1 def y = (long)1 return x << y"));
-        assertEquals(2L, exec("def x = (int)1 def y = (long)1 return x << y"));
+        assertEquals(2, exec("def x = (byte)1 def y = (long)1 return x << y"));
+        assertEquals(2, exec("def x = (short)1 def y = (long)1 return x << y"));
+        assertEquals(2, exec("def x = (char)1 def y = (long)1 return x << y"));
+        assertEquals(2, exec("def x = (int)1 def y = (long)1 return x << y"));
         assertEquals(2L, exec("def x = (long)1 def y = (long)1 return x << y"));
         assertEquals(2L, exec("def x = (float)1 def y = (long)1 return x << y"));
         assertEquals(2L, exec("def x = (double)1 def y = (long)1 return x << y"));
 
-        assertEquals(2L, exec("def x = (byte)1 def y = (float)1 return x << y"));
-        assertEquals(2L, exec("def x = (short)1 def y = (float)1 return x << y"));
-        assertEquals(2L, exec("def x = (char)1 def y = (float)1 return x << y"));
-        assertEquals(2L, exec("def x = (int)1 def y = (float)1 return x << y"));
+        assertEquals(2, exec("def x = (byte)1 def y = (float)1 return x << y"));
+        assertEquals(2, exec("def x = (short)1 def y = (float)1 return x << y"));
+        assertEquals(2, exec("def x = (char)1 def y = (float)1 return x << y"));
+        assertEquals(2, exec("def x = (int)1 def y = (float)1 return x << y"));
         assertEquals(2L, exec("def x = (long)1 def y = (float)1 return x << y"));
         assertEquals(2L, exec("def x = (float)1 def y = (float)1 return x << y"));
         assertEquals(2L, exec("def x = (double)1 def y = (float)1 return x << y"));
 
-        assertEquals(2L, exec("def x = (byte)1 def y = (double)1 return x << y"));
-        assertEquals(2L, exec("def x = (short)1 def y = (double)1 return x << y"));
-        assertEquals(2L, exec("def x = (char)1 def y = (double)1 return x << y"));
-        assertEquals(2L, exec("def x = (int)1 def y = (double)1 return x << y"));
+        assertEquals(2, exec("def x = (byte)1 def y = (double)1 return x << y"));
+        assertEquals(2, exec("def x = (short)1 def y = (double)1 return x << y"));
+        assertEquals(2, exec("def x = (char)1 def y = (double)1 return x << y"));
+        assertEquals(2, exec("def x = (int)1 def y = (double)1 return x << y"));
         assertEquals(2L, exec("def x = (long)1 def y = (double)1 return x << y"));
         assertEquals(2L, exec("def x = (float)1 def y = (double)1 return x << y"));
         assertEquals(2L, exec("def x = (double)1 def y = (double)1 return x << y"));
@@ -467,26 +467,26 @@ public class DefTests extends ScriptTestCase {
         assertEquals(2L, exec("def x = (float)4 def y = (int)1 return x >> y"));
         assertEquals(2L, exec("def x = (double)4 def y = (int)1 return x >> y"));
 
-        assertEquals(2L, exec("def x = (byte)4 def y = (long)1 return x >> y"));
-        assertEquals(2L, exec("def x = (short)4 def y = (long)1 return x >> y"));
-        assertEquals(2L, exec("def x = (char)4 def y = (long)1 return x >> y"));
-        assertEquals(2L, exec("def x = (int)4 def y = (long)1 return x >> y"));
+        assertEquals(2, exec("def x = (byte)4 def y = (long)1 return x >> y"));
+        assertEquals(2, exec("def x = (short)4 def y = (long)1 return x >> y"));
+        assertEquals(2, exec("def x = (char)4 def y = (long)1 return x >> y"));
+        assertEquals(2, exec("def x = (int)4 def y = (long)1 return x >> y"));
         assertEquals(2L, exec("def x = (long)4 def y = (long)1 return x >> y"));
         assertEquals(2L, exec("def x = (float)4 def y = (long)1 return x >> y"));
         assertEquals(2L, exec("def x = (double)4 def y = (long)1 return x >> y"));
 
-        assertEquals(2L, exec("def x = (byte)4 def y = (float)1 return x >> y"));
-        assertEquals(2L, exec("def x = (short)4 def y = (float)1 return x >> y"));
-        assertEquals(2L, exec("def x = (char)4 def y = (float)1 return x >> y"));
-        assertEquals(2L, exec("def x = (int)4 def y = (float)1 return x >> y"));
+        assertEquals(2, exec("def x = (byte)4 def y = (float)1 return x >> y"));
+        assertEquals(2, exec("def x = (short)4 def y = (float)1 return x >> y"));
+        assertEquals(2, exec("def x = (char)4 def y = (float)1 return x >> y"));
+        assertEquals(2, exec("def x = (int)4 def y = (float)1 return x >> y"));
         assertEquals(2L, exec("def x = (long)4 def y = (float)1 return x >> y"));
         assertEquals(2L, exec("def x = (float)4 def y = (float)1 return x >> y"));
         assertEquals(2L, exec("def x = (double)4 def y = (float)1 return x >> y"));
 
-        assertEquals(2L, exec("def x = (byte)4 def y = (double)1 return x >> y"));
-        assertEquals(2L, exec("def x = (short)4 def y = (double)1 return x >> y"));
-        assertEquals(2L, exec("def x = (char)4 def y = (double)1 return x >> y"));
-        assertEquals(2L, exec("def x = (int)4 def y = (double)1 return x >> y"));
+        assertEquals(2, exec("def x = (byte)4 def y = (double)1 return x >> y"));
+        assertEquals(2, exec("def x = (short)4 def y = (double)1 return x >> y"));
+        assertEquals(2, exec("def x = (char)4 def y = (double)1 return x >> y"));
+        assertEquals(2, exec("def x = (int)4 def y = (double)1 return x >> y"));
         assertEquals(2L, exec("def x = (long)4 def y = (double)1 return x >> y"));
         assertEquals(2L, exec("def x = (float)4 def y = (double)1 return x >> y"));
         assertEquals(2L, exec("def x = (double)4 def y = (double)1 return x >> y"));
@@ -533,26 +533,26 @@ public class DefTests extends ScriptTestCase {
         assertEquals(2L, exec("def x = (float)4 def y = (int)1 return x >>> y"));
         assertEquals(2L, exec("def x = (double)4 def y = (int)1 return x >>> y"));
 
-        assertEquals(2L, exec("def x = (byte)4 def y = (long)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (short)4 def y = (long)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (char)4 def y = (long)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (int)4 def y = (long)1 return x >>> y"));
+        assertEquals(2, exec("def x = (byte)4 def y = (long)1 return x >>> y"));
+        assertEquals(2, exec("def x = (short)4 def y = (long)1 return x >>> y"));
+        assertEquals(2, exec("def x = (char)4 def y = (long)1 return x >>> y"));
+        assertEquals(2, exec("def x = (int)4 def y = (long)1 return x >>> y"));
         assertEquals(2L, exec("def x = (long)4 def y = (long)1 return x >>> y"));
         assertEquals(2L, exec("def x = (float)4 def y = (long)1 return x >>> y"));
         assertEquals(2L, exec("def x = (double)4 def y = (long)1 return x >>> y"));
 
-        assertEquals(2L, exec("def x = (byte)4 def y = (float)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (short)4 def y = (float)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (char)4 def y = (float)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (int)4 def y = (float)1 return x >>> y"));
+        assertEquals(2, exec("def x = (byte)4 def y = (float)1 return x >>> y"));
+        assertEquals(2, exec("def x = (short)4 def y = (float)1 return x >>> y"));
+        assertEquals(2, exec("def x = (char)4 def y = (float)1 return x >>> y"));
+        assertEquals(2, exec("def x = (int)4 def y = (float)1 return x >>> y"));
         assertEquals(2L, exec("def x = (long)4 def y = (float)1 return x >>> y"));
         assertEquals(2L, exec("def x = (float)4 def y = (float)1 return x >>> y"));
         assertEquals(2L, exec("def x = (double)4 def y = (float)1 return x >>> y"));
 
-        assertEquals(2L, exec("def x = (byte)4 def y = (double)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (short)4 def y = (double)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (char)4 def y = (double)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (int)4 def y = (double)1 return x >>> y"));
+        assertEquals(2, exec("def x = (byte)4 def y = (double)1 return x >>> y"));
+        assertEquals(2, exec("def x = (short)4 def y = (double)1 return x >>> y"));
+        assertEquals(2, exec("def x = (char)4 def y = (double)1 return x >>> y"));
+        assertEquals(2, exec("def x = (int)4 def y = (double)1 return x >>> y"));
         assertEquals(2L, exec("def x = (long)4 def y = (double)1 return x >>> y"));
         assertEquals(2L, exec("def x = (float)4 def y = (double)1 return x >>> y"));
         assertEquals(2L, exec("def x = (double)4 def y = (double)1 return x >>> y"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java
index dd3c2c67fdb..e5284cd0eb9 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java
@@ -24,15 +24,15 @@ import java.util.HashMap;
 
 /** Tests for special reserved words such as _score */
 public class ReservedWordTests extends ScriptTestCase {
-    
+
     /** check that we can't declare a variable of _score, its really reserved! */
     public void testScoreVar() {
         IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
             exec("int _score = 5; return _score;");
         });
-        assertTrue(expected.getMessage().contains("Variable name [_score] already defined"));
+        assertTrue(expected.getMessage().contains("Variable name [_score] is reserved"));
     }
-    
+
     /** check that we can't write to _score, its read-only! */
     public void testScoreStore() {
         IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
@@ -40,15 +40,15 @@ public class ReservedWordTests extends ScriptTestCase {
         });
         assertTrue(expected.getMessage().contains("Variable [_score] is read-only"));
     }
-    
+
     /** check that we can't declare a variable of doc, its really reserved! */
     public void testDocVar() {
         IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
             exec("int doc = 5; return doc;");
         });
-        assertTrue(expected.getMessage().contains("Variable name [doc] already defined"));
+        assertTrue(expected.getMessage().contains("Variable name [doc] is reserved"));
     }
-    
+
     /** check that we can't write to doc, its read-only! */
     public void testDocStore() {
         IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
@@ -56,15 +56,15 @@ public class ReservedWordTests extends ScriptTestCase {
         });
         assertTrue(expected.getMessage().contains("Variable [doc] is read-only"));
     }
-    
+
     /** check that we can't declare a variable of ctx, its really reserved! */
     public void testCtxVar() {
         IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
             exec("int ctx = 5; return ctx;");
         });
-        assertTrue(expected.getMessage().contains("Variable name [ctx] already defined"));
+        assertTrue(expected.getMessage().contains("Variable name [ctx] is reserved"));
     }
-    
+
     /** check that we can't write to ctx, its read-only! */
     public void testCtxStore() {
         IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
@@ -72,20 +72,20 @@ public class ReservedWordTests extends ScriptTestCase {
         });
         assertTrue(expected.getMessage().contains("Variable [ctx] is read-only"));
     }
-    
+
     /** check that we can modify its contents though */
     public void testCtxStoreMap() {
         assertEquals(5, exec("ctx.foo = 5; return ctx.foo;", Collections.singletonMap("ctx", new HashMap<String,Object>())));
     }
-    
+
     /** check that we can't declare a variable of _value, its really reserved! */
     public void testAggregationValueVar() {
         IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
             exec("int _value = 5; return _value;");
         });
-        assertTrue(expected.getMessage().contains("Variable name [_value] already defined"));
+        assertTrue(expected.getMessage().contains("Variable name [_value] is reserved"));
     }
-    
+
     /** check that we can't write to _value, its read-only! */
     public void testAggregationValueStore() {
         IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
index f619d44ad81..0c2b2f4386c 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
@@ -132,26 +132,30 @@ public class StringTests extends ScriptTestCase {
 
         try {
             assertEquals("cc", exec("return (String)(char)\"cc\""));
-        } catch (final IllegalArgumentException ise) {
-            ise.getMessage().contains("Cannot cast constant from [String] to [char].");
+            fail();
+        } catch (final ClassCastException cce) {
+            assertTrue(cce.getMessage().contains("Cannot cast from [String] to [char]."));
         }
 
         try {
             assertEquals("cc", exec("return (String)(char)'cc'"));
-        } catch (final IllegalArgumentException ise) {
-            ise.getMessage().contains("Cannot cast constant from [String] to [char].");
+            fail();
+        } catch (final ClassCastException cce) {
+            assertTrue(cce.getMessage().contains("Cannot cast from [String] to [char]."));
         }
 
         try {
             assertEquals('c', exec("String s = \"cc\" (char)s"));
+            fail();
         } catch (final ClassCastException cce) {
-            cce.getMessage().contains("Cannot cast [String] with length greater than one to [char].");
+            assertTrue(cce.getMessage().contains("Cannot cast [String] with length greater than one to [char]."));
         }
 
         try {
             assertEquals('c', exec("String s = 'cc' (char)s"));
+            fail();
         } catch (final ClassCastException cce) {
-            cce.getMessage().contains("Cannot cast [String] with length greater than one to [char].");
+            assertTrue(cce.getMessage().contains("Cannot cast [String] with length greater than one to [char]."));
         }
 
         assertEquals('c', exec("return (Character)\"c\""));
@@ -164,26 +168,30 @@ public class StringTests extends ScriptTestCase {
 
         try {
             assertEquals("cc", exec("return (String)(Character)\"cc\""));
+            fail();
         } catch (final ClassCastException ise) {
-            ise.getMessage().contains("Cannot cast [String] with length greater than one to [Character].");
+            assertTrue(ise.getMessage().contains("Cannot cast [String] with length greater than one to [Character]."));
         }
 
         try {
             assertEquals("cc", exec("return (String)(Character)'cc'"));
+            fail();
         } catch (final ClassCastException ise) {
-            ise.getMessage().contains("Cannot cast [String] with length greater than one to [Character].");
+            assertTrue(ise.getMessage().contains("Cannot cast [String] with length greater than one to [Character]."));
         }
 
         try {
             assertEquals('c', exec("String s = \"cc\" (Character)s"));
+            fail();
         } catch (final ClassCastException cce) {
-            cce.getMessage().contains("Cannot cast [String] with length greater than one to [Character].");
+            assertTrue(cce.getMessage().contains("Cannot cast [String] with length greater than one to [Character]."));
         }
 
         try {
             assertEquals('c', exec("String s = 'cc' (Character)s"));
+            fail();
         } catch (final ClassCastException cce) {
-            cce.getMessage().contains("Cannot cast [String] with length greater than one to [Character].");
+            assertTrue(cce.getMessage().contains("Cannot cast [String] with length greater than one to [Character]."));
         }
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
index 2f80d1d1c39..5e4d1aa2014 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
@@ -98,7 +98,7 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
             exec("try { int x } catch (PainlessError error) {}");
             fail("should have hit ParseException");
         });
-        assertTrue(parseException.getMessage().contains("Invalid type [PainlessError]."));
+        assertTrue(parseException.getMessage().contains("Not a type [PainlessError]."));
     }
 
     public void testLoopLimits() {
@@ -126,20 +126,20 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
         // ok
         assertEquals(0, exec(new String(exactlyAtLimit)));
     }
-    
+
     public void testIllegalDynamicMethod() {
         IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
             exec("def x = 'test'; return x.getClass().toString()");
         });
         assertTrue(expected.getMessage().contains("Unable to find dynamic method"));
     }
-    
+
     public void testDynamicNPE() {
         expectThrows(NullPointerException.class, () -> {
             exec("def x = null; return x.toString()");
         });
     }
-    
+
     public void testDynamicWrongArgs() {
         expectThrows(WrongMethodTypeException.class, () -> {
             exec("def x = new ArrayList(); return x.get('bogus');");

From 9d5537d8741b0fbd850a929eaf1aead12ec0c60b Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Thu, 12 May 2016 09:40:06 +0200
Subject: [PATCH 0203/1311] Invoke scripts from Gradle with the shell
 interpreter they choose

---
 .../org/elasticsearch/gradle/test/ClusterFormationTasks.groovy | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
index 34dde6e5dad..83dd1b7e4c5 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@@ -418,8 +418,7 @@ class ClusterFormationTasks {
                 // argument are wrapped in an ExecArgWrapper that escapes commas
                 args execArgs.collect { a -> new EscapeCommaWrapper(arg: a) }
             } else {
-                executable 'sh'
-                args execArgs
+                commandLine execArgs
             }
         }
     }

From 6d5e24726f2f11ac5341e313393fb9120b16879c Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Thu, 12 May 2016 12:29:18 +0200
Subject: [PATCH 0204/1311] Fix test failures.

---
 .../query-dsl/geo-polygon-query.asciidoc      | 35 ++++++----------
 .../reference/query-dsl/nested-query.asciidoc | 30 +++++++-------
 .../query-dsl/parent-id-query.asciidoc        | 40 ++++++++++++++++---
 3 files changed, 61 insertions(+), 44 deletions(-)

diff --git a/docs/reference/query-dsl/geo-polygon-query.asciidoc b/docs/reference/query-dsl/geo-polygon-query.asciidoc
index 9a0800c7550..725abe91735 100644
--- a/docs/reference/query-dsl/geo-polygon-query.asciidoc
+++ b/docs/reference/query-dsl/geo-polygon-query.asciidoc
@@ -10,7 +10,7 @@ GET /_search
 {
     "query": {
         "bool" : {
-            "query" : {
+            "must": {
                 "geo_polygon" : {
                     "person.location" : {
                         "points" : [
@@ -59,9 +59,6 @@ GET /_search
     "query": {
         "bool" : {
             "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
                 "geo_polygon" : {
                     "person.location" : {
                         "points" : [
@@ -90,19 +87,16 @@ GET /_search
     "query": {
         "bool" : {
             "must" : {
-                "match_all" : {}
-            },
-                "filter" : {
-                    "geo_polygon" : {
-                        "person.location" : {
-                            "points" : [
-                                "40, -70",
+                "geo_polygon" : {
+                    "person.location" : {
+                        "points" : [
+                            "40, -70",
                             "30, -80",
                             "20, -90"
-                            ]
-                        }
+                        ]
                     }
                 }
+            }
         }
     }
 }
@@ -119,19 +113,16 @@ GET /_search
     "query": {
         "bool" : {
             "must" : {
-                "match_all" : {}
-            },
-                "filter" : {
-                    "geo_polygon" : {
-                        "person.location" : {
-                            "points" : [
-                                "drn5x1g8cu2y",
+                "geo_polygon" : {
+                    "person.location" : {
+                        "points" : [
+                            "drn5x1g8cu2y",
                             "30, -80",
                             "20, -90"
-                            ]
-                        }
+                        ]
                     }
                 }
+            }
         }
     }
 }
diff --git a/docs/reference/query-dsl/nested-query.asciidoc b/docs/reference/query-dsl/nested-query.asciidoc
index 17ee20b983e..06720552bde 100644
--- a/docs/reference/query-dsl/nested-query.asciidoc
+++ b/docs/reference/query-dsl/nested-query.asciidoc
@@ -12,10 +12,12 @@ will work with:
 --------------------------------------------------
 PUT /my_index
 {
-    "type1" : {
-        "properties" : {
-            "obj1" : {
-                "type" : "nested"
+    "mappings": {
+        "type1" : {
+            "properties" : {
+                "obj1" : {
+                    "type" : "nested"
+                }
             }
         }
     }
@@ -33,19 +35,15 @@ GET /_search
     "query": {
         "nested" : {
             "path" : "obj1",
-                "score_mode" : "avg",
-                "query" : {
-                    "bool" : {
-                        "must" : [
-                        {
-                            "match" : {"obj1.name" : "blue"}
-                        },
-                        {
-                            "range" : {"obj1.count" : {"gt" : 5}}
-                        }
-                        ]
-                    }
+            "score_mode" : "avg",
+            "query" : {
+                "bool" : {
+                    "must" : [
+                    { "match" : {"obj1.name" : "blue"} },
+                    { "range" : {"obj1.count" : {"gt" : 5}} }
+                    ]
                 }
+            }
         }
     }
 }
diff --git a/docs/reference/query-dsl/parent-id-query.asciidoc b/docs/reference/query-dsl/parent-id-query.asciidoc
index 321adeecb5e..10cf6d05664 100644
--- a/docs/reference/query-dsl/parent-id-query.asciidoc
+++ b/docs/reference/query-dsl/parent-id-query.asciidoc
@@ -3,32 +3,60 @@
 
 added[5.0.0]
 
-The `parent_id` query can be used to find child documents which belong to a particular parent:
+The `parent_id` query can be used to find child documents which belong to a
+particular parent. Given the following mapping definition:
 
 [source,js]
 --------------------------------------------------
 PUT /my_index
 {
-    "parent_id" : {
-        "type" : "blog_tag",
-        "id" : "1"
+    "mappings": {
+        "blog_post": {
+            "properties": {
+                "name": {
+                    "type": "keyword"
+                }
+            }
+        },
+        "blog_tag": {
+            "_parent": {
+                "type": "blog_post"
+            },
+            "_routing": {
+                "required": true
+            }
+        }
     }
 }
 --------------------------------------------------
 // CONSOLE
 // TESTSETUP
 
+[source,js]
+--------------------------------------------------
+GET /_search
+{
+    "query": {
+        "parent_id" : {
+            "type" : "blog_tag",
+                "id" : "1"
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+
 The above is functionally equivalent to using the following
 <<query-dsl-has-parent-query, `has_parent`>> query, but performs
 better as it does not need to do a join:
 
 [source,js]
 --------------------------------------------------
-GET /_search
+GET /my_index/_search
 {
     "query": {
         "has_parent": {
-            "type": "blog",
+            "type": "blog_post",
                 "query": {
                     "term": {
                         "_id": "1"

From 126ff91bf6dc0868a57d66b8274e9ddff9c266d8 Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Thu, 12 May 2016 12:30:33 +0200
Subject: [PATCH 0205/1311] Fix indent

---
 docs/reference/query-dsl/constant-score-query.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/query-dsl/constant-score-query.asciidoc b/docs/reference/query-dsl/constant-score-query.asciidoc
index 5002efcdabe..bced9fc9fbe 100644
--- a/docs/reference/query-dsl/constant-score-query.asciidoc
+++ b/docs/reference/query-dsl/constant-score-query.asciidoc
@@ -14,7 +14,7 @@ GET /_search
             "filter" : {
                 "term" : { "user" : "kimchy"}
             },
-                "boost" : 1.2
+            "boost" : 1.2
         }
     }
 }

From 0ad87b25cf78f89c108dfb17510c2d86be394172 Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Thu, 12 May 2016 12:58:22 +0200
Subject: [PATCH 0206/1311] Something messed with auto-indent. Fixed now.

---
 .../query-dsl/boosting-query.asciidoc         | 12 +--
 .../query-dsl/common-terms-query.asciidoc     | 30 ++++----
 .../query-dsl/geo-bounding-box-query.asciidoc | 76 +++++++++----------
 .../geo-distance-range-query.asciidoc         | 16 ++--
 .../query-dsl/geo-polygon-query.asciidoc      | 24 ++++--
 .../query-dsl/geohash-cell-query.asciidoc     | 18 ++---
 .../query-dsl/has-child-query.asciidoc        | 16 ++--
 docs/reference/query-dsl/ids-query.asciidoc   |  2 +-
 .../query-dsl/indices-query.asciidoc          |  8 +-
 .../query-dsl/match-phrase-query.asciidoc     |  2 +-
 docs/reference/query-dsl/match-query.asciidoc |  8 +-
 .../query-dsl/percolate-query.asciidoc        | 10 +--
 .../query-dsl/query-string-query.asciidoc     |  8 +-
 .../reference/query-dsl/regexp-query.asciidoc |  4 +-
 docs/reference/query-dsl/terms-query.asciidoc |  6 +-
 15 files changed, 124 insertions(+), 116 deletions(-)

diff --git a/docs/reference/query-dsl/boosting-query.asciidoc b/docs/reference/query-dsl/boosting-query.asciidoc
index fdeace68a49..5bb07392ab7 100644
--- a/docs/reference/query-dsl/boosting-query.asciidoc
+++ b/docs/reference/query-dsl/boosting-query.asciidoc
@@ -17,12 +17,12 @@ GET /_search
                     "field1" : "value1"
                 }
             },
-                "negative" : {
-                    "term" : {
-                        "field2" : "value2"
-                    }
-                },
-                "negative_boost" : 0.2
+            "negative" : {
+                 "term" : {
+                     "field2" : "value2"
+                }
+            },
+            "negative_boost" : 0.2
         }
     }
 }
diff --git a/docs/reference/query-dsl/common-terms-query.asciidoc b/docs/reference/query-dsl/common-terms-query.asciidoc
index 9c1dde02831..fcc4ace2ec6 100644
--- a/docs/reference/query-dsl/common-terms-query.asciidoc
+++ b/docs/reference/query-dsl/common-terms-query.asciidoc
@@ -173,11 +173,11 @@ GET /_search
                     "minimum_should_match": 2
                 }
             },
-                "should": [
-                    { "term": { "body": "the"}},
-                    { "term": { "body": "as"}},
-                    { "term": { "body": "a"}}
-                    ]
+            "should": [
+                { "term": { "body": "the"}},
+                { "term": { "body": "as"}},
+                { "term": { "body": "a"}}
+                ]
         }
     }
 }
@@ -230,17 +230,17 @@ GET /_search
                     "minimum_should_match": 2
                 }
             },
-                "should": {
-                    "bool": {
-                        "should": [
-                        { "term": { "body": "the"}},
-                        { "term": { "body": "not"}},
-                        { "term": { "body": "as"}},
-                        { "term": { "body": "a"}}
-                        ],
-                        "minimum_should_match": 3
-                    }
+            "should": {
+                "bool": {
+                    "should": [
+                    { "term": { "body": "the"}},
+                    { "term": { "body": "not"}},
+                    { "term": { "body": "as"}},
+                    { "term": { "body": "a"}}
+                    ],
+                    "minimum_should_match": 3
                 }
+            }
         }
     }
 }
diff --git a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
index dfe4ba25b39..40debb57105 100644
--- a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
+++ b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
@@ -48,20 +48,20 @@ GET /_search
             "must" : {
                 "match_all" : {}
             },
-                "filter" : {
-                    "geo_bounding_box" : {
-                        "pin.location" : {
-                            "top_left" : {
-                                "lat" : 40.73,
-                                "lon" : -74.1
-                            },
-                            "bottom_right" : {
-                                "lat" : 40.01,
-                                "lon" : -71.12
-                            }
+            "filter" : {
+                "geo_bounding_box" : {
+                    "pin.location" : {
+                        "top_left" : {
+                            "lat" : 40.73,
+                            "lon" : -74.1
+                        },
+                        "bottom_right" : {
+                            "lat" : 40.01,
+                            "lon" : -71.12
                         }
                     }
                 }
+            }
         }
     }
 }
@@ -141,14 +141,14 @@ GET /_search
             "must" : {
                 "match_all" : {}
             },
-                "filter" : {
-                    "geo_bounding_box" : {
-                        "pin.location" : {
-                            "top_left" : [-74.1, 40.73],
-                            "bottom_right" : [-71.12, 40.01]
-                        }
+            "filter" : {
+                "geo_bounding_box" : {
+                    "pin.location" : {
+                        "top_left" : [-74.1, 40.73],
+                        "bottom_right" : [-71.12, 40.01]
                     }
                 }
+            }
         }
     }
 }
@@ -169,17 +169,17 @@ GET /_search
             "must" : {
                 "match_all" : {}
             },
-                "filter" : {
-                    "geo_bounding_box" : {
-                        "pin.location" : {
-                            "top_left" : "40.73, -74.1",
-                            "bottom_right" : "40.01, -71.12"
-                        }
+            "filter" : {
+                "geo_bounding_box" : {
+                    "pin.location" : {
+                        "top_left" : "40.73, -74.1",
+                        "bottom_right" : "40.01, -71.12"
                     }
                 }
-        }
+            }
     }
 }
+}
 --------------------------------------------------
 // CONSOLE
 
@@ -195,14 +195,14 @@ GET /_search
             "must" : {
                 "match_all" : {}
             },
-                "filter" : {
-                    "geo_bounding_box" : {
-                        "pin.location" : {
-                            "top_left" : "dr5r9ydj2y73",
-                            "bottom_right" : "drj7teegpus6"
-                        }
+            "filter" : {
+                "geo_bounding_box" : {
+                    "pin.location" : {
+                        "top_left" : "dr5r9ydj2y73",
+                        "bottom_right" : "drj7teegpus6"
                     }
                 }
+            }
         }
     }
 }
@@ -228,16 +228,16 @@ GET /_search
             "must" : {
                 "match_all" : {}
             },
-                "filter" : {
-                    "geo_bounding_box" : {
-                        "pin.location" : {
-                            "top" : 40.73,
-                            "left" : -74.1,
-                            "bottom" : 40.01,
-                            "right" : -71.12
-                        }
+            "filter" : {
+                "geo_bounding_box" : {
+                    "pin.location" : {
+                        "top" : 40.73,
+                        "left" : -74.1,
+                        "bottom" : 40.01,
+                        "right" : -71.12
                     }
                 }
+            }
         }
     }
 }
diff --git a/docs/reference/query-dsl/geo-distance-range-query.asciidoc b/docs/reference/query-dsl/geo-distance-range-query.asciidoc
index 4236e2b370c..c54c50638bc 100644
--- a/docs/reference/query-dsl/geo-distance-range-query.asciidoc
+++ b/docs/reference/query-dsl/geo-distance-range-query.asciidoc
@@ -12,16 +12,16 @@ GET /_search
             "must" : {
                 "match_all" : {}
             },
-                "filter" : {
-                    "geo_distance_range" : {
-                        "from" : "200km",
-                        "to" : "400km",
-                        "pin.location" : {
-                            "lat" : 40,
-                            "lon" : -70
-                        }
+            "filter" : {
+                "geo_distance_range" : {
+                    "from" : "200km",
+                    "to" : "400km",
+                    "pin.location" : {
+                        "lat" : 40,
+                        "lon" : -70
                     }
                 }
+            }
         }
     }
 }
diff --git a/docs/reference/query-dsl/geo-polygon-query.asciidoc b/docs/reference/query-dsl/geo-polygon-query.asciidoc
index 725abe91735..001ae63c11d 100644
--- a/docs/reference/query-dsl/geo-polygon-query.asciidoc
+++ b/docs/reference/query-dsl/geo-polygon-query.asciidoc
@@ -10,7 +10,10 @@ GET /_search
 {
     "query": {
         "bool" : {
-            "must": {
+            "query" : {
+                "match_all" : {}
+            },
+            "filter" : {
                 "geo_polygon" : {
                     "person.location" : {
                         "points" : [
@@ -58,7 +61,10 @@ GET /_search
 {
     "query": {
         "bool" : {
-            "must" : {
+            "query" : {
+                "match_all" : {}
+            },
+            "filter" : {
                 "geo_polygon" : {
                     "person.location" : {
                         "points" : [
@@ -86,8 +92,11 @@ GET /_search
 {
     "query": {
         "bool" : {
-            "must" : {
-                "geo_polygon" : {
+            "query" : {
+                "match_all" : {}
+            },
+            "filter" : {
+               "geo_polygon" : {
                     "person.location" : {
                         "points" : [
                             "40, -70",
@@ -112,8 +121,11 @@ GET /_search
 {
     "query": {
         "bool" : {
-            "must" : {
-                "geo_polygon" : {
+            "query" : {
+                "match_all" : {}
+            },
+            "filter" : {
+               "geo_polygon" : {
                     "person.location" : {
                         "points" : [
                             "drn5x1g8cu2y",
diff --git a/docs/reference/query-dsl/geohash-cell-query.asciidoc b/docs/reference/query-dsl/geohash-cell-query.asciidoc
index 3728a36de5b..27e6319bc71 100644
--- a/docs/reference/query-dsl/geohash-cell-query.asciidoc
+++ b/docs/reference/query-dsl/geohash-cell-query.asciidoc
@@ -52,16 +52,16 @@ GET /_search
             "must" : {
                 "match_all" : {}
             },
-                "filter" : {
-                    "geohash_cell": {
-                        "pin": {
-                            "lat": 13.4080,
-                            "lon": 52.5186
-                        },
-                        "precision": 3,
-                        "neighbors": true
-                    }
+            "filter" : {
+                "geohash_cell": {
+                    "pin": {
+                        "lat": 13.4080,
+                        "lon": 52.5186
+                    },
+                    "precision": 3,
+                    "neighbors": true
                 }
+            }
         }
     }
 }
diff --git a/docs/reference/query-dsl/has-child-query.asciidoc b/docs/reference/query-dsl/has-child-query.asciidoc
index b863a91074c..3b8352fd4e0 100644
--- a/docs/reference/query-dsl/has-child-query.asciidoc
+++ b/docs/reference/query-dsl/has-child-query.asciidoc
@@ -67,14 +67,14 @@ GET /_search
     "query": {
         "has_child" : {
             "type" : "blog_tag",
-                "score_mode" : "min",
-                "min_children": 2, <1>
-                    "max_children": 10, <1>
-                    "query" : {
-                        "term" : {
-                            "tag" : "something"
-                        }
-                    }
+            "score_mode" : "min",
+            "min_children": 2, <1>
+            "max_children": 10, <1>
+            "query" : {
+                "term" : {
+                    "tag" : "something"
+                }
+            }
         }
     }
 }
diff --git a/docs/reference/query-dsl/ids-query.asciidoc b/docs/reference/query-dsl/ids-query.asciidoc
index acf5722fe72..09541ce51d3 100644
--- a/docs/reference/query-dsl/ids-query.asciidoc
+++ b/docs/reference/query-dsl/ids-query.asciidoc
@@ -11,7 +11,7 @@ GET /_search
     "query": {
         "ids" : {
             "type" : "my_type",
-                "values" : ["1", "4", "100"]
+            "values" : ["1", "4", "100"]
         }
     }
 }    
diff --git a/docs/reference/query-dsl/indices-query.asciidoc b/docs/reference/query-dsl/indices-query.asciidoc
index 552f5927fe1..8f2f958086e 100644
--- a/docs/reference/query-dsl/indices-query.asciidoc
+++ b/docs/reference/query-dsl/indices-query.asciidoc
@@ -14,12 +14,8 @@ GET /_search
     "query": {
         "indices" : {
             "indices" : ["index1", "index2"],
-                "query" : {
-                    "term" : { "tag" : "wow" }
-                },
-                "no_match_query" : {
-                    "term" : { "tag" : "kow" }
-                }
+            "query" : { "term" : { "tag" : "wow" } },
+            "no_match_query" : { "term" : { "tag" : "kow" } }
         }
     }
 }
diff --git a/docs/reference/query-dsl/match-phrase-query.asciidoc b/docs/reference/query-dsl/match-phrase-query.asciidoc
index 717cb94312e..943d0e84d36 100644
--- a/docs/reference/query-dsl/match-phrase-query.asciidoc
+++ b/docs/reference/query-dsl/match-phrase-query.asciidoc
@@ -32,7 +32,7 @@ GET /_search
         "match_phrase" : {
             "message" : {
                 "query" : "this is a test",
-                    "analyzer" : "my_analyzer"
+                "analyzer" : "my_analyzer"
             }
         }
     }
diff --git a/docs/reference/query-dsl/match-query.asciidoc b/docs/reference/query-dsl/match-query.asciidoc
index 8a301081445..2a1b29ea297 100644
--- a/docs/reference/query-dsl/match-query.asciidoc
+++ b/docs/reference/query-dsl/match-query.asciidoc
@@ -64,7 +64,7 @@ GET /_search
         "match" : {
             "message" : {
                 "query" : "this is a test",
-                    "operator" : "and"
+                "operator" : "and"
             }
         }
     }
@@ -87,8 +87,8 @@ GET /_search
         "match" : {
             "message" : {
                 "query" : "to be or not to be",
-                    "operator" : "and",
-                    "zero_terms_query": "all"
+                "operator" : "and",
+                "zero_terms_query": "all"
             }
         }
     }
@@ -128,7 +128,7 @@ GET /_search
         "match" : {
             "message" : {
                 "query" : "to be or not to be",
-                    "cutoff_frequency" : 0.001
+                "cutoff_frequency" : 0.001
             }
         }
     }
diff --git a/docs/reference/query-dsl/percolate-query.asciidoc b/docs/reference/query-dsl/percolate-query.asciidoc
index 8c0e4dc1feb..59d38f101ad 100644
--- a/docs/reference/query-dsl/percolate-query.asciidoc
+++ b/docs/reference/query-dsl/percolate-query.asciidoc
@@ -23,13 +23,13 @@ PUT /my-index
                 }
             }
         },
-            "queries": {
-                "properties": {
-                    "query": {
-                        "type": "percolator"
-                    }
+        "queries": {
+            "properties": {
+                "query": {
+                    "type": "percolator"
                 }
             }
+        }
     }
 }
 --------------------------------------------------
diff --git a/docs/reference/query-dsl/query-string-query.asciidoc b/docs/reference/query-dsl/query-string-query.asciidoc
index c1f5c06f65c..60477d6e28a 100644
--- a/docs/reference/query-dsl/query-string-query.asciidoc
+++ b/docs/reference/query-dsl/query-string-query.asciidoc
@@ -11,7 +11,7 @@ GET /_search
     "query": {
         "query_string" : {
             "default_field" : "content",
-                "query" : "this AND that OR thus"
+            "query" : "this AND that OR thus"
         }
     }
 }
@@ -122,7 +122,7 @@ GET /_search
     "query": {
         "query_string" : {
             "fields" : ["content", "name"],
-                "query" : "this AND that"
+            "query" : "this AND that"
         }
     }
 }
@@ -177,8 +177,8 @@ GET /_search
     "query": {
         "query_string" : {
             "fields" : ["city.*"],
-                "query" : "this AND that OR thus",
-                "use_dis_max" : true
+            "query" : "this AND that OR thus",
+            "use_dis_max" : true
         }
     }
 }
diff --git a/docs/reference/query-dsl/regexp-query.asciidoc b/docs/reference/query-dsl/regexp-query.asciidoc
index 55a1c3469c6..2d57f1d7dd0 100644
--- a/docs/reference/query-dsl/regexp-query.asciidoc
+++ b/docs/reference/query-dsl/regexp-query.asciidoc
@@ -36,7 +36,7 @@ GET /_search
         "regexp":{
             "name.first":{
                 "value":"s.*y",
-                    "boost":1.2
+                "boost":1.2
             }
         }
     }
@@ -54,7 +54,7 @@ GET /_search
         "regexp":{
             "name.first": {
                 "value": "s.*y",
-                    "flags" : "INTERSECTION|COMPLEMENT|EMPTY"
+                "flags" : "INTERSECTION|COMPLEMENT|EMPTY"
             }
         }
     }
diff --git a/docs/reference/query-dsl/terms-query.asciidoc b/docs/reference/query-dsl/terms-query.asciidoc
index a840292d242..e00c18bb56f 100644
--- a/docs/reference/query-dsl/terms-query.asciidoc
+++ b/docs/reference/query-dsl/terms-query.asciidoc
@@ -89,9 +89,9 @@ GET /tweets/_search
         "terms" : {
             "user" : {
                 "index" : "users",
-                    "type" : "user",
-                    "id" : "2",
-                    "path" : "followers"
+                "type" : "user",
+                "id" : "2",
+                "path" : "followers"
             }
         }
     }

From 36cd69c6acdfb5334999e920635bf07eeb689aa0 Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Thu, 12 May 2016 13:23:06 +0200
Subject: [PATCH 0207/1311] Fix build failure

---
 docs/reference/query-dsl/geo-polygon-query.asciidoc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/reference/query-dsl/geo-polygon-query.asciidoc b/docs/reference/query-dsl/geo-polygon-query.asciidoc
index 001ae63c11d..35278fda4a5 100644
--- a/docs/reference/query-dsl/geo-polygon-query.asciidoc
+++ b/docs/reference/query-dsl/geo-polygon-query.asciidoc
@@ -10,7 +10,7 @@ GET /_search
 {
     "query": {
         "bool" : {
-            "query" : {
+            "must" : {
                 "match_all" : {}
             },
             "filter" : {
@@ -61,7 +61,7 @@ GET /_search
 {
     "query": {
         "bool" : {
-            "query" : {
+            "must" : {
                 "match_all" : {}
             },
             "filter" : {
@@ -92,7 +92,7 @@ GET /_search
 {
     "query": {
         "bool" : {
-            "query" : {
+            "must" : {
                 "match_all" : {}
             },
             "filter" : {
@@ -121,7 +121,7 @@ GET /_search
 {
     "query": {
         "bool" : {
-            "query" : {
+            "must" : {
                 "match_all" : {}
             },
             "filter" : {

From 9ce96f57924aacaa8a624222a9b7966bfec22c11 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 12 May 2016 09:04:24 -0400
Subject: [PATCH 0208/1311] Add tests that packages depend on bash

This commit adds bats tests that the RPM and Debian packages depend on
bash.

Relates #18292
---
 .../src/test/resources/packaging/scripts/30_deb_package.bats  | 4 ++++
 .../src/test/resources/packaging/scripts/40_rpm_package.bats  | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/qa/vagrant/src/test/resources/packaging/scripts/30_deb_package.bats b/qa/vagrant/src/test/resources/packaging/scripts/30_deb_package.bats
index b1136c39b9f..1f4cdeeeb3f 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/30_deb_package.bats
+++ b/qa/vagrant/src/test/resources/packaging/scripts/30_deb_package.bats
@@ -40,6 +40,10 @@ setup() {
     export_elasticsearch_paths
 }
 
+@test "[DEB] package depends on bash" {
+    dpkg -I elasticsearch-$(cat version).deb | grep "Depends:.*bash.*"
+}
+
 ##################################
 # Install DEB package
 ##################################
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats b/qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats
index 0555f55cba6..c47f24e6c67 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats
+++ b/qa/vagrant/src/test/resources/packaging/scripts/40_rpm_package.bats
@@ -39,6 +39,10 @@ setup() {
     export_elasticsearch_paths
 }
 
+@test "[RPM] package depends on bash" {
+    rpm -qpR elasticsearch-$(cat version).rpm | grep '/bin/bash'
+}
+
 ##################################
 # Install RPM package
 ##################################

From 5ee5cc25ccaf4c11deb5a8e365df6c6071f126d2 Mon Sep 17 00:00:00 2001
From: Zachary Tong <polyfractal@elastic.co>
Date: Thu, 12 May 2016 09:34:15 -0400
Subject: [PATCH 0209/1311] Move AsciiFolding earlier in FingerprintAnalyzer
 filter chain

Rearranges the FingerprintAnalyzer so that AsciiFolding comes earlier in the chain (after lowercasing, before stop removal, for maximum deduping power)

Closes #18266
---
 .../index/analysis/FingerprintAnalyzer.java           |  2 +-
 .../index/analysis/FingerprintAnalyzerTests.java      |  6 +++++-
 .../analysis/analyzers/fingerprint-analyzer.asciidoc  | 11 +++++------
 3 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/analysis/FingerprintAnalyzer.java b/core/src/main/java/org/elasticsearch/index/analysis/FingerprintAnalyzer.java
index 66fdbeaeb39..f7bf44256cc 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/FingerprintAnalyzer.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/FingerprintAnalyzer.java
@@ -48,9 +48,9 @@ public final class FingerprintAnalyzer extends Analyzer {
         final Tokenizer tokenizer = new StandardTokenizer();
         TokenStream stream = tokenizer;
         stream = new LowerCaseFilter(stream);
+        stream = new ASCIIFoldingFilter(stream, preserveOriginal);
         stream = new StopFilter(stream, stopWords);
         stream = new FingerprintFilter(stream, maxOutputSize, separator);
-        stream = new ASCIIFoldingFilter(stream, preserveOriginal);
         return new TokenStreamComponents(tokenizer, stream);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/FingerprintAnalyzerTests.java b/core/src/test/java/org/elasticsearch/index/analysis/FingerprintAnalyzerTests.java
index 0e4ed8f4fb7..8c1d530e448 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/FingerprintAnalyzerTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/FingerprintAnalyzerTests.java
@@ -43,12 +43,15 @@ public class FingerprintAnalyzerTests extends ESTokenStreamTestCase {
         Analyzer a = new FingerprintAnalyzer(CharArraySet.EMPTY_SET, ' ', 255, false);
         assertAnalyzesTo(a, "gödel escher bach",
             new String[]{"bach escher godel"});
+
+        assertAnalyzesTo(a, "gödel godel escher bach",
+            new String[]{"bach escher godel"});
     }
 
     public void testPreserveOriginal() throws Exception {
         Analyzer a = new FingerprintAnalyzer(CharArraySet.EMPTY_SET, ' ', 255, true);
         assertAnalyzesTo(a, "gödel escher bach",
-            new String[]{"bach escher godel", "bach escher gödel"});
+            new String[]{"bach escher godel gödel"});
     }
 
     public void testLimit() throws Exception {
@@ -65,4 +68,5 @@ public class FingerprintAnalyzerTests extends ESTokenStreamTestCase {
         assertAnalyzesTo(a, "b c a",
             new String[]{"a_b_c"});
     }
+
 }
diff --git a/docs/reference/analysis/analyzers/fingerprint-analyzer.asciidoc b/docs/reference/analysis/analyzers/fingerprint-analyzer.asciidoc
index a66495acdbe..b393c883441 100644
--- a/docs/reference/analysis/analyzers/fingerprint-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/fingerprint-analyzer.asciidoc
@@ -17,11 +17,11 @@ It consists of:
 Tokenizer::
 * <<analysis-standard-tokenizer,Standard Tokenizer>>
 
-Token Filters::
-* <<analysis-lowercase-tokenfilter,Lower Case Token Filter>>
-* <<analysis-stop-tokenfilter,Stop Token Filter>> (disabled by default)
-* <<analysis-fingerprint-tokenfilter>>
-* <<analysis-asciifolding-tokenfilter>>
+Token Filters (in order)::
+1. <<analysis-lowercase-tokenfilter,Lower Case Token Filter>>
+2. <<analysis-asciifolding-tokenfilter>>
+3. <<analysis-stop-tokenfilter,Stop Token Filter>> (disabled by default)
+4. <<analysis-fingerprint-tokenfilter>>
 
 [float]
 === Example output
@@ -68,7 +68,6 @@ The `fingerprint` analyzer accepts the following parameters:
 
     A pre-defined stop words list like `_english_` or an array  containing a
     list of stop words.  Defaults to `_none_`.
-
 `stopwords_path`::
 
     The path to a file containing stop words.

From 94317b1d333e7023b9c3416cbfae3fe08cc86087 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 12 May 2016 12:20:06 -0400
Subject: [PATCH 0210/1311] Add wait_for_health=yellow to reindex snippets

This should help the tests pass more consistently. Should.

This also removes from `?pretty` from the docs. It isn't a thing
with `// CONSOLE`.

Relates to
https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+master+multijob-os-compatibility/os=oraclelinux/399/console
---
 docs/reference/docs/reindex.asciidoc         | 16 ++++++++--------
 docs/reference/docs/update-by-query.asciidoc |  4 ++--
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index 00c5d34304e..49ee8dcd283 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -174,7 +174,7 @@ POST _reindex
 }
 --------------------------------------------------
 // CONSOLE
-// TEST[s/^/PUT twitter\nPUT blog\n/]
+// TEST[s/^/PUT twitter\nPUT blog\nGET _cluster\/health?wait_for_status=yellow\n/]
 
 It's also possible to limit the number of processed documents by setting
 `size`. This will only copy a single document from `twitter` to
@@ -299,7 +299,7 @@ POST _reindex
 }
 --------------------------------------------------
 // CONSOLE
-// TEST[s/^/PUT source\n/]
+// TEST[s/^/PUT source\nGET _cluster\/health?wait_for_status=yellow\n/]
 
 By default `_reindex` uses scroll batches of 100. You can change the
 batch size with the `size` field in the `source` element:
@@ -319,7 +319,7 @@ POST _reindex
 }
 --------------------------------------------------
 // CONSOLE
-// TEST[s/^/PUT source\n/]
+// TEST[s/^/PUT source\nGET _cluster\/health?wait_for_status=yellow\n/]
 
 Reindex can also use the <<ingest>> feature by specifying a
 `pipeline` like this:
@@ -338,7 +338,7 @@ POST _reindex
 }
 --------------------------------------------------
 // CONSOLE
-// TEST[s/^/PUT source\n/]
+// TEST[s/^/PUT source\nGET _cluster\/health?wait_for_status=yellow\n/]
 
 [float]
 === URL Parameters
@@ -435,7 +435,7 @@ While Reindex is running you can fetch their status using the
 
 [source,js]
 --------------------------------------------------
-GET _tasks/?pretty&detailed=true&actions=*reindex
+GET _tasks?detailed=true&actions=*reindex
 --------------------------------------------------
 // CONSOLE
 
@@ -534,7 +534,7 @@ create an index containing documents that look like this:
 
 [source,js]
 --------------------------------------------------
-POST test/test/1?refresh&pretty
+POST test/test/1?refresh
 {
   "text": "words words",
   "flag": "foo"
@@ -547,7 +547,7 @@ But you don't like the name `flag` and want to replace it with `tag`.
 
 [source,js]
 --------------------------------------------------
-POST _reindex?pretty
+POST _reindex
 {
   "source": {
     "index": "test"
@@ -567,7 +567,7 @@ Now you can get the new document:
 
 [source,js]
 --------------------------------------------------
-GET test2/test/1?pretty
+GET test2/test/1
 --------------------------------------------------
 // CONSOLE
 // TEST[continued]
diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc
index 2de592f59a1..f049e302b34 100644
--- a/docs/reference/docs/update-by-query.asciidoc
+++ b/docs/reference/docs/update-by-query.asciidoc
@@ -134,7 +134,7 @@ types at once, just like the search API:
 POST twitter,blog/tweet,post/_update_by_query
 --------------------------------------------------
 // CONSOLE
-// TEST[s/^/PUT twitter\nPUT blog\n/]
+// TEST[s/^/PUT twitter\nPUT blog\nGET _cluster\/health?wait_for_status=yellow\n/]
 
 If you provide `routing` then the routing is copied to the scroll query,
 limiting the process to the shards that match that routing value:
@@ -266,7 +266,7 @@ While Update By Query is running you can fetch their status using the
 
 [source,js]
 --------------------------------------------------
-GET _tasks/?pretty&detailed=true&action=*byquery
+GET _tasks?detailed=true&action=*byquery
 --------------------------------------------------
 // CONSOLE
 

From 0830bd4885b8e7c7aca03f7dbbd10dd1ebe2ebfe Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 12 May 2016 12:47:57 -0400
Subject: [PATCH 0211/1311] Remove period in min master node check log message

As most of our log messages are not sentences and do not end with
periods, this commit removes a period from the end of the min master
node bootstrap check log message.
---
 .../main/java/org/elasticsearch/bootstrap/BootstrapCheck.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
index 37a89531184..925940a3903 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
@@ -335,7 +335,7 @@ final class BootstrapCheck {
         @Override
         public String errorMessage() {
             return "please set [" + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey() +
-                "] to a majority of the number of master eligible nodes in your cluster.";
+                "] to a majority of the number of master eligible nodes in your cluster";
         }
 
         @Override

From 2e25837deb6bbfc02277257e191c6887e14ab6f7 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Thu, 12 May 2016 10:32:06 -0700
Subject: [PATCH 0212/1311] Made line number available in painless nodes for
 debugging.

---
 .../elasticsearch/painless/antlr/Walker.java  | 86 ++++++++++---------
 .../painless/node/AExpression.java            | 16 ++--
 .../elasticsearch/painless/node/ALink.java    |  4 +-
 .../elasticsearch/painless/node/ANode.java    |  8 +-
 .../painless/node/AStatement.java             |  4 +-
 .../elasticsearch/painless/node/EBinary.java  |  4 +-
 .../elasticsearch/painless/node/EBool.java    |  4 +-
 .../elasticsearch/painless/node/EBoolean.java |  4 +-
 .../elasticsearch/painless/node/ECast.java    |  4 +-
 .../elasticsearch/painless/node/EChain.java   | 20 ++---
 .../elasticsearch/painless/node/EComp.java    |  4 +-
 .../painless/node/EConditional.java           |  5 +-
 .../painless/node/EConstant.java              |  4 +-
 .../elasticsearch/painless/node/EDecimal.java |  4 +-
 .../painless/node/EExplicit.java              |  4 +-
 .../elasticsearch/painless/node/ENull.java    |  4 +-
 .../elasticsearch/painless/node/ENumeric.java |  4 +-
 .../elasticsearch/painless/node/EUnary.java   |  4 +-
 .../painless/node/LArrayLength.java           |  4 +-
 .../elasticsearch/painless/node/LBrace.java   | 10 +--
 .../elasticsearch/painless/node/LCall.java    |  6 +-
 .../elasticsearch/painless/node/LCast.java    |  4 +-
 .../painless/node/LDefArray.java              |  4 +-
 .../elasticsearch/painless/node/LDefCall.java |  4 +-
 .../painless/node/LDefField.java              |  4 +-
 .../elasticsearch/painless/node/LField.java   | 16 ++--
 .../painless/node/LListShortcut.java          |  4 +-
 .../painless/node/LMapShortcut.java           |  4 +-
 .../painless/node/LNewArray.java              |  4 +-
 .../elasticsearch/painless/node/LNewObj.java  |  4 +-
 .../painless/node/LShortcut.java              |  4 +-
 .../elasticsearch/painless/node/LString.java  |  4 +-
 .../painless/node/LVariable.java              |  4 +-
 .../elasticsearch/painless/node/SBlock.java   |  4 +-
 .../elasticsearch/painless/node/SBreak.java   |  4 +-
 .../painless/node/SContinue.java              |  4 +-
 .../painless/node/SDeclBlock.java             |  4 +-
 .../painless/node/SDeclaration.java           |  4 +-
 .../org/elasticsearch/painless/node/SDo.java  |  4 +-
 .../painless/node/SExpression.java            |  4 +-
 .../org/elasticsearch/painless/node/SFor.java |  4 +-
 .../elasticsearch/painless/node/SIfElse.java  |  5 +-
 .../elasticsearch/painless/node/SReturn.java  |  4 +-
 .../elasticsearch/painless/node/SSource.java  |  4 +-
 .../elasticsearch/painless/node/SThrow.java   |  4 +-
 .../elasticsearch/painless/node/STrap.java    |  4 +-
 .../org/elasticsearch/painless/node/STry.java |  4 +-
 .../elasticsearch/painless/node/SWhile.java   |  4 +-
 48 files changed, 170 insertions(+), 158 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index 71ada2fdf9b..4f6e2f5e87c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -146,6 +146,10 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
         return parser.source();
     }
 
+    private int line(final ParserRuleContext ctx) {
+        return ctx.getStart().getLine();
+    }
+
     private String location(final ParserRuleContext ctx) {
         return "[ " + ctx.getStart().getLine() + " : " + ctx.getStart().getCharPositionInLine() + " ]";
     }
@@ -158,7 +162,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
             statements.add((AStatement)visit(statement));
         }
 
-        return new SSource(location(ctx), statements);
+        return new SSource(line(ctx), location(ctx), statements);
     }
 
     @Override
@@ -167,7 +171,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
         final AStatement ifblock = (AStatement)visit(ctx.block(0));
         final AStatement elseblock = ctx.block(1) == null ? null : (AStatement)visit(ctx.block(1));
 
-        return new SIfElse(location(ctx), condition, ifblock, elseblock);
+        return new SIfElse(line(ctx), location(ctx), condition, ifblock, elseblock);
     }
 
     @Override
@@ -177,7 +181,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
 
         reserved.usesLoop();
 
-        return new SWhile(location(ctx), condition, block);
+        return new SWhile(line(ctx), location(ctx), condition, block);
     }
 
     @Override
@@ -187,7 +191,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
 
         reserved.usesLoop();
 
-        return new SDo(location(ctx), block, condition);
+        return new SDo(line(ctx), location(ctx), block, condition);
     }
 
     @Override
@@ -199,7 +203,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
 
         reserved.usesLoop();
 
-        return new SFor(location(ctx), intializer, condition, afterthought, block);
+        return new SFor(line(ctx), location(ctx), intializer, condition, afterthought, block);
     }
 
     @Override
@@ -209,19 +213,19 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
 
     @Override
     public ANode visitContinue(final ContinueContext ctx) {
-        return new SContinue(location(ctx));
+        return new SContinue(line(ctx), location(ctx));
     }
 
     @Override
     public ANode visitBreak(final BreakContext ctx) {
-        return new SBreak(location(ctx));
+        return new SBreak(line(ctx), location(ctx));
     }
 
     @Override
     public ANode visitReturn(final ReturnContext ctx) {
         final AExpression expression = (AExpression)visit(ctx.expression());
 
-        return new SReturn(location(ctx), expression);
+        return new SReturn(line(ctx), location(ctx), expression);
     }
 
     @Override
@@ -233,21 +237,21 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
             traps.add((STrap)visit(trap));
         }
 
-        return new STry(location(ctx), block, traps);
+        return new STry(line(ctx), location(ctx), block, traps);
     }
 
     @Override
     public ANode visitThrow(final ThrowContext ctx) {
         final AExpression expression = (AExpression)visit(ctx.expression());
 
-        return new SThrow(location(ctx), expression);
+        return new SThrow(line(ctx), location(ctx), expression);
     }
 
     @Override
     public ANode visitExpr(final ExprContext ctx) {
         final AExpression expression = (AExpression)visit(ctx.expression());
 
-        return new SExpression(location(ctx), expression);
+        return new SExpression(line(ctx), location(ctx), expression);
     }
 
     @Override
@@ -258,7 +262,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
             statements.add((AStatement)visit(statement));
         }
 
-        return new SBlock(location(ctx), statements);
+        return new SBlock(line(ctx), location(ctx), statements);
     }
 
     @Override
@@ -266,7 +270,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
         final List<AStatement> statements = new ArrayList<>();
         statements.add((AStatement)visit(ctx.statement()));
 
-        return new SBlock(location(ctx), statements);
+        return new SBlock(line(ctx), location(ctx), statements);
     }
 
     @Override
@@ -303,10 +307,10 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
         for (final DeclvarContext declvar : ctx.declvar()) {
             final String name = declvar.identifier().getText();
             final AExpression expression = declvar.expression() == null ? null : (AExpression)visit(declvar.expression());
-            declarations.add(new SDeclaration(location(ctx), type, name, expression));
+            declarations.add(new SDeclaration(line(ctx), location(ctx), type, name, expression));
         }
 
-        return new SDeclBlock(location(ctx), declarations);
+        return new SDeclBlock(line(ctx), location(ctx), declarations);
     }
 
     @Override
@@ -325,7 +329,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
         final String name = ctx.identifier(1).getText();
         final AStatement block = ctx.block() == null ? null : (AStatement)visit(ctx.block());
 
-        return new STrap(location(ctx), type, name, block);
+        return new STrap(line(ctx), location(ctx), type, name, block);
     }
 
     @Override
@@ -348,13 +352,13 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
         final boolean negate = ctx.parent instanceof UnaryContext && ((UnaryContext)ctx.parent).SUB() != null;
 
         if (ctx.DECIMAL() != null) {
-            return new EDecimal(location(ctx), (negate ? "-" : "") + ctx.DECIMAL().getText());
+            return new EDecimal(line(ctx), location(ctx), (negate ? "-" : "") + ctx.DECIMAL().getText());
         } else if (ctx.HEX() != null) {
-            return new ENumeric(location(ctx), (negate ? "-" : "") + ctx.HEX().getText().substring(2), 16);
+            return new ENumeric(line(ctx), location(ctx), (negate ? "-" : "") + ctx.HEX().getText().substring(2), 16);
         } else if (ctx.INTEGER() != null) {
-            return new ENumeric(location(ctx), (negate ? "-" : "") + ctx.INTEGER().getText(), 10);
+            return new ENumeric(line(ctx), location(ctx), (negate ? "-" : "") + ctx.INTEGER().getText(), 10);
         } else if (ctx.OCTAL() != null) {
-            return new ENumeric(location(ctx), (negate ? "-" : "") + ctx.OCTAL().getText().substring(1), 8);
+            return new ENumeric(line(ctx), location(ctx), (negate ? "-" : "") + ctx.OCTAL().getText().substring(1), 8);
         } else {
             throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
         }
@@ -362,17 +366,17 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
 
     @Override
     public ANode visitTrue(final TrueContext ctx) {
-        return new EBoolean(location(ctx), true);
+        return new EBoolean(line(ctx), location(ctx), true);
     }
 
     @Override
     public ANode visitFalse(FalseContext ctx) {
-        return new EBoolean(location(ctx), false);
+        return new EBoolean(line(ctx), location(ctx), false);
     }
 
     @Override
     public ANode visitNull(final NullContext ctx) {
-        return new ENull(location(ctx));
+        return new ENull(line(ctx), location(ctx));
     }
 
     @Override
@@ -390,7 +394,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
             throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
         }
 
-        return new EChain(location(ctx), links, false, true, operation, null);
+        return new EChain(line(ctx), location(ctx), links, false, true, operation, null);
     }
 
     @Override
@@ -408,7 +412,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
             throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
         }
 
-        return new EChain(location(ctx), links, true, false, operation, null);
+        return new EChain(line(ctx), location(ctx), links, true, false, operation, null);
     }
 
     @Override
@@ -417,7 +421,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
 
         visitChain(ctx.chain(), links);
 
-        return new EChain(location(ctx), links, false, false, null, null);
+        return new EChain(line(ctx), location(ctx), links, false, false, null, null);
     }
 
     @Override
@@ -439,13 +443,13 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
                 throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
             }
 
-            return new EUnary(location(ctx), operation, (AExpression)visit(ctx.expression()));
+            return new EUnary(line(ctx), location(ctx), operation, (AExpression)visit(ctx.expression()));
         }
     }
 
     @Override
     public ANode visitCast(final CastContext ctx) {
-        return new EExplicit(location(ctx), ctx.decltype().getText(), (AExpression)visit(ctx.expression()));
+        return new EExplicit(line(ctx), location(ctx), ctx.decltype().getText(), (AExpression)visit(ctx.expression()));
     }
 
     @Override
@@ -480,7 +484,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
             throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
         }
 
-        return new EBinary(location(ctx), operation, left, right);
+        return new EBinary(line(ctx), location(ctx), operation, left, right);
     }
 
     @Override
@@ -509,7 +513,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
             throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
         }
 
-        return new EComp(location(ctx), operation, left, right);
+        return new EComp(line(ctx), location(ctx), operation, left, right);
     }
 
     @Override
@@ -526,7 +530,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
             throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
         }
 
-        return new EBool(location(ctx), operation, left, right);
+        return new EBool(line(ctx), location(ctx), operation, left, right);
     }
 
 
@@ -536,7 +540,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
         final AExpression left = (AExpression)visit(ctx.expression(1));
         final AExpression right = (AExpression)visit(ctx.expression(2));
 
-        return new EConditional(location(ctx), condition, left, right);
+        return new EConditional(line(ctx), location(ctx), condition, left, right);
     }
 
     @Override
@@ -572,7 +576,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
             operation = null;
         }
 
-        return new EChain(location(ctx), links, false, false, operation, (AExpression)visit(ctx.expression()));
+        return new EChain(line(ctx), location(ctx), links, false, false, operation, (AExpression)visit(ctx.expression()));
     }
 
     private void visitChain(final ChainContext ctx, final List<ALink> links) {
@@ -638,7 +642,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
             throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
         }
 
-        links.add(new LCast(location(ctx), ctx.decltype().getText()));
+        links.add(new LCast(line(ctx), location(ctx), ctx.decltype().getText()));
     }
 
     @Override
@@ -647,7 +651,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
     }
 
     private void visitLinkbrace(final LinkbraceContext ctx, final List<ALink> links) {
-        links.add(new LBrace(location(ctx), (AExpression)visit(ctx.expression())));
+        links.add(new LBrace(line(ctx), location(ctx), (AExpression)visit(ctx.expression())));
 
         if (ctx.linkbrace() != null) {
             visitLinkbrace(ctx.linkbrace(), links);
@@ -681,7 +685,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
             arguments.add((AExpression)visit(expression));
         }
 
-        links.add(new LCall(location(ctx), ctx.EXTID().getText(), arguments));
+        links.add(new LCall(line(ctx), location(ctx), ctx.EXTID().getText(), arguments));
 
         if (ctx.linkbrace() != null) {
             visitLinkbrace(ctx.linkbrace(), links);
@@ -700,7 +704,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
 
         reserved.markReserved(name);
 
-        links.add(new LVariable(location(ctx), name));
+        links.add(new LVariable(line(ctx), location(ctx), name));
 
         if (ctx.linkbrace() != null) {
             visitLinkbrace(ctx.linkbrace(), links);
@@ -725,7 +729,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
             throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
         }
 
-        links.add(new LField(location(ctx), value));
+        links.add(new LField(line(ctx), location(ctx), value));
 
         if (ctx.linkbrace() != null) {
             visitLinkbrace(ctx.linkbrace(), links);
@@ -747,13 +751,13 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
                 arguments.add((AExpression)visit(expression));
             }
 
-            links.add(new LNewObj(location(ctx), ctx.identifier().getText(), arguments));
+            links.add(new LNewObj(line(ctx), location(ctx), ctx.identifier().getText(), arguments));
         } else if (ctx.expression().size() > 0) {
             for (final ExpressionContext expression : ctx.expression()) {
                 arguments.add((AExpression)visit(expression));
             }
 
-            links.add(new LNewArray(location(ctx), ctx.identifier().getText(), arguments));
+            links.add(new LNewArray(line(ctx), location(ctx), ctx.identifier().getText(), arguments));
         } else {
             throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
         }
@@ -769,7 +773,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
     }
 
     private void visitLinkstring(final LinkstringContext ctx, final List<ALink> links) {
-        links.add(new LString(location(ctx), ctx.STRING().getText().substring(1, ctx.STRING().getText().length() - 1)));
+        links.add(new LString(line(ctx), location(ctx), ctx.STRING().getText().substring(1, ctx.STRING().getText().length() - 1)));
 
         if (ctx.linkbrace() != null) {
             visitLinkbrace(ctx.linkbrace(), links);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
index 185968c0a09..6f2b861b70c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
@@ -94,8 +94,8 @@ public abstract class AExpression extends ANode {
      */
     protected Label fals = null;
 
-    public AExpression(final String location) {
-        super(location);
+    public AExpression(final int line, final String location) {
+        super(line, location);
     }
 
     /**
@@ -120,7 +120,7 @@ public abstract class AExpression extends ANode {
             if (constant == null || this instanceof EConstant) {
                 return this;
             } else {
-                final EConstant econstant = new EConstant(location, constant);
+                final EConstant econstant = new EConstant(line, location, constant);
                 econstant.analyze(settings, definition, variables);
 
                 if (!expected.equals(econstant.actual)) {
@@ -131,7 +131,7 @@ public abstract class AExpression extends ANode {
             }
         } else {
             if (constant == null) {
-                final ECast ecast = new ECast(location, this, cast);
+                final ECast ecast = new ECast(line, location, this, cast);
                 ecast.statement = statement;
                 ecast.actual = expected;
                 ecast.isNull = isNull;
@@ -141,7 +141,7 @@ public abstract class AExpression extends ANode {
                 if (expected.sort.constant) {
                     constant = AnalyzerCaster.constCast(location, constant, cast);
 
-                    final EConstant econstant = new EConstant(location, constant);
+                    final EConstant econstant = new EConstant(line, location, constant);
                     econstant.analyze(settings, definition, variables);
 
                     if (!expected.equals(econstant.actual)) {
@@ -150,19 +150,19 @@ public abstract class AExpression extends ANode {
 
                     return econstant;
                 } else if (this instanceof EConstant) {
-                    final ECast ecast = new ECast(location, this, cast);
+                    final ECast ecast = new ECast(line, location, this, cast);
                     ecast.actual = expected;
 
                     return ecast;
                 } else {
-                    final EConstant econstant = new EConstant(location, constant);
+                    final EConstant econstant = new EConstant(line, location, constant);
                     econstant.analyze(settings, definition, variables);
 
                     if (!actual.equals(econstant.actual)) {
                         throw new IllegalStateException(error("Illegal tree structure."));
                     }
 
-                    final ECast ecast = new ECast(location, econstant, cast);
+                    final ECast ecast = new ECast(line, location, econstant, cast);
                     ecast.actual = expected;
 
                     return ecast;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
index f5cccfedc7d..79cbdf4a194 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
@@ -75,8 +75,8 @@ public abstract class ALink extends ANode {
      */
     String string = null;
 
-    ALink(final String location, final int size) {
-        super(location);
+    ALink(final int line, final String location, final int size) {
+        super(line, location);
 
         this.size = size;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
index 8faaaff51b3..b4c4fdd677d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
@@ -24,12 +24,18 @@ package org.elasticsearch.painless.node;
  */
 public abstract class ANode {
 
+    /**
+     * The line number in the original source used for debug messages.
+     */
+    final int line;
+
     /**
      * The location in the original source to be printed in error messages.
      */
     final String location;
 
-    ANode(final String location) {
+    ANode(final int line, final String location) {
+        this.line = line;
         this.location = location;
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
index d6830adc9bd..c4366e1af36 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
@@ -109,8 +109,8 @@ public abstract class AStatement extends ANode {
      */
     Label brake = null;
 
-    AStatement(final String location) {
-        super(location);
+    AStatement(final int line, final String location) {
+        super(line, location);
     }
 
     /**
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index a3dd45e75c4..0df1f0f29de 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -40,8 +40,8 @@ public final class EBinary extends AExpression {
 
     boolean cat = false;
 
-    public EBinary(final String location, final Operation operation, final AExpression left, final AExpression right) {
-        super(location);
+    public EBinary(final int line, final String location, final Operation operation, final AExpression left, final AExpression right) {
+        super(line, location);
 
         this.operation = operation;
         this.left = left;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
index 650c4cfc97e..7f120894603 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
@@ -35,8 +35,8 @@ public final class EBool extends AExpression {
     AExpression left;
     AExpression right;
 
-    public EBool(final String location, final Operation operation, final AExpression left, final AExpression right) {
-        super(location);
+    public EBool(final int line, final String location, final Operation operation, final AExpression left, final AExpression right) {
+        super(line, location);
 
         this.operation = operation;
         this.left = left;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
index 84bf38ea459..a5fb9c7f9e2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
@@ -29,8 +29,8 @@ import org.objectweb.asm.commons.GeneratorAdapter;
  */
 public final class EBoolean extends AExpression {
 
-    public EBoolean(final String location, final boolean constant) {
-        super(location);
+    public EBoolean(final int line, final String location, final boolean constant) {
+        super(line, location);
 
         this.constant = constant;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
index 77d39b92081..10622e758fc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
@@ -37,8 +37,8 @@ final class ECast extends AExpression {
 
     Cast cast = null;
 
-    ECast(final String location, final AExpression child, final Cast cast) {
-        super(location);
+    ECast(final int line, final String location, final AExpression child, final Cast cast) {
+        super(line, location);
 
         this.type = null;
         this.child = child;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index af3ba4fc422..bc3c326f651 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -49,9 +49,9 @@ public final class EChain extends AExpression {
     Cast there = null;
     Cast back = null;
 
-    public EChain(final String location, final List<ALink> links,
+    public EChain(final int line, final String location, final List<ALink> links,
                   final boolean pre, final boolean post, final Operation operation, final AExpression expression) {
-        super(location);
+        super(line, location);
 
         this.links = links;
         this.pre = pre;
@@ -127,25 +127,25 @@ public final class EChain extends AExpression {
 
             if (operation == Operation.INCR) {
                 if (sort == Sort.DOUBLE) {
-                    expression = new EConstant(location, 1D);
+                    expression = new EConstant(line, location, 1D);
                 } else if (sort == Sort.FLOAT) {
-                    expression = new EConstant(location, 1F);
+                    expression = new EConstant(line, location, 1F);
                 } else if (sort == Sort.LONG) {
-                    expression = new EConstant(location, 1L);
+                    expression = new EConstant(line, location, 1L);
                 } else {
-                    expression = new EConstant(location, 1);
+                    expression = new EConstant(line, location, 1);
                 }
 
                 operation = Operation.ADD;
             } else if (operation == Operation.DECR) {
                 if (sort == Sort.DOUBLE) {
-                    expression = new EConstant(location, 1D);
+                    expression = new EConstant(line, location, 1D);
                 } else if (sort == Sort.FLOAT) {
-                    expression = new EConstant(location, 1F);
+                    expression = new EConstant(line, location, 1F);
                 } else if (sort == Sort.LONG) {
-                    expression = new EConstant(location, 1L);
+                    expression = new EConstant(line, location, 1L);
                 } else {
-                    expression = new EConstant(location, 1);
+                    expression = new EConstant(line, location, 1);
                 }
 
                 operation = Operation.SUB;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index 0fd14237ef5..c7f85eb57c7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -45,8 +45,8 @@ public final class EComp extends AExpression {
     AExpression left;
     AExpression right;
 
-    public EComp(final String location, final Operation operation, final AExpression left, final AExpression right) {
-        super(location);
+    public EComp(final int line, final String location, final Operation operation, final AExpression left, final AExpression right) {
+        super(line, location);
 
         this.operation = operation;
         this.left = left;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
index 0006e5f8b97..1f751f9f7a3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
@@ -36,8 +36,9 @@ public final class EConditional extends AExpression {
     AExpression left;
     AExpression right;
 
-    public EConditional(final String location, final AExpression condition, final AExpression left, final AExpression right) {
-        super(location);
+    public EConditional(final int line, final String location,
+                        final AExpression condition, final AExpression left, final AExpression right) {
+        super(line, location);
 
         this.condition = condition;
         this.left = left;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
index 5b1e17142d9..41d5add5a31 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
@@ -32,8 +32,8 @@ import org.objectweb.asm.commons.GeneratorAdapter;
  */
 final class EConstant extends AExpression {
 
-    EConstant(final String location, final Object constant) {
-        super(location);
+    EConstant(final int line, final String location, final Object constant) {
+        super(line, location);
 
         this.constant = constant;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
index c2fc010aa51..629c3cc0aa6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
@@ -31,8 +31,8 @@ public final class EDecimal extends AExpression {
 
     final String value;
 
-    public EDecimal(final String location, final String value) {
-        super(location);
+    public EDecimal(final int line, final String location, final String value) {
+        super(line, location);
 
         this.value = value;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
index 0a3a39df7af..6dc3640a6d5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
@@ -35,8 +35,8 @@ public final class EExplicit extends AExpression {
 
     Cast cast = null;
 
-    public EExplicit(final String location, final String type, final AExpression child) {
-        super(location);
+    public EExplicit(final int line, final String location, final String type, final AExpression child) {
+        super(line, location);
 
         this.type = type;
         this.child = child;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
index 189b12d9f97..6e953b50c2e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
@@ -30,8 +30,8 @@ import org.objectweb.asm.commons.GeneratorAdapter;
  */
 public final class ENull extends AExpression {
 
-    public ENull(final String location) {
-        super(location);
+    public ENull(final int line, final String location) {
+        super(line, location);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
index 32260483b64..36ddf7f7242 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
@@ -33,8 +33,8 @@ public final class ENumeric extends AExpression {
     final String value;
     int radix;
 
-    public ENumeric(final String location, final String value, final int radix) {
-        super(location);
+    public ENumeric(final int line, final String location, final String value, final int radix) {
+        super(line, location);
 
         this.value = value;
         this.radix = radix;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index 7f6b52be89c..bc0929abdc2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -43,8 +43,8 @@ public final class EUnary extends AExpression {
     Operation operation;
     AExpression child;
 
-    public EUnary(final String location, final Operation operation, final AExpression child) {
-        super(location);
+    public EUnary(final int line, final String location, final Operation operation, final AExpression child) {
+        super(line, location);
 
         this.operation = operation;
         this.child = child;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
index b9cd0437992..34ad36480de 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
@@ -31,8 +31,8 @@ public final class LArrayLength extends ALink {
 
     final String value;
 
-    LArrayLength(final String location, final String value) {
-        super(location, -1);
+    LArrayLength(final int line, final String location, final String value) {
+        super(line, location, -1);
 
         this.value = value;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
index ff7c7fabc16..5cfaeb7466a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
@@ -35,8 +35,8 @@ public final class LBrace extends ALink {
 
     AExpression index;
 
-    public LBrace(final String location, final AExpression index) {
-        super(location, 2);
+    public LBrace(final int line, final String location, final AExpression index) {
+        super(line, location, 2);
 
         this.index = index;
     }
@@ -58,12 +58,12 @@ public final class LBrace extends ALink {
 
             return this;
         } else if (sort == Sort.DEF) {
-            return new LDefArray(location, index).copy(this).analyze(settings, definition, variables);
+            return new LDefArray(line, location, index).copy(this).analyze(settings, definition, variables);
         } else {
             try {
                 before.clazz.asSubclass(Map.class);
 
-                return new LMapShortcut(location, index).copy(this).analyze(settings, definition, variables);
+                return new LMapShortcut(line, location, index).copy(this).analyze(settings, definition, variables);
             } catch (final ClassCastException exception) {
                 // Do nothing.
             }
@@ -71,7 +71,7 @@ public final class LBrace extends ALink {
             try {
                 before.clazz.asSubclass(List.class);
 
-                return new LListShortcut(location, index).copy(this).analyze(settings, definition, variables);
+                return new LListShortcut(line, location, index).copy(this).analyze(settings, definition, variables);
             } catch (final ClassCastException exception) {
                 // Do nothing.
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
index 0d957a8b551..acb6646614a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
@@ -38,8 +38,8 @@ public final class LCall extends ALink {
 
     Method method = null;
 
-    public LCall(final String location, final String name, final List<AExpression> arguments) {
-        super(location, -1);
+    public LCall(final int line, final String location, final String name, final List<AExpression> arguments) {
+        super(line, location, -1);
 
         this.name = name;
         this.arguments = arguments;
@@ -80,7 +80,7 @@ public final class LCall extends ALink {
 
             return this;
         } else if (before.sort == Definition.Sort.DEF) {
-            final ALink link = new LDefCall(location, name, arguments);
+            final ALink link = new LDefCall(line, location, name, arguments);
             link.copy(this);
 
             return link.analyze(settings, definition, variables);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
index 52c23be993e..64b64fb9c13 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
@@ -36,8 +36,8 @@ public final class LCast extends ALink {
 
     Cast cast = null;
 
-    public LCast(final String location, final String type) {
-        super(location, -1);
+    public LCast(final int line, final String location, final String type) {
+        super(line, location, -1);
 
         this.type = type;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index 6e0b60a9227..0fa2207bc9d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -36,8 +36,8 @@ final class LDefArray extends ALink {
 
     AExpression index;
 
-    LDefArray(final String location, final AExpression index) {
-        super(location, 0);
+    LDefArray(final int line, final String location, final AExpression index) {
+        super(line, location, 0);
 
         this.index = index;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index cda802e0138..dc3e3abdcc8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -37,8 +37,8 @@ final class LDefCall extends ALink {
     final String name;
     final List<AExpression> arguments;
 
-    LDefCall(final String location, final String name, final List<AExpression> arguments) {
-        super(location, -1);
+    LDefCall(final int line, final String location, final String name, final List<AExpression> arguments) {
+        super(line, location, -1);
 
         this.name = name;
         this.arguments = arguments;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index f198e2999bd..851c1e3f631 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -36,8 +36,8 @@ final class LDefField extends ALink {
 
     final String value;
 
-    LDefField(final String location, final String value) {
-        super(location, 1);
+    LDefField(final int line, final String location, final String value) {
+        super(line, location, 1);
 
         this.value = value;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
index 577b4123152..6e3aafc0516 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
@@ -39,8 +39,8 @@ public final class LField extends ALink {
 
     Field field;
 
-    public LField(final String location, final String value) {
-        super(location, 1);
+    public LField(final int line, final String location, final String value) {
+        super(line, location, 1);
 
         this.value = value;
     }
@@ -54,9 +54,9 @@ public final class LField extends ALink {
         final Sort sort = before.sort;
 
         if (sort == Sort.ARRAY) {
-            return new LArrayLength(location, value).copy(this).analyze(settings, definition, variables);
+            return new LArrayLength(line, location, value).copy(this).analyze(settings, definition, variables);
         } else if (sort == Sort.DEF) {
-            return new LDefField(location, value).copy(this).analyze(settings, definition, variables);
+            return new LDefField(line, location, value).copy(this).analyze(settings, definition, variables);
         }
 
         final Struct struct = before.struct;
@@ -77,15 +77,15 @@ public final class LField extends ALink {
                 struct.methods.containsKey("set" + Character.toUpperCase(value.charAt(0)) + value.substring(1));
 
             if (shortcut) {
-                return new LShortcut(location, value).copy(this).analyze(settings, definition, variables);
+                return new LShortcut(line, location, value).copy(this).analyze(settings, definition, variables);
             } else {
-                final EConstant index = new EConstant(location, value);
+                final EConstant index = new EConstant(line, location, value);
                 index.analyze(settings, definition, variables);
 
                 try {
                     before.clazz.asSubclass(Map.class);
 
-                    return new LMapShortcut(location, index).copy(this).analyze(settings, definition, variables);
+                    return new LMapShortcut(line, location, index).copy(this).analyze(settings, definition, variables);
                 } catch (final ClassCastException exception) {
                     // Do nothing.
                 }
@@ -93,7 +93,7 @@ public final class LField extends ALink {
                 try {
                     before.clazz.asSubclass(List.class);
 
-                    return new LListShortcut(location, index).copy(this).analyze(settings, definition, variables);
+                    return new LListShortcut(line, location, index).copy(this).analyze(settings, definition, variables);
                 } catch (final ClassCastException exception) {
                     // Do nothing.
                 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
index 05231f81b89..cf256a947f6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
@@ -36,8 +36,8 @@ final class LListShortcut extends ALink {
     Method getter;
     Method setter;
 
-    LListShortcut(final String location, final AExpression index) {
-        super(location, 2);
+    LListShortcut(final int line, final String location, final AExpression index) {
+        super(line, location, 2);
 
         this.index = index;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
index 871950ef592..34c581f382b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
@@ -36,8 +36,8 @@ final class LMapShortcut extends ALink {
     Method getter;
     Method setter;
 
-    LMapShortcut(final String location, final AExpression index) {
-        super(location, 2);
+    LMapShortcut(final int line, final String location, final AExpression index) {
+        super(line, location, 2);
 
         this.index = index;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
index beca6057022..db0a09185d9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
@@ -35,8 +35,8 @@ public final class LNewArray extends ALink {
     final String type;
     final List<AExpression> arguments;
 
-    public LNewArray(final String location, final String type, final List<AExpression> arguments) {
-        super(location, -1);
+    public LNewArray(final int line, final String location, final String type, final List<AExpression> arguments) {
+        super(line, location, -1);
 
         this.type = type;
         this.arguments = arguments;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
index c460571f208..e5251e4130a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
@@ -39,8 +39,8 @@ public final class LNewObj extends ALink {
 
     Constructor constructor;
 
-    public LNewObj(final String location, final String type, final List<AExpression> arguments) {
-        super(location, -1);
+    public LNewObj(final int line, final String location, final String type, final List<AExpression> arguments) {
+        super(line, location, -1);
 
         this.type = type;
         this.arguments = arguments;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
index df190896940..95e3bbe3a44 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
@@ -38,8 +38,8 @@ final class LShortcut extends ALink {
     Method getter = null;
     Method setter = null;
 
-    LShortcut(final String location, final String value) {
-        super(location, 1);
+    LShortcut(final int line, final String location, final String value) {
+        super(line, location, 1);
 
         this.value = value;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
index 70c554b327a..7291b4cf8df 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
@@ -29,8 +29,8 @@ import org.objectweb.asm.commons.GeneratorAdapter;
  */
 public final class LString extends ALink {
 
-    public LString(final String location, final String string) {
-        super(location, -1);
+    public LString(final int line, final String location, final String string) {
+        super(line, location, -1);
 
         this.string = string;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
index 73ead63b961..989fa9f9ef7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
@@ -36,8 +36,8 @@ public final class LVariable extends ALink {
 
     int slot;
 
-    public LVariable(final String location, final String name) {
-        super(location, 0);
+    public LVariable(final int line, final String location, final String name) {
+        super(line, location, 0);
 
         this.name = name;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
index f24a603ad61..abaa332c7d3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
@@ -34,8 +34,8 @@ public final class SBlock extends AStatement {
 
     final List<AStatement> statements;
 
-    public SBlock(final String location, final List<AStatement> statements) {
-        super(location);
+    public SBlock(final int line, final String location, final List<AStatement> statements) {
+        super(line, location);
 
         this.statements = Collections.unmodifiableList(statements);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
index 23fd8d0603c..7b589ec740e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
@@ -29,8 +29,8 @@ import org.objectweb.asm.commons.GeneratorAdapter;
  */
 public final class SBreak extends AStatement {
 
-    public SBreak(final String location) {
-        super(location);
+    public SBreak(final int line, final String location) {
+        super(line, location);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
index 3ed7fa8905a..3da0c653129 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
@@ -29,8 +29,8 @@ import org.objectweb.asm.commons.GeneratorAdapter;
  */
 public final class SContinue extends AStatement {
 
-    public SContinue(final String location) {
-        super(location);
+    public SContinue(final int line, final String location) {
+        super(line, location);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
index 875d4857c45..8511f087ef1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
@@ -34,8 +34,8 @@ public final class SDeclBlock extends AStatement {
 
     final List<SDeclaration> declarations;
 
-    public SDeclBlock(final String location, final List<SDeclaration> declarations) {
-        super(location);
+    public SDeclBlock(final int line, final String location, final List<SDeclaration> declarations) {
+        super(line, location);
 
         this.declarations = Collections.unmodifiableList(declarations);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
index cdcdce7acc8..14e240601ce 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
@@ -38,8 +38,8 @@ public final class SDeclaration extends AStatement {
 
     Variable variable;
 
-    public SDeclaration(final String location, final String type, final String name, final AExpression expression) {
-        super(location);
+    public SDeclaration(final int line, final String location, final String type, final String name, final AExpression expression) {
+        super(line, location);
 
         this.type = type;
         this.name = name;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
index 41b2acd5fcb..329d9133d40 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -34,8 +34,8 @@ public final class SDo extends AStatement {
     final AStatement block;
     AExpression condition;
 
-    public SDo(final String location, final AStatement block, final AExpression condition) {
-        super(location);
+    public SDo(final int line, final String location, final AStatement block, final AExpression condition) {
+        super(line, location);
 
         this.condition = condition;
         this.block = block;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index b9fd106e590..17460d09870 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -33,8 +33,8 @@ public final class SExpression extends AStatement {
 
     AExpression expression;
 
-    public SExpression(final String location, final AExpression expression) {
-        super(location);
+    public SExpression(final int line, final String location, final AExpression expression) {
+        super(line, location);
 
         this.expression = expression;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index 0c26db7c63b..64d171ca611 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -36,9 +36,9 @@ public final class SFor extends AStatement {
     AExpression afterthought;
     final AStatement block;
 
-    public SFor(final String location,
+    public SFor(final int line, final String location,
                 final ANode initializer, final AExpression condition, final AExpression afterthought, final AStatement block) {
-        super(location);
+        super(line, location);
 
         this.initializer = initializer;
         this.condition = condition;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
index 747730b1c53..3914b39ba7f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
@@ -34,8 +34,9 @@ public final class SIfElse extends AStatement {
     final AStatement ifblock;
     final AStatement elseblock;
 
-    public SIfElse(final String location, final AExpression condition, final AStatement ifblock, final AStatement elseblock) {
-        super(location);
+    public SIfElse(final int line, final String location,
+                   final AExpression condition, final AStatement ifblock, final AStatement elseblock) {
+        super(line, location);
 
         this.condition = condition;
         this.ifblock = ifblock;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
index c59156b1c4a..a89abe4c5bc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -31,8 +31,8 @@ public final class SReturn extends AStatement {
 
     AExpression expression;
 
-    public SReturn(final String location, final AExpression expression) {
-        super(location);
+    public SReturn(final int line, final String location, final AExpression expression) {
+        super(line, location);
 
         this.expression = expression;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index c00fc0c7fc4..b6e731854d0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -35,8 +35,8 @@ public final class SSource extends AStatement {
 
     final List<AStatement> statements;
 
-    public SSource(final String location, final List<AStatement> statements) {
-        super(location);
+    public SSource(final int line, final String location, final List<AStatement> statements) {
+        super(line, location);
 
         this.statements = Collections.unmodifiableList(statements);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
index 613b75e6332..b54caf1bce4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
@@ -31,8 +31,8 @@ public final class SThrow extends AStatement {
 
     AExpression expression;
 
-    public SThrow(final String location, final AExpression expression) {
-        super(location);
+    public SThrow(final int line, final String location, final AExpression expression) {
+        super(line, location);
 
         this.expression = expression;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java
index 8608344ecd1..77f3c9de9d6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java
@@ -42,8 +42,8 @@ public final class STrap extends AStatement {
     Label end;
     Label exception;
 
-    public STrap(final String location, final String type, final String name, final AStatement block) {
-        super(location);
+    public STrap(final int line, final String location, final String type, final String name, final AStatement block) {
+        super(line, location);
 
         this.type = type;
         this.name = name;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
index 39aa68a0686..f9be4d75ad1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
@@ -36,8 +36,8 @@ public final class STry extends AStatement {
     final AStatement block;
     final List<STrap> traps;
 
-    public STry(final String location, final AStatement block, final List<STrap> traps) {
-        super(location);
+    public STry(final int line, final String location, final AStatement block, final List<STrap> traps) {
+        super(line, location);
 
         this.block = block;
         this.traps = Collections.unmodifiableList(traps);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
index 493ddfe0e13..29028bfe3b4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -34,8 +34,8 @@ public final class SWhile extends AStatement {
     AExpression condition;
     final AStatement block;
 
-    public SWhile(final String location, final AExpression condition, final AStatement block) {
-        super(location);
+    public SWhile(final int line, final String location, final AExpression condition, final AStatement block) {
+        super(line, location);
 
         this.condition = condition;
         this.block = block;

From 73c9cfbe8df200279b54ace45ea1e4d22b32a2e7 Mon Sep 17 00:00:00 2001
From: Shane Connelly <shane@eskibars.com>
Date: Thu, 12 May 2016 11:25:13 -0700
Subject: [PATCH 0213/1311] Adds CONSOLE tests to aliases documentation

---
 docs/reference/indices/aliases.asciidoc | 205 ++++++++++++++----------
 1 file changed, 117 insertions(+), 88 deletions(-)

diff --git a/docs/reference/indices/aliases.asciidoc b/docs/reference/indices/aliases.asciidoc
index ab4e49e7aab..2b19da01b76 100644
--- a/docs/reference/indices/aliases.asciidoc
+++ b/docs/reference/indices/aliases.asciidoc
@@ -14,25 +14,29 @@ Here is a sample of associating the alias `alias1` with index `test1`:
 
 [source,js]
 --------------------------------------------------
-curl -XPOST 'http://localhost:9200/_aliases' -d '
+POST /_aliases
 {
     "actions" : [
         { "add" : { "index" : "test1", "alias" : "alias1" } }
     ]
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT test1\n/]
 
 An alias can also be removed, for example:
 
 [source,js]
 --------------------------------------------------
-curl -XPOST 'http://localhost:9200/_aliases' -d '
+POST /_aliases
 {
     "actions" : [
         { "remove" : { "index" : "test1", "alias" : "alias1" } }
     ]
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 Renaming an alias is a simple `remove` then `add` operation within the
 same API. This operation is atomic, no need to worry about a short
@@ -40,40 +44,46 @@ period of time where the alias does not point to an index:
 
 [source,js]
 --------------------------------------------------
-curl -XPOST 'http://localhost:9200/_aliases' -d '
+POST /_aliases
 {
     "actions" : [
         { "remove" : { "index" : "test1", "alias" : "alias1" } },
         { "add" : { "index" : "test1", "alias" : "alias2" } }
     ]
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 Associating an alias with more than one index are simply several `add`
 actions:
 
 [source,js]
 --------------------------------------------------
-curl -XPOST 'http://localhost:9200/_aliases' -d '
+POST /_aliases
 {
     "actions" : [
         { "add" : { "index" : "test1", "alias" : "alias1" } },
         { "add" : { "index" : "test2", "alias" : "alias1" } }
     ]
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT test1\nPUT test2\n/]
 
 Multiple indices can be specified for an action with the `indices` array syntax:
 
 [source,js]
 --------------------------------------------------
-curl -XPOST 'http://localhost:9200/_aliases' -d '
+POST /_aliases
 {
     "actions" : [
         { "add" : { "indices" : ["test1", "test2"], "alias" : "alias1" } }
     ]
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT test1\nPUT test2\n/]
 
 To specify multiple aliases in one action, the corresponding `aliases` array
 syntax exists as well.
@@ -83,13 +93,15 @@ more than one index that share a common name:
 
 [source,js]
 --------------------------------------------------
-curl -XPOST 'http://localhost:9200/_aliases' -d '
+POST /_aliases
 {
     "actions" : [
         { "add" : { "index" : "test*", "alias" : "all_test_indices" } }
     ]
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT test1\nPUT test2\n/]
 
 In this case, the alias is a point-in-time alias that will group all
 current indices that match, it will not automatically update as new
@@ -111,7 +123,8 @@ exist in the mapping:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT 'http://localhost:9200/test1' -d '{
+PUT /test1
+{
   "mappings": {
     "type1": {
       "properties": {
@@ -123,12 +136,14 @@ curl -XPUT 'http://localhost:9200/test1' -d '{
   }
 }
 --------------------------------------------------
+// CONSOLE
 
 Now we can create an alias that uses a filter on field `user`:
 
 [source,js]
 --------------------------------------------------
-curl -XPOST 'http://localhost:9200/_aliases' -d '{
+POST /_aliases
+{
     "actions" : [
         {
             "add" : {
@@ -138,8 +153,10 @@ curl -XPOST 'http://localhost:9200/_aliases' -d '{
             }
         }
     ]
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 [float]
 [[aliases-routing]]
@@ -155,7 +172,7 @@ automatically modified to use value `1` for routing:
 
 [source,js]
 --------------------------------------------------
-curl -XPOST 'http://localhost:9200/_aliases' -d '
+POST /_aliases
 {
     "actions" : [
         {
@@ -166,15 +183,17 @@ curl -XPOST 'http://localhost:9200/_aliases' -d '
             }
         }
     ]
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT test\n/]
 
 It's also possible to specify different routing values for searching
 and indexing operations:
 
 [source,js]
 --------------------------------------------------
-curl -XPOST 'http://localhost:9200/_aliases' -d '
+POST /_aliases
 {
     "actions" : [
         {
@@ -186,8 +205,10 @@ curl -XPOST 'http://localhost:9200/_aliases' -d '
             }
         }
     ]
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT test\n/]
 
 As shown in the example above, search routing may contain several values
 separated by comma. Index routing can contain only a single value.
@@ -199,8 +220,10 @@ routing value:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'http://localhost:9200/alias2/_search?q=user:kimchy&routing=2,3'
+GET /alias2/_search?q=user:kimchy&routing=2,3
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 If an index operation that uses index routing alias also has a parent routing, the
 parent routing is ignored.
@@ -232,8 +255,10 @@ Adding time based alias::
 --
 [source,js]
 --------------------------------------------------
-curl -XPUT 'localhost:9200/logs_201305/_alias/2013'
+PUT /logs_201305/_alias/2013
 --------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT logs_201305\n/]
 --
 
 Adding a user alias::
@@ -243,7 +268,8 @@ First create the index and add a mapping for the `user_id` field:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT 'localhost:9200/users' -d '{
+PUT /users
+{
     "mappings" : {
         "user" : {
             "properties" : {
@@ -251,22 +277,26 @@ curl -XPUT 'localhost:9200/users' -d '{
             }
         }
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
 Then add the alias for a specific user:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT 'localhost:9200/users/_alias/user_12' -d '{
+PUT /users/_alias/user_12
+{
     "routing" : "12",
     "filter" : {
         "term" : {
             "user_id" : 12
         }
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 --
 
@@ -278,7 +308,8 @@ Aliases can also be specified during <<create-index-aliases,index creation>>:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT localhost:9200/logs_20142801 -d '{
+PUT /logs_20162801
+{
     "mappings" : {
         "type" : {
             "properties" : {
@@ -288,14 +319,15 @@ curl -XPUT localhost:9200/logs_20142801 -d '{
     },
     "aliases" : {
         "current_day" : {},
-        "2014" : {
+        "2016" : {
             "filter" : {
-                "term" : {"year" : 2014 }
+                "term" : {"year" : 2016 }
             }
         }
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
 [float]
 [[deleting]]
@@ -314,8 +346,10 @@ Alternatively you can use the plural `_aliases`. Example:
 
 [source,js]
 --------------------------------------------------
-curl -XDELETE 'localhost:9200/users/_alias/user_12'
+DELETE /logs_20162801/_alias/current_day
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 [float]
 [[alias-retrieving]]
@@ -351,88 +385,81 @@ All aliases for the index users:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/users/_alias/*'
+GET /logs_20162801/_alias/*
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 Response:
 
 [source,js]
 --------------------------------------------------
- {
-  "users" : {
+{
+ "logs_20162801" : {
+   "aliases" : {
+     "2016" : {
+       "filter" : {
+         "term" : {
+           "year" : 2016
+         }
+       }
+     }
+   }
+ }
+}
+--------------------------------------------------
+
+All aliases with the name 2016 in any index:
+
+[source,js]
+--------------------------------------------------
+GET /_alias/2016
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+Response:
+
+[source,js]
+--------------------------------------------------
+{
+  "logs_20162801" : {
     "aliases" : {
-      "user_13" : {
+      "2016" : {
         "filter" : {
           "term" : {
-            "user_id" : 13
+            "year" : 2016
           }
-        },
-        "index_routing" : "13",
-        "search_routing" : "13"
-      },
-      "user_14" : {
-        "filter" : {
-          "term" : {
-            "user_id" : 14
-          }
-        },
-        "index_routing" : "14",
-        "search_routing" : "14"
-      },
-      "user_12" : {
-        "filter" : {
-          "term" : {
-            "user_id" : 12
-          }
-        },
-        "index_routing" : "12",
-        "search_routing" : "12"
+        }
       }
     }
   }
 }
 --------------------------------------------------
 
-All aliases with the name 2013 in any index:
+All aliases that start with 20 in any index:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/_alias/2013'
+GET /_alias/20*
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 Response:
 
 [source,js]
 --------------------------------------------------
 {
-  "logs_201304" : {
+  "logs_20162801" : {
     "aliases" : {
-      "2013" : { }
-    }
-  },
-  "logs_201305" : {
-    "aliases" : {
-      "2013" : { }
-    }
-  }
-}
---------------------------------------------------
-
-All aliases that start with 2013_01 in any index:
-
-[source,js]
---------------------------------------------------
-curl -XGET 'localhost:9200/_alias/2013_01*'
---------------------------------------------------
-
-Response:
-
-[source,js]
---------------------------------------------------
-{
-  "logs_20130101" : {
-    "aliases" : {
-      "2013_01" : { }
+      "2016" : {
+        "filter" : {
+          "term" : {
+            "year" : 2016
+          }
+        }
+      }
     }
   }
 }
@@ -444,7 +471,9 @@ option as the get indices aliases api. Examples:
 
 [source,js]
 --------------------------------------------------
-curl -XHEAD -i 'localhost:9200/_alias/2013'
-curl -XHEAD -i 'localhost:9200/_alias/2013_01*'
-curl -XHEAD -i 'localhost:9200/users/_alias/*'
+HEAD /_alias/2016
+HEAD /_alias/20*
+HEAD /logs_20162801/_alias/*
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]

From df43230844b5c16238103a2962c8b415377b30a2 Mon Sep 17 00:00:00 2001
From: Olivier Bourgain <olivierbourgain02@gmail.com>
Date: Thu, 12 May 2016 20:32:30 +0200
Subject: [PATCH 0214/1311] Add index name and uuid in
 IndexAlreadyExistsException default message

Relates #18274
---
 .../elasticsearch/indices/IndexAlreadyExistsException.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/indices/IndexAlreadyExistsException.java b/core/src/main/java/org/elasticsearch/indices/IndexAlreadyExistsException.java
index 9d6616e4e2b..f64addfd2f1 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndexAlreadyExistsException.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndexAlreadyExistsException.java
@@ -32,7 +32,7 @@ import java.io.IOException;
 public class IndexAlreadyExistsException extends ElasticsearchException {
 
     public IndexAlreadyExistsException(Index index) {
-        this(index, "already exists");
+        this(index, "index " + index.toString() + " already exists");
     }
 
     public IndexAlreadyExistsException(Index index, String message) {
@@ -48,4 +48,4 @@ public class IndexAlreadyExistsException extends ElasticsearchException {
     public RestStatus status() {
         return RestStatus.BAD_REQUEST;
     }
-}
\ No newline at end of file
+}

From 10381de81587141b596766874215125c4c5077b5 Mon Sep 17 00:00:00 2001
From: Aaron Mildenstein <aaron@mildensteins.com>
Date: Thu, 12 May 2016 14:42:27 -0600
Subject: [PATCH 0215/1311] Removing the superfluous 's'

Pretty sure we're not making a brand new `/var/logs` directory when everything else goes into `/var/log`

Relates #18305
---
 docs/reference/setup/configuration.asciidoc | 4 ++--
 docs/reference/setup/install/deb.asciidoc   | 2 +-
 docs/reference/setup/install/rpm.asciidoc   | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/reference/setup/configuration.asciidoc b/docs/reference/setup/configuration.asciidoc
index 9c851733f6f..ceb3d8c38d3 100644
--- a/docs/reference/setup/configuration.asciidoc
+++ b/docs/reference/setup/configuration.asciidoc
@@ -39,7 +39,7 @@ example of changing the path of the data and logs directories:
 --------------------------------------------------
 path:
     data: /var/lib/elasticsearch
-    logs: /var/logs/elasticsearch
+    logs: /var/log/elasticsearch
 --------------------------------------------------
 
 Settings can also be flattened as follows:
@@ -47,7 +47,7 @@ Settings can also be flattened as follows:
 [source,yaml]
 --------------------------------------------------
 path.data: /var/lib/elasticsearch
-path.logs: /var/logs/elasticsearch
+path.logs: /var/log/elasticsearch
 --------------------------------------------------
 
 [float]
diff --git a/docs/reference/setup/install/deb.asciidoc b/docs/reference/setup/install/deb.asciidoc
index 9c7a043bdd5..0feba56b0b6 100644
--- a/docs/reference/setup/install/deb.asciidoc
+++ b/docs/reference/setup/install/deb.asciidoc
@@ -168,7 +168,7 @@ locations for a Debian-based system:
 
 | logs
   | Log files location.
-  | /var/logs/elasticsearch
+  | /var/log/elasticsearch
   | path.logs
 
 | plugins
diff --git a/docs/reference/setup/install/rpm.asciidoc b/docs/reference/setup/install/rpm.asciidoc
index c8bcf6b7b49..46a5c8739eb 100644
--- a/docs/reference/setup/install/rpm.asciidoc
+++ b/docs/reference/setup/install/rpm.asciidoc
@@ -154,7 +154,7 @@ locations for an RPM-based system:
 
 | logs
   | Log files location.
-  | /var/logs/elasticsearch
+  | /var/log/elasticsearch
   | path.logs
 
 | plugins

From 8155e1efda279c28435026fb5f0eb666060f0531 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 12 May 2016 17:53:34 -0400
Subject: [PATCH 0216/1311] [docs] Add wait_for_status=yellow

Another unstable snippet....

https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+master+multijob-os-compatibility/os=sles/402/console
---
 docs/reference/analysis/testing.asciidoc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/reference/analysis/testing.asciidoc b/docs/reference/analysis/testing.asciidoc
index a8c735f5027..ab50f6564bb 100644
--- a/docs/reference/analysis/testing.asciidoc
+++ b/docs/reference/analysis/testing.asciidoc
@@ -69,6 +69,8 @@ PUT my_index
   }
 }
 
+GET _cluster/health?wait_for_status=yellow
+
 GET my_index/_analyze <3>
 {
   "analyzer": "std_folded", <4>
@@ -88,4 +90,3 @@ GET my_index/_analyze <3>
 <3> To refer to this analyzer, the `analyze` API must specify the index name.
 <4> Refer to the analyzer by name.
 <5> Refer to the analyzer used by field `my_text`.
-

From f9667dbe3382b8ef641a461680ba5dfbd4bbd6a7 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 12 May 2016 18:13:04 -0400
Subject: [PATCH 0217/1311] Switch init.d scripts to use bash

This commit modifies the init.d scripts to use bash now that bash is a
required dependency.

Relates #18308
---
 distribution/deb/src/main/packaging/init.d/elasticsearch | 2 +-
 distribution/rpm/src/main/packaging/init.d/elasticsearch | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/distribution/deb/src/main/packaging/init.d/elasticsearch b/distribution/deb/src/main/packaging/init.d/elasticsearch
index 37bd0c3349e..82b1957b802 100755
--- a/distribution/deb/src/main/packaging/init.d/elasticsearch
+++ b/distribution/deb/src/main/packaging/init.d/elasticsearch
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 #
 # /etc/init.d/elasticsearch -- startup script for Elasticsearch
 #
diff --git a/distribution/rpm/src/main/packaging/init.d/elasticsearch b/distribution/rpm/src/main/packaging/init.d/elasticsearch
index 16c22141074..7bcb5692a88 100644
--- a/distribution/rpm/src/main/packaging/init.d/elasticsearch
+++ b/distribution/rpm/src/main/packaging/init.d/elasticsearch
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 #
 # elasticsearch <summary>
 #

From 52b09bc803baf4c938e876802dbe18445ca10561 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 12 May 2016 21:04:31 -0400
Subject: [PATCH 0218/1311] Increase default heap size to 2g

Benchmarking with the geonames data set shows that Elasticsearch truly
needs 2g of heap or the heap is a bottleneck for indexing rates. If our
goal is to satisfy the out-of-the-box-experience, we should prioritize
the out-of-the-box performance experience. Thus, the default heap should
be 2g until smaller heaps are not the bottleneck for indexing small data
sets.
---
 distribution/build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distribution/build.gradle b/distribution/build.gradle
index 44949cd0604..2483a9f625b 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -442,7 +442,7 @@ task run(type: RunTask) {}
  * </dl>
  */
 Map<String, String> expansionsForDistribution(distributionType) {
-  String heapSize = '512m'
+  String heapSize = '2g'
   String footer = "# Built for ${project.name}-${project.version} " +
       "(${distributionType})"
   Map<String, Object> expansions = [

From dd825c87305eb20727cd92268af5d4dfb3552d0d Mon Sep 17 00:00:00 2001
From: Shane Connelly <shane@eskibars.com>
Date: Thu, 12 May 2016 18:44:08 -0700
Subject: [PATCH 0219/1311] Add TESTRESPONSE code

---
 docs/reference/indices/aliases.asciidoc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/reference/indices/aliases.asciidoc b/docs/reference/indices/aliases.asciidoc
index 2b19da01b76..0a54be7cd8d 100644
--- a/docs/reference/indices/aliases.asciidoc
+++ b/docs/reference/indices/aliases.asciidoc
@@ -408,6 +408,7 @@ Response:
  }
 }
 --------------------------------------------------
+// TESTRESPONSE
 
 All aliases with the name 2016 in any index:
 
@@ -436,6 +437,7 @@ Response:
   }
 }
 --------------------------------------------------
+// TESTRESPONSE
 
 All aliases that start with 20 in any index:
 
@@ -464,6 +466,7 @@ Response:
   }
 }
 --------------------------------------------------
+// TESTRESPONSE
 
 There is also a HEAD variant of the get indices aliases api to check if
 index aliases exist. The indices aliases exists api supports the same

From 0d5a2f25d3e31dda79589d967541b810cd3e692a Mon Sep 17 00:00:00 2001
From: Britta Weber <britta.weber@elasticsearch.com>
Date: Fri, 13 May 2016 12:09:30 +0200
Subject: [PATCH 0220/1311] [TEST] muste test, we have an issue for it

https://github.com/elastic/elasticsearch/issues/18293
---
 .../org/elasticsearch/cluster/routing/DelayedAllocationIT.java   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java
index 2da75ee1115..eda6bca80f7 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java
@@ -108,6 +108,7 @@ public class DelayedAllocationIT extends ESIntegTestCase {
      * allocation to a very small value, it kicks the allocation of the unassigned shard
      * even though the node it was hosted on will not come back.
      */
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/18293")
     @TestLogging("_root:DEBUG,cluster.routing:TRACE")
     public void testDelayedAllocationChangeWithSettingTo100ms() throws Exception {
         internalCluster().startNodesAsync(3).get();

From bc46517f7e941bd09336652338bce6396769b6f8 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 13 May 2016 07:07:56 -0400
Subject: [PATCH 0221/1311] Restore unequal min and max default heap

After considerable discussion, we have elected to set the default min
heap to 256m and the default max heap to 2g. This is to balance the
desire for a good out-of-the-box performance experience (default max
heap of 2g) with a good out-of-the-box experience running on machines
with limited resources or running multiple instances on a single modern
developer laptop (default min heap of 1g).
---
 distribution/build.gradle | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/distribution/build.gradle b/distribution/build.gradle
index 2483a9f625b..62eaf70dca5 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -442,7 +442,6 @@ task run(type: RunTask) {}
  * </dl>
  */
 Map<String, String> expansionsForDistribution(distributionType) {
-  String heapSize = '2g'
   String footer = "# Built for ${project.name}-${project.version} " +
       "(${distributionType})"
   Map<String, Object> expansions = [
@@ -462,8 +461,8 @@ Map<String, String> expansionsForDistribution(distributionType) {
       'def': '',
     ],
 
-    'heap.min': "${heapSize}",
-    'heap.max': "${heapSize}",
+    'heap.min': "256m",
+    'heap.max': "2g",
 
     'stopping.timeout': [
       'rpm': 86400,

From d3efe378149bf534589b040696b2cc9224c3d29b Mon Sep 17 00:00:00 2001
From: Britta Weber <britta.weber@elasticsearch.com>
Date: Fri, 13 May 2016 14:08:17 +0200
Subject: [PATCH 0222/1311] [TEST] mute test for now, we have an issue for it

https://github.com/elastic/elasticsearch/issues/18325
---
 .../indices/memory/breaker/CircuitBreakerServiceIT.java          | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
index 539d75fdc15..a125e3842e7 100644
--- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
@@ -214,6 +214,7 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase {
      * Test that a breaker correctly redistributes to a different breaker, in
      * this case, the fielddata breaker borrows space from the request breaker
      */
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/18325")
     public void testParentChecking() throws Exception {
         if (noopBreakerUsed()) {
             logger.info("--> noop breakers used, skipping test");

From ddbfda2c68e6be7067bc8d9f779c3e75289eb99c Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@elastic.co>
Date: Fri, 13 May 2016 14:21:24 +0200
Subject: [PATCH 0223/1311] Exclude specific transport actions from request
 size limit check

We add support to explicitly exclude specific transport actions
from the request size limit check.

We also exclude the following request types currently:

*MasterPingRequest
* PingRequest
---
 .../health/TransportClusterHealthAction.java  |   3 +-
 .../state/TransportClusterStateAction.java    |   2 +-
 .../support/HandledTransportAction.java       |  13 +-
 .../master/TransportMasterNodeAction.java     |  10 +-
 .../master/TransportMasterNodeReadAction.java |   9 +-
 .../support/nodes/TransportNodesAction.java   |   3 +-
 .../TransportReplicationAction.java           |   7 +-
 .../discovery/zen/ZenDiscovery.java           |   3 +-
 .../zen/fd/MasterFaultDetection.java          |   3 +-
 .../discovery/zen/fd/NodesFaultDetection.java |   3 +-
 .../transport/RequestHandlerRegistry.java     |   9 +-
 .../transport/TransportService.java           |  18 +--
 .../transport/local/LocalTransport.java       |   8 +-
 .../netty/MessageChannelHandler.java          |  11 +-
 .../discovery/ZenFaultDetectionTests.java     | 119 ++++++++++++++++--
 .../messy/tests/IndicesRequestTests.java      |   4 +-
 16 files changed, 188 insertions(+), 37 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
index 853e6986d02..204eab7aaf4 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
@@ -54,7 +54,8 @@ public class TransportClusterHealthAction extends TransportMasterNodeReadAction<
     public TransportClusterHealthAction(Settings settings, TransportService transportService, ClusterService clusterService,
                                         ThreadPool threadPool, ClusterName clusterName, ActionFilters actionFilters,
                                         IndexNameExpressionResolver indexNameExpressionResolver, GatewayAllocator gatewayAllocator) {
-        super(settings, ClusterHealthAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, ClusterHealthRequest::new);
+        super(settings, ClusterHealthAction.NAME, false, transportService, clusterService, threadPool, actionFilters,
+            indexNameExpressionResolver, ClusterHealthRequest::new);
         this.clusterName = clusterName;
         this.gatewayAllocator = gatewayAllocator;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java
index 4bd826237c5..77a42108280 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java
@@ -47,7 +47,7 @@ public class TransportClusterStateAction extends TransportMasterNodeReadAction<C
     @Inject
     public TransportClusterStateAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
                                        ClusterName clusterName, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, ClusterStateAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, ClusterStateRequest::new);
+        super(settings, ClusterStateAction.NAME, false, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, ClusterStateRequest::new);
         this.clusterName = clusterName;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java b/core/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java
index 337d98fce68..997ed69f3fa 100644
--- a/core/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java
@@ -36,9 +36,18 @@ import java.util.function.Supplier;
  */
 public abstract class HandledTransportAction<Request extends ActionRequest<Request>, Response extends ActionResponse>
         extends TransportAction<Request, Response> {
-    protected HandledTransportAction(Settings settings, String actionName, ThreadPool threadPool, TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Supplier<Request> request) {
+    protected HandledTransportAction(Settings settings, String actionName, ThreadPool threadPool, TransportService transportService,
+                                     ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                     Supplier<Request> request) {
+        this(settings, actionName, true, threadPool, transportService, actionFilters, indexNameExpressionResolver, request);
+    }
+
+    protected HandledTransportAction(Settings settings, String actionName, boolean canTripCircuitBreaker, ThreadPool threadPool,
+                                     TransportService transportService, ActionFilters actionFilters,
+                                     IndexNameExpressionResolver indexNameExpressionResolver, Supplier<Request> request) {
         super(settings, actionName, threadPool, actionFilters, indexNameExpressionResolver, transportService.getTaskManager());
-        transportService.registerRequestHandler(actionName, request, ThreadPool.Names.SAME, new TransportHandler());
+        transportService.registerRequestHandler(actionName, request, ThreadPool.Names.SAME, false, canTripCircuitBreaker,
+            new TransportHandler());
     }
 
     class TransportHandler implements TransportRequestHandler<Request> {
diff --git a/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java b/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
index ee082610d93..f0ea4ea14f2 100644
--- a/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
@@ -58,7 +58,15 @@ public abstract class TransportMasterNodeAction<Request extends MasterNodeReques
     protected TransportMasterNodeAction(Settings settings, String actionName, TransportService transportService,
                                         ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters,
                                         IndexNameExpressionResolver indexNameExpressionResolver, Supplier<Request> request) {
-        super(settings, actionName, threadPool, transportService, actionFilters, indexNameExpressionResolver, request);
+        this(settings, actionName, true, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, request);
+    }
+
+    protected TransportMasterNodeAction(Settings settings, String actionName, boolean canTripCircuitBreaker,
+                                        TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                                        ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                        Supplier<Request> request) {
+        super(settings, actionName, canTripCircuitBreaker, threadPool, transportService, actionFilters, indexNameExpressionResolver,
+            request);
         this.transportService = transportService;
         this.clusterService = clusterService;
         this.executor = executor();
diff --git a/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeReadAction.java b/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeReadAction.java
index 5a6ddcfb34e..ceb8e1a40ac 100644
--- a/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeReadAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeReadAction.java
@@ -46,7 +46,14 @@ public abstract class TransportMasterNodeReadAction<Request extends MasterNodeRe
     protected TransportMasterNodeReadAction(Settings settings, String actionName, TransportService transportService,
                                             ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters,
                                             IndexNameExpressionResolver indexNameExpressionResolver, Supplier<Request> request) {
-        super(settings, actionName, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver,request);
+        this(settings, actionName, true, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver,request);
+    }
+
+    protected TransportMasterNodeReadAction(Settings settings, String actionName, boolean checkSizeLimit, TransportService transportService,
+                                            ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters,
+                                            IndexNameExpressionResolver indexNameExpressionResolver, Supplier<Request> request) {
+        super(settings, actionName, checkSizeLimit, transportService, clusterService, threadPool, actionFilters,
+            indexNameExpressionResolver,request);
         this.forceLocal = FORCE_LOCAL_SETTING.get(settings);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
index 2767bc80bf3..39d6bf767e1 100644
--- a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
@@ -80,7 +80,8 @@ public abstract class TransportNodesAction<NodesRequest extends BaseNodesRequest
 
         this.transportNodeAction = actionName + "[n]";
 
-        transportService.registerRequestHandler(transportNodeAction, nodeRequest, nodeExecutor, new NodeTransportHandler());
+        transportService.registerRequestHandler(
+            transportNodeAction, nodeRequest, nodeExecutor, new NodeTransportHandler());
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
index 2cc647ddd5a..8a721dfe508 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
@@ -92,8 +92,9 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
     final private WriteConsistencyLevel defaultWriteConsistencyLevel;
     final private TransportRequestOptions transportOptions;
 
-    final private String transportReplicaAction;
-    final private String transportPrimaryAction;
+    // package private for testing
+    final String transportReplicaAction;
+    final String transportPrimaryAction;
     final private ReplicasProxy replicasProxy;
 
     protected TransportReplicationAction(Settings settings, String actionName, TransportService transportService,
@@ -113,7 +114,7 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
         transportService.registerRequestHandler(actionName, request, ThreadPool.Names.SAME, new OperationTransportHandler());
         transportService.registerRequestHandler(transportPrimaryAction, request, executor, new PrimaryOperationTransportHandler());
         // we must never reject on because of thread pool capacity on replicas
-        transportService.registerRequestHandler(transportReplicaAction, replicaRequest, executor, true,
+        transportService.registerRequestHandler(transportReplicaAction, replicaRequest, executor, true, true,
             new ReplicaOperationTransportHandler());
 
         this.transportOptions = transportOptions();
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
index 596e40abfba..0e36c72e854 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
@@ -201,7 +201,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
 
         this.joinThreadControl = new JoinThreadControl(threadPool);
 
-        transportService.registerRequestHandler(DISCOVERY_REJOIN_ACTION_NAME, RejoinClusterRequest::new, ThreadPool.Names.SAME, new RejoinClusterRequestHandler());
+        transportService.registerRequestHandler(
+            DISCOVERY_REJOIN_ACTION_NAME, RejoinClusterRequest::new, ThreadPool.Names.SAME, new RejoinClusterRequestHandler());
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
index 4e7ea5be3af..5f101cf3a1f 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
@@ -81,7 +81,8 @@ public class MasterFaultDetection extends FaultDetection {
 
         logger.debug("[master] uses ping_interval [{}], ping_timeout [{}], ping_retries [{}]", pingInterval, pingRetryTimeout, pingRetryCount);
 
-        transportService.registerRequestHandler(MASTER_PING_ACTION_NAME, MasterPingRequest::new, ThreadPool.Names.SAME, new MasterPingRequestHandler());
+        transportService.registerRequestHandler(
+            MASTER_PING_ACTION_NAME, MasterPingRequest::new, ThreadPool.Names.SAME, false, false, new MasterPingRequestHandler());
     }
 
     public DiscoveryNode masterNode() {
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java b/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java
index 7dbe50577ce..74fdfe62118 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java
@@ -72,7 +72,8 @@ public class NodesFaultDetection extends FaultDetection {
 
         logger.debug("[node  ] uses ping_interval [{}], ping_timeout [{}], ping_retries [{}]", pingInterval, pingRetryTimeout, pingRetryCount);
 
-        transportService.registerRequestHandler(PING_ACTION_NAME, PingRequest::new, ThreadPool.Names.SAME, new PingRequestHandler());
+        transportService.registerRequestHandler(
+            PING_ACTION_NAME, PingRequest::new, ThreadPool.Names.SAME, false, false, new PingRequestHandler());
     }
 
     public void setLocalNode(DiscoveryNode localNode) {
diff --git a/core/src/main/java/org/elasticsearch/transport/RequestHandlerRegistry.java b/core/src/main/java/org/elasticsearch/transport/RequestHandlerRegistry.java
index 4b816de52c3..9c783c57380 100644
--- a/core/src/main/java/org/elasticsearch/transport/RequestHandlerRegistry.java
+++ b/core/src/main/java/org/elasticsearch/transport/RequestHandlerRegistry.java
@@ -33,17 +33,20 @@ public class RequestHandlerRegistry<Request extends TransportRequest> {
     private final String action;
     private final TransportRequestHandler<Request> handler;
     private final boolean forceExecution;
+    private final boolean canTripCircuitBreaker;
     private final String executor;
     private final Supplier<Request> requestFactory;
     private final TaskManager taskManager;
 
     public RequestHandlerRegistry(String action, Supplier<Request> requestFactory, TaskManager taskManager,
-                                  TransportRequestHandler<Request> handler, String executor, boolean forceExecution) {
+                                  TransportRequestHandler<Request> handler, String executor, boolean forceExecution,
+                                  boolean canTripCircuitBreaker) {
         this.action = action;
         this.requestFactory = requestFactory;
         assert newRequest() != null;
         this.handler = handler;
         this.forceExecution = forceExecution;
+        this.canTripCircuitBreaker = canTripCircuitBreaker;
         this.executor = executor;
         this.taskManager = taskManager;
     }
@@ -77,6 +80,10 @@ public class RequestHandlerRegistry<Request extends TransportRequest> {
         return forceExecution;
     }
 
+    public boolean canTripCircuitBreaker() {
+        return canTripCircuitBreaker;
+    }
+
     public String getExecutor() {
         return executor;
     }
diff --git a/core/src/main/java/org/elasticsearch/transport/TransportService.java b/core/src/main/java/org/elasticsearch/transport/TransportService.java
index 820cdfc130a..f53d63740ff 100644
--- a/core/src/main/java/org/elasticsearch/transport/TransportService.java
+++ b/core/src/main/java/org/elasticsearch/transport/TransportService.java
@@ -580,23 +580,27 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
      */
     public <Request extends TransportRequest> void registerRequestHandler(String action, Supplier<Request> requestFactory, String executor,
                                                                           TransportRequestHandler<Request> handler) {
-        RequestHandlerRegistry<Request> reg = new RequestHandlerRegistry<>(action, requestFactory, taskManager, handler, executor, false);
+        RequestHandlerRegistry<Request> reg = new RequestHandlerRegistry<>(
+            action, requestFactory, taskManager, handler, executor, false, true);
         registerRequestHandler(reg);
     }
 
     /**
      * Registers a new request handler
      *
-     * @param action         The action the request handler is associated with
-     * @param request        The request class that will be used to constrcut new instances for streaming
-     * @param executor       The executor the request handling will be executed on
-     * @param forceExecution Force execution on the executor queue and never reject it
-     * @param handler        The handler itself that implements the request handling
+     * @param action                The action the request handler is associated with
+     * @param request               The request class that will be used to constrcut new instances for streaming
+     * @param executor              The executor the request handling will be executed on
+     * @param forceExecution        Force execution on the executor queue and never reject it
+     * @param canTripCircuitBreaker Check the request size and raise an exception in case the limit is breached.
+     * @param handler               The handler itself that implements the request handling
      */
     public <Request extends TransportRequest> void registerRequestHandler(String action, Supplier<Request> request,
                                                                           String executor, boolean forceExecution,
+                                                                          boolean canTripCircuitBreaker,
                                                                           TransportRequestHandler<Request> handler) {
-        RequestHandlerRegistry<Request> reg = new RequestHandlerRegistry<>(action, request, taskManager, handler, executor, forceExecution);
+        RequestHandlerRegistry<Request> reg = new RequestHandlerRegistry<>(
+            action, request, taskManager, handler, executor, forceExecution, canTripCircuitBreaker);
         registerRequestHandler(reg);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java b/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
index 1c224e1aeef..d1ac749e4ac 100644
--- a/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
@@ -297,12 +297,16 @@ public class LocalTransport extends AbstractLifecycleComponent<Transport> implem
                                Version version) throws Exception {
         stream = new NamedWriteableAwareStreamInput(stream, namedWriteableRegistry);
         final String action = stream.readString();
+        final RequestHandlerRegistry reg = transportServiceAdapter.getRequestHandler(action);
         transportServiceAdapter.onRequestReceived(requestId, action);
-        inFlightRequestsBreaker().addEstimateBytesAndMaybeBreak(messageLengthBytes, "<transport_request>");
+        if (reg != null && reg.canTripCircuitBreaker()) {
+            inFlightRequestsBreaker().addEstimateBytesAndMaybeBreak(messageLengthBytes, "<transport_request>");
+        } else {
+            inFlightRequestsBreaker().addWithoutBreaking(messageLengthBytes);
+        }
         final LocalTransportChannel transportChannel = new LocalTransportChannel(this, transportServiceAdapter, sourceTransport, action,
             requestId, version, messageLengthBytes);
         try {
-            final RequestHandlerRegistry reg = transportServiceAdapter.getRequestHandler(action);
             if (reg == null) {
                 throw new ActionNotFoundTransportException("Action [" + action + "] not found");
             }
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java b/core/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java
index 0cd47b093ad..e45635e3349 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java
@@ -226,14 +226,17 @@ public class MessageChannelHandler extends SimpleChannelUpstreamHandler {
         transportServiceAdapter.onRequestReceived(requestId, action);
         NettyTransportChannel transportChannel = null;
         try {
-            transport.inFlightRequestsBreaker().addEstimateBytesAndMaybeBreak(messageLengthBytes, "<transport_request>");
-            transportChannel = new NettyTransportChannel(transport, transportServiceAdapter, action, channel,
-                requestId, version, profileName, messageLengthBytes);
-
             final RequestHandlerRegistry reg = transportServiceAdapter.getRequestHandler(action);
             if (reg == null) {
                 throw new ActionNotFoundTransportException(action);
             }
+            if (reg.canTripCircuitBreaker()) {
+                transport.inFlightRequestsBreaker().addEstimateBytesAndMaybeBreak(messageLengthBytes, "<transport_request>");
+            } else {
+                transport.inFlightRequestsBreaker().addWithoutBreaking(messageLengthBytes);
+            }
+            transportChannel = new NettyTransportChannel(transport, transportServiceAdapter, action, channel,
+                requestId, version, profileName, messageLengthBytes);
             final TransportRequest request = reg.newRequest();
             request.remoteAddress(new InetSocketTransportAddress((InetSocketAddress) channel.getRemoteAddress()));
             request.readFrom(buffer);
diff --git a/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java b/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
index e5199048c35..1108e8335e9 100644
--- a/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
@@ -20,40 +20,53 @@
 package org.elasticsearch.discovery;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.discovery.zen.fd.FaultDetection;
 import org.elasticsearch.discovery.zen.fd.MasterFaultDetection;
 import org.elasticsearch.discovery.zen.fd.NodesFaultDetection;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
-import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportConnectionListener;
+import org.elasticsearch.transport.TransportRequestOptions;
+import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.local.LocalTransport;
 import org.hamcrest.Matcher;
 import org.hamcrest.Matchers;
 import org.junit.After;
 import org.junit.Before;
 
+import java.util.Collections;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
+import static java.util.Collections.singleton;
 import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
 import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 public class ZenFaultDetectionTests extends ESTestCase {
     protected ThreadPool threadPool;
-    protected ClusterService clusterService;
+    protected ClusterService clusterServiceA;
+    protected ClusterService clusterServiceB;
     private CircuitBreakerService circuitBreakerService;
 
     protected static final Version version0 = Version.fromId(/*0*/99);
@@ -68,9 +81,14 @@ public class ZenFaultDetectionTests extends ESTestCase {
     @Before
     public void setUp() throws Exception {
         super.setUp();
+        Settings settings = Settings.builder()
+            .put(HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), new ByteSizeValue(0))
+            .build();
+        ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         threadPool = new ThreadPool(getClass().getName());
-        clusterService = createClusterService(threadPool);
-        circuitBreakerService = new NoneCircuitBreakerService();
+        clusterServiceA = createClusterService(threadPool);
+        clusterServiceB = createClusterService(threadPool);
+        circuitBreakerService = new HierarchyCircuitBreakerService(settings, clusterSettings);
         serviceA = build(Settings.builder().put("name", "TS_A").build(), version0);
         nodeA = new DiscoveryNode("TS_A", "TS_A", serviceA.boundAddress().publishAddress(), emptyMap(), emptySet(), version0);
         serviceB = build(Settings.builder().put("name", "TS_B").build(), version1);
@@ -109,7 +127,8 @@ public class ZenFaultDetectionTests extends ESTestCase {
         super.tearDown();
         serviceA.close();
         serviceB.close();
-        clusterService.close();
+        clusterServiceA.close();
+        clusterServiceB.close();
         terminate(threadPool);
     }
 
@@ -117,7 +136,10 @@ public class ZenFaultDetectionTests extends ESTestCase {
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
         MockTransportService transportService =
                 new MockTransportService(
-                        Settings.EMPTY,
+                        Settings.builder()
+                             // trace zenfd actions but keep the default otherwise
+                            .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), singleton(TransportLivenessAction.NAME))
+                            .build(),
                         new LocalTransport(settings, threadPool, version, namedWriteableRegistry, circuitBreakerService),
                         threadPool,
                         ClusterName.DEFAULT);
@@ -183,6 +205,9 @@ public class ZenFaultDetectionTests extends ESTestCase {
         serviceB.stop();
         notified.await(30, TimeUnit.SECONDS);
 
+        CircuitBreaker inFlightRequestsBreaker = circuitBreakerService.getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS);
+        assertThat(inFlightRequestsBreaker.getTrippedCount(), equalTo(0L));
+
         assertEquals(nodeB, failureNode[0]);
         Matcher<String> matcher = Matchers.containsString("verified");
         if (!shouldRetry) {
@@ -200,9 +225,9 @@ public class ZenFaultDetectionTests extends ESTestCase {
                 .put(FaultDetection.PING_INTERVAL_SETTING.getKey(), "5m");
         ClusterName clusterName = new ClusterName(randomAsciiOfLengthBetween(3, 20));
         final ClusterState state = ClusterState.builder(clusterName).nodes(buildNodesForA(false)).build();
-        setState(clusterService, state);
+        setState(clusterServiceA, state);
         MasterFaultDetection masterFD = new MasterFaultDetection(settings.build(), threadPool, serviceA, clusterName,
-                clusterService);
+            clusterServiceA);
         masterFD.start(nodeB, "test");
 
         final String[] failureReason = new String[1];
@@ -217,6 +242,9 @@ public class ZenFaultDetectionTests extends ESTestCase {
         serviceB.stop();
         notified.await(30, TimeUnit.SECONDS);
 
+        CircuitBreaker inFlightRequestsBreaker = circuitBreakerService.getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS);
+        assertThat(inFlightRequestsBreaker.getTrippedCount(), equalTo(0L));
+
         assertEquals(nodeB, failureNode[0]);
         Matcher<String> matcher = Matchers.containsString("verified");
         if (!shouldRetry) {
@@ -225,4 +253,79 @@ public class ZenFaultDetectionTests extends ESTestCase {
 
         assertThat(failureReason[0], matcher);
     }
+
+    public void testMasterFaultDetectionNotSizeLimited() throws InterruptedException {
+        Settings.Builder settings = Settings.builder();
+        boolean shouldRetry = randomBoolean();
+        settings
+            .put(FaultDetection.CONNECT_ON_NETWORK_DISCONNECT_SETTING.getKey(), shouldRetry)
+            .put(FaultDetection.PING_INTERVAL_SETTING.getKey(), "1s");
+        ClusterName clusterName = new ClusterName(randomAsciiOfLengthBetween(3, 20));
+        final ClusterState stateNodeA = ClusterState.builder(clusterName).nodes(buildNodesForA(false)).build();
+        setState(clusterServiceA, stateNodeA);
+
+        int minExpectedPings = 2;
+
+        PingProbe pingProbeA = new PingProbe(minExpectedPings);
+        PingProbe pingProbeB = new PingProbe(minExpectedPings);
+
+        serviceA.addTracer(pingProbeA);
+        serviceB.addTracer(pingProbeB);
+
+        MasterFaultDetection masterFDNodeA = new MasterFaultDetection(settings.build(), threadPool, serviceA, clusterName,
+            clusterServiceA);
+        masterFDNodeA.start(nodeB, "test");
+
+        final ClusterState stateNodeB = ClusterState.builder(clusterName).nodes(buildNodesForB(true)).build();
+        setState(clusterServiceB, stateNodeB);
+
+        MasterFaultDetection masterFDNodeB = new MasterFaultDetection(settings.build(), threadPool, serviceB, clusterName,
+            clusterServiceB);
+        masterFDNodeB.start(nodeB, "test");
+
+        // let's do a few pings
+        pingProbeA.awaitMinCompletedPings();
+        pingProbeB.awaitMinCompletedPings();
+
+        CircuitBreaker inFlightRequestsBreaker = circuitBreakerService.getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS);
+        assertThat(inFlightRequestsBreaker.getTrippedCount(), equalTo(0L));
+        assertThat(pingProbeA.completedPings(), greaterThanOrEqualTo(minExpectedPings));
+        assertThat(pingProbeB.completedPings(), greaterThanOrEqualTo(minExpectedPings));
+    }
+
+    private static class PingProbe extends MockTransportService.Tracer {
+        private final Set<Tuple<DiscoveryNode, Long>> inflightPings = Collections.newSetFromMap(new ConcurrentHashMap<>());
+        private final Set<Tuple<DiscoveryNode, Long>> completedPings = Collections.newSetFromMap(new ConcurrentHashMap<>());
+        private final CountDownLatch waitForPings;
+
+        public PingProbe(int minCompletedPings) {
+            this.waitForPings = new CountDownLatch(minCompletedPings);
+        }
+
+        @Override
+        public void requestSent(DiscoveryNode node, long requestId, String action, TransportRequestOptions options) {
+            if (MasterFaultDetection.MASTER_PING_ACTION_NAME.equals(action)) {
+                inflightPings.add(Tuple.tuple(node, requestId));
+            }
+        }
+
+        @Override
+        public void receivedResponse(long requestId, DiscoveryNode sourceNode, String action) {
+            if (MasterFaultDetection.MASTER_PING_ACTION_NAME.equals(action)) {
+                Tuple<DiscoveryNode, Long> ping = Tuple.tuple(sourceNode, requestId);
+                if (inflightPings.remove(ping)) {
+                    completedPings.add(ping);
+                    waitForPings.countDown();
+                }
+            }
+        }
+
+        public int completedPings() {
+            return completedPings.size();
+        }
+
+        public void awaitMinCompletedPings() throws InterruptedException {
+            waitForPings.await();
+        }
+    }
 }
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java
index c2bc877b903..c0dfcb69ead 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java
@@ -743,8 +743,8 @@ public class IndicesRequestTests extends ESIntegTestCase {
         }
 
         @Override
-        public <Request extends TransportRequest> void registerRequestHandler(String action, Supplier<Request> request, String executor, boolean forceExecution, TransportRequestHandler<Request> handler) {
-            super.registerRequestHandler(action, request, executor, forceExecution, new InterceptingRequestHandler<>(action, handler));
+        public <Request extends TransportRequest> void registerRequestHandler(String action, Supplier<Request> request, String executor, boolean forceExecution, boolean canTripCircuitBreaker, TransportRequestHandler<Request> handler) {
+            super.registerRequestHandler(action, request, executor, forceExecution, canTripCircuitBreaker, new InterceptingRequestHandler<>(action, handler));
         }
 
         @Override

From 61b1f4ad0b88323a238c3f6a1d6d539c4479675a Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Wed, 27 Apr 2016 09:17:10 +0200
Subject: [PATCH 0224/1311] Fix xcontent rendering of ip terms aggs. #18003

Currently terms on an ip address try to put their binary representation in the
json response. With this commit, they would return a formatted ip address:

```
      "buckets": [
        {
          "key": "192.168.1.7",
          "doc_count": 1
        }
      ]
```
---
 ...balOrdinalsSignificantTermsAggregator.java |   6 +-
 .../significant/InternalSignificantTerms.java |  19 ++-
 .../significant/SignificantLongTerms.java     |  13 +--
 .../significant/SignificantStringTerms.java   |  42 ++++---
 .../SignificantStringTermsAggregator.java     |   6 +-
 .../significant/UnmappedSignificantTerms.java |   5 +-
 .../bucket/terms/StringTerms.java             |   2 +-
 .../SignificanceHeuristicTests.java           |  10 +-
 .../test/search.aggregation/20_terms.yaml     | 110 ++++++++++++++++++
 .../30_sig_terms.yaml}                        |  41 +++++++
 10 files changed, 208 insertions(+), 46 deletions(-)
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml
 rename rest-api-spec/src/main/resources/rest-api-spec/test/{search/test_sig_terms.yaml => search.aggregation/30_sig_terms.yaml} (66%)

diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java
index 2372886672e..e161ecb374c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/GlobalOrdinalsSignificantTermsAggregator.java
@@ -110,7 +110,7 @@ public class GlobalOrdinalsSignificantTermsAggregator extends GlobalOrdinalsStri
             }
 
             if (spare == null) {
-                spare = new SignificantStringTerms.Bucket(new BytesRef(), 0, 0, 0, 0, null);
+                spare = new SignificantStringTerms.Bucket(new BytesRef(), 0, 0, 0, 0, null, format);
             }
             spare.bucketOrd = bucketOrd;
             copy(globalOrds.lookupOrd(globalTermOrd), spare.termBytes);
@@ -135,7 +135,7 @@ public class GlobalOrdinalsSignificantTermsAggregator extends GlobalOrdinalsStri
             list[i] = bucket;
         }
 
-        return new SignificantStringTerms(subsetSize, supersetSize, name, bucketCountThresholds.getRequiredSize(),
+        return new SignificantStringTerms(subsetSize, supersetSize, name, format, bucketCountThresholds.getRequiredSize(),
                 bucketCountThresholds.getMinDocCount(), significanceHeuristic, Arrays.asList(list), pipelineAggregators(),
                 metaData());
     }
@@ -146,7 +146,7 @@ public class GlobalOrdinalsSignificantTermsAggregator extends GlobalOrdinalsStri
         ContextIndexSearcher searcher = context.searchContext().searcher();
         IndexReader topReader = searcher.getIndexReader();
         int supersetSize = topReader.numDocs();
-        return new SignificantStringTerms(0, supersetSize, name, bucketCountThresholds.getRequiredSize(),
+        return new SignificantStringTerms(0, supersetSize, name, format, bucketCountThresholds.getRequiredSize(),
                 bucketCountThresholds.getMinDocCount(), significanceHeuristic,
                 Collections.<InternalSignificantTerms.Bucket> emptyList(), pipelineAggregators(), metaData());
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java
index 21b92e83fff..d42aa731887 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java
@@ -20,6 +20,7 @@ package org.elasticsearch.search.aggregations.bucket.significant;
 
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
@@ -33,6 +34,7 @@ import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 /**
  *
@@ -56,15 +58,19 @@ public abstract class InternalSignificantTerms<A extends InternalSignificantTerm
         long bucketOrd;
         protected InternalAggregations aggregations;
         double score;
+        transient final DocValueFormat format;
 
-        protected Bucket(long subsetSize, long supersetSize) {
+        protected Bucket(long subsetSize, long supersetSize, DocValueFormat format) {
             // for serialization
             super(subsetSize, supersetSize);
+            this.format = format;
         }
 
-        protected Bucket(long subsetDf, long subsetSize, long supersetDf, long supersetSize, InternalAggregations aggregations) {
+        protected Bucket(long subsetDf, long subsetSize, long supersetDf, long supersetSize,
+                InternalAggregations aggregations, DocValueFormat format) {
             super(subsetDf, subsetSize, supersetDf, supersetSize);
             this.aggregations = aggregations;
+            this.format = format;
         }
 
         @Override
@@ -122,9 +128,11 @@ public abstract class InternalSignificantTerms<A extends InternalSignificantTerm
         }
     }
 
-    protected InternalSignificantTerms(long subsetSize, long supersetSize, String name, int requiredSize, long minDocCount,
-            SignificanceHeuristic significanceHeuristic, List<? extends Bucket> buckets, List<PipelineAggregator> pipelineAggregators,
-            Map<String, Object> metaData) {
+    protected DocValueFormat format;
+
+    protected InternalSignificantTerms(long subsetSize, long supersetSize, String name, DocValueFormat format, int requiredSize,
+            long minDocCount, SignificanceHeuristic significanceHeuristic, List<? extends Bucket> buckets,
+            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
         super(name, pipelineAggregators, metaData);
         this.requiredSize = requiredSize;
         this.minDocCount = minDocCount;
@@ -132,6 +140,7 @@ public abstract class InternalSignificantTerms<A extends InternalSignificantTerm
         this.subsetSize = subsetSize;
         this.supersetSize = supersetSize;
         this.significanceHeuristic = significanceHeuristic;
+        this.format = Objects.requireNonNull(format);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java
index fba66e7ba0c..f2172082ae4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java
@@ -81,18 +81,15 @@ public class SignificantLongTerms extends InternalSignificantTerms<SignificantLo
     static class Bucket extends InternalSignificantTerms.Bucket {
 
         long term;
-        private transient final DocValueFormat format;
 
-        public Bucket(long subsetSize, long supersetSize, DocValueFormat formatter) {
-            super(subsetSize, supersetSize);
-            this.format = formatter;
+        public Bucket(long subsetSize, long supersetSize, DocValueFormat format) {
+            super(subsetSize, supersetSize, format);
             // for serialization
         }
 
         public Bucket(long subsetDf, long subsetSize, long supersetDf, long supersetSize, long term, InternalAggregations aggregations,
                 DocValueFormat format) {
-            super(subsetDf, subsetSize, supersetDf, supersetSize, aggregations);
-            this.format = format;
+            super(subsetDf, subsetSize, supersetDf, supersetSize, aggregations, format);
             this.term = term;
         }
 
@@ -160,7 +157,6 @@ public class SignificantLongTerms extends InternalSignificantTerms<SignificantLo
             return builder;
         }
     }
-    private DocValueFormat format;
 
     SignificantLongTerms() {
     } // for serialization
@@ -169,8 +165,7 @@ public class SignificantLongTerms extends InternalSignificantTerms<SignificantLo
             long minDocCount, SignificanceHeuristic significanceHeuristic, List<? extends InternalSignificantTerms.Bucket> buckets,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
 
-        super(subsetSize, supersetSize, name, requiredSize, minDocCount, significanceHeuristic, buckets, pipelineAggregators, metaData);
-        this.format = Objects.requireNonNull(format);
+        super(subsetSize, supersetSize, name, format, requiredSize, minDocCount, significanceHeuristic, buckets, pipelineAggregators, metaData);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java
index 5d076190bc4..b951c351702 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java
@@ -22,6 +22,7 @@ import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
@@ -55,7 +56,8 @@ public class SignificantStringTerms extends InternalSignificantTerms<Significant
     private final static BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
         @Override
         public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
-            Bucket buckets = new Bucket((long) context.attributes().get("subsetSize"), (long) context.attributes().get("supersetSize"));
+            Bucket buckets = new Bucket((long) context.attributes().get("subsetSize"),
+                    (long) context.attributes().get("supersetSize"), context.format());
             buckets.readFrom(in);
             return buckets;
         }
@@ -84,18 +86,20 @@ public class SignificantStringTerms extends InternalSignificantTerms<Significant
 
         BytesRef termBytes;
 
-        public Bucket(long subsetSize, long supersetSize) {
+        public Bucket(long subsetSize, long supersetSize, DocValueFormat format) {
             // for serialization
-            super(subsetSize, supersetSize);
+            super(subsetSize, supersetSize, format);
         }
 
-        public Bucket(BytesRef term, long subsetDf, long subsetSize, long supersetDf, long supersetSize, InternalAggregations aggregations) {
-            super(subsetDf, subsetSize, supersetDf, supersetSize, aggregations);
+        public Bucket(BytesRef term, long subsetDf, long subsetSize, long supersetDf, long supersetSize, InternalAggregations aggregations,
+                DocValueFormat format) {
+            super(subsetDf, subsetSize, supersetDf, supersetSize, aggregations, format);
             this.termBytes = term;
         }
 
-        public Bucket(BytesRef term, long subsetDf, long subsetSize, long supersetDf, long supersetSize, InternalAggregations aggregations, double score) {
-            this(term, subsetDf, subsetSize, supersetDf, supersetSize, aggregations);
+        public Bucket(BytesRef term, long subsetDf, long subsetSize, long supersetDf, long supersetSize,
+                InternalAggregations aggregations, double score, DocValueFormat format) {
+            this(term, subsetDf, subsetSize, supersetDf, supersetSize, aggregations, format);
             this.score = score;
         }
 
@@ -112,7 +116,7 @@ public class SignificantStringTerms extends InternalSignificantTerms<Significant
 
         @Override
         public String getKeyAsString() {
-            return termBytes.utf8ToString();
+            return format.format(termBytes);
         }
 
         @Override
@@ -122,7 +126,7 @@ public class SignificantStringTerms extends InternalSignificantTerms<Significant
 
         @Override
         Bucket newBucket(long subsetDf, long subsetSize, long supersetDf, long supersetSize, InternalAggregations aggregations) {
-            return new Bucket(termBytes, subsetDf, subsetSize, supersetDf, supersetSize, aggregations);
+            return new Bucket(termBytes, subsetDf, subsetSize, supersetDf, supersetSize, aggregations, format);
         }
 
         @Override
@@ -146,7 +150,7 @@ public class SignificantStringTerms extends InternalSignificantTerms<Significant
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
-            builder.utf8Field(CommonFields.KEY, termBytes);
+            builder.field(CommonFields.KEY, getKeyAsString());
             builder.field(CommonFields.DOC_COUNT, getDocCount());
             builder.field("score", score);
             builder.field("bg_count", supersetDf);
@@ -158,11 +162,11 @@ public class SignificantStringTerms extends InternalSignificantTerms<Significant
 
     SignificantStringTerms() {} // for serialization
 
-    public SignificantStringTerms(long subsetSize, long supersetSize, String name, int requiredSize, long minDocCount,
-            SignificanceHeuristic significanceHeuristic, List<? extends InternalSignificantTerms.Bucket> buckets,
+    public SignificantStringTerms(long subsetSize, long supersetSize, String name, DocValueFormat format, int requiredSize,
+            long minDocCount, SignificanceHeuristic significanceHeuristic, List<? extends InternalSignificantTerms.Bucket> buckets,
             List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) {
-        super(subsetSize, supersetSize, name, requiredSize, minDocCount, significanceHeuristic, buckets, pipelineAggregators, metaData);
+        super(subsetSize, supersetSize, name, format, requiredSize, minDocCount, significanceHeuristic, buckets, pipelineAggregators, metaData);
     }
 
     @Override
@@ -172,25 +176,26 @@ public class SignificantStringTerms extends InternalSignificantTerms<Significant
 
     @Override
     public SignificantStringTerms create(List<SignificantStringTerms.Bucket> buckets) {
-        return new SignificantStringTerms(this.subsetSize, this.supersetSize, this.name, this.requiredSize, this.minDocCount,
+        return new SignificantStringTerms(this.subsetSize, this.supersetSize, this.name, this.format, this.requiredSize, this.minDocCount,
                 this.significanceHeuristic, buckets, this.pipelineAggregators(), this.metaData);
     }
 
     @Override
     public Bucket createBucket(InternalAggregations aggregations, SignificantStringTerms.Bucket prototype) {
         return new Bucket(prototype.termBytes, prototype.subsetDf, prototype.subsetSize, prototype.supersetDf, prototype.supersetSize,
-                aggregations);
+                aggregations, prototype.format);
     }
 
     @Override
     protected SignificantStringTerms create(long subsetSize, long supersetSize, List<InternalSignificantTerms.Bucket> buckets,
             InternalSignificantTerms prototype) {
-        return new SignificantStringTerms(subsetSize, supersetSize, prototype.getName(), prototype.requiredSize, prototype.minDocCount,
-                prototype.significanceHeuristic, buckets, prototype.pipelineAggregators(), prototype.getMetaData());
+        return new SignificantStringTerms(subsetSize, supersetSize, prototype.getName(), prototype.format, prototype.requiredSize,
+                prototype.minDocCount, prototype.significanceHeuristic, buckets, prototype.pipelineAggregators(), prototype.getMetaData());
     }
 
     @Override
     protected void doReadFrom(StreamInput in) throws IOException {
+        this.format = in.readNamedWriteable(DocValueFormat.class);
         this.requiredSize = readSize(in);
         this.minDocCount = in.readVLong();
         this.subsetSize = in.readVLong();
@@ -199,7 +204,7 @@ public class SignificantStringTerms extends InternalSignificantTerms<Significant
         int size = in.readVInt();
         List<InternalSignificantTerms.Bucket> buckets = new ArrayList<>(size);
         for (int i = 0; i < size; i++) {
-            Bucket bucket = new Bucket(subsetSize, supersetSize);
+            Bucket bucket = new Bucket(subsetSize, supersetSize, format);
             bucket.readFrom(in);
             buckets.add(bucket);
         }
@@ -209,6 +214,7 @@ public class SignificantStringTerms extends InternalSignificantTerms<Significant
 
     @Override
     protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(format);
         writeSize(requiredSize, out);
         out.writeVLong(minDocCount);
         out.writeVLong(subsetSize);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsAggregator.java
index c1d4e2225ea..41d136d01dc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTermsAggregator.java
@@ -90,7 +90,7 @@ public class SignificantStringTermsAggregator extends StringTermsAggregator {
             }
 
             if (spare == null) {
-                spare = new SignificantStringTerms.Bucket(new BytesRef(), 0, 0, 0, 0, null);
+                spare = new SignificantStringTerms.Bucket(new BytesRef(), 0, 0, 0, 0, null, format);
             }
 
             bucketOrds.get(i, spare.termBytes);
@@ -117,7 +117,7 @@ public class SignificantStringTermsAggregator extends StringTermsAggregator {
             list[i] = bucket;
         }
 
-        return new SignificantStringTerms(subsetSize, supersetSize, name, bucketCountThresholds.getRequiredSize(),
+        return new SignificantStringTerms(subsetSize, supersetSize, name, format, bucketCountThresholds.getRequiredSize(),
                 bucketCountThresholds.getMinDocCount(), significanceHeuristic, Arrays.asList(list), pipelineAggregators(),
                 metaData());
     }
@@ -128,7 +128,7 @@ public class SignificantStringTermsAggregator extends StringTermsAggregator {
         ContextIndexSearcher searcher = context.searchContext().searcher();
         IndexReader topReader = searcher.getIndexReader();
         int supersetSize = topReader.numDocs();
-        return new SignificantStringTerms(0, supersetSize, name, bucketCountThresholds.getRequiredSize(),
+        return new SignificantStringTerms(0, supersetSize, name, format, bucketCountThresholds.getRequiredSize(),
                 bucketCountThresholds.getMinDocCount(), significanceHeuristic,
                 Collections.<InternalSignificantTerms.Bucket> emptyList(), pipelineAggregators(), metaData());
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/UnmappedSignificantTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/UnmappedSignificantTerms.java
index 8858050efaf..5369e269058 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/UnmappedSignificantTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/UnmappedSignificantTerms.java
@@ -21,6 +21,7 @@ package org.elasticsearch.search.aggregations.bucket.significant;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
@@ -59,8 +60,8 @@ public class UnmappedSignificantTerms extends InternalSignificantTerms<UnmappedS
     public UnmappedSignificantTerms(String name, int requiredSize, long minDocCount, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
         //We pass zero for index/subset sizes because for the purpose of significant term analysis
         // we assume an unmapped index's size is irrelevant to the proceedings.
-        super(0, 0, name, requiredSize, minDocCount, SignificantTermsAggregatorBuilder.DEFAULT_SIGNIFICANCE_HEURISTIC, BUCKETS,
-                pipelineAggregators, metaData);
+        super(0, 0, name, DocValueFormat.RAW, requiredSize, minDocCount, SignificantTermsAggregatorBuilder.DEFAULT_SIGNIFICANCE_HEURISTIC,
+                BUCKETS, pipelineAggregators, metaData);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java
index 9c7c85ef855..ddaaebe8999 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java
@@ -141,7 +141,7 @@ public class StringTerms extends InternalTerms<StringTerms, StringTerms.Bucket>
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
-            builder.utf8Field(CommonFields.KEY, termBytes);
+            builder.field(CommonFields.KEY, getKeyAsString());
             builder.field(CommonFields.DOC_COUNT, getDocCount());
             if (showDocCountError) {
                 builder.field(InternalTerms.DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME, getDocCountError());
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java
index 75970a79d1d..df449aeeaf2 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java
@@ -131,9 +131,9 @@ public class SignificanceHeuristicTests extends ESTestCase {
             sTerms[1] = new SignificantLongTerms();
         } else {
             BytesRef term = new BytesRef("someterm");
-            buckets.add(new SignificantStringTerms.Bucket(term, 1, 2, 3, 4, InternalAggregations.EMPTY));
-            sTerms[0] = new SignificantStringTerms(10, 20, "some_name", 1, 1, heuristic, buckets, Collections.emptyList(),
-                    null);
+            buckets.add(new SignificantStringTerms.Bucket(term, 1, 2, 3, 4, InternalAggregations.EMPTY, DocValueFormat.RAW));
+            sTerms[0] = new SignificantStringTerms(10, 20, "some_name", DocValueFormat.RAW, 1, 1, heuristic, buckets,
+                    Collections.emptyList(), null);
             sTerms[1] = new SignificantStringTerms();
         }
         return sTerms;
@@ -184,7 +184,7 @@ public class SignificanceHeuristicTests extends ESTestCase {
 
     private InternalSignificantTerms createAggregation(String type, SignificanceHeuristic significanceHeuristic, List<InternalSignificantTerms.Bucket> buckets, long subsetSize, long supersetSize) {
         if (type.equals("string")) {
-            return new SignificantStringTerms(subsetSize, supersetSize, "sig_terms", 2, -1, significanceHeuristic, buckets, new ArrayList<PipelineAggregator>(), new HashMap<String, Object>());
+            return new SignificantStringTerms(subsetSize, supersetSize, "sig_terms", DocValueFormat.RAW, 2, -1, significanceHeuristic, buckets, new ArrayList<PipelineAggregator>(), new HashMap<String, Object>());
         } else {
             return new SignificantLongTerms(subsetSize, supersetSize, "sig_terms", DocValueFormat.RAW, 2, -1, significanceHeuristic, buckets, new ArrayList<PipelineAggregator>(), new HashMap<String, Object>());
         }
@@ -192,7 +192,7 @@ public class SignificanceHeuristicTests extends ESTestCase {
 
     private InternalSignificantTerms.Bucket createBucket(String type, long subsetDF, long subsetSize, long supersetDF, long supersetSize, long label) {
         if (type.equals("string")) {
-            return new SignificantStringTerms.Bucket(new BytesRef(Long.toString(label).getBytes(StandardCharsets.UTF_8)), subsetDF, subsetSize, supersetDF, supersetSize, InternalAggregations.EMPTY);
+            return new SignificantStringTerms.Bucket(new BytesRef(Long.toString(label).getBytes(StandardCharsets.UTF_8)), subsetDF, subsetSize, supersetDF, supersetSize, InternalAggregations.EMPTY, DocValueFormat.RAW);
         } else {
             return new SignificantLongTerms.Bucket(subsetDF, subsetSize, supersetDF, supersetSize, label, InternalAggregations.EMPTY, DocValueFormat.RAW);
         }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml
new file mode 100644
index 00000000000..cf4771eade3
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml
@@ -0,0 +1,110 @@
+setup:
+  - do:
+      indices.create:
+          index: test_1
+          body:
+            settings:
+              number_of_replicas: 0
+            mappings:
+              test:
+                properties:
+                  str:
+                     type: keyword
+                  ip:
+                     type: ip
+
+  - do:
+      cluster.health:
+        wait_for_status: green
+
+---
+"Basic test":
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: { "str" : "abc" }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 2
+        body: { "str": "abc" }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 3
+        body: { "str": "bcd" }
+      
+  - do:
+      indices.refresh: {}
+    
+  - do:
+      search:
+        body: { "aggs" : { "str_terms" : { "terms" : { "field" : "str" } } } }
+
+  - match: { hits.total: 3 }
+  
+  - length: { aggregations.str_terms.buckets: 2 }
+  
+  - match: { aggregations.str_terms.buckets.0.key: "abc" }
+  
+  - is_false: aggregations.str_terms.buckets.0.key_as_string
+  
+  - match: { aggregations.str_terms.buckets.0.doc_count: 2 }
+  
+  - match: { aggregations.str_terms.buckets.1.key: "bcd" }
+  
+  - is_false: aggregations.str_terms.buckets.1.key_as_string
+  
+  - match: { aggregations.str_terms.buckets.1.doc_count: 1 }
+  
+---
+"IP test":
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: { "ip": "::1" }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 2
+        body: { "ip": "127.0.0.1" }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 3
+        body: { "ip": "::1" }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "aggs" : { "ip_terms" : { "terms" : { "field" : "ip" } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.ip_terms.buckets: 2 }
+
+  - match: { aggregations.ip_terms.buckets.0.key: "::1" }
+
+  - is_false: aggregations.ip_terms.buckets.0.key_as_string
+
+  - match: { aggregations.ip_terms.buckets.0.doc_count: 2 }
+
+  - match: { aggregations.ip_terms.buckets.1.key: "127.0.0.1" }
+  
+  - is_false: aggregations.ip_terms.buckets.1.key_as_string
+  
+  - match: { aggregations.ip_terms.buckets.1.doc_count: 1 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/test_sig_terms.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/30_sig_terms.yaml
similarity index 66%
rename from rest-api-spec/src/main/resources/rest-api-spec/test/search/test_sig_terms.yaml
rename to rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/30_sig_terms.yaml
index 9c01bbc2b5d..45c042baea4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/test_sig_terms.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/30_sig_terms.yaml
@@ -80,3 +80,44 @@
   - match: {aggregations.class.buckets.0.sig_terms.buckets.0.key: "bad"}
   - match: {aggregations.class.buckets.1.sig_terms.buckets.0.key: "good"}
   
+---
+"IP test":
+  - do:
+      indices.create:
+          index:  ip_index
+          body:
+            mappings:
+                doc:
+                    properties:
+                        ip:
+                            type: ip
+
+  - do:
+      index:
+          index:  ip_index
+          type:   doc
+          id:     1
+          body:   { ip: "::1" }
+  - do:
+      index:
+          index:  ip_index
+          type:   doc
+          id:     2
+          body:   { }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "query" : { "exists" : { "field" : "ip" } }, "aggs" : { "ip_terms" : { "significant_terms" : { "field" : "ip", "min_doc_count" : 1 } } } }
+
+  - match: { hits.total: 1 }
+
+  - length: { aggregations.ip_terms.buckets: 1 }
+
+  - match: { aggregations.ip_terms.buckets.0.key: "::1" }
+
+  - is_false: aggregations.ip_terms.buckets.0.key_as_string
+
+  - match: { aggregations.ip_terms.buckets.0.doc_count: 1 }

From f2797dbccb7e1f059b112e6854b3d24826135b66 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Fri, 13 May 2016 15:08:15 +0200
Subject: [PATCH 0225/1311] Fixed grammar in index-too-old exception (#18327)

---
 .../cluster/metadata/MetaDataIndexUpgradeService.java           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
index cdfb5487019..1cc6f5a44cc 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
@@ -95,7 +95,7 @@ public class MetaDataIndexUpgradeService extends AbstractComponent {
     private void checkSupportedVersion(IndexMetaData indexMetaData) {
         if (indexMetaData.getState() == IndexMetaData.State.OPEN && isSupportedVersion(indexMetaData) == false) {
             throw new IllegalStateException("The index [" + indexMetaData.getIndex() + "] was created before v2.0.0.beta1 and wasn't upgraded."
-                    + " This index should be open using a version before " + Version.CURRENT.minimumCompatibilityVersion()
+                    + " This index should be opened using a version before " + Version.CURRENT.minimumCompatibilityVersion()
                     + " and upgraded using the upgrade API.");
         }
     }

From a1d0fb5008e9d25c2ca66253a69fd108c597f869 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Fri, 13 May 2016 12:42:18 +0200
Subject: [PATCH 0226/1311] painless: cleanup of DynamicCallSite to have all
 implementation encapsulated in inner class, outer class only has bootstrap
 method

---
 .../painless/DynamicCallSite.java             | 167 +++++++++---------
 1 file changed, 83 insertions(+), 84 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DynamicCallSite.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DynamicCallSite.java
index 235bfedf8f7..330a64e2a99 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DynamicCallSite.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DynamicCallSite.java
@@ -39,10 +39,12 @@ import java.lang.invoke.MutableCallSite;
  * Based on the cascaded inlining cache from the JSR 292 cookbook
  * (https://code.google.com/archive/p/jsr292-cookbook/, BSD license)
  */
-// NOTE: this class must be public, because generated painless classes are in a different package,
+// NOTE: this class must be public, because generated painless classes are in a different classloader,
 // and it needs to be accessible by that code.
 public final class DynamicCallSite {
 
+    private DynamicCallSite() {} // no instance!
+
     // NOTE: these must be primitive types, see https://docs.oracle.com/javase/specs/jvms/se7/html/jvms-6.html#jvms-6.5.invokedynamic
     /** static bootstrap parameter indicating a dynamic method call, e.g. foo.bar(...) */
     public static final int METHOD_CALL = 0;
@@ -55,20 +57,92 @@ public final class DynamicCallSite {
     /** static bootstrap parameter indicating a dynamic array store, e.g. foo[bar] = baz */
     public static final int ARRAY_STORE = 4;
 
-    static class InliningCacheCallSite extends MutableCallSite {
+    static final class InliningCacheCallSite extends MutableCallSite {
         /** maximum number of types before we go megamorphic */
         static final int MAX_DEPTH = 5;
 
-        final Lookup lookup;
-        final String name;
-        final int flavor;
-        int depth;
+        private final String name;
+        private final int flavor;
+        int depth; // pkg-protected for testing
 
-        InliningCacheCallSite(Lookup lookup, String name, MethodType type, int flavor) {
+        InliningCacheCallSite(String name, MethodType type, int flavor) {
             super(type);
-            this.lookup = lookup;
             this.name = name;
             this.flavor = flavor;
+
+            MethodHandle fallback = FALLBACK.bindTo(this);
+            fallback = fallback.asCollector(Object[].class, type.parameterCount());
+            fallback = fallback.asType(type);
+
+            setTarget(fallback);
+        }
+        
+        /**
+         * guard method for inline caching: checks the receiver's class is the same
+         * as the cached class
+         */
+        static boolean checkClass(Class<?> clazz, Object receiver) {
+            return receiver.getClass() == clazz;
+        }
+
+        /**
+         * Does a slow lookup against the whitelist.
+         */
+        private static MethodHandle lookup(int flavor, Class<?> clazz, String name) {
+            switch(flavor) {
+                case METHOD_CALL:
+                    return Def.lookupMethod(clazz, name, Definition.INSTANCE);
+                case LOAD:
+                    return Def.lookupGetter(clazz, name, Definition.INSTANCE);
+                case STORE:
+                    return Def.lookupSetter(clazz, name, Definition.INSTANCE);
+                case ARRAY_LOAD:
+                    return Def.lookupArrayLoad(clazz);
+                case ARRAY_STORE:
+                    return Def.lookupArrayStore(clazz);
+                default: throw new AssertionError();
+            }
+        }
+
+        /**
+         * Called when a new type is encountered (or, when we have encountered more than {@code MAX_DEPTH}
+         * types at this call site and given up on caching).
+         */
+        Object fallback(Object[] args) throws Throwable {
+            MethodType type = type();
+            Object receiver = args[0];
+            Class<?> receiverClass = receiver.getClass();
+            MethodHandle target = lookup(flavor, receiverClass, name);
+            target = target.asType(type);
+
+            if (depth >= MAX_DEPTH) {
+                // revert to a vtable call
+                setTarget(target);
+                return target.invokeWithArguments(args);
+            }
+
+            MethodHandle test = CHECK_CLASS.bindTo(receiverClass);
+            test = test.asType(test.type().changeParameterType(0, type.parameterType(0)));
+
+            MethodHandle guard = MethodHandles.guardWithTest(test, target, getTarget());
+            depth++;
+
+            setTarget(guard);
+            return target.invokeWithArguments(args);
+        }
+
+        private static final MethodHandle CHECK_CLASS;
+        private static final MethodHandle FALLBACK;
+        static {
+            final Lookup lookup = MethodHandles.lookup();
+            try {
+                CHECK_CLASS = lookup.findStatic(lookup.lookupClass(), "checkClass",
+                                                MethodType.methodType(boolean.class, Class.class, Object.class));
+                FALLBACK = lookup.findVirtual(lookup.lookupClass(), "fallback",
+                                             MethodType.methodType(Object.class, Object[].class));
+            } catch (ReflectiveOperationException e) {
+                throw new AssertionError(e);
+            }
         }
     }
 
@@ -81,82 +155,7 @@ public final class DynamicCallSite {
      * see https://docs.oracle.com/javase/specs/jvms/se7/html/jvms-6.html#jvms-6.5.invokedynamic
      */
     public static CallSite bootstrap(Lookup lookup, String name, MethodType type, int flavor) {
-        InliningCacheCallSite callSite = new InliningCacheCallSite(lookup, name, type, flavor);
-
-        MethodHandle fallback = FALLBACK.bindTo(callSite);
-        fallback = fallback.asCollector(Object[].class, type.parameterCount());
-        fallback = fallback.asType(type);
-
-        callSite.setTarget(fallback);
-        return callSite;
+        return new InliningCacheCallSite(name, type, flavor);
     }
 
-    /**
-     * guard method for inline caching: checks the receiver's class is the same
-     * as the cached class
-     */
-    static boolean checkClass(Class<?> clazz, Object receiver) {
-        return receiver.getClass() == clazz;
-    }
-
-    /**
-     * Does a slow lookup against the whitelist.
-     */
-    private static MethodHandle lookup(int flavor, Class<?> clazz, String name) {
-        switch(flavor) {
-            case METHOD_CALL:
-                return Def.lookupMethod(clazz, name, Definition.INSTANCE);
-            case LOAD:
-                return Def.lookupGetter(clazz, name, Definition.INSTANCE);
-            case STORE:
-                return Def.lookupSetter(clazz, name, Definition.INSTANCE);
-            case ARRAY_LOAD:
-                return Def.lookupArrayLoad(clazz);
-            case ARRAY_STORE:
-                return Def.lookupArrayStore(clazz);
-            default: throw new AssertionError();
-        }
-    }
-
-    /**
-     * Called when a new type is encountered (or, when we have encountered more than {@code MAX_DEPTH}
-     * types at this call site and given up on caching).
-     */
-    static Object fallback(InliningCacheCallSite callSite, Object[] args) throws Throwable {
-        MethodType type = callSite.type();
-        Object receiver = args[0];
-        Class<?> receiverClass = receiver.getClass();
-        MethodHandle target = lookup(callSite.flavor, receiverClass, callSite.name);
-        target = target.asType(type);
-
-        if (callSite.depth >= InliningCacheCallSite.MAX_DEPTH) {
-            // revert to a vtable call
-            callSite.setTarget(target);
-            return target.invokeWithArguments(args);
-        }
-
-        MethodHandle test = CHECK_CLASS.bindTo(receiverClass);
-        test = test.asType(test.type().changeParameterType(0, type.parameterType(0)));
-
-        MethodHandle guard = MethodHandles.guardWithTest(test, target, callSite.getTarget());
-        callSite.depth++;
-
-        callSite.setTarget(guard);
-        return target.invokeWithArguments(args);
-    }
-
-    private static final MethodHandle CHECK_CLASS;
-    private static final MethodHandle FALLBACK;
-
-    static {
-        Lookup lookup = MethodHandles.lookup();
-        try {
-            CHECK_CLASS = lookup.findStatic(DynamicCallSite.class, "checkClass",
-                                            MethodType.methodType(boolean.class, Class.class, Object.class));
-            FALLBACK = lookup.findStatic(DynamicCallSite.class, "fallback",
-                                         MethodType.methodType(Object.class, InliningCacheCallSite.class, Object[].class));
-        } catch (ReflectiveOperationException e) {
-            throw new AssertionError(e);
-        }
-    }
 }

From 80a72652760abe3abc112997fb2f0026586a5556 Mon Sep 17 00:00:00 2001
From: Shubham Aggarwal <shubhamaggarwal391@gmail.com>
Date: Fri, 13 May 2016 19:57:40 +0530
Subject: [PATCH 0227/1311] Update CONTRIBUTING.md

Fix typo

Relates #18330
---
 CONTRIBUTING.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 9f7264acdae..a79b03f5376 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -85,7 +85,7 @@ option `Search for nested projects`. Additionally you will want to
 ensure that Eclipse is using 2048m of heap by modifying `eclipse.ini`
 accordingly to avoid GC overhead errors.
 
-IntelliJ users acn automatically configure their IDE: `gradle idea`
+IntelliJ users can automatically configure their IDE: `gradle idea`
 then `File->New Project From Existing Sources`. Point to the root of
 the source directory, select
 `Import project from external model->Gradle`, enable

From 638da06c1d5121622cf0d47c2c55b4a84a794909 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Mon, 18 Apr 2016 16:11:11 +0200
Subject: [PATCH 0228/1311] Add back support for `ip` range aggregations.
 #17859

This commit adds support for range aggregations on `ip` fields. However it will
only work on 5.x indices.

Closes #17700
---
 .../index/mapper/ip/LegacyIpFieldMapper.java  |   3 +-
 .../elasticsearch/search/SearchModule.java    |  10 +-
 .../aggregations/AggregationBuilders.java     |   8 +-
 .../bucket/range/BinaryRangeAggregator.java   | 234 ++++++
 .../range/BinaryRangeAggregatorFactory.java   |  70 ++
 .../bucket/range/InternalBinaryRange.java     | 311 ++++++++
 .../range/ip/IpRangeAggregatorBuilder.java    | 330 ++++++++
 .../bucket/range/ip/IpRangeParser.java        | 126 +++
 .../ipv4/IPv4RangeAggregatorBuilder.java      | 277 -------
 .../bucket/range/ipv4/InternalIPv4Range.java  | 164 ----
 .../bucket/range/ipv4/IpRangeParser.java      |  64 --
 .../ipv4/Ipv4RangeAggregatorFactory.java      |  44 --
 .../aggregations/support/ValueType.java       |   7 +-
 .../aggregations/BaseAggregationTestCase.java |   3 +-
 .../aggregations/bucket/IPv4RangeTests.java   |  71 --
 .../search/aggregations/bucket/IpRangeIT.java | 268 +++++++
 .../aggregations/bucket/IpRangeTests.java     |  90 +++
 .../range/BinaryRangeAggregatorTests.java     | 142 ++++
 .../bucket/iprange-aggregation.asciidoc       |   4 +-
 .../bucket/iprange-aggregation.asciidoc       |  24 +-
 .../migrate_5_0/aggregations.asciidoc         |   7 +
 .../messy/tests/IPv4RangeTests.java           | 717 ------------------
 22 files changed, 1602 insertions(+), 1372 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java
 create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorFactory.java
 create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
 create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeAggregatorBuilder.java
 create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeParser.java
 delete mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IPv4RangeAggregatorBuilder.java
 delete mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/InternalIPv4Range.java
 delete mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IpRangeParser.java
 delete mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/Ipv4RangeAggregatorFactory.java
 delete mode 100644 core/src/test/java/org/elasticsearch/search/aggregations/bucket/IPv4RangeTests.java
 create mode 100644 core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java
 create mode 100644 core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeTests.java
 create mode 100644 core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java
 delete mode 100644 modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IPv4RangeTests.java

diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpFieldMapper.java
index aef6869ffc3..01fa2106684 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpFieldMapper.java
@@ -50,7 +50,6 @@ import org.elasticsearch.index.mapper.core.LegacyLongFieldMapper.CustomLongNumer
 import org.elasticsearch.index.mapper.core.LegacyNumberFieldMapper;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.bucket.range.ipv4.InternalIPv4Range;
 import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
@@ -218,7 +217,7 @@ public class LegacyIpFieldMapper extends LegacyNumberFieldMapper {
                 }
                 if (fromTo != null) {
                     return rangeQuery(fromTo[0] == 0 ? null : fromTo[0],
-                            fromTo[1] == InternalIPv4Range.MAX_IP ? null : fromTo[1], true, false);
+                            fromTo[1] == MAX_IP ? null : fromTo[1], true, false);
                 }
             }
             return super.termQuery(value, context);
diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java
index 6fe4913d074..513a5c88a37 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -131,9 +131,9 @@ import org.elasticsearch.search.aggregations.bucket.range.date.InternalDateRange
 import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceAggregatorBuilder;
 import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceParser;
 import org.elasticsearch.search.aggregations.bucket.range.geodistance.InternalGeoDistance;
-import org.elasticsearch.search.aggregations.bucket.range.ipv4.IPv4RangeAggregatorBuilder;
-import org.elasticsearch.search.aggregations.bucket.range.ipv4.InternalIPv4Range;
-import org.elasticsearch.search.aggregations.bucket.range.ipv4.IpRangeParser;
+import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.InternalBinaryRange;
+import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeParser;
 import org.elasticsearch.search.aggregations.bucket.sampler.DiversifiedAggregatorBuilder;
 import org.elasticsearch.search.aggregations.bucket.sampler.DiversifiedSamplerParser;
 import org.elasticsearch.search.aggregations.bucket.sampler.InternalSampler;
@@ -507,7 +507,7 @@ public class SearchModule extends AbstractModule {
                 SignificantTermsAggregatorBuilder.AGGREGATION_NAME_FIELD);
         registerAggregation(RangeAggregatorBuilder::new, new RangeParser(), RangeAggregatorBuilder.AGGREGATION_NAME_FIELD);
         registerAggregation(DateRangeAggregatorBuilder::new, new DateRangeParser(), DateRangeAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(IPv4RangeAggregatorBuilder::new, new IpRangeParser(), IPv4RangeAggregatorBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(IpRangeAggregatorBuilder::new, new IpRangeParser(), IpRangeAggregatorBuilder.AGGREGATION_NAME_FIELD);
         registerAggregation(HistogramAggregatorBuilder::new, new HistogramParser(), HistogramAggregatorBuilder.AGGREGATION_NAME_FIELD);
         registerAggregation(DateHistogramAggregatorBuilder::new, new DateHistogramParser(),
                 DateHistogramAggregatorBuilder.AGGREGATION_NAME_FIELD);
@@ -736,7 +736,7 @@ public class SearchModule extends AbstractModule {
         UnmappedTerms.registerStreams();
         InternalRange.registerStream();
         InternalDateRange.registerStream();
-        InternalIPv4Range.registerStream();
+        InternalBinaryRange.registerStream();
         InternalHistogram.registerStream();
         InternalGeoDistance.registerStream();
         InternalNested.registerStream();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java
index 23fa8007d5b..b76d5d17891 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java
@@ -45,7 +45,7 @@ import org.elasticsearch.search.aggregations.bucket.range.Range;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregatorBuilder;
 import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeAggregatorBuilder;
 import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceAggregatorBuilder;
-import org.elasticsearch.search.aggregations.bucket.range.ipv4.IPv4RangeAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeAggregatorBuilder;
 import org.elasticsearch.search.aggregations.bucket.sampler.DiversifiedAggregatorBuilder;
 import org.elasticsearch.search.aggregations.bucket.sampler.Sampler;
 import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregatorBuilder;
@@ -261,11 +261,11 @@ public class AggregationBuilders {
     }
 
     /**
-     * Create a new {@link IPv4RangeAggregatorBuilder} aggregation with the
+     * Create a new {@link IpRangeAggregatorBuilder} aggregation with the
      * given name.
      */
-    public static IPv4RangeAggregatorBuilder ipRange(String name) {
-        return new IPv4RangeAggregatorBuilder(name);
+    public static IpRangeAggregatorBuilder ipRange(String name) {
+        return new IpRangeAggregatorBuilder(name);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java
new file mode 100644
index 00000000000..87a3e917ddd
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.range;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.LeafBucketCollector;
+import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
+import org.elasticsearch.search.aggregations.bucket.BucketsAggregator;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
+
+/** A range aggregator for values that are stored in SORTED_SET doc values. */
+public final class BinaryRangeAggregator extends BucketsAggregator {
+
+    public static class Range {
+
+        final String key;
+        final BytesRef from, to;
+
+        public Range(String key, BytesRef from, BytesRef to) {
+            this.key = key;
+            this.from = from;
+            this.to = to;
+        }
+    }
+
+    static final Comparator<Range> RANGE_COMPARATOR = (a, b) -> {
+        int cmp = compare(a.from, b.from, 1);
+        if (cmp == 0) {
+            cmp = compare(a.to, b.to, -1);
+        }
+        return cmp;
+    };
+
+    private static int compare(BytesRef a, BytesRef b, int m) {
+        return a == null
+                ? b == null ? 0 : -m
+                : b == null ? m : a.compareTo(b);
+    }
+
+    final ValuesSource.Bytes valuesSource;
+    final DocValueFormat format;
+    final boolean keyed;
+    final Range[] ranges;
+
+    public BinaryRangeAggregator(String name, AggregatorFactories factories,
+            ValuesSource.Bytes valuesSource, DocValueFormat format,
+            List<Range> ranges, boolean keyed, AggregationContext aggregationContext,
+            Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+            Map<String, Object> metaData) throws IOException {
+        super(name, factories, aggregationContext, parent, pipelineAggregators, metaData);
+        this.valuesSource = valuesSource;
+        this.format = format;
+        this.keyed = keyed;
+        this.ranges = ranges.toArray(new Range[0]);
+        Arrays.sort(this.ranges, RANGE_COMPARATOR);
+
+    }
+
+    @Override
+    public boolean needsScores() {
+        return (valuesSource != null && valuesSource.needsScores()) || super.needsScores();
+    }
+
+    @Override
+    protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException {
+        if (valuesSource == null) {
+            return LeafBucketCollector.NO_OP_COLLECTOR;
+        }
+        if (valuesSource instanceof ValuesSource.Bytes.WithOrdinals) {
+            SortedSetDocValues values = ((ValuesSource.Bytes.WithOrdinals) valuesSource).ordinalsValues(ctx);
+            return new SortedSetRangeLeafCollector(values, ranges, sub) {
+                @Override
+                protected void doCollect(LeafBucketCollector sub, int doc, long bucket) throws IOException {
+                    collectBucket(sub, doc, bucket);
+                }
+            };
+        }
+        throw new IllegalArgumentException("binary range aggregation expects a values source that supports ordinals");
+    }
+
+    static abstract class SortedSetRangeLeafCollector extends LeafBucketCollectorBase {
+
+        final long[] froms, tos, maxTos;
+        final SortedSetDocValues values;
+        final LeafBucketCollector sub;
+
+        SortedSetRangeLeafCollector(SortedSetDocValues values,
+                Range[] ranges, LeafBucketCollector sub) {
+            super(sub, values);
+            for (int i = 1; i < ranges.length; ++i) {
+                if (RANGE_COMPARATOR.compare(ranges[i-1], ranges[i]) > 0) {
+                    throw new IllegalArgumentException("Ranges must be sorted");
+                }
+            }
+            this.values = values;
+            this.sub = sub;
+            froms = new long[ranges.length];
+            tos = new long[ranges.length]; // inclusive
+            maxTos = new long[ranges.length];
+            for (int i = 0; i < ranges.length; ++i) {
+                if (ranges[i].from == null) {
+                    froms[i] = 0;
+                } else {
+                    froms[i] = values.lookupTerm(ranges[i].from);
+                    if (froms[i] < 0) {
+                        froms[i] = -1 - froms[i];
+                    }
+                }
+                if (ranges[i].to == null) {
+                    tos[i] = values.getValueCount() - 1;
+                } else {
+                    long ord = values.lookupTerm(ranges[i].to);
+                    if (ord < 0) {
+                        tos[i] = -2 - ord;
+                    } else {
+                        tos[i] = ord - 1;
+                    }
+                }
+            }
+            maxTos[0] = tos[0];
+            for (int i = 1; i < tos.length; ++i) {
+                maxTos[i] = Math.max(maxTos[i-1], tos[i]);
+            }
+        }
+
+        @Override
+        public void collect(int doc, long bucket) throws IOException {
+            values.setDocument(doc);
+            int lo = 0;
+            for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) {
+                lo = collect(doc, ord, bucket, lo);
+            }
+        }
+
+        private int collect(int doc, long ord, long bucket, int lowBound) throws IOException {
+            int lo = lowBound, hi = froms.length - 1; // all candidates are between these indexes
+            int mid = (lo + hi) >>> 1;
+            while (lo <= hi) {
+                if (ord < froms[mid]) {
+                    hi = mid - 1;
+                } else if (ord > maxTos[mid]) {
+                    lo = mid + 1;
+                } else {
+                    break;
+                }
+                mid = (lo + hi) >>> 1;
+            }
+            if (lo > hi) return lo; // no potential candidate
+
+            // binary search the lower bound
+            int startLo = lo, startHi = mid;
+            while (startLo <= startHi) {
+                final int startMid = (startLo + startHi) >>> 1;
+                if (ord > maxTos[startMid]) {
+                    startLo = startMid + 1;
+                } else {
+                    startHi = startMid - 1;
+                }
+            }
+
+            // binary search the upper bound
+            int endLo = mid, endHi = hi;
+            while (endLo <= endHi) {
+                final int endMid = (endLo + endHi) >>> 1;
+                if (ord < froms[endMid]) {
+                    endHi = endMid - 1;
+                } else {
+                    endLo = endMid + 1;
+                }
+            }
+
+            assert startLo == lowBound || ord > maxTos[startLo - 1];
+            assert endHi == froms.length - 1 || ord < froms[endHi + 1];
+
+            for (int i = startLo; i <= endHi; ++i) {
+                if (ord <= tos[i]) {
+                    doCollect(sub, doc, bucket * froms.length + i);
+                }
+            }
+
+            return endHi + 1;
+        }
+
+        protected abstract void doCollect(LeafBucketCollector sub, int doc, long bucket) throws IOException;
+    }
+
+    @Override
+    public InternalAggregation buildAggregation(long bucket) throws IOException {
+        InternalBinaryRange.Bucket[] buckets = new InternalBinaryRange.Bucket[ranges.length];
+        for (int i = 0; i < buckets.length; ++i) {
+            long bucketOrd = bucket * ranges.length + i;
+            buckets[i] = new InternalBinaryRange.Bucket(format, keyed,
+                    ranges[i].key, ranges[i].from, ranges[i].to,
+                    bucketDocCount(bucketOrd), bucketAggregations(bucketOrd));
+        }
+        return new InternalBinaryRange(name, format, keyed, buckets, pipelineAggregators(), metaData());
+    }
+
+    @Override
+    public InternalAggregation buildEmptyAggregation() {
+        return new InternalBinaryRange(name, format, keyed, new InternalBinaryRange.Bucket[0], pipelineAggregators(), metaData());
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorFactory.java
new file mode 100644
index 00000000000..fda822cf11b
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorFactory.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.range;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.InternalAggregation.Type;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
+import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+
+public class BinaryRangeAggregatorFactory
+        extends ValuesSourceAggregatorFactory<ValuesSource.Bytes, BinaryRangeAggregatorFactory> {
+
+    private final List<BinaryRangeAggregator.Range> ranges;
+    private final boolean keyed;
+
+    public BinaryRangeAggregatorFactory(String name, Type type,
+            ValuesSourceConfig<ValuesSource.Bytes> config,
+            List<BinaryRangeAggregator.Range> ranges, boolean keyed,
+            AggregationContext context,
+            AggregatorFactory<?> parent, Builder subFactoriesBuilder,
+            Map<String, Object> metaData) throws IOException {
+        super(name, type, config, context, parent, subFactoriesBuilder, metaData);
+        this.ranges = ranges;
+        this.keyed = keyed;
+    }
+
+    @Override
+    protected Aggregator createUnmapped(Aggregator parent,
+            List<PipelineAggregator> pipelineAggregators,
+            Map<String, Object> metaData) throws IOException {
+        return new BinaryRangeAggregator(name, factories, null, config.format(),
+                ranges, keyed, context, parent, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected Aggregator doCreateInternal(ValuesSource.Bytes valuesSource,
+            Aggregator parent,
+            boolean collectsFromSingleBucket,
+            List<PipelineAggregator> pipelineAggregators,
+            Map<String, Object> metaData) throws IOException {
+        return new BinaryRangeAggregator(name, factories, valuesSource, config.format(),
+                ranges, keyed, context, parent, pipelineAggregators, metaData);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
new file mode 100644
index 00000000000..f3f1763113e
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
@@ -0,0 +1,311 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.range;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.AggregationStreams;
+import org.elasticsearch.search.aggregations.Aggregations;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.InternalAggregations;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.bucket.BucketStreamContext;
+import org.elasticsearch.search.aggregations.bucket.BucketStreams;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+/** A range aggregation for data that is encoded in doc values using a binary representation. */
+public final class InternalBinaryRange
+        extends InternalMultiBucketAggregation<InternalBinaryRange, InternalBinaryRange.Bucket>
+        implements Range {
+
+    public static final Type TYPE = new Type("binary_range");
+
+    private final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+        @Override
+        public InternalBinaryRange readResult(StreamInput in) throws IOException {
+            InternalBinaryRange range = new InternalBinaryRange();
+            range.readFrom(in);
+            return range;
+        }
+    };
+
+    private final static BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
+        @Override
+        public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
+            Bucket bucket = new Bucket(context.format(), context.keyed());
+            bucket.readFrom(in);
+            return bucket;
+        }
+
+        @Override
+        public BucketStreamContext getBucketStreamContext(Bucket bucket) {
+            BucketStreamContext context = new BucketStreamContext();
+            context.format(bucket.format);
+            context.keyed(bucket.keyed);
+            return context;
+        }
+    };
+
+    public static void registerStream() {
+        AggregationStreams.registerStream(STREAM, TYPE.stream());
+        BucketStreams.registerStream(BUCKET_STREAM, TYPE.stream());
+    }
+
+    public static class Bucket extends InternalMultiBucketAggregation.InternalBucket implements Range.Bucket {
+
+        private final transient DocValueFormat format;
+        private final transient boolean keyed;
+        private String key;
+        private BytesRef from, to;
+        private long docCount;
+        private InternalAggregations aggregations;
+
+        public Bucket(DocValueFormat format, boolean keyed, String key, BytesRef from, BytesRef to,
+                long docCount, InternalAggregations aggregations) {
+            this(format, keyed);
+            this.key = key;
+            this.from = from;
+            this.to = to;
+            this.docCount = docCount;
+            this.aggregations = aggregations;
+        }
+
+        // for serialization
+        private Bucket(DocValueFormat format, boolean keyed) {
+            this.format = format;
+            this.keyed = keyed;
+        }
+
+        @Override
+        public Object getKey() {
+            return key;
+        }
+
+        @Override
+        public String getKeyAsString() {
+            return key;
+        }
+
+        @Override
+        public long getDocCount() {
+            return docCount;
+        }
+
+        @Override
+        public Aggregations getAggregations() {
+            return aggregations;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            String key = this.key;
+            if (keyed) {
+                if (key == null) {
+                    StringBuilder keyBuilder = new StringBuilder();
+                    keyBuilder.append(from == null ? "*" : format.format(from));
+                    keyBuilder.append("-");
+                    keyBuilder.append(to == null ? "*" : format.format(to));
+                    key = keyBuilder.toString();
+                }
+                builder.startObject(key);
+            } else {
+                builder.startObject();
+                if (key != null) {
+                    builder.field(CommonFields.KEY, key);
+                }
+            }
+            if (from != null) {
+                builder.field(CommonFields.FROM, getFrom());
+            }
+            if (to != null) {
+                builder.field(CommonFields.TO, getTo());
+            }
+            builder.field(CommonFields.DOC_COUNT, docCount);
+            aggregations.toXContentInternal(builder, params);
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            key = in.readOptionalString();
+            if (in.readBoolean()) {
+                from = in.readBytesRef();
+            } else {
+                from = null;
+            }
+            if (in.readBoolean()) {
+                to = in.readBytesRef();
+            } else {
+                to = null;
+            }
+            docCount = in.readLong();
+            aggregations = InternalAggregations.readAggregations(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeOptionalString(key);
+            out.writeBoolean(from != null);
+            if (from != null) {
+                out.writeBytesRef(from);
+            }
+            out.writeBoolean(to != null);
+            if (to != null) {
+                out.writeBytesRef(to);
+            }
+            out.writeLong(docCount);
+            aggregations.writeTo(out);
+        }
+
+        @Override
+        public Object getFrom() {
+            return getFromAsString();
+        }
+
+        @Override
+        public String getFromAsString() {
+            return from == null ? null : format.format(from);
+        }
+
+        @Override
+        public Object getTo() {
+            return getToAsString();
+        }
+
+        @Override
+        public String getToAsString() {
+            return to == null ? null : format.format(to);
+        }
+
+    }
+
+    private DocValueFormat format;
+    private boolean keyed;
+    private Bucket[] buckets;
+
+    public InternalBinaryRange(String name, DocValueFormat format, boolean keyed, Bucket[] buckets,
+            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        super(name, pipelineAggregators, metaData);
+        this.format = format;
+        this.keyed = keyed;
+        this.buckets = buckets;
+    }
+
+    private InternalBinaryRange() {} // for serialization
+
+    @Override
+    public List<Range.Bucket> getBuckets() {
+        return Arrays.asList(buckets);
+    }
+
+    @Override
+    public InternalBinaryRange create(List<Bucket> buckets) {
+        return new InternalBinaryRange(name, format, keyed, buckets.toArray(new Bucket[0]),
+                pipelineAggregators(), metaData);
+    }
+
+    @Override
+    public Bucket createBucket(InternalAggregations aggregations, Bucket prototype) {
+        return new Bucket(format, keyed, prototype.key, prototype.from, prototype.to, prototype.docCount, aggregations);
+    }
+
+    @Override
+    public Type type() {
+        return TYPE;
+    }
+
+    @Override
+    public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+        long[] docCounts = new long[buckets.length];
+        InternalAggregations[][] aggs = new InternalAggregations[buckets.length][];
+        for (int i = 0; i < aggs.length; ++i) {
+            aggs[i] = new InternalAggregations[aggregations.size()];
+        }
+        for (int i = 0; i < aggregations.size(); ++i) {
+            InternalBinaryRange range = (InternalBinaryRange) aggregations.get(i);
+            if (range.buckets.length != buckets.length) {
+                throw new IllegalStateException("Expected " + buckets.length + " buckets, but got " + range.buckets.length);
+            }
+            for (int j = 0; j < buckets.length; ++j) {
+                Bucket bucket = range.buckets[j];
+                docCounts[j] += bucket.docCount;
+                aggs[j][i] = bucket.aggregations;
+            }
+        }
+        Bucket[] buckets = new Bucket[this.buckets.length];
+        for (int i = 0; i < buckets.length; ++i) {
+            Bucket b = this.buckets[i];
+            buckets[i] = new Bucket(format, keyed, b.key, b.from, b.to, docCounts[i],
+                    InternalAggregations.reduce(Arrays.asList(aggs[i]), reduceContext));
+        }
+        return new InternalBinaryRange(name, format, keyed, buckets, pipelineAggregators(), metaData);
+    }
+
+    @Override
+    public XContentBuilder doXContentBody(XContentBuilder builder,
+            Params params) throws IOException {
+        if (keyed) {
+            builder.startObject(CommonFields.BUCKETS);
+        } else {
+            builder.startArray(CommonFields.BUCKETS);
+        }
+        for (Bucket range : buckets) {
+            range.toXContent(builder, params);
+        }
+        if (keyed) {
+            builder.endObject();
+        } else {
+            builder.endArray();
+        }
+        return builder;
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(format);
+        out.writeBoolean(keyed);
+        out.writeVInt(buckets.length);
+        for (Bucket bucket : buckets) {
+            bucket.writeTo(out);
+        }
+    }
+
+    @Override
+    protected void doReadFrom(StreamInput in) throws IOException {
+        format = in.readNamedWriteable(DocValueFormat.class);
+        keyed = in.readBoolean();
+        Bucket[] buckets = new Bucket[in.readVInt()];
+        for (int i = 0; i < buckets.length; ++i) {
+            Bucket bucket = new Bucket(format, keyed);
+            bucket.readFrom(in);
+            buckets[i] = bucket;
+        }
+        this.buckets = buckets;
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeAggregatorBuilder.java
new file mode 100644
index 00000000000..c56a2952f8d
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeAggregatorBuilder.java
@@ -0,0 +1,330 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.range.ip;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.lucene.document.InetAddressPoint;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.network.InetAddresses;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.bucket.range.BinaryRangeAggregator;
+import org.elasticsearch.search.aggregations.bucket.range.BinaryRangeAggregatorFactory;
+import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator;
+import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.aggregations.support.ValueType;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
+import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+
+
+public final class IpRangeAggregatorBuilder
+        extends ValuesSourceAggregatorBuilder<ValuesSource.Bytes, IpRangeAggregatorBuilder> {
+    private static final String NAME = "ip_range";
+    public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
+    private static final InternalAggregation.Type TYPE = new InternalAggregation.Type(NAME);
+
+    public static class Range implements ToXContent {
+        private final String key;
+        private final String from;
+        private final String to;
+
+        Range(String key, String from, String to) {
+            if (from != null) {
+                InetAddresses.forString(from);
+            }
+            if (to != null) {
+                InetAddresses.forString(to);
+            }
+            this.key = key;
+            this.from = from;
+            this.to = to;
+        }
+
+        Range(String key, String mask) {
+            String[] splits = mask.split("/");
+            if (splits.length != 2) {
+                throw new IllegalArgumentException("Expected [ip/prefix_length] but got [" + mask
+                        + "], which contains zero or more than one [/]");
+            }
+            InetAddress value = InetAddresses.forString(splits[0]);
+            int prefixLength = Integer.parseInt(splits[1]);
+            // copied from InetAddressPoint.newPrefixQuery
+            if (prefixLength < 0 || prefixLength > 8 * value.getAddress().length) {
+                throw new IllegalArgumentException("illegal prefixLength [" + prefixLength
+                        + "] in [" + mask + "]. Must be 0-32 for IPv4 ranges, 0-128 for IPv6 ranges");
+            }
+            // create the lower value by zeroing out the host portion, upper value by filling it with all ones.
+            byte lower[] = value.getAddress();
+            byte upper[] = value.getAddress();
+            for (int i = prefixLength; i < 8 * lower.length; i++) {
+                int m = 1 << (7 - (i & 7));
+                lower[i >> 3] &= ~m;
+                upper[i >> 3] |= m;
+            }
+            this.key = key;
+            try {
+                this.from = InetAddresses.toAddrString(InetAddress.getByAddress(lower));
+                this.to = InetAddresses.toAddrString(InetAddress.getByAddress(upper));
+            } catch (UnknownHostException bogus) {
+                throw new AssertionError(bogus);
+            }
+        }
+
+        private Range(StreamInput in) throws IOException {
+            this.key = in.readOptionalString();
+            this.from = in.readOptionalString();
+            this.to = in.readOptionalString();
+        }
+
+        void writeTo(StreamOutput out) throws IOException {
+            out.writeOptionalString(key);
+            out.writeOptionalString(from);
+            out.writeOptionalString(to);
+        }
+
+        public String getKey() {
+            return key;
+        }
+
+        public String getFrom() {
+            return from;
+        }
+
+        public String getTo() {
+            return to;
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null || getClass() != obj.getClass()) {
+                return false;
+            }
+            Range that = (Range) obj;
+            return Objects.equals(key, that.key)
+                    && Objects.equals(from, that.from)
+                    && Objects.equals(to, that.to);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(getClass(), key, from, to);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            if (key != null) {
+                builder.field(RangeAggregator.Range.KEY_FIELD.getPreferredName(), key);
+            }
+            if (from != null) {
+                builder.field(RangeAggregator.Range.FROM_FIELD.getPreferredName(), from);
+            }
+            if (to != null) {
+                builder.field(RangeAggregator.Range.TO_FIELD.getPreferredName(), to);
+            }
+            builder.endObject();
+            return builder;
+        }
+    }
+
+    private boolean keyed = false;
+    private List<Range> ranges = new ArrayList<>();
+
+    public IpRangeAggregatorBuilder(String name) {
+        super(name, TYPE, ValuesSourceType.BYTES, ValueType.IP);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    public IpRangeAggregatorBuilder keyed(boolean keyed) {
+        this.keyed = keyed;
+        return this;
+    }
+
+    public boolean keyed() {
+        return keyed;
+    }
+
+    /** Get the current list or ranges that are configured on this aggregation. */
+    public List<Range> getRanges() {
+        return Collections.unmodifiableList(ranges);
+    }
+
+    /** Add a new {@link Range} to this aggregation. */
+    public IpRangeAggregatorBuilder addRange(Range range) {
+        ranges.add(range);
+        return this;
+    }
+
+    /**
+     * Add a new range to this aggregation.
+     *
+     * @param key
+     *            the key to use for this range in the response
+     * @param from
+     *            the lower bound on the distances, inclusive
+     * @param to
+     *            the upper bound on the distances, exclusive
+     */
+    public IpRangeAggregatorBuilder addRange(String key, String from, String to) {
+        addRange(new Range(key, from, to));
+        return this;
+    }
+
+    /**
+     * Add a new range to this aggregation using the CIDR notation.
+     */
+    public IpRangeAggregatorBuilder addMaskRange(String key, String mask) {
+        return addRange(new Range(key, mask));
+    }
+
+    /**
+     * Same as {@link #addMaskRange(String, String)} but uses the mask itself as
+     * a key.
+     */
+    public IpRangeAggregatorBuilder addMaskRange(String mask) {
+        return addRange(new Range(mask, mask));
+    }
+
+    /**
+     * Same as {@link #addRange(String, String, String)} but the key will be
+     * automatically generated.
+     */
+    public IpRangeAggregatorBuilder addRange(String from, String to) {
+        return addRange(null, from, to);
+    }
+
+    /**
+     * Same as {@link #addRange(String, String, String)} but there will be no
+     * lower bound.
+     */
+    public IpRangeAggregatorBuilder addUnboundedTo(String key, String to) {
+        addRange(new Range(key, null, to));
+        return this;
+    }
+
+    /**
+     * Same as {@link #addUnboundedTo(String, String)} but the key will be
+     * generated automatically.
+     */
+    public IpRangeAggregatorBuilder addUnboundedTo(String to) {
+        return addUnboundedTo(null, to);
+    }
+
+    /**
+     * Same as {@link #addRange(String, String, String)} but there will be no
+     * upper bound.
+     */
+    public IpRangeAggregatorBuilder addUnboundedFrom(String key, String from) {
+        addRange(new Range(key, from, null));
+        return this;
+    }
+
+    @Override
+    public IpRangeAggregatorBuilder script(Script script) {
+        throw new IllegalArgumentException("[ip_range] does not support scripts");
+    }
+
+    /**
+     * Same as {@link #addUnboundedFrom(String, String)} but the key will be
+     * generated automatically.
+     */
+    public IpRangeAggregatorBuilder addUnboundedFrom(String from) {
+        return addUnboundedFrom(null, from);
+    }
+
+    public IpRangeAggregatorBuilder(StreamInput in) throws IOException {
+        super(in, TYPE, ValuesSourceType.BYTES, ValueType.IP);
+        final int numRanges = in.readVInt();
+        for (int i = 0; i < numRanges; ++i) {
+            addRange(new Range(in));
+        }
+        keyed = in.readBoolean();
+    }
+
+    @Override
+    protected void innerWriteTo(StreamOutput out) throws IOException {
+        out.writeVInt(ranges.size());
+        for (Range range : ranges) {
+            range.writeTo(out);
+        }
+        out.writeBoolean(keyed);
+    }
+
+    private static BytesRef toBytesRef(String ip) {
+        if (ip == null) {
+            return null;
+        }
+        InetAddress address = InetAddresses.forString(ip);
+        byte[] bytes = InetAddressPoint.encode(address);
+        return new BytesRef(bytes);
+    }
+
+    @Override
+    protected ValuesSourceAggregatorFactory<ValuesSource.Bytes, ?> innerBuild(
+            AggregationContext context, ValuesSourceConfig<ValuesSource.Bytes> config,
+            AggregatorFactory<?> parent, Builder subFactoriesBuilder)
+                    throws IOException {
+        List<BinaryRangeAggregator.Range> ranges = new ArrayList<>();
+        for (Range range : this.ranges) {
+            ranges.add(new BinaryRangeAggregator.Range(range.key, toBytesRef(range.from), toBytesRef(range.to)));
+        }
+        return new BinaryRangeAggregatorFactory(name, TYPE, config, ranges,
+                keyed, context, parent, subFactoriesBuilder, metaData);
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        builder.field(RangeAggregator.RANGES_FIELD.getPreferredName(), ranges);
+        builder.field(RangeAggregator.KEYED_FIELD.getPreferredName(), keyed);
+        return builder;
+    }
+
+    @Override
+    protected int innerHashCode() {
+        return Objects.hash(keyed, ranges);
+    }
+
+    @Override
+    protected boolean innerEquals(Object obj) {
+        IpRangeAggregatorBuilder that = (IpRangeAggregatorBuilder) obj;
+        return keyed == that.keyed
+                && ranges.equals(that.ranges);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeParser.java
new file mode 100644
index 00000000000..64ed77d42f3
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeParser.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.range.ip;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.BytesValuesSourceParser;
+import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator;
+import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeAggregatorBuilder.Range;
+import org.elasticsearch.search.aggregations.support.ValueType;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+
+/**
+ * A parser for ip range aggregations.
+ */
+public class IpRangeParser extends BytesValuesSourceParser {
+
+    private static final ParseField MASK_FIELD = new ParseField("mask");
+
+    public IpRangeParser() {
+        super(false, false);
+    }
+
+    @Override
+    protected ValuesSourceAggregatorBuilder<ValuesSource.Bytes, ?> createFactory(
+            String aggregationName, ValuesSourceType valuesSourceType,
+            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
+        IpRangeAggregatorBuilder range = new IpRangeAggregatorBuilder(aggregationName);
+        @SuppressWarnings("unchecked")
+        Iterable<Range> ranges = (Iterable<Range>) otherOptions.get(RangeAggregator.RANGES_FIELD);
+        if (otherOptions.containsKey(RangeAggregator.RANGES_FIELD)) {
+            for (Range r : ranges) {
+                range.addRange(r);
+            }
+        }
+        if (otherOptions.containsKey(RangeAggregator.KEYED_FIELD)) {
+            range.keyed((Boolean) otherOptions.get(RangeAggregator.KEYED_FIELD));
+        }
+        return range;
+    }
+
+    private Range parseRange(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
+        String key = null;
+        String from = null;
+        String to = null;
+        String mask = null;
+
+        if (parser.currentToken() != Token.START_OBJECT) {
+            throw new ParsingException(parser.getTokenLocation(), "[ranges] must contain objects, but hit a " + parser.currentToken());
+        }
+        while (parser.nextToken() != Token.END_OBJECT) {
+            if (parser.currentToken() == Token.FIELD_NAME) {
+                continue;
+            }
+            if (parseFieldMatcher.match(parser.currentName(), RangeAggregator.Range.KEY_FIELD)) {
+                key = parser.text();
+            } else if (parseFieldMatcher.match(parser.currentName(), RangeAggregator.Range.FROM_FIELD)) {
+                from = parser.text();
+            } else if (parseFieldMatcher.match(parser.currentName(), RangeAggregator.Range.TO_FIELD)) {
+                to = parser.text();
+            } else if (parseFieldMatcher.match(parser.currentName(), MASK_FIELD)) {
+                mask = parser.text();
+            } else {
+                throw new ParsingException(parser.getTokenLocation(), "Unexpected ip range parameter: [" + parser.currentName() + "]");
+            }
+        }
+        if (mask != null) {
+            if (key == null) {
+                key = mask;
+            }
+            return new Range(key, mask);
+        } else {
+            return new Range(key, from, to);
+        }
+    }
+
+    @Override
+    protected boolean token(String aggregationName, String currentFieldName,
+            Token token, XContentParser parser,
+            ParseFieldMatcher parseFieldMatcher,
+            Map<ParseField, Object> otherOptions) throws IOException {
+        if (parseFieldMatcher.match(currentFieldName, RangeAggregator.RANGES_FIELD)) {
+            if (parser.currentToken() != Token.START_ARRAY) {
+                throw new ParsingException(parser.getTokenLocation(), "[ranges] must be passed as an array, but got a " + token);
+            }
+            List<Range> ranges = new ArrayList<>();
+            while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                Range range = parseRange(parser, parseFieldMatcher);
+                ranges.add(range);
+            }
+            otherOptions.put(RangeAggregator.RANGES_FIELD, ranges);
+            return true;
+        } else if (parseFieldMatcher.match(parser.currentName(), RangeAggregator.KEYED_FIELD)) {
+            otherOptions.put(RangeAggregator.KEYED_FIELD, parser.booleanValue());
+            return true;
+        }
+        return false;
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IPv4RangeAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IPv4RangeAggregatorBuilder.java
deleted file mode 100644
index 6bafa6d5668..00000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IPv4RangeAggregatorBuilder.java
+++ /dev/null
@@ -1,277 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.aggregations.bucket.range.ipv4;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.network.Cidrs;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
-import org.elasticsearch.search.aggregations.bucket.range.AbstractRangeBuilder;
-import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
-import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
-import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
-import org.elasticsearch.search.internal.SearchContext;
-
-import java.io.IOException;
-import java.util.Objects;
-
-public class IPv4RangeAggregatorBuilder extends AbstractRangeBuilder<IPv4RangeAggregatorBuilder, IPv4RangeAggregatorBuilder.Range> {
-    public static final String NAME = InternalIPv4Range.TYPE.name();
-    public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
-
-    public IPv4RangeAggregatorBuilder(String name) {
-        super(name, InternalIPv4Range.FACTORY);
-    }
-
-    /**
-     * Read from a stream.
-     */
-    public IPv4RangeAggregatorBuilder(StreamInput in) throws IOException {
-        super(in, InternalIPv4Range.FACTORY, Range::new);
-    }
-
-    @Override
-    public String getWriteableName() {
-        return NAME;
-    }
-
-    /**
-     * Add a new range to this aggregation.
-     *
-     * @param key
-     *            the key to use for this range in the response
-     * @param from
-     *            the lower bound on the distances, inclusive
-     * @param to
-     *            the upper bound on the distances, exclusive
-     */
-    public IPv4RangeAggregatorBuilder addRange(String key, String from, String to) {
-        addRange(new Range(key, from, to));
-        return this;
-    }
-
-    /**
-     * Same as {@link #addMaskRange(String, String)} but uses the mask itself as
-     * a key.
-     */
-    public IPv4RangeAggregatorBuilder addMaskRange(String key, String mask) {
-        return addRange(new Range(key, mask));
-    }
-
-    /**
-     * Same as {@link #addMaskRange(String, String)} but uses the mask itself as
-     * a key.
-     */
-    public IPv4RangeAggregatorBuilder addMaskRange(String mask) {
-        return addRange(new Range(mask, mask));
-    }
-
-    /**
-     * Same as {@link #addRange(String, String, String)} but the key will be
-     * automatically generated.
-     */
-    public IPv4RangeAggregatorBuilder addRange(String from, String to) {
-        return addRange(null, from, to);
-    }
-
-    /**
-     * Same as {@link #addRange(String, String, String)} but there will be no
-     * lower bound.
-     */
-    public IPv4RangeAggregatorBuilder addUnboundedTo(String key, String to) {
-        addRange(new Range(key, null, to));
-        return this;
-    }
-
-    /**
-     * Same as {@link #addUnboundedTo(String, String)} but the key will be
-     * generated automatically.
-     */
-    public IPv4RangeAggregatorBuilder addUnboundedTo(String to) {
-        return addUnboundedTo(null, to);
-    }
-
-    /**
-     * Same as {@link #addRange(String, String, String)} but there will be no
-     * upper bound.
-     */
-    public IPv4RangeAggregatorBuilder addUnboundedFrom(String key, String from) {
-        addRange(new Range(key, from, null));
-        return this;
-    }
-
-    /**
-     * Same as {@link #addUnboundedFrom(String, String)} but the key will be
-     * generated automatically.
-     */
-    public IPv4RangeAggregatorBuilder addUnboundedFrom(String from) {
-        return addUnboundedFrom(null, from);
-    }
-
-    @Override
-    protected Ipv4RangeAggregatorFactory innerBuild(AggregationContext context, ValuesSourceConfig<Numeric> config,
-            AggregatorFactory<?> parent, Builder subFactoriesBuilder) throws IOException {
-        return new Ipv4RangeAggregatorFactory(name, type, config, ranges, keyed, rangeFactory, context, parent, subFactoriesBuilder,
-                metaData);
-    }
-
-    public static class Range extends RangeAggregator.Range {
-        static final ParseField MASK_FIELD = new ParseField("mask");
-
-        private final String cidr;
-
-        public Range(String key, Double from, Double to) {
-            this(key, from, null, to, null, null);
-        }
-
-        public Range(String key, String from, String to) {
-            this(key, null, from, null, to, null);
-        }
-
-        public Range(String key, String cidr) {
-            this(key, null, null, null, null, cidr);
-        }
-
-        private Range(String key, Double from, String fromAsStr, Double to, String toAsStr, String cidr) {
-            super(key, from, fromAsStr, to, toAsStr);
-            this.cidr = cidr;
-        }
-
-        /**
-         * Read from a stream.
-         */
-        public Range(StreamInput in) throws IOException {
-            super(in);
-            cidr = in.readOptionalString();
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            out.writeOptionalString(cidr);
-        }
-
-        public String mask() {
-            return cidr;
-        }
-
-        @Override
-        public Range process(DocValueFormat parser, SearchContext context) {
-            assert parser != null;
-            Double from = this.from;
-            Double to = this.to;
-            String key = this.key;
-            if (fromAsStr != null) {
-                from = parser.parseDouble(fromAsStr, false, context.nowCallable());
-            }
-            if (toAsStr != null) {
-                to = parser.parseDouble(toAsStr, false, context.nowCallable());
-            }
-            if (cidr != null) {
-                long[] fromTo = Cidrs.cidrMaskToMinMax(cidr);
-                from = fromTo[0] == 0 ? Double.NEGATIVE_INFINITY : fromTo[0];
-                to = fromTo[1] == InternalIPv4Range.MAX_IP ? Double.POSITIVE_INFINITY : fromTo[1];
-                if (this.key == null) {
-                    key = cidr;
-                }
-            }
-            return new Range(key, from, to);
-        }
-
-        public static Range fromXContent(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
-            XContentParser.Token token;
-            String currentFieldName = null;
-            double from = Double.NEGATIVE_INFINITY;
-            String fromAsStr = null;
-            double to = Double.POSITIVE_INFINITY;
-            String toAsStr = null;
-            String key = null;
-            String cidr = null;
-            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    currentFieldName = parser.currentName();
-                } else if (token == XContentParser.Token.VALUE_NUMBER) {
-                    if (parseFieldMatcher.match(currentFieldName, FROM_FIELD)) {
-                        from = parser.doubleValue();
-                    } else if (parseFieldMatcher.match(currentFieldName, TO_FIELD)) {
-                        to = parser.doubleValue();
-                    }
-                } else if (token == XContentParser.Token.VALUE_STRING) {
-                    if (parseFieldMatcher.match(currentFieldName, FROM_FIELD)) {
-                        fromAsStr = parser.text();
-                    } else if (parseFieldMatcher.match(currentFieldName, TO_FIELD)) {
-                        toAsStr = parser.text();
-                    } else if (parseFieldMatcher.match(currentFieldName, KEY_FIELD)) {
-                        key = parser.text();
-                    } else if (parseFieldMatcher.match(currentFieldName, MASK_FIELD)) {
-                        cidr = parser.text();
-                    }
-                }
-            }
-            return new Range(key, from, fromAsStr, to, toAsStr, cidr);
-        }
-
-        @Override
-        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject();
-            if (key != null) {
-                builder.field(KEY_FIELD.getPreferredName(), key);
-            }
-            if (cidr != null) {
-                builder.field(MASK_FIELD.getPreferredName(), cidr);
-            } else {
-                if (Double.isFinite(from)) {
-                    builder.field(FROM_FIELD.getPreferredName(), from);
-                }
-                if (Double.isFinite(to)) {
-                    builder.field(TO_FIELD.getPreferredName(), to);
-                }
-                if (fromAsStr != null) {
-                    builder.field(FROM_FIELD.getPreferredName(), fromAsStr);
-                }
-                if (toAsStr != null) {
-                    builder.field(TO_FIELD.getPreferredName(), toAsStr);
-                }
-            }
-            builder.endObject();
-            return builder;
-        }
-
-        @Override
-        public int hashCode() {
-            return Objects.hash(super.hashCode(), cidr);
-        }
-
-        @Override
-        public boolean equals(Object obj) {
-            return super.equals(obj)
-                    && Objects.equals(cidr, ((Range) obj).cidr);
-        }
-
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/InternalIPv4Range.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/InternalIPv4Range.java
deleted file mode 100644
index f58df79c168..00000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/InternalIPv4Range.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.bucket.range.ipv4;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationStreams;
-import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.aggregations.bucket.BucketStreamContext;
-import org.elasticsearch.search.aggregations.bucket.BucketStreams;
-import org.elasticsearch.search.aggregations.bucket.range.InternalRange;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.support.ValueType;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-/**
- *
- */
-public class InternalIPv4Range extends InternalRange<InternalIPv4Range.Bucket, InternalIPv4Range> {
-    public static final long MAX_IP = 1L << 32;
-
-    public final static Type TYPE = new Type("ip_range", "iprange");
-
-    private final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public InternalIPv4Range readResult(StreamInput in) throws IOException {
-            InternalIPv4Range range = new InternalIPv4Range();
-            range.readFrom(in);
-            return range;
-        }
-    };
-
-    private final static BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
-        @Override
-        public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
-            Bucket buckets = new Bucket(context.keyed());
-            buckets.readFrom(in);
-            return buckets;
-        }
-
-        @Override
-        public BucketStreamContext getBucketStreamContext(Bucket bucket) {
-            BucketStreamContext context = new BucketStreamContext();
-            context.keyed(bucket.keyed());
-            return context;
-        }
-    };
-
-    public static void registerStream() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-        BucketStreams.registerStream(BUCKET_STREAM, TYPE.stream());
-    }
-
-    public static final Factory FACTORY = new Factory();
-
-    public static class Bucket extends InternalRange.Bucket {
-
-        public Bucket(boolean keyed) {
-            super(keyed, DocValueFormat.IP);
-        }
-
-        public Bucket(String key, double from, double to, long docCount, List<InternalAggregation> aggregations, boolean keyed) {
-            super(key, from, to, docCount, new InternalAggregations(aggregations), keyed, DocValueFormat.IP);
-        }
-
-        public Bucket(String key, double from, double to, long docCount, InternalAggregations aggregations, boolean keyed) {
-            super(key, from, to, docCount, aggregations, keyed, DocValueFormat.IP);
-        }
-
-        @Override
-        public String getFromAsString() {
-            double from = ((Number) this.from).doubleValue();
-            return Double.isInfinite(from) ? null : from == 0 ? null : DocValueFormat.IP.format(from);
-        }
-
-        @Override
-        public String getToAsString() {
-            double to = ((Number) this.to).doubleValue();
-            return Double.isInfinite(to) ? null : MAX_IP == to ? null : DocValueFormat.IP.format(to);
-        }
-
-        @Override
-        protected InternalRange.Factory<Bucket, ?> getFactory() {
-            return FACTORY;
-        }
-
-        boolean keyed() {
-            return keyed;
-        }
-    }
-
-    public static class Factory extends InternalRange.Factory<InternalIPv4Range.Bucket, InternalIPv4Range> {
-
-        @Override
-        public Type type() {
-            return TYPE;
-        }
-
-        @Override
-        public ValueType getValueType() {
-            return ValueType.IP;
-        }
-
-        @Override
-        public InternalIPv4Range create(String name, List<Bucket> ranges, DocValueFormat formatter, boolean keyed,
-                List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
-            return new InternalIPv4Range(name, ranges, keyed, pipelineAggregators, metaData);
-        }
-
-        @Override
-        public InternalIPv4Range create(List<Bucket> ranges, InternalIPv4Range prototype) {
-            return new InternalIPv4Range(prototype.name, ranges, prototype.keyed, prototype.pipelineAggregators(), prototype.metaData);
-        }
-
-        @Override
-        public Bucket createBucket(String key, double from, double to, long docCount, InternalAggregations aggregations, boolean keyed,
-                DocValueFormat formatter) {
-            return new Bucket(key, from, to, docCount, aggregations, keyed);
-        }
-
-        @Override
-        public Bucket createBucket(InternalAggregations aggregations, Bucket prototype) {
-            return new Bucket(prototype.getKey(), ((Number) prototype.getFrom()).doubleValue(), ((Number) prototype.getTo()).doubleValue(),
-                    prototype.getDocCount(), aggregations, prototype.getKeyed());
-        }
-    }
-
-    public InternalIPv4Range() {} // for serialization
-
-    public InternalIPv4Range(String name, List<InternalIPv4Range.Bucket> ranges, boolean keyed, List<PipelineAggregator> pipelineAggregators,
-            Map<String, Object> metaData) {
-        super(name, ranges, DocValueFormat.IP, keyed, pipelineAggregators, metaData);
-    }
-
-    @Override
-    public Type type() {
-        return TYPE;
-    }
-
-    @Override
-    public InternalRange.Factory<Bucket, InternalIPv4Range> getFactory() {
-        return FACTORY;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IpRangeParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IpRangeParser.java
deleted file mode 100644
index 901300c3bfd..00000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IpRangeParser.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.aggregations.bucket.range.ipv4;
-
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator;
-import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range;
-import org.elasticsearch.search.aggregations.bucket.range.RangeParser;
-import org.elasticsearch.search.aggregations.support.ValueType;
-import org.elasticsearch.search.aggregations.support.ValuesSourceType;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-/**
- *
- */
-public class IpRangeParser extends RangeParser {
-
-    public IpRangeParser() {
-        super(true, false, false);
-    }
-
-    @Override
-    protected Range parseRange(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException {
-        return IPv4RangeAggregatorBuilder.Range.fromXContent(parser, parseFieldMatcher);
-    }
-
-    @Override
-    protected IPv4RangeAggregatorBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        IPv4RangeAggregatorBuilder factory = new IPv4RangeAggregatorBuilder(aggregationName);
-        @SuppressWarnings("unchecked")
-        List<IPv4RangeAggregatorBuilder.Range> ranges = (List<IPv4RangeAggregatorBuilder.Range>) otherOptions
-                .get(RangeAggregator.RANGES_FIELD);
-        for (IPv4RangeAggregatorBuilder.Range range : ranges) {
-            factory.addRange(range);
-        }
-        Boolean keyed = (Boolean) otherOptions.get(RangeAggregator.KEYED_FIELD);
-        if (keyed != null) {
-            factory.keyed(keyed);
-        }
-        return factory;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/Ipv4RangeAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/Ipv4RangeAggregatorFactory.java
deleted file mode 100644
index 1c059356c5d..00000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/Ipv4RangeAggregatorFactory.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.aggregations.bucket.range.ipv4;
-
-import org.elasticsearch.search.aggregations.AggregatorFactories;
-import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.bucket.range.AbstractRangeAggregatorFactory;
-import org.elasticsearch.search.aggregations.bucket.range.InternalRange.Factory;
-import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
-import org.elasticsearch.search.aggregations.support.AggregationContext;
-import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-public class Ipv4RangeAggregatorFactory
-        extends AbstractRangeAggregatorFactory<Ipv4RangeAggregatorFactory, IPv4RangeAggregatorBuilder.Range> {
-
-    public Ipv4RangeAggregatorFactory(String name, Type type, ValuesSourceConfig<Numeric> config,
-            List<IPv4RangeAggregatorBuilder.Range> ranges, boolean keyed, Factory<?, ?> rangeFactory, AggregationContext context,
-            AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder, Map<String, Object> metaData) throws IOException {
-        super(name, type, config, ranges, keyed, rangeFactory, context, parent, subFactoriesBuilder, metaData);
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java
index 021cdfb3a00..51f336a3422 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java
@@ -70,12 +70,7 @@ public enum ValueType implements Writeable {
             return true;
         }
     },
-    IP((byte) 6, "ip", "ip", ValuesSourceType.NUMERIC, IndexNumericFieldData.class, DocValueFormat.IP) {
-        @Override
-        public boolean isNumeric() {
-            return true;
-        }
-    },
+    IP((byte) 6, "ip", "ip", ValuesSourceType.BYTES, IndexFieldData.class, DocValueFormat.IP),
     NUMERIC((byte) 7, "numeric", "numeric", ValuesSourceType.NUMERIC, IndexNumericFieldData.class, DocValueFormat.RAW) {
         @Override
         public boolean isNumeric() {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
index 4af87c982e1..2cd25e0fec7 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
@@ -86,9 +86,10 @@ public abstract class BaseAggregationTestCase<AB extends AggregatorBuilder<AB>>
     protected static final String DOUBLE_FIELD_NAME = "mapped_double";
     protected static final String BOOLEAN_FIELD_NAME = "mapped_boolean";
     protected static final String DATE_FIELD_NAME = "mapped_date";
+    protected static final String IP_FIELD_NAME = "mapped_ip";
     protected static final String OBJECT_FIELD_NAME = "mapped_object";
     protected static final String[] mappedFieldNames = new String[]{STRING_FIELD_NAME, INT_FIELD_NAME,
-            DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, OBJECT_FIELD_NAME};
+            DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, IP_FIELD_NAME, OBJECT_FIELD_NAME};
 
     private static Injector injector;
     private static Index index;
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IPv4RangeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IPv4RangeTests.java
deleted file mode 100644
index c72e220cf5e..00000000000
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IPv4RangeTests.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.aggregations.bucket;
-
-import org.elasticsearch.common.network.Cidrs;
-import org.elasticsearch.index.mapper.ip.LegacyIpFieldMapper;
-import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
-import org.elasticsearch.search.aggregations.bucket.range.ipv4.IPv4RangeAggregatorBuilder;
-import org.elasticsearch.search.aggregations.bucket.range.ipv4.IPv4RangeAggregatorBuilder.Range;
-
-public class IPv4RangeTests extends BaseAggregationTestCase<IPv4RangeAggregatorBuilder> {
-
-    @Override
-    protected IPv4RangeAggregatorBuilder createTestAggregatorBuilder() {
-        int numRanges = randomIntBetween(1, 10);
-        IPv4RangeAggregatorBuilder factory = new IPv4RangeAggregatorBuilder("foo");
-        for (int i = 0; i < numRanges; i++) {
-            String key = null;
-            if (randomBoolean()) {
-                key = randomAsciiOfLengthBetween(1, 20);
-            }
-            if (randomBoolean()) {
-                double from = randomBoolean() ? Double.NEGATIVE_INFINITY : randomIntBetween(Integer.MIN_VALUE, Integer.MAX_VALUE - 1000);
-                double to = randomBoolean() ? Double.POSITIVE_INFINITY
-                        : (Double.isInfinite(from) ? randomIntBetween(Integer.MIN_VALUE, Integer.MAX_VALUE)
-                                : randomIntBetween((int) from, Integer.MAX_VALUE));
-                if (randomBoolean()) {
-                    factory.addRange(new Range(key, from, to));
-                } else {
-                    String fromAsStr = Double.isInfinite(from) ? null : LegacyIpFieldMapper.longToIp((long) from);
-                    String toAsStr = Double.isInfinite(to) ? null : LegacyIpFieldMapper.longToIp((long) to);
-                    factory.addRange(new Range(key, fromAsStr, toAsStr));
-                }
-            } else {
-                int mask = randomInt(32);
-                long ipAsLong = randomIntBetween(0, Integer.MAX_VALUE);
-
-                long blockSize = 1L << (32 - mask);
-                ipAsLong = ipAsLong - (ipAsLong & (blockSize - 1));
-                String cidr = Cidrs.createCIDR(ipAsLong, mask);
-                factory.addRange(new Range(key, cidr));
-            }
-        }
-        factory.field(INT_FIELD_NAME);
-        if (randomBoolean()) {
-            factory.keyed(randomBoolean());
-        }
-        if (randomBoolean()) {
-            factory.missing(randomIntBetween(0, 10));
-        }
-        return factory;
-    }
-
-}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java
new file mode 100644
index 00000000000..e89db4b02a1
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java
@@ -0,0 +1,268 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
+import static org.hamcrest.Matchers.containsString;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.inject.internal.Nullable;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.AbstractSearchScript;
+import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.NativeScriptFactory;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptModule;
+import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.AggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.Range;
+import org.elasticsearch.test.ESIntegTestCase;
+
+@ESIntegTestCase.SuiteScopeTestCase
+public class IpRangeIT extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(DummyScriptPlugin.class);
+    }
+
+    @Override
+    public void setupSuiteScopeCluster() throws Exception {
+        assertAcked(prepareCreate("idx")
+                .addMapping("type", "ip", "type=ip", "ips", "type=ip"));
+
+        indexRandom(true,
+                client().prepareIndex("idx", "type", "1").setSource(
+                        "ip", "192.168.1.7",
+                        "ips", Arrays.asList("192.168.0.13", "192.168.1.2")),
+                client().prepareIndex("idx", "type", "2").setSource(
+                        "ip", "192.168.1.10",
+                        "ips", Arrays.asList("192.168.1.25", "192.168.1.28")),
+                client().prepareIndex("idx", "type", "3").setSource(
+                        "ip", "2001:db8::ff00:42:8329",
+                        "ips", Arrays.asList("2001:db8::ff00:42:8329", "2001:db8::ff00:42:8380")));
+
+        assertAcked(prepareCreate("idx_unmapped"));
+    }
+
+    public void testSingleValuedField() {
+        SearchResponse rsp = client().prepareSearch("idx").addAggregation(
+                AggregationBuilders.ipRange("my_range")
+                    .field("ip")
+                    .addUnboundedTo("192.168.1.0")
+                    .addRange("192.168.1.0", "192.168.1.10")
+                    .addUnboundedFrom("192.168.1.10")).get();
+        assertSearchResponse(rsp);
+        Range range = rsp.getAggregations().get("my_range");
+        assertEquals(3, range.getBuckets().size());
+
+        Range.Bucket bucket1 = range.getBuckets().get(0);
+        assertNull(bucket1.getFrom());
+        assertEquals("192.168.1.0", bucket1.getTo());
+        assertEquals(0, bucket1.getDocCount());
+
+        Range.Bucket bucket2 = range.getBuckets().get(1);
+        assertEquals("192.168.1.0", bucket2.getFrom());
+        assertEquals("192.168.1.10", bucket2.getTo());
+        assertEquals(1, bucket2.getDocCount());
+
+        Range.Bucket bucket3 = range.getBuckets().get(2);
+        assertEquals("192.168.1.10", bucket3.getFrom());
+        assertNull(bucket3.getTo());
+        assertEquals(2, bucket3.getDocCount());
+    }
+
+    public void testMultiValuedField() {
+        SearchResponse rsp = client().prepareSearch("idx").addAggregation(
+                AggregationBuilders.ipRange("my_range")
+                    .field("ips")
+                    .addUnboundedTo("192.168.1.0")
+                    .addRange("192.168.1.0", "192.168.1.10")
+                    .addUnboundedFrom("192.168.1.10")).get();
+        assertSearchResponse(rsp);
+        Range range = rsp.getAggregations().get("my_range");
+        assertEquals(3, range.getBuckets().size());
+
+        Range.Bucket bucket1 = range.getBuckets().get(0);
+        assertNull(bucket1.getFrom());
+        assertEquals("192.168.1.0", bucket1.getTo());
+        assertEquals(1, bucket1.getDocCount());
+
+        Range.Bucket bucket2 = range.getBuckets().get(1);
+        assertEquals("192.168.1.0", bucket2.getFrom());
+        assertEquals("192.168.1.10", bucket2.getTo());
+        assertEquals(1, bucket2.getDocCount());
+
+        Range.Bucket bucket3 = range.getBuckets().get(2);
+        assertEquals("192.168.1.10", bucket3.getFrom());
+        assertNull(bucket3.getTo());
+        assertEquals(2, bucket3.getDocCount());
+    }
+
+    public void testIpMask() {
+        SearchResponse rsp = client().prepareSearch("idx").addAggregation(
+                AggregationBuilders.ipRange("my_range")
+                    .field("ips")
+                    .addMaskRange("::/0")
+                    .addMaskRange("0.0.0.0/0")
+                    .addMaskRange("2001:db8::/64")).get();
+        assertSearchResponse(rsp);
+        Range range = rsp.getAggregations().get("my_range");
+        assertEquals(3, range.getBuckets().size());
+
+        Range.Bucket bucket1 = range.getBuckets().get(0);
+        assertEquals("::/0", bucket1.getKey());
+        assertEquals(3, bucket1.getDocCount());
+
+        Range.Bucket bucket2 = range.getBuckets().get(1);
+        assertEquals("0.0.0.0/0", bucket2.getKey());
+        assertEquals(2, bucket2.getDocCount());
+
+        Range.Bucket bucket3 = range.getBuckets().get(2);
+        assertEquals("2001:db8::/64", bucket3.getKey());
+        assertEquals(1, bucket3.getDocCount());
+    }
+
+    public void testPartiallyUnmapped() {
+        SearchResponse rsp = client().prepareSearch("idx", "idx_unmapped").addAggregation(
+                AggregationBuilders.ipRange("my_range")
+                    .field("ip")
+                    .addUnboundedTo("192.168.1.0")
+                    .addRange("192.168.1.0", "192.168.1.10")
+                    .addUnboundedFrom("192.168.1.10")).get();
+        assertSearchResponse(rsp);
+        Range range = rsp.getAggregations().get("my_range");
+        assertEquals(3, range.getBuckets().size());
+
+        Range.Bucket bucket1 = range.getBuckets().get(0);
+        assertNull(bucket1.getFrom());
+        assertEquals("192.168.1.0", bucket1.getTo());
+        assertEquals(0, bucket1.getDocCount());
+
+        Range.Bucket bucket2 = range.getBuckets().get(1);
+        assertEquals("192.168.1.0", bucket2.getFrom());
+        assertEquals("192.168.1.10", bucket2.getTo());
+        assertEquals(1, bucket2.getDocCount());
+
+        Range.Bucket bucket3 = range.getBuckets().get(2);
+        assertEquals("192.168.1.10", bucket3.getFrom());
+        assertNull(bucket3.getTo());
+        assertEquals(2, bucket3.getDocCount());
+    }
+
+    public void testUnmapped() {
+        SearchResponse rsp = client().prepareSearch("idx_unmapped").addAggregation(
+                AggregationBuilders.ipRange("my_range")
+                    .field("ip")
+                    .addUnboundedTo("192.168.1.0")
+                    .addRange("192.168.1.0", "192.168.1.10")
+                    .addUnboundedFrom("192.168.1.10")).get();
+        assertSearchResponse(rsp);
+        Range range = rsp.getAggregations().get("my_range");
+        assertEquals(3, range.getBuckets().size());
+
+        Range.Bucket bucket1 = range.getBuckets().get(0);
+        assertNull(bucket1.getFrom());
+        assertEquals("192.168.1.0", bucket1.getTo());
+        assertEquals(0, bucket1.getDocCount());
+
+        Range.Bucket bucket2 = range.getBuckets().get(1);
+        assertEquals("192.168.1.0", bucket2.getFrom());
+        assertEquals("192.168.1.10", bucket2.getTo());
+        assertEquals(0, bucket2.getDocCount());
+
+        Range.Bucket bucket3 = range.getBuckets().get(2);
+        assertEquals("192.168.1.10", bucket3.getFrom());
+        assertNull(bucket3.getTo());
+        assertEquals(0, bucket3.getDocCount());
+    }
+
+    public void testRejectsScript() {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> client().prepareSearch("idx").addAggregation(
+                        AggregationBuilders.ipRange("my_range")
+                        .script(new Script(DummyScript.NAME, ScriptType.INLINE, "native", Collections.emptyMap())) ).get());
+        assertThat(e.getMessage(), containsString("[ip_range] does not support scripts"));
+    }
+
+    public void testRejectsValueScript() {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> client().prepareSearch("idx").addAggregation(
+                        AggregationBuilders.ipRange("my_range")
+                        .field("ip")
+                        .script(new Script(DummyScript.NAME, ScriptType.INLINE, "native", Collections.emptyMap())) ).get());
+        assertThat(e.getMessage(), containsString("[ip_range] does not support scripts"));
+    }
+
+    public static class DummyScriptPlugin extends Plugin {
+
+        @Override
+        public String name() {
+            return "DummyScriptPlugin";
+        }
+
+        @Override
+        public String description() {
+            return "A mock script plugin.";
+        }
+
+        public void onModule(ScriptModule module) {
+            module.registerScript(DummyScript.NAME, DummyScriptFactory.class);
+        }
+    }
+
+    public static class DummyScriptFactory implements NativeScriptFactory {
+        public DummyScriptFactory() {}
+        @Override
+        public ExecutableScript newScript(@Nullable Map<String, Object> params) {
+            return new DummyScript();
+        }
+        @Override
+        public boolean needsScores() {
+            return false;
+        }
+    }
+
+    private static class DummyScript extends AbstractSearchScript {
+
+        public static final String NAME = "dummy";
+
+        @Override
+        public Object run() {
+            return null;
+        }
+    }
+
+    public static void main(String[] args) {
+        AggregatorBuilder<?> aggregation =
+                AggregationBuilders
+                        .ipRange("agg")
+                        .field("ip")
+                        .addUnboundedTo("192.168.1.0")             // from -infinity to 192.168.1.0 (excluded)
+                        .addRange("192.168.1.0", "192.168.2.0")    // from 192.168.1.0 to 192.168.2.0 (excluded)
+                        .addUnboundedFrom("192.168.2.0");          // from 192.168.2.0 to +infinity
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeTests.java
new file mode 100644
index 00000000000..db31f576e0c
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeTests.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+import org.elasticsearch.common.network.NetworkAddress;
+import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
+import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeAggregatorBuilder;
+
+public class IpRangeTests extends BaseAggregationTestCase<IpRangeAggregatorBuilder> {
+
+    private static String randomIp(boolean v4) {
+        try {
+            if (v4) {
+                byte[] ipv4 = new byte[4];
+                random().nextBytes(ipv4);
+                return NetworkAddress.format(InetAddress.getByAddress(ipv4));
+            } else {
+                byte[] ipv6 = new byte[16];
+                random().nextBytes(ipv6);
+                return NetworkAddress.format(InetAddress.getByAddress(ipv6));
+            }
+        } catch (UnknownHostException e) {
+            throw new AssertionError();
+        }
+    }
+
+    @Override
+    protected IpRangeAggregatorBuilder createTestAggregatorBuilder() {
+        int numRanges = randomIntBetween(1, 10);
+        IpRangeAggregatorBuilder factory = new IpRangeAggregatorBuilder("foo");
+        for (int i = 0; i < numRanges; i++) {
+            String key = null;
+            if (randomBoolean()) {
+                key = randomAsciiOfLengthBetween(1, 20);
+            }
+            switch (randomInt(3)) {
+            case 0:
+                boolean v4 = randomBoolean();
+                int prefixLength;
+                if (v4) {
+                    prefixLength = randomInt(32);
+                } else {
+                    prefixLength = randomInt(128);
+                }
+                factory.addMaskRange(key, randomIp(v4) + "/" + prefixLength);
+                break;
+            case 1:
+                factory.addUnboundedFrom(key, randomIp(randomBoolean()));
+                break;
+            case 2:
+                factory.addUnboundedTo(key, randomIp(randomBoolean()));
+                break;
+            case 3:
+                factory.addRange(key, randomIp(randomBoolean()), randomIp(randomBoolean()));
+                break;
+            default:
+                fail();
+            }
+        }
+        factory.field(IP_FIELD_NAME);
+        if (randomBoolean()) {
+            factory.keyed(randomBoolean());
+        }
+        if (randomBoolean()) {
+            factory.missing(randomIp(randomBoolean()));
+        }
+        return factory;
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java
new file mode 100644
index 00000000000..b3ad7771bec
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.bucket.range;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.TestUtil;
+import org.elasticsearch.search.aggregations.LeafBucketCollector;
+import org.elasticsearch.search.aggregations.bucket.range.BinaryRangeAggregator.SortedSetRangeLeafCollector;
+import org.elasticsearch.test.ESTestCase;
+
+import com.carrotsearch.hppc.LongHashSet;
+
+public class BinaryRangeAggregatorTests extends ESTestCase {
+
+    private static class FakeSortedSetDocValues extends SortedSetDocValues {
+
+        private final BytesRef[] terms;
+        long[] ords;
+        private int i;
+
+        FakeSortedSetDocValues(BytesRef[] terms) {
+            this.terms = terms;
+        }
+
+        @Override
+        public void setDocument(int docID) {
+            i = 0;
+        }
+
+        @Override
+        public long nextOrd() {
+            if (i == ords.length) {
+                return NO_MORE_ORDS;
+            }
+            return ords[i++];
+        }
+
+        @Override
+        public BytesRef lookupOrd(long ord) {
+            return terms[(int) ord];
+        }
+
+        @Override
+        public long getValueCount() {
+            return terms.length;
+        }
+
+    }
+
+    private void doTestSortedSetRangeLeafCollector(int maxNumValuesPerDoc) throws Exception {
+        final Set<BytesRef> termSet = new HashSet<>();
+        final int numTerms = TestUtil.nextInt(random(), maxNumValuesPerDoc, 100);
+        while (termSet.size() < numTerms) {
+            termSet.add(new BytesRef(TestUtil.randomSimpleString(random(), randomInt(2))));
+        }
+        final BytesRef[] terms = termSet.toArray(new BytesRef[0]);
+        Arrays.sort(terms);
+
+        final int numRanges = randomIntBetween(1, 10);
+        BinaryRangeAggregator.Range[] ranges = new BinaryRangeAggregator.Range[numRanges];
+        for (int i = 0; i < numRanges; ++i) {
+            ranges[i] = new BinaryRangeAggregator.Range(Integer.toString(i),
+                    randomBoolean() ? null : new BytesRef(TestUtil.randomSimpleString(random(), randomInt(2))),
+                    randomBoolean() ? null : new BytesRef(TestUtil.randomSimpleString(random(), randomInt(2))));
+        }
+        Arrays.sort(ranges, BinaryRangeAggregator.RANGE_COMPARATOR);
+
+        FakeSortedSetDocValues values = new FakeSortedSetDocValues(terms);
+        final int[] counts = new int[ranges.length];
+        SortedSetRangeLeafCollector collector = new SortedSetRangeLeafCollector(values, ranges, null) {
+            @Override
+            protected void doCollect(LeafBucketCollector sub, int doc, long bucket) throws IOException {
+                counts[(int) bucket]++;
+            }
+        };
+
+        final int[] expectedCounts = new int[ranges.length];
+        final int maxDoc = randomIntBetween(5, 10);
+        for (int doc = 0; doc < maxDoc; ++doc) {
+            LongHashSet ordinalSet = new LongHashSet();
+            final int numValues = randomInt(maxNumValuesPerDoc);
+            while (ordinalSet.size() < numValues) {
+                ordinalSet.add(random().nextInt(terms.length));
+            }
+            final long[] ords = ordinalSet.toArray();
+            Arrays.sort(ords);
+            values.ords = ords;
+
+            // simulate aggregation
+            collector.collect(doc);
+
+            // now do it the naive way
+            for (int i = 0; i < ranges.length; ++i) {
+                for (long ord : ords) {
+                    BytesRef term = terms[(int) ord];
+                    if ((ranges[i].from == null || ranges[i].from.compareTo(term) <= 0)
+                            && (ranges[i].to == null || ranges[i].to.compareTo(term) > 0)) {
+                        expectedCounts[i]++;
+                        break;
+                    }
+                }
+            }
+        }
+        assertArrayEquals(expectedCounts, counts);
+    }
+
+    public void testSortedSetRangeLeafCollectorSingleValued() throws Exception {
+        final int iters = randomInt(10);
+        for (int i = 0; i < iters; ++i) {
+            doTestSortedSetRangeLeafCollector(1);
+        }
+    }
+
+    public void testSortedSetRangeLeafCollectorMultiValued() throws Exception {
+        final int iters = randomInt(10);
+        for (int i = 0; i < iters; ++i) {
+            doTestSortedSetRangeLeafCollector(5);
+        }
+    }
+}
diff --git a/docs/java-api/aggregations/bucket/iprange-aggregation.asciidoc b/docs/java-api/aggregations/bucket/iprange-aggregation.asciidoc
index eb095a2898c..7c1909298db 100644
--- a/docs/java-api/aggregations/bucket/iprange-aggregation.asciidoc
+++ b/docs/java-api/aggregations/bucket/iprange-aggregation.asciidoc
@@ -12,7 +12,7 @@ Here is an example on how to create the aggregation request:
 
 [source,java]
 --------------------------------------------------
-AggregationBuilder aggregation =
+AggregatorBuilder<?> aggregation =
         AggregationBuilders
                 .ipRange("agg")
                 .field("ip")
@@ -25,7 +25,7 @@ Note that you could also use ip masks as ranges:
 
 [source,java]
 --------------------------------------------------
-AggregationBuilder aggregation =
+AggregatorBuilder<?> aggregation =
         AggregationBuilders
                 .ipRange("agg")
                 .field("ip")
diff --git a/docs/reference/aggregations/bucket/iprange-aggregation.asciidoc b/docs/reference/aggregations/bucket/iprange-aggregation.asciidoc
index 6d06743644b..bb20b18663e 100644
--- a/docs/reference/aggregations/bucket/iprange-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/iprange-aggregation.asciidoc
@@ -1,7 +1,7 @@
 [[search-aggregations-bucket-iprange-aggregation]]
-=== IPv4 Range Aggregation
+=== IP Range Aggregation
 
-Just like the dedicated <<search-aggregations-bucket-daterange-aggregation,date>> range aggregation, there is also a dedicated range aggregation for IPv4 typed fields:
+Just like the dedicated <<search-aggregations-bucket-daterange-aggregation,date>> range aggregation, there is also a dedicated range aggregation for IP typed fields:
 
 Example:
 
@@ -33,13 +33,11 @@ Response:
         "ip_ranges": {
             "buckets" : [
                 {
-                    "to": 167772165,
-                    "to_as_string": "10.0.0.5",
+                    "to": "10.0.0.5",
                     "doc_count": 4
                 },
                 {
-                    "from": 167772165,
-                    "from_as_string": "10.0.0.5",
+                    "from": "10.0.0.5",
                     "doc_count": 6
                 }
             ]
@@ -77,22 +75,18 @@ Response:
             "buckets": [
                 {
                     "key": "10.0.0.0/25",
-                    "from": 1.6777216E+8,
-                    "from_as_string": "10.0.0.0",
-                    "to": 167772287,
-                    "to_as_string": "10.0.0.127",
+                    "from": "10.0.0.0",
+                    "to": "10.0.0.127",
                     "doc_count": 127
                 },
                 {
                     "key": "10.0.0.127/25",
-                    "from": 1.6777216E+8,
-                    "from_as_string": "10.0.0.0",
-                    "to": 167772287,
-                    "to_as_string": "10.0.0.127",
+                    "from": "10.0.0.0",
+                    "to": "10.0.0.127",
                     "doc_count": 127
                 }
             ]
         }
     }
 }
---------------------------------------------------
\ No newline at end of file
+--------------------------------------------------
diff --git a/docs/reference/migration/migrate_5_0/aggregations.asciidoc b/docs/reference/migration/migrate_5_0/aggregations.asciidoc
index 60c1060f159..d90e429732b 100644
--- a/docs/reference/migration/migrate_5_0/aggregations.asciidoc
+++ b/docs/reference/migration/migrate_5_0/aggregations.asciidoc
@@ -13,3 +13,10 @@ It is recommended to use <<keyword,`keyword`>> fields instead, either directly
 or through a <<multi-fields,multi-field>> if the numeric representation is
 still needed for sorting, range queries or numeric aggregations like
 <<search-aggregations-metrics-stats-aggregation,`stats` aggregations>>.
+
+==== `ip_range` aggregations
+
+Now that Elasticsearch supports `ipv6`, `ip` addresses are encoded in the index
+using a binary representation rather than a numeric representation. As a
+consequence, the output of `ip_range` aggregations does not give numeric values
+for `from` and `to` anymore.
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IPv4RangeTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IPv4RangeTests.java
deleted file mode 100644
index 23ecfad39b8..00000000000
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IPv4RangeTests.java
+++ /dev/null
@@ -1,717 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.messy.tests;
-
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
-import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.index.mapper.ip.LegacyIpFieldMapper;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.script.Script;
-import org.elasticsearch.script.groovy.GroovyPlugin;
-import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
-import org.elasticsearch.search.aggregations.bucket.range.Range;
-import org.elasticsearch.search.aggregations.bucket.range.Range.Bucket;
-import org.elasticsearch.search.aggregations.metrics.sum.Sum;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.hamcrest.Matchers;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
-import static org.elasticsearch.search.aggregations.AggregationBuilders.ipRange;
-import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.core.IsNull.notNullValue;
-import static org.hamcrest.core.IsNull.nullValue;
-
-/**
- *
- */
-@AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/17700")
-@ESIntegTestCase.SuiteScopeTestCase
-public class IPv4RangeTests extends ESIntegTestCase {
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(GroovyPlugin.class);
-    }
-
-    @Override
-    public void setupSuiteScopeCluster() throws Exception {
-        {
-            assertAcked(prepareCreate("idx")
-                    .addMapping("type", "ip", "type=ip", "ips", "type=ip"));
-            IndexRequestBuilder[] builders = new IndexRequestBuilder[255]; // TODO randomize the size?
-            // TODO randomize the values in the docs?
-            for (int i = 0; i < builders.length; i++) {
-                builders[i] = client().prepareIndex("idx", "type").setSource(jsonBuilder()
-                        .startObject()
-                        .field("ip", "10.0.0." + (i))
-                        .startArray("ips").value("10.0.0." + i).value("10.0.0." + (i + 1)).endArray()
-                        .field("value", (i < 100 ? 1 : i < 200 ? 2 : 3))        // 100 1's, 100 2's, and 55 3's
-                        .endObject());
-            }
-            indexRandom(true, builders);
-            createIndex("idx_unmapped");
-        }
-        {
-            assertAcked(prepareCreate("empty_bucket_idx").addMapping("type", "value", "type=integer", "ip", "type=ip"));
-            List<IndexRequestBuilder> builders = new ArrayList<>();
-            for (int i = 0; i < 2; i++) {
-                builders.add(client().prepareIndex("empty_bucket_idx", "type", "" + i).setSource(jsonBuilder()
-                        .startObject()
-                        .field("value", i * 2)
-                        .field("ip", "10.0.0.5")
-                        .endObject()));
-            }
-            indexRandom(true, builders.toArray(new IndexRequestBuilder[builders.size()]));
-        }
-        {
-            assertAcked(prepareCreate("range_idx")
-                    .addMapping("type", "ip", "type=ip", "ips", "type=ip"));
-            IndexRequestBuilder[] builders = new IndexRequestBuilder[4];
-
-            builders[0] = client().prepareIndex("range_idx", "type").setSource(jsonBuilder()
-                    .startObject()
-                    .field("ip", "0.0.0.0")
-                    .endObject());
-
-            builders[1] = client().prepareIndex("range_idx", "type").setSource(jsonBuilder()
-                    .startObject()
-                    .field("ip", "0.0.0.255")
-                    .endObject());
-
-            builders[2] = client().prepareIndex("range_idx", "type").setSource(jsonBuilder()
-                    .startObject()
-                    .field("ip", "255.255.255.0")
-                    .endObject());
-
-            builders[3] = client().prepareIndex("range_idx", "type").setSource(jsonBuilder()
-                    .startObject()
-                    .field("ip", "255.255.255.255")
-                    .endObject());
-
-            indexRandom(true, builders);
-        }
-        ensureSearchable();
-    }
-
-    public void testSingleValueField() throws Exception {
-        SearchResponse response = client().prepareSearch("idx")
-                .addAggregation(ipRange("range")
-                        .field("ip")
-                        .addUnboundedTo("10.0.0.100")
-                        .addRange("10.0.0.100", "10.0.0.200")
-                        .addUnboundedFrom("10.0.0.200"))
-                .execute().actionGet();
-
-        assertSearchResponse(response);
-
-
-        Range range = response.getAggregations().get("range");
-        assertThat(range, notNullValue());
-        assertThat(range.getName(), equalTo("range"));
-        List<? extends Bucket> buckets = range.getBuckets();
-        assertThat(buckets.size(), equalTo(3));
-
-        Range.Bucket bucket = buckets.get(0);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("*-10.0.0.100"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY));
-        assertThat(bucket.getFromAsString(), nullValue());
-        assertThat(bucket.getToAsString(), equalTo("10.0.0.100"));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.100")));
-        assertThat(bucket.getDocCount(), equalTo(100L));
-
-        bucket = buckets.get(1);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("10.0.0.100-10.0.0.200"));
-        assertThat(bucket.getFromAsString(), equalTo("10.0.0.100"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.100")));
-        assertThat(bucket.getToAsString(), equalTo("10.0.0.200"));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.200")));
-        assertThat(bucket.getDocCount(), equalTo(100L));
-
-        bucket = buckets.get(2);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("10.0.0.200-*"));
-        assertThat(bucket.getFromAsString(), equalTo("10.0.0.200"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.200")));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY));
-        assertThat(bucket.getToAsString(), nullValue());
-        assertThat(bucket.getDocCount(), equalTo(55L));
-    }
-
-    public void testSingleValueFieldWithMaskRange() throws Exception {
-        SearchResponse response = client().prepareSearch("idx")
-                .addAggregation(ipRange("range")
-                        .field("ip")
-                        .addMaskRange("10.0.0.0/25")
-                        .addMaskRange("10.0.0.128/25"))
-                .execute().actionGet();
-
-        assertSearchResponse(response);
-
-
-        Range range = response.getAggregations().get("range");
-        assertThat(range, notNullValue());
-        assertThat(range.getName(), equalTo("range"));
-        List<? extends Bucket> buckets = range.getBuckets();
-        assertThat(range.getBuckets().size(), equalTo(2));
-
-        Range.Bucket bucket = buckets.get(0);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("10.0.0.0/25"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.0")));
-        assertThat(bucket.getFromAsString(), equalTo("10.0.0.0"));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.128")));
-        assertThat(bucket.getToAsString(), equalTo("10.0.0.128"));
-        assertThat(bucket.getDocCount(), equalTo(128L));
-
-        bucket = buckets.get(1);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("10.0.0.128/25"));
-        assertThat((long) ((Number) bucket.getFrom()).doubleValue(), equalTo(LegacyIpFieldMapper.ipToLong("10.0.0.128")));
-        assertThat(bucket.getFromAsString(), equalTo("10.0.0.128"));
-        assertThat((long) ((Number) bucket.getTo()).doubleValue(), equalTo(LegacyIpFieldMapper.ipToLong("10.0.1.0"))); // range is exclusive on the to side
-        assertThat(bucket.getToAsString(), equalTo("10.0.1.0"));
-        assertThat(bucket.getDocCount(), equalTo(127L)); // include 10.0.0.128
-    }
-
-    public void testSingleValueFieldWithCustomKey() throws Exception {
-        SearchResponse response = client().prepareSearch("idx")
-                .addAggregation(ipRange("range")
-                        .field("ip")
-                        .addUnboundedTo("r1", "10.0.0.100")
-                        .addRange("r2", "10.0.0.100", "10.0.0.200")
-                        .addUnboundedFrom("r3", "10.0.0.200"))
-                .execute().actionGet();
-
-        assertSearchResponse(response);
-
-
-        Range range = response.getAggregations().get("range");
-        assertThat(range, notNullValue());
-        assertThat(range.getName(), equalTo("range"));
-        List<? extends Bucket> buckets = range.getBuckets();
-        assertThat(range.getBuckets().size(), equalTo(3));
-
-        Range.Bucket bucket = buckets.get(0);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("r1"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY));
-        assertThat(bucket.getFromAsString(), nullValue());
-        assertThat(bucket.getToAsString(), equalTo("10.0.0.100"));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.100")));
-        assertThat(bucket.getDocCount(), equalTo(100L));
-
-        bucket = buckets.get(1);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("r2"));
-        assertThat(bucket.getFromAsString(), equalTo("10.0.0.100"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.100")));
-        assertThat(bucket.getToAsString(), equalTo("10.0.0.200"));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.200")));
-        assertThat(bucket.getDocCount(), equalTo(100L));
-
-        bucket = buckets.get(2);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("r3"));
-        assertThat(bucket.getFromAsString(), equalTo("10.0.0.200"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.200")));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY));
-        assertThat(bucket.getToAsString(), nullValue());
-        assertThat(bucket.getDocCount(), equalTo(55L));
-    }
-
-    public void testSingleValuedFieldWithSubAggregation() throws Exception {
-        SearchResponse response = client().prepareSearch("idx")
-                .addAggregation(ipRange("range")
-                        .field("ip")
-                        .addUnboundedTo("10.0.0.100")
-                        .addRange("10.0.0.100", "10.0.0.200")
-                        .addUnboundedFrom("10.0.0.200")
-                        .subAggregation(sum("sum").field("value")))
-                .execute().actionGet();
-
-        assertSearchResponse(response);
-
-
-        Range range = response.getAggregations().get("range");
-        assertThat(range, notNullValue());
-        assertThat(range.getName(), equalTo("range"));
-        List<? extends Bucket> buckets = range.getBuckets();
-        assertThat(range.getBuckets().size(), equalTo(3));
-        Object[] propertiesKeys = (Object[]) range.getProperty("_key");
-        Object[] propertiesDocCounts = (Object[]) range.getProperty("_count");
-        Object[] propertiesCounts = (Object[]) range.getProperty("sum.value");
-
-        Range.Bucket bucket = buckets.get(0);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("*-10.0.0.100"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY));
-        assertThat(bucket.getFromAsString(), nullValue());
-        assertThat(bucket.getToAsString(), equalTo("10.0.0.100"));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.100")));
-        assertThat(bucket.getDocCount(), equalTo(100L));
-        Sum sum = bucket.getAggregations().get("sum");
-        assertThat(sum, notNullValue());
-        assertThat(sum.getValue(), equalTo((double) 100));
-        assertThat((String) propertiesKeys[0], equalTo("*-10.0.0.100"));
-        assertThat((long) propertiesDocCounts[0], equalTo(100L));
-        assertThat((double) propertiesCounts[0], equalTo((double) 100));
-
-        bucket = buckets.get(1);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("10.0.0.100-10.0.0.200"));
-        assertThat(bucket.getFromAsString(), equalTo("10.0.0.100"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.100")));
-        assertThat(bucket.getToAsString(), equalTo("10.0.0.200"));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.200")));
-        assertThat(bucket.getDocCount(), equalTo(100L));
-        sum = bucket.getAggregations().get("sum");
-        assertThat(sum, notNullValue());
-        assertThat(sum.getValue(), equalTo((double) 200));
-        assertThat((String) propertiesKeys[1], equalTo("10.0.0.100-10.0.0.200"));
-        assertThat((long) propertiesDocCounts[1], equalTo(100L));
-        assertThat((double) propertiesCounts[1], equalTo((double) 200));
-
-        bucket = buckets.get(2);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("10.0.0.200-*"));
-        assertThat(bucket.getFromAsString(), equalTo("10.0.0.200"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.200")));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY));
-        assertThat(bucket.getToAsString(), nullValue());
-        assertThat(bucket.getDocCount(), equalTo(55L));
-        sum = bucket.getAggregations().get("sum");
-        assertThat(sum, notNullValue());
-        assertThat(sum.getValue(), equalTo((double) 55*3));
-        assertThat((String) propertiesKeys[2], equalTo("10.0.0.200-*"));
-        assertThat((long) propertiesDocCounts[2], equalTo(55L));
-        assertThat((double) propertiesCounts[2], equalTo((double) 55 * 3));
-    }
-
-    public void testSingleValuedFieldWithValueScript() throws Exception {
-        SearchResponse response = client()
-                .prepareSearch("idx")
-                .addAggregation(
-                        ipRange("range").field("ip").script(new Script("_value")).addUnboundedTo("10.0.0.100")
-                                .addRange("10.0.0.100", "10.0.0.200").addUnboundedFrom("10.0.0.200")).execute().actionGet();
-
-        assertSearchResponse(response);
-
-        Range range = response.getAggregations().get("range");
-        assertThat(range, notNullValue());
-        assertThat(range.getName(), equalTo("range"));
-        List<? extends Bucket> buckets = range.getBuckets();
-        assertThat(range.getBuckets().size(), equalTo(3));
-
-        Range.Bucket bucket = buckets.get(0);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("*-10.0.0.100"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY));
-        assertThat(bucket.getFromAsString(), nullValue());
-        assertThat(bucket.getToAsString(), equalTo("10.0.0.100"));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.100")));
-        assertThat(bucket.getDocCount(), equalTo(100L));
-
-        bucket = buckets.get(1);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("10.0.0.100-10.0.0.200"));
-        assertThat(bucket.getFromAsString(), equalTo("10.0.0.100"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.100")));
-        assertThat(bucket.getToAsString(), equalTo("10.0.0.200"));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.200")));
-        assertThat(bucket.getDocCount(), equalTo(100L));
-
-        bucket = buckets.get(2);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("10.0.0.200-*"));
-        assertThat(bucket.getFromAsString(), equalTo("10.0.0.200"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.200")));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY));
-        assertThat(bucket.getToAsString(), nullValue());
-        assertThat(bucket.getDocCount(), equalTo(55L));
-    }
-
-    /*
-    [0, 1]
-    [1, 2]
-    [2, 3]
-    ...
-    [99, 100]
-    [100, 101]
-    [101, 102]
-    ...
-    [199, 200]
-    [200, 201]
-    [201, 202]
-    ...
-    [254, 255]
-    [255, 256]
-     */
-
-    public void testMultiValuedField() throws Exception {
-        SearchResponse response = client().prepareSearch("idx")
-                .addAggregation(ipRange("range")
-                        .field("ips")
-                        .addUnboundedTo("10.0.0.100")
-                        .addRange("10.0.0.100", "10.0.0.200")
-                        .addUnboundedFrom("10.0.0.200"))
-                .execute().actionGet();
-
-        assertSearchResponse(response);
-
-
-        Range range = response.getAggregations().get("range");
-        assertThat(range, notNullValue());
-        assertThat(range.getName(), equalTo("range"));
-        List<? extends Bucket> buckets = range.getBuckets();
-        assertThat(range.getBuckets().size(), equalTo(3));
-
-        Range.Bucket bucket = buckets.get(0);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("*-10.0.0.100"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY));
-        assertThat(bucket.getFromAsString(), nullValue());
-        assertThat(bucket.getToAsString(), equalTo("10.0.0.100"));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.100")));
-        assertThat(bucket.getDocCount(), equalTo(100L));
-
-        bucket = buckets.get(1);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("10.0.0.100-10.0.0.200"));
-        assertThat(bucket.getFromAsString(), equalTo("10.0.0.100"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.100")));
-        assertThat(bucket.getToAsString(), equalTo("10.0.0.200"));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.200")));
-        assertThat(bucket.getDocCount(), equalTo(101L));
-
-        bucket = buckets.get(2);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("10.0.0.200-*"));
-        assertThat(bucket.getFromAsString(), equalTo("10.0.0.200"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.200")));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY));
-        assertThat(bucket.getToAsString(), nullValue());
-        assertThat(bucket.getDocCount(), equalTo(56L));
-    }
-
-    public void testMultiValuedFieldWithValueScript() throws Exception {
-        SearchResponse response = client()
-                .prepareSearch("idx")
-                .addAggregation(
-                        ipRange("range").field("ips").script(new Script("_value")).addUnboundedTo("10.0.0.100")
-                                .addRange("10.0.0.100", "10.0.0.200").addUnboundedFrom("10.0.0.200")).execute().actionGet();
-
-        assertSearchResponse(response);
-
-
-        Range range = response.getAggregations().get("range");
-        assertThat(range, notNullValue());
-        assertThat(range.getName(), equalTo("range"));
-        List<? extends Bucket> buckets = range.getBuckets();
-        assertThat(range.getBuckets().size(), equalTo(3));
-
-        Range.Bucket bucket = buckets.get(0);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("*-10.0.0.100"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY));
-        assertThat(bucket.getFromAsString(), nullValue());
-        assertThat(bucket.getToAsString(), equalTo("10.0.0.100"));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.100")));
-        assertThat(bucket.getDocCount(), equalTo(100L));
-
-        bucket = buckets.get(1);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("10.0.0.100-10.0.0.200"));
-        assertThat(bucket.getFromAsString(), equalTo("10.0.0.100"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.100")));
-        assertThat(bucket.getToAsString(), equalTo("10.0.0.200"));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.200")));
-        assertThat(bucket.getDocCount(), equalTo(101L));
-
-        bucket = buckets.get(2);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("10.0.0.200-*"));
-        assertThat(bucket.getFromAsString(), equalTo("10.0.0.200"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.200")));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY));
-        assertThat(bucket.getToAsString(), nullValue());
-        assertThat(bucket.getDocCount(), equalTo(56L));
-    }
-
-    public void testScriptSingleValue() throws Exception {
-        SearchResponse response = client()
-                .prepareSearch("idx")
-                .addAggregation(
-                        ipRange("range").script(new Script("doc['ip'].value")).addUnboundedTo("10.0.0.100")
-                                .addRange("10.0.0.100", "10.0.0.200").addUnboundedFrom("10.0.0.200")).execute().actionGet();
-
-        assertSearchResponse(response);
-
-
-        Range range = response.getAggregations().get("range");
-        assertThat(range, notNullValue());
-        assertThat(range.getName(), equalTo("range"));
-        List<? extends Bucket> buckets = range.getBuckets();
-        assertThat(range.getBuckets().size(), equalTo(3));
-
-        Range.Bucket bucket = buckets.get(0);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("*-10.0.0.100"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY));
-        assertThat(bucket.getFromAsString(), nullValue());
-        assertThat(bucket.getToAsString(), equalTo("10.0.0.100"));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.100")));
-        assertThat(bucket.getDocCount(), equalTo(100L));
-
-        bucket = buckets.get(1);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("10.0.0.100-10.0.0.200"));
-        assertThat(bucket.getFromAsString(), equalTo("10.0.0.100"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.100")));
-        assertThat(bucket.getToAsString(), equalTo("10.0.0.200"));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.200")));
-        assertThat(bucket.getDocCount(), equalTo(100L));
-
-        bucket = buckets.get(2);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("10.0.0.200-*"));
-        assertThat(bucket.getFromAsString(), equalTo("10.0.0.200"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.200")));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY));
-        assertThat(bucket.getToAsString(), nullValue());
-        assertThat(bucket.getDocCount(), equalTo(55L));
-    }
-
-    public void testScriptMultiValued() throws Exception {
-        SearchResponse response = client()
-                .prepareSearch("idx")
-                .addAggregation(
-                        ipRange("range").script(new Script("doc['ips'].values")).addUnboundedTo("10.0.0.100")
-                                .addRange("10.0.0.100", "10.0.0.200").addUnboundedFrom("10.0.0.200")).execute().actionGet();
-
-        assertSearchResponse(response);
-
-
-        Range range = response.getAggregations().get("range");
-        assertThat(range, notNullValue());
-        assertThat(range.getName(), equalTo("range"));
-        List<? extends Bucket> buckets = range.getBuckets();
-        assertThat(range.getBuckets().size(), equalTo(3));
-
-        Range.Bucket bucket = buckets.get(0);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("*-10.0.0.100"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY));
-        assertThat(bucket.getFromAsString(), nullValue());
-        assertThat(bucket.getToAsString(), equalTo("10.0.0.100"));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.100")));
-        assertThat(bucket.getDocCount(), equalTo(100L));
-
-        bucket = buckets.get(1);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("10.0.0.100-10.0.0.200"));
-        assertThat(bucket.getFromAsString(), equalTo("10.0.0.100"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.100")));
-        assertThat(bucket.getToAsString(), equalTo("10.0.0.200"));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.200")));
-        assertThat(bucket.getDocCount(), equalTo(101L));
-
-        bucket = buckets.get(2);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("10.0.0.200-*"));
-        assertThat(bucket.getFromAsString(), equalTo("10.0.0.200"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.200")));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY));
-        assertThat(bucket.getToAsString(), nullValue());
-        assertThat(bucket.getDocCount(), equalTo(56L));
-    }
-
-    public void testUnmapped() throws Exception {
-        SearchResponse response = client().prepareSearch("idx_unmapped")
-                .addAggregation(ipRange("range")
-                        .field("ip")
-                        .addUnboundedTo("10.0.0.100")
-                        .addRange("10.0.0.100", "10.0.0.200")
-                        .addUnboundedFrom("10.0.0.200"))
-                .execute().actionGet();
-
-        assertSearchResponse(response);
-
-
-        Range range = response.getAggregations().get("range");
-        assertThat(range, notNullValue());
-        assertThat(range.getName(), equalTo("range"));
-        List<? extends Bucket> buckets = range.getBuckets();
-        assertThat(range.getBuckets().size(), equalTo(3));
-
-        Range.Bucket bucket = buckets.get(0);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("*-10.0.0.100"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY));
-        assertThat(bucket.getFromAsString(), nullValue());
-        assertThat(bucket.getToAsString(), equalTo("10.0.0.100"));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.100")));
-        assertThat(bucket.getDocCount(), equalTo(0L));
-
-        bucket = buckets.get(1);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("10.0.0.100-10.0.0.200"));
-        assertThat(bucket.getFromAsString(), equalTo("10.0.0.100"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.100")));
-        assertThat(bucket.getToAsString(), equalTo("10.0.0.200"));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.200")));
-        assertThat(bucket.getDocCount(), equalTo(0L));
-
-        bucket = buckets.get(2);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("10.0.0.200-*"));
-        assertThat(bucket.getFromAsString(), equalTo("10.0.0.200"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.200")));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY));
-        assertThat(bucket.getToAsString(), nullValue());
-        assertThat(bucket.getDocCount(), equalTo(0L));
-    }
-
-    public void testPartiallyUnmapped() throws Exception {
-        SearchResponse response = client().prepareSearch("idx", "idx_unmapped")
-                .addAggregation(ipRange("range")
-                        .field("ip")
-                        .addUnboundedTo("10.0.0.100")
-                        .addRange("10.0.0.100", "10.0.0.200")
-                        .addUnboundedFrom("10.0.0.200"))
-                .execute().actionGet();
-
-        assertSearchResponse(response);
-
-
-        Range range = response.getAggregations().get("range");
-        assertThat(range, notNullValue());
-        assertThat(range.getName(), equalTo("range"));
-        List<? extends Bucket> buckets = range.getBuckets();
-        assertThat(range.getBuckets().size(), equalTo(3));
-
-        Range.Bucket bucket = buckets.get(0);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("*-10.0.0.100"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo(Double.NEGATIVE_INFINITY));
-        assertThat(bucket.getFromAsString(), nullValue());
-        assertThat(bucket.getToAsString(), equalTo("10.0.0.100"));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.100")));
-        assertThat(bucket.getDocCount(), equalTo(100L));
-
-        bucket = buckets.get(1);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("10.0.0.100-10.0.0.200"));
-        assertThat(bucket.getFromAsString(), equalTo("10.0.0.100"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.100")));
-        assertThat(bucket.getToAsString(), equalTo("10.0.0.200"));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.200")));
-        assertThat(bucket.getDocCount(), equalTo(100L));
-
-        bucket = buckets.get(2);
-        assertThat(bucket, notNullValue());
-        assertThat((String) bucket.getKey(), equalTo("10.0.0.200-*"));
-        assertThat(bucket.getFromAsString(), equalTo("10.0.0.200"));
-        assertThat(((Number) bucket.getFrom()).doubleValue(), equalTo((double) LegacyIpFieldMapper.ipToLong("10.0.0.200")));
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY));
-        assertThat(bucket.getToAsString(), nullValue());
-        assertThat(bucket.getDocCount(), equalTo(55L));
-    }
-
-    public void testEmptyAggregation() throws Exception {
-        SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx")
-                .setQuery(matchAllQuery())
-                .addAggregation(histogram("histo").field("value").interval(1L).minDocCount(0)
-                        .subAggregation(ipRange("ip_range").field("ip").addRange("r1", "10.0.0.1", "10.0.0.10")))
-                .execute().actionGet();
-
-        assertThat(searchResponse.getHits().getTotalHits(), equalTo(2L));
-        Histogram histo = searchResponse.getAggregations().get("histo");
-        assertThat(histo, Matchers.notNullValue());
-        Histogram.Bucket bucket = histo.getBuckets().get(1);
-        assertThat(bucket, Matchers.notNullValue());
-
-        Range range = bucket.getAggregations().get("ip_range");
-        // TODO: use diamond once JI-9019884 is fixed
-        List<Range.Bucket> buckets = new ArrayList<Range.Bucket>(range.getBuckets());
-        assertThat(range, Matchers.notNullValue());
-        assertThat(range.getName(), equalTo("ip_range"));
-        assertThat(buckets.size(), is(1));
-        assertThat((String) buckets.get(0).getKey(), equalTo("r1"));
-        assertThat(buckets.get(0).getFromAsString(), equalTo("10.0.0.1"));
-        assertThat(buckets.get(0).getToAsString(), equalTo("10.0.0.10"));
-        assertThat(buckets.get(0).getDocCount(), equalTo(0L));
-    }
-
-    public void testMask0() {
-        SearchResponse response = client().prepareSearch("idx")
-                .addAggregation(ipRange("range")
-                        .field("ip")
-                        .addMaskRange("0.0.0.0/0"))
-                .execute().actionGet();
-
-        assertSearchResponse(response);
-
-        Range range = response.getAggregations().get("range");
-        assertThat(range, notNullValue());
-        assertThat(range.getName(), equalTo("range"));
-        List<? extends Bucket> buckets = range.getBuckets();
-        assertThat(range.getBuckets().size(), equalTo(1));
-
-        Range.Bucket bucket = buckets.get(0);
-        assertThat((String) bucket.getKey(), equalTo("0.0.0.0/0"));
-        assertThat(bucket.getFromAsString(), nullValue());
-        assertThat(bucket.getToAsString(), nullValue());
-        assertThat(((Number) bucket.getTo()).doubleValue(), equalTo(Double.POSITIVE_INFINITY));
-        assertEquals(255L, bucket.getDocCount());
-    }
-
-    public void testMask0SpecialIps() {
-        SearchResponse response = client().prepareSearch("range_idx")
-                .addAggregation(ipRange("range")
-                        .field("ip")
-                        .addMaskRange("0.0.0.0/0"))
-                .execute().actionGet();
-
-        assertSearchResponse(response);
-
-        Range range = response.getAggregations().get("range");
-
-        assertThat(range, notNullValue());
-        assertThat(range.getName(), equalTo("range"));
-        List<? extends Bucket> buckets = range.getBuckets();
-        assertThat(range.getBuckets().size(), equalTo(1));
-
-        Range.Bucket bucket = buckets.get(0);
-        assertEquals(4L, bucket.getDocCount());
-    }
-}

From a4060f743619c179f9793feb4eabb33509316136 Mon Sep 17 00:00:00 2001
From: Lee Hinman <lee@writequit.org>
Date: Mon, 9 May 2016 15:42:32 -0600
Subject: [PATCH 0229/1311] Remove vestiges of script engine sandboxing

This removes all the mentions of the sandbox from the script engine
services and permissions model. This means that the following settings
are no longer supported:

```yaml
script.inline: sandbox
script.stored: sandbox
```

Instead, only a `true` or `false` value can be specified.

Since this would otherwise break the default-allow parameter for
languages like expressions, painless, and mustache, all script engines
have been updated to have individual settings, for instance:

```yaml
script.engine.groovy.inline: true
```

Would enable all inline scripts for groovy. (they can still be
overridden on a per-operation basis).

Expressions, Painless, and Mustache all default to `true` for inline,
file, and stored scripts to preserve the old scripting behavior.

Resolves #17114
---
 .../script/NativeScriptEngineService.java     |   5 -
 .../script/ScriptEngineRegistry.java          |  20 ++++
 .../script/ScriptEngineService.java           |   2 -
 .../org/elasticsearch/script/ScriptMode.java  |   8 +-
 .../org/elasticsearch/script/ScriptModes.java |   6 +-
 .../elasticsearch/script/ScriptModule.java    |   4 +-
 .../elasticsearch/script/ScriptService.java   |   7 +-
 .../elasticsearch/script/ScriptSettings.java  |  21 +++-
 .../index/query/AbstractQueryTestCase.java    |   7 +-
 .../script/ScriptContextTests.java            |   6 +-
 .../script/ScriptModesTests.java              |  22 ++--
 .../script/ScriptServiceTests.java            | 103 +++++++++++-------
 .../script/ScriptSettingsTests.java           |   5 -
 .../BasePipelineAggregationTestCase.java      |   7 +-
 .../search/aggregations/metrics/AvgIT.java    |  15 +--
 .../search/aggregations/metrics/SumIT.java    |  15 +--
 .../aggregations/metrics/ValueCountIT.java    |   8 +-
 .../org/elasticsearch/update/UpdateIT.java    |  29 +----
 .../migration/migrate_5_0/settings.asciidoc   |  34 +++---
 docs/reference/modules/scripting.asciidoc     |  17 ---
 .../modules/scripting/groovy.asciidoc         |   6 +-
 .../modules/scripting/security.asciidoc       |  37 ++++---
 .../modules/scripting/using.asciidoc          |  15 ++-
 .../reference/search/search-template.asciidoc |   8 +-
 .../script/expression/ExpressionPlugin.java   |   4 +-
 .../ExpressionScriptEngineService.java        |   5 -
 .../groovy/GroovyScriptEngineService.java     |   5 -
 .../script/mustache/MustachePlugin.java       |   4 +-
 .../mustache/MustacheScriptEngineService.java |   5 -
 .../messy/tests/TemplateQueryParserTests.java |   1 +
 .../painless/PainlessPlugin.java              |   3 +-
 .../painless/PainlessScriptEngineService.java |   9 --
 .../JavaScriptScriptEngineService.java        |   5 -
 .../python/PythonScriptEngineService.java     |   5 -
 .../script/MockScriptEngine.java              |   9 +-
 .../elasticsearch/test/ESIntegTestCase.java   |   4 +-
 .../test/ESSingleNodeTestCase.java            |   3 -
 37 files changed, 228 insertions(+), 241 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java b/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java
index 6b9a8733f58..e1d11ee49cf 100644
--- a/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java
+++ b/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java
@@ -60,11 +60,6 @@ public class NativeScriptEngineService extends AbstractComponent implements Scri
         return Collections.emptyList();
     }
 
-    @Override
-    public boolean isSandboxed() {
-        return false;
-    }
-
     @Override
     public Object compile(String script, Map<String, String> params) {
         NativeScriptFactory scriptFactory = scripts.get(script);
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptEngineRegistry.java b/core/src/main/java/org/elasticsearch/script/ScriptEngineRegistry.java
index 4bc9a23ebae..f64c40bc773 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptEngineRegistry.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptEngineRegistry.java
@@ -29,11 +29,13 @@ public class ScriptEngineRegistry {
 
     private final Map<Class<? extends ScriptEngineService>, List<String>> registeredScriptEngineServices;
     private final Map<String, Class<? extends ScriptEngineService>> registeredLanguages;
+    private final Map<String, ScriptMode> defaultInlineScriptModes;
 
     public ScriptEngineRegistry(Iterable<ScriptEngineRegistration> registrations) {
         Objects.requireNonNull(registrations);
         Map<Class<? extends ScriptEngineService>, List<String>> registeredScriptEngineServices = new HashMap<>();
         Map<String, Class<? extends ScriptEngineService>> registeredLanguages = new HashMap<>();
+        Map<String, ScriptMode> inlineScriptModes = new HashMap<>();
         for (ScriptEngineRegistration registration : registrations) {
             List<String> languages =
                 registeredScriptEngineServices.putIfAbsent(registration.getScriptEngineService(), Collections.unmodifiableList(registration.getScriptEngineLanguages()));
@@ -47,11 +49,13 @@ public class ScriptEngineRegistry {
                 if (scriptEngineServiceClazz != null) {
                     throw new IllegalArgumentException("scripting language [" + language + "] already registered for script engine service [" + scriptEngineServiceClazz.getCanonicalName() + "]");
                 }
+                inlineScriptModes.put(language, registration.getDefaultInlineScriptMode());
             }
         }
 
         this.registeredScriptEngineServices = Collections.unmodifiableMap(registeredScriptEngineServices);
         this.registeredLanguages = Collections.unmodifiableMap(registeredLanguages);
+        this.defaultInlineScriptModes = Collections.unmodifiableMap(inlineScriptModes);
     }
 
     Iterable<Class<? extends ScriptEngineService>> getRegisteredScriptEngineServices() {
@@ -67,11 +71,22 @@ public class ScriptEngineRegistry {
         return registeredLanguages;
     }
 
+    Map<String, ScriptMode> getDefaultInlineScriptModes() {
+        return this.defaultInlineScriptModes;
+    }
+
     public static class ScriptEngineRegistration {
         private final Class<? extends ScriptEngineService> scriptEngineService;
         private final List<String> scriptEngineLanguages;
+        private final ScriptMode defaultInlineScriptMode;
 
         public ScriptEngineRegistration(Class<? extends ScriptEngineService> scriptEngineService, List<String> scriptEngineLanguages) {
+            // Default to "off/disabled" if not specified
+            this(scriptEngineService, scriptEngineLanguages, ScriptMode.OFF);
+        }
+
+        public ScriptEngineRegistration(Class<? extends ScriptEngineService> scriptEngineService, List<String> scriptEngineLanguages,
+                                        ScriptMode defaultInlineScriptMode) {
             Objects.requireNonNull(scriptEngineService);
             Objects.requireNonNull(scriptEngineLanguages);
             if (scriptEngineLanguages.isEmpty()) {
@@ -79,6 +94,7 @@ public class ScriptEngineRegistry {
             }
             this.scriptEngineService = scriptEngineService;
             this.scriptEngineLanguages = scriptEngineLanguages;
+            this.defaultInlineScriptMode = defaultInlineScriptMode;
         }
 
         Class<? extends ScriptEngineService> getScriptEngineService() {
@@ -88,6 +104,10 @@ public class ScriptEngineRegistry {
         List<String> getScriptEngineLanguages() {
             return scriptEngineLanguages;
         }
+
+        ScriptMode getDefaultInlineScriptMode() {
+            return defaultInlineScriptMode;
+        }
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java b/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java
index 1c5d0f1e9ea..29046f9d906 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java
@@ -35,8 +35,6 @@ public interface ScriptEngineService extends Closeable {
 
     List<String> getExtensions();
 
-    boolean isSandboxed();
-
     Object compile(String script, Map<String, String> params);
 
     ExecutableScript executable(CompiledScript compiledScript, @Nullable Map<String, Object> vars);
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptMode.java b/core/src/main/java/org/elasticsearch/script/ScriptMode.java
index aa9d100d43b..6508d2f1015 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptMode.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptMode.java
@@ -24,13 +24,11 @@ import java.util.Map;
 
 /**
  * Mode for a specific script, used for script settings.
- * Defines whether a certain script or category of scripts can be executed or not, or whether it can
- * only be executed by a sandboxed scripting language.
+ * Defines whether a certain script or category of scripts can be executed or not.
  */
-enum ScriptMode {
+public enum ScriptMode {
     ON("true"),
-    OFF("false"),
-    SANDBOX("sandbox");
+    OFF("false");
 
     private final String mode;
 
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptModes.java b/core/src/main/java/org/elasticsearch/script/ScriptModes.java
index baaec009350..d07a4073ba8 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptModes.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptModes.java
@@ -54,7 +54,7 @@ public class ScriptModes {
      * @param lang the language that the script is written in
      * @param scriptType the type of the script
      * @param scriptContext the operation that requires the execution of the script
-     * @return whether scripts are on, off, or enabled only for sandboxed languages
+     * @return whether scripts are on or off
      */
     public ScriptMode getScriptMode(String lang, ScriptType scriptType, ScriptContext scriptContext) {
         //native scripts are always on as they are static by definition
@@ -76,6 +76,10 @@ public class ScriptModes {
         return SCRIPT_SETTINGS_PREFIX + "." + scriptType.getScriptType();
     }
 
+    static String getGlobalKey(String lang, ScriptType scriptType) {
+        return ENGINE_SETTINGS_PREFIX + "." + lang + "." + scriptType;
+    }
+
     static String getKey(String lang, ScriptType scriptType, ScriptContext scriptContext) {
         return ENGINE_SETTINGS_PREFIX + "." + lang + "." + scriptType + "." + scriptContext.getKey();
     }
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptModule.java b/core/src/main/java/org/elasticsearch/script/ScriptModule.java
index 0a386a4a8dc..fea143a3407 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptModule.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptModule.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.inject.multibindings.MapBinder;
 import org.elasticsearch.common.inject.multibindings.Multibinder;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.SettingsModule;
+import org.elasticsearch.script.ScriptMode;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -40,7 +41,8 @@ public class ScriptModule extends AbstractModule {
     private final List<ScriptEngineRegistry.ScriptEngineRegistration> scriptEngineRegistrations = new ArrayList<>();
 
     {
-        scriptEngineRegistrations.add(new ScriptEngineRegistry.ScriptEngineRegistration(NativeScriptEngineService.class, NativeScriptEngineService.TYPES));
+        scriptEngineRegistrations.add(new ScriptEngineRegistry.ScriptEngineRegistration(NativeScriptEngineService.class,
+                        NativeScriptEngineService.TYPES, ScriptMode.ON));
     }
 
     private final Map<String, Class<? extends NativeScriptFactory>> scripts = new HashMap<>();
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java
index a1ea88a2549..299b198d9b1 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java
@@ -31,6 +31,7 @@ import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
@@ -478,8 +479,6 @@ public class ScriptService extends AbstractComponent implements Closeable {
                 return true;
             case OFF:
                 return false;
-            case SANDBOX:
-                return scriptEngineService.isSandboxed();
             default:
                 throw new IllegalArgumentException("script mode [" + mode + "] not supported");
         }
@@ -607,8 +606,8 @@ public class ScriptService extends AbstractComponent implements Closeable {
      */
     public enum ScriptType {
 
-        INLINE(0, "inline", "inline", ScriptMode.SANDBOX),
-        STORED(1, "id", "stored", ScriptMode.SANDBOX),
+        INLINE(0, "inline", "inline", ScriptMode.OFF),
+        STORED(1, "id", "stored", ScriptMode.OFF),
         FILE(2, "file", "file", ScriptMode.ON);
 
         private final int val;
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java b/core/src/main/java/org/elasticsearch/script/ScriptSettings.java
index 1bf7fdfc843..f8bcd1f542f 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptSettings.java
@@ -23,6 +23,7 @@ import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.script.ScriptService;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -96,7 +97,18 @@ public class ScriptSettings {
                     // native scripts are always enabled, and their settings can not be changed
                     continue;
                 }
-                for (ScriptService.ScriptType scriptType : ScriptService.ScriptType.values()) {
+                ScriptMode defaultNonFileScriptMode = scriptEngineRegistry.getDefaultInlineScriptModes().get(language);
+                for (final ScriptService.ScriptType scriptType : ScriptService.ScriptType.values()) {
+                    // This will be easier once ScriptMode is transitioned to booleans
+                    if (scriptType.getDefaultScriptMode() == ScriptMode.ON) {
+                        defaultNonFileScriptMode = ScriptMode.ON;
+                    }
+                    final ScriptMode defaultScriptMode = defaultNonFileScriptMode;
+                    // Like "script.engine.groovy.inline"
+                    final Setting<ScriptMode> langGlobalSetting = new Setting<>(ScriptModes.getGlobalKey(language, scriptType),
+                            defaultScriptMode.toString(), ScriptMode::parse, Property.NodeScope);
+                    scriptModeSettings.add(langGlobalSetting);
+
                     for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) {
                         Function<Settings, String> defaultSetting = settings -> {
                             // fallback logic for script mode settings
@@ -116,6 +128,11 @@ public class ScriptSettings {
                                 return languageSettings.keySet().iterator().next();
                             }
 
+                            // the next fallback is settings configured for this engine and script type
+                            if (langGlobalSetting.exists(settings)) {
+                                return langGlobalSetting.get(settings).getMode();
+                            }
+
                             // the next fallback is global operation-based settings (e.g., "script.aggs: false")
                             Setting<ScriptMode> setting = scriptContextSettingMap.get(scriptContext);
                             if (setting.exists(settings)) {
@@ -129,7 +146,7 @@ public class ScriptSettings {
                             }
 
                             // the final fallback is the default for the type
-                            return scriptType.getDefaultScriptMode().toString();
+                            return defaultScriptMode.toString();
                         };
                         Setting<ScriptMode> setting =
                             new Setting<>(
diff --git a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
index 078809b43f5..9aef2496663 100644
--- a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
@@ -22,6 +22,7 @@ package org.elasticsearch.index.query;
 import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
 import com.fasterxml.jackson.core.JsonParseException;
 import com.fasterxml.jackson.core.io.JsonStringEncoder;
+import org.elasticsearch.script.ScriptMode;
 
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.Query;
@@ -237,8 +238,10 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
                 Set<ScriptEngineService> engines = new HashSet<>();
                 engines.add(mockScriptEngine);
                 List<ScriptContext.Plugin> customContexts = new ArrayList<>();
-                ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections
-                        .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class, MockScriptEngine.TYPES)));
+                ScriptEngineRegistry scriptEngineRegistry =
+                        new ScriptEngineRegistry(Collections
+                                .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
+                                                                                                 MockScriptEngine.TYPES, ScriptMode.ON)));
                 bind(ScriptEngineRegistry.class).toInstance(scriptEngineRegistry);
                 ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts);
                 bind(ScriptContextRegistry.class).toInstance(scriptContextRegistry);
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
index 30b61ada18d..bf41f0b4213 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.script;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Arrays;
@@ -44,7 +45,10 @@ public class ScriptContextTests extends ESTestCase {
             .put("script.engine." + MockScriptEngine.NAME + ".inline." + PLUGIN_NAME + "_custom_exp_disabled_op", "false")
             .build();
         Set<ScriptEngineService> engines = new HashSet<>(Collections.singletonList(new MockScriptEngine()));
-        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class, MockScriptEngine.TYPES)));
+        ScriptEngineRegistry scriptEngineRegistry =
+                new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
+                                                                                                                     MockScriptEngine.TYPES,
+                                                                                                                     ScriptMode.ON)));
         List<ScriptContext.Plugin> customContexts = Arrays.asList(
             new ScriptContext.Plugin(PLUGIN_NAME, "custom_op"),
             new ScriptContext.Plugin(PLUGIN_NAME, "custom_exp_disabled_op"),
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java b/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
index 9470bb98b6f..134a45557e1 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
@@ -99,6 +99,7 @@ public class ScriptModesTests extends ESTestCase {
             assertThat(scriptModes, notNullValue());
             //2 is the number of engines (native excluded), custom is counted twice though as it's associated with two different names
             int numberOfSettings = 2 * ScriptType.values().length * scriptContextRegistry.scriptContexts().size();
+            numberOfSettings += 6; // for top-level inline/store/file settings
             assertThat(scriptModes.scriptModes.size(), equalTo(numberOfSettings));
             if (assertAllSettingsWereChecked) {
                 assertThat(checkedSettings.size(), equalTo(numberOfSettings));
@@ -109,7 +110,7 @@ public class ScriptModesTests extends ESTestCase {
     public void testDefaultSettings() {
         this.scriptModes = new ScriptModes(scriptSettings, Settings.EMPTY);
         assertScriptModesAllOps(ScriptMode.ON, ALL_LANGS, ScriptType.FILE);
-        assertScriptModesAllOps(ScriptMode.SANDBOX, ALL_LANGS, ScriptType.STORED, ScriptType.INLINE);
+        assertScriptModesAllOps(ScriptMode.OFF, ALL_LANGS, ScriptType.STORED, ScriptType.INLINE);
     }
 
     public void testMissingSetting() {
@@ -148,10 +149,10 @@ public class ScriptModesTests extends ESTestCase {
             assertScriptModesAllOps(ScriptMode.ON, ALL_LANGS, ScriptType.FILE);
         }
         if (randomScriptTypesSet.contains(ScriptType.STORED) == false) {
-            assertScriptModesAllOps(ScriptMode.SANDBOX, ALL_LANGS, ScriptType.STORED);
+            assertScriptModesAllOps(ScriptMode.OFF, ALL_LANGS, ScriptType.STORED);
         }
         if (randomScriptTypesSet.contains(ScriptType.INLINE) == false) {
-            assertScriptModesAllOps(ScriptMode.SANDBOX, ALL_LANGS, ScriptType.INLINE);
+            assertScriptModesAllOps(ScriptMode.OFF, ALL_LANGS, ScriptType.INLINE);
         }
     }
 
@@ -179,19 +180,19 @@ public class ScriptModesTests extends ESTestCase {
 
         ScriptContext[] complementOf = complementOf(randomScriptContexts);
         assertScriptModes(ScriptMode.ON, ALL_LANGS, new ScriptType[]{ScriptType.FILE}, complementOf);
-        assertScriptModes(ScriptMode.SANDBOX, ALL_LANGS, new ScriptType[]{ScriptType.STORED, ScriptType.INLINE}, complementOf);
+        assertScriptModes(ScriptMode.OFF, ALL_LANGS, new ScriptType[]{ScriptType.STORED, ScriptType.INLINE}, complementOf);
     }
 
     public void testConflictingScriptTypeAndOpGenericSettings() {
         ScriptContext scriptContext = randomFrom(scriptContexts);
         Settings.Builder builder = Settings.builder().put("script" + "." + scriptContext.getKey(), randomFrom(DISABLE_VALUES))
-                .put("script.stored", randomFrom(ENABLE_VALUES)).put("script.inline", "sandbox");
+                .put("script.stored", randomFrom(ENABLE_VALUES)).put("script.inline", "true");
         //operations generic settings have precedence over script type generic settings
         this.scriptModes = new ScriptModes(scriptSettings, builder.build());
         assertScriptModesAllTypes(ScriptMode.OFF, ALL_LANGS, scriptContext);
         ScriptContext[] complementOf = complementOf(scriptContext);
         assertScriptModes(ScriptMode.ON, ALL_LANGS, new ScriptType[]{ScriptType.FILE, ScriptType.STORED}, complementOf);
-        assertScriptModes(ScriptMode.SANDBOX, ALL_LANGS, new ScriptType[]{ScriptType.INLINE}, complementOf);
+        assertScriptModes(ScriptMode.ON, ALL_LANGS, new ScriptType[]{ScriptType.INLINE}, complementOf);
     }
 
     private void assertScriptModesAllOps(ScriptMode expectedScriptMode, Set<String> langs, ScriptType... scriptTypes) {
@@ -208,8 +209,10 @@ public class ScriptModesTests extends ESTestCase {
         assert scriptContexts.length > 0;
         for (String lang : langs) {
             for (ScriptType scriptType : scriptTypes) {
+                checkedSettings.add("script.engine." + lang + "." + scriptType);
                 for (ScriptContext scriptContext : scriptContexts) {
-                    assertThat(lang + "." + scriptType + "." + scriptContext.getKey() + " doesn't have the expected value", scriptModes.getScriptMode(lang, scriptType, scriptContext), equalTo(expectedScriptMode));
+                    assertThat(lang + "." + scriptType + "." + scriptContext.getKey() + " doesn't have the expected value",
+                            scriptModes.getScriptMode(lang, scriptType, scriptContext), equalTo(expectedScriptMode));
                     checkedSettings.add(lang + "." + scriptType + "." + scriptContext);
                 }
             }
@@ -251,11 +254,6 @@ public class ScriptModesTests extends ESTestCase {
             return Collections.singletonList(TYPES.get(0));
         }
 
-        @Override
-        public boolean isSandboxed() {
-            return false;
-        }
-
         @Override
         public Object compile(String script, Map<String, String> params) {
             return null;
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
index 1d5a43b8158..acbcc9ab615 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
@@ -41,11 +41,14 @@ import org.junit.Before;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -57,6 +60,8 @@ public class ScriptServiceTests extends ESTestCase {
 
     private ResourceWatcherService resourceWatcherService;
     private ScriptEngineService scriptEngineService;
+    private ScriptEngineService dangerousScriptEngineService;
+    private Set<ScriptEngineService> services;
     private Map<String, ScriptEngineService> scriptEnginesByLangMap;
     private ScriptEngineRegistry scriptEngineRegistry;
     private ScriptContextRegistry scriptContextRegistry;
@@ -70,8 +75,8 @@ public class ScriptServiceTests extends ESTestCase {
 
     static {
         DEFAULT_SCRIPT_MODES.put(ScriptType.FILE, ScriptMode.ON);
-        DEFAULT_SCRIPT_MODES.put(ScriptType.STORED, ScriptMode.SANDBOX);
-        DEFAULT_SCRIPT_MODES.put(ScriptType.INLINE, ScriptMode.SANDBOX);
+        DEFAULT_SCRIPT_MODES.put(ScriptType.STORED, ScriptMode.OFF);
+        DEFAULT_SCRIPT_MODES.put(ScriptType.INLINE, ScriptMode.OFF);
     }
 
     @Before
@@ -83,6 +88,10 @@ public class ScriptServiceTests extends ESTestCase {
                 .build();
         resourceWatcherService = new ResourceWatcherService(baseSettings, null);
         scriptEngineService = new TestEngineService();
+        dangerousScriptEngineService = new TestDangerousEngineService();
+        services = new HashSet<>(2);
+        services.add(scriptEngineService);
+        services.add(dangerousScriptEngineService);
         scriptEnginesByLangMap = ScriptModesTests.buildScriptEnginesByLangMap(Collections.singleton(scriptEngineService));
         //randomly register custom script contexts
         int randomInt = randomIntBetween(0, 3);
@@ -100,7 +109,10 @@ public class ScriptServiceTests extends ESTestCase {
             String context = plugin + "_" + operation;
             contexts.put(context, new ScriptContext.Plugin(plugin, operation));
         }
-        scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(TestEngineService.class, TestEngineService.TYPES)));
+        List<ScriptEngineRegistry.ScriptEngineRegistration> registries = new ArrayList<>(2);
+        registries.add(new ScriptEngineRegistry.ScriptEngineRegistration(TestEngineService.class, TestEngineService.TYPES, ScriptMode.ON));
+        registries.add(new ScriptEngineRegistry.ScriptEngineRegistration(TestDangerousEngineService.class, TestDangerousEngineService.TYPES));
+        scriptEngineRegistry = new ScriptEngineRegistry(registries);
         scriptContextRegistry = new ScriptContextRegistry(contexts.values());
         scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
         scriptContexts = scriptContextRegistry.scriptContexts().toArray(new ScriptContext[scriptContextRegistry.scriptContexts().size()]);
@@ -112,7 +124,7 @@ public class ScriptServiceTests extends ESTestCase {
     private void buildScriptService(Settings additionalSettings) throws IOException {
         Settings finalSettings = Settings.builder().put(baseSettings).put(additionalSettings).build();
         Environment environment = new Environment(finalSettings);
-        scriptService = new ScriptService(finalSettings, environment, Collections.singleton(scriptEngineService), resourceWatcherService, scriptEngineRegistry, scriptContextRegistry, scriptSettings) {
+        scriptService = new ScriptService(finalSettings, environment, services, resourceWatcherService, scriptEngineRegistry, scriptContextRegistry, scriptSettings) {
             @Override
             String getScriptFromClusterState(ClusterState state, String scriptLang, String id) {
                 //mock the script that gets retrieved from an index
@@ -208,20 +220,14 @@ public class ScriptServiceTests extends ESTestCase {
         if (rarely()) {
             builder.put("script.file", randomFrom(ScriptModesTests.ENABLE_VALUES));
         }
-        if (rarely()) {
-            builder.put("script.stored", "sandbox");
-        }
-        if (rarely()) {
-            builder.put("script.inline", "sandbox");
-        }
         buildScriptService(builder.build());
-        createFileScripts("groovy", "mustache", "test");
+        createFileScripts("groovy", "mustache", "dtest");
 
         for (ScriptContext scriptContext : scriptContexts) {
-            //custom engine is sandboxed, all scripts are enabled by default
-            assertCompileAccepted("test", "script", ScriptType.INLINE, scriptContext);
-            assertCompileAccepted("test", "script", ScriptType.STORED, scriptContext);
-            assertCompileAccepted("test", "file_script", ScriptType.FILE, scriptContext);
+            // only file scripts are accepted by default
+            assertCompileRejected("dtest", "script", ScriptType.INLINE, scriptContext);
+            assertCompileRejected("dtest", "script", ScriptType.STORED, scriptContext);
+            assertCompileAccepted("dtest", "file_script", ScriptType.FILE, scriptContext);
         }
     }
 
@@ -266,9 +272,6 @@ public class ScriptServiceTests extends ESTestCase {
                 case OFF:
                     builder.put("script" + "." + entry.getKey().getScriptType(), "false");
                     break;
-                case SANDBOX:
-                    builder.put("script" + "." + entry.getKey().getScriptType(), "sandbox");
-                    break;
             }
         }
         for (Map.Entry<ScriptContext, ScriptMode> entry : scriptContextSettings.entrySet()) {
@@ -276,13 +279,9 @@ public class ScriptServiceTests extends ESTestCase {
                 case ON:
                     builder.put("script" + "." + entry.getKey().getKey(), "true");
                     break;
-
                 case OFF:
                     builder.put("script" + "." + entry.getKey().getKey(), "false");
                     break;
-                case SANDBOX:
-                    builder.put("script" + "." + entry.getKey().getKey(), "sandbox");
-                    break;
             }
         }
         for (Map.Entry<String, ScriptMode> entry : engineSettings.entrySet()) {
@@ -298,14 +297,11 @@ public class ScriptServiceTests extends ESTestCase {
                 case OFF:
                     builder.put("script.engine" + "." + lang + "." + part2, "false");
                     break;
-                case SANDBOX:
-                    builder.put("script.engine" + "." + lang + "." + part2, "sandbox");
-                    break;
             }
         }
 
         buildScriptService(builder.build());
-        createFileScripts("groovy", "expression", "mustache", "test");
+        createFileScripts("groovy", "expression", "mustache", "dtest");
 
         for (ScriptType scriptType : ScriptType.values()) {
             //make sure file scripts have a different name than inline ones.
@@ -313,7 +309,7 @@ public class ScriptServiceTests extends ESTestCase {
             String script = scriptType == ScriptType.FILE ? "file_script" : "script";
             for (ScriptContext scriptContext : this.scriptContexts) {
                 //fallback mechanism: 1) engine specific settings 2) op based settings 3) source based settings
-                ScriptMode scriptMode = engineSettings.get(scriptEngineService.getTypes().get(0) + "." + scriptType + "." + scriptContext.getKey());
+                ScriptMode scriptMode = engineSettings.get(dangerousScriptEngineService.getTypes().get(0) + "." + scriptType + "." + scriptContext.getKey());
                 if (scriptMode == null) {
                     scriptMode = scriptContextSettings.get(scriptContext);
                 }
@@ -324,7 +320,7 @@ public class ScriptServiceTests extends ESTestCase {
                     scriptMode = DEFAULT_SCRIPT_MODES.get(scriptType);
                 }
 
-                for (String lang : scriptEngineService.getTypes()) {
+                for (String lang : dangerousScriptEngineService.getTypes()) {
                     switch (scriptMode) {
                         case ON:
                             assertCompileAccepted(lang, script, scriptType, scriptContext);
@@ -332,13 +328,6 @@ public class ScriptServiceTests extends ESTestCase {
                         case OFF:
                             assertCompileRejected(lang, script, scriptType, scriptContext);
                             break;
-                        case SANDBOX:
-                            if (scriptEngineService.isSandboxed()) {
-                                assertCompileAccepted(lang, script, scriptType, scriptContext);
-                            } else {
-                                assertCompileRejected(lang, script, scriptType, scriptContext);
-                            }
-                            break;
                     }
                 }
             }
@@ -401,6 +390,7 @@ public class ScriptServiceTests extends ESTestCase {
     public void testCompilationStatsOnCacheHit() throws IOException {
         Settings.Builder builder = Settings.builder();
         builder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getKey(), 1);
+        builder.put("script.inline", "true");
         buildScriptService(builder.build());
         ClusterState state = ClusterState.builder(new ClusterName("_name")).build();
         scriptService.executable(new Script("1+1", ScriptType.INLINE, "test", null), randomFrom(scriptContexts), Collections.emptyMap(), state);
@@ -426,6 +416,7 @@ public class ScriptServiceTests extends ESTestCase {
     public void testCacheEvictionCountedInCacheEvictionsStats() throws IOException {
         Settings.Builder builder = Settings.builder();
         builder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getKey(), 1);
+        builder.put("script.inline", "true");
         buildScriptService(builder.build());
         ClusterState state = ClusterState.builder(new ClusterName("_name")).build();
         scriptService.executable(new Script("1+1", ScriptType.INLINE, "test", null), randomFrom(scriptContexts), Collections.emptyMap(), state);
@@ -437,6 +428,7 @@ public class ScriptServiceTests extends ESTestCase {
     public void testDefaultLanguage() throws IOException {
         Settings.Builder builder = Settings.builder();
         builder.put("script.default_lang", "test");
+        builder.put("script.inline", "true");
         buildScriptService(builder.build());
         CompiledScript script = scriptService.compile(new Script("1 + 1", ScriptType.INLINE, null, null),
                 randomFrom(scriptContexts), Collections.emptyMap(), emptyClusterState());
@@ -552,8 +544,45 @@ public class ScriptServiceTests extends ESTestCase {
         }
 
         @Override
-        public boolean isSandboxed() {
-            return true;
+        public Object compile(String script, Map<String, String> params) {
+            return "compiled_" + script;
+        }
+
+        @Override
+        public ExecutableScript executable(final CompiledScript compiledScript, @Nullable Map<String, Object> vars) {
+            return null;
+        }
+
+        @Override
+        public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, @Nullable Map<String, Object> vars) {
+            return null;
+        }
+
+        @Override
+        public void close() {
+
+        }
+
+        @Override
+        public void scriptRemoved(CompiledScript script) {
+            // Nothing to do here
+        }
+    }
+
+    public static class TestDangerousEngineService implements ScriptEngineService {
+
+        public static final List<String> TYPES = Collections.unmodifiableList(Arrays.asList("dtest"));
+
+        public static final List<String> EXTENSIONS = Collections.unmodifiableList(Arrays.asList("dtest"));
+
+        @Override
+        public List<String> getTypes() {
+            return TYPES;
+        }
+
+        @Override
+        public List<String> getExtensions() {
+            return EXTENSIONS;
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java b/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
index eeae3622b0a..952e2598aa1 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
@@ -110,11 +110,6 @@ public class ScriptSettingsTests extends ESTestCase {
             return Collections.singletonList(TYPES.get(0));
         }
 
-        @Override
-        public boolean isSandboxed() {
-            return false;
-        }
-
         @Override
         public Object compile(String script, Map<String, String> params) {
             return null;
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
index 2acbd919484..7efef03d114 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
@@ -55,6 +55,7 @@ import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptContextRegistry;
 import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
+import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptSettings;
@@ -144,8 +145,10 @@ public abstract class BasePipelineAggregationTestCase<AF extends PipelineAggrega
                 Set<ScriptEngineService> engines = new HashSet<>();
                 engines.add(mockScriptEngine);
                 List<ScriptContext.Plugin> customContexts = new ArrayList<>();
-                ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections
-                        .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class, MockScriptEngine.TYPES)));
+                ScriptEngineRegistry scriptEngineRegistry =
+                        new ScriptEngineRegistry(Collections
+                                .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
+                                                                                                 MockScriptEngine.TYPES, ScriptMode.ON)));
                 bind(ScriptEngineRegistry.class).toInstance(scriptEngineRegistry);
                 ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts);
                 bind(ScriptContextRegistry.class).toInstance(scriptContextRegistry);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
index a8187578d71..248f75b6413 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
@@ -28,6 +28,7 @@ import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
+import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.script.SearchScript;
@@ -369,7 +370,7 @@ public class AvgIT extends AbstractNumericTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractFieldScriptEngine.class, ExtractFieldScriptEngine.TYPES));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractFieldScriptEngine.class, ExtractFieldScriptEngine.TYPES, ScriptMode.ON));
         }
 
     }
@@ -397,11 +398,6 @@ public class AvgIT extends AbstractNumericTestCase {
             return TYPES;
         }
 
-        @Override
-        public boolean isSandboxed() {
-            return true;
-        }
-
         @Override
         public Object compile(String script, Map<String, String> params) {
             return script;
@@ -501,7 +497,7 @@ public class AvgIT extends AbstractNumericTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.TYPES));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.TYPES, ScriptMode.ON));
         }
 
     }
@@ -529,11 +525,6 @@ public class AvgIT extends AbstractNumericTestCase {
             return TYPES;
         }
 
-        @Override
-        public boolean isSandboxed() {
-            return true;
-        }
-
         @Override
         public Object compile(String script, Map<String, String> params) {
             return script;
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
index 462356d4108..401a9baaa6d 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
@@ -28,6 +28,7 @@ import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
+import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.script.SearchScript;
@@ -364,7 +365,7 @@ public class SumIT extends AbstractNumericTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractFieldScriptEngine.class, ExtractFieldScriptEngine.TYPES));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractFieldScriptEngine.class, ExtractFieldScriptEngine.TYPES, ScriptMode.ON));
         }
 
     }
@@ -393,11 +394,6 @@ public class SumIT extends AbstractNumericTestCase {
             return TYPES;
         }
 
-        @Override
-        public boolean isSandboxed() {
-            return true;
-        }
-
         @Override
         public Object compile(String script, Map<String, String> params) {
             return script;
@@ -498,7 +494,7 @@ public class SumIT extends AbstractNumericTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.TYPES));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.TYPES, ScriptMode.ON));
         }
 
     }
@@ -527,11 +523,6 @@ public class SumIT extends AbstractNumericTestCase {
             return TYPES;
         }
 
-        @Override
-        public boolean isSandboxed() {
-            return true;
-        }
-
         @Override
         public Object compile(String script, Map<String, String> params) {
             return script;
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
index 381335cb005..bc75481dc3d 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
@@ -28,6 +28,7 @@ import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
+import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.script.SearchScript;
@@ -223,7 +224,7 @@ public class ValueCountIT extends ESIntegTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.TYPES));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.TYPES, ScriptMode.ON));
         }
 
     }
@@ -251,11 +252,6 @@ public class ValueCountIT extends ESIntegTestCase {
             return TYPES;
         }
 
-        @Override
-        public boolean isSandboxed() {
-            return true;
-        }
-
         @Override
         public Object compile(String script, Map<String, String> params) {
             return script;
diff --git a/core/src/test/java/org/elasticsearch/update/UpdateIT.java b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
index b6d785a4977..095b3de7678 100644
--- a/core/src/test/java/org/elasticsearch/update/UpdateIT.java
+++ b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
@@ -43,6 +43,7 @@ import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
+import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.SearchScript;
@@ -93,7 +94,7 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(PutFieldValuesScriptEngine.class, PutFieldValuesScriptEngine.TYPES));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(PutFieldValuesScriptEngine.class, PutFieldValuesScriptEngine.TYPES, ScriptMode.ON));
         }
 
     }
@@ -118,11 +119,6 @@ public class UpdateIT extends ESIntegTestCase {
             return TYPES;
         }
 
-        @Override
-        public boolean isSandboxed() {
-            return true;
-        }
-
         @Override
         public Object compile(String script, Map<String, String> params) {
             return new Object(); // unused
@@ -187,7 +183,7 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldIncrementScriptEngine.class, FieldIncrementScriptEngine.TYPES));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldIncrementScriptEngine.class, FieldIncrementScriptEngine.TYPES, ScriptMode.ON));
         }
 
     }
@@ -212,11 +208,6 @@ public class UpdateIT extends ESIntegTestCase {
             return TYPES;
         }
 
-        @Override
-        public boolean isSandboxed() {
-            return true;
-        }
-
         @Override
         public Object compile(String script, Map<String, String> params) {
             return script;
@@ -274,7 +265,7 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ScriptedUpsertScriptEngine.class, ScriptedUpsertScriptEngine.TYPES));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ScriptedUpsertScriptEngine.class, ScriptedUpsertScriptEngine.TYPES, ScriptMode.ON));
         }
 
     }
@@ -299,11 +290,6 @@ public class UpdateIT extends ESIntegTestCase {
             return TYPES;
         }
 
-        @Override
-        public boolean isSandboxed() {
-            return true;
-        }
-
         @Override
         public Object compile(String script, Map<String, String> params) {
             return new Object(); // unused
@@ -361,7 +347,7 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractContextInSourceScriptEngine.class, ExtractContextInSourceScriptEngine.TYPES));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractContextInSourceScriptEngine.class, ExtractContextInSourceScriptEngine.TYPES, ScriptMode.ON));
         }
 
     }
@@ -386,11 +372,6 @@ public class UpdateIT extends ESIntegTestCase {
             return TYPES;
         }
 
-        @Override
-        public boolean isSandboxed() {
-            return true;
-        }
-
         @Override
         public Object compile(String script, Map<String, String> params) {
             return new Object(); // unused
diff --git a/docs/reference/migration/migrate_5_0/settings.asciidoc b/docs/reference/migration/migrate_5_0/settings.asciidoc
index dbccd9dd74a..e916ec6b24c 100644
--- a/docs/reference/migration/migrate_5_0/settings.asciidoc
+++ b/docs/reference/migration/migrate_5_0/settings.asciidoc
@@ -51,16 +51,6 @@ All settings with a `netty` infix have been replaced by their already existing
 supported and should be replaced by the superseding setting
 `transport.bind_host`.
 
-==== Script mode settings
-
-Previously script mode settings (e.g., "script.inline: true",
-"script.engine.groovy.inline.aggs: false", etc.) accepted the values
-`on`, `true`, `1`, and `yes` for enabling a scripting mode, and the
-values `off`, `false`, `0`, and `no` for disabling a scripting mode.
-The variants `on`, `1`, and `yes` for enabling and `off`, `0`,
-and `no` for disabling are no longer supported.
-
-
 ==== Security manager settings
 
 The option to disable the security manager `security.manager.enabled` has been
@@ -241,6 +231,8 @@ a fallback realtime setting for the get and mget APIs when realtime
 wasn't specified. Now if the parameter isn't specified we always
 default to true.
 
+=== Script settings
+
 ==== Indexed script settings
 
 Due to the fact that indexed script has been replaced by stored
@@ -248,7 +240,21 @@ scripts the following settings have been replaced to:
 
 * `script.indexed` has been replaced by `script.stored`
 * `script.engine.*.indexed.aggs` has been replaced by `script.engine.*.stored.aggs` (where `*` represents the script language, like `groovy`, `mustache`, `plainless` etc.)
-* `script.engine.*.stored.mapping` has been replaced by `script.engine.*.stored.mapping` (where `*` represents the script language, like `groovy`, `mustache`, `plainless` etc.)
-* `script.engine.*.stored.search` has been replaced by `script.engine.*.stored.search` (where `*` represents the script language, like `groovy`, `mustache`, `plainless` etc.)
-* `script.engine.*.stored.update` has been replaced by `script.engine.*.stored.update` (where `*` represents the script language, like `groovy`, `mustache`, `plainless` etc.)
-* `script.engine.*.stored.plugin` has been replaced by `script.engine.*.stored.plugin` (where `*` represents the script language, like `groovy`, `mustache`, `plainless` etc.)
+* `script.engine.*.indexed.mapping` has been replaced by `script.engine.*.stored.mapping` (where `*` represents the script language, like `groovy`, `mustache`, `plainless` etc.)
+* `script.engine.*.indexed.search` has been replaced by `script.engine.*.stored.search` (where `*` represents the script language, like `groovy`, `mustache`, `plainless` etc.)
+* `script.engine.*.indexed.update` has been replaced by `script.engine.*.stored.update` (where `*` represents the script language, like `groovy`, `mustache`, `plainless` etc.)
+* `script.engine.*.indexed.plugin` has been replaced by `script.engine.*.stored.plugin` (where `*` represents the script language, like `groovy`, `mustache`, `plainless` etc.)
+
+==== Script mode settings
+
+Previously script mode settings (e.g., "script.inline: true",
+"script.engine.groovy.inline.aggs: false", etc.) accepted a wide range of
+"truthy" or "falsy" values. This is now much stricter and supports only the
+`true` and `false` options.
+
+
+==== Script sandbox settings removed
+
+Prior to 5.0 a third option could be specified for the `script.inline` and
+`script.stored` settings ("sandbox"). This has been removed, You can now only
+set `script.line: true` or `script.stored: true`.
diff --git a/docs/reference/modules/scripting.asciidoc b/docs/reference/modules/scripting.asciidoc
index feb8113a6da..7bc5c3aa91e 100644
--- a/docs/reference/modules/scripting.asciidoc
+++ b/docs/reference/modules/scripting.asciidoc
@@ -24,23 +24,18 @@ and give the most flexibility.
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Language
-    |Sandboxed
     |Required plugin
 
 |<<modules-scripting-painless, `painless`>>
-    |yes
     |built-in
 
 |<<modules-scripting-groovy, `groovy`>>
-    |<<modules-scripting-security, no>>
     |built-in
 
 |{plugins}/lang-javascript.html[`javascript`]
-    |<<modules-scripting-security, no>>
     |{plugins}/lang-javascript.html[`lang-javascript`]
 
 |{plugins}/lang-python.html[`python`]
-    |<<modules-scripting-security, no>>
     |{plugins}/lang-python.html[`lang-python`]
 
 |=======================================================================
@@ -54,35 +49,23 @@ certain tasks.
 [cols="<,<,<,<",options="header",]
 |=======================================================================
 |Language
-    |Sandboxed
     |Required plugin
     |Purpose
 
 |<<modules-scripting-expression, `expression`>>
-    |yes
     |built-in
     |fast custom ranking and sorting
 
 |<<search-template, `mustache`>>
-    |yes
     |built-in
     |templates
 
 |<<modules-scripting-native, `java`>>
-    |n/a
     |you write it!
     |expert API
 
 |=======================================================================
 
-[WARNING]
-.Scripts and security
-=================================================
-
-Languages that are sandboxed are designed with security in mind. However, non-
-sandboxed languages can be a security issue, please read
-<<modules-scripting-security, Scripting and security>> for more details.
-
 =================================================
 
 
diff --git a/docs/reference/modules/scripting/groovy.asciidoc b/docs/reference/modules/scripting/groovy.asciidoc
index 0c7f4676d2c..153acaf891a 100644
--- a/docs/reference/modules/scripting/groovy.asciidoc
+++ b/docs/reference/modules/scripting/groovy.asciidoc
@@ -1,9 +1,9 @@
 [[modules-scripting-groovy]]
 === Groovy Scripting Language
 
-Groovy is the default scripting language available in Elasticsearch.  Although
-limited by the <<java-security-manager,Java Security Manager>>, it is not a
-sandboxed language and only `file` scripts may be used by default.
+Groovy is the default scripting language available in Elasticsearch. Although
+limited by the <<java-security-manager,Java Security Manager>>, only `file`
+scripts may be used by default.
 
 Enabling `inline` or `stored` Groovy scripting is a security risk and should
 only be considered if your Elasticsearch cluster is protected from the outside
diff --git a/docs/reference/modules/scripting/security.asciidoc b/docs/reference/modules/scripting/security.asciidoc
index c5ad89538e3..94eb41ae152 100644
--- a/docs/reference/modules/scripting/security.asciidoc
+++ b/docs/reference/modules/scripting/security.asciidoc
@@ -53,22 +53,20 @@ Each of these settings takes one of these values:
 [horizontal]
 `false`::   Scripting is enabled.
 `true`::    Scripting is disabled.
-`sandbox`:: Scripting is enabled only for sandboxed languages.
 
 The default values are the following:
 
 [source,yaml]
 -----------------------------------
-script.inline: sandbox
-script.stored: sandbox
+script.inline: false
+script.stored: false
 script.file:   true
 -----------------------------------
 
 NOTE: Global scripting settings affect the `mustache` scripting language.
-<<search-template,Search templates>> internally use the `mustache` language,
-and will still be enabled by default as the `mustache` engine is sandboxed,
-but they will be enabled/disabled according to fine-grained settings
-specified in `elasticsearch.yml`.
+<<search-template,Search templates>> internally use the `mustache` language, and
+will be disabled by default, but they can be enabled/disabled according to
+fine-grained settings specified in `elasticsearch.yml`.
 
 [[security-script-context]]
 [float]
@@ -88,9 +86,9 @@ of using the generic `plugin` category. Those operations can be referred to
 in the following form: `${pluginName}_${operation}`.
 
 The following example disables scripting for `update` and `plugin` operations,
-regardless of the script source or language. Scripts can still be executed
-from sandboxed languages as part of `aggregations`, `search` and plugins
-execution though, as the above defaults still get applied.
+regardless of the script source or language. Scripts can be executed from
+languages as part of `aggregations`, `search` and plugins execution though,
+assuming they are enabled in the scripting settings.
 
 [source,yaml]
 -----------------------------------
@@ -114,6 +112,13 @@ Fine-grained settings have the form:
 script.engine.{lang}.{source}.{context}: true|false
 ------------------------
 
+And
+
+[source,yaml]
+------------------------
+script.engine.{lang}.{inline|file|stored}: true|false
+------------------------
+
 For example:
 
 [source,yaml]
@@ -122,14 +127,16 @@ script.inline: false <1>
 script.stored: false <1>
 script.file:   false <1>
 
-script.engine.groovy.stored.search:   true <2>
-script.engine.groovy.stored.aggs:     true <2>
+script.engine.groovy.inline:          true <2>
+script.engine.groovy.stored.search:   true <3>
+script.engine.groovy.stored.aggs:     true <3>
 
-script.engine.mustache.stored.search: true <3>
+script.engine.mustache.stored.search: true <4>
 -----------------------------------
 <1> Disable all scripting from any source.
-<2> Allow stored Groovy scripts to be used for search and aggregations.
-<3> Allow stored Mustache templates to be used for search.
+<2> Allow inline Groovy scripts for all operations
+<3> Allow stored Groovy scripts to be used for search and aggregations.
+<4> Allow stored Mustache templates to be used for search.
 
 [[java-security-manager]]
 [float]
diff --git a/docs/reference/modules/scripting/using.asciidoc b/docs/reference/modules/scripting/using.asciidoc
index 856311fac24..955bfb49b3f 100644
--- a/docs/reference/modules/scripting/using.asciidoc
+++ b/docs/reference/modules/scripting/using.asciidoc
@@ -110,12 +110,15 @@ second version is only compiled once.
 [[modules-scripting-file-scripts]]
 === File-based Scripts
 
-To increase security, non-sandboxed languages can only be specified in script
-files stored on every node in the cluster.  File scripts must be saved in the
-`scripts` directory whose default location depends on whether you use  the
-<<zip-targz-layout,`zip`/`tar.gz`>> (`$ES_HOME/config/scripts/`),
-<<rpm-layout,RPM>>, or <<deb-layout,Debian>> package.  The default may be
-changed with the `path.script` setting.
+To increase security, scripts for languages that are not deemed to be safe by
+default can only be specified in files stored on every node in the cluster. File
+scripts must be saved in the `scripts` directory whose default location depends
+on whether you use the <<zip-targz-layout,`zip`/`tar.gz`>>
+(`$ES_HOME/config/scripts/`), <<rpm-layout,RPM>>, or <<deb-layout,Debian>>
+package. The default may be changed with the `path.script` setting.
+
+The languages which are assumed to be safe by default are: painless,
+expressions, and mustache (used for query templates).
 
 Any files placed in the `scripts` directory will be compiled automatically
 when the node starts up and then <<reload-scripts,every 60 seconds thereafter>>.
diff --git a/docs/reference/search/search-template.asciidoc b/docs/reference/search/search-template.asciidoc
index 25c4c0423f1..291be9379ba 100644
--- a/docs/reference/search/search-template.asciidoc
+++ b/docs/reference/search/search-template.asciidoc
@@ -25,10 +25,10 @@ For more information on how Mustache templating and what kind of templating you
 can do with it check out the http://mustache.github.io/mustache.5.html[online
 documentation of the mustache project].
 
-NOTE: The mustache language is implemented in elasticsearch as a sandboxed
-scripting language, hence it obeys settings that may be used to enable or
-disable scripts per language, source and operation as described in
-<<enable-dynamic-scripting, scripting docs>>
+NOTE: The mustache language is implemented in elasticsearch as a scripting
+language, hence it obeys settings that may be used to enable or disable scripts
+per language, source and operation as described in <<enable-dynamic-scripting,
+scripting docs>>
 
 [float]
 ==== More template examples
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java
index a987855bae3..01e9aa18e91 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java
@@ -21,6 +21,7 @@ package org.elasticsearch.script.expression;
 
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptEngineRegistry;
+import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 
 public class ExpressionPlugin extends Plugin {
@@ -36,6 +37,7 @@ public class ExpressionPlugin extends Plugin {
     }
 
     public void onModule(ScriptModule module) {
-        module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExpressionScriptEngineService.class, ExpressionScriptEngineService.TYPES));
+        module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExpressionScriptEngineService.class,
+                        ExpressionScriptEngineService.TYPES, ScriptMode.ON));
     }
 }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
index a6ed6253263..168bf25c646 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
@@ -79,11 +79,6 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
         return TYPES;
     }
 
-    @Override
-    public boolean isSandboxed() {
-        return true;
-    }
-
     @Override
     public Object compile(String script, Map<String, String> params) {
         // classloader created here
diff --git a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
index 60a8a0c1338..086118cab38 100644
--- a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
+++ b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
@@ -170,11 +170,6 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
         return TYPES;
     }
 
-    @Override
-    public boolean isSandboxed() {
-        return false;
-    }
-
     @Override
     public Object compile(String script, Map<String, String> params) {
         try {
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
index 935dab7e439..1dc3fd96f81 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
@@ -21,6 +21,7 @@ package org.elasticsearch.script.mustache;
 
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptEngineRegistry;
+import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 
 public class MustachePlugin extends Plugin {
@@ -36,6 +37,7 @@ public class MustachePlugin extends Plugin {
     }
 
     public void onModule(ScriptModule module) {
-        module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(MustacheScriptEngineService.class, MustacheScriptEngineService.TYPES));
+        module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(MustacheScriptEngineService.class,
+                        MustacheScriptEngineService.TYPES, ScriptMode.ON));
     }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
index 647a727b2dd..4b0cb8fed4c 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
@@ -121,11 +121,6 @@ public final class MustacheScriptEngineService extends AbstractComponent impleme
         return TYPES;
     }
 
-    @Override
-    public boolean isSandboxed() {
-        return true;
-    }
-
     @Override
     public ExecutableScript executable(CompiledScript compiledScript,
             @Nullable Map<String, Object> vars) {
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
index 73a63088bf6..b579ca3cc10 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
@@ -94,6 +94,7 @@ public class TemplateQueryParserTests extends ESTestCase {
                 .put(Environment.PATH_CONF_SETTING.getKey(), this.getDataPath("config"))
                 .put("node.name", getClass().getName())
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put("script.inline", "true")
                 .build();
         final Client proxy = (Client) Proxy.newProxyInstance(
                 Client.class.getClassLoader(),
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
index 8ab6195ed18..795fbdac28b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
@@ -21,6 +21,7 @@ package org.elasticsearch.painless;
 
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptEngineRegistry;
+import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 
 /**
@@ -40,6 +41,6 @@ public final class PainlessPlugin extends Plugin {
 
     public void onModule(final ScriptModule module) {
         module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(
-            PainlessScriptEngineService.class, PainlessScriptEngineService.TYPES));
+                        PainlessScriptEngineService.class, PainlessScriptEngineService.TYPES, ScriptMode.ON));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
index c219d747c0b..cab6950fda1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
@@ -116,15 +116,6 @@ public final class PainlessScriptEngineService extends AbstractComponent impleme
         return EXTENSIONS;
     }
 
-    /**
-     * Whether or not the engine is secure.
-     * @return Always true as the engine should be secure at runtime.
-     */
-    @Override
-    public boolean isSandboxed() {
-        return true;
-    }
-
     /**
      * Compiles a Painless script with the specified parameters.
      * @param script The code to be compiled.
diff --git a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
index e835255fccb..843a6e57142 100644
--- a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
+++ b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
@@ -168,11 +168,6 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements
         return Collections.unmodifiableList(Arrays.asList("js"));
     }
 
-    @Override
-    public boolean isSandboxed() {
-        return false;
-    }
-
     @Override
     public Object compile(String script, Map<String, String> params) {
         Context ctx = Context.enter();
diff --git a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
index 13571bcd883..c9f05def941 100644
--- a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
+++ b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
@@ -105,11 +105,6 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri
         return Collections.unmodifiableList(Arrays.asList("py"));
     }
 
-    @Override
-    public boolean isSandboxed() {
-        return false;
-    }
-
     @Override
     public Object compile(String script, Map<String, String> params) {
         // classloader created here
diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
index defc8dbf6f4..dd96b7600ae 100644
--- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
+++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
@@ -23,6 +23,7 @@ import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
@@ -55,7 +56,8 @@ public class MockScriptEngine implements ScriptEngineService {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class, MockScriptEngine.TYPES));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
+                            MockScriptEngine.TYPES, ScriptMode.ON));
         }
 
     }
@@ -70,11 +72,6 @@ public class MockScriptEngine implements ScriptEngineService {
         return TYPES;
     }
 
-    @Override
-    public boolean isSandboxed() {
-        return true;
-    }
-
     @Override
     public Object compile(String script, Map<String, String> params) {
         return script;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index 247a340d8c4..4e83c1ea2bc 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -1622,9 +1622,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
                 // from failing on nodes without enough disk space
                 .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1b")
                 .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1b")
-                .put("script.stored", "true")
-                .put("script.inline", "true")
-                        // wait short time for other active shards before actually deleting, default 30s not needed in tests
+                // wait short time for other active shards before actually deleting, default 30s not needed in tests
                 .put(IndicesStore.INDICES_STORE_DELETE_SHARD_TIMEOUT.getKey(), new TimeValue(1, TimeUnit.SECONDS));
         return builder.build();
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
index 1e75f3d8261..f45a7530e7f 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
@@ -178,9 +178,6 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
             // This needs to tie into the ESIntegTestCase#indexSettings() method
             .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), createTempDir().getParent())
             .put("node.name", nodeName())
-
-            .put("script.inline", "true")
-            .put("script.stored", "true")
             .put(EsExecutors.PROCESSORS_SETTING.getKey(), 1) // limit the number of threads created
             .put("http.enabled", false)
             .put(Node.NODE_LOCAL_SETTING.getKey(), true)

From efff3918d8f07595b44384eb8277878d506c06f4 Mon Sep 17 00:00:00 2001
From: Lee Hinman <lee@writequit.org>
Date: Wed, 11 May 2016 08:06:07 -0600
Subject: [PATCH 0230/1311] Remove support for mulitple languages per scripting
 engine

---
 .../script/NativeScriptEngineService.java     |   6 +-
 .../script/ScriptEngineRegistry.java          |  56 ++++----
 .../script/ScriptEngineService.java           |   2 +-
 .../elasticsearch/script/ScriptModule.java    |   2 +-
 .../elasticsearch/script/ScriptService.java   |  11 +-
 .../elasticsearch/script/ScriptSettings.java  | 122 ++++++++----------
 .../action/bulk/BulkRequestTests.java         |   2 +-
 .../index/query/AbstractQueryTestCase.java    |   2 +-
 .../elasticsearch/script/FileScriptTests.java |   3 +-
 .../script/NativeScriptTests.java             |   3 +-
 .../script/ScriptContextTests.java            |   7 +-
 .../script/ScriptModesTests.java              |  83 ++++++------
 .../script/ScriptServiceTests.java            |  75 ++++-------
 .../script/ScriptSettingsTests.java           |  47 ++-----
 .../aggregations/AggregatorParsingTests.java  |   8 +-
 .../aggregations/BaseAggregationTestCase.java |   8 +-
 .../BasePipelineAggregationTestCase.java      |   2 +-
 .../search/aggregations/metrics/AvgIT.java    |  12 +-
 .../search/aggregations/metrics/SumIT.java    |  12 +-
 .../aggregations/metrics/ValueCountIT.java    |   6 +-
 .../builder/SearchSourceBuilderTests.java     |   5 +-
 .../search/sort/AbstractSortTestCase.java     |   2 +-
 .../org/elasticsearch/update/UpdateIT.java    |  24 ++--
 .../action/bulk/simple-bulk4.json             |   2 +-
 docs/plugins/lang-javascript.asciidoc         |   2 +-
 docs/reference/docs/bulk.asciidoc             |   2 +-
 .../migration/migrate_5_0/scripting.asciidoc  |  10 +-
 docs/reference/modules/scripting.asciidoc     |  17 +++
 .../modules/scripting/groovy.asciidoc         |   6 +-
 .../modules/scripting/security.asciidoc       |  13 +-
 .../modules/scripting/using.asciidoc          |  16 +--
 .../reference/search/search-template.asciidoc |   8 +-
 .../script/expression/ExpressionPlugin.java   |   2 +-
 .../ExpressionScriptEngineService.java        |   4 +-
 .../script/groovy/GroovyPlugin.java           |   2 +-
 .../groovy/GroovyScriptEngineService.java     |   4 +-
 .../script/mustache/MustachePlugin.java       |   2 +-
 .../mustache/MustacheScriptEngineService.java |   4 +-
 .../messy/tests/TemplateQueryParserTests.java |   4 +-
 .../painless/PainlessPlugin.java              |   2 +-
 .../painless/PainlessScriptEngineService.java |   4 +-
 .../plugin/javascript/JavaScriptPlugin.java   |   2 +-
 .../JavaScriptScriptEngineService.java        |  10 +-
 .../test/lang_javascript/20_search.yaml       |  44 +++----
 .../plugin/python/PythonPlugin.java           |   2 +-
 .../python/PythonScriptEngineService.java     |   6 +-
 .../ingest/AbstractMustacheTestCase.java      |   8 +-
 .../script/MockScriptEngine.java              |   6 +-
 .../elasticsearch/test/ESIntegTestCase.java   |   4 +-
 .../test/ESSingleNodeTestCase.java            |   3 +
 50 files changed, 336 insertions(+), 353 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java b/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java
index e1d11ee49cf..093cde4aa04 100644
--- a/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java
+++ b/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java
@@ -40,8 +40,6 @@ public class NativeScriptEngineService extends AbstractComponent implements Scri
 
     public static final String NAME = "native";
 
-    public static final List<String> TYPES = Collections.singletonList(NAME);
-
     private final Map<String, NativeScriptFactory> scripts;
 
     @Inject
@@ -51,8 +49,8 @@ public class NativeScriptEngineService extends AbstractComponent implements Scri
     }
 
     @Override
-    public List<String> getTypes() {
-        return TYPES;
+    public String getType() {
+        return NAME;
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptEngineRegistry.java b/core/src/main/java/org/elasticsearch/script/ScriptEngineRegistry.java
index f64c40bc773..226fd7b6c34 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptEngineRegistry.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptEngineRegistry.java
@@ -24,33 +24,34 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import org.elasticsearch.common.Strings;
 
 public class ScriptEngineRegistry {
 
-    private final Map<Class<? extends ScriptEngineService>, List<String>> registeredScriptEngineServices;
+    private final Map<Class<? extends ScriptEngineService>, String> registeredScriptEngineServices;
     private final Map<String, Class<? extends ScriptEngineService>> registeredLanguages;
     private final Map<String, ScriptMode> defaultInlineScriptModes;
 
     public ScriptEngineRegistry(Iterable<ScriptEngineRegistration> registrations) {
         Objects.requireNonNull(registrations);
-        Map<Class<? extends ScriptEngineService>, List<String>> registeredScriptEngineServices = new HashMap<>();
+        Map<Class<? extends ScriptEngineService>, String> registeredScriptEngineServices = new HashMap<>();
         Map<String, Class<? extends ScriptEngineService>> registeredLanguages = new HashMap<>();
         Map<String, ScriptMode> inlineScriptModes = new HashMap<>();
         for (ScriptEngineRegistration registration : registrations) {
-            List<String> languages =
-                registeredScriptEngineServices.putIfAbsent(registration.getScriptEngineService(), Collections.unmodifiableList(registration.getScriptEngineLanguages()));
-            if (languages != null) {
-                throw new IllegalArgumentException("script engine service [" + registration.getScriptEngineService() + "] already registered for languages [" + String.join(",", languages) + "]");
+            String oldLanguage = registeredScriptEngineServices.putIfAbsent(registration.getScriptEngineService(),
+                    registration.getScriptEngineLanguage());
+            if (oldLanguage != null) {
+                throw new IllegalArgumentException("script engine service [" + registration.getScriptEngineService() +
+                                "] already registered for language [" + oldLanguage + "]");
             }
-
-            for (String language : registration.getScriptEngineLanguages()) {
-                Class<? extends ScriptEngineService> scriptEngineServiceClazz =
+            String language = registration.getScriptEngineLanguage();
+            Class<? extends ScriptEngineService> scriptEngineServiceClazz =
                     registeredLanguages.putIfAbsent(language, registration.getScriptEngineService());
-                if (scriptEngineServiceClazz != null) {
-                    throw new IllegalArgumentException("scripting language [" + language + "] already registered for script engine service [" + scriptEngineServiceClazz.getCanonicalName() + "]");
-                }
-                inlineScriptModes.put(language, registration.getDefaultInlineScriptMode());
+            if (scriptEngineServiceClazz != null) {
+                throw new IllegalArgumentException("scripting language [" + language + "] already registered for script engine service [" +
+                                scriptEngineServiceClazz.getCanonicalName() + "]");
             }
+            inlineScriptModes.put(language, registration.getDefaultInlineScriptMode());
         }
 
         this.registeredScriptEngineServices = Collections.unmodifiableMap(registeredScriptEngineServices);
@@ -62,7 +63,7 @@ public class ScriptEngineRegistry {
         return registeredScriptEngineServices.keySet();
     }
 
-    List<String> getLanguages(Class<? extends ScriptEngineService> scriptEngineService) {
+    String getLanguage(Class<? extends ScriptEngineService> scriptEngineService) {
         Objects.requireNonNull(scriptEngineService);
         return registeredScriptEngineServices.get(scriptEngineService);
     }
@@ -77,23 +78,28 @@ public class ScriptEngineRegistry {
 
     public static class ScriptEngineRegistration {
         private final Class<? extends ScriptEngineService> scriptEngineService;
-        private final List<String> scriptEngineLanguages;
+        private final String scriptEngineLanguage;
         private final ScriptMode defaultInlineScriptMode;
 
-        public ScriptEngineRegistration(Class<? extends ScriptEngineService> scriptEngineService, List<String> scriptEngineLanguages) {
-            // Default to "off/disabled" if not specified
-            this(scriptEngineService, scriptEngineLanguages, ScriptMode.OFF);
+        /**
+         * Register a script engine service with the default of inline scripts disabled
+         */
+        public ScriptEngineRegistration(Class<? extends ScriptEngineService> scriptEngineService, String scriptEngineLanguage) {
+            this(scriptEngineService, scriptEngineLanguage, ScriptMode.OFF);
         }
 
-        public ScriptEngineRegistration(Class<? extends ScriptEngineService> scriptEngineService, List<String> scriptEngineLanguages,
+        /**
+         * Register a script engine service with the given default mode for inline scripts
+         */
+        public ScriptEngineRegistration(Class<? extends ScriptEngineService> scriptEngineService, String scriptEngineLanguage,
                                         ScriptMode defaultInlineScriptMode) {
             Objects.requireNonNull(scriptEngineService);
-            Objects.requireNonNull(scriptEngineLanguages);
-            if (scriptEngineLanguages.isEmpty()) {
-                throw new IllegalArgumentException("languages for script engine service [" + scriptEngineService.getCanonicalName() + "] should be non-empty");
+            if (Strings.hasText(scriptEngineLanguage) == false) {
+                throw new IllegalArgumentException("languages for script engine service [" +
+                                scriptEngineService.getCanonicalName() + "] should be a non-empty string");
             }
             this.scriptEngineService = scriptEngineService;
-            this.scriptEngineLanguages = scriptEngineLanguages;
+            this.scriptEngineLanguage = scriptEngineLanguage;
             this.defaultInlineScriptMode = defaultInlineScriptMode;
         }
 
@@ -101,8 +107,8 @@ public class ScriptEngineRegistry {
             return scriptEngineService;
         }
 
-        List<String> getScriptEngineLanguages() {
-            return scriptEngineLanguages;
+        String getScriptEngineLanguage() {
+            return scriptEngineLanguage;
         }
 
         ScriptMode getDefaultInlineScriptMode() {
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java b/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java
index 29046f9d906..b89871b4dab 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java
@@ -31,7 +31,7 @@ import java.util.Map;
  */
 public interface ScriptEngineService extends Closeable {
 
-    List<String> getTypes();
+    String getType();
 
     List<String> getExtensions();
 
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptModule.java b/core/src/main/java/org/elasticsearch/script/ScriptModule.java
index fea143a3407..0fa4d9ea731 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptModule.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptModule.java
@@ -42,7 +42,7 @@ public class ScriptModule extends AbstractModule {
 
     {
         scriptEngineRegistrations.add(new ScriptEngineRegistry.ScriptEngineRegistration(NativeScriptEngineService.class,
-                        NativeScriptEngineService.TYPES, ScriptMode.ON));
+                        NativeScriptEngineService.NAME, ScriptMode.ON));
     }
 
     private final Map<String, Class<? extends NativeScriptFactory>> scripts = new HashMap<>();
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java
index 299b198d9b1..557499d07f7 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java
@@ -168,9 +168,8 @@ public class ScriptService extends AbstractComponent implements Closeable {
         Map<String, ScriptEngineService> enginesByLangBuilder = new HashMap<>();
         Map<String, ScriptEngineService> enginesByExtBuilder = new HashMap<>();
         for (ScriptEngineService scriptEngine : scriptEngines) {
-            for (String language : scriptEngineRegistry.getLanguages(scriptEngine.getClass())) {
-                enginesByLangBuilder.put(language, scriptEngine);
-            }
+            String language = scriptEngineRegistry.getLanguage(scriptEngine.getClass());
+            enginesByLangBuilder.put(language, scriptEngine);
             for (String ext : scriptEngine.getExtensions()) {
                 enginesByExtBuilder.put(ext, scriptEngine);
             }
@@ -558,12 +557,12 @@ public class ScriptService extends AbstractComponent implements Closeable {
                 try {
                     //we don't know yet what the script will be used for, but if all of the operations for this lang
                     // with file scripts are disabled, it makes no sense to even compile it and cache it.
-                    if (isAnyScriptContextEnabled(engineService.getTypes().get(0), engineService, ScriptType.FILE)) {
+                    if (isAnyScriptContextEnabled(engineService.getType(), engineService, ScriptType.FILE)) {
                         logger.info("compiling script file [{}]", file.toAbsolutePath());
                         try (InputStreamReader reader = new InputStreamReader(Files.newInputStream(file), StandardCharsets.UTF_8)) {
                             String script = Streams.copyToString(reader);
                             CacheKey cacheKey = new CacheKey(engineService, scriptNameExt.v1(), null, Collections.emptyMap());
-                            staticCache.put(cacheKey, new CompiledScript(ScriptType.FILE, scriptNameExt.v1(), engineService.getTypes().get(0), engineService.compile(script, Collections.emptyMap())));
+                            staticCache.put(cacheKey, new CompiledScript(ScriptType.FILE, scriptNameExt.v1(), engineService.getType(), engineService.compile(script, Collections.emptyMap())));
                             scriptMetrics.onCompilation();
                         }
                     } else {
@@ -667,7 +666,7 @@ public class ScriptService extends AbstractComponent implements Closeable {
         final Map<String, String> params;
 
         private CacheKey(final ScriptEngineService service, final String name, final String code, final Map<String, String> params) {
-            this.lang = service.getTypes().get(0);
+            this.lang = service.getType();
             this.name = name;
             this.code = code;
             this.params = params;
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java b/core/src/main/java/org/elasticsearch/script/ScriptSettings.java
index f8bcd1f542f..433912e47e5 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptSettings.java
@@ -83,79 +83,65 @@ public class ScriptSettings {
         return scriptContextSettingMap;
     }
 
-    private static List<Setting<ScriptMode>> languageSettings(
-        Map<ScriptService.ScriptType, Setting<ScriptMode>> scriptTypeSettingMap,
-        Map<ScriptContext, Setting<ScriptMode>> scriptContextSettingMap,
-        ScriptEngineRegistry scriptEngineRegistry,
-        ScriptContextRegistry scriptContextRegistry) {
-        List<Setting<ScriptMode>> scriptModeSettings = new ArrayList<>();
-        for (Class<? extends ScriptEngineService> scriptEngineService : scriptEngineRegistry.getRegisteredScriptEngineServices()) {
-            List<String> languages = scriptEngineRegistry.getLanguages(scriptEngineService);
+    private static List<Setting<ScriptMode>> languageSettings(Map<ScriptService.ScriptType, Setting<ScriptMode>> scriptTypeSettingMap,
+                                                              Map<ScriptContext, Setting<ScriptMode>> scriptContextSettingMap,
+                                                              ScriptEngineRegistry scriptEngineRegistry,
+                                                              ScriptContextRegistry scriptContextRegistry) {
+        final List<Setting<ScriptMode>> scriptModeSettings = new ArrayList<>();
 
-            for (String language : languages) {
-                if (NativeScriptEngineService.TYPES.contains(language)) {
-                    // native scripts are always enabled, and their settings can not be changed
-                    continue;
+        for (final Class<? extends ScriptEngineService> scriptEngineService : scriptEngineRegistry.getRegisteredScriptEngineServices()) {
+            if (scriptEngineService == NativeScriptEngineService.class) {
+                // native scripts are always enabled, and their settings can not be changed
+                continue;
+            }
+            final String language = scriptEngineRegistry.getLanguage(scriptEngineService);
+            for (final ScriptService.ScriptType scriptType : ScriptService.ScriptType.values()) {
+                // Top level, like "script.engine.groovy.inline"
+                final ScriptMode defaultNonFileScriptMode = scriptEngineRegistry.getDefaultInlineScriptModes().get(language);
+                ScriptMode defaultLangAndType = defaultNonFileScriptMode;
+                // Files are treated differently because they are never default-deny
+                if (ScriptService.ScriptType.FILE == scriptType) {
+                    defaultLangAndType = ScriptService.ScriptType.FILE.getDefaultScriptMode();
                 }
-                ScriptMode defaultNonFileScriptMode = scriptEngineRegistry.getDefaultInlineScriptModes().get(language);
-                for (final ScriptService.ScriptType scriptType : ScriptService.ScriptType.values()) {
-                    // This will be easier once ScriptMode is transitioned to booleans
-                    if (scriptType.getDefaultScriptMode() == ScriptMode.ON) {
-                        defaultNonFileScriptMode = ScriptMode.ON;
-                    }
-                    final ScriptMode defaultScriptMode = defaultNonFileScriptMode;
-                    // Like "script.engine.groovy.inline"
-                    final Setting<ScriptMode> langGlobalSetting = new Setting<>(ScriptModes.getGlobalKey(language, scriptType),
-                            defaultScriptMode.toString(), ScriptMode::parse, Property.NodeScope);
-                    scriptModeSettings.add(langGlobalSetting);
+                final ScriptMode defaultIfNothingSet = defaultLangAndType;
 
-                    for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) {
-                        Function<Settings, String> defaultSetting = settings -> {
-                            // fallback logic for script mode settings
+                // Setting for something like "script.engine.groovy.inline"
+                final Setting<ScriptMode> langAndTypeSetting = new Setting<>(ScriptModes.getGlobalKey(language, scriptType),
+                        defaultLangAndType.toString(), ScriptMode::parse, Property.NodeScope);
+                scriptModeSettings.add(langAndTypeSetting);
 
-                            // the first fallback is other types registered by the same script engine service
-                            // e.g., "py.inline.aggs" is in the settings but a script with lang "python" is executed
-                            Map<String, List<String>> languageSettings =
-                                languages
-                                    .stream()
-                                    .map(lang -> Tuple.tuple(lang, settings.get(ScriptModes.getKey(lang, scriptType, scriptContext))))
-                                    .filter(tuple -> tuple.v2() != null)
-                                    .collect(Collectors.groupingBy(Tuple::v2, Collectors.mapping(Tuple::v1, Collectors.toList())));
-                            if (!languageSettings.isEmpty()) {
-                                if (languageSettings.size() > 1) {
-                                    throw new IllegalArgumentException("conflicting settings [" + languageSettings.toString() + "] for language [" + language + "]");
-                                }
-                                return languageSettings.keySet().iterator().next();
-                            }
+                for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) {
+                    final String langAndTypeAndContextName = ScriptModes.getKey(language, scriptType, scriptContext);
+                    // A function that, given a setting, will return what the default should be. Since the fine-grained script settings
+                    // read from a bunch of different places this is implemented in this way.
+                    Function<Settings, String> defaultSettingFn = settings -> {
+                        final Setting<ScriptMode> globalOpSetting = scriptContextSettingMap.get(scriptContext);
+                        final Setting<ScriptMode> globalTypeSetting = scriptTypeSettingMap.get(scriptType);
+                        final Setting<ScriptMode> langAndTypeAndContextSetting = new Setting<>(langAndTypeAndContextName,
+                                defaultIfNothingSet.toString(), ScriptMode::parse, Property.NodeScope);
 
-                            // the next fallback is settings configured for this engine and script type
-                            if (langGlobalSetting.exists(settings)) {
-                                return langGlobalSetting.get(settings).getMode();
-                            }
-
-                            // the next fallback is global operation-based settings (e.g., "script.aggs: false")
-                            Setting<ScriptMode> setting = scriptContextSettingMap.get(scriptContext);
-                            if (setting.exists(settings)) {
-                                return setting.get(settings).getMode();
-                            }
-
-                            // the next fallback is global source-based settings (e.g., "script.inline: false")
-                            Setting<ScriptMode> scriptTypeSetting = scriptTypeSettingMap.get(scriptType);
-                            if (scriptTypeSetting.exists(settings)) {
-                                return scriptTypeSetting.get(settings).getMode();
-                            }
-
-                            // the final fallback is the default for the type
-                            return defaultScriptMode.toString();
-                        };
-                        Setting<ScriptMode> setting =
-                            new Setting<>(
-                                ScriptModes.getKey(language, scriptType, scriptContext),
-                                defaultSetting,
-                                ScriptMode::parse,
-                                Property.NodeScope);
-                        scriptModeSettings.add(setting);
-                    }
+                        // fallback logic for script mode settings
+                        if (langAndTypeAndContextSetting.exists(settings)) {
+                            // like: "script.engine.groovy.inline.aggs: true"
+                            return langAndTypeAndContextSetting.get(settings).getMode();
+                        } else if (langAndTypeSetting.exists(settings)) {
+                            // like: "script.engine.groovy.inline: true"
+                            return langAndTypeSetting.get(settings).getMode();
+                        } else if (globalOpSetting.exists(settings)) {
+                            // like: "script.aggs: true"
+                            return globalOpSetting.get(settings).getMode();
+                        } else if (globalTypeSetting.exists(settings)) {
+                            // like: "script.inline: true"
+                            return globalTypeSetting.get(settings).getMode();
+                        } else {
+                            // Nothing is set!
+                            return defaultIfNothingSet.getMode();
+                        }
+                    };
+                    // The actual setting for finest grained script settings
+                    Setting<ScriptMode> setting = new Setting<>(langAndTypeAndContextName, defaultSettingFn,
+                            ScriptMode::parse, Property.NodeScope);
+                    scriptModeSettings.add(setting);
                 }
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
index 30552201dfd..13175ac48f8 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
@@ -87,7 +87,7 @@ public class BulkRequestTests extends ESTestCase {
         Script script = ((UpdateRequest) bulkRequest.requests().get(1)).script();
         assertThat(script, notNullValue());
         assertThat(script.getScript(), equalTo("counter += param1"));
-        assertThat(script.getLang(), equalTo("js"));
+        assertThat(script.getLang(), equalTo("javascript"));
         Map<String, Object> scriptParams = script.getParams();
         assertThat(scriptParams, notNullValue());
         assertThat(scriptParams.size(), equalTo(1));
diff --git a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
index 9aef2496663..9c47d701f57 100644
--- a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
@@ -241,7 +241,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
                 ScriptEngineRegistry scriptEngineRegistry =
                         new ScriptEngineRegistry(Collections
                                 .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
-                                                                                                 MockScriptEngine.TYPES, ScriptMode.ON)));
+                                                                                                 MockScriptEngine.NAME, ScriptMode.ON)));
                 bind(ScriptEngineRegistry.class).toInstance(scriptEngineRegistry);
                 ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts);
                 bind(ScriptContextRegistry.class).toInstance(scriptContextRegistry);
diff --git a/core/src/test/java/org/elasticsearch/script/FileScriptTests.java b/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
index e591a0f7128..de734f86013 100644
--- a/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
@@ -20,6 +20,7 @@ package org.elasticsearch.script;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.test.ESTestCase;
 
 import java.nio.file.Files;
@@ -44,7 +45,7 @@ public class FileScriptTests extends ESTestCase {
             .put(settings)
             .build();
         Set<ScriptEngineService> engines = new HashSet<>(Collections.singletonList(new MockScriptEngine()));
-        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class, MockScriptEngine.TYPES)));
+        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class, MockScriptEngine.NAME, ScriptMode.ON)));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
         return new ScriptService(settings, new Environment(settings), engines, null, scriptEngineRegistry, scriptContextRegistry, scriptSettings);
diff --git a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
index b57fc260b73..045f62ff8fe 100644
--- a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
@@ -28,6 +28,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.EnvironmentModule;
+import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -85,7 +86,7 @@ public class NativeScriptTests extends ESTestCase {
         Map<String, NativeScriptFactory> nativeScriptFactoryMap = new HashMap<>();
         nativeScriptFactoryMap.put("my", new MyNativeScriptFactory());
         Set<ScriptEngineService> scriptEngineServices = singleton(new NativeScriptEngineService(settings, nativeScriptFactoryMap));
-        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(NativeScriptEngineService.class, NativeScriptEngineService.TYPES)));
+        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(NativeScriptEngineService.class, NativeScriptEngineService.NAME, ScriptMode.ON)));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(new ArrayList<>());
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
         ScriptService scriptService = new ScriptService(settings, environment, scriptEngineServices, resourceWatcherService, scriptEngineRegistry, scriptContextRegistry, scriptSettings);
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
index bf41f0b4213..8405366f34d 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
@@ -45,10 +45,9 @@ public class ScriptContextTests extends ESTestCase {
             .put("script.engine." + MockScriptEngine.NAME + ".inline." + PLUGIN_NAME + "_custom_exp_disabled_op", "false")
             .build();
         Set<ScriptEngineService> engines = new HashSet<>(Collections.singletonList(new MockScriptEngine()));
-        ScriptEngineRegistry scriptEngineRegistry =
-                new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
-                                                                                                                     MockScriptEngine.TYPES,
-                                                                                                                     ScriptMode.ON)));
+        ScriptEngineRegistry.ScriptEngineRegistration registration =
+                new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class, MockScriptEngine.NAME, ScriptMode.ON);
+        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(registration));
         List<ScriptContext.Plugin> customContexts = Arrays.asList(
             new ScriptContext.Plugin(PLUGIN_NAME, "custom_op"),
             new ScriptContext.Plugin(PLUGIN_NAME, "custom_exp_disabled_op"),
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java b/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
index 134a45557e1..88a32cc5ddb 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
@@ -44,12 +44,6 @@ import static org.hamcrest.Matchers.containsString;
 
 // TODO: this needs to be a base test class, and all scripting engines extend it
 public class ScriptModesTests extends ESTestCase {
-    private static final Set<String> ALL_LANGS = unmodifiableSet(
-            newHashSet("custom", "test"));
-
-    static final String[] ENABLE_VALUES = new String[]{"true"};
-    static final String[] DISABLE_VALUES = new String[]{"false"};
-
     ScriptSettings scriptSettings;
     ScriptContextRegistry scriptContextRegistry;
     private ScriptContext[] scriptContexts;
@@ -78,8 +72,8 @@ public class ScriptModesTests extends ESTestCase {
                 new NativeScriptEngineService(Settings.EMPTY, Collections.<String, NativeScriptFactory>emptyMap()),
                 new CustomScriptEngineService()));
         ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Arrays.asList(
-            new ScriptEngineRegistry.ScriptEngineRegistration(NativeScriptEngineService.class, NativeScriptEngineService.TYPES),
-            new ScriptEngineRegistry.ScriptEngineRegistration(CustomScriptEngineService.class, CustomScriptEngineService.TYPES)));
+            new ScriptEngineRegistry.ScriptEngineRegistration(NativeScriptEngineService.class, NativeScriptEngineService.NAME),
+            new ScriptEngineRegistry.ScriptEngineRegistration(CustomScriptEngineService.class, CustomScriptEngineService.NAME)));
         scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
         checkedSettings = new HashSet<>();
         assertAllSettingsWereChecked = true;
@@ -97,9 +91,8 @@ public class ScriptModesTests extends ESTestCase {
     public void assertAllSettingsWereChecked() {
         if (assertScriptModesNonNull) {
             assertThat(scriptModes, notNullValue());
-            //2 is the number of engines (native excluded), custom is counted twice though as it's associated with two different names
-            int numberOfSettings = 2 * ScriptType.values().length * scriptContextRegistry.scriptContexts().size();
-            numberOfSettings += 6; // for top-level inline/store/file settings
+            int numberOfSettings = ScriptType.values().length * scriptContextRegistry.scriptContexts().size();
+            numberOfSettings += 3; // for top-level inline/store/file settings
             assertThat(scriptModes.scriptModes.size(), equalTo(numberOfSettings));
             if (assertAllSettingsWereChecked) {
                 assertThat(checkedSettings.size(), equalTo(numberOfSettings));
@@ -109,8 +102,8 @@ public class ScriptModesTests extends ESTestCase {
 
     public void testDefaultSettings() {
         this.scriptModes = new ScriptModes(scriptSettings, Settings.EMPTY);
-        assertScriptModesAllOps(ScriptMode.ON, ALL_LANGS, ScriptType.FILE);
-        assertScriptModesAllOps(ScriptMode.OFF, ALL_LANGS, ScriptType.STORED, ScriptType.INLINE);
+        assertScriptModesAllOps(ScriptMode.ON, ScriptType.FILE);
+        assertScriptModesAllOps(ScriptMode.OFF, ScriptType.STORED, ScriptType.INLINE);
     }
 
     public void testMissingSetting() {
@@ -143,16 +136,16 @@ public class ScriptModesTests extends ESTestCase {
         this.scriptModes = new ScriptModes(scriptSettings, builder.build());
 
         for (int i = 0; i < randomInt; i++) {
-            assertScriptModesAllOps(randomScriptModes[i], ALL_LANGS, randomScriptTypes[i]);
+            assertScriptModesAllOps(randomScriptModes[i], randomScriptTypes[i]);
         }
         if (randomScriptTypesSet.contains(ScriptType.FILE) == false) {
-            assertScriptModesAllOps(ScriptMode.ON, ALL_LANGS, ScriptType.FILE);
+            assertScriptModesAllOps(ScriptMode.ON, ScriptType.FILE);
         }
         if (randomScriptTypesSet.contains(ScriptType.STORED) == false) {
-            assertScriptModesAllOps(ScriptMode.OFF, ALL_LANGS, ScriptType.STORED);
+            assertScriptModesAllOps(ScriptMode.OFF, ScriptType.STORED);
         }
         if (randomScriptTypesSet.contains(ScriptType.INLINE) == false) {
-            assertScriptModesAllOps(ScriptMode.OFF, ALL_LANGS, ScriptType.INLINE);
+            assertScriptModesAllOps(ScriptMode.OFF, ScriptType.INLINE);
         }
     }
 
@@ -175,46 +168,45 @@ public class ScriptModesTests extends ESTestCase {
         this.scriptModes = new ScriptModes(scriptSettings, builder.build());
 
         for (int i = 0; i < randomInt; i++) {
-            assertScriptModesAllTypes(randomScriptModes[i], ALL_LANGS, randomScriptContexts[i]);
+            assertScriptModesAllTypes(randomScriptModes[i], randomScriptContexts[i]);
         }
 
         ScriptContext[] complementOf = complementOf(randomScriptContexts);
-        assertScriptModes(ScriptMode.ON, ALL_LANGS, new ScriptType[]{ScriptType.FILE}, complementOf);
-        assertScriptModes(ScriptMode.OFF, ALL_LANGS, new ScriptType[]{ScriptType.STORED, ScriptType.INLINE}, complementOf);
+        assertScriptModes(ScriptMode.ON, new ScriptType[]{ScriptType.FILE}, complementOf);
+        assertScriptModes(ScriptMode.OFF, new ScriptType[]{ScriptType.STORED, ScriptType.INLINE}, complementOf);
     }
 
     public void testConflictingScriptTypeAndOpGenericSettings() {
         ScriptContext scriptContext = randomFrom(scriptContexts);
-        Settings.Builder builder = Settings.builder().put("script" + "." + scriptContext.getKey(), randomFrom(DISABLE_VALUES))
-                .put("script.stored", randomFrom(ENABLE_VALUES)).put("script.inline", "true");
+        Settings.Builder builder = Settings.builder()
+                .put("script." + scriptContext.getKey(), "false")
+                .put("script.stored", "true")
+                .put("script.inline", "true");
         //operations generic settings have precedence over script type generic settings
         this.scriptModes = new ScriptModes(scriptSettings, builder.build());
-        assertScriptModesAllTypes(ScriptMode.OFF, ALL_LANGS, scriptContext);
+        assertScriptModesAllTypes(ScriptMode.OFF, scriptContext);
         ScriptContext[] complementOf = complementOf(scriptContext);
-        assertScriptModes(ScriptMode.ON, ALL_LANGS, new ScriptType[]{ScriptType.FILE, ScriptType.STORED}, complementOf);
-        assertScriptModes(ScriptMode.ON, ALL_LANGS, new ScriptType[]{ScriptType.INLINE}, complementOf);
+        assertScriptModes(ScriptMode.ON, new ScriptType[]{ScriptType.FILE, ScriptType.STORED}, complementOf);
+        assertScriptModes(ScriptMode.ON, new ScriptType[]{ScriptType.INLINE}, complementOf);
     }
 
-    private void assertScriptModesAllOps(ScriptMode expectedScriptMode, Set<String> langs, ScriptType... scriptTypes) {
-        assertScriptModes(expectedScriptMode, langs, scriptTypes, scriptContexts);
+    private void assertScriptModesAllOps(ScriptMode expectedScriptMode, ScriptType... scriptTypes) {
+        assertScriptModes(expectedScriptMode, scriptTypes, scriptContexts);
     }
 
-    private void assertScriptModesAllTypes(ScriptMode expectedScriptMode, Set<String> langs, ScriptContext... scriptContexts) {
-        assertScriptModes(expectedScriptMode, langs, ScriptType.values(), scriptContexts);
+    private void assertScriptModesAllTypes(ScriptMode expectedScriptMode, ScriptContext... scriptContexts) {
+        assertScriptModes(expectedScriptMode, ScriptType.values(), scriptContexts);
     }
 
-    private void assertScriptModes(ScriptMode expectedScriptMode, Set<String> langs, ScriptType[] scriptTypes, ScriptContext... scriptContexts) {
-        assert langs.size() > 0;
+    private void assertScriptModes(ScriptMode expectedScriptMode, ScriptType[] scriptTypes, ScriptContext... scriptContexts) {
         assert scriptTypes.length > 0;
         assert scriptContexts.length > 0;
-        for (String lang : langs) {
-            for (ScriptType scriptType : scriptTypes) {
-                checkedSettings.add("script.engine." + lang + "." + scriptType);
-                for (ScriptContext scriptContext : scriptContexts) {
-                    assertThat(lang + "." + scriptType + "." + scriptContext.getKey() + " doesn't have the expected value",
-                            scriptModes.getScriptMode(lang, scriptType, scriptContext), equalTo(expectedScriptMode));
-                    checkedSettings.add(lang + "." + scriptType + "." + scriptContext);
-                }
+        for (ScriptType scriptType : scriptTypes) {
+            checkedSettings.add("script.engine.custom." + scriptType);
+            for (ScriptContext scriptContext : scriptContexts) {
+                assertThat("custom." + scriptType + "." + scriptContext.getKey() + " doesn't have the expected value",
+                        scriptModes.getScriptMode("custom", scriptType, scriptContext), equalTo(expectedScriptMode));
+                checkedSettings.add("custom." + scriptType + "." + scriptContext);
             }
         }
     }
@@ -233,25 +225,24 @@ public class ScriptModesTests extends ESTestCase {
     static Map<String, ScriptEngineService> buildScriptEnginesByLangMap(Set<ScriptEngineService> scriptEngines) {
         Map<String, ScriptEngineService> builder = new HashMap<>();
         for (ScriptEngineService scriptEngine : scriptEngines) {
-            for (String type : scriptEngine.getTypes()) {
-                builder.put(type, scriptEngine);
-            }
+            String type = scriptEngine.getType();
+            builder.put(type, scriptEngine);
         }
         return unmodifiableMap(builder);
     }
 
     private static class CustomScriptEngineService implements ScriptEngineService {
 
-        public static final List<String> TYPES = Collections.unmodifiableList(Arrays.asList("custom", "test"));
+        public static final String NAME = "custom";
 
         @Override
-        public List<String> getTypes() {
-            return TYPES;
+        public String getType() {
+            return NAME;
         }
 
         @Override
         public List<String> getExtensions() {
-            return Collections.singletonList(TYPES.get(0));
+            return Collections.unmodifiableList(Arrays.asList("custom", "test"));
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
index acbcc9ab615..96a406fb4a7 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
@@ -110,8 +110,8 @@ public class ScriptServiceTests extends ESTestCase {
             contexts.put(context, new ScriptContext.Plugin(plugin, operation));
         }
         List<ScriptEngineRegistry.ScriptEngineRegistration> registries = new ArrayList<>(2);
-        registries.add(new ScriptEngineRegistry.ScriptEngineRegistration(TestEngineService.class, TestEngineService.TYPES, ScriptMode.ON));
-        registries.add(new ScriptEngineRegistry.ScriptEngineRegistration(TestDangerousEngineService.class, TestDangerousEngineService.TYPES));
+        registries.add(new ScriptEngineRegistry.ScriptEngineRegistration(TestEngineService.class, TestEngineService.NAME, ScriptMode.ON));
+        registries.add(new ScriptEngineRegistry.ScriptEngineRegistration(TestDangerousEngineService.class, TestDangerousEngineService.NAME));
         scriptEngineRegistry = new ScriptEngineRegistry(registries);
         scriptContextRegistry = new ScriptContextRegistry(contexts.values());
         scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
@@ -195,30 +195,11 @@ public class ScriptServiceTests extends ESTestCase {
         assertThat(compiledScript1.compiled(), sameInstance(compiledScript2.compiled()));
     }
 
-    public void testInlineScriptCompiledOnceMultipleLangAcronyms() throws IOException {
-        buildScriptService(Settings.EMPTY);
-        CompiledScript compiledScript1 = scriptService.compile(new Script("script", ScriptType.INLINE, "test", null),
-                randomFrom(scriptContexts), Collections.emptyMap(), emptyClusterState());
-        CompiledScript compiledScript2 = scriptService.compile(new Script("script", ScriptType.INLINE, "test2", null),
-                randomFrom(scriptContexts), Collections.emptyMap(), emptyClusterState());
-        assertThat(compiledScript1.compiled(), sameInstance(compiledScript2.compiled()));
-    }
-
-    public void testFileScriptCompiledOnceMultipleLangAcronyms() throws IOException {
-        buildScriptService(Settings.EMPTY);
-        createFileScripts("test");
-        CompiledScript compiledScript1 = scriptService.compile(new Script("file_script", ScriptType.FILE, "test", null),
-                randomFrom(scriptContexts), Collections.emptyMap(), emptyClusterState());
-        CompiledScript compiledScript2 = scriptService.compile(new Script("file_script", ScriptType.FILE, "test2", null),
-                randomFrom(scriptContexts), Collections.emptyMap(), emptyClusterState());
-        assertThat(compiledScript1.compiled(), sameInstance(compiledScript2.compiled()));
-    }
-
     public void testDefaultBehaviourFineGrainedSettings() throws IOException {
         Settings.Builder builder = Settings.builder();
         //rarely inject the default settings, which have no effect
         if (rarely()) {
-            builder.put("script.file", randomFrom(ScriptModesTests.ENABLE_VALUES));
+            builder.put("script.file", "true");
         }
         buildScriptService(builder.build());
         createFileScripts("groovy", "mustache", "dtest");
@@ -258,7 +239,7 @@ public class ScriptServiceTests extends ESTestCase {
             do {
                 ScriptType scriptType = randomFrom(ScriptType.values());
                 ScriptContext scriptContext = randomFrom(this.scriptContexts);
-                settingKey = scriptEngineService.getTypes().get(0) + "." + scriptType + "." + scriptContext.getKey();
+                settingKey = scriptEngineService.getType() + "." + scriptType + "." + scriptContext.getKey();
             } while (engineSettings.containsKey(settingKey));
             engineSettings.put(settingKey, randomFrom(ScriptMode.values()));
         }
@@ -289,7 +270,7 @@ public class ScriptServiceTests extends ESTestCase {
             String part1 = entry.getKey().substring(0, delimiter);
             String part2 = entry.getKey().substring(delimiter + 1);
 
-            String lang = randomFrom(scriptEnginesByLangMap.get(part1).getTypes());
+            String lang = randomFrom(scriptEnginesByLangMap.get(part1).getType());
             switch (entry.getValue()) {
                 case ON:
                     builder.put("script.engine" + "." + lang + "." + part2, "true");
@@ -309,7 +290,7 @@ public class ScriptServiceTests extends ESTestCase {
             String script = scriptType == ScriptType.FILE ? "file_script" : "script";
             for (ScriptContext scriptContext : this.scriptContexts) {
                 //fallback mechanism: 1) engine specific settings 2) op based settings 3) source based settings
-                ScriptMode scriptMode = engineSettings.get(dangerousScriptEngineService.getTypes().get(0) + "." + scriptType + "." + scriptContext.getKey());
+                ScriptMode scriptMode = engineSettings.get(dangerousScriptEngineService.getType() + "." + scriptType + "." + scriptContext.getKey());
                 if (scriptMode == null) {
                     scriptMode = scriptContextSettings.get(scriptContext);
                 }
@@ -320,15 +301,14 @@ public class ScriptServiceTests extends ESTestCase {
                     scriptMode = DEFAULT_SCRIPT_MODES.get(scriptType);
                 }
 
-                for (String lang : dangerousScriptEngineService.getTypes()) {
-                    switch (scriptMode) {
-                        case ON:
-                            assertCompileAccepted(lang, script, scriptType, scriptContext);
-                            break;
-                        case OFF:
-                            assertCompileRejected(lang, script, scriptType, scriptContext);
-                            break;
-                    }
+                String lang = dangerousScriptEngineService.getType();
+                switch (scriptMode) {
+                    case ON:
+                        assertCompileAccepted(lang, script, scriptType, scriptContext);
+                        break;
+                    case OFF:
+                        assertCompileRejected(lang, script, scriptType, scriptContext);
+                        break;
                 }
             }
         }
@@ -343,14 +323,13 @@ public class ScriptServiceTests extends ESTestCase {
             unknownContext = randomAsciiOfLength(randomIntBetween(1, 30));
         } while(scriptContextRegistry.isSupportedContext(new ScriptContext.Plugin(pluginName, unknownContext)));
 
-        for (String type : scriptEngineService.getTypes()) {
-            try {
-                scriptService.compile(new Script("test", randomFrom(ScriptType.values()), type, null), new ScriptContext.Plugin(
-                        pluginName, unknownContext), Collections.emptyMap(), emptyClusterState());
-                fail("script compilation should have been rejected");
-            } catch(IllegalArgumentException e) {
-                assertThat(e.getMessage(), containsString("script context [" + pluginName + "_" + unknownContext + "] not supported"));
-            }
+        String type = scriptEngineService.getType();
+        try {
+            scriptService.compile(new Script("test", randomFrom(ScriptType.values()), type, null), new ScriptContext.Plugin(
+                            pluginName, unknownContext), Collections.emptyMap(), emptyClusterState());
+            fail("script compilation should have been rejected");
+        } catch(IllegalArgumentException e) {
+            assertThat(e.getMessage(), containsString("script context [" + pluginName + "_" + unknownContext + "] not supported"));
         }
     }
 
@@ -529,13 +508,13 @@ public class ScriptServiceTests extends ESTestCase {
 
     public static class TestEngineService implements ScriptEngineService {
 
-        public static final List<String> TYPES = Collections.unmodifiableList(Arrays.asList("test", "test2"));
+        public static final String NAME = "test";
 
         public static final List<String> EXTENSIONS = Collections.unmodifiableList(Arrays.asList("test", "tst"));
 
         @Override
-        public List<String> getTypes() {
-            return TYPES;
+        public String getType() {
+            return NAME;
         }
 
         @Override
@@ -571,13 +550,13 @@ public class ScriptServiceTests extends ESTestCase {
 
     public static class TestDangerousEngineService implements ScriptEngineService {
 
-        public static final List<String> TYPES = Collections.unmodifiableList(Arrays.asList("dtest"));
+        public static final String NAME = "dtest";
 
         public static final List<String> EXTENSIONS = Collections.unmodifiableList(Arrays.asList("dtest"));
 
         @Override
-        public List<String> getTypes() {
-            return TYPES;
+        public String getType() {
+            return NAME;
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java b/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
index 952e2598aa1..ef432120039 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
@@ -22,6 +22,7 @@ package org.elasticsearch.script;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.test.ESTestCase;
 
@@ -36,37 +37,9 @@ import static org.hamcrest.Matchers.equalTo;
 
 public class ScriptSettingsTests extends ESTestCase {
 
-    public void testConflictingModesForLanguage() {
-        ScriptEngineRegistry scriptEngineRegistry =
-            new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(CustomScriptEngineService.class, CustomScriptEngineService.TYPES)));
-        ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
-        ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
-        ScriptService.ScriptType scriptType = randomFrom(ScriptService.ScriptType.values());
-        ScriptContext scriptContext = randomFrom(ScriptContext.Standard.values());
-        Settings settings =
-            Settings
-                .builder()
-                .put(ScriptModes.getKey("test1", scriptType, scriptContext), "off")
-                .put(ScriptModes.getKey("test2", scriptType, scriptContext), "on")
-                .build();
-        boolean sawConflictingSettings = false;
-        for (Setting<ScriptMode> scriptModeSetting : scriptSettings.getScriptLanguageSettings()) {
-            if (scriptModeSetting.getKey().startsWith(ScriptModes.getKey("test3", scriptType, scriptContext))) {
-                try {
-                    scriptModeSetting.get(settings);
-                    fail("should have seen conflicting settings");
-                } catch (IllegalArgumentException e) {
-                    assertThat(e.getMessage(), anyOf(containsString("conflicting settings [{off=[test1], on=[test2]}] for language [test3]"), containsString("conflicting settings [{on=[test2], on=[test1]}] for language [test3]")));
-                    sawConflictingSettings = true;
-                }
-            }
-        }
-        assertTrue(sawConflictingSettings);
-    }
-
     public void testDefaultLanguageIsGroovy() {
         ScriptEngineRegistry scriptEngineRegistry =
-            new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(CustomScriptEngineService.class, CustomScriptEngineService.TYPES)));
+                new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(CustomScriptEngineService.class, CustomScriptEngineService.NAME, ScriptMode.ON)));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
         assertThat(scriptSettings.getDefaultScriptLanguageSetting().get(Settings.EMPTY), equalTo("groovy"));
@@ -74,17 +47,17 @@ public class ScriptSettingsTests extends ESTestCase {
 
     public void testCustomDefaultLanguage() {
         ScriptEngineRegistry scriptEngineRegistry =
-            new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(CustomScriptEngineService.class, CustomScriptEngineService.TYPES)));
+            new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(CustomScriptEngineService.class, CustomScriptEngineService.NAME, ScriptMode.ON)));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
-        String defaultLanguage = randomFrom(CustomScriptEngineService.TYPES);
+        String defaultLanguage = CustomScriptEngineService.NAME;
         Settings settings = Settings.builder().put("script.default_lang", defaultLanguage).build();
         assertThat(scriptSettings.getDefaultScriptLanguageSetting().get(settings), equalTo(defaultLanguage));
     }
 
     public void testInvalidDefaultLanguage() {
         ScriptEngineRegistry scriptEngineRegistry =
-            new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(CustomScriptEngineService.class, CustomScriptEngineService.TYPES)));
+            new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(CustomScriptEngineService.class, CustomScriptEngineService.NAME, ScriptMode.ON)));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
         Settings settings = Settings.builder().put("script.default_lang", "C++").build();
@@ -98,16 +71,18 @@ public class ScriptSettingsTests extends ESTestCase {
 
     private static class CustomScriptEngineService implements ScriptEngineService {
 
-        public static final List<String> TYPES = Collections.unmodifiableList(Arrays.asList("test1", "test2", "test3"));
+        public static final String NAME = "custom";
+
+        public static final List<String> TYPES = Collections.unmodifiableList(Arrays.asList("test"));
 
         @Override
-        public List<String> getTypes() {
-            return TYPES;
+        public String getType() {
+            return NAME;
         }
 
         @Override
         public List<String> getExtensions() {
-            return Collections.singletonList(TYPES.get(0));
+            return Collections.unmodifiableList(Arrays.asList("test"));
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
index a7b8ed064bc..2949dadcc49 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
@@ -51,6 +51,7 @@ import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptContextRegistry;
 import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
+import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptSettings;
@@ -130,8 +131,11 @@ public class AggregatorParsingTests extends ESTestCase {
                 Set<ScriptEngineService> engines = new HashSet<>();
                 engines.add(mockScriptEngine);
                 List<ScriptContext.Plugin> customContexts = new ArrayList<>();
-                ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections
-                        .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class, MockScriptEngine.TYPES)));
+                ScriptEngineRegistry scriptEngineRegistry =
+                        new ScriptEngineRegistry(Collections
+                                .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
+                                                                                                 MockScriptEngine.NAME,
+                                                                                                 ScriptMode.ON)));
                 bind(ScriptEngineRegistry.class).toInstance(scriptEngineRegistry);
                 ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts);
                 bind(ScriptContextRegistry.class).toInstance(scriptContextRegistry);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
index 4af87c982e1..7729f421f6c 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
@@ -55,6 +55,7 @@ import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptContextRegistry;
 import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
+import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptSettings;
@@ -143,8 +144,11 @@ public abstract class BaseAggregationTestCase<AB extends AggregatorBuilder<AB>>
                 Set<ScriptEngineService> engines = new HashSet<>();
                 engines.add(mockScriptEngine);
                 List<ScriptContext.Plugin> customContexts = new ArrayList<>();
-                ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections
-                        .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class, MockScriptEngine.TYPES)));
+                ScriptEngineRegistry scriptEngineRegistry =
+                        new ScriptEngineRegistry(Collections
+                                .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
+                                                                                                 MockScriptEngine.NAME,
+                                                                                                 ScriptMode.ON)));
                 bind(ScriptEngineRegistry.class).toInstance(scriptEngineRegistry);
                 ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts);
                 bind(ScriptContextRegistry.class).toInstance(scriptContextRegistry);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
index 7efef03d114..113b52cb0b6 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
@@ -148,7 +148,7 @@ public abstract class BasePipelineAggregationTestCase<AF extends PipelineAggrega
                 ScriptEngineRegistry scriptEngineRegistry =
                         new ScriptEngineRegistry(Collections
                                 .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
-                                                                                                 MockScriptEngine.TYPES, ScriptMode.ON)));
+                                                                                                 MockScriptEngine.NAME, ScriptMode.ON)));
                 bind(ScriptEngineRegistry.class).toInstance(scriptEngineRegistry);
                 ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts);
                 bind(ScriptContextRegistry.class).toInstance(scriptContextRegistry);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
index 248f75b6413..eeb0129c2a3 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
@@ -370,7 +370,7 @@ public class AvgIT extends AbstractNumericTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractFieldScriptEngine.class, ExtractFieldScriptEngine.TYPES, ScriptMode.ON));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractFieldScriptEngine.class, ExtractFieldScriptEngine.NAME, ScriptMode.ON));
         }
 
     }
@@ -389,8 +389,8 @@ public class AvgIT extends AbstractNumericTestCase {
         }
 
         @Override
-        public List<String> getTypes() {
-            return TYPES;
+        public String getType() {
+            return NAME;
         }
 
         @Override
@@ -497,7 +497,7 @@ public class AvgIT extends AbstractNumericTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.TYPES, ScriptMode.ON));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.NAME, ScriptMode.ON));
         }
 
     }
@@ -516,8 +516,8 @@ public class AvgIT extends AbstractNumericTestCase {
         }
 
         @Override
-        public List<String> getTypes() {
-            return TYPES;
+        public String getType() {
+            return NAME;
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
index 401a9baaa6d..50e0e40d4a0 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
@@ -365,7 +365,7 @@ public class SumIT extends AbstractNumericTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractFieldScriptEngine.class, ExtractFieldScriptEngine.TYPES, ScriptMode.ON));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractFieldScriptEngine.class, ExtractFieldScriptEngine.NAME, ScriptMode.ON));
         }
 
     }
@@ -385,8 +385,8 @@ public class SumIT extends AbstractNumericTestCase {
         }
 
         @Override
-        public List<String> getTypes() {
-            return TYPES;
+        public String getType() {
+            return NAME;
         }
 
         @Override
@@ -494,7 +494,7 @@ public class SumIT extends AbstractNumericTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.TYPES, ScriptMode.ON));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.NAME, ScriptMode.ON));
         }
 
     }
@@ -514,8 +514,8 @@ public class SumIT extends AbstractNumericTestCase {
         }
 
         @Override
-        public List<String> getTypes() {
-            return TYPES;
+        public String getType() {
+            return NAME;
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
index bc75481dc3d..483f542f584 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
@@ -224,7 +224,7 @@ public class ValueCountIT extends ESIntegTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.TYPES, ScriptMode.ON));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.NAME, ScriptMode.ON));
         }
 
     }
@@ -243,8 +243,8 @@ public class ValueCountIT extends ESIntegTestCase {
         }
 
         @Override
-        public List<String> getTypes() {
-            return TYPES;
+        public String getType() {
+            return NAME;
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index df000916009..9f0113829ce 100644
--- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -62,6 +62,7 @@ import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptContextRegistry;
 import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
+import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptSettings;
@@ -151,7 +152,9 @@ public class SearchSourceBuilderTests extends ESTestCase {
                 engines.add(mockScriptEngine);
                 List<ScriptContext.Plugin> customContexts = new ArrayList<>();
                 ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections
-                        .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class, MockScriptEngine.TYPES)));
+                        .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
+                                                                                         MockScriptEngine.NAME,
+                                                                                         ScriptMode.ON)));
                 bind(ScriptEngineRegistry.class).toInstance(scriptEngineRegistry);
                 ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts);
                 bind(ScriptContextRegistry.class).toInstance(scriptContextRegistry);
diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
index 3f2bd6e9a56..21b71508bbb 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
@@ -96,7 +96,7 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST
         Environment environment = new Environment(baseSettings);
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
         ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry
-                .ScriptEngineRegistration(TestEngineService.class, TestEngineService.TYPES)));
+                .ScriptEngineRegistration(TestEngineService.class, TestEngineService.NAME)));
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
         scriptService = new ScriptService(baseSettings, environment, Collections.singleton(new TestEngineService()),
                 new ResourceWatcherService(baseSettings, null), scriptEngineRegistry, scriptContextRegistry, scriptSettings) {
diff --git a/core/src/test/java/org/elasticsearch/update/UpdateIT.java b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
index 095b3de7678..313dc1554e3 100644
--- a/core/src/test/java/org/elasticsearch/update/UpdateIT.java
+++ b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
@@ -94,7 +94,7 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(PutFieldValuesScriptEngine.class, PutFieldValuesScriptEngine.TYPES, ScriptMode.ON));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(PutFieldValuesScriptEngine.class, PutFieldValuesScriptEngine.NAME, ScriptMode.ON));
         }
 
     }
@@ -110,8 +110,8 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         @Override
-        public List<String> getTypes() {
-            return TYPES;
+        public String getType() {
+            return NAME;
         }
 
         @Override
@@ -183,7 +183,7 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldIncrementScriptEngine.class, FieldIncrementScriptEngine.TYPES, ScriptMode.ON));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldIncrementScriptEngine.class, FieldIncrementScriptEngine.NAME, ScriptMode.ON));
         }
 
     }
@@ -199,8 +199,8 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         @Override
-        public List<String> getTypes() {
-            return TYPES;
+        public String getType() {
+            return NAME;
         }
 
         @Override
@@ -265,7 +265,7 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ScriptedUpsertScriptEngine.class, ScriptedUpsertScriptEngine.TYPES, ScriptMode.ON));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ScriptedUpsertScriptEngine.class, ScriptedUpsertScriptEngine.NAME, ScriptMode.ON));
         }
 
     }
@@ -281,8 +281,8 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         @Override
-        public List<String> getTypes() {
-            return TYPES;
+        public String getType() {
+            return NAME;
         }
 
         @Override
@@ -347,7 +347,7 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractContextInSourceScriptEngine.class, ExtractContextInSourceScriptEngine.TYPES, ScriptMode.ON));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractContextInSourceScriptEngine.class, ExtractContextInSourceScriptEngine.NAME, ScriptMode.ON));
         }
 
     }
@@ -363,8 +363,8 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         @Override
-        public List<String> getTypes() {
-            return TYPES;
+        public String getType() {
+            return NAME;
         }
 
         @Override
diff --git a/core/src/test/resources/org/elasticsearch/action/bulk/simple-bulk4.json b/core/src/test/resources/org/elasticsearch/action/bulk/simple-bulk4.json
index 8b916b8feed..7fc41b1d9bb 100644
--- a/core/src/test/resources/org/elasticsearch/action/bulk/simple-bulk4.json
+++ b/core/src/test/resources/org/elasticsearch/action/bulk/simple-bulk4.json
@@ -1,7 +1,7 @@
 { "update" : {"_id" : "1", "_retry_on_conflict" : 2} }
 { "doc" : {"field" : "value"} }
 { "update" : { "_id" : "0", "_type" : "type1", "_index" : "index1" } }
-{ "script" : "counter += param1", "lang" : "js", "params" : {"param1" : 1}, "upsert" : {"counter" : 1}}
+{ "script" : "counter += param1", "lang" : "javascript", "params" : {"param1" : 1}, "upsert" : {"counter" : 1}}
 { "delete" : { "_id" : "2" } }
 { "create" : { "_id" : "3" } }
 { "field1" : "value3" }
diff --git a/docs/plugins/lang-javascript.asciidoc b/docs/plugins/lang-javascript.asciidoc
index 3877fb38d67..f5887d53e9c 100644
--- a/docs/plugins/lang-javascript.asciidoc
+++ b/docs/plugins/lang-javascript.asciidoc
@@ -36,7 +36,7 @@ The node must be stopped before removing the plugin.
 ==== Using JavaScript in Elasticsearch
 
 Once the plugin has been installed, JavaScript can be used at a scripting
-language by setting the `lang` parameter to `javascript` or `js`.
+language by setting the `lang` parameter to `javascript`.
 
 Scripting is available in many APIs, but we will use an example with the
 `function_score` for demonstration purposes:
diff --git a/docs/reference/docs/bulk.asciidoc b/docs/reference/docs/bulk.asciidoc
index 8e7045fac12..ee8f75c6cdd 100644
--- a/docs/reference/docs/bulk.asciidoc
+++ b/docs/reference/docs/bulk.asciidoc
@@ -192,7 +192,7 @@ the options. Curl example with update actions:
 { "update" : {"_id" : "1", "_type" : "type1", "_index" : "index1", "_retry_on_conflict" : 3} }
 { "doc" : {"field" : "value"} }
 { "update" : { "_id" : "0", "_type" : "type1", "_index" : "index1", "_retry_on_conflict" : 3} }
-{ "script" : { "inline": "ctx._source.counter += param1", "lang" : "js", "params" : {"param1" : 1}}, "upsert" : {"counter" : 1}}
+{ "script" : { "inline": "ctx._source.counter += param1", "lang" : "javascript", "params" : {"param1" : 1}}, "upsert" : {"counter" : 1}}
 { "update" : {"_id" : "2", "_type" : "type1", "_index" : "index1", "_retry_on_conflict" : 3} }
 { "doc" : {"field" : "value"}, "doc_as_upsert" : true }
 { "update" : {"_id" : "3", "_type" : "type1", "_index" : "index1", "fields" : ["_source"]} }
diff --git a/docs/reference/migration/migrate_5_0/scripting.asciidoc b/docs/reference/migration/migrate_5_0/scripting.asciidoc
index 12dc3e3d6fd..ca545cf432d 100644
--- a/docs/reference/migration/migrate_5_0/scripting.asciidoc
+++ b/docs/reference/migration/migrate_5_0/scripting.asciidoc
@@ -84,4 +84,12 @@ All the methods related to interacting with indexed scripts have been removed.
 The Java API methods for interacting with stored scripts have been added under `ClusterAdminClient` class.
 The sugar methods that used to exist on the indexed scripts API methods don't exist on the methods for
 stored scripts. The only way to provide scripts is by using `BytesReference` implementation, if a string needs to be
-provided the `BytesArray` class should be used.
\ No newline at end of file
+provided the `BytesArray` class should be used.
+
+==== Scripting engines now register only a single language
+
+Prior to 5.0.0, script engines could register multiple languages. The Javascript
+script engine in particular registered both `"lang": "js"` and `"lang":
+"javascript"`. Script engines can now only register a single language. All
+references to `"lang": "js"` should be changed to `"lang": "javascript"` for
+existing users of the lang-javascript plugin.
diff --git a/docs/reference/modules/scripting.asciidoc b/docs/reference/modules/scripting.asciidoc
index 7bc5c3aa91e..feb8113a6da 100644
--- a/docs/reference/modules/scripting.asciidoc
+++ b/docs/reference/modules/scripting.asciidoc
@@ -24,18 +24,23 @@ and give the most flexibility.
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Language
+    |Sandboxed
     |Required plugin
 
 |<<modules-scripting-painless, `painless`>>
+    |yes
     |built-in
 
 |<<modules-scripting-groovy, `groovy`>>
+    |<<modules-scripting-security, no>>
     |built-in
 
 |{plugins}/lang-javascript.html[`javascript`]
+    |<<modules-scripting-security, no>>
     |{plugins}/lang-javascript.html[`lang-javascript`]
 
 |{plugins}/lang-python.html[`python`]
+    |<<modules-scripting-security, no>>
     |{plugins}/lang-python.html[`lang-python`]
 
 |=======================================================================
@@ -49,23 +54,35 @@ certain tasks.
 [cols="<,<,<,<",options="header",]
 |=======================================================================
 |Language
+    |Sandboxed
     |Required plugin
     |Purpose
 
 |<<modules-scripting-expression, `expression`>>
+    |yes
     |built-in
     |fast custom ranking and sorting
 
 |<<search-template, `mustache`>>
+    |yes
     |built-in
     |templates
 
 |<<modules-scripting-native, `java`>>
+    |n/a
     |you write it!
     |expert API
 
 |=======================================================================
 
+[WARNING]
+.Scripts and security
+=================================================
+
+Languages that are sandboxed are designed with security in mind. However, non-
+sandboxed languages can be a security issue, please read
+<<modules-scripting-security, Scripting and security>> for more details.
+
 =================================================
 
 
diff --git a/docs/reference/modules/scripting/groovy.asciidoc b/docs/reference/modules/scripting/groovy.asciidoc
index 153acaf891a..0c7f4676d2c 100644
--- a/docs/reference/modules/scripting/groovy.asciidoc
+++ b/docs/reference/modules/scripting/groovy.asciidoc
@@ -1,9 +1,9 @@
 [[modules-scripting-groovy]]
 === Groovy Scripting Language
 
-Groovy is the default scripting language available in Elasticsearch. Although
-limited by the <<java-security-manager,Java Security Manager>>, only `file`
-scripts may be used by default.
+Groovy is the default scripting language available in Elasticsearch.  Although
+limited by the <<java-security-manager,Java Security Manager>>, it is not a
+sandboxed language and only `file` scripts may be used by default.
 
 Enabling `inline` or `stored` Groovy scripting is a security risk and should
 only be considered if your Elasticsearch cluster is protected from the outside
diff --git a/docs/reference/modules/scripting/security.asciidoc b/docs/reference/modules/scripting/security.asciidoc
index 94eb41ae152..89065d86b5f 100644
--- a/docs/reference/modules/scripting/security.asciidoc
+++ b/docs/reference/modules/scripting/security.asciidoc
@@ -64,9 +64,10 @@ script.file:   true
 -----------------------------------
 
 NOTE: Global scripting settings affect the `mustache` scripting language.
-<<search-template,Search templates>> internally use the `mustache` language, and
-will be disabled by default, but they can be enabled/disabled according to
-fine-grained settings specified in `elasticsearch.yml`.
+<<search-template,Search templates>> internally use the `mustache` language,
+and will still be enabled by default as the `mustache` engine is sandboxed,
+but they will be enabled/disabled according to fine-grained settings
+specified in `elasticsearch.yml`.
 
 [[security-script-context]]
 [float]
@@ -86,9 +87,9 @@ of using the generic `plugin` category. Those operations can be referred to
 in the following form: `${pluginName}_${operation}`.
 
 The following example disables scripting for `update` and `plugin` operations,
-regardless of the script source or language. Scripts can be executed from
-languages as part of `aggregations`, `search` and plugins execution though,
-assuming they are enabled in the scripting settings.
+regardless of the script source or language. Scripts can still be executed
+from sandboxed languages as part of `aggregations`, `search` and plugins
+execution though, as the above defaults still get applied.
 
 [source,yaml]
 -----------------------------------
diff --git a/docs/reference/modules/scripting/using.asciidoc b/docs/reference/modules/scripting/using.asciidoc
index 955bfb49b3f..f0ede243b78 100644
--- a/docs/reference/modules/scripting/using.asciidoc
+++ b/docs/reference/modules/scripting/using.asciidoc
@@ -110,15 +110,15 @@ second version is only compiled once.
 [[modules-scripting-file-scripts]]
 === File-based Scripts
 
-To increase security, scripts for languages that are not deemed to be safe by
-default can only be specified in files stored on every node in the cluster. File
-scripts must be saved in the `scripts` directory whose default location depends
-on whether you use the <<zip-targz-layout,`zip`/`tar.gz`>>
-(`$ES_HOME/config/scripts/`), <<rpm-layout,RPM>>, or <<deb-layout,Debian>>
-package. The default may be changed with the `path.script` setting.
+To increase security, non-sandboxed languages can only be specified in script
+files stored on every node in the cluster.  File scripts must be saved in the
+`scripts` directory whose default location depends on whether you use  the
+<<zip-targz-layout,`zip`/`tar.gz`>> (`$ES_HOME/config/scripts/`),
+<<rpm-layout,RPM>>, or <<deb-layout,Debian>> package.  The default may be
+changed with the `path.script` setting.
 
-The languages which are assumed to be safe by default are: painless,
-expressions, and mustache (used for query templates).
+The languages which are assumed to be safe by default are: `painless`,
+`expression`, and `mustache` (used for search and query templates).
 
 Any files placed in the `scripts` directory will be compiled automatically
 when the node starts up and then <<reload-scripts,every 60 seconds thereafter>>.
diff --git a/docs/reference/search/search-template.asciidoc b/docs/reference/search/search-template.asciidoc
index 291be9379ba..25c4c0423f1 100644
--- a/docs/reference/search/search-template.asciidoc
+++ b/docs/reference/search/search-template.asciidoc
@@ -25,10 +25,10 @@ For more information on how Mustache templating and what kind of templating you
 can do with it check out the http://mustache.github.io/mustache.5.html[online
 documentation of the mustache project].
 
-NOTE: The mustache language is implemented in elasticsearch as a scripting
-language, hence it obeys settings that may be used to enable or disable scripts
-per language, source and operation as described in <<enable-dynamic-scripting,
-scripting docs>>
+NOTE: The mustache language is implemented in elasticsearch as a sandboxed
+scripting language, hence it obeys settings that may be used to enable or
+disable scripts per language, source and operation as described in
+<<enable-dynamic-scripting, scripting docs>>
 
 [float]
 ==== More template examples
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java
index 01e9aa18e91..1cb21887944 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java
@@ -38,6 +38,6 @@ public class ExpressionPlugin extends Plugin {
 
     public void onModule(ScriptModule module) {
         module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExpressionScriptEngineService.class,
-                        ExpressionScriptEngineService.TYPES, ScriptMode.ON));
+                        ExpressionScriptEngineService.NAME, ScriptMode.ON));
     }
 }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
index 168bf25c646..2979001f82d 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
@@ -70,8 +70,8 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
     }
 
     @Override
-    public List<String> getTypes() {
-        return TYPES;
+    public String getType() {
+        return NAME;
     }
 
     @Override
diff --git a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyPlugin.java b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyPlugin.java
index e7f81f58a78..e5ebcf12509 100644
--- a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyPlugin.java
+++ b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyPlugin.java
@@ -36,6 +36,6 @@ public class GroovyPlugin extends Plugin {
     }
 
     public void onModule(ScriptModule module) {
-        module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(GroovyScriptEngineService.class, GroovyScriptEngineService.TYPES));
+        module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(GroovyScriptEngineService.class, GroovyScriptEngineService.NAME));
     }
 }
diff --git a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
index 086118cab38..7c0be71596c 100644
--- a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
+++ b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
@@ -161,8 +161,8 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
     }
 
     @Override
-    public List<String> getTypes() {
-        return TYPES;
+    public String getType() {
+        return NAME;
     }
 
     @Override
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
index 1dc3fd96f81..a468517a512 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
@@ -38,6 +38,6 @@ public class MustachePlugin extends Plugin {
 
     public void onModule(ScriptModule module) {
         module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(MustacheScriptEngineService.class,
-                        MustacheScriptEngineService.TYPES, ScriptMode.ON));
+                        MustacheScriptEngineService.NAME, ScriptMode.ON));
     }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
index 4b0cb8fed4c..094adb95f92 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
@@ -112,8 +112,8 @@ public final class MustacheScriptEngineService extends AbstractComponent impleme
     }
 
     @Override
-    public List<String> getTypes() {
-        return TYPES;
+    public String getType() {
+        return NAME;
     }
 
     @Override
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
index b579ca3cc10..139d8dbf0b8 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
@@ -59,6 +59,7 @@ import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.mapper.MapperRegistry;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.script.ScriptEngineRegistry;
+import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.mustache.MustacheScriptEngineService;
@@ -94,7 +95,6 @@ public class TemplateQueryParserTests extends ESTestCase {
                 .put(Environment.PATH_CONF_SETTING.getKey(), this.getDataPath("config"))
                 .put("node.name", getClass().getName())
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-                .put("script.inline", "true")
                 .build();
         final Client proxy = (Client) Proxy.newProxyInstance(
                 Client.class.getClassLoader(),
@@ -107,7 +107,7 @@ public class TemplateQueryParserTests extends ESTestCase {
         ScriptModule scriptModule = new ScriptModule();
         scriptModule.prepareSettings(settingsModule);
         // TODO: make this use a mock engine instead of mustache and it will no longer be messy!
-        scriptModule.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(MustacheScriptEngineService.class, MustacheScriptEngineService.TYPES));
+        scriptModule.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(MustacheScriptEngineService.class, MustacheScriptEngineService.NAME, ScriptMode.ON));
         settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
         injector = new ModulesBuilder().add(
                 new EnvironmentModule(new Environment(settings)),
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
index 795fbdac28b..e5998948d62 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
@@ -41,6 +41,6 @@ public final class PainlessPlugin extends Plugin {
 
     public void onModule(final ScriptModule module) {
         module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(
-                        PainlessScriptEngineService.class, PainlessScriptEngineService.TYPES, ScriptMode.ON));
+                        PainlessScriptEngineService.class, PainlessScriptEngineService.NAME, ScriptMode.ON));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
index cab6950fda1..494e71b8782 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
@@ -103,8 +103,8 @@ public final class PainlessScriptEngineService extends AbstractComponent impleme
      * @return Always contains only the single name of the language.
      */
     @Override
-    public List<String> getTypes() {
-        return TYPES;
+    public String getType() {
+        return NAME;
     }
 
     /**
diff --git a/plugins/lang-javascript/src/main/java/org/elasticsearch/plugin/javascript/JavaScriptPlugin.java b/plugins/lang-javascript/src/main/java/org/elasticsearch/plugin/javascript/JavaScriptPlugin.java
index d8c79e8d110..edd7c860244 100644
--- a/plugins/lang-javascript/src/main/java/org/elasticsearch/plugin/javascript/JavaScriptPlugin.java
+++ b/plugins/lang-javascript/src/main/java/org/elasticsearch/plugin/javascript/JavaScriptPlugin.java
@@ -45,6 +45,6 @@ public class JavaScriptPlugin extends Plugin {
     }
 
     public void onModule(ScriptModule module) {
-        module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(JavaScriptScriptEngineService.class, JavaScriptScriptEngineService.TYPES));
+        module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(JavaScriptScriptEngineService.class, JavaScriptScriptEngineService.NAME));
     }
 }
diff --git a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
index 843a6e57142..88b4fe80ce4 100644
--- a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
+++ b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
@@ -68,7 +68,9 @@ import java.util.concurrent.atomic.AtomicLong;
  */
 public class JavaScriptScriptEngineService extends AbstractComponent implements ScriptEngineService {
 
-    public static final List<String> TYPES = Collections.unmodifiableList(Arrays.asList("js", "javascript"));
+    public static final String NAME = "javascript";
+
+    public static final List<String> EXTENSIONS = Collections.unmodifiableList(Arrays.asList("js", "javascript"));
 
     private final AtomicLong counter = new AtomicLong();
 
@@ -159,13 +161,13 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements
     }
 
     @Override
-    public List<String> getTypes() {
-        return TYPES;
+    public String getType() {
+        return NAME;
     }
 
     @Override
     public List<String> getExtensions() {
-        return Collections.unmodifiableList(Arrays.asList("js"));
+        return EXTENSIONS;
     }
 
     @Override
diff --git a/plugins/lang-javascript/src/test/resources/rest-api-spec/test/lang_javascript/20_search.yaml b/plugins/lang-javascript/src/test/resources/rest-api-spec/test/lang_javascript/20_search.yaml
index 742c8f0204d..043e9690303 100644
--- a/plugins/lang-javascript/src/test/resources/rest-api-spec/test/lang_javascript/20_search.yaml
+++ b/plugins/lang-javascript/src/test/resources/rest-api-spec/test/lang_javascript/20_search.yaml
@@ -30,12 +30,12 @@
                     script:
                         script:
                             inline: "doc['num1'].value > 1"
-                            lang: js
+                            lang: javascript
                 script_fields:
                     sNum1:
                         script: 
                             inline: "doc['num1'].value"
-                            lang: js
+                            lang: javascript
                 sort:
                     num1:
                         order: asc
@@ -52,7 +52,7 @@
                     script:
                         script:
                             inline: "doc['num1'].value > param1"
-                            lang: js
+                            lang: javascript
                             params:
                                 param1: 1
 
@@ -60,7 +60,7 @@
                     sNum1:
                         script:
                             inline: "doc['num1'].value"
-                            lang: js
+                            lang: javascript
                 sort:
                     num1:
                         order: asc
@@ -77,7 +77,7 @@
                     script:
                         script:
                             inline: "doc['num1'].value > param1"
-                            lang: js
+                            lang: javascript
                             params:
                                 param1: -1
 
@@ -85,7 +85,7 @@
                     sNum1:
                         script: 
                             inline: "doc['num1'].value"
-                            lang: js
+                            lang: javascript
                 sort:
                     num1:
                         order: asc
@@ -123,19 +123,19 @@
                     s_obj1:
                         script: 
                             inline: "_source.obj1"
-                            lang: js
+                            lang: javascript
                     s_obj1_test:
                         script: 
                             inline: "_source.obj1.test"
-                            lang: js
+                            lang: javascript
                     s_obj2:
                         script: 
                             inline: "_source.obj2"
-                            lang: js
+                            lang: javascript
                     s_obj2_arr2:
                         script: 
                             inline: "_source.obj2.arr2"
-                            lang: js
+                            lang: javascript
 
     - match: { hits.total: 1 }
     - match: { hits.hits.0.fields.s_obj1.0.test: something }
@@ -175,7 +175,7 @@
                         "functions": [{
                             "script_score": {
                                 "script": {
-                                    "lang": "js",
+                                    "lang": "javascript",
                                     "inline": "doc['num1'].value"
                                 }
                             }
@@ -197,7 +197,7 @@
                         "functions": [{
                             "script_score": {
                                 "script": {
-                                    "lang": "js",
+                                    "lang": "javascript",
                                     "inline": "-doc['num1'].value"
                                 }
                             }
@@ -219,7 +219,7 @@
                         "functions": [{
                             "script_score": {
                                 "script": {
-                                    "lang": "js",
+                                    "lang": "javascript",
                                     "inline": "Math.pow(doc['num1'].value, 2)"
                                 }
                             }
@@ -241,7 +241,7 @@
                         "functions": [{
                             "script_score": {
                                 "script": {
-                                    "lang": "js",
+                                    "lang": "javascript",
                                     "inline": "Math.max(doc['num1'].value, 1)"
                                 }
                             }
@@ -263,7 +263,7 @@
                         "functions": [{
                             "script_score": {
                                 "script": {
-                                    "lang": "js",
+                                    "lang": "javascript",
                                     "inline": "doc['num1'].value * _score"
                                 }
                             }
@@ -285,7 +285,7 @@
                         "functions": [{
                             "script_score": {
                                 "script": {
-                                    "lang": "js",
+                                    "lang": "javascript",
                                     "inline": "param1 * param2 * _score",
                                     "params": {
                                         "param1": 2,
@@ -322,14 +322,14 @@
                                     {
                                         "script_score": {
                                             "script": {
-                                                "lang": "js",
+                                                "lang": "javascript",
                                                 "inline": "1"
                                             }
                                         }
                                     }, {
                                         "script_score": {
                                             "script": {
-                                                "lang": "js",
+                                                "lang": "javascript",
                                                 "inline": "_score.doubleValue()"
                                             }
                                         }
@@ -338,7 +338,7 @@
                         "functions": [{
                             "script_score": {
                                 "script": {
-                                    "lang": "js",
+                                    "lang": "javascript",
                                     "inline": "_score.doubleValue()"
                                 }
                             }
@@ -370,7 +370,7 @@
                         "functions": [{
                             "script_score": {
                                 "script": {
-                                    "lang": "js",
+                                    "lang": "javascript",
                                     "inline": "_score.doubleValue()"
                                 }
                             }
@@ -379,7 +379,7 @@
                     score_agg:
                         terms:
                             script:
-                                lang: js
+                                lang: javascript
                                 inline: "_score.doubleValue()"
 
     - match: { hits.total: 1 }
@@ -408,7 +408,7 @@
                     foobar:
                         script: 
                             inline: "doc['f'].values.length"
-                            lang: js
+                            lang: javascript
 
 
     - match: { hits.total: 1 }
diff --git a/plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java b/plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java
index 50dd3fd98b4..a9fca62dc20 100644
--- a/plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java
+++ b/plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java
@@ -40,6 +40,6 @@ public class PythonPlugin extends Plugin {
     }
 
     public void onModule(ScriptModule module) {
-        module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(PythonScriptEngineService.class, PythonScriptEngineService.TYPES));
+        module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(PythonScriptEngineService.class, PythonScriptEngineService.TYPE));
     }
 }
diff --git a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
index c9f05def941..da094808940 100644
--- a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
+++ b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
@@ -58,7 +58,7 @@ import java.util.Map;
 //TODO we can optimize the case for Map<String, Object> similar to PyStringMap
 public class PythonScriptEngineService extends AbstractComponent implements ScriptEngineService {
 
-    public static final List<String> TYPES = Collections.unmodifiableList(Arrays.asList("py", "python"));
+    public static final String TYPE = "python";
 
     private final PythonInterpreter interp;
 
@@ -96,8 +96,8 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri
     }
 
     @Override
-    public List<String> getTypes() {
-        return TYPES;
+    public String getType() {
+        return TYPE;
     }
 
     @Override
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractMustacheTestCase.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractMustacheTestCase.java
index d13c4fe0f79..f58351b2e7d 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractMustacheTestCase.java
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractMustacheTestCase.java
@@ -24,6 +24,7 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.ingest.core.TemplateService;
 import org.elasticsearch.script.ScriptContextRegistry;
 import org.elasticsearch.script.ScriptEngineRegistry;
+import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptSettings;
 import org.elasticsearch.script.mustache.MustacheScriptEngineService;
@@ -43,8 +44,11 @@ public abstract class AbstractMustacheTestCase extends ESTestCase {
             .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
             .build();
         MustacheScriptEngineService mustache = new MustacheScriptEngineService(settings);
-        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(
-                new ScriptEngineRegistry.ScriptEngineRegistration(MustacheScriptEngineService.class, MustacheScriptEngineService.TYPES)));
+        ScriptEngineRegistry scriptEngineRegistry =
+                new ScriptEngineRegistry(Collections.singletonList(
+                                new ScriptEngineRegistry.ScriptEngineRegistration(MustacheScriptEngineService.class,
+                                                                                  MustacheScriptEngineService.NAME,
+                                                                                  ScriptMode.ON)));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
         ScriptService scriptService = new ScriptService(settings, new Environment(settings), Collections.singleton(mustache), null,
diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
index dd96b7600ae..142b33f6431 100644
--- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
+++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
@@ -57,14 +57,14 @@ public class MockScriptEngine implements ScriptEngineService {
 
         public void onModule(ScriptModule module) {
             module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
-                            MockScriptEngine.TYPES, ScriptMode.ON));
+                            MockScriptEngine.NAME, ScriptMode.ON));
         }
 
     }
 
     @Override
-    public List<String> getTypes() {
-        return TYPES;
+    public String getType() {
+        return NAME;
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index 4e83c1ea2bc..247a340d8c4 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -1622,7 +1622,9 @@ public abstract class ESIntegTestCase extends ESTestCase {
                 // from failing on nodes without enough disk space
                 .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "1b")
                 .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1b")
-                // wait short time for other active shards before actually deleting, default 30s not needed in tests
+                .put("script.stored", "true")
+                .put("script.inline", "true")
+                        // wait short time for other active shards before actually deleting, default 30s not needed in tests
                 .put(IndicesStore.INDICES_STORE_DELETE_SHARD_TIMEOUT.getKey(), new TimeValue(1, TimeUnit.SECONDS));
         return builder.build();
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
index f45a7530e7f..1e75f3d8261 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
@@ -178,6 +178,9 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
             // This needs to tie into the ESIntegTestCase#indexSettings() method
             .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), createTempDir().getParent())
             .put("node.name", nodeName())
+
+            .put("script.inline", "true")
+            .put("script.stored", "true")
             .put(EsExecutors.PROCESSORS_SETTING.getKey(), 1) // limit the number of threads created
             .put("http.enabled", false)
             .put(Node.NODE_LOCAL_SETTING.getKey(), true)

From 786a6a00d9049cf0ba4a9f834633ea1ee662525a Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 13 May 2016 11:27:12 -0400
Subject: [PATCH 0231/1311] Add test for fixed executor rejected count

This commit adds a test that a fixed executors rejected count behaves as
expected. In particular, we test that if we consume the executor, then
stuff the executor queue, further tasks will be rejected and the
rejected stats are updated appropriately. This test also asserts that if
we resize the queue the rejected count is reset to zero.

Relates #18301
---
 .../threadpool/FixedThreadPoolTests.java      | 106 ++++++++++++++++++
 1 file changed, 106 insertions(+)
 create mode 100644 core/src/test/java/org/elasticsearch/threadpool/FixedThreadPoolTests.java

diff --git a/core/src/test/java/org/elasticsearch/threadpool/FixedThreadPoolTests.java b/core/src/test/java/org/elasticsearch/threadpool/FixedThreadPoolTests.java
new file mode 100644
index 00000000000..85daebb86d5
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/threadpool/FixedThreadPoolTests.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.threadpool;
+
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+
+import java.util.concurrent.CountDownLatch;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class FixedThreadPoolTests extends ESThreadPoolTestCase {
+
+    public void testRejectedExecutionCounter() throws InterruptedException {
+        final String threadPoolName = randomThreadPool(ThreadPool.ThreadPoolType.FIXED);
+        // some of the fixed thread pool are bound by the number of
+        // cores so we can not exceed that
+        final int size = randomIntBetween(1, EsExecutors.boundedNumberOfProcessors(Settings.EMPTY));
+        final int queueSize = randomIntBetween(1, 16);
+        final long rejections = randomIntBetween(1, 16);
+
+        ThreadPool threadPool = null;
+        final Settings nodeSettings =
+            Settings.builder()
+                .put("node.name", "testRejectedExecutionCounter")
+                .put("threadpool." + threadPoolName + ".size", size)
+                .put("threadpool." + threadPoolName + ".queue_size", queueSize)
+                .build();
+        try {
+            threadPool = new ThreadPool(nodeSettings);
+
+            // these tasks will consume the thread pool causing further
+            // submissions to queue
+            final CountDownLatch latch = new CountDownLatch(size);
+            final CountDownLatch block = new CountDownLatch(1);
+            for (int i = 0; i < size; i++) {
+                threadPool.executor(threadPoolName).execute(() -> {
+                    try {
+                        latch.countDown();
+                        block.await();
+                    } catch (InterruptedException e) {
+                        fail(e.toString());
+                    }
+                });
+            }
+
+            // wait for the submitted tasks to be consumed by the thread
+            // pool
+            latch.await();
+
+            // these tasks will fill the thread pool queue
+            for (int i = 0; i < queueSize; i++) {
+                threadPool.executor(threadPoolName).execute(() -> {});
+            }
+
+            // these tasks will be rejected
+            long counter = 0;
+            for (int i = 0; i < rejections; i++) {
+                try {
+                    threadPool.executor(threadPoolName).execute(() -> {});
+                } catch (EsRejectedExecutionException e) {
+                    counter++;
+                }
+            }
+
+            block.countDown();
+
+            assertThat(counter, equalTo(rejections));
+            assertThat(stats(threadPool, threadPoolName).getRejected(), equalTo(rejections));
+
+            // the rejected execution count resets to zero when the
+            // queue is resized
+            final ClusterSettings clusterSettings =
+                new ClusterSettings(nodeSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+            threadPool.setClusterSettings(clusterSettings);
+            clusterSettings.applySettings(
+                Settings.builder()
+                    .put("threadpool." + threadPoolName + ".queue_size", queueSize + 1)
+                    .build());
+            assertThat(stats(threadPool, threadPoolName).getRejected(), equalTo(0L));
+
+        } finally {
+            terminateThreadPoolIfNeeded(threadPool);
+        }
+    }
+
+}

From a40c397c670ee1e7c2dd7032baedf09007e45f3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Fri, 13 May 2016 11:08:46 +0200
Subject: [PATCH 0232/1311] Don't allow `fuzziness` for `multi_match` types
 cross_fields, phrase and phrase_prefix

Currently `fuzziness` is not supported for the `cross_fields` type
of the `multi_match` query since it complicates the logic that
blends the term queries that cross_fields uses internally. At the
moment using this combination is silently ignored, which can lead to
confusions. Instead we should throw an exception in this case.
The same is true for phrase and phrase_prefix type.

Closes #7764
---
 .../index/query/MultiMatchQueryBuilder.java   |  5 +++
 .../query/MultiMatchQueryBuilderTests.java    | 39 ++++++++++++++++++-
 .../migration/migrate_5_0/search.asciidoc     |  4 ++
 .../query-dsl/multi-match-query.asciidoc      | 14 +++++++
 4 files changed, 61 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java
index 173acff6d00..032feac9193 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java
@@ -655,6 +655,11 @@ public class MultiMatchQueryBuilder extends AbstractQueryBuilder<MultiMatchQuery
             throw new ParsingException(parser.getTokenLocation(), "No fields specified for multi_match query");
         }
 
+        if (fuzziness != null && (type == Type.CROSS_FIELDS || type == Type.PHRASE || type == Type.PHRASE_PREFIX)) {
+            throw new ParsingException(parser.getTokenLocation(),
+                    "Fuziness not allowed for type [" + type.parseField.getPreferredName() + "]");
+        }
+
         return new MultiMatchQueryBuilder(value)
                 .fields(fieldsBoosts)
                 .type(type)
diff --git a/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
index f4baaa4880c..6b921a5becb 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
@@ -31,9 +31,11 @@ import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.PointRangeQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.lucene.all.AllTermQuery;
 import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
+import org.elasticsearch.index.query.MultiMatchQueryBuilder.Type;
 import org.elasticsearch.index.search.MatchQuery;
 
 import java.io.IOException;
@@ -80,7 +82,7 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase<MultiMatc
         if (randomBoolean()) {
             query.slop(randomIntBetween(0, 5));
         }
-        if (randomBoolean()) {
+        if (randomBoolean() && (query.type() == Type.BEST_FIELDS || query.type() == Type.MOST_FIELDS)) {
             query.fuzziness(randomFuzziness(fieldName));
         }
         if (randomBoolean()) {
@@ -254,4 +256,39 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase<MultiMatc
         assertEquals(json, MultiMatchQueryBuilder.Type.MOST_FIELDS, parsed.type());
         assertEquals(json, Operator.OR, parsed.operator());
     }
+
+    /**
+     * `fuzziness` is not allowed for `cross_fields`, `phrase` and `phrase_prefix` and should throw an error
+     */
+    public void testFuzzinessNotAllowedTypes() throws IOException {
+        String[] notAllowedTypes = new String[]{ Type.CROSS_FIELDS.parseField().getPreferredName(),
+            Type.PHRASE.parseField().getPreferredName(), Type.PHRASE_PREFIX.parseField().getPreferredName()};
+        for (String type : notAllowedTypes) {
+            String json =
+                    "{\n" +
+                    "  \"multi_match\" : {\n" +
+                    "    \"query\" : \"quick brown fox\",\n" +
+                    "    \"fields\" : [ \"title^1.0\", \"title.original^1.0\", \"title.shingles^1.0\" ],\n" +
+                    "    \"type\" : \"" + type + "\",\n" +
+                    "    \"fuzziness\" : 1" +
+                    "  }\n" +
+                    "}";
+
+            ParsingException e = expectThrows(ParsingException.class, () -> parseQuery(json));
+            assertEquals("Fuziness not allowed for type [" + type +"]", e.getMessage());
+        }
+    }
+
+    public void testQueryParameterArrayException() throws IOException {
+        String json =
+                "{\n" +
+                "  \"multi_match\" : {\n" +
+                "    \"query\" : [\"quick\", \"brown\", \"fox\"]\n" +
+                "    \"fields\" : [ \"title^1.0\", \"title.original^1.0\", \"title.shingles^1.0\" ]" +
+                "  }\n" +
+                "}";
+
+        ParsingException e = expectThrows(ParsingException.class, () -> parseQuery(json));
+        assertEquals("[multi_match] unknown token [START_ARRAY] after [query]", e.getMessage());
+    }
 }
diff --git a/docs/reference/migration/migrate_5_0/search.asciidoc b/docs/reference/migration/migrate_5_0/search.asciidoc
index ef08346e505..4a6ac438d70 100644
--- a/docs/reference/migration/migrate_5_0/search.asciidoc
+++ b/docs/reference/migration/migrate_5_0/search.asciidoc
@@ -126,6 +126,9 @@ in favour of `query` and `no_match_query`.
 
 * The `exists` query will now fail if the `_field_names` field is disabled.
 
+* The `multi_match` query will fail if `fuzziness` is used for `cross_fields`, `phrase` or `phrase_prefix` type.
+This parameter was undocumented and silently ignored before for these types of `multi_match`.
+
 * Deprecated support for the coerce, normalize, ignore_malformed parameters in GeoPolygonQuery. Use parameter validation_method instead.
 
 * Deprecated support for the coerce, normalize, ignore_malformed parameters in GeoDistanceRangeQuery. Use parameter validation_method instead.
@@ -133,6 +136,7 @@ in favour of `query` and `no_match_query`.
 * Deprecated support for the coerce, normalize, ignore_malformed parameters in GeoDistanceQuery. Use parameter validation_method instead.
 
 * Deprecated support for the coerce, normalize, ignore_malformed parameters in GeoBoundingBoxQuery. Use parameter validation_method instead.
+
 ==== Top level `filter` parameter
 
 Removed support for the deprecated top level `filter` in the search api,
diff --git a/docs/reference/query-dsl/multi-match-query.asciidoc b/docs/reference/query-dsl/multi-match-query.asciidoc
index fe1a5f6b974..5c958eeaf81 100644
--- a/docs/reference/query-dsl/multi-match-query.asciidoc
+++ b/docs/reference/query-dsl/multi-match-query.asciidoc
@@ -239,6 +239,13 @@ Also, accepts `analyzer`, `boost`, `slop` and `zero_terms_query`  as explained
 in <<query-dsl-match-query>>.  Type `phrase_prefix` additionally accepts
 `max_expansions`.
 
+[IMPORTANT]
+[[phrase-fuzziness]]
+.`phrase`, `phrase_prefix` and `fuzziness`
+===================================================
+The `fuzziness` parameter cannot be used with the `phrase` or `phrase_prefix` type.
+===================================================
+
 [[type-cross-fields]]
 ==== `cross_fields`
 
@@ -432,3 +439,10 @@ per-term `blended` queries. It accepts:
                     `last_name:will`
 `0.0 < n < 1.0`::   Take the single best score plus +tie_breaker+ multiplied
                     by each of the scores from other matching fields.
+
+[IMPORTANT]
+[[crossfields-fuzziness]]
+.`cross_fields` and `fuzziness`
+===================================================
+The `fuzziness` parameter cannot be used with the `cross_fields` type.
+===================================================

From e7c17fc9fb97384d05d20e6cb52c78e6ee315f9b Mon Sep 17 00:00:00 2001
From: Britta Weber <britta.weber@elasticsearch.com>
Date: Fri, 13 May 2016 12:59:41 +0200
Subject: [PATCH 0233/1311] [TEST] increase logger level until we know what is
 going on

We have an issue for it too: https://github.com/elastic/elasticsearch/issues/18121
---
 .../snapshots/SharedClusterSnapshotRestoreIT.java             | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
index 62d9c2c4120..5ed63b519a4 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -62,6 +62,7 @@ import org.elasticsearch.index.store.IndexStore;
 import org.elasticsearch.indices.InvalidIndexNameException;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.RepositoryException;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import java.nio.channels.SeekableByteChannel;
 import java.nio.file.Files;
@@ -1289,9 +1290,10 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(0));
     }
 
+    @TestLogging("_root:DEBUG")  // this fails every now and then: https://github.com/elastic/elasticsearch/issues/18121 but without
+    // more logs we cannot find out why
     public void testReadonlyRepository() throws Exception {
         Client client = client();
-
         logger.info("-->  creating repository");
         Path repositoryLocation = randomRepoPath();
         assertAcked(client.admin().cluster().preparePutRepository("test-repo")

From 0a300320cd096f40d4ac570b7b23912592a923c5 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 13 May 2016 11:47:00 -0400
Subject: [PATCH 0234/1311] [test] Set logging to 11 on reindex test

It has failures we can't explain and we need logs to be able to do
anything useful with the failures:
https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+master+g1gc/359/consoleFull
---
 .../org/elasticsearch/index/reindex/RethrottleTests.java    | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
index ab4b76ef23f..5a0196159de 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.ListenableActionFuture;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import static org.hamcrest.Matchers.hasSize;
 
@@ -29,6 +30,9 @@ import static org.hamcrest.Matchers.hasSize;
  * too but this is the only place that tests running against multiple nodes so it is the only integration tests that checks for
  * serialization.
  */
+// Extra logging in case of failure. We couldn't explain the last failure:
+// https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+master+g1gc/359/consoleFull
+@TestLogging("_root:DEBUG")
 public class RethrottleTests extends ReindexTestCase {
 
     public void testReindex() throws Exception {
@@ -56,6 +60,8 @@ public class RethrottleTests extends ReindexTestCase {
         // Now rethrottle it so it'll finish
         ListTasksResponse rethrottleResponse = rethrottle().setActions(actionName).setRequestsPerSecond(Float.POSITIVE_INFINITY).get();
         assertThat(rethrottleResponse.getTasks(), hasSize(1));
+        BulkByScrollTask.Status status = (BulkByScrollTask.Status) rethrottleResponse.getTasks().get(0).getStatus();
+        assertEquals(Float.POSITIVE_INFINITY, status.getRequestsPerSecond(), Float.MIN_NORMAL);
 
         // Now the response should come back quickly because we've rethrottled the request
         BulkIndexByScrollResponse response = responseListener.get();

From 9303ffbf821c92658252e0a9b54f37e6744e974d Mon Sep 17 00:00:00 2001
From: Chris Earle <chris.earle@elastic.co>
Date: Fri, 13 May 2016 11:48:49 -0400
Subject: [PATCH 0235/1311] Adding REST tests to ensure key_as_string behavior
 stays consistent

---
 .../test/search.aggregation/20_terms.yaml     | 197 +++++++++++++++++-
 1 file changed, 195 insertions(+), 2 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml
index cf4771eade3..71d1a1e7ca2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml
@@ -12,6 +12,14 @@ setup:
                      type: keyword
                   ip:
                      type: ip
+                  boolean:
+                     type: boolean
+                  integer:
+                    type: long
+                  double:
+                    type: double
+                  date:
+                    type: date
 
   - do:
       cluster.health:
@@ -45,7 +53,7 @@ setup:
     
   - do:
       search:
-        body: { "aggs" : { "str_terms" : { "terms" : { "field" : "str" } } } }
+        body: { "size" : 0, "aggs" : { "str_terms" : { "terms" : { "field" : "str" } } } }
 
   - match: { hits.total: 3 }
   
@@ -91,7 +99,7 @@ setup:
 
   - do:
       search:
-        body: { "aggs" : { "ip_terms" : { "terms" : { "field" : "ip" } } } }
+        body: { "size" : 0, "aggs" : { "ip_terms" : { "terms" : { "field" : "ip" } } } }
 
   - match: { hits.total: 3 }
 
@@ -108,3 +116,188 @@ setup:
   - is_false: aggregations.ip_terms.buckets.1.key_as_string
   
   - match: { aggregations.ip_terms.buckets.1.doc_count: 1 }
+
+---
+"Boolean test":
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: { "boolean": true }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 2
+        body: { "boolean": false }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 3
+        body: { "boolean": true }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "boolean_terms" : { "terms" : { "field" : "boolean" } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.boolean_terms.buckets: 2 }
+
+  - match: { aggregations.boolean_terms.buckets.0.key: 1 }
+
+  - match: { aggregations.boolean_terms.buckets.0.key_as_string: "true" }
+
+  - match: { aggregations.boolean_terms.buckets.0.doc_count: 2 }
+
+  - match: { aggregations.boolean_terms.buckets.1.key: 0 }
+
+  - match: { aggregations.boolean_terms.buckets.1.key_as_string: "false" }
+
+  - match: { aggregations.boolean_terms.buckets.1.doc_count: 1 }
+
+---
+"Integer test":
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: { "integer": 1234 }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 2
+        body: { "integer": 5678 }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 3
+        body: { "integer": 1234 }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "integer_terms" : { "terms" : { "field" : "integer" } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.integer_terms.buckets: 2 }
+
+  - match: { aggregations.integer_terms.buckets.0.key: 1234 }
+
+  - is_false: aggregations.integer_terms.buckets.0.key_as_string
+
+  - match: { aggregations.integer_terms.buckets.0.doc_count: 2 }
+
+  - match: { aggregations.integer_terms.buckets.1.key: 5678 }
+
+  - is_false: aggregations.integer_terms.buckets.1.key_as_string
+
+  - match: { aggregations.integer_terms.buckets.1.doc_count: 1 }
+
+---
+"Double test":
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: { "double": 1234.5 }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 2
+        body: { "double": 5678.5 }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 3
+        body: { "double": 1234.5 }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "double_terms" : { "terms" : { "field" : "double" } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.double_terms.buckets: 2 }
+
+  - match: { aggregations.double_terms.buckets.0.key: 1234.5 }
+
+  - is_false: aggregations.double_terms.buckets.0.key_as_string
+
+  - match: { aggregations.double_terms.buckets.0.doc_count: 2 }
+
+  - match: { aggregations.double_terms.buckets.1.key: 5678.5 }
+
+  - is_false: aggregations.double_terms.buckets.1.key_as_string
+
+  - match: { aggregations.double_terms.buckets.1.doc_count: 1 }
+
+---
+"Date test":
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 1
+        body: { "date": "2016-05-03" }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 2
+        body: { "date": "2014-09-01" }
+
+  - do:
+      index:
+        index: test_1
+        type: test
+        id: 3
+        body: { "date": "2016-05-03" }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "date_terms" : { "terms" : { "field" : "date" } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.date_terms.buckets: 2 }
+
+  - match: { aggregations.date_terms.buckets.0.key: 1462233600000 }
+
+  - match: { aggregations.date_terms.buckets.0.key_as_string: "2016-05-03T00:00:00.000Z" }
+
+  - match: { aggregations.date_terms.buckets.0.doc_count: 2 }
+
+  - match: { aggregations.date_terms.buckets.1.key: 1409529600000 }
+
+  - match: { aggregations.date_terms.buckets.1.key_as_string: "2014-09-01T00:00:00.000Z" }
+
+  - match: { aggregations.date_terms.buckets.1.doc_count: 1 }
+

From 9bcdafedda620cbc0d03316a52146f0a33daa1e0 Mon Sep 17 00:00:00 2001
From: Lee Hinman <lee@writequit.org>
Date: Thu, 12 May 2016 14:11:23 -0600
Subject: [PATCH 0236/1311] Allow only a single extension for a scripting
 engine

Previously multiple extensions could be provided, however, this can lead
to confusion with on-disk scripts (ie, "foo.js" and "foo.javascript")
having different content. Only a single extension is now supported.

The only language currently supporting multiple extensions was the
Javascript engine ("js" and "javascript"). It now only supports the
`.js` extension.

Relates to #10598
---
 .../script/NativeScriptEngineService.java     |  4 ++--
 .../script/ScriptEngineService.java           |  2 +-
 .../elasticsearch/script/ScriptService.java   |  4 +---
 .../script/ScriptModesTests.java              |  4 ++--
 .../script/ScriptServiceTests.java            | 14 +++++------
 .../script/ScriptSettingsTests.java           |  6 ++---
 .../search/aggregations/metrics/AvgIT.java    | 12 ++++------
 .../search/aggregations/metrics/SumIT.java    | 12 ++++------
 .../aggregations/metrics/ValueCountIT.java    |  6 ++---
 .../org/elasticsearch/update/UpdateIT.java    | 24 +++++++------------
 .../migration/migrate_5_0/scripting.asciidoc  |  6 +++++
 .../ExpressionScriptEngineService.java        |  6 ++---
 .../groovy/GroovyScriptEngineService.java     |  5 ++--
 .../mustache/MustacheScriptEngineService.java |  6 ++---
 .../painless/PainlessScriptEngineService.java | 19 ++-------------
 .../JavaScriptScriptEngineService.java        |  6 ++---
 .../plugin/python/PythonPlugin.java           |  2 +-
 .../python/PythonScriptEngineService.java     |  9 +++----
 .../script/MockScriptEngine.java              |  6 ++---
 19 files changed, 57 insertions(+), 96 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java b/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java
index 093cde4aa04..51d7c96c1ac 100644
--- a/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java
+++ b/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java
@@ -54,8 +54,8 @@ public class NativeScriptEngineService extends AbstractComponent implements Scri
     }
 
     @Override
-    public List<String> getExtensions() {
-        return Collections.emptyList();
+    public String getExtension() {
+        return ""; // Native scripts have no extensions
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java b/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java
index b89871b4dab..f8ae209c0c5 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java
@@ -33,7 +33,7 @@ public interface ScriptEngineService extends Closeable {
 
     String getType();
 
-    List<String> getExtensions();
+    String getExtension();
 
     Object compile(String script, Map<String, String> params);
 
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java
index 557499d07f7..78579ea9d6c 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java
@@ -170,9 +170,7 @@ public class ScriptService extends AbstractComponent implements Closeable {
         for (ScriptEngineService scriptEngine : scriptEngines) {
             String language = scriptEngineRegistry.getLanguage(scriptEngine.getClass());
             enginesByLangBuilder.put(language, scriptEngine);
-            for (String ext : scriptEngine.getExtensions()) {
-                enginesByExtBuilder.put(ext, scriptEngine);
-            }
+            enginesByExtBuilder.put(scriptEngine.getExtension(), scriptEngine);
         }
         this.scriptEnginesByLang = unmodifiableMap(enginesByLangBuilder);
         this.scriptEnginesByExt = unmodifiableMap(enginesByExtBuilder);
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java b/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
index 88a32cc5ddb..ed224bb7bbd 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
@@ -241,8 +241,8 @@ public class ScriptModesTests extends ESTestCase {
         }
 
         @Override
-        public List<String> getExtensions() {
-            return Collections.unmodifiableList(Arrays.asList("custom", "test"));
+        public String getExtension() {
+            return NAME;
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
index 96a406fb4a7..060c45bb9e6 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
@@ -145,7 +145,7 @@ public class ScriptServiceTests extends ESTestCase {
     public void testScriptsWithoutExtensions() throws IOException {
         buildScriptService(Settings.EMPTY);
         Path testFileNoExt = scriptsFilePath.resolve("test_no_ext");
-        Path testFileWithExt = scriptsFilePath.resolve("test_script.tst");
+        Path testFileWithExt = scriptsFilePath.resolve("test_script.test");
         Streams.copy("test_file_no_ext".getBytes("UTF-8"), Files.newOutputStream(testFileNoExt));
         Streams.copy("test_file".getBytes("UTF-8"), Files.newOutputStream(testFileWithExt));
         resourceWatcherService.notifyNow();
@@ -173,7 +173,7 @@ public class ScriptServiceTests extends ESTestCase {
         Path testHiddenFile = scriptsFilePath.resolve(".hidden_file");
         Streams.copy("test_hidden_file".getBytes("UTF-8"), Files.newOutputStream(testHiddenFile));
 
-        Path testFileScript = scriptsFilePath.resolve("file_script.tst");
+        Path testFileScript = scriptsFilePath.resolve("file_script.test");
         Streams.copy("test_file_script".getBytes("UTF-8"), Files.newOutputStream(testFileScript));
         resourceWatcherService.notifyNow();
 
@@ -510,16 +510,14 @@ public class ScriptServiceTests extends ESTestCase {
 
         public static final String NAME = "test";
 
-        public static final List<String> EXTENSIONS = Collections.unmodifiableList(Arrays.asList("test", "tst"));
-
         @Override
         public String getType() {
             return NAME;
         }
 
         @Override
-        public List<String> getExtensions() {
-            return EXTENSIONS;
+        public String getExtension() {
+            return NAME;
         }
 
         @Override
@@ -560,8 +558,8 @@ public class ScriptServiceTests extends ESTestCase {
         }
 
         @Override
-        public List<String> getExtensions() {
-            return EXTENSIONS;
+        public String getExtension() {
+            return NAME;
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java b/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
index ef432120039..fcb87321063 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
@@ -73,16 +73,14 @@ public class ScriptSettingsTests extends ESTestCase {
 
         public static final String NAME = "custom";
 
-        public static final List<String> TYPES = Collections.unmodifiableList(Arrays.asList("test"));
-
         @Override
         public String getType() {
             return NAME;
         }
 
         @Override
-        public List<String> getExtensions() {
-            return Collections.unmodifiableList(Arrays.asList("test"));
+        public String getExtension() {
+            return NAME;
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
index eeb0129c2a3..ef530c5876a 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
@@ -382,8 +382,6 @@ public class AvgIT extends AbstractNumericTestCase {
 
         public static final String NAME = "extract_field";
 
-        public static final List<String> TYPES = Collections.singletonList(NAME);
-
         @Override
         public void close() throws IOException {
         }
@@ -394,8 +392,8 @@ public class AvgIT extends AbstractNumericTestCase {
         }
 
         @Override
-        public List<String> getExtensions() {
-            return TYPES;
+        public String getExtension() {
+            return NAME;
         }
 
         @Override
@@ -509,8 +507,6 @@ public class AvgIT extends AbstractNumericTestCase {
 
         public static final String NAME = "field_value";
 
-        public static final List<String> TYPES = Collections.singletonList(NAME);
-
         @Override
         public void close() throws IOException {
         }
@@ -521,8 +517,8 @@ public class AvgIT extends AbstractNumericTestCase {
         }
 
         @Override
-        public List<String> getExtensions() {
-            return TYPES;
+        public String getExtension() {
+            return NAME;
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
index 50e0e40d4a0..c39558dc120 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
@@ -378,8 +378,6 @@ public class SumIT extends AbstractNumericTestCase {
 
         public static final String NAME = "extract_field";
 
-        public static final List<String> TYPES = Collections.singletonList(NAME);
-
         @Override
         public void close() throws IOException {
         }
@@ -390,8 +388,8 @@ public class SumIT extends AbstractNumericTestCase {
         }
 
         @Override
-        public List<String> getExtensions() {
-            return TYPES;
+        public String getExtension() {
+            return NAME;
         }
 
         @Override
@@ -507,8 +505,6 @@ public class SumIT extends AbstractNumericTestCase {
 
         public static final String NAME = "field_value";
 
-        public static final List<String> TYPES = Collections.singletonList(NAME);
-
         @Override
         public void close() throws IOException {
         }
@@ -519,8 +515,8 @@ public class SumIT extends AbstractNumericTestCase {
         }
 
         @Override
-        public List<String> getExtensions() {
-            return TYPES;
+        public String getExtension() {
+            return NAME;
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
index 483f542f584..0cfdc9bd3e6 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
@@ -236,8 +236,6 @@ public class ValueCountIT extends ESIntegTestCase {
 
         public static final String NAME = "field_value";
 
-        public static final List<String> TYPES = Collections.singletonList(NAME);
-
         @Override
         public void close() throws IOException {
         }
@@ -248,8 +246,8 @@ public class ValueCountIT extends ESIntegTestCase {
         }
 
         @Override
-        public List<String> getExtensions() {
-            return TYPES;
+        public String getExtension() {
+            return NAME;
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/update/UpdateIT.java b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
index 313dc1554e3..562867ebe24 100644
--- a/core/src/test/java/org/elasticsearch/update/UpdateIT.java
+++ b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
@@ -103,8 +103,6 @@ public class UpdateIT extends ESIntegTestCase {
 
         public static final String NAME = "put_values";
 
-        public static final List<String> TYPES = Collections.singletonList(NAME);
-
         @Override
         public void close() throws IOException {
         }
@@ -115,8 +113,8 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         @Override
-        public List<String> getExtensions() {
-            return TYPES;
+        public String getExtension() {
+            return NAME;
         }
 
         @Override
@@ -192,8 +190,6 @@ public class UpdateIT extends ESIntegTestCase {
 
         public static final String NAME = "field_inc";
 
-        public static final List<String> TYPES = Collections.singletonList(NAME);
-
         @Override
         public void close() throws IOException {
         }
@@ -204,8 +200,8 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         @Override
-        public List<String> getExtensions() {
-            return TYPES;
+        public String getExtension() {
+            return NAME;
         }
 
         @Override
@@ -274,8 +270,6 @@ public class UpdateIT extends ESIntegTestCase {
 
         public static final String NAME = "scripted_upsert";
 
-        public static final List<String> TYPES = Collections.singletonList(NAME);
-
         @Override
         public void close() throws IOException {
         }
@@ -286,8 +280,8 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         @Override
-        public List<String> getExtensions() {
-            return TYPES;
+        public String getExtension() {
+            return NAME;
         }
 
         @Override
@@ -356,8 +350,6 @@ public class UpdateIT extends ESIntegTestCase {
 
         public static final String NAME = "extract_ctx";
 
-        public static final List<String> TYPES = Collections.singletonList(NAME);
-
         @Override
         public void close() throws IOException {
         }
@@ -368,8 +360,8 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         @Override
-        public List<String> getExtensions() {
-            return TYPES;
+        public String getExtension() {
+            return NAME;
         }
 
         @Override
diff --git a/docs/reference/migration/migrate_5_0/scripting.asciidoc b/docs/reference/migration/migrate_5_0/scripting.asciidoc
index ca545cf432d..2e48d3635cd 100644
--- a/docs/reference/migration/migrate_5_0/scripting.asciidoc
+++ b/docs/reference/migration/migrate_5_0/scripting.asciidoc
@@ -93,3 +93,9 @@ script engine in particular registered both `"lang": "js"` and `"lang":
 "javascript"`. Script engines can now only register a single language. All
 references to `"lang": "js"` should be changed to `"lang": "javascript"` for
 existing users of the lang-javascript plugin.
+
+==== Scripting engines now register only a single extension
+
+Prior to 5.0.0 scripting engines could register multiple extensions. The only
+engine doing this was the Javascript engine, which registered "js" and
+"javascript". It now only registers the "js" file extension for on-disk scripts.
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
index 2979001f82d..21bc8e07ab0 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
@@ -62,8 +62,6 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
 
     public static final String NAME = "expression";
 
-    public static final List<String> TYPES = Collections.singletonList(NAME);
-
     @Inject
     public ExpressionScriptEngineService(Settings settings) {
         super(settings);
@@ -75,8 +73,8 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
     }
 
     @Override
-    public List<String> getExtensions() {
-        return TYPES;
+    public String getExtension() {
+        return NAME;
     }
 
     @Override
diff --git a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
index 7c0be71596c..7a1f10adf44 100644
--- a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
+++ b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
@@ -76,7 +76,6 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
      */
     public static final String NAME = "groovy";
 
-    public static final List<String> TYPES = Collections.singletonList(NAME);
     /**
      * The name of the Groovy compiler setting to use associated with activating <code>invokedynamic</code> support.
      */
@@ -166,8 +165,8 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
     }
 
     @Override
-    public List<String> getExtensions() {
-        return TYPES;
+    public String getExtension() {
+        return NAME;
     }
 
     @Override
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
index 094adb95f92..253f5cb3fcc 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
@@ -55,8 +55,6 @@ public final class MustacheScriptEngineService extends AbstractComponent impleme
 
     public static final String NAME = "mustache";
 
-    public static final List<String> TYPES = Collections.singletonList(NAME);
-
     static final String CONTENT_TYPE_PARAM = "content_type";
     static final String JSON_CONTENT_TYPE = "application/json";
     static final String PLAIN_TEXT_CONTENT_TYPE = "text/plain";
@@ -117,8 +115,8 @@ public final class MustacheScriptEngineService extends AbstractComponent impleme
     }
 
     @Override
-    public List<String> getExtensions() {
-        return TYPES;
+    public String getExtension() {
+        return NAME;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
index 494e71b8782..cd0c0a14ff0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
@@ -53,21 +53,6 @@ public final class PainlessScriptEngineService extends AbstractComponent impleme
      */
     public static final String NAME = "painless";
 
-    /**
-     * Standard list of names for the Painless language.  (There is only one.)
-     */
-    public static final List<String> TYPES = Collections.singletonList(NAME);
-
-    /**
-     * Standard extension of the Painless language.
-     */
-    public static final String EXTENSION = "painless";
-
-    /**
-     * Standard list of extensions for the Painless language.  (There is only one.)
-     */
-    public static final List<String> EXTENSIONS = Collections.singletonList(EXTENSION);
-
     /**
      * Default compiler settings to be used.
      */
@@ -112,8 +97,8 @@ public final class PainlessScriptEngineService extends AbstractComponent impleme
      * @return Always contains only the single extension of the language.
      */
     @Override
-    public List<String> getExtensions() {
-        return EXTENSIONS;
+    public String getExtension() {
+        return NAME;
     }
 
     /**
diff --git a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
index 88b4fe80ce4..da7583b771a 100644
--- a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
+++ b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
@@ -70,7 +70,7 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements
 
     public static final String NAME = "javascript";
 
-    public static final List<String> EXTENSIONS = Collections.unmodifiableList(Arrays.asList("js", "javascript"));
+    public static final String EXTENSION = "js";
 
     private final AtomicLong counter = new AtomicLong();
 
@@ -166,8 +166,8 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements
     }
 
     @Override
-    public List<String> getExtensions() {
-        return EXTENSIONS;
+    public String getExtension() {
+        return EXTENSION;
     }
 
     @Override
diff --git a/plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java b/plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java
index a9fca62dc20..ca1d3436c11 100644
--- a/plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java
+++ b/plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java
@@ -40,6 +40,6 @@ public class PythonPlugin extends Plugin {
     }
 
     public void onModule(ScriptModule module) {
-        module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(PythonScriptEngineService.class, PythonScriptEngineService.TYPE));
+        module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(PythonScriptEngineService.class, PythonScriptEngineService.NAME));
     }
 }
diff --git a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
index da094808940..d9930284d15 100644
--- a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
+++ b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
@@ -58,7 +58,8 @@ import java.util.Map;
 //TODO we can optimize the case for Map<String, Object> similar to PyStringMap
 public class PythonScriptEngineService extends AbstractComponent implements ScriptEngineService {
 
-    public static final String TYPE = "python";
+    public static final String NAME = "python";
+    public static final String EXTENSION = "py";
 
     private final PythonInterpreter interp;
 
@@ -97,12 +98,12 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri
 
     @Override
     public String getType() {
-        return TYPE;
+        return NAME;
     }
 
     @Override
-    public List<String> getExtensions() {
-        return Collections.unmodifiableList(Arrays.asList("py"));
+    public String getExtension() {
+        return EXTENSION;
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
index 142b33f6431..d0765d6627c 100644
--- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
+++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
@@ -38,8 +38,6 @@ public class MockScriptEngine implements ScriptEngineService {
 
     public static final String NAME = "mockscript";
 
-    public static final List<String> TYPES = Collections.singletonList(NAME);
-
     public static class TestPlugin extends Plugin {
 
         public TestPlugin() {
@@ -68,8 +66,8 @@ public class MockScriptEngine implements ScriptEngineService {
     }
 
     @Override
-    public List<String> getExtensions() {
-        return TYPES;
+    public String getExtension() {
+        return NAME;
     }
 
     @Override

From 81898a2e3eda7fc28921302df9b3e3e320220ab6 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 13 May 2016 12:26:18 -0400
Subject: [PATCH 0237/1311] Avoid race while retiring executors

Today, a race condition exists when retiring executors. Namely, if an
executor is retired and then the thread pool is terminated, the retiring
of the executor and the termination of the thread pool can race to
remove the retired executor from the queue of retired executors. More
precisely, when the executor is initially retired, it is placed on a
queue of retired executors, and then removed when it is successfully
shutdown. When the pool is terminated, it will also drain the queue of
retired executors. This leads to a time-of-check-time-of-use race where
the draining can see a retired executor on the queue but that retired
executor can be removed upon successful shutdown of that executor. This
leads to the draining attempting to remove an element from the queue
when there is none. This commit addresses this race condition by instead
safely polling the queue.

Relates #18333
---
 .../org/elasticsearch/threadpool/ThreadPool.java   | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
index 1f5baec1040..0d24bb74e17 100644
--- a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
+++ b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
@@ -440,8 +440,11 @@ public class ThreadPool extends AbstractComponent implements Closeable {
                 ((ThreadPoolExecutor) executor.executor()).shutdownNow();
             }
         }
-        while (!retiredExecutors.isEmpty()) {
-            ((ThreadPoolExecutor) retiredExecutors.remove().executor()).shutdownNow();
+
+        ExecutorHolder holder;
+        while ((holder = retiredExecutors.poll()) != null) {
+            ThreadPoolExecutor executor = (ThreadPoolExecutor) holder.executor();
+            executor.shutdownNow();
         }
     }
 
@@ -452,10 +455,13 @@ public class ThreadPool extends AbstractComponent implements Closeable {
                 result &= ((ThreadPoolExecutor) executor.executor()).awaitTermination(timeout, unit);
             }
         }
-        while (!retiredExecutors.isEmpty()) {
-            ThreadPoolExecutor executor = (ThreadPoolExecutor) retiredExecutors.remove().executor();
+
+        ExecutorHolder holder;
+        while ((holder = retiredExecutors.poll()) != null) {
+            ThreadPoolExecutor executor = (ThreadPoolExecutor) holder.executor();
             result &= executor.awaitTermination(timeout, unit);
         }
+
         estimatedTimeThread.join(unit.toMillis(timeout));
         return result;
     }

From b4dec0ddbe5807a1a9ff6c02f8eba9977004a831 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Fri, 13 May 2016 18:27:12 +0200
Subject: [PATCH 0238/1311] Remove dead code.

---
 .../search/aggregations/bucket/IpRangeIT.java            | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java
index e89db4b02a1..468e8503b0a 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java
@@ -256,13 +256,4 @@ public class IpRangeIT extends ESIntegTestCase {
         }
     }
 
-    public static void main(String[] args) {
-        AggregatorBuilder<?> aggregation =
-                AggregationBuilders
-                        .ipRange("agg")
-                        .field("ip")
-                        .addUnboundedTo("192.168.1.0")             // from -infinity to 192.168.1.0 (excluded)
-                        .addRange("192.168.1.0", "192.168.2.0")    // from 192.168.1.0 to 192.168.2.0 (excluded)
-                        .addUnboundedFrom("192.168.2.0");          // from 192.168.2.0 to +infinity
-    }
 }

From 9731d5d3140414e2da7816840f0f8cb05502b8dd Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Fri, 13 May 2016 19:58:05 +0200
Subject: [PATCH 0239/1311] painless: Rename the dynamic call site factory to
 DefBootstrap and make the inner class very short (PIC = Polymorphic Inline
 Cache)

---
 .../java/org/elasticsearch/painless/Def.java  |  2 +-
 ...DynamicCallSite.java => DefBootstrap.java} | 35 ++++++++++---------
 .../painless/WriterConstants.java             |  2 +-
 .../painless/node/LDefArray.java              |  6 ++--
 .../elasticsearch/painless/node/LDefCall.java |  4 +--
 .../painless/node/LDefField.java              |  6 ++--
 ...lSiteTests.java => DefBootstrapTests.java} | 18 +++++-----
 7 files changed, 38 insertions(+), 35 deletions(-)
 rename modules/lang-painless/src/main/java/org/elasticsearch/painless/{DynamicCallSite.java => DefBootstrap.java} (85%)
 rename modules/lang-painless/src/test/java/org/elasticsearch/painless/{DynamicCallSiteTests.java => DefBootstrapTests.java} (83%)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 580363bc574..f82f04d801a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -43,7 +43,7 @@ import java.util.stream.Stream;
  * or method depending on the receiver's class. For these, we emit an {@code invokedynamic} instruction that,
  * for each new type encountered will query a corresponding {@code lookupXXX} method to retrieve the appropriate
  * method. In most cases, the {@code lookupXXX} methods here will only be called once for a given call site, because
- * caching ({@link DynamicCallSite}) generally works: usually all objects at any call site will be consistently
+ * caching ({@link DefBootstrap}) generally works: usually all objects at any call site will be consistently
  * the same type (or just a few types).  In extreme cases, if there is type explosion, they may be called every
  * single time, but simplicity is still more valuable than performance in this code.
  */
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DynamicCallSite.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
similarity index 85%
rename from modules/lang-painless/src/main/java/org/elasticsearch/painless/DynamicCallSite.java
rename to modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index 330a64e2a99..d9e96302eec 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DynamicCallSite.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -27,7 +27,7 @@ import java.lang.invoke.MethodType;
 import java.lang.invoke.MutableCallSite;
 
 /**
- * Painless invokedynamic call site.
+ * Painless invokedynamic bootstrap for the call site.
  * <p>
  * Has 5 flavors (passed as static bootstrap parameters): dynamic method call,
  * dynamic field load (getter), and dynamic field store (setter), dynamic array load,
@@ -41,9 +41,9 @@ import java.lang.invoke.MutableCallSite;
  */
 // NOTE: this class must be public, because generated painless classes are in a different classloader,
 // and it needs to be accessible by that code.
-public final class DynamicCallSite {
+public final class DefBootstrap {
 
-    private DynamicCallSite() {} // no instance!
+    private DefBootstrap() {} // no instance!
 
     // NOTE: these must be primitive types, see https://docs.oracle.com/javase/specs/jvms/se7/html/jvms-6.html#jvms-6.5.invokedynamic
     /** static bootstrap parameter indicating a dynamic method call, e.g. foo.bar(...) */
@@ -57,7 +57,10 @@ public final class DynamicCallSite {
     /** static bootstrap parameter indicating a dynamic array store, e.g. foo[bar] = baz */
     public static final int ARRAY_STORE = 4;
 
-    static final class InliningCacheCallSite extends MutableCallSite {
+    /**
+     * CallSite that implements the polymorphic inlining cache (PIC).
+     */
+    static final class PIC extends MutableCallSite {
         /** maximum number of types before we go megamorphic */
         static final int MAX_DEPTH = 5;
 
@@ -65,14 +68,14 @@ public final class DynamicCallSite {
         private final int flavor;
         int depth; // pkg-protected for testing
 
-        InliningCacheCallSite(String name, MethodType type, int flavor) {
+        PIC(String name, MethodType type, int flavor) {
             super(type);
             this.name = name;
             this.flavor = flavor;
 
-            MethodHandle fallback = FALLBACK.bindTo(this);
-            fallback = fallback.asCollector(Object[].class, type.parameterCount());
-            fallback = fallback.asType(type);
+            final MethodHandle fallback = FALLBACK.bindTo(this)
+              .asCollector(Object[].class, type.parameterCount())
+              .asType(type);
 
             setTarget(fallback);
         }
@@ -109,11 +112,10 @@ public final class DynamicCallSite {
          * types at this call site and given up on caching).
          */
         Object fallback(Object[] args) throws Throwable {
-            MethodType type = type();
-            Object receiver = args[0];
-            Class<?> receiverClass = receiver.getClass();
-            MethodHandle target = lookup(flavor, receiverClass, name);
-            target = target.asType(type);
+            final MethodType type = type();
+            final Object receiver = args[0];
+            final Class<?> receiverClass = receiver.getClass();
+            final MethodHandle target = lookup(flavor, receiverClass, name).asType(type);
 
             if (depth >= MAX_DEPTH) {
                 // revert to a vtable call
@@ -124,7 +126,8 @@ public final class DynamicCallSite {
             MethodHandle test = CHECK_CLASS.bindTo(receiverClass);
             test = test.asType(test.type().changeParameterType(0, type.parameterType(0)));
 
-            MethodHandle guard = MethodHandles.guardWithTest(test, target, getTarget());
+            final MethodHandle guard = MethodHandles.guardWithTest(test, target, getTarget());
+            
             depth++;
 
             setTarget(guard);
@@ -139,7 +142,7 @@ public final class DynamicCallSite {
                 CHECK_CLASS = lookup.findStatic(lookup.lookupClass(), "checkClass",
                                                 MethodType.methodType(boolean.class, Class.class, Object.class));
                 FALLBACK = lookup.findVirtual(lookup.lookupClass(), "fallback",
-                                             MethodType.methodType(Object.class, Object[].class));
+                                              MethodType.methodType(Object.class, Object[].class));
             } catch (ReflectiveOperationException e) {
                 throw new AssertionError(e);
             }
@@ -155,7 +158,7 @@ public final class DynamicCallSite {
      * see https://docs.oracle.com/javase/specs/jvms/se7/html/jvms-6.html#jvms-6.5.invokedynamic
      */
     public static CallSite bootstrap(Lookup lookup, String name, MethodType type, int flavor) {
-        return new InliningCacheCallSite(name, type, flavor);
+        return new PIC(name, type, flavor);
     }
 
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index b13787cece5..3386fff9c58 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -58,7 +58,7 @@ public final class WriterConstants {
     public final static MethodType DEF_BOOTSTRAP_TYPE =
         MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, int.class);
     public final static Handle DEF_BOOTSTRAP_HANDLE =
-        new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(DynamicCallSite.class),
+        new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(DefBootstrap.class),
             "bootstrap", DEF_BOOTSTRAP_TYPE.toMethodDescriptorString());
 
     public final static String DEF_DYNAMIC_LOAD_FIELD_DESC =
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index 0fa2207bc9d..6b701a77728 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -21,7 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
-import org.elasticsearch.painless.DynamicCallSite;
+import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.commons.GeneratorAdapter;
 
@@ -62,13 +62,13 @@ final class LDefArray extends ALink {
     @Override
     void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
         adapter.visitInvokeDynamicInsn(
-            "arrayLoad", DEF_DYNAMIC_ARRAY_LOAD_DESC, DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.ARRAY_LOAD });
+            "arrayLoad", DEF_DYNAMIC_ARRAY_LOAD_DESC, DEF_BOOTSTRAP_HANDLE, new Object[] { DefBootstrap.ARRAY_LOAD });
 
     }
 
     @Override
     void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
         adapter.visitInvokeDynamicInsn(
-            "arrayStore", DEF_DYNAMIC_ARRAY_STORE_DESC, DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.ARRAY_STORE });
+            "arrayStore", DEF_DYNAMIC_ARRAY_STORE_DESC, DEF_BOOTSTRAP_HANDLE, new Object[] { DefBootstrap.ARRAY_STORE });
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index dc3e3abdcc8..dae2b74160a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -21,7 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
-import org.elasticsearch.painless.DynamicCallSite;
+import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.commons.GeneratorAdapter;
 
@@ -84,7 +84,7 @@ final class LDefCall extends ALink {
         // return value
         signature.append(definition.defType.type.getDescriptor());
 
-        adapter.visitInvokeDynamicInsn(name, signature.toString(), DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.METHOD_CALL });
+        adapter.visitInvokeDynamicInsn(name, signature.toString(), DEF_BOOTSTRAP_HANDLE, new Object[] { DefBootstrap.METHOD_CALL });
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index 851c1e3f631..f8b2f3be83a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -21,7 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
-import org.elasticsearch.painless.DynamicCallSite;
+import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.commons.GeneratorAdapter;
 
@@ -57,11 +57,11 @@ final class LDefField extends ALink {
 
     @Override
     void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
-        adapter.visitInvokeDynamicInsn(value, DEF_DYNAMIC_LOAD_FIELD_DESC, DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.LOAD });
+        adapter.visitInvokeDynamicInsn(value, DEF_DYNAMIC_LOAD_FIELD_DESC, DEF_BOOTSTRAP_HANDLE, new Object[] { DefBootstrap.LOAD });
     }
 
     @Override
     void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
-        adapter.visitInvokeDynamicInsn(value, DEF_DYNAMIC_STORE_FIELD_DESC, DEF_BOOTSTRAP_HANDLE, new Object[] { DynamicCallSite.STORE });
+        adapter.visitInvokeDynamicInsn(value, DEF_DYNAMIC_STORE_FIELD_DESC, DEF_BOOTSTRAP_HANDLE, new Object[] { DefBootstrap.STORE });
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DynamicCallSiteTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
similarity index 83%
rename from modules/lang-painless/src/test/java/org/elasticsearch/painless/DynamicCallSiteTests.java
rename to modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
index de5202236db..4469279c8be 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DynamicCallSiteTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
@@ -26,14 +26,14 @@ import java.lang.invoke.MethodType;
 
 import org.elasticsearch.test.ESTestCase;
 
-public class DynamicCallSiteTests extends ESTestCase {
+public class DefBootstrapTests extends ESTestCase {
     
     /** calls toString() on integers, twice */
     public void testOneType() throws Throwable {
-        CallSite site = DynamicCallSite.bootstrap(MethodHandles.publicLookup(), 
+        CallSite site = DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                   "toString", 
                                                   MethodType.methodType(String.class, Object.class), 
-                                                  DynamicCallSite.METHOD_CALL);
+                                                  DefBootstrap.METHOD_CALL);
         MethodHandle handle = site.dynamicInvoker();
         assertDepthEquals(site, 0);
 
@@ -47,10 +47,10 @@ public class DynamicCallSiteTests extends ESTestCase {
     }
     
     public void testTwoTypes() throws Throwable {
-        CallSite site = DynamicCallSite.bootstrap(MethodHandles.publicLookup(), 
+        CallSite site = DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                   "toString", 
                                                   MethodType.methodType(String.class, Object.class), 
-                                                  DynamicCallSite.METHOD_CALL);
+                                                  DefBootstrap.METHOD_CALL);
         MethodHandle handle = site.dynamicInvoker();
         assertDepthEquals(site, 0);
 
@@ -68,11 +68,11 @@ public class DynamicCallSiteTests extends ESTestCase {
     
     public void testTooManyTypes() throws Throwable {
         // if this changes, test must be rewritten
-        assertEquals(5, DynamicCallSite.InliningCacheCallSite.MAX_DEPTH);
-        CallSite site = DynamicCallSite.bootstrap(MethodHandles.publicLookup(), 
+        assertEquals(5, DefBootstrap.PIC.MAX_DEPTH);
+        CallSite site = DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                   "toString", 
                                                   MethodType.methodType(String.class, Object.class), 
-                                                  DynamicCallSite.METHOD_CALL);
+                                                  DefBootstrap.METHOD_CALL);
         MethodHandle handle = site.dynamicInvoker();
         assertDepthEquals(site, 0);
 
@@ -91,7 +91,7 @@ public class DynamicCallSiteTests extends ESTestCase {
     }
     
     static void assertDepthEquals(CallSite site, int expected) {
-        DynamicCallSite.InliningCacheCallSite dsite = (DynamicCallSite.InliningCacheCallSite) site;
+        DefBootstrap.PIC dsite = (DefBootstrap.PIC) site;
         assertEquals(expected, dsite.depth);
     }
 }

From 2028691e66a46be52a8c2edda0b4336816b3acb1 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Fri, 13 May 2016 15:40:45 -0400
Subject: [PATCH 0240/1311] painless: improve exception stacktraces

closes #18319
---
 .../script/NativeScriptEngineService.java     |  6 +-
 .../script/ScriptEngineService.java           | 10 +++-
 .../elasticsearch/script/ScriptService.java   | 15 +++--
 .../elasticsearch/script/FileScriptTests.java |  6 +-
 .../script/ScriptModesTests.java              |  2 +-
 .../script/ScriptServiceTests.java            |  8 +--
 .../script/ScriptSettingsTests.java           |  2 +-
 .../search/aggregations/metrics/AvgIT.java    |  8 +--
 .../search/aggregations/metrics/SumIT.java    |  8 +--
 .../aggregations/metrics/ValueCountIT.java    |  4 +-
 .../search/innerhits/InnerHitsIT.java         |  1 +
 .../org/elasticsearch/update/UpdateIT.java    | 10 ++--
 .../ExpressionScriptEngineService.java        |  6 +-
 .../script/expression/ExpressionTests.java    |  8 +--
 .../groovy/GroovyScriptEngineService.java     |  6 +-
 .../script/groovy/GroovySecurityTests.java    |  2 +-
 .../mustache/MustacheScriptEngineService.java |  6 +-
 .../mustache/MustacheScriptEngineTests.java   |  4 +-
 .../script/mustache/MustacheTests.java        | 14 ++---
 .../org/elasticsearch/painless/Compiler.java  |  6 +-
 .../painless/PainlessScriptEngineService.java | 13 ++---
 .../org/elasticsearch/painless/Writer.java    | 55 +++++++++++++++++--
 .../painless/WriterConstants.java             |  2 +-
 .../elasticsearch/painless/node/ANode.java    | 14 +++++
 .../elasticsearch/painless/node/SBreak.java   |  1 +
 .../painless/node/SContinue.java              |  1 +
 .../painless/node/SDeclaration.java           |  1 +
 .../org/elasticsearch/painless/node/SDo.java  |  1 +
 .../painless/node/SExpression.java            |  1 +
 .../org/elasticsearch/painless/node/SFor.java |  1 +
 .../elasticsearch/painless/node/SIfElse.java  |  1 +
 .../elasticsearch/painless/node/SReturn.java  |  1 +
 .../elasticsearch/painless/node/SThrow.java   |  1 +
 .../elasticsearch/painless/node/STrap.java    |  1 +
 .../org/elasticsearch/painless/node/STry.java |  1 +
 .../elasticsearch/painless/node/SWhile.java   |  1 +
 .../org/elasticsearch/painless/Debugger.java  |  4 +-
 .../painless/NeedsScoreTests.java             |  8 +--
 .../painless/ScriptEngineTests.java           |  4 +-
 .../painless/ScriptTestCase.java              |  2 +-
 .../painless/WhenThingsGoWrongTests.java      | 35 ++++++++++++
 .../JavaScriptScriptEngineService.java        |  4 +-
 .../JavaScriptScriptEngineTests.java          | 24 ++++----
 .../JavaScriptScriptMultiThreadedTests.java   |  6 +-
 .../javascript/JavaScriptSecurityTests.java   |  2 +-
 .../python/PythonScriptEngineService.java     |  4 +-
 .../python/PythonScriptEngineTests.java       | 18 +++---
 .../PythonScriptMultiThreadedTests.java       |  4 +-
 .../script/python/PythonSecurityTests.java    |  2 +-
 .../script/MockScriptEngine.java              | 27 +++++++--
 50 files changed, 257 insertions(+), 115 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java b/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java
index 51d7c96c1ac..79b9725c338 100644
--- a/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java
+++ b/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java
@@ -59,12 +59,12 @@ public class NativeScriptEngineService extends AbstractComponent implements Scri
     }
 
     @Override
-    public Object compile(String script, Map<String, String> params) {
-        NativeScriptFactory scriptFactory = scripts.get(script);
+    public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
+        NativeScriptFactory scriptFactory = scripts.get(scriptSource);
         if (scriptFactory != null) {
             return scriptFactory;
         }
-        throw new IllegalArgumentException("Native script [" + script + "] not found");
+        throw new IllegalArgumentException("Native script [" + scriptSource + "] not found");
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java b/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java
index f8ae209c0c5..2a36cf645e5 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java
@@ -35,7 +35,15 @@ public interface ScriptEngineService extends Closeable {
 
     String getExtension();
 
-    Object compile(String script, Map<String, String> params);
+    /**
+     * Compiles a script.
+     * @param scriptName name of the script. {@code null} if it is anonymous (inline). 
+     *                                        For a file script, its the file name (with extension).
+     *                                        For a stored script, its the identifier.
+     * @param scriptSource actual source of the script
+     * @param params compile-time parameters (such as flags to the compiler)
+     */
+    Object compile(String scriptName, String scriptSource, Map<String, String> params);
 
     ExecutableScript executable(CompiledScript compiledScript, @Nullable Map<String, Object> vars);
 
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java
index 78579ea9d6c..f77a0aae330 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java
@@ -302,7 +302,10 @@ public class ScriptService extends AbstractComponent implements Closeable {
             //Either an un-cached inline script or indexed script
             //If the script type is inline the name will be the same as the code for identification in exceptions
             try {
-                compiledScript = new CompiledScript(type, name, lang, scriptEngineService.compile(code, params));
+                // but give the script engine the chance to be better, give it separate name + source code
+                // for the inline case, then its anonymous: null.
+                String actualName = (type == ScriptType.INLINE) ? null : name;
+                compiledScript = new CompiledScript(type, name, lang, scriptEngineService.compile(actualName, code, params));
             } catch (Exception exception) {
                 throw new ScriptException("Failed to compile " + type + " script [" + name + "] using lang [" + lang + "]", exception);
             }
@@ -351,7 +354,7 @@ public class ScriptService extends AbstractComponent implements Closeable {
                     //we don't know yet what the script will be used for, but if all of the operations for this lang with
                     //indexed scripts are disabled, it makes no sense to even compile it.
                     if (isAnyScriptContextEnabled(scriptLang, scriptEngineService, ScriptType.STORED)) {
-                        Object compiled = scriptEngineService.compile(template.getScript(), Collections.emptyMap());
+                        Object compiled = scriptEngineService.compile(id, template.getScript(), Collections.emptyMap());
                         if (compiled == null) {
                             throw new IllegalArgumentException("Unable to parse [" + template.getScript() +
                                     "] lang [" + scriptLang + "] (ScriptService.compile returned null)");
@@ -559,8 +562,12 @@ public class ScriptService extends AbstractComponent implements Closeable {
                         logger.info("compiling script file [{}]", file.toAbsolutePath());
                         try (InputStreamReader reader = new InputStreamReader(Files.newInputStream(file), StandardCharsets.UTF_8)) {
                             String script = Streams.copyToString(reader);
-                            CacheKey cacheKey = new CacheKey(engineService, scriptNameExt.v1(), null, Collections.emptyMap());
-                            staticCache.put(cacheKey, new CompiledScript(ScriptType.FILE, scriptNameExt.v1(), engineService.getType(), engineService.compile(script, Collections.emptyMap())));
+                            String name = scriptNameExt.v1();
+                            CacheKey cacheKey = new CacheKey(engineService, name, null, Collections.emptyMap());
+                            // pass the actual file name to the compiler (for script engines that care about this)
+                            Object executable = engineService.compile(file.getFileName().toString(), script, Collections.emptyMap());
+                            CompiledScript compiledScript = new CompiledScript(ScriptType.FILE, name, engineService.getType(), executable);
+                            staticCache.put(cacheKey, compiledScript);
                             scriptMetrics.onCompilation();
                         }
                     } else {
diff --git a/core/src/test/java/org/elasticsearch/script/FileScriptTests.java b/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
index de734f86013..e5f78d7474e 100644
--- a/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
@@ -20,6 +20,7 @@ package org.elasticsearch.script;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.script.MockScriptEngine.MockCompiledScript;
 import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.test.ESTestCase;
 
@@ -56,7 +57,10 @@ public class FileScriptTests extends ESTestCase {
             .put("script.engine." + MockScriptEngine.NAME + ".file.aggs", "false").build();
         ScriptService scriptService = makeScriptService(settings);
         Script script = new Script("script1", ScriptService.ScriptType.FILE, MockScriptEngine.NAME, null);
-        assertNotNull(scriptService.compile(script, ScriptContext.Standard.SEARCH, Collections.emptyMap(), null));
+        CompiledScript compiledScript = scriptService.compile(script, ScriptContext.Standard.SEARCH, Collections.emptyMap(), null);
+        assertNotNull(compiledScript);
+        MockCompiledScript executable = (MockCompiledScript) compiledScript.compiled();
+        assertEquals("script1.mockscript", executable.name);
     }
 
     public void testAllOpsDisabled() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java b/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
index ed224bb7bbd..800c079b889 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
@@ -246,7 +246,7 @@ public class ScriptModesTests extends ESTestCase {
         }
 
         @Override
-        public Object compile(String script, Map<String, String> params) {
+        public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
             return null;
         }
 
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
index 060c45bb9e6..8fd8f674c3a 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
@@ -521,8 +521,8 @@ public class ScriptServiceTests extends ESTestCase {
         }
 
         @Override
-        public Object compile(String script, Map<String, String> params) {
-            return "compiled_" + script;
+        public Object compile(String scriptName, String scriptText, Map<String, String> params) {
+            return "compiled_" + scriptText;
         }
 
         @Override
@@ -563,8 +563,8 @@ public class ScriptServiceTests extends ESTestCase {
         }
 
         @Override
-        public Object compile(String script, Map<String, String> params) {
-            return "compiled_" + script;
+        public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
+            return "compiled_" + scriptSource;
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java b/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
index fcb87321063..3d82e2f1468 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
@@ -84,7 +84,7 @@ public class ScriptSettingsTests extends ESTestCase {
         }
 
         @Override
-        public Object compile(String script, Map<String, String> params) {
+        public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
             return null;
         }
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
index ef530c5876a..5b233a9690d 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
@@ -397,8 +397,8 @@ public class AvgIT extends AbstractNumericTestCase {
         }
 
         @Override
-        public Object compile(String script, Map<String, String> params) {
-            return script;
+        public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
+            return scriptSource;
         }
 
         @Override
@@ -522,8 +522,8 @@ public class AvgIT extends AbstractNumericTestCase {
         }
 
         @Override
-        public Object compile(String script, Map<String, String> params) {
-            return script;
+        public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
+            return scriptSource;
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
index c39558dc120..d0538973173 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
@@ -393,8 +393,8 @@ public class SumIT extends AbstractNumericTestCase {
         }
 
         @Override
-        public Object compile(String script, Map<String, String> params) {
-            return script;
+        public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
+            return scriptSource;
         }
 
         @Override
@@ -520,8 +520,8 @@ public class SumIT extends AbstractNumericTestCase {
         }
 
         @Override
-        public Object compile(String script, Map<String, String> params) {
-            return script;
+        public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
+            return scriptSource;
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
index 0cfdc9bd3e6..2b957188663 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
@@ -251,8 +251,8 @@ public class ValueCountIT extends ESIntegTestCase {
         }
 
         @Override
-        public Object compile(String script, Map<String, String> params) {
-            return script;
+        public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
+            return scriptSource;
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java b/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
index 1b6c8a607ef..8d8d3194bee 100644
--- a/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
@@ -21,6 +21,7 @@ package org.elasticsearch.search.innerhits;
 
 import org.apache.lucene.search.join.ScoreMode;
 import org.apache.lucene.util.ArrayUtil;
+
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
diff --git a/core/src/test/java/org/elasticsearch/update/UpdateIT.java b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
index 562867ebe24..65553a4a90e 100644
--- a/core/src/test/java/org/elasticsearch/update/UpdateIT.java
+++ b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
@@ -118,7 +118,7 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         @Override
-        public Object compile(String script, Map<String, String> params) {
+        public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
             return new Object(); // unused
         }
 
@@ -205,8 +205,8 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         @Override
-        public Object compile(String script, Map<String, String> params) {
-            return script;
+        public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
+            return scriptSource;
         }
 
         @Override
@@ -285,7 +285,7 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         @Override
-        public Object compile(String script, Map<String, String> params) {
+        public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
             return new Object(); // unused
         }
 
@@ -365,7 +365,7 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         @Override
-        public Object compile(String script, Map<String, String> params) {
+        public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
             return new Object(); // unused
         }
 
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
index 21bc8e07ab0..b06590a56e2 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
@@ -78,7 +78,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
     }
 
     @Override
-    public Object compile(String script, Map<String, String> params) {
+    public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
         // classloader created here
         final SecurityManager sm = System.getSecurityManager();
         if (sm != null) {
@@ -105,9 +105,9 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                         };
                     }
                     // NOTE: validation is delayed to allow runtime vars, and we don't have access to per index stuff here
-                    return JavascriptCompiler.compile(script, JavascriptCompiler.DEFAULT_FUNCTIONS, loader);
+                    return JavascriptCompiler.compile(scriptSource, JavascriptCompiler.DEFAULT_FUNCTIONS, loader);
                 } catch (ParseException e) {
-                    throw new ScriptException("Failed to parse expression: " + script, e);
+                    throw new ScriptException("Failed to parse expression: " + scriptSource, e);
                 }
             }
         });
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java
index 198558381d3..209718124b6 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java
@@ -37,19 +37,19 @@ public class ExpressionTests extends ESSingleNodeTestCase {
         ExpressionScriptEngineService service = new ExpressionScriptEngineService(Settings.EMPTY);
         SearchLookup lookup = new SearchLookup(index.mapperService(), index.fieldData(), null);
 
-        Object compiled = service.compile("1.2", Collections.emptyMap());
+        Object compiled = service.compile(null, "1.2", Collections.emptyMap());
         SearchScript ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "expression", compiled), lookup, Collections.<String, Object>emptyMap());
         assertFalse(ss.needsScores());
 
-        compiled = service.compile("doc['d'].value", Collections.emptyMap());
+        compiled = service.compile(null, "doc['d'].value", Collections.emptyMap());
         ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "expression", compiled), lookup, Collections.<String, Object>emptyMap());
         assertFalse(ss.needsScores());
 
-        compiled = service.compile("1/_score", Collections.emptyMap());
+        compiled = service.compile(null, "1/_score", Collections.emptyMap());
         ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "expression", compiled), lookup, Collections.<String, Object>emptyMap());
         assertTrue(ss.needsScores());
 
-        compiled = service.compile("doc['d'].value * _score", Collections.emptyMap());
+        compiled = service.compile(null, "doc['d'].value * _score", Collections.emptyMap());
         ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "expression", compiled), lookup, Collections.<String, Object>emptyMap());
         assertTrue(ss.needsScores());
     }
diff --git a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
index 7a1f10adf44..23f1b409359 100644
--- a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
+++ b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
@@ -170,19 +170,19 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
     }
 
     @Override
-    public Object compile(String script, Map<String, String> params) {
+    public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
         try {
             // we reuse classloader, so do a security check just in case.
             SecurityManager sm = System.getSecurityManager();
             if (sm != null) {
                 sm.checkPermission(new SpecialPermission());
             }
-            String fake = MessageDigests.toHexString(MessageDigests.sha1().digest(script.getBytes(StandardCharsets.UTF_8)));
+            String fake = MessageDigests.toHexString(MessageDigests.sha1().digest(scriptSource.getBytes(StandardCharsets.UTF_8)));
             // same logic as GroovyClassLoader.parseClass() but with a different codesource string:
             return AccessController.doPrivileged(new PrivilegedAction<Object>() {
                 @Override
                 public Class<?> run() {
-                    GroovyCodeSource gcs = new GroovyCodeSource(script, fake, BootstrapInfo.UNTRUSTED_CODEBASE);
+                    GroovyCodeSource gcs = new GroovyCodeSource(scriptSource, fake, BootstrapInfo.UNTRUSTED_CODEBASE);
                     gcs.setCachable(false);
                     // TODO: we could be more complicated and paranoid, and move this to separate block, to
                     // sandbox the compilation process itself better.
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java
index 6efc0707c5f..e0e439aa6e8 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java
@@ -138,7 +138,7 @@ public class GroovySecurityTests extends ESTestCase {
         vars.put("myarray", Arrays.asList("foo"));
         vars.put("myobject", new MyObject());
 
-        se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "test", "js", se.compile(script, Collections.emptyMap())), vars).run();
+        se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "test", "js", se.compile(null, script, Collections.emptyMap())), vars).run();
     }
 
     public static class MyObject {
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
index 253f5cb3fcc..b93eaaa9b52 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
@@ -86,12 +86,12 @@ public final class MustacheScriptEngineService extends AbstractComponent impleme
      * Compile a template string to (in this case) a Mustache object than can
      * later be re-used for execution to fill in missing parameter values.
      *
-     * @param template
+     * @param templateSource
      *            a string representing the template to compile.
      * @return a compiled template object for later execution.
      * */
     @Override
-    public Object compile(String template, Map<String, String> params) {
+    public Object compile(String templateName, String templateSource, Map<String, String> params) {
         String contentType = params.getOrDefault(CONTENT_TYPE_PARAM, JSON_CONTENT_TYPE);
         final DefaultMustacheFactory mustacheFactory;
         switch (contentType){
@@ -105,7 +105,7 @@ public final class MustacheScriptEngineService extends AbstractComponent impleme
                 break;
         }
         mustacheFactory.setObjectHandler(new CustomReflectionObjectHandler());
-        Reader reader = new FastStringReader(template);
+        Reader reader = new FastStringReader(templateSource);
         return mustacheFactory.compile(reader, "query-template");
     }
 
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java
index b388f8a3ad5..254020066b5 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java
@@ -54,7 +54,7 @@ public class MustacheScriptEngineTests extends ESTestCase {
                     + "\"negative\": {\"term\": {\"body\": {\"value\": \"solr\"}" + "}}, \"negative_boost\": {{boost_val}} } }}";
             Map<String, Object> vars = new HashMap<>();
             vars.put("boost_val", "0.3");
-            BytesReference o = (BytesReference) qe.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "", "mustache", qe.compile(template, compileParams)), vars).run();
+            BytesReference o = (BytesReference) qe.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "", "mustache", qe.compile(null, template, compileParams)), vars).run();
             assertEquals("GET _search {\"query\": {\"boosting\": {\"positive\": {\"match\": {\"body\": \"gift\"}},"
                     + "\"negative\": {\"term\": {\"body\": {\"value\": \"solr\"}}}, \"negative_boost\": 0.3 } }}",
                     new String(o.toBytes(), Charset.forName("UTF-8")));
@@ -65,7 +65,7 @@ public class MustacheScriptEngineTests extends ESTestCase {
             Map<String, Object> vars = new HashMap<>();
             vars.put("boost_val", "0.3");
             vars.put("body_val", "\"quick brown\"");
-            BytesReference o = (BytesReference) qe.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "", "mustache", qe.compile(template, compileParams)), vars).run();
+            BytesReference o = (BytesReference) qe.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "", "mustache", qe.compile(null, template, compileParams)), vars).run();
             assertEquals("GET _search {\"query\": {\"boosting\": {\"positive\": {\"match\": {\"body\": \"gift\"}},"
                     + "\"negative\": {\"term\": {\"body\": {\"value\": \"\\\"quick brown\\\"\"}}}, \"negative_boost\": 0.3 } }}",
                     new String(o.toBytes(), Charset.forName("UTF-8")));
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java
index 1bbae2b36b7..f850f117cb6 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java
@@ -58,7 +58,7 @@ public class MustacheTests extends ESTestCase {
             + "}}, \"negative_boost\": {{boost_val}} } }}";
         Map<String, Object> params = Collections.singletonMap("boost_val", "0.2");
 
-        Mustache mustache = (Mustache) engine.compile(template, Collections.emptyMap());
+        Mustache mustache = (Mustache) engine.compile(null, template, Collections.emptyMap());
         CompiledScript compiledScript = new CompiledScript(ScriptService.ScriptType.INLINE, "my-name", "mustache", mustache);
         ExecutableScript result = engine.executable(compiledScript, params);
         assertEquals(
@@ -71,7 +71,7 @@ public class MustacheTests extends ESTestCase {
 
     public void testArrayAccess() throws Exception {
         String template = "{{data.0}} {{data.1}}";
-        CompiledScript mustache = new CompiledScript(ScriptService.ScriptType.INLINE, "inline", "mustache", engine.compile(template, Collections.emptyMap()));
+        CompiledScript mustache = new CompiledScript(ScriptService.ScriptType.INLINE, "inline", "mustache", engine.compile(null, template, Collections.emptyMap()));
         Map<String, Object> vars = new HashMap<>();
         Object data = randomFrom(
             new String[] { "foo", "bar" },
@@ -97,7 +97,7 @@ public class MustacheTests extends ESTestCase {
 
     public void testArrayInArrayAccess() throws Exception {
         String template = "{{data.0.0}} {{data.0.1}}";
-        CompiledScript mustache = new CompiledScript(ScriptService.ScriptType.INLINE, "inline", "mustache", engine.compile(template, Collections.emptyMap()));
+        CompiledScript mustache = new CompiledScript(ScriptService.ScriptType.INLINE, "inline", "mustache", engine.compile(null, template, Collections.emptyMap()));
         Map<String, Object> vars = new HashMap<>();
         Object data = randomFrom(
             new String[][] { new String[] { "foo", "bar" }},
@@ -114,7 +114,7 @@ public class MustacheTests extends ESTestCase {
 
     public void testMapInArrayAccess() throws Exception {
         String template = "{{data.0.key}} {{data.1.key}}";
-        CompiledScript mustache = new CompiledScript(ScriptService.ScriptType.INLINE, "inline", "mustache", engine.compile(template, Collections.emptyMap()));
+        CompiledScript mustache = new CompiledScript(ScriptService.ScriptType.INLINE, "inline", "mustache", engine.compile(null, template, Collections.emptyMap()));
         Map<String, Object> vars = new HashMap<>();
         Object data = randomFrom(
             new Object[] { singletonMap("key", "foo"), singletonMap("key", "bar") },
@@ -141,7 +141,7 @@ public class MustacheTests extends ESTestCase {
     public void testEscaping() {
         // json string escaping enabled:
         Map<String, String> params = randomBoolean() ? Collections.emptyMap() : Collections.singletonMap(CONTENT_TYPE_PARAM, JSON_CONTENT_TYPE);
-        Mustache mustache = (Mustache) engine.compile("{ \"field1\": \"{{value}}\"}", Collections.emptyMap());
+        Mustache mustache = (Mustache) engine.compile(null, "{ \"field1\": \"{{value}}\"}", Collections.emptyMap());
         CompiledScript compiledScript = new CompiledScript(ScriptService.ScriptType.INLINE, "name", "mustache", mustache);
         ExecutableScript executableScript = engine.executable(compiledScript, Collections.singletonMap("value", "a \"value\""));
         BytesReference rawResult = (BytesReference) executableScript.run();
@@ -149,7 +149,7 @@ public class MustacheTests extends ESTestCase {
         assertThat(result, equalTo("{ \"field1\": \"a \\\"value\\\"\"}"));
 
         // json string escaping disabled:
-        mustache = (Mustache) engine.compile("{ \"field1\": \"{{value}}\"}", Collections.singletonMap(CONTENT_TYPE_PARAM, PLAIN_TEXT_CONTENT_TYPE));
+        mustache = (Mustache) engine.compile(null, "{ \"field1\": \"{{value}}\"}", Collections.singletonMap(CONTENT_TYPE_PARAM, PLAIN_TEXT_CONTENT_TYPE));
         compiledScript = new CompiledScript(ScriptService.ScriptType.INLINE, "name", "mustache", mustache);
         executableScript = engine.executable(compiledScript, Collections.singletonMap("value", "a \"value\""));
         rawResult = (BytesReference) executableScript.run();
@@ -162,7 +162,7 @@ public class MustacheTests extends ESTestCase {
         List<String> randomList = Arrays.asList(generateRandomStringArray(10, 20, false));
 
         String template = "{{data.array.size}} {{data.list.size}}";
-        CompiledScript mustache = new CompiledScript(ScriptService.ScriptType.INLINE, "inline", "mustache", engine.compile(template, Collections.emptyMap()));
+        CompiledScript mustache = new CompiledScript(ScriptService.ScriptType.INLINE, "inline", "mustache", engine.compile(null, template, Collections.emptyMap()));
         Map<String, Object> data = new HashMap<>();
         data.put("array", randomArrayValues);
         data.put("list", randomList);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
index ef75e904931..c487dddba71 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
@@ -93,7 +93,7 @@ final class Compiler {
      * @return An {@link Executable} Painless script.
      */
     static Executable compile(final Loader loader, final String name, final String source, final CompilerSettings settings) {
-        byte[] bytes = compile(source, settings);
+        byte[] bytes = compile(name, source, settings);
 
         return createExecutable(loader, name, source, bytes);
     }
@@ -104,7 +104,7 @@ final class Compiler {
      * @param settings The CompilerSettings to be used during the compilation.
      * @return The bytes for compilation.
      */
-    static byte[] compile(final String source, final CompilerSettings settings) {
+    static byte[] compile(String name, String source, CompilerSettings settings) {
         if (source.length() > MAXIMUM_SOURCE_LENGTH) {
             throw new IllegalArgumentException("Scripts may be no longer than " + MAXIMUM_SOURCE_LENGTH +
                 " characters.  The passed in script is " + source.length() + " characters.  Consider using a" +
@@ -115,7 +115,7 @@ final class Compiler {
         final SSource root = Walker.buildPainlessTree(source, reserved);
         final Variables variables = Analyzer.analyze(settings, Definition.INSTANCE, reserved, root);
 
-        return Writer.write(settings, Definition.INSTANCE, source, variables, root);
+        return Writer.write(settings, Definition.INSTANCE, name, source, variables, root);
     }
 
     /**
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
index cd0c0a14ff0..dafc6aaba8a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
@@ -100,15 +100,14 @@ public final class PainlessScriptEngineService extends AbstractComponent impleme
     public String getExtension() {
         return NAME;
     }
-
+    
     /**
-     * Compiles a Painless script with the specified parameters.
-     * @param script The code to be compiled.
-     * @param params The params used to modify the compiler settings on a per script basis.
-     * @return Compiled script object represented by an {@link Executable}.
+     * When a script is anonymous (inline), we give it this name.
      */
+    static final String INLINE_NAME = "<inline>";
+
     @Override
-    public Object compile(final String script, final Map<String, String> params) {
+    public Object compile(String scriptName, final String scriptSource, final Map<String, String> params) {
         final CompilerSettings compilerSettings;
 
         if (params.isEmpty()) {
@@ -154,7 +153,7 @@ public final class PainlessScriptEngineService extends AbstractComponent impleme
         return AccessController.doPrivileged(new PrivilegedAction<Executable>() {
             @Override
             public Executable run() {
-                return Compiler.compile(loader, "unknown", script, compilerSettings);
+                return Compiler.compile(loader, scriptName == null ? INLINE_NAME : scriptName, scriptSource, compilerSettings);
             }
         }, COMPILATION_CONTEXT);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
index ab85cf27398..dab9c79de6d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
@@ -39,14 +39,15 @@ import static org.elasticsearch.painless.WriterConstants.MAP_TYPE;
 final class Writer {
 
     static byte[] write(final CompilerSettings settings, final Definition definition,
-                               final String source, final Variables variables, final SSource root) {
-        final Writer writer = new Writer(settings, definition, source, variables, root);
+                               String name, final String source, final Variables variables, final SSource root) {
+        final Writer writer = new Writer(settings, definition, name, source, variables, root);
 
         return writer.getBytes();
     }
 
     private final CompilerSettings settings;
     private final Definition definition;
+    private final String scriptName;
     private final String source;
     private final Variables variables;
     private final SSource root;
@@ -55,9 +56,10 @@ final class Writer {
     private final GeneratorAdapter adapter;
 
     private Writer(final CompilerSettings settings, final Definition definition,
-                     final String source, final Variables variables, final SSource root) {
+                     String name, final String source, final Variables variables, final SSource root) {
         this.settings = settings;
         this.definition = definition;
+        this.scriptName = name;
         this.source = source;
         this.variables = variables;
         this.root = root;
@@ -73,8 +75,11 @@ final class Writer {
         writeEnd();
     }
 
+    // This maximum length is theoretically 65535 bytes, but as it's CESU-8 encoded we dont know how large it is in bytes, so be safe
+    private static final int MAX_NAME_LENGTH = 256;
+
     private void writeBegin() {
-        final int version = Opcodes.V1_7;
+        final int version = Opcodes.V1_8;
         final int access = Opcodes.ACC_PUBLIC | Opcodes.ACC_SUPER | Opcodes.ACC_FINAL;
         final String base = BASE_CLASS_TYPE.getInternalName();
         final String name = CLASS_TYPE.getInternalName();
@@ -84,7 +89,47 @@ final class Writer {
             new String[] { WriterConstants.NEEDS_SCORE_TYPE.getInternalName() } : null;
 
         writer.visit(version, access, name, null, base, interfaces);
-        writer.visitSource(source, null);
+        writer.visitSource(computeSourceName(), null);
+    }
+
+    /** Computes the file name (mostly important for stacktraces) */
+    private String computeSourceName() {
+        StringBuilder fileName = new StringBuilder();
+        if (scriptName.equals(PainlessScriptEngineService.INLINE_NAME)) {
+            // its an anonymous script, include at least a portion of the source to help identify which one it is
+            // but don't create stacktraces with filenames that contain newlines or huge names.
+
+            // truncate to the first newline
+            int limit = source.indexOf('\n');
+            if (limit >= 0) {
+                int limit2 = source.indexOf('\r');
+                if (limit2 >= 0) {
+                    limit = Math.min(limit, limit2);
+                }
+            } else {
+                limit = source.length();
+            }
+
+            // truncate to our limit
+            limit = Math.min(limit, MAX_NAME_LENGTH);
+            fileName.append(source, 0, limit);
+
+            // if we truncated, make it obvious
+            if (limit != source.length()) {
+                fileName.append(" ...");
+            }            
+            fileName.append(" @ <inline script>");
+        } else {
+            // its a named script, just use the name
+            // but don't trust this has a reasonable length!
+            if (scriptName.length() > MAX_NAME_LENGTH) {
+                fileName.append(scriptName, 0, MAX_NAME_LENGTH);
+                fileName.append(" ...");
+            } else {
+                fileName.append(scriptName);
+            }
+        }
+        return fileName.toString();
     }
 
     private void writeConstructor() {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index 3386fff9c58..8ad57c8fd8d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -37,7 +37,7 @@ import java.util.Map;
 public final class WriterConstants {
 
     public final static String BASE_CLASS_NAME = Executable.class.getName();
-    public final static String CLASS_NAME      = BASE_CLASS_NAME + "$CompiledPainlessExecutable";
+    public final static String CLASS_NAME      = BASE_CLASS_NAME + "$Script";
     public final static Type BASE_CLASS_TYPE   = Type.getType(Executable.class);
     public final static Type CLASS_TYPE        = Type.getType("L" + CLASS_NAME.replace(".", "/") + ";");
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
index b4c4fdd677d..11c081981fe 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
@@ -19,6 +19,9 @@
 
 package org.elasticsearch.painless.node;
 
+import org.objectweb.asm.Label;
+import org.objectweb.asm.commons.GeneratorAdapter;
+
 /**
  * The superclass for all other nodes.
  */
@@ -42,4 +45,15 @@ public abstract class ANode {
     public String error(final String message) {
         return "Error " + location  + ": " + message;
     }
+    
+    /** 
+     * Writes line number information
+     * <p>
+     * Currently we emit line number data for for leaf S-nodes
+     */
+    void writeDebugInfo(GeneratorAdapter adapter) {
+        Label label = new Label();
+        adapter.visitLabel(label);
+        adapter.visitLineNumber(line, label);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
index 7b589ec740e..c680eeee8fd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
@@ -47,6 +47,7 @@ public final class SBreak extends AStatement {
 
     @Override
     void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        writeDebugInfo(adapter);
         adapter.goTo(brake);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
index 3da0c653129..bf6d7f0ad61 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
@@ -50,6 +50,7 @@ public final class SContinue extends AStatement {
 
     @Override
     void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        writeDebugInfo(adapter);
         adapter.goTo(continu);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
index 14e240601ce..62d144d4a66 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
@@ -59,6 +59,7 @@ public final class SDeclaration extends AStatement {
 
     @Override
     void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        writeDebugInfo(adapter);
         final org.objectweb.asm.Type type = variable.type.type;
         final Sort sort = variable.type.sort;
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
index 329d9133d40..25549e8bbce 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -82,6 +82,7 @@ public final class SDo extends AStatement {
 
     @Override
     void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        writeDebugInfo(adapter);
         final Label start = new Label();
         final Label begin = new Label();
         final Label end = new Label();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index 17460d09870..b8e17fd3560 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -61,6 +61,7 @@ public final class SExpression extends AStatement {
 
     @Override
     void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        writeDebugInfo(adapter);
         expression.write(settings, definition, adapter);
 
         if (methodEscape) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index 64d171ca611..b76947f8e56 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -130,6 +130,7 @@ public final class SFor extends AStatement {
 
     @Override
     void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        writeDebugInfo(adapter);
         final Label start = new Label();
         final Label begin = afterthought == null ? start : new Label();
         final Label end = new Label();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
index 3914b39ba7f..e877acd0f99 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
@@ -85,6 +85,7 @@ public final class SIfElse extends AStatement {
 
     @Override
     void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        writeDebugInfo(adapter);
         final Label end = new Label();
         final Label fals = elseblock != null ? new Label() : end;
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
index a89abe4c5bc..25f7ea17f43 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -52,6 +52,7 @@ public final class SReturn extends AStatement {
 
     @Override
     void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        writeDebugInfo(adapter);
         expression.write(settings, definition, adapter);
         adapter.returnValue();
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
index b54caf1bce4..3bcb805fb6a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
@@ -51,6 +51,7 @@ public final class SThrow extends AStatement {
 
     @Override
     void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        writeDebugInfo(adapter);
         expression.write(settings, definition, adapter);
         adapter.throwException();
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java
index 77f3c9de9d6..a7b7ce20177 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java
@@ -78,6 +78,7 @@ public final class STrap extends AStatement {
 
     @Override
     void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        writeDebugInfo(adapter);
         final Label jump = new Label();
 
         adapter.mark(jump);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
index f9be4d75ad1..c6d5a16e06c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
@@ -84,6 +84,7 @@ public final class STry extends AStatement {
 
     @Override
     void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        writeDebugInfo(adapter);
         final Label begin = new Label();
         final Label end = new Label();
         final Label exception = new Label();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
index 29028bfe3b4..fd2f359c3c6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -94,6 +94,7 @@ public final class SWhile extends AStatement {
 
     @Override
     void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        writeDebugInfo(adapter);
         final Label begin = new Label();
         final Label end = new Label();
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
index d29e5a33a04..f037903f52d 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
@@ -37,8 +37,8 @@ final class Debugger {
     }
 
     /** compiles to bytecode, and returns debugging output */
-    static String toString(final String source, final CompilerSettings settings) {
-        final byte[] bytes = Compiler.compile(source, settings);
+    static String toString(String source, CompilerSettings settings) {
+        final byte[] bytes = Compiler.compile("<debugging>", source, settings);
         final ByteArrayOutputStream output = new ByteArrayOutputStream();
         final PrintWriter outputWriter = new PrintWriter(new OutputStreamWriter(output, StandardCharsets.UTF_8));
         final ClassReader reader = new ClassReader(bytes);
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java
index 2a0dfc17242..4b56d9751b6 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java
@@ -41,22 +41,22 @@ public class NeedsScoreTests extends ESSingleNodeTestCase {
         PainlessScriptEngineService service = new PainlessScriptEngineService(Settings.EMPTY);
         SearchLookup lookup = new SearchLookup(index.mapperService(), index.fieldData(), null);
 
-        Object compiled = service.compile("1.2", Collections.emptyMap());
+        Object compiled = service.compile(null, "1.2", Collections.emptyMap());
         SearchScript ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled),
                                          lookup, Collections.<String, Object>emptyMap());
         assertFalse(ss.needsScores());
 
-        compiled = service.compile("doc['d'].value", Collections.emptyMap());
+        compiled = service.compile(null, "doc['d'].value", Collections.emptyMap());
         ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled),
                             lookup, Collections.<String, Object>emptyMap());
         assertFalse(ss.needsScores());
 
-        compiled = service.compile("1/_score", Collections.emptyMap());
+        compiled = service.compile(null, "1/_score", Collections.emptyMap());
         ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled),
                             lookup, Collections.<String, Object>emptyMap());
         assertTrue(ss.needsScores());
 
-        compiled = service.compile("doc['d'].value * _score", Collections.emptyMap());
+        compiled = service.compile(null, "doc['d'].value * _score", Collections.emptyMap());
         ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled),
                             lookup, Collections.<String, Object>emptyMap());
         assertTrue(ss.needsScores());
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptEngineTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptEngineTests.java
index 4871a72776f..828b0b8e73b 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptEngineTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptEngineTests.java
@@ -81,7 +81,7 @@ public class ScriptEngineTests extends ScriptTestCase {
         Map<String, Object> ctx = new HashMap<>();
         vars.put("ctx", ctx);
 
-        Object compiledScript = scriptEngine.compile(
+        Object compiledScript = scriptEngine.compile(null,
                 "return ctx.value;", Collections.emptyMap());
         ExecutableScript script = scriptEngine.executable(new CompiledScript(ScriptService.ScriptType.INLINE,
                 "testChangingVarsCrossExecution1", "painless", compiledScript), vars);
@@ -97,7 +97,7 @@ public class ScriptEngineTests extends ScriptTestCase {
 
     public void testChangingVarsCrossExecution2() {
         Map<String, Object> vars = new HashMap<>();
-        Object compiledScript = scriptEngine.compile("return params['value'];", Collections.emptyMap());
+        Object compiledScript = scriptEngine.compile(null, "return params['value'];", Collections.emptyMap());
 
         ExecutableScript script = scriptEngine.executable(new CompiledScript(ScriptService.ScriptType.INLINE,
                 "testChangingVarsCrossExecution2", "painless", compiledScript), vars);
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
index 462c25c6634..7c86b393c30 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
@@ -53,7 +53,7 @@ public abstract class ScriptTestCase extends ESTestCase {
 
     /** Compiles and returns the result of {@code script} with access to {@code vars} and compile-time parameters */
     public Object exec(String script, Map<String, Object> vars, Map<String,String> compileParams) {
-        Object object = scriptEngine.compile(script, compileParams);
+        Object object = scriptEngine.compile(null, script, compileParams);
         CompiledScript compiled = new CompiledScript(ScriptService.ScriptType.INLINE, getTestName(), "painless", object);
         return scriptEngine.executable(compiled, vars).run();
     }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
index 5e4d1aa2014..f4a092a5145 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
@@ -30,6 +30,41 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
         });
     }
 
+    public void testLineNumbers() {
+        // trigger NPE at line 1 of the script
+        NullPointerException exception = expectThrows(NullPointerException.class, () -> {
+            exec("String x = null; boolean y = x.isEmpty();\n" +
+                 "return y;");
+        });
+        assertEquals(1, exception.getStackTrace()[0].getLineNumber());
+
+        // trigger NPE at line 2 of the script
+        exception = expectThrows(NullPointerException.class, () -> {
+            exec("String x = null;\n" +
+                 "return x.isEmpty();");
+        });
+        assertEquals(2, exception.getStackTrace()[0].getLineNumber());
+
+        // trigger NPE at line 3 of the script
+        exception = expectThrows(NullPointerException.class, () -> {
+            exec("String x = null;\n" +
+                 "String y = x;\n" +
+                 "return y.isEmpty();");
+        });
+        assertEquals(3, exception.getStackTrace()[0].getLineNumber());
+        
+        // trigger NPE at line 4 in script (inside conditional)
+        exception = expectThrows(NullPointerException.class, () -> {
+            exec("String x = null;\n" +
+                 "boolean y = false;\n" +
+                 "if (!y) {\n" +
+                 "  y = x.isEmpty();\n" + 
+                 "}\n" +
+                 "return y;");
+        });
+        assertEquals(4, exception.getStackTrace()[0].getLineNumber());
+    }
+
     public void testInvalidShift() {
         expectThrows(ClassCastException.class, () -> {
             exec("float x = 15F; x <<= 2; return x;");
diff --git a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
index da7583b771a..d4f713d9cb5 100644
--- a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
+++ b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
@@ -171,10 +171,10 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements
     }
 
     @Override
-    public Object compile(String script, Map<String, String> params) {
+    public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
         Context ctx = Context.enter();
         try {
-            return ctx.compileString(script, generateScriptName(), 1, DOMAIN);
+            return ctx.compileString(scriptSource, generateScriptName(), 1, DOMAIN);
         } finally {
             Context.exit();
         }
diff --git a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineTests.java b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineTests.java
index 5ed2e7e4cb8..c3afe436c31 100644
--- a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineTests.java
+++ b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineTests.java
@@ -55,7 +55,7 @@ public class JavaScriptScriptEngineTests extends ESTestCase {
 
     public void testSimpleEquation() {
         Map<String, Object> vars = new HashMap<String, Object>();
-        Object o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testSimpleEquation", "js", se.compile("1 + 2", Collections.emptyMap())), vars).run();
+        Object o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testSimpleEquation", "js", se.compile(null, "1 + 2", Collections.emptyMap())), vars).run();
         assertThat(((Number) o).intValue(), equalTo(3));
     }
 
@@ -66,13 +66,13 @@ public class JavaScriptScriptEngineTests extends ESTestCase {
         Map<String, Object> obj2 = MapBuilder.<String, Object>newMapBuilder().put("prop2", "value2").map();
         Map<String, Object> obj1 = MapBuilder.<String, Object>newMapBuilder().put("prop1", "value1").put("obj2", obj2).put("l", Arrays.asList("2", "1")).map();
         vars.put("obj1", obj1);
-        Object o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testMapAccess", "js", se.compile("obj1", Collections.emptyMap())), vars).run();
+        Object o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testMapAccess", "js", se.compile(null, "obj1", Collections.emptyMap())), vars).run();
         assertThat(o, instanceOf(Map.class));
         obj1 = (Map<String, Object>) o;
         assertThat((String) obj1.get("prop1"), equalTo("value1"));
         assertThat((String) ((Map<String, Object>) obj1.get("obj2")).get("prop2"), equalTo("value2"));
 
-        o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testMapAccess", "js", se.compile("obj1.l[0]", Collections.emptyMap())), vars).run();
+        o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testMapAccess", "js", se.compile(null, "obj1.l[0]", Collections.emptyMap())), vars).run();
         assertThat(((String) o), equalTo("2"));
     }
 
@@ -80,7 +80,7 @@ public class JavaScriptScriptEngineTests extends ESTestCase {
     public void testJavaScriptObjectToMap() {
         Map<String, Object> vars = new HashMap<String, Object>();
         Object o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testJavaScriptObjectToMap", "js",
-                se.compile("var obj1 = {}; obj1.prop1 = 'value1'; obj1.obj2 = {}; obj1.obj2.prop2 = 'value2'; obj1", Collections.emptyMap())), vars).run();
+                se.compile(null, "var obj1 = {}; obj1.prop1 = 'value1'; obj1.obj2 = {}; obj1.obj2.prop2 = 'value2'; obj1", Collections.emptyMap())), vars).run();
         Map<String, Object> obj1 = (Map<String, Object>) o;
         assertThat((String) obj1.get("prop1"), equalTo("value1"));
         assertThat((String) ((Map<String, Object>) obj1.get("obj2")).get("prop2"), equalTo("value2"));
@@ -96,7 +96,7 @@ public class JavaScriptScriptEngineTests extends ESTestCase {
         vars.put("ctx", ctx);
 
         ExecutableScript executable = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testJavaScriptObjectMapInter", "js",
-                se.compile("ctx.obj2 = {}; ctx.obj2.prop2 = 'value2'; ctx.obj1.prop1 = 'uvalue1'", Collections.emptyMap())), vars);
+                se.compile(null, "ctx.obj2 = {}; ctx.obj2.prop2 = 'value2'; ctx.obj1.prop1 = 'uvalue1'", Collections.emptyMap())), vars);
         executable.run();
         ctx = (Map<String, Object>) executable.unwrap(vars.get("ctx"));
         assertThat(ctx.containsKey("obj1"), equalTo(true));
@@ -111,7 +111,7 @@ public class JavaScriptScriptEngineTests extends ESTestCase {
         Map<String, Object> doc = new HashMap<String, Object>();
         ctx.put("doc", doc);
 
-        Object compiled = se.compile("ctx.doc.field1 = ['value1', 'value2']", Collections.emptyMap());
+        Object compiled = se.compile(null, "ctx.doc.field1 = ['value1', 'value2']", Collections.emptyMap());
         ExecutableScript script = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testJavaScriptInnerArrayCreation", "js",
                 compiled), new HashMap<String, Object>());
         script.setNextVar("ctx", ctx);
@@ -130,21 +130,21 @@ public class JavaScriptScriptEngineTests extends ESTestCase {
         vars.put("l", Arrays.asList("1", "2", "3", obj1));
 
         Object o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testAccessInScript", "js",
-                se.compile("l.length", Collections.emptyMap())), vars).run();
+                se.compile(null, "l.length", Collections.emptyMap())), vars).run();
         assertThat(((Number) o).intValue(), equalTo(4));
 
         o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testAccessInScript", "js",
-                se.compile("l[0]", Collections.emptyMap())), vars).run();
+                se.compile(null, "l[0]", Collections.emptyMap())), vars).run();
         assertThat(((String) o), equalTo("1"));
 
         o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testAccessInScript", "js",
-                se.compile("l[3]", Collections.emptyMap())), vars).run();
+                se.compile(null, "l[3]", Collections.emptyMap())), vars).run();
         obj1 = (Map<String, Object>) o;
         assertThat((String) obj1.get("prop1"), equalTo("value1"));
         assertThat((String) ((Map<String, Object>) obj1.get("obj2")).get("prop2"), equalTo("value2"));
 
         o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testAccessInScript", "js",
-                se.compile("l[3].prop1", Collections.emptyMap())), vars).run();
+                se.compile(null, "l[3].prop1", Collections.emptyMap())), vars).run();
         assertThat(((String) o), equalTo("value1"));
     }
 
@@ -152,7 +152,7 @@ public class JavaScriptScriptEngineTests extends ESTestCase {
         Map<String, Object> vars = new HashMap<String, Object>();
         Map<String, Object> ctx = new HashMap<String, Object>();
         vars.put("ctx", ctx);
-        Object compiledScript = se.compile("ctx.value", Collections.emptyMap());
+        Object compiledScript = se.compile(null, "ctx.value", Collections.emptyMap());
 
         ExecutableScript script = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testChangingVarsCrossExecution1", "js",
                 compiledScript), vars);
@@ -167,7 +167,7 @@ public class JavaScriptScriptEngineTests extends ESTestCase {
 
     public void testChangingVarsCrossExecution2() {
         Map<String, Object> vars = new HashMap<String, Object>();
-        Object compiledScript = se.compile("value", Collections.emptyMap());
+        Object compiledScript = se.compile(null, "value", Collections.emptyMap());
 
         ExecutableScript script = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testChangingVarsCrossExecution2", "js",
                 compiledScript), vars);
diff --git a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptMultiThreadedTests.java b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptMultiThreadedTests.java
index b6be9f6dde0..c2f1214e72b 100644
--- a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptMultiThreadedTests.java
+++ b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptMultiThreadedTests.java
@@ -41,7 +41,7 @@ import static org.hamcrest.Matchers.equalTo;
 public class JavaScriptScriptMultiThreadedTests extends ESTestCase {
     public void testExecutableNoRuntimeParams() throws Exception {
         final JavaScriptScriptEngineService se = new JavaScriptScriptEngineService(Settings.Builder.EMPTY_SETTINGS);
-        final Object compiled = se.compile("x + y", Collections.emptyMap());
+        final Object compiled = se.compile(null, "x + y", Collections.emptyMap());
         final AtomicBoolean failed = new AtomicBoolean();
 
         Thread[] threads = new Thread[50];
@@ -83,7 +83,7 @@ public class JavaScriptScriptMultiThreadedTests extends ESTestCase {
 
     public void testExecutableWithRuntimeParams() throws Exception {
         final JavaScriptScriptEngineService se = new JavaScriptScriptEngineService(Settings.Builder.EMPTY_SETTINGS);
-        final Object compiled = se.compile("x + y", Collections.emptyMap());
+        final Object compiled = se.compile(null, "x + y", Collections.emptyMap());
         final AtomicBoolean failed = new AtomicBoolean();
 
         Thread[] threads = new Thread[50];
@@ -125,7 +125,7 @@ public class JavaScriptScriptMultiThreadedTests extends ESTestCase {
 
     public void testExecute() throws Exception {
         final JavaScriptScriptEngineService se = new JavaScriptScriptEngineService(Settings.Builder.EMPTY_SETTINGS);
-        final Object compiled = se.compile("x + y", Collections.emptyMap());
+        final Object compiled = se.compile(null, "x + y", Collections.emptyMap());
         final AtomicBoolean failed = new AtomicBoolean();
 
         Thread[] threads = new Thread[50];
diff --git a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptSecurityTests.java b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptSecurityTests.java
index ded3db2f5bb..c1eb77d4e20 100644
--- a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptSecurityTests.java
+++ b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptSecurityTests.java
@@ -54,7 +54,7 @@ public class JavaScriptSecurityTests extends ESTestCase {
     /** runs a script */
     private void doTest(String script) {
         Map<String, Object> vars = new HashMap<String, Object>();
-        se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "test", "js", se.compile(script, Collections.emptyMap())), vars).run();
+        se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "test", "js", se.compile(null, script, Collections.emptyMap())), vars).run();
     }
 
     /** asserts that a script runs without exception */
diff --git a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
index d9930284d15..3f6ceb4a1b6 100644
--- a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
+++ b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
@@ -107,7 +107,7 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri
     }
 
     @Override
-    public Object compile(String script, Map<String, String> params) {
+    public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
         // classloader created here
         SecurityManager sm = System.getSecurityManager();
         if (sm != null) {
@@ -116,7 +116,7 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri
         return AccessController.doPrivileged(new PrivilegedAction<PyCode>() {
             @Override
             public PyCode run() {
-                return interp.compile(script);
+                return interp.compile(scriptSource);
             }
         });
     }
diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java
index fd876d8cee8..b96071e3848 100644
--- a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java
+++ b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java
@@ -54,7 +54,7 @@ public class PythonScriptEngineTests extends ESTestCase {
 
     public void testSimpleEquation() {
         Map<String, Object> vars = new HashMap<String, Object>();
-        Object o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testSimpleEquation", "python", se.compile("1 + 2", Collections.emptyMap())), vars).run();
+        Object o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testSimpleEquation", "python", se.compile(null, "1 + 2", Collections.emptyMap())), vars).run();
         assertThat(((Number) o).intValue(), equalTo(3));
     }
 
@@ -65,13 +65,13 @@ public class PythonScriptEngineTests extends ESTestCase {
         Map<String, Object> obj2 = MapBuilder.<String, Object>newMapBuilder().put("prop2", "value2").map();
         Map<String, Object> obj1 = MapBuilder.<String, Object>newMapBuilder().put("prop1", "value1").put("obj2", obj2).put("l", Arrays.asList("2", "1")).map();
         vars.put("obj1", obj1);
-        Object o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testMapAccess", "python", se.compile("obj1", Collections.emptyMap())), vars).run();
+        Object o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testMapAccess", "python", se.compile(null, "obj1", Collections.emptyMap())), vars).run();
         assertThat(o, instanceOf(Map.class));
         obj1 = (Map<String, Object>) o;
         assertThat((String) obj1.get("prop1"), equalTo("value1"));
         assertThat((String) ((Map<String, Object>) obj1.get("obj2")).get("prop2"), equalTo("value2"));
 
-        o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testMapAccess", "python", se.compile("obj1['l'][0]", Collections.emptyMap())), vars).run();
+        o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testMapAccess", "python", se.compile(null, "obj1['l'][0]", Collections.emptyMap())), vars).run();
         assertThat(((String) o), equalTo("2"));
     }
 
@@ -85,7 +85,7 @@ public class PythonScriptEngineTests extends ESTestCase {
         vars.put("ctx", ctx);
 
         ExecutableScript executable = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testObjectInterMap", "python",
-                se.compile("ctx['obj2'] = { 'prop2' : 'value2' }; ctx['obj1']['prop1'] = 'uvalue1'", Collections.emptyMap())), vars);
+                se.compile(null, "ctx['obj2'] = { 'prop2' : 'value2' }; ctx['obj1']['prop1'] = 'uvalue1'", Collections.emptyMap())), vars);
         executable.run();
         ctx = (Map<String, Object>) executable.unwrap(vars.get("ctx"));
         assertThat(ctx.containsKey("obj1"), equalTo(true));
@@ -104,15 +104,15 @@ public class PythonScriptEngineTests extends ESTestCase {
 //        Object o = se.execute(se.compile("l.length"), vars);
 //        assertThat(((Number) o).intValue(), equalTo(4));
 
-        Object o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testAccessListInScript", "python", se.compile("l[0]", Collections.emptyMap())), vars).run();
+        Object o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testAccessListInScript", "python", se.compile(null, "l[0]", Collections.emptyMap())), vars).run();
         assertThat(((String) o), equalTo("1"));
 
-        o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testAccessListInScript", "python", se.compile("l[3]", Collections.emptyMap())), vars).run();
+        o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testAccessListInScript", "python", se.compile(null, "l[3]", Collections.emptyMap())), vars).run();
         obj1 = (Map<String, Object>) o;
         assertThat((String) obj1.get("prop1"), equalTo("value1"));
         assertThat((String) ((Map<String, Object>) obj1.get("obj2")).get("prop2"), equalTo("value2"));
 
-        o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testAccessListInScript", "python", se.compile("l[3]['prop1']", Collections.emptyMap())), vars).run();
+        o = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testAccessListInScript", "python", se.compile(null, "l[3]['prop1']", Collections.emptyMap())), vars).run();
         assertThat(((String) o), equalTo("value1"));
     }
 
@@ -120,7 +120,7 @@ public class PythonScriptEngineTests extends ESTestCase {
         Map<String, Object> vars = new HashMap<String, Object>();
         Map<String, Object> ctx = new HashMap<String, Object>();
         vars.put("ctx", ctx);
-        Object compiledScript = se.compile("ctx['value']", Collections.emptyMap());
+        Object compiledScript = se.compile(null, "ctx['value']", Collections.emptyMap());
 
         ExecutableScript script = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testChangingVarsCrossExecution1", "python", compiledScript), vars);
         ctx.put("value", 1);
@@ -135,7 +135,7 @@ public class PythonScriptEngineTests extends ESTestCase {
     public void testChangingVarsCrossExecution2() {
         Map<String, Object> vars = new HashMap<String, Object>();
         Map<String, Object> ctx = new HashMap<String, Object>();
-        Object compiledScript = se.compile("value", Collections.emptyMap());
+        Object compiledScript = se.compile(null, "value", Collections.emptyMap());
 
         ExecutableScript script = se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testChangingVarsCrossExecution2", "python", compiledScript), vars);
         script.setNextVar("value", 1);
diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTests.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTests.java
index f24cc889a70..abf9f661a6d 100644
--- a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTests.java
+++ b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTests.java
@@ -42,7 +42,7 @@ public class PythonScriptMultiThreadedTests extends ESTestCase {
 
     public void testExecutableNoRuntimeParams() throws Exception {
         final PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS);
-        final Object compiled = se.compile("x + y", Collections.emptyMap());
+        final Object compiled = se.compile(null, "x + y", Collections.emptyMap());
         final CompiledScript compiledScript = new CompiledScript(ScriptService.ScriptType.INLINE, "testExecutableNoRuntimeParams", "python", compiled);
         final AtomicBoolean failed = new AtomicBoolean();
 
@@ -128,7 +128,7 @@ public class PythonScriptMultiThreadedTests extends ESTestCase {
 
     public void testExecute() throws Exception {
         final PythonScriptEngineService se = new PythonScriptEngineService(Settings.Builder.EMPTY_SETTINGS);
-        final Object compiled = se.compile("x + y", Collections.emptyMap());
+        final Object compiled = se.compile(null, "x + y", Collections.emptyMap());
         final CompiledScript compiledScript = new CompiledScript(ScriptService.ScriptType.INLINE, "testExecute", "python", compiled);
         final AtomicBoolean failed = new AtomicBoolean();
 
diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonSecurityTests.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonSecurityTests.java
index 6e07daa6d0e..7ad13b31986 100644
--- a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonSecurityTests.java
+++ b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonSecurityTests.java
@@ -55,7 +55,7 @@ public class PythonSecurityTests extends ESTestCase {
     /** runs a script */
     private void doTest(String script) {
         Map<String, Object> vars = new HashMap<String, Object>();
-        se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "test", "python", se.compile(script, Collections.emptyMap())), vars).run();
+        se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "test", "python", se.compile(null, script, Collections.emptyMap())), vars).run();
     }
 
     /** asserts that a script runs without exception */
diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
index d0765d6627c..309b6622156 100644
--- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
+++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
@@ -32,12 +32,27 @@ import java.util.List;
 import java.util.Map;
 
 /**
- * A dummy script engine used for testing. Scripts must be a number. Running the script
+ * A dummy script engine used for testing. Scripts must be a number. Many 
+ * tests rely on the fact this thing returns a String as its compiled form.
+ * they even try to serialize it over the network!
  */
 public class MockScriptEngine implements ScriptEngineService {
 
     public static final String NAME = "mockscript";
 
+    /** A compiled script, just holds the scripts name, source, and params that were passed in */
+    public static class MockCompiledScript {
+        public final String name;
+        public final String source;
+        public final Map<String,String> params;
+
+        MockCompiledScript(String name, String source, Map<String,String> params) {
+            this.name = name;
+            this.source = source;
+            this.params = params;
+        }
+    }
+    
     public static class TestPlugin extends Plugin {
 
         public TestPlugin() {
@@ -71,16 +86,18 @@ public class MockScriptEngine implements ScriptEngineService {
     }
 
     @Override
-    public Object compile(String script, Map<String, String> params) {
-        return script;
+    public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
+        return new MockCompiledScript(scriptName, scriptSource, params);
     }
 
     @Override
     public ExecutableScript executable(CompiledScript compiledScript, @Nullable Map<String, Object> vars) {
+        assert compiledScript.compiled() instanceof MockCompiledScript 
+          : "do NOT pass compiled scripts from other engines to me, I will fail your test, got: " + compiledScript;
         return new AbstractExecutableScript() {
             @Override
             public Object run() {
-                return new BytesArray((String)compiledScript.compiled());
+                return new BytesArray(((MockCompiledScript)compiledScript.compiled()).source);
             }
         };
     }
@@ -94,7 +111,7 @@ public class MockScriptEngine implements ScriptEngineService {
 
                     @Override
                     public Object run() {
-                        return compiledScript.compiled();
+                        return ((MockCompiledScript)compiledScript.compiled()).source;
                     }
 
                 };

From 81a521fcb5661aa0cb8166ef00485671973bb862 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Fri, 13 May 2016 13:57:08 -0700
Subject: [PATCH 0241/1311] Build: Add developer info to generated pom files

---
 build.gradle | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/build.gradle b/build.gradle
index 459c8c8ae2e..6484a356857 100644
--- a/build.gradle
+++ b/build.gradle
@@ -42,6 +42,10 @@ subprojects {
             license.appendNode('name', 'The Apache Software License, Version 2.0')
             license.appendNode('url', 'http://www.apache.org/licenses/LICENSE-2.0.txt')
             license.appendNode('distribution', 'repo')
+
+            Node developer = node.appendNode('developers').appendNode('developer')
+            developer.appendNode('name', 'Elastic')
+            developer.appendNode('url', 'http://www.elastic.co')
           }
         }
       }

From 1b10a14e8d26dcc197a359cb142a1e4eee8df87c Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Fri, 13 May 2016 22:47:18 +0200
Subject: [PATCH 0242/1311] painless: build descriptor of array and field
 load/store in code; fix array index to adapt type not DEF

---
 .../org/elasticsearch/painless/WriterConstants.java |  9 ---------
 .../org/elasticsearch/painless/node/LDefArray.java  | 13 +++++++------
 .../org/elasticsearch/painless/node/LDefField.java  |  9 +++++----
 3 files changed, 12 insertions(+), 19 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index 8ad57c8fd8d..98aee5b7d90 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -61,15 +61,6 @@ public final class WriterConstants {
         new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(DefBootstrap.class),
             "bootstrap", DEF_BOOTSTRAP_TYPE.toMethodDescriptorString());
 
-    public final static String DEF_DYNAMIC_LOAD_FIELD_DESC =
-        MethodType.methodType(Object.class, Object.class).toMethodDescriptorString();
-    public final static String DEF_DYNAMIC_STORE_FIELD_DESC =
-        MethodType.methodType(void.class, Object.class, Object.class).toMethodDescriptorString();
-    public final static String DEF_DYNAMIC_ARRAY_LOAD_DESC =
-        MethodType.methodType(Object.class, Object.class, Object.class).toMethodDescriptorString();
-    public final static String DEF_DYNAMIC_ARRAY_STORE_DESC =
-        MethodType.methodType(void.class, Object.class, Object.class, Object.class).toMethodDescriptorString();
-
     public final static Method DEF_NOT_CALL = getAsmMethod(Object.class, "not", Object.class);
     public final static Method DEF_NEG_CALL = getAsmMethod(Object.class, "neg", Object.class);
     public final static Method DEF_MUL_CALL = getAsmMethod(Object.class, "mul", Object.class, Object.class);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index 6b701a77728..48e39457df9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -23,11 +23,10 @@ import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.Type;
 import org.objectweb.asm.commons.GeneratorAdapter;
 
 import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
-import static org.elasticsearch.painless.WriterConstants.DEF_DYNAMIC_ARRAY_LOAD_DESC;
-import static org.elasticsearch.painless.WriterConstants.DEF_DYNAMIC_ARRAY_STORE_DESC;
 
 /**
  * Represents an array load/store or shortcut on a def type.  (Internal only.)
@@ -45,8 +44,8 @@ final class LDefArray extends ALink {
 
     @Override
     ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        index.expected = definition.objectType;
         index.analyze(settings, definition, variables);
+        index.expected = index.actual;
         index = index.cast(settings, definition, variables);
 
         after = definition.defType;
@@ -61,14 +60,16 @@ final class LDefArray extends ALink {
 
     @Override
     void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        final String desc = Type.getMethodDescriptor(definition.defType.type, definition.defType.type, index.actual.type);
         adapter.visitInvokeDynamicInsn(
-            "arrayLoad", DEF_DYNAMIC_ARRAY_LOAD_DESC, DEF_BOOTSTRAP_HANDLE, new Object[] { DefBootstrap.ARRAY_LOAD });
-
+            "arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, new Object[] { DefBootstrap.ARRAY_LOAD });
     }
 
     @Override
     void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        final String desc = Type.getMethodDescriptor(definition.voidType.type, definition.defType.type,
+            index.actual.type, definition.defType.type);
         adapter.visitInvokeDynamicInsn(
-            "arrayStore", DEF_DYNAMIC_ARRAY_STORE_DESC, DEF_BOOTSTRAP_HANDLE, new Object[] { DefBootstrap.ARRAY_STORE });
+            "arrayStore", desc, DEF_BOOTSTRAP_HANDLE, new Object[] { DefBootstrap.ARRAY_STORE });
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index f8b2f3be83a..b5d7bb8b37b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -23,11 +23,10 @@ import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.Type;
 import org.objectweb.asm.commons.GeneratorAdapter;
 
 import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
-import static org.elasticsearch.painless.WriterConstants.DEF_DYNAMIC_LOAD_FIELD_DESC;
-import static org.elasticsearch.painless.WriterConstants.DEF_DYNAMIC_STORE_FIELD_DESC;
 
 /**
  * Represents a field load/store or shortcut on a def type.  (Internal only.)
@@ -57,11 +56,13 @@ final class LDefField extends ALink {
 
     @Override
     void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
-        adapter.visitInvokeDynamicInsn(value, DEF_DYNAMIC_LOAD_FIELD_DESC, DEF_BOOTSTRAP_HANDLE, new Object[] { DefBootstrap.LOAD });
+        final String desc = Type.getMethodDescriptor(definition.defType.type, definition.defType.type);
+        adapter.visitInvokeDynamicInsn(value, desc, DEF_BOOTSTRAP_HANDLE, new Object[] { DefBootstrap.LOAD });
     }
 
     @Override
     void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
-        adapter.visitInvokeDynamicInsn(value, DEF_DYNAMIC_STORE_FIELD_DESC, DEF_BOOTSTRAP_HANDLE, new Object[] { DefBootstrap.STORE });
+        final String desc = Type.getMethodDescriptor(definition.voidType.type, definition.defType.type, definition.defType.type);
+        adapter.visitInvokeDynamicInsn(value, desc, DEF_BOOTSTRAP_HANDLE, new Object[] { DefBootstrap.STORE });
     }
 }

From 1a76329d8987974b32a903f500cbc6ae8d29f113 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Fri, 13 May 2016 23:17:22 +0200
Subject: [PATCH 0243/1311] Take return type from "after" field.

---
 .../main/java/org/elasticsearch/painless/node/LDefArray.java    | 2 +-
 .../src/main/java/org/elasticsearch/painless/node/LDefCall.java | 2 +-
 .../main/java/org/elasticsearch/painless/node/LDefField.java    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index 48e39457df9..41eac59e4ff 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -60,7 +60,7 @@ final class LDefArray extends ALink {
 
     @Override
     void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
-        final String desc = Type.getMethodDescriptor(definition.defType.type, definition.defType.type, index.actual.type);
+        final String desc = Type.getMethodDescriptor(after.type, definition.defType.type, index.actual.type);
         adapter.visitInvokeDynamicInsn(
             "arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, new Object[] { DefBootstrap.ARRAY_LOAD });
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index dae2b74160a..eef3c835e45 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -82,7 +82,7 @@ final class LDefCall extends ALink {
 
         signature.append(')');
         // return value
-        signature.append(definition.defType.type.getDescriptor());
+        signature.append(after.type.getDescriptor());
 
         adapter.visitInvokeDynamicInsn(name, signature.toString(), DEF_BOOTSTRAP_HANDLE, new Object[] { DefBootstrap.METHOD_CALL });
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index b5d7bb8b37b..cb1c37a7b01 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -56,7 +56,7 @@ final class LDefField extends ALink {
 
     @Override
     void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
-        final String desc = Type.getMethodDescriptor(definition.defType.type, definition.defType.type);
+        final String desc = Type.getMethodDescriptor(after.type, definition.defType.type);
         adapter.visitInvokeDynamicInsn(value, desc, DEF_BOOTSTRAP_HANDLE, new Object[] { DefBootstrap.LOAD });
     }
 

From ba9dae820afe2f0f7d9f25b814ccc5959719e77f Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Fri, 13 May 2016 23:22:41 +0200
Subject: [PATCH 0244/1311] Add test for wrong array index

---
 .../elasticsearch/painless/WhenThingsGoWrongTests.java | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
index f4a092a5145..905766c781c 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
@@ -175,9 +175,15 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
         });
     }
 
-    public void testDynamicWrongArgs() {
+    public void testDynamicArrayWrongIndex() {
         expectThrows(WrongMethodTypeException.class, () -> {
-            exec("def x = new ArrayList(); return x.get('bogus');");
+            exec("def x = new long[1]; x[0]=1; return x['bogus'];");
+        });
+    }
+
+    public void testDynamicListWrongIndex() {
+        expectThrows(WrongMethodTypeException.class, () -> {
+            exec("def x = new ArrayList(); x.add('foo'); return x['bogus'];");
         });
     }
 }

From 5aea24137cdcfa6e0b24a54a11c586a7ff9e9e0b Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Fri, 13 May 2016 16:32:35 -0700
Subject: [PATCH 0245/1311] Build: More pom generation improvements

This adds a non empty description to all generated poms, as well as
fixing distributions so they actually have a generated pom.
---
 build.gradle                                               | 1 +
 .../groovy/org/elasticsearch/gradle/BuildPlugin.groovy     | 6 +++---
 .../elasticsearch/gradle/plugin/PluginBuildPlugin.groovy   | 7 ++++---
 distribution/build.gradle                                  | 4 ++++
 4 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/build.gradle b/build.gradle
index 6484a356857..3d52d4ab279 100644
--- a/build.gradle
+++ b/build.gradle
@@ -27,6 +27,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
 subprojects {
   group = 'org.elasticsearch'
   version = org.elasticsearch.gradle.VersionProperties.elasticsearch
+  description = "Elasticsearch subproject ${project.path}"
 
   // we only use maven publish to add tasks for pom generation
   plugins.withType(MavenPublishPlugin).whenPluginAdded {
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index 2cfd98f276f..029c80b6e25 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -19,6 +19,7 @@
 package org.elasticsearch.gradle
 
 import nebula.plugin.extraconfigurations.ProvidedBasePlugin
+import nebula.plugin.publishing.maven.MavenBasePublishPlugin
 import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.gradle.api.GradleException
 import org.gradle.api.JavaVersion
@@ -34,7 +35,6 @@ import org.gradle.api.artifacts.ResolvedArtifact
 import org.gradle.api.artifacts.dsl.RepositoryHandler
 import org.gradle.api.artifacts.maven.MavenPom
 import org.gradle.api.publish.maven.MavenPublication
-import org.gradle.api.publish.maven.plugins.MavenPublishPlugin
 import org.gradle.api.publish.maven.tasks.GenerateMavenPom
 import org.gradle.api.tasks.bundling.Jar
 import org.gradle.api.tasks.compile.JavaCompile
@@ -343,8 +343,8 @@ class BuildPlugin implements Plugin<Project> {
     }
 
     /**Configuration generation of maven poms. */
-    private static void configurePomGeneration(Project project) {
-        project.plugins.withType(MavenPublishPlugin.class).whenPluginAdded {
+    public static void configurePomGeneration(Project project) {
+        project.plugins.withType(MavenBasePublishPlugin.class).whenPluginAdded {
             project.publishing {
                 publications {
                     all { MavenPublication publication -> // we only deal with maven
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
index 36770ab35f7..7ff83f4eae3 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.gradle.plugin
 
+import nebula.plugin.publishing.maven.MavenBasePublishPlugin
 import nebula.plugin.publishing.maven.MavenManifestPlugin
 import nebula.plugin.publishing.maven.MavenScmPlugin
 import org.elasticsearch.gradle.BuildPlugin
@@ -51,7 +52,7 @@ public class PluginBuildPlugin extends BuildPlugin {
             } else {
                 project.integTest.clusterConfig.plugin(name, project.bundlePlugin.outputs.files)
                 project.tasks.run.clusterConfig.plugin(name, project.bundlePlugin.outputs.files)
-                configurePomGeneration(project)
+                addPomGeneration(project)
             }
 
             project.namingConventions {
@@ -131,9 +132,9 @@ public class PluginBuildPlugin extends BuildPlugin {
     /**
      * Adds the plugin jar and zip as publications.
      */
-    private static void configurePomGeneration(Project project) {
+    protected static void addPomGeneration(Project project) {
+        project.plugins.apply(MavenBasePublishPlugin.class)
         project.plugins.apply(MavenScmPlugin.class)
-        project.plugins.apply(MavenManifestPlugin.class)
 
         project.publishing {
             publications {
diff --git a/distribution/build.gradle b/distribution/build.gradle
index 44949cd0604..40fb6b64f99 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -20,6 +20,7 @@
 
 import org.apache.tools.ant.filters.FixCrLfFilter
 import org.apache.tools.ant.taskdefs.condition.Os
+import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.EmptyDirTask
 import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.precommit.DependencyLicensesTask
@@ -114,6 +115,7 @@ subprojects {
    *****************************************************************************/
   // note: the group must be correct before applying the nexus plugin, or it will capture the wrong value...
   project.group = "org.elasticsearch.distribution.${project.name}"
+  project.archivesBaseName = 'elasticsearch'
   apply plugin: 'com.bmuschko.nexus'
   // we must create our own install task, because it is only added when the java plugin is added
   task install(type: Upload, description: "Installs the 'archives' artifacts into the local Maven repository.", group: 'Upload') {
@@ -162,6 +164,8 @@ subprojects {
    /*****************************************************************************
    *                           Publishing setup                                *
    *****************************************************************************/
+  BuildPlugin.configurePomGeneration(project)
+  apply plugin: 'nebula.info-scm'
   apply plugin: 'nebula.maven-base-publish'
   apply plugin: 'nebula.maven-scm'
   publishing {

From d6f3d85925538bbbb466f5ed4858527d5e21f0bb Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Fri, 13 May 2016 16:36:26 -0700
Subject: [PATCH 0246/1311] Build: Make run task you full zip distribution

This change makes `gradle run` use the full zip distribution. Arguably
we could make it do the same when run inside plugins, but I started out
with this simple change. I am open to moving it into the RunTask itself
so that plugins will do this as well.
---
 distribution/build.gradle | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/distribution/build.gradle b/distribution/build.gradle
index 44949cd0604..2bb12db1413 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -409,7 +409,9 @@ task updateShas(type: UpdateShasTask) {
   parentTask = dependencyLicenses
 }
 
-task run(type: RunTask) {}
+task run(type: RunTask) {
+  distribution = 'zip'
+}
 
 /**
  * Build some variables that are replaced in the packages. This includes both

From a2381640da954d431485ed66f1acac9309206977 Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@elastic.co>
Date: Sat, 14 May 2016 09:59:53 +0200
Subject: [PATCH 0247/1311] Add not-null precondition check in BulkRequest

With this commit we add a precondition check to BulkRequest so
we fail early if users pass `null` for the request object.

For a more detailed discussion, see #12038.
This supersedes #12038.

Relates #12038.
---
 .../main/java/org/elasticsearch/action/bulk/BulkRequest.java   | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
index f008bf9a4e8..56164e3877e 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
@@ -43,6 +43,7 @@ import org.elasticsearch.index.VersionType;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
@@ -125,6 +126,7 @@ public class BulkRequest extends ActionRequest<BulkRequest> implements Composite
     }
 
     BulkRequest internalAdd(IndexRequest request, @Nullable Object payload) {
+        Objects.requireNonNull(request, "'request' must not be null");
         requests.add(request);
         addPayload(payload);
         // lack of source is validated in validate() method
@@ -144,6 +146,7 @@ public class BulkRequest extends ActionRequest<BulkRequest> implements Composite
     }
 
     BulkRequest internalAdd(UpdateRequest request, @Nullable Object payload) {
+        Objects.requireNonNull(request, "'request' must not be null");
         requests.add(request);
         addPayload(payload);
         if (request.doc() != null) {

From bfc826003bf7db97f852c8f74e74bc11946917b6 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Sat, 14 May 2016 11:20:04 +0200
Subject: [PATCH 0248/1311] Documented fuzzy_transpositions in match query

Relates to #18320
---
 docs/reference/query-dsl/match-query.asciidoc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/reference/query-dsl/match-query.asciidoc b/docs/reference/query-dsl/match-query.asciidoc
index 90936f024f6..3dbb60d1ca2 100644
--- a/docs/reference/query-dsl/match-query.asciidoc
+++ b/docs/reference/query-dsl/match-query.asciidoc
@@ -49,6 +49,9 @@ as its <<query-dsl-multi-term-rewrite,rewrite
 method>> the `fuzzy_rewrite` parameter allows to control how the query will get
 rewritten.
 
+Fuzzy transpositions (`ab` -> `ba`) are allowed by default but can be disabled
+by setting `fuzzy_transpositions` to `false`.
+
 Here is an example when providing additional parameters (note the slight
 change in structure, `message` is the field name):
 

From c8bd521ed3de788195680d97fa13896680050014 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Sat, 14 May 2016 12:00:30 +0200
Subject: [PATCH 0249/1311] painless: restore accidentally removed test

---
 .../org/elasticsearch/painless/WhenThingsGoWrongTests.java  | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
index 905766c781c..55ee490bb3f 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
@@ -175,6 +175,12 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
         });
     }
 
+    public void testDynamicWrongArgs() {
+        expectThrows(WrongMethodTypeException.class, () -> {
+            exec("def x = new ArrayList(); return x.get('bogus');");
+        });
+    }
+
     public void testDynamicArrayWrongIndex() {
         expectThrows(WrongMethodTypeException.class, () -> {
             exec("def x = new long[1]; x[0]=1; return x['bogus'];");

From 56a07f70f7ab41efde1990a7533a2e2df5f22d25 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Sat, 14 May 2016 07:29:55 -0400
Subject: [PATCH 0250/1311] Add note regarding thread stack size on Windows

This commit adds a note to the Windows service docs regarding the thread
stack size setting for the Windows service installer. As the Apache
Commons procrun daemon requires that this setting be explicitly set, we
need a value to be set when the service is installed. The right place
for this setting is the jvm.options file. We do not want to ship with a
hard-coded value here because we do not want to override the default
setting on other platforms, and the right default depends on whether or
not the end-user is on a 32-bit versus a 64-bit Windows system.

Relates #18324
---
 docs/reference/setup/install/windows.asciidoc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/reference/setup/install/windows.asciidoc b/docs/reference/setup/install/windows.asciidoc
index 3e297a9b487..e3c7622d064 100644
--- a/docs/reference/setup/install/windows.asciidoc
+++ b/docs/reference/setup/install/windows.asciidoc
@@ -106,6 +106,11 @@ Using JAVA_HOME (64-bit):  "c:\jvm\jdk1.8"
 The service 'elasticsearch-service-x64' has been installed.
 --------------------------------------------------
 
+NOTE: The service installer requires that the thread stack size setting
+be configured in jvm.options *before* you install the service. On
+32-bit Windows, you should add `-Xss320k` to the jvm.options file, and
+on 64-bit Windows you should add `-Xss1m` to the jvm.options file.
+
 NOTE: While a JRE can be used for the Elasticsearch service, due to its use of a client VM (as opposed to a server JVM which offers better performance for long-running applications) its usage is discouraged and a warning will be issued.
 
 NOTE: Upgrading (or downgrading) JVM versions does not require the service to be reinstalled. However, upgrading across JVM types (e.g. JRE versus SE) is not supported, and does require the service to be reinstalled.

From d3d57da89ff3ae4071f93af97c1f71e7b8d3364a Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Sat, 14 May 2016 00:47:43 -0400
Subject: [PATCH 0251/1311] Removes unused methods in the o/e/common/Strings
 class.

Closes #18346
---
 .../org/elasticsearch/common/Strings.java     | 86 -------------------
 1 file changed, 86 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/Strings.java b/core/src/main/java/org/elasticsearch/common/Strings.java
index 6b6c31c1522..63afe9a0323 100644
--- a/core/src/main/java/org/elasticsearch/common/Strings.java
+++ b/core/src/main/java/org/elasticsearch/common/Strings.java
@@ -37,7 +37,6 @@ import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Properties;
 import java.util.Set;
 import java.util.StringTokenizer;
 import java.util.TreeSet;
@@ -243,45 +242,6 @@ public class Strings {
         return hasText((CharSequence) str);
     }
 
-    /**
-     * Check whether the given CharSequence contains any whitespace characters.
-     *
-     * @param str the CharSequence to check (may be <code>null</code>)
-     * @return <code>true</code> if the CharSequence is not empty and
-     *         contains at least 1 whitespace character
-     * @see java.lang.Character#isWhitespace
-     */
-    public static boolean containsWhitespace(CharSequence str) {
-        if (!hasLength(str)) {
-            return false;
-        }
-        int strLen = str.length();
-        for (int i = 0; i < strLen; i++) {
-            if (Character.isWhitespace(str.charAt(i))) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    /**
-     * Trim leading whitespace from the given String.
-     *
-     * @param str the String to check
-     * @return the trimmed String
-     * @see java.lang.Character#isWhitespace
-     */
-    public static String trimLeadingWhitespace(String str) {
-        if (!hasLength(str)) {
-            return str;
-        }
-        StringBuilder sb = new StringBuilder(str);
-        while (sb.length() > 0 && Character.isWhitespace(sb.charAt(0))) {
-            sb.deleteCharAt(0);
-        }
-        return sb.toString();
-    }
-
     /**
      * Trim all occurrences of the supplied leading character from the given String.
      *
@@ -416,17 +376,6 @@ public class Strings {
         return (str != null ? "'" + str + "'" : null);
     }
 
-    /**
-     * Unqualify a string qualified by a separator character. For example,
-     * "this:name:is:qualified" returns "qualified" if using a ':' separator.
-     *
-     * @param qualifiedName the qualified name
-     * @param separator     the separator
-     */
-    public static String unqualify(String qualifiedName, char separator) {
-        return qualifiedName.substring(qualifiedName.lastIndexOf(separator) + 1);
-    }
-
     /**
      * Capitalize a <code>String</code>, changing the first letter to
      * upper case as per {@link Character#toUpperCase(char)}.
@@ -611,41 +560,6 @@ public class Strings {
         return new String[]{beforeDelimiter, afterDelimiter};
     }
 
-    /**
-     * Take an array Strings and split each element based on the given delimiter.
-     * A <code>Properties</code> instance is then generated, with the left of the
-     * delimiter providing the key, and the right of the delimiter providing the value.
-     * <p>Will trim both the key and value before adding them to the
-     * <code>Properties</code> instance.
-     *
-     * @param array         the array to process
-     * @param delimiter     to split each element using (typically the equals symbol)
-     * @param charsToDelete one or more characters to remove from each element
-     *                      prior to attempting the split operation (typically the quotation mark
-     *                      symbol), or <code>null</code> if no removal should occur
-     * @return a <code>Properties</code> instance representing the array contents,
-     *         or <code>null</code> if the array to process was <code>null</code> or empty
-     */
-    public static Properties splitArrayElementsIntoProperties(
-            String[] array, String delimiter, String charsToDelete) {
-
-        if (isEmpty(array)) {
-            return null;
-        }
-        Properties result = new Properties();
-        for (String element : array) {
-            if (charsToDelete != null) {
-                element = deleteAny(element, charsToDelete);
-            }
-            String[] splittedElement = split(element, delimiter);
-            if (splittedElement == null) {
-                continue;
-            }
-            result.setProperty(splittedElement[0].trim(), splittedElement[1].trim());
-        }
-        return result;
-    }
-
     /**
      * Tokenize the given String into a String array via a StringTokenizer.
      * Trims tokens and omits empty tokens.

From c78bf8c86738f3c0fc0f46107cbf06486f5bda17 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Sat, 14 May 2016 16:04:18 +0200
Subject: [PATCH 0252/1311] painless: Use isAssignableFrom instead of relying
 on ClassCastException

---
 .../org/elasticsearch/painless/node/LBrace.java    | 14 +++-----------
 .../org/elasticsearch/painless/node/LField.java    | 14 +++-----------
 .../org/elasticsearch/painless/node/STrap.java     |  4 +---
 3 files changed, 7 insertions(+), 25 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
index 5cfaeb7466a..c0f9ddebc68 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
@@ -60,20 +60,12 @@ public final class LBrace extends ALink {
         } else if (sort == Sort.DEF) {
             return new LDefArray(line, location, index).copy(this).analyze(settings, definition, variables);
         } else {
-            try {
-                before.clazz.asSubclass(Map.class);
-
+            if (Map.class.isAssignableFrom(before.clazz)) {
                 return new LMapShortcut(line, location, index).copy(this).analyze(settings, definition, variables);
-            } catch (final ClassCastException exception) {
-                // Do nothing.
             }
-
-            try {
-                before.clazz.asSubclass(List.class);
-
+            
+            if (List.class.isAssignableFrom(before.clazz)) {
                 return new LListShortcut(line, location, index).copy(this).analyze(settings, definition, variables);
-            } catch (final ClassCastException exception) {
-                // Do nothing.
             }
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
index 6e3aafc0516..a13991e87ba 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
@@ -82,20 +82,12 @@ public final class LField extends ALink {
                 final EConstant index = new EConstant(line, location, value);
                 index.analyze(settings, definition, variables);
 
-                try {
-                    before.clazz.asSubclass(Map.class);
-
+                if (Map.class.isAssignableFrom(before.clazz)) {
                     return new LMapShortcut(line, location, index).copy(this).analyze(settings, definition, variables);
-                } catch (final ClassCastException exception) {
-                    // Do nothing.
                 }
-
-                try {
-                    before.clazz.asSubclass(List.class);
-
+                
+                if (List.class.isAssignableFrom(before.clazz)) {
                     return new LListShortcut(line, location, index).copy(this).analyze(settings, definition, variables);
-                } catch (final ClassCastException exception) {
-                    // Do nothing.
                 }
             }
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java
index a7b7ce20177..1c3d2edc48f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java
@@ -54,9 +54,7 @@ public final class STrap extends AStatement {
     void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
         variable = variables.addVariable(location, type, name, true, false);
 
-        try {
-            variable.type.clazz.asSubclass(Exception.class);
-        } catch (final ClassCastException cce) {
+        if (!Exception.class.isAssignableFrom(variable.type.clazz)) {
             throw new ClassCastException(error("Not an exception type [" + variable.type.name + "]."));
         }
 

From 7211017813e56f4314fba09fe765a94748696af6 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Sat, 14 May 2016 17:23:29 +0200
Subject: [PATCH 0253/1311] painless: Some cleanups

---
 .../java/org/elasticsearch/painless/WriterConstants.java    | 5 +++--
 .../java/org/elasticsearch/painless/node/LDefArray.java     | 6 ++----
 .../main/java/org/elasticsearch/painless/node/LDefCall.java | 2 +-
 .../java/org/elasticsearch/painless/node/LDefField.java     | 4 ++--
 4 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index 98aee5b7d90..86111d8f1a5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -37,9 +37,10 @@ import java.util.Map;
 public final class WriterConstants {
 
     public final static String BASE_CLASS_NAME = Executable.class.getName();
-    public final static String CLASS_NAME      = BASE_CLASS_NAME + "$Script";
     public final static Type BASE_CLASS_TYPE   = Type.getType(Executable.class);
-    public final static Type CLASS_TYPE        = Type.getType("L" + CLASS_NAME.replace(".", "/") + ";");
+    
+    public final static String CLASS_NAME      = BASE_CLASS_NAME + "$Script";
+    public final static Type CLASS_TYPE        = Type.getObjectType(CLASS_NAME.replace('.', '/'));
 
     public final static Method CONSTRUCTOR = getAsmMethod(void.class, "<init>", String.class, String.class);
     public final static Method EXECUTE     =
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index 41eac59e4ff..38e8bf89cc9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -61,15 +61,13 @@ final class LDefArray extends ALink {
     @Override
     void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
         final String desc = Type.getMethodDescriptor(after.type, definition.defType.type, index.actual.type);
-        adapter.visitInvokeDynamicInsn(
-            "arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, new Object[] { DefBootstrap.ARRAY_LOAD });
+        adapter.visitInvokeDynamicInsn("arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_LOAD);
     }
 
     @Override
     void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
         final String desc = Type.getMethodDescriptor(definition.voidType.type, definition.defType.type,
             index.actual.type, definition.defType.type);
-        adapter.visitInvokeDynamicInsn(
-            "arrayStore", desc, DEF_BOOTSTRAP_HANDLE, new Object[] { DefBootstrap.ARRAY_STORE });
+        adapter.visitInvokeDynamicInsn("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_STORE);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index eef3c835e45..2bf53bcf89b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -84,7 +84,7 @@ final class LDefCall extends ALink {
         // return value
         signature.append(after.type.getDescriptor());
 
-        adapter.visitInvokeDynamicInsn(name, signature.toString(), DEF_BOOTSTRAP_HANDLE, new Object[] { DefBootstrap.METHOD_CALL });
+        adapter.visitInvokeDynamicInsn(name, signature.toString(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.METHOD_CALL);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index cb1c37a7b01..68fb7ffb95a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -57,12 +57,12 @@ final class LDefField extends ALink {
     @Override
     void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
         final String desc = Type.getMethodDescriptor(after.type, definition.defType.type);
-        adapter.visitInvokeDynamicInsn(value, desc, DEF_BOOTSTRAP_HANDLE, new Object[] { DefBootstrap.LOAD });
+        adapter.visitInvokeDynamicInsn(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.LOAD);
     }
 
     @Override
     void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
         final String desc = Type.getMethodDescriptor(definition.voidType.type, definition.defType.type, definition.defType.type);
-        adapter.visitInvokeDynamicInsn(value, desc, DEF_BOOTSTRAP_HANDLE, new Object[] { DefBootstrap.STORE });
+        adapter.visitInvokeDynamicInsn(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.STORE);
     }
 }

From cf2af8961b2a9fb4833425847aa72a324038b7c7 Mon Sep 17 00:00:00 2001
From: Mike McCandless <mikemccand@apache.org>
Date: Sat, 14 May 2016 13:39:48 -0400
Subject: [PATCH 0254/1311] Iterables.flatten should not pre-cache the first
 iterator

---
 .../common/util/iterable/Iterables.java       |  6 +---
 .../indices/IndexingMemoryController.java     |  2 +-
 .../common/util/iterable/IterablesTests.java  | 31 +++++++++++++++++--
 3 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java b/core/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java
index 654f3ae3d46..4299f161874 100644
--- a/core/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java
+++ b/core/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java
@@ -63,11 +63,7 @@ public class Iterables {
         private final Iterable<? extends Iterable<T>> inputs;
 
         FlattenedIterables(Iterable<? extends Iterable<T>> inputs) {
-            List<Iterable<T>> list = new ArrayList<>();
-            for (Iterable<T> iterable : inputs) {
-                list.add(iterable);
-            }
-            this.inputs = list;
+            this.inputs = inputs;
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java b/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
index b47172bb0ae..18bc77e69f1 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
@@ -93,7 +93,7 @@ public class IndexingMemoryController extends AbstractComponent implements Index
 
     private final ShardsIndicesStatusChecker statusChecker;
 
-    IndexingMemoryController(Settings settings, ThreadPool threadPool, Iterable<IndexShard>indexServices) {
+    IndexingMemoryController(Settings settings, ThreadPool threadPool, Iterable<IndexShard> indexServices) {
         this(settings, threadPool, indexServices, JvmInfo.jvmInfo().getMem().getHeapMax().bytes());
     }
 
diff --git a/core/src/test/java/org/elasticsearch/common/util/iterable/IterablesTests.java b/core/src/test/java/org/elasticsearch/common/util/iterable/IterablesTests.java
index 5ce816927ca..83694be2393 100644
--- a/core/src/test/java/org/elasticsearch/common/util/iterable/IterablesTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/iterable/IterablesTests.java
@@ -19,12 +19,14 @@
 
 package org.elasticsearch.common.util.iterable;
 
-import org.elasticsearch.test.ESTestCase;
-
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Iterator;
+import java.util.List;
 import java.util.NoSuchElementException;
 
+import org.elasticsearch.test.ESTestCase;
+
 import static org.hamcrest.object.HasToString.hasToString;
 
 public class IterablesTests extends ESTestCase {
@@ -56,6 +58,29 @@ public class IterablesTests extends ESTestCase {
         test(iterable);
     }
 
+    public void testFlatten() {
+        List<List<Integer>> list = new ArrayList<>();
+
+        Iterable<Integer> allInts = Iterables.flatten(list);
+        int count = 0;
+        for(int x : allInts) {
+            count++;
+        }
+        assertEquals(0, count);
+
+        list.add(new ArrayList<>());
+        for(int x : allInts) {
+            count++;
+        }
+        assertEquals(0, count);
+
+        list.get(0).add(0);
+        for(int x : allInts) {
+            count++;
+        }
+        assertEquals(1, count);
+    }
+
     private void test(Iterable<String> iterable) {
         try {
             Iterables.get(iterable, -1);
@@ -73,4 +98,4 @@ public class IterablesTests extends ESTestCase {
             assertThat(e, hasToString("java.lang.IndexOutOfBoundsException: 3"));
         }
     }
-}
\ No newline at end of file
+}

From 53c2f8b4b6ff030b0449469e0c3fa373be8fec0f Mon Sep 17 00:00:00 2001
From: Mike McCandless <mikemccand@apache.org>
Date: Sat, 14 May 2016 13:46:34 -0400
Subject: [PATCH 0255/1311] improve javadocs

---
 .../java/org/elasticsearch/common/util/iterable/Iterables.java  | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/core/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java b/core/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java
index 4299f161874..e15328089a3 100644
--- a/core/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java
+++ b/core/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java
@@ -54,6 +54,8 @@ public class Iterables {
         }
     }
 
+    /** Flattens the two level {@code Iterable} into a single {@code Iterable}.  Note that this uses the original input iterable so if it
+     *  later changes, the flattened result here will reflect the change. */
     public static <T> Iterable<T> flatten(Iterable<? extends Iterable<T>> inputs) {
         Objects.requireNonNull(inputs);
         return new FlattenedIterables<>(inputs);

From 48dca455646f886ea15363ab0f9491ca6a705384 Mon Sep 17 00:00:00 2001
From: Mike McCandless <mikemccand@apache.org>
Date: Sat, 14 May 2016 17:10:17 -0400
Subject: [PATCH 0256/1311] leave Iterables.flatten pre-caching the outer
 Iterable

---
 .../elasticsearch/common/util/iterable/Iterables.java  | 10 +++++++---
 .../java/org/elasticsearch/indices/IndicesService.java |  4 +++-
 .../common/util/iterable/IterablesTests.java           |  7 ++++++-
 3 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java b/core/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java
index e15328089a3..2852e33eb43 100644
--- a/core/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java
+++ b/core/src/main/java/org/elasticsearch/common/util/iterable/Iterables.java
@@ -54,8 +54,8 @@ public class Iterables {
         }
     }
 
-    /** Flattens the two level {@code Iterable} into a single {@code Iterable}.  Note that this uses the original input iterable so if it
-     *  later changes, the flattened result here will reflect the change. */
+    /** Flattens the two level {@code Iterable} into a single {@code Iterable}.  Note that this pre-caches the values from the outer {@code
+     *  Iterable}, but not the values from the inner one. */
     public static <T> Iterable<T> flatten(Iterable<? extends Iterable<T>> inputs) {
         Objects.requireNonNull(inputs);
         return new FlattenedIterables<>(inputs);
@@ -65,7 +65,11 @@ public class Iterables {
         private final Iterable<? extends Iterable<T>> inputs;
 
         FlattenedIterables(Iterable<? extends Iterable<T>> inputs) {
-            this.inputs = inputs;
+            List<Iterable<T>> list = new ArrayList<>();
+            for (Iterable<T> iterable : inputs) {
+                list.add(iterable);
+            }
+            this.inputs = list;
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index 9375a11c242..3b9b186ddd8 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -181,7 +181,9 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
         this.namedWriteableRegistry = namedWriteableRegistry;
         clusterSettings.addSettingsUpdateConsumer(IndexStoreConfig.INDICES_STORE_THROTTLE_TYPE_SETTING, indexStoreConfig::setRateLimitingType);
         clusterSettings.addSettingsUpdateConsumer(IndexStoreConfig.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING, indexStoreConfig::setRateLimitingThrottle);
-        indexingMemoryController = new IndexingMemoryController(settings, threadPool, Iterables.flatten(this));
+        indexingMemoryController = new IndexingMemoryController(settings, threadPool,
+                                                                // ensure we pull an iter with new shards - flatten makes a copy
+                                                                () -> Iterables.flatten(this).iterator());
         this.indexScopeSetting = indexScopedSettings;
         this.circuitBreakerService = circuitBreakerService;
         this.indicesFieldDataCache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() {
diff --git a/core/src/test/java/org/elasticsearch/common/util/iterable/IterablesTests.java b/core/src/test/java/org/elasticsearch/common/util/iterable/IterablesTests.java
index 83694be2393..4d9d89312a3 100644
--- a/core/src/test/java/org/elasticsearch/common/util/iterable/IterablesTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/iterable/IterablesTests.java
@@ -60,6 +60,7 @@ public class IterablesTests extends ESTestCase {
 
     public void testFlatten() {
         List<List<Integer>> list = new ArrayList<>();
+        list.add(new ArrayList<>());
 
         Iterable<Integer> allInts = Iterables.flatten(list);
         int count = 0;
@@ -67,13 +68,17 @@ public class IterablesTests extends ESTestCase {
             count++;
         }
         assertEquals(0, count);
-
         list.add(new ArrayList<>());
+        list.get(1).add(0);
+
+        // changes to the outer list are not seen since flatten pre-caches outer list on init:
+        count = 0;
         for(int x : allInts) {
             count++;
         }
         assertEquals(0, count);
 
+        // but changes to the original inner lists are seen:
         list.get(0).add(0);
         for(int x : allInts) {
             count++;

From ded8b400b0aaca4044fcb773c94a6ca354498d60 Mon Sep 17 00:00:00 2001
From: Mike McCandless <mikemccand@apache.org>
Date: Sat, 14 May 2016 18:47:26 -0400
Subject: [PATCH 0257/1311] Fix concurrency bug in IMC that could lead to
 negative total indexing bytes

---
 .../indices/IndexingMemoryController.java     | 24 ++++++++++++-------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java b/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
index b47172bb0ae..e9cd351b6e5 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
@@ -251,21 +251,29 @@ public class IndexingMemoryController extends AbstractComponent implements Index
         /** Shard calls this on each indexing/delete op */
         public void bytesWritten(int bytes) {
             long totalBytes = bytesWrittenSinceCheck.addAndGet(bytes);
+            assert totalBytes >= 0      ;
             while (totalBytes > indexingBuffer.bytes()/30) {
+
                 if (runLock.tryLock()) {
                     try {
-                        bytesWrittenSinceCheck.addAndGet(-totalBytes);
-                        // NOTE: this is only an approximate check, because bytes written is to the translog, vs indexing memory buffer which is
-                        // typically smaller but can be larger in extreme cases (many unique terms).  This logic is here only as a safety against
-                        // thread starvation or too infrequent checking, to ensure we are still checking periodically, in proportion to bytes
-                        // processed by indexing:
-                        runUnlocked();
+                        // Must pull this again because it may have changed since we first checked:
+                        totalBytes = bytesWrittenSinceCheck.get();
+                        if (totalBytes > indexingBuffer.bytes()/30) {
+                            bytesWrittenSinceCheck.addAndGet(-totalBytes);
+                            // NOTE: this is only an approximate check, because bytes written is to the translog, vs indexing memory buffer which is
+                            // typically smaller but can be larger in extreme cases (many unique terms).  This logic is here only as a safety against
+                            // thread starvation or too infrequent checking, to ensure we are still checking periodically, in proportion to bytes
+                            // processed by indexing:
+                            runUnlocked();
+                        }
                     } finally {
                         runLock.unlock();
                     }
-                    // Could be while we were checking, more bytes arrived:
-                    totalBytes = bytesWrittenSinceCheck.addAndGet(bytes);
+
+                    // Must get it again since other threads could have increased it while we were in runUnlocked
+                    totalBytes = bytesWrittenSinceCheck.get();
                 } else {
+                    // Another thread beat us to it: let them do all the work, yay!
                     break;
                 }
             }

From 8d7db7fd7acb4d0ee79ba9974b3774404dac4917 Mon Sep 17 00:00:00 2001
From: Mike McCandless <mikemccand@apache.org>
Date: Sat, 14 May 2016 18:50:10 -0400
Subject: [PATCH 0258/1311] remove whitespace

---
 .../org/elasticsearch/indices/IndexingMemoryController.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java b/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
index e9cd351b6e5..28345990019 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
@@ -251,7 +251,7 @@ public class IndexingMemoryController extends AbstractComponent implements Index
         /** Shard calls this on each indexing/delete op */
         public void bytesWritten(int bytes) {
             long totalBytes = bytesWrittenSinceCheck.addAndGet(bytes);
-            assert totalBytes >= 0      ;
+            assert totalBytes >= 0;
             while (totalBytes > indexingBuffer.bytes()/30) {
 
                 if (runLock.tryLock()) {

From 8195ef9e7a574a8b2e62d1fda3c0fe6a3249d21d Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Sat, 14 May 2016 18:00:58 +0200
Subject: [PATCH 0259/1311] painless: Transform more
 try-catch-ClassCastException to isAssignableFrom

---
 .../painless/AnalyzerCaster.java              | 19 +------
 .../elasticsearch/painless/Definition.java    | 56 ++++++-------------
 .../elasticsearch/painless/WriterUtility.java |  4 +-
 .../elasticsearch/painless/node/LBrace.java   | 12 ++--
 4 files changed, 25 insertions(+), 66 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
index 5042608d063..d7275909858 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
@@ -467,23 +467,10 @@ public final class AnalyzerCaster {
                 }
         }
 
-        try {
-            actual.clazz.asSubclass(expected.clazz);
-
+        if (expected.clazz.isAssignableFrom(actual.clazz) || (explicit && actual.clazz.isAssignableFrom(expected.clazz))) {
             return cast;
-        } catch (final ClassCastException cce0) {
-            try {
-                if (explicit) {
-                    expected.clazz.asSubclass(actual.clazz);
-
-                    return cast;
-                } else {
-                    throw new ClassCastException(
-                        "Error" + location + ": Cannot cast from [" + actual.name + "] to [" + expected.name + "].");
-                }
-            } catch (final ClassCastException cce1) {
-                throw new ClassCastException("Error" + location + ": Cannot cast from [" + actual.name + "] to [" + expected.name + "].");
-            }
+        } else {
+            throw new ClassCastException("Error" + location + ": Cannot cast from [" + actual.name + "] to [" + expected.name + "].");
         }
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index e2f312be7b4..ae70fef59ee 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -1481,9 +1481,7 @@ public final class Definition {
 
         for (int count = 0; count < classes.length; ++count) {
             if (genargs != null) {
-                try {
-                    genargs[count].clazz.asSubclass(args[count].clazz);
-                } catch (final ClassCastException exception) {
+                if (!args[count].clazz.isAssignableFrom(genargs[count].clazz)) {
                     throw new ClassCastException("Generic argument [" + genargs[count].name + "]" +
                         " is not a sub class of [" + args[count].name + "] in the constructor" +
                         " [" + name + " ] from the struct [" + owner.name + "].");
@@ -1550,9 +1548,7 @@ public final class Definition {
         }
 
         if (genrtn != null) {
-            try {
-                genrtn.clazz.asSubclass(rtn.clazz);
-            } catch (final ClassCastException exception) {
+            if (!rtn.clazz.isAssignableFrom(genrtn.clazz)) {
                 throw new ClassCastException("Generic return [" + genrtn.clazz.getCanonicalName() + "]" +
                     " is not a sub class of [" + rtn.clazz.getCanonicalName() + "] in the method" +
                     " [" + name + " ] from the struct [" + owner.name + "].");
@@ -1569,9 +1565,7 @@ public final class Definition {
 
         for (int count = 0; count < classes.length; ++count) {
             if (genargs != null) {
-                try {
-                    genargs[count].clazz.asSubclass(args[count].clazz);
-                } catch (final ClassCastException exception) {
+                if (!args[count].clazz.isAssignableFrom(genargs[count].clazz)) {
                     throw new ClassCastException("Generic argument [" + genargs[count].name + "] is not a sub class" +
                         " of [" + args[count].name + "] in the " + (statik ? "function" : "method") +
                         " [" + name + " ] from the struct [" + owner.name + "].");
@@ -1666,9 +1660,7 @@ public final class Definition {
         }
 
         if (generic != null) {
-            try {
-                generic.clazz.asSubclass(type.clazz);
-            } catch (final ClassCastException exception) {
+            if (!type.clazz.isAssignableFrom(generic.clazz)) {
                 throw new ClassCastException("Generic type [" + generic.clazz.getCanonicalName() + "]" +
                     " is not a sub class of [" + type.clazz.getCanonicalName() + "] for the field" +
                     " [" + name + " ] from the struct [" + owner.name + "].");
@@ -1736,9 +1728,7 @@ public final class Definition {
                     " not defined for copy to owner struct [" + owner.name + "].");
             }
 
-            try {
-                owner.clazz.asSubclass(child.clazz);
-            } catch (final ClassCastException exception) {
+            if (!child.clazz.isAssignableFrom(owner.clazz)) {
                 throw new ClassCastException("Child struct [" + child.name + "]" +
                     " is not a super type of owner struct [" + owner.name + "] in copy.");
             }
@@ -1844,13 +1834,10 @@ public final class Definition {
 
             Type argument = method.arguments.get(0);
 
-            try {
-                from.clazz.asSubclass(argument.clazz);
-            } catch (final ClassCastException cce0) {
-                try {
-                    argument.clazz.asSubclass(from.clazz);
+            if (!argument.clazz.isAssignableFrom(from.clazz)) {
+                if (from.clazz.isAssignableFrom(argument.clazz)) {
                     upcast = argument;
-                } catch (final ClassCastException cce1) {
+                } else {
                     throw new ClassCastException("Transform with owner struct [" + owner.name + "]" +
                         " and cast type from [" + from.name + "] to cast type to [" + to.name + "] using" +
                         " function [" + name + "] cannot cast from type to the function input argument type.");
@@ -1859,13 +1846,10 @@ public final class Definition {
 
             final Type rtn = method.rtn;
 
-            try {
-                rtn.clazz.asSubclass(to.clazz);
-            } catch (final ClassCastException cce0) {
-                try {
-                    to.clazz.asSubclass(rtn.clazz);
+            if (!to.clazz.isAssignableFrom(rtn.clazz)) {
+                if (rtn.clazz.isAssignableFrom(to.clazz)) {
                     downcast = to;
-                } catch (final ClassCastException cce1) {
+                } else {
                     throw new ClassCastException("Transform with owner struct [" + owner.name + "]" +
                         " and cast type from [" + from.name + "] to cast type to [" + to.name + "] using" +
                         " function [" + name + "] cannot cast to type to the function return argument type.");
@@ -1886,13 +1870,10 @@ public final class Definition {
                     "] using method [" + name + "] does not have a single type argument.");
             }
 
-            try {
-                from.clazz.asSubclass(owner.clazz);
-            } catch (final ClassCastException cce0) {
-                try {
-                    owner.clazz.asSubclass(from.clazz);
+            if (!owner.clazz.isAssignableFrom(from.clazz)) {
+                if (from.clazz.isAssignableFrom(owner.clazz)) {
                     upcast = getType(owner.name);
-                } catch (final ClassCastException cce1) {
+                } else {
                     throw new ClassCastException("Transform with owner struct [" + owner.name + "]" +
                         " and cast type from [" + from.name + "] to cast type to [" + to.name + "] using" +
                         " method [" + name + "] cannot cast from type to the method input argument type.");
@@ -1901,13 +1882,10 @@ public final class Definition {
 
             final Type rtn = method.rtn;
 
-            try {
-                rtn.clazz.asSubclass(to.clazz);
-            } catch (final ClassCastException cce0) {
-                try {
-                    to.clazz.asSubclass(rtn.clazz);
+            if (!to.clazz.isAssignableFrom(rtn.clazz)) {
+                if (rtn.clazz.isAssignableFrom(to.clazz)) {
                     downcast = to;
-                } catch (final ClassCastException cce1) {
+                } else {
                     throw new ClassCastException("Transform with owner struct [" + owner.name + "]" +
                         " and cast type from [" + from.name + "] to cast type to [" + to.name + "]" +
                         " using method [" + name + "] cannot cast to type to the method return argument type.");
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterUtility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterUtility.java
index bb0b303764d..c3c26f309ea 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterUtility.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterUtility.java
@@ -137,9 +137,7 @@ public final class WriterUtility {
             if (from.sort.numeric && from.sort.primitive && to.sort.numeric && to.sort.primitive) {
                 adapter.cast(from.type, to.type);
             } else {
-                try {
-                    from.clazz.asSubclass(to.clazz);
-                } catch (ClassCastException exception) {
+                if (!to.clazz.isAssignableFrom(from.clazz)) {
                     adapter.checkCast(to.type);
                 }
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
index c0f9ddebc68..38d31d65a93 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
@@ -59,14 +59,10 @@ public final class LBrace extends ALink {
             return this;
         } else if (sort == Sort.DEF) {
             return new LDefArray(line, location, index).copy(this).analyze(settings, definition, variables);
-        } else {
-            if (Map.class.isAssignableFrom(before.clazz)) {
-                return new LMapShortcut(line, location, index).copy(this).analyze(settings, definition, variables);
-            }
-            
-            if (List.class.isAssignableFrom(before.clazz)) {
-                return new LListShortcut(line, location, index).copy(this).analyze(settings, definition, variables);
-            }
+        } else if (Map.class.isAssignableFrom(before.clazz)) {
+            return new LMapShortcut(line, location, index).copy(this).analyze(settings, definition, variables);
+        } else if (List.class.isAssignableFrom(before.clazz)) {
+            return new LListShortcut(line, location, index).copy(this).analyze(settings, definition, variables);
         }
 
         throw new IllegalArgumentException(error("Illegal array access on type [" + before.name + "]."));

From 604bcd93202e866cad16b74ea0640f490be9548f Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Sun, 15 May 2016 10:30:11 +0200
Subject: [PATCH 0260/1311] painless: Make field stores not box; use
 GeneratorAdapter.invokeDynmaic for consistency with other method calls

---
 .../elasticsearch/painless/node/ADefLink.java | 39 +++++++++++++++++++
 .../elasticsearch/painless/node/EChain.java   | 13 ++++++-
 .../elasticsearch/painless/node/LCall.java    |  5 +--
 .../painless/node/LDefArray.java              | 11 ++++--
 .../elasticsearch/painless/node/LDefCall.java |  4 +-
 .../painless/node/LDefField.java              | 11 ++++--
 6 files changed, 67 insertions(+), 16 deletions(-)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ADefLink.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ADefLink.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ADefLink.java
new file mode 100644
index 00000000000..ad0beb23cbc
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ADefLink.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.Definition.Type;
+
+/**
+ * The superclass for all LDef* (link) nodes that store or return a DEF. (Internal only.)
+ */
+abstract class ADefLink extends ALink {
+    
+    /**
+     * The type of the original type that was pushed on stack, set by {@link EChain} during analyze.
+     * This value is only used for writing the 'store' bytecode, otherwise ignored.
+     */
+    Type storeValueType = null;
+
+    ADefLink(final int line, final String location, final int size) {
+        super(line, location, size);
+    }
+
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index bc3c326f651..4cfb35e48a8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -222,8 +222,17 @@ public final class EChain extends AExpression {
     private void analyzeWrite(final CompilerSettings settings, final Definition definition, final Variables variables) {
         final ALink last = links.get(links.size() - 1);
 
-        expression.expected = last.after;
-        expression.analyze(settings, definition, variables);
+        // If the store node is a DEF node, we remove the cast to DEF from the expression
+        // and promote the real type to it:
+        if (last instanceof ADefLink) {
+            final ADefLink lastDef = (ADefLink) last;
+            expression.analyze(settings, definition, variables);
+            lastDef.storeValueType = expression.expected = expression.actual;
+        } else {
+            // otherwise we adapt the type of the expression to the store type
+            expression.expected = last.after;
+            expression.analyze(settings, definition, variables);
+        }
         expression = expression.cast(settings, definition, variables);
 
         statement = true;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
index acb6646614a..ee9a996ac22 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
@@ -59,9 +59,6 @@ public final class LCall extends ALink {
         method = statik ? struct.functions.get(name) : struct.methods.get(name);
 
         if (method != null) {
-            final Definition.Type[] types = new Definition.Type[method.arguments.size()];
-            method.arguments.toArray(types);
-
             if (method.arguments.size() != arguments.size()) {
                 throw new IllegalArgumentException(error("When calling [" + name + "] on type [" + struct.name + "]" +
                     " expected [" + method.arguments.size() + "] arguments, but found [" + arguments.size() + "]."));
@@ -70,7 +67,7 @@ public final class LCall extends ALink {
             for (int argument = 0; argument < arguments.size(); ++argument) {
                 final AExpression expression = arguments.get(argument);
 
-                expression.expected = types[argument];
+                expression.expected = method.arguments.get(argument);
                 expression.analyze(settings, definition, variables);
                 arguments.set(argument, expression.cast(settings, definition, variables));
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index 38e8bf89cc9..d3d129d3ba9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -31,7 +31,7 @@ import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
 /**
  * Represents an array load/store or shortcut on a def type.  (Internal only.)
  */
-final class LDefArray extends ALink {
+final class LDefArray extends ADefLink {
 
     AExpression index;
 
@@ -61,13 +61,16 @@ final class LDefArray extends ALink {
     @Override
     void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
         final String desc = Type.getMethodDescriptor(after.type, definition.defType.type, index.actual.type);
-        adapter.visitInvokeDynamicInsn("arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_LOAD);
+        adapter.invokeDynamic("arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_LOAD);
     }
 
     @Override
     void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+        if (storeValueType == null) {
+            throw new IllegalStateException(error("Illegal tree structure."));
+        }
         final String desc = Type.getMethodDescriptor(definition.voidType.type, definition.defType.type,
-            index.actual.type, definition.defType.type);
-        adapter.visitInvokeDynamicInsn("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_STORE);
+            index.actual.type, storeValueType.type);
+        adapter.invokeDynamic("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_STORE);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index 2bf53bcf89b..459f7221b68 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -32,7 +32,7 @@ import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
 /**
  * Represents a method call made on a def type. (Internal only.)
  */
-final class LDefCall extends ALink {
+final class LDefCall extends ADefLink {
 
     final String name;
     final List<AExpression> arguments;
@@ -84,7 +84,7 @@ final class LDefCall extends ALink {
         // return value
         signature.append(after.type.getDescriptor());
 
-        adapter.visitInvokeDynamicInsn(name, signature.toString(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.METHOD_CALL);
+        adapter.invokeDynamic(name, signature.toString(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.METHOD_CALL);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index 68fb7ffb95a..caf78dccdd1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -31,7 +31,7 @@ import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
 /**
  * Represents a field load/store or shortcut on a def type.  (Internal only.)
  */
-final class LDefField extends ALink {
+final class LDefField extends ADefLink {
 
     final String value;
 
@@ -57,12 +57,15 @@ final class LDefField extends ALink {
     @Override
     void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
         final String desc = Type.getMethodDescriptor(after.type, definition.defType.type);
-        adapter.visitInvokeDynamicInsn(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.LOAD);
+        adapter.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.LOAD);
     }
 
     @Override
     void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
-        final String desc = Type.getMethodDescriptor(definition.voidType.type, definition.defType.type, definition.defType.type);
-        adapter.visitInvokeDynamicInsn(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.STORE);
+        if (storeValueType == null) {
+            throw new IllegalStateException(error("Illegal tree structure."));
+        }
+        final String desc = Type.getMethodDescriptor(definition.voidType.type, definition.defType.type, storeValueType.type);
+        adapter.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.STORE);
     }
 }

From 897fe9108afe13dc9c5de4579a005337124c8d3a Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Sun, 15 May 2016 19:30:59 +0200
Subject: [PATCH 0261/1311] Removed docs from mapper-attachment about copy_to

Closes #18361
---
 docs/plugins/mapper-attachments.asciidoc | 33 ------------------------
 1 file changed, 33 deletions(-)

diff --git a/docs/plugins/mapper-attachments.asciidoc b/docs/plugins/mapper-attachments.asciidoc
index 70d760e00b8..570d05a7513 100644
--- a/docs/plugins/mapper-attachments.asciidoc
+++ b/docs/plugins/mapper-attachments.asciidoc
@@ -173,39 +173,6 @@ In the above example, the actual content indexed is mapped under `fields` name `
 it will only be available in the `_all` field. The other fields map to their respective metadata names, but there is no
 need to specify the `type` (like `text` or `date`) since it is already known.
 
-[[mapper-attachments-copy-to]]
-==== Copy To feature
-
-If you want to use https://www.elastic.co/guide/en/elasticsearch/reference/current/copy-to.html[copy_to]
-feature, you need to define it on each sub-field you want to copy to another field:
-
-[source,js]
---------------------------
-PUT /test/person/_mapping
-{
-  "person": {
-    "properties": {
-      "file": {
-        "type": "attachment",
-        "fields": {
-          "content": {
-            "type": "text",
-            "copy_to": "copy"
-          }
-        }
-      },
-      "copy": {
-        "type": "text"
-      }
-    }
-  }
-}
---------------------------
-// CONSOLE
-
-In this example, the extracted content will be copy as well to `copy` field.
-
-[[mapper-attachments-querying-metadata]]
 ==== Querying or accessing metadata
 
 If you need to query on metadata fields, use the attachment field name dot the metadata field. For example:

From d221cd14d219eaf77bb2d91cb4aa7956d518f3ca Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Sun, 15 May 2016 19:49:25 +0200
Subject: [PATCH 0262/1311] painless: Also remove boxing for reads and method
 calls

---
 .../main/java/org/elasticsearch/painless/node/EChain.java    | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index 4cfb35e48a8..463c9a5a4ff 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -242,6 +242,11 @@ public final class EChain extends AExpression {
     private void analyzeRead() {
         final ALink last = links.get(links.size() - 1);
 
+        // If the load node is a DEF node, we adapt its after type to use _this_ expected output type:
+        if (last instanceof ADefLink && this.expected != null) {
+            last.after = this.expected;
+        }
+
         constant = last.string;
         statement = last.statement;
         actual = last.after;

From 65aca4f71c9b88a41e2c5d6bd7fc9cceb7c0d99d Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Sun, 15 May 2016 22:58:11 +0200
Subject: [PATCH 0263/1311] painless: make compound statement like a[1]++ work
 with def. There was also a bug with the size of LDefArray: fixed

---
 .../java/org/elasticsearch/painless/node/ADefLink.java    | 2 ++
 .../main/java/org/elasticsearch/painless/node/EChain.java | 6 ++++++
 .../java/org/elasticsearch/painless/node/LDefArray.java   | 2 +-
 .../test/java/org/elasticsearch/painless/ArrayTests.java  | 8 ++++++++
 4 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ADefLink.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ADefLink.java
index ad0beb23cbc..cf4351a63a7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ADefLink.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ADefLink.java
@@ -23,6 +23,8 @@ import org.elasticsearch.painless.Definition.Type;
 
 /**
  * The superclass for all LDef* (link) nodes that store or return a DEF. (Internal only.)
+ * For this node it is allowed to change {@link ALink#after} from outside, by default
+ * {@code after} is {@code DEF}.
  */
 abstract class ADefLink extends ALink {
     
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index 463c9a5a4ff..3dd64975757 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -215,6 +215,12 @@ public final class EChain extends AExpression {
         there = AnalyzerCaster.getLegalCast(definition, location, last.after, promote, false);
         back = AnalyzerCaster.getLegalCast(definition, location, promote, last.after, true);
 
+        if (last instanceof ADefLink) {
+            final ADefLink lastDef = (ADefLink) last;
+            // Unfortunately, we don't know the real type because we load from DEF and store to DEF!
+            lastDef.storeValueType = last.after;
+        }
+
         statement = true;
         actual = read ? last.after : definition.voidType;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index d3d129d3ba9..df1c5ffac0a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -36,7 +36,7 @@ final class LDefArray extends ADefLink {
     AExpression index;
 
     LDefArray(final int line, final String location, final AExpression index) {
-        super(line, location, 0);
+        super(line, location, 2);
 
         this.index = index;
     }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
index 51ee659e501..aa873e45d36 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
@@ -54,6 +54,14 @@ public class ArrayTests extends ScriptTestCase {
         assertEquals(5, exec("def x = new def[4]; x[0] = 5; return x[0];"));
     }
     
+    public void testArrayCompoundInt() {
+        assertEquals(6, exec("int[] x = new int[5]; x[0] = 5; x[0]++; return x[0];"));
+    }
+    
+    public void testArrayCompoundDef() {
+        assertEquals(6, exec("def x = new int[5]; x[0] = 5; x[0]++; return x[0];"));
+    }
+    
     public void testForLoop() {
         assertEquals(999*1000/2, exec("def a = new int[1000]; for (int x = 0; x < a.length; x++) { a[x] = x; } "+
             "int total = 0; for (int x = 0; x < a.length; x++) { total += a[x]; } return total;"));

From 8edf213492238a51b544d8dcddfe75ab6a14095a Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Sun, 15 May 2016 22:59:28 -0400
Subject: [PATCH 0264/1311] Remove LeafSearchScript.runAsFloat(): Nothing calls
 it.

---
 .../script/AbstractDoubleSearchScript.java    |  5 --
 .../script/AbstractFloatSearchScript.java     | 45 -------------
 .../script/AbstractLongSearchScript.java      |  5 --
 .../script/AbstractSearchScript.java          |  7 +-
 .../script/LeafSearchScript.java              |  2 -
 .../script/NativeScriptFactory.java           |  1 -
 .../function/ScriptScoreFunctionTests.java    | 66 ++++++++-----------
 .../search/aggregations/metrics/AvgIT.java    | 10 ---
 .../search/aggregations/metrics/SumIT.java    | 10 ---
 .../aggregations/metrics/ValueCountIT.java    |  5 --
 .../support/ScriptValuesTests.java            |  5 --
 .../modules/scripting/native.asciidoc         | 12 ++--
 .../expression/ExpressionSearchScript.java    |  3 -
 .../groovy/GroovyScriptEngineService.java     |  5 --
 .../elasticsearch/painless/ScriptImpl.java    |  9 ---
 .../JavaScriptScriptEngineService.java        |  5 --
 .../python/PythonScriptEngineService.java     |  5 --
 17 files changed, 33 insertions(+), 167 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/script/AbstractFloatSearchScript.java

diff --git a/core/src/main/java/org/elasticsearch/script/AbstractDoubleSearchScript.java b/core/src/main/java/org/elasticsearch/script/AbstractDoubleSearchScript.java
index 6a803ba8571..66eb0f3684d 100644
--- a/core/src/main/java/org/elasticsearch/script/AbstractDoubleSearchScript.java
+++ b/core/src/main/java/org/elasticsearch/script/AbstractDoubleSearchScript.java
@@ -37,9 +37,4 @@ public abstract class AbstractDoubleSearchScript extends AbstractSearchScript {
     public long runAsLong() {
         return (long) runAsDouble();
     }
-
-    @Override
-    public float runAsFloat() {
-        return (float) runAsDouble();
-    }
 }
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/script/AbstractFloatSearchScript.java b/core/src/main/java/org/elasticsearch/script/AbstractFloatSearchScript.java
deleted file mode 100644
index 21db0b05501..00000000000
--- a/core/src/main/java/org/elasticsearch/script/AbstractFloatSearchScript.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.script;
-
-/**
- * A simpler base class instead of {@link AbstractSearchScript} for computations
- * that return a float number.
- */
-public abstract class AbstractFloatSearchScript extends AbstractSearchScript {
-
-    @Override
-    public Object run() {
-        return runAsFloat();
-    }
-
-    @Override
-    public abstract float runAsFloat();
-
-    @Override
-    public double runAsDouble() {
-        return runAsFloat();
-    }
-
-    @Override
-    public long runAsLong() {
-        return (long) runAsFloat();
-    }
-}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/script/AbstractLongSearchScript.java b/core/src/main/java/org/elasticsearch/script/AbstractLongSearchScript.java
index 53a24002d3d..226984e0697 100644
--- a/core/src/main/java/org/elasticsearch/script/AbstractLongSearchScript.java
+++ b/core/src/main/java/org/elasticsearch/script/AbstractLongSearchScript.java
@@ -37,9 +37,4 @@ public abstract class AbstractLongSearchScript extends AbstractSearchScript {
     public double runAsDouble() {
         return runAsLong();
     }
-
-    @Override
-    public float runAsFloat() {
-        return runAsLong();
-    }
 }
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/script/AbstractSearchScript.java b/core/src/main/java/org/elasticsearch/script/AbstractSearchScript.java
index 98e5ab11fb7..929046d4dd8 100644
--- a/core/src/main/java/org/elasticsearch/script/AbstractSearchScript.java
+++ b/core/src/main/java/org/elasticsearch/script/AbstractSearchScript.java
@@ -34,7 +34,7 @@ import java.util.Map;
  * A base class for any script type that is used during the search process (custom score, aggs, and so on).
  * <p>
  * If the script returns a specific numeric type, consider overriding the type specific base classes
- * such as {@link AbstractDoubleSearchScript}, {@link AbstractFloatSearchScript} and {@link AbstractLongSearchScript}
+ * such as {@link AbstractDoubleSearchScript} and {@link AbstractLongSearchScript}
  * for better performance.
  * <p>
  * The use is required to implement the {@link #run()} method.
@@ -120,11 +120,6 @@ public abstract class AbstractSearchScript extends AbstractExecutableScript impl
         lookup.source().setSource(source);
     }
 
-    @Override
-    public float runAsFloat() {
-        return ((Number) run()).floatValue();
-    }
-
     @Override
     public long runAsLong() {
         return ((Number) run()).longValue();
diff --git a/core/src/main/java/org/elasticsearch/script/LeafSearchScript.java b/core/src/main/java/org/elasticsearch/script/LeafSearchScript.java
index cff0e42e5cf..0bf9e0d50e0 100644
--- a/core/src/main/java/org/elasticsearch/script/LeafSearchScript.java
+++ b/core/src/main/java/org/elasticsearch/script/LeafSearchScript.java
@@ -44,8 +44,6 @@ public interface LeafSearchScript extends ScorerAware, ExecutableScript {
         setNextVar("_value", value);
     }
 
-    float runAsFloat();
-
     long runAsLong();
 
     double runAsDouble();
diff --git a/core/src/main/java/org/elasticsearch/script/NativeScriptFactory.java b/core/src/main/java/org/elasticsearch/script/NativeScriptFactory.java
index 1625afe97c9..6768bc4f87a 100644
--- a/core/src/main/java/org/elasticsearch/script/NativeScriptFactory.java
+++ b/core/src/main/java/org/elasticsearch/script/NativeScriptFactory.java
@@ -29,7 +29,6 @@ import java.util.Map;
  *
  * @see AbstractExecutableScript
  * @see AbstractSearchScript
- * @see AbstractFloatSearchScript
  * @see AbstractLongSearchScript
  * @see AbstractDoubleSearchScript
  */
diff --git a/core/src/test/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunctionTests.java b/core/src/test/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunctionTests.java
index 17fbc882ba9..861476b9d47 100644
--- a/core/src/test/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunctionTests.java
+++ b/core/src/test/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunctionTests.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.common.lucene.search.function;
 
 import org.apache.lucene.index.LeafReaderContext;
-import org.elasticsearch.script.AbstractFloatSearchScript;
+import org.elasticsearch.script.AbstractDoubleSearchScript;
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptException;
@@ -29,51 +29,37 @@ import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 
-import static org.hamcrest.Matchers.equalTo;
-
 public class ScriptScoreFunctionTests extends ESTestCase {
     /**
      * Tests https://github.com/elastic/elasticsearch/issues/2426
      */
     public void testScriptScoresReturnsNaN() throws IOException {
-        ScoreFunction scoreFunction = new ScriptScoreFunction(new Script("Float.NaN"), new FloatValueScript(Float.NaN));
+        // script that always returns NaN
+        ScoreFunction scoreFunction = new ScriptScoreFunction(new Script("Double.NaN"), new SearchScript() {
+            @Override
+            public LeafSearchScript getLeafSearchScript(LeafReaderContext context) throws IOException {
+                return new AbstractDoubleSearchScript() {
+                    @Override
+                    public double runAsDouble() {
+                        return Double.NaN;
+                    }
+
+                    @Override
+                    public void setDocument(int doc) {
+                        // do nothing: we are a fake with no lookup
+                    }
+                };
+            }
+            
+            @Override
+            public boolean needsScores() {
+                return false;
+            }
+        });
         LeafScoreFunction leafScoreFunction = scoreFunction.getLeafScoreFunction(null);
-        try {
+        ScriptException expected = expectThrows(ScriptException.class, () -> {
             leafScoreFunction.score(randomInt(), randomFloat());
-            fail("should have thrown an exception about the script_score returning NaN");
-        } catch (ScriptException e) {
-            assertThat("message contains error about script_score returning NaN: " + e.getMessage(),
-                    e.getMessage().contains("NaN"), equalTo(true));
-        }
-    }
-
-    static class FloatValueScript implements SearchScript {
-
-        private final float value;
-
-        FloatValueScript(float value) {
-            this.value = value;
-        }
-
-        @Override
-        public LeafSearchScript getLeafSearchScript(LeafReaderContext context) throws IOException {
-            return new AbstractFloatSearchScript() {
-
-                @Override
-                public float runAsFloat() {
-                    return value;
-                }
-
-                @Override
-                public void setDocument(int doc) {
-                    // nothing here
-                }
-            };
-        }
-
-        @Override
-        public boolean needsScores() {
-            return false;
-        }
+        });
+        assertTrue(expected.getMessage().contains("returned NaN"));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
index 5b233a9690d..a01797ccf49 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
@@ -455,11 +455,6 @@ public class AvgIT extends AbstractNumericTestCase {
                             throw new UnsupportedOperationException();
                         }
 
-                        @Override
-                        public float runAsFloat() {
-                            throw new UnsupportedOperationException();
-                        }
-
                         @Override
                         public double runAsDouble() {
                             throw new UnsupportedOperationException();
@@ -578,11 +573,6 @@ public class AvgIT extends AbstractNumericTestCase {
                             return ((Number) vars.get("_value")).longValue() + inc;
                         }
 
-                        @Override
-                        public float runAsFloat() {
-                            throw new UnsupportedOperationException();
-                        }
-
                         @Override
                         public double runAsDouble() {
                             return ((Number) vars.get("_value")).doubleValue() + inc;
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
index d0538973173..c636508bbe1 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
@@ -452,11 +452,6 @@ public class SumIT extends AbstractNumericTestCase {
                             throw new UnsupportedOperationException();
                         }
 
-                        @Override
-                        public float runAsFloat() {
-                            throw new UnsupportedOperationException();
-                        }
-
                         @Override
                         public double runAsDouble() {
                             throw new UnsupportedOperationException();
@@ -583,11 +578,6 @@ public class SumIT extends AbstractNumericTestCase {
                             return ((Number) vars.get("_value")).longValue() + inc;
                         }
 
-                        @Override
-                        public float runAsFloat() {
-                            throw new UnsupportedOperationException();
-                        }
-
                         @Override
                         public double runAsDouble() {
                             return ((Number) vars.get("_value")).doubleValue() + inc;
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
index 2b957188663..86f531ce3e9 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
@@ -314,11 +314,6 @@ public class ValueCountIT extends ESIntegTestCase {
                             throw new UnsupportedOperationException();
                         }
 
-                        @Override
-                        public float runAsFloat() {
-                            throw new UnsupportedOperationException();
-                        }
-
                         @Override
                         public double runAsDouble() {
                             throw new UnsupportedOperationException();
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java
index 4f3b3db7c94..dcf6cfdb0ba 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java
@@ -75,11 +75,6 @@ public class ScriptValuesTests extends ESTestCase {
         public void setSource(Map<String, Object> source) {
         }
 
-        @Override
-        public float runAsFloat() {
-            throw new UnsupportedOperationException();
-        }
-
         @Override
         public long runAsLong() {
             throw new UnsupportedOperationException();
diff --git a/docs/reference/modules/scripting/native.asciidoc b/docs/reference/modules/scripting/native.asciidoc
index 1b99673a90f..9580720d967 100644
--- a/docs/reference/modules/scripting/native.asciidoc
+++ b/docs/reference/modules/scripting/native.asciidoc
@@ -12,8 +12,8 @@ To register the actual script you'll need to implement `NativeScriptFactory`
 to construct the script. The actual script will extend either
 `AbstractExecutableScript` or `AbstractSearchScript`. The second one is likely
 the most useful and has several helpful subclasses you can extend like
-`AbstractLongSearchScript`, `AbstractDoubleSearchScript`, and
-`AbstractFloatSearchScript`. Finally, your plugin should register the native
+`AbstractLongSearchScript` and `AbstractDoubleSearchScript`.
+Finally, your plugin should register the native
 script by declaring the `onModule(ScriptModule)` method.
 
 If you squashed the whole thing into one class it'd look like:
@@ -44,11 +44,11 @@ public class MyNativeScriptPlugin extends Plugin {
         }
     }
 
-    public static class MyNativeScript extends AbstractFloatSearchScript {
+    public static class MyNativeScript extends AbstractDoubleSearchScript {
         @Override
-        public float runAsFloat() {
-            float a = (float) source().get("a");
-            float b = (float) source().get("b");
+        public double runAsDouble() {
+            double a = (double) source().get("a");
+            double b = (double) source().get("b");
             return a * b;
         }
     }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java
index cb99c879058..54553565bcb 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java
@@ -80,9 +80,6 @@ class ExpressionSearchScript implements SearchScript {
             @Override
             public Object run() { return new Double(evaluate()); }
 
-            @Override
-            public float runAsFloat() { return (float)evaluate();}
-
             @Override
             public long runAsLong() { return (long)evaluate(); }
 
diff --git a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
index 23f1b409359..700a1aecbc2 100644
--- a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
+++ b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
@@ -316,11 +316,6 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
             }
         }
 
-        @Override
-        public float runAsFloat() {
-            return ((Number) run()).floatValue();
-        }
-
         @Override
         public long runAsLong() {
             return ((Number) run()).longValue();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
index 23be345ed1c..04d5a8d3d60 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
@@ -126,15 +126,6 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
         return ((Number)run()).doubleValue();
     }
 
-    /**
-     * Run the script.
-     * @return The script result as a float.
-     */
-    @Override
-    public float runAsFloat() {
-        return ((Number)run()).floatValue();
-    }
-
     /**
      * Run the script.
      * @return The script result as a long.
diff --git a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
index d4f713d9cb5..4dd24454b96 100644
--- a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
+++ b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
@@ -319,11 +319,6 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements
             }
         }
 
-        @Override
-        public float runAsFloat() {
-            return ((Number) run()).floatValue();
-        }
-
         @Override
         public long runAsLong() {
             return ((Number) run()).longValue();
diff --git a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
index 3f6ceb4a1b6..df7e9c07394 100644
--- a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
+++ b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
@@ -243,11 +243,6 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri
             return ret.__tojava__(Object.class);
         }
 
-        @Override
-        public float runAsFloat() {
-            return ((Number) run()).floatValue();
-        }
-
         @Override
         public long runAsLong() {
             return ((Number) run()).longValue();

From ab3121c8713140b9cf6df0d76cd1b78c6939d357 Mon Sep 17 00:00:00 2001
From: Colin Goodheart-Smithe <colings86@users.noreply.github.com>
Date: Wed, 27 Apr 2016 09:54:16 +0100
Subject: [PATCH 0265/1311] Adds a methods to find (and dynamically create) the
 mappers for the parents of a field with dots in the field name

---
 .../index/mapper/DocumentParser.java          | 143 +++++++++-----
 .../index/mapper/DocumentParserTests.java     | 180 ++++++++++++++++++
 2 files changed, 271 insertions(+), 52 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
index 197944aaacb..9967b087f86 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
@@ -23,6 +23,7 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -454,19 +455,24 @@ final class DocumentParser {
 
     private static ObjectMapper parseObject(final ParseContext context, ObjectMapper mapper, String currentFieldName) throws IOException {
         assert currentFieldName != null;
-        context.path().add(currentFieldName);
 
         ObjectMapper update = null;
         Mapper objectMapper = getMapper(mapper, currentFieldName);
         if (objectMapper != null) {
+            context.path().add(currentFieldName);
             parseObjectOrField(context, objectMapper);
+            context.path().remove();
         } else {
-            ObjectMapper.Dynamic dynamic = dynamicOrDefault(mapper, context);
+
+            // TODO: why Strings.splitStringToArray instead of String.split?
+            final String[] paths = Strings.splitStringToArray(currentFieldName, '.');
+            currentFieldName = paths[paths.length - 1];
+            Tuple<Integer, ObjectMapper> parentMapperTuple = getDynamicParentMapper(context, paths, mapper);
+            ObjectMapper parentMapper = parentMapperTuple.v2();
+            ObjectMapper.Dynamic dynamic = dynamicOrDefault(parentMapper, context);
             if (dynamic == ObjectMapper.Dynamic.STRICT) {
                 throw new StrictDynamicMappingException(mapper.fullPath(), currentFieldName);
             } else if (dynamic == ObjectMapper.Dynamic.TRUE) {
-                // remove the current field name from path, since template search and the object builder add it as well...
-                context.path().remove();
                 Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "object");
                 if (builder == null) {
                     builder = new ObjectMapper.Builder(currentFieldName).enabled(true);
@@ -476,13 +482,16 @@ final class DocumentParser {
                 context.addDynamicMapper(objectMapper);
                 context.path().add(currentFieldName);
                 parseObjectOrField(context, objectMapper);
+                context.path().remove();
             } else {
                 // not dynamic, read everything up to end object
                 context.parser().skipChildren();
             }
+            for (int i = 0; i < parentMapperTuple.v1(); i++) {
+                context.path().remove();
+            }
         }
 
-        context.path().remove();
         return update;
     }
 
@@ -500,6 +509,12 @@ final class DocumentParser {
             }
         } else {
 
+            // TODO: why Strings.splitStringToArray instead of String.split?
+            final String[] paths = Strings.splitStringToArray(arrayFieldName, '.');
+            arrayFieldName = paths[paths.length - 1];
+            lastFieldName = arrayFieldName;
+            Tuple<Integer, ObjectMapper> parentMapperTuple = getDynamicParentMapper(context, paths, parentMapper);
+            parentMapper = parentMapperTuple.v2();
             ObjectMapper.Dynamic dynamic = dynamicOrDefault(parentMapper, context);
             if (dynamic == ObjectMapper.Dynamic.STRICT) {
                 throw new StrictDynamicMappingException(parentMapper.fullPath(), arrayFieldName);
@@ -507,23 +522,26 @@ final class DocumentParser {
                 Mapper.Builder builder = context.root().findTemplateBuilder(context, arrayFieldName, "object");
                 if (builder == null) {
                     parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
-                    return;
-                }
-                Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings(), context.path());
-                mapper = builder.build(builderContext);
-                assert mapper != null;
-                if (mapper instanceof ArrayValueMapperParser) {
-                    context.addDynamicMapper(mapper);
-                    context.path().add(arrayFieldName);
-                    parseObjectOrField(context, mapper);
-                    context.path().remove();
                 } else {
-                    parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
+                    Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings(), context.path());
+                    mapper = builder.build(builderContext);
+                    assert mapper != null;
+                    if (mapper instanceof ArrayValueMapperParser) {
+                        context.addDynamicMapper(mapper);
+                        context.path().add(arrayFieldName);
+                        parseObjectOrField(context, mapper);
+                        context.path().remove();
+                    } else {
+                        parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
+                    }
                 }
             } else {
                 // TODO: shouldn't this skip, not parse?
                 parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName);
             }
+            for (int i = 0; i < parentMapperTuple.v1(); i++) {
+                context.path().remove();
+            }
         }
     }
 
@@ -555,7 +573,16 @@ final class DocumentParser {
         if (mapper != null) {
             parseObjectOrField(context, mapper);
         } else {
+
+            // TODO: why Strings.splitStringToArray instead of String.split?
+            final String[] paths = Strings.splitStringToArray(currentFieldName, '.');
+            currentFieldName = paths[paths.length - 1];
+            Tuple<Integer, ObjectMapper> parentMapperTuple = getDynamicParentMapper(context, paths, parentMapper);
+            parentMapper = parentMapperTuple.v2();
             parseDynamicValue(context, parentMapper, currentFieldName, token);
+            for (int i = 0; i < parentMapperTuple.v1(); i++) {
+                context.path().remove();
+            }
         }
     }
 
@@ -814,46 +841,58 @@ final class DocumentParser {
 
             final String[] paths = field.split("\\.");
             final String fieldName = paths[paths.length-1];
-            ObjectMapper mapper = context.root();
-            ObjectMapper[] mappers = new ObjectMapper[paths.length-1];
-            if (paths.length > 1) {
-                ObjectMapper parent = context.root();
-                for (int i = 0; i < paths.length-1; i++) {
-                    mapper = context.docMapper().objectMappers().get(context.path().pathAsText(paths[i]));
-                    if (mapper == null) {
-                        // One mapping is missing, check if we are allowed to create a dynamic one.
-                        ObjectMapper.Dynamic dynamic = dynamicOrDefault(parent, context);
-
-                        switch (dynamic) {
-                            case STRICT:
-                                throw new StrictDynamicMappingException(parent.fullPath(), paths[i]);
-                            case TRUE:
-                                Mapper.Builder builder = context.root().findTemplateBuilder(context, paths[i], "object");
-                                if (builder == null) {
-                                    builder = new ObjectMapper.Builder(paths[i]).enabled(true);
-                                }
-                                Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings(), context.path());
-                                mapper = (ObjectMapper) builder.build(builderContext);
-                                if (mapper.nested() != ObjectMapper.Nested.NO) {
-                                    throw new MapperParsingException("It is forbidden to create dynamic nested objects ([" + context.path().pathAsText(paths[i]) + "]) through `copy_to`");
-                                }
-                                context.addDynamicMapper(mapper);
-                                break;
-                            case FALSE:
-                              // Maybe we should log something to tell the user that the copy_to is ignored in this case.
-                              break;
-
-                        }
-                    }
-                    context.path().add(paths[i]);
-                    mappers[i] = mapper;
-                    parent = mapper;
-                }
-            }
+            Tuple<Integer, ObjectMapper> parentMapperTuple = getDynamicParentMapper(context, paths, null);
+            ObjectMapper mapper = parentMapperTuple.v2();
             parseDynamicValue(context, mapper, fieldName, context.parser().currentToken());
+            for (int i = 0; i < parentMapperTuple.v1(); i++) {
+                context.path().remove();
+            }
         }
     }
 
+    private static Tuple<Integer, ObjectMapper> getDynamicParentMapper(ParseContext context, final String[] paths,
+            ObjectMapper currentParent) {
+        ObjectMapper mapper = currentParent == null ? context.root() : currentParent;
+        ObjectMapper[] mappers = new ObjectMapper[paths.length-1];
+        int pathsAdded = 0;
+        if (paths.length > 1) {
+            ObjectMapper parent = mapper;
+            for (int i = 0; i < paths.length-1; i++) {
+                mapper = context.docMapper().objectMappers().get(context.path().pathAsText(paths[i]));
+                if (mapper == null) {
+                    // One mapping is missing, check if we are allowed to create a dynamic one.
+                    ObjectMapper.Dynamic dynamic = dynamicOrDefault(parent, context);
+
+                    switch (dynamic) {
+                        case STRICT:
+                            throw new StrictDynamicMappingException(parent.fullPath(), paths[i]);
+                        case TRUE:
+                            Mapper.Builder builder = context.root().findTemplateBuilder(context, paths[i], "object");
+                            if (builder == null) {
+                                builder = new ObjectMapper.Builder(paths[i]).enabled(true);
+                            }
+                            Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings(), context.path());
+                            mapper = (ObjectMapper) builder.build(builderContext);
+                            if (mapper.nested() != ObjectMapper.Nested.NO) {
+                                throw new MapperParsingException("It is forbidden to create dynamic nested objects ([" + context.path().pathAsText(paths[i]) + "]) through `copy_to`");
+                            }
+                            context.addDynamicMapper(mapper);
+                            break;
+                        case FALSE:
+                           // Should not dynamically create any more mappers so return the last mapper
+                        return new Tuple<Integer, ObjectMapper>(pathsAdded, parent);
+
+                    }
+                }
+                context.path().add(paths[i]);
+                pathsAdded++;
+                mappers[i] = mapper;
+                parent = mapper;
+            }
+        }
+        return new Tuple<Integer, ObjectMapper>(pathsAdded, mapper);
+    }
+
     // find what the dynamic setting is given the current parse context and parent
     private static ObjectMapper.Dynamic dynamicOrDefault(ObjectMapper parentMapper, ParseContext context) {
         ObjectMapper.Dynamic dynamic = parentMapper.dynamic();
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
index b9e5d6d3cf8..252f19f5ff7 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
@@ -497,4 +497,184 @@ public class DocumentParserTests extends ESSingleNodeTestCase {
         ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
         assertEquals(0, doc.rootDoc().getFields("foo").length);
     }
+
+    public void testDynamicDottedFieldNameLongArray() throws Exception {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .endObject().endObject().string();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = XContentFactory.jsonBuilder()
+            .startObject().startArray("foo.bar.baz")
+                .value(0)
+                .value(1)
+            .endArray().endObject().bytes();
+        ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
+        assertEquals(4, doc.rootDoc().getFields("foo.bar.baz").length);
+    }
+
+    public void testDynamicDottedFieldNameLongArrayWithParentTemplate() throws Exception {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startArray("dynamic_templates").startObject().startObject("georule")
+                .field("match", "foo*")
+                .startObject("mapping").field("type", "object").endObject()
+            .endObject().endObject().endArray().endObject().endObject().string();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = XContentFactory.jsonBuilder()
+                .startObject().startArray("foo.bar.baz")
+                .value(0)
+                .value(1)
+            .endArray().endObject().bytes();
+        ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
+        assertEquals(4, doc.rootDoc().getFields("foo.bar.baz").length);
+    }
+
+    public void testDynamicFalseDottedFieldNameLongArray() throws Exception {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").field("dynamic", "false")
+            .endObject().endObject().string();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = XContentFactory.jsonBuilder()
+                .startObject().startArray("foo.bar.baz")
+                .value(0)
+                .value(1)
+            .endArray().endObject().bytes();
+        ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
+        assertEquals(0, doc.rootDoc().getFields("foo.bar.baz").length);
+    }
+
+    public void testDynamicStrictDottedFieldNameLongArray() throws Exception {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").field("dynamic", "strict")
+            .endObject().endObject().string();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = XContentFactory.jsonBuilder()
+                .startObject().startArray("foo.bar.baz")
+                .value(0)
+                .value(1)
+            .endArray().endObject().bytes();
+        StrictDynamicMappingException exception = expectThrows(StrictDynamicMappingException.class,
+                () -> mapper.parse("test", "type", "1", bytes));
+        assertEquals("mapping set to strict, dynamic introduction of [foo] within [type] is not allowed", exception.getMessage());
+    }
+
+    public void testDynamicDottedFieldNameLong() throws Exception {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .endObject().endObject().string();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = XContentFactory.jsonBuilder()
+            .startObject().field("foo.bar.baz", 0)
+            .endObject().bytes();
+        ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
+        assertEquals(2, doc.rootDoc().getFields("foo.bar.baz").length);
+    }
+
+    public void testDynamicDottedFieldNameLongWithParentTemplate() throws Exception {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startArray("dynamic_templates").startObject().startObject("georule")
+                .field("match", "foo*")
+                .startObject("mapping").field("type", "object").endObject()
+            .endObject().endObject().endArray().endObject().endObject().string();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = XContentFactory.jsonBuilder()
+                .startObject().field("foo.bar.baz", 0)
+            .endObject().bytes();
+        ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
+        assertEquals(2, doc.rootDoc().getFields("foo.bar.baz").length);
+    }
+
+    public void testDynamicFalseDottedFieldNameLong() throws Exception {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").field("dynamic", "false")
+            .endObject().endObject().string();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = XContentFactory.jsonBuilder()
+                .startObject().field("foo.bar.baz", 0)
+            .endObject().bytes();
+        ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
+        assertEquals(0, doc.rootDoc().getFields("foo.bar.baz").length);
+    }
+
+    public void testDynamicStrictDottedFieldNameLong() throws Exception {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").field("dynamic", "strict")
+            .endObject().endObject().string();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = XContentFactory.jsonBuilder()
+                .startObject().field("foo.bar.baz", 0)
+            .endObject().bytes();
+        StrictDynamicMappingException exception = expectThrows(StrictDynamicMappingException.class,
+                () -> mapper.parse("test", "type", "1", bytes));
+        assertEquals("mapping set to strict, dynamic introduction of [foo] within [type] is not allowed", exception.getMessage());
+    }
+
+    public void testDynamicDottedFieldNameObject() throws Exception {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .endObject().endObject().string();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = XContentFactory.jsonBuilder()
+            .startObject().startObject("foo.bar.baz")
+                .field("a", 0)
+            .endObject().endObject().bytes();
+        ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
+        assertEquals(2, doc.rootDoc().getFields("foo.bar.baz.a").length);
+    }
+
+    public void testDynamicDottedFieldNameObjectWithParentTemplate() throws Exception {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startArray("dynamic_templates").startObject().startObject("georule")
+                .field("match", "foo*")
+                .startObject("mapping").field("type", "object").endObject()
+            .endObject().endObject().endArray().endObject().endObject().string();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = XContentFactory.jsonBuilder()
+                .startObject().startObject("foo.bar.baz")
+                .field("a", 0)
+            .endObject().endObject().bytes();
+        ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
+        assertEquals(2, doc.rootDoc().getFields("foo.bar.baz.a").length);
+    }
+
+    public void testDynamicFalseDottedFieldNameObject() throws Exception {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").field("dynamic", "false")
+            .endObject().endObject().string();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = XContentFactory.jsonBuilder()
+                .startObject().startObject("foo.bar.baz")
+                .field("a", 0)
+            .endObject().endObject().bytes();
+        ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
+        assertEquals(0, doc.rootDoc().getFields("foo.bar.baz.a").length);
+    }
+
+    public void testDynamicStrictDottedFieldNameObject() throws Exception {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").field("dynamic", "strict")
+            .endObject().endObject().string();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = XContentFactory.jsonBuilder()
+                .startObject().startObject("foo.bar.baz")
+                .field("a", 0)
+            .endObject().endObject().bytes();
+        StrictDynamicMappingException exception = expectThrows(StrictDynamicMappingException.class,
+                () -> mapper.parse("test", "type", "1", bytes));
+        assertEquals("mapping set to strict, dynamic introduction of [foo] within [type] is not allowed", exception.getMessage());
+    }
 }

From 66d0bdab0c808e6e753fe26489135b8ffe71cf8c Mon Sep 17 00:00:00 2001
From: Colin Goodheart-Smithe <colings86@users.noreply.github.com>
Date: Mon, 9 May 2016 10:51:50 +0100
Subject: [PATCH 0266/1311] review comments

---
 .../java/org/elasticsearch/index/mapper/DocumentParser.java   | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
index 9967b087f86..2e73223205a 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
@@ -853,9 +853,7 @@ final class DocumentParser {
     private static Tuple<Integer, ObjectMapper> getDynamicParentMapper(ParseContext context, final String[] paths,
             ObjectMapper currentParent) {
         ObjectMapper mapper = currentParent == null ? context.root() : currentParent;
-        ObjectMapper[] mappers = new ObjectMapper[paths.length-1];
         int pathsAdded = 0;
-        if (paths.length > 1) {
             ObjectMapper parent = mapper;
             for (int i = 0; i < paths.length-1; i++) {
                 mapper = context.docMapper().objectMappers().get(context.path().pathAsText(paths[i]));
@@ -886,10 +884,8 @@ final class DocumentParser {
                 }
                 context.path().add(paths[i]);
                 pathsAdded++;
-                mappers[i] = mapper;
                 parent = mapper;
             }
-        }
         return new Tuple<Integer, ObjectMapper>(pathsAdded, mapper);
     }
 

From 0c449fee4afd758af09b5e18e17f32785d541d1e Mon Sep 17 00:00:00 2001
From: Colin Goodheart-Smithe <colings86@users.noreply.github.com>
Date: Mon, 9 May 2016 10:55:15 +0100
Subject: [PATCH 0267/1311] small fix following rebase on master

---
 .../org/elasticsearch/index/mapper/DocumentParser.java   | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
index 2e73223205a..dc67c4233d6 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
@@ -464,8 +464,7 @@ final class DocumentParser {
             context.path().remove();
         } else {
 
-            // TODO: why Strings.splitStringToArray instead of String.split?
-            final String[] paths = Strings.splitStringToArray(currentFieldName, '.');
+            final String[] paths = currentFieldName.split("\\.");
             currentFieldName = paths[paths.length - 1];
             Tuple<Integer, ObjectMapper> parentMapperTuple = getDynamicParentMapper(context, paths, mapper);
             ObjectMapper parentMapper = parentMapperTuple.v2();
@@ -509,8 +508,7 @@ final class DocumentParser {
             }
         } else {
 
-            // TODO: why Strings.splitStringToArray instead of String.split?
-            final String[] paths = Strings.splitStringToArray(arrayFieldName, '.');
+            final String[] paths = arrayFieldName.split("\\.");
             arrayFieldName = paths[paths.length - 1];
             lastFieldName = arrayFieldName;
             Tuple<Integer, ObjectMapper> parentMapperTuple = getDynamicParentMapper(context, paths, parentMapper);
@@ -574,8 +572,7 @@ final class DocumentParser {
             parseObjectOrField(context, mapper);
         } else {
 
-            // TODO: why Strings.splitStringToArray instead of String.split?
-            final String[] paths = Strings.splitStringToArray(currentFieldName, '.');
+            final String[] paths = currentFieldName.split("\\.");
             currentFieldName = paths[paths.length - 1];
             Tuple<Integer, ObjectMapper> parentMapperTuple = getDynamicParentMapper(context, paths, parentMapper);
             parentMapper = parentMapperTuple.v2();

From 6eda9f5df61f3483fff1895e5cbcb1d562a5a3f6 Mon Sep 17 00:00:00 2001
From: Colin Goodheart-Smithe <colings86@users.noreply.github.com>
Date: Mon, 9 May 2016 14:41:27 +0100
Subject: [PATCH 0268/1311] more tests following review

---
 .../index/mapper/DocumentParser.java          |   9 +-
 .../index/mapper/DocumentParserTests.java     | 185 ++++++++++++++++++
 2 files changed, 193 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
index dc67c4233d6..536e6ecbca5 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
@@ -853,7 +853,14 @@ final class DocumentParser {
         int pathsAdded = 0;
             ObjectMapper parent = mapper;
             for (int i = 0; i < paths.length-1; i++) {
-                mapper = context.docMapper().objectMappers().get(context.path().pathAsText(paths[i]));
+            String currentPath = context.path().pathAsText(paths[i]);
+            FieldMapper existingFieldMapper = context.docMapper().mappers().getMapper(currentPath);
+            if (existingFieldMapper != null) {
+                throw new MapperParsingException(
+                        "Could not dynamically add mapping for field [{}]. Existing mapping for [{}] must be of type object but found [{}].",
+                        null, String.join(".", paths), currentPath, existingFieldMapper.fieldType.typeName());
+            }
+            mapper = context.docMapper().objectMappers().get(currentPath);
                 if (mapper == null) {
                     // One mapping is missing, check if we are allowed to create a dynamic one.
                     ObjectMapper.Dynamic dynamic = dynamicOrDefault(parent, context);
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
index 252f19f5ff7..197ac9ec55e 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
@@ -27,11 +27,13 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.mapper.core.NumberFieldMapper;
 import org.elasticsearch.index.mapper.internal.UidFieldMapper;
 import org.elasticsearch.index.mapper.object.ObjectMapper;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.instanceOf;
 
 // TODO: make this a real unit test
 public class DocumentParserTests extends ESSingleNodeTestCase {
@@ -511,6 +513,15 @@ public class DocumentParserTests extends ESSingleNodeTestCase {
             .endArray().endObject().bytes();
         ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
         assertEquals(4, doc.rootDoc().getFields("foo.bar.baz").length);
+        Mapper fooMapper = doc.dynamicMappingsUpdate().root().getMapper("foo");
+        assertNotNull(fooMapper);
+        assertThat(fooMapper, instanceOf(ObjectMapper.class));
+        Mapper barMapper = ((ObjectMapper) fooMapper).getMapper("bar");
+        assertNotNull(barMapper);
+        assertThat(barMapper, instanceOf(ObjectMapper.class));
+        Mapper bazMapper = ((ObjectMapper) barMapper).getMapper("baz");
+        assertNotNull(bazMapper);
+        assertThat(bazMapper, instanceOf(NumberFieldMapper.class));
     }
 
     public void testDynamicDottedFieldNameLongArrayWithParentTemplate() throws Exception {
@@ -529,6 +540,59 @@ public class DocumentParserTests extends ESSingleNodeTestCase {
             .endArray().endObject().bytes();
         ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
         assertEquals(4, doc.rootDoc().getFields("foo.bar.baz").length);
+        Mapper fooMapper = doc.dynamicMappingsUpdate().root().getMapper("foo");
+        assertNotNull(fooMapper);
+        assertThat(fooMapper, instanceOf(ObjectMapper.class));
+        Mapper barMapper = ((ObjectMapper) fooMapper).getMapper("bar");
+        assertNotNull(barMapper);
+        assertThat(barMapper, instanceOf(ObjectMapper.class));
+        Mapper bazMapper = ((ObjectMapper) barMapper).getMapper("baz");
+        assertNotNull(bazMapper);
+        assertThat(bazMapper, instanceOf(NumberFieldMapper.class));
+    }
+
+    public void testDynamicDottedFieldNameLongArrayWithExistingParent() throws Exception {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties") .startObject("foo")
+            .field("type", "object")
+            .endObject().endObject().endObject().endObject().string();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = XContentFactory.jsonBuilder()
+                .startObject().startArray("foo.bar.baz")
+                .value(0)
+                .value(1)
+            .endArray().endObject().bytes();
+        ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
+        assertEquals(4, doc.rootDoc().getFields("foo.bar.baz").length);
+        Mapper fooMapper = doc.dynamicMappingsUpdate().root().getMapper("foo");
+        assertNotNull(fooMapper);
+        assertThat(fooMapper, instanceOf(ObjectMapper.class));
+        Mapper barMapper = ((ObjectMapper) fooMapper).getMapper("bar");
+        assertNotNull(barMapper);
+        assertThat(barMapper, instanceOf(ObjectMapper.class));
+        Mapper bazMapper = ((ObjectMapper) barMapper).getMapper("baz");
+        assertNotNull(bazMapper);
+        assertThat(bazMapper, instanceOf(NumberFieldMapper.class));
+    }
+
+    public void testDynamicDottedFieldNameLongArrayWithExistingParentWrongType() throws Exception {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties") .startObject("foo")
+            .field("type", "long")
+            .endObject().endObject().endObject().endObject().string();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = XContentFactory.jsonBuilder()
+                .startObject().startArray("foo.bar.baz")
+                .value(0)
+                .value(1)
+            .endArray().endObject().bytes();
+        MapperParsingException exception = expectThrows(MapperParsingException.class, () -> mapper.parse("test", "type", "1", bytes));
+        assertEquals("Could not dynamically add mapping for field [foo.bar.baz]. "
+                + "Existing mapping for [foo] must be of type object but found [long].", exception.getMessage());
     }
 
     public void testDynamicFalseDottedFieldNameLongArray() throws Exception {
@@ -573,6 +637,15 @@ public class DocumentParserTests extends ESSingleNodeTestCase {
             .endObject().bytes();
         ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
         assertEquals(2, doc.rootDoc().getFields("foo.bar.baz").length);
+        Mapper fooMapper = doc.dynamicMappingsUpdate().root().getMapper("foo");
+        assertNotNull(fooMapper);
+        assertThat(fooMapper, instanceOf(ObjectMapper.class));
+        Mapper barMapper = ((ObjectMapper) fooMapper).getMapper("bar");
+        assertNotNull(barMapper);
+        assertThat(barMapper, instanceOf(ObjectMapper.class));
+        Mapper bazMapper = ((ObjectMapper) barMapper).getMapper("baz");
+        assertNotNull(bazMapper);
+        assertThat(bazMapper, instanceOf(NumberFieldMapper.class));
     }
 
     public void testDynamicDottedFieldNameLongWithParentTemplate() throws Exception {
@@ -589,6 +662,55 @@ public class DocumentParserTests extends ESSingleNodeTestCase {
             .endObject().bytes();
         ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
         assertEquals(2, doc.rootDoc().getFields("foo.bar.baz").length);
+        Mapper fooMapper = doc.dynamicMappingsUpdate().root().getMapper("foo");
+        assertNotNull(fooMapper);
+        assertThat(fooMapper, instanceOf(ObjectMapper.class));
+        Mapper barMapper = ((ObjectMapper) fooMapper).getMapper("bar");
+        assertNotNull(barMapper);
+        assertThat(barMapper, instanceOf(ObjectMapper.class));
+        Mapper bazMapper = ((ObjectMapper) barMapper).getMapper("baz");
+        assertNotNull(bazMapper);
+        assertThat(bazMapper, instanceOf(NumberFieldMapper.class));
+    }
+
+    public void testDynamicDottedFieldNameLongWithExistingParent() throws Exception {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties") .startObject("foo")
+            .field("type", "object")
+            .endObject().endObject().endObject().endObject().string();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = XContentFactory.jsonBuilder()
+                .startObject().field("foo.bar.baz", 0)
+            .endObject().bytes();
+        ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
+        assertEquals(2, doc.rootDoc().getFields("foo.bar.baz").length);
+        Mapper fooMapper = doc.dynamicMappingsUpdate().root().getMapper("foo");
+        assertNotNull(fooMapper);
+        assertThat(fooMapper, instanceOf(ObjectMapper.class));
+        Mapper barMapper = ((ObjectMapper) fooMapper).getMapper("bar");
+        assertNotNull(barMapper);
+        assertThat(barMapper, instanceOf(ObjectMapper.class));
+        Mapper bazMapper = ((ObjectMapper) barMapper).getMapper("baz");
+        assertNotNull(bazMapper);
+        assertThat(bazMapper, instanceOf(NumberFieldMapper.class));
+    }
+
+    public void testDynamicDottedFieldNameLongWithExistingParentWrongType() throws Exception {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties") .startObject("foo")
+            .field("type", "long")
+            .endObject().endObject().endObject().endObject().string();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = XContentFactory.jsonBuilder()
+                .startObject().field("foo.bar.baz", 0)
+            .endObject().bytes();
+        MapperParsingException exception = expectThrows(MapperParsingException.class, () -> mapper.parse("test", "type", "1", bytes));
+        assertEquals("Could not dynamically add mapping for field [foo.bar.baz]. "
+                + "Existing mapping for [foo] must be of type object but found [long].", exception.getMessage());
     }
 
     public void testDynamicFalseDottedFieldNameLong() throws Exception {
@@ -630,6 +752,18 @@ public class DocumentParserTests extends ESSingleNodeTestCase {
             .endObject().endObject().bytes();
         ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
         assertEquals(2, doc.rootDoc().getFields("foo.bar.baz.a").length);
+        Mapper fooMapper = doc.dynamicMappingsUpdate().root().getMapper("foo");
+        assertNotNull(fooMapper);
+        assertThat(fooMapper, instanceOf(ObjectMapper.class));
+        Mapper barMapper = ((ObjectMapper) fooMapper).getMapper("bar");
+        assertNotNull(barMapper);
+        assertThat(barMapper, instanceOf(ObjectMapper.class));
+        Mapper bazMapper = ((ObjectMapper) barMapper).getMapper("baz");
+        assertNotNull(bazMapper);
+        assertThat(bazMapper, instanceOf(ObjectMapper.class));
+        Mapper aMapper = ((ObjectMapper) bazMapper).getMapper("a");
+        assertNotNull(aMapper);
+        assertThat(aMapper, instanceOf(NumberFieldMapper.class));
     }
 
     public void testDynamicDottedFieldNameObjectWithParentTemplate() throws Exception {
@@ -647,6 +781,57 @@ public class DocumentParserTests extends ESSingleNodeTestCase {
             .endObject().endObject().bytes();
         ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
         assertEquals(2, doc.rootDoc().getFields("foo.bar.baz.a").length);
+        Mapper fooMapper = doc.dynamicMappingsUpdate().root().getMapper("foo");
+        assertNotNull(fooMapper);
+        assertThat(fooMapper, instanceOf(ObjectMapper.class));
+        Mapper barMapper = ((ObjectMapper) fooMapper).getMapper("bar");
+        assertNotNull(barMapper);
+        assertThat(barMapper, instanceOf(ObjectMapper.class));
+        Mapper bazMapper = ((ObjectMapper) barMapper).getMapper("baz");
+        assertNotNull(bazMapper);
+        assertThat(bazMapper, instanceOf(ObjectMapper.class));
+        Mapper aMapper = ((ObjectMapper) bazMapper).getMapper("a");
+        assertNotNull(aMapper);
+        assertThat(aMapper, instanceOf(NumberFieldMapper.class));
+    }
+
+    public void testDynamicDottedFieldNameObjectWithExistingParent() throws Exception {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties").startObject("foo")
+                .field("type", "object").endObject().endObject().endObject().endObject().string();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = XContentFactory.jsonBuilder().startObject().startObject("foo.bar.baz").field("a", 0).endObject().endObject()
+                .bytes();
+        ParsedDocument doc = mapper.parse("test", "type", "1", bytes);
+        assertEquals(2, doc.rootDoc().getFields("foo.bar.baz.a").length);
+        Mapper fooMapper = doc.dynamicMappingsUpdate().root().getMapper("foo");
+        assertNotNull(fooMapper);
+        assertThat(fooMapper, instanceOf(ObjectMapper.class));
+        Mapper barMapper = ((ObjectMapper) fooMapper).getMapper("bar");
+        assertNotNull(barMapper);
+        assertThat(barMapper, instanceOf(ObjectMapper.class));
+        Mapper bazMapper = ((ObjectMapper) barMapper).getMapper("baz");
+        assertNotNull(bazMapper);
+        assertThat(bazMapper, instanceOf(ObjectMapper.class));
+        Mapper aMapper = ((ObjectMapper) bazMapper).getMapper("a");
+        assertNotNull(aMapper);
+        assertThat(aMapper, instanceOf(NumberFieldMapper.class));
+    }
+
+    public void testDynamicDottedFieldNameObjectWithExistingParentWrongType() throws Exception {
+        DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("properties") .startObject("foo")
+            .field("type", "long")
+            .endObject().endObject().endObject().endObject().string();
+        DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
+
+        BytesReference bytes = XContentFactory.jsonBuilder().startObject().startObject("foo.bar.baz").field("a", 0).endObject().endObject()
+                .bytes();
+        MapperParsingException exception = expectThrows(MapperParsingException.class, () -> mapper.parse("test", "type", "1", bytes));
+        assertEquals("Could not dynamically add mapping for field [foo.bar.baz]. "
+                + "Existing mapping for [foo] must be of type object but found [long].", exception.getMessage());
     }
 
     public void testDynamicFalseDottedFieldNameObject() throws Exception {

From f569576c5b2e0ddac3dfc9fb489014d321e1e38b Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 13 May 2016 17:03:34 -0400
Subject: [PATCH 0269/1311] Switch default batch size for reindex to 1000

---
 docs/reference/docs/reindex.asciidoc                      | 8 ++++----
 docs/reference/docs/update-by-query.asciidoc              | 8 ++++----
 .../index/reindex/AbstractBulkByScrollRequest.java        | 2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index 49ee8dcd283..790476990d9 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -30,7 +30,7 @@ That will return something like this:
   "timed_out": false,
   "created": 120,
   "updated": 0,
-  "batches": 2,
+  "batches": 1,
   "version_conflicts": 0,
   "noops": 0,
   "retries": 0,
@@ -301,7 +301,7 @@ POST _reindex
 // CONSOLE
 // TEST[s/^/PUT source\nGET _cluster\/health?wait_for_status=yellow\n/]
 
-By default `_reindex` uses scroll batches of 100. You can change the
+By default `_reindex` uses scroll batches of 1000. You can change the
 batch size with the `size` field in the `source` element:
 
 [source,js]
@@ -310,7 +310,7 @@ POST _reindex
 {
   "source": {
     "index": "source",
-    "size": 1000
+    "size": 100
   },
   "dest": {
     "index": "dest",
@@ -465,7 +465,7 @@ The responses looks like:
             "updated" : 3500,
             "created" : 0,
             "deleted" : 0,
-            "batches" : 36,
+            "batches" : 4,
             "version_conflicts" : 0,
             "noops" : 0,
             "retries": 0,
diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc
index f049e302b34..c3eb0908398 100644
--- a/docs/reference/docs/update-by-query.asciidoc
+++ b/docs/reference/docs/update-by-query.asciidoc
@@ -23,7 +23,7 @@ That will return something like this:
   "took" : 147,
   "timed_out": false,
   "updated": 120,
-  "batches": 2,
+  "batches": 1,
   "version_conflicts": 0,
   "noops": 0,
   "retries": 0,
@@ -146,12 +146,12 @@ POST twitter/_update_by_query?routing=1
 // CONSOLE
 // TEST[setup:twitter]
 
-By default `_update_by_query` uses scroll batches of 100. You can change the
+By default `_update_by_query` uses scroll batches of 1000. You can change the
 batch size with the `scroll_size` URL parameter:
 
 [source,js]
 --------------------------------------------------
-POST twitter/_update_by_query?scroll_size=1000
+POST twitter/_update_by_query?scroll_size=100
 --------------------------------------------------
 // CONSOLE
 // TEST[setup:twitter]
@@ -296,7 +296,7 @@ The responses looks like:
             "updated" : 3500,
             "created" : 0,
             "deleted" : 0,
-            "batches" : 36,
+            "batches" : 4,
             "version_conflicts" : 0,
             "noops" : 0,
             "retries": 0,
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
index a5ec1962a55..14790e97acc 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
@@ -42,7 +42,7 @@ public abstract class AbstractBulkByScrollRequest<Self extends AbstractBulkByScr
         extends ActionRequest<Self> {
     public static final int SIZE_ALL_MATCHES = -1;
     private static final TimeValue DEFAULT_SCROLL_TIMEOUT = timeValueMinutes(5);
-    private static final int DEFAULT_SCROLL_SIZE = 100;
+    private static final int DEFAULT_SCROLL_SIZE = 1000;
 
     /**
      * The search to be executed.

From 07e4f416e63a976b8c81322e99408f7d3830ae8d Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Mon, 16 May 2016 15:52:14 +0200
Subject: [PATCH 0270/1311] painless: Small cleanup of Debugger class to use
 StringWriter

---
 .../org/elasticsearch/painless/Debugger.java    | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
index f037903f52d..3e973997229 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
@@ -22,11 +22,8 @@ package org.elasticsearch.painless;
 import org.objectweb.asm.ClassReader;
 import org.objectweb.asm.util.TraceClassVisitor;
 
-import java.io.ByteArrayOutputStream;
-import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
-import java.io.UnsupportedEncodingException;
-import java.nio.charset.StandardCharsets;
+import java.io.StringWriter;
 
 /** quick and dirty tools for debugging */
 final class Debugger {
@@ -39,17 +36,13 @@ final class Debugger {
     /** compiles to bytecode, and returns debugging output */
     static String toString(String source, CompilerSettings settings) {
         final byte[] bytes = Compiler.compile("<debugging>", source, settings);
-        final ByteArrayOutputStream output = new ByteArrayOutputStream();
-        final PrintWriter outputWriter = new PrintWriter(new OutputStreamWriter(output, StandardCharsets.UTF_8));
+        final StringWriter output = new StringWriter();
+        final PrintWriter outputWriter = new PrintWriter(output);
         final ClassReader reader = new ClassReader(bytes);
 
         reader.accept(new TraceClassVisitor(outputWriter), 0);
         outputWriter.flush();
-
-        try {
-            return output.toString("UTF-8");
-        } catch (final UnsupportedEncodingException e) {
-            throw new RuntimeException(e);
-        }
+        
+        return output.toString();
     }
 }

From 2fc7ca46e1d2e433cc1d9295fc228b9412074297 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 16 May 2016 09:56:29 -0400
Subject: [PATCH 0271/1311] expand whitelist for basic types

---
 .../elasticsearch/painless/Definition.java    | 79 +++++++++++++++----
 1 file changed, 64 insertions(+), 15 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index e2f312be7b4..b13702a67c8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -646,50 +646,106 @@ public final class Definition {
     }
 
     private void addElements() {
-        addMethod("Object", "toString", null, false, stringType, new Type[] {}, null, null);
         addMethod("Object", "equals", null, false, booleanType, new Type[] {objectType}, null, null);
         addMethod("Object", "hashCode", null, false, intType, new Type[] {}, null, null);
+        addMethod("Object", "toString", null, false, stringType, new Type[] {}, null, null);
 
-        addMethod("def", "toString", null, false, stringType, new Type[] {}, null, null);
         addMethod("def", "equals", null, false, booleanType, new Type[] {objectType}, null, null);
         addMethod("def", "hashCode", null, false, intType, new Type[] {}, null, null);
+        addMethod("def", "toString", null, false, stringType, new Type[] {}, null, null);
 
         addConstructor("Boolean", "new", new Type[] {booleanType}, null);
-        addMethod("Boolean", "valueOf", null, true, booleanobjType, new Type[] {booleanType}, null, null);
         addMethod("Boolean", "booleanValue", null, false, booleanType, new Type[] {}, null, null);
+        addMethod("Boolean", "compare", null, true, intType, new Type[] {booleanType,booleanType}, null, null);
+        addMethod("Boolean", "compareTo", null, false, intType, new Type[] {booleanobjType}, null, null);
+        addMethod("Boolean", "parseBoolean", null, true, booleanType, new Type[] {stringType}, null, null);
+        addMethod("Boolean", "valueOf", null, true, booleanobjType, new Type[] {booleanType}, null, null);
+        addField("Boolean", "FALSE", null, true, booleanobjType, null);
+        addField("Boolean", "TRUE", null, true, booleanobjType, null);
 
         addConstructor("Byte", "new", new Type[] {byteType}, null);
+        addMethod("Byte", "compare", null, true, intType, new Type[] {byteType,byteType}, null, null);
+        addMethod("Byte", "compareTo", null, false, intType, new Type[] {byteobjType}, null, null);
+        addMethod("Byte", "parseByte", null, true, byteType, new Type[] {stringType}, null, null);
         addMethod("Byte", "valueOf", null, true, byteobjType, new Type[] {byteType}, null, null);
         addField("Byte", "MIN_VALUE", null, true, byteType, null);
         addField("Byte", "MAX_VALUE", null, true, byteType, null);
 
         addConstructor("Short", "new", new Type[] {shortType}, null);
+        addMethod("Short", "compare", null, true, intType, new Type[] {shortType,shortType}, null, null);
+        addMethod("Short", "compareTo", null, false, intType, new Type[] {shortobjType}, null, null);
+        addMethod("Short", "parseShort", null, true, shortType, new Type[] {stringType}, null, null);
         addMethod("Short", "valueOf", null, true, shortobjType, new Type[] {shortType}, null, null);
         addField("Short", "MIN_VALUE", null, true, shortType, null);
         addField("Short", "MAX_VALUE", null, true, shortType, null);
 
         addConstructor("Character", "new", new Type[] {charType}, null);
-        addMethod("Character", "valueOf", null, true, charobjType, new Type[] {charType}, null, null);
+        addMethod("Character", "charCount", null, true, intType, new Type[] {intType}, null, null);
         addMethod("Character", "charValue", null, false, charType, new Type[] {}, null, null);
+        addMethod("Character", "compare", null, true, intType, new Type[] {charType,charType}, null, null);
+        addMethod("Character", "compareTo", null, false, intType, new Type[] {charobjType}, null, null);
+        addMethod("Character", "digit", null, true, intType, new Type[] {intType,intType}, null, null);
+        addMethod("Character", "forDigit", null, true, charType, new Type[] {intType,intType}, null, null);
+        addMethod("Character", "getName", null, true, stringType, new Type[] {intType}, null, null);
+        addMethod("Character", "getNumericValue", null, true, intType, new Type[] {intType}, null, null);
+        addMethod("Character", "isAlphabetic", null, true, booleanType, new Type[] {intType}, null, null);
+        addMethod("Character", "isDefined", null, true, booleanType, new Type[] {intType}, null, null);
+        addMethod("Character", "isDigit", null, true, booleanType, new Type[] {intType}, null, null);
+        addMethod("Character", "isIdeographic", null, true, booleanType, new Type[] {intType}, null, null);
+        addMethod("Character", "isLetter", null, true, booleanType, new Type[] {intType}, null, null);
+        addMethod("Character", "isLetterOrDigit", null, true, booleanType, new Type[] {intType}, null, null);
+        addMethod("Character", "isLowerCase", null, true, booleanType, new Type[] {intType}, null, null);
+        addMethod("Character", "isMirrored", null, true, booleanType, new Type[] {intType}, null, null);
+        addMethod("Character", "isSpaceChar", null, true, booleanType, new Type[] {intType}, null, null);
+        addMethod("Character", "isTitleCase", null, true, booleanType, new Type[] {intType}, null, null);
+        addMethod("Character", "isUpperCase", null, true, booleanType, new Type[] {intType}, null, null);
+        addMethod("Character", "isWhitespace", null, true, booleanType, new Type[] {intType}, null, null);
+        addMethod("Character", "valueOf", null, true, charobjType, new Type[] {charType}, null, null);
         addField("Character", "MIN_VALUE", null, true, charType, null);
         addField("Character", "MAX_VALUE", null, true, charType, null);
 
         addConstructor("Integer", "new", new Type[] {intType}, null);
+        addMethod("Integer", "compare", null, true, intType, new Type[] {intType,intType}, null, null);
+        addMethod("Integer", "compareTo", null, false, intType, new Type[] {intobjType}, null, null);
+        addMethod("Integer", "min", null, true, intType, new Type[] {intType,intType}, null, null);
+        addMethod("Integer", "max", null, true, intType, new Type[] {intType,intType}, null, null);
+        addMethod("Integer", "parseInt", null, true, intType, new Type[] {stringType}, null, null);
+        addMethod("Integer", "signum", null, true, intType, new Type[] {intType}, null, null);
+        addMethod("Integer", "toHexString", null, true, stringType, new Type[] {intType}, null, null);
         addMethod("Integer", "valueOf", null, true, intobjType, new Type[] {intType}, null, null);
         addField("Integer", "MIN_VALUE", null, true, intType, null);
         addField("Integer", "MAX_VALUE", null, true, intType, null);
 
         addConstructor("Long", "new", new Type[] {longType}, null);
+        addMethod("Long", "compare", null, true, intType, new Type[] {longType,longType}, null, null);
+        addMethod("Long", "compareTo", null, false, intType, new Type[] {longobjType}, null, null);
+        addMethod("Long", "min", null, true, longType, new Type[] {longType,longType}, null, null);
+        addMethod("Long", "max", null, true, longType, new Type[] {longType,longType}, null, null);
+        addMethod("Long", "parseLong", null, true, longType, new Type[] {stringType}, null, null);
+        addMethod("Long", "signum", null, true, intType, new Type[] {longType}, null, null);
+        addMethod("Long", "toHexString", null, true, stringType, new Type[] {longType}, null, null);
         addMethod("Long", "valueOf", null, true, longobjType, new Type[] {longType}, null, null);
         addField("Long", "MIN_VALUE", null, true, longType, null);
         addField("Long", "MAX_VALUE", null, true, longType, null);
 
         addConstructor("Float", "new", new Type[] {floatType}, null);
+        addMethod("Float", "compare", null, true, intType, new Type[] {floatType,floatType}, null, null);
+        addMethod("Float", "compareTo", null, false, intType, new Type[] {floatobjType}, null, null);
+        addMethod("Float", "min", null, true, floatType, new Type[] {floatType,floatType}, null, null);
+        addMethod("Float", "max", null, true, floatType, new Type[] {floatType,floatType}, null, null);
+        addMethod("Float", "parseFloat", null, true, floatType, new Type[] {stringType}, null, null);
+        addMethod("Float", "toHexString", null, true, stringType, new Type[] {floatType}, null, null);
         addMethod("Float", "valueOf", null, true, floatobjType, new Type[] {floatType}, null, null);
         addField("Float", "MIN_VALUE", null, true, floatType, null);
         addField("Float", "MAX_VALUE", null, true, floatType, null);
 
         addConstructor("Double", "new", new Type[] {doubleType}, null);
+        addMethod("Double", "compare", null, true, intType, new Type[] {doubleType,doubleType}, null, null);
+        addMethod("Double", "compareTo", null, false, intType, new Type[] {doubleobjType}, null, null);
+        addMethod("Double", "min", null, true, doubleType, new Type[] {doubleType,doubleType}, null, null);
+        addMethod("Double", "max", null, true, doubleType, new Type[] {doubleType,doubleType}, null, null);
+        addMethod("Double", "parseDouble", null, true, doubleType, new Type[] {stringType}, null, null);
+        addMethod("Double", "toHexString", null, true, stringType, new Type[] {doubleType}, null, null);
         addMethod("Double", "valueOf", null, true, doubleobjType, new Type[] {doubleType}, null, null);
         addField("Double", "MIN_VALUE", null, true, doubleType, null);
         addField("Double", "MAX_VALUE", null, true, doubleType, null);
@@ -830,9 +886,6 @@ public final class Definition {
         addMethod("Utility", "StringToCharacter", null, true, charobjType, new Type[] {stringType}, null, null);
 
         addMethod("Math", "abs", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "fabs", "abs", true, floatType, new Type[] {floatType}, null, null);
-        addMethod("Math", "labs", "abs", true, longType, new Type[] {longType}, null, null);
-        addMethod("Math", "iabs", "abs", true, intType, new Type[] {intType}, null, null);
         addMethod("Math", "acos", null, true, doubleType, new Type[] {doubleType}, null, null);
         addMethod("Math", "asin", null, true, doubleType, new Type[] {doubleType}, null, null);
         addMethod("Math", "atan", null, true, doubleType, new Type[] {doubleType}, null, null);
@@ -848,14 +901,8 @@ public final class Definition {
         addMethod("Math", "log", null, true, doubleType, new Type[] {doubleType}, null, null);
         addMethod("Math", "log10", null, true, doubleType, new Type[] {doubleType}, null, null);
         addMethod("Math", "log1p", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "max", "max", true, doubleType, new Type[] {doubleType, doubleType}, null, null);
-        addMethod("Math", "fmax", "max", true, floatType, new Type[] {floatType, floatType}, null, null);
-        addMethod("Math", "lmax", "max", true, longType, new Type[] {longType, longType}, null, null);
-        addMethod("Math", "imax", "max", true, intType, new Type[] {intType, intType}, null, null);
-        addMethod("Math", "min", "min", true, doubleType, new Type[] {doubleType, doubleType}, null, null);
-        addMethod("Math", "fmin", "min", true, floatType, new Type[] {floatType, floatType}, null, null);
-        addMethod("Math", "lmin", "min", true, longType, new Type[] {longType, longType}, null, null);
-        addMethod("Math", "imin", "min", true, intType, new Type[] {intType, intType}, null, null);
+        addMethod("Math", "max", null, true, doubleType, new Type[] {doubleType, doubleType}, null, null);
+        addMethod("Math", "min", null, true, doubleType, new Type[] {doubleType, doubleType}, null, null);
         addMethod("Math", "pow", null, true, doubleType, new Type[] {doubleType, doubleType}, null, null);
         addMethod("Math", "random", null, true, doubleType, new Type[] {}, null, null);
         addMethod("Math", "rint", null, true, doubleType, new Type[] {doubleType}, null, null);
@@ -867,6 +914,8 @@ public final class Definition {
         addMethod("Math", "tanh", null, true, doubleType, new Type[] {doubleType}, null, null);
         addMethod("Math", "toDegrees", null, true, doubleType, new Type[] {doubleType}, null, null);
         addMethod("Math", "toRadians", null, true, doubleType, new Type[] {doubleType}, null, null);
+        addField("Math", "E", null, true, doubleType, null);
+        addField("Math", "PI", null, true, doubleType, null);
 
         addMethod("Def", "DefToboolean", null, true, booleanType, new Type[] {defType}, null, null);
         addMethod("Def", "DefTobyte", null, true, byteType, new Type[] {defType}, null, null);

From 5a33fdbac7b374329fe173197b1d766589d7e9f7 Mon Sep 17 00:00:00 2001
From: Rodolfo Martinez <martinez@rodol.fo>
Date: Mon, 16 May 2016 15:29:06 +0100
Subject: [PATCH 0272/1311] Fix response format of /customer/external/1?pretty

The response format is displaying elements "found" and "source" in the
same lines and they should appear on their own separate lines.

Closes #18374
---
 docs/reference/getting-started.asciidoc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc
index f154e5d7a66..977cb4e5a1d 100755
--- a/docs/reference/getting-started.asciidoc
+++ b/docs/reference/getting-started.asciidoc
@@ -326,7 +326,8 @@ curl -XGET 'localhost:9200/customer/external/1?pretty'
   "_type" : "external",
   "_id" : "1",
   "_version" : 1,
-  "found" : true, "_source" : { "name": "John Doe" }
+  "found" : true,
+  "_source" : { "name": "John Doe" }
 }
 --------------------------------------------------
 

From e37e8af5e2a84e5f7538fed99cb207b4cd140d8d Mon Sep 17 00:00:00 2001
From: Colin Goodheart-Smithe <colings86@users.noreply.github.com>
Date: Mon, 16 May 2016 13:40:13 +0100
Subject: [PATCH 0273/1311] Refactor of query profile classes to make way for
 other profile implementations

---
 .../controller/SearchPhaseController.java     |  6 +--
 .../search/internal/ContextIndexSearcher.java | 13 +++---
 .../internal/InternalSearchResponse.java      |  8 ++--
 ...own.java => AbstractProfileBreakdown.java} | 35 +++++----------
 ...ree.java => InternalQueryProfileTree.java} | 20 ++++-----
 .../search/profile/ProfileResult.java         | 32 ++++++-------
 .../search/profile/ProfileScorer.java         | 18 ++++----
 .../search/profile/ProfileShardResult.java    | 18 ++++----
 .../search/profile/ProfileWeight.java         |  6 +--
 .../search/profile/Profilers.java             | 16 +++----
 .../search/profile/QueryProfileBreakdown.java | 33 ++++++++++++++
 .../{Profiler.java => QueryProfiler.java}     | 28 +++---------
 .../search/profile/QueryTimingType.java       | 36 +++++++++++++++
 ...ts.java => SearchProfileShardResults.java} | 24 ++++++++--
 .../search/query/QueryPhase.java              |  8 ++--
 .../search/profile/ProfileTests.java          | 45 +++++++++----------
 16 files changed, 202 insertions(+), 144 deletions(-)
 rename core/src/main/java/org/elasticsearch/search/profile/{ProfileBreakdown.java => AbstractProfileBreakdown.java} (78%)
 rename core/src/main/java/org/elasticsearch/search/profile/{InternalProfileTree.java => InternalQueryProfileTree.java} (92%)
 create mode 100644 core/src/main/java/org/elasticsearch/search/profile/QueryProfileBreakdown.java
 rename core/src/main/java/org/elasticsearch/search/profile/{Profiler.java => QueryProfiler.java} (75%)
 create mode 100644 core/src/main/java/org/elasticsearch/search/profile/QueryTimingType.java
 rename core/src/main/java/org/elasticsearch/search/profile/{InternalProfileShardResults.java => SearchProfileShardResults.java} (78%)

diff --git a/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java b/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java
index a8ecc7a508c..36fe562a568 100644
--- a/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java
+++ b/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java
@@ -51,7 +51,7 @@ import org.elasticsearch.search.fetch.FetchSearchResultProvider;
 import org.elasticsearch.search.internal.InternalSearchHit;
 import org.elasticsearch.search.internal.InternalSearchHits;
 import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.search.profile.InternalProfileShardResults;
+import org.elasticsearch.search.profile.SearchProfileShardResults;
 import org.elasticsearch.search.profile.ProfileShardResult;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.query.QuerySearchResultProvider;
@@ -405,14 +405,14 @@ public class SearchPhaseController extends AbstractComponent {
         }
 
         //Collect profile results
-        InternalProfileShardResults shardResults = null;
+        SearchProfileShardResults shardResults = null;
         if (!queryResults.isEmpty() && firstResult.profileResults() != null) {
             Map<String, List<ProfileShardResult>> profileResults = new HashMap<>(queryResults.size());
             for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : queryResults) {
                 String key = entry.value.queryResult().shardTarget().toString();
                 profileResults.put(key, entry.value.queryResult().profileResults());
             }
-            shardResults = new InternalProfileShardResults(profileResults);
+            shardResults = new SearchProfileShardResults(profileResults);
         }
 
         if (aggregations != null) {
diff --git a/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
index c4df30716c7..471ce26d297 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
@@ -33,9 +33,10 @@ import org.apache.lucene.search.Weight;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.search.dfs.AggregatedDfs;
-import org.elasticsearch.search.profile.ProfileBreakdown;
+import org.elasticsearch.search.profile.QueryProfileBreakdown;
 import org.elasticsearch.search.profile.ProfileWeight;
-import org.elasticsearch.search.profile.Profiler;
+import org.elasticsearch.search.profile.QueryProfiler;
+import org.elasticsearch.search.profile.QueryTimingType;
 
 import java.io.IOException;
 
@@ -54,7 +55,7 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable {
     private final Engine.Searcher engineSearcher;
 
     // TODO revisit moving the profiler to inheritance or wrapping model in the future
-    private Profiler profiler;
+    private QueryProfiler profiler;
 
     public ContextIndexSearcher(Engine.Searcher searcher,
             QueryCache queryCache, QueryCachingPolicy queryCachingPolicy) {
@@ -70,7 +71,7 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable {
     public void close() {
     }
 
-    public void setProfiler(Profiler profiler) {
+    public void setProfiler(QueryProfiler profiler) {
         this.profiler = profiler;
     }
 
@@ -114,8 +115,8 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable {
             // createWeight() is called for each query in the tree, so we tell the queryProfiler
             // each invocation so that it can build an internal representation of the query
             // tree
-            ProfileBreakdown profile = profiler.getQueryBreakdown(query);
-            profile.startTime(ProfileBreakdown.TimingType.CREATE_WEIGHT);
+            QueryProfileBreakdown profile = profiler.getQueryBreakdown(query);
+            profile.startTime(QueryTimingType.CREATE_WEIGHT);
             final Weight weight;
             try {
                 weight = super.createWeight(query, needsScores);
diff --git a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java
index 1a2e1f70191..9c33889dc9c 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java
@@ -28,7 +28,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.profile.InternalProfileShardResults;
+import org.elasticsearch.search.profile.SearchProfileShardResults;
 import org.elasticsearch.search.profile.ProfileShardResult;
 import org.elasticsearch.search.suggest.Suggest;
 
@@ -54,7 +54,7 @@ public class InternalSearchResponse implements Streamable, ToXContent {
 
     private Suggest suggest;
 
-    private InternalProfileShardResults profileResults;
+    private SearchProfileShardResults profileResults;
 
     private boolean timedOut;
 
@@ -64,7 +64,7 @@ public class InternalSearchResponse implements Streamable, ToXContent {
     }
 
     public InternalSearchResponse(InternalSearchHits hits, InternalAggregations aggregations, Suggest suggest,
-                                  InternalProfileShardResults profileResults, boolean timedOut, Boolean terminatedEarly) {
+                                  SearchProfileShardResults profileResults, boolean timedOut, Boolean terminatedEarly) {
         this.hits = hits;
         this.aggregations = aggregations;
         this.suggest = suggest;
@@ -141,7 +141,7 @@ public class InternalSearchResponse implements Streamable, ToXContent {
         terminatedEarly = in.readOptionalBoolean();
 
         if (in.getVersion().onOrAfter(Version.V_2_2_0) && in.readBoolean()) {
-            profileResults = new InternalProfileShardResults(in);
+            profileResults = new SearchProfileShardResults(in);
         } else {
             profileResults = null;
         }
diff --git a/core/src/main/java/org/elasticsearch/search/profile/ProfileBreakdown.java b/core/src/main/java/org/elasticsearch/search/profile/AbstractProfileBreakdown.java
similarity index 78%
rename from core/src/main/java/org/elasticsearch/search/profile/ProfileBreakdown.java
rename to core/src/main/java/org/elasticsearch/search/profile/AbstractProfileBreakdown.java
index a992f819874..f5cfc78d79e 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/ProfileBreakdown.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/AbstractProfileBreakdown.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.profile;
 
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.Locale;
 import java.util.Map;
 
 /**
@@ -29,22 +28,7 @@ import java.util.Map;
  * A node's time may be composed of several internal attributes (rewriting, weighting,
  * scoring, etc).
  */
-public final class ProfileBreakdown {
-
-    /** Enumeration of all supported timing types. */
-    public enum TimingType {
-        CREATE_WEIGHT,
-        BUILD_SCORER,
-        NEXT_DOC,
-        ADVANCE,
-        MATCH,
-        SCORE;
-
-        @Override
-        public String toString() {
-            return name().toLowerCase(Locale.ROOT);
-        }
-    }
+public abstract class AbstractProfileBreakdown<T extends Enum<T>> {
 
     /**
      * The accumulated timings for this query node
@@ -52,23 +36,26 @@ public final class ProfileBreakdown {
     private final long[] timings;
 
     /** Scratch to store the current timing type. */
-    private TimingType currentTimingType;
+    private T currentTimingType;
 
     /**
      * The temporary scratch space for holding start-times
      */
     private long scratch;
 
+    private T[] timingTypes;
+
     /** Sole constructor. */
-    public ProfileBreakdown() {
-        timings = new long[TimingType.values().length];
+    public AbstractProfileBreakdown(T[] timingTypes) {
+        this.timingTypes = timingTypes;
+        timings = new long[timingTypes.length];
     }
 
     /**
      * Begin timing a query for a specific Timing context
      * @param timing    The timing context being profiled
      */
-    public void startTime(TimingType timing) {
+    public void startTime(T timing) {
         assert currentTimingType == null;
         assert scratch == 0;
         currentTimingType = timing;
@@ -91,10 +78,10 @@ public final class ProfileBreakdown {
         return time;
     }
 
-    /** Convert this record to a map from {@link TimingType} to times. */
+    /** Convert this record to a map from timingType to times. */
     public Map<String, Long> toTimingMap() {
         Map<String, Long> map = new HashMap<>();
-        for (TimingType timingType : TimingType.values()) {
+        for (T timingType : timingTypes) {
             map.put(timingType.toString(), timings[timingType.ordinal()]);
         }
         return Collections.unmodifiableMap(map);
@@ -104,7 +91,7 @@ public final class ProfileBreakdown {
      * Add <code>other</code>'s timings into this breakdown
      * @param other Another Breakdown to merge with this one
      */
-    public void merge(ProfileBreakdown other) {
+    public void merge(AbstractProfileBreakdown<T> other) {
         assert(timings.length == other.timings.length);
         for (int i = 0; i < timings.length; ++i) {
             timings[i] += other.timings[i];
diff --git a/core/src/main/java/org/elasticsearch/search/profile/InternalProfileTree.java b/core/src/main/java/org/elasticsearch/search/profile/InternalQueryProfileTree.java
similarity index 92%
rename from core/src/main/java/org/elasticsearch/search/profile/InternalProfileTree.java
rename to core/src/main/java/org/elasticsearch/search/profile/InternalQueryProfileTree.java
index d7afd9c2fa0..9ebee2f1635 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/InternalProfileTree.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/InternalQueryProfileTree.java
@@ -30,12 +30,12 @@ import java.util.concurrent.LinkedBlockingDeque;
 
 /**
  * This class tracks the dependency tree for queries (scoring and rewriting) and
- * generates {@link ProfileBreakdown} for each node in the tree.  It also finalizes the tree
+ * generates {@link QueryProfileBreakdown} for each node in the tree.  It also finalizes the tree
  * and returns a list of {@link ProfileResult} that can be serialized back to the client
  */
-final class InternalProfileTree {
+final class InternalQueryProfileTree {
 
-    private ArrayList<ProfileBreakdown> timings;
+    private ArrayList<QueryProfileBreakdown> timings;
 
     /** Maps the Query to it's list of children.  This is basically the dependency tree */
     private ArrayList<ArrayList<Integer>> tree;
@@ -55,7 +55,7 @@ final class InternalProfileTree {
 
     private int currentToken = 0;
 
-    public InternalProfileTree() {
+    public InternalQueryProfileTree() {
         timings = new ArrayList<>(10);
         stack = new LinkedBlockingDeque<>(10);
         tree = new ArrayList<>(10);
@@ -64,7 +64,7 @@ final class InternalProfileTree {
     }
 
     /**
-     * Returns a {@link ProfileBreakdown} for a scoring query.  Scoring queries (e.g. those
+     * Returns a {@link QueryProfileBreakdown} for a scoring query.  Scoring queries (e.g. those
      * that are past the rewrite phase and are now being wrapped by createWeight() ) follow
      * a recursive progression.  We can track the dependency tree by a simple stack
      *
@@ -74,7 +74,7 @@ final class InternalProfileTree {
      * @param query The scoring query we wish to profile
      * @return      A ProfileBreakdown for this query
      */
-    public ProfileBreakdown getQueryBreakdown(Query query) {
+    public QueryProfileBreakdown getQueryBreakdown(Query query) {
         int token = currentToken;
 
         boolean stackEmpty = stack.isEmpty();
@@ -131,14 +131,14 @@ final class InternalProfileTree {
      * Helper method to add a new node to the dependency tree.
      *
      * Initializes a new list in the dependency tree, saves the query and
-     * generates a new {@link ProfileBreakdown} to track the timings
+     * generates a new {@link QueryProfileBreakdown} to track the timings
      * of this query
      *
      * @param query             The query to profile
      * @param token             The assigned token for this query
      * @return                  A ProfileBreakdown to profile this query
      */
-    private ProfileBreakdown addDependencyNode(Query query, int token) {
+    private QueryProfileBreakdown addDependencyNode(Query query, int token) {
 
         // Add a new slot in the dependency tree
         tree.add(new ArrayList<>(5));
@@ -146,7 +146,7 @@ final class InternalProfileTree {
         // Save our query for lookup later
         queries.add(query);
 
-        ProfileBreakdown queryTimings = new ProfileBreakdown();
+        QueryProfileBreakdown queryTimings = new QueryProfileBreakdown();
         timings.add(token, queryTimings);
         return queryTimings;
     }
@@ -180,7 +180,7 @@ final class InternalProfileTree {
      */
     private ProfileResult doGetQueryTree(int token) {
         Query query = queries.get(token);
-        ProfileBreakdown breakdown = timings.get(token);
+        QueryProfileBreakdown breakdown = timings.get(token);
         Map<String, Long> timings = breakdown.toTimingMap();
         List<Integer> children = tree.get(token);
         List<ProfileResult> childrenProfileResults = Collections.emptyList();
diff --git a/core/src/main/java/org/elasticsearch/search/profile/ProfileResult.java b/core/src/main/java/org/elasticsearch/search/profile/ProfileResult.java
index dc47f982a3c..165244e8d69 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/ProfileResult.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/ProfileResult.java
@@ -45,22 +45,22 @@ import java.util.Map;
  */
 final class ProfileResult implements Writeable, ToXContent {
 
-    private static final ParseField QUERY_TYPE = new ParseField("query_type");
-    private static final ParseField LUCENE_DESCRIPTION = new ParseField("lucene");
+    private static final ParseField TYPE = new ParseField("type");
+    private static final ParseField DESCRIPTION = new ParseField("description");
     private static final ParseField NODE_TIME = new ParseField("time");
     private static final ParseField CHILDREN = new ParseField("children");
     private static final ParseField BREAKDOWN = new ParseField("breakdown");
 
-    private final String queryType;
-    private final String luceneDescription;
+    private final String type;
+    private final String description;
     private final Map<String, Long> timings;
     private final long nodeTime;
     private final List<ProfileResult> children;
 
-    public ProfileResult(String queryType, String luceneDescription, Map<String, Long> timings, List<ProfileResult> children,
+    public ProfileResult(String type, String description, Map<String, Long> timings, List<ProfileResult> children,
             long nodeTime) {
-        this.queryType = queryType;
-        this.luceneDescription = luceneDescription;
+        this.type = type;
+        this.description = description;
         this.timings = timings;
         this.children = children;
         this.nodeTime = nodeTime;
@@ -70,8 +70,8 @@ final class ProfileResult implements Writeable, ToXContent {
      * Read from a stream.
      */
     public ProfileResult(StreamInput in) throws IOException{
-        this.queryType = in.readString();
-        this.luceneDescription = in.readString();
+        this.type = in.readString();
+        this.description = in.readString();
         this.nodeTime = in.readLong();
 
         int timingsSize = in.readVInt();
@@ -90,8 +90,8 @@ final class ProfileResult implements Writeable, ToXContent {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(queryType);
-        out.writeString(luceneDescription);
+        out.writeString(type);
+        out.writeString(description);
         out.writeLong(nodeTime);            // not Vlong because can be negative
         out.writeVInt(timings.size());
         for (Map.Entry<String, Long> entry : timings.entrySet()) {
@@ -108,14 +108,14 @@ final class ProfileResult implements Writeable, ToXContent {
      * Retrieve the lucene description of this query (e.g. the "explain" text)
      */
     public String getLuceneDescription() {
-        return luceneDescription;
+        return description;
     }
 
     /**
      * Retrieve the name of the query (e.g. "TermQuery")
      */
     public String getQueryName() {
-        return queryType;
+        return type;
     }
 
     /**
@@ -144,9 +144,9 @@ final class ProfileResult implements Writeable, ToXContent {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder = builder.startObject()
-                .field(QUERY_TYPE.getPreferredName(), queryType)
-                .field(LUCENE_DESCRIPTION.getPreferredName(), luceneDescription)
-                .field(NODE_TIME.getPreferredName(), String.format(Locale.US, "%.10gms", (double)(getTime() / 1000000.0)))
+                .field(TYPE.getPreferredName(), type)
+                .field(DESCRIPTION.getPreferredName(), description)
+                .field(NODE_TIME.getPreferredName(), String.format(Locale.US, "%.10gms", getTime() / 1000000.0))
                 .field(BREAKDOWN.getPreferredName(), timings);
 
         if (!children.isEmpty()) {
diff --git a/core/src/main/java/org/elasticsearch/search/profile/ProfileScorer.java b/core/src/main/java/org/elasticsearch/search/profile/ProfileScorer.java
index 972d176ca10..939ac85c871 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/ProfileScorer.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/ProfileScorer.java
@@ -35,9 +35,9 @@ final class ProfileScorer extends Scorer {
 
     private final Scorer scorer;
     private ProfileWeight profileWeight;
-    private final ProfileBreakdown profile;
+    private final QueryProfileBreakdown profile;
 
-    ProfileScorer(ProfileWeight w, Scorer scorer, ProfileBreakdown profile) throws IOException {
+    ProfileScorer(ProfileWeight w, Scorer scorer, QueryProfileBreakdown profile) throws IOException {
         super(w);
         this.scorer = scorer;
         this.profileWeight = w;
@@ -51,7 +51,7 @@ final class ProfileScorer extends Scorer {
 
     @Override
     public float score() throws IOException {
-        profile.startTime(ProfileBreakdown.TimingType.SCORE);
+        profile.startTime(QueryTimingType.SCORE);
         try {
             return scorer.score();
         } finally {
@@ -78,10 +78,10 @@ final class ProfileScorer extends Scorer {
     public DocIdSetIterator iterator() {
         final DocIdSetIterator in = scorer.iterator();
         return new DocIdSetIterator() {
-            
+
             @Override
             public int advance(int target) throws IOException {
-                profile.startTime(ProfileBreakdown.TimingType.ADVANCE);
+                profile.startTime(QueryTimingType.ADVANCE);
                 try {
                     return in.advance(target);
                 } finally {
@@ -91,7 +91,7 @@ final class ProfileScorer extends Scorer {
 
             @Override
             public int nextDoc() throws IOException {
-                profile.startTime(ProfileBreakdown.TimingType.NEXT_DOC);
+                profile.startTime(QueryTimingType.NEXT_DOC);
                 try {
                     return in.nextDoc();
                 } finally {
@@ -122,7 +122,7 @@ final class ProfileScorer extends Scorer {
 
             @Override
             public int advance(int target) throws IOException {
-                profile.startTime(ProfileBreakdown.TimingType.ADVANCE);
+                profile.startTime(QueryTimingType.ADVANCE);
                 try {
                     return inApproximation.advance(target);
                 } finally {
@@ -132,7 +132,7 @@ final class ProfileScorer extends Scorer {
 
             @Override
             public int nextDoc() throws IOException {
-                profile.startTime(ProfileBreakdown.TimingType.NEXT_DOC);
+                profile.startTime(QueryTimingType.NEXT_DOC);
                 try {
                     return inApproximation.nextDoc();
                 } finally {
@@ -153,7 +153,7 @@ final class ProfileScorer extends Scorer {
         return new TwoPhaseIterator(approximation) {
             @Override
             public boolean matches() throws IOException {
-                profile.startTime(ProfileBreakdown.TimingType.MATCH);
+                profile.startTime(QueryTimingType.MATCH);
                 try {
                     return in.matches();
                 } finally {
diff --git a/core/src/main/java/org/elasticsearch/search/profile/ProfileShardResult.java b/core/src/main/java/org/elasticsearch/search/profile/ProfileShardResult.java
index 1353bb505ee..e35a3c2e2aa 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/ProfileShardResult.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/ProfileShardResult.java
@@ -36,16 +36,16 @@ import java.util.List;
  */
 public final class ProfileShardResult implements Writeable, ToXContent {
 
-    private final List<ProfileResult> profileResults;
+    private final List<ProfileResult> queryProfileResults;
 
     private final CollectorResult profileCollector;
 
     private final long rewriteTime;
 
-    public ProfileShardResult(List<ProfileResult> profileResults, long rewriteTime,
+    public ProfileShardResult(List<ProfileResult> queryProfileResults, long rewriteTime,
                               CollectorResult profileCollector) {
         assert(profileCollector != null);
-        this.profileResults = profileResults;
+        this.queryProfileResults = queryProfileResults;
         this.profileCollector = profileCollector;
         this.rewriteTime = rewriteTime;
     }
@@ -55,9 +55,9 @@ public final class ProfileShardResult implements Writeable, ToXContent {
      */
     public ProfileShardResult(StreamInput in) throws IOException {
         int profileSize = in.readVInt();
-        profileResults = new ArrayList<>(profileSize);
+        queryProfileResults = new ArrayList<>(profileSize);
         for (int j = 0; j < profileSize; j++) {
-            profileResults.add(new ProfileResult(in));
+            queryProfileResults.add(new ProfileResult(in));
         }
 
         profileCollector = new CollectorResult(in);
@@ -66,8 +66,8 @@ public final class ProfileShardResult implements Writeable, ToXContent {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeVInt(profileResults.size());
-        for (ProfileResult p : profileResults) {
+        out.writeVInt(queryProfileResults.size());
+        for (ProfileResult p : queryProfileResults) {
             p.writeTo(out);
         }
         profileCollector.writeTo(out);
@@ -76,7 +76,7 @@ public final class ProfileShardResult implements Writeable, ToXContent {
 
 
     public List<ProfileResult> getQueryResults() {
-        return Collections.unmodifiableList(profileResults);
+        return Collections.unmodifiableList(queryProfileResults);
     }
 
     public long getRewriteTime() {
@@ -90,7 +90,7 @@ public final class ProfileShardResult implements Writeable, ToXContent {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startArray("query");
-        for (ProfileResult p : profileResults) {
+        for (ProfileResult p : queryProfileResults) {
             p.toXContent(builder, params);
         }
         builder.endArray();
diff --git a/core/src/main/java/org/elasticsearch/search/profile/ProfileWeight.java b/core/src/main/java/org/elasticsearch/search/profile/ProfileWeight.java
index 1ce5cd721fe..9546f691616 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/ProfileWeight.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/ProfileWeight.java
@@ -38,9 +38,9 @@ import java.util.Set;
 public final class ProfileWeight extends Weight {
 
     private final Weight subQueryWeight;
-    private final ProfileBreakdown profile;
+    private final QueryProfileBreakdown profile;
 
-    public ProfileWeight(Query query, Weight subQueryWeight, ProfileBreakdown profile) throws IOException {
+    public ProfileWeight(Query query, Weight subQueryWeight, QueryProfileBreakdown profile) throws IOException {
         super(query);
         this.subQueryWeight = subQueryWeight;
         this.profile = profile;
@@ -48,7 +48,7 @@ public final class ProfileWeight extends Weight {
 
     @Override
     public Scorer scorer(LeafReaderContext context) throws IOException {
-        profile.startTime(ProfileBreakdown.TimingType.BUILD_SCORER);
+        profile.startTime(QueryTimingType.BUILD_SCORER);
         final Scorer subQueryScorer;
         try {
             subQueryScorer = subQueryWeight.scorer(context);
diff --git a/core/src/main/java/org/elasticsearch/search/profile/Profilers.java b/core/src/main/java/org/elasticsearch/search/profile/Profilers.java
index 69a4544ccd7..38b69d38b15 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/Profilers.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/Profilers.java
@@ -25,13 +25,13 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
-/** Wrapper around several {@link Profiler}s that makes management easier. */
+/** Wrapper around several {@link QueryProfiler}s that makes management easier. */
 public final class Profilers {
 
     private final ContextIndexSearcher searcher;
-    private final List<Profiler> profilers;
+    private final List<QueryProfiler> profilers;
 
-    /** Sole constructor. This {@link Profilers} instance will initially wrap one {@link Profiler}. */
+    /** Sole constructor. This {@link Profilers} instance will initially wrap one {@link QueryProfiler}. */
     public Profilers(ContextIndexSearcher searcher) {
         this.searcher = searcher;
         this.profilers = new ArrayList<>();
@@ -39,20 +39,20 @@ public final class Profilers {
     }
 
     /** Switch to a new profile. */
-    public Profiler addProfiler() {
-        Profiler profiler = new Profiler();
+    public QueryProfiler addProfiler() {
+        QueryProfiler profiler = new QueryProfiler();
         searcher.setProfiler(profiler);
         profilers.add(profiler);
         return profiler;
     }
 
     /** Get the current profiler. */
-    public Profiler getCurrent() {
+    public QueryProfiler getCurrent() {
         return profilers.get(profilers.size() - 1);
     }
 
-    /** Return the list of all created {@link Profiler}s so far. */
-    public List<Profiler> getProfilers() {
+    /** Return the list of all created {@link QueryProfiler}s so far. */
+    public List<QueryProfiler> getProfilers() {
         return Collections.unmodifiableList(profilers);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/profile/QueryProfileBreakdown.java b/core/src/main/java/org/elasticsearch/search/profile/QueryProfileBreakdown.java
new file mode 100644
index 00000000000..fdb13a49977
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/profile/QueryProfileBreakdown.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile;
+
+/**
+ * A record of timings for the various operations that may happen during query execution.
+ * A node's time may be composed of several internal attributes (rewriting, weighting,
+ * scoring, etc).
+ */
+public final class QueryProfileBreakdown extends AbstractProfileBreakdown<QueryTimingType> {
+
+    /** Sole constructor. */
+    public QueryProfileBreakdown() {
+        super(QueryTimingType.values());
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/profile/Profiler.java b/core/src/main/java/org/elasticsearch/search/profile/QueryProfiler.java
similarity index 75%
rename from core/src/main/java/org/elasticsearch/search/profile/Profiler.java
rename to core/src/main/java/org/elasticsearch/search/profile/QueryProfiler.java
index 011b1593d35..f8fcf5fbf13 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/Profiler.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/QueryProfiler.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.profile;
 
 import org.apache.lucene.search.Query;
 
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 
@@ -36,16 +35,16 @@ import java.util.Objects;
  * request may execute two searches (query + global agg).  A Profiler just
  * represents one of those
  */
-public final class Profiler {
+public final class QueryProfiler {
 
-    private final InternalProfileTree queryTree = new InternalProfileTree();
+    private final InternalQueryProfileTree queryTree = new InternalQueryProfileTree();
 
     /**
      * The root Collector used in the search
      */
     private InternalProfileCollector collector;
 
-    public Profiler() {}
+    public QueryProfiler() {}
 
     /** Set the collector that is associated with this profiler. */
     public void setCollector(InternalProfileCollector collector) {
@@ -56,11 +55,11 @@ public final class Profiler {
     }
 
     /**
-     * Get the {@link ProfileBreakdown} for the given query, potentially creating it if it did not exist.
+     * Get the {@link QueryProfileBreakdown} for the given query, potentially creating it if it did not exist.
      * This should only be used for queries that will be undergoing scoring. Do not use it to profile the
      * rewriting phase
      */
-    public ProfileBreakdown getQueryBreakdown(Query query) {
+    public QueryProfileBreakdown getQueryBreakdown(Query query) {
         return queryTree.getQueryBreakdown(query);
     }
 
@@ -111,22 +110,5 @@ public final class Profiler {
         return collector.getCollectorTree();
     }
 
-    /**
-     * Helper method to convert Profiler into  InternalProfileShardResults, which can be
-     * serialized to other nodes, emitted as JSON, etc.
-     *
-     * @param profilers A list of Profilers to convert into InternalProfileShardResults
-     * @return          A list of corresponding InternalProfileShardResults
-     */
-    public static List<ProfileShardResult> buildShardResults(List<Profiler> profilers) {
-        List<ProfileShardResult> results = new ArrayList<>(profilers.size());
-        for (Profiler profiler : profilers) {
-            ProfileShardResult result =  new ProfileShardResult(
-                    profiler.getQueryTree(), profiler.getRewriteTime(), profiler.getCollector());
-            results.add(result);
-        }
-        return results;
-    }
-
 
 }
diff --git a/core/src/main/java/org/elasticsearch/search/profile/QueryTimingType.java b/core/src/main/java/org/elasticsearch/search/profile/QueryTimingType.java
new file mode 100644
index 00000000000..5fb3b29dee3
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/profile/QueryTimingType.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile;
+
+import java.util.Locale;
+
+public enum QueryTimingType {
+    CREATE_WEIGHT,
+    BUILD_SCORER,
+    NEXT_DOC,
+    ADVANCE,
+    MATCH,
+    SCORE;
+
+    @Override
+    public String toString() {
+        return name().toLowerCase(Locale.ROOT);
+    }
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/profile/InternalProfileShardResults.java b/core/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResults.java
similarity index 78%
rename from core/src/main/java/org/elasticsearch/search/profile/InternalProfileShardResults.java
rename to core/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResults.java
index 5aa42c45da3..a38c0813dc5 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/InternalProfileShardResults.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResults.java
@@ -37,11 +37,11 @@ import java.util.stream.Collectors;
  * A container class to hold all the profile results across all shards.  Internally
  * holds a map of shard ID -&gt; Profiled results
  */
-public final class InternalProfileShardResults implements Writeable, ToXContent{
+public final class SearchProfileShardResults implements Writeable, ToXContent{
 
     private Map<String, List<ProfileShardResult>> shardResults;
 
-    public InternalProfileShardResults(Map<String, List<ProfileShardResult>> shardResults) {
+    public SearchProfileShardResults(Map<String, List<ProfileShardResult>> shardResults) {
         Map<String, List<ProfileShardResult>> transformed =
                 shardResults.entrySet()
                         .stream()
@@ -52,7 +52,7 @@ public final class InternalProfileShardResults implements Writeable, ToXContent{
         this.shardResults =  Collections.unmodifiableMap(transformed);
     }
 
-    public InternalProfileShardResults(StreamInput in) throws IOException {
+    public SearchProfileShardResults(StreamInput in) throws IOException {
         int size = in.readInt();
         shardResults = new HashMap<>(size);
 
@@ -105,4 +105,22 @@ public final class InternalProfileShardResults implements Writeable, ToXContent{
         builder.endArray().endObject();
         return builder;
     }
+
+    /**
+     * Helper method to convert Profiler into InternalProfileShardResults, which
+     * can be serialized to other nodes, emitted as JSON, etc.
+     *
+     * @param profilers
+     *            A list of Profilers to convert into
+     *            InternalProfileShardResults
+     * @return A list of corresponding InternalProfileShardResults
+     */
+    public static List<ProfileShardResult> buildShardResults(List<QueryProfiler> profilers) {
+        List<ProfileShardResult> results = new ArrayList<>(profilers.size());
+        for (QueryProfiler profiler : profilers) {
+            ProfileShardResult result = new ProfileShardResult(profiler.getQueryTree(), profiler.getRewriteTime(), profiler.getCollector());
+            results.add(result);
+        }
+        return results;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index 1ae80337d76..da849828e7a 100644
--- a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -55,7 +55,7 @@ import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.profile.CollectorResult;
 import org.elasticsearch.search.profile.InternalProfileCollector;
 import org.elasticsearch.search.profile.ProfileShardResult;
-import org.elasticsearch.search.profile.Profiler;
+import org.elasticsearch.search.profile.SearchProfileShardResults;
 import org.elasticsearch.search.rescore.RescorePhase;
 import org.elasticsearch.search.rescore.RescoreSearchContext;
 import org.elasticsearch.search.sort.SortAndFormats;
@@ -112,7 +112,8 @@ public class QueryPhase implements SearchPhase {
         aggregationPhase.execute(searchContext);
 
         if (searchContext.getProfilers() != null) {
-            List<ProfileShardResult> shardResults = Profiler.buildShardResults(searchContext.getProfilers().getProfilers());
+            List<ProfileShardResult> shardResults = SearchProfileShardResults
+                    .buildShardResults(searchContext.getProfilers().getProfilers());
             searchContext.queryResult().profileResults(shardResults);
         }
     }
@@ -384,7 +385,8 @@ public class QueryPhase implements SearchPhase {
             queryResult.topDocs(topDocsCallable.call(), sortValueFormats);
 
             if (searchContext.getProfilers() != null) {
-                List<ProfileShardResult> shardResults = Profiler.buildShardResults(searchContext.getProfilers().getProfilers());
+                List<ProfileShardResult> shardResults = SearchProfileShardResults
+                        .buildShardResults(searchContext.getProfilers().getProfilers());
                 searchContext.queryResult().profileResults(shardResults);
             }
 
diff --git a/core/src/test/java/org/elasticsearch/search/profile/ProfileTests.java b/core/src/test/java/org/elasticsearch/search/profile/ProfileTests.java
index afc1c7b3cbb..ebae8e64f32 100644
--- a/core/src/test/java/org/elasticsearch/search/profile/ProfileTests.java
+++ b/core/src/test/java/org/elasticsearch/search/profile/ProfileTests.java
@@ -35,7 +35,6 @@ import org.apache.lucene.search.TotalHitCountCollector;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.TestUtil;
-import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.search.internal.ContextIndexSearcher;
 import org.elasticsearch.test.ESTestCase;
@@ -84,45 +83,45 @@ public class ProfileTests extends ESTestCase {
     }
 
     public void testBasic() throws IOException {
-        Profiler profiler = new Profiler();
+        QueryProfiler profiler = new QueryProfiler();
         searcher.setProfiler(profiler);
         Query query = new TermQuery(new Term("foo", "bar"));
         searcher.search(query, 1);
         List<ProfileResult> results = profiler.getQueryTree();
         assertEquals(1, results.size());
         Map<String, Long> breakdown = results.get(0).getTimeBreakdown();
-        assertThat(breakdown.get(ProfileBreakdown.TimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L));
-        assertThat(breakdown.get(ProfileBreakdown.TimingType.BUILD_SCORER.toString()).longValue(), greaterThan(0L));
-        assertThat(breakdown.get(ProfileBreakdown.TimingType.NEXT_DOC.toString()).longValue(), greaterThan(0L));
-        assertThat(breakdown.get(ProfileBreakdown.TimingType.ADVANCE.toString()).longValue(), equalTo(0L));
-        assertThat(breakdown.get(ProfileBreakdown.TimingType.SCORE.toString()).longValue(), greaterThan(0L));
-        assertThat(breakdown.get(ProfileBreakdown.TimingType.MATCH.toString()).longValue(), equalTo(0L));
+        assertThat(breakdown.get(QueryTimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdown.get(QueryTimingType.BUILD_SCORER.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdown.get(QueryTimingType.NEXT_DOC.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdown.get(QueryTimingType.ADVANCE.toString()).longValue(), equalTo(0L));
+        assertThat(breakdown.get(QueryTimingType.SCORE.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdown.get(QueryTimingType.MATCH.toString()).longValue(), equalTo(0L));
 
         long rewriteTime = profiler.getRewriteTime();
         assertThat(rewriteTime, greaterThan(0L));
     }
 
     public void testNoScoring() throws IOException {
-        Profiler profiler = new Profiler();
+        QueryProfiler profiler = new QueryProfiler();
         searcher.setProfiler(profiler);
         Query query = new TermQuery(new Term("foo", "bar"));
         searcher.search(query, 1, Sort.INDEXORDER); // scores are not needed
         List<ProfileResult> results = profiler.getQueryTree();
         assertEquals(1, results.size());
         Map<String, Long> breakdown = results.get(0).getTimeBreakdown();
-        assertThat(breakdown.get(ProfileBreakdown.TimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L));
-        assertThat(breakdown.get(ProfileBreakdown.TimingType.BUILD_SCORER.toString()).longValue(), greaterThan(0L));
-        assertThat(breakdown.get(ProfileBreakdown.TimingType.NEXT_DOC.toString()).longValue(), greaterThan(0L));
-        assertThat(breakdown.get(ProfileBreakdown.TimingType.ADVANCE.toString()).longValue(), equalTo(0L));
-        assertThat(breakdown.get(ProfileBreakdown.TimingType.SCORE.toString()).longValue(), equalTo(0L));
-        assertThat(breakdown.get(ProfileBreakdown.TimingType.MATCH.toString()).longValue(), equalTo(0L));
+        assertThat(breakdown.get(QueryTimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdown.get(QueryTimingType.BUILD_SCORER.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdown.get(QueryTimingType.NEXT_DOC.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdown.get(QueryTimingType.ADVANCE.toString()).longValue(), equalTo(0L));
+        assertThat(breakdown.get(QueryTimingType.SCORE.toString()).longValue(), equalTo(0L));
+        assertThat(breakdown.get(QueryTimingType.MATCH.toString()).longValue(), equalTo(0L));
 
         long rewriteTime = profiler.getRewriteTime();
         assertThat(rewriteTime, greaterThan(0L));
     }
 
     public void testUseIndexStats() throws IOException {
-        Profiler profiler = new Profiler();
+        QueryProfiler profiler = new QueryProfiler();
         searcher.setProfiler(profiler);
         Query query = new TermQuery(new Term("foo", "bar"));
         searcher.count(query); // will use index stats
@@ -134,7 +133,7 @@ public class ProfileTests extends ESTestCase {
     }
 
     public void testApproximations() throws IOException {
-        Profiler profiler = new Profiler();
+        QueryProfiler profiler = new QueryProfiler();
         Engine.Searcher engineSearcher = new Engine.Searcher("test", new IndexSearcher(reader));
         // disable query caching since we want to test approximations, which won't
         // be exposed on a cached entry
@@ -145,12 +144,12 @@ public class ProfileTests extends ESTestCase {
         List<ProfileResult> results = profiler.getQueryTree();
         assertEquals(1, results.size());
         Map<String, Long> breakdown = results.get(0).getTimeBreakdown();
-        assertThat(breakdown.get(ProfileBreakdown.TimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L));
-        assertThat(breakdown.get(ProfileBreakdown.TimingType.BUILD_SCORER.toString()).longValue(), greaterThan(0L));
-        assertThat(breakdown.get(ProfileBreakdown.TimingType.NEXT_DOC.toString()).longValue(), greaterThan(0L));
-        assertThat(breakdown.get(ProfileBreakdown.TimingType.ADVANCE.toString()).longValue(), equalTo(0L));
-        assertThat(breakdown.get(ProfileBreakdown.TimingType.SCORE.toString()).longValue(), equalTo(0L));
-        assertThat(breakdown.get(ProfileBreakdown.TimingType.MATCH.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdown.get(QueryTimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdown.get(QueryTimingType.BUILD_SCORER.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdown.get(QueryTimingType.NEXT_DOC.toString()).longValue(), greaterThan(0L));
+        assertThat(breakdown.get(QueryTimingType.ADVANCE.toString()).longValue(), equalTo(0L));
+        assertThat(breakdown.get(QueryTimingType.SCORE.toString()).longValue(), equalTo(0L));
+        assertThat(breakdown.get(QueryTimingType.MATCH.toString()).longValue(), greaterThan(0L));
 
         long rewriteTime = profiler.getRewriteTime();
         assertThat(rewriteTime, greaterThan(0L));

From 864ed04059f79a91a6680ee879f38182b5228865 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Wed, 11 May 2016 10:28:42 +0200
Subject: [PATCH 0274/1311] Lessen leniency of the query dsl. #18276

This change does the following:
 - Queries that are currently unsupported such as prefix queries on numeric
   fields or term queries on geo fields now throw an error rather than returning
   a query that does not match anything.
 - Fuzzy queries on numeric, date and ip fields are now unsupported: they used
   to create range queries, we now expect users to use range queries directly.
   Fuzzy, regexp and prefix queries are now only supported on text/keyword
   fields (including `_all`).
 - The `_uid` and `_id` fields do not support prefix or range queries anymore as
   it would prevent us to store them more efficiently in the future, eg. by
   using a binary encoding.

Note that it is still possible to ignore these errors by using the `lenient`
option of the `match` or `query_string` queries.
---
 .../elasticsearch/common/unit/Fuzziness.java  | 61 +-----------
 .../index/mapper/MappedFieldType.java         | 63 ++++--------
 .../index/mapper/StringFieldType.java         | 95 +++++++++++++++++++
 .../index/mapper/TermBasedFieldType.java      | 72 ++++++++++++++
 .../index/mapper/core/BinaryFieldMapper.java  |  8 ++
 .../index/mapper/core/BooleanFieldMapper.java | 16 +++-
 .../mapper/core/CompletionFieldMapper.java    |  3 +-
 .../mapper/core/CompletionFieldMapper2x.java  |  3 +-
 .../index/mapper/core/DateFieldMapper.java    | 15 ---
 .../index/mapper/core/KeywordFieldMapper.java | 20 +---
 .../mapper/core/LegacyByteFieldMapper.java    | 11 ---
 .../mapper/core/LegacyDateFieldMapper.java    | 16 ----
 .../mapper/core/LegacyDoubleFieldMapper.java  | 11 ---
 .../mapper/core/LegacyFloatFieldMapper.java   | 11 ---
 .../mapper/core/LegacyIntegerFieldMapper.java | 14 +--
 .../mapper/core/LegacyLongFieldMapper.java    | 11 ---
 .../mapper/core/LegacyNumberFieldMapper.java  |  8 +-
 .../mapper/core/LegacyShortFieldMapper.java   | 10 --
 .../index/mapper/core/NumberFieldMapper.java  | 47 ---------
 .../index/mapper/core/StringFieldMapper.java  | 11 +--
 .../index/mapper/core/TextFieldMapper.java    | 20 +---
 .../mapper/geo/BaseGeoPointFieldMapper.java   |  8 ++
 .../index/mapper/geo/GeoShapeFieldMapper.java |  7 ++
 .../index/mapper/internal/AllFieldMapper.java |  4 +-
 .../internal/FieldNamesFieldMapper.java       |  4 +-
 .../index/mapper/internal/IdFieldMapper.java  | 51 +---------
 .../mapper/internal/IndexFieldMapper.java     |  3 -
 .../mapper/internal/ParentFieldMapper.java    |  3 +-
 .../mapper/internal/RoutingFieldMapper.java   |  5 +-
 .../mapper/internal/SourceFieldMapper.java    |  8 ++
 .../mapper/internal/TypeFieldMapper.java      |  3 +-
 .../index/mapper/internal/UidFieldMapper.java |  3 +-
 .../mapper/internal/VersionFieldMapper.java   |  8 ++
 .../index/mapper/ip/IpFieldMapper.java        |  9 --
 .../index/mapper/ip/LegacyIpFieldMapper.java  | 15 ---
 .../percolator/PercolatorFieldMapper.java     |  6 ++
 .../index/search/MatchQuery.java              | 36 ++-----
 .../index/search/MultiMatchQuery.java         |  9 --
 .../common/unit/FuzzinessTests.java           | 35 +------
 .../mapper/DocumentFieldMapperTests.java      |  2 +-
 .../index/mapper/FieldTypeLookupTests.java    |  2 +-
 .../index/mapper/FieldTypeTestCase.java       |  4 +-
 .../mapper/externalvalues/ExternalMapper.java |  6 +-
 .../externalvalues/FakeStringFieldMapper.java | 18 +---
 .../internal/FieldNamesFieldMapperTests.java  |  3 +-
 .../mapper/internal/IdFieldTypeTests.java     |  8 ++
 .../mapper/internal/UidFieldTypeTests.java    |  8 ++
 .../index/query/FuzzyQueryBuilderTests.java   | 23 ++---
 .../index/query/MatchQueryBuilderTests.java   | 76 ++++++++-------
 .../query/MultiMatchQueryBuilderTests.java    | 29 +++++-
 .../index/query/PrefixQueryBuilderTests.java  | 11 +++
 .../query/QueryStringQueryBuilderTests.java   | 44 +++++++--
 .../index/query/RegexpQueryBuilderTests.java  | 11 +++
 .../index/query/TermQueryBuilderTests.java    | 10 ++
 .../index/query/TermsQueryBuilderTests.java   | 16 +++-
 .../search/query/SearchQueryIT.java           |  8 --
 .../search/simple/SimpleSearchIT.java         | 10 +-
 .../sort/GeoDistanceSortBuilderTests.java     |  5 +-
 docs/reference/api-conventions.asciidoc       | 22 +----
 .../migration/migrate_5_0/search.asciidoc     | 10 ++
 docs/reference/query-dsl/fuzzy-query.asciidoc | 43 +--------
 .../mapper/attachments/AttachmentMapper.java  |  8 ++
 .../mapper/murmur3/Murmur3FieldMapper.java    |  8 ++
 .../index/mapper/MockFieldMapper.java         |  2 +-
 64 files changed, 498 insertions(+), 632 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
 create mode 100644 core/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java

diff --git a/core/src/main/java/org/elasticsearch/common/unit/Fuzziness.java b/core/src/main/java/org/elasticsearch/common/unit/Fuzziness.java
index 8dbb949806a..d9403b52882 100644
--- a/core/src/main/java/org/elasticsearch/common/unit/Fuzziness.java
+++ b/core/src/main/java/org/elasticsearch/common/unit/Fuzziness.java
@@ -151,66 +151,7 @@ public final class Fuzziness implements ToXContent, Writeable {
                 return 1;
             }
         }
-        return Math.min(2, asInt());
-    }
-
-    public TimeValue asTimeValue() {
-        if (this.equals(AUTO)) {
-            return TimeValue.timeValueMillis(1);
-        } else {
-            return TimeValue.parseTimeValue(fuzziness.toString(), null, "fuzziness");
-        }
-    }
-
-    public long asLong() {
-        if (this.equals(AUTO)) {
-            return 1;
-        }
-        try {
-            return Long.parseLong(fuzziness.toString());
-        } catch (NumberFormatException ex) {
-            return (long) Double.parseDouble(fuzziness.toString());
-        }
-    }
-
-    public int asInt() {
-        if (this.equals(AUTO)) {
-            return 1;
-        }
-        try {
-            return Integer.parseInt(fuzziness.toString());
-        } catch (NumberFormatException ex) {
-            return (int) Float.parseFloat(fuzziness.toString());
-        }
-    }
-
-    public short asShort() {
-        if (this.equals(AUTO)) {
-            return 1;
-        }
-        try {
-            return Short.parseShort(fuzziness.toString());
-        } catch (NumberFormatException ex) {
-            return (short) Float.parseFloat(fuzziness.toString());
-        }
-    }
-
-    public byte asByte() {
-        if (this.equals(AUTO)) {
-            return 1;
-        }
-        try {
-            return Byte.parseByte(fuzziness.toString());
-        } catch (NumberFormatException ex) {
-            return (byte) Float.parseFloat(fuzziness.toString());
-        }
-    }
-
-    public double asDouble() {
-        if (this.equals(AUTO)) {
-            return 1d;
-        }
-        return Double.parseDouble(fuzziness.toString());
+        return Math.min(2, (int) asFloat());
     }
 
     public float asFloat() {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
index 3f8dc0e142c..3b4dd3bb5c6 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
@@ -25,21 +25,16 @@ import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.Terms;
-import org.apache.lucene.queries.TermsQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
-import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.MultiTermQuery;
-import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TermRangeQuery;
+import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.joda.DateMathParser;
-import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -312,13 +307,6 @@ public abstract class MappedFieldType extends FieldType {
         return value;
     }
 
-    /** Returns the indexed value used to construct search "values".
-     *  This method is used for the default implementations of most
-     *  query factory methods such as {@link #termQuery}. */
-    protected BytesRef indexedValueForSearch(Object value) {
-        return BytesRefs.toBytesRef(value);
-    }
-
     /** Returns true if the field is searchable.
      *
      */
@@ -342,50 +330,33 @@ public abstract class MappedFieldType extends FieldType {
      *  The default implementation returns a {@link TermQuery} over the value bytes,
      *  boosted by {@link #boost()}.
      *  @throws IllegalArgumentException if {@code value} cannot be converted to the expected data type */
-    public Query termQuery(Object value, @Nullable QueryShardContext context) {
-        failIfNotIndexed();
-        TermQuery query = new TermQuery(new Term(name(), indexedValueForSearch(value)));
-        if (boost == 1f ||
-            (context != null && context.indexVersionCreated().before(Version.V_5_0_0_alpha1))) {
-            return query;
-        }
-        return new BoostQuery(query, boost);
-    }
+    public abstract Query termQuery(Object value, @Nullable QueryShardContext context);
 
-    public Query termsQuery(List values, @Nullable QueryShardContext context) {
-        failIfNotIndexed();
-        BytesRef[] bytesRefs = new BytesRef[values.size()];
-        for (int i = 0; i < bytesRefs.length; i++) {
-            bytesRefs[i] = indexedValueForSearch(values.get(i));
+    /** Build a constant-scoring query that matches all values. The default implementation uses a
+     * {@link ConstantScoreQuery} around a {@link BooleanQuery} whose {@link Occur#SHOULD} clauses
+     * are generated with {@link #termQuery}. */
+    public Query termsQuery(List<?> values, @Nullable QueryShardContext context) {
+        BooleanQuery.Builder builder = new BooleanQuery.Builder();
+        for (Object value : values) {
+            builder.add(termQuery(value, context), Occur.SHOULD);
         }
-        return new TermsQuery(name(), bytesRefs);
+        return new ConstantScoreQuery(builder.build());
     }
 
     public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) {
-        failIfNotIndexed();
-        return new TermRangeQuery(name(),
-            lowerTerm == null ? null : indexedValueForSearch(lowerTerm),
-            upperTerm == null ? null : indexedValueForSearch(upperTerm),
-            includeLower, includeUpper);
+        throw new IllegalArgumentException("Field [" + name + "] of type [" + typeName() + "] does not support range queries");
     }
 
     public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions, boolean transpositions) {
-        failIfNotIndexed();
-        return new FuzzyQuery(new Term(name(), indexedValueForSearch(value)),
-                fuzziness.asDistance(BytesRefs.toString(value)), prefixLength, maxExpansions, transpositions);
+        throw new IllegalArgumentException("Can only use fuzzy queries on keyword and text fields - not on [" + name + "] which is of type [" + typeName() + "]");
     }
 
-    public Query prefixQuery(String value, @Nullable MultiTermQuery.RewriteMethod method, @Nullable QueryShardContext context) {
-        failIfNotIndexed();
-        PrefixQuery query = new PrefixQuery(new Term(name(), indexedValueForSearch(value)));
-        if (method != null) {
-            query.setRewriteMethod(method);
-        }
-        return query;
+    public Query prefixQuery(String value, @Nullable MultiTermQuery.RewriteMethod method, QueryShardContext context) {
+        throw new QueryShardException(context, "Can only use prefix queries on keyword and text fields - not on [" + name + "] which is of type [" + typeName() + "]");
     }
 
-    public Query regexpQuery(String value, int flags, int maxDeterminizedStates, @Nullable MultiTermQuery.RewriteMethod method, @Nullable QueryShardContext context) {
-        throw new QueryShardException(context, "Can only use regular expression on keyword and text fields - not on [" + name + "] which is of type [" + typeName() + "]");
+    public Query regexpQuery(String value, int flags, int maxDeterminizedStates, @Nullable MultiTermQuery.RewriteMethod method, QueryShardContext context) {
+        throw new QueryShardException(context, "Can only use regexp queries on keyword and text fields - not on [" + name + "] which is of type [" + typeName() + "]");
     }
 
     public Query nullValueQuery() {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java b/core/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
new file mode 100644
index 00000000000..9450c2a5b97
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/mapper/StringFieldType.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import java.util.List;
+
+import org.apache.lucene.index.Term;
+import org.apache.lucene.queries.TermsQuery;
+import org.apache.lucene.search.FuzzyQuery;
+import org.apache.lucene.search.MultiTermQuery;
+import org.apache.lucene.search.PrefixQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.RegexpQuery;
+import org.apache.lucene.search.TermRangeQuery;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.lucene.BytesRefs;
+import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.index.query.QueryShardContext;
+
+/** Base class for {@link MappedFieldType} implementations that use the same
+ * representation for internal index terms as the external representation so
+ * that partial matching queries such as prefix, wildcard and fuzzy queries
+ * can be implemented. */
+public abstract class StringFieldType extends TermBasedFieldType {
+
+    public StringFieldType() {}
+
+    protected StringFieldType(MappedFieldType ref) {
+        super(ref);
+    }
+
+    public Query termsQuery(List<?> values, QueryShardContext context) {
+        failIfNotIndexed();
+        BytesRef[] bytesRefs = new BytesRef[values.size()];
+        for (int i = 0; i < bytesRefs.length; i++) {
+            bytesRefs[i] = indexedValueForSearch(values.get(i));
+        }
+        return new TermsQuery(name(), bytesRefs);
+    }
+
+    @Override
+    public final Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions,
+            boolean transpositions) {
+        failIfNotIndexed();
+        return new FuzzyQuery(new Term(name(), indexedValueForSearch(value)),
+                fuzziness.asDistance(BytesRefs.toString(value)), prefixLength, maxExpansions, transpositions);
+    }
+
+    @Override
+    public final Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) {
+        failIfNotIndexed();
+        PrefixQuery query = new PrefixQuery(new Term(name(), indexedValueForSearch(value)));
+        if (method != null) {
+            query.setRewriteMethod(method);
+        }
+        return query;
+    }
+
+    @Override
+    public final Query regexpQuery(String value, int flags, int maxDeterminizedStates,
+            MultiTermQuery.RewriteMethod method, QueryShardContext context) {
+        failIfNotIndexed();
+        RegexpQuery query = new RegexpQuery(new Term(name(), indexedValueForSearch(value)), flags, maxDeterminizedStates);
+        if (method != null) {
+            query.setRewriteMethod(method);
+        }
+        return query;
+    }
+
+    @Override
+    public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) {
+        failIfNotIndexed();
+        return new TermRangeQuery(name(),
+            lowerTerm == null ? null : indexedValueForSearch(lowerTerm),
+            upperTerm == null ? null : indexedValueForSearch(upperTerm),
+            includeLower, includeUpper);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java b/core/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java
new file mode 100644
index 00000000000..4b346af6346
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/mapper/TermBasedFieldType.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import java.util.List;
+
+import org.apache.lucene.index.Term;
+import org.apache.lucene.queries.TermsQuery;
+import org.apache.lucene.search.BoostQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.lucene.BytesRefs;
+import org.elasticsearch.index.query.QueryShardContext;
+
+/** Base {@link MappedFieldType} implementation for a field that is indexed
+ *  with the inverted index. */
+public abstract class TermBasedFieldType extends MappedFieldType {
+
+    public TermBasedFieldType() {}
+
+    protected TermBasedFieldType(MappedFieldType ref) {
+        super(ref);
+    }
+
+    /** Returns the indexed value used to construct search "values".
+     *  This method is used for the default implementations of most
+     *  query factory methods such as {@link #termQuery}. */
+    protected BytesRef indexedValueForSearch(Object value) {
+        return BytesRefs.toBytesRef(value);
+    }
+
+    @Override
+    public Query termQuery(Object value, QueryShardContext context) {
+        failIfNotIndexed();
+        TermQuery query = new TermQuery(new Term(name(), indexedValueForSearch(value)));
+        if (boost() == 1f ||
+            (context != null && context.indexVersionCreated().before(Version.V_5_0_0_alpha1))) {
+            return query;
+        }
+        return new BoostQuery(query, boost());
+    }
+
+    @Override
+    public Query termsQuery(List<?> values, QueryShardContext context) {
+        failIfNotIndexed();
+        BytesRef[] bytesRefs = new BytesRef[values.size()];
+        for (int i = 0; i < bytesRefs.length; i++) {
+            bytesRefs[i] = indexedValueForSearch(values.get(i));
+        }
+        return new TermsQuery(name(), bytesRefs);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
index ed01aff290e..1ca67a8cbc0 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
@@ -22,6 +22,7 @@ package org.elasticsearch.index.mapper.core;
 import com.carrotsearch.hppc.ObjectArrayList;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.store.ByteArrayDataOutput;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchException;
@@ -40,6 +41,8 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.QueryShardException;
 
 import java.io.IOException;
 import java.util.List;
@@ -135,6 +138,11 @@ public class BinaryFieldMapper extends FieldMapper {
             failIfNoDocValues();
             return new BytesBinaryDVIndexFieldData.Builder();
         }
+
+        @Override
+        public Query termQuery(Object value, QueryShardContext context) {
+            throw new QueryShardException(context, "Binary fields do not support searching");
+        }
     }
 
     protected BinaryFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java
index 60945bae3b7..7237381159b 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java
@@ -22,10 +22,11 @@ package org.elasticsearch.index.mapper.core;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -38,6 +39,7 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.TermBasedFieldType;
 import org.elasticsearch.search.DocValueFormat;
 import org.joda.time.DateTimeZone;
 
@@ -48,7 +50,6 @@ import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
 import static org.elasticsearch.index.mapper.core.TypeParsers.parseField;
-import static org.elasticsearch.index.mapper.core.TypeParsers.parseMultiField;
 
 /**
  * A field mapper for boolean fields.
@@ -119,7 +120,7 @@ public class BooleanFieldMapper extends FieldMapper {
         }
     }
 
-    public static final class BooleanFieldType extends MappedFieldType {
+    public static final class BooleanFieldType extends TermBasedFieldType {
 
         public BooleanFieldType() {}
 
@@ -200,6 +201,15 @@ public class BooleanFieldMapper extends FieldMapper {
             }
             return DocValueFormat.BOOLEAN;
         }
+
+        @Override
+        public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) {
+            failIfNotIndexed();
+            return new TermRangeQuery(name(),
+                lowerTerm == null ? null : indexedValueForSearch(lowerTerm),
+                upperTerm == null ? null : indexedValueForSearch(upperTerm),
+                includeLower, includeUpper);
+        }
     }
 
     protected BooleanFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java
index 53162dc5353..81f890d3e0e 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java
@@ -44,6 +44,7 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.TermBasedFieldType;
 import org.elasticsearch.index.mapper.object.ArrayValueMapperParser;
 import org.elasticsearch.search.suggest.completion.CompletionSuggester;
 import org.elasticsearch.search.suggest.completion.context.ContextMapping;
@@ -178,7 +179,7 @@ public class CompletionFieldMapper extends FieldMapper implements ArrayValueMapp
         }
     }
 
-    public static final class CompletionFieldType extends MappedFieldType {
+    public static final class CompletionFieldType extends TermBasedFieldType {
 
         private static PostingsFormat postingsFormat;
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper2x.java b/core/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper2x.java
index 9b0306c6875..6fffb0e6e23 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper2x.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper2x.java
@@ -40,6 +40,7 @@ import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperException;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.TermBasedFieldType;
 import org.elasticsearch.search.suggest.completion2x.AnalyzingCompletionLookupProvider;
 import org.elasticsearch.search.suggest.completion2x.Completion090PostingsFormat;
 import org.elasticsearch.search.suggest.completion2x.CompletionTokenStream;
@@ -231,7 +232,7 @@ public class CompletionFieldMapper2x extends FieldMapper {
         }
     }
 
-    public static final class CompletionFieldType extends MappedFieldType {
+    public static final class CompletionFieldType extends TermBasedFieldType {
         private PostingsFormat postingsFormat;
         private AnalyzingCompletionLookupProvider analyzingSuggestLookupProvider;
         private SortedMap<String, ContextMapping> contextMapping = ContextMapping.EMPTY_MAPPING;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
index 8b744ce97da..a79631481d2 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
@@ -316,21 +316,6 @@ public class DateFieldMapper extends FieldMapper implements AllFieldMapper.Inclu
             return query;
         }
 
-        @Override
-        public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions, boolean transpositions) {
-            failIfNotIndexed();
-            long baseLo = parseToMilliseconds(value, false, null, dateMathParser);
-            long baseHi = parseToMilliseconds(value, true, null, dateMathParser);
-            long delta;
-            try {
-                delta = fuzziness.asTimeValue().millis();
-            } catch (Exception e) {
-                // not a time format
-                delta = fuzziness.asLong();
-            }
-            return LongPoint.newRangeQuery(name(), baseLo - delta, baseHi + delta);
-        }
-
         @Override
         public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) {
             failIfNotIndexed();
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/KeywordFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/KeywordFieldMapper.java
index bfcea8c6667..fe2b43aa512 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/KeywordFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/KeywordFieldMapper.java
@@ -22,13 +22,8 @@ package org.elasticsearch.index.mapper.core;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.RegexpQuery;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -40,8 +35,8 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.StringFieldType;
 import org.elasticsearch.index.mapper.internal.AllFieldMapper;
-import org.elasticsearch.index.query.QueryShardContext;
 
 import java.io.IOException;
 import java.util.Iterator;
@@ -143,7 +138,7 @@ public final class KeywordFieldMapper extends FieldMapper implements AllFieldMap
         }
     }
 
-    public static final class KeywordFieldType extends MappedFieldType {
+    public static final class KeywordFieldType extends StringFieldType {
 
         public KeywordFieldType() {}
 
@@ -173,17 +168,6 @@ public final class KeywordFieldMapper extends FieldMapper implements AllFieldMap
             failIfNoDocValues();
             return new DocValuesIndexFieldData.Builder();
         }
-
-        @Override
-        public Query regexpQuery(String value, int flags, int maxDeterminizedStates,
-                @Nullable MultiTermQuery.RewriteMethod method, @Nullable QueryShardContext context) {
-            failIfNotIndexed();
-            RegexpQuery query = new RegexpQuery(new Term(name(), indexedValueForSearch(value)), flags, maxDeterminizedStates);
-            if (method != null) {
-                query.setRewriteMethod(method);
-            }
-            return query;
-        }
     }
 
     private Boolean includeInAll;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyByteFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyByteFieldMapper.java
index 216fd7d359f..ff7efe76c8a 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyByteFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyByteFieldMapper.java
@@ -33,7 +33,6 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -159,16 +158,6 @@ public class LegacyByteFieldMapper extends LegacyNumberFieldMapper {
                 includeLower, includeUpper);
         }
 
-        @Override
-        public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions, boolean transpositions) {
-            byte iValue = parseValue(value);
-            byte iSim = fuzziness.asByte();
-            return LegacyNumericRangeQuery.newIntRange(name(), numericPrecisionStep(),
-                iValue - iSim,
-                iValue + iSim,
-                true, true);
-        }
-
         @Override
         public FieldStats.Long stats(IndexReader reader) throws IOException {
             int maxDoc = reader.maxDoc();
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDateFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDateFieldMapper.java
index 13d1cd23ffd..a7e44ba3654 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDateFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDateFieldMapper.java
@@ -358,22 +358,6 @@ public class LegacyDateFieldMapper extends LegacyNumberFieldMapper {
             return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, null, null);
         }
 
-        @Override
-        public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions, boolean transpositions) {
-            long iValue = parseValue(value);
-            long iSim;
-            try {
-                iSim = fuzziness.asTimeValue().millis();
-            } catch (Exception e) {
-                // not a time format
-                iSim =  fuzziness.asLong();
-            }
-            return LegacyNumericRangeQuery.newLongRange(name(), numericPrecisionStep(),
-                iValue - iSim,
-                iValue + iSim,
-                true, true);
-        }
-
         @Override
         public FieldStats.Date stats(IndexReader reader) throws IOException {
             int maxDoc = reader.maxDoc();
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDoubleFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDoubleFieldMapper.java
index 50135c0a66d..9e407a63340 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDoubleFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDoubleFieldMapper.java
@@ -36,7 +36,6 @@ import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.Numbers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -170,16 +169,6 @@ public class LegacyDoubleFieldMapper extends LegacyNumberFieldMapper {
                 includeLower, includeUpper);
         }
 
-        @Override
-        public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions, boolean transpositions) {
-            double iValue = parseDoubleValue(value);
-            double iSim = fuzziness.asDouble();
-            return LegacyNumericRangeQuery.newDoubleRange(name(), numericPrecisionStep(),
-                iValue - iSim,
-                iValue + iSim,
-                true, true);
-        }
-
         @Override
         public FieldStats.Double stats(IndexReader reader) throws IOException {
             int maxDoc = reader.maxDoc();
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyFloatFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyFloatFieldMapper.java
index 5e45cf4359d..d65801275ea 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyFloatFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyFloatFieldMapper.java
@@ -35,7 +35,6 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -155,16 +154,6 @@ public class LegacyFloatFieldMapper extends LegacyNumberFieldMapper {
                 includeLower, includeUpper);
         }
 
-        @Override
-        public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions, boolean transpositions) {
-            float iValue = parseValue(value);
-            final float iSim = fuzziness.asFloat();
-            return LegacyNumericRangeQuery.newFloatRange(name(), numericPrecisionStep(),
-                iValue - iSim,
-                iValue + iSim,
-                true, true);
-        }
-
         @Override
         public FieldStats.Double stats(IndexReader reader) throws IOException {
             int maxDoc = reader.maxDoc();
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyIntegerFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyIntegerFieldMapper.java
index be4d8cad261..f0ada9d6622 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyIntegerFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyIntegerFieldMapper.java
@@ -34,7 +34,6 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -135,8 +134,7 @@ public class LegacyIntegerFieldMapper extends LegacyNumberFieldMapper {
 
         @Override
         public String typeName() {
-            // TODO: this should be the same as the mapper type name, except fielddata expects int...
-            return "int";
+            return "integer";
         }
 
         @Override
@@ -159,16 +157,6 @@ public class LegacyIntegerFieldMapper extends LegacyNumberFieldMapper {
                 includeLower, includeUpper);
         }
 
-        @Override
-        public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions, boolean transpositions) {
-            int iValue = parseValue(value);
-            int iSim = fuzziness.asInt();
-            return LegacyNumericRangeQuery.newIntRange(name(), numericPrecisionStep(),
-                iValue - iSim,
-                iValue + iSim,
-                true, true);
-        }
-
         @Override
         public FieldStats.Long stats(IndexReader reader) throws IOException {
             int maxDoc = reader.maxDoc();
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyLongFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyLongFieldMapper.java
index 1601e009824..f53bb42fb90 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyLongFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyLongFieldMapper.java
@@ -34,7 +34,6 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.fielddata.IndexFieldData;
@@ -158,16 +157,6 @@ public class LegacyLongFieldMapper extends LegacyNumberFieldMapper {
                 includeLower, includeUpper);
         }
 
-        @Override
-        public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions, boolean transpositions) {
-            long iValue = parseLongValue(value);
-            final long iSim = fuzziness.asLong();
-            return LegacyNumericRangeQuery.newLongRange(name(), numericPrecisionStep(),
-                iValue - iSim,
-                iValue + iSim,
-                true, true);
-        }
-
         @Override
         public FieldStats stats(IndexReader reader) throws IOException {
             int maxDoc = reader.maxDoc();
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyNumberFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyNumberFieldMapper.java
index 28e68ce59d5..7278e17a3c1 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyNumberFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyNumberFieldMapper.java
@@ -27,20 +27,19 @@ import org.apache.lucene.analysis.LegacyNumericTokenStream;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.TermBasedFieldType;
 import org.elasticsearch.index.mapper.internal.AllFieldMapper;
 import org.elasticsearch.search.DocValueFormat;
 import org.joda.time.DateTimeZone;
@@ -121,7 +120,7 @@ public abstract class LegacyNumberFieldMapper extends FieldMapper implements All
         protected abstract int maxPrecisionStep();
     }
 
-    public static abstract class NumberFieldType extends MappedFieldType {
+    public static abstract class NumberFieldType extends TermBasedFieldType {
 
         public NumberFieldType(LegacyNumericType numericType) {
             setTokenized(false);
@@ -146,9 +145,6 @@ public abstract class LegacyNumberFieldMapper extends FieldMapper implements All
 
         public abstract NumberFieldType clone();
 
-        @Override
-        public abstract Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions, boolean transpositions);
-
         @Override
         public DocValueFormat docValueFormat(@Nullable String format, DateTimeZone timeZone) {
             if (timeZone != null) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyShortFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyShortFieldMapper.java
index 32c1955fc50..2c07714f452 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyShortFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyShortFieldMapper.java
@@ -163,16 +163,6 @@ public class LegacyShortFieldMapper extends LegacyNumberFieldMapper {
                 includeLower, includeUpper);
         }
 
-        @Override
-        public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions, boolean transpositions) {
-            short iValue = parseValue(value);
-            short iSim = fuzziness.asShort();
-            return LegacyNumericRangeQuery.newIntRange(name(), numericPrecisionStep(),
-                iValue - iSim,
-                iValue + iSim,
-                true, true);
-        }
-
         @Override
         public FieldStats.Long stats(IndexReader reader) throws IOException {
             int maxDoc = reader.maxDoc();
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
index 4717140277b..568b651912a 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
@@ -40,7 +40,6 @@ import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
@@ -233,13 +232,6 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
                 return FloatPoint.newRangeQuery(field, l, u);
             }
 
-            @Override
-            Query fuzzyQuery(String field, Object value, Fuzziness fuzziness) {
-                float base = parse(value);
-                float delta = fuzziness.asFloat();
-                return rangeQuery(field, base - delta, base + delta, true, true);
-            }
-
             @Override
             public List<Field> createFields(String name, Number value,
                                             boolean indexed, boolean docValued, boolean stored) {
@@ -324,13 +316,6 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
                 return DoublePoint.newRangeQuery(field, l, u);
             }
 
-            @Override
-            Query fuzzyQuery(String field, Object value, Fuzziness fuzziness) {
-                double base = parse(value);
-                double delta = fuzziness.asFloat();
-                return rangeQuery(field, base - delta, base + delta, true, true);
-            }
-
             @Override
             public List<Field> createFields(String name, Number value,
                                             boolean indexed, boolean docValued, boolean stored) {
@@ -407,11 +392,6 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
                 return INTEGER.rangeQuery(field, lowerTerm, upperTerm, includeLower, includeUpper);
             }
 
-            @Override
-            Query fuzzyQuery(String field, Object value, Fuzziness fuzziness) {
-                return INTEGER.fuzzyQuery(field, value, fuzziness);
-            }
-
             @Override
             public List<Field> createFields(String name, Number value,
                                             boolean indexed, boolean docValued, boolean stored) {
@@ -473,11 +453,6 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
                 return INTEGER.rangeQuery(field, lowerTerm, upperTerm, includeLower, includeUpper);
             }
 
-            @Override
-            Query fuzzyQuery(String field, Object value, Fuzziness fuzziness) {
-                return INTEGER.fuzzyQuery(field, value, fuzziness);
-            }
-
             @Override
             public List<Field> createFields(String name, Number value,
                                             boolean indexed, boolean docValued, boolean stored) {
@@ -560,13 +535,6 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
                 return IntPoint.newRangeQuery(field, l, u);
             }
 
-            @Override
-            Query fuzzyQuery(String field, Object value, Fuzziness fuzziness) {
-                int base = parse(value);
-                int delta = fuzziness.asInt();
-                return rangeQuery(field, base - delta, base + delta, true, true);
-            }
-
             @Override
             public List<Field> createFields(String name, Number value,
                                             boolean indexed, boolean docValued, boolean stored) {
@@ -663,13 +631,6 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
                 return LongPoint.newRangeQuery(field, l, u);
             }
 
-            @Override
-            Query fuzzyQuery(String field, Object value, Fuzziness fuzziness) {
-                long base = parse(value);
-                long delta = fuzziness.asLong();
-                return rangeQuery(field, base - delta, base + delta, true, true);
-            }
-
             @Override
             public List<Field> createFields(String name, Number value,
                                             boolean indexed, boolean docValued, boolean stored) {
@@ -722,7 +683,6 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
         abstract Query termsQuery(String field, List<Object> values);
         abstract Query rangeQuery(String field, Object lowerTerm, Object upperTerm,
                                   boolean includeLower, boolean includeUpper);
-        abstract Query fuzzyQuery(String field, Object value, Fuzziness fuzziness);
         abstract Number parse(XContentParser parser, boolean coerce) throws IOException;
         abstract Number parse(Object value);
         public abstract List<Field> createFields(String name, Number value, boolean indexed,
@@ -791,13 +751,6 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
             return query;
         }
 
-        @Override
-        public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength,
-                                int maxExpansions, boolean transpositions) {
-            failIfNotIndexed();
-            return type.fuzzyQuery(name(), value, fuzziness);
-        }
-
         @Override
         public FieldStats stats(IndexReader reader) throws IOException {
             return type.stats(reader, name(), isSearchable(), isAggregatable());
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java
index 0c98dd10c3e..197b34b67da 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java
@@ -355,7 +355,7 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
         }
     }
 
-    public static final class StringFieldType extends MappedFieldType {
+    public static final class StringFieldType extends org.elasticsearch.index.mapper.StringFieldType {
 
         private boolean fielddata;
         private double fielddataMinFrequency;
@@ -485,15 +485,6 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
                         + "use significant memory.");
             }
         }
-
-        @Override
-        public Query regexpQuery(String value, int flags, int maxDeterminizedStates, @Nullable MultiTermQuery.RewriteMethod method, @Nullable QueryShardContext context) {
-            RegexpQuery query = new RegexpQuery(new Term(name(), indexedValueForSearch(value)), flags, maxDeterminizedStates);
-            if (method != null) {
-                query.setRewriteMethod(method);
-            }
-            return query;
-        }
     }
 
     private Boolean includeInAll;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/TextFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/TextFieldMapper.java
index 3815293835a..4696bb13185 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/TextFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/TextFieldMapper.java
@@ -21,12 +21,7 @@ package org.elasticsearch.index.mapper.core;
 
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.RegexpQuery;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
@@ -39,8 +34,8 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.StringFieldType;
 import org.elasticsearch.index.mapper.internal.AllFieldMapper;
-import org.elasticsearch.index.query.QueryShardContext;
 
 import java.io.IOException;
 import java.util.Iterator;
@@ -172,7 +167,7 @@ public class TextFieldMapper extends FieldMapper implements AllFieldMapper.Inclu
         }
     }
 
-    public static final class TextFieldType extends MappedFieldType {
+    public static final class TextFieldType extends StringFieldType {
 
         private boolean fielddata;
         private double fielddataMinFrequency;
@@ -300,17 +295,6 @@ public class TextFieldMapper extends FieldMapper implements AllFieldMapper.Inclu
             }
             return new PagedBytesIndexFieldData.Builder(fielddataMinFrequency, fielddataMaxFrequency, fielddataMinSegmentSize);
         }
-
-        @Override
-        public Query regexpQuery(String value, int flags, int maxDeterminizedStates,
-                @Nullable MultiTermQuery.RewriteMethod method, @Nullable QueryShardContext context) {
-            failIfNotIndexed();
-            RegexpQuery query = new RegexpQuery(new Term(name(), indexedValueForSearch(value)), flags, maxDeterminizedStates);
-            if (method != null) {
-                query.setRewriteMethod(method);
-            }
-            return query;
-        }
     }
 
     private Boolean includeInAll;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/geo/BaseGeoPointFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/geo/BaseGeoPointFieldMapper.java
index 947733269ee..682513a9a58 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/geo/BaseGeoPointFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/geo/BaseGeoPointFieldMapper.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.mapper.geo;
 
 import org.apache.lucene.document.Field;
+import org.apache.lucene.search.Query;
 import org.elasticsearch.common.geo.GeoHashUtils;
 import org.apache.lucene.util.LegacyNumericUtils;
 import org.elasticsearch.Version;
@@ -47,6 +48,8 @@ import org.elasticsearch.index.mapper.core.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.core.LegacyNumberFieldMapper;
 import org.elasticsearch.index.mapper.core.NumberFieldMapper;
 import org.elasticsearch.index.mapper.object.ArrayValueMapperParser;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.search.DocValueFormat;
 import org.joda.time.DateTimeZone;
 
@@ -366,6 +369,11 @@ public abstract class BaseGeoPointFieldMapper extends FieldMapper implements Arr
             }
             return DocValueFormat.GEOHASH;
         }
+
+        @Override
+        public Query termQuery(Object value, QueryShardContext context) {
+            throw new QueryShardException(context, "Geo fields do not support exact searching, use dedicated geo queries instead: [" + name() + "]");
+        }
     }
 
     protected FieldMapper latMapper;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java
index f542aeb07ca..ac31c3a7243 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoShapeFieldMapper.java
@@ -23,6 +23,7 @@ import org.locationtech.spatial4j.shape.Shape;
 import org.locationtech.spatial4j.shape.jts.JtsGeometry;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.spatial.prefix.PrefixTreeStrategy;
 import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
 import org.apache.lucene.spatial.prefix.TermQueryPrefixTreeStrategy;
@@ -46,6 +47,8 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.QueryShardException;
 
 import java.io.IOException;
 import java.util.Iterator;
@@ -412,6 +415,10 @@ public class GeoShapeFieldMapper extends FieldMapper {
             throw new IllegalArgumentException("Unknown prefix tree strategy [" + strategyName + "]");
         }
 
+        @Override
+        public Query termQuery(Object value, QueryShardContext context) {
+            throw new QueryShardException(context, "Geo fields do not support exact searching, use dedicated geo queries instead");
+        }
     }
 
     protected Explicit<Boolean> coerce;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
index 8a384e94e16..02a2cb0c780 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java
@@ -23,7 +23,6 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.lucene.all.AllEntries;
 import org.elasticsearch.common.lucene.all.AllField;
@@ -36,6 +35,7 @@ import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.StringFieldType;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.similarity.SimilarityService;
 
@@ -177,7 +177,7 @@ public class AllFieldMapper extends MetadataFieldMapper {
         }
     }
 
-    static final class AllFieldType extends MappedFieldType {
+    static final class AllFieldType extends StringFieldType {
 
         public AllFieldType() {
         }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java
index c2db784e058..06dadf7c4e6 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java
@@ -23,7 +23,6 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -32,6 +31,7 @@ import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.TermBasedFieldType;
 import org.elasticsearch.index.query.QueryShardContext;
 
 import java.io.IOException;
@@ -125,7 +125,7 @@ public class FieldNamesFieldMapper extends MetadataFieldMapper {
         }
     }
 
-    public static final class FieldNamesFieldType extends MappedFieldType {
+    public static final class FieldNamesFieldType extends TermBasedFieldType {
 
         private boolean enabled = Defaults.ENABLED;
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
index 5f38a4c6a0b..67bacf93e86 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java
@@ -41,6 +41,7 @@ import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.TermBasedFieldType;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.query.QueryShardContext;
 
@@ -89,7 +90,7 @@ public class IdFieldMapper extends MetadataFieldMapper {
         }
     }
 
-    static final class IdFieldType extends MappedFieldType {
+    static final class IdFieldType extends TermBasedFieldType {
 
         public IdFieldType() {
         }
@@ -116,62 +117,14 @@ public class IdFieldMapper extends MetadataFieldMapper {
 
         @Override
         public Query termQuery(Object value, @Nullable QueryShardContext context) {
-            if (indexOptions() != IndexOptions.NONE || context == null) {
-                return super.termQuery(value, context);
-            }
             final BytesRef[] uids = Uid.createUidsForTypesAndId(context.queryTypes(), value);
             return new TermsQuery(UidFieldMapper.NAME, uids);
         }
 
         @Override
         public Query termsQuery(List values, @Nullable QueryShardContext context) {
-            if (indexOptions() != IndexOptions.NONE || context == null) {
-                return super.termsQuery(values, context);
-            }
             return new TermsQuery(UidFieldMapper.NAME, Uid.createUidsForTypesAndIds(context.queryTypes(), values));
         }
-
-        @Override
-        public Query prefixQuery(String value, @Nullable MultiTermQuery.RewriteMethod method, @Nullable QueryShardContext context) {
-            if (indexOptions() != IndexOptions.NONE || context == null) {
-                return super.prefixQuery(value, method, context);
-            }
-            Collection<String> queryTypes = context.queryTypes();
-            BooleanQuery.Builder query = new BooleanQuery.Builder();
-            for (String queryType : queryTypes) {
-                PrefixQuery prefixQuery = new PrefixQuery(new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(queryType, BytesRefs.toBytesRef(value))));
-                if (method != null) {
-                    prefixQuery.setRewriteMethod(method);
-                }
-                query.add(prefixQuery, BooleanClause.Occur.SHOULD);
-            }
-            return query.build();
-        }
-
-        @Override
-        public Query regexpQuery(String value, int flags, int maxDeterminizedStates, @Nullable MultiTermQuery.RewriteMethod method, @Nullable QueryShardContext context) {
-            if (indexOptions() != IndexOptions.NONE || context == null) {
-                return super.regexpQuery(value, flags, maxDeterminizedStates, method, context);
-            }
-            Collection<String> queryTypes = context.queryTypes();
-            if (queryTypes.size() == 1) {
-                RegexpQuery regexpQuery = new RegexpQuery(new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(Iterables.getFirst(queryTypes, null), BytesRefs.toBytesRef(value))),
-                    flags, maxDeterminizedStates);
-                if (method != null) {
-                    regexpQuery.setRewriteMethod(method);
-                }
-                return regexpQuery;
-            }
-            BooleanQuery.Builder query = new BooleanQuery.Builder();
-            for (String queryType : queryTypes) {
-                RegexpQuery regexpQuery = new RegexpQuery(new Term(UidFieldMapper.NAME, Uid.createUidAsBytes(queryType, BytesRefs.toBytesRef(value))), flags, maxDeterminizedStates);
-                if (method != null) {
-                    regexpQuery.setRewriteMethod(method);
-                }
-                query.add(regexpQuery, BooleanClause.Occur.SHOULD);
-            }
-            return query.build();
-        }
     }
 
     private IdFieldMapper(Settings indexSettings, MappedFieldType existing) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
index 3189aa9b582..408641fb67e 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
@@ -130,9 +130,6 @@ public class IndexFieldMapper extends MetadataFieldMapper {
          */
         @Override
         public Query termQuery(Object value, @Nullable QueryShardContext context) {
-            if (context == null) {
-                return super.termQuery(value, context);
-            }
             if (isSameIndex(value, context.index().getName())) {
                 return Queries.newMatchAllQuery();
             } else {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
index b98eb6720bc..ce43248443e 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
@@ -31,6 +31,7 @@ import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.loader.SettingsLoader;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -194,7 +195,7 @@ public class ParentFieldMapper extends MetadataFieldMapper {
         public Query termsQuery(List values, @Nullable QueryShardContext context) {
             BytesRef[] ids = new BytesRef[values.size()];
             for (int i = 0; i < ids.length; i++) {
-                ids[i] = indexedValueForSearch(values.get(i));
+                ids[i] = BytesRefs.toBytesRef(values.get(i));
             }
             BooleanQuery.Builder query = new BooleanQuery.Builder();
             query.add(new DocValuesTermsQuery(name(), ids), BooleanClause.Occur.MUST);
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java
index 68641ebc29a..031732f4d11 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java
@@ -19,10 +19,8 @@
 
 package org.elasticsearch.index.mapper.internal;
 
-import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -31,6 +29,7 @@ import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.TermBasedFieldType;
 
 import java.io.IOException;
 import java.util.Iterator;
@@ -107,7 +106,7 @@ public class RoutingFieldMapper extends MetadataFieldMapper {
         }
     }
 
-    static final class RoutingFieldType extends MappedFieldType {
+    static final class RoutingFieldType extends TermBasedFieldType {
 
         public RoutingFieldType() {
         }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
index a05b3a451c7..ce30490968c 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
@@ -22,6 +22,7 @@ package org.elasticsearch.index.mapper.internal;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.search.Query;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -39,6 +40,8 @@ import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.QueryShardException;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -167,6 +170,11 @@ public class SourceFieldMapper extends MetadataFieldMapper {
         public String typeName() {
             return CONTENT_TYPE;
         }
+
+        @Override
+        public Query termQuery(Object value, QueryShardContext context) {
+            throw new QueryShardException(context, "The _source field is not searchable");
+        }
     }
 
     private final boolean enabled;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
index 0dbdca87b60..7aee0aefad8 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
@@ -40,6 +40,7 @@ import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.StringFieldType;
 import org.elasticsearch.index.query.QueryShardContext;
 
 import java.io.IOException;
@@ -85,7 +86,7 @@ public class TypeFieldMapper extends MetadataFieldMapper {
         }
     }
 
-    static final class TypeFieldType extends MappedFieldType {
+    static final class TypeFieldType extends StringFieldType {
 
         public TypeFieldType() {
         }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
index 133c018cc6a..e020864c7e8 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java
@@ -33,6 +33,7 @@ import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.TermBasedFieldType;
 import org.elasticsearch.index.mapper.core.TextFieldMapper;
 import org.elasticsearch.index.mapper.Uid;
 
@@ -83,7 +84,7 @@ public class UidFieldMapper extends MetadataFieldMapper {
         }
     }
 
-    static final class UidFieldType extends MappedFieldType {
+    static final class UidFieldType extends TermBasedFieldType {
 
         public UidFieldType() {
         }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java
index 1206a058eac..98a69702d2a 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java
@@ -22,6 +22,7 @@ package org.elasticsearch.index.mapper.internal;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.search.Query;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -30,6 +31,8 @@ import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParseContext.Document;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.QueryShardException;
 
 import java.io.IOException;
 import java.util.List;
@@ -84,6 +87,11 @@ public class VersionFieldMapper extends MetadataFieldMapper {
         public String typeName() {
             return CONTENT_TYPE;
         }
+
+        @Override
+        public Query termQuery(Object value, QueryShardContext context) {
+            throw new QueryShardException(context, "The _version field is not searchable");
+        }
     }
 
     private VersionFieldMapper(Settings indexSettings) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
index 8435bdeb681..1e2a078925f 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
@@ -36,7 +36,6 @@ import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
@@ -219,14 +218,6 @@ public class IpFieldMapper extends FieldMapper implements AllFieldMapper.Include
             return InetAddressPoint.newRangeQuery(name(), lower, upper);
         }
 
-        @Override
-        public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions, boolean transpositions) {
-            failIfNotIndexed();
-            InetAddress base = parse(value);
-            int mask = fuzziness.asInt();
-            return XInetAddressPoint.newPrefixQuery(name(), base, mask);
-        }
-
         @Override
         public FieldStats.Ip stats(IndexReader reader) throws IOException {
             String field = name();
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpFieldMapper.java
index 01fa2106684..19141a675a3 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpFieldMapper.java
@@ -231,21 +231,6 @@ public class LegacyIpFieldMapper extends LegacyNumberFieldMapper {
                 includeLower, includeUpper);
         }
 
-        @Override
-        public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions, boolean transpositions) {
-            long iValue = parseValue(value);
-            long iSim;
-            try {
-                iSim = ipToLong(fuzziness.asString());
-            } catch (IllegalArgumentException e) {
-                iSim = fuzziness.asLong();
-            }
-            return LegacyNumericRangeQuery.newLongRange(name(), numericPrecisionStep(),
-                iValue - iSim,
-                iValue + iSim,
-                true, true);
-        }
-
         @Override
         public FieldStats stats(IndexReader reader) throws IOException {
             int maxDoc = reader.maxDoc();
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
index c50085857ff..59b980d59a4 100644
--- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
@@ -39,6 +39,7 @@ import org.elasticsearch.index.mapper.core.KeywordFieldMapper;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.QueryShardException;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -149,6 +150,11 @@ public class PercolatorFieldMapper extends FieldMapper {
         public String typeName() {
             return CONTENT_TYPE;
         }
+
+        @Override
+        public Query termQuery(Object value, QueryShardContext context) {
+            throw new QueryShardException(context, "Percolator fields are not searchable directly, use a percolate query instead");
+        }
     }
 
     private final boolean mapUnmappedFieldAsString;
diff --git a/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java b/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java
index 0fbc17e3ae8..47585c2210d 100644
--- a/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java
+++ b/core/src/main/java/org/elasticsearch/index/search/MatchQuery.java
@@ -231,7 +231,7 @@ public class MatchQuery {
          */
         boolean noForcedAnalyzer = this.analyzer == null;
         if (fieldType != null && fieldType.tokenized() == false && noForcedAnalyzer) {
-            return termQuery(fieldType, value);
+            return blendTermQuery(new Term(fieldName, value.toString()), fieldType);
         }
 
         Analyzer analyzer = getAnalyzer(fieldType);
@@ -265,15 +265,6 @@ public class MatchQuery {
         }
     }
 
-    /**
-     * Creates a TermQuery-like-query for MappedFieldTypes that don't support
-     * QueryBuilder which is very string-ish. Just delegates to the
-     * MappedFieldType for MatchQuery but gets more complex for blended queries.
-     */
-    protected Query termQuery(MappedFieldType fieldType, Object value) {
-        return termQuery(fieldType, value, lenient);
-    }
-
     protected final Query termQuery(MappedFieldType fieldType, Object value, boolean lenient) {
         try {
             return fieldType.termQuery(value, context);
@@ -366,8 +357,11 @@ public class MatchQuery {
                     }
                     return query;
                 } catch (RuntimeException e) {
-                    return new TermQuery(term);
-                    // See long comment below about why we're lenient here.
+                    if (lenient) {
+                        return new TermQuery(term);
+                    } else {
+                        throw e;
+                    }
                 }
             }
             int edits = fuzziness.asDistance(term.text());
@@ -376,23 +370,7 @@ public class MatchQuery {
             return query;
         }
         if (fieldType != null) {
-            /*
-             * Its a bit weird to default to lenient here but its the backwards
-             * compatible. It makes some sense when you think about what we are
-             * doing here: at this point the user has forced an analyzer and
-             * passed some string to the match query. We cut it up using the
-             * analyzer and then tried to cram whatever we get into the field.
-             * lenient=true here means that we try the terms in the query and on
-             * the off chance that they are actually valid terms then we
-             * actually try them. lenient=false would mean that we blow up the
-             * query if they aren't valid terms. "valid" in this context means
-             * "parses properly to something of the type being queried." So "1"
-             * is a valid number, etc.
-             *
-             * We use the text form here because we we've received the term from
-             * an analyzer that cut some string into text.
-             */
-            Query query = termQuery(fieldType, term.bytes(), true);
+            Query query = termQuery(fieldType, term.bytes(), lenient);
             if (query != null) {
                 return query;
             }
diff --git a/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java b/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java
index fb034690e21..115938a2c06 100644
--- a/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java
+++ b/core/src/main/java/org/elasticsearch/index/search/MultiMatchQuery.java
@@ -303,15 +303,6 @@ public class MultiMatchQuery extends MatchQuery {
         return queryBuilder.blendTerm(term, fieldType);
     }
 
-    @Override
-    protected Query termQuery(MappedFieldType fieldType, Object value) {
-        if (queryBuilder == null) {
-            // Can be null when the MultiMatchQuery collapses into a MatchQuery
-            return super.termQuery(fieldType, value);
-        }
-        return queryBuilder.termQuery(fieldType, value);
-    }
-
     static final class FieldAndFieldType {
         final MappedFieldType fieldType;
         final float boost;
diff --git a/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java b/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java
index bc0da6bf1df..2b5a7c00e5d 100644
--- a/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java
+++ b/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java
@@ -36,12 +36,7 @@ import static org.hamcrest.number.IsCloseTo.closeTo;
 public class FuzzinessTests extends ESTestCase {
     public void testNumerics() {
         String[] options = new String[]{"1.0", "1", "1.000000"};
-        assertThat(Fuzziness.build(randomFrom(options)).asByte(), equalTo((byte) 1));
-        assertThat(Fuzziness.build(randomFrom(options)).asInt(), equalTo(1));
         assertThat(Fuzziness.build(randomFrom(options)).asFloat(), equalTo(1f));
-        assertThat(Fuzziness.build(randomFrom(options)).asDouble(), equalTo(1d));
-        assertThat(Fuzziness.build(randomFrom(options)).asLong(), equalTo(1L));
-        assertThat(Fuzziness.build(randomFrom(options)).asShort(), equalTo((short) 1));
     }
 
     public void testParseFromXContent() throws IOException {
@@ -59,7 +54,6 @@ public class FuzzinessTests extends ESTestCase {
                 assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_NUMBER));
                 Fuzziness parse = Fuzziness.parse(parser);
                 assertThat(parse.asFloat(), equalTo(floatValue));
-                assertThat(parse.asDouble(), closeTo(floatValue, 0.000001));
                 assertThat(parser.nextToken(), equalTo(XContentParser.Token.END_OBJECT));
             }
             {
@@ -78,10 +72,6 @@ public class FuzzinessTests extends ESTestCase {
                 assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
                 assertThat(parser.nextToken(), anyOf(equalTo(XContentParser.Token.VALUE_NUMBER), equalTo(XContentParser.Token.VALUE_STRING)));
                 Fuzziness parse = Fuzziness.parse(parser);
-                assertThat(parse.asInt(), equalTo(value.intValue()));
-                assertThat((int) parse.asShort(), equalTo(value.intValue()));
-                assertThat((int) parse.asByte(), equalTo(value.intValue()));
-                assertThat(parse.asLong(), equalTo(value.longValue()));
                 if (value.intValue() >= 1) {
                     assertThat(parse.asDistance(), equalTo(Math.min(2, value.intValue())));
                 }
@@ -118,35 +108,12 @@ public class FuzzinessTests extends ESTestCase {
                 assertThat(parse, sameInstance(Fuzziness.AUTO));
                 assertThat(parser.nextToken(), equalTo(XContentParser.Token.END_OBJECT));
             }
-
-            {
-                String[] values = new String[]{"d", "H", "ms", "s", "S", "w"};
-                String actual = randomIntBetween(1, 3) + randomFrom(values);
-                XContent xcontent = XContentType.JSON.xContent();
-                String json = jsonBuilder().startObject()
-                        .field(Fuzziness.X_FIELD_NAME, actual)
-                        .endObject().string();
-                XContentParser parser = xcontent.createParser(json);
-                assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT));
-                assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME));
-                assertThat(parser.nextToken(), equalTo(XContentParser.Token.VALUE_STRING));
-                Fuzziness parse = Fuzziness.parse(parser);
-                assertThat(parse.asTimeValue(), equalTo(TimeValue.parseTimeValue(actual, null, "fuzziness")));
-                assertThat(parser.nextToken(), equalTo(XContentParser.Token.END_OBJECT));
-            }
         }
 
     }
 
     public void testAuto() {
-        assertThat(Fuzziness.AUTO.asByte(), equalTo((byte) 1));
-        assertThat(Fuzziness.AUTO.asInt(), equalTo(1));
         assertThat(Fuzziness.AUTO.asFloat(), equalTo(1f));
-        assertThat(Fuzziness.AUTO.asDouble(), equalTo(1d));
-        assertThat(Fuzziness.AUTO.asLong(), equalTo(1L));
-        assertThat(Fuzziness.AUTO.asShort(), equalTo((short) 1));
-        assertThat(Fuzziness.AUTO.asTimeValue(), equalTo(TimeValue.parseTimeValue("1ms", TimeValue.timeValueMillis(1), "fuzziness")));
-
     }
 
     public void testAsDistance() {
@@ -172,7 +139,7 @@ public class FuzzinessTests extends ESTestCase {
         Fuzziness fuzziness = Fuzziness.AUTO;
         Fuzziness deserializedFuzziness = doSerializeRoundtrip(fuzziness);
         assertEquals(fuzziness, deserializedFuzziness);
-        assertEquals(fuzziness.asInt(), deserializedFuzziness.asInt());
+        assertEquals(fuzziness.asFloat(), deserializedFuzziness.asFloat(), 0f);
     }
 
     private static Fuzziness doSerializeRoundtrip(Fuzziness in) throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java
index 06307614a51..43fc1d5c82a 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java
@@ -66,7 +66,7 @@ public class DocumentFieldMapperTests extends LuceneTestCase {
         
     }
 
-    static class FakeFieldType extends MappedFieldType {
+    static class FakeFieldType extends TermBasedFieldType {
 
         public FakeFieldType() {
             super();
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java
index cb9a64d357c..80e5df4081c 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java
@@ -205,7 +205,7 @@ public class FieldTypeLookupTests extends ESTestCase {
         return count;
     }
 
-    static class OtherFakeFieldType extends MappedFieldType {
+    static class OtherFakeFieldType extends TermBasedFieldType {
         public OtherFakeFieldType() {
         }
 
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
index d6a161f9ea1..68e59527982 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
@@ -273,7 +273,7 @@ public abstract class FieldTypeTestCase extends ESTestCase {
         fieldType.checkCompatibility(fieldType, conflicts, random().nextBoolean()); // no exception
         assertTrue(conflicts.toString(), conflicts.isEmpty());
 
-        MappedFieldType bogus = new MappedFieldType() {
+        MappedFieldType bogus = new TermBasedFieldType() {
             @Override
             public MappedFieldType clone() {return null;}
             @Override
@@ -287,7 +287,7 @@ public abstract class FieldTypeTestCase extends ESTestCase {
         }
         assertTrue(conflicts.toString(), conflicts.isEmpty());
 
-        MappedFieldType other = new MappedFieldType() {
+        MappedFieldType other = new TermBasedFieldType() {
             @Override
             public MappedFieldType clone() {return null;}
             @Override
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java
index e1b94967b43..a1571e47527 100755
--- a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapper.java
@@ -22,7 +22,6 @@ package org.elasticsearch.index.mapper.externalvalues;
 import org.locationtech.spatial4j.shape.Point;
 import org.apache.lucene.document.Field;
 import org.elasticsearch.Version;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.builders.ShapeBuilders;
@@ -33,9 +32,9 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.TermBasedFieldType;
 import org.elasticsearch.index.mapper.core.BinaryFieldMapper;
 import org.elasticsearch.index.mapper.core.BooleanFieldMapper;
-import org.elasticsearch.index.mapper.core.StringFieldMapper;
 import org.elasticsearch.index.mapper.core.TextFieldMapper;
 import org.elasticsearch.index.mapper.geo.BaseGeoPointFieldMapper;
 import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
@@ -50,7 +49,6 @@ import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.index.mapper.core.TypeParsers.parseField;
-import static org.elasticsearch.index.mapper.core.TypeParsers.parseMultiField;
 
 /**
  * This mapper add a new sub fields
@@ -129,7 +127,7 @@ public class ExternalMapper extends FieldMapper {
         }
     }
 
-    static class ExternalFieldType extends MappedFieldType {
+    static class ExternalFieldType extends TermBasedFieldType {
 
         public ExternalFieldType() {}
 
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/FakeStringFieldMapper.java b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/FakeStringFieldMapper.java
index e0c1243f82f..09063d685c6 100755
--- a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/FakeStringFieldMapper.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/FakeStringFieldMapper.java
@@ -22,12 +22,8 @@ package org.elasticsearch.index.mapper.externalvalues;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.RegexpQuery;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -36,8 +32,8 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.StringFieldType;
 import org.elasticsearch.index.mapper.core.StringFieldMapper;
-import org.elasticsearch.index.query.QueryShardContext;
 
 import java.io.IOException;
 import java.util.List;
@@ -93,7 +89,7 @@ public class FakeStringFieldMapper extends FieldMapper {
         }
     }
 
-    public static final class FakeStringFieldType extends MappedFieldType {
+    public static final class FakeStringFieldType extends StringFieldType {
 
 
         public FakeStringFieldType() {
@@ -119,16 +115,6 @@ public class FakeStringFieldMapper extends FieldMapper {
             }
             return termQuery(nullValue(), null);
         }
-
-        @Override
-        public Query regexpQuery(String value, int flags, int maxDeterminizedStates, @Nullable MultiTermQuery.RewriteMethod method,
-                                 @Nullable QueryShardContext context) {
-            RegexpQuery query = new RegexpQuery(new Term(name(), indexedValueForSearch(value)), flags, maxDeterminizedStates);
-            if (method != null) {
-                query.setRewriteMethod(method);
-            }
-            return query;
-        }
     }
 
     protected FakeStringFieldMapper(String simpleName, FakeStringFieldType fieldType, MappedFieldType defaultFieldType,
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java
index 6fc4c4a02ac..cd7f46844fc 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java
@@ -35,6 +35,7 @@ import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.TermBasedFieldType;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.mapper.MapperRegistry;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -177,7 +178,7 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase {
 
         }
 
-        private static class DummyFieldType extends MappedFieldType {
+        private static class DummyFieldType extends TermBasedFieldType {
 
             public DummyFieldType() {
                 super();
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/internal/IdFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/internal/IdFieldTypeTests.java
index f4da1c0dfe7..bddd55429aa 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/internal/IdFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/internal/IdFieldTypeTests.java
@@ -26,4 +26,12 @@ public class IdFieldTypeTests extends FieldTypeTestCase {
     protected MappedFieldType createDefaultFieldType() {
         return new IdFieldMapper.IdFieldType();
     }
+
+    public void testRangeQuery() {
+        MappedFieldType ft = createDefaultFieldType();
+        ft.setName("_id");
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> ft.rangeQuery(null, null, randomBoolean(), randomBoolean()));
+        assertEquals("Field [_id] of type [_id] does not support range queries", e.getMessage());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/internal/UidFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/internal/UidFieldTypeTests.java
index 93a56da74f9..7f256e822e5 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/internal/UidFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/internal/UidFieldTypeTests.java
@@ -26,4 +26,12 @@ public class UidFieldTypeTests extends FieldTypeTestCase {
     protected MappedFieldType createDefaultFieldType() {
         return new UidFieldMapper.UidFieldType();
     }
+
+    public void testRangeQuery() {
+        MappedFieldType ft = createDefaultFieldType();
+        ft.setName("_uid");
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> ft.rangeQuery(null, null, randomBoolean(), randomBoolean()));
+        assertEquals("Field [_uid] of type [_uid] does not support range queries", e.getMessage());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java
index 24dd1b32675..a1633ab16e7 100644
--- a/core/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java
@@ -41,8 +41,7 @@ public class FuzzyQueryBuilderTests extends AbstractQueryTestCase<FuzzyQueryBuil
 
     @Override
     protected FuzzyQueryBuilder doCreateTestQueryBuilder() {
-        Tuple<String, Object> fieldAndValue = getRandomFieldNameAndValue();
-        FuzzyQueryBuilder query = new FuzzyQueryBuilder(fieldAndValue.v1(), fieldAndValue.v2());
+        FuzzyQueryBuilder query = new FuzzyQueryBuilder(STRING_FIELD_NAME, getRandomValueForFieldName(STRING_FIELD_NAME));
         if (randomBoolean()) {
             query.fuzziness(randomFuzziness(query.fieldName()));
         }
@@ -63,11 +62,7 @@ public class FuzzyQueryBuilderTests extends AbstractQueryTestCase<FuzzyQueryBuil
 
     @Override
     protected void doAssertLuceneQuery(FuzzyQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
-        if (isNumericFieldName(queryBuilder.fieldName()) || queryBuilder.fieldName().equals(DATE_FIELD_NAME)) {
-            assertThat(query, either(instanceOf(LegacyNumericRangeQuery.class)).or(instanceOf(PointRangeQuery.class)));
-        } else {
-            assertThat(query, instanceOf(FuzzyQuery.class));
-        }
+        assertThat(query, instanceOf(FuzzyQuery.class));
     }
 
     public void testIllegalArguments() {
@@ -142,14 +137,10 @@ public class FuzzyQueryBuilderTests extends AbstractQueryTestCase<FuzzyQueryBuil
                 "        }\n" +
                 "    }\n" +
                 "}\n";
-        Query parsedQuery = parseQuery(query).toQuery(createShardContext());
-        Query expected;
-        if (getIndexVersionCreated().onOrAfter(Version.V_5_0_0_alpha2)) {
-            expected = IntPoint.newRangeQuery(INT_FIELD_NAME, 7, 17);
-        } else {
-            expected = LegacyNumericRangeQuery.newIntRange(INT_FIELD_NAME, 7, 17, true, true);
-        }
-        assertEquals(expected, parsedQuery);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> parseQuery(query).toQuery(createShardContext()));
+        assertEquals("Can only use fuzzy queries on keyword and text fields - not on [mapped_int] which is of type [integer]",
+                e.getMessage());
     }
 
     public void testFromJson() throws IOException {
@@ -169,6 +160,6 @@ public class FuzzyQueryBuilderTests extends AbstractQueryTestCase<FuzzyQueryBuil
         FuzzyQueryBuilder parsed = (FuzzyQueryBuilder) parseQuery(json);
         checkGeneratedJson(json, parsed);
         assertEquals(json, 42.0, parsed.boost(), 0.00001);
-        assertEquals(json, 2, parsed.fuzziness().asInt());
+        assertEquals(json, 2, parsed.fuzziness().asFloat(), 0f);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
index 543a5ea08c3..6cec3ae8c52 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
@@ -33,13 +33,11 @@ import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
 import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.search.MatchQuery;
 import org.elasticsearch.index.search.MatchQuery.Type;
 import org.elasticsearch.index.search.MatchQuery.ZeroTermsQuery;
 import org.hamcrest.Matcher;
-import org.joda.time.format.ISODateTimeFormat;
 
 import java.io.IOException;
 import java.util.Locale;
@@ -74,10 +72,15 @@ public class MatchQueryBuilderTests extends AbstractQueryTestCase<MatchQueryBuil
         matchQuery.operator(randomFrom(Operator.values()));
 
         if (randomBoolean()) {
-            matchQuery.analyzer(randomFrom("simple", "keyword", "whitespace"));
+            if (fieldName.equals(DATE_FIELD_NAME)) {
+                // tokenized dates would trigger parse errors
+                matchQuery.analyzer(randomFrom("keyword", "whitespace"));
+            } else {
+                matchQuery.analyzer(randomFrom("simple", "keyword", "whitespace"));
+            }
         }
 
-        if (fieldName.equals(BOOLEAN_FIELD_NAME) == false && randomBoolean()) {
+        if (fieldName.equals(STRING_FIELD_NAME) && randomBoolean()) {
             matchQuery.fuzziness(randomFuzziness(fieldName));
         }
 
@@ -128,17 +131,17 @@ public class MatchQueryBuilderTests extends AbstractQueryTestCase<MatchQueryBuil
         case BOOLEAN:
             assertThat(query, either(instanceOf(BooleanQuery.class)).or(instanceOf(ExtendedCommonTermsQuery.class))
                     .or(instanceOf(TermQuery.class)).or(instanceOf(FuzzyQuery.class)).or(instanceOf(MatchNoDocsQuery.class))
-                    .or(instanceOf(LegacyNumericRangeQuery.class)).or(instanceOf(PointRangeQuery.class)));
+                    .or(instanceOf(PointRangeQuery.class)));
             break;
         case PHRASE:
             assertThat(query, either(instanceOf(BooleanQuery.class)).or(instanceOf(PhraseQuery.class))
                     .or(instanceOf(TermQuery.class)).or(instanceOf(FuzzyQuery.class))
-                    .or(instanceOf(LegacyNumericRangeQuery.class)).or(instanceOf(PointRangeQuery.class)));
+                    .or(instanceOf(PointRangeQuery.class)));
             break;
         case PHRASE_PREFIX:
             assertThat(query, either(instanceOf(BooleanQuery.class)).or(instanceOf(MultiPhrasePrefixQuery.class))
                     .or(instanceOf(TermQuery.class)).or(instanceOf(FuzzyQuery.class))
-                    .or(instanceOf(LegacyNumericRangeQuery.class)).or(instanceOf(PointRangeQuery.class)));
+                    .or(instanceOf(PointRangeQuery.class)));
             break;
         }
 
@@ -193,32 +196,6 @@ public class MatchQueryBuilderTests extends AbstractQueryTestCase<MatchQueryBuil
             assertThat(queryBuilder.fuzzyTranspositions(), equalTo(fuzzyQuery.getTranspositions()));
         }
 
-        if (query instanceof LegacyNumericRangeQuery) {
-            // These are fuzzy numeric queries
-            assertTrue(queryBuilder.fuzziness() != null);
-            @SuppressWarnings("unchecked")
-            LegacyNumericRangeQuery<Number> numericRangeQuery = (LegacyNumericRangeQuery<Number>) query;
-            assertTrue(numericRangeQuery.includesMin());
-            assertTrue(numericRangeQuery.includesMax());
-
-            double value;
-            double width;
-            if (queryBuilder.fieldName().equals(DATE_FIELD_NAME) == false) {
-                value = Double.parseDouble(queryBuilder.value().toString());
-                if (queryBuilder.fuzziness().equals(Fuzziness.AUTO)) {
-                    width = 1;
-                } else {
-                    width = queryBuilder.fuzziness().asDouble();
-                }
-            } else {
-                value = ISODateTimeFormat.dateTimeParser().parseMillis(queryBuilder.value().toString());
-                width = queryBuilder.fuzziness().asTimeValue().getMillis();
-            }
-
-            assertEquals(value - width, numericRangeQuery.getMin().doubleValue(), width * .1);
-            assertEquals(value + width, numericRangeQuery.getMax().doubleValue(), width * .1);
-        }
-
         if (query instanceof PointRangeQuery) {
             // TODO
         }
@@ -414,4 +391,37 @@ public class MatchQueryBuilderTests extends AbstractQueryTestCase<MatchQueryBuil
                     containsString("Deprecated field [" + type + "] used, expected [match] instead"));
         }
     }
+
+    public void testFuzzinessOnNonStringField() throws Exception {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        MatchQueryBuilder query = new MatchQueryBuilder(INT_FIELD_NAME, 42);
+        query.fuzziness(randomFuzziness(INT_FIELD_NAME));
+        QueryShardContext context = createShardContext();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> query.toQuery(context));
+        assertEquals("Can only use fuzzy queries on keyword and text fields - not on [mapped_int] which is of type [integer]",
+                e.getMessage());
+        query.analyzer("keyword"); // triggers a different code path
+        e = expectThrows(IllegalArgumentException.class,
+                () -> query.toQuery(context));
+        assertEquals("Can only use fuzzy queries on keyword and text fields - not on [mapped_int] which is of type [integer]",
+                e.getMessage());
+
+        query.lenient(true);
+        query.toQuery(context); // no exception
+        query.analyzer(null);
+        query.toQuery(context); // no exception
+    }
+
+    public void testExactOnUnsupportedField() throws Exception {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        MatchQueryBuilder query = new MatchQueryBuilder(GEO_POINT_FIELD_NAME, "2,3");
+        QueryShardContext context = createShardContext();
+        QueryShardException e = expectThrows(QueryShardException.class,
+                () -> query.toQuery(context));
+        assertEquals("Geo fields do not support exact searching, use dedicated geo queries instead: [mapped_geo_point]",
+                e.getMessage());
+        query.lenient(true);
+        query.toQuery(context); // no exception
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
index 6b921a5becb..f81b14b5b87 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
@@ -45,6 +45,7 @@ import java.util.Map;
 
 import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBooleanSubQuery;
+import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.either;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
@@ -77,12 +78,17 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase<MultiMatc
             query.operator(randomFrom(Operator.values()));
         }
         if (randomBoolean()) {
-            query.analyzer(randomAnalyzer());
+            if (fieldName.equals(DATE_FIELD_NAME)) {
+                // tokenized dates would trigger parse errors
+                query.analyzer("keyword");
+            } else {
+                query.analyzer(randomAnalyzer());
+            }
         }
         if (randomBoolean()) {
             query.slop(randomIntBetween(0, 5));
         }
-        if (randomBoolean() && (query.type() == Type.BEST_FIELDS || query.type() == Type.MOST_FIELDS)) {
+        if (fieldName.equals(STRING_FIELD_NAME) && randomBoolean() && (query.type() == Type.BEST_FIELDS || query.type() == Type.MOST_FIELDS)) {
             query.fuzziness(randomFuzziness(fieldName));
         }
         if (randomBoolean()) {
@@ -291,4 +297,23 @@ public class MultiMatchQueryBuilderTests extends AbstractQueryTestCase<MultiMatc
         ParsingException e = expectThrows(ParsingException.class, () -> parseQuery(json));
         assertEquals("[multi_match] unknown token [START_ARRAY] after [query]", e.getMessage());
     }
+
+    public void testFuzzinessOnNonStringField() throws Exception {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        MultiMatchQueryBuilder query = new MultiMatchQueryBuilder(42).field(INT_FIELD_NAME).field(BOOLEAN_FIELD_NAME);
+        query.fuzziness(randomFuzziness(INT_FIELD_NAME));
+        QueryShardContext context = createShardContext();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> query.toQuery(context));
+        assertThat(e.getMessage(), containsString("Can only use fuzzy queries on keyword and text fields"));
+        query.analyzer("keyword"); // triggers a different code path
+        e = expectThrows(IllegalArgumentException.class,
+                () -> query.toQuery(context));
+        assertThat(e.getMessage(), containsString("Can only use fuzzy queries on keyword and text fields"));
+
+        query.lenient(true);
+        query.toQuery(context); // no exception
+        query.analyzer(null);
+        query.toQuery(context); // no exception
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
index 4db017bdd78..d49e33e91cb 100644
--- a/core/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
@@ -28,6 +28,7 @@ import java.io.IOException;
 import java.util.Arrays;
 
 import static org.elasticsearch.index.query.QueryBuilders.prefixQuery;
+import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -93,4 +94,14 @@ public class PrefixQueryBuilderTests extends AbstractQueryTestCase<PrefixQueryBu
         assertEquals(json, 2.0, parsed.boost(), 0.00001);
         assertEquals(json, "user", parsed.fieldName());
     }
+
+    public void testNumeric() throws Exception {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        PrefixQueryBuilder query = prefixQuery(INT_FIELD_NAME, "12*");
+        QueryShardContext context = createShardContext();
+        QueryShardException e = expectThrows(QueryShardException.class,
+                () -> query.toQuery(context));
+        assertEquals("Can only use prefix queries on keyword and text fields - not on [mapped_int] which is of type [integer]",
+                e.getMessage());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
index 4de7dae8c0b..375db0c1eaf 100644
--- a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
@@ -38,7 +38,6 @@ import org.apache.lucene.search.SynonymQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
-import org.elasticsearch.Version;
 import org.elasticsearch.common.lucene.all.AllTermQuery;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.hamcrest.Matchers;
@@ -393,16 +392,40 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
         }
     }
 
-    public void testToQueryNumericRangeQuery() throws Exception {
+    public void testFuzzyNumeric() throws Exception {
         assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
-        Query query = queryStringQuery("12~0.2").defaultField(INT_FIELD_NAME).toQuery(createShardContext());
-        if (getIndexVersionCreated().onOrAfter(Version.V_5_0_0_alpha2)) {
-            assertEquals(IntPoint.newExactQuery(INT_FIELD_NAME, 12), query);
-        } else {
-            LegacyNumericRangeQuery fuzzyQuery = (LegacyNumericRangeQuery) query;
-            assertThat(fuzzyQuery.getMin().longValue(), equalTo(12L));
-            assertThat(fuzzyQuery.getMax().longValue(), equalTo(12L));
-        }
+        QueryStringQueryBuilder query = queryStringQuery("12~0.2").defaultField(INT_FIELD_NAME);
+        QueryShardContext context = createShardContext();
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> query.toQuery(context));
+        assertEquals("Can only use fuzzy queries on keyword and text fields - not on [mapped_int] which is of type [integer]",
+                e.getMessage());
+        query.lenient(true);
+        query.toQuery(context); // no exception
+    }
+
+    public void testPrefixNumeric() throws Exception {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        QueryStringQueryBuilder query = queryStringQuery("12*").defaultField(INT_FIELD_NAME);
+        QueryShardContext context = createShardContext();
+        QueryShardException e = expectThrows(QueryShardException.class,
+                () -> query.toQuery(context));
+        assertEquals("Can only use prefix queries on keyword and text fields - not on [mapped_int] which is of type [integer]",
+                e.getMessage());
+        query.lenient(true);
+        query.toQuery(context); // no exception
+    }
+
+    public void testExactGeo() throws Exception {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        QueryStringQueryBuilder query = queryStringQuery("2,3").defaultField(GEO_POINT_FIELD_NAME);
+        QueryShardContext context = createShardContext();
+        QueryShardException e = expectThrows(QueryShardException.class,
+                () -> query.toQuery(context));
+        assertEquals("Geo fields do not support exact searching, use dedicated geo queries instead: [mapped_geo_point]",
+                e.getMessage());
+        query.lenient(true);
+        query.toQuery(context); // no exception
     }
 
     public void testTimezone() throws Exception {
@@ -518,4 +541,5 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
         assertEquals(json, "this AND that OR thus", parsed.queryString());
         assertEquals(json, "content", parsed.defaultField());
     }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java
index b777f419501..777d8a6800b 100644
--- a/core/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java
@@ -26,6 +26,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
+import static org.elasticsearch.index.query.QueryBuilders.prefixQuery;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -101,4 +102,14 @@ public class RegexpQueryBuilderTests extends AbstractQueryTestCase<RegexpQueryBu
         assertEquals(json, "s.*y", parsed.value());
         assertEquals(json, 20000, parsed.maxDeterminizedStates());
     }
+
+    public void testNumeric() throws Exception {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        RegexpQueryBuilder query = new RegexpQueryBuilder(INT_FIELD_NAME, "12");
+        QueryShardContext context = createShardContext();
+        QueryShardException e = expectThrows(QueryShardException.class,
+                () -> query.toQuery(context));
+        assertEquals("Can only use regexp queries on keyword and text fields - not on [mapped_int] which is of type [integer]",
+                e.getMessage());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java
index bc024686d65..a632f3803dd 100644
--- a/core/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/TermQueryBuilderTests.java
@@ -139,4 +139,14 @@ public class TermQueryBuilderTests extends AbstractTermQueryTestCase<TermQueryBu
 
         assertEquals(json, "Quick Foxes!", parsed.value());
     }
+
+    public void testGeo() throws Exception {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        TermQueryBuilder query = new TermQueryBuilder(GEO_POINT_FIELD_NAME, "2,3");
+        QueryShardContext context = createShardContext();
+        QueryShardException e = expectThrows(QueryShardException.class,
+                () -> query.toQuery(context));
+        assertEquals("Geo fields do not support exact searching, use dedicated geo queries instead: [mapped_geo_point]",
+                e.getMessage());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
index 0b4faebe18d..2dd509c159b 100644
--- a/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
@@ -45,6 +45,7 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.index.query.QueryBuilders.prefixQuery;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -74,7 +75,10 @@ public class TermsQueryBuilderTests extends AbstractQueryTestCase<TermsQueryBuil
         // terms query or lookup query
         if (randomBoolean()) {
             // make between 0 and 5 different values of the same type
-            String fieldName = getRandomFieldName();
+            String fieldName;
+            do {
+                fieldName = getRandomFieldName();
+            } while (fieldName.equals(GEO_POINT_FIELD_NAME) || fieldName.equals(GEO_SHAPE_FIELD_NAME));
             Object[] values = new Object[randomInt(5)];
             for (int i = 0; i < values.length; i++) {
                 values[i] = getRandomValueForFieldName(fieldName);
@@ -314,5 +318,15 @@ public class TermsQueryBuilderTests extends AbstractQueryTestCase<TermsQueryBuil
         assertEquals(termsQueryBuilder.rewrite(createShardContext()), new TermsQueryBuilder(STRING_FIELD_NAME,
             randomTerms.stream().filter(x -> x != null).collect(Collectors.toList()))); // terms lookup removes null values
     }
+
+    public void testGeo() throws Exception {
+        assumeTrue("test runs only when at least a type is registered", getCurrentTypes().length > 0);
+        TermsQueryBuilder query = new TermsQueryBuilder(GEO_POINT_FIELD_NAME, "2,3");
+        QueryShardContext context = createShardContext();
+        QueryShardException e = expectThrows(QueryShardException.class,
+                () -> query.toQuery(context));
+        assertEquals("Geo fields do not support exact searching, use dedicated geo queries instead: [mapped_geo_point]",
+                e.getMessage());
+    }
 }
 
diff --git a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
index 7db3022e0db..9dd32e091cd 100644
--- a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
@@ -1003,14 +1003,6 @@ public class SearchQueryIT extends ESIntegTestCase {
         assertNoFailures(searchResponse);
         assertHitCount(searchResponse, 1L);
         assertFirstHit(searchResponse, hasId("1"));
-
-        searchResponse = client().prepareSearch().setQuery(queryStringQuery("num:11~1")).get();
-        assertHitCount(searchResponse, 1L);
-        assertFirstHit(searchResponse, hasId("1"));
-
-        searchResponse = client().prepareSearch().setQuery(queryStringQuery("date:2012-02-02~1d")).get();
-        assertHitCount(searchResponse, 1L);
-        assertFirstHit(searchResponse, hasId("1"));
     }
 
     public void testQuotedQueryStringWithBoost() throws InterruptedException, ExecutionException {
diff --git a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java
index cb4e5a251a8..6dc10232b21 100644
--- a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java
@@ -192,13 +192,6 @@ public class SimpleSearchIT extends ESIntegTestCase {
 
         searchResponse = client().prepareSearch().setQuery(QueryBuilders.queryStringQuery("_id:XXX1")).execute().actionGet();
         assertHitCount(searchResponse, 1L);
-
-        // id is not index, but we can automatically support prefix as well
-        searchResponse = client().prepareSearch().setQuery(QueryBuilders.prefixQuery("_id", "XXX")).execute().actionGet();
-        assertHitCount(searchResponse, 1L);
-
-        searchResponse = client().prepareSearch().setQuery(QueryBuilders.queryStringQuery("_id:XXX*").lowercaseExpandedTerms(false)).execute().actionGet();
-        assertHitCount(searchResponse, 1L);
     }
 
     public void testSimpleDateRange() throws Exception {
@@ -421,6 +414,7 @@ public class SimpleSearchIT extends ESIntegTestCase {
                 client().prepareSearch("idx").addRescorer(new QueryRescorerBuilder(matchAllQuery()).windowSize(defaultMaxWindow + 1)).get(),
                 1);
     }
+
     public void testQueryNumericFieldWithRegex() throws Exception {
         assertAcked(prepareCreate("idx").addMapping("type", "num", "type=integer"));
         ensureGreen("idx");
@@ -429,7 +423,7 @@ public class SimpleSearchIT extends ESIntegTestCase {
             client().prepareSearch("idx").setQuery(QueryBuilders.regexpQuery("num", "34")).get();
             fail("SearchPhaseExecutionException should have been thrown");
         } catch (SearchPhaseExecutionException ex) {
-            assertThat(ex.getCause().getCause().getMessage(), containsString("Can only use regular expression on keyword and text fields"));
+            assertThat(ex.getCause().getCause().getMessage(), containsString("Can only use regexp queries on keyword and text fields"));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
index 2051233cc95..385003b7995 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java
@@ -20,6 +20,8 @@
 package org.elasticsearch.search.sort;
 
 
+import org.apache.lucene.queryparser.xml.builders.MatchAllDocsQueryBuilder;
+import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.SortField;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.ParseFieldMatcher;
@@ -34,6 +36,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
 import org.elasticsearch.index.query.GeoValidationMethod;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.DocValueFormat;
@@ -90,7 +93,7 @@ public class GeoDistanceSortBuilderTests extends AbstractSortTestCase<GeoDistanc
             result.sortMode(randomValueOtherThan(SortMode.SUM, () -> randomFrom(SortMode.values())));
         }
         if (randomBoolean()) {
-            result.setNestedFilter(randomNestedFilter());
+            result.setNestedFilter(new MatchAllQueryBuilder());
         }
         if (randomBoolean()) {
             result.setNestedPath(
diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc
index 87df2902f9c..4e487325829 100644
--- a/docs/reference/api-conventions.asciidoc
+++ b/docs/reference/api-conventions.asciidoc
@@ -428,27 +428,9 @@ precision is interpreted as the length of the geohash.
 === Fuzziness
 
 Some queries and APIs support parameters to allow inexact _fuzzy_ matching,
-using the `fuzziness` parameter. The `fuzziness` parameter is context
-sensitive which means that it depends on the type of the field being queried:
+using the `fuzziness` parameter.
 
-[float]
-==== Numeric, date and IPv4 fields
-
-When querying numeric, date and IPv4 fields, `fuzziness` is interpreted as a
-`+/-` margin. It behaves like a <<query-dsl-range-query>> where:
-
-    -fuzziness <= field value <= +fuzziness
-
-The `fuzziness` parameter should be set to a numeric value, eg `2` or `2.0`. A
-`date` field interprets a long as milliseconds, but also accepts a string
-containing a time value -- `"1h"` -- as explained in <<time-units>>. An `ip`
-field accepts a long or another IPv4 address (which will be converted into a
-long).
-
-[float]
-==== String fields
-
-When querying `string` fields, `fuzziness` is interpreted as a
+When querying `text` or `keyword` fields, `fuzziness` is interpreted as a
 http://en.wikipedia.org/wiki/Levenshtein_distance[Levenshtein Edit Distance]
 -- the number of one character changes that need to be made to one string to
 make it the same as another string.
diff --git a/docs/reference/migration/migrate_5_0/search.asciidoc b/docs/reference/migration/migrate_5_0/search.asciidoc
index 4a6ac438d70..378c86bccb7 100644
--- a/docs/reference/migration/migrate_5_0/search.asciidoc
+++ b/docs/reference/migration/migrate_5_0/search.asciidoc
@@ -91,6 +91,16 @@ The following deprecated queries have been removed:
 
 ==== Changes to queries
 
+* Unsupported queries such as term queries on `geo_point` fields will now fail
+  rather than returning no hits.
+
+* Removed support for fuzzy queries on numeric, date and ip fields, use range
+  queries instead.
+
+* Removed support for range and prefix queries on `_uid` and `_id` fields.
+
+* Querying an unindexed field will now fail rather than returning no hits.
+
 * Removed support for the deprecated `min_similarity` parameter in `fuzzy
   query`, in favour of `fuzziness`.
 
diff --git a/docs/reference/query-dsl/fuzzy-query.asciidoc b/docs/reference/query-dsl/fuzzy-query.asciidoc
index c647bab2cbb..48c9c45c0ff 100644
--- a/docs/reference/query-dsl/fuzzy-query.asciidoc
+++ b/docs/reference/query-dsl/fuzzy-query.asciidoc
@@ -1,10 +1,9 @@
 [[query-dsl-fuzzy-query]]
 === Fuzzy Query
 
-deprecated[5.0.0, Will be removed without a replacement for `string` fields. Note that the `fuzziness` parameter is still supported for match queries and in suggesters. Use range queries for `date` and `numeric` fields instead.]
+deprecated[5.0.0, Will be removed in 6.0. Use match queries with fuzziness instead]
 
-The fuzzy query uses similarity based on Levenshtein edit distance for
-`string` fields, and a `+/-` margin on numeric and date fields.
+The fuzzy query uses similarity based on Levenshtein edit distance.
 
 ==== String fields
 
@@ -63,41 +62,3 @@ WARNING: This query can be very heavy if `prefix_length` is set to `0` and if
 `max_expansions` is set to a high number. It could result in every term in the
 index being examined!
 
-[float]
-==== Numeric and date fields
-
-Performs a <<query-dsl-range-query>> ``around'' the value using the
-`fuzziness` value as a `+/-` range, where:
-
-    -fuzziness <= field value <= +fuzziness
-
-For example:
-
-[source,js]
---------------------------------------------------
-{
-    "fuzzy" : {
-        "price" : {
-            "value" : 12,
-            "fuzziness" : 2
-        }
-    }
-}
---------------------------------------------------
-
-Will result in a range query between 10 and 14. Date fields support
-<<time-units,time values>>, eg:
-
-[source,js]
---------------------------------------------------
-{
-    "fuzzy" : {
-        "created" : {
-            "value" : "2010-02-05T12:05:07",
-            "fuzziness" : "1d"
-        }
-    }
-}
---------------------------------------------------
-
-See <<fuzziness>> for more details about accepted values.
diff --git a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java
index bb3018195fa..09667733251 100644
--- a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java
+++ b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java
@@ -21,6 +21,7 @@ package org.elasticsearch.mapper.attachments;
 
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.search.Query;
 import org.apache.tika.language.LanguageIdentifier;
 import org.apache.tika.metadata.Metadata;
 import org.elasticsearch.Version;
@@ -42,6 +43,8 @@ import org.elasticsearch.index.mapper.core.DateFieldMapper;
 import org.elasticsearch.index.mapper.core.NumberFieldMapper;
 import org.elasticsearch.index.mapper.core.NumberFieldMapper.NumberType;
 import org.elasticsearch.index.mapper.core.TextFieldMapper;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.QueryShardException;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -118,6 +121,11 @@ public class AttachmentMapper extends FieldMapper {
         public String typeName() {
             return CONTENT_TYPE;
         }
+
+        @Override
+        public Query termQuery(Object value, QueryShardContext context) {
+            throw new QueryShardException(context, "Attachment fields are not searchable: [" + name() + "]");
+        }
     }
 
     public static class Builder extends FieldMapper.Builder<Builder, AttachmentMapper> {
diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java
index 46735594939..0c43efb8ecd 100644
--- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java
+++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java
@@ -27,6 +27,7 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.hash.MurmurHash3;
@@ -40,6 +41,8 @@ import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.core.TypeParsers;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.QueryShardException;
 
 public class Murmur3FieldMapper extends FieldMapper {
 
@@ -123,6 +126,11 @@ public class Murmur3FieldMapper extends FieldMapper {
             failIfNoDocValues();
             return new DocValuesIndexFieldData.Builder().numericType(NumericType.LONG);
         }
+
+        @Override
+        public Query termQuery(Object value, QueryShardContext context) {
+            throw new QueryShardException(context, "Murmur3 fields are not searchable: [" + name() + "]");
+        }
     }
 
     protected Murmur3FieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java
index f48f584ee10..d8c9c8d797b 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java
@@ -49,7 +49,7 @@ public class MockFieldMapper extends FieldMapper {
         return fullName.substring(ndx + 1);
     }
 
-    public static class FakeFieldType extends MappedFieldType {
+    public static class FakeFieldType extends TermBasedFieldType {
         public FakeFieldType() {
         }
 

From 3a5ef68ad0e90a61a3e8d94059b88c33aa7dac84 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Mon, 16 May 2016 20:31:27 +0200
Subject: [PATCH 0275/1311] painless: Fix issue with dup and cast

---
 .../main/java/org/elasticsearch/painless/node/EChain.java | 3 ++-
 .../test/java/org/elasticsearch/painless/ArrayTests.java  | 8 ++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index 3dd64975757..d2b79201878 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -234,15 +234,16 @@ public final class EChain extends AExpression {
             final ADefLink lastDef = (ADefLink) last;
             expression.analyze(settings, definition, variables);
             lastDef.storeValueType = expression.expected = expression.actual;
+            actual = read ? lastDef.storeValueType : definition.voidType;
         } else {
             // otherwise we adapt the type of the expression to the store type
             expression.expected = last.after;
             expression.analyze(settings, definition, variables);
+            actual = read ? last.after : definition.voidType;
         }
         expression = expression.cast(settings, definition, variables);
 
         statement = true;
-        actual = read ? last.after : definition.voidType;
     }
 
     private void analyzeRead() {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
index aa873e45d36..2036c4fd04c 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
@@ -62,6 +62,14 @@ public class ArrayTests extends ScriptTestCase {
         assertEquals(6, exec("def x = new int[5]; x[0] = 5; x[0]++; return x[0];"));
     }
     
+    public void testJacksCrazyExpression1() {
+        assertEquals(1, exec("int x; def[] y = new def[1]; x = y[0] = 1; return x;"));
+    }
+    
+    public void testJacksCrazyExpression2() {
+        assertEquals(1, exec("int x; def y = new def[1]; x = y[0] = 1; return x;"));
+    }
+    
     public void testForLoop() {
         assertEquals(999*1000/2, exec("def a = new int[1000]; for (int x = 0; x < a.length; x++) { a[x] = x; } "+
             "int total = 0; for (int x = 0; x < a.length; x++) { total += a[x]; } return total;"));

From b05ac87a5d412f2163f509e57cbbde5057fc6abc Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Mon, 16 May 2016 20:37:51 +0200
Subject: [PATCH 0276/1311] painless: Add Jack's test

---
 .../test/java/org/elasticsearch/painless/BasicAPITests.java  | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
index b1df5276feb..af067d15252 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
@@ -81,4 +81,9 @@ public class BasicAPITests extends ScriptTestCase {
         assertEquals(1, exec("List<String> x = new ArrayList<String>(); x.add('Hallo'); return x.length"));
         assertEquals(1, exec("List<Object> x = new ArrayList<Object>(); x.add('Hallo'); return x.length"));
     }
+    
+    public void testDefAssignments() {
+        assertEquals(2, exec("int x; def y = 2.0; x = (int)y;"));
+    }
+    
 }

From d6cbbdeb989ccdd3061685fcbe6b6fc8c2d6e40c Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Mon, 16 May 2016 21:04:03 +0200
Subject: [PATCH 0277/1311] painless: Some reformatting in EChain to make it
 clear if "this" or inner "expression" is affected

---
 .../java/org/elasticsearch/painless/node/EChain.java | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index d2b79201878..4b89fcbcb08 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -221,8 +221,8 @@ public final class EChain extends AExpression {
             lastDef.storeValueType = last.after;
         }
 
-        statement = true;
-        actual = read ? last.after : definition.voidType;
+        this.statement = true;
+        this.actual = read ? last.after : definition.voidType;
     }
 
     private void analyzeWrite(final CompilerSettings settings, final Definition definition, final Variables variables) {
@@ -234,16 +234,16 @@ public final class EChain extends AExpression {
             final ADefLink lastDef = (ADefLink) last;
             expression.analyze(settings, definition, variables);
             lastDef.storeValueType = expression.expected = expression.actual;
-            actual = read ? lastDef.storeValueType : definition.voidType;
+            this.actual = read ? lastDef.storeValueType : definition.voidType;
         } else {
             // otherwise we adapt the type of the expression to the store type
             expression.expected = last.after;
             expression.analyze(settings, definition, variables);
-            actual = read ? last.after : definition.voidType;
+            this.actual = read ? last.after : definition.voidType;
         }
+        
         expression = expression.cast(settings, definition, variables);
-
-        statement = true;
+        this.statement = true;
     }
 
     private void analyzeRead() {

From 43344eee828c919a27016aca9288a0ea29bb79f0 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Mon, 16 May 2016 12:42:25 -0700
Subject: [PATCH 0278/1311] Add more def tests including checking bytecode for
 specific optimizations.

---
 .../painless/node/package-info.java           |   1 +
 .../{DefTests.java => DefOperationTests.java} |   2 +-
 .../painless/DefOptimizationTests.java        | 226 ++++++++++++++++++
 3 files changed, 228 insertions(+), 1 deletion(-)
 rename modules/lang-painless/src/test/java/org/elasticsearch/painless/{DefTests.java => DefOperationTests.java} (99%)
 create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
index 0a53e909bf8..07fb8027a2b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
@@ -28,6 +28,7 @@
  * <p>
  * The following is a brief description of each node:
  * {@link org.elasticsearch.painless.node.AExpression} - The superclass for all E* (expression) nodes.
+ * {@link org.elasticsearch.painless.node.ADefLink} - The superclass for all LDef* (link) nodes.
  * {@link org.elasticsearch.painless.node.ALink} - The superclass for all L* (link) nodes.
  * {@link org.elasticsearch.painless.node.ANode} - The superclass for all other nodes.
  * {@link org.elasticsearch.painless.node.AStatement} - The superclass for all S* (statement) nodes.
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
similarity index 99%
rename from modules/lang-painless/src/test/java/org/elasticsearch/painless/DefTests.java
rename to modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
index 1d3906f248f..cd4bda1c071 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.painless;
 
-public class DefTests extends ScriptTestCase {
+public class DefOperationTests extends ScriptTestCase {
     public void testNot() {
         assertEquals(~1, exec("def x = (byte)1 return ~x"));
         assertEquals(~1, exec("def x = (short)1 return ~x"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
new file mode 100644
index 00000000000..c545e9e98a6
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+public class DefOptimizationTests extends ScriptTestCase {
+    public void testIntBraceArrayOptiLoad() {
+        final String script = "int x = 0; def y = new int[1]; y[0] = 5; x = y[0]; return x;";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC arrayLoad(Ljava/lang/Object;I)I"));
+        assertEquals(5, exec(script));
+    }
+
+    public void testIntBraceArrayOptiStore() {
+        final String script = "int x = 1; def y = new int[1]; y[0] = x; return y[0];";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC arrayStore(Ljava/lang/Object;II)"));
+        assertEquals(1, exec(script));
+    }
+
+    public void testIntBraceListOptiLoad() {
+        final String script = "int x = 0; def y = new ArrayList(); y.add(5); x = y[0]; return x;";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC arrayLoad(Ljava/lang/Object;I)I"));
+        assertEquals(5, exec(script));
+    }
+
+    public void testIntBraceListOptiStore() {
+        final String script = "int x = 1; def y = new ArrayList(); y.add(0); y[0] = x; return y[0];";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC arrayStore(Ljava/lang/Object;II)"));
+        assertEquals(1, exec(script));
+    }
+
+    public void testIntBraceMapOptiLoad() {
+        final String script = "int x = 0; def y = new HashMap(); y.put(0, 5); x = y[0];";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC arrayLoad(Ljava/lang/Object;I)I"));
+        assertEquals(5, exec(script));
+    }
+
+    public void testIntBraceMapOptiStore() {
+        final String script = "int x = 1; def y = new HashMap(); y.put(0, 1); y[0] = x;";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC arrayStore(Ljava/lang/Object;II)"));
+        assertEquals(1, exec(script));
+    }
+
+    public void testIntFieldListOptiLoad() {
+        final String script = "int x = 0; def y = new ArrayList(); y.add(5); x = y.0;";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC 0(Ljava/lang/Object;)I"));
+        assertEquals(5, exec(script));
+    }
+
+    public void testIntFieldListOptiStore() {
+        final String script = "int x = 1; def y = new ArrayList(); y.add(0); y.0 = x;";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC 0(Ljava/lang/Object;I)"));
+        assertEquals(1, exec(script));
+    }
+
+    public void testIntFieldMapOptiLoad() {
+        final String script = "int x = 0; def y = new HashMap(); y.put('0', 5); x = y.0; return x;";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC 0(Ljava/lang/Object;)I"));
+        assertEquals(5, exec(script));
+    }
+
+    public void testIntFieldMapOptiStore() {
+        final String script = "int x = 1; def y = new HashMap(); y.put('0', 1); y.0 = x; return y.0;";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC 0(Ljava/lang/Object;I)"));
+        assertEquals(1, exec(script));
+    }
+
+    public void testIntCall0Opti() {
+        final String script = "int x; def y = new HashMap(); y['int'] = 1; x = y.get('int'); return x;";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC get(Ljava/lang/Object;Ljava/lang/String;)I"));
+        assertEquals(1, exec(script));
+    }
+
+    public void testIntCall1Opti() {
+        final String script = "int x; def y = new HashMap(); y['int'] = 1; x = y.get('int');";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC get(Ljava/lang/Object;Ljava/lang/String;)I"));
+        assertEquals(1, exec(script));
+    }
+
+    public void testDoubleBraceArrayOptiLoad() {
+        final String script = "double x = 0; def y = new double[1]; y[0] = 5.0; x = y[0]; return x;";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC arrayLoad(Ljava/lang/Object;I)D"));
+        assertEquals(5.0, exec(script));
+    }
+
+    public void testDoubleBraceArrayOptiStore() {
+        final String script = "double x = 1; def y = new double[1]; y[0] = x; return y[0];";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC arrayStore(Ljava/lang/Object;ID)"));
+        assertEquals(1.0, exec(script));
+    }
+
+    public void testDoubleBraceListOptiLoad() {
+        final String script = "double x = 0.0; def y = new ArrayList(); y.add(5.0); x = y[0]; return x;";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC arrayLoad(Ljava/lang/Object;I)D"));
+        assertEquals(5.0, exec(script));
+    }
+
+    public void testDoubleBraceListOptiStore() {
+        final String script = "double x = 1.0; def y = new ArrayList(); y.add(0.0); y[0] = x; return y[0];";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC arrayStore(Ljava/lang/Object;ID)"));
+        assertEquals(1.0, exec(script));
+    }
+
+    public void testDoubleBraceMapOptiLoad() {
+        final String script = "double x = 0.0; def y = new HashMap(); y.put(0, 5.0); x = y[0];";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC arrayLoad(Ljava/lang/Object;I)D"));
+        assertEquals(5.0, exec(script));
+    }
+
+    public void testDoubleBraceMapOptiStore() {
+        final String script = "double x = 1.0; def y = new HashMap(); y.put(0, 2.0); y[0] = x;";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC arrayStore(Ljava/lang/Object;ID)"));
+        assertEquals(1.0, exec(script));
+    }
+
+    public void testDoubleFieldListOptiLoad() {
+        final String script = "double x = 0; def y = new ArrayList(); y.add(5.0); x = y.0;";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC 0(Ljava/lang/Object;)D"));
+        assertEquals(5.0, exec(script));
+    }
+
+    public void testDoubleFieldListOptiStore() {
+        final String script = "double x = 1.0; def y = new ArrayList(); y.add(0); y.0 = x;";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC 0(Ljava/lang/Object;D)"));
+        assertEquals(1.0, exec(script));
+    }
+
+    public void testDoubleFieldMapOptiLoad() {
+        final String script = "double x = 0; def y = new HashMap(); y.put('0', 5.0); x = y.0; return x;";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC 0(Ljava/lang/Object;)D"));
+        assertEquals(5.0, exec(script));
+    }
+
+    public void testDoubleFieldMapOptiStore() {
+        final String script = "double x = 1.0; def y = new HashMap(); y.put('0', 1.0); y.0 = x; return y.0;";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC 0(Ljava/lang/Object;D)"));
+        assertEquals(1.0, exec(script));
+    }
+
+    public void testDoubleCall0Opti() {
+        final String script = "double x; def y = new HashMap(); y['double'] = 1.0; x = y.get('double'); return x;";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC get(Ljava/lang/Object;Ljava/lang/String;)D"));
+        assertEquals(1.0, exec(script));
+    }
+
+    public void testDoubleCall1Opti() {
+        final String script = "double x; def y = new HashMap(); y['double'] = 1.0; x = y.get('double');";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC get(Ljava/lang/Object;Ljava/lang/String;)D"));
+        assertEquals(1.0, exec(script));
+    }
+
+    public void testIllegalCast() {
+        final String script = "int x;\ndef y = new HashMap();\ny['double'] = 1.0;\nx = y.get('double');\n";
+        final String asm = Debugger.toString(script);
+
+        assertTrue(asm.contains("INVOKEDYNAMIC get(Ljava/lang/Object;Ljava/lang/String;)I"));
+        final Exception exception = expectThrows(ClassCastException.class, () -> {
+            exec(script);
+        });
+
+        assertTrue(exception.getMessage().contains("Cannot cast java.lang.Double to java.lang.Integer"));
+    }
+}

From 95a1b5b2a368b30369e522c972f5e2b53462af9c Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Mon, 16 May 2016 14:18:05 -0700
Subject: [PATCH 0279/1311] Fixed a casting bug in Painless.

---
 .../java/org/elasticsearch/painless/node/EChain.java   | 10 ++++++++--
 .../org/elasticsearch/painless/node/LDefArray.java     |  2 +-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index 4b89fcbcb08..484440cd981 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -233,6 +233,7 @@ public final class EChain extends AExpression {
         if (last instanceof ADefLink) {
             final ADefLink lastDef = (ADefLink) last;
             expression.analyze(settings, definition, variables);
+            // TODO: does it make more sense to just re-use last.after instead of using storeValueType?
             lastDef.storeValueType = expression.expected = expression.actual;
             this.actual = read ? lastDef.storeValueType : definition.voidType;
         } else {
@@ -241,7 +242,7 @@ public final class EChain extends AExpression {
             expression.analyze(settings, definition, variables);
             this.actual = read ? last.after : definition.voidType;
         }
-        
+
         expression = expression.cast(settings, definition, variables);
         this.statement = true;
     }
@@ -316,7 +317,12 @@ public final class EChain extends AExpression {
                     expression.write(settings, definition, adapter);
 
                     if (link.load) {
-                        WriterUtility.writeDup(adapter, link.after.sort.size, link.size);
+                        // storeValueType may be different from after, so use storeValueType if last is an ADefLink
+                        if (last instanceof ADefLink) {
+                            WriterUtility.writeDup(adapter, ((ADefLink)last).storeValueType.sort.size, link.size);
+                        } else {
+                            WriterUtility.writeDup(adapter, link.after.sort.size, link.size);
+                        }
                     }
 
                     link.store(settings, definition, adapter);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index df1c5ffac0a..10271c35df2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -41,7 +41,6 @@ final class LDefArray extends ADefLink {
         this.index = index;
     }
 
-
     @Override
     ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
         index.analyze(settings, definition, variables);
@@ -69,6 +68,7 @@ final class LDefArray extends ADefLink {
         if (storeValueType == null) {
             throw new IllegalStateException(error("Illegal tree structure."));
         }
+
         final String desc = Type.getMethodDescriptor(definition.voidType.type, definition.defType.type,
             index.actual.type, storeValueType.type);
         adapter.invokeDynamic("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_STORE);

From 0037a579a4b075293ec0143ea0c17aaa75bd1953 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 16 May 2016 17:34:32 -0400
Subject: [PATCH 0280/1311] Add note to contributing docs about force push

This commit adds a note to the contributing docs regarding force pushing
during the review process.

Relates #18380
---
 CONTRIBUTING.md | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index a79b03f5376..4868bbc6961 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -71,6 +71,16 @@ Once your changes and tests are ready to submit for review:
 
 Then sit back and wait. There will probably be discussion about the pull request and, if any changes are needed, we would love to work with you to get your pull request merged into Elasticsearch.
 
+Please adhere to the general guideline that you should never force push
+to a publicly shared branch. Once you have opened your pull request, you
+should consider your branch publicly shared. Instead of force pushing
+you can just add incremental commits; this is generally easier on your
+reviewers. If you need to pick up changes from master, you can merge
+master into your branch. A reviewer might ask you to rebase a
+long-running pull request in which case force pushing is okay for that
+request. Note that squashing at the end of the review process should
+also not be done, we can do that if necessary.
+
 Contributing to the Elasticsearch codebase
 ------------------------------------------
 

From 3764789d9641b37d78791b83b9293611261298ff Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Mon, 16 May 2016 18:15:33 -0400
Subject: [PATCH 0281/1311] Removed unused AllocationService member in
 TransportClusterAllocationExplainAction

Closes #18381
---
 .../TransportClusterAllocationExplainAction.java   | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
index 08e899be4df..da97f492f7a 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.action.admin.cluster.allocation;
 
-import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.apache.lucene.index.CorruptIndexException;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
@@ -30,24 +29,19 @@ import org.elasticsearch.action.admin.indices.shards.TransportIndicesShardStores
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
 import org.elasticsearch.cluster.ClusterInfoService;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.metadata.MetaData.Custom;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingNodes.RoutingNodesIterator;
-import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
-import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
-import org.elasticsearch.cluster.routing.allocation.RoutingExplanations;
 import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
@@ -60,7 +54,6 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -72,7 +65,6 @@ import java.util.Set;
 public class TransportClusterAllocationExplainAction
         extends TransportMasterNodeAction<ClusterAllocationExplainRequest, ClusterAllocationExplainResponse> {
 
-    private final AllocationService allocationService;
     private final ClusterInfoService clusterInfoService;
     private final AllocationDeciders allocationDeciders;
     private final ShardsAllocator shardAllocator;
@@ -82,12 +74,10 @@ public class TransportClusterAllocationExplainAction
     public TransportClusterAllocationExplainAction(Settings settings, TransportService transportService, ClusterService clusterService,
                                                    ThreadPool threadPool, ActionFilters actionFilters,
                                                    IndexNameExpressionResolver indexNameExpressionResolver,
-                                                   AllocationService allocationService, ClusterInfoService clusterInfoService,
-                                                   AllocationDeciders allocationDeciders, ShardsAllocator shardAllocator,
-                                                   TransportIndicesShardStoresAction shardStoresAction) {
+                                                   ClusterInfoService clusterInfoService, AllocationDeciders allocationDeciders,
+                                                   ShardsAllocator shardAllocator, TransportIndicesShardStoresAction shardStoresAction) {
         super(settings, ClusterAllocationExplainAction.NAME, transportService, clusterService, threadPool, actionFilters,
                 indexNameExpressionResolver, ClusterAllocationExplainRequest::new);
-        this.allocationService = allocationService;
         this.clusterInfoService = clusterInfoService;
         this.allocationDeciders = allocationDeciders;
         this.shardAllocator = shardAllocator;

From 3fd3d367ef836f8b24f52539d4938e5060b9d691 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Mon, 16 May 2016 15:48:29 -0700
Subject: [PATCH 0282/1311] Moved ADefLink to be a marker interface instead.

---
 .../elasticsearch/painless/node/EChain.java   | 25 ++---
 .../node/{ADefLink.java => IDefLink.java}     | 20 +---
 .../painless/node/LDefArray.java              |  8 +-
 .../elasticsearch/painless/node/LDefCall.java |  2 +-
 .../painless/node/LDefField.java              |  7 +-
 .../painless/node/package-info.java           |  2 +-
 .../painless/DefOptimizationTests.java        | 95 +++++--------------
 .../painless/ScriptTestCase.java              |  9 ++
 8 files changed, 49 insertions(+), 119 deletions(-)
 rename modules/lang-painless/src/main/java/org/elasticsearch/painless/node/{ADefLink.java => IDefLink.java} (56%)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index 484440cd981..7cdbbb09e6c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -215,12 +215,6 @@ public final class EChain extends AExpression {
         there = AnalyzerCaster.getLegalCast(definition, location, last.after, promote, false);
         back = AnalyzerCaster.getLegalCast(definition, location, promote, last.after, true);
 
-        if (last instanceof ADefLink) {
-            final ADefLink lastDef = (ADefLink) last;
-            // Unfortunately, we don't know the real type because we load from DEF and store to DEF!
-            lastDef.storeValueType = last.after;
-        }
-
         this.statement = true;
         this.actual = read ? last.after : definition.voidType;
     }
@@ -230,28 +224,26 @@ public final class EChain extends AExpression {
 
         // If the store node is a DEF node, we remove the cast to DEF from the expression
         // and promote the real type to it:
-        if (last instanceof ADefLink) {
-            final ADefLink lastDef = (ADefLink) last;
+        if (last instanceof IDefLink) {
             expression.analyze(settings, definition, variables);
-            // TODO: does it make more sense to just re-use last.after instead of using storeValueType?
-            lastDef.storeValueType = expression.expected = expression.actual;
-            this.actual = read ? lastDef.storeValueType : definition.voidType;
+            last.after = expression.expected = expression.actual;
         } else {
             // otherwise we adapt the type of the expression to the store type
             expression.expected = last.after;
             expression.analyze(settings, definition, variables);
-            this.actual = read ? last.after : definition.voidType;
         }
 
         expression = expression.cast(settings, definition, variables);
+
         this.statement = true;
+        this.actual = read ? last.after : definition.voidType;
     }
 
     private void analyzeRead() {
         final ALink last = links.get(links.size() - 1);
 
         // If the load node is a DEF node, we adapt its after type to use _this_ expected output type:
-        if (last instanceof ADefLink && this.expected != null) {
+        if (last instanceof IDefLink && this.expected != null) {
             last.after = this.expected;
         }
 
@@ -317,12 +309,7 @@ public final class EChain extends AExpression {
                     expression.write(settings, definition, adapter);
 
                     if (link.load) {
-                        // storeValueType may be different from after, so use storeValueType if last is an ADefLink
-                        if (last instanceof ADefLink) {
-                            WriterUtility.writeDup(adapter, ((ADefLink)last).storeValueType.sort.size, link.size);
-                        } else {
-                            WriterUtility.writeDup(adapter, link.after.sort.size, link.size);
-                        }
+                        WriterUtility.writeDup(adapter, link.after.sort.size, link.size);
                     }
 
                     link.store(settings, definition, adapter);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ADefLink.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/IDefLink.java
similarity index 56%
rename from modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ADefLink.java
rename to modules/lang-painless/src/main/java/org/elasticsearch/painless/node/IDefLink.java
index cf4351a63a7..12c39f2e886 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ADefLink.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/IDefLink.java
@@ -19,23 +19,9 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.Definition.Type;
-
 /**
- * The superclass for all LDef* (link) nodes that store or return a DEF. (Internal only.)
- * For this node it is allowed to change {@link ALink#after} from outside, by default
- * {@code after} is {@code DEF}.
+ * A marker interface applied to LDef* nodes allowing changes to {@link ALink#after} from outside,
+ * by default {@code after} is {@code DEF}.
  */
-abstract class ADefLink extends ALink {
-    
-    /**
-     * The type of the original type that was pushed on stack, set by {@link EChain} during analyze.
-     * This value is only used for writing the 'store' bytecode, otherwise ignored.
-     */
-    Type storeValueType = null;
-
-    ADefLink(final int line, final String location, final int size) {
-        super(line, location, size);
-    }
-
+interface IDefLink {
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index 10271c35df2..f8034837837 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -31,7 +31,7 @@ import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
 /**
  * Represents an array load/store or shortcut on a def type.  (Internal only.)
  */
-final class LDefArray extends ADefLink {
+final class LDefArray extends ALink implements IDefLink {
 
     AExpression index;
 
@@ -65,12 +65,8 @@ final class LDefArray extends ADefLink {
 
     @Override
     void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
-        if (storeValueType == null) {
-            throw new IllegalStateException(error("Illegal tree structure."));
-        }
-
         final String desc = Type.getMethodDescriptor(definition.voidType.type, definition.defType.type,
-            index.actual.type, storeValueType.type);
+            index.actual.type, after.type);
         adapter.invokeDynamic("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_STORE);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index 459f7221b68..0ac3cb3dedd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -32,7 +32,7 @@ import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
 /**
  * Represents a method call made on a def type. (Internal only.)
  */
-final class LDefCall extends ADefLink {
+final class LDefCall extends ALink implements IDefLink {
 
     final String name;
     final List<AExpression> arguments;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index caf78dccdd1..0c1d8270065 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -31,7 +31,7 @@ import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
 /**
  * Represents a field load/store or shortcut on a def type.  (Internal only.)
  */
-final class LDefField extends ADefLink {
+final class LDefField extends ALink implements IDefLink {
 
     final String value;
 
@@ -62,10 +62,7 @@ final class LDefField extends ADefLink {
 
     @Override
     void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
-        if (storeValueType == null) {
-            throw new IllegalStateException(error("Illegal tree structure."));
-        }
-        final String desc = Type.getMethodDescriptor(definition.voidType.type, definition.defType.type, storeValueType.type);
+        final String desc = Type.getMethodDescriptor(definition.voidType.type, definition.defType.type, after.type);
         adapter.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.STORE);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
index 07fb8027a2b..ab6944619ca 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
@@ -28,7 +28,6 @@
  * <p>
  * The following is a brief description of each node:
  * {@link org.elasticsearch.painless.node.AExpression} - The superclass for all E* (expression) nodes.
- * {@link org.elasticsearch.painless.node.ADefLink} - The superclass for all LDef* (link) nodes.
  * {@link org.elasticsearch.painless.node.ALink} - The superclass for all L* (link) nodes.
  * {@link org.elasticsearch.painless.node.ANode} - The superclass for all other nodes.
  * {@link org.elasticsearch.painless.node.AStatement} - The superclass for all S* (statement) nodes.
@@ -45,6 +44,7 @@
  * {@link org.elasticsearch.painless.node.ENull} - Represents a null constant.
  * {@link org.elasticsearch.painless.node.ENumeric} - Respresents a non-decimal numeric constant.
  * {@link org.elasticsearch.painless.node.EUnary} - Represents a unary math expression.
+ * {@link org.elasticsearch.painless.node.IDefLink} - A marker interface for all LDef* (link) nodes.
  * {@link org.elasticsearch.painless.node.LArrayLength} - Represents an array length field load.
  * {@link org.elasticsearch.painless.node.LBrace} - Represents an array load/store or defers to possible shortcuts.
  * {@link org.elasticsearch.painless.node.LCall} - Represents a method call or deferes to a def call.
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
index c545e9e98a6..72d08b677b9 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
@@ -22,205 +22,160 @@ package org.elasticsearch.painless;
 public class DefOptimizationTests extends ScriptTestCase {
     public void testIntBraceArrayOptiLoad() {
         final String script = "int x = 0; def y = new int[1]; y[0] = 5; x = y[0]; return x;";
-        final String asm = Debugger.toString(script);
-
-        assertTrue(asm.contains("INVOKEDYNAMIC arrayLoad(Ljava/lang/Object;I)I"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC arrayLoad(Ljava/lang/Object;I)I");
         assertEquals(5, exec(script));
     }
 
     public void testIntBraceArrayOptiStore() {
         final String script = "int x = 1; def y = new int[1]; y[0] = x; return y[0];";
-        final String asm = Debugger.toString(script);
-
-        assertTrue(asm.contains("INVOKEDYNAMIC arrayStore(Ljava/lang/Object;II)"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC arrayStore(Ljava/lang/Object;II)");
         assertEquals(1, exec(script));
     }
 
     public void testIntBraceListOptiLoad() {
         final String script = "int x = 0; def y = new ArrayList(); y.add(5); x = y[0]; return x;";
-        final String asm = Debugger.toString(script);
-
-        assertTrue(asm.contains("INVOKEDYNAMIC arrayLoad(Ljava/lang/Object;I)I"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC arrayLoad(Ljava/lang/Object;I)I");
         assertEquals(5, exec(script));
     }
 
     public void testIntBraceListOptiStore() {
         final String script = "int x = 1; def y = new ArrayList(); y.add(0); y[0] = x; return y[0];";
-        final String asm = Debugger.toString(script);
-
-        assertTrue(asm.contains("INVOKEDYNAMIC arrayStore(Ljava/lang/Object;II)"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC arrayStore(Ljava/lang/Object;II)");
         assertEquals(1, exec(script));
     }
 
     public void testIntBraceMapOptiLoad() {
         final String script = "int x = 0; def y = new HashMap(); y.put(0, 5); x = y[0];";
-        final String asm = Debugger.toString(script);
-
-        assertTrue(asm.contains("INVOKEDYNAMIC arrayLoad(Ljava/lang/Object;I)I"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC arrayLoad(Ljava/lang/Object;I)I");
         assertEquals(5, exec(script));
     }
 
     public void testIntBraceMapOptiStore() {
         final String script = "int x = 1; def y = new HashMap(); y.put(0, 1); y[0] = x;";
-        final String asm = Debugger.toString(script);
-
-        assertTrue(asm.contains("INVOKEDYNAMIC arrayStore(Ljava/lang/Object;II)"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC arrayStore(Ljava/lang/Object;II)");
         assertEquals(1, exec(script));
     }
 
     public void testIntFieldListOptiLoad() {
         final String script = "int x = 0; def y = new ArrayList(); y.add(5); x = y.0;";
-        final String asm = Debugger.toString(script);
-
-        assertTrue(asm.contains("INVOKEDYNAMIC 0(Ljava/lang/Object;)I"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC 0(Ljava/lang/Object;)I");
         assertEquals(5, exec(script));
     }
 
     public void testIntFieldListOptiStore() {
         final String script = "int x = 1; def y = new ArrayList(); y.add(0); y.0 = x;";
-        final String asm = Debugger.toString(script);
-
-        assertTrue(asm.contains("INVOKEDYNAMIC 0(Ljava/lang/Object;I)"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC 0(Ljava/lang/Object;I)");
         assertEquals(1, exec(script));
     }
 
     public void testIntFieldMapOptiLoad() {
         final String script = "int x = 0; def y = new HashMap(); y.put('0', 5); x = y.0; return x;";
-        final String asm = Debugger.toString(script);
-
-        assertTrue(asm.contains("INVOKEDYNAMIC 0(Ljava/lang/Object;)I"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC 0(Ljava/lang/Object;)I");
         assertEquals(5, exec(script));
     }
 
     public void testIntFieldMapOptiStore() {
         final String script = "int x = 1; def y = new HashMap(); y.put('0', 1); y.0 = x; return y.0;";
-        final String asm = Debugger.toString(script);
-
-        assertTrue(asm.contains("INVOKEDYNAMIC 0(Ljava/lang/Object;I)"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC 0(Ljava/lang/Object;I)");
         assertEquals(1, exec(script));
     }
 
     public void testIntCall0Opti() {
         final String script = "int x; def y = new HashMap(); y['int'] = 1; x = y.get('int'); return x;";
-        final String asm = Debugger.toString(script);
-
-        assertTrue(asm.contains("INVOKEDYNAMIC get(Ljava/lang/Object;Ljava/lang/String;)I"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC get(Ljava/lang/Object;Ljava/lang/String;)I");
         assertEquals(1, exec(script));
     }
 
     public void testIntCall1Opti() {
         final String script = "int x; def y = new HashMap(); y['int'] = 1; x = y.get('int');";
-        final String asm = Debugger.toString(script);
-
-        assertTrue(asm.contains("INVOKEDYNAMIC get(Ljava/lang/Object;Ljava/lang/String;)I"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC get(Ljava/lang/Object;Ljava/lang/String;)I");
         assertEquals(1, exec(script));
     }
 
     public void testDoubleBraceArrayOptiLoad() {
         final String script = "double x = 0; def y = new double[1]; y[0] = 5.0; x = y[0]; return x;";
-        final String asm = Debugger.toString(script);
-
-        assertTrue(asm.contains("INVOKEDYNAMIC arrayLoad(Ljava/lang/Object;I)D"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC arrayLoad(Ljava/lang/Object;I)D");
         assertEquals(5.0, exec(script));
     }
 
     public void testDoubleBraceArrayOptiStore() {
         final String script = "double x = 1; def y = new double[1]; y[0] = x; return y[0];";
-        final String asm = Debugger.toString(script);
 
-        assertTrue(asm.contains("INVOKEDYNAMIC arrayStore(Ljava/lang/Object;ID)"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC arrayStore(Ljava/lang/Object;ID)");
         assertEquals(1.0, exec(script));
     }
 
     public void testDoubleBraceListOptiLoad() {
         final String script = "double x = 0.0; def y = new ArrayList(); y.add(5.0); x = y[0]; return x;";
-        final String asm = Debugger.toString(script);
 
-        assertTrue(asm.contains("INVOKEDYNAMIC arrayLoad(Ljava/lang/Object;I)D"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC arrayLoad(Ljava/lang/Object;I)D");
         assertEquals(5.0, exec(script));
     }
 
     public void testDoubleBraceListOptiStore() {
         final String script = "double x = 1.0; def y = new ArrayList(); y.add(0.0); y[0] = x; return y[0];";
-        final String asm = Debugger.toString(script);
 
-        assertTrue(asm.contains("INVOKEDYNAMIC arrayStore(Ljava/lang/Object;ID)"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC arrayStore(Ljava/lang/Object;ID)");
         assertEquals(1.0, exec(script));
     }
 
     public void testDoubleBraceMapOptiLoad() {
         final String script = "double x = 0.0; def y = new HashMap(); y.put(0, 5.0); x = y[0];";
-        final String asm = Debugger.toString(script);
 
-        assertTrue(asm.contains("INVOKEDYNAMIC arrayLoad(Ljava/lang/Object;I)D"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC arrayLoad(Ljava/lang/Object;I)D");
         assertEquals(5.0, exec(script));
     }
 
     public void testDoubleBraceMapOptiStore() {
         final String script = "double x = 1.0; def y = new HashMap(); y.put(0, 2.0); y[0] = x;";
-        final String asm = Debugger.toString(script);
 
-        assertTrue(asm.contains("INVOKEDYNAMIC arrayStore(Ljava/lang/Object;ID)"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC arrayStore(Ljava/lang/Object;ID)");
         assertEquals(1.0, exec(script));
     }
 
     public void testDoubleFieldListOptiLoad() {
         final String script = "double x = 0; def y = new ArrayList(); y.add(5.0); x = y.0;";
-        final String asm = Debugger.toString(script);
-
-        assertTrue(asm.contains("INVOKEDYNAMIC 0(Ljava/lang/Object;)D"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC 0(Ljava/lang/Object;)D");
         assertEquals(5.0, exec(script));
     }
 
     public void testDoubleFieldListOptiStore() {
         final String script = "double x = 1.0; def y = new ArrayList(); y.add(0); y.0 = x;";
-        final String asm = Debugger.toString(script);
-
-        assertTrue(asm.contains("INVOKEDYNAMIC 0(Ljava/lang/Object;D)"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC 0(Ljava/lang/Object;D)");
         assertEquals(1.0, exec(script));
     }
 
     public void testDoubleFieldMapOptiLoad() {
         final String script = "double x = 0; def y = new HashMap(); y.put('0', 5.0); x = y.0; return x;";
-        final String asm = Debugger.toString(script);
-
-        assertTrue(asm.contains("INVOKEDYNAMIC 0(Ljava/lang/Object;)D"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC 0(Ljava/lang/Object;)D");
         assertEquals(5.0, exec(script));
     }
 
     public void testDoubleFieldMapOptiStore() {
         final String script = "double x = 1.0; def y = new HashMap(); y.put('0', 1.0); y.0 = x; return y.0;";
-        final String asm = Debugger.toString(script);
-
-        assertTrue(asm.contains("INVOKEDYNAMIC 0(Ljava/lang/Object;D)"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC 0(Ljava/lang/Object;D)");
         assertEquals(1.0, exec(script));
     }
 
     public void testDoubleCall0Opti() {
         final String script = "double x; def y = new HashMap(); y['double'] = 1.0; x = y.get('double'); return x;";
-        final String asm = Debugger.toString(script);
-
-        assertTrue(asm.contains("INVOKEDYNAMIC get(Ljava/lang/Object;Ljava/lang/String;)D"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC get(Ljava/lang/Object;Ljava/lang/String;)D");
         assertEquals(1.0, exec(script));
     }
 
     public void testDoubleCall1Opti() {
         final String script = "double x; def y = new HashMap(); y['double'] = 1.0; x = y.get('double');";
-        final String asm = Debugger.toString(script);
-
-        assertTrue(asm.contains("INVOKEDYNAMIC get(Ljava/lang/Object;Ljava/lang/String;)D"));
+        assertBytecodeExists(script, "INVOKEDYNAMIC get(Ljava/lang/Object;Ljava/lang/String;)D");
         assertEquals(1.0, exec(script));
     }
 
     public void testIllegalCast() {
         final String script = "int x;\ndef y = new HashMap();\ny['double'] = 1.0;\nx = y.get('double');\n";
-        final String asm = Debugger.toString(script);
+        assertBytecodeExists(script, "INVOKEDYNAMIC get(Ljava/lang/Object;Ljava/lang/String;)I");
 
-        assertTrue(asm.contains("INVOKEDYNAMIC get(Ljava/lang/Object;Ljava/lang/String;)I"));
         final Exception exception = expectThrows(ClassCastException.class, () -> {
             exec(script);
         });
-
         assertTrue(exception.getMessage().contains("Cannot cast java.lang.Double to java.lang.Integer"));
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
index 7c86b393c30..2ccd2f1460a 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
@@ -57,4 +57,13 @@ public abstract class ScriptTestCase extends ESTestCase {
         CompiledScript compiled = new CompiledScript(ScriptService.ScriptType.INLINE, getTestName(), "painless", object);
         return scriptEngine.executable(compiled, vars).run();
     }
+
+    /**
+     * Uses the {@link Debugger} to get the bytecode output for a script and compare
+     * it against an expected bytecode passed in as a String.
+     */
+    public void assertBytecodeExists(String script, String bytecode) {
+        final String asm = Debugger.toString(script);
+        assertTrue("bytecode not found", asm.contains(bytecode));
+    }
 }

From 75f27df376cf66624bcfdf1d155edaad83a6f489 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Tue, 17 May 2016 02:25:05 +0200
Subject: [PATCH 0283/1311] painless: Refactor WriterUtils to extends ASM
 GeneratorAdapter as "MethodWriter" for easy usage (and more coming later)

---
 .../{WriterUtility.java => MethodWriter.java} | 305 ++++++++++--------
 .../org/elasticsearch/painless/Writer.java    |   4 +-
 .../painless/node/AExpression.java            |   4 +-
 .../elasticsearch/painless/node/ALink.java    |   8 +-
 .../elasticsearch/painless/node/ANode.java    |   4 +-
 .../painless/node/AStatement.java             |   4 +-
 .../elasticsearch/painless/node/EBinary.java  |  19 +-
 .../elasticsearch/painless/node/EBool.java    |   4 +-
 .../elasticsearch/painless/node/EBoolean.java |   4 +-
 .../elasticsearch/painless/node/ECast.java    |   9 +-
 .../elasticsearch/painless/node/EChain.java   |  37 ++-
 .../elasticsearch/painless/node/EComp.java    |  36 +--
 .../painless/node/EConditional.java           |   4 +-
 .../painless/node/EConstant.java              |   7 +-
 .../elasticsearch/painless/node/EDecimal.java |   4 +-
 .../painless/node/EExplicit.java              |   4 +-
 .../elasticsearch/painless/node/ENull.java    |   4 +-
 .../elasticsearch/painless/node/ENumeric.java |   4 +-
 .../elasticsearch/painless/node/EUnary.java   |  11 +-
 .../painless/node/LArrayLength.java           |   8 +-
 .../elasticsearch/painless/node/LBrace.java   |   8 +-
 .../elasticsearch/painless/node/LCall.java    |   8 +-
 .../elasticsearch/painless/node/LCast.java    |  11 +-
 .../painless/node/LDefArray.java              |   8 +-
 .../elasticsearch/painless/node/LDefCall.java |   8 +-
 .../painless/node/LDefField.java              |   8 +-
 .../elasticsearch/painless/node/LField.java   |   8 +-
 .../painless/node/LListShortcut.java          |  11 +-
 .../painless/node/LMapShortcut.java           |  11 +-
 .../painless/node/LNewArray.java              |   8 +-
 .../elasticsearch/painless/node/LNewObj.java  |   8 +-
 .../painless/node/LShortcut.java              |  11 +-
 .../elasticsearch/painless/node/LString.java  |   8 +-
 .../painless/node/LVariable.java              |   8 +-
 .../elasticsearch/painless/node/SBlock.java   |   4 +-
 .../elasticsearch/painless/node/SBreak.java   |   4 +-
 .../painless/node/SContinue.java              |   4 +-
 .../painless/node/SDeclBlock.java             |   4 +-
 .../painless/node/SDeclaration.java           |   4 +-
 .../org/elasticsearch/painless/node/SDo.java  |   7 +-
 .../painless/node/SExpression.java            |   7 +-
 .../org/elasticsearch/painless/node/SFor.java |  11 +-
 .../elasticsearch/painless/node/SIfElse.java  |   4 +-
 .../elasticsearch/painless/node/SReturn.java  |   4 +-
 .../elasticsearch/painless/node/SSource.java  |   4 +-
 .../elasticsearch/painless/node/SThrow.java   |   4 +-
 .../elasticsearch/painless/node/STrap.java    |   4 +-
 .../org/elasticsearch/painless/node/STry.java |   4 +-
 .../elasticsearch/painless/node/SWhile.java   |   9 +-
 49 files changed, 346 insertions(+), 340 deletions(-)
 rename modules/lang-painless/src/main/java/org/elasticsearch/painless/{WriterUtility.java => MethodWriter.java} (51%)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterUtility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
similarity index 51%
rename from modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterUtility.java
rename to modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index c3c26f309ea..2daa5d5589c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterUtility.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -23,9 +23,11 @@ import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Transform;
 import org.elasticsearch.painless.Definition.Type;
+import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.commons.GeneratorAdapter;
+import org.objectweb.asm.commons.Method;
 
 import static org.elasticsearch.painless.WriterConstants.ADDEXACT_INT;
 import static org.elasticsearch.painless.WriterConstants.ADDEXACT_LONG;
@@ -88,43 +90,61 @@ import static org.elasticsearch.painless.WriterConstants.TOSHORTWOOVERFLOW_DOUBL
 import static org.elasticsearch.painless.WriterConstants.TOSHORTWOOVERFLOW_FLOAT;
 
 /**
+ * Extension of {@link GeneratorAdapter} with some utility methods.
+ * <p>
  * Set of methods used during the writing phase of compilation
  * shared by the nodes of the Painless tree.
  */
-public final class WriterUtility {
+public final class MethodWriter extends GeneratorAdapter {
 
-    public static void writeLoopCounter(final GeneratorAdapter adapter, final int slot, final int count) {
+    MethodWriter(int access, Method method, org.objectweb.asm.Type[] exceptions, ClassVisitor cv) {
+        super(Opcodes.ASM5, cv.visitMethod(access, method.getName(), method.getDescriptor(), null, getInternalNames(exceptions)),
+                access, method.getName(), method.getDescriptor());
+    }
+
+    private static String[] getInternalNames(final org.objectweb.asm.Type[] types) {
+        if (types == null) {
+            return null;
+        }
+        String[] names = new String[types.length];
+        for (int i = 0; i < names.length; ++i) {
+            names[i] = types[i].getInternalName();
+        }
+        return names;
+    }
+
+    public void writeLoopCounter(final int slot, final int count) {
         if (slot > -1) {
             final Label end = new Label();
 
-            adapter.iinc(slot, -count);
-            adapter.visitVarInsn(Opcodes.ILOAD, slot);
-            adapter.push(0);
-            adapter.ifICmp(GeneratorAdapter.GT, end);
-            adapter.throwException(PAINLESS_ERROR_TYPE,
+            iinc(slot, -count);
+            visitVarInsn(Opcodes.ILOAD, slot);
+            push(0);
+            ifICmp(GeneratorAdapter.GT, end);
+            throwException(PAINLESS_ERROR_TYPE,
                 "The maximum number of statements that can be executed in a loop has been reached.");
-            adapter.mark(end);
+            mark(end);
         }
     }
 
-    public static void writeCast(final GeneratorAdapter adapter, final Cast cast) {
+    public void writeCast(final Cast cast) {
         if (cast instanceof Transform) {
             final Transform transform = (Transform)cast;
 
             if (transform.upcast != null) {
-                adapter.checkCast(transform.upcast.type);
+                checkCast(transform.upcast.type);
             }
 
             if (java.lang.reflect.Modifier.isStatic(transform.method.reflect.getModifiers())) {
-                adapter.invokeStatic(transform.method.owner.type, transform.method.method);
+                invokeStatic(transform.method.owner.type, transform.method.method);
             } else if (java.lang.reflect.Modifier.isInterface(transform.method.owner.clazz.getModifiers())) {
-                adapter.invokeInterface(transform.method.owner.type, transform.method.method);
+                invokeInterface(transform.method.owner.type, transform.method.method);
             } else {
-                adapter.invokeVirtual(transform.method.owner.type, transform.method.method);
+                invokeVirtual(transform.method.owner.type, transform.method.method);
             }
 
             if (transform.downcast != null) {
-                adapter.checkCast(transform.downcast.type);
+                checkCast(transform.downcast.type);
             }
         } else if (cast != null) {
             final Type from = cast.from;
@@ -135,50 +155,50 @@ public final class WriterUtility {
             }
 
             if (from.sort.numeric && from.sort.primitive && to.sort.numeric && to.sort.primitive) {
-                adapter.cast(from.type, to.type);
+                cast(from.type, to.type);
             } else {
                 if (!to.clazz.isAssignableFrom(from.clazz)) {
-                    adapter.checkCast(to.type);
+                    checkCast(to.type);
                 }
             }
         }
     }
 
-    public static void writeBranch(final GeneratorAdapter adapter, final Label tru, final Label fals) {
+    public void writeBranch(final Label tru, final Label fals) {
         if (tru != null) {
-            adapter.visitJumpInsn(Opcodes.IFNE, tru);
+            visitJumpInsn(Opcodes.IFNE, tru);
         } else if (fals != null) {
-            adapter.visitJumpInsn(Opcodes.IFEQ, fals);
+            visitJumpInsn(Opcodes.IFEQ, fals);
         }
     }
 
-    public static void writeNewStrings(final GeneratorAdapter adapter) {
-        adapter.newInstance(STRINGBUILDER_TYPE);
-        adapter.dup();
-        adapter.invokeConstructor(STRINGBUILDER_TYPE, STRINGBUILDER_CONSTRUCTOR);
+    public void writeNewStrings() {
+        newInstance(STRINGBUILDER_TYPE);
+        dup();
+        invokeConstructor(STRINGBUILDER_TYPE, STRINGBUILDER_CONSTRUCTOR);
     }
 
-    public static void writeAppendStrings(final GeneratorAdapter adapter, final Sort sort) {
+    public void writeAppendStrings(final Sort sort) {
         switch (sort) {
-            case BOOL:   adapter.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_BOOLEAN); break;
-            case CHAR:   adapter.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_CHAR);    break;
+            case BOOL:   invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_BOOLEAN); break;
+            case CHAR:   invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_CHAR);    break;
             case BYTE:
             case SHORT:
-            case INT:    adapter.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_INT);     break;
-            case LONG:   adapter.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_LONG);    break;
-            case FLOAT:  adapter.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_FLOAT);   break;
-            case DOUBLE: adapter.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_DOUBLE);  break;
-            case STRING: adapter.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_STRING);  break;
-            default:     adapter.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_OBJECT);
+            case INT:    invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_INT);     break;
+            case LONG:   invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_LONG);    break;
+            case FLOAT:  invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_FLOAT);   break;
+            case DOUBLE: invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_DOUBLE);  break;
+            case STRING: invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_STRING);  break;
+            default:     invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_OBJECT);
         }
     }
 
-    public static void writeToStrings(final GeneratorAdapter adapter) {
-        adapter.invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_TOSTRING);
+    public void writeToStrings() {
+        invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_TOSTRING);
     }
 
-    public static void writeBinaryInstruction(final CompilerSettings settings, final Definition definition,
-                                              final GeneratorAdapter adapter, final String location,
+    public void writeBinaryInstruction(final CompilerSettings settings, final Definition definition,
+                                              final String location,
                                               final Type type, final Operation operation) {
         final Sort sort = type.sort;
         boolean exact = !settings.getNumericOverflow() &&
@@ -193,29 +213,29 @@ public final class WriterUtility {
             switch (sort) {
                 case INT:
                     switch (operation) {
-                        case MUL: adapter.invokeStatic(definition.mathType.type,    MULEXACT_INT);     break;
-                        case DIV: adapter.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_INT); break;
-                        case ADD: adapter.invokeStatic(definition.mathType.type,    ADDEXACT_INT);     break;
-                        case SUB: adapter.invokeStatic(definition.mathType.type,    SUBEXACT_INT);     break;
+                        case MUL: invokeStatic(definition.mathType.type,    MULEXACT_INT);     break;
+                        case DIV: invokeStatic(definition.utilityType.type, DIVWOOVERLOW_INT); break;
+                        case ADD: invokeStatic(definition.mathType.type,    ADDEXACT_INT);     break;
+                        case SUB: invokeStatic(definition.mathType.type,    SUBEXACT_INT);     break;
                     }
 
                     break;
                 case LONG:
                     switch (operation) {
-                        case MUL: adapter.invokeStatic(definition.mathType.type,    MULEXACT_LONG);     break;
-                        case DIV: adapter.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_LONG); break;
-                        case ADD: adapter.invokeStatic(definition.mathType.type,    ADDEXACT_LONG);     break;
-                        case SUB: adapter.invokeStatic(definition.mathType.type,    SUBEXACT_LONG);     break;
+                        case MUL: invokeStatic(definition.mathType.type,    MULEXACT_LONG);     break;
+                        case DIV: invokeStatic(definition.utilityType.type, DIVWOOVERLOW_LONG); break;
+                        case ADD: invokeStatic(definition.mathType.type,    ADDEXACT_LONG);     break;
+                        case SUB: invokeStatic(definition.mathType.type,    SUBEXACT_LONG);     break;
                     }
 
                     break;
                 case FLOAT:
                     switch (operation) {
-                        case MUL: adapter.invokeStatic(definition.utilityType.type, MULWOOVERLOW_FLOAT); break;
-                        case DIV: adapter.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_FLOAT); break;
-                        case REM: adapter.invokeStatic(definition.utilityType.type, REMWOOVERLOW_FLOAT); break;
-                        case ADD: adapter.invokeStatic(definition.utilityType.type, ADDWOOVERLOW_FLOAT); break;
-                        case SUB: adapter.invokeStatic(definition.utilityType.type, SUBWOOVERLOW_FLOAT); break;
+                        case MUL: invokeStatic(definition.utilityType.type, MULWOOVERLOW_FLOAT); break;
+                        case DIV: invokeStatic(definition.utilityType.type, DIVWOOVERLOW_FLOAT); break;
+                        case REM: invokeStatic(definition.utilityType.type, REMWOOVERLOW_FLOAT); break;
+                        case ADD: invokeStatic(definition.utilityType.type, ADDWOOVERLOW_FLOAT); break;
+                        case SUB: invokeStatic(definition.utilityType.type, SUBWOOVERLOW_FLOAT); break;
                         default:
                             throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
                     }
@@ -223,11 +243,11 @@ public final class WriterUtility {
                     break;
                 case DOUBLE:
                     switch (operation) {
-                        case MUL: adapter.invokeStatic(definition.utilityType.type, MULWOOVERLOW_DOUBLE); break;
-                        case DIV: adapter.invokeStatic(definition.utilityType.type, DIVWOOVERLOW_DOUBLE); break;
-                        case REM: adapter.invokeStatic(definition.utilityType.type, REMWOOVERLOW_DOUBLE); break;
-                        case ADD: adapter.invokeStatic(definition.utilityType.type, ADDWOOVERLOW_DOUBLE); break;
-                        case SUB: adapter.invokeStatic(definition.utilityType.type, SUBWOOVERLOW_DOUBLE); break;
+                        case MUL: invokeStatic(definition.utilityType.type, MULWOOVERLOW_DOUBLE); break;
+                        case DIV: invokeStatic(definition.utilityType.type, DIVWOOVERLOW_DOUBLE); break;
+                        case REM: invokeStatic(definition.utilityType.type, REMWOOVERLOW_DOUBLE); break;
+                        case ADD: invokeStatic(definition.utilityType.type, ADDWOOVERLOW_DOUBLE); break;
+                        case SUB: invokeStatic(definition.utilityType.type, SUBWOOVERLOW_DOUBLE); break;
                         default:
                             throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
                     }
@@ -246,33 +266,33 @@ public final class WriterUtility {
 
             if (sort == Sort.DEF) {
                 switch (operation) {
-                    case MUL:   adapter.invokeStatic(definition.defobjType.type, DEF_MUL_CALL); break;
-                    case DIV:   adapter.invokeStatic(definition.defobjType.type, DEF_DIV_CALL); break;
-                    case REM:   adapter.invokeStatic(definition.defobjType.type, DEF_REM_CALL); break;
-                    case ADD:   adapter.invokeStatic(definition.defobjType.type, DEF_ADD_CALL); break;
-                    case SUB:   adapter.invokeStatic(definition.defobjType.type, DEF_SUB_CALL); break;
-                    case LSH:   adapter.invokeStatic(definition.defobjType.type, DEF_LSH_CALL); break;
-                    case USH:   adapter.invokeStatic(definition.defobjType.type, DEF_RSH_CALL); break;
-                    case RSH:   adapter.invokeStatic(definition.defobjType.type, DEF_USH_CALL); break;
-                    case BWAND: adapter.invokeStatic(definition.defobjType.type, DEF_AND_CALL); break;
-                    case XOR:   adapter.invokeStatic(definition.defobjType.type, DEF_XOR_CALL); break;
-                    case BWOR:  adapter.invokeStatic(definition.defobjType.type, DEF_OR_CALL);  break;
+                    case MUL:   invokeStatic(definition.defobjType.type, DEF_MUL_CALL); break;
+                    case DIV:   invokeStatic(definition.defobjType.type, DEF_DIV_CALL); break;
+                    case REM:   invokeStatic(definition.defobjType.type, DEF_REM_CALL); break;
+                    case ADD:   invokeStatic(definition.defobjType.type, DEF_ADD_CALL); break;
+                    case SUB:   invokeStatic(definition.defobjType.type, DEF_SUB_CALL); break;
+                    case LSH:   invokeStatic(definition.defobjType.type, DEF_LSH_CALL); break;
+                    case USH:   invokeStatic(definition.defobjType.type, DEF_RSH_CALL); break;
+                    case RSH:   invokeStatic(definition.defobjType.type, DEF_USH_CALL); break;
+                    case BWAND: invokeStatic(definition.defobjType.type, DEF_AND_CALL); break;
+                    case XOR:   invokeStatic(definition.defobjType.type, DEF_XOR_CALL); break;
+                    case BWOR:  invokeStatic(definition.defobjType.type, DEF_OR_CALL);  break;
                     default:
                         throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
                 }
             } else {
                 switch (operation) {
-                    case MUL:   adapter.math(GeneratorAdapter.MUL,  type.type); break;
-                    case DIV:   adapter.math(GeneratorAdapter.DIV,  type.type); break;
-                    case REM:   adapter.math(GeneratorAdapter.REM,  type.type); break;
-                    case ADD:   adapter.math(GeneratorAdapter.ADD,  type.type); break;
-                    case SUB:   adapter.math(GeneratorAdapter.SUB,  type.type); break;
-                    case LSH:   adapter.math(GeneratorAdapter.SHL,  type.type); break;
-                    case USH:   adapter.math(GeneratorAdapter.USHR, type.type); break;
-                    case RSH:   adapter.math(GeneratorAdapter.SHR,  type.type); break;
-                    case BWAND: adapter.math(GeneratorAdapter.AND,  type.type); break;
-                    case XOR:   adapter.math(GeneratorAdapter.XOR,  type.type); break;
-                    case BWOR:  adapter.math(GeneratorAdapter.OR,   type.type); break;
+                    case MUL:   math(GeneratorAdapter.MUL,  type.type); break;
+                    case DIV:   math(GeneratorAdapter.DIV,  type.type); break;
+                    case REM:   math(GeneratorAdapter.REM,  type.type); break;
+                    case ADD:   math(GeneratorAdapter.ADD,  type.type); break;
+                    case SUB:   math(GeneratorAdapter.SUB,  type.type); break;
+                    case LSH:   math(GeneratorAdapter.SHL,  type.type); break;
+                    case USH:   math(GeneratorAdapter.USHR, type.type); break;
+                    case RSH:   math(GeneratorAdapter.SHR,  type.type); break;
+                    case BWAND: math(GeneratorAdapter.AND,  type.type); break;
+                    case XOR:   math(GeneratorAdapter.XOR,  type.type); break;
+                    case BWOR:  math(GeneratorAdapter.OR,   type.type); break;
                     default:
                         throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
                 }
@@ -287,111 +307,111 @@ public final class WriterUtility {
      *
      * @return This will be true if an instruction is written, false otherwise.
      */
-    public static boolean writeExactInstruction(
-        final Definition definition, final GeneratorAdapter adapter, final Sort fsort, final Sort tsort) {
+    public boolean writeExactInstruction(
+        final Definition definition, final Sort fsort, final Sort tsort) {
         if (fsort == Sort.DOUBLE) {
             if (tsort == Sort.FLOAT) {
-                adapter.invokeStatic(definition.utilityType.type, TOFLOATWOOVERFLOW_DOUBLE);
+                invokeStatic(definition.utilityType.type, TOFLOATWOOVERFLOW_DOUBLE);
             } else if (tsort == Sort.FLOAT_OBJ) {
-                adapter.invokeStatic(definition.utilityType.type, TOFLOATWOOVERFLOW_DOUBLE);
-                adapter.checkCast(definition.floatobjType.type);
+                invokeStatic(definition.utilityType.type, TOFLOATWOOVERFLOW_DOUBLE);
+                checkCast(definition.floatobjType.type);
             } else if (tsort == Sort.LONG) {
-                adapter.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_DOUBLE);
+                invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_DOUBLE);
             } else if (tsort == Sort.LONG_OBJ) {
-                adapter.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_DOUBLE);
-                adapter.checkCast(definition.longobjType.type);
+                invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_DOUBLE);
+                checkCast(definition.longobjType.type);
             } else if (tsort == Sort.INT) {
-                adapter.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_DOUBLE);
+                invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_DOUBLE);
             } else if (tsort == Sort.INT_OBJ) {
-                adapter.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_DOUBLE);
-                adapter.checkCast(definition.intobjType.type);
+                invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_DOUBLE);
+                checkCast(definition.intobjType.type);
             } else if (tsort == Sort.CHAR) {
-                adapter.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_DOUBLE);
+                invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_DOUBLE);
             } else if (tsort == Sort.CHAR_OBJ) {
-                adapter.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_DOUBLE);
-                adapter.checkCast(definition.charobjType.type);
+                invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_DOUBLE);
+                checkCast(definition.charobjType.type);
             } else if (tsort == Sort.SHORT) {
-                adapter.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_DOUBLE);
+                invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_DOUBLE);
             } else if (tsort == Sort.SHORT_OBJ) {
-                adapter.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_DOUBLE);
-                adapter.checkCast(definition.shortobjType.type);
+                invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_DOUBLE);
+                checkCast(definition.shortobjType.type);
             } else if (tsort == Sort.BYTE) {
-                adapter.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_DOUBLE);
+                invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_DOUBLE);
             } else if (tsort == Sort.BYTE_OBJ) {
-                adapter.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_DOUBLE);
-                adapter.checkCast(definition.byteobjType.type);
+                invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_DOUBLE);
+                checkCast(definition.byteobjType.type);
             } else {
                 return false;
             }
         } else if (fsort == Sort.FLOAT) {
             if (tsort == Sort.LONG) {
-                adapter.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_FLOAT);
+                invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_FLOAT);
             } else if (tsort == Sort.LONG_OBJ) {
-                adapter.invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_FLOAT);
-                adapter.checkCast(definition.longobjType.type);
+                invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_FLOAT);
+                checkCast(definition.longobjType.type);
             } else if (tsort == Sort.INT) {
-                adapter.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_FLOAT);
+                invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_FLOAT);
             } else if (tsort == Sort.INT_OBJ) {
-                adapter.invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_FLOAT);
-                adapter.checkCast(definition.intobjType.type);
+                invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_FLOAT);
+                checkCast(definition.intobjType.type);
             } else if (tsort == Sort.CHAR) {
-                adapter.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_FLOAT);
+                invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_FLOAT);
             } else if (tsort == Sort.CHAR_OBJ) {
-                adapter.invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_FLOAT);
-                adapter.checkCast(definition.charobjType.type);
+                invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_FLOAT);
+                checkCast(definition.charobjType.type);
             } else if (tsort == Sort.SHORT) {
-                adapter.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_FLOAT);
+                invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_FLOAT);
             } else if (tsort == Sort.SHORT_OBJ) {
-                adapter.invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_FLOAT);
-                adapter.checkCast(definition.shortobjType.type);
+                invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_FLOAT);
+                checkCast(definition.shortobjType.type);
             } else if (tsort == Sort.BYTE) {
-                adapter.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_FLOAT);
+                invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_FLOAT);
             } else if (tsort == Sort.BYTE_OBJ) {
-                adapter.invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_FLOAT);
-                adapter.checkCast(definition.byteobjType.type);
+                invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_FLOAT);
+                checkCast(definition.byteobjType.type);
             } else {
                 return false;
             }
         } else if (fsort == Sort.LONG) {
             if (tsort == Sort.INT) {
-                adapter.invokeStatic(definition.mathType.type, TOINTEXACT_LONG);
+                invokeStatic(definition.mathType.type, TOINTEXACT_LONG);
             } else if (tsort == Sort.INT_OBJ) {
-                adapter.invokeStatic(definition.mathType.type, TOINTEXACT_LONG);
-                adapter.checkCast(definition.intobjType.type);
+                invokeStatic(definition.mathType.type, TOINTEXACT_LONG);
+                checkCast(definition.intobjType.type);
             } else if (tsort == Sort.CHAR) {
-                adapter.invokeStatic(definition.utilityType.type, TOCHAREXACT_LONG);
+                invokeStatic(definition.utilityType.type, TOCHAREXACT_LONG);
             } else if (tsort == Sort.CHAR_OBJ) {
-                adapter.invokeStatic(definition.utilityType.type, TOCHAREXACT_LONG);
-                adapter.checkCast(definition.charobjType.type);
+                invokeStatic(definition.utilityType.type, TOCHAREXACT_LONG);
+                checkCast(definition.charobjType.type);
             } else if (tsort == Sort.SHORT) {
-                adapter.invokeStatic(definition.utilityType.type, TOSHORTEXACT_LONG);
+                invokeStatic(definition.utilityType.type, TOSHORTEXACT_LONG);
             } else if (tsort == Sort.SHORT_OBJ) {
-                adapter.invokeStatic(definition.utilityType.type, TOSHORTEXACT_LONG);
-                adapter.checkCast(definition.shortobjType.type);
+                invokeStatic(definition.utilityType.type, TOSHORTEXACT_LONG);
+                checkCast(definition.shortobjType.type);
             } else if (tsort == Sort.BYTE) {
-                adapter.invokeStatic(definition.utilityType.type, TOBYTEEXACT_LONG);
+                invokeStatic(definition.utilityType.type, TOBYTEEXACT_LONG);
             } else if (tsort == Sort.BYTE_OBJ) {
-                adapter.invokeStatic(definition.utilityType.type, TOBYTEEXACT_LONG);
-                adapter.checkCast(definition.byteobjType.type);
+                invokeStatic(definition.utilityType.type, TOBYTEEXACT_LONG);
+                checkCast(definition.byteobjType.type);
             } else {
                 return false;
             }
         } else if (fsort == Sort.INT) {
             if (tsort == Sort.CHAR) {
-                adapter.invokeStatic(definition.utilityType.type, TOCHAREXACT_INT);
+                invokeStatic(definition.utilityType.type, TOCHAREXACT_INT);
             } else if (tsort == Sort.CHAR_OBJ) {
-                adapter.invokeStatic(definition.utilityType.type, TOCHAREXACT_INT);
-                adapter.checkCast(definition.charobjType.type);
+                invokeStatic(definition.utilityType.type, TOCHAREXACT_INT);
+                checkCast(definition.charobjType.type);
             } else if (tsort == Sort.SHORT) {
-                adapter.invokeStatic(definition.utilityType.type, TOSHORTEXACT_INT);
+                invokeStatic(definition.utilityType.type, TOSHORTEXACT_INT);
             } else if (tsort == Sort.SHORT_OBJ) {
-                adapter.invokeStatic(definition.utilityType.type, TOSHORTEXACT_INT);
-                adapter.checkCast(definition.shortobjType.type);
+                invokeStatic(definition.utilityType.type, TOSHORTEXACT_INT);
+                checkCast(definition.shortobjType.type);
             } else if (tsort == Sort.BYTE) {
-                adapter.invokeStatic(definition.utilityType.type, TOBYTEEXACT_INT);
+                invokeStatic(definition.utilityType.type, TOBYTEEXACT_INT);
             } else if (tsort == Sort.BYTE_OBJ) {
-                adapter.invokeStatic(definition.utilityType.type, TOBYTEEXACT_INT);
-                adapter.checkCast(definition.byteobjType.type);
+                invokeStatic(definition.utilityType.type, TOBYTEEXACT_INT);
+                checkCast(definition.byteobjType.type);
             } else {
                 return false;
             }
@@ -402,33 +422,32 @@ public final class WriterUtility {
         return true;
     }
 
-    public static void writeDup(final GeneratorAdapter adapter, final int size, final int xsize) {
+    public void writeDup(final int size, final int xsize) {
         if (size == 1) {
             if (xsize == 2) {
-                adapter.dupX2();
+                dupX2();
             } else if (xsize == 1) {
-                adapter.dupX1();
+                dupX1();
             } else {
-                adapter.dup();
+                dup();
             }
         } else if (size == 2) {
             if (xsize == 2) {
-                adapter.dup2X2();
+                dup2X2();
             } else if (xsize == 1) {
-                adapter.dup2X1();
+                dup2X1();
             } else {
-                adapter.dup2();
+                dup2();
             }
         }
     }
 
-    public static void writePop(final GeneratorAdapter adapter, final int size) {
+    public void writePop(final int size) {
         if (size == 1) {
-            adapter.pop();
+            pop();
         } else if (size == 2) {
-            adapter.pop2();
+            pop2();
         }
     }
 
-    private WriterUtility() {}
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
index dab9c79de6d..449361867b9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
@@ -53,7 +53,7 @@ final class Writer {
     private final SSource root;
 
     private final ClassWriter writer;
-    private final GeneratorAdapter adapter;
+    private final MethodWriter adapter;
 
     private Writer(final CompilerSettings settings, final Definition definition,
                      String name, final String source, final Variables variables, final SSource root) {
@@ -69,7 +69,7 @@ final class Writer {
         writeBegin();
         writeConstructor();
 
-        adapter = new GeneratorAdapter(Opcodes.ACC_PUBLIC, EXECUTE, null, null, writer);
+        adapter = new MethodWriter(Opcodes.ACC_PUBLIC, EXECUTE, null, writer);
 
         writeExecute();
         writeEnd();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
index 6f2b861b70c..528da4384c8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
@@ -26,7 +26,7 @@ import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * The superclass for all E* (expression) nodes.
@@ -106,7 +106,7 @@ public abstract class AExpression extends ANode {
     /**
      * Writes ASM based on the data collected during the analysis phase.
      */
-    abstract void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter);
+    abstract void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter);
 
     /**
      * Inserts {@link ECast} nodes into the tree for implicit casts.  Also replaces
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
index 79cbdf4a194..ffbfff112b3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
@@ -23,7 +23,7 @@ import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Variables;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * The superclass for all L* (link) nodes.
@@ -92,17 +92,17 @@ public abstract class ALink extends ANode {
     /**
      * Write values before a load/store occurs such as an array index.
      */
-    abstract void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter);
+    abstract void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter);
 
     /**
      * Write a load for the specific link type.
      */
-    abstract void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter);
+    abstract void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter);
 
     /**
      * Write a store for the specific link type.
      */
-    abstract void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter);
+    abstract void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter);
 
     /**
      * Used to copy link data from one to another during analysis in the case of replacement.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
index 11c081981fe..a61b002dfca 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.painless.node;
 
+import org.elasticsearch.painless.MethodWriter;
 import org.objectweb.asm.Label;
-import org.objectweb.asm.commons.GeneratorAdapter;
 
 /**
  * The superclass for all other nodes.
@@ -51,7 +51,7 @@ public abstract class ANode {
      * <p>
      * Currently we emit line number data for for leaf S-nodes
      */
-    void writeDebugInfo(GeneratorAdapter adapter) {
+    void writeDebugInfo(MethodWriter adapter) {
         Label label = new Label();
         adapter.visitLabel(label);
         adapter.visitLineNumber(line, label);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
index c4366e1af36..569e8cfb03b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
@@ -23,7 +23,7 @@ import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * The superclass for all S* (statement) nodes.
@@ -121,5 +121,5 @@ public abstract class AStatement extends ANode {
     /**
      * Writes ASM based on the data collected during the analysis phase.
      */
-    abstract void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter);
+    abstract void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter);
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 0df1f0f29de..67645211b8f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -19,15 +19,14 @@
 
 package org.elasticsearch.painless.node;
 
+import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
-import org.elasticsearch.painless.AnalyzerCaster;
+import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Variables;
-import org.elasticsearch.painless.WriterUtility;
-import org.objectweb.asm.commons.GeneratorAdapter;
 
 /**
  * Represents a binary math expression.
@@ -495,34 +494,34 @@ public final class EBinary extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         if (actual.sort == Sort.STRING && operation == Operation.ADD) {
             if (!cat) {
-                WriterUtility.writeNewStrings(adapter);
+                adapter.writeNewStrings();
             }
 
             left.write(settings, definition, adapter);
 
             if (!(left instanceof EBinary) || ((EBinary)left).operation != Operation.ADD || left.actual.sort != Sort.STRING) {
-                WriterUtility.writeAppendStrings(adapter, left.actual.sort);
+                adapter.writeAppendStrings(left.actual.sort);
             }
 
             right.write(settings, definition, adapter);
 
             if (!(right instanceof EBinary) || ((EBinary)right).operation != Operation.ADD || right.actual.sort != Sort.STRING) {
-                WriterUtility.writeAppendStrings(adapter, right.actual.sort);
+                adapter.writeAppendStrings(right.actual.sort);
             }
 
             if (!cat) {
-                WriterUtility.writeToStrings(adapter);
+                adapter.writeToStrings();
             }
         } else {
             left.write(settings, definition, adapter);
             right.write(settings, definition, adapter);
 
-            WriterUtility.writeBinaryInstruction(settings, definition, adapter, location, actual, operation);
+            adapter.writeBinaryInstruction(settings, definition, location, actual, operation);
         }
 
-        WriterUtility.writeBranch(adapter, tru, fals);
+        adapter.writeBranch(tru, fals);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
index 7f120894603..7f9f7dee000 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
@@ -24,7 +24,7 @@ import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents a boolean expression.
@@ -67,7 +67,7 @@ public final class EBool extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         if (tru != null || fals != null) {
             if (operation == Operation.AND) {
                 final Label localfals = fals == null ? new Label() : fals;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
index a5fb9c7f9e2..9b9f0917546 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents a boolean constant.
@@ -41,7 +41,7 @@ public final class EBoolean extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         throw new IllegalArgumentException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
index 10622e758fc..d68c95c910a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
@@ -23,8 +23,7 @@ import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Variables;
-import org.elasticsearch.painless.WriterUtility;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents an implicit cast in most cases, though it will replace
@@ -52,9 +51,9 @@ final class ECast extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         child.write(settings, definition, adapter);
-        WriterUtility.writeCast(adapter, cast);
-        WriterUtility.writeBranch(adapter, tru, fals);
+        adapter.writeCast(cast);
+        adapter.writeBranch(tru, fals);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index 7cdbbb09e6c..b98962a8f22 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -27,8 +27,7 @@ import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Variables;
-import org.elasticsearch.painless.WriterUtility;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 import java.util.List;
 
@@ -253,9 +252,9 @@ public final class EChain extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         if (cat) {
-            WriterUtility.writeNewStrings(adapter);
+            adapter.writeNewStrings();
         }
 
         final ALink last = links.get(links.size() - 1);
@@ -265,43 +264,43 @@ public final class EChain extends AExpression {
 
             if (link == last && link.store) {
                 if (cat) {
-                    WriterUtility.writeDup(adapter, link.size, 1);
+                    adapter.writeDup(link.size, 1);
                     link.load(settings, definition, adapter);
-                    WriterUtility.writeAppendStrings(adapter, link.after.sort);
+                    adapter.writeAppendStrings(link.after.sort);
 
                     expression.write(settings, definition, adapter);
 
                     if (!(expression instanceof EBinary) ||
                         ((EBinary)expression).operation != Operation.ADD || expression.actual.sort != Sort.STRING) {
-                        WriterUtility.writeAppendStrings(adapter, expression.actual.sort);
+                        adapter.writeAppendStrings(expression.actual.sort);
                     }
 
-                    WriterUtility.writeToStrings(adapter);
-                    WriterUtility.writeCast(adapter, back);
+                    adapter.writeToStrings();
+                    adapter.writeCast(back);
 
                     if (link.load) {
-                        WriterUtility.writeDup(adapter, link.after.sort.size, link.size);
+                        adapter.writeDup(link.after.sort.size, link.size);
                     }
 
                     link.store(settings, definition, adapter);
                 } else if (operation != null) {
-                    WriterUtility.writeDup(adapter, link.size, 0);
+                    adapter.writeDup(link.size, 0);
                     link.load(settings, definition, adapter);
 
                     if (link.load && post) {
-                        WriterUtility.writeDup(adapter, link.after.sort.size, link.size);
+                        adapter.writeDup(link.after.sort.size, link.size);
                     }
 
-                    WriterUtility.writeCast(adapter, there);
+                    adapter.writeCast(there);
                     expression.write(settings, definition, adapter);
-                    WriterUtility.writeBinaryInstruction(settings, definition, adapter, location, promote, operation);
+                    adapter.writeBinaryInstruction(settings, definition, location, promote, operation);
 
-                    if (!exact || !WriterUtility.writeExactInstruction(definition, adapter, promote.sort, link.after.sort)) {
-                        WriterUtility.writeCast(adapter, back);
+                    if (!exact || !adapter.writeExactInstruction(definition, promote.sort, link.after.sort)) {
+                        adapter.writeCast(back);
                     }
 
                     if (link.load && !post) {
-                        WriterUtility.writeDup(adapter, link.after.sort.size, link.size);
+                        adapter.writeDup(link.after.sort.size, link.size);
                     }
 
                     link.store(settings, definition, adapter);
@@ -309,7 +308,7 @@ public final class EChain extends AExpression {
                     expression.write(settings, definition, adapter);
 
                     if (link.load) {
-                        WriterUtility.writeDup(adapter, link.after.sort.size, link.size);
+                        adapter.writeDup(link.after.sort.size, link.size);
                     }
 
                     link.store(settings, definition, adapter);
@@ -319,6 +318,6 @@ public final class EChain extends AExpression {
             }
         }
 
-        WriterUtility.writeBranch(adapter, tru, fals);
+        adapter.writeBranch(tru, fals);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index c7f85eb57c7..d9337ae562b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -27,7 +27,7 @@ import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 import static org.elasticsearch.painless.WriterConstants.CHECKEQUALS;
 import static org.elasticsearch.painless.WriterConstants.DEF_EQ_CALL;
@@ -397,7 +397,7 @@ public final class EComp extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         final boolean branch = tru != null || fals != null;
         final org.objectweb.asm.Type rtype = right.actual.type;
         final Sort rsort = right.actual.sort;
@@ -429,8 +429,8 @@ public final class EComp extends AExpression {
             case CHAR:
                 throw new IllegalStateException(error("Illegal tree structure."));
             case BOOL:
-                if      (eq) adapter.ifZCmp(GeneratorAdapter.EQ, jump);
-                else if (ne) adapter.ifZCmp(GeneratorAdapter.NE, jump);
+                if      (eq) adapter.ifZCmp(MethodWriter.EQ, jump);
+                else if (ne) adapter.ifZCmp(MethodWriter.NE, jump);
                 else {
                     throw new IllegalStateException(error("Illegal tree structure."));
                 }
@@ -440,12 +440,12 @@ public final class EComp extends AExpression {
             case LONG:
             case FLOAT:
             case DOUBLE:
-                if      (eq)  adapter.ifCmp(rtype, GeneratorAdapter.EQ, jump);
-                else if (ne)  adapter.ifCmp(rtype, GeneratorAdapter.NE, jump);
-                else if (lt)  adapter.ifCmp(rtype, GeneratorAdapter.LT, jump);
-                else if (lte) adapter.ifCmp(rtype, GeneratorAdapter.LE, jump);
-                else if (gt)  adapter.ifCmp(rtype, GeneratorAdapter.GT, jump);
-                else if (gte) adapter.ifCmp(rtype, GeneratorAdapter.GE, jump);
+                if      (eq)  adapter.ifCmp(rtype, MethodWriter.EQ, jump);
+                else if (ne)  adapter.ifCmp(rtype, MethodWriter.NE, jump);
+                else if (lt)  adapter.ifCmp(rtype, MethodWriter.LT, jump);
+                else if (lte) adapter.ifCmp(rtype, MethodWriter.LE, jump);
+                else if (gt)  adapter.ifCmp(rtype, MethodWriter.GT, jump);
+                else if (gte) adapter.ifCmp(rtype, MethodWriter.GE, jump);
                 else {
                     throw new IllegalStateException(error("Illegal tree structure."));
                 }
@@ -458,16 +458,16 @@ public final class EComp extends AExpression {
                     } else if (!left.isNull && operation == Operation.EQ) {
                         adapter.invokeStatic(definition.defobjType.type, DEF_EQ_CALL);
                     } else {
-                        adapter.ifCmp(rtype, GeneratorAdapter.EQ, jump);
+                        adapter.ifCmp(rtype, MethodWriter.EQ, jump);
                     }
                 } else if (ne) {
                     if (right.isNull) {
                         adapter.ifNonNull(jump);
                     } else if (!left.isNull && operation == Operation.NE) {
                         adapter.invokeStatic(definition.defobjType.type, DEF_EQ_CALL);
-                        adapter.ifZCmp(GeneratorAdapter.EQ, jump);
+                        adapter.ifZCmp(MethodWriter.EQ, jump);
                     } else {
-                        adapter.ifCmp(rtype, GeneratorAdapter.NE, jump);
+                        adapter.ifCmp(rtype, MethodWriter.NE, jump);
                     }
                 } else if (lt) {
                     adapter.invokeStatic(definition.defobjType.type, DEF_LT_CALL);
@@ -484,7 +484,7 @@ public final class EComp extends AExpression {
                 writejump = left.isNull || ne || operation == Operation.EQR;
 
                 if (branch && !writejump) {
-                    adapter.ifZCmp(GeneratorAdapter.NE, jump);
+                    adapter.ifZCmp(MethodWriter.NE, jump);
                 }
 
                 break;
@@ -496,21 +496,21 @@ public final class EComp extends AExpression {
                         adapter.invokeStatic(definition.utilityType.type, CHECKEQUALS);
 
                         if (branch) {
-                            adapter.ifZCmp(GeneratorAdapter.NE, jump);
+                            adapter.ifZCmp(MethodWriter.NE, jump);
                         }
 
                         writejump = false;
                     } else {
-                        adapter.ifCmp(rtype, GeneratorAdapter.EQ, jump);
+                        adapter.ifCmp(rtype, MethodWriter.EQ, jump);
                     }
                 } else if (ne) {
                     if (right.isNull) {
                         adapter.ifNonNull(jump);
                     } else if (operation == Operation.NE) {
                         adapter.invokeStatic(definition.utilityType.type, CHECKEQUALS);
-                        adapter.ifZCmp(GeneratorAdapter.EQ, jump);
+                        adapter.ifZCmp(MethodWriter.EQ, jump);
                     } else {
-                        adapter.ifCmp(rtype, GeneratorAdapter.NE, jump);
+                        adapter.ifCmp(rtype, MethodWriter.NE, jump);
                     }
                 } else {
                     throw new IllegalStateException(error("Illegal tree structure."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
index 1f751f9f7a3..5853fa3242e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
@@ -25,7 +25,7 @@ import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Respresents a conditional expression.
@@ -77,7 +77,7 @@ public final class EConditional extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         final Label localfals = new Label();
         final Label end = new Label();
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
index 41d5add5a31..7afa88ffc9a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
@@ -23,8 +23,7 @@ import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Variables;
-import org.elasticsearch.painless.WriterUtility;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Respresents a constant.  Note this replaces any other expression
@@ -64,7 +63,7 @@ final class EConstant extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         final Sort sort = actual.sort;
 
         switch (sort) {
@@ -91,7 +90,7 @@ final class EConstant extends AExpression {
         }
 
         if (sort != Sort.BOOL) {
-            WriterUtility.writeBranch(adapter, tru, fals);
+            adapter.writeBranch(tru, fals);
         }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
index 629c3cc0aa6..7583d3eb158 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Respresents a decimal constant.
@@ -57,7 +57,7 @@ public final class EDecimal extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         throw new IllegalArgumentException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
index 6dc3640a6d5..22ed4a84178 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
@@ -23,7 +23,7 @@ import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Variables;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents an explicit cast.
@@ -57,7 +57,7 @@ public final class EExplicit extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         throw new IllegalArgumentException(error("Illegal tree structure."));
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
index 6e953b50c2e..0c8500c528b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
@@ -23,7 +23,7 @@ import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Opcodes;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents a null constant.
@@ -50,7 +50,7 @@ public final class ENull extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         adapter.visitInsn(Opcodes.ACONST_NULL);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
index 36ddf7f7242..ed7314b3571 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
@@ -23,7 +23,7 @@ import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Variables;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Respresents a non-decimal numeric constant.
@@ -96,7 +96,7 @@ public final class ENumeric extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         throw new IllegalArgumentException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index bc0929abdc2..f4205223a65 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -26,9 +26,8 @@ import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Variables;
-import org.elasticsearch.painless.WriterUtility;
 import org.objectweb.asm.Label;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 import static org.elasticsearch.painless.WriterConstants.DEF_NEG_CALL;
 import static org.elasticsearch.painless.WriterConstants.DEF_NOT_CALL;
@@ -169,7 +168,7 @@ public final class EUnary extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         if (operation == Operation.NOT) {
             if (tru == null && fals == null) {
                 final Label localfals = new Label();
@@ -206,14 +205,14 @@ public final class EUnary extends AExpression {
                         throw new IllegalStateException(error("Illegal tree structure."));
                     }
 
-                    adapter.math(GeneratorAdapter.XOR, type);
+                    adapter.math(MethodWriter.XOR, type);
                 }
             } else if (operation == Operation.SUB) {
                 if (sort == Sort.DEF) {
                     adapter.invokeStatic(definition.defobjType.type, DEF_NEG_CALL);
                 } else {
                     if (settings.getNumericOverflow()) {
-                        adapter.math(GeneratorAdapter.NEG, type);
+                        adapter.math(MethodWriter.NEG, type);
                     } else {
                         if (sort == Sort.INT) {
                             adapter.invokeStatic(definition.mathType.type, NEGATEEXACT_INT);
@@ -228,7 +227,7 @@ public final class EUnary extends AExpression {
                 throw new IllegalStateException(error("Illegal tree structure."));
             }
 
-            WriterUtility.writeBranch(adapter, tru, fals);
+            adapter.writeBranch(tru, fals);
         }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
index 34ad36480de..5803fcfa273 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents an array length field load.
@@ -55,17 +55,17 @@ public final class LArrayLength extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         adapter.arrayLength();
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
index 38d31d65a93..b38826f9e7e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
@@ -23,7 +23,7 @@ import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Variables;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 import java.util.List;
 import java.util.Map;
@@ -69,17 +69,17 @@ public final class LBrace extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         index.write(settings, definition, adapter);
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         adapter.arrayLoad(after.type);
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         adapter.arrayStore(after.type);
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
index ee9a996ac22..2dd03039ba7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
@@ -24,7 +24,7 @@ import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Struct;
 import org.elasticsearch.painless.Variables;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 import java.util.List;
 
@@ -87,12 +87,12 @@ public final class LCall extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         for (final AExpression argument : arguments) {
             argument.write(settings, definition, adapter);
         }
@@ -111,7 +111,7 @@ public final class LCall extends ALink {
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
index 64b64fb9c13..eb7fb3a6b10 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
@@ -24,8 +24,7 @@ import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.Variables;
-import org.elasticsearch.painless.WriterUtility;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents a cast made in a variable/method chain.
@@ -62,17 +61,17 @@ public final class LCast extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
-        WriterUtility.writeCast(adapter, cast);
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+        adapter.writeCast(cast);
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index f8034837837..98b2fbe7bf9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -24,7 +24,7 @@ import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Type;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
 
@@ -53,18 +53,18 @@ final class LDefArray extends ALink implements IDefLink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         index.write(settings, definition, adapter);
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         final String desc = Type.getMethodDescriptor(after.type, definition.defType.type, index.actual.type);
         adapter.invokeDynamic("arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_LOAD);
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         final String desc = Type.getMethodDescriptor(definition.voidType.type, definition.defType.type,
             index.actual.type, after.type);
         adapter.invokeDynamic("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_STORE);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index 0ac3cb3dedd..9dbe65110ee 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -23,7 +23,7 @@ import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Variables;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 import java.util.List;
 
@@ -61,12 +61,12 @@ final class LDefCall extends ALink implements IDefLink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         final StringBuilder signature = new StringBuilder();
 
         signature.append('(');
@@ -88,7 +88,7 @@ final class LDefCall extends ALink implements IDefLink {
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index 0c1d8270065..759b407cb5a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -24,7 +24,7 @@ import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Type;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
 
@@ -50,18 +50,18 @@ final class LDefField extends ALink implements IDefLink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         final String desc = Type.getMethodDescriptor(after.type, definition.defType.type);
         adapter.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.LOAD);
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         final String desc = Type.getMethodDescriptor(definition.voidType.type, definition.defType.type, after.type);
         adapter.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.STORE);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
index a13991e87ba..bd0079d1843 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
@@ -25,7 +25,7 @@ import org.elasticsearch.painless.Definition.Field;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Struct;
 import org.elasticsearch.painless.Variables;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 import java.util.List;
 import java.util.Map;
@@ -96,12 +96,12 @@ public final class LField extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isStatic(field.reflect.getModifiers())) {
             adapter.getStatic(field.owner.type, field.reflect.getName(), field.type.type);
 
@@ -118,7 +118,7 @@ public final class LField extends ALink {
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isStatic(field.reflect.getModifiers())) {
             adapter.putStatic(field.owner.type, field.reflect.getName(), field.type.type);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
index cf256a947f6..00e44967279 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
@@ -24,8 +24,7 @@ import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Variables;
-import org.elasticsearch.painless.WriterUtility;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents a list load/store shortcut.  (Internal only.)
@@ -75,12 +74,12 @@ final class LListShortcut extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         index.write(settings, definition, adapter);
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
             adapter.invokeInterface(getter.owner.type, getter.method);
         } else {
@@ -93,13 +92,13 @@ final class LListShortcut extends ALink {
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
             adapter.invokeInterface(setter.owner.type, setter.method);
         } else {
             adapter.invokeVirtual(setter.owner.type, setter.method);
         }
 
-        WriterUtility.writePop(adapter, setter.rtn.sort.size);
+        adapter.writePop(setter.rtn.sort.size);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
index 34c581f382b..a3d9d119eda 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
@@ -24,8 +24,7 @@ import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Variables;
-import org.elasticsearch.painless.WriterUtility;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents a map load/store shortcut. (Internal only.)
@@ -74,12 +73,12 @@ final class LMapShortcut extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         index.write(settings, definition, adapter);
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
             adapter.invokeInterface(getter.owner.type, getter.method);
         } else {
@@ -92,13 +91,13 @@ final class LMapShortcut extends ALink {
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
             adapter.invokeInterface(setter.owner.type, setter.method);
         } else {
             adapter.invokeVirtual(setter.owner.type, setter.method);
         }
 
-        WriterUtility.writePop(adapter, setter.rtn.sort.size);
+        adapter.writePop(setter.rtn.sort.size);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
index db0a09185d9..67f9769bd6a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
@@ -23,7 +23,7 @@ import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Variables;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 import java.util.List;
 
@@ -74,12 +74,12 @@ public final class LNewArray extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         for (final AExpression argument : arguments) {
             argument.write(settings, definition, adapter);
         }
@@ -92,7 +92,7 @@ public final class LNewArray extends ALink {
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
index e5251e4130a..fb91b7a6915 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
@@ -25,7 +25,7 @@ import org.elasticsearch.painless.Definition.Constructor;
 import org.elasticsearch.painless.Definition.Struct;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Variables;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 import java.util.List;
 
@@ -92,12 +92,12 @@ public final class LNewObj extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         adapter.newInstance(after.type);
 
         if (load) {
@@ -112,7 +112,7 @@ public final class LNewObj extends ALink {
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
index 95e3bbe3a44..5f18cccd512 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
@@ -25,8 +25,7 @@ import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Struct;
 import org.elasticsearch.painless.Variables;
-import org.elasticsearch.painless.WriterUtility;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents a field load/store shortcut.  (Internal only.)
@@ -75,12 +74,12 @@ final class LShortcut extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
             adapter.invokeInterface(getter.owner.type, getter.method);
         } else {
@@ -93,13 +92,13 @@ final class LShortcut extends ALink {
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
             adapter.invokeInterface(setter.owner.type, setter.method);
         } else {
             adapter.invokeVirtual(setter.owner.type, setter.method);
         }
 
-        WriterUtility.writePop(adapter, setter.rtn.sort.size);
+        adapter.writePop(setter.rtn.sort.size);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
index 7291b4cf8df..1d11652f483 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents a string constant.
@@ -51,17 +51,17 @@ public final class LString extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         adapter.push(string);
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
index 989fa9f9ef7..35a652f5b84 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
@@ -25,7 +25,7 @@ import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.Variables.Variable;
 import org.objectweb.asm.Opcodes;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents a variable load/store.
@@ -74,17 +74,17 @@ public final class LVariable extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         adapter.visitVarInsn(after.type.getOpcode(Opcodes.ILOAD), slot);
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         adapter.visitVarInsn(after.type.getOpcode(Opcodes.ISTORE), slot);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
index abaa332c7d3..802d5b6c415 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 import java.util.Collections;
 import java.util.List;
@@ -65,7 +65,7 @@ public final class SBlock extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         for (final AStatement statement : statements) {
             statement.continu = continu;
             statement.brake = brake;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
index c680eeee8fd..3998b1a9ff2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents a break statement.
@@ -46,7 +46,7 @@ public final class SBreak extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         writeDebugInfo(adapter);
         adapter.goTo(brake);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
index bf6d7f0ad61..2c4e33b6326 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents a continue statement.
@@ -49,7 +49,7 @@ public final class SContinue extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         writeDebugInfo(adapter);
         adapter.goTo(continu);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
index 8511f087ef1..5494ef9c32a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 import java.util.Collections;
 import java.util.List;
@@ -50,7 +50,7 @@ public final class SDeclBlock extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         for (final SDeclaration declaration : declarations) {
             declaration.write(settings, definition, adapter);
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
index 62d144d4a66..189ddc95936 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
@@ -25,7 +25,7 @@ import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.Variables.Variable;
 import org.objectweb.asm.Opcodes;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents a single variable declaration.
@@ -58,7 +58,7 @@ public final class SDeclaration extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         writeDebugInfo(adapter);
         final org.objectweb.asm.Type type = variable.type.type;
         final Sort sort = variable.type.sort;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
index 25549e8bbce..63d3df4bcee 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -22,9 +22,8 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
-import org.elasticsearch.painless.WriterUtility;
 import org.objectweb.asm.Label;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents a do-while loop.
@@ -81,7 +80,7 @@ public final class SDo extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         writeDebugInfo(adapter);
         final Label start = new Label();
         final Label begin = new Label();
@@ -98,7 +97,7 @@ public final class SDo extends AStatement {
         condition.fals = end;
         condition.write(settings, definition, adapter);
 
-        WriterUtility.writeLoopCounter(adapter, loopCounterSlot, Math.max(1, block.statementCount));
+        adapter.writeLoopCounter(loopCounterSlot, Math.max(1, block.statementCount));
 
         adapter.goTo(start);
         adapter.mark(end);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index b8e17fd3560..dd7cffaa970 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -23,8 +23,7 @@ import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Variables;
-import org.elasticsearch.painless.WriterUtility;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents the top-level node for an expression as a statement.
@@ -60,14 +59,14 @@ public final class SExpression extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         writeDebugInfo(adapter);
         expression.write(settings, definition, adapter);
 
         if (methodEscape) {
             adapter.returnValue();
         } else {
-            WriterUtility.writePop(adapter, expression.expected.sort.size);
+            adapter.writePop(expression.expected.sort.size);
         }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index b76947f8e56..43c978b47fa 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -22,9 +22,8 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
-import org.elasticsearch.painless.WriterUtility;
 import org.objectweb.asm.Label;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents a for loop.
@@ -129,7 +128,7 @@ public final class SFor extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         writeDebugInfo(adapter);
         final Label start = new Label();
         final Label begin = afterthought == null ? start : new Label();
@@ -141,7 +140,7 @@ public final class SFor extends AStatement {
             AExpression initializer = (AExpression)this.initializer;
 
             initializer.write(settings, definition, adapter);
-            WriterUtility.writePop(adapter, initializer.expected.sort.size);
+            adapter.writePop(initializer.expected.sort.size);
         }
 
         adapter.mark(start);
@@ -162,10 +161,10 @@ public final class SFor extends AStatement {
                 ++statementCount;
             }
 
-            WriterUtility.writeLoopCounter(adapter, loopCounterSlot, statementCount);
+            adapter.writeLoopCounter(loopCounterSlot, statementCount);
             block.write(settings, definition, adapter);
         } else {
-            WriterUtility.writeLoopCounter(adapter, loopCounterSlot, 1);
+            adapter.writeLoopCounter(loopCounterSlot, 1);
         }
 
         if (afterthought != null) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
index e877acd0f99..698d8c8126c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
@@ -23,7 +23,7 @@ import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents an if/else block.
@@ -84,7 +84,7 @@ public final class SIfElse extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         writeDebugInfo(adapter);
         final Label end = new Label();
         final Label fals = elseblock != null ? new Label() : end;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
index 25f7ea17f43..b959b47a96b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents a return statement.
@@ -51,7 +51,7 @@ public final class SReturn extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         writeDebugInfo(adapter);
         expression.write(settings, definition, adapter);
         adapter.returnValue();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index b6e731854d0..e5a80802ce8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -23,7 +23,7 @@ import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Opcodes;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 import java.util.Collections;
 import java.util.List;
@@ -63,7 +63,7 @@ public final class SSource extends AStatement {
     }
 
     @Override
-    public void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    public void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         for (final AStatement statement : statements) {
             statement.write(settings, definition, adapter);
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
index 3bcb805fb6a..eac039b998e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents a throw statement.
@@ -50,7 +50,7 @@ public final class SThrow extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         writeDebugInfo(adapter);
         expression.write(settings, definition, adapter);
         adapter.throwException();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java
index 1c3d2edc48f..acb50a2b962 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java
@@ -25,7 +25,7 @@ import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.Variables.Variable;
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents a catch block as part of a try-catch block.
@@ -75,7 +75,7 @@ public final class STrap extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         writeDebugInfo(adapter);
         final Label jump = new Label();
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
index c6d5a16e06c..0329826b02a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
@@ -23,7 +23,7 @@ import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 import java.util.Collections;
 import java.util.List;
@@ -83,7 +83,7 @@ public final class STry extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         writeDebugInfo(adapter);
         final Label begin = new Label();
         final Label end = new Label();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
index fd2f359c3c6..f6f8ddc678f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -22,9 +22,8 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
-import org.elasticsearch.painless.WriterUtility;
 import org.objectweb.asm.Label;
-import org.objectweb.asm.commons.GeneratorAdapter;
+import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents a while loop.
@@ -93,7 +92,7 @@ public final class SWhile extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final GeneratorAdapter adapter) {
+    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         writeDebugInfo(adapter);
         final Label begin = new Label();
         final Label end = new Label();
@@ -104,13 +103,13 @@ public final class SWhile extends AStatement {
         condition.write(settings, definition, adapter);
 
         if (block != null) {
-            WriterUtility.writeLoopCounter(adapter, loopCounterSlot, Math.max(1, block.statementCount));
+            adapter.writeLoopCounter(loopCounterSlot, Math.max(1, block.statementCount));
 
             block.continu = begin;
             block.brake = end;
             block.write(settings, definition, adapter);
         } else {
-            WriterUtility.writeLoopCounter(adapter, loopCounterSlot, 1);
+            adapter.writeLoopCounter(loopCounterSlot, 1);
         }
 
         if (block == null || !block.allEscape) {

From be0bbce5dfd275049d74040edb39af2286798c3a Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 16 May 2016 21:16:00 -0400
Subject: [PATCH 0284/1311] Clarify squashing of pull request commits

This commit rewords the note regarding the squashing of pull request
commits to make it clearer that the note applies to everyone and not
just external contributors.
---
 CONTRIBUTING.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 4868bbc6961..b0f1e054e46 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -79,7 +79,8 @@ reviewers. If you need to pick up changes from master, you can merge
 master into your branch. A reviewer might ask you to rebase a
 long-running pull request in which case force pushing is okay for that
 request. Note that squashing at the end of the review process should
-also not be done, we can do that if necessary.
+also not be done, that can be done when the pull request is [integrated
+via GitHub](https://github.com/blog/2141-squash-your-commits).
 
 Contributing to the Elasticsearch codebase
 ------------------------------------------

From e2367b2f12119394274dddf31a5300045a67d6f4 Mon Sep 17 00:00:00 2001
From: Colin Goodheart-Smithe <colings86@users.noreply.github.com>
Date: Tue, 17 May 2016 09:58:30 +0100
Subject: [PATCH 0285/1311] [DOCS] Updates the documentation for the recent
 changes in the profiler

---
 .../migration/migrate_5_0/search.asciidoc     |  6 +++
 docs/reference/search/profile.asciidoc        | 42 +++++++++----------
 2 files changed, 27 insertions(+), 21 deletions(-)

diff --git a/docs/reference/migration/migrate_5_0/search.asciidoc b/docs/reference/migration/migrate_5_0/search.asciidoc
index 378c86bccb7..bb450f9a819 100644
--- a/docs/reference/migration/migrate_5_0/search.asciidoc
+++ b/docs/reference/migration/migrate_5_0/search.asciidoc
@@ -176,3 +176,9 @@ The `coerce` and `ignore_malformed` parameters were deprecated in favour of `val
 * Top level inner hits syntax has been removed. Inner hits can now only be specified as part of the `nested`,
 `has_child` and `has_parent` queries. Use cases previously only possible with top level inner hits can now be done
 with inner hits defined inside the query dsl.
+
+==== Query Profiler
+
+In the response for profiling queries, the `query_type` has been renamed to `type` and `lucene` has been renamed to
+`description`. These changes have been made so the response format is more friendly to supporting other types of profiling
+in the future.
diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc
index 72fee5c40dc..48b9de5ce60 100644
--- a/docs/reference/search/profile.asciidoc
+++ b/docs/reference/search/profile.asciidoc
@@ -65,8 +65,8 @@ This will yield the following result:
               {
                  "query": [
                     {
-                       "query_type": "BooleanQuery",
-                       "lucene": "message:search message:test",
+                       "type": "BooleanQuery",
+                       "description": "message:search message:test",
                        "time": "15.52889800ms",
                        "breakdown": {
                           "score": 0,
@@ -78,8 +78,8 @@ This will yield the following result:
                        },
                        "children": [
                           {
-                             "query_type": "TermQuery",
-                             "lucene": "message:search",
+                             "type": "TermQuery",
+                             "description": "message:search",
                              "time": "4.938855000ms",
                              "breakdown": {
                                 "score": 0,
@@ -91,8 +91,8 @@ This will yield the following result:
                              }
                           },
                           {
-                             "query_type": "TermQuery",
-                             "lucene": "message:test",
+                             "type": "TermQuery",
+                             "description": "message:test",
                              "time": "0.5016660000ms",
                              "breakdown": {
                                 "score": 0,
@@ -180,20 +180,20 @@ The overall structure of this query tree will resemble your original Elasticsear
 --------------------------------------------------
 "query": [
     {
-       "query_type": "BooleanQuery",
-       "lucene": "message:search message:test",
+       "type": "BooleanQuery",
+       "description": "message:search message:test",
        "time": "15.52889800ms",
        "breakdown": {...},               <1>
        "children": [
           {
-             "query_type": "TermQuery",
-             "lucene": "message:search",
+             "type": "TermQuery",
+             "description": "message:search",
              "time": "4.938855000ms",
              "breakdown": {...}
           },
           {
-             "query_type": "TermQuery",
-             "lucene": "message:test",
+             "type": "TermQuery",
+             "description": "message:test",
              "time": "0.5016660000ms",
              "breakdown": {...}
           }
@@ -204,8 +204,8 @@ The overall structure of this query tree will resemble your original Elasticsear
 <1> The breakdown timings are omitted for simplicity
 
 Based on the profile structure, we can see that our `match` query was rewritten by Lucene into a BooleanQuery with two
-clauses (both holding a TermQuery).  The `"query_type"` field displays the Lucene class name, and often aligns with
-the equivalent name in Elasticsearch.  The `"lucene"` field displays the Lucene explanation text for the query, and
+clauses (both holding a TermQuery).  The `"type"` field displays the Lucene class name, and often aligns with
+the equivalent name in Elasticsearch.  The `"description"` field displays the Lucene explanation text for the query, and
 is made available to help differentiating between parts of your query (e.g. both `"message:search"` and `"message:test"`
 are TermQuery's and would appear identical otherwise.
 
@@ -214,7 +214,7 @@ of all children.
 
 The `"breakdown"` field will give detailed stats about how the time was spent, we'll look at
 that in a moment.  Finally, the `"children"` array lists any sub-queries that may be present.  Because we searched for two
-values ("search test"), our BooleanQuery holds two children TermQueries.  They have identical information (query_type, time,
+values ("search test"), our BooleanQuery holds two children TermQueries.  They have identical information (type, time,
 breakdown, etc).  Children are allowed to have their own children.
 
 ==== Timing Breakdown
@@ -465,8 +465,8 @@ And the response:
                   {
                      "query": [
                         {
-                           "query_type": "TermQuery",
-                           "lucene": "my_field:foo",
+                           "type": "TermQuery",
+                           "description": "my_field:foo",
                            "time": "0.4094560000ms",
                            "breakdown": {
                               "score": 0,
@@ -478,8 +478,8 @@ And the response:
                            }
                         },
                         {
-                           "query_type": "TermQuery",
-                           "lucene": "message:search",
+                           "type": "TermQuery",
+                           "description": "message:search",
                            "time": "0.3037020000ms",
                            "breakdown": {
                               "score": 0,
@@ -522,8 +522,8 @@ And the response:
                   {
                      "query": [
                         {
-                           "query_type": "MatchAllDocsQuery",
-                           "lucene": "*:*",
+                           "type": "MatchAllDocsQuery",
+                           "description": "*:*",
                            "time": "0.04829300000ms",
                            "breakdown": {
                               "score": 0,

From 48ea9137da01a589762cd7d9a0dce69790b3516c Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Tue, 17 May 2016 15:14:37 +0200
Subject: [PATCH 0286/1311] Add CONSOLE annotation to sort documentation

This adds CONSOLE to sort docs in order to automatically execute the doc
snippets. Fixes a few minor types along the way.

Relates to #18160
---
 docs/reference/search/request/sort.asciidoc | 81 ++++++++++++++++++---
 1 file changed, 69 insertions(+), 12 deletions(-)

diff --git a/docs/reference/search/request/sort.asciidoc b/docs/reference/search/request/sort.asciidoc
index dfe4b586655..36252d3caa2 100644
--- a/docs/reference/search/request/sort.asciidoc
+++ b/docs/reference/search/request/sort.asciidoc
@@ -5,8 +5,35 @@ Allows to add one or more sort on specific fields. Each sort can be
 reversed as well. The sort is defined on a per field level, with special
 field name for `_score` to sort by score, and `_doc` to sort by index order.
 
+Assuming the following index mapping:
+
 [source,js]
 --------------------------------------------------
+PUT /my_index
+{
+    "mappings": {
+        "my_type": {
+            "properties": {
+                "post_date": { "type": "date" },
+                "user": { 
+                    "type": "string",
+                    "fielddata": "true"
+                },
+                "name": {
+                    "type": "string",
+                    "fielddata": "true"
+                },
+                "age": { "type": "integer" }
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+
+[source,js]
+--------------------------------------------------
+GET /my_index/my_type/_search
 {
     "sort" : [
         { "post_date" : {"order" : "asc"}},
@@ -20,6 +47,8 @@ field name for `_score` to sort by score, and `_doc` to sort by index order.
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 NOTE: `_doc` has no real use-case besides being the most efficient sort order.
 So if you don't care about the order in which documents are returned, then you
@@ -60,20 +89,28 @@ to. The `mode` option can have the following values:
 ===== Sort mode example usage
 
 In the example below the field price has multiple prices per document.
-In this case the result hits will be sort by price ascending based on
+In this case the result hits will be sorted by price ascending based on
 the average price per document.
 
 [source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/_search' -d '{
+PUT /my_index/my_type/1
+{
+   "product": "chocolate",
+    "price": [20, 4]
+}
+
+POST /_search 
+{
    "query" : {
-    ...
+      "term" : { "product" : "chocolate" }
    },
    "sort" : [
       {"price" : {"order" : "asc", "mode" : "avg"}}
    ]
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
 [[nested-sorting]]
 ==== Sorting within nested objects.
@@ -101,9 +138,10 @@ The `nested_path` needs to be specified; otherwise, elasticsearch doesn't know o
 
 [source,js]
 --------------------------------------------------
-curl -XPOST 'localhost:9200/_search' -d '{
+POST /_search
+{
    "query" : {
-    ...
+      "term" : { "product" : "chocolate" }
    },
    "sort" : [
        {
@@ -117,8 +155,9 @@ curl -XPOST 'localhost:9200/_search' -d '{
           }
        }
     ]
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
 Nested sorting is also supported when sorting by
 scripts and sorting by geo distance.
@@ -132,15 +171,17 @@ will be used for missing docs as the sort value). For example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "sort" : [
-        { "price" : {"missing" : "_last"} },
+        { "price" : {"missing" : "_last"} }
     ],
     "query" : {
-        "term" : { "user" : "kimchy" }
+        "term" : { "product" : "chocolate" }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 NOTE: If a nested inner object doesn't match with
 the `nested_filter` then a missing value is used.
@@ -155,15 +196,17 @@ example of how it can be used:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "sort" : [
-        { "price" : {"unmapped_type" : "long"} },
+        { "price" : {"unmapped_type" : "long"} }
     ],
     "query" : {
-        "term" : { "user" : "kimchy" }
+        "term" : { "product" : "chocolate" }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 If any of the indices that are queried doesn't have a mapping for `price`
 then Elasticsearch will handle it as if there was a mapping of type
@@ -176,6 +219,7 @@ Allow to sort by `_geo_distance`. Here is an example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "sort" : [
         {
@@ -193,6 +237,7 @@ Allow to sort by `_geo_distance`. Here is an example:
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 
 
@@ -209,6 +254,7 @@ The following formats are supported in providing the coordinates:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "sort" : [
         {
@@ -227,6 +273,7 @@ The following formats are supported in providing the coordinates:
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 ===== Lat Lon as String
 
@@ -234,6 +281,7 @@ Format in `lat,lon`.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "sort" : [
         {
@@ -249,11 +297,13 @@ Format in `lat,lon`.
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 ===== Geohash
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "sort" : [
         {
@@ -269,6 +319,7 @@ Format in `lat,lon`.
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 ===== Lat Lon as Array
 
@@ -277,6 +328,7 @@ conform with http://geojson.org/[GeoJSON].
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "sort" : [
         {
@@ -292,6 +344,7 @@ conform with http://geojson.org/[GeoJSON].
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 
 ==== Multiple reference points
@@ -316,9 +369,10 @@ Allow to sort based on custom scripts, here is an example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "query" : {
-        ....
+        "term" : { "user" : "kimchy" }
     },
     "sort" : {
         "_script" : {
@@ -334,6 +388,7 @@ Allow to sort based on custom scripts, here is an example:
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 
 ==== Track Scores
@@ -343,6 +398,7 @@ When sorting on a field, scores are not computed. By setting
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "track_scores": true,
     "sort" : [
@@ -355,6 +411,7 @@ When sorting on a field, scores are not computed. By setting
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 ==== Memory Considerations
 

From 92339c476303b0451abd0774fd899ea2309fca70 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 17 May 2016 09:17:10 -0400
Subject: [PATCH 0287/1311] painless: add method overloading based on arity

Closes #18385

Squashed commit of the following:

commit b2819df4d392d69b86e5c96d358eb03424e67e02
Author: Robert Muir <rmuir@apache.org>
Date:   Tue May 17 09:15:47 2016 -0400

    add note about tuple

commit 85fcac6a0d0674da24535121eab23e2c407d683f
Author: Robert Muir <rmuir@apache.org>
Date:   Mon May 16 23:39:25 2016 -0400

    painless: add method overloading based on arity
---
 .../java/org/elasticsearch/painless/Def.java  |  58 +++---
 .../elasticsearch/painless/DefBootstrap.java  |   6 +-
 .../elasticsearch/painless/Definition.java    | 175 +++++++++++++-----
 .../elasticsearch/painless/FeatureTest.java   |  66 +++++++
 .../elasticsearch/painless/node/LCall.java    |  11 +-
 .../elasticsearch/painless/node/LField.java   |   9 +-
 .../painless/node/LListShortcut.java          |   4 +-
 .../painless/node/LMapShortcut.java           |   4 +-
 .../elasticsearch/painless/node/LNewObj.java  |   2 +-
 .../painless/node/LShortcut.java              |   4 +-
 .../elasticsearch/painless/OverloadTests.java |  52 ++++++
 11 files changed, 301 insertions(+), 90 deletions(-)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java
 create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index f82f04d801a..d9b2e02a237 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -133,37 +133,41 @@ public final class Def {
      * <p>
      * @param receiverClass Class of the object to invoke the method on.
      * @param name Name of the method.
+     * @param type Callsite signature. Need not match exactly, except the number of parameters.
      * @param definition Whitelist to check.
      * @return pointer to matching method to invoke. never returns null.
      * @throws IllegalArgumentException if no matching whitelisted method was found.
      */
-     static MethodHandle lookupMethod(Class<?> receiverClass, String name, Definition definition) {
-        // check whitelist for matching method
-        for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
-            RuntimeClass struct = definition.runtimeMap.get(clazz);
-
-            if (struct != null) {
-                Method method = struct.methods.get(name);
-                if (method != null) {
-                    return method.handle;
-                }
-            }
-
-            for (final Class<?> iface : clazz.getInterfaces()) {
-                struct = definition.runtimeMap.get(iface);
-
-                if (struct != null) {
-                    Method method = struct.methods.get(name);
-                    if (method != null) {
-                        return method.handle;
-                    }
-                }
-            }
-        }
-
-        // no matching methods in whitelist found
-        throw new IllegalArgumentException("Unable to find dynamic method [" + name + "] " +
-                                           "for class [" + receiverClass.getCanonicalName() + "].");
+     static MethodHandle lookupMethod(Class<?> receiverClass, String name, MethodType type, Definition definition) {
+         // we don't consider receiver an argument/counting towards arity
+         type = type.dropParameterTypes(0, 1);
+         Definition.MethodKey key = new Definition.MethodKey(name, type.parameterCount());
+         // check whitelist for matching method
+         for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
+             RuntimeClass struct = definition.runtimeMap.get(clazz);
+             
+             if (struct != null) {
+                 Method method = struct.methods.get(key);
+                 if (method != null) {
+                     return method.handle;
+                 }
+             }
+             
+             for (final Class<?> iface : clazz.getInterfaces()) {
+                 struct = definition.runtimeMap.get(iface);
+                 
+                 if (struct != null) {
+                     Method method = struct.methods.get(key);
+                     if (method != null) {
+                         return method.handle;
+                     }
+                 }
+             }
+         }
+         
+         // no matching methods in whitelist found
+         throw new IllegalArgumentException("Unable to find dynamic method [" + name + "] with signature [" + type + "] " +
+                 "for class [" + receiverClass.getCanonicalName() + "].");
     }
 
     /**
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index d9e96302eec..380f5455ab3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -91,10 +91,10 @@ public final class DefBootstrap {
         /**
          * Does a slow lookup against the whitelist.
          */
-        private static MethodHandle lookup(int flavor, Class<?> clazz, String name) {
+        private static MethodHandle lookup(int flavor, Class<?> clazz, String name, MethodType type) {
             switch(flavor) {
                 case METHOD_CALL:
-                    return Def.lookupMethod(clazz, name, Definition.INSTANCE);
+                    return Def.lookupMethod(clazz, name, type, Definition.INSTANCE);
                 case LOAD:
                     return Def.lookupGetter(clazz, name, Definition.INSTANCE);
                 case STORE:
@@ -115,7 +115,7 @@ public final class DefBootstrap {
             final MethodType type = type();
             final Object receiver = args[0];
             final Class<?> receiverClass = receiver.getClass();
-            final MethodHandle target = lookup(flavor, receiverClass, name).asType(type);
+            final MethodHandle target = lookup(flavor, receiverClass, name, type).asType(type);
 
             if (depth >= MAX_DEPTH) {
                 // revert to a vtable call
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 6a55b2f5042..40dc13a3368 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -33,6 +33,7 @@ import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
 /**
@@ -194,17 +195,72 @@ public final class Definition {
             this.setter = setter;
         }
     }
+    
+    // TODO: instead of hashing on this, we could have a 'next' pointer in Method itself, but it would make code more complex
+    // please do *NOT* under any circumstances change this to be the crappy Tuple from elasticsearch!
+    /**
+     * Key for looking up a method.
+     * <p>
+     * Methods are keyed on both name and arity, and can be overloaded once per arity.
+     * This allows signatures such as {@code String.indexOf(String) vs String.indexOf(String, int)}.
+     * <p>
+     * It is less flexible than full signature overloading where types can differ too, but
+     * better than just the name, and overloading types adds complexity to users, too.
+     */
+    public static final class MethodKey {
+        public final String name;
+        public final int arity;
+        
+        /**
+         * Create a new lookup key
+         * @param name name of the method
+         * @param arity number of parameters
+         */
+        public MethodKey(String name, int arity) {
+            this.name = Objects.requireNonNull(name);
+            this.arity = arity;
+        }
+
+        @Override
+        public int hashCode() {
+            final int prime = 31;
+            int result = 1;
+            result = prime * result + arity;
+            result = prime * result + name.hashCode();
+            return result;
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj) return true;
+            if (obj == null) return false;
+            if (getClass() != obj.getClass()) return false;
+            MethodKey other = (MethodKey) obj;
+            if (arity != other.arity) return false;
+            if (!name.equals(other.name)) return false;
+            return true;
+        }
+
+        @Override
+        public String toString() {
+            StringBuilder sb = new StringBuilder();
+            sb.append(name);
+            sb.append('/');
+            sb.append(arity);
+            return sb.toString();
+        }
+    }
 
     public static final class Struct {
         public final String name;
         public final Class<?> clazz;
         public final org.objectweb.asm.Type type;
 
-        public final Map<String, Constructor> constructors;
-        public final Map<String, Method> functions;
-        public final Map<String, Method> methods;
+        public final Map<MethodKey, Constructor> constructors;
+        public final Map<MethodKey, Method> staticMethods;
+        public final Map<MethodKey, Method> methods;
 
-        public final Map<String, Field> statics;
+        public final Map<String, Field> staticMembers;
         public final Map<String, Field> members;
 
         private Struct(final String name, final Class<?> clazz, final org.objectweb.asm.Type type) {
@@ -213,10 +269,10 @@ public final class Definition {
             this.type = type;
 
             constructors = new HashMap<>();
-            functions = new HashMap<>();
+            staticMethods = new HashMap<>();
             methods = new HashMap<>();
 
-            statics = new HashMap<>();
+            staticMembers = new HashMap<>();
             members = new HashMap<>();
         }
 
@@ -226,10 +282,10 @@ public final class Definition {
             type = struct.type;
 
             constructors = Collections.unmodifiableMap(struct.constructors);
-            functions = Collections.unmodifiableMap(struct.functions);
+            staticMethods = Collections.unmodifiableMap(struct.staticMethods);
             methods = Collections.unmodifiableMap(struct.methods);
 
-            statics = Collections.unmodifiableMap(struct.statics);
+            staticMembers = Collections.unmodifiableMap(struct.staticMembers);
             members = Collections.unmodifiableMap(struct.members);
         }
 
@@ -304,11 +360,11 @@ public final class Definition {
     }
 
     public static final class RuntimeClass {
-        public final Map<String, Method> methods;
+        public final Map<MethodKey, Method> methods;
         public final Map<String, MethodHandle> getters;
         public final Map<String, MethodHandle> setters;
 
-        private RuntimeClass(final Map<String, Method> methods,
+        private RuntimeClass(final Map<MethodKey, Method> methods,
                              final Map<String, MethodHandle> getters, final Map<String, MethodHandle> setters) {
             this.methods = methods;
             this.getters = getters;
@@ -395,6 +451,9 @@ public final class Definition {
     public final Type longsType;
     public final Type doublesType;
     public final Type geoPointsType;
+    
+    // for testing features not currently "used" by the whitelist (we should not rush the API for that!)
+    public final Type featureTestType;
 
     private Definition() {
         structsMap = new HashMap<>();
@@ -476,6 +535,8 @@ public final class Definition {
         longsType = getType("Longs");
         doublesType = getType("Doubles");
         geoPointsType = getType("GeoPoints");
+        
+        featureTestType = getType("FeatureTest");
 
         addElements();
         copyStructs();
@@ -567,6 +628,8 @@ public final class Definition {
         this.longsType = definition.longsType;
         this.doublesType = definition.doublesType;
         this.geoPointsType = definition.geoPointsType;
+        
+        this.featureTestType = definition.featureTestType;
     }
 
     private void addStructs() {
@@ -643,6 +706,8 @@ public final class Definition {
         addStruct( "Longs"     , ScriptDocValues.Longs.class);
         addStruct( "Doubles"   , ScriptDocValues.Doubles.class);
         addStruct( "GeoPoints" , ScriptDocValues.GeoPoints.class);
+        
+        addStruct( "FeatureTest", FeatureTest.class);
     }
 
     private void addElements() {
@@ -765,6 +830,7 @@ public final class Definition {
         addMethod("String", "compareTo", null, false, intType, new Type[] {stringType}, null, null);
         addMethod("String", "concat", null, false, stringType, new Type[] {stringType}, null, null);
         addMethod("String", "endsWith", null, false, booleanType, new Type[] {stringType}, null, null);
+        addMethod("String", "indexOf", null, false, intType, new Type[] {stringType}, null, null);
         addMethod("String", "indexOf", null, false, intType, new Type[] {stringType, intType}, null, null);
         addMethod("String", "isEmpty", null, false, booleanType, new Type[] {}, null, null);
         addMethod("String", "replace", null, false, stringType, new Type[] {charseqType, charseqType}, null, null);
@@ -1112,6 +1178,16 @@ public final class Definition {
                   new Type[] { stringType }, null, null);
         addMethod("GeoPoints", "geohashDistanceInMiles", null, false, doubleType,
                   new Type[] { stringType }, null, null);
+        
+        // currently FeatureTest exposes overloaded constructor, field load store, and overloaded static methods
+        addConstructor("FeatureTest", "new", new Type[] {}, null);
+        addConstructor("FeatureTest", "new", new Type[] {intType, intType}, null);
+        addMethod("FeatureTest", "getX", null, false, intType, new Type[] {}, null, null);
+        addMethod("FeatureTest", "getY", null, false, intType, new Type[] {}, null, null);
+        addMethod("FeatureTest", "setX", null, false, voidType, new Type[] {intType}, null, null);
+        addMethod("FeatureTest", "setY", null, false, voidType, new Type[] {intType}, null, null);
+        addMethod("FeatureTest", "overloadedStatic", null, true, booleanType, new Type[] {}, null, null);
+        addMethod("FeatureTest", "overloadedStatic", null, true, booleanType, new Type[] {booleanType}, null, null);
     }
 
     private void copyStructs() {
@@ -1165,6 +1241,8 @@ public final class Definition {
         copyStruct("Longs", "List", "Collection", "Object");
         copyStruct("Doubles", "List", "Collection", "Object");
         copyStruct("GeoPoints", "List", "Collection", "Object");
+        
+        copyStruct("FeatureTest", "Object");
     }
 
     private void addTransforms() {
@@ -1482,6 +1560,8 @@ public final class Definition {
         addRuntimeClass(longsType.struct);
         addRuntimeClass(doublesType.struct);
         addRuntimeClass(geoPointsType.struct);
+        
+        addRuntimeClass(featureTestType.struct);
     }
 
     private final void addStruct(final String name, final Class<?> clazz) {
@@ -1510,20 +1590,22 @@ public final class Definition {
             throw new IllegalArgumentException(
                 "Invalid constructor name [" + name + "] with the struct [" + owner.name + "].");
         }
+        
+        MethodKey methodKey = new MethodKey(name, args.length);
 
-        if (owner.constructors.containsKey(name)) {
+        if (owner.constructors.containsKey(methodKey)) {
             throw new IllegalArgumentException(
-                "Duplicate constructor name [" + name + "] found within the struct [" + owner.name + "].");
+                "Duplicate constructor [" + methodKey + "] found within the struct [" + owner.name + "].");
         }
 
-        if (owner.statics.containsKey(name)) {
-            throw new IllegalArgumentException("Constructors and functions may not have the same name" +
-                " [" + name + "] within the same struct [" + owner.name + "].");
+        if (owner.staticMethods.containsKey(methodKey)) {
+            throw new IllegalArgumentException("Constructors and static methods may not have the same signature" +
+                " [" + methodKey + "] within the same struct [" + owner.name + "].");
         }
 
-        if (owner.methods.containsKey(name)) {
-            throw new IllegalArgumentException("Constructors and methods may not have the same name" +
-                " [" + name + "] within the same struct [" + owner.name + "].");
+        if (owner.methods.containsKey(methodKey)) {
+            throw new IllegalArgumentException("Constructors and methods may not have the same signature" +
+                " [" + methodKey + "] within the same struct [" + owner.name + "].");
         }
 
         final Class<?>[] classes = new Class<?>[args.length];
@@ -1553,7 +1635,7 @@ public final class Definition {
         final Constructor constructor =
             new Constructor(name, owner, Arrays.asList(genargs != null ? genargs : args), asm, reflect);
 
-        owner.constructors.put(name, constructor);
+        owner.constructors.put(methodKey, constructor);
     }
 
     private final void addMethod(final String struct, final String name, final String alias, final boolean statik,
@@ -1566,32 +1648,34 @@ public final class Definition {
         }
 
         if (!name.matches("^[_a-zA-Z][_a-zA-Z0-9]*$")) {
-            throw new IllegalArgumentException("Invalid " + (statik ? "function" : "method") +
+            throw new IllegalArgumentException("Invalid " + (statik ? "static method" : "method") +
                 " name [" + name + "] with the struct [" + owner.name + "].");
         }
+        
+        MethodKey methodKey = new MethodKey(name, args.length);
 
-        if (owner.constructors.containsKey(name)) {
-            throw new IllegalArgumentException("Constructors and " + (statik ? "functions" : "methods") +
-                " may not have the same name [" + name + "] within the same struct" +
+        if (owner.constructors.containsKey(methodKey)) {
+            throw new IllegalArgumentException("Constructors and " + (statik ? "static methods" : "methods") +
+                " may not have the same signature [" + methodKey + "] within the same struct" +
                 " [" + owner.name + "].");
         }
 
-        if (owner.statics.containsKey(name)) {
+        if (owner.staticMethods.containsKey(methodKey)) {
             if (statik) {
                 throw new IllegalArgumentException(
-                    "Duplicate function name [" + name + "] found within the struct [" + owner.name + "].");
+                    "Duplicate static method signature [" + methodKey + "] found within the struct [" + owner.name + "].");
             } else {
-                throw new IllegalArgumentException("Functions and methods may not have the same name" +
-                    " [" + name + "] within the same struct [" + owner.name + "].");
+                throw new IllegalArgumentException("Static methods and methods may not have the same signature" +
+                    " [" + methodKey + "] within the same struct [" + owner.name + "].");
             }
         }
 
-        if (owner.methods.containsKey(name)) {
+        if (owner.methods.containsKey(methodKey)) {
             if (statik) {
-                throw new IllegalArgumentException("Functions and methods may not have the same name" +
-                    " [" + name + "] within the same struct [" + owner.name + "].");
+                throw new IllegalArgumentException("Static methods and methods may not have the same signature" +
+                    " [" + methodKey + "] within the same struct [" + owner.name + "].");
             } else {
-                throw new IllegalArgumentException("Duplicate method name [" + name + "]" +
+                throw new IllegalArgumentException("Duplicate method signature [" + methodKey + "]" +
                     " found within the struct [" + owner.name + "].");
             }
         }
@@ -1663,14 +1747,14 @@ public final class Definition {
                     " within the struct [" + owner.name + "] is not linked to a static Java method.");
             }
 
-            owner.functions.put(name, method);
+            owner.staticMethods.put(methodKey, method);
         } else {
             if (java.lang.reflect.Modifier.isStatic(modifiers)) {
                 throw new IllegalArgumentException("Method [" + name + "]" +
                     " within the struct [" + owner.name + "] is not linked to a non-static Java method.");
             }
 
-            owner.methods.put(name, method);
+            owner.methods.put(methodKey, method);
         }
     }
 
@@ -1688,7 +1772,7 @@ public final class Definition {
                 " name [" + name + "] with the struct [" + owner.name + "].");
         }
 
-        if (owner.statics.containsKey(name)) {
+        if (owner.staticMembers.containsKey(name)) {
             if (statik) {
                 throw new IllegalArgumentException("Duplicate static name [" + name + "]" +
                     " found within the struct [" + owner.name + "].");
@@ -1751,7 +1835,7 @@ public final class Definition {
                     " within the struct [" + owner.name + "] is not linked to static Java field.");
             }
 
-            owner.statics.put(alias == null ? name : alias, field);
+            owner.staticMembers.put(alias == null ? name : alias, field);
         } else {
             if (java.lang.reflect.Modifier.isStatic(modifiers)) {
                 throw new IllegalArgumentException("Member [" + name + "]" +
@@ -1785,8 +1869,10 @@ public final class Definition {
             final boolean object = child.clazz.equals(Object.class) &&
                 java.lang.reflect.Modifier.isInterface(owner.clazz.getModifiers());
 
-            for (final Method method : child.methods.values()) {
-                if (owner.methods.get(method.name) == null) {
+            for (Map.Entry<MethodKey,Method> kvPair : child.methods.entrySet()) {
+                MethodKey methodKey = kvPair.getKey();
+                Method method = kvPair.getValue();
+                if (owner.methods.get(methodKey) == null) {
                     final Class<?> clazz = object ? Object.class : owner.clazz;
 
                     java.lang.reflect.Method reflect;
@@ -1808,7 +1894,7 @@ public final class Definition {
                             Arrays.toString(method.reflect.getParameterTypes()) + ".");
                     }
 
-                    owner.methods.put(method.name,
+                    owner.methods.put(methodKey,
                         new Method(method.name, owner, method.rtn, method.arguments, method.method, reflect, handle));
                 }
             }
@@ -1866,8 +1952,11 @@ public final class Definition {
         Type upcast = null;
         Type downcast = null;
 
+        // transforms are implicitly arity of 0, unless a static method where its 1 (receiver passed)
+        MethodKey methodKey = new MethodKey(name, statik ? 1 : 0);
+
         if (statik) {
-            method = owner.functions.get(name);
+            method = owner.staticMethods.get(methodKey);
 
             if (method == null) {
                 throw new IllegalArgumentException("Transform with owner struct [" + owner.name + "]" +
@@ -1905,7 +1994,7 @@ public final class Definition {
                 }
             }
         } else {
-            method = owner.methods.get(name);
+            method = owner.methods.get(methodKey);
 
             if (method == null) {
                 throw new IllegalArgumentException("Transform with owner struct [" + owner.name + "]" +
@@ -1950,7 +2039,7 @@ public final class Definition {
      * Precomputes a more efficient structure for dynamic method/field access.
      */
     private void addRuntimeClass(final Struct struct) {
-        final Map<String, Method> methods = struct.methods;
+        final Map<MethodKey, Method> methods = struct.methods;
         final Map<String, MethodHandle> getters = new HashMap<>();
         final Map<String, MethodHandle> setters = new HashMap<>();
 
@@ -1961,8 +2050,8 @@ public final class Definition {
         }
 
         // add all getters/setters
-        for (final Map.Entry<String, Method> method : methods.entrySet()) {
-            final String name = method.getKey();
+        for (final Map.Entry<MethodKey, Method> method : methods.entrySet()) {
+            final String name = method.getKey().name;
             final Method m = method.getValue();
 
             if (m.arguments.size() == 0 &&
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java
new file mode 100644
index 00000000000..bef83daaad3
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java
@@ -0,0 +1,66 @@
+package org.elasticsearch.painless;
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/** Currently just a dummy class for testing a few features not yet exposed by whitelist! */
+public class FeatureTest {
+    private int x;
+    private int y;
+    
+    /** empty ctor */
+    public FeatureTest() {
+    }
+    
+    /** ctor with params */
+    public FeatureTest(int x, int y) {
+        this.x = x;
+        this.y = y;
+    }
+
+    /** getter for x */
+    public int getX() {
+        return x;
+    }
+
+    /** setter for x */
+    public void setX(int x) {
+        this.x = x;
+    }
+
+    /** getter for y */
+    public int getY() {
+        return y;
+    }
+
+    /** setter for y */
+    public void setY(int y) {
+        this.y = y;
+    }
+
+    /** static method that returns true */
+    public static boolean overloadedStatic() {
+        return true;
+    }
+    
+    /** static method that returns what you ask it */
+    public static boolean overloadedStatic(boolean whatToReturn) {
+        return whatToReturn;
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
index 2dd03039ba7..bd76aa293cc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
@@ -55,15 +55,11 @@ public final class LCall extends ALink {
             throw new IllegalArgumentException(error("Cannot assign a value to a call [" + name + "]."));
         }
 
+        Definition.MethodKey methodKey = new Definition.MethodKey(name, arguments.size());
         final Struct struct = before.struct;
-        method = statik ? struct.functions.get(name) : struct.methods.get(name);
+        method = statik ? struct.staticMethods.get(methodKey) : struct.methods.get(methodKey);
 
         if (method != null) {
-            if (method.arguments.size() != arguments.size()) {
-                throw new IllegalArgumentException(error("When calling [" + name + "] on type [" + struct.name + "]" +
-                    " expected [" + method.arguments.size() + "] arguments, but found [" + arguments.size() + "]."));
-            }
-
             for (int argument = 0; argument < arguments.size(); ++argument) {
                 final AExpression expression = arguments.get(argument);
 
@@ -83,7 +79,8 @@ public final class LCall extends ALink {
             return link.analyze(settings, definition, variables);
         }
 
-        throw new IllegalArgumentException(error("Unknown call [" + name + "] on type [" + struct.name + "]."));
+        throw new IllegalArgumentException(error("Unknown call [" + name + "] with [" + arguments.size() + 
+                                                 "] arguments on type [" + struct.name + "]."));
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
index bd0079d1843..06f820eba26 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
@@ -60,7 +60,7 @@ public final class LField extends ALink {
         }
 
         final Struct struct = before.struct;
-        field = statik ? struct.statics.get(value) : struct.members.get(value);
+        field = statik ? struct.staticMembers.get(value) : struct.members.get(value);
 
         if (field != null) {
             if (store && java.lang.reflect.Modifier.isFinal(field.reflect.getModifiers())) {
@@ -72,9 +72,12 @@ public final class LField extends ALink {
 
             return this;
         } else {
+            // TODO: improve this: the isXXX case seems missing???
             final boolean shortcut =
-                struct.methods.containsKey("get" + Character.toUpperCase(value.charAt(0)) + value.substring(1)) ||
-                struct.methods.containsKey("set" + Character.toUpperCase(value.charAt(0)) + value.substring(1));
+                struct.methods.containsKey(new Definition.MethodKey("get" + 
+                        Character.toUpperCase(value.charAt(0)) + value.substring(1), 0)) ||
+                struct.methods.containsKey(new Definition.MethodKey("set" + 
+                        Character.toUpperCase(value.charAt(0)) + value.substring(1), 1));
 
             if (shortcut) {
                 return new LShortcut(line, location, value).copy(this).analyze(settings, definition, variables);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
index 00e44967279..8526ef1297e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
@@ -43,8 +43,8 @@ final class LListShortcut extends ALink {
 
     @Override
     ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        getter = before.struct.methods.get("get");
-        setter = before.struct.methods.get("set");
+        getter = before.struct.methods.get(new Definition.MethodKey("get", 1));
+        setter = before.struct.methods.get(new Definition.MethodKey("set", 2));
 
         if (getter != null && (getter.rtn.sort == Sort.VOID || getter.arguments.size() != 1 ||
             getter.arguments.get(0).sort != Sort.INT)) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
index a3d9d119eda..4efbd4bdf0f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
@@ -43,8 +43,8 @@ final class LMapShortcut extends ALink {
 
     @Override
     ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        getter = before.struct.methods.get("get");
-        setter = before.struct.methods.get("put");
+        getter = before.struct.methods.get(new Definition.MethodKey("get", 1));
+        setter = before.struct.methods.get(new Definition.MethodKey("put", 2));
 
         if (getter != null && (getter.rtn.sort == Sort.VOID || getter.arguments.size() != 1)) {
             throw new IllegalArgumentException(error("Illegal map get shortcut for type [" + before.name + "]."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
index fb91b7a6915..227b63cf31f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
@@ -63,7 +63,7 @@ public final class LNewObj extends ALink {
         }
 
         final Struct struct = type.struct;
-        constructor = struct.constructors.get("new");
+        constructor = struct.constructors.get(new Definition.MethodKey("new", arguments.size()));
 
         if (constructor != null) {
             final Type[] types = new Type[constructor.arguments.size()];
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
index 5f18cccd512..c65077e6e28 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
@@ -47,8 +47,8 @@ final class LShortcut extends ALink {
     ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
         final Struct struct = before.struct;
 
-        getter = struct.methods.get("get" + Character.toUpperCase(value.charAt(0)) + value.substring(1));
-        setter = struct.methods.get("set" + Character.toUpperCase(value.charAt(0)) + value.substring(1));
+        getter = struct.methods.get(new Definition.MethodKey("get" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0));
+        setter = struct.methods.get(new Definition.MethodKey("set" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 1));
 
         if (getter != null && (getter.rtn.sort == Sort.VOID || !getter.arguments.isEmpty())) {
             throw new IllegalArgumentException(error(
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java
new file mode 100644
index 00000000000..3e0e4a5fc71
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java
@@ -0,0 +1,52 @@
+package org.elasticsearch.painless;
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/** Tests method overloading */
+public class OverloadTests extends ScriptTestCase {
+
+    public void testMethod() {
+        assertEquals(2, exec("return 'abc123abc'.indexOf('c');"));
+        assertEquals(8, exec("return 'abc123abc'.indexOf('c', 3);"));
+        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+            exec("return 'abc123abc'.indexOf('c', 3, 'bogus');");
+        });
+        assertTrue(expected.getMessage().contains("[indexOf] with [3] arguments"));
+    }
+    
+    public void testMethodDynamic() {
+        assertEquals(2, exec("def x = 'abc123abc'; return x.indexOf('c');"));
+        assertEquals(8, exec("def x = 'abc123abc'; return x.indexOf('c', 3);"));
+        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+            exec("def x = 'abc123abc'; return x.indexOf('c', 3, 'bogus');");
+        });
+        assertTrue(expected.getMessage().contains("dynamic method [indexOf] with signature [(String,int,String)"));
+    }
+    
+    public void testConstructor() {
+        assertEquals(true, exec("FeatureTest f = new FeatureTest(); return f.x == 0 && f.y == 0;"));
+        assertEquals(true, exec("FeatureTest f = new FeatureTest(1, 2); return f.x == 1 && f.y == 2;"));
+    }
+    
+    public void testStatic() {
+        assertEquals(true, exec("return FeatureTest.overloadedStatic();"));
+        assertEquals(false, exec("return FeatureTest.overloadedStatic(false);"));
+    }
+}

From 8c9ca8b5187e0d747104ce8097f1ff69f12d90d3 Mon Sep 17 00:00:00 2001
From: Colin Goodheart-Smithe <colings86@users.noreply.github.com>
Date: Tue, 17 May 2016 10:45:34 +0100
Subject: [PATCH 0288/1311] Moves query profiler classes into their own package

The change also renames fields and methods in the Profilers class.

Note that I had to make ProfileResult a public class (it was package private before) because now classes that call it are in a different package.
---
 .../search/aggregations/AggregationPhase.java |  6 +++---
 .../search/internal/ContextIndexSearcher.java |  8 ++++----
 .../search/profile/ProfileResult.java         |  2 +-
 .../search/profile/ProfileShardResult.java    |  1 +
 .../search/profile/Profilers.java             | 19 ++++++++++---------
 .../profile/SearchProfileShardResults.java    |  1 +
 .../profile/{ => query}/CollectorResult.java  |  2 +-
 .../{ => query}/InternalProfileCollector.java |  2 +-
 .../{ => query}/InternalQueryProfileTree.java |  3 ++-
 .../profile/{ => query}/ProfileCollector.java |  2 +-
 .../profile/{ => query}/ProfileScorer.java    |  2 +-
 .../profile/{ => query}/ProfileWeight.java    |  2 +-
 .../{ => query}/QueryProfileBreakdown.java    |  4 +++-
 .../profile/{ => query}/QueryProfiler.java    |  3 ++-
 .../profile/{ => query}/QueryTimingType.java  |  2 +-
 .../search/query/QueryPhase.java              | 10 +++++-----
 .../profile/{ => query}/ProfileTests.java     |  5 ++++-
 .../profile/{ => query}/QueryProfilerIT.java  | 16 +++++++++++-----
 .../{ => query}/RandomQueryGenerator.java     |  2 +-
 19 files changed, 54 insertions(+), 38 deletions(-)
 rename core/src/main/java/org/elasticsearch/search/profile/{ => query}/CollectorResult.java (99%)
 rename core/src/main/java/org/elasticsearch/search/profile/{ => query}/InternalProfileCollector.java (98%)
 rename core/src/main/java/org/elasticsearch/search/profile/{ => query}/InternalQueryProfileTree.java (98%)
 rename core/src/main/java/org/elasticsearch/search/profile/{ => query}/ProfileCollector.java (98%)
 rename core/src/main/java/org/elasticsearch/search/profile/{ => query}/ProfileScorer.java (99%)
 rename core/src/main/java/org/elasticsearch/search/profile/{ => query}/ProfileWeight.java (98%)
 rename core/src/main/java/org/elasticsearch/search/profile/{ => query}/QueryProfileBreakdown.java (91%)
 rename core/src/main/java/org/elasticsearch/search/profile/{ => query}/QueryProfiler.java (97%)
 rename core/src/main/java/org/elasticsearch/search/profile/{ => query}/QueryTimingType.java (95%)
 rename core/src/test/java/org/elasticsearch/search/profile/{ => query}/ProfileTests.java (97%)
 rename core/src/test/java/org/elasticsearch/search/profile/{ => query}/QueryProfilerIT.java (97%)
 rename core/src/test/java/org/elasticsearch/search/profile/{ => query}/RandomQueryGenerator.java (99%)

diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
index 4953a39d4b5..f3ad47910e4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationPhase.java
@@ -30,8 +30,8 @@ import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.SiblingPipelineAggregator;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.profile.CollectorResult;
-import org.elasticsearch.search.profile.InternalProfileCollector;
+import org.elasticsearch.search.profile.query.CollectorResult;
+import org.elasticsearch.search.profile.query.InternalProfileCollector;
 import org.elasticsearch.search.query.QueryPhaseExecutionException;
 
 import java.io.IOException;
@@ -125,7 +125,7 @@ public class AggregationPhase implements SearchPhase {
                             Collections.emptyList());
                     collector = profileCollector;
                     // start a new profile with this collector
-                    context.getProfilers().addProfiler().setCollector(profileCollector);
+                    context.getProfilers().addQueryProfiler().setCollector(profileCollector);
                 }
                 globalsCollector.preCollection();
                 context.searcher().search(query, collector);
diff --git a/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
index 471ce26d297..df1007ebc71 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
@@ -33,10 +33,10 @@ import org.apache.lucene.search.Weight;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.search.dfs.AggregatedDfs;
-import org.elasticsearch.search.profile.QueryProfileBreakdown;
-import org.elasticsearch.search.profile.ProfileWeight;
-import org.elasticsearch.search.profile.QueryProfiler;
-import org.elasticsearch.search.profile.QueryTimingType;
+import org.elasticsearch.search.profile.query.ProfileWeight;
+import org.elasticsearch.search.profile.query.QueryProfileBreakdown;
+import org.elasticsearch.search.profile.query.QueryProfiler;
+import org.elasticsearch.search.profile.query.QueryTimingType;
 
 import java.io.IOException;
 
diff --git a/core/src/main/java/org/elasticsearch/search/profile/ProfileResult.java b/core/src/main/java/org/elasticsearch/search/profile/ProfileResult.java
index 165244e8d69..4c745a5127d 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/ProfileResult.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/ProfileResult.java
@@ -43,7 +43,7 @@ import java.util.Map;
  * Each InternalProfileResult has a List of InternalProfileResults, which will contain
  * "children" queries if applicable
  */
-final class ProfileResult implements Writeable, ToXContent {
+public final class ProfileResult implements Writeable, ToXContent {
 
     private static final ParseField TYPE = new ParseField("type");
     private static final ParseField DESCRIPTION = new ParseField("description");
diff --git a/core/src/main/java/org/elasticsearch/search/profile/ProfileShardResult.java b/core/src/main/java/org/elasticsearch/search/profile/ProfileShardResult.java
index e35a3c2e2aa..9def3db7582 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/ProfileShardResult.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/ProfileShardResult.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.profile.query.CollectorResult;
 
 import java.io.IOException;
 import java.util.ArrayList;
diff --git a/core/src/main/java/org/elasticsearch/search/profile/Profilers.java b/core/src/main/java/org/elasticsearch/search/profile/Profilers.java
index 38b69d38b15..e9e6d88db18 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/Profilers.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/Profilers.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.search.profile;
 
 import org.elasticsearch.search.internal.ContextIndexSearcher;
+import org.elasticsearch.search.profile.query.QueryProfiler;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -29,31 +30,31 @@ import java.util.List;
 public final class Profilers {
 
     private final ContextIndexSearcher searcher;
-    private final List<QueryProfiler> profilers;
+    private final List<QueryProfiler> queryProfilers;
 
     /** Sole constructor. This {@link Profilers} instance will initially wrap one {@link QueryProfiler}. */
     public Profilers(ContextIndexSearcher searcher) {
         this.searcher = searcher;
-        this.profilers = new ArrayList<>();
-        addProfiler();
+        this.queryProfilers = new ArrayList<>();
+        addQueryProfiler();
     }
 
     /** Switch to a new profile. */
-    public QueryProfiler addProfiler() {
+    public QueryProfiler addQueryProfiler() {
         QueryProfiler profiler = new QueryProfiler();
         searcher.setProfiler(profiler);
-        profilers.add(profiler);
+        queryProfilers.add(profiler);
         return profiler;
     }
 
     /** Get the current profiler. */
-    public QueryProfiler getCurrent() {
-        return profilers.get(profilers.size() - 1);
+    public QueryProfiler getCurrentQueryProfiler() {
+        return queryProfilers.get(queryProfilers.size() - 1);
     }
 
     /** Return the list of all created {@link QueryProfiler}s so far. */
-    public List<QueryProfiler> getProfilers() {
-        return Collections.unmodifiableList(profilers);
+    public List<QueryProfiler> getQueryProfilers() {
+        return Collections.unmodifiableList(queryProfilers);
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResults.java b/core/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResults.java
index a38c0813dc5..bf265dd9a7e 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResults.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResults.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.profile.query.QueryProfiler;
 
 import java.io.IOException;
 import java.util.ArrayList;
diff --git a/core/src/main/java/org/elasticsearch/search/profile/CollectorResult.java b/core/src/main/java/org/elasticsearch/search/profile/query/CollectorResult.java
similarity index 99%
rename from core/src/main/java/org/elasticsearch/search/profile/CollectorResult.java
rename to core/src/main/java/org/elasticsearch/search/profile/query/CollectorResult.java
index 32b9ccfa942..c517c8730e4 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/CollectorResult.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/query/CollectorResult.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.search.profile;
+package org.elasticsearch.search.profile.query;
 
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
diff --git a/core/src/main/java/org/elasticsearch/search/profile/InternalProfileCollector.java b/core/src/main/java/org/elasticsearch/search/profile/query/InternalProfileCollector.java
similarity index 98%
rename from core/src/main/java/org/elasticsearch/search/profile/InternalProfileCollector.java
rename to core/src/main/java/org/elasticsearch/search/profile/query/InternalProfileCollector.java
index 48d165e3095..e892abaab22 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/InternalProfileCollector.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/query/InternalProfileCollector.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.search.profile;
+package org.elasticsearch.search.profile.query;
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.Collector;
diff --git a/core/src/main/java/org/elasticsearch/search/profile/InternalQueryProfileTree.java b/core/src/main/java/org/elasticsearch/search/profile/query/InternalQueryProfileTree.java
similarity index 98%
rename from core/src/main/java/org/elasticsearch/search/profile/InternalQueryProfileTree.java
rename to core/src/main/java/org/elasticsearch/search/profile/query/InternalQueryProfileTree.java
index 9ebee2f1635..5b92ef8b2a9 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/InternalQueryProfileTree.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/query/InternalQueryProfileTree.java
@@ -17,9 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.search.profile;
+package org.elasticsearch.search.profile.query;
 
 import org.apache.lucene.search.Query;
+import org.elasticsearch.search.profile.ProfileResult;
 
 import java.util.ArrayList;
 import java.util.Collections;
diff --git a/core/src/main/java/org/elasticsearch/search/profile/ProfileCollector.java b/core/src/main/java/org/elasticsearch/search/profile/query/ProfileCollector.java
similarity index 98%
rename from core/src/main/java/org/elasticsearch/search/profile/ProfileCollector.java
rename to core/src/main/java/org/elasticsearch/search/profile/query/ProfileCollector.java
index 7d7538c9117..c65d0978906 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/ProfileCollector.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/query/ProfileCollector.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.search.profile;
+package org.elasticsearch.search.profile.query;
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.Collector;
diff --git a/core/src/main/java/org/elasticsearch/search/profile/ProfileScorer.java b/core/src/main/java/org/elasticsearch/search/profile/query/ProfileScorer.java
similarity index 99%
rename from core/src/main/java/org/elasticsearch/search/profile/ProfileScorer.java
rename to core/src/main/java/org/elasticsearch/search/profile/query/ProfileScorer.java
index 939ac85c871..51d0b14fc96 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/ProfileScorer.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/query/ProfileScorer.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.search.profile;
+package org.elasticsearch.search.profile.query;
 
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Scorer;
diff --git a/core/src/main/java/org/elasticsearch/search/profile/ProfileWeight.java b/core/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java
similarity index 98%
rename from core/src/main/java/org/elasticsearch/search/profile/ProfileWeight.java
rename to core/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java
index 9546f691616..9ca33f84542 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/ProfileWeight.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/query/ProfileWeight.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.search.profile;
+package org.elasticsearch.search.profile.query;
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Term;
diff --git a/core/src/main/java/org/elasticsearch/search/profile/QueryProfileBreakdown.java b/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfileBreakdown.java
similarity index 91%
rename from core/src/main/java/org/elasticsearch/search/profile/QueryProfileBreakdown.java
rename to core/src/main/java/org/elasticsearch/search/profile/query/QueryProfileBreakdown.java
index fdb13a49977..d0608eb01af 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/QueryProfileBreakdown.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfileBreakdown.java
@@ -17,7 +17,9 @@
  * under the License.
  */
 
-package org.elasticsearch.search.profile;
+package org.elasticsearch.search.profile.query;
+
+import org.elasticsearch.search.profile.AbstractProfileBreakdown;
 
 /**
  * A record of timings for the various operations that may happen during query execution.
diff --git a/core/src/main/java/org/elasticsearch/search/profile/QueryProfiler.java b/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfiler.java
similarity index 97%
rename from core/src/main/java/org/elasticsearch/search/profile/QueryProfiler.java
rename to core/src/main/java/org/elasticsearch/search/profile/query/QueryProfiler.java
index f8fcf5fbf13..57341ee132f 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/QueryProfiler.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfiler.java
@@ -17,9 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.search.profile;
+package org.elasticsearch.search.profile.query;
 
 import org.apache.lucene.search.Query;
+import org.elasticsearch.search.profile.ProfileResult;
 
 import java.util.List;
 import java.util.Objects;
diff --git a/core/src/main/java/org/elasticsearch/search/profile/QueryTimingType.java b/core/src/main/java/org/elasticsearch/search/profile/query/QueryTimingType.java
similarity index 95%
rename from core/src/main/java/org/elasticsearch/search/profile/QueryTimingType.java
rename to core/src/main/java/org/elasticsearch/search/profile/query/QueryTimingType.java
index 5fb3b29dee3..5f194a7d5f1 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/QueryTimingType.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/query/QueryTimingType.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.search.profile;
+package org.elasticsearch.search.profile.query;
 
 import java.util.Locale;
 
diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index da849828e7a..df68064f617 100644
--- a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -52,10 +52,10 @@ import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.aggregations.AggregationPhase;
 import org.elasticsearch.search.internal.ScrollContext;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.profile.CollectorResult;
-import org.elasticsearch.search.profile.InternalProfileCollector;
 import org.elasticsearch.search.profile.ProfileShardResult;
 import org.elasticsearch.search.profile.SearchProfileShardResults;
+import org.elasticsearch.search.profile.query.CollectorResult;
+import org.elasticsearch.search.profile.query.InternalProfileCollector;
 import org.elasticsearch.search.rescore.RescorePhase;
 import org.elasticsearch.search.rescore.RescoreSearchContext;
 import org.elasticsearch.search.sort.SortAndFormats;
@@ -113,7 +113,7 @@ public class QueryPhase implements SearchPhase {
 
         if (searchContext.getProfilers() != null) {
             List<ProfileShardResult> shardResults = SearchProfileShardResults
-                    .buildShardResults(searchContext.getProfilers().getProfilers());
+                    .buildShardResults(searchContext.getProfilers().getQueryProfilers());
             searchContext.queryResult().profileResults(shardResults);
         }
     }
@@ -365,7 +365,7 @@ public class QueryPhase implements SearchPhase {
             try {
                 if (collector != null) {
                     if (doProfile) {
-                        searchContext.getProfilers().getCurrent().setCollector((InternalProfileCollector) collector);
+                        searchContext.getProfilers().getCurrentQueryProfiler().setCollector((InternalProfileCollector) collector);
                     }
                     searcher.search(query, collector);
                 }
@@ -386,7 +386,7 @@ public class QueryPhase implements SearchPhase {
 
             if (searchContext.getProfilers() != null) {
                 List<ProfileShardResult> shardResults = SearchProfileShardResults
-                        .buildShardResults(searchContext.getProfilers().getProfilers());
+                        .buildShardResults(searchContext.getProfilers().getQueryProfilers());
                 searchContext.queryResult().profileResults(shardResults);
             }
 
diff --git a/core/src/test/java/org/elasticsearch/search/profile/ProfileTests.java b/core/src/test/java/org/elasticsearch/search/profile/query/ProfileTests.java
similarity index 97%
rename from core/src/test/java/org/elasticsearch/search/profile/ProfileTests.java
rename to core/src/test/java/org/elasticsearch/search/profile/query/ProfileTests.java
index ebae8e64f32..7488dbceee7 100644
--- a/core/src/test/java/org/elasticsearch/search/profile/ProfileTests.java
+++ b/core/src/test/java/org/elasticsearch/search/profile/query/ProfileTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.search.profile;
+package org.elasticsearch.search.profile.query;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field.Store;
@@ -37,6 +37,9 @@ import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.TestUtil;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.search.internal.ContextIndexSearcher;
+import org.elasticsearch.search.profile.ProfileResult;
+import org.elasticsearch.search.profile.query.QueryProfiler;
+import org.elasticsearch.search.profile.query.QueryTimingType;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
diff --git a/core/src/test/java/org/elasticsearch/search/profile/QueryProfilerIT.java b/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java
similarity index 97%
rename from core/src/test/java/org/elasticsearch/search/profile/QueryProfilerIT.java
rename to core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java
index d9878774576..371aaadd3a7 100644
--- a/core/src/test/java/org/elasticsearch/search/profile/QueryProfilerIT.java
+++ b/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.search.profile;
+package org.elasticsearch.search.profile.query;
 
 import org.apache.lucene.util.English;
 import org.elasticsearch.action.index.IndexRequestBuilder;
@@ -29,6 +29,9 @@ import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.profile.ProfileResult;
+import org.elasticsearch.search.profile.ProfileShardResult;
+import org.elasticsearch.search.profile.query.CollectorResult;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -36,7 +39,7 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.search.profile.RandomQueryGenerator.randomQueryBuilder;
+import static org.elasticsearch.search.profile.query.RandomQueryGenerator.randomQueryBuilder;
 import static org.elasticsearch.test.hamcrest.DoubleMatcher.nearlyEqual;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -160,7 +163,8 @@ public class QueryProfilerIT extends ESIntegTestCase {
                         nearlyEqual(vanillaMaxScore, profileMaxScore, 0.001));
             }
 
-            assertThat("Profile totalHits of [" + profileResponse.getHits().totalHits() + "] is not close to Vanilla totalHits [" + vanillaResponse.getHits().totalHits() + "]",
+            assertThat("Profile totalHits of [" + profileResponse.getHits().totalHits() + "] is not close to Vanilla totalHits ["
+                    + vanillaResponse.getHits().totalHits() + "]",
                     vanillaResponse.getHits().getTotalHits(), equalTo(profileResponse.getHits().getTotalHits()));
 
             SearchHit[] vanillaHits = vanillaResponse.getHits().getHits();
@@ -237,7 +241,8 @@ public class QueryProfilerIT extends ESIntegTestCase {
 
         indexRandom(true, docs);
 
-        QueryBuilder q = QueryBuilders.boolQuery().must(QueryBuilders.matchQuery("field1", "one")).must(QueryBuilders.matchQuery("field1", "two"));
+        QueryBuilder q = QueryBuilders.boolQuery().must(QueryBuilders.matchQuery("field1", "one"))
+                .must(QueryBuilders.matchQuery("field1", "two"));
 
         SearchResponse resp = client().prepareSearch()
                 .setQuery(q)
@@ -355,7 +360,8 @@ public class QueryProfilerIT extends ESIntegTestCase {
 
         refresh();
 
-        QueryBuilder q = QueryBuilders.boolQuery().must(QueryBuilders.boolQuery().must(QueryBuilders.boolQuery().must(QueryBuilders.matchQuery("field1", "one"))));
+        QueryBuilder q = QueryBuilders.boolQuery()
+                .must(QueryBuilders.boolQuery().must(QueryBuilders.boolQuery().must(QueryBuilders.matchQuery("field1", "one"))));
 
         logger.info("Query: {}", q);
 
diff --git a/core/src/test/java/org/elasticsearch/search/profile/RandomQueryGenerator.java b/core/src/test/java/org/elasticsearch/search/profile/query/RandomQueryGenerator.java
similarity index 99%
rename from core/src/test/java/org/elasticsearch/search/profile/RandomQueryGenerator.java
rename to core/src/test/java/org/elasticsearch/search/profile/query/RandomQueryGenerator.java
index fff0b99c4d1..7fb896e1b2f 100644
--- a/core/src/test/java/org/elasticsearch/search/profile/RandomQueryGenerator.java
+++ b/core/src/test/java/org/elasticsearch/search/profile/query/RandomQueryGenerator.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.search.profile;
+package org.elasticsearch.search.profile.query;
 
 import org.apache.lucene.util.English;
 import org.elasticsearch.common.unit.Fuzziness;

From 9922931144ba319069d06c8bc4487f281db0a019 Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Tue, 17 May 2016 15:40:53 +0200
Subject: [PATCH 0289/1311] Fix occasional build error.

---
 docs/reference/query-dsl/parent-id-query.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/query-dsl/parent-id-query.asciidoc b/docs/reference/query-dsl/parent-id-query.asciidoc
index 10cf6d05664..2e095b4295c 100644
--- a/docs/reference/query-dsl/parent-id-query.asciidoc
+++ b/docs/reference/query-dsl/parent-id-query.asciidoc
@@ -34,7 +34,7 @@ PUT /my_index
 
 [source,js]
 --------------------------------------------------
-GET /_search
+GET /my_index/_search
 {
     "query": {
         "parent_id" : {

From fa5f099486f2449d22fb2fd744ec3c2b1720768d Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 17 May 2016 10:55:40 -0400
Subject: [PATCH 0290/1311] make semicolons only optional at the end

---
 .../src/main/antlr/PainlessParser.g4          |   14 +-
 .../painless/antlr/PainlessParser.java        |  828 +++++----
 .../painless/DefOperationTests.java           | 1484 ++++++++---------
 .../painless/NoSemiColonTests.java            |  183 +-
 .../elasticsearch/painless/StringTests.java   |   33 +-
 .../painless/WhenThingsGoWrongTests.java      |    6 +-
 6 files changed, 1226 insertions(+), 1322 deletions(-)

diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index e8aa3592c59..4d9a81e25a0 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -28,15 +28,15 @@ source
 statement
     : IF LP expression RP block ( ELSE block )?                                              # if
     | WHILE LP expression RP ( block | empty )                                               # while
-    | DO block WHILE LP expression RP SEMICOLON?                                             # do
+    | DO block WHILE LP expression RP ( SEMICOLON | EOF )                                    # do
     | FOR LP initializer? SEMICOLON expression? SEMICOLON afterthought? RP ( block | empty ) # for
-    | declaration SEMICOLON?                                                                 # decl
-    | CONTINUE SEMICOLON?                                                                    # continue
-    | BREAK SEMICOLON?                                                                       # break
-    | RETURN expression SEMICOLON?                                                           # return
+    | declaration ( SEMICOLON | EOF )                                                        # decl
+    | CONTINUE ( SEMICOLON | EOF )                                                           # continue
+    | BREAK ( SEMICOLON | EOF )                                                              # break
+    | RETURN expression ( SEMICOLON | EOF )                                                  # return
     | TRY block trap+                                                                        # try
-    | THROW expression SEMICOLON?                                                            # throw
-    | expression SEMICOLON?                                                                  # expr
+    | THROW expression ( SEMICOLON | EOF )                                                   # throw
+    | expression ( SEMICOLON | EOF )                                                         # expr
     ;
 
 block
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
index 9d7ae5c628d..c8e76ee4bd9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
@@ -182,6 +182,7 @@ class PainlessParser extends Parser {
       return getRuleContext(DeclarationContext.class,0);
     }
     public TerminalNode SEMICOLON() { return getToken(PainlessParser.SEMICOLON, 0); }
+    public TerminalNode EOF() { return getToken(PainlessParser.EOF, 0); }
     public DeclContext(StatementContext ctx) { copyFrom(ctx); }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
@@ -192,6 +193,7 @@ class PainlessParser extends Parser {
   public static class BreakContext extends StatementContext {
     public TerminalNode BREAK() { return getToken(PainlessParser.BREAK, 0); }
     public TerminalNode SEMICOLON() { return getToken(PainlessParser.SEMICOLON, 0); }
+    public TerminalNode EOF() { return getToken(PainlessParser.EOF, 0); }
     public BreakContext(StatementContext ctx) { copyFrom(ctx); }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
@@ -205,6 +207,7 @@ class PainlessParser extends Parser {
       return getRuleContext(ExpressionContext.class,0);
     }
     public TerminalNode SEMICOLON() { return getToken(PainlessParser.SEMICOLON, 0); }
+    public TerminalNode EOF() { return getToken(PainlessParser.EOF, 0); }
     public ThrowContext(StatementContext ctx) { copyFrom(ctx); }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
@@ -215,6 +218,7 @@ class PainlessParser extends Parser {
   public static class ContinueContext extends StatementContext {
     public TerminalNode CONTINUE() { return getToken(PainlessParser.CONTINUE, 0); }
     public TerminalNode SEMICOLON() { return getToken(PainlessParser.SEMICOLON, 0); }
+    public TerminalNode EOF() { return getToken(PainlessParser.EOF, 0); }
     public ContinueContext(StatementContext ctx) { copyFrom(ctx); }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
@@ -275,6 +279,7 @@ class PainlessParser extends Parser {
       return getRuleContext(ExpressionContext.class,0);
     }
     public TerminalNode SEMICOLON() { return getToken(PainlessParser.SEMICOLON, 0); }
+    public TerminalNode EOF() { return getToken(PainlessParser.EOF, 0); }
     public ExprContext(StatementContext ctx) { copyFrom(ctx); }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
@@ -294,6 +299,7 @@ class PainlessParser extends Parser {
     }
     public TerminalNode RP() { return getToken(PainlessParser.RP, 0); }
     public TerminalNode SEMICOLON() { return getToken(PainlessParser.SEMICOLON, 0); }
+    public TerminalNode EOF() { return getToken(PainlessParser.EOF, 0); }
     public DoContext(StatementContext ctx) { copyFrom(ctx); }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
@@ -348,6 +354,7 @@ class PainlessParser extends Parser {
       return getRuleContext(ExpressionContext.class,0);
     }
     public TerminalNode SEMICOLON() { return getToken(PainlessParser.SEMICOLON, 0); }
+    public TerminalNode EOF() { return getToken(PainlessParser.EOF, 0); }
     public ReturnContext(StatementContext ctx) { copyFrom(ctx); }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
@@ -362,8 +369,8 @@ class PainlessParser extends Parser {
     int _la;
     try {
       int _alt;
-      setState(134);
-      switch ( getInterpreter().adaptivePredict(_input,15,_ctx) ) {
+      setState(125);
+      switch ( getInterpreter().adaptivePredict(_input,8,_ctx) ) {
       case 1:
         _localctx = new IfContext(_localctx);
         enterOuterAlt(_localctx, 1);
@@ -436,69 +443,67 @@ class PainlessParser extends Parser {
         expression(0);
         setState(79);
         match(RP);
-        setState(81);
+        setState(80);
         _la = _input.LA(1);
-        if (_la==SEMICOLON) {
-          {
-          setState(80);
-          match(SEMICOLON);
-          }
+        if ( !(_la==EOF || _la==SEMICOLON) ) {
+        _errHandler.recoverInline(this);
+        } else {
+          consume();
         }
-
         }
         break;
       case 4:
         _localctx = new ForContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(83);
+        setState(82);
         match(FOR);
-        setState(84);
+        setState(83);
         match(LP);
-        setState(86);
+        setState(85);
         _la = _input.LA(1);
         if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0)) {
           {
-          setState(85);
+          setState(84);
           initializer();
           }
         }
 
-        setState(88);
+        setState(87);
         match(SEMICOLON);
-        setState(90);
+        setState(89);
         _la = _input.LA(1);
         if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0)) {
           {
-          setState(89);
+          setState(88);
           expression(0);
           }
         }
 
-        setState(92);
+        setState(91);
         match(SEMICOLON);
-        setState(94);
+        setState(93);
         _la = _input.LA(1);
         if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0)) {
           {
-          setState(93);
+          setState(92);
           afterthought();
           }
         }
 
-        setState(96);
+        setState(95);
         match(RP);
-        setState(99);
-        switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) {
+        setState(98);
+        switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) {
         case 1:
           {
-          setState(97);
+          setState(96);
           block();
           }
           break;
         case 2:
           {
-          setState(98);
+          setState(97);
           empty();
           }
           break;
@@ -509,81 +514,73 @@ class PainlessParser extends Parser {
         _localctx = new DeclContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(101);
+        setState(100);
         declaration();
-        setState(103);
+        setState(101);
         _la = _input.LA(1);
-        if (_la==SEMICOLON) {
-          {
-          setState(102);
-          match(SEMICOLON);
-          }
+        if ( !(_la==EOF || _la==SEMICOLON) ) {
+        _errHandler.recoverInline(this);
+        } else {
+          consume();
         }
-
         }
         break;
       case 6:
         _localctx = new ContinueContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(105);
+        setState(103);
         match(CONTINUE);
-        setState(107);
+        setState(104);
         _la = _input.LA(1);
-        if (_la==SEMICOLON) {
-          {
-          setState(106);
-          match(SEMICOLON);
-          }
+        if ( !(_la==EOF || _la==SEMICOLON) ) {
+        _errHandler.recoverInline(this);
+        } else {
+          consume();
         }
-
         }
         break;
       case 7:
         _localctx = new BreakContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(109);
+        setState(105);
         match(BREAK);
-        setState(111);
+        setState(106);
         _la = _input.LA(1);
-        if (_la==SEMICOLON) {
-          {
-          setState(110);
-          match(SEMICOLON);
-          }
+        if ( !(_la==EOF || _la==SEMICOLON) ) {
+        _errHandler.recoverInline(this);
+        } else {
+          consume();
         }
-
         }
         break;
       case 8:
         _localctx = new ReturnContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(113);
+        setState(107);
         match(RETURN);
-        setState(114);
+        setState(108);
         expression(0);
-        setState(116);
+        setState(109);
         _la = _input.LA(1);
-        if (_la==SEMICOLON) {
-          {
-          setState(115);
-          match(SEMICOLON);
-          }
+        if ( !(_la==EOF || _la==SEMICOLON) ) {
+        _errHandler.recoverInline(this);
+        } else {
+          consume();
         }
-
         }
         break;
       case 9:
         _localctx = new TryContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(118);
+        setState(111);
         match(TRY);
-        setState(119);
+        setState(112);
         block();
-        setState(121); 
+        setState(114); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -591,7 +588,7 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(120);
+            setState(113);
             trap();
             }
             }
@@ -599,9 +596,9 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(123); 
+          setState(116); 
           _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,12,_ctx);
+          _alt = getInterpreter().adaptivePredict(_input,7,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
         }
         break;
@@ -609,36 +606,32 @@ class PainlessParser extends Parser {
         _localctx = new ThrowContext(_localctx);
         enterOuterAlt(_localctx, 10);
         {
-        setState(125);
+        setState(118);
         match(THROW);
-        setState(126);
+        setState(119);
         expression(0);
-        setState(128);
+        setState(120);
         _la = _input.LA(1);
-        if (_la==SEMICOLON) {
-          {
-          setState(127);
-          match(SEMICOLON);
-          }
+        if ( !(_la==EOF || _la==SEMICOLON) ) {
+        _errHandler.recoverInline(this);
+        } else {
+          consume();
         }
-
         }
         break;
       case 11:
         _localctx = new ExprContext(_localctx);
         enterOuterAlt(_localctx, 11);
         {
-        setState(130);
+        setState(122);
         expression(0);
-        setState(132);
+        setState(123);
         _la = _input.LA(1);
-        if (_la==SEMICOLON) {
-          {
-          setState(131);
-          match(SEMICOLON);
-          }
+        if ( !(_la==EOF || _la==SEMICOLON) ) {
+        _errHandler.recoverInline(this);
+        } else {
+          consume();
         }
-
         }
         break;
       }
@@ -698,29 +691,29 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 4, RULE_block);
     int _la;
     try {
-      setState(145);
+      setState(136);
       switch (_input.LA(1)) {
       case LBRACK:
         _localctx = new MultipleContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(136);
+        setState(127);
         match(LBRACK);
-        setState(138); 
+        setState(129); 
         _errHandler.sync(this);
         _la = _input.LA(1);
         do {
           {
           {
-          setState(137);
+          setState(128);
           statement();
           }
           }
-          setState(140); 
+          setState(131); 
           _errHandler.sync(this);
           _la = _input.LA(1);
         } while ( (((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << IF) | (1L << WHILE) | (1L << DO) | (1L << FOR) | (1L << CONTINUE) | (1L << BREAK) | (1L << RETURN) | (1L << NEW) | (1L << TRY) | (1L << THROW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0) );
-        setState(142);
+        setState(133);
         match(RBRACK);
         }
         break;
@@ -753,7 +746,7 @@ class PainlessParser extends Parser {
         _localctx = new SingleContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(144);
+        setState(135);
         statement();
         }
         break;
@@ -792,19 +785,19 @@ class PainlessParser extends Parser {
     EmptyContext _localctx = new EmptyContext(_ctx, getState());
     enterRule(_localctx, 6, RULE_empty);
     try {
-      setState(149);
+      setState(140);
       switch (_input.LA(1)) {
       case LBRACK:
         enterOuterAlt(_localctx, 1);
         {
-        setState(147);
+        setState(138);
         emptyscope();
         }
         break;
       case SEMICOLON:
         enterOuterAlt(_localctx, 2);
         {
-        setState(148);
+        setState(139);
         match(SEMICOLON);
         }
         break;
@@ -843,9 +836,9 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(151);
+      setState(142);
       match(LBRACK);
-      setState(152);
+      setState(143);
       match(RBRACK);
       }
     }
@@ -882,19 +875,19 @@ class PainlessParser extends Parser {
     InitializerContext _localctx = new InitializerContext(_ctx, getState());
     enterRule(_localctx, 10, RULE_initializer);
     try {
-      setState(156);
-      switch ( getInterpreter().adaptivePredict(_input,19,_ctx) ) {
+      setState(147);
+      switch ( getInterpreter().adaptivePredict(_input,12,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(154);
+        setState(145);
         declaration();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(155);
+        setState(146);
         expression(0);
         }
         break;
@@ -932,7 +925,7 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(158);
+      setState(149);
       expression(0);
       }
     }
@@ -979,23 +972,23 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(160);
+      setState(151);
       decltype();
-      setState(161);
+      setState(152);
       declvar();
-      setState(166);
+      setState(157);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==COMMA) {
         {
         {
-        setState(162);
+        setState(153);
         match(COMMA);
-        setState(163);
+        setState(154);
         declvar();
         }
         }
-        setState(168);
+        setState(159);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
@@ -1042,21 +1035,21 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(169);
+      setState(160);
       identifier();
-      setState(174);
+      setState(165);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==LBRACE) {
         {
         {
-        setState(170);
+        setState(161);
         match(LBRACE);
-        setState(171);
+        setState(162);
         match(RBRACE);
         }
         }
-        setState(176);
+        setState(167);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
@@ -1099,15 +1092,15 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(177);
+      setState(168);
       identifier();
-      setState(180);
+      setState(171);
       _la = _input.LA(1);
       if (_la==ASSIGN) {
         {
-        setState(178);
+        setState(169);
         match(ASSIGN);
-        setState(179);
+        setState(170);
         expression(0);
         }
       }
@@ -1158,29 +1151,29 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(182);
+      setState(173);
       match(CATCH);
-      setState(183);
+      setState(174);
       match(LP);
       {
-      setState(184);
+      setState(175);
       identifier();
-      setState(185);
+      setState(176);
       identifier();
       }
-      setState(187);
+      setState(178);
       match(RP);
-      setState(190);
-      switch ( getInterpreter().adaptivePredict(_input,23,_ctx) ) {
+      setState(181);
+      switch ( getInterpreter().adaptivePredict(_input,16,_ctx) ) {
       case 1:
         {
-        setState(188);
+        setState(179);
         block();
         }
         break;
       case 2:
         {
-        setState(189);
+        setState(180);
         emptyscope();
         }
         break;
@@ -1220,13 +1213,13 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(192);
+      setState(183);
       match(ID);
-      setState(194);
-      switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) {
+      setState(185);
+      switch ( getInterpreter().adaptivePredict(_input,17,_ctx) ) {
       case 1:
         {
-        setState(193);
+        setState(184);
         generic();
         }
         break;
@@ -1275,27 +1268,27 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(196);
+      setState(187);
       match(LT);
-      setState(197);
+      setState(188);
       identifier();
-      setState(202);
+      setState(193);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==COMMA) {
         {
         {
-        setState(198);
+        setState(189);
         match(COMMA);
-        setState(199);
+        setState(190);
         identifier();
         }
         }
-        setState(204);
+        setState(195);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
-      setState(205);
+      setState(196);
       match(GT);
       }
     }
@@ -1563,22 +1556,22 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(233);
-      switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) {
+      setState(224);
+      switch ( getInterpreter().adaptivePredict(_input,19,_ctx) ) {
       case 1:
         {
         _localctx = new UnaryContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(208);
+        setState(199);
         _la = _input.LA(1);
         if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(209);
+        setState(200);
         expression(14);
         }
         break;
@@ -1587,13 +1580,13 @@ class PainlessParser extends Parser {
         _localctx = new CastContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(210);
+        setState(201);
         match(LP);
-        setState(211);
+        setState(202);
         decltype();
-        setState(212);
+        setState(203);
         match(RP);
-        setState(213);
+        setState(204);
         expression(13);
         }
         break;
@@ -1602,16 +1595,16 @@ class PainlessParser extends Parser {
         _localctx = new AssignmentContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(215);
+        setState(206);
         chain();
-        setState(216);
+        setState(207);
         _la = _input.LA(1);
         if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ASSIGN) | (1L << AADD) | (1L << ASUB) | (1L << AMUL) | (1L << ADIV) | (1L << AREM) | (1L << AAND) | (1L << AXOR) | (1L << AOR) | (1L << ALSH) | (1L << ARSH) | (1L << AUSH))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(217);
+        setState(208);
         expression(1);
         }
         break;
@@ -1620,11 +1613,11 @@ class PainlessParser extends Parser {
         _localctx = new PrecedenceContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(219);
+        setState(210);
         match(LP);
-        setState(220);
+        setState(211);
         expression(0);
-        setState(221);
+        setState(212);
         match(RP);
         }
         break;
@@ -1633,7 +1626,7 @@ class PainlessParser extends Parser {
         _localctx = new NumericContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(223);
+        setState(214);
         _la = _input.LA(1);
         if ( !(((((_la - 63)) & ~0x3f) == 0 && ((1L << (_la - 63)) & ((1L << (OCTAL - 63)) | (1L << (HEX - 63)) | (1L << (INTEGER - 63)) | (1L << (DECIMAL - 63)))) != 0)) ) {
         _errHandler.recoverInline(this);
@@ -1647,7 +1640,7 @@ class PainlessParser extends Parser {
         _localctx = new TrueContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(224);
+        setState(215);
         match(TRUE);
         }
         break;
@@ -1656,7 +1649,7 @@ class PainlessParser extends Parser {
         _localctx = new FalseContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(225);
+        setState(216);
         match(FALSE);
         }
         break;
@@ -1665,7 +1658,7 @@ class PainlessParser extends Parser {
         _localctx = new NullContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(226);
+        setState(217);
         match(NULL);
         }
         break;
@@ -1674,9 +1667,9 @@ class PainlessParser extends Parser {
         _localctx = new PostincContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(227);
+        setState(218);
         chain();
-        setState(228);
+        setState(219);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
@@ -1690,14 +1683,14 @@ class PainlessParser extends Parser {
         _localctx = new PreincContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(230);
+        setState(221);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(231);
+        setState(222);
         chain();
         }
         break;
@@ -1706,36 +1699,36 @@ class PainlessParser extends Parser {
         _localctx = new ReadContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(232);
+        setState(223);
         chain();
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(273);
+      setState(264);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,28,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(271);
-          switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) {
+          setState(262);
+          switch ( getInterpreter().adaptivePredict(_input,20,_ctx) ) {
           case 1:
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(235);
+            setState(226);
             if (!(precpred(_ctx, 12))) throw new FailedPredicateException(this, "precpred(_ctx, 12)");
-            setState(236);
+            setState(227);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << MUL) | (1L << DIV) | (1L << REM))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(237);
+            setState(228);
             expression(13);
             }
             break;
@@ -1743,16 +1736,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(238);
+            setState(229);
             if (!(precpred(_ctx, 11))) throw new FailedPredicateException(this, "precpred(_ctx, 11)");
-            setState(239);
+            setState(230);
             _la = _input.LA(1);
             if ( !(_la==ADD || _la==SUB) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(240);
+            setState(231);
             expression(12);
             }
             break;
@@ -1760,16 +1753,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(241);
+            setState(232);
             if (!(precpred(_ctx, 10))) throw new FailedPredicateException(this, "precpred(_ctx, 10)");
-            setState(242);
+            setState(233);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LSH) | (1L << RSH) | (1L << USH))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(243);
+            setState(234);
             expression(11);
             }
             break;
@@ -1777,16 +1770,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(244);
+            setState(235);
             if (!(precpred(_ctx, 9))) throw new FailedPredicateException(this, "precpred(_ctx, 9)");
-            setState(245);
+            setState(236);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LT) | (1L << LTE) | (1L << GT) | (1L << GTE))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(246);
+            setState(237);
             expression(10);
             }
             break;
@@ -1794,16 +1787,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(247);
+            setState(238);
             if (!(precpred(_ctx, 8))) throw new FailedPredicateException(this, "precpred(_ctx, 8)");
-            setState(248);
+            setState(239);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << EQ) | (1L << EQR) | (1L << NE) | (1L << NER))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(249);
+            setState(240);
             expression(9);
             }
             break;
@@ -1811,11 +1804,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(250);
+            setState(241);
             if (!(precpred(_ctx, 7))) throw new FailedPredicateException(this, "precpred(_ctx, 7)");
-            setState(251);
+            setState(242);
             match(BWAND);
-            setState(252);
+            setState(243);
             expression(8);
             }
             break;
@@ -1823,11 +1816,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(253);
+            setState(244);
             if (!(precpred(_ctx, 6))) throw new FailedPredicateException(this, "precpred(_ctx, 6)");
-            setState(254);
+            setState(245);
             match(XOR);
-            setState(255);
+            setState(246);
             expression(7);
             }
             break;
@@ -1835,11 +1828,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(256);
+            setState(247);
             if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)");
-            setState(257);
+            setState(248);
             match(BWOR);
-            setState(258);
+            setState(249);
             expression(6);
             }
             break;
@@ -1847,11 +1840,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(259);
+            setState(250);
             if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)");
-            setState(260);
+            setState(251);
             match(BOOLAND);
-            setState(261);
+            setState(252);
             expression(5);
             }
             break;
@@ -1859,11 +1852,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(262);
+            setState(253);
             if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)");
-            setState(263);
+            setState(254);
             match(BOOLOR);
-            setState(264);
+            setState(255);
             expression(4);
             }
             break;
@@ -1871,24 +1864,24 @@ class PainlessParser extends Parser {
             {
             _localctx = new ConditionalContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(265);
+            setState(256);
             if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
-            setState(266);
+            setState(257);
             match(COND);
-            setState(267);
+            setState(258);
             expression(0);
-            setState(268);
+            setState(259);
             match(COLON);
-            setState(269);
+            setState(260);
             expression(2);
             }
             break;
           }
           } 
         }
-        setState(275);
+        setState(266);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,28,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
       }
       }
     }
@@ -1934,40 +1927,40 @@ class PainlessParser extends Parser {
     ChainContext _localctx = new ChainContext(_ctx, getState());
     enterRule(_localctx, 28, RULE_chain);
     try {
-      setState(281);
-      switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
+      setState(272);
+      switch ( getInterpreter().adaptivePredict(_input,22,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(276);
+        setState(267);
         linkprec();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(277);
+        setState(268);
         linkcast();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(278);
+        setState(269);
         linkvar();
         }
         break;
       case 4:
         enterOuterAlt(_localctx, 4);
         {
-        setState(279);
+        setState(270);
         linknew();
         }
         break;
       case 5:
         enterOuterAlt(_localctx, 5);
         {
-        setState(280);
+        setState(271);
         linkstring();
         }
         break;
@@ -2025,54 +2018,54 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(283);
+      setState(274);
       match(LP);
-      setState(289);
-      switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
+      setState(280);
+      switch ( getInterpreter().adaptivePredict(_input,23,_ctx) ) {
       case 1:
         {
-        setState(284);
+        setState(275);
         linkprec();
         }
         break;
       case 2:
         {
-        setState(285);
+        setState(276);
         linkcast();
         }
         break;
       case 3:
         {
-        setState(286);
+        setState(277);
         linkvar();
         }
         break;
       case 4:
         {
-        setState(287);
+        setState(278);
         linknew();
         }
         break;
       case 5:
         {
-        setState(288);
+        setState(279);
         linkstring();
         }
         break;
       }
-      setState(291);
+      setState(282);
       match(RP);
-      setState(294);
-      switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) {
+      setState(285);
+      switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) {
       case 1:
         {
-        setState(292);
+        setState(283);
         linkdot();
         }
         break;
       case 2:
         {
-        setState(293);
+        setState(284);
         linkbrace();
         }
         break;
@@ -2128,41 +2121,41 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(296);
+      setState(287);
       match(LP);
-      setState(297);
+      setState(288);
       decltype();
-      setState(298);
+      setState(289);
       match(RP);
-      setState(304);
-      switch ( getInterpreter().adaptivePredict(_input,32,_ctx) ) {
+      setState(295);
+      switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) {
       case 1:
         {
-        setState(299);
+        setState(290);
         linkprec();
         }
         break;
       case 2:
         {
-        setState(300);
+        setState(291);
         linkcast();
         }
         break;
       case 3:
         {
-        setState(301);
+        setState(292);
         linkvar();
         }
         break;
       case 4:
         {
-        setState(302);
+        setState(293);
         linknew();
         }
         break;
       case 5:
         {
-        setState(303);
+        setState(294);
         linkstring();
         }
         break;
@@ -2209,23 +2202,23 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(306);
+      setState(297);
       match(LBRACE);
-      setState(307);
+      setState(298);
       expression(0);
-      setState(308);
+      setState(299);
       match(RBRACE);
-      setState(311);
-      switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) {
+      setState(302);
+      switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) {
       case 1:
         {
-        setState(309);
+        setState(300);
         linkdot();
         }
         break;
       case 2:
         {
-        setState(310);
+        setState(301);
         linkbrace();
         }
         break;
@@ -2268,19 +2261,19 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(313);
+      setState(304);
       match(DOT);
-      setState(316);
-      switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) {
+      setState(307);
+      switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) {
       case 1:
         {
-        setState(314);
+        setState(305);
         linkcall();
         }
         break;
       case 2:
         {
-        setState(315);
+        setState(306);
         linkfield();
         }
         break;
@@ -2326,21 +2319,21 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(318);
+      setState(309);
       match(EXTID);
-      setState(319);
+      setState(310);
       arguments();
-      setState(322);
-      switch ( getInterpreter().adaptivePredict(_input,35,_ctx) ) {
+      setState(313);
+      switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) {
       case 1:
         {
-        setState(320);
+        setState(311);
         linkdot();
         }
         break;
       case 2:
         {
-        setState(321);
+        setState(312);
         linkbrace();
         }
         break;
@@ -2385,19 +2378,19 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(324);
+      setState(315);
       identifier();
-      setState(327);
-      switch ( getInterpreter().adaptivePredict(_input,36,_ctx) ) {
+      setState(318);
+      switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
       case 1:
         {
-        setState(325);
+        setState(316);
         linkdot();
         }
         break;
       case 2:
         {
-        setState(326);
+        setState(317);
         linkbrace();
         }
         break;
@@ -2442,24 +2435,24 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(329);
+      setState(320);
       _la = _input.LA(1);
       if ( !(_la==EXTINTEGER || _la==EXTID) ) {
       _errHandler.recoverInline(this);
       } else {
         consume();
       }
-      setState(332);
-      switch ( getInterpreter().adaptivePredict(_input,37,_ctx) ) {
+      setState(323);
+      switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
       case 1:
         {
-        setState(330);
+        setState(321);
         linkdot();
         }
         break;
       case 2:
         {
-        setState(331);
+        setState(322);
         linkbrace();
         }
         break;
@@ -2520,22 +2513,22 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(334);
+      setState(325);
       match(NEW);
-      setState(335);
+      setState(326);
       identifier();
-      setState(351);
+      setState(342);
       switch (_input.LA(1)) {
       case LP:
         {
         {
-        setState(336);
+        setState(327);
         arguments();
-        setState(338);
-        switch ( getInterpreter().adaptivePredict(_input,38,_ctx) ) {
+        setState(329);
+        switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) {
         case 1:
           {
-          setState(337);
+          setState(328);
           linkdot();
           }
           break;
@@ -2546,7 +2539,7 @@ class PainlessParser extends Parser {
       case LBRACE:
         {
         {
-        setState(344); 
+        setState(335); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -2554,11 +2547,11 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(340);
+            setState(331);
             match(LBRACE);
-            setState(341);
+            setState(332);
             expression(0);
-            setState(342);
+            setState(333);
             match(RBRACE);
             }
             }
@@ -2566,15 +2559,15 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(346); 
+          setState(337); 
           _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,39,_ctx);
+          _alt = getInterpreter().adaptivePredict(_input,32,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
-        setState(349);
-        switch ( getInterpreter().adaptivePredict(_input,40,_ctx) ) {
+        setState(340);
+        switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) {
         case 1:
           {
-          setState(348);
+          setState(339);
           linkdot();
           }
           break;
@@ -2623,19 +2616,19 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(353);
+      setState(344);
       match(STRING);
-      setState(356);
-      switch ( getInterpreter().adaptivePredict(_input,42,_ctx) ) {
+      setState(347);
+      switch ( getInterpreter().adaptivePredict(_input,35,_ctx) ) {
       case 1:
         {
-        setState(354);
+        setState(345);
         linkdot();
         }
         break;
       case 2:
         {
-        setState(355);
+        setState(346);
         linkbrace();
         }
         break;
@@ -2685,34 +2678,34 @@ class PainlessParser extends Parser {
       enterOuterAlt(_localctx, 1);
       {
       {
-      setState(358);
+      setState(349);
       match(LP);
-      setState(367);
+      setState(358);
       _la = _input.LA(1);
       if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0)) {
         {
-        setState(359);
+        setState(350);
         expression(0);
-        setState(364);
+        setState(355);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(360);
+          setState(351);
           match(COMMA);
-          setState(361);
+          setState(352);
           expression(0);
           }
           }
-          setState(366);
+          setState(357);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
       }
 
-      setState(369);
+      setState(360);
       match(RP);
       }
       }
@@ -2764,150 +2757,145 @@ class PainlessParser extends Parser {
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3K\u0176\4\2\t\2\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3K\u016d\4\2\t\2\4"+
     "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
     "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
     "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+
     "\4\32\t\32\3\2\6\2\66\n\2\r\2\16\2\67\3\2\3\2\3\3\3\3\3\3\3\3\3\3\3\3"+
     "\3\3\5\3C\n\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3K\n\3\3\3\3\3\3\3\3\3\3\3\3\3"+
-    "\3\3\5\3T\n\3\3\3\3\3\3\3\5\3Y\n\3\3\3\3\3\5\3]\n\3\3\3\3\3\5\3a\n\3\3"+
-    "\3\3\3\3\3\5\3f\n\3\3\3\3\3\5\3j\n\3\3\3\3\3\5\3n\n\3\3\3\3\3\5\3r\n\3"+
-    "\3\3\3\3\3\3\5\3w\n\3\3\3\3\3\3\3\6\3|\n\3\r\3\16\3}\3\3\3\3\3\3\5\3\u0083"+
-    "\n\3\3\3\3\3\5\3\u0087\n\3\5\3\u0089\n\3\3\4\3\4\6\4\u008d\n\4\r\4\16"+
-    "\4\u008e\3\4\3\4\3\4\5\4\u0094\n\4\3\5\3\5\5\5\u0098\n\5\3\6\3\6\3\6\3"+
-    "\7\3\7\5\7\u009f\n\7\3\b\3\b\3\t\3\t\3\t\3\t\7\t\u00a7\n\t\f\t\16\t\u00aa"+
-    "\13\t\3\n\3\n\3\n\7\n\u00af\n\n\f\n\16\n\u00b2\13\n\3\13\3\13\3\13\5\13"+
-    "\u00b7\n\13\3\f\3\f\3\f\3\f\3\f\3\f\3\f\3\f\5\f\u00c1\n\f\3\r\3\r\5\r"+
-    "\u00c5\n\r\3\16\3\16\3\16\3\16\7\16\u00cb\n\16\f\16\16\16\u00ce\13\16"+
-    "\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
+    "\3\3\3\3\3\3\3\3\3\3\5\3X\n\3\3\3\3\3\5\3\\\n\3\3\3\3\3\5\3`\n\3\3\3\3"+
+    "\3\3\3\5\3e\n\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3"+
+    "\3\6\3u\n\3\r\3\16\3v\3\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3\u0080\n\3\3\4\3"+
+    "\4\6\4\u0084\n\4\r\4\16\4\u0085\3\4\3\4\3\4\5\4\u008b\n\4\3\5\3\5\5\5"+
+    "\u008f\n\5\3\6\3\6\3\6\3\7\3\7\5\7\u0096\n\7\3\b\3\b\3\t\3\t\3\t\3\t\7"+
+    "\t\u009e\n\t\f\t\16\t\u00a1\13\t\3\n\3\n\3\n\7\n\u00a6\n\n\f\n\16\n\u00a9"+
+    "\13\n\3\13\3\13\3\13\5\13\u00ae\n\13\3\f\3\f\3\f\3\f\3\f\3\f\3\f\3\f\5"+
+    "\f\u00b8\n\f\3\r\3\r\5\r\u00bc\n\r\3\16\3\16\3\16\3\16\7\16\u00c2\n\16"+
+    "\f\16\16\16\u00c5\13\16\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3"+
+    "\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3"+
+    "\17\3\17\3\17\3\17\3\17\5\17\u00e3\n\17\3\17\3\17\3\17\3\17\3\17\3\17"+
     "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
-    "\5\17\u00ec\n\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
     "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
-    "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\7\17\u0112\n\17"+
-    "\f\17\16\17\u0115\13\17\3\20\3\20\3\20\3\20\3\20\5\20\u011c\n\20\3\21"+
-    "\3\21\3\21\3\21\3\21\3\21\5\21\u0124\n\21\3\21\3\21\3\21\5\21\u0129\n"+
-    "\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\5\22\u0133\n\22\3\23\3\23"+
-    "\3\23\3\23\3\23\5\23\u013a\n\23\3\24\3\24\3\24\5\24\u013f\n\24\3\25\3"+
-    "\25\3\25\3\25\5\25\u0145\n\25\3\26\3\26\3\26\5\26\u014a\n\26\3\27\3\27"+
-    "\3\27\5\27\u014f\n\27\3\30\3\30\3\30\3\30\5\30\u0155\n\30\3\30\3\30\3"+
-    "\30\3\30\6\30\u015b\n\30\r\30\16\30\u015c\3\30\5\30\u0160\n\30\5\30\u0162"+
-    "\n\30\3\31\3\31\3\31\5\31\u0167\n\31\3\32\3\32\3\32\3\32\7\32\u016d\n"+
-    "\32\f\32\16\32\u0170\13\32\5\32\u0172\n\32\3\32\3\32\3\32\2\3\34\33\2"+
-    "\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,.\60\62\2\f\4\2\32\33\37"+
-    " \3\2\65@\3\2AD\3\2\63\64\3\2\34\36\3\2\37 \3\2!#\3\2$\'\3\2(+\3\2JK\u01b3"+
-    "\2\65\3\2\2\2\4\u0088\3\2\2\2\6\u0093\3\2\2\2\b\u0097\3\2\2\2\n\u0099"+
-    "\3\2\2\2\f\u009e\3\2\2\2\16\u00a0\3\2\2\2\20\u00a2\3\2\2\2\22\u00ab\3"+
-    "\2\2\2\24\u00b3\3\2\2\2\26\u00b8\3\2\2\2\30\u00c2\3\2\2\2\32\u00c6\3\2"+
-    "\2\2\34\u00eb\3\2\2\2\36\u011b\3\2\2\2 \u011d\3\2\2\2\"\u012a\3\2\2\2"+
-    "$\u0134\3\2\2\2&\u013b\3\2\2\2(\u0140\3\2\2\2*\u0146\3\2\2\2,\u014b\3"+
-    "\2\2\2.\u0150\3\2\2\2\60\u0163\3\2\2\2\62\u0168\3\2\2\2\64\66\5\4\3\2"+
-    "\65\64\3\2\2\2\66\67\3\2\2\2\67\65\3\2\2\2\678\3\2\2\289\3\2\2\29:\7\2"+
-    "\2\3:\3\3\2\2\2;<\7\16\2\2<=\7\t\2\2=>\5\34\17\2>?\7\n\2\2?B\5\6\4\2@"+
-    "A\7\17\2\2AC\5\6\4\2B@\3\2\2\2BC\3\2\2\2C\u0089\3\2\2\2DE\7\20\2\2EF\7"+
-    "\t\2\2FG\5\34\17\2GJ\7\n\2\2HK\5\6\4\2IK\5\b\5\2JH\3\2\2\2JI\3\2\2\2K"+
-    "\u0089\3\2\2\2LM\7\21\2\2MN\5\6\4\2NO\7\20\2\2OP\7\t\2\2PQ\5\34\17\2Q"+
-    "S\7\n\2\2RT\7\r\2\2SR\3\2\2\2ST\3\2\2\2T\u0089\3\2\2\2UV\7\22\2\2VX\7"+
-    "\t\2\2WY\5\f\7\2XW\3\2\2\2XY\3\2\2\2YZ\3\2\2\2Z\\\7\r\2\2[]\5\34\17\2"+
-    "\\[\3\2\2\2\\]\3\2\2\2]^\3\2\2\2^`\7\r\2\2_a\5\16\b\2`_\3\2\2\2`a\3\2"+
-    "\2\2ab\3\2\2\2be\7\n\2\2cf\5\6\4\2df\5\b\5\2ec\3\2\2\2ed\3\2\2\2f\u0089"+
-    "\3\2\2\2gi\5\20\t\2hj\7\r\2\2ih\3\2\2\2ij\3\2\2\2j\u0089\3\2\2\2km\7\23"+
-    "\2\2ln\7\r\2\2ml\3\2\2\2mn\3\2\2\2n\u0089\3\2\2\2oq\7\24\2\2pr\7\r\2\2"+
-    "qp\3\2\2\2qr\3\2\2\2r\u0089\3\2\2\2st\7\25\2\2tv\5\34\17\2uw\7\r\2\2v"+
-    "u\3\2\2\2vw\3\2\2\2w\u0089\3\2\2\2xy\7\27\2\2y{\5\6\4\2z|\5\26\f\2{z\3"+
-    "\2\2\2|}\3\2\2\2}{\3\2\2\2}~\3\2\2\2~\u0089\3\2\2\2\177\u0080\7\31\2\2"+
-    "\u0080\u0082\5\34\17\2\u0081\u0083\7\r\2\2\u0082\u0081\3\2\2\2\u0082\u0083"+
-    "\3\2\2\2\u0083\u0089\3\2\2\2\u0084\u0086\5\34\17\2\u0085\u0087\7\r\2\2"+
-    "\u0086\u0085\3\2\2\2\u0086\u0087\3\2\2\2\u0087\u0089\3\2\2\2\u0088;\3"+
-    "\2\2\2\u0088D\3\2\2\2\u0088L\3\2\2\2\u0088U\3\2\2\2\u0088g\3\2\2\2\u0088"+
-    "k\3\2\2\2\u0088o\3\2\2\2\u0088s\3\2\2\2\u0088x\3\2\2\2\u0088\177\3\2\2"+
-    "\2\u0088\u0084\3\2\2\2\u0089\5\3\2\2\2\u008a\u008c\7\5\2\2\u008b\u008d"+
-    "\5\4\3\2\u008c\u008b\3\2\2\2\u008d\u008e\3\2\2\2\u008e\u008c\3\2\2\2\u008e"+
-    "\u008f\3\2\2\2\u008f\u0090\3\2\2\2\u0090\u0091\7\6\2\2\u0091\u0094\3\2"+
-    "\2\2\u0092\u0094\5\4\3\2\u0093\u008a\3\2\2\2\u0093\u0092\3\2\2\2\u0094"+
-    "\7\3\2\2\2\u0095\u0098\5\n\6\2\u0096\u0098\7\r\2\2\u0097\u0095\3\2\2\2"+
-    "\u0097\u0096\3\2\2\2\u0098\t\3\2\2\2\u0099\u009a\7\5\2\2\u009a\u009b\7"+
-    "\6\2\2\u009b\13\3\2\2\2\u009c\u009f\5\20\t\2\u009d\u009f\5\34\17\2\u009e"+
-    "\u009c\3\2\2\2\u009e\u009d\3\2\2\2\u009f\r\3\2\2\2\u00a0\u00a1\5\34\17"+
-    "\2\u00a1\17\3\2\2\2\u00a2\u00a3\5\22\n\2\u00a3\u00a8\5\24\13\2\u00a4\u00a5"+
-    "\7\f\2\2\u00a5\u00a7\5\24\13\2\u00a6\u00a4\3\2\2\2\u00a7\u00aa\3\2\2\2"+
-    "\u00a8\u00a6\3\2\2\2\u00a8\u00a9\3\2\2\2\u00a9\21\3\2\2\2\u00aa\u00a8"+
-    "\3\2\2\2\u00ab\u00b0\5\30\r\2\u00ac\u00ad\7\7\2\2\u00ad\u00af\7\b\2\2"+
-    "\u00ae\u00ac\3\2\2\2\u00af\u00b2\3\2\2\2\u00b0\u00ae\3\2\2\2\u00b0\u00b1"+
-    "\3\2\2\2\u00b1\23\3\2\2\2\u00b2\u00b0\3\2\2\2\u00b3\u00b6\5\30\r\2\u00b4"+
-    "\u00b5\7\65\2\2\u00b5\u00b7\5\34\17\2\u00b6\u00b4\3\2\2\2\u00b6\u00b7"+
-    "\3\2\2\2\u00b7\25\3\2\2\2\u00b8\u00b9\7\30\2\2\u00b9\u00ba\7\t\2\2\u00ba"+
-    "\u00bb\5\30\r\2\u00bb\u00bc\5\30\r\2\u00bc\u00bd\3\2\2\2\u00bd\u00c0\7"+
-    "\n\2\2\u00be\u00c1\5\6\4\2\u00bf\u00c1\5\n\6\2\u00c0\u00be\3\2\2\2\u00c0"+
-    "\u00bf\3\2\2\2\u00c1\27\3\2\2\2\u00c2\u00c4\7I\2\2\u00c3\u00c5\5\32\16"+
-    "\2\u00c4\u00c3\3\2\2\2\u00c4\u00c5\3\2\2\2\u00c5\31\3\2\2\2\u00c6\u00c7"+
-    "\7$\2\2\u00c7\u00cc\5\30\r\2\u00c8\u00c9\7\f\2\2\u00c9\u00cb\5\30\r\2"+
-    "\u00ca\u00c8\3\2\2\2\u00cb\u00ce\3\2\2\2\u00cc\u00ca\3\2\2\2\u00cc\u00cd"+
-    "\3\2\2\2\u00cd\u00cf\3\2\2\2\u00ce\u00cc\3\2\2\2\u00cf\u00d0\7&\2\2\u00d0"+
-    "\33\3\2\2\2\u00d1\u00d2\b\17\1\2\u00d2\u00d3\t\2\2\2\u00d3\u00ec\5\34"+
-    "\17\20\u00d4\u00d5\7\t\2\2\u00d5\u00d6\5\22\n\2\u00d6\u00d7\7\n\2\2\u00d7"+
-    "\u00d8\5\34\17\17\u00d8\u00ec\3\2\2\2\u00d9\u00da\5\36\20\2\u00da\u00db"+
-    "\t\3\2\2\u00db\u00dc\5\34\17\3\u00dc\u00ec\3\2\2\2\u00dd\u00de\7\t\2\2"+
-    "\u00de\u00df\5\34\17\2\u00df\u00e0\7\n\2\2\u00e0\u00ec\3\2\2\2\u00e1\u00ec"+
-    "\t\4\2\2\u00e2\u00ec\7F\2\2\u00e3\u00ec\7G\2\2\u00e4\u00ec\7H\2\2\u00e5"+
-    "\u00e6\5\36\20\2\u00e6\u00e7\t\5\2\2\u00e7\u00ec\3\2\2\2\u00e8\u00e9\t"+
-    "\5\2\2\u00e9\u00ec\5\36\20\2\u00ea\u00ec\5\36\20\2\u00eb\u00d1\3\2\2\2"+
-    "\u00eb\u00d4\3\2\2\2\u00eb\u00d9\3\2\2\2\u00eb\u00dd\3\2\2\2\u00eb\u00e1"+
-    "\3\2\2\2\u00eb\u00e2\3\2\2\2\u00eb\u00e3\3\2\2\2\u00eb\u00e4\3\2\2\2\u00eb"+
-    "\u00e5\3\2\2\2\u00eb\u00e8\3\2\2\2\u00eb\u00ea\3\2\2\2\u00ec\u0113\3\2"+
-    "\2\2\u00ed\u00ee\f\16\2\2\u00ee\u00ef\t\6\2\2\u00ef\u0112\5\34\17\17\u00f0"+
-    "\u00f1\f\r\2\2\u00f1\u00f2\t\7\2\2\u00f2\u0112\5\34\17\16\u00f3\u00f4"+
-    "\f\f\2\2\u00f4\u00f5\t\b\2\2\u00f5\u0112\5\34\17\r\u00f6\u00f7\f\13\2"+
-    "\2\u00f7\u00f8\t\t\2\2\u00f8\u0112\5\34\17\f\u00f9\u00fa\f\n\2\2\u00fa"+
-    "\u00fb\t\n\2\2\u00fb\u0112\5\34\17\13\u00fc\u00fd\f\t\2\2\u00fd\u00fe"+
-    "\7,\2\2\u00fe\u0112\5\34\17\n\u00ff\u0100\f\b\2\2\u0100\u0101\7-\2\2\u0101"+
-    "\u0112\5\34\17\t\u0102\u0103\f\7\2\2\u0103\u0104\7.\2\2\u0104\u0112\5"+
-    "\34\17\b\u0105\u0106\f\6\2\2\u0106\u0107\7/\2\2\u0107\u0112\5\34\17\7"+
-    "\u0108\u0109\f\5\2\2\u0109\u010a\7\60\2\2\u010a\u0112\5\34\17\6\u010b"+
-    "\u010c\f\4\2\2\u010c\u010d\7\61\2\2\u010d\u010e\5\34\17\2\u010e\u010f"+
-    "\7\62\2\2\u010f\u0110\5\34\17\4\u0110\u0112\3\2\2\2\u0111\u00ed\3\2\2"+
-    "\2\u0111\u00f0\3\2\2\2\u0111\u00f3\3\2\2\2\u0111\u00f6\3\2\2\2\u0111\u00f9"+
-    "\3\2\2\2\u0111\u00fc\3\2\2\2\u0111\u00ff\3\2\2\2\u0111\u0102\3\2\2\2\u0111"+
-    "\u0105\3\2\2\2\u0111\u0108\3\2\2\2\u0111\u010b\3\2\2\2\u0112\u0115\3\2"+
-    "\2\2\u0113\u0111\3\2\2\2\u0113\u0114\3\2\2\2\u0114\35\3\2\2\2\u0115\u0113"+
-    "\3\2\2\2\u0116\u011c\5 \21\2\u0117\u011c\5\"\22\2\u0118\u011c\5*\26\2"+
-    "\u0119\u011c\5.\30\2\u011a\u011c\5\60\31\2\u011b\u0116\3\2\2\2\u011b\u0117"+
-    "\3\2\2\2\u011b\u0118\3\2\2\2\u011b\u0119\3\2\2\2\u011b\u011a\3\2\2\2\u011c"+
-    "\37\3\2\2\2\u011d\u0123\7\t\2\2\u011e\u0124\5 \21\2\u011f\u0124\5\"\22"+
-    "\2\u0120\u0124\5*\26\2\u0121\u0124\5.\30\2\u0122\u0124\5\60\31\2\u0123"+
-    "\u011e\3\2\2\2\u0123\u011f\3\2\2\2\u0123\u0120\3\2\2\2\u0123\u0121\3\2"+
-    "\2\2\u0123\u0122\3\2\2\2\u0124\u0125\3\2\2\2\u0125\u0128\7\n\2\2\u0126"+
-    "\u0129\5&\24\2\u0127\u0129\5$\23\2\u0128\u0126\3\2\2\2\u0128\u0127\3\2"+
-    "\2\2\u0128\u0129\3\2\2\2\u0129!\3\2\2\2\u012a\u012b\7\t\2\2\u012b\u012c"+
-    "\5\22\n\2\u012c\u0132\7\n\2\2\u012d\u0133\5 \21\2\u012e\u0133\5\"\22\2"+
-    "\u012f\u0133\5*\26\2\u0130\u0133\5.\30\2\u0131\u0133\5\60\31\2\u0132\u012d"+
-    "\3\2\2\2\u0132\u012e\3\2\2\2\u0132\u012f\3\2\2\2\u0132\u0130\3\2\2\2\u0132"+
-    "\u0131\3\2\2\2\u0133#\3\2\2\2\u0134\u0135\7\7\2\2\u0135\u0136\5\34\17"+
-    "\2\u0136\u0139\7\b\2\2\u0137\u013a\5&\24\2\u0138\u013a\5$\23\2\u0139\u0137"+
-    "\3\2\2\2\u0139\u0138\3\2\2\2\u0139\u013a\3\2\2\2\u013a%\3\2\2\2\u013b"+
-    "\u013e\7\13\2\2\u013c\u013f\5(\25\2\u013d\u013f\5,\27\2\u013e\u013c\3"+
-    "\2\2\2\u013e\u013d\3\2\2\2\u013f\'\3\2\2\2\u0140\u0141\7K\2\2\u0141\u0144"+
-    "\5\62\32\2\u0142\u0145\5&\24\2\u0143\u0145\5$\23\2\u0144\u0142\3\2\2\2"+
-    "\u0144\u0143\3\2\2\2\u0144\u0145\3\2\2\2\u0145)\3\2\2\2\u0146\u0149\5"+
-    "\30\r\2\u0147\u014a\5&\24\2\u0148\u014a\5$\23\2\u0149\u0147\3\2\2\2\u0149"+
-    "\u0148\3\2\2\2\u0149\u014a\3\2\2\2\u014a+\3\2\2\2\u014b\u014e\t\13\2\2"+
-    "\u014c\u014f\5&\24\2\u014d\u014f\5$\23\2\u014e\u014c\3\2\2\2\u014e\u014d"+
-    "\3\2\2\2\u014e\u014f\3\2\2\2\u014f-\3\2\2\2\u0150\u0151\7\26\2\2\u0151"+
-    "\u0161\5\30\r\2\u0152\u0154\5\62\32\2\u0153\u0155\5&\24\2\u0154\u0153"+
-    "\3\2\2\2\u0154\u0155\3\2\2\2\u0155\u0162\3\2\2\2\u0156\u0157\7\7\2\2\u0157"+
-    "\u0158\5\34\17\2\u0158\u0159\7\b\2\2\u0159\u015b\3\2\2\2\u015a\u0156\3"+
-    "\2\2\2\u015b\u015c\3\2\2\2\u015c\u015a\3\2\2\2\u015c\u015d\3\2\2\2\u015d"+
-    "\u015f\3\2\2\2\u015e\u0160\5&\24\2\u015f\u015e\3\2\2\2\u015f\u0160\3\2"+
-    "\2\2\u0160\u0162\3\2\2\2\u0161\u0152\3\2\2\2\u0161\u015a\3\2\2\2\u0162"+
-    "/\3\2\2\2\u0163\u0166\7E\2\2\u0164\u0167\5&\24\2\u0165\u0167\5$\23\2\u0166"+
-    "\u0164\3\2\2\2\u0166\u0165\3\2\2\2\u0166\u0167\3\2\2\2\u0167\61\3\2\2"+
-    "\2\u0168\u0171\7\t\2\2\u0169\u016e\5\34\17\2\u016a\u016b\7\f\2\2\u016b"+
-    "\u016d\5\34\17\2\u016c\u016a\3\2\2\2\u016d\u0170\3\2\2\2\u016e\u016c\3"+
-    "\2\2\2\u016e\u016f\3\2\2\2\u016f\u0172\3\2\2\2\u0170\u016e\3\2\2\2\u0171"+
-    "\u0169\3\2\2\2\u0171\u0172\3\2\2\2\u0172\u0173\3\2\2\2\u0173\u0174\7\n"+
-    "\2\2\u0174\63\3\2\2\2/\67BJSX\\`eimqv}\u0082\u0086\u0088\u008e\u0093\u0097"+
-    "\u009e\u00a8\u00b0\u00b6\u00c0\u00c4\u00cc\u00eb\u0111\u0113\u011b\u0123"+
-    "\u0128\u0132\u0139\u013e\u0144\u0149\u014e\u0154\u015c\u015f\u0161\u0166"+
-    "\u016e\u0171";
+    "\3\17\3\17\7\17\u0109\n\17\f\17\16\17\u010c\13\17\3\20\3\20\3\20\3\20"+
+    "\3\20\5\20\u0113\n\20\3\21\3\21\3\21\3\21\3\21\3\21\5\21\u011b\n\21\3"+
+    "\21\3\21\3\21\5\21\u0120\n\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22"+
+    "\5\22\u012a\n\22\3\23\3\23\3\23\3\23\3\23\5\23\u0131\n\23\3\24\3\24\3"+
+    "\24\5\24\u0136\n\24\3\25\3\25\3\25\3\25\5\25\u013c\n\25\3\26\3\26\3\26"+
+    "\5\26\u0141\n\26\3\27\3\27\3\27\5\27\u0146\n\27\3\30\3\30\3\30\3\30\5"+
+    "\30\u014c\n\30\3\30\3\30\3\30\3\30\6\30\u0152\n\30\r\30\16\30\u0153\3"+
+    "\30\5\30\u0157\n\30\5\30\u0159\n\30\3\31\3\31\3\31\5\31\u015e\n\31\3\32"+
+    "\3\32\3\32\3\32\7\32\u0164\n\32\f\32\16\32\u0167\13\32\5\32\u0169\n\32"+
+    "\3\32\3\32\3\32\2\3\34\33\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&"+
+    "(*,.\60\62\2\r\3\3\r\r\4\2\32\33\37 \3\2\65@\3\2AD\3\2\63\64\3\2\34\36"+
+    "\3\2\37 \3\2!#\3\2$\'\3\2(+\3\2JK\u01a3\2\65\3\2\2\2\4\177\3\2\2\2\6\u008a"+
+    "\3\2\2\2\b\u008e\3\2\2\2\n\u0090\3\2\2\2\f\u0095\3\2\2\2\16\u0097\3\2"+
+    "\2\2\20\u0099\3\2\2\2\22\u00a2\3\2\2\2\24\u00aa\3\2\2\2\26\u00af\3\2\2"+
+    "\2\30\u00b9\3\2\2\2\32\u00bd\3\2\2\2\34\u00e2\3\2\2\2\36\u0112\3\2\2\2"+
+    " \u0114\3\2\2\2\"\u0121\3\2\2\2$\u012b\3\2\2\2&\u0132\3\2\2\2(\u0137\3"+
+    "\2\2\2*\u013d\3\2\2\2,\u0142\3\2\2\2.\u0147\3\2\2\2\60\u015a\3\2\2\2\62"+
+    "\u015f\3\2\2\2\64\66\5\4\3\2\65\64\3\2\2\2\66\67\3\2\2\2\67\65\3\2\2\2"+
+    "\678\3\2\2\289\3\2\2\29:\7\2\2\3:\3\3\2\2\2;<\7\16\2\2<=\7\t\2\2=>\5\34"+
+    "\17\2>?\7\n\2\2?B\5\6\4\2@A\7\17\2\2AC\5\6\4\2B@\3\2\2\2BC\3\2\2\2C\u0080"+
+    "\3\2\2\2DE\7\20\2\2EF\7\t\2\2FG\5\34\17\2GJ\7\n\2\2HK\5\6\4\2IK\5\b\5"+
+    "\2JH\3\2\2\2JI\3\2\2\2K\u0080\3\2\2\2LM\7\21\2\2MN\5\6\4\2NO\7\20\2\2"+
+    "OP\7\t\2\2PQ\5\34\17\2QR\7\n\2\2RS\t\2\2\2S\u0080\3\2\2\2TU\7\22\2\2U"+
+    "W\7\t\2\2VX\5\f\7\2WV\3\2\2\2WX\3\2\2\2XY\3\2\2\2Y[\7\r\2\2Z\\\5\34\17"+
+    "\2[Z\3\2\2\2[\\\3\2\2\2\\]\3\2\2\2]_\7\r\2\2^`\5\16\b\2_^\3\2\2\2_`\3"+
+    "\2\2\2`a\3\2\2\2ad\7\n\2\2be\5\6\4\2ce\5\b\5\2db\3\2\2\2dc\3\2\2\2e\u0080"+
+    "\3\2\2\2fg\5\20\t\2gh\t\2\2\2h\u0080\3\2\2\2ij\7\23\2\2j\u0080\t\2\2\2"+
+    "kl\7\24\2\2l\u0080\t\2\2\2mn\7\25\2\2no\5\34\17\2op\t\2\2\2p\u0080\3\2"+
+    "\2\2qr\7\27\2\2rt\5\6\4\2su\5\26\f\2ts\3\2\2\2uv\3\2\2\2vt\3\2\2\2vw\3"+
+    "\2\2\2w\u0080\3\2\2\2xy\7\31\2\2yz\5\34\17\2z{\t\2\2\2{\u0080\3\2\2\2"+
+    "|}\5\34\17\2}~\t\2\2\2~\u0080\3\2\2\2\177;\3\2\2\2\177D\3\2\2\2\177L\3"+
+    "\2\2\2\177T\3\2\2\2\177f\3\2\2\2\177i\3\2\2\2\177k\3\2\2\2\177m\3\2\2"+
+    "\2\177q\3\2\2\2\177x\3\2\2\2\177|\3\2\2\2\u0080\5\3\2\2\2\u0081\u0083"+
+    "\7\5\2\2\u0082\u0084\5\4\3\2\u0083\u0082\3\2\2\2\u0084\u0085\3\2\2\2\u0085"+
+    "\u0083\3\2\2\2\u0085\u0086\3\2\2\2\u0086\u0087\3\2\2\2\u0087\u0088\7\6"+
+    "\2\2\u0088\u008b\3\2\2\2\u0089\u008b\5\4\3\2\u008a\u0081\3\2\2\2\u008a"+
+    "\u0089\3\2\2\2\u008b\7\3\2\2\2\u008c\u008f\5\n\6\2\u008d\u008f\7\r\2\2"+
+    "\u008e\u008c\3\2\2\2\u008e\u008d\3\2\2\2\u008f\t\3\2\2\2\u0090\u0091\7"+
+    "\5\2\2\u0091\u0092\7\6\2\2\u0092\13\3\2\2\2\u0093\u0096\5\20\t\2\u0094"+
+    "\u0096\5\34\17\2\u0095\u0093\3\2\2\2\u0095\u0094\3\2\2\2\u0096\r\3\2\2"+
+    "\2\u0097\u0098\5\34\17\2\u0098\17\3\2\2\2\u0099\u009a\5\22\n\2\u009a\u009f"+
+    "\5\24\13\2\u009b\u009c\7\f\2\2\u009c\u009e\5\24\13\2\u009d\u009b\3\2\2"+
+    "\2\u009e\u00a1\3\2\2\2\u009f\u009d\3\2\2\2\u009f\u00a0\3\2\2\2\u00a0\21"+
+    "\3\2\2\2\u00a1\u009f\3\2\2\2\u00a2\u00a7\5\30\r\2\u00a3\u00a4\7\7\2\2"+
+    "\u00a4\u00a6\7\b\2\2\u00a5\u00a3\3\2\2\2\u00a6\u00a9\3\2\2\2\u00a7\u00a5"+
+    "\3\2\2\2\u00a7\u00a8\3\2\2\2\u00a8\23\3\2\2\2\u00a9\u00a7\3\2\2\2\u00aa"+
+    "\u00ad\5\30\r\2\u00ab\u00ac\7\65\2\2\u00ac\u00ae\5\34\17\2\u00ad\u00ab"+
+    "\3\2\2\2\u00ad\u00ae\3\2\2\2\u00ae\25\3\2\2\2\u00af\u00b0\7\30\2\2\u00b0"+
+    "\u00b1\7\t\2\2\u00b1\u00b2\5\30\r\2\u00b2\u00b3\5\30\r\2\u00b3\u00b4\3"+
+    "\2\2\2\u00b4\u00b7\7\n\2\2\u00b5\u00b8\5\6\4\2\u00b6\u00b8\5\n\6\2\u00b7"+
+    "\u00b5\3\2\2\2\u00b7\u00b6\3\2\2\2\u00b8\27\3\2\2\2\u00b9\u00bb\7I\2\2"+
+    "\u00ba\u00bc\5\32\16\2\u00bb\u00ba\3\2\2\2\u00bb\u00bc\3\2\2\2\u00bc\31"+
+    "\3\2\2\2\u00bd\u00be\7$\2\2\u00be\u00c3\5\30\r\2\u00bf\u00c0\7\f\2\2\u00c0"+
+    "\u00c2\5\30\r\2\u00c1\u00bf\3\2\2\2\u00c2\u00c5\3\2\2\2\u00c3\u00c1\3"+
+    "\2\2\2\u00c3\u00c4\3\2\2\2\u00c4\u00c6\3\2\2\2\u00c5\u00c3\3\2\2\2\u00c6"+
+    "\u00c7\7&\2\2\u00c7\33\3\2\2\2\u00c8\u00c9\b\17\1\2\u00c9\u00ca\t\3\2"+
+    "\2\u00ca\u00e3\5\34\17\20\u00cb\u00cc\7\t\2\2\u00cc\u00cd\5\22\n\2\u00cd"+
+    "\u00ce\7\n\2\2\u00ce\u00cf\5\34\17\17\u00cf\u00e3\3\2\2\2\u00d0\u00d1"+
+    "\5\36\20\2\u00d1\u00d2\t\4\2\2\u00d2\u00d3\5\34\17\3\u00d3\u00e3\3\2\2"+
+    "\2\u00d4\u00d5\7\t\2\2\u00d5\u00d6\5\34\17\2\u00d6\u00d7\7\n\2\2\u00d7"+
+    "\u00e3\3\2\2\2\u00d8\u00e3\t\5\2\2\u00d9\u00e3\7F\2\2\u00da\u00e3\7G\2"+
+    "\2\u00db\u00e3\7H\2\2\u00dc\u00dd\5\36\20\2\u00dd\u00de\t\6\2\2\u00de"+
+    "\u00e3\3\2\2\2\u00df\u00e0\t\6\2\2\u00e0\u00e3\5\36\20\2\u00e1\u00e3\5"+
+    "\36\20\2\u00e2\u00c8\3\2\2\2\u00e2\u00cb\3\2\2\2\u00e2\u00d0\3\2\2\2\u00e2"+
+    "\u00d4\3\2\2\2\u00e2\u00d8\3\2\2\2\u00e2\u00d9\3\2\2\2\u00e2\u00da\3\2"+
+    "\2\2\u00e2\u00db\3\2\2\2\u00e2\u00dc\3\2\2\2\u00e2\u00df\3\2\2\2\u00e2"+
+    "\u00e1\3\2\2\2\u00e3\u010a\3\2\2\2\u00e4\u00e5\f\16\2\2\u00e5\u00e6\t"+
+    "\7\2\2\u00e6\u0109\5\34\17\17\u00e7\u00e8\f\r\2\2\u00e8\u00e9\t\b\2\2"+
+    "\u00e9\u0109\5\34\17\16\u00ea\u00eb\f\f\2\2\u00eb\u00ec\t\t\2\2\u00ec"+
+    "\u0109\5\34\17\r\u00ed\u00ee\f\13\2\2\u00ee\u00ef\t\n\2\2\u00ef\u0109"+
+    "\5\34\17\f\u00f0\u00f1\f\n\2\2\u00f1\u00f2\t\13\2\2\u00f2\u0109\5\34\17"+
+    "\13\u00f3\u00f4\f\t\2\2\u00f4\u00f5\7,\2\2\u00f5\u0109\5\34\17\n\u00f6"+
+    "\u00f7\f\b\2\2\u00f7\u00f8\7-\2\2\u00f8\u0109\5\34\17\t\u00f9\u00fa\f"+
+    "\7\2\2\u00fa\u00fb\7.\2\2\u00fb\u0109\5\34\17\b\u00fc\u00fd\f\6\2\2\u00fd"+
+    "\u00fe\7/\2\2\u00fe\u0109\5\34\17\7\u00ff\u0100\f\5\2\2\u0100\u0101\7"+
+    "\60\2\2\u0101\u0109\5\34\17\6\u0102\u0103\f\4\2\2\u0103\u0104\7\61\2\2"+
+    "\u0104\u0105\5\34\17\2\u0105\u0106\7\62\2\2\u0106\u0107\5\34\17\4\u0107"+
+    "\u0109\3\2\2\2\u0108\u00e4\3\2\2\2\u0108\u00e7\3\2\2\2\u0108\u00ea\3\2"+
+    "\2\2\u0108\u00ed\3\2\2\2\u0108\u00f0\3\2\2\2\u0108\u00f3\3\2\2\2\u0108"+
+    "\u00f6\3\2\2\2\u0108\u00f9\3\2\2\2\u0108\u00fc\3\2\2\2\u0108\u00ff\3\2"+
+    "\2\2\u0108\u0102\3\2\2\2\u0109\u010c\3\2\2\2\u010a\u0108\3\2\2\2\u010a"+
+    "\u010b\3\2\2\2\u010b\35\3\2\2\2\u010c\u010a\3\2\2\2\u010d\u0113\5 \21"+
+    "\2\u010e\u0113\5\"\22\2\u010f\u0113\5*\26\2\u0110\u0113\5.\30\2\u0111"+
+    "\u0113\5\60\31\2\u0112\u010d\3\2\2\2\u0112\u010e\3\2\2\2\u0112\u010f\3"+
+    "\2\2\2\u0112\u0110\3\2\2\2\u0112\u0111\3\2\2\2\u0113\37\3\2\2\2\u0114"+
+    "\u011a\7\t\2\2\u0115\u011b\5 \21\2\u0116\u011b\5\"\22\2\u0117\u011b\5"+
+    "*\26\2\u0118\u011b\5.\30\2\u0119\u011b\5\60\31\2\u011a\u0115\3\2\2\2\u011a"+
+    "\u0116\3\2\2\2\u011a\u0117\3\2\2\2\u011a\u0118\3\2\2\2\u011a\u0119\3\2"+
+    "\2\2\u011b\u011c\3\2\2\2\u011c\u011f\7\n\2\2\u011d\u0120\5&\24\2\u011e"+
+    "\u0120\5$\23\2\u011f\u011d\3\2\2\2\u011f\u011e\3\2\2\2\u011f\u0120\3\2"+
+    "\2\2\u0120!\3\2\2\2\u0121\u0122\7\t\2\2\u0122\u0123\5\22\n\2\u0123\u0129"+
+    "\7\n\2\2\u0124\u012a\5 \21\2\u0125\u012a\5\"\22\2\u0126\u012a\5*\26\2"+
+    "\u0127\u012a\5.\30\2\u0128\u012a\5\60\31\2\u0129\u0124\3\2\2\2\u0129\u0125"+
+    "\3\2\2\2\u0129\u0126\3\2\2\2\u0129\u0127\3\2\2\2\u0129\u0128\3\2\2\2\u012a"+
+    "#\3\2\2\2\u012b\u012c\7\7\2\2\u012c\u012d\5\34\17\2\u012d\u0130\7\b\2"+
+    "\2\u012e\u0131\5&\24\2\u012f\u0131\5$\23\2\u0130\u012e\3\2\2\2\u0130\u012f"+
+    "\3\2\2\2\u0130\u0131\3\2\2\2\u0131%\3\2\2\2\u0132\u0135\7\13\2\2\u0133"+
+    "\u0136\5(\25\2\u0134\u0136\5,\27\2\u0135\u0133\3\2\2\2\u0135\u0134\3\2"+
+    "\2\2\u0136\'\3\2\2\2\u0137\u0138\7K\2\2\u0138\u013b\5\62\32\2\u0139\u013c"+
+    "\5&\24\2\u013a\u013c\5$\23\2\u013b\u0139\3\2\2\2\u013b\u013a\3\2\2\2\u013b"+
+    "\u013c\3\2\2\2\u013c)\3\2\2\2\u013d\u0140\5\30\r\2\u013e\u0141\5&\24\2"+
+    "\u013f\u0141\5$\23\2\u0140\u013e\3\2\2\2\u0140\u013f\3\2\2\2\u0140\u0141"+
+    "\3\2\2\2\u0141+\3\2\2\2\u0142\u0145\t\f\2\2\u0143\u0146\5&\24\2\u0144"+
+    "\u0146\5$\23\2\u0145\u0143\3\2\2\2\u0145\u0144\3\2\2\2\u0145\u0146\3\2"+
+    "\2\2\u0146-\3\2\2\2\u0147\u0148\7\26\2\2\u0148\u0158\5\30\r\2\u0149\u014b"+
+    "\5\62\32\2\u014a\u014c\5&\24\2\u014b\u014a\3\2\2\2\u014b\u014c\3\2\2\2"+
+    "\u014c\u0159\3\2\2\2\u014d\u014e\7\7\2\2\u014e\u014f\5\34\17\2\u014f\u0150"+
+    "\7\b\2\2\u0150\u0152\3\2\2\2\u0151\u014d\3\2\2\2\u0152\u0153\3\2\2\2\u0153"+
+    "\u0151\3\2\2\2\u0153\u0154\3\2\2\2\u0154\u0156\3\2\2\2\u0155\u0157\5&"+
+    "\24\2\u0156\u0155\3\2\2\2\u0156\u0157\3\2\2\2\u0157\u0159\3\2\2\2\u0158"+
+    "\u0149\3\2\2\2\u0158\u0151\3\2\2\2\u0159/\3\2\2\2\u015a\u015d\7E\2\2\u015b"+
+    "\u015e\5&\24\2\u015c\u015e\5$\23\2\u015d\u015b\3\2\2\2\u015d\u015c\3\2"+
+    "\2\2\u015d\u015e\3\2\2\2\u015e\61\3\2\2\2\u015f\u0168\7\t\2\2\u0160\u0165"+
+    "\5\34\17\2\u0161\u0162\7\f\2\2\u0162\u0164\5\34\17\2\u0163\u0161\3\2\2"+
+    "\2\u0164\u0167\3\2\2\2\u0165\u0163\3\2\2\2\u0165\u0166\3\2\2\2\u0166\u0169"+
+    "\3\2\2\2\u0167\u0165\3\2\2\2\u0168\u0160\3\2\2\2\u0168\u0169\3\2\2\2\u0169"+
+    "\u016a\3\2\2\2\u016a\u016b\7\n\2\2\u016b\63\3\2\2\2(\67BJW[_dv\177\u0085"+
+    "\u008a\u008e\u0095\u009f\u00a7\u00ad\u00b7\u00bb\u00c3\u00e2\u0108\u010a"+
+    "\u0112\u011a\u011f\u0129\u0130\u0135\u013b\u0140\u0145\u014b\u0153\u0156"+
+    "\u0158\u015d\u0165\u0168";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
index cd4bda1c071..22a9d50b991 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
@@ -21,894 +21,894 @@ package org.elasticsearch.painless;
 
 public class DefOperationTests extends ScriptTestCase {
     public void testNot() {
-        assertEquals(~1, exec("def x = (byte)1 return ~x"));
-        assertEquals(~1, exec("def x = (short)1 return ~x"));
-        assertEquals(~1, exec("def x = (char)1 return ~x"));
-        assertEquals(~1, exec("def x = 1 return ~x"));
-        assertEquals(~1L, exec("def x = 1L return ~x"));
+        assertEquals(~1, exec("def x = (byte)1; return ~x"));
+        assertEquals(~1, exec("def x = (short)1; return ~x"));
+        assertEquals(~1, exec("def x = (char)1; return ~x"));
+        assertEquals(~1, exec("def x = 1; return ~x"));
+        assertEquals(~1L, exec("def x = 1L; return ~x"));
     }
 
     public void testNeg() {
-        assertEquals(-1, exec("def x = (byte)1 return -x"));
-        assertEquals(-1, exec("def x = (short)1 return -x"));
-        assertEquals(-1, exec("def x = (char)1 return -x"));
-        assertEquals(-1, exec("def x = 1 return -x"));
-        assertEquals(-1L, exec("def x = 1L return -x"));
-        assertEquals(-1.0F, exec("def x = 1F return -x"));
-        assertEquals(-1.0, exec("def x = 1.0 return -x"));
+        assertEquals(-1, exec("def x = (byte)1; return -x"));
+        assertEquals(-1, exec("def x = (short)1; return -x"));
+        assertEquals(-1, exec("def x = (char)1; return -x"));
+        assertEquals(-1, exec("def x = 1; return -x"));
+        assertEquals(-1L, exec("def x = 1L; return -x"));
+        assertEquals(-1.0F, exec("def x = 1F; return -x"));
+        assertEquals(-1.0, exec("def x = 1.0; return -x"));
     }
 
     public void testMul() {
-        assertEquals(4, exec("def x = (byte)2 def y = (byte)2 return x * y"));
-        assertEquals(4, exec("def x = (short)2 def y = (byte)2 return x * y"));
-        assertEquals(4, exec("def x = (char)2 def y = (byte)2 return x * y"));
-        assertEquals(4, exec("def x = (int)2 def y = (byte)2 return x * y"));
-        assertEquals(4L, exec("def x = (long)2 def y = (byte)2 return x * y"));
-        assertEquals(4F, exec("def x = (float)2 def y = (byte)2 return x * y"));
-        assertEquals(4D, exec("def x = (double)2 def y = (byte)2 return x * y"));
+        assertEquals(4, exec("def x = (byte)2; def y = (byte)2; return x * y"));
+        assertEquals(4, exec("def x = (short)2; def y = (byte)2; return x * y"));
+        assertEquals(4, exec("def x = (char)2; def y = (byte)2; return x * y"));
+        assertEquals(4, exec("def x = (int)2; def y = (byte)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; def y = (byte)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; def y = (byte)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; def y = (byte)2; return x * y"));
 
-        assertEquals(4, exec("def x = (byte)2 def y = (short)2 return x * y"));
-        assertEquals(4, exec("def x = (short)2 def y = (short)2 return x * y"));
-        assertEquals(4, exec("def x = (char)2 def y = (short)2 return x * y"));
-        assertEquals(4, exec("def x = (int)2 def y = (short)2 return x * y"));
-        assertEquals(4L, exec("def x = (long)2 def y = (short)2 return x * y"));
-        assertEquals(4F, exec("def x = (float)2 def y = (short)2 return x * y"));
-        assertEquals(4D, exec("def x = (double)2 def y = (short)2 return x * y"));
+        assertEquals(4, exec("def x = (byte)2; def y = (short)2; return x * y"));
+        assertEquals(4, exec("def x = (short)2; def y = (short)2; return x * y"));
+        assertEquals(4, exec("def x = (char)2; def y = (short)2; return x * y"));
+        assertEquals(4, exec("def x = (int)2; def y = (short)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; def y = (short)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; def y = (short)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; def y = (short)2; return x * y"));
 
-        assertEquals(4, exec("def x = (byte)2 def y = (char)2 return x * y"));
-        assertEquals(4, exec("def x = (short)2 def y = (char)2 return x * y"));
-        assertEquals(4, exec("def x = (char)2 def y = (char)2 return x * y"));
-        assertEquals(4, exec("def x = (int)2 def y = (char)2 return x * y"));
-        assertEquals(4L, exec("def x = (long)2 def y = (char)2 return x * y"));
-        assertEquals(4F, exec("def x = (float)2 def y = (char)2 return x * y"));
-        assertEquals(4D, exec("def x = (double)2 def y = (char)2 return x * y"));
+        assertEquals(4, exec("def x = (byte)2; def y = (char)2; return x * y"));
+        assertEquals(4, exec("def x = (short)2; def y = (char)2; return x * y"));
+        assertEquals(4, exec("def x = (char)2; def y = (char)2; return x * y"));
+        assertEquals(4, exec("def x = (int)2; def y = (char)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; def y = (char)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; def y = (char)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; def y = (char)2; return x * y"));
 
-        assertEquals(4, exec("def x = (byte)2 def y = (int)2 return x * y"));
-        assertEquals(4, exec("def x = (short)2 def y = (int)2 return x * y"));
-        assertEquals(4, exec("def x = (char)2 def y = (int)2 return x * y"));
-        assertEquals(4, exec("def x = (int)2 def y = (int)2 return x * y"));
-        assertEquals(4L, exec("def x = (long)2 def y = (int)2 return x * y"));
-        assertEquals(4F, exec("def x = (float)2 def y = (int)2 return x * y"));
-        assertEquals(4D, exec("def x = (double)2 def y = (int)2 return x * y"));
+        assertEquals(4, exec("def x = (byte)2; def y = (int)2; return x * y"));
+        assertEquals(4, exec("def x = (short)2; def y = (int)2; return x * y"));
+        assertEquals(4, exec("def x = (char)2; def y = (int)2; return x * y"));
+        assertEquals(4, exec("def x = (int)2; def y = (int)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; def y = (int)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; def y = (int)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; def y = (int)2; return x * y"));
 
-        assertEquals(4L, exec("def x = (byte)2 def y = (long)2 return x * y"));
-        assertEquals(4L, exec("def x = (short)2 def y = (long)2 return x * y"));
-        assertEquals(4L, exec("def x = (char)2 def y = (long)2 return x * y"));
-        assertEquals(4L, exec("def x = (int)2 def y = (long)2 return x * y"));
-        assertEquals(4L, exec("def x = (long)2 def y = (long)2 return x * y"));
-        assertEquals(4F, exec("def x = (float)2 def y = (long)2 return x * y"));
-        assertEquals(4D, exec("def x = (double)2 def y = (long)2 return x * y"));
+        assertEquals(4L, exec("def x = (byte)2; def y = (long)2; return x * y"));
+        assertEquals(4L, exec("def x = (short)2; def y = (long)2; return x * y"));
+        assertEquals(4L, exec("def x = (char)2; def y = (long)2; return x * y"));
+        assertEquals(4L, exec("def x = (int)2; def y = (long)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; def y = (long)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; def y = (long)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; def y = (long)2; return x * y"));
 
-        assertEquals(4F, exec("def x = (byte)2 def y = (float)2 return x * y"));
-        assertEquals(4F, exec("def x = (short)2 def y = (float)2 return x * y"));
-        assertEquals(4F, exec("def x = (char)2 def y = (float)2 return x * y"));
-        assertEquals(4F, exec("def x = (int)2 def y = (float)2 return x * y"));
-        assertEquals(4F, exec("def x = (long)2 def y = (float)2 return x * y"));
-        assertEquals(4F, exec("def x = (float)2 def y = (float)2 return x * y"));
-        assertEquals(4D, exec("def x = (double)2 def y = (float)2 return x * y"));
+        assertEquals(4F, exec("def x = (byte)2; def y = (float)2; return x * y"));
+        assertEquals(4F, exec("def x = (short)2; def y = (float)2; return x * y"));
+        assertEquals(4F, exec("def x = (char)2; def y = (float)2; return x * y"));
+        assertEquals(4F, exec("def x = (int)2; def y = (float)2; return x * y"));
+        assertEquals(4F, exec("def x = (long)2; def y = (float)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; def y = (float)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; def y = (float)2; return x * y"));
 
-        assertEquals(4D, exec("def x = (byte)2 def y = (double)2 return x * y"));
-        assertEquals(4D, exec("def x = (short)2 def y = (double)2 return x * y"));
-        assertEquals(4D, exec("def x = (char)2 def y = (double)2 return x * y"));
-        assertEquals(4D, exec("def x = (int)2 def y = (double)2 return x * y"));
-        assertEquals(4D, exec("def x = (long)2 def y = (double)2 return x * y"));
-        assertEquals(4D, exec("def x = (float)2 def y = (double)2 return x * y"));
-        assertEquals(4D, exec("def x = (double)2 def y = (double)2 return x * y"));
+        assertEquals(4D, exec("def x = (byte)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (short)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (char)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (int)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (long)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (float)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; def y = (double)2; return x * y"));
 
-        assertEquals(4, exec("def x = (Byte)2 def y = (byte)2 return x * y"));
-        assertEquals(4, exec("def x = (Short)2 def y = (short)2 return x * y"));
-        assertEquals(4, exec("def x = (Character)2 def y = (char)2 return x * y"));
-        assertEquals(4, exec("def x = (Integer)2 def y = (int)2 return x * y"));
-        assertEquals(4L, exec("def x = (Long)2 def y = (long)2 return x * y"));
-        assertEquals(4F, exec("def x = (Float)2 def y = (float)2 return x * y"));
-        assertEquals(4D, exec("def x = (Double)2 def y = (double)2 return x * y"));
+        assertEquals(4, exec("def x = (Byte)2; def y = (byte)2; return x * y"));
+        assertEquals(4, exec("def x = (Short)2; def y = (short)2; return x * y"));
+        assertEquals(4, exec("def x = (Character)2; def y = (char)2; return x * y"));
+        assertEquals(4, exec("def x = (Integer)2; def y = (int)2; return x * y"));
+        assertEquals(4L, exec("def x = (Long)2; def y = (long)2; return x * y"));
+        assertEquals(4F, exec("def x = (Float)2; def y = (float)2; return x * y"));
+        assertEquals(4D, exec("def x = (Double)2; def y = (double)2; return x * y"));
     }
 
     public void testDiv() {
-        assertEquals(1, exec("def x = (byte)2 def y = (byte)2 return x / y"));
-        assertEquals(1, exec("def x = (short)2 def y = (byte)2 return x / y"));
-        assertEquals(1, exec("def x = (char)2 def y = (byte)2 return x / y"));
-        assertEquals(1, exec("def x = (int)2 def y = (byte)2 return x / y"));
-        assertEquals(1L, exec("def x = (long)2 def y = (byte)2 return x / y"));
-        assertEquals(1F, exec("def x = (float)2 def y = (byte)2 return x / y"));
-        assertEquals(1D, exec("def x = (double)2 def y = (byte)2 return x / y"));
+        assertEquals(1, exec("def x = (byte)2; def y = (byte)2; return x / y"));
+        assertEquals(1, exec("def x = (short)2; def y = (byte)2; return x / y"));
+        assertEquals(1, exec("def x = (char)2; def y = (byte)2; return x / y"));
+        assertEquals(1, exec("def x = (int)2; def y = (byte)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; def y = (byte)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; def y = (byte)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; def y = (byte)2; return x / y"));
 
-        assertEquals(1, exec("def x = (byte)2 def y = (short)2 return x / y"));
-        assertEquals(1, exec("def x = (short)2 def y = (short)2 return x / y"));
-        assertEquals(1, exec("def x = (char)2 def y = (short)2 return x / y"));
-        assertEquals(1, exec("def x = (int)2 def y = (short)2 return x / y"));
-        assertEquals(1L, exec("def x = (long)2 def y = (short)2 return x / y"));
-        assertEquals(1F, exec("def x = (float)2 def y = (short)2 return x / y"));
-        assertEquals(1D, exec("def x = (double)2 def y = (short)2 return x / y"));
+        assertEquals(1, exec("def x = (byte)2; def y = (short)2; return x / y"));
+        assertEquals(1, exec("def x = (short)2; def y = (short)2; return x / y"));
+        assertEquals(1, exec("def x = (char)2; def y = (short)2; return x / y"));
+        assertEquals(1, exec("def x = (int)2; def y = (short)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; def y = (short)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; def y = (short)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; def y = (short)2; return x / y"));
 
-        assertEquals(1, exec("def x = (byte)2 def y = (char)2 return x / y"));
-        assertEquals(1, exec("def x = (short)2 def y = (char)2 return x / y"));
-        assertEquals(1, exec("def x = (char)2 def y = (char)2 return x / y"));
-        assertEquals(1, exec("def x = (int)2 def y = (char)2 return x / y"));
-        assertEquals(1L, exec("def x = (long)2 def y = (char)2 return x / y"));
-        assertEquals(1F, exec("def x = (float)2 def y = (char)2 return x / y"));
-        assertEquals(1D, exec("def x = (double)2 def y = (char)2 return x / y"));
+        assertEquals(1, exec("def x = (byte)2; def y = (char)2; return x / y"));
+        assertEquals(1, exec("def x = (short)2; def y = (char)2; return x / y"));
+        assertEquals(1, exec("def x = (char)2; def y = (char)2; return x / y"));
+        assertEquals(1, exec("def x = (int)2; def y = (char)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; def y = (char)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; def y = (char)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; def y = (char)2; return x / y"));
 
-        assertEquals(1, exec("def x = (byte)2 def y = (int)2 return x / y"));
-        assertEquals(1, exec("def x = (short)2 def y = (int)2 return x / y"));
-        assertEquals(1, exec("def x = (char)2 def y = (int)2 return x / y"));
-        assertEquals(1, exec("def x = (int)2 def y = (int)2 return x / y"));
-        assertEquals(1L, exec("def x = (long)2 def y = (int)2 return x / y"));
-        assertEquals(1F, exec("def x = (float)2 def y = (int)2 return x / y"));
-        assertEquals(1D, exec("def x = (double)2 def y = (int)2 return x / y"));
+        assertEquals(1, exec("def x = (byte)2; def y = (int)2; return x / y"));
+        assertEquals(1, exec("def x = (short)2; def y = (int)2; return x / y"));
+        assertEquals(1, exec("def x = (char)2; def y = (int)2; return x / y"));
+        assertEquals(1, exec("def x = (int)2; def y = (int)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; def y = (int)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; def y = (int)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; def y = (int)2; return x / y"));
 
-        assertEquals(1L, exec("def x = (byte)2 def y = (long)2 return x / y"));
-        assertEquals(1L, exec("def x = (short)2 def y = (long)2 return x / y"));
-        assertEquals(1L, exec("def x = (char)2 def y = (long)2 return x / y"));
-        assertEquals(1L, exec("def x = (int)2 def y = (long)2 return x / y"));
-        assertEquals(1L, exec("def x = (long)2 def y = (long)2 return x / y"));
-        assertEquals(1F, exec("def x = (float)2 def y = (long)2 return x / y"));
-        assertEquals(1D, exec("def x = (double)2 def y = (long)2 return x / y"));
+        assertEquals(1L, exec("def x = (byte)2; def y = (long)2; return x / y"));
+        assertEquals(1L, exec("def x = (short)2; def y = (long)2; return x / y"));
+        assertEquals(1L, exec("def x = (char)2; def y = (long)2; return x / y"));
+        assertEquals(1L, exec("def x = (int)2; def y = (long)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; def y = (long)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; def y = (long)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; def y = (long)2; return x / y"));
 
-        assertEquals(1F, exec("def x = (byte)2 def y = (float)2 return x / y"));
-        assertEquals(1F, exec("def x = (short)2 def y = (float)2 return x / y"));
-        assertEquals(1F, exec("def x = (char)2 def y = (float)2 return x / y"));
-        assertEquals(1F, exec("def x = (int)2 def y = (float)2 return x / y"));
-        assertEquals(1F, exec("def x = (long)2 def y = (float)2 return x / y"));
-        assertEquals(1F, exec("def x = (float)2 def y = (float)2 return x / y"));
-        assertEquals(1D, exec("def x = (double)2 def y = (float)2 return x / y"));
+        assertEquals(1F, exec("def x = (byte)2; def y = (float)2; return x / y"));
+        assertEquals(1F, exec("def x = (short)2; def y = (float)2; return x / y"));
+        assertEquals(1F, exec("def x = (char)2; def y = (float)2; return x / y"));
+        assertEquals(1F, exec("def x = (int)2; def y = (float)2; return x / y"));
+        assertEquals(1F, exec("def x = (long)2; def y = (float)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; def y = (float)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; def y = (float)2; return x / y"));
 
-        assertEquals(1D, exec("def x = (byte)2 def y = (double)2 return x / y"));
-        assertEquals(1D, exec("def x = (short)2 def y = (double)2 return x / y"));
-        assertEquals(1D, exec("def x = (char)2 def y = (double)2 return x / y"));
-        assertEquals(1D, exec("def x = (int)2 def y = (double)2 return x / y"));
-        assertEquals(1D, exec("def x = (long)2 def y = (double)2 return x / y"));
-        assertEquals(1D, exec("def x = (float)2 def y = (double)2 return x / y"));
-        assertEquals(1D, exec("def x = (double)2 def y = (double)2 return x / y"));
+        assertEquals(1D, exec("def x = (byte)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (short)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (char)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (int)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (long)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (float)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; def y = (double)2; return x / y"));
 
-        assertEquals(1, exec("def x = (Byte)2 def y = (byte)2 return x / y"));
-        assertEquals(1, exec("def x = (Short)2 def y = (short)2 return x / y"));
-        assertEquals(1, exec("def x = (Character)2 def y = (char)2 return x / y"));
-        assertEquals(1, exec("def x = (Integer)2 def y = (int)2 return x / y"));
-        assertEquals(1L, exec("def x = (Long)2 def y = (long)2 return x / y"));
-        assertEquals(1F, exec("def x = (Float)2 def y = (float)2 return x / y"));
-        assertEquals(1D, exec("def x = (Double)2 def y = (double)2 return x / y"));
+        assertEquals(1, exec("def x = (Byte)2; def y = (byte)2; return x / y"));
+        assertEquals(1, exec("def x = (Short)2; def y = (short)2; return x / y"));
+        assertEquals(1, exec("def x = (Character)2; def y = (char)2; return x / y"));
+        assertEquals(1, exec("def x = (Integer)2; def y = (int)2; return x / y"));
+        assertEquals(1L, exec("def x = (Long)2; def y = (long)2; return x / y"));
+        assertEquals(1F, exec("def x = (Float)2; def y = (float)2; return x / y"));
+        assertEquals(1D, exec("def x = (Double)2; def y = (double)2; return x / y"));
     }
 
     public void testRem() {
-        assertEquals(0, exec("def x = (byte)2 def y = (byte)2 return x % y"));
-        assertEquals(0, exec("def x = (short)2 def y = (byte)2 return x % y"));
-        assertEquals(0, exec("def x = (char)2 def y = (byte)2 return x % y"));
-        assertEquals(0, exec("def x = (int)2 def y = (byte)2 return x % y"));
-        assertEquals(0L, exec("def x = (long)2 def y = (byte)2 return x % y"));
-        assertEquals(0F, exec("def x = (float)2 def y = (byte)2 return x % y"));
-        assertEquals(0D, exec("def x = (double)2 def y = (byte)2 return x % y"));
+        assertEquals(0, exec("def x = (byte)2; def y = (byte)2; return x % y"));
+        assertEquals(0, exec("def x = (short)2; def y = (byte)2; return x % y"));
+        assertEquals(0, exec("def x = (char)2; def y = (byte)2; return x % y"));
+        assertEquals(0, exec("def x = (int)2; def y = (byte)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; def y = (byte)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; def y = (byte)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; def y = (byte)2; return x % y"));
 
-        assertEquals(0, exec("def x = (byte)2 def y = (short)2 return x % y"));
-        assertEquals(0, exec("def x = (short)2 def y = (short)2 return x % y"));
-        assertEquals(0, exec("def x = (char)2 def y = (short)2 return x % y"));
-        assertEquals(0, exec("def x = (int)2 def y = (short)2 return x % y"));
-        assertEquals(0L, exec("def x = (long)2 def y = (short)2 return x % y"));
-        assertEquals(0F, exec("def x = (float)2 def y = (short)2 return x % y"));
-        assertEquals(0D, exec("def x = (double)2 def y = (short)2 return x % y"));
+        assertEquals(0, exec("def x = (byte)2; def y = (short)2; return x % y"));
+        assertEquals(0, exec("def x = (short)2; def y = (short)2; return x % y"));
+        assertEquals(0, exec("def x = (char)2; def y = (short)2; return x % y"));
+        assertEquals(0, exec("def x = (int)2; def y = (short)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; def y = (short)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; def y = (short)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; def y = (short)2; return x % y"));
 
-        assertEquals(0, exec("def x = (byte)2 def y = (char)2 return x % y"));
-        assertEquals(0, exec("def x = (short)2 def y = (char)2 return x % y"));
-        assertEquals(0, exec("def x = (char)2 def y = (char)2 return x % y"));
-        assertEquals(0, exec("def x = (int)2 def y = (char)2 return x % y"));
-        assertEquals(0L, exec("def x = (long)2 def y = (char)2 return x % y"));
-        assertEquals(0F, exec("def x = (float)2 def y = (char)2 return x % y"));
-        assertEquals(0D, exec("def x = (double)2 def y = (char)2 return x % y"));
+        assertEquals(0, exec("def x = (byte)2; def y = (char)2; return x % y"));
+        assertEquals(0, exec("def x = (short)2; def y = (char)2; return x % y"));
+        assertEquals(0, exec("def x = (char)2; def y = (char)2; return x % y"));
+        assertEquals(0, exec("def x = (int)2; def y = (char)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; def y = (char)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; def y = (char)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; def y = (char)2; return x % y"));
 
-        assertEquals(0, exec("def x = (byte)2 def y = (int)2 return x % y"));
-        assertEquals(0, exec("def x = (short)2 def y = (int)2 return x % y"));
-        assertEquals(0, exec("def x = (char)2 def y = (int)2 return x % y"));
-        assertEquals(0, exec("def x = (int)2 def y = (int)2 return x % y"));
-        assertEquals(0L, exec("def x = (long)2 def y = (int)2 return x % y"));
-        assertEquals(0F, exec("def x = (float)2 def y = (int)2 return x % y"));
-        assertEquals(0D, exec("def x = (double)2 def y = (int)2 return x % y"));
+        assertEquals(0, exec("def x = (byte)2; def y = (int)2; return x % y"));
+        assertEquals(0, exec("def x = (short)2; def y = (int)2; return x % y"));
+        assertEquals(0, exec("def x = (char)2; def y = (int)2; return x % y"));
+        assertEquals(0, exec("def x = (int)2; def y = (int)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; def y = (int)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; def y = (int)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; def y = (int)2; return x % y"));
 
-        assertEquals(0L, exec("def x = (byte)2 def y = (long)2 return x % y"));
-        assertEquals(0L, exec("def x = (short)2 def y = (long)2 return x % y"));
-        assertEquals(0L, exec("def x = (char)2 def y = (long)2 return x % y"));
-        assertEquals(0L, exec("def x = (int)2 def y = (long)2 return x % y"));
-        assertEquals(0L, exec("def x = (long)2 def y = (long)2 return x % y"));
-        assertEquals(0F, exec("def x = (float)2 def y = (long)2 return x % y"));
-        assertEquals(0D, exec("def x = (double)2 def y = (long)2 return x % y"));
+        assertEquals(0L, exec("def x = (byte)2; def y = (long)2; return x % y"));
+        assertEquals(0L, exec("def x = (short)2; def y = (long)2; return x % y"));
+        assertEquals(0L, exec("def x = (char)2; def y = (long)2; return x % y"));
+        assertEquals(0L, exec("def x = (int)2; def y = (long)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; def y = (long)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; def y = (long)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; def y = (long)2; return x % y"));
 
-        assertEquals(0F, exec("def x = (byte)2 def y = (float)2 return x % y"));
-        assertEquals(0F, exec("def x = (short)2 def y = (float)2 return x % y"));
-        assertEquals(0F, exec("def x = (char)2 def y = (float)2 return x % y"));
-        assertEquals(0F, exec("def x = (int)2 def y = (float)2 return x % y"));
-        assertEquals(0F, exec("def x = (long)2 def y = (float)2 return x % y"));
-        assertEquals(0F, exec("def x = (float)2 def y = (float)2 return x % y"));
-        assertEquals(0D, exec("def x = (double)2 def y = (float)2 return x % y"));
+        assertEquals(0F, exec("def x = (byte)2; def y = (float)2; return x % y"));
+        assertEquals(0F, exec("def x = (short)2; def y = (float)2; return x % y"));
+        assertEquals(0F, exec("def x = (char)2; def y = (float)2; return x % y"));
+        assertEquals(0F, exec("def x = (int)2; def y = (float)2; return x % y"));
+        assertEquals(0F, exec("def x = (long)2; def y = (float)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; def y = (float)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; def y = (float)2; return x % y"));
 
-        assertEquals(0D, exec("def x = (byte)2 def y = (double)2 return x % y"));
-        assertEquals(0D, exec("def x = (short)2 def y = (double)2 return x % y"));
-        assertEquals(0D, exec("def x = (char)2 def y = (double)2 return x % y"));
-        assertEquals(0D, exec("def x = (int)2 def y = (double)2 return x % y"));
-        assertEquals(0D, exec("def x = (long)2 def y = (double)2 return x % y"));
-        assertEquals(0D, exec("def x = (float)2 def y = (double)2 return x % y"));
-        assertEquals(0D, exec("def x = (double)2 def y = (double)2 return x % y"));
+        assertEquals(0D, exec("def x = (byte)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (short)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (char)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (int)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (long)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (float)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; def y = (double)2; return x % y"));
 
-        assertEquals(0, exec("def x = (Byte)2 def y = (byte)2 return x % y"));
-        assertEquals(0, exec("def x = (Short)2 def y = (short)2 return x % y"));
-        assertEquals(0, exec("def x = (Character)2 def y = (char)2 return x % y"));
-        assertEquals(0, exec("def x = (Integer)2 def y = (int)2 return x % y"));
-        assertEquals(0L, exec("def x = (Long)2 def y = (long)2 return x % y"));
-        assertEquals(0F, exec("def x = (Float)2 def y = (float)2 return x % y"));
-        assertEquals(0D, exec("def x = (Double)2 def y = (double)2 return x % y"));
+        assertEquals(0, exec("def x = (Byte)2; def y = (byte)2; return x % y"));
+        assertEquals(0, exec("def x = (Short)2; def y = (short)2; return x % y"));
+        assertEquals(0, exec("def x = (Character)2; def y = (char)2; return x % y"));
+        assertEquals(0, exec("def x = (Integer)2; def y = (int)2; return x % y"));
+        assertEquals(0L, exec("def x = (Long)2; def y = (long)2; return x % y"));
+        assertEquals(0F, exec("def x = (Float)2; def y = (float)2; return x % y"));
+        assertEquals(0D, exec("def x = (Double)2; def y = (double)2; return x % y"));
     }
 
     public void testAdd() {
-        assertEquals(2, exec("def x = (byte)1 def y = (byte)1 return x + y"));
-        assertEquals(2, exec("def x = (short)1 def y = (byte)1 return x + y"));
-        assertEquals(2, exec("def x = (char)1 def y = (byte)1 return x + y"));
-        assertEquals(2, exec("def x = (int)1 def y = (byte)1 return x + y"));
-        assertEquals(2L, exec("def x = (long)1 def y = (byte)1 return x + y"));
-        assertEquals(2F, exec("def x = (float)1 def y = (byte)1 return x + y"));
-        assertEquals(2D, exec("def x = (double)1 def y = (byte)1 return x + y"));
+        assertEquals(2, exec("def x = (byte)1; def y = (byte)1; return x + y"));
+        assertEquals(2, exec("def x = (short)1; def y = (byte)1; return x + y"));
+        assertEquals(2, exec("def x = (char)1; def y = (byte)1; return x + y"));
+        assertEquals(2, exec("def x = (int)1; def y = (byte)1; return x + y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (byte)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; def y = (byte)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; def y = (byte)1; return x + y"));
 
-        assertEquals(2, exec("def x = (byte)1 def y = (short)1 return x + y"));
-        assertEquals(2, exec("def x = (short)1 def y = (short)1 return x + y"));
-        assertEquals(2, exec("def x = (char)1 def y = (short)1 return x + y"));
-        assertEquals(2, exec("def x = (int)1 def y = (short)1 return x + y"));
-        assertEquals(2L, exec("def x = (long)1 def y = (short)1 return x + y"));
-        assertEquals(2F, exec("def x = (float)1 def y = (short)1 return x + y"));
-        assertEquals(2D, exec("def x = (double)1 def y = (short)1 return x + y"));
+        assertEquals(2, exec("def x = (byte)1; def y = (short)1; return x + y"));
+        assertEquals(2, exec("def x = (short)1; def y = (short)1; return x + y"));
+        assertEquals(2, exec("def x = (char)1; def y = (short)1; return x + y"));
+        assertEquals(2, exec("def x = (int)1; def y = (short)1; return x + y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (short)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; def y = (short)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; def y = (short)1; return x + y"));
 
-        assertEquals(2, exec("def x = (byte)1 def y = (char)1 return x + y"));
-        assertEquals(2, exec("def x = (short)1 def y = (char)1 return x + y"));
-        assertEquals(2, exec("def x = (char)1 def y = (char)1 return x + y"));
-        assertEquals(2, exec("def x = (int)1 def y = (char)1 return x + y"));
-        assertEquals(2L, exec("def x = (long)1 def y = (char)1 return x + y"));
-        assertEquals(2F, exec("def x = (float)1 def y = (char)1 return x + y"));
-        assertEquals(2D, exec("def x = (double)1 def y = (char)1 return x + y"));
+        assertEquals(2, exec("def x = (byte)1; def y = (char)1; return x + y"));
+        assertEquals(2, exec("def x = (short)1; def y = (char)1; return x + y"));
+        assertEquals(2, exec("def x = (char)1; def y = (char)1; return x + y"));
+        assertEquals(2, exec("def x = (int)1; def y = (char)1; return x + y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (char)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; def y = (char)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; def y = (char)1; return x + y"));
 
-        assertEquals(2, exec("def x = (byte)1 def y = (int)1 return x + y"));
-        assertEquals(2, exec("def x = (short)1 def y = (int)1 return x + y"));
-        assertEquals(2, exec("def x = (char)1 def y = (int)1 return x + y"));
-        assertEquals(2, exec("def x = (int)1 def y = (int)1 return x + y"));
-        assertEquals(2L, exec("def x = (long)1 def y = (int)1 return x + y"));
-        assertEquals(2F, exec("def x = (float)1 def y = (int)1 return x + y"));
-        assertEquals(2D, exec("def x = (double)1 def y = (int)1 return x + y"));
+        assertEquals(2, exec("def x = (byte)1; def y = (int)1; return x + y"));
+        assertEquals(2, exec("def x = (short)1; def y = (int)1; return x + y"));
+        assertEquals(2, exec("def x = (char)1; def y = (int)1; return x + y"));
+        assertEquals(2, exec("def x = (int)1; def y = (int)1; return x + y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (int)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; def y = (int)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; def y = (int)1; return x + y"));
 
-        assertEquals(2L, exec("def x = (byte)1 def y = (long)1 return x + y"));
-        assertEquals(2L, exec("def x = (short)1 def y = (long)1 return x + y"));
-        assertEquals(2L, exec("def x = (char)1 def y = (long)1 return x + y"));
-        assertEquals(2L, exec("def x = (int)1 def y = (long)1 return x + y"));
-        assertEquals(2L, exec("def x = (long)1 def y = (long)1 return x + y"));
-        assertEquals(2F, exec("def x = (float)1 def y = (long)1 return x + y"));
-        assertEquals(2D, exec("def x = (double)1 def y = (long)1 return x + y"));
+        assertEquals(2L, exec("def x = (byte)1; def y = (long)1; return x + y"));
+        assertEquals(2L, exec("def x = (short)1; def y = (long)1; return x + y"));
+        assertEquals(2L, exec("def x = (char)1; def y = (long)1; return x + y"));
+        assertEquals(2L, exec("def x = (int)1; def y = (long)1; return x + y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (long)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; def y = (long)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; def y = (long)1; return x + y"));
 
-        assertEquals(2F, exec("def x = (byte)1 def y = (float)1 return x + y"));
-        assertEquals(2F, exec("def x = (short)1 def y = (float)1 return x + y"));
-        assertEquals(2F, exec("def x = (char)1 def y = (float)1 return x + y"));
-        assertEquals(2F, exec("def x = (int)1 def y = (float)1 return x + y"));
-        assertEquals(2F, exec("def x = (long)1 def y = (float)1 return x + y"));
-        assertEquals(2F, exec("def x = (float)1 def y = (float)1 return x + y"));
-        assertEquals(2D, exec("def x = (double)1 def y = (float)1 return x + y"));
+        assertEquals(2F, exec("def x = (byte)1; def y = (float)1; return x + y"));
+        assertEquals(2F, exec("def x = (short)1; def y = (float)1; return x + y"));
+        assertEquals(2F, exec("def x = (char)1; def y = (float)1; return x + y"));
+        assertEquals(2F, exec("def x = (int)1; def y = (float)1; return x + y"));
+        assertEquals(2F, exec("def x = (long)1; def y = (float)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; def y = (float)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; def y = (float)1; return x + y"));
 
-        assertEquals(2D, exec("def x = (byte)1 def y = (double)1 return x + y"));
-        assertEquals(2D, exec("def x = (short)1 def y = (double)1 return x + y"));
-        assertEquals(2D, exec("def x = (char)1 def y = (double)1 return x + y"));
-        assertEquals(2D, exec("def x = (int)1 def y = (double)1 return x + y"));
-        assertEquals(2D, exec("def x = (long)1 def y = (double)1 return x + y"));
-        assertEquals(2D, exec("def x = (float)1 def y = (double)1 return x + y"));
-        assertEquals(2D, exec("def x = (double)1 def y = (double)1 return x + y"));
+        assertEquals(2D, exec("def x = (byte)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (short)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (char)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (int)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (long)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (float)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; def y = (double)1; return x + y"));
 
-        assertEquals(2, exec("def x = (Byte)1 def y = (byte)1 return x + y"));
-        assertEquals(2, exec("def x = (Short)1 def y = (short)1 return x + y"));
-        assertEquals(2, exec("def x = (Character)1 def y = (char)1 return x + y"));
-        assertEquals(2, exec("def x = (Integer)1 def y = (int)1 return x + y"));
-        assertEquals(2L, exec("def x = (Long)1 def y = (long)1 return x + y"));
-        assertEquals(2F, exec("def x = (Float)1 def y = (float)1 return x + y"));
-        assertEquals(2D, exec("def x = (Double)1 def y = (double)1 return x + y"));
+        assertEquals(2, exec("def x = (Byte)1; def y = (byte)1; return x + y"));
+        assertEquals(2, exec("def x = (Short)1; def y = (short)1; return x + y"));
+        assertEquals(2, exec("def x = (Character)1; def y = (char)1; return x + y"));
+        assertEquals(2, exec("def x = (Integer)1; def y = (int)1; return x + y"));
+        assertEquals(2L, exec("def x = (Long)1; def y = (long)1; return x + y"));
+        assertEquals(2F, exec("def x = (Float)1; def y = (float)1; return x + y"));
+        assertEquals(2D, exec("def x = (Double)1; def y = (double)1; return x + y"));
     }
 
     public void testSub() {
-        assertEquals(0, exec("def x = (byte)1 def y = (byte)1 return x - y"));
-        assertEquals(0, exec("def x = (short)1 def y = (byte)1 return x - y"));
-        assertEquals(0, exec("def x = (char)1 def y = (byte)1 return x - y"));
-        assertEquals(0, exec("def x = (int)1 def y = (byte)1 return x - y"));
-        assertEquals(0L, exec("def x = (long)1 def y = (byte)1 return x - y"));
-        assertEquals(0F, exec("def x = (float)1 def y = (byte)1 return x - y"));
-        assertEquals(0D, exec("def x = (double)1 def y = (byte)1 return x - y"));
+        assertEquals(0, exec("def x = (byte)1; def y = (byte)1; return x - y"));
+        assertEquals(0, exec("def x = (short)1; def y = (byte)1; return x - y"));
+        assertEquals(0, exec("def x = (char)1; def y = (byte)1; return x - y"));
+        assertEquals(0, exec("def x = (int)1; def y = (byte)1; return x - y"));
+        assertEquals(0L, exec("def x = (long)1; def y = (byte)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; def y = (byte)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; def y = (byte)1; return x - y"));
 
-        assertEquals(0, exec("def x = (byte)1 def y = (short)1 return x - y"));
-        assertEquals(0, exec("def x = (short)1 def y = (short)1 return x - y"));
-        assertEquals(0, exec("def x = (char)1 def y = (short)1 return x - y"));
-        assertEquals(0, exec("def x = (int)1 def y = (short)1 return x - y"));
-        assertEquals(0L, exec("def x = (long)1 def y = (short)1 return x - y"));
-        assertEquals(0F, exec("def x = (float)1 def y = (short)1 return x - y"));
-        assertEquals(0D, exec("def x = (double)1 def y = (short)1 return x - y"));
+        assertEquals(0, exec("def x = (byte)1; def y = (short)1; return x - y"));
+        assertEquals(0, exec("def x = (short)1; def y = (short)1; return x - y"));
+        assertEquals(0, exec("def x = (char)1; def y = (short)1; return x - y"));
+        assertEquals(0, exec("def x = (int)1; def y = (short)1; return x - y"));
+        assertEquals(0L, exec("def x = (long)1; def y = (short)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; def y = (short)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; def y = (short)1; return x - y"));
 
-        assertEquals(0, exec("def x = (byte)1 def y = (char)1 return x - y"));
-        assertEquals(0, exec("def x = (short)1 def y = (char)1 return x - y"));
-        assertEquals(0, exec("def x = (char)1 def y = (char)1 return x - y"));
-        assertEquals(0, exec("def x = (int)1 def y = (char)1 return x - y"));
-        assertEquals(0L, exec("def x = (long)1 def y = (char)1 return x - y"));
-        assertEquals(0F, exec("def x = (float)1 def y = (char)1 return x - y"));
-        assertEquals(0D, exec("def x = (double)1 def y = (char)1 return x - y"));
+        assertEquals(0, exec("def x = (byte)1; def y = (char)1; return x - y"));
+        assertEquals(0, exec("def x = (short)1; def y = (char)1; return x - y"));
+        assertEquals(0, exec("def x = (char)1; def y = (char)1; return x - y"));
+        assertEquals(0, exec("def x = (int)1; def y = (char)1; return x - y"));
+        assertEquals(0L, exec("def x = (long)1; def y = (char)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; def y = (char)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; def y = (char)1; return x - y"));
 
-        assertEquals(0, exec("def x = (byte)1 def y = (int)1 return x - y"));
-        assertEquals(0, exec("def x = (short)1 def y = (int)1 return x - y"));
-        assertEquals(0, exec("def x = (char)1 def y = (int)1 return x - y"));
-        assertEquals(0, exec("def x = (int)1 def y = (int)1 return x - y"));
-        assertEquals(0L, exec("def x = (long)1 def y = (int)1 return x - y"));
-        assertEquals(0F, exec("def x = (float)1 def y = (int)1 return x - y"));
-        assertEquals(0D, exec("def x = (double)1 def y = (int)1 return x - y"));
+        assertEquals(0, exec("def x = (byte)1; def y = (int)1; return x - y"));
+        assertEquals(0, exec("def x = (short)1; def y = (int)1; return x - y"));
+        assertEquals(0, exec("def x = (char)1; def y = (int)1; return x - y"));
+        assertEquals(0, exec("def x = (int)1; def y = (int)1; return x - y"));
+        assertEquals(0L, exec("def x = (long)1; def y = (int)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; def y = (int)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; def y = (int)1; return x - y"));
 
-        assertEquals(0L, exec("def x = (byte)1 def y = (long)1 return x - y"));
-        assertEquals(0L, exec("def x = (short)1 def y = (long)1 return x - y"));
-        assertEquals(0L, exec("def x = (char)1 def y = (long)1 return x - y"));
-        assertEquals(0L, exec("def x = (int)1 def y = (long)1 return x - y"));
-        assertEquals(0L, exec("def x = (long)1 def y = (long)1 return x - y"));
-        assertEquals(0F, exec("def x = (float)1 def y = (long)1 return x - y"));
-        assertEquals(0D, exec("def x = (double)1 def y = (long)1 return x - y"));
+        assertEquals(0L, exec("def x = (byte)1; def y = (long)1; return x - y"));
+        assertEquals(0L, exec("def x = (short)1; def y = (long)1; return x - y"));
+        assertEquals(0L, exec("def x = (char)1; def y = (long)1; return x - y"));
+        assertEquals(0L, exec("def x = (int)1; def y = (long)1; return x - y"));
+        assertEquals(0L, exec("def x = (long)1; def y = (long)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; def y = (long)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; def y = (long)1; return x - y"));
 
-        assertEquals(0F, exec("def x = (byte)1 def y = (float)1 return x - y"));
-        assertEquals(0F, exec("def x = (short)1 def y = (float)1 return x - y"));
-        assertEquals(0F, exec("def x = (char)1 def y = (float)1 return x - y"));
-        assertEquals(0F, exec("def x = (int)1 def y = (float)1 return x - y"));
-        assertEquals(0F, exec("def x = (long)1 def y = (float)1 return x - y"));
-        assertEquals(0F, exec("def x = (float)1 def y = (float)1 return x - y"));
-        assertEquals(0D, exec("def x = (double)1 def y = (float)1 return x - y"));
+        assertEquals(0F, exec("def x = (byte)1; def y = (float)1; return x - y"));
+        assertEquals(0F, exec("def x = (short)1; def y = (float)1; return x - y"));
+        assertEquals(0F, exec("def x = (char)1; def y = (float)1; return x - y"));
+        assertEquals(0F, exec("def x = (int)1; def y = (float)1; return x - y"));
+        assertEquals(0F, exec("def x = (long)1; def y = (float)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; def y = (float)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; def y = (float)1; return x - y"));
 
-        assertEquals(0D, exec("def x = (byte)1 def y = (double)1 return x - y"));
-        assertEquals(0D, exec("def x = (short)1 def y = (double)1 return x - y"));
-        assertEquals(0D, exec("def x = (char)1 def y = (double)1 return x - y"));
-        assertEquals(0D, exec("def x = (int)1 def y = (double)1 return x - y"));
-        assertEquals(0D, exec("def x = (long)1 def y = (double)1 return x - y"));
-        assertEquals(0D, exec("def x = (float)1 def y = (double)1 return x - y"));
-        assertEquals(0D, exec("def x = (double)1 def y = (double)1 return x - y"));
+        assertEquals(0D, exec("def x = (byte)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (short)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (char)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (int)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (long)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (float)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; def y = (double)1; return x - y"));
 
-        assertEquals(0, exec("def x = (Byte)1 def y = (byte)1 return x - y"));
-        assertEquals(0, exec("def x = (Short)1 def y = (short)1 return x - y"));
-        assertEquals(0, exec("def x = (Character)1 def y = (char)1 return x - y"));
-        assertEquals(0, exec("def x = (Integer)1 def y = (int)1 return x - y"));
-        assertEquals(0L, exec("def x = (Long)1 def y = (long)1 return x - y"));
-        assertEquals(0F, exec("def x = (Float)1 def y = (float)1 return x - y"));
-        assertEquals(0D, exec("def x = (Double)1 def y = (double)1 return x - y"));
+        assertEquals(0, exec("def x = (Byte)1; def y = (byte)1; return x - y"));
+        assertEquals(0, exec("def x = (Short)1; def y = (short)1; return x - y"));
+        assertEquals(0, exec("def x = (Character)1; def y = (char)1; return x - y"));
+        assertEquals(0, exec("def x = (Integer)1; def y = (int)1; return x - y"));
+        assertEquals(0L, exec("def x = (Long)1; def y = (long)1; return x - y"));
+        assertEquals(0F, exec("def x = (Float)1; def y = (float)1; return x - y"));
+        assertEquals(0D, exec("def x = (Double)1; def y = (double)1; return x - y"));
     }
 
     public void testLsh() {
-        assertEquals(2, exec("def x = (byte)1 def y = (byte)1 return x << y"));
-        assertEquals(2, exec("def x = (short)1 def y = (byte)1 return x << y"));
-        assertEquals(2, exec("def x = (char)1 def y = (byte)1 return x << y"));
-        assertEquals(2, exec("def x = (int)1 def y = (byte)1 return x << y"));
-        assertEquals(2L, exec("def x = (long)1 def y = (byte)1 return x << y"));
-        assertEquals(2L, exec("def x = (float)1 def y = (byte)1 return x << y"));
-        assertEquals(2L, exec("def x = (double)1 def y = (byte)1 return x << y"));
+        assertEquals(2, exec("def x = (byte)1; def y = (byte)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; def y = (byte)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; def y = (byte)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; def y = (byte)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (byte)1; return x << y"));
+        assertEquals(2L, exec("def x = (float)1; def y = (byte)1; return x << y"));
+        assertEquals(2L, exec("def x = (double)1; def y = (byte)1; return x << y"));
 
-        assertEquals(2, exec("def x = (byte)1 def y = (short)1 return x << y"));
-        assertEquals(2, exec("def x = (short)1 def y = (short)1 return x << y"));
-        assertEquals(2, exec("def x = (char)1 def y = (short)1 return x << y"));
-        assertEquals(2, exec("def x = (int)1 def y = (short)1 return x << y"));
-        assertEquals(2L, exec("def x = (long)1 def y = (short)1 return x << y"));
-        assertEquals(2L, exec("def x = (float)1 def y = (short)1 return x << y"));
-        assertEquals(2L, exec("def x = (double)1 def y = (short)1 return x << y"));
+        assertEquals(2, exec("def x = (byte)1; def y = (short)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; def y = (short)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; def y = (short)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; def y = (short)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (short)1; return x << y"));
+        assertEquals(2L, exec("def x = (float)1; def y = (short)1; return x << y"));
+        assertEquals(2L, exec("def x = (double)1; def y = (short)1; return x << y"));
 
-        assertEquals(2, exec("def x = (byte)1 def y = (char)1 return x << y"));
-        assertEquals(2, exec("def x = (short)1 def y = (char)1 return x << y"));
-        assertEquals(2, exec("def x = (char)1 def y = (char)1 return x << y"));
-        assertEquals(2, exec("def x = (int)1 def y = (char)1 return x << y"));
-        assertEquals(2L, exec("def x = (long)1 def y = (char)1 return x << y"));
-        assertEquals(2L, exec("def x = (float)1 def y = (char)1 return x << y"));
-        assertEquals(2L, exec("def x = (double)1 def y = (char)1 return x << y"));
+        assertEquals(2, exec("def x = (byte)1; def y = (char)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; def y = (char)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; def y = (char)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; def y = (char)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (char)1; return x << y"));
+        assertEquals(2L, exec("def x = (float)1; def y = (char)1; return x << y"));
+        assertEquals(2L, exec("def x = (double)1; def y = (char)1; return x << y"));
 
-        assertEquals(2, exec("def x = (byte)1 def y = (int)1 return x << y"));
-        assertEquals(2, exec("def x = (short)1 def y = (int)1 return x << y"));
-        assertEquals(2, exec("def x = (char)1 def y = (int)1 return x << y"));
-        assertEquals(2, exec("def x = (int)1 def y = (int)1 return x << y"));
-        assertEquals(2L, exec("def x = (long)1 def y = (int)1 return x << y"));
-        assertEquals(2L, exec("def x = (float)1 def y = (int)1 return x << y"));
-        assertEquals(2L, exec("def x = (double)1 def y = (int)1 return x << y"));
+        assertEquals(2, exec("def x = (byte)1; def y = (int)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; def y = (int)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; def y = (int)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; def y = (int)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (int)1; return x << y"));
+        assertEquals(2L, exec("def x = (float)1; def y = (int)1; return x << y"));
+        assertEquals(2L, exec("def x = (double)1; def y = (int)1; return x << y"));
 
-        assertEquals(2, exec("def x = (byte)1 def y = (long)1 return x << y"));
-        assertEquals(2, exec("def x = (short)1 def y = (long)1 return x << y"));
-        assertEquals(2, exec("def x = (char)1 def y = (long)1 return x << y"));
-        assertEquals(2, exec("def x = (int)1 def y = (long)1 return x << y"));
-        assertEquals(2L, exec("def x = (long)1 def y = (long)1 return x << y"));
-        assertEquals(2L, exec("def x = (float)1 def y = (long)1 return x << y"));
-        assertEquals(2L, exec("def x = (double)1 def y = (long)1 return x << y"));
+        assertEquals(2, exec("def x = (byte)1; def y = (long)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; def y = (long)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; def y = (long)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; def y = (long)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (long)1; return x << y"));
+        assertEquals(2L, exec("def x = (float)1; def y = (long)1; return x << y"));
+        assertEquals(2L, exec("def x = (double)1; def y = (long)1; return x << y"));
 
-        assertEquals(2, exec("def x = (byte)1 def y = (float)1 return x << y"));
-        assertEquals(2, exec("def x = (short)1 def y = (float)1 return x << y"));
-        assertEquals(2, exec("def x = (char)1 def y = (float)1 return x << y"));
-        assertEquals(2, exec("def x = (int)1 def y = (float)1 return x << y"));
-        assertEquals(2L, exec("def x = (long)1 def y = (float)1 return x << y"));
-        assertEquals(2L, exec("def x = (float)1 def y = (float)1 return x << y"));
-        assertEquals(2L, exec("def x = (double)1 def y = (float)1 return x << y"));
+        assertEquals(2, exec("def x = (byte)1; def y = (float)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; def y = (float)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; def y = (float)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; def y = (float)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (float)1; return x << y"));
+        assertEquals(2L, exec("def x = (float)1; def y = (float)1; return x << y"));
+        assertEquals(2L, exec("def x = (double)1; def y = (float)1; return x << y"));
 
-        assertEquals(2, exec("def x = (byte)1 def y = (double)1 return x << y"));
-        assertEquals(2, exec("def x = (short)1 def y = (double)1 return x << y"));
-        assertEquals(2, exec("def x = (char)1 def y = (double)1 return x << y"));
-        assertEquals(2, exec("def x = (int)1 def y = (double)1 return x << y"));
-        assertEquals(2L, exec("def x = (long)1 def y = (double)1 return x << y"));
-        assertEquals(2L, exec("def x = (float)1 def y = (double)1 return x << y"));
-        assertEquals(2L, exec("def x = (double)1 def y = (double)1 return x << y"));
+        assertEquals(2, exec("def x = (byte)1; def y = (double)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; def y = (double)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; def y = (double)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; def y = (double)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (double)1; return x << y"));
+        assertEquals(2L, exec("def x = (float)1; def y = (double)1; return x << y"));
+        assertEquals(2L, exec("def x = (double)1; def y = (double)1; return x << y"));
 
-        assertEquals(2, exec("def x = (Byte)1 def y = (byte)1 return x << y"));
-        assertEquals(2, exec("def x = (Short)1 def y = (short)1 return x << y"));
-        assertEquals(2, exec("def x = (Character)1 def y = (char)1 return x << y"));
-        assertEquals(2, exec("def x = (Integer)1 def y = (int)1 return x << y"));
-        assertEquals(2L, exec("def x = (Long)1 def y = (long)1 return x << y"));
-        assertEquals(2L, exec("def x = (Float)1 def y = (float)1 return x << y"));
-        assertEquals(2L, exec("def x = (Double)1 def y = (double)1 return x << y"));
+        assertEquals(2, exec("def x = (Byte)1; def y = (byte)1; return x << y"));
+        assertEquals(2, exec("def x = (Short)1; def y = (short)1; return x << y"));
+        assertEquals(2, exec("def x = (Character)1; def y = (char)1; return x << y"));
+        assertEquals(2, exec("def x = (Integer)1; def y = (int)1; return x << y"));
+        assertEquals(2L, exec("def x = (Long)1; def y = (long)1; return x << y"));
+        assertEquals(2L, exec("def x = (Float)1; def y = (float)1; return x << y"));
+        assertEquals(2L, exec("def x = (Double)1; def y = (double)1; return x << y"));
     }
 
     public void testRsh() {
-        assertEquals(2, exec("def x = (byte)4 def y = (byte)1 return x >> y"));
-        assertEquals(2, exec("def x = (short)4 def y = (byte)1 return x >> y"));
-        assertEquals(2, exec("def x = (char)4 def y = (byte)1 return x >> y"));
-        assertEquals(2, exec("def x = (int)4 def y = (byte)1 return x >> y"));
-        assertEquals(2L, exec("def x = (long)4 def y = (byte)1 return x >> y"));
-        assertEquals(2L, exec("def x = (float)4 def y = (byte)1 return x >> y"));
-        assertEquals(2L, exec("def x = (double)4 def y = (byte)1 return x >> y"));
+        assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (byte)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (byte)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (byte)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (byte)1; return x >> y"));
+        assertEquals(2L, exec("def x = (float)4; def y = (byte)1; return x >> y"));
+        assertEquals(2L, exec("def x = (double)4; def y = (byte)1; return x >> y"));
 
-        assertEquals(2, exec("def x = (byte)4 def y = (short)1 return x >> y"));
-        assertEquals(2, exec("def x = (short)4 def y = (short)1 return x >> y"));
-        assertEquals(2, exec("def x = (char)4 def y = (short)1 return x >> y"));
-        assertEquals(2, exec("def x = (int)4 def y = (short)1 return x >> y"));
-        assertEquals(2L, exec("def x = (long)4 def y = (short)1 return x >> y"));
-        assertEquals(2L, exec("def x = (float)4 def y = (short)1 return x >> y"));
-        assertEquals(2L, exec("def x = (double)4 def y = (short)1 return x >> y"));
+        assertEquals(2, exec("def x = (byte)4; def y = (short)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (short)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (short)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (short)1; return x >> y"));
+        assertEquals(2L, exec("def x = (float)4; def y = (short)1; return x >> y"));
+        assertEquals(2L, exec("def x = (double)4; def y = (short)1; return x >> y"));
 
-        assertEquals(2, exec("def x = (byte)4 def y = (char)1 return x >> y"));
-        assertEquals(2, exec("def x = (short)4 def y = (char)1 return x >> y"));
-        assertEquals(2, exec("def x = (char)4 def y = (char)1 return x >> y"));
-        assertEquals(2, exec("def x = (int)4 def y = (char)1 return x >> y"));
-        assertEquals(2L, exec("def x = (long)4 def y = (char)1 return x >> y"));
-        assertEquals(2L, exec("def x = (float)4 def y = (char)1 return x >> y"));
-        assertEquals(2L, exec("def x = (double)4 def y = (char)1 return x >> y"));
+        assertEquals(2, exec("def x = (byte)4; def y = (char)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (char)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (char)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (char)1; return x >> y"));
+        assertEquals(2L, exec("def x = (float)4; def y = (char)1; return x >> y"));
+        assertEquals(2L, exec("def x = (double)4; def y = (char)1; return x >> y"));
 
-        assertEquals(2, exec("def x = (byte)4 def y = (int)1 return x >> y"));
-        assertEquals(2, exec("def x = (short)4 def y = (int)1 return x >> y"));
-        assertEquals(2, exec("def x = (char)4 def y = (int)1 return x >> y"));
-        assertEquals(2, exec("def x = (int)4 def y = (int)1 return x >> y"));
-        assertEquals(2L, exec("def x = (long)4 def y = (int)1 return x >> y"));
-        assertEquals(2L, exec("def x = (float)4 def y = (int)1 return x >> y"));
-        assertEquals(2L, exec("def x = (double)4 def y = (int)1 return x >> y"));
+        assertEquals(2, exec("def x = (byte)4; def y = (int)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (int)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (int)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (int)1; return x >> y"));
+        assertEquals(2L, exec("def x = (float)4; def y = (int)1; return x >> y"));
+        assertEquals(2L, exec("def x = (double)4; def y = (int)1; return x >> y"));
 
-        assertEquals(2, exec("def x = (byte)4 def y = (long)1 return x >> y"));
-        assertEquals(2, exec("def x = (short)4 def y = (long)1 return x >> y"));
-        assertEquals(2, exec("def x = (char)4 def y = (long)1 return x >> y"));
-        assertEquals(2, exec("def x = (int)4 def y = (long)1 return x >> y"));
-        assertEquals(2L, exec("def x = (long)4 def y = (long)1 return x >> y"));
-        assertEquals(2L, exec("def x = (float)4 def y = (long)1 return x >> y"));
-        assertEquals(2L, exec("def x = (double)4 def y = (long)1 return x >> y"));
+        assertEquals(2, exec("def x = (byte)4; def y = (long)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (long)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (long)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (long)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >> y"));
+        assertEquals(2L, exec("def x = (float)4; def y = (long)1; return x >> y"));
+        assertEquals(2L, exec("def x = (double)4; def y = (long)1; return x >> y"));
 
-        assertEquals(2, exec("def x = (byte)4 def y = (float)1 return x >> y"));
-        assertEquals(2, exec("def x = (short)4 def y = (float)1 return x >> y"));
-        assertEquals(2, exec("def x = (char)4 def y = (float)1 return x >> y"));
-        assertEquals(2, exec("def x = (int)4 def y = (float)1 return x >> y"));
-        assertEquals(2L, exec("def x = (long)4 def y = (float)1 return x >> y"));
-        assertEquals(2L, exec("def x = (float)4 def y = (float)1 return x >> y"));
-        assertEquals(2L, exec("def x = (double)4 def y = (float)1 return x >> y"));
+        assertEquals(2, exec("def x = (byte)4; def y = (float)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (float)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (float)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (float)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (float)1; return x >> y"));
+        assertEquals(2L, exec("def x = (float)4; def y = (float)1; return x >> y"));
+        assertEquals(2L, exec("def x = (double)4; def y = (float)1; return x >> y"));
 
-        assertEquals(2, exec("def x = (byte)4 def y = (double)1 return x >> y"));
-        assertEquals(2, exec("def x = (short)4 def y = (double)1 return x >> y"));
-        assertEquals(2, exec("def x = (char)4 def y = (double)1 return x >> y"));
-        assertEquals(2, exec("def x = (int)4 def y = (double)1 return x >> y"));
-        assertEquals(2L, exec("def x = (long)4 def y = (double)1 return x >> y"));
-        assertEquals(2L, exec("def x = (float)4 def y = (double)1 return x >> y"));
-        assertEquals(2L, exec("def x = (double)4 def y = (double)1 return x >> y"));
+        assertEquals(2, exec("def x = (byte)4; def y = (double)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (double)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (double)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (double)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (double)1; return x >> y"));
+        assertEquals(2L, exec("def x = (float)4; def y = (double)1; return x >> y"));
+        assertEquals(2L, exec("def x = (double)4; def y = (double)1; return x >> y"));
 
-        assertEquals(2, exec("def x = (Byte)4 def y = (byte)1 return x >> y"));
-        assertEquals(2, exec("def x = (Short)4 def y = (short)1 return x >> y"));
-        assertEquals(2, exec("def x = (Character)4 def y = (char)1 return x >> y"));
-        assertEquals(2, exec("def x = (Integer)4 def y = (int)1 return x >> y"));
-        assertEquals(2L, exec("def x = (Long)4 def y = (long)1 return x >> y"));
-        assertEquals(2L, exec("def x = (Float)4 def y = (float)1 return x >> y"));
-        assertEquals(2L, exec("def x = (Double)4 def y = (double)1 return x >> y"));
+        assertEquals(2, exec("def x = (Byte)4; def y = (byte)1; return x >> y"));
+        assertEquals(2, exec("def x = (Short)4; def y = (short)1; return x >> y"));
+        assertEquals(2, exec("def x = (Character)4; def y = (char)1; return x >> y"));
+        assertEquals(2, exec("def x = (Integer)4; def y = (int)1; return x >> y"));
+        assertEquals(2L, exec("def x = (Long)4; def y = (long)1; return x >> y"));
+        assertEquals(2L, exec("def x = (Float)4; def y = (float)1; return x >> y"));
+        assertEquals(2L, exec("def x = (Double)4; def y = (double)1; return x >> y"));
     }
 
     public void testUsh() {
-        assertEquals(2, exec("def x = (byte)4 def y = (byte)1 return x >>> y"));
-        assertEquals(2, exec("def x = (short)4 def y = (byte)1 return x >>> y"));
-        assertEquals(2, exec("def x = (char)4 def y = (byte)1 return x >>> y"));
-        assertEquals(2, exec("def x = (int)4 def y = (byte)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (long)4 def y = (byte)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (float)4 def y = (byte)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (double)4 def y = (byte)1 return x >>> y"));
+        assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (byte)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (byte)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (float)4; def y = (byte)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (double)4; def y = (byte)1; return x >>> y"));
 
-        assertEquals(2, exec("def x = (byte)4 def y = (short)1 return x >>> y"));
-        assertEquals(2, exec("def x = (short)4 def y = (short)1 return x >>> y"));
-        assertEquals(2, exec("def x = (char)4 def y = (short)1 return x >>> y"));
-        assertEquals(2, exec("def x = (int)4 def y = (short)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (long)4 def y = (short)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (float)4 def y = (short)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (double)4 def y = (short)1 return x >>> y"));
+        assertEquals(2, exec("def x = (byte)4; def y = (short)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (short)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (short)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (short)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (float)4; def y = (short)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (double)4; def y = (short)1; return x >>> y"));
 
-        assertEquals(2, exec("def x = (byte)4 def y = (char)1 return x >>> y"));
-        assertEquals(2, exec("def x = (short)4 def y = (char)1 return x >>> y"));
-        assertEquals(2, exec("def x = (char)4 def y = (char)1 return x >>> y"));
-        assertEquals(2, exec("def x = (int)4 def y = (char)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (long)4 def y = (char)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (float)4 def y = (char)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (double)4 def y = (char)1 return x >>> y"));
+        assertEquals(2, exec("def x = (byte)4; def y = (char)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (char)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (char)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (char)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (float)4; def y = (char)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (double)4; def y = (char)1; return x >>> y"));
 
-        assertEquals(2, exec("def x = (byte)4 def y = (int)1 return x >>> y"));
-        assertEquals(2, exec("def x = (short)4 def y = (int)1 return x >>> y"));
-        assertEquals(2, exec("def x = (char)4 def y = (int)1 return x >>> y"));
-        assertEquals(2, exec("def x = (int)4 def y = (int)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (long)4 def y = (int)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (float)4 def y = (int)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (double)4 def y = (int)1 return x >>> y"));
+        assertEquals(2, exec("def x = (byte)4; def y = (int)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (int)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (int)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (int)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (float)4; def y = (int)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (double)4; def y = (int)1; return x >>> y"));
 
-        assertEquals(2, exec("def x = (byte)4 def y = (long)1 return x >>> y"));
-        assertEquals(2, exec("def x = (short)4 def y = (long)1 return x >>> y"));
-        assertEquals(2, exec("def x = (char)4 def y = (long)1 return x >>> y"));
-        assertEquals(2, exec("def x = (int)4 def y = (long)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (long)4 def y = (long)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (float)4 def y = (long)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (double)4 def y = (long)1 return x >>> y"));
+        assertEquals(2, exec("def x = (byte)4; def y = (long)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (long)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (long)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (long)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (float)4; def y = (long)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (double)4; def y = (long)1; return x >>> y"));
 
-        assertEquals(2, exec("def x = (byte)4 def y = (float)1 return x >>> y"));
-        assertEquals(2, exec("def x = (short)4 def y = (float)1 return x >>> y"));
-        assertEquals(2, exec("def x = (char)4 def y = (float)1 return x >>> y"));
-        assertEquals(2, exec("def x = (int)4 def y = (float)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (long)4 def y = (float)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (float)4 def y = (float)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (double)4 def y = (float)1 return x >>> y"));
+        assertEquals(2, exec("def x = (byte)4; def y = (float)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (float)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (float)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (float)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (float)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (float)4; def y = (float)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (double)4; def y = (float)1; return x >>> y"));
 
-        assertEquals(2, exec("def x = (byte)4 def y = (double)1 return x >>> y"));
-        assertEquals(2, exec("def x = (short)4 def y = (double)1 return x >>> y"));
-        assertEquals(2, exec("def x = (char)4 def y = (double)1 return x >>> y"));
-        assertEquals(2, exec("def x = (int)4 def y = (double)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (long)4 def y = (double)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (float)4 def y = (double)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (double)4 def y = (double)1 return x >>> y"));
+        assertEquals(2, exec("def x = (byte)4; def y = (double)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (double)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (double)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (double)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (double)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (float)4; def y = (double)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (double)4; def y = (double)1; return x >>> y"));
 
-        assertEquals(2, exec("def x = (Byte)4 def y = (byte)1 return x >>> y"));
-        assertEquals(2, exec("def x = (Short)4 def y = (short)1 return x >>> y"));
-        assertEquals(2, exec("def x = (Character)4 def y = (char)1 return x >>> y"));
-        assertEquals(2, exec("def x = (Integer)4 def y = (int)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (Long)4 def y = (long)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (Float)4 def y = (float)1 return x >>> y"));
-        assertEquals(2L, exec("def x = (Double)4 def y = (double)1 return x >>> y"));
+        assertEquals(2, exec("def x = (Byte)4; def y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("def x = (Short)4; def y = (short)1; return x >>> y"));
+        assertEquals(2, exec("def x = (Character)4; def y = (char)1; return x >>> y"));
+        assertEquals(2, exec("def x = (Integer)4; def y = (int)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (Long)4; def y = (long)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (Float)4; def y = (float)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (Double)4; def y = (double)1; return x >>> y"));
     }
 
     public void testAnd() {
-        assertEquals(0, exec("def x = (byte)4 def y = (byte)1 return x & y"));
-        assertEquals(0, exec("def x = (short)4 def y = (byte)1 return x & y"));
-        assertEquals(0, exec("def x = (char)4 def y = (byte)1 return x & y"));
-        assertEquals(0, exec("def x = (int)4 def y = (byte)1 return x & y"));
-        assertEquals(0L, exec("def x = (long)4 def y = (byte)1 return x & y"));
-        assertEquals(0L, exec("def x = (float)4 def y = (byte)1 return x & y"));
-        assertEquals(0L, exec("def x = (double)4 def y = (byte)1 return x & y"));
+        assertEquals(0, exec("def x = (byte)4; def y = (byte)1; return x & y"));
+        assertEquals(0, exec("def x = (short)4; def y = (byte)1; return x & y"));
+        assertEquals(0, exec("def x = (char)4; def y = (byte)1; return x & y"));
+        assertEquals(0, exec("def x = (int)4; def y = (byte)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; def y = (byte)1; return x & y"));
+        assertEquals(0L, exec("def x = (float)4; def y = (byte)1; return x & y"));
+        assertEquals(0L, exec("def x = (double)4; def y = (byte)1; return x & y"));
 
-        assertEquals(0, exec("def x = (byte)4 def y = (short)1 return x & y"));
-        assertEquals(0, exec("def x = (short)4 def y = (short)1 return x & y"));
-        assertEquals(0, exec("def x = (char)4 def y = (short)1 return x & y"));
-        assertEquals(0, exec("def x = (int)4 def y = (short)1 return x & y"));
-        assertEquals(0L, exec("def x = (long)4 def y = (short)1 return x & y"));
-        assertEquals(0L, exec("def x = (float)4 def y = (short)1 return x & y"));
-        assertEquals(0L, exec("def x = (double)4 def y = (short)1 return x & y"));
+        assertEquals(0, exec("def x = (byte)4; def y = (short)1; return x & y"));
+        assertEquals(0, exec("def x = (short)4; def y = (short)1; return x & y"));
+        assertEquals(0, exec("def x = (char)4; def y = (short)1; return x & y"));
+        assertEquals(0, exec("def x = (int)4; def y = (short)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; def y = (short)1; return x & y"));
+        assertEquals(0L, exec("def x = (float)4; def y = (short)1; return x & y"));
+        assertEquals(0L, exec("def x = (double)4; def y = (short)1; return x & y"));
 
-        assertEquals(0, exec("def x = (byte)4 def y = (char)1 return x & y"));
-        assertEquals(0, exec("def x = (short)4 def y = (char)1 return x & y"));
-        assertEquals(0, exec("def x = (char)4 def y = (char)1 return x & y"));
-        assertEquals(0, exec("def x = (int)4 def y = (char)1 return x & y"));
-        assertEquals(0L, exec("def x = (long)4 def y = (char)1 return x & y"));
-        assertEquals(0L, exec("def x = (float)4 def y = (char)1 return x & y"));
-        assertEquals(0L, exec("def x = (double)4 def y = (char)1 return x & y"));
+        assertEquals(0, exec("def x = (byte)4; def y = (char)1; return x & y"));
+        assertEquals(0, exec("def x = (short)4; def y = (char)1; return x & y"));
+        assertEquals(0, exec("def x = (char)4; def y = (char)1; return x & y"));
+        assertEquals(0, exec("def x = (int)4; def y = (char)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; def y = (char)1; return x & y"));
+        assertEquals(0L, exec("def x = (float)4; def y = (char)1; return x & y"));
+        assertEquals(0L, exec("def x = (double)4; def y = (char)1; return x & y"));
 
-        assertEquals(0, exec("def x = (byte)4 def y = (int)1 return x & y"));
-        assertEquals(0, exec("def x = (short)4 def y = (int)1 return x & y"));
-        assertEquals(0, exec("def x = (char)4 def y = (int)1 return x & y"));
-        assertEquals(0, exec("def x = (int)4 def y = (int)1 return x & y"));
-        assertEquals(0L, exec("def x = (long)4 def y = (int)1 return x & y"));
-        assertEquals(0L, exec("def x = (float)4 def y = (int)1 return x & y"));
-        assertEquals(0L, exec("def x = (double)4 def y = (int)1 return x & y"));
+        assertEquals(0, exec("def x = (byte)4; def y = (int)1; return x & y"));
+        assertEquals(0, exec("def x = (short)4; def y = (int)1; return x & y"));
+        assertEquals(0, exec("def x = (char)4; def y = (int)1; return x & y"));
+        assertEquals(0, exec("def x = (int)4; def y = (int)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; def y = (int)1; return x & y"));
+        assertEquals(0L, exec("def x = (float)4; def y = (int)1; return x & y"));
+        assertEquals(0L, exec("def x = (double)4; def y = (int)1; return x & y"));
 
-        assertEquals(0L, exec("def x = (byte)4 def y = (long)1 return x & y"));
-        assertEquals(0L, exec("def x = (short)4 def y = (long)1 return x & y"));
-        assertEquals(0L, exec("def x = (char)4 def y = (long)1 return x & y"));
-        assertEquals(0L, exec("def x = (int)4 def y = (long)1 return x & y"));
-        assertEquals(0L, exec("def x = (long)4 def y = (long)1 return x & y"));
-        assertEquals(0L, exec("def x = (float)4 def y = (long)1 return x & y"));
-        assertEquals(0L, exec("def x = (double)4 def y = (long)1 return x & y"));
+        assertEquals(0L, exec("def x = (byte)4; def y = (long)1; return x & y"));
+        assertEquals(0L, exec("def x = (short)4; def y = (long)1; return x & y"));
+        assertEquals(0L, exec("def x = (char)4; def y = (long)1; return x & y"));
+        assertEquals(0L, exec("def x = (int)4; def y = (long)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; def y = (long)1; return x & y"));
+        assertEquals(0L, exec("def x = (float)4; def y = (long)1; return x & y"));
+        assertEquals(0L, exec("def x = (double)4; def y = (long)1; return x & y"));
 
-        assertEquals(0L, exec("def x = (byte)4 def y = (float)1 return x & y"));
-        assertEquals(0L, exec("def x = (short)4 def y = (float)1 return x & y"));
-        assertEquals(0L, exec("def x = (char)4 def y = (float)1 return x & y"));
-        assertEquals(0L, exec("def x = (int)4 def y = (float)1 return x & y"));
-        assertEquals(0L, exec("def x = (long)4 def y = (float)1 return x & y"));
-        assertEquals(0L, exec("def x = (float)4 def y = (float)1 return x & y"));
-        assertEquals(0L, exec("def x = (double)4 def y = (float)1 return x & y"));
+        assertEquals(0L, exec("def x = (byte)4; def y = (float)1; return x & y"));
+        assertEquals(0L, exec("def x = (short)4; def y = (float)1; return x & y"));
+        assertEquals(0L, exec("def x = (char)4; def y = (float)1; return x & y"));
+        assertEquals(0L, exec("def x = (int)4; def y = (float)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; def y = (float)1; return x & y"));
+        assertEquals(0L, exec("def x = (float)4; def y = (float)1; return x & y"));
+        assertEquals(0L, exec("def x = (double)4; def y = (float)1; return x & y"));
 
-        assertEquals(0L, exec("def x = (byte)4 def y = (double)1 return x & y"));
-        assertEquals(0L, exec("def x = (short)4 def y = (double)1 return x & y"));
-        assertEquals(0L, exec("def x = (char)4 def y = (double)1 return x & y"));
-        assertEquals(0L, exec("def x = (int)4 def y = (double)1 return x & y"));
-        assertEquals(0L, exec("def x = (long)4 def y = (double)1 return x & y"));
-        assertEquals(0L, exec("def x = (float)4 def y = (double)1 return x & y"));
-        assertEquals(0L, exec("def x = (double)4 def y = (double)1 return x & y"));
+        assertEquals(0L, exec("def x = (byte)4; def y = (double)1; return x & y"));
+        assertEquals(0L, exec("def x = (short)4; def y = (double)1; return x & y"));
+        assertEquals(0L, exec("def x = (char)4; def y = (double)1; return x & y"));
+        assertEquals(0L, exec("def x = (int)4; def y = (double)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; def y = (double)1; return x & y"));
+        assertEquals(0L, exec("def x = (float)4; def y = (double)1; return x & y"));
+        assertEquals(0L, exec("def x = (double)4; def y = (double)1; return x & y"));
 
-        assertEquals(0, exec("def x = (Byte)4 def y = (byte)1 return x & y"));
-        assertEquals(0, exec("def x = (Short)4 def y = (short)1 return x & y"));
-        assertEquals(0, exec("def x = (Character)4 def y = (char)1 return x & y"));
-        assertEquals(0, exec("def x = (Integer)4 def y = (int)1 return x & y"));
-        assertEquals(0L, exec("def x = (Long)4 def y = (long)1 return x & y"));
-        assertEquals(0L, exec("def x = (Float)4 def y = (float)1 return x & y"));
-        assertEquals(0L, exec("def x = (Double)4 def y = (double)1 return x & y"));
+        assertEquals(0, exec("def x = (Byte)4; def y = (byte)1; return x & y"));
+        assertEquals(0, exec("def x = (Short)4; def y = (short)1; return x & y"));
+        assertEquals(0, exec("def x = (Character)4; def y = (char)1; return x & y"));
+        assertEquals(0, exec("def x = (Integer)4; def y = (int)1; return x & y"));
+        assertEquals(0L, exec("def x = (Long)4; def y = (long)1; return x & y"));
+        assertEquals(0L, exec("def x = (Float)4; def y = (float)1; return x & y"));
+        assertEquals(0L, exec("def x = (Double)4; def y = (double)1; return x & y"));
     }
 
     public void testXor() {
-        assertEquals(5, exec("def x = (byte)4 def y = (byte)1 return x ^ y"));
-        assertEquals(5, exec("def x = (short)4 def y = (byte)1 return x ^ y"));
-        assertEquals(5, exec("def x = (char)4 def y = (byte)1 return x ^ y"));
-        assertEquals(5, exec("def x = (int)4 def y = (byte)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4 def y = (byte)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (float)4 def y = (byte)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (double)4 def y = (byte)1 return x ^ y"));
+        assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; def y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; def y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; def y = (byte)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (byte)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (float)4; def y = (byte)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (double)4; def y = (byte)1; return x ^ y"));
 
-        assertEquals(5, exec("def x = (byte)4 def y = (short)1 return x ^ y"));
-        assertEquals(5, exec("def x = (short)4 def y = (short)1 return x ^ y"));
-        assertEquals(5, exec("def x = (char)4 def y = (short)1 return x ^ y"));
-        assertEquals(5, exec("def x = (int)4 def y = (short)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4 def y = (short)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (float)4 def y = (short)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (double)4 def y = (short)1 return x ^ y"));
+        assertEquals(5, exec("def x = (byte)4; def y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; def y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; def y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; def y = (short)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (short)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (float)4; def y = (short)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (double)4; def y = (short)1; return x ^ y"));
 
-        assertEquals(5, exec("def x = (byte)4 def y = (char)1 return x ^ y"));
-        assertEquals(5, exec("def x = (short)4 def y = (char)1 return x ^ y"));
-        assertEquals(5, exec("def x = (char)4 def y = (char)1 return x ^ y"));
-        assertEquals(5, exec("def x = (int)4 def y = (char)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4 def y = (char)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (float)4 def y = (char)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (double)4 def y = (char)1 return x ^ y"));
+        assertEquals(5, exec("def x = (byte)4; def y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; def y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; def y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; def y = (char)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (char)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (float)4; def y = (char)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (double)4; def y = (char)1; return x ^ y"));
 
-        assertEquals(5, exec("def x = (byte)4 def y = (int)1 return x ^ y"));
-        assertEquals(5, exec("def x = (short)4 def y = (int)1 return x ^ y"));
-        assertEquals(5, exec("def x = (char)4 def y = (int)1 return x ^ y"));
-        assertEquals(5, exec("def x = (int)4 def y = (int)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4 def y = (int)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (float)4 def y = (int)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (double)4 def y = (int)1 return x ^ y"));
+        assertEquals(5, exec("def x = (byte)4; def y = (int)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; def y = (int)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; def y = (int)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; def y = (int)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (int)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (float)4; def y = (int)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (double)4; def y = (int)1; return x ^ y"));
 
-        assertEquals(5L, exec("def x = (byte)4 def y = (long)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (short)4 def y = (long)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (char)4 def y = (long)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (int)4 def y = (long)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4 def y = (long)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (float)4 def y = (long)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (double)4 def y = (long)1 return x ^ y"));
+        assertEquals(5L, exec("def x = (byte)4; def y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (short)4; def y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (char)4; def y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (int)4; def y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (float)4; def y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (double)4; def y = (long)1; return x ^ y"));
 
-        assertEquals(5L, exec("def x = (byte)4 def y = (float)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (short)4 def y = (float)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (char)4 def y = (float)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (int)4 def y = (float)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4 def y = (float)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (float)4 def y = (float)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (double)4 def y = (float)1 return x ^ y"));
+        assertEquals(5L, exec("def x = (byte)4; def y = (float)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (short)4; def y = (float)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (char)4; def y = (float)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (int)4; def y = (float)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (float)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (float)4; def y = (float)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (double)4; def y = (float)1; return x ^ y"));
 
-        assertEquals(5L, exec("def x = (byte)4 def y = (double)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (short)4 def y = (double)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (char)4 def y = (double)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (int)4 def y = (double)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4 def y = (double)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (float)4 def y = (double)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (double)4 def y = (double)1 return x ^ y"));
+        assertEquals(5L, exec("def x = (byte)4; def y = (double)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (short)4; def y = (double)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (char)4; def y = (double)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (int)4; def y = (double)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (double)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (float)4; def y = (double)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (double)4; def y = (double)1; return x ^ y"));
 
-        assertEquals(5, exec("def x = (Byte)4 def y = (byte)1 return x ^ y"));
-        assertEquals(5, exec("def x = (Short)4 def y = (short)1 return x ^ y"));
-        assertEquals(5, exec("def x = (Character)4 def y = (char)1 return x ^ y"));
-        assertEquals(5, exec("def x = (Integer)4 def y = (int)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (Long)4 def y = (long)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (Float)4 def y = (float)1 return x ^ y"));
-        assertEquals(5L, exec("def x = (Double)4 def y = (double)1 return x ^ y"));
+        assertEquals(5, exec("def x = (Byte)4; def y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (Short)4; def y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (Character)4; def y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (Integer)4; def y = (int)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (Long)4; def y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (Float)4; def y = (float)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (Double)4; def y = (double)1; return x ^ y"));
     }
 
     public void testOr() {
-        assertEquals(5, exec("def x = (byte)4 def y = (byte)1 return x | y"));
-        assertEquals(5, exec("def x = (short)4 def y = (byte)1 return x | y"));
-        assertEquals(5, exec("def x = (char)4 def y = (byte)1 return x | y"));
-        assertEquals(5, exec("def x = (int)4 def y = (byte)1 return x | y"));
-        assertEquals(5L, exec("def x = (long)4 def y = (byte)1 return x | y"));
-        assertEquals(5L, exec("def x = (float)4 def y = (byte)1 return x | y"));
-        assertEquals(5L, exec("def x = (double)4 def y = (byte)1 return x | y"));
+        assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x | y"));
+        assertEquals(5, exec("def x = (short)4; def y = (byte)1; return x | y"));
+        assertEquals(5, exec("def x = (char)4; def y = (byte)1; return x | y"));
+        assertEquals(5, exec("def x = (int)4; def y = (byte)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (byte)1; return x | y"));
+        assertEquals(5L, exec("def x = (float)4; def y = (byte)1; return x | y"));
+        assertEquals(5L, exec("def x = (double)4; def y = (byte)1; return x | y"));
 
-        assertEquals(5, exec("def x = (byte)4 def y = (short)1 return x | y"));
-        assertEquals(5, exec("def x = (short)4 def y = (short)1 return x | y"));
-        assertEquals(5, exec("def x = (char)4 def y = (short)1 return x | y"));
-        assertEquals(5, exec("def x = (int)4 def y = (short)1 return x | y"));
-        assertEquals(5L, exec("def x = (long)4 def y = (short)1 return x | y"));
-        assertEquals(5L, exec("def x = (float)4 def y = (short)1 return x | y"));
-        assertEquals(5L, exec("def x = (double)4 def y = (short)1 return x | y"));
+        assertEquals(5, exec("def x = (byte)4; def y = (short)1; return x | y"));
+        assertEquals(5, exec("def x = (short)4; def y = (short)1; return x | y"));
+        assertEquals(5, exec("def x = (char)4; def y = (short)1; return x | y"));
+        assertEquals(5, exec("def x = (int)4; def y = (short)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (short)1; return x | y"));
+        assertEquals(5L, exec("def x = (float)4; def y = (short)1; return x | y"));
+        assertEquals(5L, exec("def x = (double)4; def y = (short)1; return x | y"));
 
-        assertEquals(5, exec("def x = (byte)4 def y = (char)1 return x | y"));
-        assertEquals(5, exec("def x = (short)4 def y = (char)1 return x | y"));
-        assertEquals(5, exec("def x = (char)4 def y = (char)1 return x | y"));
-        assertEquals(5, exec("def x = (int)4 def y = (char)1 return x | y"));
-        assertEquals(5L, exec("def x = (long)4 def y = (char)1 return x | y"));
-        assertEquals(5L, exec("def x = (float)4 def y = (char)1 return x | y"));
-        assertEquals(5L, exec("def x = (double)4 def y = (char)1 return x | y"));
+        assertEquals(5, exec("def x = (byte)4; def y = (char)1; return x | y"));
+        assertEquals(5, exec("def x = (short)4; def y = (char)1; return x | y"));
+        assertEquals(5, exec("def x = (char)4; def y = (char)1; return x | y"));
+        assertEquals(5, exec("def x = (int)4; def y = (char)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (char)1; return x | y"));
+        assertEquals(5L, exec("def x = (float)4; def y = (char)1; return x | y"));
+        assertEquals(5L, exec("def x = (double)4; def y = (char)1; return x | y"));
 
-        assertEquals(5, exec("def x = (byte)4 def y = (int)1 return x | y"));
-        assertEquals(5, exec("def x = (short)4 def y = (int)1 return x | y"));
-        assertEquals(5, exec("def x = (char)4 def y = (int)1 return x | y"));
-        assertEquals(5, exec("def x = (int)4 def y = (int)1 return x | y"));
-        assertEquals(5L, exec("def x = (long)4 def y = (int)1 return x | y"));
-        assertEquals(5L, exec("def x = (float)4 def y = (int)1 return x | y"));
-        assertEquals(5L, exec("def x = (double)4 def y = (int)1 return x | y"));
+        assertEquals(5, exec("def x = (byte)4; def y = (int)1; return x | y"));
+        assertEquals(5, exec("def x = (short)4; def y = (int)1; return x | y"));
+        assertEquals(5, exec("def x = (char)4; def y = (int)1; return x | y"));
+        assertEquals(5, exec("def x = (int)4; def y = (int)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (int)1; return x | y"));
+        assertEquals(5L, exec("def x = (float)4; def y = (int)1; return x | y"));
+        assertEquals(5L, exec("def x = (double)4; def y = (int)1; return x | y"));
 
-        assertEquals(5L, exec("def x = (byte)4 def y = (long)1 return x | y"));
-        assertEquals(5L, exec("def x = (short)4 def y = (long)1 return x | y"));
-        assertEquals(5L, exec("def x = (char)4 def y = (long)1 return x | y"));
-        assertEquals(5L, exec("def x = (int)4 def y = (long)1 return x | y"));
-        assertEquals(5L, exec("def x = (long)4 def y = (long)1 return x | y"));
-        assertEquals(5L, exec("def x = (float)4 def y = (long)1 return x | y"));
-        assertEquals(5L, exec("def x = (double)4 def y = (long)1 return x | y"));
+        assertEquals(5L, exec("def x = (byte)4; def y = (long)1; return x | y"));
+        assertEquals(5L, exec("def x = (short)4; def y = (long)1; return x | y"));
+        assertEquals(5L, exec("def x = (char)4; def y = (long)1; return x | y"));
+        assertEquals(5L, exec("def x = (int)4; def y = (long)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x | y"));
+        assertEquals(5L, exec("def x = (float)4; def y = (long)1; return x | y"));
+        assertEquals(5L, exec("def x = (double)4; def y = (long)1; return x | y"));
 
-        assertEquals(5L, exec("def x = (byte)4 def y = (float)1 return x | y"));
-        assertEquals(5L, exec("def x = (short)4 def y = (float)1 return x | y"));
-        assertEquals(5L, exec("def x = (char)4 def y = (float)1 return x | y"));
-        assertEquals(5L, exec("def x = (int)4 def y = (float)1 return x | y"));
-        assertEquals(5L, exec("def x = (long)4 def y = (float)1 return x | y"));
-        assertEquals(5L, exec("def x = (float)4 def y = (float)1 return x | y"));
-        assertEquals(5L, exec("def x = (double)4 def y = (float)1 return x | y"));
+        assertEquals(5L, exec("def x = (byte)4; def y = (float)1; return x | y"));
+        assertEquals(5L, exec("def x = (short)4; def y = (float)1; return x | y"));
+        assertEquals(5L, exec("def x = (char)4; def y = (float)1; return x | y"));
+        assertEquals(5L, exec("def x = (int)4; def y = (float)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (float)1; return x | y"));
+        assertEquals(5L, exec("def x = (float)4; def y = (float)1; return x | y"));
+        assertEquals(5L, exec("def x = (double)4; def y = (float)1; return x | y"));
 
-        assertEquals(5L, exec("def x = (byte)4 def y = (double)1 return x | y"));
-        assertEquals(5L, exec("def x = (short)4 def y = (double)1 return x | y"));
-        assertEquals(5L, exec("def x = (char)4 def y = (double)1 return x | y"));
-        assertEquals(5L, exec("def x = (int)4 def y = (double)1 return x | y"));
-        assertEquals(5L, exec("def x = (long)4 def y = (double)1 return x | y"));
-        assertEquals(5L, exec("def x = (float)4 def y = (double)1 return x | y"));
-        assertEquals(5L, exec("def x = (double)4 def y = (double)1 return x | y"));
+        assertEquals(5L, exec("def x = (byte)4; def y = (double)1; return x | y"));
+        assertEquals(5L, exec("def x = (short)4; def y = (double)1; return x | y"));
+        assertEquals(5L, exec("def x = (char)4; def y = (double)1; return x | y"));
+        assertEquals(5L, exec("def x = (int)4; def y = (double)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (double)1; return x | y"));
+        assertEquals(5L, exec("def x = (float)4; def y = (double)1; return x | y"));
+        assertEquals(5L, exec("def x = (double)4; def y = (double)1; return x | y"));
 
-        assertEquals(5, exec("def x = (Byte)4 def y = (byte)1 return x | y"));
-        assertEquals(5, exec("def x = (Short)4 def y = (short)1 return x | y"));
-        assertEquals(5, exec("def x = (Character)4 def y = (char)1 return x | y"));
-        assertEquals(5, exec("def x = (Integer)4 def y = (int)1 return x | y"));
-        assertEquals(5L, exec("def x = (Long)4 def y = (long)1 return x | y"));
-        assertEquals(5L, exec("def x = (Float)4 def y = (float)1 return x | y"));
-        assertEquals(5L, exec("def x = (Double)4 def y = (double)1 return x | y"));
+        assertEquals(5, exec("def x = (Byte)4; def y = (byte)1; return x | y"));
+        assertEquals(5, exec("def x = (Short)4; def y = (short)1; return x | y"));
+        assertEquals(5, exec("def x = (Character)4; def y = (char)1; return x | y"));
+        assertEquals(5, exec("def x = (Integer)4; def y = (int)1; return x | y"));
+        assertEquals(5L, exec("def x = (Long)4; def y = (long)1; return x | y"));
+        assertEquals(5L, exec("def x = (Float)4; def y = (float)1; return x | y"));
+        assertEquals(5L, exec("def x = (Double)4; def y = (double)1; return x | y"));
     }
 
     public void testEq() {
-        assertEquals(true, exec("def x = (byte)7 def y = (int)7 return x == y"));
-        assertEquals(true, exec("def x = (short)6 def y = (int)6 return x == y"));
-        assertEquals(true, exec("def x = (char)5 def y = (int)5 return x == y"));
-        assertEquals(true, exec("def x = (int)4 def y = (int)4 return x == y"));
-        assertEquals(false, exec("def x = (long)5 def y = (int)3 return x == y"));
-        assertEquals(false, exec("def x = (float)6 def y = (int)2 return x == y"));
-        assertEquals(false, exec("def x = (double)7 def y = (int)1 return x == y"));
+        assertEquals(true, exec("def x = (byte)7; def y = (int)7; return x == y"));
+        assertEquals(true, exec("def x = (short)6; def y = (int)6; return x == y"));
+        assertEquals(true, exec("def x = (char)5; def y = (int)5; return x == y"));
+        assertEquals(true, exec("def x = (int)4; def y = (int)4; return x == y"));
+        assertEquals(false, exec("def x = (long)5; def y = (int)3; return x == y"));
+        assertEquals(false, exec("def x = (float)6; def y = (int)2; return x == y"));
+        assertEquals(false, exec("def x = (double)7; def y = (int)1; return x == y"));
 
-        assertEquals(true, exec("def x = (byte)7 def y = (double)7 return x == y"));
-        assertEquals(true, exec("def x = (short)6 def y = (double)6 return x == y"));
-        assertEquals(true, exec("def x = (char)5 def y = (double)5 return x == y"));
-        assertEquals(true, exec("def x = (int)4 def y = (double)4 return x == y"));
-        assertEquals(false, exec("def x = (long)5 def y = (double)3 return x == y"));
-        assertEquals(false, exec("def x = (float)6 def y = (double)2 return x == y"));
-        assertEquals(false, exec("def x = (double)7 def y = (double)1 return x == y"));
+        assertEquals(true, exec("def x = (byte)7; def y = (double)7; return x == y"));
+        assertEquals(true, exec("def x = (short)6; def y = (double)6; return x == y"));
+        assertEquals(true, exec("def x = (char)5; def y = (double)5; return x == y"));
+        assertEquals(true, exec("def x = (int)4; def y = (double)4; return x == y"));
+        assertEquals(false, exec("def x = (long)5; def y = (double)3; return x == y"));
+        assertEquals(false, exec("def x = (float)6; def y = (double)2; return x == y"));
+        assertEquals(false, exec("def x = (double)7; def y = (double)1; return x == y"));
 
-        assertEquals(true, exec("def x = new HashMap() def y = new HashMap() return x == y"));
-        assertEquals(false, exec("def x = new HashMap() x.put(3, 3) def y = new HashMap() return x == y"));
-        assertEquals(true, exec("def x = new HashMap() x.put(3, 3) def y = new HashMap() y.put(3, 3) return x == y"));
-        assertEquals(true, exec("def x = new HashMap() def y = x x.put(3, 3) y.put(3, 3) return x == y"));
+        assertEquals(true, exec("def x = new HashMap(); def y = new HashMap(); return x == y"));
+        assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x == y"));
+        assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x == y"));
+        assertEquals(true, exec("def x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x == y"));
     }
 
     public void testEqr() {
-        assertEquals(false, exec("def x = (byte)7 def y = (int)7 return x === y"));
-        assertEquals(false, exec("def x = (short)6 def y = (int)6 return x === y"));
-        assertEquals(false, exec("def x = (char)5 def y = (int)5 return x === y"));
-        assertEquals(true, exec("def x = (int)4 def y = (int)4 return x === y"));
-        assertEquals(false, exec("def x = (long)5 def y = (int)3 return x === y"));
-        assertEquals(false, exec("def x = (float)6 def y = (int)2 return x === y"));
-        assertEquals(false, exec("def x = (double)7 def y = (int)1 return x === y"));
+        assertEquals(false, exec("def x = (byte)7; def y = (int)7; return x === y"));
+        assertEquals(false, exec("def x = (short)6; def y = (int)6; return x === y"));
+        assertEquals(false, exec("def x = (char)5; def y = (int)5; return x === y"));
+        assertEquals(true, exec("def x = (int)4; def y = (int)4; return x === y"));
+        assertEquals(false, exec("def x = (long)5; def y = (int)3; return x === y"));
+        assertEquals(false, exec("def x = (float)6; def y = (int)2; return x === y"));
+        assertEquals(false, exec("def x = (double)7; def y = (int)1; return x === y"));
 
-        assertEquals(false, exec("def x = new HashMap() def y = new HashMap() return x === y"));
-        assertEquals(false, exec("def x = new HashMap() x.put(3, 3) def y = new HashMap() return x === y"));
-        assertEquals(false, exec("def x = new HashMap() x.put(3, 3) def y = new HashMap() y.put(3, 3) return x === y"));
-        assertEquals(true, exec("def x = new HashMap() def y = x x.put(3, 3) y.put(3, 3) return x === y"));
+        assertEquals(false, exec("def x = new HashMap(); def y = new HashMap(); return x === y"));
+        assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x === y"));
+        assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x === y"));
+        assertEquals(true, exec("def x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x === y"));
     }
 
     public void testNe() {
-        assertEquals(false, exec("def x = (byte)7 def y = (int)7 return x != y"));
-        assertEquals(false, exec("def x = (short)6 def y = (int)6 return x != y"));
-        assertEquals(false, exec("def x = (char)5 def y = (int)5 return x != y"));
-        assertEquals(false, exec("def x = (int)4 def y = (int)4 return x != y"));
-        assertEquals(true, exec("def x = (long)5 def y = (int)3 return x != y"));
-        assertEquals(true, exec("def x = (float)6 def y = (int)2 return x != y"));
-        assertEquals(true, exec("def x = (double)7 def y = (int)1 return x != y"));
+        assertEquals(false, exec("def x = (byte)7; def y = (int)7; return x != y"));
+        assertEquals(false, exec("def x = (short)6; def y = (int)6; return x != y"));
+        assertEquals(false, exec("def x = (char)5; def y = (int)5; return x != y"));
+        assertEquals(false, exec("def x = (int)4; def y = (int)4; return x != y"));
+        assertEquals(true, exec("def x = (long)5; def y = (int)3; return x != y"));
+        assertEquals(true, exec("def x = (float)6; def y = (int)2; return x != y"));
+        assertEquals(true, exec("def x = (double)7; def y = (int)1; return x != y"));
 
-        assertEquals(false, exec("def x = (byte)7 def y = (double)7 return x != y"));
-        assertEquals(false, exec("def x = (short)6 def y = (double)6 return x != y"));
-        assertEquals(false, exec("def x = (char)5 def y = (double)5 return x != y"));
-        assertEquals(false, exec("def x = (int)4 def y = (double)4 return x != y"));
-        assertEquals(true, exec("def x = (long)5 def y = (double)3 return x != y"));
-        assertEquals(true, exec("def x = (float)6 def y = (double)2 return x != y"));
-        assertEquals(true, exec("def x = (double)7 def y = (double)1 return x != y"));
+        assertEquals(false, exec("def x = (byte)7; def y = (double)7; return x != y"));
+        assertEquals(false, exec("def x = (short)6; def y = (double)6; return x != y"));
+        assertEquals(false, exec("def x = (char)5; def y = (double)5; return x != y"));
+        assertEquals(false, exec("def x = (int)4; def y = (double)4; return x != y"));
+        assertEquals(true, exec("def x = (long)5; def y = (double)3; return x != y"));
+        assertEquals(true, exec("def x = (float)6; def y = (double)2; return x != y"));
+        assertEquals(true, exec("def x = (double)7; def y = (double)1; return x != y"));
 
-        assertEquals(false, exec("def x = new HashMap() def y = new HashMap() return x != y"));
-        assertEquals(true, exec("def x = new HashMap() x.put(3, 3) def y = new HashMap() return x != y"));
-        assertEquals(false, exec("def x = new HashMap() x.put(3, 3) def y = new HashMap() y.put(3, 3) return x != y"));
-        assertEquals(false, exec("def x = new HashMap() def y = x x.put(3, 3) y.put(3, 3) return x != y"));
+        assertEquals(false, exec("def x = new HashMap(); def y = new HashMap(); return x != y"));
+        assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x != y"));
+        assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x != y"));
+        assertEquals(false, exec("def x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x != y"));
     }
 
     public void testNer() {
-        assertEquals(true, exec("def x = (byte)7 def y = (int)7 return x !== y"));
-        assertEquals(true, exec("def x = (short)6 def y = (int)6 return x !== y"));
-        assertEquals(true, exec("def x = (char)5 def y = (int)5 return x !== y"));
-        assertEquals(false, exec("def x = (int)4 def y = (int)4 return x !== y"));
-        assertEquals(true, exec("def x = (long)5 def y = (int)3 return x !== y"));
-        assertEquals(true, exec("def x = (float)6 def y = (int)2 return x !== y"));
-        assertEquals(true, exec("def x = (double)7 def y = (int)1 return x !== y"));
+        assertEquals(true, exec("def x = (byte)7; def y = (int)7; return x !== y"));
+        assertEquals(true, exec("def x = (short)6; def y = (int)6; return x !== y"));
+        assertEquals(true, exec("def x = (char)5; def y = (int)5; return x !== y"));
+        assertEquals(false, exec("def x = (int)4; def y = (int)4; return x !== y"));
+        assertEquals(true, exec("def x = (long)5; def y = (int)3; return x !== y"));
+        assertEquals(true, exec("def x = (float)6; def y = (int)2; return x !== y"));
+        assertEquals(true, exec("def x = (double)7; def y = (int)1; return x !== y"));
 
-        assertEquals(true, exec("def x = new HashMap() def y = new HashMap() return x !== y"));
-        assertEquals(true, exec("def x = new HashMap() x.put(3, 3) def y = new HashMap() return x !== y"));
-        assertEquals(true, exec("def x = new HashMap() x.put(3, 3) def y = new HashMap() y.put(3, 3) return x !== y"));
-        assertEquals(false, exec("def x = new HashMap() def y = x x.put(3, 3) y.put(3, 3) return x !== y"));
+        assertEquals(true, exec("def x = new HashMap(); def y = new HashMap(); return x !== y"));
+        assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x !== y"));
+        assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x !== y"));
+        assertEquals(false, exec("def x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x !== y"));
     }
 
     public void testLt() {
-        assertEquals(true, exec("def x = (byte)1 def y = (int)7 return x < y"));
-        assertEquals(true, exec("def x = (short)2 def y = (int)6 return x < y"));
-        assertEquals(true, exec("def x = (char)3 def y = (int)5 return x < y"));
-        assertEquals(false, exec("def x = (int)4 def y = (int)4 return x < y"));
-        assertEquals(false, exec("def x = (long)5 def y = (int)3 return x < y"));
-        assertEquals(false, exec("def x = (float)6 def y = (int)2 return x < y"));
-        assertEquals(false, exec("def x = (double)7 def y = (int)1 return x < y"));
+        assertEquals(true, exec("def x = (byte)1; def y = (int)7; return x < y"));
+        assertEquals(true, exec("def x = (short)2; def y = (int)6; return x < y"));
+        assertEquals(true, exec("def x = (char)3; def y = (int)5; return x < y"));
+        assertEquals(false, exec("def x = (int)4; def y = (int)4; return x < y"));
+        assertEquals(false, exec("def x = (long)5; def y = (int)3; return x < y"));
+        assertEquals(false, exec("def x = (float)6; def y = (int)2; return x < y"));
+        assertEquals(false, exec("def x = (double)7; def y = (int)1; return x < y"));
 
-        assertEquals(true, exec("def x = (byte)1 def y = (double)7 return x < y"));
-        assertEquals(true, exec("def x = (short)2 def y = (double)6 return x < y"));
-        assertEquals(true, exec("def x = (char)3 def y = (double)5 return x < y"));
-        assertEquals(false, exec("def x = (int)4 def y = (double)4 return x < y"));
-        assertEquals(false, exec("def x = (long)5 def y = (double)3 return x < y"));
-        assertEquals(false, exec("def x = (float)6 def y = (double)2 return x < y"));
-        assertEquals(false, exec("def x = (double)7 def y = (double)1 return x < y"));
+        assertEquals(true, exec("def x = (byte)1; def y = (double)7; return x < y"));
+        assertEquals(true, exec("def x = (short)2; def y = (double)6; return x < y"));
+        assertEquals(true, exec("def x = (char)3; def y = (double)5; return x < y"));
+        assertEquals(false, exec("def x = (int)4; def y = (double)4; return x < y"));
+        assertEquals(false, exec("def x = (long)5; def y = (double)3; return x < y"));
+        assertEquals(false, exec("def x = (float)6; def y = (double)2; return x < y"));
+        assertEquals(false, exec("def x = (double)7; def y = (double)1; return x < y"));
     }
 
     public void testLte() {
-        assertEquals(true, exec("def x = (byte)1 def y = (int)7 return x <= y"));
-        assertEquals(true, exec("def x = (short)2 def y = (int)6 return x <= y"));
-        assertEquals(true, exec("def x = (char)3 def y = (int)5 return x <= y"));
-        assertEquals(true, exec("def x = (int)4 def y = (int)4 return x <= y"));
-        assertEquals(false, exec("def x = (long)5 def y = (int)3 return x <= y"));
-        assertEquals(false, exec("def x = (float)6 def y = (int)2 return x <= y"));
-        assertEquals(false, exec("def x = (double)7 def y = (int)1 return x <= y"));
+        assertEquals(true, exec("def x = (byte)1; def y = (int)7; return x <= y"));
+        assertEquals(true, exec("def x = (short)2; def y = (int)6; return x <= y"));
+        assertEquals(true, exec("def x = (char)3; def y = (int)5; return x <= y"));
+        assertEquals(true, exec("def x = (int)4; def y = (int)4; return x <= y"));
+        assertEquals(false, exec("def x = (long)5; def y = (int)3; return x <= y"));
+        assertEquals(false, exec("def x = (float)6; def y = (int)2; return x <= y"));
+        assertEquals(false, exec("def x = (double)7; def y = (int)1; return x <= y"));
 
-        assertEquals(true, exec("def x = (byte)1 def y = (double)7 return x <= y"));
-        assertEquals(true, exec("def x = (short)2 def y = (double)6 return x <= y"));
-        assertEquals(true, exec("def x = (char)3 def y = (double)5 return x <= y"));
-        assertEquals(true, exec("def x = (int)4 def y = (double)4 return x <= y"));
-        assertEquals(false, exec("def x = (long)5 def y = (double)3 return x <= y"));
-        assertEquals(false, exec("def x = (float)6 def y = (double)2 return x <= y"));
-        assertEquals(false, exec("def x = (double)7 def y = (double)1 return x <= y"));
+        assertEquals(true, exec("def x = (byte)1; def y = (double)7; return x <= y"));
+        assertEquals(true, exec("def x = (short)2; def y = (double)6; return x <= y"));
+        assertEquals(true, exec("def x = (char)3; def y = (double)5; return x <= y"));
+        assertEquals(true, exec("def x = (int)4; def y = (double)4; return x <= y"));
+        assertEquals(false, exec("def x = (long)5; def y = (double)3; return x <= y"));
+        assertEquals(false, exec("def x = (float)6; def y = (double)2; return x <= y"));
+        assertEquals(false, exec("def x = (double)7; def y = (double)1; return x <= y"));
     }
 
     public void testGt() {
-        assertEquals(false, exec("def x = (byte)1 def y = (int)7 return x > y"));
-        assertEquals(false, exec("def x = (short)2 def y = (int)6 return x > y"));
-        assertEquals(false, exec("def x = (char)3 def y = (int)5 return x > y"));
-        assertEquals(false, exec("def x = (int)4 def y = (int)4 return x > y"));
-        assertEquals(true, exec("def x = (long)5 def y = (int)3 return x > y"));
-        assertEquals(true, exec("def x = (float)6 def y = (int)2 return x > y"));
-        assertEquals(true, exec("def x = (double)7 def y = (int)1 return x > y"));
+        assertEquals(false, exec("def x = (byte)1; def y = (int)7; return x > y"));
+        assertEquals(false, exec("def x = (short)2; def y = (int)6; return x > y"));
+        assertEquals(false, exec("def x = (char)3; def y = (int)5; return x > y"));
+        assertEquals(false, exec("def x = (int)4; def y = (int)4; return x > y"));
+        assertEquals(true, exec("def x = (long)5; def y = (int)3; return x > y"));
+        assertEquals(true, exec("def x = (float)6; def y = (int)2; return x > y"));
+        assertEquals(true, exec("def x = (double)7; def y = (int)1; return x > y"));
 
-        assertEquals(false, exec("def x = (byte)1 def y = (double)7 return x > y"));
-        assertEquals(false, exec("def x = (short)2 def y = (double)6 return x > y"));
-        assertEquals(false, exec("def x = (char)3 def y = (double)5 return x > y"));
-        assertEquals(false, exec("def x = (int)4 def y = (double)4 return x > y"));
-        assertEquals(true, exec("def x = (long)5 def y = (double)3 return x > y"));
-        assertEquals(true, exec("def x = (float)6 def y = (double)2 return x > y"));
-        assertEquals(true, exec("def x = (double)7 def y = (double)1 return x > y"));
+        assertEquals(false, exec("def x = (byte)1; def y = (double)7; return x > y"));
+        assertEquals(false, exec("def x = (short)2; def y = (double)6; return x > y"));
+        assertEquals(false, exec("def x = (char)3; def y = (double)5; return x > y"));
+        assertEquals(false, exec("def x = (int)4; def y = (double)4; return x > y"));
+        assertEquals(true, exec("def x = (long)5; def y = (double)3; return x > y"));
+        assertEquals(true, exec("def x = (float)6; def y = (double)2; return x > y"));
+        assertEquals(true, exec("def x = (double)7; def y = (double)1; return x > y"));
     }
 
     public void testGte() {
-        assertEquals(false, exec("def x = (byte)1 def y = (int)7 return x >= y"));
-        assertEquals(false, exec("def x = (short)2 def y = (int)6 return x >= y"));
-        assertEquals(false, exec("def x = (char)3 def y = (int)5 return x >= y"));
-        assertEquals(true, exec("def x = (int)4 def y = (int)4 return x >= y"));
-        assertEquals(true, exec("def x = (long)5 def y = (int)3 return x >= y"));
-        assertEquals(true, exec("def x = (float)6 def y = (int)2 return x >= y"));
-        assertEquals(true, exec("def x = (double)7 def y = (int)1 return x >= y"));
+        assertEquals(false, exec("def x = (byte)1; def y = (int)7; return x >= y"));
+        assertEquals(false, exec("def x = (short)2; def y = (int)6; return x >= y"));
+        assertEquals(false, exec("def x = (char)3; def y = (int)5; return x >= y"));
+        assertEquals(true, exec("def x = (int)4; def y = (int)4; return x >= y"));
+        assertEquals(true, exec("def x = (long)5; def y = (int)3; return x >= y"));
+        assertEquals(true, exec("def x = (float)6; def y = (int)2; return x >= y"));
+        assertEquals(true, exec("def x = (double)7; def y = (int)1; return x >= y"));
 
-        assertEquals(false, exec("def x = (byte)1 def y = (double)7 return x >= y"));
-        assertEquals(false, exec("def x = (short)2 def y = (double)6 return x >= y"));
-        assertEquals(false, exec("def x = (char)3 def y = (double)5 return x >= y"));
-        assertEquals(true, exec("def x = (int)4 def y = (double)4 return x >= y"));
-        assertEquals(true, exec("def x = (long)5 def y = (double)3 return x >= y"));
-        assertEquals(true, exec("def x = (float)6 def y = (double)2 return x >= y"));
-        assertEquals(true, exec("def x = (double)7 def y = (double)1 return x >= y"));
+        assertEquals(false, exec("def x = (byte)1; def y = (double)7; return x >= y"));
+        assertEquals(false, exec("def x = (short)2; def y = (double)6; return x >= y"));
+        assertEquals(false, exec("def x = (char)3; def y = (double)5; return x >= y"));
+        assertEquals(true, exec("def x = (int)4; def y = (double)4; return x >= y"));
+        assertEquals(true, exec("def x = (long)5; def y = (double)3; return x >= y"));
+        assertEquals(true, exec("def x = (float)6; def y = (double)2; return x >= y"));
+        assertEquals(true, exec("def x = (double)7; def y = (double)1; return x >= y"));
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NoSemiColonTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NoSemiColonTests.java
index 5ca98c2e575..f2e65fc680c 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NoSemiColonTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NoSemiColonTests.java
@@ -19,161 +19,60 @@
 
 package org.elasticsearch.painless;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
 public class NoSemiColonTests extends ScriptTestCase {
 
-    public void testIfStatement() {
-        assertEquals(1, exec("int x = 5 if (x == 5) return 1 return 0"));
-        assertEquals(0, exec("int x = 4 if (x == 5) return 1 else return 0"));
-        assertEquals(2, exec("int x = 4 if (x == 5) return 1 else if (x == 4) return 2 else return 0"));
-        assertEquals(1, exec("int x = 4 if (x == 5) return 1 else if (x == 4) return 1 else return 0"));
-
-        assertEquals(3, exec(
-                "int x = 5\n" +
-                        "if (x == 5) {\n" +
-                        "    int y = 2\n" +
-                        "    \n" +
-                        "    if (y == 2) {\n" +
-                        "        x = 3\n" +
-                        "    }\n" +
-                        "    \n" +
-                        "}\n" +
-                        "\n" +
-                        "return x\n"));
-    }
-
-    public void testWhileStatement() {
-
-        assertEquals("aaaaaa", exec("String c = \"a\" int x while (x < 5) { ++x c += \"a\" } return c"));
-
-        Object value = exec(
-                " byte[][] b = new byte[5][5]       \n" +
-                " byte x = 0, y                     \n" +
-                "                                   \n" +
-                " while (x < 5) {                   \n" +
-                "     y = 0                         \n" +
-                "                                   \n" +
-                "     while (y < 5) {               \n" +
-                "         b[x][y] = (byte)(x*y)     \n" +
-                "         ++y                       \n" +
-                "     }                             \n" +
-                "                                   \n" +
-                "     ++x                           \n" +
-                " }                                 \n" +
-                "                                   \n" +
-                " return b                          \n");
-
-        byte[][] b = (byte[][])value;
-
-        for (byte x = 0; x < 5; ++x) {
-            for (byte y = 0; y < 5; ++y) {
-                assertEquals(x*y, b[x][y]);
-            }
-        }
-    }
-
-    public void testDoWhileStatement() {
-        assertEquals("aaaaaa", exec("String c = \"a\" int x do { c += \"a\"; ++x } while (x < 5) return c"));
-
-        Object value = exec(
-                " long[][] l = new long[5][5]      \n" +
-                " long x = 0, y                    \n" +
-                "                                  \n" +
-                " do {                             \n" +
-                "     y = 0                        \n" +
-                "                                  \n" +
-                "     do {                         \n" +
-                "         l[(int)x][(int)y] = x*y; \n" +
-                "         ++y                      \n" +
-                "     } while (y < 5)              \n" +
-                "                                  \n" +
-                "     ++x                          \n" +
-                " } while (x < 5)                  \n" +
-                "                                  \n" +
-                " return l                         \n");
-
-        long[][] l = (long[][])value;
-
-        for (long x = 0; x < 5; ++x) {
-            for (long y = 0; y < 5; ++y) {
-                assertEquals(x*y, l[(int)x][(int)y]);
-            }
-        }
-    }
-
-    public void testForStatement() {
-        assertEquals("aaaaaa", exec("String c = \"a\" for (int x = 0; x < 5; ++x) c += \"a\" return c"));
-
-        Object value = exec(
-                " int[][] i = new int[5][5]         \n" +
-                " for (int x = 0; x < 5; ++x) {     \n" +
-                "     for (int y = 0; y < 5; ++y) { \n" +
-                "         i[x][y] = x*y             \n" +
-                "     }                             \n" +
-                " }                                 \n" +
-                "                                   \n" +
-                " return i                          \n");
-
-        int[][] i = (int[][])value;
-
-        for (int x = 0; x < 5; ++x) {
-            for (int y = 0; y < 5; ++y) {
-                assertEquals(x*y, i[x][y]);
-            }
-        }
-    }
-
     public void testDeclarationStatement() {
-        assertEquals((byte)2, exec("byte a = 2 return a"));
-        assertEquals((short)2, exec("short a = 2 return a"));
-        assertEquals((char)2, exec("char a = 2 return a"));
-        assertEquals(2, exec("int a = 2 return a"));
-        assertEquals(2L, exec("long a = 2 return a"));
-        assertEquals(2F, exec("float a = 2 return a"));
-        assertEquals(2.0, exec("double a = 2 return a"));
-        assertEquals(false, exec("boolean a = false return a"));
-        assertEquals("string", exec("String a = \"string\" return a"));
-        assertEquals(HashMap.class, exec("Map<String, Object> a = new HashMap<String, Object>() return a").getClass());
+        assertEquals((byte)2, exec("byte a = 2; return a"));
+        assertEquals((short)2, exec("short a = 2; return a"));
+        assertEquals((char)2, exec("char a = 2; return a"));
+        assertEquals(2, exec("int a = 2; return a"));
+        assertEquals(2L, exec("long a = 2; return a"));
+        assertEquals(2F, exec("float a = 2; return a"));
+        assertEquals(2.0, exec("double a = 2; return a"));
+        assertEquals(false, exec("boolean a = false; return a"));
+        assertEquals("string", exec("String a = \"string\"; return a"));
+        assertEquals(HashMap.class, exec("Map<String, Object> a = new HashMap<String, Object>(); return a").getClass());
 
-        assertEquals(byte[].class, exec("byte[] a = new byte[1] return a").getClass());
-        assertEquals(short[].class, exec("short[] a = new short[1] return a").getClass());
-        assertEquals(char[].class, exec("char[] a = new char[1] return a").getClass());
-        assertEquals(int[].class, exec("int[] a = new int[1] return a").getClass());
-        assertEquals(long[].class, exec("long[] a = new long[1] return a").getClass());
-        assertEquals(float[].class, exec("float[] a = new float[1] return a").getClass());
-        assertEquals(double[].class, exec("double[] a = new double[1] return a").getClass());
-        assertEquals(boolean[].class, exec("boolean[] a = new boolean[1] return a").getClass());
-        assertEquals(String[].class, exec("String[] a = new String[1] return a").getClass());
-        assertEquals(Map[].class, exec("Map<String,Object>[] a = new Map<String,Object>[1] return a").getClass());
+        assertEquals(byte[].class, exec("byte[] a = new byte[1]; return a").getClass());
+        assertEquals(short[].class, exec("short[] a = new short[1]; return a").getClass());
+        assertEquals(char[].class, exec("char[] a = new char[1]; return a").getClass());
+        assertEquals(int[].class, exec("int[] a = new int[1]; return a").getClass());
+        assertEquals(long[].class, exec("long[] a = new long[1]; return a").getClass());
+        assertEquals(float[].class, exec("float[] a = new float[1]; return a").getClass());
+        assertEquals(double[].class, exec("double[] a = new double[1]; return a").getClass());
+        assertEquals(boolean[].class, exec("boolean[] a = new boolean[1]; return a").getClass());
+        assertEquals(String[].class, exec("String[] a = new String[1]; return a").getClass());
+        assertEquals(Map[].class, exec("Map<String,Object>[] a = new Map<String,Object>[1]; return a").getClass());
 
-        assertEquals(byte[][].class, exec("byte[][] a = new byte[1][2] return a").getClass());
-        assertEquals(short[][][].class, exec("short[][][] a = new short[1][2][3] return a").getClass());
-        assertEquals(char[][][][].class, exec("char[][][][] a = new char[1][2][3][4] return a").getClass());
-        assertEquals(int[][][][][].class, exec("int[][][][][] a = new int[1][2][3][4][5] return a").getClass());
-        assertEquals(long[][].class, exec("long[][] a = new long[1][2] return a").getClass());
-        assertEquals(float[][][].class, exec("float[][][] a = new float[1][2][3] return a").getClass());
-        assertEquals(double[][][][].class, exec("double[][][][] a = new double[1][2][3][4] return a").getClass());
-        assertEquals(boolean[][][][][].class, exec("boolean[][][][][] a = new boolean[1][2][3][4][5] return a").getClass());
-        assertEquals(String[][].class, exec("String[][] a = new String[1][2] return a").getClass());
-        assertEquals(Map[][][].class, exec("Map<String,Object>[][][] a = new Map<String,Object>[1][2][3] return a").getClass());
+        assertEquals(byte[][].class, exec("byte[][] a = new byte[1][2]; return a").getClass());
+        assertEquals(short[][][].class, exec("short[][][] a = new short[1][2][3]; return a").getClass());
+        assertEquals(char[][][][].class, exec("char[][][][] a = new char[1][2][3][4]; return a").getClass());
+        assertEquals(int[][][][][].class, exec("int[][][][][] a = new int[1][2][3][4][5]; return a").getClass());
+        assertEquals(long[][].class, exec("long[][] a = new long[1][2]; return a").getClass());
+        assertEquals(float[][][].class, exec("float[][][] a = new float[1][2][3]; return a").getClass());
+        assertEquals(double[][][][].class, exec("double[][][][] a = new double[1][2][3][4]; return a").getClass());
+        assertEquals(boolean[][][][][].class, exec("boolean[][][][][] a = new boolean[1][2][3][4][5]; return a").getClass());
+        assertEquals(String[][].class, exec("String[][] a = new String[1][2]; return a").getClass());
+        assertEquals(Map[][][].class, exec("Map<String,Object>[][][] a = new Map<String,Object>[1][2][3]; return a").getClass());
     }
-
-    public void testContinueStatement() {
-        assertEquals(9, exec("int x = 0, y = 0 while (x < 10) { ++x if (x == 1) continue ++y } return y"));
-    }
-
-    public void testBreakStatement() {
-        assertEquals(4, exec("int x = 0, y = 0 while (x < 10) { ++x if (x == 5) break ++y } return y"));
+    
+    public void testExpression() {
+        assertEquals(10, exec("10"));
+        assertEquals(10, exec("5 + 5"));
+        assertEquals(10, exec("5 + 5"));
+        assertEquals(10, exec("params.param == 'yes' ? 10 : 5", Collections.singletonMap("param", "yes")));
     }
 
     @SuppressWarnings("rawtypes")
     public void testReturnStatement() {
         assertEquals(10, exec("return 10"));
-        assertEquals(5, exec("int x = 5 return x"));
-        assertEquals(4, exec("int[] x = new int[2] x[1] = 4 return x[1]"));
-        assertEquals(5, ((short[])exec("short[] s = new short[3] s[1] = 5 return s"))[1]);
-        assertEquals(10, ((Map)exec("Map<String,Object> s = new HashMap< String,Object>() s.put(\"x\", 10) return s")).get("x"));
+        assertEquals(5, exec("int x = 5; return x"));
+        assertEquals(4, exec("int[] x = new int[2]; x[1] = 4; return x[1]"));
+        assertEquals(5, ((short[])exec("short[] s = new short[3]; s[1] = 5; return s"))[1]);
+        assertEquals(10, ((Map)exec("Map<String,Object> s = new HashMap< String,Object>(); s.put(\"x\", 10); return s")).get("x"));
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
index 0c2b2f4386c..95acf1826fd 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.painless;
 
+import java.util.Locale;
+
 public class StringTests extends ScriptTestCase {
 
     public void testAppend() {
@@ -63,6 +65,21 @@ public class StringTests extends ScriptTestCase {
         assertEquals("cat" + "cat", exec("String s = 'cat'; return s + s;"));
     }
 
+    public void testAppendMultiple() {
+      assertEquals("cat" + true + "abc" + null, exec("String s = \"cat\"; return s + true + 'abc' + null;"));
+    }
+
+    public void testAppendMany() {
+      StringBuilder script = new StringBuilder("String s = \"cat\"; return s");
+      StringBuilder result = new StringBuilder("cat");
+      for (int i = 0; i < 200 /* indy limit */ + 10; i++) {
+        final String s = String.format(Locale.ROOT,  "%03d", i);
+        script.append(" + '").append(s).append("'.toString()");
+        result.append(s);
+      }
+      assertEquals(result.toString(), exec(script.toString()));
+    }
+
     public void testStringAPI() {
         assertEquals("", exec("return new String();"));
         assertEquals('x', exec("String s = \"x\"; return s.charAt(0);"));
@@ -127,8 +144,8 @@ public class StringTests extends ScriptTestCase {
         assertEquals("c", exec("return (String)(char)\"c\""));
         assertEquals("c", exec("return (String)(char)'c'"));
 
-        assertEquals('c', exec("String s = \"c\" (char)s"));
-        assertEquals('c', exec("String s = 'c' (char)s"));
+        assertEquals('c', exec("String s = \"c\"; (char)s"));
+        assertEquals('c', exec("String s = 'c'; (char)s"));
 
         try {
             assertEquals("cc", exec("return (String)(char)\"cc\""));
@@ -145,14 +162,14 @@ public class StringTests extends ScriptTestCase {
         }
 
         try {
-            assertEquals('c', exec("String s = \"cc\" (char)s"));
+            assertEquals('c', exec("String s = \"cc\"; (char)s"));
             fail();
         } catch (final ClassCastException cce) {
             assertTrue(cce.getMessage().contains("Cannot cast [String] with length greater than one to [char]."));
         }
 
         try {
-            assertEquals('c', exec("String s = 'cc' (char)s"));
+            assertEquals('c', exec("String s = 'cc'; (char)s"));
             fail();
         } catch (final ClassCastException cce) {
             assertTrue(cce.getMessage().contains("Cannot cast [String] with length greater than one to [char]."));
@@ -163,8 +180,8 @@ public class StringTests extends ScriptTestCase {
         assertEquals("c", exec("return (String)(Character)\"c\""));
         assertEquals("c", exec("return (String)(Character)'c'"));
 
-        assertEquals('c', exec("String s = \"c\" (Character)s"));
-        assertEquals('c', exec("String s = 'c' (Character)s"));
+        assertEquals('c', exec("String s = \"c\"; (Character)s"));
+        assertEquals('c', exec("String s = 'c'; (Character)s"));
 
         try {
             assertEquals("cc", exec("return (String)(Character)\"cc\""));
@@ -181,14 +198,14 @@ public class StringTests extends ScriptTestCase {
         }
 
         try {
-            assertEquals('c', exec("String s = \"cc\" (Character)s"));
+            assertEquals('c', exec("String s = \"cc\"; (Character)s"));
             fail();
         } catch (final ClassCastException cce) {
             assertTrue(cce.getMessage().contains("Cannot cast [String] with length greater than one to [Character]."));
         }
 
         try {
-            assertEquals('c', exec("String s = 'cc' (Character)s"));
+            assertEquals('c', exec("String s = 'cc'; (Character)s"));
             fail();
         } catch (final ClassCastException cce) {
             assertTrue(cce.getMessage().contains("Cannot cast [String] with length greater than one to [Character]."));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
index 55ee490bb3f..15ae321534e 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
@@ -90,7 +90,7 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
                    "The maximum number of statements that can be executed in a loop has been reached."));
 
         expected = expectThrows(PainlessError.class, () -> {
-            exec("while (true) {int y = 5}");
+            exec("while (true) {int y = 5;}");
         });
         assertTrue(expected.getMessage().contains(
                    "The maximum number of statements that can be executed in a loop has been reached."));
@@ -116,7 +116,7 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
                    "The maximum number of statements that can be executed in a loop has been reached."));
 
         expected = expectThrows(PainlessError.class, () -> {
-            exec("for (;;) {int x = 5}");
+            exec("for (;;) {int x = 5;}");
             fail("should have hit PainlessError");
         });
         assertTrue(expected.getMessage().contains(
@@ -130,7 +130,7 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
                    "The maximum number of statements that can be executed in a loop has been reached."));
 
         RuntimeException parseException = expectThrows(RuntimeException.class, () -> {
-            exec("try { int x } catch (PainlessError error) {}");
+            exec("try { int x; } catch (PainlessError error) {}");
             fail("should have hit ParseException");
         });
         assertTrue(parseException.getMessage().contains("Not a type [PainlessError]."));

From 14b87835a018fb1b65fb54f65bf1428646afd0bb Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 17 May 2016 11:03:48 -0400
Subject: [PATCH 0291/1311] fix example in doc

---
 docs/reference/modules/scripting/painless.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/modules/scripting/painless.asciidoc b/docs/reference/modules/scripting/painless.asciidoc
index 156705bfe78..d79b59cf0d6 100644
--- a/docs/reference/modules/scripting/painless.asciidoc
+++ b/docs/reference/modules/scripting/painless.asciidoc
@@ -189,7 +189,7 @@ POST hockey/player/1/_update
 {
   "script": {
     "lang": "painless",
-    "inline": "ctx._source.last = params.last ctx._source.nick = params.nick",
+    "inline": "ctx._source.last = params.last; ctx._source.nick = params.nick",
     "params": {
       "last": "gaudreau",
       "nick": "hockey"

From b12cabd2f55fd430f533be00378a4f984d1dc6ac Mon Sep 17 00:00:00 2001
From: Kyle Gochenour <kyle.gochenour@gmail.com>
Date: Tue, 17 May 2016 11:10:32 -0400
Subject: [PATCH 0292/1311] [docs] Add missing article

[docs] Add missing article to zen.asciidoc
---
 docs/reference/modules/discovery/zen.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/modules/discovery/zen.asciidoc b/docs/reference/modules/discovery/zen.asciidoc
index 12de0e3b736..60823393d11 100644
--- a/docs/reference/modules/discovery/zen.asciidoc
+++ b/docs/reference/modules/discovery/zen.asciidoc
@@ -52,7 +52,7 @@ with a timeout defaulting at 20 times the ping timeout.
 
 When the master node stops or has encountered a problem, the cluster nodes
 start pinging again and will elect a new master. This pinging round also
-serves as a protection against (partial) network failures where node may unjustly
+serves as a protection against (partial) network failures where a node may unjustly
 think that the master has failed. In this case the node will simply hear from
 other nodes about the currently active master.
 

From 5622e9c9d3d07e8916b9ad2e6c1440fa5fa1955b Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Tue, 17 May 2016 11:15:36 +0200
Subject: [PATCH 0293/1311] painless: use Java 9 Indy String Concats, if
 available

---
 .../elasticsearch/painless/MethodWriter.java  | 72 ++++++++++++++-----
 .../painless/WriterConstants.java             | 22 ++++++
 .../elasticsearch/painless/node/EBinary.java  |  4 +-
 .../elasticsearch/painless/node/EChain.java   |  4 +-
 .../elasticsearch/painless/StringTests.java   | 18 +++++
 5 files changed, 99 insertions(+), 21 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index 2daa5d5589c..a982b889194 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -29,6 +29,11 @@ import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.commons.GeneratorAdapter;
 import org.objectweb.asm.commons.Method;
 
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.List;
+
 import static org.elasticsearch.painless.WriterConstants.ADDEXACT_INT;
 import static org.elasticsearch.painless.WriterConstants.ADDEXACT_LONG;
 import static org.elasticsearch.painless.WriterConstants.ADDWOOVERLOW_DOUBLE;
@@ -48,6 +53,8 @@ import static org.elasticsearch.painless.WriterConstants.DIVWOOVERLOW_DOUBLE;
 import static org.elasticsearch.painless.WriterConstants.DIVWOOVERLOW_FLOAT;
 import static org.elasticsearch.painless.WriterConstants.DIVWOOVERLOW_INT;
 import static org.elasticsearch.painless.WriterConstants.DIVWOOVERLOW_LONG;
+import static org.elasticsearch.painless.WriterConstants.INDY_STRING_CONCAT_BOOTSTRAP_HANDLE;
+import static org.elasticsearch.painless.WriterConstants.MAX_INDY_STRING_CONCAT_ARGS;
 import static org.elasticsearch.painless.WriterConstants.MULEXACT_INT;
 import static org.elasticsearch.painless.WriterConstants.MULEXACT_LONG;
 import static org.elasticsearch.painless.WriterConstants.MULWOOVERLOW_DOUBLE;
@@ -66,6 +73,7 @@ import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_ST
 import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_CONSTRUCTOR;
 import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_TOSTRING;
 import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_TYPE;
+import static org.elasticsearch.painless.WriterConstants.STRING_TYPE;
 import static org.elasticsearch.painless.WriterConstants.SUBEXACT_INT;
 import static org.elasticsearch.painless.WriterConstants.SUBEXACT_LONG;
 import static org.elasticsearch.painless.WriterConstants.SUBWOOVERLOW_DOUBLE;
@@ -97,6 +105,8 @@ import static org.elasticsearch.painless.WriterConstants.TOSHORTWOOVERFLOW_FLOAT
  */
 public final class MethodWriter extends GeneratorAdapter {
 
+    private final Deque<List<org.objectweb.asm.Type>> stringConcatArgs = (INDY_STRING_CONCAT_BOOTSTRAP_HANDLE == null) ? null : new ArrayDeque<>();
+
     MethodWriter(int access, Method method, org.objectweb.asm.Type[] exceptions, ClassVisitor cv) {
         super(Opcodes.ASM5, cv.visitMethod(access, method.getName(), method.getDescriptor(), null, getInternalNames(exceptions)),
                 access, method.getName(), method.getDescriptor());
@@ -171,30 +181,58 @@ public final class MethodWriter extends GeneratorAdapter {
             visitJumpInsn(Opcodes.IFEQ, fals);
         }
     }
-
+    
     public void writeNewStrings() {
-        newInstance(STRINGBUILDER_TYPE);
-        dup();
-        invokeConstructor(STRINGBUILDER_TYPE, STRINGBUILDER_CONSTRUCTOR);
+        if (INDY_STRING_CONCAT_BOOTSTRAP_HANDLE != null) {
+            // Java 9+: we just push our argument collector onto deque
+            stringConcatArgs.push(new ArrayList<>());
+        } else {
+            // Java 8: create a StringBuilder in bytecode
+            newInstance(STRINGBUILDER_TYPE);
+            dup();
+            invokeConstructor(STRINGBUILDER_TYPE, STRINGBUILDER_CONSTRUCTOR);
+        }
     }
 
-    public void writeAppendStrings(final Sort sort) {
-        switch (sort) {
-            case BOOL:   invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_BOOLEAN); break;
-            case CHAR:   invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_CHAR);    break;
-            case BYTE:
-            case SHORT:
-            case INT:    invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_INT);     break;
-            case LONG:   invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_LONG);    break;
-            case FLOAT:  invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_FLOAT);   break;
-            case DOUBLE: invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_DOUBLE);  break;
-            case STRING: invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_STRING);  break;
-            default:     invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_OBJECT);
+    public void writeAppendStrings(final Type type) {
+        if (INDY_STRING_CONCAT_BOOTSTRAP_HANDLE != null) {
+            // Java 9+: record type information
+            stringConcatArgs.peek().add(type.type);
+            // prevent too many concat args.
+            // If there are too many, do the actual concat:
+            if (stringConcatArgs.peek().size() >= MAX_INDY_STRING_CONCAT_ARGS) {
+                writeToStrings();
+                writeNewStrings();
+                // add the return value type as new first param for next concat:
+                stringConcatArgs.peek().add(STRING_TYPE);
+            }
+        } else {
+            // Java 8: push a StringBuilder append
+            switch (type.sort) {
+                case BOOL:   invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_BOOLEAN); break;
+                case CHAR:   invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_CHAR);    break;
+                case BYTE:
+                case SHORT:
+                case INT:    invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_INT);     break;
+                case LONG:   invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_LONG);    break;
+                case FLOAT:  invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_FLOAT);   break;
+                case DOUBLE: invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_DOUBLE);  break;
+                case STRING: invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_STRING);  break;
+                default:     invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_APPEND_OBJECT);
+            }
         }
     }
 
     public void writeToStrings() {
-        invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_TOSTRING);
+        if (INDY_STRING_CONCAT_BOOTSTRAP_HANDLE != null) {
+            // Java 9+: use type information and push invokeDynamic
+            final String desc = org.objectweb.asm.Type.getMethodDescriptor(STRING_TYPE,
+                    stringConcatArgs.pop().stream().toArray(org.objectweb.asm.Type[]::new));
+            invokeDynamic("concat", desc, INDY_STRING_CONCAT_BOOTSTRAP_HANDLE);
+        } else {
+            // Java 8: call toString() on StringBuilder
+            invokeVirtual(STRINGBUILDER_TYPE, STRINGBUILDER_TOSTRING);
+        }
     }
 
     public void writeBinaryInstruction(final CompilerSettings settings, final Definition definition,
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index 86111d8f1a5..6bdb9856114 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -27,6 +27,7 @@ import org.objectweb.asm.Type;
 import org.objectweb.asm.commons.Method;
 
 import java.lang.invoke.CallSite;
+import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodType;
 import java.util.Map;
@@ -81,6 +82,27 @@ public final class WriterConstants {
     public final static Method DEF_GT_CALL  = getAsmMethod(boolean.class, "gt" , Object.class, Object.class);
     public final static Method DEF_GTE_CALL = getAsmMethod(boolean.class, "gte", Object.class, Object.class);
 
+    /** dynamic invokedynamic bootstrap for indy string concats (Java 9+) */
+    public final static Handle INDY_STRING_CONCAT_BOOTSTRAP_HANDLE;
+    static {
+        Handle bs;
+        try {
+            final Class<?> factory = Class.forName("java.lang.invoke.StringConcatFactory");
+            final String methodName = "makeConcat";
+            final MethodType type = MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class);
+            // ensure it is there:
+            MethodHandles.publicLookup().findStatic(factory, methodName, type);
+            bs = new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(factory), methodName, type.toMethodDescriptorString());
+        } catch (ReflectiveOperationException e) {
+            // not Java 9 - we set it null, so MethodWriter uses StringBuilder:
+            bs = null;
+        }
+        INDY_STRING_CONCAT_BOOTSTRAP_HANDLE = bs;
+    }
+    
+    public final static int MAX_INDY_STRING_CONCAT_ARGS = 200;
+    
+    public final static Type STRING_TYPE = Type.getType(String.class);
     public final static Type STRINGBUILDER_TYPE = Type.getType(StringBuilder.class);
 
     public final static Method STRINGBUILDER_CONSTRUCTOR    = getAsmMethod(void.class, "<init>");
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 67645211b8f..07fe1ff3447 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -503,13 +503,13 @@ public final class EBinary extends AExpression {
             left.write(settings, definition, adapter);
 
             if (!(left instanceof EBinary) || ((EBinary)left).operation != Operation.ADD || left.actual.sort != Sort.STRING) {
-                adapter.writeAppendStrings(left.actual.sort);
+                adapter.writeAppendStrings(left.actual);
             }
 
             right.write(settings, definition, adapter);
 
             if (!(right instanceof EBinary) || ((EBinary)right).operation != Operation.ADD || right.actual.sort != Sort.STRING) {
-                adapter.writeAppendStrings(right.actual.sort);
+                adapter.writeAppendStrings(right.actual);
             }
 
             if (!cat) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index b98962a8f22..39afcd935ad 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -266,13 +266,13 @@ public final class EChain extends AExpression {
                 if (cat) {
                     adapter.writeDup(link.size, 1);
                     link.load(settings, definition, adapter);
-                    adapter.writeAppendStrings(link.after.sort);
+                    adapter.writeAppendStrings(link.after);
 
                     expression.write(settings, definition, adapter);
 
                     if (!(expression instanceof EBinary) ||
                         ((EBinary)expression).operation != Operation.ADD || expression.actual.sort != Sort.STRING) {
-                        adapter.writeAppendStrings(expression.actual.sort);
+                        adapter.writeAppendStrings(expression.actual);
                     }
 
                     adapter.writeToStrings();
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
index 0c2b2f4386c..63fdcf315b2 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.painless;
 
+import java.util.Locale;
+
 public class StringTests extends ScriptTestCase {
 
     public void testAppend() {
@@ -63,6 +65,22 @@ public class StringTests extends ScriptTestCase {
         assertEquals("cat" + "cat", exec("String s = 'cat'; return s + s;"));
     }
 
+    public void testAppendMultiple() {
+        assertEquals("cat" + true + "abc" + null, exec("String s = \"cat\"; return s + true + 'abc' + null;"));
+    }
+    
+    public void testAppendMany() {
+        StringBuilder script = new StringBuilder("String s = \"cat\"; return s");
+        StringBuilder result = new StringBuilder("cat");
+        for (int i = 0; i < WriterConstants.MAX_INDY_STRING_CONCAT_ARGS + 10; i++) {
+            final String s = String.format(Locale.ROOT,  "%03d", i);
+            script.append(" + '").append(s).append("'.toString()");
+            result.append(s);
+        }
+        //System.out.println(Debugger.toString(script.toString()));
+        assertEquals(result.toString(), exec(script.toString()));
+    }
+    
     public void testStringAPI() {
         assertEquals("", exec("return new String();"));
         assertEquals('x', exec("String s = \"x\"; return s.charAt(0);"));

From 7c46b57ff29e85e2e222020d3ec1aae6d9689816 Mon Sep 17 00:00:00 2001
From: Zachary Tong <polyfractal@elastic.co>
Date: Tue, 17 May 2016 12:06:48 -0400
Subject: [PATCH 0294/1311] Add a Sort ingest processor

Sorts an array of values in ascending or descending order. If all elements are numerics, they will be sorted numerically. If values are strings, or mixtures of strings/numbers, the elements will be sorted lexicographically.
---
 .../ingest/processor/SortProcessor.java       | 137 +++++++++
 .../org/elasticsearch/node/NodeModule.java    |   2 +
 .../ingest/processor/SortProcessorTests.java  | 281 ++++++++++++++++++
 docs/reference/ingest/ingest-node.asciidoc    |  25 ++
 .../rest-api-spec/test/ingest/90_sort.yaml    |  35 +++
 5 files changed, 480 insertions(+)
 create mode 100644 core/src/main/java/org/elasticsearch/ingest/processor/SortProcessor.java
 create mode 100644 core/src/test/java/org/elasticsearch/ingest/processor/SortProcessorTests.java
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/ingest/90_sort.yaml

diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/SortProcessor.java b/core/src/main/java/org/elasticsearch/ingest/processor/SortProcessor.java
new file mode 100644
index 00000000000..313a8bed7fc
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/ingest/processor/SortProcessor.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.processor;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.ingest.core.AbstractProcessor;
+import org.elasticsearch.ingest.core.AbstractProcessorFactory;
+import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.core.ConfigurationUtils;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Processor that sorts an array of items.
+ * Throws exception is the specified field is not an array.
+ */
+public final class SortProcessor extends AbstractProcessor {
+
+    public static final String TYPE = "sort";
+    public static final String FIELD = "field";
+    public static final String ORDER = "order";
+    public static final String DEFAULT_ORDER = "asc";
+
+    public enum SortOrder {
+        ASCENDING("asc"), DESCENDING("desc");
+
+        private final String direction;
+
+        SortOrder(String direction) {
+            this.direction = direction;
+        }
+
+        public String toString() {
+            return this.direction;
+        }
+
+        public static SortOrder fromString(String value) {
+            if (value == null) {
+                throw new IllegalArgumentException("Sort direction cannot be null");
+            }
+
+            if (value.equals(ASCENDING.toString())) {
+                return ASCENDING;
+            } else if (value.equals(DESCENDING.toString())) {
+                return DESCENDING;
+            }
+            throw new IllegalArgumentException("Sort direction [" + value + "] not recognized."
+                    + " Valid values are: [asc, desc]");
+        }
+    }
+
+    private final String field;
+    private final SortOrder order;
+
+    SortProcessor(String tag, String field, SortOrder order) {
+        super(tag);
+        this.field = field;
+        this.order = order;
+    }
+
+    String getField() {
+        return field;
+    }
+
+    SortOrder getOrder() {
+        return order;
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public void execute(IngestDocument document) {
+        List<? extends Comparable> list = document.getFieldValue(field, List.class);
+
+        if (list == null) {
+            throw new IllegalArgumentException("field [" + field + "] is null, cannot sort.");
+        }
+
+        if (list.size() <= 1) {
+            return;
+        }
+
+        if (order.equals(SortOrder.ASCENDING)) {
+            Collections.sort(list);
+        } else {
+            Collections.sort(list, Collections.reverseOrder());
+        }
+
+        document.setFieldValue(field, list);
+    }
+
+    @Override
+    public String getType() {
+        return TYPE;
+    }
+
+    public final static class Factory extends AbstractProcessorFactory<SortProcessor> {
+
+        @Override
+        public SortProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+            String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, FIELD);
+            try {
+                SortOrder direction = SortOrder.fromString(
+                    ConfigurationUtils.readStringProperty(
+                        TYPE,
+                        processorTag,
+                        config,
+                        ORDER,
+                        DEFAULT_ORDER));
+                return new SortProcessor(processorTag, field, direction);
+            } catch (IllegalArgumentException e) {
+                throw ConfigurationUtils.newConfigurationException(TYPE, processorTag, ORDER, e.getMessage());
+            }
+        }
+    }
+}
+
diff --git a/core/src/main/java/org/elasticsearch/node/NodeModule.java b/core/src/main/java/org/elasticsearch/node/NodeModule.java
index 8565b14be8e..8c9b3e362fc 100644
--- a/core/src/main/java/org/elasticsearch/node/NodeModule.java
+++ b/core/src/main/java/org/elasticsearch/node/NodeModule.java
@@ -37,6 +37,7 @@ import org.elasticsearch.ingest.processor.LowercaseProcessor;
 import org.elasticsearch.ingest.processor.RemoveProcessor;
 import org.elasticsearch.ingest.processor.RenameProcessor;
 import org.elasticsearch.ingest.processor.SetProcessor;
+import org.elasticsearch.ingest.processor.SortProcessor;
 import org.elasticsearch.ingest.processor.SplitProcessor;
 import org.elasticsearch.ingest.processor.TrimProcessor;
 import org.elasticsearch.ingest.processor.UppercaseProcessor;
@@ -78,6 +79,7 @@ public class NodeModule extends AbstractModule {
         registerProcessor(FailProcessor.TYPE, (templateService, registry) -> new FailProcessor.Factory(templateService));
         registerProcessor(ForEachProcessor.TYPE, (templateService, registry) -> new ForEachProcessor.Factory(registry));
         registerProcessor(DateIndexNameProcessor.TYPE, (templateService, registry) -> new DateIndexNameProcessor.Factory());
+        registerProcessor(SortProcessor.TYPE, (templateService, registry) -> new SortProcessor.Factory());
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/SortProcessorTests.java b/core/src/test/java/org/elasticsearch/ingest/processor/SortProcessorTests.java
new file mode 100644
index 00000000000..f67ca3b6bee
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/ingest/processor/SortProcessorTests.java
@@ -0,0 +1,281 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.processor;
+
+import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.RandomDocumentPicks;
+import org.elasticsearch.ingest.core.Processor;
+import org.elasticsearch.ingest.processor.SortProcessor.SortOrder;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+public class SortProcessorTests extends ESTestCase {
+
+    public void testSortStrings() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        int numItems = randomIntBetween(1, 10);
+        List<String> fieldValue = new ArrayList<>(numItems);
+        List<String> expectedResult = new ArrayList<>(numItems);
+        for (int j = 0; j < numItems; j++) {
+            String value = randomAsciiOfLengthBetween(1, 10);
+            fieldValue.add(value);
+            expectedResult.add(value);
+        }
+        Collections.sort(expectedResult);
+
+        SortOrder order = randomBoolean() ? SortOrder.ASCENDING : SortOrder.DESCENDING;
+        if (order.equals(SortOrder.DESCENDING)) {
+            Collections.reverse(expectedResult);
+        }
+
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
+        Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, order);
+        processor.execute(ingestDocument);
+        assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult);
+    }
+
+    public void testSortIntegersNonRandom() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+
+        Integer[] expectedResult = new Integer[]{1,2,3,4,5,10,20,21,22,50,100};
+        List<Integer> fieldValue = new ArrayList<>(expectedResult.length);
+        fieldValue.addAll(Arrays.asList(expectedResult).subList(0, expectedResult.length));
+        Collections.shuffle(fieldValue);
+
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
+        Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, SortOrder.ASCENDING);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue(fieldName, List.class).toArray(), equalTo(expectedResult));
+    }
+
+    public void testSortIntegers() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        int numItems = randomIntBetween(1, 10);
+        List<Integer> fieldValue = new ArrayList<>(numItems);
+        List<Integer> expectedResult = new ArrayList<>(numItems);
+        for (int j = 0; j < numItems; j++) {
+            Integer value = randomIntBetween(1, 100);
+            fieldValue.add(value);
+            expectedResult.add(value);
+        }
+        Collections.sort(expectedResult);
+
+        SortOrder order = randomBoolean() ? SortOrder.ASCENDING : SortOrder.DESCENDING;
+        if (order.equals(SortOrder.DESCENDING)) {
+            Collections.reverse(expectedResult);
+        }
+
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
+        Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, order);
+        processor.execute(ingestDocument);
+        assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult);
+    }
+
+    public void testSortShorts() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        int numItems = randomIntBetween(1, 10);
+        List<Short> fieldValue = new ArrayList<>(numItems);
+        List<Short> expectedResult = new ArrayList<>(numItems);
+        for (int j = 0; j < numItems; j++) {
+            Short value = randomShort();
+            fieldValue.add(value);
+            expectedResult.add(value);
+        }
+        Collections.sort(expectedResult);
+
+        SortOrder order = randomBoolean() ? SortOrder.ASCENDING : SortOrder.DESCENDING;
+        if (order.equals(SortOrder.DESCENDING)) {
+            Collections.reverse(expectedResult);
+        }
+
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
+        Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, order);
+        processor.execute(ingestDocument);
+        assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult);
+    }
+
+    public void testSortDoubles() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        int numItems = randomIntBetween(1, 10);
+        List<Double> fieldValue = new ArrayList<>(numItems);
+        List<Double> expectedResult = new ArrayList<>(numItems);
+        for (int j = 0; j < numItems; j++) {
+            Double value = randomDoubleBetween(0.0, 100.0, true);
+            fieldValue.add(value);
+            expectedResult.add(value);
+        }
+        Collections.sort(expectedResult);
+
+        SortOrder order = randomBoolean() ? SortOrder.ASCENDING : SortOrder.DESCENDING;
+        if (order.equals(SortOrder.DESCENDING)) {
+            Collections.reverse(expectedResult);
+        }
+
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
+        Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, order);
+        processor.execute(ingestDocument);
+        assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult);
+    }
+
+    public void testSortFloats() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        int numItems = randomIntBetween(1, 10);
+        List<Float> fieldValue = new ArrayList<>(numItems);
+        List<Float> expectedResult = new ArrayList<>(numItems);
+        for (int j = 0; j < numItems; j++) {
+            Float value = randomFloat();
+            fieldValue.add(value);
+            expectedResult.add(value);
+        }
+        Collections.sort(expectedResult);
+
+        SortOrder order = randomBoolean() ? SortOrder.ASCENDING : SortOrder.DESCENDING;
+        if (order.equals(SortOrder.DESCENDING)) {
+            Collections.reverse(expectedResult);
+        }
+
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
+        Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, order);
+        processor.execute(ingestDocument);
+        assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult);
+    }
+
+    public void testSortBytes() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        int numItems = randomIntBetween(1, 10);
+        List<Byte> fieldValue = new ArrayList<>(numItems);
+        List<Byte> expectedResult = new ArrayList<>(numItems);
+        for (int j = 0; j < numItems; j++) {
+            Byte value = randomByte();
+            fieldValue.add(value);
+            expectedResult.add(value);
+        }
+        Collections.sort(expectedResult);
+
+        SortOrder order = randomBoolean() ? SortOrder.ASCENDING : SortOrder.DESCENDING;
+        if (order.equals(SortOrder.DESCENDING)) {
+            Collections.reverse(expectedResult);
+        }
+
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
+        Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, order);
+        processor.execute(ingestDocument);
+        assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult);
+    }
+
+    public void testSortBooleans() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        int numItems = randomIntBetween(1, 10);
+        List<Boolean> fieldValue = new ArrayList<>(numItems);
+        List<Boolean> expectedResult = new ArrayList<>(numItems);
+        for (int j = 0; j < numItems; j++) {
+            Boolean value = randomBoolean();
+            fieldValue.add(value);
+            expectedResult.add(value);
+        }
+        Collections.sort(expectedResult);
+
+        SortOrder order = randomBoolean() ? SortOrder.ASCENDING : SortOrder.DESCENDING;
+        if (order.equals(SortOrder.DESCENDING)) {
+            Collections.reverse(expectedResult);
+        }
+
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
+        Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, order);
+        processor.execute(ingestDocument);
+        assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult);
+    }
+
+    public void testSortMixedStrings() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        int numItems = randomIntBetween(1, 10);
+        List<String> fieldValue = new ArrayList<>(numItems);
+        List<String> expectedResult = new ArrayList<>(numItems);
+        String value;
+        for (int j = 0; j < numItems; j++) {
+            if (randomBoolean()) {
+                value = String.valueOf(randomIntBetween(0, 100));
+            } else {
+                value = randomAsciiOfLengthBetween(1, 10);
+            }
+            fieldValue.add(value);
+            expectedResult.add(value);
+        }
+        Collections.sort(expectedResult);
+
+        SortOrder order = randomBoolean() ? SortOrder.ASCENDING : SortOrder.DESCENDING;
+        if (order.equals(SortOrder.DESCENDING)) {
+            Collections.reverse(expectedResult);
+        }
+
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
+        Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, order);
+        processor.execute(ingestDocument);
+        assertEquals(ingestDocument.getFieldValue(fieldName, List.class), expectedResult);
+    }
+
+    public void testSortNonListField() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        ingestDocument.setFieldValue(fieldName, randomAsciiOfLengthBetween(1, 10));
+        SortOrder order = randomBoolean() ? SortOrder.ASCENDING : SortOrder.DESCENDING;
+        Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, order);
+        try {
+            processor.execute(ingestDocument);
+        } catch(IllegalArgumentException e) {
+            assertThat(e.getMessage(), equalTo("field [" + fieldName + "] of type [java.lang.String] cannot be cast to [java.util.List]"));
+        }
+    }
+
+    public void testSortNonExistingField() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        SortOrder order = randomBoolean() ? SortOrder.ASCENDING : SortOrder.DESCENDING;
+        Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, order);
+        try {
+            processor.execute(ingestDocument);
+        } catch(IllegalArgumentException e) {
+            assertThat(e.getMessage(), containsString("not present as part of path [" + fieldName + "]"));
+        }
+    }
+
+    public void testSortNullValue() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("field", null));
+        SortOrder order = randomBoolean() ? SortOrder.ASCENDING : SortOrder.DESCENDING;
+        Processor processor = new SortProcessor(randomAsciiOfLength(10), "field", order);
+        try {
+            processor.execute(ingestDocument);
+        } catch(IllegalArgumentException e) {
+            assertThat(e.getMessage(), equalTo("field [field] is null, cannot sort."));
+        }
+    }
+
+
+
+}
diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc
index ec6253f2d93..3dcf719f382 100644
--- a/docs/reference/ingest/ingest-node.asciidoc
+++ b/docs/reference/ingest/ingest-node.asciidoc
@@ -1282,6 +1282,31 @@ Splits a field into an array using a separator character. Only works on string f
 --------------------------------------------------
 <1> Treat all consecutive whitespace characters as a single separator
 
+[[sort-processor]]
+=== Sort Processor
+Sorts the elements of an array ascending or descending.  Homogeneous arrays of numbers will be sorted
+numerically, while arrays of strings or heterogeneous arrays of strings + numbers will be sorted lexicographically.
+Throws an error when the field is not an array.
+
+[[sort-options]]
+.Sort Options
+[options="header"]
+|======
+| Name    | Required  | Default  | Description
+| `field` | yes       | -        | The field to be sorted
+| `order` | no        | `"asc"`  | The sort order to use. Accepts `"asc"` or `"desc"`.
+|======
+
+[source,js]
+--------------------------------------------------
+{
+  "sort": {
+    "field": "field_to_sort",
+    "order": "desc"
+  }
+}
+--------------------------------------------------
+
 [[trim-processor]]
 === Trim Processor
 Trims whitespace from field.
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/90_sort.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/90_sort.yaml
new file mode 100644
index 00000000000..096e57fa4e1
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/90_sort.yaml
@@ -0,0 +1,35 @@
+---
+"Test sort Processor":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "sort" : {
+                  "field" : "values"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: >
+          {
+            "values": ["foo", "bar", "baz"]
+          }
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.values: ["bar", "baz", "foo"] }

From c755a77022ce44b1ba8f8bfad19864a0b95b301d Mon Sep 17 00:00:00 2001
From: polyfractal <zacharyjtong@gmail.com>
Date: Tue, 17 May 2016 12:55:07 -0400
Subject: [PATCH 0295/1311] [TEST] Use a reproducible source of randomness in
 shuffle

---
 .../org/elasticsearch/ingest/processor/SortProcessorTests.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/SortProcessorTests.java b/core/src/test/java/org/elasticsearch/ingest/processor/SortProcessorTests.java
index f67ca3b6bee..18e35d00a79 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/SortProcessorTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/processor/SortProcessorTests.java
@@ -30,6 +30,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Random;
 import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.containsString;
@@ -66,7 +67,7 @@ public class SortProcessorTests extends ESTestCase {
         Integer[] expectedResult = new Integer[]{1,2,3,4,5,10,20,21,22,50,100};
         List<Integer> fieldValue = new ArrayList<>(expectedResult.length);
         fieldValue.addAll(Arrays.asList(expectedResult).subList(0, expectedResult.length));
-        Collections.shuffle(fieldValue);
+        Collections.shuffle(fieldValue, random());
 
         String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, fieldValue);
         Processor processor = new SortProcessor(randomAsciiOfLength(10), fieldName, SortOrder.ASCENDING);

From 9ba554dfd2d3842ede04bfa251c00858f75d2f46 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Tue, 17 May 2016 19:02:28 +0200
Subject: [PATCH 0296/1311] Expose previous cluster state only in
 RoutingAllocation (#18390)

Instead of re-exposing index metadata and blocks in RoutingNodes (which is part of the cluster state before rerouting), expose it as part of the RoutingAllocation which is known to be only temporarily used during reroute.
---
 ...ansportClusterAllocationExplainAction.java |  2 +-
 .../cluster/routing/RoutingNodes.java         | 31 -------------------
 .../routing/allocation/AllocationService.java |  8 ++---
 .../allocation/FailedRerouteAllocation.java   |  6 ++--
 .../routing/allocation/RoutingAllocation.java | 20 +++++++++---
 .../allocation/StartedRerouteAllocation.java  |  8 ++---
 .../allocator/BalancedShardsAllocator.java    |  2 +-
 .../decider/EnableAllocationDecider.java      |  2 +-
 .../decider/FilterAllocationDecider.java      |  2 +-
 .../decider/ShardsLimitAllocationDecider.java |  4 +--
 .../SnapshotInProgressAllocationDecider.java  |  2 +-
 .../gateway/PrimaryShardAllocator.java        |  2 +-
 .../ack/AckClusterUpdateSettingsIT.java       |  2 +-
 .../decider/DiskThresholdDeciderTests.java    |  8 ++---
 .../DiskThresholdDeciderUnitTests.java        |  4 +--
 .../gateway/PrimaryShardAllocatorTests.java   | 18 +++++------
 .../gateway/ReplicaShardAllocatorTests.java   |  4 +--
 .../indices/state/RareClusterStateIT.java     |  2 +-
 18 files changed, 54 insertions(+), 73 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
index da97f492f7a..28b62083d42 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
@@ -249,7 +249,7 @@ public class TransportClusterAllocationExplainAction
     protected void masterOperation(final ClusterAllocationExplainRequest request, final ClusterState state,
                                    final ActionListener<ClusterAllocationExplainResponse> listener) {
         final RoutingNodes routingNodes = state.getRoutingNodes();
-        final RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, state.nodes(),
+        final RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, state,
                 clusterInfoService.getClusterInfo(), System.nanoTime());
 
         ShardRouting foundShard = null;
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
index 6004d2af94a..cb86c898487 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
@@ -54,10 +54,6 @@ import java.util.function.Predicate;
  */
 public class RoutingNodes implements Iterable<RoutingNode> {
 
-    private final MetaData metaData;
-
-    private final ClusterBlocks blocks;
-
     private final RoutingTable routingTable;
 
     private final Map<String, RoutingNode> nodesToShards = new HashMap<>();
@@ -66,8 +62,6 @@ public class RoutingNodes implements Iterable<RoutingNode> {
 
     private final Map<ShardId, List<ShardRouting>> assignedShards = new HashMap<>();
 
-    private final ImmutableOpenMap<String, ClusterState.Custom> customs;
-
     private final boolean readOnly;
 
     private int inactivePrimaryCount = 0;
@@ -85,10 +79,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
 
     public RoutingNodes(ClusterState clusterState, boolean readOnly) {
         this.readOnly = readOnly;
-        this.metaData = clusterState.metaData();
-        this.blocks = clusterState.blocks();
         this.routingTable = clusterState.routingTable();
-        this.customs = clusterState.customs();
 
         Map<String, LinkedHashMap<ShardId, ShardRouting>> nodesToShards = new HashMap<>();
         // fill in the nodeToShards with the "live" nodes
@@ -232,28 +223,6 @@ public class RoutingNodes implements Iterable<RoutingNode> {
         return routingTable();
     }
 
-    public MetaData metaData() {
-        return this.metaData;
-    }
-
-    public MetaData getMetaData() {
-        return metaData();
-    }
-
-    public ClusterBlocks blocks() {
-        return this.blocks;
-    }
-
-    public ClusterBlocks getBlocks() {
-        return this.blocks;
-    }
-
-    public ImmutableOpenMap<String, ClusterState.Custom> customs() {
-        return this.customs;
-    }
-
-    public <T extends ClusterState.Custom> T custom(String type) { return (T) customs.get(type); }
-
     public UnassignedShards unassigned() {
         return this.unassignedShards;
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
index 608477e55fe..e1bbbb7f4ab 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
@@ -90,7 +90,7 @@ public class AllocationService extends AbstractComponent {
         RoutingNodes routingNodes = getMutableRoutingNodes(clusterState);
         // shuffle the unassigned nodes, just so we won't have things like poison failed shards
         routingNodes.unassigned().shuffle();
-        StartedRerouteAllocation allocation = new StartedRerouteAllocation(allocationDeciders, routingNodes, clusterState.nodes(), startedShards, clusterInfoService.getClusterInfo());
+        StartedRerouteAllocation allocation = new StartedRerouteAllocation(allocationDeciders, routingNodes, clusterState, startedShards, clusterInfoService.getClusterInfo());
         boolean changed = applyStartedShards(routingNodes, startedShards);
         if (!changed) {
             return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData());
@@ -216,7 +216,7 @@ public class AllocationService extends AbstractComponent {
         RoutingNodes routingNodes = getMutableRoutingNodes(clusterState);
         // shuffle the unassigned nodes, just so we won't have things like poison failed shards
         routingNodes.unassigned().shuffle();
-        FailedRerouteAllocation allocation = new FailedRerouteAllocation(allocationDeciders, routingNodes, clusterState.nodes(), failedShards, clusterInfoService.getClusterInfo());
+        FailedRerouteAllocation allocation = new FailedRerouteAllocation(allocationDeciders, routingNodes, clusterState, failedShards, clusterInfoService.getClusterInfo());
         boolean changed = false;
         // as failing primaries also fail associated replicas, we fail replicas first here so that their nodes are added to ignore list
         List<FailedRerouteAllocation.FailedShard> orderedFailedShards = new ArrayList<>(failedShards);
@@ -266,7 +266,7 @@ public class AllocationService extends AbstractComponent {
         // we don't shuffle the unassigned shards here, to try and get as close as possible to
         // a consistent result of the effect the commands have on the routing
         // this allows systems to dry run the commands, see the resulting cluster state, and act on it
-        RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, clusterState.nodes(), clusterInfoService.getClusterInfo(), currentNanoTime());
+        RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, clusterState, clusterInfoService.getClusterInfo(), currentNanoTime());
         // don't short circuit deciders, we want a full explanation
         allocation.debugDecision(true);
         // we ignore disable allocation, because commands are explicit
@@ -305,7 +305,7 @@ public class AllocationService extends AbstractComponent {
         RoutingNodes routingNodes = getMutableRoutingNodes(clusterState);
         // shuffle the unassigned nodes, just so we won't have things like poison failed shards
         routingNodes.unassigned().shuffle();
-        RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, clusterState.nodes(), clusterInfoService.getClusterInfo(), currentNanoTime());
+        RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, clusterState, clusterInfoService.getClusterInfo(), currentNanoTime());
         allocation.debugDecision(debug);
         if (!reroute(allocation)) {
             return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData());
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java
index 7232e15f033..a13862fed26 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java
@@ -21,7 +21,7 @@ package org.elasticsearch.cluster.routing.allocation;
 
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.cluster.ClusterInfo;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
@@ -57,8 +57,8 @@ public class FailedRerouteAllocation extends RoutingAllocation {
 
     private final List<FailedShard> failedShards;
 
-    public FailedRerouteAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, DiscoveryNodes nodes, List<FailedShard> failedShards, ClusterInfo clusterInfo) {
-        super(deciders, routingNodes, nodes, clusterInfo, System.nanoTime());
+    public FailedRerouteAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, ClusterState clusterState, List<FailedShard> failedShards, ClusterInfo clusterInfo) {
+        super(deciders, routingNodes, clusterState, clusterInfo, System.nanoTime());
         this.failedShards = failedShards;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
index 536806c0830..60ca3a8d5fd 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
@@ -20,12 +20,14 @@
 package org.elasticsearch.cluster.routing.allocation;
 
 import org.elasticsearch.cluster.ClusterInfo;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.index.shard.ShardId;
 
 import java.util.HashMap;
@@ -118,8 +120,12 @@ public class RoutingAllocation {
 
     private final RoutingNodes routingNodes;
 
+    private final MetaData metaData;
+
     private final DiscoveryNodes nodes;
 
+    private final ImmutableOpenMap<String, ClusterState.Custom> customs;
+
     private final AllocationExplanation explanation = new AllocationExplanation();
 
     private final ClusterInfo clusterInfo;
@@ -139,13 +145,15 @@ public class RoutingAllocation {
      * Creates a new {@link RoutingAllocation}
      *  @param deciders {@link AllocationDeciders} to used to make decisions for routing allocations
      * @param routingNodes Routing nodes in the current cluster
-     * @param nodes TODO: Documentation
+     * @param clusterState cluster state before rerouting
      * @param currentNanoTime the nano time to use for all delay allocation calculation (typically {@link System#nanoTime()})
      */
-    public RoutingAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, DiscoveryNodes nodes, ClusterInfo clusterInfo, long currentNanoTime) {
+    public RoutingAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, ClusterState clusterState, ClusterInfo clusterInfo, long currentNanoTime) {
         this.deciders = deciders;
         this.routingNodes = routingNodes;
-        this.nodes = nodes;
+        this.metaData = clusterState.metaData();
+        this.nodes = clusterState.nodes();
+        this.customs = clusterState.customs();
         this.clusterInfo = clusterInfo;
         this.currentNanoTime = currentNanoTime;
     }
@@ -184,7 +192,7 @@ public class RoutingAllocation {
      * @return Metadata of routing nodes
      */
     public MetaData metaData() {
-        return routingNodes.metaData();
+        return metaData;
     }
 
     /**
@@ -199,6 +207,10 @@ public class RoutingAllocation {
         return clusterInfo;
     }
 
+    public <T extends ClusterState.Custom> T custom(String key) {
+        return (T)customs.get(key);
+    }
+
     /**
      * Get explanations of current routing
      * @return explanation of routing
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StartedRerouteAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StartedRerouteAllocation.java
index 00f3944ae03..e9570edd9c3 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StartedRerouteAllocation.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StartedRerouteAllocation.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.cluster.routing.allocation;
 
 import org.elasticsearch.cluster.ClusterInfo;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
@@ -35,8 +35,8 @@ public class StartedRerouteAllocation extends RoutingAllocation {
 
     private final List<? extends ShardRouting> startedShards;
 
-    public StartedRerouteAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, DiscoveryNodes nodes, List<? extends ShardRouting> startedShards, ClusterInfo clusterInfo) {
-        super(deciders, routingNodes, nodes, clusterInfo, System.nanoTime());
+    public StartedRerouteAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, ClusterState clusterState, List<? extends ShardRouting> startedShards, ClusterInfo clusterInfo) {
+        super(deciders, routingNodes, clusterState, clusterInfo, System.nanoTime());
         this.startedShards = startedShards;
     }
 
@@ -47,4 +47,4 @@ public class StartedRerouteAllocation extends RoutingAllocation {
     public List<? extends ShardRouting> startedShards() {
         return startedShards;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
index 7fd3a03a23e..b1d78fa44d0 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
@@ -225,7 +225,7 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
             this.weight = weight;
             this.threshold = threshold;
             this.routingNodes = allocation.routingNodes();
-            metaData = routingNodes.metaData();
+            this.metaData = allocation.metaData();
             avgShardsPerNode = ((float) metaData.getTotalNumberOfShards()) / routingNodes.size();
             buildModelFromAssigned();
         }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java
index 38a2a39fc7c..7dc8eff3ebd 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java
@@ -137,7 +137,7 @@ public class EnableAllocationDecider extends AllocationDecider {
             return allocation.decision(Decision.YES, NAME, "allocation is explicitly ignoring any disabling of relocation");
         }
 
-        Settings indexSettings = allocation.routingNodes().metaData().getIndexSafe(shardRouting.index()).getSettings();
+        Settings indexSettings = allocation.metaData().getIndexSafe(shardRouting.index()).getSettings();
         final Rebalance enable;
         if (INDEX_ROUTING_REBALANCE_ENABLE_SETTING.exists(indexSettings)) {
             enable = INDEX_ROUTING_REBALANCE_ENABLE_SETTING.get(indexSettings);
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java
index eb59c261214..c862865cb16 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java
@@ -105,7 +105,7 @@ public class FilterAllocationDecider extends AllocationDecider {
         Decision decision = shouldClusterFilter(node, allocation);
         if (decision != null) return decision;
 
-        decision = shouldIndexFilter(allocation.routingNodes().metaData().getIndexSafe(shardRouting.index()), node, allocation);
+        decision = shouldIndexFilter(allocation.metaData().getIndexSafe(shardRouting.index()), node, allocation);
         if (decision != null) return decision;
 
         return allocation.decision(Decision.YES, NAME, "node passes include/exclude/require filters");
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java
index eb256516353..e1741c1af7f 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ShardsLimitAllocationDecider.java
@@ -86,7 +86,7 @@ public class ShardsLimitAllocationDecider extends AllocationDecider {
 
     @Override
     public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
-        IndexMetaData indexMd = allocation.routingNodes().metaData().getIndexSafe(shardRouting.index());
+        IndexMetaData indexMd = allocation.metaData().getIndexSafe(shardRouting.index());
         final int indexShardLimit = INDEX_TOTAL_SHARDS_PER_NODE_SETTING.get(indexMd.getSettings(), settings);
         // Capture the limit here in case it changes during this method's
         // execution
@@ -125,7 +125,7 @@ public class ShardsLimitAllocationDecider extends AllocationDecider {
 
     @Override
     public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
-        IndexMetaData indexMd = allocation.routingNodes().metaData().getIndexSafe(shardRouting.index());
+        IndexMetaData indexMd = allocation.metaData().getIndexSafe(shardRouting.index());
         final int indexShardLimit = INDEX_TOTAL_SHARDS_PER_NODE_SETTING.get(indexMd.getSettings(), settings);
         // Capture the limit here in case it changes during this method's
         // execution
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java
index 54cfb6407da..e25a4e690da 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDecider.java
@@ -98,7 +98,7 @@ public class SnapshotInProgressAllocationDecider extends AllocationDecider {
         if (!enableRelocation && shardRouting.primary()) {
             // Only primary shards are snapshotted
 
-            SnapshotsInProgress snapshotsInProgress = allocation.routingNodes().custom(SnapshotsInProgress.TYPE);
+            SnapshotsInProgress snapshotsInProgress = allocation.custom(SnapshotsInProgress.TYPE);
             if (snapshotsInProgress == null) {
                 // Snapshots are not running
                 return allocation.decision(Decision.YES, NAME, "no snapshots are currently running");
diff --git a/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java b/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java
index 8cfe446a29b..73f4b9b2ef0 100644
--- a/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java
+++ b/core/src/main/java/org/elasticsearch/gateway/PrimaryShardAllocator.java
@@ -84,7 +84,7 @@ public abstract class PrimaryShardAllocator extends AbstractComponent {
     public boolean allocateUnassigned(RoutingAllocation allocation) {
         boolean changed = false;
         final RoutingNodes routingNodes = allocation.routingNodes();
-        final MetaData metaData = routingNodes.metaData();
+        final MetaData metaData = allocation.metaData();
 
         final RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator = routingNodes.unassigned().iterator();
         while (unassignedIterator.hasNext()) {
diff --git a/core/src/test/java/org/elasticsearch/cluster/ack/AckClusterUpdateSettingsIT.java b/core/src/test/java/org/elasticsearch/cluster/ack/AckClusterUpdateSettingsIT.java
index 8d13f04240e..f47134a45b8 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ack/AckClusterUpdateSettingsIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ack/AckClusterUpdateSettingsIT.java
@@ -96,7 +96,7 @@ public class AckClusterUpdateSettingsIT extends ESIntegTestCase {
 
         for (Client client : clients()) {
             ClusterState clusterState = getLocalClusterState(client);
-            assertThat(clusterState.getRoutingNodes().metaData().transientSettings().get("cluster.routing.allocation.exclude._id"), equalTo(excludedNodeId));
+            assertThat(clusterState.metaData().transientSettings().get("cluster.routing.allocation.exclude._id"), equalTo(excludedNodeId));
             for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) {
                 for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
                     for (ShardRouting shardRouting : indexShardRoutingTable) {
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
index a58c9c3f58b..2c4e86ad4b1 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
@@ -875,7 +875,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                         )
         );
         ClusterState clusterState = ClusterState.builder(baseClusterState).routingTable(builder.build()).build();
-        RoutingAllocation routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), discoveryNodes, clusterInfo,
+        RoutingAllocation routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), clusterState, clusterInfo,
                 System.nanoTime());
         Decision decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation);
         assertThat(decision.type(), equalTo(Decision.Type.NO));
@@ -896,7 +896,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                         )
         );
         clusterState = ClusterState.builder(baseClusterState).routingTable(builder.build()).build();
-        routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), discoveryNodes, clusterInfo, System.nanoTime());
+        routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), clusterState, clusterInfo, System.nanoTime());
         decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation);
         assertThat(decision.type(), equalTo(Decision.Type.YES));
 
@@ -991,7 +991,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                         )
         );
         ClusterState clusterState = ClusterState.builder(baseClusterState).routingTable(builder.build()).build();
-        RoutingAllocation routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), discoveryNodes, clusterInfo,
+        RoutingAllocation routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), clusterState, clusterInfo,
                 System.nanoTime());
         Decision decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation);
 
@@ -1051,7 +1051,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
         );
 
         clusterState = ClusterState.builder(updateClusterState).routingTable(builder.build()).build();
-        routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), discoveryNodes, clusterInfo, System.nanoTime());
+        routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), clusterState, clusterInfo, System.nanoTime());
         decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation);
         assertThat(decision.type(), equalTo(Decision.Type.YES));
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
index 3b1799e4b06..d9e9fb95445 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
@@ -136,7 +136,7 @@ public class DiskThresholdDeciderUnitTests extends ESTestCase {
         ImmutableOpenMap.Builder<String, Long> shardSizes = ImmutableOpenMap.builder();
         shardSizes.put("[test][0][p]", 10L); // 10 bytes
         final ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages.build(), mostAvailableUsage.build(), shardSizes.build(), ImmutableOpenMap.of());
-        RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, new AllocationDecider[]{decider}), clusterState.getRoutingNodes(), clusterState.nodes(), clusterInfo, System.nanoTime());
+        RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, new AllocationDecider[]{decider}), clusterState.getRoutingNodes(), clusterState, clusterInfo, System.nanoTime());
         assertEquals(mostAvailableUsage.toString(), Decision.YES, decider.canAllocate(test_0, new RoutingNode("node_0", node_0), allocation));
         assertEquals(mostAvailableUsage.toString(), Decision.NO, decider.canAllocate(test_0, new RoutingNode("node_1", node_1), allocation));
     }
@@ -204,7 +204,7 @@ public class DiskThresholdDeciderUnitTests extends ESTestCase {
         shardSizes.put("[test][2][p]", 10L);
 
         final ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages.build(), mostAvailableUsage.build(), shardSizes.build(), shardRoutingMap.build());
-        RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, new AllocationDecider[]{decider}), clusterState.getRoutingNodes(), clusterState.nodes(), clusterInfo, System.nanoTime());
+        RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, new AllocationDecider[]{decider}), clusterState.getRoutingNodes(), clusterState, clusterInfo, System.nanoTime());
         assertEquals(Decision.YES, decider.canRemain(test_0, new RoutingNode("node_0", node_0), allocation));
         assertEquals(Decision.NO, decider.canRemain(test_1, new RoutingNode("node_1", node_1), allocation));
         try {
diff --git a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
index b62d2fe3342..01e717e6fa9 100644
--- a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
@@ -346,7 +346,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
             .metaData(metaData)
             .routingTable(routingTable)
             .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build();
-        return new RoutingAllocation(allocationDeciders, new RoutingNodes(state, false), state.nodes(), null, System.nanoTime());
+        return new RoutingAllocation(allocationDeciders, new RoutingNodes(state, false), state, null, System.nanoTime());
     }
 
     /**
@@ -425,7 +425,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
             .metaData(metaData)
             .routingTable(routingTable)
             .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build();
-        return new RoutingAllocation(allocationDeciders, new RoutingNodes(state, false), state.nodes(), null, System.nanoTime());
+        return new RoutingAllocation(allocationDeciders, new RoutingNodes(state, false), state, null, System.nanoTime());
     }
 
     /**
@@ -444,7 +444,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
                 .routingTable(routingTable)
                 .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build();
 
-        RoutingAllocation allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null, System.nanoTime());
+        RoutingAllocation allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state, null, System.nanoTime());
         boolean changed = testAllocator.allocateUnassigned(allocation);
         assertThat(changed, equalTo(false));
         assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1));
@@ -452,7 +452,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(2)); // replicas
 
         testAllocator.addData(node1, 1, null, randomBoolean());
-        allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null, System.nanoTime());
+        allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state, null, System.nanoTime());
         changed = testAllocator.allocateUnassigned(allocation);
         assertThat(changed, equalTo(false));
         assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1));
@@ -460,7 +460,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(2)); // replicas
 
         testAllocator.addData(node2, 1, null, randomBoolean());
-        allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null, System.nanoTime());
+        allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state, null, System.nanoTime());
         changed = testAllocator.allocateUnassigned(allocation);
         assertThat(changed, equalTo(true));
         assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(0));
@@ -485,7 +485,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
                 .routingTable(routingTable)
                 .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build();
 
-        RoutingAllocation allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null, System.nanoTime());
+        RoutingAllocation allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state, null, System.nanoTime());
         boolean changed = testAllocator.allocateUnassigned(allocation);
         assertThat(changed, equalTo(false));
         assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1));
@@ -493,7 +493,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(2)); // replicas
 
         testAllocator.addData(node1, 1, null, randomBoolean());
-        allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null, System.nanoTime());
+        allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state, null, System.nanoTime());
         changed = testAllocator.allocateUnassigned(allocation);
         assertThat(changed, equalTo(false));
         assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1));
@@ -501,7 +501,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(2)); // replicas
 
         testAllocator.addData(node2, 2, null, randomBoolean());
-        allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null, System.nanoTime());
+        allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state, null, System.nanoTime());
         changed = testAllocator.allocateUnassigned(allocation);
         assertThat(changed, equalTo(true));
         assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(0));
@@ -525,7 +525,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
                 .metaData(metaData)
                 .routingTable(routingTableBuilder.build())
                 .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build();
-        return new RoutingAllocation(deciders, new RoutingNodes(state, false), state.nodes(), null, System.nanoTime());
+        return new RoutingAllocation(deciders, new RoutingNodes(state, false), state, null, System.nanoTime());
     }
 
     class TestAllocator extends PrimaryShardAllocator {
diff --git a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
index 96f81e2ab73..672c9de3d3e 100644
--- a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
@@ -302,7 +302,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
                 .metaData(metaData)
                 .routingTable(routingTable)
                 .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build();
-        return new RoutingAllocation(deciders, new RoutingNodes(state, false), state.nodes(), ClusterInfo.EMPTY, System.nanoTime());
+        return new RoutingAllocation(deciders, new RoutingNodes(state, false), state, ClusterInfo.EMPTY, System.nanoTime());
     }
 
     private RoutingAllocation onePrimaryOnNode1And1ReplicaRecovering(AllocationDeciders deciders) {
@@ -324,7 +324,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
                 .metaData(metaData)
                 .routingTable(routingTable)
                 .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build();
-        return new RoutingAllocation(deciders, new RoutingNodes(state, false), state.nodes(), ClusterInfo.EMPTY, System.nanoTime());
+        return new RoutingAllocation(deciders, new RoutingNodes(state, false), state, ClusterInfo.EMPTY, System.nanoTime());
     }
 
     class TestAllocator extends ReplicaShardAllocator {
diff --git a/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java b/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java
index 0b1723bd5aa..dc533737886 100644
--- a/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java
@@ -103,7 +103,7 @@ public class RareClusterStateIT extends ESIntegTestCase {
                         .nodes(DiscoveryNodes.EMPTY_NODES)
                         .build(), false
         );
-        RoutingAllocation routingAllocation = new RoutingAllocation(allocationDeciders, routingNodes, current.nodes(), ClusterInfo.EMPTY, System.nanoTime());
+        RoutingAllocation routingAllocation = new RoutingAllocation(allocationDeciders, routingNodes, current, ClusterInfo.EMPTY, System.nanoTime());
         allocator.allocateUnassigned(routingAllocation);
     }
 

From 584be0b3f82fcd498465091166f4c5dbe248e7db Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 17 May 2016 13:05:36 -0400
Subject: [PATCH 0297/1311] Refactor JvmGcMonitorService for testing

This commit refactors the JvmGcMonitorService so that it can be
tested. In particular, hooks are added to verify that the
JvmMonitorService correctly observes slow GC events, and that the
JvmGcMonitorService logs the correct messages.

Relates #18378
---
 .../resources/checkstyle_suppressions.xml     |   1 -
 .../monitor/jvm/JvmGcMonitorService.java      | 221 +++++++++++++---
 .../monitor/jvm/JvmGcMonitorServiceTests.java | 136 ++++++++++
 .../monitor/jvm/JvmMonitorTests.java          | 248 ++++++++++++++++++
 4 files changed, 572 insertions(+), 34 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceTests.java
 create mode 100644 core/src/test/java/org/elasticsearch/monitor/jvm/JvmMonitorTests.java

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 1c052597e55..48f07b1a2d5 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -517,7 +517,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]ConvertProcessor.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]GcNames.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]HotThreads.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]JvmGcMonitorService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]JvmStats.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]node[/\\]Node.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]node[/\\]internal[/\\]InternalSettingsPreparer.java" checks="LineLength" />
diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
index 5a2d591c7dc..01b4c68537e 100644
--- a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
@@ -20,9 +20,11 @@
 package org.elasticsearch.monitor.jvm;
 
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.monitor.jvm.JvmStats.GarbageCollector;
@@ -30,14 +32,13 @@ import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Objects;
 import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiFunction;
 
 import static java.util.Collections.unmodifiableMap;
-import static org.elasticsearch.monitor.jvm.JvmStats.jvmStats;
 
-/**
- *
- */
 public class JvmGcMonitorService extends AbstractLifecycleComponent<JvmGcMonitorService> {
 
     private final ThreadPool threadPool;
@@ -119,12 +120,123 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent<JvmGcMonitor
         return GC_COLLECTOR_PREFIX + key + "." + level;
     }
 
+    private static final String LOG_MESSAGE =
+        "[gc][{}][{}][{}] duration [{}], collections [{}]/[{}], total [{}]/[{}], memory [{}]->[{}]/[{}], all_pools {}";
+
     @Override
     protected void doStart() {
         if (!enabled) {
             return;
         }
-        scheduledFuture = threadPool.scheduleWithFixedDelay(new JvmMonitor(), interval);
+        scheduledFuture = threadPool.scheduleWithFixedDelay(new JvmMonitor(gcThresholds) {
+            @Override
+            void onMonitorFailure(Throwable t) {
+                logger.debug("failed to monitor", t);
+            }
+
+            @Override
+            void onSlowGc(final Threshold threshold, final long seq, final SlowGcEvent slowGcEvent) {
+                logSlowGc(logger, threshold, seq, slowGcEvent, JvmGcMonitorService::buildPools);
+            }
+        }, interval);
+    }
+
+    static void logSlowGc(
+        final ESLogger logger,
+        final JvmMonitor.Threshold threshold,
+        final long seq,
+        final JvmMonitor.SlowGcEvent slowGcEvent,
+        BiFunction<JvmStats, JvmStats, String> pools) {
+
+        final String name = slowGcEvent.currentGc.getName();
+        final long elapsed = slowGcEvent.elapsed;
+        final long totalGcCollectionCount = slowGcEvent.currentGc.getCollectionCount();
+        final long currentGcCollectionCount = slowGcEvent.collectionCount;
+        final TimeValue totalGcCollectionTime = slowGcEvent.currentGc.getCollectionTime();
+        final TimeValue currentGcCollectionTime = slowGcEvent.collectionTime;
+        final JvmStats lastJvmStats = slowGcEvent.lastJvmStats;
+        final JvmStats currentJvmStats = slowGcEvent.currentJvmStats;
+        final ByteSizeValue maxHeapUsed = slowGcEvent.maxHeapUsed;
+
+        switch (threshold) {
+            case WARN:
+                if (logger.isWarnEnabled()) {
+                    logger.warn(
+                        LOG_MESSAGE,
+                        name,
+                        seq,
+                        totalGcCollectionCount,
+                        currentGcCollectionTime,
+                        currentGcCollectionCount,
+                        TimeValue.timeValueMillis(elapsed),
+                        currentGcCollectionTime,
+                        totalGcCollectionTime,
+                        lastJvmStats.getMem().getHeapUsed(),
+                        currentJvmStats.getMem().getHeapUsed(),
+                        maxHeapUsed,
+                        pools.apply(lastJvmStats, currentJvmStats));
+                }
+                break;
+            case INFO:
+                if (logger.isInfoEnabled()) {
+                    logger.info(
+                        LOG_MESSAGE,
+                        name,
+                        seq,
+                        totalGcCollectionCount,
+                        currentGcCollectionTime,
+                        currentGcCollectionCount,
+                        TimeValue.timeValueMillis(elapsed),
+                        currentGcCollectionTime,
+                        totalGcCollectionTime,
+                        lastJvmStats.getMem().getHeapUsed(),
+                        currentJvmStats.getMem().getHeapUsed(),
+                        maxHeapUsed,
+                        pools.apply(lastJvmStats, currentJvmStats));
+                }
+                break;
+            case DEBUG:
+                if (logger.isDebugEnabled()) {
+                    logger.debug(
+                        LOG_MESSAGE,
+                        name,
+                        seq,
+                        totalGcCollectionCount,
+                        currentGcCollectionTime,
+                        currentGcCollectionCount,
+                        TimeValue.timeValueMillis(elapsed),
+                        currentGcCollectionTime,
+                        totalGcCollectionTime,
+                        lastJvmStats.getMem().getHeapUsed(),
+                        currentJvmStats.getMem().getHeapUsed(),
+                        maxHeapUsed,
+                        pools.apply(lastJvmStats, currentJvmStats));
+                }
+                break;
+        }
+    }
+
+    static String buildPools(JvmStats last, JvmStats current) {
+        StringBuilder sb = new StringBuilder();
+        for (JvmStats.MemoryPool currentPool : current.getMem()) {
+            JvmStats.MemoryPool prevPool = null;
+            for (JvmStats.MemoryPool pool : last.getMem()) {
+                if (pool.getName().equals(currentPool.getName())) {
+                    prevPool = pool;
+                    break;
+                }
+            }
+            sb.append("{[")
+                .append(currentPool.getName())
+                .append("] [")
+                .append(prevPool == null ? "?" : prevPool.getUsed())
+                .append("]->[")
+                .append(currentPool.getUsed())
+                .append("]/[")
+                .append(currentPool.getMax())
+                .append("]}");
+        }
+        return sb.toString();
     }
 
     @Override
@@ -139,12 +251,46 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent<JvmGcMonitor
     protected void doClose() {
     }
 
-    private class JvmMonitor implements Runnable {
+    static abstract class JvmMonitor implements Runnable {
 
+        enum Threshold { DEBUG, INFO, WARN }
+
+        static class SlowGcEvent {
+
+            final GarbageCollector currentGc;
+            final long collectionCount;
+            final TimeValue collectionTime;
+            final long elapsed;
+            final JvmStats lastJvmStats;
+            final JvmStats currentJvmStats;
+            final ByteSizeValue maxHeapUsed;
+
+            public SlowGcEvent(
+                final GarbageCollector currentGc,
+                final long collectionCount,
+                final TimeValue collectionTime,
+                final long elapsed,
+                final JvmStats lastJvmStats,
+                final JvmStats currentJvmStats,
+                final ByteSizeValue maxHeapUsed) {
+                this.currentGc = currentGc;
+                this.collectionCount = collectionCount;
+                this.collectionTime = collectionTime;
+                this.elapsed = elapsed;
+                this.lastJvmStats = lastJvmStats;
+                this.currentJvmStats = currentJvmStats;
+                this.maxHeapUsed = maxHeapUsed;
+            }
+
+        }
+
+        private long lastTime = now();
         private JvmStats lastJvmStats = jvmStats();
         private long seq = 0;
+        private final Map<String, GcThreshold> gcThresholds;
 
-        public JvmMonitor() {
+        public JvmMonitor(Map<String, GcThreshold> gcThresholds) {
+            this.gcThresholds = Objects.requireNonNull(gcThresholds);
         }
 
         @Override
@@ -152,24 +298,28 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent<JvmGcMonitor
             try {
                 monitorLongGc();
             } catch (Throwable t) {
-                logger.debug("failed to monitor", t);
+                onMonitorFailure(t);
             }
         }
 
-        private synchronized void monitorLongGc() {
+        abstract void onMonitorFailure(Throwable t);
+
+        synchronized void monitorLongGc() {
             seq++;
+            final long currentTime = now();
             JvmStats currentJvmStats = jvmStats();
 
+            final long elapsed = TimeUnit.NANOSECONDS.toMillis(currentTime - lastTime);
             for (int i = 0; i < currentJvmStats.getGc().getCollectors().length; i++) {
                 GarbageCollector gc = currentJvmStats.getGc().getCollectors()[i];
-                GarbageCollector prevGc = lastJvmStats.gc.collectors[i];
+                GarbageCollector prevGc = lastJvmStats.getGc().getCollectors()[i];
 
                 // no collection has happened
-                long collections = gc.collectionCount - prevGc.collectionCount;
+                long collections = gc.getCollectionCount() - prevGc.getCollectionCount();
                 if (collections == 0) {
                     continue;
                 }
-                long collectionTime = gc.collectionTime - prevGc.collectionTime;
+                long collectionTime = gc.getCollectionTime().millis() - prevGc.getCollectionTime().millis();
                 if (collectionTime == 0) {
                     continue;
                 }
@@ -181,34 +331,39 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent<JvmGcMonitor
 
                 long avgCollectionTime = collectionTime / collections;
 
+                Threshold threshold = null;
                 if (avgCollectionTime > gcThreshold.warnThreshold) {
-                    logger.warn("[gc][{}][{}][{}] duration [{}], collections [{}]/[{}], total [{}]/[{}], memory [{}]->[{}]/[{}], all_pools {}",
-                            gc.getName(), seq, gc.getCollectionCount(), TimeValue.timeValueMillis(collectionTime), collections, TimeValue.timeValueMillis(currentJvmStats.getTimestamp() - lastJvmStats.getTimestamp()), TimeValue.timeValueMillis(collectionTime), gc.getCollectionTime(), lastJvmStats.getMem().getHeapUsed(), currentJvmStats.getMem().getHeapUsed(), JvmInfo.jvmInfo().getMem().getHeapMax(), buildPools(lastJvmStats, currentJvmStats));
+                    threshold = Threshold.WARN;
                 } else if (avgCollectionTime > gcThreshold.infoThreshold) {
-                    logger.info("[gc][{}][{}][{}] duration [{}], collections [{}]/[{}], total [{}]/[{}], memory [{}]->[{}]/[{}], all_pools {}",
-                            gc.getName(), seq, gc.getCollectionCount(), TimeValue.timeValueMillis(collectionTime), collections, TimeValue.timeValueMillis(currentJvmStats.getTimestamp() - lastJvmStats.getTimestamp()), TimeValue.timeValueMillis(collectionTime), gc.getCollectionTime(), lastJvmStats.getMem().getHeapUsed(), currentJvmStats.getMem().getHeapUsed(), JvmInfo.jvmInfo().getMem().getHeapMax(), buildPools(lastJvmStats, currentJvmStats));
-                } else if (avgCollectionTime > gcThreshold.debugThreshold && logger.isDebugEnabled()) {
-                    logger.debug("[gc][{}][{}][{}] duration [{}], collections [{}]/[{}], total [{}]/[{}], memory [{}]->[{}]/[{}], all_pools {}",
-                            gc.getName(), seq, gc.getCollectionCount(), TimeValue.timeValueMillis(collectionTime), collections, TimeValue.timeValueMillis(currentJvmStats.getTimestamp() - lastJvmStats.getTimestamp()), TimeValue.timeValueMillis(collectionTime), gc.getCollectionTime(), lastJvmStats.getMem().getHeapUsed(), currentJvmStats.getMem().getHeapUsed(), JvmInfo.jvmInfo().getMem().getHeapMax(), buildPools(lastJvmStats, currentJvmStats));
+                    threshold = Threshold.INFO;
+                } else if (avgCollectionTime > gcThreshold.debugThreshold) {
+                    threshold = Threshold.DEBUG;
+                }
+                if (threshold != null) {
+                    onSlowGc(threshold, seq, new SlowGcEvent(
+                        gc,
+                        collections,
+                        TimeValue.timeValueMillis(collectionTime),
+                        elapsed,
+                        lastJvmStats,
+                        currentJvmStats,
+                        JvmInfo.jvmInfo().getMem().getHeapMax()));
                 }
             }
+            lastTime = currentTime;
             lastJvmStats = currentJvmStats;
         }
 
-        private String buildPools(JvmStats prev, JvmStats current) {
-            StringBuilder sb = new StringBuilder();
-            for (JvmStats.MemoryPool currentPool : current.getMem()) {
-                JvmStats.MemoryPool prevPool = null;
-                for (JvmStats.MemoryPool pool : prev.getMem()) {
-                    if (pool.getName().equals(currentPool.getName())) {
-                        prevPool = pool;
-                        break;
-                    }
-                }
-                sb.append("{[").append(currentPool.getName())
-                        .append("] [").append(prevPool == null ? "?" : prevPool.getUsed()).append("]->[").append(currentPool.getUsed()).append("]/[").append(currentPool.getMax()).append("]}");
-            }
-            return sb.toString();
+        JvmStats jvmStats() {
+            return JvmStats.jvmStats();
         }
+
+        long now() {
+            return System.nanoTime();
+        }
+
+        abstract void onSlowGc(final Threshold threshold, final long seq, final SlowGcEvent slowGcEvent);
+
     }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceTests.java b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceTests.java
new file mode 100644
index 00000000000..2c17fca7c8c
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceTests.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.monitor.jvm;
+
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+public class JvmGcMonitorServiceTests extends ESTestCase {
+
+    public void testSlowGcLogging() {
+        final ESLogger logger = mock(ESLogger.class);
+        when(logger.isWarnEnabled()).thenReturn(true);
+        when(logger.isInfoEnabled()).thenReturn(true);
+        when(logger.isDebugEnabled()).thenReturn(true);
+        final JvmGcMonitorService.JvmMonitor.Threshold threshold = randomFrom(JvmGcMonitorService.JvmMonitor.Threshold.values());
+        final String name = randomAsciiOfLength(16);
+        final long seq = randomIntBetween(1, 1 << 30);
+        final int elapsedValue = randomIntBetween(1, 1 << 10);
+        final long totalCollectionCount = randomIntBetween(1, 16);
+        final long currentCollectionCount = randomIntBetween(1, 16);
+        final TimeValue totalCollectionTime = TimeValue.timeValueMillis(randomIntBetween(1, elapsedValue));
+        final TimeValue currentCollectionTime = TimeValue.timeValueMillis(randomIntBetween(1, elapsedValue));
+
+        final ByteSizeValue lastHeapUsed = new ByteSizeValue(randomIntBetween(1, 1 << 10));
+        JvmStats lastJvmStats = mock(JvmStats.class);
+        JvmStats.Mem lastMem = mock(JvmStats.Mem.class);
+        when(lastMem.getHeapUsed()).thenReturn(lastHeapUsed);
+        when(lastJvmStats.getMem()).thenReturn(lastMem);
+        when(lastJvmStats.toString()).thenReturn("last");
+
+        final ByteSizeValue currentHeapUsed = new ByteSizeValue(randomIntBetween(1, 1 << 10));
+        JvmStats currentJvmStats = mock(JvmStats.class);
+        JvmStats.Mem currentMem = mock(JvmStats.Mem.class);
+        when(currentMem.getHeapUsed()).thenReturn(currentHeapUsed);
+        when(currentJvmStats.getMem()).thenReturn(currentMem);
+        when(currentJvmStats.toString()).thenReturn("current");
+
+        JvmStats.GarbageCollector gc = mock(JvmStats.GarbageCollector.class);
+        when(gc.getName()).thenReturn(name);
+        when(gc.getCollectionCount()).thenReturn(totalCollectionCount);
+        when(gc.getCollectionTime()).thenReturn(totalCollectionTime);
+
+        final ByteSizeValue maxHeapUsed = new ByteSizeValue(Math.max(lastHeapUsed.bytes(), currentHeapUsed.bytes()) + 1 << 10);
+
+        JvmGcMonitorService.JvmMonitor.SlowGcEvent slowGcEvent = new JvmGcMonitorService.JvmMonitor.SlowGcEvent(
+            gc,
+            currentCollectionCount,
+            currentCollectionTime,
+            elapsedValue,
+            lastJvmStats,
+            currentJvmStats,
+            maxHeapUsed);
+
+        JvmGcMonitorService.logSlowGc(logger, threshold, seq, slowGcEvent, (l, c) -> l.toString() + ", " + c.toString());
+
+        switch (threshold) {
+            case WARN:
+                verify(logger).isWarnEnabled();
+                verify(logger).warn(
+                    "[gc][{}][{}][{}] duration [{}], collections [{}]/[{}], total [{}]/[{}], memory [{}]->[{}]/[{}], all_pools {}",
+                    name,
+                    seq,
+                    totalCollectionCount,
+                    currentCollectionTime,
+                    currentCollectionCount,
+                    TimeValue.timeValueMillis(elapsedValue),
+                    currentCollectionTime,
+                    totalCollectionTime,
+                    lastHeapUsed,
+                    currentHeapUsed,
+                    maxHeapUsed,
+                    "last, current");
+                break;
+            case INFO:
+                verify(logger).isInfoEnabled();
+                verify(logger).info(
+                    "[gc][{}][{}][{}] duration [{}], collections [{}]/[{}], total [{}]/[{}], memory [{}]->[{}]/[{}], all_pools {}",
+                    name,
+                    seq,
+                    totalCollectionCount,
+                    currentCollectionTime,
+                    currentCollectionCount,
+                    TimeValue.timeValueMillis(elapsedValue),
+                    currentCollectionTime,
+                    totalCollectionTime,
+                    lastHeapUsed,
+                    currentHeapUsed,
+                    maxHeapUsed,
+                    "last, current");
+                break;
+            case DEBUG:
+                verify(logger).isDebugEnabled();
+                verify(logger).debug(
+                    "[gc][{}][{}][{}] duration [{}], collections [{}]/[{}], total [{}]/[{}], memory [{}]->[{}]/[{}], all_pools {}",
+                    name,
+                    seq,
+                    totalCollectionCount,
+                    currentCollectionTime,
+                    currentCollectionCount,
+                    TimeValue.timeValueMillis(elapsedValue),
+                    currentCollectionTime,
+                    totalCollectionTime,
+                    lastHeapUsed,
+                    currentHeapUsed,
+                    maxHeapUsed,
+                    "last, current");
+                break;
+        }
+        verifyNoMoreInteractions(logger);
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmMonitorTests.java b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmMonitorTests.java
new file mode 100644
index 00000000000..8d3ddeec84e
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmMonitorTests.java
@@ -0,0 +1,248 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.monitor.jvm;
+
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.hamcrest.CoreMatchers.anyOf;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.Matchers.hasToString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class JvmMonitorTests extends ESTestCase {
+
+    public void testMonitorFailure() {
+        AtomicBoolean shouldFail = new AtomicBoolean();
+        AtomicBoolean invoked = new AtomicBoolean();
+        JvmGcMonitorService.JvmMonitor monitor = new JvmGcMonitorService.JvmMonitor(Collections.emptyMap()) {
+            @Override
+            void onMonitorFailure(Throwable t) {
+                invoked.set(true);
+                assertThat(t, instanceOf(RuntimeException.class));
+                assertThat(t, hasToString(containsString("simulated")));
+            }
+
+            @Override
+            synchronized void monitorLongGc() {
+                if (shouldFail.get()) {
+                    throw new RuntimeException("simulated");
+                }
+            }
+
+            @Override
+            void onSlowGc(final Threshold threshold, final long seq, final SlowGcEvent slowGcEvent) {
+            }
+        };
+
+        monitor.run();
+        assertFalse(invoked.get());
+
+        shouldFail.set(true);
+        monitor.run();
+        assertTrue(invoked.get());
+    }
+
+    public void testSlowGc() {
+
+        final int initialYoungCollectionCount = randomIntBetween(1, 4);
+        final int initialYoungCollectionTime = randomIntBetween(initialYoungCollectionCount * 100, initialYoungCollectionCount * 200);
+
+        final int initialOldCollectionCount = randomIntBetween(1, 4);
+        final int initialOldCollectionTime = randomIntBetween(initialYoungCollectionCount * 1000, initialYoungCollectionCount * 2000);
+
+        final JvmStats.GarbageCollector initialYoungCollector = mock(JvmStats.GarbageCollector.class);
+        when(initialYoungCollector.getName()).thenReturn("young");
+        when(initialYoungCollector.getCollectionCount()).thenReturn((long) initialYoungCollectionCount);
+        when(initialYoungCollector.getCollectionTime()).thenReturn(TimeValue.timeValueMillis(initialYoungCollectionTime));
+
+        final JvmStats.GarbageCollector initialOldCollector = mock(JvmStats.GarbageCollector.class);
+        when(initialOldCollector.getName()).thenReturn("old");
+        when(initialOldCollector.getCollectionCount()).thenReturn((long) initialOldCollectionCount);
+        when(initialOldCollector.getCollectionTime()).thenReturn(TimeValue.timeValueMillis(initialOldCollectionTime));
+        JvmStats initialJvmStats = jvmStats(initialYoungCollector, initialOldCollector);
+
+        final Map<String, JvmGcMonitorService.GcThreshold> gcThresholds = new HashMap<>();
+
+        // fake debug threshold, info will be double this and warn will
+        // be triple
+        final int youngDebugThreshold = randomIntBetween(1, 10) * 100;
+        gcThresholds.put(
+            "young",
+            new JvmGcMonitorService.GcThreshold("young", youngDebugThreshold * 3, youngDebugThreshold * 2, youngDebugThreshold));
+
+        final boolean youngGcThreshold = randomBoolean();
+        final JvmGcMonitorService.JvmMonitor.Threshold youngThresholdLevel = randomFrom(JvmGcMonitorService.JvmMonitor.Threshold.values());
+        final int youngMultiplier = 1 + youngThresholdLevel.ordinal();
+        final int youngCollections = randomIntBetween(1, 4);
+
+        final JvmStats.GarbageCollector youngCollector;
+        youngCollector = mock(JvmStats.GarbageCollector.class);
+        when(youngCollector.getName()).thenReturn("young");
+        when(youngCollector.getCollectionCount()).thenReturn((long) (initialYoungCollectionCount + youngCollections));
+
+        final int youngIncrement;
+        if (youngGcThreshold) {
+            // we are faking that youngCollections collections occurred
+            // this number is chosen so that we squeak over the
+            // random threshold when computing the average collection
+            // time: note that average collection time will just be
+            // youngMultiplier * youngDebugThreshold + 1 which ensures
+            // that we are over the right threshold but below the next
+            // threshold
+            youngIncrement = youngCollections * youngMultiplier * youngDebugThreshold + youngCollections;
+        } else {
+            // fake that we did not exceed the threshold
+            youngIncrement = randomIntBetween(1, youngDebugThreshold);
+        }
+        when(youngCollector.getCollectionTime()).thenReturn(TimeValue.timeValueMillis(initialYoungCollectionTime + youngIncrement));
+
+        // fake debug threshold, info will be double this and warn will
+        // be triple
+        final int oldDebugThreshold = randomIntBetween(1, 10) * 100;
+        gcThresholds.put(
+            "old",
+            new JvmGcMonitorService.GcThreshold("old", oldDebugThreshold * 3, oldDebugThreshold * 2, oldDebugThreshold));
+
+        final boolean oldGcThreshold = randomBoolean();
+        final JvmGcMonitorService.JvmMonitor.Threshold oldThresholdLevel = randomFrom(JvmGcMonitorService.JvmMonitor.Threshold.values());
+        final int oldMultiplier = 1 + oldThresholdLevel.ordinal();
+        final int oldCollections = randomIntBetween(1, 4);
+
+        final JvmStats.GarbageCollector oldCollector = mock(JvmStats.GarbageCollector.class);
+        when(oldCollector.getName()).thenReturn("old");
+        when(oldCollector.getCollectionCount()).thenReturn((long) (initialOldCollectionCount + oldCollections));
+        final int oldIncrement;
+        if (oldGcThreshold) {
+            // we are faking that oldCollections collections occurred
+            // this number is chosen so that we squeak over the
+            // random threshold when computing the average collection
+            // time: note that average collection time will just be
+            // oldMultiplier * oldDebugThreshold + 1 which ensures
+            // that we are over the right threshold but below the next
+            // threshold
+            oldIncrement = oldCollections * oldMultiplier * oldDebugThreshold + oldCollections;
+        } else {
+            // fake that we did not exceed the threshold
+            oldIncrement = randomIntBetween(1, oldDebugThreshold);
+        }
+        when(oldCollector.getCollectionTime()).thenReturn(TimeValue.timeValueMillis(initialOldCollectionTime + oldIncrement));
+
+        final long start = randomIntBetween(1, 1 << 30);
+        final long expectedElapsed = randomIntBetween(1, 1000);
+        final AtomicLong now = new AtomicLong(start);
+
+        final AtomicReference<JvmStats> jvmStats = new AtomicReference<>();
+        jvmStats.set(initialJvmStats);
+
+        final AtomicInteger count = new AtomicInteger();
+
+        JvmGcMonitorService.JvmMonitor monitor = new JvmGcMonitorService.JvmMonitor(gcThresholds) {
+            @Override
+            void onMonitorFailure(Throwable t) {
+            }
+
+            @Override
+            void onSlowGc(final Threshold threshold, final long seq, final SlowGcEvent slowGcEvent) {
+                count.incrementAndGet();
+                assertThat(seq, equalTo(1L));
+                assertThat(slowGcEvent.elapsed, equalTo(expectedElapsed));
+                assertThat(slowGcEvent.currentGc.getName(), anyOf(equalTo("young"), equalTo("old")));
+                if ("young".equals(slowGcEvent.currentGc.getName())) {
+                    assertCollection(
+                        threshold,
+                        youngThresholdLevel,
+                        slowGcEvent,
+                        initialYoungCollectionCount,
+                        youngCollections,
+                        initialYoungCollectionTime,
+                        youngIncrement);
+                } else if ("old".equals(slowGcEvent.currentGc.getName())) {
+                    assertCollection(
+                        threshold,
+                        oldThresholdLevel,
+                        slowGcEvent,
+                        initialOldCollectionCount,
+                        oldCollections,
+                        initialOldCollectionTime,
+                        oldIncrement);
+                }
+            }
+
+            @Override
+            long now() {
+                return now.get();
+            }
+
+            @Override
+            JvmStats jvmStats() {
+                return jvmStats.get();
+            }
+        };
+
+        final JvmStats monitorJvmStats = jvmStats(youngCollector, oldCollector);
+
+        now.set(start + TimeUnit.NANOSECONDS.convert(expectedElapsed, TimeUnit.MILLISECONDS));
+        jvmStats.set(monitorJvmStats);
+        monitor.monitorLongGc();
+
+        assertThat(count.get(), equalTo((youngGcThreshold ? 1 : 0) + (oldGcThreshold ? 1 : 0)));
+    }
+
+    private void assertCollection(
+        final JvmGcMonitorService.JvmMonitor.Threshold actualThreshold,
+        final JvmGcMonitorService.JvmMonitor.Threshold expectedThreshold,
+        final JvmGcMonitorService.JvmMonitor.SlowGcEvent slowGcEvent,
+        final int initialCollectionCount,
+        final int collections,
+        final int initialCollectionTime,
+        final int increment) {
+        assertThat(actualThreshold, equalTo(expectedThreshold));
+        assertThat(slowGcEvent.currentGc.getCollectionCount(), equalTo((long) (initialCollectionCount + collections)));
+        assertThat(slowGcEvent.collectionCount, equalTo((long) collections));
+        assertThat(slowGcEvent.collectionTime, equalTo(TimeValue.timeValueMillis(increment)));
+        assertThat(slowGcEvent.currentGc.getCollectionTime(), equalTo(TimeValue.timeValueMillis(initialCollectionTime + increment)));
+    }
+
+    private JvmStats jvmStats(JvmStats.GarbageCollector youngCollector, JvmStats.GarbageCollector oldCollector) {
+        final JvmStats jvmStats = mock(JvmStats.class);
+        final JvmStats.GarbageCollectors initialGcs = mock(JvmStats.GarbageCollectors.class);
+        final JvmStats.GarbageCollector[] initialCollectors = new JvmStats.GarbageCollector[2];
+        initialCollectors[0] = youngCollector;
+        initialCollectors[1] = oldCollector;
+        when(initialGcs.getCollectors()).thenReturn(initialCollectors);
+        when(jvmStats.getGc()).thenReturn(initialGcs);
+        when(jvmStats.getMem()).thenReturn(JvmStats.jvmStats().getMem());
+        return jvmStats;
+    }
+
+}

From fe4823eae0ee6360425d696dc54bde68184875ff Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 12 May 2016 16:07:46 -0400
Subject: [PATCH 0298/1311] Reindex should retry on search failures

This uses the same backoff policy we use for bulk and just retries until
the request isn't rejected.

Instead of `{"retries": 12}` in the response to count retries this now
looks like `{"retries": {"bulk": 12", "search": 1}`.

Closes #18059
---
 docs/reference/docs/reindex.asciidoc          |  18 +-
 docs/reference/docs/update-by-query.asciidoc  |  18 +-
 docs/reference/migration/migrate_5_0.asciidoc |   1 +
 .../migration/migrate_5_0/docs.asciidoc       |  33 ++++
 .../AbstractAsyncBulkByScrollAction.java      |  88 ++++++----
 .../reindex/AbstractBulkByScrollRequest.java  |   6 +-
 .../AbstractBulkByScrollRequestBuilder.java   |  17 ++
 ...stractBulkIndexByScrollRequestBuilder.java |   7 +-
 .../index/reindex/BulkByScrollTask.java       |  52 ++++--
 .../reindex/BulkIndexByScrollResponse.java    |  14 ++
 .../index/reindex/ReindexRequest.java         |   9 +-
 .../index/reindex/ReindexRequestBuilder.java  |   2 +-
 .../index/reindex/UpdateByQueryRequest.java   |   6 +-
 .../reindex/UpdateByQueryRequestBuilder.java  |   2 +-
 .../reindex/AsyncBulkByScrollActionTests.java | 154 ++++++++++++++---
 .../index/reindex/BulkByScrollTaskTests.java  |  25 +--
 .../BulkIndexByScrollResponseMatcher.java     |   2 +-
 .../index/reindex/CancelTestUtils.java        |   7 +-
 .../index/reindex/RethrottleTests.java        |   2 +-
 .../index/reindex/RetryTests.java             | 155 ++++++++++++++++++
 .../index/reindex/RoundTripTests.java         |   5 +-
 .../reindex/UpdateByQueryRequestTests.java    |   4 +-
 22 files changed, 518 insertions(+), 109 deletions(-)
 create mode 100644 docs/reference/migration/migrate_5_0/docs.asciidoc
 create mode 100644 modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java

diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index 790476990d9..bde6ad8b611 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -33,7 +33,10 @@ That will return something like this:
   "batches": 1,
   "version_conflicts": 0,
   "noops": 0,
-  "retries": 0,
+  "retries": {
+    "bulk": 0,
+    "search": 0
+  },
   "throttled_millis": 0,
   "requests_per_second": "unlimited",
   "throttled_until_millis": 0,
@@ -386,7 +389,10 @@ The JSON response looks like this:
   "created": 123,
   "batches": 1,
   "version_conflicts": 2,
-  "retries": 0,
+  "retries": {
+    "bulk": 0,
+    "search": 0
+  }
   "throttled_millis": 0,
   "failures" : [ ]
 }
@@ -414,7 +420,8 @@ The number of version conflicts that reindex hit.
 
 `retries`::
 
-The number of retries that the reindex did in response to a full queue.
+The number of retries attempted by reindex. `bulk` is the number of bulk
+actions retried and `search` is the number of search actions retried.
 
 `throttled_millis`::
 
@@ -468,7 +475,10 @@ The responses looks like:
             "batches" : 4,
             "version_conflicts" : 0,
             "noops" : 0,
-            "retries": 0,
+            "retries": {
+              "bulk": 0,
+              "search": 0
+            },
             "throttled_millis": 0
           },
           "description" : ""
diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc
index c3eb0908398..e5a62ddf33a 100644
--- a/docs/reference/docs/update-by-query.asciidoc
+++ b/docs/reference/docs/update-by-query.asciidoc
@@ -26,7 +26,10 @@ That will return something like this:
   "batches": 1,
   "version_conflicts": 0,
   "noops": 0,
-  "retries": 0,
+  "retries": {
+    "bulk": 0,
+    "search": 0
+  },
   "throttled_millis": 0,
   "requests_per_second": "unlimited",
   "throttled_until_millis": 0,
@@ -220,7 +223,10 @@ The JSON response looks like this:
   "updated": 0,
   "batches": 1,
   "version_conflicts": 2,
-  "retries": 0,
+  "retries": {
+    "bulk": 0,
+    "search": 0
+  }
   "throttled_millis": 0,
   "failures" : [ ]
 }
@@ -244,7 +250,8 @@ The number of version conflicts that the update by query hit.
 
 `retries`::
 
-The number of retries that the update by query did in response to a full queue.
+The number of retries attempted by update-by-query. `bulk` is the number of bulk
+actions retried and `search` is the number of search actions retried.
 
 `throttled_millis`::
 
@@ -299,7 +306,10 @@ The responses looks like:
             "batches" : 4,
             "version_conflicts" : 0,
             "noops" : 0,
-            "retries": 0,
+            "retries": {
+              "bulk": 0,
+              "search": 0
+            }
             "throttled_millis": 0
           },
           "description" : ""
diff --git a/docs/reference/migration/migrate_5_0.asciidoc b/docs/reference/migration/migrate_5_0.asciidoc
index e2dae648485..71e37e49581 100644
--- a/docs/reference/migration/migrate_5_0.asciidoc
+++ b/docs/reference/migration/migrate_5_0.asciidoc
@@ -34,6 +34,7 @@ way to do this is to upgrade to Elasticsearch 2.3 or later and to use the
 * <<breaking_50_percolator>>
 * <<breaking_50_suggester>>
 * <<breaking_50_index_apis>>
+* <<breaking_50_document_api_changes>>
 * <<breaking_50_settings_changes>>
 * <<breaking_50_allocation>>
 * <<breaking_50_http_changes>>
diff --git a/docs/reference/migration/migrate_5_0/docs.asciidoc b/docs/reference/migration/migrate_5_0/docs.asciidoc
new file mode 100644
index 00000000000..85e4e901e5c
--- /dev/null
+++ b/docs/reference/migration/migrate_5_0/docs.asciidoc
@@ -0,0 +1,33 @@
+[[breaking_50_document_api_changes]]
+=== Document API changes
+
+==== Reindex and Update By Query
+Before 5.0.0 `_reindex` and `_update_by_query` only retried bulk failures so
+they used the following response format:
+
+[source,js]
+----------------------
+{
+   ...
+   "retries": 10
+   ...
+}
+----------------------
+
+Where `retries` counts the number of bulk retries. Now they retry on search
+failures as well and use this response format:
+
+[source,js]
+----------------------
+{
+   ...
+   "retries": {
+     "bulk": 10,
+     "search": 1
+   }
+   ...
+}
+----------------------
+
+Where `bulk` counts the number of bulk retries and `search` counts the number
+of search retries.
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
index efa01d78790..f8561d17acd 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
@@ -57,6 +58,7 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
 
 import static java.lang.Math.max;
 import static java.lang.Math.min;
@@ -91,7 +93,8 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
     private final ThreadPool threadPool;
     private final SearchRequest firstSearchRequest;
     private final ActionListener<Response> listener;
-    private final Retry retry;
+    private final BackoffPolicy backoffPolicy;
+    private final Retry bulkRetry;
 
     public AbstractAsyncBulkByScrollAction(BulkByScrollTask task, ESLogger logger, ParentTaskAssigningClient client,
             ThreadPool threadPool, Request mainRequest, SearchRequest firstSearchRequest, ActionListener<Response> listener) {
@@ -102,7 +105,8 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
         this.mainRequest = mainRequest;
         this.firstSearchRequest = firstSearchRequest;
         this.listener = listener;
-        retry = Retry.on(EsRejectedExecutionException.class).policy(wrapBackoffPolicy(backoffPolicy()));
+        backoffPolicy = buildBackoffPolicy();
+        bulkRetry = Retry.on(EsRejectedExecutionException.class).policy(wrapBackoffPolicy(backoffPolicy));
     }
 
     protected abstract BulkRequest buildBulk(Iterable<SearchHit> docs);
@@ -131,21 +135,14 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
                         firstSearchRequest.types() == null || firstSearchRequest.types().length == 0 ? ""
                                 : firstSearchRequest.types());
             }
-            client.search(firstSearchRequest, new ActionListener<SearchResponse>() {
-                @Override
-                public void onResponse(SearchResponse response) {
-                    logger.debug("[{}] documents match query", response.getHits().getTotalHits());
-                    onScrollResponse(timeValueSeconds(0), response);
-                }
-
-                @Override
-                public void onFailure(Throwable e) {
-                    finishHim(e);
-                }
-            });
         } catch (Throwable t) {
             finishHim(t);
+            return;
         }
+        searchWithRetry(listener -> client.search(firstSearchRequest, listener), (SearchResponse response) -> {
+            logger.debug("[{}] documents match query", response.getHits().getTotalHits());
+            onScrollResponse(timeValueSeconds(0), response);
+        });
     }
 
     /**
@@ -239,7 +236,7 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
             finishHim(null);
             return;
         }
-        retry.withAsyncBackoff(client, request, new ActionListener<BulkResponse>() {
+        bulkRetry.withAsyncBackoff(client, request, new ActionListener<BulkResponse>() {
             @Override
             public void onResponse(BulkResponse response) {
                 onBulkResponse(response);
@@ -322,16 +319,8 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
         SearchScrollRequest request = new SearchScrollRequest();
         // Add the wait time into the scroll timeout so it won't timeout while we wait for throttling
         request.scrollId(scroll.get()).scroll(timeValueNanos(firstSearchRequest.scroll().keepAlive().nanos() + waitTime));
-        client.searchScroll(request, new ActionListener<SearchResponse>() {
-            @Override
-            public void onResponse(SearchResponse response) {
-                onScrollResponse(timeValueNanos(max(0, earliestNextBatchStartTime - System.nanoTime())), response);
-            }
-
-            @Override
-            public void onFailure(Throwable e) {
-                finishHim(e);
-            }
+        searchWithRetry(listener -> client.searchScroll(request, listener), (SearchResponse response) -> {
+            onScrollResponse(timeValueNanos(max(0, earliestNextBatchStartTime - System.nanoTime())), response);
         });
     }
 
@@ -434,9 +423,9 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
     }
 
     /**
-     * Build the backoff policy for use with retries.
+     * Get the backoff policy for use with retries.
      */
-    BackoffPolicy backoffPolicy() {
+    BackoffPolicy buildBackoffPolicy() {
         return exponentialBackoff(mainRequest.getRetryBackoffInitialTime(), mainRequest.getMaxRetries());
     }
 
@@ -470,7 +459,7 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
     }
 
     /**
-     * Wraps a backoffPolicy in another policy that counts the number of backoffs acquired.
+     * Wraps a backoffPolicy in another policy that counts the number of backoffs acquired. Used to count bulk backoffs.
      */
     private BackoffPolicy wrapBackoffPolicy(BackoffPolicy backoffPolicy) {
         return new BackoffPolicy() {
@@ -488,11 +477,52 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
                         if (false == delegate.hasNext()) {
                             return null;
                         }
-                        task.countRetry();
+                        task.countBulkRetry();
                         return delegate.next();
                     }
                 };
             }
         };
     }
+
+    /**
+     * Run a search action and call onResponse when a the response comes in, retrying if the action fails with an exception caused by
+     * rejected execution.
+     *
+     * @param action consumes a listener and starts the action. The listener it consumes is rigged to retry on failure.
+     * @param onResponse consumes the response from the action
+     */
+    private <T> void searchWithRetry(Consumer<ActionListener<T>> action, Consumer<T> onResponse) {
+        class RetryHelper extends AbstractRunnable implements ActionListener<T> {
+            private final Iterator<TimeValue> retries = backoffPolicy.iterator();
+
+            @Override
+            public void onResponse(T response) {
+                onResponse.accept(response);
+            }
+
+            @Override
+            protected void doRun() throws Exception {
+                action.accept(this);
+            }
+
+            @Override
+            public void onFailure(Throwable e) {
+                if (ExceptionsHelper.unwrap(e, EsRejectedExecutionException.class) != null) {
+                    if (retries.hasNext()) {
+                        logger.trace("retrying rejected search", e);
+                        threadPool.schedule(retries.next(), ThreadPool.Names.SAME, this);
+                        task.countSearchRetry();
+                    } else {
+                        logger.warn("giving up on search because we retried {} times without success", e, retries);
+                        finishHim(e);
+                    }
+                } else {
+                    logger.warn("giving up on search because it failed with a non-retryable exception", e);
+                    finishHim(e);
+                }
+            }
+        }
+        new RetryHelper().run();
+    }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
index 14790e97acc..7505f490f45 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
@@ -19,9 +19,6 @@
 
 package org.elasticsearch.index.reindex;
 
-import java.io.IOException;
-import java.util.Arrays;
-
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.WriteConsistencyLevel;
@@ -34,6 +31,9 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
 
+import java.io.IOException;
+import java.util.Arrays;
+
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMinutes;
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
index c158a7669d2..bd1a63cb4e9 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
@@ -107,6 +107,23 @@ public abstract class AbstractBulkByScrollRequestBuilder<
         return self();
     }
 
+    /**
+     * Initial delay after a rejection before retrying a bulk request. With the default maxRetries the total backoff for retrying rejections
+     * is about one minute per bulk request. Once the entire bulk request is successful the retry counter resets.
+     */
+    public Self setRetryBackoffInitialTime(TimeValue retryBackoffInitialTime) {
+        request.setRetryBackoffInitialTime(retryBackoffInitialTime);
+        return self();
+    }
+
+    /**
+     * Total number of retries attempted for rejections. There is no way to ask for unlimited retries.
+     */
+    public Self setMaxRetries(int maxRetries) {
+        request.setMaxRetries(maxRetries);
+        return self();
+    }
+
     /**
      * Set the throttle for this request in sub-requests per second. {@link Float#POSITIVE_INFINITY} means set no throttle and that is the
      * default. Throttling is done between batches, as we start the next scroll requests. That way we can increase the scroll's timeout to
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java
index fd86b488180..3fbeea078b7 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java
@@ -26,12 +26,11 @@ import org.elasticsearch.script.Script;
 
 public abstract class AbstractBulkIndexByScrollRequestBuilder<
                 Request extends AbstractBulkIndexByScrollRequest<Request>,
-                Response extends BulkIndexByScrollResponse,
-                Self extends AbstractBulkIndexByScrollRequestBuilder<Request, Response, Self>>
-        extends AbstractBulkByScrollRequestBuilder<Request, Response, Self> {
+                Self extends AbstractBulkIndexByScrollRequestBuilder<Request, Self>>
+        extends AbstractBulkByScrollRequestBuilder<Request, BulkIndexByScrollResponse, Self> {
 
     protected AbstractBulkIndexByScrollRequestBuilder(ElasticsearchClient client,
-            Action<Request, Response, Self> action, SearchRequestBuilder search, Request request) {
+            Action<Request, BulkIndexByScrollResponse, Self> action, SearchRequestBuilder search, Request request) {
         super(client, action, search, request);
     }
 
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
index 9f4c8163b88..f062287a5de 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
@@ -58,7 +58,8 @@ public class BulkByScrollTask extends CancellableTask {
     private final AtomicLong noops = new AtomicLong(0);
     private final AtomicInteger batch = new AtomicInteger(0);
     private final AtomicLong versionConflicts = new AtomicLong(0);
-    private final AtomicLong retries = new AtomicLong(0);
+    private final AtomicLong bulkRetries = new AtomicLong(0);
+    private final AtomicLong searchRetries = new AtomicLong(0);
     private final AtomicLong throttledNanos = new AtomicLong();
     /**
      * The number of requests per second to which to throttle the request that this task represents. The other variables are all AtomicXXX
@@ -84,7 +85,8 @@ public class BulkByScrollTask extends CancellableTask {
     @Override
     public Status getStatus() {
         return new Status(total.get(), updated.get(), created.get(), deleted.get(), batch.get(), versionConflicts.get(), noops.get(),
-                retries.get(), timeValueNanos(throttledNanos.get()), getRequestsPerSecond(), getReasonCancelled(), throttledUntil());
+                bulkRetries.get(), searchRetries.get(), timeValueNanos(throttledNanos.get()), getRequestsPerSecond(), getReasonCancelled(),
+                throttledUntil());
     }
 
     private TimeValue throttledUntil() {
@@ -133,14 +135,16 @@ public class BulkByScrollTask extends CancellableTask {
         private final int batches;
         private final long versionConflicts;
         private final long noops;
-        private final long retries;
+        private final long bulkRetries;
+        private final long searchRetries;
         private final TimeValue throttled;
         private final float requestsPerSecond;
         private final String reasonCancelled;
         private final TimeValue throttledUntil;
 
-        public Status(long total, long updated, long created, long deleted, int batches, long versionConflicts, long noops, long retries,
-                TimeValue throttled, float requestsPerSecond, @Nullable String reasonCancelled, TimeValue throttledUntil) {
+        public Status(long total, long updated, long created, long deleted, int batches, long versionConflicts, long noops,
+                long bulkRetries, long searchRetries, TimeValue throttled, float requestsPerSecond, @Nullable String reasonCancelled,
+                TimeValue throttledUntil) {
             this.total = checkPositive(total, "total");
             this.updated = checkPositive(updated, "updated");
             this.created = checkPositive(created, "created");
@@ -148,7 +152,8 @@ public class BulkByScrollTask extends CancellableTask {
             this.batches = checkPositive(batches, "batches");
             this.versionConflicts = checkPositive(versionConflicts, "versionConflicts");
             this.noops = checkPositive(noops, "noops");
-            this.retries = checkPositive(retries, "retries");
+            this.bulkRetries = checkPositive(bulkRetries, "bulkRetries");
+            this.searchRetries = checkPositive(searchRetries, "searchRetries");
             this.throttled = throttled;
             this.requestsPerSecond = requestsPerSecond;
             this.reasonCancelled = reasonCancelled;
@@ -163,7 +168,8 @@ public class BulkByScrollTask extends CancellableTask {
             batches = in.readVInt();
             versionConflicts = in.readVLong();
             noops = in.readVLong();
-            retries = in.readVLong();
+            bulkRetries = in.readVLong();
+            searchRetries = in.readVLong();
             throttled = TimeValue.readTimeValue(in);
             requestsPerSecond = in.readFloat();
             reasonCancelled = in.readOptionalString();
@@ -179,7 +185,8 @@ public class BulkByScrollTask extends CancellableTask {
             out.writeVInt(batches);
             out.writeVLong(versionConflicts);
             out.writeVLong(noops);
-            out.writeVLong(retries);
+            out.writeVLong(bulkRetries);
+            out.writeVLong(searchRetries);
             throttled.writeTo(out);
             out.writeFloat(requestsPerSecond);
             out.writeOptionalString(reasonCancelled);
@@ -208,7 +215,11 @@ public class BulkByScrollTask extends CancellableTask {
             builder.field("batches", batches);
             builder.field("version_conflicts", versionConflicts);
             builder.field("noops", noops);
-            builder.field("retries", retries);
+            builder.startObject("retries"); {
+                builder.field("bulk", bulkRetries);
+                builder.field("search", searchRetries);
+            }
+            builder.endObject();
             builder.timeValueField("throttled_millis", "throttled", throttled);
             builder.field("requests_per_second", requestsPerSecond == Float.POSITIVE_INFINITY ? "unlimited" : requestsPerSecond);
             if (reasonCancelled != null) {
@@ -233,7 +244,7 @@ public class BulkByScrollTask extends CancellableTask {
             builder.append(",batches=").append(batches);
             builder.append(",versionConflicts=").append(versionConflicts);
             builder.append(",noops=").append(noops);
-            builder.append(",retries=").append(retries);
+            builder.append(",retries=").append(bulkRetries);
             if (reasonCancelled != null) {
                 builder.append(",canceled=").append(reasonCancelled);
             }
@@ -296,10 +307,17 @@ public class BulkByScrollTask extends CancellableTask {
         }
 
         /**
-         * Number of retries that had to be attempted due to rejected executions.
+         * Number of retries that had to be attempted due to bulk actions being rejected.
          */
-        public long getRetries() {
-            return retries;
+        public long getBulkRetries() {
+            return bulkRetries;
+        }
+
+        /**
+         * Number of retries that had to be attempted due to search actions being rejected.
+         */
+        public long getSearchRetries() {
+            return searchRetries;
         }
 
         /**
@@ -373,8 +391,12 @@ public class BulkByScrollTask extends CancellableTask {
         versionConflicts.incrementAndGet();
     }
 
-    void countRetry() {
-        retries.incrementAndGet();
+    void countBulkRetry() {
+        bulkRetries.incrementAndGet();
+    }
+
+    void countSearchRetry() {
+        searchRetries.incrementAndGet();
     }
 
     float getRequestsPerSecond() {
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java
index 35d45f3d5d4..8e20375d9a7 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java
@@ -98,6 +98,20 @@ public class BulkIndexByScrollResponse extends ActionResponse implements ToXCont
         return status.getReasonCancelled();
     }
 
+    /**
+     * The number of times that the request had retry bulk actions.
+     */
+    public long getBulkRetries() {
+        return status.getBulkRetries();
+    }
+
+    /**
+     * The number of times that the request had retry search actions.
+     */
+    public long getSearchRetries() {
+        return status.getSearchRetries();
+    }
+
     /**
      * All of the indexing failures. Version conflicts are only included if the request sets abortOnVersionConflict to true (the
      * default).
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
index d51fb7e8bc1..660815bbf52 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
@@ -19,10 +19,6 @@
 
 package org.elasticsearch.index.reindex;
 
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.CompositeIndicesRequest;
 import org.elasticsearch.action.IndicesRequest;
@@ -32,8 +28,11 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.uid.Versions;
 
-import static java.util.Collections.unmodifiableList;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
 
+import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 import static org.elasticsearch.index.VersionType.INTERNAL;
 
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java
index 6b5cee46c97..4f814dbc49d 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java
@@ -27,7 +27,7 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 
 public class ReindexRequestBuilder extends
-        AbstractBulkIndexByScrollRequestBuilder<ReindexRequest, BulkIndexByScrollResponse, ReindexRequestBuilder> {
+        AbstractBulkIndexByScrollRequestBuilder<ReindexRequest, ReindexRequestBuilder> {
     private final IndexRequestBuilder destination;
 
     public ReindexRequestBuilder(ElasticsearchClient client,
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
index 915921d6077..5f215a06679 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequest.java
@@ -19,14 +19,14 @@
 
 package org.elasticsearch.index.reindex;
 
-import java.util.ArrayList;
-import java.util.List;
-
 import org.elasticsearch.action.CompositeIndicesRequest;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import static java.util.Collections.unmodifiableList;
 
 /**
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java
index ef64f36b3db..8985f439fba 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java
@@ -25,7 +25,7 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 
 public class UpdateByQueryRequestBuilder extends
-        AbstractBulkIndexByScrollRequestBuilder<UpdateByQueryRequest, BulkIndexByScrollResponse, UpdateByQueryRequestBuilder> {
+        AbstractBulkIndexByScrollRequestBuilder<UpdateByQueryRequest, UpdateByQueryRequestBuilder> {
 
     public UpdateByQueryRequestBuilder(ElasticsearchClient client,
             Action<UpdateByQueryRequest, BulkIndexByScrollResponse, UpdateByQueryRequestBuilder> action) {
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
index 2409a99faf2..955cbd477d5 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionListener;
@@ -38,6 +40,8 @@ import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.search.ClearScrollRequest;
 import org.elasticsearch.action.search.ClearScrollResponse;
+import org.elasticsearch.action.search.ReduceSearchPhaseException;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchScrollRequest;
@@ -98,6 +102,7 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueMinutes;
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.emptyCollectionOf;
 import static org.hamcrest.Matchers.equalTo;
@@ -122,8 +127,8 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
     public void setupForTest() {
         client = new MyMockClient(new NoOpClient(getTestName()));
         threadPool = new ThreadPool(getTestName());
-        testRequest = new DummyAbstractBulkByScrollRequest();
-        firstSearchRequest = new SearchRequest().scroll(timeValueSeconds(10));
+        firstSearchRequest = new SearchRequest();
+        testRequest = new DummyAbstractBulkByScrollRequest(firstSearchRequest);
         listener = new PlainActionFuture<>();
         scrollId = null;
         taskManager = new TaskManager(Settings.EMPTY);
@@ -150,10 +155,62 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
      * random scroll id so it is checked instead.
      */
     private String scrollId() {
-        scrollId = randomSimpleString(random(), 1, 1000); // Empty string's get special behavior we don't want
+        scrollId = randomSimpleString(random(), 1, 1000); // Empty strings get special behavior we don't want
         return scrollId;
     }
 
+    public void testStartRetriesOnRejectionAndSucceeds() throws Exception {
+        client.searchesToReject = randomIntBetween(0, testRequest.getMaxRetries() - 1);
+        DummyAbstractAsyncBulkByScrollAction action = new DummyActionWithoutBackoff();
+        action.start();
+        assertBusy(() -> assertEquals(client.searchesToReject + 1, client.searchAttempts.get()));
+        if (listener.isDone()) {
+            Object result = listener.get();
+            fail("Expected listener not to be done but it was and had " + result);
+        }
+        assertNotNull("There should be a search attempt pending that we didn't reject", client.lastSearch.get());
+        assertEquals(client.searchesToReject, testTask.getStatus().getSearchRetries());
+    }
+
+    public void testStartRetriesOnRejectionButFailsOnTooManyRejections() throws Exception {
+        client.searchesToReject = testRequest.getMaxRetries() + randomIntBetween(1, 100);
+        DummyAbstractAsyncBulkByScrollAction action = new DummyActionWithoutBackoff();
+        action.start();
+        assertBusy(() -> assertEquals(testRequest.getMaxRetries() + 1, client.searchAttempts.get()));
+        assertBusy(() -> assertTrue(listener.isDone()));
+        ExecutionException e = expectThrows(ExecutionException.class, () -> listener.get());
+        assertThat(ExceptionsHelper.stackTrace(e), containsString(EsRejectedExecutionException.class.getSimpleName()));
+        assertNull("There shouldn't be a search attempt pending that we didn't reject", client.lastSearch.get());
+        assertEquals(testRequest.getMaxRetries(), testTask.getStatus().getSearchRetries());
+    }
+
+    public void testStartNextScrollRetriesOnRejectionAndSucceeds() throws Exception {
+        client.scrollsToReject = randomIntBetween(0, testRequest.getMaxRetries() - 1);
+        DummyAbstractAsyncBulkByScrollAction action = new DummyActionWithoutBackoff();
+        action.setScroll(scrollId());
+        action.startNextScroll(0);
+        assertBusy(() -> assertEquals(client.scrollsToReject + 1, client.scrollAttempts.get()));
+        if (listener.isDone()) {
+            Object result = listener.get();
+            fail("Expected listener not to be done but it was and had " + result);
+        }
+        assertNotNull("There should be a scroll attempt pending that we didn't reject", client.lastScroll.get());
+        assertEquals(client.scrollsToReject, testTask.getStatus().getSearchRetries());
+    }
+
+    public void testStartNextScrollRetriesOnRejectionButFailsOnTooManyRejections() throws Exception {
+        client.scrollsToReject = testRequest.getMaxRetries() + randomIntBetween(1, 100);
+        DummyAbstractAsyncBulkByScrollAction action = new DummyActionWithoutBackoff();
+        action.setScroll(scrollId());
+        action.startNextScroll(0);
+        assertBusy(() -> assertEquals(testRequest.getMaxRetries() + 1, client.scrollAttempts.get()));
+        assertBusy(() -> assertTrue(listener.isDone()));
+        ExecutionException e = expectThrows(ExecutionException.class, () -> listener.get());
+        assertThat(ExceptionsHelper.stackTrace(e), containsString(EsRejectedExecutionException.class.getSimpleName()));
+        assertNull("There shouldn't be a scroll attempt pending that we didn't reject", client.lastScroll.get());
+        assertEquals(testRequest.getMaxRetries(), testTask.getStatus().getSearchRetries());
+    }
+
     public void testScrollResponseSetsTotal() {
         // Default is 0, meaning unstarted
         assertEquals(0, testTask.getStatus().getTotal());
@@ -354,8 +411,9 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         int bulksToTry = randomIntBetween(1, 10);
         long retryAttempts = 0;
         for (int i = 0; i < bulksToTry; i++) {
-            retryAttempts += retryTestCase(false);
-            assertEquals(retryAttempts, testTask.getStatus().getRetries());
+            bulkRetryTestCase(false);
+            retryAttempts += testRequest.getMaxRetries();
+            assertEquals(retryAttempts, testTask.getStatus().getBulkRetries());
         }
     }
 
@@ -363,8 +421,8 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
      * Mimicks bulk rejections. These should be retried but we fail anyway because we run out of retries.
      */
     public void testBulkRejectionsRetryAndFailAnyway() throws Exception {
-        long retryAttempts = retryTestCase(true);
-        assertEquals(retryAttempts, testTask.getStatus().getRetries());
+        bulkRetryTestCase(true);
+        assertEquals(testRequest.getMaxRetries(), testTask.getStatus().getBulkRetries());
     }
 
     public void testPerfectlyThrottledBatchTime() {
@@ -398,6 +456,9 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction();
         action.setScroll(scrollId());
 
+        // Set the base for the scroll to wait - this is added to the figure we calculate below
+        firstSearchRequest.scroll(timeValueSeconds(10));
+
         // We'd like to get about 1 request a second
         testTask.rethrottle(1f);
         // Make the last scroll look nearly instant
@@ -405,7 +466,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         // The last batch had 100 documents
         action.startNextScroll(100);
 
-        // So the next request is going to have to wait an extra 100 seconds or so (base was 10, so 110ish)
+        // So the next request is going to have to wait an extra 100 seconds or so (base was 10 seconds, so 110ish)
         assertThat(client.lastScroll.get().request.scroll().keepAlive().seconds(), either(equalTo(110L)).or(equalTo(109L)));
 
         // Now we can simulate a response and check the delay that we used for the task
@@ -422,10 +483,14 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         assertEquals(capturedDelay.get(), testTask.getStatus().getThrottled());
     }
 
-    private long retryTestCase(boolean failWithRejection) throws Exception {
+    /**
+     * Execute a bulk retry test case. The total number of failures is random and the number of retries attempted is set to
+     * testRequest.getMaxRetries and controled by the failWithRejection parameter.
+     */
+    private void bulkRetryTestCase(boolean failWithRejection) throws Exception {
         int totalFailures = randomIntBetween(1, testRequest.getMaxRetries());
         int size = randomIntBetween(1, 100);
-        int retryAttempts = totalFailures - (failWithRejection ? 1 : 0);
+        testRequest.setMaxRetries(totalFailures - (failWithRejection ? 1 : 0));
 
         client.bulksToReject = client.bulksAttempts.get() + totalFailures;
         /*
@@ -433,13 +498,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
          * deal with it. We just wait for it to happen.
          */
         CountDownLatch successLatch = new CountDownLatch(1);
-        DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction() {
-            @Override
-            BackoffPolicy backoffPolicy() {
-                // Force a backoff time of 0 to prevent sleeping
-                return constantBackoff(timeValueMillis(0), retryAttempts);
-            }
-
+        DummyAbstractAsyncBulkByScrollAction action = new DummyActionWithoutBackoff() {
             @Override
             void startNextScroll(int lastBatchSize) {
                 successLatch.countDown();
@@ -459,14 +518,13 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         } else {
             successLatch.await(10, TimeUnit.SECONDS);
         }
-        return retryAttempts;
     }
 
     /**
      * The default retry time matches what we say it is in the javadoc for the request.
      */
     public void testDefaultRetryTimes() {
-        Iterator<TimeValue> policy = new DummyAbstractAsyncBulkByScrollAction().backoffPolicy().iterator();
+        Iterator<TimeValue> policy = new DummyAbstractAsyncBulkByScrollAction().buildBackoffPolicy().iterator();
         long millis = 0;
         while (policy.hasNext()) {
             millis += policy.next().millis();
@@ -625,7 +683,22 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         }
     }
 
+    /**
+     * An extension to {@linkplain DummyAbstractAsyncBulkByScrollAction} that uses a 0 delaying backoff policy.
+     */
+    private class DummyActionWithoutBackoff extends DummyAbstractAsyncBulkByScrollAction {
+        @Override
+        BackoffPolicy buildBackoffPolicy() {
+            // Force a backoff time of 0 to prevent sleeping
+            return constantBackoff(timeValueMillis(0), testRequest.getMaxRetries());
+        }
+    }
+
     private static class DummyAbstractBulkByScrollRequest extends AbstractBulkByScrollRequest<DummyAbstractBulkByScrollRequest> {
+        public DummyAbstractBulkByScrollRequest(SearchRequest searchRequest) {
+            super(searchRequest);
+        }
+
         @Override
         protected DummyAbstractBulkByScrollRequest self() {
             return this;
@@ -635,11 +708,23 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
     private class MyMockClient extends FilterClient {
         private final List<String> scrollsCleared = new ArrayList<>();
         private final AtomicInteger bulksAttempts = new AtomicInteger();
+        private final AtomicInteger searchAttempts = new AtomicInteger();
+        private final AtomicInteger scrollAttempts = new AtomicInteger();
         private final AtomicReference<Map<String, String>> lastHeaders = new AtomicReference<>();
         private final AtomicReference<RefreshRequest> lastRefreshRequest = new AtomicReference<>();
+        /**
+         * Last search attempt that wasn't rejected outright.
+         */
+        private final AtomicReference<RequestAndListener<SearchRequest, SearchResponse>> lastSearch = new AtomicReference<>();
+        /**
+         * Last scroll attempt that wasn't rejected outright.
+         */
         private final AtomicReference<RequestAndListener<SearchScrollRequest, SearchResponse>> lastScroll = new AtomicReference<>();
 
+
         private int bulksToReject = 0;
+        private int searchesToReject = 0;
+        private int scrollsToReject = 0;
 
         public MyMockClient(Client in) {
             super(in);
@@ -661,7 +746,19 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
                 listener.onResponse(null);
                 return;
             }
+            if (request instanceof SearchRequest) {
+                if (searchAttempts.incrementAndGet() <= searchesToReject) {
+                    listener.onFailure(wrappedRejectedException());
+                    return;
+                }
+                lastSearch.set(new RequestAndListener<>((SearchRequest) request, (ActionListener<SearchResponse>) listener));
+                return;
+            }
             if (request instanceof SearchScrollRequest) {
+                if (scrollAttempts.incrementAndGet() <= scrollsToReject) {
+                    listener.onFailure(wrappedRejectedException());
+                    return;
+                }
                 lastScroll.set(new RequestAndListener<>((SearchScrollRequest) request, (ActionListener<SearchResponse>) listener));
                 return;
             }
@@ -715,6 +812,25 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
             }
             super.doExecute(action, request, listener);
         }
+
+        private Throwable wrappedRejectedException() {
+            Exception e = new EsRejectedExecutionException();
+            int wraps = randomIntBetween(0, 4);
+            for (int i = 0; i < wraps; i++) {
+                switch (randomIntBetween(0, 2)) {
+                case 0:
+                    e = new SearchPhaseExecutionException("test", "test failure", e, new ShardSearchFailure[0]);
+                    continue;
+                case 1:
+                    e = new ReduceSearchPhaseException("test", "test failure", e, new ShardSearchFailure[0]);
+                    continue;
+                case 2:
+                    e = new ElasticsearchException(e);
+                    continue;
+                }
+            }
+            return e;
+        }
     }
 
     private static class RequestAndListener<Request extends ActionRequest<Request>, Response> {
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
index ee07c4a9278..d64c69ba362 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
@@ -127,25 +127,28 @@ public class BulkByScrollTaskTests extends ESTestCase {
     }
 
     public void testStatusHatesNegatives() {
-        expectThrows(IllegalArgumentException.class, status(-1, 0, 0, 0, 0, 0, 0, 0));
-        expectThrows(IllegalArgumentException.class, status(0, -1, 0, 0, 0, 0, 0, 0));
-        expectThrows(IllegalArgumentException.class, status(0, 0, -1, 0, 0, 0, 0, 0));
-        expectThrows(IllegalArgumentException.class, status(0, 0, 0, -1, 0, 0, 0, 0));
-        expectThrows(IllegalArgumentException.class, status(0, 0, 0, 0, -1, 0, 0, 0));
-        expectThrows(IllegalArgumentException.class, status(0, 0, 0, 0, 0, -1, 0, 0));
-        expectThrows(IllegalArgumentException.class, status(0, 0, 0, 0, 0, 0, -1, 0));
-        expectThrows(IllegalArgumentException.class, status(0, 0, 0, 0, 0, 0, 0, -1));
+        checkStatusNegatives(-1, 0, 0, 0, 0, 0, 0, 0, 0, "total");
+        checkStatusNegatives(0, -1, 0, 0, 0, 0, 0, 0, 0, "updated");
+        checkStatusNegatives(0, 0, -1, 0, 0, 0, 0, 0, 0, "created");
+        checkStatusNegatives(0, 0, 0, -1, 0, 0, 0, 0, 0, "deleted");
+        checkStatusNegatives(0, 0, 0, 0, -1, 0, 0, 0, 0, "batches");
+        checkStatusNegatives(0, 0, 0, 0, 0, -1, 0, 0, 0, "versionConflicts");
+        checkStatusNegatives(0, 0, 0, 0, 0, 0, -1, 0, 0, "noops");
+        checkStatusNegatives(0, 0, 0, 0, 0, 0, 0, -1, 0, "bulkRetries");
+        checkStatusNegatives(0, 0, 0, 0, 0, 0, 0, 0, -1, "searchRetries");
     }
 
     /**
      * Build a task status with only some values. Used for testing negative values.
      */
-    private ThrowingRunnable status(long total, long updated, long created, long deleted, int batches, long versionConflicts,
-            long noops, long retries) {
+    private void checkStatusNegatives(long total, long updated, long created, long deleted, int batches, long versionConflicts,
+            long noops, long bulkRetries, long searchRetries, String fieldName) {
         TimeValue throttle = parseTimeValue(randomPositiveTimeValue(), "test");
         TimeValue throttledUntil = parseTimeValue(randomPositiveTimeValue(), "test");
 
-        return () -> new BulkByScrollTask.Status(-1, 0, 0, 0, 0, 0, 0, 0, throttle, 0f, null, throttledUntil);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new BulkByScrollTask.Status(total, updated, created,
+                deleted, batches, versionConflicts, noops, bulkRetries, searchRetries, throttle, 0f, null, throttledUntil));
+        assertEquals(e.getMessage(), fieldName + " must be greater than 0 but was [-1]");
     }
 
     /**
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseMatcher.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseMatcher.java
index b34dfc4a595..5bce3260929 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseMatcher.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseMatcher.java
@@ -118,7 +118,7 @@ public class BulkIndexByScrollResponseMatcher extends TypeSafeMatcher<BulkIndexB
 
     @Override
     public void describeTo(Description description) {
-        description.appendText("indexed matches ").appendDescriptionOf(updatedMatcher);
+        description.appendText("updated matches ").appendDescriptionOf(updatedMatcher);
         description.appendText(" and created matches ").appendDescriptionOf(createdMatcher);
         if (batchesMatcher != null) {
             description.appendText(" and batches matches ").appendDescriptionOf(batchesMatcher);
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTestUtils.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTestUtils.java
index 763cca8c906..fc6af19e6b3 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTestUtils.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTestUtils.java
@@ -61,16 +61,15 @@ public class CancelTestUtils {
     private static final CyclicBarrier barrier = new CyclicBarrier(2);
 
     public static <Request extends AbstractBulkIndexByScrollRequest<Request>,
-                    Response extends BulkIndexByScrollResponse,
-                    Builder extends AbstractBulkIndexByScrollRequestBuilder<Request, Response, Builder>>
-            Response testCancel(ESIntegTestCase test, Builder request, String actionToCancel) throws Exception {
+                    Builder extends AbstractBulkIndexByScrollRequestBuilder<Request, Builder>>
+            BulkIndexByScrollResponse testCancel(ESIntegTestCase test, Builder request, String actionToCancel) throws Exception {
 
         test.indexRandom(true, client().prepareIndex("source", "test", "1").setSource("foo", "a"),
                 client().prepareIndex("source", "test", "2").setSource("foo", "a"));
 
         request.source("source").script(new Script("sticky", ScriptType.INLINE, "native", emptyMap()));
         request.source().setSize(1);
-        ListenableActionFuture<Response> response = request.execute();
+        ListenableActionFuture<BulkIndexByScrollResponse> response = request.execute();
 
         // Wait until the script is on the first document.
         barrier.await(30, TimeUnit.SECONDS);
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
index 5a0196159de..dfa17ae4778 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
@@ -43,7 +43,7 @@ public class RethrottleTests extends ReindexTestCase {
         testCase(updateByQuery().source("test"), UpdateByQueryAction.NAME);
     }
 
-    private void testCase(AbstractBulkIndexByScrollRequestBuilder<?, ? extends BulkIndexByScrollResponse, ?> request, String actionName)
+    private void testCase(AbstractBulkIndexByScrollRequestBuilder<?, ?> request, String actionName)
             throws Exception {
         // Use a single shard so the reindex has to happen in multiple batches
         client().admin().indices().prepareCreate("test").setSettings("index.number_of_shards", 1).get();
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
new file mode 100644
index 00000000000..0b557898552
--- /dev/null
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.ListenableActionFuture;
+import org.elasticsearch.action.bulk.BackoffPolicy;
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.bulk.Retry;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.MockSearchService;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.function.IntFunction;
+
+import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.greaterThan;
+
+/**
+ * Integration test for retry behavior. Useful because retrying relies on the way that the rest of Elasticsearch throws exceptions and unit
+ * tests won't verify that.
+ */
+public class RetryTests extends ReindexTestCase {
+    /**
+     * The number of concurrent requests to test.
+     */
+    private static final int CONCURRENT = 12;
+    /**
+     * Enough docs that the requests will likely step on each other.
+     */
+    private static final int DOC_COUNT = 200;
+
+    /**
+     * Lower the queue sizes to be small enough that both bulk and searches will time out and have to be retried.
+     */
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal));
+        settings.put("threadpool.bulk.queue_size", 1);
+        settings.put("threadpool.bulk.size", 1);
+        settings.put("threadpool.search.queue_size", 1);
+        settings.put("threadpool.search.size", 1);
+        return settings.build();
+    }
+
+    /**
+     * Disable search context leak detection because we expect leaks when there is an {@link EsRejectedExecutionException} queueing the
+     * reduce phase.
+     */
+    @Override
+    protected Collection<Class<? extends Plugin>> getMockPlugins() {
+        List<Class<? extends Plugin>> mockPlugins = new ArrayList<>();
+        for (Class<? extends Plugin> plugin: super.getMockPlugins()) {
+            if (plugin.equals(MockSearchService.TestPlugin.class)) {
+                continue;
+            }
+            mockPlugins.add(plugin);
+        }
+        return mockPlugins;
+    }
+
+    public void testReindex() throws Exception {
+        setupSourceIndex("source");
+        testCase(true, i -> reindex().source("source").destination("dest" + i));
+    }
+
+    public void testUpdateByQuery() throws Exception {
+        for (int i = 0; i < CONCURRENT; i++) {
+            setupSourceIndex("source" + i);
+        }
+        testCase(false, i -> updateByQuery().source("source" + i));
+    }
+
+    private void testCase(boolean expectCreated, IntFunction<AbstractBulkIndexByScrollRequestBuilder<?, ?>> requestBuilder)
+            throws Exception {
+        List<ListenableActionFuture<BulkIndexByScrollResponse>> futures = new ArrayList<>(CONCURRENT);
+        for (int i = 0; i < CONCURRENT; i++) {
+            AbstractBulkIndexByScrollRequestBuilder<?, ?> request = requestBuilder.apply(i);
+            // Make sure we use more than one batch so we get the full reindex behavior
+            request.source().setSize(DOC_COUNT / randomIntBetween(2, 10));
+            // Use a low, random initial wait so we are unlikely collide with others retrying.
+            request.setRetryBackoffInitialTime(timeValueMillis(randomIntBetween(10, 300)));
+            futures.add(request.execute());
+        }
+
+        // Finish all the requests
+        List<BulkIndexByScrollResponse> responses = new ArrayList<>(CONCURRENT);
+        for (ListenableActionFuture<BulkIndexByScrollResponse> future : futures) {
+            responses.add(future.get());
+        }
+
+        // Now check them
+        long bulkRetries = 0;
+        long searchRetries = 0;
+        BulkIndexByScrollResponseMatcher matcher = matcher();
+        if (expectCreated) {
+            matcher.created(DOC_COUNT);
+        } else {
+            matcher.updated(DOC_COUNT);
+        }
+        for (BulkIndexByScrollResponse response : responses) {
+            assertThat(response, matcher);
+            bulkRetries += response.getBulkRetries();
+            searchRetries += response.getSearchRetries();
+        }
+
+        // We expect at least one retry or this test isn't very useful
+        assertThat(bulkRetries, greaterThan(0L));
+        assertThat(searchRetries, greaterThan(0L));
+    }
+
+    private void setupSourceIndex(String name) {
+        try {
+            // Build the test index with a single shard so we can be sure that a search request *can* complete with the one thread
+            assertAcked(client().admin().indices().prepareCreate(name).setSettings(
+                    "index.number_of_shards", 1,
+                    "index.number_of_replicas", 0).get());
+            waitForRelocation(ClusterHealthStatus.GREEN);
+            // Build the test data. Don't use indexRandom because that won't work consistently with such small thread pools.
+            BulkRequestBuilder bulk = client().prepareBulk();
+            for (int i = 0; i < DOC_COUNT; i++) {
+                bulk.add(client().prepareIndex(name, "test").setSource("foo", "bar " + i));
+            }
+            Retry retry = Retry.on(EsRejectedExecutionException.class).policy(BackoffPolicy.exponentialBackoff());
+            BulkResponse response = retry.withSyncBackoff(client(), bulk.request());
+            assertFalse(response.buildFailureMessage(), response.hasFailures());
+            refresh(name);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
index 55a665cfa7e..d1cb77361bb 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
@@ -138,7 +138,7 @@ public class RoundTripTests extends ESTestCase {
 
     private BulkByScrollTask.Status randomStatus() {
         return new BulkByScrollTask.Status(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
-                randomInt(Integer.MAX_VALUE), randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
+                randomInt(Integer.MAX_VALUE), randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), randomPositiveLong(),
                 parseTimeValue(randomPositiveTimeValue(), "test"), abs(random().nextFloat()),
                 random().nextBoolean() ? null : randomSimpleString(random()), parseTimeValue(randomPositiveTimeValue(), "test"));
     }
@@ -210,7 +210,8 @@ public class RoundTripTests extends ESTestCase {
         assertEquals(expected.getBatches(), actual.getBatches());
         assertEquals(expected.getVersionConflicts(), actual.getVersionConflicts());
         assertEquals(expected.getNoops(), actual.getNoops());
-        assertEquals(expected.getRetries(), actual.getRetries());
+        assertEquals(expected.getBulkRetries(), actual.getBulkRetries());
+        assertEquals(expected.getSearchRetries(), actual.getSearchRetries());
         assertEquals(expected.getThrottled(), actual.getThrottled());
         assertEquals(expected.getRequestsPerSecond(), actual.getRequestsPerSecond(), 0f);
         assertEquals(expected.getReasonCancelled(), actual.getReasonCancelled());
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryRequestTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryRequestTests.java
index f6780729143..5d17e1d88fd 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryRequestTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryRequestTests.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.index.reindex;
 
-import java.util.List;
-
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.List;
+
 import static org.apache.lucene.util.TestUtil.randomSimpleString;
 import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.hasSize;

From cfb06954bac819bd3fa9cd6e1b99761f6162db2a Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 17 May 2016 14:05:35 -0400
Subject: [PATCH 0299/1311] [reindex] Add assertBusy to test

It has timing issues.
---
 .../index/reindex/AsyncBulkByScrollActionTests.java           | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
index 955cbd477d5..c9bd22d3552 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
@@ -168,7 +168,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
             Object result = listener.get();
             fail("Expected listener not to be done but it was and had " + result);
         }
-        assertNotNull("There should be a search attempt pending that we didn't reject", client.lastSearch.get());
+        assertBusy(() -> assertNotNull("There should be a search attempt pending that we didn't reject", client.lastSearch.get()));
         assertEquals(client.searchesToReject, testTask.getStatus().getSearchRetries());
     }
 
@@ -194,7 +194,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
             Object result = listener.get();
             fail("Expected listener not to be done but it was and had " + result);
         }
-        assertNotNull("There should be a scroll attempt pending that we didn't reject", client.lastScroll.get());
+        assertBusy(() -> assertNotNull("There should be a scroll attempt pending that we didn't reject", client.lastScroll.get()));
         assertEquals(client.scrollsToReject, testTask.getStatus().getSearchRetries());
     }
 

From 978c1e3e365609c435671e95592c0c1e6215d46a Mon Sep 17 00:00:00 2001
From: polyfractal <zacharyjtong@gmail.com>
Date: Tue, 17 May 2016 14:46:25 -0400
Subject: [PATCH 0300/1311] [TEST] Add missing sort processor to test

Also fixes the naming of the sort REST test
to follow the numbering convention
---
 .../resources/rest-api-spec/test/ingest_grok/10_basic.yaml | 7 ++++---
 .../test/ingest/{90_sort.yaml => 100_sort.yaml}            | 0
 2 files changed, 4 insertions(+), 3 deletions(-)
 rename rest-api-spec/src/main/resources/rest-api-spec/test/ingest/{90_sort.yaml => 100_sort.yaml} (100%)

diff --git a/modules/ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok/10_basic.yaml b/modules/ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok/10_basic.yaml
index f61027c3de3..c41793fcdb8 100644
--- a/modules/ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok/10_basic.yaml
+++ b/modules/ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok/10_basic.yaml
@@ -22,6 +22,7 @@
     - match:  { nodes.$master.ingest.processors.10.type: remove }
     - match:  { nodes.$master.ingest.processors.11.type: rename }
     - match:  { nodes.$master.ingest.processors.12.type: set }
-    - match:  { nodes.$master.ingest.processors.13.type: split }
-    - match:  { nodes.$master.ingest.processors.14.type: trim }
-    - match:  { nodes.$master.ingest.processors.15.type: uppercase }
+    - match:  { nodes.$master.ingest.processors.13.type: sort }
+    - match:  { nodes.$master.ingest.processors.14.type: split }
+    - match:  { nodes.$master.ingest.processors.15.type: trim }
+    - match:  { nodes.$master.ingest.processors.16.type: uppercase }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/90_sort.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/100_sort.yaml
similarity index 100%
rename from rest-api-spec/src/main/resources/rest-api-spec/test/ingest/90_sort.yaml
rename to rest-api-spec/src/main/resources/rest-api-spec/test/ingest/100_sort.yaml

From 20aafb1196192d4f9f7faea8ce9a36b278e501a1 Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elastic.co>
Date: Tue, 17 May 2016 20:55:56 +0200
Subject: [PATCH 0301/1311] Revert "Add Autosense annotation for query dsl
 testing"

---
 docs/reference/query-dsl/bool-query.asciidoc  |   6 +-
 .../query-dsl/boosting-query.asciidoc         |  28 +--
 .../query-dsl/common-terms-query.asciidoc     | 226 +++++++----------
 .../query-dsl/constant-score-query.asciidoc   |  14 +-
 .../query-dsl/dis-max-query.asciidoc          |  26 +-
 .../reference/query-dsl/exists-query.asciidoc |  20 +-
 .../query-dsl/function-score-query.asciidoc   | 114 ++++-----
 docs/reference/query-dsl/fuzzy-query.asciidoc |  25 +-
 .../query-dsl/geo-bounding-box-query.asciidoc | 238 +++++++-----------
 .../query-dsl/geo-distance-query.asciidoc     | 133 ++++------
 .../geo-distance-range-query.asciidoc         |  26 +-
 .../query-dsl/geo-polygon-query.asciidoc      | 104 ++++----
 .../query-dsl/geo-shape-query.asciidoc        |  40 ++-
 .../query-dsl/geohash-cell-query.asciidoc     |  31 +--
 .../query-dsl/has-child-query.asciidoc        |  54 ++--
 .../query-dsl/has-parent-query.asciidoc       |  34 +--
 docs/reference/query-dsl/ids-query.asciidoc   |  10 +-
 .../query-dsl/indices-query.asciidoc          |  14 +-
 .../query-dsl/match-all-query.asciidoc        |  24 +-
 .../match-phrase-prefix-query.asciidoc        |  22 +-
 .../query-dsl/match-phrase-query.asciidoc     |  20 +-
 docs/reference/query-dsl/match-query.asciidoc |  47 ++--
 docs/reference/query-dsl/mlt-query.asciidoc   | 142 +++++------
 .../query-dsl/multi-match-query.asciidoc      | 212 ++++++----------
 .../reference/query-dsl/nested-query.asciidoc |  39 ++-
 .../query-dsl/parent-id-query.asciidoc        |  55 +---
 .../query-dsl/percolate-query.asciidoc        |  80 +++---
 .../reference/query-dsl/prefix-query.asciidoc |  15 +-
 .../query-dsl/query-string-query.asciidoc     |  64 ++---
 .../query-dsl/query_filter_context.asciidoc   |   3 +-
 docs/reference/query-dsl/range-query.asciidoc |  54 ++--
 .../reference/query-dsl/regexp-query.asciidoc |  47 ++--
 .../reference/query-dsl/script-query.asciidoc |  40 ++-
 .../simple-query-string-query.asciidoc        |  24 +-
 .../query-dsl/span-containing-query.asciidoc  |  28 +--
 .../query-dsl/span-first-query.asciidoc       |  14 +-
 .../query-dsl/span-multi-term-query.asciidoc  |  20 +-
 .../query-dsl/span-near-query.asciidoc        |  20 +-
 .../query-dsl/span-not-query.asciidoc         |  28 +--
 .../query-dsl/span-or-query.asciidoc          |  16 +-
 .../query-dsl/span-term-query.asciidoc        |  18 +-
 .../query-dsl/span-within-query.asciidoc      |  28 +--
 .../query-dsl/template-query.asciidoc         |  15 +-
 docs/reference/query-dsl/terms-query.asciidoc |  54 ++--
 docs/reference/query-dsl/type-query.asciidoc  |   8 +-
 .../query-dsl/wildcard-query.asciidoc         |  18 +-
 46 files changed, 866 insertions(+), 1432 deletions(-)

diff --git a/docs/reference/query-dsl/bool-query.asciidoc b/docs/reference/query-dsl/bool-query.asciidoc
index 2a289910c43..8fcd66129cd 100644
--- a/docs/reference/query-dsl/bool-query.asciidoc
+++ b/docs/reference/query-dsl/bool-query.asciidoc
@@ -81,7 +81,7 @@ all documents where the `status` field contains the term `active`.
 This first query assigns a score of `0` to all documents, as no scoring
 query has been specified:
 
-[source,js]
+[source,json]
 ---------------------------------
 GET _search
 {
@@ -101,7 +101,7 @@ GET _search
 This `bool` query has a `match_all` query, which assigns a score of `1.0` to
 all documents.
 
-[source,js]
+[source,json]
 ---------------------------------
 GET _search
 {
@@ -125,7 +125,7 @@ This `constant_score` query behaves in exactly the same way as the second exampl
 The `constant_score` query assigns a score of `1.0` to all documents matched
 by the filter.
 
-[source,js]
+[source,json]
 ---------------------------------
 GET _search
 {
diff --git a/docs/reference/query-dsl/boosting-query.asciidoc b/docs/reference/query-dsl/boosting-query.asciidoc
index 5bb07392ab7..969b3bbedfe 100644
--- a/docs/reference/query-dsl/boosting-query.asciidoc
+++ b/docs/reference/query-dsl/boosting-query.asciidoc
@@ -8,23 +8,19 @@ overall score.
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "boosting" : {
-            "positive" : {
-                "term" : {
-                    "field1" : "value1"
-                }
-            },
-            "negative" : {
-                 "term" : {
-                     "field2" : "value2"
-                }
-            },
-            "negative_boost" : 0.2
-        }
+    "boosting" : {
+        "positive" : {
+            "term" : {
+                "field1" : "value1"
+            }
+        },
+        "negative" : {
+            "term" : {
+                "field2" : "value2"
+            }
+        },
+        "negative_boost" : 0.2
     }
 }
 --------------------------------------------------
-// CONSOLE
diff --git a/docs/reference/query-dsl/common-terms-query.asciidoc b/docs/reference/query-dsl/common-terms-query.asciidoc
index fcc4ace2ec6..a956c33c1ee 100644
--- a/docs/reference/query-dsl/common-terms-query.asciidoc
+++ b/docs/reference/query-dsl/common-terms-query.asciidoc
@@ -70,19 +70,15 @@ In this example, words that have a document frequency greater than 0.1%
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "common": {
-            "body": {
-                "query": "this is bonsai cool",
-                    "cutoff_frequency": 0.001
-            }
-        }
+  "common": {
+    "body": {
+      "query": "this is bonsai cool",
+      "cutoff_frequency": 0.001
     }
+  }
 }
 --------------------------------------------------
-// CONSOLE
 
 The number of terms which should match can be controlled with the
 <<query-dsl-minimum-should-match,`minimum_should_match`>>
@@ -94,44 +90,36 @@ all terms required:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "common": {
-            "body": {
-                "query": "nelly the elephant as a cartoon",
-                    "cutoff_frequency": 0.001,
-                    "low_freq_operator": "and"
-            }
-        }
+  "common": {
+    "body": {
+      "query": "nelly the elephant as a cartoon",
+      "cutoff_frequency": 0.001,
+      "low_freq_operator": "and"
     }
+  }
 }
 --------------------------------------------------
-// CONSOLE
 
 which is roughly equivalent to:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "bool": {
-            "must": [
-            { "term": { "body": "nelly"}},
-            { "term": { "body": "elephant"}},
-            { "term": { "body": "cartoon"}}
-            ],
-            "should": [
-            { "term": { "body": "the"}},
-            { "term": { "body": "as"}},
-            { "term": { "body": "a"}}
-            ]
-        }
-    }
+  "bool": {
+    "must": [
+      { "term": { "body": "nelly"}},
+      { "term": { "body": "elephant"}},
+      { "term": { "body": "cartoon"}}
+    ],
+    "should": [
+      { "term": { "body": "the"}}
+      { "term": { "body": "as"}}
+      { "term": { "body": "a"}}
+    ]
+  }
 }
 --------------------------------------------------
-// CONSOLE
 
 Alternatively use
 <<query-dsl-minimum-should-match,`minimum_should_match`>>
@@ -140,49 +128,41 @@ must be present, for instance:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "common": {
-            "body": {
-                "query": "nelly the elephant as a cartoon",
-                "cutoff_frequency": 0.001,
-                "minimum_should_match": 2
-            }
-        }
+  "common": {
+    "body": {
+      "query": "nelly the elephant as a cartoon",
+      "cutoff_frequency": 0.001,
+      "minimum_should_match": 2
     }
+  }
 }
 --------------------------------------------------
-// CONSOLE
 
 which is roughly equivalent to:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "bool": {
-            "must": {
-                "bool": {
-                    "should": [
-                    { "term": { "body": "nelly"}},
-                    { "term": { "body": "elephant"}},
-                    { "term": { "body": "cartoon"}}
-                    ],
-                    "minimum_should_match": 2
-                }
-            },
-            "should": [
-                { "term": { "body": "the"}},
-                { "term": { "body": "as"}},
-                { "term": { "body": "a"}}
-                ]
-        }
-    }
+  "bool": {
+    "must": {
+      "bool": {
+        "should": [
+          { "term": { "body": "nelly"}},
+          { "term": { "body": "elephant"}},
+          { "term": { "body": "cartoon"}}
+        ],
+        "minimum_should_match": 2
+      }
+    },
+    "should": [
+      { "term": { "body": "the"}}
+      { "term": { "body": "as"}}
+      { "term": { "body": "a"}}
+    ]
+  }
 }
 --------------------------------------------------
-// CONSOLE
 
 minimum_should_match
 
@@ -194,58 +174,50 @@ additional parameters (note the change in structure):
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "common": {
-            "body": {
-                "query": "nelly the elephant not as a cartoon",
-                    "cutoff_frequency": 0.001,
-                    "minimum_should_match": {
-                        "low_freq" : 2,
-                        "high_freq" : 3
-                    }
-            }
-        }
+  "common": {
+    "body": {
+      "query": "nelly the elephant not as a cartoon",
+      "cutoff_frequency": 0.001,
+      "minimum_should_match": {
+          "low_freq" : 2,
+          "high_freq" : 3
+       }
     }
+  }
 }
 --------------------------------------------------
-// CONSOLE
 
 which is roughly equivalent to:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "bool": {
-            "must": {
-                "bool": {
-                    "should": [
-                    { "term": { "body": "nelly"}},
-                    { "term": { "body": "elephant"}},
-                    { "term": { "body": "cartoon"}}
-                    ],
-                    "minimum_should_match": 2
-                }
-            },
-            "should": {
-                "bool": {
-                    "should": [
-                    { "term": { "body": "the"}},
-                    { "term": { "body": "not"}},
-                    { "term": { "body": "as"}},
-                    { "term": { "body": "a"}}
-                    ],
-                    "minimum_should_match": 3
-                }
-            }
-        }
+  "bool": {
+    "must": {
+      "bool": {
+        "should": [
+          { "term": { "body": "nelly"}},
+          { "term": { "body": "elephant"}},
+          { "term": { "body": "cartoon"}}
+        ],
+        "minimum_should_match": 2
+      }
+    },
+    "should": {
+      "bool": {
+        "should": [
+          { "term": { "body": "the"}},
+          { "term": { "body": "not"}},
+          { "term": { "body": "as"}},
+          { "term": { "body": "a"}}
+        ],
+        "minimum_should_match": 3
+      }
     }
+  }
 }
 --------------------------------------------------
-// CONSOLE
 
 In this case it means the high frequency terms have only an impact on
 relevance when there are at least three of them. But the most
@@ -255,44 +227,36 @@ for high frequency terms is when there are only high frequency terms:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "common": {
-            "body": {
-                "query": "how not to be",
-                    "cutoff_frequency": 0.001,
-                    "minimum_should_match": {
-                        "low_freq" : 2,
-                        "high_freq" : 3
-                    }
-            }
-        }
+  "common": {
+    "body": {
+      "query": "how not to be",
+      "cutoff_frequency": 0.001,
+      "minimum_should_match": {
+          "low_freq" : 2,
+          "high_freq" : 3
+       }
     }
+  }
 }
 --------------------------------------------------
-// CONSOLE
 
 which is roughly equivalent to:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "bool": {
-            "should": [
-            { "term": { "body": "how"}},
-            { "term": { "body": "not"}},
-            { "term": { "body": "to"}},
-            { "term": { "body": "be"}}
-            ],
-            "minimum_should_match": "3<50%"
-        }
-    }
+  "bool": {
+    "should": [
+      { "term": { "body": "how"}},
+      { "term": { "body": "not"}},
+      { "term": { "body": "to"}},
+      { "term": { "body": "be"}}
+    ],
+    "minimum_should_match": "3<50%"
+  }
 }
 --------------------------------------------------
-// CONSOLE
 
 The high frequency generated query is then slightly less restrictive
 than with an `AND`.
diff --git a/docs/reference/query-dsl/constant-score-query.asciidoc b/docs/reference/query-dsl/constant-score-query.asciidoc
index bced9fc9fbe..8e76ac13ff5 100644
--- a/docs/reference/query-dsl/constant-score-query.asciidoc
+++ b/docs/reference/query-dsl/constant-score-query.asciidoc
@@ -7,16 +7,12 @@ filter. Maps to Lucene `ConstantScoreQuery`.
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "constant_score" : {
-            "filter" : {
-                "term" : { "user" : "kimchy"}
-            },
-            "boost" : 1.2
-        }
+    "constant_score" : {
+        "filter" : {
+            "term" : { "user" : "kimchy"}
+        },
+        "boost" : 1.2
     }
 }
 --------------------------------------------------
-// CONSOLE
diff --git a/docs/reference/query-dsl/dis-max-query.asciidoc b/docs/reference/query-dsl/dis-max-query.asciidoc
index 2f82f2294b3..2938c8db8ea 100644
--- a/docs/reference/query-dsl/dis-max-query.asciidoc
+++ b/docs/reference/query-dsl/dis-max-query.asciidoc
@@ -27,22 +27,18 @@ This query maps to Lucene `DisjunctionMaxQuery`.
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "dis_max" : {
-            "tie_breaker" : 0.7,
-                "boost" : 1.2,
-                "queries" : [
-                {
-                    "term" : { "age" : 34 }
-                },
-                {
-                    "term" : { "age" : 35 }
-                }
-                ]
-        }
+    "dis_max" : {
+        "tie_breaker" : 0.7,
+        "boost" : 1.2,
+        "queries" : [
+            {
+                "term" : { "age" : 34 }
+            },
+            {
+                "term" : { "age" : 35 }
+            }
+        ]
     }
 }    
 --------------------------------------------------
-// CONSOLE
diff --git a/docs/reference/query-dsl/exists-query.asciidoc b/docs/reference/query-dsl/exists-query.asciidoc
index 4971219366f..b484d47f4b6 100644
--- a/docs/reference/query-dsl/exists-query.asciidoc
+++ b/docs/reference/query-dsl/exists-query.asciidoc
@@ -5,14 +5,10 @@ Returns documents that have at least one non-`null` value in the original field:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "exists" : { "field" : "user" }
-    }
+    "exists" : { "field" : "user" }
 }
 --------------------------------------------------
-// CONSOLE
 
 For instance, these documents would all match the above query:
 
@@ -81,20 +77,14 @@ clause as follows:
 
 [source,js]
 --------------------------------------------------
-GET /_search
-{
-    "query": {
-        "bool": {
-            "must_not": {
-                "exists": {
-                    "field": "user"
-                }
-            }
+"bool": {
+    "must_not": {
+        "exists": {
+            "field": "user"
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 This query returns documents that have no value in the user field.
 
diff --git a/docs/reference/query-dsl/function-score-query.asciidoc b/docs/reference/query-dsl/function-score-query.asciidoc
index e7e4b8f5877..5f92d8c172c 100644
--- a/docs/reference/query-dsl/function-score-query.asciidoc
+++ b/docs/reference/query-dsl/function-score-query.asciidoc
@@ -14,20 +14,13 @@ by the query.
 
 [source,js]
 --------------------------------------------------
-GET /_search
-{
-    "query": {
-        "function_score": {
-            "query": {},
-            "boost": "5",
-            "random_score": {}, <1>
-            "boost_mode":"multiply"
-        }
-    }
+"function_score": {
+    "query": {},
+    "boost": "boost for the whole query",
+    "FUNCTION": {}, <1>
+    "boost_mode":"(multiply|replace|...)"
 }
 --------------------------------------------------
-// CONSOLE
-
 <1> See <<score-functions>> for a list of supported functions.
 
 Furthermore, several functions can be combined. In this case one can
@@ -36,35 +29,30 @@ given filtering query
 
 [source,js]
 --------------------------------------------------
-GET /_search
-{
-    "query": {
-        "function_score": {
-          "query": {},
-          "boost": "5", <1>
-          "functions": [
-              {
-                  "filter": {},
-                  "random_score": {}, <2>
-                  "weight": 23
-              },
-              {
-                  "filter": {},
-                  "weight": 42 
-              }
-          ],
-          "max_boost": 42,
-          "score_mode": "max",
-          "boost_mode": "multiply",
-          "min_score" : 42
+"function_score": {
+    "query": {},
+    "boost": "boost for the whole query",
+    "functions": [
+        {
+            "filter": {},
+            "FUNCTION": {}, <1>
+            "weight": number
+        },
+        {
+            "FUNCTION": {} <1>
+        },
+        {
+            "filter": {},
+            "weight": number
         }
-    }
+    ],
+    "max_boost": number,
+    "score_mode": "(multiply|max|...)",
+    "boost_mode": "(multiply|replace|...)",
+    "min_score" : number
 }
 --------------------------------------------------
-// CONSOLE
-
-<1> Boost for the whole query.
-<2> See <<score-functions>> for a list of supported functions.
+<1> See <<score-functions>> for a list of supported functions.
 
 NOTE: The scores produced by the filtering query of each function do not matter.
 
@@ -471,36 +459,36 @@ the request would look like this:
 
 [source,js]
 --------------------------------------------------
-GET /_search
+GET _search
 {
-    "query": {
-        "function_score": {
-          "functions": [
-            {
-              "gauss": {
-                "price": {
-                  "origin": "0",
-                  "scale": "20"
-                }
-              }
-            },
-            {
-              "gauss": {
-                "location": {
-                  "origin": "11, 12",
-                  "scale": "2km"
-                }
-              }
+  "query": {
+    "function_score": {
+      "functions": [
+        {
+          "gauss": {
+            "price": {
+              "origin": "0",
+              "scale": "20"
             }
-          ],
-          "query": {
-            "match": {
-              "properties": "balcony"
+          }
+        },
+        {
+          "gauss": {
+            "location": {
+              "origin": "11, 12",
+              "scale": "2km"
             }
-          },
-          "score_mode": "multiply"
+          }
         }
+      ],
+      "query": {
+        "match": {
+          "properties": "balcony"
+        }
+      },
+      "score_mode": "multiply"
     }
+  }
 }
 --------------------------------------------------
 // CONSOLE
diff --git a/docs/reference/query-dsl/fuzzy-query.asciidoc b/docs/reference/query-dsl/fuzzy-query.asciidoc
index f320e81b579..48c9c45c0ff 100644
--- a/docs/reference/query-dsl/fuzzy-query.asciidoc
+++ b/docs/reference/query-dsl/fuzzy-query.asciidoc
@@ -16,35 +16,27 @@ Here is a simple example:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-       "fuzzy" : { "user" : "ki" }
-    }
+    "fuzzy" : { "user" : "ki" }
 }
 --------------------------------------------------
-// CONSOLE
 
 Or with more advanced settings:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "fuzzy" : {
-            "user" : {
-                "value" :         "ki",
-                    "boost" :         1.0,
-                    "fuzziness" :     2,
-                    "prefix_length" : 0,
-                    "max_expansions": 100
-            }
+    "fuzzy" : {
+        "user" : {
+            "value" :         "ki",
+            "boost" :         1.0,
+            "fuzziness" :     2,
+            "prefix_length" : 0,
+            "max_expansions": 100
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 [float]
 ===== Parameters
@@ -70,4 +62,3 @@ WARNING: This query can be very heavy if `prefix_length` is set to `0` and if
 `max_expansions` is set to a high number. It could result in every term in the
 index being examined!
 
-
diff --git a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
index 40debb57105..00120a98fc9 100644
--- a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
+++ b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
@@ -6,24 +6,6 @@ bounding box. Assuming the following indexed document:
 
 [source,js]
 --------------------------------------------------
-PUT /my_locations
-{
-    "mappings": {
-        "location": {
-            "properties": {
-                "pin": {
-                    "properties": {
-                        "location": {
-                            "type": "geo_point"
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-
-PUT /my_locations/location/1
 {
     "pin" : {
         "location" : {
@@ -33,32 +15,27 @@ PUT /my_locations/location/1
     }
 }
 --------------------------------------------------
-// CONSOLE
-// TESTSETUP
 
 Then the following simple query can be executed with a
 `geo_bounding_box` filter:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_bounding_box" : {
-                    "pin.location" : {
-                        "top_left" : {
-                            "lat" : 40.73,
-                            "lon" : -74.1
-                        },
-                        "bottom_right" : {
-                            "lat" : 40.01,
-                            "lon" : -71.12
-                        }
+    "bool" : {
+        "must" : {
+            "match_all" : {}
+        },
+        "filter" : {
+            "geo_bounding_box" : {
+                "pin.location" : {
+                    "top_left" : {
+                        "lat" : 40.73,
+                        "lon" : -74.1
+                    },
+                    "bottom_right" : {
+                        "lat" : 40.01,
+                        "lon" : -71.12
                     }
                 }
             }
@@ -66,7 +43,6 @@ GET /_search
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 [float]
 ==== Query Options
@@ -99,24 +75,21 @@ representation of the geo point, the filter can accept it as well:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_bounding_box" : {
-                    "pin.location" : {
-                        "top_left" : {
-                            "lat" : 40.73,
-                            "lon" : -74.1
-                        },
-                        "bottom_right" : {
-                            "lat" : 40.01,
-                            "lon" : -71.12
-                        }
+    "bool" : {
+        "must" : {
+            "match_all" : {}
+        },
+        "filter" : {
+            "geo_bounding_box" : {
+                "pin.location" : {
+                    "top_left" : {
+                        "lat" : 40.73,
+                        "lon" : -74.1
+                    },
+                    "bottom_right" : {
+                        "lat" : 40.01,
+                        "lon" : -71.12
                     }
                 }
             }
@@ -124,7 +97,6 @@ GET /_search
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 [float]
 ===== Lat Lon As Array
@@ -134,26 +106,22 @@ conform with http://geojson.org/[GeoJSON].
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_bounding_box" : {
-                    "pin.location" : {
-                        "top_left" : [-74.1, 40.73],
-                        "bottom_right" : [-71.12, 40.01]
-                    }
+    "bool" : {
+        "must" : {
+            "match_all" : {}
+        },
+        "filter" : {
+            "geo_bounding_box" : {
+                "pin.location" : {
+                    "top_left" : [-74.1, 40.73],
+                    "bottom_right" : [-71.12, 40.01]
                 }
             }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 [float]
 ===== Lat Lon As String
@@ -162,52 +130,44 @@ Format in `lat,lon`.
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_bounding_box" : {
-                    "pin.location" : {
-                        "top_left" : "40.73, -74.1",
-                        "bottom_right" : "40.01, -71.12"
-                    }
-                }
-            }
-    }
-}
-}
---------------------------------------------------
-// CONSOLE
-
-[float]
-===== Geohash
-
-[source,js]
---------------------------------------------------
-GET /_search
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_bounding_box" : {
-                    "pin.location" : {
-                        "top_left" : "dr5r9ydj2y73",
-                        "bottom_right" : "drj7teegpus6"
-                    }
+    "bool" : {
+        "must" : {
+            "match_all" : {}
+        },
+        "filter" : {
+            "geo_bounding_box" : {
+                "pin.location" : {
+                    "top_left" : "40.73, -74.1",
+                    "bottom_right" : "40.01, -71.12"
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+
+[float]
+===== Geohash
+
+[source,js]
+--------------------------------------------------
+{
+    "bool" : {
+        "must" : {
+            "match_all" : {}
+        },
+        "filter" : {
+            "geo_bounding_box" : {
+                "pin.location" : {
+                    "top_left" : "dr5r9ydj2y73",
+                    "bottom_right" : "drj7teegpus6"
                 }
             }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 [float]
 ==== Vertices
@@ -221,28 +181,24 @@ values separately.
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_bounding_box" : {
-                    "pin.location" : {
-                        "top" : 40.73,
-                        "left" : -74.1,
-                        "bottom" : 40.01,
-                        "right" : -71.12
-                    }
+    "bool" : {
+        "must" : {
+            "match_all" : {}
+        },
+        "filter" : {
+            "geo_bounding_box" : {
+                "pin.location" : {
+                    "top" : 40.73,
+                    "left" : -74.1,
+                    "bottom" : 40.01,
+                    "right" : -71.12
                 }
             }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 
 [float]
@@ -271,33 +227,29 @@ are not supported. Here is an example:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_bounding_box" : {
-                    "pin.location" : {
-                        "top_left" : {
-                            "lat" : 40.73,
-                            "lon" : -74.1
-                        },
-                        "bottom_right" : {
-                            "lat" : 40.10,
-                            "lon" : -71.12
-                        }
+    "bool" : {
+        "must" : {
+            "match_all" : {}
+        },
+        "filter" : {
+            "geo_bounding_box" : {
+                "pin.location" : {
+                    "top_left" : {
+                        "lat" : 40.73,
+                        "lon" : -74.1
                     },
-                    "type" : "indexed"
-                }
+                    "bottom_right" : {
+                        "lat" : 40.10,
+                        "lon" : -71.12
+                    }
+                },
+                "type" : "indexed"
             }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 [float]
 ==== Ignore Unmapped
diff --git a/docs/reference/query-dsl/geo-distance-query.asciidoc b/docs/reference/query-dsl/geo-distance-query.asciidoc
index 5e2f0ce43c7..c29391e5b57 100644
--- a/docs/reference/query-dsl/geo-distance-query.asciidoc
+++ b/docs/reference/query-dsl/geo-distance-query.asciidoc
@@ -2,29 +2,10 @@
 === Geo Distance Query
 
 Filters documents that include only hits that exists within a specific
-distance from a geo point. Assuming the following mapping and indexed
-document:
+distance from a geo point. Assuming the following indexed json:
 
 [source,js]
 --------------------------------------------------
-PUT /my_locations
-{
-    "mappings": {
-        "location": {
-            "properties": {
-                "pin": {
-                    "properties": {
-                        "location": {
-                            "type": "geo_point"
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-
-PUT /my_locations/location/1
 {
     "pin" : {
         "location" : {
@@ -34,36 +15,29 @@ PUT /my_locations/location/1
     }
 }
 --------------------------------------------------
-// CONSOLE
-// TESTSETUP
-
 
 Then the following simple query can be executed with a `geo_distance`
 filter:
 
 [source,js]
 --------------------------------------------------
-GET /my_locations/location/_search
 {
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_distance" : {
-                    "distance" : "200km",
-                    "pin.location" : {
-                        "lat" : 40,
-                        "lon" : -70
-                    }
+    "bool" : {
+        "must" : {
+            "match_all" : {}
+        },
+        "filter" : {
+            "geo_distance" : {
+                "distance" : "200km",
+                "pin.location" : {
+                    "lat" : 40,
+                    "lon" : -70
                 }
             }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 [float]
 ==== Accepted Formats
@@ -76,27 +50,23 @@ representation of the geo point, the filter can accept it as well:
 
 [source,js]
 --------------------------------------------------
-GET /my_locations/location/_search
 {
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_distance" : {
-                    "distance" : "12km",
-                    "pin.location" : {
-                        "lat" : 40,
-                        "lon" : -70
-                    }
+    "bool" : {
+        "must" : {
+            "match_all" : {}
+        },
+        "filter" : {
+            "geo_distance" : {
+                "distance" : "12km",
+                "pin.location" : {
+                    "lat" : 40,
+                    "lon" : -70
                 }
             }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 [float]
 ===== Lat Lon As Array
@@ -106,25 +76,20 @@ conform with http://geojson.org/[GeoJSON].
 
 [source,js]
 --------------------------------------------------
-GET /my_locations/location/_search
 {
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_distance" : {
-                    "distance" : "12km",
-                    "pin.location" : [-70, 40]
-                }
+    "bool" : {
+        "must" : {
+            "match_all" : {}
+        },
+        "filter" : {
+            "geo_distance" : {
+                "distance" : "12km",
+                "pin.location" : [-70, 40]
             }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
-
 
 [float]
 ===== Lat Lon As String
@@ -133,48 +98,40 @@ Format in `lat,lon`.
 
 [source,js]
 --------------------------------------------------
-GET /my_locations/location/_search
 {
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_distance" : {
-                    "distance" : "12km",
-                    "pin.location" : "40,-70"
-                }
+    "bool" : {
+        "must" : {
+            "match_all" : {}
+        },
+        "filter" : {
+            "geo_distance" : {
+                "distance" : "12km",
+                "pin.location" : "40,-70"
             }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 [float]
 ===== Geohash
 
 [source,js]
 --------------------------------------------------
-GET /my_locations/location/_search
 {
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_distance" : {
-                    "distance" : "12km",
-                    "pin.location" : "drm3btev3e86"
-                }
+    "bool" : {
+        "must" : {
+            "match_all" : {}
+        },
+        "filter" : {
+            "geo_distance" : {
+                "distance" : "12km",
+                "pin.location" : "drm3btev3e86"
             }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 [float]
 ==== Options
diff --git a/docs/reference/query-dsl/geo-distance-range-query.asciidoc b/docs/reference/query-dsl/geo-distance-range-query.asciidoc
index c54c50638bc..abf557d68c7 100644
--- a/docs/reference/query-dsl/geo-distance-range-query.asciidoc
+++ b/docs/reference/query-dsl/geo-distance-range-query.asciidoc
@@ -5,28 +5,24 @@ Filters documents that exists within a range from a specific point:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_distance_range" : {
-                    "from" : "200km",
-                    "to" : "400km",
-                    "pin.location" : {
-                        "lat" : 40,
-                        "lon" : -70
-                    }
+    "bool" : {
+        "must" : {
+            "match_all" : {}
+        },
+        "filter" : {
+            "geo_distance_range" : {
+                "from" : "200km",
+                "to" : "400km",
+                "pin.location" : {
+                    "lat" : 40,
+                    "lon" : -70
                 }
             }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 Supports the same point location parameter and query options as the
 <<query-dsl-geo-distance-query,geo_distance>>
diff --git a/docs/reference/query-dsl/geo-polygon-query.asciidoc b/docs/reference/query-dsl/geo-polygon-query.asciidoc
index 35278fda4a5..5717c5dc924 100644
--- a/docs/reference/query-dsl/geo-polygon-query.asciidoc
+++ b/docs/reference/query-dsl/geo-polygon-query.asciidoc
@@ -6,29 +6,25 @@ points. Here is an example:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_polygon" : {
-                    "person.location" : {
-                        "points" : [
+    "bool" : {
+        "query" : {
+            "match_all" : {}
+        },
+        "filter" : {
+            "geo_polygon" : {
+                "person.location" : {
+                    "points" : [
                         {"lat" : 40, "lon" : -70},
                         {"lat" : 30, "lon" : -80},
                         {"lat" : 20, "lon" : -90}
-                        ]
-                    }
+                    ]
                 }
             }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 [float]
 ==== Query Options
@@ -57,29 +53,25 @@ conform with http://geojson.org/[GeoJSON].
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_polygon" : {
-                    "person.location" : {
-                        "points" : [
-                            [-70, 40],
+    "bool" : {
+        "must" : {
+            "match_all" : {}
+        },
+        "filter" : {
+            "geo_polygon" : {
+                "person.location" : {
+                    "points" : [
+                        [-70, 40],
                         [-80, 30],
-                            [-90, 20]
-                        ]
-                    }
+                        [-90, 20]
+                    ]
                 }
             }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 [float]
 ===== Lat Lon as String
@@ -88,58 +80,50 @@ Format in `lat,lon`.
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-               "geo_polygon" : {
-                    "person.location" : {
-                        "points" : [
-                            "40, -70",
-                            "30, -80",
-                            "20, -90"
-                        ]
-                    }
+    "bool" : {
+        "must" : {
+            "match_all" : {}
+        },
+        "filter" : {
+            "geo_polygon" : {
+                "person.location" : {
+                    "points" : [
+                        "40, -70",
+                        "30, -80",
+                        "20, -90"
+                    ]
                 }
             }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 [float]
 ===== Geohash
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-               "geo_polygon" : {
-                    "person.location" : {
-                        "points" : [
-                            "drn5x1g8cu2y",
-                            "30, -80",
-                            "20, -90"
-                        ]
-                    }
+    "bool" : {
+        "must" : {
+            "match_all" : {}
+        },
+        "filter" : {
+            "geo_polygon" : {
+                "person.location" : {
+                    "points" : [
+                        "drn5x1g8cu2y",
+                        "30, -80",
+                        "20, -90"
+                    ]
                 }
             }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 [float]
 ==== geo_point Type
diff --git a/docs/reference/query-dsl/geo-shape-query.asciidoc b/docs/reference/query-dsl/geo-shape-query.asciidoc
index 12203061336..a892080dda4 100644
--- a/docs/reference/query-dsl/geo-shape-query.asciidoc
+++ b/docs/reference/query-dsl/geo-shape-query.asciidoc
@@ -26,10 +26,10 @@ Given a document that looks like this:
 --------------------------------------------------
 {
     "name": "Wind & Wetter, Berlin, Germany",
-        "location": {
-            "type": "Point",
-            "coordinates": [13.400544, 52.530286]
-        }
+    "location": {
+        "type": "Point",
+        "coordinates": [13.400544, 52.530286]
+    }
 }
 --------------------------------------------------
 
@@ -38,7 +38,6 @@ The following query will find the point using the Elasticsearch's
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
     "query":{
         "bool": {
@@ -60,7 +59,6 @@ GET /_search
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 ==== Pre-Indexed Shape
 
@@ -83,30 +81,26 @@ shape:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "bool": {
-            "must": {
-                "match_all": {}
-            },
-                "filter": {
-                    "geo_shape": {
-                        "location": {
-                            "indexed_shape": {
-                                "id": "DEU",
-                                "type": "countries",
-                                "index": "shapes",
-                                "path": "location"
-                            }
-                        }
+    "bool": {
+        "must": {
+            "match_all": {}
+        },
+        "filter": {
+            "geo_shape": {
+                "location": {
+                    "indexed_shape": {
+                        "id": "DEU",
+                        "type": "countries",
+                        "index": "shapes",
+                        "path": "location"
                     }
                 }
+            }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 ==== Spatial Relations
 
diff --git a/docs/reference/query-dsl/geohash-cell-query.asciidoc b/docs/reference/query-dsl/geohash-cell-query.asciidoc
index 27e6319bc71..2dd701fd958 100644
--- a/docs/reference/query-dsl/geohash-cell-query.asciidoc
+++ b/docs/reference/query-dsl/geohash-cell-query.asciidoc
@@ -13,7 +13,6 @@ setting the `geohash_prefix` option:
 
 [source,js]
 --------------------------------------------------
-PUT /my_index
 {
     "mappings" : {
         "location": {
@@ -29,8 +28,6 @@ PUT /my_index
     }
 }
 --------------------------------------------------
-// CONSOLE
-// TESTSETUP
 
 The geohash cell can defined by all formats of `geo_points`. If such a cell is
 defined by a latitude and longitude pair the size of the cell needs to be
@@ -45,28 +42,24 @@ next to the given cell.
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geohash_cell": {
-                    "pin": {
-                        "lat": 13.4080,
-                        "lon": 52.5186
-                    },
-                    "precision": 3,
-                    "neighbors": true
-                }
+    "bool" : {
+        "must" : {
+            "match_all" : {}
+        },
+        "filter" : {
+            "geohash_cell": {
+                "pin": {
+                    "lat": 13.4080,
+                    "lon": 52.5186
+                },
+                "precision": 3,
+                "neighbors": true
             }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 [float]
 ==== Ignore Unmapped
diff --git a/docs/reference/query-dsl/has-child-query.asciidoc b/docs/reference/query-dsl/has-child-query.asciidoc
index 3b8352fd4e0..bfadc33c06c 100644
--- a/docs/reference/query-dsl/has-child-query.asciidoc
+++ b/docs/reference/query-dsl/has-child-query.asciidoc
@@ -7,21 +7,17 @@ an example:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "has_child" : {
-            "type" : "blog_tag",
-                "query" : {
-                    "term" : {
-                        "tag" : "something"
-                    }
-                }
+    "has_child" : {
+        "type" : "blog_tag",
+        "query" : {
+            "term" : {
+                "tag" : "something"
+            }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 [float]
 ==== Scoring capabilities
@@ -36,22 +32,18 @@ inside the `has_child` query:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "has_child" : {
-            "type" : "blog_tag",
-                "score_mode" : "min",
-                "query" : {
-                    "term" : {
-                        "tag" : "something"
-                    }
-                }
+    "has_child" : {
+        "type" : "blog_tag",
+        "score_mode" : "min",
+        "query" : {
+            "term" : {
+                "tag" : "something"
+            }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 [float]
 ==== Min/Max Children
@@ -62,24 +54,20 @@ a match:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "has_child" : {
-            "type" : "blog_tag",
-            "score_mode" : "min",
-            "min_children": 2, <1>
-            "max_children": 10, <1>
-            "query" : {
-                "term" : {
-                    "tag" : "something"
-                }
+    "has_child" : {
+        "type" : "blog_tag",
+        "score_mode" : "min",
+        "min_children": 2, <1>
+        "max_children": 10, <1>
+        "query" : {
+            "term" : {
+                "tag" : "something"
             }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 <1> Both `min_children` and `max_children` are optional.
 
 The  `min_children` and `max_children` parameters can be combined with
diff --git a/docs/reference/query-dsl/has-parent-query.asciidoc b/docs/reference/query-dsl/has-parent-query.asciidoc
index 202bcaac432..9896be6a0a5 100644
--- a/docs/reference/query-dsl/has-parent-query.asciidoc
+++ b/docs/reference/query-dsl/has-parent-query.asciidoc
@@ -9,21 +9,17 @@ in the same manner as the `has_child` query.
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "has_parent" : {
-            "parent_type" : "blog",
-                "query" : {
-                    "term" : {
-                        "tag" : "something"
-                    }
-                }
+    "has_parent" : {
+        "parent_type" : "blog",
+        "query" : {
+            "term" : {
+                "tag" : "something"
+            }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 [float]
 ==== Scoring capabilities
@@ -38,22 +34,18 @@ matching parent document. The score mode can be specified with the
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "has_parent" : {
-            "parent_type" : "blog",
-                "score" : true,
-                "query" : {
-                    "term" : {
-                        "tag" : "something"
-                    }
-                }
+    "has_parent" : {
+        "parent_type" : "blog",
+        "score" : true,
+        "query" : {
+            "term" : {
+                "tag" : "something"
+            }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 [float]
 ==== Ignore Unmapped
diff --git a/docs/reference/query-dsl/ids-query.asciidoc b/docs/reference/query-dsl/ids-query.asciidoc
index 09541ce51d3..7d08243a78f 100644
--- a/docs/reference/query-dsl/ids-query.asciidoc
+++ b/docs/reference/query-dsl/ids-query.asciidoc
@@ -6,17 +6,13 @@ uses the <<mapping-uid-field,_uid>> field.
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "ids" : {
-            "type" : "my_type",
-            "values" : ["1", "4", "100"]
-        }
+    "ids" : {
+        "type" : "my_type",
+        "values" : ["1", "4", "100"]
     }
 }    
 --------------------------------------------------
-// CONSOLE
 
 The `type` is optional and can be omitted, and can also accept an array
 of values. If no type is specified, all types defined in the index mapping are tried.
diff --git a/docs/reference/query-dsl/indices-query.asciidoc b/docs/reference/query-dsl/indices-query.asciidoc
index 8f2f958086e..e3b604b7a39 100644
--- a/docs/reference/query-dsl/indices-query.asciidoc
+++ b/docs/reference/query-dsl/indices-query.asciidoc
@@ -9,18 +9,18 @@ on the list, the alternative `no_match_query` is executed.
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "indices" : {
-            "indices" : ["index1", "index2"],
-            "query" : { "term" : { "tag" : "wow" } },
-            "no_match_query" : { "term" : { "tag" : "kow" } }
+    "indices" : {
+        "indices" : ["index1", "index2"],
+        "query" : {
+            "term" : { "tag" : "wow" }
+        },
+        "no_match_query" : {
+            "term" : { "tag" : "kow" }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE 
 
 You can use the `index` field to provide a single index.
 
diff --git a/docs/reference/query-dsl/match-all-query.asciidoc b/docs/reference/query-dsl/match-all-query.asciidoc
index 6e448828676..d46b08f9e55 100644
--- a/docs/reference/query-dsl/match-all-query.asciidoc
+++ b/docs/reference/query-dsl/match-all-query.asciidoc
@@ -6,27 +6,15 @@ of `1.0`.
 
 [source,js]
 --------------------------------------------------
-GET /_search
-{ 
-    "query": {
-        "match_all": {}
-    }
-}
+{ "match_all": {} }
 --------------------------------------------------
-// CONSOLE
 
 The `_score` can be changed with the `boost` parameter:
 
 [source,js]
 --------------------------------------------------
-GET /_search
-{
-    "query": {
-        "match_all": { "boost" : 1.2 }
-    }
-}
+{ "match_all": { "boost" : 1.2 }}
 --------------------------------------------------
-// CONSOLE
 
 [[query-dsl-match-none-query]]
 [float]
@@ -36,11 +24,5 @@ This is the inverse of the `match_all` query, which matches no documents.
 
 [source,js]
 --------------------------------------------------
-GET /_search
-{
-    "query": {
-        "match_none": {} 
-    }
-}
+{ "match_none": {} }
 --------------------------------------------------
-// CONSOLE
diff --git a/docs/reference/query-dsl/match-phrase-prefix-query.asciidoc b/docs/reference/query-dsl/match-phrase-prefix-query.asciidoc
index e7e0f618652..b715bf8bd6d 100644
--- a/docs/reference/query-dsl/match-phrase-prefix-query.asciidoc
+++ b/docs/reference/query-dsl/match-phrase-prefix-query.asciidoc
@@ -6,16 +6,12 @@ allows for prefix matches on the last term in the text. For example:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "match_phrase_prefix" : {
-            "message" : "quick brown f"
-        }
+    "match_phrase_prefix" : {
+        "message" : "quick brown f"
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 It accepts the same parameters as the phrase type. In addition, it also
 accepts a `max_expansions` parameter (default `50`) that can control to how
@@ -25,19 +21,15 @@ example:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "match_phrase_prefix" : {
-            "message" : {
-                "query" : "quick brown f",
-                "max_expansions" : 10
-            }
+    "match_phrase_prefix" : {
+        "message" : {
+            "query" : "quick brown f",
+            "max_expansions" : 10
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 [IMPORTANT]
 ===================================================
@@ -61,4 +53,4 @@ For better solutions for _search-as-you-type_ see the
 <<search-suggesters-completion,completion suggester>> and
 {guide}/_index_time_search_as_you_type.html[Index-Time Search-as-You-Type].
 
-===================================================
+===================================================
\ No newline at end of file
diff --git a/docs/reference/query-dsl/match-phrase-query.asciidoc b/docs/reference/query-dsl/match-phrase-query.asciidoc
index 943d0e84d36..105866be12a 100644
--- a/docs/reference/query-dsl/match-phrase-query.asciidoc
+++ b/docs/reference/query-dsl/match-phrase-query.asciidoc
@@ -6,16 +6,12 @@ out of the analyzed text. For example:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "match_phrase" : {
-            "message" : "this is a test"
-        }
+    "match_phrase" : {
+        "message" : "this is a test"
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 A phrase query matches terms up to a configurable `slop`
 (which defaults to 0) in any order. Transposed terms have a slop of 2.
@@ -26,16 +22,12 @@ definition, or the default search analyzer, for example:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "match_phrase" : {
-            "message" : {
-                "query" : "this is a test",
-                "analyzer" : "my_analyzer"
-            }
+    "match_phrase" : {
+        "message" : {
+            "query" : "this is a test",
+            "analyzer" : "my_analyzer"
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
diff --git a/docs/reference/query-dsl/match-query.asciidoc b/docs/reference/query-dsl/match-query.asciidoc
index c0081f1de9a..3dbb60d1ca2 100644
--- a/docs/reference/query-dsl/match-query.asciidoc
+++ b/docs/reference/query-dsl/match-query.asciidoc
@@ -7,16 +7,12 @@ them, and constructs a query. For example:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "match" : {
-            "message" : "this is a test"
-        }
+    "match" : {
+        "message" : "this is a test"
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 Note, `message` is the name of a field, you can substitute the name of
 any field (including `_all`) instead.
@@ -61,19 +57,15 @@ change in structure, `message` is the field name):
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "match" : {
-            "message" : {
-                "query" : "this is a test",
-                "operator" : "and"
-            }
+    "match" : {
+        "message" : {
+            "query" : "this is a test",
+            "operator" : "and"
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 [[query-dsl-match-query-zero]]
 ===== Zero terms query
@@ -84,20 +76,16 @@ change that the `zero_terms_query` option can be used, which accepts
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "match" : {
-            "message" : {
-                "query" : "to be or not to be",
-                "operator" : "and",
-                "zero_terms_query": "all"
-            }
+    "match" : {
+        "message" : {
+            "query" : "to be or not to be",
+            "operator" : "and",
+            "zero_terms_query": "all"
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 [[query-dsl-match-query-cutoff]]
 ===== Cutoff frequency
@@ -125,19 +113,16 @@ Here is an example showing a query composed of stopwords exclusively:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "match" : {
-            "message" : {
-                "query" : "to be or not to be",
-                "cutoff_frequency" : 0.001
-            }
+    "match" : {
+        "message" : {
+            "query" : "to be or not to be",
+            "cutoff_frequency" : 0.001
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
+
 
 IMPORTANT: The `cutoff_frequency` option operates on a per-shard-level. This means
 that when trying it out on test indexes with low document numbers you
diff --git a/docs/reference/query-dsl/mlt-query.asciidoc b/docs/reference/query-dsl/mlt-query.asciidoc
index b132b49f234..dc525c2d311 100644
--- a/docs/reference/query-dsl/mlt-query.asciidoc
+++ b/docs/reference/query-dsl/mlt-query.asciidoc
@@ -15,19 +15,15 @@ fields, limiting the number of selected terms to 12.
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "more_like_this" : {
-            "fields" : ["title", "description"],
-            "like" : "Once upon a time",
-            "min_term_freq" : 1,
-            "max_query_terms" : 12
-        }
+    "more_like_this" : {
+        "fields" : ["title", "description"],
+        "like" : "Once upon a time",
+        "min_term_freq" : 1,
+        "max_query_terms" : 12
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 A more complicated use case consists of mixing texts with documents already
 existing in the index. In this case, the syntax to specify a document is
@@ -35,31 +31,27 @@ similar to the one used in the <<docs-multi-get,Multi GET API>>.
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "more_like_this" : {
-            "fields" : ["title", "description"],
-            "like" : [
-            {
-                "_index" : "imdb",
-                "_type" : "movies",
-                "_id" : "1"
-            },
-            {
-                "_index" : "imdb",
-                "_type" : "movies",
-                "_id" : "2"
-            },
-            "and potentially some more text here as well"
-            ],
-            "min_term_freq" : 1,
-            "max_query_terms" : 12
-        }
+    "more_like_this" : {
+        "fields" : ["title", "description"],
+        "like" : [
+        {
+            "_index" : "imdb",
+            "_type" : "movies",
+            "_id" : "1"
+        },
+        {
+            "_index" : "imdb",
+            "_type" : "movies",
+            "_id" : "2"
+        },
+        "and potentially some more text here as well"
+        ],
+        "min_term_freq" : 1,
+        "max_query_terms" : 12
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 Finally, users can mix some texts, a chosen set of documents but also provide
 documents not necessarily present in the index. To provide documents not
@@ -67,36 +59,32 @@ present in the index, the syntax is similar to <<docs-termvectors-artificial-doc
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "more_like_this" : {
-            "fields" : ["name.first", "name.last"],
-            "like" : [
-            {
-                "_index" : "marvel",
-                "_type" : "quotes",
-                "doc" : {
-                    "name": {
-                        "first": "Ben",
-                        "last": "Grimm"
-                    },
-                    "tweet": "You got no idea what I'd... what I'd give to be invisible."
-                  }
-            },
-            {
-                "_index" : "marvel",
-                "_type" : "quotes",
-                "_id" : "2"
-            }
-            ],
-            "min_term_freq" : 1,
-            "max_query_terms" : 12
+    "more_like_this" : {
+        "fields" : ["name.first", "name.last"],
+        "like" : [
+        {
+            "_index" : "marvel",
+            "_type" : "quotes",
+            "doc" : {
+                "name": {
+                    "first": "Ben",
+                    "last": "Grimm"
+                },
+                "tweet": "You got no idea what I'd... what I'd give to be invisible."
+              }
+        },
+        {
+            "_index" : "marvel",
+            "_type" : "quotes",
+            "_id" : "2"
         }
+        ],
+        "min_term_freq" : 1,
+        "max_query_terms" : 12
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 ==== How it Works
 
@@ -123,34 +111,32 @@ default, but there will be no speed up on analysis for these fields.
 
 [source,js]
 --------------------------------------------------
-PUT /imdb
-{
-    "mappings": {
-        "movies": {
-            "properties": {
-                "title": {
-                    "type": "text",
-                    "term_vector": "yes"
-                },
-                "description": {
-                    "type": "text"
-                },
-                "tags": {
-                    "type": "text",
-                    "fields" : {
-                        "raw": {
-                            "type" : "text",
-                            "analyzer": "keyword",
-                            "term_vector" : "yes"
-                        }
-                    }
-                }
+curl -s -XPUT 'http://localhost:9200/imdb/' -d '{
+  "mappings": {
+    "movies": {
+      "properties": {
+        "title": {
+          "type": "text",
+          "term_vector": "yes"
+         },
+         "description": {
+          "type": "text"
+        },
+        "tags": {
+          "type": "text",
+          "fields" : {
+            "raw": {
+              "type" : "text",
+              "analyzer": "keyword",
+              "term_vector" : "yes"
             }
+          }
         }
+      }
     }
+  }
 }
 --------------------------------------------------
-// CONSOLE
 
 ==== Parameters
 
diff --git a/docs/reference/query-dsl/multi-match-query.asciidoc b/docs/reference/query-dsl/multi-match-query.asciidoc
index 068156ff70f..5c958eeaf81 100644
--- a/docs/reference/query-dsl/multi-match-query.asciidoc
+++ b/docs/reference/query-dsl/multi-match-query.asciidoc
@@ -6,17 +6,13 @@ to allow multi-field queries:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-  "query": {
-    "multi_match" : {
-      "query":    "this is a test", <1>
-      "fields": [ "subject", "message" ] <2>
-    }
+  "multi_match" : {
+    "query":    "this is a test", <1>
+    "fields": [ "subject", "message" ] <2>
   }
 }
 --------------------------------------------------
-// CONSOLE
 <1> The query string.
 <2> The fields to be queried.
 
@@ -27,35 +23,26 @@ Fields can be specified with wildcards, eg:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-  "query": {
-    "multi_match" : {
-      "query":    "Will Smith",
-      "fields": [ "title", "*_name" ] <1>
-    }
+  "multi_match" : {
+    "query":    "Will Smith",
+    "fields": [ "title", "*_name" ] <1>
   }
 }
 --------------------------------------------------
-// CONSOLE
 <1> Query the `title`, `first_name` and `last_name` fields.
 
 Individual fields can be boosted with the caret (`^`) notation:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-  "query": {
-    "multi_match" : {
-      "query" : "this is a test",
-      "fields" : [ "subject^3", "message" ] <1>
-    }
+  "multi_match" : {
+    "query" : "this is a test",
+    "fields" : [ "subject^3", "message" ] <1>
   }
 }
 --------------------------------------------------
-// CONSOLE
-
 <1> The `subject` field is three times as important as the `message` field.
 
 [[multi-match-types]]
@@ -95,38 +82,30 @@ find the single best matching field.  For instance, this query:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-  "query": {
-    "multi_match" : {
-      "query":      "brown fox",
-      "type":       "best_fields",
-      "fields":     [ "subject", "message" ],
-      "tie_breaker": 0.3
-    }
+  "multi_match" : {
+    "query":      "brown fox",
+    "type":       "best_fields",
+    "fields":     [ "subject", "message" ],
+    "tie_breaker": 0.3
   }
 }
 --------------------------------------------------
-// CONSOLE
 
 would be executed as:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-  "query": {
-    "dis_max": {
-      "queries": [
-        { "match": { "subject": "brown fox" }},
-        { "match": { "message": "brown fox" }}
-      ],
-      "tie_breaker": 0.3
-    }
+  "dis_max": {
+    "queries": [
+      { "match": { "subject": "brown fox" }},
+      { "match": { "message": "brown fox" }}
+    ],
+    "tie_breaker": 0.3
   }
 }
 --------------------------------------------------
-// CONSOLE
 
 Normally the `best_fields` type uses the score of the *single* best matching
 field, but if `tie_breaker` is specified, then it calculates the score as
@@ -153,20 +132,15 @@ Take this query for example:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-  "query": {
-    "multi_match" : {
-      "query":      "Will Smith",
-      "type":       "best_fields",
-      "fields":     [ "first_name", "last_name" ],
-      "operator":   "and" <1>
-    }
+  "multi_match" : {
+    "query":      "Will Smith",
+    "type":       "best_fields",
+    "fields":     [ "first_name", "last_name" ],
+    "operator":   "and" <1>
   }
 }
 --------------------------------------------------
-// CONSOLE
-
 <1> All terms must be present.
 
 This query is executed as:
@@ -196,37 +170,29 @@ This query:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-  "query": {
-    "multi_match" : {
-      "query":      "quick brown fox",
-      "type":       "most_fields",
-      "fields":     [ "title", "title.original", "title.shingles" ]
-    }
+  "multi_match" : {
+    "query":      "quick brown fox",
+    "type":       "most_fields",
+    "fields":     [ "title", "title.original", "title.shingles" ]
   }
 }
 --------------------------------------------------
-// CONSOLE
 
 would be executed as:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-  "query": {
-    "bool": {
-      "should": [
-        { "match": { "title":          "quick brown fox" }},
-        { "match": { "title.original": "quick brown fox" }},
-        { "match": { "title.shingles": "quick brown fox" }}
-      ]
-    }
+  "bool": {
+    "should": [
+      { "match": { "title":          "quick brown fox" }},
+      { "match": { "title.original": "quick brown fox" }},
+      { "match": { "title.shingles": "quick brown fox" }}
+    ]
   }
 }
 --------------------------------------------------
-// CONSOLE
 
 The score from each `match` clause is added together, then divided by the
 number of `match` clauses.
@@ -246,36 +212,28 @@ but they use a `match_phrase` or `match_phrase_prefix` query instead of a
 This query:
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-  "query": {
-    "multi_match" : {
-      "query":      "quick brown f",
-      "type":       "phrase_prefix",
-      "fields":     [ "subject", "message" ]
-    }
+  "multi_match" : {
+    "query":      "quick brown f",
+    "type":       "phrase_prefix",
+    "fields":     [ "subject", "message" ]
   }
 }
 --------------------------------------------------
-// CONSOLE
 
 would be executed as:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-  "query": {
-    "dis_max": {
-      "queries": [
-        { "match_phrase_prefix": { "subject": "quick brown f" }},
-        { "match_phrase_prefix": { "message": "quick brown f" }}
-      ]
-    }
+  "dis_max": {
+    "queries": [
+      { "match_phrase_prefix": { "subject": "quick brown f" }},
+      { "match_phrase_prefix": { "message": "quick brown f" }}
+    ]
   }
 }
 --------------------------------------------------
-// CONSOLE
 
 Also, accepts `analyzer`, `boost`, `slop` and `zero_terms_query`  as explained
 in <<query-dsl-match-query>>.  Type `phrase_prefix` additionally accepts
@@ -330,19 +288,15 @@ A query like:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-  "query": {
-    "multi_match" : {
-      "query":      "Will Smith",
-      "type":       "cross_fields",
-      "fields":     [ "first_name", "last_name" ],
-      "operator":   "and"
-    }
+  "multi_match" : {
+    "query":      "Will Smith",
+    "type":       "cross_fields",
+    "fields":     [ "first_name", "last_name" ],
+    "operator":   "and"
   }
 }
 --------------------------------------------------
-// CONSOLE
 
 is executed as:
 
@@ -390,21 +344,17 @@ both use an `edge_ngram` analyzer, this query:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-  "query": {
-    "multi_match" : {
-      "query":      "Jon",
-      "type":       "cross_fields",
-      "fields":     [
+  "multi_match" : {
+    "query":      "Jon",
+    "type":       "cross_fields",
+    "fields":     [
         "first", "first.edge",
         "last",  "last.edge"
-      ]
-    }
+    ]
   }
 }
 --------------------------------------------------
-// CONSOLE
 
 would be executed as:
 
@@ -429,33 +379,28 @@ parameter to just one of them:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-  "query": {
     "bool": {
-      "should": [
-        {
-          "multi_match" : {
-            "query":      "Will Smith",
-            "type":       "cross_fields",
-            "fields":     [ "first", "last" ],
-            "minimum_should_match": "50%" <1>
-          }
-        },
-        {
-          "multi_match" : {
-            "query":      "Will Smith",
-            "type":       "cross_fields",
-            "fields":     [ "*.edge" ]
-          }
-        }
-      ]
+        "should": [
+            {
+              "multi_match" : {
+                "query":      "Will Smith",
+                "type":       "cross_fields",
+                "fields":     [ "first", "last" ],
+                "minimum_should_match": "50%" <1>
+              }
+            },
+            {
+              "multi_match" : {
+                "query":      "Will Smith",
+                "type":       "cross_fields",
+                "fields":     [ "*.edge" ]
+              }
+            }
+        ]
     }
-  }
 }
 --------------------------------------------------
-// CONSOLE 
-
 <1> Either `will` or `smith` must be present in either of the `first`
     or `last` fields
 
@@ -464,20 +409,15 @@ parameter in the query.
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-  "query": {
-   "multi_match" : {
-      "query":      "Jon",
-      "type":       "cross_fields",
-      "analyzer":   "standard", <1>
-      "fields":     [ "first", "last", "*.edge" ]
-    }
+  "multi_match" : {
+    "query":      "Jon",
+    "type":       "cross_fields",
+    "analyzer":   "standard", <1>
+    "fields":     [ "first", "last", "*.edge" ]
   }
 }
 --------------------------------------------------
-// CONSOLE
-
 <1> Use the `standard` analyzer for all fields.
 
 which will be executed as:
diff --git a/docs/reference/query-dsl/nested-query.asciidoc b/docs/reference/query-dsl/nested-query.asciidoc
index 06720552bde..0b861509c0e 100644
--- a/docs/reference/query-dsl/nested-query.asciidoc
+++ b/docs/reference/query-dsl/nested-query.asciidoc
@@ -10,45 +10,40 @@ will work with:
 
 [source,js]
 --------------------------------------------------
-PUT /my_index
 {
-    "mappings": {
-        "type1" : {
-            "properties" : {
-                "obj1" : {
-                    "type" : "nested"
-                }
+    "type1" : {
+        "properties" : {
+            "obj1" : {
+                "type" : "nested"
             }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
-// TESTSETUP
 
 And here is a sample nested query usage:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "nested" : {
-            "path" : "obj1",
-            "score_mode" : "avg",
-            "query" : {
-                "bool" : {
-                    "must" : [
-                    { "match" : {"obj1.name" : "blue"} },
-                    { "range" : {"obj1.count" : {"gt" : 5}} }
-                    ]
-                }
+    "nested" : {
+        "path" : "obj1",
+        "score_mode" : "avg",
+        "query" : {
+            "bool" : {
+                "must" : [
+                    {
+                        "match" : {"obj1.name" : "blue"}
+                    },
+                    {
+                        "range" : {"obj1.count" : {"gt" : 5}}
+                    }
+                ]
             }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 The query `path` points to the nested object path, and the `query`
 includes the query that will run on the nested docs matching the
diff --git a/docs/reference/query-dsl/parent-id-query.asciidoc b/docs/reference/query-dsl/parent-id-query.asciidoc
index 10cf6d05664..1aef5b1948f 100644
--- a/docs/reference/query-dsl/parent-id-query.asciidoc
+++ b/docs/reference/query-dsl/parent-id-query.asciidoc
@@ -3,48 +3,17 @@
 
 added[5.0.0]
 
-The `parent_id` query can be used to find child documents which belong to a
-particular parent. Given the following mapping definition:
+The `parent_id` query can be used to find child documents which belong to a particular parent:
 
 [source,js]
 --------------------------------------------------
-PUT /my_index
 {
-    "mappings": {
-        "blog_post": {
-            "properties": {
-                "name": {
-                    "type": "keyword"
-                }
-            }
-        },
-        "blog_tag": {
-            "_parent": {
-                "type": "blog_post"
-            },
-            "_routing": {
-                "required": true
-            }
-        }
+    "parent_id" : {
+        "type" : "blog_tag",
+        "id" : "1"
     }
 }
 --------------------------------------------------
-// CONSOLE
-// TESTSETUP
-
-[source,js]
---------------------------------------------------
-GET /_search
-{
-    "query": {
-        "parent_id" : {
-            "type" : "blog_tag",
-                "id" : "1"
-        }
-    }
-}
---------------------------------------------------
-// CONSOLE
 
 The above is functionally equivalent to using the following
 <<query-dsl-has-parent-query, `has_parent`>> query, but performs
@@ -52,21 +21,17 @@ better as it does not need to do a join:
 
 [source,js]
 --------------------------------------------------
-GET /my_index/_search
 {
+  "has_parent": {
+    "type": "blog",
     "query": {
-        "has_parent": {
-            "type": "blog_post",
-                "query": {
-                    "term": {
-                        "_id": "1"
-                    }
-                }
-        }
+      "term": {
+        "_id": "1"
+      }
     }
+  }
 }
 --------------------------------------------------
-// CONSOLE
 
 ==== Parameters
 
diff --git a/docs/reference/query-dsl/percolate-query.asciidoc b/docs/reference/query-dsl/percolate-query.asciidoc
index 59d38f101ad..267647c9dbb 100644
--- a/docs/reference/query-dsl/percolate-query.asciidoc
+++ b/docs/reference/query-dsl/percolate-query.asciidoc
@@ -13,27 +13,26 @@ Create an index with two mappings:
 
 [source,js]
 --------------------------------------------------
-PUT /my-index
+curl -XPUT "http://localhost:9200/my-index" -d'
 {
-    "mappings": {
-        "doctype": {
-            "properties": {
-                "message": {
-                    "type": "string"
-                }
-            }
-        },
-        "queries": {
-            "properties": {
-                "query": {
-                    "type": "percolator"
-                }
-            }
+  "mappings": {
+    "doctype": {
+      "properties": {
+        "message": {
+          "type": "string"
         }
+      }
+    },
+    "queries": {
+      "properties": {
+        "query": {
+          "type": "percolator"
+        }
+      }
     }
-}
+  }
+}'
 --------------------------------------------------
-// CONSOLE
 
 The `doctype` mapping is the mapping used to preprocess
 the document defined in the `percolator` query before it
@@ -51,24 +50,20 @@ Register a query in the percolator:
 
 [source,js]
 --------------------------------------------------
-PUT /my-index/queries/1
-{
+curl -XPUT 'localhost:9200/my-index/queries/1' -d '{
     "query" : {
         "match" : {
             "message" : "bonsai tree"
         }
     }
-}
+}'
 --------------------------------------------------
-// CONSOLE
-// TEST[continued]
 
 Match a document to the registered percolator queries:
 
 [source,js]
 --------------------------------------------------
-GET /my-index/_search
-{
+curl -XGET 'localhost:9200/my-index/_search' -d '{
     "query" : {
         "percolate" : {
             "field" : "query",
@@ -78,10 +73,8 @@ GET /my-index/_search
             }
         }
     }
-}
+}'
 --------------------------------------------------
-// CONSOLE
-// TEST[continued]
 
 The above request will yield the following response:
 
@@ -158,13 +151,12 @@ Index the document we want to percolate:
 
 [source,js]
 --------------------------------------------------
-PUT /my-index/message/1
+curl -XPUT "http://localhost:9200/my-index/message/1" -d'
 {
   "message" : "A new bonsai tree in the office"
-}
+}'
 --------------------------------------------------
-// CONSOLE
-// TEST[continued]
+
 Index response:
 
 [source,js]
@@ -187,7 +179,7 @@ Percolating an existing document, using the index response as basis to build to
 
 [source,js]
 --------------------------------------------------
-GET /my-index/_search
+curl -XGET "http://localhost:9200/my-index/_search" -d'
 {
     "query" : {
         "percolate" : {
@@ -199,10 +191,8 @@ GET /my-index/_search
             "version" : 1 <1>
         }
     }
-}
+}'
 --------------------------------------------------
-// CONSOLE
-// TEST[continued]
 
 <1> The version is optional, but useful in certain cases. We can then ensure that we are try to percolate
 the document we just have indexed. A change may be made after we have indexed, and if that is the
@@ -226,39 +216,35 @@ Save a query:
 
 [source,js]
 --------------------------------------------------
-PUT /my-index/queries/1
+curl -XPUT "http://localhost:9200/my-index/queries/1" -d'
 {
     "query" : {
         "match" : {
             "message" : "brown fox"
         }
     }
-}
+}'
 --------------------------------------------------
-// CONSOLE
-// TEST[continued]
 
 Save another query:
 
 [source,js]
 --------------------------------------------------
-PUT /my-index/queries/2
+curl -XPUT "http://localhost:9200/my-index/queries/2" -d'
 {
     "query" : {
         "match" : {
             "message" : "lazy dog"
         }
     }
-}
+}'
 --------------------------------------------------
-// CONSOLE
-// TEST[continued]
 
 Execute a search request with the `percolate` query and highlighting enabled:
 
 [source,js]
 --------------------------------------------------
-GET /my-index/_search
+curl -XGET "http://localhost:9200/my-index/_search" -d'
 {
     "query" : {
         "percolate" : {
@@ -274,10 +260,8 @@ GET /my-index/_search
         "message": {}
       }
     }
-}
+}'
 --------------------------------------------------
-// CONSOLE
-// TEST[continued]
 
 This will yield the following response.
 
@@ -352,4 +336,4 @@ that are registered to the index that the search request is targeted for, are go
 in-memory index. This happens on each shard the search request needs to execute.
 
 By using `routing` or additional queries the amount of percolator queries that need to be executed can be reduced and thus
-the time the search API needs to run can be decreased.
+the time the search API needs to run can be decreased.
\ No newline at end of file
diff --git a/docs/reference/query-dsl/prefix-query.asciidoc b/docs/reference/query-dsl/prefix-query.asciidoc
index d2b75d10e5f..cf26e850ad8 100644
--- a/docs/reference/query-dsl/prefix-query.asciidoc
+++ b/docs/reference/query-dsl/prefix-query.asciidoc
@@ -8,37 +8,28 @@ that starts with `ki`:
 
 [source,js]
 --------------------------------------------------
-GET /_search
-{ "query": {
+{
     "prefix" : { "user" : "ki" }
-  }
 }
 --------------------------------------------------
-// CONSOLE
 
 A boost can also be associated with the query:
 
 [source,js]
 --------------------------------------------------
-GET /_search
-{ "query": {
+{
     "prefix" : { "user" :  { "value" : "ki", "boost" : 2.0 } }
-  }
 }
 --------------------------------------------------
-// CONSOLE
 
 Or :
 
 [source,js]
 --------------------------------------------------
-GET /_search
-{ "query": {
+{
     "prefix" : { "user" :  { "prefix" : "ki", "boost" : 2.0 } }
-  }
 }
 --------------------------------------------------
-// CONSOLE
 
 This multi term query allows you to control how it gets rewritten using the
 <<query-dsl-multi-term-rewrite,rewrite>>
diff --git a/docs/reference/query-dsl/query-string-query.asciidoc b/docs/reference/query-dsl/query-string-query.asciidoc
index 60477d6e28a..9ad68ad4f73 100644
--- a/docs/reference/query-dsl/query-string-query.asciidoc
+++ b/docs/reference/query-dsl/query-string-query.asciidoc
@@ -6,17 +6,13 @@ an example:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "query_string" : {
-            "default_field" : "content",
-            "query" : "this AND that OR thus"
-        }
+    "query_string" : {
+        "default_field" : "content",
+        "query" : "this AND that OR thus"
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 The `query_string` top level parameters include:
 
@@ -117,33 +113,25 @@ For example, the following query
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "query_string" : {
-            "fields" : ["content", "name"],
-            "query" : "this AND that"
-        }
+    "query_string" : {
+        "fields" : ["content", "name"],
+        "query" : "this AND that"
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 matches the same words as
 
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "query_string": {
-            "query": "(content:this OR name:this) AND (content:that OR name:that)"
-        }
+    "query_string": {
+      "query": "(content:this OR name:this) AND (content:that OR name:that)"
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 Since several queries are generated from the individual search terms,
 combining them can be automatically done using either a `dis_max` query or a
@@ -152,18 +140,14 @@ notation):
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "query_string" : {
-            "fields" : ["content", "name^5"],
-            "query" : "this AND that OR thus",
-            "use_dis_max" : true
-        }
+    "query_string" : {
+        "fields" : ["content", "name^5"],
+        "query" : "this AND that OR thus",
+        "use_dis_max" : true
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 Simple wildcard can also be used to search "within" specific inner
 elements of the document. For example, if we have a `city` object with
@@ -172,18 +156,14 @@ search on all "city" fields:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "query_string" : {
-            "fields" : ["city.*"],
-            "query" : "this AND that OR thus",
-            "use_dis_max" : true
-        }
+    "query_string" : {
+        "fields" : ["city.*"],
+        "query" : "this AND that OR thus",
+        "use_dis_max" : true
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 Another option is to provide the wildcard fields search in the query
 string itself (properly escaping the `*` sign), for example:
@@ -208,17 +188,13 @@ introduced fields included). For example:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "query_string" : {
-            "fields" : ["content", "name.*^5"],
-            "query" : "this AND that OR thus",
-            "use_dis_max" : true
-        }
+    "query_string" : {
+        "fields" : ["content", "name.*^5"],
+        "query" : "this AND that OR thus",
+        "use_dis_max" : true
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 include::query-string-syntax.asciidoc[]
diff --git a/docs/reference/query-dsl/query_filter_context.asciidoc b/docs/reference/query-dsl/query_filter_context.asciidoc
index 79f8c4bd960..6fadc36aa73 100644
--- a/docs/reference/query-dsl/query_filter_context.asciidoc
+++ b/docs/reference/query-dsl/query_filter_context.asciidoc
@@ -47,7 +47,7 @@ conditions are met:
 
 [source,js]
 ------------------------------------
-GET /_search
+GET _search
 {
   "query": { <1>
     "bool": { <2>
@@ -63,7 +63,6 @@ GET /_search
   }
 }
 ------------------------------------
-// CONSOLE
 <1> The `query` parameter indicates query context.
 <2> The `bool` and two `match` clauses are used in query context,
     which means that they are used to score how well each document
diff --git a/docs/reference/query-dsl/range-query.asciidoc b/docs/reference/query-dsl/range-query.asciidoc
index a0005ff3ff2..eaf1dca8181 100644
--- a/docs/reference/query-dsl/range-query.asciidoc
+++ b/docs/reference/query-dsl/range-query.asciidoc
@@ -9,20 +9,16 @@ a `NumericRangeQuery`. The following example returns all documents where
 
 [source,js]
 --------------------------------------------------
-GET _search
 {
-    "query": {
-        "range" : {
-            "age" : {
-                "gte" : 10,
-                "lte" : 20,
-                "boost" : 2.0
-            }
+    "range" : {
+        "age" : {
+            "gte" : 10,
+            "lte" : 20,
+            "boost" : 2.0
         }
     }
 }
 --------------------------------------------------
-// CONSOLE 
 
 The `range` query accepts the following parameters:
 
@@ -42,19 +38,15 @@ specified using <<date-math>>:
 
 [source,js]
 --------------------------------------------------
-GET _search
 {
-    "query": {
-        "range" : {
-            "date" : {
-                "gte" : "now-1d/d",
-                "lt" :  "now/d"
-            }
+    "range" : {
+        "date" : {
+            "gte" : "now-1d/d",
+            "lt" :  "now/d"
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 ===== Date math and rounding
 
@@ -94,20 +86,16 @@ passing the `format` parameter to the `range` query:
 
 [source,js]
 --------------------------------------------------
-GET _search
 {
-    "query": {
-        "range" : {
-            "born" : {
-                "gte": "01/01/2012",
-                "lte": "2013",
-                "format": "dd/MM/yyyy||yyyy"
-            }
+    "range" : {
+        "born" : {
+            "gte": "01/01/2012",
+            "lte": "2013",
+            "format": "dd/MM/yyyy||yyyy"
         }
     }
 }
 --------------------------------------------------
-// CONSOLE 
 
 ===== Time zone in range queries
 
@@ -117,19 +105,15 @@ accepts it), or it can be specified as the `time_zone` parameter:
 
 [source,js]
 --------------------------------------------------
-GET _search
 {
-    "query": {
-        "range" : {
-            "timestamp" : {
-                "gte": "2015-01-01 00:00:00", <1>
-                "lte": "now", <2>
-                "time_zone": "+01:00"
-            }
+    "range" : {
+        "timestamp" : {
+            "gte": "2015-01-01 00:00:00", <1>
+            "lte": "now", <2>
+            "time_zone": "+01:00"
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 <1> This date will be converted to `2014-12-31T23:00:00 UTC`.
 <2> `now` is not affected by the `time_zone` parameter (dates must be stored as UTC).
diff --git a/docs/reference/query-dsl/regexp-query.asciidoc b/docs/reference/query-dsl/regexp-query.asciidoc
index 2d57f1d7dd0..692caf43480 100644
--- a/docs/reference/query-dsl/regexp-query.asciidoc
+++ b/docs/reference/query-dsl/regexp-query.asciidoc
@@ -15,52 +15,40 @@ matchers like `.*?+` will mostly lower performance.
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "regexp":{
-            "name.first": "s.*y"
-        }
+    "regexp":{
+        "name.first": "s.*y"
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 Boosting is also supported
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "regexp":{
-            "name.first":{
-                "value":"s.*y",
-                "boost":1.2
-            }
+    "regexp":{
+        "name.first":{
+            "value":"s.*y",
+            "boost":1.2
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 You can also use special flags
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "regexp":{
-            "name.first": {
-                "value": "s.*y",
-                "flags" : "INTERSECTION|COMPLEMENT|EMPTY"
-            }
+    "regexp":{
+        "name.first": {
+            "value": "s.*y",
+            "flags" : "INTERSECTION|COMPLEMENT|EMPTY"
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 Possible flags are `ALL` (default), `ANYSTRING`, `COMPLEMENT`,
 `EMPTY`, `INTERSECTION`, `INTERVAL`, or `NONE`. Please check the
@@ -76,19 +64,16 @@ this limit to allow more complex regular expressions to execute.
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "regexp":{
-            "name.first": {
-                "value": "s.*y",
-                "flags" : "INTERSECTION|COMPLEMENT|EMPTY",
-                "max_determinized_states": 20000
-            }
+    "regexp":{
+        "name.first": {
+            "value": "s.*y",
+            "flags" : "INTERSECTION|COMPLEMENT|EMPTY",
+	    "max_determinized_states": 20000
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
+
 
 include::regexp-syntax.asciidoc[]
diff --git a/docs/reference/query-dsl/script-query.asciidoc b/docs/reference/query-dsl/script-query.asciidoc
index ee06a1b64bd..223460f723d 100644
--- a/docs/reference/query-dsl/script-query.asciidoc
+++ b/docs/reference/query-dsl/script-query.asciidoc
@@ -7,20 +7,17 @@ context, for example:
 
 [source,js]
 ----------------------------------------------
-GET /_search
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "script" : {
-                    "script" : "doc['num1'].value > 1"
-                }
-            }
+"bool" : {
+    "must" : {
+        ...
+    }, 
+    "filter" : {
+        "script" : {
+            "script" : "doc['num1'].value > 1"
         }
     }
 }
 ----------------------------------------------
-// CONSOLE
 
 [float]
 ==== Custom Parameters
@@ -31,23 +28,20 @@ to use the ability to pass parameters to the script itself, for example:
 
 [source,js]
 ----------------------------------------------
-GET /_search
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "script" : {
-                    "script" : {
-                        "inline" : "doc['num1'].value > param1",
-                        "params" : {
-                            "param1" : 5
-                        }
-                    }
+"bool" : {
+    "must" : {
+        ...
+    }, 
+    "filter" : {
+        "script" : {
+            "script" : {
+                "inline" : "doc['num1'].value > param1"
+                "params" : {
+                    "param1" : 5
                 }
             }
         }
     }
 }
 ----------------------------------------------
-// CONSOLE
 
diff --git a/docs/reference/query-dsl/simple-query-string-query.asciidoc b/docs/reference/query-dsl/simple-query-string-query.asciidoc
index 796f2517fea..572c0661fa4 100644
--- a/docs/reference/query-dsl/simple-query-string-query.asciidoc
+++ b/docs/reference/query-dsl/simple-query-string-query.asciidoc
@@ -8,19 +8,15 @@ an example:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-  "query": {
     "simple_query_string" : {
         "query": "\"fried eggs\" +(eggplant | potato) -frittata",
         "analyzer": "snowball",
         "fields": ["body^5","_all"],
         "default_operator": "and"
     }
-  }
 }
 --------------------------------------------------
-// CONSOLE
 
 The `simple_query_string` top level parameters include:
 
@@ -98,17 +94,13 @@ introduced fields included). For example:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "simple_query_string" : {
-            "fields" : ["content", "name.*^5"],
-            "query" : "foo bar baz"
-        }
+    "simple_query_string" : {
+        "fields" : ["content", "name.*^5"],
+        "query" : "foo bar baz"
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 [float]
 ==== Flags
@@ -118,17 +110,13 @@ should be enabled. It is specified as a `|`-delimited string with the
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "simple_query_string" : {
-            "query" : "foo | bar + baz*",
-            "flags" : "OR|AND|PREFIX"
-        }
+    "simple_query_string" : {
+        "query" : "foo | bar + baz*",
+        "flags" : "OR|AND|PREFIX"
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 The available flags are: `ALL`, `NONE`, `AND`, `OR`, `NOT`, `PREFIX`, `PHRASE`,
 `PRECEDENCE`, `ESCAPE`, `WHITESPACE`, `FUZZY`, `NEAR`, and `SLOP`.
diff --git a/docs/reference/query-dsl/span-containing-query.asciidoc b/docs/reference/query-dsl/span-containing-query.asciidoc
index 638c6999233..965bf855b6f 100644
--- a/docs/reference/query-dsl/span-containing-query.asciidoc
+++ b/docs/reference/query-dsl/span-containing-query.asciidoc
@@ -6,28 +6,24 @@ query maps to Lucene `SpanContainingQuery`. Here is an example:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "span_containing" : {
-            "little" : {
-                "span_term" : { "field1" : "foo" }
-            },
-            "big" : {
-                "span_near" : { 
-                    "clauses" : [
-                        { "span_term" : { "field1" : "bar" } },
-                        { "span_term" : { "field1" : "baz" } }
-                    ],
-                    "slop" : 5,
-                    "in_order" : true
-                }
+    "span_containing" : {
+        "little" : {
+            "span_term" : { "field1" : "foo" }
+        },
+        "big" : {
+            "span_near" : { 
+                "clauses" : [
+                    { "span_term" : { "field1" : "bar" } },
+                    { "span_term" : { "field1" : "baz" } }
+                ],
+                "slop" : 5,
+                "in_order" : true
             }
         }
     }
 }
 --------------------------------------------------
-//  CONSOLE
 
 The `big` and `little` clauses can be any span type query. Matching
 spans from `big` that contain matches from `little` are returned.
diff --git a/docs/reference/query-dsl/span-first-query.asciidoc b/docs/reference/query-dsl/span-first-query.asciidoc
index dba7932661d..74fe7ff88ba 100644
--- a/docs/reference/query-dsl/span-first-query.asciidoc
+++ b/docs/reference/query-dsl/span-first-query.asciidoc
@@ -6,19 +6,15 @@ to Lucene `SpanFirstQuery`. Here is an example:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "span_first" : {
-            "match" : {
-                "span_term" : { "user" : "kimchy" }
-            },
-            "end" : 3
-        }
+    "span_first" : {
+        "match" : {
+            "span_term" : { "user" : "kimchy" }
+        },
+        "end" : 3
     }
 }    
 --------------------------------------------------
-// CONSOLE
 
 The `match` clause can be any other span type query. The `end` controls
 the maximum end position permitted in a match.
diff --git a/docs/reference/query-dsl/span-multi-term-query.asciidoc b/docs/reference/query-dsl/span-multi-term-query.asciidoc
index 3d1672753d2..af3da5cf7dd 100644
--- a/docs/reference/query-dsl/span-multi-term-query.asciidoc
+++ b/docs/reference/query-dsl/span-multi-term-query.asciidoc
@@ -7,32 +7,24 @@ it can be nested. Example:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "span_multi":{
-            "match":{
-                "prefix" : { "user" :  { "value" : "ki" } }
-            }
+    "span_multi":{
+        "match":{
+            "prefix" : { "user" :  { "value" : "ki" } }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 A boost can also be associated with the query:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "span_multi":{
-            "match":{
-                "prefix" : { "user" :  { "value" : "ki", "boost" : 1.08 } }
-            }
+    "span_multi":{
+        "match":{
+            "prefix" : { "user" :  { "value" : "ki", "boost" : 1.08 } }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
diff --git a/docs/reference/query-dsl/span-near-query.asciidoc b/docs/reference/query-dsl/span-near-query.asciidoc
index e69be783e3d..258bc3b51ad 100644
--- a/docs/reference/query-dsl/span-near-query.asciidoc
+++ b/docs/reference/query-dsl/span-near-query.asciidoc
@@ -8,22 +8,18 @@ matches are required to be in-order. The span near query maps to Lucene
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "span_near" : {
-            "clauses" : [
-                { "span_term" : { "field" : "value1" } },
-                { "span_term" : { "field" : "value2" } },
-                { "span_term" : { "field" : "value3" } }
-            ],
-            "slop" : 12,
-            "in_order" : false
-        }
+    "span_near" : {
+        "clauses" : [
+            { "span_term" : { "field" : "value1" } },
+            { "span_term" : { "field" : "value2" } },
+            { "span_term" : { "field" : "value3" } }
+        ],
+        "slop" : 12,
+        "in_order" : false
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 The `clauses` element is a list of one or more other span type queries
 and the `slop` controls the maximum number of intervening unmatched
diff --git a/docs/reference/query-dsl/span-not-query.asciidoc b/docs/reference/query-dsl/span-not-query.asciidoc
index 5a648bd4b0e..73186985b0d 100644
--- a/docs/reference/query-dsl/span-not-query.asciidoc
+++ b/docs/reference/query-dsl/span-not-query.asciidoc
@@ -6,28 +6,24 @@ query maps to Lucene `SpanNotQuery`. Here is an example:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "span_not" : {
-            "include" : {
-                "span_term" : { "field1" : "hoya" }
-            },
-            "exclude" : {
-                "span_near" : { 
-                    "clauses" : [
-                        { "span_term" : { "field1" : "la" } },
-                        { "span_term" : { "field1" : "hoya" } }
-                    ],
-                    "slop" : 0,
-                    "in_order" : true
-                }
+    "span_not" : {
+        "include" : {
+            "span_term" : { "field1" : "hoya" }
+        },
+        "exclude" : {
+            "span_near" : { 
+                "clauses" : [
+                    { "span_term" : { "field1" : "la" } },
+                    { "span_term" : { "field1" : "hoya" } }
+                ],
+                "slop" : 0,
+                "in_order" : true
             }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 The `include` and `exclude` clauses can be any span type query. The
 `include` clause is the span query whose matches are filtered, and the
diff --git a/docs/reference/query-dsl/span-or-query.asciidoc b/docs/reference/query-dsl/span-or-query.asciidoc
index 470935d6f5c..72a4ce8724b 100644
--- a/docs/reference/query-dsl/span-or-query.asciidoc
+++ b/docs/reference/query-dsl/span-or-query.asciidoc
@@ -6,19 +6,15 @@ Matches the union of its span clauses. The span or query maps to Lucene
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "span_or" : {
-            "clauses" : [
-                { "span_term" : { "field" : "value1" } },
-                { "span_term" : { "field" : "value2" } },
-                { "span_term" : { "field" : "value3" } }
-            ]
-        }
+    "span_or" : {
+        "clauses" : [
+            { "span_term" : { "field" : "value1" } },
+            { "span_term" : { "field" : "value2" } },
+            { "span_term" : { "field" : "value3" } }
+        ]
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 The `clauses` element is a list of one or more other span type queries.
diff --git a/docs/reference/query-dsl/span-term-query.asciidoc b/docs/reference/query-dsl/span-term-query.asciidoc
index 1b12a3c35f7..9de86d48684 100644
--- a/docs/reference/query-dsl/span-term-query.asciidoc
+++ b/docs/reference/query-dsl/span-term-query.asciidoc
@@ -6,37 +6,25 @@ Matches spans containing a term. The span term query maps to Lucene
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "span_term" : { "user" : "kimchy" }
-    }
+    "span_term" : { "user" : "kimchy" }
 }    
 --------------------------------------------------
-// CONSOLE
 
 A boost can also be associated with the query:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-       "span_term" : { "user" : { "value" : "kimchy", "boost" : 2.0 } }
-    }
+    "span_term" : { "user" : { "value" : "kimchy", "boost" : 2.0 } }
 }    
 --------------------------------------------------
-// CONSOLE
 
 Or :
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "span_term" : { "user" : { "term" : "kimchy", "boost" : 2.0 } }
-    }
+    "span_term" : { "user" : { "term" : "kimchy", "boost" : 2.0 } }
 }    
 --------------------------------------------------
-// CONSOLE
diff --git a/docs/reference/query-dsl/span-within-query.asciidoc b/docs/reference/query-dsl/span-within-query.asciidoc
index b70835c4134..dc5c4bbfdfd 100644
--- a/docs/reference/query-dsl/span-within-query.asciidoc
+++ b/docs/reference/query-dsl/span-within-query.asciidoc
@@ -6,28 +6,24 @@ query maps to Lucene `SpanWithinQuery`. Here is an example:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "span_within" : {
-            "little" : {
-                "span_term" : { "field1" : "foo" }
-            },
-            "big" : {
-                "span_near" : { 
-                    "clauses" : [
-                        { "span_term" : { "field1" : "bar" } },
-                        { "span_term" : { "field1" : "baz" } }
-                    ],
-                    "slop" : 5,
-                    "in_order" : true
-                }
+    "span_within" : {
+        "little" : {
+            "span_term" : { "field1" : "foo" }
+        },
+        "big" : {
+            "span_near" : { 
+                "clauses" : [
+                    { "span_term" : { "field1" : "bar" } },
+                    { "span_term" : { "field1" : "baz" } }
+                ],
+                "slop" : 5,
+                "in_order" : true
             }
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 The `big` and `little` clauses can be any span type query. Matching
 spans from `little` that are enclosed within `big` are returned.
diff --git a/docs/reference/query-dsl/template-query.asciidoc b/docs/reference/query-dsl/template-query.asciidoc
index f66dbe56d3e..35365e6f9b2 100644
--- a/docs/reference/query-dsl/template-query.asciidoc
+++ b/docs/reference/query-dsl/template-query.asciidoc
@@ -19,8 +19,8 @@ GET /_search
         }
     }
 }
+
 ------------------------------------------
-// CONSOLE
 
 The above request is translated into:
 
@@ -34,8 +34,8 @@ GET /_search
         }
     }
 }
+
 ------------------------------------------
-// CONSOLE
 
 Alternatively passing the template as an escaped string works as well:
 
@@ -53,8 +53,6 @@ GET /_search
     }
 }
 ------------------------------------------
-// CONSOLE
-
 <1> New line characters (`\n`) should be escaped as `\\n` or removed,
     and quotes (`"`) should be escaped as `\\"`.
 
@@ -79,8 +77,6 @@ GET /_search
     }
 }
 ------------------------------------------
-// CONSOLE
-
 <1> Name of the query template in `config/scripts/`, i.e., `my_template.mustache`.
 
 Alternatively, you can register a query template in the cluster state with:
@@ -89,10 +85,9 @@ Alternatively, you can register a query template in the cluster state with:
 ------------------------------------------
 PUT /_search/template/my_template
 {
-    "template": { "match": { "text": "{{query_string}}" }}
+    "template": { "match": { "text": "{{query_string}}" }},
 }
 ------------------------------------------
-// CONSOLE
 
 and refer to it in the `template` query with the `id` parameter:
 
@@ -111,13 +106,9 @@ GET /_search
     }
 }
 ------------------------------------------
-// CONSOLE
-// TEST[continued]
-
 <1> Name of the query template in `config/scripts/`, i.e., `my_template.mustache`.
 
 
 There is also a dedicated `template` endpoint, allows you to template an entire search request.
 Please see <<search-template>> for more details.
 
-
diff --git a/docs/reference/query-dsl/terms-query.asciidoc b/docs/reference/query-dsl/terms-query.asciidoc
index e00c18bb56f..08df1e6fece 100644
--- a/docs/reference/query-dsl/terms-query.asciidoc
+++ b/docs/reference/query-dsl/terms-query.asciidoc
@@ -6,18 +6,14 @@ Filters documents that have fields that match any of the provided terms
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "constant_score" : {
-            "filter" : {
-                "terms" : { "user" : ["kimchy", "elasticsearch"]}
-            }
+    "constant_score" : {
+        "filter" : {
+            "terms" : { "user" : ["kimchy", "elasticsearch"]}
         }
     }
 }
 --------------------------------------------------
-// CONSOLE
 
 The `terms` query is also aliased with `in` as the filter name for
 simpler usage deprecated[5.0.0,use `terms` instead].
@@ -67,37 +63,33 @@ possible, reducing the need for networking.
 
 [float]
 ===== Terms lookup twitter example
-At first we index the information for user with id 2, specifically, its
-followers, than index a tweet from user with id 1. Finally we search on
-all the tweets that match the followers of user 2.
 
 [source,js]
 --------------------------------------------------
-PUT /users/user/2
-{
-    "followers" : ["1", "3"]
-}
+# index the information for user with id 2, specifically, its followers
+curl -XPUT localhost:9200/users/user/2 -d '{
+   "followers" : ["1", "3"]
+}'
 
-PUT /tweets/tweet/1
-{
-    "user" : "1"
-}
+# index a tweet, from user with id 1
+curl -XPUT localhost:9200/tweets/tweet/1 -d '{
+   "user" : "1"
+}'
 
-GET /tweets/_search
-{
-    "query" : {
-        "terms" : {
-            "user" : {
-                "index" : "users",
-                "type" : "user",
-                "id" : "2",
-                "path" : "followers"
-            }
-        }
+# search on all the tweets that match the followers of user 2
+curl -XGET localhost:9200/tweets/_search -d '{
+  "query" : {
+    "terms" : {
+      "user" : {
+        "index" : "users",
+        "type" : "user",
+        "id" : "2",
+        "path" : "followers"
+      }
     }
-}
+  }
+}'
 --------------------------------------------------
-// CONSOLE
 
 The structure of the external terms document can also include array of
 inner objects, for example:
diff --git a/docs/reference/query-dsl/type-query.asciidoc b/docs/reference/query-dsl/type-query.asciidoc
index 05e909bc366..d3ef1a30fb1 100644
--- a/docs/reference/query-dsl/type-query.asciidoc
+++ b/docs/reference/query-dsl/type-query.asciidoc
@@ -5,13 +5,9 @@ Filters documents matching the provided document / mapping type.
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "type" : {
-            "value" : "my_type"
-        }
+    "type" : {
+        "value" : "my_type"
     }
 }    
 --------------------------------------------------
-// CONSOLE
diff --git a/docs/reference/query-dsl/wildcard-query.asciidoc b/docs/reference/query-dsl/wildcard-query.asciidoc
index ad82c029d8c..d72dbec2481 100644
--- a/docs/reference/query-dsl/wildcard-query.asciidoc
+++ b/docs/reference/query-dsl/wildcard-query.asciidoc
@@ -11,40 +11,28 @@ query maps to Lucene `WildcardQuery`.
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "wildcard" : { "user" : "ki*y" }
-    }
+    "wildcard" : { "user" : "ki*y" }
 }
 --------------------------------------------------
-// CONSOLE
 
 A boost can also be associated with the query:
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "wildcard" : { "user" : { "value" : "ki*y", "boost" : 2.0 } }
-    }
+    "wildcard" : { "user" : { "value" : "ki*y", "boost" : 2.0 } }
 }
 --------------------------------------------------
-// CONSOLE
 
 Or :
 
 [source,js]
 --------------------------------------------------
-GET /_search
 {
-    "query": {
-        "wildcard" : { "user" : { "wildcard" : "ki*y", "boost" : 2.0 } }
-    }
+    "wildcard" : { "user" : { "wildcard" : "ki*y", "boost" : 2.0 } }
 }
 --------------------------------------------------
-// CONSOLE
 
 This multi term query allows to control how it gets rewritten using the
 <<query-dsl-multi-term-rewrite,rewrite>>

From 5753bcca834c6f63ca513039ecfaab8e7a6623c2 Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Tue, 17 May 2016 21:00:15 +0200
Subject: [PATCH 0302/1311] Add Console to highlighting docs

... in order to execute the snippets through rest tests.

Relates to #18160
---
 .../search/request/highlighting.asciidoc      | 90 +++++++++++++++----
 1 file changed, 74 insertions(+), 16 deletions(-)

diff --git a/docs/reference/search/request/highlighting.asciidoc b/docs/reference/search/request/highlighting.asciidoc
index 53d6e23fa80..139ea9aa825 100644
--- a/docs/reference/search/request/highlighting.asciidoc
+++ b/docs/reference/search/request/highlighting.asciidoc
@@ -8,8 +8,11 @@ The following is an example of the search request body:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query" : {...},
+    "query" : {
+        "match_all": {}
+    },
     "highlight" : {
         "fields" : {
             "content" : {}
@@ -17,6 +20,7 @@ The following is an example of the search request body:
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 In the above case, the `content` field will be highlighted for each
 search hit (there will be another element in each search hit, called
@@ -121,7 +125,7 @@ the index to be bigger):
     }
 }
 --------------------------------------------------
-
+    "match_all": {}
 ==== Force highlighter type
 
 The `type` field allows to force a specific highlighter type. This is useful
@@ -131,8 +135,11 @@ The following is an example that forces the use of the plain highlighter:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query" : {...},
+    "query" : {
+        "match_all": {}
+    },
     "highlight" : {
         "fields" : {
             "content" : {"type" : "plain"}
@@ -140,6 +147,7 @@ The following is an example that forces the use of the plain highlighter:
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 ==== Force highlighting on source
 
@@ -148,8 +156,11 @@ stored separately. Defaults to `false`.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query" : {...},
+    "query" : {
+        "match_all": {}
+    },
     "highlight" : {
         "fields" : {
             "content" : {"force_source" : true}
@@ -157,6 +168,7 @@ stored separately. Defaults to `false`.
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [[tags]]
 ==== Highlighting Tags
@@ -167,8 +179,11 @@ for example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query" : {...},
+    "query" : {
+        "match_all": {}
+    },
     "highlight" : {
         "pre_tags" : ["<tag1>"],
         "post_tags" : ["</tag1>"],
@@ -178,14 +193,18 @@ for example:
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Using the fast vector highlighter there can be more tags, and the "importance"
 is ordered.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query" : {...},
+    "query" : {
+        "match_all": {}
+    },
     "highlight" : {
         "pre_tags" : ["<tag1>", "<tag2>"],
         "post_tags" : ["</tag1>", "</tag2>"],
@@ -195,6 +214,7 @@ is ordered.
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 There are also built in "tag" schemas, with currently a single schema
 called `styled` with the following `pre_tags`:
@@ -213,8 +233,11 @@ is an example of switching tag schemas:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query" : {...},
+    "query" : {
+        "match_all": {}
+    },
     "highlight" : {
         "tags_schema" : "styled",
         "fields" : {
@@ -223,6 +246,7 @@ is an example of switching tag schemas:
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 
 ==== Encoder
@@ -240,8 +264,11 @@ For example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query" : {...},
+    "query" : {
+        "match_all": {}
+    },
     "highlight" : {
         "fields" : {
             "content" : {"fragment_size" : 150, "number_of_fragments" : 3}
@@ -249,6 +276,7 @@ For example:
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 The `fragment_size` is ignored when using the postings highlighter, as it
 outputs sentences regardless of their length.
@@ -258,8 +286,11 @@ to be sorted by score:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query" : {...},
+    "query" : {
+        "match_all": {}
+    },
     "highlight" : {
         "order" : "score",
         "fields" : {
@@ -268,6 +299,7 @@ to be sorted by score:
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 If the `number_of_fragments` value is set to `0` then no fragments are
 produced, instead the whole content of the field is returned, and of
@@ -277,8 +309,11 @@ is required. Note that `fragment_size` is ignored in this case.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query" : {...},
+    "query" : {
+        "match_all": {}
+    },
     "highlight" : {
         "fields" : {
             "_all" : {},
@@ -287,6 +322,7 @@ is required. Note that `fragment_size` is ignored in this case.
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 When using `fvh` one can use `fragment_offset`
 parameter to control the margin to start highlighting from.
@@ -302,8 +338,11 @@ greater than `0`.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query" : {...},
+    "query" : {
+        "match_all": {}
+    },
     "highlight" : {
         "fields" : {
             "content" : {
@@ -315,6 +354,7 @@ greater than `0`.
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 
 ==== Highlight query
@@ -330,6 +370,7 @@ Generally it is better to include the search query in the
 query and the rescore query in `highlight_query`.
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "fields": [ "_id" ],
     "query" : {
@@ -385,6 +426,7 @@ query and the rescore query in `highlight_query`.
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Note that the score of text fragment in this case is calculated by the Lucene
 highlighting framework. For implementation details you can check the
@@ -400,12 +442,14 @@ at the field level.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query" : {...},
+    "query" : {
+        "match_all": {}
+    },
     "highlight" : {
         "number_of_fragments" : 3,
         "fragment_size" : 150,
-        "tag_schema" : "styled",
         "fields" : {
             "_all" : { "pre_tags" : ["<em>"], "post_tags" : ["</em>"] },
             "bio.title" : { "number_of_fragments" : 0 },
@@ -415,6 +459,7 @@ at the field level.
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [[field-match]]
 ==== Require Field Match
@@ -426,14 +471,20 @@ match will be highlighted.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query" : {...},
+    "query" : {
+        "match_all": {}
+    },
     "highlight" : {
-        "require_field_match": false
-        "fields" : {...}
+        "require_field_match": false,
+        "fields": {
+                "_all" : { "pre_tags" : ["<em>"], "post_tags" : ["</em>"] }
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [[boundary-characters]]
 ==== Boundary Characters
@@ -462,6 +513,7 @@ analyzer and `content.plain` is analyzed by the `standard` analyzer.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "query": {
         "query_string": {
@@ -480,6 +532,7 @@ analyzer and `content.plain` is analyzed by the `standard` analyzer.
     }
 }
 --------------------------------------------------
+// CONSOLE
 The above matches both "run with scissors" and "running with scissors"
 and would highlight "running" and "scissors" but not "run". If both
 phrases appear in a large document then "running with scissors" is
@@ -488,6 +541,7 @@ are more matches in that fragment.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "query": {
         "query_string": {
@@ -506,12 +560,14 @@ are more matches in that fragment.
     }
 }
 --------------------------------------------------
+// CONSOLE
 The above highlights "run" as well as "running" and "scissors" but
 still sorts "running with scissors" above "run with scissors" because
 the plain match ("running") is boosted.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "query": {
         "query_string": {
@@ -530,6 +586,8 @@ the plain match ("running") is boosted.
     }
 }
 --------------------------------------------------
+// CONSOLE
+
 The above query wouldn't highlight "run" or "scissor" but shows that
 it is just fine not to list the field to which the matches are combined
 (`content`) in the matched fields.

From ff5c7965ed3797a5e8f1a6134b2bb013f6eb20ea Mon Sep 17 00:00:00 2001
From: Lee Hinman <lee@writequit.org>
Date: Wed, 30 Mar 2016 12:42:58 -0600
Subject: [PATCH 0303/1311] Don't run `mkdir` when $DATA_DIR contains a
 comma-separated list

Resolves #16992
Resolves https://github.com/elastic/cookbook-elasticsearch/issues/441
---
 .../deb/src/main/packaging/init.d/elasticsearch |  8 +++++++-
 .../packaging/scripts/70_sysv_initd.bats        | 17 +++++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/distribution/deb/src/main/packaging/init.d/elasticsearch b/distribution/deb/src/main/packaging/init.d/elasticsearch
index 82b1957b802..a01643db2b3 100755
--- a/distribution/deb/src/main/packaging/init.d/elasticsearch
+++ b/distribution/deb/src/main/packaging/init.d/elasticsearch
@@ -117,7 +117,13 @@ case "$1" in
 	fi
 
 	# Prepare environment
-	mkdir -p "$LOG_DIR" "$DATA_DIR" && chown "$ES_USER":"$ES_GROUP" "$LOG_DIR" "$DATA_DIR"
+	# Check $DATA_DIR for a comma
+	if [ "${DATA_DIR#*,}" != "$DATA_DIR" ]; then
+		# $DATA_DIR contains a comma, so we should not mkdir it
+		mkdir -p "$LOG_DIR" && chown "$ES_USER":"$ES_GROUP" "$LOG_DIR"
+	else
+		mkdir -p "$LOG_DIR" "$DATA_DIR" && chown "$ES_USER":"$ES_GROUP" "$LOG_DIR" "$DATA_DIR"
+	fi
 
 	# Ensure that the PID_DIR exists (it is cleaned at OS startup time)
 	if [ -n "$PID_DIR" ] && [ ! -e "$PID_DIR" ]; then
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/70_sysv_initd.bats b/qa/vagrant/src/test/resources/packaging/scripts/70_sysv_initd.bats
index e8a78ee19c9..fa96882f914 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/70_sysv_initd.bats
+++ b/qa/vagrant/src/test/resources/packaging/scripts/70_sysv_initd.bats
@@ -117,6 +117,23 @@ setup() {
     [ "$status" -eq 3 ] || [ "$status" -eq 4 ]
 }
 
+@test "[INIT.D] don't mkdir when it contains a comma" {
+    # Remove these just in case they exist beforehand
+    rm -rf /tmp/aoeu,/tmp/asdf
+    rm -rf /tmp/aoeu,
+    # set DATA_DIR to DATA_DIR=/tmp/aoeu,/tmp/asdf
+    sed -i 's/DATA_DIR=.*/DATA_DIR=\/tmp\/aoeu,\/tmp\/asdf/' /etc/init.d/elasticsearch
+    cat /etc/init.d/elasticsearch | grep "DATA_DIR"
+    service elasticsearch start
+    wait_for_elasticsearch_status
+    assert_file_not_exist /tmp/aoeu,/tmp/asdf
+    assert_file_not_exist /tmp/aoeu,
+    service elasticsearch stop
+    run service elasticsearch status
+    # precise returns 4, trusty 3
+    [ "$status" -eq 3 ] || [ "$status" -eq 4 ]
+}
+
 # Simulates the behavior of a system restart:
 # the PID directory is deleted by the operating system
 # but it should not block ES from starting

From ecce53f0df1c7e63fc4f192bc620c1c416627957 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 17 May 2016 16:16:39 -0400
Subject: [PATCH 0304/1311] Add I/O statistics on Linux

This commit adds a variety of real disk metrics for the block devices
that back Elasticsearch data paths. A collection of statistics are read
from /proc/diskstats and are used to report the raw metrics for
operations and read/write bytes.

Relates #15915
---
 .../org/elasticsearch/env/ESFileStore.java    |  29 +-
 .../elasticsearch/env/NodeEnvironment.java    |   8 +-
 .../elasticsearch/monitor/MonitorService.java |   8 +-
 .../org/elasticsearch/monitor/fs/FsInfo.java  | 251 +++++++++++++++++-
 .../org/elasticsearch/monitor/fs/FsProbe.java |  82 +++++-
 .../elasticsearch/monitor/fs/FsService.java   |  45 ++--
 .../elasticsearch/bootstrap/security.policy   |   3 +
 .../elasticsearch/cluster/DiskUsageTests.java |  12 +-
 .../monitor/fs/DeviceStatsTests.java          |  62 +++++
 .../monitor/fs/FsProbeTests.java              | 121 ++++++++-
 docs/reference/cluster/nodes-stats.asciidoc   |  49 ++++
 .../MockInternalClusterInfoService.java       |   2 +-
 12 files changed, 624 insertions(+), 48 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/monitor/fs/DeviceStatsTests.java

diff --git a/core/src/main/java/org/elasticsearch/env/ESFileStore.java b/core/src/main/java/org/elasticsearch/env/ESFileStore.java
index abd9d2e072f..6df242f7ce1 100644
--- a/core/src/main/java/org/elasticsearch/env/ESFileStore.java
+++ b/core/src/main/java/org/elasticsearch/env/ESFileStore.java
@@ -32,6 +32,7 @@ import java.nio.file.Path;
 import java.nio.file.attribute.FileAttributeView;
 import java.nio.file.attribute.FileStoreAttributeView;
 import java.util.Arrays;
+import java.util.List;
 
 /** 
  * Implementation of FileStore that supports
@@ -44,6 +45,8 @@ class ESFileStore extends FileStore {
     final FileStore in;
     /** Cached result of Lucene's {@code IOUtils.spins} on path. */
     final Boolean spins;
+    int majorDeviceNumber;
+    int minorDeviceNumber;
     
     @SuppressForbidden(reason = "tries to determine if disk is spinning")
     // TODO: move PathUtils to be package-private here instead of 
@@ -58,6 +61,21 @@ class ESFileStore extends FileStore {
             } catch (Exception e) {
                 spins = null;
             }
+            try {
+                final List<String> lines = Files.readAllLines(PathUtils.get("/proc/self/mountinfo"));
+                for (final String line : lines) {
+                    final String[] fields = line.trim().split("\\s+");
+                    final String mountPoint = fields[4];
+                    if (mountPoint.equals(getMountPointLinux(in))) {
+                        final String[] deviceNumbers = fields[2].split(":");
+                        majorDeviceNumber = Integer.parseInt(deviceNumbers[0]);
+                        minorDeviceNumber = Integer.parseInt(deviceNumbers[1]);
+                    }
+                }
+            } catch (Exception e) {
+                majorDeviceNumber = -1;
+                minorDeviceNumber = -1;
+            }
         } else {
             spins = null;
         }
@@ -229,10 +247,13 @@ class ESFileStore extends FileStore {
 
     @Override
     public Object getAttribute(String attribute) throws IOException {
-        if ("lucene:spins".equals(attribute)) {
-            return spins;
-        } else {
-            return in.getAttribute(attribute);
+        switch(attribute) {
+            // for the device
+            case "lucene:spins": return spins;
+            // for the partition
+            case "lucene:major_device_number": return majorDeviceNumber;
+            case "lucene:minor_device_number": return minorDeviceNumber;
+            default: return in.getAttribute(attribute);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
index c6eec09b1c8..98b47d92246 100644
--- a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
+++ b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
@@ -64,7 +64,6 @@ import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.Semaphore;
@@ -88,14 +87,21 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
          *  not running on Linux, or we hit an exception trying), True means the device possibly spins and False means it does not. */
         public final Boolean spins;
 
+        public final int majorDeviceNumber;
+        public final int minorDeviceNumber;
+
         public NodePath(Path path) throws IOException {
             this.path = path;
             this.indicesPath = path.resolve(INDICES_FOLDER);
             this.fileStore = Environment.getFileStore(path);
             if (fileStore.supportsFileAttributeView("lucene")) {
                 this.spins = (Boolean) fileStore.getAttribute("lucene:spins");
+                this.majorDeviceNumber = (int) fileStore.getAttribute("lucene:major_device_number");
+                this.minorDeviceNumber = (int) fileStore.getAttribute("lucene:minor_device_number");
             } else {
                 this.spins = null;
+                this.majorDeviceNumber = -1;
+                this.minorDeviceNumber = -1;
             }
         }
 
diff --git a/core/src/main/java/org/elasticsearch/monitor/MonitorService.java b/core/src/main/java/org/elasticsearch/monitor/MonitorService.java
index cf033e54d7c..6ae9e2a5b67 100644
--- a/core/src/main/java/org/elasticsearch/monitor/MonitorService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/MonitorService.java
@@ -31,19 +31,12 @@ import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
 
-/**
- *
- */
 public class MonitorService extends AbstractLifecycleComponent<MonitorService> {
 
     private final JvmGcMonitorService jvmGcMonitorService;
-
     private final OsService osService;
-
     private final ProcessService processService;
-
     private final JvmService jvmService;
-
     private final FsService fsService;
 
     public MonitorService(Settings settings, NodeEnvironment nodeEnvironment, ThreadPool threadPool) throws IOException {
@@ -85,4 +78,5 @@ public class MonitorService extends AbstractLifecycleComponent<MonitorService> {
     protected void doClose() {
         jvmGcMonitorService.close();
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java b/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java
index 1491971cfe2..caa97ea7387 100644
--- a/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java
+++ b/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java
@@ -188,12 +188,234 @@ public class FsInfo implements Iterable<FsInfo.Path>, Writeable, ToXContent {
         }
     }
 
+    public static class DeviceStats implements Writeable, ToXContent {
+
+        final int majorDeviceNumber;
+        final int minorDeviceNumber;
+        final String deviceName;
+        final long currentReadsCompleted;
+        final long previousReadsCompleted;
+        final long currentSectorsRead;
+        final long previousSectorsRead;
+        final long currentWritesCompleted;
+        final long previousWritesCompleted;
+        final long currentSectorsWritten;
+        final long previousSectorsWritten;
+
+        public DeviceStats(
+                final int majorDeviceNumber,
+                final int minorDeviceNumber,
+                final String deviceName,
+                final long currentReadsCompleted,
+                final long currentSectorsRead,
+                final long currentWritesCompleted,
+                final long currentSectorsWritten,
+                final DeviceStats previousDeviceStats) {
+            this(
+                    majorDeviceNumber,
+                    minorDeviceNumber,
+                    deviceName,
+                    currentReadsCompleted,
+                    previousDeviceStats != null ? previousDeviceStats.currentReadsCompleted : -1,
+                    currentSectorsWritten,
+                    previousDeviceStats != null ? previousDeviceStats.currentSectorsWritten : -1,
+                    currentSectorsRead,
+                    previousDeviceStats != null ? previousDeviceStats.currentSectorsRead : -1,
+                    currentWritesCompleted,
+                    previousDeviceStats != null ? previousDeviceStats.currentWritesCompleted : -1);
+        }
+
+        private DeviceStats(
+                final int majorDeviceNumber,
+                final int minorDeviceNumber,
+                final String deviceName,
+                final long currentReadsCompleted,
+                final long previousReadsCompleted,
+                final long currentSectorsWritten,
+                final long previousSectorsWritten,
+                final long currentSectorsRead,
+                final long previousSectorsRead,
+                final long currentWritesCompleted,
+                final long previousWritesCompleted) {
+            this.majorDeviceNumber = majorDeviceNumber;
+            this.minorDeviceNumber = minorDeviceNumber;
+            this.deviceName = deviceName;
+            this.currentReadsCompleted = currentReadsCompleted;
+            this.previousReadsCompleted = previousReadsCompleted;
+            this.currentWritesCompleted = currentWritesCompleted;
+            this.previousWritesCompleted = previousWritesCompleted;
+            this.currentSectorsRead = currentSectorsRead;
+            this.previousSectorsRead = previousSectorsRead;
+            this.currentSectorsWritten = currentSectorsWritten;
+            this.previousSectorsWritten = previousSectorsWritten;
+        }
+
+        public DeviceStats(StreamInput in) throws IOException {
+            majorDeviceNumber = in.readVInt();
+            minorDeviceNumber = in.readVInt();
+            deviceName = in.readString();
+            currentReadsCompleted = in.readLong();
+            previousReadsCompleted = in.readLong();
+            currentWritesCompleted = in.readLong();
+            previousWritesCompleted = in.readLong();
+            currentSectorsRead = in.readLong();
+            previousSectorsRead = in.readLong();
+            currentSectorsWritten = in.readLong();
+            previousSectorsWritten = in.readLong();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeVInt(majorDeviceNumber);
+            out.writeVInt(minorDeviceNumber);
+            out.writeString(deviceName);
+            out.writeLong(currentReadsCompleted);
+            out.writeLong(previousReadsCompleted);
+            out.writeLong(currentWritesCompleted);
+            out.writeLong(previousWritesCompleted);
+            out.writeLong(currentSectorsRead);
+            out.writeLong(previousSectorsRead);
+            out.writeLong(currentSectorsWritten);
+            out.writeLong(previousSectorsWritten);
+        }
+
+        public long operations() {
+            if (previousReadsCompleted == -1 || previousWritesCompleted == -1) return -1;
+
+            return (currentReadsCompleted - previousReadsCompleted) +
+                (currentWritesCompleted - previousWritesCompleted);
+        }
+
+        public long readOperations() {
+            if (previousReadsCompleted == -1) return -1;
+
+            return (currentReadsCompleted - previousReadsCompleted);
+        }
+
+        public long writeOperations() {
+            if (previousWritesCompleted == -1) return -1;
+
+            return (currentWritesCompleted - previousWritesCompleted);
+        }
+
+        public long readKilobytes() {
+            if (previousSectorsRead == -1) return -1;
+
+            return (currentSectorsRead - previousSectorsRead) / 2;
+        }
+
+        public long writeKilobytes() {
+            if (previousSectorsWritten == -1) return -1;
+
+            return (currentSectorsWritten - previousSectorsWritten) / 2;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.field("device_name", deviceName);
+            builder.field(IoStats.OPERATIONS, operations());
+            builder.field(IoStats.READ_OPERATIONS, readOperations());
+            builder.field(IoStats.WRITE_OPERATIONS, writeOperations());
+            builder.field(IoStats.READ_KILOBYTES, readKilobytes());
+            builder.field(IoStats.WRITE_KILOBYTES, writeKilobytes());
+            return builder;
+        }
+
+    }
+
+    public static class IoStats implements Writeable, ToXContent {
+
+        private static final String OPERATIONS = "operations";
+        private static final String READ_OPERATIONS = "read_operations";
+        private static final String WRITE_OPERATIONS = "write_operations";
+        private static final String READ_KILOBYTES = "read_kilobytes";
+        private static final String WRITE_KILOBYTES = "write_kilobytes";
+
+        final DeviceStats[] devicesStats;
+        final long totalOperations;
+        final long totalReadOperations;
+        final long totalWriteOperations;
+        final long totalReadKilobytes;
+        final long totalWriteKilobytes;
+
+        public IoStats(final DeviceStats[] devicesStats) {
+            this.devicesStats = devicesStats;
+            long totalOperations = 0;
+            long totalReadOperations = 0;
+            long totalWriteOperations = 0;
+            long totalReadKilobytes = 0;
+            long totalWriteKilobytes = 0;
+            for (DeviceStats deviceStats : devicesStats) {
+                totalOperations += deviceStats.operations() != -1 ? deviceStats.operations() : 0;
+                totalReadOperations += deviceStats.readOperations() != -1 ? deviceStats.readOperations() : 0;
+                totalWriteOperations += deviceStats.writeOperations() != -1 ? deviceStats.writeOperations() : 0;
+                totalReadKilobytes += deviceStats.readKilobytes() != -1 ? deviceStats.readKilobytes() : 0;
+                totalWriteKilobytes += deviceStats.writeKilobytes() != -1 ? deviceStats.writeKilobytes() : 0;
+            }
+            this.totalOperations = totalOperations;
+            this.totalReadOperations = totalReadOperations;
+            this.totalWriteOperations = totalWriteOperations;
+            this.totalReadKilobytes = totalReadKilobytes;
+            this.totalWriteKilobytes = totalWriteKilobytes;
+        }
+
+        public IoStats(StreamInput in) throws IOException {
+            final int length = in.readVInt();
+            final DeviceStats[] devicesStats = new DeviceStats[length];
+            for (int i = 0; i < length; i++) {
+                devicesStats[i] = new DeviceStats(in);
+            }
+            this.devicesStats = devicesStats;
+            this.totalOperations = in.readLong();
+            this.totalReadOperations = in.readLong();
+            this.totalWriteOperations = in.readLong();
+            this.totalReadKilobytes = in.readLong();
+            this.totalWriteKilobytes = in.readLong();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeVInt(devicesStats.length);
+            for (int i = 0; i < devicesStats.length; i++) {
+                devicesStats[i].writeTo(out);
+            }
+            out.writeLong(totalOperations);
+            out.writeLong(totalReadOperations);
+            out.writeLong(totalWriteOperations);
+            out.writeLong(totalReadKilobytes);
+            out.writeLong(totalWriteKilobytes);
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            if (devicesStats.length > 0) {
+                builder.startArray("devices");
+                for (DeviceStats deviceStats : devicesStats) {
+                    builder.startObject();
+                    deviceStats.toXContent(builder, params);
+                    builder.endObject();
+                }
+                builder.endArray();
+                builder.startObject("total");
+                builder.field(OPERATIONS, totalOperations);
+                builder.field(READ_OPERATIONS, totalReadOperations);
+                builder.field(WRITE_OPERATIONS, totalWriteOperations);
+                builder.field(READ_KILOBYTES, totalReadKilobytes);
+                builder.field(WRITE_KILOBYTES, totalWriteKilobytes);
+            }
+            return builder;
+        }
+
+    }
+
     final long timestamp;
     final Path[] paths;
+    final IoStats ioStats;
     Path total;
 
-    public FsInfo(long timestamp, Path[] paths) {
+    public FsInfo(long timestamp, IoStats ioStats, Path[] paths) {
         this.timestamp = timestamp;
+        this.ioStats = ioStats;
         this.paths = paths;
         this.total = null;
     }
@@ -203,6 +425,7 @@ public class FsInfo implements Iterable<FsInfo.Path>, Writeable, ToXContent {
      */
     public FsInfo(StreamInput in) throws IOException {
         timestamp = in.readVLong();
+        ioStats = in.readOptionalWriteable(IoStats::new);
         paths = new Path[in.readVInt()];
         for (int i = 0; i < paths.length; i++) {
             paths[i] = new Path(in);
@@ -212,6 +435,7 @@ public class FsInfo implements Iterable<FsInfo.Path>, Writeable, ToXContent {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVLong(timestamp);
+        out.writeOptionalWriteable(ioStats);
         out.writeVInt(paths.length);
         for (Path path : paths) {
             path.writeTo(out);
@@ -244,18 +468,15 @@ public class FsInfo implements Iterable<FsInfo.Path>, Writeable, ToXContent {
         return timestamp;
     }
 
+    public IoStats getIoStats() {
+        return ioStats;
+    }
+
     @Override
     public Iterator<Path> iterator() {
         return Arrays.stream(paths).iterator();
     }
 
-    static final class Fields {
-        static final String FS = "fs";
-        static final String TIMESTAMP = "timestamp";
-        static final String DATA = "data";
-        static final String TOTAL = "total";
-    }
-
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(Fields.FS);
@@ -267,7 +488,21 @@ public class FsInfo implements Iterable<FsInfo.Path>, Writeable, ToXContent {
             path.toXContent(builder, params);
         }
         builder.endArray();
+        if (ioStats != null) {
+            builder.startObject(Fields.IO_STATS);
+            ioStats.toXContent(builder, params);
+            builder.endObject();
+        }
         builder.endObject();
         return builder;
     }
+
+    static final class Fields {
+        static final String FS = "fs";
+        static final String TIMESTAMP = "timestamp";
+        static final String DATA = "data";
+        static final String TOTAL = "total";
+        static final String IO_STATS = "io_stats";
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java b/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java
index dc1958f666b..f0b61556924 100644
--- a/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java
+++ b/core/src/main/java/org/elasticsearch/monitor/fs/FsProbe.java
@@ -19,12 +19,23 @@
 
 package org.elasticsearch.monitor.fs;
 
+import org.apache.lucene.util.Constants;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.env.NodeEnvironment.NodePath;
 
 import java.io.IOException;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 public class FsProbe extends AbstractComponent {
 
@@ -35,16 +46,80 @@ public class FsProbe extends AbstractComponent {
         this.nodeEnv = nodeEnv;
     }
 
-    public FsInfo stats() throws IOException {
+    public FsInfo stats(FsInfo previous) throws IOException {
         if (!nodeEnv.hasNodeFile()) {
-            return new FsInfo(System.currentTimeMillis(), new FsInfo.Path[0]);
+            return new FsInfo(System.currentTimeMillis(), null, new FsInfo.Path[0]);
         }
         NodePath[] dataLocations = nodeEnv.nodePaths();
         FsInfo.Path[] paths = new FsInfo.Path[dataLocations.length];
         for (int i = 0; i < dataLocations.length; i++) {
             paths[i] = getFSInfo(dataLocations[i]);
         }
-        return new FsInfo(System.currentTimeMillis(), paths);
+        FsInfo.IoStats ioStats = null;
+        if (Constants.LINUX) {
+            Set<Tuple<Integer, Integer>> devicesNumbers = new HashSet<>();
+            for (int i = 0; i < dataLocations.length; i++) {
+                if (dataLocations[i].majorDeviceNumber != -1 && dataLocations[i].minorDeviceNumber != -1) {
+                    devicesNumbers.add(Tuple.tuple(dataLocations[i].majorDeviceNumber, dataLocations[i].minorDeviceNumber));
+                }
+            }
+            ioStats = ioStats(devicesNumbers, previous);
+        }
+        return new FsInfo(System.currentTimeMillis(), ioStats, paths);
+    }
+
+    final FsInfo.IoStats ioStats(final Set<Tuple<Integer, Integer>> devicesNumbers, final FsInfo previous) {
+        try {
+            final Map<Tuple<Integer, Integer>, FsInfo.DeviceStats> deviceMap = new HashMap<>();
+            if (previous != null && previous.getIoStats() != null && previous.getIoStats().devicesStats != null) {
+                for (int i = 0; i < previous.getIoStats().devicesStats.length; i++) {
+                    FsInfo.DeviceStats deviceStats = previous.getIoStats().devicesStats[i];
+                    deviceMap.put(Tuple.tuple(deviceStats.majorDeviceNumber, deviceStats.minorDeviceNumber), deviceStats);
+                }
+            }
+
+            List<FsInfo.DeviceStats> devicesStats = new ArrayList<>();
+
+            List<String> lines = readProcDiskStats();
+            if (!lines.isEmpty()) {
+                for (String line : lines) {
+                    String fields[] = line.trim().split("\\s+");
+                    final int majorDeviceNumber = Integer.parseInt(fields[0]);
+                    final int minorDeviceNumber = Integer.parseInt(fields[1]);
+                    if (!devicesNumbers.contains(Tuple.tuple(majorDeviceNumber, minorDeviceNumber))) {
+                        continue;
+                    }
+                    final String deviceName = fields[2];
+                    final long readsCompleted = Long.parseLong(fields[3]);
+                    final long sectorsRead = Long.parseLong(fields[5]);
+                    final long writesCompleted = Long.parseLong(fields[7]);
+                    final long sectorsWritten = Long.parseLong(fields[9]);
+                    final FsInfo.DeviceStats deviceStats =
+                            new FsInfo.DeviceStats(
+                                    majorDeviceNumber,
+                                    minorDeviceNumber,
+                                    deviceName,
+                                    readsCompleted,
+                                    sectorsRead,
+                                    writesCompleted,
+                                    sectorsWritten,
+                                    deviceMap.get(Tuple.tuple(majorDeviceNumber, minorDeviceNumber)));
+                    devicesStats.add(deviceStats);
+                }
+            }
+
+            return new FsInfo.IoStats(devicesStats.toArray(new FsInfo.DeviceStats[devicesStats.size()]));
+        } catch (Exception e) {
+            // do not fail Elasticsearch if something unexpected
+            // happens here
+            logger.debug("unexpected exception processing /proc/diskstats for devices {}", e, devicesNumbers);
+            return null;
+        }
+    }
+
+    @SuppressForbidden(reason = "read /proc/diskstats")
+    List<String> readProcDiskStats() throws IOException {
+        return Files.readAllLines(PathUtils.get("/proc/diskstats"));
     }
 
     public static FsInfo.Path getFSInfo(NodePath nodePath) throws IOException {
@@ -62,4 +137,5 @@ public class FsProbe extends AbstractComponent {
         fsPath.spins = nodePath.spins;
         return fsPath;
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java b/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java
index d1058652d34..e04a06fc460 100644
--- a/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.monitor.fs;
 
 import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -29,44 +30,54 @@ import org.elasticsearch.env.NodeEnvironment;
 
 import java.io.IOException;
 
-/**
- */
 public class FsService extends AbstractComponent {
 
     private final FsProbe probe;
-
-    private final SingleObjectCache<FsInfo> fsStatsCache;
+    private final TimeValue refreshInterval;
+    private final SingleObjectCache<FsInfo> cache;
 
     public final static Setting<TimeValue> REFRESH_INTERVAL_SETTING =
-        Setting.timeSetting("monitor.fs.refresh_interval", TimeValue.timeValueSeconds(1), TimeValue.timeValueSeconds(1),
+        Setting.timeSetting(
+            "monitor.fs.refresh_interval",
+            TimeValue.timeValueSeconds(1),
+            TimeValue.timeValueSeconds(1),
             Property.NodeScope);
 
-    public FsService(Settings settings, NodeEnvironment nodeEnvironment) throws IOException {
+    public FsService(final Settings settings, final NodeEnvironment nodeEnvironment) {
         super(settings);
         this.probe = new FsProbe(settings, nodeEnvironment);
-        TimeValue refreshInterval = REFRESH_INTERVAL_SETTING.get(settings);
-        fsStatsCache = new FsInfoCache(refreshInterval, probe.stats());
+        refreshInterval = REFRESH_INTERVAL_SETTING.get(settings);
         logger.debug("using refresh_interval [{}]", refreshInterval);
+        cache = new FsInfoCache(refreshInterval, stats(probe, null, logger));
     }
 
     public FsInfo stats() {
-        return fsStatsCache.getOrRefresh();
+        return cache.getOrRefresh();
+    }
+
+    private static FsInfo stats(FsProbe probe, FsInfo initialValue, ESLogger logger) {
+        try {
+            return probe.stats(initialValue);
+        } catch (IOException e) {
+            logger.debug("unexpected exception reading filesystem info", e);
+            return null;
+        }
     }
 
     private class FsInfoCache extends SingleObjectCache<FsInfo> {
-        public FsInfoCache(TimeValue interval, FsInfo initValue) {
-            super(interval, initValue);
+
+        private final FsInfo initialValue;
+
+        public FsInfoCache(TimeValue interval, FsInfo initialValue) {
+            super(interval, initialValue);
+            this.initialValue = initialValue;
         }
 
         @Override
         protected FsInfo refresh() {
-            try {
-                return probe.stats();
-            } catch (IOException ex) {
-                logger.warn("Failed to fetch fs stats - returning empty instance");
-                return new FsInfo(0, null);
-            }
+            return stats(probe, initialValue, logger);
         }
+
     }
 
 }
diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
index 8919716bfb1..74404903e91 100644
--- a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
+++ b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
@@ -124,4 +124,7 @@ grant {
 
   // read max virtual memory areas
   permission java.io.FilePermission "/proc/sys/vm/max_map_count", "read";
+
+  // io stats on Linux
+  permission java.io.FilePermission "/proc/diskstats", "read";
 };
diff --git a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
index d7738b1abcb..9210e2a56bd 100644
--- a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
@@ -201,11 +201,11 @@ public class DiskUsageTests extends ESTestCase {
         };
         List<NodeStats> nodeStats = Arrays.asList(
                 new NodeStats(new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0,
-                        null,null,null,null,null,new FsInfo(0, node1FSInfo), null,null,null,null,null, null),
+                        null,null,null,null,null,new FsInfo(0, null, node1FSInfo), null,null,null,null,null, null),
                 new NodeStats(new DiscoveryNode("node_2", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0,
-                        null,null,null,null,null, new FsInfo(0, node2FSInfo), null,null,null,null,null, null),
+                        null,null,null,null,null, new FsInfo(0, null, node2FSInfo), null,null,null,null,null, null),
                 new NodeStats(new DiscoveryNode("node_3", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0,
-                        null,null,null,null,null, new FsInfo(0, node3FSInfo), null,null,null,null,null, null)
+                        null,null,null,null,null, new FsInfo(0, null, node3FSInfo), null,null,null,null,null, null)
         );
         InternalClusterInfoService.fillDiskUsagePerNode(logger, nodeStats, newLeastAvaiableUsages, newMostAvaiableUsages);
         DiskUsage leastNode_1 = newLeastAvaiableUsages.get("node_1");
@@ -242,11 +242,11 @@ public class DiskUsageTests extends ESTestCase {
         };
         List<NodeStats> nodeStats = Arrays.asList(
                 new NodeStats(new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0,
-                        null,null,null,null,null,new FsInfo(0, node1FSInfo), null,null,null,null,null, null),
+                        null,null,null,null,null,new FsInfo(0, null, node1FSInfo), null,null,null,null,null, null),
                 new NodeStats(new DiscoveryNode("node_2", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0,
-                        null,null,null,null,null, new FsInfo(0, node2FSInfo), null,null,null,null,null, null),
+                        null,null,null,null,null, new FsInfo(0, null, node2FSInfo), null,null,null,null,null, null),
                 new NodeStats(new DiscoveryNode("node_3", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0,
-                        null,null,null,null,null, new FsInfo(0, node3FSInfo), null,null,null,null,null, null)
+                        null,null,null,null,null, new FsInfo(0, null, node3FSInfo), null,null,null,null,null, null)
         );
         InternalClusterInfoService.fillDiskUsagePerNode(logger, nodeStats, newLeastAvailableUsages, newMostAvailableUsages);
         DiskUsage leastNode_1 = newLeastAvailableUsages.get("node_1");
diff --git a/core/src/test/java/org/elasticsearch/monitor/fs/DeviceStatsTests.java b/core/src/test/java/org/elasticsearch/monitor/fs/DeviceStatsTests.java
new file mode 100644
index 00000000000..758f67ff563
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/monitor/fs/DeviceStatsTests.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.monitor.fs;
+
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class DeviceStatsTests extends ESTestCase {
+
+    public void testDeviceStats() {
+        final int majorDeviceNumber = randomIntBetween(1, 1 << 8);
+        final int minorDeviceNumber = randomIntBetween(0, 1 << 5);
+        final String deviceName = randomAsciiOfLength(3);
+        final int readsCompleted = randomIntBetween(1, 1 << 16);
+        final int sectorsRead = randomIntBetween(8 * readsCompleted, 16 * readsCompleted);
+        final int writesCompleted = randomIntBetween(1, 1 << 16);
+        final int sectorsWritten = randomIntBetween(8 * writesCompleted, 16 * writesCompleted);
+
+        FsInfo.DeviceStats previous = new FsInfo.DeviceStats(
+            majorDeviceNumber,
+            minorDeviceNumber,
+            deviceName,
+            readsCompleted,
+            sectorsRead,
+            writesCompleted,
+            sectorsWritten,
+            null);
+        FsInfo.DeviceStats current = new FsInfo.DeviceStats(
+            majorDeviceNumber,
+            minorDeviceNumber,
+            deviceName,
+            readsCompleted + 1024,
+            sectorsRead + 16384,
+            writesCompleted + 2048,
+            sectorsWritten + 32768,
+            previous);
+        assertThat(current.operations(), equalTo(1024L + 2048L));
+        assertThat(current.readOperations(), equalTo(1024L));
+        assertThat(current.writeOperations(), equalTo(2048L));
+        assertThat(current.readKilobytes(), equalTo(16384L / 2));
+        assertThat(current.writeKilobytes(), equalTo(32768L / 2));
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java b/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java
index 26a56529e05..8eba98ae0b7 100644
--- a/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java
+++ b/core/src/test/java/org/elasticsearch/monitor/fs/FsProbeTests.java
@@ -19,25 +19,55 @@
 
 package org.elasticsearch.monitor.fs;
 
+import org.apache.lucene.util.Constants;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
 
+import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.isEmptyOrNullString;
 import static org.hamcrest.Matchers.not;
 
 public class FsProbeTests extends ESTestCase {
+
     public void testFsInfo() throws IOException {
+
         try (NodeEnvironment env = newNodeEnvironment()) {
             FsProbe probe = new FsProbe(Settings.EMPTY, env);
 
-            FsInfo stats = probe.stats();
+            FsInfo stats = probe.stats(null);
             assertNotNull(stats);
             assertThat(stats.getTimestamp(), greaterThan(0L));
 
+            if (Constants.LINUX) {
+                assertNotNull(stats.getIoStats());
+                assertNotNull(stats.getIoStats().devicesStats);
+                for (int i = 0; i < stats.getIoStats().devicesStats.length; i++) {
+                    final FsInfo.DeviceStats deviceStats = stats.getIoStats().devicesStats[i];
+                    assertNotNull(deviceStats);
+                    assertThat(deviceStats.currentReadsCompleted, greaterThanOrEqualTo(0L));
+                    assertThat(deviceStats.previousReadsCompleted, equalTo(-1L));
+                    assertThat(deviceStats.currentSectorsRead, greaterThanOrEqualTo(0L));
+                    assertThat(deviceStats.previousSectorsRead, equalTo(-1L));
+                    assertThat(deviceStats.currentWritesCompleted, greaterThanOrEqualTo(0L));
+                    assertThat(deviceStats.previousWritesCompleted, equalTo(-1L));
+                    assertThat(deviceStats.currentSectorsWritten, greaterThanOrEqualTo(0L));
+                    assertThat(deviceStats.previousSectorsWritten, equalTo(-1L));
+                }
+            } else {
+                assertNull(stats.getIoStats());
+            }
+
             FsInfo.Path total = stats.getTotal();
             assertNotNull(total);
             assertThat(total.total, greaterThan(0L));
@@ -55,4 +85,93 @@ public class FsProbeTests extends ESTestCase {
             }
         }
     }
+
+    public void testIoStats() {
+        final AtomicReference<List<String>> diskStats = new AtomicReference<>();
+        diskStats.set(Arrays.asList(
+                " 259       0 nvme0n1 336609 0 7923613 82813 10264051 0 182983933 52451441 0 2970886 52536260",
+                " 259       1 nvme0n1p1 602 0 9919 131 1 0 1 0 0 19 131",
+                " 259       2 nvme0n1p2 186 0 8626 18 24 0 60 20 0 34 38",
+                " 259       3 nvme0n1p3 335733 0 7901620 82658 9592875 0 182983872 50843431 0 1737726 50926087",
+                " 253       0 dm-0 287716 0 7184666 33457 8398869 0 118857776 18730966 0 1918440 18767169",
+                " 253       1 dm-1 112 0 4624 13 0 0 0 0 0 5 13",
+                " 253       2 dm-2 47802 0 710658 49312 1371977 0 64126096 33730596 0 1058193 33781827"));
+
+        final FsProbe probe = new FsProbe(Settings.EMPTY, null) {
+            @Override
+            List<String> readProcDiskStats() throws IOException {
+                return diskStats.get();
+            }
+        };
+
+        final Set<Tuple<Integer, Integer>> devicesNumbers = new HashSet<>();
+        devicesNumbers.add(Tuple.tuple(253, 0));
+        devicesNumbers.add(Tuple.tuple(253, 2));
+        final FsInfo.IoStats first = probe.ioStats(devicesNumbers, null);
+        assertNotNull(first);
+        assertThat(first.devicesStats[0].majorDeviceNumber, equalTo(253));
+        assertThat(first.devicesStats[0].minorDeviceNumber, equalTo(0));
+        assertThat(first.devicesStats[0].deviceName, equalTo("dm-0"));
+        assertThat(first.devicesStats[0].currentReadsCompleted, equalTo(287716L));
+        assertThat(first.devicesStats[0].previousReadsCompleted, equalTo(-1L));
+        assertThat(first.devicesStats[0].currentSectorsRead, equalTo(7184666L));
+        assertThat(first.devicesStats[0].previousSectorsRead, equalTo(-1L));
+        assertThat(first.devicesStats[0].currentWritesCompleted, equalTo(8398869L));
+        assertThat(first.devicesStats[0].previousWritesCompleted, equalTo(-1L));
+        assertThat(first.devicesStats[0].currentSectorsWritten, equalTo(118857776L));
+        assertThat(first.devicesStats[0].previousSectorsWritten, equalTo(-1L));
+        assertThat(first.devicesStats[1].majorDeviceNumber, equalTo(253));
+        assertThat(first.devicesStats[1].minorDeviceNumber, equalTo(2));
+        assertThat(first.devicesStats[1].deviceName, equalTo("dm-2"));
+        assertThat(first.devicesStats[1].currentReadsCompleted, equalTo(47802L));
+        assertThat(first.devicesStats[1].previousReadsCompleted, equalTo(-1L));
+        assertThat(first.devicesStats[1].currentSectorsRead, equalTo(710658L));
+        assertThat(first.devicesStats[1].previousSectorsRead, equalTo(-1L));
+        assertThat(first.devicesStats[1].currentWritesCompleted, equalTo(1371977L));
+        assertThat(first.devicesStats[1].previousWritesCompleted, equalTo(-1L));
+        assertThat(first.devicesStats[1].currentSectorsWritten, equalTo(64126096L));
+        assertThat(first.devicesStats[1].previousSectorsWritten, equalTo(-1L));
+
+        diskStats.set(Arrays.asList(
+                " 259       0 nvme0n1 336870 0 7928397 82876 10264393 0 182986405 52451610 0 2971042 52536492",
+                " 259       1 nvme0n1p1 602 0 9919 131 1 0 1 0 0 19 131",
+                " 259       2 nvme0n1p2 186 0 8626 18 24 0 60 20 0 34 38",
+                " 259       3 nvme0n1p3 335994 0 7906404 82721 9593184 0 182986344 50843529 0 1737840 50926248",
+                " 253       0 dm-0 287734 0 7185242 33464 8398869 0 118857776 18730966 0 1918444 18767176",
+                " 253       1 dm-1 112 0 4624 13 0 0 0 0 0 5 13",
+                " 253       2 dm-2 48045 0 714866 49369 1372291 0 64128568 33730766 0 1058347 33782056"));
+
+        final FsInfo previous = new FsInfo(System.currentTimeMillis(), first, null);
+        final FsInfo.IoStats second = probe.ioStats(devicesNumbers, previous);
+        assertNotNull(second);
+        assertThat(second.devicesStats[0].majorDeviceNumber, equalTo(253));
+        assertThat(second.devicesStats[0].minorDeviceNumber, equalTo(0));
+        assertThat(second.devicesStats[0].deviceName, equalTo("dm-0"));
+        assertThat(second.devicesStats[0].currentReadsCompleted, equalTo(287734L));
+        assertThat(second.devicesStats[0].previousReadsCompleted, equalTo(287716L));
+        assertThat(second.devicesStats[0].currentSectorsRead, equalTo(7185242L));
+        assertThat(second.devicesStats[0].previousSectorsRead, equalTo(7184666L));
+        assertThat(second.devicesStats[0].currentWritesCompleted, equalTo(8398869L));
+        assertThat(second.devicesStats[0].previousWritesCompleted, equalTo(8398869L));
+        assertThat(second.devicesStats[0].currentSectorsWritten, equalTo(118857776L));
+        assertThat(second.devicesStats[0].previousSectorsWritten, equalTo(118857776L));
+        assertThat(second.devicesStats[1].majorDeviceNumber, equalTo(253));
+        assertThat(second.devicesStats[1].minorDeviceNumber, equalTo(2));
+        assertThat(second.devicesStats[1].deviceName, equalTo("dm-2"));
+        assertThat(second.devicesStats[1].currentReadsCompleted, equalTo(48045L));
+        assertThat(second.devicesStats[1].previousReadsCompleted, equalTo(47802L));
+        assertThat(second.devicesStats[1].currentSectorsRead, equalTo(714866L));
+        assertThat(second.devicesStats[1].previousSectorsRead, equalTo(710658L));
+        assertThat(second.devicesStats[1].currentWritesCompleted, equalTo(1372291L));
+        assertThat(second.devicesStats[1].previousWritesCompleted, equalTo(1371977L));
+        assertThat(second.devicesStats[1].currentSectorsWritten, equalTo(64128568L));
+        assertThat(second.devicesStats[1].previousSectorsWritten, equalTo(64126096L));
+
+        assertThat(second.totalOperations, equalTo(575L));
+        assertThat(second.totalReadOperations, equalTo(261L));
+        assertThat(second.totalWriteOperations, equalTo(314L));
+        assertThat(second.totalReadKilobytes, equalTo(2392L));
+        assertThat(second.totalWriteKilobytes, equalTo(1236L));
+    }
+
 }
diff --git a/docs/reference/cluster/nodes-stats.asciidoc b/docs/reference/cluster/nodes-stats.asciidoc
index 8a2f27112b5..e49e2f6b8aa 100644
--- a/docs/reference/cluster/nodes-stats.asciidoc
+++ b/docs/reference/cluster/nodes-stats.asciidoc
@@ -121,6 +121,55 @@ information that concern the file system:
 	`null` means we could not determine it, `true` means the device possibly spins
 	 and `false` means it does not (ex: solid-state disks).
 
+`fs.io_stats.devices` (Linux only)::
+    Array of disk metrics for each device that is backing an
+    Elasticsearch data path. These disk metrics are probed periodically
+    and averages between the last probe and the current probe are
+    computed.
+
+`fs.io_stats.devices.device_name` (Linux only)::
+    The Linux device name.
+
+`fs.io_stats.devices.operations` (Linux only)::
+    The total number of read and write operations for the device
+    completed since starting Elasticsearch.
+
+`fs.io_stats.devices.read_operations` (Linux only)::
+    The total number of read operations for the device completed since
+    starting Elasticsearch.
+
+`fs.io_stats.devices.write_operations` (Linux only)::
+    The total number of write operations for the device completed since
+    starting Elasticsearch.
+
+`fs.io_stats.devices.read_kilobytes` (Linux only)::
+    The total number of kilobytes read for the device since starting
+    Elasticsearch.
+
+`fs.io_stats.devices.write_kilobytes` (Linux only)::
+    The total number of kilobytes written for the device since
+    starting Elasticsearch.
+
+`fs.io_stats.operations` (Linux only)::
+    The total number of read and write operations across all devices
+    used by Elasticsearch completed since starting Elasticsearch.
+
+`fs.io_stats.read_operations` (Linux only)::
+    The total number of read operations for across all devices used by
+    Elasticsearch completed since starting Elasticsearch.
+
+`fs.io_stats.write_operations` (Linux only)::
+    The total number of write operations across all devices used by
+    Elasticsearch completed since starting Elasticsearch.
+
+`fs.io_stats.read_kilobytes` (Linux only)::
+    The total number of kilobytes read across all devices used by
+    Elasticsearch since starting Elasticsearch.
+
+`fs.io_stats.write_kilobytes` (Linux only)::
+    The total number of kilobytes written across all devices used by
+    Elasticsearch since starting Elasticsearch.
+
 [float]
 [[os-stats]]
 ==== Operating System statistics
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
index ef8857475b3..cf881f4cd66 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
@@ -74,7 +74,7 @@ public class MockInternalClusterInfoService extends InternalClusterInfoService {
         FsInfo.Path path = new FsInfo.Path("/dev/null", null,
             usage.getTotalBytes(), usage.getFreeBytes(), usage.getFreeBytes());
         paths[0] = path;
-        FsInfo fsInfo = new FsInfo(System.currentTimeMillis(), paths);
+        FsInfo fsInfo = new FsInfo(System.currentTimeMillis(), null, paths);
         return new NodeStats(new DiscoveryNode(nodeName, DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT),
             System.currentTimeMillis(),
             null, null, null, null, null,

From f055ad641ade5683a2b350d2bc61dd403d2daca3 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Tue, 17 May 2016 13:04:50 -0700
Subject: [PATCH 0305/1311] Made def variable casting consistent with
 invokedynamic rules.

---
 .../src/main/antlr/PainlessParser.g4          |   4 +-
 .../painless/AnalyzerCaster.java              | 441 +-----------
 .../java/org/elasticsearch/painless/Def.java  | 243 +++++--
 .../elasticsearch/painless/Definition.java    | 628 ++++++++++--------
 .../painless/node/EExplicit.java              |   3 -
 .../painless/DefOperationTests.java           |   8 +
 6 files changed, 528 insertions(+), 799 deletions(-)

diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index 4d9a81e25a0..b36dec5afbb 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -40,8 +40,8 @@ statement
     ;
 
 block
-    : LBRACK statement+ RBRACK                 # multiple
-    | statement                                # single
+    : LBRACK statement+ RBRACK # multiple
+    | statement                # single
     ;
 
 empty
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
index d7275909858..3228ff47e92 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
@@ -35,455 +35,30 @@ public final class AnalyzerCaster {
 
     public static Cast getLegalCast(final Definition definition,
                                     final String location, final Type actual, final Type expected, final boolean explicit) {
-        final Cast cast = new Cast(actual, expected);
+        final Cast cast = new Cast(actual, expected, explicit);
 
         if (actual.equals(expected)) {
             return null;
         }
 
-        if (actual.sort == Sort.DEF && expected.sort != Sort.VOID || actual.sort != Sort.VOID && expected.sort == Sort.DEF) {
-            final Transform transform = definition.transformsMap.get(cast);
+        Cast transform = definition.transformsMap.get(cast);
 
-            if (transform != null) {
-                return transform;
-            }
-
-            return cast;
+        if (transform == null && explicit) {
+            transform = definition.transformsMap.get(new Cast(actual, expected, false));
         }
 
-        switch (actual.sort) {
-            case BOOL:
-                switch (expected.sort) {
-                    case OBJECT:
-                    case BOOL_OBJ:
-                        return checkTransform(definition, location, cast);
-                }
-
-                break;
-            case BYTE:
-                switch (expected.sort) {
-                    case SHORT:
-                    case INT:
-                    case LONG:
-                    case FLOAT:
-                    case DOUBLE:
-                        return cast;
-                    case CHAR:
-                        if (explicit)
-                            return cast;
-
-                        break;
-                    case OBJECT:
-                    case NUMBER:
-                    case BYTE_OBJ:
-                    case SHORT_OBJ:
-                    case INT_OBJ:
-                    case LONG_OBJ:
-                    case FLOAT_OBJ:
-                    case DOUBLE_OBJ:
-                        return checkTransform(definition, location, cast);
-                    case CHAR_OBJ:
-                        if (explicit)
-                            return checkTransform(definition, location, cast);
-
-                        break;
-                }
-
-                break;
-            case SHORT:
-                switch (expected.sort) {
-                    case INT:
-                    case LONG:
-                    case FLOAT:
-                    case DOUBLE:
-                        return cast;
-                    case BYTE:
-                    case CHAR:
-                        if (explicit)
-                            return cast;
-
-                        break;
-                    case OBJECT:
-                    case NUMBER:
-                    case SHORT_OBJ:
-                    case INT_OBJ:
-                    case LONG_OBJ:
-                    case FLOAT_OBJ:
-                    case DOUBLE_OBJ:
-                        return checkTransform(definition, location, cast);
-                    case BYTE_OBJ:
-                    case CHAR_OBJ:
-                        if (explicit)
-                            return checkTransform(definition, location, cast);
-
-                        break;
-                }
-
-                break;
-            case CHAR:
-                switch (expected.sort) {
-                    case INT:
-                    case LONG:
-                    case FLOAT:
-                    case DOUBLE:
-                        return cast;
-                    case BYTE:
-                    case SHORT:
-                        if (explicit)
-                            return cast;
-
-                        break;
-                    case OBJECT:
-                    case NUMBER:
-                    case CHAR_OBJ:
-                    case INT_OBJ:
-                    case LONG_OBJ:
-                    case FLOAT_OBJ:
-                    case DOUBLE_OBJ:
-                        return checkTransform(definition, location, cast);
-                    case BYTE_OBJ:
-                    case SHORT_OBJ:
-                    case STRING:
-                        if (explicit)
-                            return checkTransform(definition, location, cast);
-
-                        break;
-                }
-
-                break;
-            case INT:
-                switch (expected.sort) {
-                    case LONG:
-                    case FLOAT:
-                    case DOUBLE:
-                        return cast;
-                    case BYTE:
-                    case SHORT:
-                    case CHAR:
-                        if (explicit)
-                            return cast;
-
-                        break;
-                    case OBJECT:
-                    case NUMBER:
-                    case INT_OBJ:
-                    case LONG_OBJ:
-                    case FLOAT_OBJ:
-                    case DOUBLE_OBJ:
-                        return checkTransform(definition, location, cast);
-                    case BYTE_OBJ:
-                    case SHORT_OBJ:
-                    case CHAR_OBJ:
-                        if (explicit)
-                            return checkTransform(definition, location, cast);
-
-                        break;
-                }
-
-                break;
-            case LONG:
-                switch (expected.sort) {
-                    case FLOAT:
-                    case DOUBLE:
-                        return cast;
-                    case BYTE:
-                    case SHORT:
-                    case CHAR:
-                    case INT:
-                        if (explicit)
-                            return cast;
-
-                        break;
-                    case OBJECT:
-                    case NUMBER:
-                    case LONG_OBJ:
-                    case FLOAT_OBJ:
-                    case DOUBLE_OBJ:
-                        return checkTransform(definition, location, cast);
-                    case BYTE_OBJ:
-                    case SHORT_OBJ:
-                    case CHAR_OBJ:
-                    case INT_OBJ:
-                        if (explicit)
-                            return checkTransform(definition, location, cast);
-
-                        break;
-                }
-
-                break;
-            case FLOAT:
-                switch (expected.sort) {
-                    case DOUBLE:
-                        return cast;
-                    case BYTE:
-                    case SHORT:
-                    case CHAR:
-                    case INT:
-                    case LONG:
-                        if (explicit)
-                            return cast;
-
-                        break;
-                    case OBJECT:
-                    case NUMBER:
-                    case FLOAT_OBJ:
-                    case DOUBLE_OBJ:
-                        return checkTransform(definition, location, cast);
-                    case BYTE_OBJ:
-                    case SHORT_OBJ:
-                    case CHAR_OBJ:
-                    case INT_OBJ:
-                    case LONG_OBJ:
-                        if (explicit)
-                            return checkTransform(definition, location, cast);
-
-                        break;
-                }
-
-                break;
-            case DOUBLE:
-                switch (expected.sort) {
-                    case BYTE:
-                    case SHORT:
-                    case CHAR:
-                    case INT:
-                    case LONG:
-                    case FLOAT:
-                        if (explicit)
-                            return cast;
-
-                        break;
-                    case OBJECT:
-                    case NUMBER:
-                    case DOUBLE_OBJ:
-                        return checkTransform(definition, location, cast);
-                    case BYTE_OBJ:
-                    case SHORT_OBJ:
-                    case CHAR_OBJ:
-                    case INT_OBJ:
-                    case LONG_OBJ:
-                    case FLOAT_OBJ:
-                        if (explicit)
-                            return checkTransform(definition, location, cast);
-
-                        break;
-                }
-
-                break;
-            case OBJECT:
-            case NUMBER:
-                switch (expected.sort) {
-                    case BYTE:
-                    case SHORT:
-                    case CHAR:
-                    case INT:
-                    case LONG:
-                    case FLOAT:
-                    case DOUBLE:
-                        if (explicit)
-                            return checkTransform(definition, location, cast);
-
-                        break;
-                }
-
-                break;
-            case BOOL_OBJ:
-                switch (expected.sort) {
-                    case BOOL:
-                        return checkTransform(definition, location, cast);
-                }
-
-                break;
-            case BYTE_OBJ:
-                switch (expected.sort) {
-                    case BYTE:
-                    case SHORT:
-                    case INT:
-                    case LONG:
-                    case FLOAT:
-                    case DOUBLE:
-                    case SHORT_OBJ:
-                    case INT_OBJ:
-                    case LONG_OBJ:
-                    case FLOAT_OBJ:
-                    case DOUBLE_OBJ:
-                        return checkTransform(definition, location, cast);
-                    case CHAR:
-                    case CHAR_OBJ:
-                        if (explicit)
-                            return checkTransform(definition, location, cast);
-
-                        break;
-                }
-
-                break;
-            case SHORT_OBJ:
-                switch (expected.sort) {
-                    case SHORT:
-                    case INT:
-                    case LONG:
-                    case FLOAT:
-                    case DOUBLE:
-                    case INT_OBJ:
-                    case LONG_OBJ:
-                    case FLOAT_OBJ:
-                    case DOUBLE_OBJ:
-                        return checkTransform(definition, location, cast);
-                    case BYTE:
-                    case CHAR:
-                    case BYTE_OBJ:
-                    case CHAR_OBJ:
-                        if (explicit)
-                            return checkTransform(definition, location, cast);
-
-                        break;
-                }
-
-                break;
-            case CHAR_OBJ:
-                switch (expected.sort) {
-                    case CHAR:
-                    case INT:
-                    case LONG:
-                    case FLOAT:
-                    case DOUBLE:
-                    case INT_OBJ:
-                    case LONG_OBJ:
-                    case FLOAT_OBJ:
-                    case DOUBLE_OBJ:
-                        return checkTransform(definition, location, cast);
-                    case BYTE:
-                    case SHORT:
-                    case BYTE_OBJ:
-                    case SHORT_OBJ:
-                    case STRING:
-                        if (explicit)
-                            return checkTransform(definition, location, cast);
-
-                        break;
-                }
-
-                break;
-            case INT_OBJ:
-                switch (expected.sort) {
-                    case INT:
-                    case LONG:
-                    case FLOAT:
-                    case DOUBLE:
-                    case LONG_OBJ:
-                    case FLOAT_OBJ:
-                    case DOUBLE_OBJ:
-                        return checkTransform(definition, location, cast);
-                    case BYTE:
-                    case SHORT:
-                    case CHAR:
-                    case BYTE_OBJ:
-                    case SHORT_OBJ:
-                    case CHAR_OBJ:
-                        if (explicit)
-                            return checkTransform(definition, location, cast);
-
-                        break;
-                }
-
-                break;
-            case LONG_OBJ:
-                switch (expected.sort) {
-                    case LONG:
-                    case FLOAT:
-                    case DOUBLE:
-                    case FLOAT_OBJ:
-                    case DOUBLE_OBJ:
-                        return checkTransform(definition, location, cast);
-                    case BYTE:
-                    case SHORT:
-                    case CHAR:
-                    case INT:
-                    case BYTE_OBJ:
-                    case SHORT_OBJ:
-                    case CHAR_OBJ:
-                    case INT_OBJ:
-                        if (explicit)
-                            return checkTransform(definition, location, cast);
-
-                        break;
-                }
-
-                break;
-            case FLOAT_OBJ:
-                switch (expected.sort) {
-                    case FLOAT:
-                    case DOUBLE:
-                    case DOUBLE_OBJ:
-                        return checkTransform(definition, location, cast);
-                    case BYTE:
-                    case SHORT:
-                    case CHAR:
-                    case INT:
-                    case LONG:
-                    case BYTE_OBJ:
-                    case SHORT_OBJ:
-                    case CHAR_OBJ:
-                    case INT_OBJ:
-                    case LONG_OBJ:
-                        if (explicit)
-                            return checkTransform(definition, location, cast);
-
-                        break;
-                }
-
-                break;
-            case DOUBLE_OBJ:
-                switch (expected.sort) {
-                    case DOUBLE:
-                        return checkTransform(definition, location, cast);
-                    case BYTE:
-                    case SHORT:
-                    case CHAR:
-                    case INT:
-                    case LONG:
-                    case FLOAT:
-                    case BYTE_OBJ:
-                    case SHORT_OBJ:
-                    case CHAR_OBJ:
-                    case INT_OBJ:
-                    case LONG_OBJ:
-                    case FLOAT_OBJ:
-                        if (explicit)
-                            return checkTransform(definition, location, cast);
-
-                        break;
-                }
-
-                break;
-            case STRING:
-                switch (expected.sort) {
-                    case CHAR:
-                    case CHAR_OBJ:
-                        if (explicit)
-                            return checkTransform(definition, location, cast);
-
-                        break;
-                }
+        if (transform != null) {
+            return transform;
         }
 
-        if (expected.clazz.isAssignableFrom(actual.clazz) || (explicit && actual.clazz.isAssignableFrom(expected.clazz))) {
+        if (expected.clazz.isAssignableFrom(actual.clazz) ||
+            ((explicit || expected.sort == Sort.DEF) && actual.clazz.isAssignableFrom(expected.clazz))) {
             return cast;
         } else {
             throw new ClassCastException("Error" + location + ": Cannot cast from [" + actual.name + "] to [" + expected.name + "].");
         }
     }
 
-    private static Transform checkTransform(final Definition definition, final String location, final Cast cast) {
-        final Transform transform = definition.transformsMap.get(cast);
-
-        if (transform == null) {
-            throw new ClassCastException("Error" + location + ": Cannot cast from [" + cast.from.name + "] to [" + cast.to.name + "].");
-        }
-
-        return transform;
-    }
-
     public static Object constCast(final String location, final Object constant, final Cast cast) {
         if (cast instanceof Transform) {
             final Transform transform = (Transform)cast;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index d9b2e02a237..9226fc3f098 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -145,17 +145,17 @@ public final class Def {
          // check whitelist for matching method
          for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
              RuntimeClass struct = definition.runtimeMap.get(clazz);
-             
+
              if (struct != null) {
                  Method method = struct.methods.get(key);
                  if (method != null) {
                      return method.handle;
                  }
              }
-             
+
              for (final Class<?> iface : clazz.getInterfaces()) {
                  struct = definition.runtimeMap.get(iface);
-                 
+
                  if (struct != null) {
                      Method method = struct.methods.get(key);
                      if (method != null) {
@@ -164,7 +164,7 @@ public final class Def {
                  }
              }
          }
-         
+
          // no matching methods in whitelist found
          throw new IllegalArgumentException("Unable to find dynamic method [" + name + "] with signature [" + type + "] " +
                  "for class [" + receiverClass.getCanonicalName() + "].");
@@ -971,103 +971,218 @@ public final class Def {
 
     // Conversion methods for Def to primitive types.
 
-    public static boolean DefToboolean(final Object value) {
-        if (value instanceof Boolean) {
-            return (boolean)value;
-        } else if (value instanceof Character) {
-            return ((char)value) != 0;
+    public static byte DefTobyteImplicit(final Object value) {
+        return (byte)value;
+    }
+
+    public static short DefToshortImplicit(final Object value) {
+        if (value instanceof Byte) {
+            return (byte)value;
         } else {
-            return ((Number)value).intValue() != 0;
+            return (short)value;
         }
     }
 
-    public static byte DefTobyte(final Object value) {
-        if (value instanceof Boolean) {
-            return ((Boolean)value) ? (byte)1 : 0;
+    public static char DefTocharImplicit(final Object value) {
+        if (value instanceof Byte) {
+            return (char)(byte)value;
+        } else {
+            return (char)value;
+        }
+    }
+
+    public static int DefTointImplicit(final Object value) {
+        if (value instanceof Byte) {
+            return (byte)value;
+        } else if (value instanceof Short) {
+            return (short)value;
         } else if (value instanceof Character) {
+            return (char)value;
+        } else {
+            return (int)value;
+        }
+    }
+
+    public static long DefTolongImplicit(final Object value) {
+        if (value instanceof Byte) {
+            return (byte)value;
+        } else if (value instanceof Short) {
+            return (short)value;
+        } else if (value instanceof Character) {
+            return (char)value;
+        } else if (value instanceof Integer) {
+            return (int)value;
+        } else {
+            return (long)value;
+        }
+    }
+
+    public static float DefTofloatImplicit(final Object value) {
+        if (value instanceof Byte) {
+            return (byte)value;
+        } else if (value instanceof Short) {
+            return (short)value;
+        } else if (value instanceof Character) {
+            return (char)value;
+        } else if (value instanceof Integer) {
+            return (int)value;
+        } else if (value instanceof Long) {
+            return (long)value;
+        } else {
+            return (float)value;
+        }
+    }
+
+    public static double DefTodoubleImplicit(final Object value) {
+        if (value instanceof Byte) {
+            return (byte)value;
+        } else if (value instanceof Short) {
+            return (short)value;
+        } else if (value instanceof Character) {
+            return (char)value;
+        } else if (value instanceof Integer) {
+            return (int)value;
+        } else if (value instanceof Long) {
+            return (long)value;
+        } else if (value instanceof Float) {
+            return (float)value;
+        } else {
+            return (double)value;
+        }
+    }
+
+    public static Byte DefToByteImplicit(final Object value) {
+        return (Byte)value;
+    }
+
+    public static Short DefToShortImplicit(final Object value) {
+        if (value == null) {
+            return null;
+        } else if (value instanceof Byte) {
+            return ((Byte)value).shortValue();
+        } else {
+            return (Short)value;
+        }
+    }
+
+    public static Character DefToCharacterImplicit(final Object value) {
+        if (value == null) {
+            return null;
+        } else if (value instanceof Byte) {
+            return (char)(byte)value;
+        } else {
+            return (Character)value;
+        }
+    }
+
+    public static Integer DefToIntegerImplicit(final Object value) {
+        if (value == null) {
+            return null;
+        } else if (value instanceof Byte || value instanceof Short) {
+            return ((Number)value).intValue();
+        } else if (value instanceof Character) {
+            return (int)(char)value;
+        } else {
+            return (Integer)value;
+        }
+    }
+
+    public static Long DefToLongImplicit(final Object value) {
+        if (value == null) {
+            return null;
+        } else if (value instanceof Byte || value instanceof Short || value instanceof Integer) {
+            return ((Number)value).longValue();
+        } else if (value instanceof Character) {
+            return (long)(char)value;
+        } else {
+            return (Long)value;
+        }
+    }
+
+    public static Float DefToFloatImplicit(final Object value) {
+        if (value == null) {
+            return null;
+        } else if (value instanceof Byte || value instanceof Short || value instanceof Integer || value instanceof Long) {
+            return ((Number)value).floatValue();
+        } else if (value instanceof Character) {
+            return (float)(char)value;
+        } else {
+            return (Float)value;
+        }
+    }
+
+    public static Double DefToDoubleImplicit(final Object value) {
+        if (value == null) {
+            return null;
+        } else if (value instanceof Byte || value instanceof Short ||
+            value instanceof Integer || value instanceof Long || value instanceof  Float) {
+            return ((Number)value).doubleValue();
+        } else if (value instanceof Character) {
+            return (double)(char)value;
+        } else {
+            return (Double)value;
+        }
+    }
+
+    public static byte DefTobyteExplicit(final Object value) {
+        if (value instanceof Character) {
             return (byte)(char)value;
         } else {
             return ((Number)value).byteValue();
         }
     }
 
-    public static short DefToshort(final Object value) {
-        if (value instanceof Boolean) {
-            return ((Boolean)value) ? (short)1 : 0;
-        } else if (value instanceof Character) {
+    public static short DefToshortExplicit(final Object value) {
+        if (value instanceof Character) {
             return (short)(char)value;
         } else {
             return ((Number)value).shortValue();
         }
     }
 
-    public static char DefTochar(final Object value) {
-        if (value instanceof Boolean) {
-            return ((Boolean)value) ? (char)1 : 0;
-        } else if (value instanceof Character) {
+    public static char DefTocharExplicit(final Object value) {
+        if (value instanceof Character) {
             return ((Character)value);
         } else {
             return (char)((Number)value).intValue();
         }
     }
 
-    public static int DefToint(final Object value) {
-        if (value instanceof Boolean) {
-            return ((Boolean)value) ? 1 : 0;
-        } else if (value instanceof Character) {
+    public static int DefTointExplicit(final Object value) {
+        if (value instanceof Character) {
             return (char)value;
         } else {
             return ((Number)value).intValue();
         }
     }
 
-    public static long DefTolong(final Object value) {
-        if (value instanceof Boolean) {
-            return ((Boolean)value) ? 1L : 0;
-        } else if (value instanceof Character) {
+    public static long DefTolongExplicit(final Object value) {
+        if (value instanceof Character) {
             return (char)value;
         } else {
             return ((Number)value).longValue();
         }
     }
 
-    public static float DefTofloat(final Object value) {
-        if (value instanceof Boolean) {
-            return ((Boolean)value) ? (float)1 : 0;
-        } else if (value instanceof Character) {
+    public static float DefTofloatExplicit(final Object value) {
+        if (value instanceof Character) {
             return (char)value;
         } else {
             return ((Number)value).floatValue();
         }
     }
 
-    public static double DefTodouble(final Object value) {
-        if (value instanceof Boolean) {
-            return ((Boolean)value) ? (double)1 : 0;
-        } else if (value instanceof Character) {
+    public static double DefTodoubleExplicit(final Object value) {
+        if (value instanceof Character) {
             return (char)value;
         } else {
             return ((Number)value).doubleValue();
         }
     }
 
-    public static Boolean DefToBoolean(final Object value) {
+    public static Byte DefToByteExplicit(final Object value) {
         if (value == null) {
             return null;
-        } else if (value instanceof Boolean) {
-            return (boolean)value;
-        } else if (value instanceof Character) {
-            return ((char)value) != 0;
-        } else {
-            return ((Number)value).intValue() != 0;
-        }
-    }
-
-    public static Byte DefToByte(final Object value) {
-        if (value == null) {
-            return null;
-        } else if (value instanceof Boolean) {
-            return ((Boolean)value) ? (byte)1 : 0;
         } else if (value instanceof Character) {
             return (byte)(char)value;
         } else {
@@ -1075,11 +1190,9 @@ public final class Def {
         }
     }
 
-    public static Short DefToShort(final Object value) {
+    public static Short DefToShortExplicit(final Object value) {
         if (value == null) {
             return null;
-        } else if (value instanceof Boolean) {
-            return ((Boolean)value) ? (short)1 : 0;
         } else if (value instanceof Character) {
             return (short)(char)value;
         } else {
@@ -1087,11 +1200,9 @@ public final class Def {
         }
     }
 
-    public static Character DefToCharacter(final Object value) {
+    public static Character DefToCharacterExplicit(final Object value) {
         if (value == null) {
             return null;
-        } else if (value instanceof Boolean) {
-            return ((Boolean)value) ? (char)1 : 0;
         } else if (value instanceof Character) {
             return ((Character)value);
         } else {
@@ -1099,11 +1210,9 @@ public final class Def {
         }
     }
 
-    public static Integer DefToInteger(final Object value) {
+    public static Integer DefToIntegerExplicit(final Object value) {
         if (value == null) {
             return null;
-        } else if (value instanceof Boolean) {
-            return ((Boolean)value) ? 1 : 0;
         } else if (value instanceof Character) {
             return (int)(char)value;
         } else {
@@ -1111,11 +1220,9 @@ public final class Def {
         }
     }
 
-    public static Long DefToLong(final Object value) {
+    public static Long DefToLongExplicit(final Object value) {
         if (value == null) {
             return null;
-        } else if (value instanceof Boolean) {
-            return ((Boolean)value) ? 1L : 0;
         } else if (value instanceof Character) {
             return (long)(char)value;
         } else {
@@ -1123,11 +1230,9 @@ public final class Def {
         }
     }
 
-    public static Float DefToFloat(final Object value) {
+    public static Float DefToFloatExplicit(final Object value) {
         if (value == null) {
             return null;
-        } else if (value instanceof Boolean) {
-            return ((Boolean)value) ? (float)1 : 0;
         } else if (value instanceof Character) {
             return (float)(char)value;
         } else {
@@ -1135,11 +1240,9 @@ public final class Def {
         }
     }
 
-    public static Double DefToDouble(final Object value) {
+    public static Double DefToDoubleExplicit(final Object value) {
         if (value == null) {
             return null;
-        } else if (value instanceof Boolean) {
-            return ((Boolean)value) ? (double)1 : 0;
         } else if (value instanceof Character) {
             return (double)(char)value;
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 40dc13a3368..f44db5cb504 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -195,7 +195,7 @@ public final class Definition {
             this.setter = setter;
         }
     }
-    
+
     // TODO: instead of hashing on this, we could have a 'next' pointer in Method itself, but it would make code more complex
     // please do *NOT* under any circumstances change this to be the crappy Tuple from elasticsearch!
     /**
@@ -210,7 +210,7 @@ public final class Definition {
     public static final class MethodKey {
         public final String name;
         public final int arity;
-        
+
         /**
          * Create a new lookup key
          * @param name name of the method
@@ -313,10 +313,12 @@ public final class Definition {
     public static class Cast {
         public final Type from;
         public final Type to;
+        public final boolean explicit;
 
-        public Cast(final Type from, final Type to) {
+        public Cast(final Type from, final Type to, final boolean explicit) {
             this.from = from;
             this.to = to;
+            this.explicit = explicit;
         }
 
         @Override
@@ -331,28 +333,27 @@ public final class Definition {
 
             final Cast cast = (Cast)object;
 
-            return from.equals(cast.from) && to.equals(cast.to);
+            return from.equals(cast.from) && to.equals(cast.to) && explicit == cast.explicit;
         }
 
         @Override
         public int hashCode() {
             int result = from.hashCode();
             result = 31 * result + to.hashCode();
+            result = 31 * result + (explicit ? 1 : 0);
 
             return result;
         }
     }
 
     public static final class Transform extends Cast {
-        public final Cast cast;
         public final Method method;
         public final Type upcast;
         public final Type downcast;
 
         public Transform(final Cast cast, Method method, final Type upcast, final Type downcast) {
-            super(cast.from, cast.to);
+            super(cast.from, cast.to, cast.explicit);
 
-            this.cast = cast;
             this.method = method;
             this.upcast = upcast;
             this.downcast = downcast;
@@ -373,7 +374,7 @@ public final class Definition {
     }
 
     public final Map<String, Struct> structsMap;
-    public final Map<Cast, Transform> transformsMap;
+    public final Map<Cast, Cast> transformsMap;
     public final Map<Class<?>, RuntimeClass> runtimeMap;
 
     public final Type voidType;
@@ -451,7 +452,7 @@ public final class Definition {
     public final Type longsType;
     public final Type doublesType;
     public final Type geoPointsType;
-    
+
     // for testing features not currently "used" by the whitelist (we should not rush the API for that!)
     public final Type featureTestType;
 
@@ -535,7 +536,7 @@ public final class Definition {
         longsType = getType("Longs");
         doublesType = getType("Doubles");
         geoPointsType = getType("GeoPoints");
-        
+
         featureTestType = getType("FeatureTest");
 
         addElements();
@@ -628,7 +629,7 @@ public final class Definition {
         this.longsType = definition.longsType;
         this.doublesType = definition.doublesType;
         this.geoPointsType = definition.geoPointsType;
-        
+
         this.featureTestType = definition.featureTestType;
     }
 
@@ -706,7 +707,7 @@ public final class Definition {
         addStruct( "Longs"     , ScriptDocValues.Longs.class);
         addStruct( "Doubles"   , ScriptDocValues.Doubles.class);
         addStruct( "GeoPoints" , ScriptDocValues.GeoPoints.class);
-        
+
         addStruct( "FeatureTest", FeatureTest.class);
     }
 
@@ -983,22 +984,34 @@ public final class Definition {
         addField("Math", "E", null, true, doubleType, null);
         addField("Math", "PI", null, true, doubleType, null);
 
-        addMethod("Def", "DefToboolean", null, true, booleanType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefTobyte", null, true, byteType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToshort", null, true, shortType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefTochar", null, true, charType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToint", null, true, intType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefTolong", null, true, longType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefTofloat", null, true, floatType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefTodouble", null, true, doubleType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToBoolean", null, true, booleanobjType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToByte", null, true, byteobjType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToShort", null, true, shortobjType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToCharacter", null, true, charobjType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToInteger", null, true, intobjType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToLong", null, true, longobjType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToFloat", null, true, floatobjType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToDouble", null, true, doubleobjType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefTobyteImplicit", null, true, byteType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefToshortImplicit", null, true, shortType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefTocharImplicit", null, true, charType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefTointImplicit", null, true, intType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefTolongImplicit", null, true, longType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefTofloatImplicit", null, true, floatType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefTodoubleImplicit", null, true, doubleType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefToByteImplicit", null, true, byteobjType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefToShortImplicit", null, true, shortobjType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefToCharacterImplicit", null, true, charobjType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefToIntegerImplicit", null, true, intobjType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefToLongImplicit", null, true, longobjType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefToFloatImplicit", null, true, floatobjType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefToDoubleImplicit", null, true, doubleobjType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefTobyteExplicit", null, true, byteType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefToshortExplicit", null, true, shortType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefTocharExplicit", null, true, charType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefTointExplicit", null, true, intType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefTolongExplicit", null, true, longType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefTofloatExplicit", null, true, floatType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefTodoubleExplicit", null, true, doubleType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefToByteExplicit", null, true, byteobjType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefToShortExplicit", null, true, shortobjType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefToCharacterExplicit", null, true, charobjType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefToIntegerExplicit", null, true, intobjType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefToLongExplicit", null, true, longobjType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefToFloatExplicit", null, true, floatobjType, new Type[] {defType}, null, null);
+        addMethod("Def", "DefToDoubleExplicit", null, true, doubleobjType, new Type[] {defType}, null, null);
 
         addMethod("Iterator", "hasNext", null, false, booleanType, new Type[] {}, null, null);
         addMethod("Iterator", "next", null, false, objectType, new Type[] {}, defType, null);
@@ -1178,7 +1191,7 @@ public final class Definition {
                   new Type[] { stringType }, null, null);
         addMethod("GeoPoints", "geohashDistanceInMiles", null, false, doubleType,
                   new Type[] { stringType }, null, null);
-        
+
         // currently FeatureTest exposes overloaded constructor, field load store, and overloaded static methods
         addConstructor("FeatureTest", "new", new Type[] {}, null);
         addConstructor("FeatureTest", "new", new Type[] {intType, intType}, null);
@@ -1241,283 +1254,293 @@ public final class Definition {
         copyStruct("Longs", "List", "Collection", "Object");
         copyStruct("Doubles", "List", "Collection", "Object");
         copyStruct("GeoPoints", "List", "Collection", "Object");
-        
+
         copyStruct("FeatureTest", "Object");
     }
 
     private void addTransforms() {
-        addTransform(booleanType, byteType, "Utility", "booleanTobyte", true);
-        addTransform(booleanType, shortType, "Utility", "booleanToshort", true);
-        addTransform(booleanType, charType, "Utility", "booleanTochar", true);
-        addTransform(booleanType, intType, "Utility", "booleanToint", true);
-        addTransform(booleanType, longType, "Utility", "booleanTolong", true);
-        addTransform(booleanType, floatType, "Utility", "booleanTofloat", true);
-        addTransform(booleanType, doubleType, "Utility", "booleanTodouble", true);
-        addTransform(booleanType, objectType, "Boolean", "valueOf", true);
-        addTransform(booleanType, defType, "Boolean", "valueOf", true);
-        addTransform(booleanType, numberType, "Utility", "booleanToInteger", true);
-        addTransform(booleanType, booleanobjType, "Boolean", "valueOf", true);
+        addTransform(booleanType, objectType, "Boolean", "valueOf", true, false);
+        addTransform(booleanType, defType, "Boolean", "valueOf", true, false);
+        addTransform(booleanType, booleanobjType, "Boolean", "valueOf", true, false);
 
-        addTransform(byteType, booleanType, "Utility", "byteToboolean", true);
-        addTransform(byteType, objectType, "Byte", "valueOf", true);
-        addTransform(byteType, defType, "Byte", "valueOf", true);
-        addTransform(byteType, numberType, "Byte", "valueOf", true);
-        addTransform(byteType, byteobjType, "Byte", "valueOf", true);
-        addTransform(byteType, shortobjType, "Utility", "byteToShort", true);
-        addTransform(byteType, charobjType, "Utility", "byteToCharacter", true);
-        addTransform(byteType, intobjType, "Utility", "byteToInteger", true);
-        addTransform(byteType, longobjType, "Utility", "byteToLong", true);
-        addTransform(byteType, floatobjType, "Utility", "byteToFloat", true);
-        addTransform(byteType, doubleobjType, "Utility", "byteToDouble", true);
+        addTransform(byteType, shortType, false);
+        addTransform(byteType, charType, true);
+        addTransform(byteType, intType, false);
+        addTransform(byteType, longType, false);
+        addTransform(byteType, floatType, false);
+        addTransform(byteType, doubleType, false);
+        addTransform(byteType, objectType, "Byte", "valueOf", true, false);
+        addTransform(byteType, defType, "Byte", "valueOf", true, false);
+        addTransform(byteType, numberType, "Byte", "valueOf", true, false);
+        addTransform(byteType, byteobjType, "Byte", "valueOf", true, false);
+        addTransform(byteType, shortobjType, "Utility", "byteToShort", true, false);
+        addTransform(byteType, charobjType, "Utility", "byteToCharacter", true, true);
+        addTransform(byteType, intobjType, "Utility", "byteToInteger", true, false);
+        addTransform(byteType, longobjType, "Utility", "byteToLong", true, false);
+        addTransform(byteType, floatobjType, "Utility", "byteToFloat", true, false);
+        addTransform(byteType, doubleobjType, "Utility", "byteToDouble", true, false);
 
-        addTransform(shortType, booleanType, "Utility", "shortToboolean", true);
-        addTransform(shortType, objectType, "Short", "valueOf", true);
-        addTransform(shortType, defType, "Short", "valueOf", true);
-        addTransform(shortType, numberType, "Short", "valueOf", true);
-        addTransform(shortType, byteobjType, "Utility", "shortToByte", true);
-        addTransform(shortType, shortobjType, "Short", "valueOf", true);
-        addTransform(shortType, charobjType, "Utility", "shortToCharacter", true);
-        addTransform(shortType, intobjType, "Utility", "shortToInteger", true);
-        addTransform(shortType, longobjType, "Utility", "shortToLong", true);
-        addTransform(shortType, floatobjType, "Utility", "shortToFloat", true);
-        addTransform(shortType, doubleobjType, "Utility", "shortToDouble", true);
+        addTransform(shortType, byteType, true);
+        addTransform(shortType, charType, true);
+        addTransform(shortType, intType, false);
+        addTransform(shortType, longType, false);
+        addTransform(shortType, floatType, false);
+        addTransform(shortType, doubleType, false);
+        addTransform(shortType, objectType, "Short", "valueOf", true, false);
+        addTransform(shortType, defType, "Short", "valueOf", true, false);
+        addTransform(shortType, numberType, "Short", "valueOf", true, false);
+        addTransform(shortType, byteobjType, "Utility", "shortToByte", true, true);
+        addTransform(shortType, shortobjType, "Short", "valueOf", true, false);
+        addTransform(shortType, charobjType, "Utility", "shortToCharacter", true, true);
+        addTransform(shortType, intobjType, "Utility", "shortToInteger", true, false);
+        addTransform(shortType, longobjType, "Utility", "shortToLong", true, false);
+        addTransform(shortType, floatobjType, "Utility", "shortToFloat", true, false);
+        addTransform(shortType, doubleobjType, "Utility", "shortToDouble", true, false);
 
-        addTransform(charType, booleanType, "Utility", "charToboolean", true);
-        addTransform(charType, objectType, "Character", "valueOf", true);
-        addTransform(charType, defType, "Character", "valueOf", true);
-        addTransform(charType, numberType, "Utility", "charToInteger", true);
-        addTransform(charType, byteobjType, "Utility", "charToByte", true);
-        addTransform(charType, shortobjType, "Utility", "charToShort", true);
-        addTransform(charType, charobjType, "Character", "valueOf", true);
-        addTransform(charType, intobjType, "Utility", "charToInteger", true);
-        addTransform(charType, longobjType, "Utility", "charToLong", true);
-        addTransform(charType, floatobjType, "Utility", "charToFloat", true);
-        addTransform(charType, doubleobjType, "Utility", "charToDouble", true);
-        addTransform(charType, stringType, "Utility", "charToString", true);
+        addTransform(charType, byteType, true);
+        addTransform(charType, shortType, true);
+        addTransform(charType, intType, false);
+        addTransform(charType, longType, false);
+        addTransform(charType, floatType, false);
+        addTransform(charType, doubleType, false);
+        addTransform(charType, objectType, "Character", "valueOf", true, false);
+        addTransform(charType, defType, "Character", "valueOf", true, false);
+        addTransform(charType, numberType, "Utility", "charToInteger", true, false);
+        addTransform(charType, byteobjType, "Utility", "charToByte", true, true);
+        addTransform(charType, shortobjType, "Utility", "charToShort", true, true);
+        addTransform(charType, charobjType, "Character", "valueOf", true, false);
+        addTransform(charType, intobjType, "Utility", "charToInteger", true, false);
+        addTransform(charType, longobjType, "Utility", "charToLong", true, false);
+        addTransform(charType, floatobjType, "Utility", "charToFloat", true, false);
+        addTransform(charType, doubleobjType, "Utility", "charToDouble", true, false);
+        addTransform(charType, stringType, "Utility", "charToString", true, true);
 
-        addTransform(intType, booleanType, "Utility", "intToboolean", true);
-        addTransform(intType, objectType, "Integer", "valueOf", true);
-        addTransform(intType, defType, "Integer", "valueOf", true);
-        addTransform(intType, numberType, "Integer", "valueOf", true);
-        addTransform(intType, byteobjType, "Utility", "intToByte", true);
-        addTransform(intType, shortobjType, "Utility", "intToShort", true);
-        addTransform(intType, charobjType, "Utility", "intToCharacter", true);
-        addTransform(intType, intobjType, "Integer", "valueOf", true);
-        addTransform(intType, longobjType, "Utility", "intToLong", true);
-        addTransform(intType, floatobjType, "Utility", "intToFloat", true);
-        addTransform(intType, doubleobjType, "Utility", "intToDouble", true);
+        addTransform(intType, byteType, true);
+        addTransform(intType, shortType, true);
+        addTransform(intType, charType, true);
+        addTransform(intType, longType, false);
+        addTransform(intType, floatType, false);
+        addTransform(intType, doubleType, false);
+        addTransform(intType, objectType, "Integer", "valueOf", true, false);
+        addTransform(intType, defType, "Integer", "valueOf", true, false);
+        addTransform(intType, numberType, "Integer", "valueOf", true, false);
+        addTransform(intType, byteobjType, "Utility", "intToByte", true, true);
+        addTransform(intType, shortobjType, "Utility", "intToShort", true, true);
+        addTransform(intType, charobjType, "Utility", "intToCharacter", true, true);
+        addTransform(intType, intobjType, "Integer", "valueOf", true, false);
+        addTransform(intType, longobjType, "Utility", "intToLong", true, false);
+        addTransform(intType, floatobjType, "Utility", "intToFloat", true, false);
+        addTransform(intType, doubleobjType, "Utility", "intToDouble", true, false);
 
-        addTransform(longType, booleanType, "Utility", "longToboolean", true);
-        addTransform(longType, objectType, "Long", "valueOf", true);
-        addTransform(longType, defType, "Long", "valueOf", true);
-        addTransform(longType, numberType, "Long", "valueOf", true);
-        addTransform(longType, byteobjType, "Utility", "longToByte", true);
-        addTransform(longType, shortobjType, "Utility", "longToShort", true);
-        addTransform(longType, charobjType, "Utility", "longToCharacter", true);
-        addTransform(longType, intobjType, "Utility", "longToInteger", true);
-        addTransform(longType, longobjType, "Long", "valueOf", true);
-        addTransform(longType, floatobjType, "Utility", "longToFloat", true);
-        addTransform(longType, doubleobjType, "Utility", "longToDouble", true);
+        addTransform(longType, byteType, true);
+        addTransform(longType, shortType, true);
+        addTransform(longType, charType, true);
+        addTransform(longType, intType, false);
+        addTransform(longType, floatType, false);
+        addTransform(longType, doubleType, false);
+        addTransform(longType, objectType, "Long", "valueOf", true, false);
+        addTransform(longType, defType, "Long", "valueOf", true, false);
+        addTransform(longType, numberType, "Long", "valueOf", true, false);
+        addTransform(longType, byteobjType, "Utility", "longToByte", true, true);
+        addTransform(longType, shortobjType, "Utility", "longToShort", true, true);
+        addTransform(longType, charobjType, "Utility", "longToCharacter", true, true);
+        addTransform(longType, intobjType, "Utility", "longToInteger", true, true);
+        addTransform(longType, longobjType, "Long", "valueOf", true, false);
+        addTransform(longType, floatobjType, "Utility", "longToFloat", true, false);
+        addTransform(longType, doubleobjType, "Utility", "longToDouble", true, false);
 
-        addTransform(floatType, booleanType, "Utility", "floatToboolean", true);
-        addTransform(floatType, objectType, "Float", "valueOf", true);
-        addTransform(floatType, defType, "Float", "valueOf", true);
-        addTransform(floatType, numberType, "Float", "valueOf", true);
-        addTransform(floatType, byteobjType, "Utility", "floatToByte", true);
-        addTransform(floatType, shortobjType, "Utility", "floatToShort", true);
-        addTransform(floatType, charobjType, "Utility", "floatToCharacter", true);
-        addTransform(floatType, intobjType, "Utility", "floatToInteger", true);
-        addTransform(floatType, longobjType, "Utility", "floatToLong", true);
-        addTransform(floatType, floatobjType, "Float", "valueOf", true);
-        addTransform(floatType, doubleobjType, "Utility", "floatToDouble", true);
+        addTransform(floatType, byteType, true);
+        addTransform(floatType, shortType, true);
+        addTransform(floatType, charType, true);
+        addTransform(floatType, intType, true);
+        addTransform(floatType, longType, false);
+        addTransform(floatType, doubleType, false);
+        addTransform(floatType, objectType, "Float", "valueOf", true, false);
+        addTransform(floatType, defType, "Float", "valueOf", true, false);
+        addTransform(floatType, numberType, "Float", "valueOf", true, false);
+        addTransform(floatType, byteobjType, "Utility", "floatToByte", true, true);
+        addTransform(floatType, shortobjType, "Utility", "floatToShort", true, true);
+        addTransform(floatType, charobjType, "Utility", "floatToCharacter", true, true);
+        addTransform(floatType, intobjType, "Utility", "floatToInteger", true, true);
+        addTransform(floatType, longobjType, "Utility", "floatToLong", true, true);
+        addTransform(floatType, floatobjType, "Float", "valueOf", true, false);
+        addTransform(floatType, doubleobjType, "Utility", "floatToDouble", true, false);
 
-        addTransform(doubleType, booleanType, "Utility", "doubleToboolean", true);
-        addTransform(doubleType, objectType, "Double", "valueOf", true);
-        addTransform(doubleType, defType, "Double", "valueOf", true);
-        addTransform(doubleType, numberType, "Double", "valueOf", true);
-        addTransform(doubleType, byteobjType, "Utility", "doubleToByte", true);
-        addTransform(doubleType, shortobjType, "Utility", "doubleToShort", true);
-        addTransform(doubleType, charobjType, "Utility", "doubleToCharacter", true);
-        addTransform(doubleType, intobjType, "Utility", "doubleToInteger", true);
-        addTransform(doubleType, longobjType, "Utility", "doubleToLong", true);
-        addTransform(doubleType, floatobjType, "Utility", "doubleToFloat", true);
-        addTransform(doubleType, doubleobjType, "Double", "valueOf", true);
+        addTransform(doubleType, byteType, true);
+        addTransform(doubleType, shortType, true);
+        addTransform(doubleType, charType, true);
+        addTransform(doubleType, intType, true);
+        addTransform(doubleType, longType, true);
+        addTransform(doubleType, floatType, false);
+        addTransform(doubleType, objectType, "Double", "valueOf", true, false);
+        addTransform(doubleType, defType, "Double", "valueOf", true, false);
+        addTransform(doubleType, numberType, "Double", "valueOf", true, false);
+        addTransform(doubleType, byteobjType, "Utility", "doubleToByte", true, true);
+        addTransform(doubleType, shortobjType, "Utility", "doubleToShort", true, true);
+        addTransform(doubleType, charobjType, "Utility", "doubleToCharacter", true, true);
+        addTransform(doubleType, intobjType, "Utility", "doubleToInteger", true, true);
+        addTransform(doubleType, longobjType, "Utility", "doubleToLong", true, true);
+        addTransform(doubleType, floatobjType, "Utility", "doubleToFloat", true, true);
+        addTransform(doubleType, doubleobjType, "Double", "valueOf", true, false);
 
-        addTransform(objectType, booleanType, "Boolean", "booleanValue", false);
-        addTransform(objectType, byteType, "Number", "byteValue", false);
-        addTransform(objectType, shortType, "Number", "shortValue", false);
-        addTransform(objectType, charType, "Character", "charValue", false);
-        addTransform(objectType, intType, "Number", "intValue", false);
-        addTransform(objectType, longType, "Number", "longValue", false);
-        addTransform(objectType, floatType, "Number", "floatValue", false);
-        addTransform(objectType, doubleType, "Number", "doubleValue", false);
+        addTransform(objectType, booleanType, "Boolean", "booleanValue", false, true);
+        addTransform(objectType, byteType, "Number", "byteValue", false, true);
+        addTransform(objectType, shortType, "Number", "shortValue", false, true);
+        addTransform(objectType, charType, "Character", "charValue", false, true);
+        addTransform(objectType, intType, "Number", "intValue", false, true);
+        addTransform(objectType, longType, "Number", "longValue", false, true);
+        addTransform(objectType, floatType, "Number", "floatValue", false, true);
+        addTransform(objectType, doubleType, "Number", "doubleValue", false, true);
 
-        addTransform(defType, booleanType, "Def", "DefToboolean", true);
-        addTransform(defType, byteType, "Def", "DefTobyte", true);
-        addTransform(defType, shortType, "Def", "DefToshort", true);
-        addTransform(defType, charType, "Def", "DefTochar", true);
-        addTransform(defType, intType, "Def", "DefToint", true);
-        addTransform(defType, longType, "Def", "DefTolong", true);
-        addTransform(defType, floatType, "Def", "DefTofloat", true);
-        addTransform(defType, doubleType, "Def", "DefTodouble", true);
-        addTransform(defType, booleanobjType, "Def", "DefToBoolean", true);
-        addTransform(defType, byteobjType, "Def", "DefToByte", true);
-        addTransform(defType, shortobjType, "Def", "DefToShort", true);
-        addTransform(defType, charobjType, "Def", "DefToCharacter", true);
-        addTransform(defType, intobjType, "Def", "DefToInteger", true);
-        addTransform(defType, longobjType, "Def", "DefToLong", true);
-        addTransform(defType, floatobjType, "Def", "DefToFloat", true);
-        addTransform(defType, doubleobjType, "Def", "DefToDouble", true);
+        addTransform(defType, booleanType, "Boolean", "booleanValue", false, false);
+        addTransform(defType, byteType, "Def", "DefTobyteImplicit", true, false);
+        addTransform(defType, shortType, "Def", "DefToshortImplicit", true, false);
+        addTransform(defType, charType, "Def", "DefTocharImplicit", true, false);
+        addTransform(defType, intType, "Def", "DefTointImplicit", true, false);
+        addTransform(defType, longType, "Def", "DefTolongImplicit", true, false);
+        addTransform(defType, floatType, "Def", "DefTofloatImplicit", true, false);
+        addTransform(defType, doubleType, "Def", "DefTodoubleImplicit", true, false);
+        addTransform(defType, byteobjType, "Def", "DefToByteImplicit", true, false);
+        addTransform(defType, shortobjType, "Def", "DefToShortImplicit", true, false);
+        addTransform(defType, charobjType, "Def", "DefToCharacterImplicit", true, false);
+        addTransform(defType, intobjType, "Def", "DefToIntegerImplicit", true, false);
+        addTransform(defType, longobjType, "Def", "DefToLongImplicit", true, false);
+        addTransform(defType, floatobjType, "Def", "DefToFloatImplicit", true, false);
+        addTransform(defType, doubleobjType, "Def", "DefToDoubleImplicit", true, false);
+        addTransform(defType, byteType, "Def", "DefTobyteExplicit", true, true);
+        addTransform(defType, shortType, "Def", "DefToshortExplicit", true, true);
+        addTransform(defType, charType, "Def", "DefTocharExplicit", true, true);
+        addTransform(defType, intType, "Def", "DefTointExplicit", true, true);
+        addTransform(defType, longType, "Def", "DefTolongExplicit", true, true);
+        addTransform(defType, floatType, "Def", "DefTofloatExplicit", true, true);
+        addTransform(defType, doubleType, "Def", "DefTodoubleExplicit", true, true);
+        addTransform(defType, byteobjType, "Def", "DefToByteExplicit", true, true);
+        addTransform(defType, shortobjType, "Def", "DefToShortExplicit", true, true);
+        addTransform(defType, charobjType, "Def", "DefToCharacterExplicit", true, true);
+        addTransform(defType, intobjType, "Def", "DefToIntegerExplicit", true, true);
+        addTransform(defType, longobjType, "Def", "DefToLongExplicit", true, true);
+        addTransform(defType, floatobjType, "Def", "DefToFloatExplicit", true, true);
+        addTransform(defType, doubleobjType, "Def", "DefToDoubleExplicit", true, true);
 
-        addTransform(numberType, booleanType, "Utility", "NumberToboolean", true);
-        addTransform(numberType, byteType, "Number", "byteValue", false);
-        addTransform(numberType, shortType, "Number", "shortValue", false);
-        addTransform(numberType, charType, "Utility", "NumberTochar", true);
-        addTransform(numberType, intType, "Number", "intValue", false);
-        addTransform(numberType, longType, "Number", "longValue", false);
-        addTransform(numberType, floatType, "Number", "floatValue", false);
-        addTransform(numberType, doubleType, "Number", "doubleValue", false);
-        addTransform(numberType, booleanobjType, "Utility", "NumberToBoolean", true);
-        addTransform(numberType, byteobjType, "Utility", "NumberToByte", true);
-        addTransform(numberType, shortobjType, "Utility", "NumberToShort", true);
-        addTransform(numberType, charobjType, "Utility", "NumberToCharacter", true);
-        addTransform(numberType, intobjType, "Utility", "NumberToInteger", true);
-        addTransform(numberType, longobjType, "Utility", "NumberToLong", true);
-        addTransform(numberType, floatobjType, "Utility", "NumberToFloat", true);
-        addTransform(numberType, doubleobjType, "Utility", "NumberToDouble", true);
+        addTransform(numberType, byteType, "Number", "byteValue", false, true);
+        addTransform(numberType, shortType, "Number", "shortValue", false, true);
+        addTransform(numberType, charType, "Utility", "NumberTochar", true, true);
+        addTransform(numberType, intType, "Number", "intValue", false, true);
+        addTransform(numberType, longType, "Number", "longValue", false, true);
+        addTransform(numberType, floatType, "Number", "floatValue", false, true);
+        addTransform(numberType, doubleType, "Number", "doubleValue", false, true);
+        addTransform(numberType, booleanobjType, "Utility", "NumberToBoolean", true, true);
+        addTransform(numberType, byteobjType, "Utility", "NumberToByte", true, true);
+        addTransform(numberType, shortobjType, "Utility", "NumberToShort", true, true);
+        addTransform(numberType, charobjType, "Utility", "NumberToCharacter", true, true);
+        addTransform(numberType, intobjType, "Utility", "NumberToInteger", true, true);
+        addTransform(numberType, longobjType, "Utility", "NumberToLong", true, true);
+        addTransform(numberType, floatobjType, "Utility", "NumberToFloat", true, true);
+        addTransform(numberType, doubleobjType, "Utility", "NumberToDouble", true, true);
 
-        addTransform(booleanobjType, booleanType, "Boolean", "booleanValue", false);
-        addTransform(booleanobjType, byteType, "Utility", "BooleanTobyte", true);
-        addTransform(booleanobjType, shortType, "Utility", "BooleanToshort", true);
-        addTransform(booleanobjType, charType, "Utility", "BooleanTochar", true);
-        addTransform(booleanobjType, intType, "Utility", "BooleanToint", true);
-        addTransform(booleanobjType, longType, "Utility", "BooleanTolong", true);
-        addTransform(booleanobjType, floatType, "Utility", "BooleanTofloat", true);
-        addTransform(booleanobjType, doubleType, "Utility", "BooleanTodouble", true);
-        addTransform(booleanobjType, numberType, "Utility", "BooleanToLong", true);
-        addTransform(booleanobjType, byteobjType, "Utility", "BooleanToByte", true);
-        addTransform(booleanobjType, shortobjType, "Utility", "BooleanToShort", true);
-        addTransform(booleanobjType, charobjType, "Utility", "BooleanToCharacter", true);
-        addTransform(booleanobjType, intobjType, "Utility", "BooleanToInteger", true);
-        addTransform(booleanobjType, longobjType, "Utility", "BooleanToLong", true);
-        addTransform(booleanobjType, floatobjType, "Utility", "BooleanToFloat", true);
-        addTransform(booleanobjType, doubleobjType, "Utility", "BooleanToDouble", true);
+        addTransform(booleanobjType, booleanType, "Boolean", "booleanValue", false, false);
 
-        addTransform(byteobjType, booleanType, "Utility", "ByteToboolean", true);
-        addTransform(byteobjType, byteType, "Byte", "byteValue", false);
-        addTransform(byteobjType, shortType, "Byte", "shortValue", false);
-        addTransform(byteobjType, charType, "Utility", "ByteTochar", true);
-        addTransform(byteobjType, intType, "Byte", "intValue", false);
-        addTransform(byteobjType, longType, "Byte", "longValue", false);
-        addTransform(byteobjType, floatType, "Byte", "floatValue", false);
-        addTransform(byteobjType, doubleType, "Byte", "doubleValue", false);
-        addTransform(byteobjType, booleanobjType, "Utility", "NumberToBoolean", true);
-        addTransform(byteobjType, shortobjType, "Utility", "NumberToShort", true);
-        addTransform(byteobjType, charobjType, "Utility", "NumberToCharacter", true);
-        addTransform(byteobjType, intobjType, "Utility", "NumberToInteger", true);
-        addTransform(byteobjType, longobjType, "Utility", "NumberToLong", true);
-        addTransform(byteobjType, floatobjType, "Utility", "NumberToFloat", true);
-        addTransform(byteobjType, doubleobjType, "Utility", "NumberToDouble", true);
+        addTransform(byteobjType, byteType, "Byte", "byteValue", false, false);
+        addTransform(byteobjType, shortType, "Byte", "shortValue", false, false);
+        addTransform(byteobjType, charType, "Utility", "ByteTochar", true, false);
+        addTransform(byteobjType, intType, "Byte", "intValue", false, false);
+        addTransform(byteobjType, longType, "Byte", "longValue", false, false);
+        addTransform(byteobjType, floatType, "Byte", "floatValue", false, false);
+        addTransform(byteobjType, doubleType, "Byte", "doubleValue", false, false);
+        addTransform(byteobjType, shortobjType, "Utility", "NumberToShort", true, false);
+        addTransform(byteobjType, charobjType, "Utility", "NumberToCharacter", true, false);
+        addTransform(byteobjType, intobjType, "Utility", "NumberToInteger", true, false);
+        addTransform(byteobjType, longobjType, "Utility", "NumberToLong", true, false);
+        addTransform(byteobjType, floatobjType, "Utility", "NumberToFloat", true, false);
+        addTransform(byteobjType, doubleobjType, "Utility", "NumberToDouble", true, false);
 
-        addTransform(shortobjType, booleanType, "Utility", "ShortToboolean", true);
-        addTransform(shortobjType, byteType, "Short", "byteValue", false);
-        addTransform(shortobjType, shortType, "Short", "shortValue", false);
-        addTransform(shortobjType, charType, "Utility", "ShortTochar", true);
-        addTransform(shortobjType, intType, "Short", "intValue", false);
-        addTransform(shortobjType, longType, "Short", "longValue", false);
-        addTransform(shortobjType, floatType, "Short", "floatValue", false);
-        addTransform(shortobjType, doubleType, "Short", "doubleValue", false);
-        addTransform(shortobjType, booleanobjType, "Utility", "NumberToBoolean", true);
-        addTransform(shortobjType, byteobjType, "Utility", "NumberToByte", true);
-        addTransform(shortobjType, charobjType, "Utility", "NumberToCharacter", true);
-        addTransform(shortobjType, intobjType, "Utility", "NumberToInteger", true);
-        addTransform(shortobjType, longobjType, "Utility", "NumberToLong", true);
-        addTransform(shortobjType, floatobjType, "Utility", "NumberToFloat", true);
-        addTransform(shortobjType, doubleobjType, "Utility", "NumberToDouble", true);
+        addTransform(shortobjType, byteType, "Short", "byteValue", false, true);
+        addTransform(shortobjType, shortType, "Short", "shortValue", false, true);
+        addTransform(shortobjType, charType, "Utility", "ShortTochar", true, false);
+        addTransform(shortobjType, intType, "Short", "intValue", false, false);
+        addTransform(shortobjType, longType, "Short", "longValue", false, false);
+        addTransform(shortobjType, floatType, "Short", "floatValue", false, false);
+        addTransform(shortobjType, doubleType, "Short", "doubleValue", false, false);
+        addTransform(shortobjType, byteobjType, "Utility", "NumberToByte", true, true);
+        addTransform(shortobjType, charobjType, "Utility", "NumberToCharacter", true, true);
+        addTransform(shortobjType, intobjType, "Utility", "NumberToInteger", true, false);
+        addTransform(shortobjType, longobjType, "Utility", "NumberToLong", true, false);
+        addTransform(shortobjType, floatobjType, "Utility", "NumberToFloat", true, false);
+        addTransform(shortobjType, doubleobjType, "Utility", "NumberToDouble", true, false);
 
-        addTransform(charobjType, booleanType, "Utility", "CharacterToboolean", true);
-        addTransform(charobjType, byteType, "Utility", "CharacterTobyte", true);
-        addTransform(charobjType, shortType, "Utility", "CharacterToshort", true);
-        addTransform(charobjType, charType, "Character", "charValue", false);
-        addTransform(charobjType, intType, "Utility", "CharacterToint", true);
-        addTransform(charobjType, longType, "Utility", "CharacterTolong", true);
-        addTransform(charobjType, floatType, "Utility", "CharacterTofloat", true);
-        addTransform(charobjType, doubleType, "Utility", "CharacterTodouble", true);
-        addTransform(charobjType, booleanobjType, "Utility", "CharacterToBoolean", true);
-        addTransform(charobjType, byteobjType, "Utility", "CharacterToByte", true);
-        addTransform(charobjType, shortobjType, "Utility", "CharacterToShort", true);
-        addTransform(charobjType, intobjType, "Utility", "CharacterToInteger", true);
-        addTransform(charobjType, longobjType, "Utility", "CharacterToLong", true);
-        addTransform(charobjType, floatobjType, "Utility", "CharacterToFloat", true);
-        addTransform(charobjType, doubleobjType, "Utility", "CharacterToDouble", true);
-        addTransform(charobjType, stringType, "Utility", "CharacterToString", true);
+        addTransform(charobjType, byteType, "Utility", "CharacterTobyte", true, true);
+        addTransform(charobjType, shortType, "Utility", "CharacterToshort", true, false);
+        addTransform(charobjType, charType, "Character", "charValue", false, true);
+        addTransform(charobjType, intType, "Utility", "CharacterToint", true, false);
+        addTransform(charobjType, longType, "Utility", "CharacterTolong", true, false);
+        addTransform(charobjType, floatType, "Utility", "CharacterTofloat", true, false);
+        addTransform(charobjType, doubleType, "Utility", "CharacterTodouble", true, false);
+        addTransform(charobjType, byteobjType, "Utility", "CharacterToByte", true, true);
+        addTransform(charobjType, shortobjType, "Utility", "CharacterToShort", true, true);
+        addTransform(charobjType, intobjType, "Utility", "CharacterToInteger", true, false);
+        addTransform(charobjType, longobjType, "Utility", "CharacterToLong", true, false);
+        addTransform(charobjType, floatobjType, "Utility", "CharacterToFloat", true, false);
+        addTransform(charobjType, doubleobjType, "Utility", "CharacterToDouble", true, false);
+        addTransform(charobjType, stringType, "Utility", "CharacterToString", true, true);
 
-        addTransform(intobjType, booleanType, "Utility", "IntegerToboolean", true);
-        addTransform(intobjType, byteType, "Integer", "byteValue", false);
-        addTransform(intobjType, shortType, "Integer", "shortValue", false);
-        addTransform(intobjType, charType, "Utility", "IntegerTochar", true);
-        addTransform(intobjType, intType, "Integer", "intValue", false);
-        addTransform(intobjType, longType, "Integer", "longValue", false);
-        addTransform(intobjType, floatType, "Integer", "floatValue", false);
-        addTransform(intobjType, doubleType, "Integer", "doubleValue", false);
-        addTransform(intobjType, booleanobjType, "Utility", "NumberToBoolean", true);
-        addTransform(intobjType, byteobjType, "Utility", "NumberToByte", true);
-        addTransform(intobjType, shortobjType, "Utility", "NumberToShort", true);
-        addTransform(intobjType, charobjType, "Utility", "NumberToCharacter", true);
-        addTransform(intobjType, longobjType, "Utility", "NumberToLong", true);
-        addTransform(intobjType, floatobjType, "Utility", "NumberToFloat", true);
-        addTransform(intobjType, doubleobjType, "Utility", "NumberToDouble", true);
+        addTransform(intobjType, byteType, "Integer", "byteValue", false, true);
+        addTransform(intobjType, shortType, "Integer", "shortValue", false, true);
+        addTransform(intobjType, charType, "Utility", "IntegerTochar", true, true);
+        addTransform(intobjType, intType, "Integer", "intValue", false, false);
+        addTransform(intobjType, longType, "Integer", "longValue", false, false);
+        addTransform(intobjType, floatType, "Integer", "floatValue", false, false);
+        addTransform(intobjType, doubleType, "Integer", "doubleValue", false, false);
+        addTransform(intobjType, byteobjType, "Utility", "NumberToByte", true, true);
+        addTransform(intobjType, shortobjType, "Utility", "NumberToShort", true, true);
+        addTransform(intobjType, charobjType, "Utility", "NumberToCharacter", true, true);
+        addTransform(intobjType, longobjType, "Utility", "NumberToLong", true, false);
+        addTransform(intobjType, floatobjType, "Utility", "NumberToFloat", true, false);
+        addTransform(intobjType, doubleobjType, "Utility", "NumberToDouble", true, false);
 
-        addTransform(longobjType, booleanType, "Utility", "LongToboolean", true);
-        addTransform(longobjType, byteType, "Long", "byteValue", false);
-        addTransform(longobjType, shortType, "Long", "shortValue", false);
-        addTransform(longobjType, charType, "Utility", "LongTochar", true);
-        addTransform(longobjType, intType, "Long", "intValue", false);
-        addTransform(longobjType, longType, "Long", "longValue", false);
-        addTransform(longobjType, floatType, "Long", "floatValue", false);
-        addTransform(longobjType, doubleType, "Long", "doubleValue", false);
-        addTransform(longobjType, booleanobjType, "Utility", "NumberToBoolean", true);
-        addTransform(longobjType, byteobjType, "Utility", "NumberToByte", true);
-        addTransform(longobjType, shortobjType, "Utility", "NumberToShort", true);
-        addTransform(longobjType, charobjType, "Utility", "NumberToCharacter", true);
-        addTransform(longobjType, intobjType, "Utility", "NumberToInteger", true);
-        addTransform(longobjType, floatobjType, "Utility", "NumberToFloat", true);
-        addTransform(longobjType, doubleobjType, "Utility", "NumberToDouble", true);
+        addTransform(longobjType, byteType, "Long", "byteValue", false, true);
+        addTransform(longobjType, shortType, "Long", "shortValue", false, true);
+        addTransform(longobjType, charType, "Utility", "LongTochar", true, true);
+        addTransform(longobjType, intType, "Long", "intValue", false, true);
+        addTransform(longobjType, longType, "Long", "longValue", false, false);
+        addTransform(longobjType, floatType, "Long", "floatValue", false, false);
+        addTransform(longobjType, doubleType, "Long", "doubleValue", false, false);
+        addTransform(longobjType, byteobjType, "Utility", "NumberToByte", true, true);
+        addTransform(longobjType, shortobjType, "Utility", "NumberToShort", true, true);
+        addTransform(longobjType, charobjType, "Utility", "NumberToCharacter", true, true);
+        addTransform(longobjType, intobjType, "Utility", "NumberToInteger", true, true);
+        addTransform(longobjType, floatobjType, "Utility", "NumberToFloat", true, false);
+        addTransform(longobjType, doubleobjType, "Utility", "NumberToDouble", true, false);
 
-        addTransform(floatobjType, booleanType, "Utility", "FloatToboolean", true);
-        addTransform(floatobjType, byteType, "Float", "byteValue", false);
-        addTransform(floatobjType, shortType, "Float", "shortValue", false);
-        addTransform(floatobjType, charType, "Utility", "FloatTochar", true);
-        addTransform(floatobjType, intType, "Float", "intValue", false);
-        addTransform(floatobjType, longType, "Float", "longValue", false);
-        addTransform(floatobjType, floatType, "Float", "floatValue", false);
-        addTransform(floatobjType, doubleType, "Float", "doubleValue", false);
-        addTransform(floatobjType, booleanobjType, "Utility", "NumberToBoolean", true);
-        addTransform(floatobjType, byteobjType, "Utility", "NumberToByte", true);
-        addTransform(floatobjType, shortobjType, "Utility", "NumberToShort", true);
-        addTransform(floatobjType, charobjType, "Utility", "NumberToCharacter", true);
-        addTransform(floatobjType, intobjType, "Utility", "NumberToInteger", true);
-        addTransform(floatobjType, longobjType, "Utility", "NumberToLong", true);
-        addTransform(floatobjType, doubleobjType, "Utility", "NumberToDouble", true);
+        addTransform(floatobjType, byteType, "Float", "byteValue", false, true);
+        addTransform(floatobjType, shortType, "Float", "shortValue", false, true);
+        addTransform(floatobjType, charType, "Utility", "FloatTochar", true, true);
+        addTransform(floatobjType, intType, "Float", "intValue", false, true);
+        addTransform(floatobjType, longType, "Float", "longValue", false, true);
+        addTransform(floatobjType, floatType, "Float", "floatValue", false, false);
+        addTransform(floatobjType, doubleType, "Float", "doubleValue", false, false);
+        addTransform(floatobjType, byteobjType, "Utility", "NumberToByte", true, true);
+        addTransform(floatobjType, shortobjType, "Utility", "NumberToShort", true, true);
+        addTransform(floatobjType, charobjType, "Utility", "NumberToCharacter", true, true);
+        addTransform(floatobjType, intobjType, "Utility", "NumberToInteger", true, true);
+        addTransform(floatobjType, longobjType, "Utility", "NumberToLong", true, true);
+        addTransform(floatobjType, doubleobjType, "Utility", "NumberToDouble", true, false);
 
-        addTransform(doubleobjType, booleanType, "Utility", "DoubleToboolean", true);
-        addTransform(doubleobjType, byteType, "Double", "byteValue", false);
-        addTransform(doubleobjType, shortType, "Double", "shortValue", false);
-        addTransform(doubleobjType, charType, "Utility", "DoubleTochar", true);
-        addTransform(doubleobjType, intType, "Double", "intValue", false);
-        addTransform(doubleobjType, longType, "Double", "longValue", false);
-        addTransform(doubleobjType, floatType, "Double", "floatValue", false);
-        addTransform(doubleobjType, doubleType, "Double", "doubleValue", false);
-        addTransform(doubleobjType, booleanobjType, "Utility", "NumberToBoolean", true);
-        addTransform(doubleobjType, byteobjType, "Utility", "NumberToByte", true);
-        addTransform(doubleobjType, shortobjType, "Utility", "NumberToShort", true);
-        addTransform(doubleobjType, charobjType, "Utility", "NumberToCharacter", true);
-        addTransform(doubleobjType, intobjType, "Utility", "NumberToInteger", true);
-        addTransform(doubleobjType, longobjType, "Utility", "NumberToLong", true);
-        addTransform(doubleobjType, floatobjType, "Utility", "NumberToFloat", true);
+        addTransform(doubleobjType, byteType, "Double", "byteValue", false, true);
+        addTransform(doubleobjType, shortType, "Double", "shortValue", false, true);
+        addTransform(doubleobjType, charType, "Utility", "DoubleTochar", true, true);
+        addTransform(doubleobjType, intType, "Double", "intValue", false, true);
+        addTransform(doubleobjType, longType, "Double", "longValue", false, true);
+        addTransform(doubleobjType, floatType, "Double", "floatValue", false, true);
+        addTransform(doubleobjType, doubleType, "Double", "doubleValue", false, false);
+        addTransform(doubleobjType, byteobjType, "Utility", "NumberToByte", true, true);
+        addTransform(doubleobjType, shortobjType, "Utility", "NumberToShort", true, true);
+        addTransform(doubleobjType, charobjType, "Utility", "NumberToCharacter", true, true);
+        addTransform(doubleobjType, intobjType, "Utility", "NumberToInteger", true, true);
+        addTransform(doubleobjType, longobjType, "Utility", "NumberToLong", true, true);
+        addTransform(doubleobjType, floatobjType, "Utility", "NumberToFloat", true, true);
 
-        addTransform(stringType, charType, "Utility", "StringTochar", true);
-        addTransform(stringType, charobjType, "Utility", "StringToCharacter", true);
+        addTransform(stringType, charType, "Utility", "StringTochar", true, true);
+        addTransform(stringType, charobjType, "Utility", "StringToCharacter", true, true);
     }
 
     private void addRuntimeClasses() {
@@ -1560,7 +1583,7 @@ public final class Definition {
         addRuntimeClass(longsType.struct);
         addRuntimeClass(doublesType.struct);
         addRuntimeClass(geoPointsType.struct);
-        
+
         addRuntimeClass(featureTestType.struct);
     }
 
@@ -1590,7 +1613,7 @@ public final class Definition {
             throw new IllegalArgumentException(
                 "Invalid constructor name [" + name + "] with the struct [" + owner.name + "].");
         }
-        
+
         MethodKey methodKey = new MethodKey(name, args.length);
 
         if (owner.constructors.containsKey(methodKey)) {
@@ -1651,7 +1674,7 @@ public final class Definition {
             throw new IllegalArgumentException("Invalid " + (statik ? "static method" : "method") +
                 " name [" + name + "] with the struct [" + owner.name + "].");
         }
-        
+
         MethodKey methodKey = new MethodKey(name, args.length);
 
         if (owner.constructors.containsKey(methodKey)) {
@@ -1927,8 +1950,29 @@ public final class Definition {
         }
     }
 
+    private final void addTransform(final Type from, final Type to, final boolean explicit) {
+        if (from.equals(to)) {
+            throw new IllegalArgumentException("Transform cannot" +
+                " have cast type from [" + from.name + "] be the same as cast type to [" + to.name + "].");
+        }
+
+        if (!from.sort.primitive || !to.sort.primitive) {
+            throw new IllegalArgumentException("Only transforms between two primitives may be a simple cast, but" +
+                "found [" + from.name + "] and [" + to.name + "].");
+        }
+
+        final Cast cast = new Cast(from, to, explicit);
+
+        if (transformsMap.containsKey(cast)) {
+            throw new IllegalArgumentException("Transform with " +
+                " cast type from [" + from.name + "] to cast type to [" + to.name + "] already defined.");
+        }
+
+        transformsMap.put(cast, cast);
+    }
+
     private final void addTransform(final Type from, final Type to, final String struct,
-                                   final String name, final boolean statik) {
+                                   final String name, final boolean statik, final boolean explicit) {
         final Struct owner = structsMap.get(struct);
 
         if (owner == null) {
@@ -1941,19 +1985,21 @@ public final class Definition {
                 " have cast type from [" + from.name + "] be the same as cast type to [" + to.name + "].");
         }
 
-        final Cast cast = new Cast(from, to);
+        final Cast cast = new Cast(from, to, explicit);
 
         if (transformsMap.containsKey(cast)) {
             throw new IllegalArgumentException("Transform with owner struct [" + owner.name + "]" +
                 " and cast type from [" + from.name + "] to cast type to [" + to.name + "] already defined.");
         }
 
-        Method method;
+        final Cast transform;
+
+        final Method method;
         Type upcast = null;
         Type downcast = null;
 
         // transforms are implicitly arity of 0, unless a static method where its 1 (receiver passed)
-        MethodKey methodKey = new MethodKey(name, statik ? 1 : 0);
+        final MethodKey methodKey = new MethodKey(name, statik ? 1 : 0);
 
         if (statik) {
             method = owner.staticMethods.get(methodKey);
@@ -2031,7 +2077,7 @@ public final class Definition {
             }
         }
 
-        final Transform transform = new Transform(cast, method, upcast, downcast);
+        transform = new Transform(cast, method, upcast, downcast);
         transformsMap.put(cast, transform);
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
index 22ed4a84178..ac0b06c0a79 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
@@ -21,7 +21,6 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
-import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -33,8 +32,6 @@ public final class EExplicit extends AExpression {
     final String type;
     AExpression child;
 
-    Cast cast = null;
-
     public EExplicit(final int line, final String location, final String type, final AExpression child) {
         super(line, location);
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
index 22a9d50b991..bfc0d142045 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
@@ -20,6 +20,14 @@
 package org.elasticsearch.painless;
 
 public class DefOperationTests extends ScriptTestCase {
+    public void testIllegalCast() {
+        Exception exception = expectThrows(ClassCastException.class, () -> exec("def x = 1.0; int y = x; return y;"));
+        assertTrue(exception.getMessage().contains("java.lang.Double cannot be cast to java.lang.Integer"));
+
+        exception = expectThrows(ClassCastException.class, () -> exec("def x = (short)1; byte y = x; return y;"));
+        assertTrue(exception.getMessage().contains("java.lang.Short cannot be cast to java.lang.Byte"));
+    }
+
     public void testNot() {
         assertEquals(~1, exec("def x = (byte)1; return ~x"));
         assertEquals(~1, exec("def x = (short)1; return ~x"));

From 72094feb12b5cae9f204b0657ceca6bf00b39779 Mon Sep 17 00:00:00 2001
From: polyfractal <zacharyjtong@gmail.com>
Date: Tue, 17 May 2016 16:39:35 -0400
Subject: [PATCH 0306/1311] [TEST] Add missing sort processor to tests,
 continued

---
 .../rest-api-spec/test/ingest_attachment/10_basic.yaml     | 7 ++++---
 .../rest-api-spec/test/ingest_geoip/10_basic.yaml          | 7 ++++---
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/10_basic.yaml b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/10_basic.yaml
index 697ad558bb7..7546090d6ad 100644
--- a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/10_basic.yaml
+++ b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/10_basic.yaml
@@ -21,7 +21,8 @@
     - match:  { nodes.$master.ingest.processors.10.type: remove }
     - match:  { nodes.$master.ingest.processors.11.type: rename }
     - match:  { nodes.$master.ingest.processors.12.type: set }
-    - match:  { nodes.$master.ingest.processors.13.type: split }
-    - match:  { nodes.$master.ingest.processors.14.type: trim }
-    - match:  { nodes.$master.ingest.processors.15.type: uppercase }
+    - match:  { nodes.$master.ingest.processors.13.type: sort }
+    - match:  { nodes.$master.ingest.processors.14.type: split }
+    - match:  { nodes.$master.ingest.processors.15.type: trim }
+    - match:  { nodes.$master.ingest.processors.16.type: uppercase }
 
diff --git a/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/10_basic.yaml b/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/10_basic.yaml
index e405bbb5a6c..313a9b4544f 100644
--- a/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/10_basic.yaml
+++ b/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/10_basic.yaml
@@ -21,6 +21,7 @@
     - match:  { nodes.$master.ingest.processors.10.type: remove }
     - match:  { nodes.$master.ingest.processors.11.type: rename }
     - match:  { nodes.$master.ingest.processors.12.type: set }
-    - match:  { nodes.$master.ingest.processors.13.type: split }
-    - match:  { nodes.$master.ingest.processors.14.type: trim }
-    - match:  { nodes.$master.ingest.processors.15.type: uppercase }
+    - match:  { nodes.$master.ingest.processors.13.type: sort }
+    - match:  { nodes.$master.ingest.processors.14.type: split }
+    - match:  { nodes.$master.ingest.processors.15.type: trim }
+    - match:  { nodes.$master.ingest.processors.16.type: uppercase }

From 4e4ea5617e263e8dfabba83aecb5ddfd120692bf Mon Sep 17 00:00:00 2001
From: polyfractal <zacharyjtong@gmail.com>
Date: Tue, 17 May 2016 16:47:20 -0400
Subject: [PATCH 0307/1311] [DOCS] Add missing section include for reindex
 breaking changes

---
 docs/reference/migration/migrate_5_0.asciidoc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/reference/migration/migrate_5_0.asciidoc b/docs/reference/migration/migrate_5_0.asciidoc
index 71e37e49581..7dc79478c8a 100644
--- a/docs/reference/migration/migrate_5_0.asciidoc
+++ b/docs/reference/migration/migrate_5_0.asciidoc
@@ -58,6 +58,8 @@ include::migrate_5_0/suggest.asciidoc[]
 
 include::migrate_5_0/index-apis.asciidoc[]
 
+include::migrate_5_0/docs.asciidoc[]
+
 include::migrate_5_0/settings.asciidoc[]
 
 include::migrate_5_0/allocation.asciidoc[]

From b418d01117cf47450adc1d3c0db75733b8bf3d57 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Wed, 18 May 2016 00:26:47 +0200
Subject: [PATCH 0308/1311] painless: Improve string concat tests; add state
 machine check after MethodWriter

---
 .../elasticsearch/painless/MethodWriter.java  |  8 +++++++
 .../elasticsearch/painless/StringTests.java   | 22 +++++++++++++++----
 2 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index a982b889194..01eb28fa4d4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -488,4 +488,12 @@ public final class MethodWriter extends GeneratorAdapter {
         }
     }
 
+    @Override
+    public void visitEnd() {
+        if (stringConcatArgs != null && !stringConcatArgs.isEmpty()) {
+            throw new IllegalStateException("String concat bytecode not completed.");
+        }
+        super.visitEnd();
+    }
+
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
index 8d22323d5b3..e61541bf371 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.painless;
 
+import static org.elasticsearch.painless.WriterConstants.MAX_INDY_STRING_CONCAT_ARGS;
+
 import java.util.Locale;
 
 public class StringTests extends ScriptTestCase {
@@ -70,15 +72,27 @@ public class StringTests extends ScriptTestCase {
     }
     
     public void testAppendMany() {
+        for (int i = MAX_INDY_STRING_CONCAT_ARGS - 5; i < MAX_INDY_STRING_CONCAT_ARGS + 5; i++) {
+            doTestAppendMany(i);
+        }
+    }
+    
+    private void doTestAppendMany(int count) {
         StringBuilder script = new StringBuilder("String s = \"cat\"; return s");
         StringBuilder result = new StringBuilder("cat");
-        for (int i = 0; i < WriterConstants.MAX_INDY_STRING_CONCAT_ARGS + 10; i++) {
-            final String s = String.format(Locale.ROOT,  "%03d", i);
+        for (int i = 1; i < count; i++) {
+            final String s = String.format(Locale.ROOT, "%03d", i);
             script.append(" + '").append(s).append("'.toString()");
             result.append(s);
         }
-        //System.out.println(Debugger.toString(script.toString()));
-        assertEquals(result.toString(), exec(script.toString()));
+        final String s = script.toString();
+        assertTrue("every string part should be separatly pushed to stack.",
+                Debugger.toString(s).contains(String.format(Locale.ROOT, "LDC \"%03d\"", count/2)));
+        assertEquals(result.toString(), exec(s));
+    }
+    
+    public void testNestedConcats() {
+        assertEquals("foo1010foo", exec("String s = 'foo'; String x = '10'; return s + Integer.parseInt(x + x) + s;"));
     }
     
     public void testStringAPI() {

From ea9026e6d8e2ccbb152d9458221a44a86d14b36f Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Wed, 18 May 2016 00:31:12 +0200
Subject: [PATCH 0309/1311] painless: fix checkstyle

---
 .../src/main/java/org/elasticsearch/painless/MethodWriter.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index 01eb28fa4d4..fbb1d246a83 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -105,7 +105,8 @@ import static org.elasticsearch.painless.WriterConstants.TOSHORTWOOVERFLOW_FLOAT
  */
 public final class MethodWriter extends GeneratorAdapter {
 
-    private final Deque<List<org.objectweb.asm.Type>> stringConcatArgs = (INDY_STRING_CONCAT_BOOTSTRAP_HANDLE == null) ? null : new ArrayDeque<>();
+    private final Deque<List<org.objectweb.asm.Type>> stringConcatArgs = (INDY_STRING_CONCAT_BOOTSTRAP_HANDLE == null) ?
+            null : new ArrayDeque<>();
 
     MethodWriter(int access, Method method, org.objectweb.asm.Type[] exceptions, ClassVisitor cv) {
         super(Opcodes.ASM5, cv.visitMethod(access, method.getName(), method.getDescriptor(), null, getInternalNames(exceptions)),

From 850e9d7c57462d70b5c196172f4223ea9b17260d Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 10 May 2016 18:29:56 -0400
Subject: [PATCH 0310/1311] [docs] Add // CONSOLE to validate and uri-request

Two of the snippets in validate weren't working properly so they are
marked as skip and linked to this:
https://github.com/elastic/elasticsearch/issues/18254

We didn't properly handle empty parameter values. We were sending
them as the literal string "null". Now we do better and send them
as the empty string.
---
 .../doc/RestTestsFromSnippetsTask.groovy      |  3 +
 docs/build.gradle                             | 12 ++-
 docs/reference/search/uri-request.asciidoc    | 19 +++--
 docs/reference/search/validate.asciidoc       | 83 +++++++++++++------
 4 files changed, 84 insertions(+), 33 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
index 1a519fe7c2b..ef8c8e280ed 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
@@ -144,6 +144,9 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
             if (query != null) {
                 for (String param: query.tokenize('&')) {
                     def (String name, String value) = param.tokenize('=')
+                    if (value == null) {
+                        value = ''
+                    }
                     current.println("        $name: \"$value\"")
                 }
             }
diff --git a/docs/build.gradle b/docs/build.gradle
index f80ef1e1fd0..239eb81e820 100644
--- a/docs/build.gradle
+++ b/docs/build.gradle
@@ -45,9 +45,17 @@ Closure setupTwitter = { String name, int count ->
         refresh: true
         body: |'''
     for (int i = 0; i < count; i++) {
+      String user, text
+      if (i == 0) {
+        user = 'kimchy'
+        text = 'trying out Elasticsearch'
+      } else {
+        user = 'test'
+        text = "some message with the number $i"
+      }
       buildRestTests.setups[name] += """
-          {"index":{}}
-          {"msg": "some message with the number $i", "date": $i}"""
+          {"index":{"_id": "$i"}}
+          {"user": "$user", "message": "$text", "date": "2009-11-15T14:12:12", "likes": $i}"""
     }
   }
 setupTwitter('twitter', 5)
diff --git a/docs/reference/search/uri-request.asciidoc b/docs/reference/search/uri-request.asciidoc
index e43c3edf94a..9aa9c17727e 100644
--- a/docs/reference/search/uri-request.asciidoc
+++ b/docs/reference/search/uri-request.asciidoc
@@ -8,14 +8,18 @@ example:
 
 [source,js]
 --------------------------------------------------
-$ curl -XGET 'http://localhost:9200/twitter/tweet/_search?q=user:kimchy'
+GET twitter/tweet/_search?q=user:kimchy
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 And here is a sample response:
 
 [source,js]
 --------------------------------------------------
 {
+    "timed_out": false,
+    "took": 62,
     "_shards":{
         "total" : 5,
         "successful" : 5,
@@ -23,21 +27,25 @@ And here is a sample response:
     },
     "hits":{
         "total" : 1,
+        "max_score": 1.0,
         "hits" : [
             {
                 "_index" : "twitter",
                 "_type" : "tweet",
-                "_id" : "1",
+                "_id" : "0",
+                "_score": 1.0,
                 "_source" : {
                     "user" : "kimchy",
-                    "postDate" : "2009-11-15T14:12:12",
-                    "message" : "trying out Elasticsearch"
+                    "date" : "2009-11-15T14:12:12",
+                    "message" : "trying out Elasticsearch",
+                    "likes": 0
                 }
             }
         ]
     }
 }
 --------------------------------------------------
+// TESTRESPONSE[s/"took": 62/"took": "$body.took"/]
 
 [float]
 === Parameters
@@ -75,7 +83,7 @@ hits was computed.
 part of the document by using `_source_include` & `_source_exclude` (see the <<search-request-source-filtering, request body>>
 documentation for more details)
 
-|`fields` |The selective stored fields of the document to return for each hit, 
+|`fields` |The selective stored fields of the document to return for each hit,
 comma delimited. Not specifying any value will cause no fields to return.
 
 |`sort` |Sorting to perform. Can either be in the form of `fieldName`, or
@@ -107,4 +115,3 @@ Defaults to `query_then_fetch`. See
 <<search-request-search-type,_Search Type_>> for
 more details on the different types of search that can be performed.
 |=======================================================================
-
diff --git a/docs/reference/search/validate.asciidoc b/docs/reference/search/validate.asciidoc
index a08e183089c..8a2730c0148 100644
--- a/docs/reference/search/validate.asciidoc
+++ b/docs/reference/search/validate.asciidoc
@@ -2,24 +2,34 @@
 == Validate API
 
 The validate API allows a user to validate a potentially expensive query
-without executing it. The following example shows how it can be used:
+without executing it. We'll use the following test data to explain _validate:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT 'http://localhost:9200/twitter/tweet/1' -d '{
-    "user" : "kimchy",
-    "post_date" : "2009-11-15T14:12:12",
-    "message" : "trying out Elasticsearch"
-}'
+PUT twitter/tweet/_bulk?refresh
+{"index":{"_id":1}}
+{"user" : "kimchy", "post_date" : "2009-11-15T14:12:12", "message" : "trying out Elasticsearch"}
+{"index":{"_id":2}}
+{"user" : "kimchi", "post_date" : "2009-11-15T14:12:13", "message" : "My username is similar to @kimchy!"}
 --------------------------------------------------
+// CONSOLE
+// TESTSETUP
 
-When the query is valid, the response contains `valid:true`:
+When sent a valid query:
+
+[source,js]
+--------------------------------------------------
+GET twitter/_validate/query?q=user:foo
+--------------------------------------------------
+// CONSOLE
+
+The response contains `valid:true`:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'http://localhost:9200/twitter/_validate/query?q=user:foo'
 {"valid":true,"_shards":{"total":1,"successful":1,"failed":0}}
 --------------------------------------------------
+// TESTRESPONSE
 
 [float]
 === Request Parameters
@@ -49,11 +59,12 @@ not. Defaults to `true`.
 not. Defaults to `false`.
 |=======================================================================
 
-Or, with a request body:
+The query may also be sent in the request body:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'http://localhost:9200/twitter/tweet/_validate/query' -d '{
+GET twitter/tweet/_validate/query
+{
   "query" : {
     "bool" : {
       "must" : {
@@ -66,9 +77,9 @@ curl -XGET 'http://localhost:9200/twitter/tweet/_validate/query' -d '{
       }
     }
   }
-}'
-{"valid":true,"_shards":{"total":1,"successful":1,"failed":0}}
+}
 --------------------------------------------------
+// CONSOLE
 
 NOTE: The query being sent in the body must be nested in a `query` key, same as
 the <<search-search,search api>> works
@@ -79,16 +90,28 @@ due to dynamic mapping, and 'foo' does not correctly parse into a date:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'http://localhost:9200/twitter/tweet/_validate/query?q=post_date:foo'
+GET twitter/tweet/_validate/query?q=post_date:foo
+--------------------------------------------------
+// CONSOLE
+
+[source,js]
+--------------------------------------------------
 {"valid":false,"_shards":{"total":1,"successful":1,"failed":0}}
 --------------------------------------------------
+// TESTRESPONSE
 
 An `explain` parameter can be specified to get more detailed information
 about why a query failed:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'http://localhost:9200/twitter/tweet/_validate/query?q=post_date:foo&pretty=true&explain=true'
+GET twitter/tweet/_validate/query?q=post_date:foo&explain=true
+--------------------------------------------------
+// CONSOLE
+
+responds with:
+
+--------------------------------------------------
 {
   "valid" : false,
   "_shards" : {
@@ -99,10 +122,11 @@ curl -XGET 'http://localhost:9200/twitter/tweet/_validate/query?q=post_date:foo&
   "explanations" : [ {
     "index" : "twitter",
     "valid" : false,
-    "error" : "[twitter] QueryParsingException[Failed to parse]; nested: IllegalArgumentException[Invalid format: \"foo\"];; java.lang.IllegalArgumentException: Invalid format: \"foo\""
+    "error" : "twitter/IAEc2nIXSSunQA_suI0MLw] QueryShardException[failed to create query:...failed to parse date field [foo]"
   } ]
 }
 --------------------------------------------------
+// TESTRESPONSE[s/"error" : "[^\"]+"/"error": "$body.explanations.0.error"/]
 
 When the query is valid, the explanation defaults to the string
 representation of that query. With `rewrite` set to `true`, the explanation
@@ -112,15 +136,20 @@ For Fuzzy Queries:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'http://localhost:9200/imdb/movies/_validate/query?rewrite=true' -d '
+GET twitter/tweet/_validate/query?rewrite=true
 {
   "query": {
-    "fuzzy": {
-      "actors": "kyle"
+    "match": {
+      "user": {
+        "query": "kimchy",
+        "fuzziness": "auto"
+      }
     }
   }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[skip:https://github.com/elastic/elasticsearch/issues/18254]
 
 Response:
 
@@ -135,30 +164,33 @@ Response:
    },
    "explanations": [
       {
-         "index": "imdb",
+         "index": "twitter",
          "valid": true,
-         "explanation": "plot:kyle plot:kylie^0.75 plot:kyne^0.75 plot:lyle^0.75 plot:pyle^0.75 #_type:movies"
+         "explanation": "+user:kimchy +user:kimchi^0.75 #(ConstantScore(_type:tweet))^0.0"
       }
    ]
 }
 --------------------------------------------------
+// TESTRESPONSE
 
 For More Like This:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'http://localhost:9200/imdb/movies/_validate/query?rewrite=true'
+GET twitter/tweet/_validate/query?rewrite=true
 {
   "query": {
     "more_like_this": {
       "like": {
-        "_id": "88247"
+        "_id": "2"
       },
       "boost_terms": 1
     }
   }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[skip:https://github.com/elastic/elasticsearch/issues/18254]
 
 Response:
 
@@ -173,13 +205,14 @@ Response:
    },
    "explanations": [
       {
-         "index": "imdb",
+         "index": "twitter",
          "valid": true,
-         "explanation": "((title:terminator^3.71334 plot:future^2.763601 plot:human^2.8415773 plot:sarah^3.4193945 plot:kyle^3.8244398 plot:cyborg^3.9177752 plot:connor^4.040236 plot:reese^4.7133346 ... )~6) -ConstantScore(_uid:movies#88247) #_type:movies"
+         "explanation": "((user:terminator^3.71334 plot:future^2.763601 plot:human^2.8415773 plot:sarah^3.4193945 plot:kyle^3.8244398 plot:cyborg^3.9177752 plot:connor^4.040236 plot:reese^4.7133346 ... )~6) -ConstantScore(_uid:tweet#2)) #(ConstantScore(_type:tweet))^0.0"
       }
    ]
 }
 --------------------------------------------------
+// TESTRESPONSE
 
 CAUTION: The request is executed on a single shard only, which is randomly
 selected. The detailed explanation of the query may depend on which shard is

From c13df3b6c5020c185c232c590132151896d48d03 Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@elastic.co>
Date: Wed, 18 May 2016 09:18:10 +0200
Subject: [PATCH 0311/1311] Clear all caches after testing parent breaker

With this commit we clear all caches after testing the parent circuit breaker.
This is necessary as caches hold on to circuit breakers internally. Additionally,
due to usage of CircuitBreaker#addWithoutBreaking() in caches, it's even possible
to go above the limit. As a consequence, all subsequent requests fall victim to
the limit.

Hence, right after the parent circuit breaker tripped, we clear all caches to
reduce these circuit breakers to 0 again. We also exclude the clear caches
transport request from limit check in order to ensure it will succeed. As this is
typically a very small and low-volume request, it is deemed ok to exclude it.

Closes #18325
---
 .../TransportClearIndicesCacheAction.java     |  2 +-
 .../node/TransportBroadcastByNodeAction.java  | 20 +++++++++++++++++--
 .../breaker/CircuitBreakerServiceIT.java      |  7 +++----
 3 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java
index 59cd95044cc..fe97302060d 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java
@@ -54,7 +54,7 @@ public class TransportClearIndicesCacheAction extends TransportBroadcastByNodeAc
                                             TransportService transportService, IndicesService indicesService, ActionFilters actionFilters,
                                             IndexNameExpressionResolver indexNameExpressionResolver) {
         super(settings, ClearIndicesCacheAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
-                ClearIndicesCacheRequest::new, ThreadPool.Names.MANAGEMENT);
+                ClearIndicesCacheRequest::new, ThreadPool.Names.MANAGEMENT, false);
         this.indicesService = indicesService;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java b/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
index 29863419a4e..3356d189143 100644
--- a/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
@@ -84,6 +84,20 @@ public abstract class TransportBroadcastByNodeAction<Request extends BroadcastRe
 
     final String transportNodeBroadcastAction;
 
+    public TransportBroadcastByNodeAction(
+        Settings settings,
+        String actionName,
+        ThreadPool threadPool,
+        ClusterService clusterService,
+        TransportService transportService,
+        ActionFilters actionFilters,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        Supplier<Request> request,
+        String executor) {
+        this(settings, actionName, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver, request,
+            executor, true);
+    }
+
     public TransportBroadcastByNodeAction(
             Settings settings,
             String actionName,
@@ -93,7 +107,8 @@ public abstract class TransportBroadcastByNodeAction<Request extends BroadcastRe
             ActionFilters actionFilters,
             IndexNameExpressionResolver indexNameExpressionResolver,
             Supplier<Request> request,
-            String executor) {
+            String executor,
+            boolean canTripCircuitBreaker) {
         super(settings, actionName, threadPool, transportService, actionFilters, indexNameExpressionResolver, request);
 
         this.clusterService = clusterService;
@@ -101,7 +116,8 @@ public abstract class TransportBroadcastByNodeAction<Request extends BroadcastRe
 
         transportNodeBroadcastAction = actionName + "[n]";
 
-        transportService.registerRequestHandler(transportNodeBroadcastAction, NodeRequest::new, executor, new BroadcastByNodeTransportRequestHandler());
+        transportService.registerRequestHandler(transportNodeBroadcastAction, NodeRequest::new, executor, false, canTripCircuitBreaker,
+            new BroadcastByNodeTransportRequestHandler());
     }
 
     private Response newResponse(
diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
index a125e3842e7..0ae8f71c742 100644
--- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
@@ -75,6 +75,9 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase {
     /** Reset all breaker settings back to their defaults */
     private void reset() {
         logger.info("--> resetting breaker settings");
+        // clear all caches, we could be very close (or even above) the limit and then we will not be able to reset the breaker settings
+        client().admin().indices().prepareClearCache().setFieldDataCache(true).setQueryCache(true).setRequestCache(true).get();
+
         Settings resetSettings = Settings.builder()
                 .put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(),
                         HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING.getDefaultRaw(null))
@@ -214,7 +217,6 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase {
      * Test that a breaker correctly redistributes to a different breaker, in
      * this case, the fielddata breaker borrows space from the request breaker
      */
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/18325")
     public void testParentChecking() throws Exception {
         if (noopBreakerUsed()) {
             logger.info("--> noop breakers used, skipping test");
@@ -274,9 +276,6 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase {
                 cause.toString(), startsWith("CircuitBreakingException[[parent] Data too large"));
             assertThat("Exception: [" + cause.toString() + "] should contain a CircuitBreakingException",
                 cause.toString(), endsWith(errMsg));
-        } finally {
-            // reset before teardown as it requires properly set up breakers
-            reset();
         }
     }
 

From 6dacac49b394df9c96d20f3b4c74c8a8d7e22558 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Wed, 18 May 2016 10:51:57 +0200
Subject: [PATCH 0312/1311] Simplify recovery logic in
 IndicesClusterStateService (#18405)

- Moves recovery logic into IndexShard
- Simplifies logic to cancel peer recovery of shard where recovery source node changed
- Ensures routing entry is set on initialization of IndexShard
---
 .../cluster/routing/ShardRouting.java         |   7 +
 .../org/elasticsearch/index/IndexService.java |   5 +-
 .../elasticsearch/index/shard/IndexShard.java |  74 ++++++--
 .../index/shard/ShadowIndexShard.java         |   8 +-
 .../cluster/IndicesClusterStateService.java   | 161 ++++++------------
 .../recovery/RecoveriesCollection.java        |  25 +--
 .../recovery/RecoveryTargetService.java       |   7 +-
 .../index/shard/IndexShardTests.java          |  43 ++---
 .../IndexingMemoryControllerTests.java        |   5 +-
 .../recovery/RecoveriesCollectionTests.java   |  35 +---
 10 files changed, 159 insertions(+), 211 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
index ce31483f47f..60bb455dfe6 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
@@ -252,6 +252,13 @@ public final class ShardRouting implements Writeable, ToXContent {
         return true;
     }
 
+    /**
+     * returns true for initializing shards that recover their data from another shard copy
+     */
+    public boolean isPeerRecovery() {
+        return state == ShardRoutingState.INITIALIZING && (primary() == false || relocatingNodeId != null);
+    }
+
     /**
      * A shard iterator with just this shard in it.
      */
diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java
index ea2cfe9f106..60e3250e49d 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexService.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexService.java
@@ -336,18 +336,17 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
             store = new Store(shardId, this.indexSettings, indexStore.newDirectoryService(path), lock,
                 new StoreCloseListener(shardId, canDeleteShardContent, () -> eventListener.onStoreClosed(shardId)));
             if (useShadowEngine(primary, indexSettings)) {
-                indexShard = new ShadowIndexShard(shardId, this.indexSettings, path, store, indexCache, mapperService, similarityService,
+                indexShard = new ShadowIndexShard(routing, this.indexSettings, path, store, indexCache, mapperService, similarityService,
                     indexFieldData, engineFactory, eventListener, searcherWrapper, threadPool, bigArrays, engineWarmer,
                     searchOperationListeners);
                 // no indexing listeners - shadow  engines don't index
             } else {
-                indexShard = new IndexShard(shardId, this.indexSettings, path, store, indexCache, mapperService, similarityService,
+                indexShard = new IndexShard(routing, this.indexSettings, path, store, indexCache, mapperService, similarityService,
                     indexFieldData, engineFactory, eventListener, searcherWrapper, threadPool, bigArrays, engineWarmer,
                     searchOperationListeners, indexingOperationListeners);
             }
             eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created");
             eventListener.afterIndexShardCreated(indexShard);
-            indexShard.updateRoutingEntry(routing, true);
             shards = newMapBuilder(shards).put(shardId.id(), indexShard).immutableMap();
             success = true;
             return indexShard;
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 59ea1a29bcf..768db935308 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -36,6 +36,7 @@ import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequest;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RestoreSource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.common.Booleans;
@@ -103,6 +104,8 @@ import org.elasticsearch.index.warmer.WarmerStats;
 import org.elasticsearch.indices.IndexingMemoryController;
 import org.elasticsearch.indices.recovery.RecoveryFailedException;
 import org.elasticsearch.indices.recovery.RecoveryState;
+import org.elasticsearch.indices.recovery.RecoveryTargetService;
+import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.search.suggest.completion.CompletionFieldStats;
 import org.elasticsearch.search.suggest.completion.CompletionStats;
 import org.elasticsearch.search.suggest.completion2x.Completion090PostingsFormat;
@@ -194,12 +197,14 @@ public class IndexShard extends AbstractIndexShardComponent {
      */
     private final AtomicBoolean active = new AtomicBoolean();
 
-    public IndexShard(ShardId shardId, IndexSettings indexSettings, ShardPath path, Store store, IndexCache indexCache,
+    public IndexShard(ShardRouting shardRouting, IndexSettings indexSettings, ShardPath path, Store store, IndexCache indexCache,
                       MapperService mapperService, SimilarityService similarityService, IndexFieldDataService indexFieldDataService,
                       @Nullable EngineFactory engineFactory,
                       IndexEventListener indexEventListener, IndexSearcherWrapper indexSearcherWrapper, ThreadPool threadPool, BigArrays bigArrays,
-                      Engine.Warmer warmer, List<SearchOperationListener> searchOperationListener, List<IndexingOperationListener> listeners) {
-        super(shardId, indexSettings);
+                      Engine.Warmer warmer, List<SearchOperationListener> searchOperationListener, List<IndexingOperationListener> listeners) throws IOException {
+        super(shardRouting.shardId(), indexSettings);
+        assert shardRouting.initializing();
+        this.shardRouting = shardRouting;
         final Settings settings = indexSettings.getSettings();
         this.codecService = new CodecService(mapperService, logger);
         this.warmer = warmer;
@@ -243,6 +248,7 @@ public class IndexShard extends AbstractIndexShardComponent {
         suspendableRefContainer = new SuspendableRefContainer();
         searcherWrapper = indexSearcherWrapper;
         primaryTerm = indexSettings.getIndexMetaData().primaryTerm(shardId.id());
+        persistMetadata(shardRouting, null);
     }
 
     public Store store() {
@@ -313,8 +319,7 @@ public class IndexShard extends AbstractIndexShardComponent {
     }
 
     /**
-     * Returns the latest cluster routing entry received with this shard. Might be null if the
-     * shard was just created.
+     * Returns the latest cluster routing entry received with this shard.
      */
     public ShardRouting routingEntry() {
         return this.shardRouting;
@@ -1325,6 +1330,58 @@ public class IndexShard extends AbstractIndexShardComponent {
         return this.currentEngineReference.get();
     }
 
+    public void startRecovery(DiscoveryNode localNode, DiscoveryNode sourceNode, RecoveryTargetService recoveryTargetService,
+                              RecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService) {
+        final RestoreSource restoreSource = shardRouting.restoreSource();
+
+        if (shardRouting.isPeerRecovery()) {
+            assert sourceNode != null : "peer recovery started but sourceNode is null";
+            // we don't mark this one as relocated at the end.
+            // For primaries: requests in any case are routed to both when its relocating and that way we handle
+            //    the edge case where its mark as relocated, and we might need to roll it back...
+            // For replicas: we are recovering a backup from a primary
+            RecoveryState.Type type = shardRouting.primary() ? RecoveryState.Type.PRIMARY_RELOCATION : RecoveryState.Type.REPLICA;
+            RecoveryState recoveryState = new RecoveryState(shardId(), shardRouting.primary(), type, sourceNode, localNode);
+            try {
+                markAsRecovering("from " + sourceNode, recoveryState);
+                recoveryTargetService.startRecovery(this, type, sourceNode, recoveryListener);
+            } catch (Throwable e) {
+                failShard("corrupted preexisting index", e);
+                recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(shardId, sourceNode, localNode, e), true);
+            }
+        } else if (restoreSource == null) {
+            // recover from filesystem store
+            final RecoveryState recoveryState = new RecoveryState(shardId(), shardRouting.primary(),
+                RecoveryState.Type.STORE, localNode, localNode);
+            markAsRecovering("from store", recoveryState); // mark the shard as recovering on the cluster state thread
+            threadPool.generic().execute(() -> {
+                try {
+                    if (recoverFromStore(localNode)) {
+                        recoveryListener.onRecoveryDone(recoveryState);
+                    }
+                } catch (Throwable t) {
+                    recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(shardId, sourceNode, localNode, t), true);
+                }
+
+            });
+        } else {
+            // recover from a restore
+            final RecoveryState recoveryState = new RecoveryState(shardId(), shardRouting.primary(),
+                RecoveryState.Type.SNAPSHOT, shardRouting.restoreSource(), localNode);
+            markAsRecovering("from snapshot", recoveryState); // mark the shard as recovering on the cluster state thread
+            threadPool.generic().execute(() -> {
+                try {
+                    final IndexShardRepository indexShardRepository = repositoriesService.indexShardRepository(restoreSource.snapshotId().getRepository());
+                    if (restoreFromRepository(indexShardRepository, localNode)) {
+                        recoveryListener.onRecoveryDone(recoveryState);
+                    }
+                } catch (Throwable first) {
+                    recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(shardId, sourceNode, localNode, first), true);
+                }
+            });
+        }
+    }
+
     class ShardEventListener implements Engine.EventListener {
         private final CopyOnWriteArrayList<Callback<ShardFailure>> delegates = new CopyOnWriteArrayList<>();
 
@@ -1367,13 +1424,6 @@ public class IndexShard extends AbstractIndexShardComponent {
         return engineFactory.newReadWriteEngine(config);
     }
 
-    /**
-     * Returns <code>true</code> iff this shard allows primary promotion, otherwise <code>false</code>
-     */
-    public boolean allowsPrimaryPromotion() {
-        return true;
-    }
-
     // pkg private for testing
     void persistMetadata(ShardRouting newRouting, @Nullable ShardRouting currentRouting) throws IOException {
         assert newRouting != null : "newRouting must not be null";
diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java
index e75973f2ccc..bf35d02fea2 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java
@@ -46,12 +46,12 @@ import java.util.List;
  */
 public final class ShadowIndexShard extends IndexShard {
 
-    public ShadowIndexShard(ShardId shardId, IndexSettings indexSettings, ShardPath path, Store store, IndexCache indexCache,
+    public ShadowIndexShard(ShardRouting shardRouting, IndexSettings indexSettings, ShardPath path, Store store, IndexCache indexCache,
                             MapperService mapperService, SimilarityService similarityService, IndexFieldDataService indexFieldDataService,
                             @Nullable EngineFactory engineFactory, IndexEventListener indexEventListener, IndexSearcherWrapper wrapper,
                             ThreadPool threadPool, BigArrays bigArrays, Engine.Warmer engineWarmer,
                             List<SearchOperationListener> searchOperationListeners) throws IOException {
-        super(shardId, indexSettings, path, store, indexCache, mapperService, similarityService, indexFieldDataService, engineFactory,
+        super(shardRouting, indexSettings, path, store, indexCache, mapperService, similarityService, indexFieldDataService, engineFactory,
             indexEventListener, wrapper, threadPool, bigArrays, engineWarmer, searchOperationListeners, Collections.emptyList());
     }
 
@@ -92,10 +92,6 @@ public final class ShadowIndexShard extends IndexShard {
         return false;
     }
 
-    public boolean allowsPrimaryPromotion() {
-        return false;
-    }
-
     @Override
     public TranslogStats translogStats() {
         return null; // shadow engine has no translog
diff --git a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
index 4487a36c0ae..6f9b1684b9e 100644
--- a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
+++ b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
@@ -31,7 +31,6 @@ import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
-import org.elasticsearch.cluster.routing.RestoreSource;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -40,6 +39,7 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.Callback;
@@ -57,7 +57,6 @@ import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardNotFoundException;
-import org.elasticsearch.index.snapshots.IndexShardRepository;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.flush.SyncedFlushService;
 import org.elasticsearch.indices.recovery.RecoveryFailedException;
@@ -512,21 +511,20 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
                 assert currentRoutingEntry.isSameAllocation(shardRouting) :
                     "local shard has a different allocation id but wasn't cleaning by applyDeletedShards. "
                         + "cluster state: " + shardRouting + " local: " + currentRoutingEntry;
-                if (isPeerRecovery(shardRouting)) {
-                    final DiscoveryNode sourceNode = findSourceNodeForPeerRecovery(routingTable, nodes, shardRouting);
-                    // check if there is an existing recovery going, and if so, and the source node is not the same, cancel the recovery to restart it
-                    if (recoveryTargetService.cancelRecoveriesForShard(indexShard.shardId(), "recovery source node changed", status -> !status.sourceNode().equals(sourceNode))) {
-                        logger.debug("[{}][{}] removing shard (recovery source changed), current [{}], global [{}])", shardRouting.index(), shardRouting.id(), currentRoutingEntry, shardRouting);
-                        // closing the shard will also cancel any ongoing recovery.
-                        indexService.removeShard(shardRouting.id(), "removing shard (recovery source node changed)");
-                        shardHasBeenRemoved = true;
+                if (shardRouting.isPeerRecovery()) {
+                    RecoveryState recoveryState = indexShard.recoveryState();
+                    final DiscoveryNode sourceNode = findSourceNodeForPeerRecovery(logger, routingTable, nodes, shardRouting);
+                    if (recoveryState.getSourceNode().equals(sourceNode) == false) {
+                        if (recoveryTargetService.cancelRecoveriesForShard(currentRoutingEntry.shardId(), "recovery source node changed")) {
+                            // getting here means that the shard was still recovering
+                            logger.debug("[{}][{}] removing shard (recovery source changed), current [{}], global [{}])", shardRouting.index(), shardRouting.id(), currentRoutingEntry, shardRouting);
+                            indexService.removeShard(shardRouting.id(), "removing shard (recovery source node changed)");
+                            shardHasBeenRemoved = true;
+                        }
                     }
                 }
 
                 if (shardHasBeenRemoved == false) {
-                    // shadow replicas do not support primary promotion. The master would reinitialize the shard, giving it a new allocation, meaning we should be there.
-                    assert (shardRouting.primary() && currentRoutingEntry.primary() == false) == false || indexShard.allowsPrimaryPromotion() :
-                        "shard for doesn't support primary promotion but master promoted it with changing allocation. New routing " + shardRouting + ", current routing " + currentRoutingEntry;
                     try {
                         indexShard.updateRoutingEntry(shardRouting, event.state().blocks().disableStatePersistence() == false);
                     } catch (Throwable e) {
@@ -536,12 +534,12 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
             }
 
             if (shardRouting.initializing()) {
-                applyInitializingShard(event.state(), indexMetaData, indexService, shardRouting);
+                applyInitializingShard(event.state(), indexService, shardRouting);
             }
         }
     }
 
-    private void applyInitializingShard(final ClusterState state, final IndexMetaData indexMetaData, IndexService indexService, final ShardRouting shardRouting) {
+    private void applyInitializingShard(final ClusterState state, IndexService indexService, final ShardRouting shardRouting) {
         final RoutingTable routingTable = state.routingTable();
         final DiscoveryNodes nodes = state.getNodes();
         final int shardId = shardRouting.id();
@@ -572,8 +570,8 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
 
         // if we're in peer recovery, try to find out the source node now so in case it fails, we will not create the index shard
         DiscoveryNode sourceNode = null;
-        if (isPeerRecovery(shardRouting)) {
-            sourceNode = findSourceNodeForPeerRecovery(routingTable, nodes, shardRouting);
+        if (shardRouting.isPeerRecovery()) {
+            sourceNode = findSourceNodeForPeerRecovery(logger, routingTable, nodes, shardRouting);
             if (sourceNode == null) {
                 logger.trace("ignoring initializing shard {} - no source node can be found.", shardRouting.shardId());
                 return;
@@ -612,93 +610,27 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
             return;
         }
 
-        final RestoreSource restoreSource = shardRouting.restoreSource();
-
-        if (isPeerRecovery(shardRouting)) {
-            try {
-
-                assert sourceNode != null : "peer recovery started but sourceNode is null";
-
-                // we don't mark this one as relocated at the end.
-                // For primaries: requests in any case are routed to both when its relocating and that way we handle
-                //    the edge case where its mark as relocated, and we might need to roll it back...
-                // For replicas: we are recovering a backup from a primary
-                RecoveryState.Type type = shardRouting.primary() ? RecoveryState.Type.PRIMARY_RELOCATION : RecoveryState.Type.REPLICA;
-                RecoveryState recoveryState = new RecoveryState(indexShard.shardId(), shardRouting.primary(), type, sourceNode, nodes.getLocalNode());
-                indexShard.markAsRecovering("from " + sourceNode, recoveryState);
-                recoveryTargetService.startRecovery(indexShard, type, sourceNode, new PeerRecoveryListener(shardRouting, indexService, indexMetaData));
-            } catch (Throwable e) {
-                indexShard.failShard("corrupted preexisting index", e);
-                handleRecoveryFailure(indexService, shardRouting, true, e);
-            }
-        } else if (restoreSource == null) {
-            assert indexShard.routingEntry().equals(shardRouting); // should have already be done before
-            // recover from filesystem store
-            final RecoveryState recoveryState = new RecoveryState(indexShard.shardId(), shardRouting.primary(),
-                RecoveryState.Type.STORE,
-                nodes.getLocalNode(), nodes.getLocalNode());
-            indexShard.markAsRecovering("from store", recoveryState); // mark the shard as recovering on the cluster state thread
-            threadPool.generic().execute(() -> {
-                try {
-                    if (indexShard.recoverFromStore(nodes.getLocalNode())) {
-                        shardStateAction.shardStarted(shardRouting, "after recovery from store", SHARD_STATE_ACTION_LISTENER);
-                    }
-                } catch (Throwable t) {
-                    handleRecoveryFailure(indexService, shardRouting, true, t);
-                }
-
-            });
-        } else {
-            // recover from a restore
-            final RecoveryState recoveryState = new RecoveryState(indexShard.shardId(), shardRouting.primary(),
-                RecoveryState.Type.SNAPSHOT, shardRouting.restoreSource(), nodes.getLocalNode());
-            indexShard.markAsRecovering("from snapshot", recoveryState); // mark the shard as recovering on the cluster state thread
-            threadPool.generic().execute(() -> {
-                final ShardId sId = indexShard.shardId();
-                try {
-                    final IndexShardRepository indexShardRepository = repositoriesService.indexShardRepository(restoreSource.snapshotId().getRepository());
-                    if (indexShard.restoreFromRepository(indexShardRepository, nodes.getLocalNode())) {
-                        restoreService.indexShardRestoreCompleted(restoreSource.snapshotId(), sId);
-                        shardStateAction.shardStarted(shardRouting, "after recovery from repository", SHARD_STATE_ACTION_LISTENER);
-                    }
-                } catch (Throwable first) {
-                    try {
-                        if (Lucene.isCorruptionException(first)) {
-                            restoreService.failRestore(restoreSource.snapshotId(), sId);
-                        }
-                    } catch (Throwable second) {
-                        first.addSuppressed(second);
-                    } finally {
-                        handleRecoveryFailure(indexService, shardRouting, true, first);
-                    }
-                }
-            });
-        }
+        indexShard.startRecovery(nodes.getLocalNode(), sourceNode, recoveryTargetService,
+            new RecoveryListener(shardRouting, indexService), repositoriesService);
     }
 
     /**
      * Finds the routing source node for peer recovery, return null if its not found. Note, this method expects the shard
-     * routing to *require* peer recovery, use {@link #isPeerRecovery(org.elasticsearch.cluster.routing.ShardRouting)} to
+     * routing to *require* peer recovery, use {@link ShardRouting#isPeerRecovery()} to
      * check if its needed or not.
      */
-    private DiscoveryNode findSourceNodeForPeerRecovery(RoutingTable routingTable, DiscoveryNodes nodes, ShardRouting shardRouting) {
+    private static DiscoveryNode findSourceNodeForPeerRecovery(ESLogger logger, RoutingTable routingTable, DiscoveryNodes nodes, ShardRouting shardRouting) {
         DiscoveryNode sourceNode = null;
         if (!shardRouting.primary()) {
-            IndexShardRoutingTable shardRoutingTable = routingTable.index(shardRouting.index()).shard(shardRouting.id());
-            for (ShardRouting entry : shardRoutingTable) {
-                if (entry.primary() && entry.active()) {
-                    // only recover from started primary, if we can't find one, we will do it next round
-                    sourceNode = nodes.get(entry.currentNodeId());
-                    if (sourceNode == null) {
-                        logger.trace("can't find replica source node because primary shard {} is assigned to an unknown node.", entry);
-                        return null;
-                    }
-                    break;
+            ShardRouting primary = routingTable.shardRoutingTable(shardRouting.shardId()).primaryShard();
+            // only recover from started primary, if we can't find one, we will do it next round
+            if (primary.active()) {
+                sourceNode = nodes.get(primary.currentNodeId());
+                if (sourceNode == null) {
+                    logger.trace("can't find replica source node because primary shard {} is assigned to an unknown node.", primary);
                 }
-            }
-
-            if (sourceNode == null) {
-                logger.trace("can't find replica source node for {} because a primary shard can not be found.", shardRouting.shardId());
+            } else {
+                logger.trace("can't find replica source node because primary shard {} is not active.", primary);
             }
         } else if (shardRouting.relocatingNodeId() != null) {
             sourceNode = nodes.get(shardRouting.relocatingNodeId());
@@ -711,30 +643,49 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
         return sourceNode;
     }
 
-    private boolean isPeerRecovery(ShardRouting shardRouting) {
-        return !shardRouting.primary() || shardRouting.relocatingNodeId() != null;
-    }
-
-    private class PeerRecoveryListener implements RecoveryTargetService.RecoveryListener {
+    private class RecoveryListener implements RecoveryTargetService.RecoveryListener {
 
         private final ShardRouting shardRouting;
         private final IndexService indexService;
-        private final IndexMetaData indexMetaData;
 
-        private PeerRecoveryListener(ShardRouting shardRouting, IndexService indexService, IndexMetaData indexMetaData) {
+        private RecoveryListener(ShardRouting shardRouting, IndexService indexService) {
             this.shardRouting = shardRouting;
             this.indexService = indexService;
-            this.indexMetaData = indexMetaData;
         }
 
         @Override
         public void onRecoveryDone(RecoveryState state) {
-            shardStateAction.shardStarted(shardRouting, "after recovery (replica) from node [" + state.getSourceNode() + "]", SHARD_STATE_ACTION_LISTENER);
+            if (state.getType() == RecoveryState.Type.SNAPSHOT) {
+                restoreService.indexShardRestoreCompleted(state.getRestoreSource().snapshotId(), shardRouting.shardId());
+            }
+            shardStateAction.shardStarted(shardRouting, message(state), SHARD_STATE_ACTION_LISTENER);
+        }
+
+        private String message(RecoveryState state) {
+            switch (state.getType()) {
+                case SNAPSHOT: return "after recovery from repository";
+                case STORE: return "after recovery from store";
+                case PRIMARY_RELOCATION: return "after recovery (primary relocation) from node [" + state.getSourceNode() + "]";
+                case REPLICA: return "after recovery (replica) from node [" + state.getSourceNode() + "]";
+                default: throw new IllegalArgumentException(state.getType().name());
+            }
         }
 
         @Override
         public void onRecoveryFailure(RecoveryState state, RecoveryFailedException e, boolean sendShardFailure) {
-            handleRecoveryFailure(indexService, shardRouting, sendShardFailure, e);
+            if (state.getType() == RecoveryState.Type.SNAPSHOT) {
+                try {
+                    if (Lucene.isCorruptionException(e.getCause())) {
+                        restoreService.failRestore(state.getRestoreSource().snapshotId(), shardRouting.shardId());
+                    }
+                } catch (Throwable inner) {
+                    e.addSuppressed(inner);
+                } finally {
+                    handleRecoveryFailure(indexService, shardRouting, sendShardFailure, e);
+                }
+            } else {
+                handleRecoveryFailure(indexService, shardRouting, sendShardFailure, e);
+            }
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java
index 8494939e46d..34bd35dfeda 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java
@@ -136,37 +136,18 @@ public class RecoveriesCollection {
         return onGoingRecoveries.size();
     }
 
-    /** cancel all ongoing recoveries for the given shard. typically because the shards is closed */
-    public boolean cancelRecoveriesForShard(ShardId shardId, String reason) {
-        return cancelRecoveriesForShard(shardId, reason, status -> true);
-    }
-
     /**
-     * cancel all ongoing recoveries for the given shard, if their status match a predicate
+     * cancel all ongoing recoveries for the given shard
      *
      * @param reason       reason for cancellation
      * @param shardId      shardId for which to cancel recoveries
-     * @param shouldCancel a predicate to check if a recovery should be cancelled or not.
-     *                     Note that the recovery state can change after this check, but before it is being cancelled via other
-     *                     already issued outstanding references.
      * @return true if a recovery was cancelled
      */
-    public boolean cancelRecoveriesForShard(ShardId shardId, String reason, Predicate<RecoveryTarget> shouldCancel) {
+    public boolean cancelRecoveriesForShard(ShardId shardId, String reason) {
         boolean cancelled = false;
         for (RecoveryTarget status : onGoingRecoveries.values()) {
             if (status.shardId().equals(shardId)) {
-                boolean cancel = false;
-                // if we can't increment the status, the recovery is not there any more.
-                if (status.tryIncRef()) {
-                    try {
-                        cancel = shouldCancel.test(status);
-                    } finally {
-                        status.decRef();
-                    }
-                }
-                if (cancel && cancelRecovery(status.recoveryId(), reason)) {
-                    cancelled = true;
-                }
+                cancelled |= cancelRecovery(status.recoveryId(), reason);
             }
         }
         return cancelled;
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetService.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetService.java
index f0a0b13b872..53feb8e6135 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetService.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetService.java
@@ -124,13 +124,10 @@ public class RecoveryTargetService extends AbstractComponent implements IndexEve
      *
      * @param reason       reason for cancellation
      * @param shardId      shardId for which to cancel recoveries
-     * @param shouldCancel a predicate to check if a recovery should be cancelled or not. Null means cancel without an extra check.
-     *                     note that the recovery state can change after this check, but before it is being cancelled via other
-     *                     already issued outstanding references.
      * @return true if a recovery was cancelled
      */
-    public boolean cancelRecoveriesForShard(ShardId shardId, String reason, @Nullable Predicate<RecoveryTarget> shouldCancel) {
-        return onGoingRecoveries.cancelRecoveriesForShard(shardId, reason, shouldCancel);
+    public boolean cancelRecoveriesForShard(ShardId shardId, String reason) {
+        return onGoingRecoveries.cancelRecoveriesForShard(shardId, reason);
     }
 
     public void startRecovery(final IndexShard indexShard, final RecoveryState.Type recoveryType, final DiscoveryNode sourceNode, final
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 551b6389b84..52c16dd74d9 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -980,8 +980,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         assertEquals(translogOps, newShard.recoveryState().getTranslog().totalOperations());
         assertEquals(translogOps, newShard.recoveryState().getTranslog().totalOperationsOnStart());
         assertEquals(100.0f, newShard.recoveryState().getTranslog().recoveredPercent(), 0.01f);
-        routing = ShardRoutingHelper.moveToStarted(routing);
-        newShard.updateRoutingEntry(routing, true);
+        newShard.updateRoutingEntry(routing.moveToStarted(), true);
         SearchResponse response = client().prepareSearch().get();
         assertHitCount(response, 1);
     }
@@ -1009,8 +1008,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         assertEquals(0, newShard.recoveryState().getTranslog().totalOperations());
         assertEquals(0, newShard.recoveryState().getTranslog().totalOperationsOnStart());
         assertEquals(100.0f, newShard.recoveryState().getTranslog().recoveredPercent(), 0.01f);
-        routing = ShardRoutingHelper.moveToStarted(routing);
-        newShard.updateRoutingEntry(routing, true);
+        newShard.updateRoutingEntry(getInitializingShardRouting(routing).moveToStarted(), true);
         SearchResponse response = client().prepareSearch().get();
         assertHitCount(response, 0);
     }
@@ -1060,8 +1058,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode));
         assertTrue("recover even if there is nothing to recover", newShard.recoverFromStore(localNode));
 
-        routing = ShardRoutingHelper.moveToStarted(routing);
-        newShard.updateRoutingEntry(routing, true);
+        newShard.updateRoutingEntry(getInitializingShardRouting(routing).moveToStarted(), true);
         SearchResponse response = client().prepareSearch().get();
         assertHitCount(response, 0);
         // we can't issue this request through a client because of the inconsistencies we created with the cluster state
@@ -1147,8 +1144,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
             }
         }, localNode));
 
-        routing = ShardRoutingHelper.moveToStarted(routing);
-        test_target_shard.updateRoutingEntry(routing, true);
+        test_target_shard.updateRoutingEntry(routing.moveToStarted(), true);
         assertHitCount(client().prepareSearch("test_target").get(), 1);
         assertSearchHits(client().prepareSearch("test_target").get(), "0");
     }
@@ -1355,7 +1351,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         };
         final IndexShard newShard = newIndexShard(indexService, shard, wrapper, listener);
         shardRef.set(newShard);
-        recoverShard(newShard, shard.routingEntry());
+        recoverShard(newShard);
 
         try {
             ExceptionsHelper.rethrowAndSuppress(failures);
@@ -1396,22 +1392,20 @@ public class IndexShardTests extends ESSingleNodeTestCase {
 
     public static final IndexShard reinitWithWrapper(IndexService indexService, IndexShard shard, IndexSearcherWrapper wrapper, IndexingOperationListener... listeners) throws IOException {
         IndexShard newShard = newIndexShard(indexService, shard, wrapper, listeners);
-        return recoverShard(newShard, shard.routingEntry());
+        return recoverShard(newShard);
     }
 
-    public  static final IndexShard recoverShard(IndexShard newShard, ShardRouting oldRouting) throws IOException {
-        ShardRouting routing = ShardRoutingHelper.reinit(oldRouting);
-        newShard.updateRoutingEntry(routing, false);
+    public  static final IndexShard recoverShard(IndexShard newShard) throws IOException {
         DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
-        newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode));
+        newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), newShard.routingEntry().primary(), RecoveryState.Type.STORE, localNode, localNode));
         assertTrue(newShard.recoverFromStore(localNode));
-        routing = ShardRoutingHelper.moveToStarted(routing);
-        newShard.updateRoutingEntry(routing, true);
+        newShard.updateRoutingEntry(newShard.routingEntry().moveToStarted(), true);
         return newShard;
     }
 
     public  static final IndexShard newIndexShard(IndexService indexService, IndexShard shard, IndexSearcherWrapper wrapper, IndexingOperationListener... listeners) throws IOException {
-        IndexShard newShard = new IndexShard(shard.shardId(), indexService.getIndexSettings(), shard.shardPath(),
+        ShardRouting initializingShardRouting = getInitializingShardRouting(shard.routingEntry());
+        IndexShard newShard = new IndexShard(initializingShardRouting, indexService.getIndexSettings(), shard.shardPath(),
             shard.store(), indexService.cache(), indexService.mapperService(), indexService.similarityService(),
             indexService.fieldData(), shard.getEngineFactory(), indexService.getIndexEventListener(), wrapper,
             indexService.getThreadPool(), indexService.getBigArrays(), null, Collections.emptyList(), Arrays.asList(listeners)
@@ -1419,6 +1413,14 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         return newShard;
     }
 
+    private static ShardRouting getInitializingShardRouting(ShardRouting existingShardRouting) {
+        ShardRouting shardRouting = TestShardRouting.newShardRouting(existingShardRouting.shardId(),
+            existingShardRouting.currentNodeId(), null, existingShardRouting.primary(), ShardRoutingState.INITIALIZING,
+            existingShardRouting.allocationId());
+        shardRouting = shardRouting.updateUnassignedInfo(new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, "fake recovery"));
+        return shardRouting;
+    }
+
     public void testTranslogRecoverySyncsTranslog() throws IOException {
         createIndex("testindexfortranslogsync");
         client().admin().indices().preparePutMapping("testindexfortranslogsync").setType("testtype").setSource(jsonBuilder().startObject()
@@ -1432,7 +1434,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         IndexService test = indicesService.indexService(resolveIndex("testindexfortranslogsync"));
         IndexShard shard = test.getShardOrNull(0);
-        ShardRouting routing = shard.routingEntry();
+        ShardRouting routing = getInitializingShardRouting(shard.routingEntry());
         test.removeShard(0, "b/c britta says so");
         IndexShard newShard = test.createShard(routing);
         DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
@@ -1459,7 +1461,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         IndexService test = indicesService.indexService(resolveIndex("index"));
         IndexShard shard = test.getShardOrNull(0);
-        ShardRouting routing = shard.routingEntry();
+        ShardRouting routing = getInitializingShardRouting(shard.routingEntry());
         test.removeShard(0, "b/c britta says so");
         IndexShard newShard = test.createShard(routing);
         newShard.shardRouting = routing;
@@ -1488,10 +1490,9 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         IndexService test = indicesService.indexService(resolveIndex("index"));
         IndexShard shard = test.getShardOrNull(0);
-        ShardRouting routing = shard.routingEntry();
+        ShardRouting routing = getInitializingShardRouting(shard.routingEntry());
         test.removeShard(0, "b/c britta says so");
         IndexShard newShard = test.createShard(routing);
-        newShard.shardRouting = routing;
         DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
         newShard.markAsRecovering("for testing", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.REPLICA, localNode, localNode));
         // Shard is still inactive since we haven't started recovering yet
diff --git a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
index efd42f7f260..4b11bb1bc96 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
@@ -448,15 +448,14 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase {
         shardRef.set(newShard);
         try {
             assertEquals(0, imc.availableShards().size());
-            ShardRouting routing = ShardRoutingHelper.reinit(shard.routingEntry());
-            newShard.updateRoutingEntry(routing, false);
+            ShardRouting routing = newShard.routingEntry();
             DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
             newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode));
 
             assertEquals(1, imc.availableShards().size());
             assertTrue(newShard.recoverFromStore(localNode));
             assertTrue("we should have flushed in IMC at least once but did: " + flushes.get(), flushes.get() >= 1);
-            newShard.updateRoutingEntry(ShardRoutingHelper.moveToStarted(routing), true);
+            newShard.updateRoutingEntry(routing.moveToStarted(), true);
         } finally {
             newShard.close("simon says", false);
         }
diff --git a/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java b/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java
index 65bec885ea8..2564b31488b 100644
--- a/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java
+++ b/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java
@@ -104,7 +104,7 @@ public class RecoveriesCollectionTests extends ESSingleNodeTestCase {
 
     }
 
-    public void testRecoveryCancellationNoPredicate() throws Exception {
+    public void testRecoveryCancellation() throws Exception {
         createIndex();
         final RecoveriesCollection collection = new RecoveriesCollection(logger, getInstanceFromNode(ThreadPool.class));
         final long recoveryId = startRecovery(collection);
@@ -119,39 +119,6 @@ public class RecoveriesCollectionTests extends ESSingleNodeTestCase {
         }
     }
 
-    public void testRecoveryCancellationPredicate() throws Exception {
-        createIndex();
-        final RecoveriesCollection collection = new RecoveriesCollection(logger, getInstanceFromNode(ThreadPool.class));
-        final long recoveryId = startRecovery(collection);
-        final long recoveryId2 = startRecovery(collection);
-        final ArrayList<AutoCloseable> toClose = new ArrayList<>();
-        try {
-            RecoveriesCollection.RecoveryRef recoveryRef = collection.getRecovery(recoveryId);
-            toClose.add(recoveryRef);
-            ShardId shardId = recoveryRef.status().shardId();
-            assertFalse("should not have cancelled recoveries", collection.cancelRecoveriesForShard(shardId, "test", status -> false));
-            final Predicate<RecoveryTarget> shouldCancel = status -> status.recoveryId() == recoveryId;
-            assertTrue("failed to cancel recoveries", collection.cancelRecoveriesForShard(shardId, "test", shouldCancel));
-            assertThat("we should still have on recovery", collection.size(), equalTo(1));
-            recoveryRef = collection.getRecovery(recoveryId);
-            toClose.add(recoveryRef);
-            assertNull("recovery should have been deleted", recoveryRef);
-            recoveryRef = collection.getRecovery(recoveryId2);
-            toClose.add(recoveryRef);
-            assertNotNull("recovery should NOT have been deleted", recoveryRef);
-
-        } finally {
-            // TODO: do we want a lucene IOUtils version of this?
-            for (AutoCloseable closeable : toClose) {
-                if (closeable != null) {
-                    closeable.close();
-                }
-            }
-            collection.cancelRecovery(recoveryId, "meh");
-            collection.cancelRecovery(recoveryId2, "meh");
-        }
-    }
-
     protected void createIndex() {
         createIndex("test",
                 Settings.builder()

From 8016a2f951487ec7fa177e2d5f0e016e8e24315f Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Wed, 18 May 2016 10:03:38 +0200
Subject: [PATCH 0313/1311] Switch to keyword/text type, add refresh parameter

---
 docs/reference/search/request/sort.asciidoc | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/docs/reference/search/request/sort.asciidoc b/docs/reference/search/request/sort.asciidoc
index 36252d3caa2..2f2e8882032 100644
--- a/docs/reference/search/request/sort.asciidoc
+++ b/docs/reference/search/request/sort.asciidoc
@@ -16,12 +16,10 @@ PUT /my_index
             "properties": {
                 "post_date": { "type": "date" },
                 "user": { 
-                    "type": "string",
-                    "fielddata": "true"
+                    "type": "keyword"
                 },
                 "name": {
-                    "type": "string",
-                    "fielddata": "true"
+                    "type": "keyword"
                 },
                 "age": { "type": "integer" }
             }
@@ -94,7 +92,7 @@ the average price per document.
 
 [source,js]
 --------------------------------------------------
-PUT /my_index/my_type/1
+PUT /my_index/my_type/1?refresh
 {
    "product": "chocolate",
     "price": [20, 4]

From 947daf68d1efe1406f2a5e7b23834032cd1991fc Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Wed, 18 May 2016 11:54:41 +0200
Subject: [PATCH 0314/1311] Add CONSOLE to from/size docs

Relates to #18160
---
 docs/reference/search/request/from-size.asciidoc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/reference/search/request/from-size.asciidoc b/docs/reference/search/request/from-size.asciidoc
index 2e170dc2604..1c44a7ca8d2 100644
--- a/docs/reference/search/request/from-size.asciidoc
+++ b/docs/reference/search/request/from-size.asciidoc
@@ -12,6 +12,7 @@ defaults to `10`.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "from" : 0, "size" : 10,
     "query" : {
@@ -19,6 +20,8 @@ defaults to `10`.
     }
 }
 --------------------------------------------------
+// CONSOLE
+
 
 Note that `from` + `size` can not be more than the `index.max_result_window`
 index setting which defaults to 10,000. See the <<search-request-scroll,Scroll>> or <<search-request-search-after,Search After>>

From de3e7d161f032e7812a90389a956e2f4c9b72595 Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@elastic.co>
Date: Wed, 18 May 2016 12:10:13 +0200
Subject: [PATCH 0315/1311] Add tests for null precondition check in
 BulkRequest

Relates #18347

Checked with @javanna
---
 .../java/org/elasticsearch/action/bulk/BulkRequest.java    | 1 +
 .../org/elasticsearch/action/bulk/BulkRequestTests.java    | 7 +++++++
 2 files changed, 8 insertions(+)

diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
index 56164e3877e..a5775656475 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
@@ -169,6 +169,7 @@ public class BulkRequest extends ActionRequest<BulkRequest> implements Composite
     }
 
     public BulkRequest add(DeleteRequest request, @Nullable Object payload) {
+        Objects.requireNonNull(request, "'request' must not be null");
         requests.add(request);
         addPayload(payload);
         sizeInBytes += REQUEST_OVERHEAD;
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
index 13175ac48f8..f2afc1e7f6e 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
@@ -210,4 +210,11 @@ public class BulkRequestTests extends ESTestCase {
                 "script or doc is missing",
                 "source is missing"));
     }
+
+    public void testCannotAddNullRequests() throws Exception {
+        BulkRequest bulkRequest = new BulkRequest();
+        expectThrows(NullPointerException.class, () -> bulkRequest.add((IndexRequest) null));
+        expectThrows(NullPointerException.class, () -> bulkRequest.add((UpdateRequest) null));
+        expectThrows(NullPointerException.class, () -> bulkRequest.add((DeleteRequest) null));
+    }
 }

From 148f9af5857f287666aead37f249f204a870ab39 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 17 May 2016 22:35:48 -0400
Subject: [PATCH 0316/1311] Convert search-after tests to // CONSOLE

Relates to #18160
---
 .../search/request/search-after.asciidoc      | 20 ++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/docs/reference/search/request/search-after.asciidoc b/docs/reference/search/request/search-after.asciidoc
index bbed3eb097d..bc94af935c2 100644
--- a/docs/reference/search/request/search-after.asciidoc
+++ b/docs/reference/search/request/search-after.asciidoc
@@ -11,21 +11,22 @@ The idea is to use the results from the previous page to help the retrieval of t
 Suppose that the query to retrieve the first page looks like this:
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/twitter/tweet/_search'
+GET twitter/tweet/_search
 {
-    size: "10"
+    "size": 10,
     "query": {
         "match" : {
             "title" : "elasticsearch"
         }
     },
     "sort": [
-        {"age": "asc"},
+        {"date": "asc"},
         {"_uid": "desc"}
     ]
 }
-'
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 NOTE: A field with one unique value per document should be used as the tiebreaker of the sort specification.
 Otherwise the sort order for documents that have the same sort values would be undefined. The recommended way is to use
@@ -38,22 +39,23 @@ For instance we can use the `sort values` of the last document and pass it to `s
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/twitter/tweet/_search'
+GET twitter/tweet/_search
 {
-    "size": 10
+    "size": 10,
     "query": {
         "match" : {
             "title" : "elasticsearch"
         }
     },
-    "search_after": [18, "tweet#654323"],
+    "search_after": [1463538857, "tweet#654323"],
     "sort": [
-        {"age": "asc"},
+        {"date": "asc"},
         {"_uid": "desc"}
     ]
 }
-'
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 NOTE: The parameter `from` must be set to 0 (or -1) when `search_after` is used.
 

From de321fb159f4e9f764d3fe6f1c03c664bf15b384 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Wed, 18 May 2016 13:02:55 +0200
Subject: [PATCH 0317/1311] Disabling nodes.stats/30_discovery.yaml rest test

This is tracked in issue #18433, so temporarily disabling
the tests.
---
 .../rest-api-spec/test/nodes.stats/30_discovery.yaml          | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/30_discovery.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/30_discovery.yaml
index 2617f76941c..4769465eb1d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/30_discovery.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/30_discovery.yaml
@@ -1,5 +1,9 @@
 ---
 "Discovery stats":
+  - skip:
+      version:     "5.0.0 - "
+      reason:      Tracked in issue 18433 
+
   - do:
       cluster.state: {}
 

From f22f3c7df5e0c692e07a7841d0b43aee24a151ee Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Wed, 18 May 2016 13:15:19 +0200
Subject: [PATCH 0318/1311] Add CONSOLE to several trivial search request docs.

Relates to #18160

Touches explain, fielddata-fields, fields, index-boost, min-score,
named-queries-and-filters, query
---
 docs/reference/search/request/explain.asciidoc |  2 ++
 .../search/request/fielddata-fields.asciidoc   |  4 +++-
 docs/reference/search/request/fields.asciidoc  |  4 ++++
 .../search/request/index-boost.asciidoc        |  2 ++
 .../search/request/min-score.asciidoc          |  2 ++
 .../request/named-queries-and-filters.asciidoc | 18 +++++++++++-------
 docs/reference/search/request/query.asciidoc   |  2 ++
 7 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/docs/reference/search/request/explain.asciidoc b/docs/reference/search/request/explain.asciidoc
index 81dc110c263..9bcaecb4840 100644
--- a/docs/reference/search/request/explain.asciidoc
+++ b/docs/reference/search/request/explain.asciidoc
@@ -5,6 +5,7 @@ Enables explanation for each hit on how its score was computed.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "explain": true,
     "query" : {
@@ -12,3 +13,4 @@ Enables explanation for each hit on how its score was computed.
     }
 }
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/search/request/fielddata-fields.asciidoc b/docs/reference/search/request/fielddata-fields.asciidoc
index aaaa606980e..f3a3508b144 100644
--- a/docs/reference/search/request/fielddata-fields.asciidoc
+++ b/docs/reference/search/request/fielddata-fields.asciidoc
@@ -6,13 +6,15 @@ example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "query" : {
-        ...
+        "match_all": {}
     },
     "fielddata_fields" : ["test1", "test2"]
 }
 --------------------------------------------------
+// CONSOLE
 
 Field data fields can work on fields that are not stored.
 
diff --git a/docs/reference/search/request/fields.asciidoc b/docs/reference/search/request/fields.asciidoc
index e929928d427..3483d470ee2 100644
--- a/docs/reference/search/request/fields.asciidoc
+++ b/docs/reference/search/request/fields.asciidoc
@@ -11,6 +11,7 @@ by a search hit.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "fields" : ["user", "postDate"],
     "query" : {
@@ -18,6 +19,7 @@ by a search hit.
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 `*` can be used to load all stored fields from the document.
 
@@ -26,6 +28,7 @@ returned, for example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "fields" : [],
     "query" : {
@@ -33,6 +36,7 @@ returned, for example:
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 
 For backwards compatibility, if the fields parameter specifies fields which are not stored (`store` mapping set to
diff --git a/docs/reference/search/request/index-boost.asciidoc b/docs/reference/search/request/index-boost.asciidoc
index 29d1da3885c..bf766ce8a8c 100644
--- a/docs/reference/search/request/index-boost.asciidoc
+++ b/docs/reference/search/request/index-boost.asciidoc
@@ -8,6 +8,7 @@ graph where each user has an index).
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "indices_boost" : {
         "index1" : 1.4,
@@ -15,3 +16,4 @@ graph where each user has an index).
     }
 }
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/search/request/min-score.asciidoc b/docs/reference/search/request/min-score.asciidoc
index f5a212ebf8e..d9dbef99ddf 100644
--- a/docs/reference/search/request/min-score.asciidoc
+++ b/docs/reference/search/request/min-score.asciidoc
@@ -6,6 +6,7 @@ in `min_score`:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "min_score": 0.5,
     "query" : {
@@ -13,6 +14,7 @@ in `min_score`:
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Note, most times, this does not make much sense, but is provided for
 advanced use cases.
diff --git a/docs/reference/search/request/named-queries-and-filters.asciidoc b/docs/reference/search/request/named-queries-and-filters.asciidoc
index 96d7c1357a9..f8be0f1be69 100644
--- a/docs/reference/search/request/named-queries-and-filters.asciidoc
+++ b/docs/reference/search/request/named-queries-and-filters.asciidoc
@@ -5,21 +5,25 @@ Each filter and query can accept a `_name` in its top level definition.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "should" : [
+    "query": {
+        "bool" : {
+            "should" : [
             {"match" : { "name.first" : {"query" : "shay", "_name" : "first"} }},
             {"match" : { "name.last" : {"query" : "banon", "_name" : "last"} }}
-        ],
-        "filter" : {
-            "terms" : {
-                "name.last" : ["banon", "kimchy"],
-                "_name" : "test"
+            ],
+            "filter" : {
+                "terms" : {
+                    "name.last" : ["banon", "kimchy"],
+                    "_name" : "test"
+                }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 The search response will include for each hit the `matched_queries` it matched on. The tagging of queries and filters
 only make sense for the `bool` query.
diff --git a/docs/reference/search/request/query.asciidoc b/docs/reference/search/request/query.asciidoc
index e496320bd97..fa06d0d9bb4 100644
--- a/docs/reference/search/request/query.asciidoc
+++ b/docs/reference/search/request/query.asciidoc
@@ -6,9 +6,11 @@ query using the <<query-dsl,Query DSL>>.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "query" : {
         "term" : { "user" : "kimchy" }
     }
 }
 --------------------------------------------------
+// CONSOLE

From c03dd8a290d0ff7639895510482b1b9062d6ed25 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Wed, 18 May 2016 13:33:25 +0200
Subject: [PATCH 0319/1311] Make the index-too-old exception more explicit
 (#18438)

Closes #18418
---
 .../metadata/MetaDataIndexUpgradeService.java        | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
index 1cc6f5a44cc..4536670872d 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
@@ -88,15 +88,13 @@ public class MetaDataIndexUpgradeService extends AbstractComponent {
     }
 
     /**
-     * Elasticsearch 3.0 no longer supports indices with pre Lucene v5.0 (Elasticsearch v2.0.0.beta1) segments. All indices
-     * that were created before Elasticsearch v2.0.0.beta1 should be upgraded using upgrade API before they can
-     * be open by this version of elasticsearch.
-     */
+     * Elasticsearch 5.0 no longer supports indices with pre Lucene v5.0 (Elasticsearch v2.0.0.beta1) segments. All indices
+     * that were created before Elasticsearch v2.0.0.beta1 should be reindexed in Elasticsearch 2.x
+     * before they can be opened by this version of elasticsearch.     */
     private void checkSupportedVersion(IndexMetaData indexMetaData) {
         if (indexMetaData.getState() == IndexMetaData.State.OPEN && isSupportedVersion(indexMetaData) == false) {
-            throw new IllegalStateException("The index [" + indexMetaData.getIndex() + "] was created before v2.0.0.beta1 and wasn't upgraded."
-                    + " This index should be opened using a version before " + Version.CURRENT.minimumCompatibilityVersion()
-                    + " and upgraded using the upgrade API.");
+            throw new IllegalStateException("The index [" + indexMetaData.getIndex() + "] was created before v2.0.0.beta1."
+                    + " It should be reindexed in Elasticsearch 2.x before upgrading to " + Version.CURRENT + ".");
         }
     }
 

From 125b715e4508555b31bef7c8a56388ddf8804b0e Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Wed, 18 May 2016 13:36:19 +0200
Subject: [PATCH 0320/1311] Adds CONSOLE to count api

---
 docs/reference/search/count.asciidoc | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/docs/reference/search/count.asciidoc b/docs/reference/search/count.asciidoc
index 9be219f5e74..859455e89b7 100644
--- a/docs/reference/search/count.asciidoc
+++ b/docs/reference/search/count.asciidoc
@@ -10,15 +10,21 @@ body. Here is an example:
 
 [source,js]
 --------------------------------------------------
-$ curl -XGET 'http://localhost:9200/twitter/tweet/_count?q=user:kimchy'
+PUT /twitter/tweet/1?refresh
+{
+    "user": "kimchy"
+}
 
-$ curl -XGET 'http://localhost:9200/twitter/tweet/_count' -d '
+GET /twitter/tweet/_count?q=user:kimchy
+
+GET /twitter/tweet/_count
 {
     "query" : {
         "term" : { "user" : "kimchy" }
     }
-}'
+}
 --------------------------------------------------
+//CONSOLE
 
 NOTE: The query being sent in the body must be nested in a `query` key, same as
 the <<search-search,search api>> works
@@ -37,6 +43,7 @@ tweets from the twitter index for a certain user. The result is:
     }
 }
 --------------------------------------------------
+// TESTRESPONSE
 
 The query is optional, and when not provided, it will use `match_all` to
 count all the docs.

From a3425b4bf8eacb7cf7c340ce82c54d0596a9131b Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Wed, 18 May 2016 14:31:04 +0200
Subject: [PATCH 0321/1311] Add CONSOLE to post-filter

---
 .../search/request/post-filter.asciidoc       | 45 +++++++++++++++----
 1 file changed, 37 insertions(+), 8 deletions(-)

diff --git a/docs/reference/search/request/post-filter.asciidoc b/docs/reference/search/request/post-filter.asciidoc
index 7bd95400312..493b4261c82 100644
--- a/docs/reference/search/request/post-filter.asciidoc
+++ b/docs/reference/search/request/post-filter.asciidoc
@@ -5,14 +5,43 @@ The `post_filter` is applied to the search `hits` at the very end of a search
 request,  after aggregations have already been calculated. Its purpose is
 best explained by example:
 
-Imagine that you are selling shirts, and the user has specified two filters:
+Imagine that you are selling shirts that have the following properties:
+
+[source,js]
+-------------------------------------------------
+PUT /shirts
+{
+    "mappings": {
+        "item": {
+            "properties": {
+                "brand": { "type": "keyword"},
+                "color": { "type": "keyword"},
+                "model": { "type": "keyword"}
+            }
+        }
+    }
+}
+
+PUT /shirts/item/1?refresh
+{
+    "brand": "gucci",
+    "color": "red",
+    "model": "slim"
+}
+------------------------------------------------
+// CONSOLE
+// TESTSETUP
+
+
+Imagine a user has specified two filters:
+
 `color:red` and `brand:gucci`.  You only want to show them red shirts made by
 Gucci in the search results.  Normally you would do this with a 
 <<query-dsl-bool-query,`bool` query>>:
 
 [source,js]
 --------------------------------------------------
-curl -XGET localhost:9200/shirts/_search -d '
+GET /shirts/_search 
 {
   "query": {
     "bool": {
@@ -23,8 +52,8 @@ curl -XGET localhost:9200/shirts/_search -d '
     }
   }
 }
-'
 --------------------------------------------------
+// CONSOLE
 
 However, you would also like to use _faceted navigation_ to display a list of
 other options that the user could click on.  Perhaps you have a `model` field
@@ -36,7 +65,7 @@ This can be done with a
 
 [source,js]
 --------------------------------------------------
-curl -XGET localhost:9200/shirts/_search -d '
+GET /shirts/_search 
 {
   "query": {
     "bool": {
@@ -52,8 +81,8 @@ curl -XGET localhost:9200/shirts/_search -d '
     }
   }
 }
-'
 --------------------------------------------------
+// CONSOLE
 <1> Returns the most popular models of red shirts by Gucci.
 
 But perhaps you would also like to tell the user how many Gucci shirts are
@@ -67,12 +96,12 @@ the `post_filter`:
 
 [source,js]
 --------------------------------------------------
-curl -XGET localhost:9200/shirts/_search -d '
+GET /shirts/_search
 {
   "query": {
     "bool": {
       "filter": {
-        { "term": { "brand": "gucci" }} <1>
+        "term": { "brand": "gucci" } <1>
       }
     }
   },
@@ -95,8 +124,8 @@ curl -XGET localhost:9200/shirts/_search -d '
     "term": { "color": "red" }
   }
 }
-'
 --------------------------------------------------
+// CONSOLE
 <1> The main query now finds all shirts by Gucci, regardless of color.
 <2> The `colors` agg returns popular colors for shirts by Gucci.
 <3> The `color_red` agg limits the `models` sub-aggregation 

From 0032d4760eb8564c79630054a3a8cd16effda88c Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Wed, 18 May 2016 14:34:22 +0200
Subject: [PATCH 0322/1311] Add CONSOLE to preference docs

---
 docs/reference/search/request/preference.asciidoc | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/docs/reference/search/request/preference.asciidoc b/docs/reference/search/request/preference.asciidoc
index 0d07f29475e..3d6c6b40cb9 100644
--- a/docs/reference/search/request/preference.asciidoc
+++ b/docs/reference/search/request/preference.asciidoc
@@ -56,7 +56,7 @@ for the user:
 
 [source,js]
 ------------------------------------------------
-curl localhost:9200/_search?preference=xyzabc123 -d '
+GET /_search?preference=xyzabc123
 {
     "query": {
         "match": {
@@ -64,7 +64,6 @@ curl localhost:9200/_search?preference=xyzabc123 -d '
         }
     }
 }
-'
 ------------------------------------------------
-
+// CONSOLE
 

From a849cc97ea453d31051a0fe1d98886ecd463893c Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Wed, 18 May 2016 14:38:54 +0200
Subject: [PATCH 0323/1311] Add CONSOLE to script-fields docs

---
 docs/reference/search/request/script-fields.asciidoc | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/docs/reference/search/request/script-fields.asciidoc b/docs/reference/search/request/script-fields.asciidoc
index 596aba31d82..6e054f02e1c 100644
--- a/docs/reference/search/request/script-fields.asciidoc
+++ b/docs/reference/search/request/script-fields.asciidoc
@@ -6,9 +6,10 @@ evaluation>> (based on different fields) for each hit, for example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "query" : {
-        ...
+        "match_all": {}
     },
     "script_fields" : {
         "test1" : {
@@ -25,6 +26,8 @@ evaluation>> (based on different fields) for each hit, for example:
     }
 }
 --------------------------------------------------
+// CONSOLE
+
 
 Script fields can work on fields that are not stored (`my_field_name` in
 the above case), and allow to return custom values to be returned (the
@@ -36,9 +39,10 @@ type). Here is an example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
     {
         "query" : {
-            ...
+            "match_all": {}
         },
         "script_fields" : {
             "test1" : {
@@ -47,6 +51,7 @@ type). Here is an example:
         }
     }
 --------------------------------------------------
+// CONSOLE
 
 Note the `_source` keyword here to navigate the json-like model.
 

From 808ef6cec798fb186c8638dcddec3dc25f4e9067 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Wed, 18 May 2016 14:25:20 +0200
Subject: [PATCH 0324/1311] Fix parsing single `rescore` element in
 SearchSourceBuilder

We are currently only parsing the array-syntax for the rescore part
in SearchSourceBuilder ("rescore" : [ {...}, {...} ]) . We also need
to support "rescore" : {...}

Closes #18439
---
 .../search/builder/SearchSourceBuilder.java   |  3 ++
 .../builder/SearchSourceBuilderTests.java     | 52 +++++++++++++++++++
 2 files changed, 55 insertions(+)

diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index 61f4acb81aa..429aa36e56f 100644
--- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -1033,6 +1033,9 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                     suggestBuilder = SuggestBuilder.fromXContent(context, suggesters);
                 } else if (context.getParseFieldMatcher().match(currentFieldName, SORT_FIELD)) {
                     sorts = new ArrayList<>(SortBuilder.fromXContent(context));
+                } else if (context.getParseFieldMatcher().match(currentFieldName, RESCORE_FIELD)) {
+                    rescoreBuilders = new ArrayList<>();
+                    rescoreBuilders.add(RescoreBuilder.parseFromXContent(context));
                 } else if (context.getParseFieldMatcher().match(currentFieldName, EXT_FIELD)) {
                     XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser);
                     ext = xContentBuilder.bytes();
diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index 9f0113829ce..077d978a4ce 100644
--- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -72,6 +72,7 @@ import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.fetch.source.FetchSourceContext;
 import org.elasticsearch.search.highlight.HighlightBuilderTests;
 import org.elasticsearch.search.rescore.QueryRescoreBuilderTests;
+import org.elasticsearch.search.rescore.QueryRescorerBuilder;
 import org.elasticsearch.search.searchafter.SearchAfterBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.ScoreSortBuilder;
@@ -561,6 +562,57 @@ public class SearchSourceBuilderTests extends ESTestCase {
         }
     }
 
+    /**
+     * test that we can parse the `rescore` element either as single object or as array
+     */
+    public void testParseRescore() throws IOException {
+        {
+            String restContent = "{\n" +
+                "    \"query\" : {\n" +
+                "        \"match\": { \"content\": { \"query\": \"foo bar\" }}\n" +
+                "     },\n" +
+                "    \"rescore\": {" +
+                "        \"window_size\": 50,\n" +
+                "        \"query\": {\n" +
+                "            \"rescore_query\" : {\n" +
+                "                \"match\": { \"content\": { \"query\": \"baz\" } }\n" +
+                "            }\n" +
+                "        }\n" +
+                "    }\n" +
+                "}\n";
+            try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) {
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser),
+                        aggParsers, suggesters);
+                assertEquals(1, searchSourceBuilder.rescores().size());
+                assertEquals(new QueryRescorerBuilder(QueryBuilders.matchQuery("content", "baz")).windowSize(50),
+                        searchSourceBuilder.rescores().get(0));
+            }
+        }
+
+        {
+            String restContent = "{\n" +
+                "    \"query\" : {\n" +
+                "        \"match\": { \"content\": { \"query\": \"foo bar\" }}\n" +
+                "     },\n" +
+                "    \"rescore\": [ {" +
+                "        \"window_size\": 50,\n" +
+                "        \"query\": {\n" +
+                "            \"rescore_query\" : {\n" +
+                "                \"match\": { \"content\": { \"query\": \"baz\" } }\n" +
+                "            }\n" +
+                "        }\n" +
+                "    } ]\n" +
+                "}\n";
+            try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) {
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(createParseContext(parser),
+                        aggParsers, suggesters);
+                assertEquals(1, searchSourceBuilder.rescores().size());
+                assertEquals(new QueryRescorerBuilder(QueryBuilders.matchQuery("content", "baz")).windowSize(50),
+                        searchSourceBuilder.rescores().get(0));
+            }
+        }
+    }
+
     public void testEmptyPostFilter() throws IOException {
         SearchSourceBuilder builder = new SearchSourceBuilder();
         builder.postFilter(new EmptyQueryBuilder());

From d7a31c8cf7f411aadc7202a4a2326679ed820e88 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Wed, 18 May 2016 15:19:30 +0200
Subject: [PATCH 0325/1311] Add missing builder.endObject() in FsInfo

closes #18433
---
 core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java   | 2 ++
 .../rest-api-spec/test/nodes.stats/30_discovery.yaml          | 4 ----
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java b/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java
index caa97ea7387..641dc3a5bb3 100644
--- a/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java
+++ b/core/src/main/java/org/elasticsearch/monitor/fs/FsInfo.java
@@ -396,12 +396,14 @@ public class FsInfo implements Iterable<FsInfo.Path>, Writeable, ToXContent {
                     builder.endObject();
                 }
                 builder.endArray();
+
                 builder.startObject("total");
                 builder.field(OPERATIONS, totalOperations);
                 builder.field(READ_OPERATIONS, totalReadOperations);
                 builder.field(WRITE_OPERATIONS, totalWriteOperations);
                 builder.field(READ_KILOBYTES, totalReadKilobytes);
                 builder.field(WRITE_KILOBYTES, totalWriteKilobytes);
+                builder.endObject();
             }
             return builder;
         }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/30_discovery.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/30_discovery.yaml
index 4769465eb1d..2617f76941c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/30_discovery.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.stats/30_discovery.yaml
@@ -1,9 +1,5 @@
 ---
 "Discovery stats":
-  - skip:
-      version:     "5.0.0 - "
-      reason:      Tracked in issue 18433 
-
   - do:
       cluster.state: {}
 

From c20a669c2d9e4d1e85c2117011abdc5731164c4d Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Wed, 18 May 2016 15:20:21 +0200
Subject: [PATCH 0326/1311] Add CONSOLE to source filtering docs

---
 docs/reference/search/request/source-filtering.asciidoc | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/reference/search/request/source-filtering.asciidoc b/docs/reference/search/request/source-filtering.asciidoc
index 8458d37806c..08625751eec 100644
--- a/docs/reference/search/request/source-filtering.asciidoc
+++ b/docs/reference/search/request/source-filtering.asciidoc
@@ -13,6 +13,7 @@ To disable `_source` retrieval set to `false`:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "_source": false,
     "query" : {
@@ -20,6 +21,7 @@ To disable `_source` retrieval set to `false`:
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 The `_source` also accepts one or more wildcard patterns to control what parts of the `_source` should be returned:
 
@@ -27,6 +29,7 @@ For example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "_source": "obj.*",
     "query" : {
@@ -34,11 +37,13 @@ For example:
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Or
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "_source": [ "obj1.*", "obj2.*" ],
     "query" : {
@@ -46,11 +51,13 @@ Or
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Finally, for complete control, you can specify both include and exclude patterns:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "_source": {
         "include": [ "obj1.*", "obj2.*" ],
@@ -61,3 +68,4 @@ Finally, for complete control, you can specify both include and exclude patterns
     }
 }
 --------------------------------------------------
+// CONSOLE

From cad0608cdb28e2b8485e5c01c26579a35cb84356 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 18 May 2016 09:31:28 -0400
Subject: [PATCH 0327/1311] Add GC overhead logging

This commit adds simple GC overhead logging. This logging captures
intervals where the JVM is spending a lot of time performing GC but it
is not necessarily the case that each GC is large. For a start, this
logging is simple and does not attempt to incorporate whether or not the
collections were efficient (that is, we are only capturing that a lot of
GC is happening, not that a lot of useless GC is happening).

Relates #18419
---
 .../common/settings/ClusterSettings.java      |   3 +
 .../common/settings/Setting.java              |  30 ++--
 .../monitor/jvm/JvmGcMonitorService.java      | 154 +++++++++++++++--
 .../common/settings/SettingTests.java         |   2 +-
 .../jvm/JvmGcMonitorServiceSettingsTests.java |  40 +++++
 .../monitor/jvm/JvmGcMonitorServiceTests.java |  39 +++++
 .../monitor/jvm/JvmMonitorTests.java          | 156 ++++++++++++++++--
 7 files changed, 384 insertions(+), 40 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index dad5f48ce27..e66534a4feb 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -397,6 +397,9 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     JvmGcMonitorService.ENABLED_SETTING,
                     JvmGcMonitorService.REFRESH_INTERVAL_SETTING,
                     JvmGcMonitorService.GC_SETTING,
+                    JvmGcMonitorService.GC_OVERHEAD_WARN_SETTING,
+                    JvmGcMonitorService.GC_OVERHEAD_INFO_SETTING,
+                    JvmGcMonitorService.GC_OVERHEAD_DEBUG_SETTING,
                     PageCacheRecycler.LIMIT_HEAP_SETTING,
                     PageCacheRecycler.WEIGHT_BYTES_SETTING,
                     PageCacheRecycler.WEIGHT_INT_SETTING,
diff --git a/core/src/main/java/org/elasticsearch/common/settings/Setting.java b/core/src/main/java/org/elasticsearch/common/settings/Setting.java
index edd5d511f86..1efb65c18b1 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/Setting.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/Setting.java
@@ -18,19 +18,6 @@
  */
 package org.elasticsearch.common.settings;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.function.BiConsumer;
-import java.util.function.Consumer;
-import java.util.function.Function;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.support.ToXContentToBytes;
@@ -50,6 +37,19 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
 /**
  * A setting. Encapsulates typical stuff like default value, parsing, and scope.
  * Some (SettingsProperty.Dynamic) can by modified at run time using the API.
@@ -504,7 +504,7 @@ public class Setting<T> extends ToXContentToBytes {
             throw new IllegalArgumentException("Failed to parse value [" + s + "] for setting [" + key + "] must be >= " + minValue);
         }
         if (value > maxValue) {
-            throw new IllegalArgumentException("Failed to parse value [" + s + "] for setting [" + key + "] must be =< " + maxValue);
+            throw new IllegalArgumentException("Failed to parse value [" + s + "] for setting [" + key + "] must be <= " + maxValue);
         }
         return value;
     }
@@ -572,7 +572,7 @@ public class Setting<T> extends ToXContentToBytes {
             throw new IllegalArgumentException("Failed to parse value [" + s + "] for setting [" + key + "] must be >= " + minValue);
         }
         if (value.bytes() > maxValue.bytes()) {
-            throw new IllegalArgumentException("Failed to parse value [" + s + "] for setting [" + key + "] must be =< " + maxValue);
+            throw new IllegalArgumentException("Failed to parse value [" + s + "] for setting [" + key + "] must be <= " + maxValue);
         }
         return value;
     }
diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
index 01b4c68537e..ac75eb93aaa 100644
--- a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
@@ -31,6 +31,7 @@ import org.elasticsearch.monitor.jvm.JvmStats.GarbageCollector;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.HashMap;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.ScheduledFuture;
@@ -45,6 +46,7 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent<JvmGcMonitor
     private final boolean enabled;
     private final TimeValue interval;
     private final Map<String, GcThreshold> gcThresholds;
+    private final GcOverheadThreshold gcOverheadThreshold;
 
     private volatile ScheduledFuture scheduledFuture;
 
@@ -57,6 +59,27 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent<JvmGcMonitor
     private static String GC_COLLECTOR_PREFIX = "monitor.jvm.gc.collector.";
     public final static Setting<Settings> GC_SETTING = Setting.groupSetting(GC_COLLECTOR_PREFIX, Property.NodeScope);
 
+    public final static Setting<Integer> GC_OVERHEAD_WARN_SETTING =
+        Setting.intSetting("monitor.jvm.gc.overhead.warn", 50, 0, 100, Property.NodeScope);
+    public final static Setting<Integer> GC_OVERHEAD_INFO_SETTING =
+        Setting.intSetting("monitor.jvm.gc.overhead.info", 25, 0, 100, Property.NodeScope);
+    public final static Setting<Integer> GC_OVERHEAD_DEBUG_SETTING =
+        Setting.intSetting("monitor.jvm.gc.overhead.debug", 10, 0, 100, Property.NodeScope);
+
+    static class GcOverheadThreshold {
+        final int warnThreshold;
+        final int infoThreshold;
+        final int debugThreshold;
+
+        public GcOverheadThreshold(final int warnThreshold, final int infoThreshold, final int debugThreshold) {
+            this.warnThreshold = warnThreshold;
+            this.infoThreshold = infoThreshold;
+            this.debugThreshold = debugThreshold;
+        }
+    }
+
+
+
     static class GcThreshold {
         public final String name;
         public final long warnThreshold;
@@ -102,7 +125,42 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent<JvmGcMonitor
         gcThresholds.putIfAbsent("default", new GcThreshold("default", 10000, 5000, 2000));
         this.gcThresholds = unmodifiableMap(gcThresholds);
 
-        logger.debug("enabled [{}], interval [{}], gc_threshold [{}]", enabled, interval, this.gcThresholds);
+        if (GC_OVERHEAD_WARN_SETTING.get(settings) <= GC_OVERHEAD_INFO_SETTING.get(settings)) {
+            final String message =
+                String.format(
+                    Locale.ROOT,
+                    "[%s] must be greater than [%s] [%d] but was [%d]",
+                    GC_OVERHEAD_WARN_SETTING.getKey(),
+                    GC_OVERHEAD_INFO_SETTING.getKey(),
+                    GC_OVERHEAD_INFO_SETTING.get(settings),
+                    GC_OVERHEAD_WARN_SETTING.get(settings));
+            throw new IllegalArgumentException(message);
+        }
+        if (GC_OVERHEAD_INFO_SETTING.get(settings) <= GC_OVERHEAD_DEBUG_SETTING.get(settings)) {
+            final String message =
+                String.format(
+                    Locale.ROOT,
+                    "[%s] must be greater than [%s] [%d] but was [%d]",
+                    GC_OVERHEAD_INFO_SETTING.getKey(),
+                    GC_OVERHEAD_DEBUG_SETTING.getKey(),
+                    GC_OVERHEAD_DEBUG_SETTING.get(settings),
+                    GC_OVERHEAD_INFO_SETTING.get(settings));
+            throw new IllegalArgumentException(message);
+        }
+
+        this.gcOverheadThreshold = new GcOverheadThreshold(
+            GC_OVERHEAD_WARN_SETTING.get(settings),
+            GC_OVERHEAD_INFO_SETTING.get(settings),
+            GC_OVERHEAD_DEBUG_SETTING.get(settings));
+
+        logger.debug(
+            "enabled [{}], interval [{}], gc_threshold [{}], overhead [{}, {}, {}]",
+            this.enabled,
+            this.interval,
+            this.gcThresholds,
+            this.gcOverheadThreshold.warnThreshold,
+            this.gcOverheadThreshold.infoThreshold,
+            this.gcOverheadThreshold.debugThreshold);
     }
 
     private static TimeValue getValidThreshold(Settings settings, String key, String level) {
@@ -120,15 +178,12 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent<JvmGcMonitor
         return GC_COLLECTOR_PREFIX + key + "." + level;
     }
 
-    private static final String LOG_MESSAGE =
-        "[gc][{}][{}][{}] duration [{}], collections [{}]/[{}], total [{}]/[{}], memory [{}]->[{}]/[{}], all_pools {}";
-
     @Override
     protected void doStart() {
         if (!enabled) {
             return;
         }
-        scheduledFuture = threadPool.scheduleWithFixedDelay(new JvmMonitor(gcThresholds) {
+        scheduledFuture = threadPool.scheduleWithFixedDelay(new JvmMonitor(gcThresholds, gcOverheadThreshold) {
             @Override
             void onMonitorFailure(Throwable t) {
                 logger.debug("failed to monitor", t);
@@ -138,9 +193,17 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent<JvmGcMonitor
             void onSlowGc(final Threshold threshold, final long seq, final SlowGcEvent slowGcEvent) {
                 logSlowGc(logger, threshold, seq, slowGcEvent, JvmGcMonitorService::buildPools);
             }
+
+            @Override
+            void onGcOverhead(final Threshold threshold, final long current, final long elapsed, final long seq) {
+                logGcOverhead(logger, threshold, current, elapsed, seq);
+            }
         }, interval);
     }
 
+    private static final String SLOW_GC_LOG_MESSAGE =
+        "[gc][{}][{}][{}] duration [{}], collections [{}]/[{}], total [{}]/[{}], memory [{}]->[{}]/[{}], all_pools {}";
+
     static void logSlowGc(
         final ESLogger logger,
         final JvmMonitor.Threshold threshold,
@@ -162,7 +225,7 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent<JvmGcMonitor
             case WARN:
                 if (logger.isWarnEnabled()) {
                     logger.warn(
-                        LOG_MESSAGE,
+                        SLOW_GC_LOG_MESSAGE,
                         name,
                         seq,
                         totalGcCollectionCount,
@@ -180,7 +243,7 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent<JvmGcMonitor
             case INFO:
                 if (logger.isInfoEnabled()) {
                     logger.info(
-                        LOG_MESSAGE,
+                        SLOW_GC_LOG_MESSAGE,
                         name,
                         seq,
                         totalGcCollectionCount,
@@ -198,7 +261,7 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent<JvmGcMonitor
             case DEBUG:
                 if (logger.isDebugEnabled()) {
                     logger.debug(
-                        LOG_MESSAGE,
+                        SLOW_GC_LOG_MESSAGE,
                         name,
                         seq,
                         totalGcCollectionCount,
@@ -239,6 +302,33 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent<JvmGcMonitor
         return sb.toString();
     }
 
+    private static final String OVERHEAD_LOG_MESSAGE = "[gc][{}] overhead, spent [{}] collecting in the last [{}]";
+
+    static void logGcOverhead(
+        final ESLogger logger,
+        final JvmMonitor.Threshold threshold,
+        final long current,
+        final long elapsed,
+        final long seq) {
+        switch (threshold) {
+            case WARN:
+                if (logger.isWarnEnabled()) {
+                    logger.warn(OVERHEAD_LOG_MESSAGE, seq, TimeValue.timeValueMillis(current), TimeValue.timeValueMillis(elapsed));
+                }
+                break;
+            case INFO:
+                if (logger.isInfoEnabled()) {
+                    logger.info(OVERHEAD_LOG_MESSAGE, seq, TimeValue.timeValueMillis(current), TimeValue.timeValueMillis(elapsed));
+                }
+                break;
+            case DEBUG:
+                if (logger.isDebugEnabled()) {
+                    logger.debug(OVERHEAD_LOG_MESSAGE, seq, TimeValue.timeValueMillis(current), TimeValue.timeValueMillis(elapsed));
+                }
+                break;
+        }
+    }
+
     @Override
     protected void doStop() {
         if (!enabled) {
@@ -287,16 +377,18 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent<JvmGcMonitor
         private long lastTime = now();
         private JvmStats lastJvmStats = jvmStats();
         private long seq = 0;
-        private final Map<String, GcThreshold> gcThresholds;
+        private final Map<String, JvmGcMonitorService.GcThreshold> gcThresholds;
+        final GcOverheadThreshold gcOverheadThreshold;
 
-        public JvmMonitor(Map<String, GcThreshold> gcThresholds) {
+        public JvmMonitor(final Map<String, GcThreshold> gcThresholds, final GcOverheadThreshold gcOverheadThreshold) {
             this.gcThresholds = Objects.requireNonNull(gcThresholds);
+            this.gcOverheadThreshold = Objects.requireNonNull(gcOverheadThreshold);
         }
 
         @Override
         public void run() {
             try {
-                monitorLongGc();
+                monitorGc();
             } catch (Throwable t) {
                 onMonitorFailure(t);
             }
@@ -304,12 +396,21 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent<JvmGcMonitor
 
         abstract void onMonitorFailure(Throwable t);
 
-        synchronized void monitorLongGc() {
+        synchronized void monitorGc() {
             seq++;
             final long currentTime = now();
             JvmStats currentJvmStats = jvmStats();
 
             final long elapsed = TimeUnit.NANOSECONDS.toMillis(currentTime - lastTime);
+
+            monitorSlowGc(currentJvmStats, elapsed);
+            monitorGcOverhead(currentJvmStats, elapsed);
+
+            lastTime = currentTime;
+            lastJvmStats = currentJvmStats;
+        }
+
+        final void monitorSlowGc(JvmStats currentJvmStats, long elapsed) {
             for (int i = 0; i < currentJvmStats.getGc().getCollectors().length; i++) {
                 GarbageCollector gc = currentJvmStats.getGc().getCollectors()[i];
                 GarbageCollector prevGc = lastJvmStats.getGc().getCollectors()[i];
@@ -350,8 +451,31 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent<JvmGcMonitor
                         JvmInfo.jvmInfo().getMem().getHeapMax()));
                 }
             }
-            lastTime = currentTime;
-            lastJvmStats = currentJvmStats;
+        }
+
+        final void monitorGcOverhead(final JvmStats currentJvmStats, final long elapsed) {
+            long current = 0;
+            for (int i = 0; i < currentJvmStats.getGc().getCollectors().length; i++) {
+                GarbageCollector gc = currentJvmStats.getGc().getCollectors()[i];
+                GarbageCollector prevGc = lastJvmStats.getGc().getCollectors()[i];
+                current += gc.getCollectionTime().millis() - prevGc.getCollectionTime().millis();
+            }
+            checkGcOverhead(current, elapsed, seq);
+        }
+
+        void checkGcOverhead(final long current, final long elapsed, final long seq) {
+            final int fraction = (int) ((100 * current) / (double) elapsed);
+            Threshold overheadThreshold = null;
+            if (fraction >= gcOverheadThreshold.warnThreshold) {
+                overheadThreshold = Threshold.WARN;
+            } else if (fraction >= gcOverheadThreshold.infoThreshold) {
+                overheadThreshold = Threshold.INFO;
+            } else if (fraction >= gcOverheadThreshold.debugThreshold) {
+                overheadThreshold = Threshold.DEBUG;
+            }
+            if (overheadThreshold != null) {
+                onGcOverhead(overheadThreshold, current, elapsed, seq);
+            }
         }
 
         JvmStats jvmStats() {
@@ -364,6 +488,8 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent<JvmGcMonitor
 
         abstract void onSlowGc(final Threshold threshold, final long seq, final SlowGcEvent slowGcEvent);
 
+        abstract void onGcOverhead(final Threshold threshold, final long total, final long elapsed, final long seq);
+
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java
index 8b2dbdb4f07..78319297a98 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java
@@ -430,7 +430,7 @@ public class SettingTests extends ESTestCase {
             integerSetting.get(Settings.builder().put("foo.bar", 11).build());
             fail();
         } catch (IllegalArgumentException ex) {
-            assertEquals("Failed to parse value [11] for setting [foo.bar] must be =< 10", ex.getMessage());
+            assertEquals("Failed to parse value [11] for setting [foo.bar] must be <= 10", ex.getMessage());
         }
 
         try {
diff --git a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java
index 2c248969b2c..97a2b065627 100644
--- a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java
@@ -79,6 +79,46 @@ public class JvmGcMonitorServiceSettingsTests extends ESTestCase {
         }, true, null);
     }
 
+    public void testIllegalOverheadSettings() throws InterruptedException {
+        for (final String threshold : new String[] { "warn", "info", "debug" }) {
+            final Settings.Builder builder = Settings.builder();
+            builder.put("monitor.jvm.gc.overhead." + threshold, randomIntBetween(Integer.MIN_VALUE, -1));
+            execute(builder.build(), (command, interval) -> null, t -> {
+                assertThat(t, instanceOf(IllegalArgumentException.class));
+                assertThat(t.getMessage(), containsString("setting [monitor.jvm.gc.overhead." + threshold + "] must be >= 0"));
+            }, true, null);
+        }
+
+        for (final String threshold : new String[] { "warn", "info", "debug" }) {
+            final Settings.Builder builder = Settings.builder();
+            builder.put("monitor.jvm.gc.overhead." + threshold, randomIntBetween(100 + 1, Integer.MAX_VALUE));
+            execute(builder.build(), (command, interval) -> null, t -> {
+                assertThat(t, instanceOf(IllegalArgumentException.class));
+                assertThat(t.getMessage(), containsString("setting [monitor.jvm.gc.overhead." + threshold + "] must be <= 100"));
+            }, true, null);
+        }
+
+        final Settings.Builder infoWarnOutOfOrderBuilder = Settings.builder();
+        final int info = randomIntBetween(2, 98);
+        infoWarnOutOfOrderBuilder.put("monitor.jvm.gc.overhead.info", info);
+        final int warn = randomIntBetween(1, info - 1);
+        infoWarnOutOfOrderBuilder.put("monitor.jvm.gc.overhead.warn", warn);
+        execute(infoWarnOutOfOrderBuilder.build(), (command, interval) -> null, t -> {
+            assertThat(t, instanceOf(IllegalArgumentException.class));
+            assertThat(t.getMessage(), containsString("[monitor.jvm.gc.overhead.warn] must be greater than [monitor.jvm.gc.overhead.info] [" + info + "] but was [" + warn + "]"));
+        }, true, null);
+
+        final Settings.Builder debugInfoOutOfOrderBuilder = Settings.builder();
+        debugInfoOutOfOrderBuilder.put("monitor.jvm.gc.overhead.info", info);
+        final int debug = randomIntBetween(info + 1, 99);
+        debugInfoOutOfOrderBuilder.put("monitor.jvm.gc.overhead.debug", debug);
+        debugInfoOutOfOrderBuilder.put("monitor.jvm.gc.overhead.warn", randomIntBetween(debug + 1, 100)); // or the test will fail for the wrong reason
+        execute(debugInfoOutOfOrderBuilder.build(), (command, interval) -> null, t -> {
+            assertThat(t, instanceOf(IllegalArgumentException.class));
+            assertThat(t.getMessage(), containsString("[monitor.jvm.gc.overhead.info] must be greater than [monitor.jvm.gc.overhead.debug] [" + debug + "] but was [" + info + "]"));
+        }, true, null);
+    }
+
     private static void execute(Settings settings, BiFunction<Runnable, TimeValue, ScheduledFuture<?>> scheduler, Runnable asserts) throws InterruptedException {
         execute(settings, scheduler, null, false, asserts);
     }
diff --git a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceTests.java b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceTests.java
index 2c17fca7c8c..ab5b1ac4750 100644
--- a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceTests.java
@@ -133,4 +133,43 @@ public class JvmGcMonitorServiceTests extends ESTestCase {
         verifyNoMoreInteractions(logger);
     }
 
+    public void testGcOverheadLogging() {
+        final JvmGcMonitorService.JvmMonitor.Threshold threshold = randomFrom(JvmGcMonitorService.JvmMonitor.Threshold.values());
+        final int current = randomIntBetween(1, Integer.MAX_VALUE);
+        final long elapsed = randomIntBetween(current, Integer.MAX_VALUE);
+        final long seq = randomIntBetween(1, Integer.MAX_VALUE);
+        final ESLogger logger = mock(ESLogger.class);
+        when(logger.isWarnEnabled()).thenReturn(true);
+        when(logger.isInfoEnabled()).thenReturn(true);
+        when(logger.isDebugEnabled()).thenReturn(true);
+        JvmGcMonitorService.logGcOverhead(logger, threshold, current, elapsed, seq);
+        switch(threshold) {
+            case WARN:
+                verify(logger).isWarnEnabled();
+                verify(logger).warn(
+                    "[gc][{}] overhead, spent [{}] collecting in the last [{}]",
+                    seq,
+                    TimeValue.timeValueMillis(current),
+                    TimeValue.timeValueMillis(elapsed));
+                break;
+            case INFO:
+                verify(logger).isInfoEnabled();
+                verify(logger).info(
+                    "[gc][{}] overhead, spent [{}] collecting in the last [{}]",
+                    seq,
+                    TimeValue.timeValueMillis(current),
+                    TimeValue.timeValueMillis(elapsed));
+                break;
+            case DEBUG:
+                verify(logger).isDebugEnabled();
+                verify(logger).debug(
+                    "[gc][{}] overhead, spent [{}] collecting in the last [{}]",
+                    seq,
+                    TimeValue.timeValueMillis(current),
+                    TimeValue.timeValueMillis(elapsed));
+                break;
+        }
+        verifyNoMoreInteractions(logger);
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmMonitorTests.java b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmMonitorTests.java
index 8d3ddeec84e..91862e9cd18 100644
--- a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmMonitorTests.java
+++ b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmMonitorTests.java
@@ -41,10 +41,12 @@ import static org.mockito.Mockito.when;
 
 public class JvmMonitorTests extends ESTestCase {
 
+    private static final JvmGcMonitorService.GcOverheadThreshold IGNORE = new JvmGcMonitorService.GcOverheadThreshold(0, 0, 0);
+
     public void testMonitorFailure() {
         AtomicBoolean shouldFail = new AtomicBoolean();
         AtomicBoolean invoked = new AtomicBoolean();
-        JvmGcMonitorService.JvmMonitor monitor = new JvmGcMonitorService.JvmMonitor(Collections.emptyMap()) {
+        JvmGcMonitorService.JvmMonitor monitor = new JvmGcMonitorService.JvmMonitor(Collections.emptyMap(), IGNORE) {
             @Override
             void onMonitorFailure(Throwable t) {
                 invoked.set(true);
@@ -53,7 +55,7 @@ public class JvmMonitorTests extends ESTestCase {
             }
 
             @Override
-            synchronized void monitorLongGc() {
+            synchronized void monitorGc() {
                 if (shouldFail.get()) {
                     throw new RuntimeException("simulated");
                 }
@@ -62,6 +64,10 @@ public class JvmMonitorTests extends ESTestCase {
             @Override
             void onSlowGc(final Threshold threshold, final long seq, final SlowGcEvent slowGcEvent) {
             }
+
+            @Override
+            void onGcOverhead(Threshold threshold, long total, long elapsed, long seq) {
+            }
         };
 
         monitor.run();
@@ -166,7 +172,7 @@ public class JvmMonitorTests extends ESTestCase {
 
         final AtomicInteger count = new AtomicInteger();
 
-        JvmGcMonitorService.JvmMonitor monitor = new JvmGcMonitorService.JvmMonitor(gcThresholds) {
+        JvmGcMonitorService.JvmMonitor monitor = new JvmGcMonitorService.JvmMonitor(gcThresholds, IGNORE) {
             @Override
             void onMonitorFailure(Throwable t) {
             }
@@ -198,6 +204,10 @@ public class JvmMonitorTests extends ESTestCase {
                 }
             }
 
+            @Override
+            void onGcOverhead(Threshold threshold, long total, long elapsed, long seq) {
+            }
+
             @Override
             long now() {
                 return now.get();
@@ -213,7 +223,7 @@ public class JvmMonitorTests extends ESTestCase {
 
         now.set(start + TimeUnit.NANOSECONDS.convert(expectedElapsed, TimeUnit.MILLISECONDS));
         jvmStats.set(monitorJvmStats);
-        monitor.monitorLongGc();
+        monitor.monitorGc();
 
         assertThat(count.get(), equalTo((youngGcThreshold ? 1 : 0) + (oldGcThreshold ? 1 : 0)));
     }
@@ -235,14 +245,140 @@ public class JvmMonitorTests extends ESTestCase {
 
     private JvmStats jvmStats(JvmStats.GarbageCollector youngCollector, JvmStats.GarbageCollector oldCollector) {
         final JvmStats jvmStats = mock(JvmStats.class);
-        final JvmStats.GarbageCollectors initialGcs = mock(JvmStats.GarbageCollectors.class);
-        final JvmStats.GarbageCollector[] initialCollectors = new JvmStats.GarbageCollector[2];
-        initialCollectors[0] = youngCollector;
-        initialCollectors[1] = oldCollector;
-        when(initialGcs.getCollectors()).thenReturn(initialCollectors);
-        when(jvmStats.getGc()).thenReturn(initialGcs);
+        final JvmStats.GarbageCollectors gcs = mock(JvmStats.GarbageCollectors.class);
+        final JvmStats.GarbageCollector[] collectors = new JvmStats.GarbageCollector[2];
+        collectors[0] = youngCollector;
+        collectors[1] = oldCollector;
+        when(gcs.getCollectors()).thenReturn(collectors);
+        when(jvmStats.getGc()).thenReturn(gcs);
         when(jvmStats.getMem()).thenReturn(JvmStats.jvmStats().getMem());
         return jvmStats;
     }
 
+    public void testMonitorGc() {
+        final int youngCollectionCount = randomIntBetween(1, 16);
+        final int youngCollectionIncrement = randomIntBetween(1, 16);
+        final int youngCollectionTime = randomIntBetween(1, 1 << 10);
+        final int youngCollectionTimeIncrement = randomIntBetween(1, 1 << 10);
+        final int oldCollectionCount = randomIntBetween(1, 16);
+        final int oldCollectionIncrement = randomIntBetween(1, 16);
+        final int oldCollectionTime = randomIntBetween(1, 1 << 10);
+        final int oldCollectionTimeIncrement = randomIntBetween(1, 1 << 10);
+
+        final JvmStats.GarbageCollector lastYoungCollector = collector("young", youngCollectionCount, youngCollectionTime);
+        final JvmStats.GarbageCollector lastOldCollector = collector("old", oldCollectionCount, oldCollectionTime);
+        final JvmStats lastjvmStats = jvmStats(lastYoungCollector, lastOldCollector);
+
+        final JvmStats.GarbageCollector currentYoungCollector =
+            collector("young", youngCollectionCount + youngCollectionIncrement, youngCollectionTime + youngCollectionTimeIncrement);
+        final JvmStats.GarbageCollector currentOldCollector =
+            collector("old", oldCollectionCount + oldCollectionIncrement, oldCollectionTime + oldCollectionTimeIncrement);
+        final JvmStats currentJvmStats = jvmStats(currentYoungCollector, currentOldCollector);
+        final long expectedElapsed =
+            randomIntBetween(
+                Math.max(youngCollectionTime + youngCollectionTimeIncrement, oldCollectionTime + oldCollectionTimeIncrement),
+                Integer.MAX_VALUE);
+
+        final AtomicBoolean invoked = new AtomicBoolean();
+
+        final JvmGcMonitorService.JvmMonitor monitor = new JvmGcMonitorService.JvmMonitor(Collections.emptyMap(), IGNORE) {
+
+            @Override
+            void onMonitorFailure(Throwable t) {
+            }
+
+            @Override
+            void onSlowGc(Threshold threshold, long seq, SlowGcEvent slowGcEvent) {
+            }
+
+            @Override
+            void onGcOverhead(Threshold threshold, long total, long elapsed, long seq) {
+            }
+
+            @Override
+            void checkGcOverhead(long current, long elapsed, long seq) {
+                invoked.set(true);
+                assertThat(current, equalTo((long)(youngCollectionTimeIncrement + oldCollectionTimeIncrement)));
+                assertThat(elapsed, equalTo(expectedElapsed));
+            }
+
+            @Override
+            JvmStats jvmStats() {
+                return lastjvmStats;
+            }
+        };
+
+        monitor.monitorGcOverhead(currentJvmStats, expectedElapsed);
+        assertTrue(invoked.get());
+    }
+
+    private JvmStats.GarbageCollector collector(final String name, final int collectionCount, final int collectionTime) {
+        final JvmStats.GarbageCollector gc = mock(JvmStats.GarbageCollector.class);
+        when(gc.getName()).thenReturn(name);
+        when(gc.getCollectionCount()).thenReturn((long)collectionCount);
+        when(gc.getCollectionTime()).thenReturn(TimeValue.timeValueMillis(collectionTime));
+        return gc;
+    }
+
+    public void testCheckGcOverhead() {
+        final int debugThreshold = randomIntBetween(1, 98);
+        final int infoThreshold = randomIntBetween(debugThreshold + 1, 99);
+        final int warnThreshold = randomIntBetween(infoThreshold + 1, 100);
+        final JvmGcMonitorService.GcOverheadThreshold gcOverheadThreshold =
+            new JvmGcMonitorService.GcOverheadThreshold(warnThreshold, infoThreshold, debugThreshold);
+
+        final JvmGcMonitorService.JvmMonitor.Threshold expectedThreshold;
+        int fraction = 0;
+        final long expectedCurrent;
+        final long expectedElapsed;
+        if (randomBoolean()) {
+            expectedThreshold = randomFrom(JvmGcMonitorService.JvmMonitor.Threshold.values());
+            switch (expectedThreshold) {
+                case WARN:
+                    fraction = randomIntBetween(warnThreshold, 100);
+                    break;
+                case INFO:
+                    fraction = randomIntBetween(infoThreshold, warnThreshold - 1);
+                    break;
+                case DEBUG:
+                    fraction = randomIntBetween(debugThreshold, infoThreshold - 1);
+                    break;
+            }
+        } else {
+            expectedThreshold = null;
+            fraction = randomIntBetween(0, debugThreshold - 1);
+        }
+
+        expectedElapsed = 100 * randomIntBetween(1, 1000);
+        expectedCurrent = fraction * expectedElapsed / 100;
+
+        final AtomicBoolean invoked = new AtomicBoolean();
+        final long expectedSeq = randomIntBetween(1, Integer.MAX_VALUE);
+
+        final JvmGcMonitorService.JvmMonitor monitor = new JvmGcMonitorService.JvmMonitor(Collections.emptyMap(), gcOverheadThreshold) {
+
+            @Override
+            void onMonitorFailure(final Throwable t) {
+            }
+
+            @Override
+            void onSlowGc(Threshold threshold, long seq, SlowGcEvent slowGcEvent) {
+            }
+
+            @Override
+            void onGcOverhead(final Threshold threshold, final long current, final long elapsed, final long seq) {
+                invoked.set(true);
+                assertThat(threshold, equalTo(expectedThreshold));
+                assertThat(current, equalTo(expectedCurrent));
+                assertThat(elapsed, equalTo(expectedElapsed));
+                assertThat(seq, equalTo(expectedSeq));
+            }
+
+        };
+
+        monitor.checkGcOverhead(expectedCurrent, expectedElapsed, expectedSeq);
+
+        assertThat(invoked.get(), equalTo(expectedThreshold != null));
+    }
+
 }

From a5268cd40daa74b67045c24888d49d7602fe8ee4 Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Wed, 18 May 2016 15:32:48 +0200
Subject: [PATCH 0328/1311] Add CONSOLE to version docs

---
 docs/reference/search/request/version.asciidoc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/reference/search/request/version.asciidoc b/docs/reference/search/request/version.asciidoc
index 3b2329a828a..57c6ce27feb 100644
--- a/docs/reference/search/request/version.asciidoc
+++ b/docs/reference/search/request/version.asciidoc
@@ -5,6 +5,7 @@ Returns a version for each search hit.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "version": true,
     "query" : {
@@ -12,3 +13,4 @@ Returns a version for each search hit.
     }
 }
 --------------------------------------------------
+// CONSOLE

From 394a60f3fd2525f3efcd14d8ce981af42b54473a Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Wed, 18 May 2016 15:52:08 +0200
Subject: [PATCH 0329/1311] Switch to more match query for better illustration

---
 .../search/request/highlighting.asciidoc      | 26 +++++++++----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/docs/reference/search/request/highlighting.asciidoc b/docs/reference/search/request/highlighting.asciidoc
index 139ea9aa825..c235c37b338 100644
--- a/docs/reference/search/request/highlighting.asciidoc
+++ b/docs/reference/search/request/highlighting.asciidoc
@@ -11,7 +11,7 @@ The following is an example of the search request body:
 GET /_search
 {
     "query" : {
-        "match_all": {}
+        "match": { "user": "kimchy" }
     },
     "highlight" : {
         "fields" : {
@@ -125,7 +125,7 @@ the index to be bigger):
     }
 }
 --------------------------------------------------
-    "match_all": {}
+
 ==== Force highlighter type
 
 The `type` field allows to force a specific highlighter type. This is useful
@@ -138,7 +138,7 @@ The following is an example that forces the use of the plain highlighter:
 GET /_search
 {
     "query" : {
-        "match_all": {}
+        "match": { "user": "kimchy" }
     },
     "highlight" : {
         "fields" : {
@@ -159,7 +159,7 @@ stored separately. Defaults to `false`.
 GET /_search
 {
     "query" : {
-        "match_all": {}
+        "match": { "user": "kimchy" }
     },
     "highlight" : {
         "fields" : {
@@ -182,7 +182,7 @@ for example:
 GET /_search
 {
     "query" : {
-        "match_all": {}
+        "match": { "user": "kimchy" }
     },
     "highlight" : {
         "pre_tags" : ["<tag1>"],
@@ -203,7 +203,7 @@ is ordered.
 GET /_search
 {
     "query" : {
-        "match_all": {}
+        "match": { "user": "kimchy" }
     },
     "highlight" : {
         "pre_tags" : ["<tag1>", "<tag2>"],
@@ -236,7 +236,7 @@ is an example of switching tag schemas:
 GET /_search
 {
     "query" : {
-        "match_all": {}
+        "match": { "user": "kimchy" }
     },
     "highlight" : {
         "tags_schema" : "styled",
@@ -267,7 +267,7 @@ For example:
 GET /_search
 {
     "query" : {
-        "match_all": {}
+        "match": { "user": "kimchy" }
     },
     "highlight" : {
         "fields" : {
@@ -289,7 +289,7 @@ to be sorted by score:
 GET /_search
 {
     "query" : {
-        "match_all": {}
+        "match": { "user": "kimchy" }
     },
     "highlight" : {
         "order" : "score",
@@ -312,7 +312,7 @@ is required. Note that `fragment_size` is ignored in this case.
 GET /_search
 {
     "query" : {
-        "match_all": {}
+        "match": { "user": "kimchy" }
     },
     "highlight" : {
         "fields" : {
@@ -341,7 +341,7 @@ greater than `0`.
 GET /_search
 {
     "query" : {
-        "match_all": {}
+        "match": { "user": "kimchy" }
     },
     "highlight" : {
         "fields" : {
@@ -445,7 +445,7 @@ at the field level.
 GET /_search
 {
     "query" : {
-        "match_all": {}
+        "match": { "user": "kimchy" }
     },
     "highlight" : {
         "number_of_fragments" : 3,
@@ -474,7 +474,7 @@ match will be highlighted.
 GET /_search
 {
     "query" : {
-        "match_all": {}
+        "match": { "user": "kimchy" }
     },
     "highlight" : {
         "require_field_match": false,

From cfedda5291372002c5fa6016ad11edc51d85c85e Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Wed, 18 May 2016 16:43:28 +0200
Subject: [PATCH 0330/1311] Default azure container should be
 `elasticsearch-snapshots`

This bug has been introduced in 5.0 when we refactored settings
---
 .../elasticsearch/cloud/azure/storage/AzureStorageService.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
index 01d66c177a2..b6f5f12b266 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
@@ -32,6 +32,7 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.URISyntaxException;
 import java.util.Map;
+import java.util.function.Function;
 
 /**
  * Azure Storage Service interface
@@ -46,7 +47,7 @@ public interface AzureStorageService {
         public static final Setting<String> ACCOUNT_SETTING =
             Setting.simpleString("repositories.azure.account", Property.NodeScope, Property.Filtered);
         public static final Setting<String> CONTAINER_SETTING =
-            Setting.simpleString("repositories.azure.container", Property.NodeScope);
+            new Setting<>("repositories.azure.container", "elasticsearch-snapshots", Function.identity(), Property.NodeScope);
         public static final Setting<String> BASE_PATH_SETTING =
             Setting.simpleString("repositories.azure.base_path", Property.NodeScope);
         public static final Setting<String> LOCATION_MODE_SETTING =

From d85dac7a9a600a14676f02be207d94701674eeaa Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Wed, 18 May 2016 16:43:56 +0200
Subject: [PATCH 0331/1311] Add more logs

---
 .../cloud/azure/blobstore/AzureBlobContainer.java         | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
index cf25e5d8b77..57e7d2673dc 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
@@ -62,6 +62,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
 
     @Override
     public boolean blobExists(String blobName) {
+        logger.debug("blobExists({})", blobName);
         try {
             return blobStore.blobExists(blobStore.container(), buildKey(blobName));
         } catch (URISyntaxException | StorageException e) {
@@ -72,6 +73,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
 
     @Override
     public InputStream readBlob(String blobName) throws IOException {
+        logger.debug("readBlob({})", blobName);
         try {
             return blobStore.getInputStream(blobStore.container(), buildKey(blobName));
         } catch (StorageException e) {
@@ -86,6 +88,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
 
     @Override
     public void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException {
+        logger.debug("writeBlob({}, stream, {})", blobName, blobSize);
         try (OutputStream stream = createOutput(blobName)) {
             Streams.copy(inputStream, stream);
         }
@@ -93,6 +96,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
 
     @Override
     public void writeBlob(String blobName, BytesReference bytes) throws IOException {
+        logger.debug("writeBlob({}, bytes)", blobName);
         try (OutputStream stream = createOutput(blobName)) {
             bytes.writeTo(stream);
         }
@@ -115,6 +119,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
 
     @Override
     public void deleteBlob(String blobName) throws IOException {
+        logger.debug("deleteBlob({})", blobName);
         try {
             blobStore.deleteBlob(blobStore.container(), buildKey(blobName));
         } catch (URISyntaxException | StorageException e) {
@@ -125,6 +130,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
 
     @Override
     public Map<String, BlobMetaData> listBlobsByPrefix(@Nullable String prefix) throws IOException {
+        logger.debug("listBlobsByPrefix({})", prefix);
 
         try {
             return blobStore.listBlobsByPrefix(blobStore.container(), keyPath, prefix);
@@ -136,6 +142,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
 
     @Override
     public void move(String sourceBlobName, String targetBlobName) throws IOException {
+        logger.debug("move({}, {})", sourceBlobName, targetBlobName);
         try {
             String source = keyPath + sourceBlobName;
             String target = keyPath + targetBlobName;
@@ -154,6 +161,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
 
     @Override
     public Map<String, BlobMetaData> listBlobs() throws IOException {
+        logger.debug("listBlobs()");
         return listBlobsByPrefix(null);
     }
 

From db4809d9061e6c4158d73eab1e7a9c82aec6521b Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 18 May 2016 11:03:00 -0400
Subject: [PATCH 0332/1311] Remove last vestigates of /bin/sh shebangs

This commit removes the remaining /bin/sh shebangs in favor of
/bin/bash.

Relates #18448
---
 buildSrc/src/main/resources/deb/postinst.ftl                    | 2 +-
 buildSrc/src/main/resources/deb/preinst.ftl                     | 2 +-
 .../src/main/resources/bin/elasticsearch-systemd-pre-exec       | 2 +-
 distribution/src/main/resources/bin/elasticsearch.in.sh         | 2 +-
 plugins/jvm-example/src/main/bin/test                           | 2 +-
 qa/vagrant/src/test/resources/packaging/scripts/modules.bash    | 2 +-
 qa/vagrant/src/test/resources/packaging/scripts/os_package.bash | 2 +-
 .../test/resources/packaging/scripts/packaging_test_utils.bash  | 2 +-
 qa/vagrant/src/test/resources/packaging/scripts/plugins.bash    | 2 +-
 qa/vagrant/src/test/resources/packaging/scripts/tar.bash        | 2 +-
 10 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/buildSrc/src/main/resources/deb/postinst.ftl b/buildSrc/src/main/resources/deb/postinst.ftl
index 5f67242c265..9acfc0f084e 100644
--- a/buildSrc/src/main/resources/deb/postinst.ftl
+++ b/buildSrc/src/main/resources/deb/postinst.ftl
@@ -1,2 +1,2 @@
-#!/bin/sh -e
+#!/bin/bash -e
 <% commands.each {command -> %><%= command %><% } %>
diff --git a/buildSrc/src/main/resources/deb/preinst.ftl b/buildSrc/src/main/resources/deb/preinst.ftl
index 5f67242c265..9acfc0f084e 100644
--- a/buildSrc/src/main/resources/deb/preinst.ftl
+++ b/buildSrc/src/main/resources/deb/preinst.ftl
@@ -1,2 +1,2 @@
-#!/bin/sh -e
+#!/bin/bash -e
 <% commands.each {command -> %><%= command %><% } %>
diff --git a/distribution/src/main/resources/bin/elasticsearch-systemd-pre-exec b/distribution/src/main/resources/bin/elasticsearch-systemd-pre-exec
index a51d639bf7d..5a5877598e6 100755
--- a/distribution/src/main/resources/bin/elasticsearch-systemd-pre-exec
+++ b/distribution/src/main/resources/bin/elasticsearch-systemd-pre-exec
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # CONF_FILE setting was removed
 if [ ! -z "$CONF_FILE" ]; then
diff --git a/distribution/src/main/resources/bin/elasticsearch.in.sh b/distribution/src/main/resources/bin/elasticsearch.in.sh
index 8f1b5566f90..58b26a2d6eb 100644
--- a/distribution/src/main/resources/bin/elasticsearch.in.sh
+++ b/distribution/src/main/resources/bin/elasticsearch.in.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # check in case a user was using this mechanism
 if [ "x$ES_CLASSPATH" != "x" ]; then
diff --git a/plugins/jvm-example/src/main/bin/test b/plugins/jvm-example/src/main/bin/test
index 76ba88943ac..13fdcce1e52 100755
--- a/plugins/jvm-example/src/main/bin/test
+++ b/plugins/jvm-example/src/main/bin/test
@@ -1,3 +1,3 @@
-#!/bin/sh
+#!/bin/bash
 
 echo test
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/modules.bash b/qa/vagrant/src/test/resources/packaging/scripts/modules.bash
index bd6da680da9..2e80fd648f3 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/modules.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/modules.bash
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # This file contains some utilities to test the elasticsearch scripts,
 # the .deb/.rpm packages and the SysV/Systemd scripts.
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/os_package.bash b/qa/vagrant/src/test/resources/packaging/scripts/os_package.bash
index 72c59c39324..ee6e491d169 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/os_package.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/os_package.bash
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # This file contains some utilities to test the elasticsearch scripts with
 # the .deb/.rpm packages.
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash b/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash
index 09d0190695e..5f50dfc2850 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # This file contains some utilities to test the elasticsearch scripts,
 # the .deb/.rpm packages and the SysV/Systemd scripts.
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/plugins.bash b/qa/vagrant/src/test/resources/packaging/scripts/plugins.bash
index 4f1e574b905..afae7439057 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/plugins.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/plugins.bash
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # This file contains some utilities to test the elasticsearch scripts,
 # the .deb/.rpm packages and the SysV/Systemd scripts.
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/tar.bash b/qa/vagrant/src/test/resources/packaging/scripts/tar.bash
index 277eee60f1a..798ec6c2997 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/tar.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/tar.bash
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # This file contains some utilities to test the elasticsearch scripts,
 # the .deb/.rpm packages and the SysV/Systemd scripts.

From a846ff93e91020b5a837c45a045191b1c58a86e7 Mon Sep 17 00:00:00 2001
From: markharwood <markharwood@gmail.com>
Date: Tue, 17 May 2016 14:15:53 +0100
Subject: [PATCH 0333/1311] Aggregations fix: support include/exclude strings
 formatted for IP and date fields in terms and significant_terms aggregations.

Closes #17705
---
 .../SignificantTermsAggregatorFactory.java    | 17 +++++--
 .../bucket/terms/TermsAggregatorFactory.java  | 15 ++++--
 .../bucket/terms/support/IncludeExclude.java  | 40 ++++++++++-----
 .../test/search.aggregation/20_terms.yaml     | 50 +++++++++++++++++++
 .../test/search.aggregation/30_sig_terms.yaml | 25 ++++++++++
 5 files changed, 125 insertions(+), 22 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
index 13126029b8e..4b9e3acb873 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
@@ -211,7 +211,14 @@ public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFac
                 }
             }
             assert execution != null;
-            return execution.create(name, factories, valuesSource, config.format(), bucketCountThresholds, includeExclude, context, parent,
+            
+            DocValueFormat format = config.format();
+            if ((includeExclude != null) && (includeExclude.isRegexBased()) && format != DocValueFormat.RAW) {
+                throw new AggregationExecutionException("Aggregation [" + name + "] cannot support regular expression style include/exclude "
+                        + "settings as they can only be applied to string fields. Use an array of values for include/exclude clauses");
+            }
+            
+            return execution.create(name, factories, valuesSource, format, bucketCountThresholds, includeExclude, context, parent,
                     significanceHeuristic, this, pipelineAggregators, metaData);
         }
 
@@ -227,7 +234,7 @@ public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFac
             }
             IncludeExclude.LongFilter longFilter = null;
             if (includeExclude != null) {
-                longFilter = includeExclude.convertToLongFilter();
+                longFilter = includeExclude.convertToLongFilter(config.format());
             }
             return new SignificantLongTermsAggregator(name, factories, (ValuesSource.Numeric) valuesSource, config.format(),
                     bucketCountThresholds, context, parent, significanceHeuristic, this, longFilter, pipelineAggregators,
@@ -248,7 +255,7 @@ public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFac
                     AggregationContext aggregationContext, Aggregator parent, SignificanceHeuristic significanceHeuristic,
                     SignificantTermsAggregatorFactory termsAggregatorFactory, List<PipelineAggregator> pipelineAggregators,
                     Map<String, Object> metaData) throws IOException {
-                final IncludeExclude.StringFilter filter = includeExclude == null ? null : includeExclude.convertToStringFilter();
+                final IncludeExclude.StringFilter filter = includeExclude == null ? null : includeExclude.convertToStringFilter(format);
                 return new SignificantStringTermsAggregator(name, factories, valuesSource, format, bucketCountThresholds, filter,
                         aggregationContext, parent, significanceHeuristic, termsAggregatorFactory, pipelineAggregators, metaData);
             }
@@ -262,7 +269,7 @@ public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFac
                     AggregationContext aggregationContext, Aggregator parent, SignificanceHeuristic significanceHeuristic,
                     SignificantTermsAggregatorFactory termsAggregatorFactory, List<PipelineAggregator> pipelineAggregators,
                     Map<String, Object> metaData) throws IOException {
-                final IncludeExclude.OrdinalsFilter filter = includeExclude == null ? null : includeExclude.convertToOrdinalsFilter();
+                final IncludeExclude.OrdinalsFilter filter = includeExclude == null ? null : includeExclude.convertToOrdinalsFilter(format);
                 return new GlobalOrdinalsSignificantTermsAggregator(name, factories,
                         (ValuesSource.Bytes.WithOrdinals.FieldData) valuesSource, format, bucketCountThresholds, filter,
                         aggregationContext, parent, significanceHeuristic, termsAggregatorFactory, pipelineAggregators, metaData);
@@ -277,7 +284,7 @@ public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFac
                     AggregationContext aggregationContext, Aggregator parent, SignificanceHeuristic significanceHeuristic,
                     SignificantTermsAggregatorFactory termsAggregatorFactory, List<PipelineAggregator> pipelineAggregators,
                     Map<String, Object> metaData) throws IOException {
-                final IncludeExclude.OrdinalsFilter filter = includeExclude == null ? null : includeExclude.convertToOrdinalsFilter();
+                final IncludeExclude.OrdinalsFilter filter = includeExclude == null ? null : includeExclude.convertToOrdinalsFilter(format);
                 return new GlobalOrdinalsSignificantTermsAggregator.WithHash(name, factories,
                         (ValuesSource.Bytes.WithOrdinals.FieldData) valuesSource, format, bucketCountThresholds, filter,
                         aggregationContext, parent, significanceHeuristic, termsAggregatorFactory, pipelineAggregators, metaData);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
index 1ccf4a11570..62374ae7d19 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
@@ -150,8 +150,13 @@ public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory<Values
                     }
                 }
             }
+            DocValueFormat format = config.format();
+            if ((includeExclude != null) && (includeExclude.isRegexBased()) && format != DocValueFormat.RAW) {
+                throw new AggregationExecutionException("Aggregation [" + name + "] cannot support regular expression style include/exclude "
+                        + "settings as they can only be applied to string fields. Use an array of values for include/exclude clauses");
+            }
 
-            return execution.create(name, factories, valuesSource, order, config.format(), bucketCountThresholds, includeExclude, context, parent,
+            return execution.create(name, factories, valuesSource, order, format, bucketCountThresholds, includeExclude, context, parent, 
                     collectMode, showTermDocCountError, pipelineAggregators, metaData);
         }
 
@@ -171,7 +176,7 @@ public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory<Values
                         pipelineAggregators, metaData);
             }
             if (includeExclude != null) {
-                longFilter = includeExclude.convertToLongFilter();
+                longFilter = includeExclude.convertToLongFilter(config.format());
             }
             return new LongTermsAggregator(name, factories, (ValuesSource.Numeric) valuesSource, config.format(), order,
                     bucketCountThresholds, context, parent, collectMode, showTermDocCountError, longFilter, pipelineAggregators,
@@ -192,7 +197,7 @@ public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory<Values
                     AggregationContext aggregationContext, Aggregator parent, SubAggCollectionMode subAggCollectMode,
                     boolean showTermDocCountError, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
                             throws IOException {
-                final IncludeExclude.StringFilter filter = includeExclude == null ? null : includeExclude.convertToStringFilter();
+                final IncludeExclude.StringFilter filter = includeExclude == null ? null : includeExclude.convertToStringFilter(format);
                 return new StringTermsAggregator(name, factories, valuesSource, order, format, bucketCountThresholds, filter,
                         aggregationContext, parent, subAggCollectMode, showTermDocCountError, pipelineAggregators, metaData);
             }
@@ -211,7 +216,7 @@ public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory<Values
                     AggregationContext aggregationContext, Aggregator parent, SubAggCollectionMode subAggCollectMode,
                     boolean showTermDocCountError, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
                             throws IOException {
-                final IncludeExclude.OrdinalsFilter filter = includeExclude == null ? null : includeExclude.convertToOrdinalsFilter();
+                final IncludeExclude.OrdinalsFilter filter = includeExclude == null ? null : includeExclude.convertToOrdinalsFilter(format);
                 return new GlobalOrdinalsStringTermsAggregator(name, factories, (ValuesSource.Bytes.WithOrdinals) valuesSource, order,
                         format, bucketCountThresholds, filter, aggregationContext, parent, subAggCollectMode, showTermDocCountError,
                         pipelineAggregators, metaData);
@@ -231,7 +236,7 @@ public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory<Values
                     AggregationContext aggregationContext, Aggregator parent, SubAggCollectionMode subAggCollectMode,
                     boolean showTermDocCountError, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
                             throws IOException {
-                final IncludeExclude.OrdinalsFilter filter = includeExclude == null ? null : includeExclude.convertToOrdinalsFilter();
+                final IncludeExclude.OrdinalsFilter filter = includeExclude == null ? null : includeExclude.convertToOrdinalsFilter(format);
                 return new GlobalOrdinalsStringTermsAggregator.WithHash(name, factories, (ValuesSource.Bytes.WithOrdinals) valuesSource,
                         order, format, bucketCountThresholds, filter, aggregationContext, parent, subAggCollectMode, showTermDocCountError,
                         pipelineAggregators, metaData);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java
index 101291d01e1..209700b86d9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java
@@ -43,6 +43,7 @@ import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Bytes.WithOrdinals;
 
@@ -135,7 +136,8 @@ public class IncludeExclude implements Writeable, ToXContent {
     }
 
     public static abstract class OrdinalsFilter {
-        public abstract LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals, ValuesSource.Bytes.WithOrdinals valueSource) throws IOException;
+        public abstract LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals, ValuesSource.Bytes.WithOrdinals valueSource)
+                throws IOException;
 
     }
 
@@ -152,7 +154,8 @@ public class IncludeExclude implements Writeable, ToXContent {
          *
          */
         @Override
-        public LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals, ValuesSource.Bytes.WithOrdinals valueSource) throws IOException {
+        public LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals, ValuesSource.Bytes.WithOrdinals valueSource)
+                throws IOException {
             LongBitSet acceptedGlobalOrdinals = new LongBitSet(globalOrdinals.getValueCount());
             TermsEnum globalTermsEnum;
             Terms globalTerms = new DocValuesTerms(globalOrdinals);
@@ -179,7 +182,7 @@ public class IncludeExclude implements Writeable, ToXContent {
         @Override
         public LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals, WithOrdinals valueSource) throws IOException {
             LongBitSet acceptedGlobalOrdinals = new LongBitSet(globalOrdinals.getValueCount());
-            if(includeValues!=null){
+            if (includeValues != null) {
                 for (BytesRef term : includeValues) {
                     long ord = globalOrdinals.lookupTerm(term);
                     if (ord >= 0) {
@@ -534,33 +537,46 @@ public class IncludeExclude implements Writeable, ToXContent {
         return a;
     }
 
-    public StringFilter convertToStringFilter() {
+    public StringFilter convertToStringFilter(DocValueFormat format) {
         if (isRegexBased()) {
             return new AutomatonBackedStringFilter(toAutomaton());
         }
-        return new TermListBackedStringFilter(includeValues, excludeValues);
+        return new TermListBackedStringFilter(parseForDocValues(includeValues, format), parseForDocValues(excludeValues, format));
     }
 
-    public OrdinalsFilter convertToOrdinalsFilter() {
+    private static SortedSet<BytesRef> parseForDocValues(SortedSet<BytesRef> endUserFormattedValues, DocValueFormat format) {
+        SortedSet<BytesRef> result = endUserFormattedValues;
+        if (endUserFormattedValues != null) {
+            if (format != DocValueFormat.RAW) {
+                result = new TreeSet<>();
+                for (BytesRef formattedVal : endUserFormattedValues) {
+                    result.add(format.parseBytesRef(formattedVal.utf8ToString()));
+                }
+            }
+        }
+        return result;
+    }
+
+    public OrdinalsFilter convertToOrdinalsFilter(DocValueFormat format) {
 
         if (isRegexBased()) {
             return new AutomatonBackedOrdinalsFilter(toAutomaton());
         }
-        return new TermListBackedOrdinalsFilter(includeValues, excludeValues);
+        return new TermListBackedOrdinalsFilter(parseForDocValues(includeValues, format), parseForDocValues(excludeValues, format));
     }
 
-    public LongFilter convertToLongFilter() {
+    public LongFilter convertToLongFilter(DocValueFormat format) {
         int numValids = includeValues == null ? 0 : includeValues.size();
         int numInvalids = excludeValues == null ? 0 : excludeValues.size();
         LongFilter result = new LongFilter(numValids, numInvalids);
         if (includeValues != null) {
             for (BytesRef val : includeValues) {
-                result.addAccept(Long.parseLong(val.utf8ToString()));
+                result.addAccept(format.parseLong(val.utf8ToString(), false, null));
             }
         }
         if (excludeValues != null) {
             for (BytesRef val : excludeValues) {
-                result.addReject(Long.parseLong(val.utf8ToString()));
+                result.addReject(format.parseLong(val.utf8ToString(), false, null));
             }
         }
         return result;
@@ -572,13 +588,13 @@ public class IncludeExclude implements Writeable, ToXContent {
         LongFilter result = new LongFilter(numValids, numInvalids);
         if (includeValues != null) {
             for (BytesRef val : includeValues) {
-                double dval=Double.parseDouble(val.utf8ToString());
+                double dval = Double.parseDouble(val.utf8ToString());
                 result.addAccept(NumericUtils.doubleToSortableLong(dval));
             }
         }
         if (excludeValues != null) {
             for (BytesRef val : excludeValues) {
-                double dval=Double.parseDouble(val.utf8ToString());
+                double dval = Double.parseDouble(val.utf8ToString());
                 result.addReject(NumericUtils.doubleToSortableLong(dval));
             }
         }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml
index 71d1a1e7ca2..c35e79e6cfe 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml
@@ -117,6 +117,33 @@ setup:
   
   - match: { aggregations.ip_terms.buckets.1.doc_count: 1 }
 
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "ip_terms" : { "terms" : { "field" : "ip", "include" : [ "127.0.0.1" ] } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.ip_terms.buckets: 1 }
+
+  - match: { aggregations.ip_terms.buckets.0.key: "127.0.0.1" }
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "ip_terms" : { "terms" : { "field" : "ip", "exclude" : [ "127.0.0.1" ] } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.ip_terms.buckets: 1 }
+
+  - match: { aggregations.ip_terms.buckets.0.key: "::1" }
+
+  - do:
+      catch: request
+      search:
+        body: { "size" : 0, "aggs" : { "ip_terms" : { "terms" : { "field" : "ip", "exclude" :  "127.*"  } } } }
+
+  
+
 ---
 "Boolean test":
   - do:
@@ -300,4 +327,27 @@ setup:
   - match: { aggregations.date_terms.buckets.1.key_as_string: "2014-09-01T00:00:00.000Z" }
 
   - match: { aggregations.date_terms.buckets.1.doc_count: 1 }
+  
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "date_terms" : { "terms" : { "field" : "date", "include" : [ "2016-05-03" ] } } } }
 
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.date_terms.buckets: 1 }
+  
+  - match: { aggregations.date_terms.buckets.0.key_as_string: "2016-05-03T00:00:00.000Z" }
+  
+  - match: { aggregations.date_terms.buckets.0.doc_count: 2 }  
+  
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "date_terms" : { "terms" : { "field" : "date", "exclude" : [ "2016-05-03" ] } } } }
+
+  - match: { hits.total: 3 }
+
+  - length: { aggregations.date_terms.buckets: 1 }
+  
+  - match: { aggregations.date_terms.buckets.0.key_as_string: "2014-09-01T00:00:00.000Z" }
+
+  - match: { aggregations.date_terms.buckets.0.doc_count: 1 }  
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/30_sig_terms.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/30_sig_terms.yaml
index 45c042baea4..a708ff19d7e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/30_sig_terms.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/30_sig_terms.yaml
@@ -121,3 +121,28 @@
   - is_false: aggregations.ip_terms.buckets.0.key_as_string
 
   - match: { aggregations.ip_terms.buckets.0.doc_count: 1 }
+
+  - do:
+      search:
+        body: { "query" : { "exists" : { "field" : "ip" } }, "aggs" : { "ip_terms" : { "significant_terms" : { "field" : "ip", "min_doc_count" : 1, "include" : [ "::1" ] } } } }
+
+  - match: { hits.total: 1 }
+
+  - length: { aggregations.ip_terms.buckets: 1 }
+
+  - match: { aggregations.ip_terms.buckets.0.key: "::1" }
+ 
+  - do:
+      search:
+        body: { "query" : { "exists" : { "field" : "ip" } }, "aggs" : { "ip_terms" : { "significant_terms" : { "field" : "ip", "min_doc_count" : 1, "exclude" : [ "::1" ] } } } }
+
+  - match: { hits.total: 1 }
+
+  - length: { aggregations.ip_terms.buckets: 0 }
+  
+  - do:
+      catch: request
+      search:
+        body: { "size" : 0, "aggs" : { "ip_terms" : { "significant_terms" : { "field" : "ip", "exclude" :  "127.*"  } } } }
+  
+ 

From 9b247f982849fd844aacfe00b0f465538a63d8e1 Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Wed, 18 May 2016 17:23:33 +0200
Subject: [PATCH 0334/1311] Fix remove of azure files

Probably when we updated Azure SDK, we introduced a regression.
Actually, we are not able to remove files anymore.

For example, if you register a new azure repository, the snapshot service tries to create a temp file and then remove it.
Removing does not work and you can see in logs:

```
[2016-05-18 11:03:24,914][WARN ][org.elasticsearch.cloud.azure.blobstore] [azure] can not remove [tests-ilmRPJ8URU-sh18yj38O6g/] in container {elasticsearch-snapshots}: The specified blob does not exist.
```

This fix deals with that. It now list all the files in a flatten mode, remove in the full URL the server and the container name.

As an example, when you are removing a blob which full name is `https://dpi24329.blob.core.windows.net/elasticsearch-snapshots/bar/test` you need to actually call Azure SDK with `bar/test` as the path, `elasticsearch-snapshots` is the container.

To run the test, you need to pass some parameters: `-Dtests.thirdparty=true -Dtests.config=/path/to/elasticsearch.yml`

Where `elasticsearch.yml` contains something like:

```
cloud.azure.storage.default.account: account
cloud.azure.storage.default.key: key
```

Related to #16472
Closes #18436.
---
 .../storage/AzureStorageServiceImpl.java      | 13 +++-
 .../azure/AzureBlobStoreTests.java            | 76 +++++++++++++++++++
 2 files changed, 86 insertions(+), 3 deletions(-)
 create mode 100644 plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java

diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
index 497b0e3753a..71fcee8f6df 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
@@ -182,13 +182,20 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
         CloudBlobClient client = this.getSelectedClient(account, mode);
         CloudBlobContainer blob_container = client.getContainerReference(container);
         if (blob_container.exists()) {
-            for (ListBlobItem blobItem : blob_container.listBlobs(path)) {
-                logger.trace("removing blob [{}]", blobItem.getUri());
-                deleteBlob(account, mode, container, blobItem.getUri().toString());
+            for (ListBlobItem blobItem : blob_container.listBlobs(path, true)) {
+                String blobName = blobNameFromUri(container, blobItem.getUri());
+                logger.trace("removing blob [{}] full URI was [{}]", blobName, blobItem.getUri());
+                deleteBlob(account, mode, container, blobName);
             }
         }
     }
 
+    public static String blobNameFromUri(String container, URI uri) {
+        String path = uri.getPath();
+        // We remove the container name from the path
+        return path.replaceFirst("/" + container + "/", "");
+    }
+
     @Override
     public boolean blobExists(String account, LocationMode mode, String container, String blob) throws URISyntaxException, StorageException {
         // Container name must be lower case.
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java
new file mode 100644
index 00000000000..1ec0fdc76bb
--- /dev/null
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.azure;
+
+import com.microsoft.azure.storage.StorageException;
+import org.elasticsearch.cloud.azure.blobstore.AzureBlobStore;
+import org.elasticsearch.cloud.azure.storage.AzureStorageService;
+import org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.blobstore.BlobStore;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsException;
+import org.elasticsearch.repositories.RepositoryName;
+import org.elasticsearch.repositories.RepositorySettings;
+import org.elasticsearch.test.ESBlobStoreTestCase;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+/**
+ * You must specify {@code -Dtests.thirdparty=true -Dtests.config=/path/to/elasticsearch.yml}
+ * in order to run these tests.
+ */
+@ESIntegTestCase.ThirdParty
+public class AzureBlobStoreTests extends ESBlobStoreTestCase {
+    @Override
+    protected BlobStore newBlobStore() throws IOException {
+        try {
+            RepositoryName repositoryName = new RepositoryName("azure", "ittest");
+            Settings settings = readSettingsFromFile();
+            RepositorySettings repositorySettings = new RepositorySettings(settings, Settings.builder().build());
+            AzureStorageService storageService = new AzureStorageServiceImpl(settings);
+            AzureBlobStore blobStore = new AzureBlobStore(repositoryName, settings, repositorySettings, storageService);
+            blobStore.createContainer(blobStore.container());
+            return blobStore;
+        } catch (URISyntaxException | StorageException e) {
+            throw new IOException(e);
+        }
+    }
+
+    protected Settings readSettingsFromFile() {
+        Settings.Builder settings = Settings.builder();
+
+        // if explicit, just load it and don't load from env
+        try {
+            if (Strings.hasText(System.getProperty("tests.config"))) {
+                settings.loadFromPath(PathUtils.get((System.getProperty("tests.config"))));
+            } else {
+                throw new IllegalStateException("to run integration tests, you need to set -Dtests.thirdparty=true and -Dtests.config=/path/to/elasticsearch.yml");
+            }
+        } catch (SettingsException exception) {
+            throw new IllegalStateException("your test configuration file is incorrect: " + System.getProperty("tests.config"), exception);
+        }
+        return settings.build();
+    }
+
+}

From 7c665a010b12152fef387fac5c1ad710dc467f39 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Sun, 15 May 2016 21:26:09 +0200
Subject: [PATCH 0335/1311] Fix TimeZoneRounding#nextRoundingValue for hour,
 minute and second units

Currently rounding intervals obtained by nextRoundingValue() for hour, minute and
second units can include an extra hour when happening at DST transitions that add
an extra hour (eg CEST -> CET). This changes the rounding logic for time units
smaller or equal to an hour to fix this.

Closes #18326
---
 .../common/rounding/DateTimeUnit.java         |  9 +++
 .../common/rounding/TimeZoneRounding.java     | 25 ++++---
 .../common/rounding/DateTimeUnitTests.java    | 75 +++++++++++++++++++
 .../rounding/TimeZoneRoundingTests.java       | 58 ++++++++++++--
 .../aggregations/bucket/DateHistogramIT.java  | 24 ++++++
 5 files changed, 173 insertions(+), 18 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/common/rounding/DateTimeUnitTests.java

diff --git a/core/src/main/java/org/elasticsearch/common/rounding/DateTimeUnit.java b/core/src/main/java/org/elasticsearch/common/rounding/DateTimeUnit.java
index e02342ffdca..bd0c5506859 100644
--- a/core/src/main/java/org/elasticsearch/common/rounding/DateTimeUnit.java
+++ b/core/src/main/java/org/elasticsearch/common/rounding/DateTimeUnit.java
@@ -53,6 +53,15 @@ public enum DateTimeUnit {
         return field;
     }
 
+    /**
+     * @param unit the {@link DateTimeUnit} to check
+     * @return true if the unit is a day or longer
+     */
+    public static boolean isDayOrLonger(DateTimeUnit unit) {
+        return (unit == DateTimeUnit.HOUR_OF_DAY || unit == DateTimeUnit.MINUTES_OF_HOUR
+                || unit == DateTimeUnit.SECOND_OF_MINUTE) == false;
+    }
+
     public static DateTimeUnit resolve(byte id) {
         switch (id) {
             case 1: return WEEK_OF_WEEKYEAR;
diff --git a/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java b/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java
index 4189e412708..e0ffb89c8b0 100644
--- a/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java
+++ b/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java
@@ -46,8 +46,8 @@ public abstract class TimeZoneRounding extends Rounding {
 
     public static class Builder {
 
-        private DateTimeUnit unit;
-        private long interval = -1;
+        private final DateTimeUnit unit;
+        private final long interval;
 
         private DateTimeZone timeZone = DateTimeZone.UTC;
 
@@ -142,10 +142,15 @@ public abstract class TimeZoneRounding extends Rounding {
 
         @Override
         public long nextRoundingValue(long time) {
-            long timeLocal = time;
-            timeLocal = timeZone.convertUTCToLocal(time);
-            long nextInLocalTime = durationField.add(timeLocal, 1);
-            return timeZone.convertLocalToUTC(nextInLocalTime, false);
+            if (DateTimeUnit.isDayOrLonger(unit)) {
+                time = timeZone.convertUTCToLocal(time);
+            }
+            long next = durationField.add(time, 1);
+            if (DateTimeUnit.isDayOrLonger(unit)) {
+                return timeZone.convertLocalToUTC(next, false);
+            } else {
+                return next;
+            }
         }
 
         @Override
@@ -161,12 +166,12 @@ public abstract class TimeZoneRounding extends Rounding {
             out.writeByte(unit.id());
             out.writeString(timeZone.getID());
         }
-        
+
         @Override
         public int hashCode() {
             return Objects.hash(unit, timeZone);
         }
-        
+
         @Override
         public boolean equals(Object obj) {
             if (obj == null) {
@@ -236,12 +241,12 @@ public abstract class TimeZoneRounding extends Rounding {
             out.writeVLong(interval);
             out.writeString(timeZone.getID());
         }
-        
+
         @Override
         public int hashCode() {
             return Objects.hash(interval, timeZone);
         }
-        
+
         @Override
         public boolean equals(Object obj) {
             if (obj == null) {
diff --git a/core/src/test/java/org/elasticsearch/common/rounding/DateTimeUnitTests.java b/core/src/test/java/org/elasticsearch/common/rounding/DateTimeUnitTests.java
new file mode 100644
index 00000000000..79ef6929645
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/rounding/DateTimeUnitTests.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.common.rounding;
+
+import org.elasticsearch.test.ESTestCase;
+
+import static org.elasticsearch.common.rounding.DateTimeUnit.WEEK_OF_WEEKYEAR;
+import static org.elasticsearch.common.rounding.DateTimeUnit.YEAR_OF_CENTURY;
+import static org.elasticsearch.common.rounding.DateTimeUnit.QUARTER;
+import static org.elasticsearch.common.rounding.DateTimeUnit.MONTH_OF_YEAR;
+import static org.elasticsearch.common.rounding.DateTimeUnit.DAY_OF_MONTH;
+import static org.elasticsearch.common.rounding.DateTimeUnit.HOUR_OF_DAY;
+import static org.elasticsearch.common.rounding.DateTimeUnit.MINUTES_OF_HOUR;
+import static org.elasticsearch.common.rounding.DateTimeUnit.SECOND_OF_MINUTE;
+
+public class DateTimeUnitTests extends ESTestCase {
+
+    /**
+     * test that we don't accidentally change enum ids
+     */
+    public void testEnumIds() {
+        assertEquals(1, WEEK_OF_WEEKYEAR.id());
+        assertEquals(WEEK_OF_WEEKYEAR, DateTimeUnit.resolve((byte) 1));
+
+        assertEquals(2, YEAR_OF_CENTURY.id());
+        assertEquals(YEAR_OF_CENTURY, DateTimeUnit.resolve((byte) 2));
+
+        assertEquals(3, QUARTER.id());
+        assertEquals(QUARTER, DateTimeUnit.resolve((byte) 3));
+
+        assertEquals(4, MONTH_OF_YEAR.id());
+        assertEquals(MONTH_OF_YEAR, DateTimeUnit.resolve((byte) 4));
+
+        assertEquals(5, DAY_OF_MONTH.id());
+        assertEquals(DAY_OF_MONTH, DateTimeUnit.resolve((byte) 5));
+
+        assertEquals(6, HOUR_OF_DAY.id());
+        assertEquals(HOUR_OF_DAY, DateTimeUnit.resolve((byte) 6));
+
+        assertEquals(7, MINUTES_OF_HOUR.id());
+        assertEquals(MINUTES_OF_HOUR, DateTimeUnit.resolve((byte) 7));
+
+        assertEquals(8, SECOND_OF_MINUTE.id());
+        assertEquals(SECOND_OF_MINUTE, DateTimeUnit.resolve((byte) 8));
+    }
+
+    public void testIsDayOrLonger() {
+        for (DateTimeUnit unit : DateTimeUnit.values()) {
+            if (DateTimeUnit.isDayOrLonger(unit)) {
+                assertTrue(unit == DAY_OF_MONTH ||
+                        unit == MONTH_OF_YEAR ||
+                        unit == QUARTER ||
+                        unit == YEAR_OF_CENTURY ||
+                        unit == WEEK_OF_WEEKYEAR);
+            }
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java b/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
index 2c4d78adbd0..08a4ba11342 100644
--- a/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
+++ b/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
@@ -25,6 +25,7 @@ import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.ISODateTimeFormat;
 
+import java.util.ArrayList;
 import java.util.concurrent.TimeUnit;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -147,21 +148,37 @@ public class TimeZoneRoundingTests extends ESTestCase {
         Rounding tzRounding;
         // testing savings to non savings switch
         tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(DateTimeZone.forID("UTC")).build();
-        assertThat(tzRounding.round(time("2014-10-26T01:01:01", DateTimeZone.forID("CET"))),
-                equalTo(time("2014-10-26T01:00:00", DateTimeZone.forID("CET"))));
+        assertThat(tzRounding.round(time("2014-10-26T01:01:01", DateTimeZone.forOffsetHours(2))),  // CEST = UTC+2
+                equalTo(time("2014-10-26T01:00:00", DateTimeZone.forOffsetHours(2))));
+        assertThat(tzRounding.nextRoundingValue(time("2014-10-26T01:00:00", DateTimeZone.forOffsetHours(2))),
+                equalTo(time("2014-10-26T02:00:00", DateTimeZone.forOffsetHours(2))));
+        assertThat(tzRounding.nextRoundingValue(time("2014-10-26T02:00:00", DateTimeZone.forOffsetHours(2))),
+                equalTo(time("2014-10-26T03:00:00", DateTimeZone.forOffsetHours(2))));
 
         tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(DateTimeZone.forID("CET")).build();
-        assertThat(tzRounding.round(time("2014-10-26T01:01:01", DateTimeZone.forID("CET"))),
-                equalTo(time("2014-10-26T01:00:00", DateTimeZone.forID("CET"))));
+        assertThat(tzRounding.round(time("2014-10-26T01:01:01", DateTimeZone.forOffsetHours(2))),  // CEST = UTC+2
+                equalTo(time("2014-10-26T01:00:00", DateTimeZone.forOffsetHours(2))));
+        assertThat(tzRounding.nextRoundingValue(time("2014-10-26T01:00:00", DateTimeZone.forOffsetHours(2))),
+                equalTo(time("2014-10-26T02:00:00", DateTimeZone.forOffsetHours(2))));
+        assertThat(tzRounding.nextRoundingValue(time("2014-10-26T02:00:00", DateTimeZone.forOffsetHours(2))),
+                equalTo(time("2014-10-26T03:00:00", DateTimeZone.forOffsetHours(2))));
 
         // testing non savings to savings switch
         tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(DateTimeZone.forID("UTC")).build();
-        assertThat(tzRounding.round(time("2014-03-30T01:01:01", DateTimeZone.forID("CET"))),
-                equalTo(time("2014-03-30T01:00:00", DateTimeZone.forID("CET"))));
+        assertThat(tzRounding.round(time("2014-03-30T01:01:01", DateTimeZone.forOffsetHours(1))),  // CET = UTC+1
+                equalTo(time("2014-03-30T01:00:00", DateTimeZone.forOffsetHours(1))));
+        assertThat(tzRounding.nextRoundingValue(time("2014-03-30T01:00:00", DateTimeZone.forOffsetHours(1))),
+                equalTo(time("2014-03-30T02:00:00", DateTimeZone.forOffsetHours(1))));
+        assertThat(tzRounding.nextRoundingValue(time("2014-03-30T02:00:00", DateTimeZone.forOffsetHours(1))),
+                equalTo(time("2014-03-30T03:00:00", DateTimeZone.forOffsetHours(1))));
 
         tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(DateTimeZone.forID("CET")).build();
-        assertThat(tzRounding.round(time("2014-03-30T01:01:01", DateTimeZone.forID("CET"))),
-                equalTo(time("2014-03-30T01:00:00", DateTimeZone.forID("CET"))));
+        assertThat(tzRounding.round(time("2014-03-30T01:01:01", DateTimeZone.forOffsetHours(1))),  // CET = UTC+1
+                equalTo(time("2014-03-30T01:00:00", DateTimeZone.forOffsetHours(1))));
+        assertThat(tzRounding.nextRoundingValue(time("2014-03-30T01:00:00", DateTimeZone.forOffsetHours(1))),
+                equalTo(time("2014-03-30T02:00:00", DateTimeZone.forOffsetHours(1))));
+        assertThat(tzRounding.nextRoundingValue(time("2014-03-30T02:00:00", DateTimeZone.forOffsetHours(1))),
+                equalTo(time("2014-03-30T03:00:00", DateTimeZone.forOffsetHours(1))));
 
         // testing non savings to savings switch (America/Chicago)
         tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(DateTimeZone.forID("UTC")).build();
@@ -210,6 +227,31 @@ public class TimeZoneRoundingTests extends ESTestCase {
         }
     }
 
+    /**
+     * Test that nextRoundingValue() for hour rounding (and smaller) is equally spaced (see #18326)
+     * Start at a random date in a random time zone, then find the next zone offset transition (if any).
+     * From there, check that when we advance by using rounding#nextRoundingValue(), we always advance by the same
+     * amount of milliseconds.
+     */
+    public void testSubHourNextRoundingEquallySpaced() {
+        String timeZone = randomFrom(new ArrayList<>(DateTimeZone.getAvailableIDs()));
+        DateTimeUnit unit = randomFrom(new DateTimeUnit[] { DateTimeUnit.HOUR_OF_DAY, DateTimeUnit.MINUTES_OF_HOUR,
+                DateTimeUnit.SECOND_OF_MINUTE });
+        DateTimeZone tz = DateTimeZone.forID(timeZone);
+        TimeZoneRounding rounding = new TimeZoneRounding.TimeUnitRounding(unit, tz);
+        // move the random date to transition for timezones that have offset change due to dst transition
+        long nextTransition = tz.nextTransition(Math.abs(randomLong() % ((long) 10e11)));
+        final long millisPerUnit = unit.field().getDurationField().getUnitMillis();
+        // start ten units before transition
+        long roundedDate = rounding.round(nextTransition - (10 * millisPerUnit));
+        while (roundedDate < nextTransition + 10 * millisPerUnit) {
+            long delta = rounding.nextRoundingValue(roundedDate) - roundedDate;
+            assertEquals("Difference between rounded values not equally spaced for [" + unit.name() + "], [" + timeZone + "] at "
+                    + new DateTime(roundedDate), millisPerUnit, delta);
+            roundedDate = rounding.nextRoundingValue(roundedDate);
+        }
+    }
+
     /**
      * randomized test on TimeIntervalRounding with random interval and time zone offsets
      */
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
index 76b5558df80..cdb722ff9dd 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.joda.DateMathParser;
 import org.elasticsearch.common.joda.Joda;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.core.DateFieldMapper;
+import org.elasticsearch.index.query.MatchNoneQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
@@ -1146,4 +1147,27 @@ public class DateHistogramIT extends ESIntegTestCase {
         Histogram histo = response.getAggregations().get("histo");
         assertThat(histo.getBuckets().size(), greaterThan(0));
     }
+
+    /**
+     * When DST ends, local time turns back one hour, so between 2am and 4am wall time we should have four buckets:
+     * "2015-10-25T02:00:00.000+02:00",
+     * "2015-10-25T02:00:00.000+01:00",
+     * "2015-10-25T03:00:00.000+01:00",
+     * "2015-10-25T04:00:00.000+01:00".
+     */
+    public void testDSTEndTransition() throws Exception {
+        SearchResponse response = client().prepareSearch("idx")
+                .setQuery(new MatchNoneQueryBuilder())
+                .addAggregation(dateHistogram("histo").field("date").timeZone(DateTimeZone.forID("Europe/Oslo"))
+                        .dateHistogramInterval(DateHistogramInterval.HOUR).minDocCount(0).extendedBounds(
+                                new ExtendedBounds("2015-10-25T02:00:00.000+02:00", "2015-10-25T04:00:00.000+01:00")))
+                .execute().actionGet();
+
+        Histogram histo = response.getAggregations().get("histo");
+        List<? extends Bucket> buckets = histo.getBuckets();
+        assertThat(buckets.size(), equalTo(4));
+        assertThat(((DateTime) buckets.get(1).getKey()).getMillis() - ((DateTime) buckets.get(0).getKey()).getMillis(), equalTo(3600000L));
+        assertThat(((DateTime) buckets.get(2).getKey()).getMillis() - ((DateTime) buckets.get(1).getKey()).getMillis(), equalTo(3600000L));
+        assertThat(((DateTime) buckets.get(3).getKey()).getMillis() - ((DateTime) buckets.get(2).getKey()).getMillis(), equalTo(3600000L));
+    }
 }

From cec9a94b96b194ce35e63c5616493153dc9c19e2 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Wed, 18 May 2016 14:48:51 +0200
Subject: [PATCH 0336/1311] Added version 2.3.3 with bwc indices

---
 .../main/java/org/elasticsearch/Version.java    |   4 ++++
 .../test/resources/indices/bwc/index-2.3.3.zip  | Bin 0 -> 77612 bytes
 .../test/resources/indices/bwc/repo-2.3.3.zip   | Bin 0 -> 75600 bytes
 3 files changed, 4 insertions(+)
 create mode 100644 core/src/test/resources/indices/bwc/index-2.3.3.zip
 create mode 100644 core/src/test/resources/indices/bwc/repo-2.3.3.zip

diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java
index 0e869d06149..9cc526d8f97 100644
--- a/core/src/main/java/org/elasticsearch/Version.java
+++ b/core/src/main/java/org/elasticsearch/Version.java
@@ -69,6 +69,8 @@ public class Version {
     public static final Version V_2_3_1 = new Version(V_2_3_1_ID, org.apache.lucene.util.Version.LUCENE_5_5_0);
     public static final int V_2_3_2_ID = 2030299;
     public static final Version V_2_3_2 = new Version(V_2_3_2_ID, org.apache.lucene.util.Version.LUCENE_5_5_0);
+    public static final int V_2_3_3_ID = 2030399;
+    public static final Version V_2_3_3 = new Version(V_2_3_3_ID, org.apache.lucene.util.Version.LUCENE_5_5_0);
     public static final int V_5_0_0_alpha1_ID = 5000001;
     public static final Version V_5_0_0_alpha1 = new Version(V_5_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_6_0_0);
     public static final int V_5_0_0_alpha2_ID = 5000002;
@@ -94,6 +96,8 @@ public class Version {
                 return V_5_0_0_alpha2;
             case V_5_0_0_alpha1_ID:
                 return V_5_0_0_alpha1;
+            case V_2_3_3_ID:
+                return V_2_3_3;
             case V_2_3_2_ID:
                 return V_2_3_2;
             case V_2_3_1_ID:
diff --git a/core/src/test/resources/indices/bwc/index-2.3.3.zip b/core/src/test/resources/indices/bwc/index-2.3.3.zip
new file mode 100644
index 0000000000000000000000000000000000000000..aced41714fdb5ac31cbe04e9051ccf5571132f68
GIT binary patch
literal 77612
zcmbq)bC4&|mgcW)+qP}nc9(6tx@_C*vTfUTRhQ9ao7->RPVB^sn4Q_#j5rzjN9OnC
zJ@<Sk;@)$vq6`QqG~gc>kXwV~zdii#8yEl=U}ETE$e^YQ1po;!;nhh0YpxzJ03gsu
zAOHXq_3w*{|LNyw|Ngm=yRp8dor$T3J~KTFJ<C7g0{>s(aQ-tmJ9`sT=l>=s=)VjR
z@Ba?U_)pld|LR|1{{@!xzk=0wb}@7@{U`cAocf<Q_8$(01pEU%d{H{f<gd@${u;x7
z6W!d#-pJ5~j+Nfo<=<xTKOxdh=#~Z%LL4PQ2G#G|mIiT8$E99-K?A!5i6$?bu5Zs`
ziFzKs$|Vigb%G-8A6USM_eb{(Tp~uIeL$nFs&dhY$eLW|oQD0zI{x0wInwZ^mclZM
z#~jworbsoTNb1@Hh(tzU?(cCSM)I}puwM8x6M7*Uu=JW&=hD*SP7|o({_wxBn`t<X
zeLhFU9K51BdcJLv?lLGV8*AYMVQQBOX|T!6AIWkZ!W=hKm0K1K3OLwAj=Fo&FPEUx
zdklvv()dnV#VlI!MC&9^eoevQ6dOO*;-(`qT1aU^bcJ~Ria$=$&>9Y7MQ}X>8v@`5
zMn$rxFA9g(BCca9S!zW^!q#f7rCtcMu21KPmd0-=<KO9Kw5F(j-=5}YP4ZeZXcsuA
zB@3`)MpI}bh2ZzP9>DU5moJp~hbz66My`ml`jm^B{fL%b{E-+=4zyF4?g=V`OZ8w@
zv$H=*%5T@R;|KJARPwJ8NcLY%=3nJl8vlEM|Cf>YukcL&UHE?!DBXV$=>Ibn%m0Mx
zzpJo+9T)fijO;(^?*B5tf9mcZoaMdo^~Qfy==0Y^|Gzl@P4RS$|1QRVu>ZsUC7m|*
z=90{4|47tKobZQEuV_a_O`cGWie$OJPe!dmjdl?P1{2F@j}S{C3`qR7pjxMCrYCJB
z=3{AT1}&*MzD6i8L?WQXu{JnH@w<JXtcY@*kV-M4^2vYX5QFhRE5?}}<UeF=n14xI
zT>rBH{#)q$Gnb|OSGnvTlt)b}x@N8bKzR`WAo%}7sc&RYZ)Wn(<o2I1|LlfH#~p5F
z|Giw_;azIIC|kSGq^KED$FkUtGbF7IwFiJU_|-ll8qgnZ!wd`xPZ$r%`9qO>y$9YO
z5M-Me6QNF+k<CObBPnY9?xzrQasBrGk@W|u^Jhy(N5|O`*R|Ag(zi>MfK!|bJkNd`
z$->Gu#VpTl_@FuN&!PITJl2EwB>7z5vGFj<$N~J0;TUsV_G*#u>&6}5vyMhM(=VSA
z>U%Y@=2ZCHkK@$yRXsKjbl<5Tkpvw2t#+1fcZ(eQ&!d&0GhPmpIE^(MuT`Z4oxUq~
zIdPvZldjfjS3a~gX1~U>9*qAC=K0!O;mf%5pD&Kq=)0>0Z`STOzs^wjb(kJhrHH`r
zZ{3Y_RDxr4wVNJH9R<PgpL-fVsmI9)I8W`}%X2-fk-+m?ZbwkWt$Vs)vuX3h;5%P*
z9UMPV=EY4R|G?$8->!dI9H+>20<DqfejO@|%ZfT@vSdEKkiWn242y|zZrPMIC6qFw
z{xs!Pd0j{d!}VUEHT7A^EMAH#o+w+9EzUq?LF-JNQ9V17HAVemQ~8+1$3WMhi79$)
zVU90uOeXZxaU2=tvV%>desnQkAqbmHo^W4#ft)@^3Lj6Z>YXyD-sh1m7jL7`tDr1w
za7PvJk|?KDvl`x<C<>Otsa<KzkdE)JUR!>kc6mY<zbA!>Y(*_i{ZYNts1^CbuHqBj
zN3;IIs#mgfp9uEqp;j)oW0;btrNHbJ<i#ZqmslLBGVNCaCRG}m+cWrKff4#AgC#{=
zngMa-KvjTO^WKaayh92s^GJgxXAiF9CBQD4X3)M-9D8s47UL0sVyY1ZocyvW+f{7v
z!h&X>txdBt!m}_{5NCwWcOT|OBDd2|+0kT{o?GN}j@PyNZ6SRGg+64g_TZe0RFpd+
zZkQT$r@6>ZQka0QVW?(NQZrCEpwJ|)tv7ad8#1M!H{U(F-bTwK1lIsPlHv2~_WC>}
zQXNh~5l8DKb^pjiXtLs>Ty=2{vH@NW$-P3laZ0?hE43%0wN3iAX4PZyrbanhXPOYl
z%qV1A^@TjDWEYs6(@XwHp%nWQJC0bGsVE8pwiDfM=%Hvz$Lh9~?pkxXZ3CKFMx`q_
z!gY!Xf$e!rn*D&f5~+oq`ol`*M>VB*zR^z`Q$qOK`60Oy>o4&w5`^_Rr(vB6O+OBr
z0Bt(@A>!C(v|A39z3!6dRHLsnltZ;7PX$JS@6OX{<tZ^QL_J%kYlP4fwjAme`tUxu
zM~4z-BAaRTF5Pq}aIGSm?J(_l`h`oYR~eRY!7_~JUEOfxYs>M@z%nc9l^xmZ&8p0u
znB3HpNKPTOp$N_F3kMbCvteY~ce@T-`gt6smidZO{pBId&2ljeCP%apWyV`h=L)c`
z8{;rfW7RLtdwQee5Z>@qZG-jtlXoaqC_V@u1n8Hgj_`x6%5{u6o|OfWQZR^3+%mA+
z6z*#NM!UJLN;GG1Z4b>+S8&a9QrK$ND)HRxBJe#a>c&)>lr7f-yyA?~l41k6AQy#V
zi8(wp2p?ClZBAi=b_v{jP!s{TY%4cj^+SXl%uU!W26;sqEEWMbY|jjq=ph9j)^TAC
zs+eERrNs2B_2bTFIE#<@nkxOU>x76w;-aS{=-PcRVVTb*Mm{QVk>2yJFNyT2o;CGq
zFm=3<t)<AQ1c9Xu2aI>0jOlk(v4q&w8)kV_P}}z&i(09+xdxNgL>;I~*RE6c&YpNC
zU@AipnYwaTDTKU%xo5794xjSk=3b>6mLDA&K9Sz(Vb49ABib%ZHl%uLL&mau^m69r
zzTM(Ot*FOx%Rvtw%W78~;dg2foyH|(xvTZctHyKQMvesL`#9Y-Oe-ZaVlMg5L|a!=
zb-Xxew>!|=MkK*~g;mUIwMNFEdnfY4?-^8Y)OP0FYUUbL`&<r6Pe3bZBIfU|OtL5E
z27wJVZ}Zko5o>?Q;dVh_2|bfix0z<ljT#HX>wA!c)BIoES_6#&3lI~sv`zU~cJ2ay
zl~L^*pbXU2cxehPLPd`6=7e+%l(wxgH1A`Sto7%w-Xx7`9pM#Y;~Rkwd*V)VAY}LC
znPrt?34KiJ*_;z-{X(VhR8c$F+wW|ul-?7G_u^@s@sJQ19(i-bKxsssN-?8JALZ*O
zIJ(VQ0~Yc{YLw%>)WWcD>F!6E>xzMX)a~&{+VWrS(sqQa)AhvGJCGBrhi9U>U$V-E
zUN6E=Km?6byT?aaKn&ako~cSMbVf<KBW`i2rHWN60C#w`PVro|^o8{8mS);113%)h
zow@^`f$*ZnsJFVq;kCrh^+9^_{D|F8Un{|I5Uau}xJi4MSe`hkV_P(Wun=nZKTvZw
z+s8l)lXK;%*59126T4&MmppKPOEHChpPRpVx(*~Vqxu-nd`2~vP;FX|JQ;|()K+<c
zUCpYd6iA(BFI=ze@Tj|L_@r}XS9b^spcumQeJM6HYX6l^BX}=0McskPH(=^_zMxL|
zD~?i!*Edkh6NtqZtaK(c0CNV6!|u`MDO{oG8~7d-gt|9*e(niRu4xZ*bWn3n;E_|-
zxn5OBw5X9Ajmfgj_!1JzC3nZxw${s@CbFhdgGT6_iq_FLdXS-xrZIWOs6YrWCwl8A
z5cxOa<RxwI*eM<YW=Xi60+LEtC(|wnXUdiA4CAK_MdMw~g4We|1XqumQn%Dppm3L#
zv6L<e?~PH|S`C7+fgPP^D%&rM?<=n!?dQhY_LOJTdvdyP)~X3(*I@<%b{ZD6r{{_{
zdhzdJ_$h(KPalTZ`Q`oRM?x=xbFOv87m)64vgqkri%GNqOVnPhO3g`BS~wk*Vt2v~
zH)AyT`Ip>Z+lKQpHUr(WZqz;!&CEKdQ%|Kx5PR_H4aS7{;%a#J$Tr(ZYX^%q!?X6d
z<Zh;sCFhL@I5;f}ud)N(N9*HxkMh~gE+@2vP3tsa^1dbR1Ka>ML~{m+aBrXes-qx`
z4bbNsW9#p?6y%m^Z7z?!Iy(C*eJ+=R#YQALh9-i%I=j=kz*aBz8RE89vw_f~pwcxR
z$M?DQ7bf#lLhTvd@9SN?gm3#c4(cxj2L^_EJ@o!=M(XuBT2s`zAMw}aH@8Ynr414=
z;noU*eBsB4L##GPXHmugoGCNK8TE@V)Nq(Fo7Un9!Av?89|QIXw1B$>A9hak?|~y}
zzvS*(1B3^2D9Cji|8xhiqDviF?@XwR7;LsFibk&%s7M3m#!nBfmlrN}>R)<k0e-5T
zNNLwbB1~qVLXX>OBN%&~psYJ!@-!z{r^4!5@OGRkwxmQaFp>+idB#ITMDq^13~@^z
zPO19=-7^3HZV8jZV$h*3;rfSqb%IzLK%P$JEy)P^K9#E33BsThGib2y(}}Bl=!Mfa
z7Hs`8lUZ?>gMOzf)pmZfsjEoN_PoXu-EKP6`tieUn!$`F1!%boE^Z71=kbKkC1&pm
zimD5S^?6wK)=lx<Sx~YbMuqm~(6(u>^^A2kRv))`nQ6@ne$U?<sClLj21qYyD8Iep
zV#4Z(k#L<1JYKbdkM*Uw<6zO{Pa5#;zQQfzba`=F1EFVt)I5>_k-zGUYS4pMVtp+Z
z|4J2ehL--3(1eh5p1JQ6@BNH=y)Njgcc#{Z#Bz<;`}vZ;r=RLDMTbmlL`kF+cmLg>
zv#Nd4K1CXnBZ5f+?kyj>`0qaF@W4@>psG7wa32VuHDgvL<RB`17QB1tvaC)bA_3rJ
zp7g<vwsf}r8>hi{{DakVMWyD8PQciv5(obC;x+TNrMip=i=>%g^zn|d??=s#xcJa;
zUF}iL*Lm%upEm=IvL7id;bDZ@_7i+L3_*!A=9s@qxK`Z#9S63exx<Be;<=hVG@3l{
zlUym>zGmC4i_J)^bIMl^{$!-2r^sm11>Q{ItD-q1yu=<X-67YxfDIjzD%5W`6-ZaT
z*Q1XnsQPB!;_wu8VPCV~*)Zr<0bHcr%-rZscZK*Nx592g$H}*tZZlu&JudPLW`=f$
z*)sEfOa;q3>|q_JNmx9j9>It!;zGGJX=9{e)jZ<J=tX96=dbCp;#zx{7l!A$oQ=y@
zXhM>G(57Dk#+FRc06{pMqE2)=VC0>6aT`jP`TWLHGM?mmr?@Iz;rtZiGRL0{B$T}d
zJS&$uLa)|=lxd9ooyxB%>eD+a;DzciK6nDba*qu_eVuC-;?>S_uq5u`dG##Q0-F}-
zbj)Y#4Uu^W3p1>>V4^;$yF<=X3h%6tui76>JK{PO;WQn(c4zxqBaG}!o%=U<szQFW
z_)I_R!Z>6HW$;i_iLxAp;xL27XWdzF`CB+BnP47mD3%#|Of=|&F7NB8;ZG7vFjaq9
zU(_0hGk93AXw^(H7i<482z;7^7bE41=h@WSE}l9Yh`N@S#->&40`!bQQ&3ta)bE5S
zJ6hki{?1<mJy8J;_)H1<wjP$!jO1|8VDE=p`0Zk43UTZS_Cl9C@FKdZQ=ppUykwT1
z7u5b0NYr!Qwj2*RZ0f-`hV3X_YyDLY0nz~#Ps<G{eMWwji4#KzDwJJ0K(9jWI(^Za
zsnix!>ER0@F#L;(HV%xkY>y39XU|)s>yIWIw~Sb&F`K%5ci)izx$0yBHYEa3l3`qw
zg6pV9cqT7sbJ*(R;1PR-l$7^c$Pr+>i5)v7r0TOzrH}cq-hAt)3&R48KmgZEr7x!#
zu(BnOuq38Bv+`kZpRPvDR-Rexm}1phEiXSA>{_<4bJ0cn*o}JkrewVVm>(uuIF~~b
zt~c%2p^7p1ZdpoAXxDjt<eljb>U5rn9D%9``oliaXP~tDhK0+OnhMAgtyZxJ@~M0&
zL5$U@Ae?yyd$+HR@Ga|<&i4M`(X%?`&T$$o8q?-JzrXna=k|NHB5-|P7DEQ8ttB2t
z2?d%0M~B!a8d;7Wn0DTYV5cIQD-S%3FHt<)!_%ZnI{`Ma&w;M`Po4Y3VX(6lc(o-N
zDDRm{PjIs97(kHED>Olfet!NJnp#R-<9hL1rQ9nMkzF+urIjmM<XrrBTpgU=q$rz}
zbtL&M<V^SZ)ZA};=x|<y;WLW1QO=Gll4aGQ<NDW6`9Mi29WPEX2{Xug%WQsgc%m+N
z+rRst{}}K(bEP^<1abiQIyd8oi%^HT$frQUJB5xj<>sT65zp4%_W@TgNUI@TdSgRC
zuB*I0nw<fh8V0341AI5bwfUwqc<Wad!N)5L-oFXI=(G|^Shwdx>R++yE+{m|^{pQo
zjo678>7NR$)Tvjlz*%FsHjulG^BcYfKKJE>^O|vuH_bUwJP~eew;qAehYHp<hwmcw
z^(>6yGLyQ^P`{Dvmte-zvSY@1%PNsdpS2%P^6Ae_CUH#OR0`*hqfDUu8R#E%8>1k`
zzc44s>b`rJfs!hF3jqsaQoQ;p@$e^?)72;|DCC>mhFeZfhuuJu?k!tiKx^Q%?klAn
zy#c*B9}lTG{zpBQ0_^Uecl}@r_CQ|NK>~t8!oe~DM=kqSH#QQ)osIlrt2B<Cpnj#C
z-6)U?mrs5o9kd_%@mKaFQ7_+u+``)5X)p|L5G7H_RbPU*K(ixZf)0Vsg;g<Qq@tGS
zf(C8^&6&ydmRZT->%xUv6ZC+{wWmn{fQ9`h`ITk~>mqQ;^h;Z_s%6WaJDRC#11cue
z3rOX*KJo$xFiwc8RjOd{(+lp!k|c{k+Z`9Vn=%oxyE2Yt&kwBd1vF_tNFhieX8wNL
zX;KPLeB%Uj>aJb@cm>AwK7pXDR5}=DO#%uyNZ)xJG33Y}OjXwgn%(^x*&g8CII*)2
z%0!&Gjp14c0vEmb!8xq4h7f?wERY&<8p@77G`*jK<4D@xeOuI2nFmo2<#x)gwp-}N
zRKaglgbI*Ba<;syN{&c$MSH6N{io#sbOL1@v9(gq%}U>%IC_qAI_zXRmz7Cyk9bm`
zLz%*S3~#7A)*kWNCL$+3vhzQd29@t>H_tDmIc4C(1o_~(M!rl!Aq}-+JHSw_5p3Ai
zxsy98Lu{m1=r|v~Xx?q={Ql?&yQbno6}0U6sW%@D&0?bulpnAuq(J0dLf4X*Jd!=a
z2O}_Ph_@h~KvOqpTje1;O{B&6TNSVC)ufJcQ$HfYRtty4<F`uI`EQzD;Bp)1#RUX`
zng;^egOs!wU?gyd>Zu?y057EA_l3qy(1HvuZ$*jIF9~wOAc=D;mX+K290x0C7Ftyt
z)d#eY_g1%zMONpuvH()iw*>hnT4sm?^^Xdo9upD$@|OCn0C98VDP9#ax;oYJRGx~<
z8PF5tJ?z~T%EGIA4Y)59y>+2#vOgI@ZiH!P6AF1mzm<j`3i0F!SVjHW8l?J++C&h_
z9`{?(r!+T%Pm{j!5o9^e_pJ<$E+$h5dtnuvIZ^D517Oq;?(pjriq7`I2Ke~q%8$xf
zvrtqWLBdjhVu+(eql&?%_vJS}PJlM2=0IOb*)83``+8y)<XZhMgd0&>srBMvv5vU>
zRAzBw%yJZ!57zDDKZPfc3CP}x<k%X~tDlD9{jGd8J*?J2Je}!|dJm7mjvaa^*DT~#
zd0KG75sIX)@Ge)scE#w$)^tKP-9;lfUtaOt0}Qzg85Kc29sj6Xcn8Y@-<W@5khZi~
zInQzNR0(92N*z<y+^vn^4fNnmchmSPQ;ig2ZwxJC9Q<lao>RJS00X{%8mkLeJ76V2
zuPS)UK?%{i$V9Pz@&a47-i*(9@~mlUZq}m=ba@R(dC^DwnYO<CVEY>rKO6_eCKfeo
z`NIf<@;b4YIm@KQR$dXrGbis?q&w=W>O(#o+8YZW8jc?Z>q$NNsG)W0Hr-QI$Fc_#
z-x0scFPn4m$xf+CQ_a@iN9Or+qfiA7n$q%PdN;%%(9V>oAnz@0E)5uM*MjhNY|hP9
zMqTviOikjb@wGaHiaL5LstZaf7y`x56Rs>xF6K5)kO{a=#S4>;m9tnv?E*%7eR;__
zCWczr5hB_OFRle-ze|^D1W&}lcqHcv#_*L2L5xC!MZPmCBP6p&rP*pR0(a2y(%qx^
z)PZjs)yxV{LfAEsajZ`i7jJd|P>aVKROZKs+DkolL23lwG$$Xxb`0J?ZS-(Z(4Vv_
z>>cjFE{sFCvY3Pw7;MUW3HA=}_M!{LY&XY0WoORfJ1GLo4uLAycY-XmN6(#ht&#yj
zW9WO=GA=W=nWfNiIq8~7>D1pA>Yc5){JjLA9wEEQTZA#&-KT_YXwB`gX%i@e<n~9@
zW@t>6b1HLjNxLZr%_PkjA<cecAIKu^Zqh9)UN;g-Ae~W`j(;3Y|H^0^O5J~6FcxTK
zUppEc(7x(Qi>&J|V$uzD>zI2Q53r5z@*+@?3AK-SYV7dOVo%?{#znS8tN@tl;3+2X
z4Dzmdq<I**r9lq{?bxc-ELKa@4Z$<v=4(6=RW(`GX`Fk23teuqb*k&t$_VrI>@=IX
zdUi6(RD(iM;<re;h*Aii+S3x*pQ;WGZ^b_PzF!BfJh%`YeVD|F_7IbI)SOxigJDzG
zE`M5>f}O2!<AtA!SQJe1!tN^{r*jz4B~h`7{jj9`(E)9H+o)8P`yCQBX?59)!DAR$
zKCPQ-?8{Fb?#CbLSx84O-jCbDUk3RjDxn)Xj6P?mD#lImLQUvfpb72%SKlDNx$)d2
ze0rZxHqN1b16~ge$~a`*mI(ECN+?>r-RjWZCQv&a(@-Fifm)TrR|{PBYO&@Z>xp+1
zB{Xa){?sb5LeGWn5JOs^acyurH_lxOBvSJDv!VA4hR{<-f0qExKF5J!BzJ?STY%KI
zzs&kt-D50Y)_%&ZPj$>1X|DF>h{fg{THiV_ppGvMUe^L7Y#wb6_cO-FDyZSeX~4oF
zf;`SPyWQze<Mvq&qw~?6a4Cc&f?SI{A*;koOtu5^W)AG{XL}@_<PV0Sl?+!ZKfI@J
zn6-fw0f0_6h0UAI-xcNhHqz^*84V9eBoO}LnokU7TLrw}GQmD{LviG}ICu&hB1r!0
zo;P&xt;c|6OrYXS4Xj+0DfC_xCz%2GOJIbq_`IB?%q|2l(Go7-^34)-dnb84-$b~a
zxA+L2^=U3@(7D1valCtJ`NDM?-!yK+{8$GjCDD4mETj~Ayv-wOI&DYr007=W%X<g}
zi(S><_RkU_0glTgAmm^poR(OCVEaV!MDB!SQ-MIWcwQaQDCW#o7H>LXuwZpL0X}h~
zi)jXvp&~|*A99%H4bOOnz2z-Rwz8v<1}GAkp=g|#*Jkt9%*k%@zdW2H0km2Hjm0-X
zmhW=h*6hD{R6P`pB9!|!#P@?&i_@XR=#irWiJ(th8>z>%p5@QaB|6J}(Y~%$wztQS
z%i=LSBwU?+=PxZ5vO3bQfBwWK#J=x=7-vh!-feb4wjSbgd(tKhyrbc^b#sOoL%-8N
ztHt(hP=h#Yie1_$I?+0&xL#CNpWM-$CVeHp#M<5Rz)FU+18qn;VGRFcCC`Rsi<eA_
zgah1m$2QgvZP3qkY>+tA+j4eF98@Plrw`QZ2<r+u2X%NQr#edEDQ8W3YH7viIlr3(
zcPhyP_emg=%R6v@!EzG9U;Q}*7CZ26q5pmhU*S>}WE9inPgso+Bd!b&TwqM$hZO4D
zf)H&~<%Ji8Sj50M`YW$4{(N2QFf;IjAIB>fA9U`TVsf0nj<UruYlK(j4_(8(71I@P
zYX1fP!aOZW+PyYp9d^f$H*dQ99n(mfD?%OI?<Bl~G{Osi558E%@lxD6xQ<LpAt&Q7
zU|4O>SS^nHgGX`~kkPE_upa*$ZytQOs`luqY;CAcVsd_>q+@q_^rt96X!A0@74D=T
z!}DR>x<)FOWpAf(HdXoUK^Ek;*dsA$)-ZB}@dQe^d$56e%_dzq!QUab7%Z>^H`#rn
zJ~{ZJqEY>Y-@|yw*+@%luk=xvvS_&l(!zLdNKYNs%7}jqEjC09zJVjhPGfG|ei0Dv
z-q|r5MVi+a)+`r;i8|<Z5c`?PV7DIz?!*2L15(ZcwvKxCReWUHO1z_y+gpyE@CMjo
zF0C?h+AI_D<oOWdOo{(~%RSJw>$@S`QRwEN{KEP5J?SxfgM4~QgGU)J<<F!5^1fUU
zT61beUQn>O4H4y%&LF~7wSx7O`4|h_S)netEEGJRF6Y1AB2_wQ(~rzM_<~{E#M)~x
zc-%2oUP}KwDF$A@U9GJssd7qOiq#^!)lxGLf{7E!xvVzyj<y=@1)fAa-18gyH*Rc8
z*Z9<0CGqv0`neCp-sQy9Y{fJv>hWjry(?c6+Cs-Nr9tMvnt@UFCW=XL97^U&QXfKf
zA*&RWQ5na%MRnkw92=>WkG9%SD!pW1y&VY%I~s)%B?ER1laj3o82Rpd%Yc4scWFlZ
zm<74k;LyTM-A~Pshr>f6q{DnA7R}%mw&u=+uq1%8;n2L@Oo_prIuc}+l~Kx-w#G`J
z3z}3gg3B;>KUuws@8AZB+*)+`mJulcLi?4`An1V<qxG^fYC6)IZx8hJ5ulc3`0A6w
zbBsljC8gcJ?b)4GCw8QfNgxN*cCa84ftB_{6>+9B$zXOJu>ePQuIiau$g{n_uRHQl
zV%fS39n|X;U)|YH*6WL_mR6&R!$zt?Vz=YG0~e(>qhU`LxCm`<%W8dbURFooV4;xi
z>e4ml+!gVME;q=~sK@;3?S5|=^|oh;VyFn`&vkSeVDebtE^d{!V<J2u{>lN~hio8B
ztu-6Ojj7JX{T;Wy?G?GfA`n5Jwc?wjL88bU!mM3si9~%Pq_BR;KD~*$`SGn74)3WV
z8sOXyb~%HWl|Mgzgb!yRQTia@oU2X8uJrNhqVgPb|Iy9Ys>L#bol`2&2z<f2xue5l
zoJQ06=&E=b-)xEgd;;#5!FZI6+4MRB3so1A+K$wMZk!>`+UEWx(o&Eg!3j**>0l56
zlHzsg5o-LCv=*y;$Q;iG@(vjE)tJ25|Hg58t25{UD=*?_7!9sfNcP}hvl+BA9`GX2
z?8OPUn&kAjWjb%%^cz3=CH^G4-6iBg=}&cq`B@3U-Zai2%k)o2(xJwe+6rR}G?#{X
zGyj|YI>XrZ`H@FZz$_Od61@}dg#%(=yiuzlk^;A#!UG@pOR9MF>l=F*4gd#C5@@Xk
z0-Ho*{h~N1txL>~_0%_uF30Nc$`3n9ZpKT6dwJ%q%KRX^IA~$G<%iz1Hur-Z)MZgA
z+|OLuo~~l#o&try;70xrj+<(mPy4XV@b43Qm~Si*Ig)%NW?BFXC3EKPcM>uC&k$zl
zn^LI&<<7qaNGW|04iDw}pMfq={hYM%Y-5mDmmBhNS;-BS*VKg#8IgH%Mra>n3}(q6
z?C~J9uScE1=2Mjg$y73QndJ}Wr}PbY`I$p1_9}$iczGbW?2zBf2lhfVLN(j9E0j?T
z??K%0p@(+gxeuinjFyl#bKl}m5_7$vL}0IU6^B7QKdeWNo+^a#-(L75d)03#1@R<X
zmHZPQm{Ts*dMyg2rP(3_P_0k+CC!3(6jVNas5~jF*1bp~#v=rK#@9pBZPM8w9|$u<
zN|xWHS^n;<diZJm09FU#l~q4k;yIlF_9rBANV%+jHV=s&)lSly`(<JILM+<P-@(T}
zZg9{HNhTRbx;<PIlDci76wEcUGXS2^%1|d>$*B$U{6wMXl(L7-7`|?iGT0nI%7>k3
zSCDL%e#tk0p5@r0MNyT&Tg8nsu!8GypSy7MXAv%v+xVW<C=y@f1WZ$5m3IYtpJ(hn
z9RS>I)piI^Qlv2&dW2AE;)FqyY(}$QM2XKhXA{1|q)zhd#Zfxz9SN41SYZ$cobvGZ
z7n|%9?6L0$6;$8&T!46S!OC5uKi(`bW*>2+eZE{h&yu|O8gw6C6v*I%M^qq6{KlO6
zP*k%62KlDs;8IYTAN1-XoR}sz$|4-)W2fV!J|MBk(KiR)m#oUSS9{ie@nyA*Lt}sy
z>p-a1*GHh7`rI9{j`_m57^&!*rbTSPTwN@opgkP+^5L3yY<%)o7lP9yR4JqKb`|!t
zqFO?c@O3>oR4E7+E)j*cT?}~cC=zV^iQbmTTSs=DVs!#A@<-&yVBj<OlJ=_vn&N$&
zK^;)@Pvb}}S$C&mkUoa(R-pPStrk)#6&7YZ0%wglp7u2~#3$Vh*UX+kj<uK{nVnes
z7Sr|M`8kyQpn@2|8kpTNteqP*5(Zm08kxrY`QThsBSsuSd4KDBmCZLkEMrnfF(*TK
z8k=r(#1kC(S#Cl6Tf?`xrXvK5hDiSA57Z@x=+noGuudWIENqbW{o7GHpd<bE`@Si^
zoJdEitt1qG*u~?R=2!&%{^JkJR1jSA0i4|;oL~tm5tN`e*x0iZGt5Nb0AipoW}yA*
zFuf_o77*NcV3_mUHlkK_ZkoC3`Q9YdX8!Yz9_Ib`C_d^UEEYEO09ojzNNL19Wk0+i
zN8*YF-8$Tu4UIY%s>cVK3D=E0KI$P|g!}!_Rj6?d$5At9=!eChtdyK3?FU3Lu>6kE
zXCtawR!E9K)5W0tTZD7P7c$aCT20PVZG%m7qWB(@PI3eDSHgBvcfiI=!X+PwjY!k^
z+^YuEGQyxy_c|Taqfri_0E*tFW$BiQzyQGx2Xz{=wP&^5#!TWD&3(_d*^2P05rx{G
z7{V(DS;R4HxQ47zEEl05p-AIUBJSd;`axln8G7O**C!2ys$;5Ec6p#EmqyNyJ#wxl
z>-kG<Kolg;0o&lLThX*pycs3L&tV?z#-3l^-qgo9TS<-T=hj_1v=2PdrV~P1G$tYn
zPae+HspT5-DwU3lLjYLHmkelvhdSZoQ!5s_9touTolYU2NErS~9U-BpC)PpW8aANK
zY6<VaJKx>ZS|<Tu5Aeit%Vc?yufurf9VT-Z<2iGqUs@qWy()C6YT%8GSf<M|tU02u
z?tc~r(*lcCFd+|gozM;YwHbDm1OV(gfvzo{@jol$mJ6o=#Itj9JCB>mCf;h<d<&0-
z60(jU8eK@Koc^7MqFrmo+oTFUL%JxNRKB4Nv)je#=EZlgqLZ{@{-o?6bye8(m%HvM
zX6Wv!QUx!kd)R_?BrE%TU-{H4Psuc9;vQv68r`L1Y6w^kBvq%Syo^IOTX6-D7}&?o
zBy-~i-l;Df;z|qa@qFoI)ev&GUNKtEE;lW&mdzPtaYhL}#dj2Z`w^S1Af`lDJcSUy
zzpAECsY7*yl!FQq0peO|n?<N~2tg(&bsM$fNDM0GI-GX8VFBBV5QA+-8iPbQ$=$9S
z@6dnL8Vqha2>K1$qJN36Izk6vRCdf+abC=aO%wX!+qA)q<aPy8edE3#P9-2(9$TgR
z1lRbUGvBou{>io#t^!?KxF=f1XBw|s*S0r93O%{3e1AO}0&jjG!57VQ6<nZa0=A$E
zTM+#P88pRbb6|+}8{LS#vJg{CmKgU^8;2&4{(Cz@47;d{=961GY;xbBXFk-4XOfo_
z*uokx=SIH9hC+C;IJ&Lbn~c0m{(*#fHQnNU)rkM*xMFUb-g?=?Ze?x0cu9-rsR7i*
z#~&c5-#NZ+>xVzL-TOR*_PvSn?8yiFjgmT<=x5MQ8@t9DnKS}=YNn+AEo&O58Y&`D
z>`grl86HKrO&h^Txsxw-YNuGyNcx67W`O&cW}q|R(kb~3eQGEzq)}tE?>&X9@ahy(
z^$2UzW{Qd!QvV5i&LBOJUTBU&I4RjsnjDDUtpIh`Xgah*lme|-9{nrageyaF#il}2
zvj&!Q-A3#!GS}SqmiO&y@ywr!Sqv7(m&$OVfPWV!QbiB$(kF%3&8&gGP^)OO^67<=
zj6lEuXWk@I(+{~7WHbiTTUk=8gT`9CdL0^6yhW;h6oXFs{ue2p1KO~0p6PM1?6HO-
zEE3~FHP9#20<EChzT|Xu|1pW{0y}9^8XU68Y_pV!>f>m{1I|*SE*1MD?Y3#=-lz<$
zeWl(cm0rqr#WIoNc$m3wJ_J$wIrK?DHs_13tX+dN1>pQoykGJOC?7GC;#;DjRFlm-
z4)E4eqZ)T_bW@yy81<_=QP&rw?PIJ_7Vg4@o0H!j(y8#^vCJk_%^r_t@FZ-wwls#0
z5PdRK1uWxEDfO|a+5+!{;TG6n`|s3MASp{GC9qP)pPRZrQ}r*T?d_EqZ*7=A)N<m{
zu)gL`tw<E&=FVOT?<dbaj(~QvXfDlitCu=I9g^sssq?Q&F%PSVn)Y6p@@4A<D4==+
z6OTVG0{(`HhKc%lZ1WmCbcbF4nv5G!8opkMCPJU)MLzJhOL}sk7;*uls{_t>41cwj
zBmrahm1RYf-jF<|?N|P|K;cEX_7~#u-$oZz5sX$X`Rd;gTd?Ux?HQbS_PXlL`DKY5
z=Ex+EBGz{Ka)Tzqi$wix?ktK&qRo0%9cwU=*o`z>h2A1*A8jJHCBR8+vZc^w*2hGi
zyvR;u3>w*1mPTKC^@bG7y62d8!kHOi;xHKxaChb486A7E>BW<_<TVh=lJ!~9wI|cK
zV>v}_d3m%qgC41H$4V9_kB6{oKcXKyfzO|p;Zumi2KE+Cv8zYk?5xg_Ps%`b4&a3=
z@A2v(EeLqRhDteSLb_Yl@?C0gw7rmzvgu8PIwC^5@?s&5`c(V^PzB=mwU?dztia0Y
zog{Y!Jn_i=MMNDD(&^9@ahk?-)mh>sENnUO+<0sv?r`SpMTr^A9o2@}djaebd7=bc
zeItP1MeU@vpyG_mFbg~Y?ba=|*-oCKpatOgBv8ydRJEI2<K`2?lm+JEohXB6!lbX!
zbN~;#eLShZd}-SW7tvL>2#A|=MR_$xqZ2}wXS`UuY-^1;MtUN-v%g1Un8|{^r=s$g
zWk=Ek0oyI_y`g{U9W2n*d%zwA!TB8h2)rnX!Uh#R8OIoDYlxm=V$c;&Y#u*N;ahy$
zbeGtSKzf(2OR~6sqS^*4Q#H7JKy)S9E~?ZkfqcVQi@45R2RzIwLl~^R_j2Y?u*p#-
zcsW$rk;)i~41G^=BG#YuRrPtgOJaM>ub1|z%Cqy|Pi57*ny&guRh?+5=C4lfd_S}b
zx?Mu0XEx#$j~#z9x7DG7d;_BBE${2mC@&Hi9yXG_;NQoNnl9+=cUDE@xA8*g+(&~d
z4u1nDIWeJw3j;#yh_~yQfSfIXX`O(QD?5f}L#Y(`V)-j-Gbp^>mChPrWp>~Xh1vC1
zDHjj{vd8d7j0(S)?7DY<tf{rCD;p5<|G1~6uc-yb@|D4xlD43Cgo)1Z3AV-uLj181
zT|C<93oGeg9{qgagnSPX?3ya&HhEhED%G_9x+-aL6P4kSz%2K4WW=+|<_;1>{zn8^
zELpljWn=<Bg%ImZ-9&qmU)P{eyX0Kx4ypYE1LWv>)|Ta#=@xZ2@E2m|daNrJ?QR&6
z=bZGE^OiP{ZPL%xpwtdI67f(i*DQ3C+D%QP2^4H*oBJfIcmm$4v|ioYP-}E0su^jm
zw)l!=6Ijk{%EuGWkye7s=Fc>)VHo(`sfG?}gi7>PGO}>S?-dq&u>sI#k3q8)XW8SG
zX!yHYgh30pRw|_NdVIEF&jVEkCeJIxXdKu~Piarf#nFLX>>`u!RoHl&oGNdzA1ovF
z1~a7+izXru=CQ^dpF+OdoaEb8OcSxp@8UvY@s|)-`=d0d&=k`Zpj1$KF)f-qi-s45
zQE?-T%x0f2K8xOxmsD40V8N4%E<NpV19UDXU)nrhTn86>e5<u&kYSa@JTg1Do@e-*
zYw}i|hEGmdRJsymh}PsY+I$`>S)fmD;5CI8T=~s7X1XCb_<5YxN4sQ~2GpKsW|M*8
z?bhFT!7|$4(0hsBpnDS!1c>ws(t!>!O@w4`af()K#JH<QEM`U$Lo6vXD-2XY+ZATc
zkn{j!u|3>kfv~2#?NJ3~x<0;%vT&z24Z-H)EHd80<UO+=HGl`Ido&MNHcBdx^tps3
z$woAG>UB^n71t4SFl<xJFt4_VyCim{z9ySh;iV+Sl-LJnwTk-mYDG}DH&3fV%5qF2
zF&y<o6gPk2=4n-OnxjKZ9r{nxPbzUz6-96@#TvC@kec42i>{89yy5EuCNyTD02WHC
z)3nJVS2o3xWI$7;sMcw1v$)h88?btc)I)qW#92(kN4XvYf%5x6-dW0eq-zPcW|RNZ
z*=<I}BQ4pYNM=Crdbxyo#S`>!R-x)*YqS~B*2B2Ie-)*BCwA}H96i3`u<C^6cg2;Q
zA@{}MFEKAtZy@P?l^o`RyRfoB)I`Wgl68f!2y;zt$@Rud=}_F+@=zC=<&*d_41zQG
z%fx<SP<oW)Oj*mv^*n!yFi0a`jNaeU{fC#dO%d)u9a{!(V1m!jm>D`bsB>#zF|kXy
z=ltP{!Xd8~GCqQA1+Fl0#L!f)@2aC8<d2C0bcHw{xQz>ySSlxbbtq@Ii;S9NU98?s
z;+p)!`J}GBe~W3-G+b)7qI5Io!^(K<jG!p_Mh_z4M3~9G9jaj|T9fc$BNR;S*__b+
zmz%1-uRUGIj0dS8WPQi8QR9l??@6NvKlkOB@$?K9+#M^`Jp(f0(k)oR*(HFT#8hA{
zQo_O?e)7`u${aNoY0PkDb#PCT{>gX*Vv2nH2^gvcY-4)G{{&-+SCJ=GG9iCZ%2d@N
zuagPrU5cWybu<psp~Ghq>W{4)O*3Ev!dItU$(u)0%W9UjDM07ikav+PSs9FTabn{d
z8CCpA2BzNVEEdlS_VVe4KNJjACx{9J`WvUuYP8nyck*|8=e!|ogKcnCL8QJVU)x(X
z86F3se+^gQ?%58|FL76DAENsi<<CQ9@g52rrO++V!YeO4+hk~bXGGXt6xvSgt%7E$
zWE5KZV1w=)NK(s#dV0TQ0CLT7r(t;^EBpDWrc8AGj|$R=P*>?jGXF!j3ZgA^CHhAX
z42M^oOk*4KW~#?SH#wgj2TqqiVtkPpSs$}oDW2+ZE8;;;SmQ(Rch;S_=R!z_N$C%b
zV6Q=f6-oY{5x{N`W#3p7x|&z{SMw9=bE)+9FnuSBiMO$Hfq!>U>!95>WI{C-UsPgW
zsM2{Mk|@sWV70Oc-EJu}Hp)X7u!JU05|zbp=hqG4yo8?6RhJ?o2NW>SpnGaLf^6mn
z<<}t;5_cmTY6!WSp&uYwfKCil3ksuu3Ht5*-Oo#Z9y}wiB<C90?{%iaxs+%&OQ<Hg
zY(@o7t3h#!S}_=5o+$DdQF+0hK3}W~*%JD@l`!?I552B>QIy1<Z+J+n#OP-nw0gC#
z?3A@C+?nD`gst8#2h7AMssKbb|9o**G5%Oo_t?yvR#@JXMbnNk#aQi-?k32%VQ{mP
zz4Au3XQ^@7tgxMbLNSVtEZfGHvo(hfFRVdJx8hJG+o)Gr7=6$Sx^1#2>EDuT8ZmmT
zze&h=N1--JndZ64!Z!D;;|oo5em*A2E(sAPwKC3SAfDZF@`P8OJ8aOYnO$w96-DKr
z#qlB1(@iGW7G$sfmIH}6VwH%6h{f}^)p??)k(w!aB%%2z^e8Lno$;cJSr$3R2}$xF
zj9=FwQxvlH!-NK{V5grIw(xF)Qxp@6>5JDIHr6Or8%xI9iqmna6Tm?t-1CQ{cvCvl
zuX-J<!q(dHec(^zIQvQP8t8oUu8N>F#O$20Q%Hn1KJe;hL<P7vkt752bqfVFFW?Z;
z{XYb-@lf>a%Y5sV%YT<FFIbAl<6bAa7_8*+W+XUN%R_y{ySj>%>DK&iF;{Uou-l#Z
zbhVGyis}ez+O7;I*zXpwK|rct{@E!^EmGzGDseoGeKAG@n*r=eb$~NP*bW`_)tL~P
z7rT%@`QlG4gQVwA^_DELN6*5JBX#bpr0^XTrT^@fUbl$cP@a2_dTtomlX>vgS3;WV
z*PS!aOB7YNpC=gas!FsCt_qf=o(dwY*=elFmpS#KFshnszB0_HUY2@6ODpCCX!D%8
z$^S^DNmBYW5eo?pd-aytPWc)`q@k$^iJe?5GxK5}`K@?+c1&VbIJFv!F0hh;S<dyb
znt9{D|HUl6XDy+jZa{u@0XHGGR<uVzYk1zme+aacQoZ6+sapb#D-N)c9H>81mFaLy
zm31rxUR3elF=Lp8T$;8y=xLU5@0qjg{FB@B8q-W;PAfXu+-DE4!%;_Q-piBC%5uJ(
z8?bG6vFGP6(50w%Jnp4YnNIS9s!u)gqg09c&LGlg5q7FhV4(YDVWr#@#S0<K{LXv=
zB`;|i$7tNyO@jcEK+mZa|CY<lA7ZdcG6jWh<o+1u50@fUB~Yj$>2NKW+g<5$tif_=
zWU0{K^vja~vS9YD&`0Ogyao8C8jMLkzD{6rqmM>!3|mR^iX$?(gqd_9Pi4kcv>2(;
z^exL;UYthQ_|`PqO0$eS;Y>to7rr!@Y$CEFnN1t@p-5C?ZExGej2}T6{OUdB7=xJ)
zE>tvVf)V`J%HMX$RCw1PAbITCVBu69xri+W0o8!i4W*D<E}4?bSX}+N09v9H-_$y(
zs%MjQ7)<Gfv!mLkFV+C39W-(QPk+<>1;*(xIVoN_L5{yoksy|B@obBbV@qMb3JZ`K
zp@g+wkrH;|(!26??>hD*6(GeZNNV2wBLL)V(l4-5{YfaN##={mS-@$aPC+j=PN+YN
zI0isB8N!y0;mS~Z^MB3d5H{3;m{x(s=*eLKE2D#k+|CFMxMo$rMDCML5O)gwiIXZz
z#=Ak#{(y{sl7C=c-bX>4)apZcrRX5&mfy?Q{xasXxD*|;4;D|!Yw?66pq)sqPWq<n
zW8VBUJO|*73<5Xm#m)_h2>zm4a>DYXRVjj$P0Z=+#UG*&6b6Q*YV&_p3t19vlo=au
z>4P<zr~^lZBn2DOx)2O$ma)w+7B8qFEF?}%#HFLqhK<_)6Co8s@67Wgscm!Oq5*(T
zPmod@8utlrJs?{B#xe8jfVpqcsJvzKYLpc&+g@4)-Zk>}0Y}@GG$`@5Zt|BuuLndg
zei!5)xnqgO8t3eTdk`GwRJ<jWOg0s`kN5P4O7s6wSF7X&xk@kOSnw#8Z-2#=1|psc
zj+K12`y>9Y>2>F$NrQSq1&(?$7+o9#ytHAb7TZ7>3bA&pQbsvpJ~b0eU+czsY(ydn
zWm!|@UyP0u%U1FC00vVkAy&=083X|>75Sg73aAP(DfaUi+GH|l-=3gIq{P=eMMr{d
zD*^c#C{L|8J&;L$6R+pWWNk{RTgwUr8-(1O#$oV~=D_U4T8;RucXn28&<+=Z-D=?y
z?v(+as;8LYceGsYLw(CkA(vk2PhcUR=p$^xq0K}Iu-Eh3hV@szNX!H3->YK@hKkvb
zJ`%TjtbxKH1HG7x)JMNz@ErBxjOE#}wOS}pIRK;69!E+2VD5@dZ*+i;oywG3E4mKs
z3$oc(t&)2lFDZAk+1(bC40^pLG~X<-nOc^de4h=);03Urww<DbaOjGY93?}&)?;%8
zm9o^Wx%u(hD!I^CsYpL~_G{eXQ(E=2H{5c9VF#F5-7U3@fi)`OJUW_8lQzm%rD65h
zmIo~p6UJ-Yqz1BiE2Kj8bkJL7yJzknEhS0Z?`Y^prh``HDMBN;4-XkjXqB{Q=zc9*
zGaqC@cJIunul1REvL$`UI;$ex5Nl}OAZF8zzh@kB#ecdkG9jL;el_W9l~aydx^xZd
zc2B{`34BJtr_X2j#HjfB0nLClHTRUy7hpc>oiSV(Zkx(x+jsc#V=bIqC(pHE+4pEU
zzVsuqJ@mEvC%9W}BbwZ9tqK1$2sr|UYF)Hx>8?~THsWd8=+;p`zSG!96Tqqo$D<7s
zS->dO*{AThF)2l^n9!i+z8K}^(m2ob=-x~S=-I1i4MPg<<c@WPm+jZJ=#_sq@bCN8
ziQ=~F01Nws*Sl1!2)nDL;=`LA;aJ#rUTNEh>GtVx$vpO(i;Z`-g~<)^J(4OIVQ!f?
zl%ovM-vhMU&xA(LTtgAJa2?kT1<MR|YAa<u=jgHJZ)=7A$YeH1Az8;-la@kC$+ci#
z$+fY5(WqM+<^jHA)bxv8+)GI)U8C~t*RM)1Ii2e81ydu1bd=dF`1f^(4rgYY_}O~*
z#7;K*lz|rTCY$e&D|+x7&$d-VM*_A@`0V>cD!r@zpDkWrZp!XUiwFI{waoTSCAbW$
zsgaL=>j!oG(4^kab!J+;QJ0SO@#7NP#Z`@%h4b|Uv+>OZUPe-J*4vS3IjqP&%z^i(
zg+%<z%@yLFv=VAVXLfy~0Ovv<xmB}?_o3nV)zJkYiJT*Q)|A+s^<b#>S!LR!i|3ge
z81T%MLF0<YyUh~)ZF5frmeh8a=&`)1rvRcXzEW@zTL0|gbOw7HiL=U9{A2>b9aQUl
zB?j_%8#zq0Wagj8#7VPHq^VtM)=2=?HkHB!+HhIL^k)jmj6YQP9yD0E$!Cn}8?TdS
z=e-i3o(R_gU@GgW6DikMNfH6KoncDbp?8Ue{IIVZlW1|jbKzFa{3@sH2P?YLXE*2K
zmCujuH2<DexXI-(3Ef(O!bzRgJRwCP(EYns#Pg!4(dTp7yV))@s|BxrF5{|%y0Ojf
z#x`K=-!c1fcf3-_UX?2(x1w?@y6MT;<xbp;i*NrMkk84RAo{l1%hzA@qCa<2Te!b3
zAUH_-PQf}Q)1rd|r`%i@?Bj2Z1V#rl!3|M_jn`sFT5tD&ao7*DQ#vq*7L24Z_Pe3$
zr=JM<DG=qY{BDUn|Cm{5HKvEJza0!=Qud4!GYE+SaO#gC@I;qb7x;}u7}zl2K2=Dd
z9Ef^iJrhRgKL#PC4FR+TCN&tR4oT&(ZX@J(abp1L>sgY>LRK~KK8)l^D6Q^V)<g?v
z97^rJ2!Kv}g>O41$B2q1UvQE1^}<DugLLQ2J8#2-e#x|QjKm@pN#C#(Y$at@U^iTp
z*e7;Xvh#Qt*gZ|A#8?B1Zv@g#ZqTTg7fW;|wRw>2FzI3SGpfBH8RB6bAM7Wm#d9b%
zpOXtAWC_hXP4sb|NawXBD@mfl!96<KBJu(a$M~Q!HBULSm6J_p_Mk$3gW=IdkyTJt
z0!0mdk9h}Lr0;<{w-ZPuFg;yB5KrMV+Ru>R;I2P(>~RDeeP7kO6aIi$;V~S8XdmFc
z<(Eq$!v(L^etJUDSjG;_C_PR`a>gF+Ov+NTSW_PoX@FtsdUVXG3oqm;G%{swPYq-a
zSt)iw{6s4p_6|Llh%5Idr^>2dxuZ94G56M}n+e9+K(CT<Yt)NlR&&{qH}8@<tBO)R
zeJIw)E#cga^O~<+?#D%CK3{!1tb|Pv^XWk=!l(d@hye5n_7kv!t?Q-hc^0i<W7MC7
z8{G9-UJc!jKtl3{Hj|l^el|TDEzkB8vn>LPJYKDSc!Uc9-7U+<kEn*11Q7@@F@|Bd
zSa^uG1b4|3>>`yqY=-$-t2C_Y!qfOPE=aI}ic0mGZ1~MlJNvupxd&aKXT?e_eJqhj
z8}OBYuWu%ovdgQ`$`=~^Z0QM(AQ_*I`a?>CmgP5&`pzY5qn?gddGcGq&xUouBvFmw
zcGn~O_<`)%(kQ&a6Y>HfpjUi@b;<=Zdga7%c{Z+j_y}>3-GA4wGxE7bibV{|RI{lH
ze4Ha~`$kP2g^gSA$dPtUdbnB7*ocaCPKO+YLopSbT??PwAl13;CFI~qY;<#g!XQG-
z%AplOg;*V+a#V@>g*}LzP`n^}fPgxzldqiS`=-M%R`yc>_2`z}Ad?MK{8m02oHF*_
zUmfddS{0D+0j!o?#C41y+aw7C!+Z9&hvap{tR5x6JZD7!?R;q*uG^8#&poia+R~)+
z67>6g{*o(^5Fb>POQ?}fVK>&U!iX~_Xm{n#dD#r}dMU=6pZNuMSZ6Bwk~hBNM!J$p
z#D%9U=TPzD-~|KtAY&}(1L*2TF0$=^vG!I`bq2wjHtz23?(XjH?hb(s1lYK{OK^90
zhv4om8xQX85=j1>IkVQRnf1@i#hi<;uj{R@uP?gas)w-Twp1f>bjBCoMg^B!09n<i
zC^otd=z7RTn2KaGW<e9N!Q$JbILo95)Q-yb<z{c2ZUdP~>4{1!St>nUpvVYSnMo$<
z^yrdjve>2}YCRhIvBfFctY88<)Y|H(Pmb8WIr4OQm#rZN3*U|<r8J!#6BXz)((9Sc
zK8Aev(0&G;1qg&MRJa$0$~;xd6R)4ndj{!Dvq#n?gFY^Y3wMcNv%mMiERCnW1kdKw
zowwCpRxVoF0aMhx+SY!9y<8g+bB(Ky$or#_KV;cdhrC3SBN7i`p?xhS*sOnGItg{*
zem~&ZF|5)fr7Lulg^lr3#4I=rb_wCO^H)Ia*?Cz;O;u2~<<;VzhBJ`;!T4bu6>kV1
z_`ZQ6WXH`IvyQBZ^gg`H5ekB1y6a~7<JBL5yoFn@HNt1-VfvaPiY9YODKDZ1o}*;_
zDN?Lw>UuAg-C)ZUPHLuR0q~*JDRwu*KK!W0#A4;#GFaF_8ZKfv$*XwS{f3qXx}g*!
z=gLmhdW*&YlNr92!E@vy?;XuB8c%esz4K`{L|OG-+;=|*Meb=*M6(WSY;-6%xU?$A
zXL`y=1K}!g>VAPr^ZhokJ(1krUWDL0uN-Q64%}NFFhb)$yP|ar$v07eSbuQGa}avL
z;jx5c`l3Jg=rY@BE#BoKqHX=*D$dIxBfzXFz?^w@jzlr_L@wQ6OX2X4<}2>;^!5*?
zOR!xGSvg1VrW_j80#%8rxypZo;fE(34+fH*ykX`!aKYRLke`S5Jpoyq$}owLFep&?
zs0wL*(W^oDw>vPa9k-G3_<FWl&Yo|NxnXU(5oP=aE)iMD^z;v+qW{9sVT@8NCdVl7
z_5AqZsD(4YvsEA?FEzs(*lOr1wW1td*(gJVPxJKnSmcu5Tv|1wdYDP+fIvN6GKsiF
z?^mm{0TugBB|z>07cqZlj@RP2rV#XX#|k7u05n_3b)_s1H<m2ZzzEEjteEEHamxIL
zQ1aeG708+vSb`Bbk$^hK0!kYO=3P8tvjz1@Yuf~m<uJqvT0#^E^B^q)`gUZ(IOj-S
z-bVNtZH?fNIunA-T~|I<k2rhGU#9R8gRgAB@ldt|T!tTt5W&xbh*iM{3=#eK0GH~H
zt$Z##!Tm>-{L|K~OZuJ9m2GcLl07cHQn8q89jcn%K!<SU-+>qGf8gP|)8;SLpGd*X
zGa-Xryi1KUPr~3}D^D6lBQw)7(U`>Skl>K*oD!ByN8GTkq(rHAdO{TFE5#toMZq?R
ztP^WTBX~5^7|}@XkX~c6rb^ZFV?uVb`NeQvIfJs0%%I*9=JE$H9z~`GVrW~yAqtN!
z9!;z;HRuKefL6r4+>&<K|7?1+&HsAhbl-ijImQE`SMMu=!#ot@F5#s+HEi%OvV<@_
zOq;`2<zs0H8yjEkNWOi)u%xy|c1q8gscPfFj54Ih7R(coHE4+FfF`;D^^UL1y?J(Z
z%JY8mLEz>~_5F|p{;JbOS(Ham&#XfM;A0VL-DL1y#8dPkbrqJuh#|g&ap6PV^u)<0
z8oRQ#3PjMVgt1YkcEOzxU5297*w!?*RB)8R1O*|VMMhqVyM#{)32|@PgCQlg>?4$W
zb)5OdYDBO0q3JxQ0T<A+(=EBLQc})|RMHljy|8s$7E7it1+Znq5Sg|jW?#vPQSNLx
z>2Ns54Mqs4k#>C|UZ>rM4M%Dl@Zvdqw;xEf>n5=%E~jxg?t{fGBQA)nI5KGD=tOf+
zbQpFap;&DR9T6J&b^%`F4xk+aFhiVJUzEaYMjG$<!c1yMvV@Gx0E4#s{7<h|#l|*V
zJ*qEb2U5xjcn+mu`=EQU{Mu>#oG|UwNcs<G^_yYPcB01HD^FwMj?bTLeN}Mg2`ak#
zr%wpQ^30fF^7=8mcp(CgD!7$I3`7Kw>bx!?{~kM97ClQVQOy0E?(w~VACGm>bMRW!
z+NCuhTGBn|bshMWxX7cAla)r3F2;R9e!}`8yza{yI`_yZ5j9o?MQC1MOKWZrtFzeT
zyrFfnQybgBm*9>`T-hG}X)@sdrkgco|BV)1HR-1S(zGfG;-#yA^)<IxH!4!!2vkgm
z%(A)_x%sv>%wM@A7{h-iPpHoOgj0)_eS_|w*ADmSTE7Dz0M5OO@EEY`^0&tU%>zSP
zj3*^Pvkf#QmotKXrMf9JrbIc%uydF;6Di<Wv||=Te3v8iOEULXC6jh7G;txIun<JQ
z6Z5PX!@0;1F`%3etV{%*l{Fz|o6E;MQ^SnGGj~TQ%a89?Mcnyf{mEki92&e-!y9yN
zqQ~On=e6@^ALhwAXaT4-X{)xKo9MskE%29GY#RvF!j|A6LM0Jb|0XXdSqBO-N5SPo
zPQxrhq*0s!Po|=?*Xn*B9}=gzWXIQu_pDsf2e@)XLrq^h%AnhcL*zQ{NEmIv)~IjC
z;cxfl*vN!Csw;k@DQ5MsSabrthvQ6=?H&BS3lY5xH=iQs%O(56$zuhl_Nmt4AdG^)
zm*Ob)tJ%;Upmw#7BSY7R9QUwhBD6zPyFc59tVcB4=6|KgL6*=O3P;5bUJ2LTnpnDI
zL{?mL-k2hqV+ZSvd7Y3Z*bd9kcw#`wPH1Kq4obUq4l1aP-wV0Ic_0(rA$RI9?qA<Q
zYDNT8Bci2C3{x3rfe3Kea!IUls#~uLF?1@)9ng+hF9zz}?j_uN4T{<!{B1M-8S?(V
zW%HXRh_%nP(=+Gh^f#fnhdit~<b+zJ>IGc0vqh_|Md%H{U5|Hf9`T+>Ga6Y1+5**I
zD58saN0R5qH7&Ps?m~r+bM_||^1$YNfJ5wYo|4ff^v~z=Z?B8uJh};LBhL{Q{_$$D
z+#Xt-QyGw(;k!@+VXi(})Ga2p*ER>R<q(rHzn-i-H8tjR*~w`f(+Qh4*_C&E^YG@_
z+a2?^dgC7;Q}l-&ISN{QTgm!<)|*wo1t0x<DJluU^C~szn~>_AZ)3S6h)V_yp}0)2
z(Y%1kA-zT*8LbcYU~+%&v8*LPaVRh<aWiwZy3~>@QxQ%|ef{<$cZve09$v$HX&+<x
zqx%LryRFqnEp-eq1}Y@Ejx<^O0X0}2lCEevj6@`%GDPU!lrf~lWG2v}iQX76a)6FD
z6L}^sP_BwT@6HCNHqJW!NJisPa4O#Uj`Ej6yZ8G~=ue0XWV;r!FZpg7l9GGwVL2Mr
z;ck=TDQ+CxI3Qn5Oul(FA2x45As@(r^3Fz8!p0EX>S;~-SW6G&c1<SSUF=oBe^47U
z?c6fzKI}Vco8a&jls!*od;LWa*bAeXLCR#+p858y2i+^+XQZ0+OGf6`O*v_vMB{b<
zzJZ%2Z6^7IC>J0gjF_t1QV#c^X$_|SSsg6G6p9Gd@$JNJPxKO=k**ynLxb|ONO3X3
zvZoL)lASc>oF9;Bndi(M<JzT<s(K<(=kY`{?$B~Bw+r&KHB_zxZqyM8`Fe498gD(z
zL^-YNbU|>Wi1c}2el!IK5$_QgKdS*ei+X}Ty_#{S?hNS{f7S9pL16$aJMlC8K!Pqi
zk$-PEvEKWk{967Vtswr5-Y-rja$l#Kg=-E`Ej)nf+U5p;Z^)5JjKZW?v~SUfl>^f$
zQZA^LwrZ8+;4aYjAPO&8!MMmoCr&6vxY58H#0D9YJTa&I%EEL@=v+|$=E6gdI+Cx!
zQamkYsc1@<68OKjn>neF1j~*7*^#1+Q1%VKswdkn)Mz?1BT|<zi0zTXa&bWB5LmmE
z=206{qhFw4m0+CH6%X4cnm`CCdiSeRFLI+DN2fy|i!HF6{;_8$eb6j8VFDpH8L3C<
z#0&!_Bz~z%dA5$(YJlVs<Gez>glfPlsV82)Ef-zQT7nA{&QhT-pVEbh1{oV13tLO1
z4<UVq|5CwW3JRLgjK?|8Bgh*+%*aTd9?iAjIea@6*gtj`14GW7$_xgriA5K0Jmi|L
zBfHv9b?LB@AuV}lTZV(oh}wgYao7#$=%O?9DN|c^tA6A0!H;}m%PdD|Z(2>vOx~#r
zxL^4x<~yLLp4OnGste<nn0Lw3>zaT^6z0s8Kq4X8xG&n=Qd;GKBhla%rb#`4m`c!>
zs%T8teh=S~e1t?kQy>}~{pntW5jk9<fLVfk`f)JNdh=uP#U0O!(VziTg2{z$Q12eK
z6L;Q6<_`~nSN>#!VN??*!>AuVZm*R(WLEWp4OVYj#4x~&Fixg3xb2tr!xFTm^7#0T
zhKiTC#E$~sNbKLZwhSsiN+uHR$zbv>V1-VPJ<PtnWIhwlfEe)e4MG7gY<mFe!HMu-
z6~-f>o;-84?lA)&zsO~CF)sS#S;9@2Cb4$YKB*>QuY#h?=EI*e%IrG&XHqt9WxY2u
zn2t!W>yQQZoqC<ooRzd|s;KxMrb=%*r!_q=p|ljmW9)LhoXWFy9h3d`Z_jq|EGkO^
zGu&d{(C9+}H>bLpuDt{4qvJvA{cDBaIw3pFjjf7y#$g^+KwcnmIr_S<Ll6dp01@o3
z_hd#w^K7<~#;&h3_opYmG9L@IH`uaJ(-_!E_3x!B%J$OLJcmb`Lf6C-%aD~|hdcs%
zV=22%X!GXeqPTYK=rI=oA)l`)*5H>^)CfvG_n5<V)D0~*O!U*@-g$BHr%W6F4!gNh
zMK-@RK<Rxkiv1w1eHtvcMw?3)(svJHrcdko+1Ie+udq$bRL@sFE12j}<0PU2d~xrJ
z_=@^)pHB{Lh1}^-tK?(73j1U<s*IpI-k&`<ZkB8+(;1mSp}0k(N6iv^TTOwpqX)7N
ziZb=N@ee>n(-dp?85RqhcTzl9VO~QFukzXl-u2-y@WdZ*r!)HdhdyEhEIqgpx+xoW
zKri?t_zip%#t0X9cTT&u!PX|=l>U=(>0E{63F<@N5BJQ;1^M^buUV|!LAmiK@CM>?
zNfZ{F1Ec5*EFmR=6-zTPVQczd4Bz*a#jF`?*tD~;#eIlE;yh!>NMZP%6&c3D@qU5f
z2x+0Z){=<Fvv#v2){)UV6a(`L^{n{<xK^k=uFV?J8v#d}p_#u2k{*oCwATp6L;aLl
zZ9drWvV1$++kE(F91WE3P1wCxlz-SA@&i(@)D=l8ZR>OygU4R0VK#&y_C;OadmVo+
zG>0I`#k4wkvy3jjv-*VAN?V(;5mtbR;F8GtYZARBXqRVvc8KF18;jRk1aJjI#1D5}
z^*@YJ$)EbQixwOo>d~?0b84GG4=FWBw@Rz5FJLi3GAXiG;(`n9#wT(L!fwy)8E0;J
zLKrc8o%K=jdT(|CvN_*%3x-Vb#Jw)<zk^l>EX}w_eKK;xJ~?>lE=%YnFv180i)iOt
zLtJGA)Or-riNUVPiDms$A~$LriqBlX-WW=hql7ZC16L<A-nRM*bHlG6QLH=Ev5qs=
zH7~zR`uXHTnc8{yf$)+p>6A`Sxds9w+48YBdd!*7*R2v&g9aNi-MGJ5{+1kPvJ($-
zM`+};4B5SKm`TPTknK$3Q(<N{96SvI&<|TZNG26$1W9$y0rJ7x)%Wj9xSbBnaw-I?
zj6*0DtV9+E2$~A?NNY_%OE!4JOFtLhV_Cso73Qub0L^~%-ZFIwcHdZy5DXzn`Ezth
zrdN+*a6C%hNW&<_EM&?r!$Cicl-Wy6#Tqq>{?nLeUj$3Tq$roBDfJ%Bra)B?;Fll|
zU4UwZ@+#In72w86taTOT5jZ@Lvc`zEO<KH`(!Irk-B>d``>XH_8mqr?I?>mCCgt7{
zv5}c2=Dj}5@yJ^7Y#JTIa8jt35Nu__Qzn3$_Lp2ho4FxveZLL~XHULG^MGp&?797%
zJy;KO>age3_ifrHqMuiA?X!J<6oVXr3itX2)UvZC8y-tv-3PIErZKD`d>^W#xZV${
zdpNja(r&5hT6j50&Y>zbf^_hTJb4hcP%TzJnZW)%*2&3xd*Z}>%dsjN`>jwAx1eJF
zAc)TK29KF5W-3ep#U5ELIcUuG4IX3cNgSSxu-_|d@uiIJTPmRDPg0S7|904xLG>#}
z%6kvySz(VnKB0XTB)oNM|HMEj=VIoZVTp5=^?5hKDRLV+4Kf0GRdK!h8%Dnog7GH=
z6{&xDaVC;hisNv&6^gRlWJacQc|(Em*jwz3pkBkEpYs^?dpw;=#E(rZx0=Y)YRGR&
z;&oyzv&umU)Wj7F8g+`+D30tgD8}Ts$`DW;GnJn@mx_V)u~`vM?HzeqF3=DA8U)W|
z4k!0^v-P9OZRnAn_6H3DFL3X_>#`o~;(Z=+H%zUjv2Ho^N^UV<z@%U@e*au(D!LMm
z_gSQR5_56tMy=F_Q9qQo;bi1t<iTD43#<7QAG9KYsVGN?OLekS1^u1}ggFs%WRrJ|
zQAEuqs==xTspus2NL47*YJO??z&ebs0l63@%?gxb-nwrdaA(Pc!Mo2$K5N`E_9T69
z@1~}K3|p{N2$Kdqu_)RQENVq)Lm4iy3UX23_Z6Tis?u+)xiB{v`eT2aH`E&FLz^d+
zH?UpAP8Id&9Si$V&XQ=-Bgvy1Jj9_-ZJNpRV_!R(IdvQ+NA|Gf%>&sIKh-#->4R<K
zWK=9&zu7F$kU=6VM09SpnIZ0FADG6J;SccnA!BW!hm99?Vz_>zA;lLDuqewxZBl@m
zwZ%K4rrt7%JAC7mhB@<b?m0YW*=QNa1b-s<#Q;gRF$=6E@lS-nao<L_;<c1EsKKg0
z4U!=eU$q&aAteL@`ZuObGHcG)y97-VJrq-q?%P2ur@3x3eHTyAueL=yjinQ!`{Gbb
zvsx~twP<WKC-nA<S63}p+naxOGy5Q#miL1|+&G^mtMMFWI6kV`*ezNx6NdcG%6`UI
zEH=C^ARQ6{@fY*A`EjylD_M?=O5M4VkebU|pUiK997;IW5$0oDrF%`;O=}S|{o~hc
z)4s8gI6HE1zfr;leIOzJKG5b8a%YM7w>9qsnD&G*Mj<&C&7O9gHV$;<xEGK2M93S^
zRo-H|W2^qTl3*lj^%|FO1%W9TE?q)mZ~@mEyl0mEcv2V40C{*G89v5}VI0@!q%57U
z<lqh8GvtjN?(Hm+<@Qv}JU~Z6b+@Fgx9Ewg`Pjrpg13}b;X*SYm6!^TZUY$<*u94>
z3PO^v#yz?n|9osRpp^G0a(26mb->E-!yfj5nxATLt?P}OdS@`*#J9@yz2)YlGvy@y
z6TCQKD7fgS#*pTihq&xWfwc=|y1&NH_V;baKyBy)^K>~YO4eMXxXr#KOseRU>+zgL
zV8q$an$59I#!VmyLp2o^m1`NtspP@@gDt+z_w=Y0ySD+TKYUKe2(kp4Z1HlPT$eu3
zI;5J~NQjlz_3x|0zA#uIxYb=k!!>|?pX0##vuyGpf#XoegM3o_CdUV&1b%t$4Qu+t
zh>JBf?b?$q_9a~827(m-C;<@em5^%^@#Vvu3>D9V*w@SLSV0Eu?%9c#K`)ouFrtOJ
zEG%F^ucj(5TzBY25*a}()H^DXYwb6(W2o$TiE04R+(<qOJq&I;l@B?uW1Lf<xgf|F
zg*!<{XA>IyK<<b-v<l=w@YL(BBAY}4Z{+e<?)#B2wuDtqa9BkWq<_pL;}UWoP73xa
z<Mo*ZrpG{IU+5B@Z8z%~2I;c~0J4$k(}FEiX0)K|;{29N6biySjo+8vg6YQalJP(`
zk1c>iR+YW|XXhy7%OB21ozP<y$P7~G!B5)^)_+sW*g~cyDX|vU6Z8X%%`};NW(k!J
zz1j|qWfx(B=z8{$K1_x?xD=w-OZ8w;CFc=kLy2kadd`L#XZ?wk#$ze5Zha(`E`haf
zvIieFcm@rFB4a29)pv}_XU~dMxV1&9e0MHA=;5X=Q2h1zgz+5$h9t9LPNHbVTW${D
zATt|h?hqmd<>0bg;WMC>f=)6beU8Rh8_=5v)cuSz%mun}2vCkT4vA+3QDDZQwqFdD
zF~A_k>5Gdyc4ivy=_RmI-LTf;&g28f$_0BO^-({tgZf%zVWg_YVJ|<~1^lPxUoj3k
zEHgdrM_GdiZ3sZW7Z{I!pH+6i+MnH#bn}W1L0Tyu0s_X{KYwZl&DnJHA8|XhJ!RI>
zs&1w8+RCeLjUcaS9Bgm@O+kUJ^;o$o<D|+p<oBXRBW-IX6}6|y4&sa4c0P(r$#3Uh
zQ$RHq{Bk-V5^MiUYw*#K<f!#tFjRId3T{Yn3N;Hu5mfSr?$Z{=82KR4o+g5>Cu>nW
zwdII^a8U`G;~RdO$pY`7LQE*mw3wo&^*@(^ZSrO+jEFqQ0tpKkq1(skK$=S<+7Fh0
ztF&vm`4~sBJfIDY2C3b1k^`1#6%>;o9=r#@NgcV!IIVnu(s!kervl$D>{|12Nnm(1
znA3K!qhWM`#Sn|3ksf>t<N5|UA|quQt}o@%m$AU0-!OzgupukL?mcEC{n{eP{d918
zJ8c;|`DM$k^-)f0JhOa}#WVn<4^i5fz#2y$QdWqUxi-fd%`<=&Rj7_~0AD7m?GTRB
z3{pP%2K9i|l&bQuOZAvWl{$L)Z0lY4<7ghI#Rezk(Yf`+cHDu(1c53^nVX&#4pmUR
zWm1`XUq;vvpplTxJ92h(NdDcK$BHKw5Vi4QBafFxQVgrEq*D4=C!@n;j@9ySpi4*C
zv-#6fHiKvo&u_88myq0W5TE>`1UDE?`NDo_(yZ4?G1M>(|6^f?>OtcnJTz%S{k*PY
zEP-aLBloXbB1smmOOAoiYdNC`Aj_(EHS<>tvo@dzGQLrV8P7y`K<bI$7`V(z+A<;D
zkdmib5gbr>H_Bl|ls9xQQ<{DF4e5JC0MsZU@MWDDtDr3UitI1O5uAx-gd#6B{i@2V
z!}EHFbU$b@<F7onA$^e->!(X#XqaUeK^%-hQi|^t!*X4C;(%F&njC=e-~c6()BzT{
zzIP2=l6RSXF(*w1J-<GB4ZkJ7g^u1Rxj2&H`N(^^ORi3+>LH6mGx9DaZdKjKLU=HQ
zw2tx97YF(sB9qo@DlymyPTh2Ak;ck&%=OPmAzv>y%H}g)^&)iSv7Trj<~R+c&;v{R
z=Mw{_OeN_bRA`&C+9wx+zo9zvR{X5TE9MQ0Yh@p?Z?8$wseE!9#rKcthq!_4Azln&
zha@zqdD7ys)Ty1P7OwVPq_o#5ub_JyoE-@64?$+Q;9dKT6_iGr;34|Bz{S$sLusiW
zi+@wNpdZOOQ%opO%GsRWXn<!M+f}2oXJq@<BS$D#R2>j)nnKl1CA}-q5uG|V+@p`H
zgG?$7>>~+AANaP5;3Efkh9~~9#)8{@Jyi$@KaD}ujQNhS7Osz6-$yWU9Oa{s+3+FN
zaw?)3mDdz%$%d_k4R@InZe5hd;W>&dasAAq=7}?~4!WEM^oCh0Wp>C^O=v9j8$wE?
z8#EbB*w1@dxCHdZZCdGMI`Tz&J$orV!e0ER-N~2eHvjSr?2&i9)t%}!Ea)4PY8Ig~
z{cX5BA{?v7id{UN=Dw5WO`7rtqO5BvXpIvX-D;t%(zj-nNyXf(*4}n=r2UqR^U_RP
zJ<L#UD!$03eQ9kVk9~_4#(MYd4Cz+ctNrCO3e{73HS5YwLrM%y|MR+>BS$u!%IsMR
za_a|2)Wmh(49}b0%^+PYNSxGE9Iz1YzfmU$Qq)=kHOuMb+ya-#cPFhlqfT6C5NTK`
zTb5K&Be2MhyEHq7c=3<Cc%i&(jjVH2zb@NHS_x~(J_N}L8k4xKG#(85hRAh0z4~hA
zuptlLJ4tH6L@NkcM<JP`f>1^kox0bD432n~n^ROyB3@Aa*Dh8%&~J)S+k`j4b?OJR
zA@8j>l^BskukYck!W{`7Zx*FGZa@c4U2e{RD_?F7XnN&=t0wWB^}48GK)5VcY*RbL
zacbD=u<47dv792RhY=!Gw1o*|;OdnO<ELDdVFdjrPEI>Zq}C2@gbEPa<Ov}~p!$FY
z0r3_l=L(Af@A&sWKQ2YhvyO2O4}srTuuzT+09CG|fB&K6`}m)Md|Pv|5}#aycd8e~
zSPeB(%GC2qo()X#H_M_Z+Oui1-6n><n`vNXUK1usty-~C<f^QE-be;+_2(?QNi~RK
z+|Fj7`si1`<SFiZ8gr@^qvUaa6=S1JbU&(84iHzYx&+9s=0eZ;Oxz-Cv;&X-CQDM<
zE?$sqHndix2Z&-v>k$-oEqtYxUF2T^GtySl-H8xKImT*@dYWyTAy7SQW9<>E>ma+p
z0FXK7Zo&Gh8pXKmMw5S9@p-SOCk(J9hJbx^Q4Tq+srKJMkxTmpLbD5O(kO%d#4G)6
zN#_aft#baI+357aYt@a>qBT<AFqS!ex8mP1=4a2>Yn}mSHvM1`P<CyZ!e{?=F{rp%
z0O-X)bDXmu(sNr=7~t{jMT$_->wh5q=@2$X_igrE*gg*_US;4~*RLyWsN7b#H;u^o
zU#EqwyC7-3F#&rH#=Z~FA4Afuem~{|=aQwrUNXjSfqnpe6e12av3@Rk-S{bUYmYUe
zYAa}Vh~!3EtIl8CR?8i9*bk#+siHEADut!$0~)iPg#ecKYnX@P<0AvbLd;3z4ydsu
z3=my%6%(DQbxd<waNUfEo;XJs!8l`;Uogf7!W49o)Hl?2mcf2#26h~6xt^g?=Q*En
z?egY79A%+uXN^gB>`YUPT1Tx=3XqU$GU$*QYHa>bubC(qBBo_`p|FBS{@y46;oi7K
z?^8@yg`#hjQ<a~@-haNJ^p}6+m1xk2qaky2vl>rW{W5xTr{h_zMB)g@0L`35+1vuO
zk-%`I1GSEE&;M5P4B=eG(~<4Y+#8fAt<^lools;o_>4OQR1t{7TLzFJn}$;L<W2%K
zuHqQ!h9g|<Z=iV;%Pq_6$=*XVVqbXxF|jdg=Sf_v1FCBmAlOHpOIMvl7F!8U`%Z-0
z<%zMT^u{Snd*;3&DCPP{n>T&!R}*uMJF|$awFY!Ems#!1w+HzeLAh}Ul|a1t(OpqB
zx{XAANvybBG>pFPxuhA^lM9~LXW<7N!UuQf>~GrFQ;KnzP$f_8UV`x`6wT})69%>7
zRtkNQtBQ=JinWuPRKOl&CS>(%aD!CZz@%SVaX-|y0hkdqv?MO1lHWn6jv{i*u(?k<
z4)rVlCuYTu3Sj~37+g^f>LlMO`_xI9mi%9}n$1w{w@0}mh{a&JAMe~S0hy6YMS2~Z
zOFU%z%(23B^!d5vlMoD~{!t~9n5la(N~zOg0OVu~LUv;?8bj%dz#Ge;CqjYib8eZ9
z&;;*lp3KM%=mp<B!$+XWPcWR34rZ>Lg#<>JcPOV<@P-)p+(nL~VSvEVWq{Ao?L-w@
zuqIhs`d4D8uw2$QR3}c8J(~KHhR$_?(!GT!0b)3hjsvNV?0Z?;$Usmr3c1^luRHbT
z@j7-pcf>StPX<5Kj2X3|zv$LE@j>PV{!~9SJknwf3o4Enriz)Ewl2n#C<?#1^eLR0
z`$k^)5A1{}ZCVv#P`Np-5!7%OKTt~gO<L=e>d?uacp6YT`EOA~Gf#i31cix6t5yjP
zyzcLJAp;=?33T1@XQNE5sq#%P>`&y=rbKk;4@eRfSVKx%H&b%d1^oAlq~l%d3$J1<
z9V-&EF@jTfNa<caAUTtm;eMWGv`mGw<*}VhVfuPX<Wh+-1P+7+<f(*gl}Uci6xG(q
zBwOO9ma5;Z5&1@~##S(^2X`?s!(NY~Dxc(yh2BNdFhL0k_ti?;2tD@JV7bGj?23<1
z!0nHdPP@g-?};lVC?e>q7UAW(yRp{FQXp5%>(-VsF<r71e=VeH&hUCB%ffbZm}p!7
zG9|il7^>mr1D_$HrgM$uc!duTI^lhMqUp^u`E3-5Bq#z#{h?d?xg`d5_#2MP8b@jA
z*DoKD0tBRAY{Y-aHsc;DA+qD|&VjL@CxY;)LfK$8aDXB2GlK>B9p{xkss}R}%U7CG
zlPp6$>9rcXfs_}x+0X6i?{x2iMNyZ@Ea8Ddx!ZG-*aOkSbScixkRCz2@%lM}<o;94
z5wBsR#F2%17PI!)0ERAS+Bt)gwnA=I&7SnpU3IZN&ZeR9<#hbA`>$|dK2obgRZqkD
z0p|V(^qrA7joXW-F_8#TNeRpnt7rUSq;WfM;gnD_k7MRID%F6ss^tM}Jw-inLSPa@
zqy>D;;o$M>kOUm_H-`A*ft6~{OlHa<oxhc26%O0p{6IyebS$fEr34`uOx+q@H~EP{
zEK`CVbKxqO<4vcF{<+ZnLaNF1;RH_N+kn|AL8@?%0=y8SEePg2&c(U)z~QoUed`H?
zU(89X_#0x|*r{!CWz`;XDQCqAg>T>aFgunl0*?4}Y@J)|cEl^rjLa48f4C~`jSqug
zBsP!wXDcrL1+z#-2PTNP^iaA#`U%j+c7@kX>uZy+yZS(L#U{Uw*?;EePt`1;-$A=V
z;YGap7fdHEgkDj-J35+eS8->JJK!bjsG8VV%r7R)h!a&W(*NGu-A|*h)nN&Fn}K1Y
ze;W8Qqy2%eu$EwUP(4>oCp&cpQ^23kG!e-l9smOt$cGWTP$T#`KsFI*$+@^6S8M%h
zKPwbi9@^aTzUC{RmNFhh(dHI}J`HcAlHD?E36_LnR=2Ba4d<Ob`?|@8Y{tN^K$A(G
zYAXXitv`KEm0SX1&M2#F(Vf@3)0roH+X_+ba1xC)`i{|KOxLu&w=rxfz^bQHdN7XC
z`$iZpObgKC2Ts}wI|KAYO0IoV{<QJ`eIe^QGqot%2o#E9QTQ9Z{J>?q44<lAtP2+#
zz)^^8l0^{#iB<Mg6vzgP`2gZ6(1lBdIf{9wpj(|V(A*G1B3sP588w^D`l7!9iz>cq
ziu04ok+)Ten)YB))!~`gS;%nw!Ox|&4^o=?7N;5&o2CKpjSHsFT=Y<Jf3!x(WWcYD
z0(Pv9fOcF=ow!U7DE<EJwr9*97P~I_AJQS7A%pD?Lyb^9Xi8VF1upoxGA?@iDF^!!
zJ`Kg(5F*N}07P2W%!Quh3a>PWC`Sniadh1(NKgTltgoaUBMVVFVlSATow%Hh4j{Uz
zP+NGAIfD+UNuXU(_-ukR7el6E`*#gcZ=ydm;cm=Is!0!k5Np-nIFb7%#atOhr&EW0
zeLLh_-h~Xns1_UO`#wQ1UBNM0V)WA^%@NU;iWVBRGIXnv?{L}+@(zAgh3=Oka64MD
z?RR$@v6}@4B$vK8UJ?>r3^xBEB3+LSI&lPHc@D>ZU0FLLBijNTTC9R94yQ`N30~dg
z=O>Q;S{o`%btYu3XUNGZuew${>rdSj{p~cCJU<gH9H=m%Ok3d63{8&{NWP-or6@L&
zNW#-$J9%*mHI6LVJ1F`Xw+88FZRA3cEpIz^Q(M+am~&7_1yfJR!~4nR5yEG2tPbtL
zDoWuan+|@xydtqCj{~qT2G`wBE>8?&0F^`AyJbn2X{@WDbhV@xX-ePoLs3bL5+);N
z3!s1d%1|?EhzxXx{<&53VXluBmdhLq&wk~S-DVVw*Bl@w=@pUSRfu)O+L`bys5E>G
z+`3^gm3mwGDElQhBngyr&T>4LHO!O2AfE3%<Pwm&w6KXknWd;Yl)hP_gOV(0n7YJL
z3x#jY7_PnMli$GjpxX#hVJUIbHQ{j-ffb`Pgi6n@m<$R28a4Z|5Lsy&24U)?Mbpv;
z;E)a-$Rb4){mO~s4BQc^lr#J12VY(8=5Ji4iLtw<x1n?t|8*mc-H=M`9cqWOw3>qa
zQ|qW38h~_{k+3L-2s_F3fkrZFS{4b+Rw`<pWWvx?%9=-jqHU=o>rp*;NO?Gbn}4<(
z>Lye3XfN2UO(|(?l}ZAK_o5w&`zcliNbSudOED1;>#{JdYSv`HrX{Eq4uBTN?Hfek
z$WDpgSuoP13;m9S+8nhx(?Mq)WtnlUEM51btu7f)PfD#$79ITU^{lb+W|X-w%HK|m
z_D(^NAphEdjrF`R6Em=vSft?;Y*V+3C9qO(qDqVB3L2u}LWTKn+8Wb?=Ffw?(&%2|
zan_qd@c=5~W6~;+mX7|&2i>vG9JUEZe***<Q(eyalgn=hq$b(b>k|HpQ;jP9J(8&E
z9l^y0B~jv%=u0Sl+k;6Knnv6Ax0snsQE^HI;htOsLF<JZ>dfd!wJ&i0pMGiPnm^#s
zp&-`<8ivE~@D1T9K1?p%Gkg4rB6kmdz~WQ#Z)4KBMP3MmJVj~?y*eoqEuR|XU^OA3
z1v4m^No+j*cq&}4)4acnJGfk^WNje}ELKEM)W-^G98?p>($ES_Cs*|}t}u{&;+j5-
zQK7psvH6BV4a#!38svxx+|$^@5R`$z3eZEIVn$GVG+AF~GjLdxMY-9I-42#r@0o|3
zUuib+C#+jR5?ZZ4UISkf>svgEG6Yn>q)VhU<udfq;fHb{PM!VE>x3R+_E`-RwnmOa
zn%eTgBhEnBzS3j3iT1q1zORH%7nCoNO8TA+e}JUhRrTH>Sm}3UP~mUt)j2zkE>Rc-
zBkUco{JOc`-f7F#?mnyemKYIBUn@B=MJWL56z^oX<q3Ptd`7-T9t~P=f1Z%r7O!d`
zPlHybHr9p&gSl56&gvK|e=>ZGw{M|Br>wuW)>GS300RqLmF!PJEOxEGkTs2r=YZ1s
z!1E`&@~W=owv{Nv-oe`xHu|wU@-tOlbq{htDWcGao-O6QIa&wV`%lopIIEoL7ERaV
zpi`|?$E;{g3RPUKje10N>nP->#TtS<1zQ1Rcfn8<-PwS7aOr2M?mn(wW96ietn7gc
z0SK7F^PMeXlkL`(#)O?1vD}J(C++Lw-qPOs=TJ+1p%Azt-#A*gyTGS3O+Ilv^i}iw
z%Du*tvL|(R9d5+Hn9F;qfm-rFu=?z8Mk}chMS?nzq4xbOJ3^par!0kkJaMbjt@;R;
z=6>J}6{M}{B9kTqDiH_yy|TvKM;nZoNtVo`bhnE6Sjc=beCi4fJ0UimS~z$f6R%-0
zSJs$~bJ;<-qnOj6sGmb`DyD3Q^ZJRWp@`Hf3Sni-uM*JGZY2nKHno2}z!Y?t8|e8V
zm-;pRZCLuV00BED(4vPUe#435MsAmc`Hwz8fLwKx8V=H2>f4{P8Rdc`FtA13KAkB@
zF%5y700`}T4aXfZls`rD++t0tg?403Phu+ECDIc0Jk{3J+VrZZMSR3!#tt+%j#(UP
z2Z74=1Kue{V<}lr5ARJv48y273}%N@FZAQ7k=oy?!$=70ijA@NBQxYnLbn-%l+Q@c
z%JbB2$7$`v<*;YjSNz+9Ke7Fy<S~9O(A$}9btWpT%77d2Otb0m)4qAoY2yo3Pf4uL
zW=7n(DAf3-p&MM!_#!KFmim5agOD_1#uv{~B^`0M7^F|2y*tjZ2%#?Cz&M+k|KR0U
zw?P6%MIv}tP?q%tuso51t0niTy0iWn%vk3u3si#B=(naWzACFb<R{U)3Ydyp#o<{8
z9wEJvw27uMVK@~eynbM;u~>?~Q@DMY5v|5bH_K(j=m(E}fMMQoO{L#81!46#1|}SG
zFL%R?;ammu_b58Fo=NVCp@G)iKcWoY4c8X~sibNP7cpFQwuiS;AE0H_p!JeIGwA)t
z4!y-{N1vdz9O?*KLG79l^eakr6Z;DK=oyRVoY;9HUsg^q<*Y*)cxlE5iS;f-Vs7^s
z|Jp!O@7-9U)BgUur~6ZK!K_)UsBfL5{}_}`C%usa+qMI~LpbX}1)ZXD+EA-W@5Yub
z+n7BiXBy$9cMazGrHo3ET><DLkP;33{4#P5J66#V&Z)SUX35ODg)i~I^iQdqBUiAl
z2GKA`!dmngw#+p%6kX6dD*T(_>nQoF4Goetw*RtOk2<KUy=q<E{(AS0EUQ5vz}1<W
z4l6Wy&1x1%3LViceBxLEuifc?KCKb7#%a{>-N;)`M!z<cg?JWx!enq;_S8UJJ`z)@
ztfWD!@>zGLP^ceLHJlugd-7ac7o92ixn=vFM6*h-AR)UFQz%hYB4xN&CX0#Z@?N<s
zX|jk}&WM11fxCQXU$nzT+5WfGwhMmC`(}bnR=?`FG|axwbd%G{J({a$uf-H=&<y*V
z!gpBgm;Q-N{%P_~mL48!1nAHc^|9l3ot7Cq>M?HZfGCq9rs?ihGT#*o5&&3r^h-+4
zl_S_K_7#>AWq~eyGl|@}<h2LQ-ff{K_-$rKaITuGzNvdCK$vD9Lb0d^ujeB>Uxp-a
z<`Ug!(p*HJW<fF^s7{oPxa{(ECC@l9NmD8*U;IV1MrI|3H9#ly!7Y)@ZQNcpUK4KG
zD*5}I-XKTDeZg9^!#avKv8z%GE+XEtoS3sjrv|&qs_L!|CM@J<M<&uHgn~artkL4R
zL9Y*`v%lR+%LoRog|-Ngz`GN^ySgl<_~c`~SNt93X|GO;savK3K_ym*sP}AdtGT#^
zM)XK;eA7iyi>GoG4P?54U>&1|{pd;>Nyf=TzZE6UH>@=Q3&(uxD<!0r?QenZHj5f>
zv(nXEJ+ybUhB)QRPLkH^ID)6LRB54C4uitJXsdZ*fL37Aq`aDxS28P$;!hemsCUJ9
zCid>jUFNEoNSZHWN?C^5XwLbIaar>%BSI5?`BHnE;CatI0-RV_zUqN!M+VnC{pJ**
zzc+HuZa#8ZE(OvD`<|Jsd0Khv^Pa6(jZGLfMoy))p)02$OX{=wZ2#4bpz<Doi{2;u
z=TuOrPoavc`d!liW(umZqKjMvmv}4zyI4*5c&oK6U#%vUu_4MB+)@>kYtOY-4zpXK
zG(Ll&t84s7i$Zr^MUX~I&+&54kP2e<p+pY+b(v?pKmO$1z_yb-d&hL|khnOBQ_6P!
zW#UwUeINXT3VYduz(F{)_d{;9YHV%UABd3XS#+F{vA7Wu@EIm8>$>;@1oPK0uQu>~
za#Xk`wG^;g>6CtZ3B_*Bn!#x#vP~%qqAgV0IeP%hJ|;8Q9Y)Cx(u1VP5h93n4VC<G
zfk!lOo0RzTV>(b+!KU#TL{gVItJgu?))FXbx24*^9Rk8R6V~fCqBXXwzocpZ183=P
z^c)M@iIsu4BEu+{xAsV@_2&n;<YE{GU{T<2hS~@>Fnee+yK)5^^+ie6I2-xip<IX^
zcB*JtAPts1dM^yaYuvsaIrIW0aY^W{heUwNyBdE))qNtAsPWXgSj?4+U_svR6wQLm
z^9F*K@i6mEqGCwYL+%!@8??!VFnxFE6NEG4?Z1^BpC2?je!~5$VU7eb0_D{6*Pzvz
zwhpn}EwTTi!(fHsxzNv+st4e6T(}@r^NIO-di*8$hZT&`mTyOPj8cOsEBPK2M3_pC
zItf4T$1eYM^Mh$f&2b=$ncWxULa_2#Wn2_WMUwdnUca)!`jhsPKh%eW6bR)9M&_1B
z#5mY~UqFaQ(7+cvnhqy8NJB;)kG)up{MgTtQTJ|o_6VY7fb3igo*xWqIK(L6T?<r5
z5CV<>;&BsDDHUZdXZ!fG-kO2$4QG8pd8~EJ+7z6uk^t!|fH`3;M+=`6Sjef&1?JIm
zjgvEVCrCVH{d&6O38%(~)&aJ<yz7wms72cD38u8hlTf@n$rO&=9DpTQ+%IQP-I0$n
zXJ<7a4Rqbtke(S)aR?kpPpHQkU(8^@ZMHFCvRZ-41y_8%O?rf^{#24q@>BL?YAuAk
zJ_B_AsWqBlm}eJA=vH5lwfqZ9c5D98iq#gAu)0aNFFfiS{64BdH&meBb0$1IAE+34
z_=X-&XSgBq2RC!<oR$1IG%g7O+M=>PhsWH51kAn^4`ce@162Cz#FQ@H4$ZP94}H84
zdB-+m9B-{hQ3?9d<++<m$xl*Kn<=YSXf@de4sWQ+b*oN)>qL`zUtm*s!k4MUx3y-j
zU^LgHId&rlP67V3AriJISA|!eTrzH=IP9sZ2lrb9vGV}r5Bpf5h#AxZwK5j3ylVt<
zmzdO!fJ^$-EyTw=OFLHt<p@LUem*sclLx$+b=xG2-G%pxv3AO(sq?)`j?Y-~_O-$r
z6pj#A3aixu^Ar(B?Y^Sd$f~XK)kpL$(;k+^+Zv^0x{+LPT|=;&<C}UQf~#o*lja8<
z|EKPea!;HQHj5})p5LIhkW*AN_BgB4D!7dZ`vm4smAamW^(__0(Qel(gdmI_4>b(<
z1?dG|uM&S`v3Puz$BS&cYLv7`wFPCh!t2CpM}N)X-6{}wspUUr066rF=|!kH<DWkI
zX{`Y8B!UD`dmhiR?-99Yt*CoYBjjJ(IsZ}tG9VZ^b$j8#Hn|R?DDCD87vj<6)pxes
z@fDtE4vUb_!5HJ%Kw1CpC+(}#Zmdgs#r$uw0T&MxH6~OHWinvKgP&0<G^`*~V$psz
z>e#AO5Sj^8EX@E(+Y^L<FbhG&q*mkabIne}bF<DAZW>9>-KT+-mwkcNCiMuJBo%WH
z)24Fe3A#25X#PPuO|C+_q*e)q6urR@8&sQW{h~KOQtzoifCs<-?i(^E9YD6Joo+zS
zrXgNt7q;*A)ItLn1YpRM^$OKgeR01^oQ%2%6_;vJ{X>3<lXt+KYA=O~_{#oy_C0SK
z2aCI6*o<z_F-XW7^v%x|uLE=j1%i;n9?w?XkUdp1qSKN->IQ*S&?bza7Uy*qRa%Rk
z77_+Ln9X?%ERUN$t^C!|5%Y4Jh4C0S#_dMv2jmiO#)}wMdGf$jmp7qY^>4dh+9Gm~
zrM0HJP!|N*?4Bfbv!x_BiBj+=iWQ1#=O1Hn%RGC}LKY=R1}g+*Rxxts@VDaU+y=fU
zJ{Se$v#PRG%knO=lb}#jwpyv9{Om9(PKF|E{G)F-%`SYD>-zp~kG9PI9ay-RNc^y*
zW*j1}N^D_7sKR%3g<vfcgo+A;+t<2Tw%>rj7S=4*xVO&u8>Arg`NW#Qq>iro;YE2h
zZZd&Nx4<^ny}pCVx?2hOtxC%7w*43OKk&x?$;|x^zUz`9IGfEsUhcPl-+uw$)$6~(
ztN%BA*Z<(<vRrCr=VoWv+oKsJo2F(Tq-SNPXk4Z1869M0BpVD43=DYLGj=o?Hglp*
z*KV%yrmr@0u)%Y1v2j3@)v_SCvFUBF(_Q)s!WwNZ9t`eR(G9YFQ7wbVPqN|_lC1??
z`29Cz?*Gt7`DbP9^Pjd>sQ(ImPLBUqeMkRY-`;;H`zI>hbmsq8)c1dL5Kl93s4F0>
z@&8DILpu1$TZ=PLVp1Y1MM?9JwZQ>&!~$I8ZA^Tc3<K1iETT;aG7p0EX<;=_k_(8^
zGc~hgjPu}G4r3ITmpB;lTyh{a2QW;tvxY`S;5@l7f3Q$lamxDX+pD|S1dwpD_3TAp
zUb2SVoTyB)Do@EnNBF42BVtls79yL?<$m@68#(o1TR8QTA0FM-VOkJ3ViaS7KGyLw
ztMYa94=~+a`SdEC*;M@q<ak6_*?HSCQ(%5x{RgNRNsZ0C>t6hS$@+?3T_5kitPB2E
zRF#v5)&Di?Xa7CxR~e?MhFAZH%oAWx745Es-PD4QjWgB${+9a8jKlv~CVP85z6bwH
zz`i>cmvaAt)%&jr`2Sue7ynbhIR?p@|C^v81&0IS?DoxqLJ-jY{p0_^hW?TF+N1NI
zZ0P?{=v7Kr;CBB3oQD7d6a9Y$HM@QLKV{td$GpO@7~0sf#Rso806OvItW0r<O4&K2
z4SdjmoTNu?-Km{Bc{XB)ORGqAe-`!*xySbf`sqylj+{$Ah2naE<?1!(-NxhfXjl+n
zJcIty#A4_r(@2{CTay7!V*jhHKHHA$T8K4J#5CQa7T%)$pmO_<g7g$4Wz?o+jlX7p
zBsvuO3-MI8CCrb=EyyW|DYG*T(<!#Y`=a6?kt9dSe~r9Uu&ebbNNt=yy1PvTv{9sp
z&%~V2XKcKna5LPJm8WBg%ar(okap5=zVdgLQ8YApUL`YXkxP55Eldu8b7(l4-9BA8
zXyK{Y*gtV{thueFVPYVj>@7wl<w7EgpS5-tko6aVZHuk6D{}?&>A{kV<yc+f{OJQL
z`T)vg3Q^bQ)bE_sg*~ZcH-BgimKa>yshhP?Nt<zMbTkXF5<q{6P4NWFV}&DgQql1s
zB#4a!i4E4;-MNYKdx=H;5%v9qZ@L5j=aV`}cCf}-5)7=<1q@8_|8av&mE0|@ovgXp
zOxakhoGt%Pn|<9IsFiw}CE8PP8IbRd7h&#YgE!_vG;WSJ{%;LN<DkGFcL|{+4@090
zLz94`9#NBkBTpqzgr^QeQ-GsR)l-ZrOV?1WbFXu#YXf!FyZ>9CFT3g++CXjf4#sQc
z^}7yAUAtfTkNK}Xe|t`QPJN+ZGxy(JcGtdsimaVNd~WZjuo~2&cC4-jwu^Xfl+4yO
za0Rp%wH+Lf>0X$c>=?c3e!g@F0{Kx_r8P?%oH{>qoB>HGe_esI90q$-8B`9bRF2Io
z(gpT%n(^g8w+QP5_j<>C|GpA0f=F~BW%uGgIQDL?Oq6yz;5lsT(-$5Z-jdSwr#*LE
z9^#ig$YunyF@}aV8720j+t}#4A3K$q;$}$yCaP&3leyAzD+~S*gm3`rY(kpun-~Sn
zpQg0!G-{m!LYoO`^7bLLq<I8~+nMj689vZ!RyI}0T1i!2m7j{Mbvhu!q00}--8$)v
zQ^#D0f4!jrFxL}2{<>tsmLg)!(YNKIC4@LlAa{o!IB@zZC<w?_;!;LM92%MxgxAYB
zbcTbEKtv3%AMqnf^d(LDkZ2cMlD4W}M+7Xj?WWEx+F%ivAs;a1g?K^<`)v)=HR+z7
zb)$Rg+zin@ov?VhIwzCIA3c-wuU?acO{O`|qN<jRT|VG>6h2U6T+TMd-aV6RU}Z`v
z=M7^%UzMQ?kf|uFRlVcIm*_6RT4HyR6M@NEBg_$}p4xvCh>7YEp*TXWkS@fHX|RJ6
zG#8ESa{SKkp_w3gNi_VSz9~hjn1e1v&vai`%KQrLI~viuE~{HcsdXqCAF%GekBQcG
z=K>eHWzTb}vVgU12~6nMb+*gjCleTJif=6nn%s#k{?KO-rEFK<QrGa6G8|4iOSU{M
zocHvLt7~*(TGJsF^S@@Q<{joiFl1U_huI)?NY%b3?|i6EL(h-!ml;96>FfrUXf`pH
zn{P^Tf5G9Nx~EtOM?5a5q!ev*pz2%~@`06UvyhtMJ!#CLC1`wyQx+c5qBh+8a$jt8
z>fPZUy0mK)%+qW<A%G}z@s%{pZz}|?#eVN255TcSTi`_%OP`grvl9%5cOm;J>y~57
zOOH&^6)JRs;5nzil<pKh>kD@Cs9|^Ljb$&$baT?{=YuQolv1gSZ0LecNLwg~vPKR_
z<CRfR;#AOoqtWkJBwx}u#;v+zqgfDvsj-A|^$Suxi{lK-o)Iic_-9D~J`XHnkf5ok
z+P3hKXQv953R90pNSSEPeAh9`_(Jj3JO@9@Y4vWqV4GQv%IuKrlQ5=Rix<l&IIN87
zHpJJG=nZR+O0E%%i(&sDu^X=Kl)`L`mz&!nxaEr1AoVa$2IQ*tOdwMiiDOMT`s0-r
zV|Ioa1g1P<DNU(&A-RJMYl%b=S0=xbtsU{;s+x!K+=t|bx#Q*)Z{ek(SR8t5plGZ*
zP2R+56-3l;`^P+%%Qd}}5YFNjp~4<Iy>dOwk%M4gNita^8xSAkeBL*IPuMy$Y{W79
ziC&bi^WATzDJ_!`S3nXhe~_q?eHLq7R=;rPA0N54>zoinnVyneV>kDGhiLmfN`-Vo
zq*4M#`N}lAK$kk3IRc+sD1L?A<=A5(?$GBN;hLJjkh5X%Q@XPj#rYj=lqkTN_sLPv
zj#WjyyOB+MC3aY-J;wExYU;y0{c-cN-Keh1#83@OBb(8l2jw6=iU{i?)au(-aIesf
zcy&z<R$EuPTQSQIO;AcMn38;1$z3)RGc;zDH|zh$+C9Zs_J-kt?|*fn%eJe!Y}>YN
z+jf_+%eHOX#xC1k++~}mGc%cFCMW0O<Xo(kb^9jY#rl%>`8_aM>nnx2<N^0#2Y9zG
z*<%ewmK)<rf_eM4Xu+&hXfhy-jnUox!>kd1ywDQ$E@=3e4Z+-Xfj=;##Zyd>dGvVJ
z07O8R>781dhY8e#$sBr|k`8IyJFEKwbU{-nyt!NIQ~pYW%-xiP6MEL3Wh&piQ-X%h
z2AMWBTub2OpkzrS;>_Y;X`WE%#q4s9;O*EUn_J~PU;TQQ6%Bj4g`>)$XqJbF2B<5F
z`0UuMS8mlw=kT_1lP|njpL>}2dXUlL5g$!wrXT&9bO;|!6_m1$ucil7wFqU<#&1<n
ztTO2_39F<I^JK%{!<|!Os(D2$Vrdhhij$?%W0k8lSz?8Bk-eq{jEot%uWFBQX5Vo+
zXQ{6HrYup7^QDKLFe)T&^N}~>l8l4}m)Aq_Krtd1!7s@Y7dNefXPzC(;LQdZTEdP}
z*0rnXv?m?v3>seT0k0FfN51YOo|7Cf#Sr%}M8t1Fhco>;E`k@KG|!yb1WhVeNU!0@
zOo#0<V+l!JKj%FQ>W>jcDB_A1CYSsKxfJx4Bvw|G<e!pwBTmd+-`WJPZ`g61p9Xc!
z>34$v7;j2BiA~sTg#A~UDO#}54y$BQoj+<&#UsvV`Byg%gy52yBXTR39L}ILL^k0E
zJwj2Q=PlY!ZMzw$-bBeoKBHt7+ZKLq3sA<69cwErVCl0q`T9|vjcd-P*#bM5sKXEI
zLPT-mZeR3-aw#NQ$E+x;lYq(%m~=U8yGUyzl`zxP-}AvSKjC#5lrvx4oD19L?x_>r
zjf-L#Ce9<#t1agSf<TlhJc{Rw&as}{3_?7SkZol%9FyaZW6AX@_+m9GWf!7l*VV}!
zGGmM+F$rB7TS%y;1jS9|*J36<h^8`;q}J;}`zV+Hg)k7E;RZGNu^0yf^S!g&Vf7AO
zThHO>nDY))@a`}oN-q`ZO(G(fNMLXo=F$0|s9Rw+y)hNIy7Ad4#v1@<I7RXfJq&nv
z;B;azEcT|D9olEHduVqpN<;j^zNit9MIy$3mV{JSH$^LI%Ej2~R=E}CA-5rzWtnrf
zCHd)EJXp?pYRJ?&w^E5w^oo-^@2j1H>)Q2s1i2cNI48%-n+#J!cId+YZTqvvxd?~k
zq%JNgsVADu+z(HNvu#qMEpwbc|GJkl&;1#lB%+*B-Ypn2;{DC!n91geRlzxYOKA8P
zBg)><e3KgX{$b40uix*#FC*4vEd*xuR%ZNjjwT=_eMbHo6E{hv4gR()Mb>H5TIfg7
zAtevkBV?g-Zg0c9b}C2M42!I8FQh_><4P{3*hnoOXROu?EhYnTnS6fd8zf0^v(^T{
z@Pp)0`4N6%w-c77Ij>2v6}$y<1Sw0X1lT#AubnMoNi%*b?&JuzLclBLq>yL1174-j
z7Po>J*}h`#eyHU;-5Ej;n-xB+q>DKg(mJ#xk)lyz*XWDOwpgE0Y2$zbNUS<^*uZii
z5H6Z(&A`5=M88Od;>wW`V_=`xGWJtPYA1~&|Et;MpSfIYlC-uKnre0Oq|#J?EA0sL
z=7$wTwdS3B0b&%A--fHr7fLJYk&MZbs%Sn7?RugvWcS%II*!23IDdn9piYvVC;N0{
zQ%G!*J4d@+bWPMPYe-8g?N{Z2RN|9`d?%kjfmsEQ6XyTV?-2pcSb(dDESz|0y51Yt
zdXqiUyzE!2`+%baSI9nYVLS1vBYw@>f5|*J9FyV2a^(!;^UxCZ9I`r;?MV-T{OvQ0
z>1HFm2oAPTn0{RLgke?@m_8xUUL36erDRze{cWqt_CG&aDX$d&;<0=X-vqvPiyO%d
z8#$xH6DcsgAMQ6=ZT#+#k+y-O=XXPT3_UX0hwLaT&RnxwVvkk2H}XVK@|uV404SkD
zW9W+MouUEBf`8$V|AlSI4y_{hpx9^ADcBXape%-C=G~O*&`lIaS>OGsqk|X=h&7R|
zT2x&c)8&Sfm}>*1h2Lhe1lQLfS{UUOq5>PA61T$ib&KywtQ&Mi5?agsVOHZA2u*}~
za|RrC6mf8bzl;)A<4}z;$TnF;8r>z)uJ@yPqqCCRGOX!`&UQV+TS-$rexU0Z;tp;b
zx)v}NKs!H2XgUr3E>lWhEjGsC;M^q_MZ9;hJ7G4l67L+Wd<C}#c6mYS7@Yu_;HJoD
zjmk|T+D@(<?X!i=UMRuHS|BzJU$BY_VOtBc4ZEZQut#E<_EHpnfsiiKqRw0Aq%gE4
zG{rAP3{hJP1lOoV@K@s37h+->0h?)51fTZwg-igXuPy!+?5>Iq%(tI6@&$K9cOSBm
z=ort~^Jnd1g(BM|@-E)3;vP+<=e*Qh;at#L0zcfW^~^_RTtT>n1k_cO46T)Z<-Fh}
zr|i5@?h*gt)^~De_hxtdj+R;Vw9872&`Ws;C9_`_!|r#kNtmGFP>dk&n~$^<WVKTw
zB*{O82H~#gaIm@+ZFpr4p>+o`2dgc;$Z8PbiBNJQte3%hJ`7Z0A?xiY9&!p&m-z}t
zFZin(P?VH<L6ybyL;+6hLOQsjTG`QVe2YrQ^(Xu2b!+tF9O5-(YPqM3AJ|(?Y?0U7
z(#VTxhI@bn`Wc=Au~9bBYyKFpr3Xt+32K+f$SrZV!rE2t%&_k@Jo)&0Y;Zh&$bEcU
z87@D!*j|ddlrVW_3O2%-m}YpZ(%YDg4pZ|?JTD8V?ZRh?K>B~0HkJ&J534sL_y5z!
zS@}U0<FqQz-&AA`N7f3xAz!J{ngKIOZMy+I%usNZN=ci6Hlw3^e(|_$k(^`<-ctKn
z%|L>H_zYIga4F`(Io6tT%JTqjhhkUOA(@t8vmtx5qpMS6iyWq{;BRvEqx;n#&`G?&
z{Aat6Pt)`|<;dvJmegeeebmG?nZ+nr$K%)PgcsJ^ewAX+GPebz14~=FdkOtm#?-&Q
zvB_g@19}a+jX^l=UZH0R#fFik(E2ZS>NXen(?JYt*PM9w#;t0c!arIZQS$S;8~gOQ
zMLfnoO1J38nLG#&52E*U;vc;Z6ahY(QgKX_aB1`E^E6n4^ks5+77^$#k8DYUCNxV5
zNn5bztY8-NOamO5`*;~8@Ht**==Ae;SxC@q;lpSYb~<JwB+T%N6a3j0Dad!Sg)&&O
zPo7r5>|HV65`JfRlESM)va~7d%Kao`)ir!FicpS~;NY$IMpiC~8Ij>0`?M{u7+St+
zE>fOyG8DYx^uVrlmz!+C0|H9jB)?W?8DyD+^t^ZN9x@M_*<y^OOz4dPrrY&x>u8w(
zfgBX|YyQ%NpMOO!QF4yu?=eoCp~kNw)Wob|pA~tJyMo}|V<8qj3M3F{COPlbIEL-P
zfluTzIrdR3^vCYj*Ng?(7(>|x%c#ILzAb(>x8!VeLcj)*`3Q#SxkRz`Dx+^Ib;mdp
ztk|w0cZBj$rY0fVw&`F)^A(8m%vP;~TTjLk>Q-=nroY)$`A}bkRq^T$ov^@(+0J&F
z2?G9>v^;}ENAJ{p9P%0ArRNc!f7=eBK%$*NWClA_%wWh06LtmY49^SpP0+slapSZx
zrmzGb-p;Wl>UGepql)nNkE*H<!2VB(gGAHx)nx&gN|C-#&VQ>TEbauR`gsNWD0x-I
zS?_^8^%eNeuBirY>evXxAe41Lhjc+v>Z<w!pvdZq-<o-t2d1(brbwX=7UHOEZfA*s
z1U|Q)lH}{>Nm41-n1@P-yg@Hu#7F?M@omOszYq`6^~6WXPm^bIJD_6c+L7H(*gu#K
zV`*70RETH%T&<R^8Bg$$NKZI_=K@5|hh)2WCc*}B<XoST(Axm@w79^QDeASUM6ikW
zD(^C@%nE!|5lE!0NkplJJr&3Ld1c;JrN|M+eyOgBd%2ey<EOYvxbGf(x~^Wh%08Ru
zE^1wwIT($kb|C%5$ss6G{Y8Pj>_G`)f6eLeOJ;adnc;fj%Ij?4cMurjkTBST*CM^t
zLxVmK@r!q^>gZyBS&=w0%vd#a$@_B+CYLCs^t?w+)6TNh5tm?n8!g<1TGB!G{sPHv
z;h&Mj!mhY8(!y4k#^x^pa(9XWH<4oS3I0M+$#F;}`;gyjfueg?GL1lRQfdv?zQ22`
zhbq~J0YuukM(l$+wmewA5Q#c-CY(8>!zHJoJzIo^;MeT?ji|ANSv1~DgQ%jOZb`gU
z>-ZfQ7}}_i*5G^sr6}kHm8z|W<PIHz9E7(@3qK;fam6-=F{m8*B8#FwNL}IIsv?r{
zQ+dPKy?!K=pEA?rgq=+q{zb#<5w2OjGz&z^ERk23`qNG>FK9DP!vGgrTJD*7_8Lpq
zWq4Ln?!vNvSsOP(yrFq`xIaO~N*s0XEm^*aTz7YVj?fQUSZ$lkQEO7AZ+_}T&egs6
zv0ke6U-Z2F{sxQeQCmmw8iV`&B>n*tf%ql%q01l!`a?fg@RM?nijE_+ml-W%m|+d?
zv^4n_OZ-$L1@Yu8vjn=FQ`warUPxVKnhYULnUS*bs#5tBAA{b(hR1)sOs7Om!@tY)
zL+!zD^H+UQOVo@mxvqRUh`Wu-C_50s3s-3D&;#4c>J5`y^9y`onVd=63|C_RRh<uU
zlXcV;09>QAH~XoIZu3h0=^cE;`fIK+-O3yTWoiFwCdfNtmHtp#t^#B(ZE$}xMLj#l
zWE&khoFuwFDU+{x?n@v`Q0lJAoVl;iwZbn;gsKS)SAcdXbUPghRoo%UZmq}-MLw-K
zdBc=4#eGqOFs|%7+9^K^ryH0ARx232a7&R0MHooXdrrtTCa5+-XUZ#>DpS#OLb$d@
zT6nypSDOj{mB3667rEIUd8KX!7pW>`;(Bn^lA<+F=i7w=%6zdp#K_QVq+g|2a|Qjs
z;0nXEDqWZCQ7ld=cXtQrw0GRBAWNUJgW(3ISwn(-_c|?=VwaVb6#J(oGYZ<vi-xS!
zVI(R#4}No?N*KDusRX~m@S79Y@kwlSdjf47HmOHXD(BKJj11E*1>*+6)GrdwKH<se
zr?8TjRj=wA<@=S*w+HW08`<l2_y+j8jm`JGf8O$|9*)n0jp+NRxT4wNp`}zSuxgY@
zCdjLIDQP2HB6<|qXA30^F?VlZs2!+xR1do)s*=^#socXh`zLiR*xa;@U9eErJ#*T0
zFs;HmBz&_zLF<_!=?6<tAs#!KseBys{YGTUeBMjENxi^$i+E>ucLXde$X04-<>#10
z?r6`C*{;GrD={Cfo40#VH<l-v9>H889nbhCT;IN>=aFodRBjEGH&FO5U`}JB`O!|-
zInyUAB*|o`Mp^9H0Y-%@ONvEn3G*wROKf3Cjw!8ig;FfiedxMsG3hti+qb+CQ3HL&
z%DE*X7xo;St>X$KeIl(pM19a{aa$)rVjMZhTQaUrV7VzR$8wfTY0*upptlU%UKp3q
zu4+vJ3<%8fiL?DL25+|ia$+M|=k)a~*Bp~o+%tC-Cfw}M-{Unv+mkG_>%`aSw<hh;
zER2*U^--x*wt+6Z39~UPcX(Ys9I(<77rbPdgN`}7=kSj%=1XhNXU?ndA!39oRq3;b
z2WWP?o|;=BG0S?uMtBkDj`hyqWPML<p1MfhMb@L#uIWW!Sx04i#De9PH4AbLL$rNh
z!LZpfQ=V=<r23&-*dL0vW=E9i&urp?v-F+g6rP`UsWrm<p)?Eh6**MXrQ0_R&VtN{
zcT7j`$Ws<z;q@Hjq?9+82Rs*};tHofmamaOb2#TmbLi~W0Q}7kAAHiDCUbQlk%fx$
zU$~76e7xIj**c1LL7QdbNwqEHa7Q+;N#L<xc?-U9Qcu4NPHEE|`es|9CdV|oMP_9j
zldh<rFzkyhX1Q6MZO!gfrCFXM0g;E+=8q+<mGE$*nTm6iY3q*2<rTE-AvA}@jmg-8
z+zw@$hZ3hTeOruP;t3u)oS$--SQWK2$~p{Qkeu*wO4In-2FZ&*my=8ggr~8O4RpH0
z*$1p({&6#fZc*9Wz}%WvCF-py9A^$1rQ^f<n-eWDSyB(&YT(bb^q+MyHBP<tepzVd
zR2%x(H%ARytxy?B($wNaS-B(upkE~JIVL|KVnx5L>QlHb`7OkfV#f?Q&@+w}l;>9<
zSTwYcIAt5~hDNQjOZUCcF<yTImWNz*A1EX5_EK$tr7>*sT{T3kKtt=zIS9L*AE4aU
znf~=p@ToxfD0$Z9=5`+Ht*=5B%g3yqMMf$k52=E}(S3-aU+{^wanf`UmDhhx6h02;
z7Af$D?7XxcjWhI$A|`Y2Yc~HvH>bVOm|5Jwyn2w7tyKyDV{(eN{V-PcE3?WS(bZ4K
z_M9Lf6@mDnQz$)D_DV0@m9u-CRAu>Rv4gB)V{=}rc%x($!x;Psp9HG5?b0o%lz-U)
zJjKMUshfbdeC{*$l&SZEsuuf#LV`>)t|=s}LcVUC0lziU*ah?}7WR392C0x~+;FP4
zej<;7#yDpmCAIcYw@^0*m)K{@%EuMWMSeWAnAUdRDAC+39{eWed@43z(9tSmus2Q2
zWX*AZb-MgV(7?W`@NxZSm)J|#JdJDFA>fdiYr3<PMmS5w=fAfcMmb^5M<a|Zj3)0?
z(;>4M5O<F7DXqc94-0oj^FG0|^zoehvqrz}Xp4%5wX>X1EAZS1G7k#7>MU`i6VBzR
z$74rC)P7bOw_yEYYJ4BgpYrq3zEBjWoRu;F1x5uakP@gjvU5{+cT>%S!Ma)nZk>sm
zVfB53o>58tW!lT#udg`|5JWdeHo^g1uwo){8!|ate$j5H)E9~!=rW3iYdnC|9wq+x
zj|C8+XZ#Dd)+r0AahV`cZo-PevZ61xa=Ng>6;^)81R$R{qQgUit{14Xyp7!2cw%qN
z$UNk9<-GK+=o@S9)xK3rYI;CWGOVK65}Nm$TarhE5kTLK-@#{$2f_aS<2h-Wn`}%m
zVNr}r?29?C*EVSLbWKbuMmP9?5r+IVGf8n~zqs6WQz7#0_-!!r1}O$Md=Y~du>2Um
zdd3yu8QY=MJN~L=6sMD^_n*h_>>)5%w~d=$y47Z(kv7kC^9$y=f@?Z6(ho!rUz0AZ
zL%}Tw18$<a@fPz}SfH?H>gL*mL)EA;WSm0ZO)gtTC~qTc`}tPHhDp~o1!7v;mICl;
zJkcU$mICZCgbruK>g}ZlDdzKLLVzGWEOXc$m3b+3RnuR*tx&fPd7Df6<dsL_I^m2E
zFYQMA^2$@vO(Af;EXN4lz-d_;=XdfA>U}ZNE%Fn{Q2J&uce`jcc;(=ofj0vkeCXt4
zSr?AE>6lq5M&0R|EFhQ{+2$$pYLWj*4#B$aaV2dt4w_l2=!6z$=LkRJbSU2ZHYf2+
z)S%9qr|we7jLXzc()fS=&0YsvCiShAu@f7E?s{Ey=9(v)u>;oD8$p8~`>|VF>RF%p
zl-GNMGZ5_hzHmMrsQjZLZupZz+%f8H0n+T*Mr734eutU{ZJ4z^b0Z`=d)*>2D^Z3u
zbbW}WZv@RWT~sHos_6FU5Zjcl9X22SewYu_8h-^UxniVOKHo@5PFlmab(A<IeU7Sh
z<9$PCCb#IrzL+#;1Xq>28Z<ExIuMG!XqQ8QF<wVJN`zcAR>3oOlK*7D*4&auc={DA
ziuSNaDhg!VtpV=ey?V~N$6?G!QB>0NaK6G^{nVQLW2`DiEtkg$C+5Upsi+Y=t;Bup
zi+kFVE2ZbzHnqn&jv-N#-pE)*jlOLIo#+zClU(FENM1>Wh(1SANv=W8k7e~rRueR8
zs~#eRol;^oP@ycXc?hKXhpk<<Y6qrD$u&3B5NaNZ1}ZMOzC4=JvBZ{y!;~Cu8lvbL
zE&Qiw#$!ex^#|~CXurrW7_^!$U=5oMVq(d7>aS-e&pW@~QcoKxpXy3y4jEl_f-WO#
z_^cYf;_X2H34C54gU3A4)~2W_A^SX8#a|OA$dXuU^&f7>L7INk=u2^lwN$La(nZUL
z%NZ!Y6|o2o?MZOrx5Y*pDy3F9LR*Y_+UAf5N8BU#hW#%7TeuQ0_Gf8TS&Uz+Tc2&Y
zW%jU-_Gx<LcG8{95)RBZ1|g-D#|I<-S@3raYgLhA;EsZ0@-PYCr@p}(g?n<L>CSPN
z#+)-27+u{HglPL+={$w8<^)P=`~G;i&*>ZdUVQ&9bCB{t3qOLDvI>kj!xa3!{@%_S
z>fze@e_tPY+)xvi7fd`&YHn(+);ndHq^0z1+ZhCKnX;8k5`Mhf6|L#ALdz9*I=`nX
zxMMQx%b55i;44RmMKaxRV%E>X$<^Gri>g$U&UDRGr)QLx)A3m=M<dQy#hCTPEn21E
z%`>AvWZBHH^eW%dGfxje%5xJMiZZXQ^kq${b(N)6E%YYbgw~U~583xe`6@BL^^a||
zWW`lVD_e11(Jg*kvRZIYE6-gV?qQ>ba|T<7byV4+Bl7Sub>~+?2A@$E&A(Jj=^X(l
z-S0JHGG+|Jn96HsCY_Emb5@IWHOTWWneRSz*TpFe#!CnesCbsg8Z|~pGES{itqL;R
z0h{B5%&(SgcUTGorqSs!RWf8I`)k05;S4<KGA5atoFN5mAeYFfY;QUfOL=C{j2bTx
zgJ0b*gCW(PIWsup3=hF9L?ci^#YMGh@iYoDM_k32zw|<upIzl>{@&8d`q|8!n@|R{
zS?Sw5Q3FY8hL;7$ui;!;RV%xkk!$ZxRyaXCQR7xwE9tK8ZXUMim&b&n`>09v@4j%5
zhu<t7olzw_GfX7CxX7aR{4JbF<PRnJE5TFPgYYBHAm~GABWodBKWa<(PyiXbP*Onb
zifC|q8wxGK6Gmv+%vZJMDB(BQo_uv=rFpp)udjG;5>vkl3^yT%6RQ_dW!4zlo|QR7
z%xFhpGyU4V5%2kDoAyqOMbC2(PtVEyHG$5{XJK4vlF96+-N$xYo4;AL1@r5k2TcIC
z6%74Z(h{77JnJA5e5PvJ;)oKnyVIXWxlv$O{NHdEjhfjv5=1HZW2|M_uZwPln92dl
zIvB(Zqug2?LPp4#meTj&7Ci0TV3q{rTDYc>;j{_%6=!nOtYzRumsNc3sr0vL>4)gc
zqD;?dZ+IIo_LOdxkAW5MJtpDp+HOGi?RKx&^<&K|CBKO+@U7gr;a2pZ>xfUw2?3?9
zx@u*Z1E`KltjIzFTXZ$tDel=myVLXY=UPV6kJ$N>JrIC7rv-0Hk_443R@ED9?Qzs~
z^hX>^*3a{9hO+7OKiDF{7N<@i@Yjp{kqcTDapq-L)ZOvZZUZ%uN=;UCUE9XqB(6h(
zmC=<E%%07@HyRtvp7-VIy6&!%o`Uc59{#%9dLEwK$E1Qd{*O%#|CL!A1yN<NA0P23
znueB=krZIwzF>%v0(V`*27uvU)7Qd14&#|?WUsU!KD3U&aY7_t0Et2AI2rgv1{%v7
zM!HLt-#IO!z~J^-;$Vl!{#Rv(jY{_Q;^OgTb50b`Let6#VGVHBQuYhQ4-w&<Wczy0
zax1JZUottl_l6EFH&nO5Sw2e77fQWhuuef<>rO^Bsx|Mx3SalxkFbt^<BR6RWx4Ec
zIh-$S7$Q>l?7v2a+&a8KpGVL^7}s$b0oAh3D%V3*mP+{T;H2v8*7&4{Q{Ee$eY($H
z0BWjfNBIvg00E=-?kGNQaZVSr-aM`WV*sMaTvV}z{eH!@;p3*{LPpeKrI-t!!I=KZ
z;>Aw$jSwwY3`0atb3dJmvT4F>;<Y9=UjAxzJ@wulHZ4&8l}s*pvsC6Kl9B-<kO8nC
zkWZ#i=4v#;lT0hqW|Z+BeV9ye5t!YV)eAs*Mk5$z?@sS^Uv{sYy&U6N6Crb<7fE|0
zC>}ZHdcQTCUTsL)TE8~AC1i{`QjvUvP6d7736uiUSlx%hR0}SLR*@pOFg7}glye;t
z#Pbg`@}v?F_O*};M-a&(?KM|r_eT20!tXFg&QnBGPd=z|mO=hvdk_Sdb)l;bxKu?i
zznr5w@oV3D;(O&X1i~-BWRIxr=Y#tOo^Q*)=%n8hBngEC)CJhU{YG>=i_QEdp<dYa
zugf|a+isI?97q%<Or#8iF`1$2)o-I6Z`$@n3Z_4yVZNgwKm{2g_OD4<uMzs-!DIEZ
z{(lXSt2+cf4p{`yt)6eI#)w=Yt_%0qI}AoqTi`p*7C8M8wXPKtlXk`yr{D+`A=#SK
zu<phWdBHXt#RNuw_F8jU+6SN}g!_lYeuvaZ$+P*1ZhKzoXxcgY%0CIvm@Q!f^@uST
zIO%);AeZkBe<D&?SgSgit2;F^N!I<_A!501Xfx8$B<dIy3J6Q=uDk;z_c9idRY4si
z-pm?f81&;xLZi3TGPwWCW%HIY74Md5+9dW&z~;3^jKp4(g5FJi&Gy>ng7f<KJWlf9
z$cFBDd@*H?*%)5wnO#BU3z-kL$5jke20~LbIW`7qc%xwV?2SEfpxweYqpdiBy(!5n
zF_(?D+x!nc%8~@aNtR^7SL6i|f`H?Fex@id`RUKh0MWZQhlBO~xbZl$-66l-u+m{x
zj{?xt_y@T^pkxY84{t{3gD1C9VZczU^T7qyn@N(*rLnaLS0+FbyP&K$$mPWG_ctDV
z+YOH-O|ygHggm#@gZE#2$P<x_dpZx4KJm1)qf`Oo7l=U`LdrCX_%hvm4S%`G1BPo%
z5zh%!mHV*x)@}kRGOAOqsnl;UFROG^?TRhrS-KBYjANkMN_TkMjprj3nwNZps~2Q%
zn!vEo2hyk!jA?K0MQ>YzhQr%0z;>U7S(EiZ_UmQe7;3K4txKYRY^Z|Rna2=&grJU8
zRNaTmNI;z0dD^s%50pWHMwR}r-x_rb&2+3E_zS1cl0K-Y&5SSfD*})3&3ylgY2gBX
zR!hE-W$s9fdxa6}F^XA*|G{sMK$QIApq<1W>)XTVjcsFckbmFZAA8})<`-QcrNR*R
zyIjyi3hb^k#;NeP9Zfrha;$sB>B&Uzidcm)sba}q9stTj@ZWo2{XLZVLPww0`_L2Z
zgY0V_tnNnnVA0Jrk$m?Rv3^<*h?vsslKlrSu#D#&$Zsp4wZ)RG-=7`K6t*DZ9D&@4
zYxJAy{lL-WsM=WTJYCJ!jk@EYU~Q%9zVN09wHO&32>!*(w8k8<Dj;Xg9H%Q}eC|@N
zNiIoMOTN;-tR;Q1zyCyVEvF|(GBA;RLr|R>NE|hFj>jq845I)3!ty>%yM3u_bNq&x
zRofbE<ZArr|2NWAT6~H*cEMEod*))JCL6;Cq=Gaq!aBarEzly7z}G(Fb&~xLi{{;2
z_@Vw2$4mBI_T#&xo~B-h@p@dr-yfxk(S-S2<`V7$ef!*ITI!aCG7OSpQpRClI3<8I
z4poGqtp~dH147}Z)B&MB$PbASzaO(b6BX**Q)ssE>$X*S`XPjj?*FiS>MFB=Z2H{E
zrkP2LO3a=}!T~#j_=nE--#cVe^i*HfpSYfv7B7h1L{zOKwO;pO4UXXJPM1Fwth)AO
zX(lE$9PZtsc8{aX=r4x$zlRq=FXs!Oyc&ZiEuYJeC;kW^LK?3}s=m*s+5M&~!E#C6
zo9XG^jN~LEw!u4?X+kqCXq`iJ6wWvxyh*Zh@@zzA_Ydh>OfggSszCb#i88B=@n}F0
zHIIK(qa~*5Y}>7Bpo&i(%fz{U-o*<Tf!h}X!!Wb>+!yvl{RL*W9lU%^?BY%<V8!ab
z26THzv?3ccwY%949RoUTf1b#TSPmFnrF^%^XB(OQKt9c7nWJUw{W6t6q!r_spzT}Q
zZzeD;><oKS17w^HGd0{qJ{-*ey`(M!O<vITOx`|!vKDE2A!t-|e;81<Z#_rTrGI%M
zR(}gxBs}_VTA<(FsB_*AVa`>IZOH14%vw1)PihbsmMRVe^q1lf=x)~uGJO5v&;Mvf
z>3R5TcS}r$DGpzDlG0Wn1^WScUVS~DI!mvhH}Ckp016Mh05=7`UGD`36enL~?*7e-
ze<B7tD8E5(S#*3K89zcbcjK0Zd*8}|KaqIo`2(bqUhGb{qLLv&&&Zo%1+@G`ZpnCP
zIfQwuu&?m63)2eB-z*~8>+27YeBXJlU^|U*w;YbHw0>4J!vHDgV0WxpOkL5>=hH~c
zMw^}CMTS|gV?w5paqqN&kGK~IyopgL0%;5b6hJb?)~)BG52tK_>6C-wOsISteDMWR
zsh+3PJnHqHNH;45%cGCv8;@gwwy~#oy3Agk)d4F`8ol<5Pb~c@ok4yz)b1F%xchMN
zCB`un7DSzP$^+D{c#_mLUJ$+Cy3V0L8=X||&z{u|=t5oI?6;S#le5s>BiH9IzA~X>
zcWLdL%mRP^(=+9HUu>D^itnI{Zrd8a?Y!ImyF&^5Q$UW*87sD~!3tYRwNd~eph$v0
z$QoOe>!>f-fOx8>Rhj>n*A9aeyv3VOLeYA-MdB)ug|jLkuFBDm-xoU2i>|Ao^6eJH
zQFP`}*bofstumhOl%0H??_NTFy&O$7-)t(Ed)$_8BI|}l=<&Eu+oDloD?;BU*wH6(
zt}yb(_3Vd^Xcm)DaB1gxDNc5D8|Jm*z)ZFoYxJmfCwHLp3pn_3NJNmEq`fwa`r+}?
zGutP`V9s93zb-P$^_cex1`|Wk@HfLFJqWZrY5^EMxi^~LKqOXU7^lY``V<%oN&i#f
zzYR`3#wgDq5dS7O44I#GuW5;e$eVh`YG*$9Vj;Uof9QugvR#~!<fYKKlT`iu`<!pW
z`+x8AEg;&|Ob%z^u&j)dR;T7Nqt5&tWNN3G**t!8qkM#wJ6&(e|G4o1b!iZ^IA<g;
zPZ0jEGDSe`y{_6CpsZD08=scJoCF|Xf+O+O$e@0B({(Am!^A&hUu6xT7DZpFTzuFX
zTq0J@ScZGo6?+wJs`Xk=A01r6pVKxlFb6((tNW6OfUMj+#H=?k4RuGYS)BufJ#lgw
zOTsmM_%94^U%Qk4oo-#5=yr82gI|_R9G%o$bE$I$h+%MuC5tT8TqahyuCu*}aZGx7
zkMz7%dD0A&4pkW57<sAgv%l2?zRu&Jj68kk2Y=f|kC?lgT`nBjRkMRQI}Pn}IN~5E
zs${l>!`fNiG%L;I7JoDnpiT5vuf^?6WOi}ZFsAkbA1W4n4aFKkP~)qoEN~SQw48bQ
zvT@5pg-QI&X|F*L=8M|WS#>+JIE#5LnPSH<RV7|h)Tey+>Zvt`bXH!Rb|W7U5ExXa
z%O!X@9k6#QM+%}K)VDt4+jd{1YI;zY>x9RnnSX-&@|}d1cvQ%!?!HArMu+W8CzuvH
z7prcuoQun(tkedp47H5J0mv?H)=iZ>xrCaoI6T+fu0vJM<D5Ss!6oyvV&Yzt)^pK>
zCN;F{s#5AXU@<ObG(_=X2R*^dU8Ur{L)FD^!Dbyn?qfYAv*o|@AqOR4_OmVb<@n-#
zv9@<Q+BfFjUw|vr&ld$rDn}mBIwo%uqeDVVNBhvQF(ZW^zKwW~US5J?jJw^dbZU(I
zri!*t#d=2S>8vtl6+nw*=Ji)%a-D{izH(sV#Zs}-orNZ|w1O7;NcszGONVgPquEJf
zcjj*ZX#u)Q;!!bcWwI0thYd7YR@L;mGHXRA+&OcTQmeV8NoeZtnv0u>4K}MpBcU0v
z-&0d0GiVE5i!67(;;Y#&zUpBWzr7A!XMMeT<v?=!5HZH+VXIO_QRu~oIv@eG)R-M0
zUe^<`$=eo${0KXP6x)={gZ+W@_6G1zKNyQyXbXU2h&6x<Qv`d(FjGk;@WC})FEdM>
zgiJ`<pgYkmS7wn!)(TekjSS*+w7O)9u+hpb>~K(3467!c0-JS3?DY;nI~@-;l9oug
zfUB&6p+y^yUgKV{9F3+HRV6cxQMMwj`zu#Mx>Bkv!$GTiGP&3bn7EH<nd;&>UMn@j
zXaT?rW-^niG|J`IYHvah?)ElIEtW}Ykvay#75(=02TDmPJe?A#roucgpw$!;dzDCZ
z)eE7vS!1k$$jYdC9O?ARXB4#Vv=*kPWMq&hyk!!JH;WaRnw))@CW=cQ#H4HpshOso
z1ojizCZwFb7J}HU0O*Mi+DVJa!-I09K$_5i*g{lqt*lxmirJW%lHVhpjO9k#=2=y0
znx9$!q;gT(^4~hihiuHu$|3ShFNsN3xguGLCQ;eVL~@nE+1ZKw6eCpFt}e%%i%q8R
z7&A<fj!`OeyfHYxfI1DLi7gexY_=*N6`8he$QxqZ*i_F}%+ru{qsFSH#3l2(%f?Jc
z<kL+vhWcX4XDDdqnQZ)<cms)qnP_58SSk<N!GLo)XK9LahPxWo+$_+Fbs+}uG4ptu
z1!!QwDv&{bZYj?htQ?HvC0|*rpr@)xWQC?j%SB4(;uI!SvLvy%Sc4%aol5-d6uF#C
zlav0I0@cV`!Ajv4^ZBi=K}WmG7);a~Y_{InQb55G-Nfx5EL$hp>co3dc{-AqZhtDH
zp@pAL*BeH*K$tMvQ~H67F>@XE+KVXmp`-bnYKdQ{JYuP)-B|Q|l(u!fWH)Mo$Iz$0
zq}*<gvZ=)BL9<dBcLc!`96EjMrK0)j!t3?tIC8Kpov_&GB(08(FtF487FQF6b=L2w
zQ?nZ?&m{o1tX}JHxF=OGPI&wGFM6FjisEPEtA$XrduKgtg-Qi;`q)z5H~KVB&Zd~M
z_^lzFlJz$+-)Qu!>wMl>r6<kTg=l@+6nI3Z&E%^b2m6GYD%z{}?kS0Od|$*NUB=zx
ze%9W|U82{CSx_BQz)ZrMW2=$>@8>*2{oXI4F$<LCK+^GMU+CUTS_5}MKSNagBCAH;
zX5PJw#y{w%(7Hsga=)h)zA+}Fn~e32RQmp`yvTaq`i%Q0UB`D|CpX0ED<6!s3_UzJ
zoV1NcRod7rym{`aWZ6zDg>QR6V#w`eqN&_<UY-m=BilRmgp&gQY`?Ir&Peze>2%#x
zJ;<=)?`Z{~$L5jwo8P&i!~k+zZ16uVW%$K6Z(Ws)?m%hZD3~|K@8AeUSyP{Y&4#?X
zju*`Q)#iJQ)Dn7JiT;;;5r8!5ZqO(b#gyeopr_tFzUVi2nzk3(X+ny@$n5x=jkYgd
z?oq<HZ=1<41JD%qzr^o0NBk=R**PIn`25Pa-Gdo#BlH9|6ytYE^*7qaiBlrx7o3SA
z{ofLrIRW$>n!frQtY!$QnMcs=NoQOGuY=i#z+6I?%8Lg53rflh)c6flwN@#^7c>};
zQqLK5ZvX0O0j99a3?l*z?envv4JC86%W4VK7dC?%=`?Gax}z%wEhGIK)NrFRKPf6G
z)gMo*@Pdxo?Gs}Jd&>NVw0iw_^|?GL&G~J1-T*(eTi`cUGH-%ygFQt>ZTa{!zT-XB
z=9S@2q7+)Xxl(S^jP6qWo_q1FpQ%zxYOdW3$Mk%MNq7M}|2IDCA>Uc{!3TuhFQ2B>
zW9hpszqOw_`bo{$84ir)wLfu4rx(+X{qO??2D_NOP7{sxoxkyURnOlddLzSQ7*<(*
zp{m4hJC>h&0>fc@hX9@CCEP++q0@TJd$5$`+jvsQxM_-j-FoQz8u(d(M%HWajH5cz
zXZA#wzA$IMo%k0z=8Z;wn9sV~0#z&tRub)8s!!&ghlKQRib;)VkdaKEoV7~K2S${~
z;D?wt#>t?J3<YCgKFG(@v%#p0HK6B75BO@69MH<!@zSN4inF?dQpel4a_(>G-`MdG
z5Fs}7g8hZSg&E3nz}CjmbphDtbNX+#*fiw1kl7{J0Y(SHL-#un80s*0usq2yvGm}$
z^7C$#-r06QojZCTiY`tSsG!H}AU4_U*42J6y+rH@;iWX)1TS-D%QuJJ2LPKOS6<fX
zuCHT|mGZVi)z9lXrUkA*g)#ovlhJ=g)O<vsWsuO1#KV|UMoRytrr!AvLM~IF$Y0L#
z^Q<L{No)iF^Gq_={dbqx;NTZmKq%Tbo3xj*woZ$;P}ui%0k5o!fxOQV9e5Q6|HB2L
z6V5z3KEbDYroksWKT(oI7yUblzRo?YZNZ`V<^Ds*+r^4RE|7`O^V@nu;*eIq^Ng0@
zi`hfNW3F;_t^3)kSCG6%rYSUO`5qXKM)}FKpiCGW01aJr8dh9)Fp)RMrp;J}nSqG7
z7y51um2iG7KFL^SZsa_bX2_sbZ%f$7JYC11u3c@#$}>ER{0b4Iuw?oL8TTp0EBL&8
z9PJ$8f@u(PRTV*Jz#rl<l7xM_9d8i#`Qw~5uxFh11A-&PE0Dq1KT!OQ>I;YVwl64@
zMs=2W=_E<rRsD^@v1}D~TIC*7VejiFB@18I-xiK?-$#Z9&xFw&_Q2U`!BrVR(d-wl
z+~8fhYG)V$!<lepd71-YOk?Gl^x45PcdDct@djf#g|FuYpBH{Rs?IUND!px~=(9-k
zi$B}uiDcqzbNHFr66BCwQiDg2AbF)T9zf_8FnihU!BFE#xgh3uvK1GyT67wyvA@_n
zw-N_G-!o_W9b0_U7T*DX6wosH2?WQmS^)JKzhf5DTD`%mZa+tSrA*XA)%E=519{%C
z^5R44*?$uGOI{1pYyVxw>xVL4FPyF(15E&M1461VA0XA?=5_9#DGBs#uEpNjKQItw
z2is1BmTyF=-<j_a`VU|uk!z{k-xbQYYwoA>zTXBn!jr_>+CvRUUgI>W{RsEoS6xqf
zw5T3V6o?yhsderU{NYx+ewlqFVyIG)f8*&KyWsj&L5bebc(PgjqGBg|m+*8Oz<LGs
zhA&CD#?8fvO>2M4i&{8k9L_~Wzw=x(*rw!q{+)e5|J@fXlK#!JW?&g-R!wQp`7-d_
zfhwcpVIb`~An}sh%-QqA95lO~_I_t<;{0kOMMz}6Oyco~8SfSEyc6E|_S2W2+u@XC
z^<e5f!Rp$Vy3_p&zF9rsO2#uwgY>?t`2|W?+8g!BHke8OMs~oZ3Y3-EJl*?Wav$B@
z?qO~BjvM@5b{|NaV!Op>GCA#dseSB*<_(fSYLVW#B|J=t8~*c0%pPs@0DbZL@hkw7
zKEUx&t{dYWY2zI?ng)hBOo_QqH%lw28iR|cZDKF^5jU$TPDGF6_Z4mluIAHvz<|a{
zOUVqDU*491Xgi_{h^ER_!jnuuc+xyMPO#}A+}LQm;U+CTKkm0$Rx0i8H^p#(D9)IR
zy2}U)3uoNL14S?RHX*X_B_xvjM_ALflUOhi(ddKSUs!6?<mD%UK!h5yM`dX^y|B%T
zE@J%ZzY8;pz7Om@c>(6iBxY}5Q8ki(3??-n`6@D6jodQd2eHRiK)7Bt^>9$${s7#y
zNi4^$KzwPODS>EIB*f)s4UKPwRwpmp)!(~?@uo4egeQdPxLG-wSN@<XJANfLA;h<z
zBIQ{oXXY8I{E+wFS?QZvmOe7WtXAaS6J{vy^SYncbLax(%SUzmAt|$EA%res8o(Uf
zg|gf+fWd%vH`I_ttniHPjpv42v1r=qWz~1){r28BTZ@@#x}H^8#pCPEhSyoW=Ul5N
z<fhg9zQl1_uX#oql|F~eEDkt)V$&5L@YQ^<>->E1c|6I5jiJ`Nzx84n*j56Bj-LPF
znhDC;6?Ydz^)osE<(**6GO%Rjeek6|0OFtZ4qvtO%<_iQoW9jDeB;<U@Ivc&{KT4?
zY0+$aekzca`6l&7gj#(Ed5tUs+?|OPukMTe5v^oOZ&pq7w+7XHqCQi~-`sc|1Mb>x
zwtj#K1)qM0d@Fc4AbfwP5Ai$co?&Jc508>+h>D0UvspoAVQ)qC0sW<t-yc+(f974`
z8z)T9OtKg=$P+EeM8b>MD;W(KWgk34TVe8&g)`2S6u{onRnO|6@cdECPy2emB80vk
zwdZ$0@o0NIK|cHSvGPJBt1%}*7lto;p8Xe->-8PET+Vi7K8`9oA3r<(?oE;d^60PC
z#i4!a71mqm5KkECQr!1OK+y@f^T)0IG)0!ZE=G6=yDPSimZ$o2<bR!ANuxiwAuWU5
zgU;L+iIF*2JK_c_ezy){q32zM%!jrDOv`hF*l@5E){s}Oim7lCn|0>q*ukIU%eSIV
zwtN1S@E`zu!e$i5*qh0!>#T5!SbbIzUP-v0jR>JkDd?qR_#9m)a;7OlcZ~#y#S0Lc
z@J`8IVKAS&c$AXWSsO5ReqY%OIi=`e9D(CVBaqwrIYl9Ii++{*E4xcsr*;24Xzn#c
z<@l+RcwmB)Ca_JkPuWSS>n)*czK7J?^*Dq)PeUlaqtuzbc3)5pcuVmOe+C=D19_>I
zl@B$(FY@8Wx~OQhV&6%vIoE6*wy0(Az2^@5D{)UQQex!;jH_5<Yl}5G*sOnUt2PiO
zFdep1v?{OBqv-SVNM5DP^!sFGUSCB!#uv=XImemzjh~m&xytO2UH#KVVn0`BAbn1%
zEpf&Otlch<)bM?Xo_{4#BLb*wF^{F9qM<JdRP73b+q<ZQjEoh)6kPK-3v1E~RH9e>
zl9RG-5U6+MQqt<h4r3R8;1_oBfVu8LK1ai=L=bj&VfOo1Ziv%KA4yU5o&C_V*yES$
z&>7Lw_-=Z@lnKb$(eM!L6<iL-9vV_TpK8hJX9LEAvX|JSNE3Ah<VVnT$aZPPPq@9B
zd00DR?)>jM%3ecp@uX7sK9zf0o5La5vMl3&S{n4>p0y6|AQmULj*5)5d8Wq#3jh9T
z&Y&mymoA(S+n>{We$(QXSX>D3S|LOOog#48PMJwK#vgXG)TiRBKq_vBI(aE(6D3u}
zu^Yu3kCDZ>L@2czVJfiq((MuLRt>j|cqN!`OWBXwFEpQ|xX&TsC>%tq&Xt_FJB8&G
zSe|=r{Pr`x-boBdz-dz~>B5Az@Tj5j&+JSFTclf(oui+hJ@rE?tB<UWJYVxn;Ab;(
zSmANzuzBEV&>FWu+6;7lRZ)0^#S$N&(zKS{q}2E2F)H@Qg-lx$ehl68>fJihFQdmf
zmW_N~u-_nV!;BqhOA{YK*Dc<TUi_&%xa5oZIkMo}<|@{tu?AVv#ecMT=gwuM*o&Go
zc4WU%Bh~H+*-C80&R<+nXTx!c)-$*u2K{_v;xRomOZMS2eV1%5eHOR{`z&v}H*%|&
zOdUH9P8BT7B?WzceYu5p(I3_m9SgsW$1kFHLt;}l^blF`EHv{p?aIAJnaOy{FLj{m
z2vw@L&z)p6VSftgc8iI;{v7%EWaI^x#Y%f&26D%c<yy6rO~;urtlLr`i@DXB4r$Ko
zMo<}0Uu%x!;~1q}ApIA4-H99gm?Wk$3iA8WG??nxmuqUcEEXsD$ew&`XTD=giwiO*
z(b%b7A2eFo7Hk-4PMc7Rbpz;qyyCf&ZG(ZjIDcsfgxj<3QCzZZuS73ORpbnHWj#qZ
z0cC?JWjIAV0OJT76mm?t>@s3SH*Bl&#OfJ0fdpy5O(~DCv!{8wX<D|F=mSVU<HM~O
zQj)#BXr)(`tYgVR9cigGg*5p|UW5&WE1|>m!L5~#wClV06Hbb&4LZX8NIT46xVA3#
zsg5zO{-?N8>X-#7C+Q7Qj^Y!4X-yIqHZ`pjzD#T{j!r8L=Z;#$$Za@sL%`AM_AzuW
zvhQA3>nQ%$M3=xFbj*GD@pQ6=$6cQfI^lj8m2XO4O8XsGvMr|HT^awdho!2106>Cp
zN3%0&3?&k<Et}F|X8h;c8&AYo`M~t9vOB7)big9C$59~ajm$@e(Gu%W)k>!QTi@7t
zCoJUo+6xBUy{O-R^=Z{7EK)rH-NICX<DiMx=(LM4Wk7#}nE&>^M-&%O%K1+Mgx46C
z3PCyU=6i@_s{@fENPRX#u;`w;M_fj1r|v^#yrm1c&*`~r6Z;z1W;xE#aZ0Kg&FmDn
z&JYaqg^r<$cZ{(f_miL2zF%^Ilaz5yZ0c?S91`7)vmPS%?8M?muPi@z<!on(_KbWa
z9BH%PHR-btl$oPyTwzC6hsV%hu1ZysZwj{Qo)+K}>htXC0QOh~1j%+n`-q$<rHz=w
z$Y{}vVsBVbVxB)5jCz4s?QJ{KW-HhQd{d?!PP%vpB=Pm(_(xk@5vfVdz4CUeZ*n?@
zH*Kpk|JcgYklm_ltf?%Oa$IC-(;pleGTR`^>vqb{-*~ACa5@(Cq!s=YUum#5CVB1W
z{hj!EFLf(BEe7x@7r!s*l<3mdC{7r5Dr?Hd_0+pn7JtqXu}^Ib4kJj8p@>(6pEj#H
zr)umo9G3@2m*o_Wy8k-=#PEClFR>0V2mnU31Zu7P3d1;uUP73fH23sY<P=Q{8S85J
zh`O+B4{nOG$iqr_j-W?t^)(jdrhoNR;|B7aP~xZIv=w?PyHi*n@JBPx>!LpvpyFj3
zMH%e3^!*&d;!kDYa{IM?7b#?^*SA#EA(>QogCGo@HG8SLC710oP20kW7kj_;56HkP
z#PAAAu<aEicc`*D;AFQM%zJT}Q9dFjSK3C-E_1>w88=%>LDM$AH9G5#{@%8|N71$)
zII|=yqv~DLSC#wpNSaQZ>>|h<b5h%9l7W5(jsi?^DD-OFIQN!z0G0lia{5bJTWtN+
zJR&5s6(JiA<SWwIi55wBh;wE3sf0;gZ}pAAsLtTI{ujmm4Y!>515)+@4vSj!M@_0G
zJ+d0@#ch}9305%$8hFA)3H)P68^}yUAAUmAd^2v1vI4}dhSKs_`~+z*woh`-elzZ!
z2!4A)a>fR+1epah@QbdB?G~u!YqTsLca##E#nCXA@Xpkj^K`Z#&lwX-du(E|fN1VG
z<w3{WK5Uv+{8GDS0H{xco0zN{`J6N`vBns6HSwcHc@W;IiPGgV#4c5Ie=eklc&e@O
z$!ZHj!%9+DEv)?svny;Eydx|nsa@oeZ>>H{`q{XOl5<#cCr;*N$?k@myG+U>7ll7~
zL`Uq?oGW=P)J7UDXVB@m&^}DCEEfAvv=EJA3yd#-36^(Ymv_XaD`rN>78M$`-)4aV
z@{foyE?Ch?+Jo9^0^>u|_sHyZ>`OYVuawUPjGy&k8{roLHNKppKZD-DSaS@SO4D)?
zu54mfxZ`T_HQ2*+*bA3Zy$Xh3_f0`Rdqc*?n3zR5O+&?SX-JfYE`)$S>ql4L+r#DH
zV&3|lz<>W=^JCdOs{9}M@k8$4^MBI(a5VDzUzs2O7u0n6{|Gfm@dmny8EN`>`pH^}
z+Gz2b6aN!h2UFIqd*J`(bp-$YdE@^7cpWv_W=SN-A3u0y{}0dM;PStk<5Gvm9#sj=
zjo0btLj=B}xbPAYv69l9lCmgrR7g;A2s4ULzR7<Ve{wV7|5)aW8l!?6jot6?^;u{u
z?7gtCT>H;(J?~^p6xG#T@Ay1Dy&V4Al$*cjz)UN%sH4}<&ePA$HVDwxbui%H_P=>}
znaCMJ)adK;>(<`hxcI!Y)TY%4kK3igSanF<Y-_q+xt^bzj=irt<5J&dsOo%wIx-|~
zbMEw4dthnBzhV?vY!s-t-MS3Zt71Q35u2ykD|=f1af+EpUVkNM+`6KGU~pMs^J73C
zbqY4}(%{0w{_aoQfFq{6At+N%QCXPV(&o&76n5$J(g8{6<{|x_H%n}c86>+!_mn_U
z<jWH19%9%FTewsg(uM3${)5pfKIkeWK9X7~B*Anl_zzFh$g%o>Nex%R6E?$-Loz#^
zN4kU<63!qvVl!PZbF~i5@`tF<eFN^~+}Ca-g#T<I{US|G<)d2+vJkbK8;hBZ9v|X}
z_zLX}rE0)QymXC_)qKtXb1-zWW=kAM=8=JlTwTQ2H_TI|KB=v&^+92$l`M1PD@z?J
z=$tPbgb!9I_6KKYOMF;v(K?D+L@a-HCYhuon2k$Rn6eEzzkU?T5icHlu@4Q~R6q4@
zXl<u(;^^kGCz}IrWnZzqwE3t69{;%vaeg)bgSB@GudHdig*&!w+qP}nPF9SL(H(XN
z9ox2T+qSb}8=d6O^Za`seDA)lZy)UMV9l#Wjd?KVthuUYRo&wrernokBFuv=?e{(C
zOOj26w#sJGB%-G~0}3AQ<iiw|==z7j^gU)2iP7|jPD?#J82PivMzb$baBB{17}oKd
z#AJKSi-R{_WYV(*VPy&T_uvJ2PjE~o!EC9sN~a@-Z$E7irM7$F<I^N`n^fU@`9gkc
zf>r^UZ-|%4mqbl(QY|oUrGNU&!8KIP{0R@SYm=Sp7yL8QZcQ<|7L3rK=%(hn)Hjak
zVNP%F%V3JPx;H1)n8}n8GzFGx0DbWd-cFJCA?uVo^+N_V!YS`X`4U;0`B+d=CB6>X
zZ75?yCL){Ddd+RC&7>N!MBqyqZf38neyc&Ic+F!4lb@ml4}^LvTKnCKL0sf&o<~O}
zp$v-G45|Cq-~oPFbqx%K-$HOyjsi79dJl|ehmQ*;PNcFoC8iL>FBQhygX53Q^WBRU
z`vPm3?kR_8A!Fb13C+^xhFXTaI=WuP&&zQNsVK^P+h+tQJ39c!$Z0W)P`K+&;3A2P
z89ea>KVOt}rNd8{fzlFnS9TjR7LfuAcTy6d1_phhhJ}?!@>Kk=lg6($O<F#7C{;-k
z@=}0nk=tYp0X9<MYza)RGbji8!GlzQfD{J)HhSnDBm|dW5`-ckG<Q<E8CQwXX5JJU
zRaBcyB44FE<U(bnB7#?0-`0<RHllmRPG{OXm3d|6td;9kw$tQ0V}^(XbIG+|H`&EZ
z#P1T(-|A!L&vE-5y2%!Dn{<?jYq21cdujdo7=c{Q=S*Ju4H?#TMT@IF6Zo;W;XJjp
znJA6l%Dm1nOQ9Wie@)>|lpo7si(^T*qABcOWPN#v2!OQ|=jZSvN9i5Z4E1lYG*ca!
z{bqxVVi0Yc#iI}DV__NB?gKZh*mt-DO!%Dvf5R2z{pJgoo0z;huPfG~VoFK4cY+pv
z%`p5V9toljgbJR1&=d2Oq^Z~MBtg0?H9crB6aXjNwYZ|clcuu_Y?~ZI?92f5Rf8E&
zZi*H#Q`;MV=Hja@W_LO%gvLoBe(A=zP<DBcBksqA|Fl(_I0FfV<5bz#D~Nr;1Z(Pd
z6DKn_fChtc%!SI+rNd1K$_vn8HRz<39Vm{1dcT#d5dR5x06OBdB8f+(a)hnGb{T-Z
zazCa;U>A!*u0A$7>Ap~LKt{Fj-BM6q^!0Dz3o+}Vdd>){23H1hfR=s{7D77)egzqd
z@%{IyXBh^y6$;Z8e+>R8@>{cZ{O3O$gn7G7nb;lgKl1aCi&#1nMEXf1Fy}W8)mdG2
zyNLqqk_)0X^%7d=I4b`_6P4G*s!~ApPb6{@xU~^+MN$8dNp}%qi)<B4!jz|DakL62
znzTwBB_naoZ2q)@C`P5|`O_>y7Gjw9Nw)3yX5W3${NA+vxbFYfeR22x)XjMn;Q#4-
z{U50hd8oi_z{tLR!_xSl9hBT`{@e6#3+YfX1j~t8-IraNWPKCgu;E#MNn7`&bFq=q
zsG3;#c-W7#cF^gxk7I~b$p0oa4}AbE*Tt|w@*HKjPW^kpA+B>2ssYlrcdXGSmqUVe
zc}%xoZ3G;C9DRJAQ~|mIp64^hw%-~Q%8eTTc6>iesucQsz!G{cSWN!WW}mBf-><p-
zG*Gi`AjEp&(RO!a)ChRjIotb-bmwKnIIbZ0Su%-b<bS)$)b%_*E%Z|NaD)TjniXK_
zr$^DKrgGc(7xP8-r{)Z)5yJ=;#BIE0Om_9S7QY%XEbBIUcwnC=Jt~C6O$sVRsV6-q
z#NbT|Cd7#+JqiTHjW#W;?0&60dP3uf5=TBJO`Wv}rv*S5t!yTxjJ+6yf}1^IdvUlh
z1a<b;FH7}9tAM*LpOY?F1>3t$4A}1nV-m0rF3-D!A?Go1v#?$oG&r<j^mAw!uz0td
z_OKfIu&;CrV~2Y77^Kl2Nz@0Lq>$x0za5`6pGQ?RXcqPeFb*U7Jc~mh@|d@5Ome9w
zCa`}0gK1Q-uNrE!(TDgL<o`W^;@=ZWsUtmfwJBkNiixBQh8hV>;ie0DpepG3m>A)C
z*o`I9=oVc{zyB#|qYsuy*!;sJ{`v>2|AQ^^r1L%s2))o{|C}l-+^WfSmB~(>#xd>t
zz8Ltq9B4<8igrmTyp)OmoN%Om@t1QkC{T~nSv)wB?kbb>CmRjm6iZ-uoS?np_+*#_
z++2pjk2^N!iF^+yuZ*%)yN+I%%*!bjt@rL|XI6`PYNKH<9c*43y$5vg$>f@R7sM`C
z7TD`#xozi{eH&Fu^s<sKf*&-%I6YLJVNk0<8lqb$k>8KrGg6z8@{b9yjnoL5>rI6%
zl<cTdmW|+%1xBb(Sd<ADm@=w`kYc!|(xR58jH}BQuzC&t17-iE#s8rT|E1ANTSNzS
zGc+=o!p#=;KvnSbF|oq;up3*X(T7V0Y5v6#d-DFe|3Hd7Zb=44j6oKFb1L;8x&`Um
zbSj<6$&3TqI^0B?D)7`YcH)@EGDS6=L{f@qO4%xPVTDsV9hZsNL_dQ>%(^niQEo#c
z>SeB8tDL&RwTZt^a|Y;JRTA??Lr|?Z(Kf_LPmZM;iBCSG^#&oTGCGPQ?W$-G6fp($
zk|XQgIVK2mz*OCiW_`?R%)vVHC_vl?RoWqm0CCk<SeL#=)YGsfAa<KgEyCk)j3i>g
z>OehjuzIv|tUUq)phd!Q{@rU%S6-UesKAVL9)gqrEtL`dBxS@cR(e>xNbO&Y`484D
zU^|l)=cQydpq*ohJz<UgGp?35SWoeiM=r@ggG|8!kjbU~ts)mLE)%I6hx!k${Rhwe
z%cD@FJ{102LZfi@N*f1dk1hQ5rF|@Qq(8AQG9PcHm+`9lo00r`JE;&*b#P5s-Ay+X
zZ!w-!I7&ewp{`eDOFn*=M6cneQ%52jCOQ3a-Wj(G%WxV?SaO}T&?h<SNs;!RCFzBZ
z5XY~%o0+nYuNt?}wu%z~(2w6cNlhzgv!a}@^IR;bMu?%;+E-FE$3*uSlRge2n7i|x
z3t~llzF1Xt<?sqnk(8O#RTI+$depYS7Pc<N1-?W1)6%Jz4dN#pM$Dy*&w;Aoy6Wgs
z%bGFDMTpx>SlGxW4CyCf{>wE`N43&3pcJws7Z&ER2dPoY?!zY9+!`>9V_C2ObbeAt
zsEor(=tSyjl8>1#)Fu3b!~elanE&v<|G~2V@|0tHuzq^F0jU+zvUPe&b^1!MWR|Tz
z8ya})Kp)7TCM_WJ%+Jf?(B%<utSl-jS|{w|YFn4>DTK@d63u~i*adFV)6YO3p^X%t
zjqUBE+Po|LBz&zxiFJ8<U0=mcz`<wMn$E#z900}fGjxro<6SOoc{g(lM<LNo)0&t8
z5KgaZaT_4fufG**QBCz;d2#kZ8bay3hdLtb>+#+4mx9vR@n+&}Jo|vx+zzc8zqorT
zWBBv+QMNz8|5+maOa8y8`*}US{ZZ8Td|j?2*d+M(Z7Dn8MkalFK+1m2_tbymusmAO
zo?%|_bMB*I(yDtzeCVs2W_<Gl#?O9!RQDDV-K%p_fBu!LZ;W1VP*svNU61ym71FNG
zIVgX*?FQxOR-R;1y;B4D*QRA_-pMWeRpscGr-@mHx(yUIgc}+E<^@5V-hKX72OM``
z!s}F}_9?Z?uL~MGq<Jf{ORduzlcez}u#0|35PF?2>i~8inl+`_v8_7P^_v%uGRALG
z`p7?~owM0Y`p1J&LMb4o;&bwtx-1-|j5^1oQ3edl#H&#jD@>(iaPU~UY#juSO2o?@
zJ}uG8#-2~tag0~d<JFxH)**}|(x+FSx7Pt_X;xj$+ha2&IdQ(c;B8|KBskGm<;?L^
z!Y!U+qQoBIR~gNVV-!Rmp;mFtwPFx*FNrpfrGN|py%zCwlv)Z^srp<ewzIQ&@e)=M
zjF@x1?YeID#}2dodBua&5wn<P^h3Il_0IzP43ygP4cY*fb<;nC`Vn53@${5>LiSV$
zx@q<I7B-`EQH3aSNJ)=$nObR8_j-lmYZY$VHdS+4g@v+QiITJe>TAVriiPUBVQvUE
z*mHCR3esUQhs1s&x`A$Bh4N`X7P=F<)H~5T7TOY8)mzY;7rx&K8Qy$8%q!A3zTg#U
z%QvQ2)=rqV4Z=sHV%F0i>e1IcSJO9Wo7b+G{vH&L5Wx&$2-Svf#J0TQ>1UACj%@I>
zc3#{|8~h%Dhsni2sOQ_u5V4NwL4T>cQNLnVyKI_0=pO;5IjGXxy(@>Pi(kiW=RB}i
zGRP9Kk4Z(ZsGCvu96pa}ML(s>U&n3bytNlNC=nrtnatonj1lpMiA0~IS5c#FQ#)qb
zIEWk3s2QT%J3X%VfwZJk(Y#_@J8K$0Xc@tPsmIWyW7qu5bY8P!Q#)?jto~!tI-P~n
z%zkLq*Zu}Zi9So0zuwc~rf;uskS?N$^TKXz-7yc^Uj{1u4`-$w@48#<31EJ*s9gam
zJ&UtDFL}~DorY7*erR33Ce@m4)G}?-EnSJG`NV#HxTsn|DE)&I%nou5z7EIk&en2e
zq5;R0ZNV~S(kUH<lg!?7wW5aZ548W7RYil2Mg5v()TDbl0cXAa;mSl~s?~hx3n!c%
z<{EvSj$QqLW$7eqI*`-W{v+!<=b1g{ie1C1dHt$o@g!sVCa2zcIBDSLkY>6)XS1Ed
z`bKT5t?SZZ#-vv|9Oq|`M7kVjvVGMGf2|kbZsss{QXu_}vm5wxk}bWQbJ?zHeWRw;
z=1!Y-z;YUoD)3Pi{By)G-H-Fl9&rV~fydl+<?z>}O1c^6=YVrM8Yh*#;%Y_{4_S0|
zzg-v)qAS4%-yQxTAH9?Iam|Ry?|##;BTgne!?h8(Cav4e+)Oc_FhoQG0$v_B_v5{p
z#ZqG0=l0vAZIg^X>o68XGlC(WNRN~q=2?@te&H|?7fF+ve&n!R#D4rxzDOtR6Xr3K
zx_<Q3YUV|gw0^fR97HyPHXb*R<K3D0++;P9;A>~E@kdx7Vkf~X_x<<dqnT}!AN^io
zaEN*qndnVCb{;FcHS;FP{SILQh;jr;JoL#L>e{<Cb0&%XfG`e3TY@E?3XheYnpv-}
zcmo-L$ksOpmzDFH!QEf|Dq&`b!vs^j8Lr$UNR@puVQPpz1dlxD9^5-lGrKYUCSmip
z(sUWl&uhEC`_;m%RG|@>@C|u8T&&OLG*GYR)(fNMjFKY|lkpk(^qpHyohm^`cl{3D
z7ruw#A#xEo@NBrZ>^m**X7&4pLEReD@w<5*Tx(~!CSNiN=zF)_yWkwmDf+8hU(NLv
z=F3qfPca&hLokZzL$=?$(uL!D)82dF?6A&Q<d0J(PcrK1FSS&B*P%MXSZcR(T0O0s
zv}hk!PBvleH;`<JOnB442C}YMM2>4GJ2UPZP`1dsr5>=Z`k?`twN1ju)sqPr`3#a;
z={<CI>*g#H#{oK$AqI?h2AD1MZaN2bD;B@URg-NP=P6`h+4L>jCR|c4ScfgD#`%-0
z4YFGJ6Yr@zoV-qMXMnNeX318J(+2MC=gzBVn)p1vJ>(&NjBf^rE%<Ic2QDkXU*jst
zW{krYi5QA48E!m4mo;GIxK6UGDm0;o{^b~0zSA_YYaA^3juETi!+^0>-@SFurLk|s
zq@|A|`HYcE-=S^8rS;-=tgv#}H2Q^*3?0*x{zm(_b|k-33M%>!VJ1JXkJ}A4%~h#D
zo0r?u@$T$=ezH`nlw-6Yp<gGL3SvI`tJnP#_72O8S<J9;Gzp=YK&Th|6LuxDsKcgN
zLZrZuXXFF+5zC-i*)Tg#KJ%to=&*dW4q?5(kdORz$}!8RS>1$I%0A1oS=KO3^gN*z
z|CD#e9ru-UZ#lQiGvho4qqqKT%enL5e%UZb^cf)+KcN?Y(wqA8+J4b6L-e{TD4~Zy
zmM8y%=b^JOAxuaRVHiK65B?4Jsq@5s%P>rIJfU1?IBA$mhy~%efTm}~y*99R%`9qI
zH~Kr_8L(7yWv6!5EPmKB`l~k;X!5dqTG_2t%kr{GAZ!<S^gMr9IjY^>Zygqhej`NM
zlIFQjoHUZ%C~j9m%Koy}_8s%{e!IUpIh$8%63?C$H1i#Ljl4@ezU$jB9HxtI5`4#4
zduez&*_od$j#skEW)w8=8+wz!#>ruxuxgox&W;s~Coj^cIP!yhg}<fygWolb?q$2f
zHegjU?VSBC2<40Tihs*<=(=E4)@Png!+M>1$hK(}IxU~ABUtZuI3g3$DtPC6_(p%N
zb6h`SRXdHET_`yI_v?lUV8_3D9=n^>)A~uP_G!fIOhGzd%`pX<id&tF`XQ^zY2NGy
zK|Ej1SG!vt`sNC!0juO`him~sAHT<!j$0eN8O9vHYsxW`Ceg4Te?T%pd7<23unDev
zj&i5TvWM&X<`Dg|p7|fV4)62<y+?)o6Chw<kPs7zRMf`OJu^Sh1gZQKUoswf_Fb2b
zGWc;}kNg;)_3v8`Tvw06r!}%|1(*FC)QNd~vtMAj{2X339$WWamyfch{Yo!>IDl+|
zVN%~iALGo>8JqW%{J8zG1>y(#3V9EwO<g?-77786gpJ}vZmiU^^}`Ru40H$*Ljo}Z
zeF!2FDFvCC#aOlH;0Fze8dwp8JkkmcQy`cn<OFOA0kf1Qud!i|uT0>EWshcv1V{u}
z5;#4AHfD{`j&M)!4<HELR~y@@Y2(z;8h`Ie`|%Yx4!Q(o2fcz_BjaCK!($w=;n&mj
zqaS1)%w4(()E(kH2@V4hA`%XVh?T_Lbfh3g52P9F5_}`E<`%%a)1|J{)Bi&XBpR#>
z(G&HWeMd*x`Culd5q28Pa3mg|mDOB*<T!?f#7wL$#0^P@3Wz+%R-h&AC4ChmNa82<
z8gh@6ic8AH#$zr!GW){}WVlcYJOiE^!-;oCJ|wo^GNc~l2y_ZcAG!tGiDt*8C-Fx)
z$g=q;MitZsbPEnMvAOX`PE0R}m}qDyJltv(Gyj1_Px+4~k_*wbFgNHE>^YhO*XOS`
zt`$v=sUo2h)eNnG)gh>(G_S1asIVVF4ap>F7v&7LgI`6j<1v>TS&FeI@exg;m4{0u
zWF<4V9I1#|C2<u!44r^m#jBGy@gE6{2_cCP#gvn8m7NNsM_MKL>bE)uT|zEM_(U&*
zHxOE}T!aopM*3rpNSMeKV41`WL(bt=iCl;ej7M@~`bne;z{!8Z4vYQ?%Ye4Qn`O-N
z6CeIXN1`QG74p@>cn3B}ttc*PX<8!}F;XPaVqJ({sJH5vdb_FoTPRdwiXj<@JZLVQ
z2l^u~F>trkBX1QkawN%xX5wz)$M~}pdA8!szb<FfslyOy1xURm@58C1j|o_qO_c|B
zevzPveR;?YNk)ca6R<FvY7bohLO~%TdlkP=WG}goI3{8t-m@O4{q>FlMT#gH8J&V$
zD`#RlQ1ELMMUAvbQa&PukcG_Da-gDQZzLC`9m7&;;&%!j3zMnF!09gz6kD<-iHhhI
zGDl`IPD#7)6rx&UQ<H(bUxO%eq)C$JiOnS_<7ArR6$$ClD-2qrFTeay((^-!J|Y%!
z(CKUn#c4FgXg{+905V?Vez1C^O_FvID}=RVrd9(tztB*^ND;?LN9cTXpuEhO+MD}K
zUQ}ojxB_)Q(B$Y^ipoMr{oxyg2ap0Ve?P}go1zFE>yYjZ^NlZ%q6p3C5bq6hTV$Z#
zxTf2n2vzFPhL5>?FtLA`#sRbEQ}X99(Ere7D07!O$?i7v;jG4fr6_0L?dXFDp@LN^
zJT;fo^vt~$-Ws9hS2Z1f4rmA#q8w-oxaa@d(P5!RIIag_^8>aTeXXoP&8B)zqcD9;
zA;fY+wWZQYYPYHbT(zZoMyD`hOd>=McCxj?Nn*FGkK;rTmJ7Y8L|!Ach}Gb|Yvb_K
zO6VEO{?6XAWI{c)aL!=ur>s(<AuW)$Q{%OyA*XtwMK&N2gw|C9vJ74or-D`7qGYTY
zyNFfaB6v(XQH8c%L$W+Fb}E-uv6A@JL1XK7J8s%Dk%tydow7_`HMOEn$0A}(GZBFn
z!cKj)pia)BaSS)Hk@ie|shnOzr$}|gB@u&`PD8WwTzR#WRmq}j3^XwW5OVyzE>D8~
zOBPfoZqYo(nRsz~4$#+Z&2!NJ3OOqP+r|*QA~hUJH#A%G>!d8I#{?6*jl0J{5`$>N
z)Dg?^Rd~u=<bf??Fb@j0ZZ~9<0i$q<$*SGBR2qt<8Jax#E}FoMF)xf3VvBMY8DOHU
zUR1S4Rtdj`SJAD$gseU=Ys@bZiWW(Ou#`V3Ow6oP5(**l4{jzqudUncaoMawe!C=6
zR2FV$LpWJrK$v8!q+^sIt{=y%&HXZV4vVr`%#blcFJx3H?mT;u?Ziw<DT|z0(-3}?
z1TJp`N0K%DqIOZOzFEkSY?Q{AVfd0Yh6FgIEyp2XVm_t1R@f|e2stVjx1T-KHYIWf
zyOc%NtYHW{sup*jeaKdRCZ)uLMaHad2tA4nSL$nj0e|`H<1c22=0XAe=Ojne87>#Q
z1K@mVCBIh7EPKd53Jez!iUZLGf2pO!S%x=s2qUT-H=3Oj&|){YQc){mRzAcTb%D#r
z?hZJgYbo4y*4zIxWDw<p`@}(L&A-51=`6W#2Z7q#J$#fg<P`;X3lA*`0|^ZY3kd@W
z7YYp;3L6STC`UEpnRlbUpE~3kg@wz+aUc*YsVBlG;v)hjA|*m4VkLsrYptRpj?5y-
ztjMg-EXu4oqcNi}V=yB&qo(}ZFwjNBQG`guNJK~^pqIYat`}d#;t=)_<`51O8WR>1
z1{1Ccwh5*wf=dNlLz_yQNkNg>kXeLTg;|DKhgpJILpjkV@FWl?Fg1`OurSaf5EzIN
z7!^p*38M+C38M+O1ib{y3D*Su2Tl>Xj)!K2WQAshVufLatcARVnwy-Pp7v{&BaA2o
zE(I|RZUu1!eguI9?gQ}#{sytgP0UTrO~K8;O$N-JH?sOMC7jvIyLeJLFI8NvD3n<Z
zp#-i3F%3Qq!9*Jg4Y*dlmYvZ&s@n<QDcecgY1;|iDI!kR3tSK63v3Vc349BL3JeRB
zn)?RMaZ)sIP#mVHlxfHd<h^iPJ9C>kDe3-AI+7o+XqCyxYw$hv7=96_h)v%rcuG1`
zH*yZAl1<X8Vv0NSh8N9^@>G5<wL&eOhF9%-(TV(WYA&0aRnnAQCJ(Rc_fux2qrfSN
zOu4TaVbj_9%xY=9oK>@-v-maDf<^R{VWuDN+jqoc`~{u@SM8(FDU?ia{6(H3SN)^l
zDVa<)-n;LZ$Mg$21!~!YymD?yr}XnWRrR7)g;TVdi@dh?UvoLgibb8`dPA$ADalL)
z-t_NH$99W6mGOI#Q#zThyhnwKZdJ$ni>*bj2E~;kEEn*+xNfYTxp%5ZiBo1Y-kDUq
zO54&X1ztsW`lmfP5h6Z8h>!*d4yPM)t(9th1exu;Pi~iI{4-u9chXjsy}UucAq(YM
zQ|y=rjmvu3gH9mX;4N_{SX1nkhwy`1AvfXM@a~z9ZDz)EHgcLpi1PP-?S%QJcR{|x
zzv96&V_P$i7}xZA2O&VFBW#)c>QxRhf$T?+#ExXgwqRZ}j_lPA^1!iT-ZBpCl?+mV
z97oW^u3*=;s97taCiRTwfV_i0#G_}{wyqg5uI)t)Duf(|_aE>Jf`arwxWv)tY9WaZ
z>ZJ%8htz`q0MkOK!pY#^wsP9q3G9^!(t~V9xWxWvl0G38Bn7Dl--K(&w&GCJZ(Q8V
z7_<rb87lzkgYbxR&c$u)G_w=aYZ62TDTScxGO^Rr3lkI%Sq!g;ml0bmzGavi$^HUk
zCNG)F&-uKz^Sf6q$O>}0Fbcb)LYVYf>#vm|P6LOH73*3-y9`o%7G<aWn1Xlu-$+49
zKLyBicorN0>qJ4P3{?Cd%1rr>@F_|I`L;ASwiBzlZSx<a!to-M<+MYep;Qvr%N3lf
z#&xsi6{FnoH<W1dlxgy;sdl$q@+_%<x&iajQPy}MC7rxxnmk*o!z~G$yk*)1YpOl#
zuzA%efBXw2oIGaQ*ByT(Nl->8ETvR%teV$NnRixnNVG~gMr6+qoBogxiuk~smd{G>
zVD_@P9V@6DlxnEAcAV|q3msL6*Q0D!xJ=pLXtkOPf1^Z_CrsmyQ)PZxJNP}S7C%X;
zH$lm#ezeb7d3t2E1^0Kt$A7xo5DFr5y#o<;-S3u{+WT?#jQIQhSLGnp=ic|2t%v+&
zk<n8jKZ1m*m)^4+yK|2XZfgA3$_Z`uE*nevC_oHOMA*_;<O#?ktC`HzSZZxUdos>A
zFHTwBF_7UiL1^9{7$z&nGOBB*17xuJO_0|tO|U@U^-?R2O0Jd^4{nGGE`v-VQb=Bs
z0j?3313)#W5OGI6AQx#A0w?lCQ&+MqqhjNTOT`_PcVLZApIDH$1W>YhjmmFvv2ol2
z4uMOPziBPBO4oWuUGhs@G#hm_&X&f;M-85qAI_Fams>2|-6uCTEKM8|;?>BQszstg
z5l)i?XDf7JG%LY7-C5Q6QcL3}7E0s61D;PEyJ1c#dA`SIw8<!LekB5pQ)3z}O+HwV
zb2+8rd5F(=l~H^rFBXZLCmywlIbfG|!KLIuh|dU>QSAT9-lz#jXYq9@>I`Ls7E(GI
zv`~wIr-`=!OQ85-`5}WtVtRb0Uc01XWknhfM}4_Qb*x&k>_tV}cNg2Eg^yLYes8)L
z1V@VQ-O2;5j>NwhQE?fPGK%?M$+_d^xkqjADMaEnxP@TiG@=3$zyq{D(j5aSVHD%R
zi)8}m$$M?$cGzW{(#W|LP&0~P6z9Q<Z35@ndu`Ho*cF`8D7h8TGAv;f>%oh40_W*_
zZIX7_<($$exF=CF4q+7Uz>B>C=lOeWJ`kqXRT9Vi8%r#K=M@_r6~uhw<5QLDt+I=!
zs~WUN>In6=<E)kvJQ`){mFhHf)-2EQWju@JOH*Y=m68sf-W=tJOC`Gb4({#_4wbr#
zwhI-hiObAP)ecb=8-UOHWZU0S<-GlO)2TjI4wge*uG?2WR(8?D?sh{CvS?|h!!VRe
z!{j8jcYQI$5Wi(hOi{?G8}P!*FmI5Fk;}_eu}X*yQB6d+==Dn_+-4C>f5sp~x6uVc
zh`vv~gav~C35ltB5BL-~%5wd@6PkVoxaEyJ?F)3Hq{Zcxdgqmz=9Plxl@{8R(Oc8r
za3nUR>6R9OYO4KNO5jb?ZD|tQRSapzpcQEFgMFgouTV~v;>&f?*wqOc(_NDYNk-o)
zRmM?j*W0xSd6K1Xs*^j4g|ddzJO8MkSzY60p+id1?O<%7Xc;Wjsz#rH%ml3R>Zw_g
zom8|>(VozYE-P5_i(j$r$G*bW_ktJJ83MgDvb6rofrDwiBXEp1HsG~fq1-x@*ZCki
z5%P)wVap7+BZ}P<%i0OX-l>WU;AhE~%D}unZ5OgiTmRiMIc0NDo~Oa2hCc|1LqBco
z1v$>9oz;5_pJeraR8c6|`4r8S)K7Sa;6zk$rIcbs+G{FzkI5%`?SHmd#gE}4vEITN
z6m4ukjyj?KVg^ZNy*_AStnq`B##m{97=S&W0Uf2PfrHZ)Ha5bL7XG=Z(1Q-j7cGnm
zyI=>B4!ywrQyiU*7@KOtZT1EXY2K2~z^9u2@cNs1Z3`y&UHF;sh`F->v^%Dca70cg
zv3X+)NCS8q3OLj+u$-VAF=F#JZctrF0*GF=J<1uD8E51=qYZdo7+#3L@d4ap_G5Tx
z43P}5OsGxJK&HXUoONL=;S$vKZY?k01Ls#lpz{Z?E<8-PG^U2wM53<^JXlpBDFSzq
zJOvdL#)bKT{GQ@Gf%s@n5ax{nKZ%$~f{MICmYYsQFqKiZg?v{LMs{Wsl~JxmOLE%P
zG~zGg$1PtQ;7geE=M%ox&V@QB81{6A`g^HVd_$&XiBdhglu#~#QGiBh6&zP0PWRn;
z(ca-ze)hb>D};g4U4yWXb?oDuU>gXLvtz-#+YiJ!5?*;J>(oDUu$$t;$K07Y=_llw
zc^bOoiK1v|Xq&aQm(LUD&-c<>>8&*#zUM2aud!lc7g(R-s`;J(@3#X_M?jen7svXA
z{IsFAp!eVBwbxm(UAO}b{WTy`)e#d?RXGGh$7Sk!V7ldJno&7i<#s^zmiCT!jdFK@
zh=WBGf{LEVA?Rgyc^j#0wlah3OY_Z6>tp`qjuK9CiF_b!L9C)i%36-SdNCx2&96|C
zXG|UQ{=bY~2WrT#B6KeHOm=z<<-p*6hILr{_4~=^yCWKfPI|=nJiO;vq9PG=aT|0W
z81fKw6?m+o1QkT7rrj|VZr3kkoG>w}ojDrr@3;<~lDwR5aPv}s8_p;2-Sz8jbf2~D
zMcn>g)1gQE+*q@mN|99&NI8t(t=sHEv5@=G5jVNmVXtb=)rnHo;9HHj7}W2+lpb*_
z{++|1-GfGA+=Fjx8IfKdi&I1m7V6wT1raGefzK>-b<1+d{oa=7kPqj^BFnJdloJbl
zW(cN8!f*Q-F~z>>8N@4sFF~-5&`&RaH%mTw#~jr`&i>eHG(6<G_nFJ=)aBALbwtdO
z?SF>{MaVE(kGMFF=GC!_<oUC1(KdpjI3Zz9RrB@+?}rZsoPfh9W8(}UZ&{oUa;*?q
zi}A9Z0*V32Lm={&`DzCW?|vyLfj55CoHzdaJ4yh1$_THag93R-`nriN?Oi&Vzwx01
z)1QexLIl@fcff8{{A%kF*Sg`{SSQggop}W2LaEt_Dop7NGJ1IKDf6rEy-Ys)Y`FS(
z2+4so(L3Sg$XQ|ZrvzR=&PKR=F=3-i`+ihQDh7O^B9a9?;UC?vI1~&;rknJwYjgHJ
zEW`E85tsJsHvheb&QbFEfMBLL2Hq9Zb5*>EKRcyGx8`umGB)U1lncpaoVgh=NJu6b
z_`B2+Y!-oq+g)OxUI{K%ZNDx~+=tI3$$rj9PBalTwy7|s5-6OHL^5>a!23*2$V}WY
zV*`9*Q&%BONo%R<YfCiW_z=38XkD|z399YeAK6@w&OYK@ktREBVNc7=7UA>%UcGQ-
z=XfXx+S@tVVV7+RJ&lo*c)_piP&410s$nb^mp(CRJY<<IGj`H<`6fA*JmC{14kk|1
zeHw_je?;*bh=dr-Gyd)tEbto^D6n{Vw2yu_q`P{S|9gHxTWp<r@Z!jSI8fF7K=^iD
zj#DkD@(KL>kKoWtf%U+tFXzJCFXzJl$skuZ8>jy+IP~8j*Ykw<><j|cgf!eZ-7n+F
z|LsBQ_MaZ4E-wj`#fODyxC3}AV2v#85MkdEYgw6Le_b^pKRXOH&IiAgqrlyn`9nfV
zLaI|tJT9ly{Nl~-Cmri!HN#`;CodY6hd;-gOu52kW0~BZu%JQMZ}2;+iwSnY-o7|w
z4RfvfIARTQeIz_&4RK9sa!u{w5h!krq`+BCPGd3RvZP&|up!q01rAIcu_t6qci~Ss
z+Z?gS8~|0notl`p)ue4I6CMU$T(!cR=jv+XV!G;#iSbM@y|&UDvrZfZxTf~=tY^EX
z_VU~@xu*8<T)ef)8{-!}mA6J(;2hOVa#^)m_YJtcu_nionw66TvFfqv3F^V(c`Hx8
z4@WY92b`N)I=Nao?O2(c@o-0^%4JzlK#`4)UYtXo%C|>h>s<0%S;bj-aYUs`W&X@g
zjk+o0AR8Zcc>KnK0R@Qy28lV<381F}rf45#9;qS=vt=~860iluv4VjKf#I=&>pm=}
zdljPWYsgq8ZGU=U3@CHB)Elu|eK_<DtO=yT9a!F#1ab~VO;DvI*KL@Yd;=>=1H+9s
zVixZGW58wg?LS^AERZoDMC5PZXwm-XUMco}yb}L=E7FUUlQZxzonb4Fs!q!*Gt0-6
z&g#|G&D72-pU16C0V~TZx*Sz_Q#Ixj65R-fs9J4>Us7<Ql{2)W5#pe0s)d}WzQq(A
z;%d1M)QQi1W#R#jupcP?+k(^OOh#Epcl=S^FF*P@y)Hi{l!W5P-LpF#tBE`>Q?oA&
zg*+j$?klSSY=1lRgACtCgg!5@1VL^6H@p;uvO2*&t3S)_K$V+sZ%p;3qLuXQ_Ifov
z%#&_eXyKhPzza`BK=d;0M98UXM&+3dL>2fOn$YzmI3b#01o~bOFZp!)rxv3C=cbjB
zc-4qUZ4{nSKmT;d&bu{m<i<ELAB20es6)KAFxl6`U;2yb3)U&1KnCYgE@=WLs_L<g
zHSfB}1{HWdEiITTpUH`y%%r`Axh4C%u#lH;8k@^_sq}5hU2cN`k{A9q=LM&@a@Gss
z-#pSG_=UVPWd}EQOm|~Ud~v`UU4P2mDBnlES#QE@6C@`e@-TT}5?C#(-A#EUqIeVk
z5iJGU!sP-8AEy+A<(ku{ty=P%rZbw5MV5=^-;2&0d8|tI2IjxEw8u`sg0cJZmQ-?~
z9Vhp6*p>`saUD*gbX=IlYUaZ{FG>-Y^q9}n#NVZB)!WRLKM1)wAnVX?hYT(#FC8GA
z-ZX)2Yk0WQ8^iU^y~Fj7{J-Qb2MDPP&F(C74v)_jA@as4Ev8$IhtA=mY3UCx9y(Li
zpyp_M{h87LUcl9?TQ2QLU2V?Z9v9U%aR>TTtg)F@mc)6{LqG+lwJzlA&PJdGvP0Mv
zIPy6$8^PYFCkT6<WaOPX7G7v#q)e-$gJY|m^J3S}qSxcOG9Lmra<h{5c8fIaDR<BE
zH*0rB`&Ju*(6+3^#BwIadJJcV9f2G;&WTRbdf%&&se(SiR64!|Zx=E6T6LexgfnrB
za3;#!2V}x0+$5Yqsz&E3Y*-wp+h8*+6uRZjjw-NGj%{eLAIOcmh*T)2+_*HR40#UH
z$YTvDJ$#T}qwB?KQ&U3H$pwjpaTP|LaS~Pwibvw$N1_5Cd9d%L<1##$7`83>jW`#o
zd_wh>Qm%4*x{s^poA0CY<Le}lS((kyINjgX%QLRhc>b7f&RC4nu9{ezpVL&O8xN^t
zR95CpHaUdUGZ@L5xcnBjO_C!*%8R8L8n7E$p~JN57hWCgd8coHZAbsZOIkuyn@e9S
z{78wV+YmZ}n%EwEB1upIqBy~;iRgY>(m2DvMUV3kXvb8vdd|ptd56VX<9^3b(oe(L
z>1KK<v|ujujV5DD#M=VzQhj8ya5q}LJ4u%)5t+HLN17vfXBbVjym|mvl`zzuvC^Xg
zW$ejq?nq1N@h;M->NUtql7V6T;w*5K9fLunJqOL+)0xjPab?Sv26GPEw%Uql`62VZ
zE_0(djrUyDVNt*VX=ON8-B_4ME+*C$Y1mDj3o<{pX+*Z1H^}});RyxHpkFXvg<IC8
zPSmj$K#WEA6l=eoOoeSZOVbDwW;Ds*rd2Bw=`jlMZ3?opjIw6}B4DMQFJN%CbMBM_
z&P?4;6rLTuY_SfV2T^7d+ibNplet%FY_w<K`0!9xPLAg&Inw&K5_@lH5$iY4q`(F?
z8;*6#4k*T)gTE4tm@jb9fwfwSlo9|5isJ5iJ*7tkM0d@af5VpS1Hu{4@fs#1(TBN@
zJJ4PLA}Ddwf4@y69?pX}BvlqW6%tw$r{9tUE)0HS=woJuQ=6J6s~`s(x$KiUc0ia+
zLJ)v%z-h-wJ<^9LwqLs>5SiQBf`KZ=io@d74$9)Kh1Uz+3!Bk*D0Gpf^qCN^{APiS
zM#tjQBU!&R@-01qix1F!7lj;mK@&f6-Nk&lzLsMkk=Mi<jdXj;!44tvSmjohSWaD~
z*y;-q=%{;JKJ?t=FZ<=A0Cmo2LKL&Mq(w@AXVXtz>fBtrhP#f`vEG1Ye566B(=1b5
zKN}oP#Xea3^Ci>RKHbUdP|Yq5*jj9c%VRANE0R;v`+K#fa-rI&7qP%(0Cyj@iBN|;
z35sw-!bj@2Z;lUW6=rP1(pyScgLJft24!51jgbQ}dHuxhoH$IRQWu-;A6wWG@iwAG
z`8<vW#Xg5Zj%s*%)}&YTLt0bvgohPdfeI(f)0Wkr6k*tf5fF6m46R*GeC-hxm1=Z)
z)|P9!Z!bNHmBGB>U!v$8=|1ERc<M6Rp-Y3t-5Jce*_8Jjd)m0IXLzpQwTRzw*5x-Y
zO|wc863MC}7j^i1*C7cxn*#bF86J9kdGL@<PK!wAH{iqH-}wT#OM2$ek~Vf`^s864
z%E;Mf4OaZAiquReD?DVfnQm-tZR;jH?q-+Ja&P?C)aN0d%(s1nWLt-p#EQNgCehpO
zI2}-NP`o!TXhqbt>{=Xhz8P_wsAT=>Ie1pKpK_g}DI{86ZW-_K{!3sz?#^B>E<T|{
zz=%fiqg?R{xAGFqa->xiC&qzR<+k7)M>9_DSGRhQ;{|~gpCY)YrNj`tF&IsDRTy8t
zxzk7n637Ny@K2K;Qn5Mej4So#uvDFso5uwz7anN9xzIdAyp%aw?9i+t(uf;q7UT;}
zXZvW_757p{LKN;5-*D|&-D$+K3U1%Mh(i_w991nT1$}r^#fTv@4ES6$*@&ufQhR09
zWL^2)RIo2+gNg^hm^~<H{y3BeUnN{Ui;2!|ZxM1wv}-ZldDY8>znLQAFj&^4_YM&}
zk_L7J#2A<NY(+{1p{vBcs6}@!$g7W478v(u+m#f?8{#Na88=HS^9h?7>SBNXIu~ri
zP^UL;Jxz0(+0eXe;5!?;%FS=j=QaddF6ZgYT)aEe;mmXx`OelpN{FMEJ$@hWj-2>N
zFSsagVg{ufIyUW7!Q-^KOX>R%!#-=kdIXX%;bxrmO7|_jy2W8#9rd5b0ky>gx#$yP
z4RnPhmh|H$Q>-H}THKjp`^&I*n54sMW64^i^op`{lrA3%)^bX!EENZ!F6vT!OP41S
z9TDGL`A8%fU(I^KO~)w2v;uPDELU>-A?64GBXgDoLZvRzRzts>g^u2766(^Y0K+y^
z9~psD5BG#nPR$=YHCWLjM>9BXG6>+tv9X&)I;=RHz&xqW6C2;AhNdo_{p)16xu6o@
zBSi)@|He)9ezf{B;*?L$6Ki0%FM8ER#s^dw4Gc+^77V-^f?7QHq)u{qur1dPZw~L0
zv_4LKYTHM{bd`Ye_2n02B^>OMq@0TqUL94qjujpu)?Z3{Y_nF&UuHNfTAAG2vL;Y_
zB2VRv&NEolc4t>1>QjO6i4J_&6dM(2e56%oLIp4^Z_cPdzrIZ7IJ#xo=_Yo%fzy9q
zXS`H#&=u!{S)cRI2Jl08kRAzeYw3Ac6ZOpxV4f!bolw3Q;r!-jf~ZKgu4b_v`ZPy8
ze0-`B?1-Ul5L;sDIOci!CvVix@<bGNq_-Ug^g^HBXGy}N9%KiHXU-QkBEL#GWbTYm
zO~Krs)T3Gdke&w=X|R7cTWXk(gVI*x4xb|Hwq01161pZr(lSZX_j@3N^AKm!&s*e7
z#aD&w<V>MF67fW4P7GSt0`t;~!hy}C>4S%|feKY8ISWUVE);r`cbzd&1Y=k7RP*>d
zz61}CC+nTei@ip51jp6k%{VTLMjm?>QpmiEPTNxzJ^X|+j&I{_LF;8s4D8LnDF{!l
zSjZ?F!QOw_Df8b5Xc@h#DZ@EYFk30&=oGr!L$Go**WYc(#@vX`rxN0)$T~V#9`=s2
zR6K2ugve>Zg%J+B%Cg)yc$lwhQMko3`sa&bE$}2y<J>xT<Y73cL>q%XQk$n_WT$b!
zx6&zaj>u~iKK)WlW4tX|%D3$T?J^kMtlo!G81HG-Cuq8gM`ZAOfZoFV>o8wJi^}QO
z6g7_+cfP)`-aa<l9IXZNu#m*E7KB!<_Xw`OyW+69wu!M~nKMGLF*>*4W{@wUg$;Jp
z*jQXW8YMUX457k9=(XNvGtWC^b2M6?1mWY?ImzzIMS^8i+AwIRR$WEU0Ws@r^i2TC
zRJtUN@}>rj<wbKo-j~iYODLRC*)SVbe~r3jz=Z29ML4~mK?gQ=<i$D)pcH2-t3rm8
zg;06|Pr4Q};nQBE`ZJ?7T>U4j$&(+hL<Px*_@4-v;Sq%B^@s_RQVq4{0@>BV{<2u?
z8MWKZI$HHt9TN%6-|EnPz?L#i|8PZFr`O~Nff-Z#%=gBsYr@dei<HaP;?0e%ZdGGR
zqGLnN!&hx!4voNLqd;1A7=rp0z?+wTf;WEyPXVcAi_107O2t)$^G}(g*;&^t&G$dj
zvn~{y%eI6`zLqo$4C=am=Famp10@ci?Nfwy$^+|1km*tG+}!NDMTRRtJ_Lf=M#8xG
zuy<|5%ZpvSH)GXUQBfde+aKp76Ml>ApOqNR{sU<_n?dj1W=&Mh<{$)@O=`iJr9JPY
zwMd4%i}Ys*to;hdrg|kDRS}nmtZ#$H_Cq0+nX6TCINT{t;kJ0Yb!lK@4zd!IE27rM
z>RK;(5+I;YCiINb8x7LaE_|N+w32d!krhup<c=KTB}0K3&8BE03AdtcKl8>zh+s!0
zjPO-aV_oX&EEbe47dt1&7bCt$yvi?eK0^QU@ia;4Fc&CD)9vU#xw9)BKiM2;u^fhg
zkMZIlF0Lf(>gEC@OPiL^#slMNCCa3?3?LF(rKlC&682Xe%s9>__m11}iLJW}?%_8l
zN@eRS8SwQR5Bbf%ulMw|RHKJafw^gA?FV+dBi%}ccr4sxj|H94(p!<)y5W}`FF))G
z_H%)YF;+%<PqoqJ9WUOsr(c3LUTVGJLMV_e5gpQ~Cf||pRr4SM`c+jPn2~hFy<XV+
z_=Sm2T6s7}<pjj_hSZftQ*f3Pj_{7uBz-D~2U9dWTO^YHw7ofdjFIk!?UPIBPrN(*
zYPTS;MTTJ$u$dqN1No8DqUxW($!{L2>&T1G77J8>_UCtg7~<kKII^|TW683W90nVZ
z1BxvVy{Ms>_Nw@Kc$rl_f-3pm$T59Ry=8l?Uw<HVziMssC~y1UZFj}7_`ela^I7)j
zWqW-@WPd__4828untjIoovPL}Lh`!{`k8rH?EjXmRjc$ld(|DbW%#mp2I+cTE%X<U
zDY@?RM)9`$<4!K1idIqRH97lja2hM%E^M#Z@blgyeZSlDD$@~i+v`H?>irnw@9of&
zn9<v4z$@P?TK3mt^7*_6y`vKHeAqYg7pdRw*n7+NXzzaNBl3N?QVV7ByCPb=bH6<5
z!hGv}+I|mwtncb&@(p_Zo85I!$Fx=w;P>9hPpK^g8}Kxl@YsoE^merpLB!>;{W>--
zwB}EM1K8Q8;w88S8ZmW}cUJwa0j_4(zF$%a*#x|AR*P@>?@u-xeJl|<&I<8&ytH~0
zBE1g?jZ8me7YCfZMC^?>H;-ruo%z-6Pv><%b`o{F8@|5;T>WMGuTpM^9piP^uT?H}
z@c-#0iT_q@W)a`62r7au>|snoN{P#Ug92w3Niu|uuB?&(eD`Oh*G@%nkZM4dMfH*6
z6Y2?F%UUCv8QK~c5>lSX8ibzD;Bj>*YL8u>x&`Deb5=ze6Fbi@-78MFFyI=z%U+$%
z9e41zzn8lED*B%XbO}5V)Gyr^Dy@Xnz>xU4p1YHzQn9ybSM!H#W3|Z0|L!n|`&9b<
z*Au``J}#PMH}4wyU~n*A5J<-sQ=LH30=m2lxe=`Agy+4BxDnzr1NF2Ea^L?AzE8{(
zJlX_0t49qx)an{vGRO30${am<kA**U_=Y>H2h9^zX^)0Kr1*wCtC!6ab9#@5KkVp+
z->?VH6NPDyq9Zu@hSji_4z^({jI#Wmwo|rx&h{R+Q?hziqfJ)pgtk+$dj9O5w^PJ%
z2JaryAm7$4s%kd<p0ZQMaZc|Zvs1!xmT6*&w*?YWhnw<gj^lY<4pP9Om_8#ui129v
zl&R|d@Fn+Yx1YB6&HC^^RE8{qy;Fl<^8JXfwio_}|IOUQ&4kI+!_3IW!Q8^rh?S9z
zk&Vg0(cHq7iJ9rE&c^Jk^e@No|8xJ=T6i)UnL06=Sz7#GmozW`mu`P2BU>-VC@py}
zO-nT)Rd?e5RPEz|Aw?J(>cE15AB))7et4$f+(3F$q(Ug)$U_f~Oy(sgse%9#gek;!
z(&T@nsgAki-csc>2!!WC{7yu6wsbG?H$s0&_-kI`3$5M-H>sk0mwF{)T~t_dc!VW3
zu$q?p8ZYuElsD-DN8O1kEkBo}SCEL72n3d}MB#iJCD6u8Sp+cvda@4qW9R@&83KM9
ze6R`RKUVVQEclsqzWy?32>&CG@4rnKFsWf=jW2<notnA1K4~yVo}9M8dDB5$!hT}F
znLM)>TP}bESGg6NgCx3qv8m6qu`#qMppYOO9FC3-_aLHvEi<Uqi}g15C~>TpO)G*D
zl%shq^Ew0d-O}Bvef0fIEM%gVYkR`?F8%6l^2x8>Z5@}0Z@w&!a@0a`wcOUdT!$_#
z3&y(r#a^FDLy6*=Dm~!i8dwCHRPrdDQ`v3ISMH#-XscxSLI8eoQay^jp60t$S6*Go
z9bAFj_T*MCxpcz1*6t;YZEn88S=!Jcx@i85!UhWDTW^+*SNnX0cmDo)reeLW6!?W<
z@#?^)V!fE)<=*P)P}nt`C&pIb&!=vQ(v(+U->{1ebVf#>CNV%EWG%p9)J;GH*3JxF
z#JvaR^=Gv)y6auOnGoBA7|5aTY*m-JF2i0HMV8K)i&)@+7a5ruHQ2r54Cul>G(;TX
zy<A^@M_?}KbrynGE|V^y*{^k{J7|o$E^Y0uA)lW<ssJzn8euU4YLQ>W;G!yVmrNiv
zRz55ZGSmXonn>a&?YH|L-Qp+X3HHmj>`X08d<fCp7OweNe(aL@###a158g=3-fVt7
z%BRlkQ5F<M0I-{ZotdAzv9kv&y;4I|hyvcvBL@p+3-MLmB^25IMy&(%IGdSngOZD2
zovwev9(Sh@ogxJp?RtB0G7K^%iTyx;b=1ZA5Clww*^}gXONtreM<m8r1>cs$*1$Lw
zyP#@KbTucOE|SzJ>UQlbQ0y9TITk&GxE=Z(_U@YO2W{QYIwVTc_HB|jJ9;PvAMgP+
zq2lvXFfwxmX$FP6^^`)nsy5z?eAyU%_))ay-!zqsf1h%bUG;gwVp75i=F0m6CX*NC
zC2<gge_EyFprh7|@ZFPsYoY&1|E8yDG7~C*9c*$vXq5YFuj&3*M}ehkR{^^E(k`oo
zM3G6fa%4U+Ig*RZ_lXqstukftk}}w}Sab-P48N(CqFj3~^p{}B3FRC-TG1HiUO`x&
zu&8w$72>rbe$g>0nDXPSqCA5HCnO7;O@(K;T&|5h>_LLP4p+4x{=am%1&;nn{+(Kv
ztM8@=pg34f-(vc*a8LwH_xDepR#BwdG&Xi%Y?)g0^pHVxdxU{kB+nb`C(;dtN(I>M
zQffAs?lRZY2Z4%E^wcM+n+^J+VS<}(q?JqsTw!AVbjFnJ+91O&>T^Y|-Vg~qn@(Dw
z0nBDbmZ3y;bZ47{o6<7H>fat0{|2HBb|!Q@E21w4jQ9v(5L2F>N@(O%G%wD!id^9t
zI)Tq-?9ET|b`Q9FLy7y>Pnv8##%C58J)`c7vO04oLH|MJSh=@6>rfkXRug47D9zvU
z<>GGEsODNS@xagRp+lJrpRvCGf%(YCQ{jLLqyrC7?tWb(U`WOm@!Cf9?M@nhTb2KS
zq<LHI%h?&W*704b9V0@=5c1s#R<SCVoY$wQdK5C(X|Oc-+<%-se!i`4cK-nTuU`Va
zicq>lU!#HD*H7sGbwqHr`M(Yb=l?sAOGa8&dYaZ=YI4$aPfw5Mf9`RC3}S2NVuPn(
zx2D`i@c5#|WBd)mAZ@8f9Y=i_Q@vBYjR^i5fg10siji>_0ec&PD#v<v=b1AnF(`}}
zmYWrlCDPH#A|%d^McHd(US{P`k#250kPPOIAs@J?KCM3eL!eumsg{~2sIsq*errM;
zyrR8?y_%d2t11r4l5+&TY47h`-huj3L&n@5JY$YS38}jnn1Gm=fSO{=)Lz`c2B@JJ
z8k?D#tC}kqaeqcCgJ7v}HFrFLI7Kmr*onHNm#1;!0ajsAi$&340eU+S3rLxC3_L6@
zaN%i%4Ez78IrDg^x<8Izrm=+VNqX8xl4Yc+EJcIrVJu;?CCV0Ae#Y39RFWjJB}*hd
z5>kpRp+aPv3en@qRuT;gZNDtPdrfrjB+X;|^W$~L{NsGy_xt^x^F8N#&b{UA?D>U-
z%bvuJvMx2`=5mi)OrL5Ltj_K$pPx+aI?tJ!9m>?EnrJ9f_wvCrwKy$P8^y48kKi#`
zdp2Ety-uxLSGBw733pu}hdg>mZ-vCrWWtWg)*CDNkGEj>R~x;tOdUO6>Yk62iFzq5
zQ9B&;WfeXP_^9woRym_`UAZI(nk$zL@ajb?J;@C6NwP!Nrb2sesxP=VS_E>@y{bt3
zLOIp|Ow@eLCu+!gvnT4k+KyrxLYKBE6}5#XZVEPwdzOkjw|Z4B&!V=d{0OrkWBzCf
zQ$xX%;a!Do?lzrXS6<>ss{9JaI(G7Jk=ecb<eGIl9O+qnK_t8(Yd4@TtI<8EPbBi*
zul2IKJs#pKq5s@;F#h7_5^w19%|E)iW$YG?QCWq!B-5Pc=lwfh)?T<LkV2`;PU-C)
z(m(5$^Yt60HC|G2S8m$LPZ|^tDYpWLCW2?dYoDHXe>J7<ePVMe;8TkHh(_yw`g_Ct
z)y9YG39rOXiTan=UTBxt8biF9fHPXzsgt@&&Qq?UHnlqheO-J_cxr2G-P5NV21ojc
zE+f?9)(oR}!O7dGO=3mu8J?+u-~I_40+Y9bACJnh6=R85VgI&!dF4`-qkIjrFQS4=
zHt=t5k}V!W%WqrVBInqg@jN>}Taid14S5uZsGrtK@LH%VXcBzwZ@o>Ml}VOlN9CjM
zp9lu6JE!bTCiS(r_W8G4v_$%r`b`}3-q==h>8ryFv6~4AxmSu&9ZOL7mgAiUUNYLx
z(q9NYclYyIE^A$dx7{qv<*Rfnu8vhdR(=#cPUI_ZXV;f%ayXxIcuPVKMx;2@Jw!;1
zXdqo;ui)HtBeEvXJS5KJ6{#-8;apl1^}oV0>u>bv1>vNYI}vwBh+p&&#b(yXOGf{d
zl-SUKe<BCC^0i*qOIECJ@#WK29mv;}h}1osXN`+Kvu`MHRV1gT44d}&gX~r6+?<Iv
z{u?B}P_9+x?SC6uqQhU|kjVWw#6j|DL{vic_VVP~Dqz!bj$IFom;c2rZz>vDZ6<p=
z?7HNayi^I}gH-tv9M53Yt9*?#3Rhsk<%8U3IX~$?Daw3V7sTonl*17f%DU%+ns$ZL
z$@gEZMcc!2*^V3Mm-3%cdT7#fF7H%F_WQ-fA3aCIwk-<T^Y@aQ4G%4o>JnNOo#-9O
z%!+d0sVTPf#@wcK3S0WfVFbhZ$D95A8o3{>mb|67x;_)Hx+Ar0<%SUy4u6c?X)9Ns
z=oDNKy)XVjN+Y_kg|m=Dz<d|5RtRJ7$#Hqn@I5Rq)xz=$`@<3QVu$_hhoz}I()NoN
zNN)~M_s8#aDv-<g=j&;0*K5*Q1G$P4tMT4O!V+~d%^!73M`P?1JGrw@^t=-uaP#C`
zCJ-Db%5IpO!;QhMOd^QJl-*h&Qq$NOvBj)&1(D!MxQ14`k#?z#x8BK6vPpcTl7FEx
z`WZ@eYv_unWjo~A?TIOV=#H?Z&8nqk-f@=UpQRrZX{fhRoQJy7sCK+#rh=CXI$Q^d
z`)dq`z64lREEJP|u5~yjV~iJEO5NTb>tzXcgBOErgYBD(ZZ|()Y^lz*aN|Iz>Uh8P
zTSp9KZA;=i>(!4xxh@-%Pk(pkfd|(gLkXue%(9{{_rJ!A7bq-^I)X|xty=Q-V#6?-
zep=6=){eWtp(0}arb?qS1;4x1pNf_TNlIn7QE_H>JFo-Qo&^?V{a;tOm{0H`Q*{2l
zi0v!qCq=u;%wLKo0sluST1`-EjcwkSRD5w4!GkY4VK4Vp1JUlz0AhqiR9sig#kfd)
zYK%oxym88f_@s#H?$=9~39MMw(bq>(wHo3&q$hVrr;320TMpVMd)~YZ=4cJjIqJ;J
zo!o5foNS!QZWg%1znehS@1aec^lQOclLy_biL~*e?d_b6szA*VHnvCIs%-LlrxmuZ
zPvY0cZ3xn}CpqlDp8nWh0<ixU#&+M9C08W*u-%$JeODEFEb7XY=e!_<c_ZBOI;}Dd
z<t~JYv3}DdCnmb&{lf|^EqQ}-(h)&7!WBs^JU96W{)GItSHtFh27mA#A{=Gm5jfST
zIV?J9o`s-60APVf&hE)C-#^rtF^?z7o@^uI=wd~)PLQ>-5p=nGaRDp~K7z(yAOy-l
zorWgPMJKL2H*qrQfU}#Ui=FIjK0&J1)`Z1w09l|*n@QCvdFhQ<tPC0Cn8t|y`Q}xc
z9Yr{%{sR;TPp<)sowN@);!M5ChFEC=(ZK#s3m0{5n;po(#RT)R9<mx_!QJyjEMs-}
z$6>b_V67|xloQ*6t+YAs%mK4?E-V&q<O5_In!iqiMvDE}P(SV-3Q=-9zIFr*AQ{kJ
zcqSAlb*1HG+{lwog0!KFE<&M0e)XpNB8(%d0hY6RagQ}4Lp~ZA+3weL2)uFr(ydD$
zvh#8pQ@UzL@2!!x<f;FPPWSGwNXK9<diPe8iU&mZVMMNq#{j|=X+ZRsC2_H$@p18S
zCb99xvD!h`sy5VVYf7J?=!VE@N>@E5=ty~BW6#JC6V4k)=~FFRIL}K+*AX~zo2z&~
zROvSIe8LWKdfvcFuGieWdtn0W9u{}8^6^HYEo)F#C@Za+_WpukG<qfb_?Mde@`}2k
z1H{<oKPPT#XFR;KNvGHMtwZ9-Q=4yFdmDNV1_@9MD~XZncD0vqR^C+(Z^H9U48*l6
zD8an;rVo?LalNCDhua1kOq9fu$lji!xV}47Zp)~|`#z1Gw<#i*GI#1Fk4;c&-(PT*
zt%^(7zN<uyO<8JUg>a?$*1@jDFLFl$w30f6vV1<~nl5JfT=u%&D7`PHJ3f=gL~P@U
zZ>`ZvG8F&QR29kA-u1#RWc$|UtJ*cOB1dAW_a~^0^?~hNQAb6`E)=zp%vXMU6>Jb4
zJ@)PF=QdksDW5|*x~IkhQqB4uk6s&7X{{CvAq8Ciz*UVMZPM2_2{#>mWp~*1sQn&N
zU%N?@X5iIyjcCoiO_~M4n8%_Yf)x~0b)1~~2iu*36)px>q*s(4oftndnK67UuZ8-(
z7le&UEOeui`0wHks>A*m^^Sw_1YLA7rKiQhx*SF8L-Y!0U<uf(p&ve=e+Tq;vffKH
zL<6XRf7g^SqA8WBrszw@lTH6iN-;#!=YG{Rtt!ok0DbQH10nEgb^$CndVuDcF~9t<
zG&h|u*CHeo!4YNF%KG#;1uD%rMnOu@h|B>%N~aq%<E_Cq^TA7YS_uGB17aCevU7Af
zc)(G5)8qg(o9N7gF=kp63M3oUZ*FgD<A4!M(mP8IAyv+(6nu|jF0%3nSy`}jFslQn
z?hr1^Sme;(uOdP?VQ#|kc=~1J(&7oyPcIF>Wd%l;A9$%x>qGIxhn~}of7hpFLF`nJ
z{gQ+hWc0H%g=rEf$Ue-CWYQtDE<D4;gEm;w3<1e^-!U+^bq9d-RA@hEYOsV1VGX_?
zdJYjlleakNB4cX}P6fc8#sI}LGZw5Z=n8xALQ<%M9%7Qhi&2U-<Bg9-L4rGA2qG!g
z9$}Wk8Qc|S%odHtStVOQ3Z<WH5qFeH3TNoJJYy8TFAu(hmM_LXQE2%yO~J4&(g{?S
zL(q^9PEJ)2(aCd+NeZY6z?dnH?Rt>mq6X$3SLmbwyo(G3Fb82laJRB?w!vYcM;TnK
z7zIMUg&q#n1@3G!(_7#`lkcQw%tFZ5$-{xJKbl7%=;iW^0U=){3<pxFn?In*7Yj26
zgnXt74pjMmK7qhRCu2a!CvxCGONZwNX!`ja#(<FLao|8%D3*B%Y8nv3V-ND62ps4t
znkk^~$3_@43-Y`LoZ>Yb(-aK%C*&3>oZ>DQlN5}P!^r(WID{?_QxM;G2N`n+a$gNj
z5xR(J3WgnmEQrG?Oa+;wVB8_d;x8OxScnM-L@9VSr_8)kQ$wU^0dQM+&KVJ2bp=8G
z<!oMg8U&inhs_OSs-XFs3H^`t1hW1LcM4i$PNtY9nEC>%#?I!QA8QO`ffbH)cb<@F
z<<{9~W{#Du2p6d>nS+7g3(pw}p78Y+wD3fhTj93IWkMZZd<CuWOAeY9=!<0H(^i1i
jeHgLdSOHd)1GQ!>b)mtvBLJ)ef6s$Y{04$80>J+O&`gNr

literal 0
HcmV?d00001

diff --git a/core/src/test/resources/indices/bwc/repo-2.3.3.zip b/core/src/test/resources/indices/bwc/repo-2.3.3.zip
new file mode 100644
index 0000000000000000000000000000000000000000..b94020ee80082bd9d56a3c20a392d17135ba88e1
GIT binary patch
literal 75600
zcmbTdQ;;S=)9=~SoVIP-wr$(Ct!d+F+qP{?+qS3qv~BMDeQ|c<oV_?3dnzLTl~Gao
zi>$1=$%|5w1p|ix`L6-z(IoZX&i@by5IhhUb0<eebv0-ZunaRk&HplYFIbRY;7`9m
zK)})dvr*}P3Afb&L}2}A^}h_}{|>iyFf;f1-!XhiTpf&^T&*14TuFFzN!-j`-3*yY
z^!@)=_W#K0zbE+riEV9a?)v{D#s3+D{r|@NcdGO(46F>S|A$E2{~eO){}cLOCH?*1
zH}OYupxRLs1f;_m1ccy!4>dGo`rnoJsy9$A1(+$=lYbGA=ZzI^>}7#9>WnvLj5YQj
z4MJie!5(uCCM6C<A`3+l`At5oB=Va$1xFT|JQPXhH+hPdY-DMgifpZWt!-`VYFC~6
zfAs01tFFFvwYAPxf32);&sMH$?<?;i@1^Ib2iODj1&7Mme{<ej`|1=}1A=~T?<X_r
z)F8I6tOT|Rcy1KW)Yh{Fv=z1<9FJ<A8yf8By=Z<uxAVGjBd&<6mef0Rd}ce^B_#j2
zxXrNW{3A;zvrQqhZ(<P7w~<nfEpu}XH;;3#v(NMIEB3;PK<1NoFZ#x?adn|1wc7cW
z&9pvs?xEr>DqaWdxn=VZzTkj2!kLNI)wM`3wh`P$N8WqismKsEg3BiIP(3Dcp=6im
zP2>f&b<@}cGu$`O`!fejZry261KNc&;Zo%8gQ|&h@D8=n-$Ky5BUdkPDiF00D!j-)
z7FB7qgM~qq9hA9tQ0b?PI^*xYBH5v?$9eoXXF!#}qRmpb<{-rdJB-73haK3m`pU@g
zNLFByMus2i8s&%8N!WIT{Tc=h|I2*D4J*=@FyTX>USvwxqI?w|u-LknGP_`bhF=PQ
zK$jcr3C{1gHAK~@2|Vpa_SCo@q<TDI@N#iXB91+JBIsYaA_$#Gb(}#|EFQhM$MPt+
zCr7!MX$0Ip5v!nOh{@*;p*~%fBJ&U_$gEYqVZ|0}E<%~2cM;=(NSedU;->&@ta+j%
zdjv?1;LF7eFr(|Oe)Ad&0=n#-xII+kL@)4$-jz4S2xYU8`KamcYD?%}Abdx{d)Fm3
zOG(uZ1!Du&-S<(Ex^A6+hiut!02SuZwoTpQ`ZXP`^7e^%MjK;W3ja*(0E*tVX#`2z
zl(&>se8qH!5>As$fdzA(elfKT4s>f8ghKvTbXA-~95A|c^URPNgtjT_SHvCnRjJ5%
z;r<fC@YfyPZpEsNv}MMdqU>M4F@f&MCj8+K^9sp@8|{c1R|Q-krRofXMp%z3vq*6&
zPQT^(2i3@RH^1B$8XS6e*at7H8hCS68%}UQOPzg1b@N&aR@VSdeZ&D6mPqrQh(c*I
zqE=SCVbIP*oszEEmYme^BwZnVCorC~+KXupVKcrU*AFUIhu&y5ymZ$my?#EJJdep0
zn((^L$hedRya;Q=cBz~aGIFdk+OHJa?F+<<+WMH4w@eiC0+7|F;4XfD6i;JVL$juN
z3*!L%RBlgy7f=Y06cjC+xrnn;cuV-nN5aJnRHvOZ^wPf&d{xhW9c8z8x1KXiFGXgw
zi}s1=Q?11cW#=DOM0V?9tBLf6wnZjY^TtFozY|yu)pUrVHpEKJZsFXp#i|f`7$@1~
zDE5rQlNSo3jobTUl@y_Ngz5w)KcLA?DtE!TgAAz&MB-N@y%4P(abPMMhjQEn=Y+ar
z<`!*Xr6QOddaEF)tT;?u$Ef9p*KPYpKa@!|J{RN8U=|`m9Xh<Q-Om!QLcI`VFo@Q}
zKE(LEZGMbfI?`;!(EACVm#uT%ZKldC5#g7E;ViuqC=-1asa;e(v*#TjxwL7V;DZ|i
ziLTI_`kX*pKSs#luJII#K}cWdX6C6<W-^AMa|(nn(Yx$>OoVOwT*6&a;%KrrbUMX5
zY7iXXkVfzV962BDd94@~l)D?4)RzH6d~MM#H)NCV#%T|mpKW@zT?V>JXewE>HXH~C
zX_0tn?;&Q^TS2{i*TPlR*=Vg@X|6>KiK?r~IUsV<rNy^dbo3CYk=~3j*&C}xdK7^V
z;fHv4t~q0k#a5f+%0l@Ec4#4N)M&CmjLp%#gQM(`0KBkLjc#c8*iE6l4Z%M!qa{<!
zkogRFHULCGw%NUUx~D1Bgy}qbys|E7{0E!IB6MMM7`%mh+H=8bqwM{Zq%(T<zEv9k
zf^(v#?k2eo4P0x`<e*e(6XNXBU|GI!*yY?xuF&1s5xaZU0)N9sw>2$ChozJ1kyy5;
zs3x#Gn)v+0yib13S@-C!X^TIi#DHg*_-2sF@(CYJclIy(4e1a*ni?>517BT#TFo+y
zQ3t<GQK{Or*EGC}Cfti1e;;>Vow@c6v6!`8lsaCHTAxk6&UBd#(pBz;1~4*a?6IaZ
z!j*H+?UJpw;g`BhJ<gvIcFLrfw8Kx)f=e<I9#YW&#S6uRU<|)3M_kgp2A*|(B#So}
zY-9yHN>$&Xs@su#q&sMIy+3`E*fa9|5c!hqh$)V^k0B~?2RxentLrLs8AkiUl|#_1
zdX4lJfy{i=Av>0s+zqziRoHNXC`uV$yg0e+FT|~=zbv`Bs;uyw%ollT;r8AxbaTsr
z<MKSHXTh)=^2cOL+F5+Uelz@6RhC%cVh60UWlh1TVKuJ=ztumzcp!pnR<7usd`bkP
z@(}riKlBJ?MZS+%2aVlUkVZ2VH^r>7d0czMg&jZzJ8rDKsF1ba#`GJcCI{DoU8@y#
zFiBSc)|H6z)Wf0p8Rbe?te!<lPB#&i2QcY+)P9-XPAX}xWw7syV{yvoIw)_kv^5{T
z!_!+YvKJrCI!v5TqF-0RGYte%rSd9WEV;ya@h}SWMnSfh&vH(VKaHg{sN##)s+M1h
zmEY8)aLSG`k;Eo;Yi=W<nh}&VSKNr3`XZXiMv>ZV1RtPW{R(9yI>!xe_GdK-nJ)0j
z_JGwta%;PQqi4xKRK>f;h%CEOVla)2S|)+PWn4fPc&2HC+48|u<nF;|ryOqtoZ}QL
zIQBB)-GkGM!>~G-VRq`A$L*utw<-?_4Ev!*LKcgf{8<)OTiX(=tgR4duV3R)T!7qx
zV3A|V-H{TYZ}nt7@2w?Q@7hiyM$s=x>3XPf4yo@j;1%L-ROXr-t7tY#3*DuUumfvg
zi+2?X%}rZcR@O)|oqZUdj9}lQLR;ZHd-?q!ZIK5SlPs!|TG1mEJL2=h?3Bgsg;mKl
zd`D>X4<p*a$zqEJ_Th2N%KxwbuWw_v6>S6-jW!ni3(n?gDu&F0b!HxtDqH*=Im+y_
z==HGA;v*_v&?m?um%P5l1)Vg`@L5(ly*^0ARHxNEO!1LAey%v}SvpKc;&O$8u6Ib1
zkQVJtfYB$(lgbnP#9kLHYfFB!QX6<H<Oou>aOrf{c!5ris1@z_nS`?w=qdr9xU*uu
z)h>9oVtf25VpPYfg~yS$-%M9118jE0u(BTJSZLeOvSg}eseO|lF8fkLW|gfY3LvTa
z$Z-?PkwBz)sx1@yfeQUH4T?KgR-BPzLEFS%1F3^Fp5mWYcR<!kiD~lsdRUtE>9cBc
zA+C%Q%)38UF!j1m-X)MpSYZdQu0S}wxK}DRTe`C4JgobfrijCL*Z3q7JM-cl;*lm<
zZh`#EiCr<NS^fg;Zpkf4ue>omz3i_VFQhWR9OMVZ!YRxec)W<fFaIY5I1@qcV)6*$
z<(UQ_T$?S9D2wvnZ5{(olH8#OxJ4brYfkvJ@4r%baX2R<O5`gT#}}X_9XRE5sXCG$
zg9JKenKH~r_z)cJpfLTp9SFm%BQbqLp}jfV0Lm$Hv<5rYRULo8*r=|R{^7BH65j^B
z^+*`Y4ja3m!xJeoe;ge&S#Oec%F5WnF$lOLJ%t^c9zb@Mmt?KmFLT5xKNx!<D0?qJ
zcLJ2rp)vHt_0Q0z$wPkQQ2c{!%?Ycf@T5Fo*Dc(Wu%s%1W8vG9@6=0@K-t&>)73?c
z1H_riRWGS6kLmHiNzS(e(j)G&Swk9X5iO1Li%_SVo|Cr24fIMLNNgJQL=)R80$|n>
z7zs^<`*H^yca?B(M81s^*Wyu4Fvz#qM4LRM&~6T5_+qkC+B0n!hR*lABHBpPJVDTP
zjc^BdjNA&D3ZY$IBDI`{$jX&7)=ErpIJx#H#1J1`?N3=uttGk!tKPtErn|i%b&XG_
znc=1==8P*$Bim1}og8vR%wMTM$y*_|j9#&di(uP|a*VpA1F=WqnD<i^e*=-O(xWfh
z=A|)oB()^2M2%3}3We5bMDbS>HWp)Jn*dws)C6A+3`NWUr0;EkRqXD{PRw_(TZO`V
zqWe#|D0GY$?1l3V@gmV35(QVEHVMz>vI{;M?g(z^Z9x!s8-0tBSvMeV5dlp#6=PeK
ze+3_S=@|!Kv`6GG+=ecmoW7hMztM8*-VQm*5e8{b;S`RW64-;TbxBh+9Lf>o1B;Q?
z!t4$zgk**1uwdL(T~0Rl;!W?YA+(+#mJs#jS2;~0JW(nhgpG1oug8IEEM)zIq$4gN
znsPs(n8g4!L(0-JZ>aLv-e|z7eP}0lbQ=fStzU83xWQyUgI=vcykml<Y#q;($s<SW
zsU7l0dpbo4?QrijfkCF1U|h6q%z6L@Y}w(mbE5hcGIDGDorq4g2Mg>+EpGw-K06$*
zKXO0+cBU)XHv4OFw=yR0Y~f}EGxIE8bw)dj@ljg7sn=B@jeW!%5zydI^X9VA$x+Q#
z)WI(UoK+C=Sm!kbf#zZxIPx~=O@%7WwoI5w8oN#CVaCGiG%C7Gv{_xfi_51K%amjj
z@YcGo8b%TX#23&C#w&4GuCcb%Gu}sVdldWfPO0=v+fBLST|M1eJLGU3#Q@W5ke;^y
zKo{{M%b%Shel4?`)MMi#J5tvP^ifl{6jtL9UC-ZZ6W&<w2h~cwD?FA=j;!q&9;FOp
znNzQ;z{wLHLk3Oz%|SSwKH(QhrN)uvu!e6Ans!(Cvq20Sx7-AertKP>qCeW4(FzNC
zn+FWI#k?jUW!ntn%$|fthcWxQ2~XaKN&sIi>3C);xby{$1zM~@hI09Q%SiOsC-&q)
zQ`%+4<ZajsHc-n2<^j&E1H8;q_+0OEbcO}{Y$Ry*h+#BJdtLJp5*B!+34t8TROEZP
zB3UfCXD{pNoIP>BQUMoulA`M)^7JX2s)J+`wRL=Q$}rB=kdW<<CN^%#S<&HMhxBdl
zSUUb1Zc^R~aumFhjG*of*V`PSLjo$jWdAl7S!CJ7jQkIsUUE;`xe|=zEa=Sv=DUp@
zn;6+Z!CVxL8-cP!uzzA#D7hyJ4;ZH|P~+E;>f$!AFG_qT-NEo4aS%(Mg^~!glUxt#
zoWl;_)6W#Lxen2+3@08oH%x^&7(+RRE2z`!{M!QT?kPFwgn&&VixCX53&|3jH736_
zn$B@%Sn)k0o=BDBEG<Iz9kap4mTMr_xt)3^kG`xG)Sb}5?B5nQl_LXDHl^!(bizVs
z7JIuHW(fE@(uzzHUHvnQamW{h*WM?5fgO8<LdgzB(OK*;al;{NOxV?F7kEC<AA*jR
z&s*osF~w#0hz`zeG4I0`T{VP%f7H}`0S<pk9VMG*uCEF~Rf`RLbAPRkuzC=f8RQop
zpyXGVWPb$pHdNxfxTP7oYhWV~15q{roic^RX=@sffMV-w0UMTKUYM#Hm}13#Scv2D
z`Q2qk68OBoR3zVEr^#j9W1gy=3Wj}v5o1BjruSLbgCaacw^Ltbe=Xk0oxsZ78z&BX
zk$@0-jO7*mFk#;D3-vno7CfORB7Kp9-Af<^Ka$<jxhOltu}ec{VqYWF^U@-FmYDaJ
zGQk$wn}X|{3fuIPs$ddTZ4yci?3n}>*tJD>wGwAI$CZW_?$v%;tiRG4;ekiU*@i~Z
z8pm9chnP)O)?f^h`k~A>7pIVD%{L|XiYFD2<1M$-KZWsGb(Z^;JHM+@z)^6BQ_^rB
zUYqnv9}W5<)IY(cy0e?(byf1%D09uoHUG~IsC<&N^2<IAZ3pXiXMCc~U5rROYH26=
z$15a<Wk6;!D~HnVNGp4NI(vX5(8D<v+*F#;H{=^dHP<nX{8M4S6^i~t*(?&lS-CAj
z=i&an0jl%>1`uWI7I^^d-1cPsMkMOWop9lliIAFs_G%R#f?s#|YfOVB!m9aE7EB%e
zd`IG~RxjYl$k<Mev<~MRBu&X6q+DY&B!A=>>?pEbR`eO^gDbu@j6v<hA5|RlN$Ljw
zULBc&pT-x?;SG{lamGTM8-6}%^bZZMSEP33$~*`ut5iX8>Q4uSf{^VvEhAi5S%p{D
z`CA-)x6yfNg)8g9RbBiD@ut?%(ZK{Y8*%i5k5t7La{c|q1;St8;#&J;u6nZ?L(6j)
za-QBL$VQp=ub2gggH2Ys<Mz&wbw-beN&G`*0*Nc^BiBI;^vA#4A<rtks=7|l-sW^n
z;YPK5Gcpw4tO-+1l*E&BERyK*&gIwkc%k)G>9T~h<;E%|YswW<{EYgCo1VY=n9qos
zhsnwf!W_Ww3fBBkOVy39xUc;<iF=I8sX7rNidJdu(Sth58;nxg3JU#TnO#WRjaK9C
zbUcQ5$UEx`0dCPcTYssG@AAt67#w}Y|JL4Mx|cfz$<h64A;>>wllfF$sRm>%Z}R*w
zN5445<`^G4o+i0Ht5B?a9Y`WeQt7S9o_nm)x52MShN%k<R|0n_^}3u0RXrfe@2n||
zM8B-L_`;R5B>Yf=F|HjtJE*>jW*V6V*D9I3am$bhMHxxZdrv7eC#W~WW-BV0tJ2VO
zL%Fv{T6uk9)>;U^t-B_Ni{0&xz0<Zriq({}a6P%}NYPqm3hcuH<$hS5;^gSHGH=ps
zc|rl-a7E$TRc^};D3)hbdwYZQI=k-HkY&#~A#ejTY@s23`(0MbaVsjyN`GghG7CE@
ziifN<U?eNM4#_xCC5_zURYTrj1T2W_`6V}dyryj(w`fLBtLD=$jg8W;gyIK5H7*m+
zzu?Ikrm#|0)UNBB6#lAM><m7jHgVMN@(=L$m{{!lfZYkG9gQ!5ju`l=x}n+Qp{3R+
zvT2q|B`RoiE9)RzA$k@%<OnAYvGi<Ws2^%{)(pERsgc*!t3JTC1SEGa+TOO0U9wWu
zzi`=hGOxipCjPKJL+hI%8H7kuBc3>$tA3vFlOZx^z3eC5rd?vZM}Ba4I004^<*GEb
z3vx}Pc6AoU?AG94lv$29EIK@Cnktgbj$y8mPG<cQZ|>eR@=3NztG0(K8Yu%7F=w#R
z{OM-uT^LdnlVvm2qAmCB0iz;Sr6po@gauVDrFJkRCsa1L!l{-SzVzL7m<(GS9os&L
zs6l?>6+Dtri~Ei)Ht|JKzEL)vV!r5fxNVa_an4-iZCN*G(7e>v6L~A<^q6Kf;Cm)+
zAB<~Qca0VS1_Wls#QDKDqYwM9+_=cLc>{f`b*B_nkE}h#33q$+j|5HN&Lr#H2JsF0
zooPo5D-)GzLv$LoU63nZ;#}<NJzlpjC#;O*B_Da#pi}PN1^kn%#qzq#xy#x|s5qf&
zb;jJ$A)5W3m)3S@?27($6TGNP=SEjZih-97Z+#Ts65DZF_sr6Cd1qBe<f7HK4J&dj
zV~j&k;jsA%bG}{yq{fkZ_#evl7AKUMuN>mS^Nih-R9-Oqv|15?Fxo|i%3SK1vYp#T
z7a<nJd*<U0<S9$ghz8DaQmWglL*C0#3B@yzl^Z1BJkG`OJUWLB0Dr5~7oW7Z*+OHQ
z$WqniAKd08KHlAqTs>unknIZbr1~~;gcG~><n)PuMJv8Ya_?VPoU-P5^sV+HEzTKq
z%dDz+W<4=~5!hEd%nI{(yV|{}D)W3N0wPcCEs$mH)rbh=*~$x)8Jo_il~uHyAvDLO
z&B?gJyiOI`$5Q7p13Qd9;t5`Q958uItjaoC6<tPeNG|wz<r#b(!;~ekm1I)_ks0h0
zL*1STjsa^JZP@xE_voCR>Ac!DWt#0N92ZVn<&&d_+f!|EIZ{vDnrX1~46u6HTIaq7
z|7<i1>P-Xe+vCRVHmJ;G85#+q>^zb{;BOL-TvHH;II$n=hE(n=0ZZ}ZxG^J6^vvT$
zm4#IZR!yB_F1bd$p;7CcvICzBj5jjC%8;AhBURM>ewyucSuDFkcP$ayw2{r$JcRu&
z2r#d0_V30Q_*4*lv;x~oO9wB__IDAh)l+uw5)-wtr*vV_=mEsgZ}_CTco}+#s+&Kj
zil0aGOO*IS_TD;9CYk!hk(0UjwOjw7Thd=?%`I<X-aJXl*Q*4lV{?ml{4rJ!s<JDb
z&^6A+_MIUhm8S8-rcipR9h6^rs^<2&sLKn^;|AHp$L77&@J7iihcN^Yz6jLpI%HZ=
zss3>QcuR=c(zXEY1w7{*sZ$??)vXSNMFd&q+*3$cMf^QD1ODrzaf|5JtQ-pjjnbhr
zxDnLte~CN?n&Mr2mDM}K+{4@%UE^M;s-9N0mIUz7V%s|WqQ&yEdGVWB3aHtqgOAsk
zLVRdrCu>js)?_Gr1`iyliJUZSb&J1-FVMP`9|4Y7xM#Y`XhpJBeSf{@GRcc@JsD$U
zV>J7unGKo80(o*p&gcv;K`cF(Ec%7cGsbfZ&YS#uVk|2g*Uxjqtikgl$vr9UYqBMb
zPq|j2pH7?*QU9{Zx`!AH)8PAZfhjD+_(4&gaaG9z6qyvIfy%(XsID!&y)AW5Mw=Q{
zxD93+#<h=41}0^V*BNgQ|NfSIKrsD0`3UFqqBS##`;h7R%BxNXm4R^FK(}!WT+<<>
z&M5KcFIGULzR7RgI_GSprWJx9`3Y-AtIGbks+ponH&}%sQ-DI!h%PS)x_*%A$_{c{
z)2V|A6U&hEwafB{l3$#KcgJ=eso5bx>9DF+Ygj%Rj})&a6M&%yzmwkt4}#+Z<Ry89
zhkQ&aaY>w8{F^1e&n|fDY+YPBRxjj`35Mb=D_LpwprpcWOEK#G<b5#f7AY1sVhMu|
zu=13!cFrB?71ycUH~yw=9Iu<D|I3psX9yJ5ee?FWUX6KJl<f=s!lFf<(7Nud%p=j`
z_oOS^P)IAnfV-Gpg5|<B7BKvUrls!iNG*B{8K=l^i`$L~%E#EoVWADNanfyFk(kb|
zwGey;PpnvlwGev@q0<GiW@oukn&qN}5Fo?=%MyN1ZBa&3-TV)4JIuXP!S;$FW%Y@;
zUL-TrTc^pPqUwxvOBkF#+bL2nXhx3K<%43A=0Kcuo8lBQjG;x`!#+kGUL|CA;N4Ic
zA37yP&Xse1CU#DmNpEI0dm7Z6eCv#5tvKK`mtaHhq>8Qu2hBW9Y(kr>Yeax)CJb+3
zhl_YNdQf-WOK&-J)^%z(dHh#Ei}&HSX+v98+{EUfhkkdxh1Tg-+<=YEX7J$WLEQGX
zM)p?$)y@9kECh#vADnL|s=#QdJN~3FPpn3JpbSTjF*!|+|B;qqJ7!(){0NEeevfGE
zYP3-;eLrH^J3$L=H}$ET8oC2I#155Pr|oBeKj!0%=0CwI?pT@CuXj?C)3%5mU1csQ
z-{We%1i!G^$!&)4Z)UAop*5B6MlDQ)PK4rbx|Oi$Snp$AWkPOR>yX)dDfS_$w!G3O
zc!pIh%8u|SYD#3g?ExOLK7ALxlW-QKXlj`SI6o2Yzcg9`V{EF&tyd?BrxwJZX{eFB
zZN&W@OZz%ft7R8Cwsj}DPNC70KFC<bO@8eIUFed?liU=!NZ!dsh`z_s$!@_ePvs5D
z))TbqYo4NnUDD#UP+_cX`3R&2M{V79>W5~^DYdsW5bB;vhN`Z4e!N;Tal}@H!&IE^
znqughtpaCgCS%4xjYsecX#c2h7_{2%=~{MM#H7;kwBIkxUibceWnQ+@zBN@YoU(cv
z1l`6q@Y%KeB|AX@6Zm}7jGhZb+goC0gd7XxmH$kgAxq<EG(g->g0=i-(3j(r>ZsX7
zWQtdeRx(k@l&}bn97u2ycEm>-tE5*s!&;4d+vkx8M?9kThy5>qEnZ8M1hBTLE+wqj
zZ_Kscv3NSf_%=WBIO|Pjiv;DE0Flxw5<-ywEC#rRx2Z}q@<hWidzwZZ(A?sU!acjv
z_T+lVV9uKej;`$sL9~-qxlCcKI|Gy3e?A`{a{C8=mOg&U9i=_dB939Dt%G9EF@=6^
zes;2ldbzj%ANkDZfts+oWae#FcUNz-*)7i^En{Hc$s~ZwlB;5t^yk~DY|D@nUa7p-
zC7Y_`iOqB<XXclLuNoZ|&2q<y-8he+P<Q7ku2xGv*E3g}nN?ZIz-Ox(jXY-)XVI6i
zY?Fq!$cp)tV>idruX@kOIy(ZY%ui@4$-cEQlsBi<SCv(_GMMrZ+Dz&_<~$r1sK$~R
zoY-p1NvM`pwc)&>TasC^S@O)N%wHbuW1~iJh1i65R@<Q?^71qH6jVWmoYNF9yw*tT
zA5TwuJZQyc&KiX?SJcf;I-g|au9fI%QsiH;+<)n9NKhJ%ml7OO^RA3FX^xO&p4p^X
z7iM(;w#Err-mKW~u@nc)VlrZ@Wywts)~6qbGx21~nPqEphZJ?DxkX3i`ZAbVE3$%T
z)%m6|1T_3J8PgnCvO+S?@es^IHG>pYUDc|W&Y~f6B~(oW$}Z&uI8=`p9<01=Ud%0c
z2xWm=RepUFwUDIdc-e3Qnl5G4b#g12c@7@rMH9pmweD4QQXU!}7U4_&`OGMKPg>Of
z=@$<3@mnNfGOOighlymCmRQwaengUp0-&URCwhr^5`M-T27d}~W-sO#L~n~62_j<`
zNePNy6AkX{K%pgi!3eLI`>E9)Cz65gE7U|)SyX8A`ALK%GyheE;UVO7X7eVh${s`8
zx3++Y9qlY?VOW1K=DYZ6*V&D=?0pI5?LB?CA<%vODvB>lHl6#j|J-SB4=}H>WO>{7
zqz&Y;hG95QUWT(&U>ih&&r(ZY8c}BPaQ@RIKRVr=@GpW@vv%&C1W_9P1Zzd^`?5zd
zwrYT?9tJVfIIj+ekO?xjwd^CL6;CHGgf$Vl4z77*IDLX+)rG<=du95v+d3idOy<X|
z>{IM@Nw#;iFQT0fdrB|c*U*~p0h91<eJ`-*Zl_QD=Bf6LO2E`^`n|%X@lNcp`<P$b
z83Cofrh0Xlb6NwHSc#Pcw)lFuOTw#TZnqZ<Y&|m>ByQn!9|&N{ZN;0CB0(jOQ}Y2`
ze;Rcg1&L?P2D|8CET75vgDo0jdFBiRf4eLgxuj#2U|DfP-5WpaF;o|=(qgmFvuo;0
z<~|}=9bFy4?A_{rr?tiG{aC4~@9946E&RFY6{x>!;N{JGN-m5S_}t<QSe>&~6jK2O
z`Ak63GP072q6GEv14WDyyzd@11Pq6my%p_qn#|rHd#4BUqjd(26C(KmNDRxy$-yTw
z(OBOxGF+?uFX#{j2Y1es20KL$zN<QIRda5ZmQJo(a-(?{n^#W>Yp3U|<i1h-5fLs(
zc5e2qw!`ZSq*7A)Zt2nT!t@$l6r%P0pfnl>>lGEW?`74Z+wu>s@%3K(3F`$mziCfh
zSIYlY!1=+3BO>+8c|+6Y)#C;EJ^>HIxlhUos8@8?xF4&tRU_^OC)MV*$0t3V^WW(m
zGJN*~QPa#iD?q#f1WZ1AqxgI!x!o-K3%G_%frz5>(Iu7+2bDKQPg_!pnbAj8;;#IL
zV+N;7m%A;u!gSoRjFGu5f9X|K%o694ZnUuR3f5{GX!h^1>82Il$mK(}%4A=os2DMV
z7y$=?1>}n5ZpI_LDRi>!#+e^6M=1oCK{@@|eE^gfG=gD{o{T<^6_2X9t1;enQF2!X
z(ex*Rl93bck2|B8wZ`P_jT_TDLZ;|rRjGIAG~g$mVA*s!o5xVNTH)2u8d4-T#%33h
zN}gk)M8Q#JzH}nOfi`l{2qJlugVvhd{z(5=#69N7MXIRU=_d`&3NS!?AA;bjK5VTK
zm%8}%w@Y*ve%*U-LZ5u5V8qq8+%b*ALP-C>%U#7cz060V6rr%7h9En*|A?+viMjtI
z)GLR<O?ekn`(5&_BZ=aKskEU8CJR)9#$AllZTo>};mj8_%ufsiurM>!;Vn7)Ez$ry
zWUN6h;GZFKO{d`J5vw4&^~+uL7?B&qP0_(dr{O4SD}0yvB3A&S_Ki|f^6uEu6da)v
zBzsFb*8TVqALv$-xZo&QpAEN_Lm+BmL_lcVPiU>Q0=vK1j@PxWmc5go!m}W)`7-9T
zJ~0L(7en74<chuFFGNa98#PA@4d*6isru#FeAb)Bc4J*FqRvs_!0@D=s(V06A5$@T
zHPi{>?VJgQ;a^-SX!O=PMvpW!GrltBlD%>*+oawJ*!;H0k+>UD;D?!?`F{I+NdDh_
z&(nN3@}UP_KTO#ZcE&db7B^tUV%DSGNi`$2q3{%KuB~A@-YDn;M^kS+aIdJ{cspKj
ze@g0F+;y}4uHciOsx*;sk~M|!4S7+NAn@csfH~S*Vdg6<Q0)HQ@o?iHemtIhZ^(Zy
zylj}wvk*8n{z(x4D4l}S$D0-Y<jreR95B-EdUS>LVU}WdZE7pVl?{}_E-dd0c0F|>
z`@w^6zvY#pZE-Z3P~ee%^!bMmc`BOuK<|mtFOi;poF-`U3Nc7aNR>{RP_9>?86ZD-
z$asS(>NSC?`VgMb)<YmoPJPBbmG%SbZJmLtQ@M>iNB@b6aWbvG+7r=!>-9v9=B*Iv
z<_+1GE;uawi8N{qW7gMq+1H+^>G=K|u+wj8-fT0F^LEughMK2*=b97{7p5qF?m5H}
zDWoeMUH|Di5*V+3kv?PV3uRcSS#9u}OtXHmg`VvbfAQ=^$`=*2h3S=HRqzSEh5rF6
z6fO{~M(UkBYgcmIJDgabN!&W(4}M1^qSQAh-6ZZ<|2{@vTsyO)!pGjh*sB1xfY>4_
zHHL)$)uKL9P*1%HPGx}IX!;qHQ~euGZx(uY<Qj}=HEYhwz_e_nz=J2&za!aibo3eh
zPkpg|$o`hWnjVx-R=qq^sSiI<o99Kr$SJLEIsbo!tmFBI3OkBu?QtX<59f!oMXiW9
z$3PF_T7%{We{eK;>UOqzFE{fI<DPgZSUc(ZZ@ei&Z6-!Xg30qQ?J*~;O2|2Lr<qDw
z-}}@Xk}FcR((jCK8!12RpFfe@D;X(KjLalI5Y%UelE=+m<MGP3gXm=6SUzXzcdu3L
zPCqbn>f57D+)bZ<|8u14^n_Fk?82#xkF2F8Eq2CFNJSZ3gbjS%yJ^cL0zZeyw@Hpa
ztXlW;5l05koUb|eIZq!_`da#(CL8gE|3J!;Vh9VkEhIe#`VV-_wKc4YWErKzrA@-W
zaY_N{oN5R|+mG}ehlC={X#>Ljke`yF{vdO`6O|e~Q)qVZ8+O(C2BCyZ9^YBoG}Yz<
zIShG|&9jr1RhYd|gah`536EVLWV_^34AkE>U$|aZmam9CMAU5~b>0u*jZWYj&R1ZH
z*4_JZv=ft>jt}n9dneK643|R(Kf_DF*Na79ey!oNw(r&Fvq0oDLOP#knt|_^`NNhQ
z!AfcUyV=?PtkfhEw&4e;Sz-$vaD!896wV|tqFJhH@_a;g?+@vEYzcGCnqbExi3*#o
z$!K6O4R1hnlNF}gT>G6`kg9J!>%@gY{^ctuf%`WC<1mZF{5SSQ!zE^pJ-k9~+|q6v
zVAcAe7I=40v?>=owYSv)9Sc0`c$vtLTnQXqqx!HdU>}+LL_W)7ou^~!`!<tAq!Z_y
zpzB{gXdy5w>I#2W2V|ZPGdJEwJs!_adrMyhnZBayo4$X6u@!51BWPCkd>T@9Y`?_N
zXMB4h*8B)rCO-LXS)$+FYH&RaVa`{MZOZA7%vn3TOllGrl_?Db{w>2F(A%jOV*LIi
zQ1ID;();+&{*IU&Qv$yHG_}1@8uk<NqUL5iZH_@vf5GWz5f~A432rw1ezPAGSdwy?
zwf8SS;hA{aQRN+a+p_cL*yIVSr3bex!skvN{F%g4KL8+|{Az!;9i0LRd_mq4FQgM7
za!<iS%O%WTgMEXiTbxm3`C%2!+1Pl5<p0Tc1Kn+kzvFarqw}|>9R^6dgm_@hVd{x}
zy_`j1HrehDFEP$}pAa&SjQgYye8#^*;7yD|5y)T|q5x7Tx9_~3e7WQb&88fcX2TTP
z;Y%)&%JjXQ7f^5ZMSIvNS)Y8R-g%u0bxgc`GGzDbtq<97(iwDKed8F;=nV^Mp!UYl
zB|L^pt}srZupsJnQXirAB$B0X@PZlqH*}8z*y*MFz<Sp@p^NnRa^7FJPtU{lj@@3q
z`OAe*JY;lkvkC(O&dya9{IF$XDu046d+cicck=K4-XBTgp8;}h&)Kl`3|HApYm@^4
zfyI&n!8X`p+{gVPhQw37Z7Kr4y>}U<;Vs{N6N@(@ER)uzS-GkM<Ex$g1^l3cyy?3e
ztKRQ`oW<v!MU5e_KC0sx&N(SJ1s<goH!CsJ3oT~yc_-}|rgH9Bgq}|abgi1DcA^aJ
zLY@7R7m6eA+%Nv<h~}}0g;(}oR}$pMcj4Zvjx6L`amG*D_wt9jzo!R3kBA8Jl6BVS
zP(M9id*}Lv87(-<1U5uRxu5diKw)Aj8~<f`W&{KGMlAuOrw_(6n~22fjN=U0Ltlbp
zp&5TF19re^#+Vcs1ry%ohan5HAGEBn5c$&1+3YO_UoGX97>@i=M|Mgwlf4z2c9Uyf
z$S(LNeE!ee_rMr4b9tP_qw;bpI^Ei<%zBFtpqaf^R?GP9t;#W0-b{m;z|-a@)RkfI
z(!8;P0zt%H7Pp}KM}3VAKt;QzE+IXWB^f}#3`gRpnMw2buIE~IkBNWIvBnliBZj_O
zwe+|>xJ;~;xdQj0C;leZT<5)!F*>-6Kd)nGXfgfhqv1y)3bc0j6t~&LG}0TjVRH!-
z@xsYtDvi+c6}U9Id+SO0&+FE+jp@+PHvDbH%-Kc5J)bsTh!_rsSh~bo!)<Df>o(Vi
z7|*Pq|HQyooiD>k<yej3gOQ)+F-N8m_<fN8W$fj*Fi2(}Gh*RkezkaHU&8_9>N2v=
z<&1}*td`vo3GZNi*Qzp?Ujk_&K%3~RS&!eJ$m-^*WlHOteym*dGZJqCLXEGTvBFhO
z&~fD#$i=S=6(tL-q`w72SS;ztWY_P`;Vk91W{ID`RF`^7)12`?Xr$E|(OY|S*^hie
zKwwaxt(4;BcEaAP9xI9gQQ!Mb?mGOCY8ZgsZWEqQ<^hQwEBBJx649ZfdIy$?nVojC
zU7*?=+-!O!@~*Cva?+b@vNW=ihd}%IId?URlu{b{l88JD`%X1^&kKRXMAxjZ%83Up
zI<F;D+O)9l>niD&z@_-q(NLwwUGzk657p9#PBmBmMcWMo`Ol5itkz!_LypQK9Ov5}
zD+whB;_V;wbnh&EzX8{%U$2T%)K0v>4NSgfCdb6q&W@pB6DCT3d|QcL{rp6wSP%O*
znY37sEma-g%8kslvpE&aYJfJ$?A!08lzL5T1C^kp%jFW~drK`A8AWaMk&IW^)=rV?
zC-c*!o-8r|X(765(s2n}Rf;q#r!6#jcJ<7K3R`6t+yzUsa+`&fX;>Or?d9#nCcAZ#
zvG6P?+0+!tEZU;?66?L6#9Gd)pGJ5knfH;~oS%1}JW$>MBGv>wd`-GI8olI57bu99
z7P~u**ZoXv`o0aJFv7to%|0dj=x`{rvpN0eFDR>dSSx^Yh%JyCQxtpEC`(y3=+P}g
zKPy{<gj`t0uqVkqPj-n!&Kg$kogCt9w5D{5u*upz{Af^39IG~i5}R#R{Ouk<H<JK5
zlAc7kh^wNCp-q>7Uh7e~5`(55T`fC<QNAjp_d8Egrb@az(^0!;GNr_OI_Ut>D$Uhv
zyiR(S$r6AU!fY;GWt=CV-O-F5(&J;ERwA3+Dt$5yS4`#?FfA>u_<Tm7mIm{(h*n!d
z>|H9^-5`wGZiBH7Brm7#b)wg=m{rui*It~Nl9ffC@R3a--YQXKZg%lyo+v4O6qmLo
zq+y<M7CcB|pOAL(UJPcp2B0TB>Lf3v3=hhaPSb`3#ucIZXlK_kQ_jWCmXeKhF;y7v
zSY%hLYkg(6co2xuRgmeX9I>;osDvsoza}MH=ZR)3nMUWd5Xo1C<m4m?P>xVzySbil
zEj63LW6Ux~IYq0^^Tp!)p4M#?OKPnoX17!Qtjw}&N8S|Y!KQw(W|@JkA2m@kBQ9Og
zTQOlirkH7-H8K!aIY&XW$YK}R!W&2;%t8})#!`LM2?1QlyU0*pFy7ay<z)j`ZHh3a
zpR!J7Sf>pw*#t8wF02%|LR3O<ycMcS6!q1Vh^*1{>9|ShU7f>)OP3{=mTEENWzvXW
zoTFA!Xmc~(Q=yvJD%mLAW50ehH0kN~m_ms9Ld-Y1S_>&TW14vaLgebD+MM|=tIkG}
zG91oiHMQ|G==;LS7YP$bd&@qNF=lVV-}(^6KXtWU(yZ`{R7R}Sb()G_j?=eqmhDF^
z@fiCJmQ^|&P_~r0JZV>};*TMCL&9cGyj8W{T=~5JoJ0+_XAqVcpQhKd69#p8+~I1W
zu+8}&cWL!N<+}#LmN#hsi}0cj!HMYj`OTnvPg(L}a=jR4{@|jItyraK!4Ox*_s)>+
z#nl{Jp0GWHQ@Zgk?iYi8eN(_Ur~It-wishTmkN*Qyp?jD>*$bJTTOTU(K994f$xVn
zq{p;(@|UeIYLDn`Vh&i(95|c!?$l-+K=zVvWYG6bG-ipi5=1)Q;s@P#MQ7+C<Zpy(
zP;A}A*TT1-+4Kkf3|f!qO`dE<@dslvrrAXQShfGp>Z_dBo$q)+@=Zb~c1mNMfy&`X
z>(Jw)<7xYNbd{~`;=9+rYPQ{sa>R}&B!>KM7MkjP*VX9|G_r$pZv^S|pPg6swOL7D
zW8LoC>PJ~N{C({}^tgQT0E>HflvqGst1bSQm8^io)}5QO@jWp82L<!i<O3X`ID6^~
zu+^Af-}#DJu-5W`kygrpEBW{JKolTDx)(gkOgUxs8RVt^fG_p~p04AKc9xiGI5Ic>
zZmZ*mmv@{v?$>Vm+YmT~-7lrH!<q0#Kz>1p6tS>6ZvSY`*91L*4aM|RTJwXpdFq^m
z`3+~P#PF|_c3u!Ym$tv*7OMq9diDu;ciI)-$meMODL9|lt@^6T@QRZ93N?NURij<T
z_zexptK54Io%eU`tPoSgb(RSMhVJFX$(D+x#&xX}>KmKUopgpRUBk%@gN}*e9cs8q
zMSv6)m==JiU35uL<Nk#)f<0w%OIou*R&$|1N_%malRqE;?H=?)ox+!B*XTf5SywUs
zg75S|y>)H0n<R~vVWFJYJgc{yu<ucF=WnK*nwDoj%Q>^qX&O<;A@GCGcEo?4bNC5i
z|J%2D?L_8&+kYKQ*C4qCJJXS=q7Dp)bY?02#2-IMaIl-j`z*=$z~u*@PwnD8vM(wk
zmT`^E52{+?u5;z3Hz)$OZwSz3QOYBH9X6xivJXo|v4ba#jGL|m*lU1(sD+;sY+}0s
z&pfUtec?!Q?GJbH-%WU>XW48DfcdJwD^$ahWFyhZqyA#)eN4>wp`6r=0UFEp%iE~N
zequy>4t|R3V4M!h%2F}~6##v`yc&(m*#di?^{3x#Qv%!gI$yiB(s0&xQR?}cRxbjq
z0-8D>10%(UUa`LsxG}?6582x}yDtF;{La7TO3XrEidbAj9AR`JJoSEpro$ZP4_77`
zCzc<bR>AH^8C>iJG<aeTpy=b(rWN&B9K|Pl+`BstXO@Y*AiR}ln&D;7?fB=h`vIU6
z6e=saJq`7Ya?(E5s0R7nCv?-RP~l7<`?3aah+0nwbc~V)QFs_rDo7bWG&H-vAmp<I
ziv#4XzRp{-nZ-u{FfSzYJ^%KI4G({F2Zo{ju*-O>=;*fk2#5dN6!OWr8Y=i6(}P!I
z2s~aAI^!&$;}d*oWEp;O2oNPpb~AjC80bF0+7%v2Tpc`izF)3N=1nv6d;QpKN*>V}
zbe+=?e6x6Jdd^p^t@pfG_X$z-$~K23uRKggpizA>FRBp61wuntpM{syA5P@Yv+FRG
zV`d^E?uUKYKqX$>NK7)7TNt}cr5iD7H`oz2vCPy9Wa!jbv+)kkA-_QcD=wRTL&kqe
z^9jAIoW!_9x?&oJUROub8w!MajwEBB?Iak+e}P=E1@(^8eL`@idIvF@1O!REQ-9;o
z-Sr2D(W=c6FP|oBxM{pII+d@%&Zs_MD(-)SQL*xO|7+!}@Oxrx^hzAf<p`Ra5n7W4
z6wiI*$`9UWsC9)CFrJH4Rirxt#xz%7NM9Vi@}^3A5N|P7Qu%vd@%i9)qU)U!tuxw}
zi@%Dsz6Ek@pGhXpw}xL>tbmR=rL}kr2vXO&<AH?kfpb?qo{Y6#REy&Nr`z$NYsF_l
zng>fQ^Q-aj3%&DZKXE0u?FpUm$APVrU(?_i){DS?lMl=iI_r0MwVjv9@6?F~sQO+q
zexTPa8y`NTzQY$$fYgl$gU-JdyuVN;8$~lUW55XjZeVE5)gz<^+=A}?3l)KZ?Tz>c
z$0r7&++h2u@XD=d%?Hap!rw#KDC9b7j}OI)o!W<){Ga#1&4^_2_Kq+^lDBv*8h^t5
zk2SZ`UTx~fQ$^yYJR045gaEj;?%(F0Nf>I>6hC;nC$6~u)lg!$v|jAizo|LMKP0`}
z2e96NeG$u&Zt?T+;xjrw3SySdnMd={F(14)jCQGcUjODE(aHKlL^FPP*A1=W&1<L(
zyIu!gI#FeHJq=~t1|(neTDW?jS%T+w(m(D^OkLh=r3r~FR!BUbFcZ8JTy`Ux-og9?
zcpT42)()pW60L9iXu3SU;afBUuVuZmHAx?;TVA0=WPDJc?LwFhZsi7CtAW{BEi-+;
zQu^ud_m1j%cHQCkbNYcglsm1ylPT#Z%N=94wC|7v(n}03tr6kMJn&y2vHNr}0}Le_
zCvyNyhCrt$`5uf9q|Fc97+M&XaAlT$y=?8|8VqjU_KE$JC*17jcu{>$vTNK@T&?Gg
zzyZzE*3wxl|NL!3u?|F6AZ@jqq!+oM$fQL|yioIFgo*KZ<869IK|Gmyb{gH@59M&6
z7|xihhU*9`D_8vGBV`}>4k5DN6(o|!XL$3Cvv|lfqVXq3fQa;{=_?q4V5B;-XH{7Q
zgNW^`9%91Ue;Q^K17Fw&ibBlQNzA^W;u@rYSWFsziZx`kTKN_JPh!vQzzF>sn&II5
zg8{f3(>TsM!GyASGXk;bD2S`CT3WwK?Jhp{>wou)<IQ8{iO&cz@pJOBZvw&9_5#Z6
z!ieu+q7~Vu=N6f20+0_r*%@2fR=%>sY}OP$6Xqx%3wmES^XP&UE64Q$p{aA_p@gpD
zn$x+si{*J^0K);D9;hM9IFVVsTdz&`60!8NtLmSuhn@W&_EvMV41MeJ%BQ#6P4Dvt
zulY7F$Sv!I1Id&0K8wtBY6DK$IUI2Kq~>e>={JkP?u(1Tm+@p*cE&oNzipQ*)9t0x
z&@l@jZdt(GJqZsXRDa_`VE!q_93yLX{wIIhBOu{L|L9Fy-#mXf-T6lY!!Mq_6ECcu
z*I&H3g$~Wu7fg}7+%LH=GR*oX*n4CJ;Ne27bp22gfM_jScDrU)usx{Z8~v49@$Sy&
z6nNi$yA1*=9CG#(`lIOWi1721F(lxucaE7|GCWGEDJCku!fp+jjlCV+KOG>Q^6{wJ
z0+xS`Z<07OJIQLos6ezV8wD@wplm!~oOAdBZH>uC9>FwQS_pebUo)qR!uv<5ApQH}
znh^SC)Iq=z#k2kC6#4x3=jtnwoaVeFeK@|{Mb1A=?za!*3VFNLg?Q?m0{onW`*$f$
z$m4(3mq!j|*I4giL%iXn%ke*(fyJlbE}wS}GnCm5dKeL*9B$aU+FlwjQNOynlSe^#
zAgw|?g3mpch><zjI^zc`$=U|7(DN@t7sA>AW)*qC>^NA8>&UCuCDb@cExPmb9N;er
z72DCLJH7M_(Fg$F@L8oXju!HodTX3wHs3XbHxix~V?ro1N(Px&ekZqy+!@NSJ!3&)
zi9&>CyfgAQ7|fS$UgZ=GwnmIyvTFxn=Tu#cV{jZ91PVKU=V(M8vG1|~6%T2f^d7Lo
zmOdj?PB7J^LsOh|!5yLlsxB%$A4xrneWbqbry=A8T0)6k<*uCdhr$}bd#Yc=3+M<Q
z&|9s%VyNk3i61x4RaLVM`(Aq8rFQ$MRXu0_BX2lBnP+l|3abEMQq2}uSE9wqZUeTX
z)<~GheAGtSrn1g}Vj#dPb)7o<*Ec)s<~qhHp>RRoCElWc{GyEBO?H?3y5Jfi?rUup
z()YB+3TKSK#{CjW9p9Je<##d-B7oWs^F%s22KtIX&Auq4qnldT*hCRj(Jh~=s5YZe
zHD=X6B{};Rfo4xWHN8RnC~gS^zo?rR)NLQ~B?ewKlCY;6^KU@arUbpru{3r6If%C9
zzJPqE?ufqTPxB+DY+&xLrl(M!&`JdM(2&~2RBP^E_UQy*&N4?dX_B6x!U(!9`5v9b
zDUWvxFI!jaeZYNZ`CAw+o^;y&mr7q-O9Ui)wp9X{m0>^bdE4+VVo6HdsOU($S4JG5
z=-;1~Oa`L={4QJwKbY5ldDrHVTv`nDUL`~Wo+0qmO_@tNB^>pzHl*RJLMrWqIeRPR
z5G7Z~a~LO>jFHE?Mk;q0V=8j=(eD%ORS&n0cqdxyNIQ%=EVi7cddwr?C?3YB&6l2f
zIEUvJT3vW=lKGq5>?Q>!;<PK3c4I<Yde+hgWOb#0F3~T`%`+^_o%y4cH$>G%U95X0
z3a}eHuJXEY+CK6&YL8nYZ3Vf!sVY9fVo3~8YuU(cQ5pF08kYp%LZ+{aJcVs}_iY~=
zlr!L*$VI&@I&2cRW5x}%r%Q~W>y_-pEP<&EF8g7AjV$`LyNNezu0xh~3mh-qdvF^o
z^`YjD9Xo8+N_Ticwh<e12$WRT+j3r^^$sqIL%-abdd>{Zk$?Km+^1N`oCj^gz9`u3
zkKE~}(8Mi(Q-_FfOG97WTy3LW{tfSqiG$z46A;zEC9y3ZdW@=k5uSaXapT#i%3?Yb
zkUms%f-2KL;7K-~a5#f>zr#e{c!_#?Hui?gW}~|_2YO)0aj#j)W#G&j)o&}3$KGkr
zgtp}OAgB&#thYq*bB<CilK$t*>u&tu=Oi(;aj-u~^I)1&f1a7qig>)x6GzI4y~VB_
z9WKy<M02-pW6*eYN2qb6C4E9Y&K;ot`G)5~z5@#E=K8HE7~#OSPkF_@vl_D`U70)7
zo&7A+JS`VOCCer12^dG%q?BjQ<B%0EzGYuiAlAsd4I)UN-jenVKYw1JpP^$<jX8w$
zH#yplB_-M4k5PV8%|4MD)RmE5S4>x!<U`n0ycRyn7~Ed{OuxBLIOU?e-lQiyh_c5F
zfot#PnCcwk{`(w%MiaXz?JTn?##wS2AfrXX%C4@R%AbYp&DmwG>C#z;7_|dOVFWl{
z+c|;GL-yP6ZX3n_p6C|5hmL)SIGIV&^t|u)MJGH6r}j(jPwlwpPO-!Ezb_XU_Owzn
z2n0wH?rL=<kD){XcH~k!%}xH?_~40}s2rNzSM@}9mkn5k^*RYgzmxmQGFf3AsaeZ*
z{1}*+?1qQF+<3!)dldf-SbJXc4Uf_YM7K0k<UDNVGd}AkOdT-TBo?@P=oP~SlyL>6
zQp0PGONXMI^zc7MvDX7B5Tw6aAXxRzJR+|mchU}^GT$=<J?8aYcZmH=>av|?>A9rT
zjc0dD+GYuc`NPIg|1av^DkzgK*cQZH8h09ZcbCTD!`+<^ch_!cq>;wm-QC^&!`<DX
zai?MU|1%S3?zs`?o`-pvhpMPmSr1vcvvThpv2vAU>oCV*57kN4>jf_qSuyYEn%;VZ
zL!7%w`rWS`JBjF_KbC*D<ZY)&c8q<b92qiRwU{#YRM^5RT@eSD28J;a{uHSuT;*;s
zKFlJ=)@0jNN7&=$5+~S+?xOOb7uDempkYSNNxa~~N_c|R8h3$kTbegz%@^@<1;@==
zo%9Iz$YN?ji4N9z!;<0~x)kh|UgUL+u9}x7a|FOoer}at;*O`QmJp!Hnt^er%55Cc
zT(;1(Q4l66BIyEa$V>gH-;)t-OtV|gdfJGxpQ;x(8x4^Y&nYe#l^K(lsgIdA%PNaU
z^ffwE=OCv@xW-re22iAj(IrbmPa4#mlQegk4@-g~i!<|v+~LmtV*5RV$5lu4MS#K@
ze`qhlg<zk-&!dcwS$KLY@rWn?9B!|D3qQAP391hVX5+>_hB09_`I?CHF+F>#^MUw}
zX^4{XoAW$X+^Mbigd*8y^sw%8F$mL)!wq*EyCM5=g_5||-QYHEeu-M@chBdyO2_A2
zp?rf+pSn<6m(OsSU})qa{C&H=!J$T&hwT*@YuhD3<xp<5$HQgQm;K~2sd7L{rM!Wb
zQS3xmIAT7Zh^b?8ZG74h`LSVpi>_nedusW$n6_)#Ku!LyNBl(GSUYjru#@^Os~r3z
zXeh!ApIX1tjb~?mGosA@Lf&9rd!4hVf?tems`Tfo1J$BzMjSBy26ejFKIv<G`%6t-
z5QZ~Umj8Kyf9*97QLl`BfWw?N(?PwOX(vFlrJ(r&E7mG1SCjA?NvzQD!73VS|C^sE
zo#3Qf9YC10$w*cKSBN+n!S+Gk*>BRl4aILqRNln!H*s3-B=VfAQVS5nVwr*6<Az2`
zs~{5L0@;}kXNJ-C$aB)v(jJeLG9Z#KTBXnNx*LzaiKxh~A>yc8laG|L1MQ6bM_i=|
z#!}o{rAi;NQ$3B##ZS8=@!jd4ouuQ<bq`kS*qT<-dg>u94>;{115m9YiSaFB_kznc
z>9UU|<up74(wot8PxE$He0;?+9$Dx@L4&#ye=T?umV<3%F*Ey|4)g3oM2dgo?ThDO
zQm;b@2CyOu^lk|Zy0k}4irQkpWAxa}Qp0=~Ga-P;KTf_=Uy5b9tN$3By8Qi=g6J#b
zbB^F=z2E%xotTbDUdf+XzjwGHic+<HA)f#cmmcc4lyC`gHxcs0t6Zas?bmUY+rw3x
zx;iWYETOM09Vq%4uBittY{2o>-u-fS(Kna9aw81>k7~((%6R+{Tnn816idm4_#eo4
zu>N;49`)T><V@!Y@!1(fDhX-$aXJaQ=c&nYI=wwTJ+9XD&9%C9?5N`vt4rKT3w6%V
z2~@?0g=zQ$1j~?(tnJXDV2QPCEC^wjO<$huhZ^U8K9{2*-kSQuz(~TVQGy<pQfk8Z
zvir%#`q)ef*!wApMgd6Y1QRKj`0T6`TOe~<<h=&Jqq>-%&bV6_hisuPRqsb^K`sx(
zhit(vNlh-PJ-mX&t&x;?E6HiBhTIl(D<CThZHR*b&=E&M##9#}$jR!6BWC|V1?<$=
ztgR+(Ly7n>@Zz!+$t+h#3m@A>drXWM#Qf6AV8jMG3UEp7=UvNoN$usmWp+vJ<Gpxo
z1sD+)J(jmdn&TbSOmG9+Ec*srU)hpl$xO>hgV=Q0bcJ*g2)vah{tibnLIs?gSU9=>
z9k+pIro226sd8E7l(6LEqZj8er-1e-T<t#sKwuoui!&-!D)UEnYSeWZC;9lW{R0>)
z7Ay=JBn<Xsr!@ohL5kL4=8?)55%!Em7b5ncI5tQ~VMqctD4qMIbgx3RJ#`t2q^%Dx
ztN|rX=Xyg9;Jba_z^Y&>;=aXgNg&ry6o@({xo+Lm7!0x~4H7@zkVT~Tw>~%UKXh~c
zGfR3EV0FHJW{K_R@qZM1Vf#O4iE3I#TC%nEmyEQm^fb-g)a0b8&n)r$uUP^<h^v*0
z3zdS~nsO7t>x&VO^%jIh)>4l?j($I;a;tI^@$)SLJ>EqHE8{i-;U)rIj_vl=GiO#}
zPy{73H!C<xw4;?(Seyfgs@KY_3}|1GZe}!)4C#&q2wYH`Qk(iN*saA}OT!yf+1JOg
z2@;2@XfNTYreMdZii5S_8o_MZeVfhOS6ghznB7HU%CRpYa~FdY6cZCvRfw6~jT=~p
zFc8CFH&t~}b%7-5&q!qyDix{bi6;`LEXEQ$QIquYG)mmZDJ*I+FFGv1Y(Ky`ph`ME
zI4mx3=52)u9e{;xX=@{#PI-gCS3^Q{OTz~LSFY$k!`{*nCq(i;4Lke)S7CPs`~Okc
z#W3W^Et;MRzIE;b+`lAdXdo4;;vMY#P>(ZANZU<HO^a7KPclqM*UC#y&yE{A*vBVC
z|4O)hdg?4<dV^@G#M+=R0)YH4ai298V9|yG1H(Z6AEZ0Se;4-&bwf)+3Eb?|%#F1P
z{aK3Sw0W-U4w4d%6Me4a>D}0JK~%)b&Db2&Z%Y>&dc5oFLmPtf2_iqkF)<PEMb)lk
z1~q$eUS}U9j&-x?MA3qBG_GV`reWPI+=1<*f6v5%LABglAm7{c%h!oVzk1g-{IC3T
zWpPxa<_at2Htyxx^l4e}mhI1Wdd%vIlvmX00q<7_MG#3P57If6-A4T7_L>VeiU!X_
zP!}iFqqu8nzKeC`)s;LyE55Wny4FiBp0KU9dx_whnJsgbHgtSjFax8sf&~Zb&C>R2
zpR4fB-#gD#sMnE#x-ckS8Q4&$7ZdukyK*`ddWGnTwHf&1v0I`v<;B-G^dbY3iOHu)
z%(@V!*4lp5RZtYc))Z6Jy$Al~N3{{A%Wb}?Fgr*L{Lpu%s>@7=aW{)HOZ&`OEO6h8
zoZOTK^3GuzVtx+}I*#~Gt}nkMFc;z~3)w4|S%=s(Y|Zf&?o+EpORH<h=ZBAqHKZV|
zh!_!#Xc!69H|4lL%-}UZAJzsL8o?<IRPmGcn>~+i@ssfcyCoYA<`!mt<Y*pqmwX&Q
z4#|8Y%>eIvZ&Vg<cE28_QzwonbIKxX$m@aa=^s0>GyBWEQbW`zg5FOf`}3yr@m1X=
zl-d4<tpf~r8=0<yk_(WXF26$`b|z6ABZZi3dwcOR^fM+%{J;<DXo~ZpiI|DACn)k3
z6*9(;NR4m`z!t?;A-NQ~U~7$aG(e6QNvf1}J9ZUlwhj243!Xtd_Wky|w@r3~HtuH~
z5+!MSR!JKjJ(Pp@gw{17;&YSmGP4C~`h`37RKmF`R^Ckf*;svqQFP~E8j420PI<^L
z`#ce_sSt#60sht#$qN8UJd~e5fGIhc=rtq!cVu8K3_loNbv2BqLj-Yu8ea_>=7#My
z-Gy}&Sg3RrV5%+du$fB~8AmHc=95sMIy<|8Qq(rfl*CKQ5LRO`p=C1sCR>Vf?YuCb
zf5L#2atP?Y#W?i}A^3zwt>Gz?tQHA;8<T=BKh7%3(@$_jHOJeKe?rXVUf;zXB--t8
zQ4JD!qsK3B@K5sZ)cmvZ*8~{?52pz%rY{Q*P0(a-@8od>O{z_OeH-3}xkXp^3%E{?
z$iXG))B4(pbVH$H0dBjLsui}o%+=I>paLud&56oJgWk7Lp$%8EO6CIYP%(dcBdT^S
z@L^}Q*&-Kj=mg#kM@@(T7E?oukgvA%XB)&D(lW(rU=Is#ff$3G2^~)gm`ed8K7v>z
zRHvsB>Nyq73p1^vmjnimP%|03a}#{s1Mc3i;{Np$#v2dunMH<A=-Z=gPCQ9)ztK6D
z?<~$bR0o|@zcKEY=5P9P^E9hhb1xcu5a#yKqfLZQTi$)ge&FY=utz_jhYC>Yepw}A
zOvV-U+Cum3P8xq*0ldS|zOMA;Y!6#%`!3gxeZ|BQ_TB!e3@n$N)1$0<5H{0pu+ab5
zdzd+Xx~Xn<e~0{ES8=nt;4BuO(ri|r$NzDD=KSvp*1xN`v;Sv)*86`~Q7ygCiTXd*
zbN|~+&Bg-H#=*h{QCPu*;KHJ{!b)@I!w;*sI<womQ%cjz@|R)`JZh8~H-}`&|J3)t
zlv`eENGnKkFfbhT|AF^h|DE^q>98?GONm+Cf4VZs`#@m0NNi!!mVN2m>}0em#^ydA
zcH?Xv^xEy?SfUkxUu0$>_tr~wG3+qBM;R`Y|DNzj>Kue?4(QrDR_T(<VIaFarrIyp
z0}kJh-ak&Nth)l9<}${%UK`}g4IAG&+|H6Jg+K0bgr5o)lE1gv<?7z`YivCZ)NJVs
zvz>Uf-5wb>TL0BP+kHd5^)h4{mlygdnZPmhzgc1KdK#Y+ey+Pe!b57!3b63grEFAH
zzG-~JewO{AF->O3ID!Lx6R#1IT|KTTph|+kwuKoU*yqWB4lQw=f(~8k$$$+#c%6a`
zed5W022FXbMTa1}R|~*QXdF@G%*Up!vlQhrw-&)Dn@%a?C<dqG;Yiq87%mJ(p9u@I
zP&)($+-~}ubRo#w+;(Cixb2TgAly4Y?GOi_$HdJbc&XFk(S<V1VqhT<+-%q(sOurT
z(9e$@>e^wEMSCRC>}!z0lxzQTc+_|rRaU2+-zCC2jOg<$4u;NS*|ajwr2&C(e*J@K
zm2oc{YP2wi_!$BJK0x{J1En;P9y(f72nPy@WQ+#t3C!Up^LY@;nEBXP;d!`?CDNGY
zU5dZ{xzfTMERnGK$C4QK4_5mJo99XAy%!LBVaonKRg%9^mFp^#oji?W-t&Fd_jBIY
ziX!{gB_aP@CjJBDK=bS`=d53#7N@<ie<a;iCg)E+8o(u%!1w^7yX5>}oUp$B6M{7E
z(3~gg7EVzaWutl(JwK6`Q~a&oyQ7^&HR`dAmZNmAd3E&8x`SUP*Vs)6w_HhZx0Cgz
zT|oAAR58&D2q;1t)W<qKRGMZ~tw9~4pDzLIMeiD_PD}a61lUBX2hH}TA{0t?R4K_u
z@X8)UsDaGOL<&rpRD;Q|TvF-KOH;<xWDD552LFMw|C+`BF&F-8#wcz1HmH-Kp1~Y$
zI=>5{Oqh?26TXYv*dmQNTrx=eFOJv+`0M-wDf4(F8JVyKS*@K?X@1ksOW&kZYfnsM
z?9<iZCt6iOrIvA!#59&EsA?yYQAShAR;h`|pVDhPgJKi?^b@h`${a>{3=C<OxO;&)
zb%m>-x5rt1%*`r^Im02?*6V0164XbBQuV|~pV4~#U=<l{g^_j@jC;zMf_ll3weB2a
z<k^E%osMQb>}u@6I*KUkxOeKbLsCJKs!u|ro_f^du!VK(7Q1SM$Ke=h#5{0cEpM=T
zv~sLH0?S&Hl=IxpYgPv!&1YC(N;U^gMud^dgn5!O;u<SGEMBDgFUI}{>lCn`$%^w)
zvFX#zvc{gU#r_^w%^R$ze9ohgWTgE<$!aZ=OY=)vE?itDQYQ}mA6)wnp8eO3MwxnF
z`0o`4rIT0MI0Q#*Vc2KaZ>a;riCvM|cq@a9SJhia@~^F=LhGviE8^;I`k{F9@ub30
zN=gYe-6|W3@!KQ@bwBMoQrS?+srU2FxE%z>(^%q?tEBlp$x%<rw7*%BUYN*n0vbD+
zDSLz}aqDd>cmdXW@w+FfX$5USs<}GPg@S727zWKfMFlf#Oph_?;~=8hTi@9rHk7A}
z6%`jwuK;C9nF$?LF^vO{+7^Vu)`hshzp(yv^lD{;gb9Zcvnk`V5X$&2+B!6{rc81X
z;#MGYE7^o0y(H{^Z4ES0tqhE4g{;Yig?Su7s#LOj2#HoV`i$c^=B(D*KWHM9#}Op7
zBXu+=#!Tkx68^#A|KKF-f9${h!Lt9_sm68@{B(5!QY)lo>vR?C^b`@uEn0sxH1O3y
z+*3SGm_zHDotMX90+8{5<`osKAiKEQ)+IYiVbg#_vx7R^0$1s&rvo41^%UOqt*xZm
zyi39)Ld`;nHGrLtuR^Ew{zulT_WnnlHJZam$SQBgU%9lUoy<)<`9xO@OA<!wa0V6g
zn*fP^z0FwjYU;n07iaHe!BkGW=p(Ye9&Q$4@`_`}8;LjZ90Ojn+jOb|;_jtP;ZIjb
z+5Xo4PZH^$!WRoVA6MgB??sJISLKRAO+s(4i`fC!GU-zTQg*Apr~d1Q<<UZRjB`RC
zv+oTP!0r+8q0chS^y+sIKlAZk-COwePK}G^<4vx<F?y{*MN!gZE!u}pSgSf`zx?^8
z8-lZ2X@XhhR@K_SHZ5D@R&M@Hg|iz#8?yv^6DVSUI5H0A1x=FPeg0Z^FmBIG(5XV>
zQ)-)E7c_QA`&wk1TBkcEN$XQ!8-1T3{4!V8anOBe+LUI?zG7e3Z&p0Y6u&{`19(U~
zXSbU0kB6j!l}AY><l;4Pp5IRyb&AKJ3K*7&SEVYJpG?W%<OMo!?gx%a#LFE%F4D=y
zo=??rj#n`d)SVC3A&;Xlq*tG}*B#K&uDF=B$7V`$;jI{&*~IEgaAB^<nGq<5n?J@x
zi9H~#Fqsv{$bWl)UBNfgj6u%*^L67`>VPqz*F2t{N>jcnRge3`W@aWYUIG}wggx8a
zuH#yNY(LYVS3F1)F@tT&Fr*V%|0KA_NTmg6(6VM-Gx<HJ7vY5+&p@RsY)6f(lU9Fc
zZZ$d^Rfr~sn)E=QshL)Fr&}n#TH&f?RW++wSSZV#C`l)%wp#3}P^hL8>WXZIJ4;_6
zFC7|lNa81|6X*(A2uS-r-<{B<)`{6M-<Hs-)`HnQ?{+I}aQ$&Vr$FoQOi-i+XiTxF
z1(~!BB1NQP*E1aIGSoa(Gc;(K)h?U-8Wf2T#SUT&(L!p(wYcW(XOz^6Z1A*nTG&k+
zbc-Os=4K?;_3dShSi|;U_@lF4zie8&WRgAT9|5T`sNCDVBZs0xSjS`QG_YGT$QrSS
zP0gU7lTr5+K8FotnA8!d;{iHt?gkD@M95($Gx`r>MZ998G9>9%)M#1Nj+rzL;zu-U
z1S|DUjqARnE^1daFB{d)n8XiSL@;9OGB#=3Ha{_+*DPDrj+->AeV?#QXXP@r8(Q(T
zyM|X}$kGw0_q4z6+btZVk7(k$uw7kq$b<8jfldF-m1)bj=30AVJvUL*E{~d?#nqjc
zJYkkj%cW{Jv<9e2wPYW)NSkm?SEOw|v6~w%s+Jc{f9Ha<g;_<a!?V4$u~-H*;F++`
zTck`lrlWC@+gYqs)X@Ki^B)6NG-#XGuUbS+xTh0w)!Q8|gBnwTb0N=Mh_=|P40YPJ
z^#c~A6Kv@RTsC&^S#Dfsc3jK04J&5#D;C8QOz9h3y653!fgeK}>2_Srw)Si5wXHTT
zi-#E#Ug?NjA3YN3a$L!FRm%dkUe>qMhp`ia>91Vf2R|m*)62P*Y^&DRYg(;twde*c
zrU<A5A5@?|Mhw#ZxL)m0mI)hp&0Llb!zPr|O}Rb>oYFD4sO=P1GMaeFqpSOELwQkL
zh~D{c2@m-h9JP*XMvQ;;n}i;5G20rfjvzK^-fZM%iur`1pb!!9@w&Pn?@lk2lF&W1
z-z04rXY^TyvZ9z04e>^Lq-?Xy7{~RCgo-*#8rSrH3C%_6Ck)|_bi_Sj88fcy$4sqe
zSujrPcMZiuVJB+ib@e#jnV!o{Rwey;<>WQ~fN+4)N%X>V=XQKFy=DBp-zyXmMb|tN
zvx(Q%V|k}$&N#W>K2#7zjwp$jAz58bYo}(`II-V4loQ2<Xpy(VV|lw~#_N-5BV#SP
z31;uSd|orS6V|UBYKk&UG|89Y!b6H$*(VdKisD1`z<ch&v+X#&6Vq=TI(H*YpW*bh
zy7Q}FH58}<hr&!~z}Mkyc{Zz#emT2V7%gX*9D$Nd$i%Pb)N<-r2{F3kxBqwEEtCL-
zo5-Gb-MwYcacL*3-!ByQ#)w|P)pP$!E6XMM50jvtciWva-u|qDznbOcY;R${9Ch*}
zlRgDBlb9Y%`<)AYIH5P)od@1F+q8N9I92illdj%kONE;@^%2%$yRGBOY2Ac*`?ylF
zF;l<3WJ_eit2*uh+p2lwxK^?g)1E$63&1sXpKZkt<A6oWIDA|!nTUyBKdF_$Lwl!g
z);w|CT3a$$pXpW~yM@73d%tek{MWcjvK7-Dr3?bQo<$qTIrV~V*t}|7Ah}vUt3@F3
zj;6!W>*QwoAa>j|8OSuH@7{jyv~s3F$m`of5$wnGs*ln_=*qkAynGNguAFSjG;E%T
zrO=Y$%6s6vdJs9To$R6lN9>{ZXACmmaq6IJ95VTq38&y)pQ%;Py>-{Qv2Vn<rH?cD
zjEP&%zHQyP_2Oo%uyWWW`k9y<6Wf#FTI;xWB)?M%Hu^VlrU0Lh>oqRzWvO7Bm+Ry4
z&dgkXvQ(>-L$naFUnjRRN<QX`*WDxTHtV!$%&<{3DY2Mfh!@f$ZY9e%`wi2CNWme`
z$a~x))<M&<VGiDWmJQR8VL-GtalPP>58x{0n03^&4y2i~$GT*iHB1{lM+_90^v<~D
zxpe9+=W%{wn!{r9*1KsrcN*L)8|I8YBjy$$_7X^XReM_9D;j2uUQ>Y}_7KeS6u9?1
zbP^$k4-O&@6+rPJyyiJ|0`0X7!$-#x%XNm6g*peD6ORjOcvjqL9n`LxMh)vkyAhuq
zlxi$**Up&64_ickR#U+yFWblEom$l_FRKLNcEJbF^ZVta+O56TVZrEEV$@A(-n+yJ
zL)rD>c16_eETK-{F+cCuyX%v)ImIUN>=_|b-=UYt+tlORzP-X>`sgO1zgVl!4UZ?=
zb2G*9iok3pA$`B0SHKls4hsm_G6k0%D-=&rq(^z=2lIk-L;stwYZ%kZW}AHgSTg05
z{Z|Oq7v+WUhWF5A9$40AmQBlcm3qj&0SuV}WNQo6`yGzR1h)#^`X0VATxlQIj{s|@
z(6bAL#@{|)m<VC~v+~$pubkFT0NbZfvNMI~eKp49X)A8DFY1SYl~a7#_d*1|Traja
zybR41jsw8tDf?_eAs@eo=Z+gIf@!84zbmRS<EC$+-+zN=Lh!-5!s8NM^&I6+k!KIr
z_06LAWjzVpdmY~DS@#|l3V^^NAz`3FiPSVk(mm7PF@&i76rM94c=ucuk1_=C5sv(r
zp7ico_gz+w!l%@;ZG@Km?A1tkeY2ktxc%&(*B@H<T$YZqru<4TzT1OuKw{I}!5!mG
z(;J!flzhMWz6tIJ@d9&)s6|sf3K;^8go=yiNMWSdv-#Z*+!SI621^1Z0&@sD5;f%u
z3#*Yz&;EB>a8<}6XaMRmEps5G1q=uwg@{E;gU`sI$5$qB-J(Y$SOPo(G6{+SSqr;H
zcw3~W_xk}j{pT3ls%qiY&>6Y)q<#NvnSfY?wS`;8t&#CBtl>3^SoiB``rZ$|2I(%{
z1mO;So`i^n0v(Bn_mz#*%w(h>Mi;yp@(<K{V$F>;-*%Upc2ECzDe!2>E)-AnD~@e#
zC8z!Am_~#tNQ04hLN+!twUOf(R#H>3wqRFOZR&$Bv+M<$(q7V+F+!w%VlTmWsHym5
z-0Zw&vLiF!O~Ho?rJyp9c(5G#wgJJh{T9LX;71UXuzGMUxQ?{j&OM3W%fXk-MzN}3
z*Wp_5SV+u_Msi|$NyWZ}gdicVRIv!`oA;D|Zz8?;wi@aRcY-@hTj28aImWf$l4Gh!
z>Az`&R3K;*)lr#MR&-R@ji3i-lD2>2`e}=_f?3CFCO5JeV@K-qEr||*m`cn>Ze}r3
z5wk+-^6fALgt$Ub2Qc;@35*FQjrfKw2WXX@3}rxFA^NOa9fQun7o_~({`_1=ZpCpH
z-WMI|k2xY`rjSQq7BdJwM_l>pOtNn@k{i=cDpdeQ@e5)2+waf}I4gn~raV9K;V^nq
zO|h!r&k-irw?b`2b5>2$9Jz>*B8?X7Lh(YsQNz~VNfp>cqZU&L&OqVCaOT?A8+ndF
zyrCI+t%#8$O)fMQcMU%#oT1FK5pNFrGm}meib5wy<}G;_P7{4h#L8l#G_V~;iY8`G
zsXinb8IDWD%4DK7a21AzMo#`BewWBmau;#@m6c@Ia-cTsFB&WviezMT%9mO>W0Qe`
zuu(KsvLZ=9L<%u0xrxO<Mak|+E?PU5g%s#l3IQv#iTc247$=$y`JzNc^fI{v3ptmh
zZFtJpS`riEfxNImG&!;)$@9eK60~u04e^SE^yp<q&C%yDKeY7xkgx9%^EsIGR)ylU
z>SJ^t*#XuvUgLfUx@1j~wh_z3wd5wif$J~~v`{jXak3G5A8lAKQ|9*OKI3O)+63-E
zop%g5`j(=yU^0KC29W{O0PJ5+u~Q~!!pGWVyTknB^JHkk)7m7v!#w60=+`dkR%pVN
z+H~P#&hN||{@oo1*>frRvsjqF=`)mgN*!f)8v5{7Vn4<CId(ewpo6Iq6bnzy<TN~U
zuS7OS=mb<u#-9QjehO0!v<2J=ymfS#s}hgvLR)=DsK#6^Yf!bSo>ebQACnKZSXXJO
zbd=hu>VQ&dsh-v@j2M#$mP43mt#FjsDeL1r5kla`EGhx0rxvm4|Ls~o`~V6+;n>~U
zS(Jd(QVVDGSAWPVCK}Kk&~>W6lr-d2&$q}1B!bhqsKb;Yso+(xiJO;<HRBes>6!l=
zlTK8ot5=sSkBptnWmBjmd9hdDyxEGI@=WBV!%(9t1E{1{)M=YXjA<ky(?Q#+trXPB
znKzE%CpOZZsV$Z>sB0Igj5sG^(b21Gl%6ZCl(H$Bca1?L23rRoyVd1MFnl(|)rp%o
zk8vel+?-qMX|(1!s~-qE$se?hA$vut+n26uwC2}InOBbqC3YKikAWu!(S@p^loKlR
zmN^3sTE^h-<!xNA$*BTH5fhVDy78&i6-qNSc=Me#4l>5Puv$pW%bjHo5@mIxs@1bf
z1k}BXZuBH%^$xPe{1Rd5P}PY`1(HI=OgklEkrRL8XL9h_xXv7x&B*7sOQJ?);deHK
zlLrQbO14TmL<!;halTmHE#c;{Dw)O%86o$=M3v&taTM8rrc+8;<xHD~2%{wM`64)z
zEEyKGifZ*tgNJ0J)bYzX7cH?QpkQn`53NCSDb=+irny63qH^*3IYMkwBByanS!GQd
zhH#^5@%K1}Yyi_KCC02WrgcM@QRMhioWHFImp&JNF+;Q$@|Zs+IHS(+xjF2u&li{T
zYc)-?hy0@;@ljwoQLG3TTS}Z{_(F!TqRR23Ik>D_Y-g7%Y9&m|hq$6H@cB91t<Ps$
z3U{1z_kIuQNBQ7CauQn#%=1(_N$%M~qxW_XA7u=AMIqiG!AZiyz`-EEz{4Phz(Ir{
zguoNaQBQm3U90V-4!K0(;4^dX3&u+7it>y4h{B3WiBgLKMR9s9m6gT6uu8Hhu;{UT
zW6_vapO&B2pB9@|Rr+NR=q&0W`c>3WR9G~im!a3Tmr&IF5aAI15D^;=8vz?08?gzY
z3BD<UTNz4Si&~3WUV+7cMU+LEMTSM2MS?|LDbXtMBoHq!HIOo}Fwi{kAP_4sDv*H-
zUIRe`UITFvZV`bCu?g-sq5@nUFYPkvGVL<uGUGCN3q=bJ4+Rec-RCYx1Vsu;3VIj{
z2z>-~1dRjb1N{p13cbKX!b8JD$-~G)evmt72>d=NlG)3*a8fxZRa~tgoLLR62&D);
z1vLfDOcx1feWh|GJFRh4w;jG+ww<=!wjHuvM3Sr<xE9DC*dFK;_!<Zs7#b)w3kJn`
zQZ%Pu9IBw0X~1{Dcj3Bv<~n^+(*28UBtKpOn90Pa?-p{5w18K{t_S=%DV?bkIg3}x
zE(xrd<jK6|!*HcK1<a;asHW5Msk#-N0G3j7*;Ro_leU?>d@gRMEQ&{glM<P7pF6^)
zv-9bd(t0^yvw@TN74^J%^rS(iAK$AR$}!<QZ-I-}QOG1(rZ?dNZ;^}M(a%YlOjW*H
zH|%4EdF=w#>_I*`*Q8U1Iqj<YZ@|Jyy37SWo4e1w98AT6c5%G{FlbUTQ=Tu~t?Afy
zfwwY#H*!)t(}nM-P{Fn8SZ|@V$VI=nQk3-qi4WhE%`^8_<tTB|l-4_wnon^{8m+*q
z=vMEvCnrMGCkO>bAKCtNeYUkywT~#Xo$t~0&zZoqSIMn3u(Fpg=od^OfGx$AWzeXs
zmm}x|h8@WQAH<eor!+(u)C#kK)JAZ}a%?p{p0l3QEc!KnH*7o9H@yqyFVYJE5(};+
z%ZO1;uXhkKOgi$WaagZXkTFa@vLtRK2d+8Gs!?RGR*(lCkY&>-uvaoj9%dX_1Gj=h
z%e-c_goeyBniJ+0>5zbdMa!~g#Hh9xJ*W_79LaycF9;UK1Njf09(N0AbWks4&^U}H
z(mSLkaur?%ClAnZb33qCB1jje8Tk+HKbv%rSdbKqE>aV|E&H;4O}|lbFH_J4%tx#s
zj1Tey-Z?jqk>m7sOs{beIgAvtjx%Vxr58RZ9;O&cfgmHcSbWnUHIm~Q-V`92D!}!$
zy8WwHH3$eZRTzcaQ6WP1r1=Ikz-!>N0<x_Zw9BBzXHj(m#^k-z-y(&m{N!QMky!Dp
z*+2!IGO+Q#sWJiY;geMQfVMPO_7mXjmf81Fk$6$6a=M|95Ne65r3$VUqq-Thicy~U
zYbp!?RT_XT)%J!Pz?y1ZH(*ve$`*e>MGw$O1F)ys-;lBcEYd)1sdj9`W>upC@y}F<
z0PM8SH-1l&po)-ROsU{pF{_(2>#XRIXq9k?$etTE`7R+G@s2$O$V%^E@v^!ZE2tcl
zYN)q#nCaaO8I_OMrD~S{ld{g)3Y-mpr9uS|rwPQVuspBs{~A?|pP<qOQSqxC?QvC}
z9sxHI-#|Y8Q`H8r(3xu;C<tqQH+(eS4>M;ZZhN0M2dO`H-C{QH^Or<NPlf%65+<K}
z&vI<fJ=S?>2wy5eS{z+g7Jw-07`%wk#j(f}@C7zgnai=%+J^RIym3Cfvb^I1#*YNy
zIlF^USvl5G9b4@KM&K`^yk=>ld4{g%T5)s=)uebR19T{vFO;H%6eSr@>Tx;N=w_6n
z?&$jzqK(2(Uwtvu6fMfA**W7<@kasnZ1L%!d4PpA6}#6cV3V7j^Tzt{U~%FXow;V|
zYR{;1eu=Y2qmKI7;@J48{^Qd9*<$HZi-o)U#QM60v3)|kDmintXjBODX_C-Pg$}$%
zB~+(7n<{^5X*_7YG#)D8>C~Yc{*;Q>Ek2`7Mq%Sq0yj>Td9*Zne_qb{l$!TGKI273
z;V(t8Xxtphs8!59hqN<36)$ppMu?0;|EF%L1|q%rrz>iVWyI!E+Uj($3xTJJH`W#h
z;tS=6jP{A?@tL~qk_wd-X}p~E<?7Y3s>QMw6>V<LHb?XCE3W<C^v}o+l-)a(``jIg
zZ&*=r8Im#z`Jb}%adSMQR)mzIaqB$7@Nw!<0SQn6TJPx&fmHAc@leGwfpZkSR&m=L
zGLC5!Jo4xnMequ9P{me(a~!=^Y1<s~j%idp@)#Kw@Cx-%#oB>$482xK+Z=L^X_Pz@
z=oyFb3b#<jUV(E0y;kqYlWWR}WB!dL76<1Q>m3y&{Nv-3m1?cB3#TjUbVq8)^)};d
z781PbWongbwDgv&Pw{2E3+0QGWrmfK_MP6G<%f$UI{EhQ?)LVTItw=Q6{(3!EX>vR
zQ5EafAN9#LzoN?d`fsOFeSr2BLtQRgmp(w-=wWx;A$wVjG?QU?D#c+6(%Rd;7!v4T
zvLz;H6f_M4;bqv@Ur4@`m#N^CkQksFi*hsQl}fnIAe;P%`2yEQ9|-;J@8ol6Ak^>R
zn3}%<AA(0&E+4nTQ%}~ec_WW|g579oae1ZQd8H<Kr4V_gg;r$@mUP#giA`xbr9}`L
zs=pT#_|kM*n#6V#g4?m^1RMMi9_a-tlv1Vma~;)pw1dZVRwaUyF*i$<@D$s1cg%wy
zWf_|4<c?xtEfICk-|MGWR{2=zQB!m}m|7@X1`9Q-F+nhy)+>CvszCCSiuOsm6NYa~
z@)iQ(m+X77F9`L$P=$2{2VUw~n*Ym*lX<NpaEvZC;H6x?+%kmEX+Js<=8_S4(-glW
zio+Af(h=Uyv5MQ;&w@Xdk!5YlHh6`u-pwL8Wn&P)+hANH5OjdYFlFQgGtRD+)q8`K
z1pMD_Q7YQ{6wQ{@gS><BBC5DkO0gpCG?co>0Eu3EA1%Q6F?>|E8$|u0^>vt0NAxfj
z@Km;|{U)XwKSXJ)<p$^hg!5^LQR*5*L@f~`Lo8{L9~<&Lm=OHYBIpS7w&3Y-^E^Mq
zG1*CQsn=a+t|3w9Ea>%psu>Qiz|3k}u%T|l&qPMdoCM+Av3*1$aym)O8e6~{pxV%&
zV8bADf^x)2%-VP$bYO^}d)arXrdg+*zSJ46Bk{rWK?jZx;2(1wBf(*bW<X}bZa@Sw
z4_4-^iQtHops#gndim}<y$~Nby&u$thw7BZ)Q}iU^tD0#RFO}Lz+a$9L5F~MX1S-h
zqdZR_IhqxMf2AZ$BKayoO;I7sLoX_n$|T!Dv7-PlJH3I<B-f%TIpty!@y7Ii!{288
z*^>7A1F6>5nI`8a!s#@PTd7ohL#9QEVm*hHa4wNyfO<$3B6lKQ_w9Mn?%`#A_MH6-
zw7%kPgNTo1?8B^38~9fzhl0PZ-%(~s`2bRuslR6tHpGXIc`|d-Pbf0;)OE%azhPit
zY}D5Nc><k3-AQkzx7Kv{o-d!i#EOYs;CzUy<a=6szwUcFSeFTNbFN(grVO-%yx*Qy
zUuML15cje4Ru53Cj+jxa%ApxM{-pj5Ot<(*Gb~4}+zP1P)Y|s0QR)s5wKtDKR@N0g
zg!t24-bN;yt;Fc^+<d*=`jG!;TM;k01Q1A95UZe`vYKP3Rt&>w6&7OrgspAX|HkCC
zulnUhl-}8n*;bda{NQIl;~E0t+TFy{?GdefCj&}+9>G)W*CJ6&aVtz8c#2?5Wh9)U
z1Z5Pdrkyb~9+%HwLWPP^Z_m>5xZ&G(O7d~JBF;&@HJpP8-Sz6Nbe^>AL|xym=rN;x
zt}R(lrO2y@r0mCU*Q|D6St<PJNt&E(aaS~E>%LLf5CTV>_3L+@OOJRI-e$4rcHvN&
zc9GgzMx>X<;uO$-3U}_Ef{PZPAY~T1xMtbsy0s<R=OenZ$}(;><-{I5G5(}XB5eB+
zG0Cyw8N??_C_%J_+|K~GouQbxWr^yb;CN^?93FDn{m5l;>~e0IJR;%D_P-^7C1xD0
zM_Cxh@aot>_54w{U=u-EoRBc9qH%Lg@ZE<JQP6&rsd3smZ%LdUX0`B(CexpGN?1lz
z55dSAmdkBeg1g0_1ituDGroAYzi0s*DI<Ia_VN_L>1)O|bhqi`{zixP%)dc>#K<l`
z-K}@3;#XRaxYrD3$2!06(3?eI&zG8>sKA#_f58mTJ!N_Ey_3o3m<d-K4<_B0{`!~r
z&&XL}^oImNK+bwNpqRMPxqUCHB^3*)PyyAPf%vzMUH1i+0`qnH=9L-8E{?%k=7@9q
zRh$2AL+2<(eZWuVI7Yr@lXDe<h~L|#MK@-MOEOlNnpE@2Wn8%#@TjQ98H77D671%I
zg<D->A6^O0z_zf96ZhdWN%9|ak)S4`#x`Z<R3iEFkx0gFJS3k9kPPUW1=rdqHgyHs
zgshgjzP3c;l^?mAna(9UoT%Ea{ej)(=<Ge-1$Cm+2H~{abO9;<?edvBJI6y_$j;W$
z7Po9e_;HMa)C*~On}+53R26HXxb%@({XWZdiK&yJ%QwlP<dN`e;$Y$w{fEAI`+F3h
zzG$%i9Mi9Ep#r~Q!2<L92fOIM2K1LtfVcAtx?;=J{bvV(!-1;qd*auta=dCG<&T4p
z|3a;q*Q)H|{v-wQ|4)cP-2WZ5=3ipa@&8Q>I!`wIWOv7D#f^YL6*oB(v{Ulm)_}_V
z{7khO>HGhK>9f7v;j{Z++&O4a)wgg111l}~A25A*{yTU4I$=`?2OGK87i!ylif!fq
zp9*S9TEA-8=UNE{WOd+ozz~e`T8G4c^oH58K|rC9#6a=rDN(I-p?n7mvWtuQr9qOK
z!Ad44EpGZ6pcr+2`SSXfeuUQU*VNY5b~?{{DYFp&;aVZ=9IcANzuQ7SySPa`#eW^v
z4<z&(s142L*o%o*$nqZ<4W$Y1BW@dv0upjnihW$xZ~33L)gzm~`xepNs!KE`p=7-s
zCY>!AaC&0<gY?3`avL_=Te;uNaT`7kmxoMxJB}07SFJu*6n|~^U%bhT{_8sKW|REK
zm!ZlcZ#4bR^r%1E-}Vo&oQKfa+;Eklhx*U8>MfV&N$P+$^Sz2hF+`z_o1wOH7@Uq)
z^F7c(5TekTm+6B>w7jqjXy;ae_imXSMaXLN7j^WCmn%M}4u2G}%X!D%;R8)}GzeXf
zkk4VW_GxaEI?EZVN`dcrAU`@i;*8ab?eJXT_R1?XD$1p44PZ_pV?p=VTtM}CHa3*d
zXO_X-cQLJSKB91}WD!vKsdI_do;0a;dH^uTeCJesn<B=+)})UrxNl;MDXdQ*3D9*K
z8s@b}N~XJa1ulLK9ZwkZSbl<=I715?jj!ki0qJ)60i}{H)COfV`E?$c!roG)4C>Z{
zYhwjJ<q4`6>r-W8IxCkK?&w?}uqAIP5W|}>i<9&!=j*k@pSV<gBYWsqo;VDO=5OPm
zKl^7&CAN$c<Fpmoyo0=X6_Dc!!&N5&ilAkRL$bR1-z;%Lj#AkZC1shv4DG223uxV1
z(809HK&KsO(r4}<w>|x^kEHK+sF%dw8NJ522g5MejDSdZS_5<x8a=UNIb`V2Zw>L!
zf^wrxumx{Jy~*XbdTH7kEK;%xoX?0lmOd<H58$x}Ox5pPvd{{$h9r%Xf^M|txXANk
zu{Dj=EsLu9^7|AUBy|i%POpPOiUu>C!z(Qe{G!Np2t%p9dDoX`iQyW^ib@39Pf5E6
zo}%Mr=cQ_M({Obt@}IrZpS^98<sC^~znWWQud9|k=dP+$B6TN72rNv3H`Sh~B8s*l
zsCc{;4it;=|KdlJePb<%fJJJ@wja1F0O?v^H#1&pEwrp6u*s=*{QTtxVnyYA9FgVP
zqbo;i;-Y)AmeZ@Gk<2mivo$9{sh$~-FS2=;+#p9?nRXu3Ez=6%rvIVC$TUFqyAkV}
zTXm<i=rPIUJsD$P{ZlfFMfjurWI_cb;r+$Hj`b2X<d`#)ZjmXh2l?Kyh>g^CLZd@J
z1s+DbfPOPnCx&VE!unZ`Jxrto=W$y<4E@q-v^}uInr?9maJg2IwiT6?bR5njsy^^b
zE92Zz75#J&o#EBK&5mh?K)GqAtk`g20C%la0*BQJYe<FVn#ZLKdgID8)XP-uo#&Rx
z<S<wuEJ?>`rS|w0o&#PG))y7wX}&FNZ=-w#XPSR;R;(Brc8#zE`Z|%XQmEd3x}zM+
z1xCkHYuF7&>x=@alA}U0E298rM~1FGi9T_|ZI7rhwYaFz2sy}Au~2H72n*KN4SJL3
z+t*ep!doZ|VfPGccL9xk)J)tpqzz^TC3-w|VRwA5RQAXLMShOaZ<@4Gd5y(nOiQ(+
zE*1oH_c>asy+|u0UxFmXPsp)#dY(el9*azTRguGeX560Qn3BAzYSj^I1j3t((J{XU
z7T4{u-27!pxvBU~f?v66kxdJ~dFwf+on)6~G;TxMhN*n%26AxmA~J(k9e_>Km$yzN
z5eUpWb#rq3s~`#VE?%{IYt!@%_elwT>{=Vrab>loFi;;b1?(`%1A+dXk^{|{hw=+S
zcb*ICf4IYL)M49Aizu^}YE_m@r+rMEz5;g%I;&V0i{vC+a~?@I{(x!(2(YfV5H?N7
zfA-{8u&Gy@m_qFwD-6D-(!S8y1Nqc}nzXyTj>-?<i&$d7S2tF`@tILzUDeBsO~bF{
zBP!%=NF)-k_@qtNNuWu6epqc6`p@L=&+g5ECV{zMV$*fZ1=+W50`p2}ca1RmYO1`o
zMCahcN4GPB+xm)ImYExOaf+6EbC#~+hqVug3h{|eU<SPi$GK56y0R_Oi}6I?#tm%G
zzG~-TGPSF!@9pfiH<Zinh{bsG*H3y%i46|DIN@N_V}cSb=u?ITd%qrBXD&mC`lHp$
z3tVX9I5c(kqE2^2A>8YK>ZxvgU+B<rLax#G!Z+BHm#9TyrN5oG&OlfxAddY46|H_t
zj4}Hqa2sN>A|c-eBmRc0$+epHw|Xv&<Fieo*OHY#oPVb*>sATO0k<9K24)i0n+~Ve
z`i4Nj3O~yi?ZNBq_ioB^5ssro1zzq|^4-|N*l`W#oC&O@XzTYqbq|YO9IQ}zH~vb)
zwTT*u8%`nVJ&%_}bA;FFnX8A(KvE0Zx6!mmOj9YfhL!N+zK9DQ)hDE-^hz4xqzSJ4
zmGU;vnm=`a8Qr)v9D{yP4-ol36&jng=B3b!+{%FH+HeK?%mdD5HE8mpX><kr1GT-t
z+5Mr5CqsVVPC|3r-`hTfDHeP{++sq~btlYBKM=_`?BEXfYfTH^^8j3G)kMV$>iMu(
zty)Yk;NV@ew(RVxy&dSotI9R$MbBs%octsEnQQ3l6DCb^MTs&aH~he<KE8}!Fm#Wc
z5TW80h1n~jsfM<*ZbR}U{sBy~{I#X7zp0wl{xkZEw@Y2QQw9|HtwY;XMxR{Z$|Q8T
z3f0uep3y6bGtctlk9U{OV|{gN;v?oQ6=N7j#h9tvAoEu)dUmXb$FdhD$&W!4knr4J
zU*_L43%if^B;H@ocvqC3AUiiHBPXgY$FY7`VRqw{YmH+vAnU3YdXTKTn_{8NJZ0r=
z8qdht_H|CV)A>p@vgw|H9*WUmcTiI5Oi73()roG=Z8y=D_vUN|ryK~W+|9#_&gxMK
z2%2V}0ezhZE2G)>3K@;A#|$J5EA*iX{zV>rd|;ekfXuLAKEAsZ2SGTiP@nDCHXkpE
z=uHzkyq-HXj1Cosyso)(^=ORD4PUcs>`$fxo4vUv$y%B%`a%wZikEerUZ+={Sb--b
zI+OYzm)i!h9}X?tbnl9e%*?e0*u9-BbSu*g=9o2lF_)!R*UAmWbyA37Hi{yGVTWG^
zIBd~QBTT^vKo&}q8t3nrVTdEP&4ptkX^g7AMqIzJe%#FZa`9k)^c~0sBy?6Aq22-E
z;Z|(Fr#M0vT<8LP(%{dd@Hs)$_1;bJ;YMure?57ho_M+F@(hxH1gN#6C109|v0D5U
zz28(H!r5tu;@E;#pg+bt`KF<bV$Y*$M?w08C_OusZ8|_oI^(F%96kT$oV5F+a}o@U
zu!vQ0E@)q$WaUkxGFBoPOo37LCEf)6Hi@>;8P=#6H>khouQPAwz!Q)EZ|IeaG&ZGe
zZl<k@B)gflhK>SxyR#}UZ2O52oBKES31$oWMDT?U<meF`g8O4Z*QlL8@U$I>9FK#5
zYj>qr7ZK@NL{)~XeY=L8=2MQT--d*R3#`lDC_CSMz#Auf5Wx-N2Xb1=&d00|SV&eV
zVPe$k1Ua4>+xF&MkCGv-cNMQ$CQ1vF>qxwQ$jqP_k$zX3R13QEj;pQ46I!fbOVu_!
z5FHbh%?A2E2;5F;)apYm`J`#zNi9@Kyq?W}_X<!Oq;8XI{#6t%!`FM$@1o`qze}CW
z>V#?*OL#4aExF#~5*9eD8&q*40OJb_zHG|DiXKEu%uaNRP?FwGN-7MIz@O6J){?@x
zd*$5!O1!sprlj0>-VQdhrp!(JICsf*X{8}2#x89k5_z~~>i<@y7abE4rmr)s^*p0<
z;CIzm4|q#t4+|yHaTpWKWDbg(1mflu@h*CNck0`S<O>t+is5ba)NJr1j(4MWf1hf#
zDYPKB$t+#mJ4#JVNtDxK47>snt6@3DKK<UCzd^5Yg&x?aP^{f-$d#>lt;HUWRr61~
zCg3mVz`x|Wv1Qh;0CSafw{T}X*%lRo+X%ge8l~D`y-s_s^*qlunjF|3<V?%f1N~HR
z+`&6cma@D{IzW_EB7}Es(7{Q@tGXwUGYC)T%UL$yAhhuW=7(juo{lP%X~9vxF{E68
zjm(>4fdvuph&watLQ}O9MXxGf<OrF9<h-a1P6$;y!i1<t<qrSWk<)y(J1t&hiay%}
z(j>D8wW~ZQYD{dY!sKfp`VtBMl)tY7?`dDQl&o}-M<VwK%dTag5MHyyW@I~EsSD3W
zot@;Uh8Fit+8%HLDZX;RJ?p%&Zb|Bvh0(X^+n?@g53z8uw(njMsfh+K5VQK#gmMG=
z<%lqoNYkA}qj7&qPI<5s3N>-lup-{uQZF!fnQ1ZwUEJ2tp&Z8*;i~1?oL8HMF?(9F
zYgd8T3U&010w2avBq#)9_}8>I3qhxS5tj<G_zdbDU|k~!)HGJHwOhd|PBz!g>p9C%
z$EpxN{w4-}*bK^Og>yS<a`ht5uDe>B!ybA;KQU(YJ&7;r=BmZJeDe9SgIeDMNxRNk
z7GmHA%{>K2@SS9<ZQe^^A=}_%82I31PpST-5k!$diDs1dF{#qIO`JEUDYpcbd-}r)
z59ZM_L_^b*>~Lc0?)YeS9BFa#$w`!(a%wnq_6&SKQyY)Pr$GgeH;#@_bQ|^zOA~-<
z3|+eKKj8W$Bjd9id;qrDz=fX}T=BO@wTJEVd-JcI$PfKt0tt3GU;K1}g9IqLM<O@Z
zol*!z{_AGaXzi8Gg)331-Sp(gY~QqjpM@>n!)4OF0|NMf;l7z^6I={Pxm~s5hbTo6
zyQeERU|nYSP_?HxYA||z;SN+oH5~MfJcVK~G|pcr*HT26YPSB3s*og@7{pSU1j&<a
zv~&GjkJ7Y4<KplgCUQ!*)FoQ8Npk||yZ#{vcIL2SCx+bPZ8>0s*<9pl5}T_f{MkKt
zteNiQiEHl@2Yn)@wRp$R@)W_(H#k9|y!FHO_hX=&;Y0f_St!yJHBog@D#mM?@&htp
z1qU4NagjdgOF+Q)b1iM<j!}b{jbgsVvG9&6>f-W6Z7N=&8{Re^9}0}M@*47-CMwq3
z96G)iL2P7ilCVi7y9gJjMd^~tkWs_ShaB+u#I`5rsMtw#g9XljX(Dk~qRsW*$0H*F
z7v3ZnsX%Tp!S=P7!2--dUMdh|Si9(9n*2<p%9qpS*IlqbPiRZQ9R?!<;BHF-zFO@+
zcr=ZQeftEj2CH+-Cy7@6SpGa*ob_2Jd1us)Bj?zh39fy{t2w9E8a1@Jt2f~yWnp^A
zwbr0p`~$-g#k-2$X_`~_-uHJ`{#$k<q3Ifs2g3{X%5LKx9D5*ld2R3}+|a<%Bswj=
z(*pAY&0!vKG&v(`RG_3Bt@u&r{y2x}%xs*%>_s(y<}ktx{yQ_%y?%WJ?C3l0IAzU8
z7aMqdd3Qe8tb~k!VPXu*@x??X#v%sQ8lUlov-5r@xU@&p#yj{j41>pFF?aWm?##El
zB!ce;T~@+e9)E9op%op#y={VoMMS^-l>2eev}=83D@E2`FC?)<@6-+zP|VYb0XKW`
z5FpmZ@MakE$ALWJ=|hC?o6bivBJ&GuQ3QI$y9gop)KI91W1vfZMbrp|xD~dDk-KnX
zT0*T=dV=K2w|wm}Cb013hjB14b}m1vKUzf`a}Y%nPc4mVR!uh^SmtVVn7HsykmZ|(
z=(CUz1i^0BNg_WVo(Rw9rP+->g+8d<RY=J^R0yoP^zg!Fv19|_MBzl)gnI2JD5$-N
zO=E#{9o=9sWjL3+Uqt{iDbTo8u^2FrJ!jEma6?D9YHoG(+q+eOE{N@DiPJZlID+Zb
z!D`2^t_Csv(|99w!C<yi;Ogkf7+Z!2OaY2cL&-b0EfJs+PtqWo&BV{&8-}cca@R3Y
z%OLxyIJ5thbBALqIavQNd|3E_{Z)lPV!7CBt=zvWnu+_25kG;^b#WZVGll|uUoJlf
z#|Qp~qf4^7fz(-$^6bdUsQgv^>hXypvjk@FYYt47i9f4oa9y>;76g3rFHZc*tnn??
z0ZxiP*aUC>SUxQpLf^4bx6LI*%NV$FlCIwB8YPDBXx@+#DZr^ZL@%Xr`K7zQ?F}I&
zf4Tm@08K!$zXN#z%<vW6sfFLPB&8nRs`=TiBymm5T#1L*qs2y$-KN<-@U-|1p0m+L
zKR^Ms!h$w~QCTkwM;yelDF>MV_#_7NqhquRKreLqq>|0}kxfDalF-&^tKOhD<JJ{u
zS*$m$<17gCYVk0nYU?aa03<~EkxsNMG6&;U{Hh_oWJ6!6wy~RmMrRVtJX0ktuvDp*
zJF4x@1n3gs8sOF_qwe!p4gM2e*+H!(=b1rvAeZXOPoE*&QN{G59ZerJM1VE2Nx)*b
zLl3C+;<pyeSlfi@lm8wMOgHP|YGCE;%atI%3{P`76K`XI3sn#D9<WcO>f!~$Jw4E=
z<*78A6I5{nhn9U8MiYz_MGTn0ptJSL1htl(3hqg6)bI{|c^ILdYTTm@#!=R<dOI^~
ziS7MTGj=eWaYRmpUBEu+4knF(oZX5y-Ns(A%?mu-Qtr*gRk|Y0ns*fQ4i`2WhU87P
zL3XR@q3Ab;5mrz8O|jVSFnTq#=q1g(D?rhys`tJE2-OISi7L&I^<AU#3^NY1py*(i
z)YGfcH|+MS0cw^iji|M~S`T{x^m{Gvv-wM{5ruDK3QJ>z`E4ehr{Z7>2IA?CT@A0o
zYDX?qK=C&y2e;EPC)w!u45-<)9x&+lSY>BszET10?*NtfUl4tl+12!I+!!8+8xw4f
z6r0ujVizdylBF`6WVUTiPy>0LpQ4I)6xUSrpfwfxGd&d>e-||9u_ni1YnI+E_N=<q
zcriWYK2xG>>qg7FNvma8x4-o=(d%M{Pd6*4spc+r5Q7A}m5YOW-C8+U3tDxdh`Aa#
z+t)B%7L1x$B8$n`un(%RE@~<4C`k(+Q2ppRn^-wAxHtpK4ck!cWV+Vsjv!j0FmGQ@
zNYF7auMEZ^EUJ1r(h`5|bgd705W|oWbE+_i)~Y}kqhQlL>M3FoX7y3etwkSq1j(o1
z<<XYH|7=v6s(By`*aOKK{S$P1HUI&(c=`#N{Ki%Iu^OS4i9O9YJpgUU4q#Qs<ikLK
zB&>(~4dZTyH$tnWBQyd9%zM-f@*VN)yc1^Noq(0!&}jdYi5YGWDV~1HC5FCUcP!Ye
zFArD5|KMqlnvS(IqjA-f?=ey7fNcr;HKnQhrT|unOxrx97tg)GQ5s=uI&j-;f=dy0
z=R|D8$W%DYn$y%;w4D`Xmtzl?Z`$GkXpi#CcAJo0iU~iMFib5z$rWGjV%`Z>fYCsX
z0oLMLj0OO?*L5rsU3Wudy$SJgp6QSPZIHZrA5%+)Z;AG^x`3NDnBwmo5^WHw0A^bq
zP?0;sJlJ@bcw%;!3d4inZB?$*tw*rL4lx{{SN0*Sv}v$c>v#r1?b|iGR$i}5LOs6U
zwPtmByD&?w1PLe}wn%hDCqU_MEFo{{RK&&Ij`e=~?}FC!bRp&ZWRFC?A||@7>1#p-
z8Y-{V{b*$c>ef0Pi0MOUq02lB;!@_AH()K3DK$l0X((K}3T^STQLLW-g^SB-?Y|d0
zVS=g6U6x~hJ}QWRK8ksxEiOjE9JW5F2waPiy$Xjeoy4q-c2Mk9LH|Fr74xFN!#-zY
z(8~^(z<it=<X^HKzAF>S2(aBlD*u&+6|rvD#Npckxh*lof)QX<tZ@Fe4V>4dSi>~v
zdy6Ov45uE<*CJ28>|Mk!mx5!jgt<B!@|6e?l*#vDd(Ia@_PW8mKO5pV<6(+AvUzra
zN#1}<+1Rl4jy;>=mGOM7jo2ifTHD5G+t3xj*#ZEtJ(mt$Xak4Q70?|0jeKf@vgLBZ
zXlRKhk8PZ}a_6(&>YXri<<LT>2S^W1woV{4NBE7La1z^{4BYo`5nGh}FT~cFbt!)x
z_Wujm!qz_kTbw6s`fOzsu-V=b`R!&aI|GUV9^}}3jTpG94nWl(;tH~*it9!Ol{F9p
zp!wl@3WV`^!fB0yP|mIlO^am;;fpvWU<UYtiCsp~O-M`N2Ld6~96i<W)C#$~Cb2!r
zh)sJ&#vN<vo)m-YPvfPKcc-b+@0UHz9J0~Kx)>-S*`Q1aNfwZ7^%R&ax`e_24scm}
z!~+>NRDf;g)F2-lbjtvR<JyQUB0nGA=t$^I=p)-tA5^8$Rsgd_b2X@tWy)aEYVFDf
z%J5wnW(%29MIWGr<d=I6FzOoJdqgGH-&dCaq(9Clv&zw0HG5c{oV0-L<j^Akw`u{%
zkhBP?;5YHE;`n&1^rXiRseZDM!3M2HEfbAiCKLff7Rv8pR*kpnpwZBhb4~s%`PZwt
zxs4`F9WV5e?&|-b)YGFhaW2{YeHtJd^S%ekHIba)wR93|#5s2MS|Eb*D>`j<a|g&4
z@>dnF8h)}>2kNwr)Y_!zS>`A2(^6K;-&oBg`6lfd;N5r`ln8DFvXkgAi05ifH5qLk
zlq3-w0=V)u$iWI=!8zt&lElBZ>gghbutP0h0kv@qbp+4}<nB#W$0vDBYb5DuYCU<;
z;FAsMNIMPvA4{G*!s8b+=!HJ5{lo>1!ux1n`|%E{bgYA7jkJIutrv|(QVs&4V<&$R
zhJLmWjAE>M9g7g8FEGcVpRkbY*;(Y8g8V)kdY&Ex(C;V9$v&_swsD)rJX7Z_vh!*&
z?gEy<>>kq5EF+imS_rTjx?K8uEui0JiYx9QTO9wB9pjfE?0|SZj!?;`9Iy?#nkYeX
zV}}9^T6vCHH=yJ7CUgVGo2`ewfSr3g9u2IyjLe)`3A-XDJ|UClcP<w8i$Drzs6Ey?
zlU&8ohdNziQ|+kw>5puzpt*x+CT$wVjTLJPO(2jTQ5@k6f?l;`yc>l7h4B|N3?J~E
zz(jnV9z;Zo!K44h9TJ=o)HV4oix^B5o}o%Y9d;4+x@%Gp=V55tLty^`ipc4W@OGjf
zAmH+DGGdBnv7@flr3FNAUAiKFWJ?;k<bvW1+=l^Dn*wW#_kK|IG2N2#D^20mj>>xg
zZ5Y(oFgR@0AUmIYAREj^-1?s5U2eeeAm2~DHz@uaqW{Zy&+roI=_?&5kV$|sPXqha
zp@y#MY9^tf)7}S*beJzgIjq(h_L}vM0=ueKK-EEn$(T)m*|tfi!)(D4Jj3`GZL~DM
zV8iCR$f>6;_sc~FvEHp(P)JkhBBvR)Ox?Dw(E}J8ia6C(#QPQ3jCux>BE`Pk3fvry
zw!F#A*sCJ%@>c772jT6?%+IRL1Qh1|g!6Shv=!32)hJ+^!q{M9oZAy*gpU)N)|0>w
zt)nza31X->>u9Zl;Y~FpNqkyW#Fj2dz_D&412z>WVkj>ft}#(<WCbSR`?kVgx4oyC
zxyWcHS;NHA&ai!0h4kd)Ara)DQ8QSCwl%f8k%yB2Qeni=ZqBA)@>UTDtZHJF?pjyY
zf^-#0gb#GZI>9BeQ+~tRBTd+hsoi2E00&z6Fkpl9Bp0{stBcKwYdzrv>Gc3rn_>BU
zl{@AcBb!ND!QOejo3KUYSDGJ90l4GRLx~xeTvQOwyOUtg*$|-{Owg?NDnWU<!N9$W
z^+(lfs1^iz`5snte@uG*Ij<~NtT${)sz>0u>$)5hubHypOai18!`*7x)6q;@KjYG)
zE$;1gjq7y~TwOW?#K(Bi*YV=Ns4Cuhrcj1MIDOd`O9Cd20=zm?xaJ`pK|ZU(JoKCa
z7+KhC5ZcVU)8gNDz~0tP!=wX1eAuP%<-;M0ogkawN=K4kViAYINWfmou+hogjW>JD
zq$>iR{5DNs_}1rKT!_V)5h#2Q8|zigj@H0DT@*X!I_F)WS+5yl4A4=gSAzH*@bWDk
zFwGTP{Jg68$=kM!{q_REOBeM>joW(V1%_5ZN#2TC3bW24o8CI2L$;xyi8ulvyKrEM
z1C@I1c!pzrB(E8%#Ah9B2=W31`C}%vfben6-MfSI8J~%LhbtUvg`C5~wPp%)9Rkw<
z&+O<rRY=XrX_!2Z&H6qT?H<cEZtV!tr+rpW(a%Rfzn3_JX_)<SB*e4$uc~8c6?C%E
zW`OYGuwjn5(Z=-z0M9uv5iW8Z>~IjkkYcw$5l=d9PxL(o_?AYk`T9188vq*xNCDWc
z2O1+++3cl*m%K%8H0geuEuCr;))N|&P0o5#I@QkltY_n#rGwJNwp?D#S~~Pi3GJk(
z4Sk+WzPzOpzM)TogtI>UH}I`&{cnfci2vws3;!8IO(UQYGAsZyC^DMh|C1s%eT6a#
z@TW<EQoFy*XGvc}H+WL9eS&le!8k0CHOL6=?b=R{o0PI??<%9RnTnrHVio+zFJ_ed
zHjo1>`Q^KWXUwailqpLsnyK_>=`OMzpqj)}Z&M)N9iIanHVFUK<8MJLL9X7f)+mcF
z`-3`=hU9Mlp7f^|Flh*E(EmpEN6^0nAqDv@s^o(^Tx-VV_Nqc2`uIMIzgF=mKpjZ8
zQGk;4GL`LBzi6XMN;Qgs6l>@{NM?gOPgDJTDS0TY*?J?1$%#O|$=QaOY?w6&^dOl-
zrq%mOGcS+Scz;=30#*Y&OjhXBJL$>-fFL1omF=zfXN8Q%T9aAOqG)OOB1FMHx;;M0
zVB@t2B}k7X-o>tkmfmPlggM6F!UmbQFR(_dI9dbu_9swsmEjsNh}meB!?l4VJsOg3
zp_Fcz?LER4n{l>TC#;b?tU43I)*N*6p3>##(+!F`WAij(iXKDg1z1nlJk|>0JIMQ(
z00O;gxa1Baq*up=<qxZ5au+L<W-HmGC>}7-Y>3}6RwJVLaY~!}5ezaSPs1AqDCFG!
zHB8J5=6?Js3BbwFfkvgF*7CD}9nS(7z#@uopiQwm)J{g&3cwwU1H<%qi-Hr7+R#?S
zi?whUCfi8E)Ptyh3fI#OMOZo$(+w!~yK&230FtC}ew_|}C9HnGTF-CO_}92MSHd(I
z<3X=}d_Sow>vs^aGVF7u5v1=cwnf5cRz;Daxrhwy^zS^#$;<Iw2RLMgQJJgp*9_^Z
ztw&HG`Pr0)QG*N(k`pYsMFP;}iUrB`E^Q%uTTIa>*UA9K=R;h>g7*g0TKPv6Q1g#q
zumQDwV~SZ!cXFizz%Smn0kN!Ewj@bYGcp|?b61Z$x!4sD_Fb8En!bWJ*o|D8yBXqW
z&9MfcohUv8Ko^8p7`=@&yV|ZtVBPRnnqxko201ClVmE`IXL_frweWm1$Vugm=)@gY
z+jcD?$~T~!c7s0M#qpkH#vd?Y5rAj(DyHNX&F0e#a)ppHH3)CP-izM^#=&pmewCtf
zB*v|3M^IdDkQ-&4XoKA6fDAFAgs%JooVG%dX#_`ukzT{IzTb_-F&}UPU|&UGTKRCz
z+etNngtOyeo!o7SuT;>@GOYLWWQJ_f{qimg``pJK6w?eD8VbTChV4VAiSsDI4nQ}O
z)@Uu+4ajUORyh=S{1s$5@J=2S<SmJJ^Th6kW3D&lwK#_SXy==iozz<N5Jd%`y2tlo
zDX%mLCxT_ugP`#b>rnV5B-1QdIO$qo+h-w=zGS;5U}yOtxn*|%+W3*}Jp*h*wCm9M
zR|(Y*!(zSJEeZ8voghCGUhQe8-DE$)AKy1BmojU4txej_lKw06zTRfmLf2v^RpCVs
z`8Oqr<_w5eOk$2XAV45Pv&51d)9F^@qqJrglFK>$D<@QOmTH`-f<!r2om}B2Ikaoh
z?OFhf2s`67!_Ri4&0>&dC<lGTJ6f~8qI-KP<{RCUvsUYCyjv{vJB($@L0KzgLr?a2
zb1IgpS58x_anr;A3@G@S3Lo^aLgwjeGcCSH5%b@>L41k}KC4?nL5ub@!vd}~0o$!d
zdxG+Q;Fhm)KLU6LlBsEwP9y&0kaOQLXLMuGXJVpRg`~ezEtXXVvoIND)k`#;MEQ5;
z(!-a6rBfIP<6UwVVZmB2;88ySHl2cXwt0|!tB=*A%>W|L&`sa<+mvK`Rh&K2^`l2j
zaRXPp5tcZByW~W?*kj%#Pkh8(6Wb|%hO*DzrI=@>@(e`FEJc7MHUqG%8ZPbNca>%q
zcT`EC`03#qK(Hg$;ywPLD%SRtD`rM`s8LtFr^r<g8E_-4>8Cs}#BAM<p$Hcm=5r&^
z$zXey_%}vT(%Fzbm?f<TP4Rk-S#xyow)ttbnI(^6k-nw!Cq3ec+qVN{A?`b)2i*Rw
z%oHkyTR~ETp^^YbS(n?S6tEA2WD~I3uSO9sQ)gQ^Ze53g-qQ!exMEksAsdvvb=vz1
z=T)*`@VXztV7QE;M%Qr)ffuW~&}umB(TB|-ejaSN3&x(jf+_zT`{c}#5NYOWn0$n;
ze(N&ww_)d;Z9=RB*wXVM)gCd)tl3&`W)X(y?W_Cw<-`WH^dlZbpLIaeE-?aVDTmTT
zq6mY`J#6D)72FnLHmjo;Sxh1v?OGcvf-e8wiA5Tuyes`WN{7qhaK6!oay!dBI09&E
z0MK?O*fkR%?9<2ISih7K;3oVL81Busd+xIy`f=80ZJ4+1zBX!WXVdL09eGy)ygfdF
zKwrDb*=>Iw(B8lI#1;SB6Zd&N4F40B%aVU!xh#&@G>VhP1nFi`R{u<8k5q<3N2Plz
z%?OB#Al@vAFeu-kN$GA-s<R~e8pvQA^_O5<!gRYx{|f10N<mkyV&Z$u98dXjjjZ(y
zY`A3<5QVY*4C^mS%9x{P6Cle-vZs`S7QIgi^0Q^c-yxL>P^pZfh-5jLP}Xd!SXizM
zB-w05`ih=se(^l|?$hdkDKbS28-6Ka(xE=!8;Vn24eflDB6eo7E~8nb+pG5YC?y|1
zU>nh7idcUVw*$wG7<*DkS-LB0My=Tj1V*+=vBwuJO7o&59dH%JW1nT_rA+2mPz(_;
z(yamg3DPV;RpLm^t-<Cab<#E@la~z=WY4upWK{LXiS!%Pl3gh_^(@|Hn&HPw3~#Gm
z%PC%!-m29hP|1g9exL^-x$6q(ft)z_T}*DWN+$s8=a7Gt=mb3?F;M!FVM(-X(HjEY
z)U#DOzl^kxPem&EcOks~2yOL_Vw)V&>~?a0;SuRV!{$rdDX!r=ScJ<Ah+0Y)TR|?A
z391Y*-=`|(L{-u}%3<9F!@1m+*8xdsF;NAlFn!xyeU@MBN^fsbF!5|M{8gQh6&e1~
z>1rcSkI?G*AoI)Xc;x}$n=9>Ro~_io{cw{mb1I<jrx@g{A++K6F`!JbKPL&lf|BO^
z?18p>?pI`1>yEY94hp(%!PrZ7A(tLrrdSWaSnQwz-gKChf<+RcFm-Sn$&Eg(Sx6r+
zz3yqMv|hwc7;XNb$v>m)sapOJ=B!4ZR6`fHSx&9o2i**qq`8?m9dBJ$wV<eyH)j$f
z^+RiJpiRSMLen9^xOX#9<wUh!YP#49iW;*k-J^R$m;AUx@umWkHB6@pW^3jlc{~zw
zB`;T;YH4ErUa?^()m`ft@<N)KheivT^bPW<2AR6yj+mv(Z6>Y<Qb@6zjIZd;9LSxd
zxSyXa@WXhP4dzj3==J0bRa}hqy8`R?)ee;+hr;2~=^CxZ`fh6L66mM|TjLHwtMeVP
z2xuQWhs38i&V{|U*r0T;@LGBj<(w{LLAr^Cyq}H-<-k*affPTG;#$>me`*D(E^<iV
zKY1M{fI~!a511_$P#d(!T~;%XM`&#~9d<ixiSirJZ$u(5XK_`<HopKiiJwFt-N42G
z+@!fAZ3IRzQVY^M09v=*RnNOSq=V7|kdp)$@)T9@o@2G>VU^M{rQeHS_aKx0$Cv;+
zHgAw5fKI2l=s6+oy+Z?bbE>JWH?wjNn|Y8$xlFgNMi-cpCUbyzvxj8TUwKec{%N|C
z${#iHJo90Ir}7K|?)!n^Cm-w3F2)W-BTw2XE+sd}B)5fLP?Cf0JC2_|E2H|5xXrvp
z&OeyEesq*ZfRa_(G%>9PPJxW8CEyfE`5#WQ$hJmX^12QS*Dr?*Z+q36#Drw!ng*!}
zsO3}06H6|>rIH!vnAcX(E$t$282<RV%&Y3_-1exgU08KrldQWHmS_2LcRgHLp}lm5
zO3vDlQ08%cGPtl51OEVvE@|_z6)5bHFZ9}!dJg-@<+SWBZ{OELPTn1bEqBL*s^r`P
zlX5W@20{Raup-{DF#&UF0xWU?l&iYL&xR>bJ&r!AubC%%cTwuI3`^e*<U%%H*C^5u
z08Hi%VlYplF>bx!T-a5&R#IRfK3wxGF0QWv8GcJ0%t~l3aSM!@e1vNr0T5i$jHSoB
zehf%}sf_pIauV|ggmsolcFXau1WB~n`PWEj@Qayvkupv8<zhS6HFpDpCg(#EMU+aa
zQ^v|3l^_}FR%BYsK3!m=TBhqk@)5cG7X#(*&uugDG4T}Of}#++*^YG?EZ~PBdC*GC
zbKP12ZIgZ1gO<8YBSwZ<b(<DsuiIE+$`cwgZ1a<9kRLp(m%m-{hHDm4Db7k+xR9#V
zWDGc*DD~)h#<w5swf&bl#S0GL%(A*F533f}B_)V3|JE}eMZyHNc*ADab4=#ej1Kaz
z55s77w<!_Hu^u(WdE-<sF?sF~j2jG^c}jU{)5pT#8l+^1*9?$soveFBTr<Y8V9uvU
z(_};RXO3gK{Xu@-I40h#G0KiI`$j=UknIN<;>Rl#3Mb8~1StuhjkZ|d(_!pki;u=I
zGG_hweA9bK?J4f+1%%7fbiTO_!Yy=U{#rZ!Ipgf%J=d=#2#2esJ4@djdFme9*iLG@
zVfJzvDP2b;5NjsRT0MByn*{cD2Ck#*IiR&0W?h9F9-laC<=`dlU@Cd)X3K(zxoiI&
zgiBig3gMFf1mVi`9}q52N`i2Wv>+w=k5FngA{^IZGiG8(#50x7)-Nf8-l}GE2rd9)
zMd2K!KZmsP-p5Z;Ex<j>sEBg<S3tGqn<YG>l;LJvt^o8Y^A&guH7F?qn9z~bNV63-
zD%k{TQ+38?1sY{ohkCe)@{=}Ez_e`FLe!H`m5%gtRjRR=txyDZ`gv-jD5)4?MH^O<
z6Lx??waru}E9K-ETQ2=&{V7HyQ&0w3P_9*_1FpZM?7X_xpT+OvF%>dFfioy->6go-
zbv00?lmN7=L|N&twMI$MH3C+ZN%WAM2kMQo{Hb0qA5#0mpX!y_igl(sShRrl>hYPX
z^(-aRLz;nv*{Mfos`d)So1;|Iw~S^X-So*{fUGEYyBfd9&@LWO#Z<~^|ElgyVLI&O
zqD6WndjU$~Q=KxQ@`tFy6p{#vBiT=fiDy_kNZGUPN`@TIsfKhb)##9@7a!^l?UKPF
z1TGVlnoMc+*`I(XhX`VR7r)#}!Q|~Mw25~Cy4_&<1wi-r7@1q06wufOjgHhD>pqB3
zLWTKgOME~zp>+$7A_^(_`&7sLCiRj(Eu$O!9JA7^8B^%tuoLR>(=b>iyfk~XMp!<@
zpd@v__>oOYD>+qdL@mzi#ZnzMFa#5zV)R3ZAsCc?x>SdiS<4QG#wRhkoHz=A?b}p-
z`M$j7Fyod3CBWv-i^))k`;*7xed0|r9bRD4cX6v$d4ZXam}oPIHb(+-MN<rDm4}G*
zeLOBaOXahqSI#(_x;o1w=b7~ZF^ctl0K~5V-WYn2eFZa+Q%)pNWG3`cnp9aPuuFlz
z)Qc-^abt&DEgmsJ!H%oPmtZvk9#$yUpV1Xno3%`Ap%yvXPIO78*29l<ay2=|$54GG
z1y-|jMUXTF`2Bhw#Dod3KokK4+#A4Z$FE`Bl>fQA(P0d-ZG^0$ieG6xxcaRn9XAkw
zt~r9>>fZsPMs+HDA@j~s{p3<cz9(!^hTR0C?s^?<lnNem5DefGEV~-rp|wer6D%%-
zV7;9PBx&QZE`QYk60YXEVX31uHtXqFON_Bxsw9bqbxN}(faG?nA>9^HF7<d9aQPdW
zV{B)&DfQ%bO?=;QICSSlJ&G5b{LkH$c~%Y9MuT!0$;1xwYr7olK@sGWnDny+`GY{J
zlYn`N0(J+e{~0G;Sni<i(aPD-mM(7#!06MG@s7}f);g203U=B{hF8=0Q;zs4m^=rQ
zC+n~bS(_ks@hF+vsf8~YLCffpQbiom*<lTOLG~6@)QXD2PZt#mzOOffY|z6_*#}RO
z;EJ-YAUU-P^awKn=wMW#ql}_Jf8k=4?wI$(>LRNqImY6@b156`N3~{=ts(8T6b6^B
z6LO2zV8$}aq<F?t_!`aY)2xNvLAF^Z%%Z>+u&vWXBX2#$#95|deUA!Ot$s|*Yg7&D
zP|k^MUf?$i%EuHx2b@09rM#sc$cw$m&iYx0pY~|9-(x4puf$#31Ib~8wQ_G#+MIc(
zW2nzUZh(=c6I)C*+WvEEH(NXmV79$b#H%&OdZ>pkgZLJ0lzAi`uuqpF7LI^qg~__3
zS|g}f>txd2I?r+VE484Y7$e{#As8xB8|{J}Zq<~^dY^X;gXv~&uZn6XtLM|n#3bq4
zF$}g6dcn5iM~@;?iPDMF>$um^A@;_u%uW$O(Tf&~Y6{=Ur0mZ#O>xRdCR{N7?}f}K
zOmD>?V7CS7{i<yacEijk%F~$B?^kSV6Rg_QW8P5BkCw^;gF`ye<i{P%Tg~}ix->#-
zT9Cj7_9h$RlMYuFJ<)Yg1g;`BIF8H_AZ&aNR%Rig9Po-G!qKqOp)2?X2TH+QKN=kq
zUT>&=u~Mnrrq$AEMvxrul5}9!pFEk7aIH=W{E&5ZMW|h_+_q;^aN};^%Kde3kXehm
zgS6hOh#%s;KWq;Xsxp1MLMlV7fcZz}%^K{;D-4+ec`4%?%pkc6i~hTjil2_spv(Ax
zDoYVwfGPTvN8v6rHXBLneo;^Vi;G|Pc9^i!60)hD^Azi0ioTlT`oN|nmVdpS!oMMl
zu;QOUkabic-Ud_!n^u+sLRh<FSUsA|_)o`Fo@ds@FjlCR>?|xrasaq_&hVi8mMfD{
z{bWUjgopWiOWu|K$Pg<lu7!@v)2L>8Hpbjg@#^LyYNO28jut=GnHZ@#{MVZBfa3l#
zM&WBmS5{yq?(7Z9McAa_A1q<(wm#$m)RnE)e5+li3LHiNY)Hbf<y1?$ah6PRqXN@Y
zfZt{>&j{4bY~#MQOLxA|X}jm1zWI%{D`zZ3%eBC70B##r54GW+Oq-i??a;z)xpZ$n
zKA~MFv2o_VD^tyre+jXc<$owsjsKY;SGGcOt&uM+qG)PTw2M3kLNfa@=m}0pX^${t
zyS*z91CcH`S&;gkGCqZ1+at^g7Gihxhky;0NvuDkQ%G>{K%Tv;bmmtx?P6(9!L*`z
z9|)n(`lEbX>1VbA@T~<gPLQxa%i6#zUdS3zNbWa^gr+hi(w|e#bws5Rvt|05YfeTh
zLSyk|jA~e@CXwnxN#G8rgeA&~x|B6q6!fG-SJ>g+WM)1OQU>n(mF5>RJq?CLgUT=l
zqSkXjNR|$DTmzprS_OzHx}FfWV1cawmfBH;@i|MTtBxFB(19!=m4246Nv!u|m<t$6
z((2`}wEi@Ka^DpuKY4&<;?gk9hfPU5O+YtFpJxMUxR5o|2*}ijqNAY_FbP6g`4N%d
zkC?omUGlo&NKXhwCnK)=fQbMmHA?slNx_p7>0{mHkDCIQ1agD;Y;qKRq&LC@+b@UJ
zV-2Vjzo4SfoglKW17)lQjb2R`0#X(Ph4Ico!gZUf3yR_-$|Ac#=Z{IKlpXL7TKou*
z_D=LN)#4Kn%UQq=`6s#`y-wkvTK;1_XzfJEZ-hpbpSF1jA1un2t&{&Pz%twYVe0@L
z#=`=$UXIYjLqMV_)N&brEK{TiOp=|uzaGRVKtcitDY$_7RfW_<vrEXyw!jQx%CH6$
z2qXo_S?oZCwM%W8V@9E_AfqCdk{m53S`3Tg=ZQ&$E^|BfNLp;@bXNchn2||U#L0Y!
zx8ot#{~Kn1x(oBRSF7<gdb3PIsNU4q0^nkMd<GTXNMNRWYqWq)>#+cg)BmBM<#!oG
zj$@tT26!LlovQkgrpvUUcgeoNhDtu?R#mG!1MZlkH_~{ePPzFUN&zCwgpNq}Zs$h(
zSbFk&SSu9hDFzhi!;Gbk0@T`WRgSVKh6mX3Q>ZA)XUxuoF0Xbt=3*mA32CmZfTb22
zjy0-N66!IQAQ`UNW(OZEDJJLLstKw^NjB?^ER-b*e!hc>5t8|zP;npLYClfS33*wM
zUIWWMWP10jlv*fB@oB0LY!9CBv&9C5wSt_IuUC+p`!+N>1iEw|;H^TYI@ZEFtoDt?
z`xTx#<X>r;L3F<={Q^OJjO7|2#M_cb1^Mgovf24PiWuW6+}FsFVNjg)d`IzKG=f3{
z!oL_WD&^b@9dTZdV@@_2S+*w>Hvq@Yc;%D91$R)i`Yiyu->a4JYAw1N>?O~)S*G(l
zdP#R?&)~LXnJ!+m6@HsV&aX3_pnTd;gwh4RxZkAXhzhFcol#+4x8`S0Q>d2R(9y}S
ztWAdbNs50CHt!t8=`2?2`W&er#c&JHUENig!q_Q7pDinzWVk5#QN>n{w)9zKWH9d>
zld>hzY7wJWm;~`MZudG|wx*Lh`xO@DWy5Nzl|jay^z@n36{{@j7Js(g&io~VZu>G6
z`LUXxOs2pRTh~Lq2iO&R17^*$+|F>GMtybDF%aui{<L3NsVK*3biBh|z03<uKmCag
zn9-ShjZ=Sr0nP=qwZ5s*p&0dE>M!hJ-epXjZ@PXy8Pe$Ql+d^tZ@yV^_`wi0^uW1*
zk#}m|5M=S)*g}0^h2;bZx6^F4y{k`SVmq|jy<1l1@>klIKN+rw9V~}K(if*&;wO0U
zWTz6U%2z5n?8iDeSL>R1z1xw2zTv93hY^IkI*xS?Ou=1fzfSjHfWV@=6L#Qm3=H-i
zztpc#4B%N)d_0_qaihTNt8Ks-cKvYeGRWLHH9XI@F?JC9iYbH-(A6S_H>ePRcm%zF
zWGh~0S5O?I*&KEg8D<pB-Idv$<@L6{D-&PF%bWwFCA!$OSx=Rw??Aw&?_*!2$9A|F
z;~sH~&G>GQ_@;&}vr_T?qN{pMTdj!FWnxd56VICt``K{1b24m@&#`xsyESi>!nOvq
zZ2yJsE(Y*<xYuEEA2r(!ob!??UhcttwtD_{Qg_tT<G}*2GH<k|952_FiuKE10I)s^
zBzyO;W?4Lo?QyY?$w%I$tg{%S(Xk)TkhP(DVo8r{-inu<G!lF=g7fL4AwJc$sz!M%
zM^(h?ZotO?bEAAs*JjH5#Kn-;$I<~uo#k&dm5!a+gbBINQY@3~_s;MO2AbBxW2*J?
z+ooUMXL(OEmbr9{$e+yifgzhcrv^hTfSje6FK=u|>Z`XNlpjFf6l<VSFD8&|O&ArF
zGJWPElh5`fmb_JKlK?chQ=`%>h)htJ_a{i1=P4d|D;XS=>M&N?`O6jSzefal5AT5m
zQ`wfvmGAD8M}hIXhf&^!@{*%I4Efs26_4DW4cBIWtITi1s=S!b+t83w>*j7)Ie6jl
zo;PHM-PQ>kmd{w|BoiN9IpgdddMB=5J#^vl-lJx(2C?_3IjafqZJfOtU@*boW?b^-
z)}uD8o<U94Q}IOGc{sdxBDEYIZ`=T&I6RR=`rE%f!KC<K(D1LL;^IF*!z}wJG|Wr3
zTN@2hwXlSIfNK#2Eepyz2c#Muwr(V`ZuA%A!3*D~!q63j5m(6Evb=qNLni3~DDh6<
zrcOS{W`?a8cz%Fx3m}u6&T=vXBToXBT*L;FbdGcm+S3aRhXTM<2mzf3u_9h%Fbd~j
zg{QKCETLqs7wK@5sSI|;Ci0KS04nd#BZ{LmRR+Kpl*<WOy{|NqXbKxBTEK|_Tac=L
zxfnObQ&>HOq)PA7?7nbHyUzPOvh-i3%~Fj=(<vjs#YqgSD4T-44U0wn8myFa$%h5=
zE1E~hvNL{(5#OGJ$9j4*jG8KEgG|LF<gND<Z$Kpw<gh&q{Wxg%hAKUBbRY<(EBh;q
z=AT2m?`bq6l27CF78TPH(knN~!%`|csWNJEY`dJpPoT14<?N(tgRGG!t}ec3x)q9k
z<99l81FHfqErEkOqxThkh1>>pTw8A&K(l`NRlSkt18N;F<^x*e9r~cDND>W%*suM0
z2`g!i!p>3k&5AgV<loCoDl=;;<RMoD7+rX}&{jgypHH(fn%>O9n#5{Qya#<1qs4oM
z>qn2NdnPHHvDWe~XSQd1SFp~68QK=COLntfk1|zs+D>P@N$RYNRO$4kUmT`5;E#II
zuhqdE6f*1A`sAw&kwtvI6{Huc0LDWAUmxO;8iZ@HO0hfBYYdH6_Cm6EUo-h(cI5+w
zpTyggGEsfB_hQx0zNIzO0z>ug*Zg>hK>_mBPLRe_h^B)dKQS<e7j$TNM7D&yP9NYA
zr*PYcerr>**If=Pe6!L=H3URSzhv5f&?<9)bzXOa>=v7jN!0w*CX8S`rUu#Dl&3aM
zQ!GxSV12f}goz``&5P$GD;1l6$rSrJOl($|6|3mnPk(GQ(qxiVFY(}boaXc;_i18?
zvUv#X4}rf%+BD2`7>w5Fan;X{wZ-KS!#056f4hqJwn;NZ8D&_stOm(9m)`JIRueTk
zK;>~P*h<B<USnb@G|*cn6E{$n8a7!TCSaCxxcG(R_C?3r!Y5#fMQU)j4^tvm08{0w
z6#g2+5|O3SOyds}#ap0Kmwx(NFUY8UKNRKhHei}H7)J3<oCYY4^MF<v_RUlPk^BNx
zoTNGC4@|U33ky8z`r#3J<uX=_KQhp29}08rV~Ab4oIX0j;McaaTlj<A>(T8wk{})g
zR5?MjEl=Q%ZmME9jf3FU@^jT@81K}Ldp<Jw9K~DAjO{#-y6{S?DMNHSsGa0c?Bn<s
z0^^y;gZu&R@J))i8>8=}QisLNdl_a<7d1K!PkNz?iLG>U<1ibP3}(h+CWhHAd$CK+
zELe0^h?;vfF}2IJ)?h?fKSfMqO!VpwMm(utLyuOxS0R`#XO#<7^ILo{j7w(J6KuCo
zwXG%Q<S=P%HoAGG*9(C?@5YdmvFPO-pIIv@SYXZbf^c?K_15TqI)%`ub)bVr^r%T4
z*yqo8fL6;fhpm#<`s>4>-y8>;aJ&Z6$CUPg&6}<1wXN%@(PL<IlTdiI*ZTnW?OYu+
z$5F&deGnn?nP#ej_K6`29l{h4{nCzR*!mN0Jh~Tf;x^q|saPK<plwG7ihN5GpW}gX
zf;DJ;N)No<_R|lPN=;`sR5i;8T1Sy9`dx^PTM2*p*$zBtH7knP4O0yH!{0K4)(9-{
zyDt8CzlIIl7p_#qIB0fX_me*mM(ZO<K2QOhOlAB&qF7&Y@kE-hXDo#uU{GRgdLve%
zbzl|a`l=HYFEiU((A^Bfp)Tf~r}E!v5S7;MBS0ZEnvYcYi4p)t!|g^GZ8Rah#g|;i
zk8-T=da_#%)p^n3b?MfJ;8P-rFT+yVw@Pbqv`vkecSc2Y2&WdU-qxg^eAyLu40HJQ
zJP3E}YpcBCb+hskgN^*UiB?-7+75IS{ujra3cUdd6g`T*iTvOwn6@Wk`ACL|XMG5#
z>81mQ0A(ToZ#LcJKNDC!>vIRB2p9~{r|}qA;o{4q6>(cHGsW=AYp_s%PaP#RruzUH
zg^=aJe<0@g6P;@`2Je0$>4m=ZbS`eC6C4Z@A`UntpeEiaPy&6nYb{XawIDWoceG>8
z>D<P16xwqfm-}?m9Arkkzrha?-V#<Th)bd&5}$i_`jKr^7Qi;!pGHYU%r}3uTXICy
zO^K+mMjRO?^z-wfah|fIsmxE~m?LWnCCj793)?AttXA0si}@rvA$ACc&+uUwvu)Q(
zfc@|<bQ}t7vSLWOlhebnqthk^I?tojUz?pXM*9LxdEgH^naeuT20rFO1nCrXMO%NJ
z>hu*bfWAdlW`V|=DvUd00-rrL4AX4B-A05n&K~yo#5pTxTnG%fj(}lu)cV!rGrVE;
z${7oR>}CNN4%<3$!}6gE0qKApPXWMN3w$_i>rrjAIB_<CzvQSn%VjUpS-AD6wfM0^
zym7rmixbzkPg8kxYlThwf9f&*2?)ED&+Ae29{^#P1A%dQq+OWiga`+y=CD79n3b3i
zEI?=s2n}d%j?!Pma}?}EKPBC~f=Vf<Km(>f0NEU6kF3RpHdB==5FYRh_ludzq)?uU
zg5R1sKo<8ifSC+n1o)25p1>0GKFYv~C$j+ppycZFUStzeD8M>&WF5UVLueH~A(JPv
zfK<*``Fa&H$9{>`@uz`=c1w)w2}QV4W*8EgJQH?bvV_%1izgefu=i*lX>jUeC1nk6
zmYd9}wKfxoTPYPgojyleuZFh;9PZY=$JEaFce=PqH~csP7oJ4EkwbLetyd1wNeWSz
ztqyD%6dP3TmpAH-BEAf|prr;t3-*LG83gXUj7g7Oaxb(xh9^#hFe8+@2<9B?gAl7{
ztFW`DGNlH9fDPz}hwDLcg~>cTHQQ<c43M+o3Z{6s=zNP(iPgRkR`r#1&=8uIV>RrP
zlj$5zmC&+y6;^T4qs!|$6-R^%EZc_7_aPk)@Mkk97Bk0M#2O_G-+G4k%XPzB#H_Cv
zjWT=O@*Y*K$rK{%%N8>6fk9)2>KtS;mnKLkKbsC#D&GSHFJ~KV3a~ieY<S0O*4<hF
zTNaCZ4xsHTCV7pZm?do>j+dBzTD9?>{CKoWu(cOhB>i1$A0x;KGEBp2fbjsHtjk%O
zgjJM>Df$Ib?!(sHk5c&eHn5M~dj6VOPXR5+Bq;Ip`~pj+n2N(8uEuDMJQ^~o2-J@f
z3kudx&sQ5c;4u)HDK4}eit)}@<?%XvHf5ppf!B!F>t$?-Lv@??*mU%d(KcBYnUyA%
zKsD+i<}dm^*n2O7&hslmh{mY|8)2@jCt+HPR?)F<*;MA;tc$NWj&&0qsJ4atUci0$
zSDDIcX(tg+khwYx%TFE)>vB3}?KMEV4-CG<1d6vt(@|t0Vcb;)|45JqTM4@&1i0~z
z=VG(rTD!UJYw;0O`+fp!Z@tk85(n5x@8y27z`|53=A(3A*$_-|=o-IFt;!HAk4SBV
z=?1l4M0Cc9ZJ7d%x@`sak~Q0?ELE%p9iVKWpw=nfiuFs1GtnxBz)hZXsE^j@rLhZ|
zVM&VhB?iU`ExU$INyp!*hVXk7{+%vt<68Er30k<?5sMiaowhle<DExqRBR<vC#l?T
zu_<X-&^e@H+*&%U2csAZ1dtu7`yr+t*mOHBhzOno%LdT`l_KbL=E{kh*RA{I;~kZ8
znvJbNywvL{esVI21k3GhMiFR10-SB#LR{Ggt5MvH8Qi{`W=4f=QA=qiqAzx0(QelH
zg%-&m;Ntu6f<NnpVag#moFM))He?7;{7|=RJq{V*lWjoWvRid*Q0(CrMku}wjPloB
zJl!0u;my7cSRBo@ee6pzAda+y2qZB9s0h2XkEGIEAfl`3u#-QVXa{){?T(c@wg)P(
z=2{~$8(?UK)|KyXfS;_5;DYQH@*qFmA#S%_zu4UjvI_uE949D`gigmU0)A`TagVER
zt=Or8TW+JAz*UOaqI6jA;EwTl!9QpU|FvDYg8BK|X8R_9FTQfPcbo3#`x`-aNe5vA
z$WVt*>IkxLVzg&sgYIl~SU2F$3-LBni49H#Vh?>hhPoZyLpG7^CpBH1(5HHDn*0<E
zz}nA#Ym4QbOkCOpx%H-L_t8N%SB_A`^@d`7jZlj<c2qOUmjl{`-vq#fa(YxHvtF$2
zX5t5?0?9_+^kjEaQqET~Atr5>8?o9kP2GE)7|t=aZJpb_jq(GNS;K8IpR;?kAilY-
zV5<0m<hk~tqk<TCq+Y(rM7__oF6$bMlGefwamX+x_5er>_l68y*ulILHE_IRtQrn$
zpIoDJU~**ZR7j;xJ&=_5X_1`DA_lGrQhCzh64Uy-?gg5)w0cmK#%#vcpdRgqUe}QX
z!KQ+iOJBnb970aQ8+Eal!b~h+I<r#|uztki5fIFW=s`NBPZ7^`FbGh`FURvBCU&3a
zLAd!2{~}eraJdEi0_qKDvmpIRyCxo^@ioIpCQrk=UggR)78i_kIID`DVX8P4H-4Zv
z)@vj1W=}$9TLEPB*G1)2Sk`mxAU(UHc%M1qtcuHTv&D78d&OstYyE(l&md>*{ThG0
z22+<GEM&Hr`1M!?y*hkcPmtb5M&9nGADu$5+<qBab5kdnA;>>8Y<`3tB)els7vizk
z2Cnr{0ES-#91m^y=4jwQ>j^6;whWh2BmALD(S~EZpcscSAByQYY?x!6Z#9Z?x>*lT
z=0dG@^d`q^v*}3R)_iAMO|5j<tsFdicrtfg+x13>ZrE130y2|6zIGG(v4Mac;2K@^
zP;7Uhyn+09vKPwmTPe}rtTqG$5^Q6*8JB$4J8|8I|L!vs`M(5wv;XAO3;&7XqLLhS
z*d)yevVu>b;s(0hHEaRdNFYMR29sj8+a^G!be1BM>Lk+JtV~D<>Ji`qpzALHG$237
z<|4#35CN}1!7Gq=6Uuz0gjb^|aS#8T2?fP%A=d{Y^wF74Q$?g2efhRvZcZM7NX7o$
zL??z#dxh+-Al#r*vEyJIQ9@dayhs#rYYj3w*Xu<#P{vRx&>}BFE6!Izgxnm+3WA~e
ze5p~Pb0^0UC<-8ppJU6B;9EuFvOCCL0!Yb+SR-yN1^_%k>rW%-*b}r`40eid)Fsjn
zS}#LbL9X0LiK;K77j!+DPAR6!R|1LFU0UTmWkQz14E%-OP8KYHfXGvg7U`WDoyX&s
zmy*lHoMJZP_~$y|<#j*+4VcQ9f!5bRiK5x^oL|@ta%#%DRxcn7(s`O=h1Oskwtx*5
zF<K}niI-U;l(XNd`xrP0#QYUT(}q>rWXR0P@KHK-?0id}YY~*^s}7V=JsG26x=ATl
zUIB)no6e&H3=C#<(Pr->WSs#FCKsTUMD*4yqY=`nCSTY4!|)#Zf`|1Y3tHs<yU_+d
z<I*Aq(gXC7;?FPck<s>b7<emMuhydzG!a^^b%@=J>AoO+2?7|V^kx!VtBgh~S%t@H
zRFuMCsrD5Bu3w=lz%2^LAXdTLe3g-7*4s|=^Jyj|HyxvVA{ksjOw(MKBzz+*9#zSK
zA-Z&@R=jEk;c7#kEEd2(bo}@NN*ILPzy$Jc8{*Xlc8;zgnANTGml+wMaoW+0iRSe}
zb3G=HtlDajtB&BOBBf^feKUw)?TNXNnir_9c`X1|K7n$=VioXFtz6DzMRJrsLH;bL
zc_?nzSO7eglS}0lrdXS(=(bi~;kIe@XpJsT(mKpFKzRib!hY`OSD13Wz?Q|wfY`s+
zplP@RxM5vnq20#@^G?=y)<H-qt^MQH5&}9L2j2OTD;r^&Fr%=#{pfY{^f;w@Vo>-8
zrGsCJ2gy-zq)tt0MXEOA<WoHD9mvxIr~u<OI^f2W-)<1kVIe;kXfY9r>VAU!_mgD-
z>m(%#OnUJX!>uK+vt}$Q%RzaajE(0k*`!{+&ZJ8)eAjHG@uCiD!6#~!2Q^Z)ak>m}
zb{#DIWv<QlgysE&+7py__c&w*iN#%z9G@9Md8du1u7!I1GF3{P%Y$T+<yfa`k`N@9
zfUP&VycTm~Fiy*tm~=dh(OmOWqZwyYI_Tu`Ao?PuQO?KuY^SlxFNNq7E--nJjR2sX
zuZlNK*ZLUk<HL03eQfYn#ru(23$F($0bCJZFcp5i0m%XG35XKJGy%JtwpCi&+HB`3
z{43JwP9F3G`dNGz=x%Wh`^MhQ6yL!}$MgZO9i~GLE@t9fiRu(QOd8Z<U;sU3Dc&k@
z;Y~ngE2}V7niWBM9D<B@iXa~bIQ2^^#)TN(02Vw{4|1#?L5(lFE|0AsCu^c~ouF7{
zIDRzLC3fv#uo19q<Wqx~`@oR$T<BHT8pnftO)qBpFc_J8xVE)=I3{gcP4QkadDKmj
zXyZ9-l<ZMU9;j!nS1W<W_o@cjJObfAa{0Y&xzMtL3`mN!Zh0=n(fKcWgRDhmU@#k}
z)*J|sAit}Rc|s@C%zE7&jPtOf*RwhoJAy&u-*Q>{T@NAjzg3~oZSt-W6wjNivexpG
zs|@c<)i3q{iabV>N9;j6(6(LXNJ>3(^|qJjbDcaAia5Bgh)G!ZXnseVa95wgUl>7n
z>uPY;)xcw5HN|?DoP_)`Hz@0)C_)m18#|#V&$NPaM><fvOA#Fa?I9gUeUpdTt81d@
za{c%!qfLlBm=%B3$(Q)szM!0i?eh+0w^0=3m)KM5{a(K~NrNo5HaT<&LvfZPZuFd5
zI>eeui|$x0m<8>$LaD)8*+#n;`!4^Z<(M%Y!8L|bo-V>@_A+2Yjp88P$r&UQn6KjW
z!E6!Zw00iJOUyeTR_Sa>fmN#U`O+7-Yb{7l8r3Vp4yP|1V&V!@N+}H2v2<fF30v!Y
zaW~~hJH1XVy3%YW;fdhWC8HI-#;ZIA8y=?hiKCc!yi0+N4xPlW#UjlElipjwd=it*
z>k~_>P-4I5iuHR6V_T4jak0u#yi>7%Pu6VSJ2ohOPp3`Q6!9c%?ruP|)eiG?!_N-a
zD-hftqSuqV+b$KbtHaISpy&k@zp%@Xdv(l{@ofvREiQ`ru7qvmWlQMG5fWR;JiQwL
z)W~;6el;iFNPe|pTPLo+Rvt6Fo?^1Va;pdFYR+l^ye*W&cgk?wC$Sv4><G5azWz&q
zZFEZUS)V0G&0g7d(*3*Qi|}8HFM>GwPsJB`@=xFv7eGaGlu=X|n~-mVPOCd82}?Ge
z0;~kHWVtK=M1#smNqKh`jA@%>p~m}4pN!B>Ve&f>GWZD-oIqbaA_OsqszbqDFlh-A
z#`Hl6ARE7UnbG#8C~1^ztNxIoS-;$b+ghsk5|jD76X>A4Nf{ezsRIdRl)uQpgEP^X
z&Re5znPus_3g->A5he`^A*rISb@&p$Tnb(M0wf=OrlOb(RFiGPg=7v*NXeI&Z>F!G
zUB>u}%sW`G!F+GVCHajgPKJ@%6mzSG=c)r~OhK(cv&~I_jwKzZ&{C~SWe#mIiVj^N
z?@a;(x6G*gR_jZDu8Wz5^tQL1RB;9J;Zhp)AZzCFXw9z0O|=n+cWUCVI`#<p`>mPs
zXpMUEoY`Mw%M|Z2SflR~_5ef%m;1TUv7zieje5}}^gwZLyAP;U22tZlW-}``D>fI{
z{|8g_jeyib6H7=N7bNE0sRj9+bOz=;Z7@l4ic1fg9IF(n)=pNCyhNu;tW*6|dIxL8
z*K{;1Ltvjxiuk+k@)yk@JpsGu9{j6wm*7Ks7x2o<bSm2iBnrt|yo(N10xnRY5W<&n
zi%Lj{aN$Y?BB_?{Wi%7nD!+^SVXv%OSp{K`rmrySi^`$ASmix>Bh4ODDGW&wTJnq$
z%GzK_ITh_~U~aeLl83x5ozHYIkNi+FMJa)2j7Cmla%pDM%pd~|N`|=FwO|Y2n&MxI
zr5V6Dd<_)a5roH~xDs+{CLb9=eqyDY53x!F&1V?oB_-QljN|w-MF!}4Z@lB>F4?od
zm;v+}-YNqi+0AsU9(|7j7IoYaWH)0)TB{VN4}eA-LDgo!CLbFBbbYXs$t_dQu5p0L
z9P*+i#o=fI)0J}EEO0@NS*m9c=Zh#jDE%@}m1NVTCt|R$VH<gTjZsggP~gw?b71-$
z%Sn6!3)$(*Agg!7D&^#>+25hiU_AqHFEQviqCmspIz@tl0{;{HVdaujv>=;CPOa%o
zJWz4?zDD~Rn<EG>#4Y6)Oz~4k+GRdqS>^YdU%tkIjA$VNe8n5DS#%p2;dEx}1KrXc
z`(-!C09D9c5}+XzCg4?!RTc%P$9Oh*$b$HEa!F#`x7)7us2Yr&<8!A`^X6mQwr$%J
z+xEno*vZ7UZQHh;Z*1Gv&Tj3S-)_}Y&zt)X=u@ZrK7DmxpP}^SD@@!a7`gzc6cLgB
zIZQL)u4(3@aSb4C_`z9t=m=AqQB2+A;uNmJ{a0+az*l0Z*V9mjn-d`;Uv&|Mox<ks
z{715eBYkrb&LT?ba~0ntd@?MmHF$tu=PtS+0Pa@>=Hbog=R>_V=`YuOd*|CoYqV5v
z^g&Os*@;@G>h7qCHyVR=Y|}Ka8+KMI16KU+fcY^U-g$3Dnq=E-*hO0sv>h;my%lbz
zpVuvI#esL!lg0EfNh9TgW~;(rvHVZAhcgDAA$xB#CfjBSXC8kPg(OHswnYrP!h554
zrs!s`lfx$T?phbE!82T1fQgI#1}EF`Rna}Ab&`R(h)_{=@17FmGmSBfbM*x{R4tIh
z3=7)#qW--IhIKUu{Bgmn6c>Qh<;!h%P{jvASg5{!$BJmaC+0lM4<Pd`YXfs93tt(F
z{VUi`N9N3*d9}on5un}jCnNU4|M@}<LNHH*fdR<Hy}sNX^AGqufdnra;w_oLmD(%O
z5m?5oNQDjF%uo&jH3Vi0nI|!)ZIqp#5ih_Cfjv%LeH|QlU+R!Ns2t#m^Vt1IUNVjV
zTG#PM%Hfb7UBomqAh<LR)F)z`b^*QzBN2U>_UhCa)wQp#Cuo7nqLc9yh44w)2DFau
z)0inuVmP<s{OpEJ5Ddg4ncIunnC@EVf_7gro5`0zQh~Xpy=@ru`J44YJ?KatG?fs%
z|I;Fs@t<oMQ{bd1DcbyMjFw-4p$dK1G_LG{d-H*w<UE8wRo5=uyZ&G+n{@bUktQUf
z=q#*c5I&`4*Xcmrv=5%_Xe24xji;#Wg-eyQ<o>%kmUeBwzzBkN#VxJe>5~i@W>vld
z*R5k0@?Qf-Fz%Wh+~`&w9fIj#J3*v^4QK1$plNkew=g07QcxL9(5c|E{>Q1Io`)lh
zwa5*9O5S>@Mm(JuI0%Pp2l!LG2oR%So6kCOC_o^i)CC2tTT^v+)FNm}&S)!9r@#D0
zN_e|MwGiLY{d*cEA;ii@Aum3ed3+{jUr_d2P14+~h8g{F&2a!9bF@bvr)8~>R;RZF
zot%OLpr$ehHol`RpY1CCGv=+mhwRqPk7?DE3L7b$7QYlWhTvC}_cu3x5)mM)T$e73
zS;?|=xZTN-2%DP-1+6GD{JBCm?GK|8b6U7pq!IOazwGw$gj#+mwcl&wY*pWK2a1mb
zfpu_Bz@{Nc{0qOSJ}v&S9`<9cC_<>Z(&vSf8V|Yq=VifJeq$%=&vEukM+9L^3dy*c
z{hKndNLWvT5cmZ!hC_1+(dePICd{G`>;=lXk=-#_e~2Pk>{A0r0#)le&IF1z^^b@V
z4%oH9NE$j%Jt?~f({iGWCIjBg>sax0jG=kZo>8;5q@Z<#L=lRh5bu8tX8R2~Bq3(-
zS4+&XCv}cSt9AeeXH8Oo*<(ak{H2kX{c-=~X3`{b{L6w}^}U2pZ))*8ovzP?I#70P
z3~dyCKu$VZ;>rkZI9nTBP`V<_8h8<}s+B)V#b56DH`qN|eUkkB4%tI8S<>*ulZ8iK
z{NXG{qd7+6gMHJn#i%ulJ`7o$96L276e6#1<G398o&>*+jdDx|=g{fl0kMNUhbc#-
zP1xGA`7f+wf&xe-S^1)eY6*2ZBeceUhAyqaPezXm88m|a93Mjc&w*Kg0bIWxMA(5a
zO6K;8;-=k~3cv;_xF7Ob753{6puq`aYGyTTBXLxkY}vP~@I)EdE?D}4uB3EBTo{%;
zD(Jr==+$iUL8I%`>9O?r`@|k`j$9TQ2^+_RYZJ2-N&|fJZiiWP@v;Z*B#QoF>ERqg
ze8GlsU7lCz(Q=E!FNuCo4x#i-LS#6}sh8znte;j}#d`tssXxEab*S^*89yEUf`Uyt
zaH1fz;}X3tX%?&h#`YPODoWYl?(ZXn5?Vuo*L1G{i*hb9&u1n}Am`MCuV6RkG*VIP
zCKQCyJRN#Wc1Tt8mEWhcsD$1oMlCCO8uRxD5?0fGdSQUSfuvEoPs9f3LMa(6%u|>e
zj5vJ{<#BbhBdkAhRm_8j9%%~pppH_2^4&AEd_K~kN|X_PBZ8ZsRy{i6`~<20GUaAG
zS~9AgUnzc%e0_-vPvVkVE4X_|Il%N|4s@ppJ|Lh-$`%)nBu{EPF?O=*Af&uXd;#2<
zV{C!2zw^>V1?*U@Eg{rV1Po9|`OOz)9f*s?&;KN{fj<zlChC(QlrY)7Qn;M1ZI%y9
zo)Ybu4ILs}lC^>~tMFCW6?QLyhqS4ivkyNk_tVMOG7rV*zGGX=0}t(E=^Xn+>hW&&
zbd|%vwCe%LX>)8Njh!CY9EMOaY=4EpGhu@&W|oH2%B@IO5e=I0>+I0S+&IdPLbK$X
zV0s&d&Elt`?RVJqX$~@$No)}*=u?<z)dm)d*Q(I!GoN)au<>Y)nm18NwElzcxpkAe
z20!~u{vlqVTL0nb+a>OJtv=EGhmq1FRLMuA`>DM+#2u+Yi=021WWN>XjGORrL0DCl
zRvpFBy-`J2rfy0v7K^xEuDa=DN&XOxa#u-SK1h|TFF4Pjd~T{Gj(iR0M|<;X3+$BN
zt=je+2J0%ioOWWSAjAi!ZoevF$&^eXGkg*Q-H2xi8@tM$;&`>Z?x%_b2oo9z+ssA#
ztX1;@WK<Ub4N~fvH!cgrJL9ITVaJXXuoR4>jSC8hA!tNL9V)E@oY;r%oM7%2x@MUQ
zUl%PyO}JG=@4Un~b#d&b%J(`w1H>9_?md+==%D*=Z3I<7f~B~O!=UtG{s_Y|cAcvO
z+J_vA4T<u{A<u|DE9XnC$kzpk&HU@Y>NWitpm%2Lvb1o5S9egRe{FFduII&CuL1jZ
z9nSVPmtO2F;M8(`m-WJ#tJPtHHh(42BI{c~j*^0x2MwN`^rU1EU3Fo}!j1Jo{gy8!
zXg{UGbV8^<F)~{iLRGggL*!k+^&eppc`EiPU|??`GB43+u#P_d&Et|(KB?<<ao~7=
z1@L7`*dR)^_U_#mI*fkz<yaUA6?$fAzmYx5L@Fy9kS3j7aIB#UznbKSQJzj3?$p!t
zTu-{BW!Iw;R4Eq7hAvC~${tF^toWQp)~^7Nj9QxSksogNik@J;CDSLV(uyASmebZr
zg!dwfWx8MrRTR1~D>_oMKH)b?>TW?}ze*AmwFu`X7!IuDYubb{BX#leJLSEQOU`pI
z0O_j9YHWoFBOIYMg*{H!PvIz>HZyh!RksqIqu3DHXKg@wDd<Kx?t~M6nsRxpro{9y
z#RdYscaRR)ttj+fgAt2+`++m_tW(GVwPO{1m{7R^d&r+T(CePuyDvM_8aIV%=|s|}
z?3DgXW8PL=-A1WEhLiWkzQtE2iCk7+=lwG4xi;M>NVYRp1Deh&(tR9m-EblDn!Wdg
z?bg8~RKKUs_^q<xqUHOpG`u^K2TCmXyOZ(k|7}`W{NW|6(ZgZRMA`G?_|_rZ==G-G
zw=Z1yahK40_4BsTLcn8D6zXN8)`*@kGIL$Qt1ty;21%%+G;P~vH(hL{Lcbp_P7;)u
zm(MFw>QkO>%d=s4yMnkcI6Bmp$wM87Zv`7!KmkxCl+#fem_;-s2UJf9XbQ6g<BZal
zdk3Jb!AyYXi~UAyVd(D#r(woWlj<5MvY+w%t5(wR%~Bkscv=^C%S<;xt9sZ3CJhRz
zD1i)`s>tN?_>zWzB4AK_8wAOF=;MK4!`~e>?{R|aB%S}eY@+NF-{boWOlvVdyHJ@z
z7zv)GlTmNXbX)iF4;9C9861mmDq!j~%>2ei4GsuH+)wog^X#XLV*ukanu=&=>Q1{*
zcBS$u>X;<G)^pU_w;V?p+QgR#-XMsqD{I_F`7(-@YB0pf>KdFwro^PAhUhIQHS&eS
zCL%Io<t&bExldu`902*Ce&M7Z%U~hGYSjjFvp6<VpHer0YDM2O047%xYW}LF_M&g3
zd}|nzzEX>9=s2yG_Im%Tl2>ZfTHX(9c6dink!megOB5|C3kjvCb0%(z@%WtM<%$0u
z1NZ)qea3IKs|lGXRItLw7I)rg1d;}3fIf|4K@*9Vz-4LbLg~tJWs=P<Xc}n6OF*qy
za^JXjazQWHrZ$i+IJhV#sI2#Xo4O2q#GsL93I_QL_a|y;d?`PVSp=pa3wfN^gjLeG
zL}Si&l}ZCx%gtey0BiwJ*83Yfgl}5tLcV6}`T_^h9(^P~6?INl$v6lNp-))hIBL=^
zgly8JkPUo-F)p(n5QUC-so%AU|06EX)fv0QT2PEf1xH%w8u*;ouFivtemfAxP%AxK
z=3ERd#2c913vg`&bk;n};h+uA;f1f~;mue%Q-BIlbIMn25WiIVCRiIry%mzuqq6!{
zuI!z$AP($b4s~lnb;-Bl=Al0S0t8~`_^(@~hS6$f%RjKm!frI)h^bSG13$=SnbH16
zxjtm^%C5<gI=Q8XG!q4MbQ|ZRaU^-a9ebotjJ!h6efqWnW!FvfP{^EZSN`GY3hqfo
zy~oYei`1zkkKMFM?R+*!!s#b}mjQwW#1+bU`(F0;I^bPE`*Ae>U{8k`n33feoLe3L
zN}do<r`{)sm1YbqYFbasRO0d3%@>b$s>!>IFtII-%|HoA+9ITSeg|cZqlaogNo}0?
z%aqM@CWh+eCXz)aMB~>N?3*ngxKS+nIh9{kDUo1;nN*~7y@KZzwj5bXuN2ThM-O>5
zj3{@UJrZ;qN<jz4$KO*WYR>o2U5WMwB5_A}bPQ^5lyK5HVs=+pHbw?UOCcXC>yI;I
zl_Uv#>8wUo5gpwHQ^D6<lFAgPTY@BHCyTzC*|q`RrS(81Cl~M(9yyg$B+Co5FW)ie
z!y`p^w*H51C>*Z<5c#`C)#ru~*ue)Bn;C}e!uGbOKrRg2HWU6g(R$Q<8AwL-?U_p?
z;1TEVM4n^-6R?dA=M#-F@h$767NRRX5yMxqY`r8+4dIm{w6>T#uwnb=qyyC(Z+_TC
z0>fWFzO2oearD0ML8?T1dr((@&S<SnUSgjK`jD64Vf@fMP2*`RbQ_uud&(K@!sa}7
z1(mLp;T<KRUDo=6(Zv+(;=8ZEE;(>b*5zHbXZxso@8Gw(!W7QWZhCkEaD{~s3yf~j
z2cdc`oOu&`4IH*<qljdEW(p?zkToPV_%VKQG@-`O5eNN8F9RY_^uKAMkNTD>+|uYt
z2h@Mch)S(DJ-A(DWK+;gGh}1<AW$_bIh}uv^`jZ!Y#H&FLmaK!o%hZJ<>Zl#rwqoh
z;@|jAPw<lcb<M>J#M=O&zh#}DS@jz%KGQNAgW0B!TgF}!+C)!kjw-Hjl}bD<h{=0(
z;6iO(H1<8@Qn#>gwA>OdJ=HamzKeH~-5nhSK96k}_Q{Z${{b>iKz4}{aO@)e^We=x
z8QJl-dQwY`fZ53toGmipWyI<;CugE^0r?i(2@EUb)hBl{b}s0W?9JBJaI>5}ebgE&
zL0v)L+<10AW=a^ZVxIbAcV{n|x=NiP@O27;iTbhc%aAf2TY4qNaKB=vf=Y7Y6e5>9
zhi)vCM%Whu$d3ypa;}p1vyW)Z&xCb;FRIGy#cG<*uOz6U^=-xLS90R0KS{H*Kk_8B
zu6#!0v<Xlgf?@TJf*F)Y#`McN7rY@2w=_i>d6I<$@TAt{8CgOhfIhXjtWjfD^HzNp
z_jMysq18?>RM!EeOOL94b$4yhgoja6y=Z?Fq5Bp0FF&P?Cb!GD1;4$Gra<A9SK@aw
z_s=s?$EksF{+b_O7=!dr_~JdA#UgZ)Qh^3kq%TVzx_&xI2q;?dW4<2~B<ekYBUb||
z3F0u~jf84>Oj~752##nz{d(ANI{l0K8YrybvOdaNEc2IzeAuKboq{^Y*w$RCZ9F%d
z>K;IL;&+rnSY)y?v<D`T7JdGG>D}Q9E}b^F8UoOf5)9H&0eS2qwM~)3@0+d>D@gR}
zgnvkfXqr@}Z<<QJ8o-2xW+PO<Q(08_=3^%M1#~ikkq$_hdoF;mxPc8h&Iwv}23Cp`
z6y)%#i5I67EI~_FEkY8cXvm5;BO`V(1K9@ex>RxQUg8wow=#xuN&2%M%18*FjOkw+
zphjP7VC;_`BcTel4Tw;a*4nYuZ!!9^Fe>$G^sAcz`;rcL8<Yy6z8;4$oXJv_;X>VZ
z*JN8*FEUDS#Im4`I<A9BchFntWqGP?8JEp)ndXnqW_)L37Em@VVXQbfst9!M16Zmq
zb7cGw+>%U|z3So?T3V(#D5OYf1q@dC++(cj@y|~TpOt1rh>A4OD!0Jn6HX=77RGjs
zM6JzahHP(rRSd9TzBCJ$g(-?IJHW5f7RUU^G&~VE>&=Au3B)L(1P}l4Bg{&;_Nvgi
zd<)JN^!n!X<6wLLz*4%d!27r3^+TA?f=G4B{bhu_2PSpwnqM;bDje1>JrS6Hyrr^*
zQ2G#A)I1s&H0VY;a*LJ=yWu9Z+}>qmg-9V%BQ|WbZeD0Ahjrjx{(!e{<i8tfAq8jA
zM?y1SI%YKM24FS#3W>Uh#JK079Wu7XJn_m79=U8>GZ=`yF1;6TOAUy+NZF^`o=NIt
zOQ7J-b|0|uh+P<)M;}j<lply+FHnJr=GIPJpeY7H*QE|tU2*+dLwToK3zTOlbk@-4
zuoZw5BGmy)$tWEU4EP#0jGqfFGYAGTuv4XI?6F}H59&)NgcbbCjAHfM5-5{0{5KE2
zyx7TIw@49Tc}HzdYAd{bEskEBgzph#iLtPp2;Z)H*a>a}cbghBF9i!Z&i0N(Fl<m9
z>XIRw-!x8#q9U6<3j;>kSWVQWuz#O;zYjJ0WHQi6r0Cj`yHk}|Sl1*L2Mp~_IS|z@
zRBV&folTUe&m+`fY*5~yLW53;Q^oHKE{xgJ55tm?7``>9t3nmz0EgHRwm#KLWfo?V
zdL<`b9p7A?fTby>SS^VR{Q7cQS9d*3pBLt1DMWcI&5M(BWzEERmY0U=*Nrbwdjhns
z(ZS$X#yeK7%5ez}QhP2>|C78zcdydEpIsE*jX%nGeIV>hhJQp@4p3Fs8hWQXQlCNB
zXX&j40ivtUJbQHfu!gIbT)rygK0i?|*V-irtJvb5uay<VFATo`)3VqfXP~IFczcbQ
zN)r?&mFDltg5foryCzQy4^{jE_G$M{Hd2X)LIwj}<*6O~dyB0DP4cdP@h7dz2QPGI
zKOPdBlzS7E(mC{;C-5;;jqk-y4sY>9I}@!Q4k>^}+E8TeG5#@cwOZxwHfsN3u8gsf
z$ge;KIaUiTu&!TG7)@E#FNIjaO|Mj2@{w)wG(w*0Qs4YH0y!Yv`m&!T!f#i37ll^>
z0?k(weu5rG@j-uel}X!rUJ~JYGjcOPa<yv~batsh-v_sP365{M=4b_YMW|=~Figj$
z6e2|=u|A8YhYCB01$N@}BfAZJfZlUCnBNRO3T|S<6AM2TX7f^$<~rQ%7X7XaJcajH
zp;(+l2J}9hMo0NuD{q<ip?0Z{fqUEZD6&Xi7zDpZwA{=3YD=31Tg#tmmDkviNa`xl
zu?bQhNV{k|oeek0Bl=U~72<HfYRl7@)TVHGEpalq9J!tvC=kS*%wT%!NXetlL$p;R
z88T_jm6@jEhBOe6-?C_LB5Z+E&AFsOXfz9$>N}PX(WQGu6}yE<9{Lv6I=^oG&QN>O
zFNIyuKG~2wPim&bw}x<ac#m)Y{ZU3KgAIy~hkm;%)7ELh%0#lLDs!cfil$-EPvaGs
zUnESqpq;q`<y5DAMgc{iMH+jUnssI4o|2OLjyM39L=Fe0kaeaTCklOLeAp6G?ycmn
z_d63Sv-W|;nmj&W8Lm;J&Obb#;uJaf(cqU2Z%cQ|3vzDc8P%8}eIt%<McRr9eE~|-
zAGD@ofigJN03EfvX=WHd=Qc?apJ@ChyBnn;HkCcUYcfy^gLyg?8bmx6;yXFznfGQ0
zA^miT2k}mMqmjVb1n8tC3T9k%D#gFR*>s#b1#Iaf=Jv(=e{F^A`USnMyOU5QTkThm
z-E;)RmJx8v8n+7p3p-^1m(z*8t3EpagDgL{cd4YW$=`$GpSdvT5q`#9EYWLrEZ0&y
z1oYopHax@%>*P?NMq<Cei>Ksr<A8wXF?-Y}K!ucfGJQeRa+Gbigb=>-XW51774j^J
z>K}#V*$c%*YB(zXvB;?v5c9e4h4ic`Fl^IV6!-n)tol3>bw?7@AMfAl`Dg|aGieMD
zCZ4HBlS0*gR|FH_Ru|}@?}euRDh%4B^_M#(I4#Lmycwmm6qZ7tW?u4X3HU_!4*P}D
zK1Xe7xX~6Xy(|H&%`wTO&Q1C1N~MO)S1}>7I-M4B>nL66m5i)?HRS~_$6Dm|r3Qdg
zi5Q(fLlm{e+@KLZ2KQ(^MZ*O<e+6P~p#R3osb~hZ85RiPTtZmX;z4tR3#bs?BkN56
zH(<s%Tb3&ykW9TXasE|S(fVr~xg(dZph*~-vF`!Y15OQZ5*30~TEsmbWre{+_>IK*
z-4Jg%QoKPbH9{+3_#FuKmTe;Cw%#AD%hoUEfPJwOVg%#zADUdoy6IGOM+gb9^5;EF
z`%Pzc-j7VIDsLXeNquv0Bk3MoLJ?du?lYCzXXL;`sA~8TT-CZ7rwP!a0z$ncTRpZX
zt%aOAZ^Vk8E%0S(2T{T}kcyS8w;x;Mh$rNHhw^U?B<aqXAw2o#U!U%~@SI+yN=C~p
zPV2!xg-U!a6S8>=cnf#hl?*&l{-m}_h1!`ZL$WSoLdqb-UGoaa?Mn`kB%{=(2S-dW
z=<`e0KKMvpoj<emPMjev{RX<w71bxPdWKlqtO7tmBMNEUrPDamK$Cw?X)FED!`D`H
zyBP_TF|zlfL6h9SqosUR$?9t7mMFcJ$Jfc8oC+-{Va0UXg%CWXlmFPZ6k4s#=WJ5h
ze}z@I)<M@pN<ynDhyi~Zcuc>4Q}RSx_*W>ZY;j?&YT1*<R32Y1sKQ@jSoZNVH4S9C
zfTzaII|7w*&D@xbGE}};1(C$TZi#d{j*C0Fj=1rBdMRBT>N)n3tv$h3M>(qxu}w$p
z#<%qtiS%BDQE`Yp&&hhbr8^`iw{GJJM*k`1S7`@G^yl8OH10{_Hij+^GZ^roW2KR!
zH}%FTEb<X{HQzA(e7ecbCL*sTV*(qXity*e%u8FK8}v&wS<+k$=mr9*Gtnzoirt$$
z72un+)_^QUCoO|NK{ospdmu9TU07Z3895RJ*;5zDp5sOWS`>4lIW9_g8L*3vUzfjV
z$Hpm&M1K{0;jIvv3ZeB;@x8N)B(UqXl#f>aHE0t3IHT6il=w4eCfI5gMj6{trV14j
zZBl~ITBu%$UT#``TMZE$*xs53w+<rh0}`n_f2Q5-Nowz7xzsp>LTRif;DY1P2Hjav
z98qxmzS=GP2JyICtxDG^Q3@j;$%of{y1UU(&`2S8s5!dsD5J_zwu}TYSb{N&P(*)l
zA`B&B<)GdO6XqIJ9fO3Tzwr{|Q_b)(Ms}V?j5c5DXs8(2JzRmE@M0!N?zSDml3yq@
z)+~WQV4k;7Io3wX)vuRZj>|5bmPGI&4DHvvq&*dS^I|V{l8GhEkuV@FMyxYp-KJes
zc})#bfnK~&+r)X={SyL=&;P6Z9&bwm(<tTo1g5t;bjETvbWti1)D!)Vo~U6`ZsYTg
zsX&p5A2LEpzNoe%vo>Aqv*L8`#TlpU&IS{?NAmlGm#;^<oUGzaMH^xQtgN(ySO61$
zBnG`ek^g9;sW?ZmGKsb}Ob^&Z0gz?Iwo(GIQz|<;g`%OM_dtn2bykj(Oi9i1d`FW6
zp!Xz&_uIb6*4rC>^r&UpN|?T-yR(j+AIB(SI{VVMD@DHxh$ll|bj7je59)rG8ZIAM
zS@dy%iFM0AN==<#3-tXA7MFCIj|V{f=ww&<y^Rm^S0)wNESK4(++2V$n=z)c>I!U<
zN&+bJ6t_<AfijMWj5G!jGW<0`Niz9xBAtRn-yNZ0wVlUBzT+qT_@&M3jsOJJY15jm
z_|1)eqLv#9wd{caj8lHiPF+ep%bE*{mTxE%AKj-&$TqZ8*d+;C-t3hJO4aXpVA1(t
z6q|XTpH#&mc9)ES@r<%1bi`*_NxcmCJL?iYX2^;BL7rqt=J4HM6!%f97WklZgxCeX
z*DeAcGLH)EA%#C<L3nj1W(7h{Y&dhjyibsf+1#!{IB5^kUPVd=1YM<WusQ+jY%r6z
z2cG^IQyxAW8PPfZ!=uOS+qE+UuptO1ZrlB)r)ui>QaAWM^Va>PI%j<Cjpg?)Pf>q)
z$r_H0R#T!cfZt6abT_<OGa_Vb4H$`c!9arLyOapHIp24<uNv+R0xOUF7z{1HK<5JF
zPRpXg5K7~Wmaux|q*ou+9(};xMZ^FI@hE8<t|6m9dp*8^uKs;r%t-32zyM_lB`oFw
zMdBlGTUw3V$>~Fo#y+AmRcLM?u)#oG-#1l29!?-I41mL#N4`jqzJ%%hd$lPQ+XKq%
zob*Wbh_OB(K|Ti5iwAYgOo|dZ&M%Kujt$7Q@d_hz;+7YG!tCW_!3|203#k=od2z=&
z`9YPi#SKVyg(Ie5XPoXYdV?<-Z$Yn=c13Fr!i=S9pSX+Dp0fDVki4~DUrJ03#^`)1
z4Q7M6KAq_jSQfC%%T3$^XvL?jc$~MK8(mW#=+&uB+jo`j7|kp*PfVxMoTSMQB+-r0
zdlOo7M9lI!)gJ$_SHRn_GSxt?Qr9W};H;la=)o&;G?mVpz&B>v$QdK?QBYJI2)J=s
zO(uttD*=#I5f;kiKLILbd2B)hYl~vhgo8SokGOS{Hj1;#HNOz~@Z!u_zObFpz<k2A
z30zUKmc;|CY}K^Q$1GNV*2|^-Z_Jm0$nTW~wg4ojxEW?$YgQia<N*SvFem93jw~W}
zyeRaEiTgh{FhXa(@b6ZUd?8baxr)UM?%7u`#Eub3t-cr3%Nwu{w<eZOFmfR}=)GKu
zBFFbwQ>zwnC_8g+r6Vn*3lnF%Wh|eO#4Rg%*9a_uP9&zwxkiZswrV~3FQMfdCCd-U
z9R^(t^EZ{U2~<N_z#2M0*GJbiE-+38wR9@)RNS8$hjLv}y66moNZH=~YJ7HK;pn4`
zcFVx#0?cEm?ef)KwW}NQw!@uHmoWY)U9O5K&~xH*obH7_@Iukp3=iiSmK6xe4~lbg
zig{PD6}CPqgFEE__9Byi%m85UDTDJMBiipC>Pb~!;5eKZKub2qk=G%yTb1A+|2ohg
ziy5CH9wHYYV)B-}{Y_$ZT0zS7XExZw@k<A$tkEToa2Dgh&;AI#NIyxR&PTP&lTNe?
zYMGqhlD_BnB$fJPG{q7?di|ea@)V2!1AM_=Me@k<Bmj~=SR_RsLGvRB4?hD=>9}g$
z$C*l-&Y59bB0GgB>&|1}((@kAa=lWBM4Y^lt3iE<+!$4}F*tWWl?q#)Wn7bpbfRYe
zyE&qHg;xHnO<ea0kFP7Y&(146E0v98eG65erg?3&#13T7&55xxCcuU!Thcv9MdA6+
zW$bv^d62MJqrx}w1xEHhdy<tHBJ2zE=cz;XCI%XN>7XH1zpX!?8Q{0K6ILtW6bt|&
zg+7`gvnF|>s7s|Pe%J{B$RmyEg3ZsW&&xLz*v-ZCxzZbPXqz0>f13IzA;V^8HSlB6
zu8mp_QTIv3UXSL}EOTT7D=e;qIceQ=J~xMC9f_+>b|B92GFd)~YGjCsu;L|RkrYVh
zSIs^|Vivn~p9ap05cHSwN-U#f&R}mu&p7uv#NO-r<}fNSR7i5pGZP??lQx+uA+)#Z
z7bJk;)_rSPPBQZ!W;wmRSs^VlJGkWS#^T>EDCz|YE8rXJ;>vSfUc#73_#+~^K<%ks
zFIugEp!2H~DBqc-{`gDteLgbA&?zA+yt|WLj_QvgQq3{VbgpipGVYYwd?*kyJ8%9#
zD$0O@q5VV0LIJ_KHb_yV4l${K0RbVw{}=RD?*Bw@omQ8#!4*f#;pFYyj?QKa*U4BY
zVvivqZiYwLPHIcg3DrbSrqWhL4UXII*#2X-=U%wQ?j(fF?Y}F5%*h}n6--Jg4^>``
z6zU|R3UUOb%qV_zqpYTe*f-^F(fA$dMWqbTzR65~>GAmA-{Vb%=0QjUo22Rv_FV0T
z6~L9m)>LX2O2O=OZNb^my>gUy!`sK~MkWo9WOHfov-qqlopZa@vaDPCq36j($#RkU
zoLNY~a{=q4_=dyPS*)Mv_rr|r0d5GkUypg}4i#65T3r^q2GO2;5izqH^YB#BQ6zW;
z1!-K-M%wiJJnhZnxtjgsW%ov<#zlA@|K&1n=T4<t(ch2FzxN}dx7^<Fg8qbG*-Yqu
z+g&X~bfce0i;QbY;XSm-9LtQ|_u#DjB&*CeE0uuSwpt#wS*Lr~c1*D_nJjl01yrr@
zvUNo^4n8MC7aKm`+d0%w${0Qm7pL=8$8*0OgVjW@Uhf}&50>qooHwgA+`6o+=E82!
zT`W+M)xbR!cLuqI@-;NTA8$_rM$<77a}IP^FW&l$X%OW!)=%(nHKd^(NM*WZtM~$I
z&a4J<%<kwvk!ZMTD)c?Sx$Tx~5+_^qA~ReMoTdw`VoH}U-_aSUYfcnH+35&u)ua+A
z(p|BcswP=!l1p;47Z*k5n1LLTHYybJcxG0;kmQ7}+Z3o06{1>9*^?u5@Bb1fLQ6A1
z$bI+M5{tn>XPf!a(!9==jkf5YM)$}g5C0SF#v`q0<`a^#CIp7^rpAES0-+@PH3>@K
zShcYeCv+GaV=HK0TZ*VOjW1by*x_%+4%);^XJP-g=NrRCK|d?cbyjUnhl03=Hb5ms
zgDBmJ$_?8F-@ck+@5DAk1ZuYRB1iBJXr08ggzVPdVst;bea7-5L@KRg)h!Mf=1ym=
zA_Sd1R#hii!B2J@dXP8H?x7ocGLxgmA%Cy<c`w0@kkHigkjzwx2niPr645?_vWxh;
z?3pvqWZQ<fC+lYR7c7BytU`A{q3B5hk)e_Qc7w`Cc<5q6yfiQ@gdBL(>$cM0v8KNi
zF4d=Kr{@R|^AnM<ZD+3Bd<&CvETgZ!ZswsP5%JJKr<!J1c~a>8OtIkEgR2cXxj?^m
zc7w)43IB@?soLBCN6t=^D~p{nSPzCdxwMzSsk29@ChI0!2F_bknzM~KW2;KDOz}91
z48sS$0E*2OO*lU&%nJEit-u?6Hx_3_sCUpK!!YiJzQ0pkY<t%3e!uk60`W^5HNSn=
z@kUL5QuY2dz+@N(mmVNvol^OQSW1`GD3IoE`0pC5-9X&zaCA1V05A;2U}0FUqlLb&
zOiyjdXUSHxFHA|spB?eOSj#V(`dwl1@3TV0b*hu|uMR~h+PQI(d_c3c$FNS*7at&$
zOHV8}c3XigJJn{`Ml}Yxy;~=qI|)K*=qjBr*>zfXnXV_?+6@M;-U#jWeh~j>s$v%C
zpBes_xFUi7i&SOwpHkKT;a*?=U+(p8R<2H*K}PC+hK6!dy7tunPrt?mMU2$f*Mb2B
zI~D$I{q3HHeFx!5o(`^fCkr(^HeHaKq6~bLBt$N{n<49;p*-P~e@~g$#2=9l9-fTk
zXzp6(Yk>Ba^l4h=4W-%xJFTSnkbWa<SyEhfe1a)Dw3d<oo+um>!jp1^t?EFTQJ7EM
zFF;5`2n<6|CU?1m9QfNqQ5eAwYP#Xbf8-ET5gcw7Y`6vZ|DkZJ{o4pff&c>Ill?D(
zWBi|jbEU&;hpL43htCo8F$`Z(LS%`ESV?J4Nm&dzA~-N9m<7cv&jk3=KPLm;-!f0k
z7!}ND^kJL7*Fsxi@0FGH#&?GMWjlSmu(tMQ+w1xH_3-j=Z~lQ3Gqu#BmO(!&S3f7q
zz)xG(-auf>_xAC1JbMsPqqo<mOM7em^6TDGn@%G%c84Bg#Xe=DwebdUGe0>U^H6)v
zt-i%r+41pwWJuiV)Zwf4$l8K`%_O+kAXt94c@?5p$#K9cK2N(>`n(2ohM7Q7cP(Vx
z0#HCOxGJ{>>eok|gblwkxOB6-_mAy&z;rb{$<R|&7U8k9J~tqRUHZDRM-sk$OnczV
z6dz?dk=>+!j;Ad2W({x+Htc~dSgH-~M7A#jV)`8ycpV%UPNNhYZ#o&|&)YbBtln=@
z%^m-Y%?NZzZmaV|A0JJ^6$nRcrVC=O)~;Fh81Z-CfF~*EtqTbO*apHU+|*P)s@WhD
zQM;+3h{fpXF_wtGz|QbjH5iG9t`V}D*Lma|485$`5+{;*cz_~zCo%Rd%VddHN()<E
zV8~eo>)aS%sa*w~>vf&*@i&Uy!TI?TKh~cp9Yrl7R{!k@X6Y~%<6;%&EQ5}3paMDK
z#bXbS!6ECa=bm+~tz<47-5ic2^Qk-8H*61Wermy|6z76ipK2c!O%-9L;kM4lJ~V(t
zOR=@0i4?KO`R<UMn=9EUc{Q5uaWGw<33+lX-LV6pj~l&k9?4+-EgE*+juqW9af^s_
zpJ{pc&Vy8Hz9_UJ>G2V)sNe;b(I}WTeO}>w?D!+#H-h9&KU`vlxOR&&TtDyMaCMNH
zBc?l|6*7Q`@m;zZ`n^<u*8*%)&0I{xUz-l;g+YOsu}(|!@%3Q%CV3YXXTZQDf}1Iw
ztv9_f?%Mu>WOFuSR?rM+z8=&S5S)!X&r{AhSNiualt_nyH^pltDW+2a38lmaB$tt_
z&0mqZ9G2@Y+Z{%=2xa_K=$9t8imLZ&r1H1i7SM%h3UEh|A0_MI?dU`$&ZY&lq~eMo
zxUCS{pL$PlE6VGj$b4o3<1*wZSyKC;+`GJ-&<VoT{b_N<z&`2Fo^I?hzh58TG}x9{
zDzwkpMT!{)PS2<TTbn9bvZ`o0RRPzN<dV@8h1Rd|kTy0}>|^Jp%z_clcY({qzf9nW
zrucZHEvxMUpohxKRGry=lQIhznYoe>AE}|!6|0$9xTVe{jykA?b*R(uvOy|K5R;W#
zIhVLh$Khik7SESK=R1P1qa8j;`teJm<L#h@>_dQa3Z#I`TZQCLOSR%CF#KLLhC&h1
zBo!}Is{DJUG*%VKqo`}`!#5w<J7=Rc>zU59I(N~|`7YgU^oJo!Se&Ws)~A>BYA!Oo
zOe9=&!t^!api4W|OlFIg0%1KKczQo$un;|v)A5qgL$@i*vY}*ot#1l1{yu`co+cZ)
z8K}bJ0;3$telKhWXR7j421^W6svTAC;40^z@Xim~R$7?HhZL=IST{1b$=phLViL{@
z5lt`BF^^04cYv9BQgZ-Izv{s5+G@(@;wT)ps34p-LS}0E=CYwwgOV{V>Cpj7=sipS
zhiEK_DiAVw_DM(7TY|b#x0@L8y4?7%NuM8#bkFRD?m>#yJg{SW0--w##9IY=NU<fB
z-$Z49@|BafzLd@3tQZPAjp(fx{Yug4Nrq?;2kysOVd?@n1eQbTK&L4F4Fj}g&_#^Y
zR1XRi+CCq$K${jPDX73ti$$-SMtZ0;8uII2qDm|P_7G&uVO0W`Qt1Rsj`i9PYxQwL
z1K%bdnM`$JdfIiV>X4Lj>5sX9tjPOk@*5G$v1;BJiW+AYlAnfd2_}3eI$jkivf<;O
znb%+RDy!titG?)b(PZ}~op`S?>;wgSE!kLIU;c#!NF~hONy399kr<1c$EqyO+P#E+
zHmOC?TRKVY3+&aOP=u9r@yg_ogHy>I_%0oUoY7SNzcQT!S)<woQZQs`ne8ni2&XNQ
z$4QBuvs-^Gz)Mlc`(j#!N&o5>{E+V0f7te3wSKkiJa71Z^j<xDz4UV2`1$@g-u^%D
zg>u<TV9S36m=WZ^s52)2sm_?i&q@ypA&o2=zjLbDfX6<c7RxM<$=L@s1NUna6Ld+f
z+O@DJOowc-spc!}O+((ocX>U7Kb|Vx60`kEB)RHixO~ZcGk1MC9OSjpn?ioBXV7t%
zs3XkzP@#bm+52j)$*?54;$!p^Fi0`3f;Mj1FWdZ<7N4Lc4O=&<^ik;zMFvBD#-GSA
zfrt;?0G$AtFg#T@m|!}%%P;U3h_e;_X9q$Cxm<$)*UTE<*{RQ?h9HK2DrAQ|W$q4!
znd+P%HyMdvEX(Z=x0Q_Xm9w>opsd32BAQwSU({u0tiSIvgM^{d>DiHq^fw6|y&WUd
zjNMEeA_C;m%6LdrDlnwrNp))uUTfZ^dA^BqX(o3zB|vn(1g%4uJEd<)%Z4<8M9`@r
z$$^zTuPce@`kPXFfySwYyg?0-umPh|T_qPS2JnM#fW=!9$sd}Tgp36ug0IVquf5Xp
zhaE4c8(+`|R?D0Jy3_A}{zPE{Pk1390RhpV{uk$G`k&74D(&n7EL3aM!mXy;{Kmxm
zIjy^PZEY*P`^M{eJDcCa{DwA98O~UR>5N!Al0Ld#Q!Y#rR-}54MkG=UWL>$K1I4?P
zoLx*M|A{L3b)Z7b&mQI**>^`^wvy2x=j4Gey7$e0ki+BpdrCntand!n+rF02{W?AO
zN?*_&Jm;~x){6DByD&)qYfSLx3R3{Y+IQ1KUNEN{^r!Zx(gsAa_5RLSXC_ub$7a7@
z-OV)Ro|y*D5gn}fTo_m<+eVm-vTj_KQBOpUuc-x1N1Ow^6`FtG4dI$sdvIns+UnA{
zItsTI;iQAyJvtz!OM20zi9J8gf$1>9lUWt~y^Ybf5$@VoR9B!u4jCf2fC8XyHC5Ay
zWvKqpLpm(S{bO#%Sp7;y_+ldEDa0i)*n^3*a@X8i!9%HQP3CkTY$b8!`}?x!97o1-
zDFVnX6P!=bGh2FibJut;&d3`ZwAnd8=1%c3>cesidIvu>^_ZK{0|Vb;Mde|}Eg9L9
zC`P3G$Qm}^ir{%hPDrLTbJn7*uw^!@1xa|NWbv!yvYFeWY=3BR!nk?j3^W*Ppa7th
z4`n~Suf+<`m&UO>i`H^t60KVdb-yY{0O&AXW{7=B)vI=xtb7x2u|qVV-H+&9QC!<W
zIDDudb*$s!NNtWbI`)q?KJ$giTn`aY6`MSm=N+G3%7Yh7Qkc!Q8;)GU#?sIoUOjcE
zt3WQ$^!qYqSa}?+<=k^>Mrmtu^!GU_cZk{1rDIOatuZGriX2;2VOVNIyzg!XnjzVR
z-hd%p60zd%kGli26-Y!qsAA%VBuD*fx3{xzw{cwV2`G6#U8wNFcOf$=>+Cek(428~
zul%rdWw34kjUUpHlbl@1$k2%HNWaUU2g@<lZQSU6Gd5E+AdpVWyX5I43Rkb{b)9q}
zh91F4k^h85@Pd<qJxtl`Sc3(F?QkD#f{9GKvei`sI?lcW1?rE~tc^g4e9nbKZA@Qa
zCxtZ8l-9=!;W55ZnlUpYD3w~2T%1s4(48P|u_S*Y26iIC4_pBARX+KP8w1_Ct*{yU
zN|{%%(OlA5hFAM}?Q-jDTy}DU7$PUT6$-oek7{MsO$K+2@z$K#IL(@orRgPgO{U?9
zDtdKw-gJxI-$r@^X(Oj_A?p+wLd1f2>X9LvkyTm@i$S5a;l3}rCYVmN7#@-`!uou=
zdZA|uOzozSF_h%a;4=yQDq#639(4rQ^RnhSzHK_}r$8IVlC?_)mg@&hmO9r5`m#Z4
zj&2v@Yr!Q`K_Kd^ZDCI{+-udb>EgXu)!r0s!ek_-;yx+%)ZI~3<;vP29A$zKSB7f0
zD&&b5mxU7zh3AJThnn{w4+(nu$*YUNaW-^%;m$l%TX#oZ`{dPaYif)IEbCee!j-4&
z$A;|9{tTW=WxHj5JH*w01aU(lZkf1vXT(t#RZfV)_?9v0N}eFwH@O#NOua#YL?te1
zrv?%GdMhGK+Lw6Som5IJ^LgrK=um@cdKZoQUr}!3R^BZ^Hs;Z`j7RX8X_rgr9Gx7y
zl~xzVu4i(u_8!)l$Bx6u^T{37n(C=st9Ac;Jz#lpkyp=77bw^>2Dg*@?`aSkw=N_>
zhqjtdwJHwDCme$ra3xKb*l9uA&4tVHkMK(p9{Sy-#`s0{Oj^TX$_}g|7%p*}rX<it
zxlX%K->ig@6J|ewrjm~rf$dVNOC5>{%t|xwi3690f#?UASYTCV7D=nfK*z2Jex2IE
zPp821Lp5P{qNktef|uH^U*ik^8zzK?EX7Q~<k1Yu;i-qy3E2;w)3qyhlBV#Q5~~I>
zLqerx_Ue;p1dIV?CUNpw^*%%+C0tP_j@|Y!U2m-C>4_J#@WdkCpR==p3qRMmR3ukY
z)yTJdTk&@_e5@S1Z}C-xdC5UuG8hrYtphYj@Ns_+Qk6Tl*01AiAa-pup&Fj35oopk
zDs7w(j-_N9t`B(2Hnh!j@HkenNjPdRHNoMw<c}B5E9(zmtE*nBHRwkuG8)1;fN3Gn
zB1?fJ*cA7Y4EN6S0;$1>ZvuRzg*HjWDydN<<o!0VBO+^@+FKBVj#B7hwT`idDHH1;
zT$U|hZ;~IdD`u~Sqhm>VM?0o5CQEu+wdSvKFh6fw3m^~0DvkuF{i1L0ap3KYtg2R_
z)v+{R*Zz3xldlfui3pid+?5(Y>Vl)Hpc%Q=Yu=l~SeQ@y%Cn_O*nWlM3|@~6Pq3`K
zb84Aa5EoBX7QSl0+rRyrl()sN`!~x?hqnL@!ogt~@$wFA^v@TsA6Hr50&2?U?woGz
z>UISg>%88oFJ*~}@pP5juUy7EYisL<DYu7t0BZi7@4D(D_>1X|m!Nd}2tc&tPsKc1
z#{-8Q3O2Im<`s>wiiS;_T^^7DmyuFVSl{8RqV0_H0(CLr%1Ya0pXVpO<)kZHqnOx~
z7Cr+ixqqen4NmnnsQFmCGIpFDjnaM5CANBkOjxgKko^_D1+P4qySexXoFOQ6ZcQlf
zpsB-H79#K_YjA*(zhr!#D#L1{DGX)z^ww#S(v=(P(LzXpK5p6q4OU1_2}$G~6f@G5
zx}$9@%&Kd-JpnS;s&|Cuy!I?YMHQF#e&jK;9=5WEq?|6Cv3%r+2|8Rps&r({B#Et}
za;moMUOH&3lULP~)r2hwNZ}-;8*epiBeRj#UVjNvSFCd>?Pbl|m9L3B!zgIZwC64%
z9HJUlB=`iU=X_OK6@jz(frwdeKJdGjMGh#}PsfcU`Uk=|V+9v;JJT7f3Ci+d;RYv6
z(@3``P9t?|hRMjHbKn;%o6=pl`)db04X5L5c0TTd@o07?v}||#00sET+diL{XIEZg
zlm~3IClS5E9WAToxxi`0!nM@nU;P0~tBpuRA%d+0%hld{I#sjdh6bvb=OLBlLz&n!
zLp3zH6z0s+7Gum4P#T=MQrqj$FX)uxT0@C?#LTLSOynLfa+XR8${Z!TBW)BwV_T0q
zA}t~BLgiQ#C~w_H(OuU#_^cdK^E_v2=P|~Z6?*nOGq_SivbCCSB{MCZ!!+czR}s2(
zh%OR5hYrpefsBeTSbDI$Tb_Dw!gP?83;X6?4#}weXcE)3DtCNhhYG5ySZ>(aUTaac
zm6s&xktq-t;n&I9+n7Tk8F##%&4I{G2PyB7+<4&MR4IYLn-R$6OLwXirzh)5&4|{B
z9tq3S^p}nUR19ZvNN;aG0TzPc9tnztXrZ-nx!ZW5F(O?+#&d_IO5qCqMak;){<bB)
z$_rUKM{I%KvZgDW65)UnxL0i8)0XHsfAceqA|r|w{mRyy64d+Kbe_FSj*WJ5w+k5E
zpACj<B|B{~Ug(Vl-&`v`a5s_@el86i&sxHP#UYII)Xyo!t1%8BA0q^L(hU`}osgFW
zqS4cHrC@t>O}+RsbNdPR^O%BhAM-O2n6ds&Xpk#iIxm2@TO;r;Hur)zOk`n=;@^b}
z0u?z^UlO-g-D5g#5X9lZy<EvrUUmv=wFf-%ocm585elffND1>43GY9F^o}DODFOFL
z+4AplxvANL1;k>>OdRMm&PAr>SH(kHDYJ)96+>0Z4l-u;Mm@-MMqgSJBJhUJWa*}f
z54=fkZZDR**;o6`s_^z}qgx4_X3gBT%p?#6SKZd<N;-H+73@Hh9YGrv4)koT;p7Bo
zH_W6I&7falHi~?A{2B)DDvGcU<V+Uw*jmM|w%{!6t&I<x(s6epi|GV-Y0~zN)yMsl
z%vCQtV}E5dU_%K;ou!!{o7_y-G{{{N8GH*xF_*YgXR+@cy9&@9(_#%lo~caJvT`%n
z;o52CIL2hvieJK1G8pbl0EN~)AU%5HTeSy}a+7`Sy7(<Oi3s#QPf*(!pLUC7G$<TC
zEzyez36~p78=VvLt+5)wPfIDx>p`fsI?rILd#iR^>s#on=6PfIo8t>hE_#K+8d#tw
z&CR8?<IysUui#4D1Rfh5zZZFCexHmtrht3-bWgK6a}r}3ls66AsMOZbv4hV$8UXPl
z8cUTWP~6p_GQX)WCVJDF=LkkHDC*~;=&n<>4H<EMqzPpX(rdxQkG)w&TPY-1ODmCL
z=fIbr!I7-TO?h>eC<kP<N2ms{7`^!5h*uGRi^W7jkB-5|ZbVKQm8+?=7D=xa4_3ru
z&8ggPHPEQOYZ-}SgsVag9JN)b`$ou1JG`ex^3NIDX8(J@-4cYHU!`5Ymu_ut_bMAo
z5T5F*AHQoJ<x%rLH}hxYhQh1fSb6f$P4N_N;>sbmZF4#oSSUFwal}-}n_TqFQ~!yP
zns*}KTCpZb@iwQXr&rbXF?C#|9xAf~>6{_3Q5@Phfyj(@<>F%7E74y);)TbrZzf1c
z4E@qXxW3xM{V-IH7ZEw4=!kJlG2%1J4X8$M^&QH{+Y0*fG-;u1H3i1MZcz!&F7Nvy
zsYlf3S*E*yXBm_`HP$I(uZg@qX89O4v>o|doxN6*fX$WW5MhnG*N_1^VJ9t4u_|I|
zsH*W+APx-jVnoLvwb>*!>%{BMM<XFa5LNZkM`q71RyGot)oP47mUJ)D5l}Gsmk?%L
z+yGA*CElsA!E8y<e7SpyY&r5z<hv|@;~6T<%iSog%ap$;L%VD6?7^md@@#9U&3qIb
zF3y9UsI;1(r<c<zRm!-GCh;hdM!Z65TMs;`U6M-fBk5qx&V>D9djGTokI1sO=n-yf
zs$9CUnjTNL`Iyi2&qm)sTP<3|45*7n&Ou<WE8@N6U$>>Z+=-wI8afM7YZtt-)0L+^
zfk93%QHJVR&zTOIg45-P&dh6&=4*`)9B?^O0O2vUa_R#yZ!I^1)u6J{6BDAgn8zF2
z0G|-iSvxn!xD3CT&WNhQcp5f9?gaNlMZ&9!XgE#Hy-hqNrsKoWZGvPk^ngrUck0U_
ztkVqN8VQ<}|MwIjD6oHCo3d{b2cKz(wmlCXYy6QMlrNv-(+DS*-ih^Z9p)TsiBZrY
znIqAakvA1&<9;O{HxHAVXAlMNI~m5GnU7qLjoWX;-gk{1Zp9tnhn=1TX5WwET3+)$
zom`La$lM=@?~#wFACsSi&zV|v14N&Ppn&Y-Qs0kMje3Qj`J3L*ZT+|X3kc`iTES0T
z#?*$NJNf(G?}vYG7c}yM@2R;T!?T!v525>|`ah3unFqb@H`(?OJ04e}H(#gdpZ6m#
zq6Q!1e(${RsJZ{b<n{9i@<1u*{&ZmAE8Mu#wf~Xp*4g_qK<NE+qY}dCb3?fN;Cg-1
zgYnV-vhx-A+}P94=pFR_ncMS7%eY?V=kwLfN1-VQ<M%S1^xTbU@Nu&mNyzE8^FA>t
zxbBOOZMAzq$%B7;WWd->)?M>iceIvU|8-3%_}lMmt5$5=_h7o!;2S__KQGAF_15lI
zjQBMqI5zu~Tk3c57P&v!+B&8oc;VA<Fk8_3+)dc)s{i%ock{{kKf4US^W5t4|4B9C
z|BX~*_Mf^8|68h=QOe55$^0kJO!f8kxmwY-*6TE|B2HGVuW+U;JF%pjH#V>c0ZsZq
zK#Uv-L9i-@aSKrABFnMUhARAavie`0oe4OU{}#u`z9q6R6Cq2s!C1>KVeFKhv5qA>
z$-a#xOJVGZxY;Gi5?MlINwTX@*|VhVd+w{-|6bGoKFq!IJTLF#;e6iTIj=MCIp5!T
z#;mxgGl&ahzZZw!gutqz&Aw^NrsBu)!t3m06_hBIihY)PZn~ho0OuAJdy4h!oFF4T
z-H(<wRaK~Nt9t;YhuR}<RrxcX0rE(w)YO`(zwUW5JGDS8Qi_N{*c$yGG7R1q^HC3A
z(boY<oe2a6x!bt9K<u5Z?X7G)j=tM?^9k`o`JoU22po<$Y2h}BbsE@mRI<0tSy(w3
zgjddySjDourlZo*PMSFg<mWX=Cv{e;B!^M>sa}z3iujoIk-7Kvr`FdpTHRS~gaZ#G
zJe>3E5=MGf9dbrQpWHRS;5IP4uJNvhkHPc+_I;uMYrSOML6MigM&MFVqZBXzI=r5z
z`9Az95leu_k{3%lr|=7dGRg4Iv^6SfQT3*%L(QmtZwJYXZ{$-P2RtG+OsC)EhVnq;
z%aYh@utz5F$HI(V8GR?tj)nX65^POit-k`DX;JXTjk6%kYSN{Pu>AqPmTkGxV@(w|
zDJU7yz9OF0MCnbTIeAGkykJE*s~PG|^KMc?GIDN3(yWQZ`y6DFy*?~;MaZm~hq&@n
zBuB|QcLS`f|K>VF!?m)$8!uq`^V|&@Wdn=rmmB0<dKuU8O&{I#y!)gt0~qB}ap^Z+
zr)f}h>4S9lT&}@CYbegK)i1I=s){e=3<#TvNT#hpY{-*?*`1uNiC5gsdv>#bbUKF+
z(S-`YJ**BKr-st@q)`5n8|Qz^jpXrg6I6Pt8Q3hz5M@LaZ_xR7!I6hn?N)s=*4<7B
zUAAoQ)l|km?lK9hSd&&vl~B7=FAZ}gl;B7SHG<(CDWUV^$CXfye<`8*DcCdX^6aJB
z!g<r23NxCrGj+XZ-i8xG@#N;)xK@7&qA6us2aT>il<_+@S_h9cwEUK|H+~7B4F3{B
z9W4E@sbfLDGGyEx&Nx+WBwT8+3*cokt>i1ck0X5_+TwNR;xHt_Wj~w|f`^k}p{E|d
zamz(zz&8e%gw6g`-z@wN+fph#$#^Nl2$hC1Kz00=kl|c;6+~z^VICYxOb+bD0hC&S
zgv_maJ)A#c$M$pZ^*(TIm6&7*hf}X_`Y#nbCH;QlWO`}}{3uC!|B)o03Aq9vyE&Ii
z!(M=GL;q6&3+JZ4ij5pM8zmcw+8ssF3sr=Uda{9TDhM;*a^pa4XUixvhP26GLrwzS
zS=2*tN}BHdXwyt0{;6oyfqo$#I=2iQ-6jh7{q&aBR>IrDlnMM7t)WW(h7Q_ph(IQ&
zK>e2p%0)=%@~lQDM7>J|FTz)wh=h`DF`Lx9=fQrx=ToR5K?T$hwYui<9>0R*L$qpi
z@a}v1w4y9M!%0dHcX5M47XeLw24zuch@e<iS|Wb!(veTkXbD)nSdsg~r}kcyvqFGB
z<%vNc)|0dDzjCm^Nq(KtdlNT$72Ly-(T=Fd_VT$y$ukX8wB)-|z4k@c9<0Q@praKX
z79THC-d-+N-M;dkMXorEn1^_>N!4x$2m5Ru0j`Oxl>01mFe5l&o8aT)Vw=}--!9MR
zm|o<Mt?We1?nxmUKY!1V*vdGVX7CD$RHyo4!vB1AzhY=|U+UE}mPV<d5xlcYh1}a6
zP}}$Y!KMA_S(kqpybbmf^^6SZ9BlN4Lvuu1at5Xjo-Qj_kbPP$y%7B5a@s9+J<D3;
zj?bj>BG&?_{5>jN7w+Y^PgnWa!Z?a$1WX4S!hK(FyfXprbX>D6l%s{2FrPjD6`$4w
zrw<%-U~l@>-rZ?qH~D4IOSe4bS@*W-xY(|Y$STQ}<n!51rVB0&jg2H6!I`r9N<ZgS
zrF$J~*x5cSNE7l-52I|D6Pc;AoXQ`3q++^jo9Ss$%3rQ-;ny_8T>j{Bf4F7RUOvtN
z%NW)`ssd*x&D`@Hp%&gBA{D|;B7+gz2GfIN<4x}Oz=QqqgFZx$dPmF!-JdfBYzEU@
zka_2EW0JT0(i-^9+T+S==O;kopYS1~xJ8#lB+Qb+?hZ_+4Xtlqzc-XnElzD<mX5Oc
zIxxe|Y2?o7i%mqs!qk$N)-}b$N-ENqTJV6~Pe4_^HRsHTq0jCemuD7*TfBCQHIe=V
zxQ~&Z2H&_2-Z&gmE6)FP->i7)-nqT@O;^vv4B1oi#=fhYyGHRQsr#&5F&Lus-iL_O
zx~N|nQOxW`&HKTLl>O<a@kyrARv0nc1dL?nc}nWFw%m31lA!qA{gi#(3%YoyZk=+y
zeh&#Xe32_AbO{j4Wxo5VOkMu<c4j`VYx{^VIJBd7g=eD_$9Tdow>yE~M&{(nOe}$n
zeXVFrW~(8M!_WB(3+r&+G0ze7+}~UAS)>m{Stg_>7f5sN3v)DO)zAk<yzviB@18EZ
zO-)3)5Ro9qxV5s}(G3-=BR-=b=V;TbSCBf$?6%Do<r3@I_u;L%S~R_GQDRibtz??d
z_p=>|zP)x<-s$R2&uG2NRb3F8)ciwr^g;tCG!CApJ*xhY7V7Fur13%@(ZwjN`k7k!
ziuDg{>LIs_#fy8mEhBBc55`Nrd}oqZ<smV;|EBEv;Y~iNaytgEJ8uc}I=vdSqTiK*
z6JrxyymeSU?!6bXge@ddd}m_kbK!1Y>Eya}5X5CKe2LvmEh<%O@_-T67`VJ3CdjVS
zU$}c6LHy7w*|qn{T3~)ctH$axmfI0N$W3?3?rj02ef~4KTZ^`#g6&`?giVRt!$8he
zEy+EpW(8VJTqx3|KuBRKP;t;9+FOtZS}MF3Jf>r-yj$OT7fVC%_LK8cle2zU5(dc+
zUwU8Zr{I>Uyilak%h#WneW|}x>!q-y&xGQ0&VX#~<a{x<A>HDp>v%;*6h(C&uCA{8
zuAe^yO(X{Wk&XH~n$mcIY{Um-Bkq&4k%zN|E1##0hbLT+-`c|S<gBC`H+r3_$7(fA
zEu#25(?T)ZbONy@@0Id3b7=29sLH$+kd?Su7GF2slPV9e`n(u_{l5AeLi`(r*kQ%|
z<WITS!m>$ZO}*ZR@-3N1wK50m+0_K9x2w=*vX(catmZy_m-KJ^4)*(IyOz#c-AulC
zCs(IvJ?34q`__|>O`#MaX&w6_uk<DinB-^~I}ocQw*|5I6QV4dlh@bAHp>$~n{FHm
z_%XlF8bUsa6*h#b-AQjS4J8z?aWskOE>eXzQ1Tx7e9wbE>o#RvpmgN&9F5z2F{ZHU
zqZuMfC6R7*BX=pScbPKt67Q!#{k|NbrmUe!(>e#NXM5E*$+4Yk#SCc;6K0kPG<NRV
z%e2x}rx!f&G^!tdTvzY9EKvW9rL%!0=B{XkD^i}}RXJfM;c6BhQT(mojh7zqvcAiF
zZO(O1UF0%69Z7fC_g4189aJTZv+8OO+9A)Y*F5=>V>Avsa#iWWVlre68+%OyJuFCK
zL03YnqsJ&|O|sKmcX-_+JMhBvYQRneGyJQL$eC&14TBnoHpVsTgi?KdKa&|EXb1PF
z4!l}gazseU*B!S<93x?zI5$Ts0<tWvB{skLby`i=5_j9@M{ZD5-IK-3vhvBIo(oWg
z)r*)+ha25^9CPd5r@@Myr@?rK|F9E((Xw8Iv^+O0Uu7e!+r~Govur?8^wjg(f>}~s
znM(ie_yF?(Q!@9)4vBsvVQtawm^fL@1W#yzCabfV1Ot29AsPGIXw4`}oyn_7;)fBa
z%*6;A9ahIm*88Kefido^gUw-E6nyNvbgd2xHE*qDmXyuP3@65Mb2sBTsy?bMuZs<`
zSFRKmbN<-q&6Cm@Rof+q?mK^W>l<$Vxkz@W{7U($Rn|GsgC^mCSjxOZL#0rwR+1@P
z+*VD(QMDaPhOQ0n6mb=EU+6s9xealGWvzM~iGcX^qHXzWA@OlR3ktHYzKW5Bne1;8
z%J--f)ou?5?!2T~Md6K0kKs<~OYwBk*y`=H58wXc%(Cr-6%K>G@`&TTUH-z`a`UZV
z!&NwTxN>rZu^x+#J}%1uIm|`c`p1uZUm_WbDpZGKrP8jlv}oOTl5&Rp^xCO8R}51x
z6o5r+ws*!AwK}kw-$RjNi}g(0;WY&Ja`?Ptv*>f={@ScOZb;II&q_y{`VjTm{~CCD
zlK%1*&(K`zI)`fNGvRdla+;~7%}1}qMju|rh7bCSuQ5rXYN60I=5fi2t0v}Flp8bB
zRfK-*#uGJd$XoBAg9WAP=FzDf)2z>Ze*(oxLf<Zj|Bir+tIo@F!^VDuGM*h$bxL#h
z=5B;v3(fqwq=uC!nppcG;p$+Coe_3vJCgT-jt<f#R?Gb>-AZ?^AoTe`*~5@a2F66)
zdcshrXu}U{?X|sN{WiL_k%L#7o0QeVhkrZ~lU=kfM*%gd1h8d;>!f_-WaDXpVK%bS
zQrD!CBR_wZBLb=a-8Td$<HLD_rkIS!6cj%$Sc-c&6vT{c_`YagX0njz67UBf+;QKG
z^Zp@|woS56q~(!g>&qIoIlob&UuHJ6247ua%MoWPv;G>4L~=#bWbr|yvrC%Dd{rPh
z3sMX`mAi&U1ZUeXE8TMLlgP<Wk*@AHb0B{}EBGB*cQ9n}th!Nj%m6-47f4Ec!<tf9
zjadj;OLghJ_WsOgRA$J8*Wk-D;r}4tccNH4a3bY#sV1l+n3@WK+DrNzNN~Myw2)pc
zm3z?IIg1F$lFJgqa`}oga>;b|^_-|icHThLFiK-wvb_|awQx6j(ULX6&<Wh&OCPuq
zKIrh#Z3=(Hc9`B}ksT}PQ`3tXyLAELvduVHo;t!l@(D8fJdv0cUuVtJFAY2J7^z~d
zs=>q;KR1IE^y`^FI5$43w91z4(TxAz_j3T|@TGAd+afwYO8c{(lDSY2%Yy8~F2qeg
z4?n~$3!U6m(-7dJcj#+8U(3QAuJFWTKYp3qE|VgCFrO0hVE^adS@bH1%tgqX7l?TT
za8Ugf^Zy~frg6<jn{y(j<jCPob)3`;jxoe$l}2&IFzOA8{0~v4W9lu-*Nj>d)s3FE
zAt7HVx6f=WkL&+9=ecP~F%aWH7vU0dcxd}w_c5=Ea8pz8lsHM<;+q&@e&tfxr16}7
zNn)8S{{%Z^mnub1k7nrbM$hJqp6!Jlyz01jq2_A~p1FN=OrPw0OP-igv*WqR-RU4m
z<f(oCF7?bf>;sO~^|}es@%;0QN#!8q{rt5)9)00hDg_b7w<{sdcNG{`<1HR$y}IhP
z=BWK;o{1}eYjODA#a6e3*^CGTxmBi+rc%^ug3y^9p7>Iun?XY4U{B}t6a{2|aE#;?
zsabWMk}d>iUaEe8r{|lz&x~p!p{ss}g_&ci?POItsHKOVnU01PRjH7I;;qV+T@rN9
zPnkpD`9OWw0v!ML{QvMi|GVLpYM}KX0p0@Ca&~#0ZM!AsIaW&~fKq6v>S9&R#ZONw
zc&iG<p)wF@sA9N;j62*IVFp9EJFJjFF0fU}k0`%0c(-k<>ZW3;Qf%y6SW$7))s-_|
z|2hR6m7Y=DRcNF21|m@^B?7@yk##V=trDnWsp0Q-%~Bh-l0(#%*D0$IGXR^A|7IVR
zE|I4wn-Tc)XMn8LDhWQ9`9hTK(dJ_;Y$hDw0S9;;jgkC1`TYeCMECoQ9P~}@==4_?
z?8bFG5Kfx4r2Z{$5)c;d25@KonnU^1)J>z9$Rh(s2k~!H!0&=`LuCFk^{a@()E@9h
z|9+98U0Me4bF>EC|6`IK&E({tCXe_0{vY#<Xy!!#N1$Etaq|L@lL7x3`UzHQE^z^R
zTQzP+t*yVJb>!mclL2rzP&uCz#{V#urA9-00BbqFHVPhRXe*w$2sm62Q%17}fe|yn
za0CwYb=+1ug&~9~qoy7uec`_{mVvwegkZqoP)r&9Hj)B{K*d1$YY6MOuMn_P7{VAc
zjIjg|z};yN9O%BH)jWkEf+<6S6UQEh4FsaS@h4;GG=?aqj6s9aMloOwSN^sR0gqGG
zA%-bqrc&1waSH@0%KdX4FHd8LW6D4uygMHAfcaC_0gT~c_CJU|F>{>3wR{Q#n4iIn
zfj)S0oT2#r6b3Mcf*AvS)A4ae>BcDxU<)#44D^k$#~Dsrr!atRvY0W@WwGOo0RpU(
za|s}lVa7mLppG-T2~S}FdK6|1bZO)`BbE3R1|XDR#z5CNjx&TvPGJCw2WAX(t>8FA
z<IE`xKsmsSf!^;w&R9N;0d)T{W1#oHk2B=Yp16*Gcf&E`pm(;8b9yOG<oxaV7c(Au
z<$9cFPJ06HZ*>laG_;!OIPE(9Z`%K98NDVrZn=lyw`Ghyga-`y9la|tfq=(oPzxjQ
HXaf2tRZAf3

literal 0
HcmV?d00001


From 6e3b49c522c6cfffe6b797b63315ea6391ba66e4 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 18 May 2016 12:04:46 -0400
Subject: [PATCH 0337/1311] Fix inequality symbol in test assertion

This commit fixes the inequality symbol used in a test assertion in
RepositoryS3SettingsTests#testInvalidChunkBufferSizeRepositorySettings. The
inequality symbol was previously backwards but fixed in commit
cad0608cdb28e2b8485e5c01c26579a35cb84356 but fixing the inequality
symbol here was missed in that commit.

Closes #18449
---
 .../org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java
index c5c617e8591..4cb8e4d3abb 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java
@@ -316,7 +316,7 @@ public class RepositoryS3SettingsTests extends ESTestCase {
             "Failed to parse value [4mb] for setting [buffer_size] must be >= 5mb");
         // chunk > 5tb should fail
         internalTestInvalidChunkBufferSizeSettings(new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(6, ByteSizeUnit.TB),
-            "Failed to parse value [6tb] for setting [chunk_size] must be =< 5tb");
+            "Failed to parse value [6tb] for setting [chunk_size] must be <= 5tb");
     }
 
     private Settings buildSettings(Settings... global) {

From ee4e470f6000fda030ad1f6d9216efec70119227 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 18 May 2016 15:10:01 -0400
Subject: [PATCH 0338/1311] Add a wait_for_stats=yellow to a docs snippet

It was making unstable tests.
---
 docs/reference/query-dsl/parent-id-query.asciidoc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/reference/query-dsl/parent-id-query.asciidoc b/docs/reference/query-dsl/parent-id-query.asciidoc
index 713e19c26ce..a29073dafa9 100644
--- a/docs/reference/query-dsl/parent-id-query.asciidoc
+++ b/docs/reference/query-dsl/parent-id-query.asciidoc
@@ -28,6 +28,8 @@ PUT /my_index
         }
     }
 }
+
+GET /_cluster/health?wait_for_status=yellow
 ------------------------------------------
 // CONSOLE
 // TESTSETUP
@@ -73,7 +75,7 @@ This query has two required parameters:
 
 `id`::    The required parent id select documents must referrer to.
 
-`ignore_unmapped`::  When set to `true` this will ignore an unmapped `type` and will not match any 
+`ignore_unmapped`::  When set to `true` this will ignore an unmapped `type` and will not match any
 documents for this query. This can be useful when querying multiple indexes
 which might have different mappings. When set to `false` (the default value)
 the query will throw an exception if the `type` is not mapped.

From 9a9301f7d8acd94f621e4bf88522c6e58cf63af2 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Wed, 18 May 2016 23:00:57 +0200
Subject: [PATCH 0339/1311] Remove dead BloomFilter code

We don't use this class for a quite a while. lets trash it.
---
 .../common/util/BloomFilter.java              | 629 ------------------
 1 file changed, 629 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/common/util/BloomFilter.java

diff --git a/core/src/main/java/org/elasticsearch/common/util/BloomFilter.java b/core/src/main/java/org/elasticsearch/common/util/BloomFilter.java
deleted file mode 100644
index 6c471cddb55..00000000000
--- a/core/src/main/java/org/elasticsearch/common/util/BloomFilter.java
+++ /dev/null
@@ -1,629 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.common.util;
-
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.RamUsageEstimator;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.hash.MurmurHash3;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.unit.SizeValue;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Comparator;
-
-/**
- * A bloom filter. Inspired by Guava bloom filter implementation though with some optimizations.
- */
-public class BloomFilter {
-
-    /**
-     * A factory that can use different fpp based on size.
-     */
-    public static class Factory {
-
-        public static final Factory DEFAULT = buildDefault();
-
-        private static Factory buildDefault() {
-            // Some numbers:
-            // 10k =0.001: 140.4kb  , 10 Hashes
-            // 10k =0.01 :  93.6kb  ,  6 Hashes
-            // 100k=0.01 : 936.0kb  ,  6 Hashes
-            // 100k=0.03 : 712.7kb  ,  5 Hashes
-            // 500k=0.01 :   4.5mb  ,  6 Hashes
-            // 500k=0.03 :   3.4mb  ,  5 Hashes
-            // 500k=0.05 :   2.9mb  ,  4 Hashes
-            //   1m=0.01 :   9.1mb  ,  6 Hashes
-            //   1m=0.03 :   6.9mb  ,  5 Hashes
-            //   1m=0.05 :   5.9mb  ,  4 Hashes
-            //   5m=0.01 :  45.7mb  ,  6 Hashes
-            //   5m=0.03 :  34.8mb  ,  5 Hashes
-            //   5m=0.05 :  29.7mb  ,  4 Hashes
-            //  50m=0.01 : 457.0mb  ,  6 Hashes
-            //  50m=0.03 : 297.3mb  ,  4 Hashes
-            //  50m=0.10 : 228.5mb  ,  3 Hashes
-            return buildFromString("10k=0.01,1m=0.03");
-        }
-
-        /**
-         * Supports just passing fpp, as in "0.01", and also ranges, like "50k=0.01,1m=0.05". If
-         * its null, returns {@link #buildDefault()}.
-         */
-        public static Factory buildFromString(@Nullable String config) {
-            if (config == null) {
-                return buildDefault();
-            }
-            String[] sEntries = config.split(",");
-            if (sEntries.length == 0) {
-                if (config.length() > 0) {
-                    return new Factory(new Entry[]{new Entry(0, Double.parseDouble(config))});
-                }
-                return buildDefault();
-            }
-            Entry[] entries = new Entry[sEntries.length];
-            for (int i = 0; i < sEntries.length; i++) {
-                int index = sEntries[i].indexOf('=');
-                entries[i] = new Entry(
-                        (int) SizeValue.parseSizeValue(sEntries[i].substring(0, index).trim()).singles(),
-                        Double.parseDouble(sEntries[i].substring(index + 1).trim())
-                );
-            }
-            return new Factory(entries);
-        }
-
-        private final Entry[] entries;
-
-        public Factory(Entry[] entries) {
-            this.entries = entries;
-            // the order is from the upper most expected insertions to the lowest
-            Arrays.sort(this.entries, new Comparator<Entry>() {
-                @Override
-                public int compare(Entry o1, Entry o2) {
-                    return o2.expectedInsertions - o1.expectedInsertions;
-                }
-            });
-        }
-
-        public BloomFilter createFilter(int expectedInsertions) {
-            for (Entry entry : entries) {
-                if (expectedInsertions > entry.expectedInsertions) {
-                    return BloomFilter.create(expectedInsertions, entry.fpp);
-                }
-            }
-            return BloomFilter.create(expectedInsertions, 0.03);
-        }
-
-        public static class Entry {
-            public final int expectedInsertions;
-            public final double fpp;
-
-            Entry(int expectedInsertions, double fpp) {
-                this.expectedInsertions = expectedInsertions;
-                this.fpp = fpp;
-            }
-        }
-    }
-
-    /**
-     * Creates a bloom filter based on the with the expected number
-     * of insertions and expected false positive probability.
-     *
-     * @param expectedInsertions the number of expected insertions to the constructed
-     * @param fpp                the desired false positive probability (must be positive and less than 1.0)
-     */
-    public static BloomFilter create(int expectedInsertions, double fpp) {
-        return create(expectedInsertions, fpp, -1);
-    }
-
-    /**
-     * Creates a bloom filter based on the expected number of insertions, expected false positive probability,
-     * and number of hash functions.
-     *
-     * @param expectedInsertions the number of expected insertions to the constructed
-     * @param fpp                the desired false positive probability (must be positive and less than 1.0)
-     * @param numHashFunctions   the number of hash functions to use (must be less than or equal to 255)
-     */
-    public static BloomFilter create(int expectedInsertions, double fpp, int numHashFunctions) {
-        if (expectedInsertions == 0) {
-            expectedInsertions = 1;
-        }
-        /*
-         * TODO(user): Put a warning in the javadoc about tiny fpp values,
-         * since the resulting size is proportional to -log(p), but there is not
-         * much of a point after all, e.g. optimalM(1000, 0.0000000000000001) = 76680
-         * which is less that 10kb. Who cares!
-         */
-        long numBits = optimalNumOfBits(expectedInsertions, fpp);
-
-        // calculate the optimal number of hash functions
-        if (numHashFunctions == -1) {
-            numHashFunctions = optimalNumOfHashFunctions(expectedInsertions, numBits);
-        }
-
-        try {
-            return new BloomFilter(new BitArray(numBits), numHashFunctions, Hashing.DEFAULT);
-        } catch (IllegalArgumentException e) {
-            throw new IllegalArgumentException("Could not create BloomFilter of " + numBits + " bits", e);
-        }
-    }
-
-    public static void skipBloom(IndexInput in) throws IOException {
-        int version = in.readInt(); // we do nothing with this now..., defaults to 0
-        final int numLongs = in.readInt();
-        in.seek(in.getFilePointer() + (numLongs * 8) + 4 + 4); // filter + numberOfHashFunctions + hashType
-    }
-
-    public static BloomFilter deserialize(DataInput in) throws IOException {
-        int version = in.readInt(); // we do nothing with this now..., defaults to 0
-        int numLongs = in.readInt();
-        long[] data = new long[numLongs];
-        for (int i = 0; i < numLongs; i++) {
-            data[i] = in.readLong();
-        }
-        int numberOfHashFunctions = in.readInt();
-        int hashType = in.readInt();
-        return new BloomFilter(new BitArray(data), numberOfHashFunctions, Hashing.fromType(hashType));
-    }
-
-    public static void serilaize(BloomFilter filter, DataOutput out) throws IOException {
-        out.writeInt(0); // version
-        BitArray bits = filter.bits;
-        out.writeInt(bits.data.length);
-        for (long l : bits.data) {
-            out.writeLong(l);
-        }
-        out.writeInt(filter.numHashFunctions);
-        out.writeInt(filter.hashing.type()); // hashType
-    }
-
-    public static BloomFilter readFrom(StreamInput in) throws IOException {
-        int version = in.readVInt(); // we do nothing with this now..., defaults to 0
-        int numLongs = in.readVInt();
-        long[] data = new long[numLongs];
-        for (int i = 0; i < numLongs; i++) {
-            data[i] = in.readLong();
-        }
-        int numberOfHashFunctions = in.readVInt();
-        int hashType = in.readVInt(); // again, nothing to do now...
-        return new BloomFilter(new BitArray(data), numberOfHashFunctions, Hashing.fromType(hashType));
-    }
-
-    public static void writeTo(BloomFilter filter, StreamOutput out) throws IOException {
-        out.writeVInt(0); // version
-        BitArray bits = filter.bits;
-        out.writeVInt(bits.data.length);
-        for (long l : bits.data) {
-            out.writeLong(l);
-        }
-        out.writeVInt(filter.numHashFunctions);
-        out.writeVInt(filter.hashing.type()); // hashType
-    }
-
-    /**
-     * The bit set of the BloomFilter (not necessarily power of 2!)
-     */
-    final BitArray bits;
-    /**
-     * Number of hashes per element
-     */
-    final int numHashFunctions;
-
-    final Hashing hashing;
-
-    BloomFilter(BitArray bits, int numHashFunctions, Hashing hashing) {
-        this.bits = bits;
-        this.numHashFunctions = numHashFunctions;
-        this.hashing = hashing;
-    /*
-     * This only exists to forbid BFs that cannot use the compact persistent representation.
-     * If it ever throws, at a user who was not intending to use that representation, we should
-     * reconsider
-     */
-        if (numHashFunctions > 255) {
-            throw new IllegalArgumentException("Currently we don't allow BloomFilters that would use more than 255 hash functions");
-        }
-    }
-
-    public boolean put(BytesRef value) {
-        return hashing.put(value, numHashFunctions, bits);
-    }
-
-    public boolean mightContain(BytesRef value) {
-        return hashing.mightContain(value, numHashFunctions, bits);
-    }
-
-    public int getNumHashFunctions() {
-        return this.numHashFunctions;
-    }
-
-    public long getSizeInBytes() {
-        return bits.ramBytesUsed();
-    }
-
-    @Override
-    public int hashCode() {
-        return bits.hashCode() + numHashFunctions;
-    }
-
-  /*
-   * Cheat sheet:
-   *
-   * m: total bits
-   * n: expected insertions
-   * b: m/n, bits per insertion
-
-   * p: expected false positive probability
-   *
-   * 1) Optimal k = b * ln2
-   * 2) p = (1 - e ^ (-kn/m))^k
-   * 3) For optimal k: p = 2 ^ (-k) ~= 0.6185^b
-   * 4) For optimal k: m = -nlnp / ((ln2) ^ 2)
-   */
-
-    /**
-     * Computes the optimal k (number of hashes per element inserted in Bloom filter), given the
-     * expected insertions and total number of bits in the Bloom filter.
-     * <p>
-     * See http://en.wikipedia.org/wiki/File:Bloom_filter_fp_probability.svg for the formula.
-     *
-     * @param n expected insertions (must be positive)
-     * @param m total number of bits in Bloom filter (must be positive)
-     */
-    static int optimalNumOfHashFunctions(long n, long m) {
-        return Math.max(1, (int) Math.round(m / n * Math.log(2)));
-    }
-
-    /**
-     * Computes m (total bits of Bloom filter) which is expected to achieve, for the specified
-     * expected insertions, the required false positive probability.
-     * <p>
-     * See http://en.wikipedia.org/wiki/Bloom_filter#Probability_of_false_positives for the formula.
-     *
-     * @param n expected insertions (must be positive)
-     * @param p false positive rate (must be 0 &lt; p &lt; 1)
-     */
-    static long optimalNumOfBits(long n, double p) {
-        if (p == 0) {
-            p = Double.MIN_VALUE;
-        }
-        return (long) (-n * Math.log(p) / (Math.log(2) * Math.log(2)));
-    }
-
-    // Note: We use this instead of java.util.BitSet because we need access to the long[] data field
-    static final class BitArray {
-        final long[] data;
-        final long bitSize;
-        long bitCount;
-
-        BitArray(long bits) {
-            this(new long[size(bits)]);
-        }
-
-        private static int size(long bits) {
-            long quotient = bits / 64;
-            long remainder = bits - quotient * 64;
-            return Math.toIntExact(remainder == 0 ? quotient : 1 + quotient);
-        }
-
-        // Used by serialization
-        BitArray(long[] data) {
-            this.data = data;
-            long bitCount = 0;
-            for (long value : data) {
-                bitCount += Long.bitCount(value);
-            }
-            this.bitCount = bitCount;
-            this.bitSize = data.length * Long.SIZE;
-        }
-
-        /** Returns true if the bit changed value. */
-        boolean set(long index) {
-            if (!get(index)) {
-                data[(int) (index >>> 6)] |= (1L << index);
-                bitCount++;
-                return true;
-            }
-            return false;
-        }
-
-        boolean get(long index) {
-            return (data[(int) (index >>> 6)] & (1L << index)) != 0;
-        }
-
-        /** Number of bits */
-        long bitSize() {
-            return bitSize;
-        }
-
-        /** Number of set bits (1s) */
-        long bitCount() {
-            return bitCount;
-        }
-
-        BitArray copy() {
-            return new BitArray(data.clone());
-        }
-
-        /** Combines the two BitArrays using bitwise OR. */
-        void putAll(BitArray array) {
-            bitCount = 0;
-            for (int i = 0; i < data.length; i++) {
-                data[i] |= array.data[i];
-                bitCount += Long.bitCount(data[i]);
-            }
-        }
-
-        @Override public boolean equals(Object o) {
-            if (o instanceof BitArray) {
-                BitArray bitArray = (BitArray) o;
-                return Arrays.equals(data, bitArray.data);
-            }
-            return false;
-        }
-
-        @Override public int hashCode() {
-            return Arrays.hashCode(data);
-        }
-
-        public long ramBytesUsed() {
-            return Long.BYTES * data.length + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + 16;
-        }
-    }
-
-    static enum Hashing {
-
-        V0() {
-            @Override
-            protected boolean put(BytesRef value, int numHashFunctions, BitArray bits) {
-                long bitSize = bits.bitSize();
-                long hash64 = hash3_x64_128(value.bytes, value.offset, value.length, 0);
-                int hash1 = (int) hash64;
-                int hash2 = (int) (hash64 >>> 32);
-                boolean bitsChanged = false;
-                for (int i = 1; i <= numHashFunctions; i++) {
-                    int nextHash = hash1 + i * hash2;
-                    if (nextHash < 0) {
-                        nextHash = ~nextHash;
-                    }
-                    bitsChanged |= bits.set(nextHash % bitSize);
-                }
-                return bitsChanged;
-            }
-
-            @Override
-            protected boolean mightContain(BytesRef value, int numHashFunctions, BitArray bits) {
-                long bitSize = bits.bitSize();
-                long hash64 = hash3_x64_128(value.bytes, value.offset, value.length, 0);
-                int hash1 = (int) hash64;
-                int hash2 = (int) (hash64 >>> 32);
-                for (int i = 1; i <= numHashFunctions; i++) {
-                    int nextHash = hash1 + i * hash2;
-                    if (nextHash < 0) {
-                        nextHash = ~nextHash;
-                    }
-                    if (!bits.get(nextHash % bitSize)) {
-                        return false;
-                    }
-                }
-                return true;
-            }
-
-            @Override
-            protected int type() {
-                return 0;
-            }
-        },
-        V1() {
-            @Override
-            protected boolean put(BytesRef value, int numHashFunctions, BitArray bits) {
-                long bitSize = bits.bitSize();
-                MurmurHash3.Hash128 hash128 = MurmurHash3.hash128(value.bytes, value.offset, value.length, 0, new MurmurHash3.Hash128());
-
-                boolean bitsChanged = false;
-                long combinedHash = hash128.h1;
-                for (int i = 0; i < numHashFunctions; i++) {
-                    // Make the combined hash positive and indexable
-                    bitsChanged |= bits.set((combinedHash & Long.MAX_VALUE) % bitSize);
-                    combinedHash += hash128.h2;
-                }
-                return bitsChanged;
-            }
-
-            @Override
-            protected boolean mightContain(BytesRef value, int numHashFunctions, BitArray bits) {
-                long bitSize = bits.bitSize();
-                MurmurHash3.Hash128 hash128 = MurmurHash3.hash128(value.bytes, value.offset, value.length, 0, new MurmurHash3.Hash128());
-
-                long combinedHash = hash128.h1;
-                for (int i = 0; i < numHashFunctions; i++) {
-                    // Make the combined hash positive and indexable
-                    if (!bits.get((combinedHash & Long.MAX_VALUE) % bitSize)) {
-                        return false;
-                    }
-                    combinedHash += hash128.h2;
-                }
-                return true;
-            }
-
-            @Override
-            protected int type() {
-                return 1;
-            }
-        }
-        ;
-
-        protected abstract boolean put(BytesRef value, int numHashFunctions, BitArray bits);
-
-        protected abstract boolean mightContain(BytesRef value, int numHashFunctions, BitArray bits);
-
-        protected abstract int type();
-
-        public static final Hashing DEFAULT = Hashing.V1;
-
-        public static Hashing fromType(int type) {
-            if (type == 0) {
-                return Hashing.V0;
-            } if (type == 1) {
-                return Hashing.V1;
-            } else {
-                throw new IllegalArgumentException("no hashing type matching " + type);
-            }
-        }
-    }
-
-    // START : MURMUR 3_128 USED FOR Hashing.V0
-    // NOTE: don't replace this code with the o.e.common.hashing.MurmurHash3 method which returns a different hash
-
-    protected static long getblock(byte[] key, int offset, int index) {
-        int i_8 = index << 3;
-        int blockOffset = offset + i_8;
-        return ((long) key[blockOffset + 0] & 0xff) + (((long) key[blockOffset + 1] & 0xff) << 8) +
-                (((long) key[blockOffset + 2] & 0xff) << 16) + (((long) key[blockOffset + 3] & 0xff) << 24) +
-                (((long) key[blockOffset + 4] & 0xff) << 32) + (((long) key[blockOffset + 5] & 0xff) << 40) +
-                (((long) key[blockOffset + 6] & 0xff) << 48) + (((long) key[blockOffset + 7] & 0xff) << 56);
-    }
-
-    protected static long rotl64(long v, int n) {
-        return ((v << n) | (v >>> (64 - n)));
-    }
-
-    protected static long fmix(long k) {
-        k ^= k >>> 33;
-        k *= 0xff51afd7ed558ccdL;
-        k ^= k >>> 33;
-        k *= 0xc4ceb9fe1a85ec53L;
-        k ^= k >>> 33;
-
-        return k;
-    }
-
-    @SuppressWarnings("fallthrough") // Uses fallthrough to implement a well know hashing algorithm
-    public static long hash3_x64_128(byte[] key, int offset, int length, long seed) {
-        final int nblocks = length >> 4; // Process as 128-bit blocks.
-
-        long h1 = seed;
-        long h2 = seed;
-
-        long c1 = 0x87c37b91114253d5L;
-        long c2 = 0x4cf5ad432745937fL;
-
-        //----------
-        // body
-
-        for (int i = 0; i < nblocks; i++) {
-            long k1 = getblock(key, offset, i * 2 + 0);
-            long k2 = getblock(key, offset, i * 2 + 1);
-
-            k1 *= c1;
-            k1 = rotl64(k1, 31);
-            k1 *= c2;
-            h1 ^= k1;
-
-            h1 = rotl64(h1, 27);
-            h1 += h2;
-            h1 = h1 * 5 + 0x52dce729;
-
-            k2 *= c2;
-            k2 = rotl64(k2, 33);
-            k2 *= c1;
-            h2 ^= k2;
-
-            h2 = rotl64(h2, 31);
-            h2 += h1;
-            h2 = h2 * 5 + 0x38495ab5;
-        }
-
-        //----------
-        // tail
-
-        // Advance offset to the unprocessed tail of the data.
-        offset += nblocks * 16;
-
-        long k1 = 0;
-        long k2 = 0;
-
-        switch (length & 15) {
-            case 15:
-                k2 ^= ((long) key[offset + 14]) << 48;
-            case 14:
-                k2 ^= ((long) key[offset + 13]) << 40;
-            case 13:
-                k2 ^= ((long) key[offset + 12]) << 32;
-            case 12:
-                k2 ^= ((long) key[offset + 11]) << 24;
-            case 11:
-                k2 ^= ((long) key[offset + 10]) << 16;
-            case 10:
-                k2 ^= ((long) key[offset + 9]) << 8;
-            case 9:
-                k2 ^= ((long) key[offset + 8]) << 0;
-                k2 *= c2;
-                k2 = rotl64(k2, 33);
-                k2 *= c1;
-                h2 ^= k2;
-
-            case 8:
-                k1 ^= ((long) key[offset + 7]) << 56;
-            case 7:
-                k1 ^= ((long) key[offset + 6]) << 48;
-            case 6:
-                k1 ^= ((long) key[offset + 5]) << 40;
-            case 5:
-                k1 ^= ((long) key[offset + 4]) << 32;
-            case 4:
-                k1 ^= ((long) key[offset + 3]) << 24;
-            case 3:
-                k1 ^= ((long) key[offset + 2]) << 16;
-            case 2:
-                k1 ^= ((long) key[offset + 1]) << 8;
-            case 1:
-                k1 ^= (key[offset]);
-                k1 *= c1;
-                k1 = rotl64(k1, 31);
-                k1 *= c2;
-                h1 ^= k1;
-        }
-
-        //----------
-        // finalization
-
-        h1 ^= length;
-        h2 ^= length;
-
-        h1 += h2;
-        h2 += h1;
-
-        h1 = fmix(h1);
-        h2 = fmix(h2);
-
-        h1 += h2;
-        h2 += h1;
-
-        //return (new long[]{h1, h2});
-        // SAME AS GUAVA, they take the first long out of the 128bit
-        return h1;
-    }
-
-    // END: MURMUR 3_128
-}

From 27e6908c8d9b0f69d219c54a9162f1da4b036211 Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Thu, 19 May 2016 09:33:29 +0200
Subject: [PATCH 0340/1311] Add indent

---
 .../search/request/named-queries-and-filters.asciidoc         | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/reference/search/request/named-queries-and-filters.asciidoc b/docs/reference/search/request/named-queries-and-filters.asciidoc
index f8be0f1be69..0fb60253938 100644
--- a/docs/reference/search/request/named-queries-and-filters.asciidoc
+++ b/docs/reference/search/request/named-queries-and-filters.asciidoc
@@ -10,8 +10,8 @@ GET /_search
     "query": {
         "bool" : {
             "should" : [
-            {"match" : { "name.first" : {"query" : "shay", "_name" : "first"} }},
-            {"match" : { "name.last" : {"query" : "banon", "_name" : "last"} }}
+                {"match" : { "name.first" : {"query" : "shay", "_name" : "first"} }},
+                {"match" : { "name.last" : {"query" : "banon", "_name" : "last"} }}
             ],
             "filter" : {
                 "terms" : {

From 2b972f1f75c03eaeca28abbaf912170b30cec732 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Thu, 19 May 2016 09:40:10 +0200
Subject: [PATCH 0341/1311] FSync translog outside of the writers global lock
 (#18360)

FSync translog outside of the writers global lock

Today we aquire a write global lock that blocks all modification to the
translog file while we fsync / checkpoint the file. Yet, we don't necessarily
needt to block concurrent operations here. This can lead to a lot of blocked
threads if the machine has high concurrency (lot os CPUs) but uses slow disks
(spinning disks) which is absolutely unnecessary. We just need to protect from
fsyncing / checkpointing concurrently but we can fill buffers and write to the
underlying file in a concurrent fashion.

This change introduces an additional lock that we hold while fsyncing but moves
the checkpointing code outside of the writers global lock.
---
 .../index/translog/TranslogWriter.java        | 64 +++++++++++--------
 1 file changed, 37 insertions(+), 27 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java b/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
index e215669761c..84278fa92b3 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
@@ -39,6 +39,7 @@ import java.nio.file.OpenOption;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantLock;
 
 public class TranslogWriter extends BaseTranslogReader implements Closeable {
 
@@ -60,7 +61,8 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
     private volatile long totalOffset;
 
     protected final AtomicBoolean closed = new AtomicBoolean(false);
-
+    // lock order synchronized(syncLock) -> synchronized(this)
+    private final Object syncLock = new Object();
 
     public TranslogWriter(ShardId shardId, long generation, FileChannel channel, Path path, ByteSizeValue bufferSize) throws IOException {
         super(generation, channel, path, channel.position());
@@ -146,23 +148,7 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
      * raising the exception.
      */
     public void sync() throws IOException {
-        if (syncNeeded()) {
-            synchronized (this) {
-                ensureOpen();
-                final long offsetToSync;
-                final int opsCounter;
-                try {
-                    outputStream.flush();
-                    offsetToSync = totalOffset;
-                    opsCounter = operationCounter;
-                    checkpoint(offsetToSync, opsCounter, generation, channel, path);
-                } catch (Throwable ex) {
-                    closeWithTragicEvent(ex);
-                    throw ex;
-                }
-                lastSyncedOffset = offsetToSync;
-            }
-        }
+        syncUpTo(Long.MAX_VALUE);
     }
 
     /**
@@ -229,9 +215,38 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
      * @return <code>true</code> if this call caused an actual sync operation
      */
     public boolean syncUpTo(long offset) throws IOException {
-        if (lastSyncedOffset < offset) {
-            sync();
-            return true;
+        if (lastSyncedOffset < offset && syncNeeded()) {
+            synchronized (syncLock) { // only one sync/checkpoint should happen concurrently but we wait
+                if (lastSyncedOffset < offset && syncNeeded()) {
+                    // double checked locking - we don't want to fsync unless we have to and now that we have
+                    // the lock we should check again since if this code is busy we might have fsynced enough already
+                    final long offsetToSync;
+                    final int opsCounter;
+                    synchronized (this) {
+                        ensureOpen();
+                        try {
+                            outputStream.flush();
+                            offsetToSync = totalOffset;
+                            opsCounter = operationCounter;
+                        } catch (Throwable ex) {
+                            closeWithTragicEvent(ex);
+                            throw ex;
+                        }
+                    }
+                    // now do the actual fsync outside of the synchronized block such that
+                    // we can continue writing to the buffer etc.
+                    try {
+                        channel.force(false);
+                        writeCheckpoint(offsetToSync, opsCounter, path.getParent(), generation, StandardOpenOption.WRITE);
+                    } catch (Throwable ex) {
+                        closeWithTragicEvent(ex);
+                        throw ex;
+                    }
+                    assert lastSyncedOffset <= offsetToSync : "illegal state: " + lastSyncedOffset + " <= " + offsetToSync;
+                    lastSyncedOffset = offsetToSync; // write protected by syncLock
+                    return true;
+                }
+            }
         }
         return false;
     }
@@ -254,11 +269,6 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
         Channels.readFromFileChannelWithEofException(channel, position, targetBuffer);
     }
 
-    private synchronized void checkpoint(long lastSyncPosition, int operationCounter, long generation, FileChannel translogFileChannel, Path translogFilePath) throws IOException {
-        translogFileChannel.force(false);
-        writeCheckpoint(lastSyncPosition, operationCounter, translogFilePath.getParent(), generation, StandardOpenOption.WRITE);
-    }
-
     private static void writeCheckpoint(long syncPosition, int numOperations, Path translogFile, long generation, OpenOption... options) throws IOException {
         final Path checkpointFile = translogFile.resolve(Translog.CHECKPOINT_FILE_NAME);
         Checkpoint checkpoint = new Checkpoint(syncPosition, numOperations, generation);
@@ -269,7 +279,7 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
 
         static final ChannelFactory DEFAULT = new ChannelFactory();
 
-        // only for testing until we have a disk-full FileSystemt
+        // only for testing until we have a disk-full FileSystem
         public FileChannel open(Path file) throws IOException {
             return FileChannel.open(file, StandardOpenOption.WRITE, StandardOpenOption.READ, StandardOpenOption.CREATE_NEW);
         }

From d77c299cb9b2a55b47fff539b539ebb9846bd858 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Thu, 19 May 2016 10:42:35 +0200
Subject: [PATCH 0342/1311] Register `indices.query.bool.max_clause_count`
 setting (#18341)

* Register `indices.query.bool.max_clause_count` setting

This commit registers `indices.query.bool.max_clause_count` as a node
level setting and removes support for its synonym setting
`index.query.bool.max_clause_count`.

Closes #18336
---
 .../elasticsearch/common/settings/ClusterSettings.java   | 4 +++-
 .../elasticsearch/common/settings/SettingsModule.java    | 7 ++++++-
 .../main/java/org/elasticsearch/search/SearchModule.java | 6 ++++--
 .../common/settings/SettingsModuleTests.java             | 9 +++++++++
 docs/reference/migration/migrate_5_0/settings.asciidoc   | 6 ++++++
 5 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index e66534a4feb..5b6130281d4 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -87,6 +87,7 @@ import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.repositories.uri.URLRepository;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.Transport;
@@ -420,6 +421,7 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     ResourceWatcherService.ENABLED,
                     ResourceWatcherService.RELOAD_INTERVAL_HIGH,
                     ResourceWatcherService.RELOAD_INTERVAL_MEDIUM,
-                    ResourceWatcherService.RELOAD_INTERVAL_LOW
+                    ResourceWatcherService.RELOAD_INTERVAL_LOW,
+                    SearchModule.INDICES_MAX_CLAUSE_COUNT_SETTING
             )));
 }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java b/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
index 2e7acd6ae8c..2ed5ffd86cd 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
@@ -65,7 +65,12 @@ public class SettingsModule extends AbstractModule {
     protected void configure() {
         final IndexScopedSettings indexScopedSettings = new IndexScopedSettings(settings, new HashSet<>(this.indexSettings.values()));
         final ClusterSettings clusterSettings = new ClusterSettings(settings, new HashSet<>(this.nodeSettings.values()));
-        Settings indexSettings = settings.filter((s) -> s.startsWith("index.") && clusterSettings.get(s) == null);
+        Settings indexSettings = settings.filter((s) -> (s.startsWith("index.") &&
+            // special case - we want to get Did you mean indices.query.bool.max_clause_count
+            // which means we need to by-pass this check for this setting
+            // TODO remove in 6.0!!
+            "index.query.bool.max_clause_count".equals(s) == false)
+            && clusterSettings.get(s) == null);
         if (indexSettings.isEmpty() == false) {
             try {
                 String separator = IntStream.range(0, 85).mapToObj(s -> "*").collect(Collectors.joining("")).trim();
diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java
index 513a5c88a37..59e9c94c27d 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -29,6 +29,7 @@ import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.lucene.search.function.ScoreFunction;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ParseFieldRegistry;
 import org.elasticsearch.index.percolator.PercolatorHighlightSubFetchPhase;
@@ -290,6 +291,8 @@ public class SearchModule extends AbstractModule {
 
     private final Settings settings;
     private final NamedWriteableRegistry namedWriteableRegistry;
+    public static final Setting<Integer> INDICES_MAX_CLAUSE_COUNT_SETTING = Setting.intSetting("indices.query.bool.max_clause_count",
+        1024, 1, Integer.MAX_VALUE, Setting.Property.NodeScope);
 
     // pkg private so tests can mock
     Class<? extends SearchService> searchServiceImpl = SearchService.class;
@@ -650,8 +653,7 @@ public class SearchModule extends AbstractModule {
         registerQuery(MatchAllQueryBuilder::new, MatchAllQueryBuilder::fromXContent, MatchAllQueryBuilder.QUERY_NAME_FIELD);
         registerQuery(QueryStringQueryBuilder::new, QueryStringQueryBuilder::fromXContent, QueryStringQueryBuilder.QUERY_NAME_FIELD);
         registerQuery(BoostingQueryBuilder::new, BoostingQueryBuilder::fromXContent, BoostingQueryBuilder.QUERY_NAME_FIELD);
-        BooleanQuery.setMaxClauseCount(settings.getAsInt("index.query.bool.max_clause_count",
-                settings.getAsInt("indices.query.bool.max_clause_count", BooleanQuery.getMaxClauseCount())));
+        BooleanQuery.setMaxClauseCount(INDICES_MAX_CLAUSE_COUNT_SETTING.get(settings));
         registerQuery(BoolQueryBuilder::new, BoolQueryBuilder::fromXContent, BoolQueryBuilder.QUERY_NAME_FIELD);
         registerQuery(TermQueryBuilder::new, TermQueryBuilder::fromXContent, TermQueryBuilder.QUERY_NAME_FIELD);
         registerQuery(TermsQueryBuilder::new, TermsQueryBuilder::fromXContent, TermsQueryBuilder.QUERY_NAME_FIELD);
diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
index 977d5fa7b09..5e992fc947c 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
@@ -208,4 +208,13 @@ public class SettingsModuleTests extends ModuleTestCase {
             assertThat(e.getMessage(), containsString("Cannot register setting [foo.bar] twice"));
         }
     }
+
+    public void testOldMaxClauseCountSetting() {
+            Settings settings = Settings.builder().put("index.query.bool.max_clause_count", 1024).build();
+            SettingsModule module = new SettingsModule(settings);
+            IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
+                () -> assertInstanceBinding(module, Settings.class, (s) -> s == settings));
+            assertEquals("unknown setting [index.query.bool.max_clause_count] did you mean [indices.query.bool.max_clause_count]?",
+                ex.getMessage());
+    }
 }
diff --git a/docs/reference/migration/migrate_5_0/settings.asciidoc b/docs/reference/migration/migrate_5_0/settings.asciidoc
index e916ec6b24c..85895d65b67 100644
--- a/docs/reference/migration/migrate_5_0/settings.asciidoc
+++ b/docs/reference/migration/migrate_5_0/settings.asciidoc
@@ -258,3 +258,9 @@ Previously script mode settings (e.g., "script.inline: true",
 Prior to 5.0 a third option could be specified for the `script.inline` and
 `script.stored` settings ("sandbox"). This has been removed, You can now only
 set `script.line: true` or `script.stored: true`.
+
+==== Search settings
+
+The setting `index.query.bool.max_clause_count` has been removed. In order to
+set the maximum number of boolean clauses `indices.query.bool.max_clause_count`
+should be used instead.

From 050145f61b3bf6666712059bea07b54ca1076ed6 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Wed, 18 May 2016 16:13:52 +0200
Subject: [PATCH 0343/1311] parent/child: Allow adding additional child types
 that point to an existing parent type

From 2.0 adding child types to existing types was forbidden because the`_parent` field stores the join between parent and child at index time.
This is to protect from the fact that types that weren't a parent before become a parent while previously indexed documents would not have a join field.
 This would break the parent/child queries.

The restriction was a bit too strict in the sense that also if a type was a parent type the restriction would forbid adding child types that point to a parent type (so child points already point to it).
This change make sure that the restriction only applies if that type isn't a parent type already.

Closes #17956
---
 .../metadata/MetaDataMappingService.java      |  7 +-
 .../metadata/MetaDataMappingServiceTests.java | 66 +++++++++++++++++++
 .../search/child/ChildQuerySearchIT.java      | 23 -------
 3 files changed, 71 insertions(+), 25 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataMappingServiceTests.java

diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
index 311ffae71fd..d3b5e7ecbad 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
@@ -281,8 +281,11 @@ public class MetaDataMappingService extends AbstractComponent {
                         // Also the order of the mappings may be backwards.
                         if (newMapper.parentFieldMapper().active()) {
                             for (ObjectCursor<MappingMetaData> mapping : indexMetaData.getMappings().values()) {
-                                if (newMapper.parentFieldMapper().type().equals(mapping.value.type())) {
-                                    throw new IllegalArgumentException("can't add a _parent field that points to an already existing type");
+                                String parentType = newMapper.parentFieldMapper().type();
+                                if (parentType.equals(mapping.value.type()) &&
+                                        indexService.mapperService().getParentTypes().contains(parentType) == false) {
+                                    throw new IllegalArgumentException("can't add a _parent field that points to an " +
+                                            "already existing type, that isn't already a parent");
                                 }
                             }
                         }
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataMappingServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataMappingServiceTests.java
new file mode 100644
index 00000000000..c6ce30e2a52
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataMappingServiceTests.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+public class MetaDataMappingServiceTests extends ESSingleNodeTestCase {
+
+    // Tests _parent meta field logic, because part of the validation is in MetaDataMappingService
+    public void testAddChildTypePointingToAlreadyExistingType() throws Exception {
+        createIndex("test", Settings.EMPTY, "type", "field", "type=keyword");
+
+        // Shouldn't be able the add the _parent field pointing to an already existing type, which isn't a parent type
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> client().admin()
+                .indices()
+                .preparePutMapping("test")
+                .setType("child")
+                .setSource("_parent", "type=type")
+                .get());
+        assertThat(e.getMessage(),
+                equalTo("can't add a _parent field that points to an already existing type, that isn't already a parent"));
+    }
+
+    // Tests _parent meta field logic, because part of the validation is in MetaDataMappingService
+    public void testAddExtraChildTypePointingToAlreadyParentExistingType() throws Exception {
+        IndexService indexService = createIndex("test", client().admin().indices().prepareCreate("test")
+                .addMapping("parent")
+                .addMapping("child1", "_parent", "type=parent")
+        );
+
+        // adding the extra child type that points to an already existing parent type is allowed:
+        client().admin()
+                .indices()
+                .preparePutMapping("test")
+                .setType("child2")
+                .setSource("_parent", "type=parent")
+                .get();
+
+        DocumentMapper documentMapper = indexService.mapperService().documentMapper("child2");
+        assertThat(documentMapper.parentFieldMapper().type(), equalTo("parent"));
+        assertThat(documentMapper.parentFieldMapper().active(), is(true));
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
index 1a258abc9cc..90f5c65c066 100644
--- a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
@@ -1321,29 +1321,6 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         }
     }
 
-    public void testAddParentFieldAfterIndexingParentDocButBeforeIndexingChildDoc() throws Exception {
-        assertAcked(prepareCreate("test")
-                .setSettings(Settings.builder()
-                        .put(indexSettings())
-                        .put("index.refresh_interval", -1)));
-        ensureGreen();
-
-        String parentId = "p1";
-        client().prepareIndex("test", "parent", parentId).setSource("p_field", "1").get();
-        refresh();
-
-        try {
-            assertAcked(client().admin()
-                    .indices()
-                    .preparePutMapping("test")
-                    .setType("child")
-                    .setSource("_parent", "type=parent"));
-            fail("Shouldn't be able the add the _parent field pointing to an already existing parent type");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("can't add a _parent field that points to an already existing type"));
-        }
-    }
-
     public void testParentChildCaching() throws Exception {
         assertAcked(prepareCreate("test")
                 .setSettings(

From e2691d7e5ca6006a93ae421bf28183c21047910b Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Thu, 19 May 2016 13:08:30 +0200
Subject: [PATCH 0344/1311] test: Don't generate a value of 0, because
 FuzzyQuery constructor does't allow that

---
 .../org/elasticsearch/index/query/MatchQueryBuilderTests.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
index 6cec3ae8c52..87f8f3981d3 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
@@ -89,7 +89,7 @@ public class MatchQueryBuilderTests extends AbstractQueryTestCase<MatchQueryBuil
         }
 
         if (randomBoolean()) {
-            matchQuery.maxExpansions(randomIntBetween(0, 1000));
+            matchQuery.maxExpansions(randomIntBetween(1, 1000));
         }
 
         if (randomBoolean()) {

From 35d3bdab84fa05c71e8ae019aaf661759c8b1622 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Tue, 15 Sep 2015 17:35:10 +0200
Subject: [PATCH 0345/1311] Add Google Cloud Storage repository plugin

Closes #12880
---
 .../plugins/InstallPluginCommand.java         |   1 +
 .../blobstore/FsBlobStoreContainerTests.java  |   2 +-
 .../common/blobstore/FsBlobStoreTests.java    |   2 +-
 .../snapshots/FsBlobStoreRepositoryIT.java    |   2 +-
 dev-tools/smoke_test_rc.py                    |   1 +
 docs/plugins/repository-gcs.asciidoc          | 216 +++++++
 docs/plugins/repository.asciidoc              |   5 +
 docs/reference/modules/snapshots.asciidoc     |   1 +
 plugins/repository-gcs/build.gradle           |  54 ++
 .../licenses/commons-codec-1.10.jar.sha1      |   1 +
 .../licenses/commons-codec-LICENSE.txt        | 202 +++++++
 .../licenses/commons-codec-NOTICE.txt         |  17 +
 .../licenses/commons-logging-1.1.3.jar.sha1   |   1 +
 .../licenses/commons-logging-LICENSE.txt      | 202 +++++++
 .../licenses/commons-logging-NOTICE.txt       |   5 +
 .../licenses/google-LICENSE.txt               | 201 +++++++
 .../repository-gcs/licenses/google-NOTICE.txt |   1 +
 .../google-api-client-1.21.0.jar.sha1         |   1 +
 ...-services-storage-v1-rev66-1.21.0.jar.sha1 |   1 +
 .../google-http-client-1.21.0.jar.sha1        |   1 +
 ...oogle-http-client-jackson2-1.21.0.jar.sha1 |   1 +
 .../google-oauth-client-1.21.0.jar.sha1       |   1 +
 .../licenses/httpclient-4.3.6.jar.sha1        |   1 +
 .../licenses/httpclient-LICENSE.txt           | 558 ++++++++++++++++++
 .../licenses/httpclient-NOTICE.txt            |   5 +
 .../licenses/httpcore-4.3.3.jar.sha1          |   1 +
 .../licenses/httpcore-LICENSE.txt             | 241 ++++++++
 .../licenses/httpcore-NOTICE.txt              |   8 +
 .../gcs/GoogleCloudStorageBlobContainer.java  | 120 ++++
 .../gcs/GoogleCloudStorageBlobStore.java      | 432 ++++++++++++++
 .../gcs/GoogleCloudStorageModule.java         |  31 +
 .../gcs/GoogleCloudStoragePlugin.java         | 131 ++++
 .../gcs/GoogleCloudStorageRepository.java     | 154 +++++
 .../gcs/GoogleCloudStorageService.java        | 176 ++++++
 .../plugin-metadata/plugin-security.policy    |  26 +
 ...leCloudStorageBlobStoreContainerTests.java |  36 ++
 .../gcs/GoogleCloudStorageBlobStoreTests.java |  36 ++
 .../blobstore/gcs/MockHttpTransport.java      | 432 ++++++++++++++
 ...eCloudStorageBlobStoreRepositoryTests.java |  96 +++
 .../GoogleCloudStorageRepositoryRestIT.java   |  41 ++
 .../test/repository_gcs/10_basic.yaml         |  13 +
 .../scripts/module_and_plugin_test_cases.bash |   8 +
 settings.gradle                               |   1 +
 .../ESBlobStoreContainerTestCase.java         |   9 +-
 .../ESBlobStoreRepositoryIntegTestCase.java   |  21 +-
 .../repositories}/ESBlobStoreTestCase.java    |   3 +-
 46 files changed, 3487 insertions(+), 12 deletions(-)
 create mode 100644 docs/plugins/repository-gcs.asciidoc
 create mode 100644 plugins/repository-gcs/build.gradle
 create mode 100644 plugins/repository-gcs/licenses/commons-codec-1.10.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/commons-codec-LICENSE.txt
 create mode 100644 plugins/repository-gcs/licenses/commons-codec-NOTICE.txt
 create mode 100644 plugins/repository-gcs/licenses/commons-logging-1.1.3.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/commons-logging-LICENSE.txt
 create mode 100644 plugins/repository-gcs/licenses/commons-logging-NOTICE.txt
 create mode 100644 plugins/repository-gcs/licenses/google-LICENSE.txt
 create mode 100644 plugins/repository-gcs/licenses/google-NOTICE.txt
 create mode 100644 plugins/repository-gcs/licenses/google-api-client-1.21.0.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-api-services-storage-v1-rev66-1.21.0.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-http-client-1.21.0.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-http-client-jackson2-1.21.0.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/google-oauth-client-1.21.0.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/httpclient-4.3.6.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/httpclient-LICENSE.txt
 create mode 100644 plugins/repository-gcs/licenses/httpclient-NOTICE.txt
 create mode 100644 plugins/repository-gcs/licenses/httpcore-4.3.3.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/httpcore-LICENSE.txt
 create mode 100644 plugins/repository-gcs/licenses/httpcore-NOTICE.txt
 create mode 100644 plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobContainer.java
 create mode 100644 plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStore.java
 create mode 100644 plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStorageModule.java
 create mode 100644 plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStoragePlugin.java
 create mode 100644 plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
 create mode 100644 plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
 create mode 100644 plugins/repository-gcs/src/main/plugin-metadata/plugin-security.policy
 create mode 100644 plugins/repository-gcs/src/test/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStoreContainerTests.java
 create mode 100644 plugins/repository-gcs/src/test/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStoreTests.java
 create mode 100644 plugins/repository-gcs/src/test/java/org/elasticsearch/common/blobstore/gcs/MockHttpTransport.java
 create mode 100644 plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
 create mode 100644 plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepositoryRestIT.java
 create mode 100644 plugins/repository-gcs/src/test/resources/rest-api-spec/test/repository_gcs/10_basic.yaml
 rename {core/src/test/java/org/elasticsearch/test => test/framework/src/main/java/org/elasticsearch/repositories}/ESBlobStoreContainerTestCase.java (94%)
 rename {core/src/test/java/org/elasticsearch/test => test/framework/src/main/java/org/elasticsearch/repositories}/ESBlobStoreRepositoryIntegTestCase.java (91%)
 rename {core/src/test/java/org/elasticsearch/test => test/framework/src/main/java/org/elasticsearch/repositories}/ESBlobStoreTestCase.java (97%)

diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
index 2e5faa6bfae..e81d376f3b5 100644
--- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
@@ -126,6 +126,7 @@ class InstallPluginCommand extends Command {
             "mapper-murmur3",
             "mapper-size",
             "repository-azure",
+            "repository-gcs",
             "repository-hdfs",
             "repository-s3",
             "store-smb",
diff --git a/core/src/test/java/org/elasticsearch/common/blobstore/FsBlobStoreContainerTests.java b/core/src/test/java/org/elasticsearch/common/blobstore/FsBlobStoreContainerTests.java
index 63c04b1c5e1..b08b81db11a 100644
--- a/core/src/test/java/org/elasticsearch/common/blobstore/FsBlobStoreContainerTests.java
+++ b/core/src/test/java/org/elasticsearch/common/blobstore/FsBlobStoreContainerTests.java
@@ -23,7 +23,7 @@ import org.elasticsearch.common.blobstore.fs.FsBlobStore;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.test.ESBlobStoreContainerTestCase;
+import org.elasticsearch.repositories.ESBlobStoreContainerTestCase;
 
 import java.io.IOException;
 import java.nio.file.Path;
diff --git a/core/src/test/java/org/elasticsearch/common/blobstore/FsBlobStoreTests.java b/core/src/test/java/org/elasticsearch/common/blobstore/FsBlobStoreTests.java
index f6f53549ce4..7d4ac1acc07 100644
--- a/core/src/test/java/org/elasticsearch/common/blobstore/FsBlobStoreTests.java
+++ b/core/src/test/java/org/elasticsearch/common/blobstore/FsBlobStoreTests.java
@@ -23,7 +23,7 @@ import org.elasticsearch.common.blobstore.fs.FsBlobStore;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.test.ESBlobStoreTestCase;
+import org.elasticsearch.repositories.ESBlobStoreTestCase;
 
 import java.io.IOException;
 import java.nio.file.Path;
diff --git a/core/src/test/java/org/elasticsearch/snapshots/FsBlobStoreRepositoryIT.java b/core/src/test/java/org/elasticsearch/snapshots/FsBlobStoreRepositoryIT.java
index fceedff8e70..84c3a03f2c8 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/FsBlobStoreRepositoryIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/FsBlobStoreRepositoryIT.java
@@ -20,7 +20,7 @@ package org.elasticsearch.snapshots;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.test.ESBlobStoreRepositoryIntegTestCase;
+import org.elasticsearch.repositories.ESBlobStoreRepositoryIntegTestCase;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 
diff --git a/dev-tools/smoke_test_rc.py b/dev-tools/smoke_test_rc.py
index 244ab8caa25..39db9929a54 100644
--- a/dev-tools/smoke_test_rc.py
+++ b/dev-tools/smoke_test_rc.py
@@ -75,6 +75,7 @@ DEFAULT_PLUGINS = ["analysis-icu",
                    "mapper-murmur3",
                    "mapper-size",
                    "repository-azure",
+                   "repository-gcs",
                    "repository-hdfs",
                    "repository-s3",
                    "store-smb"]
diff --git a/docs/plugins/repository-gcs.asciidoc b/docs/plugins/repository-gcs.asciidoc
new file mode 100644
index 00000000000..bed78b4cbbf
--- /dev/null
+++ b/docs/plugins/repository-gcs.asciidoc
@@ -0,0 +1,216 @@
+[[repository-gcs]]
+=== Google Cloud Storage Repository Plugin
+
+The GCS repository plugin adds support for using the https://cloud.google.com/storage/[Google Cloud Storage]
+service as a repository for {ref}/modules-snapshots.html[Snapshot/Restore].
+
+[[repository-gcs-install]]
+[float]
+==== Installation
+
+This plugin can be installed using the plugin manager:
+
+[source,sh]
+----------------------------------------------------------------
+sudo bin/elasticsearch-plugin install repository-gcs
+----------------------------------------------------------------
+
+NOTE: The plugin requires new permission to be installed in order to work
+
+The plugin must be installed on every node in the cluster, and each node must
+be restarted after installation.
+
+[[repository-gcs-remove]]
+[float]
+==== Removal
+
+The plugin can be removed with the following command:
+
+[source,sh]
+----------------------------------------------------------------
+sudo bin/elasticsearch-plugin remove repository-gcs
+----------------------------------------------------------------
+
+The node must be stopped before removing the plugin.
+
+[[repository-gcs-usage]]
+==== Getting started
+
+The plugin uses the https://cloud.google.com/storage/docs/json_api/[Google Cloud Storage JSON API] (v1)
+to connect to the Storage service. If this is the first time you use Google Cloud Storage, you first
+need to connect to the https://console.cloud.google.com/[Google Cloud Platform Console] and create a new
+project. Once your project is created, you must enable the Cloud Storage Service for your project.
+
+[[repository-gcs-creating-bucket]]
+===== Creating a Bucket
+
+Google Cloud Storage service uses the concept of https://cloud.google.com/storage/docs/key-terms[Bucket]
+as a container for all the data. Buckets are usually created using the
+https://console.cloud.google.com/[Google Cloud Platform Console]. The plugin will not automatically
+create buckets.
+
+To create a new bucket:
+
+1. Connect to the https://console.cloud.google.com/[Google Cloud Platform Console]
+2. Select your project
+3. Got to the https://console.cloud.google.com/storage/browser[Storage Browser]
+4. Click the "Create Bucket" button
+5. Enter a the name of the new bucket
+6. Select a storage class
+7. Select a location
+8. Click the "Create" button
+
+The bucket should now be created.
+
+[[repository-gcs-service-authentication]]
+===== Service Authentication
+
+The plugin supports two authentication modes:
+
+* the built-in <<repository-gcs-using-compute-engine, Compute Engine authentication>>. This mode is
+recommended if your elasticsearch node is running on a Compute Engine virtual machine.
+
+* the <<repository-gcs-using-service-account, Service Account>> authentication mode.
+
+[[repository-gcs-using-compute-engine]]
+===== Using Compute Engine
+When running on Compute Engine, the plugin use the Google's built-in authentication mechanism to
+authenticate on the Storage service. Compute Engine virtual machines are usually associated to a
+default service account. This service account can be found in the VM instance details in the
+https://console.cloud.google.com/compute/[Compute Engine console].
+
+To indicate that a repository should use the built-in authentication,
+the repository `service_account` setting must be set to `_default_`:
+
+[source,json]
+----
+PUT _snapshot/my_gcs_repository_on_compute_engine
+{
+  "type": "gcs",
+  "settings": {
+    "bucket": "my_bucket",
+    "service_account": "_default_"
+  }
+}
+----
+// CONSOLE
+
+NOTE: The Compute Engine VM must be allowed to use the Storage service. This can be done only at VM
+creation time, when "Storage" access can be configured to "Read/Write" permission. Check your
+instance details at the section "Cloud API access scopes".
+
+[[repository-gcs-using-service-account]]
+===== Using a Service Account
+If your elasticsearch node is not running on Compute Engine, or if you don't want to use Google
+built-in authentication mechanism, you can authenticate on the Storage service using a
+https://cloud.google.com/iam/docs/overview#service_account[Service Account] file.
+
+To create a service account file:
+1. Connect to the https://console.cloud.google.com/[Google Cloud Platform Console]
+2. Select your project
+3. Got to the https://console.cloud.google.com/permissions[Permission] tab
+4. Select the https://console.cloud.google.com/permissions/serviceaccounts[Service Accounts] tab
+5. Click on "Create service account"
+6. Once created, select the new service account and download a JSON key file
+
+A service account file looks like this:
+
+[source,json]
+----
+{
+  "type": "service_account",
+  "project_id": "your-project-id",
+  "private_key_id": "...",
+  "private_key": "-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----\n",
+  "client_email": "service-account-for-your-repository@your-project-id.iam.gserviceaccount.com",
+  "client_id": "...",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://accounts.google.com/o/oauth2/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "..."
+}
+----
+
+This file must be copied in the `config` directory of the elasticsearch installation and on
+every node of the cluster.
+
+To indicate that a repository should use a service account file:
+
+[source,json]
+----
+PUT _snapshot/my_gcs_repository
+{
+  "type": "gcs",
+  "settings": {
+    "bucket": "my_bucket",
+    "service_account": "service_account.json"
+  }
+}
+----
+// CONSOLE
+
+
+[[repository-gcs-bucket-permission]]
+===== Set Bucket Permission
+
+The service account used to access the bucket must have the "Writer" access to the bucket:
+
+1. Connect to the https://console.cloud.google.com/[Google Cloud Platform Console]
+2. Select your project
+3. Got to the [https://console.cloud.google.com/storage/browser]Storage Browser
+4. Select the bucket and "Edit bucket permission"
+5. The service account must be configured as a "User" with "Writer" access
+
+
+[[repository-gcs-repository]]
+==== Create a Repository
+
+Once everything is installed and every node is started, you can create a new repository that
+uses Google Cloud Storage to store snapshots:
+
+[source,json]
+----
+PUT _snapshot/my_gcs_repository
+{
+  "type": "gcs",
+  "settings": {
+    "bucket": "my_bucket",
+    "service_account": "service_account.json"
+  }
+}
+----
+// CONSOLE
+
+The following settings are supported:
+
+`bucket`::
+
+    The name of the bucket to be used for snapshots. (Mandatory)
+
+`service_account`::
+
+    The service account to use. It can be a relative path to a service account JSON file
+    or the value `_default_` that indicate to use built-in Compute Engine service account.
+
+`base_path`::
+
+    Specifies the path within bucket to repository data. Defaults to
+    the root of the bucket.
+
+`chunk_size`::
+
+    Big files can be broken down into chunks during snapshotting if needed.
+    The chunk size can be specified in bytes or by using size value notation,
+    i.e. `1g`, `10m`, `5k`. Defaults to `100m`.
+
+`compress`::
+
+    When set to `true` metadata files are stored in compressed format. This
+    setting doesn't affect index files that are already compressed by default.
+    Defaults to `false`.
+
+`application_name`::
+
+    Name used by the plugin when it uses the Google Cloud JSON API. Setting
+    a custom name can be useful to authenticate your cluster when requests
+    statistics are logged in the Google Cloud Platform. Default to `repository-gcs`
diff --git a/docs/plugins/repository.asciidoc b/docs/plugins/repository.asciidoc
index 08557b9e03e..9a4e90bebd7 100644
--- a/docs/plugins/repository.asciidoc
+++ b/docs/plugins/repository.asciidoc
@@ -22,6 +22,10 @@ The Azure repository plugin adds support for using Azure as a repository.
 
 The Hadoop HDFS Repository plugin adds support for using HDFS as a repository.
 
+<<repository-gcs,Google Cloud Storage Repository>>::
+
+The GCS repository plugin adds support for using Google Cloud Storage service as a repository.
+
 
 [float]
 === Community contributed repository plugins
@@ -37,3 +41,4 @@ include::repository-s3.asciidoc[]
 
 include::repository-hdfs.asciidoc[]
 
+include::repository-gcs.asciidoc[]
diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc
index 6c5245ce646..0f87744d317 100644
--- a/docs/reference/modules/snapshots.asciidoc
+++ b/docs/reference/modules/snapshots.asciidoc
@@ -162,6 +162,7 @@ Other repository backends are available in these official plugins:
 * {plugins}/repository-s3.html[repository-s3] for S3 repository support
 * {plugins}/repository-hdfs.html[repository-hdfs] for HDFS repository support in Hadoop environments
 * {plugins}/repository-azure.html[repository-azure] for Azure storage repositories
+* {plugins}/repository-gcs.html[repository-gcs] for Google Cloud Storage repositories
 
 [float]
 ===== Repository Verification
diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle
new file mode 100644
index 00000000000..9968d4408e4
--- /dev/null
+++ b/plugins/repository-gcs/build.gradle
@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+esplugin {
+  description 'The GCS repository plugin adds Google Cloud Storage support for repositories.'
+  classname 'org.elasticsearch.plugin.repository.gcs.GoogleCloudStoragePlugin'
+}
+
+versions << [
+        'google': '1.21.0',
+]
+
+dependencies {
+  compile "com.google.apis:google-api-services-storage:v1-rev66-${versions.google}"
+  compile "com.google.api-client:google-api-client:${versions.google}"
+  compile "com.google.oauth-client:google-oauth-client:${versions.google}"
+  compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
+  compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
+  compile "commons-logging:commons-logging:${versions.commonslogging}"
+  compile "commons-codec:commons-codec:${versions.commonscodec}"
+  compile "com.google.http-client:google-http-client:${versions.google}"
+  compile "com.google.http-client:google-http-client-jackson2:${versions.google}"
+}
+
+dependencyLicenses {
+  mapping from: /google-.*/, to: 'google'
+}
+
+thirdPartyAudit.excludes = [
+        // classes are missing
+        'com.google.common.base.Splitter',
+        'com.google.common.collect.Lists',
+        'javax.servlet.ServletContextEvent',
+        'javax.servlet.ServletContextListener',
+        'org.apache.avalon.framework.logger.Logger',
+        'org.apache.log.Hierarchy',
+        'org.apache.log.Logger',
+]
diff --git a/plugins/repository-gcs/licenses/commons-codec-1.10.jar.sha1 b/plugins/repository-gcs/licenses/commons-codec-1.10.jar.sha1
new file mode 100644
index 00000000000..3fe8682a1b0
--- /dev/null
+++ b/plugins/repository-gcs/licenses/commons-codec-1.10.jar.sha1
@@ -0,0 +1 @@
+4b95f4897fa13f2cd904aee711aeafc0c5295cd8
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/commons-codec-LICENSE.txt b/plugins/repository-gcs/licenses/commons-codec-LICENSE.txt
new file mode 100644
index 00000000000..d6456956733
--- /dev/null
+++ b/plugins/repository-gcs/licenses/commons-codec-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/repository-gcs/licenses/commons-codec-NOTICE.txt b/plugins/repository-gcs/licenses/commons-codec-NOTICE.txt
new file mode 100644
index 00000000000..56916449bbe
--- /dev/null
+++ b/plugins/repository-gcs/licenses/commons-codec-NOTICE.txt
@@ -0,0 +1,17 @@
+Apache Commons Codec
+Copyright 2002-2015 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java
+contains test data from http://aspell.net/test/orig/batch0.tab.
+Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org)
+
+===============================================================================
+
+The content of package org.apache.commons.codec.language.bm has been translated
+from the original php source code available at http://stevemorse.org/phoneticinfo.htm
+with permission from the original authors.
+Original source copyright:
+Copyright (c) 2008 Alexander Beider & Stephen P. Morse.
diff --git a/plugins/repository-gcs/licenses/commons-logging-1.1.3.jar.sha1 b/plugins/repository-gcs/licenses/commons-logging-1.1.3.jar.sha1
new file mode 100644
index 00000000000..5b8f029e582
--- /dev/null
+++ b/plugins/repository-gcs/licenses/commons-logging-1.1.3.jar.sha1
@@ -0,0 +1 @@
+f6f66e966c70a83ffbdb6f17a0919eaf7c8aca7f
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/commons-logging-LICENSE.txt b/plugins/repository-gcs/licenses/commons-logging-LICENSE.txt
new file mode 100644
index 00000000000..57bc88a15a0
--- /dev/null
+++ b/plugins/repository-gcs/licenses/commons-logging-LICENSE.txt
@@ -0,0 +1,202 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
diff --git a/plugins/repository-gcs/licenses/commons-logging-NOTICE.txt b/plugins/repository-gcs/licenses/commons-logging-NOTICE.txt
new file mode 100644
index 00000000000..72eb32a9024
--- /dev/null
+++ b/plugins/repository-gcs/licenses/commons-logging-NOTICE.txt
@@ -0,0 +1,5 @@
+Apache Commons CLI
+Copyright 2001-2009 The Apache Software Foundation
+
+This product includes software developed by
+The Apache Software Foundation (http://www.apache.org/).
diff --git a/plugins/repository-gcs/licenses/google-LICENSE.txt b/plugins/repository-gcs/licenses/google-LICENSE.txt
new file mode 100644
index 00000000000..980a15ac24e
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-LICENSE.txt
@@ -0,0 +1,201 @@
+                                Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/repository-gcs/licenses/google-NOTICE.txt b/plugins/repository-gcs/licenses/google-NOTICE.txt
new file mode 100644
index 00000000000..8d1c8b69c3f
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-NOTICE.txt
@@ -0,0 +1 @@
+ 
diff --git a/plugins/repository-gcs/licenses/google-api-client-1.21.0.jar.sha1 b/plugins/repository-gcs/licenses/google-api-client-1.21.0.jar.sha1
new file mode 100644
index 00000000000..56988521028
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-api-client-1.21.0.jar.sha1
@@ -0,0 +1 @@
+16a6b3c680f3bf7b81bb42790ff5c1b72c5bbedc
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev66-1.21.0.jar.sha1 b/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev66-1.21.0.jar.sha1
new file mode 100644
index 00000000000..2a97aed2b79
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-api-services-storage-v1-rev66-1.21.0.jar.sha1
@@ -0,0 +1 @@
+eb753d716e4f8dec203deb0f8fdca86913a79029
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-1.21.0.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-1.21.0.jar.sha1
new file mode 100644
index 00000000000..401abd444ce
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-http-client-1.21.0.jar.sha1
@@ -0,0 +1 @@
+42631630fe1276d4d6d6397bb07d53a4e4fec278
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-jackson2-1.21.0.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-jackson2-1.21.0.jar.sha1
new file mode 100644
index 00000000000..e7ca5887412
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-http-client-jackson2-1.21.0.jar.sha1
@@ -0,0 +1 @@
+8ce17bdd15fff0fd8cf359757f29e778fc7191ad
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-oauth-client-1.21.0.jar.sha1 b/plugins/repository-gcs/licenses/google-oauth-client-1.21.0.jar.sha1
new file mode 100644
index 00000000000..7e3de94a9bc
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-oauth-client-1.21.0.jar.sha1
@@ -0,0 +1 @@
+61ec42bbfc51aafde5eb8b4923c602c5b5965bc2
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/httpclient-4.3.6.jar.sha1 b/plugins/repository-gcs/licenses/httpclient-4.3.6.jar.sha1
new file mode 100644
index 00000000000..2c18ef0f54c
--- /dev/null
+++ b/plugins/repository-gcs/licenses/httpclient-4.3.6.jar.sha1
@@ -0,0 +1 @@
+4c47155e3e6c9a41a28db36680b828ced53b8af4
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/httpclient-LICENSE.txt b/plugins/repository-gcs/licenses/httpclient-LICENSE.txt
new file mode 100644
index 00000000000..32f01eda18f
--- /dev/null
+++ b/plugins/repository-gcs/licenses/httpclient-LICENSE.txt
@@ -0,0 +1,558 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+=========================================================================
+
+This project includes Public Suffix List copied from
+<https://publicsuffix.org/list/effective_tld_names.dat>
+licensed under the terms of the Mozilla Public License, v. 2.0
+
+Full license text: <http://mozilla.org/MPL/2.0/>
+
+Mozilla Public License Version 2.0
+==================================
+
+1. Definitions
+--------------
+
+1.1. "Contributor"
+    means each individual or legal entity that creates, contributes to
+    the creation of, or owns Covered Software.
+
+1.2. "Contributor Version"
+    means the combination of the Contributions of others (if any) used
+    by a Contributor and that particular Contributor's Contribution.
+
+1.3. "Contribution"
+    means Covered Software of a particular Contributor.
+
+1.4. "Covered Software"
+    means Source Code Form to which the initial Contributor has attached
+    the notice in Exhibit A, the Executable Form of such Source Code
+    Form, and Modifications of such Source Code Form, in each case
+    including portions thereof.
+
+1.5. "Incompatible With Secondary Licenses"
+    means
+
+    (a) that the initial Contributor has attached the notice described
+        in Exhibit B to the Covered Software; or
+
+    (b) that the Covered Software was made available under the terms of
+        version 1.1 or earlier of the License, but not also under the
+        terms of a Secondary License.
+
+1.6. "Executable Form"
+    means any form of the work other than Source Code Form.
+
+1.7. "Larger Work"
+    means a work that combines Covered Software with other material, in
+    a separate file or files, that is not Covered Software.
+
+1.8. "License"
+    means this document.
+
+1.9. "Licensable"
+    means having the right to grant, to the maximum extent possible,
+    whether at the time of the initial grant or subsequently, any and
+    all of the rights conveyed by this License.
+
+1.10. "Modifications"
+    means any of the following:
+
+    (a) any file in Source Code Form that results from an addition to,
+        deletion from, or modification of the contents of Covered
+        Software; or
+
+    (b) any new file in Source Code Form that contains any Covered
+        Software.
+
+1.11. "Patent Claims" of a Contributor
+    means any patent claim(s), including without limitation, method,
+    process, and apparatus claims, in any patent Licensable by such
+    Contributor that would be infringed, but for the grant of the
+    License, by the making, using, selling, offering for sale, having
+    made, import, or transfer of either its Contributions or its
+    Contributor Version.
+
+1.12. "Secondary License"
+    means either the GNU General Public License, Version 2.0, the GNU
+    Lesser General Public License, Version 2.1, the GNU Affero General
+    Public License, Version 3.0, or any later versions of those
+    licenses.
+
+1.13. "Source Code Form"
+    means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your")
+    means an individual or a legal entity exercising rights under this
+    License. For legal entities, "You" includes any entity that
+    controls, is controlled by, or is under common control with You. For
+    purposes of this definition, "control" means (a) the power, direct
+    or indirect, to cause the direction or management of such entity,
+    whether by contract or otherwise, or (b) ownership of more than
+    fifty percent (50%) of the outstanding shares or beneficial
+    ownership of such entity.
+
+2. License Grants and Conditions
+--------------------------------
+
+2.1. Grants
+
+Each Contributor hereby grants You a world-wide, royalty-free,
+non-exclusive license:
+
+(a) under intellectual property rights (other than patent or trademark)
+    Licensable by such Contributor to use, reproduce, make available,
+    modify, display, perform, distribute, and otherwise exploit its
+    Contributions, either on an unmodified basis, with Modifications, or
+    as part of a Larger Work; and
+
+(b) under Patent Claims of such Contributor to make, use, sell, offer
+    for sale, have made, import, and otherwise transfer either its
+    Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+The licenses granted in Section 2.1 with respect to any Contribution
+become effective for each Contribution on the date the Contributor first
+distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+The licenses granted in this Section 2 are the only rights granted under
+this License. No additional rights or licenses will be implied from the
+distribution or licensing of Covered Software under this License.
+Notwithstanding Section 2.1(b) above, no patent license is granted by a
+Contributor:
+
+(a) for any code that a Contributor has removed from Covered Software;
+    or
+
+(b) for infringements caused by: (i) Your and any other third party's
+    modifications of Covered Software, or (ii) the combination of its
+    Contributions with other software (except as part of its Contributor
+    Version); or
+
+(c) under Patent Claims infringed by Covered Software in the absence of
+    its Contributions.
+
+This License does not grant any rights in the trademarks, service marks,
+or logos of any Contributor (except as may be necessary to comply with
+the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+No Contributor makes additional grants as a result of Your choice to
+distribute the Covered Software under a subsequent version of this
+License (see Section 10.2) or under the terms of a Secondary License (if
+permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+Each Contributor represents that the Contributor believes its
+Contributions are its original creation(s) or it has sufficient rights
+to grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+This License is not intended to limit any rights You have under
+applicable copyright doctrines of fair use, fair dealing, or other
+equivalents.
+
+2.7. Conditions
+
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+in Section 2.1.
+
+3. Responsibilities
+-------------------
+
+3.1. Distribution of Source Form
+
+All distribution of Covered Software in Source Code Form, including any
+Modifications that You create or to which You contribute, must be under
+the terms of this License. You must inform recipients that the Source
+Code Form of the Covered Software is governed by the terms of this
+License, and how they can obtain a copy of this License. You may not
+attempt to alter or restrict the recipients' rights in the Source Code
+Form.
+
+3.2. Distribution of Executable Form
+
+If You distribute Covered Software in Executable Form then:
+
+(a) such Covered Software must also be made available in Source Code
+    Form, as described in Section 3.1, and You must inform recipients of
+    the Executable Form how they can obtain a copy of such Source Code
+    Form by reasonable means in a timely manner, at a charge no more
+    than the cost of distribution to the recipient; and
+
+(b) You may distribute such Executable Form under the terms of this
+    License, or sublicense it under different terms, provided that the
+    license for the Executable Form does not attempt to limit or alter
+    the recipients' rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+You may create and distribute a Larger Work under terms of Your choice,
+provided that You also comply with the requirements of this License for
+the Covered Software. If the Larger Work is a combination of Covered
+Software with a work governed by one or more Secondary Licenses, and the
+Covered Software is not Incompatible With Secondary Licenses, this
+License permits You to additionally distribute such Covered Software
+under the terms of such Secondary License(s), so that the recipient of
+the Larger Work may, at their option, further distribute the Covered
+Software under the terms of either this License or such Secondary
+License(s).
+
+3.4. Notices
+
+You may not remove or alter the substance of any license notices
+(including copyright notices, patent notices, disclaimers of warranty,
+or limitations of liability) contained within the Source Code Form of
+the Covered Software, except that You may alter any license notices to
+the extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+You may choose to offer, and to charge a fee for, warranty, support,
+indemnity or liability obligations to one or more recipients of Covered
+Software. However, You may do so only on Your own behalf, and not on
+behalf of any Contributor. You must make it absolutely clear that any
+such warranty, support, indemnity, or liability obligation is offered by
+You alone, and You hereby agree to indemnify every Contributor for any
+liability incurred by such Contributor as a result of warranty, support,
+indemnity or liability terms You offer. You may include additional
+disclaimers of warranty and limitations of liability specific to any
+jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+---------------------------------------------------
+
+If it is impossible for You to comply with any of the terms of this
+License with respect to some or all of the Covered Software due to
+statute, judicial order, or regulation then You must: (a) comply with
+the terms of this License to the maximum extent possible; and (b)
+describe the limitations and the code they affect. Such description must
+be placed in a text file included with all distributions of the Covered
+Software under this License. Except to the extent prohibited by statute
+or regulation, such description must be sufficiently detailed for a
+recipient of ordinary skill to be able to understand it.
+
+5. Termination
+--------------
+
+5.1. The rights granted under this License will terminate automatically
+if You fail to comply with any of its terms. However, if You become
+compliant, then the rights granted under this License from a particular
+Contributor are reinstated (a) provisionally, unless and until such
+Contributor explicitly and finally terminates Your grants, and (b) on an
+ongoing basis, if such Contributor fails to notify You of the
+non-compliance by some reasonable means prior to 60 days after You have
+come back into compliance. Moreover, Your grants from a particular
+Contributor are reinstated on an ongoing basis if such Contributor
+notifies You of the non-compliance by some reasonable means, this is the
+first time You have received notice of non-compliance with this License
+from such Contributor, and You become compliant prior to 30 days after
+Your receipt of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+infringement claim (excluding declaratory judgment actions,
+counter-claims, and cross-claims) alleging that a Contributor Version
+directly or indirectly infringes any patent, then the rights granted to
+You by any and all Contributors for the Covered Software under Section
+2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+end user license agreements (excluding distributors and resellers) which
+have been validly granted by You or Your distributors under this License
+prior to termination shall survive termination.
+
+************************************************************************
+*                                                                      *
+*  6. Disclaimer of Warranty                                           *
+*  -------------------------                                           *
+*                                                                      *
+*  Covered Software is provided under this License on an "as is"       *
+*  basis, without warranty of any kind, either expressed, implied, or  *
+*  statutory, including, without limitation, warranties that the       *
+*  Covered Software is free of defects, merchantable, fit for a        *
+*  particular purpose or non-infringing. The entire risk as to the     *
+*  quality and performance of the Covered Software is with You.        *
+*  Should any Covered Software prove defective in any respect, You     *
+*  (not any Contributor) assume the cost of any necessary servicing,   *
+*  repair, or correction. This disclaimer of warranty constitutes an   *
+*  essential part of this License. No use of any Covered Software is   *
+*  authorized under this License except under this disclaimer.         *
+*                                                                      *
+************************************************************************
+
+************************************************************************
+*                                                                      *
+*  7. Limitation of Liability                                          *
+*  --------------------------                                          *
+*                                                                      *
+*  Under no circumstances and under no legal theory, whether tort      *
+*  (including negligence), contract, or otherwise, shall any           *
+*  Contributor, or anyone who distributes Covered Software as          *
+*  permitted above, be liable to You for any direct, indirect,         *
+*  special, incidental, or consequential damages of any character      *
+*  including, without limitation, damages for lost profits, loss of    *
+*  goodwill, work stoppage, computer failure or malfunction, or any    *
+*  and all other commercial damages or losses, even if such party      *
+*  shall have been informed of the possibility of such damages. This   *
+*  limitation of liability shall not apply to liability for death or   *
+*  personal injury resulting from such party's negligence to the       *
+*  extent applicable law prohibits such limitation. Some               *
+*  jurisdictions do not allow the exclusion or limitation of           *
+*  incidental or consequential damages, so this exclusion and          *
+*  limitation may not apply to You.                                    *
+*                                                                      *
+************************************************************************
+
+8. Litigation
+-------------
+
+Any litigation relating to this License may be brought only in the
+courts of a jurisdiction where the defendant maintains its principal
+place of business and such litigation shall be governed by laws of that
+jurisdiction, without reference to its conflict-of-law provisions.
+Nothing in this Section shall prevent a party's ability to bring
+cross-claims or counter-claims.
+
+9. Miscellaneous
+----------------
+
+This License represents the complete agreement concerning the subject
+matter hereof. If any provision of this License is held to be
+unenforceable, such provision shall be reformed only to the extent
+necessary to make it enforceable. Any law or regulation which provides
+that the language of a contract shall be construed against the drafter
+shall not be used to construe this License against a Contributor.
+
+10. Versions of the License
+---------------------------
+
+10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in Section
+10.3, no one other than the license steward has the right to modify or
+publish new versions of this License. Each version will be given a
+distinguishing version number.
+
+10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version
+of the License under which You originally received the Covered Software,
+or under the terms of any subsequent version published by the license
+steward.
+
+10.3. Modified Versions
+
+If you create software not governed by this License, and you want to
+create a new license for such software, you may create and use a
+modified version of this License if you rename the license and remove
+any references to the name of the license steward (except to note that
+such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With
+Secondary Licenses under the terms of this version of the License, the
+notice described in Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+-------------------------------------------
+
+  This Source Code Form is subject to the terms of the Mozilla Public
+  License, v. 2.0. If a copy of the MPL was not distributed with this
+  file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular
+file, then You may include the notice in a location (such as a LICENSE
+file in a relevant directory) where a recipient would be likely to look
+for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+---------------------------------------------------------
+
+  This Source Code Form is "Incompatible With Secondary Licenses", as
+  defined by the Mozilla Public License, v. 2.0.
diff --git a/plugins/repository-gcs/licenses/httpclient-NOTICE.txt b/plugins/repository-gcs/licenses/httpclient-NOTICE.txt
new file mode 100644
index 00000000000..4f6058178b2
--- /dev/null
+++ b/plugins/repository-gcs/licenses/httpclient-NOTICE.txt
@@ -0,0 +1,5 @@
+Apache HttpComponents Client
+Copyright 1999-2015 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
diff --git a/plugins/repository-gcs/licenses/httpcore-4.3.3.jar.sha1 b/plugins/repository-gcs/licenses/httpcore-4.3.3.jar.sha1
new file mode 100644
index 00000000000..0ad1d24aa9f
--- /dev/null
+++ b/plugins/repository-gcs/licenses/httpcore-4.3.3.jar.sha1
@@ -0,0 +1 @@
+f91b7a4aadc5cf486df6e4634748d7dd7a73f06d
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/httpcore-LICENSE.txt b/plugins/repository-gcs/licenses/httpcore-LICENSE.txt
new file mode 100644
index 00000000000..72819a9f06f
--- /dev/null
+++ b/plugins/repository-gcs/licenses/httpcore-LICENSE.txt
@@ -0,0 +1,241 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+=========================================================================
+
+This project contains annotations in the package org.apache.http.annotation
+which are derived from JCIP-ANNOTATIONS
+Copyright (c) 2005 Brian Goetz and Tim Peierls.
+See http://www.jcip.net and the Creative Commons Attribution License
+(http://creativecommons.org/licenses/by/2.5)
+Full text: http://creativecommons.org/licenses/by/2.5/legalcode
+
+License
+
+THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED.
+
+BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE BOUND BY THE TERMS OF THIS LICENSE. THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS.
+
+1. Definitions
+
+    "Collective Work" means a work, such as a periodical issue, anthology or encyclopedia, in which the Work in its entirety in unmodified form, along with a number of other contributions, constituting separate and independent works in themselves, are assembled into a collective whole. A work that constitutes a Collective Work will not be considered a Derivative Work (as defined below) for the purposes of this License.
+    "Derivative Work" means a work based upon the Work or upon the Work and other pre-existing works, such as a translation, musical arrangement, dramatization, fictionalization, motion picture version, sound recording, art reproduction, abridgment, condensation, or any other form in which the Work may be recast, transformed, or adapted, except that a work that constitutes a Collective Work will not be considered a Derivative Work for the purpose of this License. For the avoidance of doubt, where the Work is a musical composition or sound recording, the synchronization of the Work in timed-relation with a moving image ("synching") will be considered a Derivative Work for the purpose of this License.
+    "Licensor" means the individual or entity that offers the Work under the terms of this License.
+    "Original Author" means the individual or entity who created the Work.
+    "Work" means the copyrightable work of authorship offered under the terms of this License.
+    "You" means an individual or entity exercising rights under this License who has not previously violated the terms of this License with respect to the Work, or who has received express permission from the Licensor to exercise rights under this License despite a previous violation.
+
+2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or restrict any rights arising from fair use, first sale or other limitations on the exclusive rights of the copyright owner under copyright law or other applicable laws.
+
+3. License Grant. Subject to the terms and conditions of this License, Licensor hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the duration of the applicable copyright) license to exercise the rights in the Work as stated below:
+
+    to reproduce the Work, to incorporate the Work into one or more Collective Works, and to reproduce the Work as incorporated in the Collective Works;
+    to create and reproduce Derivative Works;
+    to distribute copies or phonorecords of, display publicly, perform publicly, and perform publicly by means of a digital audio transmission the Work including as incorporated in Collective Works;
+    to distribute copies or phonorecords of, display publicly, perform publicly, and perform publicly by means of a digital audio transmission Derivative Works.
+
+    For the avoidance of doubt, where the work is a musical composition:
+        Performance Royalties Under Blanket Licenses. Licensor waives the exclusive right to collect, whether individually or via a performance rights society (e.g. ASCAP, BMI, SESAC), royalties for the public performance or public digital performance (e.g. webcast) of the Work.
+        Mechanical Rights and Statutory Royalties. Licensor waives the exclusive right to collect, whether individually or via a music rights agency or designated agent (e.g. Harry Fox Agency), royalties for any phonorecord You create from the Work ("cover version") and distribute, subject to the compulsory license created by 17 USC Section 115 of the US Copyright Act (or the equivalent in other jurisdictions).
+    Webcasting Rights and Statutory Royalties. For the avoidance of doubt, where the Work is a sound recording, Licensor waives the exclusive right to collect, whether individually or via a performance-rights society (e.g. SoundExchange), royalties for the public digital performance (e.g. webcast) of the Work, subject to the compulsory license created by 17 USC Section 114 of the US Copyright Act (or the equivalent in other jurisdictions).
+
+The above rights may be exercised in all media and formats whether now known or hereafter devised. The above rights include the right to make such modifications as are technically necessary to exercise the rights in other media and formats. All rights not expressly granted by Licensor are hereby reserved.
+
+4. Restrictions.The license granted in Section 3 above is expressly made subject to and limited by the following restrictions:
+
+    You may distribute, publicly display, publicly perform, or publicly digitally perform the Work only under the terms of this License, and You must include a copy of, or the Uniform Resource Identifier for, this License with every copy or phonorecord of the Work You distribute, publicly display, publicly perform, or publicly digitally perform. You may not offer or impose any terms on the Work that alter or restrict the terms of this License or the recipients' exercise of the rights granted hereunder. You may not sublicense the Work. You must keep intact all notices that refer to this License and to the disclaimer of warranties. You may not distribute, publicly display, publicly perform, or publicly digitally perform the Work with any technological measures that control access or use of the Work in a manner inconsistent with the terms of this License Agreement. The above applies to the Work as incorporated in a Collective Work, but this does not require the Collective Work apart from the Work itself to be made subject to the terms of this License. If You create a Collective Work, upon notice from any Licensor You must, to the extent practicable, remove from the Collective Work any credit as required by clause 4(b), as requested. If You create a Derivative Work, upon notice from any Licensor You must, to the extent practicable, remove from the Derivative Work any credit as required by clause 4(b), as requested.
+    If you distribute, publicly display, publicly perform, or publicly digitally perform the Work or any Derivative Works or Collective Works, You must keep intact all copyright notices for the Work and provide, reasonable to the medium or means You are utilizing: (i) the name of the Original Author (or pseudonym, if applicable) if supplied, and/or (ii) if the Original Author and/or Licensor designate another party or parties (e.g. a sponsor institute, publishing entity, journal) for attribution in Licensor's copyright notice, terms of service or by other reasonable means, the name of such party or parties; the title of the Work if supplied; to the extent reasonably practicable, the Uniform Resource Identifier, if any, that Licensor specifies to be associated with the Work, unless such URI does not refer to the copyright notice or licensing information for the Work; and in the case of a Derivative Work, a credit identifying the use of the Work in the Derivative Work (e.g., "French translation of the Work by Original Author," or "Screenplay based on original Work by Original Author"). Such credit may be implemented in any reasonable manner; provided, however, that in the case of a Derivative Work or Collective Work, at a minimum such credit will appear where any other comparable authorship credit appears and in a manner at least as prominent as such other comparable authorship credit.
+
+5. Representations, Warranties and Disclaimer
+
+UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU.
+
+6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+7. Termination
+
+    This License and the rights granted hereunder will terminate automatically upon any breach by You of the terms of this License. Individuals or entities who have received Derivative Works or Collective Works from You under this License, however, will not have their licenses terminated provided such individuals or entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will survive any termination of this License.
+    Subject to the above terms and conditions, the license granted here is perpetual (for the duration of the applicable copyright in the Work). Notwithstanding the above, Licensor reserves the right to release the Work under different license terms or to stop distributing the Work at any time; provided, however that any such election will not serve to withdraw this License (or any other license that has been, or is required to be, granted under the terms of this License), and this License will continue in full force and effect unless terminated as stated above.
+
+8. Miscellaneous
+
+    Each time You distribute or publicly digitally perform the Work or a Collective Work, the Licensor offers to the recipient a license to the Work on the same terms and conditions as the license granted to You under this License.
+    Each time You distribute or publicly digitally perform a Derivative Work, Licensor offers to the recipient a license to the original Work on the same terms and conditions as the license granted to You under this License.
+    If any provision of this License is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this License, and without further action by the parties to this agreement, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable.
+    No term or provision of this License shall be deemed waived and no breach consented to unless such waiver or consent shall be in writing and signed by the party to be charged with such waiver or consent.
+    This License constitutes the entire agreement between the parties with respect to the Work licensed here. There are no understandings, agreements or representations with respect to the Work not specified here. Licensor shall not be bound by any additional provisions that may appear in any communication from You. This License may not be modified without the mutual written agreement of the Licensor and You.
diff --git a/plugins/repository-gcs/licenses/httpcore-NOTICE.txt b/plugins/repository-gcs/licenses/httpcore-NOTICE.txt
new file mode 100644
index 00000000000..c0be50a505e
--- /dev/null
+++ b/plugins/repository-gcs/licenses/httpcore-NOTICE.txt
@@ -0,0 +1,8 @@
+Apache HttpComponents Core
+Copyright 2005-2014 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+This project contains annotations derived from JCIP-ANNOTATIONS
+Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobContainer.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobContainer.java
new file mode 100644
index 00000000000..d8117180ce3
--- /dev/null
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobContainer.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.blobstore.gcs;
+
+import org.elasticsearch.common.blobstore.BlobMetaData;
+import org.elasticsearch.common.blobstore.BlobPath;
+import org.elasticsearch.common.blobstore.BlobStoreException;
+import org.elasticsearch.common.blobstore.support.AbstractBlobContainer;
+import org.elasticsearch.common.bytes.BytesReference;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+
+public class GoogleCloudStorageBlobContainer extends AbstractBlobContainer {
+
+    private final GoogleCloudStorageBlobStore blobStore;
+    private final String path;
+
+    GoogleCloudStorageBlobContainer(BlobPath path, GoogleCloudStorageBlobStore blobStore) {
+        super(path);
+        this.blobStore = blobStore;
+
+        String keyPath = path.buildAsString("/");
+        // TODO Move this keyPath logic to the buildAsString() method
+        if (!keyPath.isEmpty()) {
+            keyPath = keyPath + "/";
+        }
+        this.path = keyPath;
+    }
+
+    @Override
+    public boolean blobExists(String blobName) {
+        try {
+            return blobStore.blobExists(buildKey(blobName));
+        } catch (Exception e) {
+            throw new BlobStoreException("Failed to check if blob [" + blobName + "] exists", e);
+        }
+    }
+
+    @Override
+    public Map<String, BlobMetaData> listBlobs() throws IOException {
+        return blobStore.listBlobs(path);
+    }
+
+    @Override
+    public Map<String, BlobMetaData> listBlobsByPrefix(String prefix) throws IOException {
+        return blobStore.listBlobsByPrefix(path, prefix);
+    }
+
+    @Override
+    public InputStream readBlob(String blobName) throws IOException {
+        return blobStore.readBlob(buildKey(blobName));
+    }
+
+    @Override
+    public void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException {
+        blobStore.writeBlob(buildKey(blobName), inputStream, blobSize);
+    }
+
+    @Override
+    public void writeBlob(String blobName, BytesReference bytes) throws IOException {
+        writeBlob(blobName, bytes.streamInput(), bytes.length());
+    }
+
+    @Override
+    public void deleteBlob(String blobName) throws IOException {
+        blobStore.deleteBlob(buildKey(blobName));
+    }
+
+    @Override
+    public void deleteBlobsByPrefix(String prefix) throws IOException {
+        blobStore.deleteBlobsByPrefix(buildKey(prefix));
+    }
+
+    @Override
+    public void deleteBlobs(Collection<String> blobNames) throws IOException {
+        blobStore.deleteBlobs(buildKeys(blobNames));
+    }
+
+    @Override
+    public void move(String sourceBlobName, String targetBlobName) throws IOException {
+        blobStore.moveBlob(buildKey(sourceBlobName), buildKey(targetBlobName));
+    }
+
+    protected String buildKey(String blobName) {
+        assert blobName != null;
+        return path + blobName;
+    }
+
+    protected Set<String> buildKeys(Collection<String> blobNames) {
+        Set<String> keys = new HashSet<>();
+        if (blobNames != null) {
+            keys.addAll(blobNames.stream().map(this::buildKey).collect(Collectors.toList()));
+        }
+        return keys;
+    }
+}
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStore.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStore.java
new file mode 100644
index 00000000000..7bf79494440
--- /dev/null
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStore.java
@@ -0,0 +1,432 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.blobstore.gcs;
+
+import com.google.api.client.googleapis.batch.BatchRequest;
+import com.google.api.client.googleapis.batch.json.JsonBatchCallback;
+import com.google.api.client.googleapis.json.GoogleJsonError;
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.InputStreamContent;
+import com.google.api.services.storage.Storage;
+import com.google.api.services.storage.model.Bucket;
+import com.google.api.services.storage.model.Objects;
+import com.google.api.services.storage.model.StorageObject;
+import org.elasticsearch.SpecialPermission;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.blobstore.BlobContainer;
+import org.elasticsearch.common.blobstore.BlobMetaData;
+import org.elasticsearch.common.blobstore.BlobPath;
+import org.elasticsearch.common.blobstore.BlobStore;
+import org.elasticsearch.common.blobstore.BlobStoreException;
+import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.CountDown;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Spliterator;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
+
+public class GoogleCloudStorageBlobStore extends AbstractComponent implements BlobStore {
+
+    /**
+     * Google Cloud Storage batch requests are limited to 1000 operations
+     **/
+    private static final int MAX_BATCHING_REQUESTS = 999;
+
+    private final Storage client;
+    private final String bucket;
+
+    public GoogleCloudStorageBlobStore(Settings settings, String bucket, Storage storageClient) {
+        super(settings);
+        this.bucket = bucket;
+        this.client = storageClient;
+
+        if (doesBucketExist(bucket) == false) {
+            throw new BlobStoreException("Bucket [" + bucket + "] does not exist");
+        }
+    }
+
+    @Override
+    public BlobContainer blobContainer(BlobPath path) {
+        return new GoogleCloudStorageBlobContainer(path, this);
+    }
+
+    @Override
+    public void delete(BlobPath path) throws IOException {
+        String keyPath = path.buildAsString("/");
+        // TODO Move this keyPath logic to the buildAsString() method
+        if (!keyPath.isEmpty()) {
+            keyPath = keyPath + "/";
+        }
+        deleteBlobsByPrefix(keyPath);
+    }
+
+    @Override
+    public void close() {
+    }
+
+    /**
+     * Return true if the given bucket exists
+     *
+     * @param bucketName name of the bucket
+     * @return true if the bucket exists, false otherwise
+     */
+    boolean doesBucketExist(String bucketName) {
+        try {
+            return doPrivileged(() -> {
+                try {
+                    Bucket bucket = client.buckets().get(bucketName).execute();
+                    if (bucket != null) {
+                        return Strings.hasText(bucket.getId());
+                    }
+                } catch (GoogleJsonResponseException e) {
+                    GoogleJsonError error = e.getDetails();
+                    if ((e.getStatusCode() == HTTP_NOT_FOUND) || ((error != null) && (error.getCode() == HTTP_NOT_FOUND))) {
+                        return false;
+                    }
+                    throw e;
+                }
+                return false;
+            });
+        } catch (IOException e) {
+            throw new BlobStoreException("Unable to check if bucket [" + bucketName + "] exists", e);
+        }
+    }
+
+    /**
+     * List all blobs in the bucket
+     *
+     * @param path base path of the blobs to list
+     * @return a map of blob names and their metadata
+     */
+    Map<String, BlobMetaData> listBlobs(String path) throws IOException {
+        return doPrivileged(() -> listBlobsByPath(bucket, path, path));
+    }
+
+    /**
+     * List all blobs in the bucket which have a prefix
+     *
+     * @param path   base path of the blobs to list
+     * @param prefix prefix of the blobs to list
+     * @return a map of blob names and their metadata
+     */
+    Map<String, BlobMetaData> listBlobsByPrefix(String path, String prefix) throws IOException {
+        return doPrivileged(() -> listBlobsByPath(bucket, buildKey(path, prefix), path));
+    }
+
+    /**
+     * Lists all blobs in a given bucket
+     *
+     * @param bucketName   name of the bucket
+     * @param path         base path of the blobs to list
+     * @param pathToRemove if true, this path part is removed from blob name
+     * @return a map of blob names and their metadata
+     */
+    private Map<String, BlobMetaData> listBlobsByPath(String bucketName, String path, String pathToRemove) throws IOException {
+        return blobsStream(client, bucketName, path, MAX_BATCHING_REQUESTS)
+                .map(new BlobMetaDataConverter(pathToRemove))
+                .collect(Collectors.toMap(PlainBlobMetaData::name, Function.identity()));
+    }
+
+    /**
+     * Returns true if the blob exists in the bucket
+     *
+     * @param blobName name of the blob
+     * @return true if the blob exists, false otherwise
+     */
+    boolean blobExists(String blobName) throws IOException {
+        return doPrivileged(() -> {
+            try {
+                StorageObject blob = client.objects().get(bucket, blobName).execute();
+                if (blob != null) {
+                    return Strings.hasText(blob.getId());
+                }
+            } catch (GoogleJsonResponseException e) {
+                GoogleJsonError error = e.getDetails();
+                if ((e.getStatusCode() == HTTP_NOT_FOUND) || ((error != null) && (error.getCode() == HTTP_NOT_FOUND))) {
+                    return false;
+                }
+                throw e;
+            }
+            return false;
+        });
+    }
+
+    /**
+     * Returns an {@link java.io.InputStream} for a given blob
+     *
+     * @param blobName name of the blob
+     * @return an InputStream
+     */
+    InputStream readBlob(String blobName) throws IOException {
+        return doPrivileged(() -> {
+            try {
+                Storage.Objects.Get object = client.objects().get(bucket, blobName);
+                return object.executeMediaAsInputStream();
+            } catch (GoogleJsonResponseException e) {
+                GoogleJsonError error = e.getDetails();
+                if ((e.getStatusCode() == HTTP_NOT_FOUND) || ((error != null) && (error.getCode() == HTTP_NOT_FOUND))) {
+                    throw new FileNotFoundException(e.getMessage());
+                }
+                throw e;
+            }
+        });
+    }
+
+    /**
+     * Writes a blob in the bucket.
+     *
+     * @param inputStream content of the blob to be written
+     * @param blobSize    expected size of the blob to be written
+     */
+    void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException {
+        doPrivileged(() -> {
+            InputStreamContent stream = new InputStreamContent(null, inputStream);
+            stream.setLength(blobSize);
+
+            Storage.Objects.Insert insert = client.objects().insert(bucket, null, stream);
+            insert.setName(blobName);
+            insert.execute();
+            return null;
+        });
+    }
+
+    /**
+     * Deletes a blob in the bucket
+     *
+     * @param blobName name of the blob
+     */
+    void deleteBlob(String blobName) throws IOException {
+        doPrivileged(() -> client.objects().delete(bucket, blobName).execute());
+    }
+
+    /**
+     * Deletes multiple blobs in the bucket that have a given prefix
+     *
+     * @param prefix prefix of the buckets to delete
+     */
+    void deleteBlobsByPrefix(String prefix) throws IOException {
+        doPrivileged(() -> {
+            deleteBlobs(listBlobsByPath(bucket, prefix, null).keySet());
+            return null;
+        });
+    }
+
+    /**
+     * Deletes multiple blobs in the given bucket (uses a batch request to perform this)
+     *
+     * @param blobNames names of the bucket to delete
+     */
+    void deleteBlobs(Collection<String> blobNames) throws IOException {
+        if (blobNames == null || blobNames.isEmpty()) {
+            return;
+        }
+
+        if (blobNames.size() == 1) {
+            deleteBlob(blobNames.iterator().next());
+            return;
+        }
+
+        doPrivileged(() -> {
+            final List<Storage.Objects.Delete> deletions = new ArrayList<>();
+            final Iterator<String> blobs = blobNames.iterator();
+
+            while (blobs.hasNext()) {
+                // Create a delete request for each blob to delete
+                deletions.add(client.objects().delete(bucket, blobs.next()));
+
+                if (blobs.hasNext() == false || deletions.size() == MAX_BATCHING_REQUESTS) {
+                    try {
+                        // Deletions are executed using a batch request
+                        BatchRequest batch = client.batch();
+
+                        // Used to track successful deletions
+                        CountDown countDown = new CountDown(deletions.size());
+
+                        for (Storage.Objects.Delete delete : deletions) {
+                            // Queue the delete request in batch
+                            delete.queue(batch, new JsonBatchCallback<Void>() {
+                                @Override
+                                public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) throws IOException {
+                                    logger.error("failed to delete blob [{}] in bucket [{}]: {}", delete.getObject(), delete.getBucket(), e
+                                            .getMessage());
+                                }
+
+                                @Override
+                                public void onSuccess(Void aVoid, HttpHeaders responseHeaders) throws IOException {
+                                    countDown.countDown();
+                                }
+                            });
+                        }
+
+                        batch.execute();
+
+                        if (countDown.isCountedDown() == false) {
+                            throw new IOException("Failed to delete all [" + deletions.size() + "] blobs");
+                        }
+                    } finally {
+                        deletions.clear();
+                    }
+                }
+            }
+            return null;
+        });
+    }
+
+    /**
+     * Moves a blob within the same bucket
+     *
+     * @param sourceBlob name of the blob to move
+     * @param targetBlob new name of the blob in the target bucket
+     */
+    void moveBlob(String sourceBlob, String targetBlob) throws IOException {
+        doPrivileged(() -> {
+            // There's no atomic "move" in GCS so we need to copy and delete
+            client.objects().copy(bucket, sourceBlob, bucket, targetBlob, null).execute();
+            client.objects().delete(bucket, sourceBlob).execute();
+            return null;
+        });
+    }
+
+    /**
+     * Executes a {@link PrivilegedExceptionAction} with privileges enabled.
+     */
+    <T> T doPrivileged(PrivilegedExceptionAction<T> operation) throws IOException {
+        SecurityManager sm = System.getSecurityManager();
+        if (sm != null) {
+            sm.checkPermission(new SpecialPermission());
+        }
+        try {
+            return AccessController.doPrivileged((PrivilegedExceptionAction<T>) operation::run);
+        } catch (PrivilegedActionException e) {
+            throw (IOException) e.getException();
+        }
+    }
+
+    private String buildKey(String keyPath, String s) {
+        assert s != null;
+        return keyPath + s;
+    }
+
+    /**
+     * Converts a {@link StorageObject} to a {@link PlainBlobMetaData}
+     */
+    class BlobMetaDataConverter implements Function<StorageObject, PlainBlobMetaData> {
+
+        private final String pathToRemove;
+
+        BlobMetaDataConverter(String pathToRemove) {
+            this.pathToRemove = pathToRemove;
+        }
+
+        @Override
+        public PlainBlobMetaData apply(StorageObject storageObject) {
+            String blobName = storageObject.getName();
+            if (Strings.hasLength(pathToRemove)) {
+                blobName = blobName.substring(pathToRemove.length());
+            }
+            return new PlainBlobMetaData(blobName, storageObject.getSize().longValue());
+        }
+    }
+
+    /**
+     * Spliterator can be used to list storage objects stored in a bucket.
+     */
+    static class StorageObjectsSpliterator implements Spliterator<StorageObject> {
+
+        private final Storage.Objects.List list;
+
+        StorageObjectsSpliterator(Storage client, String bucketName, String prefix, long pageSize) throws IOException {
+            list = client.objects().list(bucketName);
+            list.setMaxResults(pageSize);
+            if (prefix != null) {
+                list.setPrefix(prefix);
+            }
+        }
+
+        @Override
+        public boolean tryAdvance(Consumer<? super StorageObject> action) {
+            try {
+                // Retrieves the next page of items
+                Objects objects = list.execute();
+
+                if ((objects == null) || (objects.getItems() == null) || (objects.getItems().isEmpty())) {
+                    return false;
+                }
+
+                // Consumes all the items
+                objects.getItems().forEach(action::accept);
+
+                // Sets the page token of the next page,
+                // null indicates that all items have been consumed
+                String next = objects.getNextPageToken();
+                if (next != null) {
+                    list.setPageToken(next);
+                    return true;
+                }
+
+                return false;
+            } catch (Exception e) {
+                throw new BlobStoreException("Exception while listing objects", e);
+            }
+        }
+
+        @Override
+        public Spliterator<StorageObject> trySplit() {
+            return null;
+        }
+
+        @Override
+        public long estimateSize() {
+            return Long.MAX_VALUE;
+        }
+
+        @Override
+        public int characteristics() {
+            return 0;
+        }
+    }
+
+    /**
+     * Returns a {@link Stream} of {@link StorageObject}s that are stored in a given bucket.
+     */
+    static Stream<StorageObject> blobsStream(Storage client, String bucketName, String prefix, long pageSize) throws IOException {
+        return StreamSupport.stream(new StorageObjectsSpliterator(client, bucketName, prefix, pageSize), false);
+    }
+
+}
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStorageModule.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStorageModule.java
new file mode 100644
index 00000000000..8a4bf88ed74
--- /dev/null
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStorageModule.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugin.repository.gcs;
+
+import org.elasticsearch.common.inject.AbstractModule;
+import org.elasticsearch.repositories.gcs.GoogleCloudStorageService;
+
+public class GoogleCloudStorageModule extends AbstractModule {
+
+    @Override
+    protected void configure() {
+        bind(GoogleCloudStorageService.class).to(GoogleCloudStorageService.InternalGoogleCloudStorageService.class).asEagerSingleton();
+    }
+}
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStoragePlugin.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStoragePlugin.java
new file mode 100644
index 00000000000..477a083fad5
--- /dev/null
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStoragePlugin.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugin.repository.gcs;
+
+import com.google.api.client.auth.oauth2.TokenRequest;
+import com.google.api.client.auth.oauth2.TokenResponse;
+import com.google.api.client.googleapis.json.GoogleJsonError;
+import com.google.api.client.http.GenericUrl;
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.json.GenericJson;
+import com.google.api.client.json.webtoken.JsonWebSignature;
+import com.google.api.client.json.webtoken.JsonWebToken;
+import com.google.api.client.util.ClassInfo;
+import com.google.api.client.util.Data;
+import com.google.api.services.storage.Storage;
+import com.google.api.services.storage.model.Bucket;
+import com.google.api.services.storage.model.Objects;
+import com.google.api.services.storage.model.StorageObject;
+import org.elasticsearch.SpecialPermission;
+import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.RepositoriesModule;
+import org.elasticsearch.repositories.gcs.GoogleCloudStorageRepository;
+
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.Collection;
+import java.util.Collections;
+
+public class GoogleCloudStoragePlugin extends Plugin {
+
+    public static final String NAME = "repository-gcs";
+
+    static {
+        /*
+         * Google HTTP client changes access levels because its silly and we
+         * can't allow that on any old stack stack so we pull it here, up front,
+         * so we can cleanly check the permissions for it. Without this changing
+         * the permission can fail if any part of core is on the stack because
+         * our plugin permissions don't allow core to "reach through" plugins to
+         * change the permission. Because that'd be silly.
+         */
+        SecurityManager sm = System.getSecurityManager();
+        if (sm != null) {
+            sm.checkPermission(new SpecialPermission());
+        }
+        AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
+            // ClassInfo put in cache all the fields of a given class
+            // that are annoted with @Key; at the same time it changes
+            // the field access level using setAccessible(). Calling
+            // them here put the ClassInfo in cache (they are never evicted)
+            // before the SecurityManager is installed.
+            ClassInfo.of(HttpHeaders.class, true);
+
+            ClassInfo.of(JsonWebSignature.Header.class, false);
+            ClassInfo.of(JsonWebToken.Payload.class, false);
+
+            ClassInfo.of(TokenRequest.class, false);
+            ClassInfo.of(TokenResponse.class, false);
+
+            ClassInfo.of(GenericJson.class, false);
+            ClassInfo.of(GenericUrl.class, false);
+
+            Data.nullOf(GoogleJsonError.ErrorInfo.class);
+            ClassInfo.of(GoogleJsonError.class, false);
+
+            Data.nullOf(Bucket.Cors.class);
+            ClassInfo.of(Bucket.class, false);
+            ClassInfo.of(Bucket.Cors.class, false);
+            ClassInfo.of(Bucket.Lifecycle.class, false);
+            ClassInfo.of(Bucket.Logging.class, false);
+            ClassInfo.of(Bucket.Owner.class, false);
+            ClassInfo.of(Bucket.Versioning.class, false);
+            ClassInfo.of(Bucket.Website.class, false);
+
+            ClassInfo.of(StorageObject.class, false);
+            ClassInfo.of(StorageObject.Owner.class, false);
+
+            ClassInfo.of(Objects.class, false);
+
+            ClassInfo.of(Storage.Buckets.Get.class, false);
+            ClassInfo.of(Storage.Buckets.Insert.class, false);
+
+            ClassInfo.of(Storage.Objects.Get.class, false);
+            ClassInfo.of(Storage.Objects.Insert.class, false);
+            ClassInfo.of(Storage.Objects.Delete.class, false);
+            ClassInfo.of(Storage.Objects.Copy.class, false);
+            ClassInfo.of(Storage.Objects.List.class, false);
+
+            return null;
+        });
+    }
+
+    @Override
+    public String name() {
+        return NAME;
+    }
+
+    @Override
+    public String description() {
+        return "Google Cloud Storage Repository Plugin";
+    }
+
+    @Override
+    public Collection<Module> nodeModules() {
+        return Collections.singletonList(new GoogleCloudStorageModule());
+    }
+
+    public void onModule(RepositoriesModule repositoriesModule) {
+        repositoriesModule.registerRepository(GoogleCloudStorageRepository.TYPE,
+                GoogleCloudStorageRepository.class, BlobStoreIndexShardRepository.class);
+    }
+}
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
new file mode 100644
index 00000000000..337fbcf8d72
--- /dev/null
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.gcs;
+
+import com.google.api.services.storage.Storage;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.blobstore.BlobPath;
+import org.elasticsearch.common.blobstore.BlobStore;
+import org.elasticsearch.common.blobstore.gcs.GoogleCloudStorageBlobStore;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.snapshots.IndexShardRepository;
+import org.elasticsearch.plugin.repository.gcs.GoogleCloudStoragePlugin;
+import org.elasticsearch.repositories.RepositoryException;
+import org.elasticsearch.repositories.RepositoryName;
+import org.elasticsearch.repositories.RepositorySettings;
+import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
+
+import java.util.function.Function;
+
+import static org.elasticsearch.common.settings.Setting.Property;
+import static org.elasticsearch.common.settings.Setting.boolSetting;
+import static org.elasticsearch.common.settings.Setting.byteSizeSetting;
+import static org.elasticsearch.common.settings.Setting.simpleString;
+import static org.elasticsearch.common.settings.Setting.timeSetting;
+import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
+
+public class GoogleCloudStorageRepository extends BlobStoreRepository {
+
+    public static final String TYPE = "gcs";
+
+    public static final TimeValue NO_TIMEOUT = timeValueMillis(-1);
+
+    public static final Setting<String> BUCKET =
+            simpleString("bucket", Property.NodeScope, Property.Dynamic);
+    public static final Setting<String> BASE_PATH =
+            simpleString("base_path", Property.NodeScope, Property.Dynamic);
+    public static final Setting<Boolean> COMPRESS =
+            boolSetting("compress", false, Property.NodeScope, Property.Dynamic);
+    public static final Setting<ByteSizeValue> CHUNK_SIZE =
+            byteSizeSetting("chunk_size", new ByteSizeValue(100, ByteSizeUnit.MB), Property.NodeScope, Property.Dynamic);
+    public static final Setting<String> APPLICATION_NAME =
+            new Setting<>("application_name", GoogleCloudStoragePlugin.NAME, Function.identity(), Property.NodeScope, Property.Dynamic);
+    public static final Setting<String> SERVICE_ACCOUNT =
+            simpleString("service_account", Property.NodeScope, Property.Dynamic, Property.Filtered);
+    public static final Setting<TimeValue> HTTP_READ_TIMEOUT =
+            timeSetting("http.read_timeout", NO_TIMEOUT, Property.NodeScope, Property.Dynamic);
+    public static final Setting<TimeValue> HTTP_CONNECT_TIMEOUT =
+            timeSetting("http.connect_timeout", NO_TIMEOUT, Property.NodeScope, Property.Dynamic);
+
+    private final ByteSizeValue chunkSize;
+    private final boolean compress;
+    private final BlobPath basePath;
+    private final GoogleCloudStorageBlobStore blobStore;
+
+    @Inject
+    public GoogleCloudStorageRepository(RepositoryName repositoryName, RepositorySettings repositorySettings,
+                                        IndexShardRepository indexShardRepository,
+                                        GoogleCloudStorageService storageService) throws Exception {
+        super(repositoryName.getName(), repositorySettings, indexShardRepository);
+
+        String bucket = get(BUCKET, repositoryName, repositorySettings);
+        String application = get(APPLICATION_NAME, repositoryName, repositorySettings);
+        String serviceAccount = get(SERVICE_ACCOUNT, repositoryName, repositorySettings);
+
+        String basePath = BASE_PATH.get(repositorySettings.settings());
+        if (Strings.hasLength(basePath)) {
+            BlobPath path = new BlobPath();
+            for (String elem : basePath.split("/")) {
+                path = path.add(elem);
+            }
+            this.basePath = path;
+        } else {
+            this.basePath = BlobPath.cleanPath();
+        }
+
+        TimeValue connectTimeout = null;
+        TimeValue readTimeout = null;
+
+        TimeValue timeout = HTTP_CONNECT_TIMEOUT.get(repositorySettings.settings());
+        if ((timeout != null) && (timeout.millis() != NO_TIMEOUT.millis())) {
+            connectTimeout = timeout;
+        }
+
+        timeout = HTTP_READ_TIMEOUT.get(repositorySettings.settings());
+        if ((timeout != null) && (timeout.millis() != NO_TIMEOUT.millis())) {
+            readTimeout = timeout;
+        }
+
+        this.compress = get(COMPRESS, repositoryName, repositorySettings);
+        this.chunkSize = get(CHUNK_SIZE, repositoryName, repositorySettings);
+
+        logger.debug("using bucket [{}], base_path [{}], chunk_size [{}], compress [{}], application [{}]",
+                bucket, basePath, chunkSize, compress, application);
+
+        Storage client = storageService.createClient(serviceAccount, application, connectTimeout, readTimeout);
+        this.blobStore = new GoogleCloudStorageBlobStore(settings, bucket, client);
+    }
+
+
+    @Override
+    protected BlobStore blobStore() {
+        return blobStore;
+    }
+
+    @Override
+    protected BlobPath basePath() {
+        return basePath;
+    }
+
+    @Override
+    protected boolean isCompress() {
+        return compress;
+    }
+
+    @Override
+    protected ByteSizeValue chunkSize() {
+        return chunkSize;
+    }
+
+    /**
+     * Get a given setting from the repository settings, throwing a {@link RepositoryException} if the setting does not exist or is empty.
+     */
+    static <T> T get(Setting<T> setting, RepositoryName name, RepositorySettings repositorySettings) {
+        T value = setting.get(repositorySettings.settings());
+        if (value == null) {
+            throw new RepositoryException(name.getName(), "Setting [" + setting.getKey() + "] is not defined for repository");
+        }
+        if ((value instanceof String) && (Strings.hasText((String) value)) == false) {
+            throw new RepositoryException(name.getName(), "Setting [" + setting.getKey() + "] is empty for repository");
+        }
+        return value;
+    }
+}
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
new file mode 100644
index 00000000000..098ce5f1504
--- /dev/null
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.gcs;
+
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpBackOffIOExceptionHandler;
+import com.google.api.client.http.HttpBackOffUnsuccessfulResponseHandler;
+import com.google.api.client.http.HttpIOExceptionHandler;
+import com.google.api.client.http.HttpRequest;
+import com.google.api.client.http.HttpRequestInitializer;
+import com.google.api.client.http.HttpUnsuccessfulResponseHandler;
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.client.util.ExponentialBackOff;
+import com.google.api.services.storage.Storage;
+import com.google.api.services.storage.StorageScopes;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.env.Environment;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+
+public interface GoogleCloudStorageService {
+
+    /**
+     * Creates a client that can be used to manage Google Cloud Storage objects.
+     *
+     * @param serviceAccount path to service account file
+     * @param application    name of the application
+     * @param connectTimeout connection timeout for HTTP requests
+     * @param readTimeout    read timeout for HTTP requests
+     * @return a Client instance that can be used to manage objects
+     */
+    Storage createClient(String serviceAccount, String application, TimeValue connectTimeout, TimeValue readTimeout) throws Exception;
+
+    /**
+     * Default implementation
+     */
+    class InternalGoogleCloudStorageService extends AbstractComponent implements GoogleCloudStorageService {
+
+        private static final String DEFAULT = "_default_";
+
+        private final Environment environment;
+
+        @Inject
+        public InternalGoogleCloudStorageService(Settings settings, Environment environment) {
+            super(settings);
+            this.environment = environment;
+        }
+
+        @Override
+        public Storage createClient(String serviceAccount, String application, TimeValue connectTimeout, TimeValue readTimeout)
+                throws Exception {
+            try {
+                GoogleCredential credentials = (DEFAULT.equalsIgnoreCase(serviceAccount)) ? loadDefault() : loadCredentials(serviceAccount);
+                NetHttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport();
+
+                Storage.Builder storage = new Storage.Builder(httpTransport, JacksonFactory.getDefaultInstance(),
+                        new DefaultHttpRequestInitializer(credentials, connectTimeout, readTimeout));
+                storage.setApplicationName(application);
+
+                logger.debug("initializing client with service account [{}/{}]",
+                        credentials.getServiceAccountId(), credentials.getServiceAccountUser());
+                return storage.build();
+            } catch (IOException e) {
+                throw new ElasticsearchException("Error when loading Google Cloud Storage credentials file", e);
+            }
+        }
+
+        /**
+         * HTTP request initializer that loads credentials from the service account file
+         * and manages authentication for HTTP requests
+         */
+        private GoogleCredential loadCredentials(String serviceAccount) throws IOException {
+            if (serviceAccount == null) {
+                throw new ElasticsearchException("Cannot load Google Cloud Storage service account file from a null path");
+            }
+
+            Path account = environment.configFile().resolve(serviceAccount);
+            if (Files.exists(account) == false) {
+                throw new ElasticsearchException("Unable to find service account file [" + serviceAccount
+                        + "] defined for repository");
+            }
+
+            try (InputStream is = Files.newInputStream(account)) {
+                GoogleCredential credential = GoogleCredential.fromStream(is);
+                if (credential.createScopedRequired()) {
+                    credential = credential.createScoped(Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL));
+                }
+                return credential;
+            }
+        }
+
+        /**
+         * HTTP request initializer that loads default credentials when running on Compute Engine
+         */
+        private GoogleCredential loadDefault() throws IOException {
+            return GoogleCredential.getApplicationDefault();
+        }
+
+        /**
+         * HTTP request initializer that set timeouts and backoff handler while deferring authentication to GoogleCredential.
+         * See https://cloud.google.com/storage/transfer/create-client#retry
+         */
+        class DefaultHttpRequestInitializer implements HttpRequestInitializer {
+
+            private final TimeValue connectTimeout;
+            private final TimeValue readTimeout;
+            private final GoogleCredential credential;
+            private final HttpUnsuccessfulResponseHandler handler;
+            private final HttpIOExceptionHandler ioHandler;
+
+            DefaultHttpRequestInitializer(GoogleCredential credential, TimeValue connectTimeout, TimeValue readTimeout) {
+                this.credential = credential;
+                this.connectTimeout = connectTimeout;
+                this.readTimeout = readTimeout;
+                this.handler = new HttpBackOffUnsuccessfulResponseHandler(newBackOff());
+                this.ioHandler = new HttpBackOffIOExceptionHandler(newBackOff());
+            }
+
+            @Override
+            public void initialize(HttpRequest request) throws IOException {
+                if (connectTimeout != null) {
+                    request.setConnectTimeout((int) connectTimeout.millis());
+                }
+                if (readTimeout != null) {
+                    request.setReadTimeout((int) readTimeout.millis());
+                }
+
+                request.setIOExceptionHandler(ioHandler);
+                request.setInterceptor(credential);
+
+                request.setUnsuccessfulResponseHandler((req, resp, supportsRetry) -> {
+                            // Let the credential handle the response. If it failed, we rely on our backoff handler
+                            return credential.handleResponse(req, resp, supportsRetry) || handler.handleResponse(req, resp, supportsRetry);
+                        }
+                );
+            }
+
+            private ExponentialBackOff newBackOff() {
+                return new ExponentialBackOff.Builder()
+                        .setInitialIntervalMillis(100)
+                        .setMaxIntervalMillis(6000)
+                        .setMaxElapsedTimeMillis(900000)
+                        .setMultiplier(1.5)
+                        .setRandomizationFactor(0.5)
+                        .build();
+            }
+        }
+    }
+}
diff --git a/plugins/repository-gcs/src/main/plugin-metadata/plugin-security.policy b/plugins/repository-gcs/src/main/plugin-metadata/plugin-security.policy
new file mode 100644
index 00000000000..bc7acd60602
--- /dev/null
+++ b/plugins/repository-gcs/src/main/plugin-metadata/plugin-security.policy
@@ -0,0 +1,26 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+grant {
+  permission java.lang.RuntimePermission "accessDeclaredMembers";
+  permission java.lang.RuntimePermission "setFactory";
+  permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+  permission java.net.URLPermission "http://www.googleapis.com/*", "*";
+  permission java.net.URLPermission "https://www.googleapis.com/*", "*";
+};
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStoreContainerTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStoreContainerTests.java
new file mode 100644
index 00000000000..4fe8c718345
--- /dev/null
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStoreContainerTests.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.blobstore.gcs;
+
+import org.elasticsearch.common.blobstore.BlobStore;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.repositories.ESBlobStoreContainerTestCase;
+
+import java.io.IOException;
+import java.util.Locale;
+
+public class GoogleCloudStorageBlobStoreContainerTests extends ESBlobStoreContainerTestCase {
+
+    @Override
+    protected BlobStore newBlobStore() throws IOException {
+        String bucket = randomAsciiOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
+        return new GoogleCloudStorageBlobStore(Settings.EMPTY, bucket, MockHttpTransport.newStorage(bucket, getTestName()));
+    }
+}
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStoreTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStoreTests.java
new file mode 100644
index 00000000000..b5489466b51
--- /dev/null
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStoreTests.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.blobstore.gcs;
+
+import org.elasticsearch.common.blobstore.BlobStore;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.repositories.ESBlobStoreTestCase;
+
+import java.io.IOException;
+import java.util.Locale;
+
+public class GoogleCloudStorageBlobStoreTests extends ESBlobStoreTestCase {
+
+    @Override
+    protected BlobStore newBlobStore() throws IOException {
+        String bucket = randomAsciiOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
+        return new GoogleCloudStorageBlobStore(Settings.EMPTY, bucket, MockHttpTransport.newStorage(bucket, getTestName()));
+    }
+}
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/common/blobstore/gcs/MockHttpTransport.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/common/blobstore/gcs/MockHttpTransport.java
new file mode 100644
index 00000000000..196fcf12f87
--- /dev/null
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/common/blobstore/gcs/MockHttpTransport.java
@@ -0,0 +1,432 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.blobstore.gcs;
+
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.http.LowLevelHttpRequest;
+import com.google.api.client.http.LowLevelHttpResponse;
+import com.google.api.client.json.Json;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.client.testing.http.MockLowLevelHttpRequest;
+import com.google.api.client.testing.http.MockLowLevelHttpResponse;
+import com.google.api.services.storage.Storage;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.path.PathTrie;
+import org.elasticsearch.common.util.Callback;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.rest.support.RestUtils;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+
+/**
+ * Mock for {@link HttpTransport} to test Google Cloud Storage service.
+ * <p>
+ * This basically handles each type of request used by the {@link GoogleCloudStorageBlobStore} and provides appropriate responses like
+ * the Google Cloud Storage service would do. It is largely based on official documentation available at https://cloud.google
+ * .com/storage/docs/json_api/v1/.
+ */
+public class MockHttpTransport extends com.google.api.client.testing.http.MockHttpTransport {
+
+    private final AtomicInteger objectsCount = new AtomicInteger(0);
+    private final Map<String, String> objectsNames = ConcurrentCollections.newConcurrentMap();
+    private final Map<String, byte[]> objectsContent = ConcurrentCollections.newConcurrentMap();
+
+    private final PathTrie<Handler> handlers = new PathTrie<>(RestUtils.REST_DECODER);
+
+    public MockHttpTransport(String bucket) {
+
+        // GET Bucket
+        //
+        // https://cloud.google.com/storage/docs/json_api/v1/buckets/get
+        handlers.insert("GET https://www.googleapis.com/storage/v1/b/{bucket}", (url, params, req) -> {
+            String name = params.get("bucket");
+            if (Strings.hasText(name) == false) {
+                return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "bucket name is missing");
+            }
+
+            if (name.equals(bucket)) {
+                return newMockResponse().setContent(buildBucketResource(bucket));
+            } else {
+                return newMockError(RestStatus.NOT_FOUND, "bucket not found");
+            }
+        });
+
+        // GET Object
+        //
+        // https://cloud.google.com/storage/docs/json_api/v1/objects/get
+        handlers.insert("GET https://www.googleapis.com/storage/v1/b/{bucket}/o/{object}", (url, params, req) -> {
+            String name = params.get("object");
+            if (Strings.hasText(name) == false) {
+                return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "object name is missing");
+            }
+
+            for (Map.Entry<String, String> object : objectsNames.entrySet()) {
+                if (object.getValue().equals(name)) {
+                    byte[] content = objectsContent.get(object.getKey());
+                    if (content != null) {
+                        return newMockResponse().setContent(buildObjectResource(bucket, name, object.getKey(), content.length));
+                    }
+                }
+            }
+            return newMockError(RestStatus.NOT_FOUND, "object not found");
+        });
+
+        // Download Object
+        //
+        // https://cloud.google.com/storage/docs/request-endpoints
+        handlers.insert("GET https://www.googleapis.com/download/storage/v1/b/{bucket}/o/{object}", (url, params, req) -> {
+            String name = params.get("object");
+            if (Strings.hasText(name) == false) {
+                return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "object name is missing");
+            }
+
+            for (Map.Entry<String, String> object : objectsNames.entrySet()) {
+                if (object.getValue().equals(name)) {
+                    byte[] content = objectsContent.get(object.getKey());
+                    if (content == null) {
+                        return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "object content is missing");
+                    }
+                    return newMockResponse().setContent(new ByteArrayInputStream(content));
+                }
+            }
+            return newMockError(RestStatus.NOT_FOUND, "object not found");
+        });
+
+        // Insert Object (initialization)
+        //
+        // https://cloud.google.com/storage/docs/json_api/v1/objects/insert
+        handlers.insert("POST https://www.googleapis.com/upload/storage/v1/b/{bucket}/o", (url, params, req) -> {
+            if ("resumable".equals(params.get("uploadType")) == false) {
+                return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "upload type must be resumable");
+            }
+
+            String name = params.get("name");
+            if (Strings.hasText(name) == false) {
+                return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "object name is missing");
+            }
+
+            String objectId = String.valueOf(objectsCount.getAndIncrement());
+            objectsNames.put(objectId, name);
+
+            return newMockResponse()
+                    .setStatusCode(RestStatus.CREATED.getStatus())
+                    .addHeader("Location", "https://www.googleapis.com/upload/storage/v1/b/" + bucket +
+                            "/o?uploadType=resumable&upload_id=" + objectId);
+        });
+
+        // Insert Object (upload)
+        //
+        // https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload
+        handlers.insert("PUT https://www.googleapis.com/upload/storage/v1/b/{bucket}/o", (url, params, req) -> {
+            String objectId = params.get("upload_id");
+            if (Strings.hasText(objectId) == false) {
+                return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "upload id is missing");
+            }
+
+            String name = objectsNames.get(objectId);
+            if (Strings.hasText(name) == false) {
+                return newMockError(RestStatus.NOT_FOUND, "object name not found");
+            }
+
+            ByteArrayOutputStream os = new ByteArrayOutputStream((int) req.getContentLength());
+            try {
+                req.getStreamingContent().writeTo(os);
+                os.close();
+            } catch (IOException e) {
+                return newMockError(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage());
+            }
+
+            byte[] content = os.toByteArray();
+            objectsContent.put(objectId, content);
+            return newMockResponse().setContent(buildObjectResource(bucket, name, objectId, content.length));
+        });
+
+        // List Objects
+        //
+        // https://cloud.google.com/storage/docs/json_api/v1/objects/list
+        handlers.insert("GET https://www.googleapis.com/storage/v1/b/{bucket}/o", (url, params, req) -> {
+            String prefix = params.get("prefix");
+
+            try (XContentBuilder builder = jsonBuilder()) {
+                builder.startObject();
+                builder.field("kind", "storage#objects");
+                builder.startArray("items");
+                for (Map.Entry<String, String> o : objectsNames.entrySet()) {
+                    if (prefix != null && o.getValue().startsWith(prefix) == false) {
+                        continue;
+                    }
+                    buildObjectResource(builder, bucket, o.getValue(), o.getKey(), objectsContent.get(o.getKey()).length);
+                }
+                builder.endArray();
+                builder.endObject();
+                return newMockResponse().setContent(builder.string());
+            } catch (IOException e) {
+                return newMockError(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage());
+            }
+        });
+
+        // Delete Object
+        //
+        // https://cloud.google.com/storage/docs/json_api/v1/objects/delete
+        handlers.insert("DELETE https://www.googleapis.com/storage/v1/b/{bucket}/o/{object}", (url, params, req) -> {
+            String name = params.get("object");
+            if (Strings.hasText(name) == false) {
+                return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "object name is missing");
+            }
+
+            String objectId = null;
+            for (Map.Entry<String, String> object : objectsNames.entrySet()) {
+                if (object.getValue().equals(name)) {
+                    objectId = object.getKey();
+                    break;
+                }
+            }
+
+            if (objectId != null) {
+                objectsNames.remove(objectId);
+                objectsContent.remove(objectId);
+                return newMockResponse().setStatusCode(RestStatus.NO_CONTENT.getStatus());
+            }
+            return newMockError(RestStatus.NOT_FOUND, "object not found");
+        });
+
+        // Copy Object
+        //
+        // https://cloud.google.com/storage/docs/json_api/v1/objects/copy
+        handlers.insert("POST https://www.googleapis.com/storage/v1/b/{srcBucket}/o/{srcObject}/copyTo/b/{destBucket}/o/{destObject}",
+                (url, params, req) -> {
+                    String source = params.get("srcObject");
+                    if (Strings.hasText(source) == false) {
+                        return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "source object name is missing");
+                    }
+
+                    String dest = params.get("destObject");
+                    if (Strings.hasText(dest) == false) {
+                        return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "destination object name is missing");
+                    }
+
+                    String srcObjectId = null;
+                    for (Map.Entry<String, String> object : objectsNames.entrySet()) {
+                        if (object.getValue().equals(source)) {
+                            srcObjectId = object.getKey();
+                            break;
+                        }
+                    }
+
+                    if (srcObjectId == null) {
+                        return newMockError(RestStatus.NOT_FOUND, "source object not found");
+                    }
+
+                    byte[] content = objectsContent.get(srcObjectId);
+                    if (content == null) {
+                        return newMockError(RestStatus.NOT_FOUND, "source content can not be found");
+                    }
+
+                    String destObjectId = String.valueOf(objectsCount.getAndIncrement());
+                    objectsNames.put(destObjectId, dest);
+                    objectsContent.put(destObjectId, content);
+
+                    return newMockResponse().setContent(buildObjectResource(bucket, dest, destObjectId, content.length));
+                });
+
+        // Batch
+        //
+        // https://cloud.google.com/storage/docs/json_api/v1/how-tos/batch
+        handlers.insert("POST https://www.googleapis.com/batch", (url, params, req) -> {
+            List<MockLowLevelHttpResponse> responses = new ArrayList<>();
+
+            // A batch request body looks like this:
+            //
+            //            --__END_OF_PART__
+            //            Content-Length: 71
+            //            Content-Type: application/http
+            //            content-id: 1
+            //            content-transfer-encoding: binary
+            //
+            //            DELETE https://www.googleapis.com/storage/v1/b/ohifkgu/o/foo%2Ftest
+            //
+            //
+            //            --__END_OF_PART__
+            //            Content-Length: 71
+            //            Content-Type: application/http
+            //            content-id: 2
+            //            content-transfer-encoding: binary
+            //
+            //            DELETE https://www.googleapis.com/storage/v1/b/ohifkgu/o/bar%2Ftest
+            //
+            //
+            //            --__END_OF_PART__--
+
+            // Here we simply process the request body line by line and delegate to other handlers
+            // if possible.
+            try (ByteArrayOutputStream os = new ByteArrayOutputStream((int) req.getContentLength())) {
+                req.getStreamingContent().writeTo(os);
+
+                Streams.readAllLines(new ByteArrayInputStream(os.toByteArray()), new Callback<String>() {
+                    @Override
+                    public void handle(String line) {
+                        Handler handler = handlers.retrieve(line, params);
+                        if (handler != null) {
+                            try {
+                                responses.add(handler.execute(line, params, req));
+                            } catch (IOException e) {
+                                responses.add(newMockError(RestStatus.INTERNAL_SERVER_ERROR, e.getMessage()));
+                            }
+                        }
+                    }
+                });
+            }
+
+            // Now we can build the response
+            String boundary = "__END_OF_PART__";
+            String sep = "--";
+            String line = "\r\n";
+
+            StringBuilder builder = new StringBuilder();
+            for (MockLowLevelHttpResponse resp : responses) {
+                builder.append(sep).append(boundary).append(line);
+                builder.append(line);
+                builder.append("HTTP/1.1 ").append(resp.getStatusCode()).append(' ').append(resp.getReasonPhrase()).append(line);
+                builder.append("Content-Length: ").append(resp.getContentLength()).append(line);
+                builder.append(line);
+            }
+            builder.append(line);
+            builder.append(sep).append(boundary).append(sep);
+
+            return newMockResponse().setContentType("multipart/mixed; boundary=" + boundary).setContent(builder.toString());
+        });
+    }
+
+    @Override
+    public LowLevelHttpRequest buildRequest(String method, String url) throws IOException {
+        return new MockLowLevelHttpRequest() {
+            @Override
+            public LowLevelHttpResponse execute() throws IOException {
+                String rawPath = url;
+                Map<String, String> params = new HashMap<>();
+
+                int pathEndPos = url.indexOf('?');
+                if (pathEndPos != -1) {
+                    rawPath = url.substring(0, pathEndPos);
+                    RestUtils.decodeQueryString(url, pathEndPos + 1, params);
+                }
+
+                Handler handler = handlers.retrieve(method + " " + rawPath, params);
+                if (handler != null) {
+                    return handler.execute(rawPath, params, this);
+                }
+                return newMockError(RestStatus.INTERNAL_SERVER_ERROR, "Unable to handle request [method=" + method + ", url=" + url + "]");
+            }
+        };
+    }
+
+    private static MockLowLevelHttpResponse newMockResponse() {
+        return new MockLowLevelHttpResponse()
+                .setContentType(Json.MEDIA_TYPE)
+                .setStatusCode(RestStatus.OK.getStatus())
+                .setReasonPhrase(RestStatus.OK.name());
+    }
+
+    private static MockLowLevelHttpResponse newMockError(RestStatus status, String message) {
+        MockLowLevelHttpResponse response = newMockResponse().setStatusCode(status.getStatus()).setReasonPhrase(status.name());
+        try {
+            response.setContent(buildErrorResource(status, message));
+        } catch (IOException e) {
+            response.setContent("Failed to build error resource [" + message + "] because of: " + e.getMessage());
+        }
+        return response;
+    }
+
+    /**
+     * Storage Error JSON representation
+     */
+    private static String buildErrorResource(RestStatus status, String message) throws IOException {
+        return jsonBuilder()
+                .startObject()
+                    .startObject("error")
+                        .field("code", status.getStatus())
+                        .field("message", message)
+                        .startArray("errors")
+                            .startObject()
+                                .field("domain", "global")
+                                .field("reason", status.toString())
+                                .field("message", message)
+                            .endObject()
+                        .endArray()
+                    .endObject()
+                .endObject()
+                .string();
+    }
+
+    /**
+     * Storage Bucket JSON representation as defined in
+     * https://cloud.google.com/storage/docs/json_api/v1/bucket#resource
+     */
+    private static String buildBucketResource(String name) throws IOException {
+        return jsonBuilder().startObject()
+                                .field("kind", "storage#bucket")
+                                .field("id", name)
+                            .endObject()
+                            .string();
+    }
+
+    /**
+     * Storage Object JSON representation as defined in
+     * https://cloud.google.com/storage/docs/json_api/v1/objects#resource
+     */
+    private static XContentBuilder buildObjectResource(XContentBuilder builder, String bucket, String name, String id, int size)
+            throws IOException {
+        return builder.startObject()
+                        .field("kind", "storage#object")
+                        .field("id", String.join("/", bucket, name, id))
+                        .field("name", name)
+                        .field("size", String.valueOf(size))
+                    .endObject();
+    }
+
+    private static String buildObjectResource(String bucket, String name, String id, int size) throws IOException {
+        return buildObjectResource(jsonBuilder(), bucket, name, id, size).string();
+    }
+
+    interface Handler {
+        MockLowLevelHttpResponse execute(String url, Map<String, String> params, MockLowLevelHttpRequest request) throws IOException;
+    }
+
+    /**
+     * Instanciates a mocked Storage client for tests.
+     */
+    public static Storage newStorage(String bucket, String applicationName) {
+        return new Storage.Builder(new MockHttpTransport(bucket), new JacksonFactory(), null)
+                .setApplicationName(applicationName)
+                .build();
+    }
+}
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
new file mode 100644
index 00000000000..c5b57ba6cd6
--- /dev/null
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.gcs;
+
+import com.google.api.services.storage.Storage;
+import org.elasticsearch.common.blobstore.gcs.MockHttpTransport;
+import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.plugin.repository.gcs.GoogleCloudStorageModule;
+import org.elasticsearch.plugin.repository.gcs.GoogleCloudStoragePlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.ESBlobStoreRepositoryIntegTestCase;
+import org.junit.BeforeClass;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+
+public class GoogleCloudStorageBlobStoreRepositoryTests extends ESBlobStoreRepositoryIntegTestCase {
+
+    private static final String BUCKET = "gcs-repository-test";
+
+    // Static storage client shared among all nodes in order to act like a remote repository service:
+    // all nodes must see the same content
+    private static final AtomicReference<Storage> storage = new AtomicReference<>();
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return pluginList(MockGoogleCloudStoragePlugin.class);
+    }
+
+    @Override
+    protected void createTestRepository(String name) {
+        assertAcked(client().admin().cluster().preparePutRepository(name)
+                .setType(GoogleCloudStorageRepository.TYPE)
+                .setSettings(Settings.builder()
+                        .put("bucket", BUCKET)
+                        .put("base_path", GoogleCloudStorageBlobStoreRepositoryTests.class.getSimpleName())
+                        .put("service_account", "_default_")
+                        .put("compress", randomBoolean())
+                        .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
+    }
+
+    @BeforeClass
+    public static void setUpStorage() {
+        storage.set(MockHttpTransport.newStorage(BUCKET, GoogleCloudStorageBlobStoreRepositoryTests.class.getName()));
+    }
+
+    public static class MockGoogleCloudStoragePlugin extends GoogleCloudStoragePlugin {
+
+        public MockGoogleCloudStoragePlugin() {
+        }
+
+        @Override
+        public Collection<Module> nodeModules() {
+            return Collections.singletonList(new MockGoogleCloudStorageModule());
+        }
+    }
+
+    public static class MockGoogleCloudStorageModule extends GoogleCloudStorageModule {
+        @Override
+        protected void configure() {
+            bind(GoogleCloudStorageService.class).to(MockGoogleCloudStorageService.class).asEagerSingleton();
+        }
+    }
+
+    public static class MockGoogleCloudStorageService implements GoogleCloudStorageService {
+
+        @Override
+        public Storage createClient(String serviceAccount, String application, TimeValue connectTimeout, TimeValue readTimeout) throws
+                Exception {
+            return storage.get();
+        }
+    }
+}
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepositoryRestIT.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepositoryRestIT.java
new file mode 100644
index 00000000000..18862d05aa0
--- /dev/null
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepositoryRestIT.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.gcs;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.RestTestCandidate;
+import org.elasticsearch.test.rest.parser.RestTestParseException;
+
+import java.io.IOException;
+
+public class GoogleCloudStorageRepositoryRestIT extends ESRestTestCase {
+
+    public GoogleCloudStorageRepositoryRestIT(@Name("yaml") RestTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws IOException, RestTestParseException {
+        return createParameters(0, 1);
+    }
+}
+
diff --git a/plugins/repository-gcs/src/test/resources/rest-api-spec/test/repository_gcs/10_basic.yaml b/plugins/repository-gcs/src/test/resources/rest-api-spec/test/repository_gcs/10_basic.yaml
new file mode 100644
index 00000000000..a37fb779549
--- /dev/null
+++ b/plugins/repository-gcs/src/test/resources/rest-api-spec/test/repository_gcs/10_basic.yaml
@@ -0,0 +1,13 @@
+# Integration tests for Repository GCS component
+#
+"Repository GCS loaded":
+    - do:
+        cluster.state: {}
+
+    # Get master node id
+    - set: { master_node: master }
+
+    - do:
+        nodes.info: {}
+
+    - match:  { nodes.$master.plugins.0.name: repository-gcs  }
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
index bd5cd499015..07fea76bd8b 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
@@ -289,6 +289,10 @@ fi
     install_and_check_plugin repository azure azure-storage-*.jar
 }
 
+@test "[$GROUP] install repository-gcs plugin" {
+    install_and_check_plugin repository gcs google-api-services-storage-*.jar
+}
+
 @test "[$GROUP] install repository-s3 plugin" {
     install_and_check_plugin repository s3 aws-java-sdk-core-*.jar
 }
@@ -387,6 +391,10 @@ fi
     remove_plugin repository-azure
 }
 
+@test "[$GROUP] remove repository-gcs plugin" {
+    remove_plugin repository-gcs
+}
+
 @test "[$GROUP] remove repository-hdfs plugin" {
     remove_plugin repository-hdfs
 }
diff --git a/settings.gradle b/settings.gradle
index 3a8b0f66210..88217a9dde5 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -37,6 +37,7 @@ List projects = [
   'plugins:mapper-murmur3',
   'plugins:mapper-size',
   'plugins:repository-azure',
+  'plugins:repository-gcs',
   'plugins:repository-hdfs',
   'plugins:repository-s3',
   'plugins:jvm-example',
diff --git a/core/src/test/java/org/elasticsearch/test/ESBlobStoreContainerTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java
similarity index 94%
rename from core/src/test/java/org/elasticsearch/test/ESBlobStoreContainerTestCase.java
rename to test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java
index 291d00a8dde..8462cf007f0 100644
--- a/core/src/test/java/org/elasticsearch/test/ESBlobStoreContainerTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.test;
+package org.elasticsearch.repositories;
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
@@ -25,6 +25,7 @@ import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -32,9 +33,9 @@ import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.elasticsearch.test.ESBlobStoreTestCase.writeRandomBlob;
-import static org.elasticsearch.test.ESBlobStoreTestCase.randomBytes;
-import static org.elasticsearch.test.ESBlobStoreTestCase.readBlobFully;
+import static org.elasticsearch.repositories.ESBlobStoreTestCase.writeRandomBlob;
+import static org.elasticsearch.repositories.ESBlobStoreTestCase.randomBytes;
+import static org.elasticsearch.repositories.ESBlobStoreTestCase.readBlobFully;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.notNullValue;
 
diff --git a/core/src/test/java/org/elasticsearch/test/ESBlobStoreRepositoryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreRepositoryIntegTestCase.java
similarity index 91%
rename from core/src/test/java/org/elasticsearch/test/ESBlobStoreRepositoryIntegTestCase.java
rename to test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreRepositoryIntegTestCase.java
index dc49683de63..2ffd30fa470 100644
--- a/core/src/test/java/org/elasticsearch/test/ESBlobStoreRepositoryIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreRepositoryIntegTestCase.java
@@ -16,13 +16,14 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.test;
+package org.elasticsearch.repositories;
 
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequestBuilder;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequestBuilder;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.Arrays;
 import java.util.HashSet;
@@ -59,7 +60,8 @@ public abstract class ESBlobStoreRepositoryIntegTestCase extends ESIntegTestCase
 
         String snapshotName = randomAsciiName();
         logger.info("-->  create snapshot {}:{}", repoName, snapshotName);
-        assertSuccessfulSnapshot(client().admin().cluster().prepareCreateSnapshot(repoName, snapshotName).setWaitForCompletion(true).setIndices(indexNames));
+        assertSuccessfulSnapshot(client().admin().cluster().prepareCreateSnapshot(repoName, snapshotName)
+                .setWaitForCompletion(true).setIndices(indexNames));
 
         List<String> deleteIndices = randomSubsetOf(randomIntBetween(0, indexCount), indexNames);
         if (deleteIndices.size() > 0) {
@@ -99,6 +101,9 @@ public abstract class ESBlobStoreRepositoryIntegTestCase extends ESIntegTestCase
         for (int i = 0; i < indexCount; i++) {
             assertHitCount(client().prepareSearch(indexNames[i]).setSize(0).get(), docCounts[i]);
         }
+
+        logger.info("-->  delete snapshot {}:{}", repoName, snapshotName);
+        assertAcked(client().admin().cluster().prepareDeleteSnapshot(repoName, snapshotName).get());
     }
 
     public void testMultipleSnapshotAndRollback() throws Exception {
@@ -130,7 +135,8 @@ public abstract class ESBlobStoreRepositoryIntegTestCase extends ESIntegTestCase
             // Check number of documents in this iteration
             docCounts[i] = (int) client().prepareSearch(indexName).setSize(0).get().getHits().totalHits();
             logger.info("-->  create snapshot {}:{} with {} documents", repoName, snapshotName + "-" + i, docCounts[i]);
-            assertSuccessfulSnapshot(client().admin().cluster().prepareCreateSnapshot(repoName, snapshotName + "-" + i).setWaitForCompletion(true).setIndices(indexName));
+            assertSuccessfulSnapshot(client().admin().cluster().prepareCreateSnapshot(repoName, snapshotName + "-" + i)
+                    .setWaitForCompletion(true).setIndices(indexName));
         }
 
         int restoreOperations = randomIntBetween(1, 3);
@@ -142,10 +148,17 @@ public abstract class ESBlobStoreRepositoryIntegTestCase extends ESIntegTestCase
             assertAcked(client().admin().indices().prepareClose(indexName));
 
             logger.info("--> restore index from the snapshot");
-            assertSuccessfulRestore(client().admin().cluster().prepareRestoreSnapshot(repoName, snapshotName + "-" + iterationToRestore).setWaitForCompletion(true));
+            assertSuccessfulRestore(client().admin().cluster().prepareRestoreSnapshot(repoName, snapshotName + "-" + iterationToRestore)
+                    .setWaitForCompletion(true));
+
             ensureGreen();
             assertHitCount(client().prepareSearch(indexName).setSize(0).get(), docCounts[iterationToRestore]);
         }
+
+        for (int i = 0; i < iterationCount; i++) {
+            logger.info("-->  delete snapshot {}:{}", repoName, snapshotName + "-" + i);
+            assertAcked(client().admin().cluster().prepareDeleteSnapshot(repoName, snapshotName + "-" + i).get());
+        }
     }
 
     protected void addRandomDocuments(String name, int numDocs) throws ExecutionException, InterruptedException {
diff --git a/core/src/test/java/org/elasticsearch/test/ESBlobStoreTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreTestCase.java
similarity index 97%
rename from core/src/test/java/org/elasticsearch/test/ESBlobStoreTestCase.java
rename to test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreTestCase.java
index 80432d628ef..be7431795b2 100644
--- a/core/src/test/java/org/elasticsearch/test/ESBlobStoreTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreTestCase.java
@@ -16,12 +16,13 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.test;
+package org.elasticsearch.repositories;
 
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.test.ESTestCase;
 import org.junit.Test;
 
 import java.io.IOException;

From fc6df23feab09c55c93b27e727ef78ba772f5586 Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Thu, 19 May 2016 09:24:45 -0400
Subject: [PATCH 0346/1311] Rename AggregatorBuilder and all of its subclasses
 to AggregationBuilder, in keeping consistent with the Java APIs.

Closes #18377
Closes #18367
---
 .../percolate/PercolateRequestBuilder.java    |   6 +-
 .../percolate/PercolateSourceBuilder.java     |   8 +-
 .../action/search/SearchRequestBuilder.java   |   4 +-
 .../elasticsearch/search/SearchModule.java    | 170 +++++++--------
 ...orBuilder.java => AggregationBuilder.java} |  14 +-
 .../aggregations/AggregationBuilders.java     | 202 +++++++++---------
 .../search/aggregations/Aggregator.java       |   4 +-
 .../aggregations/AggregatorFactories.java     |  50 ++---
 .../aggregations/AggregatorParsers.java       |   2 +-
 ...r.java => ChildrenAggregationBuilder.java} |  16 +-
 ...der.java => FilterAggregationBuilder.java} |  17 +-
 ...er.java => FiltersAggregationBuilder.java} |  28 +--
 ...er.java => GeoGridAggregationBuilder.java} |  16 +-
 .../bucket/geogrid/GeoHashGridAggregator.java |   4 +-
 .../geogrid/GeoHashGridAggregatorFactory.java |   2 +-
 .../bucket/geogrid/GeoHashGridParser.java     |   4 +-
 ...der.java => GlobalAggregationBuilder.java} |  14 +-
 .../histogram/AbstractHistogramBuilder.java   |   6 +-
 ...a => DateHistogramAggregationBuilder.java} |  18 +-
 .../bucket/histogram/DateHistogramParser.java |   8 +-
 ....java => HistogramAggregationBuilder.java} |   8 +-
 .../bucket/histogram/HistogramParser.java     |   2 +-
 ...er.java => MissingAggregationBuilder.java} |  10 +-
 .../bucket/missing/MissingParser.java         |   6 +-
 ...der.java => NestedAggregationBuilder.java} |  16 +-
 ...a => ReverseNestedAggregationBuilder.java} |  18 +-
 .../bucket/range/AbstractRangeBuilder.java    |   6 +-
 ...lder.java => RangeAggregationBuilder.java} |  20 +-
 .../bucket/range/RangeParser.java             |   2 +-
 ....java => DateRangeAggregationBuilder.java} |  42 ++--
 .../bucket/range/date/DateRangeParser.java    |   6 +-
 ...ava => GeoDistanceAggregationBuilder.java} |  36 ++--
 .../range/geodistance/GeoDistanceParser.java  |   6 +-
 ...er.java => IpRangeAggregationBuilder.java} |  34 +--
 .../bucket/range/ip/IpRangeParser.java        |   8 +-
 ...ava => DiversifiedAggregationBuilder.java} |  20 +-
 .../sampler/DiversifiedSamplerParser.java     |   6 +-
 ...er.java => SamplerAggregationBuilder.java} |  18 +-
 ...> SignificantTermsAggregationBuilder.java} |  32 +--
 .../SignificantTermsAggregatorFactory.java    |   2 +-
 .../significant/SignificantTermsParser.java   |  22 +-
 .../significant/UnmappedSignificantTerms.java |   2 +-
 .../bucket/terms/AbstractTermsParser.java     |  20 +-
 ...lder.java => TermsAggregationBuilder.java} |  34 +--
 .../bucket/terms/TermsAggregator.java         |   8 +-
 .../bucket/terms/TermsAggregatorFactory.java  |   2 +-
 .../bucket/terms/TermsParser.java             |  27 +--
 ...uilder.java => AvgAggregationBuilder.java} |  10 +-
 .../aggregations/metrics/avg/AvgParser.java   |   6 +-
 ...ava => CardinalityAggregationBuilder.java} |  14 +-
 .../cardinality/CardinalityParser.java        |  12 +-
 ....java => GeoBoundsAggregationBuilder.java} |  14 +-
 .../metrics/geobounds/GeoBoundsParser.java    |   6 +-
 ...ava => GeoCentroidAggregationBuilder.java} |  12 +-
 .../geocentroid/GeoCentroidParser.java        |   6 +-
 ...uilder.java => MaxAggregationBuilder.java} |  10 +-
 .../aggregations/metrics/max/MaxParser.java   |   6 +-
 ...uilder.java => MinAggregationBuilder.java} |  10 +-
 .../aggregations/metrics/min/MinParser.java   |   6 +-
 .../AbstractPercentilesParser.java            |  14 +-
 ...=> PercentileRanksAggregationBuilder.java} |  20 +-
 .../percentiles/PercentileRanksParser.java    |   9 +-
 ...ava => PercentilesAggregationBuilder.java} |  20 +-
 .../percentiles/PercentilesParser.java        |   9 +-
 ... => ScriptedMetricAggregationBuilder.java} |  26 +--
 ...lder.java => StatsAggregationBuilder.java} |  10 +-
 .../metrics/stats/StatsParser.java            |   6 +-
 ...a => ExtendedStatsAggregationBuilder.java} |  16 +-
 .../stats/extended/ExtendedStatsParser.java   |   6 +-
 ...uilder.java => SumAggregationBuilder.java} |  10 +-
 .../aggregations/metrics/sum/SumParser.java   |   6 +-
 ...er.java => TopHitsAggregationBuilder.java} |  62 +++---
 ...java => ValueCountAggregationBuilder.java} |  10 +-
 .../metrics/valuecount/ValueCountParser.java  |   6 +-
 .../support/AbstractValuesSourceParser.java   |  10 +-
 ...va => ValuesSourceAggregationBuilder.java} |  18 +-
 .../search/builder/SearchSourceBuilder.java   |   4 +-
 .../aggregations/BaseAggregationTestCase.java |   8 +-
 .../aggregations/bucket/ChildrenTests.java    |   8 +-
 .../bucket/DateHistogramTests.java            |   8 +-
 .../aggregations/bucket/DateRangeIT.java      |   4 +-
 .../aggregations/bucket/DateRangeTests.java   |   8 +-
 .../bucket/DiversifiedSamplerIT.java          |  18 +-
 .../bucket/DiversifiedSamplerTests.java       |   8 +-
 .../bucket/GeoDistanceRangeTests.java         |   8 +-
 .../aggregations/bucket/GeoHashGridTests.java |   8 +-
 .../aggregations/bucket/GlobalTests.java      |   8 +-
 .../aggregations/bucket/HistogramTests.java   |   8 +-
 .../search/aggregations/bucket/IpRangeIT.java |   1 -
 .../aggregations/bucket/IpRangeTests.java     |   8 +-
 .../aggregations/bucket/NaNSortingIT.java     |  20 +-
 .../aggregations/bucket/RangeTests.java       |   8 +-
 .../search/aggregations/bucket/SamplerIT.java |   8 +-
 .../aggregations/bucket/SamplerTests.java     |   8 +-
 .../bucket/SignificantTermsTests.java         |   8 +-
 .../aggregations/bucket/TermsTests.java       |   8 +-
 .../bucket/nested/NestedAggregatorTests.java  |   2 +-
 .../bucket/nested/NestedTests.java            |   6 +-
 .../bucket/nested/ReverseNestedTests.java     |   6 +-
 .../SignificanceHeuristicTests.java           |   4 +-
 .../AbstractNumericMetricTestCase.java        |   4 +-
 .../search/aggregations/metrics/AvgTests.java |   8 +-
 .../metrics/ExtendedStatsTests.java           |   8 +-
 .../aggregations/metrics/FilterTests.java     |   8 +-
 .../aggregations/metrics/FiltersTests.java    |  16 +-
 .../aggregations/metrics/GeoBoundsTests.java  |   8 +-
 .../metrics/GeoCentroidTests.java             |   8 +-
 .../search/aggregations/metrics/MaxTests.java |   8 +-
 .../search/aggregations/metrics/MinTests.java |   8 +-
 .../aggregations/metrics/MissingTests.java    |   8 +-
 .../metrics/PercentileRanksTests.java         |   8 +-
 .../metrics/PercentilesTests.java             |   8 +-
 .../metrics/ScriptedMetricTests.java          |   8 +-
 .../aggregations/metrics/StatsTests.java      |   8 +-
 .../search/aggregations/metrics/SumTests.java |   8 +-
 .../aggregations/metrics/TopHitsTests.java    |   8 +-
 .../aggregations/metrics/ValueCountTests.java |   8 +-
 .../metrics/cardinality/CardinalityTests.java |   6 +-
 .../PipelineAggregationHelperTests.java       |  20 +-
 .../pipeline/moving/avg/MovAvgIT.java         |   8 +-
 .../pipeline/serialdiff/SerialDiffIT.java     |   6 +-
 .../messy/tests/EquivalenceTests.java         |   4 +-
 .../messy/tests/MinDocCountTests.java         |   8 +-
 .../tests/TDigestPercentileRanksTests.java    |   6 +-
 .../messy/tests/TDigestPercentilesTests.java  |   6 +-
 125 files changed, 890 insertions(+), 875 deletions(-)
 rename core/src/main/java/org/elasticsearch/search/aggregations/{AggregatorBuilder.java => AggregationBuilder.java} (93%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/{ChildrenAggregatorBuilder.java => ChildrenAggregationBuilder.java} (92%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/{FilterAggregatorBuilder.java => FilterAggregationBuilder.java} (86%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/{FiltersAggregatorBuilder.java => FiltersAggregationBuilder.java} (92%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/{GeoGridAggregatorBuilder.java => GeoGridAggregationBuilder.java} (93%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/{GlobalAggregatorBuilder.java => GlobalAggregationBuilder.java} (84%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/{DateHistogramAggregatorBuilder.java => DateHistogramAggregationBuilder.java} (86%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/{HistogramAggregatorBuilder.java => HistogramAggregationBuilder.java} (90%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/{MissingAggregatorBuilder.java => MissingAggregationBuilder.java} (91%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/{NestedAggregatorBuilder.java => NestedAggregationBuilder.java} (89%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/{ReverseNestedAggregatorBuilder.java => ReverseNestedAggregationBuilder.java} (87%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/{RangeAggregatorBuilder.java => RangeAggregationBuilder.java} (85%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/{DateRangeAggregatorBuilder.java => DateRangeAggregationBuilder.java} (81%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/{GeoDistanceAggregatorBuilder.java => GeoDistanceAggregationBuilder.java} (85%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/{IpRangeAggregatorBuilder.java => IpRangeAggregationBuilder.java} (90%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/{DiversifiedAggregatorBuilder.java => DiversifiedAggregationBuilder.java} (88%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/{SamplerAggregatorBuilder.java => SamplerAggregationBuilder.java} (87%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/{SignificantTermsAggregatorBuilder.java => SignificantTermsAggregationBuilder.java} (87%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/{TermsAggregatorBuilder.java => TermsAggregationBuilder.java} (89%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/{AvgAggregatorBuilder.java => AvgAggregationBuilder.java} (91%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/{CardinalityAggregatorBuilder.java => CardinalityAggregationBuilder.java} (90%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/{GeoBoundsAggregatorBuilder.java => GeoBoundsAggregationBuilder.java} (88%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/{GeoCentroidAggregatorBuilder.java => GeoCentroidAggregationBuilder.java} (90%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/{MaxAggregatorBuilder.java => MaxAggregationBuilder.java} (91%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/{MinAggregatorBuilder.java => MinAggregationBuilder.java} (91%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/{PercentileRanksAggregatorBuilder.java => PercentileRanksAggregationBuilder.java} (91%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/{PercentilesAggregatorBuilder.java => PercentilesAggregationBuilder.java} (91%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/{ScriptedMetricAggregatorBuilder.java => ScriptedMetricAggregationBuilder.java} (93%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/{StatsAggregatorBuilder.java => StatsAggregationBuilder.java} (91%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/{ExtendedStatsAggregatorBuilder.java => ExtendedStatsAggregationBuilder.java} (88%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/{SumAggregatorBuilder.java => SumAggregationBuilder.java} (91%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/{TopHitsAggregatorBuilder.java => TopHitsAggregationBuilder.java} (92%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/{ValueCountAggregatorBuilder.java => ValueCountAggregationBuilder.java} (90%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/support/{ValuesSourceAggregatorBuilder.java => ValuesSourceAggregationBuilder.java} (95%)

diff --git a/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java
index 9286601da69..9490abd0b68 100644
--- a/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java
@@ -27,7 +27,7 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.search.aggregations.AggregatorBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
 import org.elasticsearch.search.highlight.HighlightBuilder;
 import org.elasticsearch.search.sort.SortBuilder;
@@ -165,9 +165,9 @@ public class PercolateRequestBuilder extends ActionRequestBuilder<PercolateReque
 
     /**
      * Delegates to
-     * {@link PercolateSourceBuilder#addAggregation(AggregatorBuilder)}
+     * {@link PercolateSourceBuilder#addAggregation(AggregationBuilder)}
      */
-    public PercolateRequestBuilder addAggregation(AggregatorBuilder<?> aggregationBuilder) {
+    public PercolateRequestBuilder addAggregation(AggregationBuilder<?> aggregationBuilder) {
         sourceBuilder().addAggregation(aggregationBuilder);
         return this;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java b/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java
index a6ee99a476c..5c69d3be50b 100644
--- a/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java
@@ -29,7 +29,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.search.aggregations.AggregatorBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.highlight.HighlightBuilder;
@@ -53,7 +53,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes {
     private List<SortBuilder<?>> sorts;
     private Boolean trackScores;
     private HighlightBuilder highlightBuilder;
-    private List<AggregatorBuilder<?>> aggregationBuilders;
+    private List<AggregationBuilder<?>> aggregationBuilders;
     private List<PipelineAggregatorBuilder<?>> pipelineAggregationBuilders;
 
     /**
@@ -126,7 +126,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes {
     /**
      * Add an aggregation definition.
      */
-    public PercolateSourceBuilder addAggregation(AggregatorBuilder<?> aggregationBuilder) {
+    public PercolateSourceBuilder addAggregation(AggregationBuilder<?> aggregationBuilder) {
         if (aggregationBuilders == null) {
             aggregationBuilders = new ArrayList<>();
         }
@@ -175,7 +175,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes {
             builder.field("aggregations");
             builder.startObject();
             if (aggregationBuilders != null) {
-                for (AggregatorBuilder<?> aggregation : aggregationBuilders) {
+                for (AggregationBuilder<?> aggregation : aggregationBuilders) {
                     aggregation.toXContent(builder, params);
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
index 9830f7be203..5732d43b4c7 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
@@ -28,7 +28,7 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.Template;
 import org.elasticsearch.search.Scroll;
-import org.elasticsearch.search.aggregations.AggregatorBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.highlight.HighlightBuilder;
@@ -373,7 +373,7 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
     /**
      * Adds an aggregation to the search operation.
      */
-    public SearchRequestBuilder addAggregation(AggregatorBuilder<?> aggregation) {
+    public SearchRequestBuilder addAggregation(AggregationBuilder<?> aggregation) {
         sourceBuilder().aggregation(aggregation);
         return this;
     }
diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java
index 59e9c94c27d..e76597c256c 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -98,51 +98,51 @@ import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.action.SearchTransportService;
 import org.elasticsearch.search.aggregations.AggregationPhase;
 import org.elasticsearch.search.aggregations.Aggregator;
-import org.elasticsearch.search.aggregations.AggregatorBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
-import org.elasticsearch.search.aggregations.bucket.children.ChildrenAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.children.ChildrenAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.children.InternalChildren;
-import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter;
-import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.filters.InternalFilters;
-import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridParser;
 import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid;
-import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.global.InternalGlobal;
-import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramParser;
-import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.HistogramParser;
 import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram;
 import org.elasticsearch.search.aggregations.bucket.missing.InternalMissing;
-import org.elasticsearch.search.aggregations.bucket.missing.MissingAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.missing.MissingAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.missing.MissingParser;
 import org.elasticsearch.search.aggregations.bucket.nested.InternalNested;
 import org.elasticsearch.search.aggregations.bucket.nested.InternalReverseNested;
-import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregatorBuilder;
-import org.elasticsearch.search.aggregations.bucket.nested.ReverseNestedAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.nested.ReverseNestedAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.range.InternalRange;
-import org.elasticsearch.search.aggregations.bucket.range.RangeAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.range.RangeParser;
-import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeParser;
 import org.elasticsearch.search.aggregations.bucket.range.date.InternalDateRange;
-import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceParser;
 import org.elasticsearch.search.aggregations.bucket.range.geodistance.InternalGeoDistance;
-import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.range.InternalBinaryRange;
 import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeParser;
-import org.elasticsearch.search.aggregations.bucket.sampler.DiversifiedAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.sampler.DiversifiedAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.sampler.DiversifiedSamplerParser;
 import org.elasticsearch.search.aggregations.bucket.sampler.InternalSampler;
-import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.sampler.UnmappedSampler;
 import org.elasticsearch.search.aggregations.bucket.significant.SignificantLongTerms;
 import org.elasticsearch.search.aggregations.bucket.significant.SignificantStringTerms;
-import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsParser;
 import org.elasticsearch.search.aggregations.bucket.significant.UnmappedSignificantTerms;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.ChiSquare;
@@ -156,50 +156,50 @@ import org.elasticsearch.search.aggregations.bucket.significant.heuristics.Signi
 import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
-import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsParser;
 import org.elasticsearch.search.aggregations.bucket.terms.UnmappedTerms;
-import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.avg.AvgParser;
 import org.elasticsearch.search.aggregations.metrics.avg.InternalAvg;
-import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityParser;
 import org.elasticsearch.search.aggregations.metrics.cardinality.InternalCardinality;
-import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBoundsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBoundsAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBoundsParser;
 import org.elasticsearch.search.aggregations.metrics.geobounds.InternalGeoBounds;
-import org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroidAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroidAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroidParser;
 import org.elasticsearch.search.aggregations.metrics.geocentroid.InternalGeoCentroid;
 import org.elasticsearch.search.aggregations.metrics.max.InternalMax;
-import org.elasticsearch.search.aggregations.metrics.max.MaxAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.max.MaxParser;
 import org.elasticsearch.search.aggregations.metrics.min.InternalMin;
-import org.elasticsearch.search.aggregations.metrics.min.MinAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.min.MinParser;
-import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanksAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanksAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanksParser;
-import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesParser;
 import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentileRanks;
 import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.InternalHDRPercentiles;
 import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentileRanks;
 import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentiles;
 import org.elasticsearch.search.aggregations.metrics.scripted.InternalScriptedMetric;
-import org.elasticsearch.search.aggregations.metrics.scripted.ScriptedMetricAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.scripted.ScriptedMetricAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.stats.InternalStats;
-import org.elasticsearch.search.aggregations.metrics.stats.StatsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.stats.StatsAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.stats.StatsParser;
-import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStatsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStatsAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStatsParser;
 import org.elasticsearch.search.aggregations.metrics.stats.extended.InternalExtendedStats;
 import org.elasticsearch.search.aggregations.metrics.sum.InternalSum;
-import org.elasticsearch.search.aggregations.metrics.sum.SumAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.sum.SumParser;
 import org.elasticsearch.search.aggregations.metrics.tophits.InternalTopHits;
-import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.valuecount.InternalValueCount;
-import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountParser;
 import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -424,10 +424,10 @@ public class SearchModule extends AbstractModule {
      * @param aggregationName names by which the aggregation may be parsed. The first name is special because it is the name that the reader
      *        is registered under.
      */
-    public <AB extends AggregatorBuilder<AB>> void registerAggregation(Writeable.Reader<AB> reader, Aggregator.Parser aggregationParser,
-            ParseField aggregationName) {
+    public <AB extends AggregationBuilder<AB>> void registerAggregation(Writeable.Reader<AB> reader, Aggregator.Parser aggregationParser,
+                                                                        ParseField aggregationName) {
         aggregationParserRegistry.register(aggregationParser, aggregationName);
-        namedWriteableRegistry.register(AggregatorBuilder.class, aggregationName.getPreferredName(), reader);
+        namedWriteableRegistry.register(AggregationBuilder.class, aggregationName.getPreferredName(), reader);
     }
 
     /**
@@ -481,55 +481,57 @@ public class SearchModule extends AbstractModule {
     }
 
     protected void configureAggs() {
-        registerAggregation(AvgAggregatorBuilder::new, new AvgParser(), AvgAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(SumAggregatorBuilder::new, new SumParser(), SumAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(MinAggregatorBuilder::new, new MinParser(), MinAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(MaxAggregatorBuilder::new, new MaxParser(), MaxAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(StatsAggregatorBuilder::new, new StatsParser(), StatsAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(ExtendedStatsAggregatorBuilder::new, new ExtendedStatsParser(),
-        ExtendedStatsAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(ValueCountAggregatorBuilder::new, new ValueCountParser(), ValueCountAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(PercentilesAggregatorBuilder::new, new PercentilesParser(),
-                PercentilesAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(PercentileRanksAggregatorBuilder::new, new PercentileRanksParser(),
-                PercentileRanksAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(CardinalityAggregatorBuilder::new, new CardinalityParser(),
-                CardinalityAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(GlobalAggregatorBuilder::new, GlobalAggregatorBuilder::parse, GlobalAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(MissingAggregatorBuilder::new, new MissingParser(), MissingAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(FilterAggregatorBuilder::new, FilterAggregatorBuilder::parse, FilterAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(FiltersAggregatorBuilder::new, FiltersAggregatorBuilder::parse,
-                FiltersAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(SamplerAggregatorBuilder::new, SamplerAggregatorBuilder::parse,
-                SamplerAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(DiversifiedAggregatorBuilder::new, new DiversifiedSamplerParser(),
-                DiversifiedAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(TermsAggregatorBuilder::new, new TermsParser(), TermsAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(SignificantTermsAggregatorBuilder::new,
+        registerAggregation(AvgAggregationBuilder::new, new AvgParser(), AvgAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(SumAggregationBuilder::new, new SumParser(), SumAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(MinAggregationBuilder::new, new MinParser(), MinAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(MaxAggregationBuilder::new, new MaxParser(), MaxAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(StatsAggregationBuilder::new, new StatsParser(), StatsAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(ExtendedStatsAggregationBuilder::new, new ExtendedStatsParser(),
+        ExtendedStatsAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(ValueCountAggregationBuilder::new, new ValueCountParser(), ValueCountAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(PercentilesAggregationBuilder::new, new PercentilesParser(),
+                PercentilesAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(PercentileRanksAggregationBuilder::new, new PercentileRanksParser(),
+                PercentileRanksAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(CardinalityAggregationBuilder::new, new CardinalityParser(),
+                CardinalityAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(GlobalAggregationBuilder::new, GlobalAggregationBuilder::parse,
+                GlobalAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(MissingAggregationBuilder::new, new MissingParser(), MissingAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(FilterAggregationBuilder::new, FilterAggregationBuilder::parse,
+                FilterAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(FiltersAggregationBuilder::new, FiltersAggregationBuilder::parse,
+                FiltersAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(SamplerAggregationBuilder::new, SamplerAggregationBuilder::parse,
+                SamplerAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(DiversifiedAggregationBuilder::new, new DiversifiedSamplerParser(),
+                DiversifiedAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(TermsAggregationBuilder::new, new TermsParser(), TermsAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(SignificantTermsAggregationBuilder::new,
                 new SignificantTermsParser(significanceHeuristicParserRegistry, queryParserRegistry),
-                SignificantTermsAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(RangeAggregatorBuilder::new, new RangeParser(), RangeAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(DateRangeAggregatorBuilder::new, new DateRangeParser(), DateRangeAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(IpRangeAggregatorBuilder::new, new IpRangeParser(), IpRangeAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(HistogramAggregatorBuilder::new, new HistogramParser(), HistogramAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(DateHistogramAggregatorBuilder::new, new DateHistogramParser(),
-                DateHistogramAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(GeoDistanceAggregatorBuilder::new, new GeoDistanceParser(),
-                GeoDistanceAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(GeoGridAggregatorBuilder::new, new GeoHashGridParser(), GeoGridAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(NestedAggregatorBuilder::new, NestedAggregatorBuilder::parse, NestedAggregatorBuilder.AGGREGATION_FIELD_NAME);
-        registerAggregation(ReverseNestedAggregatorBuilder::new, ReverseNestedAggregatorBuilder::parse,
-                ReverseNestedAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(TopHitsAggregatorBuilder::new, TopHitsAggregatorBuilder::parse,
-                TopHitsAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(GeoBoundsAggregatorBuilder::new, new GeoBoundsParser(), GeoBoundsAggregatorBuilder.AGGREGATION_NAME_FIED);
-        registerAggregation(GeoCentroidAggregatorBuilder::new, new GeoCentroidParser(),
-                GeoCentroidAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(ScriptedMetricAggregatorBuilder::new, ScriptedMetricAggregatorBuilder::parse,
-                ScriptedMetricAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(ChildrenAggregatorBuilder::new, ChildrenAggregatorBuilder::parse,
-                ChildrenAggregatorBuilder.AGGREGATION_NAME_FIELD);
-
+                SignificantTermsAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(RangeAggregationBuilder::new, new RangeParser(), RangeAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(DateRangeAggregationBuilder::new, new DateRangeParser(), DateRangeAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(IpRangeAggregationBuilder::new, new IpRangeParser(), IpRangeAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(HistogramAggregationBuilder::new, new HistogramParser(), HistogramAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(DateHistogramAggregationBuilder::new, new DateHistogramParser(),
+                DateHistogramAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(GeoDistanceAggregationBuilder::new, new GeoDistanceParser(),
+                GeoDistanceAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(GeoGridAggregationBuilder::new, new GeoHashGridParser(), GeoGridAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(NestedAggregationBuilder::new, NestedAggregationBuilder::parse,
+                NestedAggregationBuilder.AGGREGATION_FIELD_NAME);
+        registerAggregation(ReverseNestedAggregationBuilder::new, ReverseNestedAggregationBuilder::parse,
+                ReverseNestedAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(TopHitsAggregationBuilder::new, TopHitsAggregationBuilder::parse,
+                TopHitsAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(GeoBoundsAggregationBuilder::new, new GeoBoundsParser(), GeoBoundsAggregationBuilder.AGGREGATION_NAME_FIED);
+        registerAggregation(GeoCentroidAggregationBuilder::new, new GeoCentroidParser(),
+                GeoCentroidAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(ScriptedMetricAggregationBuilder::new, ScriptedMetricAggregationBuilder::parse,
+                ScriptedMetricAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(ChildrenAggregationBuilder::new, ChildrenAggregationBuilder::parse,
+                ChildrenAggregationBuilder.AGGREGATION_NAME_FIELD);
         registerPipelineAggregation(DerivativePipelineAggregatorBuilder::new, DerivativePipelineAggregatorBuilder::parse,
                 DerivativePipelineAggregatorBuilder.AGGREGATION_NAME_FIELD);
         registerPipelineAggregation(MaxBucketPipelineAggregatorBuilder::new, MaxBucketPipelineAggregatorBuilder.PARSER,
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
similarity index 93%
rename from core/src/main/java/org/elasticsearch/search/aggregations/AggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
index 159ab5a8a10..e0336247c75 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
@@ -36,7 +36,9 @@ import java.util.Objects;
 /**
  * A factory that knows how to create an {@link Aggregator} of a specific type.
  */
-public abstract class AggregatorBuilder<AB extends AggregatorBuilder<AB>> extends ToXContentToBytes implements NamedWriteable, ToXContent {
+public abstract class AggregationBuilder<AB extends AggregationBuilder<AB>>
+    extends ToXContentToBytes
+    implements NamedWriteable, ToXContent {
 
     protected String name;
     protected Type type;
@@ -44,12 +46,12 @@ public abstract class AggregatorBuilder<AB extends AggregatorBuilder<AB>> extend
     protected Map<String, Object> metaData;
 
     /**
-     * Constructs a new aggregator factory.
+     * Constructs a new aggregation builder.
      *
      * @param name  The aggregation name
      * @param type  The aggregation type
      */
-    public AggregatorBuilder(String name, Type type) {
+    public AggregationBuilder(String name, Type type) {
         if (name == null) {
             throw new IllegalArgumentException("[name] must not be null: [" + name + "]");
         }
@@ -63,7 +65,7 @@ public abstract class AggregatorBuilder<AB extends AggregatorBuilder<AB>> extend
     /**
      * Read from a stream.
      */
-    protected AggregatorBuilder(StreamInput in, Type type) throws IOException {
+    protected AggregationBuilder(StreamInput in, Type type) throws IOException {
         name = in.readString();
         this.type = type;
         factoriesBuilder = new AggregatorFactories.Builder(in);
@@ -84,7 +86,7 @@ public abstract class AggregatorBuilder<AB extends AggregatorBuilder<AB>> extend
      * Add a sub aggregation to this aggregation.
      */
     @SuppressWarnings("unchecked")
-    public AB subAggregation(AggregatorBuilder<?> aggregation) {
+    public AB subAggregation(AggregationBuilder<?> aggregation) {
         if (aggregation == null) {
             throw new IllegalArgumentException("[aggregation] must not be null: [" + name + "]");
         }
@@ -178,7 +180,7 @@ public abstract class AggregatorBuilder<AB extends AggregatorBuilder<AB>> extend
         if (getClass() != obj.getClass())
             return false;
         @SuppressWarnings("unchecked")
-        AggregatorBuilder<AB> other = (AggregatorBuilder<AB>) obj;
+        AggregationBuilder<AB> other = (AggregationBuilder<AB>) obj;
         if (!Objects.equals(name, other.name))
             return false;
         if (!Objects.equals(type, other.type))
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java
index b76d5d17891..b1818971d6b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java
@@ -22,65 +22,65 @@ import org.elasticsearch.common.geo.GeoDistance;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.aggregations.bucket.children.Children;
-import org.elasticsearch.search.aggregations.bucket.children.ChildrenAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.children.ChildrenAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
-import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.filters.Filters;
 import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregator.KeyedFilter;
-import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregatorBuilder;
-import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGrid;
 import org.elasticsearch.search.aggregations.bucket.global.Global;
-import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregatorBuilder;
-import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
-import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.missing.Missing;
-import org.elasticsearch.search.aggregations.bucket.missing.MissingAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.missing.MissingAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.nested.Nested;
-import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.nested.ReverseNested;
-import org.elasticsearch.search.aggregations.bucket.nested.ReverseNestedAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.nested.ReverseNestedAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.range.Range;
-import org.elasticsearch.search.aggregations.bucket.range.RangeAggregatorBuilder;
-import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeAggregatorBuilder;
-import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceAggregatorBuilder;
-import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeAggregatorBuilder;
-import org.elasticsearch.search.aggregations.bucket.sampler.DiversifiedAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.sampler.DiversifiedAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.sampler.Sampler;
-import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms;
-import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
-import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.avg.Avg;
-import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.cardinality.Cardinality;
-import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBounds;
-import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBoundsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBoundsAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroid;
-import org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroidAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroidAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.max.Max;
-import org.elasticsearch.search.aggregations.metrics.max.MaxAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.min.Min;
-import org.elasticsearch.search.aggregations.metrics.min.MinAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanks;
-import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanksAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanksAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles;
-import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.scripted.ScriptedMetric;
-import org.elasticsearch.search.aggregations.metrics.scripted.ScriptedMetricAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.scripted.ScriptedMetricAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.stats.Stats;
-import org.elasticsearch.search.aggregations.metrics.stats.StatsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.stats.StatsAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats;
-import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStatsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStatsAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.sum.Sum;
-import org.elasticsearch.search.aggregations.metrics.sum.SumAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.tophits.TopHits;
-import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCount;
-import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder;
 
 /**
  * Utility class to create aggregations.
@@ -93,234 +93,234 @@ public class AggregationBuilders {
     /**
      * Create a new {@link ValueCount} aggregation with the given name.
      */
-    public static ValueCountAggregatorBuilder count(String name) {
-        return new ValueCountAggregatorBuilder(name, null);
+    public static ValueCountAggregationBuilder count(String name) {
+        return new ValueCountAggregationBuilder(name, null);
     }
 
     /**
      * Create a new {@link Avg} aggregation with the given name.
      */
-    public static AvgAggregatorBuilder avg(String name) {
-        return new AvgAggregatorBuilder(name);
+    public static AvgAggregationBuilder avg(String name) {
+        return new AvgAggregationBuilder(name);
     }
 
     /**
      * Create a new {@link Max} aggregation with the given name.
      */
-    public static MaxAggregatorBuilder max(String name) {
-        return new MaxAggregatorBuilder(name);
+    public static MaxAggregationBuilder max(String name) {
+        return new MaxAggregationBuilder(name);
     }
 
     /**
      * Create a new {@link Min} aggregation with the given name.
      */
-    public static MinAggregatorBuilder min(String name) {
-        return new MinAggregatorBuilder(name);
+    public static MinAggregationBuilder min(String name) {
+        return new MinAggregationBuilder(name);
     }
 
     /**
      * Create a new {@link Sum} aggregation with the given name.
      */
-    public static SumAggregatorBuilder sum(String name) {
-        return new SumAggregatorBuilder(name);
+    public static SumAggregationBuilder sum(String name) {
+        return new SumAggregationBuilder(name);
     }
 
     /**
      * Create a new {@link Stats} aggregation with the given name.
      */
-    public static StatsAggregatorBuilder stats(String name) {
-        return new StatsAggregatorBuilder(name);
+    public static StatsAggregationBuilder stats(String name) {
+        return new StatsAggregationBuilder(name);
     }
 
     /**
      * Create a new {@link ExtendedStats} aggregation with the given name.
      */
-    public static ExtendedStatsAggregatorBuilder extendedStats(String name) {
-        return new ExtendedStatsAggregatorBuilder(name);
+    public static ExtendedStatsAggregationBuilder extendedStats(String name) {
+        return new ExtendedStatsAggregationBuilder(name);
     }
 
     /**
      * Create a new {@link Filter} aggregation with the given name.
      */
-    public static FilterAggregatorBuilder filter(String name, QueryBuilder filter) {
-        return new FilterAggregatorBuilder(name, filter);
+    public static FilterAggregationBuilder filter(String name, QueryBuilder filter) {
+        return new FilterAggregationBuilder(name, filter);
     }
 
     /**
      * Create a new {@link Filters} aggregation with the given name.
      */
-    public static FiltersAggregatorBuilder filters(String name, KeyedFilter... filters) {
-        return new FiltersAggregatorBuilder(name, filters);
+    public static FiltersAggregationBuilder filters(String name, KeyedFilter... filters) {
+        return new FiltersAggregationBuilder(name, filters);
     }
 
     /**
      * Create a new {@link Filters} aggregation with the given name.
      */
-    public static FiltersAggregatorBuilder filters(String name, QueryBuilder... filters) {
-        return new FiltersAggregatorBuilder(name, filters);
+    public static FiltersAggregationBuilder filters(String name, QueryBuilder... filters) {
+        return new FiltersAggregationBuilder(name, filters);
     }
 
     /**
      * Create a new {@link Sampler} aggregation with the given name.
      */
-    public static SamplerAggregatorBuilder sampler(String name) {
-        return new SamplerAggregatorBuilder(name);
+    public static SamplerAggregationBuilder sampler(String name) {
+        return new SamplerAggregationBuilder(name);
     }
 
     /**
      * Create a new {@link Sampler} aggregation with the given name.
      */
-    public static DiversifiedAggregatorBuilder diversifiedSampler(String name) {
-        return new DiversifiedAggregatorBuilder(name);
+    public static DiversifiedAggregationBuilder diversifiedSampler(String name) {
+        return new DiversifiedAggregationBuilder(name);
     }
 
     /**
      * Create a new {@link Global} aggregation with the given name.
      */
-    public static GlobalAggregatorBuilder global(String name) {
-        return new GlobalAggregatorBuilder(name);
+    public static GlobalAggregationBuilder global(String name) {
+        return new GlobalAggregationBuilder(name);
     }
 
     /**
      * Create a new {@link Missing} aggregation with the given name.
      */
-    public static MissingAggregatorBuilder missing(String name) {
-        return new MissingAggregatorBuilder(name, null);
+    public static MissingAggregationBuilder missing(String name) {
+        return new MissingAggregationBuilder(name, null);
     }
 
     /**
      * Create a new {@link Nested} aggregation with the given name.
      */
-    public static NestedAggregatorBuilder nested(String name, String path) {
-        return new NestedAggregatorBuilder(name, path);
+    public static NestedAggregationBuilder nested(String name, String path) {
+        return new NestedAggregationBuilder(name, path);
     }
 
     /**
      * Create a new {@link ReverseNested} aggregation with the given name.
      */
-    public static ReverseNestedAggregatorBuilder reverseNested(String name) {
-        return new ReverseNestedAggregatorBuilder(name);
+    public static ReverseNestedAggregationBuilder reverseNested(String name) {
+        return new ReverseNestedAggregationBuilder(name);
     }
 
     /**
      * Create a new {@link Children} aggregation with the given name.
      */
-    public static ChildrenAggregatorBuilder children(String name, String childType) {
-        return new ChildrenAggregatorBuilder(name, childType);
+    public static ChildrenAggregationBuilder children(String name, String childType) {
+        return new ChildrenAggregationBuilder(name, childType);
     }
 
     /**
      * Create a new {@link GeoDistance} aggregation with the given name.
      */
-    public static GeoDistanceAggregatorBuilder geoDistance(String name, GeoPoint origin) {
-        return new GeoDistanceAggregatorBuilder(name, origin);
+    public static GeoDistanceAggregationBuilder geoDistance(String name, GeoPoint origin) {
+        return new GeoDistanceAggregationBuilder(name, origin);
     }
 
     /**
      * Create a new {@link Histogram} aggregation with the given name.
      */
-    public static HistogramAggregatorBuilder histogram(String name) {
-        return new HistogramAggregatorBuilder(name);
+    public static HistogramAggregationBuilder histogram(String name) {
+        return new HistogramAggregationBuilder(name);
     }
 
     /**
      * Create a new {@link GeoHashGrid} aggregation with the given name.
      */
-    public static GeoGridAggregatorBuilder geohashGrid(String name) {
-        return new GeoGridAggregatorBuilder(name);
+    public static GeoGridAggregationBuilder geohashGrid(String name) {
+        return new GeoGridAggregationBuilder(name);
     }
 
     /**
      * Create a new {@link SignificantTerms} aggregation with the given name.
      */
-    public static SignificantTermsAggregatorBuilder significantTerms(String name) {
-        return new SignificantTermsAggregatorBuilder(name, null);
+    public static SignificantTermsAggregationBuilder significantTerms(String name) {
+        return new SignificantTermsAggregationBuilder(name, null);
     }
 
     /**
-     * Create a new {@link DateHistogramAggregatorBuilder} aggregation with the given
+     * Create a new {@link DateHistogramAggregationBuilder} aggregation with the given
      * name.
      */
-    public static DateHistogramAggregatorBuilder dateHistogram(String name) {
-        return new DateHistogramAggregatorBuilder(name);
+    public static DateHistogramAggregationBuilder dateHistogram(String name) {
+        return new DateHistogramAggregationBuilder(name);
     }
 
     /**
      * Create a new {@link Range} aggregation with the given name.
      */
-    public static RangeAggregatorBuilder range(String name) {
-        return new RangeAggregatorBuilder(name);
+    public static RangeAggregationBuilder range(String name) {
+        return new RangeAggregationBuilder(name);
     }
 
     /**
-     * Create a new {@link DateRangeAggregatorBuilder} aggregation with the
+     * Create a new {@link DateRangeAggregationBuilder} aggregation with the
      * given name.
      */
-    public static DateRangeAggregatorBuilder dateRange(String name) {
-        return new DateRangeAggregatorBuilder(name);
+    public static DateRangeAggregationBuilder dateRange(String name) {
+        return new DateRangeAggregationBuilder(name);
     }
 
     /**
-     * Create a new {@link IpRangeAggregatorBuilder} aggregation with the
+     * Create a new {@link IpRangeAggregationBuilder} aggregation with the
      * given name.
      */
-    public static IpRangeAggregatorBuilder ipRange(String name) {
-        return new IpRangeAggregatorBuilder(name);
+    public static IpRangeAggregationBuilder ipRange(String name) {
+        return new IpRangeAggregationBuilder(name);
     }
 
     /**
      * Create a new {@link Terms} aggregation with the given name.
      */
-    public static TermsAggregatorBuilder terms(String name) {
-        return new TermsAggregatorBuilder(name, null);
+    public static TermsAggregationBuilder terms(String name) {
+        return new TermsAggregationBuilder(name, null);
     }
 
     /**
      * Create a new {@link Percentiles} aggregation with the given name.
      */
-    public static PercentilesAggregatorBuilder percentiles(String name) {
-        return new PercentilesAggregatorBuilder(name);
+    public static PercentilesAggregationBuilder percentiles(String name) {
+        return new PercentilesAggregationBuilder(name);
     }
 
     /**
      * Create a new {@link PercentileRanks} aggregation with the given name.
      */
-    public static PercentileRanksAggregatorBuilder percentileRanks(String name) {
-        return new PercentileRanksAggregatorBuilder(name);
+    public static PercentileRanksAggregationBuilder percentileRanks(String name) {
+        return new PercentileRanksAggregationBuilder(name);
     }
 
     /**
      * Create a new {@link Cardinality} aggregation with the given name.
      */
-    public static CardinalityAggregatorBuilder cardinality(String name) {
-        return new CardinalityAggregatorBuilder(name, null);
+    public static CardinalityAggregationBuilder cardinality(String name) {
+        return new CardinalityAggregationBuilder(name, null);
     }
 
     /**
      * Create a new {@link TopHits} aggregation with the given name.
      */
-    public static TopHitsAggregatorBuilder topHits(String name) {
-        return new TopHitsAggregatorBuilder(name);
+    public static TopHitsAggregationBuilder topHits(String name) {
+        return new TopHitsAggregationBuilder(name);
     }
 
     /**
      * Create a new {@link GeoBounds} aggregation with the given name.
      */
-    public static GeoBoundsAggregatorBuilder geoBounds(String name) {
-        return new GeoBoundsAggregatorBuilder(name);
+    public static GeoBoundsAggregationBuilder geoBounds(String name) {
+        return new GeoBoundsAggregationBuilder(name);
     }
 
     /**
      * Create a new {@link GeoCentroid} aggregation with the given name.
      */
-    public static GeoCentroidAggregatorBuilder geoCentroid(String name) {
-        return new GeoCentroidAggregatorBuilder(name);
+    public static GeoCentroidAggregationBuilder geoCentroid(String name) {
+        return new GeoCentroidAggregationBuilder(name);
     }
 
     /**
      * Create a new {@link ScriptedMetric} aggregation with the given name.
      */
-    public static ScriptedMetricAggregatorBuilder scriptedMetric(String name) {
-        return new ScriptedMetricAggregatorBuilder(name);
+    public static ScriptedMetricAggregationBuilder scriptedMetric(String name) {
+        return new ScriptedMetricAggregationBuilder(name);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java
index 426f148e38e..faceada6415 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java
@@ -42,7 +42,7 @@ public abstract class Aggregator extends BucketCollector implements Releasable {
     /**
      * Parses the aggregation request and creates the appropriate aggregator factory for it.
      *
-     * @see AggregatorBuilder
+     * @see AggregationBuilder
      */
     @FunctionalInterface
     public interface Parser {
@@ -55,7 +55,7 @@ public abstract class Aggregator extends BucketCollector implements Releasable {
          * @return                  The resolved aggregator factory or {@code null} in case the aggregation should be skipped
          * @throws java.io.IOException      When parsing fails
          */
-        AggregatorBuilder<?> parse(String aggregationName, QueryParseContext context) throws IOException;
+        AggregationBuilder<?> parse(String aggregationName, QueryParseContext context) throws IOException;
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
index f1236cd5cce..4e07ffcc4d8 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
@@ -124,7 +124,7 @@ public class AggregatorFactories {
 
     public static class Builder extends ToXContentToBytes implements Writeable {
         private final Set<String> names = new HashSet<>();
-        private final List<AggregatorBuilder<?>> aggregatorBuilders = new ArrayList<>();
+        private final List<AggregationBuilder<?>> aggregationBuilders = new ArrayList<>();
         private final List<PipelineAggregatorBuilder<?>> pipelineAggregatorBuilders = new ArrayList<>();
         private boolean skipResolveOrder;
 
@@ -140,7 +140,7 @@ public class AggregatorFactories {
         public Builder(StreamInput in) throws IOException {
             int factoriesSize = in.readVInt();
             for (int i = 0; i < factoriesSize; i++) {
-                addAggregator(in.readNamedWriteable(AggregatorBuilder.class));
+                addAggregator(in.readNamedWriteable(AggregationBuilder.class));
             }
             int pipelineFactoriesSize = in.readVInt();
             for (int i = 0; i < pipelineFactoriesSize; i++) {
@@ -150,8 +150,8 @@ public class AggregatorFactories {
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            out.writeVInt(this.aggregatorBuilders.size());
-            for (AggregatorBuilder<?> factory : aggregatorBuilders) {
+            out.writeVInt(this.aggregationBuilders.size());
+            for (AggregationBuilder<?> factory : aggregationBuilders) {
                 out.writeNamedWriteable(factory);
             }
             out.writeVInt(this.pipelineAggregatorBuilders.size());
@@ -164,11 +164,11 @@ public class AggregatorFactories {
             throw new UnsupportedOperationException("This needs to be removed");
         }
 
-        public Builder addAggregator(AggregatorBuilder<?> factory) {
+        public Builder addAggregator(AggregationBuilder<?> factory) {
             if (!names.add(factory.name)) {
                 throw new IllegalArgumentException("Two sibling aggregations cannot have the same name: [" + factory.name + "]");
             }
-            aggregatorBuilders.add(factory);
+            aggregationBuilders.add(factory);
             return this;
         }
 
@@ -186,30 +186,30 @@ public class AggregatorFactories {
         }
 
         public AggregatorFactories build(AggregationContext context, AggregatorFactory<?> parent) throws IOException {
-            if (aggregatorBuilders.isEmpty() && pipelineAggregatorBuilders.isEmpty()) {
+            if (aggregationBuilders.isEmpty() && pipelineAggregatorBuilders.isEmpty()) {
                 return EMPTY;
             }
             List<PipelineAggregatorBuilder<?>> orderedpipelineAggregators = null;
             if (skipResolveOrder) {
                 orderedpipelineAggregators = new ArrayList<>(pipelineAggregatorBuilders);
             } else {
-                orderedpipelineAggregators = resolvePipelineAggregatorOrder(this.pipelineAggregatorBuilders, this.aggregatorBuilders);
+                orderedpipelineAggregators = resolvePipelineAggregatorOrder(this.pipelineAggregatorBuilders, this.aggregationBuilders);
             }
-            AggregatorFactory<?>[] aggFactories = new AggregatorFactory<?>[aggregatorBuilders.size()];
-            for (int i = 0; i < aggregatorBuilders.size(); i++) {
-                aggFactories[i] = aggregatorBuilders.get(i).build(context, parent);
+            AggregatorFactory<?>[] aggFactories = new AggregatorFactory<?>[aggregationBuilders.size()];
+            for (int i = 0; i < aggregationBuilders.size(); i++) {
+                aggFactories[i] = aggregationBuilders.get(i).build(context, parent);
             }
             return new AggregatorFactories(parent, aggFactories, orderedpipelineAggregators);
         }
 
         private List<PipelineAggregatorBuilder<?>> resolvePipelineAggregatorOrder(
-                List<PipelineAggregatorBuilder<?>> pipelineAggregatorBuilders, List<AggregatorBuilder<?>> aggBuilders) {
+                List<PipelineAggregatorBuilder<?>> pipelineAggregatorBuilders, List<AggregationBuilder<?>> aggBuilders) {
             Map<String, PipelineAggregatorBuilder<?>> pipelineAggregatorBuildersMap = new HashMap<>();
             for (PipelineAggregatorBuilder<?> builder : pipelineAggregatorBuilders) {
                 pipelineAggregatorBuildersMap.put(builder.getName(), builder);
             }
-            Map<String, AggregatorBuilder<?>> aggBuildersMap = new HashMap<>();
-            for (AggregatorBuilder<?> aggBuilder : aggBuilders) {
+            Map<String, AggregationBuilder<?>> aggBuildersMap = new HashMap<>();
+            for (AggregationBuilder<?> aggBuilder : aggBuilders) {
                 aggBuildersMap.put(aggBuilder.name, aggBuilder);
             }
             List<PipelineAggregatorBuilder<?>> orderedPipelineAggregatorrs = new LinkedList<>();
@@ -223,7 +223,7 @@ public class AggregatorFactories {
             return orderedPipelineAggregatorrs;
         }
 
-        private void resolvePipelineAggregatorOrder(Map<String, AggregatorBuilder<?>> aggBuildersMap,
+        private void resolvePipelineAggregatorOrder(Map<String, AggregationBuilder<?>> aggBuildersMap,
                 Map<String, PipelineAggregatorBuilder<?>> pipelineAggregatorBuildersMap,
                 List<PipelineAggregatorBuilder<?>> orderedPipelineAggregators, List<PipelineAggregatorBuilder<?>> unmarkedBuilders,
                 Set<PipelineAggregatorBuilder<?>> temporarilyMarked, PipelineAggregatorBuilder<?> builder) {
@@ -238,7 +238,7 @@ public class AggregatorFactories {
                     if (bucketsPath.equals("_count") || bucketsPath.equals("_key")) {
                         continue;
                     } else if (aggBuildersMap.containsKey(firstAggName)) {
-                        AggregatorBuilder<?> aggBuilder = aggBuildersMap.get(firstAggName);
+                        AggregationBuilder<?> aggBuilder = aggBuildersMap.get(firstAggName);
                         for (int i = 1; i < bucketsPathElements.size(); i++) {
                             PathElement pathElement = bucketsPathElements.get(i);
                             String aggName = pathElement.name;
@@ -247,9 +247,9 @@ public class AggregatorFactories {
                             } else {
                                 // Check the non-pipeline sub-aggregator
                                 // factories
-                                AggregatorBuilder<?>[] subBuilders = aggBuilder.factoriesBuilder.getAggregatorFactories();
+                                AggregationBuilder<?>[] subBuilders = aggBuilder.factoriesBuilder.getAggregatorFactories();
                                 boolean foundSubBuilder = false;
-                                for (AggregatorBuilder<?> subBuilder : subBuilders) {
+                                for (AggregationBuilder<?> subBuilder : subBuilders) {
                                     if (aggName.equals(subBuilder.name)) {
                                         aggBuilder = subBuilder;
                                         foundSubBuilder = true;
@@ -289,8 +289,8 @@ public class AggregatorFactories {
             }
         }
 
-        AggregatorBuilder<?>[] getAggregatorFactories() {
-            return this.aggregatorBuilders.toArray(new AggregatorBuilder<?>[this.aggregatorBuilders.size()]);
+        AggregationBuilder<?>[] getAggregatorFactories() {
+            return this.aggregationBuilders.toArray(new AggregationBuilder<?>[this.aggregationBuilders.size()]);
         }
 
         List<PipelineAggregatorBuilder<?>> getPipelineAggregatorFactories() {
@@ -298,14 +298,14 @@ public class AggregatorFactories {
         }
 
         public int count() {
-            return aggregatorBuilders.size() + pipelineAggregatorBuilders.size();
+            return aggregationBuilders.size() + pipelineAggregatorBuilders.size();
         }
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
-            if (aggregatorBuilders != null) {
-                for (AggregatorBuilder<?> subAgg : aggregatorBuilders) {
+            if (aggregationBuilders != null) {
+                for (AggregationBuilder<?> subAgg : aggregationBuilders) {
                     subAgg.toXContent(builder, params);
                 }
             }
@@ -320,7 +320,7 @@ public class AggregatorFactories {
 
         @Override
         public int hashCode() {
-            return Objects.hash(aggregatorBuilders, pipelineAggregatorBuilders);
+            return Objects.hash(aggregationBuilders, pipelineAggregatorBuilders);
         }
 
         @Override
@@ -330,7 +330,7 @@ public class AggregatorFactories {
             if (getClass() != obj.getClass())
                 return false;
             Builder other = (Builder) obj;
-            if (!Objects.equals(aggregatorBuilders, other.aggregatorBuilders))
+            if (!Objects.equals(aggregationBuilders, other.aggregationBuilders))
                 return false;
             if (!Objects.equals(pipelineAggregatorBuilders, other.pipelineAggregatorBuilders))
                 return false;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorParsers.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorParsers.java
index 2fd1f63d620..55345d6e5ec 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorParsers.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorParsers.java
@@ -104,7 +104,7 @@ public class AggregatorParsers {
                         + token + "], expected a [" + XContentParser.Token.START_OBJECT + "].");
             }
 
-            AggregatorBuilder<?> aggFactory = null;
+            AggregationBuilder<?> aggFactory = null;
             PipelineAggregatorBuilder<?> pipelineAggregatorFactory = null;
             AggregatorFactories.Builder subFactories = null;
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenAggregationBuilder.java
similarity index 92%
rename from core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenAggregationBuilder.java
index c5982d1000e..3749d2b2edd 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/ChildrenAggregationBuilder.java
@@ -36,7 +36,7 @@ import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.FieldContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Bytes.ParentChild;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
@@ -44,7 +44,7 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import java.io.IOException;
 import java.util.Objects;
 
-public class ChildrenAggregatorBuilder extends ValuesSourceAggregatorBuilder<ParentChild, ChildrenAggregatorBuilder> {
+public class ChildrenAggregationBuilder extends ValuesSourceAggregationBuilder<ParentChild, ChildrenAggregationBuilder> {
     public static final String NAME = InternalChildren.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
@@ -59,7 +59,7 @@ public class ChildrenAggregatorBuilder extends ValuesSourceAggregatorBuilder<Par
      * @param childType
      *            the type of children documents
      */
-    public ChildrenAggregatorBuilder(String name, String childType) {
+    public ChildrenAggregationBuilder(String name, String childType) {
         super(name, InternalChildren.TYPE, ValuesSourceType.BYTES, ValueType.STRING);
         if (childType == null) {
             throw new IllegalArgumentException("[childType] must not be null: [" + name + "]");
@@ -70,7 +70,7 @@ public class ChildrenAggregatorBuilder extends ValuesSourceAggregatorBuilder<Par
     /**
      * Read from a stream.
      */
-    public ChildrenAggregatorBuilder(StreamInput in) throws IOException {
+    public ChildrenAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalChildren.TYPE, ValuesSourceType.BYTES, ValueType.STRING);
         childType = in.readString();
     }
@@ -121,7 +121,7 @@ public class ChildrenAggregatorBuilder extends ValuesSourceAggregatorBuilder<Par
         return builder;
     }
 
-    public static ChildrenAggregatorBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+    public static ChildrenAggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
         String childType = null;
 
         XContentParser.Token token;
@@ -148,7 +148,7 @@ public class ChildrenAggregatorBuilder extends ValuesSourceAggregatorBuilder<Par
         }
 
 
-        return new ChildrenAggregatorBuilder(aggregationName, childType);
+        return new ChildrenAggregationBuilder(aggregationName, childType);
     }
 
     @Override
@@ -158,7 +158,7 @@ public class ChildrenAggregatorBuilder extends ValuesSourceAggregatorBuilder<Par
 
     @Override
     protected boolean innerEquals(Object obj) {
-        ChildrenAggregatorBuilder other = (ChildrenAggregatorBuilder) obj;
+        ChildrenAggregationBuilder other = (ChildrenAggregationBuilder) obj;
         return Objects.equals(childType, other.childType);
     }
 
@@ -166,4 +166,4 @@ public class ChildrenAggregatorBuilder extends ValuesSourceAggregatorBuilder<Par
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
similarity index 86%
rename from core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
index 9024e902d7d..1a73471c32e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
@@ -24,12 +24,11 @@ import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.EmptyQueryBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregatorBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
@@ -37,7 +36,7 @@ import org.elasticsearch.search.aggregations.support.AggregationContext;
 import java.io.IOException;
 import java.util.Objects;
 
-public class FilterAggregatorBuilder extends AggregatorBuilder<FilterAggregatorBuilder> {
+public class FilterAggregationBuilder extends AggregationBuilder<FilterAggregationBuilder> {
     public static final String NAME = InternalFilter.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
@@ -51,7 +50,7 @@ public class FilterAggregatorBuilder extends AggregatorBuilder<FilterAggregatorB
      *            filter will fall into the bucket defined by this
      *            {@link Filter} aggregation.
      */
-    public FilterAggregatorBuilder(String name, QueryBuilder filter) {
+    public FilterAggregationBuilder(String name, QueryBuilder filter) {
         super(name, InternalFilter.TYPE);
         if (filter == null) {
             throw new IllegalArgumentException("[filter] must not be null: [" + name + "]");
@@ -66,7 +65,7 @@ public class FilterAggregatorBuilder extends AggregatorBuilder<FilterAggregatorB
     /**
      * Read from a stream.
      */
-    public FilterAggregatorBuilder(StreamInput in) throws IOException {
+    public FilterAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalFilter.TYPE);
         filter = in.readNamedWriteable(QueryBuilder.class);
     }
@@ -90,7 +89,7 @@ public class FilterAggregatorBuilder extends AggregatorBuilder<FilterAggregatorB
         return builder;
     }
 
-    public static FilterAggregatorBuilder parse(String aggregationName, QueryParseContext context)
+    public static FilterAggregationBuilder parse(String aggregationName, QueryParseContext context)
             throws IOException {
         QueryBuilder filter = context.parseInnerQueryBuilder();
 
@@ -98,7 +97,7 @@ public class FilterAggregatorBuilder extends AggregatorBuilder<FilterAggregatorB
             throw new ParsingException(null, "filter cannot be null in filter aggregation [{}]", aggregationName);
         }
 
-        return new FilterAggregatorBuilder(aggregationName, filter);
+        return new FilterAggregationBuilder(aggregationName, filter);
     }
 
 
@@ -109,7 +108,7 @@ public class FilterAggregatorBuilder extends AggregatorBuilder<FilterAggregatorB
 
     @Override
     protected boolean doEquals(Object obj) {
-        FilterAggregatorBuilder other = (FilterAggregatorBuilder) obj;
+        FilterAggregationBuilder other = (FilterAggregationBuilder) obj;
         return Objects.equals(filter, other.filter);
     }
 
@@ -117,4 +116,4 @@ public class FilterAggregatorBuilder extends AggregatorBuilder<FilterAggregatorB
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregationBuilder.java
similarity index 92%
rename from core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregationBuilder.java
index 52a1b949b4b..45744db3e30 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregationBuilder.java
@@ -28,7 +28,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregatorBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregator.KeyedFilter;
@@ -43,7 +43,7 @@ import java.util.Objects;
 
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 
-public class FiltersAggregatorBuilder extends AggregatorBuilder<FiltersAggregatorBuilder> {
+public class FiltersAggregationBuilder extends AggregationBuilder<FiltersAggregationBuilder> {
     public static final String NAME = InternalFilters.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
@@ -62,11 +62,11 @@ public class FiltersAggregatorBuilder extends AggregatorBuilder<FiltersAggregato
      * @param filters
      *            the KeyedFilters to use with this aggregation.
      */
-    public FiltersAggregatorBuilder(String name, KeyedFilter... filters) {
+    public FiltersAggregationBuilder(String name, KeyedFilter... filters) {
         this(name, Arrays.asList(filters));
     }
 
-    private FiltersAggregatorBuilder(String name, List<KeyedFilter> filters) {
+    private FiltersAggregationBuilder(String name, List<KeyedFilter> filters) {
         super(name, InternalFilters.TYPE);
         // internally we want to have a fixed order of filters, regardless of the order of the filters in the request
         this.filters = new ArrayList<>(filters);
@@ -80,7 +80,7 @@ public class FiltersAggregatorBuilder extends AggregatorBuilder<FiltersAggregato
      * @param filters
      *            the filters to use with this aggregation
      */
-    public FiltersAggregatorBuilder(String name, QueryBuilder... filters) {
+    public FiltersAggregationBuilder(String name, QueryBuilder... filters) {
         super(name, InternalFilters.TYPE);
         List<KeyedFilter> keyedFilters = new ArrayList<>(filters.length);
         for (int i = 0; i < filters.length; i++) {
@@ -93,7 +93,7 @@ public class FiltersAggregatorBuilder extends AggregatorBuilder<FiltersAggregato
     /**
      * Read from a stream.
      */
-    public FiltersAggregatorBuilder(StreamInput in) throws IOException {
+    public FiltersAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalFilters.TYPE);
         keyed = in.readBoolean();
         int filtersSize = in.readVInt();
@@ -131,7 +131,7 @@ public class FiltersAggregatorBuilder extends AggregatorBuilder<FiltersAggregato
     /**
      * Set whether to include a bucket for documents not matching any filter
      */
-    public FiltersAggregatorBuilder otherBucket(boolean otherBucket) {
+    public FiltersAggregationBuilder otherBucket(boolean otherBucket) {
         this.otherBucket = otherBucket;
         return this;
     }
@@ -154,7 +154,7 @@ public class FiltersAggregatorBuilder extends AggregatorBuilder<FiltersAggregato
      * Set the key to use for the bucket for documents not matching any
      * filter.
      */
-    public FiltersAggregatorBuilder otherBucketKey(String otherBucketKey) {
+    public FiltersAggregationBuilder otherBucketKey(String otherBucketKey) {
         if (otherBucketKey == null) {
             throw new IllegalArgumentException("[otherBucketKey] must not be null: [" + name + "]");
         }
@@ -199,7 +199,7 @@ public class FiltersAggregatorBuilder extends AggregatorBuilder<FiltersAggregato
         return builder;
     }
 
-    public static FiltersAggregatorBuilder parse(String aggregationName, QueryParseContext context)
+    public static FiltersAggregationBuilder parse(String aggregationName, QueryParseContext context)
             throws IOException {
         XContentParser parser = context.parser();
 
@@ -264,12 +264,12 @@ public class FiltersAggregatorBuilder extends AggregatorBuilder<FiltersAggregato
             otherBucketKey = "_other_";
         }
 
-        FiltersAggregatorBuilder factory;
+        FiltersAggregationBuilder factory;
         if (keyedFilters != null) {
-            factory = new FiltersAggregatorBuilder(aggregationName,
+            factory = new FiltersAggregationBuilder(aggregationName,
                     keyedFilters.toArray(new FiltersAggregator.KeyedFilter[keyedFilters.size()]));
         } else {
-            factory = new FiltersAggregatorBuilder(aggregationName,
+            factory = new FiltersAggregationBuilder(aggregationName,
                     nonKeyedFilters.toArray(new QueryBuilder[nonKeyedFilters.size()]));
         }
         if (otherBucket != null) {
@@ -288,7 +288,7 @@ public class FiltersAggregatorBuilder extends AggregatorBuilder<FiltersAggregato
 
     @Override
     protected boolean doEquals(Object obj) {
-        FiltersAggregatorBuilder other = (FiltersAggregatorBuilder) obj;
+        FiltersAggregationBuilder other = (FiltersAggregationBuilder) obj;
         return Objects.equals(filters, other.filters)
                 && Objects.equals(keyed, other.keyed)
                 && Objects.equals(otherBucket, other.otherBucket)
@@ -299,4 +299,4 @@ public class FiltersAggregatorBuilder extends AggregatorBuilder<FiltersAggregato
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
similarity index 93%
rename from core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
index 76227ba5704..68e93f41991 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
@@ -37,7 +37,7 @@ import org.elasticsearch.search.aggregations.bucket.BucketUtils;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
@@ -45,7 +45,7 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import java.io.IOException;
 import java.util.Objects;
 
-public class GeoGridAggregatorBuilder extends ValuesSourceAggregatorBuilder<ValuesSource.GeoPoint, GeoGridAggregatorBuilder> {
+public class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource.GeoPoint, GeoGridAggregationBuilder> {
     public static final String NAME = InternalGeoHashGrid.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
@@ -53,14 +53,14 @@ public class GeoGridAggregatorBuilder extends ValuesSourceAggregatorBuilder<Valu
     private int requiredSize = GeoHashGridParser.DEFAULT_MAX_NUM_CELLS;
     private int shardSize = -1;
 
-    public GeoGridAggregatorBuilder(String name) {
+    public GeoGridAggregationBuilder(String name) {
         super(name, InternalGeoHashGrid.TYPE, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
     }
 
     /**
      * Read from a stream.
      */
-    public GeoGridAggregatorBuilder(StreamInput in) throws IOException {
+    public GeoGridAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalGeoHashGrid.TYPE, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
         precision = in.readVInt();
         requiredSize = in.readVInt();
@@ -74,7 +74,7 @@ public class GeoGridAggregatorBuilder extends ValuesSourceAggregatorBuilder<Valu
         out.writeVInt(shardSize);
     }
 
-    public GeoGridAggregatorBuilder precision(int precision) {
+    public GeoGridAggregationBuilder precision(int precision) {
         this.precision = GeoHashGridParams.checkPrecision(precision);
         return this;
     }
@@ -83,7 +83,7 @@ public class GeoGridAggregatorBuilder extends ValuesSourceAggregatorBuilder<Valu
         return precision;
     }
 
-    public GeoGridAggregatorBuilder size(int size) {
+    public GeoGridAggregationBuilder size(int size) {
         if (size < -1) {
             throw new IllegalArgumentException(
                     "[size] must be greater than or equal to 0. Found [" + shardSize + "] in [" + name + "]");
@@ -96,7 +96,7 @@ public class GeoGridAggregatorBuilder extends ValuesSourceAggregatorBuilder<Valu
         return requiredSize;
     }
 
-    public GeoGridAggregatorBuilder shardSize(int shardSize) {
+    public GeoGridAggregationBuilder shardSize(int shardSize) {
         if (shardSize < -1) {
             throw new IllegalArgumentException(
                     "[shardSize] must be greater than or equal to 0. Found [" + shardSize + "] in [" + name + "]");
@@ -145,7 +145,7 @@ public class GeoGridAggregatorBuilder extends ValuesSourceAggregatorBuilder<Valu
 
     @Override
     protected boolean innerEquals(Object obj) {
-        GeoGridAggregatorBuilder other = (GeoGridAggregatorBuilder) obj;
+        GeoGridAggregationBuilder other = (GeoGridAggregationBuilder) obj;
         if (precision != other.precision) {
             return false;
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java
index d63c30272bc..d3653e87422 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java
@@ -46,10 +46,10 @@ public class GeoHashGridAggregator extends BucketsAggregator {
 
     private final int requiredSize;
     private final int shardSize;
-    private final GeoGridAggregatorBuilder.CellIdSource valuesSource;
+    private final GeoGridAggregationBuilder.CellIdSource valuesSource;
     private final LongHash bucketOrds;
 
-    public GeoHashGridAggregator(String name, AggregatorFactories factories, GeoGridAggregatorBuilder.CellIdSource valuesSource,
+    public GeoHashGridAggregator(String name, AggregatorFactories factories, GeoGridAggregationBuilder.CellIdSource valuesSource,
             int requiredSize, int shardSize, AggregationContext aggregationContext, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) throws IOException {
         super(name, factories, aggregationContext, parent, pipelineAggregators, metaData);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java
index ba4f84017c1..1b2c4c26372 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java
@@ -25,7 +25,7 @@ import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.NonCollectingAggregator;
-import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregatorBuilder.CellIdSource;
+import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder.CellIdSource;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParser.java
index 2ecf4953e78..1ae31e09ba0 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridParser.java
@@ -45,10 +45,10 @@ public class GeoHashGridParser extends GeoPointValuesSourceParser {
     }
 
     @Override
-    protected GeoGridAggregatorBuilder createFactory(
+    protected GeoGridAggregationBuilder createFactory(
             String aggregationName, ValuesSourceType valuesSourceType,
             ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        GeoGridAggregatorBuilder factory = new GeoGridAggregatorBuilder(aggregationName);
+        GeoGridAggregationBuilder factory = new GeoGridAggregationBuilder(aggregationName);
         Integer precision = (Integer) otherOptions.get(GeoHashGridParams.FIELD_PRECISION);
         if (precision != null) {
             factory.precision(precision);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java
similarity index 84%
rename from core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java
index 7a60dcdab93..0f7e0713598 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java
@@ -24,25 +24,25 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregatorBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 
 import java.io.IOException;
 
-public class GlobalAggregatorBuilder extends AggregatorBuilder<GlobalAggregatorBuilder> {
+public class GlobalAggregationBuilder extends AggregationBuilder<GlobalAggregationBuilder> {
     public static final String NAME = InternalGlobal.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
-    public GlobalAggregatorBuilder(String name) {
+    public GlobalAggregationBuilder(String name) {
         super(name, InternalGlobal.TYPE);
     }
 
     /**
      * Read from a stream.
      */
-    public GlobalAggregatorBuilder(StreamInput in) throws IOException {
+    public GlobalAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalGlobal.TYPE);
     }
 
@@ -64,9 +64,9 @@ public class GlobalAggregatorBuilder extends AggregatorBuilder<GlobalAggregatorB
         return builder;
     }
 
-    public static GlobalAggregatorBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+    public static GlobalAggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
         context.parser().nextToken();
-        return new GlobalAggregatorBuilder(aggregationName);
+        return new GlobalAggregationBuilder(aggregationName);
     }
 
     @Override
@@ -83,4 +83,4 @@ public class GlobalAggregatorBuilder extends AggregatorBuilder<GlobalAggregatorB
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramBuilder.java
index 707cdabb6f9..703748f14a9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramBuilder.java
@@ -24,14 +24,14 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.rounding.Rounding;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
 import java.util.Objects;
 
 public abstract class AbstractHistogramBuilder<AB extends AbstractHistogramBuilder<AB>>
-        extends ValuesSourceAggregatorBuilder<ValuesSource.Numeric, AB> {
+        extends ValuesSourceAggregationBuilder<ValuesSource.Numeric, AB> {
 
     protected long interval;
     protected long offset = 0;
@@ -200,4 +200,4 @@ public abstract class AbstractHistogramBuilder<AB extends AbstractHistogramBuild
                 && Objects.equals(minDocCount, other.minDocCount)
                 && Objects.equals(extendedBounds, other.extendedBounds);
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
similarity index 86%
rename from core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
index 2442e9f5aab..f07bac1a7d1 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramAggregationBuilder.java
@@ -33,21 +33,21 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import java.io.IOException;
 import java.util.Objects;
 
-public class DateHistogramAggregatorBuilder extends AbstractHistogramBuilder<DateHistogramAggregatorBuilder> {
+public class DateHistogramAggregationBuilder extends AbstractHistogramBuilder<DateHistogramAggregationBuilder> {
 
     public static final String NAME = InternalDateHistogram.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     private DateHistogramInterval dateHistogramInterval;
 
-    public DateHistogramAggregatorBuilder(String name) {
+    public DateHistogramAggregationBuilder(String name) {
         super(name, InternalDateHistogram.HISTOGRAM_FACTORY);
     }
 
     /**
      * Read from a stream.
      */
-    public DateHistogramAggregatorBuilder(StreamInput in) throws IOException {
+    public DateHistogramAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalDateHistogram.HISTOGRAM_FACTORY);
         dateHistogramInterval = in.readOptionalWriteable(DateHistogramInterval::new);
     }
@@ -61,7 +61,7 @@ public class DateHistogramAggregatorBuilder extends AbstractHistogramBuilder<Dat
     /**
      * Set the interval.
      */
-    public DateHistogramAggregatorBuilder dateHistogramInterval(DateHistogramInterval dateHistogramInterval) {
+    public DateHistogramAggregationBuilder dateHistogramInterval(DateHistogramInterval dateHistogramInterval) {
         if (dateHistogramInterval == null) {
             throw new IllegalArgumentException("[dateHistogramInterval] must not be null: [" + name + "]");
         }
@@ -69,7 +69,7 @@ public class DateHistogramAggregatorBuilder extends AbstractHistogramBuilder<Dat
         return this;
     }
 
-    public DateHistogramAggregatorBuilder offset(String offset) {
+    public DateHistogramAggregationBuilder offset(String offset) {
         if (offset == null) {
             throw new IllegalArgumentException("[offset] must not be null: [" + name + "]");
         }
@@ -79,12 +79,12 @@ public class DateHistogramAggregatorBuilder extends AbstractHistogramBuilder<Dat
     protected static long parseStringOffset(String offset) {
         if (offset.charAt(0) == '-') {
             return -TimeValue
-                    .parseTimeValue(offset.substring(1), null, DateHistogramAggregatorBuilder.class.getSimpleName() + ".parseOffset")
+                    .parseTimeValue(offset.substring(1), null, DateHistogramAggregationBuilder.class.getSimpleName() + ".parseOffset")
                     .millis();
         }
         int beginIndex = offset.charAt(0) == '+' ? 1 : 0;
         return TimeValue
-                .parseTimeValue(offset.substring(beginIndex), null, DateHistogramAggregatorBuilder.class.getSimpleName() + ".parseOffset")
+                .parseTimeValue(offset.substring(beginIndex), null, DateHistogramAggregationBuilder.class.getSimpleName() + ".parseOffset")
                 .millis();
     }
 
@@ -121,7 +121,7 @@ public class DateHistogramAggregatorBuilder extends AbstractHistogramBuilder<Dat
 
     @Override
     protected boolean innerEquals(Object obj) {
-        DateHistogramAggregatorBuilder other = (DateHistogramAggregatorBuilder) obj;
+        DateHistogramAggregationBuilder other = (DateHistogramAggregationBuilder) obj;
         return super.innerEquals(obj) && Objects.equals(dateHistogramInterval, other.dateHistogramInterval);
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramParser.java
index 79149573313..b033de5597e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramParser.java
@@ -42,9 +42,9 @@ public class DateHistogramParser extends HistogramParser {
     }
 
     @Override
-    protected DateHistogramAggregatorBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        DateHistogramAggregatorBuilder factory = new DateHistogramAggregatorBuilder(aggregationName);
+    protected DateHistogramAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
+                                                            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
+        DateHistogramAggregationBuilder factory = new DateHistogramAggregationBuilder(aggregationName);
         Object interval = otherOptions.get(Rounding.Interval.INTERVAL_FIELD);
         if (interval == null) {
             throw new ParsingException(null, "Missing required field [interval] for histogram aggregation [" + aggregationName + "]");
@@ -89,6 +89,6 @@ public class DateHistogramParser extends HistogramParser {
 
     @Override
     protected long parseStringOffset(String offset) throws IOException {
-        return DateHistogramAggregatorBuilder.parseStringOffset(offset);
+        return DateHistogramAggregationBuilder.parseStringOffset(offset);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java
similarity index 90%
rename from core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java
index 49bbd7160cf..54d52466bbb 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramAggregationBuilder.java
@@ -29,18 +29,18 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 
 import java.io.IOException;
 
-public class HistogramAggregatorBuilder extends AbstractHistogramBuilder<HistogramAggregatorBuilder> {
+public class HistogramAggregationBuilder extends AbstractHistogramBuilder<HistogramAggregationBuilder> {
     public static final String NAME = InternalHistogram.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
-    public HistogramAggregatorBuilder(String name) {
+    public HistogramAggregationBuilder(String name) {
         super(name, InternalHistogram.HISTOGRAM_FACTORY);
     }
 
     /**
      * Read from a stream.
      */
-    public HistogramAggregatorBuilder(StreamInput in) throws IOException {
+    public HistogramAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalHistogram.HISTOGRAM_FACTORY);
     }
 
@@ -55,4 +55,4 @@ public class HistogramAggregatorBuilder extends AbstractHistogramBuilder<Histogr
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramParser.java
index 0553b2d339d..371dfad07d7 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramParser.java
@@ -47,7 +47,7 @@ public class HistogramParser extends NumericValuesSourceParser {
     @Override
     protected AbstractHistogramBuilder<?> createFactory(String aggregationName, ValuesSourceType valuesSourceType,
             ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        HistogramAggregatorBuilder factory = new HistogramAggregatorBuilder(aggregationName);
+        HistogramAggregationBuilder factory = new HistogramAggregationBuilder(aggregationName);
         Long interval = (Long) otherOptions.get(Rounding.Interval.INTERVAL_FIELD);
         if (interval == null) {
             throw new ParsingException(null, "Missing required field [interval] for histogram aggregation [" + aggregationName + "]");
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java
similarity index 91%
rename from core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java
index 34263980bf4..f10f7683841 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java
@@ -28,25 +28,25 @@ import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
 
-public class MissingAggregatorBuilder extends ValuesSourceAggregatorBuilder<ValuesSource, MissingAggregatorBuilder> {
+public class MissingAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource, MissingAggregationBuilder> {
     public static final String NAME = InternalMissing.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
-    public MissingAggregatorBuilder(String name, ValueType targetValueType) {
+    public MissingAggregationBuilder(String name, ValueType targetValueType) {
         super(name, InternalMissing.TYPE, ValuesSourceType.ANY, targetValueType);
     }
 
     /**
      * Read from a stream.
      */
-    public MissingAggregatorBuilder(StreamInput in) throws IOException {
+    public MissingAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalMissing.TYPE, ValuesSourceType.ANY);
     }
 
@@ -85,4 +85,4 @@ public class MissingAggregatorBuilder extends ValuesSourceAggregatorBuilder<Valu
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingParser.java
index 7bfddbd8e53..fff81db1301 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingParser.java
@@ -41,8 +41,8 @@ public class MissingParser extends AnyValuesSourceParser {
     }
 
     @Override
-    protected MissingAggregatorBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        return new MissingAggregatorBuilder(aggregationName, targetValueType);
+    protected MissingAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
+                                                      ValueType targetValueType, Map<ParseField, Object> otherOptions) {
+        return new MissingAggregationBuilder(aggregationName, targetValueType);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java
similarity index 89%
rename from core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java
index f01a78e9211..33771910f16 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java
@@ -26,7 +26,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregatorBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
@@ -34,7 +34,7 @@ import org.elasticsearch.search.aggregations.support.AggregationContext;
 import java.io.IOException;
 import java.util.Objects;
 
-public class NestedAggregatorBuilder extends AggregatorBuilder<NestedAggregatorBuilder> {
+public class NestedAggregationBuilder extends AggregationBuilder<NestedAggregationBuilder> {
     public static final String NAME = InternalNested.TYPE.name();
     public static final ParseField AGGREGATION_FIELD_NAME = new ParseField(NAME);
 
@@ -47,7 +47,7 @@ public class NestedAggregatorBuilder extends AggregatorBuilder<NestedAggregatorB
      *            the path to use for this nested aggregation. The path must
      *            match the path to a nested object in the mappings.
      */
-    public NestedAggregatorBuilder(String name, String path) {
+    public NestedAggregationBuilder(String name, String path) {
         super(name, InternalNested.TYPE);
         if (path == null) {
             throw new IllegalArgumentException("[path] must not be null: [" + name + "]");
@@ -58,7 +58,7 @@ public class NestedAggregatorBuilder extends AggregatorBuilder<NestedAggregatorB
     /**
      * Read from a stream.
      */
-    public NestedAggregatorBuilder(StreamInput in) throws IOException {
+    public NestedAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalNested.TYPE);
         path = in.readString();
     }
@@ -89,7 +89,7 @@ public class NestedAggregatorBuilder extends AggregatorBuilder<NestedAggregatorB
         return builder;
     }
 
-    public static NestedAggregatorBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+    public static NestedAggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
         String path = null;
 
         XContentParser.Token token;
@@ -115,7 +115,7 @@ public class NestedAggregatorBuilder extends AggregatorBuilder<NestedAggregatorB
             throw new ParsingException(parser.getTokenLocation(), "Missing [path] field for nested aggregation [" + aggregationName + "]");
         }
 
-        return new NestedAggregatorBuilder(aggregationName, path);
+        return new NestedAggregationBuilder(aggregationName, path);
     }
 
 
@@ -126,7 +126,7 @@ public class NestedAggregatorBuilder extends AggregatorBuilder<NestedAggregatorB
 
     @Override
     protected boolean doEquals(Object obj) {
-        NestedAggregatorBuilder other = (NestedAggregatorBuilder) obj;
+        NestedAggregationBuilder other = (NestedAggregationBuilder) obj;
         return Objects.equals(path, other.path);
     }
 
@@ -134,4 +134,4 @@ public class NestedAggregatorBuilder extends AggregatorBuilder<NestedAggregatorB
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java
similarity index 87%
rename from core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java
index 349b610e877..6257f219daf 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java
@@ -26,7 +26,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregatorBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
@@ -34,20 +34,20 @@ import org.elasticsearch.search.aggregations.support.AggregationContext;
 import java.io.IOException;
 import java.util.Objects;
 
-public class ReverseNestedAggregatorBuilder extends AggregatorBuilder<ReverseNestedAggregatorBuilder> {
+public class ReverseNestedAggregationBuilder extends AggregationBuilder<ReverseNestedAggregationBuilder> {
     public static final String NAME = InternalReverseNested.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     private String path;
 
-    public ReverseNestedAggregatorBuilder(String name) {
+    public ReverseNestedAggregationBuilder(String name) {
         super(name, InternalReverseNested.TYPE);
     }
 
     /**
      * Read from a stream.
      */
-    public ReverseNestedAggregatorBuilder(StreamInput in) throws IOException {
+    public ReverseNestedAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalReverseNested.TYPE);
         path = in.readOptionalString();
     }
@@ -62,7 +62,7 @@ public class ReverseNestedAggregatorBuilder extends AggregatorBuilder<ReverseNes
      * the path to a nested object in the mappings. If it is not specified
      * then this aggregation will go back to the root document.
      */
-    public ReverseNestedAggregatorBuilder path(String path) {
+    public ReverseNestedAggregationBuilder path(String path) {
         if (path == null) {
             throw new IllegalArgumentException("[path] must not be null: [" + name + "]");
         }
@@ -93,7 +93,7 @@ public class ReverseNestedAggregatorBuilder extends AggregatorBuilder<ReverseNes
         return builder;
     }
 
-    public static ReverseNestedAggregatorBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+    public static ReverseNestedAggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
         String path = null;
 
         XContentParser.Token token;
@@ -114,7 +114,7 @@ public class ReverseNestedAggregatorBuilder extends AggregatorBuilder<ReverseNes
             }
         }
 
-        ReverseNestedAggregatorBuilder factory = new ReverseNestedAggregatorBuilder(
+        ReverseNestedAggregationBuilder factory = new ReverseNestedAggregationBuilder(
                 aggregationName);
         if (path != null) {
             factory.path(path);
@@ -130,7 +130,7 @@ public class ReverseNestedAggregatorBuilder extends AggregatorBuilder<ReverseNes
 
     @Override
     protected boolean doEquals(Object obj) {
-        ReverseNestedAggregatorBuilder other = (ReverseNestedAggregatorBuilder) obj;
+        ReverseNestedAggregationBuilder other = (ReverseNestedAggregationBuilder) obj;
         return Objects.equals(path, other.path);
     }
 
@@ -138,4 +138,4 @@ public class ReverseNestedAggregatorBuilder extends AggregatorBuilder<ReverseNes
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java
index fbf4269c26e..0121da2fa90 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java
@@ -25,7 +25,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -33,7 +33,7 @@ import java.util.List;
 import java.util.Objects;
 
 public abstract class AbstractRangeBuilder<AB extends AbstractRangeBuilder<AB, R>, R extends Range>
-        extends ValuesSourceAggregatorBuilder<ValuesSource.Numeric, AB> {
+        extends ValuesSourceAggregationBuilder<ValuesSource.Numeric, AB> {
 
     protected final InternalRange.Factory<?, ?> rangeFactory;
     protected List<R> ranges = new ArrayList<>();
@@ -103,4 +103,4 @@ public abstract class AbstractRangeBuilder<AB extends AbstractRangeBuilder<AB, R
         return Objects.equals(ranges, other.ranges)
                 && Objects.equals(keyed, other.keyed);
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java
similarity index 85%
rename from core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java
index 8347625b09a..c4ba10ed6dc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregationBuilder.java
@@ -30,18 +30,18 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 
 import java.io.IOException;
 
-public class RangeAggregatorBuilder extends AbstractRangeBuilder<RangeAggregatorBuilder, Range> {
+public class RangeAggregationBuilder extends AbstractRangeBuilder<RangeAggregationBuilder, Range> {
     public static final String NAME = InternalRange.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
-    public RangeAggregatorBuilder(String name) {
+    public RangeAggregationBuilder(String name) {
         super(name, InternalRange.FACTORY);
     }
 
     /**
      * Read from a stream.
      */
-    public RangeAggregatorBuilder(StreamInput in) throws IOException {
+    public RangeAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalRange.FACTORY, Range::new);
     }
 
@@ -55,7 +55,7 @@ public class RangeAggregatorBuilder extends AbstractRangeBuilder<RangeAggregator
      * @param to
      *            the upper bound on the distances, exclusive
      */
-    public RangeAggregatorBuilder addRange(String key, double from, double to) {
+    public RangeAggregationBuilder addRange(String key, double from, double to) {
         addRange(new Range(key, from, to));
         return this;
     }
@@ -65,7 +65,7 @@ public class RangeAggregatorBuilder extends AbstractRangeBuilder<RangeAggregator
      * automatically generated based on <code>from</code> and
      * <code>to</code>.
      */
-    public RangeAggregatorBuilder addRange(double from, double to) {
+    public RangeAggregationBuilder addRange(double from, double to) {
         return addRange(null, from, to);
     }
 
@@ -77,7 +77,7 @@ public class RangeAggregatorBuilder extends AbstractRangeBuilder<RangeAggregator
      * @param to
      *            the upper bound on the distances, exclusive
      */
-    public RangeAggregatorBuilder addUnboundedTo(String key, double to) {
+    public RangeAggregationBuilder addUnboundedTo(String key, double to) {
         addRange(new Range(key, null, to));
         return this;
     }
@@ -86,7 +86,7 @@ public class RangeAggregatorBuilder extends AbstractRangeBuilder<RangeAggregator
      * Same as {@link #addUnboundedTo(String, double)} but the key will be
      * computed automatically.
      */
-    public RangeAggregatorBuilder addUnboundedTo(double to) {
+    public RangeAggregationBuilder addUnboundedTo(double to) {
         return addUnboundedTo(null, to);
     }
 
@@ -98,7 +98,7 @@ public class RangeAggregatorBuilder extends AbstractRangeBuilder<RangeAggregator
      * @param from
      *            the lower bound on the distances, inclusive
      */
-    public RangeAggregatorBuilder addUnboundedFrom(String key, double from) {
+    public RangeAggregationBuilder addUnboundedFrom(String key, double from) {
         addRange(new Range(key, from, null));
         return this;
     }
@@ -107,7 +107,7 @@ public class RangeAggregatorBuilder extends AbstractRangeBuilder<RangeAggregator
      * Same as {@link #addUnboundedFrom(String, double)} but the key will be
      * computed automatically.
      */
-    public RangeAggregatorBuilder addUnboundedFrom(double from) {
+    public RangeAggregationBuilder addUnboundedFrom(double from) {
         return addUnboundedFrom(null, from);
     }
 
@@ -122,4 +122,4 @@ public class RangeAggregatorBuilder extends AbstractRangeBuilder<RangeAggregator
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeParser.java
index 8a441f2f3a9..6ebd413d2d4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeParser.java
@@ -51,7 +51,7 @@ public class RangeParser extends NumericValuesSourceParser {
     @Override
     protected AbstractRangeBuilder<?, ?> createFactory(String aggregationName, ValuesSourceType valuesSourceType,
             ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        RangeAggregatorBuilder factory = new RangeAggregatorBuilder(aggregationName);
+        RangeAggregationBuilder factory = new RangeAggregationBuilder(aggregationName);
         @SuppressWarnings("unchecked")
         List<? extends Range> ranges = (List<? extends Range>) otherOptions.get(RangeAggregator.RANGES_FIELD);
         for (Range range : ranges) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeAggregationBuilder.java
similarity index 81%
rename from core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeAggregationBuilder.java
index 9c28461df6c..392744a4f1c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeAggregationBuilder.java
@@ -33,18 +33,18 @@ import org.joda.time.DateTime;
 
 import java.io.IOException;
 
-public class DateRangeAggregatorBuilder extends AbstractRangeBuilder<DateRangeAggregatorBuilder, RangeAggregator.Range> {
+public class DateRangeAggregationBuilder extends AbstractRangeBuilder<DateRangeAggregationBuilder, RangeAggregator.Range> {
     public static final String NAME = InternalDateRange.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
-    public DateRangeAggregatorBuilder(String name) {
+    public DateRangeAggregationBuilder(String name) {
         super(name, InternalDateRange.FACTORY);
     }
 
     /**
      * Read from a stream.
      */
-    public DateRangeAggregatorBuilder(StreamInput in) throws IOException {
+    public DateRangeAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalDateRange.FACTORY, Range::new);
     }
 
@@ -63,7 +63,7 @@ public class DateRangeAggregatorBuilder extends AbstractRangeBuilder<DateRangeAg
      * @param to
      *            the upper bound on the dates, exclusive
      */
-    public DateRangeAggregatorBuilder addRange(String key, String from, String to) {
+    public DateRangeAggregationBuilder addRange(String key, String from, String to) {
         addRange(new Range(key, from, to));
         return this;
     }
@@ -72,7 +72,7 @@ public class DateRangeAggregatorBuilder extends AbstractRangeBuilder<DateRangeAg
      * Same as {@link #addRange(String, String, String)} but the key will be
      * automatically generated based on <code>from</code> and <code>to</code>.
      */
-    public DateRangeAggregatorBuilder addRange(String from, String to) {
+    public DateRangeAggregationBuilder addRange(String from, String to) {
         return addRange(null, from, to);
     }
 
@@ -84,7 +84,7 @@ public class DateRangeAggregatorBuilder extends AbstractRangeBuilder<DateRangeAg
      * @param to
      *            the upper bound on the dates, exclusive
      */
-    public DateRangeAggregatorBuilder addUnboundedTo(String key, String to) {
+    public DateRangeAggregationBuilder addUnboundedTo(String key, String to) {
         addRange(new Range(key, null, to));
         return this;
     }
@@ -93,7 +93,7 @@ public class DateRangeAggregatorBuilder extends AbstractRangeBuilder<DateRangeAg
      * Same as {@link #addUnboundedTo(String, String)} but the key will be
      * computed automatically.
      */
-    public DateRangeAggregatorBuilder addUnboundedTo(String to) {
+    public DateRangeAggregationBuilder addUnboundedTo(String to) {
         return addUnboundedTo(null, to);
     }
 
@@ -105,7 +105,7 @@ public class DateRangeAggregatorBuilder extends AbstractRangeBuilder<DateRangeAg
      * @param from
      *            the lower bound on the distances, inclusive
      */
-    public DateRangeAggregatorBuilder addUnboundedFrom(String key, String from) {
+    public DateRangeAggregationBuilder addUnboundedFrom(String key, String from) {
         addRange(new Range(key, from, null));
         return this;
     }
@@ -114,7 +114,7 @@ public class DateRangeAggregatorBuilder extends AbstractRangeBuilder<DateRangeAg
      * Same as {@link #addUnboundedFrom(String, String)} but the key will be
      * computed automatically.
      */
-    public DateRangeAggregatorBuilder addUnboundedFrom(String from) {
+    public DateRangeAggregationBuilder addUnboundedFrom(String from) {
         return addUnboundedFrom(null, from);
     }
 
@@ -128,7 +128,7 @@ public class DateRangeAggregatorBuilder extends AbstractRangeBuilder<DateRangeAg
      * @param to
      *            the upper bound on the dates, exclusive
      */
-    public DateRangeAggregatorBuilder addRange(String key, double from, double to) {
+    public DateRangeAggregationBuilder addRange(String key, double from, double to) {
         addRange(new Range(key, from, to));
         return this;
     }
@@ -137,7 +137,7 @@ public class DateRangeAggregatorBuilder extends AbstractRangeBuilder<DateRangeAg
      * Same as {@link #addRange(String, double, double)} but the key will be
      * automatically generated based on <code>from</code> and <code>to</code>.
      */
-    public DateRangeAggregatorBuilder addRange(double from, double to) {
+    public DateRangeAggregationBuilder addRange(double from, double to) {
         return addRange(null, from, to);
     }
 
@@ -149,7 +149,7 @@ public class DateRangeAggregatorBuilder extends AbstractRangeBuilder<DateRangeAg
      * @param to
      *            the upper bound on the dates, exclusive
      */
-    public DateRangeAggregatorBuilder addUnboundedTo(String key, double to) {
+    public DateRangeAggregationBuilder addUnboundedTo(String key, double to) {
         addRange(new Range(key, null, to));
         return this;
     }
@@ -158,7 +158,7 @@ public class DateRangeAggregatorBuilder extends AbstractRangeBuilder<DateRangeAg
      * Same as {@link #addUnboundedTo(String, double)} but the key will be
      * computed automatically.
      */
-    public DateRangeAggregatorBuilder addUnboundedTo(double to) {
+    public DateRangeAggregationBuilder addUnboundedTo(double to) {
         return addUnboundedTo(null, to);
     }
 
@@ -170,7 +170,7 @@ public class DateRangeAggregatorBuilder extends AbstractRangeBuilder<DateRangeAg
      * @param from
      *            the lower bound on the distances, inclusive
      */
-    public DateRangeAggregatorBuilder addUnboundedFrom(String key, double from) {
+    public DateRangeAggregationBuilder addUnboundedFrom(String key, double from) {
         addRange(new Range(key, from, null));
         return this;
     }
@@ -179,7 +179,7 @@ public class DateRangeAggregatorBuilder extends AbstractRangeBuilder<DateRangeAg
      * Same as {@link #addUnboundedFrom(String, double)} but the key will be
      * computed automatically.
      */
-    public DateRangeAggregatorBuilder addUnboundedFrom(double from) {
+    public DateRangeAggregationBuilder addUnboundedFrom(double from) {
         return addUnboundedFrom(null, from);
     }
 
@@ -193,7 +193,7 @@ public class DateRangeAggregatorBuilder extends AbstractRangeBuilder<DateRangeAg
      * @param to
      *            the upper bound on the dates, exclusive
      */
-    public DateRangeAggregatorBuilder addRange(String key, DateTime from, DateTime to) {
+    public DateRangeAggregationBuilder addRange(String key, DateTime from, DateTime to) {
         addRange(new Range(key, convertDateTime(from), convertDateTime(to)));
         return this;
     }
@@ -210,7 +210,7 @@ public class DateRangeAggregatorBuilder extends AbstractRangeBuilder<DateRangeAg
      * Same as {@link #addRange(String, DateTime, DateTime)} but the key will be
      * automatically generated based on <code>from</code> and <code>to</code>.
      */
-    public DateRangeAggregatorBuilder addRange(DateTime from, DateTime to) {
+    public DateRangeAggregationBuilder addRange(DateTime from, DateTime to) {
         return addRange(null, from, to);
     }
 
@@ -222,7 +222,7 @@ public class DateRangeAggregatorBuilder extends AbstractRangeBuilder<DateRangeAg
      * @param to
      *            the upper bound on the dates, exclusive
      */
-    public DateRangeAggregatorBuilder addUnboundedTo(String key, DateTime to) {
+    public DateRangeAggregationBuilder addUnboundedTo(String key, DateTime to) {
         addRange(new Range(key, null, convertDateTime(to)));
         return this;
     }
@@ -231,7 +231,7 @@ public class DateRangeAggregatorBuilder extends AbstractRangeBuilder<DateRangeAg
      * Same as {@link #addUnboundedTo(String, DateTime)} but the key will be
      * computed automatically.
      */
-    public DateRangeAggregatorBuilder addUnboundedTo(DateTime to) {
+    public DateRangeAggregationBuilder addUnboundedTo(DateTime to) {
         return addUnboundedTo(null, to);
     }
 
@@ -243,7 +243,7 @@ public class DateRangeAggregatorBuilder extends AbstractRangeBuilder<DateRangeAg
      * @param from
      *            the lower bound on the distances, inclusive
      */
-    public DateRangeAggregatorBuilder addUnboundedFrom(String key, DateTime from) {
+    public DateRangeAggregationBuilder addUnboundedFrom(String key, DateTime from) {
         addRange(new Range(key, convertDateTime(from), null));
         return this;
     }
@@ -252,7 +252,7 @@ public class DateRangeAggregatorBuilder extends AbstractRangeBuilder<DateRangeAg
      * Same as {@link #addUnboundedFrom(String, DateTime)} but the key will be
      * computed automatically.
      */
-    public DateRangeAggregatorBuilder addUnboundedFrom(DateTime from) {
+    public DateRangeAggregationBuilder addUnboundedFrom(DateTime from) {
         return addUnboundedFrom(null, from);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeParser.java
index cc6dca1e543..f8eff715abb 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/DateRangeParser.java
@@ -38,9 +38,9 @@ public class DateRangeParser extends RangeParser {
     }
 
     @Override
-    protected DateRangeAggregatorBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        DateRangeAggregatorBuilder factory = new DateRangeAggregatorBuilder(aggregationName);
+    protected DateRangeAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
+                                                        ValueType targetValueType, Map<ParseField, Object> otherOptions) {
+        DateRangeAggregationBuilder factory = new DateRangeAggregationBuilder(aggregationName);
         @SuppressWarnings("unchecked")
         List<Range> ranges = (List<Range>) otherOptions.get(RangeAggregator.RANGES_FIELD);
         for (Range range : ranges) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceAggregationBuilder.java
similarity index 85%
rename from core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceAggregationBuilder.java
index a72b4fd322e..e82a769431a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceAggregationBuilder.java
@@ -33,7 +33,7 @@ import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator;
 import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceParser.Range;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 
@@ -42,7 +42,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 
-public class GeoDistanceAggregatorBuilder extends ValuesSourceAggregatorBuilder<ValuesSource.GeoPoint, GeoDistanceAggregatorBuilder> {
+public class GeoDistanceAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource.GeoPoint, GeoDistanceAggregationBuilder> {
     public static final String NAME = InternalGeoDistance.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
@@ -52,12 +52,12 @@ public class GeoDistanceAggregatorBuilder extends ValuesSourceAggregatorBuilder<
     private GeoDistance distanceType = GeoDistance.DEFAULT;
     private boolean keyed = false;
 
-    public GeoDistanceAggregatorBuilder(String name, GeoPoint origin) {
+    public GeoDistanceAggregationBuilder(String name, GeoPoint origin) {
         this(name, origin, InternalGeoDistance.FACTORY);
     }
 
-    private GeoDistanceAggregatorBuilder(String name, GeoPoint origin,
-            InternalRange.Factory<InternalGeoDistance.Bucket, InternalGeoDistance> rangeFactory) {
+    private GeoDistanceAggregationBuilder(String name, GeoPoint origin,
+                                          InternalRange.Factory<InternalGeoDistance.Bucket, InternalGeoDistance> rangeFactory) {
         super(name, rangeFactory.type(), rangeFactory.getValueSourceType(), rangeFactory.getValueType());
         if (origin == null) {
             throw new IllegalArgumentException("[origin] must not be null: [" + name + "]");
@@ -68,7 +68,7 @@ public class GeoDistanceAggregatorBuilder extends ValuesSourceAggregatorBuilder<
     /**
      * Read from a stream.
      */
-    public GeoDistanceAggregatorBuilder(StreamInput in) throws IOException {
+    public GeoDistanceAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalGeoDistance.FACTORY.type(), InternalGeoDistance.FACTORY.getValueSourceType(),
                 InternalGeoDistance.FACTORY.getValueType());
         origin = new GeoPoint(in.readDouble(), in.readDouble());
@@ -95,7 +95,7 @@ public class GeoDistanceAggregatorBuilder extends ValuesSourceAggregatorBuilder<
         unit.writeTo(out);
     }
 
-    public GeoDistanceAggregatorBuilder addRange(Range range) {
+    public GeoDistanceAggregationBuilder addRange(Range range) {
         if (range == null) {
             throw new IllegalArgumentException("[range] must not be null: [" + name + "]");
         }
@@ -113,7 +113,7 @@ public class GeoDistanceAggregatorBuilder extends ValuesSourceAggregatorBuilder<
      * @param to
      *            the upper bound on the distances, exclusive
      */
-    public GeoDistanceAggregatorBuilder addRange(String key, double from, double to) {
+    public GeoDistanceAggregationBuilder addRange(String key, double from, double to) {
         ranges.add(new Range(key, from, to));
         return this;
     }
@@ -123,7 +123,7 @@ public class GeoDistanceAggregatorBuilder extends ValuesSourceAggregatorBuilder<
      * automatically generated based on <code>from</code> and
      * <code>to</code>.
      */
-    public GeoDistanceAggregatorBuilder addRange(double from, double to) {
+    public GeoDistanceAggregationBuilder addRange(double from, double to) {
         return addRange(null, from, to);
     }
 
@@ -135,7 +135,7 @@ public class GeoDistanceAggregatorBuilder extends ValuesSourceAggregatorBuilder<
      * @param to
      *            the upper bound on the distances, exclusive
      */
-    public GeoDistanceAggregatorBuilder addUnboundedTo(String key, double to) {
+    public GeoDistanceAggregationBuilder addUnboundedTo(String key, double to) {
         ranges.add(new Range(key, null, to));
         return this;
     }
@@ -144,7 +144,7 @@ public class GeoDistanceAggregatorBuilder extends ValuesSourceAggregatorBuilder<
      * Same as {@link #addUnboundedTo(String, double)} but the key will be
      * computed automatically.
      */
-    public GeoDistanceAggregatorBuilder addUnboundedTo(double to) {
+    public GeoDistanceAggregationBuilder addUnboundedTo(double to) {
         return addUnboundedTo(null, to);
     }
 
@@ -156,7 +156,7 @@ public class GeoDistanceAggregatorBuilder extends ValuesSourceAggregatorBuilder<
      * @param from
      *            the lower bound on the distances, inclusive
      */
-    public GeoDistanceAggregatorBuilder addUnboundedFrom(String key, double from) {
+    public GeoDistanceAggregationBuilder addUnboundedFrom(String key, double from) {
         addRange(new Range(key, from, null));
         return this;
     }
@@ -165,7 +165,7 @@ public class GeoDistanceAggregatorBuilder extends ValuesSourceAggregatorBuilder<
      * Same as {@link #addUnboundedFrom(String, double)} but the key will be
      * computed automatically.
      */
-    public GeoDistanceAggregatorBuilder addUnboundedFrom(double from) {
+    public GeoDistanceAggregationBuilder addUnboundedFrom(double from) {
         return addUnboundedFrom(null, from);
     }
 
@@ -178,7 +178,7 @@ public class GeoDistanceAggregatorBuilder extends ValuesSourceAggregatorBuilder<
         return NAME;
     }
 
-    public GeoDistanceAggregatorBuilder unit(DistanceUnit unit) {
+    public GeoDistanceAggregationBuilder unit(DistanceUnit unit) {
         if (unit == null) {
             throw new IllegalArgumentException("[unit] must not be null: [" + name + "]");
         }
@@ -190,7 +190,7 @@ public class GeoDistanceAggregatorBuilder extends ValuesSourceAggregatorBuilder<
         return unit;
     }
 
-    public GeoDistanceAggregatorBuilder distanceType(GeoDistance distanceType) {
+    public GeoDistanceAggregationBuilder distanceType(GeoDistance distanceType) {
         if (distanceType == null) {
             throw new IllegalArgumentException("[distanceType] must not be null: [" + name + "]");
         }
@@ -202,7 +202,7 @@ public class GeoDistanceAggregatorBuilder extends ValuesSourceAggregatorBuilder<
         return distanceType;
     }
 
-    public GeoDistanceAggregatorBuilder keyed(boolean keyed) {
+    public GeoDistanceAggregationBuilder keyed(boolean keyed) {
         this.keyed = keyed;
         return this;
     }
@@ -236,7 +236,7 @@ public class GeoDistanceAggregatorBuilder extends ValuesSourceAggregatorBuilder<
 
     @Override
     protected boolean innerEquals(Object obj) {
-        GeoDistanceAggregatorBuilder other = (GeoDistanceAggregatorBuilder) obj;
+        GeoDistanceAggregationBuilder other = (GeoDistanceAggregationBuilder) obj;
         return Objects.equals(origin, other.origin)
                 && Objects.equals(ranges, other.ranges)
                 && Objects.equals(keyed, other.keyed)
@@ -244,4 +244,4 @@ public class GeoDistanceAggregatorBuilder extends ValuesSourceAggregatorBuilder<
                 && Objects.equals(unit, other.unit);
     }
 
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceParser.java
index ed6d6a67e2a..b98757aae5d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/GeoDistanceParser.java
@@ -85,10 +85,10 @@ public class GeoDistanceParser extends GeoPointValuesSourceParser {
     }
 
     @Override
-    protected GeoDistanceAggregatorBuilder createFactory(
+    protected GeoDistanceAggregationBuilder createFactory(
             String aggregationName, ValuesSourceType valuesSourceType, ValueType targetValueType, Map<ParseField, Object> otherOptions) {
         GeoPoint origin = (GeoPoint) otherOptions.get(ORIGIN_FIELD);
-        GeoDistanceAggregatorBuilder factory = new GeoDistanceAggregatorBuilder(aggregationName, origin);
+        GeoDistanceAggregationBuilder factory = new GeoDistanceAggregationBuilder(aggregationName, origin);
         @SuppressWarnings("unchecked")
         List<Range> ranges = (List<Range>) otherOptions.get(RangeAggregator.RANGES_FIELD);
         for (Range range : ranges) {
@@ -171,4 +171,4 @@ public class GeoDistanceParser extends GeoPointValuesSourceParser {
         }
         return false;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeAggregationBuilder.java
similarity index 90%
rename from core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeAggregationBuilder.java
index c56a2952f8d..243db5f75e3 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeAggregationBuilder.java
@@ -44,14 +44,14 @@ import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 
-public final class IpRangeAggregatorBuilder
-        extends ValuesSourceAggregatorBuilder<ValuesSource.Bytes, IpRangeAggregatorBuilder> {
+public final class IpRangeAggregationBuilder
+        extends ValuesSourceAggregationBuilder<ValuesSource.Bytes, IpRangeAggregationBuilder> {
     private static final String NAME = "ip_range";
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
     private static final InternalAggregation.Type TYPE = new InternalAggregation.Type(NAME);
@@ -163,7 +163,7 @@ public final class IpRangeAggregatorBuilder
     private boolean keyed = false;
     private List<Range> ranges = new ArrayList<>();
 
-    public IpRangeAggregatorBuilder(String name) {
+    public IpRangeAggregationBuilder(String name) {
         super(name, TYPE, ValuesSourceType.BYTES, ValueType.IP);
     }
 
@@ -172,7 +172,7 @@ public final class IpRangeAggregatorBuilder
         return NAME;
     }
 
-    public IpRangeAggregatorBuilder keyed(boolean keyed) {
+    public IpRangeAggregationBuilder keyed(boolean keyed) {
         this.keyed = keyed;
         return this;
     }
@@ -187,7 +187,7 @@ public final class IpRangeAggregatorBuilder
     }
 
     /** Add a new {@link Range} to this aggregation. */
-    public IpRangeAggregatorBuilder addRange(Range range) {
+    public IpRangeAggregationBuilder addRange(Range range) {
         ranges.add(range);
         return this;
     }
@@ -202,7 +202,7 @@ public final class IpRangeAggregatorBuilder
      * @param to
      *            the upper bound on the distances, exclusive
      */
-    public IpRangeAggregatorBuilder addRange(String key, String from, String to) {
+    public IpRangeAggregationBuilder addRange(String key, String from, String to) {
         addRange(new Range(key, from, to));
         return this;
     }
@@ -210,7 +210,7 @@ public final class IpRangeAggregatorBuilder
     /**
      * Add a new range to this aggregation using the CIDR notation.
      */
-    public IpRangeAggregatorBuilder addMaskRange(String key, String mask) {
+    public IpRangeAggregationBuilder addMaskRange(String key, String mask) {
         return addRange(new Range(key, mask));
     }
 
@@ -218,7 +218,7 @@ public final class IpRangeAggregatorBuilder
      * Same as {@link #addMaskRange(String, String)} but uses the mask itself as
      * a key.
      */
-    public IpRangeAggregatorBuilder addMaskRange(String mask) {
+    public IpRangeAggregationBuilder addMaskRange(String mask) {
         return addRange(new Range(mask, mask));
     }
 
@@ -226,7 +226,7 @@ public final class IpRangeAggregatorBuilder
      * Same as {@link #addRange(String, String, String)} but the key will be
      * automatically generated.
      */
-    public IpRangeAggregatorBuilder addRange(String from, String to) {
+    public IpRangeAggregationBuilder addRange(String from, String to) {
         return addRange(null, from, to);
     }
 
@@ -234,7 +234,7 @@ public final class IpRangeAggregatorBuilder
      * Same as {@link #addRange(String, String, String)} but there will be no
      * lower bound.
      */
-    public IpRangeAggregatorBuilder addUnboundedTo(String key, String to) {
+    public IpRangeAggregationBuilder addUnboundedTo(String key, String to) {
         addRange(new Range(key, null, to));
         return this;
     }
@@ -243,7 +243,7 @@ public final class IpRangeAggregatorBuilder
      * Same as {@link #addUnboundedTo(String, String)} but the key will be
      * generated automatically.
      */
-    public IpRangeAggregatorBuilder addUnboundedTo(String to) {
+    public IpRangeAggregationBuilder addUnboundedTo(String to) {
         return addUnboundedTo(null, to);
     }
 
@@ -251,13 +251,13 @@ public final class IpRangeAggregatorBuilder
      * Same as {@link #addRange(String, String, String)} but there will be no
      * upper bound.
      */
-    public IpRangeAggregatorBuilder addUnboundedFrom(String key, String from) {
+    public IpRangeAggregationBuilder addUnboundedFrom(String key, String from) {
         addRange(new Range(key, from, null));
         return this;
     }
 
     @Override
-    public IpRangeAggregatorBuilder script(Script script) {
+    public IpRangeAggregationBuilder script(Script script) {
         throw new IllegalArgumentException("[ip_range] does not support scripts");
     }
 
@@ -265,11 +265,11 @@ public final class IpRangeAggregatorBuilder
      * Same as {@link #addUnboundedFrom(String, String)} but the key will be
      * generated automatically.
      */
-    public IpRangeAggregatorBuilder addUnboundedFrom(String from) {
+    public IpRangeAggregationBuilder addUnboundedFrom(String from) {
         return addUnboundedFrom(null, from);
     }
 
-    public IpRangeAggregatorBuilder(StreamInput in) throws IOException {
+    public IpRangeAggregationBuilder(StreamInput in) throws IOException {
         super(in, TYPE, ValuesSourceType.BYTES, ValueType.IP);
         final int numRanges = in.readVInt();
         for (int i = 0; i < numRanges; ++i) {
@@ -323,7 +323,7 @@ public final class IpRangeAggregatorBuilder
 
     @Override
     protected boolean innerEquals(Object obj) {
-        IpRangeAggregatorBuilder that = (IpRangeAggregatorBuilder) obj;
+        IpRangeAggregationBuilder that = (IpRangeAggregationBuilder) obj;
         return keyed == that.keyed
                 && ranges.equals(that.ranges);
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeParser.java
index 64ed77d42f3..8445fb2d459 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ip/IpRangeParser.java
@@ -30,10 +30,10 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.BytesValuesSourceParser;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator;
-import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeAggregatorBuilder.Range;
+import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeAggregationBuilder.Range;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 /**
@@ -48,10 +48,10 @@ public class IpRangeParser extends BytesValuesSourceParser {
     }
 
     @Override
-    protected ValuesSourceAggregatorBuilder<ValuesSource.Bytes, ?> createFactory(
+    protected ValuesSourceAggregationBuilder<ValuesSource.Bytes, ?> createFactory(
             String aggregationName, ValuesSourceType valuesSourceType,
             ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        IpRangeAggregatorBuilder range = new IpRangeAggregatorBuilder(aggregationName);
+        IpRangeAggregationBuilder range = new IpRangeAggregationBuilder(aggregationName);
         @SuppressWarnings("unchecked")
         Iterable<Range> ranges = (Iterable<Range>) otherOptions.get(RangeAggregator.RANGES_FIELD);
         if (otherOptions.containsKey(RangeAggregator.RANGES_FIELD)) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java
similarity index 88%
rename from core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java
index f0c923e6efc..804574eea10 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedAggregationBuilder.java
@@ -28,7 +28,7 @@ import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
@@ -36,25 +36,25 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import java.io.IOException;
 import java.util.Objects;
 
-public class DiversifiedAggregatorBuilder extends ValuesSourceAggregatorBuilder<ValuesSource, DiversifiedAggregatorBuilder> {
+public class DiversifiedAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource, DiversifiedAggregationBuilder> {
     public static final String NAME = "diversified_sampler";
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
     public static final Type TYPE = new Type(NAME);
 
     public static final int MAX_DOCS_PER_VALUE_DEFAULT = 1;
 
-    private int shardSize = SamplerAggregatorBuilder.DEFAULT_SHARD_SAMPLE_SIZE;
+    private int shardSize = SamplerAggregationBuilder.DEFAULT_SHARD_SAMPLE_SIZE;
     private int maxDocsPerValue = MAX_DOCS_PER_VALUE_DEFAULT;
     private String executionHint = null;
 
-    public DiversifiedAggregatorBuilder(String name) {
+    public DiversifiedAggregationBuilder(String name) {
         super(name, TYPE, ValuesSourceType.ANY, null);
     }
 
     /**
      * Read from a stream.
      */
-    public DiversifiedAggregatorBuilder(StreamInput in) throws IOException {
+    public DiversifiedAggregationBuilder(StreamInput in) throws IOException {
         super(in, TYPE, ValuesSourceType.ANY, null);
         shardSize = in.readVInt();
         maxDocsPerValue = in.readVInt();
@@ -71,7 +71,7 @@ public class DiversifiedAggregatorBuilder extends ValuesSourceAggregatorBuilder<
     /**
      * Set the max num docs to be returned from each shard.
      */
-    public DiversifiedAggregatorBuilder shardSize(int shardSize) {
+    public DiversifiedAggregationBuilder shardSize(int shardSize) {
         if (shardSize < 0) {
             throw new IllegalArgumentException(
                     "[shardSize] must be greater than or equal to 0. Found [" + shardSize + "] in [" + name + "]");
@@ -90,7 +90,7 @@ public class DiversifiedAggregatorBuilder extends ValuesSourceAggregatorBuilder<
     /**
      * Set the max num docs to be returned per value.
      */
-    public DiversifiedAggregatorBuilder maxDocsPerValue(int maxDocsPerValue) {
+    public DiversifiedAggregationBuilder maxDocsPerValue(int maxDocsPerValue) {
         if (maxDocsPerValue < 0) {
             throw new IllegalArgumentException(
                     "[maxDocsPerValue] must be greater than or equal to 0. Found [" + maxDocsPerValue + "] in [" + name + "]");
@@ -109,7 +109,7 @@ public class DiversifiedAggregatorBuilder extends ValuesSourceAggregatorBuilder<
     /**
      * Set the execution hint.
      */
-    public DiversifiedAggregatorBuilder executionHint(String executionHint) {
+    public DiversifiedAggregationBuilder executionHint(String executionHint) {
         this.executionHint = executionHint;
         return this;
     }
@@ -145,7 +145,7 @@ public class DiversifiedAggregatorBuilder extends ValuesSourceAggregatorBuilder<
 
     @Override
     protected boolean innerEquals(Object obj) {
-        DiversifiedAggregatorBuilder other = (DiversifiedAggregatorBuilder) obj;
+        DiversifiedAggregationBuilder other = (DiversifiedAggregationBuilder) obj;
         return Objects.equals(shardSize, other.shardSize)
                 && Objects.equals(maxDocsPerValue, other.maxDocsPerValue)
                 && Objects.equals(executionHint, other.executionHint);
@@ -155,4 +155,4 @@ public class DiversifiedAggregatorBuilder extends ValuesSourceAggregatorBuilder<
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedSamplerParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedSamplerParser.java
index cb87e53f2c0..f495071f6d2 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedSamplerParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/DiversifiedSamplerParser.java
@@ -38,9 +38,9 @@ public class DiversifiedSamplerParser extends AnyValuesSourceParser {
     }
 
     @Override
-    protected DiversifiedAggregatorBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        DiversifiedAggregatorBuilder factory = new DiversifiedAggregatorBuilder(aggregationName);
+    protected DiversifiedAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
+                                                          ValueType targetValueType, Map<ParseField, Object> otherOptions) {
+        DiversifiedAggregationBuilder factory = new DiversifiedAggregationBuilder(aggregationName);
         Integer shardSize = (Integer) otherOptions.get(SamplerAggregator.SHARD_SIZE_FIELD);
         if (shardSize != null) {
             factory.shardSize(shardSize);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java
similarity index 87%
rename from core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java
index 2cc3bb4c303..1220a2ddd42 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java
@@ -26,7 +26,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregatorBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
@@ -34,7 +34,7 @@ import org.elasticsearch.search.aggregations.support.AggregationContext;
 import java.io.IOException;
 import java.util.Objects;
 
-public class SamplerAggregatorBuilder extends AggregatorBuilder<SamplerAggregatorBuilder> {
+public class SamplerAggregationBuilder extends AggregationBuilder<SamplerAggregationBuilder> {
     public static final String NAME = InternalSampler.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
@@ -42,14 +42,14 @@ public class SamplerAggregatorBuilder extends AggregatorBuilder<SamplerAggregato
 
     private int shardSize = DEFAULT_SHARD_SAMPLE_SIZE;
 
-    public SamplerAggregatorBuilder(String name) {
+    public SamplerAggregationBuilder(String name) {
         super(name, InternalSampler.TYPE);
     }
 
     /**
      * Read from a stream.
      */
-    public SamplerAggregatorBuilder(StreamInput in) throws IOException {
+    public SamplerAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalSampler.TYPE);
         shardSize = in.readVInt();
     }
@@ -62,7 +62,7 @@ public class SamplerAggregatorBuilder extends AggregatorBuilder<SamplerAggregato
     /**
      * Set the max num docs to be returned from each shard.
      */
-    public SamplerAggregatorBuilder shardSize(int shardSize) {
+    public SamplerAggregationBuilder shardSize(int shardSize) {
         this.shardSize = shardSize;
         return this;
     }
@@ -88,7 +88,7 @@ public class SamplerAggregatorBuilder extends AggregatorBuilder<SamplerAggregato
         return builder;
     }
 
-    public static SamplerAggregatorBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+    public static SamplerAggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
         XContentParser.Token token;
         String currentFieldName = null;
         Integer shardSize = null;
@@ -110,7 +110,7 @@ public class SamplerAggregatorBuilder extends AggregatorBuilder<SamplerAggregato
             }
         }
 
-        SamplerAggregatorBuilder factory = new SamplerAggregatorBuilder(aggregationName);
+        SamplerAggregationBuilder factory = new SamplerAggregationBuilder(aggregationName);
         if (shardSize != null) {
             factory.shardSize(shardSize);
         }
@@ -124,7 +124,7 @@ public class SamplerAggregatorBuilder extends AggregatorBuilder<SamplerAggregato
 
     @Override
     protected boolean doEquals(Object obj) {
-        SamplerAggregatorBuilder other = (SamplerAggregatorBuilder) obj;
+        SamplerAggregationBuilder other = (SamplerAggregationBuilder) obj;
         return Objects.equals(shardSize, other.shardSize);
     }
 
@@ -132,4 +132,4 @@ public class SamplerAggregatorBuilder extends AggregatorBuilder<SamplerAggregato
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java
similarity index 87%
rename from core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java
index 55b85bef0d9..64a9c607729 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java
@@ -29,12 +29,12 @@ import org.elasticsearch.search.aggregations.bucket.significant.heuristics.JLHSc
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds;
-import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
@@ -45,7 +45,7 @@ import java.util.Objects;
 /**
  *
  */
-public class SignificantTermsAggregatorBuilder extends ValuesSourceAggregatorBuilder<ValuesSource, SignificantTermsAggregatorBuilder> {
+public class SignificantTermsAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource, SignificantTermsAggregationBuilder> {
     public static final String NAME = SignificantStringTerms.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
@@ -62,14 +62,14 @@ public class SignificantTermsAggregatorBuilder extends ValuesSourceAggregatorBui
     private TermsAggregator.BucketCountThresholds bucketCountThresholds = new BucketCountThresholds(DEFAULT_BUCKET_COUNT_THRESHOLDS);
     private SignificanceHeuristic significanceHeuristic = DEFAULT_SIGNIFICANCE_HEURISTIC;
 
-    public SignificantTermsAggregatorBuilder(String name, ValueType valueType) {
+    public SignificantTermsAggregationBuilder(String name, ValueType valueType) {
         super(name, SignificantStringTerms.TYPE, ValuesSourceType.ANY, valueType);
     }
 
     /**
      * Read from a Stream.
      */
-    public SignificantTermsAggregatorBuilder(StreamInput in) throws IOException {
+    public SignificantTermsAggregationBuilder(StreamInput in) throws IOException {
         super(in, SignificantStringTerms.TYPE, ValuesSourceType.ANY);
         bucketCountThresholds = new BucketCountThresholds(in);
         executionHint = in.readOptionalString();
@@ -100,7 +100,7 @@ public class SignificantTermsAggregatorBuilder extends ValuesSourceAggregatorBui
         return bucketCountThresholds;
     }
 
-    public SignificantTermsAggregatorBuilder bucketCountThresholds(TermsAggregator.BucketCountThresholds bucketCountThresholds) {
+    public SignificantTermsAggregationBuilder bucketCountThresholds(TermsAggregator.BucketCountThresholds bucketCountThresholds) {
         if (bucketCountThresholds == null) {
             throw new IllegalArgumentException("[bucketCountThresholds] must not be null: [" + name + "]");
         }
@@ -112,7 +112,7 @@ public class SignificantTermsAggregatorBuilder extends ValuesSourceAggregatorBui
      * Sets the size - indicating how many term buckets should be returned
      * (defaults to 10)
      */
-    public SignificantTermsAggregatorBuilder size(int size) {
+    public SignificantTermsAggregationBuilder size(int size) {
         if (size < 0) {
             throw new IllegalArgumentException("[size] must be greater than or equal to 0. Found [" + size + "] in [" + name + "]");
         }
@@ -126,7 +126,7 @@ public class SignificantTermsAggregatorBuilder extends ValuesSourceAggregatorBui
      * search execution). The higher the shard size is, the more accurate the
      * results are.
      */
-    public SignificantTermsAggregatorBuilder shardSize(int shardSize) {
+    public SignificantTermsAggregationBuilder shardSize(int shardSize) {
         if (shardSize < 0) {
             throw new IllegalArgumentException(
                     "[shardSize] must be greater than or equal to 0. Found [" + shardSize + "] in [" + name + "]");
@@ -139,7 +139,7 @@ public class SignificantTermsAggregatorBuilder extends ValuesSourceAggregatorBui
      * Set the minimum document count terms should have in order to appear in
      * the response.
      */
-    public SignificantTermsAggregatorBuilder minDocCount(long minDocCount) {
+    public SignificantTermsAggregationBuilder minDocCount(long minDocCount) {
         if (minDocCount < 0) {
             throw new IllegalArgumentException(
                     "[minDocCount] must be greater than or equal to 0. Found [" + minDocCount + "] in [" + name + "]");
@@ -152,7 +152,7 @@ public class SignificantTermsAggregatorBuilder extends ValuesSourceAggregatorBui
      * Set the minimum document count terms should have on the shard in order to
      * appear in the response.
      */
-    public SignificantTermsAggregatorBuilder shardMinDocCount(long shardMinDocCount) {
+    public SignificantTermsAggregationBuilder shardMinDocCount(long shardMinDocCount) {
         if (shardMinDocCount < 0) {
             throw new IllegalArgumentException(
                     "[shardMinDocCount] must be greater than or equal to 0. Found [" + shardMinDocCount + "] in [" + name + "]");
@@ -164,7 +164,7 @@ public class SignificantTermsAggregatorBuilder extends ValuesSourceAggregatorBui
     /**
      * Expert: sets an execution hint to the aggregation.
      */
-    public SignificantTermsAggregatorBuilder executionHint(String executionHint) {
+    public SignificantTermsAggregationBuilder executionHint(String executionHint) {
         this.executionHint = executionHint;
         return this;
     }
@@ -176,7 +176,7 @@ public class SignificantTermsAggregatorBuilder extends ValuesSourceAggregatorBui
         return executionHint;
     }
 
-    public SignificantTermsAggregatorBuilder backgroundFilter(QueryBuilder backgroundFilter) {
+    public SignificantTermsAggregationBuilder backgroundFilter(QueryBuilder backgroundFilter) {
         if (backgroundFilter == null) {
             throw new IllegalArgumentException("[backgroundFilter] must not be null: [" + name + "]");
         }
@@ -191,7 +191,7 @@ public class SignificantTermsAggregatorBuilder extends ValuesSourceAggregatorBui
     /**
      * Set terms to include and exclude from the aggregation results
      */
-    public SignificantTermsAggregatorBuilder includeExclude(IncludeExclude includeExclude) {
+    public SignificantTermsAggregationBuilder includeExclude(IncludeExclude includeExclude) {
         this.includeExclude = includeExclude;
         return this;
     }
@@ -203,7 +203,7 @@ public class SignificantTermsAggregatorBuilder extends ValuesSourceAggregatorBui
         return includeExclude;
     }
 
-    public SignificantTermsAggregatorBuilder significanceHeuristic(SignificanceHeuristic significanceHeuristic) {
+    public SignificantTermsAggregationBuilder significanceHeuristic(SignificanceHeuristic significanceHeuristic) {
         if (significanceHeuristic == null) {
             throw new IllegalArgumentException("[significanceHeuristic] must not be null: [" + name + "]");
         }
@@ -226,7 +226,7 @@ public class SignificantTermsAggregatorBuilder extends ValuesSourceAggregatorBui
     protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         bucketCountThresholds.toXContent(builder, params);
         if (executionHint != null) {
-            builder.field(TermsAggregatorBuilder.EXECUTION_HINT_FIELD_NAME.getPreferredName(), executionHint);
+            builder.field(TermsAggregationBuilder.EXECUTION_HINT_FIELD_NAME.getPreferredName(), executionHint);
         }
         if (filterBuilder != null) {
             builder.field(BACKGROUND_FILTER.getPreferredName(), filterBuilder);
@@ -245,7 +245,7 @@ public class SignificantTermsAggregatorBuilder extends ValuesSourceAggregatorBui
 
     @Override
     protected boolean innerEquals(Object obj) {
-        SignificantTermsAggregatorBuilder other = (SignificantTermsAggregatorBuilder) obj;
+        SignificantTermsAggregationBuilder other = (SignificantTermsAggregationBuilder) obj;
         return Objects.equals(bucketCountThresholds, other.bucketCountThresholds)
                 && Objects.equals(executionHint, other.executionHint)
                 && Objects.equals(filterBuilder, other.filterBuilder)
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
index 4b9e3acb873..ab30e1b2d4a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java
@@ -178,7 +178,7 @@ public class SignificantTermsAggregatorFactory extends ValuesSourceAggregatorFac
 
         numberOfAggregatorsCreated++;
         BucketCountThresholds bucketCountThresholds = new BucketCountThresholds(this.bucketCountThresholds);
-        if (bucketCountThresholds.getShardSize() == SignificantTermsAggregatorBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS.getShardSize()) {
+        if (bucketCountThresholds.getShardSize() == SignificantTermsAggregationBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS.getShardSize()) {
             // The user has not made a shardSize selection .
             // Use default heuristic to avoid any wrong-ranking caused by
             // distributed counting
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParser.java
index 60805bea692..33db8f97335 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParser.java
@@ -53,10 +53,11 @@ public class SignificantTermsParser extends AbstractTermsParser {
     }
 
     @Override
-    protected SignificantTermsAggregatorBuilder doCreateFactory(String aggregationName, ValuesSourceType valuesSourceType,
-            ValueType targetValueType, BucketCountThresholds bucketCountThresholds, SubAggCollectionMode collectMode, String executionHint,
-            IncludeExclude incExc, Map<ParseField, Object> otherOptions) {
-        SignificantTermsAggregatorBuilder factory = new SignificantTermsAggregatorBuilder(aggregationName, targetValueType);
+    protected SignificantTermsAggregationBuilder doCreateFactory(String aggregationName, ValuesSourceType valuesSourceType,
+                                                                 ValueType targetValueType, BucketCountThresholds bucketCountThresholds,
+                                                                 SubAggCollectionMode collectMode, String executionHint,
+                                                                 IncludeExclude incExc, Map<ParseField, Object> otherOptions) {
+        SignificantTermsAggregationBuilder factory = new SignificantTermsAggregationBuilder(aggregationName, targetValueType);
         if (bucketCountThresholds != null) {
             factory.bucketCountThresholds(bucketCountThresholds);
         }
@@ -66,11 +67,12 @@ public class SignificantTermsParser extends AbstractTermsParser {
         if (incExc != null) {
             factory.includeExclude(incExc);
         }
-        QueryBuilder backgroundFilter = (QueryBuilder) otherOptions.get(SignificantTermsAggregatorBuilder.BACKGROUND_FILTER);
+        QueryBuilder backgroundFilter = (QueryBuilder) otherOptions.get(SignificantTermsAggregationBuilder.BACKGROUND_FILTER);
         if (backgroundFilter != null) {
             factory.backgroundFilter(backgroundFilter);
         }
-        SignificanceHeuristic significanceHeuristic = (SignificanceHeuristic) otherOptions.get(SignificantTermsAggregatorBuilder.HEURISTIC);
+        SignificanceHeuristic significanceHeuristic =
+            (SignificanceHeuristic) otherOptions.get(SignificantTermsAggregationBuilder.HEURISTIC);
         if (significanceHeuristic != null) {
             factory.significanceHeuristic(significanceHeuristic);
         }
@@ -85,12 +87,12 @@ public class SignificantTermsParser extends AbstractTermsParser {
                     .lookupReturningNullIfNotFound(currentFieldName, parseFieldMatcher);
             if (significanceHeuristicParser != null) {
                 SignificanceHeuristic significanceHeuristic = significanceHeuristicParser.parse(parser, parseFieldMatcher);
-                otherOptions.put(SignificantTermsAggregatorBuilder.HEURISTIC, significanceHeuristic);
+                otherOptions.put(SignificantTermsAggregationBuilder.HEURISTIC, significanceHeuristic);
                 return true;
-            } else if (parseFieldMatcher.match(currentFieldName, SignificantTermsAggregatorBuilder.BACKGROUND_FILTER)) {
+            } else if (parseFieldMatcher.match(currentFieldName, SignificantTermsAggregationBuilder.BACKGROUND_FILTER)) {
                 QueryParseContext queryParseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
                 QueryBuilder filter = queryParseContext.parseInnerQueryBuilder();
-                otherOptions.put(SignificantTermsAggregatorBuilder.BACKGROUND_FILTER, filter);
+                otherOptions.put(SignificantTermsAggregationBuilder.BACKGROUND_FILTER, filter);
                 return true;
             }
         }
@@ -99,6 +101,6 @@ public class SignificantTermsParser extends AbstractTermsParser {
 
     @Override
     protected BucketCountThresholds getDefaultBucketCountThresholds() {
-        return new TermsAggregator.BucketCountThresholds(SignificantTermsAggregatorBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS);
+        return new TermsAggregator.BucketCountThresholds(SignificantTermsAggregationBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/UnmappedSignificantTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/UnmappedSignificantTerms.java
index 5369e269058..b5781aa34be 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/UnmappedSignificantTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/UnmappedSignificantTerms.java
@@ -60,7 +60,7 @@ public class UnmappedSignificantTerms extends InternalSignificantTerms<UnmappedS
     public UnmappedSignificantTerms(String name, int requiredSize, long minDocCount, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
         //We pass zero for index/subset sizes because for the purpose of significant term analysis
         // we assume an unmapped index's size is irrelevant to the proceedings.
-        super(0, 0, name, DocValueFormat.RAW, requiredSize, minDocCount, SignificantTermsAggregatorBuilder.DEFAULT_SIGNIFICANCE_HEURISTIC,
+        super(0, 0, name, DocValueFormat.RAW, requiredSize, minDocCount, SignificantTermsAggregationBuilder.DEFAULT_SIGNIFICANCE_HEURISTIC,
                 BUCKETS, pipelineAggregators, metaData);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractTermsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractTermsParser.java
index a15c7d28427..3f27c4f1c6f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractTermsParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractTermsParser.java
@@ -29,7 +29,7 @@ import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude
 import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.AnyValuesSourceParser;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
@@ -50,8 +50,10 @@ public abstract class AbstractTermsParser extends AnyValuesSourceParser {
     }
 
     @Override
-    protected final ValuesSourceAggregatorBuilder<ValuesSource, ?> createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
+    protected final ValuesSourceAggregationBuilder<ValuesSource, ?> createFactory(String aggregationName,
+                                                                                  ValuesSourceType valuesSourceType,
+                                                                                  ValueType targetValueType,
+                                                                                  Map<ParseField, Object> otherOptions) {
         BucketCountThresholds bucketCountThresholds = getDefaultBucketCountThresholds();
         Integer requiredSize = (Integer) otherOptions.get(REQUIRED_SIZE_FIELD_NAME);
         if (requiredSize != null && requiredSize != -1) {
@@ -77,10 +79,14 @@ public abstract class AbstractTermsParser extends AnyValuesSourceParser {
                 otherOptions);
     }
 
-    protected abstract ValuesSourceAggregatorBuilder<ValuesSource, ?> doCreateFactory(String aggregationName,
-            ValuesSourceType valuesSourceType,
-            ValueType targetValueType, BucketCountThresholds bucketCountThresholds, SubAggCollectionMode collectMode, String executionHint,
-            IncludeExclude incExc, Map<ParseField, Object> otherOptions);
+    protected abstract ValuesSourceAggregationBuilder<ValuesSource, ?> doCreateFactory(String aggregationName,
+                                                                                       ValuesSourceType valuesSourceType,
+                                                                                       ValueType targetValueType,
+                                                                                       BucketCountThresholds bucketCountThresholds,
+                                                                                       SubAggCollectionMode collectMode,
+                                                                                       String executionHint,
+                                                                                       IncludeExclude incExc,
+                                                                                       Map<ParseField, Object> otherOptions);
 
     @Override
     protected boolean token(String aggregationName, String currentFieldName, Token token, XContentParser parser,
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
similarity index 89%
rename from core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
index 53887d8b20c..f4cb133c499 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
@@ -30,7 +30,7 @@ import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
@@ -38,7 +38,7 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 
-public class TermsAggregatorBuilder extends ValuesSourceAggregatorBuilder<ValuesSource, TermsAggregatorBuilder> {
+public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource, TermsAggregationBuilder> {
     public static final String NAME = StringTerms.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
@@ -61,14 +61,14 @@ public class TermsAggregatorBuilder extends ValuesSourceAggregatorBuilder<Values
             DEFAULT_BUCKET_COUNT_THRESHOLDS);
     private boolean showTermDocCountError = false;
 
-    public TermsAggregatorBuilder(String name, ValueType valueType) {
+    public TermsAggregationBuilder(String name, ValueType valueType) {
         super(name, StringTerms.TYPE, ValuesSourceType.ANY, valueType);
     }
 
     /**
      * Read from a stream.
      */
-    public TermsAggregatorBuilder(StreamInput in) throws IOException {
+    public TermsAggregationBuilder(StreamInput in) throws IOException {
         super(in, StringTerms.TYPE, ValuesSourceType.ANY);
         bucketCountThresholds = new BucketCountThresholds(in);
         collectMode = SubAggCollectionMode.readFromStream(in);
@@ -97,7 +97,7 @@ public class TermsAggregatorBuilder extends ValuesSourceAggregatorBuilder<Values
         return bucketCountThresholds;
     }
 
-    public TermsAggregatorBuilder bucketCountThresholds(TermsAggregator.BucketCountThresholds bucketCountThresholds) {
+    public TermsAggregationBuilder bucketCountThresholds(TermsAggregator.BucketCountThresholds bucketCountThresholds) {
         if (bucketCountThresholds == null) {
             throw new IllegalArgumentException("[bucketCountThresholds] must not be null: [" + name + "]");
         }
@@ -109,7 +109,7 @@ public class TermsAggregatorBuilder extends ValuesSourceAggregatorBuilder<Values
      * Sets the size - indicating how many term buckets should be returned
      * (defaults to 10)
      */
-    public TermsAggregatorBuilder size(int size) {
+    public TermsAggregationBuilder size(int size) {
         if (size < 0) {
             throw new IllegalArgumentException("[size] must be greater than or equal to 0. Found [" + size + "] in [" + name + "]");
         }
@@ -123,7 +123,7 @@ public class TermsAggregatorBuilder extends ValuesSourceAggregatorBuilder<Values
      * search execution). The higher the shard size is, the more accurate the
      * results are.
      */
-    public TermsAggregatorBuilder shardSize(int shardSize) {
+    public TermsAggregationBuilder shardSize(int shardSize) {
         if (shardSize < 0) {
             throw new IllegalArgumentException(
                     "[shardSize] must be greater than or equal to 0. Found [" + shardSize + "] in [" + name + "]");
@@ -136,7 +136,7 @@ public class TermsAggregatorBuilder extends ValuesSourceAggregatorBuilder<Values
      * Set the minimum document count terms should have in order to appear in
      * the response.
      */
-    public TermsAggregatorBuilder minDocCount(long minDocCount) {
+    public TermsAggregationBuilder minDocCount(long minDocCount) {
         if (minDocCount < 0) {
             throw new IllegalArgumentException(
                     "[minDocCount] must be greater than or equal to 0. Found [" + minDocCount + "] in [" + name + "]");
@@ -149,7 +149,7 @@ public class TermsAggregatorBuilder extends ValuesSourceAggregatorBuilder<Values
      * Set the minimum document count terms should have on the shard in order to
      * appear in the response.
      */
-    public TermsAggregatorBuilder shardMinDocCount(long shardMinDocCount) {
+    public TermsAggregationBuilder shardMinDocCount(long shardMinDocCount) {
         if (shardMinDocCount < 0) {
             throw new IllegalArgumentException(
                     "[shardMinDocCount] must be greater than or equal to 0. Found [" + shardMinDocCount + "] in [" + name + "]");
@@ -161,7 +161,7 @@ public class TermsAggregatorBuilder extends ValuesSourceAggregatorBuilder<Values
     /**
      * Sets the order in which the buckets will be returned.
      */
-    public TermsAggregatorBuilder order(Terms.Order order) {
+    public TermsAggregationBuilder order(Terms.Order order) {
         if (order == null) {
             throw new IllegalArgumentException("[order] must not be null: [" + name + "]");
         }
@@ -172,7 +172,7 @@ public class TermsAggregatorBuilder extends ValuesSourceAggregatorBuilder<Values
     /**
      * Sets the order in which the buckets will be returned.
      */
-    public TermsAggregatorBuilder order(List<Terms.Order> orders) {
+    public TermsAggregationBuilder order(List<Terms.Order> orders) {
         if (orders == null) {
             throw new IllegalArgumentException("[orders] must not be null: [" + name + "]");
         }
@@ -190,7 +190,7 @@ public class TermsAggregatorBuilder extends ValuesSourceAggregatorBuilder<Values
     /**
      * Expert: sets an execution hint to the aggregation.
      */
-    public TermsAggregatorBuilder executionHint(String executionHint) {
+    public TermsAggregationBuilder executionHint(String executionHint) {
         this.executionHint = executionHint;
         return this;
     }
@@ -205,7 +205,7 @@ public class TermsAggregatorBuilder extends ValuesSourceAggregatorBuilder<Values
     /**
      * Expert: set the collection mode.
      */
-    public TermsAggregatorBuilder collectMode(SubAggCollectionMode collectMode) {
+    public TermsAggregationBuilder collectMode(SubAggCollectionMode collectMode) {
         if (collectMode == null) {
             throw new IllegalArgumentException("[collectMode] must not be null: [" + name + "]");
         }
@@ -223,7 +223,7 @@ public class TermsAggregatorBuilder extends ValuesSourceAggregatorBuilder<Values
     /**
      * Set terms to include and exclude from the aggregation results
      */
-    public TermsAggregatorBuilder includeExclude(IncludeExclude includeExclude) {
+    public TermsAggregationBuilder includeExclude(IncludeExclude includeExclude) {
         this.includeExclude = includeExclude;
         return this;
     }
@@ -245,7 +245,7 @@ public class TermsAggregatorBuilder extends ValuesSourceAggregatorBuilder<Values
     /**
      * Set whether doc count error will be return for individual terms
      */
-    public TermsAggregatorBuilder showTermDocCountError(boolean showTermDocCountError) {
+    public TermsAggregationBuilder showTermDocCountError(boolean showTermDocCountError) {
         this.showTermDocCountError = showTermDocCountError;
         return this;
     }
@@ -262,7 +262,7 @@ public class TermsAggregatorBuilder extends ValuesSourceAggregatorBuilder<Values
         bucketCountThresholds.toXContent(builder, params);
         builder.field(SHOW_TERM_DOC_COUNT_ERROR.getPreferredName(), showTermDocCountError);
         if (executionHint != null) {
-            builder.field(TermsAggregatorBuilder.EXECUTION_HINT_FIELD_NAME.getPreferredName(), executionHint);
+            builder.field(TermsAggregationBuilder.EXECUTION_HINT_FIELD_NAME.getPreferredName(), executionHint);
         }
         builder.field(ORDER_FIELD.getPreferredName());
         order.toXContent(builder, params);
@@ -280,7 +280,7 @@ public class TermsAggregatorBuilder extends ValuesSourceAggregatorBuilder<Values
 
     @Override
     protected boolean innerEquals(Object obj) {
-        TermsAggregatorBuilder other = (TermsAggregatorBuilder) obj;
+        TermsAggregationBuilder other = (TermsAggregationBuilder) obj;
         return Objects.equals(bucketCountThresholds, other.bucketCountThresholds)
                 && Objects.equals(collectMode, other.collectMode)
                 && Objects.equals(executionHint, other.executionHint)
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java
index eefaf63c62b..bd4fe874de7 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java
@@ -139,10 +139,10 @@ public abstract class TermsAggregator extends BucketsAggregator {
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.field(TermsAggregatorBuilder.REQUIRED_SIZE_FIELD_NAME.getPreferredName(), requiredSize);
-            builder.field(TermsAggregatorBuilder.SHARD_SIZE_FIELD_NAME.getPreferredName(), shardSize);
-            builder.field(TermsAggregatorBuilder.MIN_DOC_COUNT_FIELD_NAME.getPreferredName(), minDocCount);
-            builder.field(TermsAggregatorBuilder.SHARD_MIN_DOC_COUNT_FIELD_NAME.getPreferredName(), shardMinDocCount);
+            builder.field(TermsAggregationBuilder.REQUIRED_SIZE_FIELD_NAME.getPreferredName(), requiredSize);
+            builder.field(TermsAggregationBuilder.SHARD_SIZE_FIELD_NAME.getPreferredName(), shardSize);
+            builder.field(TermsAggregationBuilder.MIN_DOC_COUNT_FIELD_NAME.getPreferredName(), minDocCount);
+            builder.field(TermsAggregationBuilder.SHARD_MIN_DOC_COUNT_FIELD_NAME.getPreferredName(), shardMinDocCount);
             return builder;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
index 62374ae7d19..14df012d4b7 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
@@ -93,7 +93,7 @@ public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory<Values
         }
         BucketCountThresholds bucketCountThresholds = new BucketCountThresholds(this.bucketCountThresholds);
         if (!(order == InternalOrder.TERM_ASC || order == InternalOrder.TERM_DESC)
-                && bucketCountThresholds.getShardSize() == TermsAggregatorBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS.getShardSize()) {
+                && bucketCountThresholds.getShardSize() == TermsAggregationBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS.getShardSize()) {
             // The user has not made a shardSize selection. Use default
             // heuristic to avoid any wrong-ranking caused by distributed
             // counting
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsParser.java
index 343ec5e2d78..2a67dbe2218 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsParser.java
@@ -41,12 +41,13 @@ import java.util.Map;
  */
 public class TermsParser extends AbstractTermsParser {
     @Override
-    protected TermsAggregatorBuilder doCreateFactory(String aggregationName, ValuesSourceType valuesSourceType,
-            ValueType targetValueType, BucketCountThresholds bucketCountThresholds, SubAggCollectionMode collectMode, String executionHint,
-            IncludeExclude incExc, Map<ParseField, Object> otherOptions) {
-        TermsAggregatorBuilder factory = new TermsAggregatorBuilder(aggregationName, targetValueType);
+    protected TermsAggregationBuilder doCreateFactory(String aggregationName, ValuesSourceType valuesSourceType,
+                                                      ValueType targetValueType, BucketCountThresholds bucketCountThresholds,
+                                                      SubAggCollectionMode collectMode, String executionHint,
+                                                      IncludeExclude incExc, Map<ParseField, Object> otherOptions) {
+        TermsAggregationBuilder factory = new TermsAggregationBuilder(aggregationName, targetValueType);
         @SuppressWarnings("unchecked")
-        List<OrderElement> orderElements = (List<OrderElement>) otherOptions.get(TermsAggregatorBuilder.ORDER_FIELD);
+        List<OrderElement> orderElements = (List<OrderElement>) otherOptions.get(TermsAggregationBuilder.ORDER_FIELD);
         if (orderElements != null) {
             List<Terms.Order> orders = new ArrayList<>(orderElements.size());
             for (OrderElement orderElement : orderElements) {
@@ -66,7 +67,7 @@ public class TermsParser extends AbstractTermsParser {
         if (incExc != null) {
             factory.includeExclude(incExc);
         }
-        Boolean showTermDocCountError = (Boolean) otherOptions.get(TermsAggregatorBuilder.SHOW_TERM_DOC_COUNT_ERROR);
+        Boolean showTermDocCountError = (Boolean) otherOptions.get(TermsAggregationBuilder.SHOW_TERM_DOC_COUNT_ERROR);
         if (showTermDocCountError != null) {
             factory.showTermDocCountError(showTermDocCountError);
         }
@@ -77,12 +78,12 @@ public class TermsParser extends AbstractTermsParser {
     public boolean parseSpecial(String aggregationName, XContentParser parser, ParseFieldMatcher parseFieldMatcher, Token token,
             String currentFieldName, Map<ParseField, Object> otherOptions) throws IOException {
         if (token == XContentParser.Token.START_OBJECT) {
-            if (parseFieldMatcher.match(currentFieldName, TermsAggregatorBuilder.ORDER_FIELD)) {
-                otherOptions.put(TermsAggregatorBuilder.ORDER_FIELD, Collections.singletonList(parseOrderParam(aggregationName, parser)));
+            if (parseFieldMatcher.match(currentFieldName, TermsAggregationBuilder.ORDER_FIELD)) {
+                otherOptions.put(TermsAggregationBuilder.ORDER_FIELD, Collections.singletonList(parseOrderParam(aggregationName, parser)));
                 return true;
             }
         } else if (token == XContentParser.Token.START_ARRAY) {
-            if (parseFieldMatcher.match(currentFieldName, TermsAggregatorBuilder.ORDER_FIELD)) {
+            if (parseFieldMatcher.match(currentFieldName, TermsAggregationBuilder.ORDER_FIELD)) {
                 List<OrderElement> orderElements = new ArrayList<>();
                 while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                     if (token == XContentParser.Token.START_OBJECT) {
@@ -93,12 +94,12 @@ public class TermsParser extends AbstractTermsParser {
                                 "Order elements must be of type object in [" + aggregationName + "] found token of type [" + token + "].");
                     }
                 }
-                otherOptions.put(TermsAggregatorBuilder.ORDER_FIELD, orderElements);
+                otherOptions.put(TermsAggregationBuilder.ORDER_FIELD, orderElements);
                 return true;
             }
         } else if (token == XContentParser.Token.VALUE_BOOLEAN) {
-            if (parseFieldMatcher.match(currentFieldName, TermsAggregatorBuilder.SHOW_TERM_DOC_COUNT_ERROR)) {
-                otherOptions.put(TermsAggregatorBuilder.SHOW_TERM_DOC_COUNT_ERROR, parser.booleanValue());
+            if (parseFieldMatcher.match(currentFieldName, TermsAggregationBuilder.SHOW_TERM_DOC_COUNT_ERROR)) {
+                otherOptions.put(TermsAggregationBuilder.SHOW_TERM_DOC_COUNT_ERROR, parser.booleanValue());
                 return true;
             }
         }
@@ -158,7 +159,7 @@ public class TermsParser extends AbstractTermsParser {
 
     @Override
     public TermsAggregator.BucketCountThresholds getDefaultBucketCountThresholds() {
-        return new TermsAggregator.BucketCountThresholds(TermsAggregatorBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS);
+        return new TermsAggregator.BucketCountThresholds(TermsAggregationBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS);
     }
 
     static Terms.Order resolveOrder(String key, boolean asc) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregationBuilder.java
similarity index 91%
rename from core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregationBuilder.java
index da3733d13a9..ce098177a0b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregationBuilder.java
@@ -29,24 +29,24 @@ import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
 
-public class AvgAggregatorBuilder extends ValuesSourceAggregatorBuilder.LeafOnly<ValuesSource.Numeric, AvgAggregatorBuilder> {
+public class AvgAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, AvgAggregationBuilder> {
     public static final String NAME = InternalAvg.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
-    public AvgAggregatorBuilder(String name) {
+    public AvgAggregationBuilder(String name) {
         super(name, InternalAvg.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
-    public AvgAggregatorBuilder(StreamInput in) throws IOException {
+    public AvgAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalAvg.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
@@ -80,4 +80,4 @@ public class AvgAggregatorBuilder extends ValuesSourceAggregatorBuilder.LeafOnly
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgParser.java
index edb3d8f6620..b4f9261b1eb 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgParser.java
@@ -44,8 +44,8 @@ public class AvgParser extends NumericValuesSourceParser {
     }
 
     @Override
-    protected AvgAggregatorBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        return new AvgAggregatorBuilder(aggregationName);
+    protected AvgAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
+                                                  ValueType targetValueType, Map<ParseField, Object> otherOptions) {
+        return new AvgAggregationBuilder(aggregationName);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregationBuilder.java
similarity index 90%
rename from core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregationBuilder.java
index c9465cfb94d..a7850c23475 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregationBuilder.java
@@ -28,14 +28,16 @@ import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
 import java.util.Objects;
 
-public final class CardinalityAggregatorBuilder extends ValuesSourceAggregatorBuilder.LeafOnly<ValuesSource, CardinalityAggregatorBuilder> {
+public final class CardinalityAggregationBuilder
+    extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource, CardinalityAggregationBuilder> {
+
     public static final String NAME = InternalCardinality.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
@@ -43,14 +45,14 @@ public final class CardinalityAggregatorBuilder extends ValuesSourceAggregatorBu
 
     private Long precisionThreshold = null;
 
-    public CardinalityAggregatorBuilder(String name, ValueType targetValueType) {
+    public CardinalityAggregationBuilder(String name, ValueType targetValueType) {
         super(name, InternalCardinality.TYPE, ValuesSourceType.ANY, targetValueType);
     }
 
     /**
      * Read from a stream.
      */
-    public CardinalityAggregatorBuilder(StreamInput in) throws IOException {
+    public CardinalityAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalCardinality.TYPE, ValuesSourceType.ANY);
         if (in.readBoolean()) {
             precisionThreshold = in.readLong();
@@ -75,7 +77,7 @@ public final class CardinalityAggregatorBuilder extends ValuesSourceAggregatorBu
      * Set a precision threshold. Higher values improve accuracy but also
      * increase memory usage.
      */
-    public CardinalityAggregatorBuilder precisionThreshold(long precisionThreshold) {
+    public CardinalityAggregationBuilder precisionThreshold(long precisionThreshold) {
         if (precisionThreshold < 0) {
             throw new IllegalArgumentException(
                     "[precisionThreshold] must be greater than or equal to 0. Found [" + precisionThreshold + "] in [" + name + "]");
@@ -122,7 +124,7 @@ public final class CardinalityAggregatorBuilder extends ValuesSourceAggregatorBu
 
     @Override
     protected boolean innerEquals(Object obj) {
-        CardinalityAggregatorBuilder other = (CardinalityAggregatorBuilder) obj;
+        CardinalityAggregationBuilder other = (CardinalityAggregationBuilder) obj;
         return Objects.equals(precisionThreshold, other.precisionThreshold);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityParser.java
index 3272d90eede..3a2e6a2072a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityParser.java
@@ -40,10 +40,10 @@ public class CardinalityParser extends AnyValuesSourceParser {
     }
 
     @Override
-    protected CardinalityAggregatorBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        CardinalityAggregatorBuilder factory = new CardinalityAggregatorBuilder(aggregationName, targetValueType);
-        Long precisionThreshold = (Long) otherOptions.get(CardinalityAggregatorBuilder.PRECISION_THRESHOLD_FIELD);
+    protected CardinalityAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
+                                                          ValueType targetValueType, Map<ParseField, Object> otherOptions) {
+        CardinalityAggregationBuilder factory = new CardinalityAggregationBuilder(aggregationName, targetValueType);
+        Long precisionThreshold = (Long) otherOptions.get(CardinalityAggregationBuilder.PRECISION_THRESHOLD_FIELD);
         if (precisionThreshold != null) {
             factory.precisionThreshold(precisionThreshold);
         }
@@ -54,8 +54,8 @@ public class CardinalityParser extends AnyValuesSourceParser {
     protected boolean token(String aggregationName, String currentFieldName, Token token, XContentParser parser,
             ParseFieldMatcher parseFieldMatcher, Map<ParseField, Object> otherOptions) throws IOException {
         if (token.isValue()) {
-            if (parseFieldMatcher.match(currentFieldName, CardinalityAggregatorBuilder.PRECISION_THRESHOLD_FIELD)) {
-                otherOptions.put(CardinalityAggregatorBuilder.PRECISION_THRESHOLD_FIELD, parser.longValue());
+            if (parseFieldMatcher.match(currentFieldName, CardinalityAggregationBuilder.PRECISION_THRESHOLD_FIELD)) {
+                otherOptions.put(CardinalityAggregationBuilder.PRECISION_THRESHOLD_FIELD, parser.longValue());
                 return true;
             } else if (parseFieldMatcher.match(currentFieldName, REHASH)) {
                 // ignore
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregationBuilder.java
similarity index 88%
rename from core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregationBuilder.java
index ea4681ed686..eff020ec610 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsAggregationBuilder.java
@@ -28,27 +28,27 @@ import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
 import java.util.Objects;
 
-public class GeoBoundsAggregatorBuilder extends ValuesSourceAggregatorBuilder<ValuesSource.GeoPoint, GeoBoundsAggregatorBuilder> {
+public class GeoBoundsAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource.GeoPoint, GeoBoundsAggregationBuilder> {
     public static final String NAME = InternalGeoBounds.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIED = new ParseField(NAME);
 
     private boolean wrapLongitude = true;
 
-    public GeoBoundsAggregatorBuilder(String name) {
+    public GeoBoundsAggregationBuilder(String name) {
         super(name, InternalGeoBounds.TYPE, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
     }
 
     /**
      * Read from a stream.
      */
-    public GeoBoundsAggregatorBuilder(StreamInput in) throws IOException {
+    public GeoBoundsAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalGeoBounds.TYPE, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
         wrapLongitude = in.readBoolean();
     }
@@ -61,7 +61,7 @@ public class GeoBoundsAggregatorBuilder extends ValuesSourceAggregatorBuilder<Va
     /**
      * Set whether to wrap longitudes. Defaults to true.
      */
-    public GeoBoundsAggregatorBuilder wrapLongitude(boolean wrapLongitude) {
+    public GeoBoundsAggregationBuilder wrapLongitude(boolean wrapLongitude) {
         this.wrapLongitude = wrapLongitude;
         return this;
     }
@@ -92,7 +92,7 @@ public class GeoBoundsAggregatorBuilder extends ValuesSourceAggregatorBuilder<Va
 
     @Override
     protected boolean innerEquals(Object obj) {
-        GeoBoundsAggregatorBuilder other = (GeoBoundsAggregatorBuilder) obj;
+        GeoBoundsAggregationBuilder other = (GeoBoundsAggregationBuilder) obj;
         return Objects.equals(wrapLongitude, other.wrapLongitude);
     }
 
@@ -100,4 +100,4 @@ public class GeoBoundsAggregatorBuilder extends ValuesSourceAggregatorBuilder<Va
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsParser.java
index 60c0d082463..7420fc0149e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/GeoBoundsParser.java
@@ -37,9 +37,9 @@ public class GeoBoundsParser extends GeoPointValuesSourceParser {
     }
 
     @Override
-    protected GeoBoundsAggregatorBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        GeoBoundsAggregatorBuilder factory = new GeoBoundsAggregatorBuilder(aggregationName);
+    protected GeoBoundsAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
+                                                        ValueType targetValueType, Map<ParseField, Object> otherOptions) {
+        GeoBoundsAggregationBuilder factory = new GeoBoundsAggregationBuilder(aggregationName);
         Boolean wrapLongitude = (Boolean) otherOptions.get(GeoBoundsAggregator.WRAP_LONGITUDE_FIELD);
         if (wrapLongitude != null) {
             factory.wrapLongitude(wrapLongitude);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregationBuilder.java
similarity index 90%
rename from core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregationBuilder.java
index ea8e54cdba9..f9bf2e0a346 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregationBuilder.java
@@ -28,25 +28,25 @@ import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
 
-public class GeoCentroidAggregatorBuilder
-        extends ValuesSourceAggregatorBuilder.LeafOnly<ValuesSource.GeoPoint, GeoCentroidAggregatorBuilder> {
+public class GeoCentroidAggregationBuilder
+        extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.GeoPoint, GeoCentroidAggregationBuilder> {
     public static final String NAME = InternalGeoCentroid.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
-    public GeoCentroidAggregatorBuilder(String name) {
+    public GeoCentroidAggregationBuilder(String name) {
         super(name, InternalGeoCentroid.TYPE, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
     }
 
     /**
      * Read from a stream.
      */
-    public GeoCentroidAggregatorBuilder(StreamInput in) throws IOException {
+    public GeoCentroidAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalGeoCentroid.TYPE, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
     }
 
@@ -80,4 +80,4 @@ public class GeoCentroidAggregatorBuilder
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidParser.java
index b056920b141..6c9e9ba67b0 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidParser.java
@@ -46,8 +46,8 @@ public class GeoCentroidParser extends GeoPointValuesSourceParser {
     }
 
     @Override
-    protected GeoCentroidAggregatorBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        return new GeoCentroidAggregatorBuilder(aggregationName);
+    protected GeoCentroidAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
+                                                          ValueType targetValueType, Map<ParseField, Object> otherOptions) {
+        return new GeoCentroidAggregationBuilder(aggregationName);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregationBuilder.java
similarity index 91%
rename from core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregationBuilder.java
index 7258e77aea5..9fa919fcf9e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregationBuilder.java
@@ -29,24 +29,24 @@ import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
 
-public class MaxAggregatorBuilder extends ValuesSourceAggregatorBuilder.LeafOnly<ValuesSource.Numeric, MaxAggregatorBuilder> {
+public class MaxAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, MaxAggregationBuilder> {
     public static final String NAME = InternalMax.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
-    public MaxAggregatorBuilder(String name) {
+    public MaxAggregationBuilder(String name) {
         super(name, InternalMax.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
-    public MaxAggregatorBuilder(StreamInput in) throws IOException {
+    public MaxAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalMax.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
@@ -80,4 +80,4 @@ public class MaxAggregatorBuilder extends ValuesSourceAggregatorBuilder.LeafOnly
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxParser.java
index 41c321acf33..d2ddd4daa08 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxParser.java
@@ -44,8 +44,8 @@ public class MaxParser extends NumericValuesSourceParser {
     }
 
     @Override
-    protected MaxAggregatorBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        return new MaxAggregatorBuilder(aggregationName);
+    protected MaxAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
+                                                  ValueType targetValueType, Map<ParseField, Object> otherOptions) {
+        return new MaxAggregationBuilder(aggregationName);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregationBuilder.java
similarity index 91%
rename from core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregationBuilder.java
index c51e97f0538..af4f204bddb 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregationBuilder.java
@@ -29,24 +29,24 @@ import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
 
-public class MinAggregatorBuilder extends ValuesSourceAggregatorBuilder.LeafOnly<ValuesSource.Numeric, MinAggregatorBuilder> {
+public class MinAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, MinAggregationBuilder> {
     public static final String NAME = InternalMin.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
-    public MinAggregatorBuilder(String name) {
+    public MinAggregationBuilder(String name) {
         super(name, InternalMin.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
-    public MinAggregatorBuilder(StreamInput in) throws IOException {
+    public MinAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalMin.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
@@ -80,4 +80,4 @@ public class MinAggregatorBuilder extends ValuesSourceAggregatorBuilder.LeafOnly
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinParser.java
index 9f9eafc5035..194c08fc49b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinParser.java
@@ -45,8 +45,8 @@ public class MinParser extends NumericValuesSourceParser {
     }
 
     @Override
-    protected MinAggregatorBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        return new MinAggregatorBuilder(aggregationName);
+    protected MinAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
+                                                  ValueType targetValueType, Map<ParseField, Object> otherOptions) {
+        return new MinAggregationBuilder(aggregationName);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesParser.java
index f29615a593f..ec145754a04 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractPercentilesParser.java
@@ -27,7 +27,7 @@ import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.NumericValuesSourceParser;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
@@ -115,8 +115,8 @@ public abstract class AbstractPercentilesParser extends NumericValuesSourceParse
     }
 
     @Override
-    protected ValuesSourceAggregatorBuilder<Numeric, ?> createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
+    protected ValuesSourceAggregationBuilder<Numeric, ?> createFactory(String aggregationName, ValuesSourceType valuesSourceType,
+                                                                       ValueType targetValueType, Map<ParseField, Object> otherOptions) {
         PercentilesMethod method = (PercentilesMethod) otherOptions.getOrDefault(METHOD_FIELD, PercentilesMethod.TDIGEST);
 
         double[] cdfValues = (double[]) otherOptions.get(keysField());
@@ -126,10 +126,10 @@ public abstract class AbstractPercentilesParser extends NumericValuesSourceParse
         return buildFactory(aggregationName, cdfValues, method, compression, numberOfSignificantValueDigits, keyed);
     }
 
-    protected abstract ValuesSourceAggregatorBuilder<Numeric, ?> buildFactory(String aggregationName, double[] cdfValues,
-            PercentilesMethod method,
-            Double compression,
-            Integer numberOfSignificantValueDigits, Boolean keyed);
+    protected abstract ValuesSourceAggregationBuilder<Numeric, ?> buildFactory(String aggregationName, double[] cdfValues,
+                                                                               PercentilesMethod method,
+                                                                               Double compression,
+                                                                               Integer numberOfSignificantValueDigits, Boolean keyed);
 
     protected abstract ParseField keysField();
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksAggregationBuilder.java
similarity index 91%
rename from core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksAggregationBuilder.java
index ea03451cf67..d36dcdecb7b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksAggregationBuilder.java
@@ -32,7 +32,7 @@ import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder.LeafOnly;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder.LeafOnly;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
@@ -41,7 +41,7 @@ import java.io.IOException;
 import java.util.Arrays;
 import java.util.Objects;
 
-public class PercentileRanksAggregatorBuilder extends LeafOnly<ValuesSource.Numeric, PercentileRanksAggregatorBuilder> {
+public class PercentileRanksAggregationBuilder extends LeafOnly<ValuesSource.Numeric, PercentileRanksAggregationBuilder> {
     public static final String NAME = InternalTDigestPercentileRanks.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
@@ -51,14 +51,14 @@ public class PercentileRanksAggregatorBuilder extends LeafOnly<ValuesSource.Nume
     private double compression = 100.0;
     private boolean keyed = true;
 
-    public PercentileRanksAggregatorBuilder(String name) {
+    public PercentileRanksAggregationBuilder(String name) {
         super(name, InternalTDigestPercentileRanks.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
-    public PercentileRanksAggregatorBuilder(StreamInput in) throws IOException {
+    public PercentileRanksAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalTDigestPercentileRanks.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
         values = in.readDoubleArray();
         keyed = in.readBoolean();
@@ -79,7 +79,7 @@ public class PercentileRanksAggregatorBuilder extends LeafOnly<ValuesSource.Nume
     /**
      * Set the values to compute percentiles from.
      */
-    public PercentileRanksAggregatorBuilder values(double... values) {
+    public PercentileRanksAggregationBuilder values(double... values) {
         if (values == null) {
             throw new IllegalArgumentException("[values] must not be null: [" + name + "]");
         }
@@ -99,7 +99,7 @@ public class PercentileRanksAggregatorBuilder extends LeafOnly<ValuesSource.Nume
     /**
      * Set whether the XContent response should be keyed
      */
-    public PercentileRanksAggregatorBuilder keyed(boolean keyed) {
+    public PercentileRanksAggregationBuilder keyed(boolean keyed) {
         this.keyed = keyed;
         return this;
     }
@@ -115,7 +115,7 @@ public class PercentileRanksAggregatorBuilder extends LeafOnly<ValuesSource.Nume
      * Expert: set the number of significant digits in the values. Only relevant
      * when using {@link PercentilesMethod#HDR}.
      */
-    public PercentileRanksAggregatorBuilder numberOfSignificantValueDigits(int numberOfSignificantValueDigits) {
+    public PercentileRanksAggregationBuilder numberOfSignificantValueDigits(int numberOfSignificantValueDigits) {
         if (numberOfSignificantValueDigits < 0 || numberOfSignificantValueDigits > 5) {
             throw new IllegalArgumentException("[numberOfSignificantValueDigits] must be between 0 and 5: [" + name + "]");
         }
@@ -135,7 +135,7 @@ public class PercentileRanksAggregatorBuilder extends LeafOnly<ValuesSource.Nume
      * Expert: set the compression. Higher values improve accuracy but also
      * memory usage. Only relevant when using {@link PercentilesMethod#TDIGEST}.
      */
-    public PercentileRanksAggregatorBuilder compression(double compression) {
+    public PercentileRanksAggregationBuilder compression(double compression) {
         if (compression < 0.0) {
             throw new IllegalArgumentException(
                     "[compression] must be greater than or equal to 0. Found [" + compression + "] in [" + name + "]");
@@ -152,7 +152,7 @@ public class PercentileRanksAggregatorBuilder extends LeafOnly<ValuesSource.Nume
         return compression;
     }
 
-    public PercentileRanksAggregatorBuilder method(PercentilesMethod method) {
+    public PercentileRanksAggregationBuilder method(PercentilesMethod method) {
         if (method == null) {
             throw new IllegalArgumentException("[method] must not be null: [" + name + "]");
         }
@@ -195,7 +195,7 @@ public class PercentileRanksAggregatorBuilder extends LeafOnly<ValuesSource.Nume
 
     @Override
     protected boolean innerEquals(Object obj) {
-        PercentileRanksAggregatorBuilder other = (PercentileRanksAggregatorBuilder) obj;
+        PercentileRanksAggregationBuilder other = (PercentileRanksAggregationBuilder) obj;
         if (!Objects.equals(method, other.method)) {
             return false;
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksParser.java
index ab2a4b01723..313b8e7b7c0 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksParser.java
@@ -20,7 +20,7 @@ package org.elasticsearch.search.aggregations.metrics.percentiles;
 
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 
 /**
  *
@@ -39,9 +39,10 @@ public class PercentileRanksParser extends AbstractPercentilesParser {
     }
 
     @Override
-    protected ValuesSourceAggregatorBuilder<Numeric, ?> buildFactory(String aggregationName, double[] keys, PercentilesMethod method,
-            Double compression, Integer numberOfSignificantValueDigits, Boolean keyed) {
-        PercentileRanksAggregatorBuilder factory = new PercentileRanksAggregatorBuilder(aggregationName);
+    protected ValuesSourceAggregationBuilder<Numeric, ?> buildFactory(String aggregationName, double[] keys, PercentilesMethod method,
+                                                                      Double compression, Integer numberOfSignificantValueDigits,
+                                                                      Boolean keyed) {
+        PercentileRanksAggregationBuilder factory = new PercentileRanksAggregationBuilder(aggregationName);
         if (keys != null) {
             factory.values(keys);
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesAggregationBuilder.java
similarity index 91%
rename from core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesAggregationBuilder.java
index 45f3d37732f..d2d1e8d9f8a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesAggregationBuilder.java
@@ -32,7 +32,7 @@ import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder.LeafOnly;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder.LeafOnly;
 import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
@@ -41,7 +41,7 @@ import java.io.IOException;
 import java.util.Arrays;
 import java.util.Objects;
 
-public class PercentilesAggregatorBuilder extends LeafOnly<ValuesSource.Numeric, PercentilesAggregatorBuilder> {
+public class PercentilesAggregationBuilder extends LeafOnly<ValuesSource.Numeric, PercentilesAggregationBuilder> {
     public static final String NAME = InternalTDigestPercentiles.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
@@ -51,14 +51,14 @@ public class PercentilesAggregatorBuilder extends LeafOnly<ValuesSource.Numeric,
     private double compression = 100.0;
     private boolean keyed = true;
 
-    public PercentilesAggregatorBuilder(String name) {
+    public PercentilesAggregationBuilder(String name) {
         super(name, InternalTDigestPercentiles.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
-    public PercentilesAggregatorBuilder(StreamInput in) throws IOException {
+    public PercentilesAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalTDigestPercentiles.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
         percents = in.readDoubleArray();
         keyed = in.readBoolean();
@@ -79,7 +79,7 @@ public class PercentilesAggregatorBuilder extends LeafOnly<ValuesSource.Numeric,
     /**
      * Set the values to compute percentiles from.
      */
-    public PercentilesAggregatorBuilder percentiles(double... percents) {
+    public PercentilesAggregationBuilder percentiles(double... percents) {
         if (percents == null) {
             throw new IllegalArgumentException("[percents] must not be null: [" + name + "]");
         }
@@ -99,7 +99,7 @@ public class PercentilesAggregatorBuilder extends LeafOnly<ValuesSource.Numeric,
     /**
      * Set whether the XContent response should be keyed
      */
-    public PercentilesAggregatorBuilder keyed(boolean keyed) {
+    public PercentilesAggregationBuilder keyed(boolean keyed) {
         this.keyed = keyed;
         return this;
     }
@@ -115,7 +115,7 @@ public class PercentilesAggregatorBuilder extends LeafOnly<ValuesSource.Numeric,
      * Expert: set the number of significant digits in the values. Only relevant
      * when using {@link PercentilesMethod#HDR}.
      */
-    public PercentilesAggregatorBuilder numberOfSignificantValueDigits(int numberOfSignificantValueDigits) {
+    public PercentilesAggregationBuilder numberOfSignificantValueDigits(int numberOfSignificantValueDigits) {
         if (numberOfSignificantValueDigits < 0 || numberOfSignificantValueDigits > 5) {
             throw new IllegalArgumentException("[numberOfSignificantValueDigits] must be between 0 and 5: [" + name + "]");
         }
@@ -135,7 +135,7 @@ public class PercentilesAggregatorBuilder extends LeafOnly<ValuesSource.Numeric,
      * Expert: set the compression. Higher values improve accuracy but also
      * memory usage. Only relevant when using {@link PercentilesMethod#TDIGEST}.
      */
-    public PercentilesAggregatorBuilder compression(double compression) {
+    public PercentilesAggregationBuilder compression(double compression) {
         if (compression < 0.0) {
             throw new IllegalArgumentException(
                     "[compression] must be greater than or equal to 0. Found [" + compression + "] in [" + name + "]");
@@ -152,7 +152,7 @@ public class PercentilesAggregatorBuilder extends LeafOnly<ValuesSource.Numeric,
         return compression;
     }
 
-    public PercentilesAggregatorBuilder method(PercentilesMethod method) {
+    public PercentilesAggregationBuilder method(PercentilesMethod method) {
         if (method == null) {
             throw new IllegalArgumentException("[method] must not be null: [" + name + "]");
         }
@@ -195,7 +195,7 @@ public class PercentilesAggregatorBuilder extends LeafOnly<ValuesSource.Numeric,
 
     @Override
     protected boolean innerEquals(Object obj) {
-        PercentilesAggregatorBuilder other = (PercentilesAggregatorBuilder) obj;
+        PercentilesAggregationBuilder other = (PercentilesAggregationBuilder) obj;
         if (!Objects.equals(method, other.method)) {
             return false;
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesParser.java
index a53a9c8670c..ffdc4d9a631 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesParser.java
@@ -20,7 +20,7 @@ package org.elasticsearch.search.aggregations.metrics.percentiles;
 
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 
 /**
  *
@@ -41,9 +41,10 @@ public class PercentilesParser extends AbstractPercentilesParser {
     }
 
     @Override
-    protected ValuesSourceAggregatorBuilder<Numeric, ?> buildFactory(String aggregationName, double[] keys, PercentilesMethod method,
-            Double compression, Integer numberOfSignificantValueDigits, Boolean keyed) {
-        PercentilesAggregatorBuilder factory = new PercentilesAggregatorBuilder(aggregationName);
+    protected ValuesSourceAggregationBuilder<Numeric, ?> buildFactory(String aggregationName, double[] keys, PercentilesMethod method,
+                                                                      Double compression, Integer numberOfSignificantValueDigits,
+                                                                      Boolean keyed) {
+        PercentilesAggregationBuilder factory = new PercentilesAggregationBuilder(aggregationName);
         if (keys != null) {
             factory.percentiles(keys);
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java
similarity index 93%
rename from core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java
index 753052b4fe0..d5bdf2f5626 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java
@@ -29,7 +29,7 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptParameterParser;
 import org.elasticsearch.script.ScriptParameterParser.ScriptParameterValue;
-import org.elasticsearch.search.aggregations.AggregatorBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
@@ -40,7 +40,7 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 
-public class ScriptedMetricAggregatorBuilder extends AggregatorBuilder<ScriptedMetricAggregatorBuilder> {
+public class ScriptedMetricAggregationBuilder extends AggregationBuilder<ScriptedMetricAggregationBuilder> {
 
     public static final String NAME = InternalScriptedMetric.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
@@ -58,14 +58,14 @@ public class ScriptedMetricAggregatorBuilder extends AggregatorBuilder<ScriptedM
     private Script reduceScript;
     private Map<String, Object> params;
 
-    public ScriptedMetricAggregatorBuilder(String name) {
+    public ScriptedMetricAggregationBuilder(String name) {
         super(name, InternalScriptedMetric.TYPE);
     }
 
     /**
      * Read from a stream.
      */
-    public ScriptedMetricAggregatorBuilder(StreamInput in) throws IOException {
+    public ScriptedMetricAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalScriptedMetric.TYPE);
         initScript = in.readOptionalWriteable(Script::new);
         mapScript = in.readOptionalWriteable(Script::new);
@@ -92,7 +92,7 @@ public class ScriptedMetricAggregatorBuilder extends AggregatorBuilder<ScriptedM
     /**
      * Set the <tt>init</tt> script.
      */
-    public ScriptedMetricAggregatorBuilder initScript(Script initScript) {
+    public ScriptedMetricAggregationBuilder initScript(Script initScript) {
         if (initScript == null) {
             throw new IllegalArgumentException("[initScript] must not be null: [" + name + "]");
         }
@@ -110,7 +110,7 @@ public class ScriptedMetricAggregatorBuilder extends AggregatorBuilder<ScriptedM
     /**
      * Set the <tt>map</tt> script.
      */
-    public ScriptedMetricAggregatorBuilder mapScript(Script mapScript) {
+    public ScriptedMetricAggregationBuilder mapScript(Script mapScript) {
         if (mapScript == null) {
             throw new IllegalArgumentException("[mapScript] must not be null: [" + name + "]");
         }
@@ -128,7 +128,7 @@ public class ScriptedMetricAggregatorBuilder extends AggregatorBuilder<ScriptedM
     /**
      * Set the <tt>combine</tt> script.
      */
-    public ScriptedMetricAggregatorBuilder combineScript(Script combineScript) {
+    public ScriptedMetricAggregationBuilder combineScript(Script combineScript) {
         if (combineScript == null) {
             throw new IllegalArgumentException("[combineScript] must not be null: [" + name + "]");
         }
@@ -146,7 +146,7 @@ public class ScriptedMetricAggregatorBuilder extends AggregatorBuilder<ScriptedM
     /**
      * Set the <tt>reduce</tt> script.
      */
-    public ScriptedMetricAggregatorBuilder reduceScript(Script reduceScript) {
+    public ScriptedMetricAggregationBuilder reduceScript(Script reduceScript) {
         if (reduceScript == null) {
             throw new IllegalArgumentException("[reduceScript] must not be null: [" + name + "]");
         }
@@ -165,7 +165,7 @@ public class ScriptedMetricAggregatorBuilder extends AggregatorBuilder<ScriptedM
      * Set parameters that will be available in the <tt>init</tt>,
      * <tt>map</tt> and <tt>combine</tt> phases.
      */
-    public ScriptedMetricAggregatorBuilder params(Map<String, Object> params) {
+    public ScriptedMetricAggregationBuilder params(Map<String, Object> params) {
         if (params == null) {
             throw new IllegalArgumentException("[params] must not be null: [" + name + "]");
         }
@@ -214,7 +214,7 @@ public class ScriptedMetricAggregatorBuilder extends AggregatorBuilder<ScriptedM
         return builder;
     }
 
-    public static ScriptedMetricAggregatorBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+    public static ScriptedMetricAggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
         Script initScript = null;
         Script mapScript = null;
         Script combineScript = null;
@@ -305,7 +305,7 @@ public class ScriptedMetricAggregatorBuilder extends AggregatorBuilder<ScriptedM
             throw new ParsingException(parser.getTokenLocation(), "map_script field is required in [" + aggregationName + "].");
         }
 
-        ScriptedMetricAggregatorBuilder factory = new ScriptedMetricAggregatorBuilder(aggregationName);
+        ScriptedMetricAggregationBuilder factory = new ScriptedMetricAggregationBuilder(aggregationName);
         if (initScript != null) {
             factory.initScript(initScript);
         }
@@ -336,7 +336,7 @@ public class ScriptedMetricAggregatorBuilder extends AggregatorBuilder<ScriptedM
 
     @Override
     protected boolean doEquals(Object obj) {
-        ScriptedMetricAggregatorBuilder other = (ScriptedMetricAggregatorBuilder) obj;
+        ScriptedMetricAggregationBuilder other = (ScriptedMetricAggregationBuilder) obj;
         return Objects.equals(initScript, other.initScript)
                 && Objects.equals(mapScript, other.mapScript)
                 && Objects.equals(combineScript, other.combineScript)
@@ -344,4 +344,4 @@ public class ScriptedMetricAggregatorBuilder extends AggregatorBuilder<ScriptedM
                 && Objects.equals(params, other.params);
     }
 
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregationBuilder.java
similarity index 91%
rename from core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregationBuilder.java
index ab9e4a8d34f..4fd80e04494 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregationBuilder.java
@@ -29,24 +29,24 @@ import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
 
-public class StatsAggregatorBuilder extends ValuesSourceAggregatorBuilder.LeafOnly<ValuesSource.Numeric, StatsAggregatorBuilder> {
+public class StatsAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, StatsAggregationBuilder> {
     public static final String NAME = InternalStats.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
-    public StatsAggregatorBuilder(String name) {
+    public StatsAggregationBuilder(String name) {
         super(name, InternalStats.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
-    public StatsAggregatorBuilder(StreamInput in) throws IOException {
+    public StatsAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalStats.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
@@ -80,4 +80,4 @@ public class StatsAggregatorBuilder extends ValuesSourceAggregatorBuilder.LeafOn
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsParser.java
index a57ba89b676..eacfc0068b4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsParser.java
@@ -44,8 +44,8 @@ public class StatsParser extends NumericValuesSourceParser {
     }
 
     @Override
-    protected StatsAggregatorBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        return new StatsAggregatorBuilder(aggregationName);
+    protected StatsAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
+                                                    ValueType targetValueType, Map<ParseField, Object> otherOptions) {
+        return new StatsAggregationBuilder(aggregationName);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregationBuilder.java
similarity index 88%
rename from core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregationBuilder.java
index e6f49d719d6..a2b961f1fc3 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregationBuilder.java
@@ -29,28 +29,28 @@ import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
 import java.util.Objects;
 
-public class ExtendedStatsAggregatorBuilder
-        extends ValuesSourceAggregatorBuilder.LeafOnly<ValuesSource.Numeric, ExtendedStatsAggregatorBuilder> {
+public class ExtendedStatsAggregationBuilder
+        extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, ExtendedStatsAggregationBuilder> {
     public static final String NAME = InternalExtendedStats.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     private double sigma = 2.0;
 
-    public ExtendedStatsAggregatorBuilder(String name) {
+    public ExtendedStatsAggregationBuilder(String name) {
         super(name, InternalExtendedStats.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
-    public ExtendedStatsAggregatorBuilder(StreamInput in) throws IOException {
+    public ExtendedStatsAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalExtendedStats.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
         sigma = in.readDouble();
     }
@@ -60,7 +60,7 @@ public class ExtendedStatsAggregatorBuilder
         out.writeDouble(sigma);
     }
 
-    public ExtendedStatsAggregatorBuilder sigma(double sigma) {
+    public ExtendedStatsAggregationBuilder sigma(double sigma) {
         if (sigma < 0.0) {
             throw new IllegalArgumentException("[sigma] must be greater than or equal to 0. Found [" + sigma + "] in [" + name + "]");
         }
@@ -91,7 +91,7 @@ public class ExtendedStatsAggregatorBuilder
 
     @Override
     protected boolean innerEquals(Object obj) {
-        ExtendedStatsAggregatorBuilder other = (ExtendedStatsAggregatorBuilder) obj;
+        ExtendedStatsAggregationBuilder other = (ExtendedStatsAggregationBuilder) obj;
         return Objects.equals(sigma, other.sigma);
     }
 
@@ -99,4 +99,4 @@ public class ExtendedStatsAggregatorBuilder
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsParser.java
index 76e6beac2da..c650847360f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsParser.java
@@ -50,9 +50,9 @@ public class ExtendedStatsParser extends NumericValuesSourceParser {
     }
 
     @Override
-    protected ExtendedStatsAggregatorBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        ExtendedStatsAggregatorBuilder factory = new ExtendedStatsAggregatorBuilder(aggregationName);
+    protected ExtendedStatsAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
+                                                            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
+        ExtendedStatsAggregationBuilder factory = new ExtendedStatsAggregationBuilder(aggregationName);
         Double sigma = (Double) otherOptions.get(ExtendedStatsAggregator.SIGMA_FIELD);
         if (sigma != null) {
             factory.sigma(sigma);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregationBuilder.java
similarity index 91%
rename from core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregationBuilder.java
index 30246452330..25dd1a3f214 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregationBuilder.java
@@ -29,24 +29,24 @@ import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
 
-public class SumAggregatorBuilder extends ValuesSourceAggregatorBuilder.LeafOnly<ValuesSource.Numeric, SumAggregatorBuilder> {
+public class SumAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, SumAggregationBuilder> {
     public static final String NAME = InternalSum.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
-    public SumAggregatorBuilder(String name) {
+    public SumAggregationBuilder(String name) {
         super(name, InternalSum.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
-    public SumAggregatorBuilder(StreamInput in) throws IOException {
+    public SumAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalSum.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
@@ -80,4 +80,4 @@ public class SumAggregatorBuilder extends ValuesSourceAggregatorBuilder.LeafOnly
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumParser.java
index 7971adba4eb..6edc6cc8905 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumParser.java
@@ -44,8 +44,8 @@ public class SumParser extends NumericValuesSourceParser {
     }
 
     @Override
-    protected SumAggregatorBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-            ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        return new SumAggregatorBuilder(aggregationName);
+    protected SumAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
+                                                  ValueType targetValueType, Map<ParseField, Object> otherOptions) {
+        return new SumAggregationBuilder(aggregationName);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java
similarity index 92%
rename from core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java
index 8f15437cc54..7195482f147 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java
@@ -30,7 +30,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.AggregationInitializationException;
-import org.elasticsearch.search.aggregations.AggregatorBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
@@ -51,7 +51,7 @@ import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 
-public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregatorBuilder> {
+public class TopHitsAggregationBuilder extends AggregationBuilder<TopHitsAggregationBuilder> {
     public static final String NAME = InternalTopHits.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
@@ -67,14 +67,14 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
     private Set<ScriptField> scriptFields;
     private FetchSourceContext fetchSourceContext;
 
-    public TopHitsAggregatorBuilder(String name) {
+    public TopHitsAggregationBuilder(String name) {
         super(name, InternalTopHits.TYPE);
     }
 
     /**
      * Read from a stream.
      */
-    public TopHitsAggregatorBuilder(StreamInput in) throws IOException {
+    public TopHitsAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalTopHits.TYPE);
         explain = in.readBoolean();
         fetchSourceContext = in.readOptionalStreamable(FetchSourceContext::new);
@@ -159,7 +159,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
     /**
      * From index to start the search from. Defaults to <tt>0</tt>.
      */
-    public TopHitsAggregatorBuilder from(int from) {
+    public TopHitsAggregationBuilder from(int from) {
         if (from < 0) {
             throw new IllegalArgumentException("[from] must be greater than or equal to 0. Found [" + from + "] in [" + name + "]");
         }
@@ -177,7 +177,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
     /**
      * The number of search hits to return. Defaults to <tt>10</tt>.
      */
-    public TopHitsAggregatorBuilder size(int size) {
+    public TopHitsAggregationBuilder size(int size) {
         if (size < 0) {
             throw new IllegalArgumentException("[size] must be greater than or equal to 0. Found [" + size + "] in [" + name + "]");
         }
@@ -200,7 +200,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
      * @param order
      *            The sort ordering
      */
-    public TopHitsAggregatorBuilder sort(String name, SortOrder order) {
+    public TopHitsAggregationBuilder sort(String name, SortOrder order) {
         if (name == null) {
             throw new IllegalArgumentException("sort [name] must not be null: [" + name + "]");
         }
@@ -220,7 +220,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
      * @param name
      *            The name of the field to sort by
      */
-    public TopHitsAggregatorBuilder sort(String name) {
+    public TopHitsAggregationBuilder sort(String name) {
         if (name == null) {
             throw new IllegalArgumentException("sort [name] must not be null: [" + name + "]");
         }
@@ -234,7 +234,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
     /**
      * Adds a sort builder.
      */
-    public TopHitsAggregatorBuilder sort(SortBuilder<?> sort) {
+    public TopHitsAggregationBuilder sort(SortBuilder<?> sort) {
         if (sort == null) {
             throw new IllegalArgumentException("[sort] must not be null: [" + name + "]");
         }
@@ -248,7 +248,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
     /**
      * Adds a sort builder.
      */
-    public TopHitsAggregatorBuilder sorts(List<SortBuilder<?>> sorts) {
+    public TopHitsAggregationBuilder sorts(List<SortBuilder<?>> sorts) {
         if (sorts == null) {
             throw new IllegalArgumentException("[sorts] must not be null: [" + name + "]");
         }
@@ -271,7 +271,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
     /**
      * Adds highlight to perform as part of the search.
      */
-    public TopHitsAggregatorBuilder highlighter(HighlightBuilder highlightBuilder) {
+    public TopHitsAggregationBuilder highlighter(HighlightBuilder highlightBuilder) {
         if (highlightBuilder == null) {
             throw new IllegalArgumentException("[highlightBuilder] must not be null: [" + name + "]");
         }
@@ -290,7 +290,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
      * Indicates whether the response should contain the stored _source for
      * every hit
      */
-    public TopHitsAggregatorBuilder fetchSource(boolean fetch) {
+    public TopHitsAggregationBuilder fetchSource(boolean fetch) {
         if (this.fetchSourceContext == null) {
             this.fetchSourceContext = new FetchSourceContext(fetch);
         } else {
@@ -311,7 +311,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
      *            An optional exclude (optionally wildcarded) pattern to
      *            filter the returned _source
      */
-    public TopHitsAggregatorBuilder fetchSource(@Nullable String include, @Nullable String exclude) {
+    public TopHitsAggregationBuilder fetchSource(@Nullable String include, @Nullable String exclude) {
         fetchSource(include == null ? Strings.EMPTY_ARRAY : new String[] { include },
                 exclude == null ? Strings.EMPTY_ARRAY : new String[] { exclude });
         return this;
@@ -329,7 +329,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
      *            An optional list of exclude (optionally wildcarded)
      *            pattern to filter the returned _source
      */
-    public TopHitsAggregatorBuilder fetchSource(@Nullable String[] includes, @Nullable String[] excludes) {
+    public TopHitsAggregationBuilder fetchSource(@Nullable String[] includes, @Nullable String[] excludes) {
         fetchSourceContext = new FetchSourceContext(includes, excludes);
         return this;
     }
@@ -337,7 +337,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
     /**
      * Indicate how the _source should be fetched.
      */
-    public TopHitsAggregatorBuilder fetchSource(@Nullable FetchSourceContext fetchSourceContext) {
+    public TopHitsAggregationBuilder fetchSource(@Nullable FetchSourceContext fetchSourceContext) {
         if (fetchSourceContext == null) {
             throw new IllegalArgumentException("[fetchSourceContext] must not be null: [" + name + "]");
         }
@@ -358,7 +358,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
      * the search request. If none are specified, the source of the document
      * will be return.
      */
-    public TopHitsAggregatorBuilder field(String field) {
+    public TopHitsAggregationBuilder field(String field) {
         if (field == null) {
             throw new IllegalArgumentException("[field] must not be null: [" + name + "]");
         }
@@ -373,7 +373,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
      * Sets the fields to load and return as part of the search request. If
      * none are specified, the source of the document will be returned.
      */
-    public TopHitsAggregatorBuilder fields(List<String> fields) {
+    public TopHitsAggregationBuilder fields(List<String> fields) {
         if (fields == null) {
             throw new IllegalArgumentException("[fields] must not be null: [" + name + "]");
         }
@@ -385,7 +385,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
      * Sets no fields to be loaded, resulting in only id and type to be
      * returned per field.
      */
-    public TopHitsAggregatorBuilder noFields() {
+    public TopHitsAggregationBuilder noFields() {
         this.fieldNames = Collections.emptyList();
         return this;
     }
@@ -401,7 +401,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
      * Adds a field to load from the field data cache and return as part of
      * the search request.
      */
-    public TopHitsAggregatorBuilder fieldDataField(String fieldDataField) {
+    public TopHitsAggregationBuilder fieldDataField(String fieldDataField) {
         if (fieldDataField == null) {
             throw new IllegalArgumentException("[fieldDataField] must not be null: [" + name + "]");
         }
@@ -416,7 +416,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
      * Adds fields to load from the field data cache and return as part of
      * the search request.
      */
-    public TopHitsAggregatorBuilder fieldDataFields(List<String> fieldDataFields) {
+    public TopHitsAggregationBuilder fieldDataFields(List<String> fieldDataFields) {
         if (fieldDataFields == null) {
             throw new IllegalArgumentException("[fieldDataFields] must not be null: [" + name + "]");
         }
@@ -442,7 +442,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
      * @param script
      *            The script
      */
-    public TopHitsAggregatorBuilder scriptField(String name, Script script) {
+    public TopHitsAggregationBuilder scriptField(String name, Script script) {
         if (name == null) {
             throw new IllegalArgumentException("scriptField [name] must not be null: [" + name + "]");
         }
@@ -461,7 +461,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
      * @param script
      *            The script
      */
-    public TopHitsAggregatorBuilder scriptField(String name, Script script, boolean ignoreFailure) {
+    public TopHitsAggregationBuilder scriptField(String name, Script script, boolean ignoreFailure) {
         if (name == null) {
             throw new IllegalArgumentException("scriptField [name] must not be null: [" + name + "]");
         }
@@ -475,7 +475,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
         return this;
     }
 
-    public TopHitsAggregatorBuilder scriptFields(List<ScriptField> scriptFields) {
+    public TopHitsAggregationBuilder scriptFields(List<ScriptField> scriptFields) {
         if (scriptFields == null) {
             throw new IllegalArgumentException("[scriptFields] must not be null: [" + name + "]");
         }
@@ -497,7 +497,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
      * Should each {@link org.elasticsearch.search.SearchHit} be returned
      * with an explanation of the hit (ranking).
      */
-    public TopHitsAggregatorBuilder explain(boolean explain) {
+    public TopHitsAggregationBuilder explain(boolean explain) {
         this.explain = explain;
         return this;
     }
@@ -514,7 +514,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
      * Should each {@link org.elasticsearch.search.SearchHit} be returned
      * with a version associated with it.
      */
-    public TopHitsAggregatorBuilder version(boolean version) {
+    public TopHitsAggregationBuilder version(boolean version) {
         this.version = version;
         return this;
     }
@@ -531,7 +531,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
      * Applies when sorting, and controls if scores will be tracked as well.
      * Defaults to <tt>false</tt>.
      */
-    public TopHitsAggregatorBuilder trackScores(boolean trackScores) {
+    public TopHitsAggregationBuilder trackScores(boolean trackScores) {
         this.trackScores = trackScores;
         return this;
     }
@@ -544,7 +544,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
     }
 
     @Override
-    public TopHitsAggregatorBuilder subAggregations(Builder subFactories) {
+    public TopHitsAggregationBuilder subAggregations(Builder subFactories) {
         throw new AggregationInitializationException("Aggregator [" + name + "] of type [" + type + "] cannot accept sub-aggregations");
     }
 
@@ -607,8 +607,8 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
         return builder;
     }
 
-    public static TopHitsAggregatorBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
-        TopHitsAggregatorBuilder factory = new TopHitsAggregatorBuilder(aggregationName);
+    public static TopHitsAggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        TopHitsAggregationBuilder factory = new TopHitsAggregationBuilder(aggregationName);
         XContentParser.Token token;
         String currentFieldName = null;
         XContentParser parser = context.parser();
@@ -741,7 +741,7 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
 
     @Override
     protected boolean doEquals(Object obj) {
-        TopHitsAggregatorBuilder other = (TopHitsAggregatorBuilder) obj;
+        TopHitsAggregationBuilder other = (TopHitsAggregationBuilder) obj;
         return Objects.equals(explain, other.explain)
                 && Objects.equals(fetchSourceContext, other.fetchSourceContext)
                 && Objects.equals(fieldDataFields, other.fieldDataFields)
@@ -759,4 +759,4 @@ public class TopHitsAggregatorBuilder extends AggregatorBuilder<TopHitsAggregato
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregationBuilder.java
similarity index 90%
rename from core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregationBuilder.java
index bfe34f8b3da..8cc7a5b39c8 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregationBuilder.java
@@ -28,24 +28,24 @@ import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
 
-public class ValueCountAggregatorBuilder extends ValuesSourceAggregatorBuilder.LeafOnly<ValuesSource, ValueCountAggregatorBuilder> {
+public class ValueCountAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource, ValueCountAggregationBuilder> {
     public static final String NAME = InternalValueCount.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
-    public ValueCountAggregatorBuilder(String name, ValueType targetValueType) {
+    public ValueCountAggregationBuilder(String name, ValueType targetValueType) {
         super(name, InternalValueCount.TYPE, ValuesSourceType.ANY, targetValueType);
     }
 
     /**
      * Read from a stream.
      */
-    public ValueCountAggregatorBuilder(StreamInput in) throws IOException {
+    public ValueCountAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalValueCount.TYPE, ValuesSourceType.ANY);
     }
 
@@ -84,4 +84,4 @@ public class ValueCountAggregatorBuilder extends ValuesSourceAggregatorBuilder.L
     public String getWriteableName() {
         return NAME;
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountParser.java
index 2f0e7e6f263..fe8a34f242b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountParser.java
@@ -24,7 +24,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.aggregations.support.AbstractValuesSourceParser.AnyValuesSourceParser;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
@@ -46,8 +46,8 @@ public class ValueCountParser extends AnyValuesSourceParser {
     }
 
     @Override
-    protected ValuesSourceAggregatorBuilder<ValuesSource, ValueCountAggregatorBuilder> createFactory(
+    protected ValuesSourceAggregationBuilder<ValuesSource, ValueCountAggregationBuilder> createFactory(
             String aggregationName, ValuesSourceType valuesSourceType, ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        return new ValueCountAggregatorBuilder(aggregationName, targetValueType);
+        return new ValueCountAggregationBuilder(aggregationName, targetValueType);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/AbstractValuesSourceParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/AbstractValuesSourceParser.java
index 030f5143af3..51d2ea2e8c9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/AbstractValuesSourceParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/AbstractValuesSourceParser.java
@@ -84,7 +84,7 @@ public abstract class AbstractValuesSourceParser<VS extends ValuesSource>
     }
 
     @Override
-    public final ValuesSourceAggregatorBuilder<VS, ?> parse(String aggregationName, QueryParseContext context)
+    public final ValuesSourceAggregationBuilder<VS, ?> parse(String aggregationName, QueryParseContext context)
             throws IOException {
 
         XContentParser parser = context.parser();
@@ -147,7 +147,7 @@ public abstract class AbstractValuesSourceParser<VS extends ValuesSource>
             }
         }
 
-        ValuesSourceAggregatorBuilder<VS, ?> factory = createFactory(aggregationName, this.valuesSourceType, this.targetValueType,
+        ValuesSourceAggregationBuilder<VS, ?> factory = createFactory(aggregationName, this.valuesSourceType, this.targetValueType,
                 otherOptions);
         if (field != null) {
             factory.field(field);
@@ -171,7 +171,7 @@ public abstract class AbstractValuesSourceParser<VS extends ValuesSource>
     }
 
     /**
-     * Creates a {@link ValuesSourceAggregatorBuilder} from the information
+     * Creates a {@link ValuesSourceAggregationBuilder} from the information
      * gathered by the subclass. Options parsed in
      * {@link AbstractValuesSourceParser} itself will be added to the factory
      * after it has been returned by this method.
@@ -189,8 +189,8 @@ public abstract class AbstractValuesSourceParser<VS extends ValuesSource>
      *            method
      * @return the created factory
      */
-    protected abstract ValuesSourceAggregatorBuilder<VS, ?> createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-            ValueType targetValueType, Map<ParseField, Object> otherOptions);
+    protected abstract ValuesSourceAggregationBuilder<VS, ?> createFactory(String aggregationName, ValuesSourceType valuesSourceType,
+                                                                           ValueType targetValueType, Map<ParseField, Object> otherOptions);
 
     /**
      * Allows subclasses of {@link AbstractValuesSourceParser} to parse extra
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
similarity index 95%
rename from core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
index c92faa09613..78d2a2da10f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
@@ -31,7 +31,7 @@ import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregationInitializationException;
-import org.elasticsearch.search.aggregations.AggregatorBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
@@ -46,11 +46,11 @@ import java.util.Objects;
 /**
  *
  */
-public abstract class ValuesSourceAggregatorBuilder<VS extends ValuesSource, AB extends ValuesSourceAggregatorBuilder<VS, AB>>
-        extends AggregatorBuilder<AB> {
+public abstract class ValuesSourceAggregationBuilder<VS extends ValuesSource, AB extends ValuesSourceAggregationBuilder<VS, AB>>
+        extends AggregationBuilder<AB> {
 
-    public static abstract class LeafOnly<VS extends ValuesSource, AB extends ValuesSourceAggregatorBuilder<VS, AB>>
-            extends ValuesSourceAggregatorBuilder<VS, AB> {
+    public static abstract class LeafOnly<VS extends ValuesSource, AB extends ValuesSourceAggregationBuilder<VS, AB>>
+            extends ValuesSourceAggregationBuilder<VS, AB> {
 
         protected LeafOnly(String name, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType) {
             super(name, type, valuesSourceType, targetValueType);
@@ -87,7 +87,7 @@ public abstract class ValuesSourceAggregatorBuilder<VS extends ValuesSource, AB
     private DateTimeZone timeZone = null;
     protected ValuesSourceConfig<VS> config;
 
-    protected ValuesSourceAggregatorBuilder(String name, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType) {
+    protected ValuesSourceAggregationBuilder(String name, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType) {
         super(name, type);
         if (valuesSourceType == null) {
             throw new IllegalArgumentException("[valuesSourceType] must not be null: [" + name + "]");
@@ -99,7 +99,7 @@ public abstract class ValuesSourceAggregatorBuilder<VS extends ValuesSource, AB
     /**
      * Read an aggregation from a stream that does not serialize its targetValueType. This should be used by most subclasses.
      */
-    protected ValuesSourceAggregatorBuilder(StreamInput in, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType)
+    protected ValuesSourceAggregationBuilder(StreamInput in, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType)
             throws IOException {
         super(in, type);
         assert false == serializeTargetValueType() : "Wrong read constructor called for subclass that provides its targetValueType";
@@ -112,7 +112,7 @@ public abstract class ValuesSourceAggregatorBuilder<VS extends ValuesSource, AB
      * Read an aggregation from a stream that serializes its targetValueType. This should only be used by subclasses that override
      * {@link #serializeTargetValueType()} to return true.
      */
-    protected ValuesSourceAggregatorBuilder(StreamInput in, Type type, ValuesSourceType valuesSourceType) throws IOException {
+    protected ValuesSourceAggregationBuilder(StreamInput in, Type type, ValuesSourceType valuesSourceType) throws IOException {
         super(in, type);
         assert serializeTargetValueType() : "Wrong read constructor called for subclass that serializes its targetValueType";
         this.valuesSourceType = valuesSourceType;
@@ -430,7 +430,7 @@ public abstract class ValuesSourceAggregatorBuilder<VS extends ValuesSource, AB
 
     @Override
     protected final boolean doEquals(Object obj) {
-        ValuesSourceAggregatorBuilder<?, ?> other = (ValuesSourceAggregatorBuilder<?, ?>) obj;
+        ValuesSourceAggregationBuilder<?, ?> other = (ValuesSourceAggregationBuilder<?, ?>) obj;
         if (!Objects.equals(field, other.field))
             return false;
         if (!Objects.equals(format, other.format))
diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index 429aa36e56f..dc45a99fa87 100644
--- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -41,7 +41,7 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.search.aggregations.AggregatorBuilder;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
@@ -600,7 +600,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
     /**
      * Add an aggregation to perform as part of the search.
      */
-    public SearchSourceBuilder aggregation(AggregatorBuilder<?> aggregation) {
+    public SearchSourceBuilder aggregation(AggregationBuilder<?> aggregation) {
             if (aggregations == null) {
             aggregations = AggregatorFactories.builder();
             }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
index 6c221e4eb36..e986ab1288f 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
@@ -80,7 +80,7 @@ import static org.elasticsearch.cluster.service.ClusterServiceUtils.createCluste
 import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.equalTo;
 
-public abstract class BaseAggregationTestCase<AB extends AggregatorBuilder<AB>> extends ESTestCase {
+public abstract class BaseAggregationTestCase<AB extends AggregationBuilder<AB>> extends ESTestCase {
 
     protected static final String STRING_FIELD_NAME = "mapped_string";
     protected static final String INT_FIELD_NAME = "mapped_int";
@@ -238,7 +238,7 @@ public abstract class BaseAggregationTestCase<AB extends AggregatorBuilder<AB>>
         assertSame(XContentParser.Token.FIELD_NAME, parser.nextToken());
         assertEquals(testAgg.type.name(), parser.currentName());
         assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        AggregatorBuilder<?> newAgg = aggParsers.parser(testAgg.getType(), ParseFieldMatcher.STRICT).parse(testAgg.name, parseContext);
+        AggregationBuilder<?> newAgg = aggParsers.parser(testAgg.getType(), ParseFieldMatcher.STRICT).parse(testAgg.name, parseContext);
         assertSame(XContentParser.Token.END_OBJECT, parser.currentToken());
         assertSame(XContentParser.Token.END_OBJECT, parser.nextToken());
         assertSame(XContentParser.Token.END_OBJECT, parser.nextToken());
@@ -258,7 +258,7 @@ public abstract class BaseAggregationTestCase<AB extends AggregatorBuilder<AB>>
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             output.writeNamedWriteable(testAgg);
             try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
-                AggregatorBuilder<?> deserialized = in.readNamedWriteable(AggregatorBuilder.class);
+                AggregationBuilder<?> deserialized = in.readNamedWriteable(AggregationBuilder.class);
                 assertEquals(testAgg, deserialized);
                 assertEquals(testAgg.hashCode(), deserialized.hashCode());
                 assertNotSame(testAgg, deserialized);
@@ -299,7 +299,7 @@ public abstract class BaseAggregationTestCase<AB extends AggregatorBuilder<AB>>
             agg.writeTo(output);
             try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
                 @SuppressWarnings("unchecked")
-                AB secondAgg = (AB) namedWriteableRegistry.getReader(AggregatorBuilder.class, agg.getWriteableName()).read(in);
+                AB secondAgg = (AB) namedWriteableRegistry.getReader(AggregationBuilder.class, agg.getWriteableName()).read(in);
                 return secondAgg;
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenTests.java
index 3f8684c36db..c7844f29d05 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenTests.java
@@ -20,15 +20,15 @@
 package org.elasticsearch.search.aggregations.bucket;
 
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
-import org.elasticsearch.search.aggregations.bucket.children.ChildrenAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.children.ChildrenAggregationBuilder;
 
-public class ChildrenTests extends BaseAggregationTestCase<ChildrenAggregatorBuilder> {
+public class ChildrenTests extends BaseAggregationTestCase<ChildrenAggregationBuilder> {
 
     @Override
-    protected ChildrenAggregatorBuilder createTestAggregatorBuilder() {
+    protected ChildrenAggregationBuilder createTestAggregatorBuilder() {
         String name = randomAsciiOfLengthBetween(3, 20);
         String childType = randomAsciiOfLengthBetween(5, 40);
-        ChildrenAggregatorBuilder factory = new ChildrenAggregatorBuilder(name, childType);
+        ChildrenAggregationBuilder factory = new ChildrenAggregationBuilder(name, childType);
         return factory;
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramTests.java
index 58641a43b68..74ea18cc1d1 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramTests.java
@@ -20,16 +20,16 @@
 package org.elasticsearch.search.aggregations.bucket;
 
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
-import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Order;
 
-public class DateHistogramTests extends BaseAggregationTestCase<DateHistogramAggregatorBuilder> {
+public class DateHistogramTests extends BaseAggregationTestCase<DateHistogramAggregationBuilder> {
 
     @Override
-    protected DateHistogramAggregatorBuilder createTestAggregatorBuilder() {
-        DateHistogramAggregatorBuilder factory = new DateHistogramAggregatorBuilder("foo");
+    protected DateHistogramAggregationBuilder createTestAggregatorBuilder() {
+        DateHistogramAggregationBuilder factory = new DateHistogramAggregationBuilder("foo");
         factory.field(INT_FIELD_NAME);
         if (randomBoolean()) {
             factory.interval(randomIntBetween(1, 100000));
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java
index b1dc61a9b9e..a95b3cd4871 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java
@@ -27,7 +27,7 @@ import org.elasticsearch.search.aggregations.bucket.DateScriptMocks.DateScriptsM
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.range.Range;
 import org.elasticsearch.search.aggregations.bucket.range.Range.Bucket;
-import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.sum.Sum;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.hamcrest.Matchers;
@@ -117,7 +117,7 @@ public class DateRangeIT extends ESIntegTestCase {
     public void testDateMath() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("fieldname", "date");
-        DateRangeAggregatorBuilder rangeBuilder = dateRange("range");
+        DateRangeAggregationBuilder rangeBuilder = dateRange("range");
         if (randomBoolean()) {
             rangeBuilder.field("date");
         } else {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeTests.java
index 71b61c0e6e6..eba3790330b 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeTests.java
@@ -21,17 +21,17 @@ package org.elasticsearch.search.aggregations.bucket;
 
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range;
-import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeAggregationBuilder;
 import org.joda.time.DateTimeZone;
 
-public class DateRangeTests extends BaseAggregationTestCase<DateRangeAggregatorBuilder> {
+public class DateRangeTests extends BaseAggregationTestCase<DateRangeAggregationBuilder> {
 
     private final static String[] timeZoneIds = DateTimeZone.getAvailableIDs().toArray(new String[DateTimeZone.getAvailableIDs().size()]);
 
     @Override
-    protected DateRangeAggregatorBuilder createTestAggregatorBuilder() {
+    protected DateRangeAggregationBuilder createTestAggregatorBuilder() {
         int numRanges = randomIntBetween(1, 10);
-        DateRangeAggregatorBuilder factory = new DateRangeAggregatorBuilder("foo");
+        DateRangeAggregationBuilder factory = new DateRangeAggregationBuilder("foo");
         for (int i = 0; i < numRanges; i++) {
             String key = null;
             if (randomBoolean()) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerIT.java
index 6635a674b6a..96fbf17480c 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerIT.java
@@ -22,12 +22,12 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.index.query.TermQueryBuilder;
-import org.elasticsearch.search.aggregations.bucket.sampler.DiversifiedAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.sampler.DiversifiedAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.sampler.Sampler;
 import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregator;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
-import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.max.Max;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -128,7 +128,7 @@ public class DiversifiedSamplerIT extends ESIntegTestCase {
 
     public void testSimpleDiversity() throws Exception {
         int MAX_DOCS_PER_AUTHOR = 1;
-        DiversifiedAggregatorBuilder sampleAgg = new DiversifiedAggregatorBuilder("sample").shardSize(100);
+        DiversifiedAggregationBuilder sampleAgg = new DiversifiedAggregationBuilder("sample").shardSize(100);
         sampleAgg.field("author").maxDocsPerValue(MAX_DOCS_PER_AUTHOR).executionHint(randomExecutionHint());
         sampleAgg.subAggregation(terms("authors").field("author"));
         SearchResponse response = client().prepareSearch("test")
@@ -151,9 +151,9 @@ public class DiversifiedSamplerIT extends ESIntegTestCase {
     public void testNestedDiversity() throws Exception {
         // Test multiple samples gathered under buckets made by a parent agg
         int MAX_DOCS_PER_AUTHOR = 1;
-        TermsAggregatorBuilder rootTerms = terms("genres").field("genre");
+        TermsAggregationBuilder rootTerms = terms("genres").field("genre");
 
-        DiversifiedAggregatorBuilder sampleAgg = new DiversifiedAggregatorBuilder("sample").shardSize(100);
+        DiversifiedAggregationBuilder sampleAgg = new DiversifiedAggregationBuilder("sample").shardSize(100);
         sampleAgg.field("author").maxDocsPerValue(MAX_DOCS_PER_AUTHOR).executionHint(randomExecutionHint());
         sampleAgg.subAggregation(terms("authors").field("author"));
 
@@ -178,11 +178,11 @@ public class DiversifiedSamplerIT extends ESIntegTestCase {
         // Test samples nested under samples
         int MAX_DOCS_PER_AUTHOR = 1;
         int MAX_DOCS_PER_GENRE = 2;
-        DiversifiedAggregatorBuilder rootSample = new DiversifiedAggregatorBuilder("genreSample").shardSize(100)
+        DiversifiedAggregationBuilder rootSample = new DiversifiedAggregationBuilder("genreSample").shardSize(100)
                 .field("genre")
                 .maxDocsPerValue(MAX_DOCS_PER_GENRE);
 
-        DiversifiedAggregatorBuilder sampleAgg = new DiversifiedAggregatorBuilder("sample").shardSize(100);
+        DiversifiedAggregationBuilder sampleAgg = new DiversifiedAggregationBuilder("sample").shardSize(100);
         sampleAgg.field("author").maxDocsPerValue(MAX_DOCS_PER_AUTHOR).executionHint(randomExecutionHint());
         sampleAgg.subAggregation(terms("authors").field("author"));
         sampleAgg.subAggregation(terms("genres").field("genre"));
@@ -210,7 +210,7 @@ public class DiversifiedSamplerIT extends ESIntegTestCase {
     public void testPartiallyUnmappedDiversifyField() throws Exception {
         // One of the indexes is missing the "author" field used for
         // diversifying results
-        DiversifiedAggregatorBuilder sampleAgg = new DiversifiedAggregatorBuilder("sample").shardSize(100).field("author")
+        DiversifiedAggregationBuilder sampleAgg = new DiversifiedAggregationBuilder("sample").shardSize(100).field("author")
                 .maxDocsPerValue(1);
         sampleAgg.subAggregation(terms("authors").field("author"));
         SearchResponse response = client().prepareSearch("idx_unmapped_author", "test").setSearchType(SearchType.QUERY_AND_FETCH)
@@ -226,7 +226,7 @@ public class DiversifiedSamplerIT extends ESIntegTestCase {
     public void testWhollyUnmappedDiversifyField() throws Exception {
         //All of the indices are missing the "author" field used for diversifying results
         int MAX_DOCS_PER_AUTHOR = 1;
-        DiversifiedAggregatorBuilder sampleAgg = new DiversifiedAggregatorBuilder("sample").shardSize(100);
+        DiversifiedAggregationBuilder sampleAgg = new DiversifiedAggregationBuilder("sample").shardSize(100);
         sampleAgg.field("author").maxDocsPerValue(MAX_DOCS_PER_AUTHOR).executionHint(randomExecutionHint());
         sampleAgg.subAggregation(terms("authors").field("author"));
         SearchResponse response = client().prepareSearch("idx_unmapped", "idx_unmapped_author").setSearchType(SearchType.QUERY_AND_FETCH)
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerTests.java
index 512d7a8d69b..b68caad0ea7 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DiversifiedSamplerTests.java
@@ -21,14 +21,14 @@ package org.elasticsearch.search.aggregations.bucket;
 
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
-import org.elasticsearch.search.aggregations.bucket.sampler.DiversifiedAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.sampler.DiversifiedAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregator.ExecutionMode;
 
-public class DiversifiedSamplerTests extends BaseAggregationTestCase<DiversifiedAggregatorBuilder> {
+public class DiversifiedSamplerTests extends BaseAggregationTestCase<DiversifiedAggregationBuilder> {
 
     @Override
-    protected final DiversifiedAggregatorBuilder createTestAggregatorBuilder() {
-        DiversifiedAggregatorBuilder factory = new DiversifiedAggregatorBuilder("foo");
+    protected final DiversifiedAggregationBuilder createTestAggregatorBuilder() {
+        DiversifiedAggregationBuilder factory = new DiversifiedAggregationBuilder("foo");
         String field = randomNumericField();
         int randomFieldBranch = randomInt(3);
         switch (randomFieldBranch) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceRangeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceRangeTests.java
index 89dd3e3b137..65b80537c7a 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceRangeTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoDistanceRangeTests.java
@@ -23,17 +23,17 @@ import org.elasticsearch.common.geo.GeoDistance;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
-import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceParser.Range;
 import org.elasticsearch.test.geo.RandomShapeGenerator;
 
-public class GeoDistanceRangeTests extends BaseAggregationTestCase<GeoDistanceAggregatorBuilder> {
+public class GeoDistanceRangeTests extends BaseAggregationTestCase<GeoDistanceAggregationBuilder> {
 
     @Override
-    protected GeoDistanceAggregatorBuilder createTestAggregatorBuilder() {
+    protected GeoDistanceAggregationBuilder createTestAggregatorBuilder() {
         int numRanges = randomIntBetween(1, 10);
         GeoPoint origin = RandomShapeGenerator.randomPoint(random());
-        GeoDistanceAggregatorBuilder factory = new GeoDistanceAggregatorBuilder("foo", origin);
+        GeoDistanceAggregationBuilder factory = new GeoDistanceAggregationBuilder("foo", origin);
         for (int i = 0; i < numRanges; i++) {
             String key = null;
             if (randomBoolean()) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridTests.java
index 34e3e266d6a..c3c8f6902b3 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridTests.java
@@ -20,14 +20,14 @@
 package org.elasticsearch.search.aggregations.bucket;
 
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
-import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder;
 
-public class GeoHashGridTests extends BaseAggregationTestCase<GeoGridAggregatorBuilder> {
+public class GeoHashGridTests extends BaseAggregationTestCase<GeoGridAggregationBuilder> {
 
     @Override
-    protected GeoGridAggregatorBuilder createTestAggregatorBuilder() {
+    protected GeoGridAggregationBuilder createTestAggregatorBuilder() {
         String name = randomAsciiOfLengthBetween(3, 20);
-        GeoGridAggregatorBuilder factory = new GeoGridAggregatorBuilder(name);
+        GeoGridAggregationBuilder factory = new GeoGridAggregationBuilder(name);
         if (randomBoolean()) {
             int precision = randomIntBetween(1, 12);
             factory.precision(precision);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GlobalTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GlobalTests.java
index ca2b5c9d6c0..a874eff839b 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GlobalTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GlobalTests.java
@@ -20,13 +20,13 @@
 package org.elasticsearch.search.aggregations.bucket;
 
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
-import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
 
-public class GlobalTests extends BaseAggregationTestCase<GlobalAggregatorBuilder> {
+public class GlobalTests extends BaseAggregationTestCase<GlobalAggregationBuilder> {
 
     @Override
-    protected GlobalAggregatorBuilder createTestAggregatorBuilder() {
-        return new GlobalAggregatorBuilder(randomAsciiOfLengthBetween(3, 20));
+    protected GlobalAggregationBuilder createTestAggregatorBuilder() {
+        return new GlobalAggregationBuilder(randomAsciiOfLengthBetween(3, 20));
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramTests.java
index 1cd930ecc31..ac0d6d0df8b 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramTests.java
@@ -22,13 +22,13 @@ package org.elasticsearch.search.aggregations.bucket;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Order;
-import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
 
-public class HistogramTests extends BaseAggregationTestCase<HistogramAggregatorBuilder> {
+public class HistogramTests extends BaseAggregationTestCase<HistogramAggregationBuilder> {
 
     @Override
-    protected HistogramAggregatorBuilder createTestAggregatorBuilder() {
-        HistogramAggregatorBuilder factory = new HistogramAggregatorBuilder("foo");
+    protected HistogramAggregationBuilder createTestAggregatorBuilder() {
+        HistogramAggregationBuilder factory = new HistogramAggregationBuilder("foo");
         factory.field(INT_FIELD_NAME);
         factory.interval(randomIntBetween(1, 100000));
         if (randomBoolean()) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java
index 468e8503b0a..3a2abda6aa7 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java
@@ -37,7 +37,6 @@ import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
-import org.elasticsearch.search.aggregations.AggregatorBuilder;
 import org.elasticsearch.search.aggregations.bucket.range.Range;
 import org.elasticsearch.test.ESIntegTestCase;
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeTests.java
index db31f576e0c..5d86571f08f 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeTests.java
@@ -24,9 +24,9 @@ import java.net.UnknownHostException;
 
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
-import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeAggregationBuilder;
 
-public class IpRangeTests extends BaseAggregationTestCase<IpRangeAggregatorBuilder> {
+public class IpRangeTests extends BaseAggregationTestCase<IpRangeAggregationBuilder> {
 
     private static String randomIp(boolean v4) {
         try {
@@ -45,9 +45,9 @@ public class IpRangeTests extends BaseAggregationTestCase<IpRangeAggregatorBuild
     }
 
     @Override
-    protected IpRangeAggregatorBuilder createTestAggregatorBuilder() {
+    protected IpRangeAggregationBuilder createTestAggregatorBuilder() {
         int numRanges = randomIntBetween(1, 10);
-        IpRangeAggregatorBuilder factory = new IpRangeAggregatorBuilder("foo");
+        IpRangeAggregationBuilder factory = new IpRangeAggregationBuilder("foo");
         for (int i = 0; i < numRanges; i++) {
             String key = null;
             if (randomBoolean()) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NaNSortingIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NaNSortingIT.java
index 84edfbfe359..f6db12a7f6d 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NaNSortingIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/NaNSortingIT.java
@@ -27,11 +27,11 @@ import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.metrics.avg.Avg;
-import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats;
-import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStatsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStatsAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -49,8 +49,8 @@ public class NaNSortingIT extends ESIntegTestCase {
     private enum SubAggregation {
         AVG("avg") {
             @Override
-            public AvgAggregatorBuilder builder() {
-                AvgAggregatorBuilder factory = avg(name);
+            public AvgAggregationBuilder builder() {
+                AvgAggregationBuilder factory = avg(name);
                 factory.field("numeric_field");
                 return factory;
             }
@@ -61,8 +61,8 @@ public class NaNSortingIT extends ESIntegTestCase {
         },
         VARIANCE("variance") {
             @Override
-            public ExtendedStatsAggregatorBuilder builder() {
-                ExtendedStatsAggregatorBuilder factory = extendedStats(name);
+            public ExtendedStatsAggregationBuilder builder() {
+                ExtendedStatsAggregationBuilder factory = extendedStats(name);
                 factory.field("numeric_field");
                 return factory;
             }
@@ -77,8 +77,8 @@ public class NaNSortingIT extends ESIntegTestCase {
         },
         STD_DEVIATION("std_deviation"){
             @Override
-            public ExtendedStatsAggregatorBuilder builder() {
-                ExtendedStatsAggregatorBuilder factory = extendedStats(name);
+            public ExtendedStatsAggregationBuilder builder() {
+                ExtendedStatsAggregationBuilder factory = extendedStats(name);
                 factory.field("numeric_field");
                 return factory;
             }
@@ -98,7 +98,7 @@ public class NaNSortingIT extends ESIntegTestCase {
 
         public String name;
 
-        public abstract ValuesSourceAggregatorBuilder.LeafOnly<ValuesSource.Numeric, ? extends ValuesSourceAggregatorBuilder.LeafOnly<ValuesSource.Numeric, ?>> builder();
+        public abstract ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, ? extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, ?>> builder();
 
         public String sortKey() {
             return name;
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeTests.java
index b9c60dab1be..22b4eae8421 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/RangeTests.java
@@ -21,14 +21,14 @@ package org.elasticsearch.search.aggregations.bucket;
 
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range;
-import org.elasticsearch.search.aggregations.bucket.range.RangeAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
 
-public class RangeTests extends BaseAggregationTestCase<RangeAggregatorBuilder> {
+public class RangeTests extends BaseAggregationTestCase<RangeAggregationBuilder> {
 
     @Override
-    protected RangeAggregatorBuilder createTestAggregatorBuilder() {
+    protected RangeAggregationBuilder createTestAggregatorBuilder() {
         int numRanges = randomIntBetween(1, 10);
-        RangeAggregatorBuilder factory = new RangeAggregatorBuilder("foo");
+        RangeAggregationBuilder factory = new RangeAggregationBuilder("foo");
         for (int i = 0; i < numRanges; i++) {
             String key = null;
             if (randomBoolean()) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java
index f42d213bb57..980d792013a 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SamplerIT.java
@@ -24,7 +24,7 @@ import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.search.aggregations.bucket.sampler.Sampler;
 import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregator;
-import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
 import org.elasticsearch.search.aggregations.metrics.max.Max;
@@ -123,7 +123,7 @@ public class SamplerIT extends ESIntegTestCase {
     }
 
     public void testSimpleSampler() throws Exception {
-        SamplerAggregatorBuilder sampleAgg = sampler("sample").shardSize(100);
+        SamplerAggregationBuilder sampleAgg = sampler("sample").shardSize(100);
         sampleAgg.subAggregation(terms("authors").field("author"));
         SearchResponse response = client().prepareSearch("test").setSearchType(SearchType.QUERY_AND_FETCH)
                 .setQuery(new TermQueryBuilder("genre", "fantasy")).setFrom(0).setSize(60).addAggregation(sampleAgg).execute().actionGet();
@@ -140,7 +140,7 @@ public class SamplerIT extends ESIntegTestCase {
     }
 
     public void testUnmappedChildAggNoDiversity() throws Exception {
-        SamplerAggregatorBuilder sampleAgg = sampler("sample").shardSize(100);
+        SamplerAggregationBuilder sampleAgg = sampler("sample").shardSize(100);
         sampleAgg.subAggregation(terms("authors").field("author"));
         SearchResponse response = client().prepareSearch("idx_unmapped")
                 .setSearchType(SearchType.QUERY_AND_FETCH)
@@ -157,7 +157,7 @@ public class SamplerIT extends ESIntegTestCase {
     }
 
     public void testPartiallyUnmappedChildAggNoDiversity() throws Exception {
-        SamplerAggregatorBuilder sampleAgg = sampler("sample").shardSize(100);
+        SamplerAggregationBuilder sampleAgg = sampler("sample").shardSize(100);
         sampleAgg.subAggregation(terms("authors").field("author"));
         SearchResponse response = client().prepareSearch("idx_unmapped", "test")
                 .setSearchType(SearchType.QUERY_AND_FETCH)
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SamplerTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SamplerTests.java
index 8d792fd72ff..e4de490f6b2 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SamplerTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SamplerTests.java
@@ -20,13 +20,13 @@
 package org.elasticsearch.search.aggregations.bucket;
 
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
-import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.sampler.SamplerAggregationBuilder;
 
-public class SamplerTests extends BaseAggregationTestCase<SamplerAggregatorBuilder> {
+public class SamplerTests extends BaseAggregationTestCase<SamplerAggregationBuilder> {
 
     @Override
-    protected final SamplerAggregatorBuilder createTestAggregatorBuilder() {
-        SamplerAggregatorBuilder factory = new SamplerAggregatorBuilder("foo");
+    protected final SamplerAggregationBuilder createTestAggregatorBuilder() {
+        SamplerAggregationBuilder factory = new SamplerAggregationBuilder("foo");
         if (randomBoolean()) {
             factory.shardSize(randomIntBetween(1, 1000));
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsTests.java
index 373eb0e6e96..897125ee2fa 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsTests.java
@@ -24,7 +24,7 @@ import org.apache.lucene.util.automaton.RegExp;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
-import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.significant.SignificantTermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.ChiSquare;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.GND;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.JLHScore;
@@ -37,7 +37,7 @@ import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude
 import java.util.SortedSet;
 import java.util.TreeSet;
 
-public class SignificantTermsTests extends BaseAggregationTestCase<SignificantTermsAggregatorBuilder> {
+public class SignificantTermsTests extends BaseAggregationTestCase<SignificantTermsAggregationBuilder> {
 
     private static final String[] executionHints;
 
@@ -50,9 +50,9 @@ public class SignificantTermsTests extends BaseAggregationTestCase<SignificantTe
     }
 
     @Override
-    protected SignificantTermsAggregatorBuilder createTestAggregatorBuilder() {
+    protected SignificantTermsAggregationBuilder createTestAggregatorBuilder() {
         String name = randomAsciiOfLengthBetween(3, 20);
-        SignificantTermsAggregatorBuilder factory = new SignificantTermsAggregatorBuilder(name, null);
+        SignificantTermsAggregationBuilder factory = new SignificantTermsAggregationBuilder(name, null);
         String field = randomAsciiOfLengthBetween(3, 20);
         int randomFieldBranch = randomInt(2);
         switch (randomFieldBranch) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsTests.java
index ea12f6aebb6..d8600758f29 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsTests.java
@@ -25,7 +25,7 @@ import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
-import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory.ExecutionMode;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
 import java.util.ArrayList;
@@ -33,7 +33,7 @@ import java.util.List;
 import java.util.SortedSet;
 import java.util.TreeSet;
 
-public class TermsTests extends BaseAggregationTestCase<TermsAggregatorBuilder> {
+public class TermsTests extends BaseAggregationTestCase<TermsAggregationBuilder> {
 
     private static final String[] executionHints;
 
@@ -46,9 +46,9 @@ public class TermsTests extends BaseAggregationTestCase<TermsAggregatorBuilder>
     }
 
     @Override
-    protected TermsAggregatorBuilder createTestAggregatorBuilder() {
+    protected TermsAggregationBuilder createTestAggregatorBuilder() {
         String name = randomAsciiOfLengthBetween(3, 20);
-        TermsAggregatorBuilder factory = new TermsAggregatorBuilder(name, null);
+        TermsAggregationBuilder factory = new TermsAggregationBuilder(name, null);
         String field = randomAsciiOfLengthBetween(3, 20);
         int randomFieldBranch = randomInt(2);
         switch (randomFieldBranch) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
index dbe10e2ff9d..272aa70d48b 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java
@@ -123,7 +123,7 @@ public class NestedAggregatorTests extends ESSingleNodeTestCase {
         AggregationContext context = new AggregationContext(searchContext);
 
         AggregatorFactories.Builder builder = AggregatorFactories.builder();
-        NestedAggregatorBuilder factory = new NestedAggregatorBuilder("test", "nested_field");
+        NestedAggregationBuilder factory = new NestedAggregationBuilder("test", "nested_field");
         builder.addAggregator(factory);
         AggregatorFactories factories = builder.build(context, null);
         searchContext.aggregations(new SearchContextAggregations(factories));
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedTests.java
index 6ea5b3791d8..29dde100a08 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedTests.java
@@ -21,11 +21,11 @@ package org.elasticsearch.search.aggregations.bucket.nested;
 
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 
-public class NestedTests extends BaseAggregationTestCase<NestedAggregatorBuilder> {
+public class NestedTests extends BaseAggregationTestCase<NestedAggregationBuilder> {
 
     @Override
-    protected NestedAggregatorBuilder createTestAggregatorBuilder() {
-        return new NestedAggregatorBuilder(randomAsciiOfLengthBetween(1, 20), randomAsciiOfLengthBetween(3, 40));
+    protected NestedAggregationBuilder createTestAggregatorBuilder() {
+        return new NestedAggregationBuilder(randomAsciiOfLengthBetween(1, 20), randomAsciiOfLengthBetween(3, 40));
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedTests.java
index 1a45c550bc1..97dbf3718af 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedTests.java
@@ -21,11 +21,11 @@ package org.elasticsearch.search.aggregations.bucket.nested;
 
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 
-public class ReverseNestedTests extends BaseAggregationTestCase<ReverseNestedAggregatorBuilder> {
+public class ReverseNestedTests extends BaseAggregationTestCase<ReverseNestedAggregationBuilder> {
 
     @Override
-    protected ReverseNestedAggregatorBuilder createTestAggregatorBuilder() {
-        ReverseNestedAggregatorBuilder factory = new ReverseNestedAggregatorBuilder(randomAsciiOfLengthBetween(1, 20));
+    protected ReverseNestedAggregationBuilder createTestAggregatorBuilder() {
+        ReverseNestedAggregationBuilder factory = new ReverseNestedAggregationBuilder(randomAsciiOfLengthBetween(1, 20));
         if (randomBoolean()) {
             factory.path(randomAsciiOfLengthBetween(3, 40));
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java
index df449aeeaf2..8dc015b30ed 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java
@@ -257,7 +257,7 @@ public class SignificanceHeuristicTests extends ESTestCase {
 
     protected SignificanceHeuristic parseFromBuilder(ParseFieldRegistry<SignificanceHeuristicParser> significanceHeuristicParserRegistry,
             SearchContext searchContext, SignificanceHeuristic significanceHeuristic) throws IOException {
-        SignificantTermsAggregatorBuilder stBuilder = significantTerms("testagg");
+        SignificantTermsAggregationBuilder stBuilder = significantTerms("testagg");
         stBuilder.significanceHeuristic(significanceHeuristic).field("text").minDocCount(200);
         XContentBuilder stXContentBuilder = XContentFactory.jsonBuilder();
         stBuilder.internalXContent(stXContentBuilder, null);
@@ -271,7 +271,7 @@ public class SignificanceHeuristicTests extends ESTestCase {
         IndicesQueriesRegistry registry = new IndicesQueriesRegistry();
         QueryParseContext parseContext = new QueryParseContext(registry, stParser, ParseFieldMatcher.STRICT);
         stParser.nextToken();
-        SignificantTermsAggregatorBuilder aggregatorFactory = (SignificantTermsAggregatorBuilder) new SignificantTermsParser(
+        SignificantTermsAggregationBuilder aggregatorFactory = (SignificantTermsAggregationBuilder) new SignificantTermsParser(
                 significanceHeuristicParserRegistry, registry).parse("testagg", parseContext);
         stParser.nextToken();
         assertThat(aggregatorFactory.getBucketCountThresholds().getMinDocCount(), equalTo(200L));
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericMetricTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericMetricTestCase.java
index 58d7fa70d62..f1ccf344a7c 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericMetricTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractNumericMetricTestCase.java
@@ -22,9 +22,9 @@ package org.elasticsearch.search.aggregations.metrics;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 
-public abstract class AbstractNumericMetricTestCase<AF extends ValuesSourceAggregatorBuilder.LeafOnly<ValuesSource.Numeric, AF>>
+public abstract class AbstractNumericMetricTestCase<AF extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, AF>>
         extends BaseAggregationTestCase<AF> {
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgTests.java
index 61e685169f6..df90dc4f7c3 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgTests.java
@@ -19,13 +19,13 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
 
-public class AvgTests extends AbstractNumericMetricTestCase<AvgAggregatorBuilder> {
+public class AvgTests extends AbstractNumericMetricTestCase<AvgAggregationBuilder> {
 
     @Override
-    protected AvgAggregatorBuilder doCreateTestAggregatorFactory() {
-        return new AvgAggregatorBuilder("foo");
+    protected AvgAggregationBuilder doCreateTestAggregatorFactory() {
+        return new AvgAggregationBuilder("foo");
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsTests.java
index 4a7ca7e8b38..3f78cc17aa9 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsTests.java
@@ -19,13 +19,13 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStatsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStatsAggregationBuilder;
 
-public class ExtendedStatsTests extends AbstractNumericMetricTestCase<ExtendedStatsAggregatorBuilder> {
+public class ExtendedStatsTests extends AbstractNumericMetricTestCase<ExtendedStatsAggregationBuilder> {
 
     @Override
-    protected ExtendedStatsAggregatorBuilder doCreateTestAggregatorFactory() {
-        ExtendedStatsAggregatorBuilder factory = new ExtendedStatsAggregatorBuilder("foo");
+    protected ExtendedStatsAggregationBuilder doCreateTestAggregatorFactory() {
+        ExtendedStatsAggregationBuilder factory = new ExtendedStatsAggregationBuilder("foo");
         if (randomBoolean()) {
             factory.sigma(randomDoubleBetween(0.0, 10.0, true));
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FilterTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FilterTests.java
index 8a6a4373691..1b563d531a8 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FilterTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FilterTests.java
@@ -21,13 +21,13 @@ package org.elasticsearch.search.aggregations.metrics;
 
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
-import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
 
-public class FilterTests extends BaseAggregationTestCase<FilterAggregatorBuilder> {
+public class FilterTests extends BaseAggregationTestCase<FilterAggregationBuilder> {
 
     @Override
-    protected FilterAggregatorBuilder createTestAggregatorBuilder() {
-        FilterAggregatorBuilder factory = new FilterAggregatorBuilder(randomAsciiOfLengthBetween(1, 20),
+    protected FilterAggregationBuilder createTestAggregatorBuilder() {
+        FilterAggregationBuilder factory = new FilterAggregationBuilder(randomAsciiOfLengthBetween(1, 20),
                 QueryBuilders.termQuery(randomAsciiOfLengthBetween(5, 20), randomAsciiOfLengthBetween(5, 20)));
         // NORELEASE make RandomQueryBuilder work outside of the
         // AbstractQueryTestCase
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FiltersTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FiltersTests.java
index cd2dae53327..89fc38b7cd8 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FiltersTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/FiltersTests.java
@@ -24,15 +24,15 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregator.KeyedFilter;
-import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregationBuilder;
 
-public class FiltersTests extends BaseAggregationTestCase<FiltersAggregatorBuilder> {
+public class FiltersTests extends BaseAggregationTestCase<FiltersAggregationBuilder> {
 
     @Override
-    protected FiltersAggregatorBuilder createTestAggregatorBuilder() {
+    protected FiltersAggregationBuilder createTestAggregatorBuilder() {
 
         int size = randomIntBetween(1, 20);
-        FiltersAggregatorBuilder factory;
+        FiltersAggregationBuilder factory;
         if (randomBoolean()) {
             KeyedFilter[] filters = new KeyedFilter[size];
             int i = 0;
@@ -40,13 +40,13 @@ public class FiltersTests extends BaseAggregationTestCase<FiltersAggregatorBuild
                 filters[i++] = new KeyedFilter(key,
                         QueryBuilders.termQuery(randomAsciiOfLengthBetween(5, 20), randomAsciiOfLengthBetween(5, 20)));
             }
-            factory = new FiltersAggregatorBuilder(randomAsciiOfLengthBetween(1, 20), filters);
+            factory = new FiltersAggregationBuilder(randomAsciiOfLengthBetween(1, 20), filters);
         } else {
             QueryBuilder[] filters = new QueryBuilder[size];
             for (int i = 0; i < size; i++) {
                 filters[i] = QueryBuilders.termQuery(randomAsciiOfLengthBetween(5, 20), randomAsciiOfLengthBetween(5, 20));
             }
-            factory = new FiltersAggregatorBuilder(randomAsciiOfLengthBetween(1, 20), filters);
+            factory = new FiltersAggregationBuilder(randomAsciiOfLengthBetween(1, 20), filters);
         }
         if (randomBoolean()) {
             factory.otherBucket(randomBoolean());
@@ -64,8 +64,8 @@ public class FiltersTests extends BaseAggregationTestCase<FiltersAggregatorBuild
     public void testFiltersSortedByKey() {
         KeyedFilter[] original = new KeyedFilter[]{new KeyedFilter("bbb", new MatchNoneQueryBuilder()),
                 new KeyedFilter("aaa", new MatchNoneQueryBuilder())};
-        FiltersAggregatorBuilder builder;
-        builder = new FiltersAggregatorBuilder("my-agg", original);
+        FiltersAggregationBuilder builder;
+        builder = new FiltersAggregationBuilder("my-agg", original);
         assertEquals("aaa", builder.filters().get(0).key());
         assertEquals("bbb", builder.filters().get(1).key());
         // original should be unchanged
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsTests.java
index 9bbd28c412d..e086598dbfe 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsTests.java
@@ -20,13 +20,13 @@
 package org.elasticsearch.search.aggregations.metrics;
 
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
-import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBoundsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBoundsAggregationBuilder;
 
-public class GeoBoundsTests extends BaseAggregationTestCase<GeoBoundsAggregatorBuilder> {
+public class GeoBoundsTests extends BaseAggregationTestCase<GeoBoundsAggregationBuilder> {
 
     @Override
-    protected GeoBoundsAggregatorBuilder createTestAggregatorBuilder() {
-        GeoBoundsAggregatorBuilder factory = new GeoBoundsAggregatorBuilder(randomAsciiOfLengthBetween(1, 20));
+    protected GeoBoundsAggregationBuilder createTestAggregatorBuilder() {
+        GeoBoundsAggregationBuilder factory = new GeoBoundsAggregationBuilder(randomAsciiOfLengthBetween(1, 20));
         String field = randomAsciiOfLengthBetween(3, 20);
         factory.field(field);
         if (randomBoolean()) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidTests.java
index c912c18a82e..1ea21a1ff1d 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidTests.java
@@ -21,13 +21,13 @@ package org.elasticsearch.search.aggregations.metrics;
 
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
-import org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroidAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroidAggregationBuilder;
 
-public class GeoCentroidTests extends BaseAggregationTestCase<GeoCentroidAggregatorBuilder> {
+public class GeoCentroidTests extends BaseAggregationTestCase<GeoCentroidAggregationBuilder> {
 
     @Override
-    protected GeoCentroidAggregatorBuilder createTestAggregatorBuilder() {
-        GeoCentroidAggregatorBuilder factory = new GeoCentroidAggregatorBuilder(randomAsciiOfLengthBetween(1, 20));
+    protected GeoCentroidAggregationBuilder createTestAggregatorBuilder() {
+        GeoCentroidAggregationBuilder factory = new GeoCentroidAggregationBuilder(randomAsciiOfLengthBetween(1, 20));
         String field = randomNumericField();
         int randomFieldBranch = randomInt(3);
         switch (randomFieldBranch) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxTests.java
index a9fe4654c9d..6ffd824aa3c 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxTests.java
@@ -19,13 +19,13 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.search.aggregations.metrics.max.MaxAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
 
-public class MaxTests extends AbstractNumericMetricTestCase<MaxAggregatorBuilder> {
+public class MaxTests extends AbstractNumericMetricTestCase<MaxAggregationBuilder> {
 
     @Override
-    protected MaxAggregatorBuilder doCreateTestAggregatorFactory() {
-        return new MaxAggregatorBuilder("foo");
+    protected MaxAggregationBuilder doCreateTestAggregatorFactory() {
+        return new MaxAggregationBuilder("foo");
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MinTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MinTests.java
index 54512f579f3..eed4059ade7 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MinTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MinTests.java
@@ -19,13 +19,13 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.search.aggregations.metrics.min.MinAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
 
-public class MinTests extends AbstractNumericMetricTestCase<MinAggregatorBuilder> {
+public class MinTests extends AbstractNumericMetricTestCase<MinAggregationBuilder> {
 
     @Override
-    protected MinAggregatorBuilder doCreateTestAggregatorFactory() {
-        return new MinAggregatorBuilder("foo");
+    protected MinAggregationBuilder doCreateTestAggregatorFactory() {
+        return new MinAggregationBuilder("foo");
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MissingTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MissingTests.java
index 3f49da5eb6e..979747ade2e 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MissingTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/MissingTests.java
@@ -21,13 +21,13 @@ package org.elasticsearch.search.aggregations.metrics;
 
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
-import org.elasticsearch.search.aggregations.bucket.missing.MissingAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.missing.MissingAggregationBuilder;
 
-public class MissingTests extends BaseAggregationTestCase<MissingAggregatorBuilder> {
+public class MissingTests extends BaseAggregationTestCase<MissingAggregationBuilder> {
 
     @Override
-    protected final MissingAggregatorBuilder createTestAggregatorBuilder() {
-        MissingAggregatorBuilder factory = new MissingAggregatorBuilder("foo", null);
+    protected final MissingAggregationBuilder createTestAggregatorBuilder() {
+        MissingAggregationBuilder factory = new MissingAggregationBuilder("foo", null);
         String field = randomNumericField();
         int randomFieldBranch = randomInt(3);
         switch (randomFieldBranch) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksTests.java
index 4636e4ed174..1907733fbd8 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/PercentileRanksTests.java
@@ -21,13 +21,13 @@ package org.elasticsearch.search.aggregations.metrics;
 
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
-import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanksAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanksAggregationBuilder;
 
-public class PercentileRanksTests extends BaseAggregationTestCase<PercentileRanksAggregatorBuilder> {
+public class PercentileRanksTests extends BaseAggregationTestCase<PercentileRanksAggregationBuilder> {
 
     @Override
-    protected PercentileRanksAggregatorBuilder createTestAggregatorBuilder() {
-        PercentileRanksAggregatorBuilder factory = new PercentileRanksAggregatorBuilder(randomAsciiOfLengthBetween(1, 20));
+    protected PercentileRanksAggregationBuilder createTestAggregatorBuilder() {
+        PercentileRanksAggregationBuilder factory = new PercentileRanksAggregationBuilder(randomAsciiOfLengthBetween(1, 20));
         if (randomBoolean()) {
             factory.keyed(randomBoolean());
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/PercentilesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/PercentilesTests.java
index 674197cffbf..b5539f8c1be 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/PercentilesTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/PercentilesTests.java
@@ -21,13 +21,13 @@ package org.elasticsearch.search.aggregations.metrics;
 
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
-import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesAggregationBuilder;
 
-public class PercentilesTests extends BaseAggregationTestCase<PercentilesAggregatorBuilder> {
+public class PercentilesTests extends BaseAggregationTestCase<PercentilesAggregationBuilder> {
 
     @Override
-    protected PercentilesAggregatorBuilder createTestAggregatorBuilder() {
-        PercentilesAggregatorBuilder factory = new PercentilesAggregatorBuilder(randomAsciiOfLengthBetween(1, 20));
+    protected PercentilesAggregationBuilder createTestAggregatorBuilder() {
+        PercentilesAggregationBuilder factory = new PercentilesAggregationBuilder(randomAsciiOfLengthBetween(1, 20));
         if (randomBoolean()) {
             factory.keyed(randomBoolean());
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricTests.java
index a4e12b56d8e..e4f96fae762 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricTests.java
@@ -22,16 +22,16 @@ package org.elasticsearch.search.aggregations.metrics;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
-import org.elasticsearch.search.aggregations.metrics.scripted.ScriptedMetricAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.scripted.ScriptedMetricAggregationBuilder;
 
 import java.util.HashMap;
 import java.util.Map;
 
-public class ScriptedMetricTests extends BaseAggregationTestCase<ScriptedMetricAggregatorBuilder> {
+public class ScriptedMetricTests extends BaseAggregationTestCase<ScriptedMetricAggregationBuilder> {
 
     @Override
-    protected ScriptedMetricAggregatorBuilder createTestAggregatorBuilder() {
-        ScriptedMetricAggregatorBuilder factory = new ScriptedMetricAggregatorBuilder(randomAsciiOfLengthBetween(1, 20));
+    protected ScriptedMetricAggregationBuilder createTestAggregatorBuilder() {
+        ScriptedMetricAggregationBuilder factory = new ScriptedMetricAggregationBuilder(randomAsciiOfLengthBetween(1, 20));
         if (randomBoolean()) {
             factory.initScript(randomScript("initScript"));
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsTests.java
index 5db4e1e332b..76a8e9aa98a 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsTests.java
@@ -19,13 +19,13 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.search.aggregations.metrics.stats.StatsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.stats.StatsAggregationBuilder;
 
-public class StatsTests extends AbstractNumericMetricTestCase<StatsAggregatorBuilder> {
+public class StatsTests extends AbstractNumericMetricTestCase<StatsAggregationBuilder> {
 
     @Override
-    protected StatsAggregatorBuilder doCreateTestAggregatorFactory() {
-        return new StatsAggregatorBuilder("foo");
+    protected StatsAggregationBuilder doCreateTestAggregatorFactory() {
+        return new StatsAggregationBuilder("foo");
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumTests.java
index a6d9f0bd270..edc6d4edef0 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumTests.java
@@ -19,13 +19,13 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
-import org.elasticsearch.search.aggregations.metrics.sum.SumAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder;
 
-public class SumTests extends AbstractNumericMetricTestCase<SumAggregatorBuilder> {
+public class SumTests extends AbstractNumericMetricTestCase<SumAggregationBuilder> {
 
     @Override
-    protected SumAggregatorBuilder doCreateTestAggregatorFactory() {
-        return new SumAggregatorBuilder("foo");
+    protected SumAggregationBuilder doCreateTestAggregatorFactory() {
+        return new SumAggregationBuilder("foo");
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java
index 00bb04dde9f..c79ab04e492 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java
@@ -26,7 +26,7 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.AggregationInitializationException;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
-import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationBuilder;
 import org.elasticsearch.search.fetch.source.FetchSourceContext;
 import org.elasticsearch.search.highlight.HighlightBuilderTests;
 import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
@@ -38,11 +38,11 @@ import java.util.List;
 
 import static org.hamcrest.Matchers.containsString;
 
-public class TopHitsTests extends BaseAggregationTestCase<TopHitsAggregatorBuilder> {
+public class TopHitsTests extends BaseAggregationTestCase<TopHitsAggregationBuilder> {
 
     @Override
-    protected final TopHitsAggregatorBuilder createTestAggregatorBuilder() {
-        TopHitsAggregatorBuilder factory = new TopHitsAggregatorBuilder("foo");
+    protected final TopHitsAggregationBuilder createTestAggregatorBuilder() {
+        TopHitsAggregationBuilder factory = new TopHitsAggregationBuilder("foo");
         if (randomBoolean()) {
             factory.from(randomIntBetween(0, 10000));
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountTests.java
index c9b601c4e8b..99d4d41839c 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountTests.java
@@ -21,13 +21,13 @@ package org.elasticsearch.search.aggregations.metrics;
 
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
-import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder;
 
-public class ValueCountTests extends BaseAggregationTestCase<ValueCountAggregatorBuilder> {
+public class ValueCountTests extends BaseAggregationTestCase<ValueCountAggregationBuilder> {
 
     @Override
-    protected final ValueCountAggregatorBuilder createTestAggregatorBuilder() {
-        ValueCountAggregatorBuilder factory = new ValueCountAggregatorBuilder("foo", null);
+    protected final ValueCountAggregationBuilder createTestAggregatorBuilder() {
+        ValueCountAggregationBuilder factory = new ValueCountAggregationBuilder("foo", null);
         String field = randomNumericField();
         int randomFieldBranch = randomInt(3);
         switch (randomFieldBranch) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityTests.java
index a769a71b8e5..ab0377c6331 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityTests.java
@@ -22,11 +22,11 @@ package org.elasticsearch.search.aggregations.metrics.cardinality;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 
-public class CardinalityTests extends BaseAggregationTestCase<CardinalityAggregatorBuilder> {
+public class CardinalityTests extends BaseAggregationTestCase<CardinalityAggregationBuilder> {
 
     @Override
-    protected final CardinalityAggregatorBuilder createTestAggregatorBuilder() {
-        CardinalityAggregatorBuilder factory = new CardinalityAggregatorBuilder("foo", null);
+    protected final CardinalityAggregationBuilder createTestAggregatorBuilder() {
+        CardinalityAggregationBuilder factory = new CardinalityAggregationBuilder("foo", null);
         String field = randomNumericField();
         int randomFieldBranch = randomInt(3);
         switch (randomFieldBranch) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregationHelperTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregationHelperTests.java
index 43b3b4d357c..ce9394692de 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregationHelperTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregationHelperTests.java
@@ -20,11 +20,11 @@
 package org.elasticsearch.search.aggregations.pipeline;
 
 
-import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregatorBuilder;
-import org.elasticsearch.search.aggregations.metrics.max.MaxAggregatorBuilder;
-import org.elasticsearch.search.aggregations.metrics.min.MinAggregatorBuilder;
-import org.elasticsearch.search.aggregations.metrics.sum.SumAggregatorBuilder;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
@@ -109,27 +109,27 @@ public class PipelineAggregationHelperTests extends ESTestCase {
      * @param values Array of values to compute metric for
      * @param metric A metric builder which defines what kind of metric should be returned for the values
      */
-    public static double calculateMetric(double[] values, ValuesSourceAggregatorBuilder<?, ?> metric) {
+    public static double calculateMetric(double[] values, ValuesSourceAggregationBuilder<?, ?> metric) {
 
-        if (metric instanceof MinAggregatorBuilder) {
+        if (metric instanceof MinAggregationBuilder) {
             double accumulator = Double.POSITIVE_INFINITY;
             for (double value : values) {
                 accumulator = Math.min(accumulator, value);
             }
             return accumulator;
-        } else if (metric instanceof MaxAggregatorBuilder) {
+        } else if (metric instanceof MaxAggregationBuilder) {
             double accumulator = Double.NEGATIVE_INFINITY;
             for (double value : values) {
                 accumulator = Math.max(accumulator, value);
             }
             return accumulator;
-        } else if (metric instanceof SumAggregatorBuilder) {
+        } else if (metric instanceof SumAggregationBuilder) {
             double accumulator = 0;
             for (double value : values) {
                 accumulator += value;
             }
             return accumulator;
-        } else if (metric instanceof AvgAggregatorBuilder) {
+        } else if (metric instanceof AvgAggregationBuilder) {
             double accumulator = 0;
             for (double value : values) {
                 accumulator += value;
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgIT.java
index 94ac6fc10ab..c16d8e8062e 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgIT.java
@@ -39,7 +39,7 @@ import org.elasticsearch.search.aggregations.pipeline.movavg.models.LinearModel;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.MovAvgModelBuilder;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.SimpleModel;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.hamcrest.Matchers;
 
@@ -79,7 +79,7 @@ public class MovAvgIT extends ESIntegTestCase {
     static int period;
     static HoltWintersModel.SeasonalityType seasonalityType;
     static BucketHelpers.GapPolicy gapPolicy;
-    static ValuesSourceAggregatorBuilder<? extends ValuesSource, ? extends ValuesSourceAggregatorBuilder<?, ?>> metric;
+    static ValuesSourceAggregationBuilder<? extends ValuesSource, ? extends ValuesSourceAggregationBuilder<?, ?>> metric;
     static List<PipelineAggregationHelperTests.MockBucket> mockHisto;
 
     static Map<String, ArrayList<Double>> testValues;
@@ -1289,8 +1289,8 @@ public class MovAvgIT extends ESIntegTestCase {
         }
     }
 
-    private ValuesSourceAggregatorBuilder<? extends ValuesSource, ? extends ValuesSourceAggregatorBuilder<?, ?>> randomMetric(String name,
-            String field) {
+    private ValuesSourceAggregationBuilder<? extends ValuesSource, ? extends ValuesSourceAggregationBuilder<?, ?>> randomMetric(String name,
+                                                                                                                                String field) {
         int rand = randomIntBetween(0,3);
 
         switch (rand) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffIT.java
index 66961c2fcbc..1b263d1af09 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffIT.java
@@ -29,7 +29,7 @@ import org.elasticsearch.search.aggregations.pipeline.BucketHelpers;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregationHelperTests;
 import org.elasticsearch.search.aggregations.pipeline.SimpleValue;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
-import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.hamcrest.Matchers;
 
@@ -61,7 +61,7 @@ public class SerialDiffIT extends ESIntegTestCase {
     static int numBuckets;
     static int lag;
     static BucketHelpers.GapPolicy gapPolicy;
-    static ValuesSourceAggregatorBuilder<? extends ValuesSource, ? extends ValuesSourceAggregatorBuilder<?, ?>> metric;
+    static ValuesSourceAggregationBuilder<? extends ValuesSource, ? extends ValuesSourceAggregationBuilder<?, ?>> metric;
     static List<PipelineAggregationHelperTests.MockBucket> mockHisto;
 
     static Map<String, ArrayList<Double>> testValues;
@@ -81,7 +81,7 @@ public class SerialDiffIT extends ESIntegTestCase {
         }
     }
 
-    private ValuesSourceAggregatorBuilder<? extends ValuesSource, ? extends ValuesSourceAggregatorBuilder<?, ?>> randomMetric(String name, String field) {
+    private ValuesSourceAggregationBuilder<? extends ValuesSource, ? extends ValuesSourceAggregationBuilder<?, ?>> randomMetric(String name, String field) {
         int rand = randomIntBetween(0,3);
 
         switch (rand) {
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java
index 6f83746d4ce..deed43abf12 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java
@@ -36,7 +36,7 @@ import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.range.Range;
 import org.elasticsearch.search.aggregations.bucket.range.Range.Bucket;
-import org.elasticsearch.search.aggregations.bucket.range.RangeAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorFactory;
 import org.elasticsearch.search.aggregations.metrics.sum.Sum;
@@ -122,7 +122,7 @@ public class EquivalenceTests extends ESIntegTestCase {
             }
         }
 
-        RangeAggregatorBuilder query = range("range").field("values");
+        RangeAggregationBuilder query = range("range").field("values");
         for (int i = 0; i < ranges.length; ++i) {
             String key = Integer.toString(i);
             if (ranges[i][0] == Double.NEGATIVE_INFINITY) {
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java
index 1c17c1966e5..662d4d2f30c 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java
@@ -35,7 +35,7 @@ import org.elasticsearch.search.aggregations.bucket.AbstractTermsTestCase;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
-import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.joda.time.DateTime;
@@ -113,17 +113,17 @@ public class MinDocCountTests extends AbstractTermsTestCase {
     private enum Script {
         NO {
             @Override
-            TermsAggregatorBuilder apply(TermsAggregatorBuilder builder, String field) {
+            TermsAggregationBuilder apply(TermsAggregationBuilder builder, String field) {
                 return builder.field(field);
             }
         },
         YES {
             @Override
-            TermsAggregatorBuilder apply(TermsAggregatorBuilder builder, String field) {
+            TermsAggregationBuilder apply(TermsAggregationBuilder builder, String field) {
                 return builder.script(new org.elasticsearch.script.Script("doc['" + field + "'].values"));
             }
         };
-        abstract TermsAggregatorBuilder apply(TermsAggregatorBuilder builder, String field);
+        abstract TermsAggregationBuilder apply(TermsAggregationBuilder builder, String field);
     }
 
     // check that terms2 is a subset of terms1
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentileRanksTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentileRanksTests.java
index 2e59b798297..7e76b3f03eb 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentileRanksTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentileRanksTests.java
@@ -33,7 +33,7 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.metrics.AbstractNumericTestCase;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
 import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanks;
-import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanksAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanksAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesMethod;
 
 import java.util.Arrays;
@@ -88,7 +88,7 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase {
         return percents;
     }
 
-    private static PercentileRanksAggregatorBuilder randomCompression(PercentileRanksAggregatorBuilder builder) {
+    private static PercentileRanksAggregationBuilder randomCompression(PercentileRanksAggregationBuilder builder) {
         if (randomBoolean()) {
             builder.compression(randomIntBetween(20, 120) + randomDouble());
         }
@@ -462,4 +462,4 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase {
         }
     }
 
-}
\ No newline at end of file
+}
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentilesTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentilesTests.java
index 69d3c281ca8..712c9ebd951 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentilesTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentilesTests.java
@@ -33,7 +33,7 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.metrics.AbstractNumericTestCase;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles;
-import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesAggregatorBuilder;
+import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesMethod;
 
 import java.util.Arrays;
@@ -87,7 +87,7 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase {
         return percentiles;
     }
 
-    private static PercentilesAggregatorBuilder randomCompression(PercentilesAggregatorBuilder builder) {
+    private static PercentilesAggregationBuilder randomCompression(PercentilesAggregationBuilder builder) {
         if (randomBoolean()) {
             builder.compression(randomIntBetween(20, 120) + randomDouble());
         }
@@ -446,4 +446,4 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase {
 
         }
     }
-}
\ No newline at end of file
+}

From 915ab16176b95524968b07a2faa5b217a03cfcaa Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Thu, 19 May 2016 09:29:48 -0400
Subject: [PATCH 0347/1311] First steps at whitelist cleanup

---
 .../painless/AnalyzerCaster.java              |   96 +-
 .../elasticsearch/painless/Definition.java    | 1267 +++--------------
 .../elasticsearch/painless/MethodWriter.java  |  166 +--
 .../painless/PainlessPlugin.java              |   11 +
 .../painless/PainlessScriptEngineService.java |    2 -
 .../org/elasticsearch/painless/Variables.java |   18 +-
 .../elasticsearch/painless/node/EBinary.java  |    6 +-
 .../elasticsearch/painless/node/EBool.java    |    6 +-
 .../elasticsearch/painless/node/EBoolean.java |    2 +-
 .../elasticsearch/painless/node/EChain.java   |    6 +-
 .../elasticsearch/painless/node/EComp.java    |   32 +-
 .../painless/node/EConditional.java           |    2 +-
 .../painless/node/EConstant.java              |   18 +-
 .../elasticsearch/painless/node/EDecimal.java |    4 +-
 .../elasticsearch/painless/node/ENull.java    |    2 +-
 .../elasticsearch/painless/node/ENumeric.java |   14 +-
 .../elasticsearch/painless/node/EUnary.java   |   12 +-
 .../painless/node/LArrayLength.java           |    2 +-
 .../elasticsearch/painless/node/LBrace.java   |    2 +-
 .../painless/node/LDefArray.java              |    6 +-
 .../elasticsearch/painless/node/LDefCall.java |    4 +-
 .../painless/node/LDefField.java              |    6 +-
 .../painless/node/LListShortcut.java          |    2 +-
 .../painless/node/LNewArray.java              |    2 +-
 .../elasticsearch/painless/node/LString.java  |    2 +-
 .../org/elasticsearch/painless/node/SDo.java  |    2 +-
 .../painless/node/SExpression.java            |    2 +-
 .../org/elasticsearch/painless/node/SFor.java |    2 +-
 .../elasticsearch/painless/node/SIfElse.java  |    2 +-
 .../elasticsearch/painless/node/SReturn.java  |    2 +-
 .../elasticsearch/painless/node/SThrow.java   |    2 +-
 .../elasticsearch/painless/node/SWhile.java   |    2 +-
 .../org/elasticsearch/painless/definition.txt |  529 +++++++
 .../elasticsearch/painless/BasicAPITests.java |   14 +-
 .../painless/BasicStatementTests.java         |    8 +-
 .../painless/NoSemiColonTests.java            |    8 +-
 36 files changed, 997 insertions(+), 1266 deletions(-)
 create mode 100644 modules/lang-painless/src/main/resources/org/elasticsearch/painless/definition.txt

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
index 3228ff47e92..d6f099e47f9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
@@ -119,18 +119,18 @@ public final class AnalyzerCaster {
         final Sort sort = from.sort;
 
         if (sort == Sort.DEF) {
-            return definition.defType;
+            return definition.getType("def");
         } else if ((sort == Sort.DOUBLE || sort == Sort.DOUBLE_OBJ) && decimal) {
-            return primitive ? definition.doubleType : definition.doubleobjType;
+            return primitive ? definition.getType("double") : definition.getType("Double");
         } else if ((sort == Sort.FLOAT || sort == Sort.FLOAT_OBJ) && decimal) {
-            return primitive ? definition.floatType : definition.floatobjType;
+            return primitive ? definition.getType("float") : definition.getType("Float");
         } else if (sort == Sort.LONG || sort == Sort.LONG_OBJ) {
-            return primitive ? definition.longType : definition.longobjType;
+            return primitive ? definition.getType("long") : definition.getType("Long");
         } else if (sort == Sort.INT   || sort == Sort.INT_OBJ   ||
                    sort == Sort.CHAR  || sort == Sort.CHAR_OBJ  ||
                    sort == Sort.SHORT || sort == Sort.SHORT_OBJ ||
                    sort == Sort.BYTE  || sort == Sort.BYTE_OBJ) {
-            return primitive ? definition.intType : definition.intobjType;
+            return primitive ? definition.getType("int") : definition.getType("Integer");
         }
 
         return null;
@@ -142,21 +142,21 @@ public final class AnalyzerCaster {
         final Sort sort1 = from1.sort;
 
         if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return definition.defType;
+            return definition.getType("def");
         }
 
         if (decimal) {
             if (sort0 == Sort.DOUBLE || sort0 == Sort.DOUBLE_OBJ ||
                 sort1 == Sort.DOUBLE || sort1 == Sort.DOUBLE_OBJ) {
-                return primitive ? definition.doubleType : definition.doubleobjType;
+                return primitive ? definition.getType("double") : definition.getType("Double");
             } else if (sort0 == Sort.FLOAT || sort0 == Sort.FLOAT_OBJ || sort1 == Sort.FLOAT || sort1 == Sort.FLOAT_OBJ) {
-                return primitive ? definition.floatType : definition.floatobjType;
+                return primitive ? definition.getType("float") : definition.getType("Float");
             }
         }
 
         if (sort0 == Sort.LONG || sort0 == Sort.LONG_OBJ ||
             sort1 == Sort.LONG || sort1 == Sort.LONG_OBJ) {
-            return primitive ? definition.longType : definition.longobjType;
+            return primitive ? definition.getType("long") : definition.getType("Long");
         } else if (sort0 == Sort.INT   || sort0 == Sort.INT_OBJ   ||
                    sort1 == Sort.INT   || sort1 == Sort.INT_OBJ   ||
                    sort0 == Sort.CHAR  || sort0 == Sort.CHAR_OBJ  ||
@@ -165,7 +165,7 @@ public final class AnalyzerCaster {
                    sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ ||
                    sort0 == Sort.BYTE  || sort0 == Sort.BYTE_OBJ  ||
                    sort1 == Sort.BYTE  || sort1 == Sort.BYTE_OBJ) {
-            return primitive ? definition.intType : definition.intobjType;
+            return primitive ? definition.getType("int") : definition.getType("Integer");
         }
 
         return null;
@@ -176,7 +176,7 @@ public final class AnalyzerCaster {
         final Sort sort1 = from1.sort;
 
         if (sort0 == Sort.STRING || sort1 == Sort.STRING) {
-            return definition.stringType;
+            return definition.getType("String");
         }
 
         return promoteNumeric(definition, from0, from1, true, true);
@@ -187,7 +187,7 @@ public final class AnalyzerCaster {
         final Sort sort1 = from1.sort;
 
         if (sort0.bool || sort1.bool) {
-            return definition.booleanType;
+            return definition.getType("boolean");
         }
 
         return promoteNumeric(definition, from0, from1, false, true);
@@ -198,20 +198,20 @@ public final class AnalyzerCaster {
         final Sort sort1 = from1.sort;
 
         if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return definition.defType;
+            return definition.getType("def");
         }
 
         final boolean primitive = sort0.primitive && sort1.primitive;
 
         if (sort0.bool && sort1.bool) {
-            return primitive ? definition.booleanType : definition.booleanobjType;
+            return primitive ? definition.getType("boolean") : definition.getType("Boolean");
         }
 
         if (sort0.numeric && sort1.numeric) {
             return promoteNumeric(definition, from0, from1, true, primitive);
         }
 
-        return definition.objectType;
+        return definition.getType("Object");
     }
 
     public static Type promoteReference(final Definition definition, final Type from0, final Type from1) {
@@ -219,12 +219,12 @@ public final class AnalyzerCaster {
         final Sort sort1 = from1.sort;
 
         if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return definition.defType;
+            return definition.getType("def");
         }
 
         if (sort0.primitive && sort1.primitive) {
             if (sort0.bool && sort1.bool) {
-                return definition.booleanType;
+                return definition.getType("boolean");
             }
 
             if (sort0.numeric && sort1.numeric) {
@@ -232,7 +232,7 @@ public final class AnalyzerCaster {
             }
         }
 
-        return definition.objectType;
+        return definition.getType("Object");
     }
 
     public static Type promoteConditional(final Definition definition,
@@ -245,48 +245,48 @@ public final class AnalyzerCaster {
         final Sort sort1 = from1.sort;
 
         if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return definition.defType;
+            return definition.getType("def");
         }
 
         final boolean primitive = sort0.primitive && sort1.primitive;
 
         if (sort0.bool && sort1.bool) {
-            return primitive ? definition.booleanType : definition.booleanobjType;
+            return primitive ? definition.getType("boolean") : definition.getType("Boolean");
         }
 
         if (sort0.numeric && sort1.numeric) {
             if (sort0 == Sort.DOUBLE || sort0 == Sort.DOUBLE_OBJ || sort1 == Sort.DOUBLE || sort1 == Sort.DOUBLE_OBJ) {
-                return primitive ? definition.doubleType : definition.doubleobjType;
+                return primitive ? definition.getType("double") : definition.getType("Double");
             } else if (sort0 == Sort.FLOAT || sort0 == Sort.FLOAT_OBJ || sort1 == Sort.FLOAT || sort1 == Sort.FLOAT_OBJ) {
-                return primitive ? definition.floatType : definition.floatobjType;
+                return primitive ? definition.getType("float") : definition.getType("Float");
             } else if (sort0 == Sort.LONG || sort0 == Sort.LONG_OBJ || sort1 == Sort.LONG || sort1 == Sort.LONG_OBJ) {
-                return sort0.primitive && sort1.primitive ? definition.longType : definition.longobjType;
+                return sort0.primitive && sort1.primitive ? definition.getType("long") : definition.getType("Long");
             } else {
                 if (sort0 == Sort.BYTE || sort0 == Sort.BYTE_OBJ) {
                     if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
-                        return primitive ? definition.byteType : definition.byteobjType;
+                        return primitive ? definition.getType("byte") : definition.getType("Byte");
                     } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
                         if (const1 != null) {
                             final short constant = (short)const1;
 
                             if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? definition.byteType : definition.byteobjType;
+                                return primitive ? definition.getType("byte") : definition.getType("Byte");
                             }
                         }
 
-                        return primitive ? definition.shortType : definition.shortobjType;
+                        return primitive ? definition.getType("short") : definition.getType("Short");
                     } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
-                        return primitive ? definition.intType : definition.intobjType;
+                        return primitive ? definition.getType("int") : definition.getType("Integer");
                     } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
                         if (const1 != null) {
                             final int constant = (int)const1;
 
                             if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? definition.byteType : definition.byteobjType;
+                                return primitive ? definition.getType("byte") : definition.getType("Byte");
                             }
                         }
 
-                        return primitive ? definition.intType : definition.intobjType;
+                        return primitive ? definition.getType("int") : definition.getType("Integer");
                     }
                 } else if (sort0 == Sort.SHORT || sort0 == Sort.SHORT_OBJ) {
                     if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
@@ -294,43 +294,43 @@ public final class AnalyzerCaster {
                             final short constant = (short)const0;
 
                             if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? definition.byteType : definition.byteobjType;
+                                return primitive ? definition.getType("byte") : definition.getType("Byte");
                             }
                         }
 
-                        return primitive ? definition.shortType : definition.shortobjType;
+                        return primitive ? definition.getType("short") : definition.getType("Short");
                     } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
-                        return primitive ? definition.shortType : definition.shortobjType;
+                        return primitive ? definition.getType("short") : definition.getType("Short");
                     } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
-                        return primitive ? definition.intType : definition.intobjType;
+                        return primitive ? definition.getType("int") : definition.getType("Integer");
                     } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
                         if (const1 != null) {
                             final int constant = (int)const1;
 
                             if (constant <= Short.MAX_VALUE && constant >= Short.MIN_VALUE) {
-                                return primitive ? definition.shortType : definition.shortobjType;
+                                return primitive ? definition.getType("short") : definition.getType("Short");
                             }
                         }
 
-                        return primitive ? definition.intType : definition.intobjType;
+                        return primitive ? definition.getType("int") : definition.getType("Integer");
                     }
                 } else if (sort0 == Sort.CHAR || sort0 == Sort.CHAR_OBJ) {
                     if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
-                        return primitive ? definition.intType : definition.intobjType;
+                        return primitive ? definition.getType("int") : definition.getType("Integer");
                     } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
-                        return primitive ? definition.intType : definition.intobjType;
+                        return primitive ? definition.getType("int") : definition.getType("Integer");
                     } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
-                        return primitive ? definition.charType : definition.charobjType;
+                        return primitive ? definition.getType("char") : definition.getType("Character");
                     } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
                         if (const1 != null) {
                             final int constant = (int)const1;
 
                             if (constant <= Character.MAX_VALUE && constant >= Character.MIN_VALUE) {
-                                return primitive ? definition.byteType : definition.byteobjType;
+                                return primitive ? definition.getType("byte") : definition.getType("Byte");
                             }
                         }
 
-                        return primitive ? definition.intType : definition.intobjType;
+                        return primitive ? definition.getType("int") : definition.getType("Integer");
                     }
                 } else if (sort0 == Sort.INT || sort0 == Sort.INT_OBJ) {
                     if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
@@ -338,33 +338,33 @@ public final class AnalyzerCaster {
                             final int constant = (int)const0;
 
                             if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? definition.byteType : definition.byteobjType;
+                                return primitive ? definition.getType("byte") : definition.getType("Byte");
                             }
                         }
 
-                        return primitive ? definition.intType : definition.intobjType;
+                        return primitive ? definition.getType("int") : definition.getType("Integer");
                     } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
                         if (const0 != null) {
                             final int constant = (int)const0;
 
                             if (constant <= Short.MAX_VALUE && constant >= Short.MIN_VALUE) {
-                                return primitive ? definition.byteType : definition.byteobjType;
+                                return primitive ? definition.getType("byte") : definition.getType("Byte");
                             }
                         }
 
-                        return primitive ? definition.intType : definition.intobjType;
+                        return primitive ? definition.getType("int") : definition.getType("Integer");
                     } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
                         if (const0 != null) {
                             final int constant = (int)const0;
 
                             if (constant <= Character.MAX_VALUE && constant >= Character.MIN_VALUE) {
-                                return primitive ? definition.byteType : definition.byteobjType;
+                                return primitive ? definition.getType("byte") : definition.getType("Byte");
                             }
                         }
 
-                        return primitive ? definition.intType : definition.intobjType;
+                        return primitive ? definition.getType("int") : definition.getType("Integer");
                     } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
-                        return primitive ? definition.intType : definition.intobjType;
+                        return primitive ? definition.getType("int") : definition.getType("Integer");
                     }
                 }
             }
@@ -374,7 +374,7 @@ public final class AnalyzerCaster {
         //       to calculate the highest upper bound for the two types and return that.
         //       However, for now we just return objectType that may require an extra cast.
 
-        return definition.objectType;
+        return definition.getType("Object");
     }
 
     private AnalyzerCaster() {}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index f44db5cb504..68fc3a32262 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -19,22 +19,18 @@
 
 package org.elasticsearch.painless;
 
-import org.elasticsearch.common.geo.GeoPoint;
-import org.elasticsearch.index.fielddata.ScriptDocValues;
-
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.LineNumberReader;
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
+import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
 
 /**
  * The entire API for Painless.  Also used as a whitelist for checking for legal
@@ -373,176 +369,30 @@ public final class Definition {
         }
     }
 
-    public final Map<String, Struct> structsMap;
-    public final Map<Cast, Cast> transformsMap;
-    public final Map<Class<?>, RuntimeClass> runtimeMap;
-
-    public final Type voidType;
-    public final Type booleanType;
-    public final Type byteType;
-    public final Type shortType;
-    public final Type charType;
-    public final Type intType;
-    public final Type longType;
-    public final Type floatType;
-    public final Type doubleType;
-
-    public final Type voidobjType;
-    public final Type booleanobjType;
-    public final Type byteobjType;
-    public final Type shortobjType;
-    public final Type charobjType;
-    public final Type intobjType;
-    public final Type longobjType;
-    public final Type floatobjType;
-    public final Type doubleobjType;
-
-    public final Type objectType;
-    public final Type defType;
-    public final Type numberType;
-    public final Type charseqType;
-    public final Type stringType;
-    public final Type mathType;
-    public final Type utilityType;
-    public final Type defobjType;
-
-    public final Type itrType;
-    public final Type oitrType;
-    public final Type sitrType;
-
-    public final Type collectionType;
-    public final Type ocollectionType;
-    public final Type scollectionType;
-
-    public final Type listType;
-    public final Type arraylistType;
-    public final Type olistType;
-    public final Type oarraylistType;
-    public final Type slistType;
-    public final Type sarraylistType;
-
-    public final Type setType;
-    public final Type hashsetType;
-    public final Type osetType;
-    public final Type ohashsetType;
-    public final Type ssetType;
-    public final Type shashsetType;
-
-    public final Type mapType;
-    public final Type hashmapType;
-    public final Type oomapType;
-    public final Type oohashmapType;
-    public final Type smapType;
-    public final Type shashmapType;
-    public final Type somapType;
-    public final Type sohashmapType;
-
-    public final Type execType;
-
-    public final Type exceptionType;
-    public final Type arithexcepType;
-    public final Type iargexcepType;
-    public final Type istateexceptType;
-    public final Type nfexcepType;
-
-    // docvalues accessors
-    public final Type geoPointType;
-    public final Type stringsType;
-    // TODO: add ReadableDateTime? or don't expose the joda stuff?
-    public final Type longsType;
-    public final Type doublesType;
-    public final Type geoPointsType;
-
-    // for testing features not currently "used" by the whitelist (we should not rush the API for that!)
-    public final Type featureTestType;
+    final Map<Cast, Cast> transformsMap;
+    final Map<Class<?>, RuntimeClass> runtimeMap;
+    private final Map<String, Struct> structsMap;
+    private final Map<String, Type> simpleTypesMap;
 
     private Definition() {
         structsMap = new HashMap<>();
+        simpleTypesMap = new HashMap<>();
         transformsMap = new HashMap<>();
         runtimeMap = new HashMap<>();
 
-        addStructs();
-
-        voidType = getType("void");
-        booleanType = getType("boolean");
-        byteType = getType("byte");
-        shortType = getType("short");
-        charType = getType("char");
-        intType = getType("int");
-        longType = getType("long");
-        floatType = getType("float");
-        doubleType = getType("double");
-
-        voidobjType = getType("Void");
-        booleanobjType = getType("Boolean");
-        byteobjType = getType("Byte");
-        shortobjType = getType("Short");
-        charobjType = getType("Character");
-        intobjType = getType("Integer");
-        longobjType = getType("Long");
-        floatobjType = getType("Float");
-        doubleobjType = getType("Double");
-
-        objectType = getType("Object");
-        defType = getType("def");
-        numberType = getType("Number");
-        charseqType = getType("CharSequence");
-        stringType = getType("String");
-        mathType = getType("Math");
-        utilityType = getType("Utility");
-        defobjType = getType("Def");
-
-        itrType = getType("Iterator");
-        oitrType = getType("Iterator<Object>");
-        sitrType = getType("Iterator<String>");
-
-        collectionType = getType("Collection");
-        ocollectionType = getType("Collection<Object>");
-        scollectionType = getType("Collection<String>");
-
-        listType = getType("List");
-        arraylistType = getType("ArrayList");
-        olistType = getType("List<Object>");
-        oarraylistType = getType("ArrayList<Object>");
-        slistType = getType("List<String>");
-        sarraylistType = getType("ArrayList<String>");
-
-        setType = getType("Set");
-        hashsetType = getType("HashSet");
-        osetType = getType("Set<Object>");
-        ohashsetType = getType("HashSet<Object>");
-        ssetType = getType("Set<String>");
-        shashsetType = getType("HashSet<String>");
-
-        mapType = getType("Map");
-        hashmapType = getType("HashMap");
-        oomapType = getType("Map<Object,Object>");
-        oohashmapType = getType("HashMap<Object,Object>");
-        smapType = getType("Map<String,def>");
-        shashmapType = getType("HashMap<String,def>");
-        somapType = getType("Map<String,Object>");
-        sohashmapType = getType("HashMap<String,Object>");
-
-        execType = getType("Executable");
-
-        exceptionType = getType("Exception");
-        arithexcepType = getType("ArithmeticException");
-        iargexcepType = getType("IllegalArgumentException");
-        istateexceptType = getType("IllegalStateException");
-        nfexcepType = getType("NumberFormatException");
-
-        geoPointType = getType("GeoPoint");
-        stringsType = getType("Strings");
-        longsType = getType("Longs");
-        doublesType = getType("Doubles");
-        geoPointsType = getType("GeoPoints");
-
-        featureTestType = getType("FeatureTest");
-
+        // parse the classes and return hierarchy (map of class name -> superclasses/interfaces)
+        Map<String, List<String>> hierarchy = addStructs();
+        // add every method for each class
         addElements();
-        copyStructs();
+        // apply hierarchy: this means e.g. copying Object's methods into String (thats how subclasses work)
+        for (Map.Entry<String,List<String>> clazz : hierarchy.entrySet()) {
+            copyStruct(clazz.getKey(), clazz.getValue());
+        }
         addTransforms();
-        addRuntimeClasses();
+        // precompute runtime classes
+        for (Struct struct : structsMap.values()) {
+          addRuntimeClass(struct);
+        }
     }
 
     private Definition(final Definition definition) {
@@ -555,710 +405,128 @@ public final class Definition {
         this.structsMap = Collections.unmodifiableMap(structs);
         this.transformsMap = Collections.unmodifiableMap(definition.transformsMap);
         this.runtimeMap = Collections.unmodifiableMap(definition.runtimeMap);
-
-        this.voidType = definition.voidType;
-        this.booleanType = definition.booleanType;
-        this.byteType = definition.byteType;
-        this.shortType = definition.shortType;
-        this.charType = definition.charType;
-        this.intType = definition.intType;
-        this.longType = definition.longType;
-        this.floatType = definition.floatType;
-        this.doubleType = definition.doubleType;
-
-        this.voidobjType = definition.voidobjType;
-        this.booleanobjType = definition.booleanobjType;
-        this.byteobjType = definition.byteobjType;
-        this.shortobjType = definition.shortobjType;
-        this.charobjType = definition.charobjType;
-        this.intobjType = definition.intobjType;
-        this.longobjType = definition.longobjType;
-        this.floatobjType = definition.floatobjType;
-        this.doubleobjType = definition.doubleobjType;
-
-        this.objectType = definition.objectType;
-        this.defType = definition.defType;
-        this.numberType = definition.numberType;
-        this.charseqType = definition.charseqType;
-        this.stringType = definition.stringType;
-        this.mathType = definition.mathType;
-        this.utilityType = definition.utilityType;
-        this.defobjType = definition.defobjType;
-
-        this.itrType = definition.itrType;
-        this.oitrType = definition.oitrType;
-        this.sitrType = definition.sitrType;
-
-        this.collectionType = definition.collectionType;
-        this.ocollectionType = definition.ocollectionType;
-        this.scollectionType = definition.scollectionType;
-
-        this.listType = definition.listType;
-        this.arraylistType = definition.arraylistType;
-        this.olistType = definition.olistType;
-        this.oarraylistType = definition.oarraylistType;
-        this.slistType = definition.slistType;
-        this.sarraylistType = definition.sarraylistType;
-
-        this.setType = definition.setType;
-        this.hashsetType = definition.hashsetType;
-        this.osetType = definition.osetType;
-        this.ohashsetType = definition.ohashsetType;
-        this.ssetType = definition.ssetType;
-        this.shashsetType = definition.shashsetType;
-
-        this.mapType = definition.mapType;
-        this.hashmapType = definition.hashmapType;
-        this.oomapType = definition.oomapType;
-        this.oohashmapType = definition.oohashmapType;
-        this.smapType = definition.smapType;
-        this.shashmapType = definition.shashmapType;
-        this.somapType = definition.somapType;
-        this.sohashmapType = definition.sohashmapType;
-
-        this.execType = definition.execType;
-
-        this.exceptionType = definition.exceptionType;
-        this.arithexcepType = definition.arithexcepType;
-        this.iargexcepType = definition.iargexcepType;
-        this.istateexceptType = definition.istateexceptType;
-        this.nfexcepType = definition.nfexcepType;
-
-        this.geoPointType = definition.geoPointType;
-        this.stringsType = definition.stringsType;
-        this.longsType = definition.longsType;
-        this.doublesType = definition.doublesType;
-        this.geoPointsType = definition.geoPointsType;
-
-        this.featureTestType = definition.featureTestType;
+        this.simpleTypesMap = Collections.unmodifiableMap(definition.simpleTypesMap);
     }
 
-    private void addStructs() {
-        addStruct( "void"    , void.class    );
-        addStruct( "boolean" , boolean.class );
-        addStruct( "byte"    , byte.class    );
-        addStruct( "short"   , short.class   );
-        addStruct( "char"    , char.class    );
-        addStruct( "int"     , int.class     );
-        addStruct( "long"    , long.class    );
-        addStruct( "float"   , float.class   );
-        addStruct( "double"  , double.class  );
-
-        addStruct( "Void"      , Void.class      );
-        addStruct( "Boolean"   , Boolean.class   );
-        addStruct( "Byte"      , Byte.class      );
-        addStruct( "Short"     , Short.class     );
-        addStruct( "Character" , Character.class );
-        addStruct( "Integer"   , Integer.class   );
-        addStruct( "Long"      , Long.class      );
-        addStruct( "Float"     , Float.class     );
-        addStruct( "Double"    , Double.class    );
-
-        addStruct( "Object"       , Object.class       );
-        addStruct( "def"          , Object.class       );
-        addStruct( "Number"       , Number.class       );
-        addStruct( "CharSequence" , CharSequence.class );
-        addStruct( "String"       , String.class       );
-        addStruct( "Math"         , Math.class         );
-        addStruct( "Utility"      , Utility.class      );
-        addStruct( "Def"          , Def.class          );
-
-        addStruct( "Iterator"         , Iterator.class );
-        addStruct( "Iterator<Object>" , Iterator.class );
-        addStruct( "Iterator<String>" , Iterator.class );
-
-        addStruct( "Collection"         , Collection.class );
-        addStruct( "Collection<Object>" , Collection.class );
-        addStruct( "Collection<String>" , Collection.class );
-
-        addStruct( "List"              , List.class      );
-        addStruct( "ArrayList"         , ArrayList.class );
-        addStruct( "List<Object>"      , List.class      );
-        addStruct( "ArrayList<Object>" , ArrayList.class );
-        addStruct( "List<String>"      , List.class      );
-        addStruct( "ArrayList<String>" , ArrayList.class );
-
-        addStruct( "Set"             , Set.class     );
-        addStruct( "HashSet"         , HashSet.class );
-        addStruct( "Set<Object>"     , Set.class     );
-        addStruct( "HashSet<Object>" , HashSet.class );
-        addStruct( "Set<String>"     , Set.class     );
-        addStruct( "HashSet<String>" , HashSet.class );
-
-        addStruct( "Map"                    , Map.class     );
-        addStruct( "HashMap"                , HashMap.class );
-        addStruct( "Map<Object,Object>"     , Map.class     );
-        addStruct( "HashMap<Object,Object>" , HashMap.class );
-        addStruct( "Map<String,def>"        , Map.class     );
-        addStruct( "HashMap<String,def>"    , HashMap.class );
-        addStruct( "Map<String,Object>"     , Map.class     );
-        addStruct( "HashMap<String,Object>" , HashMap.class );
-
-        addStruct( "Executable" , Executable.class );
-
-        addStruct( "Exception"                , Exception.class);
-        addStruct( "ArithmeticException"      , ArithmeticException.class);
-        addStruct( "IllegalArgumentException" , IllegalArgumentException.class);
-        addStruct( "IllegalStateException"    , IllegalStateException.class);
-        addStruct( "NumberFormatException"    , NumberFormatException.class);
-
-        addStruct( "GeoPoint"  , GeoPoint.class);
-        addStruct( "Strings"   , ScriptDocValues.Strings.class);
-        addStruct( "Longs"     , ScriptDocValues.Longs.class);
-        addStruct( "Doubles"   , ScriptDocValues.Doubles.class);
-        addStruct( "GeoPoints" , ScriptDocValues.GeoPoints.class);
-
-        addStruct( "FeatureTest", FeatureTest.class);
+    /** adds classes from definition. returns hierarchy */
+    private Map<String,List<String>> addStructs() {
+        final Map<String,List<String>> hierarchy = new HashMap<>();
+        int currentLine = -1;
+        try {
+            try (InputStream stream = Definition.class.getResourceAsStream("definition.txt");
+                    LineNumberReader reader = new LineNumberReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
+                String line = null;
+                while ((line = reader.readLine()) != null) {
+                    currentLine = reader.getLineNumber();
+                    line = line.trim();
+                    if (line.length() == 0 || line.charAt(0) == '#') {
+                        continue;
+                    }
+                    if (line.startsWith("class ")) {
+                        String elements[] = line.split("\u0020");
+                        assert elements[2].equals("->");
+                        if (elements.length == 7) {
+                            hierarchy.put(elements[1], Arrays.asList(elements[5].split(",")));
+                        } else {
+                            assert elements.length == 5;
+                        }
+                        String className = elements[1];
+                        String javaPeer = elements[3];
+                        final Class<?> javaClazz;
+                        switch (javaPeer) {
+                            case "void":
+                                javaClazz = void.class;
+                                break;
+                            case "boolean":
+                                javaClazz = boolean.class;
+                                break;
+                            case "byte":
+                                javaClazz = byte.class;
+                                break;
+                            case "short":
+                                javaClazz = short.class;
+                                break;
+                            case "char":
+                                javaClazz = char.class;
+                                break;
+                            case "int":
+                                javaClazz = int.class;
+                                break;
+                            case "long":
+                                javaClazz = long.class;
+                                break;
+                            case "float":
+                                javaClazz = float.class;
+                                break;
+                            case "double":
+                                javaClazz = double.class;
+                                break;
+                            default:
+                                javaClazz = Class.forName(javaPeer);
+                                break;
+                        }
+                        addStruct(className, javaClazz);
+                    }
+                }
+            }
+        } catch (Exception e) {
+            throw new RuntimeException("syntax error in definition line: " + currentLine, e);
+        }
+        return hierarchy;
     }
 
+    /** adds class methods/fields/ctors */
     private void addElements() {
-        addMethod("Object", "equals", null, false, booleanType, new Type[] {objectType}, null, null);
-        addMethod("Object", "hashCode", null, false, intType, new Type[] {}, null, null);
-        addMethod("Object", "toString", null, false, stringType, new Type[] {}, null, null);
-
-        addMethod("def", "equals", null, false, booleanType, new Type[] {objectType}, null, null);
-        addMethod("def", "hashCode", null, false, intType, new Type[] {}, null, null);
-        addMethod("def", "toString", null, false, stringType, new Type[] {}, null, null);
-
-        addConstructor("Boolean", "new", new Type[] {booleanType}, null);
-        addMethod("Boolean", "booleanValue", null, false, booleanType, new Type[] {}, null, null);
-        addMethod("Boolean", "compare", null, true, intType, new Type[] {booleanType,booleanType}, null, null);
-        addMethod("Boolean", "compareTo", null, false, intType, new Type[] {booleanobjType}, null, null);
-        addMethod("Boolean", "parseBoolean", null, true, booleanType, new Type[] {stringType}, null, null);
-        addMethod("Boolean", "valueOf", null, true, booleanobjType, new Type[] {booleanType}, null, null);
-        addField("Boolean", "FALSE", null, true, booleanobjType, null);
-        addField("Boolean", "TRUE", null, true, booleanobjType, null);
-
-        addConstructor("Byte", "new", new Type[] {byteType}, null);
-        addMethod("Byte", "compare", null, true, intType, new Type[] {byteType,byteType}, null, null);
-        addMethod("Byte", "compareTo", null, false, intType, new Type[] {byteobjType}, null, null);
-        addMethod("Byte", "parseByte", null, true, byteType, new Type[] {stringType}, null, null);
-        addMethod("Byte", "valueOf", null, true, byteobjType, new Type[] {byteType}, null, null);
-        addField("Byte", "MIN_VALUE", null, true, byteType, null);
-        addField("Byte", "MAX_VALUE", null, true, byteType, null);
-
-        addConstructor("Short", "new", new Type[] {shortType}, null);
-        addMethod("Short", "compare", null, true, intType, new Type[] {shortType,shortType}, null, null);
-        addMethod("Short", "compareTo", null, false, intType, new Type[] {shortobjType}, null, null);
-        addMethod("Short", "parseShort", null, true, shortType, new Type[] {stringType}, null, null);
-        addMethod("Short", "valueOf", null, true, shortobjType, new Type[] {shortType}, null, null);
-        addField("Short", "MIN_VALUE", null, true, shortType, null);
-        addField("Short", "MAX_VALUE", null, true, shortType, null);
-
-        addConstructor("Character", "new", new Type[] {charType}, null);
-        addMethod("Character", "charCount", null, true, intType, new Type[] {intType}, null, null);
-        addMethod("Character", "charValue", null, false, charType, new Type[] {}, null, null);
-        addMethod("Character", "compare", null, true, intType, new Type[] {charType,charType}, null, null);
-        addMethod("Character", "compareTo", null, false, intType, new Type[] {charobjType}, null, null);
-        addMethod("Character", "digit", null, true, intType, new Type[] {intType,intType}, null, null);
-        addMethod("Character", "forDigit", null, true, charType, new Type[] {intType,intType}, null, null);
-        addMethod("Character", "getName", null, true, stringType, new Type[] {intType}, null, null);
-        addMethod("Character", "getNumericValue", null, true, intType, new Type[] {intType}, null, null);
-        addMethod("Character", "isAlphabetic", null, true, booleanType, new Type[] {intType}, null, null);
-        addMethod("Character", "isDefined", null, true, booleanType, new Type[] {intType}, null, null);
-        addMethod("Character", "isDigit", null, true, booleanType, new Type[] {intType}, null, null);
-        addMethod("Character", "isIdeographic", null, true, booleanType, new Type[] {intType}, null, null);
-        addMethod("Character", "isLetter", null, true, booleanType, new Type[] {intType}, null, null);
-        addMethod("Character", "isLetterOrDigit", null, true, booleanType, new Type[] {intType}, null, null);
-        addMethod("Character", "isLowerCase", null, true, booleanType, new Type[] {intType}, null, null);
-        addMethod("Character", "isMirrored", null, true, booleanType, new Type[] {intType}, null, null);
-        addMethod("Character", "isSpaceChar", null, true, booleanType, new Type[] {intType}, null, null);
-        addMethod("Character", "isTitleCase", null, true, booleanType, new Type[] {intType}, null, null);
-        addMethod("Character", "isUpperCase", null, true, booleanType, new Type[] {intType}, null, null);
-        addMethod("Character", "isWhitespace", null, true, booleanType, new Type[] {intType}, null, null);
-        addMethod("Character", "valueOf", null, true, charobjType, new Type[] {charType}, null, null);
-        addField("Character", "MIN_VALUE", null, true, charType, null);
-        addField("Character", "MAX_VALUE", null, true, charType, null);
-
-        addConstructor("Integer", "new", new Type[] {intType}, null);
-        addMethod("Integer", "compare", null, true, intType, new Type[] {intType,intType}, null, null);
-        addMethod("Integer", "compareTo", null, false, intType, new Type[] {intobjType}, null, null);
-        addMethod("Integer", "min", null, true, intType, new Type[] {intType,intType}, null, null);
-        addMethod("Integer", "max", null, true, intType, new Type[] {intType,intType}, null, null);
-        addMethod("Integer", "parseInt", null, true, intType, new Type[] {stringType}, null, null);
-        addMethod("Integer", "signum", null, true, intType, new Type[] {intType}, null, null);
-        addMethod("Integer", "toHexString", null, true, stringType, new Type[] {intType}, null, null);
-        addMethod("Integer", "valueOf", null, true, intobjType, new Type[] {intType}, null, null);
-        addField("Integer", "MIN_VALUE", null, true, intType, null);
-        addField("Integer", "MAX_VALUE", null, true, intType, null);
-
-        addConstructor("Long", "new", new Type[] {longType}, null);
-        addMethod("Long", "compare", null, true, intType, new Type[] {longType,longType}, null, null);
-        addMethod("Long", "compareTo", null, false, intType, new Type[] {longobjType}, null, null);
-        addMethod("Long", "min", null, true, longType, new Type[] {longType,longType}, null, null);
-        addMethod("Long", "max", null, true, longType, new Type[] {longType,longType}, null, null);
-        addMethod("Long", "parseLong", null, true, longType, new Type[] {stringType}, null, null);
-        addMethod("Long", "signum", null, true, intType, new Type[] {longType}, null, null);
-        addMethod("Long", "toHexString", null, true, stringType, new Type[] {longType}, null, null);
-        addMethod("Long", "valueOf", null, true, longobjType, new Type[] {longType}, null, null);
-        addField("Long", "MIN_VALUE", null, true, longType, null);
-        addField("Long", "MAX_VALUE", null, true, longType, null);
-
-        addConstructor("Float", "new", new Type[] {floatType}, null);
-        addMethod("Float", "compare", null, true, intType, new Type[] {floatType,floatType}, null, null);
-        addMethod("Float", "compareTo", null, false, intType, new Type[] {floatobjType}, null, null);
-        addMethod("Float", "min", null, true, floatType, new Type[] {floatType,floatType}, null, null);
-        addMethod("Float", "max", null, true, floatType, new Type[] {floatType,floatType}, null, null);
-        addMethod("Float", "parseFloat", null, true, floatType, new Type[] {stringType}, null, null);
-        addMethod("Float", "toHexString", null, true, stringType, new Type[] {floatType}, null, null);
-        addMethod("Float", "valueOf", null, true, floatobjType, new Type[] {floatType}, null, null);
-        addField("Float", "MIN_VALUE", null, true, floatType, null);
-        addField("Float", "MAX_VALUE", null, true, floatType, null);
-
-        addConstructor("Double", "new", new Type[] {doubleType}, null);
-        addMethod("Double", "compare", null, true, intType, new Type[] {doubleType,doubleType}, null, null);
-        addMethod("Double", "compareTo", null, false, intType, new Type[] {doubleobjType}, null, null);
-        addMethod("Double", "min", null, true, doubleType, new Type[] {doubleType,doubleType}, null, null);
-        addMethod("Double", "max", null, true, doubleType, new Type[] {doubleType,doubleType}, null, null);
-        addMethod("Double", "parseDouble", null, true, doubleType, new Type[] {stringType}, null, null);
-        addMethod("Double", "toHexString", null, true, stringType, new Type[] {doubleType}, null, null);
-        addMethod("Double", "valueOf", null, true, doubleobjType, new Type[] {doubleType}, null, null);
-        addField("Double", "MIN_VALUE", null, true, doubleType, null);
-        addField("Double", "MAX_VALUE", null, true, doubleType, null);
-
-        addMethod("Number", "byteValue", null, false, byteType, new Type[] {}, null, null);
-        addMethod("Number", "shortValue", null, false, shortType, new Type[] {}, null, null);
-        addMethod("Number", "intValue", null, false, intType, new Type[] {}, null, null);
-        addMethod("Number", "longValue", null, false, longType, new Type[] {}, null, null);
-        addMethod("Number", "floatValue", null, false, floatType, new Type[] {}, null, null);
-        addMethod("Number", "doubleValue", null, false, doubleType, new Type[] {}, null, null);
-
-        addMethod("CharSequence", "charAt", null, false, charType, new Type[] {intType}, null, null);
-        addMethod("CharSequence", "length", null, false, intType, new Type[] {}, null, null);
-
-        addConstructor("String", "new", new Type[] {}, null);
-        addMethod("String", "codePointAt", null, false, intType, new Type[] {intType}, null, null);
-        addMethod("String", "compareTo", null, false, intType, new Type[] {stringType}, null, null);
-        addMethod("String", "concat", null, false, stringType, new Type[] {stringType}, null, null);
-        addMethod("String", "endsWith", null, false, booleanType, new Type[] {stringType}, null, null);
-        addMethod("String", "indexOf", null, false, intType, new Type[] {stringType}, null, null);
-        addMethod("String", "indexOf", null, false, intType, new Type[] {stringType, intType}, null, null);
-        addMethod("String", "isEmpty", null, false, booleanType, new Type[] {}, null, null);
-        addMethod("String", "replace", null, false, stringType, new Type[] {charseqType, charseqType}, null, null);
-        addMethod("String", "startsWith", null, false, booleanType, new Type[] {stringType}, null, null);
-        addMethod("String", "substring", null, false, stringType, new Type[] {intType, intType}, null, null);
-        addMethod("String", "toCharArray", null, false, getType(charType.struct, 1), new Type[] {}, null, null);
-        addMethod("String", "trim", null, false, stringType, new Type[] {}, null, null);
-
-        addMethod("Utility", "NumberToboolean", null, true, booleanType, new Type[] {numberType}, null, null);
-        addMethod("Utility", "NumberTochar", null, true, charType, new Type[] {numberType}, null, null);
-        addMethod("Utility", "NumberToBoolean", null, true, booleanobjType, new Type[] {numberType}, null, null);
-        addMethod("Utility", "NumberToByte", null, true, byteobjType, new Type[] {numberType}, null, null);
-        addMethod("Utility", "NumberToShort", null, true, shortobjType, new Type[] {numberType}, null, null);
-        addMethod("Utility", "NumberToCharacter", null, true, charobjType, new Type[] {numberType}, null, null);
-        addMethod("Utility", "NumberToInteger", null, true, intobjType, new Type[] {numberType}, null, null);
-        addMethod("Utility", "NumberToLong", null, true, longobjType, new Type[] {numberType}, null, null);
-        addMethod("Utility", "NumberToFloat", null, true, floatobjType, new Type[] {numberType}, null, null);
-        addMethod("Utility", "NumberToDouble", null, true, doubleobjType, new Type[] {numberType}, null, null);
-        addMethod("Utility", "booleanTobyte", null, true, byteType, new Type[] {booleanType}, null, null);
-        addMethod("Utility", "booleanToshort", null, true, shortType, new Type[] {booleanType}, null, null);
-        addMethod("Utility", "booleanTochar", null, true, charType, new Type[] {booleanType}, null, null);
-        addMethod("Utility", "booleanToint", null, true, intType, new Type[] {booleanType}, null, null);
-        addMethod("Utility", "booleanTolong", null, true, longType, new Type[] {booleanType}, null, null);
-        addMethod("Utility", "booleanTofloat", null, true, floatType, new Type[] {booleanType}, null, null);
-        addMethod("Utility", "booleanTodouble", null, true, doubleType, new Type[] {booleanType}, null, null);
-        addMethod("Utility", "booleanToInteger", null, true, intobjType, new Type[] {booleanType}, null, null);
-        addMethod("Utility", "BooleanTobyte", null, true, byteType, new Type[] {booleanobjType}, null, null);
-        addMethod("Utility", "BooleanToshort", null, true, shortType, new Type[] {booleanobjType}, null, null);
-        addMethod("Utility", "BooleanTochar", null, true, charType, new Type[] {booleanobjType}, null, null);
-        addMethod("Utility", "BooleanToint", null, true, intType, new Type[] {booleanobjType}, null, null);
-        addMethod("Utility", "BooleanTolong", null, true, longType, new Type[] {booleanobjType}, null, null);
-        addMethod("Utility", "BooleanTofloat", null, true, floatType, new Type[] {booleanobjType}, null, null);
-        addMethod("Utility", "BooleanTodouble", null, true, doubleType, new Type[] {booleanobjType}, null, null);
-        addMethod("Utility", "BooleanToByte", null, true, byteobjType, new Type[] {booleanobjType}, null, null);
-        addMethod("Utility", "BooleanToShort", null, true, shortobjType, new Type[] {booleanobjType}, null, null);
-        addMethod("Utility", "BooleanToCharacter", null, true, charobjType, new Type[] {booleanobjType}, null, null);
-        addMethod("Utility", "BooleanToInteger", null, true, intobjType, new Type[] {booleanobjType}, null, null);
-        addMethod("Utility", "BooleanToLong", null, true, longobjType, new Type[] {booleanobjType}, null, null);
-        addMethod("Utility", "BooleanToFloat", null, true, floatobjType, new Type[] {booleanobjType}, null, null);
-        addMethod("Utility", "BooleanToDouble", null, true, doubleobjType, new Type[] {booleanobjType}, null, null);
-        addMethod("Utility", "byteToboolean", null, true, booleanType, new Type[] {byteType}, null, null);
-        addMethod("Utility", "byteToShort", null, true, shortobjType, new Type[] {byteType}, null, null);
-        addMethod("Utility", "byteToCharacter", null, true, charobjType, new Type[] {byteType}, null, null);
-        addMethod("Utility", "byteToInteger", null, true, intobjType, new Type[] {byteType}, null, null);
-        addMethod("Utility", "byteToLong", null, true, longobjType, new Type[] {byteType}, null, null);
-        addMethod("Utility", "byteToFloat", null, true, floatobjType, new Type[] {byteType}, null, null);
-        addMethod("Utility", "byteToDouble", null, true, doubleobjType, new Type[] {byteType}, null, null);
-        addMethod("Utility", "ByteToboolean", null, true, booleanType, new Type[] {byteobjType}, null, null);
-        addMethod("Utility", "ByteTochar", null, true, charType, new Type[] {byteobjType}, null, null);
-        addMethod("Utility", "shortToboolean", null, true, booleanType, new Type[] {shortType}, null, null);
-        addMethod("Utility", "shortToByte", null, true, byteobjType, new Type[] {shortType}, null, null);
-        addMethod("Utility", "shortToCharacter", null, true, charobjType, new Type[] {shortType}, null, null);
-        addMethod("Utility", "shortToInteger", null, true, intobjType, new Type[] {shortType}, null, null);
-        addMethod("Utility", "shortToLong", null, true, longobjType, new Type[] {shortType}, null, null);
-        addMethod("Utility", "shortToFloat", null, true, floatobjType, new Type[] {shortType}, null, null);
-        addMethod("Utility", "shortToDouble", null, true, doubleobjType, new Type[] {shortType}, null, null);
-        addMethod("Utility", "ShortToboolean", null, true, booleanType, new Type[] {shortobjType}, null, null);
-        addMethod("Utility", "ShortTochar", null, true, charType, new Type[] {shortobjType}, null, null);
-        addMethod("Utility", "charToboolean", null, true, booleanType, new Type[] {charType}, null, null);
-        addMethod("Utility", "charToByte", null, true, byteobjType, new Type[] {charType}, null, null);
-        addMethod("Utility", "charToShort", null, true, shortobjType, new Type[] {charType}, null, null);
-        addMethod("Utility", "charToInteger", null, true, intobjType, new Type[] {charType}, null, null);
-        addMethod("Utility", "charToLong", null, true, longobjType, new Type[] {charType}, null, null);
-        addMethod("Utility", "charToFloat", null, true, floatobjType, new Type[] {charType}, null, null);
-        addMethod("Utility", "charToDouble", null, true, doubleobjType, new Type[] {charType}, null, null);
-        addMethod("Utility", "charToString", null, true, stringType, new Type[] {charType}, null, null);
-        addMethod("Utility", "CharacterToboolean", null, true, booleanType, new Type[] {charobjType}, null, null);
-        addMethod("Utility", "CharacterTobyte", null, true, byteType, new Type[] {charobjType}, null, null);
-        addMethod("Utility", "CharacterToshort", null, true, shortType, new Type[] {charobjType}, null, null);
-        addMethod("Utility", "CharacterToint", null, true, intType, new Type[] {charobjType}, null, null);
-        addMethod("Utility", "CharacterTolong", null, true, longType, new Type[] {charobjType}, null, null);
-        addMethod("Utility", "CharacterTofloat", null, true, floatType, new Type[] {charobjType}, null, null);
-        addMethod("Utility", "CharacterTodouble", null, true, doubleType, new Type[] {charobjType}, null, null);
-        addMethod("Utility", "CharacterToBoolean", null, true, booleanobjType, new Type[] {charobjType}, null, null);
-        addMethod("Utility", "CharacterToByte", null, true, byteobjType, new Type[] {charobjType}, null, null);
-        addMethod("Utility", "CharacterToShort", null, true, shortobjType, new Type[] {charobjType}, null, null);
-        addMethod("Utility", "CharacterToInteger", null, true, intobjType, new Type[] {charobjType}, null, null);
-        addMethod("Utility", "CharacterToLong", null, true, longobjType, new Type[] {charobjType}, null, null);
-        addMethod("Utility", "CharacterToFloat", null, true, floatobjType, new Type[] {charobjType}, null, null);
-        addMethod("Utility", "CharacterToDouble", null, true, doubleobjType, new Type[] {charobjType}, null, null);
-        addMethod("Utility", "CharacterToString", null, true, stringType, new Type[] {charobjType}, null, null);
-        addMethod("Utility", "intToboolean", null, true, booleanType, new Type[] {intType}, null, null);
-        addMethod("Utility", "intToByte", null, true, byteobjType, new Type[] {intType}, null, null);
-        addMethod("Utility", "intToShort", null, true, shortobjType, new Type[] {intType}, null, null);
-        addMethod("Utility", "intToCharacter", null, true, charobjType, new Type[] {intType}, null, null);
-        addMethod("Utility", "intToLong", null, true, longobjType, new Type[] {intType}, null, null);
-        addMethod("Utility", "intToFloat", null, true, floatobjType, new Type[] {intType}, null, null);
-        addMethod("Utility", "intToDouble", null, true, doubleobjType, new Type[] {intType}, null, null);
-        addMethod("Utility", "IntegerToboolean", null, true, booleanType, new Type[] {intobjType}, null, null);
-        addMethod("Utility", "IntegerTochar", null, true, charType, new Type[] {intobjType}, null, null);
-        addMethod("Utility", "longToboolean", null, true, booleanType, new Type[] {longType}, null, null);
-        addMethod("Utility", "longToByte", null, true, byteobjType, new Type[] {longType}, null, null);
-        addMethod("Utility", "longToShort", null, true, shortobjType, new Type[] {longType}, null, null);
-        addMethod("Utility", "longToCharacter", null, true, charobjType, new Type[] {longType}, null, null);
-        addMethod("Utility", "longToInteger", null, true, intobjType, new Type[] {longType}, null, null);
-        addMethod("Utility", "longToFloat", null, true, floatobjType, new Type[] {longType}, null, null);
-        addMethod("Utility", "longToDouble", null, true, doubleobjType, new Type[] {longType}, null, null);
-        addMethod("Utility", "LongToboolean", null, true, booleanType, new Type[] {longobjType}, null, null);
-        addMethod("Utility", "LongTochar", null, true, charType, new Type[] {longobjType}, null, null);
-        addMethod("Utility", "floatToboolean", null, true, booleanType, new Type[] {floatType}, null, null);
-        addMethod("Utility", "floatToByte", null, true, byteobjType, new Type[] {floatType}, null, null);
-        addMethod("Utility", "floatToShort", null, true, shortobjType, new Type[] {floatType}, null, null);
-        addMethod("Utility", "floatToCharacter", null, true, charobjType, new Type[] {floatType}, null, null);
-        addMethod("Utility", "floatToInteger", null, true, intobjType, new Type[] {floatType}, null, null);
-        addMethod("Utility", "floatToLong", null, true, longobjType, new Type[] {floatType}, null, null);
-        addMethod("Utility", "floatToDouble", null, true, doubleobjType, new Type[] {floatType}, null, null);
-        addMethod("Utility", "FloatToboolean", null, true, booleanType, new Type[] {floatobjType}, null, null);
-        addMethod("Utility", "FloatTochar", null, true, charType, new Type[] {floatobjType}, null, null);
-        addMethod("Utility", "doubleToboolean", null, true, booleanType, new Type[] {doubleType}, null, null);
-        addMethod("Utility", "doubleToByte", null, true, byteobjType, new Type[] {doubleType}, null, null);
-        addMethod("Utility", "doubleToShort", null, true, shortobjType, new Type[] {doubleType}, null, null);
-        addMethod("Utility", "doubleToCharacter", null, true, charobjType, new Type[] {doubleType}, null, null);
-        addMethod("Utility", "doubleToInteger", null, true, intobjType, new Type[] {doubleType}, null, null);
-        addMethod("Utility", "doubleToLong", null, true, longobjType, new Type[] {doubleType}, null, null);
-        addMethod("Utility", "doubleToFloat", null, true, floatobjType, new Type[] {doubleType}, null, null);
-        addMethod("Utility", "DoubleToboolean", null, true, booleanType, new Type[] {doubleobjType}, null, null);
-        addMethod("Utility", "DoubleTochar", null, true, charType, new Type[] {doubleobjType}, null, null);
-        addMethod("Utility", "StringTochar", null, true, charType, new Type[] {stringType}, null, null);
-        addMethod("Utility", "StringToCharacter", null, true, charobjType, new Type[] {stringType}, null, null);
-
-        addMethod("Math", "abs", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "acos", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "asin", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "atan", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "atan2", null, true, doubleType, new Type[] {doubleType, doubleType}, null, null);
-        addMethod("Math", "cbrt", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "ceil", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "cos", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "cosh", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "exp", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "expm1", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "floor", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "hypot", null, true, doubleType, new Type[] {doubleType, doubleType}, null, null);
-        addMethod("Math", "log", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "log10", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "log1p", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "max", null, true, doubleType, new Type[] {doubleType, doubleType}, null, null);
-        addMethod("Math", "min", null, true, doubleType, new Type[] {doubleType, doubleType}, null, null);
-        addMethod("Math", "pow", null, true, doubleType, new Type[] {doubleType, doubleType}, null, null);
-        addMethod("Math", "random", null, true, doubleType, new Type[] {}, null, null);
-        addMethod("Math", "rint", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "round", null, true, longType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "sin", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "sinh", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "sqrt", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "tan", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "tanh", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "toDegrees", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addMethod("Math", "toRadians", null, true, doubleType, new Type[] {doubleType}, null, null);
-        addField("Math", "E", null, true, doubleType, null);
-        addField("Math", "PI", null, true, doubleType, null);
-
-        addMethod("Def", "DefTobyteImplicit", null, true, byteType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToshortImplicit", null, true, shortType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefTocharImplicit", null, true, charType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefTointImplicit", null, true, intType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefTolongImplicit", null, true, longType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefTofloatImplicit", null, true, floatType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefTodoubleImplicit", null, true, doubleType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToByteImplicit", null, true, byteobjType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToShortImplicit", null, true, shortobjType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToCharacterImplicit", null, true, charobjType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToIntegerImplicit", null, true, intobjType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToLongImplicit", null, true, longobjType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToFloatImplicit", null, true, floatobjType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToDoubleImplicit", null, true, doubleobjType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefTobyteExplicit", null, true, byteType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToshortExplicit", null, true, shortType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefTocharExplicit", null, true, charType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefTointExplicit", null, true, intType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefTolongExplicit", null, true, longType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefTofloatExplicit", null, true, floatType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefTodoubleExplicit", null, true, doubleType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToByteExplicit", null, true, byteobjType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToShortExplicit", null, true, shortobjType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToCharacterExplicit", null, true, charobjType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToIntegerExplicit", null, true, intobjType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToLongExplicit", null, true, longobjType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToFloatExplicit", null, true, floatobjType, new Type[] {defType}, null, null);
-        addMethod("Def", "DefToDoubleExplicit", null, true, doubleobjType, new Type[] {defType}, null, null);
-
-        addMethod("Iterator", "hasNext", null, false, booleanType, new Type[] {}, null, null);
-        addMethod("Iterator", "next", null, false, objectType, new Type[] {}, defType, null);
-        addMethod("Iterator", "remove", null, false, voidType, new Type[] {}, null, null);
-
-        addMethod("Iterator<Object>", "hasNext", null, false, booleanType, new Type[] {}, null, null);
-        addMethod("Iterator<Object>", "next", null, false, objectType, new Type[] {}, null, null);
-        addMethod("Iterator<Object>", "remove", null, false, voidType, new Type[] {}, null, null);
-
-        addMethod("Iterator<String>", "hasNext", null, false, booleanType, new Type[] {}, null, null);
-        addMethod("Iterator<String>", "next", null, false, objectType, new Type[] {}, stringType, null);
-        addMethod("Iterator<String>", "remove", null, false, voidType, new Type[] {}, null, null);
-
-        addMethod("Collection", "add", null, false, booleanType, new Type[] {objectType}, null, new Type[] {defType});
-        addMethod("Collection", "clear", null, false, voidType, new Type[] {}, null, null);
-        addMethod("Collection", "contains", null, false, booleanType, new Type[] {objectType}, null, new Type[] {defType});
-        addMethod("Collection", "isEmpty", null, false, booleanType, new Type[] {}, null, null);
-        addMethod("Collection", "iterator", null, false, itrType, new Type[] {}, null, null);
-        addMethod("Collection", "remove", null, false, booleanType, new Type[] {objectType}, null, new Type[] {defType});
-        addMethod("Collection", "size", null, false, intType, new Type[] {}, null, null);
-
-        addMethod("Collection<Object>", "add", null, false, booleanType, new Type[] {objectType}, null, null);
-        addMethod("Collection<Object>", "clear", null, false, voidType, new Type[] {}, null, null);
-        addMethod("Collection<Object>", "contains", null, false, booleanType, new Type[] {objectType}, null, null);
-        addMethod("Collection<Object>", "isEmpty", null, false, booleanType, new Type[] {}, null, null);
-        addMethod("Collection<Object>", "iterator", null, false, oitrType, new Type[] {}, null, null);
-        addMethod("Collection<Object>", "remove", null, false, booleanType, new Type[] {objectType}, null, null);
-        addMethod("Collection<Object>", "size", null, false, intType, new Type[] {}, null, null);
-
-        addMethod("Collection<String>", "add", null, false, booleanType, new Type[] {objectType}, null, new Type[] {stringType});
-        addMethod("Collection<String>", "clear", null, false, voidType, new Type[] {}, null, null);
-        addMethod("Collection<String>", "contains", null, false, booleanType, new Type[] {objectType}, null, new Type[] {stringType});
-        addMethod("Collection<String>", "isEmpty", null, false, booleanType, new Type[] {}, null, null);
-        addMethod("Collection<String>", "iterator", null, false, sitrType, new Type[] {}, null, null);
-        addMethod("Collection<String>", "remove", null, false, booleanType, new Type[] {objectType}, null, new Type[] {stringType});
-        addMethod("Collection<String>", "size", null, false, intType, new Type[] {}, null, null);
-
-        addMethod("List", "set", null, false, objectType, new Type[] {intType, objectType}, defType, new Type[] {intType, defType});
-        addMethod("List", "get", null, false, objectType, new Type[] {intType}, defType, null);
-        addMethod("List", "remove", null, false, objectType, new Type[] {intType}, defType, null);
-        addMethod("List", "getLength", "size", false, intType, new Type[] {}, null, null);
-
-        addConstructor("ArrayList", "new", new Type[] {}, null);
-
-        addMethod("List<Object>", "set", null, false, objectType, new Type[] {intType, objectType}, null, null);
-        addMethod("List<Object>", "get", null, false, objectType, new Type[] {intType}, null, null);
-        addMethod("List<Object>", "remove", null, false, objectType, new Type[] {intType}, null, null);
-        addMethod("List<Object>", "getLength", "size", false, intType, new Type[] {}, null, null);
-
-        addConstructor("ArrayList<Object>", "new", new Type[] {}, null);
-
-        addMethod("List<String>", "set", null, false, objectType, new Type[] {intType, objectType}, stringType,
-            new Type[] {intType, stringType});
-        addMethod("List<String>", "get", null, false, objectType, new Type[] {intType}, stringType, null);
-        addMethod("List<String>", "remove", null, false, objectType, new Type[] {intType}, stringType, null);
-        addMethod("List<String>", "getLength", "size", false, intType, new Type[] {}, null, null);
-
-        addConstructor("ArrayList<String>", "new", new Type[] {}, null);
-
-        addConstructor("HashSet", "new", new Type[] {}, null);
-
-        addConstructor("HashSet<Object>", "new", new Type[] {}, null);
-
-        addConstructor("HashSet<String>", "new", new Type[] {}, null);
-
-        addMethod("Map", "put", null, false, objectType, new Type[] {objectType, objectType}, defType, new Type[] {defType, defType});
-        addMethod("Map", "get", null, false, objectType, new Type[] {objectType}, defType, new Type[] {defType});
-        addMethod("Map", "remove", null, false, objectType, new Type[] {objectType}, null, null);
-        addMethod("Map", "isEmpty", null, false, booleanType, new Type[] {}, null, null);
-        addMethod("Map", "size", null, false, intType, new Type[] {}, null, null);
-        addMethod("Map", "containsKey", null, false, booleanType, new Type[] {objectType}, null, new Type[] {defType});
-        addMethod("Map", "containsValue", null, false, booleanType, new Type[] {objectType}, null, new Type[] {defType});
-        addMethod("Map", "keySet", null, false, osetType, new Type[] {}, setType, null);
-        addMethod("Map", "values", null, false, ocollectionType, new Type[] {}, collectionType, null);
-
-        addConstructor("HashMap", "new", new Type[] {}, null);
-
-        addMethod("Map<Object,Object>", "put", null, false, objectType, new Type[] {objectType, objectType}, null, null);
-        addMethod("Map<Object,Object>", "get", null, false, objectType, new Type[] {objectType}, null, null);
-        addMethod("Map<Object,Object>", "remove", null, false, objectType, new Type[] {objectType}, null, null);
-        addMethod("Map<Object,Object>", "isEmpty", null, false, booleanType, new Type[] {}, null, null);
-        addMethod("Map<Object,Object>", "size", null, false, intType, new Type[] {}, null, null);
-        addMethod("Map<Object,Object>", "containsKey", null, false, booleanType, new Type[] {objectType}, null, null);
-        addMethod("Map<Object,Object>", "containsValue", null, false, booleanType, new Type[] {objectType}, null, null);
-        addMethod("Map<Object,Object>", "keySet", null, false, osetType, new Type[] {}, null, null);
-        addMethod("Map<Object,Object>", "values", null, false, ocollectionType, new Type[] {}, null, null);
-
-        addConstructor("HashMap<Object,Object>", "new", new Type[] {}, null);
-
-        addMethod("Map<String,def>", "put", null, false, objectType, new Type[] {objectType, objectType}, defType,
-            new Type[] {stringType, defType});
-        addMethod("Map<String,def>", "get", null, false, objectType, new Type[] {objectType}, defType, new Type[] {stringType});
-        addMethod("Map<String,def>", "remove", null, false, objectType, new Type[] {objectType}, defType, new Type[] {stringType});
-        addMethod("Map<String,def>", "isEmpty", null, false, booleanType, new Type[] {}, null, null);
-        addMethod("Map<String,def>", "size", null, false, intType, new Type[] {}, null, null);
-        addMethod("Map<String,def>", "containsKey", null, false, booleanType, new Type[] {objectType}, null, new Type[] {stringType});
-        addMethod("Map<String,def>", "containsValue", null, false, booleanType, new Type[] {objectType}, null, new Type[] {defType});
-        addMethod("Map<String,def>", "keySet", null, false, osetType, new Type[] {}, ssetType, null);
-        addMethod("Map<String,def>", "values", null, false, ocollectionType, new Type[] {}, collectionType, null);
-
-        addConstructor("HashMap<String,def>", "new", new Type[] {}, null);
-
-        addMethod("Map<String,Object>", "put", null, false, objectType, new Type[] {objectType, objectType}, null,
-            new Type[] {stringType, objectType});
-        addMethod("Map<String,Object>", "get", null, false, objectType, new Type[] {objectType}, null, new Type[] {stringType});
-        addMethod("Map<String,Object>", "remove", null, false, objectType, new Type[] {objectType}, null, new Type[] {stringType});
-        addMethod("Map<String,Object>", "isEmpty", null, false, booleanType, new Type[] {}, null, null);
-        addMethod("Map<String,Object>", "size", null, false, intType, new Type[] {}, null, null);
-        addMethod("Map<String,Object>", "containsKey", null, false, booleanType, new Type[] {objectType}, null, new Type[] {stringType});
-        addMethod("Map<String,Object>", "containsValue", null, false, booleanType, new Type[] {objectType}, null, null);
-        addMethod("Map<String,Object>", "keySet", null, false, osetType, new Type[] {}, ssetType, null);
-        addMethod("Map<String,Object>", "values", null, false, ocollectionType, new Type[] {}, null, null);
-
-        addConstructor("HashMap<String,Object>", "new", new Type[] {}, null);
-
-        addMethod("Exception", "getMessage", null, false, stringType, new Type[] {}, null, null);
-
-        addConstructor("ArithmeticException", "new", new Type[] {stringType}, null);
-
-        addConstructor("IllegalArgumentException", "new", new Type[] {stringType}, null);
-
-        addConstructor("IllegalStateException", "new", new Type[] {stringType}, null);
-
-        addConstructor("NumberFormatException", "new", new Type[] {stringType}, null);
-
-        addMethod("GeoPoint", "getLat", null, false, doubleType, new Type[] {}, null, null);
-        addMethod("GeoPoint", "getLon", null, false, doubleType, new Type[] {}, null, null);
-        addMethod("Strings", "getValue", null, false, stringType, new Type[] {}, null, null);
-        addMethod("Strings", "getValues", null, false, slistType, new Type[] {}, null, null);
-        addMethod("Longs", "getValue", null, false, longType, new Type[] {}, null, null);
-        addMethod("Longs", "getValues", null, false, olistType, new Type[] {}, null, null);
-        // TODO: add better date support for Longs here? (carefully?)
-        addMethod("Doubles", "getValue", null, false, doubleType, new Type[] {}, null, null);
-        addMethod("Doubles", "getValues", null, false, olistType, new Type[] {}, null, null);
-        addMethod("GeoPoints", "getValue", null, false, geoPointType, new Type[] {}, null, null);
-        addMethod("GeoPoints", "getValues", null, false, olistType, new Type[] {}, null, null);
-        addMethod("GeoPoints", "getLat", null, false, doubleType, new Type[] {}, null, null);
-        addMethod("GeoPoints", "getLon", null, false, doubleType, new Type[] {}, null, null);
-        addMethod("GeoPoints", "getLats", null, false, getType(doubleType.struct, 1), new Type[] {}, null, null);
-        addMethod("GeoPoints", "getLons", null, false, getType(doubleType.struct, 1), new Type[] {}, null, null);
-        // geo distance functions... so many...
-        addMethod("GeoPoints", "factorDistance", null, false, doubleType,
-                  new Type[] { doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "factorDistanceWithDefault", null, false, doubleType,
-                  new Type[] { doubleType, doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "factorDistance02", null, false, doubleType,
-                  new Type[] { doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "factorDistance13", null, false, doubleType,
-                  new Type[] { doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "arcDistance", null, false, doubleType,
-                  new Type[] { doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "arcDistanceWithDefault", null, false, doubleType,
-                  new Type[] { doubleType, doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "arcDistanceInKm", null, false, doubleType,
-                  new Type[] { doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "arcDistanceInKmWithDefault", null, false, doubleType,
-                  new Type[] { doubleType, doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "arcDistanceInMiles", null, false, doubleType,
-                  new Type[] { doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "arcDistanceInMilesWithDefault", null, false, doubleType,
-                  new Type[] { doubleType, doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "distance", null, false, doubleType,
-                  new Type[] { doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "distanceWithDefault", null, false, doubleType,
-                  new Type[] { doubleType, doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "distanceInKm", null, false, doubleType,
-                  new Type[] { doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "distanceInKmWithDefault", null, false, doubleType,
-                  new Type[] { doubleType, doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "distanceInMiles", null, false, doubleType,
-                  new Type[] { doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "distanceInMilesWithDefault", null, false, doubleType,
-                  new Type[] { doubleType, doubleType, doubleType }, null, null);
-        addMethod("GeoPoints", "geohashDistance", null, false, doubleType,
-                  new Type[] { stringType }, null, null);
-        addMethod("GeoPoints", "geohashDistanceInKm", null, false, doubleType,
-                  new Type[] { stringType }, null, null);
-        addMethod("GeoPoints", "geohashDistanceInMiles", null, false, doubleType,
-                  new Type[] { stringType }, null, null);
-
-        // currently FeatureTest exposes overloaded constructor, field load store, and overloaded static methods
-        addConstructor("FeatureTest", "new", new Type[] {}, null);
-        addConstructor("FeatureTest", "new", new Type[] {intType, intType}, null);
-        addMethod("FeatureTest", "getX", null, false, intType, new Type[] {}, null, null);
-        addMethod("FeatureTest", "getY", null, false, intType, new Type[] {}, null, null);
-        addMethod("FeatureTest", "setX", null, false, voidType, new Type[] {intType}, null, null);
-        addMethod("FeatureTest", "setY", null, false, voidType, new Type[] {intType}, null, null);
-        addMethod("FeatureTest", "overloadedStatic", null, true, booleanType, new Type[] {}, null, null);
-        addMethod("FeatureTest", "overloadedStatic", null, true, booleanType, new Type[] {booleanType}, null, null);
-    }
-
-    private void copyStructs() {
-        copyStruct("Void", "Object");
-        copyStruct("Boolean", "Object");
-        copyStruct("Byte", "Number", "Object");
-        copyStruct("Short", "Number", "Object");
-        copyStruct("Character", "Object");
-        copyStruct("Integer", "Number", "Object");
-        copyStruct("Long", "Number", "Object");
-        copyStruct("Float", "Number", "Object");
-        copyStruct("Double", "Number", "Object");
-
-        copyStruct("Number", "Object");
-        copyStruct("CharSequence", "Object");
-        copyStruct("String", "CharSequence", "Object");
-
-        copyStruct("List", "Collection", "Object");
-        copyStruct("ArrayList", "List", "Collection", "Object");
-        copyStruct("List<Object>", "Collection<Object>", "Object");
-        copyStruct("ArrayList<Object>", "List<Object>", "Collection<Object>", "Object");
-        copyStruct("List<String>", "Collection<String>", "Object");
-        copyStruct("ArrayList<String>", "List<String>", "Collection<String>", "Object");
-
-        copyStruct("Set", "Collection", "Object");
-        copyStruct("HashSet", "Set", "Collection", "Object");
-        copyStruct("Set<Object>", "Collection<Object>", "Object");
-        copyStruct("HashSet<Object>", "Set<Object>", "Collection<Object>", "Object");
-        copyStruct("Set<String>", "Collection<String>", "Object");
-        copyStruct("HashSet<String>", "Set<String>", "Collection<String>", "Object");
-
-        copyStruct("Map", "Object");
-        copyStruct("HashMap", "Map", "Object");
-        copyStruct("Map<Object,Object>", "Object");
-        copyStruct("HashMap<Object,Object>", "Map<Object,Object>", "Object");
-        copyStruct("Map<String,def>", "Object");
-        copyStruct("HashMap<String,def>", "Map<String,def>", "Object");
-        copyStruct("Map<String,Object>", "Object");
-        copyStruct("HashMap<String,Object>", "Map<String,Object>", "Object");
-
-        copyStruct("Executable", "Object");
-
-        copyStruct("Exception", "Object");
-        copyStruct("ArithmeticException", "Exception", "Object");
-        copyStruct("IllegalArgumentException", "Exception", "Object");
-        copyStruct("IllegalStateException", "Exception", "Object");
-        copyStruct("NumberFormatException", "Exception", "Object");
-
-        copyStruct("GeoPoint", "Object");
-        copyStruct("Strings", "List<String>", "Collection<String>", "Object");
-        copyStruct("Longs", "List", "Collection", "Object");
-        copyStruct("Doubles", "List", "Collection", "Object");
-        copyStruct("GeoPoints", "List", "Collection", "Object");
-
-        copyStruct("FeatureTest", "Object");
+        int currentLine = -1;
+        try {
+            try (InputStream stream = Definition.class.getResourceAsStream("definition.txt");
+                 LineNumberReader reader = new LineNumberReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
+                String line = null;
+                String currentClass = null;
+                while ((line = reader.readLine()) != null) {
+                    currentLine = reader.getLineNumber();
+                    line = line.trim();
+                    if (line.length() == 0 || line.charAt(0) == '#') {
+                        continue;
+                    } else if (line.startsWith("class ")) {
+                        assert currentClass == null;
+                        currentClass = line.split("\u0020")[1];
+                    } else if (line.equals("}")) {
+                        assert currentClass != null;
+                        currentClass = null;
+                    } else {
+                        assert currentClass != null;
+                        addSignature(currentClass, line);
+                    }
+                }
+            }
+        } catch (Exception e) {
+            throw new RuntimeException("syntax error in definition line: " + currentLine, e);
+        }
     }
 
     private void addTransforms() {
+        Type booleanType = getType("boolean");
+        Type objectType = getType("Object");
+        Type defType = getType("def");
+        Type booleanobjType = getType("Boolean");
+        Type byteType = getType("byte");
+        Type shortType = getType("short");
+        Type intType = getType("int");
+        Type charType = getType("char");
+        Type longType = getType("long");
+        Type floatType = getType("float");
+        Type doubleType = getType("double");
+        Type numberType = getType("Number");
+        Type byteobjType = getType("Byte");
+        Type shortobjType = getType("Short");
+        Type charobjType = getType("Character");
+        Type intobjType = getType("Integer");
+        Type longobjType = getType("Long");
+        Type floatobjType = getType("Float");
+        Type doubleobjType = getType("Double");
+        Type stringType = getType("String");
+        
         addTransform(booleanType, objectType, "Boolean", "valueOf", true, false);
         addTransform(booleanType, defType, "Boolean", "valueOf", true, false);
         addTransform(booleanType, booleanobjType, "Boolean", "valueOf", true, false);
@@ -1543,50 +811,6 @@ public final class Definition {
         addTransform(stringType, charobjType, "Utility", "StringToCharacter", true, true);
     }
 
-    private void addRuntimeClasses() {
-        addRuntimeClass(booleanType.struct);
-        addRuntimeClass(byteType.struct);
-        addRuntimeClass(shortType.struct);
-        addRuntimeClass(charType.struct);
-        addRuntimeClass(intType.struct);
-        addRuntimeClass(longType.struct);
-        addRuntimeClass(floatType.struct);
-        addRuntimeClass(doubleType.struct);
-
-        addRuntimeClass(booleanobjType.struct);
-        addRuntimeClass(byteobjType.struct);
-        addRuntimeClass(shortobjType.struct);
-        addRuntimeClass(charobjType.struct);
-        addRuntimeClass(intobjType.struct);
-        addRuntimeClass(longobjType.struct);
-        addRuntimeClass(floatobjType.struct);
-        addRuntimeClass(doubleobjType.struct);
-
-        addRuntimeClass(objectType.struct);
-        addRuntimeClass(numberType.struct);
-        addRuntimeClass(charseqType.struct);
-        addRuntimeClass(stringType.struct);
-
-        addRuntimeClass(oitrType.struct);
-        addRuntimeClass(ocollectionType.struct);
-        addRuntimeClass(olistType.struct);
-        addRuntimeClass(oarraylistType.struct);
-        addRuntimeClass(osetType.struct);
-        addRuntimeClass(ohashsetType.struct);
-        addRuntimeClass(oomapType.struct);
-        addRuntimeClass(oohashmapType.struct);
-
-        addRuntimeClass(exceptionType.struct);
-
-        addRuntimeClass(geoPointType.struct);
-        addRuntimeClass(stringsType.struct);
-        addRuntimeClass(longsType.struct);
-        addRuntimeClass(doublesType.struct);
-        addRuntimeClass(geoPointsType.struct);
-
-        addRuntimeClass(featureTestType.struct);
-    }
-
     private final void addStruct(final String name, final Class<?> clazz) {
         if (!name.matches("^[_a-zA-Z][<>,_a-zA-Z0-9]*$")) {
             throw new IllegalArgumentException("Invalid struct name [" + name + "].");
@@ -1599,9 +823,10 @@ public final class Definition {
         final Struct struct = new Struct(name, clazz, org.objectweb.asm.Type.getType(clazz));
 
         structsMap.put(name, struct);
+        simpleTypesMap.put(name, getType(name));
     }
-
-    private final void addConstructor(final String struct, final String name, final Type[] args, final Type[] genargs) {
+    
+    private final void addConstructorInternal(final String struct, final String name, final Type[] args) {
         final Struct owner = structsMap.get(struct);
 
         if (owner == null) {
@@ -1634,14 +859,6 @@ public final class Definition {
         final Class<?>[] classes = new Class<?>[args.length];
 
         for (int count = 0; count < classes.length; ++count) {
-            if (genargs != null) {
-                if (!args[count].clazz.isAssignableFrom(genargs[count].clazz)) {
-                    throw new ClassCastException("Generic argument [" + genargs[count].name + "]" +
-                        " is not a sub class of [" + args[count].name + "] in the constructor" +
-                        " [" + name + " ] from the struct [" + owner.name + "].");
-                }
-            }
-
             classes[count] = args[count].clazz;
         }
 
@@ -1655,79 +872,96 @@ public final class Definition {
         }
 
         final org.objectweb.asm.commons.Method asm = org.objectweb.asm.commons.Method.getMethod(reflect);
-        final Constructor constructor =
-            new Constructor(name, owner, Arrays.asList(genargs != null ? genargs : args), asm, reflect);
+        final Constructor constructor = new Constructor(name, owner, Arrays.asList(args), asm, reflect);
 
         owner.constructors.put(methodKey, constructor);
     }
+    
+    /** 
+     * Adds a new signature to the definition.
+     * <p>
+     * Signatures have the following forms:
+     * <ul>
+     *   <li>{@code void method(String,int)}
+     *   <li>{@code boolean field}
+     *   <li>{@code Class <init>(String)}
+     * </ul>
+     * no spaces allowed.
+     */
+    private final void addSignature(String className, String signature) {
+        String elements[] = signature.split("\u0020");
+        if (elements.length != 2) {
+            throw new IllegalArgumentException("Malformed signature: " + signature);
+        }
+        // method or field type (e.g. return type)
+        Type rtn = getType(elements[0]);
+        int parenIndex = elements[1].indexOf('(');
+        if (parenIndex != -1) {
+            // method or ctor
+            int parenEnd = elements[1].indexOf(')');
+            final Type args[];
+            if (parenEnd > parenIndex + 1) {
+                String arguments[] = elements[1].substring(parenIndex + 1, parenEnd).split(",");
+                args = new Type[arguments.length];
+                for (int i = 0; i < arguments.length; i++) {
+                    args[i] = getType(arguments[i]);
+                }
+            } else {
+                args = new Type[0];
+            }
+            String methodName = elements[1].substring(0, parenIndex);
+            if (methodName.equals("<init>")) {
+                if (!elements[0].equals(className)) {
+                    throw new IllegalArgumentException("Constructors must return their own type");
+                }
+                addConstructorInternal(className, "new", args);
+            } else {
+                if (methodName.indexOf('/') >= 0) {
+                    String nameAndAlias[] = methodName.split("/");
+                    if (nameAndAlias.length != 2) {
+                        throw new IllegalArgumentException("Currently only two aliases are allowed!");
+                    }
+                    addMethodInternal(className, nameAndAlias[0], nameAndAlias[1], rtn, args);
+                } else {
+                    addMethodInternal(className, methodName, null, rtn, args);
+                }
+            }
+        } else {
+            // field
+            addFieldInternal(className, elements[1], null, rtn);
+        }
+    }
 
-    private final void addMethod(final String struct, final String name, final String alias, final boolean statik,
-                                final Type rtn, final Type[] args, final Type genrtn, final Type[] genargs) {
+    private final void addMethodInternal(final String struct, final String name, final String alias, 
+                                         final Type rtn, final Type[] args) {
         final Struct owner = structsMap.get(struct);
 
         if (owner == null) {
             throw new IllegalArgumentException("Owner struct [" + struct + "] not defined" +
-                " for " + (statik ? "function" : "method") + " [" + name + "].");
+                " for method [" + name + "].");
         }
 
         if (!name.matches("^[_a-zA-Z][_a-zA-Z0-9]*$")) {
-            throw new IllegalArgumentException("Invalid " + (statik ? "static method" : "method") +
-                " name [" + name + "] with the struct [" + owner.name + "].");
+            throw new IllegalArgumentException("Invalid method name" +
+                " [" + name + "] with the struct [" + owner.name + "].");
         }
 
         MethodKey methodKey = new MethodKey(name, args.length);
 
         if (owner.constructors.containsKey(methodKey)) {
-            throw new IllegalArgumentException("Constructors and " + (statik ? "static methods" : "methods") +
+            throw new IllegalArgumentException("Constructors and methods" +
                 " may not have the same signature [" + methodKey + "] within the same struct" +
                 " [" + owner.name + "].");
         }
 
-        if (owner.staticMethods.containsKey(methodKey)) {
-            if (statik) {
-                throw new IllegalArgumentException(
-                    "Duplicate static method signature [" + methodKey + "] found within the struct [" + owner.name + "].");
-            } else {
-                throw new IllegalArgumentException("Static methods and methods may not have the same signature" +
-                    " [" + methodKey + "] within the same struct [" + owner.name + "].");
-            }
-        }
-
-        if (owner.methods.containsKey(methodKey)) {
-            if (statik) {
-                throw new IllegalArgumentException("Static methods and methods may not have the same signature" +
-                    " [" + methodKey + "] within the same struct [" + owner.name + "].");
-            } else {
-                throw new IllegalArgumentException("Duplicate method signature [" + methodKey + "]" +
-                    " found within the struct [" + owner.name + "].");
-            }
-        }
-
-        if (genrtn != null) {
-            if (!rtn.clazz.isAssignableFrom(genrtn.clazz)) {
-                throw new ClassCastException("Generic return [" + genrtn.clazz.getCanonicalName() + "]" +
-                    " is not a sub class of [" + rtn.clazz.getCanonicalName() + "] in the method" +
-                    " [" + name + " ] from the struct [" + owner.name + "].");
-            }
-        }
-
-        if (genargs != null && genargs.length != args.length) {
-            throw new IllegalArgumentException("Generic arguments arity [" +  genargs.length + "] is not the same as " +
-                (statik ? "function" : "method") + " [" + name + "] arguments arity" +
-                " [" + args.length + "] within the struct [" + owner.name + "].");
+        if (owner.staticMethods.containsKey(methodKey) || owner.methods.containsKey(methodKey)) {
+            throw new IllegalArgumentException(
+                "Duplicate  method signature [" + methodKey + "] found within the struct [" + owner.name + "].");
         }
 
         final Class<?>[] classes = new Class<?>[args.length];
 
         for (int count = 0; count < classes.length; ++count) {
-            if (genargs != null) {
-                if (!args[count].clazz.isAssignableFrom(genargs[count].clazz)) {
-                    throw new ClassCastException("Generic argument [" + genargs[count].name + "] is not a sub class" +
-                        " of [" + args[count].name + "] in the " + (statik ? "function" : "method") +
-                        " [" + name + " ] from the struct [" + owner.name + "].");
-                }
-            }
-
             classes[count] = args[count].clazz;
         }
 
@@ -1736,15 +970,15 @@ public final class Definition {
         try {
             reflect = owner.clazz.getMethod(alias == null ? name : alias, classes);
         } catch (final NoSuchMethodException exception) {
-            throw new IllegalArgumentException((statik ? "Function" : "Method") +
-                " [" + (alias == null ? name : alias) + "] not found for class [" + owner.clazz.getName() + "]" +
+            throw new IllegalArgumentException("Method [" + (alias == null ? name : alias) + 
+                "] not found for class [" + owner.clazz.getName() + "]" +
                 " with arguments " + Arrays.toString(classes) + ".");
         }
 
         if (!reflect.getReturnType().equals(rtn.clazz)) {
             throw new IllegalArgumentException("Specified return type class [" + rtn.clazz + "]" +
-                " does not match the found return type class [" + reflect.getReturnType() + "] for the " +
-                (statik ? "function" : "method") + " [" + name + "]" +
+                " does not match the found return type class [" + reflect.getReturnType() + "] for the" +
+                " method [" + name + "]" +
                 " within the struct [" + owner.name + "].");
         }
 
@@ -1760,67 +994,33 @@ public final class Definition {
                 " with arguments " + Arrays.toString(classes) + ".");
         }
 
-        final Method method = new Method(name, owner, genrtn != null ? genrtn : rtn,
-            Arrays.asList(genargs != null ? genargs : args), asm, reflect, handle);
+        final Method method = new Method(name, owner, rtn, Arrays.asList(args), asm, reflect, handle);
         final int modifiers = reflect.getModifiers();
 
-        if (statik) {
-            if (!java.lang.reflect.Modifier.isStatic(modifiers)) {
-                throw new IllegalArgumentException("Function [" + name + "]" +
-                    " within the struct [" + owner.name + "] is not linked to a static Java method.");
-            }
-
+        if (java.lang.reflect.Modifier.isStatic(modifiers)) {
             owner.staticMethods.put(methodKey, method);
         } else {
-            if (java.lang.reflect.Modifier.isStatic(modifiers)) {
-                throw new IllegalArgumentException("Method [" + name + "]" +
-                    " within the struct [" + owner.name + "] is not linked to a non-static Java method.");
-            }
-
             owner.methods.put(methodKey, method);
         }
     }
-
-    private final void addField(final String struct, final String name, final String alias,
-                               final boolean statik, final Type type, final Type generic) {
+    
+    private final void addFieldInternal(final String struct, final String name, final String alias,
+                                        final Type type) {
         final Struct owner = structsMap.get(struct);
 
         if (owner == null) {
             throw new IllegalArgumentException("Owner struct [" + struct + "] not defined for " +
-                (statik ? "static" : "member") + " [" + name + "].");
+                " field [" + name + "].");
         }
 
         if (!name.matches("^[_a-zA-Z][_a-zA-Z0-9]*$")) {
-            throw new IllegalArgumentException("Invalid " + (statik ? "static" : "member") +
+            throw new IllegalArgumentException("Invalid field " +
                 " name [" + name + "] with the struct [" + owner.name + "].");
         }
 
-        if (owner.staticMembers.containsKey(name)) {
-            if (statik) {
-                throw new IllegalArgumentException("Duplicate static name [" + name + "]" +
-                    " found within the struct [" + owner.name + "].");
-            } else {
-                throw new IllegalArgumentException("Statics and members may not have the same name " +
-                    "[" + name + "] within the same struct [" + owner.name + "].");
-            }
-        }
-
-        if (owner.members.containsKey(name)) {
-            if (statik) {
-                throw new IllegalArgumentException("Statics and members may not have the same name " +
-                    "[" + name + "] within the same struct [" + owner.name + "].");
-            } else {
-                throw new IllegalArgumentException("Duplicate member name [" + name + "]" +
-                    " found within the struct [" + owner.name + "].");
-            }
-        }
-
-        if (generic != null) {
-            if (!type.clazz.isAssignableFrom(generic.clazz)) {
-                throw new ClassCastException("Generic type [" + generic.clazz.getCanonicalName() + "]" +
-                    " is not a sub class of [" + type.clazz.getCanonicalName() + "] for the field" +
-                    " [" + name + " ] from the struct [" + owner.name + "].");
-            }
+        if (owner.staticMembers.containsKey(name) || owner.members.containsKey(name)) {
+             throw new IllegalArgumentException("Duplicate field name [" + name + "]" +
+                     " found within the struct [" + owner.name + "].");
         }
 
         java.lang.reflect.Field reflect;
@@ -1831,12 +1031,15 @@ public final class Definition {
             throw new IllegalArgumentException("Field [" + (alias == null ? name : alias) + "]" +
                 " not found for class [" + owner.clazz.getName() + "].");
         }
+        
+        final int modifiers = reflect.getModifiers();
+        boolean isStatic = java.lang.reflect.Modifier.isStatic(modifiers);
 
         MethodHandle getter = null;
         MethodHandle setter = null;
 
         try {
-            if (!statik) {
+            if (!isStatic) {
                 getter = MethodHandles.publicLookup().unreflectGetter(reflect);
                 setter = MethodHandles.publicLookup().unreflectSetter(reflect);
             }
@@ -1845,42 +1048,33 @@ public final class Definition {
                 " not found for class [" + owner.clazz.getName() + "].");
         }
 
-        final Field field = new Field(name, owner, generic == null ? type : generic, type, reflect, getter, setter);
-        final int modifiers = reflect.getModifiers();
-
-        if (statik) {
-            if (!java.lang.reflect.Modifier.isStatic(modifiers)) {
-                throw new IllegalArgumentException();
-            }
+        final Field field = new Field(name, owner, type, type, reflect, getter, setter);
 
+        if (isStatic) {
+            // require that all static fields are static final
             if (!java.lang.reflect.Modifier.isFinal(modifiers)) {
                 throw new IllegalArgumentException("Static [" + name + "]" +
-                    " within the struct [" + owner.name + "] is not linked to static Java field.");
+                    " within the struct [" + owner.name + "] is not final.");
             }
 
             owner.staticMembers.put(alias == null ? name : alias, field);
         } else {
-            if (java.lang.reflect.Modifier.isStatic(modifiers)) {
-                throw new IllegalArgumentException("Member [" + name + "]" +
-                    " within the struct [" + owner.name + "] is not linked to non-static Java field.");
-            }
-
             owner.members.put(alias == null ? name : alias, field);
         }
     }
 
-    private final void copyStruct(final String struct, final String... children) {
+    private final void copyStruct(final String struct, List<String> children) {
         final Struct owner = structsMap.get(struct);
 
         if (owner == null) {
             throw new IllegalArgumentException("Owner struct [" + struct + "] not defined for copy.");
         }
 
-        for (int count = 0; count < children.length; ++count) {
-            final Struct child = structsMap.get(children[count]);
+        for (int count = 0; count < children.size(); ++count) {
+            final Struct child = structsMap.get(children.get(count));
 
             if (struct == null) {
-                throw new IllegalArgumentException("Child struct [" + children[count] + "]" +
+                throw new IllegalArgumentException("Child struct [" + children.get(count) + "]" +
                     " not defined for copy to owner struct [" + owner.name + "].");
             }
 
@@ -2133,6 +1327,11 @@ public final class Definition {
     }
 
     public final Type getType(final String name) {
+        // simple types (e.g. 0 array dimensions) are a simple hash lookup for speed
+        Type simple = simpleTypesMap.get(name);
+        if (simple != null) {
+            return simple;
+        }
         final int dimensions = getDimensions(name);
         final String structstr = dimensions == 0 ? name : name.substring(0, name.indexOf('['));
         final Struct struct = structsMap.get(structstr);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index fbb1d246a83..bb362558796 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -252,29 +252,29 @@ public final class MethodWriter extends GeneratorAdapter {
             switch (sort) {
                 case INT:
                     switch (operation) {
-                        case MUL: invokeStatic(definition.mathType.type,    MULEXACT_INT);     break;
-                        case DIV: invokeStatic(definition.utilityType.type, DIVWOOVERLOW_INT); break;
-                        case ADD: invokeStatic(definition.mathType.type,    ADDEXACT_INT);     break;
-                        case SUB: invokeStatic(definition.mathType.type,    SUBEXACT_INT);     break;
+                        case MUL: invokeStatic(definition.getType("Math").type,    MULEXACT_INT);     break;
+                        case DIV: invokeStatic(definition.getType("Utility").type, DIVWOOVERLOW_INT); break;
+                        case ADD: invokeStatic(definition.getType("Math").type,    ADDEXACT_INT);     break;
+                        case SUB: invokeStatic(definition.getType("Math").type,    SUBEXACT_INT);     break;
                     }
 
                     break;
                 case LONG:
                     switch (operation) {
-                        case MUL: invokeStatic(definition.mathType.type,    MULEXACT_LONG);     break;
-                        case DIV: invokeStatic(definition.utilityType.type, DIVWOOVERLOW_LONG); break;
-                        case ADD: invokeStatic(definition.mathType.type,    ADDEXACT_LONG);     break;
-                        case SUB: invokeStatic(definition.mathType.type,    SUBEXACT_LONG);     break;
+                        case MUL: invokeStatic(definition.getType("Math").type,    MULEXACT_LONG);     break;
+                        case DIV: invokeStatic(definition.getType("Utility").type, DIVWOOVERLOW_LONG); break;
+                        case ADD: invokeStatic(definition.getType("Math").type,    ADDEXACT_LONG);     break;
+                        case SUB: invokeStatic(definition.getType("Math").type,    SUBEXACT_LONG);     break;
                     }
 
                     break;
                 case FLOAT:
                     switch (operation) {
-                        case MUL: invokeStatic(definition.utilityType.type, MULWOOVERLOW_FLOAT); break;
-                        case DIV: invokeStatic(definition.utilityType.type, DIVWOOVERLOW_FLOAT); break;
-                        case REM: invokeStatic(definition.utilityType.type, REMWOOVERLOW_FLOAT); break;
-                        case ADD: invokeStatic(definition.utilityType.type, ADDWOOVERLOW_FLOAT); break;
-                        case SUB: invokeStatic(definition.utilityType.type, SUBWOOVERLOW_FLOAT); break;
+                        case MUL: invokeStatic(definition.getType("Utility").type, MULWOOVERLOW_FLOAT); break;
+                        case DIV: invokeStatic(definition.getType("Utility").type, DIVWOOVERLOW_FLOAT); break;
+                        case REM: invokeStatic(definition.getType("Utility").type, REMWOOVERLOW_FLOAT); break;
+                        case ADD: invokeStatic(definition.getType("Utility").type, ADDWOOVERLOW_FLOAT); break;
+                        case SUB: invokeStatic(definition.getType("Utility").type, SUBWOOVERLOW_FLOAT); break;
                         default:
                             throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
                     }
@@ -282,11 +282,11 @@ public final class MethodWriter extends GeneratorAdapter {
                     break;
                 case DOUBLE:
                     switch (operation) {
-                        case MUL: invokeStatic(definition.utilityType.type, MULWOOVERLOW_DOUBLE); break;
-                        case DIV: invokeStatic(definition.utilityType.type, DIVWOOVERLOW_DOUBLE); break;
-                        case REM: invokeStatic(definition.utilityType.type, REMWOOVERLOW_DOUBLE); break;
-                        case ADD: invokeStatic(definition.utilityType.type, ADDWOOVERLOW_DOUBLE); break;
-                        case SUB: invokeStatic(definition.utilityType.type, SUBWOOVERLOW_DOUBLE); break;
+                        case MUL: invokeStatic(definition.getType("Utility").type, MULWOOVERLOW_DOUBLE); break;
+                        case DIV: invokeStatic(definition.getType("Utility").type, DIVWOOVERLOW_DOUBLE); break;
+                        case REM: invokeStatic(definition.getType("Utility").type, REMWOOVERLOW_DOUBLE); break;
+                        case ADD: invokeStatic(definition.getType("Utility").type, ADDWOOVERLOW_DOUBLE); break;
+                        case SUB: invokeStatic(definition.getType("Utility").type, SUBWOOVERLOW_DOUBLE); break;
                         default:
                             throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
                     }
@@ -305,17 +305,17 @@ public final class MethodWriter extends GeneratorAdapter {
 
             if (sort == Sort.DEF) {
                 switch (operation) {
-                    case MUL:   invokeStatic(definition.defobjType.type, DEF_MUL_CALL); break;
-                    case DIV:   invokeStatic(definition.defobjType.type, DEF_DIV_CALL); break;
-                    case REM:   invokeStatic(definition.defobjType.type, DEF_REM_CALL); break;
-                    case ADD:   invokeStatic(definition.defobjType.type, DEF_ADD_CALL); break;
-                    case SUB:   invokeStatic(definition.defobjType.type, DEF_SUB_CALL); break;
-                    case LSH:   invokeStatic(definition.defobjType.type, DEF_LSH_CALL); break;
-                    case USH:   invokeStatic(definition.defobjType.type, DEF_RSH_CALL); break;
-                    case RSH:   invokeStatic(definition.defobjType.type, DEF_USH_CALL); break;
-                    case BWAND: invokeStatic(definition.defobjType.type, DEF_AND_CALL); break;
-                    case XOR:   invokeStatic(definition.defobjType.type, DEF_XOR_CALL); break;
-                    case BWOR:  invokeStatic(definition.defobjType.type, DEF_OR_CALL);  break;
+                    case MUL:   invokeStatic(definition.getType("Def").type, DEF_MUL_CALL); break;
+                    case DIV:   invokeStatic(definition.getType("Def").type, DEF_DIV_CALL); break;
+                    case REM:   invokeStatic(definition.getType("Def").type, DEF_REM_CALL); break;
+                    case ADD:   invokeStatic(definition.getType("Def").type, DEF_ADD_CALL); break;
+                    case SUB:   invokeStatic(definition.getType("Def").type, DEF_SUB_CALL); break;
+                    case LSH:   invokeStatic(definition.getType("Def").type, DEF_LSH_CALL); break;
+                    case USH:   invokeStatic(definition.getType("Def").type, DEF_RSH_CALL); break;
+                    case RSH:   invokeStatic(definition.getType("Def").type, DEF_USH_CALL); break;
+                    case BWAND: invokeStatic(definition.getType("Def").type, DEF_AND_CALL); break;
+                    case XOR:   invokeStatic(definition.getType("Def").type, DEF_XOR_CALL); break;
+                    case BWOR:  invokeStatic(definition.getType("Def").type, DEF_OR_CALL);  break;
                     default:
                         throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
                 }
@@ -350,107 +350,107 @@ public final class MethodWriter extends GeneratorAdapter {
         final Definition definition, final Sort fsort, final Sort tsort) {
         if (fsort == Sort.DOUBLE) {
             if (tsort == Sort.FLOAT) {
-                invokeStatic(definition.utilityType.type, TOFLOATWOOVERFLOW_DOUBLE);
+                invokeStatic(definition.getType("Utility").type, TOFLOATWOOVERFLOW_DOUBLE);
             } else if (tsort == Sort.FLOAT_OBJ) {
-                invokeStatic(definition.utilityType.type, TOFLOATWOOVERFLOW_DOUBLE);
-                checkCast(definition.floatobjType.type);
+                invokeStatic(definition.getType("Utility").type, TOFLOATWOOVERFLOW_DOUBLE);
+                checkCast(definition.getType("Float").type);
             } else if (tsort == Sort.LONG) {
-                invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_DOUBLE);
+                invokeStatic(definition.getType("Utility").type, TOLONGWOOVERFLOW_DOUBLE);
             } else if (tsort == Sort.LONG_OBJ) {
-                invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_DOUBLE);
-                checkCast(definition.longobjType.type);
+                invokeStatic(definition.getType("Utility").type, TOLONGWOOVERFLOW_DOUBLE);
+                checkCast(definition.getType("Long").type);
             } else if (tsort == Sort.INT) {
-                invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_DOUBLE);
+                invokeStatic(definition.getType("Utility").type, TOINTWOOVERFLOW_DOUBLE);
             } else if (tsort == Sort.INT_OBJ) {
-                invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_DOUBLE);
-                checkCast(definition.intobjType.type);
+                invokeStatic(definition.getType("Utility").type, TOINTWOOVERFLOW_DOUBLE);
+                checkCast(definition.getType("Integer").type);
             } else if (tsort == Sort.CHAR) {
-                invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_DOUBLE);
+                invokeStatic(definition.getType("Utility").type, TOCHARWOOVERFLOW_DOUBLE);
             } else if (tsort == Sort.CHAR_OBJ) {
-                invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_DOUBLE);
-                checkCast(definition.charobjType.type);
+                invokeStatic(definition.getType("Utility").type, TOCHARWOOVERFLOW_DOUBLE);
+                checkCast(definition.getType("Character").type);
             } else if (tsort == Sort.SHORT) {
-                invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_DOUBLE);
+                invokeStatic(definition.getType("Utility").type, TOSHORTWOOVERFLOW_DOUBLE);
             } else if (tsort == Sort.SHORT_OBJ) {
-                invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_DOUBLE);
-                checkCast(definition.shortobjType.type);
+                invokeStatic(definition.getType("Utility").type, TOSHORTWOOVERFLOW_DOUBLE);
+                checkCast(definition.getType("Short").type);
             } else if (tsort == Sort.BYTE) {
-                invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_DOUBLE);
+                invokeStatic(definition.getType("Utility").type, TOBYTEWOOVERFLOW_DOUBLE);
             } else if (tsort == Sort.BYTE_OBJ) {
-                invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_DOUBLE);
-                checkCast(definition.byteobjType.type);
+                invokeStatic(definition.getType("Utility").type, TOBYTEWOOVERFLOW_DOUBLE);
+                checkCast(definition.getType("Byte").type);
             } else {
                 return false;
             }
         } else if (fsort == Sort.FLOAT) {
             if (tsort == Sort.LONG) {
-                invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_FLOAT);
+                invokeStatic(definition.getType("Utility").type, TOLONGWOOVERFLOW_FLOAT);
             } else if (tsort == Sort.LONG_OBJ) {
-                invokeStatic(definition.utilityType.type, TOLONGWOOVERFLOW_FLOAT);
-                checkCast(definition.longobjType.type);
+                invokeStatic(definition.getType("Utility").type, TOLONGWOOVERFLOW_FLOAT);
+                checkCast(definition.getType("Long").type);
             } else if (tsort == Sort.INT) {
-                invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_FLOAT);
+                invokeStatic(definition.getType("Utility").type, TOINTWOOVERFLOW_FLOAT);
             } else if (tsort == Sort.INT_OBJ) {
-                invokeStatic(definition.utilityType.type, TOINTWOOVERFLOW_FLOAT);
-                checkCast(definition.intobjType.type);
+                invokeStatic(definition.getType("Utility").type, TOINTWOOVERFLOW_FLOAT);
+                checkCast(definition.getType("Integer").type);
             } else if (tsort == Sort.CHAR) {
-                invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_FLOAT);
+                invokeStatic(definition.getType("Utility").type, TOCHARWOOVERFLOW_FLOAT);
             } else if (tsort == Sort.CHAR_OBJ) {
-                invokeStatic(definition.utilityType.type, TOCHARWOOVERFLOW_FLOAT);
-                checkCast(definition.charobjType.type);
+                invokeStatic(definition.getType("Utility").type, TOCHARWOOVERFLOW_FLOAT);
+                checkCast(definition.getType("Character").type);
             } else if (tsort == Sort.SHORT) {
-                invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_FLOAT);
+                invokeStatic(definition.getType("Utility").type, TOSHORTWOOVERFLOW_FLOAT);
             } else if (tsort == Sort.SHORT_OBJ) {
-                invokeStatic(definition.utilityType.type, TOSHORTWOOVERFLOW_FLOAT);
-                checkCast(definition.shortobjType.type);
+                invokeStatic(definition.getType("Utility").type, TOSHORTWOOVERFLOW_FLOAT);
+                checkCast(definition.getType("Short").type);
             } else if (tsort == Sort.BYTE) {
-                invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_FLOAT);
+                invokeStatic(definition.getType("Utility").type, TOBYTEWOOVERFLOW_FLOAT);
             } else if (tsort == Sort.BYTE_OBJ) {
-                invokeStatic(definition.utilityType.type, TOBYTEWOOVERFLOW_FLOAT);
-                checkCast(definition.byteobjType.type);
+                invokeStatic(definition.getType("Utility").type, TOBYTEWOOVERFLOW_FLOAT);
+                checkCast(definition.getType("Byte").type);
             } else {
                 return false;
             }
         } else if (fsort == Sort.LONG) {
             if (tsort == Sort.INT) {
-                invokeStatic(definition.mathType.type, TOINTEXACT_LONG);
+                invokeStatic(definition.getType("Math").type, TOINTEXACT_LONG);
             } else if (tsort == Sort.INT_OBJ) {
-                invokeStatic(definition.mathType.type, TOINTEXACT_LONG);
-                checkCast(definition.intobjType.type);
+                invokeStatic(definition.getType("Math").type, TOINTEXACT_LONG);
+                checkCast(definition.getType("Integer").type);
             } else if (tsort == Sort.CHAR) {
-                invokeStatic(definition.utilityType.type, TOCHAREXACT_LONG);
+                invokeStatic(definition.getType("Utility").type, TOCHAREXACT_LONG);
             } else if (tsort == Sort.CHAR_OBJ) {
-                invokeStatic(definition.utilityType.type, TOCHAREXACT_LONG);
-                checkCast(definition.charobjType.type);
+                invokeStatic(definition.getType("Utility").type, TOCHAREXACT_LONG);
+                checkCast(definition.getType("Character").type);
             } else if (tsort == Sort.SHORT) {
-                invokeStatic(definition.utilityType.type, TOSHORTEXACT_LONG);
+                invokeStatic(definition.getType("Utility").type, TOSHORTEXACT_LONG);
             } else if (tsort == Sort.SHORT_OBJ) {
-                invokeStatic(definition.utilityType.type, TOSHORTEXACT_LONG);
-                checkCast(definition.shortobjType.type);
+                invokeStatic(definition.getType("Utility").type, TOSHORTEXACT_LONG);
+                checkCast(definition.getType("Short").type);
             } else if (tsort == Sort.BYTE) {
-                invokeStatic(definition.utilityType.type, TOBYTEEXACT_LONG);
+                invokeStatic(definition.getType("Utility").type, TOBYTEEXACT_LONG);
             } else if (tsort == Sort.BYTE_OBJ) {
-                invokeStatic(definition.utilityType.type, TOBYTEEXACT_LONG);
-                checkCast(definition.byteobjType.type);
+                invokeStatic(definition.getType("Utility").type, TOBYTEEXACT_LONG);
+                checkCast(definition.getType("Byte").type);
             } else {
                 return false;
             }
         } else if (fsort == Sort.INT) {
             if (tsort == Sort.CHAR) {
-                invokeStatic(definition.utilityType.type, TOCHAREXACT_INT);
+                invokeStatic(definition.getType("Utility").type, TOCHAREXACT_INT);
             } else if (tsort == Sort.CHAR_OBJ) {
-                invokeStatic(definition.utilityType.type, TOCHAREXACT_INT);
-                checkCast(definition.charobjType.type);
+                invokeStatic(definition.getType("Utility").type, TOCHAREXACT_INT);
+                checkCast(definition.getType("Character").type);
             } else if (tsort == Sort.SHORT) {
-                invokeStatic(definition.utilityType.type, TOSHORTEXACT_INT);
+                invokeStatic(definition.getType("Utility").type, TOSHORTEXACT_INT);
             } else if (tsort == Sort.SHORT_OBJ) {
-                invokeStatic(definition.utilityType.type, TOSHORTEXACT_INT);
-                checkCast(definition.shortobjType.type);
+                invokeStatic(definition.getType("Utility").type, TOSHORTEXACT_INT);
+                checkCast(definition.getType("Short").type);
             } else if (tsort == Sort.BYTE) {
-                invokeStatic(definition.utilityType.type, TOBYTEEXACT_INT);
+                invokeStatic(definition.getType("Utility").type, TOBYTEEXACT_INT);
             } else if (tsort == Sort.BYTE_OBJ) {
-                invokeStatic(definition.utilityType.type, TOBYTEEXACT_INT);
-                checkCast(definition.byteobjType.type);
+                invokeStatic(definition.getType("Utility").type, TOBYTEEXACT_INT);
+                checkCast(definition.getType("Byte").type);
             } else {
                 return false;
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
index e5998948d62..de774344ee2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.painless;
 
+
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptMode;
@@ -29,6 +30,16 @@ import org.elasticsearch.script.ScriptModule;
  */
 public final class PainlessPlugin extends Plugin {
 
+    // parse our definition at startup (not on the user's first script)
+    // compilation process is sandboxed and has no file access.
+    static {
+        try {
+            Class.forName("org.elasticsearch.painless.Definition");
+        } catch (ClassNotFoundException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     @Override
     public String name() {
         return "lang-painless";
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
index dafc6aaba8a..1d11dae720f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
@@ -38,9 +38,7 @@ import java.security.AccessController;
 import java.security.Permissions;
 import java.security.PrivilegedAction;
 import java.security.ProtectionDomain;
-import java.util.Collections;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 /**
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
index adf930b017c..d0b4d706f51 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
@@ -101,35 +101,35 @@ public final class Variables {
         // Method variables.
 
         // This reference.  Internal use only.
-        addVariable("[" + Reserved.THIS + "]"  , definition.execType.name, Reserved.THIS  , true, true);
+        addVariable("[" + Reserved.THIS + "]"  , "Executable", Reserved.THIS  , true, true);
 
         // Input map of variables passed to the script.  TODO: Rename to 'params' since that will be its use.
-        addVariable("[" + Reserved.PARAMS + "]", definition.smapType.name, Reserved.PARAMS, true, true);
+        addVariable("[" + Reserved.PARAMS + "]", "Map", Reserved.PARAMS, true, true);
 
         // Scorer parameter passed to the script.  Internal use only.
-        addVariable("[" + Reserved.SCORER + "]", definition.defType.name , Reserved.SCORER, true, true);
+        addVariable("[" + Reserved.SCORER + "]", "def", Reserved.SCORER, true, true);
 
-        // Doc parameter passed to the script. TODO: Currently working as a Map<String,Def>, we can do better?
-        addVariable("[" + Reserved.DOC + "]"   , definition.smapType.name, Reserved.DOC   , true, true);
+        // Doc parameter passed to the script. TODO: Currently working as a Map, we can do better?
+        addVariable("[" + Reserved.DOC + "]"   , "Map", Reserved.DOC   , true, true);
 
         // Aggregation _value parameter passed to the script.
-        addVariable("[" + Reserved.VALUE + "]" , definition.defType.name , Reserved.VALUE , true, true);
+        addVariable("[" + Reserved.VALUE + "]" , "def", Reserved.VALUE , true, true);
 
         // Shortcut variables.
 
         // Document's score as a read-only double.
         if (reserved.score) {
-            addVariable("[" + Reserved.SCORE + "]", definition.doubleType.name, Reserved.SCORE, true, true);
+            addVariable("[" + Reserved.SCORE + "]", "double", Reserved.SCORE, true, true);
         }
 
         // The ctx map set by executable scripts as a read-only map.
         if (reserved.ctx) {
-            addVariable("[" + Reserved.CTX + "]", definition.smapType.name, Reserved.CTX, true, true);
+            addVariable("[" + Reserved.CTX + "]", "Map", Reserved.CTX, true, true);
         }
 
         // Loop counter to catch infinite loops.  Internal use only.
         if (reserved.loop && settings.getMaxLoopCounter() > 0) {
-            addVariable("[" + Reserved.LOOP + "]", definition.intType.name, Reserved.LOOP, true, true);
+            addVariable("[" + Reserved.LOOP + "]", "int", Reserved.LOOP, true, true);
         }
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 07fe1ff3447..18b9de80fb6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -308,7 +308,7 @@ public final class EBinary extends AExpression {
         }
 
         left.expected = promote;
-        right.expected = definition.intType;
+        right.expected = definition.getType("int");
         right.explicit = true;
 
         left = left.cast(settings, definition, variables);
@@ -341,7 +341,7 @@ public final class EBinary extends AExpression {
         }
 
         left.expected = promote;
-        right.expected = definition.intType;
+        right.expected = definition.getType("int");
         right.explicit = true;
 
         left = left.cast(settings, definition, variables);
@@ -374,7 +374,7 @@ public final class EBinary extends AExpression {
         }
 
         left.expected = promote;
-        right.expected = definition.intType;
+        right.expected = definition.getType("int");
         right.explicit = true;
 
         left = left.cast(settings, definition, variables);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
index 7f9f7dee000..4a6967c5285 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
@@ -45,11 +45,11 @@ public final class EBool extends AExpression {
 
     @Override
     void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        left.expected = definition.booleanType;
+        left.expected = definition.getType("boolean");
         left.analyze(settings, definition, variables);
         left = left.cast(settings, definition, variables);
 
-        right.expected = definition.booleanType;
+        right.expected = definition.getType("boolean");
         right.analyze(settings, definition, variables);
         right = right.cast(settings, definition, variables);
 
@@ -63,7 +63,7 @@ public final class EBool extends AExpression {
             }
         }
 
-        actual = definition.booleanType;
+        actual = definition.getType("boolean");
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
index 9b9f0917546..631b47db5d1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
@@ -37,7 +37,7 @@ public final class EBoolean extends AExpression {
 
     @Override
     void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        actual = definition.booleanType;
+        actual = definition.getType("boolean");
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index 39afcd935ad..763005d17ba 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -200,7 +200,7 @@ public final class EChain extends AExpression {
 
             expression.expected = expression.actual;
         } else if (operation == Operation.LSH || operation == Operation.RSH || operation == Operation.USH) {
-            expression.expected = definition.intType;
+            expression.expected = definition.getType("int");
             expression.explicit = true;
         } else {
             expression.expected = promote;
@@ -215,7 +215,7 @@ public final class EChain extends AExpression {
         back = AnalyzerCaster.getLegalCast(definition, location, promote, last.after, true);
 
         this.statement = true;
-        this.actual = read ? last.after : definition.voidType;
+        this.actual = read ? last.after : definition.getType("void");
     }
 
     private void analyzeWrite(final CompilerSettings settings, final Definition definition, final Variables variables) {
@@ -235,7 +235,7 @@ public final class EChain extends AExpression {
         expression = expression.cast(settings, definition, variables);
 
         this.statement = true;
-        this.actual = read ? last.after : definition.voidType;
+        this.actual = read ? last.after : definition.getType("void");
     }
 
     private void analyzeRead() {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index d9337ae562b..a65e82e02fa 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -119,7 +119,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = definition.booleanType;
+        actual = definition.getType("boolean");
     }
 
     private void analyzeEqR(final CompilerSettings settings, final Definition definition, final Variables variables) {
@@ -161,7 +161,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = definition.booleanType;
+        actual = definition.getType("boolean");
     }
 
     private void analyzeNE(final CompilerSettings settings, final Definition definition, final Variables variables) {
@@ -207,7 +207,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = definition.booleanType;
+        actual = definition.getType("boolean");
     }
 
     private void analyzeNER(final CompilerSettings settings, final Definition definition, final Variables variables) {
@@ -249,7 +249,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = definition.booleanType;
+        actual = definition.getType("boolean");
     }
 
     private void analyzeGTE(final CompilerSettings settings, final Definition definition, final Variables variables) {
@@ -285,7 +285,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = definition.booleanType;
+        actual = definition.getType("boolean");
     }
 
     private void analyzeGT(final CompilerSettings settings, final Definition definition, final Variables variables) {
@@ -321,7 +321,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = definition.booleanType;
+        actual = definition.getType("boolean");
     }
 
     private void analyzeLTE(final CompilerSettings settings, final Definition definition, final Variables variables) {
@@ -357,7 +357,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = definition.booleanType;
+        actual = definition.getType("boolean");
     }
 
     private void analyzeLT(final CompilerSettings settings, final Definition definition, final Variables variables) {
@@ -393,7 +393,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = definition.booleanType;
+        actual = definition.getType("boolean");
     }
 
     @Override
@@ -456,7 +456,7 @@ public final class EComp extends AExpression {
                     if (right.isNull) {
                         adapter.ifNull(jump);
                     } else if (!left.isNull && operation == Operation.EQ) {
-                        adapter.invokeStatic(definition.defobjType.type, DEF_EQ_CALL);
+                        adapter.invokeStatic(definition.getType("Def").type, DEF_EQ_CALL);
                     } else {
                         adapter.ifCmp(rtype, MethodWriter.EQ, jump);
                     }
@@ -464,19 +464,19 @@ public final class EComp extends AExpression {
                     if (right.isNull) {
                         adapter.ifNonNull(jump);
                     } else if (!left.isNull && operation == Operation.NE) {
-                        adapter.invokeStatic(definition.defobjType.type, DEF_EQ_CALL);
+                        adapter.invokeStatic(definition.getType("Def").type, DEF_EQ_CALL);
                         adapter.ifZCmp(MethodWriter.EQ, jump);
                     } else {
                         adapter.ifCmp(rtype, MethodWriter.NE, jump);
                     }
                 } else if (lt) {
-                    adapter.invokeStatic(definition.defobjType.type, DEF_LT_CALL);
+                    adapter.invokeStatic(definition.getType("Def").type, DEF_LT_CALL);
                 } else if (lte) {
-                    adapter.invokeStatic(definition.defobjType.type, DEF_LTE_CALL);
+                    adapter.invokeStatic(definition.getType("Def").type, DEF_LTE_CALL);
                 } else if (gt) {
-                    adapter.invokeStatic(definition.defobjType.type, DEF_GT_CALL);
+                    adapter.invokeStatic(definition.getType("Def").type, DEF_GT_CALL);
                 } else if (gte) {
-                    adapter.invokeStatic(definition.defobjType.type, DEF_GTE_CALL);
+                    adapter.invokeStatic(definition.getType("Def").type, DEF_GTE_CALL);
                 } else {
                     throw new IllegalStateException(error("Illegal tree structure."));
                 }
@@ -493,7 +493,7 @@ public final class EComp extends AExpression {
                     if (right.isNull) {
                         adapter.ifNull(jump);
                     } else if (operation == Operation.EQ) {
-                        adapter.invokeStatic(definition.utilityType.type, CHECKEQUALS);
+                        adapter.invokeStatic(definition.getType("Utility").type, CHECKEQUALS);
 
                         if (branch) {
                             adapter.ifZCmp(MethodWriter.NE, jump);
@@ -507,7 +507,7 @@ public final class EComp extends AExpression {
                     if (right.isNull) {
                         adapter.ifNonNull(jump);
                     } else if (operation == Operation.NE) {
-                        adapter.invokeStatic(definition.utilityType.type, CHECKEQUALS);
+                        adapter.invokeStatic(definition.getType("Utility").type, CHECKEQUALS);
                         adapter.ifZCmp(MethodWriter.EQ, jump);
                     } else {
                         adapter.ifCmp(rtype, MethodWriter.NE, jump);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
index 5853fa3242e..45976c13414 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
@@ -47,7 +47,7 @@ public final class EConditional extends AExpression {
 
     @Override
     void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        condition.expected = definition.booleanType;
+        condition.expected = definition.getType("boolean");
         condition.analyze(settings, definition, variables);
         condition = condition.cast(settings, definition, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
index 7afa88ffc9a..f0ee3a8de51 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
@@ -40,23 +40,23 @@ final class EConstant extends AExpression {
     @Override
     void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
         if (constant instanceof String) {
-            actual = definition.stringType;
+            actual = definition.getType("String");
         } else if (constant instanceof Double) {
-            actual = definition.doubleType;
+            actual = definition.getType("double");
         } else if (constant instanceof Float) {
-            actual = definition.floatType;
+            actual = definition.getType("float");
         } else if (constant instanceof Long) {
-            actual = definition.longType;
+            actual = definition.getType("long");
         } else if (constant instanceof Integer) {
-            actual = definition.intType;
+            actual = definition.getType("int");
         } else if (constant instanceof Character) {
-            actual = definition.charType;
+            actual = definition.getType("char");
         } else if (constant instanceof Short) {
-            actual = definition.shortType;
+            actual = definition.getType("short");
         } else if (constant instanceof Byte) {
-            actual = definition.byteType;
+            actual = definition.getType("byte");
         } else if (constant instanceof Boolean) {
-            actual = definition.booleanType;
+            actual = definition.getType("boolean");
         } else {
             throw new IllegalStateException(error("Illegal tree structure."));
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
index 7583d3eb158..9019d370895 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
@@ -42,14 +42,14 @@ public final class EDecimal extends AExpression {
         if (value.endsWith("f") || value.endsWith("F")) {
             try {
                 constant = Float.parseFloat(value.substring(0, value.length() - 1));
-                actual = definition.floatType;
+                actual = definition.getType("float");
             } catch (final NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid float constant [" + value + "]."));
             }
         } else {
             try {
                 constant = Double.parseDouble(value);
-                actual = definition.doubleType;
+                actual = definition.getType("double");
             } catch (final NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid double constant [" + value + "]."));
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
index 0c8500c528b..7098e583c5a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
@@ -45,7 +45,7 @@ public final class ENull extends AExpression {
 
             actual = expected;
         } else {
-            actual = definition.objectType;
+            actual = definition.getType("Object");
         }
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
index ed7314b3571..489334ae24e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
@@ -49,7 +49,7 @@ public final class ENumeric extends AExpression {
 
             try {
                 constant = Double.parseDouble(value.substring(0, value.length() - 1));
-                actual = definition.doubleType;
+                actual = definition.getType("double");
             } catch (final NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid double constant [" + value + "]."));
             }
@@ -60,14 +60,14 @@ public final class ENumeric extends AExpression {
 
             try {
                 constant = Float.parseFloat(value.substring(0, value.length() - 1));
-                actual = definition.floatType;
+                actual = definition.getType("float");
             } catch (final NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid float constant [" + value + "]."));
             }
         } else if (value.endsWith("l") || value.endsWith("L")) {
             try {
                 constant = Long.parseLong(value.substring(0, value.length() - 1), radix);
-                actual = definition.longType;
+                actual = definition.getType("long");
             } catch (final NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid long constant [" + value + "]."));
             }
@@ -78,16 +78,16 @@ public final class ENumeric extends AExpression {
 
                 if (sort == Sort.BYTE && integer >= Byte.MIN_VALUE && integer <= Byte.MAX_VALUE) {
                     constant = (byte)integer;
-                    actual = definition.byteType;
+                    actual = definition.getType("byte");
                 } else if (sort == Sort.CHAR && integer >= Character.MIN_VALUE && integer <= Character.MAX_VALUE) {
                     constant = (char)integer;
-                    actual = definition.charType;
+                    actual = definition.getType("char");
                 } else if (sort == Sort.SHORT && integer >= Short.MIN_VALUE && integer <= Short.MAX_VALUE) {
                     constant = (short)integer;
-                    actual = definition.shortType;
+                    actual = definition.getType("short");
                 } else {
                     constant = integer;
-                    actual = definition.intType;
+                    actual = definition.getType("int");
                 }
             } catch (final NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid int constant [" + value + "]."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index f4205223a65..cbf76373136 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -65,7 +65,7 @@ public final class EUnary extends AExpression {
     }
 
     void analyzeNot(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        child.expected = definition.booleanType;
+        child.expected = definition.getType("boolean");
         child.analyze(settings, definition, variables);
         child = child.cast(settings, definition, variables);
 
@@ -73,7 +73,7 @@ public final class EUnary extends AExpression {
             constant = !(boolean)child.constant;
         }
 
-        actual = definition.booleanType;
+        actual = definition.getType("boolean");
     }
 
     void analyzeBWNot(final CompilerSettings settings, final Definition definition, final Variables variables) {
@@ -195,7 +195,7 @@ public final class EUnary extends AExpression {
 
             if (operation == Operation.BWNOT) {
                 if (sort == Sort.DEF) {
-                    adapter.invokeStatic(definition.defobjType.type, DEF_NOT_CALL);
+                    adapter.invokeStatic(definition.getType("Def").type, DEF_NOT_CALL);
                 } else {
                     if (sort == Sort.INT) {
                         adapter.push(-1);
@@ -209,15 +209,15 @@ public final class EUnary extends AExpression {
                 }
             } else if (operation == Operation.SUB) {
                 if (sort == Sort.DEF) {
-                    adapter.invokeStatic(definition.defobjType.type, DEF_NEG_CALL);
+                    adapter.invokeStatic(definition.getType("Def").type, DEF_NEG_CALL);
                 } else {
                     if (settings.getNumericOverflow()) {
                         adapter.math(MethodWriter.NEG, type);
                     } else {
                         if (sort == Sort.INT) {
-                            adapter.invokeStatic(definition.mathType.type, NEGATEEXACT_INT);
+                            adapter.invokeStatic(definition.getType("Math").type, NEGATEEXACT_INT);
                         } else if (sort == Sort.LONG) {
-                            adapter.invokeStatic(definition.mathType.type, NEGATEEXACT_LONG);
+                            adapter.invokeStatic(definition.getType("Math").type, NEGATEEXACT_LONG);
                         } else {
                             throw new IllegalStateException(error("Illegal tree structure."));
                         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
index 5803fcfa273..d41128f60bb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
@@ -46,7 +46,7 @@ public final class LArrayLength extends ALink {
                 throw new IllegalArgumentException(error("Cannot write to read-only array field [length]."));
             }
 
-            after = definition.intType;
+            after = definition.getType("int");
         } else {
             throw new IllegalArgumentException(error("Illegal field access [" + value + "]."));
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
index b38826f9e7e..1e38161ae87 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
@@ -50,7 +50,7 @@ public final class LBrace extends ALink {
         final Sort sort = before.sort;
 
         if (sort == Sort.ARRAY) {
-            index.expected = definition.intType;
+            index.expected = definition.getType("int");
             index.analyze(settings, definition, variables);
             index = index.cast(settings, definition, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index 98b2fbe7bf9..68f60ef6f8f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -47,7 +47,7 @@ final class LDefArray extends ALink implements IDefLink {
         index.expected = index.actual;
         index = index.cast(settings, definition, variables);
 
-        after = definition.defType;
+        after = definition.getType("def");
 
         return this;
     }
@@ -59,13 +59,13 @@ final class LDefArray extends ALink implements IDefLink {
 
     @Override
     void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
-        final String desc = Type.getMethodDescriptor(after.type, definition.defType.type, index.actual.type);
+        final String desc = Type.getMethodDescriptor(after.type, definition.getType("def").type, index.actual.type);
         adapter.invokeDynamic("arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_LOAD);
     }
 
     @Override
     void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
-        final String desc = Type.getMethodDescriptor(definition.voidType.type, definition.defType.type,
+        final String desc = Type.getMethodDescriptor(definition.getType("void").type, definition.getType("def").type,
             index.actual.type, after.type);
         adapter.invokeDynamic("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_STORE);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index 9dbe65110ee..1285d548163 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -55,7 +55,7 @@ final class LDefCall extends ALink implements IDefLink {
         }
 
         statement = true;
-        after = definition.defType;
+        after = definition.getType("def");
 
         return this;
     }
@@ -71,7 +71,7 @@ final class LDefCall extends ALink implements IDefLink {
 
         signature.append('(');
         // first parameter is the receiver, we never know its type: always Object
-        signature.append(definition.defType.type.getDescriptor());
+        signature.append(definition.getType("def").type.getDescriptor());
 
         // TODO: remove our explicit conversions and feed more type information for return value,
         // it can avoid some unnecessary boxing etc.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index 759b407cb5a..9dbe1cc3367 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -44,7 +44,7 @@ final class LDefField extends ALink implements IDefLink {
 
     @Override
     ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        after = definition.defType;
+        after = definition.getType("def");
 
         return this;
     }
@@ -56,13 +56,13 @@ final class LDefField extends ALink implements IDefLink {
 
     @Override
     void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
-        final String desc = Type.getMethodDescriptor(after.type, definition.defType.type);
+        final String desc = Type.getMethodDescriptor(after.type, definition.getType("def").type);
         adapter.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.LOAD);
     }
 
     @Override
     void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
-        final String desc = Type.getMethodDescriptor(definition.voidType.type, definition.defType.type, after.type);
+        final String desc = Type.getMethodDescriptor(definition.getType("void").type, definition.getType("def").type, after.type);
         adapter.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.STORE);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
index 8526ef1297e..f1649f3bf37 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
@@ -61,7 +61,7 @@ final class LListShortcut extends ALink {
         }
 
         if ((load || store) && (!load || getter != null) && (!store || setter != null)) {
-            index.expected = definition.intType;
+            index.expected = definition.getType("int");
             index.analyze(settings, definition, variables);
             index = index.cast(settings, definition, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
index 67f9769bd6a..acafc44ede1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
@@ -63,7 +63,7 @@ public final class LNewArray extends ALink {
         for (int argument = 0; argument < arguments.size(); ++argument) {
             final AExpression expression = arguments.get(argument);
 
-            expression.expected = definition.intType;
+            expression.expected = definition.getType("int");
             expression.analyze(settings, definition, variables);
             arguments.set(argument, expression.cast(settings, definition, variables));
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
index 1d11652f483..6b91cf5e87a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
@@ -45,7 +45,7 @@ public final class LString extends ALink {
             throw new IllegalArgumentException(error("Must read String constant [" + string + "]."));
         }
 
-        after = definition.stringType;
+        after = definition.getType("String");
 
         return this;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
index 63d3df4bcee..7a79fe9c02c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -53,7 +53,7 @@ public final class SDo extends AStatement {
             throw new IllegalArgumentException(error("Extraneous do while loop."));
         }
 
-        condition.expected = definition.booleanType;
+        condition.expected = definition.getType("boolean");
         condition.analyze(settings, definition, variables);
         condition = condition.cast(settings, definition, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index dd7cffaa970..39ab0c099a6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -49,7 +49,7 @@ public final class SExpression extends AStatement {
 
         final boolean rtn = lastSource && expression.actual.sort != Sort.VOID;
 
-        expression.expected = rtn ? definition.objectType : expression.actual;
+        expression.expected = rtn ? definition.getType("Object") : expression.actual;
         expression = expression.cast(settings, definition, variables);
 
         methodEscape = rtn;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index 43c978b47fa..239e302c72d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -70,7 +70,7 @@ public final class SFor extends AStatement {
 
         if (condition != null) {
 
-            condition.expected = definition.booleanType;
+            condition.expected = definition.getType("boolean");
             condition.analyze(settings, definition, variables);
             condition = condition.cast(settings, definition, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
index 698d8c8126c..89dad5a96c4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
@@ -45,7 +45,7 @@ public final class SIfElse extends AStatement {
 
     @Override
     void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        condition.expected = definition.booleanType;
+        condition.expected = definition.getType("boolean");
         condition.analyze(settings, definition, variables);
         condition = condition.cast(settings, definition, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
index b959b47a96b..68084a9a70c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -39,7 +39,7 @@ public final class SReturn extends AStatement {
 
     @Override
     void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        expression.expected = definition.objectType;
+        expression.expected = definition.getType("Object");
         expression.analyze(settings, definition, variables);
         expression = expression.cast(settings, definition, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
index eac039b998e..b840fce3a83 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
@@ -39,7 +39,7 @@ public final class SThrow extends AStatement {
 
     @Override
     void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        expression.expected = definition.exceptionType;
+        expression.expected = definition.getType("Exception");
         expression.analyze(settings, definition, variables);
         expression = expression.cast(settings, definition, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
index f6f8ddc678f..5cabb1b0f7c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -44,7 +44,7 @@ public final class SWhile extends AStatement {
     void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
         variables.incrementScope();
 
-        condition.expected = definition.booleanType;
+        condition.expected = definition.getType("boolean");
         condition.analyze(settings, definition, variables);
         condition = condition.cast(settings, definition, variables);
 
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/definition.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/definition.txt
new file mode 100644
index 00000000000..4ca37223043
--- /dev/null
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/definition.txt
@@ -0,0 +1,529 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#
+# Painless definition file. This defines the hierarchy of classes,
+# what methods and fields they have, etc.
+#
+
+# primitive types
+
+class void -> void {
+}
+
+class boolean -> boolean {
+}
+
+class byte -> byte {
+}
+
+class short -> short {
+}
+
+class char -> char {
+}
+
+class int -> int {
+}
+
+class long -> long {
+}
+
+class float -> float {
+}
+
+class double -> double {
+}
+
+# basic JDK classes
+
+class Object -> java.lang.Object {
+  boolean equals(Object)
+  int hashCode()
+  String toString()
+}
+
+class def -> java.lang.Object {
+  boolean equals(Object)
+  int hashCode()
+  String toString()
+}
+
+class Void -> java.lang.Void extends Object {
+}
+
+class Boolean -> java.lang.Boolean extends Object {
+  Boolean <init>(boolean)
+  Boolean TRUE
+  Boolean FALSE
+  int compare(boolean,boolean)
+  boolean parseBoolean(String)
+  Boolean valueOf(boolean)
+  boolean booleanValue()
+  int compareTo(Boolean)
+} 
+
+class Byte -> java.lang.Byte extends Number,Object {
+  Byte <init>(byte)
+  byte MIN_VALUE
+  byte MAX_VALUE
+  int compare(byte,byte)
+  int compareTo(Byte)
+  byte parseByte(String)
+  Byte valueOf(byte)
+}
+
+class Short -> java.lang.Short extends Number,Object {
+  Short <init>(short)
+  short MIN_VALUE
+  short MAX_VALUE
+  int compare(short,short)
+  int compareTo(Short)
+  short parseShort(String)
+  Short valueOf(short)
+}
+
+class Character -> java.lang.Character extends Object {
+  Character <init>(char)
+  char MIN_VALUE
+  char MAX_VALUE
+  int charCount(int)
+  char charValue()
+  int compare(char,char)
+  int compareTo(Character)
+  int digit(int,int)
+  char forDigit(int,int)
+  String getName(int)
+  int getNumericValue(int)
+  boolean isAlphabetic(int)
+  boolean isDefined(int)
+  boolean isDigit(int)
+  boolean isIdeographic(int)
+  boolean isLetter(int)
+  boolean isLetterOrDigit(int)
+  boolean isLowerCase(int)
+  boolean isMirrored(int)
+  boolean isSpaceChar(int)
+  boolean isTitleCase(int)
+  boolean isUpperCase(int)
+  boolean isWhitespace(int)
+  Character valueOf(char)
+}
+
+class Integer -> java.lang.Integer extends Number,Object {
+  Integer <init>(int)
+  int MIN_VALUE
+  int MAX_VALUE
+  int compare(int,int)
+  int compareTo(Integer)
+  int min(int,int)
+  int max(int,int)
+  int parseInt(String)
+  int signum(int)
+  String toHexString(int)
+  Integer valueOf(int)
+}
+
+class Long -> java.lang.Long extends Number,Object {
+  Long <init>(long)
+  long MIN_VALUE
+  long MAX_VALUE
+  int compare(long,long)
+  int compareTo(Long)
+  long min(long,long)
+  long max(long,long)
+  long parseLong(String)
+  int signum(long)
+  String toHexString(long)
+  Long valueOf(long)
+}
+
+class Float -> java.lang.Float extends Number,Object {
+  Float <init>(float)
+  float MIN_VALUE
+  float MAX_VALUE
+  int compare(float,float)
+  int compareTo(Float)
+  float min(float,float)
+  float max(float,float)
+  float parseFloat(String)
+  String toHexString(float)
+  Float valueOf(float)
+}
+
+class Double -> java.lang.Double extends Number,Object {
+  Double <init>(double)
+  double MIN_VALUE
+  double MAX_VALUE
+  int compare(double,double)
+  int compareTo(Double)
+  double min(double,double)
+  double max(double,double)
+  double parseDouble(String)
+  String toHexString(double)
+  Double valueOf(double)
+}
+
+class Number -> java.lang.Number extends Object {
+  byte byteValue()
+  short shortValue()
+  int intValue()
+  long longValue()
+  float floatValue()
+  double doubleValue()
+}
+
+class CharSequence -> java.lang.CharSequence extends Object {
+  char charAt(int)
+  int length()
+}
+
+class String -> java.lang.String extends CharSequence,Object {
+  String <init>()
+  int codePointAt(int)
+  int compareTo(String)
+  String concat(String)
+  boolean endsWith(String)
+  int indexOf(String)
+  int indexOf(String,int)
+  boolean isEmpty()
+  String replace(CharSequence,CharSequence)
+  boolean startsWith(String)
+  String substring(int,int)
+  char[] toCharArray()
+  String trim()
+}
+
+class Math -> java.lang.Math {
+  double E
+  double PI
+  double abs(double)
+  double acos(double)
+  double asin(double)
+  double atan(double)
+  double atan2(double,double)
+  double cbrt(double)
+  double ceil(double)
+  double cos(double)
+  double cosh(double)
+  double exp(double)
+  double expm1(double)
+  double floor(double)
+  double hypot(double,double)
+  double log(double)
+  double log10(double)
+  double log1p(double)
+  double max(double,double)
+  double min(double,double)
+  double pow(double,double)
+  double random()
+  double rint(double)
+  long round(double)
+  double sin(double)
+  double sinh(double)
+  double sqrt(double)
+  double tan(double)
+  double tanh(double)
+  double toDegrees(double)
+  double toRadians(double)
+}
+
+class Iterator -> java.util.Iterator extends Object {
+  boolean hasNext()
+  def next()
+  void remove()
+}
+
+class Collection -> java.util.Collection extends Object {
+  boolean add(def)
+  void clear()
+  boolean contains(def)
+  boolean isEmpty()
+  Iterator iterator()
+  boolean remove(def)
+  int size()
+}
+
+class List -> java.util.List extends Collection,Object {
+  def set(int,def)
+  def get(int)
+  def remove(int)
+  int getLength/size()
+}
+
+class ArrayList -> java.util.ArrayList extends List,Collection,Object {
+  ArrayList <init>()
+}
+
+class Set -> java.util.Set extends Collection,Object {
+}
+
+class HashSet -> java.util.HashSet extends Set,Collection,Object {
+  HashSet <init>()
+}
+
+class Map -> java.util.Map extends Object {
+  def put(def,def)
+  def get(def)
+  def remove(def)
+  boolean isEmpty()
+  int size()
+  boolean containsKey(def)
+  Set keySet()
+  Collection values()
+}
+
+class HashMap -> java.util.HashMap extends Map,Object {
+  HashMap <init>()
+}
+
+class Exception -> java.lang.Exception extends Object {
+  String getMessage()
+}
+
+class ArithmeticException -> java.lang.ArithmeticException extends Exception,Object {
+  ArithmeticException <init>()
+}
+
+class IllegalArgumentException -> java.lang.IllegalArgumentException extends Exception,Object {
+  IllegalArgumentException <init>()
+}
+
+class IllegalStateException -> java.lang.IllegalStateException extends Exception,Object {
+  IllegalStateException <init>()
+}
+
+class NumberFormatException -> java.lang.NumberFormatException extends Exception,Object {
+  NumberFormatException <init>()
+}
+
+# ES Scripting API
+
+class GeoPoint -> org.elasticsearch.common.geo.GeoPoint extends Object {
+  double getLat()
+  double getLon()
+}
+
+class Strings -> org.elasticsearch.index.fielddata.ScriptDocValues$Strings extends List,Collection,Object {
+  String getValue()
+  List getValues()
+}
+
+class Longs -> org.elasticsearch.index.fielddata.ScriptDocValues$Longs extends List,Collection,Object {
+  long getValue()
+  List getValues()
+}
+
+class Doubles -> org.elasticsearch.index.fielddata.ScriptDocValues$Doubles extends List,Collection,Object {
+  double getValue()
+  List getValues()
+}
+
+class GeoPoints -> org.elasticsearch.index.fielddata.ScriptDocValues$GeoPoints extends List,Collection,Object {
+  GeoPoint getValue()
+  List getValues()
+  double getLat()
+  double getLon()
+  double[] getLats()
+  double[] getLons()
+
+  # geo distance functions... so many...
+  double factorDistance(double,double)
+  double factorDistanceWithDefault(double,double,double)
+  double factorDistance02(double,double)
+  double factorDistance13(double,double)
+  double arcDistance(double,double)
+  double arcDistanceWithDefault(double,double,double)
+  double arcDistanceInKm(double,double)
+  double arcDistanceInKmWithDefault(double,double,double)
+  double arcDistanceInMiles(double,double)
+  double arcDistanceInMilesWithDefault(double,double,double)
+  double distance(double,double)
+  double distanceWithDefault(double,double,double)
+  double distanceInKm(double,double)
+  double distanceInKmWithDefault(double,double,double)
+  double distanceInMiles(double,double)
+  double distanceInMilesWithDefault(double,double,double)
+  double geohashDistance(String)
+  double geohashDistanceInKm(String)
+  double geohashDistanceInMiles(String)
+}
+
+# internal conversion methods
+class Utility -> org.elasticsearch.painless.Utility {
+  boolean NumberToboolean(Number)
+  char NumberTochar(Number)
+  Boolean NumberToBoolean(Number)
+  Byte NumberToByte(Number)
+  Short NumberToShort(Number)
+  Character NumberToCharacter(Number)
+  Integer NumberToInteger(Number)
+  Long NumberToLong(Number)
+  Float NumberToFloat(Number)
+  Double NumberToDouble(Number)
+  byte booleanTobyte(boolean)
+  short booleanToshort(boolean)
+  char booleanTochar(boolean)
+  int booleanToint(boolean)
+  long booleanTolong(boolean)
+  float booleanTofloat(boolean)
+  double booleanTodouble(boolean)
+  Integer booleanToInteger(boolean)
+  byte BooleanTobyte(Boolean)
+  short BooleanToshort(Boolean)
+  char BooleanTochar(Boolean)
+  int BooleanToint(Boolean)
+  long BooleanTolong(Boolean)
+  float BooleanTofloat(Boolean)
+  double BooleanTodouble(Boolean)
+  Byte BooleanToByte(Boolean)
+  Short BooleanToShort(Boolean)
+  Character BooleanToCharacter(Boolean)
+  Integer BooleanToInteger(Boolean)
+  Long BooleanToLong(Boolean)
+  Float BooleanToFloat(Boolean)
+  Double BooleanToDouble(Boolean)
+  boolean byteToboolean(byte)
+  Short byteToShort(byte)
+  Character byteToCharacter(byte)
+  Integer byteToInteger(byte)
+  Long byteToLong(byte)
+  Float byteToFloat(byte)
+  Double byteToDouble(byte)
+  boolean ByteToboolean(Byte)
+  char ByteTochar(Byte)
+  boolean shortToboolean(short)
+  Byte shortToByte(short)
+  Character shortToCharacter(short)
+  Integer shortToInteger(short)
+  Long shortToLong(short)
+  Float shortToFloat(short)
+  Double shortToDouble(short)
+  boolean ShortToboolean(Short)
+  char ShortTochar(Short)
+  boolean charToboolean(char)
+  Byte charToByte(char)
+  Short charToShort(char)
+  Integer charToInteger(char)
+  Long charToLong(char)
+  Float charToFloat(char)
+  Double charToDouble(char)
+  String charToString(char)
+  boolean CharacterToboolean(Character)
+  byte CharacterTobyte(Character)
+  short CharacterToshort(Character)
+  int CharacterToint(Character)
+  long CharacterTolong(Character)
+  float CharacterTofloat(Character)
+  double CharacterTodouble(Character)
+  Boolean CharacterToBoolean(Character)
+  Byte CharacterToByte(Character)
+  Short CharacterToShort(Character)
+  Integer CharacterToInteger(Character)
+  Long CharacterToLong(Character)
+  Float CharacterToFloat(Character)
+  Double CharacterToDouble(Character)
+  String CharacterToString(Character)
+  boolean intToboolean(int)
+  Byte intToByte(int)
+  Short intToShort(int)
+  Character intToCharacter(int)
+  Long intToLong(int)
+  Float intToFloat(int)
+  Double intToDouble(int)
+  boolean IntegerToboolean(Integer)
+  char IntegerTochar(Integer)
+  boolean longToboolean(long)
+  Byte longToByte(long)
+  Short longToShort(long)
+  Character longToCharacter(long)
+  Integer longToInteger(long)
+  Float longToFloat(long)
+  Double longToDouble(long)
+  boolean LongToboolean(Long)
+  char LongTochar(Long)
+  boolean floatToboolean(float)
+  Byte floatToByte(float)
+  Short floatToShort(float)
+  Character floatToCharacter(float)
+  Integer floatToInteger(float)
+  Long floatToLong(float)
+  Double floatToDouble(float)
+  boolean FloatToboolean(Float)
+  char FloatTochar(Float)
+  boolean doubleToboolean(double)
+  Byte doubleToByte(double)
+  Short doubleToShort(double)
+  Character doubleToCharacter(double)
+  Integer doubleToInteger(double)
+  Long doubleToLong(double)
+  Float doubleToFloat(double)
+  boolean DoubleToboolean(Double)
+  char DoubleTochar(Double)
+  char StringTochar(String)
+  Character StringToCharacter(String)
+}
+
+class Def -> org.elasticsearch.painless.Def {
+  byte DefTobyteImplicit(def)
+  short DefToshortImplicit(def)
+  char DefTocharImplicit(def)
+  int DefTointImplicit(def)
+  long DefTolongImplicit(def)
+  float DefTofloatImplicit(def)
+  double DefTodoubleImplicit(def)
+  Byte DefToByteImplicit(def)
+  Short DefToShortImplicit(def)
+  Character DefToCharacterImplicit(def)
+  Integer DefToIntegerImplicit(def)
+  Long DefToLongImplicit(def)
+  Float DefToFloatImplicit(def)
+  Double DefToDoubleImplicit(def)
+  byte DefTobyteExplicit(def)
+  short DefToshortExplicit(def)
+  char DefTocharExplicit(def)
+  int DefTointExplicit(def)
+  long DefTolongExplicit(def)
+  float DefTofloatExplicit(def)
+  double DefTodoubleExplicit(def)
+  Byte DefToByteExplicit(def)
+  Short DefToShortExplicit(def)
+  Character DefToCharacterExplicit(def)
+  Integer DefToIntegerExplicit(def)
+  Long DefToLongExplicit(def)
+  Float DefToFloatExplicit(def)
+  Double DefToDoubleExplicit(def)
+}
+
+# for testing.
+# currently FeatureTest exposes overloaded constructor, field load store, and overloaded static methods
+class FeatureTest -> org.elasticsearch.painless.FeatureTest extends Object {
+  FeatureTest <init>()
+  FeatureTest <init>(int,int)
+  int getX()
+  int getY()
+  void setX(int)
+  void setY(int)
+  boolean overloadedStatic()
+  boolean overloadedStatic(boolean)
+}
+
+# currently needed internally
+class Executable -> org.elasticsearch.painless.Executable {
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
index af067d15252..b7479b505fe 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
@@ -24,10 +24,8 @@ public class BasicAPITests extends ScriptTestCase {
     public void testListIterator() {
         assertEquals(3, exec("List x = new ArrayList(); x.add(2); x.add(3); x.add(-2); Iterator y = x.iterator(); " +
             "int total = 0; while (y.hasNext()) total += y.next(); return total;"));
-        assertEquals(3, exec("List<Object> x = new ArrayList(); x.add(2); x.add(3); x.add(-2); Iterator<Object> y = x.iterator(); " +
-            "int total = 0; while (y.hasNext()) total += (int)y.next(); return total;"));
-        assertEquals("abc", exec("List<String> x = new ArrayList(); x.add(\"a\"); x.add(\"b\"); x.add(\"c\"); " +
-            "Iterator<String> y = x.iterator(); String total = \"\"; while (y.hasNext()) total += y.next(); return total;"));
+        assertEquals("abc", exec("List x = new ArrayList(); x.add(\"a\"); x.add(\"b\"); x.add(\"c\"); " +
+            "Iterator y = x.iterator(); String total = \"\"; while (y.hasNext()) total += y.next(); return total;"));
         assertEquals(3, exec("def x = new ArrayList(); x.add(2); x.add(3); x.add(-2); def y = x.iterator(); " +
             "def total = 0; while (y.hasNext()) total += y.next(); return total;"));
     }
@@ -35,10 +33,8 @@ public class BasicAPITests extends ScriptTestCase {
     public void testSetIterator() {
         assertEquals(3, exec("Set x = new HashSet(); x.add(2); x.add(3); x.add(-2); Iterator y = x.iterator(); " +
             "int total = 0; while (y.hasNext()) total += y.next(); return total;"));
-        assertEquals(3, exec("Set<Object> x = new HashSet(); x.add(2); x.add(3); x.add(-2); Iterator<Object> y = x.iterator(); " +
-            "int total = 0; while (y.hasNext()) total += (int)y.next(); return total;"));
-        assertEquals("abc", exec("Set<String> x = new HashSet(); x.add(\"a\"); x.add(\"b\"); x.add(\"c\"); " +
-            "Iterator<String> y = x.iterator(); String total = \"\"; while (y.hasNext()) total += y.next(); return total;"));
+        assertEquals("abc", exec("Set x = new HashSet(); x.add(\"a\"); x.add(\"b\"); x.add(\"c\"); " +
+            "Iterator y = x.iterator(); String total = \"\"; while (y.hasNext()) total += y.next(); return total;"));
         assertEquals(3, exec("def x = new HashSet(); x.add(2); x.add(3); x.add(-2); def y = x.iterator(); " +
             "def total = 0; while (y.hasNext()) total += (int)y.next(); return total;"));
     }
@@ -78,8 +74,6 @@ public class BasicAPITests extends ScriptTestCase {
         assertEquals(1, exec("def x = new ArrayList(); x.add(5); return x.length"));
         assertEquals(5, exec("def x = new ArrayList(); x.add(5); return x[0]"));
         assertEquals(1, exec("List x = new ArrayList(); x.add('Hallo'); return x.length"));
-        assertEquals(1, exec("List<String> x = new ArrayList<String>(); x.add('Hallo'); return x.length"));
-        assertEquals(1, exec("List<Object> x = new ArrayList<Object>(); x.add('Hallo'); return x.length"));
     }
     
     public void testDefAssignments() {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
index f0022e6bcf1..0d6a54b515b 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
@@ -135,7 +135,7 @@ public class BasicStatementTests extends ScriptTestCase {
         assertEquals(2.0, exec("double a = 2; return a;"));
         assertEquals(false, exec("boolean a = false; return a;"));
         assertEquals("string", exec("String a = \"string\"; return a;"));
-        assertEquals(HashMap.class, exec("Map<String,Object> a = new HashMap<String,Object>(); return a;").getClass());
+        assertEquals(HashMap.class, exec("Map a = new HashMap(); return a;").getClass());
 
         assertEquals(byte[].class, exec("byte[] a = new byte[1]; return a;").getClass());
         assertEquals(short[].class, exec("short[] a = new short[1]; return a;").getClass());
@@ -146,7 +146,7 @@ public class BasicStatementTests extends ScriptTestCase {
         assertEquals(double[].class, exec("double[] a = new double[1]; return a;").getClass());
         assertEquals(boolean[].class, exec("boolean[] a = new boolean[1]; return a;").getClass());
         assertEquals(String[].class, exec("String[] a = new String[1]; return a;").getClass());
-        assertEquals(Map[].class, exec("Map<String,Object>[] a = new Map<String,Object>[1]; return a;").getClass());
+        assertEquals(Map[].class, exec("Map[] a = new Map[1]; return a;").getClass());
 
         assertEquals(byte[][].class, exec("byte[][] a = new byte[1][2]; return a;").getClass());
         assertEquals(short[][][].class, exec("short[][][] a = new short[1][2][3]; return a;").getClass());
@@ -157,7 +157,7 @@ public class BasicStatementTests extends ScriptTestCase {
         assertEquals(double[][][][].class, exec("double[][][][] a = new double[1][2][3][4]; return a;").getClass());
         assertEquals(boolean[][][][][].class, exec("boolean[][][][][] a = new boolean[1][2][3][4][5]; return a;").getClass());
         assertEquals(String[][].class, exec("String[][] a = new String[1][2]; return a;").getClass());
-        assertEquals(Map[][][].class, exec("Map<String,Object>[][][] a = new Map<String,Object>[1][2][3]; return a;").getClass());
+        assertEquals(Map[][][].class, exec("Map[][][] a = new Map[1][2][3]; return a;").getClass());
     }
 
     public void testContinueStatement() {
@@ -174,6 +174,6 @@ public class BasicStatementTests extends ScriptTestCase {
         assertEquals(5, exec("int x = 5; return x;"));
         assertEquals(4, exec("int[] x = new int[2]; x[1] = 4; return x[1];"));
         assertEquals(5, ((short[])exec("short[] s = new short[3]; s[1] = 5; return s;"))[1]);
-        assertEquals(10, ((Map)exec("Map<String, Object> s = new HashMap< String , Object >(); s.put(\"x\", 10); return s;")).get("x"));
+        assertEquals(10, ((Map)exec("Map s = new HashMap(); s.put(\"x\", 10); return s;")).get("x"));
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NoSemiColonTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NoSemiColonTests.java
index f2e65fc680c..e4af7a2d166 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NoSemiColonTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NoSemiColonTests.java
@@ -35,7 +35,7 @@ public class NoSemiColonTests extends ScriptTestCase {
         assertEquals(2.0, exec("double a = 2; return a"));
         assertEquals(false, exec("boolean a = false; return a"));
         assertEquals("string", exec("String a = \"string\"; return a"));
-        assertEquals(HashMap.class, exec("Map<String, Object> a = new HashMap<String, Object>(); return a").getClass());
+        assertEquals(HashMap.class, exec("Map a = new HashMap(); return a").getClass());
 
         assertEquals(byte[].class, exec("byte[] a = new byte[1]; return a").getClass());
         assertEquals(short[].class, exec("short[] a = new short[1]; return a").getClass());
@@ -46,7 +46,7 @@ public class NoSemiColonTests extends ScriptTestCase {
         assertEquals(double[].class, exec("double[] a = new double[1]; return a").getClass());
         assertEquals(boolean[].class, exec("boolean[] a = new boolean[1]; return a").getClass());
         assertEquals(String[].class, exec("String[] a = new String[1]; return a").getClass());
-        assertEquals(Map[].class, exec("Map<String,Object>[] a = new Map<String,Object>[1]; return a").getClass());
+        assertEquals(Map[].class, exec("Map[] a = new Map[1]; return a").getClass());
 
         assertEquals(byte[][].class, exec("byte[][] a = new byte[1][2]; return a").getClass());
         assertEquals(short[][][].class, exec("short[][][] a = new short[1][2][3]; return a").getClass());
@@ -57,7 +57,7 @@ public class NoSemiColonTests extends ScriptTestCase {
         assertEquals(double[][][][].class, exec("double[][][][] a = new double[1][2][3][4]; return a").getClass());
         assertEquals(boolean[][][][][].class, exec("boolean[][][][][] a = new boolean[1][2][3][4][5]; return a").getClass());
         assertEquals(String[][].class, exec("String[][] a = new String[1][2]; return a").getClass());
-        assertEquals(Map[][][].class, exec("Map<String,Object>[][][] a = new Map<String,Object>[1][2][3]; return a").getClass());
+        assertEquals(Map[][][].class, exec("Map[][][] a = new Map[1][2][3]; return a").getClass());
     }
     
     public void testExpression() {
@@ -73,6 +73,6 @@ public class NoSemiColonTests extends ScriptTestCase {
         assertEquals(5, exec("int x = 5; return x"));
         assertEquals(4, exec("int[] x = new int[2]; x[1] = 4; return x[1]"));
         assertEquals(5, ((short[])exec("short[] s = new short[3]; s[1] = 5; return s"))[1]);
-        assertEquals(10, ((Map)exec("Map<String,Object> s = new HashMap< String,Object>(); s.put(\"x\", 10); return s")).get("x"));
+        assertEquals(10, ((Map)exec("Map s = new HashMap(); s.put(\"x\", 10); return s")).get("x"));
     }
 }

From 3cf4214255e872612d9f99e1900ccadd55437e87 Mon Sep 17 00:00:00 2001
From: Jeff Evans <wayne.mokane+github@gmail.com>
Date: Thu, 19 May 2016 08:47:07 -0500
Subject: [PATCH 0348/1311] Add better error message when analyzer created
 without tokenizer or analyzer type (#18455)

Closes #15492
---
 .../index/analysis/AnalysisRegistry.java         |  4 ++--
 .../index/analysis/AnalysisServiceTests.java     | 16 ++++++++++++++++
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
index c33eef4ac61..2d73df76f07 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
@@ -321,7 +321,7 @@ public final class AnalysisRegistry implements Closeable {
                     if (currentSettings.get("tokenizer") != null) {
                         factory = (T) new CustomAnalyzerProvider(settings, name, currentSettings);
                     } else {
-                        throw new IllegalArgumentException(toBuild + " [" + name + "] must have a type associated with it");
+                        throw new IllegalArgumentException(toBuild + " [" + name + "] must specify either an analyzer type, or a tokenizer");
                     }
                 } else if (typeName.equals("custom")) {
                     factory = (T) new CustomAnalyzerProvider(settings, name, currentSettings);
@@ -335,7 +335,7 @@ public final class AnalysisRegistry implements Closeable {
                 factories.put(name, factory);
             }  else {
                 if (typeName == null) {
-                    throw new IllegalArgumentException(toBuild + " [" + name + "] must have a type associated with it");
+                    throw new IllegalArgumentException(toBuild + " [" + name + "] must specify either an analyzer type, or a tokenizer");
                 }
                 AnalysisModule.AnalysisProvider<T> type = providerMap.get(typeName);
                 if (type == null) {
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java
index b667c256019..b72996bd1a1 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java
@@ -41,6 +41,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
 public class AnalysisServiceTests extends ESTestCase {
@@ -183,4 +184,19 @@ public class AnalysisServiceTests extends ESTestCase {
             assertSame(analysisService.analyzer(preBuiltAnalyzers.name()), otherAnalysisSergice.analyzer(preBuiltAnalyzers.name()));
         }
     }
+
+    public void testNoTypeOrTokenizerErrorMessage() throws IOException {
+        Version version = VersionUtils.randomVersion(random());
+        Settings settings = Settings
+            .builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, version)
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .putArray("index.analysis.analyzer.test_analyzer.filter", new String[] {"lowercase", "stop", "shingle"})
+            .putArray("index.analysis.analyzer.test_analyzer.char_filter", new String[] {"html_strip"})
+            .build();
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new AnalysisRegistry(null, new Environment(settings)).build(idxSettings));
+        assertThat(e.getMessage(), equalTo("analyzer [test_analyzer] must specify either an analyzer type, or a tokenizer"));
+    }
 }

From a01ecb20ea3edaee2db670763a3bb58d79ab5f03 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Fri, 13 May 2016 15:27:02 +0200
Subject: [PATCH 0349/1311] Port Delete By Query to Reindex infrastructure

closes #16883
---
 docs/reference/docs.asciidoc                  |   2 +
 docs/reference/docs/delete-by-query.asciidoc  | 318 ++++++++++
 docs/reference/docs/update-by-query.asciidoc  |   4 +-
 .../AbstractAsyncBulkByScrollAction.java      |  16 +-
 .../AbstractAsyncBulkIndexByScrollAction.java | 542 +++++++++++++-----
 .../AbstractBaseReindexRestHandler.java       | 111 ++--
 .../AbstractBulkByQueryRestHandler.java       | 118 ++++
 .../index/reindex/DeleteByQueryAction.java    |  43 ++
 .../index/reindex/DeleteByQueryRequest.java   |  79 +++
 .../reindex/DeleteByQueryRequestBuilder.java  |  51 ++
 .../index/reindex/ReindexPlugin.java          |   2 +
 .../reindex/RestDeleteByQueryAction.java      |  77 +++
 .../index/reindex/RestReindexAction.java      |  46 +-
 .../index/reindex/RestRethrottleAction.java   |   1 +
 .../reindex/RestUpdateByQueryAction.java      |  75 +--
 .../reindex/TransportDeleteByQueryAction.java | 109 ++++
 .../index/reindex/TransportReindexAction.java | 188 +++---
 .../reindex/TransportUpdateByQueryAction.java |  99 ++--
 ...BulkIndexByScrollActionScriptTestCase.java |  30 +-
 ...tAsyncBulkIndexByScrollActionTestCase.java |   2 -
 ...lkIndexbyScrollActionMetadataTestCase.java |   6 +-
 .../reindex/AsyncBulkByScrollActionTests.java |   2 +-
 .../BulkIndexByScrollResponseMatcher.java     |  13 +
 .../reindex/DeleteByQueryBasicTests.java      | 211 +++++++
 .../reindex/DeleteByQueryCancelTests.java     | 184 ++++++
 .../reindex/DeleteByQueryConcurrentTests.java | 119 ++++
 .../index/reindex/ReindexMetadataTests.java   |  12 +-
 .../index/reindex/ReindexScriptTests.java     |  11 +-
 .../index/reindex/ReindexTestCase.java        |   4 +
 .../reindex/UpdateByQueryMetadataTests.java   |   5 +-
 .../reindex/UpdateByQueryWithScriptTests.java |   8 +-
 .../test/delete_by_query/10_basic.yaml        | 304 ++++++++++
 .../test/delete_by_query/20_validation.yaml   |  99 ++++
 .../test/delete_by_query/30_by_type.yaml      |  72 +++
 .../test/delete_by_query/50_consistency.yaml  |  62 ++
 .../test/delete_by_query/70_throttle.yaml     | 202 +++++++
 .../rest-api-spec/test/reindex/10_basic.yaml  |   3 +
 .../test/reindex/20_validation.yaml           |   2 +-
 .../test/update_by_query/10_basic.yaml        |   2 +
 .../test/update_by_query/10_script.yaml       |   4 +-
 .../rest-api-spec/api/delete_by_query.json    | 207 +++++++
 .../rest-api-spec/api/reindex.rethrottle.json |   2 +-
 42 files changed, 3007 insertions(+), 440 deletions(-)
 create mode 100644 docs/reference/docs/delete-by-query.asciidoc
 create mode 100644 modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java
 create mode 100644 modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryAction.java
 create mode 100644 modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
 create mode 100644 modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java
 create mode 100644 modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java
 create mode 100644 modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java
 create mode 100644 modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java
 create mode 100644 modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryCancelTests.java
 create mode 100644 modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryConcurrentTests.java
 create mode 100644 modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml
 create mode 100644 modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml
 create mode 100644 modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/30_by_type.yaml
 create mode 100644 modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/50_consistency.yaml
 create mode 100644 modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yaml
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json

diff --git a/docs/reference/docs.asciidoc b/docs/reference/docs.asciidoc
index 465d2e60c77..f3b30e7f0c3 100644
--- a/docs/reference/docs.asciidoc
+++ b/docs/reference/docs.asciidoc
@@ -27,6 +27,8 @@ include::docs/get.asciidoc[]
 
 include::docs/delete.asciidoc[]
 
+include::docs/delete-by-query.asciidoc[]
+
 include::docs/update.asciidoc[]
 
 include::docs/update-by-query.asciidoc[]
diff --git a/docs/reference/docs/delete-by-query.asciidoc b/docs/reference/docs/delete-by-query.asciidoc
new file mode 100644
index 00000000000..1562d8c515d
--- /dev/null
+++ b/docs/reference/docs/delete-by-query.asciidoc
@@ -0,0 +1,318 @@
+[[docs-delete-by-query]]
+== Delete By Query API
+
+experimental[The delete-by-query API is new and should still be considered experimental.  The API may change in ways that are not backwards compatible]
+
+The simplest usage of `_delete_by_query` just performs a deletion on every
+document that match a query. Here is the API:
+
+[source,js]
+--------------------------------------------------
+POST twitter/_delete_by_query
+{
+  "query": { <1>
+    "match": {
+      "message": "some message"
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:big_twitter]
+
+<1> The query must be passed as a value to the `query` key, in the same
+way as the <<search-search,Search API>>. You can also use the `q`
+parameter in the same way as the search api.
+
+That will return something like this:
+
+[source,js]
+--------------------------------------------------
+{
+  "took" : 147,
+  "timed_out": false,
+  "deleted": 119,
+  "batches": 1,
+  "version_conflicts": 0,
+  "noops": 0,
+  "retries": {
+    "bulk": 0,
+    "search": 0
+  },
+  "throttled_millis": 0,
+  "requests_per_second": "unlimited",
+  "throttled_until_millis": 0,
+  "total": 119,
+  "failures" : [ ]
+}
+--------------------------------------------------
+// TESTRESPONSE[s/"took" : 147/"took" : "$body.took"/]
+
+`_delete_by_query` gets a snapshot of the index when it starts and deletes what
+it finds using `internal` versioning. That means that you'll get a version
+conflict if the document changes between the time when the snapshot was taken
+and when the delete request is processed. When the versions match the document
+is deleted.
+
+During the `_delete_by_query` execution, multiple search requests are sequentially
+executed in order to find all the matching documents to delete. Every time a batch
+of documents is found, a corresponding bulk request is executed to delete all
+these documents. In case a search or bulk request got rejected, `_delete_by_query`
+ relies on a default policy to retry rejected requests (up to 10 times, with
+ exponential back off). Reaching the maximum retries limit causes the `_delete_by_query`
+ to abort and all failures are returned in the `failures` of the response.
+ The deletions that have been performed still stick. In other words, the process
+ is not rolled back, only aborted. While the first failure causes the abort all
+ failures that are returned by the failing bulk request are returned in the `failures`
+ element so it's possible for there to be quite a few.
+
+If you'd like to count version conflicts rather than cause them to abort then
+set `conflicts=proceed` on the url or `"conflicts": "proceed"` in the request body.
+
+Back to the API format, you can limit `_delete_by_query` to a single type. This
+will only delete `tweet` documents from the `twitter` index:
+
+[source,js]
+--------------------------------------------------
+POST twitter/tweet/_delete_by_query?conflicts=proceed
+{
+  "query": {
+    "match_all": {}
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+It's also possible to delete documents of multiple indexes and multiple
+types at once, just like the search API:
+
+[source,js]
+--------------------------------------------------
+POST twitter,blog/tweet,post/_delete_by_query
+{
+  "query": {
+    "match_all": {}
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT twitter\nPUT blog\nGET _cluster\/health?wait_for_status=yellow\n/]
+
+If you provide `routing` then the routing is copied to the scroll query,
+limiting the process to the shards that match that routing value:
+
+[source,js]
+--------------------------------------------------
+POST twitter/_delete_by_query?routing=1
+{
+  "query": {
+    "range" : {
+        "age" : {
+           "gte" : 10
+        }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+By default `_delete_by_query` uses scroll batches of 1000. You can change the
+batch size with the `scroll_size` URL parameter:
+
+[source,js]
+--------------------------------------------------
+POST twitter/_delete_by_query?scroll_size=5000
+{
+  "query": {
+    "term": {
+      "user": "kimchy"
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+
+[float]
+=== URL Parameters
+
+In addition to the standard parameters like `pretty`, the Delete By Query API
+also supports `refresh`, `wait_for_completion`, `consistency`, and `timeout`.
+
+Sending the `refresh` will refresh all shards involved in the delete by query
+once the request completes. This is different than the Delete API's `refresh`
+parameter which causes just the shard that received the delete request
+to be refreshed.
+
+If the request contains `wait_for_completion=false` then Elasticsearch will
+perform some preflight checks, launch the request, and then return a `task`
+which can be used with <<docs-delete-by-query-task-api,Tasks APIs>> to cancel
+or get the status of the task. For now, once the request is finished the task
+is gone and the only place to look for the ultimate result of the task is in
+the Elasticsearch log file. This will be fixed soon.
+
+`consistency` controls how many copies of a shard must respond to each write
+request. `timeout` controls how long each write request waits for unavailable
+shards to become available. Both work exactly how they work in the
+<<docs-bulk,Bulk API>>.
+
+`requests_per_second` can be set to any decimal number (`1.4`, `6`, `1000`, etc)
+and throttles the number of requests per second that the delete by query issues.
+The throttling is done waiting between bulk batches so that it can manipulate
+the scroll timeout. The wait time is the difference between the time it took the
+batch to complete and the time `requests_per_second * requests_in_the_batch`.
+Since the batch isn't broken into multiple bulk requests large batch sizes will
+cause Elasticsearch to create many requests and then wait for a while before
+starting the next set. This is "bursty" instead of "smooth". The default is
+`unlimited` which is also the only non-number value that it accepts.
+
+[float]
+=== Response body
+
+The JSON response looks like this:
+
+[source,js]
+--------------------------------------------------
+{
+  "took" : 639,
+  "deleted": 0,
+  "batches": 1,
+  "version_conflicts": 2,
+  "retries": 0,
+  "throttled_millis": 0,
+  "failures" : [ ]
+}
+--------------------------------------------------
+
+`took`::
+
+The number of milliseconds from start to end of the whole operation.
+
+`deleted`::
+
+The number of documents that were successfully deleted.
+
+`batches`::
+
+The number of scroll responses pulled back by the the delete by query.
+
+`version_conflicts`::
+
+The number of version conflicts that the delete by query hit.
+
+`retries`::
+
+The number of retries that the delete by query did in response to a full queue.
+
+`throttled_millis`::
+
+Number of milliseconds the request slept to conform to `requests_per_second`.
+
+`failures`::
+
+Array of all indexing failures. If this is non-empty then the request aborted
+because of those failures. See `conflicts` for how to prevent version conflicts
+from aborting the operation.
+
+
+[float]
+[[docs-delete-by-query-task-api]]
+=== Works with the Task API
+
+While Delete By Query is running you can fetch their status using the
+<<tasks,Task API>>:
+
+[source,js]
+--------------------------------------------------
+GET _tasks?detailed=true&action=*/delete/byquery
+--------------------------------------------------
+// CONSOLE
+
+The responses looks like:
+
+[source,js]
+--------------------------------------------------
+{
+  "nodes" : {
+    "r1A2WoRbTwKZ516z6NEs5A" : {
+      "name" : "Tyrannus",
+      "transport_address" : "127.0.0.1:9300",
+      "host" : "127.0.0.1",
+      "ip" : "127.0.0.1:9300",
+      "attributes" : {
+        "testattr" : "test",
+        "portsfile" : "true"
+      },
+      "tasks" : {
+        "r1A2WoRbTwKZ516z6NEs5A:36619" : {
+          "node" : "r1A2WoRbTwKZ516z6NEs5A",
+          "id" : 36619,
+          "type" : "transport",
+          "action" : "indices:data/write/delete/byquery",
+          "status" : {    <1>
+            "total" : 6154,
+            "updated" : 0,
+            "created" : 0,
+            "deleted" : 3500,
+            "batches" : 36,
+            "version_conflicts" : 0,
+            "noops" : 0,
+            "retries": 0,
+            "throttled_millis": 0
+          },
+          "description" : ""
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+
+<1> this object contains the actual status. It is just like the response json
+with the important addition of the `total` field. `total` is the total number
+of operations that the reindex expects to perform. You can estimate the
+progress by adding the `updated`, `created`, and `deleted` fields. The request
+will finish when their sum is equal to the `total` field.
+
+
+[float]
+[[docs-delete-by-query-cancel-task-api]]
+=== Works with the Cancel Task API
+
+Any Delete By Query can be canceled using the <<tasks,Task Cancel API>>:
+
+[source,js]
+--------------------------------------------------
+POST _tasks/taskid:1/_cancel
+--------------------------------------------------
+// CONSOLE
+
+The `task_id` can be found using the tasks API above.
+
+Cancelation should happen quickly but might take a few seconds. The task status
+API above will continue to list the task until it is wakes to cancel itself.
+
+
+[float]
+[[docs-delete-by-query-rethrottle]]
+=== Rethrottling
+
+The value of `requests_per_second` can be changed on a running delete by query
+using the `_rethrottle` API:
+
+[source,js]
+--------------------------------------------------
+POST _delete_by_query/taskid:1/_rethrottle?requests_per_second=unlimited
+--------------------------------------------------
+// CONSOLE
+
+The `task_id` can be found using the tasks API above.
+
+Just like when setting it on the `_delete_by_query` API `requests_per_second`
+can be either `unlimited` to disable throttling or any decimal number like `1.7`
+or `12` to throttle to that level. Rethrottling that speeds up the query takes
+effect immediately but rethrotting that slows down the query will take effect
+on after completing the current batch. This prevents scroll timeouts.
diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc
index e5a62ddf33a..ac4da4251be 100644
--- a/docs/reference/docs/update-by-query.asciidoc
+++ b/docs/reference/docs/update-by-query.asciidoc
@@ -61,7 +61,7 @@ and the time when it attempted to update the document. This is fine because
 that update will have picked up the online mapping update.
 
 Back to the API format, you can limit `_update_by_query` to a single type. This
-will only update `tweet`s from the `twitter` index:
+will only update `tweet` documents from the `twitter` index:
 
 [source,js]
 --------------------------------------------------
@@ -119,7 +119,7 @@ Just as in <<docs-update,Update API>> you can set `ctx.op = "noop"` if
 your script decides that it doesn't have to make any changes. That will cause
 `_update_by_query` to omit that document from its updates. Setting `ctx.op` to
 anything else is an error. If you want to delete by a query you can use the
-{plugins}/plugins-delete-by-query.html[Delete by Query plugin] instead. Setting any
+<<docs-delete-by-query,Delete By Query API>> instead. Setting any
 other field in `ctx` is an error.
 
 Note that we stopped specifying `conflicts=proceed`. In this case we want a
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
index f8561d17acd..ee2f5484737 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
@@ -75,7 +75,7 @@ import static org.elasticsearch.search.sort.SortBuilders.fieldSort;
  * Abstract base for scrolling across a search and executing bulk actions on all results. All package private methods are package private so
  * their tests can use them. Most methods run in the listener thread pool because the are meant to be fast and don't expect to block.
  */
-public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBulkByScrollRequest<Request>, Response> {
+public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBulkByScrollRequest<Request>> {
     /**
      * The request for this action. Named mainRequest because we create lots of <code>request</code> variables all representing child
      * requests of this mainRequest.
@@ -92,12 +92,13 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
     private final ParentTaskAssigningClient client;
     private final ThreadPool threadPool;
     private final SearchRequest firstSearchRequest;
-    private final ActionListener<Response> listener;
+    private final ActionListener<BulkIndexByScrollResponse> listener;
     private final BackoffPolicy backoffPolicy;
     private final Retry bulkRetry;
 
     public AbstractAsyncBulkByScrollAction(BulkByScrollTask task, ESLogger logger, ParentTaskAssigningClient client,
-            ThreadPool threadPool, Request mainRequest, SearchRequest firstSearchRequest, ActionListener<Response> listener) {
+                                           ThreadPool threadPool, Request mainRequest, SearchRequest firstSearchRequest,
+                                           ActionListener<BulkIndexByScrollResponse> listener) {
         this.task = task;
         this.logger = logger;
         this.client = client;
@@ -111,8 +112,13 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
 
     protected abstract BulkRequest buildBulk(Iterable<SearchHit> docs);
 
-    protected abstract Response buildResponse(TimeValue took, List<Failure> indexingFailures, List<ShardSearchFailure> searchFailures,
-            boolean timedOut);
+    /**
+     * Build the response for reindex actions.
+     */
+    protected BulkIndexByScrollResponse buildResponse(TimeValue took, List<BulkItemResponse.Failure> indexingFailures,
+                                                      List<ShardSearchFailure> searchFailures, boolean timedOut) {
+        return new BulkIndexByScrollResponse(took, task.getStatus(), indexingFailures, searchFailures, timedOut);
+    }
 
     /**
      * Start the action by firing the initial search request.
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java
index df9245346b3..0e3f3678851 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java
@@ -20,15 +20,16 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.client.ParentTaskAssigningClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.mapper.internal.IdFieldMapper;
 import org.elasticsearch.index.mapper.internal.IndexFieldMapper;
 import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
@@ -40,6 +41,7 @@ import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
 import org.elasticsearch.index.mapper.internal.VersionFieldMapper;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchHit;
@@ -47,9 +49,9 @@ import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.function.BiFunction;
 
 import static java.util.Collections.emptyMap;
 
@@ -57,91 +59,106 @@ import static java.util.Collections.emptyMap;
  * Abstract base for scrolling across a search and executing bulk indexes on all
  * results.
  */
-public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends AbstractBulkIndexByScrollRequest<Request>>
-        extends AbstractAsyncBulkByScrollAction<Request, BulkIndexByScrollResponse> {
+public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends AbstractBulkByScrollRequest<Request>>
+        extends AbstractAsyncBulkByScrollAction<Request> {
 
-    private final ScriptService scriptService;
-    private final CompiledScript script;
+    protected final ScriptService scriptService;
+    protected final ClusterState clusterState;
 
-    public AbstractAsyncBulkIndexByScrollAction(BulkByScrollTask task, ESLogger logger, ScriptService scriptService, ClusterState state,
-            ParentTaskAssigningClient client, ThreadPool threadPool, Request mainRequest, SearchRequest firstSearchRequest,
-            ActionListener<BulkIndexByScrollResponse> listener) {
+    /**
+     * This BiFunction is used to apply various changes depending of the Reindex action and  the search hit,
+     * from copying search hit metadata (parent, routing, etc) to potentially transforming the
+     * {@link RequestWrapper} completely.
+     */
+    private final BiFunction<RequestWrapper<?>, SearchHit, RequestWrapper<?>> scriptApplier;
+
+    public AbstractAsyncBulkIndexByScrollAction(BulkByScrollTask task, ESLogger logger, ParentTaskAssigningClient client,
+                                                ThreadPool threadPool, Request mainRequest, SearchRequest firstSearchRequest,
+                                                ActionListener<BulkIndexByScrollResponse> listener,
+                                                ScriptService scriptService, ClusterState clusterState) {
         super(task, logger, client, threadPool, mainRequest, firstSearchRequest, listener);
         this.scriptService = scriptService;
-        if (mainRequest.getScript() == null) {
-            script = null;
-        } else {
-            script = scriptService.compile(mainRequest.getScript(), ScriptContext.Standard.UPDATE, emptyMap(), state);
-        }
-    }
-
-    @Override
-    protected BulkIndexByScrollResponse buildResponse(TimeValue took, List<BulkItemResponse.Failure> indexingFailures,
-                                     List<ShardSearchFailure> searchFailures, boolean timedOut) {
-        return new BulkIndexByScrollResponse(took, task.getStatus(), indexingFailures, searchFailures, timedOut);
+        this.clusterState = clusterState;
+        this.scriptApplier = Objects.requireNonNull(buildScriptApplier(), "script applier must not be null");
     }
 
     /**
-     * Build the IndexRequest for a single search hit. This shouldn't handle
-     * metadata or the script. That will be handled by copyMetadata and
-     * applyScript functions that can be overridden.
+     * Build the {@link BiFunction} to apply to all {@link RequestWrapper}.
      */
-    protected abstract IndexRequest buildIndexRequest(SearchHit doc);
+    protected BiFunction<RequestWrapper<?>, SearchHit, RequestWrapper<?>> buildScriptApplier() {
+        // The default script applier executes a no-op
+        return (request, searchHit) -> request;
+    }
 
     @Override
     protected BulkRequest buildBulk(Iterable<SearchHit> docs) {
         BulkRequest bulkRequest = new BulkRequest();
-        ExecutableScript executableScript = null;
-        Map<String, Object> scriptCtx = null;
-
         for (SearchHit doc : docs) {
-            if (doc.hasSource()) {
-                /*
-                 * Either the document didn't store _source or we didn't fetch it for some reason. Since we don't allow the user to
-                 * change the "fields" part of the search request it is unlikely that we got here because we didn't fetch _source.
-                 * Thus the error message assumes that it wasn't stored.
-                 */
-                throw new IllegalArgumentException("[" + doc.index() + "][" + doc.type() + "][" + doc.id() + "] didn't store _source");
-            }
-            IndexRequest index = buildIndexRequest(doc);
-            copyMetadata(index, doc);
-            if (script != null) {
-                if (executableScript == null) {
-                    executableScript = scriptService.executable(script, mainRequest.getScript().getParams());
-                    scriptCtx = new HashMap<>();
-                }
-                if (false == applyScript(index, doc, executableScript, scriptCtx)) {
-                    continue;
+            if (accept(doc)) {
+                RequestWrapper<?> request = scriptApplier.apply(copyMetadata(buildRequest(doc), doc), doc);
+                if (request != null) {
+                    bulkRequest.add(request.self());
                 }
             }
-            bulkRequest.add(index);
         }
-
         return bulkRequest;
     }
 
     /**
-     * Copies the metadata from a hit to the index request.
+     * Used to accept or ignore a search hit. Ignored search hits will be excluded
+     * from the bulk request. It is also where we fail on invalid search hits, like
+     * when the document has no source but it's required.
      */
-    protected void copyMetadata(IndexRequest index, SearchHit doc) {
-        index.parent(fieldValue(doc, ParentFieldMapper.NAME));
-        copyRouting(index, doc);
-        // Comes back as a Long but needs to be a string
-        Long timestamp = fieldValue(doc, TimestampFieldMapper.NAME);
-        if (timestamp != null) {
-            index.timestamp(timestamp.toString());
-        }
-        Long ttl = fieldValue(doc, TTLFieldMapper.NAME);
-        if (ttl != null) {
-            index.ttl(ttl);
+    protected boolean accept(SearchHit doc) {
+        if (doc.hasSource()) {
+            /*
+             * Either the document didn't store _source or we didn't fetch it for some reason. Since we don't allow the user to
+             * change the "fields" part of the search request it is unlikely that we got here because we didn't fetch _source.
+             * Thus the error message assumes that it wasn't stored.
+             */
+            throw new IllegalArgumentException("[" + doc.index() + "][" + doc.type() + "][" + doc.id() + "] didn't store _source");
         }
+        return true;
     }
 
     /**
-     * Part of copyMetadata but called out individual for easy overwriting.
+     * Build the {@link RequestWrapper} for a single search hit. This shouldn't handle
+     * metadata or scripting. That will be handled by copyMetadata and
+     * apply functions that can be overridden.
      */
-    protected void copyRouting(IndexRequest index, SearchHit doc) {
-        index.routing(fieldValue(doc, RoutingFieldMapper.NAME));
+    protected abstract RequestWrapper<?> buildRequest(SearchHit doc);
+
+    /**
+     * Copies the metadata from a hit to the request.
+     */
+    protected RequestWrapper<?> copyMetadata(RequestWrapper<?> request, SearchHit doc) {
+        copyParent(request, fieldValue(doc, ParentFieldMapper.NAME));
+        copyRouting(request, fieldValue(doc, RoutingFieldMapper.NAME));
+
+        // Comes back as a Long but needs to be a string
+        Long timestamp = fieldValue(doc, TimestampFieldMapper.NAME);
+        if (timestamp != null) {
+            request.setTimestamp(timestamp.toString());
+        }
+        Long ttl = fieldValue(doc, TTLFieldMapper.NAME);
+        if (ttl != null) {
+            request.setTtl(ttl);
+        }
+        return request;
+    }
+
+    /**
+     * Copy the parent from a search hit to the request.
+     */
+    protected void copyParent(RequestWrapper<?> request, String parent) {
+        request.setParent(parent);
+    }
+
+    /**
+     * Copy the routing from a search hit to the request.
+     */
+    protected void copyRouting(RequestWrapper<?> request, String routing) {
+        request.setRouting(routing);
     }
 
     protected <T> T fieldValue(SearchHit doc, String fieldName) {
@@ -150,106 +167,327 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
     }
 
     /**
-     * Apply a script to the request.
-     *
-     * @return is this request still ok to apply (true) or is it a noop (false)
+     * Wrapper for the {@link ActionRequest} that are used in this action class.
      */
-    @SuppressWarnings("unchecked")
-    protected boolean applyScript(IndexRequest index, SearchHit doc, ExecutableScript script, final Map<String, Object> ctx) {
-        if (script == null) {
-            return true;
-        }
-        ctx.put(IndexFieldMapper.NAME, doc.index());
-        ctx.put(TypeFieldMapper.NAME, doc.type());
-        ctx.put(IdFieldMapper.NAME, doc.id());
-        Long oldVersion = doc.getVersion();
-        ctx.put(VersionFieldMapper.NAME, oldVersion);
-        String oldParent = fieldValue(doc, ParentFieldMapper.NAME);
-        ctx.put(ParentFieldMapper.NAME, oldParent);
-        String oldRouting = fieldValue(doc, RoutingFieldMapper.NAME);
-        ctx.put(RoutingFieldMapper.NAME, oldRouting);
-        Long oldTimestamp = fieldValue(doc, TimestampFieldMapper.NAME);
-        ctx.put(TimestampFieldMapper.NAME, oldTimestamp);
-        Long oldTTL = fieldValue(doc, TTLFieldMapper.NAME);
-        ctx.put(TTLFieldMapper.NAME, oldTTL);
-        ctx.put(SourceFieldMapper.NAME, index.sourceAsMap());
-        ctx.put("op", "update");
-        script.setNextVar("ctx", ctx);
-        script.run();
-        Map<String, Object> resultCtx = (Map<String, Object>) script.unwrap(ctx);
-        String newOp = (String) resultCtx.remove("op");
-        if (newOp == null) {
-            throw new IllegalArgumentException("Script cleared op!");
-        }
-        if ("noop".equals(newOp)) {
-            task.countNoop();
-            return false;
-        }
-        if (false == "update".equals(newOp)) {
-            throw new IllegalArgumentException("Invalid op [" + newOp + ']');
-        }
+    interface RequestWrapper<Self extends ActionRequest<Self>> {
 
-        /*
-         * It'd be lovely to only set the source if we know its been modified
-         * but it isn't worth keeping two copies of it around just to check!
-         */
-        index.source((Map<String, Object>) resultCtx.remove(SourceFieldMapper.NAME));
+        void setIndex(String index);
 
-        Object newValue = ctx.remove(IndexFieldMapper.NAME);
-        if (false == doc.index().equals(newValue)) {
-            scriptChangedIndex(index, newValue);
-        }
-        newValue = ctx.remove(TypeFieldMapper.NAME);
-        if (false == doc.type().equals(newValue)) {
-            scriptChangedType(index, newValue);
-        }
-        newValue = ctx.remove(IdFieldMapper.NAME);
-        if (false == doc.id().equals(newValue)) {
-            scriptChangedId(index, newValue);
-        }
-        newValue = ctx.remove(VersionFieldMapper.NAME);
-        if (false == Objects.equals(oldVersion, newValue)) {
-            scriptChangedVersion(index, newValue);
-        }
-        newValue = ctx.remove(ParentFieldMapper.NAME);
-        if (false == Objects.equals(oldParent, newValue)) {
-            scriptChangedParent(index, newValue);
-        }
-        /*
-         * Its important that routing comes after parent in case you want to
-         * change them both.
-         */
-        newValue = ctx.remove(RoutingFieldMapper.NAME);
-        if (false == Objects.equals(oldRouting, newValue)) {
-            scriptChangedRouting(index, newValue);
-        }
-        newValue = ctx.remove(TimestampFieldMapper.NAME);
-        if (false == Objects.equals(oldTimestamp, newValue)) {
-            scriptChangedTimestamp(index, newValue);
-        }
-        newValue = ctx.remove(TTLFieldMapper.NAME);
-        if (false == Objects.equals(oldTTL, newValue)) {
-            scriptChangedTTL(index, newValue);
-        }
-        if (false == ctx.isEmpty()) {
-            throw new IllegalArgumentException("Invalid fields added to ctx [" + String.join(",", ctx.keySet()) + ']');
-        }
-        return true;
+        void setType(String type);
+
+        void setId(String id);
+
+        void setVersion(long version);
+
+        void setVersionType(VersionType versionType);
+
+        void setParent(String parent);
+
+        void setRouting(String routing);
+
+        void setTimestamp(String timestamp);
+
+        void setTtl(Long ttl);
+
+        void setSource(Map<String, Object> source);
+
+        Map<String, Object> getSource();
+
+        Self self();
     }
 
-    protected abstract void scriptChangedIndex(IndexRequest index, Object to);
+    /**
+     * {@link RequestWrapper} for {@link IndexRequest}
+     */
+    public static class IndexRequestWrapper implements RequestWrapper<IndexRequest> {
 
-    protected abstract void scriptChangedType(IndexRequest index, Object to);
+        private final IndexRequest request;
 
-    protected abstract void scriptChangedId(IndexRequest index, Object to);
+        IndexRequestWrapper(IndexRequest request) {
+            this.request = Objects.requireNonNull(request, "Wrapped IndexRequest can not be null");
+        }
 
-    protected abstract void scriptChangedVersion(IndexRequest index, Object to);
+        @Override
+        public void setIndex(String index) {
+            request.index(index);
+        }
 
-    protected abstract void scriptChangedRouting(IndexRequest index, Object to);
+        @Override
+        public void setType(String type) {
+            request.type(type);
+        }
 
-    protected abstract void scriptChangedParent(IndexRequest index, Object to);
+        @Override
+        public void setId(String id) {
+            request.id(id);
+        }
 
-    protected abstract void scriptChangedTimestamp(IndexRequest index, Object to);
+        @Override
+        public void setVersion(long version) {
+            request.version(version);
+        }
 
-    protected abstract void scriptChangedTTL(IndexRequest index, Object to);
+        @Override
+        public void setVersionType(VersionType versionType) {
+            request.versionType(versionType);
+        }
+
+        @Override
+        public void setParent(String parent) {
+            request.parent(parent);
+        }
+
+        @Override
+        public void setRouting(String routing) {
+            request.routing(routing);
+        }
+
+        @Override
+        public void setTimestamp(String timestamp) {
+            request.timestamp(timestamp);
+        }
+
+        @Override
+        public void setTtl(Long ttl) {
+            if (ttl == null) {
+                request.ttl((TimeValue) null);
+            } else {
+                request.ttl(ttl);
+            }
+        }
+
+        @Override
+        public Map<String, Object> getSource() {
+            return request.sourceAsMap();
+        }
+
+        @Override
+        public void setSource(Map<String, Object> source) {
+            request.source(source);
+        }
+
+        @Override
+        public IndexRequest self() {
+            return request;
+        }
+    }
+
+    /**
+     * Wraps a {@link IndexRequest} in a {@link RequestWrapper}
+     */
+    static RequestWrapper<IndexRequest> wrap(IndexRequest request) {
+        return new IndexRequestWrapper(request);
+    }
+
+    /**
+     * {@link RequestWrapper} for {@link DeleteRequest}
+     */
+    public static class DeleteRequestWrapper implements RequestWrapper<DeleteRequest> {
+
+        private final DeleteRequest request;
+
+        DeleteRequestWrapper(DeleteRequest request) {
+            this.request = Objects.requireNonNull(request, "Wrapped DeleteRequest can not be null");
+        }
+
+        @Override
+        public void setIndex(String index) {
+            request.index(index);
+        }
+
+        @Override
+        public void setType(String type) {
+            request.type(type);
+        }
+
+        @Override
+        public void setId(String id) {
+            request.id(id);
+        }
+
+        @Override
+        public void setVersion(long version) {
+            request.version(version);
+        }
+
+        @Override
+        public void setVersionType(VersionType versionType) {
+            request.versionType(versionType);
+        }
+
+        @Override
+        public void setParent(String parent) {
+            request.parent(parent);
+        }
+
+        @Override
+        public void setRouting(String routing) {
+            request.routing(routing);
+        }
+
+        @Override
+        public void setTimestamp(String timestamp) {
+            throw new UnsupportedOperationException("unable to set [timestamp] on action request [" + request.getClass() + "]");
+        }
+
+        @Override
+        public void setTtl(Long ttl) {
+            throw new UnsupportedOperationException("unable to set [ttl] on action request [" + request.getClass() + "]");
+        }
+
+        @Override
+        public Map<String, Object> getSource() {
+            throw new UnsupportedOperationException("unable to get source from action request [" + request.getClass() + "]");
+        }
+
+        @Override
+        public void setSource(Map<String, Object> source) {
+            throw new UnsupportedOperationException("unable to set [source] on action request [" + request.getClass() + "]");
+        }
+
+        @Override
+        public DeleteRequest self() {
+            return request;
+        }
+    }
+
+    /**
+     * Wraps a {@link DeleteRequest} in a {@link RequestWrapper}
+     */
+    static RequestWrapper<DeleteRequest> wrap(DeleteRequest request) {
+        return new DeleteRequestWrapper(request);
+    }
+
+    /**
+     * Apply a {@link Script} to a {@link RequestWrapper}
+     */
+    public abstract class ScriptApplier implements BiFunction<RequestWrapper<?>, SearchHit, RequestWrapper<?>> {
+
+        private final BulkByScrollTask task;
+        private final ScriptService scriptService;
+        private final ClusterState state;
+        private final Script script;
+        private final Map<String, Object> params;
+
+        private ExecutableScript executable;
+        private Map<String, Object> context;
+
+        public ScriptApplier(BulkByScrollTask task, ScriptService scriptService, Script script, ClusterState state,
+                             Map<String, Object> params) {
+            this.task = task;
+            this.scriptService = scriptService;
+            this.script = script;
+            this.state = state;
+            this.params = params;
+        }
+
+        @Override
+        @SuppressWarnings("unchecked")
+        public RequestWrapper<?> apply(RequestWrapper<?> request, SearchHit doc) {
+            if (script == null) {
+                return request;
+            }
+            if (executable == null) {
+                CompiledScript compiled = scriptService.compile(script, ScriptContext.Standard.UPDATE, emptyMap(), state);
+                executable = scriptService.executable(compiled, params);
+            }
+            if (context == null) {
+                context = new HashMap<>();
+            }
+
+            context.put(IndexFieldMapper.NAME, doc.index());
+            context.put(TypeFieldMapper.NAME, doc.type());
+            context.put(IdFieldMapper.NAME, doc.id());
+            Long oldVersion = doc.getVersion();
+            context.put(VersionFieldMapper.NAME, oldVersion);
+            String oldParent = fieldValue(doc, ParentFieldMapper.NAME);
+            context.put(ParentFieldMapper.NAME, oldParent);
+            String oldRouting = fieldValue(doc, RoutingFieldMapper.NAME);
+            context.put(RoutingFieldMapper.NAME, oldRouting);
+            Long oldTimestamp = fieldValue(doc, TimestampFieldMapper.NAME);
+            context.put(TimestampFieldMapper.NAME, oldTimestamp);
+            Long oldTTL = fieldValue(doc, TTLFieldMapper.NAME);
+            context.put(TTLFieldMapper.NAME, oldTTL);
+            context.put(SourceFieldMapper.NAME, request.getSource());
+            context.put("op", "update");
+            executable.setNextVar("ctx", context);
+            executable.run();
+
+            Map<String, Object> resultCtx = (Map<String, Object>) executable.unwrap(context);
+            String newOp = (String) resultCtx.remove("op");
+            if (newOp == null) {
+                throw new IllegalArgumentException("Script cleared op!");
+            }
+            if ("noop".equals(newOp)) {
+                task.countNoop();
+                return null;
+            }
+            if (false == "update".equals(newOp)) {
+                throw new IllegalArgumentException("Invalid op [" + newOp + ']');
+            }
+
+            /*
+             * It'd be lovely to only set the source if we know its been modified
+             * but it isn't worth keeping two copies of it around just to check!
+             */
+            request.setSource((Map<String, Object>) resultCtx.remove(SourceFieldMapper.NAME));
+
+            Object newValue = context.remove(IndexFieldMapper.NAME);
+            if (false == doc.index().equals(newValue)) {
+                scriptChangedIndex(request, newValue);
+            }
+            newValue = context.remove(TypeFieldMapper.NAME);
+            if (false == doc.type().equals(newValue)) {
+                scriptChangedType(request, newValue);
+            }
+            newValue = context.remove(IdFieldMapper.NAME);
+            if (false == doc.id().equals(newValue)) {
+                scriptChangedId(request, newValue);
+            }
+            newValue = context.remove(VersionFieldMapper.NAME);
+            if (false == Objects.equals(oldVersion, newValue)) {
+                scriptChangedVersion(request, newValue);
+            }
+            newValue = context.remove(ParentFieldMapper.NAME);
+            if (false == Objects.equals(oldParent, newValue)) {
+                scriptChangedParent(request, newValue);
+            }
+            /*
+             * Its important that routing comes after parent in case you want to
+             * change them both.
+             */
+            newValue = context.remove(RoutingFieldMapper.NAME);
+            if (false == Objects.equals(oldRouting, newValue)) {
+                scriptChangedRouting(request, newValue);
+            }
+            newValue = context.remove(TimestampFieldMapper.NAME);
+            if (false == Objects.equals(oldTimestamp, newValue)) {
+                scriptChangedTimestamp(request, newValue);
+            }
+            newValue = context.remove(TTLFieldMapper.NAME);
+            if (false == Objects.equals(oldTTL, newValue)) {
+                scriptChangedTTL(request, newValue);
+            }
+            if (false == context.isEmpty()) {
+                throw new IllegalArgumentException("Invalid fields added to context [" + String.join(",", context.keySet()) + ']');
+            }
+            return request;
+        }
+
+        protected abstract void scriptChangedIndex(RequestWrapper<?> request, Object to);
+
+        protected abstract void scriptChangedType(RequestWrapper<?> request, Object to);
+
+        protected abstract void scriptChangedId(RequestWrapper<?> request, Object to);
+
+        protected abstract void scriptChangedVersion(RequestWrapper<?> request, Object to);
+
+        protected abstract void scriptChangedRouting(RequestWrapper<?> request, Object to);
+
+        protected abstract void scriptChangedParent(RequestWrapper<?> request, Object to);
+
+        protected abstract void scriptChangedTimestamp(RequestWrapper<?> request, Object to);
+
+        protected abstract void scriptChangedTTL(RequestWrapper<?> request, Object to);
+    }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
index d98735d3fb4..e78a6a9c350 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.WriteConsistencyLevel;
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -45,32 +46,6 @@ public abstract class AbstractBaseReindexRestHandler<
                 TA extends TransportAction<Request, BulkIndexByScrollResponse>
             > extends BaseRestHandler {
 
-    /**
-     * @return requests_per_second from the request as a float if it was on the request, null otherwise
-     */
-    public static Float parseRequestsPerSecond(RestRequest request) {
-        String requestsPerSecondString = request.param("requests_per_second");
-        if (requestsPerSecondString == null) {
-            return null;
-        }
-        if ("unlimited".equals(requestsPerSecondString)) {
-            return Float.POSITIVE_INFINITY;
-        }
-        float requestsPerSecond;
-        try {
-            requestsPerSecond = Float.parseFloat(requestsPerSecondString);
-        } catch (NumberFormatException e) {
-            throw new IllegalArgumentException(
-                    "[requests_per_second] must be a float greater than 0. Use \"unlimited\" to disable throttling.", e);
-        }
-        if (requestsPerSecond <= 0) {
-            // We validate here and in the setters because the setters use "Float.POSITIVE_INFINITY" instead of "unlimited"
-            throw new IllegalArgumentException(
-                    "[requests_per_second] must be a float greater than 0. Use \"unlimited\" to disable throttling.");
-        }
-        return requestsPerSecond;
-    }
-
     protected final IndicesQueriesRegistry indicesQueriesRegistry;
     protected final AggregatorParsers aggParsers;
     protected final Suggesters suggesters;
@@ -88,41 +63,95 @@ public abstract class AbstractBaseReindexRestHandler<
         this.action = action;
     }
 
-    protected void execute(RestRequest request, Request internalRequest, RestChannel channel,
-                           boolean includeCreated, boolean includeUpdated, boolean includeDeleted) throws IOException {
-        Float requestsPerSecond = parseRequestsPerSecond(request);
-        if (requestsPerSecond != null) {
-            internalRequest.setRequestsPerSecond(requestsPerSecond);
-        }
+    protected void handleRequest(RestRequest request, RestChannel channel,
+                                 boolean includeCreated, boolean includeUpdated, boolean includeDeleted) throws IOException {
 
+        // Build the internal request
+        Request internal = setCommonOptions(request, buildRequest(request));
+
+        // Executes the request and waits for completion
         if (request.paramAsBoolean("wait_for_completion", true)) {
             Map<String, String> params = new HashMap<>();
             params.put(BulkByScrollTask.Status.INCLUDE_CREATED, Boolean.toString(includeCreated));
             params.put(BulkByScrollTask.Status.INCLUDE_UPDATED, Boolean.toString(includeUpdated));
             params.put(BulkByScrollTask.Status.INCLUDE_DELETED, Boolean.toString(includeDeleted));
 
-            action.execute(internalRequest, new BulkIndexByScrollResponseContentListener<>(channel, params));
+            action.execute(internal, new BulkIndexByScrollResponseContentListener<>(channel, params));
             return;
         }
+
         /*
          * Lets try and validate before forking so the user gets some error. The
          * task can't totally validate until it starts but this is better than
          * nothing.
          */
-        ActionRequestValidationException validationException = internalRequest.validate();
+        ActionRequestValidationException validationException = internal.validate();
         if (validationException != null) {
             channel.sendResponse(new BytesRestResponse(channel, validationException));
             return;
         }
-        Task task = action.execute(internalRequest, LoggingTaskListener.instance());
-        sendTask(channel, task);
+        sendTask(channel, action.execute(internal, LoggingTaskListener.instance()));
+    }
+
+    /**
+     * Build the Request based on the RestRequest.
+     */
+    protected abstract Request buildRequest(RestRequest request) throws IOException;
+
+    /**
+     * Sets common options of {@link AbstractBulkByScrollRequest} requests.
+     */
+    protected Request setCommonOptions(RestRequest restRequest, Request request) {
+        assert restRequest != null : "RestRequest should not be null";
+        assert request != null : "Request should not be null";
+
+        request.setRefresh(restRequest.paramAsBoolean("refresh", request.isRefresh()));
+        request.setTimeout(restRequest.paramAsTime("timeout", request.getTimeout()));
+
+        String consistency = restRequest.param("consistency");
+        if (consistency != null) {
+            request.setConsistency(WriteConsistencyLevel.fromString(consistency));
+        }
+
+        Float requestsPerSecond = parseRequestsPerSecond(restRequest);
+        if (requestsPerSecond != null) {
+            request.setRequestsPerSecond(requestsPerSecond);
+        }
+        return request;
     }
 
     private void sendTask(RestChannel channel, Task task) throws IOException {
-        XContentBuilder builder = channel.newBuilder();
-        builder.startObject();
-        builder.field("task", clusterService.localNode().getId() + ":" + task.getId());
-        builder.endObject();
-        channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder));
+        try (XContentBuilder builder = channel.newBuilder()) {
+            builder.startObject();
+            builder.field("task", clusterService.localNode().getId() + ":" + task.getId());
+            builder.endObject();
+            channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder));
+        }
+    }
+
+    /**
+     * @return requests_per_second from the request as a float if it was on the request, null otherwise
+     */
+    public static Float parseRequestsPerSecond(RestRequest request) {
+        String requestsPerSecondString = request.param("requests_per_second");
+        if (requestsPerSecondString == null) {
+            return null;
+        }
+        if ("unlimited".equals(requestsPerSecondString)) {
+            return  Float.POSITIVE_INFINITY;
+        }
+        float requestsPerSecond;
+        try {
+            requestsPerSecond = Float.parseFloat(requestsPerSecondString);
+        } catch (NumberFormatException e) {
+            throw new IllegalArgumentException(
+                    "[requests_per_second] must be a float greater than 0. Use \"unlimited\" to disable throttling.", e);
+        }
+        if (requestsPerSecond <= 0) {
+            // We validate here and in the setters because the setters use "Float.POSITIVE_INFINITY" instead of "unlimited"
+            throw new IllegalArgumentException(
+                    "[requests_per_second] must be a float greater than 0. Use \"unlimited\" to disable throttling.");
+        }
+        return requestsPerSecond;
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java
new file mode 100644
index 00000000000..926da3befdd
--- /dev/null
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.support.TransportAction;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.search.RestSearchAction;
+import org.elasticsearch.rest.action.support.RestActions;
+import org.elasticsearch.search.aggregations.AggregatorParsers;
+import org.elasticsearch.search.suggest.Suggesters;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.function.Consumer;
+
+import static org.elasticsearch.index.reindex.AbstractBulkByScrollRequest.SIZE_ALL_MATCHES;
+
+/**
+ * Rest handler for reindex actions that accepts a search request like Update-By-Query or Delete-By-Query
+ */
+public abstract class AbstractBulkByQueryRestHandler<
+        Request extends AbstractBulkByScrollRequest<Request>,
+        TA extends TransportAction<Request, BulkIndexByScrollResponse>> extends AbstractBaseReindexRestHandler<Request, TA> {
+
+    protected AbstractBulkByQueryRestHandler(Settings settings, Client client, IndicesQueriesRegistry indicesQueriesRegistry,
+                                             AggregatorParsers aggParsers, Suggesters suggesters, ClusterService clusterService,
+                                             TA action) {
+        super(settings, client, indicesQueriesRegistry, aggParsers, suggesters, clusterService, action);
+    }
+
+    protected void parseInternalRequest(Request internal, RestRequest restRequest,
+                                        Map<String, Consumer<Object>> consumers) throws IOException {
+        assert internal != null : "Request should not be null";
+        assert restRequest != null : "RestRequest should not be null";
+
+        SearchRequest searchRequest = internal.getSearchRequest();
+        int scrollSize = searchRequest.source().size();
+        searchRequest.source().size(SIZE_ALL_MATCHES);
+
+        parseSearchRequest(searchRequest, restRequest, consumers);
+
+        internal.setSize(searchRequest.source().size());
+        searchRequest.source().size(restRequest.paramAsInt("scroll_size", scrollSize));
+
+        String conflicts = restRequest.param("conflicts");
+        if (conflicts != null) {
+            internal.setConflicts(conflicts);
+        }
+
+        // Let the requester set search timeout. It is probably only going to be useful for testing but who knows.
+        if (restRequest.hasParam("search_timeout")) {
+            searchRequest.source().timeout(restRequest.paramAsTime("search_timeout", null));
+        }
+    }
+
+    protected void parseSearchRequest(SearchRequest searchRequest, RestRequest restRequest,
+                                      Map<String, Consumer<Object>> consumers) throws IOException {
+        assert searchRequest != null : "SearchRequest should not be null";
+        assert restRequest != null : "RestRequest should not be null";
+
+        /*
+         * We can't send parseSearchRequest REST content that it doesn't support
+         * so we will have to remove the content that is valid in addition to
+         * what it supports from the content first. This is a temporary hack and
+         * should get better when SearchRequest has full ObjectParser support
+         * then we can delegate and stuff.
+         */
+        BytesReference content = RestActions.hasBodyContent(restRequest) ? RestActions.getRestContent(restRequest) : null;
+        if ((content != null) && (consumers != null && consumers.size() > 0)) {
+            Tuple<XContentType, Map<String, Object>> body = XContentHelper.convertToMap(content, false);
+            boolean modified = false;
+            for (Map.Entry<String, Consumer<Object>> consumer : consumers.entrySet()) {
+                Object value = body.v2().remove(consumer.getKey());
+                if (value != null) {
+                    consumer.getValue().accept(value);
+                    modified = true;
+                }
+            }
+
+            if (modified) {
+                try (XContentBuilder builder = XContentFactory.contentBuilder(body.v1())) {
+                    content = builder.map(body.v2()).bytes();
+                }
+            }
+        }
+
+        RestSearchAction.parseSearchRequest(searchRequest, indicesQueriesRegistry, restRequest, parseFieldMatcher, aggParsers,
+                suggesters, content);
+    }
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryAction.java
new file mode 100644
index 00000000000..c789e9c77b4
--- /dev/null
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryAction.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.client.ElasticsearchClient;
+
+public class DeleteByQueryAction extends Action<DeleteByQueryRequest, BulkIndexByScrollResponse, DeleteByQueryRequestBuilder> {
+
+    public static final DeleteByQueryAction INSTANCE = new DeleteByQueryAction();
+    public static final String NAME = "indices:data/write/delete/byquery";
+
+    private DeleteByQueryAction() {
+        super(NAME);
+    }
+
+    @Override
+    public DeleteByQueryRequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new DeleteByQueryRequestBuilder(client, this);
+    }
+
+    @Override
+    public BulkIndexByScrollResponse newResponse() {
+        return new BulkIndexByScrollResponse();
+    }
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
new file mode 100644
index 00000000000..327459bd339
--- /dev/null
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.search.SearchRequest;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * Creates a new {@link DeleteByQueryRequest} that uses scrolling and bulk requests to delete all documents matching
+ * the query. This can have performance as well as visibility implications.
+ *
+ * Delete-by-query now has the following semantics:
+ * <ul>
+ *     <li>it's <tt>non-atomic</tt>, a delete-by-query may fail at any time while some documents matching the query have already been
+ *     deleted</li>
+ *     <li>it's <tt>syntactic sugar</tt>, a delete-by-query is equivalent to a scroll search and corresponding bulk-deletes by ID</li>
+ *     <li>it's executed on a <tt>point-in-time</tt> snapshot, a delete-by-query will only delete the documents that are visible at the
+ *     point in time the delete-by-query was started, equivalent to the scroll API</li>
+ *     <li>it's <tt>consistent</tt>, a delete-by-query will yield consistent results across all replicas of a shard</li>
+ *     <li>it's <tt>forward-compatible</tt>, a delete-by-query will only send IDs to the shards as deletes such that no queries are
+ *     stored in the transaction logs that might not be supported in the future.</li>
+ *     <li>it's results won't be visible until the index is refreshed.</li>
+ * </ul>
+ */
+public class DeleteByQueryRequest extends AbstractBulkByScrollRequest<DeleteByQueryRequest> {
+
+    public DeleteByQueryRequest() {
+    }
+
+    public DeleteByQueryRequest(SearchRequest search) {
+        super(search);
+        // Delete-By-Query does not require the source
+        search.source().fetchSource(false);
+    }
+
+    @Override
+    protected DeleteByQueryRequest self() {
+        return this;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException e = super.validate();
+        if (getSearchRequest().indices() == null || getSearchRequest().indices().length == 0) {
+            e = addValidationError("use _all if you really want to delete from all existing indexes", e);
+        }
+        if (getSearchRequest() == null || getSearchRequest().source() == null) {
+            e = addValidationError("source is missing", e);
+        }
+        return e;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder b = new StringBuilder();
+        b.append("delete-by-query ");
+        searchToString(b);
+        return b.toString();
+    }
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java
new file mode 100644
index 00000000000..f4d8a91f4cb
--- /dev/null
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+
+public class DeleteByQueryRequestBuilder extends
+        AbstractBulkByScrollRequestBuilder<DeleteByQueryRequest, BulkIndexByScrollResponse, DeleteByQueryRequestBuilder> {
+
+    public DeleteByQueryRequestBuilder(ElasticsearchClient client,
+                                       Action<DeleteByQueryRequest, BulkIndexByScrollResponse, DeleteByQueryRequestBuilder> action) {
+        this(client, action, new SearchRequestBuilder(client, SearchAction.INSTANCE));
+    }
+
+    private DeleteByQueryRequestBuilder(ElasticsearchClient client,
+                                        Action<DeleteByQueryRequest, BulkIndexByScrollResponse, DeleteByQueryRequestBuilder> action,
+                                        SearchRequestBuilder search) {
+        super(client, action, search, new DeleteByQueryRequest(search.request()));
+    }
+
+    @Override
+    protected DeleteByQueryRequestBuilder self() {
+        return this;
+    }
+
+    @Override
+    public DeleteByQueryRequestBuilder abortOnVersionConflict(boolean abortOnVersionConflict) {
+        request.setAbortOnVersionConflict(abortOnVersionConflict);
+        return this;
+    }
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
index 6e42f56dece..e3a826dbdad 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
@@ -39,12 +39,14 @@ public class ReindexPlugin extends Plugin {
     public void onModule(ActionModule actionModule) {
         actionModule.registerAction(ReindexAction.INSTANCE, TransportReindexAction.class);
         actionModule.registerAction(UpdateByQueryAction.INSTANCE, TransportUpdateByQueryAction.class);
+        actionModule.registerAction(DeleteByQueryAction.INSTANCE, TransportDeleteByQueryAction.class);
         actionModule.registerAction(RethrottleAction.INSTANCE, TransportRethrottleAction.class);
     }
 
     public void onModule(NetworkModule networkModule) {
         networkModule.registerRestHandler(RestReindexAction.class);
         networkModule.registerRestHandler(RestUpdateByQueryAction.class);
+        networkModule.registerRestHandler(RestDeleteByQueryAction.class);
         networkModule.registerRestHandler(RestRethrottleAction.class);
         networkModule.registerTaskStatus(BulkByScrollTask.Status.NAME, BulkByScrollTask.Status::new);
     }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java
new file mode 100644
index 00000000000..4750fe22313
--- /dev/null
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.search.aggregations.AggregatorParsers;
+import org.elasticsearch.search.suggest.Suggesters;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Consumer;
+
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+
+public class RestDeleteByQueryAction extends AbstractBulkByQueryRestHandler<DeleteByQueryRequest, TransportDeleteByQueryAction> {
+
+    @Inject
+    public RestDeleteByQueryAction(Settings settings, RestController controller, Client client,
+                                   IndicesQueriesRegistry indicesQueriesRegistry, AggregatorParsers aggParsers, Suggesters suggesters,
+                                   ClusterService clusterService, TransportDeleteByQueryAction action) {
+        super(settings, client, indicesQueriesRegistry, aggParsers, suggesters, clusterService, action);
+        controller.registerHandler(POST, "/{index}/_delete_by_query", this);
+        controller.registerHandler(POST, "/{index}/{type}/_delete_by_query", this);
+    }
+
+    @Override
+    protected void handleRequest(RestRequest request, RestChannel channel, Client client) throws Exception {
+        if (false == request.hasContent()) {
+            throw new ElasticsearchException("_delete_by_query requires a request body");
+        }
+        handleRequest(request, channel, false, false, true);
+    }
+
+    @Override
+    protected DeleteByQueryRequest buildRequest(RestRequest request) throws IOException {
+         /*
+         * Passing the search request through DeleteByQueryRequest first allows
+         * it to set its own defaults which differ from SearchRequest's
+         * defaults. Then the parseInternalRequest can override them.
+         */
+        DeleteByQueryRequest internal = new DeleteByQueryRequest(new SearchRequest());
+
+        Map<String, Consumer<Object>> consumers = new HashMap<>();
+        consumers.put("conflicts", o -> internal.setConflicts((String) o));
+
+        parseInternalRequest(internal, request, consumers);
+
+        return internal;
+    }
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
index d61980fb8ce..267994672d4 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.action.WriteConsistencyLevel;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.Client;
@@ -27,7 +27,6 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParseFieldMatcherSupplier;
-import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -39,7 +38,6 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
@@ -53,13 +51,14 @@ import java.util.Map;
 
 import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
-import static org.elasticsearch.rest.RestStatus.BAD_REQUEST;
 
 /**
  * Expose IndexBySearchRequest over rest.
  */
 public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexRequest, TransportReindexAction> {
+
     private static final ObjectParser<ReindexRequest, ReindexParseContext> PARSER = new ObjectParser<>("reindex");
+
     static {
         ObjectParser.Parser<SearchRequest, ReindexParseContext> sourceParser = (parser, search, context) -> {
             /*
@@ -114,41 +113,18 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexReq
     @Override
     public void handleRequest(RestRequest request, RestChannel channel, Client client) throws IOException {
         if (false == request.hasContent()) {
-            badRequest(channel, "body required");
-            return;
+            throw new ElasticsearchException("_reindex requires a request body");
         }
+        handleRequest(request, channel, true, true, false);
+    }
 
-        ReindexRequest internalRequest = new ReindexRequest(new SearchRequest(), new IndexRequest());
-
+    @Override
+    protected ReindexRequest buildRequest(RestRequest request) throws IOException {
+        ReindexRequest internal = new ReindexRequest(new SearchRequest(), new IndexRequest());
         try (XContentParser xcontent = XContentFactory.xContent(request.content()).createParser(request.content())) {
-            PARSER.parse(xcontent, internalRequest, new ReindexParseContext(indicesQueriesRegistry, aggParsers,
-                    suggesters, parseFieldMatcher));
-        } catch (ParsingException e) {
-            logger.warn("Bad request", e);
-            badRequest(channel, e.getDetailedMessage());
-            return;
-        }
-        parseCommon(internalRequest, request);
-
-        execute(request, internalRequest, channel, true, true, false);
-    }
-
-    private void badRequest(RestChannel channel, String message) {
-        try {
-            XContentBuilder builder = channel.newErrorBuilder();
-            channel.sendResponse(new BytesRestResponse(BAD_REQUEST, builder.startObject().field("error", message).endObject()));
-        } catch (IOException e) {
-            logger.warn("Failed to send response", e);
-        }
-    }
-
-    public static void parseCommon(AbstractBulkByScrollRequest<?> internalRequest, RestRequest request) {
-        internalRequest.setRefresh(request.paramAsBoolean("refresh", internalRequest.isRefresh()));
-        internalRequest.setTimeout(request.paramAsTime("timeout", internalRequest.getTimeout()));
-        String consistency = request.param("consistency");
-        if (consistency != null) {
-            internalRequest.setConsistency(WriteConsistencyLevel.fromString(consistency));
+            PARSER.parse(xcontent, internal, new ReindexParseContext(indicesQueriesRegistry, aggParsers, suggesters, parseFieldMatcher));
         }
+        return internal;
     }
 
     /**
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java
index a7c29d40a7d..382f5b51726 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java
@@ -39,6 +39,7 @@ public class RestRethrottleAction extends BaseRestHandler {
         super(settings, client);
         this.action = action;
         controller.registerHandler(POST, "/_update_by_query/{taskId}/_rethrottle", this);
+        controller.registerHandler(POST, "/_delete_by_query/{taskId}/_rethrottle", this);
         controller.registerHandler(POST, "/_reindex/{taskId}/_rethrottle", this);
     }
 
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
index 9f76be3f0ff..f7dbbf893a8 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
@@ -22,31 +22,24 @@ package org.elasticsearch.index.reindex;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.search.RestSearchAction;
-import org.elasticsearch.rest.action.support.RestActions;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.suggest.Suggesters;
 
+import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
+import java.util.function.Consumer;
 
-import static org.elasticsearch.index.reindex.AbstractBulkByScrollRequest.SIZE_ALL_MATCHES;
-import static org.elasticsearch.index.reindex.RestReindexAction.parseCommon;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
-public class RestUpdateByQueryAction extends AbstractBaseReindexRestHandler<UpdateByQueryRequest, TransportUpdateByQueryAction> {
+public class RestUpdateByQueryAction extends AbstractBulkByQueryRestHandler<UpdateByQueryRequest, TransportUpdateByQueryAction> {
 
     @Inject
     public RestUpdateByQueryAction(Settings settings, RestController controller, Client client,
@@ -59,60 +52,26 @@ public class RestUpdateByQueryAction extends AbstractBaseReindexRestHandler<Upda
 
     @Override
     protected void handleRequest(RestRequest request, RestChannel channel, Client client) throws Exception {
+        handleRequest(request, channel, false, true, false);
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    protected UpdateByQueryRequest buildRequest(RestRequest request) throws IOException {
         /*
          * Passing the search request through UpdateByQueryRequest first allows
          * it to set its own defaults which differ from SearchRequest's
          * defaults. Then the parse can override them.
          */
-        UpdateByQueryRequest internalRequest = new UpdateByQueryRequest(new SearchRequest());
-        int scrollSize = internalRequest.getSearchRequest().source().size();
-        internalRequest.getSearchRequest().source().size(SIZE_ALL_MATCHES);
-        /*
-         * We can't send parseSearchRequest REST content that it doesn't support
-         * so we will have to remove the content that is valid in addition to
-         * what it supports from the content first. This is a temporary hack and
-         * should get better when SearchRequest has full ObjectParser support
-         * then we can delegate and stuff.
-         */
-        BytesReference bodyContent = null;
-        if (RestActions.hasBodyContent(request)) {
-            bodyContent = RestActions.getRestContent(request);
-            Tuple<XContentType, Map<String, Object>> body = XContentHelper.convertToMap(bodyContent, false);
-            boolean modified = false;
-            String conflicts = (String) body.v2().remove("conflicts");
-            if (conflicts != null) {
-                internalRequest.setConflicts(conflicts);
-                modified = true;
-            }
-            @SuppressWarnings("unchecked")
-            Map<String, Object> script = (Map<String, Object>) body.v2().remove("script");
-            if (script != null) {
-                internalRequest.setScript(Script.parse(script, false, parseFieldMatcher));
-                modified = true;
-            }
-            if (modified) {
-                XContentBuilder builder = XContentFactory.contentBuilder(body.v1());
-                builder.map(body.v2());
-                bodyContent = builder.bytes();
-            }
-        }
-        RestSearchAction.parseSearchRequest(internalRequest.getSearchRequest(), indicesQueriesRegistry, request,
-                parseFieldMatcher, aggParsers, suggesters, bodyContent);
+        UpdateByQueryRequest internal = new UpdateByQueryRequest(new SearchRequest());
 
-        String conflicts = request.param("conflicts");
-        if (conflicts != null) {
-            internalRequest.setConflicts(conflicts);
-        }
-        parseCommon(internalRequest, request);
+        Map<String, Consumer<Object>> consumers = new HashMap<>();
+        consumers.put("conflicts", o -> internal.setConflicts((String) o));
+        consumers.put("script", o -> internal.setScript(Script.parse((Map<String, Object>)o, false, parseFieldMatcher)));
 
-        internalRequest.setSize(internalRequest.getSearchRequest().source().size());
-        internalRequest.setPipeline(request.param("pipeline"));
-        internalRequest.getSearchRequest().source().size(request.paramAsInt("scroll_size", scrollSize));
-        // Let the requester set search timeout. It is probably only going to be useful for testing but who knows.
-        if (request.hasParam("search_timeout")) {
-            internalRequest.getSearchRequest().source().timeout(request.paramAsTime("search_timeout", null));
-        }
+        parseInternalRequest(internal, request, consumers);
 
-        execute(request, internalRequest, channel, false, true, false);
+        internal.setPipeline(request.param("pipeline"));
+        return internal;
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java
new file mode 100644
index 00000000000..471bd066f94
--- /dev/null
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.ParentTaskAssigningClient;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
+import org.elasticsearch.index.mapper.internal.RoutingFieldMapper;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+public class TransportDeleteByQueryAction extends HandledTransportAction<DeleteByQueryRequest, BulkIndexByScrollResponse> {
+    private final Client client;
+    private final ScriptService scriptService;
+    private final ClusterService clusterService;
+
+    @Inject
+    public TransportDeleteByQueryAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters,
+                                        IndexNameExpressionResolver resolver, Client client, TransportService transportService,
+                                        ScriptService scriptService, ClusterService clusterService) {
+        super(settings, DeleteByQueryAction.NAME, threadPool, transportService, actionFilters, resolver, DeleteByQueryRequest::new);
+        this.client = client;
+        this.scriptService = scriptService;
+        this.clusterService = clusterService;
+    }
+
+    @Override
+    protected void doExecute(Task task, DeleteByQueryRequest request, ActionListener<BulkIndexByScrollResponse> listener) {
+        ClusterState state = clusterService.state();
+        ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task);
+        new AsyncDeleteBySearchAction((BulkByScrollTask) task, logger, client, threadPool, request, listener, scriptService, state).start();
+    }
+
+    @Override
+    protected void doExecute(DeleteByQueryRequest request, ActionListener<BulkIndexByScrollResponse> listener) {
+        throw new UnsupportedOperationException("task required");
+    }
+
+    /**
+     * Implementation of delete-by-query using scrolling and bulk.
+     */
+    static class AsyncDeleteBySearchAction extends AbstractAsyncBulkIndexByScrollAction<DeleteByQueryRequest> {
+
+        public AsyncDeleteBySearchAction(BulkByScrollTask task, ESLogger logger, ParentTaskAssigningClient client, ThreadPool threadPool,
+                                         DeleteByQueryRequest request, ActionListener<BulkIndexByScrollResponse> listener,
+                                         ScriptService scriptService, ClusterState clusterState) {
+            super(task, logger, client, threadPool, request, request.getSearchRequest(), listener, scriptService, clusterState);
+        }
+
+        @Override
+        protected boolean accept(SearchHit doc) {
+            // Delete-by-query does not require the source to delete a document
+            // and the default implementation checks for it
+            return true;
+        }
+
+        @Override
+        protected RequestWrapper<DeleteRequest> buildRequest(SearchHit doc) {
+            DeleteRequest delete = new DeleteRequest();
+            delete.index(doc.index());
+            delete.type(doc.type());
+            delete.id(doc.id());
+            delete.version(doc.version());
+            return wrap(delete);
+        }
+
+        /**
+         * Overrides the parent {@link AbstractAsyncBulkIndexByScrollAction#copyMetadata(RequestWrapper, SearchHit)}
+         * method that is much more Update/Reindex oriented and so also copies things like timestamp/ttl which we
+         * don't care for a deletion.
+         */
+        @Override
+        protected RequestWrapper<?> copyMetadata(RequestWrapper<?> request, SearchHit doc) {
+            copyParent(request, fieldValue(doc, ParentFieldMapper.NAME));
+            copyRouting(request, fieldValue(doc, RoutingFieldMapper.NAME));
+            return request;
+        }
+    }
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
index 0f07cc560c8..a49ba0a3b32 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
@@ -35,16 +35,18 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.mapper.internal.TTLFieldMapper;
 import org.elasticsearch.index.mapper.internal.VersionFieldMapper;
+import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.util.Map;
 import java.util.Objects;
+import java.util.function.BiFunction;
 
 import static java.util.Objects.requireNonNull;
 import static org.elasticsearch.index.VersionType.INTERNAL;
@@ -72,7 +74,7 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
         ClusterState state = clusterService.state();
         validateAgainstAliases(request.getSearchRequest(), request.getDestination(), indexNameExpressionResolver, autoCreateIndex, state);
         ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task);
-        new AsyncIndexBySearchAction((BulkByScrollTask) task, logger, scriptService, client, state, threadPool, request, listener).start();
+        new AsyncIndexBySearchAction((BulkByScrollTask) task, logger, client, threadPool, request, listener, scriptService, state).start();
     }
 
     @Override
@@ -87,7 +89,8 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
      * isn't available then. Package private for testing.
      */
     static String validateAgainstAliases(SearchRequest source, IndexRequest destination,
-            IndexNameExpressionResolver indexNameExpressionResolver, AutoCreateIndex autoCreateIndex, ClusterState clusterState) {
+                                         IndexNameExpressionResolver indexNameExpressionResolver, AutoCreateIndex autoCreateIndex,
+                                         ClusterState clusterState) {
         String target = destination.index();
         if (false == autoCreateIndex.shouldAutoCreate(target, clusterState)) {
             /*
@@ -97,7 +100,7 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
              */
             target = indexNameExpressionResolver.concreteIndexNames(clusterState, destination)[0];
         }
-        for (String sourceIndex: indexNameExpressionResolver.concreteIndexNames(clusterState, source)) {
+        for (String sourceIndex : indexNameExpressionResolver.concreteIndexNames(clusterState, source)) {
             if (sourceIndex.equals(target)) {
                 ActionRequestValidationException e = new ActionRequestValidationException();
                 e.addValidationError("reindex cannot write into an index its reading from [" + target + ']');
@@ -114,14 +117,24 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
      * possible.
      */
     static class AsyncIndexBySearchAction extends AbstractAsyncBulkIndexByScrollAction<ReindexRequest> {
-        public AsyncIndexBySearchAction(BulkByScrollTask task, ESLogger logger, ScriptService scriptService,
-                ParentTaskAssigningClient client, ClusterState state, ThreadPool threadPool, ReindexRequest request,
-                ActionListener<BulkIndexByScrollResponse> listener) {
-            super(task, logger, scriptService, state, client, threadPool, request, request.getSearchRequest(), listener);
+
+        public AsyncIndexBySearchAction(BulkByScrollTask task, ESLogger logger, ParentTaskAssigningClient client, ThreadPool threadPool,
+                                        ReindexRequest request, ActionListener<BulkIndexByScrollResponse> listener,
+                                        ScriptService scriptService, ClusterState clusterState) {
+            super(task, logger, client, threadPool, request, request.getSearchRequest(), listener, scriptService, clusterState);
         }
 
         @Override
-        protected IndexRequest buildIndexRequest(SearchHit doc) {
+        protected BiFunction<RequestWrapper<?>, SearchHit, RequestWrapper<?>> buildScriptApplier() {
+            Script script = mainRequest.getScript();
+            if (script != null) {
+                return new ReindexScriptApplier(task, scriptService, script, clusterState, script.getParams());
+            }
+            return super.buildScriptApplier();
+        }
+
+        @Override
+        protected RequestWrapper<IndexRequest> buildRequest(SearchHit doc) {
             IndexRequest index = new IndexRequest();
 
             // Copy the index from the request so we always write where it asked to write
@@ -161,109 +174,120 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
             index.setPipeline(mainRequest.getDestination().getPipeline());
             // OpType is synthesized from version so it is handled when we copy version above.
 
-            return index;
+            return wrap(index);
         }
 
         /**
          * Override the simple copy behavior to allow more fine grained control.
          */
         @Override
-        protected void copyRouting(IndexRequest index, SearchHit doc) {
+        protected void copyRouting(RequestWrapper<?> request, String routing) {
             String routingSpec = mainRequest.getDestination().routing();
             if (routingSpec == null) {
-                super.copyRouting(index, doc);
+                super.copyRouting(request, routing);
                 return;
             }
             if (routingSpec.startsWith("=")) {
-                index.routing(mainRequest.getDestination().routing().substring(1));
+                super.copyRouting(request, mainRequest.getDestination().routing().substring(1));
                 return;
             }
             switch (routingSpec) {
             case "keep":
-                super.copyRouting(index, doc);
+                super.copyRouting(request, routing);
                 break;
             case "discard":
-                index.routing(null);
+                super.copyRouting(request, null);
                 break;
             default:
                 throw new IllegalArgumentException("Unsupported routing command");
             }
         }
 
-        /*
-         * Methods below here handle script updating the index request. They try
-         * to be pretty liberal with regards to types because script are often
-         * dynamically typed.
-         */
-        @Override
-        protected void scriptChangedIndex(IndexRequest index, Object to) {
-            requireNonNull(to, "Can't reindex without a destination index!");
-            index.index(to.toString());
-        }
+        class ReindexScriptApplier extends ScriptApplier {
 
-        @Override
-        protected void scriptChangedType(IndexRequest index, Object to) {
-            requireNonNull(to, "Can't reindex without a destination type!");
-            index.type(to.toString());
-        }
-
-        @Override
-        protected void scriptChangedId(IndexRequest index, Object to) {
-            index.id(Objects.toString(to, null));
-        }
-
-        @Override
-        protected void scriptChangedVersion(IndexRequest index, Object to) {
-            if (to == null) {
-                index.version(Versions.MATCH_ANY).versionType(INTERNAL);
-                return;
+            ReindexScriptApplier(BulkByScrollTask task, ScriptService scriptService, Script script, ClusterState state,
+                                 Map<String, Object> params) {
+                super(task, scriptService, script, state, params);
             }
-            index.version(asLong(to, VersionFieldMapper.NAME));
-        }
 
-        @Override
-        protected void scriptChangedParent(IndexRequest index, Object to) {
-            // Have to override routing with parent just in case its changed
-            String routing = Objects.toString(to, null);
-            index.parent(routing).routing(routing);
-        }
-
-        @Override
-        protected void scriptChangedRouting(IndexRequest index, Object to) {
-            index.routing(Objects.toString(to, null));
-        }
-
-        @Override
-        protected void scriptChangedTimestamp(IndexRequest index, Object to) {
-            index.timestamp(Objects.toString(to, null));
-        }
-
-        @Override
-        protected void scriptChangedTTL(IndexRequest index, Object to) {
-            if (to == null) {
-                index.ttl((TimeValue) null);
-                return;
-            }
-            index.ttl(asLong(to, TTLFieldMapper.NAME));
-        }
-
-        private long asLong(Object from, String name) {
             /*
-             * Stuffing a number into the map will have converted it to
-             * some Number.
+             * Methods below here handle script updating the index request. They try
+             * to be pretty liberal with regards to types because script are often
+             * dynamically typed.
              */
-            Number fromNumber;
-            try {
-                fromNumber = (Number) from;
-            } catch (ClassCastException e) {
-                throw new IllegalArgumentException(name + " may only be set to an int or a long but was [" + from + "]", e);
+
+            @Override
+            protected void scriptChangedIndex(RequestWrapper<?> request, Object to) {
+                requireNonNull(to, "Can't reindex without a destination index!");
+                request.setIndex(to.toString());
             }
-            long l = fromNumber.longValue();
-            // Check that we didn't round when we fetched the value.
-            if (fromNumber.doubleValue() != l) {
-                throw new IllegalArgumentException(name + " may only be set to an int or a long but was [" + from + "]");
+
+            @Override
+            protected void scriptChangedType(RequestWrapper<?> request, Object to) {
+                requireNonNull(to, "Can't reindex without a destination type!");
+                request.setType(to.toString());
+            }
+
+            @Override
+            protected void scriptChangedId(RequestWrapper<?> request, Object to) {
+                request.setId(Objects.toString(to, null));
+            }
+
+            @Override
+            protected void scriptChangedVersion(RequestWrapper<?> request, Object to) {
+                if (to == null) {
+                    request.setVersion(Versions.MATCH_ANY);
+                    request.setVersionType(INTERNAL);
+                } else {
+                    request.setVersion(asLong(to, VersionFieldMapper.NAME));
+                }
+            }
+
+            @Override
+            protected void scriptChangedParent(RequestWrapper<?> request, Object to) {
+                // Have to override routing with parent just in case its changed
+                String routing = Objects.toString(to, null);
+                request.setParent(routing);
+                request.setRouting(routing);
+            }
+
+            @Override
+            protected void scriptChangedRouting(RequestWrapper<?> request, Object to) {
+                request.setRouting(Objects.toString(to, null));
+            }
+
+            @Override
+            protected void scriptChangedTimestamp(RequestWrapper<?> request, Object to) {
+                request.setTimestamp(Objects.toString(to, null));
+            }
+
+            @Override
+            protected void scriptChangedTTL(RequestWrapper<?> request, Object to) {
+                if (to == null) {
+                    request.setTtl(null);
+                } else {
+                    request.setTtl(asLong(to, TTLFieldMapper.NAME));
+                }
+            }
+
+            private long asLong(Object from, String name) {
+                /*
+                 * Stuffing a number into the map will have converted it to
+                 * some Number.
+                 * */
+                Number fromNumber;
+                try {
+                    fromNumber = (Number) from;
+                } catch (ClassCastException e) {
+                    throw new IllegalArgumentException(name + " may only be set to an int or a long but was [" + from + "]", e);
+                }
+                long l = fromNumber.longValue();
+                // Check that we didn't round when we fetched the value.
+                if (fromNumber.doubleValue() != l) {
+                    throw new IllegalArgumentException(name + " may only be set to an int or a long but was [" + from + "]");
+                }
+                return l;
             }
-            return l;
         }
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
index 4d702be4e9d..7459972ce64 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
@@ -39,12 +39,16 @@ import org.elasticsearch.index.mapper.internal.RoutingFieldMapper;
 import org.elasticsearch.index.mapper.internal.TTLFieldMapper;
 import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
 import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
+import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.util.Map;
+import java.util.function.BiFunction;
+
 public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateByQueryRequest, BulkIndexByScrollResponse> {
     private final Client client;
     private final ScriptService scriptService;
@@ -65,8 +69,7 @@ public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateB
     protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener<BulkIndexByScrollResponse> listener) {
         ClusterState state = clusterService.state();
         ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task);
-        new AsyncIndexBySearchAction((BulkByScrollTask) task, logger, scriptService, client, threadPool, state, request, listener)
-                .start();
+        new AsyncIndexBySearchAction((BulkByScrollTask) task, logger, client, threadPool, request, listener, scriptService, state).start();
     }
 
     @Override
@@ -78,14 +81,24 @@ public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateB
      * Simple implementation of update-by-query using scrolling and bulk.
      */
     static class AsyncIndexBySearchAction extends AbstractAsyncBulkIndexByScrollAction<UpdateByQueryRequest> {
-        public AsyncIndexBySearchAction(BulkByScrollTask task, ESLogger logger, ScriptService scriptService,
-                ParentTaskAssigningClient client, ThreadPool threadPool, ClusterState clusterState, UpdateByQueryRequest request,
-                ActionListener<BulkIndexByScrollResponse> listener) {
-            super(task, logger, scriptService, clusterState, client, threadPool, request, request.getSearchRequest(), listener);
+
+        public AsyncIndexBySearchAction(BulkByScrollTask task, ESLogger logger, ParentTaskAssigningClient client, ThreadPool threadPool,
+                                        UpdateByQueryRequest request, ActionListener<BulkIndexByScrollResponse> listener,
+                                        ScriptService scriptService, ClusterState clusterState) {
+            super(task, logger, client, threadPool, request, request.getSearchRequest(), listener, scriptService, clusterState);
         }
 
         @Override
-        protected IndexRequest buildIndexRequest(SearchHit doc) {
+        protected BiFunction<RequestWrapper<?>, SearchHit, RequestWrapper<?>> buildScriptApplier() {
+            Script script = mainRequest.getScript();
+            if (script != null) {
+                return new UpdateByQueryScriptApplier(task, scriptService, script, clusterState, script.getParams());
+            }
+            return super.buildScriptApplier();
+        }
+
+        @Override
+        protected RequestWrapper<IndexRequest> buildRequest(SearchHit doc) {
             IndexRequest index = new IndexRequest();
             index.index(doc.index());
             index.type(doc.type());
@@ -94,47 +107,55 @@ public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateB
             index.versionType(VersionType.INTERNAL);
             index.version(doc.version());
             index.setPipeline(mainRequest.getPipeline());
-            return index;
+            return wrap(index);
         }
 
-        @Override
-        protected void scriptChangedIndex(IndexRequest index, Object to) {
-            throw new IllegalArgumentException("Modifying [" + IndexFieldMapper.NAME + "] not allowed");
-        }
+        class UpdateByQueryScriptApplier extends ScriptApplier {
 
-        @Override
-        protected void scriptChangedType(IndexRequest index, Object to) {
-            throw new IllegalArgumentException("Modifying [" + TypeFieldMapper.NAME + "] not allowed");
-        }
+            UpdateByQueryScriptApplier(BulkByScrollTask task, ScriptService scriptService, Script script, ClusterState state,
+                                 Map<String, Object> params) {
+                super(task, scriptService, script, state, params);
+            }
 
-        @Override
-        protected void scriptChangedId(IndexRequest index, Object to) {
-            throw new IllegalArgumentException("Modifying [" + IdFieldMapper.NAME + "] not allowed");
-        }
+            @Override
+            protected void scriptChangedIndex(RequestWrapper<?> request, Object to) {
+                throw new IllegalArgumentException("Modifying [" + IndexFieldMapper.NAME + "] not allowed");
+            }
 
-        @Override
-        protected void scriptChangedVersion(IndexRequest index, Object to) {
-            throw new IllegalArgumentException("Modifying [_version] not allowed");
-        }
+            @Override
+            protected void scriptChangedType(RequestWrapper<?> request, Object to) {
+                throw new IllegalArgumentException("Modifying [" + TypeFieldMapper.NAME + "] not allowed");
+            }
 
-        @Override
-        protected void scriptChangedRouting(IndexRequest index, Object to) {
-            throw new IllegalArgumentException("Modifying [" + RoutingFieldMapper.NAME + "] not allowed");
-        }
+            @Override
+            protected void scriptChangedId(RequestWrapper<?> request, Object to) {
+                throw new IllegalArgumentException("Modifying [" + IdFieldMapper.NAME + "] not allowed");
+            }
 
-        @Override
-        protected void scriptChangedParent(IndexRequest index, Object to) {
-            throw new IllegalArgumentException("Modifying [" + ParentFieldMapper.NAME + "] not allowed");
-        }
+            @Override
+            protected void scriptChangedVersion(RequestWrapper<?> request, Object to) {
+                throw new IllegalArgumentException("Modifying [_version] not allowed");
+            }
 
-        @Override
-        protected void scriptChangedTimestamp(IndexRequest index, Object to) {
-            throw new IllegalArgumentException("Modifying [" + TimestampFieldMapper.NAME + "] not allowed");
-        }
+            @Override
+            protected void scriptChangedRouting(RequestWrapper<?> request, Object to) {
+                throw new IllegalArgumentException("Modifying [" + RoutingFieldMapper.NAME + "] not allowed");
+            }
 
-        @Override
-        protected void scriptChangedTTL(IndexRequest index, Object to) {
-            throw new IllegalArgumentException("Modifying [" + TTLFieldMapper.NAME + "] not allowed");
+            @Override
+            protected void scriptChangedParent(RequestWrapper<?> request, Object to) {
+                throw new IllegalArgumentException("Modifying [" + ParentFieldMapper.NAME + "] not allowed");
+            }
+
+            @Override
+            protected void scriptChangedTimestamp(RequestWrapper<?> request, Object to) {
+                throw new IllegalArgumentException("Modifying [" + TimestampFieldMapper.NAME + "] not allowed");
+            }
+
+            @Override
+            protected void scriptChangedTTL(RequestWrapper<?> request, Object to) {
+                throw new IllegalArgumentException("Modifying [" + TTLFieldMapper.NAME + "] not allowed");
+            }
         }
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionScriptTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionScriptTestCase.java
index b8f389d171a..27a8a42d5e6 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionScriptTestCase.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionScriptTestCase.java
@@ -22,10 +22,15 @@ package org.elasticsearch.index.reindex;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.InternalSearchHit;
+import org.junit.Before;
+import org.mockito.Matchers;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -33,18 +38,35 @@ import java.util.function.Consumer;
 
 import static java.util.Collections.singletonMap;
 import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public abstract class AbstractAsyncBulkIndexByScrollActionScriptTestCase<
                 Request extends AbstractBulkIndexByScrollRequest<Request>,
                 Response extends BulkIndexByScrollResponse>
         extends AbstractAsyncBulkIndexByScrollActionTestCase<Request, Response> {
+
+    private static final Script EMPTY_SCRIPT = new Script("");
+
+    protected ScriptService scriptService;
+
+    @Before
+    public void setupScriptService() {
+        scriptService = mock(ScriptService.class);
+    }
+
     protected IndexRequest applyScript(Consumer<Map<String, Object>> scriptBody) {
         IndexRequest index = new IndexRequest("index", "type", "1").source(singletonMap("foo", "bar"));
         Map<String, SearchHitField> fields = new HashMap<>();
         InternalSearchHit doc = new InternalSearchHit(0, "id", new Text("type"), fields);
         doc.shardTarget(new SearchShardTarget("nodeid", new Index("index", "uuid"), 1));
-        ExecutableScript script = new SimpleExecutableScript(scriptBody);
-        action().applyScript(index, doc, script, new HashMap<>());
+        ExecutableScript executableScript = new SimpleExecutableScript(scriptBody);
+
+        when(scriptService.executable(any(CompiledScript.class), Matchers.<Map<String, Object>>any()))
+                .thenReturn(executableScript);
+        AbstractAsyncBulkIndexByScrollAction<Request> action = action(scriptService, request().setScript(EMPTY_SCRIPT));
+        action.buildScriptApplier().apply(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc);
         return index;
     }
 
@@ -53,7 +75,7 @@ public abstract class AbstractAsyncBulkIndexByScrollActionScriptTestCase<
             applyScript((Map<String, Object> ctx) -> ctx.put("junk", "junk"));
             fail("Expected error");
         } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("Invalid fields added to ctx [junk]"));
+            assertThat(e.getMessage(), equalTo("Invalid fields added to context [junk]"));
         }
     }
 
@@ -65,4 +87,6 @@ public abstract class AbstractAsyncBulkIndexByScrollActionScriptTestCase<
         });
         assertEquals("cat", index.sourceAsMap().get("bar"));
     }
+
+    protected abstract AbstractAsyncBulkIndexByScrollAction<Request> action(ScriptService scriptService, Request request);
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionTestCase.java
index ab5f8f0d748..b9489e9f5d9 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionTestCase.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionTestCase.java
@@ -46,8 +46,6 @@ public abstract class AbstractAsyncBulkIndexByScrollActionTestCase<
         threadPool.shutdown();
     }
 
-    protected abstract AbstractAsyncBulkIndexByScrollAction<Request> action();
-
     protected abstract Request request();
 
     protected PlainActionFuture<Response> listener() {
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexbyScrollActionMetadataTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexbyScrollActionMetadataTestCase.java
index 37386abf12e..5a9976fc005 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexbyScrollActionMetadataTestCase.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexbyScrollActionMetadataTestCase.java
@@ -49,13 +49,15 @@ public abstract class AbstractAsyncBulkIndexbyScrollActionMetadataTestCase<
 
     public void testTimestampIsCopied() {
         IndexRequest index = new IndexRequest();
-        action().copyMetadata(index, doc(TimestampFieldMapper.NAME, 10L));
+        action().copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc(TimestampFieldMapper.NAME, 10L));
         assertEquals("10", index.timestamp());
     }
 
     public void testTTL() throws Exception {
         IndexRequest index = new IndexRequest();
-        action().copyMetadata(index, doc(TTLFieldMapper.NAME, 10L));
+        action().copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc(TTLFieldMapper.NAME, 10L));
         assertEquals(timeValueMillis(10), index.ttl());
     }
+
+    protected abstract AbstractAsyncBulkIndexByScrollAction<Request> action();
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
index c9bd22d3552..d78fcfe69ca 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
@@ -665,7 +665,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
     }
 
     private class DummyAbstractAsyncBulkByScrollAction
-            extends AbstractAsyncBulkByScrollAction<DummyAbstractBulkByScrollRequest, BulkIndexByScrollResponse> {
+            extends AbstractAsyncBulkByScrollAction<DummyAbstractBulkByScrollRequest> {
         public DummyAbstractAsyncBulkByScrollAction() {
             super(testTask, logger, new ParentTaskAssigningClient(client, localNode, testTask), threadPool, testRequest, firstSearchRequest,
                     listener);
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseMatcher.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseMatcher.java
index 5bce3260929..4ef16c59141 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseMatcher.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseMatcher.java
@@ -30,6 +30,8 @@ public class BulkIndexByScrollResponseMatcher extends TypeSafeMatcher<BulkIndexB
 
     private Matcher<Long> createdMatcher = equalTo(0L);
     private Matcher<Long> updatedMatcher = equalTo(0L);
+    private Matcher<Long> deletedMatcher = equalTo(0L);
+
     /**
      * Matches for number of batches. Optional.
      */
@@ -56,6 +58,15 @@ public class BulkIndexByScrollResponseMatcher extends TypeSafeMatcher<BulkIndexB
         return updated(equalTo(updated));
     }
 
+    public BulkIndexByScrollResponseMatcher deleted(Matcher<Long> deletedMatcher) {
+        this.deletedMatcher = deletedMatcher;
+        return this;
+    }
+
+    public BulkIndexByScrollResponseMatcher deleted(long deleted) {
+        return deleted(equalTo(deleted));
+    }
+
     /**
      * Set the matches for the number of batches. Defaults to matching any
      * integer because we usually don't care about how many batches the job
@@ -110,6 +121,7 @@ public class BulkIndexByScrollResponseMatcher extends TypeSafeMatcher<BulkIndexB
     protected boolean matchesSafely(BulkIndexByScrollResponse item) {
         return updatedMatcher.matches(item.getUpdated()) &&
                 createdMatcher.matches(item.getCreated()) &&
+                deletedMatcher.matches(item.getDeleted()) &&
                 (batchesMatcher == null || batchesMatcher.matches(item.getBatches())) &&
                 versionConflictsMatcher.matches(item.getVersionConflicts()) &&
                 failuresMatcher.matches(item.getIndexingFailures().size()) &&
@@ -120,6 +132,7 @@ public class BulkIndexByScrollResponseMatcher extends TypeSafeMatcher<BulkIndexB
     public void describeTo(Description description) {
         description.appendText("updated matches ").appendDescriptionOf(updatedMatcher);
         description.appendText(" and created matches ").appendDescriptionOf(createdMatcher);
+        description.appendText(" and deleted matches ").appendDescriptionOf(deletedMatcher);
         if (batchesMatcher != null) {
             description.appendText(" and batches matches ").appendDescriptionOf(batchesMatcher);
         }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java
new file mode 100644
index 00000000000..a52b46233bc
--- /dev/null
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryBasicTests.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.sort.SortOrder;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
+import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+
+public class DeleteByQueryBasicTests extends ReindexTestCase {
+
+    public void testBasics() throws Exception {
+        indexRandom(true,
+                client().prepareIndex("test", "test", "1").setSource("foo", "a"),
+                client().prepareIndex("test", "test", "2").setSource("foo", "a"),
+                client().prepareIndex("test", "test", "3").setSource("foo", "b"),
+                client().prepareIndex("test", "test", "4").setSource("foo", "c"),
+                client().prepareIndex("test", "test", "5").setSource("foo", "d"),
+                client().prepareIndex("test", "test", "6").setSource("foo", "e"),
+                client().prepareIndex("test", "test", "7").setSource("foo", "f")
+        );
+
+        assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 7);
+
+        // Deletes two docs that matches "foo:a"
+        assertThat(deleteByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).get(), matcher().deleted(2));
+        assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 5);
+
+        // Deletes the two first docs with limit by size
+        DeleteByQueryRequestBuilder request = deleteByQuery().source("test").size(2).refresh(true);
+        request.source().addSort("foo.keyword", SortOrder.ASC);
+        assertThat(request.get(), matcher().deleted(2));
+        assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 3);
+
+        // Deletes but match no docs
+        assertThat(deleteByQuery().source("test").filter(termQuery("foo", "no_match")).refresh(true).get(), matcher().deleted(0));
+        assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 3);
+
+        // Deletes all remaining docs
+        assertThat(deleteByQuery().source("test").refresh(true).get(), matcher().deleted(3));
+        assertHitCount(client().prepareSearch("test").setTypes("test").setSize(0).get(), 0);
+    }
+
+    public void testDeleteByQueryWithOneIndex() throws Exception {
+        final long docs = randomIntBetween(1, 50);
+
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+        for (int i = 0; i < docs; i++) {
+            builders.add(client().prepareIndex("test", "doc", String.valueOf(i)).setSource("fields1", 1));
+        }
+        indexRandom(true, true, true, builders);
+
+        assertThat(deleteByQuery().source("t*").refresh(true).get(), matcher().deleted(docs));
+        assertHitCount(client().prepareSearch("test").setSize(0).get(), 0);
+    }
+
+    public void testDeleteByQueryWithMultipleIndices() throws Exception {
+        final int indices = randomIntBetween(2, 5);
+        final int docs = randomIntBetween(2, 10) * 2;
+        long[] candidates = new long[indices];
+
+        // total number of expected deletions
+        long deletions = 0;
+
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+        for (int i = 0; i < indices; i++) {
+            // number of documents to be deleted with the upcoming delete-by-query
+            // (this number differs for each index)
+            candidates[i] = randomIntBetween(1, docs);
+            deletions = deletions + candidates[i];
+
+            for (int j = 0; j < docs; j++) {
+                boolean candidate = (j < candidates[i]);
+                builders.add(client().prepareIndex("test-" + i, "doc", String.valueOf(j)).setSource("candidate", candidate));
+            }
+        }
+        indexRandom(true, true, true, builders);
+
+        // Deletes all the documents with candidate=true
+        assertThat(deleteByQuery().source("test-*").filter(termQuery("candidate", true)).refresh(true).get(),
+                matcher().deleted(deletions));
+
+        for (int i = 0; i < indices; i++) {
+            long remaining = docs - candidates[i];
+            assertHitCount(client().prepareSearch("test-" + i).setSize(0).get(), remaining);
+        }
+
+        assertHitCount(client().prepareSearch().setSize(0).get(), (indices * docs) - deletions);
+    }
+
+    public void testDeleteByQueryWithMissingIndex() throws Exception {
+        indexRandom(true, client().prepareIndex("test", "test", "1").setSource("foo", "a"));
+        assertHitCount(client().prepareSearch().setSize(0).get(), 1);
+
+        try {
+            deleteByQuery().source("missing").get();
+            fail("should have thrown an exception because of a missing index");
+        } catch (IndexNotFoundException e) {
+            // Ok
+        }
+    }
+
+    public void testDeleteByQueryWithRouting() throws Exception {
+        assertAcked(prepareCreate("test").setSettings("number_of_shards", 2));
+        ensureGreen("test");
+
+        final int docs = randomIntBetween(2, 10);
+        logger.info("--> indexing [{}] documents with routing", docs);
+
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+        for (int i = 0; i < docs; i++) {
+            builders.add(client().prepareIndex("test", "test", String.valueOf(i)).setRouting(String.valueOf(i)).setSource("field1", 1));
+        }
+        indexRandom(true, true, true, builders);
+
+        logger.info("--> counting documents with no routing, should be equal to [{}]", docs);
+        assertHitCount(client().prepareSearch().setSize(0).get(), docs);
+
+        String routing = String.valueOf(randomIntBetween(2, docs));
+
+        logger.info("--> counting documents with routing [{}]", routing);
+        long expected = client().prepareSearch().setSize(0).setRouting(routing).get().getHits().totalHits();
+
+        logger.info("--> delete all documents with routing [{}] with a delete-by-query", routing);
+        DeleteByQueryRequestBuilder delete = deleteByQuery().source("test");
+        delete.source().setRouting(routing);
+        assertThat(delete.refresh(true).get(), matcher().deleted(expected));
+
+        assertHitCount(client().prepareSearch().setSize(0).get(), docs - expected);
+    }
+
+    public void testDeleteByMatchQuery() throws Exception {
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias")));
+
+        final int docs = scaledRandomIntBetween(10, 100);
+
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+        for (int i = 0; i < docs; i++) {
+            builders.add(client().prepareIndex("test", "test", Integer.toString(i))
+                    .setRouting(randomAsciiOfLengthBetween(1, 5))
+                    .setSource("foo", "bar"));
+        }
+        indexRandom(true, true, true, builders);
+
+        int n = between(0, docs - 1);
+        assertHitCount(client().prepareSearch("test").setSize(0).setQuery(matchQuery("_id", Integer.toString(n))).get(), 1);
+        assertHitCount(client().prepareSearch("test").setSize(0).setQuery(QueryBuilders.matchAllQuery()).get(), docs);
+
+        DeleteByQueryRequestBuilder delete = deleteByQuery().source("alias").filter(matchQuery("_id", Integer.toString(n)));
+        assertThat(delete.refresh(true).get(), matcher().deleted(1L));
+
+        assertHitCount(client().prepareSearch("test").setSize(0).setQuery(QueryBuilders.matchAllQuery()).get(), docs - 1);
+    }
+
+    public void testDeleteByQueryWithDateMath() throws Exception {
+        indexRandom(true, client().prepareIndex("test", "type", "1").setSource("d", "2013-01-01"));
+
+        DeleteByQueryRequestBuilder delete = deleteByQuery().source("test").filter(rangeQuery("d").to("now-1h"));
+        assertThat(delete.refresh(true).get(), matcher().deleted(1L));
+
+        assertHitCount(client().prepareSearch("test").setSize(0).get(), 0);
+    }
+
+    public void testDeleteByQueryOnReadOnlyIndex() throws Exception {
+        createIndex("test");
+
+        final int docs = randomIntBetween(1, 50);
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+        for (int i = 0; i < docs; i++) {
+            builders.add(client().prepareIndex("test", "test", Integer.toString(i)).setSource("field", 1));
+        }
+        indexRandom(true, true, true, builders);
+
+        try {
+            enableIndexBlock("test", IndexMetaData.SETTING_READ_ONLY);
+            assertThat(deleteByQuery().source("test").refresh(true).get(), matcher().deleted(0).failures(docs));
+        } finally {
+            disableIndexBlock("test", IndexMetaData.SETTING_READ_ONLY);
+        }
+
+        assertHitCount(client().prepareSearch("test").setSize(0).get(), docs);
+    }
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryCancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryCancelTests.java
new file mode 100644
index 00000000000..6007b646429
--- /dev/null
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryCancelTests.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.ListenableActionFuture;
+import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo;
+import org.elasticsearch.common.util.concurrent.CountDown;
+import org.elasticsearch.index.IndexModule;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.shard.IndexingOperationListener;
+import org.elasticsearch.plugins.Plugin;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+/**
+ * Tests that you can actually cancel a delete-by-query request and all the plumbing works. Doesn't test all of the different cancellation
+ * places - that is the responsibility of {@link AsyncBulkByScrollActionTests} which have more precise control to simulate failures but do
+ * not exercise important portion of the stack like transport and task management.
+ */
+public class DeleteByQueryCancelTests extends ReindexTestCase {
+
+    private static final String INDEX = "test-delete-by-query";
+    private static final String TYPE = "test";
+
+    private static final int MAX_DELETIONS = 10;
+    private static final CyclicBarrier barrier = new CyclicBarrier(2);
+
+    @Override
+    protected int numberOfShards() {
+        // Only 1 shard and no replica so that test execution
+        // can be easily controlled within a {@link IndexingOperationListener#preDelete}
+        return 1;
+    }
+
+    @Override
+    protected int numberOfReplicas() {
+        // Only 1 shard and no replica so that test execution
+        // can be easily controlled within a {@link IndexingOperationListener#preDelete}
+        return 0;
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        Collection<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
+        plugins.add(DeleteByQueryCancellationPlugin.class);
+        return plugins;
+    }
+
+    public void testCancel() throws Exception {
+        createIndex(INDEX);
+
+        int totalNumShards = getNumShards(INDEX).totalNumShards;
+
+        // Number of documents to be deleted in this test
+        final int nbDocsToDelete = totalNumShards * MAX_DELETIONS;
+
+        // Total number of documents that will be created in this test
+        final int nbDocs = nbDocsToDelete * randomIntBetween(1, 5);
+        for (int i = 0; i < nbDocs; i++) {
+            indexRandom(false, client().prepareIndex(INDEX, TYPE, String.valueOf(i)).setSource("n", i));
+        }
+
+        refresh(INDEX);
+        assertHitCount(client().prepareSearch(INDEX).setSize(0).get(), nbDocs);
+
+        // Executes the delete by query; each shard will block after MAX_DELETIONS
+        DeleteByQueryRequestBuilder deleteByQuery = deleteByQuery().source("_all");
+        deleteByQuery.source().setSize(1);
+
+        ListenableActionFuture<BulkIndexByScrollResponse> future = deleteByQuery.execute();
+
+        // Waits for the indexing operation listener to block
+        barrier.await(30, TimeUnit.SECONDS);
+
+        // Status should show running
+        ListTasksResponse tasksList = client().admin().cluster().prepareListTasks()
+                .setActions(DeleteByQueryAction.NAME).setDetailed(true).get();
+        assertThat(tasksList.getNodeFailures(), empty());
+        assertThat(tasksList.getTaskFailures(), empty());
+        assertThat(tasksList.getTasks(), hasSize(1));
+        BulkByScrollTask.Status status = (BulkByScrollTask.Status) tasksList.getTasks().get(0).getStatus();
+        assertNull(status.getReasonCancelled());
+
+        // Cancel the request while the deletions are blocked. This will prevent further deletions requests from being sent.
+        List<TaskInfo> cancelledTasks = client().admin().cluster().prepareCancelTasks()
+                .setActions(DeleteByQueryAction.NAME).get().getTasks();
+        assertThat(cancelledTasks, hasSize(1));
+
+        // The status should now show canceled. The request will still be in the list because the script is still blocked.
+        tasksList = client().admin().cluster().prepareListTasks().setActions(DeleteByQueryAction.NAME).setDetailed(true).get();
+        assertThat(tasksList.getNodeFailures(), empty());
+        assertThat(tasksList.getTaskFailures(), empty());
+        assertThat(tasksList.getTasks(), hasSize(1));
+        status = (BulkByScrollTask.Status) tasksList.getTasks().get(0).getStatus();
+        assertEquals(CancelTasksRequest.DEFAULT_REASON, status.getReasonCancelled());
+
+        // Now unblock the listener so that it can proceed
+        barrier.await();
+
+        // And check the status of the response
+        BulkIndexByScrollResponse response = future.get();
+        assertThat(response, matcher()
+                .deleted(lessThanOrEqualTo((long) MAX_DELETIONS)).batches(MAX_DELETIONS).reasonCancelled(equalTo("by user request")));
+    }
+
+
+    public static class DeleteByQueryCancellationPlugin extends Plugin {
+
+        @Override
+        public String name() {
+            return "delete-by-query-cancellation";
+        }
+
+        @Override
+        public String description() {
+            return "See " + DeleteByQueryCancellationPlugin.class.getName();
+        }
+
+        @Override
+        public void onIndexModule(IndexModule indexModule) {
+            indexModule.addIndexOperationListener(new BlockingDeleteListener());
+        }
+    }
+
+    /**
+     * A {@link IndexingOperationListener} that allows a given number of documents to be deleted
+     * and then blocks until it is notified to proceed.
+     */
+    public static class BlockingDeleteListener implements IndexingOperationListener {
+
+        private final CountDown blockAfter = new CountDown(MAX_DELETIONS);
+
+        @Override
+        public Engine.Delete preDelete(Engine.Delete delete) {
+            if (blockAfter.isCountedDown() || (TYPE.equals(delete.type()) == false)) {
+                return delete;
+            }
+
+            if (blockAfter.countDown()) {
+                try {
+                    // Tell the test we've deleted enough documents.
+                    barrier.await(30, TimeUnit.SECONDS);
+
+                    // Wait for the test to tell us to proceed.
+                    barrier.await(30, TimeUnit.SECONDS);
+                } catch (InterruptedException | BrokenBarrierException | TimeoutException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+            return delete;
+        }
+    }
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryConcurrentTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryConcurrentTests.java
new file mode 100644
index 00000000000..40a776626a3
--- /dev/null
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryConcurrentTests.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.index.query.MatchQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
+import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.equalTo;
+
+public class DeleteByQueryConcurrentTests extends ReindexTestCase {
+
+    public void testConcurrentDeleteByQueriesOnDifferentDocs() throws Throwable {
+        final Thread[] threads =  new Thread[scaledRandomIntBetween(2, 5)];
+        final long docs = randomIntBetween(1, 50);
+
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+        for (int i = 0; i < docs; i++) {
+            for (int t = 0; t < threads.length; t++) {
+                builders.add(client().prepareIndex("test", "doc").setSource("field", t));
+            }
+        }
+        indexRandom(true, true, true, builders);
+
+        final CountDownLatch start = new CountDownLatch(1);
+        for (int t = 0; t < threads.length; t++) {
+            final int threadNum = t;
+            assertHitCount(client().prepareSearch("test").setSize(0).setQuery(QueryBuilders.termQuery("field", threadNum)).get(), docs);
+
+            Runnable r = () -> {
+                try {
+                    start.await();
+
+                    assertThat(deleteByQuery().source("_all").filter(termQuery("field", threadNum)).refresh(true).get(),
+                            matcher().deleted(docs));
+                } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                }
+            };
+            threads[t] = new Thread(r);
+            threads[t].start();
+        }
+
+        start.countDown();
+        for (Thread thread : threads) {
+            thread.join();
+        }
+
+        for (int t = 0; t < threads.length; t++) {
+            assertHitCount(client().prepareSearch("test").setSize(0).setQuery(QueryBuilders.termQuery("field", t)).get(), 0);
+        }
+    }
+
+    public void testConcurrentDeleteByQueriesOnSameDocs() throws Throwable {
+        final long docs = randomIntBetween(50, 100);
+
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+        for (int i = 0; i < docs; i++) {
+            builders.add(client().prepareIndex("test", "doc", String.valueOf(i)).setSource("foo", "bar"));
+        }
+        indexRandom(true, true, true, builders);
+
+        final Thread[] threads =  new Thread[scaledRandomIntBetween(2, 9)];
+
+        final CountDownLatch start = new CountDownLatch(1);
+        final MatchQueryBuilder query = matchQuery("foo", "bar");
+        final AtomicLong deleted = new AtomicLong(0);
+
+        for (int t = 0; t < threads.length; t++) {
+            Runnable r = () -> {
+                try {
+                    start.await();
+
+                    BulkIndexByScrollResponse response = deleteByQuery().source("test").filter(query).refresh(true).get();
+                    // Some deletions might fail due to version conflict, but
+                    // what matters here is the total of successful deletions
+                    deleted.addAndGet(response.getDeleted());
+                } catch (InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                }
+            };
+            threads[t] = new Thread(r);
+            threads[t].start();
+        }
+
+        start.countDown();
+        for (Thread thread : threads) {
+            thread.join();
+        }
+
+        assertHitCount(client().prepareSearch("test").setSize(0).get(), 0L);
+        assertThat(deleted.get(), equalTo(docs));
+    }
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java
index e82dd12a407..670fcefbf55 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java
@@ -29,7 +29,7 @@ import org.elasticsearch.index.mapper.internal.RoutingFieldMapper;
 public class ReindexMetadataTests extends AbstractAsyncBulkIndexbyScrollActionMetadataTestCase<ReindexRequest, BulkIndexByScrollResponse> {
     public void testRoutingCopiedByDefault() throws Exception {
         IndexRequest index = new IndexRequest();
-        action().copyMetadata(index, doc(RoutingFieldMapper.NAME, "foo"));
+        action().copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc(RoutingFieldMapper.NAME, "foo"));
         assertEquals("foo", index.routing());
     }
 
@@ -37,7 +37,7 @@ public class ReindexMetadataTests extends AbstractAsyncBulkIndexbyScrollActionMe
         TransportReindexAction.AsyncIndexBySearchAction action = action();
         action.mainRequest.getDestination().routing("keep");
         IndexRequest index = new IndexRequest();
-        action.copyMetadata(index, doc(RoutingFieldMapper.NAME, "foo"));
+        action.copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc(RoutingFieldMapper.NAME, "foo"));
         assertEquals("foo", index.routing());
     }
 
@@ -45,7 +45,7 @@ public class ReindexMetadataTests extends AbstractAsyncBulkIndexbyScrollActionMe
         TransportReindexAction.AsyncIndexBySearchAction action = action();
         action.mainRequest.getDestination().routing("discard");
         IndexRequest index = new IndexRequest();
-        action.copyMetadata(index, doc(RoutingFieldMapper.NAME, "foo"));
+        action.copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc(RoutingFieldMapper.NAME, "foo"));
         assertEquals(null, index.routing());
     }
 
@@ -53,7 +53,7 @@ public class ReindexMetadataTests extends AbstractAsyncBulkIndexbyScrollActionMe
         TransportReindexAction.AsyncIndexBySearchAction action = action();
         action.mainRequest.getDestination().routing("=cat");
         IndexRequest index = new IndexRequest();
-        action.copyMetadata(index, doc(RoutingFieldMapper.NAME, "foo"));
+        action.copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc(RoutingFieldMapper.NAME, "foo"));
         assertEquals("cat", index.routing());
     }
 
@@ -61,13 +61,13 @@ public class ReindexMetadataTests extends AbstractAsyncBulkIndexbyScrollActionMe
         TransportReindexAction.AsyncIndexBySearchAction action = action();
         action.mainRequest.getDestination().routing("==]");
         IndexRequest index = new IndexRequest();
-        action.copyMetadata(index, doc(RoutingFieldMapper.NAME, "foo"));
+        action.copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc(RoutingFieldMapper.NAME, "foo"));
         assertEquals("=]", index.routing());
     }
 
     @Override
     protected TransportReindexAction.AsyncIndexBySearchAction action() {
-        return new TransportReindexAction.AsyncIndexBySearchAction(task, logger, null, null, null, threadPool, request(), listener());
+        return new TransportReindexAction.AsyncIndexBySearchAction(task, logger, null, threadPool, request(), listener(), null, null);
     }
 
     @Override
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java
index b805dbd2772..74b7548cd63 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java
@@ -20,7 +20,10 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.lucene.uid.Versions;
+import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.ScriptService;
 
 import java.util.Map;
 
@@ -31,6 +34,7 @@ import static org.hamcrest.Matchers.containsString;
  * Tests index-by-search with a script modifying the documents.
  */
 public class ReindexScriptTests extends AbstractAsyncBulkIndexByScrollActionScriptTestCase<ReindexRequest, BulkIndexByScrollResponse> {
+
     public void testSetIndex() throws Exception {
         Object dest = randomFrom(new Object[] {234, 234L, "pancake"});
         IndexRequest index = applyScript((Map<String, Object> ctx) -> ctx.put("_index", dest));
@@ -129,11 +133,12 @@ public class ReindexScriptTests extends AbstractAsyncBulkIndexByScrollActionScri
 
     @Override
     protected ReindexRequest request() {
-        return new ReindexRequest();
+        return new ReindexRequest(new SearchRequest(), new IndexRequest());
     }
 
     @Override
-    protected AbstractAsyncBulkIndexByScrollAction<ReindexRequest> action() {
-        return new TransportReindexAction.AsyncIndexBySearchAction(task, logger, null, null, null, threadPool, request(), listener());
+    protected AbstractAsyncBulkIndexByScrollAction<ReindexRequest> action(ScriptService scriptService, ReindexRequest request) {
+        return new TransportReindexAction.AsyncIndexBySearchAction(task, logger, null, threadPool, request, listener(), scriptService,
+                null);
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java
index 33c72baa7cb..f4a777a1973 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java
@@ -43,6 +43,10 @@ public abstract class ReindexTestCase extends ESIntegTestCase {
         return UpdateByQueryAction.INSTANCE.newRequestBuilder(client());
     }
 
+    protected DeleteByQueryRequestBuilder deleteByQuery() {
+        return DeleteByQueryAction.INSTANCE.newRequestBuilder(client());
+    }
+
     protected RethrottleRequestBuilder rethrottle() {
         return RethrottleAction.INSTANCE.newRequestBuilder(client());
     }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java
index 5c5e45993b9..bb6a33b593a 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java
@@ -27,14 +27,13 @@ public class UpdateByQueryMetadataTests
         extends AbstractAsyncBulkIndexbyScrollActionMetadataTestCase<UpdateByQueryRequest, BulkIndexByScrollResponse> {
     public void testRoutingIsCopied() throws Exception {
         IndexRequest index = new IndexRequest();
-        action().copyMetadata(index, doc(RoutingFieldMapper.NAME, "foo"));
+        action().copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc(RoutingFieldMapper.NAME, "foo"));
         assertEquals("foo", index.routing());
     }
 
     @Override
     protected TransportUpdateByQueryAction.AsyncIndexBySearchAction action() {
-        return new TransportUpdateByQueryAction.AsyncIndexBySearchAction(task, logger, null, null, threadPool, null, request(),
-                listener());
+        return new TransportUpdateByQueryAction.AsyncIndexBySearchAction(task, logger, null, threadPool, request(), listener(), null, null);
     }
 
     @Override
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java
index 62f405cb0cd..1c57c202766 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.script.ScriptService;
+
 import java.util.Date;
 import java.util.Map;
 
@@ -26,6 +28,7 @@ import static org.hamcrest.Matchers.containsString;
 
 public class UpdateByQueryWithScriptTests
         extends AbstractAsyncBulkIndexByScrollActionScriptTestCase<UpdateByQueryRequest, BulkIndexByScrollResponse> {
+
     public void testModifyingCtxNotAllowed() {
         /*
          * Its important that none of these actually match any of the fields.
@@ -49,7 +52,8 @@ public class UpdateByQueryWithScriptTests
     }
 
     @Override
-    protected AbstractAsyncBulkIndexByScrollAction<UpdateByQueryRequest> action() {
-        return new TransportUpdateByQueryAction.AsyncIndexBySearchAction(task, logger, null, null, threadPool, null, request(), listener());
+    protected AbstractAsyncBulkIndexByScrollAction<UpdateByQueryRequest> action(ScriptService scriptService, UpdateByQueryRequest request) {
+        return new TransportUpdateByQueryAction.AsyncIndexBySearchAction(task, logger, null, threadPool, request, listener(),
+                scriptService, null);
     }
 }
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml
new file mode 100644
index 00000000000..bdad5f581bc
--- /dev/null
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml
@@ -0,0 +1,304 @@
+---
+"Basic response":
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      delete_by_query:
+        index: test
+        body:
+          query:
+            match_all: {}
+
+  - is_false: timed_out
+  - match: {deleted: 1}
+  - is_false: created
+  - is_false: updated
+  - match: {version_conflicts: 0}
+  - match: {batches: 1}
+  - match: {failures: []}
+  - match: {noops: 0}
+  - match: {throttled_millis: 0}
+  - gte: { took: 0 }
+  - is_false: task
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      count:
+        index: test
+
+  - match: {count: 0}
+
+---
+"wait_for_completion=false":
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      delete_by_query:
+        wait_for_completion: false
+        index: test
+        body:
+          query:
+            match_all: {}
+
+  - match: {task: '/.+:\d+/'}
+  - set: {task: task}
+  - is_false: version_conflicts
+  - is_false: batches
+  - is_false: failures
+  - is_false: noops
+  - is_false: took
+  - is_false: throttled_millis
+  - is_false: created
+  - is_false: updated
+  - is_false: deleted
+
+  - do:
+      tasks.list:
+        wait_for_completion: true
+        task_id: $task
+  - is_false: node_failures
+
+---
+"Response for version conflict":
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index.refresh_interval: -1
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+  # Creates a new version for reindex to miss on scan.
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      1
+        body:    { "text": "test2" }
+
+  - do:
+      catch: conflict
+      delete_by_query:
+        index: test
+        body:
+          query:
+            match_all: {}
+
+  - match: {deleted: 0}
+  - match: {version_conflicts: 1}
+  - match: {batches: 1}
+  - match: {failures.0.index:  test}
+  - match: {failures.0.type:   foo}
+  - match: {failures.0.id:     "1"}
+  - match: {failures.0.status: 409}
+  - match: {failures.0.cause.type:   version_conflict_engine_exception}
+  # Use a regex so we don't mind if the current version isn't always 1. Sometimes it comes out 2.
+  - match: {failures.0.cause.reason: "/\\[foo\\]\\[1\\]:.version.conflict,.current.version.\\[\\d+\\].is.different.than.the.one.provided.\\[\\d+\\]/"}
+  - match: {failures.0.cause.shard:  /\d+/}
+  - match: {failures.0.cause.index:  test}
+  - gte: { took: 0 }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      count:
+        index: test
+
+  - match: {count: 1}
+
+---
+"Response for version conflict with conflicts=proceed":
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index.refresh_interval: -1
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+  # Creates a new version for reindex to miss on scan.
+  - do:
+      index:
+        index:   test
+        type:    foo
+        id:      1
+        body:    { "text": "test2" }
+
+  - do:
+      delete_by_query:
+        index: test
+        conflicts: proceed
+        body:
+          query:
+            match_all: {}
+
+  - match: {deleted: 0}
+  - match: {version_conflicts: 1}
+  - match: {batches: 1}
+  - match: {noops: 0}
+  - match: {failures: []}
+  - match: {throttled_millis: 0}
+  - gte: { took: 0 }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      count:
+        index: test
+
+  - match: {count: 1}
+
+---
+"Limit by query":
+  - do:
+      index:
+        index:   twitter
+        type:    tweet
+        id:      1
+        body:    { "user": "kimchy" }
+  - do:
+      index:
+        index:   twitter
+        type:    tweet
+        id:      2
+        body:    { "user": "junk" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      delete_by_query:
+        index: twitter
+        refresh: true
+        body:
+          query:
+            match:
+              user: kimchy
+  - match: {deleted: 1}
+  - match: {version_conflicts: 0}
+  - match: {batches: 1}
+  - match: {failures: []}
+  - gte: { took: 0 }
+
+  - do:
+      count:
+        index: twitter
+
+  - match: {count: 1}
+
+---
+"Limit by size":
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     1
+        body:   { "user": "kimchy" }
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     2
+        body:   { "user": "kimchy" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      delete_by_query:
+        index: twitter
+        size: 1
+        body:
+          query:
+            match_all: {}
+
+  - match: {deleted: 1}
+  - match: {version_conflicts: 0}
+  - match: {batches: 1}
+  - match: {failures: []}
+  - match: {throttled_millis: 0}
+  - gte: { took: 0 }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      count:
+        index: twitter
+
+  - match: {count: 1}
+
+---
+"Can override scroll_size":
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 1
+  - do:
+      cluster.health:
+          wait_for_status: yellow
+  - do:
+      index:
+        index:   test
+        type:    foo
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      delete_by_query:
+        index: test
+        refresh: true
+        scroll_size: 1
+        body:
+          query:
+            match_all: {}
+
+  - match: {batches: 3}
+  - match: {deleted: 3}
+
+  - do:
+      count:
+        index: test
+
+  - match: {count: 0}
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml
new file mode 100644
index 00000000000..8648c9034ee
--- /dev/null
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml
@@ -0,0 +1,99 @@
+---
+"no body fails":
+  - do:
+      catch: param
+      delete_by_query:
+        index: _all
+
+---
+"invalid conflicts fails":
+  - do:
+      index:
+        index:   test
+        type:    test
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      catch: /conflicts may only be .* but was \[cat\]/
+      delete_by_query:
+        index: test
+        conflicts: cat
+        body:
+          query:
+            match_all: {}
+
+---
+"invalid size fails":
+  - do:
+      index:
+        index:   test
+        type:    test
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      catch: /size should be greater than 0 if the request is limited to some number of documents or -1 if it isn't but it was \[-4\]/
+      delete_by_query:
+        index: test
+        size: -4
+        body:
+          query:
+            match_all: {}
+
+---
+"invalid scroll_size fails":
+  - do:
+      index:
+        index:   test
+        type:    test
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      catch: /Failed to parse int parameter \[scroll_size\] with value \[asdf\]/
+      delete_by_query:
+        index: test
+        scroll_size: asdf
+        body:
+          query:
+            match_all: {}
+
+---
+"source fields may not be modified":
+  - do:
+      catch: /fields is not supported in this context/
+      delete_by_query:
+        index: test
+        body:
+          fields: [_id]
+
+---
+"requests_per_second cannot be an empty string":
+  - do:
+      catch: /\[requests_per_second\] must be a float greater than 0. Use "unlimited" to disable throttling./
+      delete_by_query:
+        requests_per_second: ""
+        index: test
+        body:
+          query:
+            match_all: {}
+
+---
+"requests_per_second cannot be negative":
+  - do:
+      catch: /\[requests_per_second\] must be a float greater than 0. Use "unlimited" to disable throttling./
+      delete_by_query:
+        requests_per_second: -12
+        index: test
+        body:
+          query:
+            match_all: {}
+
+---
+"requests_per_second cannot be zero":
+  - do:
+      catch: /\[requests_per_second\] must be a float greater than 0. Use "unlimited" to disable throttling./
+      delete_by_query:
+        requests_per_second: 0
+        index: test
+        body:
+          query:
+            match_all: {}
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/30_by_type.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/30_by_type.yaml
new file mode 100644
index 00000000000..1ee249fc6bb
--- /dev/null
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/30_by_type.yaml
@@ -0,0 +1,72 @@
+---
+"Delete by type":
+  - do:
+      index:
+          index:  test
+          type:   t1
+          id:     1
+          body:   { foo: bar }
+  - do:
+      index:
+          index:  test
+          type:   t1
+          id:     2
+          body:   { foo: bar }
+  - do:
+      index:
+          index:  test
+          type:   t2
+          id:     1
+          body:   { foo: bar }
+  - do:
+      index:
+          index:  test
+          type:   t2
+          id:     2
+          body:   { foo: bar }
+  - do:
+      index:
+          index:  test
+          type:   t2
+          id:     3
+          body:   { foo: baz }
+  - do:
+      indices.refresh: {}
+  - do:
+      count:
+        index: test
+        type: t2
+
+  - match: {count: 3}
+
+  - do:
+      delete_by_query:
+        index: test
+        type: t2
+        body:
+          query:
+            match:
+              foo: bar
+
+  - is_false: timed_out
+  - match: {deleted: 2}
+  - is_false: created
+  - is_false: updated
+  - match: {version_conflicts: 0}
+  - match: {batches: 1}
+  - match: {failures: []}
+  - match: {noops: 0}
+  - match: {throttled_millis: 0}
+  - gte: { took: 0 }
+  - is_false: task
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      count:
+        index: test
+        type: t2
+
+  - match: {count: 1}
+
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/50_consistency.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/50_consistency.yaml
new file mode 100644
index 00000000000..a69dfdfac04
--- /dev/null
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/50_consistency.yaml
@@ -0,0 +1,62 @@
+---
+"can override consistency":
+  - do:
+      indices.create:
+          index: test
+          body:
+            settings:
+              number_of_replicas: 5
+  - do:
+      cluster.health:
+          wait_for_status: yellow
+  - do:
+      index:
+        index:       test
+        type:        test
+        id:          1
+        body:        {"text": "test"}
+        consistency: one
+  - do:
+      indices.refresh: {}
+
+  - do:
+      catch: unavailable
+      delete_by_query:
+        index: test
+        timeout: 1s
+        body:
+          query:
+            match_all: {}
+
+  - match:
+      failures.0.cause.reason: /Not.enough.active.copies.to.meet.write.consistency.of.\[QUORUM\].\(have.1,.needed.4\)..Timeout\:.\[1s\],.request:.\[BulkShardRequest.to.\[test\].containing.\[1\].requests\]/
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      count:
+        index: test
+
+  - match: {count: 1}
+
+  - do:
+      delete_by_query:
+        index: test
+        consistency: one
+        body:
+          query:
+            match_all: {}
+
+  - match: {failures: []}
+  - match: {deleted: 1}
+  - match: {version_conflicts: 0}
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      count:
+        index: test
+
+  - match: {count: 0}
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yaml
new file mode 100644
index 00000000000..0ff382ff751
--- /dev/null
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yaml
@@ -0,0 +1,202 @@
+"Throttle the request":
+  # Throttling happens between each scroll batch so we need to control the size of the batch by using a single shard
+  # and a small batch size on the request
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 1
+  - do:
+      cluster.health:
+          wait_for_status: yellow
+  - do:
+      index:
+        index:   test
+        type:    foo
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      delete_by_query:
+        index: test
+        scroll_size: 1
+        requests_per_second: 1
+        body:
+          query:
+            match_all: {}
+
+  - match: {batches: 3}
+  - match: {deleted: 3}
+  - gt: {throttled_millis: 1000}
+  - lt: {throttled_millis: 4000}
+
+---
+"requests_per_second supports unlimited which turns off throttling":
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 1
+  - do:
+      cluster.health:
+          wait_for_status: yellow
+  - do:
+      index:
+        index:   test
+        type:    foo
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      delete_by_query:
+        index: test
+        scroll_size: 1
+        requests_per_second: unlimited
+        body:
+          query:
+            match_all: {}
+
+  - match: {batches: 3}
+  - match: {deleted: 3}
+  - match: {throttled_millis: 0}
+
+---
+"Rethrottle":
+  # Throttling happens between each scroll batch so we need to control the size of the batch by using a single shard
+  # and a small batch size on the request
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 1
+  - do:
+      cluster.health:
+          wait_for_status: yellow
+  - do:
+      index:
+        index:   test
+        type:    foo
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      delete_by_query:
+        requests_per_second: .00000001 # About 9.5 years to complete the request
+        wait_for_completion: false
+        index: test
+        scroll_size: 1
+        body:
+          query:
+            match_all: {}
+
+  - match: {task: '/.+:\d+/'}
+  - set: {task: task}
+
+  - do:
+      reindex.rethrottle:
+        requests_per_second: unlimited
+        task_id: $task
+
+  - do:
+      tasks.list:
+        wait_for_completion: true
+        task_id: $task
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      count:
+        index: test
+
+  - match: {count: 0}
+
+---
+"Rethrottle but not unlimited":
+  # Throttling happens between each scroll batch so we need to control the size of the batch by using a single shard
+  # and a small batch size on the request
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 1
+  - do:
+      cluster.health:
+          wait_for_status: yellow
+  - do:
+      index:
+        index:   test
+        type:    foo
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   test
+        type:    foo
+        body:    { "text": "test" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      delete_by_query:
+        requests_per_second: .00000001 # About 9.5 years to complete the request
+        wait_for_completion: false
+        index: test
+        scroll_size: 1
+        body:
+          query:
+            match_all: {}
+
+  - match: {task: '/.+:\d+/'}
+  - set: {task: task}
+
+  - do:
+      reindex.rethrottle:
+        requests_per_second: 1
+        task_id: $task
+
+  - do:
+      tasks.list:
+        wait_for_completion: true
+        task_id: $task
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml
index 63d0edefe14..f4025383321 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml
@@ -24,6 +24,7 @@
   - match: {throttled_millis: 0}
   - gte: { took: 0 }
   - is_false: task
+  - is_false: deleted
 
 ---
 "Response format for updated":
@@ -57,6 +58,7 @@
   - match: {throttled_millis: 0}
   - gte: { took: 0 }
   - is_false: task
+  - is_false: deleted
 
 ---
 "wait_for_completion=false":
@@ -88,6 +90,7 @@
   - is_false: took
   - is_false: throttled_millis
   - is_false: created
+  - is_false: deleted
 
   - do:
       tasks.list:
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
index dc54c0d91cf..a7a5198e430 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
@@ -59,7 +59,7 @@
 ---
 "search size fails if not a number":
   - do:
-      catch: '/NumberFormatException: For input string: "cat"/'
+      catch: '/number_format_exception.*For input string: \"cat\"/'
       reindex:
         body:
           source:
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yaml
index 89266198f2c..843bb9b6eb5 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yaml
@@ -23,6 +23,7 @@
   # Update by query can't create
   - is_false: created
   - is_false: task
+  - is_false: deleted
 
 ---
 "wait_for_completion=false":
@@ -49,6 +50,7 @@
   - is_false: took
   - is_false: throttled_millis
   - is_false: created
+  - is_false: deleted
 
   - do:
       tasks.list:
diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml b/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml
index 00c6e814eed..54a79ac1e32 100644
--- a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml
+++ b/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml
@@ -102,7 +102,7 @@
   - match: {batches: 1}
 
 ---
-"Setting bogus ctx is an error":
+"Setting bogus context is an error":
   - do:
       index:
         index:  twitter
@@ -113,7 +113,7 @@
       indices.refresh: {}
 
   - do:
-      catch: /Invalid fields added to ctx \[junk\]/
+      catch: /Invalid fields added to context \[junk\]/
       update_by_query:
         index: twitter
         body:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
new file mode 100644
index 00000000000..981aea79a1c
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
@@ -0,0 +1,207 @@
+{
+  "delete_by_query": {
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/plugins/master/plugins-reindex.html",
+    "methods": ["POST"],
+    "url": {
+      "path": "/{index}/_delete_by_query",
+      "paths": ["/{index}/_delete_by_query", "/{index}/{type}/_delete_by_query"],
+      "comment": "most things below this are just copied from search.json",
+      "parts": {
+        "index": {
+         "required" : true,
+         "type" : "list",
+         "description" : "A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices"
+        },
+        "type": {
+          "type" : "list",
+          "description" : "A comma-separated list of document types to search; leave empty to perform the operation on all types"
+        }
+      },
+      "params": {
+        "analyzer": {
+          "type" : "string",
+          "description" : "The analyzer to use for the query string"
+        },
+        "analyze_wildcard": {
+          "type" : "boolean",
+          "description" : "Specify whether wildcard and prefix queries should be analyzed (default: false)"
+        },
+        "default_operator": {
+          "type" : "enum",
+          "options" : ["AND","OR"],
+          "default" : "OR",
+          "description" : "The default operator for query string query (AND or OR)"
+        },
+        "df": {
+          "type" : "string",
+          "description" : "The field to use as default where no field prefix is given in the query string"
+        },
+        "explain": {
+          "type" : "boolean",
+          "description" : "Specify whether to return detailed information about score computation as part of a hit"
+        },
+        "fields": {
+          "type" : "list",
+          "description" : "A comma-separated list of fields to return as part of a hit"
+        },
+        "fielddata_fields": {
+          "type" : "list",
+          "description" : "A comma-separated list of fields to return as the field data representation of a field for each hit"
+        },
+        "from": {
+          "type" : "number",
+          "description" : "Starting offset (default: 0)"
+        },
+        "ignore_unavailable": {
+            "type" : "boolean",
+            "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
+        },
+        "allow_no_indices": {
+            "type" : "boolean",
+            "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
+        },
+        "conflicts": {
+            "note": "This is not copied from search",
+            "type" : "enum",
+            "options": ["abort", "proceed"],
+            "default": "abort",
+            "description" : "What to do when the delete-by-query hits version conflicts?"
+        },
+        "expand_wildcards": {
+            "type" : "enum",
+            "options" : ["open","closed","none","all"],
+            "default" : "open",
+            "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both."
+        },
+        "lenient": {
+          "type" : "boolean",
+          "description" : "Specify whether format-based query failures (such as providing text to a numeric field) should be ignored"
+        },
+        "lowercase_expanded_terms": {
+          "type" : "boolean",
+          "description" : "Specify whether query terms should be lowercased"
+        },
+        "preference": {
+          "type" : "string",
+          "description" : "Specify the node or shard the operation should be performed on (default: random)"
+        },
+        "q": {
+          "type" : "string",
+          "description" : "Query in the Lucene query string syntax"
+        },
+        "routing": {
+          "type" : "list",
+          "description" : "A comma-separated list of specific routing values"
+        },
+        "scroll": {
+          "type" : "duration",
+          "description" : "Specify how long a consistent view of the index should be maintained for scrolled search"
+        },
+        "search_type": {
+          "type" : "enum",
+          "options" : ["query_then_fetch", "dfs_query_then_fetch"],
+          "description" : "Search operation type"
+        },
+        "search_timeout": {
+          "type" : "time",
+          "description" : "Explicit timeout for each search request. Defaults to no timeout."
+        },
+        "size": {
+          "type" : "number",
+          "description" : "Number of hits to return (default: 10)"
+        },
+        "sort": {
+          "type" : "list",
+          "description" : "A comma-separated list of <field>:<direction> pairs"
+        },
+        "_source": {
+          "type" : "list",
+          "description" : "True or false to return the _source field or not, or a list of fields to return"
+        },
+        "_source_exclude": {
+          "type" : "list",
+          "description" : "A list of fields to exclude from the returned _source field"
+        },
+        "_source_include": {
+          "type" : "list",
+          "description" : "A list of fields to extract and return from the _source field"
+        },
+        "terminate_after": {
+          "type" : "number",
+          "description" : "The maximum number of documents to collect for each shard, upon reaching which the query execution will terminate early."
+        },
+        "stats": {
+          "type" : "list",
+          "description" : "Specific 'tag' of the request for logging and statistical purposes"
+        },
+        "suggest_field": {
+          "type" : "string",
+          "description" : "Specify which field to use for suggestions"
+        },
+        "suggest_mode": {
+          "type" : "enum",
+          "options" : ["missing", "popular", "always"],
+          "default" : "missing",
+          "description" : "Specify suggest mode"
+        },
+        "suggest_size": {
+          "type" : "number",
+          "description" : "How many suggestions to return in response"
+        },
+        "suggest_text": {
+          "type" : "text",
+          "description" : "The source text for which the suggestions should be returned"
+        },
+        "timeout": {
+          "type" : "time",
+          "description" : "Explicit operation timeout"
+        },
+        "track_scores": {
+          "type" : "boolean",
+          "description": "Whether to calculate and return scores even if they are not used for sorting"
+        },
+        "version": {
+          "type" : "boolean",
+          "description" : "Specify whether to return document version as part of a hit"
+        },
+        "request_cache": {
+          "type" : "boolean",
+          "description" : "Specify if request cache should be used for this request or not, defaults to index level setting"
+        },
+        "refresh": {
+          "type" : "boolean",
+          "description" : "Should the effected indexes be refreshed?"
+        },
+        "timeout": {
+          "type" : "time",
+          "default": "1m",
+          "description" : "Time each individual bulk request should wait for shards that are unavailable."
+        },
+        "consistency": {
+          "type" : "enum",
+          "options" : ["one", "quorum", "all"],
+          "description" : "Explicit write consistency setting for the operation"
+        },
+        "scroll_size": {
+          "type": "integer",
+          "defaut_value": 100,
+          "description": "Size on the scroll request powering the update_by_query"
+        },
+        "wait_for_completion": {
+           "type" : "boolean",
+           "default": false,
+           "description" : "Should the request should block until the delete-by-query is complete."
+        },
+        "requests_per_second": {
+          "type": "float",
+          "default": 0,
+          "description": "The throttle for this request in sub-requests per second. 0 means set no throttle."
+        }
+      }
+    },
+    "body": {
+      "description": "The search definition using the Query DSL",
+      "required": true
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.rethrottle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.rethrottle.json
index 4903c7598c3..921249ab4cd 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.rethrottle.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.rethrottle.json
@@ -4,7 +4,7 @@
     "methods": ["POST"],
     "url": {
       "path": "/_reindex/{task_id}/_rethrottle",
-      "paths": ["/_reindex/{task_id}/_rethrottle", "/_update_by_query/{task_id}/_rethrottle"],
+      "paths": ["/_reindex/{task_id}/_rethrottle", "/_update_by_query/{task_id}/_rethrottle", "/_delete_by_query/{task_id}/_rethrottle"],
       "parts": {
         "task_id": {
           "type": "string",

From e289de6e96520aaba42f98fb284ccd4dac7fc7d5 Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Thu, 19 May 2016 16:50:37 +0200
Subject: [PATCH 0350/1311] Move `throttle_retries` under `repositories.s3.`
 prefix or per repository

I initially wrongly put this setting under `cloud.aws.s3.` prefix which does not make sense. It should be placed at the same place as `max_retries`.

Also applied @tlrx comments. We should set this even if max_retries is not set (when using default values).

Also added some documentation about this setting.
---
 docs/plugins/repository-s3.asciidoc           | 13 ++++++++++--
 .../elasticsearch/cloud/aws/AwsS3Service.java |  7 ++-----
 .../cloud/aws/InternalAwsS3Service.java       | 10 +++++----
 .../repository/s3/S3RepositoryPlugin.java     |  3 ++-
 .../repositories/s3/S3Repository.java         | 21 ++++++++++++++++---
 .../cloud/aws/TestAwsS3Service.java           |  5 +++--
 .../s3/AbstractS3SnapshotRestoreTest.java     |  5 +++--
 7 files changed, 45 insertions(+), 19 deletions(-)

diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc
index 8780b3a710e..f862fcb7cdb 100644
--- a/docs/plugins/repository-s3.asciidoc
+++ b/docs/plugins/repository-s3.asciidoc
@@ -39,7 +39,8 @@ http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
 credentials for authentication. These can be overridden by, in increasing
 order of precedence, system properties `aws.accessKeyId` and `aws.secretKey`,
 environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_KEY`, or the
-elasticsearch config using `cloud.aws.access_key` and `cloud.aws.secret_key`:
+elasticsearch config using `cloud.aws.access_key` and `cloud.aws.secret_key` or
+if you wish to set credentials specifically for s3 `cloud.aws.s3.access_key` and `cloud.aws.s3.secret_key`:
 
 [source,yaml]
 ----
@@ -106,6 +107,7 @@ cloud:
 ===== Region
 
 The `cloud.aws.region` can be set to a region and will automatically use the relevant settings for both `ec2` and `s3`.
+You can specifically set it for s3 only using `cloud.aws.s3.region`.
 The available values are:
 
 * `us-east` (`us-east-1`)
@@ -216,9 +218,13 @@ The following settings are supported:
 
     Number of retries in case of S3 errors. Defaults to `3`.
 
+`throttle_retries`::
+
+    Set to `true` if you want to throttle retries. Defaults to AWS SDK default value (`false`).
+
 `read_only`::
 
-    Makes repository read-only. coming[2.1.0]  Defaults to `false`.
+    Makes repository read-only. Defaults to `false`.
 
 `canned_acl`::
 
@@ -236,6 +242,9 @@ The following settings are supported:
     currently supported by the plugin. For more information about the
     different classes, see http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html[AWS Storage Classes Guide]
 
+Note that you can define S3 repository settings for all S3 repositories in `elasticsearch.yml` configuration file.
+They are all prefixed with `repositories.s3.`.
+
 The S3 repositories use the same credentials as the rest of the AWS services
 provided by this plugin (`discovery`). See <<repository-s3-usage>> for details.
 
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
index 383be6825c2..59a1dbff45f 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
@@ -151,11 +151,8 @@ public interface AwsS3Service extends LifecycleComponent<AwsS3Service> {
          * cloud.aws.s3.endpoint: Endpoint. If not set, endpoint will be guessed based on region setting.
          */
         Setting<String> ENDPOINT_SETTING = Setting.simpleString("cloud.aws.s3.endpoint", Property.NodeScope);
-        /**
-         * cloud.aws.s3.throttle_retries: Set to `true` if you want to throttle retries. Defaults to `true`.
-         */
-        Setting<Boolean> THROTTLE_RETRIES_SETTING = Setting.boolSetting("cloud.aws.s3.throttle_retries", true, Property.NodeScope);
     }
 
-    AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries);
+    AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries,
+                    boolean throttleRetries);
 }
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
index 67fae30657d..887e8ba61d9 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
@@ -57,7 +57,8 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
     }
 
     @Override
-    public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries) {
+    public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries,
+                                        boolean throttleRetries) {
         if (Strings.isNullOrEmpty(endpoint)) {
             // We need to set the endpoint based on the region
             if (region != null) {
@@ -69,10 +70,11 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
             }
         }
 
-        return getClient(endpoint, protocol, account, key, maxRetries);
+        return getClient(endpoint, protocol, account, key, maxRetries, throttleRetries);
     }
 
-    private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, String account, String key, Integer maxRetries) {
+    private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, String account, String key, Integer maxRetries,
+                                            boolean throttleRetries) {
         Tuple<String, String> clientDescriptor = new Tuple<>(endpoint, account);
         AmazonS3Client client = clients.get(clientDescriptor);
         if (client != null) {
@@ -101,8 +103,8 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
         if (maxRetries != null) {
             // If not explicitly set, default to 3 with exponential backoff policy
             clientConfiguration.setMaxErrorRetry(maxRetries);
-            clientConfiguration.setUseThrottleRetries(AwsS3Service.CLOUD_S3.THROTTLE_RETRIES_SETTING.get(settings));
         }
+        clientConfiguration.setUseThrottleRetries(throttleRetries);
 
         // #155: we might have 3rd party users using older S3 API version
         String awsSigner = CLOUD_S3.SIGNER_SETTING.get(settings);
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
index 4da73bf3353..9fd9b0d21c7 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
@@ -112,7 +112,6 @@ public class S3RepositoryPlugin extends Plugin {
         settingsModule.registerSetting(AwsS3Service.CLOUD_S3.SIGNER_SETTING);
         settingsModule.registerSetting(AwsS3Service.CLOUD_S3.REGION_SETTING);
         settingsModule.registerSetting(AwsS3Service.CLOUD_S3.ENDPOINT_SETTING);
-        settingsModule.registerSetting(AwsS3Service.CLOUD_S3.THROTTLE_RETRIES_SETTING);
 
         // Register S3 repositories settings: repositories.s3
         settingsModule.registerSetting(S3Repository.Repositories.KEY_SETTING);
@@ -129,6 +128,7 @@ public class S3RepositoryPlugin extends Plugin {
         settingsModule.registerSetting(S3Repository.Repositories.STORAGE_CLASS_SETTING);
         settingsModule.registerSetting(S3Repository.Repositories.CANNED_ACL_SETTING);
         settingsModule.registerSetting(S3Repository.Repositories.BASE_PATH_SETTING);
+        settingsModule.registerSetting(S3Repository.Repositories.THROTTLE_RETRIES_SETTING);
 
         // Register S3 single repository settings
         settingsModule.registerSetting(S3Repository.Repository.KEY_SETTING);
@@ -145,6 +145,7 @@ public class S3RepositoryPlugin extends Plugin {
         settingsModule.registerSetting(S3Repository.Repository.STORAGE_CLASS_SETTING);
         settingsModule.registerSetting(S3Repository.Repository.CANNED_ACL_SETTING);
         settingsModule.registerSetting(S3Repository.Repository.BASE_PATH_SETTING);
+        settingsModule.registerSetting(S3Repository.Repository.THROTTLE_RETRIES_SETTING);
     }
 
     /**
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
index a09d57ebc93..b275c45ef1d 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.repositories.s3;
 
+import com.amazonaws.ClientConfiguration;
 import com.amazonaws.Protocol;
 import org.elasticsearch.cloud.aws.AwsS3Service;
 import org.elasticsearch.cloud.aws.AwsS3Service.CLOUD_S3;
@@ -115,6 +116,11 @@ public class S3Repository extends BlobStoreRepository {
          * repositories.s3.max_retries: Number of retries in case of S3 errors. Defaults to 3.
          */
         Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("repositories.s3.max_retries", 3, Property.NodeScope);
+        /**
+         * repositories.s3.throttle_retries: Set to `true` if you want to throttle retries. Defaults to AWS SDK default value (`false`).
+         */
+        Setting<Boolean> THROTTLE_RETRIES_SETTING = Setting.boolSetting("repositories.s3.throttle_retries",
+            ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope);
         /**
          * repositories.s3.chunk_size: Big files can be broken down into chunks during snapshotting if needed. Defaults to 1g.
          */
@@ -196,6 +202,12 @@ public class S3Repository extends BlobStoreRepository {
          * @see  Repositories#MAX_RETRIES_SETTING
          */
         Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("max_retries", 3, Property.NodeScope);
+        /**
+         * throttle_retries
+         * @see  Repositories#THROTTLE_RETRIES_SETTING
+         */
+        Setting<Boolean> THROTTLE_RETRIES_SETTING = Setting.boolSetting("throttle_retries",
+            ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope);
         /**
          * chunk_size
          * @see  Repositories#CHUNK_SIZE_SETTING
@@ -262,6 +274,7 @@ public class S3Repository extends BlobStoreRepository {
         boolean serverSideEncryption = getValue(repositorySettings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING);
         ByteSizeValue bufferSize = getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING);
         Integer maxRetries = getValue(repositorySettings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING);
+        boolean throttleRetries = getValue(repositorySettings, Repository.THROTTLE_RETRIES_SETTING, Repositories.THROTTLE_RETRIES_SETTING);
         this.chunkSize = getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING);
         this.compress = getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING);
 
@@ -275,13 +288,15 @@ public class S3Repository extends BlobStoreRepository {
         String storageClass = getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING);
         String cannedACL = getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING);
 
-        logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], buffer_size [{}], max_retries [{}], cannedACL [{}], storageClass [{}]",
-                bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
+        logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], " +
+            "buffer_size [{}], max_retries [{}], throttle_retries [{}], cannedACL [{}], storageClass [{}]",
+            bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, throttleRetries, cannedACL,
+            storageClass);
 
         String key = getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING);
         String secret = getValue(repositorySettings, Repository.SECRET_SETTING, Repositories.SECRET_SETTING);
 
-        blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, key, secret, maxRetries),
+        blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, key, secret, maxRetries, throttleRetries),
                 bucket, region, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
 
         String basePath = getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING);
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
index 47e884d73bd..1a3ce418c0b 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
@@ -51,8 +51,9 @@ public class TestAwsS3Service extends InternalAwsS3Service {
 
 
     @Override
-    public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries) {
-        return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries));
+    public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries,
+                                        boolean throttleRetries) {
+        return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries, throttleRetries));
     }
 
     private AmazonS3 cachedWrapper(AmazonS3 client) {
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
index 743e2645ab2..2e196610c1a 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
@@ -200,7 +200,7 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
             S3Repository.Repositories.REGION_SETTING.get(settings),
             S3Repository.Repositories.KEY_SETTING.get(settings),
             S3Repository.Repositories.SECRET_SETTING.get(settings),
-            null);
+            null, randomBoolean());
 
         String bucketName = bucket.get("bucket");
         logger.info("--> verify encryption for bucket [{}], prefix [{}]", bucketName, basePath);
@@ -475,7 +475,8 @@ abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
             // We check that settings has been set in elasticsearch.yml integration test file
             // as described in README
             assertThat("Your settings in elasticsearch.yml are incorrects. Check README file.", bucketName, notNullValue());
-            AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(endpoint, protocol, region, accessKey, secretKey, null);
+            AmazonS3 client = internalCluster().getInstance(AwsS3Service.class).client(endpoint, protocol, region, accessKey, secretKey,
+                null, randomBoolean());
             try {
                 ObjectListing prevListing = null;
                 //From http://docs.amazonwebservices.com/AmazonS3/latest/dev/DeletingMultipleObjectsUsingJava.html

From 757ccf00b296228e4b2e63485f7aa0d37a131d7d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Thu, 19 May 2016 14:53:55 +0200
Subject: [PATCH 0351/1311] Enforce MatchQueryBuilder#maxExpansions() to be
 strictly positive

---
 .../index/query/MatchQueryBuilder.java        |  6 +-
 .../index/query/MatchQueryBuilderTests.java   | 67 +++++++------------
 2 files changed, 27 insertions(+), 46 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java
index 58207f7ca54..6c867590653 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java
@@ -267,7 +267,7 @@ public class MatchQueryBuilder extends AbstractQueryBuilder<MatchQueryBuilder> {
      */
     public MatchQueryBuilder prefixLength(int prefixLength) {
         if (prefixLength < 0 ) {
-            throw new IllegalArgumentException("No negative prefix length allowed.");
+            throw new IllegalArgumentException("[" + NAME + "] requires prefix length to be non-negative.");
         }
         this.prefixLength = prefixLength;
         return this;
@@ -284,8 +284,8 @@ public class MatchQueryBuilder extends AbstractQueryBuilder<MatchQueryBuilder> {
      * When using fuzzy or prefix type query, the number of term expansions to use.
      */
     public MatchQueryBuilder maxExpansions(int maxExpansions) {
-        if (maxExpansions < 0 ) {
-            throw new IllegalArgumentException("No negative maxExpansions allowed.");
+        if (maxExpansions <= 0 ) {
+            throw new IllegalArgumentException("[" + NAME + "] requires maxExpansions to be positive.");
         }
         this.maxExpansions = maxExpansions;
         return this;
diff --git a/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
index 87f8f3981d3..c5e01c1d0ff 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
@@ -23,7 +23,6 @@ import org.apache.lucene.queries.ExtendedCommonTermsQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.FuzzyQuery;
-import org.apache.lucene.search.LegacyNumericRangeQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.PointRangeQuery;
@@ -202,64 +201,46 @@ public class MatchQueryBuilderTests extends AbstractQueryTestCase<MatchQueryBuil
     }
 
     public void testIllegalValues() {
-        try {
-            new MatchQueryBuilder(null, "value");
-            fail("value must not be non-null");
-        } catch (IllegalArgumentException ex) {
-            // expected
+        {
+            IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new MatchQueryBuilder(null, "value"));
+            assertEquals("[match] requires fieldName", e.getMessage());
         }
 
-        try {
-            new MatchQueryBuilder("fieldName", null);
-            fail("value must not be non-null");
-        } catch (IllegalArgumentException ex) {
-            // expected
+        {
+            IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new MatchQueryBuilder("fieldName", null));
+            assertEquals("[match] requires query value", e.getMessage());
         }
 
         MatchQueryBuilder matchQuery = new MatchQueryBuilder("fieldName", "text");
-        try {
-            matchQuery.prefixLength(-1);
-            fail("must not be positive");
-        } catch (IllegalArgumentException ex) {
-            // expected
+        {
+            IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> matchQuery.prefixLength(-1));
+            assertEquals("[match] requires prefix length to be non-negative.", e.getMessage());
         }
 
-        try {
-            matchQuery.maxExpansions(-1);
-            fail("must not be positive");
-        } catch (IllegalArgumentException ex) {
-            // expected
+        {
+            IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                    () -> matchQuery.maxExpansions(randomIntBetween(-10, 0)));
+            assertEquals("[match] requires maxExpansions to be positive.", e.getMessage());
         }
 
-        try {
-            matchQuery.operator(null);
-            fail("must not be non-null");
-        } catch (IllegalArgumentException ex) {
-            // expected
+        {
+            IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> matchQuery.operator(null));
+            assertEquals("[match] requires operator to be non-null", e.getMessage());
         }
 
-        try {
-            matchQuery.type(null);
-            fail("must not be non-null");
-        } catch (IllegalArgumentException ex) {
-            // expected
+        {
+            IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> matchQuery.type(null));
+            assertEquals("[match] requires type to be non-null", e.getMessage());
         }
 
-        try {
-            matchQuery.zeroTermsQuery(null);
-            fail("must not be non-null");
-        } catch (IllegalArgumentException ex) {
-            // expected
+        {
+            IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> matchQuery.zeroTermsQuery(null));
+            assertEquals("[match] requires zeroTermsQuery to be non-null", e.getMessage());
         }
-    }
 
-    public void testBadAnalyzer() throws IOException {
-        MatchQueryBuilder matchQuery = new MatchQueryBuilder("fieldName", "text");
         matchQuery.analyzer("bogusAnalyzer");
-        try {
-            matchQuery.toQuery(createShardContext());
-            fail("Expected QueryShardException");
-        } catch (QueryShardException e) {
+        {
+            QueryShardException e = expectThrows(QueryShardException.class, () -> matchQuery.toQuery(createShardContext()));
             assertThat(e.getMessage(), containsString("analyzer [bogusAnalyzer] not found"));
         }
     }

From 4d6887075f2a57944001032630d4f55fd9616d9f Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Thu, 19 May 2016 17:12:37 +0200
Subject: [PATCH 0352/1311] Log IndexShard.refresh logs under trace (#18435)

We log them every second...
---
 .../org/elasticsearch/index/shard/IndexShard.java    | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 768db935308..2cadc1ad24b 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -576,16 +576,22 @@ public class IndexShard extends AbstractIndexShardComponent {
             long bytes = getEngine().getIndexBufferRAMBytesUsed();
             writingBytes.addAndGet(bytes);
             try {
-                logger.debug("refresh with source [{}] indexBufferRAMBytesUsed [{}]", source, new ByteSizeValue(bytes));
+                if (logger.isTraceEnabled()) {
+                    logger.trace("refresh with source [{}] indexBufferRAMBytesUsed [{}]", source, new ByteSizeValue(bytes));
+                }
                 long time = System.nanoTime();
                 getEngine().refresh(source);
                 refreshMetric.inc(System.nanoTime() - time);
             } finally {
-                logger.debug("remove [{}] writing bytes for shard [{}]", new ByteSizeValue(bytes), shardId());
+                if (logger.isTraceEnabled()) {
+                    logger.trace("remove [{}] writing bytes for shard [{}]", new ByteSizeValue(bytes), shardId());
+                }
                 writingBytes.addAndGet(-bytes);
             }
         } else {
-            logger.debug("refresh with source [{}]", source);
+            if (logger.isTraceEnabled()) {
+                logger.trace("refresh with source [{}]", source);
+            }
             long time = System.nanoTime();
             getEngine().refresh(source);
             refreshMetric.inc(System.nanoTime() - time);

From d2515727d09b16a97357f7963538ce40928112cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Thu, 19 May 2016 16:09:57 +0200
Subject: [PATCH 0353/1311] Improve random DateTimeZone creation in tests

We often require a random joda DateTimeZone in our tests. Currently
there are a few options for generating such a random DateTimeZone
from the set of available ids. Currently most random picks are not
really reproducable across different jvms because they rely on order
in the ids set implementation. The helper in DateProcessorFactoryTests
thus performs a sort on the set of ids before random picking from
the result, so I moved this to ESTestCase to make it publicly
available and changed all other tests to use that method.
---
 .../rounding/TimeZoneRoundingTests.java       | 10 ++++------
 .../index/query/AbstractQueryTestCase.java    |  6 ------
 .../query/QueryStringQueryBuilderTests.java   |  4 +---
 .../index/query/RangeQueryBuilderTests.java   |  2 +-
 .../processor/DateProcessorFactoryTests.java  | 10 +---------
 .../aggregations/bucket/DateRangeIT.java      |  3 +--
 .../aggregations/bucket/DateRangeTests.java   |  5 +----
 .../elasticsearch/test/ESIntegTestCase.java   | 19 -------------------
 .../org/elasticsearch/test/ESTestCase.java    | 10 ++++++++++
 9 files changed, 19 insertions(+), 50 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java b/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
index 08a4ba11342..61710b726d4 100644
--- a/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
+++ b/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
@@ -25,7 +25,6 @@ import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.ISODateTimeFormat;
 
-import java.util.ArrayList;
 import java.util.concurrent.TimeUnit;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -234,19 +233,18 @@ public class TimeZoneRoundingTests extends ESTestCase {
      * amount of milliseconds.
      */
     public void testSubHourNextRoundingEquallySpaced() {
-        String timeZone = randomFrom(new ArrayList<>(DateTimeZone.getAvailableIDs()));
         DateTimeUnit unit = randomFrom(new DateTimeUnit[] { DateTimeUnit.HOUR_OF_DAY, DateTimeUnit.MINUTES_OF_HOUR,
                 DateTimeUnit.SECOND_OF_MINUTE });
-        DateTimeZone tz = DateTimeZone.forID(timeZone);
-        TimeZoneRounding rounding = new TimeZoneRounding.TimeUnitRounding(unit, tz);
+        DateTimeZone timezone = randomDateTimeZone();
+        TimeZoneRounding rounding = new TimeZoneRounding.TimeUnitRounding(unit, timezone);
         // move the random date to transition for timezones that have offset change due to dst transition
-        long nextTransition = tz.nextTransition(Math.abs(randomLong() % ((long) 10e11)));
+        long nextTransition = timezone.nextTransition(Math.abs(randomLong() % ((long) 10e11)));
         final long millisPerUnit = unit.field().getDurationField().getUnitMillis();
         // start ten units before transition
         long roundedDate = rounding.round(nextTransition - (10 * millisPerUnit));
         while (roundedDate < nextTransition + 10 * millisPerUnit) {
             long delta = rounding.nextRoundingValue(roundedDate) - roundedDate;
-            assertEquals("Difference between rounded values not equally spaced for [" + unit.name() + "], [" + timeZone + "] at "
+            assertEquals("Difference between rounded values not equally spaced for [" + unit.name() + "], [" + timezone + "] at "
                     + new DateTime(roundedDate), millisPerUnit, delta);
             roundedDate = rounding.nextRoundingValue(roundedDate);
         }
diff --git a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
index 9c47d701f57..22bdc2c7290 100644
--- a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
@@ -903,12 +903,6 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
         return randomFrom("1", "-1", "75%", "-25%", "2<75%", "2<-25%");
     }
 
-    protected static String randomTimeZone() {
-        return randomFrom(TIMEZONE_IDS);
-    }
-
-    private static final List<String> TIMEZONE_IDS = new ArrayList<>(DateTimeZone.getAvailableIDs());
-
     private static class ClientInvocationHandler implements InvocationHandler {
         AbstractQueryTestCase<?> delegate;
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
index 375db0c1eaf..cf16456b2bf 100644
--- a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.index.query;
 
-import org.apache.lucene.document.IntPoint;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.queryparser.classic.MapperQueryParser;
 import org.apache.lucene.queryparser.classic.QueryParserSettings;
@@ -27,7 +26,6 @@ import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.DisjunctionMaxQuery;
-import org.apache.lucene.search.LegacyNumericRangeQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.PhraseQuery;
@@ -148,7 +146,7 @@ public class QueryStringQueryBuilderTests extends AbstractQueryTestCase<QueryStr
             queryStringQueryBuilder.locale(randomLocale(random()));
         }
         if (randomBoolean()) {
-            queryStringQueryBuilder.timeZone(randomTimeZone());
+            queryStringQueryBuilder.timeZone(randomDateTimeZone().getID());
         }
         return queryStringQueryBuilder;
     }
diff --git a/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
index c3db90f1f6d..dd97e7a68eb 100644
--- a/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
@@ -69,7 +69,7 @@ public class RangeQueryBuilderTests extends AbstractQueryTestCase<RangeQueryBuil
                 // otherwise we could trigger exception.
                 if (createShardContext().getMapperService().fullName(DATE_FIELD_NAME) != null) {
                     if (randomBoolean()) {
-                        query.timeZone(randomTimeZone());
+                        query.timeZone(randomDateTimeZone().getID());
                     }
                     if (randomBoolean()) {
                         query.format("yyyy-MM-dd'T'HH:mm:ss.SSSZZ");
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/DateProcessorFactoryTests.java b/core/src/test/java/org/elasticsearch/ingest/processor/DateProcessorFactoryTests.java
index 1eecb1397d3..68b5d175509 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/DateProcessorFactoryTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/processor/DateProcessorFactoryTests.java
@@ -120,7 +120,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("field", sourceField);
         config.put("formats", Collections.singletonList("dd/MM/yyyyy"));
 
-        DateTimeZone timezone = randomTimezone();
+        DateTimeZone timezone = randomDateTimeZone();
         config.put("timezone", timezone.getID());
         DateProcessor processor = factory.create(config);
         assertThat(processor.getTimezone(), equalTo(timezone));
@@ -141,14 +141,6 @@ public class DateProcessorFactoryTests extends ESTestCase {
         }
     }
 
-    //we generate a timezone out of the available ones in joda, some available in the jdk are not available in joda by default
-    private static DateTimeZone randomTimezone() {
-        List<String> ids = new ArrayList<>(DateTimeZone.getAvailableIDs());
-        Collections.sort(ids);
-        return DateTimeZone.forID(randomFrom(ids));
-    }
-
-
     public void testParseMatchFormats() throws Exception {
         DateProcessor.Factory factory = new DateProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java
index a95b3cd4871..36613cfa784 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java
@@ -295,8 +295,7 @@ public class DateRangeIT extends ESIntegTestCase {
     }
 
     public void testSingleValueFieldWithDateMath() throws Exception {
-        String[] ids = DateTimeZone.getAvailableIDs().toArray(new String[DateTimeZone.getAvailableIDs().size()]);
-        DateTimeZone timezone = DateTimeZone.forID(randomFrom(ids));
+        DateTimeZone timezone = randomDateTimeZone();
         int timeZoneOffset = timezone.getOffset(date(2, 15));
         // if time zone is UTC (or equivalent), time zone suffix is "Z", else something like "+03:00", which we get with the "ZZ" format
         String feb15Suffix = timeZoneOffset == 0 ? "Z" : date(2,15, timezone).toString("ZZ");
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeTests.java
index eba3790330b..ee32915fdd7 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeTests.java
@@ -22,12 +22,9 @@ package org.elasticsearch.search.aggregations.bucket;
 import org.elasticsearch.search.aggregations.BaseAggregationTestCase;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range;
 import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeAggregationBuilder;
-import org.joda.time.DateTimeZone;
 
 public class DateRangeTests extends BaseAggregationTestCase<DateRangeAggregationBuilder> {
 
-    private final static String[] timeZoneIds = DateTimeZone.getAvailableIDs().toArray(new String[DateTimeZone.getAvailableIDs().size()]);
-
     @Override
     protected DateRangeAggregationBuilder createTestAggregatorBuilder() {
         int numRanges = randomIntBetween(1, 10);
@@ -60,7 +57,7 @@ public class DateRangeTests extends BaseAggregationTestCase<DateRangeAggregation
             factory.missing(randomIntBetween(0, 10));
         }
         if (randomBoolean()) {
-            factory.timeZone(DateTimeZone.forID(randomFrom(timeZoneIds)));
+            factory.timeZone(randomDateTimeZone());
         }
         return factory;
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index 247a340d8c4..918ecf0ce9a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.test;
 
 import com.carrotsearch.randomizedtesting.RandomizedContext;
-import com.carrotsearch.randomizedtesting.RandomizedTest;
 import com.carrotsearch.randomizedtesting.annotations.TestGroup;
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
@@ -124,7 +123,6 @@ import org.elasticsearch.test.store.MockFSIndexStore;
 import org.elasticsearch.test.transport.AssertingLocalTransport;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.hamcrest.Matchers;
-import org.joda.time.DateTimeZone;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -1826,23 +1824,6 @@ public abstract class ESIntegTestCase extends ESTestCase {
         return perTestRatio;
     }
 
-    /**
-     * Returns a random JODA Time Zone based on Java Time Zones
-     */
-    public static DateTimeZone randomDateTimeZone() {
-        DateTimeZone timeZone;
-
-        // It sounds like some Java Time Zones are unknown by JODA. For example: Asia/Riyadh88
-        // We need to fallback in that case to a known time zone
-        try {
-            timeZone = DateTimeZone.forTimeZone(RandomizedTest.randomTimeZone());
-        } catch (IllegalArgumentException e) {
-            timeZone = DateTimeZone.forOffsetHours(randomIntBetween(-12, 12));
-        }
-
-        return timeZone;
-    }
-
     /**
      * Returns path to a random directory that can be used to create a temporary file system repo
      */
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 3dbbf25e202..67d93c6887a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -61,6 +61,7 @@ import org.elasticsearch.search.MockSearchService;
 import org.elasticsearch.test.junit.listeners.LoggingListener;
 import org.elasticsearch.test.junit.listeners.ReproduceInfoPrinter;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.joda.time.DateTimeZone;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -398,6 +399,15 @@ public abstract class ESTestCase extends LuceneTestCase {
         return randomTimeValue(1, 1000);
     }
 
+    /**
+     * generate a random DateTimeZone from the ones available in joda library
+     */
+    public static DateTimeZone randomDateTimeZone() {
+        List<String> ids = new ArrayList<>(DateTimeZone.getAvailableIDs());
+        Collections.sort(ids);
+        return DateTimeZone.forID(randomFrom(ids));
+    }
+
     /**
      * helper to randomly perform on <code>consumer</code> with <code>value</code>
      */

From 848648862736843c219de1036114ddf4a911e6cc Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Thu, 19 May 2016 18:44:53 +0200
Subject: [PATCH 0354/1311] Disable DeleteByQueryRestIT
 delete_by_query/10_basic/Basic delete_by_query

Because of a REST test namespace conflict introduced by 18329. Issue tracked in 18469
---
 .../rest-api-spec/test/delete_by_query/10_basic.yaml          | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/plugins/delete-by-query/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml b/plugins/delete-by-query/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml
index 063e959a807..124122a44bf 100644
--- a/plugins/delete-by-query/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml
+++ b/plugins/delete-by-query/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml
@@ -25,6 +25,10 @@ setup:
 
 ---
 "Basic delete_by_query":
+  - skip:
+      version: all
+      reason: "Test muted because of a REST test namespace conflict, see https://github.com/elastic/elasticsearch/issues/18469"
+
   - do:
       delete_by_query:
         index:  test_1

From dc33a8323188416df4e85d0ab762a750a482611b Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Thu, 19 May 2016 19:37:13 +0200
Subject: [PATCH 0355/1311] Remove the preserve_original option from the
 FingerprintAnalyzer (#18471)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The preserve_original option to the ASCIIFoldingFilter doesn't
play well with the FingerprintFilter, as it ends up producing
fingerprints like:

    "and consistent godel gödel is said sentence this yes"

The goal of the OpenRefine algorithm is to product a small normalized
ASCII fingerprint. There's no need to expose preserve_original.
---
 .../index/analysis/FingerprintAnalyzer.java   |  6 ++----
 .../analysis/FingerprintAnalyzerProvider.java |  5 +----
 .../analysis/FingerprintAnalyzerTests.java    | 20 ++++---------------
 3 files changed, 7 insertions(+), 24 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/analysis/FingerprintAnalyzer.java b/core/src/main/java/org/elasticsearch/index/analysis/FingerprintAnalyzer.java
index f7bf44256cc..985a081ccc8 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/FingerprintAnalyzer.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/FingerprintAnalyzer.java
@@ -33,13 +33,11 @@ import org.apache.lucene.analysis.util.CharArraySet;
 public final class FingerprintAnalyzer extends Analyzer {
     private final char separator;
     private final int maxOutputSize;
-    private final boolean preserveOriginal;
     private final CharArraySet stopWords;
 
-    public FingerprintAnalyzer(CharArraySet stopWords, char separator, int maxOutputSize, boolean preserveOriginal) {
+    public FingerprintAnalyzer(CharArraySet stopWords, char separator, int maxOutputSize) {
         this.separator = separator;
         this.maxOutputSize = maxOutputSize;
-        this.preserveOriginal = preserveOriginal;
         this.stopWords = stopWords;
     }
 
@@ -48,7 +46,7 @@ public final class FingerprintAnalyzer extends Analyzer {
         final Tokenizer tokenizer = new StandardTokenizer();
         TokenStream stream = tokenizer;
         stream = new LowerCaseFilter(stream);
-        stream = new ASCIIFoldingFilter(stream, preserveOriginal);
+        stream = new ASCIIFoldingFilter(stream, false);
         stream = new StopFilter(stream, stopWords);
         stream = new FingerprintFilter(stream, maxOutputSize, separator);
         return new TokenStreamComponents(tokenizer, stream);
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/FingerprintAnalyzerProvider.java b/core/src/main/java/org/elasticsearch/index/analysis/FingerprintAnalyzerProvider.java
index 897068cbf8b..bb8a51e0969 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/FingerprintAnalyzerProvider.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/FingerprintAnalyzerProvider.java
@@ -34,10 +34,8 @@ import org.elasticsearch.index.IndexSettings;
 public class FingerprintAnalyzerProvider extends AbstractIndexAnalyzerProvider<Analyzer> {
 
     public static ParseField MAX_OUTPUT_SIZE = FingerprintTokenFilterFactory.MAX_OUTPUT_SIZE;
-    public static ParseField PRESERVE_ORIGINAL = ASCIIFoldingTokenFilterFactory.PRESERVE_ORIGINAL;
 
     public static int DEFAULT_MAX_OUTPUT_SIZE = FingerprintTokenFilterFactory.DEFAULT_MAX_OUTPUT_SIZE;
-    public static boolean DEFAULT_PRESERVE_ORIGINAL = ASCIIFoldingTokenFilterFactory.DEFAULT_PRESERVE_ORIGINAL;
     public static CharArraySet DEFAULT_STOP_WORDS = CharArraySet.EMPTY_SET;
 
     private final FingerprintAnalyzer analyzer;
@@ -47,10 +45,9 @@ public class FingerprintAnalyzerProvider extends AbstractIndexAnalyzerProvider<A
 
         char separator = FingerprintTokenFilterFactory.parseSeparator(settings);
         int maxOutputSize = settings.getAsInt(MAX_OUTPUT_SIZE.getPreferredName(),DEFAULT_MAX_OUTPUT_SIZE);
-        boolean preserveOriginal = settings.getAsBoolean(PRESERVE_ORIGINAL.getPreferredName(), DEFAULT_PRESERVE_ORIGINAL);
         CharArraySet stopWords = Analysis.parseStopWords(env, settings, DEFAULT_STOP_WORDS);
 
-        this.analyzer = new FingerprintAnalyzer(stopWords, separator, maxOutputSize, preserveOriginal);
+        this.analyzer = new FingerprintAnalyzer(stopWords, separator, maxOutputSize);
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/FingerprintAnalyzerTests.java b/core/src/test/java/org/elasticsearch/index/analysis/FingerprintAnalyzerTests.java
index 8c1d530e448..ac412207021 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/FingerprintAnalyzerTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/FingerprintAnalyzerTests.java
@@ -26,13 +26,13 @@ import org.elasticsearch.test.ESTokenStreamTestCase;
 public class FingerprintAnalyzerTests extends ESTokenStreamTestCase {
 
     public void testFingerprint() throws Exception {
-        Analyzer a = new FingerprintAnalyzer(CharArraySet.EMPTY_SET, ' ', 255, false);
+        Analyzer a = new FingerprintAnalyzer(CharArraySet.EMPTY_SET, ' ', 255);
         assertAnalyzesTo(a, "foo bar@baz Baz $ foo foo FOO. FoO",
             new String[]{"bar baz foo"});
     }
 
     public void testReusableTokenStream() throws Exception {
-        Analyzer a = new FingerprintAnalyzer(CharArraySet.EMPTY_SET, ' ', 255, false);
+        Analyzer a = new FingerprintAnalyzer(CharArraySet.EMPTY_SET, ' ', 255);
         assertAnalyzesTo(a, "foo bar baz Baz foo foo FOO. FoO",
             new String[]{"bar baz foo"});
         assertAnalyzesTo(a, "xyz XYZ abc 123.2 abc",
@@ -40,7 +40,7 @@ public class FingerprintAnalyzerTests extends ESTokenStreamTestCase {
     }
 
     public void testAsciifolding() throws Exception {
-        Analyzer a = new FingerprintAnalyzer(CharArraySet.EMPTY_SET, ' ', 255, false);
+        Analyzer a = new FingerprintAnalyzer(CharArraySet.EMPTY_SET, ' ', 255);
         assertAnalyzesTo(a, "gödel escher bach",
             new String[]{"bach escher godel"});
 
@@ -48,14 +48,8 @@ public class FingerprintAnalyzerTests extends ESTokenStreamTestCase {
             new String[]{"bach escher godel"});
     }
 
-    public void testPreserveOriginal() throws Exception {
-        Analyzer a = new FingerprintAnalyzer(CharArraySet.EMPTY_SET, ' ', 255, true);
-        assertAnalyzesTo(a, "gödel escher bach",
-            new String[]{"bach escher godel gödel"});
-    }
-
     public void testLimit() throws Exception {
-        Analyzer a = new FingerprintAnalyzer(CharArraySet.EMPTY_SET, ' ', 3, false);
+        Analyzer a = new FingerprintAnalyzer(CharArraySet.EMPTY_SET, ' ', 3);
         assertAnalyzesTo(a, "e d c b a",
             new String[]{});
 
@@ -63,10 +57,4 @@ public class FingerprintAnalyzerTests extends ESTokenStreamTestCase {
             new String[]{"a b"});
     }
 
-    public void testSeparator() throws Exception {
-        Analyzer a = new FingerprintAnalyzer(CharArraySet.EMPTY_SET, '_', 255, true);
-        assertAnalyzesTo(a, "b c a",
-            new String[]{"a_b_c"});
-    }
-
 }

From 5da9e5dcbcd7676e09060ddb0a64d4328d959a64 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Thu, 19 May 2016 19:42:23 +0200
Subject: [PATCH 0356/1311] Docs: Improved tokenizer docs (#18356)

* Docs: Improved tokenizer docs

Added descriptions and runnable examples

* Addressed Nik's comments

* Added TESTRESPONSEs for all tokenizer examples

* Added TESTRESPONSEs for all analyzer examples too

* Added docs, examples, and TESTRESPONSES for character filters

* Skipping two tests:

One interprets "$1" as a stack variable - same problem exists with the REST tests

The other because the "took" value is always different

* Fixed tests with "took"

* Fixed failing tests and removed preserve_original from fingerprint analyzer
---
 .../analysis/analyzers/configuring.asciidoc   |  35 ++
 .../analyzers/custom-analyzer.asciidoc        |  88 ++++-
 .../analyzers/fingerprint-analyzer.asciidoc   |  59 ++-
 .../analyzers/keyword-analyzer.asciidoc       |  21 ++
 .../analyzers/pattern-analyzer.asciidoc       | 203 ++++++++++
 .../analyzers/simple-analyzer.asciidoc        |  91 +++++
 .../analyzers/standard-analyzer.asciidoc      | 174 +++++++++
 .../analysis/analyzers/stop-analyzer.asciidoc | 147 ++++++++
 .../analyzers/whitespace-analyzer.asciidoc    |  84 +++++
 docs/reference/analysis/charfilters.asciidoc  |  34 +-
 .../charfilters/htmlstrip-charfilter.asciidoc | 134 ++++++-
 .../charfilters/mapping-charfilter.asciidoc   | 210 +++++++++--
 .../pattern-replace-charfilter.asciidoc       | 266 +++++++++++--
 docs/reference/analysis/tokenizers.asciidoc   | 132 ++++++-
 .../tokenizers/classic-tokenizer.asciidoc     | 276 +++++++++++++-
 .../tokenizers/edgengram-tokenizer.asciidoc   | 355 +++++++++++++++---
 .../tokenizers/keyword-tokenizer.asciidoc     |  63 +++-
 .../tokenizers/letter-tokenizer.asciidoc      | 124 +++++-
 .../tokenizers/lowercase-tokenizer.asciidoc   | 137 ++++++-
 .../tokenizers/ngram-tokenizer.asciidoc       | 345 ++++++++++++++---
 .../pathhierarchy-tokenizer.asciidoc          | 183 ++++++++-
 .../tokenizers/pattern-tokenizer.asciidoc     | 288 ++++++++++++--
 .../tokenizers/standard-tokenizer.asciidoc    | 280 +++++++++++++-
 .../tokenizers/thai-tokenizer.asciidoc        | 107 +++++-
 .../tokenizers/uaxurlemail-tokenizer.asciidoc | 205 +++++++++-
 .../tokenizers/whitespace-tokenizer.asciidoc  | 114 +++++-
 26 files changed, 3844 insertions(+), 311 deletions(-)

diff --git a/docs/reference/analysis/analyzers/configuring.asciidoc b/docs/reference/analysis/analyzers/configuring.asciidoc
index c93d800afb9..2ce13702e00 100644
--- a/docs/reference/analysis/analyzers/configuring.asciidoc
+++ b/docs/reference/analysis/analyzers/configuring.asciidoc
@@ -64,3 +64,38 @@ POST my_index/_analyze
     English stop words will be removed.  The resulting terms are:
     `[ old, brown, cow ]`
 
+
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "old",
+      "start_offset": 4,
+      "end_offset": 7,
+      "type": "<ALPHANUM>",
+      "position": 1
+    },
+    {
+      "token": "brown",
+      "start_offset": 8,
+      "end_offset": 13,
+      "type": "<ALPHANUM>",
+      "position": 2
+    },
+    {
+      "token": "cow",
+      "start_offset": 14,
+      "end_offset": 17,
+      "type": "<ALPHANUM>",
+      "position": 3
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
diff --git a/docs/reference/analysis/analyzers/custom-analyzer.asciidoc b/docs/reference/analysis/analyzers/custom-analyzer.asciidoc
index eccd16c23be..1707a9a399b 100644
--- a/docs/reference/analysis/analyzers/custom-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/custom-analyzer.asciidoc
@@ -84,6 +84,48 @@ POST my_index/_analyze
 --------------------------------
 // CONSOLE
 
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "is",
+      "start_offset": 0,
+      "end_offset": 2,
+      "type": "<ALPHANUM>",
+      "position": 0
+    },
+    {
+      "token": "this",
+      "start_offset": 3,
+      "end_offset": 7,
+      "type": "<ALPHANUM>",
+      "position": 1
+    },
+    {
+      "token": "deja",
+      "start_offset": 11,
+      "end_offset": 15,
+      "type": "<ALPHANUM>",
+      "position": 2
+    },
+    {
+      "token": "vu",
+      "start_offset": 16,
+      "end_offset": 22,
+      "type": "<ALPHANUM>",
+      "position": 3
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
 The above example produces the following terms:
 
 [source,text]
@@ -119,13 +161,10 @@ PUT my_index
       "analyzer": {
         "my_custom_analyzer": {
           "type": "custom",
-
           "char_filter": [
             "emoticons" <1>
           ],
-
           "tokenizer": "punctuation", <1>
-
           "filter": [
             "lowercase",
             "english_stop" <1>
@@ -165,11 +204,54 @@ POST my_index/_analyze
   "text":     "I'm a :) person, and you?"
 }
 --------------------------------------------------
+// CONSOLE
 
 <1> The `emoticon` character filter, `punctuation` tokenizer and
     `english_stop` token filter are custom implementations which are defined
     in the same index settings.
 
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "i'm",
+      "start_offset": 0,
+      "end_offset": 3,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "_happy_",
+      "start_offset": 6,
+      "end_offset": 8,
+      "type": "word",
+      "position": 2
+    },
+    {
+      "token": "person",
+      "start_offset": 9,
+      "end_offset": 15,
+      "type": "word",
+      "position": 3
+    },
+    {
+      "token": "you",
+      "start_offset": 21,
+      "end_offset": 24,
+      "type": "word",
+      "position": 5
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
 The above example produces the following terms:
 
 [source,text]
diff --git a/docs/reference/analysis/analyzers/fingerprint-analyzer.asciidoc b/docs/reference/analysis/analyzers/fingerprint-analyzer.asciidoc
index b393c883441..24dc92380bb 100644
--- a/docs/reference/analysis/analyzers/fingerprint-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/fingerprint-analyzer.asciidoc
@@ -36,6 +36,27 @@ POST _analyze
 ---------------------------
 // CONSOLE
 
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "and consistent godel is said sentence this yes",
+      "start_offset": 0,
+      "end_offset": 52,
+      "type": "fingerprint",
+      "position": 0
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
 The above sentence would produce the following single term:
 
 [source,text]
@@ -58,16 +79,11 @@ The `fingerprint` analyzer accepts the following parameters:
     The maximum token size to emit.  Defaults to `255`. Tokens larger than
     this size will be discarded.
 
-`preserve_original`::
-
-    If `true`, emits two tokens: one with ASCII-folding of terms that contain
-    extended characters (if any) and one with the original characters.
-    Defaults to `false`.
-
 `stopwords`::
 
     A pre-defined stop words list like `_english_` or an array  containing a
     list of stop words.  Defaults to `_none_`.
+
 `stopwords_path`::
 
     The path to a file containing stop words.
@@ -80,8 +96,7 @@ about stop word configuration.
 === Example configuration
 
 In this example, we configure the `fingerprint` analyzer to use the
-pre-defined list of English stop words, and to emit a second token in
-the presence of non-ASCII characters:
+pre-defined list of English stop words:
 
 [source,js]
 ----------------------------
@@ -92,8 +107,7 @@ PUT my_index
       "analyzer": {
         "my_fingerprint_analyzer": {
           "type": "fingerprint",
-          "stopwords": "_english_",
-          "preserve_original": true
+          "stopwords": "_english_"
         }
       }
     }
@@ -110,9 +124,30 @@ POST my_index/_analyze
 ----------------------------
 // CONSOLE
 
-The above example produces the following two terms:
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "consistent godel said sentence yes",
+      "start_offset": 0,
+      "end_offset": 52,
+      "type": "fingerprint",
+      "position": 0
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
+The above example produces the following term:
 
 [source,text]
 ---------------------------
-[ consistent godel said sentence yes, consistent gödel said sentence yes ]
+[ consistent godel said sentence yes ]
 ---------------------------
diff --git a/docs/reference/analysis/analyzers/keyword-analyzer.asciidoc b/docs/reference/analysis/analyzers/keyword-analyzer.asciidoc
index a0c1b1b0a6a..cc94f3b757e 100644
--- a/docs/reference/analysis/analyzers/keyword-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/keyword-analyzer.asciidoc
@@ -25,6 +25,27 @@ POST _analyze
 ---------------------------
 // CONSOLE
 
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone.",
+      "start_offset": 0,
+      "end_offset": 56,
+      "type": "word",
+      "position": 0
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
 The above sentence would produce the following single term:
 
 [source,text]
diff --git a/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc b/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc
index 6a4ca274416..2d5741c2b9e 100644
--- a/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc
@@ -30,6 +30,104 @@ POST _analyze
 ---------------------------
 // CONSOLE
 
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "the",
+      "start_offset": 0,
+      "end_offset": 3,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "2",
+      "start_offset": 4,
+      "end_offset": 5,
+      "type": "word",
+      "position": 1
+    },
+    {
+      "token": "quick",
+      "start_offset": 6,
+      "end_offset": 11,
+      "type": "word",
+      "position": 2
+    },
+    {
+      "token": "brown",
+      "start_offset": 12,
+      "end_offset": 17,
+      "type": "word",
+      "position": 3
+    },
+    {
+      "token": "foxes",
+      "start_offset": 18,
+      "end_offset": 23,
+      "type": "word",
+      "position": 4
+    },
+    {
+      "token": "jumped",
+      "start_offset": 24,
+      "end_offset": 30,
+      "type": "word",
+      "position": 5
+    },
+    {
+      "token": "over",
+      "start_offset": 31,
+      "end_offset": 35,
+      "type": "word",
+      "position": 6
+    },
+    {
+      "token": "the",
+      "start_offset": 36,
+      "end_offset": 39,
+      "type": "word",
+      "position": 7
+    },
+    {
+      "token": "lazy",
+      "start_offset": 40,
+      "end_offset": 44,
+      "type": "word",
+      "position": 8
+    },
+    {
+      "token": "dog",
+      "start_offset": 45,
+      "end_offset": 48,
+      "type": "word",
+      "position": 9
+    },
+    {
+      "token": "s",
+      "start_offset": 49,
+      "end_offset": 50,
+      "type": "word",
+      "position": 10
+    },
+    {
+      "token": "bone",
+      "start_offset": 51,
+      "end_offset": 55,
+      "type": "word",
+      "position": 11
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
 The above sentence would produce the following terms:
 
 [source,text]
@@ -110,6 +208,55 @@ POST my_index/_analyze
 <1> The backslashes in the pattern need to be escaped when specifying the
     pattern as a JSON string.
 
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "john",
+      "start_offset": 0,
+      "end_offset": 4,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "smith",
+      "start_offset": 5,
+      "end_offset": 10,
+      "type": "word",
+      "position": 1
+    },
+    {
+      "token": "foo",
+      "start_offset": 11,
+      "end_offset": 14,
+      "type": "word",
+      "position": 2
+    },
+    {
+      "token": "bar",
+      "start_offset": 15,
+      "end_offset": 18,
+      "type": "word",
+      "position": 3
+    },
+    {
+      "token": "com",
+      "start_offset": 19,
+      "end_offset": 22,
+      "type": "word",
+      "position": 4
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
 The above example produces the following terms:
 
 [source,text]
@@ -148,6 +295,62 @@ GET my_index/_analyze
 --------------------------------------------------
 // CONSOLE
 
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "moose",
+      "start_offset": 0,
+      "end_offset": 5,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "x",
+      "start_offset": 5,
+      "end_offset": 6,
+      "type": "word",
+      "position": 1
+    },
+    {
+      "token": "ftp",
+      "start_offset": 8,
+      "end_offset": 11,
+      "type": "word",
+      "position": 2
+    },
+    {
+      "token": "class",
+      "start_offset": 11,
+      "end_offset": 16,
+      "type": "word",
+      "position": 3
+    },
+    {
+      "token": "2",
+      "start_offset": 16,
+      "end_offset": 17,
+      "type": "word",
+      "position": 4
+    },
+    {
+      "token": "beta",
+      "start_offset": 18,
+      "end_offset": 22,
+      "type": "word",
+      "position": 5
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
 The above example produces the following terms:
 
 [source,text]
diff --git a/docs/reference/analysis/analyzers/simple-analyzer.asciidoc b/docs/reference/analysis/analyzers/simple-analyzer.asciidoc
index 4c932bb5d3e..a57c30d8dd6 100644
--- a/docs/reference/analysis/analyzers/simple-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/simple-analyzer.asciidoc
@@ -25,6 +25,97 @@ POST _analyze
 ---------------------------
 // CONSOLE
 
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "the",
+      "start_offset": 0,
+      "end_offset": 3,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "quick",
+      "start_offset": 6,
+      "end_offset": 11,
+      "type": "word",
+      "position": 1
+    },
+    {
+      "token": "brown",
+      "start_offset": 12,
+      "end_offset": 17,
+      "type": "word",
+      "position": 2
+    },
+    {
+      "token": "foxes",
+      "start_offset": 18,
+      "end_offset": 23,
+      "type": "word",
+      "position": 3
+    },
+    {
+      "token": "jumped",
+      "start_offset": 24,
+      "end_offset": 30,
+      "type": "word",
+      "position": 4
+    },
+    {
+      "token": "over",
+      "start_offset": 31,
+      "end_offset": 35,
+      "type": "word",
+      "position": 5
+    },
+    {
+      "token": "the",
+      "start_offset": 36,
+      "end_offset": 39,
+      "type": "word",
+      "position": 6
+    },
+    {
+      "token": "lazy",
+      "start_offset": 40,
+      "end_offset": 44,
+      "type": "word",
+      "position": 7
+    },
+    {
+      "token": "dog",
+      "start_offset": 45,
+      "end_offset": 48,
+      "type": "word",
+      "position": 8
+    },
+    {
+      "token": "s",
+      "start_offset": 49,
+      "end_offset": 50,
+      "type": "word",
+      "position": 9
+    },
+    {
+      "token": "bone",
+      "start_offset": 51,
+      "end_offset": 55,
+      "type": "word",
+      "position": 10
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
 The above sentence would produce the following terms:
 
 [source,text]
diff --git a/docs/reference/analysis/analyzers/standard-analyzer.asciidoc b/docs/reference/analysis/analyzers/standard-analyzer.asciidoc
index 72292e1d40d..3b948892483 100644
--- a/docs/reference/analysis/analyzers/standard-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/standard-analyzer.asciidoc
@@ -33,6 +33,97 @@ POST _analyze
 ---------------------------
 // CONSOLE
 
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "the",
+      "start_offset": 0,
+      "end_offset": 3,
+      "type": "<ALPHANUM>",
+      "position": 0
+    },
+    {
+      "token": "2",
+      "start_offset": 4,
+      "end_offset": 5,
+      "type": "<NUM>",
+      "position": 1
+    },
+    {
+      "token": "quick",
+      "start_offset": 6,
+      "end_offset": 11,
+      "type": "<ALPHANUM>",
+      "position": 2
+    },
+    {
+      "token": "brown",
+      "start_offset": 12,
+      "end_offset": 17,
+      "type": "<ALPHANUM>",
+      "position": 3
+    },
+    {
+      "token": "foxes",
+      "start_offset": 18,
+      "end_offset": 23,
+      "type": "<ALPHANUM>",
+      "position": 4
+    },
+    {
+      "token": "jumped",
+      "start_offset": 24,
+      "end_offset": 30,
+      "type": "<ALPHANUM>",
+      "position": 5
+    },
+    {
+      "token": "over",
+      "start_offset": 31,
+      "end_offset": 35,
+      "type": "<ALPHANUM>",
+      "position": 6
+    },
+    {
+      "token": "the",
+      "start_offset": 36,
+      "end_offset": 39,
+      "type": "<ALPHANUM>",
+      "position": 7
+    },
+    {
+      "token": "lazy",
+      "start_offset": 40,
+      "end_offset": 44,
+      "type": "<ALPHANUM>",
+      "position": 8
+    },
+    {
+      "token": "dog's",
+      "start_offset": 45,
+      "end_offset": 50,
+      "type": "<ALPHANUM>",
+      "position": 9
+    },
+    {
+      "token": "bone",
+      "start_offset": 51,
+      "end_offset": 55,
+      "type": "<ALPHANUM>",
+      "position": 10
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
 The above sentence would produce the following terms:
 
 [source,text]
@@ -98,6 +189,89 @@ POST my_index/_analyze
 ----------------------------
 // CONSOLE
 
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "2",
+      "start_offset": 4,
+      "end_offset": 5,
+      "type": "<NUM>",
+      "position": 1
+    },
+    {
+      "token": "quick",
+      "start_offset": 6,
+      "end_offset": 11,
+      "type": "<ALPHANUM>",
+      "position": 2
+    },
+    {
+      "token": "brown",
+      "start_offset": 12,
+      "end_offset": 17,
+      "type": "<ALPHANUM>",
+      "position": 3
+    },
+    {
+      "token": "foxes",
+      "start_offset": 18,
+      "end_offset": 23,
+      "type": "<ALPHANUM>",
+      "position": 4
+    },
+    {
+      "token": "jumpe",
+      "start_offset": 24,
+      "end_offset": 29,
+      "type": "<ALPHANUM>",
+      "position": 5
+    },
+    {
+      "token": "d",
+      "start_offset": 29,
+      "end_offset": 30,
+      "type": "<ALPHANUM>",
+      "position": 6
+    },
+    {
+      "token": "over",
+      "start_offset": 31,
+      "end_offset": 35,
+      "type": "<ALPHANUM>",
+      "position": 7
+    },
+    {
+      "token": "lazy",
+      "start_offset": 40,
+      "end_offset": 44,
+      "type": "<ALPHANUM>",
+      "position": 9
+    },
+    {
+      "token": "dog's",
+      "start_offset": 45,
+      "end_offset": 50,
+      "type": "<ALPHANUM>",
+      "position": 10
+    },
+    {
+      "token": "bone",
+      "start_offset": 51,
+      "end_offset": 55,
+      "type": "<ALPHANUM>",
+      "position": 11
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
 The above example produces the following terms:
 
 [source,text]
diff --git a/docs/reference/analysis/analyzers/stop-analyzer.asciidoc b/docs/reference/analysis/analyzers/stop-analyzer.asciidoc
index ada9022a287..e40436342d7 100644
--- a/docs/reference/analysis/analyzers/stop-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/stop-analyzer.asciidoc
@@ -29,6 +29,83 @@ POST _analyze
 ---------------------------
 // CONSOLE
 
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "quick",
+      "start_offset": 6,
+      "end_offset": 11,
+      "type": "word",
+      "position": 1
+    },
+    {
+      "token": "brown",
+      "start_offset": 12,
+      "end_offset": 17,
+      "type": "word",
+      "position": 2
+    },
+    {
+      "token": "foxes",
+      "start_offset": 18,
+      "end_offset": 23,
+      "type": "word",
+      "position": 3
+    },
+    {
+      "token": "jumped",
+      "start_offset": 24,
+      "end_offset": 30,
+      "type": "word",
+      "position": 4
+    },
+    {
+      "token": "over",
+      "start_offset": 31,
+      "end_offset": 35,
+      "type": "word",
+      "position": 5
+    },
+    {
+      "token": "lazy",
+      "start_offset": 40,
+      "end_offset": 44,
+      "type": "word",
+      "position": 7
+    },
+    {
+      "token": "dog",
+      "start_offset": 45,
+      "end_offset": 48,
+      "type": "word",
+      "position": 8
+    },
+    {
+      "token": "s",
+      "start_offset": 49,
+      "end_offset": 50,
+      "type": "word",
+      "position": 9
+    },
+    {
+      "token": "bone",
+      "start_offset": 51,
+      "end_offset": 55,
+      "type": "word",
+      "position": 10
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
 The above sentence would produce the following terms:
 
 [source,text]
@@ -87,6 +164,76 @@ POST my_index/_analyze
 ----------------------------
 // CONSOLE
 
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "quick",
+      "start_offset": 6,
+      "end_offset": 11,
+      "type": "word",
+      "position": 1
+    },
+    {
+      "token": "brown",
+      "start_offset": 12,
+      "end_offset": 17,
+      "type": "word",
+      "position": 2
+    },
+    {
+      "token": "foxes",
+      "start_offset": 18,
+      "end_offset": 23,
+      "type": "word",
+      "position": 3
+    },
+    {
+      "token": "jumped",
+      "start_offset": 24,
+      "end_offset": 30,
+      "type": "word",
+      "position": 4
+    },
+    {
+      "token": "lazy",
+      "start_offset": 40,
+      "end_offset": 44,
+      "type": "word",
+      "position": 7
+    },
+    {
+      "token": "dog",
+      "start_offset": 45,
+      "end_offset": 48,
+      "type": "word",
+      "position": 8
+    },
+    {
+      "token": "s",
+      "start_offset": 49,
+      "end_offset": 50,
+      "type": "word",
+      "position": 9
+    },
+    {
+      "token": "bone",
+      "start_offset": 51,
+      "end_offset": 55,
+      "type": "word",
+      "position": 10
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
 The above example produces the following terms:
 
 [source,text]
diff --git a/docs/reference/analysis/analyzers/whitespace-analyzer.asciidoc b/docs/reference/analysis/analyzers/whitespace-analyzer.asciidoc
index 0dce8db1c99..f95e5c6e4ab 100644
--- a/docs/reference/analysis/analyzers/whitespace-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/whitespace-analyzer.asciidoc
@@ -25,6 +25,90 @@ POST _analyze
 ---------------------------
 // CONSOLE
 
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "The",
+      "start_offset": 0,
+      "end_offset": 3,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "2",
+      "start_offset": 4,
+      "end_offset": 5,
+      "type": "word",
+      "position": 1
+    },
+    {
+      "token": "QUICK",
+      "start_offset": 6,
+      "end_offset": 11,
+      "type": "word",
+      "position": 2
+    },
+    {
+      "token": "Brown-Foxes",
+      "start_offset": 12,
+      "end_offset": 23,
+      "type": "word",
+      "position": 3
+    },
+    {
+      "token": "jumped",
+      "start_offset": 24,
+      "end_offset": 30,
+      "type": "word",
+      "position": 4
+    },
+    {
+      "token": "over",
+      "start_offset": 31,
+      "end_offset": 35,
+      "type": "word",
+      "position": 5
+    },
+    {
+      "token": "the",
+      "start_offset": 36,
+      "end_offset": 39,
+      "type": "word",
+      "position": 6
+    },
+    {
+      "token": "lazy",
+      "start_offset": 40,
+      "end_offset": 44,
+      "type": "word",
+      "position": 7
+    },
+    {
+      "token": "dog's",
+      "start_offset": 45,
+      "end_offset": 50,
+      "type": "word",
+      "position": 8
+    },
+    {
+      "token": "bone.",
+      "start_offset": 51,
+      "end_offset": 56,
+      "type": "word",
+      "position": 9
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
 The above sentence would produce the following terms:
 
 [source,text]
diff --git a/docs/reference/analysis/charfilters.asciidoc b/docs/reference/analysis/charfilters.asciidoc
index c9f5805284c..cd24f5bf571 100644
--- a/docs/reference/analysis/charfilters.asciidoc
+++ b/docs/reference/analysis/charfilters.asciidoc
@@ -1,16 +1,36 @@
 [[analysis-charfilters]]
 == Character Filters
 
-Character filters are used to preprocess the string of 
-characters before it is passed to the <<analysis-tokenizers,tokenizer>>.
-A character filter may be used to strip out HTML markup, or to convert
-`"&"` characters to the word `"and"`.    
+_Character filters_ are used to preprocess the stream of characters before it
+is passed to the <<analysis-tokenizers,tokenizer>>.
 
-Elasticsearch has built in characters filters which can be
-used to build <<analysis-custom-analyzer,custom analyzers>>.
+A character filter receives the original text as a stream of characters and
+can transform the stream by adding, removing, or changing characters.  For
+instance, a character filter could be used to convert Arabic numerals
+(٠‎١٢٣٤٥٦٧٨‎٩‎) into their Latin equivalents (0123456789), or to strip HTML
+elements like `<b>` from the stream.
 
-include::charfilters/mapping-charfilter.asciidoc[]
+
+Elasticsearch has a number of built in character filters which can be used to build
+<<analysis-custom-analyzer,custom analyzers>>.
+
+<<analysis-htmlstrip-charfilter,HTML Strip Character Filter>>::
+
+The `html_strip` character filter strips out HTML elements like `<b>` and
+decodes HTML entities like `&amp;`.
+
+<<analysis-mapping-charfilter,Mapping Character Filter>>::
+
+The `mapping` character filter replaces any occurrences of the specified
+strings with the specified replacements.
+
+<<analysis-pattern-replace-charfilter,Pattern Replace Character Filter>>::
+
+The `pattern_replace` character filter replaces any characters matching a
+regular expression with the specified replacement.
 
 include::charfilters/htmlstrip-charfilter.asciidoc[]
 
+include::charfilters/mapping-charfilter.asciidoc[]
+
 include::charfilters/pattern-replace-charfilter.asciidoc[]
diff --git a/docs/reference/analysis/charfilters/htmlstrip-charfilter.asciidoc b/docs/reference/analysis/charfilters/htmlstrip-charfilter.asciidoc
index f12238a36ad..3d8b187d772 100644
--- a/docs/reference/analysis/charfilters/htmlstrip-charfilter.asciidoc
+++ b/docs/reference/analysis/charfilters/htmlstrip-charfilter.asciidoc
@@ -1,5 +1,135 @@
 [[analysis-htmlstrip-charfilter]]
 === HTML Strip Char Filter
 
-A char filter of type `html_strip` stripping out HTML elements from an
-analyzed text.
+The `html_strip` character filter strips HTML elements from the text and
+replaces HTML entities with their decoded value (e.g. replacing `&amp;` with
+`&`).
+
+[float]
+=== Example output
+
+[source,js]
+---------------------------
+POST _analyze
+{
+  "tokenizer":      "keyword", <1>
+  "char_filter":  [ "html_strip" ],
+  "text": "<p>I&apos;m so <b>happy</b>!</p>"
+}
+---------------------------
+// CONSOLE
+<1> The <<analysis-keyword-tokenizer,`keyword` tokenizer>> returns a single term.
+
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "\nI'm so happy!\n",
+      "start_offset": 0,
+      "end_offset": 32,
+      "type": "word",
+      "position": 0
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
+The above example returns the term:
+
+[source,js]
+---------------------------
+[ \nI'm so happy!\n ]
+---------------------------
+
+The same example with the `standard` tokenizer would return the following terms:
+
+[source,js]
+---------------------------
+[ I'm, so, happy ]
+---------------------------
+
+[float]
+=== Configuration
+
+The `html_strip` character filter accepts the following parameter:
+
+[horizontal]
+`escaped_tags`::
+
+    An array of HTML tags which should not be stripped from the original text.
+
+[float]
+=== Example configuration
+
+In this example, we configure the `html_strip` character filter to leave `<b>`
+tags in place:
+
+[source,js]
+----------------------------
+PUT my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "keyword",
+          "char_filter": ["my_char_filter"]
+        }
+      },
+      "char_filter": {
+        "my_char_filter": {
+          "type": "html_strip",
+          "escaped_tags": ["b"]
+        }
+      }
+    }
+  }
+}
+
+GET _cluster/health?wait_for_status=yellow
+
+POST my_index/_analyze
+{
+  "analyzer": "my_analyzer",
+  "text": "<p>I&apos;m so <b>happy</b>!</p>"
+}
+----------------------------
+// CONSOLE
+
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "\nI'm so <b>happy</b>!\n",
+      "start_offset": 0,
+      "end_offset": 32,
+      "type": "word",
+      "position": 0
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
+The above example produces the following term:
+
+[source,text]
+---------------------------
+[ \nI'm so <b>happy</b>!\n ]
+---------------------------
+
+
+
diff --git a/docs/reference/analysis/charfilters/mapping-charfilter.asciidoc b/docs/reference/analysis/charfilters/mapping-charfilter.asciidoc
index 14c316dcac5..ed90e9f6ab6 100644
--- a/docs/reference/analysis/charfilters/mapping-charfilter.asciidoc
+++ b/docs/reference/analysis/charfilters/mapping-charfilter.asciidoc
@@ -1,42 +1,202 @@
 [[analysis-mapping-charfilter]]
 === Mapping Char Filter
 
-A char filter of type `mapping` replacing characters of an analyzed text
-with given mapping.
+The `mapping` character filter accepts a map of keys and values.  Whenever it
+encounters a string of characters that is the same as a key, it replaces them
+with the value associated with that key.
+
+Matching is greedy; the longest pattern matching at a given point wins.
+Replacements are allowed to be the empty string.
+
+[float]
+=== Configuration
+
+The `mapping` character filter accepts the following parameters:
 
 [horizontal]
 `mappings`::
 
-    A list of mappings to use.
+    A array of mappings, with each element having the form `key => value`.
 
 `mappings_path`::
 
-    A path, relative to the `config` directory, to a mappings file
-    configuration.
+    A path, either absolute or relative to the `config` directory, to a UTF-8
+    encoded text mappings file containing a `key => value` mapping per line.
 
-Here is a sample configuration:
+Either the `mappings` or `mappings_path` parameter must be provided.
+
+[float]
+=== Example configuration
+
+In this example, we configure the `mapping` character filter to replace Arabic
+numerals with their Latin equivalents:
 
 [source,js]
---------------------------------------------------
+----------------------------
+PUT my_index
 {
-    "index" : {
-        "analysis" : {
-            "char_filter" : {
-                "my_mapping" : {
-                    "type" : "mapping",
-                    "mappings" : [
-                      "ph => f",
-                      "qu => k"
-                    ]
-                }
-            },
-            "analyzer" : {
-                "custom_with_char_filter" : {
-                    "tokenizer" : "standard",
-                    "char_filter" : ["my_mapping"]
-                }
-            }
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "keyword",
+          "char_filter": [
+            "my_char_filter"
+          ]
         }
+      },
+      "char_filter": {
+        "my_char_filter": {
+          "type": "mapping",
+          "mappings": [
+            "Ù  => 0",
+            "Ù¡ => 1",
+            "Ù¢ => 2",
+            "Ù£ => 3",
+            "Ù¤ => 4",
+            "Ù¥ => 5",
+            "Ù¦ => 6",
+            "Ù§ => 7",
+            "Ù¨ => 8",
+            "Ù© => 9"
+          ]
+        }
+      }
     }
+  }
 }
---------------------------------------------------
+
+GET _cluster/health?wait_for_status=yellow
+
+POST my_index/_analyze
+{
+  "analyzer": "my_analyzer",
+  "text": "My license plate is ٢٥٠١٥"
+}
+----------------------------
+// CONSOLE
+
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "My license plate is 25015",
+      "start_offset": 0,
+      "end_offset": 25,
+      "type": "word",
+      "position": 0
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
+The above example produces the following term:
+
+[source,text]
+---------------------------
+[ My license plate is 25015 ]
+---------------------------
+
+Keys and values can be strings with multiple characters.  The following
+example replaces the `:)` and `:(` emoticons with a text equivalent:
+
+[source,js]
+----------------------------
+PUT my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "standard",
+          "char_filter": [
+            "my_char_filter"
+          ]
+        }
+      },
+      "char_filter": {
+        "my_char_filter": {
+          "type": "mapping",
+          "mappings": [
+            ":) => _happy_",
+            ":( => _sad_"
+          ]
+        }
+      }
+    }
+  }
+}
+
+GET _cluster/health?wait_for_status=yellow
+
+POST my_index/_analyze
+{
+  "analyzer": "my_analyzer",
+  "text": "I'm delighted about it :("
+}
+----------------------------
+// CONSOLE
+
+
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "I'm",
+      "start_offset": 0,
+      "end_offset": 3,
+      "type": "<ALPHANUM>",
+      "position": 0
+    },
+    {
+      "token": "delighted",
+      "start_offset": 4,
+      "end_offset": 13,
+      "type": "<ALPHANUM>",
+      "position": 1
+    },
+    {
+      "token": "about",
+      "start_offset": 14,
+      "end_offset": 19,
+      "type": "<ALPHANUM>",
+      "position": 2
+    },
+    {
+      "token": "it",
+      "start_offset": 20,
+      "end_offset": 22,
+      "type": "<ALPHANUM>",
+      "position": 3
+    },
+    {
+      "token": "_sad_",
+      "start_offset": 23,
+      "end_offset": 25,
+      "type": "<ALPHANUM>",
+      "position": 4
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
+The above example produces the following terms:
+
+[source,text]
+---------------------------
+[ I'm, delighted, about, it, _sad_ ]
+---------------------------
diff --git a/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc b/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc
index e3b85fd7bd1..72adefa5aec 100644
--- a/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc
+++ b/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc
@@ -1,37 +1,249 @@
 [[analysis-pattern-replace-charfilter]]
 === Pattern Replace Char Filter
 
-The `pattern_replace` char filter allows the use of a regex to
-manipulate the characters in a string before analysis. The regular
-expression is defined using the `pattern` parameter, and the replacement
-string can be provided using the `replacement` parameter (supporting
-referencing the original text, as explained
-http://docs.oracle.com/javase/6/docs/api/java/util/regex/Matcher.html#appendReplacement(java.lang.StringBuffer,%20java.lang.String)[here]).
-For more information check the
-http://lucene.apache.org/core/4_3_1/analyzers-common/org/apache/lucene/analysis/pattern/PatternReplaceCharFilter.html[lucene
-documentation]
+The `pattern_replace` character filter uses a regular expression to match
+characters which should be replaced with the specified replacement string.
+The replacement string can refer to capture groups in the regular expression.
 
-Here is a sample configuration:
+[float]
+=== Configuration
+
+The `pattern_replace` character filter accepts the following parameters:
+
+[horizontal]
+`pattern`::
+
+    A http://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html[Java regular expression]. Required.
+
+`replacement`::
+
+    The replacement string, which can reference capture groups using the
+    `$1`..`$9` syntax, as explained
+    http://docs.oracle.com/javase/8/docs/api/java/util/regex/Matcher.html#appendReplacement-java.lang.StringBuffer-java.lang.String-[here].
+
+[float]
+=== Example configuration
+
+In this example, we configure the `pattern_replace` character filter to
+replace any embedded dashes in numbers with underscores, i.e `123-456-789` ->
+`123_456_789`:
 
 [source,js]
---------------------------------------------------
+----------------------------
+PUT my_index
 {
-    "index" : {
-        "analysis" : {
-            "char_filter" : {
-                "my_pattern":{
-                    "type":"pattern_replace",
-                    "pattern":"sample(.*)",
-                    "replacement":"replacedSample $1"
-                }
-            },
-            "analyzer" : {
-                "custom_with_char_filter" : {
-                    "tokenizer" : "standard",
-                    "char_filter" : ["my_pattern"]
-                }
-            }
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "standard",
+          "char_filter": [
+            "my_char_filter"
+          ]
         }
+      },
+      "char_filter": {
+        "my_char_filter": {
+          "type": "pattern_replace",
+          "pattern": "(\\d+)-(?=\\d)",
+          "replacement": "$1_"
+        }
+      }
     }
+  }
 }
---------------------------------------------------
+
+GET _cluster/health?wait_for_status=yellow
+
+POST my_index/_analyze
+{
+  "analyzer": "my_analyzer",
+  "text": "My credit card is 123-456-789"
+}
+----------------------------
+// CONSOLE
+// TEST[skip:Test interprets $1 as a stashed variable]
+
+The above example produces the following term:
+
+[source,text]
+---------------------------
+[ My, credit, card, is 123_456_789 ]
+---------------------------
+
+
+WARNING: Using a replacement string that changes the length of the original
+text will work for search purposes, but will result in incorrect highlighting,
+as can be seen in the following example.
+
+This example inserts a space whenever it encounters a lower-case letter
+followed by an upper-case letter (i.e. `fooBarBaz` -> `foo Bar Baz`), allowing
+camelCase words to be queried individually:
+
+[source,js]
+----------------------------
+PUT my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "standard",
+          "char_filter": [
+            "my_char_filter"
+          ],
+          "filter": [
+            "lowercase"
+          ]
+        }
+      },
+      "char_filter": {
+        "my_char_filter": {
+          "type": "pattern_replace",
+          "pattern": "(?<=\\p{Lower})(?=\\p{Upper})",
+          "replacement": " "
+        }
+      }
+    }
+  },
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "text": {
+          "type": "text",
+          "analyzer": "my_analyzer"
+        }
+      }
+    }
+  }
+}
+
+GET _cluster/health?wait_for_status=yellow
+
+POST my_index/_analyze
+{
+  "analyzer": "my_analyzer",
+  "text": "The fooBarBaz method"
+}
+----------------------------
+// CONSOLE
+
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "the",
+      "start_offset": 0,
+      "end_offset": 3,
+      "type": "<ALPHANUM>",
+      "position": 0
+    },
+    {
+      "token": "foo",
+      "start_offset": 4,
+      "end_offset": 6,
+      "type": "<ALPHANUM>",
+      "position": 1
+    },
+    {
+      "token": "bar",
+      "start_offset": 7,
+      "end_offset": 9,
+      "type": "<ALPHANUM>",
+      "position": 2
+    },
+    {
+      "token": "baz",
+      "start_offset": 10,
+      "end_offset": 13,
+      "type": "<ALPHANUM>",
+      "position": 3
+    },
+    {
+      "token": "method",
+      "start_offset": 14,
+      "end_offset": 20,
+      "type": "<ALPHANUM>",
+      "position": 4
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+The above returns the following terms:
+
+[source,js]
+----------------------------
+[ the, foo, bar, baz, method ]
+----------------------------
+
+Querying for `bar` will find the document correctly, but highlighting on the
+result will produce incorrect highlights, because our character filter changed
+the length of the original text:
+
+[source,js]
+----------------------------
+PUT my_index/my_doc/1?refresh
+{
+  "text": "The fooBarBaz method"
+}
+
+GET my_index/_search
+{
+  "query": {
+    "match": {
+      "text": "bar"
+    }
+  },
+  "highlight": {
+    "fields": {
+      "text": {}
+    }
+  }
+}
+----------------------------
+// CONSOLE
+// TEST[continued]
+
+The output from the above is:
+
+[source,js]
+----------------------------
+{
+  "timed_out": false,
+  "took": $body.took,
+  "_shards": {
+    "total": 5,
+    "successful": 5,
+    "failed": 0
+  },
+  "hits": {
+    "total": 1,
+    "max_score": 0.4375,
+    "hits": [
+      {
+        "_index": "my_index",
+        "_type": "my_doc",
+        "_id": "1",
+        "_score": 0.4375,
+        "_source": {
+          "text": "The fooBarBaz method"
+        },
+        "highlight": {
+          "text": [
+            "The foo<em>Ba</em>rBaz method" <1>
+          ]
+        }
+      }
+    ]
+  }
+}
+----------------------------
+// TESTRESPONSE[s/"took".*/"took": "$body.took",/]
+<1> Note the incorrect highlight.
diff --git a/docs/reference/analysis/tokenizers.asciidoc b/docs/reference/analysis/tokenizers.asciidoc
index 46c02f9a4fc..b30822b6a0b 100644
--- a/docs/reference/analysis/tokenizers.asciidoc
+++ b/docs/reference/analysis/tokenizers.asciidoc
@@ -1,34 +1,136 @@
 [[analysis-tokenizers]]
 == Tokenizers
 
-Tokenizers are used to break a string down into a stream of terms
-or tokens. A simple tokenizer might split the string up into terms 
-wherever it encounters whitespace or punctuation.
+A _tokenizer_  receives a stream of characters, breaks it up into individual
+_tokens_ (usually individual words), and outputs a stream of _tokens_. For
+instance, a <<analysis-whitespace-tokenizer,`whitespace`>> tokenizer breaks
+text into tokens whenever it sees any whitespace.  It would convert the text
+`"Quick brown fox!"` into the terms `[Quick, brown, fox!]`.
+
+The tokenizer is also responsible for recording the order or _position_ of
+each term (used for phrase and word proximity queries) and the start and end
+_character offsets_ of the original word which the term represents (used for
+highlighting search snippets).
+
+Elasticsearch has a number of built in tokenizers which can be used to build
+<<analysis-custom-analyzer,custom analyzers>>.
+
+[float]
+=== Word Oriented Tokenizers
+
+The following tokenizers are usually used for tokenizing full text into
+individual words:
+
+<<analysis-standard-tokenizer,Standard Tokenizer>>::
+
+The `standard` tokenizer divides text into terms on word boundaries, as
+defined by the Unicode Text Segmentation algorithm. It removes most
+punctuation symbols. It is the best choice for most languages.
+
+<<analysis-letter-tokenizer,Letter Tokenizer>>::
+
+The `letter` tokenizer divides text into terms whenever it encounters a
+character which is not a letter.
+
+<<analysis-letter-tokenizer,Lowercase Tokenizer>>::
+
+The `lowercase` tokenizer, like the `letter` tokenizer,  divides text into
+terms whenever it encounters a character which is not a letter, but it also
+lowercases all terms.
+
+<<analysis-whitespace-tokenizer,Whitespace Tokenizer>>::
+
+The `whitespace` tokenizer divides text into terms whenever it encounters any
+whitespace character.
+
+<<analysis-uaxurlemail-tokenizer,UAX URL Email Tokenizer>>::
+
+The `uax_url_email` tokenizer is like the `standard` tokenizer except that it
+recognises URLs and email addresses as single tokens.
+
+<<analysis-classic-tokenizer,Classic Tokenizer>>::
+
+The `classic` tokenizer is a grammar based tokenizer for the English Language.
+
+<<analysis-thai-tokenizer,Thai Tokenizer>>::
+
+The `thai` tokenizer segments Thai text into words.
+
+[float]
+=== Partial Word Tokenizers
+
+These tokenizers break up text or words into small fragments, for partial word
+matching:
+
+<<analysis-ngram-tokenizer,N-Gram Tokenizer>>::
+
+The `ngram` tokenizer can break up text into words when it encounters any of
+a list of specified characters (e.g. whitespace or punctuation), then it returns
+n-grams of each word: a sliding window of continuous letters, e.g. `quick` ->
+`[qu, ui, ic, ck]`.
+
+<<analysis-edgengram-tokenizer,Edge N-Gram Tokenizer>>::
+
+The `edge_ngram` tokenizer can break up text into words when it encounters any of
+a list of specified characters (e.g. whitespace or punctuation), then it returns
+n-grams of each word which are anchored to the start of the word, e.g. `quick` ->
+`[q, qu, qui, quic, quick]`.
+
+
+[float]
+=== Structured Text Tokenizers
+
+The following tokenizers are usually used with structured text like
+identifiers, email addresses, zip codes, and paths, rather than with full
+text:
+
+<<analysis-keyword-tokenizer,Keyword Tokenizer>>::
+
+The `keyword` tokenizer is a ``noop'' tokenizer that accepts whatever text it
+is given and outputs the exact same text as a single term.  It can be combined
+with token filters like <<analysis-lowercase-tokenfilter,`lowercase`>> to
+normalise the analysed terms.
+
+<<analysis-pattern-tokenizer,Pattern Tokenizer>>::
+
+The `pattern` tokenizer uses a regular expression to either split text into
+terms whenever it matches a word separator, or to capture matching text as
+terms.
+
+<<analysis-pathhierarchy-tokenizer,Path Tokenizer>>::
+
+The `path_hierarchy` tokenizer takes a hierarchical value like a filesystem
+path, splits on the path separator, and emits a term for each component in the
+tree, e.g. `/foo/bar/baz` -> `[/foo, /foo/bar, /foo/bar/baz ]`.
+
+
+
 
-Elasticsearch has a number of built in tokenizers which can be
-used to build <<analysis-custom-analyzer,custom analyzers>>.
 
 include::tokenizers/standard-tokenizer.asciidoc[]
 
-include::tokenizers/edgengram-tokenizer.asciidoc[]
-
-include::tokenizers/keyword-tokenizer.asciidoc[]
-
 include::tokenizers/letter-tokenizer.asciidoc[]
 
 include::tokenizers/lowercase-tokenizer.asciidoc[]
 
-include::tokenizers/ngram-tokenizer.asciidoc[]
-
 include::tokenizers/whitespace-tokenizer.asciidoc[]
 
-include::tokenizers/pattern-tokenizer.asciidoc[]
-
 include::tokenizers/uaxurlemail-tokenizer.asciidoc[]
 
-include::tokenizers/pathhierarchy-tokenizer.asciidoc[]
-
 include::tokenizers/classic-tokenizer.asciidoc[]
 
 include::tokenizers/thai-tokenizer.asciidoc[]
 
+
+include::tokenizers/ngram-tokenizer.asciidoc[]
+
+include::tokenizers/edgengram-tokenizer.asciidoc[]
+
+
+include::tokenizers/keyword-tokenizer.asciidoc[]
+
+include::tokenizers/pattern-tokenizer.asciidoc[]
+
+include::tokenizers/pathhierarchy-tokenizer.asciidoc[]
+
+
diff --git a/docs/reference/analysis/tokenizers/classic-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/classic-tokenizer.asciidoc
index 9b6315cec96..45d4ad41526 100644
--- a/docs/reference/analysis/tokenizers/classic-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/classic-tokenizer.asciidoc
@@ -1,19 +1,269 @@
 [[analysis-classic-tokenizer]]
 === Classic Tokenizer
 
-A tokenizer of type `classic` providing grammar based tokenizer that is
-a good tokenizer for English language documents. This tokenizer has 
-heuristics for special treatment of acronyms, company names, email addresses,
-and internet host names. However, these rules don't always work, and 
-the tokenizer doesn't work well for most languages other than English.
+The `classic` tokenizer is a grammar based tokenizer that is good for English
+language documents. This tokenizer has heuristics for special treatment of
+acronyms, company names, email addresses, and internet host names. However,
+these rules don't always work, and the tokenizer doesn't work well for most
+languages other than English:
+
+* It splits words at most punctuation characters, removing punctuation. However, a
+  dot that's not followed by whitespace is considered part of a token.
+
+* It splits words at hyphens, unless there's a number in the token, in which case
+  the whole token is interpreted as a product number and is not split.
+
+* It recognizes email addresses and internet hostnames as one token.
+
+[float]
+=== Example output
+
+[source,js]
+---------------------------
+POST _analyze
+{
+  "tokenizer": "classic",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+---------------------------
+// CONSOLE
+
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "The",
+      "start_offset": 0,
+      "end_offset": 3,
+      "type": "<ALPHANUM>",
+      "position": 0
+    },
+    {
+      "token": "2",
+      "start_offset": 4,
+      "end_offset": 5,
+      "type": "<ALPHANUM>",
+      "position": 1
+    },
+    {
+      "token": "QUICK",
+      "start_offset": 6,
+      "end_offset": 11,
+      "type": "<ALPHANUM>",
+      "position": 2
+    },
+    {
+      "token": "Brown",
+      "start_offset": 12,
+      "end_offset": 17,
+      "type": "<ALPHANUM>",
+      "position": 3
+    },
+    {
+      "token": "Foxes",
+      "start_offset": 18,
+      "end_offset": 23,
+      "type": "<ALPHANUM>",
+      "position": 4
+    },
+    {
+      "token": "jumped",
+      "start_offset": 24,
+      "end_offset": 30,
+      "type": "<ALPHANUM>",
+      "position": 5
+    },
+    {
+      "token": "over",
+      "start_offset": 31,
+      "end_offset": 35,
+      "type": "<ALPHANUM>",
+      "position": 6
+    },
+    {
+      "token": "the",
+      "start_offset": 36,
+      "end_offset": 39,
+      "type": "<ALPHANUM>",
+      "position": 7
+    },
+    {
+      "token": "lazy",
+      "start_offset": 40,
+      "end_offset": 44,
+      "type": "<ALPHANUM>",
+      "position": 8
+    },
+    {
+      "token": "dog's",
+      "start_offset": 45,
+      "end_offset": 50,
+      "type": "<APOSTROPHE>",
+      "position": 9
+    },
+    {
+      "token": "bone",
+      "start_offset": 51,
+      "end_offset": 55,
+      "type": "<ALPHANUM>",
+      "position": 10
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
+The above sentence would produce the following terms:
+
+[source,text]
+---------------------------
+[ The, 2, QUICK, Brown, Foxes, jumped, over, the, lazy, dog's, bone ]
+---------------------------
+
+[float]
+=== Configuration
+
+The `classic` tokenizer accepts the following parameters:
+
+[horizontal]
+`max_token_length`::
+
+    The maximum token length. If a token is seen that exceeds this length then
+    it is split at `max_token_length` intervals. Defaults to `255`.
+
+[float]
+=== Example configuration
+
+In this example, we configure the `classic` tokenizer to have a
+`max_token_length` of 5 (for demonstration purposes):
+
+[source,js]
+----------------------------
+PUT my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "my_tokenizer"
+        }
+      },
+      "tokenizer": {
+        "my_tokenizer": {
+          "type": "classic",
+          "max_token_length": 5
+        }
+      }
+    }
+  }
+}
+
+GET _cluster/health?wait_for_status=yellow
+
+POST my_index/_analyze
+{
+  "analyzer": "my_analyzer",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+----------------------------
+// CONSOLE
+
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "The",
+      "start_offset": 0,
+      "end_offset": 3,
+      "type": "<ALPHANUM>",
+      "position": 0
+    },
+    {
+      "token": "2",
+      "start_offset": 4,
+      "end_offset": 5,
+      "type": "<ALPHANUM>",
+      "position": 1
+    },
+    {
+      "token": "QUICK",
+      "start_offset": 6,
+      "end_offset": 11,
+      "type": "<ALPHANUM>",
+      "position": 2
+    },
+    {
+      "token": "Brown",
+      "start_offset": 12,
+      "end_offset": 17,
+      "type": "<ALPHANUM>",
+      "position": 3
+    },
+    {
+      "token": "Foxes",
+      "start_offset": 18,
+      "end_offset": 23,
+      "type": "<ALPHANUM>",
+      "position": 4
+    },
+    {
+      "token": "over",
+      "start_offset": 31,
+      "end_offset": 35,
+      "type": "<ALPHANUM>",
+      "position": 6
+    },
+    {
+      "token": "the",
+      "start_offset": 36,
+      "end_offset": 39,
+      "type": "<ALPHANUM>",
+      "position": 7
+    },
+    {
+      "token": "lazy",
+      "start_offset": 40,
+      "end_offset": 44,
+      "type": "<ALPHANUM>",
+      "position": 8
+    },
+    {
+      "token": "dog's",
+      "start_offset": 45,
+      "end_offset": 50,
+      "type": "<APOSTROPHE>",
+      "position": 9
+    },
+    {
+      "token": "bone",
+      "start_offset": 51,
+      "end_offset": 55,
+      "type": "<ALPHANUM>",
+      "position": 10
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
+The above example produces the following terms:
+
+[source,text]
+---------------------------
+[ The, 2, QUICK, Brown, Foxes, jumpe, d, over, the, lazy, dog's, bone ]
+---------------------------
 
-The following are settings that can be set for a `classic` tokenizer
-type:
 
-[cols="<,<",options="header",]
-|=======================================================================
-|Setting |Description
-|`max_token_length` |The maximum token length. If a token is seen that
-exceeds this length then it is discarded. Defaults to `255`.
-|=======================================================================
 
diff --git a/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc
index 41cc2337940..2328354998e 100644
--- a/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc
@@ -1,80 +1,323 @@
 [[analysis-edgengram-tokenizer]]
 === Edge NGram Tokenizer
 
-A tokenizer of type `edgeNGram`.
+The `edge_ngram` tokenizer first breaks text down into words whenever it
+encounters one of a list of specified characters, then it emits
+https://en.wikipedia.org/wiki/N-gram[N-grams] of each word where the start of
+the N-gram is anchored to the beginning of the word.
 
-This tokenizer is very similar to `nGram` but only keeps n-grams which
-start at the beginning of a token.
+Edge N-Grams are useful for _search-as-you-type_ queries.
 
-The following are settings that can be set for a `edgeNGram` tokenizer
-type:
+TIP: When you need _search-as-you-type_ for text which has a widely known
+order, such as movie or song titles, the
+<<search-suggesters-completion,completion suggester>> is a much more efficient
+choice than edge N-grams.  Edge N-grams have the advantage when trying to
+autocomplete words that can appear in any order.
 
-[cols="<,<,<",options="header",]
-|=======================================================================
-|Setting |Description |Default value
-|`min_gram` |Minimum size in codepoints of a single n-gram |`1`.
+[float]
+=== Example output
 
-|`max_gram` |Maximum size in codepoints of a single n-gram |`2`.
+With the default settings, the `edge_ngram` tokenizer treats the initial text as a
+single token and produces N-grams with minimum length `1` and maximum length
+`2`:
 
-|`token_chars` | Characters classes to keep in the
-tokens, Elasticsearch will split on characters that don't belong to any
-of these classes. |`[]` (Keep all characters)
-|=======================================================================
+[source,js]
+---------------------------
+POST _analyze
+{
+  "tokenizer": "edge_ngram",
+  "text": "Quick Fox"
+}
+---------------------------
+// CONSOLE
+
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "Q",
+      "start_offset": 0,
+      "end_offset": 1,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "Qu",
+      "start_offset": 0,
+      "end_offset": 2,
+      "type": "word",
+      "position": 1
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
 
 
-`token_chars` accepts the following character classes:
+The above sentence would produce the following terms:
+
+[source,text]
+---------------------------
+[ Q, Qu ]
+---------------------------
+
+NOTE: These default gram lengths are almost entirely useless.  You need to
+configure the `edge_ngram` before using it.
+
+[float]
+=== Configuration
+
+The `edge_ngram` tokenizer accepts the following parameters:
 
 [horizontal]
-`letter`::      for example `a`, `b`, `ï` or `京`
-`digit`::       for example `3` or `7`
-`whitespace`::  for example `" "` or `"\n"`
-`punctuation`:: for example `!` or `"`
-`symbol`::      for example `$` or `√`
+`min_gram`::
+    Minimum length of characters in a gram.  Defaults to `1`.
+
+`max_gram`::
+    Maximum length of characters in a gram.  Defaults to `2`.
+
+`token_chars`::
+
+    Character classes that should be included in a token.  Elasticsearch
+    will split on characters that don't belong to the classes specified.
+    Defaults to `[]` (keep all characters).
++
+Character classes may be any of the following:
++
+* `letter` --      for example `a`, `b`, `ï` or `京`
+* `digit` --       for example `3` or `7`
+* `whitespace` --  for example `" "` or `"\n"`
+* `punctuation` -- for example `!` or `"`
+* `symbol` --      for example `$` or `√`
 
 [float]
-==== Example
+=== Example configuration
+
+In this example, we configure the `edge_ngram` tokenizer to treat letters and
+digits as tokens, and to produce grams with minimum length `2` and maximum
+length `10`:
 
 [source,js]
---------------------------------------------------
-    curl -XPUT 'localhost:9200/test' -d '
-    {
-        "settings" : {
-            "analysis" : {
-                "analyzer" : {
-                    "my_edge_ngram_analyzer" : {
-                        "tokenizer" : "my_edge_ngram_tokenizer"
-                    }
-                },
-                "tokenizer" : {
-                    "my_edge_ngram_tokenizer" : {
-                        "type" : "edgeNGram",
-                        "min_gram" : "2",
-                        "max_gram" : "5",
-                        "token_chars": [ "letter", "digit" ]
-                    }
-                }
-            }
+----------------------------
+PUT my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "my_tokenizer"
         }
-    }'
+      },
+      "tokenizer": {
+        "my_tokenizer": {
+          "type": "edge_ngram",
+          "min_gram": 2,
+          "max_gram": 10,
+          "token_chars": [
+            "letter",
+            "digit"
+          ]
+        }
+      }
+    }
+  }
+}
 
-    curl 'localhost:9200/test/_analyze?pretty=1&analyzer=my_edge_ngram_analyzer' -d 'FC Schalke 04'
-    # FC, Sc, Sch, Scha, Schal, 04
---------------------------------------------------
+GET _cluster/health?wait_for_status=yellow
 
-[float]
-==== `side` deprecated
+POST my_index/_analyze
+{
+  "analyzer": "my_analyzer",
+  "text": "2 Quick Foxes."
+}
+----------------------------
+// CONSOLE
 
-There used to be a `side` parameter up to `0.90.1` but it is now deprecated. In
-order to emulate the behavior of `"side" : "BACK"` a
-<<analysis-reverse-tokenfilter,`reverse` token filter>>  should be used together
-with the <<analysis-edgengram-tokenfilter,`edgeNGram` token filter>>. The
-`edgeNGram` filter must be enclosed in `reverse` filters like this:
+/////////////////////
 
 [source,js]
---------------------------------------------------
-    "filter" : ["reverse", "edgeNGram", "reverse"]
---------------------------------------------------
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "Qu",
+      "start_offset": 2,
+      "end_offset": 4,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "Qui",
+      "start_offset": 2,
+      "end_offset": 5,
+      "type": "word",
+      "position": 1
+    },
+    {
+      "token": "Quic",
+      "start_offset": 2,
+      "end_offset": 6,
+      "type": "word",
+      "position": 2
+    },
+    {
+      "token": "Quick",
+      "start_offset": 2,
+      "end_offset": 7,
+      "type": "word",
+      "position": 3
+    },
+    {
+      "token": "Fo",
+      "start_offset": 8,
+      "end_offset": 10,
+      "type": "word",
+      "position": 4
+    },
+    {
+      "token": "Fox",
+      "start_offset": 8,
+      "end_offset": 11,
+      "type": "word",
+      "position": 5
+    },
+    {
+      "token": "Foxe",
+      "start_offset": 8,
+      "end_offset": 12,
+      "type": "word",
+      "position": 6
+    },
+    {
+      "token": "Foxes",
+      "start_offset": 8,
+      "end_offset": 13,
+      "type": "word",
+      "position": 7
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+The above example produces the following terms:
+
+[source,text]
+---------------------------
+[ Qu, Qui, Quic, Quick, Fo, Fox, Foxe, Foxes ]
+---------------------------
+
+Usually we recommend using the same `analyzer` at index time and at search
+time. In the case of the `edge_ngram` tokenizer, the advice is different.  It
+only makes sense to use the `edge_ngram` tokenizer at index time, to ensure
+that partial words are available for matching in the index.  At search time,
+just search for the terms the user has typed in, for instance: `Quick Fo`.
+
+Below is an example of how to set up a field for _search-as-you-type_:
+
+[source,js]
+-----------------------------------
+PUT my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "autocomplete": {
+          "tokenizer": "autocomplete",
+          "filter": [
+            "lowercase"
+          ]
+        },
+        "autocomplete_search": {
+          "tokenizer": "lowercase"
+        }
+      },
+      "tokenizer": {
+        "autocomplete": {
+          "type": "edge_ngram",
+          "min_gram": 2,
+          "max_gram": 10,
+          "token_chars": [
+            "letter"
+          ]
+        }
+      }
+    }
+  },
+  "mappings": {
+    "doc": {
+      "properties": {
+        "title": {
+          "type": "text",
+          "analyzer": "autocomplete",
+          "search_analyzer": "autocomplete_search"
+        }
+      }
+    }
+  }
+}
+
+PUT my_index/doc/1
+{
+  "title": "Quick Foxes" <1>
+}
+
+POST my_index/_refresh
+
+GET my_index/_search
+{
+  "query": {
+    "match": {
+      "title": {
+        "query": "Quick Fo", <2>
+        "operator": "and"
+      }
+    }
+  }
+}
+-----------------------------------
+// CONSOLE
+
+<1> The `autocomplete` analyzer indexes the terms `[qu, qui, quic, quick, fo, fox, foxe, foxes]`.
+<2> The `autocomplete_search` analyzer searches for the terms `[quick, fo]`, both of which appear in the index.
+
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "took": $body.took,
+  "timed_out": false,
+  "_shards": {
+    "total": 5,
+    "successful": 5,
+    "failed": 0
+  },
+  "hits": {
+    "total": 1,
+    "max_score": 0.44194174,
+    "hits": [
+      {
+        "_index": "my_index",
+        "_type": "doc",
+        "_id": "1",
+        "_score": 0.44194174,
+        "_source": {
+          "title": "Quick Foxes"
+        }
+      }
+    ]
+  }
+}
+----------------------------
+// TESTRESPONSE[s/"took".*/"took": "$body.took",/]
+/////////////////////
 
-which essentially reverses the token, builds front `EdgeNGrams` and reverses
-the ngram again. This has the same effect as the previous `"side" : "BACK"` setting.
 
diff --git a/docs/reference/analysis/tokenizers/keyword-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/keyword-tokenizer.asciidoc
index ad1652466be..27515516fe5 100644
--- a/docs/reference/analysis/tokenizers/keyword-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/keyword-tokenizer.asciidoc
@@ -1,15 +1,60 @@
 [[analysis-keyword-tokenizer]]
 === Keyword Tokenizer
 
-A tokenizer of type `keyword` that emits the entire input as a single
-output.
+The `keyword` tokenizer  is a ``noop'' tokenizer that accepts whatever text it
+is given and outputs the exact same text as a single term.  It can be combined
+with token filters to normalise output, e.g. lower-casing email addresses.
 
-The following are settings that can be set for a `keyword` tokenizer
-type:
+[float]
+=== Example output
 
-[cols="<,<",options="header",]
-|=======================================================
-|Setting |Description
-|`buffer_size` |The term buffer size. Defaults to `256`.
-|=======================================================
+[source,js]
+---------------------------
+POST _analyze
+{
+  "tokenizer": "keyword",
+  "text": "New York"
+}
+---------------------------
+// CONSOLE
+
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "New York",
+      "start_offset": 0,
+      "end_offset": 8,
+      "type": "word",
+      "position": 0
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
+The above sentence would produce the following term:
+
+[source,text]
+---------------------------
+[ New York ]
+---------------------------
+
+[float]
+=== Configuration
+
+The `keyword` tokenizer accepts the following parameters:
+
+[horizontal]
+`buffer_size`::
+
+    The number of characters read into the term buffer in a single pass.
+    Defaults to `256`.  The term buffer will grow by this size until all the
+    text has been consumed.  It is advisable not to change this setting.
 
diff --git a/docs/reference/analysis/tokenizers/letter-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/letter-tokenizer.asciidoc
index 03025ccd303..7423a68732d 100644
--- a/docs/reference/analysis/tokenizers/letter-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/letter-tokenizer.asciidoc
@@ -1,7 +1,123 @@
 [[analysis-letter-tokenizer]]
 === Letter Tokenizer
 
-A tokenizer of type `letter` that divides text at non-letters. That's to
-say, it defines tokens as maximal strings of adjacent letters. Note,
-this does a decent job for most European languages, but does a terrible
-job for some Asian languages, where words are not separated by spaces.
+The `letter` tokenizer breaks text into terms whenever it encounters a
+character which is not a letter. It does a reasonable job for most European
+languages, but does a terrible job for some Asian languages, where words are
+not separated by spaces.
+
+[float]
+=== Example output
+
+[source,js]
+---------------------------
+POST _analyze
+{
+  "tokenizer": "letter",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+---------------------------
+// CONSOLE
+
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "The",
+      "start_offset": 0,
+      "end_offset": 3,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "QUICK",
+      "start_offset": 6,
+      "end_offset": 11,
+      "type": "word",
+      "position": 1
+    },
+    {
+      "token": "Brown",
+      "start_offset": 12,
+      "end_offset": 17,
+      "type": "word",
+      "position": 2
+    },
+    {
+      "token": "Foxes",
+      "start_offset": 18,
+      "end_offset": 23,
+      "type": "word",
+      "position": 3
+    },
+    {
+      "token": "jumped",
+      "start_offset": 24,
+      "end_offset": 30,
+      "type": "word",
+      "position": 4
+    },
+    {
+      "token": "over",
+      "start_offset": 31,
+      "end_offset": 35,
+      "type": "word",
+      "position": 5
+    },
+    {
+      "token": "the",
+      "start_offset": 36,
+      "end_offset": 39,
+      "type": "word",
+      "position": 6
+    },
+    {
+      "token": "lazy",
+      "start_offset": 40,
+      "end_offset": 44,
+      "type": "word",
+      "position": 7
+    },
+    {
+      "token": "dog",
+      "start_offset": 45,
+      "end_offset": 48,
+      "type": "word",
+      "position": 8
+    },
+    {
+      "token": "s",
+      "start_offset": 49,
+      "end_offset": 50,
+      "type": "word",
+      "position": 9
+    },
+    {
+      "token": "bone",
+      "start_offset": 51,
+      "end_offset": 55,
+      "type": "word",
+      "position": 10
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
+The above sentence would produce the following terms:
+
+[source,text]
+---------------------------
+[ The, QUICK, Brown, Foxes, jumped, over, the, lazy, dog, s, bone ]
+---------------------------
+
+[float]
+=== Configuration
+
+The `letter` tokenizer is not configurable.
diff --git a/docs/reference/analysis/tokenizers/lowercase-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/lowercase-tokenizer.asciidoc
index 0cdbbc387a4..5aad28b4394 100644
--- a/docs/reference/analysis/tokenizers/lowercase-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/lowercase-tokenizer.asciidoc
@@ -1,15 +1,128 @@
 [[analysis-lowercase-tokenizer]]
 === Lowercase Tokenizer
 
-A tokenizer of type `lowercase` that performs the function of
-<<analysis-letter-tokenizer,Letter
-Tokenizer>> and
-<<analysis-lowercase-tokenfilter,Lower
-Case Token Filter>> together. It divides text at non-letters and converts
-them to lower case. While it is functionally equivalent to the
-combination of
-<<analysis-letter-tokenizer,Letter
-Tokenizer>> and
-<<analysis-lowercase-tokenfilter,Lower
-Case Token Filter>>, there is a performance advantage to doing the two
-tasks at once, hence this (redundant) implementation.
+
+The `lowercase` toknenizer, like the
+<<analysis-letter-tokenizer, `letter` tokenizer>> breaks text into terms
+whenever it encounters a character which is not a letter, but it also
+lowecases all terms.  It is functionally equivalent to the
+<<analysis-letter-tokenizer, `letter` tokenizer>> combined with the
+<<analysis-lowercase-tokenfilter, `lowercase` token filter>>, but is more
+efficient as it performs both steps in a single pass.
+
+
+[float]
+=== Example output
+
+[source,js]
+---------------------------
+POST _analyze
+{
+  "tokenizer": "lowercase",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+---------------------------
+// CONSOLE
+
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "the",
+      "start_offset": 0,
+      "end_offset": 3,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "quick",
+      "start_offset": 6,
+      "end_offset": 11,
+      "type": "word",
+      "position": 1
+    },
+    {
+      "token": "brown",
+      "start_offset": 12,
+      "end_offset": 17,
+      "type": "word",
+      "position": 2
+    },
+    {
+      "token": "foxes",
+      "start_offset": 18,
+      "end_offset": 23,
+      "type": "word",
+      "position": 3
+    },
+    {
+      "token": "jumped",
+      "start_offset": 24,
+      "end_offset": 30,
+      "type": "word",
+      "position": 4
+    },
+    {
+      "token": "over",
+      "start_offset": 31,
+      "end_offset": 35,
+      "type": "word",
+      "position": 5
+    },
+    {
+      "token": "the",
+      "start_offset": 36,
+      "end_offset": 39,
+      "type": "word",
+      "position": 6
+    },
+    {
+      "token": "lazy",
+      "start_offset": 40,
+      "end_offset": 44,
+      "type": "word",
+      "position": 7
+    },
+    {
+      "token": "dog",
+      "start_offset": 45,
+      "end_offset": 48,
+      "type": "word",
+      "position": 8
+    },
+    {
+      "token": "s",
+      "start_offset": 49,
+      "end_offset": 50,
+      "type": "word",
+      "position": 9
+    },
+    {
+      "token": "bone",
+      "start_offset": 51,
+      "end_offset": 55,
+      "type": "word",
+      "position": 10
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
+The above sentence would produce the following terms:
+
+[source,text]
+---------------------------
+[ the, quick, brown, foxes, jumped, over, the, lazy, dog, s, bone ]
+---------------------------
+
+[float]
+=== Configuration
+
+The `lowercase` tokenizer is not configurable.
diff --git a/docs/reference/analysis/tokenizers/ngram-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/ngram-tokenizer.asciidoc
index 23e6bc52dda..cf45da0627e 100644
--- a/docs/reference/analysis/tokenizers/ngram-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/ngram-tokenizer.asciidoc
@@ -1,57 +1,306 @@
 [[analysis-ngram-tokenizer]]
 === NGram Tokenizer
 
-A tokenizer of type `nGram`.
+The `ngram` tokenizer first breaks text down into words whenever it encounters
+one of a list of specified characters, then it emits
+https://en.wikipedia.org/wiki/N-gram[N-grams] of each word of the specified
+length.
 
-The following are settings that can be set for a `nGram` tokenizer type:
-
-[cols="<,<,<",options="header",]
-|=======================================================================
-|Setting |Description |Default value
-|`min_gram` |Minimum size in codepoints of a single n-gram |`1`.
-
-|`max_gram` |Maximum size in codepoints of a single n-gram |`2`.
-
-|`token_chars` |Characters classes to keep in the
-tokens, Elasticsearch will split on characters that don't belong to any
-of these classes. |`[]` (Keep all characters)
-|=======================================================================
-
-`token_chars` accepts the following character classes:
-
-[horizontal]
-`letter`::      for example `a`, `b`, `ï` or `京`
-`digit`::       for example `3` or `7`
-`whitespace`::  for example `" "` or `"\n"`
-`punctuation`:: for example `!` or `"`
-`symbol`::      for example `$` or `√`
+N-grams are like a sliding window that moves across the word - a continuous
+sequence of characters of the specified length. They are useful for querying
+languages that don't use spaces or that have long compound words, like German.
 
 [float]
-==== Example
+=== Example output
+
+With the default settings, the `ngram` tokenizer treats the initial text as a
+single token and produces N-grams with minimum length `1` and maximum length
+`2`:
 
 [source,js]
---------------------------------------------------
-    curl -XPUT 'localhost:9200/test' -d '
-    {
-        "settings" : {
-            "analysis" : {
-                "analyzer" : {
-                    "my_ngram_analyzer" : {
-                        "tokenizer" : "my_ngram_tokenizer"
-                    }
-                },
-                "tokenizer" : {
-                    "my_ngram_tokenizer" : {
-                        "type" : "nGram",
-                        "min_gram" : "2",
-                        "max_gram" : "3",
-                        "token_chars": [ "letter", "digit" ]
-                    }
-                }
-            }
-        }
-    }'
+---------------------------
+POST _analyze
+{
+  "tokenizer": "ngram",
+  "text": "Quick Fox"
+}
+---------------------------
+// CONSOLE
+
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "Q",
+      "start_offset": 0,
+      "end_offset": 1,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "Qu",
+      "start_offset": 0,
+      "end_offset": 2,
+      "type": "word",
+      "position": 1
+    },
+    {
+      "token": "u",
+      "start_offset": 1,
+      "end_offset": 2,
+      "type": "word",
+      "position": 2
+    },
+    {
+      "token": "ui",
+      "start_offset": 1,
+      "end_offset": 3,
+      "type": "word",
+      "position": 3
+    },
+    {
+      "token": "i",
+      "start_offset": 2,
+      "end_offset": 3,
+      "type": "word",
+      "position": 4
+    },
+    {
+      "token": "ic",
+      "start_offset": 2,
+      "end_offset": 4,
+      "type": "word",
+      "position": 5
+    },
+    {
+      "token": "c",
+      "start_offset": 3,
+      "end_offset": 4,
+      "type": "word",
+      "position": 6
+    },
+    {
+      "token": "ck",
+      "start_offset": 3,
+      "end_offset": 5,
+      "type": "word",
+      "position": 7
+    },
+    {
+      "token": "k",
+      "start_offset": 4,
+      "end_offset": 5,
+      "type": "word",
+      "position": 8
+    },
+    {
+      "token": "k ",
+      "start_offset": 4,
+      "end_offset": 6,
+      "type": "word",
+      "position": 9
+    },
+    {
+      "token": " ",
+      "start_offset": 5,
+      "end_offset": 6,
+      "type": "word",
+      "position": 10
+    },
+    {
+      "token": " F",
+      "start_offset": 5,
+      "end_offset": 7,
+      "type": "word",
+      "position": 11
+    },
+    {
+      "token": "F",
+      "start_offset": 6,
+      "end_offset": 7,
+      "type": "word",
+      "position": 12
+    },
+    {
+      "token": "Fo",
+      "start_offset": 6,
+      "end_offset": 8,
+      "type": "word",
+      "position": 13
+    },
+    {
+      "token": "o",
+      "start_offset": 7,
+      "end_offset": 8,
+      "type": "word",
+      "position": 14
+    },
+    {
+      "token": "ox",
+      "start_offset": 7,
+      "end_offset": 9,
+      "type": "word",
+      "position": 15
+    },
+    {
+      "token": "x",
+      "start_offset": 8,
+      "end_offset": 9,
+      "type": "word",
+      "position": 16
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
+The above sentence would produce the following terms:
+
+[source,text]
+---------------------------
+[ Q, Qu, u, ui, i, ic, c, ck, k, "k ", " ", " F", F, Fo, o, ox, x ]
+---------------------------
+
+[float]
+=== Configuration
+
+The `ngram` tokenizer accepts the following parameters:
+
+[horizontal]
+`min_gram`::
+    Minimum length of characters in a gram.  Defaults to `1`.
+
+`max_gram`::
+    Maximum length of characters in a gram.  Defaults to `2`.
+
+`token_chars`::
+
+    Character classes that should be included in a token.  Elasticsearch
+    will split on characters that don't belong to the classes specified.
+    Defaults to `[]` (keep all characters).
++
+Character classes may be any of the following:
++
+* `letter` --      for example `a`, `b`, `ï` or `京`
+* `digit` --       for example `3` or `7`
+* `whitespace` --  for example `" "` or `"\n"`
+* `punctuation` -- for example `!` or `"`
+* `symbol` --      for example `$` or `√`
+
+TIP:  It usually makes sense to set `min_gram` and `max_gram` to the same
+value.  The smaller the length, the more documents will match but the lower
+the quality of the matches.  The longer the length, the more specific the
+matches.  A tri-gram (length `3`) is a good place to start.
+
+[float]
+=== Example configuration
+
+In this example, we configure the `ngram` tokenizer to treat letters and
+digits as tokens, and to produce tri-grams (grams of length `3`):
+
+[source,js]
+----------------------------
+PUT my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "my_tokenizer"
+        }
+      },
+      "tokenizer": {
+        "my_tokenizer": {
+          "type": "ngram",
+          "min_gram": 3,
+          "max_gram": 3,
+          "token_chars": [
+            "letter",
+            "digit"
+          ]
+        }
+      }
+    }
+  }
+}
+
+GET _cluster/health?wait_for_status=yellow
+
+POST my_index/_analyze
+{
+  "analyzer": "my_analyzer",
+  "text": "2 Quick Foxes."
+}
+----------------------------
+// CONSOLE
+
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "Qui",
+      "start_offset": 2,
+      "end_offset": 5,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "uic",
+      "start_offset": 3,
+      "end_offset": 6,
+      "type": "word",
+      "position": 1
+    },
+    {
+      "token": "ick",
+      "start_offset": 4,
+      "end_offset": 7,
+      "type": "word",
+      "position": 2
+    },
+    {
+      "token": "Fox",
+      "start_offset": 8,
+      "end_offset": 11,
+      "type": "word",
+      "position": 3
+    },
+    {
+      "token": "oxe",
+      "start_offset": 9,
+      "end_offset": 12,
+      "type": "word",
+      "position": 4
+    },
+    {
+      "token": "xes",
+      "start_offset": 10,
+      "end_offset": 13,
+      "type": "word",
+      "position": 5
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
+The above example produces the following terms:
+
+[source,text]
+---------------------------
+[ Qui, uic, ick, Fox, oxe, xes ]
+---------------------------
+
 
-    curl 'localhost:9200/test/_analyze?pretty=1&analyzer=my_ngram_analyzer' -d 'FC Schalke 04'
-    # FC, Sc, Sch, ch, cha, ha, hal, al, alk, lk, lke, ke, 04
---------------------------------------------------
diff --git a/docs/reference/analysis/tokenizers/pathhierarchy-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/pathhierarchy-tokenizer.asciidoc
index e6876f55bc6..b656e67eaec 100644
--- a/docs/reference/analysis/tokenizers/pathhierarchy-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/pathhierarchy-tokenizer.asciidoc
@@ -1,32 +1,175 @@
 [[analysis-pathhierarchy-tokenizer]]
 === Path Hierarchy Tokenizer
 
-The `path_hierarchy` tokenizer takes something like this:
+The `path_hierarchy` tokenizer takes a hierarchical value like a filesystem
+path, splits on the path separator, and emits a term for each component in the
+tree.
 
--------------------------
-/something/something/else
--------------------------
+[float]
+=== Example output
 
-And produces tokens:
+[source,js]
+---------------------------
+POST _analyze
+{
+  "tokenizer": "path_hierarchy",
+  "text": "/one/two/three"
+}
+---------------------------
+// CONSOLE
 
--------------------------
-/something
-/something/something
-/something/something/else
--------------------------
+/////////////////////
 
-[cols="<,<",options="header",]
-|=======================================================================
-|Setting |Description
-|`delimiter` |The character delimiter to use, defaults to `/`.
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "/one",
+      "start_offset": 0,
+      "end_offset": 4,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "/one/two",
+      "start_offset": 0,
+      "end_offset": 8,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "/one/two/three",
+      "start_offset": 0,
+      "end_offset": 14,
+      "type": "word",
+      "position": 0
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
 
-|`replacement` |An optional replacement character to use. Defaults to
-the `delimiter`.
+/////////////////////
 
-|`buffer_size` |The buffer size to use, defaults to `1024`.
 
-|`reverse` |Generates tokens in reverse order, defaults to `false`.
 
-|`skip` |Controls initial tokens to skip, defaults to `0`.
-|=======================================================================
+The above text would produce the following terms:
+
+[source,text]
+---------------------------
+[ /one, /one/two, /one/two/three ]
+---------------------------
+
+[float]
+=== Configuration
+
+The `path_hierarchy` tokenizer accepts the following parameters:
+
+[horizontal]
+`delimiter`::
+    The character to use as the path separator.  Defaults to `/`.
+
+`replacement`::
+    An optional replacement character to use for the delimiter.
+    Defaults to the `delimiter`.
+
+`buffer_size`::
+    The number of characters read into the term buffer in a single pass.
+    Defaults to `1024`.  The term buffer will grow by this size until all the
+    text has been consumed.  It is advisable not to change this setting.
+
+`reverse`::
+    If set to `true`, emits the tokens in reverse order.  Defaults to `false`.
+
+`skip`::
+    The number of initial tokens to skip.  Defaults to `0`.
+
+[float]
+=== Example configuration
+
+In this example, we configure the `path_hierarchy` tokenizer to split on `-`
+characters, and to replace them with `/`.  The first two tokens are skipped:
+
+[source,js]
+----------------------------
+PUT my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "my_tokenizer"
+        }
+      },
+      "tokenizer": {
+        "my_tokenizer": {
+          "type": "path_hierarchy",
+          "delimiter": "-",
+          "replacement": "/",
+          "skip": 2
+        }
+      }
+    }
+  }
+}
+
+GET _cluster/health?wait_for_status=yellow
+
+POST my_index/_analyze
+{
+  "analyzer": "my_analyzer",
+  "text": "one-two-three-four-five"
+}
+----------------------------
+// CONSOLE
+
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "/three",
+      "start_offset": 7,
+      "end_offset": 13,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "/three/four",
+      "start_offset": 7,
+      "end_offset": 18,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "/three/four/five",
+      "start_offset": 7,
+      "end_offset": 23,
+      "type": "word",
+      "position": 0
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
+The above example produces the following terms:
+
+[source,text]
+---------------------------
+[ /three, /three/four, /three/four/five ]
+---------------------------
+
+If we were to set `reverse` to `true`, it would produce the following:
+
+[source,text]
+---------------------------
+[ one/two/three/, two/three/, three/ ]
+---------------------------
 
diff --git a/docs/reference/analysis/tokenizers/pattern-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/pattern-tokenizer.asciidoc
index 9a148456195..ca902a4e5f2 100644
--- a/docs/reference/analysis/tokenizers/pattern-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/pattern-tokenizer.asciidoc
@@ -1,38 +1,268 @@
 [[analysis-pattern-tokenizer]]
 === Pattern Tokenizer
 
-A tokenizer of type `pattern` that can flexibly separate text into terms
-via a regular expression. Accepts the following settings:
+The `pattern` tokenizer uses a regular expression to either split text into
+terms whenever it matches a word separator, or to capture matching text as
+terms.
 
-[cols="<,<",options="header",]
-|======================================================================
-|Setting |Description
-|`pattern` |The regular expression pattern, defaults to `\W+`.
-|`flags` |The regular expression flags.
-|`group` |Which group to extract into tokens. Defaults to `-1` (split).
-|======================================================================
+The default pattern is `\W+`, which splits text whenever it encounters
+non-word characters.
 
-*IMPORTANT*: The regular expression should match the *token separators*,
-not the tokens themselves.
+[float]
+=== Example output
 
-*********************************************
-Note that you may need to escape `pattern` string literal according to
-your client language rules. For example, in many programming languages
-a string literal for `\W+` pattern is written as `"\\W+"`.
-There is nothing special about `pattern` (you may have to escape other
-string literals as well); escaping `pattern` is common just because it
-often contains characters that should be escaped.
-*********************************************
+[source,js]
+---------------------------
+POST _analyze
+{
+  "tokenizer": "pattern",
+  "text": "The foo_bar_size's default is 5."
+}
+---------------------------
+// CONSOLE
 
-`group` set to `-1` (the default) is equivalent to "split". Using group
->= 0 selects the matching group as the token. For example, if you have:
+/////////////////////
 
-------------------------
-pattern = '([^']+)'
-group   = 0
-input   = aaa 'bbb' 'ccc'
-------------------------
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "The",
+      "start_offset": 0,
+      "end_offset": 3,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "foo_bar_size",
+      "start_offset": 4,
+      "end_offset": 16,
+      "type": "word",
+      "position": 1
+    },
+    {
+      "token": "s",
+      "start_offset": 17,
+      "end_offset": 18,
+      "type": "word",
+      "position": 2
+    },
+    {
+      "token": "default",
+      "start_offset": 19,
+      "end_offset": 26,
+      "type": "word",
+      "position": 3
+    },
+    {
+      "token": "is",
+      "start_offset": 27,
+      "end_offset": 29,
+      "type": "word",
+      "position": 4
+    },
+    {
+      "token": "5",
+      "start_offset": 30,
+      "end_offset": 31,
+      "type": "word",
+      "position": 5
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
 
-the output will be two tokens: `'bbb'` and `'ccc'` (including the `'`
-marks). With the same input but using group=1, the output would be:
-`bbb` and `ccc` (no `'` marks).
+/////////////////////
+
+
+The above sentence would produce the following terms:
+
+[source,text]
+---------------------------
+[ The, foo_bar_size, s, default, is, 5 ]
+---------------------------
+
+[float]
+=== Configuration
+
+The `pattern` tokenizer accepts the following parameters:
+
+[horizontal]
+`pattern`::
+
+    A http://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html[Java regular expression], defaults to `\W+`.
+
+`flags`::
+
+    Java regular expression http://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html#field.summary[flags].
+    lags should be pipe-separated, eg `"CASE_INSENSITIVE|COMMENTS"`.
+
+`group`::
+
+    Which capture group to extract as tokens.  Defaults to `-1` (split).
+
+[float]
+=== Example configuration
+
+In this example, we configure the `pattern` tokenizer to break text into
+tokens when it encounters commas:
+
+[source,js]
+----------------------------
+PUT my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "my_tokenizer"
+        }
+      },
+      "tokenizer": {
+        "my_tokenizer": {
+          "type": "pattern",
+          "pattern": ","
+        }
+      }
+    }
+  }
+}
+
+GET _cluster/health?wait_for_status=yellow
+
+POST my_index/_analyze
+{
+  "analyzer": "my_analyzer",
+  "text": "comma,separated,values"
+}
+----------------------------
+// CONSOLE
+
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "comma",
+      "start_offset": 0,
+      "end_offset": 5,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "separated",
+      "start_offset": 6,
+      "end_offset": 15,
+      "type": "word",
+      "position": 1
+    },
+    {
+      "token": "values",
+      "start_offset": 16,
+      "end_offset": 22,
+      "type": "word",
+      "position": 2
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
+The above example produces the following terms:
+
+[source,text]
+---------------------------
+[ comma, separated, values ]
+---------------------------
+
+In the next example, we configure the `pattern` tokenizer to capture values
+enclosed in double quotes (ignoring embedded escaped quotes `\"`).  The regex
+itself looks like this:
+
+    "((?:\\"|[^"]|\\")*)"
+
+And reads as follows:
+
+* A literal `"`
+* Start capturing:
+** A literal `\"` OR any character except `"`
+** Repeat until no more characters match
+* A literal closing `"`
+
+When the pattern is specified in JSON, the `"` and `\` characters need to be
+escaped, so the pattern ends up looking like:
+
+    \"((?:\\\\\"|[^\"]|\\\\\")+)\"
+
+[source,js]
+----------------------------
+PUT my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "my_tokenizer"
+        }
+      },
+      "tokenizer": {
+        "my_tokenizer": {
+          "type": "pattern",
+          "pattern": "\"((?:\\\\\"|[^\"]|\\\\\")+)\"",
+          "group": 1
+        }
+      }
+    }
+  }
+}
+
+GET _cluster/health?wait_for_status=yellow
+
+POST my_index/_analyze
+{
+  "analyzer": "my_analyzer",
+  "text": "\"value\", \"value with embedded \\\" quote\""
+}
+----------------------------
+// CONSOLE
+
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "value",
+      "start_offset": 1,
+      "end_offset": 6,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "value with embedded \\\" quote",
+      "start_offset": 10,
+      "end_offset": 38,
+      "type": "word",
+      "position": 1
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+The above example produces the following two terms:
+
+[source,text]
+---------------------------
+[ value, value with embedded \" quote ]
+---------------------------
diff --git a/docs/reference/analysis/tokenizers/standard-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/standard-tokenizer.asciidoc
index 42dbe5a864a..ee052529b43 100644
--- a/docs/reference/analysis/tokenizers/standard-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/standard-tokenizer.asciidoc
@@ -1,18 +1,274 @@
 [[analysis-standard-tokenizer]]
 === Standard Tokenizer
 
-A tokenizer of type `standard` providing grammar based tokenizer that is
-a good tokenizer for most European language documents. The tokenizer
-implements the Unicode Text Segmentation algorithm, as specified in
-http://unicode.org/reports/tr29/[Unicode Standard Annex #29].
+The `standard` tokenizer provides grammar based tokenization (based on the
+Unicode Text Segmentation algorithm, as specified in
+http://unicode.org/reports/tr29/[Unicode Standard Annex #29]) and works well
+for most languages.
+
+[float]
+=== Example output
+
+[source,js]
+---------------------------
+POST _analyze
+{
+  "tokenizer": "standard",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+---------------------------
+// CONSOLE
+
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "The",
+      "start_offset": 0,
+      "end_offset": 3,
+      "type": "<ALPHANUM>",
+      "position": 0
+    },
+    {
+      "token": "2",
+      "start_offset": 4,
+      "end_offset": 5,
+      "type": "<NUM>",
+      "position": 1
+    },
+    {
+      "token": "QUICK",
+      "start_offset": 6,
+      "end_offset": 11,
+      "type": "<ALPHANUM>",
+      "position": 2
+    },
+    {
+      "token": "Brown",
+      "start_offset": 12,
+      "end_offset": 17,
+      "type": "<ALPHANUM>",
+      "position": 3
+    },
+    {
+      "token": "Foxes",
+      "start_offset": 18,
+      "end_offset": 23,
+      "type": "<ALPHANUM>",
+      "position": 4
+    },
+    {
+      "token": "jumped",
+      "start_offset": 24,
+      "end_offset": 30,
+      "type": "<ALPHANUM>",
+      "position": 5
+    },
+    {
+      "token": "over",
+      "start_offset": 31,
+      "end_offset": 35,
+      "type": "<ALPHANUM>",
+      "position": 6
+    },
+    {
+      "token": "the",
+      "start_offset": 36,
+      "end_offset": 39,
+      "type": "<ALPHANUM>",
+      "position": 7
+    },
+    {
+      "token": "lazy",
+      "start_offset": 40,
+      "end_offset": 44,
+      "type": "<ALPHANUM>",
+      "position": 8
+    },
+    {
+      "token": "dog's",
+      "start_offset": 45,
+      "end_offset": 50,
+      "type": "<ALPHANUM>",
+      "position": 9
+    },
+    {
+      "token": "bone",
+      "start_offset": 51,
+      "end_offset": 55,
+      "type": "<ALPHANUM>",
+      "position": 10
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
+The above sentence would produce the following terms:
+
+[source,text]
+---------------------------
+[ The, 2, QUICK, Brown, Foxes, jumped, over, the, lazy, dog's, bone ]
+---------------------------
+
+[float]
+=== Configuration
+
+The `standard` tokenizer accepts the following parameters:
+
+[horizontal]
+`max_token_length`::
+
+    The maximum token length. If a token is seen that exceeds this length then
+    it is split at `max_token_length` intervals. Defaults to `255`.
+
+[float]
+=== Example configuration
+
+In this example, we configure the `standard` tokenizer to have a
+`max_token_length` of 5 (for demonstration purposes):
+
+[source,js]
+----------------------------
+PUT my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "my_tokenizer"
+        }
+      },
+      "tokenizer": {
+        "my_tokenizer": {
+          "type": "standard",
+          "max_token_length": 5
+        }
+      }
+    }
+  }
+}
+
+GET _cluster/health?wait_for_status=yellow
+
+POST my_index/_analyze
+{
+  "analyzer": "my_analyzer",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+----------------------------
+// CONSOLE
+
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "The",
+      "start_offset": 0,
+      "end_offset": 3,
+      "type": "<ALPHANUM>",
+      "position": 0
+    },
+    {
+      "token": "2",
+      "start_offset": 4,
+      "end_offset": 5,
+      "type": "<NUM>",
+      "position": 1
+    },
+    {
+      "token": "QUICK",
+      "start_offset": 6,
+      "end_offset": 11,
+      "type": "<ALPHANUM>",
+      "position": 2
+    },
+    {
+      "token": "Brown",
+      "start_offset": 12,
+      "end_offset": 17,
+      "type": "<ALPHANUM>",
+      "position": 3
+    },
+    {
+      "token": "Foxes",
+      "start_offset": 18,
+      "end_offset": 23,
+      "type": "<ALPHANUM>",
+      "position": 4
+    },
+    {
+      "token": "jumpe",
+      "start_offset": 24,
+      "end_offset": 29,
+      "type": "<ALPHANUM>",
+      "position": 5
+    },
+    {
+      "token": "d",
+      "start_offset": 29,
+      "end_offset": 30,
+      "type": "<ALPHANUM>",
+      "position": 6
+    },
+    {
+      "token": "over",
+      "start_offset": 31,
+      "end_offset": 35,
+      "type": "<ALPHANUM>",
+      "position": 7
+    },
+    {
+      "token": "the",
+      "start_offset": 36,
+      "end_offset": 39,
+      "type": "<ALPHANUM>",
+      "position": 8
+    },
+    {
+      "token": "lazy",
+      "start_offset": 40,
+      "end_offset": 44,
+      "type": "<ALPHANUM>",
+      "position": 9
+    },
+    {
+      "token": "dog's",
+      "start_offset": 45,
+      "end_offset": 50,
+      "type": "<ALPHANUM>",
+      "position": 10
+    },
+    {
+      "token": "bone",
+      "start_offset": 51,
+      "end_offset": 55,
+      "type": "<ALPHANUM>",
+      "position": 11
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
+The above example produces the following terms:
+
+[source,text]
+---------------------------
+[ The, 2, QUICK, Brown, Foxes, jumpe, d, over, the, lazy, dog's, bone ]
+---------------------------
 
-The following are settings that can be set for a `standard` tokenizer
-type:
 
-[cols="<,<",options="header",]
-|=======================================================================
-|Setting |Description
-|`max_token_length` |The maximum token length. If a token is seen that
-exceeds this length then it is split at `max_token_length` intervals. Defaults to `255`.
-|=======================================================================
 
diff --git a/docs/reference/analysis/tokenizers/thai-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/thai-tokenizer.asciidoc
index 06f0b6892e7..3e9904d116e 100644
--- a/docs/reference/analysis/tokenizers/thai-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/thai-tokenizer.asciidoc
@@ -1,7 +1,106 @@
 [[analysis-thai-tokenizer]]
 === Thai Tokenizer
 
-A tokenizer of type `thai` that segments Thai text into words. This tokenizer
-uses the built-in Thai segmentation algorithm included with Java to divide
-up Thai text. Text in other languages in general will be treated the same
-as `standard`.
+The `thai` tokenizer segments Thai text into words, using the Thai
+segmentation algorithm included with Java. Text in other languages in general
+will be treated the same as the
+<<analysis-standard-tokenizer,`standard` tokenizer>>.
+
+WARNING: This tokenizer may not be supported by all JREs. It is known to work
+with Sun/Oracle and OpenJDK. If your application needs to be fully portable,
+consider using the {plugins}/analysis-icu-tokenizer.html[ICU Tokenizer] instead.
+
+[float]
+=== Example output
+
+[source,js]
+---------------------------
+POST _analyze
+{
+  "tokenizer": "thai",
+  "text": "�ารที่ได้ต้อง�สดงว่างานดี"
+}
+---------------------------
+// CONSOLE
+
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "�าร",
+      "start_offset": 0,
+      "end_offset": 3,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "ที่",
+      "start_offset": 3,
+      "end_offset": 6,
+      "type": "word",
+      "position": 1
+    },
+    {
+      "token": "ได้",
+      "start_offset": 6,
+      "end_offset": 9,
+      "type": "word",
+      "position": 2
+    },
+    {
+      "token": "ต้อง",
+      "start_offset": 9,
+      "end_offset": 13,
+      "type": "word",
+      "position": 3
+    },
+    {
+      "token": "�สดง",
+      "start_offset": 13,
+      "end_offset": 17,
+      "type": "word",
+      "position": 4
+    },
+    {
+      "token": "ว่า",
+      "start_offset": 17,
+      "end_offset": 20,
+      "type": "word",
+      "position": 5
+    },
+    {
+      "token": "งาน",
+      "start_offset": 20,
+      "end_offset": 23,
+      "type": "word",
+      "position": 6
+    },
+    {
+      "token": "ดี",
+      "start_offset": 23,
+      "end_offset": 25,
+      "type": "word",
+      "position": 7
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
+The above sentence would produce the following terms:
+
+[source,text]
+---------------------------
+[ �าร, ที่, ได้, ต้อง, �สดง, ว่า, งาน, ดี ]
+---------------------------
+
+[float]
+=== Configuration
+
+The `thai` tokenizer is not configurable.
diff --git a/docs/reference/analysis/tokenizers/uaxurlemail-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/uaxurlemail-tokenizer.asciidoc
index 9ed28e60b91..500a5e191f1 100644
--- a/docs/reference/analysis/tokenizers/uaxurlemail-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/uaxurlemail-tokenizer.asciidoc
@@ -1,16 +1,199 @@
 [[analysis-uaxurlemail-tokenizer]]
-=== UAX Email URL Tokenizer
+=== UAX URL Email  Tokenizer
 
-A tokenizer of type `uax_url_email` which works exactly like the
-`standard` tokenizer, but tokenizes emails and urls as single tokens.
+The `uax_url_email` tokenizer is like the <<analysis-standard-tokenizer,`standard` tokenizer>> except that it
+recognises URLs and email addresses as single tokens.
 
-The following are settings that can be set for a `uax_url_email`
-tokenizer type:
+[float]
+=== Example output
 
-[cols="<,<",options="header",]
-|=======================================================================
-|Setting |Description
-|`max_token_length` |The maximum token length. If a token is seen that
-exceeds this length then it is discarded. Defaults to `255`.
-|=======================================================================
+[source,js]
+---------------------------
+POST _analyze
+{
+  "tokenizer": "uax_url_email",
+  "text": "Email me at john.smith@global-international.com"
+}
+---------------------------
+// CONSOLE
 
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "Email",
+      "start_offset": 0,
+      "end_offset": 5,
+      "type": "<ALPHANUM>",
+      "position": 0
+    },
+    {
+      "token": "me",
+      "start_offset": 6,
+      "end_offset": 8,
+      "type": "<ALPHANUM>",
+      "position": 1
+    },
+    {
+      "token": "at",
+      "start_offset": 9,
+      "end_offset": 11,
+      "type": "<ALPHANUM>",
+      "position": 2
+    },
+    {
+      "token": "john.smith@global-international.com",
+      "start_offset": 12,
+      "end_offset": 47,
+      "type": "<EMAIL>",
+      "position": 3
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
+The above sentence would produce the following terms:
+
+[source,text]
+---------------------------
+[ Email, me, at, john.smith@global-international.com ]
+---------------------------
+
+while the `standard` tokenizer would produce:
+
+[source,text]
+---------------------------
+[ Email, me, at, john.smith, global, international.com ]
+---------------------------
+
+[float]
+=== Configuration
+
+The `uax_url_email` tokenizer accepts the following parameters:
+
+[horizontal]
+`max_token_length`::
+
+    The maximum token length. If a token is seen that exceeds this length then
+    it is split at `max_token_length` intervals. Defaults to `255`.
+
+[float]
+=== Example configuration
+
+In this example, we configure the `uax_url_email` tokenizer to have a
+`max_token_length` of 5 (for demonstration purposes):
+
+[source,js]
+----------------------------
+PUT my_index
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "my_tokenizer"
+        }
+      },
+      "tokenizer": {
+        "my_tokenizer": {
+          "type": "uax_url_email",
+          "max_token_length": 5
+        }
+      }
+    }
+  }
+}
+
+GET _cluster/health?wait_for_status=yellow
+
+POST my_index/_analyze
+{
+  "analyzer": "my_analyzer",
+  "text": "john.smith@global-international.com"
+}
+----------------------------
+// CONSOLE
+
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "john",
+      "start_offset": 0,
+      "end_offset": 4,
+      "type": "<ALPHANUM>",
+      "position": 0
+    },
+    {
+      "token": "smith",
+      "start_offset": 5,
+      "end_offset": 10,
+      "type": "<ALPHANUM>",
+      "position": 1
+    },
+    {
+      "token": "globa",
+      "start_offset": 11,
+      "end_offset": 16,
+      "type": "<ALPHANUM>",
+      "position": 2
+    },
+    {
+      "token": "l",
+      "start_offset": 16,
+      "end_offset": 17,
+      "type": "<ALPHANUM>",
+      "position": 3
+    },
+    {
+      "token": "inter",
+      "start_offset": 18,
+      "end_offset": 23,
+      "type": "<ALPHANUM>",
+      "position": 4
+    },
+    {
+      "token": "natio",
+      "start_offset": 23,
+      "end_offset": 28,
+      "type": "<ALPHANUM>",
+      "position": 5
+    },
+    {
+      "token": "nal.c",
+      "start_offset": 28,
+      "end_offset": 33,
+      "type": "<ALPHANUM>",
+      "position": 6
+    },
+    {
+      "token": "om",
+      "start_offset": 33,
+      "end_offset": 35,
+      "type": "<ALPHANUM>",
+      "position": 7
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
+The above example produces the following terms:
+
+[source,text]
+---------------------------
+[ john, smith, globa, l, inter, natio, nal.c, om ]
+---------------------------
diff --git a/docs/reference/analysis/tokenizers/whitespace-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/whitespace-tokenizer.asciidoc
index f0e1ce28a12..9d06ea28d55 100644
--- a/docs/reference/analysis/tokenizers/whitespace-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/whitespace-tokenizer.asciidoc
@@ -1,4 +1,114 @@
 [[analysis-whitespace-tokenizer]]
-=== Whitespace Tokenizer
+=== Whitespace Analyzer
 
-A tokenizer of type `whitespace` that divides text at whitespace.
+The `whitespace` tokenizer breaks text into terms whenever it encounters a
+whitespace character.
+
+[float]
+=== Example output
+
+[source,js]
+---------------------------
+POST _analyze
+{
+  "tokenizer": "whitespace",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+---------------------------
+// CONSOLE
+
+/////////////////////
+
+[source,js]
+----------------------------
+{
+  "tokens": [
+    {
+      "token": "The",
+      "start_offset": 0,
+      "end_offset": 3,
+      "type": "word",
+      "position": 0
+    },
+    {
+      "token": "2",
+      "start_offset": 4,
+      "end_offset": 5,
+      "type": "word",
+      "position": 1
+    },
+    {
+      "token": "QUICK",
+      "start_offset": 6,
+      "end_offset": 11,
+      "type": "word",
+      "position": 2
+    },
+    {
+      "token": "Brown-Foxes",
+      "start_offset": 12,
+      "end_offset": 23,
+      "type": "word",
+      "position": 3
+    },
+    {
+      "token": "jumped",
+      "start_offset": 24,
+      "end_offset": 30,
+      "type": "word",
+      "position": 4
+    },
+    {
+      "token": "over",
+      "start_offset": 31,
+      "end_offset": 35,
+      "type": "word",
+      "position": 5
+    },
+    {
+      "token": "the",
+      "start_offset": 36,
+      "end_offset": 39,
+      "type": "word",
+      "position": 6
+    },
+    {
+      "token": "lazy",
+      "start_offset": 40,
+      "end_offset": 44,
+      "type": "word",
+      "position": 7
+    },
+    {
+      "token": "dog's",
+      "start_offset": 45,
+      "end_offset": 50,
+      "type": "word",
+      "position": 8
+    },
+    {
+      "token": "bone.",
+      "start_offset": 51,
+      "end_offset": 56,
+      "type": "word",
+      "position": 9
+    }
+  ]
+}
+----------------------------
+// TESTRESPONSE
+
+/////////////////////
+
+
+The above sentence would produce the following terms:
+
+[source,text]
+---------------------------
+[ The, 2, QUICK, Brown-Foxes, jumped, over, the, lazy, dog's, bone. ]
+---------------------------
+
+[float]
+=== Configuration
+
+The `whitespace` tokenizer is not configurable.

From c257e2c51f235853c4453a86e10e463813140fc9 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 19 May 2016 14:08:08 -0400
Subject: [PATCH 0357/1311] Remove settings and system properties entanglement

Today when parsing settings during bootstrap, we add a system property
for every Elasticsearch setting. Additionally, settings can be set via
system properties. This commit simplifies this situation.
 - settings are no longer propogated to system properties
 - system properties can not be used to set settings
 - the "es." prefix on settings is no longer required (nor permitted)
 - test logging has a dedicated system property (tests.logger.level)

Relates #18198
---
 TESTING.asciidoc                              |   2 +-
 .../elasticsearch/gradle/BuildPlugin.groovy   |   2 +-
 .../elasticsearch/gradle/test/NodeInfo.groovy |   4 +-
 .../elasticsearch/bootstrap/Bootstrap.java    |  27 +--
 .../bootstrap/BootstrapInfo.java              |   4 +
 .../bootstrap/Elasticsearch.java              |  27 +--
 .../java/org/elasticsearch/cli/Command.java   |   7 +-
 .../org/elasticsearch/cli/SettingCommand.java |  77 +++++++
 .../common/logging/LogConfigurator.java       |   6 +-
 .../common/settings/ClusterSettings.java      |   1 -
 .../common/settings/Settings.java             |  62 ++----
 .../internal/InternalSettingsPreparer.java    |  48 +++--
 .../plugins/InstallPluginCommand.java         |  26 +--
 .../plugins/ListPluginsCommand.java           |  23 ++-
 .../org/elasticsearch/plugins/PluginCli.java  |  21 +-
 .../plugins/RemovePluginCommand.java          |  18 +-
 .../elasticsearch/bootstrap/security.policy   |   2 +-
 .../bootstrap/ElasticsearchCliTests.java      |  72 +------
 .../client/transport/TransportClientIT.java   |   1 -
 .../transport/TransportClientRetryIT.java     |   1 -
 .../common/settings/ScopedSettingsTests.java  |   9 +-
 .../common/settings/SettingsTests.java        |  41 ++--
 .../InternalSettingsPreparerTests.java        |  12 +-
 .../common/logging/config/logging.yml         |   7 +-
 dev-tools/smoke_test_rc.py                    |   2 +-
 .../src/main/packaging/init.d/elasticsearch   |   2 +-
 .../src/main/packaging/init.d/elasticsearch   |   2 +-
 .../packaging/systemd/elasticsearch.service   |   6 +-
 .../main/resources/bin/elasticsearch-plugin   |   6 +-
 .../src/main/resources/config/logging.yml     |   7 +-
 docs/plugins/plugin-script.asciidoc           |   2 +-
 docs/reference/getting-started.asciidoc       |   2 +-
 .../allocation/filtering.asciidoc             |   2 +-
 .../migration/migrate_5_0/packaging.asciidoc  |  12 ++
 .../migration/migrate_5_0/settings.asciidoc   |  11 +-
 .../cluster/allocation_awareness.asciidoc     |   2 +-
 docs/reference/modules/node.asciidoc          |   2 +-
 docs/reference/setup/configuration.asciidoc   |   9 +-
 docs/reference/setup/install/windows.asciidoc |   7 +-
 .../setup/install/zip-targz.asciidoc          |   2 +-
 .../bootstrap/EvilElasticsearchCliTests.java  |  62 ++++++
 .../plugins/InstallPluginCommandTests.java    | 195 ++++++++----------
 .../plugins/ListPluginsCommandTests.java      |  58 +++---
 .../plugins/RemovePluginCommandTests.java     |  42 ++--
 .../elasticsearch/tribe/TribeUnitTests.java   |  19 --
 .../smoketest/ESSmokeClientTestCase.java      |   1 -
 .../scripts/module_and_plugin_test_cases.bash |   2 +-
 .../scripts/packaging_test_utils.bash         |   2 +-
 .../bootstrap/ESElasticsearchCliTestCase.java |  65 ++++++
 .../test/ESSingleNodeTestCase.java            |   1 -
 .../org/elasticsearch/test/ExternalNode.java  |   1 -
 .../test/ExternalTestCluster.java             |   1 -
 .../test/InternalTestCluster.java             |  16 +-
 .../junit/listeners/ReproduceInfoPrinter.java |   4 +-
 .../src/main/resources/log4j.properties       |   4 +-
 55 files changed, 554 insertions(+), 495 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/cli/SettingCommand.java
 create mode 100644 qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilElasticsearchCliTests.java
 create mode 100644 test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java

diff --git a/TESTING.asciidoc b/TESTING.asciidoc
index 5eea0b8c163..af46c2e567b 100644
--- a/TESTING.asciidoc
+++ b/TESTING.asciidoc
@@ -201,7 +201,7 @@ gradle test -Dtests.timeoutSuite=5000! ...
 Change the logging level of ES (not gradle)
 
 --------------------------------
-gradle test -Des.logger.level=DEBUG
+gradle test -Dtests.logger.level=DEBUG
 --------------------------------
 
 Print all the logging output from the test runs to the commandline
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index 029c80b6e25..3207aa73613 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -456,7 +456,7 @@ class BuildPlugin implements Plugin<Project> {
             // default test sysprop values
             systemProperty 'tests.ifNoTests', 'fail'
             // TODO: remove setting logging level via system property
-            systemProperty 'es.logger.level', 'WARN'
+            systemProperty 'tests.logger.level', 'WARN'
             for (Map.Entry<String, String> property : System.properties.entrySet()) {
                 if (property.getKey().startsWith('tests.') ||
                     property.getKey().startsWith('es.')) {
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
index 2ff5e333139..014686e0207 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
@@ -129,7 +129,7 @@ class NodeInfo {
         }
 
         env = [ 'JAVA_HOME' : project.javaHome ]
-        args.addAll("-E", "es.node.portsfile=true")
+        args.addAll("-E", "node.portsfile=true")
         String collectedSystemProperties = config.systemProperties.collect { key, value -> "-D${key}=${value}" }.join(" ")
         String esJavaOpts = config.jvmArgs.isEmpty() ? collectedSystemProperties : collectedSystemProperties + " " + config.jvmArgs
         env.put('ES_JAVA_OPTS', esJavaOpts)
@@ -140,7 +140,7 @@ class NodeInfo {
             }
         }
         env.put('ES_JVM_OPTIONS', new File(confDir, 'jvm.options'))
-        args.addAll("-E", "es.path.conf=${confDir}")
+        args.addAll("-E", "path.conf=${confDir}")
         if (Os.isFamily(Os.FAMILY_WINDOWS)) {
             args.add('"') // end the entire command, quoted
         }
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
index 6d35cafd088..305a4fd30ae 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
@@ -177,15 +177,7 @@ final class Bootstrap {
         // install SM after natives, shutdown hooks, etc.
         Security.configure(environment, BootstrapSettings.SECURITY_FILTER_BAD_DEFAULTS_SETTING.get(settings));
 
-        // We do not need to reload system properties here as we have already applied them in building the settings and
-        // reloading could cause multiple prompts to the user for values if a system property was specified with a prompt
-        // placeholder
-        Settings nodeSettings = Settings.builder()
-                .put(settings)
-                .put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING.getKey(), true)
-                .build();
-
-        node = new Node(nodeSettings) {
+        node = new Node(settings) {
             @Override
             protected void validateNodeBeforeAcceptingRequests(Settings settings, BoundTransportAddress boundTransportAddress) {
                 BootstrapCheck.check(settings, boundTransportAddress);
@@ -193,13 +185,13 @@ final class Bootstrap {
         };
     }
 
-    private static Environment initialSettings(boolean foreground, String pidFile) {
+    private static Environment initialSettings(boolean foreground, String pidFile, Map<String, String> esSettings) {
         Terminal terminal = foreground ? Terminal.DEFAULT : null;
         Settings.Builder builder = Settings.builder();
         if (Strings.hasLength(pidFile)) {
             builder.put(Environment.PIDFILE_SETTING.getKey(), pidFile);
         }
-        return InternalSettingsPreparer.prepareEnvironment(builder.build(), terminal);
+        return InternalSettingsPreparer.prepareEnvironment(builder.build(), terminal, esSettings);
     }
 
     private void start() {
@@ -233,11 +225,13 @@ final class Bootstrap {
         // Set the system property before anything has a chance to trigger its use
         initLoggerPrefix();
 
-        elasticsearchSettings(esSettings);
+        // force the class initializer for BootstrapInfo to run before
+        // the security manager is installed
+        BootstrapInfo.init();
 
         INSTANCE = new Bootstrap();
 
-        Environment environment = initialSettings(foreground, pidFile);
+        Environment environment = initialSettings(foreground, pidFile, esSettings);
         Settings settings = environment.settings();
         LogConfigurator.configure(settings, true);
         checkForCustomConfFile();
@@ -295,13 +289,6 @@ final class Bootstrap {
         }
     }
 
-    @SuppressForbidden(reason = "Sets system properties passed as CLI parameters")
-    private static void elasticsearchSettings(Map<String, String> esSettings) {
-        for (Map.Entry<String, String> esSetting : esSettings.entrySet()) {
-            System.setProperty(esSetting.getKey(), esSetting.getValue());
-        }
-    }
-
     @SuppressForbidden(reason = "System#out")
     private static void closeSystOut() {
         System.out.close();
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java
index bd693951eb2..791836bf8a4 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapInfo.java
@@ -120,4 +120,8 @@ public final class BootstrapInfo {
         }
         return SYSTEM_PROPERTIES;
     }
+
+    public static void init() {
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
index bb1f6cc87d5..b3259129473 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
@@ -21,28 +21,25 @@ package org.elasticsearch.bootstrap;
 
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
-import joptsimple.util.KeyValuePair;
 import org.elasticsearch.Build;
-import org.elasticsearch.cli.Command;
 import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.SettingCommand;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserError;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.HashMap;
 import java.util.Map;
 
 /**
  * This class starts elasticsearch.
  */
-class Elasticsearch extends Command {
+class Elasticsearch extends SettingCommand {
 
     private final OptionSpec<Void> versionOption;
     private final OptionSpec<Void> daemonizeOption;
     private final OptionSpec<String> pidfileOption;
-    private final OptionSpec<KeyValuePair> propertyOption;
 
     // visible for testing
     Elasticsearch() {
@@ -56,7 +53,6 @@ class Elasticsearch extends Command {
         pidfileOption = parser.acceptsAll(Arrays.asList("p", "pidfile"),
             "Creates a pid file in the specified path on start")
             .withRequiredArg();
-        propertyOption = parser.accepts("E", "Configure an Elasticsearch setting").withRequiredArg().ofType(KeyValuePair.class);
     }
 
     /**
@@ -75,7 +71,7 @@ class Elasticsearch extends Command {
     }
 
     @Override
-    protected void execute(Terminal terminal, OptionSet options) throws Exception {
+    protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
         if (options.nonOptionArguments().isEmpty() == false) {
             throw new UserError(ExitCodes.USAGE, "Positional arguments not allowed, found " + options.nonOptionArguments());
         }
@@ -84,26 +80,15 @@ class Elasticsearch extends Command {
                 throw new UserError(ExitCodes.USAGE, "Elasticsearch version option is mutually exclusive with any other option");
             }
             terminal.println("Version: " + org.elasticsearch.Version.CURRENT
-                + ", Build: " + Build.CURRENT.shortHash() + "/" + Build.CURRENT.date()
-                + ", JVM: " + JvmInfo.jvmInfo().version());
+                    + ", Build: " + Build.CURRENT.shortHash() + "/" + Build.CURRENT.date()
+                    + ", JVM: " + JvmInfo.jvmInfo().version());
             return;
         }
 
         final boolean daemonize = options.has(daemonizeOption);
         final String pidFile = pidfileOption.value(options);
 
-        final Map<String, String> esSettings = new HashMap<>();
-        for (final KeyValuePair kvp : propertyOption.values(options)) {
-            if (!kvp.key.startsWith("es.")) {
-                throw new UserError(ExitCodes.USAGE, "Elasticsearch settings must be prefixed with [es.] but was [" + kvp.key + "]");
-            }
-            if (kvp.value.isEmpty()) {
-                throw new UserError(ExitCodes.USAGE, "Elasticsearch setting [" + kvp.key + "] must not be empty");
-            }
-            esSettings.put(kvp.key, kvp.value);
-        }
-
-        init(daemonize, pidFile, esSettings);
+        init(daemonize, pidFile, settings);
     }
 
     void init(final boolean daemonize, final String pidFile, final Map<String, String> esSettings) {
diff --git a/core/src/main/java/org/elasticsearch/cli/Command.java b/core/src/main/java/org/elasticsearch/cli/Command.java
index 1fc7c9fe74f..3e2faf13657 100644
--- a/core/src/main/java/org/elasticsearch/cli/Command.java
+++ b/core/src/main/java/org/elasticsearch/cli/Command.java
@@ -19,15 +19,15 @@
 
 package org.elasticsearch.cli;
 
-import java.io.IOException;
-import java.util.Arrays;
-
 import joptsimple.OptionException;
 import joptsimple.OptionParser;
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
 import org.elasticsearch.common.SuppressForbidden;
 
+import java.io.IOException;
+import java.util.Arrays;
+
 /**
  * An action to execute within a cli.
  */
@@ -112,4 +112,5 @@ public abstract class Command {
      *
      * Any runtime user errors (like an input file that does not exist), should throw a {@link UserError}. */
     protected abstract void execute(Terminal terminal, OptionSet options) throws Exception;
+
 }
diff --git a/core/src/main/java/org/elasticsearch/cli/SettingCommand.java b/core/src/main/java/org/elasticsearch/cli/SettingCommand.java
new file mode 100644
index 00000000000..868975ac6ff
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cli/SettingCommand.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cli;
+
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import joptsimple.util.KeyValuePair;
+
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+public abstract class SettingCommand extends Command {
+
+    private final OptionSpec<KeyValuePair> settingOption;
+
+    public SettingCommand(String description) {
+        super(description);
+        this.settingOption = parser.accepts("E", "Configure a setting").withRequiredArg().ofType(KeyValuePair.class);
+    }
+
+    @Override
+    protected void execute(Terminal terminal, OptionSet options) throws Exception {
+        final Map<String, String> settings = new HashMap<>();
+        for (final KeyValuePair kvp : settingOption.values(options)) {
+            if (kvp.value.isEmpty()) {
+                throw new UserError(ExitCodes.USAGE, "Setting [" + kvp.key + "] must not be empty");
+            }
+            settings.put(kvp.key, kvp.value);
+        }
+
+        putSystemPropertyIfSettingIsMissing(settings, "path.conf", "es.path.conf");
+        putSystemPropertyIfSettingIsMissing(settings, "path.data", "es.path.data");
+        putSystemPropertyIfSettingIsMissing(settings, "path.home", "es.path.home");
+        putSystemPropertyIfSettingIsMissing(settings, "path.logs", "es.path.logs");
+
+        execute(terminal, options, settings);
+    }
+
+    protected static void putSystemPropertyIfSettingIsMissing(final Map<String, String> settings, final String setting, final String key) {
+        final String value = System.getProperty(key);
+        if (value != null) {
+            if (settings.containsKey(setting)) {
+                final String message =
+                        String.format(
+                                Locale.ROOT,
+                                "duplicate setting [%s] found via command-line [%s] and system property [%s]",
+                                setting,
+                                settings.get(setting),
+                                value);
+                throw new IllegalArgumentException(message);
+            } else {
+                settings.put(setting, value);
+            }
+        }
+    }
+
+    protected abstract void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception;
+
+}
diff --git a/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java b/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
index eba89c2e02a..b29cab9fbfe 100644
--- a/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
+++ b/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
@@ -21,7 +21,6 @@ package org.elasticsearch.common.logging;
 
 import org.apache.log4j.PropertyConfigurator;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.bootstrap.BootstrapInfo;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.env.Environment;
@@ -93,8 +92,7 @@ public class LogConfigurator {
 
     /**
      * Consolidates settings and converts them into actual log4j settings, then initializes loggers and appenders.
-     *
-     * @param settings      custom settings that should be applied
+     *  @param settings      custom settings that should be applied
      * @param resolveConfig controls whether the logging conf file should be read too or not.
      */
     public static void configure(Settings settings, boolean resolveConfig) {
@@ -109,7 +107,7 @@ public class LogConfigurator {
         if (resolveConfig) {
             resolveConfig(environment, settingsBuilder);
         }
-        settingsBuilder.putProperties("es.", BootstrapInfo.getSystemProperties());
+
         // add custom settings after config was added so that they are not overwritten by config
         settingsBuilder.put(settings);
         settingsBuilder.replacePropertyPlaceholders();
diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index 5b6130281d4..36ee01484e6 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -375,7 +375,6 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     BaseRestHandler.MULTI_ALLOW_EXPLICIT_INDEX,
                     ClusterName.CLUSTER_NAME_SETTING,
                     Client.CLIENT_TYPE_SETTING_S,
-                    InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING,
                     ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING,
                     EsExecutors.PROCESSORS_SETTING,
                     ThreadContext.DEFAULT_HEADERS_SETTING,
diff --git a/core/src/main/java/org/elasticsearch/common/settings/Settings.java b/core/src/main/java/org/elasticsearch/common/settings/Settings.java
index 8488ca75c73..7a335aa1f32 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/Settings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/Settings.java
@@ -58,9 +58,11 @@ import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.common.unit.ByteSizeValue.parseBytesSizeValue;
 import static org.elasticsearch.common.unit.SizeValue.parseSizeValue;
@@ -942,66 +944,27 @@ public final class Settings implements ToXContent {
             return this;
         }
 
-        /**
-         * Puts all the properties with keys starting with the provided <tt>prefix</tt>.
-         *
-         * @param prefix     The prefix to filter property key by
-         * @param properties The properties to put
-         * @return The builder
-         */
-        public Builder putProperties(String prefix, Dictionary<Object, Object> properties) {
-            for (Object property : Collections.list(properties.keys())) {
-                String key = Objects.toString(property);
-                String value = Objects.toString(properties.get(property));
-                if (key.startsWith(prefix)) {
-                    map.put(key.substring(prefix.length()), value);
+        public Builder putProperties(Map<String, String> esSettings, Predicate<String> keyPredicate, Function<String, String> keyFunction) {
+            for (final Map.Entry<String, String> esSetting : esSettings.entrySet()) {
+                final String key = esSetting.getKey();
+                if (keyPredicate.test(key)) {
+                    map.put(keyFunction.apply(key), esSetting.getValue());
                 }
             }
             return this;
         }
 
         /**
-         * Puts all the properties with keys starting with the provided <tt>prefix</tt>.
-         *
-         * @param prefix     The prefix to filter property key by
-         * @param properties The properties to put
-         * @return The builder
-         */
-        public Builder putProperties(String prefix, Dictionary<Object, Object> properties, String ignorePrefix) {
-            for (Object property : Collections.list(properties.keys())) {
-                String key = Objects.toString(property);
-                String value = Objects.toString(properties.get(property));
-                if (key.startsWith(prefix)) {
-                    if (!key.startsWith(ignorePrefix)) {
-                        map.put(key.substring(prefix.length()), value);
-                    }
-                }
-            }
-            return this;
-        }
-
-        /**
-         * Runs across all the settings set on this builder and replaces <tt>${...}</tt> elements in the
-         * each setting value according to the following logic:
-         * <p>
-         * First, tries to resolve it against a System property ({@link System#getProperty(String)}), next,
-         * tries and resolve it against an environment variable ({@link System#getenv(String)}), and last, tries
-         * and replace it with another setting already set on this builder.
+         * Runs across all the settings set on this builder and
+         * replaces <tt>${...}</tt> elements in each setting with
+         * another setting already set on this builder.
          */
         public Builder replacePropertyPlaceholders() {
             PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false);
             PropertyPlaceholder.PlaceholderResolver placeholderResolver = new PropertyPlaceholder.PlaceholderResolver() {
                     @Override
                     public String resolvePlaceholder(String placeholderName) {
-                        if (placeholderName.startsWith("env.")) {
-                            // explicit env var prefix
-                            return System.getenv(placeholderName.substring("env.".length()));
-                        }
-                        String value = System.getProperty(placeholderName);
-                        if (value != null) {
-                            return value;
-                        }
-                        value = System.getenv(placeholderName);
+                        final String value = System.getenv(placeholderName);
                         if (value != null) {
                             return value;
                         }
@@ -1010,8 +973,7 @@ public final class Settings implements ToXContent {
 
                     @Override
                     public boolean shouldIgnoreMissing(String placeholderName) {
-                        // if its an explicit env var, we are ok with not having a value for it and treat it as optional
-                        if (placeholderName.startsWith("env.") || placeholderName.startsWith("prompt.")) {
+                        if (placeholderName.startsWith("prompt.")) {
                             return true;
                         }
                         return false;
diff --git a/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java b/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java
index 124c8b2dbdd..24a9cf589bd 100644
--- a/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java
+++ b/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java
@@ -19,14 +19,11 @@
 
 package org.elasticsearch.node.internal;
 
-import org.elasticsearch.bootstrap.BootstrapInfo;
+import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.env.Environment;
@@ -39,10 +36,13 @@ import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
+import java.util.function.Predicate;
 
 import static org.elasticsearch.common.Strings.cleanPath;
 
@@ -52,20 +52,18 @@ import static org.elasticsearch.common.Strings.cleanPath;
 public class InternalSettingsPreparer {
 
     private static final String[] ALLOWED_SUFFIXES = {".yml", ".yaml", ".json", ".properties"};
-    static final String PROPERTY_PREFIX = "es.";
-    static final String PROPERTY_DEFAULTS_PREFIX = "es.default.";
+    static final String PROPERTY_DEFAULTS_PREFIX = "default.";
+    static final Predicate<String> PROPERTY_DEFAULTS_PREDICATE = key -> key.startsWith(PROPERTY_DEFAULTS_PREFIX);
 
     public static final String SECRET_PROMPT_VALUE = "${prompt.secret}";
     public static final String TEXT_PROMPT_VALUE = "${prompt.text}";
-    public static final Setting<Boolean> IGNORE_SYSTEM_PROPERTIES_SETTING =
-        Setting.boolSetting("config.ignore_system_properties", false, Property.NodeScope);
 
     /**
      * Prepares the settings by gathering all elasticsearch system properties and setting defaults.
      */
     public static Settings prepareSettings(Settings input) {
         Settings.Builder output = Settings.builder();
-        initializeSettings(output, input, true);
+        initializeSettings(output, input, true, Collections.emptyMap());
         finalizeSettings(output, null, null);
         return output.build();
     }
@@ -80,9 +78,23 @@ public class InternalSettingsPreparer {
      * @return the {@link Settings} and {@link Environment} as a {@link Tuple}
      */
     public static Environment prepareEnvironment(Settings input, Terminal terminal) {
+        return prepareEnvironment(input, terminal, Collections.emptyMap());
+    }
+
+    /**
+     * Prepares the settings by gathering all elasticsearch system properties, optionally loading the configuration settings,
+     * and then replacing all property placeholders. If a {@link Terminal} is provided and configuration settings are loaded,
+     * settings with a value of <code>${prompt.text}</code> or <code>${prompt.secret}</code> will result in a prompt for
+     * the setting to the user.
+     * @param input The custom settings to use. These are not overwritten by settings in the configuration file.
+     * @param terminal the Terminal to use for input/output
+     * @param properties Map of properties key/value pairs (usually from the command-line)
+     * @return the {@link Settings} and {@link Environment} as a {@link Tuple}
+     */
+    public static Environment prepareEnvironment(Settings input, Terminal terminal, Map<String, String> properties) {
         // just create enough settings to build the environment, to get the config dir
         Settings.Builder output = Settings.builder();
-        initializeSettings(output, input, true);
+        initializeSettings(output, input, true, properties);
         Environment environment = new Environment(output.build());
 
         boolean settingsFileFound = false;
@@ -103,7 +115,7 @@ public class InternalSettingsPreparer {
 
         // re-initialize settings now that the config file has been loaded
         // TODO: only re-initialize if a config file was actually loaded
-        initializeSettings(output, input, false);
+        initializeSettings(output, input, false, properties);
         finalizeSettings(output, terminal, environment.configFile());
 
         environment = new Environment(output.build());
@@ -113,22 +125,16 @@ public class InternalSettingsPreparer {
         return new Environment(output.build());
     }
 
-    private static boolean useSystemProperties(Settings input) {
-        return !IGNORE_SYSTEM_PROPERTIES_SETTING.get(input);
-    }
-
     /**
      * Initializes the builder with the given input settings, and loads system properties settings if allowed.
      * If loadDefaults is true, system property default settings are loaded.
      */
-    private static void initializeSettings(Settings.Builder output, Settings input, boolean loadDefaults) {
+    private static void initializeSettings(Settings.Builder output, Settings input, boolean loadDefaults, Map<String, String> esSettings) {
         output.put(input);
-        if (useSystemProperties(input)) {
-            if (loadDefaults) {
-                output.putProperties(PROPERTY_DEFAULTS_PREFIX, BootstrapInfo.getSystemProperties());
-            }
-            output.putProperties(PROPERTY_PREFIX, BootstrapInfo.getSystemProperties(), PROPERTY_DEFAULTS_PREFIX);
+        if (loadDefaults) {
+            output.putProperties(esSettings, PROPERTY_DEFAULTS_PREDICATE, key -> key.substring(PROPERTY_DEFAULTS_PREFIX.length()));
         }
+        output.putProperties(esSettings, PROPERTY_DEFAULTS_PREDICATE.negate(), Function.identity());
         output.replacePropertyPlaceholders();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
index e81d376f3b5..645d07bfb64 100644
--- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
@@ -27,11 +27,14 @@ import org.elasticsearch.Version;
 import org.elasticsearch.bootstrap.JarHell;
 import org.elasticsearch.cli.Command;
 import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.SettingCommand;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserError;
 import org.elasticsearch.common.hash.MessageDigests;
 import org.elasticsearch.common.io.FileSystemUtils;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.internal.InternalSettingsPreparer;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -56,6 +59,7 @@ import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.zip.ZipEntry;
@@ -95,7 +99,7 @@ import static org.elasticsearch.common.util.set.Sets.newHashSet;
  * elasticsearch config directory, using the name of the plugin. If any files to be installed
  * already exist, they will be skipped.
  */
-class InstallPluginCommand extends Command {
+class InstallPluginCommand extends SettingCommand {
 
     private static final String PROPERTY_SUPPORT_STAGING_URLS = "es.plugins.staging";
 
@@ -132,7 +136,6 @@ class InstallPluginCommand extends Command {
             "store-smb",
             "x-pack")));
 
-    private final Environment env;
     private final OptionSpec<Void> batchOption;
     private final OptionSpec<String> arguments;
 
@@ -160,9 +163,8 @@ class InstallPluginCommand extends Command {
         FILE_PERMS = Collections.unmodifiableSet(filePerms);
     }
 
-    InstallPluginCommand(Environment env) {
+    InstallPluginCommand() {
         super("Install a plugin");
-        this.env = env;
         this.batchOption = parser.acceptsAll(Arrays.asList("b", "batch"),
                 "Enable batch mode explicitly, automatic confirmation of security permission");
         this.arguments = parser.nonOptions("plugin id");
@@ -178,7 +180,7 @@ class InstallPluginCommand extends Command {
     }
 
     @Override
-    protected void execute(Terminal terminal, OptionSet options) throws Exception {
+    protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
         // TODO: in jopt-simple 5.0 we can enforce a min/max number of positional args
         List<String> args = arguments.values(options);
         if (args.size() != 1) {
@@ -186,12 +188,12 @@ class InstallPluginCommand extends Command {
         }
         String pluginId = args.get(0);
         boolean isBatch = options.has(batchOption) || System.console() == null;
-        execute(terminal, pluginId, isBatch);
+        execute(terminal, pluginId, isBatch, settings);
     }
 
     // pkg private for testing
-    void execute(Terminal terminal, String pluginId, boolean isBatch) throws Exception {
-
+    void execute(Terminal terminal, String pluginId, boolean isBatch, Map<String, String> settings) throws Exception {
+        final Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, terminal, settings);
         // TODO: remove this leniency!! is it needed anymore?
         if (Files.exists(env.pluginsFile()) == false) {
             terminal.println("Plugins directory [" + env.pluginsFile() + "] does not exist. Creating...");
@@ -200,7 +202,7 @@ class InstallPluginCommand extends Command {
 
         Path pluginZip = download(terminal, pluginId, env.tmpFile());
         Path extractedZip = unzip(pluginZip, env.pluginsFile());
-        install(terminal, isBatch, extractedZip);
+        install(terminal, isBatch, extractedZip, env);
     }
 
     /** Downloads the plugin and returns the file it was downloaded to. */
@@ -349,7 +351,7 @@ class InstallPluginCommand extends Command {
     }
 
     /** Load information about the plugin, and verify it can be installed with no errors. */
-    private PluginInfo verify(Terminal terminal, Path pluginRoot, boolean isBatch) throws Exception {
+    private PluginInfo verify(Terminal terminal, Path pluginRoot, boolean isBatch, Environment env) throws Exception {
         // read and validate the plugin descriptor
         PluginInfo info = PluginInfo.readFromProperties(pluginRoot);
         terminal.println(VERBOSE, info.toString());
@@ -398,12 +400,12 @@ class InstallPluginCommand extends Command {
      * Installs the plugin from {@code tmpRoot} into the plugins dir.
      * If the plugin has a bin dir and/or a config dir, those are copied.
      */
-    private void install(Terminal terminal, boolean isBatch, Path tmpRoot) throws Exception {
+    private void install(Terminal terminal, boolean isBatch, Path tmpRoot, Environment env) throws Exception {
         List<Path> deleteOnFailure = new ArrayList<>();
         deleteOnFailure.add(tmpRoot);
 
         try {
-            PluginInfo info = verify(terminal, tmpRoot, isBatch);
+            PluginInfo info = verify(terminal, tmpRoot, isBatch, env);
 
             final Path destination = env.pluginsFile().resolve(info.getName());
             if (Files.exists(destination)) {
diff --git a/core/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java b/core/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java
index c03e70ad4da..bd2f853bac0 100644
--- a/core/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/ListPluginsCommand.java
@@ -19,6 +19,13 @@
 
 package org.elasticsearch.plugins;
 
+import joptsimple.OptionSet;
+import org.elasticsearch.cli.SettingCommand;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.internal.InternalSettingsPreparer;
+
 import java.io.IOException;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
@@ -26,26 +33,20 @@ import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-
-import joptsimple.OptionSet;
-import org.elasticsearch.cli.Command;
-import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.env.Environment;
+import java.util.Map;
 
 /**
  * A command for the plugin cli to list plugins installed in elasticsearch.
  */
-class ListPluginsCommand extends Command {
+class ListPluginsCommand extends SettingCommand {
 
-    private final Environment env;
-
-    ListPluginsCommand(Environment env) {
+    ListPluginsCommand() {
         super("Lists installed elasticsearch plugins");
-        this.env = env;
     }
 
     @Override
-    protected void execute(Terminal terminal, OptionSet options) throws Exception {
+    protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
+        final Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, terminal, settings);
         if (Files.exists(env.pluginsFile()) == false) {
             throw new IOException("Plugins directory missing: " + env.pluginsFile());
         }
diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginCli.java b/core/src/main/java/org/elasticsearch/plugins/PluginCli.java
index be06ea7db1c..3a88c4d0083 100644
--- a/core/src/main/java/org/elasticsearch/plugins/PluginCli.java
+++ b/core/src/main/java/org/elasticsearch/plugins/PluginCli.java
@@ -26,21 +26,24 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
 
+import java.util.Collections;
+
 /**
  * A cli tool for adding, removing and listing plugins for elasticsearch.
  */
 public class PluginCli extends MultiCommand {
 
-    public PluginCli(Environment env) {
+    public PluginCli() {
         super("A tool for managing installed elasticsearch plugins");
-        subcommands.put("list", new ListPluginsCommand(env));
-        subcommands.put("install", new InstallPluginCommand(env));
-        subcommands.put("remove", new RemovePluginCommand(env));
+        subcommands.put("list", new ListPluginsCommand());
+        subcommands.put("install", new InstallPluginCommand());
+        subcommands.put("remove", new RemovePluginCommand());
     }
 
     public static void main(String[] args) throws Exception {
         // initialize default for es.logger.level because we will not read the logging.yml
         String loggerLevel = System.getProperty("es.logger.level", "INFO");
+        String pathHome = System.getProperty("es.path.home");
         // Set the appender for all potential log files to terminal so that other components that use the logger print out the
         // same terminal.
         // The reason for this is that the plugin cli cannot be configured with a file appender because when the plugin command is
@@ -48,12 +51,14 @@ public class PluginCli extends MultiCommand {
         // is run as service then the logs should be at /var/log/elasticsearch but when started from the tar they should be at es.home/logs.
         // Therefore we print to Terminal.
         Environment loggingEnvironment = InternalSettingsPreparer.prepareEnvironment(Settings.builder()
+                .put("path.home", pathHome)
                 .put("appender.terminal.type", "terminal")
-                .put("rootLogger", "${es.logger.level}, terminal")
-                .put("es.logger.level", loggerLevel)
+                .put("rootLogger", "${logger.level}, terminal")
+                .put("logger.level", loggerLevel)
                 .build(), Terminal.DEFAULT);
         LogConfigurator.configure(loggingEnvironment.settings(), false);
-        Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, Terminal.DEFAULT);
-        exit(new PluginCli(env).main(args, Terminal.DEFAULT));
+
+        exit(new PluginCli().main(args, Terminal.DEFAULT));
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java
index a3e6c375f83..af48c1d8207 100644
--- a/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java
@@ -24,45 +24,49 @@ import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.cli.Command;
 import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.SettingCommand;
 import org.elasticsearch.cli.UserError;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.internal.InternalSettingsPreparer;
 
 import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE;
 
 /**
  * A command for the plugin cli to remove a plugin from elasticsearch.
  */
-class RemovePluginCommand extends Command {
+class RemovePluginCommand extends SettingCommand {
 
-    private final Environment env;
     private final OptionSpec<String> arguments;
 
-    RemovePluginCommand(Environment env) {
+    RemovePluginCommand() {
         super("Removes a plugin from elasticsearch");
-        this.env = env;
         this.arguments = parser.nonOptions("plugin name");
     }
 
     @Override
-    protected void execute(Terminal terminal, OptionSet options) throws Exception {
+    protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
         // TODO: in jopt-simple 5.0 we can enforce a min/max number of positional args
         List<String> args = arguments.values(options);
         if (args.size() != 1) {
             throw new UserError(ExitCodes.USAGE, "Must supply a single plugin id argument");
         }
-        execute(terminal, args.get(0));
+        execute(terminal, args.get(0), settings);
     }
 
     // pkg private for testing
-    void execute(Terminal terminal, String pluginName) throws Exception {
+    void execute(Terminal terminal, String pluginName, Map<String, String> settings) throws Exception {
+        final Environment env = InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, terminal, settings);
+
         terminal.println("-> Removing " + Strings.coalesceToEmpty(pluginName) + "...");
 
         Path pluginDir = env.pluginsFile().resolve(pluginName);
diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
index 74404903e91..32f7b2bf0dd 100644
--- a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
+++ b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
@@ -71,7 +71,7 @@ grant {
 
   // set by ESTestCase to improve test reproducibility
   // TODO: set this with gradle or some other way that repros with seed?
-  permission java.util.PropertyPermission "es.processors.override", "write";
+  permission java.util.PropertyPermission "processors.override", "write";
 
   // TODO: these simply trigger a noisy warning if its unable to clear the properties
   // fix that in randomizedtesting
diff --git a/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java b/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java
index 632646146fe..8b8a4d947a9 100644
--- a/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java
+++ b/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java
@@ -22,25 +22,15 @@ package org.elasticsearch.bootstrap;
 import org.elasticsearch.Build;
 import org.elasticsearch.Version;
 import org.elasticsearch.cli.ExitCodes;
-import org.elasticsearch.cli.MockTerminal;
-import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.monitor.jvm.JvmInfo;
-import org.elasticsearch.test.ESTestCase;
-import org.junit.After;
-import org.junit.Before;
 
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.hasEntry;
 
-public class ElasticsearchCliTests extends ESTestCase {
+public class ElasticsearchCliTests extends ESElasticsearchCliTestCase {
 
     public void testVersion() throws Exception {
         runTestThatVersionIsMutuallyExclusiveToOtherOptions("-V", "-d");
@@ -96,7 +86,7 @@ public class ElasticsearchCliTests extends ESTestCase {
             false,
             output -> assertThat(output, containsString("Positional arguments not allowed, found [foo]")),
             (foreground, pidFile, esSettings) -> {},
-            "-E", "something", "foo", "-E", "somethingelse"
+            "-E", "foo=bar", "foo", "-E", "baz=qux"
         );
     }
 
@@ -138,26 +128,10 @@ public class ElasticsearchCliTests extends ESTestCase {
                 output -> {},
                 (foreground, pidFile, esSettings) -> {
                     assertThat(esSettings.size(), equalTo(2));
-                    assertThat(esSettings, hasEntry("es.foo", "bar"));
-                    assertThat(esSettings, hasEntry("es.baz", "qux"));
+                    assertThat(esSettings, hasEntry("foo", "bar"));
+                    assertThat(esSettings, hasEntry("baz", "qux"));
                 },
-                "-Ees.foo=bar", "-E", "es.baz=qux"
-        );
-    }
-
-    public void testElasticsearchSettingPrefix() throws Exception {
-        runElasticsearchSettingPrefixTest("-E", "foo");
-        runElasticsearchSettingPrefixTest("-E", "foo=bar");
-        runElasticsearchSettingPrefixTest("-E", "=bar");
-    }
-
-    private void runElasticsearchSettingPrefixTest(String... args) throws Exception {
-        runTest(
-                ExitCodes.USAGE,
-                false,
-                output -> assertThat(output, containsString("Elasticsearch settings must be prefixed with [es.] but was [")),
-                (foreground, pidFile, esSettings) -> {},
-                args
+                "-Efoo=bar", "-E", "baz=qux"
         );
     }
 
@@ -165,9 +139,9 @@ public class ElasticsearchCliTests extends ESTestCase {
         runTest(
                 ExitCodes.USAGE,
                 false,
-                output -> assertThat(output, containsString("Elasticsearch setting [es.foo] must not be empty")),
+                output -> assertThat(output, containsString("Setting [foo] must not be empty")),
                 (foreground, pidFile, esSettings) -> {},
-                "-E", "es.foo="
+                "-E", "foo="
         );
     }
 
@@ -180,36 +154,4 @@ public class ElasticsearchCliTests extends ESTestCase {
                 "--network.host");
     }
 
-    private interface InitConsumer {
-        void accept(final boolean foreground, final String pidFile, final Map<String, String> esSettings);
-    }
-
-    private void runTest(
-            final int expectedStatus,
-            final boolean expectedInit,
-            final Consumer<String> outputConsumer,
-            final InitConsumer initConsumer,
-            String... args) throws Exception {
-        final MockTerminal terminal = new MockTerminal();
-        try {
-            final AtomicBoolean init = new AtomicBoolean();
-            final int status = Elasticsearch.main(args, new Elasticsearch() {
-                @Override
-                void init(final boolean daemonize, final String pidFile, final Map<String, String> esSettings) {
-                    init.set(true);
-                    initConsumer.accept(!daemonize, pidFile, esSettings);
-                }
-            }, terminal);
-            assertThat(status, equalTo(expectedStatus));
-            assertThat(init.get(), equalTo(expectedInit));
-            outputConsumer.accept(terminal.getOutput());
-        } catch (Throwable t) {
-            // if an unexpected exception is thrown, we log
-            // terminal output to aid debugging
-            logger.info(terminal.getOutput());
-            // rethrow so the test fails
-            throw t;
-        }
-    }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientIT.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientIT.java
index a20a5247ed6..9cdeef2a7ff 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientIT.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientIT.java
@@ -59,7 +59,6 @@ public class TransportClientIT extends ESIntegTestCase {
                 .put("http.enabled", false)
                 .put(Node.NODE_DATA_SETTING.getKey(), false)
                 .put("cluster.name", "foobar")
-                .put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING.getKey(), true) // make sure we get what we set :)
                 .build());
         node.start();
         try {
diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientRetryIT.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientRetryIT.java
index 2fcadb51a10..4ec1f66df57 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientRetryIT.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientRetryIT.java
@@ -55,7 +55,6 @@ public class TransportClientRetryIT extends ESIntegTestCase {
                 .put("node.name", "transport_client_retry_test")
                 .put(Node.NODE_MODE_SETTING.getKey(), internalCluster().getNodeMode())
                 .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), internalCluster().getClusterName())
-                .put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING.getKey(), true)
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir());
 
         try (TransportClient client = TransportClient.builder().settings(builder.build()).build()) {
diff --git a/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java b/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
index aa8614aee71..3afd60d86e4 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
@@ -302,11 +302,8 @@ public class ScopedSettingsTests extends ESTestCase {
     public void testLoggingUpdates() {
         final String level = ESLoggerFactory.getRootLogger().getLevel();
         final String testLevel = ESLoggerFactory.getLogger("test").getLevel();
-        String property = System.getProperty("es.logger.level");
-        Settings.Builder builder = Settings.builder();
-        if (property != null) {
-            builder.put("logger.level", property);
-        }
+        String property = randomFrom(ESLoggerFactory.LogLevel.values()).toString();
+        Settings.Builder builder = Settings.builder().put("logger.level", property);
         try {
             ClusterSettings settings = new ClusterSettings(builder.build(), ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
             try {
@@ -319,7 +316,7 @@ public class ScopedSettingsTests extends ESTestCase {
             settings.applySettings(Settings.builder().put("logger._root", "TRACE").build());
             assertEquals("TRACE", ESLoggerFactory.getRootLogger().getLevel());
             settings.applySettings(Settings.builder().build());
-            assertEquals(level, ESLoggerFactory.getRootLogger().getLevel());
+            assertEquals(property, ESLoggerFactory.getRootLogger().getLevel());
             settings.applySettings(Settings.builder().put("logger.test", "TRACE").build());
             assertEquals("TRACE", ESLoggerFactory.getLogger("test").getLevel());
             settings.applySettings(Settings.builder().build());
diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
index 3539e54d943..fe7dcda3b25 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
@@ -31,7 +31,9 @@ import java.util.Set;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
@@ -42,31 +44,32 @@ public class SettingsTests extends ESTestCase {
         String value = System.getProperty("java.home");
         assertFalse(value.isEmpty());
         Settings settings = Settings.builder()
-                 .put("setting1", "${java.home}")
+                 .put("property.placeholder", value)
+                 .put("setting1", "${property.placeholder}")
                  .replacePropertyPlaceholders()
                  .build();
         assertThat(settings.get("setting1"), equalTo(value));
-
-        assertNull(System.getProperty("_test_property_should_not_exist"));
-        settings = Settings.builder()
-                .put("setting1", "${_test_property_should_not_exist:defaultVal1}")
-                .replacePropertyPlaceholders()
-                .build();
-        assertThat(settings.get("setting1"), equalTo("defaultVal1"));
-
-        settings = Settings.builder()
-                .put("setting1", "${_test_property_should_not_exist:}")
-                .replacePropertyPlaceholders()
-                .build();
-        assertThat(settings.get("setting1"), is(nullValue()));
     }
 
-    public void testReplacePropertiesPlaceholderIgnoreEnvUnset() {
-        Settings settings = Settings.builder()
-                .put("setting1", "${env.UNSET_ENV_VAR}")
+    public void testReplacePropertiesPlaceholderSystemVariablesHaveNoEffect() {
+        final String value = System.getProperty("java.home");
+        assertNotNull(value);
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> Settings.builder()
+                .put("setting1", "${java.home}")
                 .replacePropertyPlaceholders()
-                .build();
-        assertThat(settings.get("setting1"), is(nullValue()));
+                .build());
+        assertThat(e, hasToString(containsString("Could not resolve placeholder 'java.home'")));
+    }
+
+    public void testReplacePropertiesPlaceholderByEnvironmentVariables() {
+        final Map.Entry<String, String> entry = randomSubsetOf(1, System.getenv().entrySet()).get(0);
+        assertNotNull(entry.getValue());
+
+        final Settings implicitEnvSettings = Settings.builder()
+            .put("setting1", "${" + entry.getKey() + "}")
+            .replacePropertyPlaceholders()
+            .build();
+        assertThat(implicitEnvSettings.get("setting1"), equalTo(entry.getValue()));
     }
 
     public void testReplacePropertiesPlaceholderIgnoresPrompt() {
diff --git a/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java b/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java
index e1397ca47f1..87abc20a0de 100644
--- a/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java
+++ b/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java
@@ -19,11 +19,6 @@
 
 package org.elasticsearch.node.internal;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-
 import org.elasticsearch.cli.MockTerminal;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.settings.Settings;
@@ -33,6 +28,11 @@ import org.elasticsearch.test.ESTestCase;
 import org.junit.After;
 import org.junit.Before;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -134,7 +134,6 @@ public class InternalSettingsPreparerTests extends ESTestCase {
             Files.createDirectory(config);
             Files.copy(garbage, config.resolve("elasticsearch.yml"));
             InternalSettingsPreparer.prepareEnvironment(Settings.builder()
-                .put("config.ignore_system_properties", true)
                 .put(baseEnvSettings)
                 .build(), null);
         } catch (SettingsException e) {
@@ -153,7 +152,6 @@ public class InternalSettingsPreparerTests extends ESTestCase {
 
         try {
             InternalSettingsPreparer.prepareEnvironment(Settings.builder()
-                .put("config.ignore_system_properties", true)
                 .put(baseEnvSettings)
                 .build(), null);
         } catch (SettingsException e) {
diff --git a/core/src/test/resources/org/elasticsearch/common/logging/config/logging.yml b/core/src/test/resources/org/elasticsearch/common/logging/config/logging.yml
index 515e4320fd2..548b186e46f 100644
--- a/core/src/test/resources/org/elasticsearch/common/logging/config/logging.yml
+++ b/core/src/test/resources/org/elasticsearch/common/logging/config/logging.yml
@@ -1,6 +1,7 @@
-# you can override this using by setting a system property, for example -Ees.logger.level=DEBUG
-es.logger.level: INFO
-rootLogger: ${es.logger.level}, console
+# you can override using a command-line parameter
+# -E logger.level=(ERROR|WARN|INFO|DEBUG|TRACE)
+logger.level: INFO
+rootLogger: ${logger.level}, console
 logger:
   test: TRACE, console
 
diff --git a/dev-tools/smoke_test_rc.py b/dev-tools/smoke_test_rc.py
index 39db9929a54..ac5e9afec47 100644
--- a/dev-tools/smoke_test_rc.py
+++ b/dev-tools/smoke_test_rc.py
@@ -203,7 +203,7 @@ def smoke_test_release(release, files, expected_hash, plugins):
       headers = {}
     print('  Starting elasticsearch deamon from [%s]' % es_dir)
     try:
-      run('%s; %s -Ees.node.name=smoke_tester -Ees.cluster.name=prepare_release -Ees.script.inline=true -Ees.script.stored=true -Ees.repositories.url.allowed_urls=http://snapshot.test* %s -Ees.pidfile=%s -Ees.node.portsfile=true'
+      run('%s; %s -Enode.name=smoke_tester -Ecluster.name=prepare_release -Escript.inline=true -Escript.stored=true -Erepositories.url.allowed_urls=http://snapshot.test* %s -Epidfile=%s -Enode.portsfile=true'
           % (java_exe(), es_run_path, '-d', os.path.join(es_dir, 'es-smoke.pid')))
       if not wait_for_node_startup(es_dir, header=headers):
         print("elasticsearch logs:")
diff --git a/distribution/deb/src/main/packaging/init.d/elasticsearch b/distribution/deb/src/main/packaging/init.d/elasticsearch
index a01643db2b3..fdeb6db308e 100755
--- a/distribution/deb/src/main/packaging/init.d/elasticsearch
+++ b/distribution/deb/src/main/packaging/init.d/elasticsearch
@@ -79,7 +79,7 @@ fi
 # Define other required variables
 PID_FILE="$PID_DIR/$NAME.pid"
 DAEMON=$ES_HOME/bin/elasticsearch
-DAEMON_OPTS="-d -p $PID_FILE -Ees.default.path.logs=$LOG_DIR -Ees.default.path.data=$DATA_DIR -Ees.default.path.conf=$CONF_DIR"
+DAEMON_OPTS="-d -p $PID_FILE -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR"
 
 export ES_JAVA_OPTS
 export JAVA_HOME
diff --git a/distribution/rpm/src/main/packaging/init.d/elasticsearch b/distribution/rpm/src/main/packaging/init.d/elasticsearch
index 7bcb5692a88..8f1d93dcbdc 100644
--- a/distribution/rpm/src/main/packaging/init.d/elasticsearch
+++ b/distribution/rpm/src/main/packaging/init.d/elasticsearch
@@ -114,7 +114,7 @@ start() {
     cd $ES_HOME
     echo -n $"Starting $prog: "
     # if not running, start it up here, usually something like "daemon $exec"
-    daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Ees.default.path.home=$ES_HOME -Ees.default.path.logs=$LOG_DIR -Ees.default.path.data=$DATA_DIR -Ees.default.path.conf=$CONF_DIR
+    daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR
     retval=$?
     echo
     [ $retval -eq 0 ] && touch $lockfile
diff --git a/distribution/src/main/packaging/systemd/elasticsearch.service b/distribution/src/main/packaging/systemd/elasticsearch.service
index ccbf4650a22..0c99464c4f6 100644
--- a/distribution/src/main/packaging/systemd/elasticsearch.service
+++ b/distribution/src/main/packaging/systemd/elasticsearch.service
@@ -21,9 +21,9 @@ ExecStartPre=/usr/share/elasticsearch/bin/elasticsearch-systemd-pre-exec
 
 ExecStart=/usr/share/elasticsearch/bin/elasticsearch \
                                                 -p ${PID_DIR}/elasticsearch.pid \
-                                                -Ees.default.path.logs=${LOG_DIR} \
-                                                -Ees.default.path.data=${DATA_DIR} \
-                                                -Ees.default.path.conf=${CONF_DIR}
+                                                -Edefault.path.logs=${LOG_DIR} \
+                                                -Edefault.path.data=${DATA_DIR} \
+                                                -Edefault.path.conf=${CONF_DIR}
 
 StandardOutput=journal
 StandardError=inherit
diff --git a/distribution/src/main/resources/bin/elasticsearch-plugin b/distribution/src/main/resources/bin/elasticsearch-plugin
index 8a3b6676a98..06f8c5b8c27 100755
--- a/distribution/src/main/resources/bin/elasticsearch-plugin
+++ b/distribution/src/main/resources/bin/elasticsearch-plugin
@@ -81,10 +81,10 @@ fi
 HOSTNAME=`hostname | cut -d. -f1`
 export HOSTNAME
 
-declare -a properties=(-Delasticsearch -Des.path.home="$ES_HOME")
+declare -a args=("$@")
 
 if [ -e "$CONF_DIR" ]; then
-  properties=("${properties[@]}" -Des.default.path.conf="$CONF_DIR")
+  args=("${args[@]}" -Edefault.path.conf="$CONF_DIR")
 fi
 
-exec "$JAVA" $ES_JAVA_OPTS "${properties[@]}" -cp "$ES_HOME/lib/*" org.elasticsearch.plugins.PluginCli "$@"
+exec "$JAVA" $ES_JAVA_OPTS -Delasticsearch -Des.path.home="$ES_HOME" -cp "$ES_HOME/lib/*" org.elasticsearch.plugins.PluginCli "${args[@]}"
diff --git a/distribution/src/main/resources/config/logging.yml b/distribution/src/main/resources/config/logging.yml
index 187e79cffa0..11cd181ebd0 100644
--- a/distribution/src/main/resources/config/logging.yml
+++ b/distribution/src/main/resources/config/logging.yml
@@ -1,6 +1,7 @@
-# you can override this using by setting a system property, for example -Ees.logger.level=DEBUG
-es.logger.level: INFO
-rootLogger: ${es.logger.level}, console, file
+# you can override using a command-line parameter
+# -E logger.level=(ERROR|WARN|INFO|DEBUG|TRACE)
+logger.level: INFO
+rootLogger: ${logger.level}, console, file
 logger:
   # log action execution errors for easier debugging
   action: DEBUG
diff --git a/docs/plugins/plugin-script.asciidoc b/docs/plugins/plugin-script.asciidoc
index 7cb7f396608..08ad129f22f 100644
--- a/docs/plugins/plugin-script.asciidoc
+++ b/docs/plugins/plugin-script.asciidoc
@@ -135,7 +135,7 @@ can do this as follows:
 
 [source,sh]
 ---------------------
-sudo bin/elasticsearch-plugin -Ees.path.conf=/path/to/custom/config/dir install <plugin name>
+sudo bin/elasticsearch-plugin -Epath.conf=/path/to/custom/config/dir install <plugin name>
 ---------------------
 
 You can also set the `CONF_DIR` environment variable to the custom config
diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc
index 977cb4e5a1d..132b287bd46 100755
--- a/docs/reference/getting-started.asciidoc
+++ b/docs/reference/getting-started.asciidoc
@@ -163,7 +163,7 @@ As mentioned previously, we can override either the cluster or node name. This c
 
 [source,sh]
 --------------------------------------------------
-./elasticsearch -Ees.cluster.name=my_cluster_name -Ees.node.name=my_node_name
+./elasticsearch -Ecluster.name=my_cluster_name -Enode.name=my_node_name
 --------------------------------------------------
 
 Also note the line marked http with information about the HTTP address (`192.168.8.112`) and port (`9200`) that our node is reachable from. By default, Elasticsearch uses port `9200` to provide access to its REST API. This port is configurable if necessary.
diff --git a/docs/reference/index-modules/allocation/filtering.asciidoc b/docs/reference/index-modules/allocation/filtering.asciidoc
index be45cd2a1ac..05007b46188 100644
--- a/docs/reference/index-modules/allocation/filtering.asciidoc
+++ b/docs/reference/index-modules/allocation/filtering.asciidoc
@@ -14,7 +14,7 @@ attribute as follows:
 
 [source,sh]
 ------------------------
-bin/elasticsearch -Ees.node.attr.rack=rack1 -Ees.node.attr.size=big  <1>
+bin/elasticsearch -Enode.attr.rack=rack1 -Enode.attr.size=big  <1>
 ------------------------
 <1> These attribute settings can also be specified in the `elasticsearch.yml` config file.
 
diff --git a/docs/reference/migration/migrate_5_0/packaging.asciidoc b/docs/reference/migration/migrate_5_0/packaging.asciidoc
index 5911b964b6b..977e20a76b1 100644
--- a/docs/reference/migration/migrate_5_0/packaging.asciidoc
+++ b/docs/reference/migration/migrate_5_0/packaging.asciidoc
@@ -43,3 +43,15 @@ Previously, the scripts used to start Elasticsearch and run plugin
 commands only required a Bourne-compatible shell. Starting in
 Elasticsearch 5.0.0, the bash shell is now required and `/bin/bash` is a
 hard-dependency for the RPM and Debian packages.
+
+==== Environmental Settings
+
+Previously, Elasticsearch could be configured via environment variables
+in two ways: first by using the placeholder syntax
+`${env.ENV_VAR_NAME}` and the second by using the same syntax without
+the `env` prefix: `${ENV_VAR_NAME}`. The first method has been removed
+from Elasticsearch.
+
+Additionally, it was previously possible to set any setting in
+Elasticsearch via JVM system properties. This has been removed from
+Elasticsearch.
diff --git a/docs/reference/migration/migrate_5_0/settings.asciidoc b/docs/reference/migration/migrate_5_0/settings.asciidoc
index 85895d65b67..0fa7d42e874 100644
--- a/docs/reference/migration/migrate_5_0/settings.asciidoc
+++ b/docs/reference/migration/migrate_5_0/settings.asciidoc
@@ -202,19 +202,14 @@ the cache implementation used for the request cache and the field data cache.
 
 ==== Using system properties to configure Elasticsearch
 
-Elasticsearch can be configured by setting system properties on the
-command line via `-Des.name.of.property=value.of.property`. This will be
-removed in a future version of Elasticsearch. Instead, use
-`-E es.name.of.setting=value.of.setting`. Note that in all cases the
-name of the setting must be prefixed with `es.`.
+Elasticsearch can no longer be configured by setting system properties.
+Instead, use `-Ename.of.setting=value.of.setting`.
 
 ==== Removed using double-dashes to configure Elasticsearch
 
 Elasticsearch could previously be configured on the command line by
 setting settings via `--name.of.setting value.of.setting`. This feature
-has been removed. Instead, use
-`-Ees.name.of.setting=value.of.setting`. Note that in all cases the
-name of the setting must be prefixed with `es.`.
+has been removed. Instead, use `-Ename.of.setting=value.of.setting`.
 
 ==== Discovery Settings
 
diff --git a/docs/reference/modules/cluster/allocation_awareness.asciidoc b/docs/reference/modules/cluster/allocation_awareness.asciidoc
index 383252e23b3..f4e61fb0da1 100644
--- a/docs/reference/modules/cluster/allocation_awareness.asciidoc
+++ b/docs/reference/modules/cluster/allocation_awareness.asciidoc
@@ -21,7 +21,7 @@ attribute called `rack_id` -- we could use any attribute name.  For example:
 
 [source,sh]
 ----------------------
-./bin/elasticsearch -Ees.node.attr.rack_id=rack_one <1>
+./bin/elasticsearch -Enode.attr.rack_id=rack_one <1>
 ----------------------
 <1> This setting could also be specified in the `elasticsearch.yml` config file.
 
diff --git a/docs/reference/modules/node.asciidoc b/docs/reference/modules/node.asciidoc
index 124d68f1d6d..2f1caa42ad8 100644
--- a/docs/reference/modules/node.asciidoc
+++ b/docs/reference/modules/node.asciidoc
@@ -265,7 +265,7 @@ Like all node settings, it can also be specified on the command line as:
 
 [source,sh]
 -----------------------
-./bin/elasticsearch -Ees.path.data=/var/elasticsearch/data
+./bin/elasticsearch -Epath.data=/var/elasticsearch/data
 -----------------------
 
 TIP: When using the `.zip` or `.tar.gz` distributions, the `path.data` setting
diff --git a/docs/reference/setup/configuration.asciidoc b/docs/reference/setup/configuration.asciidoc
index ceb3d8c38d3..68f73fc96b8 100644
--- a/docs/reference/setup/configuration.asciidoc
+++ b/docs/reference/setup/configuration.asciidoc
@@ -26,7 +26,7 @@ setting, as follows:
 
 [source,sh]
 -------------------------------
-./bin/elasticsearch -E es.path.conf=/path/to/my/config/
+./bin/elasticsearch -Epath.conf=/path/to/my/config/
 -------------------------------
 
 [float]
@@ -93,15 +93,14 @@ is used in the settings and the process is run as a service or in the background
 === Setting default settings
 
 New default settings may be specified on the command line using the
-`es.default.` prefix instead of the `es.` prefix.  This will specify a value
-that will be used by default unless another value is specified in the config
-file.
+`default.` prefix.  This will specify a value that will be used by
+default unless another value is specified in the config file.
 
 For instance, if Elasticsearch is started as follows:
 
 [source,sh]
 ---------------------------
-./bin/elasticsearch -E es.default.node.name=My_Node
+./bin/elasticsearch -Edefault.node.name=My_Node
 ---------------------------
 
 the value for `node.name` will be `My_Node`, unless it is overwritten on the
diff --git a/docs/reference/setup/install/windows.asciidoc b/docs/reference/setup/install/windows.asciidoc
index e3c7622d064..0d2e8bf04f6 100644
--- a/docs/reference/setup/install/windows.asciidoc
+++ b/docs/reference/setup/install/windows.asciidoc
@@ -45,15 +45,14 @@ file by default.  The format of this config file is explained in
 <<settings>>.
 
 Any settings that can be specified in the config file can also be specified on
-the command line, using the `-E` syntax, and prepending `es.` to the setting
-name, as follows:
+the command line, using the `-E` syntax as follows:
 
 [source,sh]
 --------------------------------------------
-./bin/elasticsearch -E es.cluster.name=my_cluster -E es.node.name=node_1
+./bin/elasticsearch -Ecluster.name=my_cluster -Enode.name=node_1
 --------------------------------------------
 
-NOTE: Values that contain spaces must be surrounded with quotes.  For instance `-E es.path.logs="C:\My Logs\logs"`.
+NOTE: Values that contain spaces must be surrounded with quotes.  For instance `-Epath.logs="C:\My Logs\logs"`.
 
 TIP: Typically, any cluster-wide settings (like `cluster.name`) should be
 added to the `elasticsearch.yml` config file, while any node-specific settings
diff --git a/docs/reference/setup/install/zip-targz.asciidoc b/docs/reference/setup/install/zip-targz.asciidoc
index 0ed67cb9ce9..7fc41a0f3f8 100644
--- a/docs/reference/setup/install/zip-targz.asciidoc
+++ b/docs/reference/setup/install/zip-targz.asciidoc
@@ -93,7 +93,7 @@ name, as follows:
 
 [source,sh]
 --------------------------------------------
-./bin/elasticsearch -d -E es.cluster.name=my_cluster -E es.node.name=node_1
+./bin/elasticsearch -d -Ecluster.name=my_cluster -Enode.name=node_1
 --------------------------------------------
 
 TIP: Typically, any cluster-wide settings (like `cluster.name`) should be
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilElasticsearchCliTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilElasticsearchCliTests.java
new file mode 100644
index 00000000000..8bd2451da57
--- /dev/null
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilElasticsearchCliTests.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.bootstrap;
+
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.hasEntry;
+
+public class EvilElasticsearchCliTests extends ESElasticsearchCliTestCase {
+
+    @SuppressForbidden(reason = "manipulates system properties for testing")
+    public void testPathHome() throws Exception {
+        final String pathHome = System.getProperty("es.path.home");
+        final String value = randomAsciiOfLength(16);
+        System.setProperty("es.path.home", value);
+
+        runTest(
+                ExitCodes.OK,
+                true,
+                output -> {},
+                (foreground, pidFile, esSettings) -> {
+                    assertThat(esSettings.size(), equalTo(1));
+                    assertThat(esSettings, hasEntry("path.home", value));
+                });
+
+        System.clearProperty("es.path.home");
+        final String commandLineValue = randomAsciiOfLength(16);
+        runTest(
+                ExitCodes.OK,
+                true,
+                output -> {},
+                (foreground, pidFile, esSettings) -> {
+                    assertThat(esSettings.size(), equalTo(1));
+                    assertThat(esSettings, hasEntry("path.home", commandLineValue));
+                },
+                "-Epath.home=" + commandLineValue);
+
+        if (pathHome != null) System.setProperty("es.path.home", pathHome);
+        else System.clearProperty("es.path.home");
+    }
+
+}
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
index af36d96f442..22b2ef39a88 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
@@ -27,6 +27,7 @@ import org.apache.lucene.util.SuppressForbidden;
 import org.elasticsearch.Version;
 import org.elasticsearch.cli.MockTerminal;
 import org.elasticsearch.cli.UserError;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.io.PathUtilsForTesting;
 import org.elasticsearch.common.settings.Settings;
@@ -54,8 +55,10 @@ import java.nio.file.attribute.PosixFileAttributeView;
 import java.nio.file.attribute.PosixFileAttributes;
 import java.nio.file.attribute.PosixFilePermission;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.function.Supplier;
@@ -129,7 +132,7 @@ public class InstallPluginCommandTests extends ESTestCase {
     }
 
     /** Creates a test environment with bin, config and plugins directories. */
-    static Environment createEnv(FileSystem fs, Function<String, Path> temp) throws IOException {
+    static Tuple<Path, Environment> createEnv(FileSystem fs, Function<String, Path> temp) throws IOException {
         Path home = temp.apply("install-plugin-command-tests");
         Files.createDirectories(home.resolve("bin"));
         Files.createFile(home.resolve("bin").resolve("elasticsearch"));
@@ -140,7 +143,7 @@ public class InstallPluginCommandTests extends ESTestCase {
         Settings settings = Settings.builder()
             .put("path.home", home)
             .build();
-        return new Environment(settings);
+        return Tuple.tuple(home, new Environment(settings));
     }
 
     static Path createPluginDir(Function<String, Path> temp) throws IOException {
@@ -185,20 +188,22 @@ public class InstallPluginCommandTests extends ESTestCase {
         return writeZip(structure, "elasticsearch");
     }
 
-    static MockTerminal installPlugin(String pluginUrl, Environment env) throws Exception {
-        return installPlugin(pluginUrl, env, false);
+    static MockTerminal installPlugin(String pluginUrl, Path home) throws Exception {
+        return installPlugin(pluginUrl, home, false);
     }
 
-    static MockTerminal installPlugin(String pluginUrl, Environment env, boolean jarHellCheck) throws Exception {
+    static MockTerminal installPlugin(String pluginUrl, Path home, boolean jarHellCheck) throws Exception {
+        Map<String, String> settings = new HashMap<>();
+        settings.put("path.home", home.toString());
         MockTerminal terminal = new MockTerminal();
-        new InstallPluginCommand(env) {
+        new InstallPluginCommand() {
             @Override
             void jarHellCheck(Path candidate, Path pluginsDir) throws Exception {
                 if (jarHellCheck) {
                     super.jarHellCheck(candidate, pluginsDir);
                 }
             }
-        }.execute(terminal, pluginUrl, true);
+        }.execute(terminal, pluginUrl, true, settings);
         return terminal;
     }
 
@@ -275,192 +280,176 @@ public class InstallPluginCommandTests extends ESTestCase {
     }
 
     public void testSomethingWorks() throws Exception {
-        Environment env = createEnv(fs, temp);
+        Tuple<Path, Environment> env = createEnv(fs, temp);
         Path pluginDir = createPluginDir(temp);
         String pluginZip = createPlugin("fake", pluginDir);
-        installPlugin(pluginZip, env);
-        assertPlugin("fake", pluginDir, env);
+        installPlugin(pluginZip, env.v1());
+        assertPlugin("fake", pluginDir, env.v2());
     }
 
     public void testSpaceInUrl() throws Exception {
-        Environment env = createEnv(fs, temp);
+        Tuple<Path, Environment> env = createEnv(fs, temp);
         Path pluginDir = createPluginDir(temp);
         String pluginZip = createPlugin("fake", pluginDir);
         Path pluginZipWithSpaces = createTempFile("foo bar", ".zip");
         try (InputStream in = new URL(pluginZip).openStream()) {
             Files.copy(in, pluginZipWithSpaces, StandardCopyOption.REPLACE_EXISTING);
         }
-        installPlugin(pluginZipWithSpaces.toUri().toURL().toString(), env);
-        assertPlugin("fake", pluginDir, env);
+        installPlugin(pluginZipWithSpaces.toUri().toURL().toString(), env.v1());
+        assertPlugin("fake", pluginDir, env.v2());
     }
 
     public void testMalformedUrlNotMaven() throws Exception {
-        Environment env = createEnv(fs, temp);
+        Tuple<Path, Environment> env = createEnv(fs, temp);
         // has two colons, so it appears similar to maven coordinates
-        MalformedURLException e = expectThrows(MalformedURLException.class, () -> {
-            installPlugin("://host:1234", env);
-        });
+        MalformedURLException e = expectThrows(MalformedURLException.class, () -> installPlugin("://host:1234", env.v1()));
         assertTrue(e.getMessage(), e.getMessage().contains("no protocol"));
     }
 
     public void testPluginsDirMissing() throws Exception {
-        Environment env = createEnv(fs, temp);
-        Files.delete(env.pluginsFile());
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Files.delete(env.v2().pluginsFile());
         Path pluginDir = createPluginDir(temp);
         String pluginZip = createPlugin("fake", pluginDir);
-        installPlugin(pluginZip, env);
-        assertPlugin("fake", pluginDir, env);
+        installPlugin(pluginZip, env.v1());
+        assertPlugin("fake", pluginDir, env.v2());
     }
 
     public void testPluginsDirReadOnly() throws Exception {
         assumeTrue("posix and filesystem", isPosix && isReal);
-        Environment env = createEnv(fs, temp);
+        Tuple<Path, Environment> env = createEnv(fs, temp);
         Path pluginDir = createPluginDir(temp);
-        try (PosixPermissionsResetter pluginsAttrs = new PosixPermissionsResetter(env.pluginsFile())) {
+        try (PosixPermissionsResetter pluginsAttrs = new PosixPermissionsResetter(env.v2().pluginsFile())) {
             pluginsAttrs.setPermissions(new HashSet<>());
             String pluginZip = createPlugin("fake", pluginDir);
-            IOException e = expectThrows(IOException.class, () -> {
-                installPlugin(pluginZip, env);
-            });
-            assertTrue(e.getMessage(), e.getMessage().contains(env.pluginsFile().toString()));
+            IOException e = expectThrows(IOException.class, () -> installPlugin(pluginZip, env.v1()));
+            assertTrue(e.getMessage(), e.getMessage().contains(env.v2().pluginsFile().toString()));
         }
-        assertInstallCleaned(env);
+        assertInstallCleaned(env.v2());
     }
 
     public void testBuiltinModule() throws Exception {
-        Environment env = createEnv(fs, temp);
+        Tuple<Path, Environment> env = createEnv(fs, temp);
         Path pluginDir = createPluginDir(temp);
         String pluginZip = createPlugin("lang-groovy", pluginDir);
-        UserError e = expectThrows(UserError.class, () -> {
-            installPlugin(pluginZip, env);
-        });
+        UserError e = expectThrows(UserError.class, () -> installPlugin(pluginZip, env.v1()));
         assertTrue(e.getMessage(), e.getMessage().contains("is a system module"));
-        assertInstallCleaned(env);
+        assertInstallCleaned(env.v2());
     }
 
     public void testJarHell() throws Exception {
         // jar hell test needs a real filesystem
         assumeTrue("real filesystem", isReal);
-        Environment environment = createEnv(fs, temp);
+        Tuple<Path, Environment> environment = createEnv(fs, temp);
         Path pluginDirectory = createPluginDir(temp);
         writeJar(pluginDirectory.resolve("other.jar"), "FakePlugin");
         String pluginZip = createPlugin("fake", pluginDirectory); // adds plugin.jar with FakePlugin
-        IllegalStateException e = expectThrows(IllegalStateException.class, () -> {
-            installPlugin(pluginZip, environment, true);
-        });
+        IllegalStateException e = expectThrows(IllegalStateException.class, () -> installPlugin(pluginZip, environment.v1(), true));
         assertTrue(e.getMessage(), e.getMessage().contains("jar hell"));
-        assertInstallCleaned(environment);
+        assertInstallCleaned(environment.v2());
     }
 
     public void testIsolatedPlugins() throws Exception {
-        Environment env = createEnv(fs, temp);
+        Tuple<Path, Environment> env = createEnv(fs, temp);
         // these both share the same FakePlugin class
         Path pluginDir1 = createPluginDir(temp);
         String pluginZip1 = createPlugin("fake1", pluginDir1);
-        installPlugin(pluginZip1, env);
+        installPlugin(pluginZip1, env.v1());
         Path pluginDir2 = createPluginDir(temp);
         String pluginZip2 = createPlugin("fake2", pluginDir2);
-        installPlugin(pluginZip2, env);
-        assertPlugin("fake1", pluginDir1, env);
-        assertPlugin("fake2", pluginDir2, env);
+        installPlugin(pluginZip2, env.v1());
+        assertPlugin("fake1", pluginDir1, env.v2());
+        assertPlugin("fake2", pluginDir2, env.v2());
     }
 
     public void testExistingPlugin() throws Exception {
-        Environment env = createEnv(fs, temp);
+        Tuple<Path, Environment> env = createEnv(fs, temp);
         Path pluginDir = createPluginDir(temp);
         String pluginZip = createPlugin("fake", pluginDir);
-        installPlugin(pluginZip, env);
-        UserError e = expectThrows(UserError.class, () -> {
-            installPlugin(pluginZip, env);
-        });
+        installPlugin(pluginZip, env.v1());
+        UserError e = expectThrows(UserError.class, () -> installPlugin(pluginZip, env.v1()));
         assertTrue(e.getMessage(), e.getMessage().contains("already exists"));
-        assertInstallCleaned(env);
+        assertInstallCleaned(env.v2());
     }
 
     public void testBin() throws Exception {
-        Environment env = createEnv(fs, temp);
+        Tuple<Path, Environment> env = createEnv(fs, temp);
         Path pluginDir = createPluginDir(temp);
         Path binDir = pluginDir.resolve("bin");
         Files.createDirectory(binDir);
         Files.createFile(binDir.resolve("somescript"));
         String pluginZip = createPlugin("fake", pluginDir);
-        installPlugin(pluginZip, env);
-        assertPlugin("fake", pluginDir, env);
+        installPlugin(pluginZip, env.v1());
+        assertPlugin("fake", pluginDir, env.v2());
     }
 
     public void testBinNotDir() throws Exception {
-        Environment env = createEnv(fs, temp);
+        Tuple<Path, Environment> env = createEnv(fs, temp);
         Path pluginDir = createPluginDir(temp);
         Path binDir = pluginDir.resolve("bin");
         Files.createFile(binDir);
         String pluginZip = createPlugin("fake", pluginDir);
-        UserError e = expectThrows(UserError.class, () -> {
-            installPlugin(pluginZip, env);
-        });
+        UserError e = expectThrows(UserError.class, () -> installPlugin(pluginZip, env.v1()));
         assertTrue(e.getMessage(), e.getMessage().contains("not a directory"));
-        assertInstallCleaned(env);
+        assertInstallCleaned(env.v2());
     }
 
     public void testBinContainsDir() throws Exception {
-        Environment env = createEnv(fs, temp);
+        Tuple<Path, Environment> env = createEnv(fs, temp);
         Path pluginDir = createPluginDir(temp);
         Path dirInBinDir = pluginDir.resolve("bin").resolve("foo");
         Files.createDirectories(dirInBinDir);
         Files.createFile(dirInBinDir.resolve("somescript"));
         String pluginZip = createPlugin("fake", pluginDir);
-        UserError e = expectThrows(UserError.class, () -> {
-            installPlugin(pluginZip, env);
-        });
+        UserError e = expectThrows(UserError.class, () -> installPlugin(pluginZip, env.v1()));
         assertTrue(e.getMessage(), e.getMessage().contains("Directories not allowed in bin dir for plugin"));
-        assertInstallCleaned(env);
+        assertInstallCleaned(env.v2());
     }
 
     public void testBinConflict() throws Exception {
-        Environment env = createEnv(fs, temp);
+        Tuple<Path, Environment> env = createEnv(fs, temp);
         Path pluginDir = createPluginDir(temp);
         Path binDir = pluginDir.resolve("bin");
         Files.createDirectory(binDir);
         Files.createFile(binDir.resolve("somescript"));
         String pluginZip = createPlugin("elasticsearch", pluginDir);
-        FileAlreadyExistsException e = expectThrows(FileAlreadyExistsException.class, () -> {
-            installPlugin(pluginZip, env);
-        });
-        assertTrue(e.getMessage(), e.getMessage().contains(env.binFile().resolve("elasticsearch").toString()));
-        assertInstallCleaned(env);
+        FileAlreadyExistsException e = expectThrows(FileAlreadyExistsException.class, () -> installPlugin(pluginZip, env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains(env.v2().binFile().resolve("elasticsearch").toString()));
+        assertInstallCleaned(env.v2());
     }
 
     public void testBinPermissions() throws Exception {
         assumeTrue("posix filesystem", isPosix);
-        Environment env = createEnv(fs, temp);
+        Tuple<Path, Environment> env = createEnv(fs, temp);
         Path pluginDir = createPluginDir(temp);
         Path binDir = pluginDir.resolve("bin");
         Files.createDirectory(binDir);
         Files.createFile(binDir.resolve("somescript"));
         String pluginZip = createPlugin("fake", pluginDir);
-        try (PosixPermissionsResetter binAttrs = new PosixPermissionsResetter(env.binFile())) {
+        try (PosixPermissionsResetter binAttrs = new PosixPermissionsResetter(env.v2().binFile())) {
             Set<PosixFilePermission> perms = binAttrs.getCopyPermissions();
             // make sure at least one execute perm is missing, so we know we forced it during installation
             perms.remove(PosixFilePermission.GROUP_EXECUTE);
             binAttrs.setPermissions(perms);
-            installPlugin(pluginZip, env);
-            assertPlugin("fake", pluginDir, env);
+            installPlugin(pluginZip, env.v1());
+            assertPlugin("fake", pluginDir, env.v2());
         }
     }
 
     public void testConfig() throws Exception {
-        Environment env = createEnv(fs, temp);
+        Tuple<Path, Environment> env = createEnv(fs, temp);
         Path pluginDir = createPluginDir(temp);
         Path configDir = pluginDir.resolve("config");
         Files.createDirectory(configDir);
         Files.createFile(configDir.resolve("custom.yaml"));
         String pluginZip = createPlugin("fake", pluginDir);
-        installPlugin(pluginZip, env);
-        assertPlugin("fake", pluginDir, env);
+        installPlugin(pluginZip, env.v1());
+        assertPlugin("fake", pluginDir, env.v2());
     }
 
     public void testExistingConfig() throws Exception {
-        Environment env = createEnv(fs, temp);
-        Path envConfigDir = env.configFile().resolve("fake");
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path envConfigDir = env.v2().configFile().resolve("fake");
         Files.createDirectories(envConfigDir);
         Files.write(envConfigDir.resolve("custom.yaml"), "existing config".getBytes(StandardCharsets.UTF_8));
         Path pluginDir = createPluginDir(temp);
@@ -469,8 +458,8 @@ public class InstallPluginCommandTests extends ESTestCase {
         Files.write(configDir.resolve("custom.yaml"), "new config".getBytes(StandardCharsets.UTF_8));
         Files.createFile(configDir.resolve("other.yaml"));
         String pluginZip = createPlugin("fake", pluginDir);
-        installPlugin(pluginZip, env);
-        assertPlugin("fake", pluginDir, env);
+        installPlugin(pluginZip, env.v1());
+        assertPlugin("fake", pluginDir, env.v2());
         List<String> configLines = Files.readAllLines(envConfigDir.resolve("custom.yaml"), StandardCharsets.UTF_8);
         assertEquals(1, configLines.size());
         assertEquals("existing config", configLines.get(0));
@@ -478,80 +467,68 @@ public class InstallPluginCommandTests extends ESTestCase {
     }
 
     public void testConfigNotDir() throws Exception {
-        Environment env = createEnv(fs, temp);
+        Tuple<Path, Environment> env = createEnv(fs, temp);
         Path pluginDir = createPluginDir(temp);
         Path configDir = pluginDir.resolve("config");
         Files.createFile(configDir);
         String pluginZip = createPlugin("fake", pluginDir);
-        UserError e = expectThrows(UserError.class, () -> {
-            installPlugin(pluginZip, env);
-        });
+        UserError e = expectThrows(UserError.class, () -> installPlugin(pluginZip, env.v1()));
         assertTrue(e.getMessage(), e.getMessage().contains("not a directory"));
-        assertInstallCleaned(env);
+        assertInstallCleaned(env.v2());
     }
 
     public void testConfigContainsDir() throws Exception {
-        Environment env = createEnv(fs, temp);
+        Tuple<Path, Environment> env = createEnv(fs, temp);
         Path pluginDir = createPluginDir(temp);
         Path dirInConfigDir = pluginDir.resolve("config").resolve("foo");
         Files.createDirectories(dirInConfigDir);
         Files.createFile(dirInConfigDir.resolve("myconfig.yml"));
         String pluginZip = createPlugin("fake", pluginDir);
-        UserError e = expectThrows(UserError.class, () -> {
-            installPlugin(pluginZip, env);
-        });
+        UserError e = expectThrows(UserError.class, () -> installPlugin(pluginZip, env.v1()));
         assertTrue(e.getMessage(), e.getMessage().contains("Directories not allowed in config dir for plugin"));
-        assertInstallCleaned(env);
+        assertInstallCleaned(env.v2());
     }
 
     public void testConfigConflict() throws Exception {
-        Environment env = createEnv(fs, temp);
+        Tuple<Path, Environment> env = createEnv(fs, temp);
         Path pluginDir = createPluginDir(temp);
         Path configDir = pluginDir.resolve("config");
         Files.createDirectory(configDir);
         Files.createFile(configDir.resolve("myconfig.yml"));
         String pluginZip = createPlugin("elasticsearch.yml", pluginDir);
-        FileAlreadyExistsException e = expectThrows(FileAlreadyExistsException.class, () -> {
-            installPlugin(pluginZip, env);
-        });
-        assertTrue(e.getMessage(), e.getMessage().contains(env.configFile().resolve("elasticsearch.yml").toString()));
-        assertInstallCleaned(env);
+        FileAlreadyExistsException e = expectThrows(FileAlreadyExistsException.class, () -> installPlugin(pluginZip, env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains(env.v2().configFile().resolve("elasticsearch.yml").toString()));
+        assertInstallCleaned(env.v2());
     }
 
     public void testMissingDescriptor() throws Exception {
-        Environment env = createEnv(fs, temp);
+        Tuple<Path, Environment> env = createEnv(fs, temp);
         Path pluginDir = createPluginDir(temp);
         Files.createFile(pluginDir.resolve("fake.yml"));
         String pluginZip = writeZip(pluginDir, "elasticsearch");
-        NoSuchFileException e = expectThrows(NoSuchFileException.class, () -> {
-            installPlugin(pluginZip, env);
-        });
+        NoSuchFileException e = expectThrows(NoSuchFileException.class, () -> installPlugin(pluginZip, env.v1()));
         assertTrue(e.getMessage(), e.getMessage().contains("plugin-descriptor.properties"));
-        assertInstallCleaned(env);
+        assertInstallCleaned(env.v2());
     }
 
     public void testMissingDirectory() throws Exception {
-        Environment env = createEnv(fs, temp);
+        Tuple<Path, Environment> env = createEnv(fs, temp);
         Path pluginDir = createPluginDir(temp);
         Files.createFile(pluginDir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES));
         String pluginZip = writeZip(pluginDir, null);
-        UserError e = expectThrows(UserError.class, () -> {
-            installPlugin(pluginZip, env);
-        });
+        UserError e = expectThrows(UserError.class, () -> installPlugin(pluginZip, env.v1()));
         assertTrue(e.getMessage(), e.getMessage().contains("`elasticsearch` directory is missing in the plugin zip"));
-        assertInstallCleaned(env);
+        assertInstallCleaned(env.v2());
     }
 
     public void testZipRelativeOutsideEntryName() throws Exception {
-        Environment env = createEnv(fs, temp);
+        Tuple<Path, Environment> env = createEnv(fs, temp);
         Path zip = createTempDir().resolve("broken.zip");
         try (ZipOutputStream stream = new ZipOutputStream(Files.newOutputStream(zip))) {
             stream.putNextEntry(new ZipEntry("elasticsearch/../blah"));
         }
         String pluginZip = zip.toUri().toURL().toString();
-        IOException e = expectThrows(IOException.class, () -> {
-            installPlugin(pluginZip, env);
-        });
+        IOException e = expectThrows(IOException.class, () -> installPlugin(pluginZip, env.v1()));
         assertTrue(e.getMessage(), e.getMessage().contains("resolving outside of plugin directory"));
     }
 
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java
index f26857e19af..1422280165c 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/ListPluginsCommandTests.java
@@ -25,35 +25,47 @@ import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.MockTerminal;
+import org.elasticsearch.common.inject.spi.HasDependencies;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.Version;
+import org.junit.Before;
 
 @LuceneTestCase.SuppressFileSystems("*")
 public class ListPluginsCommandTests extends ESTestCase {
 
-    Environment createEnv() throws IOException {
-        Path home = createTempDir();
+    private Path home;
+    private Environment env;
+
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        home = createTempDir();
         Files.createDirectories(home.resolve("plugins"));
         Settings settings = Settings.builder()
-            .put("path.home", home)
-            .build();
-        return new Environment(settings);
+                .put("path.home", home)
+                .build();
+        env = new Environment(settings);
     }
 
-    static MockTerminal listPlugins(Environment env) throws Exception {
-        return listPlugins(env, new String[0]);
+    static MockTerminal listPlugins(Path home) throws Exception {
+        return listPlugins(home, new String[0]);
     }
     
-    static MockTerminal listPlugins(Environment env, String[] args) throws Exception {
+    static MockTerminal listPlugins(Path home, String[] args) throws Exception {
+        String[] argsAndHome = new String[args.length + 1];
+        System.arraycopy(args, 0, argsAndHome, 0, args.length);
+        argsAndHome[args.length] = "-Epath.home=" + home;
         MockTerminal terminal = new MockTerminal();
-        int status = new ListPluginsCommand(env).main(args, terminal);
+        int status = new ListPluginsCommand().main(argsAndHome, terminal);
         assertEquals(ExitCodes.OK, status);
         return terminal;
     }
@@ -74,49 +86,42 @@ public class ListPluginsCommandTests extends ESTestCase {
 
 
     public void testPluginsDirMissing() throws Exception {
-        Environment env = createEnv();
         Files.delete(env.pluginsFile());
-        IOException e = expectThrows(IOException.class, () -> {
-           listPlugins(env);
-        });
+        IOException e = expectThrows(IOException.class, () -> listPlugins(home));
         assertEquals(e.getMessage(), "Plugins directory missing: " + env.pluginsFile());
     }
 
     public void testNoPlugins() throws Exception {
-        MockTerminal terminal = listPlugins(createEnv());
+        MockTerminal terminal = listPlugins(home);
         assertTrue(terminal.getOutput(), terminal.getOutput().isEmpty());
     }
 
     public void testOnePlugin() throws Exception {
-        Environment env = createEnv();
         buildFakePlugin(env, "fake desc", "fake", "org.fake");
-        MockTerminal terminal = listPlugins(env);
+        MockTerminal terminal = listPlugins(home);
         assertEquals(terminal.getOutput(), buildMultiline("fake"));
     }
 
     public void testTwoPlugins() throws Exception {
-        Environment env = createEnv();
         buildFakePlugin(env, "fake desc", "fake1", "org.fake");
         buildFakePlugin(env, "fake desc 2", "fake2", "org.fake");
-        MockTerminal terminal = listPlugins(env);
+        MockTerminal terminal = listPlugins(home);
         assertEquals(terminal.getOutput(), buildMultiline("fake1", "fake2"));
     }
     
     public void testPluginWithVerbose() throws Exception {
-        Environment env = createEnv();
         buildFakePlugin(env, "fake desc", "fake_plugin", "org.fake");
         String[] params = { "-v" };
-        MockTerminal terminal = listPlugins(env, params);
+        MockTerminal terminal = listPlugins(home, params);
         assertEquals(terminal.getOutput(), buildMultiline("Plugins directory: " + env.pluginsFile(), "fake_plugin",
                 "- Plugin information:", "Name: fake_plugin", "Description: fake desc", "Version: 1.0", " * Classname: org.fake"));
     }
     
     public void testPluginWithVerboseMultiplePlugins() throws Exception {
-        Environment env = createEnv();
         buildFakePlugin(env, "fake desc 1", "fake_plugin1", "org.fake");
         buildFakePlugin(env, "fake desc 2", "fake_plugin2", "org.fake2");
         String[] params = { "-v" };
-        MockTerminal terminal = listPlugins(env, params);
+        MockTerminal terminal = listPlugins(home, params);
         assertEquals(terminal.getOutput(), buildMultiline("Plugins directory: " + env.pluginsFile(),
                 "fake_plugin1", "- Plugin information:", "Name: fake_plugin1", "Description: fake desc 1", "Version: 1.0",
                 " * Classname: org.fake", "fake_plugin2", "- Plugin information:", "Name: fake_plugin2",
@@ -124,26 +129,23 @@ public class ListPluginsCommandTests extends ESTestCase {
     }
         
     public void testPluginWithoutVerboseMultiplePlugins() throws Exception {
-        Environment env = createEnv();
         buildFakePlugin(env, "fake desc 1", "fake_plugin1", "org.fake");
         buildFakePlugin(env, "fake desc 2", "fake_plugin2", "org.fake2");
-        MockTerminal terminal = listPlugins(env, new String[0]);
+        MockTerminal terminal = listPlugins(home, new String[0]);
         String output = terminal.getOutput();
         assertEquals(output, buildMultiline("fake_plugin1", "fake_plugin2"));
     }
     
     public void testPluginWithoutDescriptorFile() throws Exception{
-        Environment env = createEnv();
         Files.createDirectories(env.pluginsFile().resolve("fake1"));
-        NoSuchFileException e = expectThrows(NoSuchFileException.class, () -> listPlugins(env));
+        NoSuchFileException e = expectThrows(NoSuchFileException.class, () -> listPlugins(home));
         assertEquals(e.getFile(), env.pluginsFile().resolve("fake1").resolve(PluginInfo.ES_PLUGIN_PROPERTIES).toString());
     }
     
     public void testPluginWithWrongDescriptorFile() throws Exception{
-        Environment env = createEnv();        
         PluginTestUtil.writeProperties(env.pluginsFile().resolve("fake1"),
                 "description", "fake desc");
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> listPlugins(env));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> listPlugins(home));
         assertEquals(e.getMessage(), "Property [name] is missing in [" +
                 env.pluginsFile().resolve("fake1").resolve(PluginInfo.ES_PLUGIN_PROPERTIES).toString() + "]");
     }
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java
index d9d5661b834..6528bbc0911 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java
@@ -23,6 +23,8 @@ import java.io.IOException;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
 
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.cli.UserError;
@@ -30,25 +32,32 @@ import org.elasticsearch.cli.MockTerminal;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
 
 @LuceneTestCase.SuppressFileSystems("*")
 public class RemovePluginCommandTests extends ESTestCase {
 
-    /** Creates a test environment with bin, config and plugins directories. */
-    static Environment createEnv() throws IOException {
-        Path home = createTempDir();
+    private Path home;
+    private Environment env;
+
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        home = createTempDir();
         Files.createDirectories(home.resolve("bin"));
         Files.createFile(home.resolve("bin").resolve("elasticsearch"));
         Files.createDirectories(home.resolve("plugins"));
         Settings settings = Settings.builder()
-            .put("path.home", home)
-            .build();
-        return new Environment(settings);
+                .put("path.home", home)
+                .build();
+        env = new Environment(settings);
     }
 
-    static MockTerminal removePlugin(String name, Environment env) throws Exception {
+    static MockTerminal removePlugin(String name, Path home) throws Exception {
+        Map<String, String> settings = new HashMap<>();
+        settings.put("path.home", home.toString());
         MockTerminal terminal = new MockTerminal();
-        new RemovePluginCommand(env).execute(terminal, name);
+        new RemovePluginCommand().execute(terminal, name, settings);
         return terminal;
     }
 
@@ -63,33 +72,28 @@ public class RemovePluginCommandTests extends ESTestCase {
     }
 
     public void testMissing() throws Exception {
-        Environment env = createEnv();
-        UserError e = expectThrows(UserError.class, () -> {
-           removePlugin("dne", env);
-        });
+        UserError e = expectThrows(UserError.class, () -> removePlugin("dne", home));
         assertTrue(e.getMessage(), e.getMessage().contains("Plugin dne not found"));
         assertRemoveCleaned(env);
     }
 
     public void testBasic() throws Exception {
-        Environment env = createEnv();
         Files.createDirectory(env.pluginsFile().resolve("fake"));
         Files.createFile(env.pluginsFile().resolve("fake").resolve("plugin.jar"));
         Files.createDirectory(env.pluginsFile().resolve("fake").resolve("subdir"));
         Files.createDirectory(env.pluginsFile().resolve("other"));
-        removePlugin("fake", env);
+        removePlugin("fake", home);
         assertFalse(Files.exists(env.pluginsFile().resolve("fake")));
         assertTrue(Files.exists(env.pluginsFile().resolve("other")));
         assertRemoveCleaned(env);
     }
 
     public void testBin() throws Exception {
-        Environment env = createEnv();
         Files.createDirectories(env.pluginsFile().resolve("fake"));
         Path binDir = env.binFile().resolve("fake");
         Files.createDirectories(binDir);
         Files.createFile(binDir.resolve("somescript"));
-        removePlugin("fake", env);
+        removePlugin("fake", home);
         assertFalse(Files.exists(env.pluginsFile().resolve("fake")));
         assertTrue(Files.exists(env.binFile().resolve("elasticsearch")));
         assertFalse(Files.exists(binDir));
@@ -97,14 +101,12 @@ public class RemovePluginCommandTests extends ESTestCase {
     }
 
     public void testBinNotDir() throws Exception {
-        Environment env = createEnv();
         Files.createDirectories(env.pluginsFile().resolve("elasticsearch"));
-        UserError e = expectThrows(UserError.class, () -> {
-            removePlugin("elasticsearch", env);
-        });
+        UserError e = expectThrows(UserError.class, () -> removePlugin("elasticsearch", home));
         assertTrue(e.getMessage(), e.getMessage().contains("not a directory"));
         assertTrue(Files.exists(env.pluginsFile().resolve("elasticsearch"))); // did not remove
         assertTrue(Files.exists(env.binFile().resolve("elasticsearch")));
         assertRemoveCleaned(env);
     }
+
 }
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java
index 63c09890acc..f9cdf5b4f66 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java
@@ -84,29 +84,10 @@ public class TribeUnitTests extends ESTestCase {
         tribe2 = null;
     }
 
-    public void testThatTribeClientsIgnoreGlobalSysProps() throws Exception {
-        System.setProperty("es.cluster.name", "tribe_node_cluster");
-        System.setProperty("es.tribe.t1.cluster.name", "tribe1");
-        System.setProperty("es.tribe.t2.cluster.name", "tribe2");
-        System.setProperty("es.tribe.t1.node_id.seed", Long.toString(random().nextLong()));
-        System.setProperty("es.tribe.t2.node_id.seed", Long.toString(random().nextLong()));
-
-        try {
-            assertTribeNodeSuccessfullyCreated(Settings.EMPTY);
-        } finally {
-            System.clearProperty("es.cluster.name");
-            System.clearProperty("es.tribe.t1.cluster.name");
-            System.clearProperty("es.tribe.t2.cluster.name");
-            System.clearProperty("es.tribe.t1.node_id.seed");
-            System.clearProperty("es.tribe.t2.node_id.seed");
-        }
-    }
-
     public void testThatTribeClientsIgnoreGlobalConfig() throws Exception {
         Path pathConf = getDataPath("elasticsearch.yml").getParent();
         Settings settings = Settings
             .builder()
-            .put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING.getKey(), true)
             .put(Environment.PATH_CONF_SETTING.getKey(), pathConf)
             .build();
         assertTribeNodeSuccessfullyCreated(settings);
diff --git a/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java b/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java
index 9976f072d42..6297ce244f9 100644
--- a/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java
+++ b/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java
@@ -75,7 +75,6 @@ public abstract class ESSmokeClientTestCase extends LuceneTestCase {
     private static Client startClient(Path tempDir, TransportAddress... transportAddresses) {
         Settings clientSettings = Settings.builder()
                 .put("node.name", "qa_smoke_client_" + counter.getAndIncrement())
-                .put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING.getKey(), true) // prevents any settings to be replaced by system properties.
                 .put("client.transport.ignore_cluster_name", true)
                 .put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
                 .put(Node.NODE_MODE_SETTING.getKey(), "network").build(); // we require network here!
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
index 07fea76bd8b..362b5d60341 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
@@ -103,7 +103,7 @@ fi
         echo "CONF_FILE=$CONF_FILE" >> /etc/sysconfig/elasticsearch;
     fi
 
-    run_elasticsearch_service 1 -Ees.default.config="$CONF_FILE"
+    run_elasticsearch_service 1 -Edefault.config="$CONF_FILE"
 
     # remove settings again otherwise cleaning up before next testrun will fail
     if is_dpkg ; then
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash b/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash
index 5f50dfc2850..c4dc8c96f58 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/packaging_test_utils.bash
@@ -340,7 +340,7 @@ run_elasticsearch_service() {
             local CONF_DIR=""
             local ES_PATH_CONF=""
         else
-            local ES_PATH_CONF="-Ees.path.conf=$CONF_DIR"
+            local ES_PATH_CONF="-Epath.conf=$CONF_DIR"
         fi
         # we must capture the exit code to compare so we don't want to start as background process in case we expect something other than 0
         local background=""
diff --git a/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java b/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java
new file mode 100644
index 00000000000..aa327ae2546
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.bootstrap;
+
+import org.elasticsearch.cli.MockTerminal;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+
+abstract class ESElasticsearchCliTestCase extends ESTestCase {
+
+    interface InitConsumer {
+        void accept(final boolean foreground, final String pidFile, final Map<String, String> esSettings);
+    }
+
+    void runTest(
+            final int expectedStatus,
+            final boolean expectedInit,
+            final Consumer<String> outputConsumer,
+            final InitConsumer initConsumer,
+            String... args) throws Exception {
+        final MockTerminal terminal = new MockTerminal();
+        try {
+            final AtomicBoolean init = new AtomicBoolean();
+            final int status = Elasticsearch.main(args, new Elasticsearch() {
+                @Override
+                void init(final boolean daemonize, final String pidFile, final Map<String, String> esSettings) {
+                    init.set(true);
+                    initConsumer.accept(!daemonize, pidFile, esSettings);
+                }
+            }, terminal);
+            assertThat(status, equalTo(expectedStatus));
+            assertThat(init.get(), equalTo(expectedInit));
+            outputConsumer.accept(terminal.getOutput());
+        } catch (Throwable t) {
+            // if an unexpected exception is thrown, we log
+            // terminal output to aid debugging
+            logger.info(terminal.getOutput());
+            // rethrow so the test fails
+            throw t;
+        }
+    }
+
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
index 1e75f3d8261..7875f8fd20b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
@@ -185,7 +185,6 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
             .put("http.enabled", false)
             .put(Node.NODE_LOCAL_SETTING.getKey(), true)
             .put(Node.NODE_DATA_SETTING.getKey(), true)
-            .put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING.getKey(), true) // make sure we get what we set :)
             .put(nodeSettings()) // allow test cases to provide their own settings or override these
             .build();
         Node build = new MockNode(settings, getVersion(), getPlugins());
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java b/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java
index 4625aa77e25..efdd6bad90f 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java
@@ -51,7 +51,6 @@ import java.util.concurrent.TimeUnit;
 final class ExternalNode implements Closeable {
 
     public static final Settings REQUIRED_SETTINGS = Settings.builder()
-            .put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING.getKey(), true)
             .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "zen")
             .put(Node.NODE_MODE_SETTING.getKey(), "network").build(); // we need network mode for this
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java
index 5372c319dae..71fe622d8c0 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ExternalTestCluster.java
@@ -73,7 +73,6 @@ public final class ExternalTestCluster extends TestCluster {
         Settings clientSettings = Settings.builder()
                 .put(additionalSettings)
                 .put("node.name", InternalTestCluster.TRANSPORT_CLIENT_PREFIX + EXTERNAL_CLUSTER_PREFIX + counter.getAndIncrement())
-                .put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING.getKey(), true) // prevents any settings to be replaced by system properties.
                 .put("client.transport.ignore_cluster_name", true)
                 .put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
                 .put(Node.NODE_MODE_SETTING.getKey(), "network").build(); // we require network here!
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index 3199a27b9a5..995ca5480f3 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -291,11 +291,10 @@ public final class InternalTestCluster extends TestCluster {
         builder.put(Environment.PATH_REPO_SETTING.getKey(), baseDir.resolve("repos"));
         builder.put(TransportSettings.PORT.getKey(), TRANSPORT_BASE_PORT + "-" + (TRANSPORT_BASE_PORT + PORTS_PER_CLUSTER));
         builder.put("http.port", HTTP_BASE_PORT + "-" + (HTTP_BASE_PORT + PORTS_PER_CLUSTER));
-        builder.put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING.getKey(), true);
         builder.put(Node.NODE_MODE_SETTING.getKey(), nodeMode);
         builder.put("http.pipelining", enableHttpPipelining);
-        if (Strings.hasLength(System.getProperty("es.logger.level"))) {
-            builder.put("logger.level", System.getProperty("es.logger.level"));
+        if (Strings.hasLength(System.getProperty("tests.logger.level"))) {
+            builder.put("logger.level", System.getProperty("tests.logger.level"));
         }
         if (Strings.hasLength(System.getProperty("es.logger.prefix"))) {
             builder.put("logger.prefix", System.getProperty("es.logger.prefix"));
@@ -319,14 +318,14 @@ public final class InternalTestCluster extends TestCluster {
 
     public static String configuredNodeMode() {
         Builder builder = Settings.builder();
-        if (Strings.isEmpty(System.getProperty("es.node.mode")) && Strings.isEmpty(System.getProperty("es.node.local"))) {
+        if (Strings.isEmpty(System.getProperty("node.mode")) && Strings.isEmpty(System.getProperty("node.local"))) {
             return "local"; // default if nothing is specified
         }
-        if (Strings.hasLength(System.getProperty("es.node.mode"))) {
-            builder.put(Node.NODE_MODE_SETTING.getKey(), System.getProperty("es.node.mode"));
+        if (Strings.hasLength(System.getProperty("node.mode"))) {
+            builder.put(Node.NODE_MODE_SETTING.getKey(), System.getProperty("node.mode"));
         }
-        if (Strings.hasLength(System.getProperty("es.node.local"))) {
-            builder.put(Node.NODE_LOCAL_SETTING.getKey(), System.getProperty("es.node.local"));
+        if (Strings.hasLength(System.getProperty("node.local"))) {
+            builder.put(Node.NODE_LOCAL_SETTING.getKey(), System.getProperty("node.local"));
         }
         if (DiscoveryNode.isLocalNode(builder.build())) {
             return "local";
@@ -882,7 +881,6 @@ public final class InternalTestCluster extends TestCluster {
                     .put(Node.NODE_MODE_SETTING.getKey(), Node.NODE_MODE_SETTING.exists(nodeSettings) ? Node.NODE_MODE_SETTING.get(nodeSettings) : nodeMode)
                     .put("logger.prefix", nodeSettings.get("logger.prefix", ""))
                     .put("logger.level", nodeSettings.get("logger.level", "INFO"))
-                    .put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING.getKey(), true)
                     .put(settings);
 
             if (Node.NODE_LOCAL_SETTING.exists(nodeSettings)) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java
index 6142edb9394..75bc916ab94 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java
@@ -137,10 +137,10 @@ public class ReproduceInfoPrinter extends RunListener {
         }
 
         public ReproduceErrorMessageBuilder appendESProperties() {
-            appendProperties("es.logger.level");
+            appendProperties("tests.logger.level");
             if (inVerifyPhase()) {
                 // these properties only make sense for integration tests
-                appendProperties("es.node.mode", "es.node.local", TESTS_CLUSTER,
+                appendProperties("node.mode", "node.local", TESTS_CLUSTER,
                     ESIntegTestCase.TESTS_ENABLE_MOCK_MODULES);
             }
             appendProperties("tests.assertion.disabled", "tests.security.manager", "tests.nightly", "tests.jvms",
diff --git a/test/framework/src/main/resources/log4j.properties b/test/framework/src/main/resources/log4j.properties
index 22f54ef68e5..11a864df0f3 100644
--- a/test/framework/src/main/resources/log4j.properties
+++ b/test/framework/src/main/resources/log4j.properties
@@ -1,5 +1,5 @@
-es.logger.level=INFO
-log4j.rootLogger=${es.logger.level}, out
+tests.logger.level=INFO
+log4j.rootLogger=${tests.logger.level}, out
 
 log4j.logger.org.apache.http=INFO, out
 log4j.additivity.org.apache.http=false

From b9c7dbcfbd9c7862b4524d8dac35224f71a13e89 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Thu, 19 May 2016 15:25:31 -0400
Subject: [PATCH 0358/1311] nuke overflow detection as we cannot guarantee for
 def. simplify life :)

---
 .../painless/CompilerSettings.java            |  31 --
 .../elasticsearch/painless/MethodWriter.java  | 286 ++---------
 .../painless/PainlessScriptEngineService.java |   8 +-
 .../org/elasticsearch/painless/Utility.java   | 341 --------------
 .../painless/WriterConstants.java             |  52 --
 .../elasticsearch/painless/node/EBinary.java  |  62 +--
 .../elasticsearch/painless/node/EChain.java   |  10 +-
 .../elasticsearch/painless/node/EUnary.java   |  19 +-
 .../painless/FloatOverflowDisabledTests.java  | 293 ------------
 ...bledTests.java => FloatOverflowTests.java} |  13 +-
 .../IntegerOverflowDisabledTests.java         | 444 ------------------
 ...edTests.java => IntegerOverflowTests.java} |  13 +-
 .../painless/ScriptTestCase.java              |   2 +-
 .../elasticsearch/painless/UtilityTests.java  | 250 ----------
 14 files changed, 66 insertions(+), 1758 deletions(-)
 delete mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/FloatOverflowDisabledTests.java
 rename modules/lang-painless/src/test/java/org/elasticsearch/painless/{FloatOverflowEnabledTests.java => FloatOverflowTests.java} (94%)
 delete mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/IntegerOverflowDisabledTests.java
 rename modules/lang-painless/src/test/java/org/elasticsearch/painless/{IntegerOverflowEnabledTests.java => IntegerOverflowTests.java} (95%)
 delete mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/UtilityTests.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java
index 9ec26bf345a..4cafe32bb56 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java
@@ -24,47 +24,16 @@ package org.elasticsearch.painless;
  */
 public final class CompilerSettings {
 
-    /**
-     * Constant to be used when specifying numeric overflow when compiling a script.
-     */
-    public static final String NUMERIC_OVERFLOW = "numeric_overflow";
-
     /**
      * Constant to be used when specifying the maximum loop counter when compiling a script.
      */
     public static final String MAX_LOOP_COUNTER = "max_loop_counter";
 
-    /**
-     * Whether or not to allow numeric values to overflow without exception.
-     */
-    private boolean numericOverflow = true;
-
     /**
      * The maximum number of statements allowed to be run in a loop.
      */
     private int maxLoopCounter = 10000;
 
-    /**
-     * Returns {@code true} if numeric operations should overflow, {@code false}
-     * if they should signal an exception.
-     * <p>
-     * If this value is {@code true} (default), then things behave like java:
-     * overflow for integer types can result in unexpected values / unexpected
-     * signs, and overflow for floating point types can result in infinite or
-     * {@code NaN} values.
-     */
-    public final boolean getNumericOverflow() {
-        return numericOverflow;
-    }
-
-    /**
-     * Set {@code true} for numerics to overflow, false to deliver exceptions.
-     * @see #getNumericOverflow
-     */
-    public final void setNumericOverflow(boolean allow) {
-        this.numericOverflow = allow;
-    }
-
     /**
      * Returns the value for the cumulative total number of statements that can be made in all loops
      * in a script before an exception is thrown.  This attempts to prevent infinite loops.  Note if
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index bb362558796..cda8c23e3bf 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -34,10 +34,6 @@ import java.util.ArrayList;
 import java.util.Deque;
 import java.util.List;
 
-import static org.elasticsearch.painless.WriterConstants.ADDEXACT_INT;
-import static org.elasticsearch.painless.WriterConstants.ADDEXACT_LONG;
-import static org.elasticsearch.painless.WriterConstants.ADDWOOVERLOW_DOUBLE;
-import static org.elasticsearch.painless.WriterConstants.ADDWOOVERLOW_FLOAT;
 import static org.elasticsearch.painless.WriterConstants.DEF_ADD_CALL;
 import static org.elasticsearch.painless.WriterConstants.DEF_AND_CALL;
 import static org.elasticsearch.painless.WriterConstants.DEF_DIV_CALL;
@@ -49,19 +45,9 @@ import static org.elasticsearch.painless.WriterConstants.DEF_RSH_CALL;
 import static org.elasticsearch.painless.WriterConstants.DEF_SUB_CALL;
 import static org.elasticsearch.painless.WriterConstants.DEF_USH_CALL;
 import static org.elasticsearch.painless.WriterConstants.DEF_XOR_CALL;
-import static org.elasticsearch.painless.WriterConstants.DIVWOOVERLOW_DOUBLE;
-import static org.elasticsearch.painless.WriterConstants.DIVWOOVERLOW_FLOAT;
-import static org.elasticsearch.painless.WriterConstants.DIVWOOVERLOW_INT;
-import static org.elasticsearch.painless.WriterConstants.DIVWOOVERLOW_LONG;
 import static org.elasticsearch.painless.WriterConstants.INDY_STRING_CONCAT_BOOTSTRAP_HANDLE;
 import static org.elasticsearch.painless.WriterConstants.MAX_INDY_STRING_CONCAT_ARGS;
-import static org.elasticsearch.painless.WriterConstants.MULEXACT_INT;
-import static org.elasticsearch.painless.WriterConstants.MULEXACT_LONG;
-import static org.elasticsearch.painless.WriterConstants.MULWOOVERLOW_DOUBLE;
-import static org.elasticsearch.painless.WriterConstants.MULWOOVERLOW_FLOAT;
 import static org.elasticsearch.painless.WriterConstants.PAINLESS_ERROR_TYPE;
-import static org.elasticsearch.painless.WriterConstants.REMWOOVERLOW_DOUBLE;
-import static org.elasticsearch.painless.WriterConstants.REMWOOVERLOW_FLOAT;
 import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_BOOLEAN;
 import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_CHAR;
 import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_DOUBLE;
@@ -74,28 +60,6 @@ import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_CONSTRUCT
 import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_TOSTRING;
 import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_TYPE;
 import static org.elasticsearch.painless.WriterConstants.STRING_TYPE;
-import static org.elasticsearch.painless.WriterConstants.SUBEXACT_INT;
-import static org.elasticsearch.painless.WriterConstants.SUBEXACT_LONG;
-import static org.elasticsearch.painless.WriterConstants.SUBWOOVERLOW_DOUBLE;
-import static org.elasticsearch.painless.WriterConstants.SUBWOOVERLOW_FLOAT;
-import static org.elasticsearch.painless.WriterConstants.TOBYTEEXACT_INT;
-import static org.elasticsearch.painless.WriterConstants.TOBYTEEXACT_LONG;
-import static org.elasticsearch.painless.WriterConstants.TOBYTEWOOVERFLOW_DOUBLE;
-import static org.elasticsearch.painless.WriterConstants.TOBYTEWOOVERFLOW_FLOAT;
-import static org.elasticsearch.painless.WriterConstants.TOCHAREXACT_INT;
-import static org.elasticsearch.painless.WriterConstants.TOCHAREXACT_LONG;
-import static org.elasticsearch.painless.WriterConstants.TOCHARWOOVERFLOW_DOUBLE;
-import static org.elasticsearch.painless.WriterConstants.TOCHARWOOVERFLOW_FLOAT;
-import static org.elasticsearch.painless.WriterConstants.TOFLOATWOOVERFLOW_DOUBLE;
-import static org.elasticsearch.painless.WriterConstants.TOINTEXACT_LONG;
-import static org.elasticsearch.painless.WriterConstants.TOINTWOOVERFLOW_DOUBLE;
-import static org.elasticsearch.painless.WriterConstants.TOINTWOOVERFLOW_FLOAT;
-import static org.elasticsearch.painless.WriterConstants.TOLONGWOOVERFLOW_DOUBLE;
-import static org.elasticsearch.painless.WriterConstants.TOLONGWOOVERFLOW_FLOAT;
-import static org.elasticsearch.painless.WriterConstants.TOSHORTEXACT_INT;
-import static org.elasticsearch.painless.WriterConstants.TOSHORTEXACT_LONG;
-import static org.elasticsearch.painless.WriterConstants.TOSHORTWOOVERFLOW_DOUBLE;
-import static org.elasticsearch.painless.WriterConstants.TOSHORTWOOVERFLOW_FLOAT;
 
 /**
  * Extension of {@link GeneratorAdapter} with some utility methods.
@@ -236,231 +200,53 @@ public final class MethodWriter extends GeneratorAdapter {
         }
     }
 
-    public void writeBinaryInstruction(final CompilerSettings settings, final Definition definition,
+    public void writeBinaryInstruction(final Definition definition,
                                               final String location,
                                               final Type type, final Operation operation) {
         final Sort sort = type.sort;
-        boolean exact = !settings.getNumericOverflow() &&
-            ((sort == Sort.INT || sort == Sort.LONG) &&
-                (operation == Operation.MUL || operation == Operation.DIV ||
-                    operation == Operation.ADD || operation == Operation.SUB) ||
-                (sort == Sort.FLOAT || sort == Sort.DOUBLE) &&
-                    (operation == Operation.MUL || operation == Operation.DIV || operation == Operation.REM ||
-                        operation == Operation.ADD || operation == Operation.SUB));
-
-        if (exact) {
-            switch (sort) {
-                case INT:
-                    switch (operation) {
-                        case MUL: invokeStatic(definition.getType("Math").type,    MULEXACT_INT);     break;
-                        case DIV: invokeStatic(definition.getType("Utility").type, DIVWOOVERLOW_INT); break;
-                        case ADD: invokeStatic(definition.getType("Math").type,    ADDEXACT_INT);     break;
-                        case SUB: invokeStatic(definition.getType("Math").type,    SUBEXACT_INT);     break;
-                    }
-
-                    break;
-                case LONG:
-                    switch (operation) {
-                        case MUL: invokeStatic(definition.getType("Math").type,    MULEXACT_LONG);     break;
-                        case DIV: invokeStatic(definition.getType("Utility").type, DIVWOOVERLOW_LONG); break;
-                        case ADD: invokeStatic(definition.getType("Math").type,    ADDEXACT_LONG);     break;
-                        case SUB: invokeStatic(definition.getType("Math").type,    SUBEXACT_LONG);     break;
-                    }
-
-                    break;
-                case FLOAT:
-                    switch (operation) {
-                        case MUL: invokeStatic(definition.getType("Utility").type, MULWOOVERLOW_FLOAT); break;
-                        case DIV: invokeStatic(definition.getType("Utility").type, DIVWOOVERLOW_FLOAT); break;
-                        case REM: invokeStatic(definition.getType("Utility").type, REMWOOVERLOW_FLOAT); break;
-                        case ADD: invokeStatic(definition.getType("Utility").type, ADDWOOVERLOW_FLOAT); break;
-                        case SUB: invokeStatic(definition.getType("Utility").type, SUBWOOVERLOW_FLOAT); break;
-                        default:
-                            throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
-                    }
-
-                    break;
-                case DOUBLE:
-                    switch (operation) {
-                        case MUL: invokeStatic(definition.getType("Utility").type, MULWOOVERLOW_DOUBLE); break;
-                        case DIV: invokeStatic(definition.getType("Utility").type, DIVWOOVERLOW_DOUBLE); break;
-                        case REM: invokeStatic(definition.getType("Utility").type, REMWOOVERLOW_DOUBLE); break;
-                        case ADD: invokeStatic(definition.getType("Utility").type, ADDWOOVERLOW_DOUBLE); break;
-                        case SUB: invokeStatic(definition.getType("Utility").type, SUBWOOVERLOW_DOUBLE); break;
-                        default:
-                            throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
-                    }
-
-                    break;
+        
+        if ((sort == Sort.FLOAT || sort == Sort.DOUBLE) &&
+                (operation == Operation.LSH || operation == Operation.USH ||
+                operation == Operation.RSH || operation == Operation.BWAND ||
+                operation == Operation.XOR || operation == Operation.BWOR)) {
+            throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
+        }
+        
+        if (sort == Sort.DEF) {
+            switch (operation) {
+                case MUL:   invokeStatic(definition.getType("Def").type, DEF_MUL_CALL); break;
+                case DIV:   invokeStatic(definition.getType("Def").type, DEF_DIV_CALL); break;
+                case REM:   invokeStatic(definition.getType("Def").type, DEF_REM_CALL); break;
+                case ADD:   invokeStatic(definition.getType("Def").type, DEF_ADD_CALL); break;
+                case SUB:   invokeStatic(definition.getType("Def").type, DEF_SUB_CALL); break;
+                case LSH:   invokeStatic(definition.getType("Def").type, DEF_LSH_CALL); break;
+                case USH:   invokeStatic(definition.getType("Def").type, DEF_RSH_CALL); break;
+                case RSH:   invokeStatic(definition.getType("Def").type, DEF_USH_CALL); break;
+                case BWAND: invokeStatic(definition.getType("Def").type, DEF_AND_CALL); break;
+                case XOR:   invokeStatic(definition.getType("Def").type, DEF_XOR_CALL); break;
+                case BWOR:  invokeStatic(definition.getType("Def").type, DEF_OR_CALL);  break;
                 default:
                     throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
             }
         } else {
-            if ((sort == Sort.FLOAT || sort == Sort.DOUBLE) &&
-                (operation == Operation.LSH || operation == Operation.USH ||
-                    operation == Operation.RSH || operation == Operation.BWAND ||
-                    operation == Operation.XOR || operation == Operation.BWOR)) {
-                throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
-            }
-
-            if (sort == Sort.DEF) {
-                switch (operation) {
-                    case MUL:   invokeStatic(definition.getType("Def").type, DEF_MUL_CALL); break;
-                    case DIV:   invokeStatic(definition.getType("Def").type, DEF_DIV_CALL); break;
-                    case REM:   invokeStatic(definition.getType("Def").type, DEF_REM_CALL); break;
-                    case ADD:   invokeStatic(definition.getType("Def").type, DEF_ADD_CALL); break;
-                    case SUB:   invokeStatic(definition.getType("Def").type, DEF_SUB_CALL); break;
-                    case LSH:   invokeStatic(definition.getType("Def").type, DEF_LSH_CALL); break;
-                    case USH:   invokeStatic(definition.getType("Def").type, DEF_RSH_CALL); break;
-                    case RSH:   invokeStatic(definition.getType("Def").type, DEF_USH_CALL); break;
-                    case BWAND: invokeStatic(definition.getType("Def").type, DEF_AND_CALL); break;
-                    case XOR:   invokeStatic(definition.getType("Def").type, DEF_XOR_CALL); break;
-                    case BWOR:  invokeStatic(definition.getType("Def").type, DEF_OR_CALL);  break;
-                    default:
-                        throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
-                }
-            } else {
-                switch (operation) {
-                    case MUL:   math(GeneratorAdapter.MUL,  type.type); break;
-                    case DIV:   math(GeneratorAdapter.DIV,  type.type); break;
-                    case REM:   math(GeneratorAdapter.REM,  type.type); break;
-                    case ADD:   math(GeneratorAdapter.ADD,  type.type); break;
-                    case SUB:   math(GeneratorAdapter.SUB,  type.type); break;
-                    case LSH:   math(GeneratorAdapter.SHL,  type.type); break;
-                    case USH:   math(GeneratorAdapter.USHR, type.type); break;
-                    case RSH:   math(GeneratorAdapter.SHR,  type.type); break;
-                    case BWAND: math(GeneratorAdapter.AND,  type.type); break;
-                    case XOR:   math(GeneratorAdapter.XOR,  type.type); break;
-                    case BWOR:  math(GeneratorAdapter.OR,   type.type); break;
-                    default:
-                        throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
-                }
+            switch (operation) {
+                case MUL:   math(GeneratorAdapter.MUL,  type.type); break;
+                case DIV:   math(GeneratorAdapter.DIV,  type.type); break;
+                case REM:   math(GeneratorAdapter.REM,  type.type); break;
+                case ADD:   math(GeneratorAdapter.ADD,  type.type); break;
+                case SUB:   math(GeneratorAdapter.SUB,  type.type); break;
+                case LSH:   math(GeneratorAdapter.SHL,  type.type); break;
+                case USH:   math(GeneratorAdapter.USHR, type.type); break;
+                case RSH:   math(GeneratorAdapter.SHR,  type.type); break;
+                case BWAND: math(GeneratorAdapter.AND,  type.type); break;
+                case XOR:   math(GeneratorAdapter.XOR,  type.type); break;
+                case BWOR:  math(GeneratorAdapter.OR,   type.type); break;
+                default:
+                    throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
             }
         }
     }
 
-    /**
-     * Called for any compound assignment (including increment/decrement instructions).
-     * We have to be stricter than writeBinary and do overflow checks against the original type's size
-     * instead of the promoted type's size, since the result will be implicitly cast back.
-     *
-     * @return This will be true if an instruction is written, false otherwise.
-     */
-    public boolean writeExactInstruction(
-        final Definition definition, final Sort fsort, final Sort tsort) {
-        if (fsort == Sort.DOUBLE) {
-            if (tsort == Sort.FLOAT) {
-                invokeStatic(definition.getType("Utility").type, TOFLOATWOOVERFLOW_DOUBLE);
-            } else if (tsort == Sort.FLOAT_OBJ) {
-                invokeStatic(definition.getType("Utility").type, TOFLOATWOOVERFLOW_DOUBLE);
-                checkCast(definition.getType("Float").type);
-            } else if (tsort == Sort.LONG) {
-                invokeStatic(definition.getType("Utility").type, TOLONGWOOVERFLOW_DOUBLE);
-            } else if (tsort == Sort.LONG_OBJ) {
-                invokeStatic(definition.getType("Utility").type, TOLONGWOOVERFLOW_DOUBLE);
-                checkCast(definition.getType("Long").type);
-            } else if (tsort == Sort.INT) {
-                invokeStatic(definition.getType("Utility").type, TOINTWOOVERFLOW_DOUBLE);
-            } else if (tsort == Sort.INT_OBJ) {
-                invokeStatic(definition.getType("Utility").type, TOINTWOOVERFLOW_DOUBLE);
-                checkCast(definition.getType("Integer").type);
-            } else if (tsort == Sort.CHAR) {
-                invokeStatic(definition.getType("Utility").type, TOCHARWOOVERFLOW_DOUBLE);
-            } else if (tsort == Sort.CHAR_OBJ) {
-                invokeStatic(definition.getType("Utility").type, TOCHARWOOVERFLOW_DOUBLE);
-                checkCast(definition.getType("Character").type);
-            } else if (tsort == Sort.SHORT) {
-                invokeStatic(definition.getType("Utility").type, TOSHORTWOOVERFLOW_DOUBLE);
-            } else if (tsort == Sort.SHORT_OBJ) {
-                invokeStatic(definition.getType("Utility").type, TOSHORTWOOVERFLOW_DOUBLE);
-                checkCast(definition.getType("Short").type);
-            } else if (tsort == Sort.BYTE) {
-                invokeStatic(definition.getType("Utility").type, TOBYTEWOOVERFLOW_DOUBLE);
-            } else if (tsort == Sort.BYTE_OBJ) {
-                invokeStatic(definition.getType("Utility").type, TOBYTEWOOVERFLOW_DOUBLE);
-                checkCast(definition.getType("Byte").type);
-            } else {
-                return false;
-            }
-        } else if (fsort == Sort.FLOAT) {
-            if (tsort == Sort.LONG) {
-                invokeStatic(definition.getType("Utility").type, TOLONGWOOVERFLOW_FLOAT);
-            } else if (tsort == Sort.LONG_OBJ) {
-                invokeStatic(definition.getType("Utility").type, TOLONGWOOVERFLOW_FLOAT);
-                checkCast(definition.getType("Long").type);
-            } else if (tsort == Sort.INT) {
-                invokeStatic(definition.getType("Utility").type, TOINTWOOVERFLOW_FLOAT);
-            } else if (tsort == Sort.INT_OBJ) {
-                invokeStatic(definition.getType("Utility").type, TOINTWOOVERFLOW_FLOAT);
-                checkCast(definition.getType("Integer").type);
-            } else if (tsort == Sort.CHAR) {
-                invokeStatic(definition.getType("Utility").type, TOCHARWOOVERFLOW_FLOAT);
-            } else if (tsort == Sort.CHAR_OBJ) {
-                invokeStatic(definition.getType("Utility").type, TOCHARWOOVERFLOW_FLOAT);
-                checkCast(definition.getType("Character").type);
-            } else if (tsort == Sort.SHORT) {
-                invokeStatic(definition.getType("Utility").type, TOSHORTWOOVERFLOW_FLOAT);
-            } else if (tsort == Sort.SHORT_OBJ) {
-                invokeStatic(definition.getType("Utility").type, TOSHORTWOOVERFLOW_FLOAT);
-                checkCast(definition.getType("Short").type);
-            } else if (tsort == Sort.BYTE) {
-                invokeStatic(definition.getType("Utility").type, TOBYTEWOOVERFLOW_FLOAT);
-            } else if (tsort == Sort.BYTE_OBJ) {
-                invokeStatic(definition.getType("Utility").type, TOBYTEWOOVERFLOW_FLOAT);
-                checkCast(definition.getType("Byte").type);
-            } else {
-                return false;
-            }
-        } else if (fsort == Sort.LONG) {
-            if (tsort == Sort.INT) {
-                invokeStatic(definition.getType("Math").type, TOINTEXACT_LONG);
-            } else if (tsort == Sort.INT_OBJ) {
-                invokeStatic(definition.getType("Math").type, TOINTEXACT_LONG);
-                checkCast(definition.getType("Integer").type);
-            } else if (tsort == Sort.CHAR) {
-                invokeStatic(definition.getType("Utility").type, TOCHAREXACT_LONG);
-            } else if (tsort == Sort.CHAR_OBJ) {
-                invokeStatic(definition.getType("Utility").type, TOCHAREXACT_LONG);
-                checkCast(definition.getType("Character").type);
-            } else if (tsort == Sort.SHORT) {
-                invokeStatic(definition.getType("Utility").type, TOSHORTEXACT_LONG);
-            } else if (tsort == Sort.SHORT_OBJ) {
-                invokeStatic(definition.getType("Utility").type, TOSHORTEXACT_LONG);
-                checkCast(definition.getType("Short").type);
-            } else if (tsort == Sort.BYTE) {
-                invokeStatic(definition.getType("Utility").type, TOBYTEEXACT_LONG);
-            } else if (tsort == Sort.BYTE_OBJ) {
-                invokeStatic(definition.getType("Utility").type, TOBYTEEXACT_LONG);
-                checkCast(definition.getType("Byte").type);
-            } else {
-                return false;
-            }
-        } else if (fsort == Sort.INT) {
-            if (tsort == Sort.CHAR) {
-                invokeStatic(definition.getType("Utility").type, TOCHAREXACT_INT);
-            } else if (tsort == Sort.CHAR_OBJ) {
-                invokeStatic(definition.getType("Utility").type, TOCHAREXACT_INT);
-                checkCast(definition.getType("Character").type);
-            } else if (tsort == Sort.SHORT) {
-                invokeStatic(definition.getType("Utility").type, TOSHORTEXACT_INT);
-            } else if (tsort == Sort.SHORT_OBJ) {
-                invokeStatic(definition.getType("Utility").type, TOSHORTEXACT_INT);
-                checkCast(definition.getType("Short").type);
-            } else if (tsort == Sort.BYTE) {
-                invokeStatic(definition.getType("Utility").type, TOBYTEEXACT_INT);
-            } else if (tsort == Sort.BYTE_OBJ) {
-                invokeStatic(definition.getType("Utility").type, TOBYTEEXACT_INT);
-                checkCast(definition.getType("Byte").type);
-            } else {
-                return false;
-            }
-        } else {
-            return false;
-        }
-
-        return true;
-    }
-
     public void writeDup(final int size, final int xsize) {
         if (size == 1) {
             if (xsize == 2) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
index 1d11dae720f..4c627d08eca 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
@@ -115,13 +115,7 @@ public final class PainlessScriptEngineService extends AbstractComponent impleme
             // Use custom settings specified by params.
             compilerSettings = new CompilerSettings();
             Map<String, String> copy = new HashMap<>(params);
-            String value = copy.remove(CompilerSettings.NUMERIC_OVERFLOW);
-
-            if (value != null) {
-                compilerSettings.setNumericOverflow(Boolean.parseBoolean(value));
-            }
-
-            value = copy.remove(CompilerSettings.MAX_LOOP_COUNTER);
+            String value = copy.remove(CompilerSettings.MAX_LOOP_COUNTER);
 
             if (value != null) {
                 compilerSettings.setMaxLoopCounter(Integer.parseInt(value));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java
index 32641649827..559e909ed77 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java
@@ -481,347 +481,6 @@ public class Utility {
         return value.charAt(0);
     }
 
-    // although divide by zero is guaranteed, the special overflow case is not caught.
-    // its not needed for remainder because it is not possible there.
-    // see https://docs.oracle.com/javase/specs/jls/se8/html/jls-15.html#jls-15.17.2
-
-    /**
-     * Integer divide without overflow
-     * @throws ArithmeticException on overflow or divide-by-zero
-     */
-    public static int divideWithoutOverflow(int x, int y) {
-       if (x == Integer.MIN_VALUE && y == -1) {
-           throw new ArithmeticException("integer overflow");
-       }
-       return x / y;
-    }
-
-    /**
-     * Long divide without overflow
-     * @throws ArithmeticException on overflow or divide-by-zero
-     */
-    public static long divideWithoutOverflow(long x, long y) {
-        if (x == Long.MIN_VALUE && y == -1L) {
-            throw new ArithmeticException("long overflow");
-        }
-        return x / y;
-    }
-
-    // byte, short, and char are promoted to int for normal operations,
-    // so the JDK exact methods are typically used, and the result has a wider range.
-    // but compound assignments and increment/decrement operators (e.g. byte b = Byte.MAX_VALUE; b++;)
-    // implicitly cast back to the original type: so these need to be checked against the original range.
-
-    /**
-     * Like {@link Math#toIntExact(long)} but for byte range.
-     */
-    public static byte toByteExact(int value) {
-        byte s = (byte) value;
-        if (s != value) {
-            throw new ArithmeticException("byte overflow");
-        }
-        return s;
-    }
-
-    /**
-     * Like {@link Math#toIntExact(long)} but for byte range.
-     */
-    public static byte toByteExact(long value) {
-        byte s = (byte) value;
-        if (s != value) {
-            throw new ArithmeticException("byte overflow");
-        }
-        return s;
-    }
-
-    /**
-     * Like {@link Math#toIntExact(long)} but for byte range.
-     */
-    public static byte toByteWithoutOverflow(float value) {
-        if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) {
-            throw new ArithmeticException("byte overflow");
-        }
-        return (byte)value;
-    }
-
-    /**
-     * Like {@link Math#toIntExact(long)} but for byte range.
-     */
-    public static byte toByteWithoutOverflow(double value) {
-        if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) {
-            throw new ArithmeticException("byte overflow");
-        }
-        return (byte)value;
-    }
-
-    /**
-     * Like {@link Math#toIntExact(long)} but for short range.
-     */
-    public static short toShortExact(int value) {
-        short s = (short) value;
-        if (s != value) {
-            throw new ArithmeticException("short overflow");
-        }
-        return s;
-    }
-
-    /**
-     * Like {@link Math#toIntExact(long)} but for short range.
-     */
-    public static short toShortExact(long value) {
-        short s = (short) value;
-        if (s != value) {
-            throw new ArithmeticException("short overflow");
-        }
-        return s;
-    }
-
-    /**
-     * Like {@link Math#toIntExact(long)} but for short range.
-     */
-    public static short toShortWithoutOverflow(float value) {
-        if (value < Short.MIN_VALUE || value > Short.MAX_VALUE) {
-            throw new ArithmeticException("short overflow");
-        }
-        return (short)value;
-    }
-
-    /**
-     * Like {@link Math#toIntExact(long)} but for short range.
-     */
-    public static short toShortExact(double value) {
-        if (value < Short.MIN_VALUE || value > Short.MAX_VALUE) {
-            throw new ArithmeticException("short overflow");
-        }
-        return (short)value;
-    }
-
-    /**
-     * Like {@link Math#toIntExact(long)} but for char range.
-     */
-    public static char toCharExact(int value) {
-        char s = (char) value;
-        if (s != value) {
-            throw new ArithmeticException("char overflow");
-        }
-        return s;
-    }
-
-    /**
-     * Like {@link Math#toIntExact(long)} but for char range.
-     */
-    public static char toCharExact(long value) {
-        char s = (char) value;
-        if (s != value) {
-            throw new ArithmeticException("char overflow");
-        }
-        return s;
-    }
-
-    /**
-     * Like {@link Math#toIntExact(long)} but for char range.
-     */
-    public static char toCharWithoutOverflow(float value) {
-        if (value < Character.MIN_VALUE || value > Character.MAX_VALUE) {
-            throw new ArithmeticException("char overflow");
-        }
-        return (char)value;
-    }
-
-    /**
-     * Like {@link Math#toIntExact(long)} but for char range.
-     */
-    public static char toCharWithoutOverflow(double value) {
-        if (value < Character.MIN_VALUE || value > Character.MAX_VALUE) {
-            throw new ArithmeticException("char overflow");
-        }
-        return (char)value;
-    }
-
-    /**
-     * Like {@link Math#toIntExact(long)} but for int range.
-     */
-    public static int toIntWithoutOverflow(float value) {
-        if (value < Integer.MIN_VALUE || value > Integer.MAX_VALUE) {
-            throw new ArithmeticException("int overflow");
-        }
-        return (int)value;
-    }
-
-    /**
-     * Like {@link Math#toIntExact(long)} but for int range.
-     */
-    public static int toIntWithoutOverflow(double value) {
-        if (value < Integer.MIN_VALUE || value > Integer.MAX_VALUE) {
-            throw new ArithmeticException("int overflow");
-        }
-        return (int)value;
-    }
-
-    /**
-     * Like {@link Math#toIntExact(long)} but for long range.
-     */
-    public static long toLongWithoutOverflow(float value) {
-        if (value < Long.MIN_VALUE || value > Long.MAX_VALUE) {
-            throw new ArithmeticException("long overflow");
-        }
-        return (long)value;
-    }
-
-    /**
-     * Like {@link Math#toIntExact(long)} but for long range.
-     */
-    public static float toLongWithoutOverflow(double value) {
-        if (value < Long.MIN_VALUE || value > Long.MAX_VALUE) {
-            throw new ArithmeticException("long overflow");
-        }
-        return (long)value;
-    }
-
-    /**
-     * Like {@link Math#toIntExact(long)} but for float range.
-     */
-    public static float toFloatWithoutOverflow(double value) {
-        if (value < Float.MIN_VALUE || value > Float.MAX_VALUE) {
-            throw new ArithmeticException("float overflow");
-        }
-        return (float)value;
-    }
-
-    /**
-     * Checks for overflow, result is infinite but operands are finite
-     * @throws ArithmeticException if overflow occurred
-     */
-    private static float checkInfFloat(float x, float y, float z) {
-        if (Float.isInfinite(z)) {
-            if (Float.isFinite(x) && Float.isFinite(y)) {
-                throw new ArithmeticException("float overflow");
-            }
-        }
-        return z;
-    }
-
-    /**
-     * Checks for NaN, result is NaN but operands are finite
-     * @throws ArithmeticException if overflow occurred
-     */
-    private static float checkNaNFloat(float x, float y, float z) {
-        if (Float.isNaN(z)) {
-            if (Float.isFinite(x) && Float.isFinite(y)) {
-                throw new ArithmeticException("NaN");
-            }
-        }
-        return z;
-    }
-
-    /**
-     * Checks for NaN, result is infinite but operands are finite
-     * @throws ArithmeticException if overflow occurred
-     */
-    private static double checkInfDouble(double x, double y, double z) {
-        if (Double.isInfinite(z)) {
-            if (Double.isFinite(x) && Double.isFinite(y)) {
-                throw new ArithmeticException("double overflow");
-            }
-        }
-        return z;
-    }
-
-    /**
-     * Checks for NaN, result is NaN but operands are finite
-     * @throws ArithmeticException if overflow occurred
-     */
-    private static double checkNaNDouble(double x, double y, double z) {
-        if (Double.isNaN(z)) {
-            if (Double.isFinite(x) && Double.isFinite(y)) {
-                throw new ArithmeticException("NaN");
-            }
-        }
-        return z;
-    }
-
-    /**
-     * Adds two floats but throws {@code ArithmeticException}
-     * if the result overflows.
-     */
-    public static float addWithoutOverflow(float x, float y) {
-        return checkInfFloat(x, y, x + y);
-    }
-
-    /**
-     * Adds two doubles but throws {@code ArithmeticException}
-     * if the result overflows.
-     */
-    public static double addWithoutOverflow(double x, double y) {
-        return checkInfDouble(x, y, x + y);
-    }
-
-    /**
-     * Subtracts two floats but throws {@code ArithmeticException}
-     * if the result overflows.
-     */
-    public static float subtractWithoutOverflow(float x, float y) {
-        return checkInfFloat(x, y, x - y);
-    }
-
-    /**
-     * Subtracts two doubles but throws {@code ArithmeticException}
-     * if the result overflows.
-     */
-    public static double subtractWithoutOverflow(double x, double y) {
-        return checkInfDouble(x, y , x - y);
-    }
-
-    /**
-     * Multiplies two floats but throws {@code ArithmeticException}
-     * if the result overflows.
-     */
-    public static float multiplyWithoutOverflow(float x, float y) {
-        return checkInfFloat(x, y, x * y);
-    }
-
-    /**
-     * Multiplies two doubles but throws {@code ArithmeticException}
-     * if the result overflows.
-     */
-    public static double multiplyWithoutOverflow(double x, double y) {
-        return checkInfDouble(x, y, x * y);
-    }
-
-    /**
-     * Divides two floats but throws {@code ArithmeticException}
-     * if the result overflows, or would create NaN from finite
-     * inputs ({@code x == 0, y == 0})
-     */
-    public static float divideWithoutOverflow(float x, float y) {
-        return checkNaNFloat(x, y, checkInfFloat(x, y, x / y));
-    }
-
-    /**
-     * Divides two doubles but throws {@code ArithmeticException}
-     * if the result overflows, or would create NaN from finite
-     * inputs ({@code x == 0, y == 0})
-     */
-    public static double divideWithoutOverflow(double x, double y) {
-        return checkNaNDouble(x, y, checkInfDouble(x, y, x / y));
-    }
-
-    /**
-     * Takes remainder two floats but throws {@code ArithmeticException}
-     * if the result would create NaN from finite inputs ({@code y == 0})
-     */
-    public static float remainderWithoutOverflow(float x, float y) {
-        return checkNaNFloat(x, y, x % y);
-    }
-
-    /**
-     * Divides two doubles but throws {@code ArithmeticException}
-     * if the result would create NaN from finite inputs ({@code y == 0})
-     */
-    public static double remainderWithoutOverflow(double x, double y) {
-        return checkNaNDouble(x, y, x % y);
-    }
-
     public static boolean checkEquals(final Object left, final Object right) {
         if (left != null) {
             return left.equals(right);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index 6bdb9856114..d167646e9dd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -27,7 +27,6 @@ import org.objectweb.asm.Type;
 import org.objectweb.asm.commons.Method;
 
 import java.lang.invoke.CallSite;
-import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodType;
 import java.util.Map;
@@ -116,59 +115,8 @@ public final class WriterConstants {
     public final static Method STRINGBUILDER_APPEND_OBJECT  = getAsmMethod(StringBuilder.class, "append", Object.class);
     public final static Method STRINGBUILDER_TOSTRING       = getAsmMethod(String.class, "toString");
 
-    public final static Method TOINTEXACT_LONG  = getAsmMethod(int.class,  "toIntExact",    long.class);
-    public final static Method NEGATEEXACT_INT  = getAsmMethod(int.class,  "negateExact",   int.class);
-    public final static Method NEGATEEXACT_LONG = getAsmMethod(long.class, "negateExact",   long.class);
-    public final static Method MULEXACT_INT     = getAsmMethod(int.class,  "multiplyExact", int.class,  int.class);
-    public final static Method MULEXACT_LONG    = getAsmMethod(long.class, "multiplyExact", long.class, long.class);
-    public final static Method ADDEXACT_INT     = getAsmMethod(int.class,  "addExact",      int.class,  int.class);
-    public final static Method ADDEXACT_LONG    = getAsmMethod(long.class, "addExact",      long.class, long.class);
-    public final static Method SUBEXACT_INT     = getAsmMethod(int.class,  "subtractExact", int.class,  int.class);
-    public final static Method SUBEXACT_LONG    = getAsmMethod(long.class, "subtractExact", long.class, long.class);
-
     public final static Method CHECKEQUALS              =
         getAsmMethod(boolean.class, "checkEquals",              Object.class, Object.class);
-    public final static Method TOBYTEEXACT_INT          = getAsmMethod(byte.class,    "toByteExact",              int.class);
-    public final static Method TOBYTEEXACT_LONG         = getAsmMethod(byte.class,    "toByteExact",              long.class);
-    public final static Method TOBYTEWOOVERFLOW_FLOAT   = getAsmMethod(byte.class,    "toByteWithoutOverflow",    float.class);
-    public final static Method TOBYTEWOOVERFLOW_DOUBLE  = getAsmMethod(byte.class,    "toByteWithoutOverflow",    double.class);
-    public final static Method TOSHORTEXACT_INT         = getAsmMethod(short.class,   "toShortExact",             int.class);
-    public final static Method TOSHORTEXACT_LONG        = getAsmMethod(short.class,   "toShortExact",             long.class);
-    public final static Method TOSHORTWOOVERFLOW_FLOAT  = getAsmMethod(short.class,   "toShortWithoutOverflow",   float.class);
-    public final static Method TOSHORTWOOVERFLOW_DOUBLE = getAsmMethod(short.class,   "toShortWihtoutOverflow",   double.class);
-    public final static Method TOCHAREXACT_INT          = getAsmMethod(char.class,    "toCharExact",              int.class);
-    public final static Method TOCHAREXACT_LONG         = getAsmMethod(char.class,    "toCharExact",              long.class);
-    public final static Method TOCHARWOOVERFLOW_FLOAT   = getAsmMethod(char.class,    "toCharWithoutOverflow",    float.class);
-    public final static Method TOCHARWOOVERFLOW_DOUBLE  = getAsmMethod(char.class,    "toCharWithoutOverflow",    double.class);
-    public final static Method TOINTWOOVERFLOW_FLOAT    = getAsmMethod(int.class,     "toIntWithoutOverflow",     float.class);
-    public final static Method TOINTWOOVERFLOW_DOUBLE   = getAsmMethod(int.class,     "toIntWithoutOverflow",     double.class);
-    public final static Method TOLONGWOOVERFLOW_FLOAT   = getAsmMethod(long.class,    "toLongWithoutOverflow",    float.class);
-    public final static Method TOLONGWOOVERFLOW_DOUBLE  = getAsmMethod(long.class,    "toLongWithoutOverflow",    double.class);
-    public final static Method TOFLOATWOOVERFLOW_DOUBLE = getAsmMethod(float.class ,  "toFloatWihtoutOverflow",   double.class);
-    public final static Method MULWOOVERLOW_FLOAT       =
-        getAsmMethod(float.class,   "multiplyWithoutOverflow",  float.class,  float.class);
-    public final static Method MULWOOVERLOW_DOUBLE      =
-        getAsmMethod(double.class,  "multiplyWithoutOverflow",  double.class, double.class);
-    public final static Method DIVWOOVERLOW_INT         =
-        getAsmMethod(int.class,     "divideWithoutOverflow",    int.class,    int.class);
-    public final static Method DIVWOOVERLOW_LONG        =
-        getAsmMethod(long.class,    "divideWithoutOverflow",    long.class,   long.class);
-    public final static Method DIVWOOVERLOW_FLOAT       =
-        getAsmMethod(float.class,   "divideWithoutOverflow",    float.class,  float.class);
-    public final static Method DIVWOOVERLOW_DOUBLE      =
-        getAsmMethod(double.class,  "divideWithoutOverflow",    double.class, double.class);
-    public final static Method REMWOOVERLOW_FLOAT       =
-        getAsmMethod(float.class,   "remainderWithoutOverflow", float.class,  float.class);
-    public final static Method REMWOOVERLOW_DOUBLE      =
-        getAsmMethod(double.class,  "remainderWithoutOverflow", double.class, double.class);
-    public final static Method ADDWOOVERLOW_FLOAT       =
-        getAsmMethod(float.class,   "addWithoutOverflow",       float.class,  float.class);
-    public final static Method ADDWOOVERLOW_DOUBLE      =
-        getAsmMethod(double.class,  "addWithoutOverflow",       double.class, double.class);
-    public final static Method SUBWOOVERLOW_FLOAT       =
-        getAsmMethod(float.class,   "subtractWithoutOverflow",  float.class,  float.class);
-    public final static Method SUBWOOVERLOW_DOUBLE      =
-        getAsmMethod(double.class,  "subtractWithoutOverflow",  double.class, double.class);
 
     private static Method getAsmMethod(final Class<?> rtype, final String name, final Class<?>... ptypes) {
         return new Method(name, MethodType.methodType(rtype, ptypes).toMethodDescriptorString());
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 18b9de80fb6..f22b7394ca0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -94,21 +94,16 @@ public final class EBinary extends AExpression {
         right = right.cast(settings, definition, variables);
 
         if (left.constant != null && right.constant != null) {
-            final boolean overflow = settings.getNumericOverflow();
             final Sort sort = promote.sort;
 
             if (sort == Sort.INT) {
-                constant = overflow ? (int)left.constant * (int)right.constant :
-                    Math.multiplyExact((int)left.constant, (int)right.constant);
+                constant = (int)left.constant * (int)right.constant;
             } else if (sort == Sort.LONG) {
-                constant = overflow ? (long)left.constant * (long)right.constant :
-                    Math.multiplyExact((long)left.constant, (long)right.constant);
+                constant = (long)left.constant * (long)right.constant;
             } else if (sort == Sort.FLOAT) {
-                constant = overflow ? (float)left.constant * (float)right.constant :
-                    org.elasticsearch.painless.Utility.multiplyWithoutOverflow((float)left.constant, (float)right.constant);
+                constant = (float)left.constant * (float)right.constant;
             } else if (sort == Sort.DOUBLE) {
-                constant = overflow ? (double)left.constant * (double)right.constant :
-                    org.elasticsearch.painless.Utility.multiplyWithoutOverflow((double)left.constant, (double)right.constant);
+                constant = (double)left.constant * (double)right.constant;
             } else {
                 throw new IllegalStateException(error("Illegal tree structure."));
             }
@@ -135,21 +130,16 @@ public final class EBinary extends AExpression {
         right = right.cast(settings, definition, variables);
 
         if (left.constant != null && right.constant != null) {
-            final boolean overflow = settings.getNumericOverflow();
             final Sort sort = promote.sort;
 
             if (sort == Sort.INT) {
-                constant = overflow ? (int)left.constant / (int)right.constant :
-                    org.elasticsearch.painless.Utility.divideWithoutOverflow((int)left.constant, (int)right.constant);
+                constant = (int)left.constant / (int)right.constant;
             } else if (sort == Sort.LONG) {
-                constant = overflow ? (long)left.constant / (long)right.constant :
-                    org.elasticsearch.painless.Utility.divideWithoutOverflow((long)left.constant, (long)right.constant);
+                constant = (long)left.constant / (long)right.constant;
             } else if (sort == Sort.FLOAT) {
-                constant = overflow ? (float)left.constant / (float)right.constant :
-                    org.elasticsearch.painless.Utility.divideWithoutOverflow((float)left.constant, (float)right.constant);
+                constant = (float)left.constant / (float)right.constant;
             } else if (sort == Sort.DOUBLE) {
-                constant = overflow ? (double)left.constant / (double)right.constant :
-                    org.elasticsearch.painless.Utility.divideWithoutOverflow((double)left.constant, (double)right.constant);
+                constant = (double)left.constant / (double)right.constant;
             } else {
                 throw new IllegalStateException(error("Illegal tree structure."));
             }
@@ -176,7 +166,6 @@ public final class EBinary extends AExpression {
         right = right.cast(settings, definition, variables);
 
         if (left.constant != null && right.constant != null) {
-            final boolean overflow = settings.getNumericOverflow();
             final Sort sort = promote.sort;
 
             if (sort == Sort.INT) {
@@ -184,11 +173,9 @@ public final class EBinary extends AExpression {
             } else if (sort == Sort.LONG) {
                 constant = (long)left.constant % (long)right.constant;
             } else if (sort == Sort.FLOAT) {
-                constant = overflow ? (float)left.constant % (float)right.constant :
-                    org.elasticsearch.painless.Utility.remainderWithoutOverflow((float)left.constant, (float)right.constant);
+                constant = (float)left.constant % (float)right.constant;
             } else if (sort == Sort.DOUBLE) {
-                constant = overflow ? (double)left.constant % (double)right.constant :
-                    org.elasticsearch.painless.Utility.remainderWithoutOverflow((double)left.constant, (double)right.constant);
+                constant = (double)left.constant % (double)right.constant;
             } else {
                 throw new IllegalStateException(error("Illegal tree structure."));
             }
@@ -231,20 +218,14 @@ public final class EBinary extends AExpression {
         right = right.cast(settings, definition, variables);
 
         if (left.constant != null && right.constant != null) {
-            final boolean overflow = settings.getNumericOverflow();
-
             if (sort == Sort.INT) {
-                constant = overflow ? (int)left.constant + (int)right.constant :
-                    Math.addExact((int)left.constant, (int)right.constant);
+                constant = (int)left.constant + (int)right.constant;
             } else if (sort == Sort.LONG) {
-                constant = overflow ? (long)left.constant + (long)right.constant :
-                    Math.addExact((long)left.constant, (long)right.constant);
+                constant = (long)left.constant + (long)right.constant;
             } else if (sort == Sort.FLOAT) {
-                constant = overflow ? (float)left.constant + (float)right.constant :
-                    org.elasticsearch.painless.Utility.addWithoutOverflow((float)left.constant, (float)right.constant);
+                constant = (float)left.constant + (float)right.constant;
             } else if (sort == Sort.DOUBLE) {
-                constant = overflow ? (double)left.constant + (double)right.constant :
-                    org.elasticsearch.painless.Utility.addWithoutOverflow((double)left.constant, (double)right.constant);
+                constant = (double)left.constant + (double)right.constant;
             } else if (sort == Sort.STRING) {
                 constant = "" + left.constant + right.constant;
             } else {
@@ -273,21 +254,16 @@ public final class EBinary extends AExpression {
         right = right.cast(settings, definition, variables);
 
         if (left.constant != null && right.constant != null) {
-            final boolean overflow = settings.getNumericOverflow();
             final Sort sort = promote.sort;
 
             if (sort == Sort.INT) {
-                constant = overflow ? (int)left.constant - (int)right.constant :
-                    Math.subtractExact((int)left.constant, (int)right.constant);
+                constant = (int)left.constant - (int)right.constant;
             } else if (sort == Sort.LONG) {
-                constant = overflow ? (long)left.constant - (long)right.constant :
-                    Math.subtractExact((long)left.constant, (long)right.constant);
+                constant = (long)left.constant - (long)right.constant;
             } else if (sort == Sort.FLOAT) {
-                constant = overflow ? (float)left.constant - (float)right.constant :
-                    org.elasticsearch.painless.Utility.subtractWithoutOverflow((float)left.constant, (float)right.constant);
+                constant = (float)left.constant - (float)right.constant;
             } else if (sort == Sort.DOUBLE) {
-                constant = overflow ? (double)left.constant - (double)right.constant :
-                    org.elasticsearch.painless.Utility.subtractWithoutOverflow((double)left.constant, (double)right.constant);
+                constant = (double)left.constant - (double)right.constant;
             } else {
                 throw new IllegalStateException(error("Illegal tree structure."));
             }
@@ -519,7 +495,7 @@ public final class EBinary extends AExpression {
             left.write(settings, definition, adapter);
             right.write(settings, definition, adapter);
 
-            adapter.writeBinaryInstruction(settings, definition, location, actual, operation);
+            adapter.writeBinaryInstruction(definition, location, actual, operation);
         }
 
         adapter.writeBranch(tru, fals);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index 763005d17ba..933295f9beb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -44,7 +44,6 @@ public final class EChain extends AExpression {
 
     boolean cat = false;
     Type promote = null;
-    boolean exact = false;
     Cast there = null;
     Cast back = null;
 
@@ -208,9 +207,6 @@ public final class EChain extends AExpression {
 
         expression = expression.cast(settings, definition, variables);
 
-        exact = !settings.getNumericOverflow() &&
-            (operation == Operation.MUL || operation == Operation.DIV || operation == Operation.REM ||
-                operation == Operation.ADD || operation == Operation.SUB);
         there = AnalyzerCaster.getLegalCast(definition, location, last.after, promote, false);
         back = AnalyzerCaster.getLegalCast(definition, location, promote, last.after, true);
 
@@ -293,11 +289,9 @@ public final class EChain extends AExpression {
 
                     adapter.writeCast(there);
                     expression.write(settings, definition, adapter);
-                    adapter.writeBinaryInstruction(settings, definition, location, promote, operation);
+                    adapter.writeBinaryInstruction(definition, location, promote, operation);
 
-                    if (!exact || !adapter.writeExactInstruction(definition, promote.sort, link.after.sort)) {
-                        adapter.writeCast(back);
-                    }
+                    adapter.writeCast(back);
 
                     if (link.load && !post) {
                         adapter.writeDup(link.after.sort.size, link.size);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index cbf76373136..409d1081ae9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -31,8 +31,6 @@ import org.elasticsearch.painless.MethodWriter;
 
 import static org.elasticsearch.painless.WriterConstants.DEF_NEG_CALL;
 import static org.elasticsearch.painless.WriterConstants.DEF_NOT_CALL;
-import static org.elasticsearch.painless.WriterConstants.NEGATEEXACT_INT;
-import static org.elasticsearch.painless.WriterConstants.NEGATEEXACT_LONG;
 
 /**
  * Represents a unary math expression.
@@ -147,14 +145,13 @@ public final class EUnary extends AExpression {
         child = child.cast(settings, definition, variables);
 
         if (child.constant != null) {
-            final boolean overflow = settings.getNumericOverflow();
             final Sort sort = promote.sort;
 
 
             if (sort == Sort.INT) {
-                constant = overflow ? -(int)child.constant : Math.negateExact((int)child.constant);
+                constant = -(int)child.constant;
             } else if (sort == Sort.LONG) {
-                constant = overflow ? -(long)child.constant : Math.negateExact((long)child.constant);
+                constant = -(long)child.constant;
             } else if (sort == Sort.FLOAT) {
                 constant = -(float)child.constant;
             } else if (sort == Sort.DOUBLE) {
@@ -211,17 +208,7 @@ public final class EUnary extends AExpression {
                 if (sort == Sort.DEF) {
                     adapter.invokeStatic(definition.getType("Def").type, DEF_NEG_CALL);
                 } else {
-                    if (settings.getNumericOverflow()) {
-                        adapter.math(MethodWriter.NEG, type);
-                    } else {
-                        if (sort == Sort.INT) {
-                            adapter.invokeStatic(definition.getType("Math").type, NEGATEEXACT_INT);
-                        } else if (sort == Sort.LONG) {
-                            adapter.invokeStatic(definition.getType("Math").type, NEGATEEXACT_LONG);
-                        } else {
-                            throw new IllegalStateException(error("Illegal tree structure."));
-                        }
-                    }
+                    adapter.math(MethodWriter.NEG, type);
                 }
             } else if (operation != Operation.ADD) {
                 throw new IllegalStateException(error("Illegal tree structure."));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FloatOverflowDisabledTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FloatOverflowDisabledTests.java
deleted file mode 100644
index 7bec0b110df..00000000000
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FloatOverflowDisabledTests.java
+++ /dev/null
@@ -1,293 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless;
-
-import java.util.Collections;
-import java.util.Map;
-
-/** Tests floating point overflow with numeric overflow disabled */
-public class FloatOverflowDisabledTests extends ScriptTestCase {
-
-    /** wire overflow to false for all tests */
-    @Override
-    public Object exec(String script, Map<String, Object> vars) {
-        return exec(script, vars, Collections.singletonMap(CompilerSettings.NUMERIC_OVERFLOW, "false"));
-    }
-
-    public void testAssignmentAdditionOverflow() {
-        // float
-        try {
-            exec("float x = 3.4028234663852886E38f; x += 3.4028234663852886E38f; return x;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("float x = -3.4028234663852886E38f; x += -3.4028234663852886E38f; return x;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-
-        // double
-        try {
-            exec("double x = 1.7976931348623157E308; x += 1.7976931348623157E308; return x;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("double x = -1.7976931348623157E308; x += -1.7976931348623157E308; return x;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testAssignmentSubtractionOverflow() {
-        // float
-        try {
-            exec("float x = 3.4028234663852886E38f; x -= -3.4028234663852886E38f; return x;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("float x = -3.4028234663852886E38f; x -= 3.4028234663852886E38f; return x;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-
-        // double
-        try {
-            exec("double x = 1.7976931348623157E308; x -= -1.7976931348623157E308; return x;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("double x = -1.7976931348623157E308; x -= 1.7976931348623157E308; return x;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testAssignmentMultiplicationOverflow() {
-        // float
-        try {
-            exec("float x = 3.4028234663852886E38f; x *= 3.4028234663852886E38f; return x;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("float x = 3.4028234663852886E38f; x *= -3.4028234663852886E38f; return x;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-
-        // double
-        try {
-            exec("double x = 1.7976931348623157E308; x *= 1.7976931348623157E308; return x;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("double x = 1.7976931348623157E308; x *= -1.7976931348623157E308; return x;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testAssignmentDivisionOverflow() {
-        // float
-        try {
-            exec("float x = 3.4028234663852886E38f; x /= 1.401298464324817E-45f; return x;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("float x = 3.4028234663852886E38f; x /= -1.401298464324817E-45f; return x;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("float x = 1.0f; x /= 0.0f; return x;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-
-        // double
-        try {
-            exec("double x = 1.7976931348623157E308; x /= 4.9E-324; return x;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("double x = 1.7976931348623157E308; x /= -4.9E-324; return x;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("double x = 1.0f; x /= 0.0; return x;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testAddition() throws Exception {
-        try {
-            exec("float x = 3.4028234663852886E38f; float y = 3.4028234663852886E38f; return x + y;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("double x = 1.7976931348623157E308; double y = 1.7976931348623157E308; return x + y;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testAdditionConst() throws Exception {
-        try {
-            exec("return 3.4028234663852886E38f + 3.4028234663852886E38f;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("return 1.7976931348623157E308 + 1.7976931348623157E308;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testSubtraction() throws Exception {
-        try {
-            exec("float x = -3.4028234663852886E38f; float y = 3.4028234663852886E38f; return x - y;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("double x = -1.7976931348623157E308; double y = 1.7976931348623157E308; return x - y;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testSubtractionConst() throws Exception {
-        try {
-            exec("return -3.4028234663852886E38f - 3.4028234663852886E38f;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("return -1.7976931348623157E308 - 1.7976931348623157E308;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testMultiplication() throws Exception {
-        try {
-            exec("float x = 3.4028234663852886E38f; float y = 3.4028234663852886E38f; return x * y;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("double x = 1.7976931348623157E308; double y = 1.7976931348623157E308; return x * y;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testMultiplicationConst() throws Exception {
-        try {
-            exec("return 3.4028234663852886E38f * 3.4028234663852886E38f;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("return 1.7976931348623157E308 * 1.7976931348623157E308;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testDivision() throws Exception {
-        try {
-            exec("float x = 3.4028234663852886E38f; float y = 1.401298464324817E-45f; return x / y;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("float x = 1.0f; float y = 0.0f; return x / y;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("double x = 1.7976931348623157E308; double y = 4.9E-324; return x / y;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("double x = 1.0; double y = 0.0; return x / y;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testDivisionConst() throws Exception {
-        try {
-            exec("return 3.4028234663852886E38f / 1.401298464324817E-45f;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("return 1.0f / 0.0f;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("return 1.7976931348623157E308 / 4.9E-324;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("return 1.0 / 0.0;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testDivisionNaN() throws Exception {
-        // float division, constant division, and assignment
-        try {
-            exec("float x = 0f; float y = 0f; return x / y;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("return 0f / 0f;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("float x = 0f; x /= 0f; return x;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-
-        // double division, constant division, and assignment
-        try {
-            exec("double x = 0.0; double y = 0.0; return x / y;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("return 0.0 / 0.0;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("double x = 0.0; x /= 0.0; return x;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testRemainderNaN() throws Exception {
-        // float division, constant division, and assignment
-        try {
-            exec("float x = 1f; float y = 0f; return x % y;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("return 1f % 0f;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("float x = 1f; x %= 0f; return x;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-
-        // double division, constant division, and assignment
-        try {
-            exec("double x = 1.0; double y = 0.0; return x % y;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("return 1.0 % 0.0;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-        try {
-            exec("double x = 1.0; x %= 0.0; return x;");
-            fail("didn't hit expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FloatOverflowEnabledTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FloatOverflowTests.java
similarity index 94%
rename from modules/lang-painless/src/test/java/org/elasticsearch/painless/FloatOverflowEnabledTests.java
rename to modules/lang-painless/src/test/java/org/elasticsearch/painless/FloatOverflowTests.java
index ccfd2232e88..4b3eb8f0e7f 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FloatOverflowEnabledTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FloatOverflowTests.java
@@ -19,17 +19,8 @@
 
 package org.elasticsearch.painless;
 
-import java.util.Collections;
-import java.util.Map;
-
-/** Tests floating point overflow with numeric overflow enabled */
-public class FloatOverflowEnabledTests extends ScriptTestCase {
-
-    /** wire overflow to true for all tests */
-    @Override
-    public Object exec(String script, Map<String, Object> vars) {
-        return exec(script, vars, Collections.singletonMap(CompilerSettings.NUMERIC_OVERFLOW, "true"));
-    }
+/** Tests floating point overflow cases */
+public class FloatOverflowTests extends ScriptTestCase {
 
     public void testAssignmentAdditionOverflow() {
         // float
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/IntegerOverflowDisabledTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/IntegerOverflowDisabledTests.java
deleted file mode 100644
index f4adcfce878..00000000000
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/IntegerOverflowDisabledTests.java
+++ /dev/null
@@ -1,444 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless;
-
-import java.util.Collections;
-import java.util.Map;
-
-/** Tests integer overflow with numeric overflow disabled */
-public class IntegerOverflowDisabledTests extends ScriptTestCase {
-
-    /** wire overflow to true for all tests */
-    @Override
-    public Object exec(String script, Map<String, Object> vars) {
-        return exec(script, vars, Collections.singletonMap(CompilerSettings.NUMERIC_OVERFLOW, "false"));
-    }
-
-    public void testAssignmentAdditionOverflow() {
-        // byte
-        try {
-            exec("byte x = 0; x += 128; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("byte x = 0; x += -129; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        // short
-        try {
-            exec("short x = 0; x += 32768; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("byte x = 0; x += -32769; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        // char
-        try {
-            exec("char x = 0; x += 65536; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("char x = 0; x += -65536; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        // int
-        try {
-            exec("int x = 1; x += 2147483647; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("int x = -2; x += -2147483647; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        // long
-        try {
-            exec("long x = 1; x += 9223372036854775807L; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("long x = -2; x += -9223372036854775807L; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testAssignmentSubtractionOverflow() {
-        // byte
-        try {
-            exec("byte x = 0; x -= -128; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("byte x = 0; x -= 129; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        // short
-        try {
-            exec("short x = 0; x -= -32768; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("byte x = 0; x -= 32769; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        // char
-        try {
-            exec("char x = 0; x -= -65536; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("char x = 0; x -= 65536; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        // int
-        try {
-            exec("int x = 1; x -= -2147483647; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("int x = -2; x -= 2147483647; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        // long
-        try {
-            exec("long x = 1; x -= -9223372036854775807L; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("long x = -2; x -= 9223372036854775807L; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testAssignmentMultiplicationOverflow() {
-        // byte
-        try {
-            exec("byte x = 2; x *= 128; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("byte x = 2; x *= -128; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        // char
-        try {
-            exec("char x = 2; x *= 65536; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("char x = 2; x *= -65536; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        // int
-        try {
-            exec("int x = 2; x *= 2147483647; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("int x = 2; x *= -2147483647; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        // long
-        try {
-            exec("long x = 2; x *= 9223372036854775807L; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("long x = 2; x *= -9223372036854775807L; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testAssignmentDivisionOverflow() {
-        // byte
-        try {
-            exec("byte x = (byte) -128; x /= -1; return x;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
-
-        // short
-        try {
-            exec("short x = (short) -32768; x /= -1; return x;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
-
-        // cannot happen for char: unsigned
-
-        // int
-        try {
-            exec("int x = -2147483647 - 1; x /= -1; return x;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
-
-        // long
-        try {
-            exec("long x = -9223372036854775807L - 1L; x /=-1L; return x;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testIncrementOverFlow() throws Exception {
-        // byte
-        try {
-            exec("byte x = 127; ++x; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("byte x = 127; x++; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("byte x = (byte) -128; --x; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("byte x = (byte) -128; x--; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        // short
-        try {
-            exec("short x = 32767; ++x; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("short x = 32767; x++; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("short x = (short) -32768; --x; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("short x = (short) -32768; x--; return x;");
-        } catch (ArithmeticException expected) {}
-
-        // char
-        try {
-            exec("char x = 65535; ++x; return x;");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("char x = 65535; x++; return x;");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("char x = (char) 0; --x; return x;");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("char x = (char) 0; x--; return x;");
-        } catch (ArithmeticException expected) {}
-
-        // int
-        try {
-            exec("int x = 2147483647; ++x; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("int x = 2147483647; x++; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("int x = (int) -2147483648L; --x; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("int x = (int) -2147483648L; x--; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        // long
-        try {
-            exec("long x = 9223372036854775807L; ++x; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("long x = 9223372036854775807L; x++; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("long x = -9223372036854775807L - 1L; --x; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("long x = -9223372036854775807L - 1L; x--; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testAddition() throws Exception {
-        try {
-            exec("int x = 2147483647; int y = 2147483647; return x + y;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("long x = 9223372036854775807L; long y = 9223372036854775807L; return x + y;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testAdditionConst() throws Exception {
-        try {
-            exec("return 2147483647 + 2147483647;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("return 9223372036854775807L + 9223372036854775807L;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-
-    public void testSubtraction() throws Exception {
-        try {
-            exec("int x = -10; int y = 2147483647; return x - y;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("long x = -10L; long y = 9223372036854775807L; return x - y;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testSubtractionConst() throws Exception {
-        try {
-            exec("return -10 - 2147483647;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("return -10L - 9223372036854775807L;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testMultiplication() throws Exception {
-        try {
-            exec("int x = 2147483647; int y = 2147483647; return x * y;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("long x = 9223372036854775807L; long y = 9223372036854775807L; return x * y;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testMultiplicationConst() throws Exception {
-        try {
-            exec("return 2147483647 * 2147483647;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("return 9223372036854775807L * 9223372036854775807L;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testDivision() throws Exception {
-        try {
-            exec("int x = -2147483647 - 1; int y = -1; return x / y;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("long x = -9223372036854775808L; long y = -1L; return x / y;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testDivisionConst() throws Exception {
-        try {
-            exec("return (-2147483648) / -1;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("return (-9223372036854775808L) / -1L;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testNegationOverflow() throws Exception {
-        try {
-            exec("int x = -2147483648; x = -x; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("long x = -9223372036854775808L; x = -x; return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testNegationOverflowConst() throws Exception {
-        try {
-            exec("int x = -(-2147483648); return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            exec("long x = -(-9223372036854775808L); return x;");
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/IntegerOverflowEnabledTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/IntegerOverflowTests.java
similarity index 95%
rename from modules/lang-painless/src/test/java/org/elasticsearch/painless/IntegerOverflowEnabledTests.java
rename to modules/lang-painless/src/test/java/org/elasticsearch/painless/IntegerOverflowTests.java
index 41b3f857c0a..1165547bf5a 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/IntegerOverflowEnabledTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/IntegerOverflowTests.java
@@ -19,17 +19,8 @@
 
 package org.elasticsearch.painless;
 
-import java.util.Collections;
-import java.util.Map;
-
-/** Tests integer overflow with numeric overflow enabled */
-public class IntegerOverflowEnabledTests extends ScriptTestCase {
-
-    /** wire overflow to true for all tests */
-    @Override
-    public Object exec(String script, Map<String, Object> vars) {
-        return exec(script, vars, Collections.singletonMap(CompilerSettings.NUMERIC_OVERFLOW, "true"));
-    }
+/** Tests integer overflow cases */
+public class IntegerOverflowTests extends ScriptTestCase {
 
     public void testAssignmentAdditionOverflow() {
         // byte
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
index 2ccd2f1460a..c3ce127034e 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
@@ -48,7 +48,7 @@ public abstract class ScriptTestCase extends ESTestCase {
 
     /** Compiles and returns the result of {@code script} with access to {@code vars} */
     public Object exec(String script, Map<String, Object> vars) {
-        return exec(script, vars, Collections.singletonMap(CompilerSettings.NUMERIC_OVERFLOW, Boolean.toString(random().nextBoolean())));
+        return exec(script, vars, Collections.emptyMap());
     }
 
     /** Compiles and returns the result of {@code script} with access to {@code vars} and compile-time parameters */
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/UtilityTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/UtilityTests.java
deleted file mode 100644
index ba476fac7f2..00000000000
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/UtilityTests.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless;
-
-import org.elasticsearch.test.ESTestCase;
-
-/**
- * Tests utility methods (typically built-ins)
- */
-public class UtilityTests extends ESTestCase {
-
-    public void testDivideWithoutOverflowInt() {
-        assertEquals(5 / 2, Utility.divideWithoutOverflow(5, 2));
-
-        try {
-            Utility.divideWithoutOverflow(Integer.MIN_VALUE, -1);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            Utility.divideWithoutOverflow(5, 0);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testDivideWithoutOverflowLong() {
-        assertEquals(5L / 2L, Utility.divideWithoutOverflow(5L, 2L));
-
-        try {
-            Utility.divideWithoutOverflow(Long.MIN_VALUE, -1L);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            Utility.divideWithoutOverflow(5L, 0L);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testToByteExact() {
-        for (int b = Byte.MIN_VALUE; b < Byte.MAX_VALUE; b++) {
-            assertEquals((byte)b, Utility.toByteExact(b));
-        }
-
-        try {
-            Utility.toByteExact(Byte.MIN_VALUE - 1);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            Utility.toByteExact(Byte.MAX_VALUE + 1);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testToShortExact() {
-        for (int s = Short.MIN_VALUE; s < Short.MAX_VALUE; s++) {
-            assertEquals((short)s, Utility.toShortExact(s));
-        }
-
-        try {
-            Utility.toShortExact(Short.MIN_VALUE - 1);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            Utility.toShortExact(Short.MAX_VALUE + 1);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testToCharExact() {
-        for (int c = Character.MIN_VALUE; c < Character.MAX_VALUE; c++) {
-            assertEquals((char)c, Utility.toCharExact(c));
-        }
-
-        try {
-            Utility.toCharExact(Character.MIN_VALUE - 1);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            Utility.toCharExact(Character.MAX_VALUE + 1);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testAddWithoutOverflowFloat() {
-        assertEquals(10F, Utility.addWithoutOverflow(5F, 5F), 0F);
-        assertTrue(Float.isNaN(Utility.addWithoutOverflow(5F, Float.NaN)));
-        assertTrue(Float.isNaN(Utility.addWithoutOverflow(Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY)));
-
-        try {
-            Utility.addWithoutOverflow(Float.MAX_VALUE, Float.MAX_VALUE);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            Utility.addWithoutOverflow(-Float.MAX_VALUE, -Float.MAX_VALUE);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testAddWithoutOverflowDouble() {
-        assertEquals(10D, Utility.addWithoutOverflow(5D, 5D), 0D);
-        assertTrue(Double.isNaN(Utility.addWithoutOverflow(5D, Double.NaN)));
-        assertTrue(Double.isNaN(Utility.addWithoutOverflow(Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY)));
-
-        try {
-            Utility.addWithoutOverflow(Double.MAX_VALUE, Double.MAX_VALUE);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            Utility.addWithoutOverflow(-Double.MAX_VALUE, -Double.MAX_VALUE);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testSubtractWithoutOverflowFloat() {
-        assertEquals(5F, Utility.subtractWithoutOverflow(10F, 5F), 0F);
-        assertTrue(Float.isNaN(Utility.subtractWithoutOverflow(5F, Float.NaN)));
-        assertTrue(Float.isNaN(Utility.subtractWithoutOverflow(Float.POSITIVE_INFINITY, Float.POSITIVE_INFINITY)));
-
-        try {
-            Utility.subtractWithoutOverflow(Float.MAX_VALUE, -Float.MAX_VALUE);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            Utility.subtractWithoutOverflow(-Float.MAX_VALUE, Float.MAX_VALUE);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testSubtractWithoutOverflowDouble() {
-        assertEquals(5D, Utility.subtractWithoutOverflow(10D, 5D), 0D);
-        assertTrue(Double.isNaN(Utility.subtractWithoutOverflow(5D, Double.NaN)));
-        assertTrue(Double.isNaN(Utility.subtractWithoutOverflow(Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY)));
-
-        try {
-            Utility.subtractWithoutOverflow(Double.MAX_VALUE, -Double.MAX_VALUE);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            Utility.subtractWithoutOverflow(-Double.MAX_VALUE, Double.MAX_VALUE);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testMultiplyWithoutOverflowFloat() {
-        assertEquals(25F, Utility.multiplyWithoutOverflow(5F, 5F), 0F);
-        assertTrue(Float.isNaN(Utility.multiplyWithoutOverflow(5F, Float.NaN)));
-        assertEquals(Float.POSITIVE_INFINITY, Utility.multiplyWithoutOverflow(5F, Float.POSITIVE_INFINITY), 0F);
-
-        try {
-            Utility.multiplyWithoutOverflow(Float.MAX_VALUE, Float.MAX_VALUE);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testMultiplyWithoutOverflowDouble() {
-        assertEquals(25D, Utility.multiplyWithoutOverflow(5D, 5D), 0D);
-        assertTrue(Double.isNaN(Utility.multiplyWithoutOverflow(5D, Double.NaN)));
-        assertEquals(Double.POSITIVE_INFINITY, Utility.multiplyWithoutOverflow(5D, Double.POSITIVE_INFINITY), 0D);
-
-        try {
-            Utility.multiplyWithoutOverflow(Double.MAX_VALUE, Double.MAX_VALUE);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testDivideWithoutOverflowFloat() {
-        assertEquals(5F, Utility.divideWithoutOverflow(25F, 5F), 0F);
-        assertTrue(Float.isNaN(Utility.divideWithoutOverflow(5F, Float.NaN)));
-        assertEquals(Float.POSITIVE_INFINITY, Utility.divideWithoutOverflow(Float.POSITIVE_INFINITY, 5F), 0F);
-
-        try {
-            Utility.divideWithoutOverflow(Float.MAX_VALUE, Float.MIN_VALUE);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            Utility.divideWithoutOverflow(0F, 0F);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            Utility.divideWithoutOverflow(5F, 0F);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testDivideWithoutOverflowDouble() {
-        assertEquals(5D, Utility.divideWithoutOverflow(25D, 5D), 0D);
-        assertTrue(Double.isNaN(Utility.divideWithoutOverflow(5D, Double.NaN)));
-        assertEquals(Double.POSITIVE_INFINITY, Utility.divideWithoutOverflow(Double.POSITIVE_INFINITY, 5D), 0D);
-
-        try {
-            Utility.divideWithoutOverflow(Double.MAX_VALUE, Double.MIN_VALUE);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            Utility.divideWithoutOverflow(0D, 0D);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-
-        try {
-            Utility.divideWithoutOverflow(5D, 0D);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testRemainderWithoutOverflowFloat() {
-        assertEquals(1F, Utility.remainderWithoutOverflow(25F, 4F), 0F);
-
-        try {
-            Utility.remainderWithoutOverflow(5F, 0F);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-
-    public void testRemainderWithoutOverflowDouble() {
-        assertEquals(1D, Utility.remainderWithoutOverflow(25D, 4D), 0D);
-
-        try {
-            Utility.remainderWithoutOverflow(5D, 0D);
-            fail("did not get expected exception");
-        } catch (ArithmeticException expected) {}
-    }
-}

From a569130368deca90ff939708de8f001ff548fd72 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Thu, 19 May 2016 15:28:59 -0400
Subject: [PATCH 0359/1311] fix typo

---
 .../src/main/java/org/elasticsearch/painless/Def.java           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 9226fc3f098..7cc3d2f870e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -52,7 +52,7 @@ public final class Def {
     // TODO: Once Java has a factory for those in java.lang.invoke.MethodHandles, use it:
 
     /** Helper class for isolating MethodHandles and methods to get the length of arrays
-     * (to emulate a "arraystore" byteoode using MethodHandles).
+     * (to emulate a "arraystore" bytecode using MethodHandles).
      * This should really be a method in {@link MethodHandles} class!
      */
     private static final class ArrayLengthHelper {

From 0ee726ed0dfe352dc8362149ca80f4969088a607 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Thu, 19 May 2016 15:35:26 -0400
Subject: [PATCH 0360/1311] nuke eclipse warnings

---
 .../src/main/java/org/elasticsearch/painless/Def.java  | 10 ++++++++++
 .../painless/PainlessScriptEngineService.java          |  2 +-
 .../java/org/elasticsearch/painless/node/EUnary.java   |  1 -
 .../org/elasticsearch/painless/NeedsScoreTests.java    |  1 +
 4 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 7cc3d2f870e..e5967501664 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -72,14 +72,24 @@ public final class Def {
 
         private static final MethodHandle OBJECT_ARRAY_MH = ARRAY_TYPE_MH_MAPPING.get(Object[].class);
 
+        // NOTE: the following are actually used, javac just does not know :)
+        @SuppressWarnings("unused")
         static int getArrayLength(final boolean[] array) { return array.length; }
+        @SuppressWarnings("unused")
         static int getArrayLength(final byte[] array)    { return array.length; }
+        @SuppressWarnings("unused")
         static int getArrayLength(final short[] array)   { return array.length; }
+        @SuppressWarnings("unused")
         static int getArrayLength(final int[] array)     { return array.length; }
+        @SuppressWarnings("unused")
         static int getArrayLength(final long[] array)    { return array.length; }
+        @SuppressWarnings("unused")
         static int getArrayLength(final char[] array)    { return array.length; }
+        @SuppressWarnings("unused")
         static int getArrayLength(final float[] array)   { return array.length; }
+        @SuppressWarnings("unused")
         static int getArrayLength(final double[] array)  { return array.length; }
+        @SuppressWarnings("unused")
         static int getArrayLength(final Object[] array)  { return array.length; }
 
         static MethodHandle arrayLengthGetter(Class<?> arrayType) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
index 4c627d08eca..72a657cd7f0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
@@ -204,7 +204,7 @@ public final class PainlessScriptEngineService extends AbstractComponent impleme
      * Action taken when the engine is closed.
      */
     @Override
-    public void close() throws IOException {
+    public void close() {
         // Nothing to do.
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index 409d1081ae9..7e52790fa50 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -147,7 +147,6 @@ public final class EUnary extends AExpression {
         if (child.constant != null) {
             final Sort sort = promote.sort;
 
-
             if (sort == Sort.INT) {
                 constant = -(int)child.constant;
             } else if (sort == Sort.LONG) {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java
index 4b56d9751b6..3fe071c5221 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java
@@ -60,6 +60,7 @@ public class NeedsScoreTests extends ESSingleNodeTestCase {
         ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "painless", compiled),
                             lookup, Collections.<String, Object>emptyMap());
         assertTrue(ss.needsScores());
+        service.close();
     }
 
 }

From afa9ebad28095ea91a615acdc8a023dc734c377c Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Thu, 19 May 2016 16:03:11 -0400
Subject: [PATCH 0361/1311] dce

---
 .../main/java/org/elasticsearch/painless/Definition.java  | 8 +++-----
 .../main/java/org/elasticsearch/painless/node/LField.java | 8 --------
 2 files changed, 3 insertions(+), 13 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 68fc3a32262..bde6063cfe1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -174,17 +174,15 @@ public final class Definition {
     public static final class Field {
         public final String name;
         public final Struct owner;
-        public final Type generic;
         public final Type type;
         public final java.lang.reflect.Field reflect;
         public final MethodHandle getter;
         public final MethodHandle setter;
 
-        private Field(final String name, final Struct owner, final Type generic, final Type type,
+        private Field(final String name, final Struct owner, final Type type,
                       final java.lang.reflect.Field reflect, final MethodHandle getter, final MethodHandle setter) {
             this.name = name;
             this.owner = owner;
-            this.generic = generic;
             this.type = type;
             this.reflect = reflect;
             this.getter = getter;
@@ -1048,7 +1046,7 @@ public final class Definition {
                 " not found for class [" + owner.clazz.getName() + "].");
         }
 
-        final Field field = new Field(name, owner, type, type, reflect, getter, setter);
+        final Field field = new Field(name, owner, type, reflect, getter, setter);
 
         if (isStatic) {
             // require that all static fields are static final
@@ -1138,7 +1136,7 @@ public final class Definition {
                     }
 
                     owner.members.put(field.name,
-                        new Field(field.name, owner, field.type, field.generic, reflect, getter, setter));
+                        new Field(field.name, owner, field.type, reflect, getter, setter));
                 }
             }
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
index 06f820eba26..3ec456c69a4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
@@ -107,16 +107,8 @@ public final class LField extends ALink {
     void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isStatic(field.reflect.getModifiers())) {
             adapter.getStatic(field.owner.type, field.reflect.getName(), field.type.type);
-
-            if (!field.generic.clazz.equals(field.type.clazz)) {
-                adapter.checkCast(field.generic.type);
-            }
         } else {
             adapter.getField(field.owner.type, field.reflect.getName(), field.type.type);
-
-            if (!field.generic.clazz.equals(field.type.clazz)) {
-                adapter.checkCast(field.generic.type);
-            }
         }
     }
 

From 223cb6a7f0a3403aa60b48886a1bda980bc28fed Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 19 May 2016 18:03:36 -0400
Subject: [PATCH 0362/1311] [reindex] Mark test awaits fix because it is
 unstable

Fix coming.
---
 .../test/java/org/elasticsearch/index/reindex/RetryTests.java   | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
index 0b557898552..ff821724316 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
 import org.elasticsearch.action.ListenableActionFuture;
 import org.elasticsearch.action.bulk.BackoffPolicy;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
@@ -43,6 +44,7 @@ import static org.hamcrest.Matchers.greaterThan;
  * Integration test for retry behavior. Useful because retrying relies on the way that the rest of Elasticsearch throws exceptions and unit
  * tests won't verify that.
  */
+@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/18456")
 public class RetryTests extends ReindexTestCase {
     /**
      * The number of concurrent requests to test.

From cfe555d216120818b4823e967659b13a30f3b3a5 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Fri, 20 May 2016 00:46:05 +0200
Subject: [PATCH 0363/1311] move unused @SuppressWarnings one level up

---
 .../src/main/java/org/elasticsearch/painless/Def.java | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index e5967501664..889328e58db 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -55,6 +55,7 @@ public final class Def {
      * (to emulate a "arraystore" bytecode using MethodHandles).
      * This should really be a method in {@link MethodHandles} class!
      */
+    @SuppressWarnings("unused") // getArrayLength() methods are are actually used, javac just does not know :)
     private static final class ArrayLengthHelper {
         private static final Lookup PRIV_LOOKUP = MethodHandles.lookup();
 
@@ -72,24 +73,14 @@ public final class Def {
 
         private static final MethodHandle OBJECT_ARRAY_MH = ARRAY_TYPE_MH_MAPPING.get(Object[].class);
 
-        // NOTE: the following are actually used, javac just does not know :)
-        @SuppressWarnings("unused")
         static int getArrayLength(final boolean[] array) { return array.length; }
-        @SuppressWarnings("unused")
         static int getArrayLength(final byte[] array)    { return array.length; }
-        @SuppressWarnings("unused")
         static int getArrayLength(final short[] array)   { return array.length; }
-        @SuppressWarnings("unused")
         static int getArrayLength(final int[] array)     { return array.length; }
-        @SuppressWarnings("unused")
         static int getArrayLength(final long[] array)    { return array.length; }
-        @SuppressWarnings("unused")
         static int getArrayLength(final char[] array)    { return array.length; }
-        @SuppressWarnings("unused")
         static int getArrayLength(final float[] array)   { return array.length; }
-        @SuppressWarnings("unused")
         static int getArrayLength(final double[] array)  { return array.length; }
-        @SuppressWarnings("unused")
         static int getArrayLength(final Object[] array)  { return array.length; }
 
         static MethodHandle arrayLengthGetter(Class<?> arrayType) {

From 3962553551ed2c8366161e26da2d18c963eabc0d Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Thu, 19 May 2016 18:51:55 -0700
Subject: [PATCH 0364/1311] Removed user-facing boxing.

---
 .../painless/AnalyzerCaster.java              | 189 +++++++----------
 .../elasticsearch/painless/Definition.java    | 198 +-----------------
 .../elasticsearch/painless/node/EBinary.java  |  18 +-
 .../elasticsearch/painless/node/EChain.java   |  14 +-
 .../elasticsearch/painless/node/EComp.java    |  12 +-
 .../elasticsearch/painless/node/EUnary.java   |   6 +-
 .../painless/DefOperationTests.java           | 158 +++++++-------
 7 files changed, 190 insertions(+), 405 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
index d6f099e47f9..398dc0a7a61 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
@@ -115,29 +115,25 @@ public final class AnalyzerCaster {
         }
     }
 
-    public static Type promoteNumeric(final Definition definition, final Type from, final boolean decimal, final boolean primitive) {
+    public static Type promoteNumeric(final Definition definition, final Type from, final boolean decimal) {
         final Sort sort = from.sort;
 
         if (sort == Sort.DEF) {
             return definition.getType("def");
-        } else if ((sort == Sort.DOUBLE || sort == Sort.DOUBLE_OBJ) && decimal) {
-            return primitive ? definition.getType("double") : definition.getType("Double");
-        } else if ((sort == Sort.FLOAT || sort == Sort.FLOAT_OBJ) && decimal) {
-            return primitive ? definition.getType("float") : definition.getType("Float");
-        } else if (sort == Sort.LONG || sort == Sort.LONG_OBJ) {
-            return primitive ? definition.getType("long") : definition.getType("Long");
-        } else if (sort == Sort.INT   || sort == Sort.INT_OBJ   ||
-                   sort == Sort.CHAR  || sort == Sort.CHAR_OBJ  ||
-                   sort == Sort.SHORT || sort == Sort.SHORT_OBJ ||
-                   sort == Sort.BYTE  || sort == Sort.BYTE_OBJ) {
-            return primitive ? definition.getType("int") : definition.getType("Integer");
+        } else if ((sort == Sort.DOUBLE) && decimal) {
+            return definition.getType("double");
+        } else if ((sort == Sort.FLOAT) && decimal) {
+            return  definition.getType("float");
+        } else if (sort == Sort.LONG) {
+            return definition.getType("long");
+        } else if (sort == Sort.INT || sort == Sort.CHAR || sort == Sort.SHORT || sort == Sort.BYTE) {
+            return definition.getType("int");
         }
 
         return null;
     }
 
-    public static Type promoteNumeric(final Definition definition,
-                                      final Type from0, final Type from1, final boolean decimal, final boolean primitive) {
+    public static Type promoteNumeric(final Definition definition, final Type from0, final Type from1, final boolean decimal) {
         final Sort sort0 = from0.sort;
         final Sort sort1 = from1.sort;
 
@@ -146,26 +142,20 @@ public final class AnalyzerCaster {
         }
 
         if (decimal) {
-            if (sort0 == Sort.DOUBLE || sort0 == Sort.DOUBLE_OBJ ||
-                sort1 == Sort.DOUBLE || sort1 == Sort.DOUBLE_OBJ) {
-                return primitive ? definition.getType("double") : definition.getType("Double");
-            } else if (sort0 == Sort.FLOAT || sort0 == Sort.FLOAT_OBJ || sort1 == Sort.FLOAT || sort1 == Sort.FLOAT_OBJ) {
-                return primitive ? definition.getType("float") : definition.getType("Float");
+            if (sort0 == Sort.DOUBLE || sort1 == Sort.DOUBLE) {
+                return definition.getType("double");
+            } else if (sort0 == Sort.FLOAT || sort1 == Sort.FLOAT) {
+                return definition.getType("float");
             }
         }
 
-        if (sort0 == Sort.LONG || sort0 == Sort.LONG_OBJ ||
-            sort1 == Sort.LONG || sort1 == Sort.LONG_OBJ) {
-            return primitive ? definition.getType("long") : definition.getType("Long");
-        } else if (sort0 == Sort.INT   || sort0 == Sort.INT_OBJ   ||
-                   sort1 == Sort.INT   || sort1 == Sort.INT_OBJ   ||
-                   sort0 == Sort.CHAR  || sort0 == Sort.CHAR_OBJ  ||
-                   sort1 == Sort.CHAR  || sort1 == Sort.CHAR_OBJ  ||
-                   sort0 == Sort.SHORT || sort0 == Sort.SHORT_OBJ ||
-                   sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ ||
-                   sort0 == Sort.BYTE  || sort0 == Sort.BYTE_OBJ  ||
-                   sort1 == Sort.BYTE  || sort1 == Sort.BYTE_OBJ) {
-            return primitive ? definition.getType("int") : definition.getType("Integer");
+        if (sort0 == Sort.LONG || sort1 == Sort.LONG) {
+            return definition.getType("long");
+        } else if (sort0 == Sort.INT   || sort1 == Sort.INT   ||
+                   sort0 == Sort.CHAR  || sort1 == Sort.CHAR  ||
+                   sort0 == Sort.SHORT || sort1 == Sort.SHORT ||
+                   sort0 == Sort.BYTE  || sort1 == Sort.BYTE) {
+            return definition.getType("int");
         }
 
         return null;
@@ -179,7 +169,7 @@ public final class AnalyzerCaster {
             return definition.getType("String");
         }
 
-        return promoteNumeric(definition, from0, from1, true, true);
+        return promoteNumeric(definition, from0, from1, true);
     }
 
     public static Type promoteXor(final Definition definition, final Type from0, final Type from1) {
@@ -190,7 +180,7 @@ public final class AnalyzerCaster {
             return definition.getType("boolean");
         }
 
-        return promoteNumeric(definition, from0, from1, false, true);
+        return promoteNumeric(definition, from0, from1, false);
     }
 
     public static Type promoteEquality(final Definition definition, final Type from0, final Type from1) {
@@ -201,34 +191,13 @@ public final class AnalyzerCaster {
             return definition.getType("def");
         }
 
-        final boolean primitive = sort0.primitive && sort1.primitive;
-
-        if (sort0.bool && sort1.bool) {
-            return primitive ? definition.getType("boolean") : definition.getType("Boolean");
-        }
-
-        if (sort0.numeric && sort1.numeric) {
-            return promoteNumeric(definition, from0, from1, true, primitive);
-        }
-
-        return definition.getType("Object");
-    }
-
-    public static Type promoteReference(final Definition definition, final Type from0, final Type from1) {
-        final Sort sort0 = from0.sort;
-        final Sort sort1 = from1.sort;
-
-        if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return definition.getType("def");
-        }
-
         if (sort0.primitive && sort1.primitive) {
             if (sort0.bool && sort1.bool) {
                 return definition.getType("boolean");
             }
 
             if (sort0.numeric && sort1.numeric) {
-                return promoteNumeric(definition, from0, from1, true, true);
+                return promoteNumeric(definition, from0, from1, true);
             }
         }
 
@@ -248,123 +217,121 @@ public final class AnalyzerCaster {
             return definition.getType("def");
         }
 
-        final boolean primitive = sort0.primitive && sort1.primitive;
+        if (sort0.primitive && sort1.primitive) {
+            if (sort0.bool && sort1.bool) {
+                return definition.getType("boolean");
+            }
 
-        if (sort0.bool && sort1.bool) {
-            return primitive ? definition.getType("boolean") : definition.getType("Boolean");
-        }
-
-        if (sort0.numeric && sort1.numeric) {
-            if (sort0 == Sort.DOUBLE || sort0 == Sort.DOUBLE_OBJ || sort1 == Sort.DOUBLE || sort1 == Sort.DOUBLE_OBJ) {
-                return primitive ? definition.getType("double") : definition.getType("Double");
-            } else if (sort0 == Sort.FLOAT || sort0 == Sort.FLOAT_OBJ || sort1 == Sort.FLOAT || sort1 == Sort.FLOAT_OBJ) {
-                return primitive ? definition.getType("float") : definition.getType("Float");
-            } else if (sort0 == Sort.LONG || sort0 == Sort.LONG_OBJ || sort1 == Sort.LONG || sort1 == Sort.LONG_OBJ) {
-                return sort0.primitive && sort1.primitive ? definition.getType("long") : definition.getType("Long");
+            if (sort0 == Sort.DOUBLE || sort1 == Sort.DOUBLE) {
+                return definition.getType("double");
+            } else if (sort0 == Sort.FLOAT || sort1 == Sort.FLOAT) {
+                return definition.getType("float");
+            } else if (sort0 == Sort.LONG || sort1 == Sort.LONG) {
+                definition.getType("long");
             } else {
-                if (sort0 == Sort.BYTE || sort0 == Sort.BYTE_OBJ) {
-                    if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
-                        return primitive ? definition.getType("byte") : definition.getType("Byte");
-                    } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
+                if (sort0 == Sort.BYTE) {
+                    if (sort1 == Sort.BYTE) {
+                        return definition.getType("byte");
+                    } else if (sort1 == Sort.SHORT) {
                         if (const1 != null) {
                             final short constant = (short)const1;
 
                             if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? definition.getType("byte") : definition.getType("Byte");
+                                return definition.getType("byte");
                             }
                         }
 
-                        return primitive ? definition.getType("short") : definition.getType("Short");
-                    } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
-                        return primitive ? definition.getType("int") : definition.getType("Integer");
-                    } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
+                        return definition.getType("short");
+                    } else if (sort1 == Sort.CHAR) {
+                        return definition.getType("int");
+                    } else if (sort1 == Sort.INT) {
                         if (const1 != null) {
                             final int constant = (int)const1;
 
                             if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? definition.getType("byte") : definition.getType("Byte");
+                                return definition.getType("byte");
                             }
                         }
 
-                        return primitive ? definition.getType("int") : definition.getType("Integer");
+                        return definition.getType("int");
                     }
-                } else if (sort0 == Sort.SHORT || sort0 == Sort.SHORT_OBJ) {
-                    if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
+                } else if (sort0 == Sort.SHORT) {
+                    if (sort1 == Sort.BYTE) {
                         if (const0 != null) {
                             final short constant = (short)const0;
 
                             if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? definition.getType("byte") : definition.getType("Byte");
+                                return definition.getType("byte");
                             }
                         }
 
-                        return primitive ? definition.getType("short") : definition.getType("Short");
-                    } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
-                        return primitive ? definition.getType("short") : definition.getType("Short");
-                    } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
-                        return primitive ? definition.getType("int") : definition.getType("Integer");
-                    } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
+                        return definition.getType("short");
+                    } else if (sort1 == Sort.SHORT) {
+                        return definition.getType("short");
+                    } else if (sort1 == Sort.CHAR) {
+                        return definition.getType("int");
+                    } else if (sort1 == Sort.INT) {
                         if (const1 != null) {
                             final int constant = (int)const1;
 
                             if (constant <= Short.MAX_VALUE && constant >= Short.MIN_VALUE) {
-                                return primitive ? definition.getType("short") : definition.getType("Short");
+                                return definition.getType("short");
                             }
                         }
 
-                        return primitive ? definition.getType("int") : definition.getType("Integer");
+                        return definition.getType("int");
                     }
-                } else if (sort0 == Sort.CHAR || sort0 == Sort.CHAR_OBJ) {
-                    if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
-                        return primitive ? definition.getType("int") : definition.getType("Integer");
-                    } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
-                        return primitive ? definition.getType("int") : definition.getType("Integer");
-                    } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
-                        return primitive ? definition.getType("char") : definition.getType("Character");
-                    } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
+                } else if (sort0 == Sort.CHAR) {
+                    if (sort1 == Sort.BYTE) {
+                        return definition.getType("int");
+                    } else if (sort1 == Sort.SHORT) {
+                        return definition.getType("int");
+                    } else if (sort1 == Sort.CHAR) {
+                        return definition.getType("char");
+                    } else if (sort1 == Sort.INT) {
                         if (const1 != null) {
                             final int constant = (int)const1;
 
                             if (constant <= Character.MAX_VALUE && constant >= Character.MIN_VALUE) {
-                                return primitive ? definition.getType("byte") : definition.getType("Byte");
+                                return definition.getType("byte");
                             }
                         }
 
-                        return primitive ? definition.getType("int") : definition.getType("Integer");
+                        return definition.getType("int");
                     }
-                } else if (sort0 == Sort.INT || sort0 == Sort.INT_OBJ) {
-                    if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
+                } else if (sort0 == Sort.INT) {
+                    if (sort1 == Sort.BYTE) {
                         if (const0 != null) {
                             final int constant = (int)const0;
 
                             if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? definition.getType("byte") : definition.getType("Byte");
+                                return definition.getType("byte");
                             }
                         }
 
-                        return primitive ? definition.getType("int") : definition.getType("Integer");
-                    } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
+                        return definition.getType("int");
+                    } else if (sort1 == Sort.SHORT) {
                         if (const0 != null) {
                             final int constant = (int)const0;
 
                             if (constant <= Short.MAX_VALUE && constant >= Short.MIN_VALUE) {
-                                return primitive ? definition.getType("byte") : definition.getType("Byte");
+                                return definition.getType("byte");
                             }
                         }
 
-                        return primitive ? definition.getType("int") : definition.getType("Integer");
-                    } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
+                        return definition.getType("int");
+                    } else if (sort1 == Sort.CHAR) {
                         if (const0 != null) {
                             final int constant = (int)const0;
 
                             if (constant <= Character.MAX_VALUE && constant >= Character.MIN_VALUE) {
-                                return primitive ? definition.getType("byte") : definition.getType("Byte");
+                                return definition.getType("byte");
                             }
                         }
 
-                        return primitive ? definition.getType("int") : definition.getType("Integer");
-                    } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
-                        return primitive ? definition.getType("int") : definition.getType("Integer");
+                        return definition.getType("int");
+                    } else if (sort1 == Sort.INT) {
+                        return definition.getType("int");
                     }
                 }
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index bde6063cfe1..95d169cb886 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -524,7 +524,7 @@ public final class Definition {
         Type floatobjType = getType("Float");
         Type doubleobjType = getType("Double");
         Type stringType = getType("String");
-        
+
         addTransform(booleanType, objectType, "Boolean", "valueOf", true, false);
         addTransform(booleanType, defType, "Boolean", "valueOf", true, false);
         addTransform(booleanType, booleanobjType, "Boolean", "valueOf", true, false);
@@ -539,12 +539,6 @@ public final class Definition {
         addTransform(byteType, defType, "Byte", "valueOf", true, false);
         addTransform(byteType, numberType, "Byte", "valueOf", true, false);
         addTransform(byteType, byteobjType, "Byte", "valueOf", true, false);
-        addTransform(byteType, shortobjType, "Utility", "byteToShort", true, false);
-        addTransform(byteType, charobjType, "Utility", "byteToCharacter", true, true);
-        addTransform(byteType, intobjType, "Utility", "byteToInteger", true, false);
-        addTransform(byteType, longobjType, "Utility", "byteToLong", true, false);
-        addTransform(byteType, floatobjType, "Utility", "byteToFloat", true, false);
-        addTransform(byteType, doubleobjType, "Utility", "byteToDouble", true, false);
 
         addTransform(shortType, byteType, true);
         addTransform(shortType, charType, true);
@@ -555,13 +549,7 @@ public final class Definition {
         addTransform(shortType, objectType, "Short", "valueOf", true, false);
         addTransform(shortType, defType, "Short", "valueOf", true, false);
         addTransform(shortType, numberType, "Short", "valueOf", true, false);
-        addTransform(shortType, byteobjType, "Utility", "shortToByte", true, true);
         addTransform(shortType, shortobjType, "Short", "valueOf", true, false);
-        addTransform(shortType, charobjType, "Utility", "shortToCharacter", true, true);
-        addTransform(shortType, intobjType, "Utility", "shortToInteger", true, false);
-        addTransform(shortType, longobjType, "Utility", "shortToLong", true, false);
-        addTransform(shortType, floatobjType, "Utility", "shortToFloat", true, false);
-        addTransform(shortType, doubleobjType, "Utility", "shortToDouble", true, false);
 
         addTransform(charType, byteType, true);
         addTransform(charType, shortType, true);
@@ -572,13 +560,7 @@ public final class Definition {
         addTransform(charType, objectType, "Character", "valueOf", true, false);
         addTransform(charType, defType, "Character", "valueOf", true, false);
         addTransform(charType, numberType, "Utility", "charToInteger", true, false);
-        addTransform(charType, byteobjType, "Utility", "charToByte", true, true);
-        addTransform(charType, shortobjType, "Utility", "charToShort", true, true);
         addTransform(charType, charobjType, "Character", "valueOf", true, false);
-        addTransform(charType, intobjType, "Utility", "charToInteger", true, false);
-        addTransform(charType, longobjType, "Utility", "charToLong", true, false);
-        addTransform(charType, floatobjType, "Utility", "charToFloat", true, false);
-        addTransform(charType, doubleobjType, "Utility", "charToDouble", true, false);
         addTransform(charType, stringType, "Utility", "charToString", true, true);
 
         addTransform(intType, byteType, true);
@@ -590,13 +572,7 @@ public final class Definition {
         addTransform(intType, objectType, "Integer", "valueOf", true, false);
         addTransform(intType, defType, "Integer", "valueOf", true, false);
         addTransform(intType, numberType, "Integer", "valueOf", true, false);
-        addTransform(intType, byteobjType, "Utility", "intToByte", true, true);
-        addTransform(intType, shortobjType, "Utility", "intToShort", true, true);
-        addTransform(intType, charobjType, "Utility", "intToCharacter", true, true);
         addTransform(intType, intobjType, "Integer", "valueOf", true, false);
-        addTransform(intType, longobjType, "Utility", "intToLong", true, false);
-        addTransform(intType, floatobjType, "Utility", "intToFloat", true, false);
-        addTransform(intType, doubleobjType, "Utility", "intToDouble", true, false);
 
         addTransform(longType, byteType, true);
         addTransform(longType, shortType, true);
@@ -607,13 +583,7 @@ public final class Definition {
         addTransform(longType, objectType, "Long", "valueOf", true, false);
         addTransform(longType, defType, "Long", "valueOf", true, false);
         addTransform(longType, numberType, "Long", "valueOf", true, false);
-        addTransform(longType, byteobjType, "Utility", "longToByte", true, true);
-        addTransform(longType, shortobjType, "Utility", "longToShort", true, true);
-        addTransform(longType, charobjType, "Utility", "longToCharacter", true, true);
-        addTransform(longType, intobjType, "Utility", "longToInteger", true, true);
         addTransform(longType, longobjType, "Long", "valueOf", true, false);
-        addTransform(longType, floatobjType, "Utility", "longToFloat", true, false);
-        addTransform(longType, doubleobjType, "Utility", "longToDouble", true, false);
 
         addTransform(floatType, byteType, true);
         addTransform(floatType, shortType, true);
@@ -624,13 +594,7 @@ public final class Definition {
         addTransform(floatType, objectType, "Float", "valueOf", true, false);
         addTransform(floatType, defType, "Float", "valueOf", true, false);
         addTransform(floatType, numberType, "Float", "valueOf", true, false);
-        addTransform(floatType, byteobjType, "Utility", "floatToByte", true, true);
-        addTransform(floatType, shortobjType, "Utility", "floatToShort", true, true);
-        addTransform(floatType, charobjType, "Utility", "floatToCharacter", true, true);
-        addTransform(floatType, intobjType, "Utility", "floatToInteger", true, true);
-        addTransform(floatType, longobjType, "Utility", "floatToLong", true, true);
         addTransform(floatType, floatobjType, "Float", "valueOf", true, false);
-        addTransform(floatType, doubleobjType, "Utility", "floatToDouble", true, false);
 
         addTransform(doubleType, byteType, true);
         addTransform(doubleType, shortType, true);
@@ -641,23 +605,8 @@ public final class Definition {
         addTransform(doubleType, objectType, "Double", "valueOf", true, false);
         addTransform(doubleType, defType, "Double", "valueOf", true, false);
         addTransform(doubleType, numberType, "Double", "valueOf", true, false);
-        addTransform(doubleType, byteobjType, "Utility", "doubleToByte", true, true);
-        addTransform(doubleType, shortobjType, "Utility", "doubleToShort", true, true);
-        addTransform(doubleType, charobjType, "Utility", "doubleToCharacter", true, true);
-        addTransform(doubleType, intobjType, "Utility", "doubleToInteger", true, true);
-        addTransform(doubleType, longobjType, "Utility", "doubleToLong", true, true);
-        addTransform(doubleType, floatobjType, "Utility", "doubleToFloat", true, true);
         addTransform(doubleType, doubleobjType, "Double", "valueOf", true, false);
 
-        addTransform(objectType, booleanType, "Boolean", "booleanValue", false, true);
-        addTransform(objectType, byteType, "Number", "byteValue", false, true);
-        addTransform(objectType, shortType, "Number", "shortValue", false, true);
-        addTransform(objectType, charType, "Character", "charValue", false, true);
-        addTransform(objectType, intType, "Number", "intValue", false, true);
-        addTransform(objectType, longType, "Number", "longValue", false, true);
-        addTransform(objectType, floatType, "Number", "floatValue", false, true);
-        addTransform(objectType, doubleType, "Number", "doubleValue", false, true);
-
         addTransform(defType, booleanType, "Boolean", "booleanValue", false, false);
         addTransform(defType, byteType, "Def", "DefTobyteImplicit", true, false);
         addTransform(defType, shortType, "Def", "DefToshortImplicit", true, false);
@@ -666,13 +615,6 @@ public final class Definition {
         addTransform(defType, longType, "Def", "DefTolongImplicit", true, false);
         addTransform(defType, floatType, "Def", "DefTofloatImplicit", true, false);
         addTransform(defType, doubleType, "Def", "DefTodoubleImplicit", true, false);
-        addTransform(defType, byteobjType, "Def", "DefToByteImplicit", true, false);
-        addTransform(defType, shortobjType, "Def", "DefToShortImplicit", true, false);
-        addTransform(defType, charobjType, "Def", "DefToCharacterImplicit", true, false);
-        addTransform(defType, intobjType, "Def", "DefToIntegerImplicit", true, false);
-        addTransform(defType, longobjType, "Def", "DefToLongImplicit", true, false);
-        addTransform(defType, floatobjType, "Def", "DefToFloatImplicit", true, false);
-        addTransform(defType, doubleobjType, "Def", "DefToDoubleImplicit", true, false);
         addTransform(defType, byteType, "Def", "DefTobyteExplicit", true, true);
         addTransform(defType, shortType, "Def", "DefToshortExplicit", true, true);
         addTransform(defType, charType, "Def", "DefTocharExplicit", true, true);
@@ -680,130 +622,6 @@ public final class Definition {
         addTransform(defType, longType, "Def", "DefTolongExplicit", true, true);
         addTransform(defType, floatType, "Def", "DefTofloatExplicit", true, true);
         addTransform(defType, doubleType, "Def", "DefTodoubleExplicit", true, true);
-        addTransform(defType, byteobjType, "Def", "DefToByteExplicit", true, true);
-        addTransform(defType, shortobjType, "Def", "DefToShortExplicit", true, true);
-        addTransform(defType, charobjType, "Def", "DefToCharacterExplicit", true, true);
-        addTransform(defType, intobjType, "Def", "DefToIntegerExplicit", true, true);
-        addTransform(defType, longobjType, "Def", "DefToLongExplicit", true, true);
-        addTransform(defType, floatobjType, "Def", "DefToFloatExplicit", true, true);
-        addTransform(defType, doubleobjType, "Def", "DefToDoubleExplicit", true, true);
-
-        addTransform(numberType, byteType, "Number", "byteValue", false, true);
-        addTransform(numberType, shortType, "Number", "shortValue", false, true);
-        addTransform(numberType, charType, "Utility", "NumberTochar", true, true);
-        addTransform(numberType, intType, "Number", "intValue", false, true);
-        addTransform(numberType, longType, "Number", "longValue", false, true);
-        addTransform(numberType, floatType, "Number", "floatValue", false, true);
-        addTransform(numberType, doubleType, "Number", "doubleValue", false, true);
-        addTransform(numberType, booleanobjType, "Utility", "NumberToBoolean", true, true);
-        addTransform(numberType, byteobjType, "Utility", "NumberToByte", true, true);
-        addTransform(numberType, shortobjType, "Utility", "NumberToShort", true, true);
-        addTransform(numberType, charobjType, "Utility", "NumberToCharacter", true, true);
-        addTransform(numberType, intobjType, "Utility", "NumberToInteger", true, true);
-        addTransform(numberType, longobjType, "Utility", "NumberToLong", true, true);
-        addTransform(numberType, floatobjType, "Utility", "NumberToFloat", true, true);
-        addTransform(numberType, doubleobjType, "Utility", "NumberToDouble", true, true);
-
-        addTransform(booleanobjType, booleanType, "Boolean", "booleanValue", false, false);
-
-        addTransform(byteobjType, byteType, "Byte", "byteValue", false, false);
-        addTransform(byteobjType, shortType, "Byte", "shortValue", false, false);
-        addTransform(byteobjType, charType, "Utility", "ByteTochar", true, false);
-        addTransform(byteobjType, intType, "Byte", "intValue", false, false);
-        addTransform(byteobjType, longType, "Byte", "longValue", false, false);
-        addTransform(byteobjType, floatType, "Byte", "floatValue", false, false);
-        addTransform(byteobjType, doubleType, "Byte", "doubleValue", false, false);
-        addTransform(byteobjType, shortobjType, "Utility", "NumberToShort", true, false);
-        addTransform(byteobjType, charobjType, "Utility", "NumberToCharacter", true, false);
-        addTransform(byteobjType, intobjType, "Utility", "NumberToInteger", true, false);
-        addTransform(byteobjType, longobjType, "Utility", "NumberToLong", true, false);
-        addTransform(byteobjType, floatobjType, "Utility", "NumberToFloat", true, false);
-        addTransform(byteobjType, doubleobjType, "Utility", "NumberToDouble", true, false);
-
-        addTransform(shortobjType, byteType, "Short", "byteValue", false, true);
-        addTransform(shortobjType, shortType, "Short", "shortValue", false, true);
-        addTransform(shortobjType, charType, "Utility", "ShortTochar", true, false);
-        addTransform(shortobjType, intType, "Short", "intValue", false, false);
-        addTransform(shortobjType, longType, "Short", "longValue", false, false);
-        addTransform(shortobjType, floatType, "Short", "floatValue", false, false);
-        addTransform(shortobjType, doubleType, "Short", "doubleValue", false, false);
-        addTransform(shortobjType, byteobjType, "Utility", "NumberToByte", true, true);
-        addTransform(shortobjType, charobjType, "Utility", "NumberToCharacter", true, true);
-        addTransform(shortobjType, intobjType, "Utility", "NumberToInteger", true, false);
-        addTransform(shortobjType, longobjType, "Utility", "NumberToLong", true, false);
-        addTransform(shortobjType, floatobjType, "Utility", "NumberToFloat", true, false);
-        addTransform(shortobjType, doubleobjType, "Utility", "NumberToDouble", true, false);
-
-        addTransform(charobjType, byteType, "Utility", "CharacterTobyte", true, true);
-        addTransform(charobjType, shortType, "Utility", "CharacterToshort", true, false);
-        addTransform(charobjType, charType, "Character", "charValue", false, true);
-        addTransform(charobjType, intType, "Utility", "CharacterToint", true, false);
-        addTransform(charobjType, longType, "Utility", "CharacterTolong", true, false);
-        addTransform(charobjType, floatType, "Utility", "CharacterTofloat", true, false);
-        addTransform(charobjType, doubleType, "Utility", "CharacterTodouble", true, false);
-        addTransform(charobjType, byteobjType, "Utility", "CharacterToByte", true, true);
-        addTransform(charobjType, shortobjType, "Utility", "CharacterToShort", true, true);
-        addTransform(charobjType, intobjType, "Utility", "CharacterToInteger", true, false);
-        addTransform(charobjType, longobjType, "Utility", "CharacterToLong", true, false);
-        addTransform(charobjType, floatobjType, "Utility", "CharacterToFloat", true, false);
-        addTransform(charobjType, doubleobjType, "Utility", "CharacterToDouble", true, false);
-        addTransform(charobjType, stringType, "Utility", "CharacterToString", true, true);
-
-        addTransform(intobjType, byteType, "Integer", "byteValue", false, true);
-        addTransform(intobjType, shortType, "Integer", "shortValue", false, true);
-        addTransform(intobjType, charType, "Utility", "IntegerTochar", true, true);
-        addTransform(intobjType, intType, "Integer", "intValue", false, false);
-        addTransform(intobjType, longType, "Integer", "longValue", false, false);
-        addTransform(intobjType, floatType, "Integer", "floatValue", false, false);
-        addTransform(intobjType, doubleType, "Integer", "doubleValue", false, false);
-        addTransform(intobjType, byteobjType, "Utility", "NumberToByte", true, true);
-        addTransform(intobjType, shortobjType, "Utility", "NumberToShort", true, true);
-        addTransform(intobjType, charobjType, "Utility", "NumberToCharacter", true, true);
-        addTransform(intobjType, longobjType, "Utility", "NumberToLong", true, false);
-        addTransform(intobjType, floatobjType, "Utility", "NumberToFloat", true, false);
-        addTransform(intobjType, doubleobjType, "Utility", "NumberToDouble", true, false);
-
-        addTransform(longobjType, byteType, "Long", "byteValue", false, true);
-        addTransform(longobjType, shortType, "Long", "shortValue", false, true);
-        addTransform(longobjType, charType, "Utility", "LongTochar", true, true);
-        addTransform(longobjType, intType, "Long", "intValue", false, true);
-        addTransform(longobjType, longType, "Long", "longValue", false, false);
-        addTransform(longobjType, floatType, "Long", "floatValue", false, false);
-        addTransform(longobjType, doubleType, "Long", "doubleValue", false, false);
-        addTransform(longobjType, byteobjType, "Utility", "NumberToByte", true, true);
-        addTransform(longobjType, shortobjType, "Utility", "NumberToShort", true, true);
-        addTransform(longobjType, charobjType, "Utility", "NumberToCharacter", true, true);
-        addTransform(longobjType, intobjType, "Utility", "NumberToInteger", true, true);
-        addTransform(longobjType, floatobjType, "Utility", "NumberToFloat", true, false);
-        addTransform(longobjType, doubleobjType, "Utility", "NumberToDouble", true, false);
-
-        addTransform(floatobjType, byteType, "Float", "byteValue", false, true);
-        addTransform(floatobjType, shortType, "Float", "shortValue", false, true);
-        addTransform(floatobjType, charType, "Utility", "FloatTochar", true, true);
-        addTransform(floatobjType, intType, "Float", "intValue", false, true);
-        addTransform(floatobjType, longType, "Float", "longValue", false, true);
-        addTransform(floatobjType, floatType, "Float", "floatValue", false, false);
-        addTransform(floatobjType, doubleType, "Float", "doubleValue", false, false);
-        addTransform(floatobjType, byteobjType, "Utility", "NumberToByte", true, true);
-        addTransform(floatobjType, shortobjType, "Utility", "NumberToShort", true, true);
-        addTransform(floatobjType, charobjType, "Utility", "NumberToCharacter", true, true);
-        addTransform(floatobjType, intobjType, "Utility", "NumberToInteger", true, true);
-        addTransform(floatobjType, longobjType, "Utility", "NumberToLong", true, true);
-        addTransform(floatobjType, doubleobjType, "Utility", "NumberToDouble", true, false);
-
-        addTransform(doubleobjType, byteType, "Double", "byteValue", false, true);
-        addTransform(doubleobjType, shortType, "Double", "shortValue", false, true);
-        addTransform(doubleobjType, charType, "Utility", "DoubleTochar", true, true);
-        addTransform(doubleobjType, intType, "Double", "intValue", false, true);
-        addTransform(doubleobjType, longType, "Double", "longValue", false, true);
-        addTransform(doubleobjType, floatType, "Double", "floatValue", false, true);
-        addTransform(doubleobjType, doubleType, "Double", "doubleValue", false, false);
-        addTransform(doubleobjType, byteobjType, "Utility", "NumberToByte", true, true);
-        addTransform(doubleobjType, shortobjType, "Utility", "NumberToShort", true, true);
-        addTransform(doubleobjType, charobjType, "Utility", "NumberToCharacter", true, true);
-        addTransform(doubleobjType, intobjType, "Utility", "NumberToInteger", true, true);
-        addTransform(doubleobjType, longobjType, "Utility", "NumberToLong", true, true);
-        addTransform(doubleobjType, floatobjType, "Utility", "NumberToFloat", true, true);
 
         addTransform(stringType, charType, "Utility", "StringTochar", true, true);
         addTransform(stringType, charobjType, "Utility", "StringToCharacter", true, true);
@@ -823,7 +641,7 @@ public final class Definition {
         structsMap.put(name, struct);
         simpleTypesMap.put(name, getType(name));
     }
-    
+
     private final void addConstructorInternal(final String struct, final String name, final Type[] args) {
         final Struct owner = structsMap.get(struct);
 
@@ -874,8 +692,8 @@ public final class Definition {
 
         owner.constructors.put(methodKey, constructor);
     }
-    
-    /** 
+
+    /**
      * Adds a new signature to the definition.
      * <p>
      * Signatures have the following forms:
@@ -930,7 +748,7 @@ public final class Definition {
         }
     }
 
-    private final void addMethodInternal(final String struct, final String name, final String alias, 
+    private final void addMethodInternal(final String struct, final String name, final String alias,
                                          final Type rtn, final Type[] args) {
         final Struct owner = structsMap.get(struct);
 
@@ -968,7 +786,7 @@ public final class Definition {
         try {
             reflect = owner.clazz.getMethod(alias == null ? name : alias, classes);
         } catch (final NoSuchMethodException exception) {
-            throw new IllegalArgumentException("Method [" + (alias == null ? name : alias) + 
+            throw new IllegalArgumentException("Method [" + (alias == null ? name : alias) +
                 "] not found for class [" + owner.clazz.getName() + "]" +
                 " with arguments " + Arrays.toString(classes) + ".");
         }
@@ -1001,7 +819,7 @@ public final class Definition {
             owner.methods.put(methodKey, method);
         }
     }
-    
+
     private final void addFieldInternal(final String struct, final String name, final String alias,
                                         final Type type) {
         final Struct owner = structsMap.get(struct);
@@ -1029,7 +847,7 @@ public final class Definition {
             throw new IllegalArgumentException("Field [" + (alias == null ? name : alias) + "]" +
                 " not found for class [" + owner.clazz.getName() + "].");
         }
-        
+
         final int modifiers = reflect.getModifiers();
         boolean isStatic = java.lang.reflect.Modifier.isStatic(modifiers);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index f22b7394ca0..483e82cc500 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -80,7 +80,7 @@ public final class EBinary extends AExpression {
         left.analyze(settings, definition, variables);
         right.analyze(settings, definition, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply multiply [*] to types " +
@@ -116,7 +116,7 @@ public final class EBinary extends AExpression {
         left.analyze(settings, definition, variables);
         right.analyze(settings, definition, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply divide [/] to types " +
@@ -152,7 +152,7 @@ public final class EBinary extends AExpression {
         left.analyze(settings, definition, variables);
         right.analyze(settings, definition, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply remainder [%] to types " +
@@ -240,7 +240,7 @@ public final class EBinary extends AExpression {
         left.analyze(settings, definition, variables);
         right.analyze(settings, definition, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply subtract [-] to types " +
@@ -276,7 +276,7 @@ public final class EBinary extends AExpression {
         left.analyze(settings, definition, variables);
         right.analyze(settings, definition, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, false, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, false);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply left shift [<<] to types " +
@@ -309,7 +309,7 @@ public final class EBinary extends AExpression {
         left.analyze(settings, definition, variables);
         right.analyze(settings, definition, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, false, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, false);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply right shift [>>] to types " +
@@ -342,7 +342,7 @@ public final class EBinary extends AExpression {
         left.analyze(settings, definition, variables);
         right.analyze(settings, definition, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, false, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, false);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply unsigned shift [>>>] to types " +
@@ -375,7 +375,7 @@ public final class EBinary extends AExpression {
         left.analyze(settings, definition, variables);
         right.analyze(settings, definition, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, false, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, false);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply and [&] to types " +
@@ -441,7 +441,7 @@ public final class EBinary extends AExpression {
         left.analyze(settings, definition, variables);
         right.analyze(settings, definition, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, false, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, false);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply or [|] to types " +
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index 933295f9beb..0b91ee406e8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -159,21 +159,21 @@ public final class EChain extends AExpression {
         expression.analyze(settings, definition, variables);
 
         if (operation == Operation.MUL) {
-            promote = AnalyzerCaster.promoteNumeric(definition, last.after, expression.actual, true, true);
+            promote = AnalyzerCaster.promoteNumeric(definition, last.after, expression.actual, true);
         } else if (operation == Operation.DIV) {
-            promote = AnalyzerCaster.promoteNumeric(definition, last.after, expression.actual, true, true);
+            promote = AnalyzerCaster.promoteNumeric(definition, last.after, expression.actual, true);
         } else if (operation == Operation.REM) {
-            promote = AnalyzerCaster.promoteNumeric(definition, last.after, expression.actual, true, true);
+            promote = AnalyzerCaster.promoteNumeric(definition, last.after, expression.actual, true);
         } else if (operation == Operation.ADD) {
             promote = AnalyzerCaster.promoteAdd(definition, last.after, expression.actual);
         } else if (operation == Operation.SUB) {
-            promote = AnalyzerCaster.promoteNumeric(definition, last.after, expression.actual, true, true);
+            promote = AnalyzerCaster.promoteNumeric(definition, last.after, expression.actual, true);
         } else if (operation == Operation.LSH) {
-            promote = AnalyzerCaster.promoteNumeric(definition, last.after, false, true);
+            promote = AnalyzerCaster.promoteNumeric(definition, last.after, false);
         } else if (operation == Operation.RSH) {
-            promote = AnalyzerCaster.promoteNumeric(definition, last.after, false, true);
+            promote = AnalyzerCaster.promoteNumeric(definition, last.after, false);
         } else if (operation == Operation.USH) {
-            promote = AnalyzerCaster.promoteNumeric(definition, last.after, false, true);
+            promote = AnalyzerCaster.promoteNumeric(definition, last.after, false);
         } else if (operation == Operation.BWAND) {
             promote = AnalyzerCaster.promoteXor(definition, last.after, expression.actual);
         } else if (operation == Operation.XOR) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index a65e82e02fa..270f95e0a92 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -126,7 +126,7 @@ public final class EComp extends AExpression {
         left.analyze(settings, definition, variables);
         right.analyze(settings, definition, variables);
 
-        final Type promote = AnalyzerCaster.promoteReference(definition, left.actual, right.actual);
+        final Type promote = AnalyzerCaster.promoteEquality(definition, left.actual, right.actual);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply reference equals [===] to types " +
@@ -214,7 +214,7 @@ public final class EComp extends AExpression {
         left.analyze(settings, definition, variables);
         right.analyze(settings, definition, variables);
 
-        final Type promote = AnalyzerCaster.promoteReference(definition, left.actual, right.actual);
+        final Type promote = AnalyzerCaster.promoteEquality(definition, left.actual, right.actual);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply reference not equals [!==] to types " +
@@ -256,7 +256,7 @@ public final class EComp extends AExpression {
         left.analyze(settings, definition, variables);
         right.analyze(settings, definition, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply greater than or equals [>=] to types " +
@@ -292,7 +292,7 @@ public final class EComp extends AExpression {
         left.analyze(settings, definition, variables);
         right.analyze(settings, definition, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply greater than [>] to types " +
@@ -328,7 +328,7 @@ public final class EComp extends AExpression {
         left.analyze(settings, definition, variables);
         right.analyze(settings, definition, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply less than or equals [<=] to types " +
@@ -364,7 +364,7 @@ public final class EComp extends AExpression {
         left.analyze(settings, definition, variables);
         right.analyze(settings, definition, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply less than [>=] to types " +
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index 7e52790fa50..3e34c16ab97 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -77,7 +77,7 @@ public final class EUnary extends AExpression {
     void analyzeBWNot(final CompilerSettings settings, final Definition definition, final Variables variables) {
         child.analyze(settings, definition, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, child.actual, false, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, child.actual, false);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply not [~] to type [" + child.actual.name + "]."));
@@ -104,7 +104,7 @@ public final class EUnary extends AExpression {
     void analyzerAdd(final CompilerSettings settings, final Definition definition, final Variables variables) {
         child.analyze(settings, definition, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, child.actual, true, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, child.actual, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply positive [+] to type [" + child.actual.name + "]."));
@@ -135,7 +135,7 @@ public final class EUnary extends AExpression {
     void analyzerSub(final CompilerSettings settings, final Definition definition, final Variables variables) {
         child.analyze(settings, definition, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, child.actual, true, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(definition, child.actual, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply negative [-] to type [" + child.actual.name + "]."));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
index bfc0d142045..9ae657f7c1b 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
@@ -22,10 +22,10 @@ package org.elasticsearch.painless;
 public class DefOperationTests extends ScriptTestCase {
     public void testIllegalCast() {
         Exception exception = expectThrows(ClassCastException.class, () -> exec("def x = 1.0; int y = x; return y;"));
-        assertTrue(exception.getMessage().contains("java.lang.Double cannot be cast to java.lang.Integer"));
+        assertTrue(exception.getMessage().contains("java.lang.double cannot be cast to java.lang.int"));
 
         exception = expectThrows(ClassCastException.class, () -> exec("def x = (short)1; byte y = x; return y;"));
-        assertTrue(exception.getMessage().contains("java.lang.Short cannot be cast to java.lang.Byte"));
+        assertTrue(exception.getMessage().contains("java.lang.short cannot be cast to java.lang.byte"));
     }
 
     public void testNot() {
@@ -103,13 +103,13 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(4D, exec("def x = (float)2; def y = (double)2; return x * y"));
         assertEquals(4D, exec("def x = (double)2; def y = (double)2; return x * y"));
 
-        assertEquals(4, exec("def x = (Byte)2; def y = (byte)2; return x * y"));
-        assertEquals(4, exec("def x = (Short)2; def y = (short)2; return x * y"));
-        assertEquals(4, exec("def x = (Character)2; def y = (char)2; return x * y"));
-        assertEquals(4, exec("def x = (Integer)2; def y = (int)2; return x * y"));
-        assertEquals(4L, exec("def x = (Long)2; def y = (long)2; return x * y"));
-        assertEquals(4F, exec("def x = (Float)2; def y = (float)2; return x * y"));
-        assertEquals(4D, exec("def x = (Double)2; def y = (double)2; return x * y"));
+        assertEquals(4, exec("def x = (byte)2; def y = (byte)2; return x * y"));
+        assertEquals(4, exec("def x = (short)2; def y = (short)2; return x * y"));
+        assertEquals(4, exec("def x = (char)2; def y = (char)2; return x * y"));
+        assertEquals(4, exec("def x = (int)2; def y = (int)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; def y = (long)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; def y = (float)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; def y = (double)2; return x * y"));
     }
 
     public void testDiv() {
@@ -169,13 +169,13 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(1D, exec("def x = (float)2; def y = (double)2; return x / y"));
         assertEquals(1D, exec("def x = (double)2; def y = (double)2; return x / y"));
 
-        assertEquals(1, exec("def x = (Byte)2; def y = (byte)2; return x / y"));
-        assertEquals(1, exec("def x = (Short)2; def y = (short)2; return x / y"));
-        assertEquals(1, exec("def x = (Character)2; def y = (char)2; return x / y"));
-        assertEquals(1, exec("def x = (Integer)2; def y = (int)2; return x / y"));
-        assertEquals(1L, exec("def x = (Long)2; def y = (long)2; return x / y"));
-        assertEquals(1F, exec("def x = (Float)2; def y = (float)2; return x / y"));
-        assertEquals(1D, exec("def x = (Double)2; def y = (double)2; return x / y"));
+        assertEquals(1, exec("def x = (byte)2; def y = (byte)2; return x / y"));
+        assertEquals(1, exec("def x = (short)2; def y = (short)2; return x / y"));
+        assertEquals(1, exec("def x = (char)2; def y = (char)2; return x / y"));
+        assertEquals(1, exec("def x = (int)2; def y = (int)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; def y = (long)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; def y = (float)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; def y = (double)2; return x / y"));
     }
 
     public void testRem() {
@@ -235,13 +235,13 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(0D, exec("def x = (float)2; def y = (double)2; return x % y"));
         assertEquals(0D, exec("def x = (double)2; def y = (double)2; return x % y"));
 
-        assertEquals(0, exec("def x = (Byte)2; def y = (byte)2; return x % y"));
-        assertEquals(0, exec("def x = (Short)2; def y = (short)2; return x % y"));
-        assertEquals(0, exec("def x = (Character)2; def y = (char)2; return x % y"));
-        assertEquals(0, exec("def x = (Integer)2; def y = (int)2; return x % y"));
-        assertEquals(0L, exec("def x = (Long)2; def y = (long)2; return x % y"));
-        assertEquals(0F, exec("def x = (Float)2; def y = (float)2; return x % y"));
-        assertEquals(0D, exec("def x = (Double)2; def y = (double)2; return x % y"));
+        assertEquals(0, exec("def x = (byte)2; def y = (byte)2; return x % y"));
+        assertEquals(0, exec("def x = (short)2; def y = (short)2; return x % y"));
+        assertEquals(0, exec("def x = (char)2; def y = (char)2; return x % y"));
+        assertEquals(0, exec("def x = (int)2; def y = (int)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; def y = (long)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; def y = (float)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; def y = (double)2; return x % y"));
     }
 
     public void testAdd() {
@@ -301,13 +301,13 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(2D, exec("def x = (float)1; def y = (double)1; return x + y"));
         assertEquals(2D, exec("def x = (double)1; def y = (double)1; return x + y"));
 
-        assertEquals(2, exec("def x = (Byte)1; def y = (byte)1; return x + y"));
-        assertEquals(2, exec("def x = (Short)1; def y = (short)1; return x + y"));
-        assertEquals(2, exec("def x = (Character)1; def y = (char)1; return x + y"));
-        assertEquals(2, exec("def x = (Integer)1; def y = (int)1; return x + y"));
-        assertEquals(2L, exec("def x = (Long)1; def y = (long)1; return x + y"));
-        assertEquals(2F, exec("def x = (Float)1; def y = (float)1; return x + y"));
-        assertEquals(2D, exec("def x = (Double)1; def y = (double)1; return x + y"));
+        assertEquals(2, exec("def x = (byte)1; def y = (byte)1; return x + y"));
+        assertEquals(2, exec("def x = (short)1; def y = (short)1; return x + y"));
+        assertEquals(2, exec("def x = (char)1; def y = (char)1; return x + y"));
+        assertEquals(2, exec("def x = (int)1; def y = (int)1; return x + y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (long)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; def y = (float)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; def y = (double)1; return x + y"));
     }
 
     public void testSub() {
@@ -367,13 +367,13 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(0D, exec("def x = (float)1; def y = (double)1; return x - y"));
         assertEquals(0D, exec("def x = (double)1; def y = (double)1; return x - y"));
 
-        assertEquals(0, exec("def x = (Byte)1; def y = (byte)1; return x - y"));
-        assertEquals(0, exec("def x = (Short)1; def y = (short)1; return x - y"));
-        assertEquals(0, exec("def x = (Character)1; def y = (char)1; return x - y"));
-        assertEquals(0, exec("def x = (Integer)1; def y = (int)1; return x - y"));
-        assertEquals(0L, exec("def x = (Long)1; def y = (long)1; return x - y"));
-        assertEquals(0F, exec("def x = (Float)1; def y = (float)1; return x - y"));
-        assertEquals(0D, exec("def x = (Double)1; def y = (double)1; return x - y"));
+        assertEquals(0, exec("def x = (byte)1; def y = (byte)1; return x - y"));
+        assertEquals(0, exec("def x = (short)1; def y = (short)1; return x - y"));
+        assertEquals(0, exec("def x = (char)1; def y = (char)1; return x - y"));
+        assertEquals(0, exec("def x = (int)1; def y = (int)1; return x - y"));
+        assertEquals(0L, exec("def x = (long)1; def y = (long)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; def y = (float)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; def y = (double)1; return x - y"));
     }
 
     public void testLsh() {
@@ -433,13 +433,13 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(2L, exec("def x = (float)1; def y = (double)1; return x << y"));
         assertEquals(2L, exec("def x = (double)1; def y = (double)1; return x << y"));
 
-        assertEquals(2, exec("def x = (Byte)1; def y = (byte)1; return x << y"));
-        assertEquals(2, exec("def x = (Short)1; def y = (short)1; return x << y"));
-        assertEquals(2, exec("def x = (Character)1; def y = (char)1; return x << y"));
-        assertEquals(2, exec("def x = (Integer)1; def y = (int)1; return x << y"));
-        assertEquals(2L, exec("def x = (Long)1; def y = (long)1; return x << y"));
-        assertEquals(2L, exec("def x = (Float)1; def y = (float)1; return x << y"));
-        assertEquals(2L, exec("def x = (Double)1; def y = (double)1; return x << y"));
+        assertEquals(2, exec("def x = (byte)1; def y = (byte)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; def y = (short)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; def y = (char)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; def y = (int)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (long)1; return x << y"));
+        assertEquals(2L, exec("def x = (float)1; def y = (float)1; return x << y"));
+        assertEquals(2L, exec("def x = (double)1; def y = (double)1; return x << y"));
     }
 
     public void testRsh() {
@@ -499,13 +499,13 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(2L, exec("def x = (float)4; def y = (double)1; return x >> y"));
         assertEquals(2L, exec("def x = (double)4; def y = (double)1; return x >> y"));
 
-        assertEquals(2, exec("def x = (Byte)4; def y = (byte)1; return x >> y"));
-        assertEquals(2, exec("def x = (Short)4; def y = (short)1; return x >> y"));
-        assertEquals(2, exec("def x = (Character)4; def y = (char)1; return x >> y"));
-        assertEquals(2, exec("def x = (Integer)4; def y = (int)1; return x >> y"));
-        assertEquals(2L, exec("def x = (Long)4; def y = (long)1; return x >> y"));
-        assertEquals(2L, exec("def x = (Float)4; def y = (float)1; return x >> y"));
-        assertEquals(2L, exec("def x = (Double)4; def y = (double)1; return x >> y"));
+        assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >> y"));
+        assertEquals(2L, exec("def x = (float)4; def y = (float)1; return x >> y"));
+        assertEquals(2L, exec("def x = (double)4; def y = (double)1; return x >> y"));
     }
 
     public void testUsh() {
@@ -565,13 +565,13 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(2L, exec("def x = (float)4; def y = (double)1; return x >>> y"));
         assertEquals(2L, exec("def x = (double)4; def y = (double)1; return x >>> y"));
 
-        assertEquals(2, exec("def x = (Byte)4; def y = (byte)1; return x >>> y"));
-        assertEquals(2, exec("def x = (Short)4; def y = (short)1; return x >>> y"));
-        assertEquals(2, exec("def x = (Character)4; def y = (char)1; return x >>> y"));
-        assertEquals(2, exec("def x = (Integer)4; def y = (int)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (Long)4; def y = (long)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (Float)4; def y = (float)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (Double)4; def y = (double)1; return x >>> y"));
+        assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (float)4; def y = (float)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (double)4; def y = (double)1; return x >>> y"));
     }
 
     public void testAnd() {
@@ -631,13 +631,13 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(0L, exec("def x = (float)4; def y = (double)1; return x & y"));
         assertEquals(0L, exec("def x = (double)4; def y = (double)1; return x & y"));
 
-        assertEquals(0, exec("def x = (Byte)4; def y = (byte)1; return x & y"));
-        assertEquals(0, exec("def x = (Short)4; def y = (short)1; return x & y"));
-        assertEquals(0, exec("def x = (Character)4; def y = (char)1; return x & y"));
-        assertEquals(0, exec("def x = (Integer)4; def y = (int)1; return x & y"));
-        assertEquals(0L, exec("def x = (Long)4; def y = (long)1; return x & y"));
-        assertEquals(0L, exec("def x = (Float)4; def y = (float)1; return x & y"));
-        assertEquals(0L, exec("def x = (Double)4; def y = (double)1; return x & y"));
+        assertEquals(0, exec("def x = (byte)4; def y = (byte)1; return x & y"));
+        assertEquals(0, exec("def x = (short)4; def y = (short)1; return x & y"));
+        assertEquals(0, exec("def x = (char)4; def y = (char)1; return x & y"));
+        assertEquals(0, exec("def x = (int)4; def y = (int)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; def y = (long)1; return x & y"));
+        assertEquals(0L, exec("def x = (float)4; def y = (float)1; return x & y"));
+        assertEquals(0L, exec("def x = (double)4; def y = (double)1; return x & y"));
     }
 
     public void testXor() {
@@ -697,13 +697,13 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(5L, exec("def x = (float)4; def y = (double)1; return x ^ y"));
         assertEquals(5L, exec("def x = (double)4; def y = (double)1; return x ^ y"));
 
-        assertEquals(5, exec("def x = (Byte)4; def y = (byte)1; return x ^ y"));
-        assertEquals(5, exec("def x = (Short)4; def y = (short)1; return x ^ y"));
-        assertEquals(5, exec("def x = (Character)4; def y = (char)1; return x ^ y"));
-        assertEquals(5, exec("def x = (Integer)4; def y = (int)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (Long)4; def y = (long)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (Float)4; def y = (float)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (Double)4; def y = (double)1; return x ^ y"));
+        assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; def y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; def y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; def y = (int)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (float)4; def y = (float)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (double)4; def y = (double)1; return x ^ y"));
     }
 
     public void testOr() {
@@ -763,13 +763,13 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(5L, exec("def x = (float)4; def y = (double)1; return x | y"));
         assertEquals(5L, exec("def x = (double)4; def y = (double)1; return x | y"));
 
-        assertEquals(5, exec("def x = (Byte)4; def y = (byte)1; return x | y"));
-        assertEquals(5, exec("def x = (Short)4; def y = (short)1; return x | y"));
-        assertEquals(5, exec("def x = (Character)4; def y = (char)1; return x | y"));
-        assertEquals(5, exec("def x = (Integer)4; def y = (int)1; return x | y"));
-        assertEquals(5L, exec("def x = (Long)4; def y = (long)1; return x | y"));
-        assertEquals(5L, exec("def x = (Float)4; def y = (float)1; return x | y"));
-        assertEquals(5L, exec("def x = (Double)4; def y = (double)1; return x | y"));
+        assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x | y"));
+        assertEquals(5, exec("def x = (short)4; def y = (short)1; return x | y"));
+        assertEquals(5, exec("def x = (char)4; def y = (char)1; return x | y"));
+        assertEquals(5, exec("def x = (int)4; def y = (int)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x | y"));
+        assertEquals(5L, exec("def x = (float)4; def y = (float)1; return x | y"));
+        assertEquals(5L, exec("def x = (double)4; def y = (double)1; return x | y"));
     }
 
     public void testEq() {

From 64e2ef58078eb9066eff76656cffaa2e0db91b44 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Thu, 19 May 2016 23:38:14 -0700
Subject: [PATCH 0365/1311] Removed transforms.  No user-facing boxing.

---
 .../painless/AnalyzerCaster.java              | 711 ++++++++++++++++--
 .../java/org/elasticsearch/painless/Def.java  | 147 +---
 .../elasticsearch/painless/Definition.java    | 311 +-------
 .../elasticsearch/painless/MethodWriter.java  | 139 ++--
 .../org/elasticsearch/painless/Utility.java   |  12 -
 .../painless/WriterConstants.java             |  53 +-
 .../painless/node/AExpression.java            |   8 +-
 .../elasticsearch/painless/node/EBinary.java  |   2 +-
 .../elasticsearch/painless/node/EChain.java   |   6 +-
 .../painless/node/EConditional.java           |   2 +
 .../elasticsearch/painless/node/LCall.java    |   3 +-
 .../elasticsearch/painless/node/LCast.java    |   2 +-
 .../elasticsearch/painless/node/LDefCall.java |   1 +
 .../elasticsearch/painless/node/LNewObj.java  |   1 +
 .../painless/node/SExpression.java            |   1 +
 .../elasticsearch/painless/node/SReturn.java  |   1 +
 .../painless/node/package-info.java           |   1 +
 .../org/elasticsearch/painless/definition.txt | 154 ----
 .../painless/BasicExpressionTests.java        |   6 +-
 .../painless/CompoundAssignmentTests.java     |  48 +-
 .../painless/ConditionalTests.java            |   3 -
 .../elasticsearch/painless/EqualsTests.java   |  12 +-
 .../elasticsearch/painless/StringTests.java   |  44 +-
 23 files changed, 852 insertions(+), 816 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
index 398dc0a7a61..11a334eff1d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
@@ -20,98 +20,677 @@
 package org.elasticsearch.painless;
 
 import org.elasticsearch.painless.Definition.Cast;
-import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
-import org.elasticsearch.painless.Definition.Transform;
 import org.elasticsearch.painless.Definition.Type;
 
-import java.lang.reflect.InvocationTargetException;
-
 /**
  * Used during the analysis phase to collect legal type casts and promotions
  * for type-checking and later to write necessary casts in the bytecode.
  */
 public final class AnalyzerCaster {
 
-    public static Cast getLegalCast(final Definition definition,
-                                    final String location, final Type actual, final Type expected, final boolean explicit) {
-        final Cast cast = new Cast(actual, expected, explicit);
-
+    public static Cast getLegalCast(final Definition definition, final String location, final Type actual,
+                                    final Type expected, final boolean explicit, final boolean internal) {
         if (actual.equals(expected)) {
             return null;
         }
 
-        Cast transform = definition.transformsMap.get(cast);
+        switch (actual.sort) {
+            case BOOL:
+                switch (expected.sort) {
+                    case DEF:
+                        return new Cast(actual, definition.getType("def"), explicit, false, false, true, false);
+                    case OBJECT:
+                    case BOOL_OBJ:
+                        if (internal)
+                            return new Cast(actual, actual, explicit, false, false, false, true);
+                }
 
-        if (transform == null && explicit) {
-            transform = definition.transformsMap.get(new Cast(actual, expected, false));
-        }
+                break;
+            case BYTE:
+                switch (expected.sort) {
+                    case SHORT:
+                    case INT:
+                    case LONG:
+                    case FLOAT:
+                    case DOUBLE:
+                        return new Cast(actual, expected, explicit);
+                    case CHAR:
+                        if (explicit)
+                            return new Cast(actual, expected, true);
 
-        if (transform != null) {
-            return transform;
+                        break;
+                    case DEF:
+                        return new Cast(actual, definition.getType("def"), explicit, false, false, true, false);
+                    case OBJECT:
+                    case NUMBER:
+                    case BYTE_OBJ:
+                        if (internal)
+                            return new Cast(actual, actual, explicit, false, false, false, true);
+
+                        break;
+                    case SHORT_OBJ:
+                        if (internal)
+                            return new Cast(actual, definition.getType("short"), explicit, false, false, false, true);
+
+                        break;
+                    case INT_OBJ:
+                        if (internal)
+                            return new Cast(actual, definition.getType("int"), explicit, false, false, false, true);
+
+                        break;
+                    case LONG_OBJ:
+                        if (internal)
+                            return new Cast(actual, definition.getType("long"), explicit, false, false, false, true);
+
+                        break;
+                    case FLOAT_OBJ:
+                        if (internal)
+                            return new Cast(actual, definition.getType("float"), explicit, false, false, false, true);
+
+                        break;
+                    case DOUBLE_OBJ:
+                        if (internal)
+                            return new Cast(actual, definition.getType("double"), explicit, false, false, false, true);
+
+                        break;
+                    case CHAR_OBJ:
+                        if (explicit && internal)
+                            return new Cast(actual, definition.getType("char"), explicit, false, false, false, true);
+
+                        break;
+                }
+
+                break;
+            case SHORT:
+                switch (expected.sort) {
+                    case INT:
+                    case LONG:
+                    case FLOAT:
+                    case DOUBLE:
+                        return new Cast(actual, expected, explicit);
+                    case BYTE:
+                    case CHAR:
+                        if (explicit)
+                            return new Cast(actual, expected, true);
+
+                        break;
+                    case DEF:
+                        return new Cast(actual, definition.getType("def"), explicit, false, false, true, false);
+                    case OBJECT:
+                    case NUMBER:
+                    case SHORT_OBJ:
+                        if (internal)
+                            return new Cast(actual, actual, explicit, false, false, false, true);
+
+                        break;
+                    case INT_OBJ:
+                        if (internal)
+                            return new Cast(actual, definition.getType("int"), explicit, false, false, false, true);
+
+                        break;
+                    case LONG_OBJ:
+                        if (internal)
+                            return new Cast(actual, definition.getType("long"), explicit, false, false, false, true);
+
+                        break;
+                    case FLOAT_OBJ:
+                        if (internal)
+                            return new Cast(actual, definition.getType("float"), explicit, false, false, false, true);
+
+                        break;
+                    case DOUBLE_OBJ:
+                        if (internal)
+                            return new Cast(actual, definition.getType("double"), explicit, false, false, false, true);
+
+                        break;
+                    case BYTE_OBJ:
+                        if (explicit && internal)
+                            return new Cast(actual, definition.getType("byte"), true, false, false, false, true);
+
+                        break;
+                    case CHAR_OBJ:
+                        if (explicit && internal)
+                            return new Cast(actual, definition.getType("char"), true, false, false, false, true);
+
+                        break;
+                }
+
+                break;
+            case CHAR:
+                switch (expected.sort) {
+                    case INT:
+                    case LONG:
+                    case FLOAT:
+                    case DOUBLE:
+                        return new Cast(actual, expected, explicit);
+                    case BYTE:
+                    case SHORT:
+                        if (explicit)
+                            return new Cast(actual, expected, true);
+
+                        break;
+                    case DEF:
+                        return new Cast(actual, definition.getType("def"), explicit, false, false, true, false);
+                    case OBJECT:
+                    case NUMBER:
+                    case CHAR_OBJ:
+                        if (internal)
+                            return new Cast(actual, actual, explicit, false, false, false, true);
+
+                        break;
+                    case STRING:
+                        return new Cast(actual, definition.getType("String"), explicit, false, false, false, false);
+                    case INT_OBJ:
+                        if (internal)
+                            return new Cast(actual, definition.getType("int"), explicit, false, false, false, true);
+
+                        break;
+                    case LONG_OBJ:
+                        if (internal)
+                            return new Cast(actual, definition.getType("long"), explicit, false, false, false, true);
+
+                        break;
+                    case FLOAT_OBJ:
+                        if (internal)
+                            return new Cast(actual, definition.getType("float"), explicit, false, false, false, true);
+
+                        break;
+                    case DOUBLE_OBJ:
+                        if (internal)
+                            return new Cast(actual, definition.getType("double"), explicit, false, false, false, true);
+
+                        break;
+                    case BYTE_OBJ:
+                        if (explicit && internal)
+                            return new Cast(actual, definition.getType("byte"), true, false, false, false, true);
+
+                        break;
+                    case SHORT_OBJ:
+                        if (explicit && internal)
+                            return new Cast(actual, definition.getType("short"), true, false, false, false, true);
+
+                        break;
+                }
+
+                break;
+            case INT:
+                switch (expected.sort) {
+                    case LONG:
+                    case FLOAT:
+                    case DOUBLE:
+                        return new Cast(actual, expected, explicit);
+                    case BYTE:
+                    case SHORT:
+                    case CHAR:
+                        if (explicit)
+                            return new Cast(actual, expected, true);
+
+                        break;
+                    case DEF:
+                        return new Cast(actual, definition.getType("def"), explicit, false, false, true, false);
+                    case OBJECT:
+                    case NUMBER:
+                    case INT_OBJ:
+                        if (internal)
+                            return new Cast(actual, actual, explicit, false, false, false, true);
+
+                        break;
+                    case LONG_OBJ:
+                        if (internal)
+                            return new Cast(actual, definition.getType("long"), explicit, false, false, false, true);
+
+                        break;
+                    case FLOAT_OBJ:
+                        if (internal)
+                            return new Cast(actual, definition.getType("float"), explicit, false, false, false, true);
+
+                        break;
+                    case DOUBLE_OBJ:
+                        if (internal)
+                            return new Cast(actual, definition.getType("double"), explicit, false, false, false, true);
+
+                        break;
+                    case BYTE_OBJ:
+                        if (explicit && internal)
+                            return new Cast(actual, definition.getType("byte"), true, false, false, false, true);
+
+                        break;
+                    case SHORT_OBJ:
+                        if (explicit && internal)
+                            return new Cast(actual, definition.getType("short"), true, false, false, false, true);
+
+                        break;
+                    case CHAR_OBJ:
+                        if (explicit && internal)
+                            return new Cast(actual, definition.getType("char"), true, false, false, false, true);
+
+                        break;
+                }
+
+                break;
+            case LONG:
+                switch (expected.sort) {
+                    case FLOAT:
+                    case DOUBLE:
+                        return new Cast(actual, expected, explicit);
+                    case BYTE:
+                    case SHORT:
+                    case CHAR:
+                    case INT:
+                        if (explicit)
+                            return new Cast(actual, expected, true);
+
+                        break;
+                    case DEF:
+                        return new Cast(actual, definition.getType("def"), explicit, false, false, true, false);
+                    case OBJECT:
+                    case NUMBER:
+                    case LONG_OBJ:
+                        if (internal)
+                            return new Cast(actual, actual, explicit, false, false, false, true);
+
+                        break;
+                    case FLOAT_OBJ:
+                        if (internal)
+                            return new Cast(actual, definition.getType("float"), explicit, false, false, false, true);
+
+                        break;
+                    case DOUBLE_OBJ:
+                        if (internal)
+                            return new Cast(actual, definition.getType("double"), explicit, false, false, false, true);
+
+                        break;
+                    case BYTE_OBJ:
+                        if (explicit && internal)
+                            return new Cast(actual, definition.getType("byte"), true, false, false, false, true);
+
+                        break;
+                    case SHORT_OBJ:
+                        if (explicit && internal)
+                            return new Cast(actual, definition.getType("short"), true, false, false, false, true);
+
+                        break;
+                    case CHAR_OBJ:
+                        if (explicit && internal)
+                            return new Cast(actual, definition.getType("char"), true, false, false, false, true);
+
+                        break;
+                    case INT_OBJ:
+                        if (explicit && internal)
+                            return new Cast(actual, definition.getType("int"), true, false, false, false, true);
+
+                        break;
+                }
+
+                break;
+            case FLOAT:
+                switch (expected.sort) {
+                    case DOUBLE:
+                        return new Cast(actual, expected, explicit);
+                    case BYTE:
+                    case SHORT:
+                    case CHAR:
+                    case INT:
+                    case FLOAT:
+                        if (explicit)
+                            return new Cast(actual, expected, true);
+
+                        break;
+                    case DEF:
+                        return new Cast(actual, definition.getType("def"), explicit, false, false, true, false);
+                    case OBJECT:
+                    case NUMBER:
+                    case FLOAT_OBJ:
+                        if (internal)
+                            return new Cast(actual, actual, explicit, false, false, false, true);
+
+                        break;
+                    case DOUBLE_OBJ:
+                        if (internal)
+                            return new Cast(actual, definition.getType("double"), explicit, false, false, false, true);
+
+                        break;
+                    case BYTE_OBJ:
+                        if (explicit && internal)
+                            return new Cast(actual, definition.getType("byte"), true, false, false, false, true);
+
+                        break;
+                    case SHORT_OBJ:
+                        if (explicit && internal)
+                            return new Cast(actual, definition.getType("short"), true, false, false, false, true);
+
+                        break;
+                    case CHAR_OBJ:
+                        if (explicit && internal)
+                            return new Cast(actual, definition.getType("char"), true, false, false, false, true);
+
+                        break;
+                    case INT_OBJ:
+                        if (explicit && internal)
+                            return new Cast(actual, definition.getType("int"), true, false, false, false, true);
+
+                        break;
+                    case LONG_OBJ:
+                        if (explicit && internal)
+                            return new Cast(actual, definition.getType("long"), true, false, false, false, true);
+
+                        break;
+                }
+
+                break;
+            case DOUBLE:
+                switch (expected.sort) {
+                    case BYTE:
+                    case SHORT:
+                    case CHAR:
+                    case INT:
+                    case FLOAT:
+                        if (explicit)
+                            return new Cast(actual, expected, true);
+
+                        break;
+                    case DEF:
+                        return new Cast(actual, definition.getType("def"), explicit, false, false, true, false);
+                    case OBJECT:
+                    case NUMBER:
+                    case DOUBLE_OBJ:
+                        if (internal)
+                            return new Cast(actual, actual, explicit, false, false, false, true);
+
+                        break;
+                    case BYTE_OBJ:
+                        if (explicit && internal)
+                            return new Cast(actual, definition.getType("byte"), true, false, false, false, true);
+
+                        break;
+                    case SHORT_OBJ:
+                        if (explicit && internal)
+                            return new Cast(actual, definition.getType("short"), true, false, false, false, true);
+
+                        break;
+                    case CHAR_OBJ:
+                        if (explicit && internal)
+                            return new Cast(actual, definition.getType("char"), true, false, false, false, true);
+
+                        break;
+                    case INT_OBJ:
+                        if (explicit && internal)
+                            return new Cast(actual, definition.getType("int"), true, false, false, false, true);
+
+                        break;
+                    case LONG_OBJ:
+                        if (explicit && internal)
+                            return new Cast(actual, definition.getType("long"), true, false, false, false, true);
+
+                        break;
+                    case FLOAT_OBJ:
+                        if (explicit && internal)
+                            return new Cast(actual, definition.getType("float"), true, false, false, false, true);
+
+                        break;
+                }
+
+                break;
+            case OBJECT:
+            case NUMBER:
+                switch (expected.sort) {
+                    case BYTE:
+                        if (internal && explicit)
+                            return new Cast(actual, definition.getType("Byte"), true, false, true, false, false);
+
+                        break;
+                    case SHORT:
+                        if (internal && explicit)
+                            return new Cast(actual, definition.getType("Short"), true, false, true, false, false);
+
+                        break;
+                    case CHAR:
+                        if (internal && explicit)
+                            return new Cast(actual, definition.getType("Character"), true, false, true, false, false);
+
+                        break;
+                    case INT:
+                        if (internal && explicit)
+                            return new Cast(actual, definition.getType("Integer"), true, false, true, false, false);
+
+                        break;
+                    case LONG:
+                        if (internal && explicit)
+                            return new Cast(actual, definition.getType("Long"), true, false, true, false, false);
+
+                        break;
+                    case FLOAT:
+                        if (internal && explicit)
+                            return new Cast(actual, definition.getType("Float"), true, false, true, false, false);
+
+                        break;
+                    case DOUBLE:
+                        if (internal && explicit)
+                            return new Cast(actual, definition.getType("Double"), true, false, true, false, false);
+
+                        break;
+                }
+
+                break;
+            case BOOL_OBJ:
+                switch (expected.sort) {
+                    case BOOL:
+                        if (internal)
+                            return new Cast(actual, expected, explicit, true, false, false, false);
+
+                        break;
+                }
+
+                break;
+            case BYTE_OBJ:
+                switch (expected.sort) {
+                    case BYTE:
+                    case SHORT:
+                    case INT:
+                    case LONG:
+                    case FLOAT:
+                    case DOUBLE:
+                        if (internal)
+                            return new Cast(actual, expected, explicit, true, false, false, false);
+
+                        break;
+                    case CHAR:
+                        if (internal && explicit)
+                            return new Cast(actual, expected, true, true, false, false, false);
+
+                        break;
+                }
+
+                break;
+            case SHORT_OBJ:
+                switch (expected.sort) {
+                    case SHORT:
+                    case INT:
+                    case LONG:
+                    case FLOAT:
+                    case DOUBLE:
+                        if (internal)
+                            return new Cast(actual, expected, explicit, true, false, false, false);
+
+                        break;
+                    case BYTE:
+                    case CHAR:
+                        if (internal && explicit)
+                            return new Cast(actual, expected, true, true, false, false, false);
+
+                        break;
+                }
+
+                break;
+            case CHAR_OBJ:
+                switch (expected.sort) {
+                    case CHAR:
+                    case INT:
+                    case LONG:
+                    case FLOAT:
+                    case DOUBLE:
+                        if (internal)
+                            return new Cast(actual, expected, explicit, true, false, false, false);
+
+                        break;
+                    case BYTE:
+                    case SHORT:
+                        if (internal && explicit)
+                            return new Cast(actual, expected, true, true, false, false, false);
+
+                        break;
+                }
+
+                break;
+            case INT_OBJ:
+                switch (expected.sort) {
+                    case INT:
+                    case LONG:
+                    case FLOAT:
+                    case DOUBLE:
+                        if (internal)
+                            return new Cast(actual, expected, explicit, true, false, false, false);
+
+                        break;
+                    case BYTE:
+                    case SHORT:
+                    case CHAR:
+                        if (internal && explicit)
+                            return new Cast(actual, expected, true, true, false, false, false);
+
+                        break;
+                }
+
+                break;
+            case LONG_OBJ:
+                switch (expected.sort) {
+                    case LONG:
+                    case FLOAT:
+                    case DOUBLE:
+                        if (internal)
+                            return new Cast(actual, expected, explicit, true, false, false, false);
+
+                        break;
+                    case BYTE:
+                    case SHORT:
+                    case CHAR:
+                    case INT:
+                        if (internal && explicit)
+                            return new Cast(actual, expected, true, true, false, false, false);
+
+                        break;
+                }
+
+                break;
+            case FLOAT_OBJ:
+                switch (expected.sort) {
+                    case FLOAT:
+                    case DOUBLE:
+                        if (internal)
+                            return new Cast(actual, expected, explicit, true, false, false, false);
+
+                        break;
+                    case BYTE:
+                    case SHORT:
+                    case CHAR:
+                    case INT:
+                    case LONG:
+                        if (internal && explicit)
+                            return new Cast(actual, expected, true, true, false, false, false);
+
+                        break;
+                }
+
+                break;
+            case DOUBLE_OBJ:
+                switch (expected.sort) {
+                    case FLOAT:
+                    case DOUBLE:
+                        if (internal)
+                            return new Cast(actual, expected, explicit, true, false, false, false);
+
+                        break;
+                    case BYTE:
+                    case SHORT:
+                    case CHAR:
+                    case INT:
+                    case LONG:
+                        if (internal && explicit)
+                            return new Cast(actual, expected, true, true, false, false, false);
+
+                        break;
+                }
+
+                break;
+            case DEF:
+                switch (expected.sort) {
+                    case BOOL:
+                    case BYTE:
+                    case SHORT:
+                    case CHAR:
+                    case INT:
+                    case LONG:
+                    case FLOAT:
+                    case DOUBLE:
+                            return new Cast(actual, expected, true, true, false, false, false);
+                }
+
+                break;
+            case STRING:
+                switch (expected.sort) {
+                    case CHAR:
+                        if (explicit)
+                            return new Cast(actual, expected, true, false, false, false, false);
+
+                        break;
+                }
+
+                break;
         }
 
         if (expected.clazz.isAssignableFrom(actual.clazz) ||
             ((explicit || expected.sort == Sort.DEF) && actual.clazz.isAssignableFrom(expected.clazz))) {
-            return cast;
+            return new Cast(actual, expected, explicit);
         } else {
             throw new ClassCastException("Error" + location + ": Cannot cast from [" + actual.name + "] to [" + expected.name + "].");
         }
     }
 
     public static Object constCast(final String location, final Object constant, final Cast cast) {
-        if (cast instanceof Transform) {
-            final Transform transform = (Transform)cast;
-            return invokeTransform(location, transform, constant);
+        final Sort fsort = cast.from.sort;
+        final Sort tsort = cast.to.sort;
+
+        if (fsort == tsort) {
+            return constant;
+        } else if (fsort == Sort.STRING && tsort == Sort.CHAR) {
+            return Utility.StringTochar((String)constant);
+        } else if (fsort == Sort.CHAR && tsort == Sort.STRING) {
+            return Utility.charToString((char)constant);
+        } else if (fsort.numeric && tsort.numeric) {
+            final Number number;
+
+            if (fsort == Sort.CHAR) {
+                number = (int)(char)constant;
+            } else {
+                number = (Number)constant;
+            }
+
+            switch (tsort) {
+                case BYTE:   return number.byteValue();
+                case SHORT:  return number.shortValue();
+                case CHAR:   return (char)number.intValue();
+                case INT:    return number.intValue();
+                case LONG:   return number.longValue();
+                case FLOAT:  return number.floatValue();
+                case DOUBLE: return number.doubleValue();
+                default:
+                    throw new IllegalStateException("Error" + location + ": Cannot cast from " +
+                        "[" + cast.from.clazz.getCanonicalName() + "] to [" + cast.to.clazz.getCanonicalName() + "].");
+            }
         } else {
-            final Sort fsort = cast.from.sort;
-            final Sort tsort = cast.to.sort;
-
-            if (fsort == tsort) {
-                return constant;
-            } else if (fsort.numeric && tsort.numeric) {
-                Number number;
-
-                if (fsort == Sort.CHAR) {
-                    number = (int)(char)constant;
-                } else {
-                    number = (Number)constant;
-                }
-
-                switch (tsort) {
-                    case BYTE:   return number.byteValue();
-                    case SHORT:  return number.shortValue();
-                    case CHAR:   return (char)number.intValue();
-                    case INT:    return number.intValue();
-                    case LONG:   return number.longValue();
-                    case FLOAT:  return number.floatValue();
-                    case DOUBLE: return number.doubleValue();
-                    default:
-                        throw new IllegalStateException("Error" + location + ": Cannot cast from " +
-                            "[" + cast.from.clazz.getCanonicalName() + "] to [" + cast.to.clazz.getCanonicalName() + "].");
-                }
-            } else {
-                throw new IllegalStateException("Error" + location + ": Cannot cast from " +
-                    "[" + cast.from.clazz.getCanonicalName() + "] to [" + cast.to.clazz.getCanonicalName() + "].");
-            }
-        }
-    }
-
-    private static Object invokeTransform(final String location, final Transform transform, final Object object) {
-        final Method method = transform.method;
-        final java.lang.reflect.Method jmethod = method.reflect;
-        final int modifiers = jmethod.getModifiers();
-
-        try {
-            if (java.lang.reflect.Modifier.isStatic(modifiers)) {
-                return jmethod.invoke(null, object);
-            } else {
-                return jmethod.invoke(object);
-            }
-        } catch (final IllegalAccessException | IllegalArgumentException |
-            InvocationTargetException | NullPointerException | ExceptionInInitializerError exception) {
-            throw new ClassCastException(
-                "Error" + location + ": Cannot cast from [" + transform.from.name + "] to [" + transform.to.name + "].");
+            throw new IllegalStateException("Error" + location + ": Cannot cast from " +
+                "[" + cast.from.clazz.getCanonicalName() + "] to [" + cast.to.clazz.getCanonicalName() + "].");
         }
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 889328e58db..1d804a264ae 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -972,6 +972,10 @@ public final class Def {
 
     // Conversion methods for Def to primitive types.
 
+    public static boolean DefToboolean(final Object value) {
+        return (boolean)value;
+    }
+
     public static byte DefTobyteImplicit(final Object value) {
         return (byte)value;
     }
@@ -1052,79 +1056,6 @@ public final class Def {
         }
     }
 
-    public static Byte DefToByteImplicit(final Object value) {
-        return (Byte)value;
-    }
-
-    public static Short DefToShortImplicit(final Object value) {
-        if (value == null) {
-            return null;
-        } else if (value instanceof Byte) {
-            return ((Byte)value).shortValue();
-        } else {
-            return (Short)value;
-        }
-    }
-
-    public static Character DefToCharacterImplicit(final Object value) {
-        if (value == null) {
-            return null;
-        } else if (value instanceof Byte) {
-            return (char)(byte)value;
-        } else {
-            return (Character)value;
-        }
-    }
-
-    public static Integer DefToIntegerImplicit(final Object value) {
-        if (value == null) {
-            return null;
-        } else if (value instanceof Byte || value instanceof Short) {
-            return ((Number)value).intValue();
-        } else if (value instanceof Character) {
-            return (int)(char)value;
-        } else {
-            return (Integer)value;
-        }
-    }
-
-    public static Long DefToLongImplicit(final Object value) {
-        if (value == null) {
-            return null;
-        } else if (value instanceof Byte || value instanceof Short || value instanceof Integer) {
-            return ((Number)value).longValue();
-        } else if (value instanceof Character) {
-            return (long)(char)value;
-        } else {
-            return (Long)value;
-        }
-    }
-
-    public static Float DefToFloatImplicit(final Object value) {
-        if (value == null) {
-            return null;
-        } else if (value instanceof Byte || value instanceof Short || value instanceof Integer || value instanceof Long) {
-            return ((Number)value).floatValue();
-        } else if (value instanceof Character) {
-            return (float)(char)value;
-        } else {
-            return (Float)value;
-        }
-    }
-
-    public static Double DefToDoubleImplicit(final Object value) {
-        if (value == null) {
-            return null;
-        } else if (value instanceof Byte || value instanceof Short ||
-            value instanceof Integer || value instanceof Long || value instanceof  Float) {
-            return ((Number)value).doubleValue();
-        } else if (value instanceof Character) {
-            return (double)(char)value;
-        } else {
-            return (Double)value;
-        }
-    }
-
     public static byte DefTobyteExplicit(final Object value) {
         if (value instanceof Character) {
             return (byte)(char)value;
@@ -1180,74 +1111,4 @@ public final class Def {
             return ((Number)value).doubleValue();
         }
     }
-
-    public static Byte DefToByteExplicit(final Object value) {
-        if (value == null) {
-            return null;
-        } else if (value instanceof Character) {
-            return (byte)(char)value;
-        } else {
-            return ((Number)value).byteValue();
-        }
-    }
-
-    public static Short DefToShortExplicit(final Object value) {
-        if (value == null) {
-            return null;
-        } else if (value instanceof Character) {
-            return (short)(char)value;
-        } else {
-            return ((Number)value).shortValue();
-        }
-    }
-
-    public static Character DefToCharacterExplicit(final Object value) {
-        if (value == null) {
-            return null;
-        } else if (value instanceof Character) {
-            return ((Character)value);
-        } else {
-            return (char)((Number)value).intValue();
-        }
-    }
-
-    public static Integer DefToIntegerExplicit(final Object value) {
-        if (value == null) {
-            return null;
-        } else if (value instanceof Character) {
-            return (int)(char)value;
-        } else {
-            return ((Number)value).intValue();
-        }
-    }
-
-    public static Long DefToLongExplicit(final Object value) {
-        if (value == null) {
-            return null;
-        } else if (value instanceof Character) {
-            return (long)(char)value;
-        } else {
-            return ((Number)value).longValue();
-        }
-    }
-
-    public static Float DefToFloatExplicit(final Object value) {
-        if (value == null) {
-            return null;
-        } else if (value instanceof Character) {
-            return (float)(char)value;
-        } else {
-            return ((Number)value).floatValue();
-        }
-    }
-
-    public static Double DefToDoubleExplicit(final Object value) {
-        if (value == null) {
-            return null;
-        } else if (value instanceof Character) {
-            return (double)(char)value;
-        } else {
-            return ((Number)value).doubleValue();
-        }
-    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 95d169cb886..f20393fb6ed 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -308,50 +308,32 @@ public final class Definition {
         public final Type from;
         public final Type to;
         public final boolean explicit;
+        public final boolean unboxFrom;
+        public final boolean unboxTo;
+        public final boolean boxFrom;
+        public final boolean boxTo;
 
         public Cast(final Type from, final Type to, final boolean explicit) {
             this.from = from;
             this.to = to;
             this.explicit = explicit;
+            this.unboxFrom = false;
+            this.unboxTo = false;
+            this.boxFrom = false;
+            this.boxTo = false;
         }
 
-        @Override
-        public boolean equals(final Object object) {
-            if (this == object) {
-                return true;
-            }
-
-            if (object == null || getClass() != object.getClass()) {
-                return false;
-            }
-
-            final Cast cast = (Cast)object;
-
-            return from.equals(cast.from) && to.equals(cast.to) && explicit == cast.explicit;
+        public Cast(final Type from, final Type to, final boolean explicit,
+                    final boolean unboxFrom, final boolean unboxTo, final boolean boxFrom, final boolean boxTo) {
+            this.from = from;
+            this.to = to;
+            this.explicit = explicit;
+            this.unboxFrom = unboxFrom;
+            this.unboxTo = unboxTo;
+            this.boxFrom = boxFrom;
+            this.boxTo = boxTo;
         }
 
-        @Override
-        public int hashCode() {
-            int result = from.hashCode();
-            result = 31 * result + to.hashCode();
-            result = 31 * result + (explicit ? 1 : 0);
-
-            return result;
-        }
-    }
-
-    public static final class Transform extends Cast {
-        public final Method method;
-        public final Type upcast;
-        public final Type downcast;
-
-        public Transform(final Cast cast, Method method, final Type upcast, final Type downcast) {
-            super(cast.from, cast.to, cast.explicit);
-
-            this.method = method;
-            this.upcast = upcast;
-            this.downcast = downcast;
-        }
     }
 
     public static final class RuntimeClass {
@@ -367,7 +349,6 @@ public final class Definition {
         }
     }
 
-    final Map<Cast, Cast> transformsMap;
     final Map<Class<?>, RuntimeClass> runtimeMap;
     private final Map<String, Struct> structsMap;
     private final Map<String, Type> simpleTypesMap;
@@ -375,7 +356,6 @@ public final class Definition {
     private Definition() {
         structsMap = new HashMap<>();
         simpleTypesMap = new HashMap<>();
-        transformsMap = new HashMap<>();
         runtimeMap = new HashMap<>();
 
         // parse the classes and return hierarchy (map of class name -> superclasses/interfaces)
@@ -386,7 +366,6 @@ public final class Definition {
         for (Map.Entry<String,List<String>> clazz : hierarchy.entrySet()) {
             copyStruct(clazz.getKey(), clazz.getValue());
         }
-        addTransforms();
         // precompute runtime classes
         for (Struct struct : structsMap.values()) {
           addRuntimeClass(struct);
@@ -401,7 +380,6 @@ public final class Definition {
         }
 
         this.structsMap = Collections.unmodifiableMap(structs);
-        this.transformsMap = Collections.unmodifiableMap(definition.transformsMap);
         this.runtimeMap = Collections.unmodifiableMap(definition.runtimeMap);
         this.simpleTypesMap = Collections.unmodifiableMap(definition.simpleTypesMap);
     }
@@ -503,130 +481,6 @@ public final class Definition {
         }
     }
 
-    private void addTransforms() {
-        Type booleanType = getType("boolean");
-        Type objectType = getType("Object");
-        Type defType = getType("def");
-        Type booleanobjType = getType("Boolean");
-        Type byteType = getType("byte");
-        Type shortType = getType("short");
-        Type intType = getType("int");
-        Type charType = getType("char");
-        Type longType = getType("long");
-        Type floatType = getType("float");
-        Type doubleType = getType("double");
-        Type numberType = getType("Number");
-        Type byteobjType = getType("Byte");
-        Type shortobjType = getType("Short");
-        Type charobjType = getType("Character");
-        Type intobjType = getType("Integer");
-        Type longobjType = getType("Long");
-        Type floatobjType = getType("Float");
-        Type doubleobjType = getType("Double");
-        Type stringType = getType("String");
-
-        addTransform(booleanType, objectType, "Boolean", "valueOf", true, false);
-        addTransform(booleanType, defType, "Boolean", "valueOf", true, false);
-        addTransform(booleanType, booleanobjType, "Boolean", "valueOf", true, false);
-
-        addTransform(byteType, shortType, false);
-        addTransform(byteType, charType, true);
-        addTransform(byteType, intType, false);
-        addTransform(byteType, longType, false);
-        addTransform(byteType, floatType, false);
-        addTransform(byteType, doubleType, false);
-        addTransform(byteType, objectType, "Byte", "valueOf", true, false);
-        addTransform(byteType, defType, "Byte", "valueOf", true, false);
-        addTransform(byteType, numberType, "Byte", "valueOf", true, false);
-        addTransform(byteType, byteobjType, "Byte", "valueOf", true, false);
-
-        addTransform(shortType, byteType, true);
-        addTransform(shortType, charType, true);
-        addTransform(shortType, intType, false);
-        addTransform(shortType, longType, false);
-        addTransform(shortType, floatType, false);
-        addTransform(shortType, doubleType, false);
-        addTransform(shortType, objectType, "Short", "valueOf", true, false);
-        addTransform(shortType, defType, "Short", "valueOf", true, false);
-        addTransform(shortType, numberType, "Short", "valueOf", true, false);
-        addTransform(shortType, shortobjType, "Short", "valueOf", true, false);
-
-        addTransform(charType, byteType, true);
-        addTransform(charType, shortType, true);
-        addTransform(charType, intType, false);
-        addTransform(charType, longType, false);
-        addTransform(charType, floatType, false);
-        addTransform(charType, doubleType, false);
-        addTransform(charType, objectType, "Character", "valueOf", true, false);
-        addTransform(charType, defType, "Character", "valueOf", true, false);
-        addTransform(charType, numberType, "Utility", "charToInteger", true, false);
-        addTransform(charType, charobjType, "Character", "valueOf", true, false);
-        addTransform(charType, stringType, "Utility", "charToString", true, true);
-
-        addTransform(intType, byteType, true);
-        addTransform(intType, shortType, true);
-        addTransform(intType, charType, true);
-        addTransform(intType, longType, false);
-        addTransform(intType, floatType, false);
-        addTransform(intType, doubleType, false);
-        addTransform(intType, objectType, "Integer", "valueOf", true, false);
-        addTransform(intType, defType, "Integer", "valueOf", true, false);
-        addTransform(intType, numberType, "Integer", "valueOf", true, false);
-        addTransform(intType, intobjType, "Integer", "valueOf", true, false);
-
-        addTransform(longType, byteType, true);
-        addTransform(longType, shortType, true);
-        addTransform(longType, charType, true);
-        addTransform(longType, intType, false);
-        addTransform(longType, floatType, false);
-        addTransform(longType, doubleType, false);
-        addTransform(longType, objectType, "Long", "valueOf", true, false);
-        addTransform(longType, defType, "Long", "valueOf", true, false);
-        addTransform(longType, numberType, "Long", "valueOf", true, false);
-        addTransform(longType, longobjType, "Long", "valueOf", true, false);
-
-        addTransform(floatType, byteType, true);
-        addTransform(floatType, shortType, true);
-        addTransform(floatType, charType, true);
-        addTransform(floatType, intType, true);
-        addTransform(floatType, longType, false);
-        addTransform(floatType, doubleType, false);
-        addTransform(floatType, objectType, "Float", "valueOf", true, false);
-        addTransform(floatType, defType, "Float", "valueOf", true, false);
-        addTransform(floatType, numberType, "Float", "valueOf", true, false);
-        addTransform(floatType, floatobjType, "Float", "valueOf", true, false);
-
-        addTransform(doubleType, byteType, true);
-        addTransform(doubleType, shortType, true);
-        addTransform(doubleType, charType, true);
-        addTransform(doubleType, intType, true);
-        addTransform(doubleType, longType, true);
-        addTransform(doubleType, floatType, false);
-        addTransform(doubleType, objectType, "Double", "valueOf", true, false);
-        addTransform(doubleType, defType, "Double", "valueOf", true, false);
-        addTransform(doubleType, numberType, "Double", "valueOf", true, false);
-        addTransform(doubleType, doubleobjType, "Double", "valueOf", true, false);
-
-        addTransform(defType, booleanType, "Boolean", "booleanValue", false, false);
-        addTransform(defType, byteType, "Def", "DefTobyteImplicit", true, false);
-        addTransform(defType, shortType, "Def", "DefToshortImplicit", true, false);
-        addTransform(defType, charType, "Def", "DefTocharImplicit", true, false);
-        addTransform(defType, intType, "Def", "DefTointImplicit", true, false);
-        addTransform(defType, longType, "Def", "DefTolongImplicit", true, false);
-        addTransform(defType, floatType, "Def", "DefTofloatImplicit", true, false);
-        addTransform(defType, doubleType, "Def", "DefTodoubleImplicit", true, false);
-        addTransform(defType, byteType, "Def", "DefTobyteExplicit", true, true);
-        addTransform(defType, shortType, "Def", "DefToshortExplicit", true, true);
-        addTransform(defType, charType, "Def", "DefTocharExplicit", true, true);
-        addTransform(defType, intType, "Def", "DefTointExplicit", true, true);
-        addTransform(defType, longType, "Def", "DefTolongExplicit", true, true);
-        addTransform(defType, floatType, "Def", "DefTofloatExplicit", true, true);
-        addTransform(defType, doubleType, "Def", "DefTodoubleExplicit", true, true);
-
-        addTransform(stringType, charType, "Utility", "StringTochar", true, true);
-        addTransform(stringType, charobjType, "Utility", "StringToCharacter", true, true);
-    }
-
     private final void addStruct(final String name, final Class<?> clazz) {
         if (!name.matches("^[_a-zA-Z][<>,_a-zA-Z0-9]*$")) {
             throw new IllegalArgumentException("Invalid struct name [" + name + "].");
@@ -960,137 +814,6 @@ public final class Definition {
         }
     }
 
-    private final void addTransform(final Type from, final Type to, final boolean explicit) {
-        if (from.equals(to)) {
-            throw new IllegalArgumentException("Transform cannot" +
-                " have cast type from [" + from.name + "] be the same as cast type to [" + to.name + "].");
-        }
-
-        if (!from.sort.primitive || !to.sort.primitive) {
-            throw new IllegalArgumentException("Only transforms between two primitives may be a simple cast, but" +
-                "found [" + from.name + "] and [" + to.name + "].");
-        }
-
-        final Cast cast = new Cast(from, to, explicit);
-
-        if (transformsMap.containsKey(cast)) {
-            throw new IllegalArgumentException("Transform with " +
-                " cast type from [" + from.name + "] to cast type to [" + to.name + "] already defined.");
-        }
-
-        transformsMap.put(cast, cast);
-    }
-
-    private final void addTransform(final Type from, final Type to, final String struct,
-                                   final String name, final boolean statik, final boolean explicit) {
-        final Struct owner = structsMap.get(struct);
-
-        if (owner == null) {
-            throw new IllegalArgumentException("Owner struct [" + struct + "] not defined for" +
-                " transform with cast type from [" + from.name + "] and cast type to [" + to.name + "].");
-        }
-
-        if (from.equals(to)) {
-            throw new IllegalArgumentException("Transform with owner struct [" + owner.name + "] cannot" +
-                " have cast type from [" + from.name + "] be the same as cast type to [" + to.name + "].");
-        }
-
-        final Cast cast = new Cast(from, to, explicit);
-
-        if (transformsMap.containsKey(cast)) {
-            throw new IllegalArgumentException("Transform with owner struct [" + owner.name + "]" +
-                " and cast type from [" + from.name + "] to cast type to [" + to.name + "] already defined.");
-        }
-
-        final Cast transform;
-
-        final Method method;
-        Type upcast = null;
-        Type downcast = null;
-
-        // transforms are implicitly arity of 0, unless a static method where its 1 (receiver passed)
-        final MethodKey methodKey = new MethodKey(name, statik ? 1 : 0);
-
-        if (statik) {
-            method = owner.staticMethods.get(methodKey);
-
-            if (method == null) {
-                throw new IllegalArgumentException("Transform with owner struct [" + owner.name + "]" +
-                    " and cast type from [" + from.name + "] to cast type to [" + to.name +
-                    "] using a function [" + name + "] that is not defined.");
-            }
-
-            if (method.arguments.size() != 1) {
-                throw new IllegalArgumentException("Transform with owner struct [" + owner.name + "]" +
-                    " and cast type from [" + from.name + "] to cast type to [" + to.name +
-                    "] using function [" + name + "] does not have a single type argument.");
-            }
-
-            Type argument = method.arguments.get(0);
-
-            if (!argument.clazz.isAssignableFrom(from.clazz)) {
-                if (from.clazz.isAssignableFrom(argument.clazz)) {
-                    upcast = argument;
-                } else {
-                    throw new ClassCastException("Transform with owner struct [" + owner.name + "]" +
-                        " and cast type from [" + from.name + "] to cast type to [" + to.name + "] using" +
-                        " function [" + name + "] cannot cast from type to the function input argument type.");
-                }
-            }
-
-            final Type rtn = method.rtn;
-
-            if (!to.clazz.isAssignableFrom(rtn.clazz)) {
-                if (rtn.clazz.isAssignableFrom(to.clazz)) {
-                    downcast = to;
-                } else {
-                    throw new ClassCastException("Transform with owner struct [" + owner.name + "]" +
-                        " and cast type from [" + from.name + "] to cast type to [" + to.name + "] using" +
-                        " function [" + name + "] cannot cast to type to the function return argument type.");
-                }
-            }
-        } else {
-            method = owner.methods.get(methodKey);
-
-            if (method == null) {
-                throw new IllegalArgumentException("Transform with owner struct [" + owner.name + "]" +
-                    " and cast type from [" + from.name + "] to cast type to [" + to.name +
-                    "] using a method [" + name + "] that is not defined.");
-            }
-
-            if (!method.arguments.isEmpty()) {
-                throw new IllegalArgumentException("Transform with owner struct [" + owner.name + "]" +
-                    " and cast type from [" + from.name + "] to cast type to [" + to.name +
-                    "] using method [" + name + "] does not have a single type argument.");
-            }
-
-            if (!owner.clazz.isAssignableFrom(from.clazz)) {
-                if (from.clazz.isAssignableFrom(owner.clazz)) {
-                    upcast = getType(owner.name);
-                } else {
-                    throw new ClassCastException("Transform with owner struct [" + owner.name + "]" +
-                        " and cast type from [" + from.name + "] to cast type to [" + to.name + "] using" +
-                        " method [" + name + "] cannot cast from type to the method input argument type.");
-                }
-            }
-
-            final Type rtn = method.rtn;
-
-            if (!to.clazz.isAssignableFrom(rtn.clazz)) {
-                if (rtn.clazz.isAssignableFrom(to.clazz)) {
-                    downcast = to;
-                } else {
-                    throw new ClassCastException("Transform with owner struct [" + owner.name + "]" +
-                        " and cast type from [" + from.name + "] to cast type to [" + to.name + "]" +
-                        " using method [" + name + "] cannot cast to type to the method return argument type.");
-                }
-            }
-        }
-
-        transform = new Transform(cast, method, upcast, downcast);
-        transformsMap.put(cast, transform);
-    }
-
     /**
      * Precomputes a more efficient structure for dynamic method/field access.
      */
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index cda8c23e3bf..52070ee3f77 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -21,7 +21,6 @@ package org.elasticsearch.painless;
 
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Definition.Sort;
-import org.elasticsearch.painless.Definition.Transform;
 import org.elasticsearch.painless.Definition.Type;
 import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.Label;
@@ -34,6 +33,7 @@ import java.util.ArrayList;
 import java.util.Deque;
 import java.util.List;
 
+import static org.elasticsearch.painless.WriterConstants.CHAR_TO_STRING;
 import static org.elasticsearch.painless.WriterConstants.DEF_ADD_CALL;
 import static org.elasticsearch.painless.WriterConstants.DEF_AND_CALL;
 import static org.elasticsearch.painless.WriterConstants.DEF_DIV_CALL;
@@ -43,6 +43,22 @@ import static org.elasticsearch.painless.WriterConstants.DEF_OR_CALL;
 import static org.elasticsearch.painless.WriterConstants.DEF_REM_CALL;
 import static org.elasticsearch.painless.WriterConstants.DEF_RSH_CALL;
 import static org.elasticsearch.painless.WriterConstants.DEF_SUB_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_TO_BOOLEAN;
+import static org.elasticsearch.painless.WriterConstants.DEF_TO_BYTE_EXPLICIT;
+import static org.elasticsearch.painless.WriterConstants.DEF_TO_BYTE_IMPLICIT;
+import static org.elasticsearch.painless.WriterConstants.DEF_TO_CHAR_EXPLICIT;
+import static org.elasticsearch.painless.WriterConstants.DEF_TO_CHAR_IMPLICIT;
+import static org.elasticsearch.painless.WriterConstants.DEF_TO_DOUBLE_EXPLICIT;
+import static org.elasticsearch.painless.WriterConstants.DEF_TO_DOUBLE_IMPLICIT;
+import static org.elasticsearch.painless.WriterConstants.DEF_TO_FLOAT_EXPLICIT;
+import static org.elasticsearch.painless.WriterConstants.DEF_TO_FLOAT_IMPLICIT;
+import static org.elasticsearch.painless.WriterConstants.DEF_TO_INT_EXPLICIT;
+import static org.elasticsearch.painless.WriterConstants.DEF_TO_INT_IMPLICIT;
+import static org.elasticsearch.painless.WriterConstants.DEF_TO_LONG_EXPLICIT;
+import static org.elasticsearch.painless.WriterConstants.DEF_TO_LONG_IMPLICIT;
+import static org.elasticsearch.painless.WriterConstants.DEF_TO_SHORT_EXPLICIT;
+import static org.elasticsearch.painless.WriterConstants.DEF_TO_SHORT_IMPLICIT;
+import static org.elasticsearch.painless.WriterConstants.DEF_TYPE;
 import static org.elasticsearch.painless.WriterConstants.DEF_USH_CALL;
 import static org.elasticsearch.painless.WriterConstants.DEF_XOR_CALL;
 import static org.elasticsearch.painless.WriterConstants.INDY_STRING_CONCAT_BOOTSTRAP_HANDLE;
@@ -59,7 +75,9 @@ import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_ST
 import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_CONSTRUCTOR;
 import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_TOSTRING;
 import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_TYPE;
+import static org.elasticsearch.painless.WriterConstants.STRING_TO_CHAR;
 import static org.elasticsearch.painless.WriterConstants.STRING_TYPE;
+import static org.elasticsearch.painless.WriterConstants.UTILITY_TYPE;
 
 /**
  * Extension of {@link GeneratorAdapter} with some utility methods.
@@ -96,45 +114,72 @@ public final class MethodWriter extends GeneratorAdapter {
             visitVarInsn(Opcodes.ILOAD, slot);
             push(0);
             ifICmp(GeneratorAdapter.GT, end);
-            throwException(PAINLESS_ERROR_TYPE,
-                "The maximum number of statements that can be executed in a loop has been reached.");
+            throwException(PAINLESS_ERROR_TYPE, "The maximum number of statements that can be executed in a loop has been reached.");
             mark(end);
         }
     }
 
     public void writeCast(final Cast cast) {
-        if (cast instanceof Transform) {
-            final Transform transform = (Transform)cast;
-
-            if (transform.upcast != null) {
-                checkCast(transform.upcast.type);
-            }
-
-            if (java.lang.reflect.Modifier.isStatic(transform.method.reflect.getModifiers())) {
-                invokeStatic(transform.method.owner.type, transform.method.method);
-            } else if (java.lang.reflect.Modifier.isInterface(transform.method.owner.clazz.getModifiers())) {
-                invokeInterface(transform.method.owner.type, transform.method.method);
-            } else {
-                invokeVirtual(transform.method.owner.type, transform.method.method);
-            }
-
-            if (transform.downcast != null) {
-                checkCast(transform.downcast.type);
-            }
-        } else if (cast != null) {
+        if (cast != null) {
             final Type from = cast.from;
             final Type to = cast.to;
 
-            if (from.equals(to)) {
-                return;
-            }
-
-            if (from.sort.numeric && from.sort.primitive && to.sort.numeric && to.sort.primitive) {
-                cast(from.type, to.type);
-            } else {
-                if (!to.clazz.isAssignableFrom(from.clazz)) {
-                    checkCast(to.type);
+            if (from.sort == Sort.CHAR && to.sort == Sort.STRING) {
+                invokeStatic(UTILITY_TYPE, CHAR_TO_STRING);
+            } else if (from.sort == Sort.STRING && to.sort == Sort.CHAR) {
+                invokeStatic(UTILITY_TYPE, STRING_TO_CHAR);
+            } else if (cast.unboxFrom) {
+                if (from.sort == Sort.DEF) {
+                    if (cast.explicit) {
+                        if      (to.sort == Sort.BOOL)   invokeStatic(DEF_TYPE, DEF_TO_BOOLEAN);
+                        else if (to.sort == Sort.BYTE)   invokeStatic(DEF_TYPE, DEF_TO_BYTE_EXPLICIT);
+                        else if (to.sort == Sort.SHORT)  invokeStatic(DEF_TYPE, DEF_TO_SHORT_EXPLICIT);
+                        else if (to.sort == Sort.CHAR)   invokeStatic(DEF_TYPE, DEF_TO_CHAR_EXPLICIT);
+                        else if (to.sort == Sort.INT)    invokeStatic(DEF_TYPE, DEF_TO_INT_EXPLICIT);
+                        else if (to.sort == Sort.LONG)   invokeStatic(DEF_TYPE, DEF_TO_LONG_EXPLICIT);
+                        else if (to.sort == Sort.FLOAT)  invokeStatic(DEF_TYPE, DEF_TO_FLOAT_EXPLICIT);
+                        else if (to.sort == Sort.DOUBLE) invokeStatic(DEF_TYPE, DEF_TO_DOUBLE_EXPLICIT);
+                        else throw new IllegalStateException("Illegal tree structure.");
+                    } else {
+                        if      (to.sort == Sort.BOOL)   invokeStatic(DEF_TYPE, DEF_TO_BOOLEAN);
+                        else if (to.sort == Sort.BYTE)   invokeStatic(DEF_TYPE, DEF_TO_BYTE_IMPLICIT);
+                        else if (to.sort == Sort.SHORT)  invokeStatic(DEF_TYPE, DEF_TO_SHORT_IMPLICIT);
+                        else if (to.sort == Sort.CHAR)   invokeStatic(DEF_TYPE, DEF_TO_CHAR_IMPLICIT);
+                        else if (to.sort == Sort.INT)    invokeStatic(DEF_TYPE, DEF_TO_INT_IMPLICIT);
+                        else if (to.sort == Sort.LONG)   invokeStatic(DEF_TYPE, DEF_TO_LONG_IMPLICIT);
+                        else if (to.sort == Sort.FLOAT)  invokeStatic(DEF_TYPE, DEF_TO_FLOAT_IMPLICIT);
+                        else if (to.sort == Sort.DOUBLE) invokeStatic(DEF_TYPE, DEF_TO_DOUBLE_IMPLICIT);
+                        else throw new IllegalStateException("Illegal tree structure.");
+                    }
+                } else {
+                    unbox(from.type);
+                    writeCast(from, to);
                 }
+            } else if (cast.unboxTo) {
+                writeCast(from, to);
+                unbox(to.type);
+            } else if (cast.boxFrom) {
+                box(from.type);
+                writeCast(from, to);
+            } else if (cast.boxTo) {
+                writeCast(from, to);
+                box(to.type);
+            } else {
+                writeCast(from, to);
+            }
+        }
+    }
+
+    private void writeCast(final Type from, final Type to) {
+        if (from.equals(to)) {
+            return;
+        }
+
+        if (from.sort.numeric && from.sort.primitive && to.sort.numeric && to.sort.primitive) {
+            cast(from.type, to.type);
+        } else {
+            if (!to.clazz.isAssignableFrom(from.clazz)) {
+                checkCast(to.type);
             }
         }
     }
@@ -146,7 +191,7 @@ public final class MethodWriter extends GeneratorAdapter {
             visitJumpInsn(Opcodes.IFEQ, fals);
         }
     }
-    
+
     public void writeNewStrings() {
         if (INDY_STRING_CONCAT_BOOTSTRAP_HANDLE != null) {
             // Java 9+: we just push our argument collector onto deque
@@ -200,31 +245,29 @@ public final class MethodWriter extends GeneratorAdapter {
         }
     }
 
-    public void writeBinaryInstruction(final Definition definition,
-                                              final String location,
-                                              final Type type, final Operation operation) {
+    public void writeBinaryInstruction(final String location, final Type type, final Operation operation) {
         final Sort sort = type.sort;
-        
+
         if ((sort == Sort.FLOAT || sort == Sort.DOUBLE) &&
                 (operation == Operation.LSH || operation == Operation.USH ||
                 operation == Operation.RSH || operation == Operation.BWAND ||
                 operation == Operation.XOR || operation == Operation.BWOR)) {
             throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
         }
-        
+
         if (sort == Sort.DEF) {
             switch (operation) {
-                case MUL:   invokeStatic(definition.getType("Def").type, DEF_MUL_CALL); break;
-                case DIV:   invokeStatic(definition.getType("Def").type, DEF_DIV_CALL); break;
-                case REM:   invokeStatic(definition.getType("Def").type, DEF_REM_CALL); break;
-                case ADD:   invokeStatic(definition.getType("Def").type, DEF_ADD_CALL); break;
-                case SUB:   invokeStatic(definition.getType("Def").type, DEF_SUB_CALL); break;
-                case LSH:   invokeStatic(definition.getType("Def").type, DEF_LSH_CALL); break;
-                case USH:   invokeStatic(definition.getType("Def").type, DEF_RSH_CALL); break;
-                case RSH:   invokeStatic(definition.getType("Def").type, DEF_USH_CALL); break;
-                case BWAND: invokeStatic(definition.getType("Def").type, DEF_AND_CALL); break;
-                case XOR:   invokeStatic(definition.getType("Def").type, DEF_XOR_CALL); break;
-                case BWOR:  invokeStatic(definition.getType("Def").type, DEF_OR_CALL);  break;
+                case MUL:   invokeStatic(DEF_TYPE, DEF_MUL_CALL); break;
+                case DIV:   invokeStatic(DEF_TYPE, DEF_DIV_CALL); break;
+                case REM:   invokeStatic(DEF_TYPE, DEF_REM_CALL); break;
+                case ADD:   invokeStatic(DEF_TYPE, DEF_ADD_CALL); break;
+                case SUB:   invokeStatic(DEF_TYPE, DEF_SUB_CALL); break;
+                case LSH:   invokeStatic(DEF_TYPE, DEF_LSH_CALL); break;
+                case USH:   invokeStatic(DEF_TYPE, DEF_RSH_CALL); break;
+                case RSH:   invokeStatic(DEF_TYPE, DEF_USH_CALL); break;
+                case BWAND: invokeStatic(DEF_TYPE, DEF_AND_CALL); break;
+                case XOR:   invokeStatic(DEF_TYPE, DEF_XOR_CALL); break;
+                case BWOR:  invokeStatic(DEF_TYPE, DEF_OR_CALL);  break;
                 default:
                     throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java
index 559e909ed77..c8429f9458a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java
@@ -469,18 +469,6 @@ public class Utility {
         return value.charAt(0);
     }
 
-    public static Character StringToCharacter(final String value) {
-        if (value == null) {
-            return null;
-        }
-
-        if (value.length() != 1) {
-            throw new ClassCastException("Cannot cast [String] with length greater than one to [Character].");
-        }
-
-        return value.charAt(0);
-    }
-
     public static boolean checkEquals(final Object left, final Object right) {
         if (left != null) {
             return left.equals(right);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index d167646e9dd..6330e1603c5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -38,7 +38,7 @@ public final class WriterConstants {
 
     public final static String BASE_CLASS_NAME = Executable.class.getName();
     public final static Type BASE_CLASS_TYPE   = Type.getType(Executable.class);
-    
+
     public final static String CLASS_NAME      = BASE_CLASS_NAME + "$Script";
     public final static Type CLASS_TYPE        = Type.getObjectType(CLASS_NAME.replace('.', '/'));
 
@@ -55,6 +55,10 @@ public final class WriterConstants {
     public final static Type MAP_TYPE  = Type.getType(Map.class);
     public final static Method MAP_GET = getAsmMethod(Object.class, "get", Object.class);
 
+    public final static Type UTILITY_TYPE = Type.getType(Utility.class);
+    public final static Method STRING_TO_CHAR = getAsmMethod(char.class, "StringTochar", String.class);
+    public final static Method CHAR_TO_STRING = getAsmMethod(String.class, "charToString", char.class);
+
     /** dynamic callsite bootstrap signature */
     public final static MethodType DEF_BOOTSTRAP_TYPE =
         MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, int.class);
@@ -62,19 +66,36 @@ public final class WriterConstants {
         new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(DefBootstrap.class),
             "bootstrap", DEF_BOOTSTRAP_TYPE.toMethodDescriptorString());
 
-    public final static Method DEF_NOT_CALL = getAsmMethod(Object.class, "not", Object.class);
-    public final static Method DEF_NEG_CALL = getAsmMethod(Object.class, "neg", Object.class);
-    public final static Method DEF_MUL_CALL = getAsmMethod(Object.class, "mul", Object.class, Object.class);
-    public final static Method DEF_DIV_CALL = getAsmMethod(Object.class, "div", Object.class, Object.class);
-    public final static Method DEF_REM_CALL = getAsmMethod(Object.class, "rem", Object.class, Object.class);
-    public final static Method DEF_ADD_CALL = getAsmMethod(Object.class, "add", Object.class, Object.class);
-    public final static Method DEF_SUB_CALL = getAsmMethod(Object.class, "sub", Object.class, Object.class);
-    public final static Method DEF_LSH_CALL = getAsmMethod(Object.class, "lsh", Object.class, int.class);
-    public final static Method DEF_RSH_CALL = getAsmMethod(Object.class, "rsh", Object.class, int.class);
-    public final static Method DEF_USH_CALL = getAsmMethod(Object.class, "ush", Object.class, int.class);
-    public final static Method DEF_AND_CALL = getAsmMethod(Object.class, "and", Object.class, Object.class);
-    public final static Method DEF_XOR_CALL = getAsmMethod(Object.class, "xor", Object.class, Object.class);
-    public final static Method DEF_OR_CALL  = getAsmMethod(Object.class, "or" , Object.class, Object.class);
+
+    public final static Type DEF_TYPE = Type.getType(Def.class);
+    public final static Method DEF_TO_BOOLEAN         = getAsmMethod(boolean.class, "DefToboolean"       , Object.class);
+    public final static Method DEF_TO_BYTE_IMPLICIT   = getAsmMethod(byte.class   , "DefTobyteImplicit"  , Object.class);
+    public final static Method DEF_TO_SHORT_IMPLICIT  = getAsmMethod(short.class  , "DefToshortImplicit" , Object.class);
+    public final static Method DEF_TO_CHAR_IMPLICIT   = getAsmMethod(char.class   , "DefTocharImplicit"  , Object.class);
+    public final static Method DEF_TO_INT_IMPLICIT    = getAsmMethod(int.class    , "DefTointImplicit"   , Object.class);
+    public final static Method DEF_TO_LONG_IMPLICIT   = getAsmMethod(long.class   , "DefTolongImplicit"  , Object.class);
+    public final static Method DEF_TO_FLOAT_IMPLICIT  = getAsmMethod(float.class  , "DefTofloatImplicit" , Object.class);
+    public final static Method DEF_TO_DOUBLE_IMPLICIT = getAsmMethod(double.class , "DefTodoubleImplicit", Object.class);
+    public final static Method DEF_TO_BYTE_EXPLICIT   = getAsmMethod(byte.class   , "DefTobyteExplicit"  , Object.class);
+    public final static Method DEF_TO_SHORT_EXPLICIT  = getAsmMethod(short.class  , "DefToshortExplicit" , Object.class);
+    public final static Method DEF_TO_CHAR_EXPLICIT   = getAsmMethod(char.class   , "DefTocharExplicit"  , Object.class);
+    public final static Method DEF_TO_INT_EXPLICIT    = getAsmMethod(int.class    , "DefTointExplicit"   , Object.class);
+    public final static Method DEF_TO_LONG_EXPLICIT   = getAsmMethod(long.class   , "DefTolongExplicit"  , Object.class);
+    public final static Method DEF_TO_FLOAT_EXPLICIT  = getAsmMethod(float.class  , "DefTofloatExplicit" , Object.class);
+    public final static Method DEF_TO_DOUBLE_EXPLICIT = getAsmMethod(double.class , "DefTodoubleExplicit", Object.class);
+    public final static Method DEF_NOT_CALL = getAsmMethod(Object.class , "not", Object.class);
+    public final static Method DEF_NEG_CALL = getAsmMethod(Object.class , "neg", Object.class);
+    public final static Method DEF_MUL_CALL = getAsmMethod(Object.class , "mul", Object.class, Object.class);
+    public final static Method DEF_DIV_CALL = getAsmMethod(Object.class , "div", Object.class, Object.class);
+    public final static Method DEF_REM_CALL = getAsmMethod(Object.class , "rem", Object.class, Object.class);
+    public final static Method DEF_ADD_CALL = getAsmMethod(Object.class , "add", Object.class, Object.class);
+    public final static Method DEF_SUB_CALL = getAsmMethod(Object.class , "sub", Object.class, Object.class);
+    public final static Method DEF_LSH_CALL = getAsmMethod(Object.class , "lsh", Object.class, int.class);
+    public final static Method DEF_RSH_CALL = getAsmMethod(Object.class , "rsh", Object.class, int.class);
+    public final static Method DEF_USH_CALL = getAsmMethod(Object.class , "ush", Object.class, int.class);
+    public final static Method DEF_AND_CALL = getAsmMethod(Object.class , "and", Object.class, Object.class);
+    public final static Method DEF_XOR_CALL = getAsmMethod(Object.class , "xor", Object.class, Object.class);
+    public final static Method DEF_OR_CALL  = getAsmMethod(Object.class , "or" , Object.class, Object.class);
     public final static Method DEF_EQ_CALL  = getAsmMethod(boolean.class, "eq" , Object.class, Object.class);
     public final static Method DEF_LT_CALL  = getAsmMethod(boolean.class, "lt" , Object.class, Object.class);
     public final static Method DEF_LTE_CALL = getAsmMethod(boolean.class, "lte", Object.class, Object.class);
@@ -98,9 +119,9 @@ public final class WriterConstants {
         }
         INDY_STRING_CONCAT_BOOTSTRAP_HANDLE = bs;
     }
-    
+
     public final static int MAX_INDY_STRING_CONCAT_ARGS = 200;
-    
+
     public final static Type STRING_TYPE = Type.getType(String.class);
     public final static Type STRINGBUILDER_TYPE = Type.getType(StringBuilder.class);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
index 528da4384c8..df03dfcef02 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
@@ -67,6 +67,12 @@ public abstract class AExpression extends ANode {
      */
     protected boolean explicit = false;
 
+    /**
+     * Set to true if a cast is allowed to boxed/unboxed.  This is used
+     * for method arguments because casting may be required.
+     */
+    protected boolean internal = false;
+
     /**
      * Set to the value of the constant this expression node represents if
      * and only if the node represents a constant.  If this is not null
@@ -114,7 +120,7 @@ public abstract class AExpression extends ANode {
      * @return The new child node for the parent node calling this method.
      */
     AExpression cast(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        final Cast cast = AnalyzerCaster.getLegalCast(definition, location, actual, expected, explicit);
+        final Cast cast = AnalyzerCaster.getLegalCast(definition, location, actual, expected, explicit, internal);
 
         if (cast == null) {
             if (constant == null || this instanceof EConstant) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 483e82cc500..f7177613857 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -495,7 +495,7 @@ public final class EBinary extends AExpression {
             left.write(settings, definition, adapter);
             right.write(settings, definition, adapter);
 
-            adapter.writeBinaryInstruction(definition, location, actual, operation);
+            adapter.writeBinaryInstruction(location, actual, operation);
         }
 
         adapter.writeBranch(tru, fals);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index 0b91ee406e8..a7471f07b9c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -207,8 +207,8 @@ public final class EChain extends AExpression {
 
         expression = expression.cast(settings, definition, variables);
 
-        there = AnalyzerCaster.getLegalCast(definition, location, last.after, promote, false);
-        back = AnalyzerCaster.getLegalCast(definition, location, promote, last.after, true);
+        there = AnalyzerCaster.getLegalCast(definition, location, last.after, promote, false, false);
+        back = AnalyzerCaster.getLegalCast(definition, location, promote, last.after, true, false);
 
         this.statement = true;
         this.actual = read ? last.after : definition.getType("void");
@@ -289,7 +289,7 @@ public final class EChain extends AExpression {
 
                     adapter.writeCast(there);
                     expression.write(settings, definition, adapter);
-                    adapter.writeBinaryInstruction(definition, location, promote, operation);
+                    adapter.writeBinaryInstruction(location, promote, operation);
 
                     adapter.writeCast(back);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
index 45976c13414..69f22e828cd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
@@ -57,8 +57,10 @@ public final class EConditional extends AExpression {
 
         left.expected = expected;
         left.explicit = explicit;
+        left.internal = internal;
         right.expected = expected;
         right.explicit = explicit;
+        right.internal = internal;
         actual = expected;
 
         left.analyze(settings, definition, variables);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
index bd76aa293cc..9e519714a97 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
@@ -64,6 +64,7 @@ public final class LCall extends ALink {
                 final AExpression expression = arguments.get(argument);
 
                 expression.expected = method.arguments.get(argument);
+                expression.internal = true;
                 expression.analyze(settings, definition, variables);
                 arguments.set(argument, expression.cast(settings, definition, variables));
             }
@@ -79,7 +80,7 @@ public final class LCall extends ALink {
             return link.analyze(settings, definition, variables);
         }
 
-        throw new IllegalArgumentException(error("Unknown call [" + name + "] with [" + arguments.size() + 
+        throw new IllegalArgumentException(error("Unknown call [" + name + "] with [" + arguments.size() +
                                                  "] arguments on type [" + struct.name + "]."));
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
index eb7fb3a6b10..126b8b2711b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
@@ -55,7 +55,7 @@ public final class LCast extends ALink {
             throw new IllegalArgumentException(error("Not a type [" + type + "]."));
         }
 
-        cast = AnalyzerCaster.getLegalCast(definition, location, before, after, true);
+        cast = AnalyzerCaster.getLegalCast(definition, location, before, after, true, false);
 
         return cast != null ? this : null;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index 1285d548163..1f7544d32c1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -49,6 +49,7 @@ final class LDefCall extends ALink implements IDefLink {
         for (int argument = 0; argument < arguments.size(); ++argument) {
             final AExpression expression = arguments.get(argument);
 
+            expression.internal = true;
             expression.analyze(settings, definition, variables);
             expression.expected = expression.actual;
             arguments.set(argument, expression.cast(settings, definition, variables));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
index 227b63cf31f..5610c8dd7e7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
@@ -78,6 +78,7 @@ public final class LNewObj extends ALink {
                 final AExpression expression = arguments.get(argument);
 
                 expression.expected = types[argument];
+                expression.internal = true;
                 expression.analyze(settings, definition, variables);
                 arguments.set(argument, expression.cast(settings, definition, variables));
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index 39ab0c099a6..1af2e213ebc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -50,6 +50,7 @@ public final class SExpression extends AStatement {
         final boolean rtn = lastSource && expression.actual.sort != Sort.VOID;
 
         expression.expected = rtn ? definition.getType("Object") : expression.actual;
+        expression.internal = rtn;
         expression = expression.cast(settings, definition, variables);
 
         methodEscape = rtn;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
index 68084a9a70c..dc76ca4c33d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -40,6 +40,7 @@ public final class SReturn extends AStatement {
     @Override
     void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
         expression.expected = definition.getType("Object");
+        expression.internal = true;
         expression.analyze(settings, definition, variables);
         expression = expression.cast(settings, definition, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
index ab6944619ca..0d4e993b39b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
@@ -22,6 +22,7 @@
  * <p>
  * The following are the types of nodes:
  * A* (abstract) - These are the abstract nodes that are the superclasses for the other types.
+ * I* (interface) -- Thse are marker interfaces to denote a property of the node.
  * S* (statement) - These are nodes that represent a statement in Painless.  These are the highest level nodes.
  * E* (expression) - These are nodess that represent an expression in Painless.  These are the middle level nodes.
  * L* (link) - These are nodes that respresent a piece of a variable/method chain.  The are the lowest level nodes.
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/definition.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/definition.txt
index 4ca37223043..379fadb9865 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/definition.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/definition.txt
@@ -69,7 +69,6 @@ class Void -> java.lang.Void extends Object {
 }
 
 class Boolean -> java.lang.Boolean extends Object {
-  Boolean <init>(boolean)
   Boolean TRUE
   Boolean FALSE
   int compare(boolean,boolean)
@@ -80,7 +79,6 @@ class Boolean -> java.lang.Boolean extends Object {
 } 
 
 class Byte -> java.lang.Byte extends Number,Object {
-  Byte <init>(byte)
   byte MIN_VALUE
   byte MAX_VALUE
   int compare(byte,byte)
@@ -90,7 +88,6 @@ class Byte -> java.lang.Byte extends Number,Object {
 }
 
 class Short -> java.lang.Short extends Number,Object {
-  Short <init>(short)
   short MIN_VALUE
   short MAX_VALUE
   int compare(short,short)
@@ -100,7 +97,6 @@ class Short -> java.lang.Short extends Number,Object {
 }
 
 class Character -> java.lang.Character extends Object {
-  Character <init>(char)
   char MIN_VALUE
   char MAX_VALUE
   int charCount(int)
@@ -127,7 +123,6 @@ class Character -> java.lang.Character extends Object {
 }
 
 class Integer -> java.lang.Integer extends Number,Object {
-  Integer <init>(int)
   int MIN_VALUE
   int MAX_VALUE
   int compare(int,int)
@@ -141,7 +136,6 @@ class Integer -> java.lang.Integer extends Number,Object {
 }
 
 class Long -> java.lang.Long extends Number,Object {
-  Long <init>(long)
   long MIN_VALUE
   long MAX_VALUE
   int compare(long,long)
@@ -155,7 +149,6 @@ class Long -> java.lang.Long extends Number,Object {
 }
 
 class Float -> java.lang.Float extends Number,Object {
-  Float <init>(float)
   float MIN_VALUE
   float MAX_VALUE
   int compare(float,float)
@@ -168,7 +161,6 @@ class Float -> java.lang.Float extends Number,Object {
 }
 
 class Double -> java.lang.Double extends Number,Object {
-  Double <init>(double)
   double MIN_VALUE
   double MAX_VALUE
   int compare(double,double)
@@ -365,152 +357,6 @@ class GeoPoints -> org.elasticsearch.index.fielddata.ScriptDocValues$GeoPoints e
   double geohashDistanceInMiles(String)
 }
 
-# internal conversion methods
-class Utility -> org.elasticsearch.painless.Utility {
-  boolean NumberToboolean(Number)
-  char NumberTochar(Number)
-  Boolean NumberToBoolean(Number)
-  Byte NumberToByte(Number)
-  Short NumberToShort(Number)
-  Character NumberToCharacter(Number)
-  Integer NumberToInteger(Number)
-  Long NumberToLong(Number)
-  Float NumberToFloat(Number)
-  Double NumberToDouble(Number)
-  byte booleanTobyte(boolean)
-  short booleanToshort(boolean)
-  char booleanTochar(boolean)
-  int booleanToint(boolean)
-  long booleanTolong(boolean)
-  float booleanTofloat(boolean)
-  double booleanTodouble(boolean)
-  Integer booleanToInteger(boolean)
-  byte BooleanTobyte(Boolean)
-  short BooleanToshort(Boolean)
-  char BooleanTochar(Boolean)
-  int BooleanToint(Boolean)
-  long BooleanTolong(Boolean)
-  float BooleanTofloat(Boolean)
-  double BooleanTodouble(Boolean)
-  Byte BooleanToByte(Boolean)
-  Short BooleanToShort(Boolean)
-  Character BooleanToCharacter(Boolean)
-  Integer BooleanToInteger(Boolean)
-  Long BooleanToLong(Boolean)
-  Float BooleanToFloat(Boolean)
-  Double BooleanToDouble(Boolean)
-  boolean byteToboolean(byte)
-  Short byteToShort(byte)
-  Character byteToCharacter(byte)
-  Integer byteToInteger(byte)
-  Long byteToLong(byte)
-  Float byteToFloat(byte)
-  Double byteToDouble(byte)
-  boolean ByteToboolean(Byte)
-  char ByteTochar(Byte)
-  boolean shortToboolean(short)
-  Byte shortToByte(short)
-  Character shortToCharacter(short)
-  Integer shortToInteger(short)
-  Long shortToLong(short)
-  Float shortToFloat(short)
-  Double shortToDouble(short)
-  boolean ShortToboolean(Short)
-  char ShortTochar(Short)
-  boolean charToboolean(char)
-  Byte charToByte(char)
-  Short charToShort(char)
-  Integer charToInteger(char)
-  Long charToLong(char)
-  Float charToFloat(char)
-  Double charToDouble(char)
-  String charToString(char)
-  boolean CharacterToboolean(Character)
-  byte CharacterTobyte(Character)
-  short CharacterToshort(Character)
-  int CharacterToint(Character)
-  long CharacterTolong(Character)
-  float CharacterTofloat(Character)
-  double CharacterTodouble(Character)
-  Boolean CharacterToBoolean(Character)
-  Byte CharacterToByte(Character)
-  Short CharacterToShort(Character)
-  Integer CharacterToInteger(Character)
-  Long CharacterToLong(Character)
-  Float CharacterToFloat(Character)
-  Double CharacterToDouble(Character)
-  String CharacterToString(Character)
-  boolean intToboolean(int)
-  Byte intToByte(int)
-  Short intToShort(int)
-  Character intToCharacter(int)
-  Long intToLong(int)
-  Float intToFloat(int)
-  Double intToDouble(int)
-  boolean IntegerToboolean(Integer)
-  char IntegerTochar(Integer)
-  boolean longToboolean(long)
-  Byte longToByte(long)
-  Short longToShort(long)
-  Character longToCharacter(long)
-  Integer longToInteger(long)
-  Float longToFloat(long)
-  Double longToDouble(long)
-  boolean LongToboolean(Long)
-  char LongTochar(Long)
-  boolean floatToboolean(float)
-  Byte floatToByte(float)
-  Short floatToShort(float)
-  Character floatToCharacter(float)
-  Integer floatToInteger(float)
-  Long floatToLong(float)
-  Double floatToDouble(float)
-  boolean FloatToboolean(Float)
-  char FloatTochar(Float)
-  boolean doubleToboolean(double)
-  Byte doubleToByte(double)
-  Short doubleToShort(double)
-  Character doubleToCharacter(double)
-  Integer doubleToInteger(double)
-  Long doubleToLong(double)
-  Float doubleToFloat(double)
-  boolean DoubleToboolean(Double)
-  char DoubleTochar(Double)
-  char StringTochar(String)
-  Character StringToCharacter(String)
-}
-
-class Def -> org.elasticsearch.painless.Def {
-  byte DefTobyteImplicit(def)
-  short DefToshortImplicit(def)
-  char DefTocharImplicit(def)
-  int DefTointImplicit(def)
-  long DefTolongImplicit(def)
-  float DefTofloatImplicit(def)
-  double DefTodoubleImplicit(def)
-  Byte DefToByteImplicit(def)
-  Short DefToShortImplicit(def)
-  Character DefToCharacterImplicit(def)
-  Integer DefToIntegerImplicit(def)
-  Long DefToLongImplicit(def)
-  Float DefToFloatImplicit(def)
-  Double DefToDoubleImplicit(def)
-  byte DefTobyteExplicit(def)
-  short DefToshortExplicit(def)
-  char DefTocharExplicit(def)
-  int DefTointExplicit(def)
-  long DefTolongExplicit(def)
-  float DefTofloatExplicit(def)
-  double DefTodoubleExplicit(def)
-  Byte DefToByteExplicit(def)
-  Short DefToShortExplicit(def)
-  Character DefToCharacterExplicit(def)
-  Integer DefToIntegerExplicit(def)
-  Long DefToLongExplicit(def)
-  Float DefToFloatExplicit(def)
-  Double DefToDoubleExplicit(def)
-}
-
 # for testing.
 # currently FeatureTest exposes overloaded constructor, field load store, and overloaded static methods
 class FeatureTest -> org.elasticsearch.painless.FeatureTest extends Object {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java
index 2fb676bf299..2a8f3674ac1 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java
@@ -96,15 +96,15 @@ public class BasicExpressionTests extends ScriptTestCase {
     }
 
     /**
-     * Test boxed objects in various places
+     * Test boxed def objects in various places
      */
     public void testBoxing() {
         // return
         assertEquals(4, exec("return params.get(\"x\");", Collections.singletonMap("x", 4)));
         // assignment
-        assertEquals(4, exec("int y = (Integer)params.get(\"x\"); return y;", Collections.singletonMap("x", 4)));
+        assertEquals(4, exec("int y = params.get(\"x\"); return y;", Collections.singletonMap("x", 4)));
         // comparison
-        assertEquals(true, exec("return 5 > (Integer)params.get(\"x\");", Collections.singletonMap("x", 4)));
+        assertEquals(true, exec("return 5 > params.get(\"x\");", Collections.singletonMap("x", 4)));
     }
 
     public void testBool() {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/CompoundAssignmentTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/CompoundAssignmentTests.java
index d54b976d65d..03593116538 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/CompoundAssignmentTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/CompoundAssignmentTests.java
@@ -230,18 +230,18 @@ public class CompoundAssignmentTests extends ScriptTestCase {
         assertEquals(false, exec("boolean x = true; x &= false; return x;"));
         assertEquals(false, exec("boolean x = false; x &= true; return x;"));
         assertEquals(false, exec("boolean x = false; x &= false; return x;"));
-        assertEquals(true, exec("Boolean x = true; x &= true; return x;"));
-        assertEquals(false, exec("Boolean x = true; x &= false; return x;"));
-        assertEquals(false, exec("Boolean x = false; x &= true; return x;"));
-        assertEquals(false, exec("Boolean x = false; x &= false; return x;"));
+        assertEquals(true, exec("def x = true; x &= true; return x;"));
+        assertEquals(false, exec("def x = true; x &= false; return x;"));
+        assertEquals(false, exec("def x = false; x &= true; return x;"));
+        assertEquals(false, exec("def x = false; x &= false; return x;"));
         assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] &= true; return x[0];"));
         assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] &= false; return x[0];"));
         assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] &= true; return x[0];"));
         assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] &= false; return x[0];"));
-        assertEquals(true, exec("Boolean[] x = new Boolean[1]; x[0] = true; x[0] &= true; return x[0];"));
-        assertEquals(false, exec("Boolean[] x = new Boolean[1]; x[0] = true; x[0] &= false; return x[0];"));
-        assertEquals(false, exec("Boolean[] x = new Boolean[1]; x[0] = false; x[0] &= true; return x[0];"));
-        assertEquals(false, exec("Boolean[] x = new Boolean[1]; x[0] = false; x[0] &= false; return x[0];"));
+        assertEquals(true, exec("def[] x = new def[1]; x[0] = true; x[0] &= true; return x[0];"));
+        assertEquals(false, exec("def[] x = new def[1]; x[0] = true; x[0] &= false; return x[0];"));
+        assertEquals(false, exec("def[] x = new def[1]; x[0] = false; x[0] &= true; return x[0];"));
+        assertEquals(false, exec("def[] x = new def[1]; x[0] = false; x[0] &= false; return x[0];"));
 
         // byte
         assertEquals((byte) (13 & 14), exec("byte x = 13; x &= 14; return x;"));
@@ -261,18 +261,18 @@ public class CompoundAssignmentTests extends ScriptTestCase {
         assertEquals(true, exec("boolean x = true; x |= false; return x;"));
         assertEquals(true, exec("boolean x = false; x |= true; return x;"));
         assertEquals(false, exec("boolean x = false; x |= false; return x;"));
-        assertEquals(true, exec("Boolean x = true; x |= true; return x;"));
-        assertEquals(true, exec("Boolean x = true; x |= false; return x;"));
-        assertEquals(true, exec("Boolean x = false; x |= true; return x;"));
-        assertEquals(false, exec("Boolean x = false; x |= false; return x;"));
+        assertEquals(true, exec("def x = true; x |= true; return x;"));
+        assertEquals(true, exec("def x = true; x |= false; return x;"));
+        assertEquals(true, exec("def x = false; x |= true; return x;"));
+        assertEquals(false, exec("def x = false; x |= false; return x;"));
         assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] |= true; return x[0];"));
         assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] |= false; return x[0];"));
         assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] |= true; return x[0];"));
         assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] |= false; return x[0];"));
-        assertEquals(true, exec("Boolean[] x = new Boolean[1]; x[0] = true; x[0] |= true; return x[0];"));
-        assertEquals(true, exec("Boolean[] x = new Boolean[1]; x[0] = true; x[0] |= false; return x[0];"));
-        assertEquals(true, exec("Boolean[] x = new Boolean[1]; x[0] = false; x[0] |= true; return x[0];"));
-        assertEquals(false, exec("Boolean[] x = new Boolean[1]; x[0] = false; x[0] |= false; return x[0];"));
+        assertEquals(true, exec("def[] x = new def[1]; x[0] = true; x[0] |= true; return x[0];"));
+        assertEquals(true, exec("def[] x = new def[1]; x[0] = true; x[0] |= false; return x[0];"));
+        assertEquals(true, exec("def[] x = new def[1]; x[0] = false; x[0] |= true; return x[0];"));
+        assertEquals(false, exec("def[] x = new def[1]; x[0] = false; x[0] |= false; return x[0];"));
 
         // byte
         assertEquals((byte) (13 | 14), exec("byte x = 13; x |= 14; return x;"));
@@ -292,18 +292,18 @@ public class CompoundAssignmentTests extends ScriptTestCase {
         assertEquals(true, exec("boolean x = true; x ^= false; return x;"));
         assertEquals(true, exec("boolean x = false; x ^= true; return x;"));
         assertEquals(false, exec("boolean x = false; x ^= false; return x;"));
-        assertEquals(false, exec("Boolean x = true; x ^= true; return x;"));
-        assertEquals(true, exec("Boolean x = true; x ^= false; return x;"));
-        assertEquals(true, exec("Boolean x = false; x ^= true; return x;"));
-        assertEquals(false, exec("Boolean x = false; x ^= false; return x;"));
+        assertEquals(false, exec("def x = true; x ^= true; return x;"));
+        assertEquals(true, exec("def x = true; x ^= false; return x;"));
+        assertEquals(true, exec("def x = false; x ^= true; return x;"));
+        assertEquals(false, exec("def x = false; x ^= false; return x;"));
         assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] ^= true; return x[0];"));
         assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] ^= false; return x[0];"));
         assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] ^= true; return x[0];"));
         assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] ^= false; return x[0];"));
-        assertEquals(false, exec("Boolean[] x = new Boolean[1]; x[0] = true; x[0] ^= true; return x[0];"));
-        assertEquals(true, exec("Boolean[] x = new Boolean[1]; x[0] = true; x[0] ^= false; return x[0];"));
-        assertEquals(true, exec("Boolean[] x = new Boolean[1]; x[0] = false; x[0] ^= true; return x[0];"));
-        assertEquals(false, exec("Boolean[] x = new Boolean[1]; x[0] = false; x[0] ^= false; return x[0];"));
+        assertEquals(false, exec("def[] x = new def[1]; x[0] = true; x[0] ^= true; return x[0];"));
+        assertEquals(true, exec("def[] x = new def[1]; x[0] = true; x[0] ^= false; return x[0];"));
+        assertEquals(true, exec("def[] x = new def[1]; x[0] = false; x[0] ^= true; return x[0];"));
+        assertEquals(false, exec("def[] x = new def[1]; x[0] = false; x[0] ^= false; return x[0];"));
 
         // byte
         assertEquals((byte) (13 ^ 14), exec("byte x = 13; x ^= 14; return x;"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ConditionalTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ConditionalTests.java
index 859825f129a..bfcadef3b84 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ConditionalTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ConditionalTests.java
@@ -65,9 +65,6 @@ public class ConditionalTests extends ScriptTestCase {
 
     public void testPromotion() {
         assertEquals(false, exec("boolean x = false; boolean y = true; return (x ? 2 : 4.0F) == (y ? 2 : 4.0F);"));
-        assertEquals(false, exec("boolean x = false; boolean y = true; return (x ? 2 : 4.0F) == (y ? new Long(2) : new Float(4.0F));"));
-        assertEquals(false, exec("boolean x = false; boolean y = true; " +
-                "return (x ? new HashMap() : new ArrayList()) == (y ? new Long(2) : new Float(4.0F));"));
         assertEquals(false, exec("boolean x = false; boolean y = true; return (x ? 2 : 4.0F) == (y ? new HashMap() : new ArrayList());"));
     }
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java
index 8043d9da915..bf7d41dfa62 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java
@@ -94,7 +94,7 @@ public class EqualsTests extends ScriptTestCase {
     }
 
     public void testEquals() {
-        assertEquals(true, exec("return new Long(3) == new Long(3);"));
+        assertEquals(true, exec("return Long.valueOf(3) == 3L;"));
         assertEquals(false, exec("return new Long(3) === new Long(3);"));
         assertEquals(true, exec("Integer x = new Integer(3); Object y = x; return x == y;"));
         assertEquals(true, exec("Integer x = new Integer(3); Object y = x; return x === y;"));
@@ -143,11 +143,11 @@ public class EqualsTests extends ScriptTestCase {
     public void testBranchNotEquals() {
         assertEquals(1, exec("Character a = (char)'a'; Character b = (char)'b'; if (a != b) return 1; else return 0;"));
         assertEquals(0, exec("Character a = (char)'a'; Character b = (char)'a'; if (a != b) return 1; else return 0;"));
-        assertEquals(1, exec("Integer a = new Integer(1); Integer b = 1; if (a !== b) return 1; else return 0;"));
-        assertEquals(1, exec("Character a = (char)'a'; Character b = new Character((char)'a'); if (a !== b) return 1; else return 0;"));
-        assertEquals(0, exec("Character a = (char)'a'; Object b = a; if (a !== b) return 1; else return 0;"));
-        assertEquals(0, exec("Integer a = 1; Number b = a; Number c = a; if (c !== b) return 1; else return 0;"));
-        assertEquals(1, exec("Integer a = 1; Character b = (char)'a'; if (a !== (Object)b) return 1; else return 0;"));
+        assertEquals(1, exec("def a = 1; def b = 1; if (a !== b) return 1; else return 0;"));
+        assertEquals(1, exec("def a = (char)'a'; Character b = new Character((char)'a'); if (a !== b) return 1; else return 0;"));
+        assertEquals(0, exec("def a = (char)'a'; Object b = a; if (a !== b) return 1; else return 0;"));
+        assertEquals(0, exec("def a = 1; Number b = a; Number c = a; if (c !== b) return 1; else return 0;"));
+        assertEquals(1, exec("def a = 1; Character b = (char)'a'; if (a !== (Object)b) return 1; else return 0;"));
     }
 
     public void testRightHandNull() {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
index e61541bf371..5cd3c64925c 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
@@ -70,13 +70,13 @@ public class StringTests extends ScriptTestCase {
     public void testAppendMultiple() {
         assertEquals("cat" + true + "abc" + null, exec("String s = \"cat\"; return s + true + 'abc' + null;"));
     }
-    
+
     public void testAppendMany() {
         for (int i = MAX_INDY_STRING_CONCAT_ARGS - 5; i < MAX_INDY_STRING_CONCAT_ARGS + 5; i++) {
             doTestAppendMany(i);
         }
     }
-    
+
     private void doTestAppendMany(int count) {
         StringBuilder script = new StringBuilder("String s = \"cat\"; return s");
         StringBuilder result = new StringBuilder("cat");
@@ -90,11 +90,11 @@ public class StringTests extends ScriptTestCase {
                 Debugger.toString(s).contains(String.format(Locale.ROOT, "LDC \"%03d\"", count/2)));
         assertEquals(result.toString(), exec(s));
     }
-    
+
     public void testNestedConcats() {
         assertEquals("foo1010foo", exec("String s = 'foo'; String x = '10'; return s + Integer.parseInt(x + x) + s;"));
     }
-    
+
     public void testStringAPI() {
         assertEquals("", exec("return new String();"));
         assertEquals('x', exec("String s = \"x\"; return s.charAt(0);"));
@@ -189,41 +189,5 @@ public class StringTests extends ScriptTestCase {
         } catch (final ClassCastException cce) {
             assertTrue(cce.getMessage().contains("Cannot cast [String] with length greater than one to [char]."));
         }
-
-        assertEquals('c', exec("return (Character)\"c\""));
-        assertEquals('c', exec("return (Character)'c'"));
-        assertEquals("c", exec("return (String)(Character)\"c\""));
-        assertEquals("c", exec("return (String)(Character)'c'"));
-
-        assertEquals('c', exec("String s = \"c\"; (Character)s"));
-        assertEquals('c', exec("String s = 'c'; (Character)s"));
-
-        try {
-            assertEquals("cc", exec("return (String)(Character)\"cc\""));
-            fail();
-        } catch (final ClassCastException ise) {
-            assertTrue(ise.getMessage().contains("Cannot cast [String] with length greater than one to [Character]."));
-        }
-
-        try {
-            assertEquals("cc", exec("return (String)(Character)'cc'"));
-            fail();
-        } catch (final ClassCastException ise) {
-            assertTrue(ise.getMessage().contains("Cannot cast [String] with length greater than one to [Character]."));
-        }
-
-        try {
-            assertEquals('c', exec("String s = \"cc\"; (Character)s"));
-            fail();
-        } catch (final ClassCastException cce) {
-            assertTrue(cce.getMessage().contains("Cannot cast [String] with length greater than one to [Character]."));
-        }
-
-        try {
-            assertEquals('c', exec("String s = 'cc'; (Character)s"));
-            fail();
-        } catch (final ClassCastException cce) {
-            assertTrue(cce.getMessage().contains("Cannot cast [String] with length greater than one to [Character]."));
-        }
     }
 }

From 82aed083016d64a7856f3e5a0d3aae52f8debbe0 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Fri, 20 May 2016 10:48:41 +0200
Subject: [PATCH 0366/1311] Restore constants as static final (first step)

---
 .../painless/AnalyzerCaster.java              | 96 +++++++++----------
 .../elasticsearch/painless/Definition.java    | 32 ++++++-
 .../elasticsearch/painless/node/EBinary.java  |  6 +-
 .../elasticsearch/painless/node/EBool.java    |  6 +-
 .../elasticsearch/painless/node/EBoolean.java |  2 +-
 .../elasticsearch/painless/node/EChain.java   |  6 +-
 .../elasticsearch/painless/node/EComp.java    | 16 ++--
 .../painless/node/EConditional.java           |  2 +-
 .../painless/node/EConstant.java              | 18 ++--
 .../elasticsearch/painless/node/EDecimal.java |  4 +-
 .../elasticsearch/painless/node/ENull.java    |  2 +-
 .../elasticsearch/painless/node/ENumeric.java | 14 +--
 .../elasticsearch/painless/node/EUnary.java   |  4 +-
 .../painless/node/LArrayLength.java           |  2 +-
 .../elasticsearch/painless/node/LBrace.java   |  2 +-
 .../painless/node/LDefArray.java              |  6 +-
 .../elasticsearch/painless/node/LDefCall.java |  4 +-
 .../painless/node/LDefField.java              |  6 +-
 .../painless/node/LListShortcut.java          |  2 +-
 .../painless/node/LNewArray.java              |  2 +-
 .../elasticsearch/painless/node/LString.java  |  2 +-
 .../org/elasticsearch/painless/node/SDo.java  |  2 +-
 .../painless/node/SExpression.java            |  2 +-
 .../org/elasticsearch/painless/node/SFor.java |  2 +-
 .../elasticsearch/painless/node/SIfElse.java  |  2 +-
 .../elasticsearch/painless/node/SReturn.java  |  2 +-
 .../elasticsearch/painless/node/SThrow.java   |  2 +-
 .../elasticsearch/painless/node/SWhile.java   |  2 +-
 28 files changed, 137 insertions(+), 111 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
index d6f099e47f9..608f947016f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
@@ -119,18 +119,18 @@ public final class AnalyzerCaster {
         final Sort sort = from.sort;
 
         if (sort == Sort.DEF) {
-            return definition.getType("def");
+            return Definition.defType;
         } else if ((sort == Sort.DOUBLE || sort == Sort.DOUBLE_OBJ) && decimal) {
-            return primitive ? definition.getType("double") : definition.getType("Double");
+            return primitive ? Definition.doubleType : Definition.doubleobjType;
         } else if ((sort == Sort.FLOAT || sort == Sort.FLOAT_OBJ) && decimal) {
-            return primitive ? definition.getType("float") : definition.getType("Float");
+            return primitive ? Definition.floatType : Definition.floatobjType;
         } else if (sort == Sort.LONG || sort == Sort.LONG_OBJ) {
-            return primitive ? definition.getType("long") : definition.getType("Long");
+            return primitive ? Definition.longType : Definition.longobjType;
         } else if (sort == Sort.INT   || sort == Sort.INT_OBJ   ||
                    sort == Sort.CHAR  || sort == Sort.CHAR_OBJ  ||
                    sort == Sort.SHORT || sort == Sort.SHORT_OBJ ||
                    sort == Sort.BYTE  || sort == Sort.BYTE_OBJ) {
-            return primitive ? definition.getType("int") : definition.getType("Integer");
+            return primitive ? Definition.intType : Definition.intobjType;
         }
 
         return null;
@@ -142,21 +142,21 @@ public final class AnalyzerCaster {
         final Sort sort1 = from1.sort;
 
         if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return definition.getType("def");
+            return Definition.defType;
         }
 
         if (decimal) {
             if (sort0 == Sort.DOUBLE || sort0 == Sort.DOUBLE_OBJ ||
                 sort1 == Sort.DOUBLE || sort1 == Sort.DOUBLE_OBJ) {
-                return primitive ? definition.getType("double") : definition.getType("Double");
+                return primitive ? Definition.doubleType : Definition.doubleobjType;
             } else if (sort0 == Sort.FLOAT || sort0 == Sort.FLOAT_OBJ || sort1 == Sort.FLOAT || sort1 == Sort.FLOAT_OBJ) {
-                return primitive ? definition.getType("float") : definition.getType("Float");
+                return primitive ? Definition.floatType : Definition.floatobjType;
             }
         }
 
         if (sort0 == Sort.LONG || sort0 == Sort.LONG_OBJ ||
             sort1 == Sort.LONG || sort1 == Sort.LONG_OBJ) {
-            return primitive ? definition.getType("long") : definition.getType("Long");
+            return primitive ? Definition.longType : Definition.longobjType;
         } else if (sort0 == Sort.INT   || sort0 == Sort.INT_OBJ   ||
                    sort1 == Sort.INT   || sort1 == Sort.INT_OBJ   ||
                    sort0 == Sort.CHAR  || sort0 == Sort.CHAR_OBJ  ||
@@ -165,7 +165,7 @@ public final class AnalyzerCaster {
                    sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ ||
                    sort0 == Sort.BYTE  || sort0 == Sort.BYTE_OBJ  ||
                    sort1 == Sort.BYTE  || sort1 == Sort.BYTE_OBJ) {
-            return primitive ? definition.getType("int") : definition.getType("Integer");
+            return primitive ? Definition.intType : Definition.intobjType;
         }
 
         return null;
@@ -176,7 +176,7 @@ public final class AnalyzerCaster {
         final Sort sort1 = from1.sort;
 
         if (sort0 == Sort.STRING || sort1 == Sort.STRING) {
-            return definition.getType("String");
+            return Definition.stringType;
         }
 
         return promoteNumeric(definition, from0, from1, true, true);
@@ -187,7 +187,7 @@ public final class AnalyzerCaster {
         final Sort sort1 = from1.sort;
 
         if (sort0.bool || sort1.bool) {
-            return definition.getType("boolean");
+            return Definition.booleanType;
         }
 
         return promoteNumeric(definition, from0, from1, false, true);
@@ -198,20 +198,20 @@ public final class AnalyzerCaster {
         final Sort sort1 = from1.sort;
 
         if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return definition.getType("def");
+            return Definition.defType;
         }
 
         final boolean primitive = sort0.primitive && sort1.primitive;
 
         if (sort0.bool && sort1.bool) {
-            return primitive ? definition.getType("boolean") : definition.getType("Boolean");
+            return primitive ? Definition.booleanType : Definition.booleanobjType;
         }
 
         if (sort0.numeric && sort1.numeric) {
             return promoteNumeric(definition, from0, from1, true, primitive);
         }
 
-        return definition.getType("Object");
+        return Definition.objectType;
     }
 
     public static Type promoteReference(final Definition definition, final Type from0, final Type from1) {
@@ -219,12 +219,12 @@ public final class AnalyzerCaster {
         final Sort sort1 = from1.sort;
 
         if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return definition.getType("def");
+            return Definition.defType;
         }
 
         if (sort0.primitive && sort1.primitive) {
             if (sort0.bool && sort1.bool) {
-                return definition.getType("boolean");
+                return Definition.booleanType;
             }
 
             if (sort0.numeric && sort1.numeric) {
@@ -232,7 +232,7 @@ public final class AnalyzerCaster {
             }
         }
 
-        return definition.getType("Object");
+        return Definition.objectType;
     }
 
     public static Type promoteConditional(final Definition definition,
@@ -245,48 +245,48 @@ public final class AnalyzerCaster {
         final Sort sort1 = from1.sort;
 
         if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return definition.getType("def");
+            return Definition.defType;
         }
 
         final boolean primitive = sort0.primitive && sort1.primitive;
 
         if (sort0.bool && sort1.bool) {
-            return primitive ? definition.getType("boolean") : definition.getType("Boolean");
+            return primitive ? Definition.booleanType : Definition.booleanobjType;
         }
 
         if (sort0.numeric && sort1.numeric) {
             if (sort0 == Sort.DOUBLE || sort0 == Sort.DOUBLE_OBJ || sort1 == Sort.DOUBLE || sort1 == Sort.DOUBLE_OBJ) {
-                return primitive ? definition.getType("double") : definition.getType("Double");
+                return primitive ? Definition.doubleType : Definition.doubleobjType;
             } else if (sort0 == Sort.FLOAT || sort0 == Sort.FLOAT_OBJ || sort1 == Sort.FLOAT || sort1 == Sort.FLOAT_OBJ) {
-                return primitive ? definition.getType("float") : definition.getType("Float");
+                return primitive ? Definition.floatType : Definition.floatobjType;
             } else if (sort0 == Sort.LONG || sort0 == Sort.LONG_OBJ || sort1 == Sort.LONG || sort1 == Sort.LONG_OBJ) {
-                return sort0.primitive && sort1.primitive ? definition.getType("long") : definition.getType("Long");
+                return sort0.primitive && sort1.primitive ? Definition.longType : Definition.longobjType;
             } else {
                 if (sort0 == Sort.BYTE || sort0 == Sort.BYTE_OBJ) {
                     if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
-                        return primitive ? definition.getType("byte") : definition.getType("Byte");
+                        return primitive ? Definition.byteType : Definition.byteobjType;
                     } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
                         if (const1 != null) {
                             final short constant = (short)const1;
 
                             if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? definition.getType("byte") : definition.getType("Byte");
+                                return primitive ? Definition.byteType : Definition.byteobjType;
                             }
                         }
 
-                        return primitive ? definition.getType("short") : definition.getType("Short");
+                        return primitive ? Definition.shortType : Definition.shortobjType;
                     } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
-                        return primitive ? definition.getType("int") : definition.getType("Integer");
+                        return primitive ? Definition.intType : Definition.intobjType;
                     } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
                         if (const1 != null) {
                             final int constant = (int)const1;
 
                             if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? definition.getType("byte") : definition.getType("Byte");
+                                return primitive ? Definition.byteType : Definition.byteobjType;
                             }
                         }
 
-                        return primitive ? definition.getType("int") : definition.getType("Integer");
+                        return primitive ? Definition.intType : Definition.intobjType;
                     }
                 } else if (sort0 == Sort.SHORT || sort0 == Sort.SHORT_OBJ) {
                     if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
@@ -294,43 +294,43 @@ public final class AnalyzerCaster {
                             final short constant = (short)const0;
 
                             if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? definition.getType("byte") : definition.getType("Byte");
+                                return primitive ? Definition.byteType : Definition.byteobjType;
                             }
                         }
 
-                        return primitive ? definition.getType("short") : definition.getType("Short");
+                        return primitive ? Definition.shortType : Definition.shortobjType;
                     } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
-                        return primitive ? definition.getType("short") : definition.getType("Short");
+                        return primitive ? Definition.shortType : Definition.shortobjType;
                     } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
-                        return primitive ? definition.getType("int") : definition.getType("Integer");
+                        return primitive ? Definition.intType : Definition.intobjType;
                     } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
                         if (const1 != null) {
                             final int constant = (int)const1;
 
                             if (constant <= Short.MAX_VALUE && constant >= Short.MIN_VALUE) {
-                                return primitive ? definition.getType("short") : definition.getType("Short");
+                                return primitive ? Definition.shortType : Definition.shortobjType;
                             }
                         }
 
-                        return primitive ? definition.getType("int") : definition.getType("Integer");
+                        return primitive ? Definition.intType : Definition.intobjType;
                     }
                 } else if (sort0 == Sort.CHAR || sort0 == Sort.CHAR_OBJ) {
                     if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
-                        return primitive ? definition.getType("int") : definition.getType("Integer");
+                        return primitive ? Definition.intType : Definition.intobjType;
                     } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
-                        return primitive ? definition.getType("int") : definition.getType("Integer");
+                        return primitive ? Definition.intType : Definition.intobjType;
                     } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
-                        return primitive ? definition.getType("char") : definition.getType("Character");
+                        return primitive ? Definition.charType : Definition.charobjType;
                     } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
                         if (const1 != null) {
                             final int constant = (int)const1;
 
                             if (constant <= Character.MAX_VALUE && constant >= Character.MIN_VALUE) {
-                                return primitive ? definition.getType("byte") : definition.getType("Byte");
+                                return primitive ? Definition.byteType : Definition.byteobjType;
                             }
                         }
 
-                        return primitive ? definition.getType("int") : definition.getType("Integer");
+                        return primitive ? Definition.intType : Definition.intobjType;
                     }
                 } else if (sort0 == Sort.INT || sort0 == Sort.INT_OBJ) {
                     if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
@@ -338,33 +338,33 @@ public final class AnalyzerCaster {
                             final int constant = (int)const0;
 
                             if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? definition.getType("byte") : definition.getType("Byte");
+                                return primitive ? Definition.byteType : Definition.byteobjType;
                             }
                         }
 
-                        return primitive ? definition.getType("int") : definition.getType("Integer");
+                        return primitive ? Definition.intType : Definition.intobjType;
                     } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
                         if (const0 != null) {
                             final int constant = (int)const0;
 
                             if (constant <= Short.MAX_VALUE && constant >= Short.MIN_VALUE) {
-                                return primitive ? definition.getType("byte") : definition.getType("Byte");
+                                return primitive ? Definition.byteType : Definition.byteobjType;
                             }
                         }
 
-                        return primitive ? definition.getType("int") : definition.getType("Integer");
+                        return primitive ? Definition.intType : Definition.intobjType;
                     } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
                         if (const0 != null) {
                             final int constant = (int)const0;
 
                             if (constant <= Character.MAX_VALUE && constant >= Character.MIN_VALUE) {
-                                return primitive ? definition.getType("byte") : definition.getType("Byte");
+                                return primitive ? Definition.byteType : Definition.byteobjType;
                             }
                         }
 
-                        return primitive ? definition.getType("int") : definition.getType("Integer");
+                        return primitive ? Definition.intType : Definition.intobjType;
                     } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
-                        return primitive ? definition.getType("int") : definition.getType("Integer");
+                        return primitive ? Definition.intType : Definition.intobjType;
                     }
                 }
             }
@@ -374,7 +374,7 @@ public final class AnalyzerCaster {
         //       to calculate the highest upper bound for the two types and return that.
         //       However, for now we just return objectType that may require an extra cast.
 
-        return definition.getType("Object");
+        return Definition.objectType;
     }
 
     private AnalyzerCaster() {}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index bde6063cfe1..827c268793a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -37,12 +37,38 @@ import java.util.Objects;
  * methods and fields during at both compile-time and runtime.
  */
 public final class Definition {
+    
+    private static final String DEFINITION_FILE = "definition.txt";
 
     /**
      * The default language API to be used with Painless.  The second construction is used
      * to finalize all the variables, so there is no mistake of modification afterwards.
      */
-    static Definition INSTANCE = new Definition(new Definition());
+    public static final Definition INSTANCE = new Definition(new Definition());
+    
+    /** Some native types as constants: */
+    public static final Type booleanType = INSTANCE.getType("boolean");
+    public static final Type objectType = INSTANCE.getType("Object");
+    public static final Type defType = INSTANCE.getType("def");
+    public static final Type booleanobjType = INSTANCE.getType("Boolean");
+    public static final Type byteType = INSTANCE.getType("byte");
+    public static final Type shortType = INSTANCE.getType("short");
+    public static final Type intType = INSTANCE.getType("int");
+    public static final Type charType = INSTANCE.getType("char");
+    public static final Type longType = INSTANCE.getType("long");
+    public static final Type floatType = INSTANCE.getType("float");
+    public static final Type doubleType = INSTANCE.getType("double");
+    public static final Type numberType = INSTANCE.getType("Number");
+    public static final Type byteobjType = INSTANCE.getType("Byte");
+    public static final Type shortobjType = INSTANCE.getType("Short");
+    public static final Type charobjType = INSTANCE.getType("Character");
+    public static final Type intobjType = INSTANCE.getType("Integer");
+    public static final Type longobjType = INSTANCE.getType("Long");
+    public static final Type floatobjType = INSTANCE.getType("Float");
+    public static final Type doubleobjType = INSTANCE.getType("Double");
+    public static final Type stringType = INSTANCE.getType("String");
+    public static final Type exceptionType = INSTANCE.getType("Exception");
+    public static final Type voidType = INSTANCE.getType("void");
 
     public enum Sort {
         VOID(       void.class      , 0 , true  , false , false , false ),
@@ -411,7 +437,7 @@ public final class Definition {
         final Map<String,List<String>> hierarchy = new HashMap<>();
         int currentLine = -1;
         try {
-            try (InputStream stream = Definition.class.getResourceAsStream("definition.txt");
+            try (InputStream stream = Definition.class.getResourceAsStream(DEFINITION_FILE);
                     LineNumberReader reader = new LineNumberReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
                 String line = null;
                 while ((line = reader.readLine()) != null) {
@@ -477,7 +503,7 @@ public final class Definition {
     private void addElements() {
         int currentLine = -1;
         try {
-            try (InputStream stream = Definition.class.getResourceAsStream("definition.txt");
+            try (InputStream stream = Definition.class.getResourceAsStream(DEFINITION_FILE);
                  LineNumberReader reader = new LineNumberReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
                 String line = null;
                 String currentClass = null;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index f22b7394ca0..92ccf60a200 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -284,7 +284,7 @@ public final class EBinary extends AExpression {
         }
 
         left.expected = promote;
-        right.expected = definition.getType("int");
+        right.expected = Definition.intType;
         right.explicit = true;
 
         left = left.cast(settings, definition, variables);
@@ -317,7 +317,7 @@ public final class EBinary extends AExpression {
         }
 
         left.expected = promote;
-        right.expected = definition.getType("int");
+        right.expected = Definition.intType;
         right.explicit = true;
 
         left = left.cast(settings, definition, variables);
@@ -350,7 +350,7 @@ public final class EBinary extends AExpression {
         }
 
         left.expected = promote;
-        right.expected = definition.getType("int");
+        right.expected = Definition.intType;
         right.explicit = true;
 
         left = left.cast(settings, definition, variables);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
index 4a6967c5285..1cd2564b0e1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
@@ -45,11 +45,11 @@ public final class EBool extends AExpression {
 
     @Override
     void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        left.expected = definition.getType("boolean");
+        left.expected = Definition.booleanType;
         left.analyze(settings, definition, variables);
         left = left.cast(settings, definition, variables);
 
-        right.expected = definition.getType("boolean");
+        right.expected = Definition.booleanType;
         right.analyze(settings, definition, variables);
         right = right.cast(settings, definition, variables);
 
@@ -63,7 +63,7 @@ public final class EBool extends AExpression {
             }
         }
 
-        actual = definition.getType("boolean");
+        actual = Definition.booleanType;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
index 631b47db5d1..24f1ac899e8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
@@ -37,7 +37,7 @@ public final class EBoolean extends AExpression {
 
     @Override
     void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        actual = definition.getType("boolean");
+        actual = Definition.booleanType;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index 933295f9beb..a3c0c656fda 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -199,7 +199,7 @@ public final class EChain extends AExpression {
 
             expression.expected = expression.actual;
         } else if (operation == Operation.LSH || operation == Operation.RSH || operation == Operation.USH) {
-            expression.expected = definition.getType("int");
+            expression.expected = Definition.intType;
             expression.explicit = true;
         } else {
             expression.expected = promote;
@@ -211,7 +211,7 @@ public final class EChain extends AExpression {
         back = AnalyzerCaster.getLegalCast(definition, location, promote, last.after, true);
 
         this.statement = true;
-        this.actual = read ? last.after : definition.getType("void");
+        this.actual = read ? last.after : Definition.voidType;
     }
 
     private void analyzeWrite(final CompilerSettings settings, final Definition definition, final Variables variables) {
@@ -231,7 +231,7 @@ public final class EChain extends AExpression {
         expression = expression.cast(settings, definition, variables);
 
         this.statement = true;
-        this.actual = read ? last.after : definition.getType("void");
+        this.actual = read ? last.after : Definition.voidType;
     }
 
     private void analyzeRead() {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index a65e82e02fa..0d781abf9b7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -119,7 +119,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = definition.getType("boolean");
+        actual = Definition.booleanType;
     }
 
     private void analyzeEqR(final CompilerSettings settings, final Definition definition, final Variables variables) {
@@ -161,7 +161,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = definition.getType("boolean");
+        actual = Definition.booleanType;
     }
 
     private void analyzeNE(final CompilerSettings settings, final Definition definition, final Variables variables) {
@@ -207,7 +207,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = definition.getType("boolean");
+        actual = Definition.booleanType;
     }
 
     private void analyzeNER(final CompilerSettings settings, final Definition definition, final Variables variables) {
@@ -249,7 +249,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = definition.getType("boolean");
+        actual = Definition.booleanType;
     }
 
     private void analyzeGTE(final CompilerSettings settings, final Definition definition, final Variables variables) {
@@ -285,7 +285,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = definition.getType("boolean");
+        actual = Definition.booleanType;
     }
 
     private void analyzeGT(final CompilerSettings settings, final Definition definition, final Variables variables) {
@@ -321,7 +321,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = definition.getType("boolean");
+        actual = Definition.booleanType;
     }
 
     private void analyzeLTE(final CompilerSettings settings, final Definition definition, final Variables variables) {
@@ -357,7 +357,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = definition.getType("boolean");
+        actual = Definition.booleanType;
     }
 
     private void analyzeLT(final CompilerSettings settings, final Definition definition, final Variables variables) {
@@ -393,7 +393,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = definition.getType("boolean");
+        actual = Definition.booleanType;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
index 45976c13414..c1b37dfeee0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
@@ -47,7 +47,7 @@ public final class EConditional extends AExpression {
 
     @Override
     void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        condition.expected = definition.getType("boolean");
+        condition.expected = Definition.booleanType;
         condition.analyze(settings, definition, variables);
         condition = condition.cast(settings, definition, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
index f0ee3a8de51..cbf547a0417 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
@@ -40,23 +40,23 @@ final class EConstant extends AExpression {
     @Override
     void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
         if (constant instanceof String) {
-            actual = definition.getType("String");
+            actual = Definition.stringType;
         } else if (constant instanceof Double) {
-            actual = definition.getType("double");
+            actual = Definition.doubleType;
         } else if (constant instanceof Float) {
-            actual = definition.getType("float");
+            actual = Definition.floatType;
         } else if (constant instanceof Long) {
-            actual = definition.getType("long");
+            actual = Definition.longType;
         } else if (constant instanceof Integer) {
-            actual = definition.getType("int");
+            actual = Definition.intType;
         } else if (constant instanceof Character) {
-            actual = definition.getType("char");
+            actual = Definition.charType;
         } else if (constant instanceof Short) {
-            actual = definition.getType("short");
+            actual = Definition.shortType;
         } else if (constant instanceof Byte) {
-            actual = definition.getType("byte");
+            actual = Definition.byteType;
         } else if (constant instanceof Boolean) {
-            actual = definition.getType("boolean");
+            actual = Definition.booleanType;
         } else {
             throw new IllegalStateException(error("Illegal tree structure."));
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
index 9019d370895..820c3b3926e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
@@ -42,14 +42,14 @@ public final class EDecimal extends AExpression {
         if (value.endsWith("f") || value.endsWith("F")) {
             try {
                 constant = Float.parseFloat(value.substring(0, value.length() - 1));
-                actual = definition.getType("float");
+                actual = Definition.floatType;
             } catch (final NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid float constant [" + value + "]."));
             }
         } else {
             try {
                 constant = Double.parseDouble(value);
-                actual = definition.getType("double");
+                actual = Definition.doubleType;
             } catch (final NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid double constant [" + value + "]."));
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
index 7098e583c5a..3e003fefe5c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
@@ -45,7 +45,7 @@ public final class ENull extends AExpression {
 
             actual = expected;
         } else {
-            actual = definition.getType("Object");
+            actual = Definition.objectType;
         }
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
index 489334ae24e..e8cf202d91d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
@@ -49,7 +49,7 @@ public final class ENumeric extends AExpression {
 
             try {
                 constant = Double.parseDouble(value.substring(0, value.length() - 1));
-                actual = definition.getType("double");
+                actual = Definition.doubleType;
             } catch (final NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid double constant [" + value + "]."));
             }
@@ -60,14 +60,14 @@ public final class ENumeric extends AExpression {
 
             try {
                 constant = Float.parseFloat(value.substring(0, value.length() - 1));
-                actual = definition.getType("float");
+                actual = Definition.floatType;
             } catch (final NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid float constant [" + value + "]."));
             }
         } else if (value.endsWith("l") || value.endsWith("L")) {
             try {
                 constant = Long.parseLong(value.substring(0, value.length() - 1), radix);
-                actual = definition.getType("long");
+                actual = Definition.longType;
             } catch (final NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid long constant [" + value + "]."));
             }
@@ -78,16 +78,16 @@ public final class ENumeric extends AExpression {
 
                 if (sort == Sort.BYTE && integer >= Byte.MIN_VALUE && integer <= Byte.MAX_VALUE) {
                     constant = (byte)integer;
-                    actual = definition.getType("byte");
+                    actual = Definition.byteType;
                 } else if (sort == Sort.CHAR && integer >= Character.MIN_VALUE && integer <= Character.MAX_VALUE) {
                     constant = (char)integer;
-                    actual = definition.getType("char");
+                    actual = Definition.charType;
                 } else if (sort == Sort.SHORT && integer >= Short.MIN_VALUE && integer <= Short.MAX_VALUE) {
                     constant = (short)integer;
-                    actual = definition.getType("short");
+                    actual = Definition.shortType;
                 } else {
                     constant = integer;
-                    actual = definition.getType("int");
+                    actual = Definition.intType;
                 }
             } catch (final NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid int constant [" + value + "]."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index 7e52790fa50..11694c344f7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -63,7 +63,7 @@ public final class EUnary extends AExpression {
     }
 
     void analyzeNot(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        child.expected = definition.getType("boolean");
+        child.expected = Definition.booleanType;
         child.analyze(settings, definition, variables);
         child = child.cast(settings, definition, variables);
 
@@ -71,7 +71,7 @@ public final class EUnary extends AExpression {
             constant = !(boolean)child.constant;
         }
 
-        actual = definition.getType("boolean");
+        actual = Definition.booleanType;
     }
 
     void analyzeBWNot(final CompilerSettings settings, final Definition definition, final Variables variables) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
index d41128f60bb..a8047047f93 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
@@ -46,7 +46,7 @@ public final class LArrayLength extends ALink {
                 throw new IllegalArgumentException(error("Cannot write to read-only array field [length]."));
             }
 
-            after = definition.getType("int");
+            after = Definition.intType;
         } else {
             throw new IllegalArgumentException(error("Illegal field access [" + value + "]."));
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
index 1e38161ae87..fb1b11d1d07 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
@@ -50,7 +50,7 @@ public final class LBrace extends ALink {
         final Sort sort = before.sort;
 
         if (sort == Sort.ARRAY) {
-            index.expected = definition.getType("int");
+            index.expected = Definition.intType;
             index.analyze(settings, definition, variables);
             index = index.cast(settings, definition, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index 68f60ef6f8f..eae33809e70 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -47,7 +47,7 @@ final class LDefArray extends ALink implements IDefLink {
         index.expected = index.actual;
         index = index.cast(settings, definition, variables);
 
-        after = definition.getType("def");
+        after = Definition.defType;
 
         return this;
     }
@@ -59,13 +59,13 @@ final class LDefArray extends ALink implements IDefLink {
 
     @Override
     void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
-        final String desc = Type.getMethodDescriptor(after.type, definition.getType("def").type, index.actual.type);
+        final String desc = Type.getMethodDescriptor(after.type, Definition.defType.type, index.actual.type);
         adapter.invokeDynamic("arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_LOAD);
     }
 
     @Override
     void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
-        final String desc = Type.getMethodDescriptor(definition.getType("void").type, definition.getType("def").type,
+        final String desc = Type.getMethodDescriptor(Definition.voidType.type, Definition.defType.type,
             index.actual.type, after.type);
         adapter.invokeDynamic("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_STORE);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index 1285d548163..360f78eb4fd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -55,7 +55,7 @@ final class LDefCall extends ALink implements IDefLink {
         }
 
         statement = true;
-        after = definition.getType("def");
+        after = Definition.defType;
 
         return this;
     }
@@ -71,7 +71,7 @@ final class LDefCall extends ALink implements IDefLink {
 
         signature.append('(');
         // first parameter is the receiver, we never know its type: always Object
-        signature.append(definition.getType("def").type.getDescriptor());
+        signature.append(Definition.defType.type.getDescriptor());
 
         // TODO: remove our explicit conversions and feed more type information for return value,
         // it can avoid some unnecessary boxing etc.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index 9dbe1cc3367..af20e152c70 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -44,7 +44,7 @@ final class LDefField extends ALink implements IDefLink {
 
     @Override
     ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        after = definition.getType("def");
+        after = Definition.defType;
 
         return this;
     }
@@ -56,13 +56,13 @@ final class LDefField extends ALink implements IDefLink {
 
     @Override
     void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
-        final String desc = Type.getMethodDescriptor(after.type, definition.getType("def").type);
+        final String desc = Type.getMethodDescriptor(after.type, Definition.defType.type);
         adapter.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.LOAD);
     }
 
     @Override
     void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
-        final String desc = Type.getMethodDescriptor(definition.getType("void").type, definition.getType("def").type, after.type);
+        final String desc = Type.getMethodDescriptor(Definition.voidType.type, Definition.defType.type, after.type);
         adapter.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.STORE);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
index f1649f3bf37..cc20fc6dec4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
@@ -61,7 +61,7 @@ final class LListShortcut extends ALink {
         }
 
         if ((load || store) && (!load || getter != null) && (!store || setter != null)) {
-            index.expected = definition.getType("int");
+            index.expected = Definition.intType;
             index.analyze(settings, definition, variables);
             index = index.cast(settings, definition, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
index acafc44ede1..ffc90990cd6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
@@ -63,7 +63,7 @@ public final class LNewArray extends ALink {
         for (int argument = 0; argument < arguments.size(); ++argument) {
             final AExpression expression = arguments.get(argument);
 
-            expression.expected = definition.getType("int");
+            expression.expected = Definition.intType;
             expression.analyze(settings, definition, variables);
             arguments.set(argument, expression.cast(settings, definition, variables));
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
index 6b91cf5e87a..91f379cce76 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
@@ -45,7 +45,7 @@ public final class LString extends ALink {
             throw new IllegalArgumentException(error("Must read String constant [" + string + "]."));
         }
 
-        after = definition.getType("String");
+        after = Definition.stringType;
 
         return this;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
index 7a79fe9c02c..99f8b634d49 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -53,7 +53,7 @@ public final class SDo extends AStatement {
             throw new IllegalArgumentException(error("Extraneous do while loop."));
         }
 
-        condition.expected = definition.getType("boolean");
+        condition.expected = Definition.booleanType;
         condition.analyze(settings, definition, variables);
         condition = condition.cast(settings, definition, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index 39ab0c099a6..2ed0d10b930 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -49,7 +49,7 @@ public final class SExpression extends AStatement {
 
         final boolean rtn = lastSource && expression.actual.sort != Sort.VOID;
 
-        expression.expected = rtn ? definition.getType("Object") : expression.actual;
+        expression.expected = rtn ? Definition.objectType : expression.actual;
         expression = expression.cast(settings, definition, variables);
 
         methodEscape = rtn;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index 239e302c72d..ef97fbac06b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -70,7 +70,7 @@ public final class SFor extends AStatement {
 
         if (condition != null) {
 
-            condition.expected = definition.getType("boolean");
+            condition.expected = Definition.booleanType;
             condition.analyze(settings, definition, variables);
             condition = condition.cast(settings, definition, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
index 89dad5a96c4..34f4a80645c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
@@ -45,7 +45,7 @@ public final class SIfElse extends AStatement {
 
     @Override
     void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        condition.expected = definition.getType("boolean");
+        condition.expected = Definition.booleanType;
         condition.analyze(settings, definition, variables);
         condition = condition.cast(settings, definition, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
index 68084a9a70c..4442e5b9ee4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -39,7 +39,7 @@ public final class SReturn extends AStatement {
 
     @Override
     void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        expression.expected = definition.getType("Object");
+        expression.expected = Definition.objectType;
         expression.analyze(settings, definition, variables);
         expression = expression.cast(settings, definition, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
index b840fce3a83..99fb5c6a973 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
@@ -39,7 +39,7 @@ public final class SThrow extends AStatement {
 
     @Override
     void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        expression.expected = definition.getType("Exception");
+        expression.expected = Definition.exceptionType;
         expression.analyze(settings, definition, variables);
         expression = expression.cast(settings, definition, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
index 5cabb1b0f7c..83afe469320 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -44,7 +44,7 @@ public final class SWhile extends AStatement {
     void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
         variables.incrementScope();
 
-        condition.expected = definition.getType("boolean");
+        condition.expected = Definition.booleanType;
         condition.analyze(settings, definition, variables);
         condition = condition.cast(settings, definition, variables);
 

From 87ab39613d5d963232b1c94b223d9cf7f03aed79 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Fri, 20 May 2016 10:56:50 +0200
Subject: [PATCH 0367/1311] more constants

---
 .../elasticsearch/painless/Definition.java    | 22 +++++++++----------
 .../elasticsearch/painless/MethodWriter.java  | 22 +++++++++----------
 .../elasticsearch/painless/node/EComp.java    | 12 +++++-----
 .../elasticsearch/painless/node/EUnary.java   |  4 ++--
 4 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 827c268793a..6e38bf2b7a4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -47,28 +47,28 @@ public final class Definition {
     public static final Definition INSTANCE = new Definition(new Definition());
     
     /** Some native types as constants: */
+    public static final Type voidType = INSTANCE.getType("void");
     public static final Type booleanType = INSTANCE.getType("boolean");
-    public static final Type objectType = INSTANCE.getType("Object");
-    public static final Type defType = INSTANCE.getType("def");
     public static final Type booleanobjType = INSTANCE.getType("Boolean");
     public static final Type byteType = INSTANCE.getType("byte");
-    public static final Type shortType = INSTANCE.getType("short");
-    public static final Type intType = INSTANCE.getType("int");
-    public static final Type charType = INSTANCE.getType("char");
-    public static final Type longType = INSTANCE.getType("long");
-    public static final Type floatType = INSTANCE.getType("float");
-    public static final Type doubleType = INSTANCE.getType("double");
-    public static final Type numberType = INSTANCE.getType("Number");
     public static final Type byteobjType = INSTANCE.getType("Byte");
+    public static final Type shortType = INSTANCE.getType("short");
     public static final Type shortobjType = INSTANCE.getType("Short");
-    public static final Type charobjType = INSTANCE.getType("Character");
+    public static final Type intType = INSTANCE.getType("int");
     public static final Type intobjType = INSTANCE.getType("Integer");
+    public static final Type longType = INSTANCE.getType("long");
     public static final Type longobjType = INSTANCE.getType("Long");
+    public static final Type floatType = INSTANCE.getType("float");
     public static final Type floatobjType = INSTANCE.getType("Float");
+    public static final Type doubleType = INSTANCE.getType("double");
     public static final Type doubleobjType = INSTANCE.getType("Double");
+    public static final Type charType = INSTANCE.getType("char");
+    public static final Type charobjType = INSTANCE.getType("Character");
+    public static final Type objectType = INSTANCE.getType("Object");
+    public static final Type defType = INSTANCE.getType("def");
+    public static final Type defobjType = INSTANCE.getType("Def");
     public static final Type stringType = INSTANCE.getType("String");
     public static final Type exceptionType = INSTANCE.getType("Exception");
-    public static final Type voidType = INSTANCE.getType("void");
 
     public enum Sort {
         VOID(       void.class      , 0 , true  , false , false , false ),
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index cda8c23e3bf..ce1bb531655 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -214,17 +214,17 @@ public final class MethodWriter extends GeneratorAdapter {
         
         if (sort == Sort.DEF) {
             switch (operation) {
-                case MUL:   invokeStatic(definition.getType("Def").type, DEF_MUL_CALL); break;
-                case DIV:   invokeStatic(definition.getType("Def").type, DEF_DIV_CALL); break;
-                case REM:   invokeStatic(definition.getType("Def").type, DEF_REM_CALL); break;
-                case ADD:   invokeStatic(definition.getType("Def").type, DEF_ADD_CALL); break;
-                case SUB:   invokeStatic(definition.getType("Def").type, DEF_SUB_CALL); break;
-                case LSH:   invokeStatic(definition.getType("Def").type, DEF_LSH_CALL); break;
-                case USH:   invokeStatic(definition.getType("Def").type, DEF_RSH_CALL); break;
-                case RSH:   invokeStatic(definition.getType("Def").type, DEF_USH_CALL); break;
-                case BWAND: invokeStatic(definition.getType("Def").type, DEF_AND_CALL); break;
-                case XOR:   invokeStatic(definition.getType("Def").type, DEF_XOR_CALL); break;
-                case BWOR:  invokeStatic(definition.getType("Def").type, DEF_OR_CALL);  break;
+                case MUL:   invokeStatic(Definition.defobjType.type, DEF_MUL_CALL); break;
+                case DIV:   invokeStatic(Definition.defobjType.type, DEF_DIV_CALL); break;
+                case REM:   invokeStatic(Definition.defobjType.type, DEF_REM_CALL); break;
+                case ADD:   invokeStatic(Definition.defobjType.type, DEF_ADD_CALL); break;
+                case SUB:   invokeStatic(Definition.defobjType.type, DEF_SUB_CALL); break;
+                case LSH:   invokeStatic(Definition.defobjType.type, DEF_LSH_CALL); break;
+                case USH:   invokeStatic(Definition.defobjType.type, DEF_RSH_CALL); break;
+                case RSH:   invokeStatic(Definition.defobjType.type, DEF_USH_CALL); break;
+                case BWAND: invokeStatic(Definition.defobjType.type, DEF_AND_CALL); break;
+                case XOR:   invokeStatic(Definition.defobjType.type, DEF_XOR_CALL); break;
+                case BWOR:  invokeStatic(Definition.defobjType.type, DEF_OR_CALL);  break;
                 default:
                     throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index 0d781abf9b7..9d7862e404a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -456,7 +456,7 @@ public final class EComp extends AExpression {
                     if (right.isNull) {
                         adapter.ifNull(jump);
                     } else if (!left.isNull && operation == Operation.EQ) {
-                        adapter.invokeStatic(definition.getType("Def").type, DEF_EQ_CALL);
+                        adapter.invokeStatic(Definition.defobjType.type, DEF_EQ_CALL);
                     } else {
                         adapter.ifCmp(rtype, MethodWriter.EQ, jump);
                     }
@@ -464,19 +464,19 @@ public final class EComp extends AExpression {
                     if (right.isNull) {
                         adapter.ifNonNull(jump);
                     } else if (!left.isNull && operation == Operation.NE) {
-                        adapter.invokeStatic(definition.getType("Def").type, DEF_EQ_CALL);
+                        adapter.invokeStatic(Definition.defobjType.type, DEF_EQ_CALL);
                         adapter.ifZCmp(MethodWriter.EQ, jump);
                     } else {
                         adapter.ifCmp(rtype, MethodWriter.NE, jump);
                     }
                 } else if (lt) {
-                    adapter.invokeStatic(definition.getType("Def").type, DEF_LT_CALL);
+                    adapter.invokeStatic(Definition.defobjType.type, DEF_LT_CALL);
                 } else if (lte) {
-                    adapter.invokeStatic(definition.getType("Def").type, DEF_LTE_CALL);
+                    adapter.invokeStatic(Definition.defobjType.type, DEF_LTE_CALL);
                 } else if (gt) {
-                    adapter.invokeStatic(definition.getType("Def").type, DEF_GT_CALL);
+                    adapter.invokeStatic(Definition.defobjType.type, DEF_GT_CALL);
                 } else if (gte) {
-                    adapter.invokeStatic(definition.getType("Def").type, DEF_GTE_CALL);
+                    adapter.invokeStatic(Definition.defobjType.type, DEF_GTE_CALL);
                 } else {
                     throw new IllegalStateException(error("Illegal tree structure."));
                 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index 11694c344f7..627eb85e968 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -191,7 +191,7 @@ public final class EUnary extends AExpression {
 
             if (operation == Operation.BWNOT) {
                 if (sort == Sort.DEF) {
-                    adapter.invokeStatic(definition.getType("Def").type, DEF_NOT_CALL);
+                    adapter.invokeStatic(Definition.defobjType.type, DEF_NOT_CALL);
                 } else {
                     if (sort == Sort.INT) {
                         adapter.push(-1);
@@ -205,7 +205,7 @@ public final class EUnary extends AExpression {
                 }
             } else if (operation == Operation.SUB) {
                 if (sort == Sort.DEF) {
-                    adapter.invokeStatic(definition.getType("Def").type, DEF_NEG_CALL);
+                    adapter.invokeStatic(Definition.defobjType.type, DEF_NEG_CALL);
                 } else {
                     adapter.math(MethodWriter.NEG, type);
                 }

From 7e00a1c1a33e7f0c7ed0735e1a991f2b7970c8be Mon Sep 17 00:00:00 2001
From: eratio08 <eratio@gmx.net>
Date: Fri, 20 May 2016 11:01:07 +0200
Subject: [PATCH 0368/1311] Added Type name for DFI (#18480)

---
 docs/reference/index-modules/similarity.asciidoc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/reference/index-modules/similarity.asciidoc b/docs/reference/index-modules/similarity.asciidoc
index 07591dc277b..8173b98d505 100644
--- a/docs/reference/index-modules/similarity.asciidoc
+++ b/docs/reference/index-modules/similarity.asciidoc
@@ -118,6 +118,8 @@ This similarity has the following options:
 [horizontal]
 `independence_measure`:: Possible values `standardized`, `saturated`, `chisquared`.
 
+Type name: `DFI`
+
 [float]
 [[ib]]
 ==== IB similarity.

From 4e454439bb3d9b9a4e949dfed98736303a584e65 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Fri, 20 May 2016 02:04:38 -0700
Subject: [PATCH 0369/1311] Fixed bugs in comparison with Def.  Fixed may
 tests.n

---
 .../painless/AnalyzerCaster.java              |   7 +-
 .../org/elasticsearch/painless/Utility.java   | 432 ------------------
 .../org/elasticsearch/painless/Writer.java    |   2 +-
 .../painless/WriterConstants.java             |   1 -
 .../elasticsearch/painless/node/EComp.java    |  33 +-
 .../painless/node/EExplicit.java              |   1 +
 .../elasticsearch/painless/node/EUnary.java   |   5 +-
 .../elasticsearch/painless/ArrayTests.java    |  18 +-
 .../painless/ConditionalTests.java            |   3 +-
 .../painless/DefOperationTests.java           |   8 +-
 .../elasticsearch/painless/EqualsTests.java   |  79 +---
 .../elasticsearch/painless/StringTests.java   |   4 +-
 12 files changed, 68 insertions(+), 525 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
index 11a334eff1d..0e490054a25 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
@@ -633,7 +633,7 @@ public final class AnalyzerCaster {
                     case LONG:
                     case FLOAT:
                     case DOUBLE:
-                            return new Cast(actual, expected, true, true, false, false, false);
+                            return new Cast(actual, expected, explicit, true, false, false, false);
                 }
 
                 break;
@@ -649,8 +649,9 @@ public final class AnalyzerCaster {
                 break;
         }
 
-        if (expected.clazz.isAssignableFrom(actual.clazz) ||
-            ((explicit || expected.sort == Sort.DEF) && actual.clazz.isAssignableFrom(expected.clazz))) {
+        if (actual.sort == Sort.DEF || expected.sort == Sort.DEF ||
+            expected.clazz.isAssignableFrom(actual.clazz) ||
+            explicit && actual.clazz.isAssignableFrom(expected.clazz)) {
             return new Cast(actual, expected, explicit);
         } else {
             throw new ClassCastException("Error" + location + ": Cannot cast from [" + actual.name + "] to [" + expected.name + "].");
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java
index c8429f9458a..5ab3450db7e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java
@@ -25,442 +25,10 @@ package org.elasticsearch.painless;
  */
 public class Utility {
 
-    public static boolean NumberToboolean(final Number value) {
-        return value.longValue() != 0;
-    }
-
-    public static char NumberTochar(final Number value) {
-        return (char)value.intValue();
-    }
-
-    public static Boolean NumberToBoolean(final Number value) {
-        return value.longValue() != 0;
-    }
-
-    public static Byte NumberToByte(final Number value) {
-        return value == null ? null : value.byteValue();
-    }
-
-    public static Short NumberToShort(final Number value) {
-        return value == null ? null : value.shortValue();
-    }
-
-    public static Character NumberToCharacter(final Number value) {
-        return value == null ? null : (char)value.intValue();
-    }
-
-    public static Integer NumberToInteger(final Number value) {
-        return value == null ? null : value.intValue();
-    }
-
-    public static Long NumberToLong(final Number value) {
-        return value == null ? null : value.longValue();
-    }
-
-    public static Float NumberToFloat(final Number value) {
-        return value == null ? null : value.floatValue();
-    }
-
-    public static Double NumberToDouble(final Number value) {
-        return value == null ? null : value.doubleValue();
-    }
-
-    public static byte booleanTobyte(final boolean value) {
-        return (byte)(value ? 1 : 0);
-    }
-
-    public static short booleanToshort(final boolean value) {
-        return (short)(value ? 1 : 0);
-    }
-
-    public static char booleanTochar(final boolean value) {
-        return (char)(value ? 1 : 0);
-    }
-
-    public static int booleanToint(final boolean value) {
-        return value ? 1 : 0;
-    }
-
-    public static long booleanTolong(final boolean value) {
-        return value ? 1 : 0;
-    }
-
-    public static float booleanTofloat(final boolean value) {
-        return value ? 1 : 0;
-    }
-
-    public static double booleanTodouble(final boolean value) {
-        return value ? 1 : 0;
-    }
-
-    public static Integer booleanToInteger(final boolean value) {
-        return value ? 1 : 0;
-    }
-
-    public static byte BooleanTobyte(final Boolean value) {
-        return (byte)(value ? 1 : 0);
-    }
-
-    public static short BooleanToshort(final Boolean value) {
-        return (short)(value ? 1 : 0);
-    }
-
-    public static char BooleanTochar(final Boolean value) {
-        return (char)(value ? 1 : 0);
-    }
-
-    public static int BooleanToint(final Boolean value) {
-        return value ? 1 : 0;
-    }
-
-    public static long BooleanTolong(final Boolean value) {
-        return value ? 1 : 0;
-    }
-
-    public static float BooleanTofloat(final Boolean value) {
-        return value ? 1 : 0;
-    }
-
-    public static double BooleanTodouble(final Boolean value) {
-        return value ? 1 : 0;
-    }
-
-    public static Byte BooleanToByte(final Boolean value) {
-        return value == null ? null : (byte)(value ? 1 : 0);
-    }
-
-    public static Short BooleanToShort(final Boolean value) {
-        return value == null ? null : (short)(value ? 1 : 0);
-    }
-
-    public static Character BooleanToCharacter(final Boolean value) {
-        return value == null ? null : (char)(value ? 1 : 0);
-    }
-
-    public static Integer BooleanToInteger(final Boolean value) {
-        return value == null ? null : value ? 1 : 0;
-    }
-
-    public static Long BooleanToLong(final Boolean value) {
-        return value == null ? null : value ? 1L : 0L;
-    }
-
-    public static Float BooleanToFloat(final Boolean value) {
-        return value == null ? null : value ? 1F : 0F;
-    }
-
-    public static Double BooleanToDouble(final Boolean value) {
-        return value == null ? null : value ? 1D : 0D;
-    }
-
-    public static boolean byteToboolean(final byte value) {
-        return value != 0;
-    }
-
-    public static Short byteToShort(final byte value) {
-        return (short)value;
-    }
-
-    public static Character byteToCharacter(final byte value) {
-        return (char)value;
-    }
-
-    public static Integer byteToInteger(final byte value) {
-        return (int)value;
-    }
-
-    public static Long byteToLong(final byte value) {
-        return (long)value;
-    }
-
-    public static Float byteToFloat(final byte value) {
-        return (float)value;
-    }
-
-    public static Double byteToDouble(final byte value) {
-        return (double)value;
-    }
-
-    public static boolean ByteToboolean(final Byte value) {
-        return value != 0;
-    }
-
-    public static char ByteTochar(final Byte value) {
-        return (char)value.byteValue();
-    }
-
-    public static boolean shortToboolean(final short value) {
-        return value != 0;
-    }
-
-    public static Byte shortToByte(final short value) {
-        return (byte)value;
-    }
-
-    public static Character shortToCharacter(final short value) {
-        return (char)value;
-    }
-
-    public static Integer shortToInteger(final short value) {
-        return (int)value;
-    }
-
-    public static Long shortToLong(final short value) {
-        return (long)value;
-    }
-
-    public static Float shortToFloat(final short value) {
-        return (float)value;
-    }
-
-    public static Double shortToDouble(final short value) {
-        return (double)value;
-    }
-
-    public static boolean ShortToboolean(final Short value) {
-        return value != 0;
-    }
-
-    public static char ShortTochar(final Short value) {
-        return (char)value.shortValue();
-    }
-
-    public static boolean charToboolean(final char value) {
-        return value != 0;
-    }
-
-    public static Byte charToByte(final char value) {
-        return (byte)value;
-    }
-
-    public static Short charToShort(final char value) {
-        return (short)value;
-    }
-
-    public static Integer charToInteger(final char value) {
-        return (int)value;
-    }
-
-    public static Long charToLong(final char value) {
-        return (long)value;
-    }
-
-    public static Float charToFloat(final char value) {
-        return (float)value;
-    }
-
-    public static Double charToDouble(final char value) {
-        return (double)value;
-    }
-
     public static String charToString(final char value) {
         return String.valueOf(value);
     }
 
-    public static boolean CharacterToboolean(final Character value) {
-        return value != 0;
-    }
-
-    public static byte CharacterTobyte(final Character value) {
-        return (byte)value.charValue();
-    }
-
-    public static short CharacterToshort(final Character value) {
-        return (short)value.charValue();
-    }
-
-    public static int CharacterToint(final Character value) {
-        return value;
-    }
-
-    public static long CharacterTolong(final Character value) {
-        return value;
-    }
-
-    public static float CharacterTofloat(final Character value) {
-        return value;
-    }
-
-    public static double CharacterTodouble(final Character value) {
-        return value;
-    }
-
-    public static Boolean CharacterToBoolean(final Character value) {
-        return value == null ? null : value != 0;
-    }
-
-    public static Byte CharacterToByte(final Character value) {
-        return value == null ? null : (byte)value.charValue();
-    }
-
-    public static Short CharacterToShort(final Character value) {
-        return value == null ? null : (short)value.charValue();
-    }
-
-    public static Integer CharacterToInteger(final Character value) {
-        return value == null ? null : (int)value;
-    }
-
-    public static Long CharacterToLong(final Character value) {
-        return value == null ? null : (long)value;
-    }
-
-    public static Float CharacterToFloat(final Character value) {
-        return value == null ? null : (float)value;
-    }
-
-    public static Double CharacterToDouble(final Character value) {
-        return value == null ? null : (double)value;
-    }
-
-    public static String CharacterToString(final Character value) {
-        return value == null ? null : value.toString();
-    }
-
-    public static boolean intToboolean(final int value) {
-        return value != 0;
-    }
-
-    public static Byte intToByte(final int value) {
-        return (byte)value;
-    }
-
-    public static Short intToShort(final int value) {
-        return (short)value;
-    }
-
-    public static Character intToCharacter(final int value) {
-        return (char)value;
-    }
-
-    public static Long intToLong(final int value) {
-        return (long)value;
-    }
-
-    public static Float intToFloat(final int value) {
-        return (float)value;
-    }
-
-    public static Double intToDouble(final int value) {
-        return (double)value;
-    }
-
-    public static boolean IntegerToboolean(final Integer value) {
-        return value != 0;
-    }
-
-    public static char IntegerTochar(final Integer value) {
-        return (char)value.intValue();
-    }
-
-    public static boolean longToboolean(final long value) {
-        return value != 0;
-    }
-
-    public static Byte longToByte(final long value) {
-        return (byte)value;
-    }
-
-    public static Short longToShort(final long value) {
-        return (short)value;
-    }
-
-    public static Character longToCharacter(final long value) {
-        return (char)value;
-    }
-
-    public static Integer longToInteger(final long value) {
-        return (int)value;
-    }
-
-    public static Float longToFloat(final long value) {
-        return (float)value;
-    }
-
-    public static Double longToDouble(final long value) {
-        return (double)value;
-    }
-
-    public static boolean LongToboolean(final Long value) {
-        return value != 0;
-    }
-
-    public static char LongTochar(final Long value) {
-        return (char)value.longValue();
-    }
-
-    public static boolean floatToboolean(final float value) {
-        return value != 0;
-    }
-
-    public static Byte floatToByte(final float value) {
-        return (byte)value;
-    }
-
-    public static Short floatToShort(final float value) {
-        return (short)value;
-    }
-
-    public static Character floatToCharacter(final float value) {
-        return (char)value;
-    }
-
-    public static Integer floatToInteger(final float value) {
-        return (int)value;
-    }
-
-    public static Long floatToLong(final float value) {
-        return (long)value;
-    }
-
-    public static Double floatToDouble(final float value) {
-        return (double)value;
-    }
-
-    public static boolean FloatToboolean(final Float value) {
-        return value != 0;
-    }
-
-    public static char FloatTochar(final Float value) {
-        return (char)value.floatValue();
-    }
-
-    public static boolean doubleToboolean(final double value) {
-        return value != 0;
-    }
-
-    public static Byte doubleToByte(final double value) {
-        return (byte)value;
-    }
-
-    public static Short doubleToShort(final double value) {
-        return (short)value;
-    }
-
-    public static Character doubleToCharacter(final double value) {
-        return (char)value;
-    }
-
-    public static Integer doubleToInteger(final double value) {
-        return (int)value;
-    }
-
-    public static Long doubleToLong(final double value) {
-        return (long)value;
-    }
-
-    public static Float doubleToFloat(final double value) {
-        return (float)value;
-    }
-
-    public static boolean DoubleToboolean(final Double value) {
-        return value != 0;
-    }
-
-    public static char DoubleTochar(final Double value) {
-        return (char)value.doubleValue();
-    }
-
     public static char StringTochar(final String value) {
         if (value.length() != 1) {
             throw new ClassCastException("Cannot cast [String] with length greater than one to [char].");
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
index 449361867b9..d290e758ea9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
@@ -117,7 +117,7 @@ final class Writer {
             // if we truncated, make it obvious
             if (limit != source.length()) {
                 fileName.append(" ...");
-            }            
+            }
             fileName.append(" @ <inline script>");
         } else {
             // its a named script, just use the name
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index 6330e1603c5..e223efe36a4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -66,7 +66,6 @@ public final class WriterConstants {
         new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(DefBootstrap.class),
             "bootstrap", DEF_BOOTSTRAP_TYPE.toMethodDescriptorString());
 
-
     public final static Type DEF_TYPE = Type.getType(Def.class);
     public final static Method DEF_TO_BOOLEAN         = getAsmMethod(boolean.class, "DefToboolean"       , Object.class);
     public final static Method DEF_TO_BYTE_IMPLICIT   = getAsmMethod(byte.class   , "DefTobyteImplicit"  , Object.class);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index 270f95e0a92..e7b610bfc92 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -35,6 +35,8 @@ import static org.elasticsearch.painless.WriterConstants.DEF_GTE_CALL;
 import static org.elasticsearch.painless.WriterConstants.DEF_GT_CALL;
 import static org.elasticsearch.painless.WriterConstants.DEF_LTE_CALL;
 import static org.elasticsearch.painless.WriterConstants.DEF_LT_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_TYPE;
+import static org.elasticsearch.painless.WriterConstants.UTILITY_TYPE;
 
 /**
  * Represents a comparison expression.
@@ -455,34 +457,37 @@ public final class EComp extends AExpression {
                 if (eq) {
                     if (right.isNull) {
                         adapter.ifNull(jump);
-                    } else if (!left.isNull && operation == Operation.EQ) {
-                        adapter.invokeStatic(definition.getType("Def").type, DEF_EQ_CALL);
+                    } else if (!left.isNull && (operation == Operation.EQ || operation == Operation.NE)) {
+                        adapter.invokeStatic(DEF_TYPE, DEF_EQ_CALL);
+                        writejump = false;
                     } else {
                         adapter.ifCmp(rtype, MethodWriter.EQ, jump);
                     }
                 } else if (ne) {
                     if (right.isNull) {
                         adapter.ifNonNull(jump);
-                    } else if (!left.isNull && operation == Operation.NE) {
-                        adapter.invokeStatic(definition.getType("Def").type, DEF_EQ_CALL);
+                    } else if (!left.isNull && (operation == Operation.EQ || operation == Operation.NE)) {
+                        adapter.invokeStatic(DEF_TYPE, DEF_EQ_CALL);
                         adapter.ifZCmp(MethodWriter.EQ, jump);
                     } else {
                         adapter.ifCmp(rtype, MethodWriter.NE, jump);
                     }
                 } else if (lt) {
-                    adapter.invokeStatic(definition.getType("Def").type, DEF_LT_CALL);
+                    adapter.invokeStatic(DEF_TYPE, DEF_LT_CALL);
+                    writejump = false;
                 } else if (lte) {
-                    adapter.invokeStatic(definition.getType("Def").type, DEF_LTE_CALL);
+                    adapter.invokeStatic(DEF_TYPE, DEF_LTE_CALL);
+                    writejump = false;
                 } else if (gt) {
-                    adapter.invokeStatic(definition.getType("Def").type, DEF_GT_CALL);
+                    adapter.invokeStatic(DEF_TYPE, DEF_GT_CALL);
+                    writejump = false;
                 } else if (gte) {
-                    adapter.invokeStatic(definition.getType("Def").type, DEF_GTE_CALL);
+                    adapter.invokeStatic(DEF_TYPE, DEF_GTE_CALL);
+                    writejump = false;
                 } else {
                     throw new IllegalStateException(error("Illegal tree structure."));
                 }
 
-                writejump = left.isNull || ne || operation == Operation.EQR;
-
                 if (branch && !writejump) {
                     adapter.ifZCmp(MethodWriter.NE, jump);
                 }
@@ -492,8 +497,8 @@ public final class EComp extends AExpression {
                 if (eq) {
                     if (right.isNull) {
                         adapter.ifNull(jump);
-                    } else if (operation == Operation.EQ) {
-                        adapter.invokeStatic(definition.getType("Utility").type, CHECKEQUALS);
+                    } else if (operation == Operation.EQ || operation == Operation.NE) {
+                        adapter.invokeStatic(UTILITY_TYPE, CHECKEQUALS);
 
                         if (branch) {
                             adapter.ifZCmp(MethodWriter.NE, jump);
@@ -506,8 +511,8 @@ public final class EComp extends AExpression {
                 } else if (ne) {
                     if (right.isNull) {
                         adapter.ifNonNull(jump);
-                    } else if (operation == Operation.NE) {
-                        adapter.invokeStatic(definition.getType("Utility").type, CHECKEQUALS);
+                    } else if (operation == Operation.EQ || operation == Operation.NE) {
+                        adapter.invokeStatic(UTILITY_TYPE, CHECKEQUALS);
                         adapter.ifZCmp(MethodWriter.EQ, jump);
                     } else {
                         adapter.ifCmp(rtype, MethodWriter.NE, jump);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
index ac0b06c0a79..d685ed9eb01 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
@@ -61,6 +61,7 @@ public final class EExplicit extends AExpression {
     AExpression cast(final CompilerSettings settings, final Definition definition, final Variables variables) {
         child.expected = expected;
         child.explicit = explicit;
+        child.internal = internal;
 
         return child.cast(settings, definition, variables);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index 3e34c16ab97..418611ba657 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -31,6 +31,7 @@ import org.elasticsearch.painless.MethodWriter;
 
 import static org.elasticsearch.painless.WriterConstants.DEF_NEG_CALL;
 import static org.elasticsearch.painless.WriterConstants.DEF_NOT_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_TYPE;
 
 /**
  * Represents a unary math expression.
@@ -191,7 +192,7 @@ public final class EUnary extends AExpression {
 
             if (operation == Operation.BWNOT) {
                 if (sort == Sort.DEF) {
-                    adapter.invokeStatic(definition.getType("Def").type, DEF_NOT_CALL);
+                    adapter.invokeStatic(DEF_TYPE, DEF_NOT_CALL);
                 } else {
                     if (sort == Sort.INT) {
                         adapter.push(-1);
@@ -205,7 +206,7 @@ public final class EUnary extends AExpression {
                 }
             } else if (operation == Operation.SUB) {
                 if (sort == Sort.DEF) {
-                    adapter.invokeStatic(definition.getType("Def").type, DEF_NEG_CALL);
+                    adapter.invokeStatic(DEF_TYPE, DEF_NEG_CALL);
                 } else {
                     adapter.math(MethodWriter.NEG, type);
                 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
index 2036c4fd04c..8dabacdd5f9 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
@@ -34,7 +34,7 @@ public class ArrayTests extends ScriptTestCase {
         assertArrayLength(10, new Integer[10]);
         assertArrayLength(11, new String[11][2]);
     }
-    
+
     private void assertArrayLength(int length, Object array) throws Throwable {
         assertEquals(length, (int) Def.arrayLengthGetter(array.getClass()).invoke(array));
     }
@@ -43,36 +43,36 @@ public class ArrayTests extends ScriptTestCase {
         assertEquals(5, exec("def x = new int[5]; return x.length"));
         assertEquals(5, exec("def x = new int[4]; x[0] = 5; return x[0];"));
     }
-    
+
     public void testArrayLoadStoreString() {
         assertEquals(5, exec("def x = new String[5]; return x.length"));
         assertEquals("foobar", exec("def x = new String[4]; x[0] = 'foobar'; return x[0];"));
     }
-    
+
     public void testArrayLoadStoreDef() {
         assertEquals(5, exec("def x = new def[5]; return x.length"));
         assertEquals(5, exec("def x = new def[4]; x[0] = 5; return x[0];"));
     }
-    
+
     public void testArrayCompoundInt() {
         assertEquals(6, exec("int[] x = new int[5]; x[0] = 5; x[0]++; return x[0];"));
     }
-    
+
     public void testArrayCompoundDef() {
         assertEquals(6, exec("def x = new int[5]; x[0] = 5; x[0]++; return x[0];"));
     }
-    
+
     public void testJacksCrazyExpression1() {
         assertEquals(1, exec("int x; def[] y = new def[1]; x = y[0] = 1; return x;"));
     }
-    
+
     public void testJacksCrazyExpression2() {
         assertEquals(1, exec("int x; def y = new def[1]; x = y[0] = 1; return x;"));
     }
-    
+
     public void testForLoop() {
         assertEquals(999*1000/2, exec("def a = new int[1000]; for (int x = 0; x < a.length; x++) { a[x] = x; } "+
             "int total = 0; for (int x = 0; x < a.length; x++) { total += a[x]; } return total;"));
     }
-    
+
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ConditionalTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ConditionalTests.java
index bfcadef3b84..a3a09fee425 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ConditionalTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ConditionalTests.java
@@ -65,7 +65,8 @@ public class ConditionalTests extends ScriptTestCase {
 
     public void testPromotion() {
         assertEquals(false, exec("boolean x = false; boolean y = true; return (x ? 2 : 4.0F) == (y ? 2 : 4.0F);"));
-        assertEquals(false, exec("boolean x = false; boolean y = true; return (x ? 2 : 4.0F) == (y ? new HashMap() : new ArrayList());"));
+        assertEquals(false, exec("boolean x = false; boolean y = true; " +
+            "return (x ? new HashMap() : new ArrayList()) == (y ? new HashMap() : new ArrayList());"));
     }
 
     public void testIncompatibleAssignment() {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
index 9ae657f7c1b..1b548efa3b8 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
@@ -22,10 +22,10 @@ package org.elasticsearch.painless;
 public class DefOperationTests extends ScriptTestCase {
     public void testIllegalCast() {
         Exception exception = expectThrows(ClassCastException.class, () -> exec("def x = 1.0; int y = x; return y;"));
-        assertTrue(exception.getMessage().contains("java.lang.double cannot be cast to java.lang.int"));
+        assertTrue(exception.getMessage().contains("cannot be cast"));
 
         exception = expectThrows(ClassCastException.class, () -> exec("def x = (short)1; byte y = x; return y;"));
-        assertTrue(exception.getMessage().contains("java.lang.short cannot be cast to java.lang.byte"));
+        assertTrue(exception.getMessage().contains("cannot be cast"));
     }
 
     public void testNot() {
@@ -799,7 +799,7 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(false, exec("def x = (byte)7; def y = (int)7; return x === y"));
         assertEquals(false, exec("def x = (short)6; def y = (int)6; return x === y"));
         assertEquals(false, exec("def x = (char)5; def y = (int)5; return x === y"));
-        assertEquals(true, exec("def x = (int)4; def y = (int)4; return x === y"));
+        assertEquals(false, exec("def x = (int)4; def y = (int)4; return x === y"));
         assertEquals(false, exec("def x = (long)5; def y = (int)3; return x === y"));
         assertEquals(false, exec("def x = (float)6; def y = (int)2; return x === y"));
         assertEquals(false, exec("def x = (double)7; def y = (int)1; return x === y"));
@@ -837,7 +837,7 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(true, exec("def x = (byte)7; def y = (int)7; return x !== y"));
         assertEquals(true, exec("def x = (short)6; def y = (int)6; return x !== y"));
         assertEquals(true, exec("def x = (char)5; def y = (int)5; return x !== y"));
-        assertEquals(false, exec("def x = (int)4; def y = (int)4; return x !== y"));
+        assertEquals(true, exec("def x = (int)4; def y = (int)4; return x !== y"));
         assertEquals(true, exec("def x = (long)5; def y = (int)3; return x !== y"));
         assertEquals(true, exec("def x = (float)6; def y = (int)2; return x !== y"));
         assertEquals(true, exec("def x = (double)7; def y = (int)1; return x !== y"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java
index bf7d41dfa62..59f9aadc6aa 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java
@@ -94,17 +94,8 @@ public class EqualsTests extends ScriptTestCase {
     }
 
     public void testEquals() {
-        assertEquals(true, exec("return Long.valueOf(3) == 3L;"));
-        assertEquals(false, exec("return new Long(3) === new Long(3);"));
-        assertEquals(true, exec("Integer x = new Integer(3); Object y = x; return x == y;"));
-        assertEquals(true, exec("Integer x = new Integer(3); Object y = x; return x === y;"));
-        assertEquals(true, exec("Integer x = new Integer(3); Object y = new Integer(3); return x == y;"));
-        assertEquals(false, exec("Integer x = new Integer(3); Object y = new Integer(3); return x === y;"));
-        assertEquals(true, exec("Integer x = new Integer(3); int y = 3; return x == y;"));
-        assertEquals(true, exec("Integer x = new Integer(3); short y = 3; return x == y;"));
-        assertEquals(true, exec("Integer x = new Integer(3); Short y = (short)3; return x == y;"));
-        assertEquals(false, exec("Integer x = new Integer(3); int y = 3; return x === y;"));
-        assertEquals(false, exec("Integer x = new Integer(3); double y = 3; return x === y;"));
+        assertEquals(true, exec("return 3 == 3;"));
+        assertEquals(false, exec("int x = 4; int y = 5; x == y"));
         assertEquals(true, exec("int[] x = new int[1]; Object y = x; return x == y;"));
         assertEquals(true, exec("int[] x = new int[1]; Object y = x; return x === y;"));
         assertEquals(false, exec("int[] x = new int[1]; Object y = new int[1]; return x == y;"));
@@ -114,14 +105,8 @@ public class EqualsTests extends ScriptTestCase {
     }
 
     public void testNotEquals() {
-        assertEquals(false, exec("return new Long(3) != new Long(3);"));
-        assertEquals(true, exec("return new Long(3) !== new Long(3);"));
-        assertEquals(false, exec("Integer x = new Integer(3); Object y = x; return x != y;"));
-        assertEquals(false, exec("Integer x = new Integer(3); Object y = x; return x !== y;"));
-        assertEquals(false, exec("Integer x = new Integer(3); Object y = new Integer(3); return x != y;"));
-        assertEquals(true, exec("Integer x = new Integer(3); Object y = new Integer(3); return x !== y;"));
-        assertEquals(true, exec("Integer x = new Integer(3); int y = 3; return x !== y;"));
-        assertEquals(true, exec("Integer x = new Integer(3); double y = 3; return x !== y;"));
+        assertEquals(false, exec("return 3 != 3;"));
+        assertEquals(true, exec("int x = 4; int y = 5; x != y"));
         assertEquals(false, exec("int[] x = new int[1]; Object y = x; return x != y;"));
         assertEquals(false, exec("int[] x = new int[1]; Object y = x; return x !== y;"));
         assertEquals(true, exec("int[] x = new int[1]; Object y = new int[1]; return x != y;"));
@@ -131,54 +116,36 @@ public class EqualsTests extends ScriptTestCase {
     }
 
     public void testBranchEquals() {
-        assertEquals(0, exec("Character a = (char)'a'; Character b = (char)'b'; if (a == b) return 1; else return 0;"));
-        assertEquals(1, exec("Character a = (char)'a'; Character b = (char)'a'; if (a == b) return 1; else return 0;"));
-        assertEquals(0, exec("Integer a = new Integer(1); Integer b = 1; if (a === b) return 1; else return 0;"));
-        assertEquals(0, exec("Character a = (char)'a'; Character b = new Character((char)'a'); if (a === b) return 1; else return 0;"));
-        assertEquals(1, exec("Character a = (char)'a'; Object b = a; if (a === b) return 1; else return 0;"));
-        assertEquals(1, exec("Integer a = 1; Number b = a; Number c = a; if (c === b) return 1; else return 0;"));
-        assertEquals(0, exec("Integer a = 1; Character b = (char)'a'; if (a === (Object)b) return 1; else return 0;"));
+        assertEquals(0, exec("def a = (char)'a'; def b = (char)'b'; if (a == b) return 1; else return 0;"));
+        assertEquals(1, exec("def a = (char)'a'; def b = (char)'a'; if (a == b) return 1; else return 0;"));
+        assertEquals(0, exec("def a = 1; def b = 1; if (a === b) return 1; else return 0;"));
+        assertEquals(0, exec("def a = (char)'a'; def b = (char)'a'; if (a === b) return 1; else return 0;"));
+        assertEquals(1, exec("def a = (char)'a'; Object b = a; if (a === b) return 1; else return 0;"));
+        assertEquals(1, exec("def a = 1; Number b = a; Number c = a; if (c === b) return 1; else return 0;"));
+        assertEquals(0, exec("def a = 1; Object b = new HashMap(); if (a === (Object)b) return 1; else return 0;"));
     }
 
     public void testBranchNotEquals() {
-        assertEquals(1, exec("Character a = (char)'a'; Character b = (char)'b'; if (a != b) return 1; else return 0;"));
-        assertEquals(0, exec("Character a = (char)'a'; Character b = (char)'a'; if (a != b) return 1; else return 0;"));
+        assertEquals(1, exec("def a = (char)'a'; def b = (char)'b'; if (a != b) return 1; else return 0;"));
+        assertEquals(0, exec("def a = (char)'a'; def b = (char)'a'; if (a != b) return 1; else return 0;"));
         assertEquals(1, exec("def a = 1; def b = 1; if (a !== b) return 1; else return 0;"));
-        assertEquals(1, exec("def a = (char)'a'; Character b = new Character((char)'a'); if (a !== b) return 1; else return 0;"));
+        assertEquals(1, exec("def a = (char)'a'; def b = (char)'a'; if (a !== b) return 1; else return 0;"));
         assertEquals(0, exec("def a = (char)'a'; Object b = a; if (a !== b) return 1; else return 0;"));
         assertEquals(0, exec("def a = 1; Number b = a; Number c = a; if (c !== b) return 1; else return 0;"));
-        assertEquals(1, exec("def a = 1; Character b = (char)'a'; if (a !== (Object)b) return 1; else return 0;"));
+        assertEquals(1, exec("def a = 1; Object b = new HashMap(); if (a !== (Object)b) return 1; else return 0;"));
     }
 
     public void testRightHandNull() {
-        assertEquals(false, exec("Character a = (char)'a'; return a == null;"));
-        assertEquals(false, exec("Character a = (char)'a'; return a === null;"));
-        assertEquals(true, exec("Character a = (char)'a'; return a != null;"));
-        assertEquals(true, exec("Character a = (char)'a'; return a !== null;"));
-        assertEquals(true, exec("Character a = null; return a == null;"));
-        assertEquals(false, exec("Character a = null; return a != null;"));
-        assertEquals(false, exec("Character a = (char)'a'; Character b = null; return a == b;"));
-        assertEquals(true, exec("Character a = null; Character b = null; return a === b;"));
-        assertEquals(true, exec("Character a = (char)'a'; Character b = null; return a != b;"));
-        assertEquals(false, exec("Character a = null; Character b = null; return a !== b;"));
-        assertEquals(false, exec("Integer x = null; double y = 2.0; return x == y;"));
-        assertEquals(true, exec("Integer x = null; Short y = null; return x == y;"));
+        assertEquals(false, exec("HashMap a = new HashMap(); return a == null;"));
+        assertEquals(false, exec("HashMap a = new HashMap(); return a === null;"));
+        assertEquals(true, exec("HashMap a = new HashMap(); return a != null;"));
+        assertEquals(true, exec("HashMap a = new HashMap(); return a !== null;"));
     }
 
     public void testLeftHandNull() {
-        assertEquals(false, exec("Character a = (char)'a'; return null == a;"));
-        assertEquals(false, exec("Character a = (char)'a'; return null === a;"));
-        assertEquals(true, exec("Character a = (char)'a'; return null != a;"));
-        assertEquals(true, exec("Character a = (char)'a'; return null !== a;"));
-        assertEquals(true, exec("Character a = null; return null == a;"));
-        assertEquals(false, exec("Character a = null; return null != a;"));
-        assertEquals(false, exec("Character a = null; Character b = (char)'a'; return a == b;"));
-        assertEquals(true, exec("Character a = null; Character b = null; return a == b;"));
-        assertEquals(true, exec("Character a = null; Character b = null; return b === a;"));
-        assertEquals(true, exec("Character a = null; Character b = (char)'a'; return a != b;"));
-        assertEquals(false, exec("Character a = null; Character b = null; return b != a;"));
-        assertEquals(false, exec("Character a = null; Character b = null; return b !== a;"));
-        assertEquals(false, exec("Integer x = null; double y = 2.0; return y == x;"));
-        assertEquals(true, exec("Integer x = null; Short y = null; return y == x;"));
+        assertEquals(false, exec("HashMap a = new HashMap(); return null == a;"));
+        assertEquals(false, exec("HashMap a = new HashMap(); return null === a;"));
+        assertEquals(true, exec("HashMap a = new HashMap(); return null != a;"));
+        assertEquals(true, exec("HashMap a = new HashMap(); return null !== a;"));
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
index 5cd3c64925c..b06199cf903 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
@@ -166,14 +166,14 @@ public class StringTests extends ScriptTestCase {
             assertEquals("cc", exec("return (String)(char)\"cc\""));
             fail();
         } catch (final ClassCastException cce) {
-            assertTrue(cce.getMessage().contains("Cannot cast from [String] to [char]."));
+            assertTrue(cce.getMessage().contains("Cannot cast [String] with length greater than one to [char]."));
         }
 
         try {
             assertEquals("cc", exec("return (String)(char)'cc'"));
             fail();
         } catch (final ClassCastException cce) {
-            assertTrue(cce.getMessage().contains("Cannot cast from [String] to [char]."));
+            assertTrue(cce.getMessage().contains("Cannot cast [String] with length greater than one to [char]."));
         }
 
         try {

From f5fc60ac678432fded7cd64548dbb9dc4bb01d52 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Fri, 20 May 2016 11:05:40 +0200
Subject: [PATCH 0370/1311] Add utility type constant

---
 .../src/main/java/org/elasticsearch/painless/Definition.java  | 1 +
 .../src/main/java/org/elasticsearch/painless/node/EComp.java  | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 6e38bf2b7a4..8013361dd9b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -69,6 +69,7 @@ public final class Definition {
     public static final Type defobjType = INSTANCE.getType("Def");
     public static final Type stringType = INSTANCE.getType("String");
     public static final Type exceptionType = INSTANCE.getType("Exception");
+    public static final Type utilityType = INSTANCE.getType("Utility");
 
     public enum Sort {
         VOID(       void.class      , 0 , true  , false , false , false ),
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index 9d7862e404a..82033124e1c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -493,7 +493,7 @@ public final class EComp extends AExpression {
                     if (right.isNull) {
                         adapter.ifNull(jump);
                     } else if (operation == Operation.EQ) {
-                        adapter.invokeStatic(definition.getType("Utility").type, CHECKEQUALS);
+                        adapter.invokeStatic(Definition.utilityType.type, CHECKEQUALS);
 
                         if (branch) {
                             adapter.ifZCmp(MethodWriter.NE, jump);
@@ -507,7 +507,7 @@ public final class EComp extends AExpression {
                     if (right.isNull) {
                         adapter.ifNonNull(jump);
                     } else if (operation == Operation.NE) {
-                        adapter.invokeStatic(definition.getType("Utility").type, CHECKEQUALS);
+                        adapter.invokeStatic(Definition.utilityType.type, CHECKEQUALS);
                         adapter.ifZCmp(MethodWriter.EQ, jump);
                     } else {
                         adapter.ifCmp(rtype, MethodWriter.NE, jump);

From 91f4bba04275223c0205b27bb8b447a158f4a346 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Fri, 20 May 2016 11:22:03 +0200
Subject: [PATCH 0371/1311] Make Definition's public API completely static.
 TODO: Remove Definition arguments everywhere and hide INSTANCE field!

---
 .../painless/AnalyzerCaster.java              |   4 +-
 .../java/org/elasticsearch/painless/Def.java  |  12 +-
 .../elasticsearch/painless/Definition.java    | 125 ++++++++++--------
 .../org/elasticsearch/painless/Variables.java |   4 +-
 .../painless/node/EExplicit.java              |   2 +-
 .../elasticsearch/painless/node/LBrace.java   |   2 +-
 .../elasticsearch/painless/node/LCast.java    |   2 +-
 .../painless/node/LNewArray.java              |   6 +-
 .../elasticsearch/painless/node/LNewObj.java  |   2 +-
 .../painless/node/LVariable.java              |   2 +-
 10 files changed, 91 insertions(+), 70 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
index 608f947016f..bff486a60c5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
@@ -41,10 +41,10 @@ public final class AnalyzerCaster {
             return null;
         }
 
-        Cast transform = definition.transformsMap.get(cast);
+        Cast transform = Definition.getTransform(cast);
 
         if (transform == null && explicit) {
-            transform = definition.transformsMap.get(new Cast(actual, expected, false));
+            transform = Definition.getTransform(new Cast(actual, expected, false));
         }
 
         if (transform != null) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 889328e58db..de9e1c43bea 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -145,7 +145,7 @@ public final class Def {
          Definition.MethodKey key = new Definition.MethodKey(name, type.parameterCount());
          // check whitelist for matching method
          for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
-             RuntimeClass struct = definition.runtimeMap.get(clazz);
+             RuntimeClass struct = Definition.getRuntimeClass(clazz);
 
              if (struct != null) {
                  Method method = struct.methods.get(key);
@@ -155,7 +155,7 @@ public final class Def {
              }
 
              for (final Class<?> iface : clazz.getInterfaces()) {
-                 struct = definition.runtimeMap.get(iface);
+                 struct = Definition.getRuntimeClass(iface);
 
                  if (struct != null) {
                      Method method = struct.methods.get(key);
@@ -200,7 +200,7 @@ public final class Def {
     static MethodHandle lookupGetter(Class<?> receiverClass, String name, Definition definition) {
         // first try whitelist
         for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
-            RuntimeClass struct = definition.runtimeMap.get(clazz);
+            RuntimeClass struct = Definition.getRuntimeClass(clazz);
 
             if (struct != null) {
                 MethodHandle handle = struct.getters.get(name);
@@ -210,7 +210,7 @@ public final class Def {
             }
 
             for (final Class<?> iface : clazz.getInterfaces()) {
-                struct = definition.runtimeMap.get(iface);
+                struct = Definition.getRuntimeClass(iface);
 
                 if (struct != null) {
                     MethodHandle handle = struct.getters.get(name);
@@ -271,7 +271,7 @@ public final class Def {
     static MethodHandle lookupSetter(Class<?> receiverClass, String name, Definition definition) {
         // first try whitelist
         for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
-            RuntimeClass struct = definition.runtimeMap.get(clazz);
+            RuntimeClass struct = Definition.getRuntimeClass(clazz);
 
             if (struct != null) {
                 MethodHandle handle = struct.setters.get(name);
@@ -281,7 +281,7 @@ public final class Def {
             }
 
             for (final Class<?> iface : clazz.getInterfaces()) {
-                struct = definition.runtimeMap.get(iface);
+                struct = Definition.getRuntimeClass(iface);
 
                 if (struct != null) {
                     MethodHandle handle = struct.setters.get(name);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 8013361dd9b..2557d41f6ae 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -47,29 +47,29 @@ public final class Definition {
     public static final Definition INSTANCE = new Definition(new Definition());
     
     /** Some native types as constants: */
-    public static final Type voidType = INSTANCE.getType("void");
-    public static final Type booleanType = INSTANCE.getType("boolean");
-    public static final Type booleanobjType = INSTANCE.getType("Boolean");
-    public static final Type byteType = INSTANCE.getType("byte");
-    public static final Type byteobjType = INSTANCE.getType("Byte");
-    public static final Type shortType = INSTANCE.getType("short");
-    public static final Type shortobjType = INSTANCE.getType("Short");
-    public static final Type intType = INSTANCE.getType("int");
-    public static final Type intobjType = INSTANCE.getType("Integer");
-    public static final Type longType = INSTANCE.getType("long");
-    public static final Type longobjType = INSTANCE.getType("Long");
-    public static final Type floatType = INSTANCE.getType("float");
-    public static final Type floatobjType = INSTANCE.getType("Float");
-    public static final Type doubleType = INSTANCE.getType("double");
-    public static final Type doubleobjType = INSTANCE.getType("Double");
-    public static final Type charType = INSTANCE.getType("char");
-    public static final Type charobjType = INSTANCE.getType("Character");
-    public static final Type objectType = INSTANCE.getType("Object");
-    public static final Type defType = INSTANCE.getType("def");
-    public static final Type defobjType = INSTANCE.getType("Def");
-    public static final Type stringType = INSTANCE.getType("String");
-    public static final Type exceptionType = INSTANCE.getType("Exception");
-    public static final Type utilityType = INSTANCE.getType("Utility");
+    public static final Type voidType = getType("void");
+    public static final Type booleanType = getType("boolean");
+    public static final Type booleanobjType = getType("Boolean");
+    public static final Type byteType = getType("byte");
+    public static final Type byteobjType = getType("Byte");
+    public static final Type shortType = getType("short");
+    public static final Type shortobjType = getType("Short");
+    public static final Type intType = getType("int");
+    public static final Type intobjType = getType("Integer");
+    public static final Type longType = getType("long");
+    public static final Type longobjType = getType("Long");
+    public static final Type floatType = getType("float");
+    public static final Type floatobjType = getType("Float");
+    public static final Type doubleType = getType("double");
+    public static final Type doubleobjType = getType("Double");
+    public static final Type charType = getType("char");
+    public static final Type charobjType = getType("Character");
+    public static final Type objectType = getType("Object");
+    public static final Type defType = getType("def");
+    public static final Type defobjType = getType("Def");
+    public static final Type stringType = getType("String");
+    public static final Type exceptionType = getType("Exception");
+    public static final Type utilityType = getType("Utility");
 
     public enum Sort {
         VOID(       void.class      , 0 , true  , false , false , false ),
@@ -394,8 +394,29 @@ public final class Definition {
         }
     }
 
-    final Map<Cast, Cast> transformsMap;
-    final Map<Class<?>, RuntimeClass> runtimeMap;
+    
+    /** Gets the type given by its name */
+    public static Type getType(final String name) {
+        return INSTANCE.getTypeInternal(name);
+    }
+
+    /** Creates an array type from the given Struct. */
+    public static Type getType(final Struct struct, final int dimensions) {
+        return INSTANCE.getTypeInternal(struct, dimensions);
+    }
+    
+    public static RuntimeClass getRuntimeClass(Class<?> clazz) {
+        return INSTANCE.runtimeMap.get(clazz);
+    }
+    
+    public static Cast getTransform(Cast cast) {
+        return INSTANCE.transformsMap.get(cast);
+    }
+    
+    // INTERNAL IMPLEMENTATION:
+    
+    private final Map<Cast, Cast> transformsMap;
+    private final Map<Class<?>, RuntimeClass> runtimeMap;
     private final Map<String, Struct> structsMap;
     private final Map<String, Type> simpleTypesMap;
 
@@ -531,26 +552,26 @@ public final class Definition {
     }
 
     private void addTransforms() {
-        Type booleanType = getType("boolean");
-        Type objectType = getType("Object");
-        Type defType = getType("def");
-        Type booleanobjType = getType("Boolean");
-        Type byteType = getType("byte");
-        Type shortType = getType("short");
-        Type intType = getType("int");
-        Type charType = getType("char");
-        Type longType = getType("long");
-        Type floatType = getType("float");
-        Type doubleType = getType("double");
-        Type numberType = getType("Number");
-        Type byteobjType = getType("Byte");
-        Type shortobjType = getType("Short");
-        Type charobjType = getType("Character");
-        Type intobjType = getType("Integer");
-        Type longobjType = getType("Long");
-        Type floatobjType = getType("Float");
-        Type doubleobjType = getType("Double");
-        Type stringType = getType("String");
+        Type booleanType = getTypeInternal("boolean");
+        Type objectType = getTypeInternal("Object");
+        Type defType = getTypeInternal("def");
+        Type booleanobjType = getTypeInternal("Boolean");
+        Type byteType = getTypeInternal("byte");
+        Type shortType = getTypeInternal("short");
+        Type intType = getTypeInternal("int");
+        Type charType = getTypeInternal("char");
+        Type longType = getTypeInternal("long");
+        Type floatType = getTypeInternal("float");
+        Type doubleType = getTypeInternal("double");
+        Type numberType = getTypeInternal("Number");
+        Type byteobjType = getTypeInternal("Byte");
+        Type shortobjType = getTypeInternal("Short");
+        Type charobjType = getTypeInternal("Character");
+        Type intobjType = getTypeInternal("Integer");
+        Type longobjType = getTypeInternal("Long");
+        Type floatobjType = getTypeInternal("Float");
+        Type doubleobjType = getTypeInternal("Double");
+        Type stringType = getTypeInternal("String");
         
         addTransform(booleanType, objectType, "Boolean", "valueOf", true, false);
         addTransform(booleanType, defType, "Boolean", "valueOf", true, false);
@@ -848,7 +869,7 @@ public final class Definition {
         final Struct struct = new Struct(name, clazz, org.objectweb.asm.Type.getType(clazz));
 
         structsMap.put(name, struct);
-        simpleTypesMap.put(name, getType(name));
+        simpleTypesMap.put(name, getTypeInternal(name));
     }
     
     private final void addConstructorInternal(final String struct, final String name, final Type[] args) {
@@ -919,7 +940,7 @@ public final class Definition {
             throw new IllegalArgumentException("Malformed signature: " + signature);
         }
         // method or field type (e.g. return type)
-        Type rtn = getType(elements[0]);
+        Type rtn = getTypeInternal(elements[0]);
         int parenIndex = elements[1].indexOf('(');
         if (parenIndex != -1) {
             // method or ctor
@@ -929,7 +950,7 @@ public final class Definition {
                 String arguments[] = elements[1].substring(parenIndex + 1, parenEnd).split(",");
                 args = new Type[arguments.length];
                 for (int i = 0; i < arguments.length; i++) {
-                    args[i] = getType(arguments[i]);
+                    args[i] = getTypeInternal(arguments[i]);
                 }
             } else {
                 args = new Type[0];
@@ -1275,7 +1296,7 @@ public final class Definition {
 
             if (!owner.clazz.isAssignableFrom(from.clazz)) {
                 if (from.clazz.isAssignableFrom(owner.clazz)) {
-                    upcast = getType(owner.name);
+                    upcast = getTypeInternal(owner.name);
                 } else {
                     throw new ClassCastException("Transform with owner struct [" + owner.name + "]" +
                         " and cast type from [" + from.name + "] to cast type to [" + to.name + "] using" +
@@ -1351,7 +1372,7 @@ public final class Definition {
         runtimeMap.put(struct.clazz, new RuntimeClass(methods, getters, setters));
     }
 
-    public final Type getType(final String name) {
+    private Type getTypeInternal(final String name) {
         // simple types (e.g. 0 array dimensions) are a simple hash lookup for speed
         Type simple = simpleTypesMap.get(name);
         if (simple != null) {
@@ -1365,10 +1386,10 @@ public final class Definition {
             throw new IllegalArgumentException("The struct with name [" + name + "] has not been defined.");
         }
 
-        return getType(struct, dimensions);
+        return getTypeInternal(struct, dimensions);
     }
 
-    public final Type getType(final Struct struct, final int dimensions) {
+    private Type getTypeInternal(final Struct struct, final int dimensions) {
         String name = struct.name;
         org.objectweb.asm.Type type = struct.type;
         Class<?> clazz = struct.clazz;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
index d0b4d706f51..417eff84249 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
@@ -182,7 +182,7 @@ public final class Variables {
         final Type type;
 
         try {
-            type = definition.getType(typestr);
+            type = Definition.getType(typestr);
         } catch (final IllegalArgumentException exception) {
             throw new IllegalArgumentException("Error " + location + ": Not a type [" + typestr + "].");
         }
@@ -190,7 +190,7 @@ public final class Variables {
         boolean legal = !name.contains("<");
 
         try {
-            definition.getType(name);
+            Definition.getType(name);
             legal = false;
         } catch (final IllegalArgumentException exception) {
             // Do nothing.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
index ac0b06c0a79..0ed6fccf35f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
@@ -42,7 +42,7 @@ public final class EExplicit extends AExpression {
     @Override
     void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
         try {
-            actual = definition.getType(this.type);
+            actual = Definition.getType(this.type);
         } catch (final IllegalArgumentException exception) {
             throw new IllegalArgumentException(error("Not a type [" + this.type + "]."));
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
index fb1b11d1d07..045c5d0e5fa 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
@@ -54,7 +54,7 @@ public final class LBrace extends ALink {
             index.analyze(settings, definition, variables);
             index = index.cast(settings, definition, variables);
 
-            after = definition.getType(before.struct, before.dimensions - 1);
+            after = Definition.getType(before.struct, before.dimensions - 1);
 
             return this;
         } else if (sort == Sort.DEF) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
index eb7fb3a6b10..6d618eb4a26 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
@@ -50,7 +50,7 @@ public final class LCast extends ALink {
         }
 
         try {
-            after = definition.getType(type);
+            after = Definition.getType(type);
         } catch (final IllegalArgumentException exception) {
             throw new IllegalArgumentException(error("Not a type [" + type + "]."));
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
index ffc90990cd6..36e4b83c117 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
@@ -55,7 +55,7 @@ public final class LNewArray extends ALink {
         final Type type;
 
         try {
-            type = definition.getType(this.type);
+            type = Definition.getType(this.type);
         } catch (final IllegalArgumentException exception) {
             throw new IllegalArgumentException(error("Not a type [" + this.type + "]."));
         }
@@ -68,7 +68,7 @@ public final class LNewArray extends ALink {
             arguments.set(argument, expression.cast(settings, definition, variables));
         }
 
-        after = definition.getType(type.struct, arguments.size());
+        after = Definition.getType(type.struct, arguments.size());
 
         return this;
     }
@@ -87,7 +87,7 @@ public final class LNewArray extends ALink {
         if (arguments.size() > 1) {
             adapter.visitMultiANewArrayInsn(after.type.getDescriptor(), after.type.getDimensions());
         } else {
-            adapter.newArray(definition.getType(after.struct, 0).type);
+            adapter.newArray(Definition.getType(after.struct, 0).type);
         }
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
index 227b63cf31f..784935e2aee 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
@@ -57,7 +57,7 @@ public final class LNewObj extends ALink {
         final Type type;
 
         try {
-            type = definition.getType(this.type);
+            type = Definition.getType(this.type);
         } catch (final IllegalArgumentException exception) {
             throw new IllegalArgumentException(error("Not a type [" + this.type + "]."));
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
index 35a652f5b84..2c6ded3b4ec 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
@@ -51,7 +51,7 @@ public final class LVariable extends ALink {
         Type type = null;
 
         try {
-            type = definition.getType(name);
+            type = Definition.getType(name);
         } catch (final IllegalArgumentException exception) {
             // Do nothing.
         }

From fcee329332d30b26df4b4380be1f64dc327c419a Mon Sep 17 00:00:00 2001
From: Luca Cavanna <javanna@users.noreply.github.com>
Date: Fri, 20 May 2016 12:02:42 +0200
Subject: [PATCH 0372/1311] update http client version to 4.5.2 and http-core
 4.4.4 (#18399)

StrictHostnameVerifier can now be removed
---
 .../resources/checkstyle_suppressions.xml     |   1 -
 buildSrc/version.properties                   |   6 +-
 .../test/rest/client/RestClient.java          |  18 +--
 .../rest/client/StrictHostnameVerifier.java   |  76 -----------
 .../client/StrictHostnameVerifierTests.java   | 120 ------------------
 5 files changed, 11 insertions(+), 210 deletions(-)
 delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/client/StrictHostnameVerifier.java
 delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/client/StrictHostnameVerifierTests.java

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 48f07b1a2d5..5c776083279 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -1335,7 +1335,6 @@
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]junit[/\\]listeners[/\\]LoggingListener.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]ESRestTestCase.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]RestTestExecutionContext.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]client[/\\]RestClient.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]client[/\\]http[/\\]HttpRequestBuilder.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]json[/\\]JsonPath.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]parser[/\\]GreaterThanEqualToParser.java" checks="LineLength" />
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index fee8404080a..b6e64a3c263 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -13,9 +13,7 @@ jna               = 4.1.0
 # test dependencies
 randomizedrunner  = 2.3.2
 junit             = 4.11
-# TODO: Upgrade httpclient to a version > 4.5.1 once released. Then remove o.e.test.rest.client.StrictHostnameVerifier* and use
-#  DefaultHostnameVerifier instead since we no longer need to workaround https://issues.apache.org/jira/browse/HTTPCLIENT-1698
-httpclient        = 4.3.6
-httpcore          = 4.3.3
+httpclient        = 4.5.2
+httpcore          = 4.4.4
 commonslogging    = 1.1.3
 commonscodec      = 1.10
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestClient.java
index 5fb6e199b17..cb35653b103 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestClient.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestClient.java
@@ -19,16 +19,15 @@
 package org.elasticsearch.test.rest.client;
 
 import com.carrotsearch.randomizedtesting.RandomizedTest;
-
 import org.apache.http.config.Registry;
 import org.apache.http.config.RegistryBuilder;
 import org.apache.http.conn.socket.ConnectionSocketFactory;
 import org.apache.http.conn.socket.PlainConnectionSocketFactory;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.SSLContexts;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.ssl.SSLContexts;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
@@ -134,7 +133,8 @@ public class RestClient implements Closeable {
      * @throws RestException if the obtained status code is non ok, unless the specific error code needs to be ignored
      * according to the ignore parameter received as input (which won't get sent to elasticsearch)
      */
-    public RestResponse callApi(String apiName, Map<String, String> params, String body, Map<String, String> headers) throws IOException, RestException {
+    public RestResponse callApi(String apiName, Map<String, String> params, String body, Map<String, String> headers)
+            throws IOException, RestException {
 
         List<Integer> ignores = new ArrayList<>();
         Map<String, String> requestParams = null;
@@ -220,7 +220,8 @@ public class RestClient implements Closeable {
                     if (restApi.getParams().contains(entry.getKey()) || ALWAYS_ACCEPTED_QUERY_STRING_PARAMS.contains(entry.getKey())) {
                         httpRequestBuilder.addParam(entry.getKey(), entry.getValue());
                     } else {
-                        throw new IllegalArgumentException("param [" + entry.getKey() + "] not supported in [" + restApi.getName() + "] api");
+                        throw new IllegalArgumentException("param [" + entry.getKey() +
+                                "] not supported in [" + restApi.getName() + "] api");
                     }
                 }
             }
@@ -293,10 +294,8 @@ public class RestClient implements Closeable {
                 try (InputStream is = Files.newInputStream(path)) {
                     keyStore.load(is, keystorePass.toCharArray());
                 }
-                SSLContext sslcontext = SSLContexts.custom()
-                        .loadTrustMaterial(keyStore, null)
-                        .build();
-                sslsf = new SSLConnectionSocketFactory(sslcontext, StrictHostnameVerifier.INSTANCE);
+                SSLContext sslcontext = SSLContexts.custom().loadTrustMaterial(keyStore, null).build();
+                sslsf = new SSLConnectionSocketFactory(sslcontext);
             } catch (KeyStoreException|NoSuchAlgorithmException|KeyManagementException|CertificateException e) {
                 throw new RuntimeException(e);
             }
@@ -308,7 +307,8 @@ public class RestClient implements Closeable {
                 .register("http", PlainConnectionSocketFactory.getSocketFactory())
                 .register("https", sslsf)
                 .build();
-        return HttpClients.createMinimal(new PoolingHttpClientConnectionManager(socketFactoryRegistry, null, null, null, 15, TimeUnit.SECONDS));
+        return HttpClients.createMinimal(
+                new PoolingHttpClientConnectionManager(socketFactoryRegistry, null, null, null, 15, TimeUnit.SECONDS));
     }
 
     /**
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/StrictHostnameVerifier.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/StrictHostnameVerifier.java
deleted file mode 100644
index 33a92ceb417..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/StrictHostnameVerifier.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.client;
-
-import org.apache.http.conn.ssl.X509HostnameVerifier;
-import org.apache.http.conn.util.InetAddressUtils;
-
-import javax.net.ssl.SSLException;
-import javax.net.ssl.SSLSession;
-import javax.net.ssl.SSLSocket;
-import java.io.IOException;
-import java.security.cert.X509Certificate;
-
-/**
- * A custom {@link X509HostnameVerifier} implementation that wraps calls to the {@link org.apache.http.conn.ssl.StrictHostnameVerifier} and
- * properly handles IPv6 addresses that come from a URL in the form <code>http://[::1]:9200/</code> by removing the surrounding brackets.
- *
- * This is a variation of the fix for <a href="https://issues.apache.org/jira/browse/HTTPCLIENT-1698">HTTPCLIENT-1698</a>, which is not
- * released yet as of Apache HttpClient 4.5.1
- */
-final class StrictHostnameVerifier implements X509HostnameVerifier {
-
-    static final StrictHostnameVerifier INSTANCE = new StrictHostnameVerifier();
-
-    // We need to wrap the default verifier for HttpClient since we use an older version and the following issue is not
-    // fixed in a released version yet https://issues.apache.org/jira/browse/HTTPCLIENT-1698
-    // TL;DR we need to strip '[' and ']' from IPv6 addresses if they come from a URL
-    private final X509HostnameVerifier verifier = new org.apache.http.conn.ssl.StrictHostnameVerifier();
-
-    private StrictHostnameVerifier() {}
-
-    @Override
-    public boolean verify(String host, SSLSession sslSession) {
-        return verifier.verify(stripBracketsIfNecessary(host), sslSession);
-    }
-
-    @Override
-    public void verify(String host, SSLSocket ssl) throws IOException {
-        verifier.verify(stripBracketsIfNecessary(host), ssl);
-    }
-
-    @Override
-    public void verify(String host, X509Certificate cert) throws SSLException {
-        verifier.verify(stripBracketsIfNecessary(host), cert);
-    }
-
-    @Override
-    public void verify(String host, String[] cns, String[] subjectAlts) throws SSLException {
-        verifier.verify(stripBracketsIfNecessary(host), cns, subjectAlts);
-    }
-
-    private String stripBracketsIfNecessary(String host) {
-        if (host.startsWith("[") && host.endsWith("]")) {
-            String newHost = host.substring(1, host.length() - 1);
-            assert InetAddressUtils.isIPv6Address(newHost);
-            return newHost;
-        }
-        return host;
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/StrictHostnameVerifierTests.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/StrictHostnameVerifierTests.java
deleted file mode 100644
index 7bbda67fbdb..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/StrictHostnameVerifierTests.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.client;
-
-import org.elasticsearch.test.ESTestCase;
-import org.junit.Before;
-
-import javax.net.ssl.SSLSession;
-import javax.net.ssl.SSLSocket;
-import javax.security.auth.x500.X500Principal;
-import java.security.cert.Certificate;
-import java.security.cert.X509Certificate;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-/**
- * Tests for the {@link StrictHostnameVerifier} to validate that it can verify IPv6 addresses with and without bracket notation, in
- * addition to other address types.
- */
-public class StrictHostnameVerifierTests extends ESTestCase {
-
-    private static final int IP_SAN_TYPE = 7;
-    private static final int DNS_SAN_TYPE = 2;
-
-    private static final String[] CNS = new String[] { "my node" };
-    private static final String[] IP_SANS = new String[] { "127.0.0.1", "192.168.1.1", "::1" };
-    private static final String[] DNS_SANS = new String[] { "localhost", "computer", "localhost6" };
-
-    private SSLSocket sslSocket;
-    private SSLSession sslSession;
-    private X509Certificate certificate;
-
-    @Before
-    public void setupMocks() throws Exception {
-        sslSocket = mock(SSLSocket.class);
-        sslSession = mock(SSLSession.class);
-        certificate = mock(X509Certificate.class);
-        Collection<List<?>> subjectAlternativeNames = new ArrayList<>();
-        for (String san : IP_SANS) {
-            subjectAlternativeNames.add(Arrays.asList(IP_SAN_TYPE, san));
-        }
-        for (String san : DNS_SANS) {
-            subjectAlternativeNames.add(Arrays.asList(DNS_SAN_TYPE, san));
-        }
-
-        when(sslSocket.getSession()).thenReturn(sslSession);
-        when(sslSession.getPeerCertificates()).thenReturn(new Certificate[] { certificate });
-        when(certificate.getSubjectX500Principal()).thenReturn(new X500Principal("CN=" + CNS[0]));
-        when(certificate.getSubjectAlternativeNames()).thenReturn(subjectAlternativeNames);
-    }
-
-    public void testThatIPv6WithBracketsWorks() throws Exception {
-        final String ipv6Host = "[::1]";
-
-        // an exception will be thrown if verification fails
-        StrictHostnameVerifier.INSTANCE.verify(ipv6Host, CNS, IP_SANS);
-        StrictHostnameVerifier.INSTANCE.verify(ipv6Host, sslSocket);
-        StrictHostnameVerifier.INSTANCE.verify(ipv6Host, certificate);
-
-        // this is the only one we can assert on
-        assertTrue(StrictHostnameVerifier.INSTANCE.verify(ipv6Host, sslSession));
-    }
-
-    public void testThatIPV6WithoutBracketWorks() throws Exception {
-        final String ipv6Host = "::1";
-
-        // an exception will be thrown if verification fails
-        StrictHostnameVerifier.INSTANCE.verify(ipv6Host, CNS, IP_SANS);
-        StrictHostnameVerifier.INSTANCE.verify(ipv6Host, sslSocket);
-        StrictHostnameVerifier.INSTANCE.verify(ipv6Host, certificate);
-
-        // this is the only one we can assert on
-        assertTrue(StrictHostnameVerifier.INSTANCE.verify(ipv6Host, sslSession));
-    }
-
-    public void testThatIPV4Works() throws Exception {
-        final String ipv4Host = randomFrom("127.0.0.1", "192.168.1.1");
-
-        // an exception will be thrown if verification fails
-        StrictHostnameVerifier.INSTANCE.verify(ipv4Host, CNS, IP_SANS);
-        StrictHostnameVerifier.INSTANCE.verify(ipv4Host, sslSocket);
-        StrictHostnameVerifier.INSTANCE.verify(ipv4Host, certificate);
-
-        // this is the only one we can assert on
-        assertTrue(StrictHostnameVerifier.INSTANCE.verify(ipv4Host, sslSession));
-    }
-
-    public void testThatHostnameWorks() throws Exception {
-        final String host = randomFrom(DNS_SANS);
-
-        // an exception will be thrown if verification fails
-        StrictHostnameVerifier.INSTANCE.verify(host, CNS, DNS_SANS);
-        StrictHostnameVerifier.INSTANCE.verify(host, sslSocket);
-        StrictHostnameVerifier.INSTANCE.verify(host, certificate);
-
-        // this is the only one we can assert on
-        assertTrue(StrictHostnameVerifier.INSTANCE.verify(host, sslSession));
-    }
-}

From 4ffa92c7c14b57cf7b9723ca1a1fddfda2f35940 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Fri, 20 May 2016 12:35:08 +0200
Subject: [PATCH 0373/1311] Remove the Definition instance passed around
 everywhere

---
 .../org/elasticsearch/painless/Analyzer.java  |   6 +-
 .../painless/AnalyzerCaster.java              |  27 ++-
 .../org/elasticsearch/painless/Compiler.java  |   4 +-
 .../java/org/elasticsearch/painless/Def.java  |   9 +-
 .../elasticsearch/painless/DefBootstrap.java  |   6 +-
 .../elasticsearch/painless/Definition.java    |   7 +-
 .../elasticsearch/painless/MethodWriter.java  |   4 +-
 .../org/elasticsearch/painless/Variables.java |   4 +-
 .../org/elasticsearch/painless/Writer.java    |  10 +-
 .../painless/node/AExpression.java            |  15 +-
 .../elasticsearch/painless/node/ALink.java    |   9 +-
 .../painless/node/AStatement.java             |   5 +-
 .../elasticsearch/painless/node/EBinary.java  | 168 +++++++++---------
 .../elasticsearch/painless/node/EBool.java    |  28 +--
 .../elasticsearch/painless/node/EBoolean.java |   4 +-
 .../elasticsearch/painless/node/ECast.java    |   7 +-
 .../elasticsearch/painless/node/EChain.java   |  76 ++++----
 .../elasticsearch/painless/node/EComp.java    | 120 ++++++-------
 .../painless/node/EConditional.java           |  24 +--
 .../painless/node/EConstant.java              |   4 +-
 .../elasticsearch/painless/node/EDecimal.java |   4 +-
 .../painless/node/EExplicit.java              |  12 +-
 .../elasticsearch/painless/node/ENull.java    |   4 +-
 .../elasticsearch/painless/node/ENumeric.java |   4 +-
 .../elasticsearch/painless/node/EUnary.java   |  48 ++---
 .../painless/node/LArrayLength.java           |   8 +-
 .../elasticsearch/painless/node/LBrace.java   |  20 +--
 .../elasticsearch/painless/node/LCall.java    |  16 +-
 .../elasticsearch/painless/node/LCast.java    |  10 +-
 .../painless/node/LDefArray.java              |  14 +-
 .../elasticsearch/painless/node/LDefCall.java |  14 +-
 .../painless/node/LDefField.java              |   8 +-
 .../elasticsearch/painless/node/LField.java   |  20 +--
 .../painless/node/LListShortcut.java          |  14 +-
 .../painless/node/LMapShortcut.java           |  14 +-
 .../painless/node/LNewArray.java              |  14 +-
 .../elasticsearch/painless/node/LNewObj.java  |  14 +-
 .../painless/node/LShortcut.java              |   8 +-
 .../elasticsearch/painless/node/LString.java  |   8 +-
 .../painless/node/LVariable.java              |   8 +-
 .../elasticsearch/painless/node/SBlock.java   |   9 +-
 .../elasticsearch/painless/node/SBreak.java   |   5 +-
 .../painless/node/SContinue.java              |   5 +-
 .../painless/node/SDeclBlock.java             |   9 +-
 .../painless/node/SDeclaration.java           |  11 +-
 .../org/elasticsearch/painless/node/SDo.java  |  14 +-
 .../painless/node/SExpression.java            |  10 +-
 .../org/elasticsearch/painless/node/SFor.java |  26 +--
 .../elasticsearch/painless/node/SIfElse.java  |  18 +-
 .../elasticsearch/painless/node/SReturn.java  |  10 +-
 .../elasticsearch/painless/node/SSource.java  |   9 +-
 .../elasticsearch/painless/node/SThrow.java   |  10 +-
 .../elasticsearch/painless/node/STrap.java    |   9 +-
 .../org/elasticsearch/painless/node/STry.java |  13 +-
 .../elasticsearch/painless/node/SWhile.java   |  14 +-
 55 files changed, 472 insertions(+), 499 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
index f215e39ad01..2cd62b4c3d4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
@@ -26,10 +26,10 @@ import org.elasticsearch.painless.node.SSource;
  * Runs the analysis phase of compilation using the Painless AST.
  */
 final class Analyzer {
-    static Variables analyze(final CompilerSettings settings, final Definition definition,
+    static Variables analyze(final CompilerSettings settings,
                                     final Reserved shortcut, final SSource root) {
-        final Variables variables = new Variables(settings, definition, shortcut);
-        root.analyze(settings, definition, variables);
+        final Variables variables = new Variables(settings, shortcut);
+        root.analyze(settings, variables);
 
         return variables;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
index bff486a60c5..2495d2edacd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
@@ -33,8 +33,7 @@ import java.lang.reflect.InvocationTargetException;
  */
 public final class AnalyzerCaster {
 
-    public static Cast getLegalCast(final Definition definition,
-                                    final String location, final Type actual, final Type expected, final boolean explicit) {
+    public static Cast getLegalCast(final String location, final Type actual, final Type expected, final boolean explicit) {
         final Cast cast = new Cast(actual, expected, explicit);
 
         if (actual.equals(expected)) {
@@ -115,7 +114,7 @@ public final class AnalyzerCaster {
         }
     }
 
-    public static Type promoteNumeric(final Definition definition, final Type from, final boolean decimal, final boolean primitive) {
+    public static Type promoteNumeric(final Type from, final boolean decimal, final boolean primitive) {
         final Sort sort = from.sort;
 
         if (sort == Sort.DEF) {
@@ -136,8 +135,7 @@ public final class AnalyzerCaster {
         return null;
     }
 
-    public static Type promoteNumeric(final Definition definition,
-                                      final Type from0, final Type from1, final boolean decimal, final boolean primitive) {
+    public static Type promoteNumeric(final Type from0, final Type from1, final boolean decimal, final boolean primitive) {
         final Sort sort0 = from0.sort;
         final Sort sort1 = from1.sort;
 
@@ -171,7 +169,7 @@ public final class AnalyzerCaster {
         return null;
     }
 
-    public static Type promoteAdd(final Definition definition, final Type from0, final Type from1) {
+    public static Type promoteAdd(final Type from0, final Type from1) {
         final Sort sort0 = from0.sort;
         final Sort sort1 = from1.sort;
 
@@ -179,10 +177,10 @@ public final class AnalyzerCaster {
             return Definition.stringType;
         }
 
-        return promoteNumeric(definition, from0, from1, true, true);
+        return promoteNumeric(from0, from1, true, true);
     }
 
-    public static Type promoteXor(final Definition definition, final Type from0, final Type from1) {
+    public static Type promoteXor(final Type from0, final Type from1) {
         final Sort sort0 = from0.sort;
         final Sort sort1 = from1.sort;
 
@@ -190,10 +188,10 @@ public final class AnalyzerCaster {
             return Definition.booleanType;
         }
 
-        return promoteNumeric(definition, from0, from1, false, true);
+        return promoteNumeric(from0, from1, false, true);
     }
 
-    public static Type promoteEquality(final Definition definition, final Type from0, final Type from1) {
+    public static Type promoteEquality(final Type from0, final Type from1) {
         final Sort sort0 = from0.sort;
         final Sort sort1 = from1.sort;
 
@@ -208,13 +206,13 @@ public final class AnalyzerCaster {
         }
 
         if (sort0.numeric && sort1.numeric) {
-            return promoteNumeric(definition, from0, from1, true, primitive);
+            return promoteNumeric(from0, from1, true, primitive);
         }
 
         return Definition.objectType;
     }
 
-    public static Type promoteReference(final Definition definition, final Type from0, final Type from1) {
+    public static Type promoteReference(final Type from0, final Type from1) {
         final Sort sort0 = from0.sort;
         final Sort sort1 = from1.sort;
 
@@ -228,15 +226,14 @@ public final class AnalyzerCaster {
             }
 
             if (sort0.numeric && sort1.numeric) {
-                return promoteNumeric(definition, from0, from1, true, true);
+                return promoteNumeric(from0, from1, true, true);
             }
         }
 
         return Definition.objectType;
     }
 
-    public static Type promoteConditional(final Definition definition,
-                                          final Type from0, final Type from1, final Object const0, final Object const1) {
+    public static Type promoteConditional(final Type from0, final Type from1, final Object const0, final Object const1) {
         if (from0.equals(from1)) {
             return from0;
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
index c487dddba71..65d7c78555c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
@@ -113,9 +113,9 @@ final class Compiler {
 
         final Reserved reserved = new Reserved();
         final SSource root = Walker.buildPainlessTree(source, reserved);
-        final Variables variables = Analyzer.analyze(settings, Definition.INSTANCE, reserved, root);
+        final Variables variables = Analyzer.analyze(settings, reserved, root);
 
-        return Writer.write(settings, Definition.INSTANCE, name, source, variables, root);
+        return Writer.write(settings, name, source, variables, root);
     }
 
     /**
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index de9e1c43bea..c4bebee7443 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -135,11 +135,10 @@ public final class Def {
      * @param receiverClass Class of the object to invoke the method on.
      * @param name Name of the method.
      * @param type Callsite signature. Need not match exactly, except the number of parameters.
-     * @param definition Whitelist to check.
      * @return pointer to matching method to invoke. never returns null.
      * @throws IllegalArgumentException if no matching whitelisted method was found.
      */
-     static MethodHandle lookupMethod(Class<?> receiverClass, String name, MethodType type, Definition definition) {
+     static MethodHandle lookupMethod(Class<?> receiverClass, String name, MethodType type) {
          // we don't consider receiver an argument/counting towards arity
          type = type.dropParameterTypes(0, 1);
          Definition.MethodKey key = new Definition.MethodKey(name, type.parameterCount());
@@ -193,11 +192,10 @@ public final class Def {
      * <p>
      * @param receiverClass Class of the object to retrieve the field from.
      * @param name Name of the field.
-     * @param definition Whitelist to check.
      * @return pointer to matching field. never returns null.
      * @throws IllegalArgumentException if no matching whitelisted field was found.
      */
-    static MethodHandle lookupGetter(Class<?> receiverClass, String name, Definition definition) {
+    static MethodHandle lookupGetter(Class<?> receiverClass, String name) {
         // first try whitelist
         for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
             RuntimeClass struct = Definition.getRuntimeClass(clazz);
@@ -264,11 +262,10 @@ public final class Def {
      * <p>
      * @param receiverClass Class of the object to retrieve the field from.
      * @param name Name of the field.
-     * @param definition Whitelist to check.
      * @return pointer to matching field. never returns null.
      * @throws IllegalArgumentException if no matching whitelisted field was found.
      */
-    static MethodHandle lookupSetter(Class<?> receiverClass, String name, Definition definition) {
+    static MethodHandle lookupSetter(Class<?> receiverClass, String name) {
         // first try whitelist
         for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
             RuntimeClass struct = Definition.getRuntimeClass(clazz);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index 380f5455ab3..40b9cc6cbe8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -94,11 +94,11 @@ public final class DefBootstrap {
         private static MethodHandle lookup(int flavor, Class<?> clazz, String name, MethodType type) {
             switch(flavor) {
                 case METHOD_CALL:
-                    return Def.lookupMethod(clazz, name, type, Definition.INSTANCE);
+                    return Def.lookupMethod(clazz, name, type);
                 case LOAD:
-                    return Def.lookupGetter(clazz, name, Definition.INSTANCE);
+                    return Def.lookupGetter(clazz, name);
                 case STORE:
-                    return Def.lookupSetter(clazz, name, Definition.INSTANCE);
+                    return Def.lookupSetter(clazz, name);
                 case ARRAY_LOAD:
                     return Def.lookupArrayLoad(clazz);
                 case ARRAY_STORE:
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 2557d41f6ae..0a81188e776 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -40,11 +40,8 @@ public final class Definition {
     
     private static final String DEFINITION_FILE = "definition.txt";
 
-    /**
-     * The default language API to be used with Painless.  The second construction is used
-     * to finalize all the variables, so there is no mistake of modification afterwards.
-     */
-    public static final Definition INSTANCE = new Definition(new Definition());
+    // The second construction is used to finalize all the variables, so there is no mistake of modification afterwards.
+    private static final Definition INSTANCE = new Definition(new Definition());
     
     /** Some native types as constants: */
     public static final Type voidType = getType("void");
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index ce1bb531655..cddd1326c74 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -200,9 +200,7 @@ public final class MethodWriter extends GeneratorAdapter {
         }
     }
 
-    public void writeBinaryInstruction(final Definition definition,
-                                              final String location,
-                                              final Type type, final Operation operation) {
+    public void writeBinaryInstruction(final String location, final Type type, final Operation operation) {
         final Sort sort = type.sort;
         
         if ((sort == Sort.FLOAT || sort == Sort.DOUBLE) &&
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
index 417eff84249..28a7416d800 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
@@ -86,14 +86,12 @@ public final class Variables {
         }
     }
 
-    private final Definition definition;
     final Reserved reserved;
 
     private final Deque<Integer> scopes = new ArrayDeque<>();
     private final Deque<Variable> variables = new ArrayDeque<>();
 
-    public Variables(final CompilerSettings settings, final Definition definition, final Reserved reserved) {
-        this.definition = definition;
+    public Variables(final CompilerSettings settings, final Reserved reserved) {
         this.reserved = reserved;
 
         incrementScope();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
index 449361867b9..977a591b7af 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
@@ -38,15 +38,14 @@ import static org.elasticsearch.painless.WriterConstants.MAP_TYPE;
  */
 final class Writer {
 
-    static byte[] write(final CompilerSettings settings, final Definition definition,
+    static byte[] write(final CompilerSettings settings,
                                String name, final String source, final Variables variables, final SSource root) {
-        final Writer writer = new Writer(settings, definition, name, source, variables, root);
+        final Writer writer = new Writer(settings, name, source, variables, root);
 
         return writer.getBytes();
     }
 
     private final CompilerSettings settings;
-    private final Definition definition;
     private final String scriptName;
     private final String source;
     private final Variables variables;
@@ -55,10 +54,9 @@ final class Writer {
     private final ClassWriter writer;
     private final MethodWriter adapter;
 
-    private Writer(final CompilerSettings settings, final Definition definition,
+    private Writer(final CompilerSettings settings,
                      String name, final String source, final Variables variables, final SSource root) {
         this.settings = settings;
-        this.definition = definition;
         this.scriptName = name;
         this.source = source;
         this.variables = variables;
@@ -177,7 +175,7 @@ final class Writer {
             adapter.visitVarInsn(Opcodes.ISTORE, loop.slot);
         }
 
-        root.write(settings, definition, adapter);
+        root.write(settings, adapter);
         adapter.endMethod();
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
index 528da4384c8..f741d62921f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.CompilerSettings;
-import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.AnalyzerCaster;
@@ -101,27 +100,27 @@ public abstract class AExpression extends ANode {
     /**
      * Checks for errors and collects data for the writing phase.
      */
-    abstract void analyze(final CompilerSettings settings, final Definition definition, final Variables variables);
+    abstract void analyze(final CompilerSettings settings, final Variables variables);
 
     /**
      * Writes ASM based on the data collected during the analysis phase.
      */
-    abstract void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter);
+    abstract void write(final CompilerSettings settings, final MethodWriter adapter);
 
     /**
      * Inserts {@link ECast} nodes into the tree for implicit casts.  Also replaces
      * nodes with the constant variable set to a non-null value with {@link EConstant}.
      * @return The new child node for the parent node calling this method.
      */
-    AExpression cast(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        final Cast cast = AnalyzerCaster.getLegalCast(definition, location, actual, expected, explicit);
+    AExpression cast(final CompilerSettings settings, final Variables variables) {
+        final Cast cast = AnalyzerCaster.getLegalCast(location, actual, expected, explicit);
 
         if (cast == null) {
             if (constant == null || this instanceof EConstant) {
                 return this;
             } else {
                 final EConstant econstant = new EConstant(line, location, constant);
-                econstant.analyze(settings, definition, variables);
+                econstant.analyze(settings, variables);
 
                 if (!expected.equals(econstant.actual)) {
                     throw new IllegalStateException(error("Illegal tree structure."));
@@ -142,7 +141,7 @@ public abstract class AExpression extends ANode {
                     constant = AnalyzerCaster.constCast(location, constant, cast);
 
                     final EConstant econstant = new EConstant(line, location, constant);
-                    econstant.analyze(settings, definition, variables);
+                    econstant.analyze(settings, variables);
 
                     if (!expected.equals(econstant.actual)) {
                         throw new IllegalStateException(error("Illegal tree structure."));
@@ -156,7 +155,7 @@ public abstract class AExpression extends ANode {
                     return ecast;
                 } else {
                     final EConstant econstant = new EConstant(line, location, constant);
-                    econstant.analyze(settings, definition, variables);
+                    econstant.analyze(settings, variables);
 
                     if (!actual.equals(econstant.actual)) {
                         throw new IllegalStateException(error("Illegal tree structure."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
index ffbfff112b3..8f8e226006e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.CompilerSettings;
-import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
@@ -87,22 +86,22 @@ public abstract class ALink extends ANode {
      * def or a shortcut is used. Otherwise, returns itself.  This will be
      * updated into the {@link EChain} node's list of links.
      */
-    abstract ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables);
+    abstract ALink analyze(final CompilerSettings settings, final Variables variables);
 
     /**
      * Write values before a load/store occurs such as an array index.
      */
-    abstract void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter);
+    abstract void write(final CompilerSettings settings, final MethodWriter adapter);
 
     /**
      * Write a load for the specific link type.
      */
-    abstract void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter);
+    abstract void load(final CompilerSettings settings, final MethodWriter adapter);
 
     /**
      * Write a store for the specific link type.
      */
-    abstract void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter);
+    abstract void store(final CompilerSettings settings, final MethodWriter adapter);
 
     /**
      * Used to copy link data from one to another during analysis in the case of replacement.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
index 569e8cfb03b..f80d5e7b4fd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.CompilerSettings;
-import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.MethodWriter;
@@ -116,10 +115,10 @@ public abstract class AStatement extends ANode {
     /**
      * Checks for errors and collects data for the writing phase.
      */
-    abstract void analyze(final CompilerSettings settings, final Definition definition, final Variables variables);
+    abstract void analyze(final CompilerSettings settings, final Variables variables);
 
     /**
      * Writes ASM based on the data collected during the analysis phase.
      */
-    abstract void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter);
+    abstract void write(final CompilerSettings settings, final MethodWriter adapter);
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 92ccf60a200..38871d007f4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -48,39 +48,39 @@ public final class EBinary extends AExpression {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         if (operation == Operation.MUL) {
-            analyzeMul(settings, definition, variables);
+            analyzeMul(settings, variables);
         } else if (operation == Operation.DIV) {
-            analyzeDiv(settings, definition, variables);
+            analyzeDiv(settings, variables);
         } else if (operation == Operation.REM) {
-            analyzeRem(settings, definition, variables);
+            analyzeRem(settings, variables);
         } else if (operation == Operation.ADD) {
-            analyzeAdd(settings, definition, variables);
+            analyzeAdd(settings, variables);
         } else if (operation == Operation.SUB) {
-            analyzeSub(settings, definition, variables);
+            analyzeSub(settings, variables);
         } else if (operation == Operation.LSH) {
-            analyzeLSH(settings, definition, variables);
+            analyzeLSH(settings, variables);
         } else if (operation == Operation.RSH) {
-            analyzeRSH(settings, definition, variables);
+            analyzeRSH(settings, variables);
         } else if (operation == Operation.USH) {
-            analyzeUSH(settings, definition, variables);
+            analyzeUSH(settings, variables);
         } else if (operation == Operation.BWAND) {
-            analyzeBWAnd(settings, definition, variables);
+            analyzeBWAnd(settings, variables);
         } else if (operation == Operation.XOR) {
-            analyzeXor(settings, definition, variables);
+            analyzeXor(settings, variables);
         } else if (operation == Operation.BWOR) {
-            analyzeBWOr(settings, definition, variables);
+            analyzeBWOr(settings, variables);
         } else {
             throw new IllegalStateException(error("Illegal tree structure."));
         }
     }
 
-    private void analyzeMul(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        left.analyze(settings, definition, variables);
-        right.analyze(settings, definition, variables);
+    private void analyzeMul(final CompilerSettings settings, final Variables variables) {
+        left.analyze(settings, variables);
+        right.analyze(settings, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply multiply [*] to types " +
@@ -90,8 +90,8 @@ public final class EBinary extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, definition, variables);
-        right = right.cast(settings, definition, variables);
+        left = left.cast(settings, variables);
+        right = right.cast(settings, variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -112,11 +112,11 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeDiv(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        left.analyze(settings, definition, variables);
-        right.analyze(settings, definition, variables);
+    private void analyzeDiv(final CompilerSettings settings, final Variables variables) {
+        left.analyze(settings, variables);
+        right.analyze(settings, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply divide [/] to types " +
@@ -126,8 +126,8 @@ public final class EBinary extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, definition, variables);
-        right = right.cast(settings, definition, variables);
+        left = left.cast(settings, variables);
+        right = right.cast(settings, variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -148,11 +148,11 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeRem(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        left.analyze(settings, definition, variables);
-        right.analyze(settings, definition, variables);
+    private void analyzeRem(final CompilerSettings settings, final Variables variables) {
+        left.analyze(settings, variables);
+        right.analyze(settings, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply remainder [%] to types " +
@@ -162,8 +162,8 @@ public final class EBinary extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, definition, variables);
-        right = right.cast(settings, definition, variables);
+        left = left.cast(settings, variables);
+        right = right.cast(settings, variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -184,11 +184,11 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeAdd(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        left.analyze(settings, definition, variables);
-        right.analyze(settings, definition, variables);
+    private void analyzeAdd(final CompilerSettings settings, final Variables variables) {
+        left.analyze(settings, variables);
+        right.analyze(settings, variables);
 
-        final Type promote = AnalyzerCaster.promoteAdd(definition, left.actual, right.actual);
+        final Type promote = AnalyzerCaster.promoteAdd(left.actual, right.actual);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply add [+] to types " +
@@ -214,8 +214,8 @@ public final class EBinary extends AExpression {
             right.expected = promote;
         }
 
-        left = left.cast(settings, definition, variables);
-        right = right.cast(settings, definition, variables);
+        left = left.cast(settings, variables);
+        right = right.cast(settings, variables);
 
         if (left.constant != null && right.constant != null) {
             if (sort == Sort.INT) {
@@ -236,11 +236,11 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeSub(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        left.analyze(settings, definition, variables);
-        right.analyze(settings, definition, variables);
+    private void analyzeSub(final CompilerSettings settings, final Variables variables) {
+        left.analyze(settings, variables);
+        right.analyze(settings, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply subtract [-] to types " +
@@ -250,8 +250,8 @@ public final class EBinary extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, definition, variables);
-        right = right.cast(settings, definition, variables);
+        left = left.cast(settings, variables);
+        right = right.cast(settings, variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -272,11 +272,11 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeLSH(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        left.analyze(settings, definition, variables);
-        right.analyze(settings, definition, variables);
+    private void analyzeLSH(final CompilerSettings settings, final Variables variables) {
+        left.analyze(settings, variables);
+        right.analyze(settings, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, false, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, false, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply left shift [<<] to types " +
@@ -287,8 +287,8 @@ public final class EBinary extends AExpression {
         right.expected = Definition.intType;
         right.explicit = true;
 
-        left = left.cast(settings, definition, variables);
-        right = right.cast(settings, definition, variables);
+        left = left.cast(settings, variables);
+        right = right.cast(settings, variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -305,11 +305,11 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeRSH(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        left.analyze(settings, definition, variables);
-        right.analyze(settings, definition, variables);
+    private void analyzeRSH(final CompilerSettings settings, final Variables variables) {
+        left.analyze(settings, variables);
+        right.analyze(settings, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, false, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, false, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply right shift [>>] to types " +
@@ -320,8 +320,8 @@ public final class EBinary extends AExpression {
         right.expected = Definition.intType;
         right.explicit = true;
 
-        left = left.cast(settings, definition, variables);
-        right = right.cast(settings, definition, variables);
+        left = left.cast(settings, variables);
+        right = right.cast(settings, variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -338,11 +338,11 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeUSH(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        left.analyze(settings, definition, variables);
-        right.analyze(settings, definition, variables);
+    private void analyzeUSH(final CompilerSettings settings, final Variables variables) {
+        left.analyze(settings, variables);
+        right.analyze(settings, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, false, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, false, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply unsigned shift [>>>] to types " +
@@ -353,8 +353,8 @@ public final class EBinary extends AExpression {
         right.expected = Definition.intType;
         right.explicit = true;
 
-        left = left.cast(settings, definition, variables);
-        right = right.cast(settings, definition, variables);
+        left = left.cast(settings, variables);
+        right = right.cast(settings, variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -371,11 +371,11 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeBWAnd(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        left.analyze(settings, definition, variables);
-        right.analyze(settings, definition, variables);
+    private void analyzeBWAnd(final CompilerSettings settings, final Variables variables) {
+        left.analyze(settings, variables);
+        right.analyze(settings, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, false, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply and [&] to types " +
@@ -385,8 +385,8 @@ public final class EBinary extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, definition, variables);
-        right = right.cast(settings, definition, variables);
+        left = left.cast(settings, variables);
+        right = right.cast(settings, variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -403,11 +403,11 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeXor(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        left.analyze(settings, definition, variables);
-        right.analyze(settings, definition, variables);
+    private void analyzeXor(final CompilerSettings settings, final Variables variables) {
+        left.analyze(settings, variables);
+        right.analyze(settings, variables);
 
-        final Type promote = AnalyzerCaster.promoteXor(definition, left.actual, right.actual);
+        final Type promote = AnalyzerCaster.promoteXor(left.actual, right.actual);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply xor [^] to types " +
@@ -417,8 +417,8 @@ public final class EBinary extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, definition, variables);
-        right = right.cast(settings, definition, variables);
+        left = left.cast(settings, variables);
+        right = right.cast(settings, variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -437,11 +437,11 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeBWOr(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        left.analyze(settings, definition, variables);
-        right.analyze(settings, definition, variables);
+    private void analyzeBWOr(final CompilerSettings settings, final Variables variables) {
+        left.analyze(settings, variables);
+        right.analyze(settings, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, false, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply or [|] to types " +
@@ -451,8 +451,8 @@ public final class EBinary extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, definition, variables);
-        right = right.cast(settings, definition, variables);
+        left = left.cast(settings, variables);
+        right = right.cast(settings, variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -470,19 +470,19 @@ public final class EBinary extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         if (actual.sort == Sort.STRING && operation == Operation.ADD) {
             if (!cat) {
                 adapter.writeNewStrings();
             }
 
-            left.write(settings, definition, adapter);
+            left.write(settings, adapter);
 
             if (!(left instanceof EBinary) || ((EBinary)left).operation != Operation.ADD || left.actual.sort != Sort.STRING) {
                 adapter.writeAppendStrings(left.actual);
             }
 
-            right.write(settings, definition, adapter);
+            right.write(settings, adapter);
 
             if (!(right instanceof EBinary) || ((EBinary)right).operation != Operation.ADD || right.actual.sort != Sort.STRING) {
                 adapter.writeAppendStrings(right.actual);
@@ -492,10 +492,10 @@ public final class EBinary extends AExpression {
                 adapter.writeToStrings();
             }
         } else {
-            left.write(settings, definition, adapter);
-            right.write(settings, definition, adapter);
+            left.write(settings, adapter);
+            right.write(settings, adapter);
 
-            adapter.writeBinaryInstruction(definition, location, actual, operation);
+            adapter.writeBinaryInstruction(location, actual, operation);
         }
 
         adapter.writeBranch(tru, fals);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
index 1cd2564b0e1..28072b9d9bd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
@@ -44,14 +44,14 @@ public final class EBool extends AExpression {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         left.expected = Definition.booleanType;
-        left.analyze(settings, definition, variables);
-        left = left.cast(settings, definition, variables);
+        left.analyze(settings, variables);
+        left = left.cast(settings, variables);
 
         right.expected = Definition.booleanType;
-        right.analyze(settings, definition, variables);
-        right = right.cast(settings, definition, variables);
+        right.analyze(settings, variables);
+        right = right.cast(settings, variables);
 
         if (left.constant != null && right.constant != null) {
             if (operation == Operation.AND) {
@@ -67,7 +67,7 @@ public final class EBool extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         if (tru != null || fals != null) {
             if (operation == Operation.AND) {
                 final Label localfals = fals == null ? new Label() : fals;
@@ -76,8 +76,8 @@ public final class EBool extends AExpression {
                 right.tru = tru;
                 right.fals = fals;
 
-                left.write(settings, definition, adapter);
-                right.write(settings, definition, adapter);
+                left.write(settings, adapter);
+                right.write(settings, adapter);
 
                 if (fals == null) {
                     adapter.mark(localfals);
@@ -89,8 +89,8 @@ public final class EBool extends AExpression {
                 right.tru = tru;
                 right.fals = fals;
 
-                left.write(settings, definition, adapter);
-                right.write(settings, definition, adapter);
+                left.write(settings, adapter);
+                right.write(settings, adapter);
 
                 if (tru == null) {
                     adapter.mark(localtru);
@@ -106,8 +106,8 @@ public final class EBool extends AExpression {
                 left.fals = localfals;
                 right.fals = localfals;
 
-                left.write(settings, definition, adapter);
-                right.write(settings, definition, adapter);
+                left.write(settings, adapter);
+                right.write(settings, adapter);
 
                 adapter.push(true);
                 adapter.goTo(end);
@@ -122,8 +122,8 @@ public final class EBool extends AExpression {
                 left.tru = localtru;
                 right.fals = localfals;
 
-                left.write(settings, definition, adapter);
-                right.write(settings, definition, adapter);
+                left.write(settings, adapter);
+                right.write(settings, adapter);
 
                 adapter.mark(localtru);
                 adapter.push(true);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
index 24f1ac899e8..979fbed03e3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
@@ -36,12 +36,12 @@ public final class EBoolean extends AExpression {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         actual = Definition.booleanType;
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         throw new IllegalArgumentException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
index d68c95c910a..c46957f78d9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.CompilerSettings;
-import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
@@ -46,13 +45,13 @@ final class ECast extends AExpression {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
-        child.write(settings, definition, adapter);
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
+        child.write(settings, adapter);
         adapter.writeCast(cast);
         adapter.writeBranch(tru, fals);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index a3c0c656fda..d992edd8505 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -59,20 +59,20 @@ public final class EChain extends AExpression {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        analyzeLinks(settings, definition, variables);
+    void analyze(final CompilerSettings settings, final Variables variables) {
+        analyzeLinks(settings, variables);
         analyzeIncrDecr();
 
         if (operation != null) {
-            analyzeCompound(settings, definition, variables);
+            analyzeCompound(settings, variables);
         } else if (expression != null) {
-            analyzeWrite(settings, definition, variables);
+            analyzeWrite(settings, variables);
         } else {
             analyzeRead();
         }
     }
 
-    private void analyzeLinks(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    private void analyzeLinks(final CompilerSettings settings, final Variables variables) {
         ALink previous = null;
         int index = 0;
 
@@ -92,7 +92,7 @@ public final class EChain extends AExpression {
                 current.store = expression != null || pre || post;
             }
 
-            final ALink analyzed = current.analyze(settings, definition, variables);
+            final ALink analyzed = current.analyze(settings, variables);
 
             if (analyzed == null) {
                 links.remove(index);
@@ -153,33 +153,33 @@ public final class EChain extends AExpression {
         }
     }
 
-    private void analyzeCompound(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    private void analyzeCompound(final CompilerSettings settings, final Variables variables) {
         final ALink last = links.get(links.size() - 1);
 
-        expression.analyze(settings, definition, variables);
+        expression.analyze(settings, variables);
 
         if (operation == Operation.MUL) {
-            promote = AnalyzerCaster.promoteNumeric(definition, last.after, expression.actual, true, true);
+            promote = AnalyzerCaster.promoteNumeric(last.after, expression.actual, true, true);
         } else if (operation == Operation.DIV) {
-            promote = AnalyzerCaster.promoteNumeric(definition, last.after, expression.actual, true, true);
+            promote = AnalyzerCaster.promoteNumeric(last.after, expression.actual, true, true);
         } else if (operation == Operation.REM) {
-            promote = AnalyzerCaster.promoteNumeric(definition, last.after, expression.actual, true, true);
+            promote = AnalyzerCaster.promoteNumeric(last.after, expression.actual, true, true);
         } else if (operation == Operation.ADD) {
-            promote = AnalyzerCaster.promoteAdd(definition, last.after, expression.actual);
+            promote = AnalyzerCaster.promoteAdd(last.after, expression.actual);
         } else if (operation == Operation.SUB) {
-            promote = AnalyzerCaster.promoteNumeric(definition, last.after, expression.actual, true, true);
+            promote = AnalyzerCaster.promoteNumeric(last.after, expression.actual, true, true);
         } else if (operation == Operation.LSH) {
-            promote = AnalyzerCaster.promoteNumeric(definition, last.after, false, true);
+            promote = AnalyzerCaster.promoteNumeric(last.after, false, true);
         } else if (operation == Operation.RSH) {
-            promote = AnalyzerCaster.promoteNumeric(definition, last.after, false, true);
+            promote = AnalyzerCaster.promoteNumeric(last.after, false, true);
         } else if (operation == Operation.USH) {
-            promote = AnalyzerCaster.promoteNumeric(definition, last.after, false, true);
+            promote = AnalyzerCaster.promoteNumeric(last.after, false, true);
         } else if (operation == Operation.BWAND) {
-            promote = AnalyzerCaster.promoteXor(definition, last.after, expression.actual);
+            promote = AnalyzerCaster.promoteXor(last.after, expression.actual);
         } else if (operation == Operation.XOR) {
-            promote = AnalyzerCaster.promoteXor(definition, last.after, expression.actual);
+            promote = AnalyzerCaster.promoteXor(last.after, expression.actual);
         } else if (operation == Operation.BWOR) {
-            promote = AnalyzerCaster.promoteXor(definition, last.after, expression.actual);
+            promote = AnalyzerCaster.promoteXor(last.after, expression.actual);
         } else {
             throw new IllegalStateException(error("Illegal tree structure."));
         }
@@ -205,30 +205,30 @@ public final class EChain extends AExpression {
             expression.expected = promote;
         }
 
-        expression = expression.cast(settings, definition, variables);
+        expression = expression.cast(settings, variables);
 
-        there = AnalyzerCaster.getLegalCast(definition, location, last.after, promote, false);
-        back = AnalyzerCaster.getLegalCast(definition, location, promote, last.after, true);
+        there = AnalyzerCaster.getLegalCast(location, last.after, promote, false);
+        back = AnalyzerCaster.getLegalCast(location, promote, last.after, true);
 
         this.statement = true;
         this.actual = read ? last.after : Definition.voidType;
     }
 
-    private void analyzeWrite(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    private void analyzeWrite(final CompilerSettings settings, final Variables variables) {
         final ALink last = links.get(links.size() - 1);
 
         // If the store node is a DEF node, we remove the cast to DEF from the expression
         // and promote the real type to it:
         if (last instanceof IDefLink) {
-            expression.analyze(settings, definition, variables);
+            expression.analyze(settings, variables);
             last.after = expression.expected = expression.actual;
         } else {
             // otherwise we adapt the type of the expression to the store type
             expression.expected = last.after;
-            expression.analyze(settings, definition, variables);
+            expression.analyze(settings, variables);
         }
 
-        expression = expression.cast(settings, definition, variables);
+        expression = expression.cast(settings, variables);
 
         this.statement = true;
         this.actual = read ? last.after : Definition.voidType;
@@ -248,7 +248,7 @@ public final class EChain extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings,final MethodWriter adapter) {
         if (cat) {
             adapter.writeNewStrings();
         }
@@ -256,15 +256,15 @@ public final class EChain extends AExpression {
         final ALink last = links.get(links.size() - 1);
 
         for (final ALink link : links) {
-            link.write(settings, definition, adapter);
+            link.write(settings, adapter);
 
             if (link == last && link.store) {
                 if (cat) {
                     adapter.writeDup(link.size, 1);
-                    link.load(settings, definition, adapter);
+                    link.load(settings, adapter);
                     adapter.writeAppendStrings(link.after);
 
-                    expression.write(settings, definition, adapter);
+                    expression.write(settings, adapter);
 
                     if (!(expression instanceof EBinary) ||
                         ((EBinary)expression).operation != Operation.ADD || expression.actual.sort != Sort.STRING) {
@@ -278,18 +278,18 @@ public final class EChain extends AExpression {
                         adapter.writeDup(link.after.sort.size, link.size);
                     }
 
-                    link.store(settings, definition, adapter);
+                    link.store(settings, adapter);
                 } else if (operation != null) {
                     adapter.writeDup(link.size, 0);
-                    link.load(settings, definition, adapter);
+                    link.load(settings, adapter);
 
                     if (link.load && post) {
                         adapter.writeDup(link.after.sort.size, link.size);
                     }
 
                     adapter.writeCast(there);
-                    expression.write(settings, definition, adapter);
-                    adapter.writeBinaryInstruction(definition, location, promote, operation);
+                    expression.write(settings, adapter);
+                    adapter.writeBinaryInstruction(location, promote, operation);
 
                     adapter.writeCast(back);
 
@@ -297,18 +297,18 @@ public final class EChain extends AExpression {
                         adapter.writeDup(link.after.sort.size, link.size);
                     }
 
-                    link.store(settings, definition, adapter);
+                    link.store(settings, adapter);
                 } else {
-                    expression.write(settings, definition, adapter);
+                    expression.write(settings, adapter);
 
                     if (link.load) {
                         adapter.writeDup(link.after.sort.size, link.size);
                     }
 
-                    link.store(settings, definition, adapter);
+                    link.store(settings, adapter);
                 }
             } else {
-                link.load(settings, definition, adapter);
+                link.load(settings, adapter);
             }
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index 82033124e1c..68c45f4870e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -54,33 +54,33 @@ public final class EComp extends AExpression {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         if (operation == Operation.EQ) {
-            analyzeEq(settings, definition, variables);
+            analyzeEq(settings, variables);
         } else if (operation == Operation.EQR) {
-            analyzeEqR(settings, definition, variables);
+            analyzeEqR(settings, variables);
         } else if (operation == Operation.NE) {
-            analyzeNE(settings, definition, variables);
+            analyzeNE(settings, variables);
         } else if (operation == Operation.NER) {
-            analyzeNER(settings, definition, variables);
+            analyzeNER(settings, variables);
         } else if (operation == Operation.GTE) {
-            analyzeGTE(settings, definition, variables);
+            analyzeGTE(settings, variables);
         } else if (operation == Operation.GT) {
-            analyzeGT(settings, definition, variables);
+            analyzeGT(settings, variables);
         } else if (operation == Operation.LTE) {
-            analyzeLTE(settings, definition, variables);
+            analyzeLTE(settings, variables);
         } else if (operation == Operation.LT) {
-            analyzeLT(settings, definition, variables);
+            analyzeLT(settings, variables);
         } else {
             throw new IllegalStateException(error("Illegal tree structure."));
         }
     }
 
-    private void analyzeEq(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        left.analyze(settings, definition, variables);
-        right.analyze(settings, definition, variables);
+    private void analyzeEq(final CompilerSettings settings, final Variables variables) {
+        left.analyze(settings, variables);
+        right.analyze(settings, variables);
 
-        final Type promote = AnalyzerCaster.promoteEquality(definition, left.actual, right.actual);
+        final Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply equals [==] to types " +
@@ -90,8 +90,8 @@ public final class EComp extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, definition, variables);
-        right = right.cast(settings, definition, variables);
+        left = left.cast(settings, variables);
+        right = right.cast(settings, variables);
 
         if (left.isNull && right.isNull) {
             throw new IllegalArgumentException(error("Extraneous comparison of null constants."));
@@ -122,11 +122,11 @@ public final class EComp extends AExpression {
         actual = Definition.booleanType;
     }
 
-    private void analyzeEqR(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        left.analyze(settings, definition, variables);
-        right.analyze(settings, definition, variables);
+    private void analyzeEqR(final CompilerSettings settings, final Variables variables) {
+        left.analyze(settings, variables);
+        right.analyze(settings, variables);
 
-        final Type promote = AnalyzerCaster.promoteReference(definition, left.actual, right.actual);
+        final Type promote = AnalyzerCaster.promoteReference(left.actual, right.actual);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply reference equals [===] to types " +
@@ -136,8 +136,8 @@ public final class EComp extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, definition, variables);
-        right = right.cast(settings, definition, variables);
+        left = left.cast(settings, variables);
+        right = right.cast(settings, variables);
 
         if (left.isNull && right.isNull) {
             throw new IllegalArgumentException(error("Extraneous comparison of null constants."));
@@ -164,11 +164,11 @@ public final class EComp extends AExpression {
         actual = Definition.booleanType;
     }
 
-    private void analyzeNE(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        left.analyze(settings, definition, variables);
-        right.analyze(settings, definition, variables);
+    private void analyzeNE(final CompilerSettings settings, final Variables variables) {
+        left.analyze(settings, variables);
+        right.analyze(settings, variables);
 
-        final Type promote = AnalyzerCaster.promoteEquality(definition, left.actual, right.actual);
+        final Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply not equals [!=] to types " +
@@ -178,8 +178,8 @@ public final class EComp extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, definition, variables);
-        right = right.cast(settings, definition, variables);
+        left = left.cast(settings, variables);
+        right = right.cast(settings, variables);
 
         if (left.isNull && right.isNull) {
             throw new IllegalArgumentException(error("Extraneous comparison of null constants."));
@@ -210,11 +210,11 @@ public final class EComp extends AExpression {
         actual = Definition.booleanType;
     }
 
-    private void analyzeNER(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        left.analyze(settings, definition, variables);
-        right.analyze(settings, definition, variables);
+    private void analyzeNER(final CompilerSettings settings, final Variables variables) {
+        left.analyze(settings, variables);
+        right.analyze(settings, variables);
 
-        final Type promote = AnalyzerCaster.promoteReference(definition, left.actual, right.actual);
+        final Type promote = AnalyzerCaster.promoteReference(left.actual, right.actual);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply reference not equals [!==] to types " +
@@ -224,8 +224,8 @@ public final class EComp extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, definition, variables);
-        right = right.cast(settings, definition, variables);
+        left = left.cast(settings, variables);
+        right = right.cast(settings, variables);
 
         if (left.isNull && right.isNull) {
             throw new IllegalArgumentException(error("Extraneous comparison of null constants."));
@@ -252,11 +252,11 @@ public final class EComp extends AExpression {
         actual = Definition.booleanType;
     }
 
-    private void analyzeGTE(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        left.analyze(settings, definition, variables);
-        right.analyze(settings, definition, variables);
+    private void analyzeGTE(final CompilerSettings settings, final Variables variables) {
+        left.analyze(settings, variables);
+        right.analyze(settings, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply greater than or equals [>=] to types " +
@@ -266,8 +266,8 @@ public final class EComp extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, definition, variables);
-        right = right.cast(settings, definition, variables);
+        left = left.cast(settings, variables);
+        right = right.cast(settings, variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -288,11 +288,11 @@ public final class EComp extends AExpression {
         actual = Definition.booleanType;
     }
 
-    private void analyzeGT(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        left.analyze(settings, definition, variables);
-        right.analyze(settings, definition, variables);
+    private void analyzeGT(final CompilerSettings settings, final Variables variables) {
+        left.analyze(settings, variables);
+        right.analyze(settings, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply greater than [>] to types " +
@@ -302,8 +302,8 @@ public final class EComp extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, definition, variables);
-        right = right.cast(settings, definition, variables);
+        left = left.cast(settings, variables);
+        right = right.cast(settings, variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -324,11 +324,11 @@ public final class EComp extends AExpression {
         actual = Definition.booleanType;
     }
 
-    private void analyzeLTE(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        left.analyze(settings, definition, variables);
-        right.analyze(settings, definition, variables);
+    private void analyzeLTE(final CompilerSettings settings, final Variables variables) {
+        left.analyze(settings, variables);
+        right.analyze(settings, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply less than or equals [<=] to types " +
@@ -338,8 +338,8 @@ public final class EComp extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, definition, variables);
-        right = right.cast(settings, definition, variables);
+        left = left.cast(settings, variables);
+        right = right.cast(settings, variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -360,11 +360,11 @@ public final class EComp extends AExpression {
         actual = Definition.booleanType;
     }
 
-    private void analyzeLT(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        left.analyze(settings, definition, variables);
-        right.analyze(settings, definition, variables);
+    private void analyzeLT(final CompilerSettings settings, final Variables variables) {
+        left.analyze(settings, variables);
+        right.analyze(settings, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, left.actual, right.actual, true, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply less than [>=] to types " +
@@ -374,8 +374,8 @@ public final class EComp extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, definition, variables);
-        right = right.cast(settings, definition, variables);
+        left = left.cast(settings, variables);
+        right = right.cast(settings, variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -397,15 +397,15 @@ public final class EComp extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         final boolean branch = tru != null || fals != null;
         final org.objectweb.asm.Type rtype = right.actual.type;
         final Sort rsort = right.actual.sort;
 
-        left.write(settings, definition, adapter);
+        left.write(settings, adapter);
 
         if (!right.isNull) {
-            right.write(settings, definition, adapter);
+            right.write(settings, adapter);
         }
 
         final Label jump = tru != null ? tru : fals != null ? fals : new Label();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
index c1b37dfeee0..e084974fb33 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
@@ -46,10 +46,10 @@ public final class EConditional extends AExpression {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         condition.expected = Definition.booleanType;
-        condition.analyze(settings, definition, variables);
-        condition = condition.cast(settings, definition, variables);
+        condition.analyze(settings, variables);
+        condition = condition.cast(settings, variables);
 
         if (condition.constant != null) {
             throw new IllegalArgumentException(error("Extraneous conditional statement."));
@@ -61,23 +61,23 @@ public final class EConditional extends AExpression {
         right.explicit = explicit;
         actual = expected;
 
-        left.analyze(settings, definition, variables);
-        right.analyze(settings, definition, variables);
+        left.analyze(settings, variables);
+        right.analyze(settings, variables);
 
         if (expected == null) {
-            final Type promote = AnalyzerCaster.promoteConditional(definition, left.actual, right.actual, left.constant, right.constant);
+            final Type promote = AnalyzerCaster.promoteConditional(left.actual, right.actual, left.constant, right.constant);
 
             left.expected = promote;
             right.expected = promote;
             actual = promote;
         }
 
-        left = left.cast(settings, definition, variables);
-        right = right.cast(settings, definition, variables);
+        left = left.cast(settings, variables);
+        right = right.cast(settings, variables);
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         final Label localfals = new Label();
         final Label end = new Label();
 
@@ -85,11 +85,11 @@ public final class EConditional extends AExpression {
         left.tru = right.tru = tru;
         left.fals = right.fals = fals;
 
-        condition.write(settings, definition, adapter);
-        left.write(settings, definition, adapter);
+        condition.write(settings, adapter);
+        left.write(settings, adapter);
         adapter.goTo(end);
         adapter.mark(localfals);
-        right.write(settings, definition, adapter);
+        right.write(settings, adapter);
         adapter.mark(end);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
index cbf547a0417..e590ee41852 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
@@ -38,7 +38,7 @@ final class EConstant extends AExpression {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         if (constant instanceof String) {
             actual = Definition.stringType;
         } else if (constant instanceof Double) {
@@ -63,7 +63,7 @@ final class EConstant extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         final Sort sort = actual.sort;
 
         switch (sort) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
index 820c3b3926e..3f1d3a3fc26 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
@@ -38,7 +38,7 @@ public final class EDecimal extends AExpression {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         if (value.endsWith("f") || value.endsWith("F")) {
             try {
                 constant = Float.parseFloat(value.substring(0, value.length() - 1));
@@ -57,7 +57,7 @@ public final class EDecimal extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         throw new IllegalArgumentException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
index 0ed6fccf35f..e0afd8f822a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
@@ -40,7 +40,7 @@ public final class EExplicit extends AExpression {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         try {
             actual = Definition.getType(this.type);
         } catch (final IllegalArgumentException exception) {
@@ -49,19 +49,19 @@ public final class EExplicit extends AExpression {
 
         child.expected = actual;
         child.explicit = true;
-        child.analyze(settings, definition, variables);
-        child = child.cast(settings, definition, variables);
+        child.analyze(settings, variables);
+        child = child.cast(settings, variables);
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         throw new IllegalArgumentException(error("Illegal tree structure."));
     }
 
-    AExpression cast(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    AExpression cast(final CompilerSettings settings, final Variables variables) {
         child.expected = expected;
         child.explicit = explicit;
 
-        return child.cast(settings, definition, variables);
+        return child.cast(settings, variables);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
index 3e003fefe5c..3fcabccff17 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
@@ -35,7 +35,7 @@ public final class ENull extends AExpression {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         isNull = true;
 
         if (expected != null) {
@@ -50,7 +50,7 @@ public final class ENull extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         adapter.visitInsn(Opcodes.ACONST_NULL);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
index e8cf202d91d..e68de5b4128 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
@@ -41,7 +41,7 @@ public final class ENumeric extends AExpression {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         if (value.endsWith("d") || value.endsWith("D")) {
             if (radix != 10) {
                 throw new IllegalStateException(error("Invalid tree structure."));
@@ -96,7 +96,7 @@ public final class ENumeric extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         throw new IllegalArgumentException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index 627eb85e968..8ab97dcbde6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -48,24 +48,24 @@ public final class EUnary extends AExpression {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         if (operation == Operation.NOT) {
-            analyzeNot(settings, definition, variables);
+            analyzeNot(settings, variables);
         } else if (operation == Operation.BWNOT) {
-            analyzeBWNot(settings, definition, variables);
+            analyzeBWNot(settings, variables);
         } else if (operation == Operation.ADD) {
-            analyzerAdd(settings, definition, variables);
+            analyzerAdd(settings, variables);
         } else if (operation == Operation.SUB) {
-            analyzerSub(settings, definition, variables);
+            analyzerSub(settings, variables);
         } else {
             throw new IllegalStateException(error("Illegal tree structure."));
         }
     }
 
-    void analyzeNot(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyzeNot(final CompilerSettings settings, final Variables variables) {
         child.expected = Definition.booleanType;
-        child.analyze(settings, definition, variables);
-        child = child.cast(settings, definition, variables);
+        child.analyze(settings, variables);
+        child = child.cast(settings, variables);
 
         if (child.constant != null) {
             constant = !(boolean)child.constant;
@@ -74,17 +74,17 @@ public final class EUnary extends AExpression {
         actual = Definition.booleanType;
     }
 
-    void analyzeBWNot(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        child.analyze(settings, definition, variables);
+    void analyzeBWNot(final CompilerSettings settings, final Variables variables) {
+        child.analyze(settings, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, child.actual, false, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(child.actual, false, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply not [~] to type [" + child.actual.name + "]."));
         }
 
         child.expected = promote;
-        child = child.cast(settings, definition, variables);
+        child = child.cast(settings, variables);
 
         if (child.constant != null) {
             final Sort sort = promote.sort;
@@ -101,17 +101,17 @@ public final class EUnary extends AExpression {
         actual = promote;
     }
 
-    void analyzerAdd(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        child.analyze(settings, definition, variables);
+    void analyzerAdd(final CompilerSettings settings, final Variables variables) {
+        child.analyze(settings, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, child.actual, true, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(child.actual, true, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply positive [+] to type [" + child.actual.name + "]."));
         }
 
         child.expected = promote;
-        child = child.cast(settings, definition, variables);
+        child = child.cast(settings, variables);
 
         if (child.constant != null) {
             final Sort sort = promote.sort;
@@ -132,17 +132,17 @@ public final class EUnary extends AExpression {
         actual = promote;
     }
 
-    void analyzerSub(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        child.analyze(settings, definition, variables);
+    void analyzerSub(final CompilerSettings settings, final Variables variables) {
+        child.analyze(settings, variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(definition, child.actual, true, true);
+        final Type promote = AnalyzerCaster.promoteNumeric(child.actual, true, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply negative [-] to type [" + child.actual.name + "]."));
         }
 
         child.expected = promote;
-        child = child.cast(settings, definition, variables);
+        child = child.cast(settings, variables);
 
         if (child.constant != null) {
             final Sort sort = promote.sort;
@@ -164,14 +164,14 @@ public final class EUnary extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         if (operation == Operation.NOT) {
             if (tru == null && fals == null) {
                 final Label localfals = new Label();
                 final Label end = new Label();
 
                 child.fals = localfals;
-                child.write(settings, definition, adapter);
+                child.write(settings, adapter);
 
                 adapter.push(false);
                 adapter.goTo(end);
@@ -181,13 +181,13 @@ public final class EUnary extends AExpression {
             } else {
                 child.tru = fals;
                 child.fals = tru;
-                child.write(settings, definition, adapter);
+                child.write(settings, adapter);
             }
         } else {
             final org.objectweb.asm.Type type = actual.type;
             final Sort sort = actual.sort;
 
-            child.write(settings, definition, adapter);
+            child.write(settings, adapter);
 
             if (operation == Operation.BWNOT) {
                 if (sort == Sort.DEF) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
index a8047047f93..c4026bb529e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
@@ -38,7 +38,7 @@ public final class LArrayLength extends ALink {
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    ALink analyze(final CompilerSettings settings, final Variables variables) {
         if ("length".equals(value)) {
             if (!load) {
                 throw new IllegalArgumentException(error("Must read array field [length]."));
@@ -55,17 +55,17 @@ public final class LArrayLength extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void load(final CompilerSettings settings, final MethodWriter adapter) {
         adapter.arrayLength();
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void store(final CompilerSettings settings, final MethodWriter adapter) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
index 045c5d0e5fa..2a8da6e1892 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
@@ -42,7 +42,7 @@ public final class LBrace extends ALink {
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    ALink analyze(final CompilerSettings settings, final Variables variables) {
         if (before == null) {
             throw new IllegalStateException(error("Illegal tree structure."));
         }
@@ -51,35 +51,35 @@ public final class LBrace extends ALink {
 
         if (sort == Sort.ARRAY) {
             index.expected = Definition.intType;
-            index.analyze(settings, definition, variables);
-            index = index.cast(settings, definition, variables);
+            index.analyze(settings, variables);
+            index = index.cast(settings, variables);
 
             after = Definition.getType(before.struct, before.dimensions - 1);
 
             return this;
         } else if (sort == Sort.DEF) {
-            return new LDefArray(line, location, index).copy(this).analyze(settings, definition, variables);
+            return new LDefArray(line, location, index).copy(this).analyze(settings, variables);
         } else if (Map.class.isAssignableFrom(before.clazz)) {
-            return new LMapShortcut(line, location, index).copy(this).analyze(settings, definition, variables);
+            return new LMapShortcut(line, location, index).copy(this).analyze(settings, variables);
         } else if (List.class.isAssignableFrom(before.clazz)) {
-            return new LListShortcut(line, location, index).copy(this).analyze(settings, definition, variables);
+            return new LListShortcut(line, location, index).copy(this).analyze(settings, variables);
         }
 
         throw new IllegalArgumentException(error("Illegal array access on type [" + before.name + "]."));
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
-        index.write(settings, definition, adapter);
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
+        index.write(settings, adapter);
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void load(final CompilerSettings settings, final MethodWriter adapter) {
         adapter.arrayLoad(after.type);
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void store(final CompilerSettings settings, final MethodWriter adapter) {
         adapter.arrayStore(after.type);
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
index bd76aa293cc..150c9097ddd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
@@ -46,7 +46,7 @@ public final class LCall extends ALink {
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    ALink analyze(final CompilerSettings settings, final Variables variables) {
         if (before == null) {
             throw new IllegalStateException(error("Illegal tree structure."));
         } else if (before.sort == Definition.Sort.ARRAY) {
@@ -64,8 +64,8 @@ public final class LCall extends ALink {
                 final AExpression expression = arguments.get(argument);
 
                 expression.expected = method.arguments.get(argument);
-                expression.analyze(settings, definition, variables);
-                arguments.set(argument, expression.cast(settings, definition, variables));
+                expression.analyze(settings, variables);
+                arguments.set(argument, expression.cast(settings, variables));
             }
 
             statement = true;
@@ -76,7 +76,7 @@ public final class LCall extends ALink {
             final ALink link = new LDefCall(line, location, name, arguments);
             link.copy(this);
 
-            return link.analyze(settings, definition, variables);
+            return link.analyze(settings, variables);
         }
 
         throw new IllegalArgumentException(error("Unknown call [" + name + "] with [" + arguments.size() + 
@@ -84,14 +84,14 @@ public final class LCall extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void load(final CompilerSettings settings, final MethodWriter adapter) {
         for (final AExpression argument : arguments) {
-            argument.write(settings, definition, adapter);
+            argument.write(settings, adapter);
         }
 
         if (java.lang.reflect.Modifier.isStatic(method.reflect.getModifiers())) {
@@ -108,7 +108,7 @@ public final class LCall extends ALink {
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void store(final CompilerSettings settings, final MethodWriter adapter) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
index 6d618eb4a26..f8a03085124 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
@@ -42,7 +42,7 @@ public final class LCast extends ALink {
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    ALink analyze(final CompilerSettings settings, final Variables variables) {
         if (before == null) {
             throw new IllegalStateException(error("Illegal tree structure."));
         } else if (store) {
@@ -55,23 +55,23 @@ public final class LCast extends ALink {
             throw new IllegalArgumentException(error("Not a type [" + type + "]."));
         }
 
-        cast = AnalyzerCaster.getLegalCast(definition, location, before, after, true);
+        cast = AnalyzerCaster.getLegalCast(location, before, after, true);
 
         return cast != null ? this : null;
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         adapter.writeCast(cast);
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void load(final CompilerSettings settings, final MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void store(final CompilerSettings settings, final MethodWriter adapter) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index eae33809e70..842cc7c41bb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -42,10 +42,10 @@ final class LDefArray extends ALink implements IDefLink {
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
-        index.analyze(settings, definition, variables);
+    ALink analyze(final CompilerSettings settings, final Variables variables) {
+        index.analyze(settings, variables);
         index.expected = index.actual;
-        index = index.cast(settings, definition, variables);
+        index = index.cast(settings, variables);
 
         after = Definition.defType;
 
@@ -53,18 +53,18 @@ final class LDefArray extends ALink implements IDefLink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
-        index.write(settings, definition, adapter);
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
+        index.write(settings, adapter);
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void load(final CompilerSettings settings, final MethodWriter adapter) {
         final String desc = Type.getMethodDescriptor(after.type, Definition.defType.type, index.actual.type);
         adapter.invokeDynamic("arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_LOAD);
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void store(final CompilerSettings settings, final MethodWriter adapter) {
         final String desc = Type.getMethodDescriptor(Definition.voidType.type, Definition.defType.type,
             index.actual.type, after.type);
         adapter.invokeDynamic("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_STORE);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index 360f78eb4fd..e3b7c6c583d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -45,13 +45,13 @@ final class LDefCall extends ALink implements IDefLink {
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    ALink analyze(final CompilerSettings settings, final Variables variables) {
         for (int argument = 0; argument < arguments.size(); ++argument) {
             final AExpression expression = arguments.get(argument);
 
-            expression.analyze(settings, definition, variables);
+            expression.analyze(settings, variables);
             expression.expected = expression.actual;
-            arguments.set(argument, expression.cast(settings, definition, variables));
+            arguments.set(argument, expression.cast(settings, variables));
         }
 
         statement = true;
@@ -61,12 +61,12 @@ final class LDefCall extends ALink implements IDefLink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void load(final CompilerSettings settings, final MethodWriter adapter) {
         final StringBuilder signature = new StringBuilder();
 
         signature.append('(');
@@ -77,7 +77,7 @@ final class LDefCall extends ALink implements IDefLink {
         // it can avoid some unnecessary boxing etc.
         for (final AExpression argument : arguments) {
             signature.append(argument.actual.type.getDescriptor());
-            argument.write(settings, definition, adapter);
+            argument.write(settings, adapter);
         }
 
         signature.append(')');
@@ -88,7 +88,7 @@ final class LDefCall extends ALink implements IDefLink {
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void store(final CompilerSettings settings, final MethodWriter adapter) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index af20e152c70..f54bf316254 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -43,25 +43,25 @@ final class LDefField extends ALink implements IDefLink {
 
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    ALink analyze(final CompilerSettings settings, final Variables variables) {
         after = Definition.defType;
 
         return this;
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void load(final CompilerSettings settings, final MethodWriter adapter) {
         final String desc = Type.getMethodDescriptor(after.type, Definition.defType.type);
         adapter.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.LOAD);
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void store(final CompilerSettings settings, final MethodWriter adapter) {
         final String desc = Type.getMethodDescriptor(Definition.voidType.type, Definition.defType.type, after.type);
         adapter.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.STORE);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
index 3ec456c69a4..3c0368666d0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
@@ -46,7 +46,7 @@ public final class LField extends ALink {
     }
 
     @Override
-    ALink analyze(CompilerSettings settings, Definition definition, Variables variables) {
+    ALink analyze(CompilerSettings settings, Variables variables) {
         if (before == null) {
             throw new IllegalStateException(error("Illegal tree structure."));
         }
@@ -54,9 +54,9 @@ public final class LField extends ALink {
         final Sort sort = before.sort;
 
         if (sort == Sort.ARRAY) {
-            return new LArrayLength(line, location, value).copy(this).analyze(settings, definition, variables);
+            return new LArrayLength(line, location, value).copy(this).analyze(settings, variables);
         } else if (sort == Sort.DEF) {
-            return new LDefField(line, location, value).copy(this).analyze(settings, definition, variables);
+            return new LDefField(line, location, value).copy(this).analyze(settings, variables);
         }
 
         final Struct struct = before.struct;
@@ -80,17 +80,17 @@ public final class LField extends ALink {
                         Character.toUpperCase(value.charAt(0)) + value.substring(1), 1));
 
             if (shortcut) {
-                return new LShortcut(line, location, value).copy(this).analyze(settings, definition, variables);
+                return new LShortcut(line, location, value).copy(this).analyze(settings, variables);
             } else {
                 final EConstant index = new EConstant(line, location, value);
-                index.analyze(settings, definition, variables);
+                index.analyze(settings, variables);
 
                 if (Map.class.isAssignableFrom(before.clazz)) {
-                    return new LMapShortcut(line, location, index).copy(this).analyze(settings, definition, variables);
+                    return new LMapShortcut(line, location, index).copy(this).analyze(settings, variables);
                 }
                 
                 if (List.class.isAssignableFrom(before.clazz)) {
-                    return new LListShortcut(line, location, index).copy(this).analyze(settings, definition, variables);
+                    return new LListShortcut(line, location, index).copy(this).analyze(settings, variables);
                 }
             }
         }
@@ -99,12 +99,12 @@ public final class LField extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void load(final CompilerSettings settings, final MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isStatic(field.reflect.getModifiers())) {
             adapter.getStatic(field.owner.type, field.reflect.getName(), field.type.type);
         } else {
@@ -113,7 +113,7 @@ public final class LField extends ALink {
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void store(final CompilerSettings settings, final MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isStatic(field.reflect.getModifiers())) {
             adapter.putStatic(field.owner.type, field.reflect.getName(), field.type.type);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
index cc20fc6dec4..1ee0f8fa3ed 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
@@ -42,7 +42,7 @@ final class LListShortcut extends ALink {
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    ALink analyze(final CompilerSettings settings, final Variables variables) {
         getter = before.struct.methods.get(new Definition.MethodKey("get", 1));
         setter = before.struct.methods.get(new Definition.MethodKey("set", 2));
 
@@ -62,8 +62,8 @@ final class LListShortcut extends ALink {
 
         if ((load || store) && (!load || getter != null) && (!store || setter != null)) {
             index.expected = Definition.intType;
-            index.analyze(settings, definition, variables);
-            index = index.cast(settings, definition, variables);
+            index.analyze(settings, variables);
+            index = index.cast(settings, variables);
 
             after = setter != null ? setter.arguments.get(1) : getter.rtn;
         } else {
@@ -74,12 +74,12 @@ final class LListShortcut extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
-        index.write(settings, definition, adapter);
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
+        index.write(settings, adapter);
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void load(final CompilerSettings settings, final MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
             adapter.invokeInterface(getter.owner.type, getter.method);
         } else {
@@ -92,7 +92,7 @@ final class LListShortcut extends ALink {
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void store(final CompilerSettings settings, final MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
             adapter.invokeInterface(setter.owner.type, setter.method);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
index 4efbd4bdf0f..f06cb46152b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
@@ -42,7 +42,7 @@ final class LMapShortcut extends ALink {
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    ALink analyze(final CompilerSettings settings, final Variables variables) {
         getter = before.struct.methods.get(new Definition.MethodKey("get", 1));
         setter = before.struct.methods.get(new Definition.MethodKey("put", 2));
 
@@ -61,8 +61,8 @@ final class LMapShortcut extends ALink {
 
         if ((load || store) && (!load || getter != null) && (!store || setter != null)) {
             index.expected = setter != null ? setter.arguments.get(0) : getter.arguments.get(0);
-            index.analyze(settings, definition, variables);
-            index = index.cast(settings, definition, variables);
+            index.analyze(settings, variables);
+            index = index.cast(settings, variables);
 
             after = setter != null ? setter.arguments.get(1) : getter.rtn;
         } else {
@@ -73,12 +73,12 @@ final class LMapShortcut extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
-        index.write(settings, definition, adapter);
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
+        index.write(settings, adapter);
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void load(final CompilerSettings settings, final MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
             adapter.invokeInterface(getter.owner.type, getter.method);
         } else {
@@ -91,7 +91,7 @@ final class LMapShortcut extends ALink {
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void store(final CompilerSettings settings, final MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
             adapter.invokeInterface(setter.owner.type, setter.method);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
index 36e4b83c117..9d3e5d307fe 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
@@ -43,7 +43,7 @@ public final class LNewArray extends ALink {
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    ALink analyze(final CompilerSettings settings, final Variables variables) {
         if (before != null) {
             throw new IllegalStateException(error("Illegal tree structure."));
         } else if (store) {
@@ -64,8 +64,8 @@ public final class LNewArray extends ALink {
             final AExpression expression = arguments.get(argument);
 
             expression.expected = Definition.intType;
-            expression.analyze(settings, definition, variables);
-            arguments.set(argument, expression.cast(settings, definition, variables));
+            expression.analyze(settings, variables);
+            arguments.set(argument, expression.cast(settings, variables));
         }
 
         after = Definition.getType(type.struct, arguments.size());
@@ -74,14 +74,14 @@ public final class LNewArray extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void load(final CompilerSettings settings, final MethodWriter adapter) {
         for (final AExpression argument : arguments) {
-            argument.write(settings, definition, adapter);
+            argument.write(settings, adapter);
         }
 
         if (arguments.size() > 1) {
@@ -92,7 +92,7 @@ public final class LNewArray extends ALink {
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void store(final CompilerSettings settings, final MethodWriter adapter) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
index 784935e2aee..a47ced1e096 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
@@ -47,7 +47,7 @@ public final class LNewObj extends ALink {
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    ALink analyze(final CompilerSettings settings, final Variables variables) {
         if (before != null) {
             throw new IllegalStateException(error("Illegal tree structure"));
         } else if (store) {
@@ -78,8 +78,8 @@ public final class LNewObj extends ALink {
                 final AExpression expression = arguments.get(argument);
 
                 expression.expected = types[argument];
-                expression.analyze(settings, definition, variables);
-                arguments.set(argument, expression.cast(settings, definition, variables));
+                expression.analyze(settings, variables);
+                arguments.set(argument, expression.cast(settings, variables));
             }
 
             statement = true;
@@ -92,12 +92,12 @@ public final class LNewObj extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void load(final CompilerSettings settings, final MethodWriter adapter) {
         adapter.newInstance(after.type);
 
         if (load) {
@@ -105,14 +105,14 @@ public final class LNewObj extends ALink {
         }
 
         for (final AExpression argument : arguments) {
-            argument.write(settings, definition, adapter);
+            argument.write(settings, adapter);
         }
 
         adapter.invokeConstructor(constructor.owner.type, constructor.method);
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void store(final CompilerSettings settings, final MethodWriter adapter) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
index c65077e6e28..79feef8813b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
@@ -44,7 +44,7 @@ final class LShortcut extends ALink {
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    ALink analyze(final CompilerSettings settings, final Variables variables) {
         final Struct struct = before.struct;
 
         getter = struct.methods.get(new Definition.MethodKey("get" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0));
@@ -74,12 +74,12 @@ final class LShortcut extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void load(final CompilerSettings settings, final MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
             adapter.invokeInterface(getter.owner.type, getter.method);
         } else {
@@ -92,7 +92,7 @@ final class LShortcut extends ALink {
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void store(final CompilerSettings settings, final MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
             adapter.invokeInterface(setter.owner.type, setter.method);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
index 91f379cce76..5fe7c22a457 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
@@ -36,7 +36,7 @@ public final class LString extends ALink {
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    ALink analyze(final CompilerSettings settings, final Variables variables) {
         if (before != null) {
             throw new IllegalStateException("Illegal tree structure.");
         } else if (store) {
@@ -51,17 +51,17 @@ public final class LString extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void load(final CompilerSettings settings, final MethodWriter adapter) {
         adapter.push(string);
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void store(final CompilerSettings settings, final MethodWriter adapter) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
index 2c6ded3b4ec..6d4254f76ba 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
@@ -43,7 +43,7 @@ public final class LVariable extends ALink {
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    ALink analyze(final CompilerSettings settings, final Variables variables) {
         if (before != null) {
             throw new IllegalStateException(error("Illegal tree structure."));
         }
@@ -74,17 +74,17 @@ public final class LVariable extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void load(final CompilerSettings settings, final MethodWriter adapter) {
         adapter.visitVarInsn(after.type.getOpcode(Opcodes.ILOAD), slot);
     }
 
     @Override
-    void store(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void store(final CompilerSettings settings, final MethodWriter adapter) {
         adapter.visitVarInsn(after.type.getOpcode(Opcodes.ISTORE), slot);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
index 802d5b6c415..7b9aed850b6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.CompilerSettings;
-import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -41,7 +40,7 @@ public final class SBlock extends AStatement {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         final AStatement last = statements.get(statements.size() - 1);
 
         for (final AStatement statement : statements) {
@@ -53,7 +52,7 @@ public final class SBlock extends AStatement {
             statement.lastSource = lastSource && statement == last;
             statement.lastLoop = (beginLoop || lastLoop) && statement == last;
 
-            statement.analyze(settings, definition, variables);
+            statement.analyze(settings, variables);
 
             methodEscape = statement.methodEscape;
             loopEscape = statement.loopEscape;
@@ -65,11 +64,11 @@ public final class SBlock extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         for (final AStatement statement : statements) {
             statement.continu = continu;
             statement.brake = brake;
-            statement.write(settings, definition, adapter);
+            statement.write(settings, adapter);
         }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
index 3998b1a9ff2..08f8e811b05 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.CompilerSettings;
-import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -34,7 +33,7 @@ public final class SBreak extends AStatement {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         if (!inLoop) {
             throw new IllegalArgumentException(error("Break statement outside of a loop."));
         }
@@ -46,7 +45,7 @@ public final class SBreak extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         writeDebugInfo(adapter);
         adapter.goTo(brake);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
index 2c4e33b6326..35317e8de8a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.CompilerSettings;
-import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -34,7 +33,7 @@ public final class SContinue extends AStatement {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         if (!inLoop) {
             throw new IllegalArgumentException(error("Continue statement outside of a loop."));
         }
@@ -49,7 +48,7 @@ public final class SContinue extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         writeDebugInfo(adapter);
         adapter.goTo(continu);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
index 5494ef9c32a..ee878146364 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.CompilerSettings;
-import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -41,18 +40,18 @@ public final class SDeclBlock extends AStatement {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         for (final SDeclaration declaration : declarations) {
-            declaration.analyze(settings, definition, variables);
+            declaration.analyze(settings, variables);
         }
 
         statementCount = declarations.size();
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         for (final SDeclaration declaration : declarations) {
-            declaration.write(settings, definition, adapter);
+            declaration.write(settings, adapter);
         }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
index 189ddc95936..adf24509f6c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.CompilerSettings;
-import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.Variables.Variable;
@@ -47,18 +46,18 @@ public final class SDeclaration extends AStatement {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         variable = variables.addVariable(location, type, name, false, false);
 
         if (expression != null) {
             expression.expected = variable.type;
-            expression.analyze(settings, definition, variables);
-            expression = expression.cast(settings, definition, variables);
+            expression.analyze(settings, variables);
+            expression = expression.cast(settings, variables);
         }
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         writeDebugInfo(adapter);
         final org.objectweb.asm.Type type = variable.type.type;
         final Sort sort = variable.type.sort;
@@ -66,7 +65,7 @@ public final class SDeclaration extends AStatement {
         final boolean initialize = expression == null;
 
         if (!initialize) {
-            expression.write(settings, definition, adapter);
+            expression.write(settings, adapter);
         }
 
         switch (sort) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
index 99f8b634d49..ade9f9ba221 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -41,21 +41,21 @@ public final class SDo extends AStatement {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         variables.incrementScope();
 
         block.beginLoop = true;
         block.inLoop = true;
 
-        block.analyze(settings, definition, variables);
+        block.analyze(settings, variables);
 
         if (block.loopEscape && !block.anyContinue) {
             throw new IllegalArgumentException(error("Extraneous do while loop."));
         }
 
         condition.expected = Definition.booleanType;
-        condition.analyze(settings, definition, variables);
-        condition = condition.cast(settings, definition, variables);
+        condition.analyze(settings, variables);
+        condition = condition.cast(settings, variables);
 
         if (condition.constant != null) {
             final boolean continuous = (boolean)condition.constant;
@@ -80,7 +80,7 @@ public final class SDo extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         writeDebugInfo(adapter);
         final Label start = new Label();
         final Label begin = new Label();
@@ -90,12 +90,12 @@ public final class SDo extends AStatement {
 
         block.continu = begin;
         block.brake = end;
-        block.write(settings, definition, adapter);
+        block.write(settings, adapter);
 
         adapter.mark(begin);
 
         condition.fals = end;
-        condition.write(settings, definition, adapter);
+        condition.write(settings, adapter);
 
         adapter.writeLoopCounter(loopCounterSlot, Math.max(1, block.statementCount));
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index 2ed0d10b930..1f107548dff 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -39,9 +39,9 @@ public final class SExpression extends AStatement {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         expression.read = lastSource;
-        expression.analyze(settings, definition, variables);
+        expression.analyze(settings, variables);
 
         if (!lastSource && !expression.statement) {
             throw new IllegalArgumentException(error("Not a statement."));
@@ -50,7 +50,7 @@ public final class SExpression extends AStatement {
         final boolean rtn = lastSource && expression.actual.sort != Sort.VOID;
 
         expression.expected = rtn ? Definition.objectType : expression.actual;
-        expression = expression.cast(settings, definition, variables);
+        expression = expression.cast(settings, variables);
 
         methodEscape = rtn;
         loopEscape = rtn;
@@ -59,9 +59,9 @@ public final class SExpression extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         writeDebugInfo(adapter);
-        expression.write(settings, definition, adapter);
+        expression.write(settings, adapter);
 
         if (methodEscape) {
             adapter.returnValue();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index ef97fbac06b..ae59382039d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -46,19 +46,19 @@ public final class SFor extends AStatement {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         variables.incrementScope();
 
         boolean continuous = false;
 
         if (initializer != null) {
             if (initializer instanceof SDeclBlock) {
-                ((SDeclBlock)initializer).analyze(settings, definition, variables);
+                ((SDeclBlock)initializer).analyze(settings, variables);
             } else if (initializer instanceof AExpression) {
                 final AExpression initializer = (AExpression)this.initializer;
 
                 initializer.read = false;
-                initializer.analyze(settings, definition, variables);
+                initializer.analyze(settings, variables);
 
                 if (!initializer.statement) {
                     throw new IllegalArgumentException(initializer.error("Not a statement."));
@@ -71,8 +71,8 @@ public final class SFor extends AStatement {
         if (condition != null) {
 
             condition.expected = Definition.booleanType;
-            condition.analyze(settings, definition, variables);
-            condition = condition.cast(settings, definition, variables);
+            condition.analyze(settings, variables);
+            condition = condition.cast(settings, variables);
 
             if (condition.constant != null) {
                 continuous = (boolean)condition.constant;
@@ -91,7 +91,7 @@ public final class SFor extends AStatement {
 
         if (afterthought != null) {
             afterthought.read = false;
-            afterthought.analyze(settings, definition, variables);
+            afterthought.analyze(settings, variables);
 
             if (!afterthought.statement) {
                 throw new IllegalArgumentException(afterthought.error("Not a statement."));
@@ -104,7 +104,7 @@ public final class SFor extends AStatement {
             block.beginLoop = true;
             block.inLoop = true;
 
-            block.analyze(settings, definition, variables);
+            block.analyze(settings, variables);
 
             if (block.loopEscape && !block.anyContinue) {
                 throw new IllegalArgumentException(error("Extraneous for loop."));
@@ -128,18 +128,18 @@ public final class SFor extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         writeDebugInfo(adapter);
         final Label start = new Label();
         final Label begin = afterthought == null ? start : new Label();
         final Label end = new Label();
 
         if (initializer instanceof SDeclBlock) {
-            ((SDeclBlock)initializer).write(settings, definition, adapter);
+            ((SDeclBlock)initializer).write(settings, adapter);
         } else if (initializer instanceof AExpression) {
             AExpression initializer = (AExpression)this.initializer;
 
-            initializer.write(settings, definition, adapter);
+            initializer.write(settings, adapter);
             adapter.writePop(initializer.expected.sort.size);
         }
 
@@ -147,7 +147,7 @@ public final class SFor extends AStatement {
 
         if (condition != null) {
             condition.fals = end;
-            condition.write(settings, definition, adapter);
+            condition.write(settings, adapter);
         }
 
         boolean allEscape = false;
@@ -162,14 +162,14 @@ public final class SFor extends AStatement {
             }
 
             adapter.writeLoopCounter(loopCounterSlot, statementCount);
-            block.write(settings, definition, adapter);
+            block.write(settings, adapter);
         } else {
             adapter.writeLoopCounter(loopCounterSlot, 1);
         }
 
         if (afterthought != null) {
             adapter.mark(begin);
-            afterthought.write(settings, definition, adapter);
+            afterthought.write(settings, adapter);
         }
 
         if (afterthought != null || !allEscape) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
index 34f4a80645c..40c20a1cb87 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
@@ -44,10 +44,10 @@ public final class SIfElse extends AStatement {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         condition.expected = Definition.booleanType;
-        condition.analyze(settings, definition, variables);
-        condition = condition.cast(settings, definition, variables);
+        condition.analyze(settings, variables);
+        condition = condition.cast(settings, variables);
 
         if (condition.constant != null) {
             throw new IllegalArgumentException(error("Extraneous if statement."));
@@ -58,7 +58,7 @@ public final class SIfElse extends AStatement {
         ifblock.lastLoop = lastLoop;
 
         variables.incrementScope();
-        ifblock.analyze(settings, definition, variables);
+        ifblock.analyze(settings, variables);
         variables.decrementScope();
 
         anyContinue = ifblock.anyContinue;
@@ -71,7 +71,7 @@ public final class SIfElse extends AStatement {
             elseblock.lastLoop = lastLoop;
 
             variables.incrementScope();
-            elseblock.analyze(settings, definition, variables);
+            elseblock.analyze(settings, variables);
             variables.decrementScope();
 
             methodEscape = ifblock.methodEscape && elseblock.methodEscape;
@@ -84,17 +84,17 @@ public final class SIfElse extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         writeDebugInfo(adapter);
         final Label end = new Label();
         final Label fals = elseblock != null ? new Label() : end;
 
         condition.fals = fals;
-        condition.write(settings, definition, adapter);
+        condition.write(settings, adapter);
 
         ifblock.continu = continu;
         ifblock.brake = brake;
-        ifblock.write(settings, definition, adapter);
+        ifblock.write(settings, adapter);
 
         if (elseblock != null) {
             if (!ifblock.allEscape) {
@@ -105,7 +105,7 @@ public final class SIfElse extends AStatement {
 
             elseblock.continu = continu;
             elseblock.brake = brake;
-            elseblock.write(settings, definition, adapter);
+            elseblock.write(settings, adapter);
         }
 
         adapter.mark(end);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
index 4442e5b9ee4..4caf57f463b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -38,10 +38,10 @@ public final class SReturn extends AStatement {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         expression.expected = Definition.objectType;
-        expression.analyze(settings, definition, variables);
-        expression = expression.cast(settings, definition, variables);
+        expression.analyze(settings, variables);
+        expression = expression.cast(settings, variables);
 
         methodEscape = true;
         loopEscape = true;
@@ -51,9 +51,9 @@ public final class SReturn extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         writeDebugInfo(adapter);
-        expression.write(settings, definition, adapter);
+        expression.write(settings, adapter);
         adapter.returnValue();
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index e5a80802ce8..e77ce8330a6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.CompilerSettings;
-import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Opcodes;
 import org.elasticsearch.painless.MethodWriter;
@@ -42,7 +41,7 @@ public final class SSource extends AStatement {
     }
 
     @Override
-    public void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    public void analyze(final CompilerSettings settings, final Variables variables) {
         variables.incrementScope();
 
         final AStatement last = statements.get(statements.size() - 1);
@@ -53,7 +52,7 @@ public final class SSource extends AStatement {
             }
 
             statement.lastSource = statement == last;
-            statement.analyze(settings, definition, variables);
+            statement.analyze(settings, variables);
 
             methodEscape = statement.methodEscape;
             allEscape = statement.allEscape;
@@ -63,9 +62,9 @@ public final class SSource extends AStatement {
     }
 
     @Override
-    public void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    public void write(final CompilerSettings settings, final MethodWriter adapter) {
         for (final AStatement statement : statements) {
-            statement.write(settings, definition, adapter);
+            statement.write(settings, adapter);
         }
 
         if (!methodEscape) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
index 99fb5c6a973..8a2909d0a3b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
@@ -38,10 +38,10 @@ public final class SThrow extends AStatement {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         expression.expected = Definition.exceptionType;
-        expression.analyze(settings, definition, variables);
-        expression = expression.cast(settings, definition, variables);
+        expression.analyze(settings, variables);
+        expression = expression.cast(settings, variables);
 
         methodEscape = true;
         loopEscape = true;
@@ -50,9 +50,9 @@ public final class SThrow extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         writeDebugInfo(adapter);
-        expression.write(settings, definition, adapter);
+        expression.write(settings, adapter);
         adapter.throwException();
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java
index acb50a2b962..a1ee15a050f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.CompilerSettings;
-import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.Variables.Variable;
 import org.objectweb.asm.Label;
@@ -51,7 +50,7 @@ public final class STrap extends AStatement {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         variable = variables.addVariable(location, type, name, true, false);
 
         if (!Exception.class.isAssignableFrom(variable.type.clazz)) {
@@ -63,7 +62,7 @@ public final class STrap extends AStatement {
             block.inLoop = inLoop;
             block.lastLoop = lastLoop;
 
-            block.analyze(settings, definition, variables);
+            block.analyze(settings, variables);
 
             methodEscape = block.methodEscape;
             loopEscape = block.loopEscape;
@@ -75,7 +74,7 @@ public final class STrap extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         writeDebugInfo(adapter);
         final Label jump = new Label();
 
@@ -85,7 +84,7 @@ public final class STrap extends AStatement {
         if (block != null) {
             block.continu = continu;
             block.brake = brake;
-            block.write(settings, definition, adapter);
+            block.write(settings, adapter);
         }
 
         adapter.visitTryCatchBlock(begin, end, jump, variable.type.type.getInternalName());
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
index 0329826b02a..70a5b2164da 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.CompilerSettings;
-import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.MethodWriter;
@@ -44,13 +43,13 @@ public final class STry extends AStatement {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         block.lastSource = lastSource;
         block.inLoop = inLoop;
         block.lastLoop = lastLoop;
 
         variables.incrementScope();
-        block.analyze(settings, definition, variables);
+        block.analyze(settings, variables);
         variables.decrementScope();
 
         methodEscape = block.methodEscape;
@@ -67,7 +66,7 @@ public final class STry extends AStatement {
             trap.lastLoop = lastLoop;
 
             variables.incrementScope();
-            trap.analyze(settings, definition, variables);
+            trap.analyze(settings, variables);
             variables.decrementScope();
 
             methodEscape &= trap.methodEscape;
@@ -83,7 +82,7 @@ public final class STry extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         writeDebugInfo(adapter);
         final Label begin = new Label();
         final Label end = new Label();
@@ -93,7 +92,7 @@ public final class STry extends AStatement {
 
         block.continu = continu;
         block.brake = brake;
-        block.write(settings, definition, adapter);
+        block.write(settings, adapter);
 
         if (!block.allEscape) {
             adapter.goTo(exception);
@@ -105,7 +104,7 @@ public final class STry extends AStatement {
             trap.begin = begin;
             trap.end = end;
             trap.exception = traps.size() > 1 ? exception : null;
-            trap.write(settings, definition, adapter);
+            trap.write(settings, adapter);
         }
 
         if (!block.allEscape || traps.size() > 1) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
index 83afe469320..a6d88fdcced 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -41,12 +41,12 @@ public final class SWhile extends AStatement {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Definition definition, final Variables variables) {
+    void analyze(final CompilerSettings settings, final Variables variables) {
         variables.incrementScope();
 
         condition.expected = Definition.booleanType;
-        condition.analyze(settings, definition, variables);
-        condition = condition.cast(settings, definition, variables);
+        condition.analyze(settings, variables);
+        condition = condition.cast(settings, variables);
 
         boolean continuous = false;
 
@@ -68,7 +68,7 @@ public final class SWhile extends AStatement {
             block.beginLoop = true;
             block.inLoop = true;
 
-            block.analyze(settings, definition, variables);
+            block.analyze(settings, variables);
 
             if (block.loopEscape && !block.anyContinue) {
                 throw new IllegalArgumentException(error("Extranous while loop."));
@@ -92,7 +92,7 @@ public final class SWhile extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final Definition definition, final MethodWriter adapter) {
+    void write(final CompilerSettings settings, final MethodWriter adapter) {
         writeDebugInfo(adapter);
         final Label begin = new Label();
         final Label end = new Label();
@@ -100,14 +100,14 @@ public final class SWhile extends AStatement {
         adapter.mark(begin);
 
         condition.fals = end;
-        condition.write(settings, definition, adapter);
+        condition.write(settings, adapter);
 
         if (block != null) {
             adapter.writeLoopCounter(loopCounterSlot, Math.max(1, block.statementCount));
 
             block.continu = begin;
             block.brake = end;
-            block.write(settings, definition, adapter);
+            block.write(settings, adapter);
         } else {
             adapter.writeLoopCounter(loopCounterSlot, 1);
         }

From 34ef5306d2ead72f8faa6df23195829d5c522de9 Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Fri, 20 May 2016 12:56:24 +0200
Subject: [PATCH 0374/1311] Snapshotting and sync could cause a dead lock
 TranslogWriter (#18481)

 #18360 introduced an extra lock in order to allow writes while syncing the translog. This caused a potential deadlock with snapshotting code where we first acquire the instance lock, followed by a sync (which acquires the syncLock). However, the sync logic acquires the syncLock first, followed by the instance lock.

I considered solving this by not syncing the translog on snapshot - I think we can get away with just flushing it. That however will create subtleties around snapshoting and whether operations in them are persisted. I opted instead to have slightly uglier code with nest synchronized, where the scope of the change is contained to the TranslogWriter class alone.
---
 .../index/translog/TranslogWriter.java        | 72 +++++++++++--------
 .../index/translog/TranslogTests.java         | 29 ++++----
 2 files changed, 60 insertions(+), 41 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java b/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
index 84278fa92b3..0109995f80f 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
@@ -39,7 +39,6 @@ import java.nio.file.OpenOption;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.ReentrantLock;
 
 public class TranslogWriter extends BaseTranslogReader implements Closeable {
 
@@ -154,7 +153,9 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
     /**
      * returns true if there are buffered ops
      */
-    public boolean syncNeeded() { return totalOffset != lastSyncedOffset; }
+    public boolean syncNeeded() {
+        return totalOffset != lastSyncedOffset;
+    }
 
     @Override
     public int totalOperations() {
@@ -169,40 +170,55 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
     /**
      * closes this writer and transfers it's underlying file channel to a new immutable reader
      */
-    public synchronized TranslogReader closeIntoReader() throws IOException {
-        try {
-            sync(); // sync before we close..
-        } catch (IOException e) {
-            closeWithTragicEvent(e);
-            throw e;
-        }
-        if (closed.compareAndSet(false, true)) {
-            boolean success = false;
-            try {
-                final TranslogReader reader = new TranslogReader(generation, channel, path, firstOperationOffset, getWrittenOffset(), operationCounter);
-                success = true;
-                return reader;
-            } finally {
-                if (success == false) {
-                    // close the channel, as we are closed and failed to create a new reader
-                    IOUtils.closeWhileHandlingException(channel);
+    public TranslogReader closeIntoReader() throws IOException {
+        // make sure to acquire the sync lock first, to prevent dead locks with threads calling
+        // syncUpTo() , where the sync lock is acquired first, following by the synchronize(this)
+        //
+        // Note: While this is not strictly needed as this method is called while blocking all ops on the translog,
+        //       we do this to for correctness and preventing future issues.
+        synchronized (syncLock) {
+            synchronized (this) {
+                try {
+                    sync(); // sync before we close..
+                } catch (IOException e) {
+                    closeWithTragicEvent(e);
+                    throw e;
+                }
+                if (closed.compareAndSet(false, true)) {
+                    boolean success = false;
+                    try {
+                        final TranslogReader reader = new TranslogReader(generation, channel, path, firstOperationOffset, getWrittenOffset(), operationCounter);
+                        success = true;
+                        return reader;
+                    } finally {
+                        if (success == false) {
+                            // close the channel, as we are closed and failed to create a new reader
+                            IOUtils.closeWhileHandlingException(channel);
+                        }
+                    }
+                } else {
+                    throw new AlreadyClosedException("translog [" + getGeneration() + "] is already closed (path [" + path + "]", tragedy);
                 }
             }
-        } else {
-            throw new AlreadyClosedException("translog [" + getGeneration() + "] is already closed (path [" + path + "]", tragedy);
         }
     }
 
 
     @Override
-    public synchronized Translog.Snapshot newSnapshot() {
-        ensureOpen();
-        try {
-            sync();
-        } catch (IOException e) {
-            throw new TranslogException(shardId, "exception while syncing before creating a snapshot", e);
+    public Translog.Snapshot newSnapshot() {
+        // make sure to acquire the sync lock first, to prevent dead locks with threads calling
+        // syncUpTo() , where the sync lock is acquired first, following by the synchronize(this)
+        synchronized (syncLock) {
+            synchronized (this) {
+                ensureOpen();
+                try {
+                    sync();
+                } catch (IOException e) {
+                    throw new TranslogException(shardId, "exception while syncing before creating a snapshot", e);
+                }
+                return super.newSnapshot();
+            }
         }
-        return super.newSnapshot();
     }
 
     private long getWrittenOffset() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
index 9b6e4670794..b4d2423921c 100644
--- a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
+++ b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
@@ -42,7 +42,6 @@ import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
-import org.elasticsearch.index.Index;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
@@ -138,8 +137,8 @@ public class TranslogTests extends ESTestCase {
 
     private TranslogConfig getTranslogConfig(Path path) {
         Settings build = Settings.builder()
-                .put(IndexMetaData.SETTING_VERSION_CREATED, org.elasticsearch.Version.CURRENT)
-                .build();
+            .put(IndexMetaData.SETTING_VERSION_CREATED, org.elasticsearch.Version.CURRENT)
+            .build();
         ByteSizeValue bufferSize = randomBoolean() ? TranslogConfig.DEFAULT_BUFFER_SIZE : new ByteSizeValue(10 + randomInt(128 * 1024), ByteSizeUnit.BYTES);
         return new TranslogConfig(shardId, path, IndexSettingsModule.newIndexSettings(shardId.getIndex(), build), BigArrays.NON_RECYCLING_INSTANCE, bufferSize);
     }
@@ -330,9 +329,9 @@ public class TranslogTests extends ESTestCase {
         assertEquals(6, copy.estimatedNumberOfOperations());
         assertEquals(431, copy.getTranslogSizeInBytes());
         assertEquals("\"translog\"{\n" +
-                "  \"operations\" : 6,\n" +
-                "  \"size_in_bytes\" : 431\n" +
-                "}", copy.toString().trim());
+            "  \"operations\" : 6,\n" +
+            "  \"size_in_bytes\" : 431\n" +
+            "}", copy.toString().trim());
 
         try {
             new TranslogStats(1, -1);
@@ -604,7 +603,8 @@ public class TranslogTests extends ESTestCase {
         final List<Throwable> errors = new CopyOnWriteArrayList<>();
         logger.debug("using [{}] readers. [{}] writers. flushing every ~[{}] ops.", readers.length, writers.length, flushEveryOps);
         for (int i = 0; i < writers.length; i++) {
-            final String threadId = "writer_" + i;
+            final String threadName = "writer_" + i;
+            final int threadId = i;
             writers[i] = new Thread(new AbstractRunnable() {
                 @Override
                 public void doRun() throws BrokenBarrierException, InterruptedException, IOException {
@@ -629,18 +629,21 @@ public class TranslogTests extends ESTestCase {
                         if (existing != null) {
                             fail("duplicate op [" + op + "], old entry at " + location);
                         }
+                        if (id % writers.length == threadId) {
+                            translog.ensureSynced(location);
+                        }
                         writtenOpsLatch.get().countDown();
                         counter++;
                     }
-                    logger.debug("--> [{}] done. wrote [{}] ops.", threadId, counter);
+                    logger.debug("--> [{}] done. wrote [{}] ops.", threadName, counter);
                 }
 
                 @Override
                 public void onFailure(Throwable t) {
-                    logger.error("--> writer [{}] had an error", t, threadId);
+                    logger.error("--> writer [{}] had an error", t, threadName);
                     errors.add(t);
                 }
-            }, threadId);
+            }, threadName);
             writers[i].start();
         }
 
@@ -1262,12 +1265,12 @@ public class TranslogTests extends ESTestCase {
                         case CREATE:
                         case INDEX:
                             op = new Translog.Index("test", threadId + "_" + opCount,
-                                    randomUnicodeOfLengthBetween(1, 20 * 1024).getBytes("UTF-8"));
+                                randomUnicodeOfLengthBetween(1, 20 * 1024).getBytes("UTF-8"));
                             break;
                         case DELETE:
                             op = new Translog.Delete(new Term("_uid", threadId + "_" + opCount),
-                                    1 + randomInt(100000),
-                                    randomFrom(VersionType.values()));
+                                1 + randomInt(100000),
+                                randomFrom(VersionType.values()));
                             break;
                         default:
                             throw new ElasticsearchException("not supported op type");

From d3fe22c99034f141965c14316b90174257a07199 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Fri, 20 May 2016 13:36:55 +0200
Subject: [PATCH 0375/1311] Improve adding clauses to `span_near` and `span_or`
 query

Currently the query builders expose the clauses of the span
query as a modifiable list. Instead we should make the that
getter return an unmodifiable list. Also renaming the method
used to add a clause from `clause(spanQuery)` to
`addClause(spanQuery)`.
---
 .../index/query/SpanNearQueryBuilder.java     | 14 +++++++----
 .../index/query/SpanOrQueryBuilder.java       | 14 +++++++----
 .../query/QueryDSLDocumentationTests.java     | 12 +++++-----
 .../query/SpanNearQueryBuilderTests.java      | 24 +++++++++----------
 .../index/query/SpanNotQueryBuilderTests.java |  6 ++---
 .../index/query/SpanOrQueryBuilderTests.java  | 21 +++++++++-------
 .../percolator/PercolatorQuerySearchIT.java   | 20 ++++++++--------
 .../search/query/SearchQueryIT.java           |  8 +++----
 .../migration/migrate_5_0/java.asciidoc       |  2 ++
 9 files changed, 66 insertions(+), 55 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
index a503b708633..cebc72c077b 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
@@ -31,6 +31,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
@@ -64,7 +65,7 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
      */
     public SpanNearQueryBuilder(SpanQueryBuilder initialClause, int slop) {
         if (initialClause == null) {
-            throw new IllegalArgumentException("query must include at least one clause");
+            throw new IllegalArgumentException("[" + NAME + "] must include at least one clause");
         }
         this.clauses.add(initialClause);
         this.slop = slop;
@@ -96,9 +97,12 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
         return this.slop;
     }
 
-    public SpanNearQueryBuilder clause(SpanQueryBuilder clause) {
+    /**
+     * Add a span clause to the current list of clauses
+     */
+    public SpanNearQueryBuilder addClause(SpanQueryBuilder clause) {
         if (clause == null) {
-            throw new IllegalArgumentException("query clauses cannot be null");
+            throw new IllegalArgumentException("[" + NAME + "]  clauses cannot be null");
         }
         clauses.add(clause);
         return this;
@@ -108,7 +112,7 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
      * @return the {@link SpanQueryBuilder} clauses that were set for this query
      */
     public List<SpanQueryBuilder> clauses() {
-        return this.clauses;
+        return Collections.unmodifiableList(this.clauses);
     }
 
     /**
@@ -198,7 +202,7 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
 
         SpanNearQueryBuilder queryBuilder = new SpanNearQueryBuilder(clauses.get(0), slop);
         for (int i = 1; i < clauses.size(); i++) {
-            queryBuilder.clause(clauses.get(i));
+            queryBuilder.addClause(clauses.get(i));
         }
         queryBuilder.inOrder(inOrder);
         queryBuilder.boost(boost);
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java
index 2a67a84be81..98698307d0c 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java
@@ -31,6 +31,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
@@ -48,7 +49,7 @@ public class SpanOrQueryBuilder extends AbstractQueryBuilder<SpanOrQueryBuilder>
 
     public SpanOrQueryBuilder(SpanQueryBuilder initialClause) {
         if (initialClause == null) {
-            throw new IllegalArgumentException("query must include at least one clause");
+            throw new IllegalArgumentException("[" + NAME + "] must include at least one clause");
         }
         clauses.add(initialClause);
     }
@@ -68,9 +69,12 @@ public class SpanOrQueryBuilder extends AbstractQueryBuilder<SpanOrQueryBuilder>
         writeQueries(out, clauses);
     }
 
-    public SpanOrQueryBuilder clause(SpanQueryBuilder clause) {
+    /**
+     * Add a span clause to the current list of clauses
+     */
+    public SpanOrQueryBuilder addClause(SpanQueryBuilder clause) {
         if (clause == null) {
-            throw new IllegalArgumentException("inner bool query clause cannot be null");
+            throw new IllegalArgumentException("[" + NAME + "] inner clause cannot be null");
         }
         clauses.add(clause);
         return this;
@@ -80,7 +84,7 @@ public class SpanOrQueryBuilder extends AbstractQueryBuilder<SpanOrQueryBuilder>
      * @return the {@link SpanQueryBuilder} clauses that were set for this query
      */
     public List<SpanQueryBuilder> clauses() {
-        return this.clauses;
+        return Collections.unmodifiableList(this.clauses);
     }
 
     @Override
@@ -137,7 +141,7 @@ public class SpanOrQueryBuilder extends AbstractQueryBuilder<SpanOrQueryBuilder>
 
         SpanOrQueryBuilder queryBuilder = new SpanOrQueryBuilder(clauses.get(0));
         for (int i = 1; i < clauses.size(); i++) {
-            queryBuilder.clause(clauses.get(i));
+            queryBuilder.addClause(clauses.get(i));
         }
         queryBuilder.boost(boost);
         queryBuilder.queryName(queryName);
diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryDSLDocumentationTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryDSLDocumentationTests.java
index f82561a4179..d0f2236ce06 100644
--- a/core/src/test/java/org/elasticsearch/index/query/QueryDSLDocumentationTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/QueryDSLDocumentationTests.java
@@ -314,7 +314,7 @@ public class QueryDSLDocumentationTests extends ESTestCase {
     public void testSpanContaining() {
         spanContainingQuery(
                 spanNearQuery(spanTermQuery("field1","bar"), 5)
-                    .clause(spanTermQuery("field1","baz"))
+                    .addClause(spanTermQuery("field1","baz"))
                     .inOrder(true),
                 spanTermQuery("field1","foo"));
     }
@@ -332,8 +332,8 @@ public class QueryDSLDocumentationTests extends ESTestCase {
 
     public void testSpanNear() {
         spanNearQuery(spanTermQuery("field","value1"), 12)
-        .clause(spanTermQuery("field","value2"))
-        .clause(spanTermQuery("field","value3"))
+        .addClause(spanTermQuery("field","value2"))
+        .addClause(spanTermQuery("field","value3"))
         .inOrder(false);
     }
 
@@ -344,8 +344,8 @@ public class QueryDSLDocumentationTests extends ESTestCase {
 
     public void testSpanOr() {
         spanOrQuery(spanTermQuery("field","value1"))
-        .clause(spanTermQuery("field","value2"))
-        .clause(spanTermQuery("field","value3"));
+        .addClause(spanTermQuery("field","value2"))
+        .addClause(spanTermQuery("field","value3"));
     }
 
     public void testSpanTerm() {
@@ -355,7 +355,7 @@ public class QueryDSLDocumentationTests extends ESTestCase {
     public void testSpanWithin() {
         spanWithinQuery(
                 spanNearQuery(spanTermQuery("field1", "bar"), 5)
-                    .clause(spanTermQuery("field1", "baz"))
+                    .addClause(spanTermQuery("field1", "baz"))
                     .inOrder(true),
                 spanTermQuery("field1", "foo"));
     }
diff --git a/core/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java
index 202623d1980..cbad5a60aa3 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java
@@ -37,7 +37,7 @@ public class SpanNearQueryBuilderTests extends AbstractQueryTestCase<SpanNearQue
         SpanTermQueryBuilder[] spanTermQueries = new SpanTermQueryBuilderTests().createSpanTermQueryBuilders(randomIntBetween(1, 6));
         SpanNearQueryBuilder queryBuilder = new SpanNearQueryBuilder(spanTermQueries[0], randomIntBetween(-10, 10));
         for (int i = 1; i < spanTermQueries.length; i++) {
-            queryBuilder.clause(spanTermQueries[i]);
+            queryBuilder.addClause(spanTermQueries[i]);
         }
         queryBuilder.inOrder(randomBoolean());
         return queryBuilder;
@@ -57,20 +57,18 @@ public class SpanNearQueryBuilderTests extends AbstractQueryTestCase<SpanNearQue
     }
 
     public void testIllegalArguments() {
-        try {
-            new SpanNearQueryBuilder(null, 1);
-            fail("cannot be null");
-        } catch (IllegalArgumentException e) {
-            // ecpected
-        }
+            IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new SpanNearQueryBuilder(null, 1));
+            assertEquals("[span_near] must include at least one clause", e.getMessage());
 
-        try {
             SpanNearQueryBuilder spanNearQueryBuilder = new SpanNearQueryBuilder(new SpanTermQueryBuilder("field", "value"), 1);
-            spanNearQueryBuilder.clause(null);
-            fail("cannot be null");
-        } catch (IllegalArgumentException e) {
-            // ecpected
-        }
+            e = expectThrows(IllegalArgumentException.class, () -> spanNearQueryBuilder.addClause(null));
+            assertEquals("[span_near]  clauses cannot be null", e.getMessage());
+    }
+
+    public void testClausesUnmodifiable() {
+        SpanNearQueryBuilder spanNearQueryBuilder = new SpanNearQueryBuilder(new SpanTermQueryBuilder("field", "value"), 1);
+        expectThrows(UnsupportedOperationException.class,
+                () -> spanNearQueryBuilder.clauses().add(new SpanTermQueryBuilder("field", "value2")));
     }
 
     public void testFromJson() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/index/query/SpanNotQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SpanNotQueryBuilderTests.java
index 527b7e5e83a..e536dec0598 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SpanNotQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SpanNotQueryBuilderTests.java
@@ -110,7 +110,7 @@ public class SpanNotQueryBuilderTests extends AbstractQueryTestCase<SpanNotQuery
         spanTermQuery("description", "jumped").toXContent(builder, null);
         builder.field("include");
         spanNearQuery(QueryBuilders.spanTermQuery("description", "quick"), 1)
-                .clause(QueryBuilders.spanTermQuery("description", "fox")).toXContent(builder, null);
+                .addClause(QueryBuilders.spanTermQuery("description", "fox")).toXContent(builder, null);
         builder.field("dist", 3);
         builder.endObject();
         builder.endObject();
@@ -148,7 +148,7 @@ public class SpanNotQueryBuilderTests extends AbstractQueryTestCase<SpanNotQuery
             builder.startObject(SpanNotQueryBuilder.NAME);
             builder.field("include");
             spanNearQuery(QueryBuilders.spanTermQuery("description", "quick"), 1)
-                    .clause(QueryBuilders.spanTermQuery("description", "fox")).toXContent(builder, null);
+                    .addClause(QueryBuilders.spanTermQuery("description", "fox")).toXContent(builder, null);
             builder.field("dist", 2);
             builder.endObject();
             builder.endObject();
@@ -166,7 +166,7 @@ public class SpanNotQueryBuilderTests extends AbstractQueryTestCase<SpanNotQuery
             builder.startObject(SpanNotQueryBuilder.NAME);
             builder.field("include");
             spanNearQuery(QueryBuilders.spanTermQuery("description", "quick"), 1)
-                    .clause(QueryBuilders.spanTermQuery("description", "fox")).toXContent(builder, null);
+                    .addClause(QueryBuilders.spanTermQuery("description", "fox")).toXContent(builder, null);
             builder.field("exclude");
             spanTermQuery("description", "jumped").toXContent(builder, null);
             builder.field("dist", 2);
diff --git a/core/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java
index a9c4ec7d7dc..1622210b731 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java
@@ -35,7 +35,7 @@ public class SpanOrQueryBuilderTests extends AbstractQueryTestCase<SpanOrQueryBu
         SpanTermQueryBuilder[] spanTermQueries = new SpanTermQueryBuilderTests().createSpanTermQueryBuilders(randomIntBetween(1, 6));
         SpanOrQueryBuilder queryBuilder = new SpanOrQueryBuilder(spanTermQueries[0]);
         for (int i = 1; i < spanTermQueries.length; i++) {
-            queryBuilder.clause(spanTermQueries[i]);
+            queryBuilder.addClause(spanTermQueries[i]);
         }
         return queryBuilder;
     }
@@ -52,15 +52,18 @@ public class SpanOrQueryBuilderTests extends AbstractQueryTestCase<SpanOrQueryBu
     }
 
     public void testIllegalArguments() {
-        expectThrows(IllegalArgumentException.class, () -> new SpanOrQueryBuilder((SpanQueryBuilder) null));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new SpanOrQueryBuilder((SpanQueryBuilder) null));
+        assertEquals("[span_or] must include at least one clause", e.getMessage());
 
-        try {
-            SpanOrQueryBuilder spanOrBuilder = new SpanOrQueryBuilder(new SpanTermQueryBuilder("field", "value"));
-            spanOrBuilder.clause(null);
-            fail("cannot be null");
-        } catch (IllegalArgumentException e) {
-            // expected
-        }
+        SpanOrQueryBuilder spanOrBuilder = new SpanOrQueryBuilder(new SpanTermQueryBuilder("field", "value"));
+        e = expectThrows(IllegalArgumentException.class, () -> spanOrBuilder.addClause(null));
+        assertEquals("[span_or] inner clause cannot be null", e.getMessage());
+    }
+
+    public void testClausesUnmodifiable() {
+        SpanNearQueryBuilder spanNearQueryBuilder = new SpanNearQueryBuilder(new SpanTermQueryBuilder("field", "value"), 1);
+        expectThrows(UnsupportedOperationException.class,
+                () -> spanNearQueryBuilder.clauses().add(new SpanTermQueryBuilder("field", "value2")));
     }
 
     public void testFromJson() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/search/percolator/PercolatorQuerySearchIT.java b/core/src/test/java/org/elasticsearch/search/percolator/PercolatorQuerySearchIT.java
index 0065e46875a..067d84d5e00 100644
--- a/core/src/test/java/org/elasticsearch/search/percolator/PercolatorQuerySearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/percolator/PercolatorQuerySearchIT.java
@@ -161,8 +161,8 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
         client().prepareIndex("test", "queries", "3")
                 .setSource(jsonBuilder().startObject().field("query",
                         spanNearQuery(spanTermQuery("field1", "quick"), 0)
-                                .clause(spanTermQuery("field1", "brown"))
-                                .clause(spanTermQuery("field1", "fox"))
+                                .addClause(spanTermQuery("field1", "brown"))
+                                .addClause(spanTermQuery("field1", "fox"))
                                 .inOrder(true)
                 ).endObject())
                 .get();
@@ -172,12 +172,12 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
                 .setSource(jsonBuilder().startObject().field("query",
                         spanNotQuery(
                                 spanNearQuery(spanTermQuery("field1", "quick"), 0)
-                                        .clause(spanTermQuery("field1", "brown"))
-                                        .clause(spanTermQuery("field1", "fox"))
+                                        .addClause(spanTermQuery("field1", "brown"))
+                                        .addClause(spanTermQuery("field1", "fox"))
                                         .inOrder(true),
                                 spanNearQuery(spanTermQuery("field1", "the"), 0)
-                                        .clause(spanTermQuery("field1", "lazy"))
-                                        .clause(spanTermQuery("field1", "dog"))
+                                        .addClause(spanTermQuery("field1", "lazy"))
+                                        .addClause(spanTermQuery("field1", "dog"))
                                         .inOrder(true)).dist(2)
                 ).endObject())
                 .get();
@@ -187,12 +187,12 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
                 .setSource(jsonBuilder().startObject().field("query",
                         spanNotQuery(
                                 spanNearQuery(spanTermQuery("field1", "quick"), 0)
-                                        .clause(spanTermQuery("field1", "brown"))
-                                        .clause(spanTermQuery("field1", "fox"))
+                                        .addClause(spanTermQuery("field1", "brown"))
+                                        .addClause(spanTermQuery("field1", "fox"))
                                         .inOrder(true),
                                 spanNearQuery(spanTermQuery("field1", "the"), 0)
-                                        .clause(spanTermQuery("field1", "lazy"))
-                                        .clause(spanTermQuery("field1", "dog"))
+                                        .addClause(spanTermQuery("field1", "lazy"))
+                                        .addClause(spanTermQuery("field1", "dog"))
                                         .inOrder(true)).dist(3)
                 ).endObject())
                 .get();
diff --git a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
index 9dd32e091cd..882226afe63 100644
--- a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
@@ -1436,7 +1436,7 @@ public class SearchQueryIT extends ESIntegTestCase {
 
         searchResponse = client().prepareSearch("test").setQuery(
                 spanNearQuery(spanTermQuery("description", "foo"), 3)
-                        .clause(spanTermQuery("description", "other"))).get();
+                        .addClause(spanTermQuery("description", "other"))).get();
         assertHitCount(searchResponse, 3L);
     }
 
@@ -1481,17 +1481,17 @@ public class SearchQueryIT extends ESIntegTestCase {
 
         SearchResponse searchResponse = client().prepareSearch("test")
                 .setQuery(spanNotQuery(spanNearQuery(QueryBuilders.spanTermQuery("description", "quick"), 1)
-                        .clause(QueryBuilders.spanTermQuery("description", "fox")), spanTermQuery("description", "brown"))).get();
+                        .addClause(QueryBuilders.spanTermQuery("description", "fox")), spanTermQuery("description", "brown"))).get();
         assertHitCount(searchResponse, 1L);
 
         searchResponse = client().prepareSearch("test")
                 .setQuery(spanNotQuery(spanNearQuery(QueryBuilders.spanTermQuery("description", "quick"), 1)
-                        .clause(QueryBuilders.spanTermQuery("description", "fox")), spanTermQuery("description", "sleeping")).dist(5)).get();
+                        .addClause(QueryBuilders.spanTermQuery("description", "fox")), spanTermQuery("description", "sleeping")).dist(5)).get();
         assertHitCount(searchResponse, 1L);
 
         searchResponse = client().prepareSearch("test")
                 .setQuery(spanNotQuery(spanNearQuery(QueryBuilders.spanTermQuery("description", "quick"), 1)
-                        .clause(QueryBuilders.spanTermQuery("description", "fox")), spanTermQuery("description", "jumped")).pre(1).post(1)).get();
+                        .addClause(QueryBuilders.spanTermQuery("description", "fox")), spanTermQuery("description", "jumped")).pre(1).post(1)).get();
         assertHitCount(searchResponse, 1L);
     }
 
diff --git a/docs/reference/migration/migrate_5_0/java.asciidoc b/docs/reference/migration/migrate_5_0/java.asciidoc
index cdc471d19f2..da97d360b43 100644
--- a/docs/reference/migration/migrate_5_0/java.asciidoc
+++ b/docs/reference/migration/migrate_5_0/java.asciidoc
@@ -84,12 +84,14 @@ static factory methods in QueryBuilders accordingly.
 
 Making sure that query contains at least one clause by making initial clause mandatory
 in constructor.
+Renaming method to add clauses from `clause(SpanQueryBuilder)` to `addClause(SpanQueryBuilder)`. 
 
 ===== SpanNearQueryBuilder
 
 Removed setter for mandatory slop parameter, needs to be set in constructor now. Also
 making sure that query contains at least one clause by making initial clause mandatory
 in constructor. Updated the static factory methods in QueryBuilders accordingly.
+Renaming method to add clauses from `clause(SpanQueryBuilder)` to `addClause(SpanQueryBuilder)`. 
 
 ===== SpanNotQueryBuilder
 

From 6205f79554933bb1f10266d8b04c98c0841cc2a7 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Fri, 20 May 2016 14:03:46 +0200
Subject: [PATCH 0376/1311] Remove Definitions's copy-ctor; fix RuntimeClass to
 be unmodifiable Please note: The maps inside the pirvate singleton instance
 of Defininition are no longer unmodifiable, but nothing from the outside can
 modify it! All private :-)

---
 .../elasticsearch/painless/Definition.java    | 30 ++++++++-----------
 1 file changed, 12 insertions(+), 18 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 0a81188e776..5c45aaaf407 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -40,8 +40,7 @@ public final class Definition {
     
     private static final String DEFINITION_FILE = "definition.txt";
 
-    // The second construction is used to finalize all the variables, so there is no mistake of modification afterwards.
-    private static final Definition INSTANCE = new Definition(new Definition());
+    private static final Definition INSTANCE = new Definition();
     
     /** Some native types as constants: */
     public static final Type voidType = getType("void");
@@ -293,7 +292,7 @@ public final class Definition {
             staticMembers = new HashMap<>();
             members = new HashMap<>();
         }
-
+        
         private Struct(final Struct struct) {
             name = struct.name;
             clazz = struct.clazz;
@@ -307,6 +306,10 @@ public final class Definition {
             members = Collections.unmodifiableMap(struct.members);
         }
 
+        private Struct freeze() {
+            return new Struct(this);
+        }
+
         @Override
         public boolean equals(Object object) {
             if (this == object) {
@@ -385,9 +388,9 @@ public final class Definition {
 
         private RuntimeClass(final Map<MethodKey, Method> methods,
                              final Map<String, MethodHandle> getters, final Map<String, MethodHandle> setters) {
-            this.methods = methods;
-            this.getters = getters;
-            this.setters = setters;
+            this.methods = Collections.unmodifiableMap(methods);
+            this.getters = Collections.unmodifiableMap(getters);
+            this.setters = Collections.unmodifiableMap(setters);
         }
     }
 
@@ -436,19 +439,10 @@ public final class Definition {
         for (Struct struct : structsMap.values()) {
           addRuntimeClass(struct);
         }
-    }
-
-    private Definition(final Definition definition) {
-        final Map<String, Struct> structs = new HashMap<>();
-
-        for (final Struct struct : definition.structsMap.values()) {
-            structs.put(struct.name, new Struct(struct));
+        // copy all structs to make them unmodifiable for outside users:
+        for (final Map.Entry<String,Struct> entry : structsMap.entrySet()) {
+            entry.setValue(entry.getValue().freeze());
         }
-
-        this.structsMap = Collections.unmodifiableMap(structs);
-        this.transformsMap = Collections.unmodifiableMap(definition.transformsMap);
-        this.runtimeMap = Collections.unmodifiableMap(definition.runtimeMap);
-        this.simpleTypesMap = Collections.unmodifiableMap(definition.simpleTypesMap);
     }
 
     /** adds classes from definition. returns hierarchy */

From 63c5b314497c764ef05bb4ba0dc43f3bd2672da0 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 20 May 2016 14:10:19 +0200
Subject: [PATCH 0377/1311] update shas for httpclient and httpcore

---
 plugins/discovery-azure/licenses/httpclient-4.3.6.jar.sha1 | 1 -
 plugins/discovery-azure/licenses/httpclient-4.5.2.jar.sha1 | 1 +
 plugins/discovery-azure/licenses/httpcore-4.3.3.jar.sha1   | 1 -
 plugins/discovery-azure/licenses/httpcore-4.4.4.jar.sha1   | 1 +
 plugins/discovery-ec2/licenses/httpclient-4.3.6.jar.sha1   | 1 -
 plugins/discovery-ec2/licenses/httpclient-4.5.2.jar.sha1   | 1 +
 plugins/discovery-ec2/licenses/httpcore-4.3.3.jar.sha1     | 1 -
 plugins/discovery-ec2/licenses/httpcore-4.4.4.jar.sha1     | 1 +
 plugins/discovery-gce/licenses/httpclient-4.3.6.jar.sha1   | 1 -
 plugins/discovery-gce/licenses/httpclient-4.5.2.jar.sha1   | 1 +
 plugins/discovery-gce/licenses/httpcore-4.3.3.jar.sha1     | 1 -
 plugins/discovery-gce/licenses/httpcore-4.4.4.jar.sha1     | 1 +
 plugins/repository-gcs/licenses/httpclient-4.3.6.jar.sha1  | 1 -
 plugins/repository-gcs/licenses/httpclient-4.5.2.jar.sha1  | 1 +
 plugins/repository-gcs/licenses/httpcore-4.3.3.jar.sha1    | 1 -
 plugins/repository-gcs/licenses/httpcore-4.4.4.jar.sha1    | 1 +
 plugins/repository-s3/licenses/httpclient-4.3.6.jar.sha1   | 1 -
 plugins/repository-s3/licenses/httpclient-4.5.2.jar.sha1   | 1 +
 plugins/repository-s3/licenses/httpcore-4.3.3.jar.sha1     | 1 -
 plugins/repository-s3/licenses/httpcore-4.4.4.jar.sha1     | 1 +
 20 files changed, 10 insertions(+), 10 deletions(-)
 delete mode 100644 plugins/discovery-azure/licenses/httpclient-4.3.6.jar.sha1
 create mode 100644 plugins/discovery-azure/licenses/httpclient-4.5.2.jar.sha1
 delete mode 100644 plugins/discovery-azure/licenses/httpcore-4.3.3.jar.sha1
 create mode 100644 plugins/discovery-azure/licenses/httpcore-4.4.4.jar.sha1
 delete mode 100644 plugins/discovery-ec2/licenses/httpclient-4.3.6.jar.sha1
 create mode 100644 plugins/discovery-ec2/licenses/httpclient-4.5.2.jar.sha1
 delete mode 100644 plugins/discovery-ec2/licenses/httpcore-4.3.3.jar.sha1
 create mode 100644 plugins/discovery-ec2/licenses/httpcore-4.4.4.jar.sha1
 delete mode 100644 plugins/discovery-gce/licenses/httpclient-4.3.6.jar.sha1
 create mode 100644 plugins/discovery-gce/licenses/httpclient-4.5.2.jar.sha1
 delete mode 100644 plugins/discovery-gce/licenses/httpcore-4.3.3.jar.sha1
 create mode 100644 plugins/discovery-gce/licenses/httpcore-4.4.4.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/httpclient-4.3.6.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/httpclient-4.5.2.jar.sha1
 delete mode 100644 plugins/repository-gcs/licenses/httpcore-4.3.3.jar.sha1
 create mode 100644 plugins/repository-gcs/licenses/httpcore-4.4.4.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/httpclient-4.3.6.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/httpclient-4.5.2.jar.sha1
 delete mode 100644 plugins/repository-s3/licenses/httpcore-4.3.3.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/httpcore-4.4.4.jar.sha1

diff --git a/plugins/discovery-azure/licenses/httpclient-4.3.6.jar.sha1 b/plugins/discovery-azure/licenses/httpclient-4.3.6.jar.sha1
deleted file mode 100644
index 3d35ee99d07..00000000000
--- a/plugins/discovery-azure/licenses/httpclient-4.3.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4c47155e3e6c9a41a28db36680b828ced53b8af4
diff --git a/plugins/discovery-azure/licenses/httpclient-4.5.2.jar.sha1 b/plugins/discovery-azure/licenses/httpclient-4.5.2.jar.sha1
new file mode 100644
index 00000000000..6937112a09f
--- /dev/null
+++ b/plugins/discovery-azure/licenses/httpclient-4.5.2.jar.sha1
@@ -0,0 +1 @@
+733db77aa8d9b2d68015189df76ab06304406e50
\ No newline at end of file
diff --git a/plugins/discovery-azure/licenses/httpcore-4.3.3.jar.sha1 b/plugins/discovery-azure/licenses/httpcore-4.3.3.jar.sha1
deleted file mode 100644
index 5d9c0e26c09..00000000000
--- a/plugins/discovery-azure/licenses/httpcore-4.3.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f91b7a4aadc5cf486df6e4634748d7dd7a73f06d
diff --git a/plugins/discovery-azure/licenses/httpcore-4.4.4.jar.sha1 b/plugins/discovery-azure/licenses/httpcore-4.4.4.jar.sha1
new file mode 100644
index 00000000000..ef0c257e012
--- /dev/null
+++ b/plugins/discovery-azure/licenses/httpcore-4.4.4.jar.sha1
@@ -0,0 +1 @@
+b31526a230871fbe285fbcbe2813f9c0839ae9b0
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/httpclient-4.3.6.jar.sha1 b/plugins/discovery-ec2/licenses/httpclient-4.3.6.jar.sha1
deleted file mode 100644
index 3d35ee99d07..00000000000
--- a/plugins/discovery-ec2/licenses/httpclient-4.3.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4c47155e3e6c9a41a28db36680b828ced53b8af4
diff --git a/plugins/discovery-ec2/licenses/httpclient-4.5.2.jar.sha1 b/plugins/discovery-ec2/licenses/httpclient-4.5.2.jar.sha1
new file mode 100644
index 00000000000..6937112a09f
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/httpclient-4.5.2.jar.sha1
@@ -0,0 +1 @@
+733db77aa8d9b2d68015189df76ab06304406e50
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/httpcore-4.3.3.jar.sha1 b/plugins/discovery-ec2/licenses/httpcore-4.3.3.jar.sha1
deleted file mode 100644
index 5d9c0e26c09..00000000000
--- a/plugins/discovery-ec2/licenses/httpcore-4.3.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f91b7a4aadc5cf486df6e4634748d7dd7a73f06d
diff --git a/plugins/discovery-ec2/licenses/httpcore-4.4.4.jar.sha1 b/plugins/discovery-ec2/licenses/httpcore-4.4.4.jar.sha1
new file mode 100644
index 00000000000..ef0c257e012
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/httpcore-4.4.4.jar.sha1
@@ -0,0 +1 @@
+b31526a230871fbe285fbcbe2813f9c0839ae9b0
\ No newline at end of file
diff --git a/plugins/discovery-gce/licenses/httpclient-4.3.6.jar.sha1 b/plugins/discovery-gce/licenses/httpclient-4.3.6.jar.sha1
deleted file mode 100644
index 3d35ee99d07..00000000000
--- a/plugins/discovery-gce/licenses/httpclient-4.3.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4c47155e3e6c9a41a28db36680b828ced53b8af4
diff --git a/plugins/discovery-gce/licenses/httpclient-4.5.2.jar.sha1 b/plugins/discovery-gce/licenses/httpclient-4.5.2.jar.sha1
new file mode 100644
index 00000000000..6937112a09f
--- /dev/null
+++ b/plugins/discovery-gce/licenses/httpclient-4.5.2.jar.sha1
@@ -0,0 +1 @@
+733db77aa8d9b2d68015189df76ab06304406e50
\ No newline at end of file
diff --git a/plugins/discovery-gce/licenses/httpcore-4.3.3.jar.sha1 b/plugins/discovery-gce/licenses/httpcore-4.3.3.jar.sha1
deleted file mode 100644
index 5d9c0e26c09..00000000000
--- a/plugins/discovery-gce/licenses/httpcore-4.3.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f91b7a4aadc5cf486df6e4634748d7dd7a73f06d
diff --git a/plugins/discovery-gce/licenses/httpcore-4.4.4.jar.sha1 b/plugins/discovery-gce/licenses/httpcore-4.4.4.jar.sha1
new file mode 100644
index 00000000000..ef0c257e012
--- /dev/null
+++ b/plugins/discovery-gce/licenses/httpcore-4.4.4.jar.sha1
@@ -0,0 +1 @@
+b31526a230871fbe285fbcbe2813f9c0839ae9b0
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/httpclient-4.3.6.jar.sha1 b/plugins/repository-gcs/licenses/httpclient-4.3.6.jar.sha1
deleted file mode 100644
index 2c18ef0f54c..00000000000
--- a/plugins/repository-gcs/licenses/httpclient-4.3.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4c47155e3e6c9a41a28db36680b828ced53b8af4
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/httpclient-4.5.2.jar.sha1 b/plugins/repository-gcs/licenses/httpclient-4.5.2.jar.sha1
new file mode 100644
index 00000000000..6937112a09f
--- /dev/null
+++ b/plugins/repository-gcs/licenses/httpclient-4.5.2.jar.sha1
@@ -0,0 +1 @@
+733db77aa8d9b2d68015189df76ab06304406e50
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/httpcore-4.3.3.jar.sha1 b/plugins/repository-gcs/licenses/httpcore-4.3.3.jar.sha1
deleted file mode 100644
index 0ad1d24aa9f..00000000000
--- a/plugins/repository-gcs/licenses/httpcore-4.3.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f91b7a4aadc5cf486df6e4634748d7dd7a73f06d
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/httpcore-4.4.4.jar.sha1 b/plugins/repository-gcs/licenses/httpcore-4.4.4.jar.sha1
new file mode 100644
index 00000000000..ef0c257e012
--- /dev/null
+++ b/plugins/repository-gcs/licenses/httpcore-4.4.4.jar.sha1
@@ -0,0 +1 @@
+b31526a230871fbe285fbcbe2813f9c0839ae9b0
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/httpclient-4.3.6.jar.sha1 b/plugins/repository-s3/licenses/httpclient-4.3.6.jar.sha1
deleted file mode 100644
index 3d35ee99d07..00000000000
--- a/plugins/repository-s3/licenses/httpclient-4.3.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4c47155e3e6c9a41a28db36680b828ced53b8af4
diff --git a/plugins/repository-s3/licenses/httpclient-4.5.2.jar.sha1 b/plugins/repository-s3/licenses/httpclient-4.5.2.jar.sha1
new file mode 100644
index 00000000000..6937112a09f
--- /dev/null
+++ b/plugins/repository-s3/licenses/httpclient-4.5.2.jar.sha1
@@ -0,0 +1 @@
+733db77aa8d9b2d68015189df76ab06304406e50
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/httpcore-4.3.3.jar.sha1 b/plugins/repository-s3/licenses/httpcore-4.3.3.jar.sha1
deleted file mode 100644
index 5d9c0e26c09..00000000000
--- a/plugins/repository-s3/licenses/httpcore-4.3.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f91b7a4aadc5cf486df6e4634748d7dd7a73f06d
diff --git a/plugins/repository-s3/licenses/httpcore-4.4.4.jar.sha1 b/plugins/repository-s3/licenses/httpcore-4.4.4.jar.sha1
new file mode 100644
index 00000000000..ef0c257e012
--- /dev/null
+++ b/plugins/repository-s3/licenses/httpcore-4.4.4.jar.sha1
@@ -0,0 +1 @@
+b31526a230871fbe285fbcbe2813f9c0839ae9b0
\ No newline at end of file

From 7bc91c00d9329268ac5bb612f442d74fcf04ae73 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Fri, 20 May 2016 14:12:15 +0200
Subject: [PATCH 0378/1311] More elegant way to force class initialization

---
 .../java/org/elasticsearch/painless/PainlessPlugin.java  | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
index de774344ee2..7533707cf63 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
@@ -30,14 +30,9 @@ import org.elasticsearch.script.ScriptModule;
  */
 public final class PainlessPlugin extends Plugin {
 
-    // parse our definition at startup (not on the user's first script)
-    // compilation process is sandboxed and has no file access.
+    // force to pare our definition at startup (not on the user's first script)
     static {
-        try {
-            Class.forName("org.elasticsearch.painless.Definition");
-        } catch (ClassNotFoundException e) {
-            throw new RuntimeException(e);
-        }
+        Definition.voidType.hashCode();
     }
 
     @Override

From 4de284866875b987110df66229fb7c7d0b9cd4d8 Mon Sep 17 00:00:00 2001
From: Julien Pivotto <roidelapluie@inuits.eu>
Date: Fri, 20 May 2016 14:01:59 +0200
Subject: [PATCH 0379/1311] Doc: /_reindex: Add a note about the source size
 parameter

Closes #18484
---
 docs/reference/docs/reindex.asciidoc | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index bde6ad8b611..2dbecf922d6 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -131,6 +131,23 @@ POST _reindex
 // CONSOLE
 // TEST[setup:twitter]
 
+You can increase the default batch size by setting the `size` parameter (which defaults to `1000`) to the `source`. Note that it is different than the `size` parameter available at the root, which limits the number of documents (see below).
+
+[source,js]
+--------------------------------------------------
+POST /_reindex
+{
+  "source": {
+    "index": "twitter",
+    "size": 2000
+  },
+  "dest": {
+    "index": "new_twitter"
+  }
+}
+--------------------------------------------------
+// AUTOSENSE
+
 You can limit the documents by adding a type to the `source` or by adding a
 query. This will only copy ++tweet++&apos;s made by `kimchy` into `new_twitter`:
 

From dafa78ec63a3e7c20f510d95fef286308444430d Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Fri, 20 May 2016 14:39:18 +0200
Subject: [PATCH 0380/1311] Revert "Doc: /_reindex: Add a note about the source
 size parameter"

This reverts commit 4de284866875b987110df66229fb7c7d0b9cd4d8.
---
 docs/reference/docs/reindex.asciidoc | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index 2dbecf922d6..bde6ad8b611 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -131,23 +131,6 @@ POST _reindex
 // CONSOLE
 // TEST[setup:twitter]
 
-You can increase the default batch size by setting the `size` parameter (which defaults to `1000`) to the `source`. Note that it is different than the `size` parameter available at the root, which limits the number of documents (see below).
-
-[source,js]
---------------------------------------------------
-POST /_reindex
-{
-  "source": {
-    "index": "twitter",
-    "size": 2000
-  },
-  "dest": {
-    "index": "new_twitter"
-  }
-}
---------------------------------------------------
-// AUTOSENSE
-
 You can limit the documents by adding a type to the `source` or by adding a
 query. This will only copy ++tweet++&apos;s made by `kimchy` into `new_twitter`:
 

From b3c71c1e3ad8b3c2516b0898dbaa696b69cb7abb Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Fri, 20 May 2016 08:42:29 -0400
Subject: [PATCH 0381/1311] improve style for these constants

---
 .../painless/AnalyzerCaster.java              | 96 +++++++++----------
 .../elasticsearch/painless/Definition.java    | 46 ++++-----
 .../elasticsearch/painless/MethodWriter.java  | 22 ++---
 .../painless/PainlessPlugin.java              |  2 +-
 .../elasticsearch/painless/node/EBinary.java  |  6 +-
 .../elasticsearch/painless/node/EBool.java    |  6 +-
 .../elasticsearch/painless/node/EBoolean.java |  2 +-
 .../elasticsearch/painless/node/EChain.java   |  6 +-
 .../elasticsearch/painless/node/EComp.java    | 32 +++----
 .../painless/node/EConditional.java           |  2 +-
 .../painless/node/EConstant.java              | 18 ++--
 .../elasticsearch/painless/node/EDecimal.java |  4 +-
 .../elasticsearch/painless/node/ENull.java    |  2 +-
 .../elasticsearch/painless/node/ENumeric.java | 14 +--
 .../elasticsearch/painless/node/EUnary.java   |  8 +-
 .../painless/node/LArrayLength.java           |  2 +-
 .../elasticsearch/painless/node/LBrace.java   |  2 +-
 .../painless/node/LDefArray.java              |  6 +-
 .../elasticsearch/painless/node/LDefCall.java |  4 +-
 .../painless/node/LDefField.java              |  6 +-
 .../painless/node/LListShortcut.java          |  2 +-
 .../painless/node/LNewArray.java              |  2 +-
 .../elasticsearch/painless/node/LString.java  |  2 +-
 .../org/elasticsearch/painless/node/SDo.java  |  2 +-
 .../painless/node/SExpression.java            |  2 +-
 .../org/elasticsearch/painless/node/SFor.java |  2 +-
 .../elasticsearch/painless/node/SIfElse.java  |  2 +-
 .../elasticsearch/painless/node/SReturn.java  |  2 +-
 .../elasticsearch/painless/node/SThrow.java   |  2 +-
 .../elasticsearch/painless/node/SWhile.java   |  2 +-
 30 files changed, 153 insertions(+), 153 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
index 2495d2edacd..da7debb7e05 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
@@ -118,18 +118,18 @@ public final class AnalyzerCaster {
         final Sort sort = from.sort;
 
         if (sort == Sort.DEF) {
-            return Definition.defType;
+            return Definition.DEF_TYPE;
         } else if ((sort == Sort.DOUBLE || sort == Sort.DOUBLE_OBJ) && decimal) {
-            return primitive ? Definition.doubleType : Definition.doubleobjType;
+            return primitive ? Definition.DOUBLE_TYPE : Definition.DOUBLE_OBJ_TYPE;
         } else if ((sort == Sort.FLOAT || sort == Sort.FLOAT_OBJ) && decimal) {
-            return primitive ? Definition.floatType : Definition.floatobjType;
+            return primitive ? Definition.FLOAT_TYPE : Definition.FLOAT_OBJ_TYPE;
         } else if (sort == Sort.LONG || sort == Sort.LONG_OBJ) {
-            return primitive ? Definition.longType : Definition.longobjType;
+            return primitive ? Definition.LONG_TYPE : Definition.LONG_OBJ_TYPE;
         } else if (sort == Sort.INT   || sort == Sort.INT_OBJ   ||
                    sort == Sort.CHAR  || sort == Sort.CHAR_OBJ  ||
                    sort == Sort.SHORT || sort == Sort.SHORT_OBJ ||
                    sort == Sort.BYTE  || sort == Sort.BYTE_OBJ) {
-            return primitive ? Definition.intType : Definition.intobjType;
+            return primitive ? Definition.INT_TYPE : Definition.INT_OBJ_TYPE;
         }
 
         return null;
@@ -140,21 +140,21 @@ public final class AnalyzerCaster {
         final Sort sort1 = from1.sort;
 
         if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return Definition.defType;
+            return Definition.DEF_TYPE;
         }
 
         if (decimal) {
             if (sort0 == Sort.DOUBLE || sort0 == Sort.DOUBLE_OBJ ||
                 sort1 == Sort.DOUBLE || sort1 == Sort.DOUBLE_OBJ) {
-                return primitive ? Definition.doubleType : Definition.doubleobjType;
+                return primitive ? Definition.DOUBLE_TYPE : Definition.DOUBLE_OBJ_TYPE;
             } else if (sort0 == Sort.FLOAT || sort0 == Sort.FLOAT_OBJ || sort1 == Sort.FLOAT || sort1 == Sort.FLOAT_OBJ) {
-                return primitive ? Definition.floatType : Definition.floatobjType;
+                return primitive ? Definition.FLOAT_TYPE : Definition.FLOAT_OBJ_TYPE;
             }
         }
 
         if (sort0 == Sort.LONG || sort0 == Sort.LONG_OBJ ||
             sort1 == Sort.LONG || sort1 == Sort.LONG_OBJ) {
-            return primitive ? Definition.longType : Definition.longobjType;
+            return primitive ? Definition.LONG_TYPE : Definition.LONG_OBJ_TYPE;
         } else if (sort0 == Sort.INT   || sort0 == Sort.INT_OBJ   ||
                    sort1 == Sort.INT   || sort1 == Sort.INT_OBJ   ||
                    sort0 == Sort.CHAR  || sort0 == Sort.CHAR_OBJ  ||
@@ -163,7 +163,7 @@ public final class AnalyzerCaster {
                    sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ ||
                    sort0 == Sort.BYTE  || sort0 == Sort.BYTE_OBJ  ||
                    sort1 == Sort.BYTE  || sort1 == Sort.BYTE_OBJ) {
-            return primitive ? Definition.intType : Definition.intobjType;
+            return primitive ? Definition.INT_TYPE : Definition.INT_OBJ_TYPE;
         }
 
         return null;
@@ -174,7 +174,7 @@ public final class AnalyzerCaster {
         final Sort sort1 = from1.sort;
 
         if (sort0 == Sort.STRING || sort1 == Sort.STRING) {
-            return Definition.stringType;
+            return Definition.STRING_TYPE;
         }
 
         return promoteNumeric(from0, from1, true, true);
@@ -185,7 +185,7 @@ public final class AnalyzerCaster {
         final Sort sort1 = from1.sort;
 
         if (sort0.bool || sort1.bool) {
-            return Definition.booleanType;
+            return Definition.BOOLEAN_TYPE;
         }
 
         return promoteNumeric(from0, from1, false, true);
@@ -196,20 +196,20 @@ public final class AnalyzerCaster {
         final Sort sort1 = from1.sort;
 
         if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return Definition.defType;
+            return Definition.DEF_TYPE;
         }
 
         final boolean primitive = sort0.primitive && sort1.primitive;
 
         if (sort0.bool && sort1.bool) {
-            return primitive ? Definition.booleanType : Definition.booleanobjType;
+            return primitive ? Definition.BOOLEAN_TYPE : Definition.BOOLEAN_OBJ_TYPE;
         }
 
         if (sort0.numeric && sort1.numeric) {
             return promoteNumeric(from0, from1, true, primitive);
         }
 
-        return Definition.objectType;
+        return Definition.OBJECT_TYPE;
     }
 
     public static Type promoteReference(final Type from0, final Type from1) {
@@ -217,12 +217,12 @@ public final class AnalyzerCaster {
         final Sort sort1 = from1.sort;
 
         if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return Definition.defType;
+            return Definition.DEF_TYPE;
         }
 
         if (sort0.primitive && sort1.primitive) {
             if (sort0.bool && sort1.bool) {
-                return Definition.booleanType;
+                return Definition.BOOLEAN_TYPE;
             }
 
             if (sort0.numeric && sort1.numeric) {
@@ -230,7 +230,7 @@ public final class AnalyzerCaster {
             }
         }
 
-        return Definition.objectType;
+        return Definition.OBJECT_TYPE;
     }
 
     public static Type promoteConditional(final Type from0, final Type from1, final Object const0, final Object const1) {
@@ -242,48 +242,48 @@ public final class AnalyzerCaster {
         final Sort sort1 = from1.sort;
 
         if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
-            return Definition.defType;
+            return Definition.DEF_TYPE;
         }
 
         final boolean primitive = sort0.primitive && sort1.primitive;
 
         if (sort0.bool && sort1.bool) {
-            return primitive ? Definition.booleanType : Definition.booleanobjType;
+            return primitive ? Definition.BOOLEAN_TYPE : Definition.BOOLEAN_OBJ_TYPE;
         }
 
         if (sort0.numeric && sort1.numeric) {
             if (sort0 == Sort.DOUBLE || sort0 == Sort.DOUBLE_OBJ || sort1 == Sort.DOUBLE || sort1 == Sort.DOUBLE_OBJ) {
-                return primitive ? Definition.doubleType : Definition.doubleobjType;
+                return primitive ? Definition.DOUBLE_TYPE : Definition.DOUBLE_OBJ_TYPE;
             } else if (sort0 == Sort.FLOAT || sort0 == Sort.FLOAT_OBJ || sort1 == Sort.FLOAT || sort1 == Sort.FLOAT_OBJ) {
-                return primitive ? Definition.floatType : Definition.floatobjType;
+                return primitive ? Definition.FLOAT_TYPE : Definition.FLOAT_OBJ_TYPE;
             } else if (sort0 == Sort.LONG || sort0 == Sort.LONG_OBJ || sort1 == Sort.LONG || sort1 == Sort.LONG_OBJ) {
-                return sort0.primitive && sort1.primitive ? Definition.longType : Definition.longobjType;
+                return sort0.primitive && sort1.primitive ? Definition.LONG_TYPE : Definition.LONG_OBJ_TYPE;
             } else {
                 if (sort0 == Sort.BYTE || sort0 == Sort.BYTE_OBJ) {
                     if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
-                        return primitive ? Definition.byteType : Definition.byteobjType;
+                        return primitive ? Definition.BYTE_TYPE : Definition.BYTE_OBJ_TYPE;
                     } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
                         if (const1 != null) {
                             final short constant = (short)const1;
 
                             if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? Definition.byteType : Definition.byteobjType;
+                                return primitive ? Definition.BYTE_TYPE : Definition.BYTE_OBJ_TYPE;
                             }
                         }
 
-                        return primitive ? Definition.shortType : Definition.shortobjType;
+                        return primitive ? Definition.SHORT_TYPE : Definition.SHORT_OBJ_TYPE;
                     } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
-                        return primitive ? Definition.intType : Definition.intobjType;
+                        return primitive ? Definition.INT_TYPE : Definition.INT_OBJ_TYPE;
                     } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
                         if (const1 != null) {
                             final int constant = (int)const1;
 
                             if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? Definition.byteType : Definition.byteobjType;
+                                return primitive ? Definition.BYTE_TYPE : Definition.BYTE_OBJ_TYPE;
                             }
                         }
 
-                        return primitive ? Definition.intType : Definition.intobjType;
+                        return primitive ? Definition.INT_TYPE : Definition.INT_OBJ_TYPE;
                     }
                 } else if (sort0 == Sort.SHORT || sort0 == Sort.SHORT_OBJ) {
                     if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
@@ -291,43 +291,43 @@ public final class AnalyzerCaster {
                             final short constant = (short)const0;
 
                             if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? Definition.byteType : Definition.byteobjType;
+                                return primitive ? Definition.BYTE_TYPE : Definition.BYTE_OBJ_TYPE;
                             }
                         }
 
-                        return primitive ? Definition.shortType : Definition.shortobjType;
+                        return primitive ? Definition.SHORT_TYPE : Definition.SHORT_OBJ_TYPE;
                     } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
-                        return primitive ? Definition.shortType : Definition.shortobjType;
+                        return primitive ? Definition.SHORT_TYPE : Definition.SHORT_OBJ_TYPE;
                     } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
-                        return primitive ? Definition.intType : Definition.intobjType;
+                        return primitive ? Definition.INT_TYPE : Definition.INT_OBJ_TYPE;
                     } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
                         if (const1 != null) {
                             final int constant = (int)const1;
 
                             if (constant <= Short.MAX_VALUE && constant >= Short.MIN_VALUE) {
-                                return primitive ? Definition.shortType : Definition.shortobjType;
+                                return primitive ? Definition.SHORT_TYPE : Definition.SHORT_OBJ_TYPE;
                             }
                         }
 
-                        return primitive ? Definition.intType : Definition.intobjType;
+                        return primitive ? Definition.INT_TYPE : Definition.INT_OBJ_TYPE;
                     }
                 } else if (sort0 == Sort.CHAR || sort0 == Sort.CHAR_OBJ) {
                     if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
-                        return primitive ? Definition.intType : Definition.intobjType;
+                        return primitive ? Definition.INT_TYPE : Definition.INT_OBJ_TYPE;
                     } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
-                        return primitive ? Definition.intType : Definition.intobjType;
+                        return primitive ? Definition.INT_TYPE : Definition.INT_OBJ_TYPE;
                     } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
-                        return primitive ? Definition.charType : Definition.charobjType;
+                        return primitive ? Definition.CHAR_TYPE : Definition.CHAR_OBJ_TYPE;
                     } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
                         if (const1 != null) {
                             final int constant = (int)const1;
 
                             if (constant <= Character.MAX_VALUE && constant >= Character.MIN_VALUE) {
-                                return primitive ? Definition.byteType : Definition.byteobjType;
+                                return primitive ? Definition.BYTE_TYPE : Definition.BYTE_OBJ_TYPE;
                             }
                         }
 
-                        return primitive ? Definition.intType : Definition.intobjType;
+                        return primitive ? Definition.INT_TYPE : Definition.INT_OBJ_TYPE;
                     }
                 } else if (sort0 == Sort.INT || sort0 == Sort.INT_OBJ) {
                     if (sort1 == Sort.BYTE || sort1 == Sort.BYTE_OBJ) {
@@ -335,33 +335,33 @@ public final class AnalyzerCaster {
                             final int constant = (int)const0;
 
                             if (constant <= Byte.MAX_VALUE && constant >= Byte.MIN_VALUE) {
-                                return primitive ? Definition.byteType : Definition.byteobjType;
+                                return primitive ? Definition.BYTE_TYPE : Definition.BYTE_OBJ_TYPE;
                             }
                         }
 
-                        return primitive ? Definition.intType : Definition.intobjType;
+                        return primitive ? Definition.INT_TYPE : Definition.INT_OBJ_TYPE;
                     } else if (sort1 == Sort.SHORT || sort1 == Sort.SHORT_OBJ) {
                         if (const0 != null) {
                             final int constant = (int)const0;
 
                             if (constant <= Short.MAX_VALUE && constant >= Short.MIN_VALUE) {
-                                return primitive ? Definition.byteType : Definition.byteobjType;
+                                return primitive ? Definition.BYTE_TYPE : Definition.BYTE_OBJ_TYPE;
                             }
                         }
 
-                        return primitive ? Definition.intType : Definition.intobjType;
+                        return primitive ? Definition.INT_TYPE : Definition.INT_OBJ_TYPE;
                     } else if (sort1 == Sort.CHAR || sort1 == Sort.CHAR_OBJ) {
                         if (const0 != null) {
                             final int constant = (int)const0;
 
                             if (constant <= Character.MAX_VALUE && constant >= Character.MIN_VALUE) {
-                                return primitive ? Definition.byteType : Definition.byteobjType;
+                                return primitive ? Definition.BYTE_TYPE : Definition.BYTE_OBJ_TYPE;
                             }
                         }
 
-                        return primitive ? Definition.intType : Definition.intobjType;
+                        return primitive ? Definition.INT_TYPE : Definition.INT_OBJ_TYPE;
                     } else if (sort1 == Sort.INT || sort1 == Sort.INT_OBJ) {
-                        return primitive ? Definition.intType : Definition.intobjType;
+                        return primitive ? Definition.INT_TYPE : Definition.INT_OBJ_TYPE;
                     }
                 }
             }
@@ -371,7 +371,7 @@ public final class AnalyzerCaster {
         //       to calculate the highest upper bound for the two types and return that.
         //       However, for now we just return objectType that may require an extra cast.
 
-        return Definition.objectType;
+        return Definition.OBJECT_TYPE;
     }
 
     private AnalyzerCaster() {}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 5c45aaaf407..9e665e8280f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -43,29 +43,29 @@ public final class Definition {
     private static final Definition INSTANCE = new Definition();
     
     /** Some native types as constants: */
-    public static final Type voidType = getType("void");
-    public static final Type booleanType = getType("boolean");
-    public static final Type booleanobjType = getType("Boolean");
-    public static final Type byteType = getType("byte");
-    public static final Type byteobjType = getType("Byte");
-    public static final Type shortType = getType("short");
-    public static final Type shortobjType = getType("Short");
-    public static final Type intType = getType("int");
-    public static final Type intobjType = getType("Integer");
-    public static final Type longType = getType("long");
-    public static final Type longobjType = getType("Long");
-    public static final Type floatType = getType("float");
-    public static final Type floatobjType = getType("Float");
-    public static final Type doubleType = getType("double");
-    public static final Type doubleobjType = getType("Double");
-    public static final Type charType = getType("char");
-    public static final Type charobjType = getType("Character");
-    public static final Type objectType = getType("Object");
-    public static final Type defType = getType("def");
-    public static final Type defobjType = getType("Def");
-    public static final Type stringType = getType("String");
-    public static final Type exceptionType = getType("Exception");
-    public static final Type utilityType = getType("Utility");
+    public static final Type VOID_TYPE = getType("void");
+    public static final Type BOOLEAN_TYPE = getType("boolean");
+    public static final Type BOOLEAN_OBJ_TYPE = getType("Boolean");
+    public static final Type BYTE_TYPE = getType("byte");
+    public static final Type BYTE_OBJ_TYPE = getType("Byte");
+    public static final Type SHORT_TYPE = getType("short");
+    public static final Type SHORT_OBJ_TYPE = getType("Short");
+    public static final Type INT_TYPE = getType("int");
+    public static final Type INT_OBJ_TYPE = getType("Integer");
+    public static final Type LONG_TYPE = getType("long");
+    public static final Type LONG_OBJ_TYPE = getType("Long");
+    public static final Type FLOAT_TYPE = getType("float");
+    public static final Type FLOAT_OBJ_TYPE = getType("Float");
+    public static final Type DOUBLE_TYPE = getType("double");
+    public static final Type DOUBLE_OBJ_TYPE = getType("Double");
+    public static final Type CHAR_TYPE = getType("char");
+    public static final Type CHAR_OBJ_TYPE = getType("Character");
+    public static final Type OBJECT_TYPE = getType("Object");
+    public static final Type DEF_TYPE = getType("def");
+    public static final Type DEF_UTIL_TYPE = getType("Def");
+    public static final Type STRING_TYPE = getType("String");
+    public static final Type EXCEPTION_TYPE = getType("Exception");
+    public static final Type UTILITY_TYPE = getType("Utility");
 
     public enum Sort {
         VOID(       void.class      , 0 , true  , false , false , false ),
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index cddd1326c74..ebc7d7b9f61 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -212,17 +212,17 @@ public final class MethodWriter extends GeneratorAdapter {
         
         if (sort == Sort.DEF) {
             switch (operation) {
-                case MUL:   invokeStatic(Definition.defobjType.type, DEF_MUL_CALL); break;
-                case DIV:   invokeStatic(Definition.defobjType.type, DEF_DIV_CALL); break;
-                case REM:   invokeStatic(Definition.defobjType.type, DEF_REM_CALL); break;
-                case ADD:   invokeStatic(Definition.defobjType.type, DEF_ADD_CALL); break;
-                case SUB:   invokeStatic(Definition.defobjType.type, DEF_SUB_CALL); break;
-                case LSH:   invokeStatic(Definition.defobjType.type, DEF_LSH_CALL); break;
-                case USH:   invokeStatic(Definition.defobjType.type, DEF_RSH_CALL); break;
-                case RSH:   invokeStatic(Definition.defobjType.type, DEF_USH_CALL); break;
-                case BWAND: invokeStatic(Definition.defobjType.type, DEF_AND_CALL); break;
-                case XOR:   invokeStatic(Definition.defobjType.type, DEF_XOR_CALL); break;
-                case BWOR:  invokeStatic(Definition.defobjType.type, DEF_OR_CALL);  break;
+                case MUL:   invokeStatic(Definition.DEF_UTIL_TYPE.type, DEF_MUL_CALL); break;
+                case DIV:   invokeStatic(Definition.DEF_UTIL_TYPE.type, DEF_DIV_CALL); break;
+                case REM:   invokeStatic(Definition.DEF_UTIL_TYPE.type, DEF_REM_CALL); break;
+                case ADD:   invokeStatic(Definition.DEF_UTIL_TYPE.type, DEF_ADD_CALL); break;
+                case SUB:   invokeStatic(Definition.DEF_UTIL_TYPE.type, DEF_SUB_CALL); break;
+                case LSH:   invokeStatic(Definition.DEF_UTIL_TYPE.type, DEF_LSH_CALL); break;
+                case USH:   invokeStatic(Definition.DEF_UTIL_TYPE.type, DEF_RSH_CALL); break;
+                case RSH:   invokeStatic(Definition.DEF_UTIL_TYPE.type, DEF_USH_CALL); break;
+                case BWAND: invokeStatic(Definition.DEF_UTIL_TYPE.type, DEF_AND_CALL); break;
+                case XOR:   invokeStatic(Definition.DEF_UTIL_TYPE.type, DEF_XOR_CALL); break;
+                case BWOR:  invokeStatic(Definition.DEF_UTIL_TYPE.type, DEF_OR_CALL);  break;
                 default:
                     throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
index 7533707cf63..8f116c3c6f7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
@@ -32,7 +32,7 @@ public final class PainlessPlugin extends Plugin {
 
     // force to pare our definition at startup (not on the user's first script)
     static {
-        Definition.voidType.hashCode();
+        Definition.VOID_TYPE.hashCode();
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 38871d007f4..0046d498074 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -284,7 +284,7 @@ public final class EBinary extends AExpression {
         }
 
         left.expected = promote;
-        right.expected = Definition.intType;
+        right.expected = Definition.INT_TYPE;
         right.explicit = true;
 
         left = left.cast(settings, variables);
@@ -317,7 +317,7 @@ public final class EBinary extends AExpression {
         }
 
         left.expected = promote;
-        right.expected = Definition.intType;
+        right.expected = Definition.INT_TYPE;
         right.explicit = true;
 
         left = left.cast(settings, variables);
@@ -350,7 +350,7 @@ public final class EBinary extends AExpression {
         }
 
         left.expected = promote;
-        right.expected = Definition.intType;
+        right.expected = Definition.INT_TYPE;
         right.explicit = true;
 
         left = left.cast(settings, variables);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
index 28072b9d9bd..1e2af41c0c0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
@@ -45,11 +45,11 @@ public final class EBool extends AExpression {
 
     @Override
     void analyze(final CompilerSettings settings, final Variables variables) {
-        left.expected = Definition.booleanType;
+        left.expected = Definition.BOOLEAN_TYPE;
         left.analyze(settings, variables);
         left = left.cast(settings, variables);
 
-        right.expected = Definition.booleanType;
+        right.expected = Definition.BOOLEAN_TYPE;
         right.analyze(settings, variables);
         right = right.cast(settings, variables);
 
@@ -63,7 +63,7 @@ public final class EBool extends AExpression {
             }
         }
 
-        actual = Definition.booleanType;
+        actual = Definition.BOOLEAN_TYPE;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
index 979fbed03e3..2e019c576e0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
@@ -37,7 +37,7 @@ public final class EBoolean extends AExpression {
 
     @Override
     void analyze(final CompilerSettings settings, final Variables variables) {
-        actual = Definition.booleanType;
+        actual = Definition.BOOLEAN_TYPE;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index d992edd8505..2f8ad66f101 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -199,7 +199,7 @@ public final class EChain extends AExpression {
 
             expression.expected = expression.actual;
         } else if (operation == Operation.LSH || operation == Operation.RSH || operation == Operation.USH) {
-            expression.expected = Definition.intType;
+            expression.expected = Definition.INT_TYPE;
             expression.explicit = true;
         } else {
             expression.expected = promote;
@@ -211,7 +211,7 @@ public final class EChain extends AExpression {
         back = AnalyzerCaster.getLegalCast(location, promote, last.after, true);
 
         this.statement = true;
-        this.actual = read ? last.after : Definition.voidType;
+        this.actual = read ? last.after : Definition.VOID_TYPE;
     }
 
     private void analyzeWrite(final CompilerSettings settings, final Variables variables) {
@@ -231,7 +231,7 @@ public final class EChain extends AExpression {
         expression = expression.cast(settings, variables);
 
         this.statement = true;
-        this.actual = read ? last.after : Definition.voidType;
+        this.actual = read ? last.after : Definition.VOID_TYPE;
     }
 
     private void analyzeRead() {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index 68c45f4870e..75dd1bb1f14 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -119,7 +119,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = Definition.booleanType;
+        actual = Definition.BOOLEAN_TYPE;
     }
 
     private void analyzeEqR(final CompilerSettings settings, final Variables variables) {
@@ -161,7 +161,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = Definition.booleanType;
+        actual = Definition.BOOLEAN_TYPE;
     }
 
     private void analyzeNE(final CompilerSettings settings, final Variables variables) {
@@ -207,7 +207,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = Definition.booleanType;
+        actual = Definition.BOOLEAN_TYPE;
     }
 
     private void analyzeNER(final CompilerSettings settings, final Variables variables) {
@@ -249,7 +249,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = Definition.booleanType;
+        actual = Definition.BOOLEAN_TYPE;
     }
 
     private void analyzeGTE(final CompilerSettings settings, final Variables variables) {
@@ -285,7 +285,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = Definition.booleanType;
+        actual = Definition.BOOLEAN_TYPE;
     }
 
     private void analyzeGT(final CompilerSettings settings, final Variables variables) {
@@ -321,7 +321,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = Definition.booleanType;
+        actual = Definition.BOOLEAN_TYPE;
     }
 
     private void analyzeLTE(final CompilerSettings settings, final Variables variables) {
@@ -357,7 +357,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = Definition.booleanType;
+        actual = Definition.BOOLEAN_TYPE;
     }
 
     private void analyzeLT(final CompilerSettings settings, final Variables variables) {
@@ -393,7 +393,7 @@ public final class EComp extends AExpression {
             }
         }
 
-        actual = Definition.booleanType;
+        actual = Definition.BOOLEAN_TYPE;
     }
 
     @Override
@@ -456,7 +456,7 @@ public final class EComp extends AExpression {
                     if (right.isNull) {
                         adapter.ifNull(jump);
                     } else if (!left.isNull && operation == Operation.EQ) {
-                        adapter.invokeStatic(Definition.defobjType.type, DEF_EQ_CALL);
+                        adapter.invokeStatic(Definition.DEF_UTIL_TYPE.type, DEF_EQ_CALL);
                     } else {
                         adapter.ifCmp(rtype, MethodWriter.EQ, jump);
                     }
@@ -464,19 +464,19 @@ public final class EComp extends AExpression {
                     if (right.isNull) {
                         adapter.ifNonNull(jump);
                     } else if (!left.isNull && operation == Operation.NE) {
-                        adapter.invokeStatic(Definition.defobjType.type, DEF_EQ_CALL);
+                        adapter.invokeStatic(Definition.DEF_UTIL_TYPE.type, DEF_EQ_CALL);
                         adapter.ifZCmp(MethodWriter.EQ, jump);
                     } else {
                         adapter.ifCmp(rtype, MethodWriter.NE, jump);
                     }
                 } else if (lt) {
-                    adapter.invokeStatic(Definition.defobjType.type, DEF_LT_CALL);
+                    adapter.invokeStatic(Definition.DEF_UTIL_TYPE.type, DEF_LT_CALL);
                 } else if (lte) {
-                    adapter.invokeStatic(Definition.defobjType.type, DEF_LTE_CALL);
+                    adapter.invokeStatic(Definition.DEF_UTIL_TYPE.type, DEF_LTE_CALL);
                 } else if (gt) {
-                    adapter.invokeStatic(Definition.defobjType.type, DEF_GT_CALL);
+                    adapter.invokeStatic(Definition.DEF_UTIL_TYPE.type, DEF_GT_CALL);
                 } else if (gte) {
-                    adapter.invokeStatic(Definition.defobjType.type, DEF_GTE_CALL);
+                    adapter.invokeStatic(Definition.DEF_UTIL_TYPE.type, DEF_GTE_CALL);
                 } else {
                     throw new IllegalStateException(error("Illegal tree structure."));
                 }
@@ -493,7 +493,7 @@ public final class EComp extends AExpression {
                     if (right.isNull) {
                         adapter.ifNull(jump);
                     } else if (operation == Operation.EQ) {
-                        adapter.invokeStatic(Definition.utilityType.type, CHECKEQUALS);
+                        adapter.invokeStatic(Definition.UTILITY_TYPE.type, CHECKEQUALS);
 
                         if (branch) {
                             adapter.ifZCmp(MethodWriter.NE, jump);
@@ -507,7 +507,7 @@ public final class EComp extends AExpression {
                     if (right.isNull) {
                         adapter.ifNonNull(jump);
                     } else if (operation == Operation.NE) {
-                        adapter.invokeStatic(Definition.utilityType.type, CHECKEQUALS);
+                        adapter.invokeStatic(Definition.UTILITY_TYPE.type, CHECKEQUALS);
                         adapter.ifZCmp(MethodWriter.EQ, jump);
                     } else {
                         adapter.ifCmp(rtype, MethodWriter.NE, jump);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
index e084974fb33..c20391a9d41 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
@@ -47,7 +47,7 @@ public final class EConditional extends AExpression {
 
     @Override
     void analyze(final CompilerSettings settings, final Variables variables) {
-        condition.expected = Definition.booleanType;
+        condition.expected = Definition.BOOLEAN_TYPE;
         condition.analyze(settings, variables);
         condition = condition.cast(settings, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
index e590ee41852..6ee486c9470 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
@@ -40,23 +40,23 @@ final class EConstant extends AExpression {
     @Override
     void analyze(final CompilerSettings settings, final Variables variables) {
         if (constant instanceof String) {
-            actual = Definition.stringType;
+            actual = Definition.STRING_TYPE;
         } else if (constant instanceof Double) {
-            actual = Definition.doubleType;
+            actual = Definition.DOUBLE_TYPE;
         } else if (constant instanceof Float) {
-            actual = Definition.floatType;
+            actual = Definition.FLOAT_TYPE;
         } else if (constant instanceof Long) {
-            actual = Definition.longType;
+            actual = Definition.LONG_TYPE;
         } else if (constant instanceof Integer) {
-            actual = Definition.intType;
+            actual = Definition.INT_TYPE;
         } else if (constant instanceof Character) {
-            actual = Definition.charType;
+            actual = Definition.CHAR_TYPE;
         } else if (constant instanceof Short) {
-            actual = Definition.shortType;
+            actual = Definition.SHORT_TYPE;
         } else if (constant instanceof Byte) {
-            actual = Definition.byteType;
+            actual = Definition.BYTE_TYPE;
         } else if (constant instanceof Boolean) {
-            actual = Definition.booleanType;
+            actual = Definition.BOOLEAN_TYPE;
         } else {
             throw new IllegalStateException(error("Illegal tree structure."));
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
index 3f1d3a3fc26..7c942dff32c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
@@ -42,14 +42,14 @@ public final class EDecimal extends AExpression {
         if (value.endsWith("f") || value.endsWith("F")) {
             try {
                 constant = Float.parseFloat(value.substring(0, value.length() - 1));
-                actual = Definition.floatType;
+                actual = Definition.FLOAT_TYPE;
             } catch (final NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid float constant [" + value + "]."));
             }
         } else {
             try {
                 constant = Double.parseDouble(value);
-                actual = Definition.doubleType;
+                actual = Definition.DOUBLE_TYPE;
             } catch (final NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid double constant [" + value + "]."));
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
index 3fcabccff17..c7c93951223 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
@@ -45,7 +45,7 @@ public final class ENull extends AExpression {
 
             actual = expected;
         } else {
-            actual = Definition.objectType;
+            actual = Definition.OBJECT_TYPE;
         }
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
index e68de5b4128..75f85703bc8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
@@ -49,7 +49,7 @@ public final class ENumeric extends AExpression {
 
             try {
                 constant = Double.parseDouble(value.substring(0, value.length() - 1));
-                actual = Definition.doubleType;
+                actual = Definition.DOUBLE_TYPE;
             } catch (final NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid double constant [" + value + "]."));
             }
@@ -60,14 +60,14 @@ public final class ENumeric extends AExpression {
 
             try {
                 constant = Float.parseFloat(value.substring(0, value.length() - 1));
-                actual = Definition.floatType;
+                actual = Definition.FLOAT_TYPE;
             } catch (final NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid float constant [" + value + "]."));
             }
         } else if (value.endsWith("l") || value.endsWith("L")) {
             try {
                 constant = Long.parseLong(value.substring(0, value.length() - 1), radix);
-                actual = Definition.longType;
+                actual = Definition.LONG_TYPE;
             } catch (final NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid long constant [" + value + "]."));
             }
@@ -78,16 +78,16 @@ public final class ENumeric extends AExpression {
 
                 if (sort == Sort.BYTE && integer >= Byte.MIN_VALUE && integer <= Byte.MAX_VALUE) {
                     constant = (byte)integer;
-                    actual = Definition.byteType;
+                    actual = Definition.BYTE_TYPE;
                 } else if (sort == Sort.CHAR && integer >= Character.MIN_VALUE && integer <= Character.MAX_VALUE) {
                     constant = (char)integer;
-                    actual = Definition.charType;
+                    actual = Definition.CHAR_TYPE;
                 } else if (sort == Sort.SHORT && integer >= Short.MIN_VALUE && integer <= Short.MAX_VALUE) {
                     constant = (short)integer;
-                    actual = Definition.shortType;
+                    actual = Definition.SHORT_TYPE;
                 } else {
                     constant = integer;
-                    actual = Definition.intType;
+                    actual = Definition.INT_TYPE;
                 }
             } catch (final NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid int constant [" + value + "]."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index 8ab97dcbde6..115db1499f4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -63,7 +63,7 @@ public final class EUnary extends AExpression {
     }
 
     void analyzeNot(final CompilerSettings settings, final Variables variables) {
-        child.expected = Definition.booleanType;
+        child.expected = Definition.BOOLEAN_TYPE;
         child.analyze(settings, variables);
         child = child.cast(settings, variables);
 
@@ -71,7 +71,7 @@ public final class EUnary extends AExpression {
             constant = !(boolean)child.constant;
         }
 
-        actual = Definition.booleanType;
+        actual = Definition.BOOLEAN_TYPE;
     }
 
     void analyzeBWNot(final CompilerSettings settings, final Variables variables) {
@@ -191,7 +191,7 @@ public final class EUnary extends AExpression {
 
             if (operation == Operation.BWNOT) {
                 if (sort == Sort.DEF) {
-                    adapter.invokeStatic(Definition.defobjType.type, DEF_NOT_CALL);
+                    adapter.invokeStatic(Definition.DEF_UTIL_TYPE.type, DEF_NOT_CALL);
                 } else {
                     if (sort == Sort.INT) {
                         adapter.push(-1);
@@ -205,7 +205,7 @@ public final class EUnary extends AExpression {
                 }
             } else if (operation == Operation.SUB) {
                 if (sort == Sort.DEF) {
-                    adapter.invokeStatic(Definition.defobjType.type, DEF_NEG_CALL);
+                    adapter.invokeStatic(Definition.DEF_UTIL_TYPE.type, DEF_NEG_CALL);
                 } else {
                     adapter.math(MethodWriter.NEG, type);
                 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
index c4026bb529e..4678e9766f3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
@@ -46,7 +46,7 @@ public final class LArrayLength extends ALink {
                 throw new IllegalArgumentException(error("Cannot write to read-only array field [length]."));
             }
 
-            after = Definition.intType;
+            after = Definition.INT_TYPE;
         } else {
             throw new IllegalArgumentException(error("Illegal field access [" + value + "]."));
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
index 2a8da6e1892..1a1d4b5927e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
@@ -50,7 +50,7 @@ public final class LBrace extends ALink {
         final Sort sort = before.sort;
 
         if (sort == Sort.ARRAY) {
-            index.expected = Definition.intType;
+            index.expected = Definition.INT_TYPE;
             index.analyze(settings, variables);
             index = index.cast(settings, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index 842cc7c41bb..e07dd9ab6ff 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -47,7 +47,7 @@ final class LDefArray extends ALink implements IDefLink {
         index.expected = index.actual;
         index = index.cast(settings, variables);
 
-        after = Definition.defType;
+        after = Definition.DEF_TYPE;
 
         return this;
     }
@@ -59,13 +59,13 @@ final class LDefArray extends ALink implements IDefLink {
 
     @Override
     void load(final CompilerSettings settings, final MethodWriter adapter) {
-        final String desc = Type.getMethodDescriptor(after.type, Definition.defType.type, index.actual.type);
+        final String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type, index.actual.type);
         adapter.invokeDynamic("arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_LOAD);
     }
 
     @Override
     void store(final CompilerSettings settings, final MethodWriter adapter) {
-        final String desc = Type.getMethodDescriptor(Definition.voidType.type, Definition.defType.type,
+        final String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type,
             index.actual.type, after.type);
         adapter.invokeDynamic("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_STORE);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index e3b7c6c583d..f15f132cbec 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -55,7 +55,7 @@ final class LDefCall extends ALink implements IDefLink {
         }
 
         statement = true;
-        after = Definition.defType;
+        after = Definition.DEF_TYPE;
 
         return this;
     }
@@ -71,7 +71,7 @@ final class LDefCall extends ALink implements IDefLink {
 
         signature.append('(');
         // first parameter is the receiver, we never know its type: always Object
-        signature.append(Definition.defType.type.getDescriptor());
+        signature.append(Definition.DEF_TYPE.type.getDescriptor());
 
         // TODO: remove our explicit conversions and feed more type information for return value,
         // it can avoid some unnecessary boxing etc.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index f54bf316254..84850826e6f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -44,7 +44,7 @@ final class LDefField extends ALink implements IDefLink {
 
     @Override
     ALink analyze(final CompilerSettings settings, final Variables variables) {
-        after = Definition.defType;
+        after = Definition.DEF_TYPE;
 
         return this;
     }
@@ -56,13 +56,13 @@ final class LDefField extends ALink implements IDefLink {
 
     @Override
     void load(final CompilerSettings settings, final MethodWriter adapter) {
-        final String desc = Type.getMethodDescriptor(after.type, Definition.defType.type);
+        final String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type);
         adapter.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.LOAD);
     }
 
     @Override
     void store(final CompilerSettings settings, final MethodWriter adapter) {
-        final String desc = Type.getMethodDescriptor(Definition.voidType.type, Definition.defType.type, after.type);
+        final String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, after.type);
         adapter.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.STORE);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
index 1ee0f8fa3ed..9541459a615 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
@@ -61,7 +61,7 @@ final class LListShortcut extends ALink {
         }
 
         if ((load || store) && (!load || getter != null) && (!store || setter != null)) {
-            index.expected = Definition.intType;
+            index.expected = Definition.INT_TYPE;
             index.analyze(settings, variables);
             index = index.cast(settings, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
index 9d3e5d307fe..2faad220db6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
@@ -63,7 +63,7 @@ public final class LNewArray extends ALink {
         for (int argument = 0; argument < arguments.size(); ++argument) {
             final AExpression expression = arguments.get(argument);
 
-            expression.expected = Definition.intType;
+            expression.expected = Definition.INT_TYPE;
             expression.analyze(settings, variables);
             arguments.set(argument, expression.cast(settings, variables));
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
index 5fe7c22a457..115c4c57ab3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
@@ -45,7 +45,7 @@ public final class LString extends ALink {
             throw new IllegalArgumentException(error("Must read String constant [" + string + "]."));
         }
 
-        after = Definition.stringType;
+        after = Definition.STRING_TYPE;
 
         return this;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
index ade9f9ba221..249ff75c744 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -53,7 +53,7 @@ public final class SDo extends AStatement {
             throw new IllegalArgumentException(error("Extraneous do while loop."));
         }
 
-        condition.expected = Definition.booleanType;
+        condition.expected = Definition.BOOLEAN_TYPE;
         condition.analyze(settings, variables);
         condition = condition.cast(settings, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index 1f107548dff..aaf63a3c923 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -49,7 +49,7 @@ public final class SExpression extends AStatement {
 
         final boolean rtn = lastSource && expression.actual.sort != Sort.VOID;
 
-        expression.expected = rtn ? Definition.objectType : expression.actual;
+        expression.expected = rtn ? Definition.OBJECT_TYPE : expression.actual;
         expression = expression.cast(settings, variables);
 
         methodEscape = rtn;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index ae59382039d..cbce549f1cb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -70,7 +70,7 @@ public final class SFor extends AStatement {
 
         if (condition != null) {
 
-            condition.expected = Definition.booleanType;
+            condition.expected = Definition.BOOLEAN_TYPE;
             condition.analyze(settings, variables);
             condition = condition.cast(settings, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
index 40c20a1cb87..1fadc4f0ac4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
@@ -45,7 +45,7 @@ public final class SIfElse extends AStatement {
 
     @Override
     void analyze(final CompilerSettings settings, final Variables variables) {
-        condition.expected = Definition.booleanType;
+        condition.expected = Definition.BOOLEAN_TYPE;
         condition.analyze(settings, variables);
         condition = condition.cast(settings, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
index 4caf57f463b..f2c051ac1f6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -39,7 +39,7 @@ public final class SReturn extends AStatement {
 
     @Override
     void analyze(final CompilerSettings settings, final Variables variables) {
-        expression.expected = Definition.objectType;
+        expression.expected = Definition.OBJECT_TYPE;
         expression.analyze(settings, variables);
         expression = expression.cast(settings, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
index 8a2909d0a3b..f9a719a5f77 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
@@ -39,7 +39,7 @@ public final class SThrow extends AStatement {
 
     @Override
     void analyze(final CompilerSettings settings, final Variables variables) {
-        expression.expected = Definition.exceptionType;
+        expression.expected = Definition.EXCEPTION_TYPE;
         expression.analyze(settings, variables);
         expression = expression.cast(settings, variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
index a6d88fdcced..98616d11499 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -44,7 +44,7 @@ public final class SWhile extends AStatement {
     void analyze(final CompilerSettings settings, final Variables variables) {
         variables.incrementScope();
 
-        condition.expected = Definition.booleanType;
+        condition.expected = Definition.BOOLEAN_TYPE;
         condition.analyze(settings, variables);
         condition = condition.cast(settings, variables);
 

From 80fee8666ff5dd61ba29b175857cf42ce3b9eab9 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Fri, 29 Apr 2016 16:23:01 +0200
Subject: [PATCH 0382/1311] percolator: Removed percolator cache

Before 5.0 for it was required that the percolator queries were cached in jvm heap as Lucene queries for two reasons:
1) Performance. The percolator evaluated all percolator queries all the time. There was no pre-selecting queries that are likely to match like we have today.
2) Updates made to percolator queries were visible in realtime, Today these changes are visible in near realtime. So updating no longer requires the percolator to have the queries in jvm heap.

So having the percolator queries in jvm heap via the percolator cache is now less attractive. Especially when there are many percolator queries then these queries can consume many GBs of jvm heap.
Removing the percolator cache does make the percolate query slower compared to how the execution time in 5.0.0-alpha1 and alpha2, but it is still faster compared to 2.x and before.
---
 .../cluster/stats/ClusterStatsIndices.java    |   9 -
 .../stats/TransportClusterStatsAction.java    |   5 +-
 .../admin/indices/stats/CommonStats.java      |  39 +-
 .../admin/indices/stats/CommonStatsFlags.java |   1 -
 .../indices/stats/IndicesStatsRequest.java    |   9 -
 .../stats/IndicesStatsRequestBuilder.java     |   5 -
 .../stats/TransportIndicesStatsAction.java    |   5 +-
 .../common/settings/IndexScopedSettings.java  |   4 +-
 .../org/elasticsearch/index/IndexService.java |  12 +-
 .../elasticsearch/index/cache/IndexCache.java |  12 +-
 .../percolator/PercolatorFieldMapper.java     |  16 +-
 .../PercolatorHighlightSubFetchPhase.java     |  14 +-
 .../percolator/PercolatorQueryCache.java      | 294 -------------
 .../percolator/PercolatorQueryCacheStats.java |  89 ----
 .../index/query/PercolateQuery.java           |  44 +-
 .../index/query/PercolateQueryBuilder.java    | 126 +++++-
 .../index/query/QueryShardContext.java        |  11 +-
 .../elasticsearch/indices/IndicesService.java |   2 +-
 .../indices/NodeIndicesStats.java             |   6 -
 .../indices/stats/RestIndicesStatsAction.java |   1 -
 .../rest/action/cat/RestIndicesAction.java    |   6 -
 .../rest/action/cat/RestNodesAction.java      |   6 -
 .../rest/action/cat/RestShardsAction.java     |   4 -
 .../search/internal/DefaultSearchContext.java |   8 +-
 .../internal/FilteredSearchContext.java       |   6 -
 .../search/internal/SearchContext.java        |   3 -
 .../PercolatorFieldMapperTests.java           |   4 +-
 .../percolator/PercolatorQueryCacheTests.java | 390 ------------------
 .../index/query/AbstractQueryTestCase.java    |   6 +-
 .../query/PercolateQueryBuilderTests.java     |   2 +-
 .../index/query/PercolateQueryTests.java      |  12 +-
 .../index/query/QueryShardContextTests.java   |   2 +-
 .../index/shard/IndexShardTests.java          |   2 +-
 .../indices/stats/IndexStatsIT.java           |   7 +-
 .../percolator/PercolatorIT.java              |  32 --
 .../highlight/HighlightBuilderTests.java      |   2 +-
 .../rescore/QueryRescoreBuilderTests.java     |   2 +-
 .../search/sort/AbstractSortTestCase.java     |   2 +-
 .../migration/migrate_5_0/percolator.asciidoc |   2 +-
 .../query-dsl/percolate-query.asciidoc        |  13 +-
 .../messy/tests/TemplateQueryParserTests.java |   2 +-
 .../test/cat.shards/10_basic.yaml             |   1 -
 .../elasticsearch/test/TestSearchContext.java |   9 -
 .../search/MockSearchServiceTests.java        |   2 +-
 44 files changed, 198 insertions(+), 1031 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCache.java
 delete mode 100644 core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCacheStats.java
 delete mode 100644 core/src/test/java/org/elasticsearch/index/percolator/PercolatorQueryCacheTests.java

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java
index 8c0c427beea..9a7bb5c8f3d 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java
@@ -27,7 +27,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.cache.query.QueryCacheStats;
 import org.elasticsearch.index.engine.SegmentsStats;
 import org.elasticsearch.index.fielddata.FieldDataStats;
-import org.elasticsearch.index.percolator.PercolatorQueryCacheStats;
 import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.store.StoreStats;
 import org.elasticsearch.search.suggest.completion.CompletionStats;
@@ -45,7 +44,6 @@ public class ClusterStatsIndices implements ToXContent {
     private QueryCacheStats queryCache;
     private CompletionStats completion;
     private SegmentsStats segments;
-    private PercolatorQueryCacheStats percolatorCache;
 
     public ClusterStatsIndices(List<ClusterStatsNodeResponse> nodeResponses) {
         ObjectObjectHashMap<String, ShardStats> countsPerIndex = new ObjectObjectHashMap<>();
@@ -56,7 +54,6 @@ public class ClusterStatsIndices implements ToXContent {
         this.queryCache = new QueryCacheStats();
         this.completion = new CompletionStats();
         this.segments = new SegmentsStats();
-        this.percolatorCache = new PercolatorQueryCacheStats();
 
         for (ClusterStatsNodeResponse r : nodeResponses) {
             for (org.elasticsearch.action.admin.indices.stats.ShardStats shardStats : r.shardsStats()) {
@@ -79,7 +76,6 @@ public class ClusterStatsIndices implements ToXContent {
                 queryCache.add(shardCommonStats.queryCache);
                 completion.add(shardCommonStats.completion);
                 segments.add(shardCommonStats.segments);
-                percolatorCache.add(shardCommonStats.percolatorCache);
             }
         }
 
@@ -122,10 +118,6 @@ public class ClusterStatsIndices implements ToXContent {
         return segments;
     }
 
-    public PercolatorQueryCacheStats getPercolatorCache() {
-        return percolatorCache;
-    }
-
     static final class Fields {
         static final String COUNT = "count";
     }
@@ -140,7 +132,6 @@ public class ClusterStatsIndices implements ToXContent {
         queryCache.toXContent(builder, params);
         completion.toXContent(builder, params);
         segments.toXContent(builder, params);
-        percolatorCache.toXContent(builder, params);
         return builder;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
index bae7b20694d..3a0b1455209 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
@@ -55,8 +55,7 @@ public class TransportClusterStatsAction extends TransportNodesAction<ClusterSta
         TransportClusterStatsAction.ClusterStatsNodeRequest, ClusterStatsNodeResponse> {
 
     private static final CommonStatsFlags SHARD_STATS_FLAGS = new CommonStatsFlags(CommonStatsFlags.Flag.Docs, CommonStatsFlags.Flag.Store,
-            CommonStatsFlags.Flag.FieldData, CommonStatsFlags.Flag.QueryCache, CommonStatsFlags.Flag.Completion, CommonStatsFlags.Flag.Segments,
-            CommonStatsFlags.Flag.PercolatorCache);
+            CommonStatsFlags.Flag.FieldData, CommonStatsFlags.Flag.QueryCache, CommonStatsFlags.Flag.Completion, CommonStatsFlags.Flag.Segments);
 
     private final NodeService nodeService;
     private final IndicesService indicesService;
@@ -100,7 +99,7 @@ public class TransportClusterStatsAction extends TransportNodesAction<ClusterSta
             for (IndexShard indexShard : indexService) {
                 if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) {
                     // only report on fully started shards
-                    shardsStats.add(new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), indexService.cache().getPercolatorQueryCache(), indexShard, SHARD_STATS_FLAGS), indexShard.commitStats()));
+                    shardsStats.add(new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), indexShard, SHARD_STATS_FLAGS), indexShard.commitStats()));
                 }
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java
index b130e6b378f..14fc6c05e5c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java
@@ -32,10 +32,8 @@ import org.elasticsearch.index.engine.SegmentsStats;
 import org.elasticsearch.index.fielddata.FieldDataStats;
 import org.elasticsearch.index.flush.FlushStats;
 import org.elasticsearch.index.get.GetStats;
-import org.elasticsearch.index.percolator.PercolatorQueryCache;
 import org.elasticsearch.index.shard.IndexingStats;
 import org.elasticsearch.index.merge.MergeStats;
-import org.elasticsearch.index.percolator.PercolatorQueryCacheStats;
 import org.elasticsearch.index.recovery.RecoveryStats;
 import org.elasticsearch.index.refresh.RefreshStats;
 import org.elasticsearch.index.search.stats.SearchStats;
@@ -101,9 +99,6 @@ public class CommonStats implements Streamable, ToXContent {
                 case Segments:
                     segments = new SegmentsStats();
                     break;
-                case PercolatorCache:
-                    percolatorCache = new PercolatorQueryCacheStats();
-                    break;
                 case Translog:
                     translog = new TranslogStats();
                     break;
@@ -123,8 +118,7 @@ public class CommonStats implements Streamable, ToXContent {
     }
 
 
-    public CommonStats(IndicesQueryCache indicesQueryCache, PercolatorQueryCache percolatorQueryCache,
-                       IndexShard indexShard, CommonStatsFlags flags) {
+    public CommonStats(IndicesQueryCache indicesQueryCache, IndexShard indexShard, CommonStatsFlags flags) {
 
         CommonStatsFlags.Flag[] setFlags = flags.getFlags();
 
@@ -169,9 +163,6 @@ public class CommonStats implements Streamable, ToXContent {
                 case Segments:
                     segments = indexShard.segmentStats(flags.includeSegmentFileSizes());
                     break;
-                case PercolatorCache:
-                    percolatorCache = percolatorQueryCache.getStats(indexShard.shardId());
-                    break;
                 case Translog:
                     translog = indexShard.translogStats();
                     break;
@@ -223,9 +214,6 @@ public class CommonStats implements Streamable, ToXContent {
     @Nullable
     public FieldDataStats fieldData;
 
-    @Nullable
-    public PercolatorQueryCacheStats percolatorCache;
-
     @Nullable
     public CompletionStats completion;
 
@@ -331,14 +319,6 @@ public class CommonStats implements Streamable, ToXContent {
         } else {
             fieldData.add(stats.getFieldData());
         }
-        if (percolatorCache == null) {
-            if (stats.getPercolatorCache() != null) {
-                percolatorCache = new PercolatorQueryCacheStats();
-                percolatorCache.add(stats.getPercolatorCache());
-            }
-        } else {
-            percolatorCache.add(stats.getPercolatorCache());
-        }
         if (completion == null) {
             if (stats.getCompletion() != null) {
                 completion = new CompletionStats();
@@ -436,11 +416,6 @@ public class CommonStats implements Streamable, ToXContent {
         return this.fieldData;
     }
 
-    @Nullable
-    public PercolatorQueryCacheStats getPercolatorCache() {
-        return percolatorCache;
-    }
-
     @Nullable
     public CompletionStats getCompletion() {
         return completion;
@@ -528,9 +503,6 @@ public class CommonStats implements Streamable, ToXContent {
         if (in.readBoolean()) {
             fieldData = FieldDataStats.readFieldDataStats(in);
         }
-        if (in.readBoolean()) {
-            percolatorCache = PercolatorQueryCacheStats.readPercolateStats(in);
-        }
         if (in.readBoolean()) {
             completion = CompletionStats.readCompletionStats(in);
         }
@@ -610,12 +582,6 @@ public class CommonStats implements Streamable, ToXContent {
             out.writeBoolean(true);
             fieldData.writeTo(out);
         }
-        if (percolatorCache == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            percolatorCache.writeTo(out);
-        }
         if (completion == null) {
             out.writeBoolean(false);
         } else {
@@ -669,9 +635,6 @@ public class CommonStats implements Streamable, ToXContent {
         if (fieldData != null) {
             fieldData.toXContent(builder, params);
         }
-        if (percolatorCache != null) {
-            percolatorCache.toXContent(builder, params);
-        }
         if (completion != null) {
             completion.toXContent(builder, params);
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java
index e2f250dd577..3be3de00f61 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java
@@ -240,7 +240,6 @@ public class CommonStatsFlags implements Streamable, Cloneable {
         FieldData("fielddata"),
         Docs("docs"),
         Warmer("warmer"),
-        PercolatorCache("percolator_cache"),
         Completion("completion"),
         Segments("segments"),
         Translog("translog"),
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java
index 4a2d137593e..2308c9bae6d 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java
@@ -184,15 +184,6 @@ public class IndicesStatsRequest extends BroadcastRequest<IndicesStatsRequest> {
         return flags.isSet(Flag.FieldData);
     }
 
-    public IndicesStatsRequest percolate(boolean percolate) {
-        flags.set(Flag.PercolatorCache, percolate);
-        return this;
-    }
-
-    public boolean percolate() {
-        return flags.isSet(Flag.PercolatorCache);
-    }
-
     public IndicesStatsRequest segments(boolean segments) {
         flags.set(Flag.Segments, segments);
         return this;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java
index cad919cbd18..8e7afe3e7e3 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java
@@ -127,11 +127,6 @@ public class IndicesStatsRequestBuilder extends BroadcastOperationRequestBuilder
         return this;
     }
 
-    public IndicesStatsRequestBuilder setPercolate(boolean percolate) {
-        request.percolate(percolate);
-        return this;
-    }
-
     public IndicesStatsRequestBuilder setSegments(boolean segments) {
         request.segments(segments);
         return this;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java
index 8c12dfa9fda..7e8ccd30a8a 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java
@@ -139,9 +139,6 @@ public class TransportIndicesStatsAction extends TransportBroadcastByNodeAction<
             flags.set(CommonStatsFlags.Flag.FieldData);
             flags.fieldDataFields(request.fieldDataFields());
         }
-        if (request.percolate()) {
-            flags.set(CommonStatsFlags.Flag.PercolatorCache);
-        }
         if (request.segments()) {
             flags.set(CommonStatsFlags.Flag.Segments);
             flags.includeSegmentFileSizes(request.includeSegmentFileSizes());
@@ -163,6 +160,6 @@ public class TransportIndicesStatsAction extends TransportBroadcastByNodeAction<
             flags.set(CommonStatsFlags.Flag.Recovery);
         }
 
-        return new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), indexService.cache().getPercolatorQueryCache(), indexShard, flags), indexShard.commitStats());
+        return new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), indexShard, flags), indexShard.commitStats());
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
index 1b795239457..9ebf6853166 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
@@ -35,7 +35,7 @@ import org.elasticsearch.index.engine.EngineConfig;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.percolator.PercolatorQueryCache;
+import org.elasticsearch.index.percolator.PercolatorFieldMapper;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.index.store.FsDirectoryService;
 import org.elasticsearch.index.store.IndexStore;
@@ -126,7 +126,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         FieldMapper.IGNORE_MALFORMED_SETTING,
         FieldMapper.COERCE_SETTING,
         Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING,
-        PercolatorQueryCache.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING,
+        PercolatorFieldMapper.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING,
         MapperService.INDEX_MAPPER_DYNAMIC_SETTING,
         MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING,
         MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING,
diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java
index 60e3250e49d..f5e5ce91d80 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexService.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexService.java
@@ -50,7 +50,6 @@ import org.elasticsearch.index.engine.EngineFactory;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.percolator.PercolatorQueryCache;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.IndexEventListener;
@@ -151,11 +150,9 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
         this.indexStore = indexStore;
         indexFieldData.setListener(new FieldDataCacheListener(this));
         this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this));
-        PercolatorQueryCache percolatorQueryCache = new PercolatorQueryCache(indexSettings, IndexService.this::newQueryShardContext);
         this.warmer = new IndexWarmer(indexSettings.getSettings(), threadPool,
-            bitsetFilterCache.createListener(threadPool),
-            percolatorQueryCache.createListener(threadPool));
-        this.indexCache = new IndexCache(indexSettings, queryCache, bitsetFilterCache, percolatorQueryCache);
+            bitsetFilterCache.createListener(threadPool));
+        this.indexCache = new IndexCache(indexSettings, queryCache, bitsetFilterCache);
         this.engineFactory = engineFactory;
         // initialize this last -- otherwise if the wrapper requires any other member to be non-null we fail with an NPE
         this.searcherWrapper = wrapperFactory.newWrapper(this);
@@ -239,8 +236,7 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
                     }
                 }
             } finally {
-                IOUtils.close(bitsetFilterCache, indexCache, indexFieldData, analysisService, refreshTask, fsyncTask,
-                    cache().getPercolatorQueryCache());
+                IOUtils.close(bitsetFilterCache, indexCache, indexFieldData, analysisService, refreshTask, fsyncTask);
             }
         }
     }
@@ -443,7 +439,7 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
         return new QueryShardContext(
                 indexSettings, indexCache.bitsetFilterCache(), indexFieldData, mapperService(),
                 similarityService(), nodeServicesProvider.getScriptService(), nodeServicesProvider.getIndicesQueriesRegistry(),
-                nodeServicesProvider.getClient(), indexCache.getPercolatorQueryCache(), indexReader,
+                nodeServicesProvider.getClient(), indexReader,
                 nodeServicesProvider.getClusterService().state()
         );
     }
diff --git a/core/src/main/java/org/elasticsearch/index/cache/IndexCache.java b/core/src/main/java/org/elasticsearch/index/cache/IndexCache.java
index b41f5bc0125..61733f24695 100644
--- a/core/src/main/java/org/elasticsearch/index/cache/IndexCache.java
+++ b/core/src/main/java/org/elasticsearch/index/cache/IndexCache.java
@@ -24,7 +24,6 @@ import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.cache.query.QueryCache;
-import org.elasticsearch.index.percolator.PercolatorQueryCache;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -36,14 +35,11 @@ public class IndexCache extends AbstractIndexComponent implements Closeable {
 
     private final QueryCache queryCache;
     private final BitsetFilterCache bitsetFilterCache;
-    private final PercolatorQueryCache percolatorQueryCache;
 
-    public IndexCache(IndexSettings indexSettings, QueryCache queryCache, BitsetFilterCache bitsetFilterCache,
-                      PercolatorQueryCache percolatorQueryCache) {
+    public IndexCache(IndexSettings indexSettings, QueryCache queryCache, BitsetFilterCache bitsetFilterCache) {
         super(indexSettings);
         this.queryCache = queryCache;
         this.bitsetFilterCache = bitsetFilterCache;
-        this.percolatorQueryCache = percolatorQueryCache;
     }
 
     public QueryCache query() {
@@ -57,13 +53,9 @@ public class IndexCache extends AbstractIndexComponent implements Closeable {
         return bitsetFilterCache;
     }
 
-    public PercolatorQueryCache getPercolatorQueryCache() {
-        return percolatorQueryCache;
-    }
-
     @Override
     public void close() throws IOException {
-        IOUtils.close(queryCache, bitsetFilterCache, percolatorQueryCache);
+        IOUtils.close(queryCache, bitsetFilterCache);
     }
 
     public void clear(String reason) {
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
index 59b980d59a4..decb91e960a 100644
--- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
@@ -23,11 +23,13 @@ import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -36,6 +38,7 @@ import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.core.BinaryFieldMapper;
 import org.elasticsearch.index.mapper.core.KeywordFieldMapper;
+import org.elasticsearch.index.query.PercolateQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
@@ -50,14 +53,17 @@ import java.util.Map;
 
 public class PercolatorFieldMapper extends FieldMapper {
 
+    public final static XContentType QUERY_BUILDER_CONTENT_TYPE = XContentType.SMILE;
+    public final static Setting<Boolean> INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING =
+            Setting.boolSetting("index.percolator.map_unmapped_fields_as_string", false, Setting.Property.IndexScope);
     @Deprecated
     public static final String LEGACY_TYPE_NAME = ".percolator";
     public static final String CONTENT_TYPE = "percolator";
     private static final PercolatorFieldType FIELD_TYPE = new PercolatorFieldType();
 
-    private static final String EXTRACTED_TERMS_FIELD_NAME = "extracted_terms";
-    private static final String UNKNOWN_QUERY_FIELD_NAME = "unknown_query";
-    static final String QUERY_BUILDER_FIELD_NAME = "query_builder_field";
+    public static final String EXTRACTED_TERMS_FIELD_NAME = "extracted_terms";
+    public static final String UNKNOWN_QUERY_FIELD_NAME = "unknown_query";
+    public static final String QUERY_BUILDER_FIELD_NAME = "query_builder_field";
 
     public static class Builder extends FieldMapper.Builder<Builder, PercolatorFieldMapper> {
 
@@ -172,7 +178,7 @@ public class PercolatorFieldMapper extends FieldMapper {
         this.queryTermsField = queryTermsField;
         this.unknownQueryField = unknownQueryField;
         this.queryBuilderField = queryBuilderField;
-        this.mapUnmappedFieldAsString = PercolatorQueryCache.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING.get(indexSettings);
+        this.mapUnmappedFieldAsString = INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING.get(indexSettings);
     }
 
     @Override
@@ -196,7 +202,7 @@ public class PercolatorFieldMapper extends FieldMapper {
         // Fetching of terms, shapes and indexed scripts happen during this rewrite:
         queryBuilder = queryBuilder.rewrite(queryShardContext);
 
-        try (XContentBuilder builder = XContentFactory.contentBuilder(PercolatorQueryCache.QUERY_BUILDER_CONTENT_TYPE)) {
+        try (XContentBuilder builder = XContentFactory.contentBuilder(QUERY_BUILDER_CONTENT_TYPE)) {
             queryBuilder.toXContent(builder, new MapParams(Collections.emptyMap()));
             builder.flush();
             byte[] queryBuilderAsBytes = builder.bytes().toBytes();
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorHighlightSubFetchPhase.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorHighlightSubFetchPhase.java
index 9fb6934cca4..2c231593860 100644
--- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorHighlightSubFetchPhase.java
+++ b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorHighlightSubFetchPhase.java
@@ -40,6 +40,7 @@ import org.elasticsearch.search.internal.InternalSearchHit;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.SubSearchContext;
 
+import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -69,8 +70,8 @@ public class PercolatorHighlightSubFetchPhase implements FetchSubPhase {
         }
 
         List<LeafReaderContext> ctxs = context.searcher().getIndexReader().leaves();
-        PercolatorQueryCache queriesRegistry = context.percolatorQueryCache();
         IndexSearcher percolatorIndexSearcher = percolateQuery.getPercolatorIndexSearcher();
+        PercolateQuery.QueryStore queryStore = percolateQuery.getQueryStore();
 
         LeafReaderContext percolatorLeafReaderContext = percolatorIndexSearcher.getIndexReader().leaves().get(0);
         FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext();
@@ -78,9 +79,14 @@ public class PercolatorHighlightSubFetchPhase implements FetchSubPhase {
                 createSubSearchContext(context, percolatorLeafReaderContext, percolateQuery.getDocumentSource());
 
         for (InternalSearchHit hit : hits) {
-            LeafReaderContext ctx = ctxs.get(ReaderUtil.subIndex(hit.docId(), ctxs));
-            int segmentDocId = hit.docId() - ctx.docBase;
-            Query query = queriesRegistry.getQueries(ctx).getQuery(segmentDocId);
+            final Query query;
+            try {
+                LeafReaderContext ctx = ctxs.get(ReaderUtil.subIndex(hit.docId(), ctxs));
+                int segmentDocId = hit.docId() - ctx.docBase;
+                query = queryStore.getQueries(ctx).getQuery(segmentDocId);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
             if (query != null) {
                 subSearchContext.parsedQuery(new ParsedQuery(query));
                 hitContext.reset(
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCache.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCache.java
deleted file mode 100644
index e7ca6c3d427..00000000000
--- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCache.java
+++ /dev/null
@@ -1,294 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.percolator;
-
-import com.carrotsearch.hppc.IntObjectHashMap;
-import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.StoredFieldVisitor;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Weight;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.cache.Cache;
-import org.elasticsearch.common.cache.CacheBuilder;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.XContent;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.AbstractIndexComponent;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.IndexWarmer;
-import org.elasticsearch.index.IndexWarmer.TerminationHandle;
-import org.elasticsearch.index.engine.Engine.Searcher;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
-import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
-import org.elasticsearch.index.query.PercolateQuery;
-import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.index.shard.ShardUtils;
-import org.elasticsearch.threadpool.ThreadPool;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Executor;
-import java.util.function.Supplier;
-
-import static org.elasticsearch.index.percolator.PercolatorFieldMapper.LEGACY_TYPE_NAME;
-import static org.elasticsearch.index.percolator.PercolatorFieldMapper.PercolatorFieldType;
-import static org.elasticsearch.index.percolator.PercolatorFieldMapper.parseQuery;
-
-public final class PercolatorQueryCache extends AbstractIndexComponent
-        implements Closeable, LeafReader.CoreClosedListener, PercolateQuery.QueryRegistry {
-
-    public final static Setting<Boolean> INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING =
-            Setting.boolSetting("index.percolator.map_unmapped_fields_as_string", false, Setting.Property.IndexScope);
-
-    public final static XContentType QUERY_BUILDER_CONTENT_TYPE = XContentType.SMILE;
-
-    private final Supplier<QueryShardContext> queryShardContextSupplier;
-    private final Cache<Object, QueriesLeaf> cache;
-    private final boolean mapUnmappedFieldsAsString;
-
-    public PercolatorQueryCache(IndexSettings indexSettings, Supplier<QueryShardContext> queryShardContextSupplier) {
-        super(indexSettings);
-        this.queryShardContextSupplier = queryShardContextSupplier;
-        cache = CacheBuilder.<Object, QueriesLeaf>builder().build();
-        this.mapUnmappedFieldsAsString = indexSettings.getValue(INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING);
-    }
-
-    @Override
-    public Leaf getQueries(LeafReaderContext ctx) {
-        QueriesLeaf percolatorQueries = cache.get(ctx.reader().getCoreCacheKey());
-        if (percolatorQueries == null) {
-            throw new IllegalStateException("queries not loaded, queries should be have been preloaded during index warming...");
-        }
-        return percolatorQueries;
-    }
-
-    public IndexWarmer.Listener createListener(ThreadPool threadPool) {
-        return new IndexWarmer.Listener() {
-
-            final Executor executor = threadPool.executor(ThreadPool.Names.WARMER);
-
-            @Override
-            public TerminationHandle warmReader(IndexShard indexShard, Searcher searcher) {
-                final CountDownLatch latch = new CountDownLatch(searcher.reader().leaves().size());
-                for (final LeafReaderContext ctx : searcher.reader().leaves()) {
-                    if (cache.get(ctx.reader().getCoreCacheKey()) != null) {
-                        latch.countDown();
-                        continue;
-                    }
-                    executor.execute(() -> {
-                        try {
-                            final long start = System.nanoTime();
-                            QueriesLeaf queries = loadQueries(ctx, indexShard);
-                            cache.put(ctx.reader().getCoreCacheKey(), queries);
-                            if (indexShard.warmerService().logger().isTraceEnabled()) {
-                                indexShard.warmerService().logger().trace(
-                                        "loading percolator queries took [{}]",
-                                        TimeValue.timeValueNanos(System.nanoTime() - start)
-                                );
-                            }
-                        } catch (Throwable t) {
-                            indexShard.warmerService().logger().warn("failed to load percolator queries", t);
-                        } finally {
-                            latch.countDown();
-                        }
-                    });
-                }
-                return () -> latch.await();
-            }
-        };
-    }
-
-    QueriesLeaf loadQueries(LeafReaderContext context, IndexShard indexShard) throws IOException {
-        Version indexVersionCreated = indexShard.indexSettings().getIndexVersionCreated();
-        MapperService mapperService = indexShard.mapperService();
-        LeafReader leafReader = context.reader();
-        ShardId shardId = ShardUtils.extractShardId(leafReader);
-        if (shardId == null) {
-            throw new IllegalStateException("can't resolve shard id");
-        }
-        if (indexSettings.getIndex().equals(shardId.getIndex()) == false) {
-            // percolator cache insanity
-            String message = "Trying to load queries for index " + shardId.getIndex() + " with cache of index " +
-                    indexSettings.getIndex();
-            throw new IllegalStateException(message);
-        }
-
-        IntObjectHashMap<Query> queries = new IntObjectHashMap<>();
-        boolean legacyLoading = indexVersionCreated.before(Version.V_5_0_0_alpha1);
-        if (legacyLoading) {
-            PostingsEnum postings = leafReader.postings(new Term(TypeFieldMapper.NAME, LEGACY_TYPE_NAME), PostingsEnum.NONE);
-            if (postings != null) {
-                LegacyQueryFieldVisitor visitor = new LegacyQueryFieldVisitor();
-                for (int docId = postings.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = postings.nextDoc()) {
-                    leafReader.document(docId, visitor);
-                    queries.put(docId, parseLegacyPercolatorDocument(docId, visitor.source));
-                    visitor.source = null; // reset
-                }
-            }
-        } else {
-            // Each type can have one percolator field mapper,
-            // So for each type we check if there is a percolator field mapper
-            // and parse all the queries for the documents of that type.
-            IndexSearcher indexSearcher = new IndexSearcher(leafReader);
-            for (DocumentMapper documentMapper : mapperService.docMappers(false)) {
-                Weight queryWeight = indexSearcher.createNormalizedWeight(documentMapper.typeFilter(), false);
-                for (FieldMapper fieldMapper : documentMapper.mappers()) {
-                    if (fieldMapper instanceof PercolatorFieldMapper) {
-                        PercolatorFieldType fieldType = (PercolatorFieldType) fieldMapper.fieldType();
-                        BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues(fieldType.getQueryBuilderFieldName());
-                        if (binaryDocValues != null) {
-                            // use the same leaf reader context the indexSearcher is using too:
-                            Scorer scorer = queryWeight.scorer(leafReader.getContext());
-                            if (scorer != null) {
-                                DocIdSetIterator iterator = scorer.iterator();
-                                for (int docId = iterator.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = iterator.nextDoc()) {
-                                    BytesRef qbSource = binaryDocValues.get(docId);
-                                    if (qbSource.length > 0) {
-                                        queries.put(docId, parseQueryBuilder(docId, qbSource));
-                                    }
-                                }
-                            }
-                        }
-                        break;
-                    }
-                }
-            }
-        }
-        leafReader.addCoreClosedListener(this);
-        return new QueriesLeaf(shardId, queries);
-    }
-
-    private Query parseQueryBuilder(int docId, BytesRef qbSource) {
-        XContent xContent = QUERY_BUILDER_CONTENT_TYPE.xContent();
-        try (XContentParser sourceParser = xContent.createParser(qbSource.bytes, qbSource.offset, qbSource.length)) {
-            QueryShardContext context = queryShardContextSupplier.get();
-            return parseQuery(context, mapUnmappedFieldsAsString, sourceParser);
-        } catch (IOException e) {
-            throw new PercolatorException(index(), "failed to parse query builder for document  [" + docId   + "]", e);
-        }
-    }
-
-    private Query parseLegacyPercolatorDocument(int docId, BytesReference source) {
-        try (XContentParser sourceParser = XContentHelper.createParser(source)) {
-            String currentFieldName = null;
-            XContentParser.Token token = sourceParser.nextToken(); // move the START_OBJECT
-            if (token != XContentParser.Token.START_OBJECT) {
-                throw new ElasticsearchException("failed to parse query [" + docId + "], not starting with OBJECT");
-            }
-            while ((token = sourceParser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    currentFieldName = sourceParser.currentName();
-                } else if (token == XContentParser.Token.START_OBJECT) {
-                    if ("query".equals(currentFieldName)) {
-                        QueryShardContext context = queryShardContextSupplier.get();
-                        return parseQuery(context, mapUnmappedFieldsAsString, sourceParser);
-                    } else {
-                        sourceParser.skipChildren();
-                    }
-                } else if (token == XContentParser.Token.START_ARRAY) {
-                    sourceParser.skipChildren();
-                }
-            }
-        } catch (Exception e) {
-            throw new PercolatorException(index(), "failed to parse query [" + docId + "]", e);
-        }
-        return null;
-    }
-
-    public PercolatorQueryCacheStats getStats(ShardId shardId) {
-        int numberOfQueries = 0;
-        for (QueriesLeaf queries : cache.values()) {
-            if (shardId.equals(queries.shardId)) {
-                numberOfQueries += queries.queries.size();
-            }
-        }
-        return new PercolatorQueryCacheStats(numberOfQueries);
-    }
-
-    @Override
-    public void onClose(Object cacheKey) throws IOException {
-        cache.invalidate(cacheKey);
-    }
-
-    @Override
-    public void close() throws IOException {
-        cache.invalidateAll();
-    }
-
-    final static class LegacyQueryFieldVisitor extends StoredFieldVisitor {
-
-        private BytesArray source;
-
-        @Override
-        public void binaryField(FieldInfo fieldInfo, byte[] bytes) throws IOException {
-            source = new BytesArray(bytes);
-        }
-
-        @Override
-        public Status needsField(FieldInfo fieldInfo) throws IOException {
-            if (source != null)  {
-                return Status.STOP;
-            }
-            if (SourceFieldMapper.NAME.equals(fieldInfo.name)) {
-                return Status.YES;
-            } else {
-                return Status.NO;
-            }
-        }
-
-    }
-
-    final static class QueriesLeaf implements Leaf {
-
-        final ShardId shardId;
-        final IntObjectHashMap<Query> queries;
-
-        QueriesLeaf(ShardId shardId, IntObjectHashMap<Query> queries) {
-            this.shardId = shardId;
-            this.queries = queries;
-        }
-
-        @Override
-        public Query getQuery(int docId) {
-            return queries.get(docId);
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCacheStats.java b/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCacheStats.java
deleted file mode 100644
index cbc21286e20..00000000000
--- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorQueryCacheStats.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.index.percolator;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-
-import java.io.IOException;
-
-/**
- * Exposes percolator query cache statistics.
- */
-public class PercolatorQueryCacheStats implements Streamable, ToXContent {
-
-    private long numQueries;
-
-    /**
-     * Noop constructor for serialization purposes.
-     */
-    public PercolatorQueryCacheStats() {
-    }
-
-    PercolatorQueryCacheStats(long numQueries) {
-        this.numQueries = numQueries;
-    }
-
-    /**
-     * @return The total number of loaded percolate queries.
-     */
-    public long getNumQueries() {
-        return numQueries;
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(Fields.PERCOLATOR);
-        builder.field(Fields.QUERIES, getNumQueries());
-        builder.endObject();
-        return builder;
-    }
-
-    public void add(PercolatorQueryCacheStats percolate) {
-        if (percolate == null) {
-            return;
-        }
-
-        numQueries += percolate.getNumQueries();
-    }
-
-    static final class Fields {
-        static final String PERCOLATOR = "percolator";
-        static final String QUERIES = "num_queries";
-    }
-
-    public static PercolatorQueryCacheStats readPercolateStats(StreamInput in) throws IOException {
-        PercolatorQueryCacheStats stats = new PercolatorQueryCacheStats();
-        stats.readFrom(in);
-        return stats;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        numQueries = in.readVLong();
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeVLong(numQueries);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/query/PercolateQuery.java b/core/src/main/java/org/elasticsearch/index/query/PercolateQuery.java
index e99687422db..41448e14ef7 100644
--- a/core/src/main/java/org/elasticsearch/index/query/PercolateQuery.java
+++ b/core/src/main/java/org/elasticsearch/index/query/PercolateQuery.java
@@ -50,7 +50,7 @@ public final class PercolateQuery extends Query implements Accountable {
     public static class Builder {
 
         private final String docType;
-        private final QueryRegistry queryRegistry;
+        private final QueryStore queryStore;
         private final BytesReference documentSource;
         private final IndexSearcher percolatorIndexSearcher;
 
@@ -59,15 +59,15 @@ public final class PercolateQuery extends Query implements Accountable {
 
         /**
          * @param docType                   The type of the document being percolated
-         * @param queryRegistry             The registry holding all the percolator queries as Lucene queries.
+         * @param queryStore                The lookup holding all the percolator queries as Lucene queries.
          * @param documentSource            The source of the document being percolated
          * @param percolatorIndexSearcher   The index searcher on top of the in-memory index that holds the document being percolated
          */
-        public Builder(String docType, QueryRegistry queryRegistry, BytesReference documentSource, IndexSearcher percolatorIndexSearcher) {
+        public Builder(String docType, QueryStore queryStore, BytesReference documentSource, IndexSearcher percolatorIndexSearcher) {
             this.docType = Objects.requireNonNull(docType);
             this.documentSource = Objects.requireNonNull(documentSource);
             this.percolatorIndexSearcher = Objects.requireNonNull(percolatorIndexSearcher);
-            this.queryRegistry = Objects.requireNonNull(queryRegistry);
+            this.queryStore = Objects.requireNonNull(queryStore);
         }
 
         /**
@@ -94,7 +94,6 @@ public final class PercolateQuery extends Query implements Accountable {
             if (percolateTypeQuery != null && queriesMetaDataQuery != null) {
                 throw new IllegalStateException("Either filter by deprecated percolator type or by query metadata");
             }
-
             // The query that selects which percolator queries will be evaluated by MemoryIndex:
             BooleanQuery.Builder builder = new BooleanQuery.Builder();
             if (percolateTypeQuery != null) {
@@ -103,24 +102,23 @@ public final class PercolateQuery extends Query implements Accountable {
             if (queriesMetaDataQuery != null) {
                 builder.add(queriesMetaDataQuery, FILTER);
             }
-
-            return new PercolateQuery(docType, queryRegistry, documentSource, builder.build(), percolatorIndexSearcher);
+            return new PercolateQuery(docType, queryStore, documentSource, builder.build(), percolatorIndexSearcher);
         }
 
     }
 
     private final String documentType;
-    private final QueryRegistry queryRegistry;
+    private final QueryStore queryStore;
     private final BytesReference documentSource;
     private final Query percolatorQueriesQuery;
     private final IndexSearcher percolatorIndexSearcher;
 
-    private PercolateQuery(String documentType, QueryRegistry queryRegistry, BytesReference documentSource,
+    private PercolateQuery(String documentType, QueryStore queryStore, BytesReference documentSource,
                            Query percolatorQueriesQuery, IndexSearcher percolatorIndexSearcher) {
         this.documentType = documentType;
         this.documentSource = documentSource;
         this.percolatorQueriesQuery = percolatorQueriesQuery;
-        this.queryRegistry = queryRegistry;
+        this.queryStore = queryStore;
         this.percolatorIndexSearcher = percolatorIndexSearcher;
     }
 
@@ -128,7 +126,7 @@ public final class PercolateQuery extends Query implements Accountable {
     public Query rewrite(IndexReader reader) throws IOException {
         Query rewritten = percolatorQueriesQuery.rewrite(reader);
         if (rewritten != percolatorQueriesQuery) {
-            return new PercolateQuery(documentType, queryRegistry, documentSource, rewritten, percolatorIndexSearcher);
+            return new PercolateQuery(documentType, queryStore, documentSource, rewritten, percolatorIndexSearcher);
         } else {
             return this;
         }
@@ -151,7 +149,7 @@ public final class PercolateQuery extends Query implements Accountable {
                     if (result == docId) {
                         if (twoPhaseIterator.matches()) {
                             if (needsScores) {
-                                QueryRegistry.Leaf percolatorQueries = queryRegistry.getQueries(leafReaderContext);
+                                QueryStore.Leaf percolatorQueries = queryStore.getQueries(leafReaderContext);
                                 Query query = percolatorQueries.getQuery(docId);
                                 Explanation detail = percolatorIndexSearcher.explain(query, 0);
                                 return Explanation.match(scorer.score(), "PercolateQuery", detail);
@@ -181,9 +179,9 @@ public final class PercolateQuery extends Query implements Accountable {
                     return null;
                 }
 
-                final QueryRegistry.Leaf percolatorQueries = queryRegistry.getQueries(leafReaderContext);
+                final QueryStore.Leaf queries = queryStore.getQueries(leafReaderContext);
                 if (needsScores) {
-                    return new BaseScorer(this, approximation, percolatorQueries, percolatorIndexSearcher) {
+                    return new BaseScorer(this, approximation, queries, percolatorIndexSearcher) {
 
                         float score;
 
@@ -209,7 +207,7 @@ public final class PercolateQuery extends Query implements Accountable {
                         }
                     };
                 } else {
-                    return new BaseScorer(this, approximation, percolatorQueries, percolatorIndexSearcher) {
+                    return new BaseScorer(this, approximation, queries, percolatorIndexSearcher) {
 
                         @Override
                         public float score() throws IOException {
@@ -238,6 +236,10 @@ public final class PercolateQuery extends Query implements Accountable {
         return documentSource;
     }
 
+    public QueryStore getQueryStore() {
+        return queryStore;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -276,13 +278,15 @@ public final class PercolateQuery extends Query implements Accountable {
         return sizeInBytes;
     }
 
-    public interface QueryRegistry {
+    @FunctionalInterface
+    public interface QueryStore {
 
-        Leaf getQueries(LeafReaderContext ctx);
+        Leaf getQueries(LeafReaderContext ctx) throws IOException;
 
+        @FunctionalInterface
         interface Leaf {
 
-            Query getQuery(int docId);
+            Query getQuery(int docId) throws IOException;
 
         }
 
@@ -291,10 +295,10 @@ public final class PercolateQuery extends Query implements Accountable {
     static abstract class BaseScorer extends Scorer {
 
         final Scorer approximation;
-        final QueryRegistry.Leaf percolatorQueries;
+        final QueryStore.Leaf percolatorQueries;
         final IndexSearcher percolatorIndexSearcher;
 
-        BaseScorer(Weight weight, Scorer approximation, QueryRegistry.Leaf percolatorQueries, IndexSearcher percolatorIndexSearcher) {
+        BaseScorer(Weight weight, Scorer approximation, QueryStore.Leaf percolatorQueries, IndexSearcher percolatorIndexSearcher) {
             super(weight);
             this.approximation = approximation;
             this.percolatorQueries = percolatorQueries;
diff --git a/core/src/main/java/org/elasticsearch/index/query/PercolateQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/PercolateQueryBuilder.java
index 8341459b155..59be8d7a2d0 100644
--- a/core/src/main/java/org/elasticsearch/index/query/PercolateQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/PercolateQueryBuilder.java
@@ -21,10 +21,13 @@ package org.elasticsearch.index.query;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.MultiReader;
 import org.apache.lucene.index.SlowCompositeReaderWrapper;
+import org.apache.lucene.index.StoredFieldVisitor;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.memory.MemoryIndex;
 import org.apache.lucene.search.BooleanClause;
@@ -33,23 +36,27 @@ import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.search.Queries;
-import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.FieldNameAnalyzer;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperForType;
@@ -57,15 +64,16 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
 import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
 import org.elasticsearch.index.percolator.PercolatorFieldMapper;
-import org.elasticsearch.index.percolator.PercolatorQueryCache;
 
 import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 
 import static org.elasticsearch.index.mapper.SourceToParse.source;
+import static org.elasticsearch.index.percolator.PercolatorFieldMapper.parseQuery;
 
 public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBuilder> {
 
@@ -388,16 +396,14 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
             docSearcher.setQueryCache(null);
         }
 
-        PercolatorQueryCache registry = context.getPercolatorQueryCache();
-        if (registry == null) {
-            throw new QueryShardException(context, "no percolator query registry");
-        }
+        IndexSettings indexSettings = context.getIndexSettings();
+        boolean mapUnmappedFieldsAsString = indexSettings.getValue(PercolatorFieldMapper.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING);
+        return buildQuery(indexSettings.getIndexVersionCreated(), context, docSearcher, mapUnmappedFieldsAsString);
+    }
 
-        PercolateQuery.Builder builder = new PercolateQuery.Builder(
-                documentType, registry, document, docSearcher
-        );
-        Settings indexSettings = registry.getIndexSettings().getSettings();
-        if (indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null).onOrAfter(Version.V_5_0_0_alpha1)) {
+    Query buildQuery(Version indexVersionCreated, QueryShardContext context, IndexSearcher docSearcher,
+                     boolean mapUnmappedFieldsAsString) throws IOException {
+        if (indexVersionCreated.onOrAfter(Version.V_5_0_0_alpha1)) {
             MappedFieldType fieldType = context.fieldMapper(field);
             if (fieldType == null) {
                 throw new QueryShardException(context, "field [" + field + "] does not exist");
@@ -408,12 +414,19 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
                         "] to be of type [percolator], but is of type [" + fieldType.typeName() + "]");
             }
             PercolatorFieldMapper.PercolatorFieldType pft = (PercolatorFieldMapper.PercolatorFieldType) fieldType;
+            PercolateQuery.Builder builder = new PercolateQuery.Builder(
+                    documentType, createStore(pft, context, mapUnmappedFieldsAsString), document, docSearcher
+            );
             builder.extractQueryTermsQuery(pft.getExtractedTermsField(), pft.getUnknownQueryFieldName());
+            return builder.build();
         } else {
             Query percolateTypeQuery = new TermQuery(new Term(TypeFieldMapper.NAME, PercolatorFieldMapper.LEGACY_TYPE_NAME));
+            PercolateQuery.Builder builder = new PercolateQuery.Builder(
+                    documentType, createLegacyStore(context, mapUnmappedFieldsAsString), document, docSearcher
+            );
             builder.setPercolateTypeQuery(percolateTypeQuery);
+            return builder.build();
         }
-        return builder.build();
     }
 
     public String getField() {
@@ -459,4 +472,91 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
         }
     }
 
+    private static PercolateQuery.QueryStore createStore(PercolatorFieldMapper.PercolatorFieldType fieldType,
+                                                         QueryShardContext context,
+                                                         boolean mapUnmappedFieldsAsString) {
+        return ctx -> {
+            LeafReader leafReader = ctx.reader();
+            BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues(fieldType.getQueryBuilderFieldName());
+            if (binaryDocValues == null) {
+                return docId -> null;
+            }
+
+            Bits bits = leafReader.getDocsWithField(fieldType.getQueryBuilderFieldName());
+            return docId -> {
+                if (bits.get(docId)) {
+                    BytesRef qbSource = binaryDocValues.get(docId);
+                    if (qbSource.length > 0) {
+                        XContent xContent = PercolatorFieldMapper.QUERY_BUILDER_CONTENT_TYPE.xContent();
+                        try (XContentParser sourceParser = xContent.createParser(qbSource.bytes, qbSource.offset, qbSource.length)) {
+                            return parseQuery(context, mapUnmappedFieldsAsString, sourceParser);
+                        }
+                    } else {
+                        return null;
+                    }
+                } else {
+                    return null;
+                }
+            };
+        };
+    }
+
+    private static PercolateQuery.QueryStore createLegacyStore(QueryShardContext context, boolean mapUnmappedFieldsAsString) {
+        return ctx -> {
+            LeafReader leafReader = ctx.reader();
+            return docId -> {
+                LegacyQueryFieldVisitor visitor = new LegacyQueryFieldVisitor();
+                leafReader.document(docId, visitor);
+                if (visitor.source == null) {
+                    throw new IllegalStateException("No source found for document with docid [" + docId + "]");
+                }
+
+                try (XContentParser sourceParser = XContentHelper.createParser(visitor.source)) {
+                    String currentFieldName = null;
+                    XContentParser.Token token = sourceParser.nextToken(); // move the START_OBJECT
+                    if (token != XContentParser.Token.START_OBJECT) {
+                        throw new ElasticsearchException("failed to parse query [" + docId + "], not starting with OBJECT");
+                    }
+                    while ((token = sourceParser.nextToken()) != XContentParser.Token.END_OBJECT) {
+                        if (token == XContentParser.Token.FIELD_NAME) {
+                            currentFieldName = sourceParser.currentName();
+                        } else if (token == XContentParser.Token.START_OBJECT) {
+                            if ("query".equals(currentFieldName)) {
+                                return parseQuery(context, mapUnmappedFieldsAsString, sourceParser);
+                            } else {
+                                sourceParser.skipChildren();
+                            }
+                        } else if (token == XContentParser.Token.START_ARRAY) {
+                            sourceParser.skipChildren();
+                        }
+                    }
+                }
+                return null;
+            };
+        };
+    }
+
+    private final static class LegacyQueryFieldVisitor extends StoredFieldVisitor {
+
+        private BytesArray source;
+
+        @Override
+        public void binaryField(FieldInfo fieldInfo, byte[] bytes) throws IOException {
+            source = new BytesArray(bytes);
+        }
+
+        @Override
+        public Status needsField(FieldInfo fieldInfo) throws IOException {
+            if (source != null)  {
+                return Status.STOP;
+            }
+            if (SourceFieldMapper.NAME.equals(fieldInfo.name)) {
+                return Status.YES;
+            } else {
+                return Status.NO;
+            }
+        }
+
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
index 4aa72728bc8..3cdabb74d7e 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
@@ -51,7 +51,6 @@ import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.core.TextFieldMapper;
 import org.elasticsearch.index.mapper.object.ObjectMapper;
-import org.elasticsearch.index.percolator.PercolatorQueryCache;
 import org.elasticsearch.index.query.support.NestedScope;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
@@ -82,7 +81,6 @@ public class QueryShardContext extends QueryRewriteContext {
     private final Map<String, Query> namedQueries = new HashMap<>();
     private final MapperQueryParser queryParser = new MapperQueryParser(this);
     private final IndicesQueriesRegistry indicesQueriesRegistry;
-    private final PercolatorQueryCache percolatorQueryCache;
     private boolean allowUnmappedFields;
     private boolean mapUnmappedFieldAsString;
     private NestedScope nestedScope;
@@ -90,7 +88,7 @@ public class QueryShardContext extends QueryRewriteContext {
 
     public QueryShardContext(IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache, IndexFieldDataService indexFieldDataService,
                              MapperService mapperService, SimilarityService similarityService, ScriptService scriptService,
-                             final IndicesQueriesRegistry indicesQueriesRegistry, Client client, PercolatorQueryCache percolatorQueryCache,
+                             final IndicesQueriesRegistry indicesQueriesRegistry, Client client,
                              IndexReader reader, ClusterState clusterState) {
         super(indexSettings, mapperService, scriptService, indicesQueriesRegistry, client, reader, clusterState);
         this.indexSettings = indexSettings;
@@ -100,14 +98,13 @@ public class QueryShardContext extends QueryRewriteContext {
         this.indexFieldDataService = indexFieldDataService;
         this.allowUnmappedFields = indexSettings.isDefaultAllowUnmappedFields();
         this.indicesQueriesRegistry = indicesQueriesRegistry;
-        this.percolatorQueryCache = percolatorQueryCache;
         this.nestedScope = new NestedScope();
     }
 
     public QueryShardContext(QueryShardContext source) {
         this(source.indexSettings, source.bitsetFilterCache, source.indexFieldDataService, source.mapperService,
                 source.similarityService, source.scriptService, source.indicesQueriesRegistry, source.client,
-                source.percolatorQueryCache, source.reader, source.clusterState);
+                source.reader, source.clusterState);
         this.types = source.getTypes();
     }
 
@@ -123,10 +120,6 @@ public class QueryShardContext extends QueryRewriteContext {
         return mapperService.analysisService();
     }
 
-    public PercolatorQueryCache getPercolatorQueryCache() {
-        return percolatorQueryCache;
-    }
-
     public Similarity getSearchSimilarity() {
         return similarityService != null ? similarityService.similarity(mapperService) : null;
     }
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index 3b9b186ddd8..6b1d24e47f1 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -280,7 +280,7 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
                     if (indexShard.routingEntry() == null) {
                         continue;
                     }
-                    IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesQueryCache, indexService.cache().getPercolatorQueryCache(), indexShard, flags), indexShard.commitStats()) });
+                    IndexShardStats indexShardStats = new IndexShardStats(indexShard.shardId(), new ShardStats[] { new ShardStats(indexShard.routingEntry(), indexShard.shardPath(), new CommonStats(indicesQueryCache, indexShard, flags), indexShard.commitStats()) });
                     if (!statsByShard.containsKey(indexService.index())) {
                         statsByShard.put(indexService.index(), arrayAsArrayList(indexShardStats));
                     } else {
diff --git a/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java b/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java
index f4736f0332e..f205cdd8562 100644
--- a/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java
+++ b/core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java
@@ -37,7 +37,6 @@ import org.elasticsearch.index.flush.FlushStats;
 import org.elasticsearch.index.get.GetStats;
 import org.elasticsearch.index.shard.IndexingStats;
 import org.elasticsearch.index.merge.MergeStats;
-import org.elasticsearch.index.percolator.PercolatorQueryCacheStats;
 import org.elasticsearch.index.recovery.RecoveryStats;
 import org.elasticsearch.index.refresh.RefreshStats;
 import org.elasticsearch.index.search.stats.SearchStats;
@@ -102,11 +101,6 @@ public class NodeIndicesStats implements Streamable, ToXContent {
         return stats.getSearch();
     }
 
-    @Nullable
-    public PercolatorQueryCacheStats getPercolate() {
-        return stats.getPercolatorCache();
-    }
-
     @Nullable
     public MergeStats getMerge() {
         return stats.getMerge();
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java
index dbda83709ba..55ed1d8dda4 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java
@@ -78,7 +78,6 @@ public class RestIndicesStatsAction extends BaseRestHandler {
             indicesStatsRequest.flush(metrics.contains("flush"));
             indicesStatsRequest.warmer(metrics.contains("warmer"));
             indicesStatsRequest.queryCache(metrics.contains("query_cache"));
-            indicesStatsRequest.percolate(metrics.contains("percolator_cache"));
             indicesStatsRequest.segments(metrics.contains("segments"));
             indicesStatsRequest.fieldData(metrics.contains("fielddata"));
             indicesStatsRequest.completion(metrics.contains("completion"));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
index 958fa40b54b..e266efa4357 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
@@ -222,9 +222,6 @@ public class RestIndicesAction extends AbstractCatAction {
         table.addCell("merges.total_time", "sibling:pri;alias:mtt,mergesTotalTime;default:false;text-align:right;desc:time spent in merges");
         table.addCell("pri.merges.total_time", "default:false;text-align:right;desc:time spent in merges");
 
-        table.addCell("percolate.queries", "sibling:pri;alias:pq,percolateQueries;default:false;text-align:right;desc:number of registered percolation queries");
-        table.addCell("pri.percolate.queries", "default:false;text-align:right;desc:number of registered percolation queries");
-
         table.addCell("refresh.total", "sibling:pri;alias:rto,refreshTotal;default:false;text-align:right;desc:total refreshes");
         table.addCell("pri.refresh.total", "default:false;text-align:right;desc:total refreshes");
 
@@ -424,9 +421,6 @@ public class RestIndicesAction extends AbstractCatAction {
             table.addCell(indexStats == null ? null : indexStats.getTotal().getMerge().getTotalTime());
             table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getTotalTime());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getPercolatorCache().getNumQueries());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getPercolatorCache().getNumQueries());
-
             table.addCell(indexStats == null ? null : indexStats.getTotal().getRefresh().getTotal());
             table.addCell(indexStats == null ? null : indexStats.getPrimaries().getRefresh().getTotal());
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
index 15656733baf..f23f0e0a029 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
@@ -45,7 +45,6 @@ import org.elasticsearch.index.fielddata.FieldDataStats;
 import org.elasticsearch.index.flush.FlushStats;
 import org.elasticsearch.index.get.GetStats;
 import org.elasticsearch.index.merge.MergeStats;
-import org.elasticsearch.index.percolator.PercolatorQueryCacheStats;
 import org.elasticsearch.index.refresh.RefreshStats;
 import org.elasticsearch.index.search.stats.SearchStats;
 import org.elasticsearch.index.shard.IndexingStats;
@@ -185,8 +184,6 @@ public class RestNodesAction extends AbstractCatAction {
         table.addCell("merges.total_size", "alias:mts,mergesTotalSize;default:false;text-align:right;desc:size merged");
         table.addCell("merges.total_time", "alias:mtt,mergesTotalTime;default:false;text-align:right;desc:time spent in merges");
 
-        table.addCell("percolate.queries", "alias:pq,percolateQueries;default:false;text-align:right;desc:number of registered percolation queries");
-
         table.addCell("refresh.total", "alias:rto,refreshTotal;default:false;text-align:right;desc:total refreshes");
         table.addCell("refresh.time", "alias:rti,refreshTime;default:false;text-align:right;desc:time spent in refreshes");
 
@@ -338,9 +335,6 @@ public class RestNodesAction extends AbstractCatAction {
             table.addCell(mergeStats == null ? null : mergeStats.getTotalSize());
             table.addCell(mergeStats == null ? null : mergeStats.getTotalTime());
 
-            PercolatorQueryCacheStats percolatorQueryCacheStats = indicesStats == null ? null : indicesStats.getPercolate();
-            table.addCell(percolatorQueryCacheStats == null ? null : percolatorQueryCacheStats.getNumQueries());
-
             RefreshStats refreshStats = indicesStats == null ? null : indicesStats.getRefresh();
             table.addCell(refreshStats == null ? null : refreshStats.getTotal());
             table.addCell(refreshStats == null ? null : refreshStats.getTotalTime());
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
index 9954bd3098b..6538b405fce 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
@@ -139,8 +139,6 @@ public class RestShardsAction extends AbstractCatAction {
         table.addCell("merges.total_size", "alias:mts,mergesTotalSize;default:false;text-align:right;desc:size merged");
         table.addCell("merges.total_time", "alias:mtt,mergesTotalTime;default:false;text-align:right;desc:time spent in merges");
 
-        table.addCell("percolate.queries", "alias:pq,percolateQueries;default:false;text-align:right;desc:number of registered percolation queries");
-
         table.addCell("refresh.total", "alias:rto,refreshTotal;default:false;text-align:right;desc:total refreshes");
         table.addCell("refresh.time", "alias:rti,refreshTime;default:false;text-align:right;desc:time spent in refreshes");
 
@@ -278,8 +276,6 @@ public class RestShardsAction extends AbstractCatAction {
             table.addCell(commonStats == null ? null : commonStats.getMerge().getTotalSize());
             table.addCell(commonStats == null ? null : commonStats.getMerge().getTotalTime());
 
-            table.addCell(commonStats == null ? null : commonStats.getPercolatorCache().getNumQueries());
-
             table.addCell(commonStats == null ? null : commonStats.getRefresh().getTotal());
             table.addCell(commonStats == null ? null : commonStats.getRefresh().getTotalTime());
 
diff --git a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
index 96319303420..e69e4094f44 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
@@ -48,7 +48,6 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
 import org.elasticsearch.index.mapper.object.ObjectMapper;
-import org.elasticsearch.index.percolator.PercolatorQueryCache;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.QueryShardContext;
@@ -216,7 +215,7 @@ public class DefaultSearchContext extends SearchContext {
                             + "be less than [" + maxWindow + "]. This prevents allocating massive heaps for storing the results to be "
                             + "rescored. This limit can be set by chaning the [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey()
                             + "] index level setting.");
-                
+
                 }
             }
         }
@@ -495,11 +494,6 @@ public class DefaultSearchContext extends SearchContext {
         return indexService.fieldData();
     }
 
-    @Override
-    public PercolatorQueryCache percolatorQueryCache() {
-        return indexService.cache().getPercolatorQueryCache();
-    }
-
     @Override
     public long timeoutInMillis() {
         return timeoutInMillis;
diff --git a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
index 8009d0b5fe4..ac283f1f7ef 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
@@ -33,7 +33,6 @@ import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.object.ObjectMapper;
-import org.elasticsearch.index.percolator.PercolatorQueryCache;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.IndexShard;
@@ -270,11 +269,6 @@ public abstract class FilteredSearchContext extends SearchContext {
         return in.fieldData();
     }
 
-    @Override
-    public PercolatorQueryCache percolatorQueryCache() {
-        return in.percolatorQueryCache();
-    }
-
     @Override
     public long timeoutInMillis() {
         return in.timeoutInMillis();
diff --git a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
index 550a5f76caf..55b66c42a60 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
@@ -38,7 +38,6 @@ import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.object.ObjectMapper;
-import org.elasticsearch.index.percolator.PercolatorQueryCache;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.IndexShard;
@@ -231,8 +230,6 @@ public abstract class SearchContext implements Releasable {
 
     public abstract IndexFieldDataService fieldData();
 
-    public abstract PercolatorQueryCache percolatorQueryCache();
-
     public abstract long timeoutInMillis();
 
     public abstract void timeoutInMillis(long timeoutInMillis);
diff --git a/core/src/test/java/org/elasticsearch/index/percolator/PercolatorFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/percolator/PercolatorFieldMapperTests.java
index 604670f179e..8faad63c0a4 100644
--- a/core/src/test/java/org/elasticsearch/index/percolator/PercolatorFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/percolator/PercolatorFieldMapperTests.java
@@ -38,7 +38,6 @@ import org.junit.Before;
 import java.io.IOException;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.index.percolator.PercolatorQueryCache.QUERY_BUILDER_CONTENT_TYPE;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchPhraseQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
@@ -251,7 +250,8 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
     }
 
     private void assertQueryBuilder(BytesRef actual, QueryBuilder expected) throws IOException {
-        XContentParser sourceParser = QUERY_BUILDER_CONTENT_TYPE.xContent().createParser(actual.bytes, actual.offset, actual.length);
+        XContentParser sourceParser = PercolatorFieldMapper.QUERY_BUILDER_CONTENT_TYPE.xContent()
+                .createParser(actual.bytes, actual.offset, actual.length);
         QueryParseContext qsc = indexService.newQueryShardContext().newParseContext(sourceParser);
         assertThat(qsc.parseInnerQueryBuilder(), equalTo(expected));
     }
diff --git a/core/src/test/java/org/elasticsearch/index/percolator/PercolatorQueryCacheTests.java b/core/src/test/java/org/elasticsearch/index/percolator/PercolatorQueryCacheTests.java
deleted file mode 100644
index 30986e45e4b..00000000000
--- a/core/src/test/java/org/elasticsearch/index/percolator/PercolatorQueryCacheTests.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.percolator;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.core.SimpleAnalyzer;
-import org.apache.lucene.document.BinaryDocValuesField;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.StoredField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.NoMergePolicy;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TieredMergePolicy;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.WildcardQuery;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
-import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.IndexWarmer;
-import org.elasticsearch.index.analysis.AnalysisRegistry;
-import org.elasticsearch.index.analysis.AnalysisService;
-import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.mapper.DocumentFieldMappers;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
-import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
-import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.PercolateQuery;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryParser;
-import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.index.query.TermQueryBuilder;
-import org.elasticsearch.index.query.WildcardQueryBuilder;
-import org.elasticsearch.index.shard.IndexShard;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.index.similarity.SimilarityService;
-import org.elasticsearch.index.warmer.ShardIndexWarmerService;
-import org.elasticsearch.indices.IndicesModule;
-import org.elasticsearch.indices.mapper.MapperRegistry;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.IndexSettingsModule;
-import org.elasticsearch.threadpool.ThreadPool;
-
-import java.io.IOException;
-import java.util.Collections;
-
-import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
-import static org.elasticsearch.index.query.QueryBuilders.termQuery;
-import static org.elasticsearch.index.query.QueryBuilders.wildcardQuery;
-import static org.hamcrest.Matchers.equalTo;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class PercolatorQueryCacheTests extends ESTestCase {
-
-    private QueryShardContext queryShardContext;
-    private PercolatorQueryCache cache;
-
-    void initialize(Object... fields) throws IOException {
-        Settings settings = Settings.builder()
-                .put("node.name", PercolatorQueryCacheTests.class.toString())
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-                .build();
-
-        IndicesQueriesRegistry indicesQueriesRegistry = new IndicesQueriesRegistry();
-        QueryParser<TermQueryBuilder> termParser = TermQueryBuilder::fromXContent;
-        indicesQueriesRegistry.register(termParser, TermQueryBuilder.QUERY_NAME_FIELD);
-        QueryParser<WildcardQueryBuilder> wildcardParser = WildcardQueryBuilder::fromXContent;
-        indicesQueriesRegistry.register(wildcardParser, WildcardQueryBuilder.QUERY_NAME_FIELD);
-        QueryParser<BoolQueryBuilder> boolQueryParser = BoolQueryBuilder::fromXContent;
-        indicesQueriesRegistry.register(boolQueryParser, BoolQueryBuilder.QUERY_NAME_FIELD);
-
-        Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
-        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(new Index("_index", ClusterState.UNKNOWN_UUID), indexSettings);
-        SimilarityService similarityService = new SimilarityService(idxSettings, Collections.emptyMap());
-        AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
-        MapperRegistry mapperRegistry = new IndicesModule().getMapperRegistry();
-        MapperService mapperService = new MapperService(idxSettings, analysisService, similarityService, mapperRegistry,
-                () -> queryShardContext);
-        mapperService.merge("type", new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef("type", fields).string()),
-                MapperService.MergeReason.MAPPING_UPDATE, false);
-        cache = new PercolatorQueryCache(idxSettings, () -> queryShardContext);
-        ClusterState state = ClusterState.builder(new ClusterName("_name")).build();
-        queryShardContext = new QueryShardContext(idxSettings, null, null, mapperService, similarityService, null,
-                    indicesQueriesRegistry, null, cache, null, state);
-    }
-
-    public void testLoadQueries() throws Exception {
-        Directory directory = newDirectory();
-        IndexWriter indexWriter = new IndexWriter(
-                directory,
-                new IndexWriterConfig(new MockAnalyzer(random()))
-                        .setMergePolicy(NoMergePolicy.INSTANCE)
-        );
-
-        boolean legacyFormat = randomBoolean();
-        Version version = legacyFormat ? Version.V_2_0_0 : Version.CURRENT;
-        IndexShard indexShard = mockIndexShard(version, legacyFormat);
-
-        storeQuery("0", indexWriter, termQuery("field1", "value1"), true, legacyFormat);
-        storeQuery("1", indexWriter, wildcardQuery("field1", "v*"), true, legacyFormat);
-        storeQuery("2", indexWriter, boolQuery().must(termQuery("field1", "value1")).must(termQuery("field2", "value2")),
-                true, legacyFormat);
-        // dymmy docs should be skipped during loading:
-        Document doc = new Document();
-        doc.add(new StringField("dummy", "value", Field.Store.YES));
-        indexWriter.addDocument(doc);
-        storeQuery("4", indexWriter, termQuery("field2", "value2"), true, legacyFormat);
-        // only documents that .percolator type should be loaded:
-        storeQuery("5", indexWriter, termQuery("field2", "value2"), false, legacyFormat);
-        storeQuery("6", indexWriter, termQuery("field3", "value3"), true, legacyFormat);
-        indexWriter.forceMerge(1);
-
-        // also include queries for percolator docs marked as deleted:
-        indexWriter.deleteDocuments(new Term("id", "6"));
-        indexWriter.close();
-
-        ShardId shardId = new ShardId("_index", ClusterState.UNKNOWN_UUID, 0);
-        IndexReader indexReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(directory), shardId);
-        assertThat(indexReader.leaves().size(), equalTo(1));
-        assertThat(indexReader.numDeletedDocs(), equalTo(1));
-        assertThat(indexReader.maxDoc(), equalTo(7));
-
-        initialize("field1", "type=keyword", "field2", "type=keyword", "field3", "type=keyword");
-
-        PercolatorQueryCache.QueriesLeaf leaf = cache.loadQueries(indexReader.leaves().get(0), indexShard);
-        assertThat(leaf.queries.size(), equalTo(5));
-        assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("field1", "value1"))));
-        assertThat(leaf.getQuery(1), equalTo(new WildcardQuery(new Term("field1", "v*"))));
-        assertThat(leaf.getQuery(2), equalTo(new BooleanQuery.Builder()
-                .add(new TermQuery(new Term("field1", "value1")), BooleanClause.Occur.MUST)
-                .add(new TermQuery(new Term("field2", "value2")), BooleanClause.Occur.MUST)
-                .build()
-        ));
-        assertThat(leaf.getQuery(4), equalTo(new TermQuery(new Term("field2", "value2"))));
-        assertThat(leaf.getQuery(6), equalTo(new TermQuery(new Term("field3", "value3"))));
-
-        indexReader.close();
-        directory.close();
-    }
-
-    public void testGetQueries() throws Exception {
-        Directory directory = newDirectory();
-        IndexWriter indexWriter = new IndexWriter(
-                directory,
-                new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE)
-        );
-
-        storeQuery("0", indexWriter, termQuery("a", "0"), true, false);
-        storeQuery("1", indexWriter, termQuery("a", "1"), true, false);
-        storeQuery("2", indexWriter, termQuery("a", "2"), true, false);
-        indexWriter.flush();
-        storeQuery("3", indexWriter, termQuery("a", "3"), true, false);
-        storeQuery("4", indexWriter, termQuery("a", "4"), true, false);
-        storeQuery("5", indexWriter, termQuery("a", "5"), true, false);
-        indexWriter.flush();
-        storeQuery("6", indexWriter, termQuery("a", "6"), true, false);
-        storeQuery("7", indexWriter, termQuery("a", "7"), true, false);
-        storeQuery("8", indexWriter, termQuery("a", "8"), true, false);
-        indexWriter.flush();
-        indexWriter.close();
-
-        ShardId shardId = new ShardId("_index", ClusterState.UNKNOWN_UUID , 0);
-        IndexReader indexReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(directory), shardId);
-        assertThat(indexReader.leaves().size(), equalTo(3));
-        assertThat(indexReader.maxDoc(), equalTo(9));
-
-        initialize("a", "type=keyword");
-
-        try {
-            cache.getQueries(indexReader.leaves().get(0));
-            fail("IllegalStateException expected");
-        } catch (IllegalStateException e) {
-            assertThat(e.getMessage(), equalTo("queries not loaded, queries should be have been preloaded during index warming..."));
-        }
-
-        IndexShard indexShard = mockIndexShard(Version.CURRENT, false);
-        ThreadPool threadPool = mockThreadPool();
-        IndexWarmer.Listener listener = cache.createListener(threadPool);
-        listener.warmReader(indexShard, new Engine.Searcher("test", new IndexSearcher(indexReader)));
-        PercolatorQueryCacheStats stats = cache.getStats(shardId);
-        assertThat(stats.getNumQueries(), equalTo(9L));
-
-        PercolateQuery.QueryRegistry.Leaf leaf = cache.getQueries(indexReader.leaves().get(0));
-        assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "0"))));
-        assertThat(leaf.getQuery(1), equalTo(new TermQuery(new Term("a", "1"))));
-        assertThat(leaf.getQuery(2), equalTo(new TermQuery(new Term("a", "2"))));
-
-        leaf = cache.getQueries(indexReader.leaves().get(1));
-        assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "3"))));
-        assertThat(leaf.getQuery(1), equalTo(new TermQuery(new Term("a", "4"))));
-        assertThat(leaf.getQuery(2), equalTo(new TermQuery(new Term("a", "5"))));
-
-        leaf = cache.getQueries(indexReader.leaves().get(2));
-        assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "6"))));
-        assertThat(leaf.getQuery(1), equalTo(new TermQuery(new Term("a", "7"))));
-        assertThat(leaf.getQuery(2), equalTo(new TermQuery(new Term("a", "8"))));
-
-        indexReader.close();
-        directory.close();
-    }
-
-    public void testInvalidateEntries() throws Exception {
-        Directory directory = newDirectory();
-        IndexWriter indexWriter = new IndexWriter(
-                directory,
-                new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE)
-        );
-
-        storeQuery("0", indexWriter, termQuery("a", "0"), true, false);
-        indexWriter.flush();
-        storeQuery("1", indexWriter, termQuery("a", "1"), true, false);
-        indexWriter.flush();
-        storeQuery("2", indexWriter, termQuery("a", "2"), true, false);
-        indexWriter.flush();
-
-        ShardId shardId = new ShardId("_index", ClusterState.UNKNOWN_UUID, 0);
-        IndexReader indexReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(indexWriter), shardId);
-        assertThat(indexReader.leaves().size(), equalTo(3));
-        assertThat(indexReader.maxDoc(), equalTo(3));
-
-        initialize("a", "type=keyword");
-
-        IndexShard indexShard = mockIndexShard(Version.CURRENT, false);
-        ThreadPool threadPool = mockThreadPool();
-        IndexWarmer.Listener listener = cache.createListener(threadPool);
-        listener.warmReader(indexShard, new Engine.Searcher("test", new IndexSearcher(indexReader)));
-        assertThat(cache.getStats(shardId).getNumQueries(), equalTo(3L));
-
-        PercolateQuery.QueryRegistry.Leaf leaf = cache.getQueries(indexReader.leaves().get(0));
-        assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "0"))));
-
-        leaf = cache.getQueries(indexReader.leaves().get(1));
-        assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "1"))));
-
-        leaf = cache.getQueries(indexReader.leaves().get(2));
-        assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "2"))));
-
-        // change merge policy, so that merges will actually happen:
-        indexWriter.getConfig().setMergePolicy(new TieredMergePolicy());
-        indexWriter.deleteDocuments(new Term("id", "1"));
-        indexWriter.forceMergeDeletes();
-        indexReader.close();
-        indexReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(indexWriter), shardId);
-        assertThat(indexReader.leaves().size(), equalTo(2));
-        assertThat(indexReader.maxDoc(), equalTo(2));
-        listener.warmReader(indexShard, new Engine.Searcher("test", new IndexSearcher(indexReader)));
-        assertThat(cache.getStats(shardId).getNumQueries(), equalTo(2L));
-
-        leaf = cache.getQueries(indexReader.leaves().get(0));
-        assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "0"))));
-
-        leaf = cache.getQueries(indexReader.leaves().get(1));
-        assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "2"))));
-
-        indexWriter.forceMerge(1);
-        indexReader.close();
-        indexReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(indexWriter), shardId);
-        assertThat(indexReader.leaves().size(), equalTo(1));
-        assertThat(indexReader.maxDoc(), equalTo(2));
-        listener.warmReader(indexShard, new Engine.Searcher("test", new IndexSearcher(indexReader)));
-        assertThat(cache.getStats(shardId).getNumQueries(), equalTo(2L));
-
-        leaf = cache.getQueries(indexReader.leaves().get(0));
-        assertThat(leaf.getQuery(0), equalTo(new TermQuery(new Term("a", "0"))));
-        assertThat(leaf.getQuery(1), equalTo(new TermQuery(new Term("a", "2"))));
-
-        indexWriter.close();
-        indexReader.close();
-        directory.close();
-    }
-
-    void storeQuery(String id, IndexWriter indexWriter, QueryBuilder queryBuilder, boolean typeField, boolean legacy) throws IOException {
-        Document doc = new Document();
-        doc.add(new StringField("id", id, Field.Store.NO));
-        if (typeField) {
-            if (legacy) {
-                doc.add(new StringField(TypeFieldMapper.NAME, PercolatorFieldMapper.LEGACY_TYPE_NAME, Field.Store.NO));
-            } else {
-                doc.add(new StringField(TypeFieldMapper.NAME, "query", Field.Store.NO));
-            }
-        }
-        if (legacy) {
-            BytesReference percolatorQuery = XContentFactory.jsonBuilder().startObject()
-                    .field("query", queryBuilder)
-                    .endObject().bytes();
-            doc.add(new StoredField(
-                    SourceFieldMapper.NAME,
-                    percolatorQuery.array(), percolatorQuery.arrayOffset(), percolatorQuery.length())
-            );
-        } else {
-            BytesRef queryBuilderAsBytes = new BytesRef(
-                    XContentFactory.contentBuilder(PercolatorQueryCache.QUERY_BUILDER_CONTENT_TYPE).value(queryBuilder).bytes().toBytes()
-            );
-            doc.add(new BinaryDocValuesField(PercolatorFieldMapper.QUERY_BUILDER_FIELD_NAME, queryBuilderAsBytes));
-        }
-        indexWriter.addDocument(doc);
-    }
-
-    IndexShard mockIndexShard(Version version, boolean legacyFormat) {
-        IndexShard indexShard = mock(IndexShard.class);
-        ShardIndexWarmerService shardIndexWarmerService = mock(ShardIndexWarmerService.class);
-        when(shardIndexWarmerService.logger()).thenReturn(logger);
-        when(indexShard.warmerService()).thenReturn(shardIndexWarmerService);
-        IndexSettings indexSettings = new IndexSettings(
-                IndexMetaData.builder("_index").settings(Settings.builder()
-                        .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-                        .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
-                        .put(IndexMetaData.SETTING_VERSION_CREATED, version)
-                ).build(),
-                Settings.EMPTY
-        );
-        when(indexShard.indexSettings()).thenReturn(indexSettings);
-
-        PercolatorFieldMapper.PercolatorFieldType fieldType = mock(PercolatorFieldMapper.PercolatorFieldType.class);
-        when(fieldType.name()).thenReturn("query");
-        when(fieldType.getQueryBuilderFieldName()).thenReturn(PercolatorFieldMapper.QUERY_BUILDER_FIELD_NAME);
-        PercolatorFieldMapper percolatorFieldMapper = mock(PercolatorFieldMapper.class);
-        when(percolatorFieldMapper.fieldType()).thenReturn(fieldType);
-        MapperService mapperService = mock(MapperService.class);
-        DocumentMapper documentMapper = mock(DocumentMapper.class);
-        if (legacyFormat) {
-            when(documentMapper.type()).thenReturn(PercolatorFieldMapper.LEGACY_TYPE_NAME);
-            when(documentMapper.typeFilter())
-                    .thenReturn(new TermQuery(new Term(TypeFieldMapper.NAME, PercolatorFieldMapper.LEGACY_TYPE_NAME)));
-        } else {
-            when(documentMapper.type()).thenReturn("query");
-            when(documentMapper.typeFilter()).thenReturn(new TermQuery(new Term(TypeFieldMapper.NAME, "query")));
-        }
-
-        Analyzer analyzer = new SimpleAnalyzer();
-        DocumentFieldMappers documentFieldMappers =
-                new DocumentFieldMappers(Collections.singleton(percolatorFieldMapper), analyzer, analyzer, analyzer);
-        when(documentMapper.mappers()).thenReturn(documentFieldMappers);
-
-        when(mapperService.docMappers(false)).thenReturn(Collections.singleton(documentMapper));
-
-        when(indexShard.mapperService()).thenReturn(mapperService);
-
-        return indexShard;
-    }
-
-    ThreadPool mockThreadPool() {
-        ThreadPool threadPool = mock(ThreadPool.class);
-        when(threadPool.executor(anyString())).thenReturn(Runnable::run);
-        return threadPool;
-    }
-
-}
diff --git a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
index 22bdc2c7290..b150000daf1 100644
--- a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
@@ -76,7 +76,6 @@ import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.percolator.PercolatorQueryCache;
 import org.elasticsearch.index.query.support.QueryParsers;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.similarity.SimilarityService;
@@ -190,7 +189,6 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
     private static IndexSettings idxSettings;
     private static SimilarityService similarityService;
     private static MapperService mapperService;
-    private static PercolatorQueryCache percolatorQueryCache;
     private static BitsetFilterCache bitsetFilterCache;
     private static ScriptService scriptService;
 
@@ -308,7 +306,6 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
 
             }
         });
-        percolatorQueryCache = new PercolatorQueryCache(idxSettings, () -> createShardContext());
         indicesQueriesRegistry = injector.getInstance(IndicesQueriesRegistry.class);
         //create some random type with some default field, those types will stick around for all of the subclasses
         currentTypes = new String[randomIntBetween(0, 5)];
@@ -349,7 +346,6 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
         idxSettings = null;
         similarityService = null;
         mapperService = null;
-        percolatorQueryCache = null;
         bitsetFilterCache = null;
         scriptService = null;
     }
@@ -750,7 +746,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
         ClusterState state = ClusterState.builder(new ClusterName("_name")).build();
         Client client = injector.getInstance(Client.class);
         return new QueryShardContext(idxSettings, bitsetFilterCache, indexFieldDataService, mapperService, similarityService,
-                scriptService, indicesQueriesRegistry, client, percolatorQueryCache, null, state);
+                scriptService, indicesQueriesRegistry, client, null, state);
     }
 
     /**
diff --git a/core/src/test/java/org/elasticsearch/index/query/PercolateQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/PercolateQueryBuilderTests.java
index 690b2c03a2f..92a9369aed5 100644
--- a/core/src/test/java/org/elasticsearch/index/query/PercolateQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/PercolateQueryBuilderTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.index.query;
 
 import com.fasterxml.jackson.core.JsonParseException;
-
 import org.apache.lucene.search.Query;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ResourceNotFoundException;
@@ -42,6 +41,7 @@ import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.util.Collections;
+
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/PercolateQueryTests.java b/core/src/test/java/org/elasticsearch/index/query/PercolateQueryTests.java
index acb4917b421..8dde535e3cc 100644
--- a/core/src/test/java/org/elasticsearch/index/query/PercolateQueryTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/PercolateQueryTests.java
@@ -84,14 +84,14 @@ public class PercolateQueryTests extends ESTestCase {
     private Directory directory;
     private IndexWriter indexWriter;
     private Map<String, Query> queries;
-    private PercolateQuery.QueryRegistry queryRegistry;
+    private PercolateQuery.QueryStore queryStore;
     private DirectoryReader directoryReader;
 
     @Before
     public void init() throws Exception {
         directory = newDirectory();
         queries = new HashMap<>();
-        queryRegistry = ctx -> docId -> {
+        queryStore = ctx -> docId -> {
             try {
                 String val = ctx.reader().document(docId).get(UidFieldMapper.NAME);
                 return queries.get(Uid.createUid(val).id());
@@ -145,7 +145,7 @@ public class PercolateQueryTests extends ESTestCase {
 
         PercolateQuery.Builder builder = new PercolateQuery.Builder(
                 "docType",
-                queryRegistry,
+                queryStore,
                 new BytesArray("{}"),
                 percolateSearcher
         );
@@ -219,7 +219,7 @@ public class PercolateQueryTests extends ESTestCase {
 
         PercolateQuery.Builder builder = new PercolateQuery.Builder(
                 "docType",
-                queryRegistry,
+                queryStore,
                 new BytesArray("{}"),
                 percolateSearcher
         );
@@ -336,7 +336,7 @@ public class PercolateQueryTests extends ESTestCase {
         IndexSearcher percolateSearcher = memoryIndex.createSearcher();
         PercolateQuery.Builder builder1 = new PercolateQuery.Builder(
                 "docType",
-                queryRegistry,
+                queryStore,
                 new BytesArray("{}"),
                 percolateSearcher
         );
@@ -346,7 +346,7 @@ public class PercolateQueryTests extends ESTestCase {
 
         PercolateQuery.Builder builder2 = new PercolateQuery.Builder(
                 "docType",
-                queryRegistry,
+                queryStore,
                 new BytesArray("{}"),
                 percolateSearcher
         );
diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java
index b6d69374661..9a4c9aece5c 100644
--- a/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java
@@ -47,7 +47,7 @@ public class QueryShardContextTests extends ESTestCase {
         MapperService mapperService = mock(MapperService.class);
         when(mapperService.getIndexSettings()).thenReturn(indexSettings);
         QueryShardContext context = new QueryShardContext(
-            indexSettings, null, null, mapperService, null, null, null, null, null, null, null
+            indexSettings, null, null, mapperService, null, null, null, null, null, null
         );
 
         context.setAllowUnmappedFields(false);
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 52c16dd74d9..f96b546855b 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -644,7 +644,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         IndexService test = indicesService.indexService(resolveIndex("test"));
         IndexShard shard = test.getShardOrNull(0);
-        ShardStats stats = new ShardStats(shard.routingEntry(), shard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), test.cache().getPercolatorQueryCache(), shard, new CommonStatsFlags()), shard.commitStats());
+        ShardStats stats = new ShardStats(shard.routingEntry(), shard.shardPath(), new CommonStats(indicesService.getIndicesQueryCache(), shard, new CommonStatsFlags()), shard.commitStats());
         assertEquals(shard.shardPath().getRootDataPath().toString(), stats.getDataPath());
         assertEquals(shard.shardPath().getRootStatePath().toString(), stats.getStatePath());
         assertEquals(shard.shardPath().isCustomDataPath(), stats.isCustomDataPath());
diff --git a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
index 4716e7dba78..92c3260aeb0 100644
--- a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
@@ -670,7 +670,7 @@ public class IndexStatsIT extends ESIntegTestCase {
 
     public void testFlagOrdinalOrder() {
         Flag[] flags = new Flag[]{Flag.Store, Flag.Indexing, Flag.Get, Flag.Search, Flag.Merge, Flag.Flush, Flag.Refresh,
-                Flag.QueryCache, Flag.FieldData, Flag.Docs, Flag.Warmer, Flag.PercolatorCache, Flag.Completion, Flag.Segments,
+                Flag.QueryCache, Flag.FieldData, Flag.Docs, Flag.Warmer, Flag.Completion, Flag.Segments,
                 Flag.Translog, Flag.Suggest, Flag.RequestCache, Flag.Recovery};
 
         assertThat(flags.length, equalTo(Flag.values().length));
@@ -913,9 +913,6 @@ public class IndexStatsIT extends ESIntegTestCase {
             case Warmer:
                 builder.setWarmer(set);
                 break;
-            case PercolatorCache:
-                builder.setPercolate(set);
-                break;
             case Completion:
                 builder.setCompletion(set);
                 break;
@@ -963,8 +960,6 @@ public class IndexStatsIT extends ESIntegTestCase {
                 return response.getStore() != null;
             case Warmer:
                 return response.getWarmer() != null;
-            case PercolatorCache:
-                return response.getPercolatorCache() != null;
             case Completion:
                 return response.getCompletion() != null;
             case Segments:
diff --git a/core/src/test/java/org/elasticsearch/percolator/PercolatorIT.java b/core/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
index 77b67227401..c489025a943 100644
--- a/core/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
+++ b/core/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
@@ -514,38 +514,6 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(percolate.getMatches(), emptyArray());
     }
 
-    public void testPercolateStatistics() throws Exception {
-        client().admin().indices().prepareCreate(INDEX_NAME)
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .get();
-        client().admin().indices().prepareCreate("test2")
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .get();
-        ensureGreen();
-
-        logger.info("--> register a query");
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "1")
-                .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
-                .execute().actionGet();
-        client().prepareIndex("test2", TYPE_NAME, "1")
-                .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
-                .execute().actionGet();
-        refresh();
-
-        logger.info("--> First percolate request");
-        PercolateResponse response = client().preparePercolate()
-                .setIndices(INDEX_NAME).setDocumentType("type")
-                .setSource(jsonBuilder().startObject().startObject("doc").field("field", "val").endObject().endObject())
-                .execute().actionGet();
-        assertMatchCount(response, 1L);
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContaining("1"));
-
-        NumShards numShards = getNumShards(INDEX_NAME);
-
-        IndicesStatsResponse indicesResponse = client().admin().indices().prepareStats(INDEX_NAME).execute().actionGet();
-        assertThat(indicesResponse.getTotal().getPercolatorCache().getNumQueries(), equalTo((long)numShards.dataCopies)); // number of copies
-    }
-
     public void testPercolatingExistingDocs() throws Exception {
         client().admin().indices().prepareCreate(INDEX_NAME)
                 .addMapping(TYPE_NAME, "query", "type=percolator")
diff --git a/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java b/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java
index d5c6bf98835..9ebbb5b42e0 100644
--- a/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java
@@ -289,7 +289,7 @@ public class HighlightBuilderTests extends ESTestCase {
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings);
         // shard context will only need indicesQueriesRegistry for building Query objects nested in highlighter
         QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, null, null, indicesQueriesRegistry,
-                null, null, null, null) {
+                null, null, null) {
             @Override
             public MappedFieldType fieldMapper(String name) {
                 TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name);
diff --git a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
index d0f2c0492da..5cc59046433 100644
--- a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
@@ -158,7 +158,7 @@ public class QueryRescoreBuilderTests extends ESTestCase {
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAsciiOfLengthBetween(1, 10), indexSettings);
         // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer
         QueryShardContext mockShardContext = new QueryShardContext(idxSettings, null, null, null, null, null, indicesQueriesRegistry,
-                null, null, null, null) {
+                null, null, null) {
             @Override
             public MappedFieldType fieldMapper(String name) {
                 TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name);
diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
index 21b71508bbb..e964b975bb2 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
@@ -232,7 +232,7 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST
             }
         });
         return new QueryShardContext(idxSettings, bitsetFilterCache, ifds, null, null, scriptService,
-                indicesQueriesRegistry, null, null, null, null) {
+                indicesQueriesRegistry, null, null, null) {
             @Override
             public MappedFieldType fieldMapper(String name) {
                 return provideMappedFieldType(name);
diff --git a/docs/reference/migration/migrate_5_0/percolator.asciidoc b/docs/reference/migration/migrate_5_0/percolator.asciidoc
index 7a3fa97fe92..d3b1ccb97a3 100644
--- a/docs/reference/migration/migrate_5_0/percolator.asciidoc
+++ b/docs/reference/migration/migrate_5_0/percolator.asciidoc
@@ -46,4 +46,4 @@ the existing document.
 
 ==== Percolate Stats
 
-Percolate stats have been replaced with `percolate` query cache stats in nodes stats and cluster stats APIs.
\ No newline at end of file
+The percolate stats have been removed. This is because the percolator no longer caches the percolator queries.
\ No newline at end of file
diff --git a/docs/reference/query-dsl/percolate-query.asciidoc b/docs/reference/query-dsl/percolate-query.asciidoc
index 267647c9dbb..48ba500c98d 100644
--- a/docs/reference/query-dsl/percolate-query.asciidoc
+++ b/docs/reference/query-dsl/percolate-query.asciidoc
@@ -327,13 +327,10 @@ the document defined in the `percolate` query.
 ==== How it Works Under the Hood
 
 When indexing a document into an index that has the <<percolator,percolator field type>> mapping configured, the query
-part of the documents gets parsed into a Lucene query and is kept in memory until that percolator document is removed.
-So, all the active percolator queries are kept in memory.
+part of the documents gets parsed into a Lucene query and are stored into the Lucene index. A binary representation
+of the query gets stored, but also the query's terms are analyzed and stored into an indexed field.
 
 At search time, the document specified in the request gets parsed into a Lucene document and is stored in a in-memory
-temporary Lucene index. This in-memory index can just hold this one document and it is optimized for that. Then all the queries
-that are registered to the index that the search request is targeted for, are going to be executed on this single document
-in-memory index. This happens on each shard the search request needs to execute.
-
-By using `routing` or additional queries the amount of percolator queries that need to be executed can be reduced and thus
-the time the search API needs to run can be decreased.
\ No newline at end of file
+temporary Lucene index. This in-memory index can just hold this one document and it is optimized for that. After this
+a special query is build based on the terms in the in-memory index that select candidate percolator queries based on
+their indexed query terms. These queries are then evaluated by the in-memory index if they actually match.
\ No newline at end of file
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
index 139d8dbf0b8..1bdb87778f5 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
@@ -153,7 +153,7 @@ public class TemplateQueryParserTests extends ESTestCase {
         });
         IndicesQueriesRegistry indicesQueriesRegistry = injector.getInstance(IndicesQueriesRegistry.class);
         contextFactory =  () -> new QueryShardContext(idxSettings, bitsetFilterCache, indexFieldDataService, mapperService,
-                similarityService, scriptService, indicesQueriesRegistry, proxy, null, null, null);
+                similarityService, scriptService, indicesQueriesRegistry, proxy, null, null);
     }
 
     @Override
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml
index dfafd833509..97ffae1f802 100755
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml
@@ -48,7 +48,6 @@
                     merges.total_docs                .+   \n
                     merges.total_size                .+   \n
                     merges.total_time                .+   \n
-                    percolate.queries                .+   \n
                     refresh.total                    .+   \n
                     refresh.time                     .+   \n
                     search.fetch_current             .+   \n
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
index 5cac904df35..d8ea1f60bbd 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
@@ -38,7 +38,6 @@ import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.object.ObjectMapper;
-import org.elasticsearch.index.percolator.PercolatorQueryCache;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.IndexShard;
@@ -73,7 +72,6 @@ public class TestSearchContext extends SearchContext {
     final IndexService indexService;
     final IndexFieldDataService indexFieldDataService;
     final BitsetFilterCache fixedBitSetFilterCache;
-    final PercolatorQueryCache percolatorQueryCache;
     final ThreadPool threadPool;
     final Map<Class<?>, Collector> queryCollectors = new HashMap<>();
     final IndexShard indexShard;
@@ -101,7 +99,6 @@ public class TestSearchContext extends SearchContext {
         this.indexService = indexService;
         this.indexFieldDataService = indexService.fieldData();
         this.fixedBitSetFilterCache = indexService.cache().bitsetFilterCache();
-        this.percolatorQueryCache = indexService.cache().getPercolatorQueryCache();
         this.threadPool = threadPool;
         this.indexShard = indexService.getShardOrNull(0);
         this.scriptService = scriptService;
@@ -116,7 +113,6 @@ public class TestSearchContext extends SearchContext {
         this.indexFieldDataService = null;
         this.threadPool = null;
         this.fixedBitSetFilterCache = null;
-        this.percolatorQueryCache = null;
         this.indexShard = null;
         scriptService = null;
         this.queryShardContext = queryShardContext;
@@ -323,11 +319,6 @@ public class TestSearchContext extends SearchContext {
         return indexFieldDataService;
     }
 
-    @Override
-    public PercolatorQueryCache percolatorQueryCache() {
-        return percolatorQueryCache;
-    }
-
     @Override
     public long timeoutInMillis() {
         return 0;
diff --git a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java
index ee567aae779..d6cd3eea5ac 100644
--- a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java
@@ -34,7 +34,7 @@ import org.elasticsearch.test.TestSearchContext;
 public class MockSearchServiceTests extends ESTestCase {
     public void testAssertNoInFlightContext() {
         SearchContext s = new TestSearchContext(new QueryShardContext(new IndexSettings(IndexMetaData.PROTO, Settings.EMPTY), null, null,
-                null, null, null, null, null, null, null, null)) {
+                null, null, null, null, null, null, null)) {
             @Override
             public SearchShardTarget shardTarget() {
                 return new SearchShardTarget("node", new Index("idx", "ignored"), 0);

From 140f9dfe5f660aac896d15c4b977ee29a91e1b0d Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 20 May 2016 08:54:13 -0400
Subject: [PATCH 0383/1311] Fix scaling thread pool test bug

This commit fixes a test bug in the scaling thread pool configuration
test. In particular, the test randomization could select min and max for
a thread pool configuration where both are equal to zero. This is a
violation of the requirements of the ThreadPoolExecutor. With this
commit, we now ensure that the max is bounded below by one.
---
 .../org/elasticsearch/threadpool/ScalingThreadPoolTests.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java b/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
index 94d6d075589..86223ccd1fc 100644
--- a/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
+++ b/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
@@ -59,7 +59,7 @@ public class ScalingThreadPoolTests extends ESThreadPoolTestCase {
 
         final int expectedSize;
         if (sizeBasedOnNumberOfProcessors < min || randomBoolean()) {
-            expectedSize = randomIntBetween(min, 16);
+            expectedSize = randomIntBetween(Math.max(1, min), 16);
             builder.put("threadpool." + threadPoolName + ".size", expectedSize);
         }  else {
             expectedSize = sizeBasedOnNumberOfProcessors;

From 8b962bb234cd9d57b42d85024b175ee52378713d Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@elastic.co>
Date: Fri, 20 May 2016 15:12:52 +0200
Subject: [PATCH 0384/1311] Increase log level for NettyHttpRequestSizeLimitIT

This test fails spuriosly in CI and is not reproducible locally.

With this commit we temporarily increase the log level in a few
packages that are suspected to reveal the cause.
---
 .../elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java   | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java
index 632df308e99..eeda96743df 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java
@@ -29,6 +29,7 @@ import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.jboss.netty.handler.codec.http.HttpResponse;
 import org.jboss.netty.handler.codec.http.HttpResponseStatus;
 
@@ -53,6 +54,7 @@ public class NettyHttpRequestSizeLimitIT extends ESIntegTestCase {
             .build();
     }
 
+    @TestLogging("_root:DEBUG,org.elasticsearch.common.breaker:TRACE,org.elasticsearch.test:TRACE,org.elasticsearch.transport:TRACE")
     public void testLimitsInFlightRequests() throws Exception {
         ensureGreen();
 

From ce9b36590aa884c9bcabe183a974175d41901314 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Fri, 20 May 2016 09:36:09 -0400
Subject: [PATCH 0385/1311] simplify tree node use of compiler settings

---
 .../org/elasticsearch/painless/Analyzer.java  |   2 +-
 .../org/elasticsearch/painless/Compiler.java  |   2 +-
 .../org/elasticsearch/painless/Writer.java    |   2 +-
 .../elasticsearch/painless/antlr/Walker.java  |  15 +-
 .../painless/node/AExpression.java            |  13 +-
 .../elasticsearch/painless/node/ALink.java    |  13 +-
 .../painless/node/AStatement.java             |   7 +-
 .../elasticsearch/painless/node/EBinary.java  | 147 +++++++++---------
 .../elasticsearch/painless/node/EBool.java    |  31 ++--
 .../elasticsearch/painless/node/EBoolean.java |   7 +-
 .../elasticsearch/painless/node/ECast.java    |   9 +-
 .../elasticsearch/painless/node/EChain.java   |  53 ++++---
 .../elasticsearch/painless/node/EComp.java    | 107 +++++++------
 .../painless/node/EConditional.java           |  26 ++--
 .../painless/node/EConstant.java              |   7 +-
 .../elasticsearch/painless/node/EDecimal.java |   7 +-
 .../painless/node/EExplicit.java              |  15 +-
 .../elasticsearch/painless/node/ENull.java    |   7 +-
 .../elasticsearch/painless/node/ENumeric.java |   7 +-
 .../elasticsearch/painless/node/EUnary.java   |  45 +++---
 .../painless/node/LArrayLength.java           |  11 +-
 .../elasticsearch/painless/node/LBrace.java   |  23 ++-
 .../elasticsearch/painless/node/LCall.java    |  19 ++-
 .../elasticsearch/painless/node/LCast.java    |  11 +-
 .../painless/node/LDefArray.java              |  17 +-
 .../elasticsearch/painless/node/LDefCall.java |  17 +-
 .../painless/node/LDefField.java              |  11 +-
 .../elasticsearch/painless/node/LField.java   |  23 ++-
 .../painless/node/LListShortcut.java          |  17 +-
 .../painless/node/LMapShortcut.java           |  17 +-
 .../painless/node/LNewArray.java              |  17 +-
 .../elasticsearch/painless/node/LNewObj.java  |  19 ++-
 .../painless/node/LShortcut.java              |  11 +-
 .../elasticsearch/painless/node/LString.java  |  11 +-
 .../painless/node/LVariable.java              |  11 +-
 .../elasticsearch/painless/node/SBlock.java   |  15 +-
 .../elasticsearch/painless/node/SBreak.java   |   7 +-
 .../painless/node/SContinue.java              |   7 +-
 .../painless/node/SDeclBlock.java             |  11 +-
 .../painless/node/SDeclaration.java           |  13 +-
 .../org/elasticsearch/painless/node/SDo.java  |  21 +--
 .../painless/node/SExpression.java            |  13 +-
 .../org/elasticsearch/painless/node/SFor.java |  35 +++--
 .../elasticsearch/painless/node/SIfElse.java  |  22 ++-
 .../elasticsearch/painless/node/SReturn.java  |  13 +-
 .../elasticsearch/painless/node/SSource.java  |  11 +-
 .../elasticsearch/painless/node/SThrow.java   |  13 +-
 .../elasticsearch/painless/node/STrap.java    |  11 +-
 .../org/elasticsearch/painless/node/STry.java |  15 +-
 .../elasticsearch/painless/node/SWhile.java   |  21 +--
 50 files changed, 473 insertions(+), 512 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
index 2cd62b4c3d4..e81e828c875 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
@@ -29,7 +29,7 @@ final class Analyzer {
     static Variables analyze(final CompilerSettings settings,
                                     final Reserved shortcut, final SSource root) {
         final Variables variables = new Variables(settings, shortcut);
-        root.analyze(settings, variables);
+        root.analyze(variables);
 
         return variables;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
index 65d7c78555c..16ad355177f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
@@ -112,7 +112,7 @@ final class Compiler {
         }
 
         final Reserved reserved = new Reserved();
-        final SSource root = Walker.buildPainlessTree(source, reserved);
+        final SSource root = Walker.buildPainlessTree(source, reserved, settings);
         final Variables variables = Analyzer.analyze(settings, reserved, root);
 
         return Writer.write(settings, name, source, variables, root);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
index 977a591b7af..57103d53764 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
@@ -175,7 +175,7 @@ final class Writer {
             adapter.visitVarInsn(Opcodes.ISTORE, loop.slot);
         }
 
-        root.write(settings, adapter);
+        root.write(adapter);
         adapter.endMethod();
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index 4f6e2f5e87c..9ecf6754248 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -22,6 +22,7 @@ package org.elasticsearch.painless.antlr;
 import org.antlr.v4.runtime.ANTLRInputStream;
 import org.antlr.v4.runtime.CommonTokenStream;
 import org.antlr.v4.runtime.ParserRuleContext;
+import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Variables.Reserved;
 import org.elasticsearch.painless.antlr.PainlessParser.AfterthoughtContext;
@@ -121,15 +122,17 @@ import java.util.List;
  */
 public final class Walker extends PainlessParserBaseVisitor<ANode> {
 
-    public static SSource buildPainlessTree(final String source, final Reserved reserved) {
-        return new Walker(source, reserved).source;
+    public static SSource buildPainlessTree(String source, Reserved reserved, CompilerSettings settings) {
+        return new Walker(source, reserved, settings).source;
     }
 
     private final Reserved reserved;
     private final SSource source;
+    private final CompilerSettings settings;
 
-    private Walker(final String source, final Reserved reserved) {
+    private Walker(String source, Reserved reserved, CompilerSettings settings) {
         this.reserved = reserved;
+        this.settings = settings;
         this.source = (SSource)visit(buildAntlrTree(source));
     }
 
@@ -181,7 +184,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
 
         reserved.usesLoop();
 
-        return new SWhile(line(ctx), location(ctx), condition, block);
+        return new SWhile(line(ctx), location(ctx), condition, block, settings.getMaxLoopCounter());
     }
 
     @Override
@@ -191,7 +194,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
 
         reserved.usesLoop();
 
-        return new SDo(line(ctx), location(ctx), block, condition);
+        return new SDo(line(ctx), location(ctx), block, condition, settings.getMaxLoopCounter());
     }
 
     @Override
@@ -203,7 +206,7 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
 
         reserved.usesLoop();
 
-        return new SFor(line(ctx), location(ctx), intializer, condition, afterthought, block);
+        return new SFor(line(ctx), location(ctx), intializer, condition, afterthought, block, settings.getMaxLoopCounter());
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
index f741d62921f..110b5a54189 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.AnalyzerCaster;
@@ -100,19 +99,19 @@ public abstract class AExpression extends ANode {
     /**
      * Checks for errors and collects data for the writing phase.
      */
-    abstract void analyze(final CompilerSettings settings, final Variables variables);
+    abstract void analyze(Variables variables);
 
     /**
      * Writes ASM based on the data collected during the analysis phase.
      */
-    abstract void write(final CompilerSettings settings, final MethodWriter adapter);
+    abstract void write(MethodWriter adapter);
 
     /**
      * Inserts {@link ECast} nodes into the tree for implicit casts.  Also replaces
      * nodes with the constant variable set to a non-null value with {@link EConstant}.
      * @return The new child node for the parent node calling this method.
      */
-    AExpression cast(final CompilerSettings settings, final Variables variables) {
+    AExpression cast(Variables variables) {
         final Cast cast = AnalyzerCaster.getLegalCast(location, actual, expected, explicit);
 
         if (cast == null) {
@@ -120,7 +119,7 @@ public abstract class AExpression extends ANode {
                 return this;
             } else {
                 final EConstant econstant = new EConstant(line, location, constant);
-                econstant.analyze(settings, variables);
+                econstant.analyze(variables);
 
                 if (!expected.equals(econstant.actual)) {
                     throw new IllegalStateException(error("Illegal tree structure."));
@@ -141,7 +140,7 @@ public abstract class AExpression extends ANode {
                     constant = AnalyzerCaster.constCast(location, constant, cast);
 
                     final EConstant econstant = new EConstant(line, location, constant);
-                    econstant.analyze(settings, variables);
+                    econstant.analyze(variables);
 
                     if (!expected.equals(econstant.actual)) {
                         throw new IllegalStateException(error("Illegal tree structure."));
@@ -155,7 +154,7 @@ public abstract class AExpression extends ANode {
                     return ecast;
                 } else {
                     final EConstant econstant = new EConstant(line, location, constant);
-                    econstant.analyze(settings, variables);
+                    econstant.analyze(variables);
 
                     if (!actual.equals(econstant.actual)) {
                         throw new IllegalStateException(error("Illegal tree structure."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
index 8f8e226006e..e134e7fbcb9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
@@ -74,7 +73,7 @@ public abstract class ALink extends ANode {
      */
     String string = null;
 
-    ALink(final int line, final String location, final int size) {
+    ALink(int line, String location, int size) {
         super(line, location);
 
         this.size = size;
@@ -86,27 +85,27 @@ public abstract class ALink extends ANode {
      * def or a shortcut is used. Otherwise, returns itself.  This will be
      * updated into the {@link EChain} node's list of links.
      */
-    abstract ALink analyze(final CompilerSettings settings, final Variables variables);
+    abstract ALink analyze(Variables variables);
 
     /**
      * Write values before a load/store occurs such as an array index.
      */
-    abstract void write(final CompilerSettings settings, final MethodWriter adapter);
+    abstract void write(MethodWriter adapter);
 
     /**
      * Write a load for the specific link type.
      */
-    abstract void load(final CompilerSettings settings, final MethodWriter adapter);
+    abstract void load(MethodWriter adapter);
 
     /**
      * Write a store for the specific link type.
      */
-    abstract void store(final CompilerSettings settings, final MethodWriter adapter);
+    abstract void store(MethodWriter adapter);
 
     /**
      * Used to copy link data from one to another during analysis in the case of replacement.
      */
-    final ALink copy(final ALink link) {
+    final ALink copy(ALink link) {
         load       = link.load;
         store      = link.store;
         statik     = link.statik;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
index f80d5e7b4fd..ebc4a166268 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.MethodWriter;
@@ -108,17 +107,17 @@ public abstract class AStatement extends ANode {
      */
     Label brake = null;
 
-    AStatement(final int line, final String location) {
+    AStatement(int line, String location) {
         super(line, location);
     }
 
     /**
      * Checks for errors and collects data for the writing phase.
      */
-    abstract void analyze(final CompilerSettings settings, final Variables variables);
+    abstract void analyze(Variables variables);
 
     /**
      * Writes ASM based on the data collected during the analysis phase.
      */
-    abstract void write(final CompilerSettings settings, final MethodWriter adapter);
+    abstract void write(MethodWriter adapter);
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 0046d498074..b94ef09c042 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.AnalyzerCaster;
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
@@ -39,7 +38,7 @@ public final class EBinary extends AExpression {
 
     boolean cat = false;
 
-    public EBinary(final int line, final String location, final Operation operation, final AExpression left, final AExpression right) {
+    public EBinary(int line, String location, Operation operation, AExpression left, AExpression right) {
         super(line, location);
 
         this.operation = operation;
@@ -48,37 +47,37 @@ public final class EBinary extends AExpression {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         if (operation == Operation.MUL) {
-            analyzeMul(settings, variables);
+            analyzeMul(variables);
         } else if (operation == Operation.DIV) {
-            analyzeDiv(settings, variables);
+            analyzeDiv(variables);
         } else if (operation == Operation.REM) {
-            analyzeRem(settings, variables);
+            analyzeRem(variables);
         } else if (operation == Operation.ADD) {
-            analyzeAdd(settings, variables);
+            analyzeAdd(variables);
         } else if (operation == Operation.SUB) {
-            analyzeSub(settings, variables);
+            analyzeSub(variables);
         } else if (operation == Operation.LSH) {
-            analyzeLSH(settings, variables);
+            analyzeLSH(variables);
         } else if (operation == Operation.RSH) {
-            analyzeRSH(settings, variables);
+            analyzeRSH(variables);
         } else if (operation == Operation.USH) {
-            analyzeUSH(settings, variables);
+            analyzeUSH(variables);
         } else if (operation == Operation.BWAND) {
-            analyzeBWAnd(settings, variables);
+            analyzeBWAnd(variables);
         } else if (operation == Operation.XOR) {
-            analyzeXor(settings, variables);
+            analyzeXor(variables);
         } else if (operation == Operation.BWOR) {
-            analyzeBWOr(settings, variables);
+            analyzeBWOr(variables);
         } else {
             throw new IllegalStateException(error("Illegal tree structure."));
         }
     }
 
-    private void analyzeMul(final CompilerSettings settings, final Variables variables) {
-        left.analyze(settings, variables);
-        right.analyze(settings, variables);
+    private void analyzeMul(Variables variables) {
+        left.analyze(variables);
+        right.analyze(variables);
 
         final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true, true);
 
@@ -90,8 +89,8 @@ public final class EBinary extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, variables);
-        right = right.cast(settings, variables);
+        left = left.cast(variables);
+        right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -112,9 +111,9 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeDiv(final CompilerSettings settings, final Variables variables) {
-        left.analyze(settings, variables);
-        right.analyze(settings, variables);
+    private void analyzeDiv(Variables variables) {
+        left.analyze(variables);
+        right.analyze(variables);
 
         final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true, true);
 
@@ -126,8 +125,8 @@ public final class EBinary extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, variables);
-        right = right.cast(settings, variables);
+        left = left.cast(variables);
+        right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -148,9 +147,9 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeRem(final CompilerSettings settings, final Variables variables) {
-        left.analyze(settings, variables);
-        right.analyze(settings, variables);
+    private void analyzeRem(Variables variables) {
+        left.analyze(variables);
+        right.analyze(variables);
 
         final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true, true);
 
@@ -162,8 +161,8 @@ public final class EBinary extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, variables);
-        right = right.cast(settings, variables);
+        left = left.cast(variables);
+        right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -184,9 +183,9 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeAdd(final CompilerSettings settings, final Variables variables) {
-        left.analyze(settings, variables);
-        right.analyze(settings, variables);
+    private void analyzeAdd(Variables variables) {
+        left.analyze(variables);
+        right.analyze(variables);
 
         final Type promote = AnalyzerCaster.promoteAdd(left.actual, right.actual);
 
@@ -214,8 +213,8 @@ public final class EBinary extends AExpression {
             right.expected = promote;
         }
 
-        left = left.cast(settings, variables);
-        right = right.cast(settings, variables);
+        left = left.cast(variables);
+        right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
             if (sort == Sort.INT) {
@@ -236,9 +235,9 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeSub(final CompilerSettings settings, final Variables variables) {
-        left.analyze(settings, variables);
-        right.analyze(settings, variables);
+    private void analyzeSub(Variables variables) {
+        left.analyze(variables);
+        right.analyze(variables);
 
         final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true, true);
 
@@ -250,8 +249,8 @@ public final class EBinary extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, variables);
-        right = right.cast(settings, variables);
+        left = left.cast(variables);
+        right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -272,9 +271,9 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeLSH(final CompilerSettings settings, final Variables variables) {
-        left.analyze(settings, variables);
-        right.analyze(settings, variables);
+    private void analyzeLSH(Variables variables) {
+        left.analyze(variables);
+        right.analyze(variables);
 
         final Type promote = AnalyzerCaster.promoteNumeric(left.actual, false, true);
 
@@ -287,8 +286,8 @@ public final class EBinary extends AExpression {
         right.expected = Definition.INT_TYPE;
         right.explicit = true;
 
-        left = left.cast(settings, variables);
-        right = right.cast(settings, variables);
+        left = left.cast(variables);
+        right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -305,9 +304,9 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeRSH(final CompilerSettings settings, final Variables variables) {
-        left.analyze(settings, variables);
-        right.analyze(settings, variables);
+    private void analyzeRSH(Variables variables) {
+        left.analyze(variables);
+        right.analyze(variables);
 
         final Type promote = AnalyzerCaster.promoteNumeric(left.actual, false, true);
 
@@ -320,8 +319,8 @@ public final class EBinary extends AExpression {
         right.expected = Definition.INT_TYPE;
         right.explicit = true;
 
-        left = left.cast(settings, variables);
-        right = right.cast(settings, variables);
+        left = left.cast(variables);
+        right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -338,9 +337,9 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeUSH(final CompilerSettings settings, final Variables variables) {
-        left.analyze(settings, variables);
-        right.analyze(settings, variables);
+    private void analyzeUSH(Variables variables) {
+        left.analyze(variables);
+        right.analyze(variables);
 
         final Type promote = AnalyzerCaster.promoteNumeric(left.actual, false, true);
 
@@ -353,8 +352,8 @@ public final class EBinary extends AExpression {
         right.expected = Definition.INT_TYPE;
         right.explicit = true;
 
-        left = left.cast(settings, variables);
-        right = right.cast(settings, variables);
+        left = left.cast(variables);
+        right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -371,9 +370,9 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeBWAnd(final CompilerSettings settings, final Variables variables) {
-        left.analyze(settings, variables);
-        right.analyze(settings, variables);
+    private void analyzeBWAnd(Variables variables) {
+        left.analyze(variables);
+        right.analyze(variables);
 
         final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false, true);
 
@@ -385,8 +384,8 @@ public final class EBinary extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, variables);
-        right = right.cast(settings, variables);
+        left = left.cast(variables);
+        right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -403,9 +402,9 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeXor(final CompilerSettings settings, final Variables variables) {
-        left.analyze(settings, variables);
-        right.analyze(settings, variables);
+    private void analyzeXor(Variables variables) {
+        left.analyze(variables);
+        right.analyze(variables);
 
         final Type promote = AnalyzerCaster.promoteXor(left.actual, right.actual);
 
@@ -417,8 +416,8 @@ public final class EBinary extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, variables);
-        right = right.cast(settings, variables);
+        left = left.cast(variables);
+        right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -437,9 +436,9 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeBWOr(final CompilerSettings settings, final Variables variables) {
-        left.analyze(settings, variables);
-        right.analyze(settings, variables);
+    private void analyzeBWOr(Variables variables) {
+        left.analyze(variables);
+        right.analyze(variables);
 
         final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false, true);
 
@@ -451,8 +450,8 @@ public final class EBinary extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, variables);
-        right = right.cast(settings, variables);
+        left = left.cast(variables);
+        right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -470,19 +469,19 @@ public final class EBinary extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         if (actual.sort == Sort.STRING && operation == Operation.ADD) {
             if (!cat) {
                 adapter.writeNewStrings();
             }
 
-            left.write(settings, adapter);
+            left.write(adapter);
 
             if (!(left instanceof EBinary) || ((EBinary)left).operation != Operation.ADD || left.actual.sort != Sort.STRING) {
                 adapter.writeAppendStrings(left.actual);
             }
 
-            right.write(settings, adapter);
+            right.write(adapter);
 
             if (!(right instanceof EBinary) || ((EBinary)right).operation != Operation.ADD || right.actual.sort != Sort.STRING) {
                 adapter.writeAppendStrings(right.actual);
@@ -492,8 +491,8 @@ public final class EBinary extends AExpression {
                 adapter.writeToStrings();
             }
         } else {
-            left.write(settings, adapter);
-            right.write(settings, adapter);
+            left.write(adapter);
+            right.write(adapter);
 
             adapter.writeBinaryInstruction(location, actual, operation);
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
index 1e2af41c0c0..27cd3018ded 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Variables;
@@ -35,7 +34,7 @@ public final class EBool extends AExpression {
     AExpression left;
     AExpression right;
 
-    public EBool(final int line, final String location, final Operation operation, final AExpression left, final AExpression right) {
+    public EBool(int line, String location, Operation operation, AExpression left, AExpression right) {
         super(line, location);
 
         this.operation = operation;
@@ -44,14 +43,14 @@ public final class EBool extends AExpression {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         left.expected = Definition.BOOLEAN_TYPE;
-        left.analyze(settings, variables);
-        left = left.cast(settings, variables);
+        left.analyze(variables);
+        left = left.cast(variables);
 
         right.expected = Definition.BOOLEAN_TYPE;
-        right.analyze(settings, variables);
-        right = right.cast(settings, variables);
+        right.analyze(variables);
+        right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
             if (operation == Operation.AND) {
@@ -67,7 +66,7 @@ public final class EBool extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         if (tru != null || fals != null) {
             if (operation == Operation.AND) {
                 final Label localfals = fals == null ? new Label() : fals;
@@ -76,8 +75,8 @@ public final class EBool extends AExpression {
                 right.tru = tru;
                 right.fals = fals;
 
-                left.write(settings, adapter);
-                right.write(settings, adapter);
+                left.write(adapter);
+                right.write(adapter);
 
                 if (fals == null) {
                     adapter.mark(localfals);
@@ -89,8 +88,8 @@ public final class EBool extends AExpression {
                 right.tru = tru;
                 right.fals = fals;
 
-                left.write(settings, adapter);
-                right.write(settings, adapter);
+                left.write(adapter);
+                right.write(adapter);
 
                 if (tru == null) {
                     adapter.mark(localtru);
@@ -106,8 +105,8 @@ public final class EBool extends AExpression {
                 left.fals = localfals;
                 right.fals = localfals;
 
-                left.write(settings, adapter);
-                right.write(settings, adapter);
+                left.write(adapter);
+                right.write(adapter);
 
                 adapter.push(true);
                 adapter.goTo(end);
@@ -122,8 +121,8 @@ public final class EBool extends AExpression {
                 left.tru = localtru;
                 right.fals = localfals;
 
-                left.write(settings, adapter);
-                right.write(settings, adapter);
+                left.write(adapter);
+                right.write(adapter);
 
                 adapter.mark(localtru);
                 adapter.push(true);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
index 2e019c576e0..27d7bb9a626 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
@@ -29,19 +28,19 @@ import org.elasticsearch.painless.MethodWriter;
  */
 public final class EBoolean extends AExpression {
 
-    public EBoolean(final int line, final String location, final boolean constant) {
+    public EBoolean(int line, String location, boolean constant) {
         super(line, location);
 
         this.constant = constant;
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         actual = Definition.BOOLEAN_TYPE;
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         throw new IllegalArgumentException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
index c46957f78d9..7a8c4a29b60 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
@@ -35,7 +34,7 @@ final class ECast extends AExpression {
 
     Cast cast = null;
 
-    ECast(final int line, final String location, final AExpression child, final Cast cast) {
+    ECast(int line, String location, AExpression child, Cast cast) {
         super(line, location);
 
         this.type = null;
@@ -45,13 +44,13 @@ final class ECast extends AExpression {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
-        child.write(settings, adapter);
+    void write(MethodWriter adapter) {
+        child.write(adapter);
         adapter.writeCast(cast);
         adapter.writeBranch(tru, fals);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index 2f8ad66f101..8f2d3812943 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Definition.Sort;
@@ -47,8 +46,8 @@ public final class EChain extends AExpression {
     Cast there = null;
     Cast back = null;
 
-    public EChain(final int line, final String location, final List<ALink> links,
-                  final boolean pre, final boolean post, final Operation operation, final AExpression expression) {
+    public EChain(int line, String location, List<ALink> links,
+                  boolean pre, boolean post, Operation operation, AExpression expression) {
         super(line, location);
 
         this.links = links;
@@ -59,20 +58,20 @@ public final class EChain extends AExpression {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
-        analyzeLinks(settings, variables);
+    void analyze(Variables variables) {
+        analyzeLinks(variables);
         analyzeIncrDecr();
 
         if (operation != null) {
-            analyzeCompound(settings, variables);
+            analyzeCompound(variables);
         } else if (expression != null) {
-            analyzeWrite(settings, variables);
+            analyzeWrite(variables);
         } else {
             analyzeRead();
         }
     }
 
-    private void analyzeLinks(final CompilerSettings settings, final Variables variables) {
+    private void analyzeLinks(Variables variables) {
         ALink previous = null;
         int index = 0;
 
@@ -92,7 +91,7 @@ public final class EChain extends AExpression {
                 current.store = expression != null || pre || post;
             }
 
-            final ALink analyzed = current.analyze(settings, variables);
+            final ALink analyzed = current.analyze(variables);
 
             if (analyzed == null) {
                 links.remove(index);
@@ -153,10 +152,10 @@ public final class EChain extends AExpression {
         }
     }
 
-    private void analyzeCompound(final CompilerSettings settings, final Variables variables) {
+    private void analyzeCompound(Variables variables) {
         final ALink last = links.get(links.size() - 1);
 
-        expression.analyze(settings, variables);
+        expression.analyze(variables);
 
         if (operation == Operation.MUL) {
             promote = AnalyzerCaster.promoteNumeric(last.after, expression.actual, true, true);
@@ -205,7 +204,7 @@ public final class EChain extends AExpression {
             expression.expected = promote;
         }
 
-        expression = expression.cast(settings, variables);
+        expression = expression.cast(variables);
 
         there = AnalyzerCaster.getLegalCast(location, last.after, promote, false);
         back = AnalyzerCaster.getLegalCast(location, promote, last.after, true);
@@ -214,21 +213,21 @@ public final class EChain extends AExpression {
         this.actual = read ? last.after : Definition.VOID_TYPE;
     }
 
-    private void analyzeWrite(final CompilerSettings settings, final Variables variables) {
+    private void analyzeWrite(Variables variables) {
         final ALink last = links.get(links.size() - 1);
 
         // If the store node is a DEF node, we remove the cast to DEF from the expression
         // and promote the real type to it:
         if (last instanceof IDefLink) {
-            expression.analyze(settings, variables);
+            expression.analyze(variables);
             last.after = expression.expected = expression.actual;
         } else {
             // otherwise we adapt the type of the expression to the store type
             expression.expected = last.after;
-            expression.analyze(settings, variables);
+            expression.analyze(variables);
         }
 
-        expression = expression.cast(settings, variables);
+        expression = expression.cast(variables);
 
         this.statement = true;
         this.actual = read ? last.after : Definition.VOID_TYPE;
@@ -248,7 +247,7 @@ public final class EChain extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings,final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         if (cat) {
             adapter.writeNewStrings();
         }
@@ -256,15 +255,15 @@ public final class EChain extends AExpression {
         final ALink last = links.get(links.size() - 1);
 
         for (final ALink link : links) {
-            link.write(settings, adapter);
+            link.write(adapter);
 
             if (link == last && link.store) {
                 if (cat) {
                     adapter.writeDup(link.size, 1);
-                    link.load(settings, adapter);
+                    link.load(adapter);
                     adapter.writeAppendStrings(link.after);
 
-                    expression.write(settings, adapter);
+                    expression.write(adapter);
 
                     if (!(expression instanceof EBinary) ||
                         ((EBinary)expression).operation != Operation.ADD || expression.actual.sort != Sort.STRING) {
@@ -278,17 +277,17 @@ public final class EChain extends AExpression {
                         adapter.writeDup(link.after.sort.size, link.size);
                     }
 
-                    link.store(settings, adapter);
+                    link.store(adapter);
                 } else if (operation != null) {
                     adapter.writeDup(link.size, 0);
-                    link.load(settings, adapter);
+                    link.load(adapter);
 
                     if (link.load && post) {
                         adapter.writeDup(link.after.sort.size, link.size);
                     }
 
                     adapter.writeCast(there);
-                    expression.write(settings, adapter);
+                    expression.write(adapter);
                     adapter.writeBinaryInstruction(location, promote, operation);
 
                     adapter.writeCast(back);
@@ -297,18 +296,18 @@ public final class EChain extends AExpression {
                         adapter.writeDup(link.after.sort.size, link.size);
                     }
 
-                    link.store(settings, adapter);
+                    link.store(adapter);
                 } else {
-                    expression.write(settings, adapter);
+                    expression.write(adapter);
 
                     if (link.load) {
                         adapter.writeDup(link.after.sort.size, link.size);
                     }
 
-                    link.store(settings, adapter);
+                    link.store(adapter);
                 }
             } else {
-                link.load(settings, adapter);
+                link.load(adapter);
             }
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index 75dd1bb1f14..046ab1bfdf6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
@@ -45,7 +44,7 @@ public final class EComp extends AExpression {
     AExpression left;
     AExpression right;
 
-    public EComp(final int line, final String location, final Operation operation, final AExpression left, final AExpression right) {
+    public EComp(int line, String location, Operation operation, AExpression left, AExpression right) {
         super(line, location);
 
         this.operation = operation;
@@ -54,31 +53,31 @@ public final class EComp extends AExpression {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         if (operation == Operation.EQ) {
-            analyzeEq(settings, variables);
+            analyzeEq(variables);
         } else if (operation == Operation.EQR) {
-            analyzeEqR(settings, variables);
+            analyzeEqR(variables);
         } else if (operation == Operation.NE) {
-            analyzeNE(settings, variables);
+            analyzeNE(variables);
         } else if (operation == Operation.NER) {
-            analyzeNER(settings, variables);
+            analyzeNER(variables);
         } else if (operation == Operation.GTE) {
-            analyzeGTE(settings, variables);
+            analyzeGTE(variables);
         } else if (operation == Operation.GT) {
-            analyzeGT(settings, variables);
+            analyzeGT(variables);
         } else if (operation == Operation.LTE) {
-            analyzeLTE(settings, variables);
+            analyzeLTE(variables);
         } else if (operation == Operation.LT) {
-            analyzeLT(settings, variables);
+            analyzeLT(variables);
         } else {
             throw new IllegalStateException(error("Illegal tree structure."));
         }
     }
 
-    private void analyzeEq(final CompilerSettings settings, final Variables variables) {
-        left.analyze(settings, variables);
-        right.analyze(settings, variables);
+    private void analyzeEq(Variables variables) {
+        left.analyze(variables);
+        right.analyze(variables);
 
         final Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual);
 
@@ -90,8 +89,8 @@ public final class EComp extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, variables);
-        right = right.cast(settings, variables);
+        left = left.cast(variables);
+        right = right.cast(variables);
 
         if (left.isNull && right.isNull) {
             throw new IllegalArgumentException(error("Extraneous comparison of null constants."));
@@ -122,9 +121,9 @@ public final class EComp extends AExpression {
         actual = Definition.BOOLEAN_TYPE;
     }
 
-    private void analyzeEqR(final CompilerSettings settings, final Variables variables) {
-        left.analyze(settings, variables);
-        right.analyze(settings, variables);
+    private void analyzeEqR(Variables variables) {
+        left.analyze(variables);
+        right.analyze(variables);
 
         final Type promote = AnalyzerCaster.promoteReference(left.actual, right.actual);
 
@@ -136,8 +135,8 @@ public final class EComp extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, variables);
-        right = right.cast(settings, variables);
+        left = left.cast(variables);
+        right = right.cast(variables);
 
         if (left.isNull && right.isNull) {
             throw new IllegalArgumentException(error("Extraneous comparison of null constants."));
@@ -164,9 +163,9 @@ public final class EComp extends AExpression {
         actual = Definition.BOOLEAN_TYPE;
     }
 
-    private void analyzeNE(final CompilerSettings settings, final Variables variables) {
-        left.analyze(settings, variables);
-        right.analyze(settings, variables);
+    private void analyzeNE(Variables variables) {
+        left.analyze(variables);
+        right.analyze(variables);
 
         final Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual);
 
@@ -178,8 +177,8 @@ public final class EComp extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, variables);
-        right = right.cast(settings, variables);
+        left = left.cast(variables);
+        right = right.cast(variables);
 
         if (left.isNull && right.isNull) {
             throw new IllegalArgumentException(error("Extraneous comparison of null constants."));
@@ -210,9 +209,9 @@ public final class EComp extends AExpression {
         actual = Definition.BOOLEAN_TYPE;
     }
 
-    private void analyzeNER(final CompilerSettings settings, final Variables variables) {
-        left.analyze(settings, variables);
-        right.analyze(settings, variables);
+    private void analyzeNER(Variables variables) {
+        left.analyze(variables);
+        right.analyze(variables);
 
         final Type promote = AnalyzerCaster.promoteReference(left.actual, right.actual);
 
@@ -224,8 +223,8 @@ public final class EComp extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, variables);
-        right = right.cast(settings, variables);
+        left = left.cast(variables);
+        right = right.cast(variables);
 
         if (left.isNull && right.isNull) {
             throw new IllegalArgumentException(error("Extraneous comparison of null constants."));
@@ -252,9 +251,9 @@ public final class EComp extends AExpression {
         actual = Definition.BOOLEAN_TYPE;
     }
 
-    private void analyzeGTE(final CompilerSettings settings, final Variables variables) {
-        left.analyze(settings, variables);
-        right.analyze(settings, variables);
+    private void analyzeGTE(Variables variables) {
+        left.analyze(variables);
+        right.analyze(variables);
 
         final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true, true);
 
@@ -266,8 +265,8 @@ public final class EComp extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, variables);
-        right = right.cast(settings, variables);
+        left = left.cast(variables);
+        right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -288,9 +287,9 @@ public final class EComp extends AExpression {
         actual = Definition.BOOLEAN_TYPE;
     }
 
-    private void analyzeGT(final CompilerSettings settings, final Variables variables) {
-        left.analyze(settings, variables);
-        right.analyze(settings, variables);
+    private void analyzeGT(Variables variables) {
+        left.analyze(variables);
+        right.analyze(variables);
 
         final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true, true);
 
@@ -302,8 +301,8 @@ public final class EComp extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, variables);
-        right = right.cast(settings, variables);
+        left = left.cast(variables);
+        right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -324,9 +323,9 @@ public final class EComp extends AExpression {
         actual = Definition.BOOLEAN_TYPE;
     }
 
-    private void analyzeLTE(final CompilerSettings settings, final Variables variables) {
-        left.analyze(settings, variables);
-        right.analyze(settings, variables);
+    private void analyzeLTE(Variables variables) {
+        left.analyze(variables);
+        right.analyze(variables);
 
         final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true, true);
 
@@ -338,8 +337,8 @@ public final class EComp extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, variables);
-        right = right.cast(settings, variables);
+        left = left.cast(variables);
+        right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -360,9 +359,9 @@ public final class EComp extends AExpression {
         actual = Definition.BOOLEAN_TYPE;
     }
 
-    private void analyzeLT(final CompilerSettings settings, final Variables variables) {
-        left.analyze(settings, variables);
-        right.analyze(settings, variables);
+    private void analyzeLT(Variables variables) {
+        left.analyze(variables);
+        right.analyze(variables);
 
         final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true, true);
 
@@ -374,8 +373,8 @@ public final class EComp extends AExpression {
         left.expected = promote;
         right.expected = promote;
 
-        left = left.cast(settings, variables);
-        right = right.cast(settings, variables);
+        left = left.cast(variables);
+        right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
             final Sort sort = promote.sort;
@@ -397,15 +396,15 @@ public final class EComp extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         final boolean branch = tru != null || fals != null;
         final org.objectweb.asm.Type rtype = right.actual.type;
         final Sort rsort = right.actual.sort;
 
-        left.write(settings, adapter);
+        left.write(adapter);
 
         if (!right.isNull) {
-            right.write(settings, adapter);
+            right.write(adapter);
         }
 
         final Label jump = tru != null ? tru : fals != null ? fals : new Label();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
index c20391a9d41..bb2fc4d9d44 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.AnalyzerCaster;
@@ -36,8 +35,7 @@ public final class EConditional extends AExpression {
     AExpression left;
     AExpression right;
 
-    public EConditional(final int line, final String location,
-                        final AExpression condition, final AExpression left, final AExpression right) {
+    public EConditional(int line, String location, AExpression condition, AExpression left, AExpression right) {
         super(line, location);
 
         this.condition = condition;
@@ -46,10 +44,10 @@ public final class EConditional extends AExpression {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         condition.expected = Definition.BOOLEAN_TYPE;
-        condition.analyze(settings, variables);
-        condition = condition.cast(settings, variables);
+        condition.analyze(variables);
+        condition = condition.cast(variables);
 
         if (condition.constant != null) {
             throw new IllegalArgumentException(error("Extraneous conditional statement."));
@@ -61,8 +59,8 @@ public final class EConditional extends AExpression {
         right.explicit = explicit;
         actual = expected;
 
-        left.analyze(settings, variables);
-        right.analyze(settings, variables);
+        left.analyze(variables);
+        right.analyze(variables);
 
         if (expected == null) {
             final Type promote = AnalyzerCaster.promoteConditional(left.actual, right.actual, left.constant, right.constant);
@@ -72,12 +70,12 @@ public final class EConditional extends AExpression {
             actual = promote;
         }
 
-        left = left.cast(settings, variables);
-        right = right.cast(settings, variables);
+        left = left.cast(variables);
+        right = right.cast(variables);
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         final Label localfals = new Label();
         final Label end = new Label();
 
@@ -85,11 +83,11 @@ public final class EConditional extends AExpression {
         left.tru = right.tru = tru;
         left.fals = right.fals = fals;
 
-        condition.write(settings, adapter);
-        left.write(settings, adapter);
+        condition.write(adapter);
+        left.write(adapter);
         adapter.goTo(end);
         adapter.mark(localfals);
-        right.write(settings, adapter);
+        right.write(adapter);
         adapter.mark(end);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
index 6ee486c9470..ac18b849162 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Variables;
@@ -31,14 +30,14 @@ import org.elasticsearch.painless.MethodWriter;
  */
 final class EConstant extends AExpression {
 
-    EConstant(final int line, final String location, final Object constant) {
+    EConstant(int line, String location, Object constant) {
         super(line, location);
 
         this.constant = constant;
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         if (constant instanceof String) {
             actual = Definition.STRING_TYPE;
         } else if (constant instanceof Double) {
@@ -63,7 +62,7 @@ final class EConstant extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         final Sort sort = actual.sort;
 
         switch (sort) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
index 7c942dff32c..85e857da3c4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
@@ -31,14 +30,14 @@ public final class EDecimal extends AExpression {
 
     final String value;
 
-    public EDecimal(final int line, final String location, final String value) {
+    public EDecimal(int line, String location, String value) {
         super(line, location);
 
         this.value = value;
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         if (value.endsWith("f") || value.endsWith("F")) {
             try {
                 constant = Float.parseFloat(value.substring(0, value.length() - 1));
@@ -57,7 +56,7 @@ public final class EDecimal extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         throw new IllegalArgumentException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
index e0afd8f822a..c17d74f14b9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
@@ -32,7 +31,7 @@ public final class EExplicit extends AExpression {
     final String type;
     AExpression child;
 
-    public EExplicit(final int line, final String location, final String type, final AExpression child) {
+    public EExplicit(int line, String location, String type, AExpression child) {
         super(line, location);
 
         this.type = type;
@@ -40,7 +39,7 @@ public final class EExplicit extends AExpression {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         try {
             actual = Definition.getType(this.type);
         } catch (final IllegalArgumentException exception) {
@@ -49,19 +48,19 @@ public final class EExplicit extends AExpression {
 
         child.expected = actual;
         child.explicit = true;
-        child.analyze(settings, variables);
-        child = child.cast(settings, variables);
+        child.analyze(variables);
+        child = child.cast(variables);
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         throw new IllegalArgumentException(error("Illegal tree structure."));
     }
 
-    AExpression cast(final CompilerSettings settings, final Variables variables) {
+    AExpression cast(Variables variables) {
         child.expected = expected;
         child.explicit = explicit;
 
-        return child.cast(settings, variables);
+        return child.cast(variables);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
index c7c93951223..3a8005b20d6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Opcodes;
@@ -30,12 +29,12 @@ import org.elasticsearch.painless.MethodWriter;
  */
 public final class ENull extends AExpression {
 
-    public ENull(final int line, final String location) {
+    public ENull(int line, String location) {
         super(line, location);
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         isNull = true;
 
         if (expected != null) {
@@ -50,7 +49,7 @@ public final class ENull extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         adapter.visitInsn(Opcodes.ACONST_NULL);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
index 75f85703bc8..ee70bb77d44 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Variables;
@@ -33,7 +32,7 @@ public final class ENumeric extends AExpression {
     final String value;
     int radix;
 
-    public ENumeric(final int line, final String location, final String value, final int radix) {
+    public ENumeric(int line, String location, String value, int radix) {
         super(line, location);
 
         this.value = value;
@@ -41,7 +40,7 @@ public final class ENumeric extends AExpression {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         if (value.endsWith("d") || value.endsWith("D")) {
             if (radix != 10) {
                 throw new IllegalStateException(error("Invalid tree structure."));
@@ -96,7 +95,7 @@ public final class ENumeric extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         throw new IllegalArgumentException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index 115db1499f4..353dc8ee885 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
@@ -40,7 +39,7 @@ public final class EUnary extends AExpression {
     Operation operation;
     AExpression child;
 
-    public EUnary(final int line, final String location, final Operation operation, final AExpression child) {
+    public EUnary(int line, String location, Operation operation, AExpression child) {
         super(line, location);
 
         this.operation = operation;
@@ -48,24 +47,24 @@ public final class EUnary extends AExpression {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         if (operation == Operation.NOT) {
-            analyzeNot(settings, variables);
+            analyzeNot(variables);
         } else if (operation == Operation.BWNOT) {
-            analyzeBWNot(settings, variables);
+            analyzeBWNot(variables);
         } else if (operation == Operation.ADD) {
-            analyzerAdd(settings, variables);
+            analyzerAdd(variables);
         } else if (operation == Operation.SUB) {
-            analyzerSub(settings, variables);
+            analyzerSub(variables);
         } else {
             throw new IllegalStateException(error("Illegal tree structure."));
         }
     }
 
-    void analyzeNot(final CompilerSettings settings, final Variables variables) {
+    void analyzeNot(Variables variables) {
         child.expected = Definition.BOOLEAN_TYPE;
-        child.analyze(settings, variables);
-        child = child.cast(settings, variables);
+        child.analyze(variables);
+        child = child.cast(variables);
 
         if (child.constant != null) {
             constant = !(boolean)child.constant;
@@ -74,8 +73,8 @@ public final class EUnary extends AExpression {
         actual = Definition.BOOLEAN_TYPE;
     }
 
-    void analyzeBWNot(final CompilerSettings settings, final Variables variables) {
-        child.analyze(settings, variables);
+    void analyzeBWNot(Variables variables) {
+        child.analyze(variables);
 
         final Type promote = AnalyzerCaster.promoteNumeric(child.actual, false, true);
 
@@ -84,7 +83,7 @@ public final class EUnary extends AExpression {
         }
 
         child.expected = promote;
-        child = child.cast(settings, variables);
+        child = child.cast(variables);
 
         if (child.constant != null) {
             final Sort sort = promote.sort;
@@ -101,8 +100,8 @@ public final class EUnary extends AExpression {
         actual = promote;
     }
 
-    void analyzerAdd(final CompilerSettings settings, final Variables variables) {
-        child.analyze(settings, variables);
+    void analyzerAdd(Variables variables) {
+        child.analyze(variables);
 
         final Type promote = AnalyzerCaster.promoteNumeric(child.actual, true, true);
 
@@ -111,7 +110,7 @@ public final class EUnary extends AExpression {
         }
 
         child.expected = promote;
-        child = child.cast(settings, variables);
+        child = child.cast(variables);
 
         if (child.constant != null) {
             final Sort sort = promote.sort;
@@ -132,8 +131,8 @@ public final class EUnary extends AExpression {
         actual = promote;
     }
 
-    void analyzerSub(final CompilerSettings settings, final Variables variables) {
-        child.analyze(settings, variables);
+    void analyzerSub(Variables variables) {
+        child.analyze(variables);
 
         final Type promote = AnalyzerCaster.promoteNumeric(child.actual, true, true);
 
@@ -142,7 +141,7 @@ public final class EUnary extends AExpression {
         }
 
         child.expected = promote;
-        child = child.cast(settings, variables);
+        child = child.cast(variables);
 
         if (child.constant != null) {
             final Sort sort = promote.sort;
@@ -164,14 +163,14 @@ public final class EUnary extends AExpression {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         if (operation == Operation.NOT) {
             if (tru == null && fals == null) {
                 final Label localfals = new Label();
                 final Label end = new Label();
 
                 child.fals = localfals;
-                child.write(settings, adapter);
+                child.write(adapter);
 
                 adapter.push(false);
                 adapter.goTo(end);
@@ -181,13 +180,13 @@ public final class EUnary extends AExpression {
             } else {
                 child.tru = fals;
                 child.fals = tru;
-                child.write(settings, adapter);
+                child.write(adapter);
             }
         } else {
             final org.objectweb.asm.Type type = actual.type;
             final Sort sort = actual.sort;
 
-            child.write(settings, adapter);
+            child.write(adapter);
 
             if (operation == Operation.BWNOT) {
                 if (sort == Sort.DEF) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
index 4678e9766f3..4a6df96073e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
@@ -31,14 +30,14 @@ public final class LArrayLength extends ALink {
 
     final String value;
 
-    LArrayLength(final int line, final String location, final String value) {
+    LArrayLength(int line, String location, String value) {
         super(line, location, -1);
 
         this.value = value;
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Variables variables) {
+    ALink analyze(Variables variables) {
         if ("length".equals(value)) {
             if (!load) {
                 throw new IllegalArgumentException(error("Must read array field [length]."));
@@ -55,17 +54,17 @@ public final class LArrayLength extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final MethodWriter adapter) {
+    void load(MethodWriter adapter) {
         adapter.arrayLength();
     }
 
     @Override
-    void store(final CompilerSettings settings, final MethodWriter adapter) {
+    void store(MethodWriter adapter) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
index 1a1d4b5927e..95cc02602f6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Variables;
@@ -35,14 +34,14 @@ public final class LBrace extends ALink {
 
     AExpression index;
 
-    public LBrace(final int line, final String location, final AExpression index) {
+    public LBrace(int line, String location, AExpression index) {
         super(line, location, 2);
 
         this.index = index;
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Variables variables) {
+    ALink analyze(Variables variables) {
         if (before == null) {
             throw new IllegalStateException(error("Illegal tree structure."));
         }
@@ -51,35 +50,35 @@ public final class LBrace extends ALink {
 
         if (sort == Sort.ARRAY) {
             index.expected = Definition.INT_TYPE;
-            index.analyze(settings, variables);
-            index = index.cast(settings, variables);
+            index.analyze(variables);
+            index = index.cast(variables);
 
             after = Definition.getType(before.struct, before.dimensions - 1);
 
             return this;
         } else if (sort == Sort.DEF) {
-            return new LDefArray(line, location, index).copy(this).analyze(settings, variables);
+            return new LDefArray(line, location, index).copy(this).analyze(variables);
         } else if (Map.class.isAssignableFrom(before.clazz)) {
-            return new LMapShortcut(line, location, index).copy(this).analyze(settings, variables);
+            return new LMapShortcut(line, location, index).copy(this).analyze(variables);
         } else if (List.class.isAssignableFrom(before.clazz)) {
-            return new LListShortcut(line, location, index).copy(this).analyze(settings, variables);
+            return new LListShortcut(line, location, index).copy(this).analyze(variables);
         }
 
         throw new IllegalArgumentException(error("Illegal array access on type [" + before.name + "]."));
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
-        index.write(settings, adapter);
+    void write(MethodWriter adapter) {
+        index.write(adapter);
     }
 
     @Override
-    void load(final CompilerSettings settings, final MethodWriter adapter) {
+    void load(MethodWriter adapter) {
         adapter.arrayLoad(after.type);
     }
 
     @Override
-    void store(final CompilerSettings settings, final MethodWriter adapter) {
+    void store(MethodWriter adapter) {
         adapter.arrayStore(after.type);
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
index 150c9097ddd..6ea2eb2d3b3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Struct;
@@ -38,7 +37,7 @@ public final class LCall extends ALink {
 
     Method method = null;
 
-    public LCall(final int line, final String location, final String name, final List<AExpression> arguments) {
+    public LCall(int line, String location, String name, List<AExpression> arguments) {
         super(line, location, -1);
 
         this.name = name;
@@ -46,7 +45,7 @@ public final class LCall extends ALink {
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Variables variables) {
+    ALink analyze(Variables variables) {
         if (before == null) {
             throw new IllegalStateException(error("Illegal tree structure."));
         } else if (before.sort == Definition.Sort.ARRAY) {
@@ -64,8 +63,8 @@ public final class LCall extends ALink {
                 final AExpression expression = arguments.get(argument);
 
                 expression.expected = method.arguments.get(argument);
-                expression.analyze(settings, variables);
-                arguments.set(argument, expression.cast(settings, variables));
+                expression.analyze(variables);
+                arguments.set(argument, expression.cast(variables));
             }
 
             statement = true;
@@ -76,7 +75,7 @@ public final class LCall extends ALink {
             final ALink link = new LDefCall(line, location, name, arguments);
             link.copy(this);
 
-            return link.analyze(settings, variables);
+            return link.analyze(variables);
         }
 
         throw new IllegalArgumentException(error("Unknown call [" + name + "] with [" + arguments.size() + 
@@ -84,14 +83,14 @@ public final class LCall extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final MethodWriter adapter) {
+    void load(MethodWriter adapter) {
         for (final AExpression argument : arguments) {
-            argument.write(settings, adapter);
+            argument.write(adapter);
         }
 
         if (java.lang.reflect.Modifier.isStatic(method.reflect.getModifiers())) {
@@ -108,7 +107,7 @@ public final class LCall extends ALink {
     }
 
     @Override
-    void store(final CompilerSettings settings, final MethodWriter adapter) {
+    void store(MethodWriter adapter) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
index f8a03085124..5650dd77b4e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.AnalyzerCaster;
@@ -35,14 +34,14 @@ public final class LCast extends ALink {
 
     Cast cast = null;
 
-    public LCast(final int line, final String location, final String type) {
+    public LCast(int line, String location, String type) {
         super(line, location, -1);
 
         this.type = type;
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Variables variables) {
+    ALink analyze(Variables variables) {
         if (before == null) {
             throw new IllegalStateException(error("Illegal tree structure."));
         } else if (store) {
@@ -61,17 +60,17 @@ public final class LCast extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         adapter.writeCast(cast);
     }
 
     @Override
-    void load(final CompilerSettings settings, final MethodWriter adapter) {
+    void load(MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void store(final CompilerSettings settings, final MethodWriter adapter) {
+    void store(MethodWriter adapter) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index e07dd9ab6ff..1ef12026e65 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Variables;
@@ -35,17 +34,17 @@ final class LDefArray extends ALink implements IDefLink {
 
     AExpression index;
 
-    LDefArray(final int line, final String location, final AExpression index) {
+    LDefArray(int line, String location, AExpression index) {
         super(line, location, 2);
 
         this.index = index;
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Variables variables) {
-        index.analyze(settings, variables);
+    ALink analyze(Variables variables) {
+        index.analyze(variables);
         index.expected = index.actual;
-        index = index.cast(settings, variables);
+        index = index.cast(variables);
 
         after = Definition.DEF_TYPE;
 
@@ -53,18 +52,18 @@ final class LDefArray extends ALink implements IDefLink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
-        index.write(settings, adapter);
+    void write(MethodWriter adapter) {
+        index.write(adapter);
     }
 
     @Override
-    void load(final CompilerSettings settings, final MethodWriter adapter) {
+    void load(MethodWriter adapter) {
         final String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type, index.actual.type);
         adapter.invokeDynamic("arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_LOAD);
     }
 
     @Override
-    void store(final CompilerSettings settings, final MethodWriter adapter) {
+    void store(MethodWriter adapter) {
         final String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type,
             index.actual.type, after.type);
         adapter.invokeDynamic("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_STORE);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index f15f132cbec..f9ab0934eb5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Variables;
@@ -37,7 +36,7 @@ final class LDefCall extends ALink implements IDefLink {
     final String name;
     final List<AExpression> arguments;
 
-    LDefCall(final int line, final String location, final String name, final List<AExpression> arguments) {
+    LDefCall(int line, String location, String name, List<AExpression> arguments) {
         super(line, location, -1);
 
         this.name = name;
@@ -45,13 +44,13 @@ final class LDefCall extends ALink implements IDefLink {
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Variables variables) {
+    ALink analyze(Variables variables) {
         for (int argument = 0; argument < arguments.size(); ++argument) {
             final AExpression expression = arguments.get(argument);
 
-            expression.analyze(settings, variables);
+            expression.analyze(variables);
             expression.expected = expression.actual;
-            arguments.set(argument, expression.cast(settings, variables));
+            arguments.set(argument, expression.cast(variables));
         }
 
         statement = true;
@@ -61,12 +60,12 @@ final class LDefCall extends ALink implements IDefLink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final MethodWriter adapter) {
+    void load(MethodWriter adapter) {
         final StringBuilder signature = new StringBuilder();
 
         signature.append('(');
@@ -77,7 +76,7 @@ final class LDefCall extends ALink implements IDefLink {
         // it can avoid some unnecessary boxing etc.
         for (final AExpression argument : arguments) {
             signature.append(argument.actual.type.getDescriptor());
-            argument.write(settings, adapter);
+            argument.write(adapter);
         }
 
         signature.append(')');
@@ -88,7 +87,7 @@ final class LDefCall extends ALink implements IDefLink {
     }
 
     @Override
-    void store(final CompilerSettings settings, final MethodWriter adapter) {
+    void store(MethodWriter adapter) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index 84850826e6f..ee0cf990c5b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Variables;
@@ -35,7 +34,7 @@ final class LDefField extends ALink implements IDefLink {
 
     final String value;
 
-    LDefField(final int line, final String location, final String value) {
+    LDefField(int line, String location, String value) {
         super(line, location, 1);
 
         this.value = value;
@@ -43,25 +42,25 @@ final class LDefField extends ALink implements IDefLink {
 
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Variables variables) {
+    ALink analyze(Variables variables) {
         after = Definition.DEF_TYPE;
 
         return this;
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final MethodWriter adapter) {
+    void load(MethodWriter adapter) {
         final String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type);
         adapter.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.LOAD);
     }
 
     @Override
-    void store(final CompilerSettings settings, final MethodWriter adapter) {
+    void store(MethodWriter adapter) {
         final String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, after.type);
         adapter.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.STORE);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
index 3c0368666d0..44ba33acdda 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Field;
 import org.elasticsearch.painless.Definition.Sort;
@@ -39,14 +38,14 @@ public final class LField extends ALink {
 
     Field field;
 
-    public LField(final int line, final String location, final String value) {
+    public LField(int line, String location, String value) {
         super(line, location, 1);
 
         this.value = value;
     }
 
     @Override
-    ALink analyze(CompilerSettings settings, Variables variables) {
+    ALink analyze(Variables variables) {
         if (before == null) {
             throw new IllegalStateException(error("Illegal tree structure."));
         }
@@ -54,9 +53,9 @@ public final class LField extends ALink {
         final Sort sort = before.sort;
 
         if (sort == Sort.ARRAY) {
-            return new LArrayLength(line, location, value).copy(this).analyze(settings, variables);
+            return new LArrayLength(line, location, value).copy(this).analyze(variables);
         } else if (sort == Sort.DEF) {
-            return new LDefField(line, location, value).copy(this).analyze(settings, variables);
+            return new LDefField(line, location, value).copy(this).analyze(variables);
         }
 
         final Struct struct = before.struct;
@@ -80,17 +79,17 @@ public final class LField extends ALink {
                         Character.toUpperCase(value.charAt(0)) + value.substring(1), 1));
 
             if (shortcut) {
-                return new LShortcut(line, location, value).copy(this).analyze(settings, variables);
+                return new LShortcut(line, location, value).copy(this).analyze(variables);
             } else {
                 final EConstant index = new EConstant(line, location, value);
-                index.analyze(settings, variables);
+                index.analyze(variables);
 
                 if (Map.class.isAssignableFrom(before.clazz)) {
-                    return new LMapShortcut(line, location, index).copy(this).analyze(settings, variables);
+                    return new LMapShortcut(line, location, index).copy(this).analyze(variables);
                 }
                 
                 if (List.class.isAssignableFrom(before.clazz)) {
-                    return new LListShortcut(line, location, index).copy(this).analyze(settings, variables);
+                    return new LListShortcut(line, location, index).copy(this).analyze(variables);
                 }
             }
         }
@@ -99,12 +98,12 @@ public final class LField extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final MethodWriter adapter) {
+    void load(MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isStatic(field.reflect.getModifiers())) {
             adapter.getStatic(field.owner.type, field.reflect.getName(), field.type.type);
         } else {
@@ -113,7 +112,7 @@ public final class LField extends ALink {
     }
 
     @Override
-    void store(final CompilerSettings settings, final MethodWriter adapter) {
+    void store(MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isStatic(field.reflect.getModifiers())) {
             adapter.putStatic(field.owner.type, field.reflect.getName(), field.type.type);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
index 9541459a615..52f82c9d1ca 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
@@ -35,14 +34,14 @@ final class LListShortcut extends ALink {
     Method getter;
     Method setter;
 
-    LListShortcut(final int line, final String location, final AExpression index) {
+    LListShortcut(int line, String location, AExpression index) {
         super(line, location, 2);
 
         this.index = index;
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Variables variables) {
+    ALink analyze(Variables variables) {
         getter = before.struct.methods.get(new Definition.MethodKey("get", 1));
         setter = before.struct.methods.get(new Definition.MethodKey("set", 2));
 
@@ -62,8 +61,8 @@ final class LListShortcut extends ALink {
 
         if ((load || store) && (!load || getter != null) && (!store || setter != null)) {
             index.expected = Definition.INT_TYPE;
-            index.analyze(settings, variables);
-            index = index.cast(settings, variables);
+            index.analyze(variables);
+            index = index.cast(variables);
 
             after = setter != null ? setter.arguments.get(1) : getter.rtn;
         } else {
@@ -74,12 +73,12 @@ final class LListShortcut extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
-        index.write(settings, adapter);
+    void write(MethodWriter adapter) {
+        index.write(adapter);
     }
 
     @Override
-    void load(final CompilerSettings settings, final MethodWriter adapter) {
+    void load(MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
             adapter.invokeInterface(getter.owner.type, getter.method);
         } else {
@@ -92,7 +91,7 @@ final class LListShortcut extends ALink {
     }
 
     @Override
-    void store(final CompilerSettings settings, final MethodWriter adapter) {
+    void store(MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
             adapter.invokeInterface(setter.owner.type, setter.method);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
index f06cb46152b..9fbc39d72b3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
@@ -35,14 +34,14 @@ final class LMapShortcut extends ALink {
     Method getter;
     Method setter;
 
-    LMapShortcut(final int line, final String location, final AExpression index) {
+    LMapShortcut(int line, String location, AExpression index) {
         super(line, location, 2);
 
         this.index = index;
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Variables variables) {
+    ALink analyze(Variables variables) {
         getter = before.struct.methods.get(new Definition.MethodKey("get", 1));
         setter = before.struct.methods.get(new Definition.MethodKey("put", 2));
 
@@ -61,8 +60,8 @@ final class LMapShortcut extends ALink {
 
         if ((load || store) && (!load || getter != null) && (!store || setter != null)) {
             index.expected = setter != null ? setter.arguments.get(0) : getter.arguments.get(0);
-            index.analyze(settings, variables);
-            index = index.cast(settings, variables);
+            index.analyze(variables);
+            index = index.cast(variables);
 
             after = setter != null ? setter.arguments.get(1) : getter.rtn;
         } else {
@@ -73,12 +72,12 @@ final class LMapShortcut extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
-        index.write(settings, adapter);
+    void write(MethodWriter adapter) {
+        index.write(adapter);
     }
 
     @Override
-    void load(final CompilerSettings settings, final MethodWriter adapter) {
+    void load(MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
             adapter.invokeInterface(getter.owner.type, getter.method);
         } else {
@@ -91,7 +90,7 @@ final class LMapShortcut extends ALink {
     }
 
     @Override
-    void store(final CompilerSettings settings, final MethodWriter adapter) {
+    void store(MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
             adapter.invokeInterface(setter.owner.type, setter.method);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
index 2faad220db6..da88b45e72d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Variables;
@@ -35,7 +34,7 @@ public final class LNewArray extends ALink {
     final String type;
     final List<AExpression> arguments;
 
-    public LNewArray(final int line, final String location, final String type, final List<AExpression> arguments) {
+    public LNewArray(int line, String location, String type, List<AExpression> arguments) {
         super(line, location, -1);
 
         this.type = type;
@@ -43,7 +42,7 @@ public final class LNewArray extends ALink {
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Variables variables) {
+    ALink analyze(Variables variables) {
         if (before != null) {
             throw new IllegalStateException(error("Illegal tree structure."));
         } else if (store) {
@@ -64,8 +63,8 @@ public final class LNewArray extends ALink {
             final AExpression expression = arguments.get(argument);
 
             expression.expected = Definition.INT_TYPE;
-            expression.analyze(settings, variables);
-            arguments.set(argument, expression.cast(settings, variables));
+            expression.analyze(variables);
+            arguments.set(argument, expression.cast(variables));
         }
 
         after = Definition.getType(type.struct, arguments.size());
@@ -74,14 +73,14 @@ public final class LNewArray extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final MethodWriter adapter) {
+    void load(MethodWriter adapter) {
         for (final AExpression argument : arguments) {
-            argument.write(settings, adapter);
+            argument.write(adapter);
         }
 
         if (arguments.size() > 1) {
@@ -92,7 +91,7 @@ public final class LNewArray extends ALink {
     }
 
     @Override
-    void store(final CompilerSettings settings, final MethodWriter adapter) {
+    void store(MethodWriter adapter) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
index a47ced1e096..ad3fc4627a5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Constructor;
 import org.elasticsearch.painless.Definition.Struct;
@@ -39,7 +38,7 @@ public final class LNewObj extends ALink {
 
     Constructor constructor;
 
-    public LNewObj(final int line, final String location, final String type, final List<AExpression> arguments) {
+    public LNewObj(int line, String location, String type, List<AExpression> arguments) {
         super(line, location, -1);
 
         this.type = type;
@@ -47,7 +46,7 @@ public final class LNewObj extends ALink {
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Variables variables) {
+    ALink analyze(Variables variables) {
         if (before != null) {
             throw new IllegalStateException(error("Illegal tree structure"));
         } else if (store) {
@@ -78,8 +77,8 @@ public final class LNewObj extends ALink {
                 final AExpression expression = arguments.get(argument);
 
                 expression.expected = types[argument];
-                expression.analyze(settings, variables);
-                arguments.set(argument, expression.cast(settings, variables));
+                expression.analyze(variables);
+                arguments.set(argument, expression.cast(variables));
             }
 
             statement = true;
@@ -92,27 +91,27 @@ public final class LNewObj extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final MethodWriter adapter) {
+    void load(MethodWriter adapter) {
         adapter.newInstance(after.type);
 
         if (load) {
             adapter.dup();
         }
 
-        for (final AExpression argument : arguments) {
-            argument.write(settings, adapter);
+        for (AExpression argument : arguments) {
+            argument.write(adapter);
         }
 
         adapter.invokeConstructor(constructor.owner.type, constructor.method);
     }
 
     @Override
-    void store(final CompilerSettings settings, final MethodWriter adapter) {
+    void store(MethodWriter adapter) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
index 79feef8813b..85a7d70c46e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
@@ -37,14 +36,14 @@ final class LShortcut extends ALink {
     Method getter = null;
     Method setter = null;
 
-    LShortcut(final int line, final String location, final String value) {
+    LShortcut(int line, String location, String value) {
         super(line, location, 1);
 
         this.value = value;
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Variables variables) {
+    ALink analyze(Variables variables) {
         final Struct struct = before.struct;
 
         getter = struct.methods.get(new Definition.MethodKey("get" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0));
@@ -74,12 +73,12 @@ final class LShortcut extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final MethodWriter adapter) {
+    void load(MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
             adapter.invokeInterface(getter.owner.type, getter.method);
         } else {
@@ -92,7 +91,7 @@ final class LShortcut extends ALink {
     }
 
     @Override
-    void store(final CompilerSettings settings, final MethodWriter adapter) {
+    void store(MethodWriter adapter) {
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
             adapter.invokeInterface(setter.owner.type, setter.method);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
index 115c4c57ab3..07d76d42676 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
@@ -29,14 +28,14 @@ import org.elasticsearch.painless.MethodWriter;
  */
 public final class LString extends ALink {
 
-    public LString(final int line, final String location, final String string) {
+    public LString(int line, String location, String string) {
         super(line, location, -1);
 
         this.string = string;
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Variables variables) {
+    ALink analyze(Variables variables) {
         if (before != null) {
             throw new IllegalStateException("Illegal tree structure.");
         } else if (store) {
@@ -51,17 +50,17 @@ public final class LString extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final MethodWriter adapter) {
+    void load(MethodWriter adapter) {
         adapter.push(string);
     }
 
     @Override
-    void store(final CompilerSettings settings, final MethodWriter adapter) {
+    void store(MethodWriter adapter) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
index 6d4254f76ba..85a2048bdc6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Variables;
@@ -36,14 +35,14 @@ public final class LVariable extends ALink {
 
     int slot;
 
-    public LVariable(final int line, final String location, final String name) {
+    public LVariable(int line, String location, String name) {
         super(line, location, 0);
 
         this.name = name;
     }
 
     @Override
-    ALink analyze(final CompilerSettings settings, final Variables variables) {
+    ALink analyze(Variables variables) {
         if (before != null) {
             throw new IllegalStateException(error("Illegal tree structure."));
         }
@@ -74,17 +73,17 @@ public final class LVariable extends ALink {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         // Do nothing.
     }
 
     @Override
-    void load(final CompilerSettings settings, final MethodWriter adapter) {
+    void load(MethodWriter adapter) {
         adapter.visitVarInsn(after.type.getOpcode(Opcodes.ILOAD), slot);
     }
 
     @Override
-    void store(final CompilerSettings settings, final MethodWriter adapter) {
+    void store(MethodWriter adapter) {
         adapter.visitVarInsn(after.type.getOpcode(Opcodes.ISTORE), slot);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
index 7b9aed850b6..a925c411a2a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -33,17 +32,17 @@ public final class SBlock extends AStatement {
 
     final List<AStatement> statements;
 
-    public SBlock(final int line, final String location, final List<AStatement> statements) {
+    public SBlock(int line, String location, List<AStatement> statements) {
         super(line, location);
 
         this.statements = Collections.unmodifiableList(statements);
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         final AStatement last = statements.get(statements.size() - 1);
 
-        for (final AStatement statement : statements) {
+        for (AStatement statement : statements) {
             if (allEscape) {
                 throw new IllegalArgumentException(error("Unreachable statement."));
             }
@@ -52,7 +51,7 @@ public final class SBlock extends AStatement {
             statement.lastSource = lastSource && statement == last;
             statement.lastLoop = (beginLoop || lastLoop) && statement == last;
 
-            statement.analyze(settings, variables);
+            statement.analyze(variables);
 
             methodEscape = statement.methodEscape;
             loopEscape = statement.loopEscape;
@@ -64,11 +63,11 @@ public final class SBlock extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
-        for (final AStatement statement : statements) {
+    void write(MethodWriter adapter) {
+        for (AStatement statement : statements) {
             statement.continu = continu;
             statement.brake = brake;
-            statement.write(settings, adapter);
+            statement.write(adapter);
         }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
index 08f8e811b05..fb12a2b452e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -28,12 +27,12 @@ import org.elasticsearch.painless.MethodWriter;
  */
 public final class SBreak extends AStatement {
 
-    public SBreak(final int line, final String location) {
+    public SBreak(int line, String location) {
         super(line, location);
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         if (!inLoop) {
             throw new IllegalArgumentException(error("Break statement outside of a loop."));
         }
@@ -45,7 +44,7 @@ public final class SBreak extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         writeDebugInfo(adapter);
         adapter.goTo(brake);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
index 35317e8de8a..01ca3f5697d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -28,12 +27,12 @@ import org.elasticsearch.painless.MethodWriter;
  */
 public final class SContinue extends AStatement {
 
-    public SContinue(final int line, final String location) {
+    public SContinue(int line, String location) {
         super(line, location);
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         if (!inLoop) {
             throw new IllegalArgumentException(error("Continue statement outside of a loop."));
         }
@@ -48,7 +47,7 @@ public final class SContinue extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         writeDebugInfo(adapter);
         adapter.goTo(continu);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
index ee878146364..917dd9b8e5a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -33,25 +32,25 @@ public final class SDeclBlock extends AStatement {
 
     final List<SDeclaration> declarations;
 
-    public SDeclBlock(final int line, final String location, final List<SDeclaration> declarations) {
+    public SDeclBlock(int line, String location, List<SDeclaration> declarations) {
         super(line, location);
 
         this.declarations = Collections.unmodifiableList(declarations);
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         for (final SDeclaration declaration : declarations) {
-            declaration.analyze(settings, variables);
+            declaration.analyze(variables);
         }
 
         statementCount = declarations.size();
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         for (final SDeclaration declaration : declarations) {
-            declaration.write(settings, adapter);
+            declaration.write(adapter);
         }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
index adf24509f6c..0e366dfc19b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.Variables.Variable;
@@ -37,7 +36,7 @@ public final class SDeclaration extends AStatement {
 
     Variable variable;
 
-    public SDeclaration(final int line, final String location, final String type, final String name, final AExpression expression) {
+    public SDeclaration(int line, String location, String type, String name, AExpression expression) {
         super(line, location);
 
         this.type = type;
@@ -46,18 +45,18 @@ public final class SDeclaration extends AStatement {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         variable = variables.addVariable(location, type, name, false, false);
 
         if (expression != null) {
             expression.expected = variable.type;
-            expression.analyze(settings, variables);
-            expression = expression.cast(settings, variables);
+            expression.analyze(variables);
+            expression = expression.cast(variables);
         }
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         writeDebugInfo(adapter);
         final org.objectweb.asm.Type type = variable.type.type;
         final Sort sort = variable.type.sort;
@@ -65,7 +64,7 @@ public final class SDeclaration extends AStatement {
         final boolean initialize = expression == null;
 
         if (!initialize) {
-            expression.write(settings, adapter);
+            expression.write(adapter);
         }
 
         switch (sort) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
index 249ff75c744..6c8785a923a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
@@ -32,30 +31,32 @@ public final class SDo extends AStatement {
 
     final AStatement block;
     AExpression condition;
+    final int maxLoopCounter;
 
-    public SDo(final int line, final String location, final AStatement block, final AExpression condition) {
+    public SDo(int line, String location, AStatement block, AExpression condition, int maxLoopCounter) {
         super(line, location);
 
         this.condition = condition;
         this.block = block;
+        this.maxLoopCounter = maxLoopCounter;
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         variables.incrementScope();
 
         block.beginLoop = true;
         block.inLoop = true;
 
-        block.analyze(settings, variables);
+        block.analyze(variables);
 
         if (block.loopEscape && !block.anyContinue) {
             throw new IllegalArgumentException(error("Extraneous do while loop."));
         }
 
         condition.expected = Definition.BOOLEAN_TYPE;
-        condition.analyze(settings, variables);
-        condition = condition.cast(settings, variables);
+        condition.analyze(variables);
+        condition = condition.cast(variables);
 
         if (condition.constant != null) {
             final boolean continuous = (boolean)condition.constant;
@@ -72,7 +73,7 @@ public final class SDo extends AStatement {
 
         statementCount = 1;
 
-        if (settings.getMaxLoopCounter() > 0) {
+        if (maxLoopCounter > 0) {
             loopCounterSlot = variables.getVariable(location, "#loop").slot;
         }
 
@@ -80,7 +81,7 @@ public final class SDo extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         writeDebugInfo(adapter);
         final Label start = new Label();
         final Label begin = new Label();
@@ -90,12 +91,12 @@ public final class SDo extends AStatement {
 
         block.continu = begin;
         block.brake = end;
-        block.write(settings, adapter);
+        block.write(adapter);
 
         adapter.mark(begin);
 
         condition.fals = end;
-        condition.write(settings, adapter);
+        condition.write(adapter);
 
         adapter.writeLoopCounter(loopCounterSlot, Math.max(1, block.statementCount));
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index aaf63a3c923..46922fdcd9e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Variables;
@@ -32,16 +31,16 @@ public final class SExpression extends AStatement {
 
     AExpression expression;
 
-    public SExpression(final int line, final String location, final AExpression expression) {
+    public SExpression(int line, String location, AExpression expression) {
         super(line, location);
 
         this.expression = expression;
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         expression.read = lastSource;
-        expression.analyze(settings, variables);
+        expression.analyze(variables);
 
         if (!lastSource && !expression.statement) {
             throw new IllegalArgumentException(error("Not a statement."));
@@ -50,7 +49,7 @@ public final class SExpression extends AStatement {
         final boolean rtn = lastSource && expression.actual.sort != Sort.VOID;
 
         expression.expected = rtn ? Definition.OBJECT_TYPE : expression.actual;
-        expression = expression.cast(settings, variables);
+        expression = expression.cast(variables);
 
         methodEscape = rtn;
         loopEscape = rtn;
@@ -59,9 +58,9 @@ public final class SExpression extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         writeDebugInfo(adapter);
-        expression.write(settings, adapter);
+        expression.write(adapter);
 
         if (methodEscape) {
             adapter.returnValue();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index cbce549f1cb..56073e6e8db 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
@@ -34,31 +33,33 @@ public final class SFor extends AStatement {
     AExpression condition;
     AExpression afterthought;
     final AStatement block;
+    final int maxLoopCounter;
 
-    public SFor(final int line, final String location,
-                final ANode initializer, final AExpression condition, final AExpression afterthought, final AStatement block) {
+    public SFor(int line, String location,
+                ANode initializer, AExpression condition, AExpression afterthought, AStatement block, int maxLoopCounter) {
         super(line, location);
 
         this.initializer = initializer;
         this.condition = condition;
         this.afterthought = afterthought;
         this.block = block;
+        this.maxLoopCounter = maxLoopCounter;
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         variables.incrementScope();
 
         boolean continuous = false;
 
         if (initializer != null) {
             if (initializer instanceof SDeclBlock) {
-                ((SDeclBlock)initializer).analyze(settings, variables);
+                ((SDeclBlock)initializer).analyze(variables);
             } else if (initializer instanceof AExpression) {
                 final AExpression initializer = (AExpression)this.initializer;
 
                 initializer.read = false;
-                initializer.analyze(settings, variables);
+                initializer.analyze(variables);
 
                 if (!initializer.statement) {
                     throw new IllegalArgumentException(initializer.error("Not a statement."));
@@ -71,8 +72,8 @@ public final class SFor extends AStatement {
         if (condition != null) {
 
             condition.expected = Definition.BOOLEAN_TYPE;
-            condition.analyze(settings, variables);
-            condition = condition.cast(settings, variables);
+            condition.analyze(variables);
+            condition = condition.cast(variables);
 
             if (condition.constant != null) {
                 continuous = (boolean)condition.constant;
@@ -91,7 +92,7 @@ public final class SFor extends AStatement {
 
         if (afterthought != null) {
             afterthought.read = false;
-            afterthought.analyze(settings, variables);
+            afterthought.analyze(variables);
 
             if (!afterthought.statement) {
                 throw new IllegalArgumentException(afterthought.error("Not a statement."));
@@ -104,7 +105,7 @@ public final class SFor extends AStatement {
             block.beginLoop = true;
             block.inLoop = true;
 
-            block.analyze(settings, variables);
+            block.analyze(variables);
 
             if (block.loopEscape && !block.anyContinue) {
                 throw new IllegalArgumentException(error("Extraneous for loop."));
@@ -120,7 +121,7 @@ public final class SFor extends AStatement {
 
         statementCount = 1;
 
-        if (settings.getMaxLoopCounter() > 0) {
+        if (maxLoopCounter > 0) {
             loopCounterSlot = variables.getVariable(location, "#loop").slot;
         }
 
@@ -128,18 +129,18 @@ public final class SFor extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         writeDebugInfo(adapter);
         final Label start = new Label();
         final Label begin = afterthought == null ? start : new Label();
         final Label end = new Label();
 
         if (initializer instanceof SDeclBlock) {
-            ((SDeclBlock)initializer).write(settings, adapter);
+            ((SDeclBlock)initializer).write(adapter);
         } else if (initializer instanceof AExpression) {
             AExpression initializer = (AExpression)this.initializer;
 
-            initializer.write(settings, adapter);
+            initializer.write(adapter);
             adapter.writePop(initializer.expected.sort.size);
         }
 
@@ -147,7 +148,7 @@ public final class SFor extends AStatement {
 
         if (condition != null) {
             condition.fals = end;
-            condition.write(settings, adapter);
+            condition.write(adapter);
         }
 
         boolean allEscape = false;
@@ -162,14 +163,14 @@ public final class SFor extends AStatement {
             }
 
             adapter.writeLoopCounter(loopCounterSlot, statementCount);
-            block.write(settings, adapter);
+            block.write(adapter);
         } else {
             adapter.writeLoopCounter(loopCounterSlot, 1);
         }
 
         if (afterthought != null) {
             adapter.mark(begin);
-            afterthought.write(settings, adapter);
+            afterthought.write(adapter);
         }
 
         if (afterthought != null || !allEscape) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
index 1fadc4f0ac4..560ce44e415 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
@@ -34,8 +33,7 @@ public final class SIfElse extends AStatement {
     final AStatement ifblock;
     final AStatement elseblock;
 
-    public SIfElse(final int line, final String location,
-                   final AExpression condition, final AStatement ifblock, final AStatement elseblock) {
+    public SIfElse(int line, String location, AExpression condition, AStatement ifblock, AStatement elseblock) {
         super(line, location);
 
         this.condition = condition;
@@ -44,10 +42,10 @@ public final class SIfElse extends AStatement {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         condition.expected = Definition.BOOLEAN_TYPE;
-        condition.analyze(settings, variables);
-        condition = condition.cast(settings, variables);
+        condition.analyze(variables);
+        condition = condition.cast(variables);
 
         if (condition.constant != null) {
             throw new IllegalArgumentException(error("Extraneous if statement."));
@@ -58,7 +56,7 @@ public final class SIfElse extends AStatement {
         ifblock.lastLoop = lastLoop;
 
         variables.incrementScope();
-        ifblock.analyze(settings, variables);
+        ifblock.analyze(variables);
         variables.decrementScope();
 
         anyContinue = ifblock.anyContinue;
@@ -71,7 +69,7 @@ public final class SIfElse extends AStatement {
             elseblock.lastLoop = lastLoop;
 
             variables.incrementScope();
-            elseblock.analyze(settings, variables);
+            elseblock.analyze(variables);
             variables.decrementScope();
 
             methodEscape = ifblock.methodEscape && elseblock.methodEscape;
@@ -84,17 +82,17 @@ public final class SIfElse extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         writeDebugInfo(adapter);
         final Label end = new Label();
         final Label fals = elseblock != null ? new Label() : end;
 
         condition.fals = fals;
-        condition.write(settings, adapter);
+        condition.write(adapter);
 
         ifblock.continu = continu;
         ifblock.brake = brake;
-        ifblock.write(settings, adapter);
+        ifblock.write(adapter);
 
         if (elseblock != null) {
             if (!ifblock.allEscape) {
@@ -105,7 +103,7 @@ public final class SIfElse extends AStatement {
 
             elseblock.continu = continu;
             elseblock.brake = brake;
-            elseblock.write(settings, adapter);
+            elseblock.write(adapter);
         }
 
         adapter.mark(end);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
index f2c051ac1f6..d8c388e7e60 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
@@ -31,17 +30,17 @@ public final class SReturn extends AStatement {
 
     AExpression expression;
 
-    public SReturn(final int line, final String location, final AExpression expression) {
+    public SReturn(int line, String location, AExpression expression) {
         super(line, location);
 
         this.expression = expression;
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         expression.expected = Definition.OBJECT_TYPE;
-        expression.analyze(settings, variables);
-        expression = expression.cast(settings, variables);
+        expression.analyze(variables);
+        expression = expression.cast(variables);
 
         methodEscape = true;
         loopEscape = true;
@@ -51,9 +50,9 @@ public final class SReturn extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         writeDebugInfo(adapter);
-        expression.write(settings, adapter);
+        expression.write(adapter);
         adapter.returnValue();
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index e77ce8330a6..0932ae6e2f0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Opcodes;
 import org.elasticsearch.painless.MethodWriter;
@@ -34,14 +33,14 @@ public final class SSource extends AStatement {
 
     final List<AStatement> statements;
 
-    public SSource(final int line, final String location, final List<AStatement> statements) {
+    public SSource(int line, String location, List<AStatement> statements) {
         super(line, location);
 
         this.statements = Collections.unmodifiableList(statements);
     }
 
     @Override
-    public void analyze(final CompilerSettings settings, final Variables variables) {
+    public void analyze(Variables variables) {
         variables.incrementScope();
 
         final AStatement last = statements.get(statements.size() - 1);
@@ -52,7 +51,7 @@ public final class SSource extends AStatement {
             }
 
             statement.lastSource = statement == last;
-            statement.analyze(settings, variables);
+            statement.analyze(variables);
 
             methodEscape = statement.methodEscape;
             allEscape = statement.allEscape;
@@ -62,9 +61,9 @@ public final class SSource extends AStatement {
     }
 
     @Override
-    public void write(final CompilerSettings settings, final MethodWriter adapter) {
+    public void write(MethodWriter adapter) {
         for (final AStatement statement : statements) {
-            statement.write(settings, adapter);
+            statement.write(adapter);
         }
 
         if (!methodEscape) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
index f9a719a5f77..a0bfa12e267 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
@@ -31,17 +30,17 @@ public final class SThrow extends AStatement {
 
     AExpression expression;
 
-    public SThrow(final int line, final String location, final AExpression expression) {
+    public SThrow(int line, String location, AExpression expression) {
         super(line, location);
 
         this.expression = expression;
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         expression.expected = Definition.EXCEPTION_TYPE;
-        expression.analyze(settings, variables);
-        expression = expression.cast(settings, variables);
+        expression.analyze(variables);
+        expression = expression.cast(variables);
 
         methodEscape = true;
         loopEscape = true;
@@ -50,9 +49,9 @@ public final class SThrow extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         writeDebugInfo(adapter);
-        expression.write(settings, adapter);
+        expression.write(adapter);
         adapter.throwException();
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java
index a1ee15a050f..9a791b1615d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.Variables.Variable;
 import org.objectweb.asm.Label;
@@ -41,7 +40,7 @@ public final class STrap extends AStatement {
     Label end;
     Label exception;
 
-    public STrap(final int line, final String location, final String type, final String name, final AStatement block) {
+    public STrap(int line, String location, String type, String name, AStatement block) {
         super(line, location);
 
         this.type = type;
@@ -50,7 +49,7 @@ public final class STrap extends AStatement {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         variable = variables.addVariable(location, type, name, true, false);
 
         if (!Exception.class.isAssignableFrom(variable.type.clazz)) {
@@ -62,7 +61,7 @@ public final class STrap extends AStatement {
             block.inLoop = inLoop;
             block.lastLoop = lastLoop;
 
-            block.analyze(settings, variables);
+            block.analyze(variables);
 
             methodEscape = block.methodEscape;
             loopEscape = block.loopEscape;
@@ -74,7 +73,7 @@ public final class STrap extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         writeDebugInfo(adapter);
         final Label jump = new Label();
 
@@ -84,7 +83,7 @@ public final class STrap extends AStatement {
         if (block != null) {
             block.continu = continu;
             block.brake = brake;
-            block.write(settings, adapter);
+            block.write(adapter);
         }
 
         adapter.visitTryCatchBlock(begin, end, jump, variable.type.type.getInternalName());
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
index 70a5b2164da..4f97d5835cc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.MethodWriter;
@@ -35,7 +34,7 @@ public final class STry extends AStatement {
     final AStatement block;
     final List<STrap> traps;
 
-    public STry(final int line, final String location, final AStatement block, final List<STrap> traps) {
+    public STry(int line, String location, AStatement block, List<STrap> traps) {
         super(line, location);
 
         this.block = block;
@@ -43,13 +42,13 @@ public final class STry extends AStatement {
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         block.lastSource = lastSource;
         block.inLoop = inLoop;
         block.lastLoop = lastLoop;
 
         variables.incrementScope();
-        block.analyze(settings, variables);
+        block.analyze(variables);
         variables.decrementScope();
 
         methodEscape = block.methodEscape;
@@ -66,7 +65,7 @@ public final class STry extends AStatement {
             trap.lastLoop = lastLoop;
 
             variables.incrementScope();
-            trap.analyze(settings, variables);
+            trap.analyze(variables);
             variables.decrementScope();
 
             methodEscape &= trap.methodEscape;
@@ -82,7 +81,7 @@ public final class STry extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         writeDebugInfo(adapter);
         final Label begin = new Label();
         final Label end = new Label();
@@ -92,7 +91,7 @@ public final class STry extends AStatement {
 
         block.continu = continu;
         block.brake = brake;
-        block.write(settings, adapter);
+        block.write(adapter);
 
         if (!block.allEscape) {
             adapter.goTo(exception);
@@ -104,7 +103,7 @@ public final class STry extends AStatement {
             trap.begin = begin;
             trap.end = end;
             trap.exception = traps.size() > 1 ? exception : null;
-            trap.write(settings, adapter);
+            trap.write(adapter);
         }
 
         if (!block.allEscape || traps.size() > 1) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
index 98616d11499..48f59c55189 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
@@ -32,21 +31,23 @@ public final class SWhile extends AStatement {
 
     AExpression condition;
     final AStatement block;
+    final int maxLoopCounter;
 
-    public SWhile(final int line, final String location, final AExpression condition, final AStatement block) {
+    public SWhile(int line, String location, AExpression condition, AStatement block, int maxLoopCounter) {
         super(line, location);
 
         this.condition = condition;
         this.block = block;
+        this.maxLoopCounter = maxLoopCounter;
     }
 
     @Override
-    void analyze(final CompilerSettings settings, final Variables variables) {
+    void analyze(Variables variables) {
         variables.incrementScope();
 
         condition.expected = Definition.BOOLEAN_TYPE;
-        condition.analyze(settings, variables);
-        condition = condition.cast(settings, variables);
+        condition.analyze(variables);
+        condition = condition.cast(variables);
 
         boolean continuous = false;
 
@@ -68,7 +69,7 @@ public final class SWhile extends AStatement {
             block.beginLoop = true;
             block.inLoop = true;
 
-            block.analyze(settings, variables);
+            block.analyze(variables);
 
             if (block.loopEscape && !block.anyContinue) {
                 throw new IllegalArgumentException(error("Extranous while loop."));
@@ -84,7 +85,7 @@ public final class SWhile extends AStatement {
 
         statementCount = 1;
 
-        if (settings.getMaxLoopCounter() > 0) {
+        if (maxLoopCounter > 0) {
             loopCounterSlot = variables.getVariable(location, "#loop").slot;
         }
 
@@ -92,7 +93,7 @@ public final class SWhile extends AStatement {
     }
 
     @Override
-    void write(final CompilerSettings settings, final MethodWriter adapter) {
+    void write(MethodWriter adapter) {
         writeDebugInfo(adapter);
         final Label begin = new Label();
         final Label end = new Label();
@@ -100,14 +101,14 @@ public final class SWhile extends AStatement {
         adapter.mark(begin);
 
         condition.fals = end;
-        condition.write(settings, adapter);
+        condition.write(adapter);
 
         if (block != null) {
             adapter.writeLoopCounter(loopCounterSlot, Math.max(1, block.statementCount));
 
             block.continu = begin;
             block.brake = end;
-            block.write(settings, adapter);
+            block.write(adapter);
         } else {
             adapter.writeLoopCounter(loopCounterSlot, 1);
         }

From 6772517f4d927fbcabd9fb9528fc1049a3f95578 Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Fri, 20 May 2016 16:04:39 +0200
Subject: [PATCH 0386/1311] Cleanup the PR and apply advices from the review

* ESBlobStore tests must move to the test framework if we want to be able to reuse them in the context of plugins.
* To be able to identify more easily what are Integration Tests vs Unit Tests, this commit renames `*AzureTestCase` to `*AzureIntegTestCase`.
* Move some debug level logs to trace level
* Collapse when possible identical catch blocks
* `blobNameFromUri()` does not need anymore to get the container name. We just split the URI after 3 `/` and simply get the remaining part.
* Added a Unit test for that
* As we renamed some existing classes, checkstyle is now complaining about the lines width.
* While we are at it, let's replace all calls to `execute().actionGet()` with `get()`
* Move `readSettingsFromFile()` in a Util class. Note that this class might be useful for other plugins (S3/EC2/Azure-discovery for instance) so may be we should move it to the test framework?
* Replace some part of the code with lambdas
---
 .../azure/blobstore/AzureBlobContainer.java   |  21 ++-
 .../storage/AzureStorageServiceImpl.java      |  20 ++-
 ...e.java => AbstractAzureIntegTestCase.java} |   2 +-
 ...tAzureRepositoryServiceIntegTestCase.java} |   6 +-
 ...ractAzureWithThirdPartyIntegTestCase.java} |  24 +--
 .../cloud/azure/AzureTestUtils.java           |  49 ++++++
 .../storage/AzureStorageServiceTests.java     |  14 +-
 .../azure/AzureBlobStoreTests.java            |  22 +--
 ...zureSnapshotRestoreServiceIntegTests.java} |  21 ++-
 .../azure/AzureSnapshotRestoreTests.java      | 145 ++++++++++--------
 .../test/ESBlobStoreContainerTestCase.java    |   0
 .../ESBlobStoreRepositoryIntegTestCase.java   |   0
 .../test/ESBlobStoreTestCase.java             |   0
 13 files changed, 190 insertions(+), 134 deletions(-)
 rename plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/{AbstractAzureTestCase.java => AbstractAzureIntegTestCase.java} (94%)
 rename plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/{AbstractAzureRepositoryServiceTestCase.java => AbstractAzureRepositoryServiceIntegTestCase.java} (95%)
 rename plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/{AbstractAzureWithThirdPartyTestCase.java => AbstractAzureWithThirdPartyIntegTestCase.java} (60%)
 create mode 100644 plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AzureTestUtils.java
 rename plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/{AzureSnapshotRestoreServiceTests.java => AzureSnapshotRestoreServiceIntegTests.java} (91%)
 rename {core/src/test => test/framework/src/main}/java/org/elasticsearch/test/ESBlobStoreContainerTestCase.java (100%)
 rename {core/src/test => test/framework/src/main}/java/org/elasticsearch/test/ESBlobStoreRepositoryIntegTestCase.java (100%)
 rename {core/src/test => test/framework/src/main}/java/org/elasticsearch/test/ESBlobStoreTestCase.java (100%)

diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
index 57e7d2673dc..6f4654161ee 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
@@ -62,7 +62,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
 
     @Override
     public boolean blobExists(String blobName) {
-        logger.debug("blobExists({})", blobName);
+        logger.trace("blobExists({})", blobName);
         try {
             return blobStore.blobExists(blobStore.container(), buildKey(blobName));
         } catch (URISyntaxException | StorageException e) {
@@ -73,7 +73,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
 
     @Override
     public InputStream readBlob(String blobName) throws IOException {
-        logger.debug("readBlob({})", blobName);
+        logger.trace("readBlob({})", blobName);
         try {
             return blobStore.getInputStream(blobStore.container(), buildKey(blobName));
         } catch (StorageException e) {
@@ -88,7 +88,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
 
     @Override
     public void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException {
-        logger.debug("writeBlob({}, stream, {})", blobName, blobSize);
+        logger.trace("writeBlob({}, stream, {})", blobName, blobSize);
         try (OutputStream stream = createOutput(blobName)) {
             Streams.copy(inputStream, stream);
         }
@@ -96,7 +96,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
 
     @Override
     public void writeBlob(String blobName, BytesReference bytes) throws IOException {
-        logger.debug("writeBlob({}, bytes)", blobName);
+        logger.trace("writeBlob({}, bytes)", blobName);
         try (OutputStream stream = createOutput(blobName)) {
             bytes.writeTo(stream);
         }
@@ -119,7 +119,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
 
     @Override
     public void deleteBlob(String blobName) throws IOException {
-        logger.debug("deleteBlob({})", blobName);
+        logger.trace("deleteBlob({})", blobName);
         try {
             blobStore.deleteBlob(blobStore.container(), buildKey(blobName));
         } catch (URISyntaxException | StorageException e) {
@@ -130,7 +130,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
 
     @Override
     public Map<String, BlobMetaData> listBlobsByPrefix(@Nullable String prefix) throws IOException {
-        logger.debug("listBlobsByPrefix({})", prefix);
+        logger.trace("listBlobsByPrefix({})", prefix);
 
         try {
             return blobStore.listBlobsByPrefix(blobStore.container(), keyPath, prefix);
@@ -142,7 +142,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
 
     @Override
     public void move(String sourceBlobName, String targetBlobName) throws IOException {
-        logger.debug("move({}, {})", sourceBlobName, targetBlobName);
+        logger.trace("move({}, {})", sourceBlobName, targetBlobName);
         try {
             String source = keyPath + sourceBlobName;
             String target = keyPath + targetBlobName;
@@ -150,10 +150,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
             logger.debug("moving blob [{}] to [{}] in container {{}}", source, target, blobStore.container());
 
             blobStore.moveBlob(blobStore.container(), source, target);
-        } catch (URISyntaxException e) {
-            logger.warn("can not move blob [{}] to [{}] in container {{}}: {}", sourceBlobName, targetBlobName, blobStore.container(), e.getMessage());
-            throw new IOException(e);
-        } catch (StorageException e) {
+        } catch (URISyntaxException | StorageException e) {
             logger.warn("can not move blob [{}] to [{}] in container {{}}: {}", sourceBlobName, targetBlobName, blobStore.container(), e.getMessage());
             throw new IOException(e);
         }
@@ -161,7 +158,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
 
     @Override
     public Map<String, BlobMetaData> listBlobs() throws IOException {
-        logger.debug("listBlobs()");
+        logger.trace("listBlobs()");
         return listBlobsByPrefix(null);
     }
 
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
index 71fcee8f6df..ce5a06fdefa 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
@@ -182,18 +182,32 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
         CloudBlobClient client = this.getSelectedClient(account, mode);
         CloudBlobContainer blob_container = client.getContainerReference(container);
         if (blob_container.exists()) {
+            // We list the blobs using a flat blob listing mode
             for (ListBlobItem blobItem : blob_container.listBlobs(path, true)) {
-                String blobName = blobNameFromUri(container, blobItem.getUri());
+                String blobName = blobNameFromUri(blobItem.getUri());
                 logger.trace("removing blob [{}] full URI was [{}]", blobName, blobItem.getUri());
                 deleteBlob(account, mode, container, blobName);
             }
         }
     }
 
-    public static String blobNameFromUri(String container, URI uri) {
+    /**
+     * Extract the blob name from a URI like https://myservice.azure.net/container/path/to/myfile
+     * It should remove the container part (first part of the path) and gives path/to/myfile
+     * @param uri URI to parse
+     * @return The blob name relative to the container
+     */
+    public static String blobNameFromUri(URI uri) {
         String path = uri.getPath();
+
         // We remove the container name from the path
-        return path.replaceFirst("/" + container + "/", "");
+        // The 3 magic number cames from the fact we have // in the first part of the URI (protocol)
+        // Then a / after the server address
+        // And we finally split after the container/
+        String[] splits = path.split("/", 3);
+
+        // We return the remaining end of the string
+        return splits[2];
     }
 
     @Override
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureTestCase.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureIntegTestCase.java
similarity index 94%
rename from plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureTestCase.java
rename to plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureIntegTestCase.java
index b1bc5ccf6d4..9645ef3461f 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureTestCase.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureIntegTestCase.java
@@ -28,7 +28,7 @@ import java.util.Collection;
 /**
  * Base class for Azure tests.
  */
-public abstract class AbstractAzureTestCase extends ESIntegTestCase {
+public abstract class AbstractAzureIntegTestCase extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceTestCase.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceIntegTestCase.java
similarity index 95%
rename from plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceTestCase.java
rename to plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceIntegTestCase.java
index cd7effa417f..ae1bdd19a54 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceTestCase.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceIntegTestCase.java
@@ -38,7 +38,7 @@ import org.junit.Before;
 import java.net.URISyntaxException;
 import java.util.Collection;
 
-public abstract class AbstractAzureRepositoryServiceTestCase extends AbstractAzureTestCase {
+public abstract class AbstractAzureRepositoryServiceIntegTestCase extends AbstractAzureIntegTestCase {
 
     public static class TestPlugin extends Plugin {
         @Override
@@ -56,7 +56,7 @@ public abstract class AbstractAzureRepositoryServiceTestCase extends AbstractAzu
 
     protected String basePath;
 
-    public AbstractAzureRepositoryServiceTestCase(String basePath) {
+    public AbstractAzureRepositoryServiceIntegTestCase(String basePath) {
         this.basePath = basePath;
     }
 
@@ -70,7 +70,7 @@ public abstract class AbstractAzureRepositoryServiceTestCase extends AbstractAzu
         }
         for (String repository : repositories) {
             try {
-                client().admin().cluster().prepareDeleteRepository(repository).execute().actionGet();
+                client().admin().cluster().prepareDeleteRepository(repository).get();
             } catch (RepositoryMissingException ex) {
                 // ignore
             }
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureWithThirdPartyTestCase.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureWithThirdPartyIntegTestCase.java
similarity index 60%
rename from plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureWithThirdPartyTestCase.java
rename to plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureWithThirdPartyIntegTestCase.java
index b7c2d373a58..8cae9424950 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureWithThirdPartyTestCase.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureWithThirdPartyIntegTestCase.java
@@ -19,16 +19,15 @@
 
 package org.elasticsearch.cloud.azure;
 
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.plugin.repository.azure.AzureRepositoryPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase.ThirdParty;
 
 import java.util.Collection;
 
+import static org.elasticsearch.cloud.azure.AzureTestUtils.readSettingsFromFile;
+
 /**
  * Base class for Azure tests that require credentials.
  * <p>
@@ -36,14 +35,13 @@ import java.util.Collection;
  * in order to run these tests.
  */
 @ThirdParty
-public abstract class AbstractAzureWithThirdPartyTestCase extends AbstractAzureTestCase {
+public abstract class AbstractAzureWithThirdPartyIntegTestCase extends AbstractAzureIntegTestCase {
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
                 .put(readSettingsFromFile())
-//                .put("path.home", createTempDir())
                 .build();
     }
 
@@ -51,20 +49,4 @@ public abstract class AbstractAzureWithThirdPartyTestCase extends AbstractAzureT
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return pluginList(AzureRepositoryPlugin.class);
     }
-
-    protected Settings readSettingsFromFile() {
-        Settings.Builder settings = Settings.builder();
-
-        // if explicit, just load it and don't load from env
-        try {
-            if (Strings.hasText(System.getProperty("tests.config"))) {
-                settings.loadFromPath(PathUtils.get((System.getProperty("tests.config"))));
-            } else {
-                throw new IllegalStateException("to run integration tests, you need to set -Dtests.thirdparty=true and -Dtests.config=/path/to/elasticsearch.yml");
-            }
-        } catch (SettingsException exception) {
-          throw new IllegalStateException("your test configuration file is incorrect: " + System.getProperty("tests.config"), exception);
-        }
-        return settings.build();
-    }
 }
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AzureTestUtils.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AzureTestUtils.java
new file mode 100644
index 00000000000..80840db587f
--- /dev/null
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AzureTestUtils.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cloud.azure;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsException;
+
+public class AzureTestUtils {
+    /**
+     * Read settings from file when running integration tests with ThirdParty annotation.
+     * elasticsearch.yml file path has to be set with -Dtests.config=/path/to/elasticsearch.yml.
+     * @return Settings from elasticsearch.yml integration test file (for 3rd party tests)
+     */
+    public static Settings readSettingsFromFile() {
+        Settings.Builder settings = Settings.builder();
+
+        // if explicit, just load it and don't load from env
+        try {
+            if (Strings.hasText(System.getProperty("tests.config"))) {
+                settings.loadFromPath(PathUtils.get((System.getProperty("tests.config"))));
+            } else {
+                throw new IllegalStateException("to run integration tests, you need to set -Dtests.thirdparty=true and " +
+                    "-Dtests.config=/path/to/elasticsearch.yml");
+            }
+        } catch (SettingsException exception) {
+            throw new IllegalStateException("your test configuration file is incorrect: " + System.getProperty("tests.config"), exception);
+        }
+        return settings.build();
+    }
+}
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java
index 865b611e95d..5959f964644 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java
@@ -25,9 +25,10 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 
 import java.net.URI;
+import java.net.URISyntaxException;
 
+import static org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl.blobNameFromUri;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.nullValue;
 
 public class AzureStorageServiceTests extends ESTestCase {
@@ -169,4 +170,15 @@ public class AzureStorageServiceTests extends ESTestCase {
                     new CloudBlobClient(URI.create("https://" + azureStorageSettings.getName())));
         }
     }
+
+    public void testBlobNameFromUri() throws URISyntaxException {
+        String name = blobNameFromUri(new URI("https://myservice.azure.net/container/path/to/myfile"));
+        assertThat(name, is("path/to/myfile"));
+        name = blobNameFromUri(new URI("http://myservice.azure.net/container/path/to/myfile"));
+        assertThat(name, is("path/to/myfile"));
+        name = blobNameFromUri(new URI("http://127.0.0.1/container/path/to/myfile"));
+        assertThat(name, is("path/to/myfile"));
+        name = blobNameFromUri(new URI("https://127.0.0.1/container/path/to/myfile"));
+        assertThat(name, is("path/to/myfile"));
+    }
 }
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java
index 1ec0fdc76bb..f851a0e55cb 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java
@@ -23,11 +23,8 @@ import com.microsoft.azure.storage.StorageException;
 import org.elasticsearch.cloud.azure.blobstore.AzureBlobStore;
 import org.elasticsearch.cloud.azure.storage.AzureStorageService;
 import org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.blobstore.BlobStore;
-import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.repositories.RepositoryName;
 import org.elasticsearch.repositories.RepositorySettings;
 import org.elasticsearch.test.ESBlobStoreTestCase;
@@ -36,6 +33,8 @@ import org.elasticsearch.test.ESIntegTestCase;
 import java.io.IOException;
 import java.net.URISyntaxException;
 
+import static org.elasticsearch.cloud.azure.AzureTestUtils.readSettingsFromFile;
+
 /**
  * You must specify {@code -Dtests.thirdparty=true -Dtests.config=/path/to/elasticsearch.yml}
  * in order to run these tests.
@@ -56,21 +55,4 @@ public class AzureBlobStoreTests extends ESBlobStoreTestCase {
             throw new IOException(e);
         }
     }
-
-    protected Settings readSettingsFromFile() {
-        Settings.Builder settings = Settings.builder();
-
-        // if explicit, just load it and don't load from env
-        try {
-            if (Strings.hasText(System.getProperty("tests.config"))) {
-                settings.loadFromPath(PathUtils.get((System.getProperty("tests.config"))));
-            } else {
-                throw new IllegalStateException("to run integration tests, you need to set -Dtests.thirdparty=true and -Dtests.config=/path/to/elasticsearch.yml");
-            }
-        } catch (SettingsException exception) {
-            throw new IllegalStateException("your test configuration file is incorrect: " + System.getProperty("tests.config"), exception);
-        }
-        return settings.build();
-    }
-
 }
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceIntegTests.java
similarity index 91%
rename from plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceTests.java
rename to plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceIntegTests.java
index 99e20e1ee4a..2c50deafb59 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceIntegTests.java
@@ -24,7 +24,7 @@ import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResp
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.cloud.azure.AbstractAzureRepositoryServiceTestCase;
+import org.elasticsearch.cloud.azure.AbstractAzureRepositoryServiceIntegTestCase;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -39,8 +39,8 @@ import static org.hamcrest.Matchers.greaterThan;
         numDataNodes = 1,
         numClientNodes = 0,
         transportClientRatio = 0.0)
-public class AzureSnapshotRestoreServiceTests extends AbstractAzureRepositoryServiceTestCase {
-    public AzureSnapshotRestoreServiceTests() {
+public class AzureSnapshotRestoreServiceIntegTests extends AbstractAzureRepositoryServiceIntegTestCase {
+    public AzureSnapshotRestoreServiceIntegTests() {
         super("/snapshot-test/repo-" + randomInt());
     }
 
@@ -69,11 +69,14 @@ public class AzureSnapshotRestoreServiceTests extends AbstractAzureRepositorySer
         assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(100L));
 
         logger.info("--> snapshot");
-        CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get();
+        CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap")
+            .setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get();
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
-        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
+        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(),
+            equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
 
-        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
+        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(),
+            equalTo(SnapshotState.SUCCESS));
 
         logger.info("--> delete some data");
         for (int i = 0; i < 50; i++) {
@@ -94,7 +97,8 @@ public class AzureSnapshotRestoreServiceTests extends AbstractAzureRepositorySer
         client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get();
 
         logger.info("--> restore all indices from the snapshot");
-        RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet();
+        RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap")
+            .setWaitForCompletion(true).get();
         assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
 
         ensureGreen();
@@ -106,7 +110,8 @@ public class AzureSnapshotRestoreServiceTests extends AbstractAzureRepositorySer
         logger.info("--> delete indices");
         cluster().wipeIndices("test-idx-1", "test-idx-2");
         logger.info("--> restore one index after deletion");
-        restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-2").execute().actionGet();
+        restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true)
+            .setIndices("test-idx-*", "-test-idx-2").get();
         assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
         ensureGreen();
         assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L));
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java
index 24b76047699..79b41a355b7 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java
@@ -28,7 +28,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRes
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.ClusterAdminClient;
-import org.elasticsearch.cloud.azure.AbstractAzureWithThirdPartyTestCase;
+import org.elasticsearch.cloud.azure.AbstractAzureWithThirdPartyIntegTestCase;
 import org.elasticsearch.cloud.azure.storage.AzureStorageService;
 import org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl;
 import org.elasticsearch.cluster.ClusterState;
@@ -51,19 +51,20 @@ import java.net.URISyntaxException;
 import java.util.Locale;
 import java.util.concurrent.TimeUnit;
 
+import static org.elasticsearch.cloud.azure.AzureTestUtils.readSettingsFromFile;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 
 /**
  * This test needs Azure to run and -Dtests.thirdparty=true to be set
  * and -Dtests.config=/path/to/elasticsearch.yml
- * @see AbstractAzureWithThirdPartyTestCase
+ * @see AbstractAzureWithThirdPartyIntegTestCase
  */
 @ClusterScope(
         scope = ESIntegTestCase.Scope.SUITE,
         numDataNodes = 1,
         transportClientRatio = 0.0)
-public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCase {
+public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyIntegTestCase {
     private String getRepositoryPath() {
         String testName = "it-" + getTestName();
         return testName.contains(" ") ? Strings.split(testName, " ")[0] : testName;
@@ -127,11 +128,14 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
         assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().totalHits(), equalTo(100L));
 
         logger.info("--> snapshot");
-        CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get();
+        CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap")
+            .setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get();
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
-        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
+        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(),
+            equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
 
-        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
+        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots()
+            .get(0).state(), equalTo(SnapshotState.SUCCESS));
 
         logger.info("--> delete some data");
         for (int i = 0; i < 50; i++) {
@@ -152,7 +156,8 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
         client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get();
 
         logger.info("--> restore all indices from the snapshot");
-        RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet();
+        RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap")
+            .setWaitForCompletion(true).get();
         assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
 
         ensureGreen();
@@ -164,7 +169,8 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
         logger.info("--> delete indices");
         cluster().wipeIndices("test-idx-1", "test-idx-2");
         logger.info("--> restore one index after deletion");
-        restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-2").execute().actionGet();
+        restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true)
+            .setIndices("test-idx-*", "-test-idx-2").get();
         assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
         ensureGreen();
         assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L));
@@ -204,11 +210,14 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
         assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
 
         logger.info("creating snapshot [{}]", snapshot1Name);
-        CreateSnapshotResponse createSnapshotResponse1 = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshot1Name).setWaitForCompletion(true).setIndices(indexName).get();
+        CreateSnapshotResponse createSnapshotResponse1 = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshot1Name)
+            .setWaitForCompletion(true).setIndices(indexName).get();
         assertThat(createSnapshotResponse1.getSnapshotInfo().successfulShards(), greaterThan(0));
-        assertThat(createSnapshotResponse1.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse1.getSnapshotInfo().totalShards()));
+        assertThat(createSnapshotResponse1.getSnapshotInfo().successfulShards(),
+            equalTo(createSnapshotResponse1.getSnapshotInfo().totalShards()));
 
-        assertThat(client.admin().cluster().prepareGetSnapshots(repositoryName).setSnapshots(snapshot1Name).get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
+        assertThat(client.admin().cluster().prepareGetSnapshots(repositoryName).setSnapshots(snapshot1Name).get().getSnapshots()
+            .get(0).state(), equalTo(SnapshotState.SUCCESS));
 
         logger.info("indexing second document");
         index(indexName, typeName, Integer.toString(2), "foo", "bar " + Integer.toString(2));
@@ -216,17 +225,21 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
         assertThat(client.prepareSearch(indexName).setSize(0).get().getHits().totalHits(), equalTo(2L));
 
         logger.info("creating snapshot [{}]", snapshot2Name);
-        CreateSnapshotResponse createSnapshotResponse2 = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshot2Name).setWaitForCompletion(true).setIndices(indexName).get();
+        CreateSnapshotResponse createSnapshotResponse2 = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshot2Name)
+            .setWaitForCompletion(true).setIndices(indexName).get();
         assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(), greaterThan(0));
-        assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse2.getSnapshotInfo().totalShards()));
+        assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(),
+            equalTo(createSnapshotResponse2.getSnapshotInfo().totalShards()));
 
-        assertThat(client.admin().cluster().prepareGetSnapshots(repositoryName).setSnapshots(snapshot2Name).get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
+        assertThat(client.admin().cluster().prepareGetSnapshots(repositoryName).setSnapshots(snapshot2Name).get().getSnapshots()
+            .get(0).state(), equalTo(SnapshotState.SUCCESS));
 
         logger.info("closing index [{}]", indexName);
         client.admin().indices().prepareClose(indexName).get();
 
         logger.info("attempting restore from snapshot [{}]", snapshot1Name);
-        RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot(repositoryName, snapshot1Name).setWaitForCompletion(true).execute().actionGet();
+        RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot(repositoryName, snapshot1Name)
+            .setWaitForCompletion(true).get();
         assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
         ensureGreen();
         assertThat(client.prepareSearch(indexName).setSize(0).get().getHits().totalHits(), equalTo(1L));
@@ -263,23 +276,30 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
         assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(100L));
 
         logger.info("--> snapshot 1");
-        CreateSnapshotResponse createSnapshotResponse1 = client.admin().cluster().prepareCreateSnapshot("test-repo1", "test-snap").setWaitForCompletion(true).setIndices("test-idx-1").get();
+        CreateSnapshotResponse createSnapshotResponse1 = client.admin().cluster().prepareCreateSnapshot("test-repo1", "test-snap")
+            .setWaitForCompletion(true).setIndices("test-idx-1").get();
         assertThat(createSnapshotResponse1.getSnapshotInfo().successfulShards(), greaterThan(0));
-        assertThat(createSnapshotResponse1.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse1.getSnapshotInfo().totalShards()));
+        assertThat(createSnapshotResponse1.getSnapshotInfo().successfulShards(),
+            equalTo(createSnapshotResponse1.getSnapshotInfo().totalShards()));
 
         logger.info("--> snapshot 2");
-        CreateSnapshotResponse createSnapshotResponse2 = client.admin().cluster().prepareCreateSnapshot("test-repo2", "test-snap").setWaitForCompletion(true).setIndices("test-idx-2").get();
+        CreateSnapshotResponse createSnapshotResponse2 = client.admin().cluster().prepareCreateSnapshot("test-repo2", "test-snap")
+            .setWaitForCompletion(true).setIndices("test-idx-2").get();
         assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(), greaterThan(0));
-        assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse2.getSnapshotInfo().totalShards()));
+        assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(),
+            equalTo(createSnapshotResponse2.getSnapshotInfo().totalShards()));
 
-        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo1").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
-        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo2").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
+        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo1").setSnapshots("test-snap").get().getSnapshots().get(0).state(),
+            equalTo(SnapshotState.SUCCESS));
+        assertThat(client.admin().cluster().prepareGetSnapshots("test-repo2").setSnapshots("test-snap").get().getSnapshots().get(0).state(),
+            equalTo(SnapshotState.SUCCESS));
 
         // Test restore after index deletion
         logger.info("--> delete indices");
         cluster().wipeIndices("test-idx-1", "test-idx-2");
         logger.info("--> restore one index after deletion from snapshot 1");
-        RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin().cluster().prepareRestoreSnapshot("test-repo1", "test-snap").setWaitForCompletion(true).setIndices("test-idx-1").execute().actionGet();
+        RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin().cluster().prepareRestoreSnapshot("test-repo1", "test-snap")
+            .setWaitForCompletion(true).setIndices("test-idx-1").get();
         assertThat(restoreSnapshotResponse1.getRestoreInfo().totalShards(), greaterThan(0));
         ensureGreen();
         assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().totalHits(), equalTo(100L));
@@ -288,7 +308,8 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
         assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false));
 
         logger.info("--> restore other index after deletion from snapshot 2");
-        RestoreSnapshotResponse restoreSnapshotResponse2 = client.admin().cluster().prepareRestoreSnapshot("test-repo2", "test-snap").setWaitForCompletion(true).setIndices("test-idx-2").execute().actionGet();
+        RestoreSnapshotResponse restoreSnapshotResponse2 = client.admin().cluster().prepareRestoreSnapshot("test-repo2", "test-snap")
+            .setWaitForCompletion(true).setIndices("test-idx-2").get();
         assertThat(restoreSnapshotResponse2.getRestoreInfo().totalShards(), greaterThan(0));
         ensureGreen();
         assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().totalHits(), equalTo(100L));
@@ -324,7 +345,8 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
         assertThat(client.prepareGetSnapshots("test-repo").get().getSnapshots().size(), equalTo(0));
 
         logger.info("--> snapshot");
-        CreateSnapshotResponse createSnapshotResponse = client.prepareCreateSnapshot("test-repo", "test-snap-26").setWaitForCompletion(true).setIndices("test-idx-*").get();
+        CreateSnapshotResponse createSnapshotResponse = client.prepareCreateSnapshot("test-repo", "test-snap-26")
+            .setWaitForCompletion(true).setIndices("test-idx-*").get();
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
 
         // Get all snapshots - should have one
@@ -346,7 +368,8 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
         assertThat(client.prepareGetSnapshots("test-repo").get().getSnapshots().size(), equalTo(0));
 
         logger.info("--> snapshot");
-        createSnapshotResponse = client.prepareCreateSnapshot("test-repo", "test-snap-26").setWaitForCompletion(true).setIndices("test-idx-*").get();
+        createSnapshotResponse = client.prepareCreateSnapshot("test-repo", "test-snap-26").setWaitForCompletion(true)
+            .setIndices("test-idx-*").get();
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
 
         // Get all snapshots - should have one
@@ -407,30 +430,26 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
         logger.info("-->  creating azure repository with container name [{}]", container);
         // It could happen that we just removed from a previous test the same container so
         // we can not create it yet.
-        assertBusy(new Runnable() {
-
-            @Override
-            public void run() {
+        assertBusy(() -> {
+            try {
+                PutRepositoryResponse putRepositoryResponse = client().admin().cluster().preparePutRepository("test-repo")
+                        .setType("azure").setSettings(Settings.builder()
+                                        .put(Repository.CONTAINER_SETTING.getKey(), container)
+                                        .put(Repository.BASE_PATH_SETTING.getKey(), getRepositoryPath())
+                                        .put(Repository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(1000, 10000), ByteSizeUnit.BYTES)
+                        ).get();
+                client().admin().cluster().prepareDeleteRepository("test-repo").get();
                 try {
-                    PutRepositoryResponse putRepositoryResponse = client().admin().cluster().preparePutRepository("test-repo")
-                            .setType("azure").setSettings(Settings.builder()
-                                            .put(Repository.CONTAINER_SETTING.getKey(), container)
-                                            .put(Repository.BASE_PATH_SETTING.getKey(), getRepositoryPath())
-                                            .put(Repository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(1000, 10000), ByteSizeUnit.BYTES)
-                            ).get();
-                    client().admin().cluster().prepareDeleteRepository("test-repo").get();
-                    try {
-                        logger.info("--> remove container [{}]", container);
-                        cleanRepositoryFiles(container);
-                    } catch (StorageException | URISyntaxException e) {
-                        // We can ignore that as we just try to clean after the test
-                    }
-                    assertTrue(putRepositoryResponse.isAcknowledged() == correct);
-                } catch (RepositoryVerificationException e) {
-                    if (correct) {
-                        logger.debug(" -> container is being removed. Let's wait a bit...");
-                        fail();
-                    }
+                    logger.info("--> remove container [{}]", container);
+                    cleanRepositoryFiles(container);
+                } catch (StorageException | URISyntaxException e) {
+                    // We can ignore that as we just try to clean after the test
+                }
+                assertTrue(putRepositoryResponse.isAcknowledged() == correct);
+            } catch (RepositoryVerificationException e) {
+                if (correct) {
+                    logger.debug(" -> container is being removed. Let's wait a bit...");
+                    fail();
                 }
             }
         }, 5, TimeUnit.MINUTES);
@@ -452,7 +471,7 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
 
         logger.info("--> restore non existing snapshot");
         try {
-            client.admin().cluster().prepareRestoreSnapshot("test-repo", "no-existing-snapshot").setWaitForCompletion(true).execute().actionGet();
+            client.admin().cluster().prepareRestoreSnapshot("test-repo", "no-existing-snapshot").setWaitForCompletion(true).get();
             fail("Shouldn't be here");
         } catch (SnapshotMissingException ex) {
             // Expected
@@ -468,21 +487,17 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
 
         // It could happen that we run this test really close to a previous one
         // so we might need some time to be able to create the container
-        assertBusy(new Runnable() {
-
-            @Override
-            public void run()  {
-                try {
-                    storageService.createContainer(null, LocationMode.PRIMARY_ONLY, container);
-                    logger.debug(" -> container created...");
-                } catch (URISyntaxException e) {
-                    // Incorrect URL. This should never happen.
-                    fail();
-                } catch (StorageException e) {
-                    // It could happen. Let's wait for a while.
-                    logger.debug(" -> container is being removed. Let's wait a bit...");
-                    fail();
-                }
+        assertBusy(() -> {
+            try {
+                storageService.createContainer(null, LocationMode.PRIMARY_ONLY, container);
+                logger.debug(" -> container created...");
+            } catch (URISyntaxException e) {
+                // Incorrect URL. This should never happen.
+                fail();
+            } catch (StorageException e) {
+                // It could happen. Let's wait for a while.
+                logger.debug(" -> container is being removed. Let's wait a bit...");
+                fail();
             }
         }, 30, TimeUnit.SECONDS);
         storageService.removeContainer(null, LocationMode.PRIMARY_ONLY, container);
@@ -510,7 +525,7 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyTestCa
         }
         for (String repository : repositories) {
             try {
-                client().admin().cluster().prepareDeleteRepository(repository).execute().actionGet();
+                client().admin().cluster().prepareDeleteRepository(repository).get();
             } catch (RepositoryMissingException ex) {
                 // ignore
             }
diff --git a/core/src/test/java/org/elasticsearch/test/ESBlobStoreContainerTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESBlobStoreContainerTestCase.java
similarity index 100%
rename from core/src/test/java/org/elasticsearch/test/ESBlobStoreContainerTestCase.java
rename to test/framework/src/main/java/org/elasticsearch/test/ESBlobStoreContainerTestCase.java
diff --git a/core/src/test/java/org/elasticsearch/test/ESBlobStoreRepositoryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESBlobStoreRepositoryIntegTestCase.java
similarity index 100%
rename from core/src/test/java/org/elasticsearch/test/ESBlobStoreRepositoryIntegTestCase.java
rename to test/framework/src/main/java/org/elasticsearch/test/ESBlobStoreRepositoryIntegTestCase.java
diff --git a/core/src/test/java/org/elasticsearch/test/ESBlobStoreTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESBlobStoreTestCase.java
similarity index 100%
rename from core/src/test/java/org/elasticsearch/test/ESBlobStoreTestCase.java
rename to test/framework/src/main/java/org/elasticsearch/test/ESBlobStoreTestCase.java

From 76f310ea15e2b91f0a6954622df9e3aa7103cee7 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 20 May 2016 10:40:55 -0400
Subject: [PATCH 0387/1311] Passthrough test logger level to nodes

This commit passes the system property tests.logger.level down to the
external nodes launched in integration tests. Specific tests that want
to override the default logging level should push down a setting to
the nodes using cluster configuration instead of pushing down a system
property to the nodes using cluster configuration.

Relates #18489
---
 .../main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
index 014686e0207..cd0475d2aea 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
@@ -130,6 +130,10 @@ class NodeInfo {
 
         env = [ 'JAVA_HOME' : project.javaHome ]
         args.addAll("-E", "node.portsfile=true")
+        String loggerLevel = System.getProperty("tests.logger.level")
+        if (loggerLevel != null) {
+            args.addAll("-E", "logger.level=${loggerLevel}")
+        }
         String collectedSystemProperties = config.systemProperties.collect { key, value -> "-D${key}=${value}" }.join(" ")
         String esJavaOpts = config.jvmArgs.isEmpty() ? collectedSystemProperties : collectedSystemProperties + " " + config.jvmArgs
         env.put('ES_JAVA_OPTS', esJavaOpts)

From ce88d7f9ab8932b0d0db32021c622c3f5a86cd6e Mon Sep 17 00:00:00 2001
From: Igor Motov <igor@motovs.org>
Date: Tue, 17 May 2016 12:37:49 -0400
Subject: [PATCH 0388/1311] Fix race condition in snapshot initialization

When a snapshot initialization fails, the create snapshot method may return before the snapshot metadata in the cluster state is removed. This can cause follow up snapshot-API related calls to fail due to a snapshot still running. This is causing CI failures when we try to delete indices that were participating in failed snapshot to a read-only repository.

Closes #18121
---
 .../snapshots/SnapshotsService.java           | 68 +++++++++++++++----
 1 file changed, 55 insertions(+), 13 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
index 8e6681893c9..5b52e915195 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
@@ -23,6 +23,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterChangedEvent;
@@ -44,6 +45,7 @@ import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.collect.Tuple;
@@ -366,14 +368,7 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
                 @Override
                 public void onFailure(String source, Throwable t) {
                     logger.warn("[{}] failed to create snapshot", t, snapshot.snapshotId());
-                    removeSnapshotFromClusterState(snapshot.snapshotId(), null, t);
-                    try {
-                        repositoriesService.repository(snapshot.snapshotId().getRepository()).finalizeSnapshot(
-                                snapshot.snapshotId(), snapshot.indices(), snapshot.startTime(), ExceptionsHelper.detailedMessage(t), 0, Collections.<SnapshotShardFailure>emptyList());
-                    } catch (Throwable t2) {
-                        logger.warn("[{}] failed to close snapshot in repository", snapshot.snapshotId());
-                    }
-                    userCreateSnapshotListener.onFailure(t);
+                    removeSnapshotFromClusterState(snapshot.snapshotId(), null, t, new CleanupAfterErrorListener(snapshot, true, userCreateSnapshotListener, t));
                 }
 
                 @Override
@@ -395,17 +390,46 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
             });
         } catch (Throwable t) {
             logger.warn("failed to create snapshot [{}]", t, snapshot.snapshotId());
-            removeSnapshotFromClusterState(snapshot.snapshotId(), null, t);
-            if (snapshotCreated) {
+            removeSnapshotFromClusterState(snapshot.snapshotId(), null, t, new CleanupAfterErrorListener(snapshot, snapshotCreated, userCreateSnapshotListener, t));
+        }
+    }
+
+    private class CleanupAfterErrorListener implements ActionListener<SnapshotInfo> {
+
+        private final SnapshotsInProgress.Entry snapshot;
+        private final boolean snapshotCreated;
+        private final CreateSnapshotListener userCreateSnapshotListener;
+        private final Throwable t;
+
+        public CleanupAfterErrorListener(SnapshotsInProgress.Entry snapshot, boolean snapshotCreated, CreateSnapshotListener userCreateSnapshotListener, Throwable t) {
+            this.snapshot = snapshot;
+            this.snapshotCreated = snapshotCreated;
+            this.userCreateSnapshotListener = userCreateSnapshotListener;
+            this.t = t;
+        }
+
+        @Override
+        public void onResponse(SnapshotInfo snapshotInfo) {
+            cleanupAfterError();
+        }
+
+        @Override
+        public void onFailure(Throwable e) {
+            cleanupAfterError();
+        }
+
+        private void cleanupAfterError() {
+            if(snapshotCreated) {
                 try {
-                    repositoriesService.repository(snapshot.snapshotId().getRepository()).finalizeSnapshot(snapshot.snapshotId(), snapshot.indices(), snapshot.startTime(),
-                            ExceptionsHelper.detailedMessage(t), 0, Collections.<SnapshotShardFailure>emptyList());
+                    repositoriesService.repository(snapshot.snapshotId().getRepository()).finalizeSnapshot(
+                        snapshot.snapshotId(), snapshot.indices(), snapshot.startTime(), ExceptionsHelper.detailedMessage(t), 0, Collections.<SnapshotShardFailure>emptyList());
                 } catch (Throwable t2) {
                     logger.warn("[{}] failed to close snapshot in repository", snapshot.snapshotId());
                 }
             }
             userCreateSnapshotListener.onFailure(t);
         }
+
     }
 
     private SnapshotInfo inProgressSnapshot(SnapshotsInProgress.Entry entry) {
@@ -818,6 +842,19 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
      * @param t          exception if snapshot failed
      */
     private void removeSnapshotFromClusterState(final SnapshotId snapshotId, final SnapshotInfo snapshot, final Throwable t) {
+        removeSnapshotFromClusterState(snapshotId, snapshot, t, null);
+    }
+
+    /**
+     * Removes record of running snapshot from cluster state and notifies the listener when this action is complete
+     *
+     * @param snapshotId snapshot id
+     * @param snapshot   snapshot info if snapshot was successful
+     * @param t          exception if snapshot failed
+     * @param listener   listener to notify when snapshot information is removed from the cluster state
+     */
+    private void removeSnapshotFromClusterState(final SnapshotId snapshotId, final SnapshotInfo snapshot, final Throwable t,
+                                                @Nullable ActionListener<SnapshotInfo> listener) {
         clusterService.submitStateUpdateTask("remove snapshot metadata", new ClusterStateUpdateTask() {
             @Override
             public ClusterState execute(ClusterState currentState) {
@@ -843,6 +880,9 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
             @Override
             public void onFailure(String source, Throwable t) {
                 logger.warn("[{}] failed to remove snapshot metadata", t, snapshotId);
+                if (listener != null) {
+                    listener.onFailure(t);
+                }
             }
 
             @Override
@@ -858,7 +898,9 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
                         logger.warn("failed to notify listener [{}]", t, listener);
                     }
                 }
-
+                if (listener != null) {
+                    listener.onResponse(snapshot);
+                }
             }
         });
     }

From 61f40156d3c06cf3907b41542a050071b451fa3d Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 20 May 2016 12:15:30 -0400
Subject: [PATCH 0389/1311] Do not decode path when sending error

Today when sending a REST error to a client, we send the decoded
path. But decoding that path can already be the cause of the error in
which case decoding it again will just throw an exception leading to us
never sending an error back to the client. It would be better to send
the entire raw path to the client and that is what we do in this commit.

Relates #18477
---
 .../elasticsearch/rest/BytesRestResponse.java |  4 +--
 .../rest/BytesRestResponseTests.java          | 30 +++++++++++++++++++
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java b/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java
index 2194732445d..8cdbca512e7 100644
--- a/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java
+++ b/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java
@@ -123,9 +123,9 @@ public class BytesRestResponse extends RestResponse {
                 params =  new ToXContent.DelegatingMapParams(Collections.singletonMap(ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE, "false"), channel.request());
             } else {
                 if (status.getStatus() < 500) {
-                    SUPPRESSED_ERROR_LOGGER.debug("{} Params: {}", t, channel.request().path(), channel.request().params());
+                    SUPPRESSED_ERROR_LOGGER.debug("path: {}, params: {}", t, channel.request().rawPath(), channel.request().params());
                 } else {
-                    SUPPRESSED_ERROR_LOGGER.warn("{} Params: {}", t, channel.request().path(), channel.request().params());
+                    SUPPRESSED_ERROR_LOGGER.warn("path: {}, params: {}", t, channel.request().rawPath(), channel.request().params());
                 }
                 params = channel.request();
             }
diff --git a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java
index 306f3813e9b..aa3b11e6250 100644
--- a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java
@@ -25,6 +25,7 @@ import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.rest.support.RestUtils;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
@@ -35,8 +36,11 @@ import java.io.IOException;
 
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 /**
  *
@@ -147,6 +151,32 @@ public class BytesRestResponseTests extends ESTestCase {
         assertTrue(stackTrace.contains("Caused by: ParsingException[foobar]"));
     }
 
+    public void testResponseWhenPathContainsEncodingError() throws IOException {
+        final String path = "%a";
+        final RestRequest request = mock(RestRequest.class);
+        when(request.rawPath()).thenReturn(path);
+        final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RestUtils.decodeComponent(request.rawPath()));
+        final RestChannel channel = new DetailedExceptionRestChannel(request);
+        // if we try to decode the path, this will throw an IllegalArgumentException again
+        final BytesRestResponse response = new BytesRestResponse(channel, e);
+        assertNotNull(response.content());
+        final String content = response.content().toUtf8();
+        assertThat(content, containsString("\"type\":\"illegal_argument_exception\""));
+        assertThat(content, containsString("\"reason\":\"partial escape sequence at end of string: %a\""));
+        assertThat(content, containsString("\"status\":" + 400));
+    }
+
+    public void testResponseWhenInternalServerError() throws IOException {
+        final RestRequest request = new FakeRestRequest();
+        final RestChannel channel = new DetailedExceptionRestChannel(request);
+        final BytesRestResponse response = new BytesRestResponse(channel, new ElasticsearchException("simulated"));
+        assertNotNull(response.content());
+        final String content = response.content().toUtf8();
+        assertThat(content, containsString("\"type\":\"exception\""));
+        assertThat(content, containsString("\"reason\":\"simulated\""));
+        assertThat(content, containsString("\"status\":" + 500));
+    }
+
     public static class WithHeadersException extends ElasticsearchException {
 
         WithHeadersException() {

From 2b793c1e06766aebb930b2ad226ae3d81beb8cd3 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Fri, 20 May 2016 10:38:06 -0700
Subject: [PATCH 0390/1311] Fix boxing.

---
 .../elasticsearch/painless/MethodWriter.java  | 32 +++++++++++++++++++
 .../painless/WriterConstants.java             | 19 +++++++++++
 .../elasticsearch/painless/BasicAPITests.java | 24 ++++++++++----
 .../painless/DefOperationTests.java           |  4 +--
 .../elasticsearch/painless/EqualsTests.java   |  4 +--
 5 files changed, 72 insertions(+), 11 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index d17330eea8f..757041429a4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -33,6 +33,12 @@ import java.util.ArrayList;
 import java.util.Deque;
 import java.util.List;
 
+import static org.elasticsearch.painless.WriterConstants.BOOLEAN_OBJECT;
+import static org.elasticsearch.painless.WriterConstants.BOOLEAN_VALUE_OF;
+import static org.elasticsearch.painless.WriterConstants.BYTE_OBJECT;
+import static org.elasticsearch.painless.WriterConstants.BYTE_VALUE_OF;
+import static org.elasticsearch.painless.WriterConstants.CHARACTER_OBJECT;
+import static org.elasticsearch.painless.WriterConstants.CHARACTER_VALUE_OF;
 import static org.elasticsearch.painless.WriterConstants.CHAR_TO_STRING;
 import static org.elasticsearch.painless.WriterConstants.DEF_ADD_CALL;
 import static org.elasticsearch.painless.WriterConstants.DEF_AND_CALL;
@@ -61,9 +67,19 @@ import static org.elasticsearch.painless.WriterConstants.DEF_TO_SHORT_IMPLICIT;
 import static org.elasticsearch.painless.WriterConstants.DEF_UTIL_TYPE;
 import static org.elasticsearch.painless.WriterConstants.DEF_USH_CALL;
 import static org.elasticsearch.painless.WriterConstants.DEF_XOR_CALL;
+import static org.elasticsearch.painless.WriterConstants.DOUBLE_OBJECT;
+import static org.elasticsearch.painless.WriterConstants.DOUBLE_VALUE_OF;
+import static org.elasticsearch.painless.WriterConstants.FLOAT_OBJECT;
+import static org.elasticsearch.painless.WriterConstants.FLOAT_VALUE_OF;
 import static org.elasticsearch.painless.WriterConstants.INDY_STRING_CONCAT_BOOTSTRAP_HANDLE;
+import static org.elasticsearch.painless.WriterConstants.INTEGER_OBJECT;
+import static org.elasticsearch.painless.WriterConstants.INTEGER_VALUE_OF;
+import static org.elasticsearch.painless.WriterConstants.LONG_OBJECT;
+import static org.elasticsearch.painless.WriterConstants.LONG_VALUE_OF;
 import static org.elasticsearch.painless.WriterConstants.MAX_INDY_STRING_CONCAT_ARGS;
 import static org.elasticsearch.painless.WriterConstants.PAINLESS_ERROR_TYPE;
+import static org.elasticsearch.painless.WriterConstants.SHORT_OBJECT;
+import static org.elasticsearch.painless.WriterConstants.SHORT_VALUE_OF;
 import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_BOOLEAN;
 import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_CHAR;
 import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_DOUBLE;
@@ -184,6 +200,22 @@ public final class MethodWriter extends GeneratorAdapter {
         }
     }
 
+    @Override
+    public void box(final org.objectweb.asm.Type type) {
+        switch (type.getSort()) {
+            case org.objectweb.asm.Type.BOOLEAN: invokeStatic(BOOLEAN_OBJECT  , BOOLEAN_VALUE_OF);   break;
+            case org.objectweb.asm.Type.BYTE:    invokeStatic(BYTE_OBJECT     , BYTE_VALUE_OF);      break;
+            case org.objectweb.asm.Type.SHORT:   invokeStatic(SHORT_OBJECT    , SHORT_VALUE_OF);     break;
+            case org.objectweb.asm.Type.CHAR:    invokeStatic(CHARACTER_OBJECT, CHARACTER_VALUE_OF); break;
+            case org.objectweb.asm.Type.INT:     invokeStatic(INTEGER_OBJECT  , INTEGER_VALUE_OF);   break;
+            case org.objectweb.asm.Type.LONG:    invokeStatic(LONG_OBJECT     , LONG_VALUE_OF);      break;
+            case org.objectweb.asm.Type.FLOAT:   invokeStatic(FLOAT_OBJECT    , FLOAT_VALUE_OF);     break;
+            case org.objectweb.asm.Type.DOUBLE:  invokeStatic(DOUBLE_OBJECT   , DOUBLE_VALUE_OF);    break;
+            default:
+                throw new IllegalArgumentException("Illegal tree structure.");
+        }
+    }
+
     public void writeBranch(final Label tru, final Label fals) {
         if (tru != null) {
             visitJumpInsn(Opcodes.IFNE, tru);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index b50ec7b8040..c9d24212bbc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -138,6 +138,25 @@ public final class WriterConstants {
     public final static Method CHECKEQUALS              =
         getAsmMethod(boolean.class, "checkEquals",              Object.class, Object.class);
 
+    public final static Type BOOLEAN_OBJECT   = Type.getType(Boolean.class);
+    public final static Type BYTE_OBJECT      = Type.getType(Byte.class);
+    public final static Type SHORT_OBJECT     = Type.getType(Short.class);
+    public final static Type CHARACTER_OBJECT = Type.getType(Character.class);
+    public final static Type INTEGER_OBJECT   = Type.getType(Integer.class);
+    public final static Type LONG_OBJECT      = Type.getType(Long.class);
+    public final static Type FLOAT_OBJECT     = Type.getType(Float.class);
+    public final static Type DOUBLE_OBJECT    = Type.getType(Double.class);
+
+    /** box methods to replace the GeneratorAdapter's slow way of boxing */
+    public final static Method BOOLEAN_VALUE_OF   = getAsmMethod(Boolean.class  , "valueOf", boolean.class);
+    public final static Method BYTE_VALUE_OF      = getAsmMethod(Byte.class     , "valueOf", byte.class);
+    public final static Method SHORT_VALUE_OF     = getAsmMethod(Short.class    , "valueOf", short.class);
+    public final static Method CHARACTER_VALUE_OF = getAsmMethod(Character.class, "valueOf", char.class);
+    public final static Method INTEGER_VALUE_OF   = getAsmMethod(Integer.class  , "valueOf", int.class);
+    public final static Method LONG_VALUE_OF      = getAsmMethod(Long.class     , "valueOf", long.class);
+    public final static Method FLOAT_VALUE_OF     = getAsmMethod(Float.class    , "valueOf", float.class);
+    public final static Method DOUBLE_VALUE_OF    = getAsmMethod(Double.class   , "valueOf", double.class);
+
     private static Method getAsmMethod(final Class<?> rtype, final String name, final Class<?>... ptypes) {
         return new Method(name, MethodType.methodType(rtype, ptypes).toMethodDescriptorString());
     }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
index b7479b505fe..338e3f00113 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
@@ -45,39 +45,49 @@ public class BasicAPITests extends ScriptTestCase {
         assertEquals(3, exec("Map x = new HashMap(); x.put(2, 2); x.put(3, 3); x.put(-2, -2); Iterator y = x.values().iterator(); " +
             "int total = 0; while (y.hasNext()) total += (int)y.next(); return total;"));
     }
-    
+
     /** Test loads and stores with a map */
     public void testMapLoadStore() {
         assertEquals(5, exec("def x = new HashMap(); x.abc = 5; return x.abc;"));
         assertEquals(5, exec("def x = new HashMap(); x['abc'] = 5; return x['abc'];"));
     }
-    
+
     /** Test loads and stores with a list */
     public void testListLoadStore() {
         assertEquals(5, exec("def x = new ArrayList(); x.add(3); x.0 = 5; return x.0;"));
         assertEquals(5, exec("def x = new ArrayList(); x.add(3); x[0] = 5; return x[0];"));
     }
-    
+
     /** Test shortcut for getters with isXXXX */
     public void testListEmpty() {
         assertEquals(true, exec("def x = new ArrayList(); return x.empty;"));
         assertEquals(true, exec("def x = new HashMap(); return x.empty;"));
     }
-    
+
     /** Test list method invocation */
     public void testListGet() {
         assertEquals(5, exec("def x = new ArrayList(); x.add(5); return x.get(0);"));
         assertEquals(5, exec("def x = new ArrayList(); x.add(5); def index = 0; return x.get(index);"));
     }
-    
+
     public void testListAsArray() {
         assertEquals(1, exec("def x = new ArrayList(); x.add(5); return x.length"));
         assertEquals(5, exec("def x = new ArrayList(); x.add(5); return x[0]"));
         assertEquals(1, exec("List x = new ArrayList(); x.add('Hallo'); return x.length"));
     }
-    
+
     public void testDefAssignments() {
         assertEquals(2, exec("int x; def y = 2.0; x = (int)y;"));
     }
-    
+
+    public void testInternalBoxing() {
+        assertBytecodeExists("def x = true", "INVOKESTATIC java/lang/Boolean.valueOf (Z)Ljava/lang/Boolean;");
+        assertBytecodeExists("def x = (byte)1", "INVOKESTATIC java/lang/Byte.valueOf (B)Ljava/lang/Byte;");
+        assertBytecodeExists("def x = (short)1", "INVOKESTATIC java/lang/Short.valueOf (S)Ljava/lang/Short;");
+        assertBytecodeExists("def x = (char)1", "INVOKESTATIC java/lang/Character.valueOf (C)Ljava/lang/Character;");
+        assertBytecodeExists("def x = 1", "INVOKESTATIC java/lang/Integer.valueOf (I)Ljava/lang/Integer;");
+        assertBytecodeExists("def x = 1L", "INVOKESTATIC java/lang/Long.valueOf (J)Ljava/lang/Long;");
+        assertBytecodeExists("def x = 1F", "INVOKESTATIC java/lang/Float.valueOf (F)Ljava/lang/Float;");
+        assertBytecodeExists("def x = 1D", "INVOKESTATIC java/lang/Double.valueOf (D)Ljava/lang/Double;");
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
index 1b548efa3b8..9f171a96889 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
@@ -799,7 +799,7 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(false, exec("def x = (byte)7; def y = (int)7; return x === y"));
         assertEquals(false, exec("def x = (short)6; def y = (int)6; return x === y"));
         assertEquals(false, exec("def x = (char)5; def y = (int)5; return x === y"));
-        assertEquals(false, exec("def x = (int)4; def y = (int)4; return x === y"));
+        assertEquals(true, exec("def x = (int)4; def y = (int)4; return x === y"));
         assertEquals(false, exec("def x = (long)5; def y = (int)3; return x === y"));
         assertEquals(false, exec("def x = (float)6; def y = (int)2; return x === y"));
         assertEquals(false, exec("def x = (double)7; def y = (int)1; return x === y"));
@@ -837,7 +837,7 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(true, exec("def x = (byte)7; def y = (int)7; return x !== y"));
         assertEquals(true, exec("def x = (short)6; def y = (int)6; return x !== y"));
         assertEquals(true, exec("def x = (char)5; def y = (int)5; return x !== y"));
-        assertEquals(true, exec("def x = (int)4; def y = (int)4; return x !== y"));
+        assertEquals(false, exec("def x = (int)4; def y = (int)4; return x !== y"));
         assertEquals(true, exec("def x = (long)5; def y = (int)3; return x !== y"));
         assertEquals(true, exec("def x = (float)6; def y = (int)2; return x !== y"));
         assertEquals(true, exec("def x = (double)7; def y = (int)1; return x !== y"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java
index 59f9aadc6aa..cc308c70ded 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java
@@ -118,7 +118,7 @@ public class EqualsTests extends ScriptTestCase {
     public void testBranchEquals() {
         assertEquals(0, exec("def a = (char)'a'; def b = (char)'b'; if (a == b) return 1; else return 0;"));
         assertEquals(1, exec("def a = (char)'a'; def b = (char)'a'; if (a == b) return 1; else return 0;"));
-        assertEquals(0, exec("def a = 1; def b = 1; if (a === b) return 1; else return 0;"));
+        assertEquals(1, exec("def a = 1; def b = 1; if (a === b) return 1; else return 0;"));
         assertEquals(0, exec("def a = (char)'a'; def b = (char)'a'; if (a === b) return 1; else return 0;"));
         assertEquals(1, exec("def a = (char)'a'; Object b = a; if (a === b) return 1; else return 0;"));
         assertEquals(1, exec("def a = 1; Number b = a; Number c = a; if (c === b) return 1; else return 0;"));
@@ -128,7 +128,7 @@ public class EqualsTests extends ScriptTestCase {
     public void testBranchNotEquals() {
         assertEquals(1, exec("def a = (char)'a'; def b = (char)'b'; if (a != b) return 1; else return 0;"));
         assertEquals(0, exec("def a = (char)'a'; def b = (char)'a'; if (a != b) return 1; else return 0;"));
-        assertEquals(1, exec("def a = 1; def b = 1; if (a !== b) return 1; else return 0;"));
+        assertEquals(0, exec("def a = 1; def b = 1; if (a !== b) return 1; else return 0;"));
         assertEquals(1, exec("def a = (char)'a'; def b = (char)'a'; if (a !== b) return 1; else return 0;"));
         assertEquals(0, exec("def a = (char)'a'; Object b = a; if (a !== b) return 1; else return 0;"));
         assertEquals(0, exec("def a = 1; Number b = a; Number c = a; if (c !== b) return 1; else return 0;"));

From d836194095a3be1eee117d9303a7055db8feedb9 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Fri, 20 May 2016 11:04:09 -0700
Subject: [PATCH 0391/1311] Fixed tests and added a comment to the box method.

---
 .../java/org/elasticsearch/painless/MethodWriter.java     | 8 ++++++--
 .../test/java/org/elasticsearch/painless/EqualsTests.java | 4 ++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index 757041429a4..b03de2ef4df 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -200,9 +200,15 @@ public final class MethodWriter extends GeneratorAdapter {
         }
     }
 
+    /**
+     * ASM does boxing in an evil way to be compatible with Java prior to verison 5.
+     * Supporting versions prior to 5 is not a requirement of this project, so the modern
+     * boxing methods are used instead.
+     */
     @Override
     public void box(final org.objectweb.asm.Type type) {
         switch (type.getSort()) {
+            case org.objectweb.asm.Type.VOID:    visitInsn(Opcodes.ACONST_NULL);                     break;
             case org.objectweb.asm.Type.BOOLEAN: invokeStatic(BOOLEAN_OBJECT  , BOOLEAN_VALUE_OF);   break;
             case org.objectweb.asm.Type.BYTE:    invokeStatic(BYTE_OBJECT     , BYTE_VALUE_OF);      break;
             case org.objectweb.asm.Type.SHORT:   invokeStatic(SHORT_OBJECT    , SHORT_VALUE_OF);     break;
@@ -211,8 +217,6 @@ public final class MethodWriter extends GeneratorAdapter {
             case org.objectweb.asm.Type.LONG:    invokeStatic(LONG_OBJECT     , LONG_VALUE_OF);      break;
             case org.objectweb.asm.Type.FLOAT:   invokeStatic(FLOAT_OBJECT    , FLOAT_VALUE_OF);     break;
             case org.objectweb.asm.Type.DOUBLE:  invokeStatic(DOUBLE_OBJECT   , DOUBLE_VALUE_OF);    break;
-            default:
-                throw new IllegalArgumentException("Illegal tree structure.");
         }
     }
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java
index cc308c70ded..7e4448495a9 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/EqualsTests.java
@@ -119,7 +119,7 @@ public class EqualsTests extends ScriptTestCase {
         assertEquals(0, exec("def a = (char)'a'; def b = (char)'b'; if (a == b) return 1; else return 0;"));
         assertEquals(1, exec("def a = (char)'a'; def b = (char)'a'; if (a == b) return 1; else return 0;"));
         assertEquals(1, exec("def a = 1; def b = 1; if (a === b) return 1; else return 0;"));
-        assertEquals(0, exec("def a = (char)'a'; def b = (char)'a'; if (a === b) return 1; else return 0;"));
+        assertEquals(1, exec("def a = (char)'a'; def b = (char)'a'; if (a === b) return 1; else return 0;"));
         assertEquals(1, exec("def a = (char)'a'; Object b = a; if (a === b) return 1; else return 0;"));
         assertEquals(1, exec("def a = 1; Number b = a; Number c = a; if (c === b) return 1; else return 0;"));
         assertEquals(0, exec("def a = 1; Object b = new HashMap(); if (a === (Object)b) return 1; else return 0;"));
@@ -129,7 +129,7 @@ public class EqualsTests extends ScriptTestCase {
         assertEquals(1, exec("def a = (char)'a'; def b = (char)'b'; if (a != b) return 1; else return 0;"));
         assertEquals(0, exec("def a = (char)'a'; def b = (char)'a'; if (a != b) return 1; else return 0;"));
         assertEquals(0, exec("def a = 1; def b = 1; if (a !== b) return 1; else return 0;"));
-        assertEquals(1, exec("def a = (char)'a'; def b = (char)'a'; if (a !== b) return 1; else return 0;"));
+        assertEquals(0, exec("def a = (char)'a'; def b = (char)'a'; if (a !== b) return 1; else return 0;"));
         assertEquals(0, exec("def a = (char)'a'; Object b = a; if (a !== b) return 1; else return 0;"));
         assertEquals(0, exec("def a = 1; Number b = a; Number c = a; if (c !== b) return 1; else return 0;"));
         assertEquals(1, exec("def a = 1; Object b = new HashMap(); if (a !== (Object)b) return 1; else return 0;"));

From b493f0defef0a6dc07a71d428c7702be85a3182e Mon Sep 17 00:00:00 2001
From: Britta Weber <britta.weber@elasticsearch.com>
Date: Fri, 20 May 2016 19:28:04 +0200
Subject: [PATCH 0392/1311] skip all geo point queries in plain highlighter

Geo queries and plain highlighter do not seem to work well
together (https://issues.apache.org/jira/browse/LUCENE-7293)
so we need to skip all geo related queries when we highlight.

closes #17537
---
 .../search/highlight/CustomQueryScorer.java   |  9 +++-
 .../search/highlight/HighlighterSearchIT.java | 21 ++++++--
 .../highlight/PlainHighlighterTests.java      | 48 +++++++++++++++++++
 3 files changed, 72 insertions(+), 6 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/highlight/CustomQueryScorer.java b/core/src/main/java/org/elasticsearch/search/highlight/CustomQueryScorer.java
index 7a15f67dbd6..03c7a535140 100644
--- a/core/src/main/java/org/elasticsearch/search/highlight/CustomQueryScorer.java
+++ b/core/src/main/java/org/elasticsearch/search/highlight/CustomQueryScorer.java
@@ -24,6 +24,7 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.search.highlight.QueryScorer;
 import org.apache.lucene.search.highlight.WeightedSpanTerm;
 import org.apache.lucene.search.highlight.WeightedSpanTermExtractor;
+import org.apache.lucene.spatial.geopoint.search.GeoPointInBBoxQuery;
 import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery;
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 
@@ -87,6 +88,12 @@ public final class CustomQueryScorer extends QueryScorer {
             }
         }
 
+        protected void extract(Query query, float boost, Map<String, WeightedSpanTerm> terms) throws IOException {
+            // skip all geo queries, see https://issues.apache.org/jira/browse/LUCENE-7293 and
+            // https://github.com/elastic/elasticsearch/issues/17537
+            if (query instanceof GeoPointInBBoxQuery == false) {
+                super.extract(query, boost, terms);
+            }
+        }
     }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java b/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java
index 869182cb51a..ab4d7655aa6 100644
--- a/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java
@@ -2553,9 +2553,9 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         assertHighlight(response, 0, "field1", 0, 1, highlightedMatcher);
     }
 
-    public void testGeoFieldHighlighting() throws IOException {
+    public void testGeoFieldHighlightingWithDifferentHighlighters() throws IOException {
         // check that we do not get an exception for geo_point fields in case someone tries to highlight
-        // it accidential with a wildcard
+        // it accidentially with a wildcard
         // see https://github.com/elastic/elasticsearch/issues/17537
         XContentBuilder mappings = jsonBuilder();
         mappings.startObject();
@@ -2563,6 +2563,12 @@ public class HighlighterSearchIT extends ESIntegTestCase {
             .startObject("properties")
             .startObject("geo_point")
             .field("type", "geo_point")
+            .field("geohash", true)
+            .endObject()
+            .startObject("text")
+            .field("type", "text")
+            .field("term_vector", "with_positions_offsets_payloads")
+            .field("index_options", "offsets")
             .endObject()
             .endObject()
             .endObject();
@@ -2572,14 +2578,19 @@ public class HighlighterSearchIT extends ESIntegTestCase {
         ensureYellow();
 
         client().prepareIndex("test", "type", "1")
-            .setSource(jsonBuilder().startObject().field("geo_point", "60.12,100.34").endObject())
+            .setSource(jsonBuilder().startObject().field("text", "Arbitrary text field which will should not cause a failure").endObject())
             .get();
         refresh();
+        String highlighterType = randomFrom("plain", "fvh", "postings");
+        QueryBuilder query = QueryBuilders.boolQuery().should(QueryBuilders.geoBoundingBoxQuery("geo_point")
+            .setCorners(61.10078883158897, -170.15625, -64.92354174306496, 118.47656249999999))
+            .should(QueryBuilders.termQuery("text", "failure"));
         SearchResponse search = client().prepareSearch().setSource(
-            new SearchSourceBuilder().query(QueryBuilders.geoBoundingBoxQuery("geo_point").setCorners(61.10078883158897, -170.15625,
-                -64.92354174306496, 118.47656249999999)).highlighter(new HighlightBuilder().field("*"))).get();
+            new SearchSourceBuilder().query(query)
+                .highlighter(new HighlightBuilder().field("*").highlighterType(highlighterType))).get();
         assertNoFailures(search);
         assertThat(search.getHits().totalHits(), equalTo(1L));
+        assertThat(search.getHits().getAt(0).highlightFields().get("text").fragments().length, equalTo(1));
     }
 
     public void testKeywordFieldHighlighting() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/search/highlight/PlainHighlighterTests.java b/core/src/test/java/org/elasticsearch/search/highlight/PlainHighlighterTests.java
index 5156209d6f1..9c14e2116d1 100644
--- a/core/src/test/java/org/elasticsearch/search/highlight/PlainHighlighterTests.java
+++ b/core/src/test/java/org/elasticsearch/search/highlight/PlainHighlighterTests.java
@@ -19,12 +19,28 @@
 
 package org.elasticsearch.search.highlight;
 
+import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
 import org.apache.lucene.search.highlight.QueryScorer;
+import org.apache.lucene.spatial.geopoint.search.GeoPointDistanceQuery;
+import org.apache.lucene.spatial.geopoint.search.GeoPointInBBoxQuery;
+import org.apache.lucene.spatial.geopoint.search.GeoPointInPolygonQuery;
 import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.index.analysis.FieldNameAnalyzer;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
 
 public class PlainHighlighterTests extends LuceneTestCase {
 
@@ -39,4 +55,36 @@ public class PlainHighlighterTests extends LuceneTestCase {
         assertArrayEquals(new String[] {"bar <B>foo</B> <B>bar</B> foo"}, frags);
     }
 
+    public void checkGeoQueryHighlighting(Query geoQuery) throws IOException, InvalidTokenOffsetsException {
+        Map analysers = new HashMap<String, Analyzer>();
+        analysers.put("text", new StandardAnalyzer());
+        FieldNameAnalyzer fieldNameAnalyzer = new FieldNameAnalyzer(analysers);
+        Query termQuery = new TermQuery(new Term("text", "failure"));
+        Query boolQuery = new BooleanQuery.Builder().add(new BooleanClause(geoQuery, BooleanClause.Occur.SHOULD))
+            .add(new BooleanClause(termQuery, BooleanClause.Occur.SHOULD)).build();
+        org.apache.lucene.search.highlight.Highlighter highlighter =
+            new org.apache.lucene.search.highlight.Highlighter(new CustomQueryScorer(boolQuery));
+        String fragment = highlighter.getBestFragment(fieldNameAnalyzer.tokenStream("text", "Arbitrary text field which should not cause " +
+            "a failure"), "Arbitrary text field which should not cause a failure");
+        assertThat(fragment, equalTo("Arbitrary text field which should not cause a <B>failure</B>"));
+        // TODO: This test will fail if we pass in an instance of GeoPointInBBoxQueryImpl too. Should we also find a way to work around that
+        // or can the query not be rewritten before it is passed into the highlighter?
+    }
+
+    public void testGeoPointInBBoxQueryHighlighting() throws IOException, InvalidTokenOffsetsException {
+        Query geoQuery = new GeoPointDistanceQuery("geo_point", -64.92354174306496, -170.15625, 5576757);
+        checkGeoQueryHighlighting(geoQuery);
+    }
+
+    public void testGeoPointDistanceQueryHighlighting() throws IOException, InvalidTokenOffsetsException {
+        Query geoQuery = new GeoPointInBBoxQuery("geo_point", -64.92354174306496, 61.10078883158897, -170.15625, 118.47656249999999);
+        checkGeoQueryHighlighting(geoQuery);
+    }
+
+    public void testGeoPointInPolygonQueryHighlighting() throws IOException, InvalidTokenOffsetsException {
+        double[] polyLats = new double[]{0, 60, 0, 0};
+        double[] polyLons = new double[]{0, 60, 90, 0};
+        Query geoQuery = new GeoPointInPolygonQuery("geo_point", polyLats, polyLons);
+        checkGeoQueryHighlighting(geoQuery);
+    }
 }

From 0e24ed49e699e4aa6890daaaf720ee5cc3a6f5e8 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Fri, 20 May 2016 11:13:43 -0700
Subject: [PATCH 0393/1311] Use the build-in valueOf method instead of the
 custom one.

---
 .../elasticsearch/painless/MethodWriter.java  | 42 ++-----------------
 .../painless/WriterConstants.java             | 10 -----
 2 files changed, 3 insertions(+), 49 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index b03de2ef4df..f0477478fda 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -33,12 +33,6 @@ import java.util.ArrayList;
 import java.util.Deque;
 import java.util.List;
 
-import static org.elasticsearch.painless.WriterConstants.BOOLEAN_OBJECT;
-import static org.elasticsearch.painless.WriterConstants.BOOLEAN_VALUE_OF;
-import static org.elasticsearch.painless.WriterConstants.BYTE_OBJECT;
-import static org.elasticsearch.painless.WriterConstants.BYTE_VALUE_OF;
-import static org.elasticsearch.painless.WriterConstants.CHARACTER_OBJECT;
-import static org.elasticsearch.painless.WriterConstants.CHARACTER_VALUE_OF;
 import static org.elasticsearch.painless.WriterConstants.CHAR_TO_STRING;
 import static org.elasticsearch.painless.WriterConstants.DEF_ADD_CALL;
 import static org.elasticsearch.painless.WriterConstants.DEF_AND_CALL;
@@ -64,22 +58,12 @@ import static org.elasticsearch.painless.WriterConstants.DEF_TO_LONG_EXPLICIT;
 import static org.elasticsearch.painless.WriterConstants.DEF_TO_LONG_IMPLICIT;
 import static org.elasticsearch.painless.WriterConstants.DEF_TO_SHORT_EXPLICIT;
 import static org.elasticsearch.painless.WriterConstants.DEF_TO_SHORT_IMPLICIT;
-import static org.elasticsearch.painless.WriterConstants.DEF_UTIL_TYPE;
 import static org.elasticsearch.painless.WriterConstants.DEF_USH_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_UTIL_TYPE;
 import static org.elasticsearch.painless.WriterConstants.DEF_XOR_CALL;
-import static org.elasticsearch.painless.WriterConstants.DOUBLE_OBJECT;
-import static org.elasticsearch.painless.WriterConstants.DOUBLE_VALUE_OF;
-import static org.elasticsearch.painless.WriterConstants.FLOAT_OBJECT;
-import static org.elasticsearch.painless.WriterConstants.FLOAT_VALUE_OF;
 import static org.elasticsearch.painless.WriterConstants.INDY_STRING_CONCAT_BOOTSTRAP_HANDLE;
-import static org.elasticsearch.painless.WriterConstants.INTEGER_OBJECT;
-import static org.elasticsearch.painless.WriterConstants.INTEGER_VALUE_OF;
-import static org.elasticsearch.painless.WriterConstants.LONG_OBJECT;
-import static org.elasticsearch.painless.WriterConstants.LONG_VALUE_OF;
 import static org.elasticsearch.painless.WriterConstants.MAX_INDY_STRING_CONCAT_ARGS;
 import static org.elasticsearch.painless.WriterConstants.PAINLESS_ERROR_TYPE;
-import static org.elasticsearch.painless.WriterConstants.SHORT_OBJECT;
-import static org.elasticsearch.painless.WriterConstants.SHORT_VALUE_OF;
 import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_BOOLEAN;
 import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_CHAR;
 import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_DOUBLE;
@@ -175,11 +159,11 @@ public final class MethodWriter extends GeneratorAdapter {
                 writeCast(from, to);
                 unbox(to.type);
             } else if (cast.boxFrom) {
-                box(from.type);
+                valueOf(from.type);
                 writeCast(from, to);
             } else if (cast.boxTo) {
                 writeCast(from, to);
-                box(to.type);
+                valueOf(to.type);
             } else {
                 writeCast(from, to);
             }
@@ -200,26 +184,6 @@ public final class MethodWriter extends GeneratorAdapter {
         }
     }
 
-    /**
-     * ASM does boxing in an evil way to be compatible with Java prior to verison 5.
-     * Supporting versions prior to 5 is not a requirement of this project, so the modern
-     * boxing methods are used instead.
-     */
-    @Override
-    public void box(final org.objectweb.asm.Type type) {
-        switch (type.getSort()) {
-            case org.objectweb.asm.Type.VOID:    visitInsn(Opcodes.ACONST_NULL);                     break;
-            case org.objectweb.asm.Type.BOOLEAN: invokeStatic(BOOLEAN_OBJECT  , BOOLEAN_VALUE_OF);   break;
-            case org.objectweb.asm.Type.BYTE:    invokeStatic(BYTE_OBJECT     , BYTE_VALUE_OF);      break;
-            case org.objectweb.asm.Type.SHORT:   invokeStatic(SHORT_OBJECT    , SHORT_VALUE_OF);     break;
-            case org.objectweb.asm.Type.CHAR:    invokeStatic(CHARACTER_OBJECT, CHARACTER_VALUE_OF); break;
-            case org.objectweb.asm.Type.INT:     invokeStatic(INTEGER_OBJECT  , INTEGER_VALUE_OF);   break;
-            case org.objectweb.asm.Type.LONG:    invokeStatic(LONG_OBJECT     , LONG_VALUE_OF);      break;
-            case org.objectweb.asm.Type.FLOAT:   invokeStatic(FLOAT_OBJECT    , FLOAT_VALUE_OF);     break;
-            case org.objectweb.asm.Type.DOUBLE:  invokeStatic(DOUBLE_OBJECT   , DOUBLE_VALUE_OF);    break;
-        }
-    }
-
     public void writeBranch(final Label tru, final Label fals) {
         if (tru != null) {
             visitJumpInsn(Opcodes.IFNE, tru);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index c9d24212bbc..47927438804 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -147,16 +147,6 @@ public final class WriterConstants {
     public final static Type FLOAT_OBJECT     = Type.getType(Float.class);
     public final static Type DOUBLE_OBJECT    = Type.getType(Double.class);
 
-    /** box methods to replace the GeneratorAdapter's slow way of boxing */
-    public final static Method BOOLEAN_VALUE_OF   = getAsmMethod(Boolean.class  , "valueOf", boolean.class);
-    public final static Method BYTE_VALUE_OF      = getAsmMethod(Byte.class     , "valueOf", byte.class);
-    public final static Method SHORT_VALUE_OF     = getAsmMethod(Short.class    , "valueOf", short.class);
-    public final static Method CHARACTER_VALUE_OF = getAsmMethod(Character.class, "valueOf", char.class);
-    public final static Method INTEGER_VALUE_OF   = getAsmMethod(Integer.class  , "valueOf", int.class);
-    public final static Method LONG_VALUE_OF      = getAsmMethod(Long.class     , "valueOf", long.class);
-    public final static Method FLOAT_VALUE_OF     = getAsmMethod(Float.class    , "valueOf", float.class);
-    public final static Method DOUBLE_VALUE_OF    = getAsmMethod(Double.class   , "valueOf", double.class);
-
     private static Method getAsmMethod(final Class<?> rtype, final String name, final Class<?>... ptypes) {
         return new Method(name, MethodType.methodType(rtype, ptypes).toMethodDescriptorString());
     }

From b156438957f22183b8c0ae3b45204af599c91e3f Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Fri, 20 May 2016 11:22:18 -0700
Subject: [PATCH 0394/1311] Proxy box method to use valueOf.

---
 .../org/elasticsearch/painless/MethodWriter.java     | 12 ++++++++++--
 .../org/elasticsearch/painless/WriterConstants.java  | 12 +-----------
 2 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index f0477478fda..90c02b7e801 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -159,11 +159,11 @@ public final class MethodWriter extends GeneratorAdapter {
                 writeCast(from, to);
                 unbox(to.type);
             } else if (cast.boxFrom) {
-                valueOf(from.type);
+                box(from.type);
                 writeCast(from, to);
             } else if (cast.boxTo) {
                 writeCast(from, to);
-                valueOf(to.type);
+                box(to.type);
             } else {
                 writeCast(from, to);
             }
@@ -184,6 +184,14 @@ public final class MethodWriter extends GeneratorAdapter {
         }
     }
 
+    /**
+     * Proxy the box method to use valueOf instead to ensure that the modern boxing methods are used.
+     */
+    @Override
+    public void box(org.objectweb.asm.Type type) {
+        valueOf(type);
+    }
+
     public void writeBranch(final Label tru, final Label fals) {
         if (tru != null) {
             visitJumpInsn(Opcodes.IFNE, tru);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index 47927438804..410c06e6fd7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -135,17 +135,7 @@ public final class WriterConstants {
     public final static Method STRINGBUILDER_APPEND_OBJECT  = getAsmMethod(StringBuilder.class, "append", Object.class);
     public final static Method STRINGBUILDER_TOSTRING       = getAsmMethod(String.class, "toString");
 
-    public final static Method CHECKEQUALS              =
-        getAsmMethod(boolean.class, "checkEquals",              Object.class, Object.class);
-
-    public final static Type BOOLEAN_OBJECT   = Type.getType(Boolean.class);
-    public final static Type BYTE_OBJECT      = Type.getType(Byte.class);
-    public final static Type SHORT_OBJECT     = Type.getType(Short.class);
-    public final static Type CHARACTER_OBJECT = Type.getType(Character.class);
-    public final static Type INTEGER_OBJECT   = Type.getType(Integer.class);
-    public final static Type LONG_OBJECT      = Type.getType(Long.class);
-    public final static Type FLOAT_OBJECT     = Type.getType(Float.class);
-    public final static Type DOUBLE_OBJECT    = Type.getType(Double.class);
+    public final static Method CHECKEQUALS = getAsmMethod(boolean.class, "checkEquals", Object.class, Object.class);
 
     private static Method getAsmMethod(final Class<?> rtype, final String name, final Class<?>... ptypes) {
         return new Method(name, MethodType.methodType(rtype, ptypes).toMethodDescriptorString());

From 35e705877be8e18946bb110a6f3e38a58905a793 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Fri, 20 May 2016 20:37:45 +0200
Subject: [PATCH 0395/1311] Limit retries of failed allocations per index
 (#18467)

Today if a shard fails during initialization phase due to misconfiguration, broken disks,
missing analyzers, not installed plugins etc. elasticsaerch keeps on trying to initialize
or rather allocate that shard. Yet, in the worst case scenario this ends in an endless
allocation loop. To prevent this loop and all it's sideeffects like spamming log files over
and over again this commit adds an allocation decider that stops allocating a shard that
failed more than N times in a row to allocate. The number or retries can be configured via
`index.allocation.max_retry` and it's default is set to `5`. Once the setting is updated
shards with less failures than the number set per index will be allowed to allocate again.

Internally we maintain a counter on the UnassignedInfo that is reset to `0` once the shards
has been started.

Relates to #18417
---
 ...ansportClusterAllocationExplainAction.java |   2 +-
 .../reroute/ClusterRerouteRequest.java        |  33 ++-
 .../reroute/ClusterRerouteRequestBuilder.java |   9 +
 .../TransportClusterRerouteAction.java        |  86 ++++---
 .../elasticsearch/cluster/ClusterModule.java  |   2 +
 .../cluster/routing/UnassignedInfo.java       |  32 ++-
 .../routing/allocation/AllocationService.java |  22 +-
 .../allocation/FailedRerouteAllocation.java   |   2 +-
 .../routing/allocation/RoutingAllocation.java |   9 +-
 .../allocation/StartedRerouteAllocation.java  |   2 +-
 ...AllocateEmptyPrimaryAllocationCommand.java |   2 +-
 .../decider/MaxRetryAllocationDecider.java    |  83 +++++++
 .../common/settings/IndexScopedSettings.java  |   3 +-
 .../gateway/ReplicaShardAllocator.java        |   2 +-
 .../reroute/RestClusterRerouteAction.java     |   1 +
 .../cluster/reroute/ClusterRerouteTests.java  | 181 +++++++++++++++
 .../cluster/routing/UnassignedInfoTests.java  |  16 +-
 .../allocation/AllocationCommandsTests.java   |  46 ++--
 .../allocation/AwarenessAllocationTests.java  |   2 +-
 .../allocation/DeadNodesAllocationTests.java  |   8 +-
 .../ExpectedShardSizeAllocationTests.java     |   2 +-
 .../allocation/FailedShardsRoutingTests.java  |   8 +-
 .../MaxRetryAllocationDeciderTests.java       | 210 ++++++++++++++++++
 .../NodeVersionAllocationDeciderTests.java    |   4 +-
 .../allocation/ThrottlingAllocationTests.java |   6 +-
 .../decider/DiskThresholdDeciderTests.java    |  14 +-
 .../DiskThresholdDeciderUnitTests.java        |   4 +-
 .../gateway/PrimaryShardAllocatorTests.java   |  18 +-
 .../gateway/ReplicaShardAllocatorTests.java   |   4 +-
 .../indices/state/RareClusterStateIT.java     |   2 +-
 .../SharedClusterSnapshotRestoreIT.java       |   2 +-
 docs/reference/cluster/reroute.asciidoc       |  13 ++
 .../rest-api-spec/api/cluster.reroute.json    |   4 +
 33 files changed, 711 insertions(+), 123 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java
 create mode 100644 core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
 create mode 100644 core/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
index 28b62083d42..f986a5679a9 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
@@ -250,7 +250,7 @@ public class TransportClusterAllocationExplainAction
                                    final ActionListener<ClusterAllocationExplainResponse> listener) {
         final RoutingNodes routingNodes = state.getRoutingNodes();
         final RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, state,
-                clusterInfoService.getClusterInfo(), System.nanoTime());
+                clusterInfoService.getClusterInfo(), System.nanoTime(), false);
 
         ShardRouting foundShard = null;
         if (request.useAnyUnassignedShard()) {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequest.java
index a241f01ea28..dcc45ab21b9 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequest.java
@@ -38,9 +38,10 @@ import java.io.IOException;
  * Request to submit cluster reroute allocation commands
  */
 public class ClusterRerouteRequest extends AcknowledgedRequest<ClusterRerouteRequest> {
-    AllocationCommands commands = new AllocationCommands();
-    boolean dryRun;
-    boolean explain;
+    private AllocationCommands commands = new AllocationCommands();
+    private boolean dryRun;
+    private boolean explain;
+    private boolean retryFailed;
 
     public ClusterRerouteRequest() {
     }
@@ -81,6 +82,15 @@ public class ClusterRerouteRequest extends AcknowledgedRequest<ClusterRerouteReq
         return this;
     }
 
+    /**
+     * Sets the retry failed flag (defaults to <tt>false</tt>). If true, the
+     * request will retry allocating shards that can't currently be allocated due to too many allocation failures.
+     */
+    public ClusterRerouteRequest setRetryFailed(boolean retryFailed) {
+        this.retryFailed = retryFailed;
+        return this;
+    }
+
     /**
      * Returns the current explain flag
      */
@@ -88,6 +98,14 @@ public class ClusterRerouteRequest extends AcknowledgedRequest<ClusterRerouteReq
         return this.explain;
     }
 
+    /**
+     * Returns the current retry failed flag
+     */
+    public boolean isRetryFailed() {
+        return this.retryFailed;
+    }
+
+
     /**
      * Set the allocation commands to execute.
      */
@@ -96,6 +114,13 @@ public class ClusterRerouteRequest extends AcknowledgedRequest<ClusterRerouteReq
         return this;
     }
 
+    /**
+     * Returns the allocation commands to execute
+     */
+    public AllocationCommands getCommands() {
+        return commands;
+    }
+
     /**
      * Sets the source for the request.
      */
@@ -136,6 +161,7 @@ public class ClusterRerouteRequest extends AcknowledgedRequest<ClusterRerouteReq
         commands = AllocationCommands.readFrom(in);
         dryRun = in.readBoolean();
         explain = in.readBoolean();
+        retryFailed = in.readBoolean();
         readTimeout(in);
     }
 
@@ -145,6 +171,7 @@ public class ClusterRerouteRequest extends AcknowledgedRequest<ClusterRerouteReq
         AllocationCommands.writeTo(commands, out);
         out.writeBoolean(dryRun);
         out.writeBoolean(explain);
+        out.writeBoolean(retryFailed);
         writeTimeout(out);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestBuilder.java
index 7f0ad81a00d..e5d130cf812 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestBuilder.java
@@ -60,6 +60,15 @@ public class ClusterRerouteRequestBuilder extends AcknowledgedRequestBuilder<Clu
         return this;
     }
 
+    /**
+     * Sets the retry failed flag (defaults to <tt>false</tt>). If true, the
+     * request will retry allocating shards that can't currently be allocated due to too many allocation failures.
+     */
+    public ClusterRerouteRequestBuilder setRetryFailed(boolean retryFailed) {
+        request.setRetryFailed(retryFailed);
+        return this;
+    }
+
     /**
      * Sets the commands for the request to execute.
      */
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java
index e6116dbfbc4..b0b676f6e2e 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java
@@ -33,6 +33,7 @@ import org.elasticsearch.cluster.routing.allocation.RoutingExplanations;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -68,38 +69,55 @@ public class TransportClusterRerouteAction extends TransportMasterNodeAction<Clu
 
     @Override
     protected void masterOperation(final ClusterRerouteRequest request, final ClusterState state, final ActionListener<ClusterRerouteResponse> listener) {
-        clusterService.submitStateUpdateTask("cluster_reroute (api)", new AckedClusterStateUpdateTask<ClusterRerouteResponse>(Priority.IMMEDIATE, request, listener) {
-
-            private volatile ClusterState clusterStateToSend;
-            private volatile RoutingExplanations explanations;
-
-            @Override
-            protected ClusterRerouteResponse newResponse(boolean acknowledged) {
-                return new ClusterRerouteResponse(acknowledged, clusterStateToSend, explanations);
-            }
-
-            @Override
-            public void onAckTimeout() {
-                listener.onResponse(new ClusterRerouteResponse(false, clusterStateToSend, new RoutingExplanations()));
-            }
-
-            @Override
-            public void onFailure(String source, Throwable t) {
-                logger.debug("failed to perform [{}]", t, source);
-                super.onFailure(source, t);
-            }
-
-            @Override
-            public ClusterState execute(ClusterState currentState) {
-                RoutingAllocation.Result routingResult = allocationService.reroute(currentState, request.commands, request.explain());
-                ClusterState newState = ClusterState.builder(currentState).routingResult(routingResult).build();
-                clusterStateToSend = newState;
-                explanations = routingResult.explanations();
-                if (request.dryRun) {
-                    return currentState;
-                }
-                return newState;
-            }
-        });
+        clusterService.submitStateUpdateTask("cluster_reroute (api)", new ClusterRerouteResponseAckedClusterStateUpdateTask(logger,
+            allocationService, request, listener));
     }
-}
\ No newline at end of file
+
+    static class ClusterRerouteResponseAckedClusterStateUpdateTask extends AckedClusterStateUpdateTask<ClusterRerouteResponse> {
+
+        private final ClusterRerouteRequest request;
+        private final ActionListener<ClusterRerouteResponse> listener;
+        private final ESLogger logger;
+        private final AllocationService allocationService;
+        private volatile ClusterState clusterStateToSend;
+        private volatile RoutingExplanations explanations;
+
+        ClusterRerouteResponseAckedClusterStateUpdateTask(ESLogger logger, AllocationService allocationService, ClusterRerouteRequest request,
+                                                                 ActionListener<ClusterRerouteResponse> listener) {
+            super(Priority.IMMEDIATE, request, listener);
+            this.request = request;
+            this.listener = listener;
+            this.logger = logger;
+            this.allocationService = allocationService;
+        }
+
+        @Override
+        protected ClusterRerouteResponse newResponse(boolean acknowledged) {
+            return new ClusterRerouteResponse(acknowledged, clusterStateToSend, explanations);
+        }
+
+        @Override
+        public void onAckTimeout() {
+            listener.onResponse(new ClusterRerouteResponse(false, clusterStateToSend, new RoutingExplanations()));
+        }
+
+        @Override
+        public void onFailure(String source, Throwable t) {
+            logger.debug("failed to perform [{}]", t, source);
+            super.onFailure(source, t);
+        }
+
+        @Override
+        public ClusterState execute(ClusterState currentState) {
+            RoutingAllocation.Result routingResult = allocationService.reroute(currentState, request.getCommands(), request.explain(),
+                request.isRetryFailed());
+            ClusterState newState = ClusterState.builder(currentState).routingResult(routingResult).build();
+            clusterStateToSend = newState;
+            explanations = routingResult.explanations();
+            if (request.dryRun()) {
+                return currentState;
+            }
+            return newState;
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index 47dd2ce9ae6..a02e399ac0c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -49,6 +49,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDeci
 import org.elasticsearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.RebalanceOnlyWhenActiveAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider;
@@ -79,6 +80,7 @@ public class ClusterModule extends AbstractModule {
         new Setting<>("cluster.routing.allocation.type", BALANCED_ALLOCATOR, Function.identity(), Property.NodeScope);
     public static final List<Class<? extends AllocationDecider>> DEFAULT_ALLOCATION_DECIDERS =
         Collections.unmodifiableList(Arrays.asList(
+            MaxRetryAllocationDecider.class,
             SameShardAllocationDecider.class,
             FilterAllocationDecider.class,
             ReplicaAfterPrimaryActiveAllocationDecider.class,
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java b/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
index 2670363364d..bc44cd1701c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
@@ -48,7 +48,6 @@ public final class UnassignedInfo implements ToXContent, Writeable {
     public static final Setting<TimeValue> INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING =
         Setting.timeSetting("index.unassigned.node_left.delayed_timeout", DEFAULT_DELAYED_NODE_LEFT_TIMEOUT, Property.Dynamic,
             Property.IndexScope);
-
     /**
      * Reason why the shard is in unassigned state.
      * <p>
@@ -103,7 +102,11 @@ public final class UnassignedInfo implements ToXContent, Writeable {
         /**
          * A better replica location is identified and causes the existing replica allocation to be cancelled.
          */
-        REALLOCATED_REPLICA;
+        REALLOCATED_REPLICA,
+        /**
+         * Unassigned as a result of a failed primary while the replica was initializing.
+         */
+        PRIMARY_FAILED;
     }
 
     private final Reason reason;
@@ -112,6 +115,7 @@ public final class UnassignedInfo implements ToXContent, Writeable {
     private final long lastComputedLeftDelayNanos; // how long to delay shard allocation, not serialized (always positive, 0 means no delay)
     private final String message;
     private final Throwable failure;
+    private final int failedAllocations;
 
     /**
      * creates an UnassingedInfo object based **current** time
@@ -120,7 +124,7 @@ public final class UnassignedInfo implements ToXContent, Writeable {
      * @param message more information about cause.
      **/
     public UnassignedInfo(Reason reason, String message) {
-        this(reason, message, null, System.nanoTime(), System.currentTimeMillis());
+        this(reason, message, null, reason == Reason.ALLOCATION_FAILED ? 1 : 0, System.nanoTime(), System.currentTimeMillis());
     }
 
     /**
@@ -130,13 +134,16 @@ public final class UnassignedInfo implements ToXContent, Writeable {
      * @param unassignedTimeNanos  the time to use as the base for any delayed re-assignment calculation
      * @param unassignedTimeMillis the time of unassignment used to display to in our reporting.
      */
-    public UnassignedInfo(Reason reason, @Nullable String message, @Nullable Throwable failure, long unassignedTimeNanos, long unassignedTimeMillis) {
+    public UnassignedInfo(Reason reason, @Nullable String message, @Nullable Throwable failure, int failedAllocations, long unassignedTimeNanos, long unassignedTimeMillis) {
         this.reason = reason;
         this.unassignedTimeMillis = unassignedTimeMillis;
         this.unassignedTimeNanos = unassignedTimeNanos;
         this.lastComputedLeftDelayNanos = 0L;
         this.message = message;
         this.failure = failure;
+        this.failedAllocations = failedAllocations;
+        assert (failedAllocations > 0) == (reason == Reason.ALLOCATION_FAILED):
+            "failedAllocations: " + failedAllocations + " for reason " + reason;
         assert !(message == null && failure != null) : "provide a message if a failure exception is provided";
     }
 
@@ -147,17 +154,19 @@ public final class UnassignedInfo implements ToXContent, Writeable {
         this.lastComputedLeftDelayNanos = newComputedLeftDelayNanos;
         this.message = unassignedInfo.message;
         this.failure = unassignedInfo.failure;
+        this.failedAllocations = unassignedInfo.failedAllocations;
     }
 
     public UnassignedInfo(StreamInput in) throws IOException {
         this.reason = Reason.values()[(int) in.readByte()];
         this.unassignedTimeMillis = in.readLong();
         // As System.nanoTime() cannot be compared across different JVMs, reset it to now.
-        // This means that in master failover situations, elapsed delay time is forgotten.
+        // This means that in master fail-over situations, elapsed delay time is forgotten.
         this.unassignedTimeNanos = System.nanoTime();
         this.lastComputedLeftDelayNanos = 0L;
         this.message = in.readOptionalString();
         this.failure = in.readThrowable();
+        this.failedAllocations = in.readVInt();
     }
 
     public void writeTo(StreamOutput out) throws IOException {
@@ -166,12 +175,18 @@ public final class UnassignedInfo implements ToXContent, Writeable {
         // Do not serialize unassignedTimeNanos as System.nanoTime() cannot be compared across different JVMs
         out.writeOptionalString(message);
         out.writeThrowable(failure);
+        out.writeVInt(failedAllocations);
     }
 
     public UnassignedInfo readFrom(StreamInput in) throws IOException {
         return new UnassignedInfo(in);
     }
 
+    /**
+     * Returns the number of previously failed allocations of this shard.
+     */
+    public int getNumFailedAllocations() { return failedAllocations; }
+
     /**
      * The reason why the shard is unassigned.
      */
@@ -325,7 +340,11 @@ public final class UnassignedInfo implements ToXContent, Writeable {
         StringBuilder sb = new StringBuilder();
         sb.append("[reason=").append(reason).append("]");
         sb.append(", at[").append(DATE_TIME_FORMATTER.printer().print(unassignedTimeMillis)).append("]");
+        if (failedAllocations >  0) {
+            sb.append(", failed_attempts[").append(failedAllocations).append("]");
+        }
         String details = getDetails();
+
         if (details != null) {
             sb.append(", details[").append(details).append("]");
         }
@@ -342,6 +361,9 @@ public final class UnassignedInfo implements ToXContent, Writeable {
         builder.startObject("unassigned_info");
         builder.field("reason", reason);
         builder.field("at", DATE_TIME_FORMATTER.printer().print(unassignedTimeMillis));
+        if (failedAllocations >  0) {
+            builder.field("failed_attempts", failedAllocations);
+        }
         String details = getDetails();
         if (details != null) {
             builder.field("details", details);
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
index e1bbbb7f4ab..d59113675d8 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
@@ -222,8 +222,10 @@ public class AllocationService extends AbstractComponent {
         List<FailedRerouteAllocation.FailedShard> orderedFailedShards = new ArrayList<>(failedShards);
         orderedFailedShards.sort(Comparator.comparing(failedShard -> failedShard.shard.primary()));
         for (FailedRerouteAllocation.FailedShard failedShard : orderedFailedShards) {
+            UnassignedInfo unassignedInfo = failedShard.shard.unassignedInfo();
+            final int failedAllocations = unassignedInfo != null ? unassignedInfo.getNumFailedAllocations() : 0;
             changed |= applyFailedShard(allocation, failedShard.shard, true, new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, failedShard.message, failedShard.failure,
-                    System.nanoTime(), System.currentTimeMillis()));
+                    failedAllocations + 1, System.nanoTime(), System.currentTimeMillis()));
         }
         if (!changed) {
             return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData());
@@ -257,16 +259,13 @@ public class AllocationService extends AbstractComponent {
                 .collect(Collectors.joining(", "));
     }
 
-    public RoutingAllocation.Result reroute(ClusterState clusterState, AllocationCommands commands) {
-        return reroute(clusterState, commands, false);
-    }
-
-    public RoutingAllocation.Result reroute(ClusterState clusterState, AllocationCommands commands, boolean explain) {
+    public RoutingAllocation.Result reroute(ClusterState clusterState, AllocationCommands commands, boolean explain, boolean retryFailed) {
         RoutingNodes routingNodes = getMutableRoutingNodes(clusterState);
         // we don't shuffle the unassigned shards here, to try and get as close as possible to
         // a consistent result of the effect the commands have on the routing
         // this allows systems to dry run the commands, see the resulting cluster state, and act on it
-        RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, clusterState, clusterInfoService.getClusterInfo(), currentNanoTime());
+        RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, clusterState,
+            clusterInfoService.getClusterInfo(), currentNanoTime(), retryFailed);
         // don't short circuit deciders, we want a full explanation
         allocation.debugDecision(true);
         // we ignore disable allocation, because commands are explicit
@@ -305,7 +304,8 @@ public class AllocationService extends AbstractComponent {
         RoutingNodes routingNodes = getMutableRoutingNodes(clusterState);
         // shuffle the unassigned nodes, just so we won't have things like poison failed shards
         routingNodes.unassigned().shuffle();
-        RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, clusterState, clusterInfoService.getClusterInfo(), currentNanoTime());
+        RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, clusterState,
+            clusterInfoService.getClusterInfo(), currentNanoTime(), false);
         allocation.debugDecision(debug);
         if (!reroute(allocation)) {
             return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData());
@@ -437,7 +437,7 @@ public class AllocationService extends AbstractComponent {
             // now, go over all the shards routing on the node, and fail them
             for (ShardRouting shardRouting : node.copyShards()) {
                 UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, "node_left[" + node.nodeId() + "]", null,
-                        allocation.getCurrentNanoTime(), System.currentTimeMillis());
+                        0, allocation.getCurrentNanoTime(), System.currentTimeMillis());
                 applyFailedShard(allocation, shardRouting, false, unassignedInfo);
             }
             // its a dead node, remove it, note, its important to remove it *after* we apply failed shard
@@ -457,8 +457,8 @@ public class AllocationService extends AbstractComponent {
         boolean changed = false;
         for (ShardRouting routing : replicas) {
             changed |= applyFailedShard(allocation, routing, false,
-                    new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, "primary failed while replica initializing",
-                            null, allocation.getCurrentNanoTime(), System.currentTimeMillis()));
+                    new UnassignedInfo(UnassignedInfo.Reason.PRIMARY_FAILED, "primary failed while replica initializing",
+                            null, 0, allocation.getCurrentNanoTime(), System.currentTimeMillis()));
         }
         return changed;
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java
index a13862fed26..ef2e42eed76 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java
@@ -58,7 +58,7 @@ public class FailedRerouteAllocation extends RoutingAllocation {
     private final List<FailedShard> failedShards;
 
     public FailedRerouteAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, ClusterState clusterState, List<FailedShard> failedShards, ClusterInfo clusterInfo) {
-        super(deciders, routingNodes, clusterState, clusterInfo, System.nanoTime());
+        super(deciders, routingNodes, clusterState, clusterInfo, System.nanoTime(), false);
         this.failedShards = failedShards;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
index 60ca3a8d5fd..0df8074e14c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
@@ -134,6 +134,8 @@ public class RoutingAllocation {
 
     private boolean ignoreDisable = false;
 
+    private final boolean retryFailed;
+
     private boolean debugDecision = false;
 
     private boolean hasPendingAsyncFetch = false;
@@ -148,7 +150,7 @@ public class RoutingAllocation {
      * @param clusterState cluster state before rerouting
      * @param currentNanoTime the nano time to use for all delay allocation calculation (typically {@link System#nanoTime()})
      */
-    public RoutingAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, ClusterState clusterState, ClusterInfo clusterInfo, long currentNanoTime) {
+    public RoutingAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, ClusterState clusterState, ClusterInfo clusterInfo, long currentNanoTime, boolean retryFailed) {
         this.deciders = deciders;
         this.routingNodes = routingNodes;
         this.metaData = clusterState.metaData();
@@ -156,6 +158,7 @@ public class RoutingAllocation {
         this.customs = clusterState.customs();
         this.clusterInfo = clusterInfo;
         this.currentNanoTime = currentNanoTime;
+        this.retryFailed = retryFailed;
     }
 
     /** returns the nano time captured at the beginning of the allocation. used to make sure all time based decisions are aligned */
@@ -297,4 +300,8 @@ public class RoutingAllocation {
     public void setHasPendingAsyncFetch() {
         this.hasPendingAsyncFetch = true;
     }
+
+    public boolean isRetryFailed() {
+        return retryFailed;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StartedRerouteAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StartedRerouteAllocation.java
index e9570edd9c3..0f55ab4fda1 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StartedRerouteAllocation.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StartedRerouteAllocation.java
@@ -36,7 +36,7 @@ public class StartedRerouteAllocation extends RoutingAllocation {
     private final List<? extends ShardRouting> startedShards;
 
     public StartedRerouteAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, ClusterState clusterState, List<? extends ShardRouting> startedShards, ClusterInfo clusterInfo) {
-        super(deciders, routingNodes, clusterState, clusterInfo, System.nanoTime());
+        super(deciders, routingNodes, clusterState, clusterInfo, System.nanoTime(), false);
         this.startedShards = startedShards;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java
index d4191292cfc..c80afde3086 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java
@@ -125,7 +125,7 @@ public class AllocateEmptyPrimaryAllocationCommand extends BasePrimaryAllocation
             // we need to move the unassigned info back to treat it as if it was index creation
             unassignedInfoToUpdate = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED,
                 "force empty allocation from previous reason " + shardRouting.unassignedInfo().getReason() + ", " + shardRouting.unassignedInfo().getMessage(),
-                shardRouting.unassignedInfo().getFailure(), System.nanoTime(), System.currentTimeMillis());
+                shardRouting.unassignedInfo().getFailure(), 0, System.nanoTime(), System.currentTimeMillis());
         }
 
         initializeUnassignedShard(allocation, routingNodes, routingNode, shardRouting, unassignedInfoToUpdate);
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java
new file mode 100644
index 00000000000..6a8a0ccc5fa
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/MaxRetryAllocationDecider.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing.allocation.decider;
+
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.RoutingNode;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+
+/**
+ * An allocation decider that prevents shards from being allocated on any node if the shards allocation has been retried N times without
+ * success. This means if a shard has been INITIALIZING N times in a row without being moved to STARTED the shard will be ignored until
+ * the setting for <tt>index.allocation.max_retry</tt> is raised. The default value is <tt>5</tt>.
+ * Note: This allocation decider also allows allocation of repeatedly failing shards when the <tt>/_cluster/reroute?retry_failed=true</tt>
+ * API is manually invoked. This allows single retries without raising the limits.
+ *
+ * @see RoutingAllocation#isRetryFailed()
+ */
+public class MaxRetryAllocationDecider extends AllocationDecider {
+
+    public static final Setting<Integer> SETTING_ALLOCATION_MAX_RETRY = Setting.intSetting("index.allocation.max_retries", 5, 0,
+        Setting.Property.Dynamic, Setting.Property.IndexScope);
+
+    public static final String NAME = "max_retry";
+
+    /**
+     * Initializes a new {@link MaxRetryAllocationDecider}
+     *
+     * @param settings {@link Settings} used by this {@link AllocationDecider}
+     */
+    @Inject
+    public MaxRetryAllocationDecider(Settings settings) {
+        super(settings);
+    }
+
+    @Override
+    public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocation) {
+        UnassignedInfo unassignedInfo = shardRouting.unassignedInfo();
+        if (unassignedInfo != null && unassignedInfo.getNumFailedAllocations() > 0) {
+            final IndexMetaData indexMetaData = allocation.metaData().getIndexSafe(shardRouting.index());
+            final int maxRetry = SETTING_ALLOCATION_MAX_RETRY.get(indexMetaData.getSettings());
+            if (allocation.isRetryFailed()) { // manual allocation - retry
+                // if we are called via the _reroute API we ignore the failure counter and try to allocate
+                // this improves the usability since people don't need to raise the limits to issue retries since a simple _reroute call is
+                // enough to manually retry.
+                return allocation.decision(Decision.YES, NAME, "shard has already failed allocating ["
+                    + unassignedInfo.getNumFailedAllocations() + "] times vs. [" + maxRetry + "] retries allowed "
+                    + unassignedInfo.toString() + " - retrying once on manual allocation");
+            } else if (unassignedInfo.getNumFailedAllocations() >= maxRetry) {
+                return allocation.decision(Decision.NO, NAME, "shard has already failed allocating ["
+                    + unassignedInfo.getNumFailedAllocations() + "] times vs. [" + maxRetry + "] retries allowed "
+                    + unassignedInfo.toString() + " - manually call [/_cluster/reroute?retry_failed=true] to retry");
+            }
+        }
+        return allocation.decision(Decision.YES, NAME, "shard has no previous failures");
+    }
+
+    @Override
+    public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
+        return canAllocate(shardRouting, allocation);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
index 9ebf6853166..203d1db76b3 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
@@ -21,6 +21,7 @@ package org.elasticsearch.common.settings;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.gateway.PrimaryShardAllocator;
@@ -40,7 +41,6 @@ import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.index.store.FsDirectoryService;
 import org.elasticsearch.index.store.IndexStore;
 import org.elasticsearch.index.store.Store;
-import org.elasticsearch.index.IndexWarmer;
 import org.elasticsearch.indices.IndicesRequestCache;
 
 import java.util.Arrays;
@@ -59,6 +59,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
     public static final Predicate<String> INDEX_SETTINGS_KEY_PREDICATE = (s) -> s.startsWith(IndexMetaData.INDEX_SETTING_PREFIX);
 
     public static final Set<Setting<?>> BUILT_IN_INDEX_SETTINGS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
+        MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY,
         IndexSettings.INDEX_TTL_DISABLE_PURGE_SETTING,
         IndexStore.INDEX_STORE_THROTTLE_TYPE_SETTING,
         IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING,
diff --git a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
index d2e3d7f42cf..8b6e425c26a 100644
--- a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
+++ b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
@@ -108,7 +108,7 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
                                 currentNode, nodeWithHighestMatch);
                         it.moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.REALLOCATED_REPLICA,
                                 "existing allocation of replica to [" + currentNode + "] cancelled, sync id match found on node [" + nodeWithHighestMatch + "]",
-                                null, allocation.getCurrentNanoTime(), System.currentTimeMillis()));
+                                null, 0, allocation.getCurrentNanoTime(), System.currentTimeMillis()));
                         changed = true;
                     }
                 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/reroute/RestClusterRerouteAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/reroute/RestClusterRerouteAction.java
index 8a4afd89ac4..b34c6726c09 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/reroute/RestClusterRerouteAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/reroute/RestClusterRerouteAction.java
@@ -64,6 +64,7 @@ public class RestClusterRerouteAction extends BaseRestHandler {
     public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws Exception {
         final ClusterRerouteRequest clusterRerouteRequest = Requests.clusterRerouteRequest();
         clusterRerouteRequest.dryRun(request.paramAsBoolean("dry_run", clusterRerouteRequest.dryRun()));
+        clusterRerouteRequest.setRetryFailed(request.paramAsBoolean("retry_failed", clusterRerouteRequest.isRetryFailed()));
         clusterRerouteRequest.explain(request.paramAsBoolean("explain", clusterRerouteRequest.explain()));
         clusterRerouteRequest.timeout(request.paramAsTime("timeout", clusterRerouteRequest.timeout()));
         clusterRerouteRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterRerouteRequest.masterNodeTimeout()));
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
new file mode 100644
index 00000000000..927f572487c
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.admin.cluster.reroute;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
+import org.elasticsearch.cluster.routing.allocation.command.AllocateEmptyPrimaryAllocationCommand;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESAllocationTestCase;
+import org.elasticsearch.test.gateway.NoopGatewayAllocator;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
+import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
+
+public class ClusterRerouteTests extends ESAllocationTestCase {
+
+    public void testSerializeRequest() throws IOException {
+        ClusterRerouteRequest req = new ClusterRerouteRequest();
+        req.setRetryFailed(randomBoolean());
+        req.dryRun(randomBoolean());
+        req.explain(randomBoolean());
+        req.commands(new AllocateEmptyPrimaryAllocationCommand("foo", 1, "bar", randomBoolean()));
+        req.timeout(TimeValue.timeValueMillis(randomIntBetween(0, 100)));
+        BytesStreamOutput out = new BytesStreamOutput();
+        req.writeTo(out);
+        BytesReference bytes = out.bytes();
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
+        new NetworkModule(null, Settings.EMPTY, true, namedWriteableRegistry);
+        StreamInput wrap = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes.toBytes()),
+            namedWriteableRegistry);
+        ClusterRerouteRequest deserializedReq = new ClusterRerouteRequest();
+        deserializedReq.readFrom(wrap);
+
+        assertEquals(req.isRetryFailed(), deserializedReq.isRetryFailed());
+        assertEquals(req.dryRun(), deserializedReq.dryRun());
+        assertEquals(req.explain(), deserializedReq.explain());
+        assertEquals(req.timeout(), deserializedReq.timeout());
+        assertEquals(1, deserializedReq.getCommands().commands().size()); // allocation commands have their own tests
+        assertEquals(req.getCommands().commands().size(), deserializedReq.getCommands().commands().size());
+    }
+
+    public void testClusterStateUpdateTask() {
+        AllocationService allocationService = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
+            Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))),
+            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
+        ClusterState clusterState = createInitialClusterState(allocationService);
+        ClusterRerouteRequest req = new ClusterRerouteRequest();
+        req.dryRun(true);
+        AtomicReference<ClusterRerouteResponse> responseRef = new AtomicReference<>();
+        ActionListener<ClusterRerouteResponse> responseActionListener = new ActionListener<ClusterRerouteResponse>() {
+            @Override
+            public void onResponse(ClusterRerouteResponse clusterRerouteResponse) {
+                responseRef.set(clusterRerouteResponse);
+            }
+
+            @Override
+            public void onFailure(Throwable e) {
+
+            }
+        };
+        TransportClusterRerouteAction.ClusterRerouteResponseAckedClusterStateUpdateTask task =
+            new TransportClusterRerouteAction.ClusterRerouteResponseAckedClusterStateUpdateTask(logger, allocationService, req,
+                responseActionListener );
+        ClusterState execute = task.execute(clusterState);
+        assertSame(execute, clusterState); // dry-run
+        task.onAllNodesAcked(null);
+        assertNotSame(responseRef.get().getState(), execute);
+
+        req.dryRun(false);// now we allocate
+
+        final int retries = MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY.get(Settings.EMPTY);
+        // now fail it N-1 times
+        for (int i = 0; i < retries; i++) {
+            ClusterState newState = task.execute(clusterState);
+            assertNotSame(newState, clusterState); // dry-run=false
+            clusterState = newState;
+            RoutingTable routingTable = clusterState.routingTable();
+            assertEquals(routingTable.index("idx").shards().size(), 1);
+            assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING);
+            assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getNumFailedAllocations(), i);
+            List<FailedRerouteAllocation.FailedShard> failedShards = Collections.singletonList(
+                new FailedRerouteAllocation.FailedShard(routingTable.index("idx").shard(0).shards().get(0), "boom" + i,
+                    new UnsupportedOperationException()));
+            RoutingAllocation.Result result = allocationService.applyFailedShards(clusterState, failedShards);
+            assertTrue(result.changed());
+            clusterState = ClusterState.builder(clusterState).routingTable(result.routingTable()).build();
+            routingTable = clusterState.routingTable();
+            assertEquals(routingTable.index("idx").shards().size(), 1);
+            if (i == retries-1) {
+                assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), UNASSIGNED);
+            } else {
+                assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING);
+            }
+            assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getNumFailedAllocations(), i+1);
+        }
+
+
+        // without retry_failed we won't allocate that shard
+        ClusterState newState = task.execute(clusterState);
+        assertNotSame(newState, clusterState); // dry-run=false
+        task.onAllNodesAcked(null);
+        assertSame(responseRef.get().getState(), newState);
+        RoutingTable routingTable = clusterState.routingTable();
+        assertEquals(routingTable.index("idx").shards().size(), 1);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), UNASSIGNED);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getNumFailedAllocations(), retries);
+
+        req.setRetryFailed(true); // now we manually retry and get the shard back into initializing
+        newState = task.execute(clusterState);
+        assertNotSame(newState, clusterState); // dry-run=false
+        clusterState = newState;
+        routingTable = clusterState.routingTable();
+        assertEquals(routingTable.index("idx").shards().size(), 1);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getNumFailedAllocations(), retries);
+    }
+
+    private ClusterState createInitialClusterState(AllocationService service) {
+        MetaData.Builder metaBuilder = MetaData.builder();
+        metaBuilder.put(IndexMetaData.builder("idx").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0));
+        MetaData metaData = metaBuilder.build();
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        routingTableBuilder.addAsNew(metaData.index("idx"));
+
+        RoutingTable routingTable = routingTableBuilder.build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+            .metaData(metaData).routingTable(routingTable).build();
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")))
+            .build();
+        RoutingTable prevRoutingTable = routingTable;
+        routingTable = service.reroute(clusterState, "reroute").routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+
+        assertEquals(prevRoutingTable.index("idx").shards().size(), 1);
+        assertEquals(prevRoutingTable.index("idx").shard(0).shards().get(0).state(), UNASSIGNED);
+
+        assertEquals(routingTable.index("idx").shards().size(), 1);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING);
+        return clusterState;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
index 708f8ca5079..f46224570b0 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
@@ -64,7 +64,8 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
                 UnassignedInfo.Reason.NODE_LEFT,
                 UnassignedInfo.Reason.REROUTE_CANCELLED,
                 UnassignedInfo.Reason.REINITIALIZED,
-                UnassignedInfo.Reason.REALLOCATED_REPLICA};
+                UnassignedInfo.Reason.REALLOCATED_REPLICA,
+                UnassignedInfo.Reason.PRIMARY_FAILED};
         for (int i = 0; i < order.length; i++) {
             assertThat(order[i].ordinal(), equalTo(i));
         }
@@ -72,7 +73,10 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
     }
 
     public void testSerialization() throws Exception {
-        UnassignedInfo meta = new UnassignedInfo(RandomPicks.randomFrom(random(), UnassignedInfo.Reason.values()), randomBoolean() ? randomAsciiOfLength(4) : null);
+        UnassignedInfo.Reason reason = RandomPicks.randomFrom(random(), UnassignedInfo.Reason.values());
+        UnassignedInfo meta = reason == UnassignedInfo.Reason.ALLOCATION_FAILED ?
+            new UnassignedInfo(reason, randomBoolean() ? randomAsciiOfLength(4) : null, null, randomIntBetween(1, 100), System.nanoTime(), System.currentTimeMillis()):
+            new UnassignedInfo(reason, randomBoolean() ? randomAsciiOfLength(4) : null);
         BytesStreamOutput out = new BytesStreamOutput();
         meta.writeTo(out);
         out.close();
@@ -82,6 +86,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
         assertThat(read.getUnassignedTimeInMillis(), equalTo(meta.getUnassignedTimeInMillis()));
         assertThat(read.getMessage(), equalTo(meta.getMessage()));
         assertThat(read.getDetails(), equalTo(meta.getDetails()));
+        assertThat(read.getNumFailedAllocations(), equalTo(meta.getNumFailedAllocations()));
     }
 
     public void testIndexCreated() {
@@ -273,7 +278,10 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
     public void testUnassignedDelayOnlyNodeLeftNonNodeLeftReason() throws Exception {
         EnumSet<UnassignedInfo.Reason> reasons = EnumSet.allOf(UnassignedInfo.Reason.class);
         reasons.remove(UnassignedInfo.Reason.NODE_LEFT);
-        UnassignedInfo unassignedInfo = new UnassignedInfo(RandomPicks.randomFrom(random(), reasons), null);
+        UnassignedInfo.Reason reason = RandomPicks.randomFrom(random(), reasons);
+        UnassignedInfo unassignedInfo = reason == UnassignedInfo.Reason.ALLOCATION_FAILED ?
+            new UnassignedInfo(reason, null, null, 1, System.nanoTime(), System.currentTimeMillis()):
+            new UnassignedInfo(reason, null);
         unassignedInfo = unassignedInfo.updateDelay(unassignedInfo.getUnassignedTimeInNanos() + 1, // add 1 tick delay
                 Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "10h").build(), Settings.EMPTY);
         long delay = unassignedInfo.getLastComputedLeftDelayNanos();
@@ -287,7 +295,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
      */
     public void testLeftDelayCalculation() throws Exception {
         final long baseTime = System.nanoTime();
-        UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, "test", null, baseTime, System.currentTimeMillis());
+        UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, "test", null, 0, baseTime, System.currentTimeMillis());
         final long totalDelayNanos = TimeValue.timeValueMillis(10).nanos();
         final Settings settings = Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), TimeValue.timeValueNanos(totalDelayNanos)).build();
         unassignedInfo = unassignedInfo.updateDelay(baseTime, settings, Settings.EMPTY);
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
index 7aa8576ece3..b63692e0d2a 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
@@ -94,7 +94,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
         } else {
             toNodeId = "node1";
         }
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", 0, existingNodeId, toNodeId)));
+        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", 0, existingNodeId, toNodeId)), false, false);
         assertThat(rerouteResult.changed(), equalTo(true));
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
         assertThat(clusterState.getRoutingNodes().node(existingNodeId).iterator().next().state(), equalTo(ShardRoutingState.RELOCATING));
@@ -148,7 +148,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
 
         logger.info("--> allocating to non-existent node, should fail");
         try {
-            allocation.reroute(clusterState, new AllocationCommands(randomAllocateCommand(index, shardId.id(), "node42")));
+            allocation.reroute(clusterState, new AllocationCommands(randomAllocateCommand(index, shardId.id(), "node42")), false, false);
             fail("expected IllegalArgumentException when allocating to non-existing node");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), containsString("failed to resolve [node42], no matching nodes"));
@@ -156,7 +156,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
 
         logger.info("--> allocating to non-data node, should fail");
         try {
-            allocation.reroute(clusterState, new AllocationCommands(randomAllocateCommand(index, shardId.id(), "node4")));
+            allocation.reroute(clusterState, new AllocationCommands(randomAllocateCommand(index, shardId.id(), "node4")), false, false);
             fail("expected IllegalArgumentException when allocating to non-data node");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), containsString("allocation can only be done on data nodes"));
@@ -164,7 +164,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
 
         logger.info("--> allocating non-existing shard, should fail");
         try {
-            allocation.reroute(clusterState, new AllocationCommands(randomAllocateCommand("test", 1, "node2")));
+            allocation.reroute(clusterState, new AllocationCommands(randomAllocateCommand("test", 1, "node2")), false, false);
             fail("expected ShardNotFoundException when allocating non-existing shard");
         } catch (ShardNotFoundException e) {
             assertThat(e.getMessage(), containsString("no such shard"));
@@ -172,7 +172,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
 
         logger.info("--> allocating non-existing index, should fail");
         try {
-            allocation.reroute(clusterState, new AllocationCommands(randomAllocateCommand("test2", 0, "node2")));
+            allocation.reroute(clusterState, new AllocationCommands(randomAllocateCommand("test2", 0, "node2")), false, false);
             fail("expected ShardNotFoundException when allocating non-existing index");
         } catch (IndexNotFoundException e) {
             assertThat(e.getMessage(), containsString("no such index"));
@@ -180,7 +180,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
 
         logger.info("--> allocating empty primary with acceptDataLoss flag set to false");
         try {
-            allocation.reroute(clusterState, new AllocationCommands(new AllocateEmptyPrimaryAllocationCommand("test", 0, "node1", false)));
+            allocation.reroute(clusterState, new AllocationCommands(new AllocateEmptyPrimaryAllocationCommand("test", 0, "node1", false)), false, false);
             fail("expected IllegalArgumentException when allocating empty primary with acceptDataLoss flag set to false");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), containsString("allocating an empty primary for " + shardId + " can result in data loss. Please confirm by setting the accept_data_loss parameter to true"));
@@ -188,14 +188,14 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
 
         logger.info("--> allocating stale primary with acceptDataLoss flag set to false");
         try {
-            allocation.reroute(clusterState, new AllocationCommands(new AllocateStalePrimaryAllocationCommand(index, shardId.id(), "node1", false)));
+            allocation.reroute(clusterState, new AllocationCommands(new AllocateStalePrimaryAllocationCommand(index, shardId.id(), "node1", false)), false, false);
             fail("expected IllegalArgumentException when allocating stale primary with acceptDataLoss flag set to false");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), containsString("allocating an empty primary for " + shardId + " can result in data loss. Please confirm by setting the accept_data_loss parameter to true"));
         }
 
         logger.info("--> allocating empty primary with acceptDataLoss flag set to true");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new AllocateEmptyPrimaryAllocationCommand("test", 0, "node1", true)));
+        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new AllocateEmptyPrimaryAllocationCommand("test", 0, "node1", true)), false, false);
         assertThat(rerouteResult.changed(), equalTo(true));
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
@@ -211,13 +211,13 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
 
         logger.info("--> allocate the replica shard on the primary shard node, should fail");
         try {
-            allocation.reroute(clusterState, new AllocationCommands(new AllocateReplicaAllocationCommand("test", 0, "node1")));
+            allocation.reroute(clusterState, new AllocationCommands(new AllocateReplicaAllocationCommand("test", 0, "node1")), false, false);
             fail("expected IllegalArgumentException when allocating replica shard on the primary shard node");
         } catch (IllegalArgumentException e) {
         }
 
         logger.info("--> allocate the replica shard on on the second node");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new AllocateReplicaAllocationCommand("test", 0, "node2")));
+        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new AllocateReplicaAllocationCommand("test", 0, "node2")), false, false);
         assertThat(rerouteResult.changed(), equalTo(true));
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
@@ -236,7 +236,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
 
         logger.info("--> verify that we fail when there are no unassigned shards");
         try {
-            allocation.reroute(clusterState, new AllocationCommands(randomAllocateCommand("test", 0, "node3")));
+            allocation.reroute(clusterState, new AllocationCommands(randomAllocateCommand("test", 0, "node3")), false, false);
             fail("expected IllegalArgumentException when allocating shard while no unassigned shard available");
         } catch (IllegalArgumentException e) {
         }
@@ -268,7 +268,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
         assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(0));
 
         logger.info("--> allocating empty primary shard with accept_data_loss flag set to true");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new AllocateEmptyPrimaryAllocationCommand("test", 0, "node1", true)));
+        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new AllocateEmptyPrimaryAllocationCommand("test", 0, "node1", true)), false, false);
         assertThat(rerouteResult.changed(), equalTo(true));
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
@@ -277,7 +277,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
 
         logger.info("--> cancel primary allocation, make sure it fails...");
         try {
-            allocation.reroute(clusterState, new AllocationCommands(new CancelAllocationCommand("test", 0, "node1", false)));
+            allocation.reroute(clusterState, new AllocationCommands(new CancelAllocationCommand("test", 0, "node1", false)), false, false);
             fail();
         } catch (IllegalArgumentException e) {
         }
@@ -291,13 +291,13 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
 
         logger.info("--> cancel primary allocation, make sure it fails...");
         try {
-            allocation.reroute(clusterState, new AllocationCommands(new CancelAllocationCommand("test", 0, "node1", false)));
+            allocation.reroute(clusterState, new AllocationCommands(new CancelAllocationCommand("test", 0, "node1", false)), false, false);
             fail();
         } catch (IllegalArgumentException e) {
         }
 
         logger.info("--> allocate the replica shard on on the second node");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new AllocateReplicaAllocationCommand("test", 0, "node2")));
+        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new AllocateReplicaAllocationCommand("test", 0, "node2")), false, false);
         assertThat(rerouteResult.changed(), equalTo(true));
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
@@ -306,7 +306,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
         assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(INITIALIZING).size(), equalTo(1));
 
         logger.info("--> cancel the relocation allocation");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new CancelAllocationCommand("test", 0, "node2", false)));
+        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new CancelAllocationCommand("test", 0, "node2", false)), false, false);
         assertThat(rerouteResult.changed(), equalTo(true));
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
@@ -315,7 +315,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
         assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(0));
 
         logger.info("--> allocate the replica shard on on the second node");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new AllocateReplicaAllocationCommand("test", 0, "node2")));
+        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new AllocateReplicaAllocationCommand("test", 0, "node2")), false, false);
         assertThat(rerouteResult.changed(), equalTo(true));
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
@@ -325,7 +325,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
 
         logger.info("--> cancel the primary being replicated, make sure it fails");
         try {
-            allocation.reroute(clusterState, new AllocationCommands(new CancelAllocationCommand("test", 0, "node1", false)));
+            allocation.reroute(clusterState, new AllocationCommands(new CancelAllocationCommand("test", 0, "node1", false)), false, false);
             fail();
         } catch (IllegalArgumentException e) {
         }
@@ -339,7 +339,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
         assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).size(), equalTo(1));
 
         logger.info("--> cancel allocation of the replica shard");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new CancelAllocationCommand("test", 0, "node2", false)));
+        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new CancelAllocationCommand("test", 0, "node2", false)), false, false);
         assertThat(rerouteResult.changed(), equalTo(true));
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
@@ -348,7 +348,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
         assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(0));
 
         logger.info("--> allocate the replica shard on on the second node");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new AllocateReplicaAllocationCommand("test", 0, "node2")));
+        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new AllocateReplicaAllocationCommand("test", 0, "node2")), false, false);
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
         assertThat(rerouteResult.changed(), equalTo(true));
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
@@ -364,7 +364,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
         assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).size(), equalTo(1));
 
         logger.info("--> move the replica shard");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", 0, "node2", "node3")));
+        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", 0, "node2", "node3")), false, false);
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
@@ -374,7 +374,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
         assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState(INITIALIZING).size(), equalTo(1));
 
         logger.info("--> cancel the move of the replica shard");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new CancelAllocationCommand("test", 0, "node3", false)));
+        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new CancelAllocationCommand("test", 0, "node3", false)), false, false);
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
@@ -383,7 +383,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
 
 
         logger.info("--> cancel the primary allocation (with allow_primary set to true)");
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new CancelAllocationCommand("test", 0, "node1", true)));
+        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new CancelAllocationCommand("test", 0, "node1", true)), false, false);
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
         assertThat(rerouteResult.changed(), equalTo(true));
         logger.error(clusterState.prettyPrint());
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java
index ee993bf3ebd..805ab0321ba 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java
@@ -868,7 +868,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase {
         }
         commands.add(new MoveAllocationCommand("test", 0, primaryNode, "A-4"));
 
-        routingTable = strategy.reroute(clusterState, commands).routingTable();
+        routingTable = strategy.reroute(clusterState, commands, false, false).routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
 
         assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(0));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java
index 11b78d2ae6a..b14aeca890e 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java
@@ -149,8 +149,8 @@ public class DeadNodesAllocationTests extends ESAllocationTestCase {
 
         logger.info("--> moving primary shard to node3");
         rerouteResult = allocation.reroute(clusterState, new AllocationCommands(
-                new MoveAllocationCommand("test", 0, clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), "node3"))
-        );
+                new MoveAllocationCommand("test", 0, clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), "node3")),
+            false, false);
         assertThat(rerouteResult.changed(), equalTo(true));
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
         assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).iterator().next().state(), equalTo(RELOCATING));
@@ -223,8 +223,8 @@ public class DeadNodesAllocationTests extends ESAllocationTestCase {
 
         logger.info("--> moving primary shard to node3");
         rerouteResult = allocation.reroute(clusterState, new AllocationCommands(
-                new MoveAllocationCommand("test",0 , clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), "node3"))
-        );
+                new MoveAllocationCommand("test",0 , clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), "node3")),
+            false, false);
         assertThat(rerouteResult.changed(), equalTo(true));
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
         assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).iterator().next().state(), equalTo(RELOCATING));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java
index bfa27a36d8b..29644f07944 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java
@@ -149,7 +149,7 @@ public class ExpectedShardSizeAllocationTests extends ESAllocationTestCase {
         } else {
             toNodeId = "node1";
         }
-        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", 0, existingNodeId, toNodeId)));
+        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", 0, existingNodeId, toNodeId)), false, false);
         assertThat(rerouteResult.changed(), equalTo(true));
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
         assertEquals(clusterState.getRoutingNodes().node(existingNodeId).iterator().next().state(), ShardRoutingState.RELOCATING);
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java
index 58e2397b043..e859c5811c3 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java
@@ -109,8 +109,8 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase {
 
         logger.info("--> moving primary shard to node3");
         rerouteResult = allocation.reroute(clusterState, new AllocationCommands(
-                        new MoveAllocationCommand("test", 0, clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), "node3"))
-        );
+                        new MoveAllocationCommand("test", 0, clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), "node3")),
+            false, false);
         assertThat(rerouteResult.changed(), equalTo(true));
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
         assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).iterator().next().state(), equalTo(RELOCATING));
@@ -125,8 +125,8 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase {
 
         logger.info("--> moving primary shard to node3");
         rerouteResult = allocation.reroute(clusterState, new AllocationCommands(
-                        new MoveAllocationCommand("test", 0, clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), "node3"))
-        );
+                        new MoveAllocationCommand("test", 0, clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), "node3")),
+            false, false);
         assertThat(rerouteResult.changed(), equalTo(true));
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
         assertThat(clusterState.getRoutingNodes().node(origPrimaryNodeId).iterator().next().state(), equalTo(RELOCATING));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java
new file mode 100644
index 00000000000..f76851cfef9
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing.allocation;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
+import org.elasticsearch.cluster.routing.allocation.command.AllocateEmptyPrimaryAllocationCommand;
+import org.elasticsearch.cluster.routing.allocation.command.AllocateReplicaAllocationCommand;
+import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESAllocationTestCase;
+import org.elasticsearch.test.gateway.NoopGatewayAllocator;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
+import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
+import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
+
+public class MaxRetryAllocationDeciderTests extends ESAllocationTestCase {
+
+    private AllocationService strategy;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        strategy = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
+            Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))),
+            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
+    }
+
+    private ClusterState createInitialClusterState() {
+        MetaData.Builder metaBuilder = MetaData.builder();
+        metaBuilder.put(IndexMetaData.builder("idx").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0));
+        MetaData metaData = metaBuilder.build();
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        routingTableBuilder.addAsNew(metaData.index("idx"));
+
+        RoutingTable routingTable = routingTableBuilder.build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+            .metaData(metaData).routingTable(routingTable).build();
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")))
+            .build();
+        RoutingTable prevRoutingTable = routingTable;
+        routingTable = strategy.reroute(clusterState, "reroute", false).routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+
+        assertEquals(prevRoutingTable.index("idx").shards().size(), 1);
+        assertEquals(prevRoutingTable.index("idx").shard(0).shards().get(0).state(), UNASSIGNED);
+
+        assertEquals(routingTable.index("idx").shards().size(), 1);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING);
+        return clusterState;
+    }
+
+    public void testSingleRetryOnIgnore() {
+        ClusterState clusterState = createInitialClusterState();
+        RoutingTable routingTable = clusterState.routingTable();
+        final int retries = MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY.get(Settings.EMPTY);
+        // now fail it N-1 times
+        for (int i = 0; i < retries-1; i++) {
+            List<FailedRerouteAllocation.FailedShard> failedShards = Collections.singletonList(
+                new FailedRerouteAllocation.FailedShard(routingTable.index("idx").shard(0).shards().get(0), "boom" + i,
+                    new UnsupportedOperationException()));
+            RoutingAllocation.Result result = strategy.applyFailedShards(clusterState, failedShards);
+            assertTrue(result.changed());
+            routingTable = result.routingTable();
+            clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+            assertEquals(routingTable.index("idx").shards().size(), 1);
+            assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING);
+            assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getNumFailedAllocations(), i+1);
+            assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getMessage(), "boom" + i);
+        }
+        // now we go and check that we are actually stick to unassigned on the next failure
+        List<FailedRerouteAllocation.FailedShard> failedShards = Collections.singletonList(
+            new FailedRerouteAllocation.FailedShard(routingTable.index("idx").shard(0).shards().get(0), "boom",
+                new UnsupportedOperationException()));
+        RoutingAllocation.Result result = strategy.applyFailedShards(clusterState, failedShards);
+        assertTrue(result.changed());
+        routingTable = result.routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        assertEquals(routingTable.index("idx").shards().size(), 1);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getNumFailedAllocations(), retries);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), UNASSIGNED);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getMessage(), "boom");
+
+        result = strategy.reroute(clusterState, new AllocationCommands(), false, true); // manual reroute should retry once
+        assertTrue(result.changed());
+        routingTable = result.routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        assertEquals(routingTable.index("idx").shards().size(), 1);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getNumFailedAllocations(), retries);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getMessage(), "boom");
+
+        // now we go and check that we are actually stick to unassigned on the next failure ie. no retry
+        failedShards = Collections.singletonList(
+            new FailedRerouteAllocation.FailedShard(routingTable.index("idx").shard(0).shards().get(0), "boom",
+                new UnsupportedOperationException()));
+        result = strategy.applyFailedShards(clusterState, failedShards);
+        assertTrue(result.changed());
+        routingTable = result.routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        assertEquals(routingTable.index("idx").shards().size(), 1);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getNumFailedAllocations(), retries+1);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), UNASSIGNED);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getMessage(), "boom");
+
+    }
+
+    public void testFailedAllocation() {
+        ClusterState clusterState = createInitialClusterState();
+        RoutingTable routingTable = clusterState.routingTable();
+        final int retries = MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY.get(Settings.EMPTY);
+        // now fail it N-1 times
+        for (int i = 0; i < retries-1; i++) {
+            List<FailedRerouteAllocation.FailedShard> failedShards = Collections.singletonList(
+                new FailedRerouteAllocation.FailedShard(routingTable.index("idx").shard(0).shards().get(0), "boom" + i,
+                    new UnsupportedOperationException()));
+            RoutingAllocation.Result result = strategy.applyFailedShards(clusterState, failedShards);
+            assertTrue(result.changed());
+            routingTable = result.routingTable();
+            clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+            assertEquals(routingTable.index("idx").shards().size(), 1);
+            assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING);
+            assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getNumFailedAllocations(), i+1);
+            assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getMessage(), "boom" + i);
+        }
+        // now we go and check that we are actually stick to unassigned on the next failure
+        {
+            List<FailedRerouteAllocation.FailedShard> failedShards = Collections.singletonList(
+                new FailedRerouteAllocation.FailedShard(routingTable.index("idx").shard(0).shards().get(0), "boom",
+                    new UnsupportedOperationException()));
+            RoutingAllocation.Result result = strategy.applyFailedShards(clusterState, failedShards);
+            assertTrue(result.changed());
+            routingTable = result.routingTable();
+            clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+            assertEquals(routingTable.index("idx").shards().size(), 1);
+            assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getNumFailedAllocations(), retries);
+            assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), UNASSIGNED);
+            assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getMessage(), "boom");
+        }
+
+        // change the settings and ensure we can do another round of allocation for that index.
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable)
+            .metaData(MetaData.builder(clusterState.metaData())
+                .put(IndexMetaData.builder(clusterState.metaData().index("idx")).settings(
+                    Settings.builder().put(clusterState.metaData().index("idx").getSettings()).put("index.allocation.max_retries",
+                        retries+1).build()
+                ).build(), true).build()).build();
+        RoutingAllocation.Result result = strategy.reroute(clusterState, "settings changed", false);
+        assertTrue(result.changed());
+        routingTable = result.routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        // good we are initializing and we are maintaining failure information
+        assertEquals(routingTable.index("idx").shards().size(), 1);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getNumFailedAllocations(), retries);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getMessage(), "boom");
+
+        // now we start the shard
+        routingTable = strategy.applyStartedShards(clusterState, Collections.singletonList(routingTable.index("idx")
+            .shard(0).shards().get(0))).routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+
+        // all counters have been reset to 0 ie. no unassigned info
+        assertEquals(routingTable.index("idx").shards().size(), 1);
+        assertNull(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo());
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), STARTED);
+
+        // now fail again and see if it has a new counter
+        List<FailedRerouteAllocation.FailedShard> failedShards = Collections.singletonList(
+            new FailedRerouteAllocation.FailedShard(routingTable.index("idx").shard(0).shards().get(0), "ZOOOMG",
+                new UnsupportedOperationException()));
+        result = strategy.applyFailedShards(clusterState, failedShards);
+        assertTrue(result.changed());
+        routingTable = result.routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        assertEquals(routingTable.index("idx").shards().size(), 1);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getNumFailedAllocations(), 1);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).unassignedInfo().getMessage(), "ZOOOMG");
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
index 97a3003ab2f..d0fc64b4b6b 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
@@ -337,7 +337,7 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase {
         AllocationService strategy = new MockAllocationService(Settings.EMPTY,
             allocationDeciders,
             NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
-        RoutingAllocation.Result result = strategy.reroute(state, new AllocationCommands(), true);
+        RoutingAllocation.Result result = strategy.reroute(state, new AllocationCommands(), true, false);
         // the two indices must stay as is, the replicas cannot move to oldNode2 because versions don't match
         state = ClusterState.builder(state).routingResult(result).build();
         assertThat(result.routingTable().index(shard2.getIndex()).shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(0));
@@ -369,7 +369,7 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase {
         AllocationService strategy = new MockAllocationService(Settings.EMPTY,
             allocationDeciders,
             NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
-        RoutingAllocation.Result result = strategy.reroute(state, new AllocationCommands(), true);
+        RoutingAllocation.Result result = strategy.reroute(state, new AllocationCommands(), true, false);
 
         // Make sure that primary shards are only allocated on the new node
         for (int i = 0; i < numberOfShards; i++) {
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java
index 28d916e20c1..61a72bc352a 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java
@@ -283,7 +283,7 @@ public class ThrottlingAllocationTests extends ESAllocationTestCase {
         assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node2"), 0);
         assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node3"), 0);
 
-        RoutingAllocation.Result reroute = strategy.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", clusterState.getRoutingNodes().node("node1").iterator().next().shardId().id(), "node1", "node2")));
+        RoutingAllocation.Result reroute = strategy.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", clusterState.getRoutingNodes().node("node1").iterator().next().shardId().id(), "node1", "node2")), false, false);
         assertEquals(reroute.explanations().explanations().size(), 1);
         assertEquals(reroute.explanations().explanations().get(0).decisions().type(), Decision.Type.YES);
         routingTable = reroute.routingTable();
@@ -296,7 +296,7 @@ public class ThrottlingAllocationTests extends ESAllocationTestCase {
         assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node3"), 0);
 
         // outgoing throttles
-        reroute = strategy.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", clusterState.getRoutingNodes().node("node3").iterator().next().shardId().id(), "node3", "node1")), true);
+        reroute = strategy.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", clusterState.getRoutingNodes().node("node3").iterator().next().shardId().id(), "node3", "node1")), true, false);
         assertEquals(reroute.explanations().explanations().size(), 1);
         assertEquals(reroute.explanations().explanations().get(0).decisions().type(), Decision.Type.THROTTLE);
         assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node1"), 0);
@@ -311,7 +311,7 @@ public class ThrottlingAllocationTests extends ESAllocationTestCase {
         assertThat(routingTable.shardsWithState(UNASSIGNED).size(), equalTo(0));
 
         // incoming throttles
-        reroute = strategy.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", clusterState.getRoutingNodes().node("node3").iterator().next().shardId().id(), "node3", "node2")), true);
+        reroute = strategy.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", clusterState.getRoutingNodes().node("node3").iterator().next().shardId().id(), "node3", "node2")), true, false);
         assertEquals(reroute.explanations().explanations().size(), 1);
         assertEquals(reroute.explanations().explanations().get(0).decisions().type(), Decision.Type.THROTTLE);
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
index 2c4e86ad4b1..579e87150a7 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
@@ -796,7 +796,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
         AllocationCommand relocate1 = new MoveAllocationCommand("test", 0, "node2", "node3");
         AllocationCommands cmds = new AllocationCommands(relocate1);
 
-        routingTable = strategy.reroute(clusterState, cmds).routingTable();
+        routingTable = strategy.reroute(clusterState, cmds, false, false).routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
         logShardStates(clusterState);
 
@@ -808,7 +808,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
             // node3, which will put it over the low watermark when it
             // completes, with shard relocations taken into account this should
             // throw an exception about not being able to complete
-            strategy.reroute(clusterState, cmds).routingTable();
+            strategy.reroute(clusterState, cmds, false, false).routingTable();
             fail("should not have been able to reroute the shard");
         } catch (IllegalArgumentException e) {
             assertThat("can't allocated because there isn't enough room: " + e.getMessage(),
@@ -876,7 +876,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
         );
         ClusterState clusterState = ClusterState.builder(baseClusterState).routingTable(builder.build()).build();
         RoutingAllocation routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), clusterState, clusterInfo,
-                System.nanoTime());
+                System.nanoTime(), false);
         Decision decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation);
         assertThat(decision.type(), equalTo(Decision.Type.NO));
 
@@ -896,7 +896,8 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                         )
         );
         clusterState = ClusterState.builder(baseClusterState).routingTable(builder.build()).build();
-        routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), clusterState, clusterInfo, System.nanoTime());
+        routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), clusterState, clusterInfo, System.nanoTime(),
+            false);
         decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation);
         assertThat(decision.type(), equalTo(Decision.Type.YES));
 
@@ -992,7 +993,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
         );
         ClusterState clusterState = ClusterState.builder(baseClusterState).routingTable(builder.build()).build();
         RoutingAllocation routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), clusterState, clusterInfo,
-                System.nanoTime());
+                System.nanoTime(), false);
         Decision decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation);
 
         // Two shards should start happily
@@ -1051,7 +1052,8 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
         );
 
         clusterState = ClusterState.builder(updateClusterState).routingTable(builder.build()).build();
-        routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), clusterState, clusterInfo, System.nanoTime());
+        routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), clusterState, clusterInfo, System.nanoTime(),
+            false);
         decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation);
         assertThat(decision.type(), equalTo(Decision.Type.YES));
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
index d9e9fb95445..008884cbb8d 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
@@ -136,7 +136,7 @@ public class DiskThresholdDeciderUnitTests extends ESTestCase {
         ImmutableOpenMap.Builder<String, Long> shardSizes = ImmutableOpenMap.builder();
         shardSizes.put("[test][0][p]", 10L); // 10 bytes
         final ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages.build(), mostAvailableUsage.build(), shardSizes.build(), ImmutableOpenMap.of());
-        RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, new AllocationDecider[]{decider}), clusterState.getRoutingNodes(), clusterState, clusterInfo, System.nanoTime());
+        RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, new AllocationDecider[]{decider}), clusterState.getRoutingNodes(), clusterState, clusterInfo, System.nanoTime(), false);
         assertEquals(mostAvailableUsage.toString(), Decision.YES, decider.canAllocate(test_0, new RoutingNode("node_0", node_0), allocation));
         assertEquals(mostAvailableUsage.toString(), Decision.NO, decider.canAllocate(test_0, new RoutingNode("node_1", node_1), allocation));
     }
@@ -204,7 +204,7 @@ public class DiskThresholdDeciderUnitTests extends ESTestCase {
         shardSizes.put("[test][2][p]", 10L);
 
         final ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages.build(), mostAvailableUsage.build(), shardSizes.build(), shardRoutingMap.build());
-        RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, new AllocationDecider[]{decider}), clusterState.getRoutingNodes(), clusterState, clusterInfo, System.nanoTime());
+        RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, new AllocationDecider[]{decider}), clusterState.getRoutingNodes(), clusterState, clusterInfo, System.nanoTime(), false);
         assertEquals(Decision.YES, decider.canRemain(test_0, new RoutingNode("node_0", node_0), allocation));
         assertEquals(Decision.NO, decider.canRemain(test_1, new RoutingNode("node_1", node_1), allocation));
         try {
diff --git a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
index 01e717e6fa9..ee1cf7280e7 100644
--- a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
@@ -346,7 +346,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
             .metaData(metaData)
             .routingTable(routingTable)
             .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build();
-        return new RoutingAllocation(allocationDeciders, new RoutingNodes(state, false), state, null, System.nanoTime());
+        return new RoutingAllocation(allocationDeciders, new RoutingNodes(state, false), state, null, System.nanoTime(), false);
     }
 
     /**
@@ -425,7 +425,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
             .metaData(metaData)
             .routingTable(routingTable)
             .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build();
-        return new RoutingAllocation(allocationDeciders, new RoutingNodes(state, false), state, null, System.nanoTime());
+        return new RoutingAllocation(allocationDeciders, new RoutingNodes(state, false), state, null, System.nanoTime(), false);
     }
 
     /**
@@ -444,7 +444,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
                 .routingTable(routingTable)
                 .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build();
 
-        RoutingAllocation allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state, null, System.nanoTime());
+        RoutingAllocation allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state, null, System.nanoTime(), false);
         boolean changed = testAllocator.allocateUnassigned(allocation);
         assertThat(changed, equalTo(false));
         assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1));
@@ -452,7 +452,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(2)); // replicas
 
         testAllocator.addData(node1, 1, null, randomBoolean());
-        allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state, null, System.nanoTime());
+        allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state, null, System.nanoTime(), false);
         changed = testAllocator.allocateUnassigned(allocation);
         assertThat(changed, equalTo(false));
         assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1));
@@ -460,7 +460,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(2)); // replicas
 
         testAllocator.addData(node2, 1, null, randomBoolean());
-        allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state, null, System.nanoTime());
+        allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state, null, System.nanoTime(), false);
         changed = testAllocator.allocateUnassigned(allocation);
         assertThat(changed, equalTo(true));
         assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(0));
@@ -485,7 +485,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
                 .routingTable(routingTable)
                 .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build();
 
-        RoutingAllocation allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state, null, System.nanoTime());
+        RoutingAllocation allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state, null, System.nanoTime(), false);
         boolean changed = testAllocator.allocateUnassigned(allocation);
         assertThat(changed, equalTo(false));
         assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1));
@@ -493,7 +493,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(2)); // replicas
 
         testAllocator.addData(node1, 1, null, randomBoolean());
-        allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state, null, System.nanoTime());
+        allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state, null, System.nanoTime(), false);
         changed = testAllocator.allocateUnassigned(allocation);
         assertThat(changed, equalTo(false));
         assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1));
@@ -501,7 +501,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(2)); // replicas
 
         testAllocator.addData(node2, 2, null, randomBoolean());
-        allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state, null, System.nanoTime());
+        allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state, null, System.nanoTime(), false);
         changed = testAllocator.allocateUnassigned(allocation);
         assertThat(changed, equalTo(true));
         assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(0));
@@ -525,7 +525,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
                 .metaData(metaData)
                 .routingTable(routingTableBuilder.build())
                 .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build();
-        return new RoutingAllocation(deciders, new RoutingNodes(state, false), state, null, System.nanoTime());
+        return new RoutingAllocation(deciders, new RoutingNodes(state, false), state, null, System.nanoTime(), false);
     }
 
     class TestAllocator extends PrimaryShardAllocator {
diff --git a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
index 672c9de3d3e..20eb6286813 100644
--- a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
@@ -302,7 +302,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
                 .metaData(metaData)
                 .routingTable(routingTable)
                 .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build();
-        return new RoutingAllocation(deciders, new RoutingNodes(state, false), state, ClusterInfo.EMPTY, System.nanoTime());
+        return new RoutingAllocation(deciders, new RoutingNodes(state, false), state, ClusterInfo.EMPTY, System.nanoTime(), false);
     }
 
     private RoutingAllocation onePrimaryOnNode1And1ReplicaRecovering(AllocationDeciders deciders) {
@@ -324,7 +324,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
                 .metaData(metaData)
                 .routingTable(routingTable)
                 .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build();
-        return new RoutingAllocation(deciders, new RoutingNodes(state, false), state, ClusterInfo.EMPTY, System.nanoTime());
+        return new RoutingAllocation(deciders, new RoutingNodes(state, false), state, ClusterInfo.EMPTY, System.nanoTime(), false);
     }
 
     class TestAllocator extends ReplicaShardAllocator {
diff --git a/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java b/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java
index dc533737886..81c50cc4f9c 100644
--- a/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java
@@ -103,7 +103,7 @@ public class RareClusterStateIT extends ESIntegTestCase {
                         .nodes(DiscoveryNodes.EMPTY_NODES)
                         .build(), false
         );
-        RoutingAllocation routingAllocation = new RoutingAllocation(allocationDeciders, routingNodes, current, ClusterInfo.EMPTY, System.nanoTime());
+        RoutingAllocation routingAllocation = new RoutingAllocation(allocationDeciders, routingNodes, current, ClusterInfo.EMPTY, System.nanoTime(), false);
         allocator.allocateUnassigned(routingAllocation);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
index 5ed63b519a4..7ca30132a4a 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -640,7 +640,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         assertAcked(client.admin().cluster().preparePutRepository("test-repo")
                 .setType("fs").setSettings(Settings.builder().put("location", repositoryLocation)));
 
-        createIndex("test-idx");
+        prepareCreate("test-idx").setSettings(Settings.builder().put("index.allocation.max_retries", Integer.MAX_VALUE)).get();
         ensureGreen();
 
         logger.info("--> indexing some data");
diff --git a/docs/reference/cluster/reroute.asciidoc b/docs/reference/cluster/reroute.asciidoc
index 99e754df529..bb48a00fbe5 100644
--- a/docs/reference/cluster/reroute.asciidoc
+++ b/docs/reference/cluster/reroute.asciidoc
@@ -103,3 +103,16 @@ are available:
     To ensure that these implications are well-understood,
     this command requires the special field `accept_data_loss` to be
     explicitly set to `true` for it to work.
+
+[float]
+=== Retry failed shards
+
+The cluster will attempt to allocate a shard a maximum of
+`index.allocation.max_retries` times in a row (defaults to `5`), before giving
+up and leaving the shard unallocated. This scenario can be caused by
+structural problems such as having an analyzer which refers to a stopwords
+file which doesn't exist on all nodes.
+
+Once the problem has been corrected, allocation can be manually retried by
+calling the <<cluster-reroute,`_reroute`>> API with `?retry_failed`, which
+will attempt a single retry round for these shards.
\ No newline at end of file
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.reroute.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.reroute.json
index 2ae42c089d3..8bb85ca087a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.reroute.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.reroute.json
@@ -16,6 +16,10 @@
           "type" : "boolean",
           "description" : "Return an explanation of why the commands can or cannot be executed"
         },
+        "retry_failed": {
+          "type" : "boolean",
+          "description" : "Retries allocation of shards that are blocked due to too many subsequent allocation failures"
+        },
         "metric": {
           "type": "list",
           "options": ["_all", "blocks", "metadata", "nodes", "routing_table", "master_node", "version"],

From 923d90d434cf4f049a396536087be213d8dea6d4 Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Fri, 20 May 2016 15:04:32 -0400
Subject: [PATCH 0396/1311] Remove use of a Fields class in snapshot responses
 that contains x-content keys, in favor of declaring/using the keys directly.

Closes #18497
---
 .../cluster/snapshots/create/CreateSnapshotResponse.java | 9 ++-------
 .../cluster/snapshots/get/GetSnapshotsResponse.java      | 6 +-----
 .../snapshots/restore/RestoreSnapshotResponse.java       | 9 ++-------
 .../snapshots/status/SnapshotsStatusResponse.java        | 6 +-----
 4 files changed, 6 insertions(+), 24 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
index 0a7a8a9ce80..d49be2d294e 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
@@ -81,18 +81,13 @@ public class CreateSnapshotResponse extends ActionResponse implements ToXContent
         return snapshotInfo.status();
     }
 
-    static final class Fields {
-        static final String SNAPSHOT = "snapshot";
-        static final String ACCEPTED = "accepted";
-    }
-
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         if (snapshotInfo != null) {
-            builder.field(Fields.SNAPSHOT);
+            builder.field("snapshot");
             snapshotInfo.toExternalXContent(builder, params);
         } else {
-            builder.field(Fields.ACCEPTED, true);
+            builder.field("accepted", true);
         }
         return builder;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
index a5db19684b2..7c4b255ff7c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
@@ -74,13 +74,9 @@ public class GetSnapshotsResponse extends ActionResponse implements ToXContent {
         }
     }
 
-    static final class Fields {
-        static final String SNAPSHOTS = "snapshots";
-    }
-
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
-        builder.startArray(Fields.SNAPSHOTS);
+        builder.startArray("snapshots");
         for (SnapshotInfo snapshotInfo : snapshots) {
             snapshotInfo.toExternalXContent(builder, params);
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java
index a54c01ed15a..70f4f2aa4f2 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotResponse.java
@@ -73,18 +73,13 @@ public class RestoreSnapshotResponse extends ActionResponse implements ToXConten
         return restoreInfo.status();
     }
 
-    static final class Fields {
-        static final String SNAPSHOT = "snapshot";
-        static final String ACCEPTED = "accepted";
-    }
-
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
         if (restoreInfo != null) {
-            builder.field(Fields.SNAPSHOT);
+            builder.field("snapshot");
             restoreInfo.toXContent(builder, params);
         } else {
-            builder.field(Fields.ACCEPTED, true);
+            builder.field("accepted", true);
         }
         return builder;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java
index 34e503224ce..b9800a2d9ed 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotsStatusResponse.java
@@ -73,13 +73,9 @@ public class SnapshotsStatusResponse extends ActionResponse implements ToXConten
         }
     }
 
-    static final class Fields {
-        static final String SNAPSHOTS = "snapshots";
-    }
-
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startArray(Fields.SNAPSHOTS);
+        builder.startArray("snapshots");
         for (SnapshotStatus snapshot : snapshots) {
             snapshot.toXContent(builder, params);
         }

From 332f6ffe59b2313df38ee2a597869ec1db6584f9 Mon Sep 17 00:00:00 2001
From: Spencer <spalger@users.noreply.github.com>
Date: Fri, 20 May 2016 12:22:56 -0700
Subject: [PATCH 0397/1311] [rest api spec] fix url for reindex api docs

---
 .../main/resources/rest-api-spec/api/reindex.rethrottle.json    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.rethrottle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.rethrottle.json
index 921249ab4cd..10d6321212d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.rethrottle.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.rethrottle.json
@@ -1,6 +1,6 @@
 {
   "reindex.rethrottle": {
-    "documentation": "https://www.elastic.co/guide/en/elasticsearch/plugins/master/plugins-reindex.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-reindex.html",
     "methods": ["POST"],
     "url": {
       "path": "/_reindex/{task_id}/_rethrottle",

From 452faa220c453c00218ec5ad89bd70d8ffaa88c2 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 20 May 2016 15:25:45 -0400
Subject: [PATCH 0398/1311] Filter client/server VM options from jvm.options

The default jvm.options file ships with the -server flag to force the
server VM on systems where the server VM is not the default. However,
the method of starting the JVM via the Windows service does not support
the command-line flags for selecting the VM because it starts from a
DLL that is specific to the server or client VM. Thus, we need to
filter these options from the jvm.options configuration file when
installing the Windows service.

Relates #18473
---
 distribution/src/main/resources/bin/service.bat | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distribution/src/main/resources/bin/service.bat b/distribution/src/main/resources/bin/service.bat
index 3d73c37b823..81b6c8a5df5 100644
--- a/distribution/src/main/resources/bin/service.bat
+++ b/distribution/src/main/resources/bin/service.bat
@@ -163,7 +163,7 @@ set ES_JVM_OPTIONS="%ES_HOME%\config\jvm.options"
 if not "%ES_JAVA_OPTS%" == "" set ES_JAVA_OPTS=%ES_JAVA_OPTS: =;%
 
 @setlocal
-for /F "usebackq delims=" %%a in (`findstr /b \- "%ES_JVM_OPTIONS%"`) do set JVM_OPTIONS=!JVM_OPTIONS!%%a;
+for /F "usebackq delims=" %%a in (`findstr /b \- "%ES_JVM_OPTIONS%" ^| findstr /b /v "\-server \-client"`) do set JVM_OPTIONS=!JVM_OPTIONS!%%a;
 @endlocal & set ES_JAVA_OPTS=%JVM_OPTIONS%%ES_JAVA_OPTS%
 
 if "%ES_JAVA_OPTS:~-1%"==";" set ES_JAVA_OPTS=%ES_JAVA_OPTS:~0,-1%

From 4c7993ea719b2d06360135bb57ac43e28ae1fe09 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 20 May 2016 16:26:46 -0400
Subject: [PATCH 0399/1311] Netty request/response tracer should wait for send

We write to Netty channels in an async fashion, but notify listeners via
a transport service adapter before we are certain that the channel write
succeeded. In particular, the tracer logs are implemented via a
transport service adapter and this means that we can write tracer logs
before a write was successful and in some cases the write might fail
leading to misleading logs. This commit attaches the transport service
adapters to channel writes as a listener so that the notification occurs
only after a successful write.

Relates #18500
---
 .../transport/netty/NettyTransport.java              |  5 ++++-
 .../transport/netty/NettyTransportChannel.java       | 12 +++++++++---
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
index 59649bff783..c5d8f29e39a 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
@@ -925,7 +925,10 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
             ReleaseChannelFutureListener listener = new ReleaseChannelFutureListener(bytes);
             future.addListener(listener);
             addedReleaseListener = true;
-            transportServiceAdapter.onRequestSent(node, requestId, action, request, options);
+            final TransportRequestOptions finalOptions = options;
+            ChannelFutureListener channelFutureListener =
+                f -> transportServiceAdapter.onRequestSent(node, requestId, action, request, finalOptions);
+            future.addListener(channelFutureListener);
         } finally {
             if (!addedReleaseListener) {
                 Releasables.close(bStream.bytes());
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java
index 57893ff1908..91b6bc120ad 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java
@@ -37,6 +37,7 @@ import org.elasticsearch.transport.support.TransportStatus;
 import org.jboss.netty.buffer.ChannelBuffer;
 import org.jboss.netty.channel.Channel;
 import org.jboss.netty.channel.ChannelFuture;
+import org.jboss.netty.channel.ChannelFutureListener;
 
 import java.io.IOException;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -114,7 +115,10 @@ public class NettyTransportChannel implements TransportChannel {
             ReleaseChannelFutureListener listener = new ReleaseChannelFutureListener(bytes);
             future.addListener(listener);
             addedReleaseListener = true;
-            transportServiceAdapter.onResponseSent(requestId, action, response, options);
+            final TransportResponseOptions finalOptions = options;
+            ChannelFutureListener onResponseSentListener =
+                f -> transportServiceAdapter.onResponseSent(requestId, action, response, finalOptions);
+            future.addListener(onResponseSentListener);
         } finally {
             if (!addedReleaseListener && bStream != null) {
                 Releasables.close(bStream.bytes());
@@ -137,8 +141,10 @@ public class NettyTransportChannel implements TransportChannel {
         BytesReference bytes = stream.bytes();
         ChannelBuffer buffer = bytes.toChannelBuffer();
         NettyHeader.writeHeader(buffer, requestId, status, version);
-        channel.write(buffer);
-        transportServiceAdapter.onResponseSent(requestId, action, error);
+        ChannelFuture future = channel.write(buffer);
+        ChannelFutureListener onResponseSentListener =
+            f -> transportServiceAdapter.onResponseSent(requestId, action, error);
+        future.addListener(onResponseSentListener);
     }
 
     private void close() {

From be2ba53fca62465693e7b24a06b20580789081d4 Mon Sep 17 00:00:00 2001
From: spalger <email@spalger.com>
Date: Fri, 20 May 2016 13:48:16 -0700
Subject: [PATCH 0400/1311] [rest api spec] fix doc urls

---
 .../src/main/resources/rest-api-spec/api/delete_by_query.json   | 2 +-
 rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json | 2 +-
 .../src/main/resources/rest-api-spec/api/tasks.cancel.json      | 2 +-
 .../src/main/resources/rest-api-spec/api/update_by_query.json   | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
index 981aea79a1c..834133138b2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
@@ -1,6 +1,6 @@
 {
   "delete_by_query": {
-    "documentation": "https://www.elastic.co/guide/en/elasticsearch/plugins/master/plugins-reindex.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/plugins/master/plugins-delete-by-query.html",
     "methods": ["POST"],
     "url": {
       "path": "/{index}/_delete_by_query",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json
index 0dc0088dd5c..80210a2048e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json
@@ -1,6 +1,6 @@
 {
   "reindex": {
-    "documentation": "https://www.elastic.co/guide/en/elasticsearch/plugins/master/plugins-reindex.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-reindex.html",
     "methods": ["POST"],
     "url": {
       "path": "/_reindex",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.cancel.json b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.cancel.json
index 506828beaf7..69d21f4ec1d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.cancel.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.cancel.json
@@ -1,6 +1,6 @@
 {
   "tasks.cancel": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/tasks-cancel.html",
+    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html",
     "methods": ["POST"],
     "url": {
       "path": "/_tasks",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json
index a2e2b5e916c..313dbe00c46 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json
@@ -1,6 +1,6 @@
 {
   "update_by_query": {
-    "documentation": "https://www.elastic.co/guide/en/elasticsearch/plugins/master/plugins-reindex.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-update-by-query.html",
     "methods": ["POST"],
     "url": {
       "path": "/{index}/_update_by_query",

From fdfd2a2f187a817b891ebdbe67952061e10bc278 Mon Sep 17 00:00:00 2001
From: Lee Hinman <lee@writequit.org>
Date: Fri, 20 May 2016 15:01:30 -0600
Subject: [PATCH 0401/1311] Remove ScriptMode class in favor of boolean
 true/false

This removes the ScriptMode class entirely, which was an enum with two
options (ON and OFF) which essentially boiled down to true and false.
Now the boolean values are used instead.
---
 .../common/settings/Setting.java              |  4 +
 .../script/ScriptEngineRegistry.java          | 24 +++---
 .../org/elasticsearch/script/ScriptMode.java  | 64 --------------
 .../org/elasticsearch/script/ScriptModes.java | 28 +++---
 .../elasticsearch/script/ScriptModule.java    |  3 +-
 .../elasticsearch/script/ScriptService.java   | 26 ++----
 .../elasticsearch/script/ScriptSettings.java  | 74 ++++++++--------
 .../index/query/AbstractQueryTestCase.java    |  3 +-
 .../elasticsearch/script/FileScriptTests.java |  3 +-
 .../script/NativeScriptTests.java             |  7 +-
 .../script/ScriptContextTests.java            |  3 +-
 .../script/ScriptModesTests.java              | 46 +++++-----
 .../script/ScriptServiceTests.java            | 86 ++++++++-----------
 .../script/ScriptSettingsTests.java           |  7 +-
 .../aggregations/AggregatorParsingTests.java  |  3 +-
 .../aggregations/BaseAggregationTestCase.java |  3 +-
 .../BasePipelineAggregationTestCase.java      |  3 +-
 .../search/aggregations/metrics/AvgIT.java    |  5 +-
 .../search/aggregations/metrics/SumIT.java    |  5 +-
 .../aggregations/metrics/ValueCountIT.java    |  3 +-
 .../builder/SearchSourceBuilderTests.java     |  3 +-
 .../org/elasticsearch/update/UpdateIT.java    |  9 +-
 .../script/expression/ExpressionPlugin.java   |  3 +-
 .../script/mustache/MustachePlugin.java       |  3 +-
 .../messy/tests/TemplateQueryParserTests.java |  3 +-
 .../painless/PainlessPlugin.java              |  3 +-
 .../ingest/AbstractMustacheTestCase.java      |  3 +-
 .../script/MockScriptEngine.java              |  3 +-
 28 files changed, 162 insertions(+), 268 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/script/ScriptMode.java

diff --git a/core/src/main/java/org/elasticsearch/common/settings/Setting.java b/core/src/main/java/org/elasticsearch/common/settings/Setting.java
index 1efb65c18b1..1be1fa8f3f4 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/Setting.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/Setting.java
@@ -537,6 +537,10 @@ public class Setting<T> extends ToXContentToBytes {
         return new Setting<>(key, fallbackSetting, Booleans::parseBooleanExact, properties);
     }
 
+    public static Setting<Boolean> boolSetting(String key, Function<Settings, String> defaultValueFn, Property... properties) {
+        return new Setting<>(key, defaultValueFn, Booleans::parseBooleanExact, properties);
+    }
+
     public static Setting<ByteSizeValue> byteSizeSetting(String key, String percentage, Property... properties) {
         return new Setting<>(key, (s) -> percentage, (s) -> MemorySizeValue.parseBytesSizeValueOrHeapRatio(s, key), properties);
     }
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptEngineRegistry.java b/core/src/main/java/org/elasticsearch/script/ScriptEngineRegistry.java
index 226fd7b6c34..31ff9ac0b61 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptEngineRegistry.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptEngineRegistry.java
@@ -30,13 +30,13 @@ public class ScriptEngineRegistry {
 
     private final Map<Class<? extends ScriptEngineService>, String> registeredScriptEngineServices;
     private final Map<String, Class<? extends ScriptEngineService>> registeredLanguages;
-    private final Map<String, ScriptMode> defaultInlineScriptModes;
+    private final Map<String, Boolean> defaultInlineScriptEnableds;
 
     public ScriptEngineRegistry(Iterable<ScriptEngineRegistration> registrations) {
         Objects.requireNonNull(registrations);
         Map<Class<? extends ScriptEngineService>, String> registeredScriptEngineServices = new HashMap<>();
         Map<String, Class<? extends ScriptEngineService>> registeredLanguages = new HashMap<>();
-        Map<String, ScriptMode> inlineScriptModes = new HashMap<>();
+        Map<String, Boolean> inlineScriptEnableds = new HashMap<>();
         for (ScriptEngineRegistration registration : registrations) {
             String oldLanguage = registeredScriptEngineServices.putIfAbsent(registration.getScriptEngineService(),
                     registration.getScriptEngineLanguage());
@@ -51,12 +51,12 @@ public class ScriptEngineRegistry {
                 throw new IllegalArgumentException("scripting language [" + language + "] already registered for script engine service [" +
                                 scriptEngineServiceClazz.getCanonicalName() + "]");
             }
-            inlineScriptModes.put(language, registration.getDefaultInlineScriptMode());
+            inlineScriptEnableds.put(language, registration.getDefaultInlineScriptEnabled());
         }
 
         this.registeredScriptEngineServices = Collections.unmodifiableMap(registeredScriptEngineServices);
         this.registeredLanguages = Collections.unmodifiableMap(registeredLanguages);
-        this.defaultInlineScriptModes = Collections.unmodifiableMap(inlineScriptModes);
+        this.defaultInlineScriptEnableds = Collections.unmodifiableMap(inlineScriptEnableds);
     }
 
     Iterable<Class<? extends ScriptEngineService>> getRegisteredScriptEngineServices() {
@@ -72,27 +72,27 @@ public class ScriptEngineRegistry {
         return registeredLanguages;
     }
 
-    Map<String, ScriptMode> getDefaultInlineScriptModes() {
-        return this.defaultInlineScriptModes;
+    Map<String, Boolean> getDefaultInlineScriptEnableds() {
+        return this.defaultInlineScriptEnableds;
     }
 
     public static class ScriptEngineRegistration {
         private final Class<? extends ScriptEngineService> scriptEngineService;
         private final String scriptEngineLanguage;
-        private final ScriptMode defaultInlineScriptMode;
+        private final boolean defaultInlineScriptEnabled;
 
         /**
          * Register a script engine service with the default of inline scripts disabled
          */
         public ScriptEngineRegistration(Class<? extends ScriptEngineService> scriptEngineService, String scriptEngineLanguage) {
-            this(scriptEngineService, scriptEngineLanguage, ScriptMode.OFF);
+            this(scriptEngineService, scriptEngineLanguage, false);
         }
 
         /**
          * Register a script engine service with the given default mode for inline scripts
          */
         public ScriptEngineRegistration(Class<? extends ScriptEngineService> scriptEngineService, String scriptEngineLanguage,
-                                        ScriptMode defaultInlineScriptMode) {
+                                        boolean defaultInlineScriptEnabled) {
             Objects.requireNonNull(scriptEngineService);
             if (Strings.hasText(scriptEngineLanguage) == false) {
                 throw new IllegalArgumentException("languages for script engine service [" +
@@ -100,7 +100,7 @@ public class ScriptEngineRegistry {
             }
             this.scriptEngineService = scriptEngineService;
             this.scriptEngineLanguage = scriptEngineLanguage;
-            this.defaultInlineScriptMode = defaultInlineScriptMode;
+            this.defaultInlineScriptEnabled = defaultInlineScriptEnabled;
         }
 
         Class<? extends ScriptEngineService> getScriptEngineService() {
@@ -111,8 +111,8 @@ public class ScriptEngineRegistry {
             return scriptEngineLanguage;
         }
 
-        ScriptMode getDefaultInlineScriptMode() {
-            return defaultInlineScriptMode;
+        boolean getDefaultInlineScriptEnabled() {
+            return defaultInlineScriptEnabled;
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptMode.java b/core/src/main/java/org/elasticsearch/script/ScriptMode.java
deleted file mode 100644
index 6508d2f1015..00000000000
--- a/core/src/main/java/org/elasticsearch/script/ScriptMode.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.script;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Mode for a specific script, used for script settings.
- * Defines whether a certain script or category of scripts can be executed or not.
- */
-public enum ScriptMode {
-    ON("true"),
-    OFF("false");
-
-    private final String mode;
-
-    ScriptMode(String mode) {
-        this.mode = mode;
-    }
-
-    private static final Map<String, ScriptMode> SCRIPT_MODES;
-
-    static {
-        SCRIPT_MODES = new HashMap<>();
-        for (ScriptMode scriptMode : ScriptMode.values()) {
-            SCRIPT_MODES.put(scriptMode.mode, scriptMode);
-        }
-    }
-
-    static ScriptMode parse(String input) {
-        ScriptMode scriptMode = SCRIPT_MODES.get(input);
-        if (scriptMode == null) {
-            throw new IllegalArgumentException("script mode [" + input + "] not supported");
-        }
-        return scriptMode;
-    }
-
-    public String getMode() {
-        return mode;
-    }
-
-    @Override
-    public String toString() {
-        return mode;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptModes.java b/core/src/main/java/org/elasticsearch/script/ScriptModes.java
index d07a4073ba8..46ab2a44d21 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptModes.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptModes.java
@@ -29,22 +29,22 @@ import java.util.Map;
 import java.util.TreeMap;
 
 /**
- * Holds the {@link org.elasticsearch.script.ScriptMode}s for each of the different scripting languages available,
- * each script source and each scripted operation.
+ * Holds the boolean indicating the enabled mode for each of the different scripting languages available, each script source and each
+ * scripted operation.
  */
 public class ScriptModes {
 
     private static final String SCRIPT_SETTINGS_PREFIX = "script";
     private static final String ENGINE_SETTINGS_PREFIX = "script.engine";
 
-    final Map<String, ScriptMode> scriptModes;
+    final Map<String, Boolean> scriptEnabled;
 
     ScriptModes(ScriptSettings scriptSettings, Settings settings) {
-        HashMap<String, ScriptMode> scriptModes = new HashMap<>();
-        for (Setting<ScriptMode> scriptModeSetting : scriptSettings.getScriptLanguageSettings()) {
+        HashMap<String, Boolean> scriptModes = new HashMap<>();
+        for (Setting<Boolean> scriptModeSetting : scriptSettings.getScriptLanguageSettings()) {
             scriptModes.put(scriptModeSetting.getKey(), scriptModeSetting.get(settings));
         }
-        this.scriptModes = Collections.unmodifiableMap(scriptModes);
+        this.scriptEnabled = Collections.unmodifiableMap(scriptModes);
     }
 
     /**
@@ -54,14 +54,14 @@ public class ScriptModes {
      * @param lang the language that the script is written in
      * @param scriptType the type of the script
      * @param scriptContext the operation that requires the execution of the script
-     * @return whether scripts are on or off
+     * @return whether scripts are enabled (true) or disabled (false)
      */
-    public ScriptMode getScriptMode(String lang, ScriptType scriptType, ScriptContext scriptContext) {
-        //native scripts are always on as they are static by definition
+    public boolean getScriptEnabled(String lang, ScriptType scriptType, ScriptContext scriptContext) {
+        //native scripts are always enabled as they are static by definition
         if (NativeScriptEngineService.NAME.equals(lang)) {
-            return ScriptMode.ON;
+            return true;
         }
-        ScriptMode scriptMode = scriptModes.get(getKey(lang, scriptType, scriptContext));
+        Boolean scriptMode = scriptEnabled.get(getKey(lang, scriptType, scriptContext));
         if (scriptMode == null) {
             throw new IllegalArgumentException("script mode not found for lang [" + lang + "], script_type [" + scriptType + "], operation [" + scriptContext.getKey() + "]");
         }
@@ -87,10 +87,10 @@ public class ScriptModes {
     @Override
     public String toString() {
         //order settings by key before printing them out, for readability
-        TreeMap<String, ScriptMode> scriptModesTreeMap = new TreeMap<>();
-        scriptModesTreeMap.putAll(scriptModes);
+        TreeMap<String, Boolean> scriptModesTreeMap = new TreeMap<>();
+        scriptModesTreeMap.putAll(scriptEnabled);
         StringBuilder stringBuilder = new StringBuilder();
-        for (Map.Entry<String, ScriptMode> stringScriptModeEntry : scriptModesTreeMap.entrySet()) {
+        for (Map.Entry<String, Boolean> stringScriptModeEntry : scriptModesTreeMap.entrySet()) {
             stringBuilder.append(stringScriptModeEntry.getKey()).append(": ").append(stringScriptModeEntry.getValue()).append("\n");
         }
         return stringBuilder.toString();
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptModule.java b/core/src/main/java/org/elasticsearch/script/ScriptModule.java
index 0fa4d9ea731..c08c31c9bf2 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptModule.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptModule.java
@@ -24,7 +24,6 @@ import org.elasticsearch.common.inject.multibindings.MapBinder;
 import org.elasticsearch.common.inject.multibindings.Multibinder;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.SettingsModule;
-import org.elasticsearch.script.ScriptMode;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -42,7 +41,7 @@ public class ScriptModule extends AbstractModule {
 
     {
         scriptEngineRegistrations.add(new ScriptEngineRegistry.ScriptEngineRegistration(NativeScriptEngineService.class,
-                        NativeScriptEngineService.NAME, ScriptMode.ON));
+                        NativeScriptEngineService.NAME, true));
     }
 
     private final Map<String, Class<? extends NativeScriptFactory>> scripts = new HashMap<>();
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java
index f77a0aae330..cf0ba1ca941 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java
@@ -473,15 +473,7 @@ public class ScriptService extends AbstractComponent implements Closeable {
         if (scriptContextRegistry.isSupportedContext(scriptContext) == false) {
             throw new IllegalArgumentException("script context [" + scriptContext.getKey() + "] not supported");
         }
-        ScriptMode mode = scriptModes.getScriptMode(lang, scriptType, scriptContext);
-        switch (mode) {
-            case ON:
-                return true;
-            case OFF:
-                return false;
-            default:
-                throw new IllegalArgumentException("script mode [" + mode + "] not supported");
-        }
+        return scriptModes.getScriptEnabled(lang, scriptType, scriptContext);
     }
 
     public ScriptStats stats() {
@@ -610,14 +602,14 @@ public class ScriptService extends AbstractComponent implements Closeable {
      */
     public enum ScriptType {
 
-        INLINE(0, "inline", "inline", ScriptMode.OFF),
-        STORED(1, "id", "stored", ScriptMode.OFF),
-        FILE(2, "file", "file", ScriptMode.ON);
+        INLINE(0, "inline", "inline", false),
+        STORED(1, "id", "stored", false),
+        FILE(2, "file", "file", true);
 
         private final int val;
         private final ParseField parseField;
         private final String scriptType;
-        private final ScriptMode defaultScriptMode;
+        private final boolean defaultScriptEnabled;
 
         public static ScriptType readFrom(StreamInput in) throws IOException {
             int scriptTypeVal = in.readVInt();
@@ -638,19 +630,19 @@ public class ScriptService extends AbstractComponent implements Closeable {
             }
         }
 
-        ScriptType(int val, String name, String scriptType, ScriptMode defaultScriptMode) {
+        ScriptType(int val, String name, String scriptType, boolean defaultScriptEnabled) {
             this.val = val;
             this.parseField = new ParseField(name);
             this.scriptType = scriptType;
-            this.defaultScriptMode = defaultScriptMode;
+            this.defaultScriptEnabled = defaultScriptEnabled;
         }
 
         public ParseField getParseField() {
             return parseField;
         }
 
-        public ScriptMode getDefaultScriptMode() {
-            return defaultScriptMode;
+        public boolean getDefaultScriptEnabled() {
+            return defaultScriptEnabled;
         }
 
         public String getScriptType() {
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java b/core/src/main/java/org/elasticsearch/script/ScriptSettings.java
index 433912e47e5..71013f28a06 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptSettings.java
@@ -37,29 +37,28 @@ public class ScriptSettings {
 
     public final static String DEFAULT_LANG = "groovy";
 
-    private final static Map<ScriptService.ScriptType, Setting<ScriptMode>> SCRIPT_TYPE_SETTING_MAP;
+    private final static Map<ScriptService.ScriptType, Setting<Boolean>> SCRIPT_TYPE_SETTING_MAP;
 
     static {
-        Map<ScriptService.ScriptType, Setting<ScriptMode>> scriptTypeSettingMap = new HashMap<>();
+        Map<ScriptService.ScriptType, Setting<Boolean>> scriptTypeSettingMap = new HashMap<>();
         for (ScriptService.ScriptType scriptType : ScriptService.ScriptType.values()) {
-            scriptTypeSettingMap.put(scriptType, new Setting<>(
+            scriptTypeSettingMap.put(scriptType, Setting.boolSetting(
                 ScriptModes.sourceKey(scriptType),
-                scriptType.getDefaultScriptMode().getMode(),
-                ScriptMode::parse,
+                scriptType.getDefaultScriptEnabled(),
                 Property.NodeScope));
         }
         SCRIPT_TYPE_SETTING_MAP = Collections.unmodifiableMap(scriptTypeSettingMap);
     }
 
-    private final Map<ScriptContext, Setting<ScriptMode>> scriptContextSettingMap;
-    private final List<Setting<ScriptMode>> scriptLanguageSettings;
+    private final Map<ScriptContext, Setting<Boolean>> scriptContextSettingMap;
+    private final List<Setting<Boolean>> scriptLanguageSettings;
     private final Setting<String> defaultScriptLanguageSetting;
 
     public ScriptSettings(ScriptEngineRegistry scriptEngineRegistry, ScriptContextRegistry scriptContextRegistry) {
-        Map<ScriptContext, Setting<ScriptMode>> scriptContextSettingMap = contextSettings(scriptContextRegistry);
+        Map<ScriptContext, Setting<Boolean>> scriptContextSettingMap = contextSettings(scriptContextRegistry);
         this.scriptContextSettingMap = Collections.unmodifiableMap(scriptContextSettingMap);
 
-        List<Setting<ScriptMode>> scriptLanguageSettings = languageSettings(SCRIPT_TYPE_SETTING_MAP, scriptContextSettingMap, scriptEngineRegistry, scriptContextRegistry);
+        List<Setting<Boolean>> scriptLanguageSettings = languageSettings(SCRIPT_TYPE_SETTING_MAP, scriptContextSettingMap, scriptEngineRegistry, scriptContextRegistry);
         this.scriptLanguageSettings = Collections.unmodifiableList(scriptLanguageSettings);
 
         this.defaultScriptLanguageSetting = new Setting<>("script.default_lang", DEFAULT_LANG, setting -> {
@@ -70,24 +69,20 @@ public class ScriptSettings {
         }, Property.NodeScope);
     }
 
-    private static Map<ScriptContext, Setting<ScriptMode>> contextSettings(ScriptContextRegistry scriptContextRegistry) {
-        Map<ScriptContext, Setting<ScriptMode>> scriptContextSettingMap = new HashMap<>();
+    private static Map<ScriptContext, Setting<Boolean>> contextSettings(ScriptContextRegistry scriptContextRegistry) {
+        Map<ScriptContext, Setting<Boolean>> scriptContextSettingMap = new HashMap<>();
         for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) {
-            scriptContextSettingMap.put(scriptContext, new Setting<>(
-                ScriptModes.operationKey(scriptContext),
-                ScriptMode.OFF.getMode(),
-                ScriptMode::parse,
-                Property.NodeScope
-            ));
+            scriptContextSettingMap.put(scriptContext,
+                    Setting.boolSetting(ScriptModes.operationKey(scriptContext), false, Property.NodeScope));
         }
         return scriptContextSettingMap;
     }
 
-    private static List<Setting<ScriptMode>> languageSettings(Map<ScriptService.ScriptType, Setting<ScriptMode>> scriptTypeSettingMap,
-                                                              Map<ScriptContext, Setting<ScriptMode>> scriptContextSettingMap,
+    private static List<Setting<Boolean>> languageSettings(Map<ScriptService.ScriptType, Setting<Boolean>> scriptTypeSettingMap,
+                                                              Map<ScriptContext, Setting<Boolean>> scriptContextSettingMap,
                                                               ScriptEngineRegistry scriptEngineRegistry,
                                                               ScriptContextRegistry scriptContextRegistry) {
-        final List<Setting<ScriptMode>> scriptModeSettings = new ArrayList<>();
+        final List<Setting<Boolean>> scriptModeSettings = new ArrayList<>();
 
         for (final Class<? extends ScriptEngineService> scriptEngineService : scriptEngineRegistry.getRegisteredScriptEngineServices()) {
             if (scriptEngineService == NativeScriptEngineService.class) {
@@ -97,17 +92,17 @@ public class ScriptSettings {
             final String language = scriptEngineRegistry.getLanguage(scriptEngineService);
             for (final ScriptService.ScriptType scriptType : ScriptService.ScriptType.values()) {
                 // Top level, like "script.engine.groovy.inline"
-                final ScriptMode defaultNonFileScriptMode = scriptEngineRegistry.getDefaultInlineScriptModes().get(language);
-                ScriptMode defaultLangAndType = defaultNonFileScriptMode;
+                final boolean defaultNonFileScriptMode = scriptEngineRegistry.getDefaultInlineScriptEnableds().get(language);
+                boolean defaultLangAndType = defaultNonFileScriptMode;
                 // Files are treated differently because they are never default-deny
                 if (ScriptService.ScriptType.FILE == scriptType) {
-                    defaultLangAndType = ScriptService.ScriptType.FILE.getDefaultScriptMode();
+                    defaultLangAndType = ScriptService.ScriptType.FILE.getDefaultScriptEnabled();
                 }
-                final ScriptMode defaultIfNothingSet = defaultLangAndType;
+                final boolean defaultIfNothingSet = defaultLangAndType;
 
                 // Setting for something like "script.engine.groovy.inline"
-                final Setting<ScriptMode> langAndTypeSetting = new Setting<>(ScriptModes.getGlobalKey(language, scriptType),
-                        defaultLangAndType.toString(), ScriptMode::parse, Property.NodeScope);
+                final Setting<Boolean> langAndTypeSetting = Setting.boolSetting(ScriptModes.getGlobalKey(language, scriptType),
+                        defaultLangAndType, Property.NodeScope);
                 scriptModeSettings.add(langAndTypeSetting);
 
                 for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) {
@@ -115,32 +110,31 @@ public class ScriptSettings {
                     // A function that, given a setting, will return what the default should be. Since the fine-grained script settings
                     // read from a bunch of different places this is implemented in this way.
                     Function<Settings, String> defaultSettingFn = settings -> {
-                        final Setting<ScriptMode> globalOpSetting = scriptContextSettingMap.get(scriptContext);
-                        final Setting<ScriptMode> globalTypeSetting = scriptTypeSettingMap.get(scriptType);
-                        final Setting<ScriptMode> langAndTypeAndContextSetting = new Setting<>(langAndTypeAndContextName,
-                                defaultIfNothingSet.toString(), ScriptMode::parse, Property.NodeScope);
+                        final Setting<Boolean> globalOpSetting = scriptContextSettingMap.get(scriptContext);
+                        final Setting<Boolean> globalTypeSetting = scriptTypeSettingMap.get(scriptType);
+                        final Setting<Boolean> langAndTypeAndContextSetting = Setting.boolSetting(langAndTypeAndContextName,
+                                defaultIfNothingSet, Property.NodeScope);
 
                         // fallback logic for script mode settings
                         if (langAndTypeAndContextSetting.exists(settings)) {
                             // like: "script.engine.groovy.inline.aggs: true"
-                            return langAndTypeAndContextSetting.get(settings).getMode();
+                            return langAndTypeAndContextSetting.get(settings).toString();
                         } else if (langAndTypeSetting.exists(settings)) {
                             // like: "script.engine.groovy.inline: true"
-                            return langAndTypeSetting.get(settings).getMode();
+                            return langAndTypeSetting.get(settings).toString();
                         } else if (globalOpSetting.exists(settings)) {
                             // like: "script.aggs: true"
-                            return globalOpSetting.get(settings).getMode();
+                            return globalOpSetting.get(settings).toString();
                         } else if (globalTypeSetting.exists(settings)) {
                             // like: "script.inline: true"
-                            return globalTypeSetting.get(settings).getMode();
+                            return globalTypeSetting.get(settings).toString();
                         } else {
                             // Nothing is set!
-                            return defaultIfNothingSet.getMode();
+                            return Boolean.toString(defaultIfNothingSet);
                         }
                     };
                     // The actual setting for finest grained script settings
-                    Setting<ScriptMode> setting = new Setting<>(langAndTypeAndContextName, defaultSettingFn,
-                            ScriptMode::parse, Property.NodeScope);
+                    Setting<Boolean> setting = Setting.boolSetting(langAndTypeAndContextName, defaultSettingFn, Property.NodeScope);
                     scriptModeSettings.add(setting);
                 }
             }
@@ -148,15 +142,15 @@ public class ScriptSettings {
         return scriptModeSettings;
     }
 
-    public Iterable<Setting<ScriptMode>> getScriptTypeSettings() {
+    public Iterable<Setting<Boolean>> getScriptTypeSettings() {
         return Collections.unmodifiableCollection(SCRIPT_TYPE_SETTING_MAP.values());
     }
 
-    public Iterable<Setting<ScriptMode>> getScriptContextSettings() {
+    public Iterable<Setting<Boolean>> getScriptContextSettings() {
         return Collections.unmodifiableCollection(scriptContextSettingMap.values());
     }
 
-    public Iterable<Setting<ScriptMode>> getScriptLanguageSettings() {
+    public Iterable<Setting<Boolean>> getScriptLanguageSettings() {
         return scriptLanguageSettings;
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
index b150000daf1..b49488bd7c2 100644
--- a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
@@ -22,7 +22,6 @@ package org.elasticsearch.index.query;
 import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
 import com.fasterxml.jackson.core.JsonParseException;
 import com.fasterxml.jackson.core.io.JsonStringEncoder;
-import org.elasticsearch.script.ScriptMode;
 
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.Query;
@@ -239,7 +238,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
                 ScriptEngineRegistry scriptEngineRegistry =
                         new ScriptEngineRegistry(Collections
                                 .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
-                                                                                                 MockScriptEngine.NAME, ScriptMode.ON)));
+                                                                                                 MockScriptEngine.NAME, true)));
                 bind(ScriptEngineRegistry.class).toInstance(scriptEngineRegistry);
                 ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts);
                 bind(ScriptContextRegistry.class).toInstance(scriptContextRegistry);
diff --git a/core/src/test/java/org/elasticsearch/script/FileScriptTests.java b/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
index e5f78d7474e..fefe55b9ec3 100644
--- a/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
@@ -21,7 +21,6 @@ package org.elasticsearch.script;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.script.MockScriptEngine.MockCompiledScript;
-import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.test.ESTestCase;
 
 import java.nio.file.Files;
@@ -46,7 +45,7 @@ public class FileScriptTests extends ESTestCase {
             .put(settings)
             .build();
         Set<ScriptEngineService> engines = new HashSet<>(Collections.singletonList(new MockScriptEngine()));
-        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class, MockScriptEngine.NAME, ScriptMode.ON)));
+        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class, MockScriptEngine.NAME, true)));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
         return new ScriptService(settings, new Environment(settings), engines, null, scriptEngineRegistry, scriptContextRegistry, scriptSettings);
diff --git a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
index 045f62ff8fe..5e1dc740f9e 100644
--- a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
@@ -28,7 +28,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.EnvironmentModule;
-import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -75,10 +74,10 @@ public class NativeScriptTests extends ESTestCase {
         Settings.Builder builder = Settings.builder();
         if (randomBoolean()) {
             ScriptType scriptType = randomFrom(ScriptType.values());
-            builder.put("script" + "." + scriptType.getScriptType(), randomFrom(ScriptMode.values()));
+            builder.put("script" + "." + scriptType.getScriptType(), randomBoolean());
         } else {
             ScriptContext scriptContext = randomFrom(ScriptContext.Standard.values());
-            builder.put("script" + "." + scriptContext.getKey(), randomFrom(ScriptMode.values()));
+            builder.put("script" + "." + scriptContext.getKey(), randomBoolean());
         }
         Settings settings = builder.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build();
         Environment environment = new Environment(settings);
@@ -86,7 +85,7 @@ public class NativeScriptTests extends ESTestCase {
         Map<String, NativeScriptFactory> nativeScriptFactoryMap = new HashMap<>();
         nativeScriptFactoryMap.put("my", new MyNativeScriptFactory());
         Set<ScriptEngineService> scriptEngineServices = singleton(new NativeScriptEngineService(settings, nativeScriptFactoryMap));
-        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(NativeScriptEngineService.class, NativeScriptEngineService.NAME, ScriptMode.ON)));
+        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(NativeScriptEngineService.class, NativeScriptEngineService.NAME, true)));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(new ArrayList<>());
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
         ScriptService scriptService = new ScriptService(settings, environment, scriptEngineServices, resourceWatcherService, scriptEngineRegistry, scriptContextRegistry, scriptSettings);
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
index 8405366f34d..715694fe890 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
@@ -21,7 +21,6 @@ package org.elasticsearch.script;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Arrays;
@@ -46,7 +45,7 @@ public class ScriptContextTests extends ESTestCase {
             .build();
         Set<ScriptEngineService> engines = new HashSet<>(Collections.singletonList(new MockScriptEngine()));
         ScriptEngineRegistry.ScriptEngineRegistration registration =
-                new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class, MockScriptEngine.NAME, ScriptMode.ON);
+                new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class, MockScriptEngine.NAME, true);
         ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(registration));
         List<ScriptContext.Plugin> customContexts = Arrays.asList(
             new ScriptContext.Plugin(PLUGIN_NAME, "custom_op"),
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java b/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
index 800c079b889..8f02bee234d 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
@@ -83,7 +83,7 @@ public class ScriptModesTests extends ESTestCase {
     @After
     public void assertNativeScriptsAreAlwaysAllowed() {
         if (assertScriptModesNonNull) {
-            assertThat(scriptModes.getScriptMode(NativeScriptEngineService.NAME, randomFrom(ScriptType.values()), randomFrom(scriptContexts)), equalTo(ScriptMode.ON));
+            assertThat(scriptModes.getScriptEnabled(NativeScriptEngineService.NAME, randomFrom(ScriptType.values()), randomFrom(scriptContexts)), equalTo(true));
         }
     }
 
@@ -93,7 +93,7 @@ public class ScriptModesTests extends ESTestCase {
             assertThat(scriptModes, notNullValue());
             int numberOfSettings = ScriptType.values().length * scriptContextRegistry.scriptContexts().size();
             numberOfSettings += 3; // for top-level inline/store/file settings
-            assertThat(scriptModes.scriptModes.size(), equalTo(numberOfSettings));
+            assertThat(scriptModes.scriptEnabled.size(), equalTo(numberOfSettings));
             if (assertAllSettingsWereChecked) {
                 assertThat(checkedSettings.size(), equalTo(numberOfSettings));
             }
@@ -102,15 +102,15 @@ public class ScriptModesTests extends ESTestCase {
 
     public void testDefaultSettings() {
         this.scriptModes = new ScriptModes(scriptSettings, Settings.EMPTY);
-        assertScriptModesAllOps(ScriptMode.ON, ScriptType.FILE);
-        assertScriptModesAllOps(ScriptMode.OFF, ScriptType.STORED, ScriptType.INLINE);
+        assertScriptModesAllOps(true, ScriptType.FILE);
+        assertScriptModesAllOps(false, ScriptType.STORED, ScriptType.INLINE);
     }
 
     public void testMissingSetting() {
         assertAllSettingsWereChecked = false;
         this.scriptModes = new ScriptModes(scriptSettings, Settings.EMPTY);
         try {
-            scriptModes.getScriptMode("non_existing", randomFrom(ScriptType.values()), randomFrom(scriptContexts));
+            scriptModes.getScriptEnabled("non_existing", randomFrom(ScriptType.values()), randomFrom(scriptContexts));
             fail("Expected IllegalArgumentException");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), containsString("not found for lang [non_existing]"));
@@ -120,13 +120,13 @@ public class ScriptModesTests extends ESTestCase {
     public void testScriptTypeGenericSettings() {
         int randomInt = randomIntBetween(1, ScriptType.values().length - 1);
         Set<ScriptType> randomScriptTypesSet = new HashSet<>();
-        ScriptMode[] randomScriptModes = new ScriptMode[randomInt];
+        boolean[] randomScriptModes = new boolean[randomInt];
         for (int i = 0; i < randomInt; i++) {
             boolean added = false;
             while (added == false) {
                 added = randomScriptTypesSet.add(randomFrom(ScriptType.values()));
             }
-            randomScriptModes[i] = randomFrom(ScriptMode.values());
+            randomScriptModes[i] = randomBoolean();
         }
         ScriptType[] randomScriptTypes = randomScriptTypesSet.toArray(new ScriptType[randomScriptTypesSet.size()]);
         Settings.Builder builder = Settings.builder();
@@ -139,26 +139,26 @@ public class ScriptModesTests extends ESTestCase {
             assertScriptModesAllOps(randomScriptModes[i], randomScriptTypes[i]);
         }
         if (randomScriptTypesSet.contains(ScriptType.FILE) == false) {
-            assertScriptModesAllOps(ScriptMode.ON, ScriptType.FILE);
+            assertScriptModesAllOps(true, ScriptType.FILE);
         }
         if (randomScriptTypesSet.contains(ScriptType.STORED) == false) {
-            assertScriptModesAllOps(ScriptMode.OFF, ScriptType.STORED);
+            assertScriptModesAllOps(false, ScriptType.STORED);
         }
         if (randomScriptTypesSet.contains(ScriptType.INLINE) == false) {
-            assertScriptModesAllOps(ScriptMode.OFF, ScriptType.INLINE);
+            assertScriptModesAllOps(false, ScriptType.INLINE);
         }
     }
 
     public void testScriptContextGenericSettings() {
         int randomInt = randomIntBetween(1, scriptContexts.length - 1);
         Set<ScriptContext> randomScriptContextsSet = new HashSet<>();
-        ScriptMode[] randomScriptModes = new ScriptMode[randomInt];
+        boolean[] randomScriptModes = new boolean[randomInt];
         for (int i = 0; i < randomInt; i++) {
             boolean added = false;
             while (added == false) {
                 added = randomScriptContextsSet.add(randomFrom(scriptContexts));
             }
-            randomScriptModes[i] = randomFrom(ScriptMode.values());
+            randomScriptModes[i] = randomBoolean();
         }
         ScriptContext[] randomScriptContexts = randomScriptContextsSet.toArray(new ScriptContext[randomScriptContextsSet.size()]);
         Settings.Builder builder = Settings.builder();
@@ -172,8 +172,8 @@ public class ScriptModesTests extends ESTestCase {
         }
 
         ScriptContext[] complementOf = complementOf(randomScriptContexts);
-        assertScriptModes(ScriptMode.ON, new ScriptType[]{ScriptType.FILE}, complementOf);
-        assertScriptModes(ScriptMode.OFF, new ScriptType[]{ScriptType.STORED, ScriptType.INLINE}, complementOf);
+        assertScriptModes(true, new ScriptType[]{ScriptType.FILE}, complementOf);
+        assertScriptModes(false, new ScriptType[]{ScriptType.STORED, ScriptType.INLINE}, complementOf);
     }
 
     public void testConflictingScriptTypeAndOpGenericSettings() {
@@ -184,28 +184,28 @@ public class ScriptModesTests extends ESTestCase {
                 .put("script.inline", "true");
         //operations generic settings have precedence over script type generic settings
         this.scriptModes = new ScriptModes(scriptSettings, builder.build());
-        assertScriptModesAllTypes(ScriptMode.OFF, scriptContext);
+        assertScriptModesAllTypes(false, scriptContext);
         ScriptContext[] complementOf = complementOf(scriptContext);
-        assertScriptModes(ScriptMode.ON, new ScriptType[]{ScriptType.FILE, ScriptType.STORED}, complementOf);
-        assertScriptModes(ScriptMode.ON, new ScriptType[]{ScriptType.INLINE}, complementOf);
+        assertScriptModes(true, new ScriptType[]{ScriptType.FILE, ScriptType.STORED}, complementOf);
+        assertScriptModes(true, new ScriptType[]{ScriptType.INLINE}, complementOf);
     }
 
-    private void assertScriptModesAllOps(ScriptMode expectedScriptMode, ScriptType... scriptTypes) {
-        assertScriptModes(expectedScriptMode, scriptTypes, scriptContexts);
+    private void assertScriptModesAllOps(boolean expectedScriptEnabled, ScriptType... scriptTypes) {
+        assertScriptModes(expectedScriptEnabled, scriptTypes, scriptContexts);
     }
 
-    private void assertScriptModesAllTypes(ScriptMode expectedScriptMode, ScriptContext... scriptContexts) {
-        assertScriptModes(expectedScriptMode, ScriptType.values(), scriptContexts);
+    private void assertScriptModesAllTypes(boolean expectedScriptEnabled, ScriptContext... scriptContexts) {
+        assertScriptModes(expectedScriptEnabled, ScriptType.values(), scriptContexts);
     }
 
-    private void assertScriptModes(ScriptMode expectedScriptMode, ScriptType[] scriptTypes, ScriptContext... scriptContexts) {
+    private void assertScriptModes(boolean expectedScriptEnabled, ScriptType[] scriptTypes, ScriptContext... scriptContexts) {
         assert scriptTypes.length > 0;
         assert scriptContexts.length > 0;
         for (ScriptType scriptType : scriptTypes) {
             checkedSettings.add("script.engine.custom." + scriptType);
             for (ScriptContext scriptContext : scriptContexts) {
                 assertThat("custom." + scriptType + "." + scriptContext.getKey() + " doesn't have the expected value",
-                        scriptModes.getScriptMode("custom", scriptType, scriptContext), equalTo(expectedScriptMode));
+                        scriptModes.getScriptEnabled("custom", scriptType, scriptContext), equalTo(expectedScriptEnabled));
                 checkedSettings.add("custom." + scriptType + "." + scriptContext);
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
index 8fd8f674c3a..890ffccc514 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
@@ -71,12 +71,12 @@ public class ScriptServiceTests extends ESTestCase {
     private Path scriptsFilePath;
     private Settings baseSettings;
 
-    private static final Map<ScriptType, ScriptMode> DEFAULT_SCRIPT_MODES = new HashMap<>();
+    private static final Map<ScriptType, Boolean> DEFAULT_SCRIPT_ENABLED = new HashMap<>();
 
     static {
-        DEFAULT_SCRIPT_MODES.put(ScriptType.FILE, ScriptMode.ON);
-        DEFAULT_SCRIPT_MODES.put(ScriptType.STORED, ScriptMode.OFF);
-        DEFAULT_SCRIPT_MODES.put(ScriptType.INLINE, ScriptMode.OFF);
+        DEFAULT_SCRIPT_ENABLED.put(ScriptType.FILE, true);
+        DEFAULT_SCRIPT_ENABLED.put(ScriptType.STORED, false);
+        DEFAULT_SCRIPT_ENABLED.put(ScriptType.INLINE, false);
     }
 
     @Before
@@ -110,7 +110,7 @@ public class ScriptServiceTests extends ESTestCase {
             contexts.put(context, new ScriptContext.Plugin(plugin, operation));
         }
         List<ScriptEngineRegistry.ScriptEngineRegistration> registries = new ArrayList<>(2);
-        registries.add(new ScriptEngineRegistry.ScriptEngineRegistration(TestEngineService.class, TestEngineService.NAME, ScriptMode.ON));
+        registries.add(new ScriptEngineRegistry.ScriptEngineRegistration(TestEngineService.class, TestEngineService.NAME, true));
         registries.add(new ScriptEngineRegistry.ScriptEngineRegistration(TestDangerousEngineService.class, TestDangerousEngineService.NAME));
         scriptEngineRegistry = new ScriptEngineRegistry(registries);
         scriptContextRegistry = new ScriptContextRegistry(contexts.values());
@@ -215,25 +215,25 @@ public class ScriptServiceTests extends ESTestCase {
     public void testFineGrainedSettings() throws IOException {
         //collect the fine-grained settings to set for this run
         int numScriptSettings = randomIntBetween(0, ScriptType.values().length);
-        Map<ScriptType, ScriptMode> scriptSourceSettings = new HashMap<>();
+        Map<ScriptType, Boolean> scriptSourceSettings = new HashMap<>();
         for (int i = 0; i < numScriptSettings; i++) {
             ScriptType scriptType;
             do {
                 scriptType = randomFrom(ScriptType.values());
             } while (scriptSourceSettings.containsKey(scriptType));
-            scriptSourceSettings.put(scriptType, randomFrom(ScriptMode.values()));
+            scriptSourceSettings.put(scriptType, randomBoolean());
         }
         int numScriptContextSettings = randomIntBetween(0, this.scriptContextRegistry.scriptContexts().size());
-        Map<ScriptContext, ScriptMode> scriptContextSettings = new HashMap<>();
+        Map<ScriptContext, Boolean> scriptContextSettings = new HashMap<>();
         for (int i = 0; i < numScriptContextSettings; i++) {
             ScriptContext scriptContext;
             do {
                 scriptContext = randomFrom(this.scriptContexts);
             } while (scriptContextSettings.containsKey(scriptContext));
-            scriptContextSettings.put(scriptContext, randomFrom(ScriptMode.values()));
+            scriptContextSettings.put(scriptContext, randomBoolean());
         }
         int numEngineSettings = randomIntBetween(0, ScriptType.values().length * scriptContexts.length);
-        Map<String, ScriptMode> engineSettings = new HashMap<>();
+        Map<String, Boolean> engineSettings = new HashMap<>();
         for (int i = 0; i < numEngineSettings; i++) {
             String settingKey;
             do {
@@ -241,43 +241,34 @@ public class ScriptServiceTests extends ESTestCase {
                 ScriptContext scriptContext = randomFrom(this.scriptContexts);
                 settingKey = scriptEngineService.getType() + "." + scriptType + "." + scriptContext.getKey();
             } while (engineSettings.containsKey(settingKey));
-            engineSettings.put(settingKey, randomFrom(ScriptMode.values()));
+            engineSettings.put(settingKey, randomBoolean());
         }
         //set the selected fine-grained settings
         Settings.Builder builder = Settings.builder();
-        for (Map.Entry<ScriptType, ScriptMode> entry : scriptSourceSettings.entrySet()) {
-            switch (entry.getValue()) {
-                case ON:
-                    builder.put("script" + "." + entry.getKey().getScriptType(), "true");
-                    break;
-                case OFF:
-                    builder.put("script" + "." + entry.getKey().getScriptType(), "false");
-                    break;
+        for (Map.Entry<ScriptType, Boolean> entry : scriptSourceSettings.entrySet()) {
+            if (entry.getValue()) {
+                builder.put("script" + "." + entry.getKey().getScriptType(), "true");
+            } else {
+                builder.put("script" + "." + entry.getKey().getScriptType(), "false");
             }
         }
-        for (Map.Entry<ScriptContext, ScriptMode> entry : scriptContextSettings.entrySet()) {
-            switch (entry.getValue()) {
-                case ON:
-                    builder.put("script" + "." + entry.getKey().getKey(), "true");
-                    break;
-                case OFF:
-                    builder.put("script" + "." + entry.getKey().getKey(), "false");
-                    break;
+        for (Map.Entry<ScriptContext, Boolean> entry : scriptContextSettings.entrySet()) {
+            if (entry.getValue()) {
+                builder.put("script" + "." + entry.getKey().getKey(), "true");
+            } else {
+                builder.put("script" + "." + entry.getKey().getKey(), "false");
             }
         }
-        for (Map.Entry<String, ScriptMode> entry : engineSettings.entrySet()) {
+        for (Map.Entry<String, Boolean> entry : engineSettings.entrySet()) {
             int delimiter = entry.getKey().indexOf('.');
             String part1 = entry.getKey().substring(0, delimiter);
             String part2 = entry.getKey().substring(delimiter + 1);
 
             String lang = randomFrom(scriptEnginesByLangMap.get(part1).getType());
-            switch (entry.getValue()) {
-                case ON:
-                    builder.put("script.engine" + "." + lang + "." + part2, "true");
-                    break;
-                case OFF:
-                    builder.put("script.engine" + "." + lang + "." + part2, "false");
-                    break;
+            if (entry.getValue()) {
+                builder.put("script.engine" + "." + lang + "." + part2, "true");
+            } else {
+                builder.put("script.engine" + "." + lang + "." + part2, "false");
             }
         }
 
@@ -290,25 +281,22 @@ public class ScriptServiceTests extends ESTestCase {
             String script = scriptType == ScriptType.FILE ? "file_script" : "script";
             for (ScriptContext scriptContext : this.scriptContexts) {
                 //fallback mechanism: 1) engine specific settings 2) op based settings 3) source based settings
-                ScriptMode scriptMode = engineSettings.get(dangerousScriptEngineService.getType() + "." + scriptType + "." + scriptContext.getKey());
-                if (scriptMode == null) {
-                    scriptMode = scriptContextSettings.get(scriptContext);
+                Boolean scriptEnabled = engineSettings.get(dangerousScriptEngineService.getType() + "." + scriptType + "." + scriptContext.getKey());
+                if (scriptEnabled == null) {
+                    scriptEnabled = scriptContextSettings.get(scriptContext);
                 }
-                if (scriptMode == null) {
-                    scriptMode = scriptSourceSettings.get(scriptType);
+                if (scriptEnabled == null) {
+                    scriptEnabled = scriptSourceSettings.get(scriptType);
                 }
-                if (scriptMode == null) {
-                    scriptMode = DEFAULT_SCRIPT_MODES.get(scriptType);
+                if (scriptEnabled == null) {
+                    scriptEnabled = DEFAULT_SCRIPT_ENABLED.get(scriptType);
                 }
 
                 String lang = dangerousScriptEngineService.getType();
-                switch (scriptMode) {
-                    case ON:
-                        assertCompileAccepted(lang, script, scriptType, scriptContext);
-                        break;
-                    case OFF:
-                        assertCompileRejected(lang, script, scriptType, scriptContext);
-                        break;
+                if (scriptEnabled) {
+                    assertCompileAccepted(lang, script, scriptType, scriptContext);
+                } else {
+                    assertCompileRejected(lang, script, scriptType, scriptContext);
                 }
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java b/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
index 3d82e2f1468..92598ec8dd2 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
@@ -22,7 +22,6 @@ package org.elasticsearch.script;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.test.ESTestCase;
 
@@ -39,7 +38,7 @@ public class ScriptSettingsTests extends ESTestCase {
 
     public void testDefaultLanguageIsGroovy() {
         ScriptEngineRegistry scriptEngineRegistry =
-                new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(CustomScriptEngineService.class, CustomScriptEngineService.NAME, ScriptMode.ON)));
+                new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(CustomScriptEngineService.class, CustomScriptEngineService.NAME, true)));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
         assertThat(scriptSettings.getDefaultScriptLanguageSetting().get(Settings.EMPTY), equalTo("groovy"));
@@ -47,7 +46,7 @@ public class ScriptSettingsTests extends ESTestCase {
 
     public void testCustomDefaultLanguage() {
         ScriptEngineRegistry scriptEngineRegistry =
-            new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(CustomScriptEngineService.class, CustomScriptEngineService.NAME, ScriptMode.ON)));
+            new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(CustomScriptEngineService.class, CustomScriptEngineService.NAME, true)));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
         String defaultLanguage = CustomScriptEngineService.NAME;
@@ -57,7 +56,7 @@ public class ScriptSettingsTests extends ESTestCase {
 
     public void testInvalidDefaultLanguage() {
         ScriptEngineRegistry scriptEngineRegistry =
-            new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(CustomScriptEngineService.class, CustomScriptEngineService.NAME, ScriptMode.ON)));
+            new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(CustomScriptEngineService.class, CustomScriptEngineService.NAME, true)));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
         Settings settings = Settings.builder().put("script.default_lang", "C++").build();
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
index 2949dadcc49..60185dbae28 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
@@ -51,7 +51,6 @@ import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptContextRegistry;
 import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
-import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptSettings;
@@ -135,7 +134,7 @@ public class AggregatorParsingTests extends ESTestCase {
                         new ScriptEngineRegistry(Collections
                                 .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
                                                                                                  MockScriptEngine.NAME,
-                                                                                                 ScriptMode.ON)));
+                                                                                                 true)));
                 bind(ScriptEngineRegistry.class).toInstance(scriptEngineRegistry);
                 ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts);
                 bind(ScriptContextRegistry.class).toInstance(scriptContextRegistry);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
index e986ab1288f..8df37e25f76 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
@@ -55,7 +55,6 @@ import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptContextRegistry;
 import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
-import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptSettings;
@@ -149,7 +148,7 @@ public abstract class BaseAggregationTestCase<AB extends AggregationBuilder<AB>>
                         new ScriptEngineRegistry(Collections
                                 .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
                                                                                                  MockScriptEngine.NAME,
-                                                                                                 ScriptMode.ON)));
+                                                                                                 true)));
                 bind(ScriptEngineRegistry.class).toInstance(scriptEngineRegistry);
                 ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts);
                 bind(ScriptContextRegistry.class).toInstance(scriptContextRegistry);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
index 113b52cb0b6..c20344f35fc 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
@@ -55,7 +55,6 @@ import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptContextRegistry;
 import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
-import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptSettings;
@@ -148,7 +147,7 @@ public abstract class BasePipelineAggregationTestCase<AF extends PipelineAggrega
                 ScriptEngineRegistry scriptEngineRegistry =
                         new ScriptEngineRegistry(Collections
                                 .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
-                                                                                                 MockScriptEngine.NAME, ScriptMode.ON)));
+                                                                                                 MockScriptEngine.NAME, true)));
                 bind(ScriptEngineRegistry.class).toInstance(scriptEngineRegistry);
                 ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts);
                 bind(ScriptContextRegistry.class).toInstance(scriptContextRegistry);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
index a01797ccf49..a19b274c4db 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
@@ -28,7 +28,6 @@ import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
-import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.script.SearchScript;
@@ -370,7 +369,7 @@ public class AvgIT extends AbstractNumericTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractFieldScriptEngine.class, ExtractFieldScriptEngine.NAME, ScriptMode.ON));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractFieldScriptEngine.class, ExtractFieldScriptEngine.NAME, true));
         }
 
     }
@@ -490,7 +489,7 @@ public class AvgIT extends AbstractNumericTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.NAME, ScriptMode.ON));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.NAME, true));
         }
 
     }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
index c636508bbe1..641696722c9 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
@@ -28,7 +28,6 @@ import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
-import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.script.SearchScript;
@@ -365,7 +364,7 @@ public class SumIT extends AbstractNumericTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractFieldScriptEngine.class, ExtractFieldScriptEngine.NAME, ScriptMode.ON));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractFieldScriptEngine.class, ExtractFieldScriptEngine.NAME, true));
         }
 
     }
@@ -487,7 +486,7 @@ public class SumIT extends AbstractNumericTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.NAME, ScriptMode.ON));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.NAME, true));
         }
 
     }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
index 86f531ce3e9..04bef292051 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
@@ -28,7 +28,6 @@ import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
-import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.script.SearchScript;
@@ -224,7 +223,7 @@ public class ValueCountIT extends ESIntegTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.NAME, ScriptMode.ON));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.NAME, true));
         }
 
     }
diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index 077d978a4ce..661643b0e8b 100644
--- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -62,7 +62,6 @@ import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptContextRegistry;
 import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
-import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptSettings;
@@ -155,7 +154,7 @@ public class SearchSourceBuilderTests extends ESTestCase {
                 ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections
                         .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
                                                                                          MockScriptEngine.NAME,
-                                                                                         ScriptMode.ON)));
+                                                                                         true)));
                 bind(ScriptEngineRegistry.class).toInstance(scriptEngineRegistry);
                 ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts);
                 bind(ScriptContextRegistry.class).toInstance(scriptContextRegistry);
diff --git a/core/src/test/java/org/elasticsearch/update/UpdateIT.java b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
index 65553a4a90e..0445da61096 100644
--- a/core/src/test/java/org/elasticsearch/update/UpdateIT.java
+++ b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
@@ -43,7 +43,6 @@ import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
-import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.SearchScript;
@@ -94,7 +93,7 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(PutFieldValuesScriptEngine.class, PutFieldValuesScriptEngine.NAME, ScriptMode.ON));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(PutFieldValuesScriptEngine.class, PutFieldValuesScriptEngine.NAME, true));
         }
 
     }
@@ -181,7 +180,7 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldIncrementScriptEngine.class, FieldIncrementScriptEngine.NAME, ScriptMode.ON));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldIncrementScriptEngine.class, FieldIncrementScriptEngine.NAME, true));
         }
 
     }
@@ -261,7 +260,7 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ScriptedUpsertScriptEngine.class, ScriptedUpsertScriptEngine.NAME, ScriptMode.ON));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ScriptedUpsertScriptEngine.class, ScriptedUpsertScriptEngine.NAME, true));
         }
 
     }
@@ -341,7 +340,7 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractContextInSourceScriptEngine.class, ExtractContextInSourceScriptEngine.NAME, ScriptMode.ON));
+            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractContextInSourceScriptEngine.class, ExtractContextInSourceScriptEngine.NAME, true));
         }
 
     }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java
index 1cb21887944..cd95f13f63a 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java
@@ -21,7 +21,6 @@ package org.elasticsearch.script.expression;
 
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptEngineRegistry;
-import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 
 public class ExpressionPlugin extends Plugin {
@@ -38,6 +37,6 @@ public class ExpressionPlugin extends Plugin {
 
     public void onModule(ScriptModule module) {
         module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExpressionScriptEngineService.class,
-                        ExpressionScriptEngineService.NAME, ScriptMode.ON));
+                        ExpressionScriptEngineService.NAME, true));
     }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
index a468517a512..5ef3ad766e6 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
@@ -21,7 +21,6 @@ package org.elasticsearch.script.mustache;
 
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptEngineRegistry;
-import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 
 public class MustachePlugin extends Plugin {
@@ -38,6 +37,6 @@ public class MustachePlugin extends Plugin {
 
     public void onModule(ScriptModule module) {
         module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(MustacheScriptEngineService.class,
-                        MustacheScriptEngineService.NAME, ScriptMode.ON));
+                        MustacheScriptEngineService.NAME, true));
     }
 }
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
index 1bdb87778f5..7097718c6fc 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
@@ -59,7 +59,6 @@ import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.mapper.MapperRegistry;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.script.ScriptEngineRegistry;
-import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.mustache.MustacheScriptEngineService;
@@ -107,7 +106,7 @@ public class TemplateQueryParserTests extends ESTestCase {
         ScriptModule scriptModule = new ScriptModule();
         scriptModule.prepareSettings(settingsModule);
         // TODO: make this use a mock engine instead of mustache and it will no longer be messy!
-        scriptModule.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(MustacheScriptEngineService.class, MustacheScriptEngineService.NAME, ScriptMode.ON));
+        scriptModule.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(MustacheScriptEngineService.class, MustacheScriptEngineService.NAME, true));
         settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
         injector = new ModulesBuilder().add(
                 new EnvironmentModule(new Environment(settings)),
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
index 8f116c3c6f7..67f889b7a72 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
@@ -22,7 +22,6 @@ package org.elasticsearch.painless;
 
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptEngineRegistry;
-import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptModule;
 
 /**
@@ -47,6 +46,6 @@ public final class PainlessPlugin extends Plugin {
 
     public void onModule(final ScriptModule module) {
         module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(
-                        PainlessScriptEngineService.class, PainlessScriptEngineService.NAME, ScriptMode.ON));
+                        PainlessScriptEngineService.class, PainlessScriptEngineService.NAME, true));
     }
 }
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractMustacheTestCase.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractMustacheTestCase.java
index f58351b2e7d..8a24ac2408f 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractMustacheTestCase.java
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractMustacheTestCase.java
@@ -24,7 +24,6 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.ingest.core.TemplateService;
 import org.elasticsearch.script.ScriptContextRegistry;
 import org.elasticsearch.script.ScriptEngineRegistry;
-import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptSettings;
 import org.elasticsearch.script.mustache.MustacheScriptEngineService;
@@ -48,7 +47,7 @@ public abstract class AbstractMustacheTestCase extends ESTestCase {
                 new ScriptEngineRegistry(Collections.singletonList(
                                 new ScriptEngineRegistry.ScriptEngineRegistration(MustacheScriptEngineService.class,
                                                                                   MustacheScriptEngineService.NAME,
-                                                                                  ScriptMode.ON)));
+                                                                                  true)));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
         ScriptService scriptService = new ScriptService(settings, new Environment(settings), Collections.singleton(mustache), null,
diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
index 309b6622156..4e06bbe8b45 100644
--- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
+++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
@@ -23,7 +23,6 @@ import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.script.ScriptMode;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
@@ -70,7 +69,7 @@ public class MockScriptEngine implements ScriptEngineService {
 
         public void onModule(ScriptModule module) {
             module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
-                            MockScriptEngine.NAME, ScriptMode.ON));
+                            MockScriptEngine.NAME, true));
         }
 
     }

From 115f983827c0c29652bd444c072b658b76651317 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 20 May 2016 17:11:37 -0400
Subject: [PATCH 0402/1311] Fix env. var placeholder test so it's reproducible

This commit modifies the settings test for environment variables
placeholders so that it is reproducible. The underlying issue is that
the set of environment variables from system to system can vary, and
this means that's we can never be sure that a failing test will be
reproducible. This commit simplifies this test to not rely on external
forces that could influence reproducibility.

Relates #18501
---
 .../elasticsearch/common/settings/Settings.java  | 10 ++++++++--
 .../common/settings/SettingsTests.java           | 16 ++++++++++------
 2 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/settings/Settings.java b/core/src/main/java/org/elasticsearch/common/settings/Settings.java
index 7a335aa1f32..2ceafc376c5 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/Settings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/Settings.java
@@ -602,7 +602,8 @@ public final class Settings implements ToXContent {
 
         private final Map<String, String> map = new LinkedHashMap<>();
 
-        private Builder() {
+        // visible for testing
+        Builder() {
 
         }
 
@@ -964,7 +965,7 @@ public final class Settings implements ToXContent {
             PropertyPlaceholder.PlaceholderResolver placeholderResolver = new PropertyPlaceholder.PlaceholderResolver() {
                     @Override
                     public String resolvePlaceholder(String placeholderName) {
-                        final String value = System.getenv(placeholderName);
+                        final String value = getenv(placeholderName);
                         if (value != null) {
                             return value;
                         }
@@ -1000,6 +1001,11 @@ public final class Settings implements ToXContent {
             return this;
         }
 
+        // visible for testing
+        String getenv(String placeholderName) {
+            return System.getenv(placeholderName);
+        }
+
         /**
          * Checks that all settings in the builder start with the specified prefix.
          *
diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
index fe7dcda3b25..5b3a698d328 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
@@ -62,14 +62,18 @@ public class SettingsTests extends ESTestCase {
     }
 
     public void testReplacePropertiesPlaceholderByEnvironmentVariables() {
-        final Map.Entry<String, String> entry = randomSubsetOf(1, System.getenv().entrySet()).get(0);
-        assertNotNull(entry.getValue());
-
-        final Settings implicitEnvSettings = Settings.builder()
-            .put("setting1", "${" + entry.getKey() + "}")
+        final String hostname = randomAsciiOfLength(16);
+        final Settings.Builder builder = new Settings.Builder() {
+            @Override
+            protected String getenv(String placeholderName) {
+                return "HOSTNAME".equals(placeholderName) ? hostname : null;
+            }
+        };
+        final Settings implicitEnvSettings = builder
+            .put("setting1", "${HOSTNAME}")
             .replacePropertyPlaceholders()
             .build();
-        assertThat(implicitEnvSettings.get("setting1"), equalTo(entry.getValue()));
+        assertThat(implicitEnvSettings.get("setting1"), equalTo(hostname));
     }
 
     public void testReplacePropertiesPlaceholderIgnoresPrompt() {

From 688254de41bd78ab22508754a9aa95764ae42d2b Mon Sep 17 00:00:00 2001
From: Lee Hinman <lee@writequit.org>
Date: Fri, 20 May 2016 15:45:32 -0600
Subject: [PATCH 0403/1311] Don't mkdir directly in deb init script

Elasticsearch will create the log and data path directories when it
starts, there is no need to do this in the init script

Resolves #18307
---
 distribution/deb/src/main/packaging/init.d/elasticsearch | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/distribution/deb/src/main/packaging/init.d/elasticsearch b/distribution/deb/src/main/packaging/init.d/elasticsearch
index fdeb6db308e..f04008ba796 100755
--- a/distribution/deb/src/main/packaging/init.d/elasticsearch
+++ b/distribution/deb/src/main/packaging/init.d/elasticsearch
@@ -116,15 +116,6 @@ case "$1" in
 		exit 0
 	fi
 
-	# Prepare environment
-	# Check $DATA_DIR for a comma
-	if [ "${DATA_DIR#*,}" != "$DATA_DIR" ]; then
-		# $DATA_DIR contains a comma, so we should not mkdir it
-		mkdir -p "$LOG_DIR" && chown "$ES_USER":"$ES_GROUP" "$LOG_DIR"
-	else
-		mkdir -p "$LOG_DIR" "$DATA_DIR" && chown "$ES_USER":"$ES_GROUP" "$LOG_DIR" "$DATA_DIR"
-	fi
-
 	# Ensure that the PID_DIR exists (it is cleaned at OS startup time)
 	if [ -n "$PID_DIR" ] && [ ! -e "$PID_DIR" ]; then
 		mkdir -p "$PID_DIR" && chown "$ES_USER":"$ES_GROUP" "$PID_DIR"

From 1d40c4bbc1f8c3414cfc64a30e08ab097aed19d4 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Thu, 19 May 2016 23:56:06 -0700
Subject: [PATCH 0404/1311] Make java9 work again

This change makes ES compile with java9 again, build 118.
* There are a handful of changes due to failure to determine types during compile.
* The attachment plugins which use tika needed to have tika upgraded in order to pickup fixes there for java 9.
* azure discovery and s3 repository indirectly depend on jaxb, which is no longer in the default modules. They now add a jaxb dependency externally, and make JarHell allow for this package.
---
 .../elasticsearch/gradle/BuildPlugin.groovy   |    2 +-
 .../threadpool/ScalingThreadPoolTests.java    |    3 +-
 .../expression/ExpressionSearchScript.java    |    2 +-
 .../groovy/GroovyScriptEngineService.java     |    8 +-
 .../plugin-metadata/plugin-security.policy    |    3 +
 .../painless/DefOperationTests.java           |    4 +-
 plugins/discovery-azure/build.gradle          |  109 +-
 .../licenses/jaxb-api-2.2.2.jar.sha1          |    1 +
 plugins/ingest-attachment/build.gradle        |  627 +--
 .../licenses/bcmail-jdk15on-1.52.jar.sha1     |    1 -
 .../licenses/bcmail-jdk15on-1.54.jar.sha1     |    1 +
 .../licenses/bcpkix-jdk15on-1.52.jar.sha1     |    1 -
 .../licenses/bcpkix-jdk15on-1.54.jar.sha1     |    1 +
 .../licenses/bcprov-jdk15on-1.52.jar.sha1     |    1 -
 .../licenses/bcprov-jdk15on-1.54.jar.sha1     |    1 +
 .../licenses/fontbox-1.8.10.jar.sha1          |    1 -
 .../licenses/fontbox-2.0.1.jar.sha1           |    1 +
 .../licenses/jempbox-1.8.10.jar.sha1          |    1 -
 .../licenses/jempbox-1.8.12.jar.sha1          |    1 +
 .../licenses/pdfbox-1.8.10.jar.sha1           |    1 -
 .../licenses/pdfbox-2.0.1.jar.sha1            |    1 +
 .../licenses/poi-3.13.jar.sha1                |    1 -
 .../licenses/poi-3.15-beta1.jar.sha1          |    1 +
 .../licenses/poi-ooxml-3.13.jar.sha1          |    1 -
 .../licenses/poi-ooxml-3.15-beta1.jar.sha1    |    1 +
 .../licenses/poi-ooxml-schemas-3.13.jar.sha1  |    1 -
 .../poi-ooxml-schemas-3.15-beta1.jar.sha1     |    1 +
 .../licenses/poi-scratchpad-3.13.jar.sha1     |    1 -
 .../poi-scratchpad-3.15-beta1.jar.sha1        |    1 +
 .../licenses/tika-core-1.11.jar.sha1          |    1 -
 .../licenses/tika-core-1.13.jar.sha1          |    1 +
 .../licenses/tika-parsers-1.11.jar.sha1       |    1 -
 .../licenses/tika-parsers-1.13.jar.sha1       |    1 +
 plugins/mapper-attachments/build.gradle       | 3901 +++++++++--------
 .../licenses/bcmail-jdk15on-1.52.jar.sha1     |    1 -
 .../licenses/bcmail-jdk15on-1.54.jar.sha1     |    1 +
 .../licenses/bcpkix-jdk15on-1.52.jar.sha1     |    1 -
 .../licenses/bcpkix-jdk15on-1.54.jar.sha1     |    1 +
 .../licenses/bcprov-jdk15on-1.52.jar.sha1     |    1 -
 .../licenses/bcprov-jdk15on-1.54.jar.sha1     |    1 +
 .../licenses/fontbox-1.8.10.jar.sha1          |    1 -
 .../licenses/fontbox-2.0.1.jar.sha1           |    1 +
 .../licenses/jempbox-1.8.10.jar.sha1          |    1 -
 .../licenses/jempbox-1.8.12.jar.sha1          |    1 +
 .../licenses/pdfbox-1.8.10.jar.sha1           |    1 -
 .../licenses/pdfbox-2.0.1.jar.sha1            |    1 +
 .../licenses/poi-3.13.jar.sha1                |    1 -
 .../licenses/poi-3.15-beta1.jar.sha1          |    1 +
 .../licenses/poi-ooxml-3.13.jar.sha1          |    1 -
 .../licenses/poi-ooxml-3.15-beta1.jar.sha1    |    1 +
 .../licenses/poi-ooxml-schemas-3.13.jar.sha1  |    1 -
 .../poi-ooxml-schemas-3.15-beta1.jar.sha1     |    1 +
 .../licenses/poi-scratchpad-3.13.jar.sha1     |    1 -
 .../poi-scratchpad-3.15-beta1.jar.sha1        |    1 +
 .../licenses/tika-core-1.11.jar.sha1          |    1 -
 .../licenses/tika-core-1.13.jar.sha1          |    1 +
 .../licenses/tika-parsers-1.11.jar.sha1       |    1 -
 .../licenses/tika-parsers-1.13.jar.sha1       |    1 +
 plugins/repository-s3/build.gradle            |  110 +-
 .../repository-s3/licenses/jaxb-LICENSE.txt   |  705 +++
 .../repository-s3/licenses/jaxb-NOTICE.txt    |    1 +
 .../licenses/jaxb-api-2.2.2.jar.sha1          |    1 +
 .../elasticsearch/test/ESIntegTestCase.java   |    2 +-
 .../test/test/SuiteScopeClusterIT.java        |    2 +-
 .../test/test/TestScopeClusterIT.java         |    2 +-
 65 files changed, 3286 insertions(+), 2245 deletions(-)
 create mode 100644 plugins/discovery-azure/licenses/jaxb-api-2.2.2.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/bcmail-jdk15on-1.52.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/bcmail-jdk15on-1.54.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/bcpkix-jdk15on-1.52.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/bcpkix-jdk15on-1.54.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/bcprov-jdk15on-1.52.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/bcprov-jdk15on-1.54.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/fontbox-1.8.10.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/fontbox-2.0.1.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/jempbox-1.8.10.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/jempbox-1.8.12.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/pdfbox-1.8.10.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/pdfbox-2.0.1.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/poi-3.13.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/poi-3.15-beta1.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/poi-ooxml-3.13.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/poi-ooxml-3.15-beta1.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/poi-ooxml-schemas-3.13.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/poi-ooxml-schemas-3.15-beta1.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/poi-scratchpad-3.13.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/poi-scratchpad-3.15-beta1.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/tika-core-1.11.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/tika-core-1.13.jar.sha1
 delete mode 100644 plugins/ingest-attachment/licenses/tika-parsers-1.11.jar.sha1
 create mode 100644 plugins/ingest-attachment/licenses/tika-parsers-1.13.jar.sha1
 delete mode 100644 plugins/mapper-attachments/licenses/bcmail-jdk15on-1.52.jar.sha1
 create mode 100644 plugins/mapper-attachments/licenses/bcmail-jdk15on-1.54.jar.sha1
 delete mode 100644 plugins/mapper-attachments/licenses/bcpkix-jdk15on-1.52.jar.sha1
 create mode 100644 plugins/mapper-attachments/licenses/bcpkix-jdk15on-1.54.jar.sha1
 delete mode 100644 plugins/mapper-attachments/licenses/bcprov-jdk15on-1.52.jar.sha1
 create mode 100644 plugins/mapper-attachments/licenses/bcprov-jdk15on-1.54.jar.sha1
 delete mode 100644 plugins/mapper-attachments/licenses/fontbox-1.8.10.jar.sha1
 create mode 100644 plugins/mapper-attachments/licenses/fontbox-2.0.1.jar.sha1
 delete mode 100644 plugins/mapper-attachments/licenses/jempbox-1.8.10.jar.sha1
 create mode 100644 plugins/mapper-attachments/licenses/jempbox-1.8.12.jar.sha1
 delete mode 100644 plugins/mapper-attachments/licenses/pdfbox-1.8.10.jar.sha1
 create mode 100644 plugins/mapper-attachments/licenses/pdfbox-2.0.1.jar.sha1
 delete mode 100644 plugins/mapper-attachments/licenses/poi-3.13.jar.sha1
 create mode 100644 plugins/mapper-attachments/licenses/poi-3.15-beta1.jar.sha1
 delete mode 100644 plugins/mapper-attachments/licenses/poi-ooxml-3.13.jar.sha1
 create mode 100644 plugins/mapper-attachments/licenses/poi-ooxml-3.15-beta1.jar.sha1
 delete mode 100644 plugins/mapper-attachments/licenses/poi-ooxml-schemas-3.13.jar.sha1
 create mode 100644 plugins/mapper-attachments/licenses/poi-ooxml-schemas-3.15-beta1.jar.sha1
 delete mode 100644 plugins/mapper-attachments/licenses/poi-scratchpad-3.13.jar.sha1
 create mode 100644 plugins/mapper-attachments/licenses/poi-scratchpad-3.15-beta1.jar.sha1
 delete mode 100644 plugins/mapper-attachments/licenses/tika-core-1.11.jar.sha1
 create mode 100644 plugins/mapper-attachments/licenses/tika-core-1.13.jar.sha1
 delete mode 100644 plugins/mapper-attachments/licenses/tika-parsers-1.11.jar.sha1
 create mode 100644 plugins/mapper-attachments/licenses/tika-parsers-1.13.jar.sha1
 create mode 100644 plugins/repository-s3/licenses/jaxb-LICENSE.txt
 create mode 100644 plugins/repository-s3/licenses/jaxb-NOTICE.txt
 create mode 100644 plugins/repository-s3/licenses/jaxb-api-2.2.2.jar.sha1

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index 029c80b6e25..8e1afd7d1a2 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -378,7 +378,7 @@ class BuildPlugin implements Plugin<Project> {
                  * -serial because we don't use java serialization.
                  */
                 // don't even think about passing args with -J-xxx, oracle will ask you to submit a bug report :)
-                options.compilerArgs << '-Werror' << '-Xlint:all,-path,-serial' << '-Xdoclint:all' << '-Xdoclint:-missing'
+                options.compilerArgs << '-Werror' << '-Xlint:all,-path,-serial,-options' << '-Xdoclint:all' << '-Xdoclint:-missing'
                 // compile with compact 3 profile by default
                 // NOTE: this is just a compile time check: does not replace testing with a compact3 JRE
                 if (project.compactProfile != 'full') {
diff --git a/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java b/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
index 94d6d075589..e331678c1e9 100644
--- a/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
+++ b/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
@@ -177,7 +177,8 @@ public class ScalingThreadPoolTests extends ESThreadPoolTestCase {
                     }
                 });
             }
-            assertThat(stats(threadPool, threadPoolName).getThreads(), equalTo(128));
+            int threads = stats(threadPool, threadPoolName).getThreads();
+            assertEquals(128, threads);
             latch.countDown();
             // this while loop is the core of this test; if threads
             // are correctly idled down by the pool, the number of
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java
index 54553565bcb..edc6a7c3924 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java
@@ -78,7 +78,7 @@ class ExpressionSearchScript implements SearchScript {
             }
 
             @Override
-            public Object run() { return new Double(evaluate()); }
+            public Object run() { return Double.valueOf(evaluate()); }
 
             @Override
             public long runAsLong() { return (long)evaluate(); }
diff --git a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
index 700a1aecbc2..5d06196a83b 100644
--- a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
+++ b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
@@ -201,9 +201,9 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
      * Return a script object with the given vars from the compiled script object
      */
     @SuppressWarnings("unchecked")
-    private Script createScript(Object compiledScript, Map<String, Object> vars) throws InstantiationException, IllegalAccessException {
+    private Script createScript(Object compiledScript, Map<String, Object> vars) throws ReflectiveOperationException {
         Class<?> scriptClass = (Class<?>) compiledScript;
-        Script scriptObject = (Script) scriptClass.newInstance();
+        Script scriptObject = (Script) scriptClass.getConstructor().newInstance();
         Binding binding = new Binding();
         binding.getVariables().putAll(vars);
         scriptObject.setBinding(binding);
@@ -218,7 +218,7 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
                 allVars.putAll(vars);
             }
             return new GroovyScript(compiledScript, createScript(compiledScript.compiled(), allVars), this.logger);
-        } catch (Exception e) {
+        } catch (ReflectiveOperationException e) {
             throw new ScriptException("failed to build executable " + compiledScript, e);
         }
     }
@@ -238,7 +238,7 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
                 Script scriptObject;
                 try {
                     scriptObject = createScript(compiledScript.compiled(), allVars);
-                } catch (InstantiationException | IllegalAccessException e) {
+                } catch (ReflectiveOperationException e) {
                     throw new ScriptException("failed to build search " + compiledScript, e);
                 }
                 return new GroovyScript(compiledScript, scriptObject, leafLookup, logger);
diff --git a/modules/lang-groovy/src/main/plugin-metadata/plugin-security.policy b/modules/lang-groovy/src/main/plugin-metadata/plugin-security.policy
index a2318191540..82482448677 100644
--- a/modules/lang-groovy/src/main/plugin-metadata/plugin-security.policy
+++ b/modules/lang-groovy/src/main/plugin-metadata/plugin-security.policy
@@ -25,6 +25,7 @@ grant {
   // needed by groovy engine
   permission java.lang.RuntimePermission "accessDeclaredMembers";
   permission java.lang.RuntimePermission "accessClassInPackage.sun.reflect";
+  permission java.lang.RuntimePermission "accessClassInPackage.jdk.internal.reflect";
   // needed by GroovyScriptEngineService to close its classloader (why?)
   permission java.lang.RuntimePermission "closeClassLoader";
   // Allow executing groovy scripts with codesource of /untrusted
@@ -49,6 +50,8 @@ grant {
   permission org.elasticsearch.script.ClassPermission "org.codehaus.groovy.vmplugin.v7.IndyInterface";
   permission org.elasticsearch.script.ClassPermission "sun.reflect.ConstructorAccessorImpl";
   permission org.elasticsearch.script.ClassPermission "sun.reflect.MethodAccessorImpl";
+  permission org.elasticsearch.script.ClassPermission "jdk.internal.reflect.ConstructorAccessorImpl";
+  permission org.elasticsearch.script.ClassPermission "jdk.internal.reflect.MethodAccessorImpl";
 
   permission org.elasticsearch.script.ClassPermission "groovy.lang.Closure";
   permission org.elasticsearch.script.ClassPermission "org.codehaus.groovy.runtime.GeneratedClosure";
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
index bfc0d142045..58b37469f1b 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
@@ -22,10 +22,10 @@ package org.elasticsearch.painless;
 public class DefOperationTests extends ScriptTestCase {
     public void testIllegalCast() {
         Exception exception = expectThrows(ClassCastException.class, () -> exec("def x = 1.0; int y = x; return y;"));
-        assertTrue(exception.getMessage().contains("java.lang.Double cannot be cast to java.lang.Integer"));
+        assertTrue(exception.getMessage(), exception.getMessage().contains("cannot be cast to java.lang.Integer"));
 
         exception = expectThrows(ClassCastException.class, () -> exec("def x = (short)1; byte y = x; return y;"));
-        assertTrue(exception.getMessage().contains("java.lang.Short cannot be cast to java.lang.Byte"));
+        assertTrue(exception.getMessage(), exception.getMessage().contains("cannot be cast to java.lang.Byte"));
     }
 
     public void testNot() {
diff --git a/plugins/discovery-azure/build.gradle b/plugins/discovery-azure/build.gradle
index ceddbc2ff44..8f0c641e150 100644
--- a/plugins/discovery-azure/build.gradle
+++ b/plugins/discovery-azure/build.gradle
@@ -20,7 +20,7 @@ import org.elasticsearch.gradle.LoggedExec
  */
 
 esplugin {
-  description 'The Azure Discovery plugin allows to use Azure API for the unicast discovery mechanism.'
+  description 'The Azure Discovery plugin allows to use Azure API for the unicast discovery mechanism'
   classname 'org.elasticsearch.plugin.discovery.azure.AzureDiscoveryPlugin'
 }
 
@@ -49,6 +49,10 @@ dependencies {
   compile 'org.codehaus.jackson:jackson-mapper-asl:1.9.2'
   compile 'org.codehaus.jackson:jackson-jaxrs:1.9.2'
   compile 'org.codehaus.jackson:jackson-xc:1.9.2'  
+
+  // HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
+  // and whitelist this hack in JarHell 
+  compile 'javax.xml.bind:jaxb-api:2.2.2'
 }
 
 // needed to be consistent with ssl host checking
@@ -121,4 +125,107 @@ thirdPartyAudit.excludes = [
   'org.osgi.framework.SynchronousBundleListener',
   'com.sun.xml.fastinfoset.stax.StAXDocumentParser', 
   'com.sun.xml.fastinfoset.stax.StAXDocumentSerializer',
+
+  // jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9)
+  'javax.xml.bind.Binder',
+  'javax.xml.bind.ContextFinder$1',
+  'javax.xml.bind.ContextFinder',
+  'javax.xml.bind.DataBindingException',
+  'javax.xml.bind.DatatypeConverter',
+  'javax.xml.bind.DatatypeConverterImpl$CalendarFormatter',
+  'javax.xml.bind.DatatypeConverterImpl',
+  'javax.xml.bind.DatatypeConverterInterface',
+  'javax.xml.bind.Element',
+  'javax.xml.bind.GetPropertyAction',
+  'javax.xml.bind.JAXB$Cache',
+  'javax.xml.bind.JAXB',
+  'javax.xml.bind.JAXBContext',
+  'javax.xml.bind.JAXBElement$GlobalScope',
+  'javax.xml.bind.JAXBElement',
+  'javax.xml.bind.JAXBException',
+  'javax.xml.bind.JAXBIntrospector',
+  'javax.xml.bind.JAXBPermission',
+  'javax.xml.bind.MarshalException',
+  'javax.xml.bind.Marshaller$Listener',
+  'javax.xml.bind.Marshaller',
+  'javax.xml.bind.Messages',
+  'javax.xml.bind.NotIdentifiableEvent',
+  'javax.xml.bind.ParseConversionEvent',
+  'javax.xml.bind.PrintConversionEvent',
+  'javax.xml.bind.PropertyException',
+  'javax.xml.bind.SchemaOutputResolver',
+  'javax.xml.bind.TypeConstraintException',
+  'javax.xml.bind.UnmarshalException',
+  'javax.xml.bind.Unmarshaller$Listener',
+  'javax.xml.bind.Unmarshaller',
+  'javax.xml.bind.UnmarshallerHandler',
+  'javax.xml.bind.ValidationEvent',
+  'javax.xml.bind.ValidationEventHandler',
+  'javax.xml.bind.ValidationEventLocator',
+  'javax.xml.bind.ValidationException',
+  'javax.xml.bind.Validator',
+  'javax.xml.bind.WhiteSpaceProcessor',
+  'javax.xml.bind.annotation.DomHandler',
+  'javax.xml.bind.annotation.W3CDomHandler',
+  'javax.xml.bind.annotation.XmlAccessOrder',
+  'javax.xml.bind.annotation.XmlAccessType',
+  'javax.xml.bind.annotation.XmlAccessorOrder',
+  'javax.xml.bind.annotation.XmlAccessorType',
+  'javax.xml.bind.annotation.XmlAnyAttribute',
+  'javax.xml.bind.annotation.XmlAnyElement',
+  'javax.xml.bind.annotation.XmlAttachmentRef',
+  'javax.xml.bind.annotation.XmlAttribute',
+  'javax.xml.bind.annotation.XmlElement$DEFAULT',
+  'javax.xml.bind.annotation.XmlElement',
+  'javax.xml.bind.annotation.XmlElementDecl$GLOBAL',
+  'javax.xml.bind.annotation.XmlElementDecl',
+  'javax.xml.bind.annotation.XmlElementRef$DEFAULT',
+  'javax.xml.bind.annotation.XmlElementRef',
+  'javax.xml.bind.annotation.XmlElementRefs',
+  'javax.xml.bind.annotation.XmlElementWrapper',
+  'javax.xml.bind.annotation.XmlElements',
+  'javax.xml.bind.annotation.XmlEnum',
+  'javax.xml.bind.annotation.XmlEnumValue',
+  'javax.xml.bind.annotation.XmlID',
+  'javax.xml.bind.annotation.XmlIDREF',
+  'javax.xml.bind.annotation.XmlInlineBinaryData',
+  'javax.xml.bind.annotation.XmlList',
+  'javax.xml.bind.annotation.XmlMimeType',
+  'javax.xml.bind.annotation.XmlMixed',
+  'javax.xml.bind.annotation.XmlNs',
+  'javax.xml.bind.annotation.XmlNsForm',
+  'javax.xml.bind.annotation.XmlRegistry',
+  'javax.xml.bind.annotation.XmlRootElement',
+  'javax.xml.bind.annotation.XmlSchema',
+  'javax.xml.bind.annotation.XmlSchemaType$DEFAULT',
+  'javax.xml.bind.annotation.XmlSchemaType',
+  'javax.xml.bind.annotation.XmlSchemaTypes',
+  'javax.xml.bind.annotation.XmlSeeAlso',
+  'javax.xml.bind.annotation.XmlTransient',
+  'javax.xml.bind.annotation.XmlType$DEFAULT',
+  'javax.xml.bind.annotation.XmlType',
+  'javax.xml.bind.annotation.XmlValue',
+  'javax.xml.bind.annotation.adapters.CollapsedStringAdapter',
+  'javax.xml.bind.annotation.adapters.HexBinaryAdapter',
+  'javax.xml.bind.annotation.adapters.NormalizedStringAdapter',
+  'javax.xml.bind.annotation.adapters.XmlAdapter',
+  'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter$DEFAULT',
+  'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter',
+  'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapters',
+  'javax.xml.bind.attachment.AttachmentMarshaller',
+  'javax.xml.bind.attachment.AttachmentUnmarshaller',
+  'javax.xml.bind.helpers.AbstractMarshallerImpl',
+  'javax.xml.bind.helpers.AbstractUnmarshallerImpl',
+  'javax.xml.bind.helpers.DefaultValidationEventHandler',
+  'javax.xml.bind.helpers.Messages',
+  'javax.xml.bind.helpers.NotIdentifiableEventImpl',
+  'javax.xml.bind.helpers.ParseConversionEventImpl',
+  'javax.xml.bind.helpers.PrintConversionEventImpl',
+  'javax.xml.bind.helpers.ValidationEventImpl',
+  'javax.xml.bind.helpers.ValidationEventLocatorImpl',
+  'javax.xml.bind.util.JAXBResult',
+  'javax.xml.bind.util.JAXBSource$1',
+  'javax.xml.bind.util.JAXBSource',
+  'javax.xml.bind.util.Messages',
+  'javax.xml.bind.util.ValidationEventCollector'
 ]
diff --git a/plugins/discovery-azure/licenses/jaxb-api-2.2.2.jar.sha1 b/plugins/discovery-azure/licenses/jaxb-api-2.2.2.jar.sha1
new file mode 100644
index 00000000000..a37e1872389
--- /dev/null
+++ b/plugins/discovery-azure/licenses/jaxb-api-2.2.2.jar.sha1
@@ -0,0 +1 @@
+aeb3021ca93dde265796d82015beecdcff95bf09
\ No newline at end of file
diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle
index e7a48cd1763..3d9545d7aed 100644
--- a/plugins/ingest-attachment/build.gradle
+++ b/plugins/ingest-attachment/build.gradle
@@ -23,10 +23,10 @@ esplugin {
 }
 
 versions << [
-  'tika': '1.11',
-  'pdfbox': '1.8.10',
-  'bouncycastle': '1.52',
-  'poi': '3.13'
+  'tika': '1.13',
+  'pdfbox': '2.0.1',
+  'bouncycastle': '1.54',
+  'poi': '3.15-beta1'
 ]
 
 dependencies {
@@ -44,7 +44,7 @@ dependencies {
   // Adobe PDF
   compile "org.apache.pdfbox:pdfbox:${versions.pdfbox}"
   compile "org.apache.pdfbox:fontbox:${versions.pdfbox}"
-  compile "org.apache.pdfbox:jempbox:${versions.pdfbox}"
+  compile "org.apache.pdfbox:jempbox:1.8.12"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
   compile "org.bouncycastle:bcmail-jdk15on:${versions.bouncycastle}"
   compile "org.bouncycastle:bcprov-jdk15on:${versions.bouncycastle}"
@@ -61,6 +61,9 @@ dependencies {
   compile 'org.apache.commons:commons-compress:1.10'
 }
 
+// TODO: stop using LanguageIdentifier...
+compileJava.options.compilerArgs << "-Xlint:-deprecation"
+
 forbiddenPatterns {
   exclude '**/*.docx'
   exclude '**/*.pdf'
@@ -99,9 +102,10 @@ thirdPartyAudit.excludes = [
   'com.drew.metadata.iptc.IptcDirectory',
   'com.drew.metadata.jpeg.JpegCommentDirectory',
   'com.drew.metadata.jpeg.JpegDirectory',
-  'com.drew.metadata.xmp.XmpReader',
   'com.github.junrar.Archive',
   'com.github.junrar.rarfile.FileHeader',
+  'com.google.common.reflect.TypeToken',
+  'com.google.gson.Gson',
   'com.googlecode.mp4parser.DataSource',
   'com.googlecode.mp4parser.boxes.apple.AppleAlbumBox',
   'com.googlecode.mp4parser.boxes.apple.AppleArtist2Box',
@@ -117,6 +121,12 @@ thirdPartyAudit.excludes = [
   'com.googlecode.mp4parser.boxes.apple.AppleTrackNumberBox',
   'com.googlecode.mp4parser.boxes.apple.Utf8AppleDataBox',
   'com.googlecode.mp4parser.util.CastUtils',
+  'com.graphbuilder.curve.ControlPath',
+  'com.graphbuilder.curve.GroupIterator',
+  'com.graphbuilder.curve.NURBSpline',
+  'com.graphbuilder.curve.ShapeMultiPath',
+  'com.graphbuilder.curve.ValueVector',
+  'com.graphbuilder.geom.PointFactory',
   'com.healthmarketscience.jackcess.Column',
   'com.healthmarketscience.jackcess.CryptCodecProvider',
   'com.healthmarketscience.jackcess.DataType',
@@ -136,12 +146,216 @@ thirdPartyAudit.excludes = [
   'com.healthmarketscience.jackcess.util.OleBlob$SimplePackageContent',
   'com.healthmarketscience.jackcess.util.OleBlob',
   'com.healthmarketscience.jackcess.util.TableIterableBuilder',
-  'com.ibm.icu.text.Bidi',
-  'com.ibm.icu.text.Normalizer',
   'com.jmatio.io.MatFileHeader',
   'com.jmatio.io.MatFileReader',
   'com.jmatio.types.MLArray',
   'com.jmatio.types.MLStructure',
+  'com.microsoft.schemas.office.excel.STCF',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1Accel2List',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1AccelList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1AnchorList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1AutoFillList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1AutoLineList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1AutoPictList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1AutoScaleList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1CFList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1CameraList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1CancelList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1CheckedList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ColHiddenList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ColoredList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ColumnList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1DDEList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1DefaultList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1DefaultSizeList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1DisabledList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1DismissList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1DropLinesList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1DropStyleList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1DxList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1FirstButtonList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1FmlaGroupList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1FmlaLinkList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1FmlaMacroList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1FmlaPictList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1FmlaRangeList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1FmlaTxbxList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1HelpList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1HorizList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1IncList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1JustLastXList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1LCTList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ListItemList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1LockTextList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1LockedList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1MapOCXList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1MaxList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1MinList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1MoveWithCellsList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1MultiLineList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1MultiSelList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1NoThreeD2List',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1NoThreeDList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1PageList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1PrintObjectList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1RecalcAlwaysList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1RowHiddenList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1RowList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ScriptExtendedList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ScriptLanguageList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ScriptLocationList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ScriptTextList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1SecretEditList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1SelList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1SelTypeList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1SizeWithCellsList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1TextHAlignList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1TextVAlignList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1UIObjList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1VScrollList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1VTEditList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ValList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ValidIdsList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1VisibleList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1WidthMinList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2Accel2List',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2AccelList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2AnchorList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2AutoFillList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2AutoLineList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2AutoPictList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2AutoScaleList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2CFList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2CameraList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2CancelList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2CheckedList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2ColHiddenList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2ColoredList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2ColumnList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2DDEList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2DefaultList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2DefaultSizeList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2DisabledList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2DismissList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2DropLinesList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2DropStyleList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2DxList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2FirstButtonList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2FmlaGroupList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2FmlaLinkList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2FmlaMacroList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2FmlaPictList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2FmlaRangeList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2FmlaTxbxList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2HelpList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2HorizList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2IncList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2JustLastXList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2LCTList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2ListItemList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2LockTextList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2LockedList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2MapOCXList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2MaxList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2MinList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2MoveWithCellsList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2MultiLineList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2MultiSelList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2NoThreeD2List',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2NoThreeDList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2PageList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2PrintObjectList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2RecalcAlwaysList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2RowHiddenList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2RowList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2ScriptExtendedList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2ScriptLanguageList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2ScriptLocationList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2ScriptTextList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2SecretEditList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2SelList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2SelTypeList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2SizeWithCellsList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2TextHAlignList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2TextVAlignList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2UIObjList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2VScrollList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2VTEditList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2ValList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2ValidIdsList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2VisibleList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2WidthMinList',
+  'com.microsoft.schemas.office.office.CTCallout',
+  'com.microsoft.schemas.office.office.CTClipPath',
+  'com.microsoft.schemas.office.office.CTComplex',
+  'com.microsoft.schemas.office.office.CTDiagram',
+  'com.microsoft.schemas.office.office.CTExtrusion',
+  'com.microsoft.schemas.office.office.CTFill',
+  'com.microsoft.schemas.office.office.CTInk',
+  'com.microsoft.schemas.office.office.CTRegroupTable',
+  'com.microsoft.schemas.office.office.CTRules',
+  'com.microsoft.schemas.office.office.CTSignatureLine',
+  'com.microsoft.schemas.office.office.CTSkew',
+  'com.microsoft.schemas.office.office.CTStrokeChild',
+  'com.microsoft.schemas.office.office.STBWMode',
+  'com.microsoft.schemas.office.office.STConnectorType',
+  'com.microsoft.schemas.office.office.STHrAlign',
+  'com.microsoft.schemas.office.office.STRelationshipId',
+  'com.microsoft.schemas.office.office.STTrueFalse',
+  'com.microsoft.schemas.office.office.STTrueFalseBlank',
+  'com.microsoft.schemas.office.powerpoint.CTEmpty',
+  'com.microsoft.schemas.office.powerpoint.CTRel',
+  'com.microsoft.schemas.office.visio.x2012.main.AttachedToolbarsType',
+  'com.microsoft.schemas.office.visio.x2012.main.ColorsType',
+  'com.microsoft.schemas.office.visio.x2012.main.ConnectType',
+  'com.microsoft.schemas.office.visio.x2012.main.ConnectsType',
+  'com.microsoft.schemas.office.visio.x2012.main.CpType',
+  'com.microsoft.schemas.office.visio.x2012.main.CustomMenusFileType',
+  'com.microsoft.schemas.office.visio.x2012.main.CustomToolbarsFileType',
+  'com.microsoft.schemas.office.visio.x2012.main.DataType',
+  'com.microsoft.schemas.office.visio.x2012.main.DocumentSheetType',
+  'com.microsoft.schemas.office.visio.x2012.main.DynamicGridEnabledType',
+  'com.microsoft.schemas.office.visio.x2012.main.EventListType',
+  'com.microsoft.schemas.office.visio.x2012.main.FaceNamesType',
+  'com.microsoft.schemas.office.visio.x2012.main.FldType',
+  'com.microsoft.schemas.office.visio.x2012.main.ForeignDataType',
+  'com.microsoft.schemas.office.visio.x2012.main.GlueSettingsType',
+  'com.microsoft.schemas.office.visio.x2012.main.HeaderFooterType',
+  'com.microsoft.schemas.office.visio.x2012.main.IconType',
+  'com.microsoft.schemas.office.visio.x2012.main.MasterShortcutType',
+  'com.microsoft.schemas.office.visio.x2012.main.PpType',
+  'com.microsoft.schemas.office.visio.x2012.main.ProtectBkgndsType',
+  'com.microsoft.schemas.office.visio.x2012.main.ProtectMastersType',
+  'com.microsoft.schemas.office.visio.x2012.main.ProtectShapesType',
+  'com.microsoft.schemas.office.visio.x2012.main.ProtectStylesType',
+  'com.microsoft.schemas.office.visio.x2012.main.PublishSettingsType',
+  'com.microsoft.schemas.office.visio.x2012.main.RefByType',
+  'com.microsoft.schemas.office.visio.x2012.main.SnapAnglesType',
+  'com.microsoft.schemas.office.visio.x2012.main.SnapExtensionsType',
+  'com.microsoft.schemas.office.visio.x2012.main.SnapSettingsType',
+  'com.microsoft.schemas.office.visio.x2012.main.TpType',
+  'com.microsoft.schemas.office.visio.x2012.main.TriggerType',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.CellTypeImpl$1RefByList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.MastersTypeImpl$1MasterList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.MastersTypeImpl$1MasterShortcutList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.PagesTypeImpl$1PageList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.RowTypeImpl$1CellList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.RowTypeImpl$1TriggerList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.SectionTypeImpl$1CellList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.SectionTypeImpl$1RowList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.SectionTypeImpl$1TriggerList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.ShapesTypeImpl$1ShapeList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.SheetTypeImpl$1CellList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.SheetTypeImpl$1SectionList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.SheetTypeImpl$1TriggerList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.StyleSheetsTypeImpl$1StyleSheetList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.TextTypeImpl$1CpList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.TextTypeImpl$1FldList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.TextTypeImpl$1PpList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.TextTypeImpl$1TpList',
+  'com.microsoft.schemas.office.word.CTAnchorLock',
+  'com.microsoft.schemas.office.word.CTBorder',
+  'com.microsoft.schemas.office.word.CTWrap',
   'com.microsoft.schemas.office.x2006.digsig.STPositiveInteger',
   'com.microsoft.schemas.office.x2006.digsig.STSignatureComments',
   'com.microsoft.schemas.office.x2006.digsig.STSignatureProviderUrl',
@@ -149,14 +363,118 @@ thirdPartyAudit.excludes = [
   'com.microsoft.schemas.office.x2006.digsig.STSignatureType',
   'com.microsoft.schemas.office.x2006.digsig.STUniqueIdentifierWithBraces',
   'com.microsoft.schemas.office.x2006.digsig.STVersion',
+  'com.microsoft.schemas.vml.CTArc',
+  'com.microsoft.schemas.vml.CTCurve',
+  'com.microsoft.schemas.vml.CTImage',
+  'com.microsoft.schemas.vml.CTImageData',
+  'com.microsoft.schemas.vml.CTLine',
+  'com.microsoft.schemas.vml.CTOval',
+  'com.microsoft.schemas.vml.CTPolyLine',
+  'com.microsoft.schemas.vml.CTRect',
+  'com.microsoft.schemas.vml.CTRoundRect',
+  'com.microsoft.schemas.vml.STEditAs',
+  'com.microsoft.schemas.vml.STFillMethod',
+  'com.microsoft.schemas.vml.STFillType',
+  'com.microsoft.schemas.vml.STImageAspect',
+  'com.microsoft.schemas.vml.STShadowType',
+  'com.microsoft.schemas.vml.STStrokeArrowLength',
+  'com.microsoft.schemas.vml.STStrokeArrowType',
+  'com.microsoft.schemas.vml.STStrokeArrowWidth',
+  'com.microsoft.schemas.vml.STStrokeEndCap',
+  'com.microsoft.schemas.vml.STStrokeLineStyle',
+  'com.microsoft.schemas.vml.STTrueFalseBlank',
+  'com.microsoft.schemas.vml.impl.CTFormulasImpl$1FList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1AnchorlockList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1ArcList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1BorderbottomList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1BorderleftList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1BorderrightList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1BordertopList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1CalloutList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1ClientDataList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1ClippathList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1CurveList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1DiagramList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1ExtrusionList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1FillList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1FormulasList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1GroupList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1HandlesList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1ImageList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1ImagedataList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1LineList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1LockList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1OvalList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1PathList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1PolylineList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1RectList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1RoundrectList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1ShadowList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1ShapeList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1ShapetypeList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1SignaturelineList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1SkewList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1StrokeList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1TextboxList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1TextdataList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1TextpathList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1WrapList',
+  'com.microsoft.schemas.vml.impl.CTHandlesImpl$1HList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1AnchorlockList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1BorderbottomList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1BorderleftList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1BorderrightList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1BordertopList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1CalloutList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1ClippathList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1ExtrusionList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1FillList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1FormulasList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1HandlesList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1ImagedataList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1InkList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1IscommentList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1LockList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1PathList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1ShadowList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1SignaturelineList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1SkewList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1StrokeList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1TextboxList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1TextdataList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1TextpathList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1WrapList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1AnchorlockList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1BorderbottomList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1BorderleftList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1BorderrightList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1BordertopList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1CalloutList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1ClientDataList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1ClippathList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1ExtrusionList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1FillList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1FormulasList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1HandlesList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1ImagedataList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1LockList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1PathList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1ShadowList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1SignaturelineList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1SkewList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1StrokeList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1TextboxList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1TextdataList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1TextpathList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1WrapList',
   'com.pff.PSTAttachment',
   'com.pff.PSTFile',
   'com.pff.PSTFolder',
   'com.pff.PSTMessage',
-  'com.sun.syndication.feed.synd.SyndContent',
-  'com.sun.syndication.feed.synd.SyndEntry',
-  'com.sun.syndication.feed.synd.SyndFeed',
-  'com.sun.syndication.io.SyndFeedInput',
+  'com.rometools.rome.feed.synd.SyndContent',
+  'com.rometools.rome.feed.synd.SyndEntry',
+  'com.rometools.rome.feed.synd.SyndFeed',
+  'com.rometools.rome.io.SyndFeedInput',
   'com.uwyn.jhighlight.renderer.Renderer',
   'com.uwyn.jhighlight.renderer.XhtmlRendererFactory',
   'de.l3s.boilerpipe.BoilerpipeExtractor',
@@ -202,6 +520,13 @@ thirdPartyAudit.excludes = [
   'org.apache.cxf.jaxrs.ext.multipart.Attachment',
   'org.apache.cxf.jaxrs.ext.multipart.ContentDisposition',
   'org.apache.cxf.jaxrs.ext.multipart.MultipartBody',
+  'org.apache.http.HttpEntity',
+  'org.apache.http.HttpResponse',
+  'org.apache.http.StatusLine',
+  'org.apache.http.client.HttpClient',
+  'org.apache.http.client.methods.HttpGet',
+  'org.apache.http.client.utils.URIBuilder',
+  'org.apache.http.impl.client.DefaultHttpClient',
   'org.apache.james.mime4j.MimeException',
   'org.apache.james.mime4j.codec.DecodeMonitor',
   'org.apache.james.mime4j.codec.DecoderUtil',
@@ -227,6 +552,7 @@ thirdPartyAudit.excludes = [
   'org.apache.jcp.xml.dsig.internal.dom.DOMSignedInfo',
   'org.apache.log.Hierarchy',
   'org.apache.log.Logger',
+  'org.apache.pdfbox.tools.imageio.ImageIOUtil',
   'org.apache.sis.internal.util.CheckedArrayList',
   'org.apache.sis.internal.util.CheckedHashSet',
   'org.apache.sis.metadata.iso.DefaultMetadata',
@@ -263,7 +589,6 @@ thirdPartyAudit.excludes = [
   'org.apache.xml.security.Init',
   'org.apache.xml.security.c14n.Canonicalizer',
   'org.apache.xml.security.utils.Base64',
-  'org.bouncycastle.asn1.DERObject',
   'org.etsi.uri.x01903.v13.AnyType',
   'org.etsi.uri.x01903.v13.ClaimedRolesListType',
   'org.etsi.uri.x01903.v13.CounterSignatureType',
@@ -312,7 +637,7 @@ thirdPartyAudit.excludes = [
   'org.json.XML',
   'org.json.simple.JSONArray',
   'org.json.simple.JSONObject',
-  'org.json.simple.JSONValue',
+  'org.json.simple.parser.JSONParser',
   'org.junit.Test',
   'org.junit.internal.TextListener',
   'org.junit.runner.JUnitCore',
@@ -812,8 +1137,6 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.presentationml.x2006.main.impl.CTCommentAuthorListImpl$1CmAuthorList',
   'org.openxmlformats.schemas.presentationml.x2006.main.impl.CTCommentListImpl$1CmList',
   'org.openxmlformats.schemas.presentationml.x2006.main.impl.CTCustomerDataListImpl$1CustDataList',
-  'org.openxmlformats.schemas.presentationml.x2006.main.impl.CTGroupShapeImpl$1GraphicFrameList',
-  'org.openxmlformats.schemas.presentationml.x2006.main.impl.CTGroupShapeImpl$1PicList',
   'org.openxmlformats.schemas.schemaLibrary.x2006.main.CTSchemaLibrary',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTAutoSortScope',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTBoolean',
@@ -853,7 +1176,6 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTFormats',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTFunctionGroups',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTGradientFill',
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTIgnoredErrors',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTMeasureDimensionMaps',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTMeasureGroups',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTMissing',
@@ -878,7 +1200,6 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTSortState',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTString',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTTableFormula',
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTTableStyleInfo',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTTableStyles',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTTupleCache',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTWebPublishItems',
@@ -908,7 +1229,6 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTBordersImpl$1BorderList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTCacheFieldImpl$1MpMapList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTCacheFieldsImpl$1CacheFieldList',
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTCalcChainImpl$1CList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTCellStyleXfsImpl$1XfList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTCellXfsImpl$1XfList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTCfRuleImpl$1FormulaList',
@@ -916,7 +1236,6 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTColFieldsImpl$1FieldList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTColorScaleImpl$1CfvoList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTColorScaleImpl$1ColorList',
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTColsImpl$1ColList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTCommentListImpl$1CommentList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTConditionalFormattingImpl$1CfRuleList',
   'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTDataBarImpl$1CfvoList',
@@ -1023,7 +1342,6 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTFramePr',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTFtnDocProps',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTFtnProps',
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTHighlight',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTKinsoku',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTLevelSuffix',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTLineNumber',
@@ -1265,7 +1583,6 @@ thirdPartyAudit.excludes = [
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTNumberingImpl$1NumList',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTNumberingImpl$1NumPicBulletList',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1BookmarkEndList',
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1BookmarkStartList',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CommentRangeEndList',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CommentRangeStartList',
   'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CustomXmlDelRangeEndList',
@@ -1696,6 +2013,8 @@ thirdPartyAudit.excludes = [
   'org.osgi.framework.ServiceRegistration',
   'org.osgi.util.tracker.ServiceTracker',
   'org.osgi.util.tracker.ServiceTrackerCustomizer',
+  'org.slf4j.Logger',
+  'org.slf4j.LoggerFactory',
   'org.sqlite.SQLiteConfig',
   'org.tukaani.xz.ARMOptions',
   'org.tukaani.xz.ARMThumbOptions',
@@ -1722,273 +2041,11 @@ thirdPartyAudit.excludes = [
   'org.w3.x2000.x09.xmldsig.impl.SignedInfoTypeImpl$1ReferenceList',
   'org.w3.x2000.x09.xmldsig.impl.TransformTypeImpl$1XPathList',
   'org.w3.x2000.x09.xmldsig.impl.TransformTypeImpl$2XPathList',
-  'schemasMicrosoftComOfficeExcel.STCF',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1Accel2List',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1AccelList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1AnchorList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1AutoFillList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1AutoLineList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1AutoPictList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1AutoScaleList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1CFList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1CameraList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1CancelList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1CheckedList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1ColHiddenList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1ColoredList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1ColumnList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1DDEList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1DefaultList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1DefaultSizeList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1DisabledList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1DismissList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1DropLinesList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1DropStyleList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1DxList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1FirstButtonList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1FmlaGroupList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1FmlaLinkList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1FmlaMacroList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1FmlaPictList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1FmlaRangeList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1FmlaTxbxList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1HelpList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1HorizList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1IncList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1JustLastXList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1LCTList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1ListItemList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1LockTextList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1LockedList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1MapOCXList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1MaxList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1MinList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1MoveWithCellsList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1MultiLineList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1MultiSelList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1NoThreeD2List',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1NoThreeDList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1PageList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1PrintObjectList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1RecalcAlwaysList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1RowHiddenList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1RowList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1ScriptExtendedList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1ScriptLanguageList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1ScriptLocationList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1ScriptTextList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1SecretEditList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1SelList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1SelTypeList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1SizeWithCellsList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1TextHAlignList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1TextVAlignList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1UIObjList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1VScrollList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1VTEditList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1ValList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1ValidIdsList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1VisibleList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1WidthMinList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2Accel2List',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2AccelList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2AnchorList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2AutoFillList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2AutoLineList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2AutoPictList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2AutoScaleList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2CFList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2CameraList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2CancelList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2CheckedList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2ColHiddenList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2ColoredList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2ColumnList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2DDEList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2DefaultList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2DefaultSizeList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2DisabledList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2DismissList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2DropLinesList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2DropStyleList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2DxList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2FirstButtonList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2FmlaGroupList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2FmlaLinkList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2FmlaMacroList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2FmlaPictList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2FmlaRangeList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2FmlaTxbxList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2HelpList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2HorizList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2IncList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2JustLastXList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2LCTList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2ListItemList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2LockTextList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2LockedList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2MapOCXList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2MaxList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2MinList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2MoveWithCellsList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2MultiLineList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2MultiSelList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2NoThreeD2List',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2NoThreeDList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2PageList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2PrintObjectList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2RecalcAlwaysList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2RowHiddenList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2RowList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2ScriptExtendedList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2ScriptLanguageList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2ScriptLocationList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2ScriptTextList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2SecretEditList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2SelList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2SelTypeList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2SizeWithCellsList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2TextHAlignList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2TextVAlignList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2UIObjList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2VScrollList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2VTEditList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2ValList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2ValidIdsList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2VisibleList',
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2WidthMinList',
-  'schemasMicrosoftComOfficeOffice.CTCallout',
-  'schemasMicrosoftComOfficeOffice.CTClipPath',
-  'schemasMicrosoftComOfficeOffice.CTComplex',
-  'schemasMicrosoftComOfficeOffice.CTDiagram',
-  'schemasMicrosoftComOfficeOffice.CTExtrusion',
-  'schemasMicrosoftComOfficeOffice.CTFill',
-  'schemasMicrosoftComOfficeOffice.CTInk',
-  'schemasMicrosoftComOfficeOffice.CTRegroupTable',
-  'schemasMicrosoftComOfficeOffice.CTRules',
-  'schemasMicrosoftComOfficeOffice.CTSignatureLine',
-  'schemasMicrosoftComOfficeOffice.CTSkew',
-  'schemasMicrosoftComOfficeOffice.CTStrokeChild',
-  'schemasMicrosoftComOfficeOffice.STBWMode',
-  'schemasMicrosoftComOfficeOffice.STConnectorType',
-  'schemasMicrosoftComOfficeOffice.STHrAlign',
-  'schemasMicrosoftComOfficeOffice.STRelationshipId',
-  'schemasMicrosoftComOfficeOffice.STTrueFalse',
-  'schemasMicrosoftComOfficeOffice.STTrueFalseBlank',
-  'schemasMicrosoftComOfficePowerpoint.CTEmpty',
-  'schemasMicrosoftComOfficePowerpoint.CTRel',
-  'schemasMicrosoftComOfficeWord.CTAnchorLock',
-  'schemasMicrosoftComOfficeWord.CTBorder',
-  'schemasMicrosoftComOfficeWord.CTWrap',
-  'schemasMicrosoftComVml.CTArc',
-  'schemasMicrosoftComVml.CTCurve',
-  'schemasMicrosoftComVml.CTImage',
-  'schemasMicrosoftComVml.CTImageData',
-  'schemasMicrosoftComVml.CTLine',
-  'schemasMicrosoftComVml.CTOval',
-  'schemasMicrosoftComVml.CTPolyLine',
-  'schemasMicrosoftComVml.CTRect',
-  'schemasMicrosoftComVml.CTRoundRect',
-  'schemasMicrosoftComVml.STEditAs',
-  'schemasMicrosoftComVml.STFillMethod',
-  'schemasMicrosoftComVml.STFillType',
-  'schemasMicrosoftComVml.STImageAspect',
-  'schemasMicrosoftComVml.STShadowType',
-  'schemasMicrosoftComVml.STStrokeArrowLength',
-  'schemasMicrosoftComVml.STStrokeArrowType',
-  'schemasMicrosoftComVml.STStrokeArrowWidth',
-  'schemasMicrosoftComVml.STStrokeEndCap',
-  'schemasMicrosoftComVml.STStrokeLineStyle',
-  'schemasMicrosoftComVml.STTrueFalseBlank',
-  'schemasMicrosoftComVml.impl.CTFormulasImpl$1FList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1AnchorlockList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1ArcList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1BorderbottomList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1BorderleftList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1BorderrightList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1BordertopList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1CalloutList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1ClientDataList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1ClippathList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1CurveList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1DiagramList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1ExtrusionList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1FillList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1FormulasList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1GroupList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1HandlesList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1ImageList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1ImagedataList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1LineList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1LockList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1OvalList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1PathList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1PolylineList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1RectList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1RoundrectList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1ShadowList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1ShapeList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1ShapetypeList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1SignaturelineList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1SkewList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1StrokeList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1TextboxList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1TextdataList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1TextpathList',
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1WrapList',
-  'schemasMicrosoftComVml.impl.CTHandlesImpl$1HList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1AnchorlockList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1BorderbottomList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1BorderleftList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1BorderrightList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1BordertopList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1CalloutList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1ClippathList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1ExtrusionList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1FillList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1FormulasList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1HandlesList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1ImagedataList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1InkList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1IscommentList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1LockList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1PathList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1ShadowList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1SignaturelineList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1SkewList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1StrokeList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1TextboxList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1TextdataList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1TextpathList',
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1WrapList',
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1AnchorlockList',
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1BorderbottomList',
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1BorderleftList',
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1BorderrightList',
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1BordertopList',
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1CalloutList',
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1ClientDataList',
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1ClippathList',
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1ExtrusionList',
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1FillList',
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1FormulasList',
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1HandlesList',
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1ImagedataList',
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1LockList',
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1PathList',
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1ShadowList',
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1SignaturelineList',
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1SkewList',
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1StrokeList',
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1TextboxList',
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1TextdataList',
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1TextpathList',
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1WrapList',
   'ucar.ma2.DataType',
   'ucar.nc2.Attribute',
   'ucar.nc2.Dimension',
   'ucar.nc2.Group',
   'ucar.nc2.NetcdfFile',
   'ucar.nc2.Variable',
-  'ucar.nc2.dataset.NetcdfDataset',
+  'ucar.nc2.dataset.NetcdfDataset'
 ]
diff --git a/plugins/ingest-attachment/licenses/bcmail-jdk15on-1.52.jar.sha1 b/plugins/ingest-attachment/licenses/bcmail-jdk15on-1.52.jar.sha1
deleted file mode 100644
index de084c948f4..00000000000
--- a/plugins/ingest-attachment/licenses/bcmail-jdk15on-1.52.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4995a870400e1554d1c7ed2afcb5d198fae12db9
diff --git a/plugins/ingest-attachment/licenses/bcmail-jdk15on-1.54.jar.sha1 b/plugins/ingest-attachment/licenses/bcmail-jdk15on-1.54.jar.sha1
new file mode 100644
index 00000000000..79da45c5c42
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/bcmail-jdk15on-1.54.jar.sha1
@@ -0,0 +1 @@
+9d9b5432b4b29ef4a853223bc6e19379ef116cca
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/bcpkix-jdk15on-1.52.jar.sha1 b/plugins/ingest-attachment/licenses/bcpkix-jdk15on-1.52.jar.sha1
deleted file mode 100644
index 489ceeaaf36..00000000000
--- a/plugins/ingest-attachment/licenses/bcpkix-jdk15on-1.52.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b8ffac2bbc6626f86909589c8cc63637cc936504
diff --git a/plugins/ingest-attachment/licenses/bcpkix-jdk15on-1.54.jar.sha1 b/plugins/ingest-attachment/licenses/bcpkix-jdk15on-1.54.jar.sha1
new file mode 100644
index 00000000000..2d0c3cf4e27
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/bcpkix-jdk15on-1.54.jar.sha1
@@ -0,0 +1 @@
+b11bfee99bb11eea344de6e4a07fe89212c55c02
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/bcprov-jdk15on-1.52.jar.sha1 b/plugins/ingest-attachment/licenses/bcprov-jdk15on-1.52.jar.sha1
deleted file mode 100644
index 14ecc1be40b..00000000000
--- a/plugins/ingest-attachment/licenses/bcprov-jdk15on-1.52.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-88a941faf9819d371e3174b5ed56a3f3f7d73269
diff --git a/plugins/ingest-attachment/licenses/bcprov-jdk15on-1.54.jar.sha1 b/plugins/ingest-attachment/licenses/bcprov-jdk15on-1.54.jar.sha1
new file mode 100644
index 00000000000..fcda646b42a
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/bcprov-jdk15on-1.54.jar.sha1
@@ -0,0 +1 @@
+1acdedeb89f1d950d67b73d481eb7736df65eedb
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/fontbox-1.8.10.jar.sha1 b/plugins/ingest-attachment/licenses/fontbox-1.8.10.jar.sha1
deleted file mode 100644
index ce7f9f5d49c..00000000000
--- a/plugins/ingest-attachment/licenses/fontbox-1.8.10.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-41776c7713e3f3a1ce688bd96459fc597298c340
diff --git a/plugins/ingest-attachment/licenses/fontbox-2.0.1.jar.sha1 b/plugins/ingest-attachment/licenses/fontbox-2.0.1.jar.sha1
new file mode 100644
index 00000000000..0668199b242
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/fontbox-2.0.1.jar.sha1
@@ -0,0 +1 @@
+b9d4f0993e015f3f1ce0be9e7300cf62dd7a7f15
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/jempbox-1.8.10.jar.sha1 b/plugins/ingest-attachment/licenses/jempbox-1.8.10.jar.sha1
deleted file mode 100644
index 5a7b1997208..00000000000
--- a/plugins/ingest-attachment/licenses/jempbox-1.8.10.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-40df4e4ca884aadc20b82d5abd0a3679774c55a6
diff --git a/plugins/ingest-attachment/licenses/jempbox-1.8.12.jar.sha1 b/plugins/ingest-attachment/licenses/jempbox-1.8.12.jar.sha1
new file mode 100644
index 00000000000..0e3dcf4573b
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/jempbox-1.8.12.jar.sha1
@@ -0,0 +1 @@
+426450c573c19f6f2c751a7a52c11931b712c9f6
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/pdfbox-1.8.10.jar.sha1 b/plugins/ingest-attachment/licenses/pdfbox-1.8.10.jar.sha1
deleted file mode 100644
index 98ce1f9d98c..00000000000
--- a/plugins/ingest-attachment/licenses/pdfbox-1.8.10.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-bc5d1254495be36d0a3b3d6c35f88d05200b9311
diff --git a/plugins/ingest-attachment/licenses/pdfbox-2.0.1.jar.sha1 b/plugins/ingest-attachment/licenses/pdfbox-2.0.1.jar.sha1
new file mode 100644
index 00000000000..1014db34044
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/pdfbox-2.0.1.jar.sha1
@@ -0,0 +1 @@
+dbc69649118b7eff278f228c070a40ee559e1f62
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-3.13.jar.sha1 b/plugins/ingest-attachment/licenses/poi-3.13.jar.sha1
deleted file mode 100644
index 09063c1e5e0..00000000000
--- a/plugins/ingest-attachment/licenses/poi-3.13.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0f59f504ba8c521e61e25f417ec652fd485010f3
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-3.15-beta1.jar.sha1 b/plugins/ingest-attachment/licenses/poi-3.15-beta1.jar.sha1
new file mode 100644
index 00000000000..6049604dd97
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/poi-3.15-beta1.jar.sha1
@@ -0,0 +1 @@
+048bb8326b81323631d9ceb4236cfbd382e56da2
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-ooxml-3.13.jar.sha1 b/plugins/ingest-attachment/licenses/poi-ooxml-3.13.jar.sha1
deleted file mode 100644
index 16784299855..00000000000
--- a/plugins/ingest-attachment/licenses/poi-ooxml-3.13.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c364a8f5422d613e3a56db3b4b889f2989d7ee73
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-ooxml-3.15-beta1.jar.sha1 b/plugins/ingest-attachment/licenses/poi-ooxml-3.15-beta1.jar.sha1
new file mode 100644
index 00000000000..c3cf49d9246
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/poi-ooxml-3.15-beta1.jar.sha1
@@ -0,0 +1 @@
+81085a47fdf0d74d473d605c6b3784e26731842e
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-ooxml-schemas-3.13.jar.sha1 b/plugins/ingest-attachment/licenses/poi-ooxml-schemas-3.13.jar.sha1
deleted file mode 100644
index b5a3a05c489..00000000000
--- a/plugins/ingest-attachment/licenses/poi-ooxml-schemas-3.13.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-56fb0b9f3ffc3d7f7fc9b59e17b5fa2c3ab921e7
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-ooxml-schemas-3.15-beta1.jar.sha1 b/plugins/ingest-attachment/licenses/poi-ooxml-schemas-3.15-beta1.jar.sha1
new file mode 100644
index 00000000000..afd3b676d08
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/poi-ooxml-schemas-3.15-beta1.jar.sha1
@@ -0,0 +1 @@
+f8bc979ad79908a99483337f1ca2edf78558ac20
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-scratchpad-3.13.jar.sha1 b/plugins/ingest-attachment/licenses/poi-scratchpad-3.13.jar.sha1
deleted file mode 100644
index cc61780e2a5..00000000000
--- a/plugins/ingest-attachment/licenses/poi-scratchpad-3.13.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-09d763275e6c7fa05d47e2581606748669e88c55
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-scratchpad-3.15-beta1.jar.sha1 b/plugins/ingest-attachment/licenses/poi-scratchpad-3.15-beta1.jar.sha1
new file mode 100644
index 00000000000..7056a9fa49e
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/poi-scratchpad-3.15-beta1.jar.sha1
@@ -0,0 +1 @@
+f4e276aaf97a60a1156388c9e38069122b7ea914
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-core-1.11.jar.sha1 b/plugins/ingest-attachment/licenses/tika-core-1.11.jar.sha1
deleted file mode 100644
index a6dfd778a9c..00000000000
--- a/plugins/ingest-attachment/licenses/tika-core-1.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d37a6b9080c8361e47b2050f69833fd61501ede9
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-core-1.13.jar.sha1 b/plugins/ingest-attachment/licenses/tika-core-1.13.jar.sha1
new file mode 100644
index 00000000000..cfc36a450bd
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/tika-core-1.13.jar.sha1
@@ -0,0 +1 @@
+1305c798d41d1d7bbf12cb7c0ca184c98eed25ad
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-parsers-1.11.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parsers-1.11.jar.sha1
deleted file mode 100644
index fbbd59efaf9..00000000000
--- a/plugins/ingest-attachment/licenses/tika-parsers-1.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-355dc05d842ed223fc682da472229473ba706d68
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-parsers-1.13.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parsers-1.13.jar.sha1
new file mode 100644
index 00000000000..7fb2755d545
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/tika-parsers-1.13.jar.sha1
@@ -0,0 +1 @@
+374fde67b9d35f785534b0e6c4953533c31bab5f
\ No newline at end of file
diff --git a/plugins/mapper-attachments/build.gradle b/plugins/mapper-attachments/build.gradle
index 340832af6f8..b00f61867e9 100644
--- a/plugins/mapper-attachments/build.gradle
+++ b/plugins/mapper-attachments/build.gradle
@@ -23,10 +23,10 @@ esplugin {
 }
 
 versions << [
-  'tika': '1.11',
-  'pdfbox': '1.8.10',
-  'bouncycastle': '1.52',
-  'poi': '3.13'
+  'tika': '1.13',
+  'pdfbox': '2.0.1',
+  'bouncycastle': '1.54',
+  'poi': '3.15-beta1'
 ]
 
 dependencies {
@@ -44,7 +44,7 @@ dependencies {
   // Adobe PDF
   compile "org.apache.pdfbox:pdfbox:${versions.pdfbox}"
   compile "org.apache.pdfbox:fontbox:${versions.pdfbox}"
-  compile "org.apache.pdfbox:jempbox:${versions.pdfbox}"
+  compile "org.apache.pdfbox:jempbox:1.8.12"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
   compile "org.bouncycastle:bcmail-jdk15on:${versions.bouncycastle}"
   compile "org.bouncycastle:bcprov-jdk15on:${versions.bouncycastle}"
@@ -61,6 +61,9 @@ dependencies {
   compile 'org.apache.commons:commons-compress:1.10'
 }
 
+// TODO: stop using LanguageIdentifier...
+compileJava.options.compilerArgs << "-Xlint:-deprecation"
+
 forbiddenPatterns {
   exclude '**/*.docx'
   exclude '**/*.pdf'
@@ -69,1926 +72,1980 @@ forbiddenPatterns {
 
 thirdPartyAudit.excludes = [
   // classes are missing: some due to our whitelisting of parsers
-  'com.coremedia.iso.IsoFile', 
-  'com.coremedia.iso.boxes.Box', 
-  'com.coremedia.iso.boxes.Container', 
-  'com.coremedia.iso.boxes.FileTypeBox', 
-  'com.coremedia.iso.boxes.MetaBox', 
-  'com.coremedia.iso.boxes.MovieBox', 
-  'com.coremedia.iso.boxes.MovieHeaderBox', 
-  'com.coremedia.iso.boxes.SampleTableBox', 
-  'com.coremedia.iso.boxes.TrackBox', 
-  'com.coremedia.iso.boxes.TrackHeaderBox', 
-  'com.coremedia.iso.boxes.UserDataBox', 
-  'com.coremedia.iso.boxes.apple.AppleItemListBox', 
-  'com.coremedia.iso.boxes.sampleentry.AudioSampleEntry', 
-  'com.drew.imaging.jpeg.JpegMetadataReader', 
-  'com.drew.imaging.tiff.TiffMetadataReader', 
-  'com.drew.imaging.webp.WebpMetadataReader', 
-  'com.drew.lang.ByteArrayReader', 
-  'com.drew.lang.GeoLocation', 
-  'com.drew.lang.Rational', 
-  'com.drew.metadata.Directory', 
-  'com.drew.metadata.Metadata', 
-  'com.drew.metadata.Tag', 
-  'com.drew.metadata.exif.ExifIFD0Directory', 
-  'com.drew.metadata.exif.ExifReader', 
-  'com.drew.metadata.exif.ExifSubIFDDirectory', 
-  'com.drew.metadata.exif.ExifThumbnailDirectory', 
-  'com.drew.metadata.exif.GpsDirectory', 
-  'com.drew.metadata.iptc.IptcDirectory', 
-  'com.drew.metadata.jpeg.JpegCommentDirectory', 
-  'com.drew.metadata.jpeg.JpegDirectory', 
-  'com.drew.metadata.xmp.XmpReader', 
-  'com.github.junrar.Archive', 
-  'com.github.junrar.rarfile.FileHeader', 
-  'com.googlecode.mp4parser.DataSource', 
-  'com.googlecode.mp4parser.boxes.apple.AppleAlbumBox', 
-  'com.googlecode.mp4parser.boxes.apple.AppleArtist2Box', 
-  'com.googlecode.mp4parser.boxes.apple.AppleArtistBox', 
-  'com.googlecode.mp4parser.boxes.apple.AppleCommentBox', 
-  'com.googlecode.mp4parser.boxes.apple.AppleCompilationBox', 
-  'com.googlecode.mp4parser.boxes.apple.AppleDiskNumberBox', 
-  'com.googlecode.mp4parser.boxes.apple.AppleEncoderBox', 
-  'com.googlecode.mp4parser.boxes.apple.AppleGenreBox', 
-  'com.googlecode.mp4parser.boxes.apple.AppleNameBox', 
-  'com.googlecode.mp4parser.boxes.apple.AppleRecordingYear2Box', 
-  'com.googlecode.mp4parser.boxes.apple.AppleTrackAuthorBox', 
-  'com.googlecode.mp4parser.boxes.apple.AppleTrackNumberBox', 
-  'com.googlecode.mp4parser.boxes.apple.Utf8AppleDataBox', 
-  'com.googlecode.mp4parser.util.CastUtils', 
-  'com.healthmarketscience.jackcess.Column', 
-  'com.healthmarketscience.jackcess.CryptCodecProvider', 
-  'com.healthmarketscience.jackcess.DataType', 
-  'com.healthmarketscience.jackcess.Database', 
-  'com.healthmarketscience.jackcess.DatabaseBuilder', 
-  'com.healthmarketscience.jackcess.PropertyMap$Property', 
-  'com.healthmarketscience.jackcess.PropertyMap', 
-  'com.healthmarketscience.jackcess.Row', 
-  'com.healthmarketscience.jackcess.Table', 
-  'com.healthmarketscience.jackcess.query.Query', 
-  'com.healthmarketscience.jackcess.util.LinkResolver', 
-  'com.healthmarketscience.jackcess.util.OleBlob$CompoundContent', 
-  'com.healthmarketscience.jackcess.util.OleBlob$Content', 
-  'com.healthmarketscience.jackcess.util.OleBlob$ContentType', 
-  'com.healthmarketscience.jackcess.util.OleBlob$LinkContent', 
-  'com.healthmarketscience.jackcess.util.OleBlob$OtherContent', 
-  'com.healthmarketscience.jackcess.util.OleBlob$SimplePackageContent', 
-  'com.healthmarketscience.jackcess.util.OleBlob', 
-  'com.healthmarketscience.jackcess.util.TableIterableBuilder', 
-  'com.ibm.icu.text.Bidi', 
-  'com.ibm.icu.text.Normalizer', 
-  'com.jmatio.io.MatFileHeader', 
-  'com.jmatio.io.MatFileReader', 
-  'com.jmatio.types.MLArray', 
-  'com.jmatio.types.MLStructure', 
-  'com.microsoft.schemas.office.x2006.digsig.STPositiveInteger', 
-  'com.microsoft.schemas.office.x2006.digsig.STSignatureComments', 
-  'com.microsoft.schemas.office.x2006.digsig.STSignatureProviderUrl', 
-  'com.microsoft.schemas.office.x2006.digsig.STSignatureText', 
-  'com.microsoft.schemas.office.x2006.digsig.STSignatureType', 
-  'com.microsoft.schemas.office.x2006.digsig.STUniqueIdentifierWithBraces', 
-  'com.microsoft.schemas.office.x2006.digsig.STVersion', 
-  'com.pff.PSTAttachment', 
-  'com.pff.PSTFile', 
-  'com.pff.PSTFolder', 
-  'com.pff.PSTMessage', 
-  'com.sun.syndication.feed.synd.SyndContent', 
-  'com.sun.syndication.feed.synd.SyndEntry',
-  'com.sun.syndication.feed.synd.SyndFeed', 
-  'com.sun.syndication.io.SyndFeedInput', 
-  'com.uwyn.jhighlight.renderer.Renderer', 
-  'com.uwyn.jhighlight.renderer.XhtmlRendererFactory', 
-  'de.l3s.boilerpipe.BoilerpipeExtractor', 
-  'de.l3s.boilerpipe.document.TextBlock', 
-  'de.l3s.boilerpipe.document.TextDocument', 
-  'de.l3s.boilerpipe.extractors.DefaultExtractor', 
-  'de.l3s.boilerpipe.sax.BoilerpipeHTMLContentHandler', 
-  'javax.mail.BodyPart', 
-  'javax.mail.Header', 
-  'javax.mail.Message$RecipientType', 
-  'javax.mail.MessagingException', 
-  'javax.mail.Multipart', 
-  'javax.mail.Part', 
-  'javax.mail.Session', 
-  'javax.mail.Transport', 
-  'javax.mail.internet.ContentType', 
-  'javax.mail.internet.InternetAddress', 
-  'javax.mail.internet.InternetHeaders', 
-  'javax.mail.internet.MimeBodyPart', 
-  'javax.mail.internet.MimeMessage', 
-  'javax.mail.internet.MimeMultipart', 
-  'javax.mail.internet.MimePart', 
-  'javax.mail.internet.SharedInputStream', 
-  'javax.servlet.ServletContextEvent', 
-  'javax.servlet.ServletContextListener', 
-  'javax.ws.rs.core.Response', 
-  'junit.framework.TestCase', 
-  'opennlp.tools.namefind.NameFinderME', 
-  'opennlp.tools.namefind.TokenNameFinderModel', 
-  'opennlp.tools.util.Span', 
-  'org.apache.avalon.framework.logger.Logger', 
-  'org.apache.commons.csv.CSVFormat', 
-  'org.apache.commons.csv.CSVParser', 
-  'org.apache.commons.csv.CSVRecord', 
-  'org.apache.commons.exec.CommandLine', 
-  'org.apache.commons.exec.DefaultExecutor', 
-  'org.apache.commons.exec.ExecuteWatchdog', 
-  'org.apache.commons.exec.PumpStreamHandler', 
-  'org.apache.commons.exec.environment.EnvironmentUtils', 
-  'org.apache.ctakes.typesystem.type.refsem.UmlsConcept', 
-  'org.apache.ctakes.typesystem.type.textsem.IdentifiedAnnotation', 
-  'org.apache.cxf.jaxrs.client.WebClient', 
-  'org.apache.cxf.jaxrs.ext.multipart.Attachment', 
-  'org.apache.cxf.jaxrs.ext.multipart.ContentDisposition', 
-  'org.apache.cxf.jaxrs.ext.multipart.MultipartBody', 
-  'org.apache.james.mime4j.MimeException', 
-  'org.apache.james.mime4j.codec.DecodeMonitor', 
-  'org.apache.james.mime4j.codec.DecoderUtil', 
-  'org.apache.james.mime4j.dom.FieldParser', 
-  'org.apache.james.mime4j.dom.address.Address', 
-  'org.apache.james.mime4j.dom.address.AddressList', 
-  'org.apache.james.mime4j.dom.address.Mailbox', 
-  'org.apache.james.mime4j.dom.address.MailboxList', 
-  'org.apache.james.mime4j.dom.field.AddressListField', 
-  'org.apache.james.mime4j.dom.field.DateTimeField', 
-  'org.apache.james.mime4j.dom.field.MailboxListField', 
-  'org.apache.james.mime4j.dom.field.ParsedField', 
-  'org.apache.james.mime4j.dom.field.UnstructuredField', 
-  'org.apache.james.mime4j.field.LenientFieldParser', 
-  'org.apache.james.mime4j.parser.ContentHandler', 
-  'org.apache.james.mime4j.parser.MimeStreamParser', 
-  'org.apache.james.mime4j.stream.BodyDescriptor', 
-  'org.apache.james.mime4j.stream.Field', 
-  'org.apache.james.mime4j.stream.MimeConfig', 
-  'org.apache.jcp.xml.dsig.internal.dom.DOMDigestMethod', 
-  'org.apache.jcp.xml.dsig.internal.dom.DOMKeyInfo', 
-  'org.apache.jcp.xml.dsig.internal.dom.DOMReference', 
-  'org.apache.jcp.xml.dsig.internal.dom.DOMSignedInfo', 
-  'org.apache.log.Hierarchy', 
-  'org.apache.log.Logger', 
-  'org.apache.sis.internal.util.CheckedArrayList', 
-  'org.apache.sis.internal.util.CheckedHashSet', 
-  'org.apache.sis.metadata.iso.DefaultMetadata', 
-  'org.apache.sis.metadata.iso.DefaultMetadataScope', 
-  'org.apache.sis.metadata.iso.constraint.DefaultLegalConstraints', 
-  'org.apache.sis.metadata.iso.extent.DefaultGeographicBoundingBox', 
-  'org.apache.sis.metadata.iso.extent.DefaultGeographicDescription', 
-  'org.apache.sis.metadata.iso.identification.DefaultDataIdentification', 
-  'org.apache.sis.storage.DataStore', 
-  'org.apache.sis.storage.DataStores', 
-  'org.apache.sis.util.collection.CodeListSet', 
-  'org.apache.tools.ant.BuildException', 
-  'org.apache.tools.ant.FileScanner', 
-  'org.apache.tools.ant.Project', 
-  'org.apache.tools.ant.taskdefs.Jar', 
-  'org.apache.tools.ant.taskdefs.Javac', 
-  'org.apache.tools.ant.taskdefs.MatchingTask', 
-  'org.apache.tools.ant.types.FileSet', 
-  'org.apache.tools.ant.types.Path$PathElement', 
-  'org.apache.tools.ant.types.Path', 
-  'org.apache.tools.ant.types.Reference', 
-  'org.apache.uima.UIMAFramework', 
-  'org.apache.uima.analysis_engine.AnalysisEngine', 
-  'org.apache.uima.cas.Type', 
-  'org.apache.uima.cas.impl.XCASSerializer', 
-  'org.apache.uima.cas.impl.XmiCasSerializer', 
-  'org.apache.uima.cas.impl.XmiSerializationSharedData', 
-  'org.apache.uima.fit.util.JCasUtil', 
-  'org.apache.uima.jcas.JCas', 
-  'org.apache.uima.jcas.cas.FSArray', 
-  'org.apache.uima.util.XMLInputSource', 
-  'org.apache.uima.util.XMLParser', 
-  'org.apache.uima.util.XmlCasSerializer', 
-  'org.apache.xml.security.Init', 
-  'org.apache.xml.security.c14n.Canonicalizer', 
-  'org.apache.xml.security.utils.Base64', 
-  'org.bouncycastle.asn1.DERObject', 
-  'org.etsi.uri.x01903.v13.AnyType', 
-  'org.etsi.uri.x01903.v13.ClaimedRolesListType', 
-  'org.etsi.uri.x01903.v13.CounterSignatureType', 
-  'org.etsi.uri.x01903.v13.DataObjectFormatType$Factory', 
-  'org.etsi.uri.x01903.v13.DataObjectFormatType', 
-  'org.etsi.uri.x01903.v13.IdentifierType', 
-  'org.etsi.uri.x01903.v13.IncludeType', 
-  'org.etsi.uri.x01903.v13.ObjectIdentifierType', 
-  'org.etsi.uri.x01903.v13.OtherCertStatusRefsType', 
-  'org.etsi.uri.x01903.v13.OtherCertStatusValuesType', 
-  'org.etsi.uri.x01903.v13.ReferenceInfoType', 
-  'org.etsi.uri.x01903.v13.SigPolicyQualifiersListType', 
-  'org.etsi.uri.x01903.v13.SignaturePolicyIdType', 
-  'org.etsi.uri.x01903.v13.SignatureProductionPlaceType', 
-  'org.etsi.uri.x01903.v13.SignedDataObjectPropertiesType', 
-  'org.etsi.uri.x01903.v13.SignerRoleType', 
-  'org.etsi.uri.x01903.v13.UnsignedDataObjectPropertiesType', 
-  'org.etsi.uri.x01903.v13.impl.CRLRefsTypeImpl$1CRLRefList', 
-  'org.etsi.uri.x01903.v13.impl.CRLValuesTypeImpl$1EncapsulatedCRLValueList', 
-  'org.etsi.uri.x01903.v13.impl.CertIDListTypeImpl$1CertList', 
-  'org.etsi.uri.x01903.v13.impl.CertificateValuesTypeImpl$1EncapsulatedX509CertificateList', 
-  'org.etsi.uri.x01903.v13.impl.CertificateValuesTypeImpl$1OtherCertificateList', 
-  'org.etsi.uri.x01903.v13.impl.GenericTimeStampTypeImpl$1EncapsulatedTimeStampList', 
-  'org.etsi.uri.x01903.v13.impl.GenericTimeStampTypeImpl$1IncludeList', 
-  'org.etsi.uri.x01903.v13.impl.GenericTimeStampTypeImpl$1ReferenceInfoList', 
-  'org.etsi.uri.x01903.v13.impl.GenericTimeStampTypeImpl$1XMLTimeStampList', 
-  'org.etsi.uri.x01903.v13.impl.OCSPRefsTypeImpl$1OCSPRefList', 
-  'org.etsi.uri.x01903.v13.impl.OCSPValuesTypeImpl$1EncapsulatedOCSPValueList', 
-  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1ArchiveTimeStampList', 
-  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1AttrAuthoritiesCertValuesList', 
-  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1AttributeCertificateRefsList', 
-  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1AttributeRevocationRefsList', 
-  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1AttributeRevocationValuesList', 
-  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1CertificateValuesList', 
-  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1CompleteCertificateRefsList', 
-  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1CompleteRevocationRefsList', 
-  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1CounterSignatureList', 
-  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1RefsOnlyTimeStampList', 
-  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1RevocationValuesList', 
-  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1SigAndRefsTimeStampList', 
-  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1SignatureTimeStampList', 
-  'org.etsi.uri.x01903.v14.ValidationDataType$Factory', 
-  'org.etsi.uri.x01903.v14.ValidationDataType', 
-  'org.json.JSONArray', 
-  'org.json.JSONObject', 
-  'org.json.XML', 
-  'org.json.simple.JSONArray', 
-  'org.json.simple.JSONObject', 
-  'org.json.simple.JSONValue', 
-  'org.junit.Test', 
-  'org.junit.internal.TextListener', 
-  'org.junit.runner.JUnitCore', 
+  'com.coremedia.iso.IsoFile',
+  'com.coremedia.iso.boxes.Box',
+  'com.coremedia.iso.boxes.Container',
+  'com.coremedia.iso.boxes.FileTypeBox',
+  'com.coremedia.iso.boxes.MetaBox',
+  'com.coremedia.iso.boxes.MovieBox',
+  'com.coremedia.iso.boxes.MovieHeaderBox',
+  'com.coremedia.iso.boxes.SampleTableBox',
+  'com.coremedia.iso.boxes.TrackBox',
+  'com.coremedia.iso.boxes.TrackHeaderBox',
+  'com.coremedia.iso.boxes.UserDataBox',
+  'com.coremedia.iso.boxes.apple.AppleItemListBox',
+  'com.coremedia.iso.boxes.sampleentry.AudioSampleEntry',
+  'com.drew.imaging.jpeg.JpegMetadataReader',
+  'com.drew.imaging.tiff.TiffMetadataReader',
+  'com.drew.imaging.webp.WebpMetadataReader',
+  'com.drew.lang.ByteArrayReader',
+  'com.drew.lang.GeoLocation',
+  'com.drew.lang.Rational',
+  'com.drew.metadata.Directory',
+  'com.drew.metadata.Metadata',
+  'com.drew.metadata.Tag',
+  'com.drew.metadata.exif.ExifIFD0Directory',
+  'com.drew.metadata.exif.ExifReader',
+  'com.drew.metadata.exif.ExifSubIFDDirectory',
+  'com.drew.metadata.exif.ExifThumbnailDirectory',
+  'com.drew.metadata.exif.GpsDirectory',
+  'com.drew.metadata.iptc.IptcDirectory',
+  'com.drew.metadata.jpeg.JpegCommentDirectory',
+  'com.drew.metadata.jpeg.JpegDirectory',
+  'com.github.junrar.Archive',
+  'com.github.junrar.rarfile.FileHeader',
+  'com.google.common.reflect.TypeToken',
+  'com.google.gson.Gson',
+  'com.googlecode.mp4parser.DataSource',
+  'com.googlecode.mp4parser.boxes.apple.AppleAlbumBox',
+  'com.googlecode.mp4parser.boxes.apple.AppleArtist2Box',
+  'com.googlecode.mp4parser.boxes.apple.AppleArtistBox',
+  'com.googlecode.mp4parser.boxes.apple.AppleCommentBox',
+  'com.googlecode.mp4parser.boxes.apple.AppleCompilationBox',
+  'com.googlecode.mp4parser.boxes.apple.AppleDiskNumberBox',
+  'com.googlecode.mp4parser.boxes.apple.AppleEncoderBox',
+  'com.googlecode.mp4parser.boxes.apple.AppleGenreBox',
+  'com.googlecode.mp4parser.boxes.apple.AppleNameBox',
+  'com.googlecode.mp4parser.boxes.apple.AppleRecordingYear2Box',
+  'com.googlecode.mp4parser.boxes.apple.AppleTrackAuthorBox',
+  'com.googlecode.mp4parser.boxes.apple.AppleTrackNumberBox',
+  'com.googlecode.mp4parser.boxes.apple.Utf8AppleDataBox',
+  'com.googlecode.mp4parser.util.CastUtils',
+  'com.graphbuilder.curve.ControlPath',
+  'com.graphbuilder.curve.GroupIterator',
+  'com.graphbuilder.curve.NURBSpline',
+  'com.graphbuilder.curve.ShapeMultiPath',
+  'com.graphbuilder.curve.ValueVector',
+  'com.graphbuilder.geom.PointFactory',
+  'com.healthmarketscience.jackcess.Column',
+  'com.healthmarketscience.jackcess.CryptCodecProvider',
+  'com.healthmarketscience.jackcess.DataType',
+  'com.healthmarketscience.jackcess.Database',
+  'com.healthmarketscience.jackcess.DatabaseBuilder',
+  'com.healthmarketscience.jackcess.PropertyMap$Property',
+  'com.healthmarketscience.jackcess.PropertyMap',
+  'com.healthmarketscience.jackcess.Row',
+  'com.healthmarketscience.jackcess.Table',
+  'com.healthmarketscience.jackcess.query.Query',
+  'com.healthmarketscience.jackcess.util.LinkResolver',
+  'com.healthmarketscience.jackcess.util.OleBlob$CompoundContent',
+  'com.healthmarketscience.jackcess.util.OleBlob$Content',
+  'com.healthmarketscience.jackcess.util.OleBlob$ContentType',
+  'com.healthmarketscience.jackcess.util.OleBlob$LinkContent',
+  'com.healthmarketscience.jackcess.util.OleBlob$OtherContent',
+  'com.healthmarketscience.jackcess.util.OleBlob$SimplePackageContent',
+  'com.healthmarketscience.jackcess.util.OleBlob',
+  'com.healthmarketscience.jackcess.util.TableIterableBuilder',
+  'com.jmatio.io.MatFileHeader',
+  'com.jmatio.io.MatFileReader',
+  'com.jmatio.types.MLArray',
+  'com.jmatio.types.MLStructure',
+  'com.microsoft.schemas.office.excel.STCF',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1Accel2List',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1AccelList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1AnchorList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1AutoFillList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1AutoLineList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1AutoPictList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1AutoScaleList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1CFList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1CameraList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1CancelList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1CheckedList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ColHiddenList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ColoredList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ColumnList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1DDEList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1DefaultList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1DefaultSizeList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1DisabledList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1DismissList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1DropLinesList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1DropStyleList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1DxList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1FirstButtonList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1FmlaGroupList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1FmlaLinkList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1FmlaMacroList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1FmlaPictList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1FmlaRangeList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1FmlaTxbxList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1HelpList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1HorizList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1IncList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1JustLastXList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1LCTList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ListItemList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1LockTextList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1LockedList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1MapOCXList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1MaxList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1MinList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1MoveWithCellsList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1MultiLineList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1MultiSelList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1NoThreeD2List',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1NoThreeDList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1PageList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1PrintObjectList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1RecalcAlwaysList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1RowHiddenList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1RowList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ScriptExtendedList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ScriptLanguageList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ScriptLocationList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ScriptTextList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1SecretEditList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1SelList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1SelTypeList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1SizeWithCellsList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1TextHAlignList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1TextVAlignList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1UIObjList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1VScrollList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1VTEditList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ValList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1ValidIdsList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1VisibleList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$1WidthMinList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2Accel2List',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2AccelList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2AnchorList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2AutoFillList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2AutoLineList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2AutoPictList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2AutoScaleList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2CFList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2CameraList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2CancelList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2CheckedList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2ColHiddenList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2ColoredList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2ColumnList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2DDEList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2DefaultList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2DefaultSizeList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2DisabledList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2DismissList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2DropLinesList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2DropStyleList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2DxList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2FirstButtonList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2FmlaGroupList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2FmlaLinkList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2FmlaMacroList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2FmlaPictList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2FmlaRangeList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2FmlaTxbxList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2HelpList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2HorizList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2IncList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2JustLastXList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2LCTList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2ListItemList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2LockTextList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2LockedList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2MapOCXList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2MaxList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2MinList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2MoveWithCellsList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2MultiLineList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2MultiSelList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2NoThreeD2List',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2NoThreeDList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2PageList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2PrintObjectList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2RecalcAlwaysList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2RowHiddenList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2RowList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2ScriptExtendedList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2ScriptLanguageList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2ScriptLocationList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2ScriptTextList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2SecretEditList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2SelList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2SelTypeList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2SizeWithCellsList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2TextHAlignList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2TextVAlignList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2UIObjList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2VScrollList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2VTEditList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2ValList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2ValidIdsList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2VisibleList',
+  'com.microsoft.schemas.office.excel.impl.CTClientDataImpl$2WidthMinList',
+  'com.microsoft.schemas.office.office.CTCallout',
+  'com.microsoft.schemas.office.office.CTClipPath',
+  'com.microsoft.schemas.office.office.CTComplex',
+  'com.microsoft.schemas.office.office.CTDiagram',
+  'com.microsoft.schemas.office.office.CTExtrusion',
+  'com.microsoft.schemas.office.office.CTFill',
+  'com.microsoft.schemas.office.office.CTInk',
+  'com.microsoft.schemas.office.office.CTRegroupTable',
+  'com.microsoft.schemas.office.office.CTRules',
+  'com.microsoft.schemas.office.office.CTSignatureLine',
+  'com.microsoft.schemas.office.office.CTSkew',
+  'com.microsoft.schemas.office.office.CTStrokeChild',
+  'com.microsoft.schemas.office.office.STBWMode',
+  'com.microsoft.schemas.office.office.STConnectorType',
+  'com.microsoft.schemas.office.office.STHrAlign',
+  'com.microsoft.schemas.office.office.STRelationshipId',
+  'com.microsoft.schemas.office.office.STTrueFalse',
+  'com.microsoft.schemas.office.office.STTrueFalseBlank',
+  'com.microsoft.schemas.office.powerpoint.CTEmpty',
+  'com.microsoft.schemas.office.powerpoint.CTRel',
+  'com.microsoft.schemas.office.visio.x2012.main.AttachedToolbarsType',
+  'com.microsoft.schemas.office.visio.x2012.main.ColorsType',
+  'com.microsoft.schemas.office.visio.x2012.main.ConnectType',
+  'com.microsoft.schemas.office.visio.x2012.main.ConnectsType',
+  'com.microsoft.schemas.office.visio.x2012.main.CpType',
+  'com.microsoft.schemas.office.visio.x2012.main.CustomMenusFileType',
+  'com.microsoft.schemas.office.visio.x2012.main.CustomToolbarsFileType',
+  'com.microsoft.schemas.office.visio.x2012.main.DataType',
+  'com.microsoft.schemas.office.visio.x2012.main.DocumentSheetType',
+  'com.microsoft.schemas.office.visio.x2012.main.DynamicGridEnabledType',
+  'com.microsoft.schemas.office.visio.x2012.main.EventListType',
+  'com.microsoft.schemas.office.visio.x2012.main.FaceNamesType',
+  'com.microsoft.schemas.office.visio.x2012.main.FldType',
+  'com.microsoft.schemas.office.visio.x2012.main.ForeignDataType',
+  'com.microsoft.schemas.office.visio.x2012.main.GlueSettingsType',
+  'com.microsoft.schemas.office.visio.x2012.main.HeaderFooterType',
+  'com.microsoft.schemas.office.visio.x2012.main.IconType',
+  'com.microsoft.schemas.office.visio.x2012.main.MasterShortcutType',
+  'com.microsoft.schemas.office.visio.x2012.main.PpType',
+  'com.microsoft.schemas.office.visio.x2012.main.ProtectBkgndsType',
+  'com.microsoft.schemas.office.visio.x2012.main.ProtectMastersType',
+  'com.microsoft.schemas.office.visio.x2012.main.ProtectShapesType',
+  'com.microsoft.schemas.office.visio.x2012.main.ProtectStylesType',
+  'com.microsoft.schemas.office.visio.x2012.main.PublishSettingsType',
+  'com.microsoft.schemas.office.visio.x2012.main.RefByType',
+  'com.microsoft.schemas.office.visio.x2012.main.SnapAnglesType',
+  'com.microsoft.schemas.office.visio.x2012.main.SnapExtensionsType',
+  'com.microsoft.schemas.office.visio.x2012.main.SnapSettingsType',
+  'com.microsoft.schemas.office.visio.x2012.main.TpType',
+  'com.microsoft.schemas.office.visio.x2012.main.TriggerType',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.CellTypeImpl$1RefByList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.MastersTypeImpl$1MasterList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.MastersTypeImpl$1MasterShortcutList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.PagesTypeImpl$1PageList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.RowTypeImpl$1CellList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.RowTypeImpl$1TriggerList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.SectionTypeImpl$1CellList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.SectionTypeImpl$1RowList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.SectionTypeImpl$1TriggerList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.ShapesTypeImpl$1ShapeList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.SheetTypeImpl$1CellList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.SheetTypeImpl$1SectionList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.SheetTypeImpl$1TriggerList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.StyleSheetsTypeImpl$1StyleSheetList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.TextTypeImpl$1CpList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.TextTypeImpl$1FldList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.TextTypeImpl$1PpList',
+  'com.microsoft.schemas.office.visio.x2012.main.impl.TextTypeImpl$1TpList',
+  'com.microsoft.schemas.office.word.CTAnchorLock',
+  'com.microsoft.schemas.office.word.CTBorder',
+  'com.microsoft.schemas.office.word.CTWrap',
+  'com.microsoft.schemas.office.x2006.digsig.STPositiveInteger',
+  'com.microsoft.schemas.office.x2006.digsig.STSignatureComments',
+  'com.microsoft.schemas.office.x2006.digsig.STSignatureProviderUrl',
+  'com.microsoft.schemas.office.x2006.digsig.STSignatureText',
+  'com.microsoft.schemas.office.x2006.digsig.STSignatureType',
+  'com.microsoft.schemas.office.x2006.digsig.STUniqueIdentifierWithBraces',
+  'com.microsoft.schemas.office.x2006.digsig.STVersion',
+  'com.microsoft.schemas.vml.CTArc',
+  'com.microsoft.schemas.vml.CTCurve',
+  'com.microsoft.schemas.vml.CTImage',
+  'com.microsoft.schemas.vml.CTImageData',
+  'com.microsoft.schemas.vml.CTLine',
+  'com.microsoft.schemas.vml.CTOval',
+  'com.microsoft.schemas.vml.CTPolyLine',
+  'com.microsoft.schemas.vml.CTRect',
+  'com.microsoft.schemas.vml.CTRoundRect',
+  'com.microsoft.schemas.vml.STEditAs',
+  'com.microsoft.schemas.vml.STFillMethod',
+  'com.microsoft.schemas.vml.STFillType',
+  'com.microsoft.schemas.vml.STImageAspect',
+  'com.microsoft.schemas.vml.STShadowType',
+  'com.microsoft.schemas.vml.STStrokeArrowLength',
+  'com.microsoft.schemas.vml.STStrokeArrowType',
+  'com.microsoft.schemas.vml.STStrokeArrowWidth',
+  'com.microsoft.schemas.vml.STStrokeEndCap',
+  'com.microsoft.schemas.vml.STStrokeLineStyle',
+  'com.microsoft.schemas.vml.STTrueFalseBlank',
+  'com.microsoft.schemas.vml.impl.CTFormulasImpl$1FList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1AnchorlockList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1ArcList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1BorderbottomList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1BorderleftList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1BorderrightList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1BordertopList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1CalloutList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1ClientDataList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1ClippathList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1CurveList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1DiagramList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1ExtrusionList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1FillList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1FormulasList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1GroupList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1HandlesList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1ImageList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1ImagedataList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1LineList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1LockList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1OvalList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1PathList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1PolylineList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1RectList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1RoundrectList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1ShadowList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1ShapeList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1ShapetypeList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1SignaturelineList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1SkewList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1StrokeList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1TextboxList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1TextdataList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1TextpathList',
+  'com.microsoft.schemas.vml.impl.CTGroupImpl$1WrapList',
+  'com.microsoft.schemas.vml.impl.CTHandlesImpl$1HList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1AnchorlockList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1BorderbottomList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1BorderleftList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1BorderrightList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1BordertopList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1CalloutList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1ClippathList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1ExtrusionList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1FillList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1FormulasList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1HandlesList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1ImagedataList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1InkList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1IscommentList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1LockList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1PathList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1ShadowList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1SignaturelineList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1SkewList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1StrokeList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1TextboxList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1TextdataList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1TextpathList',
+  'com.microsoft.schemas.vml.impl.CTShapeImpl$1WrapList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1AnchorlockList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1BorderbottomList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1BorderleftList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1BorderrightList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1BordertopList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1CalloutList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1ClientDataList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1ClippathList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1ExtrusionList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1FillList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1FormulasList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1HandlesList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1ImagedataList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1LockList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1PathList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1ShadowList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1SignaturelineList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1SkewList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1StrokeList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1TextboxList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1TextdataList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1TextpathList',
+  'com.microsoft.schemas.vml.impl.CTShapetypeImpl$1WrapList',
+  'com.pff.PSTAttachment',
+  'com.pff.PSTFile',
+  'com.pff.PSTFolder',
+  'com.pff.PSTMessage',
+  'com.rometools.rome.feed.synd.SyndContent',
+  'com.rometools.rome.feed.synd.SyndEntry',
+  'com.rometools.rome.feed.synd.SyndFeed',
+  'com.rometools.rome.io.SyndFeedInput',
+  'com.uwyn.jhighlight.renderer.Renderer',
+  'com.uwyn.jhighlight.renderer.XhtmlRendererFactory',
+  'de.l3s.boilerpipe.BoilerpipeExtractor',
+  'de.l3s.boilerpipe.document.TextBlock',
+  'de.l3s.boilerpipe.document.TextDocument',
+  'de.l3s.boilerpipe.extractors.DefaultExtractor',
+  'de.l3s.boilerpipe.sax.BoilerpipeHTMLContentHandler',
+  'javax.mail.BodyPart',
+  'javax.mail.Header',
+  'javax.mail.Message$RecipientType',
+  'javax.mail.MessagingException',
+  'javax.mail.Multipart',
+  'javax.mail.Part',
+  'javax.mail.Session',
+  'javax.mail.Transport',
+  'javax.mail.internet.ContentType',
+  'javax.mail.internet.InternetAddress',
+  'javax.mail.internet.InternetHeaders',
+  'javax.mail.internet.MimeBodyPart',
+  'javax.mail.internet.MimeMessage',
+  'javax.mail.internet.MimeMultipart',
+  'javax.mail.internet.MimePart',
+  'javax.mail.internet.SharedInputStream',
+  'javax.servlet.ServletContextEvent',
+  'javax.servlet.ServletContextListener',
+  'javax.ws.rs.core.Response',
+  'junit.framework.TestCase',
+  'opennlp.tools.namefind.NameFinderME',
+  'opennlp.tools.namefind.TokenNameFinderModel',
+  'opennlp.tools.util.Span',
+  'org.apache.avalon.framework.logger.Logger',
+  'org.apache.commons.csv.CSVFormat',
+  'org.apache.commons.csv.CSVParser',
+  'org.apache.commons.csv.CSVRecord',
+  'org.apache.commons.exec.CommandLine',
+  'org.apache.commons.exec.DefaultExecutor',
+  'org.apache.commons.exec.ExecuteWatchdog',
+  'org.apache.commons.exec.PumpStreamHandler',
+  'org.apache.commons.exec.environment.EnvironmentUtils',
+  'org.apache.ctakes.typesystem.type.refsem.UmlsConcept',
+  'org.apache.ctakes.typesystem.type.textsem.IdentifiedAnnotation',
+  'org.apache.cxf.jaxrs.client.WebClient',
+  'org.apache.cxf.jaxrs.ext.multipart.Attachment',
+  'org.apache.cxf.jaxrs.ext.multipart.ContentDisposition',
+  'org.apache.cxf.jaxrs.ext.multipart.MultipartBody',
+  'org.apache.http.HttpEntity',
+  'org.apache.http.HttpResponse',
+  'org.apache.http.StatusLine',
+  'org.apache.http.client.HttpClient',
+  'org.apache.http.client.methods.HttpGet',
+  'org.apache.http.client.utils.URIBuilder',
+  'org.apache.http.impl.client.DefaultHttpClient',
+  'org.apache.james.mime4j.MimeException',
+  'org.apache.james.mime4j.codec.DecodeMonitor',
+  'org.apache.james.mime4j.codec.DecoderUtil',
+  'org.apache.james.mime4j.dom.FieldParser',
+  'org.apache.james.mime4j.dom.address.Address',
+  'org.apache.james.mime4j.dom.address.AddressList',
+  'org.apache.james.mime4j.dom.address.Mailbox',
+  'org.apache.james.mime4j.dom.address.MailboxList',
+  'org.apache.james.mime4j.dom.field.AddressListField',
+  'org.apache.james.mime4j.dom.field.DateTimeField',
+  'org.apache.james.mime4j.dom.field.MailboxListField',
+  'org.apache.james.mime4j.dom.field.ParsedField',
+  'org.apache.james.mime4j.dom.field.UnstructuredField',
+  'org.apache.james.mime4j.field.LenientFieldParser',
+  'org.apache.james.mime4j.parser.ContentHandler',
+  'org.apache.james.mime4j.parser.MimeStreamParser',
+  'org.apache.james.mime4j.stream.BodyDescriptor',
+  'org.apache.james.mime4j.stream.Field',
+  'org.apache.james.mime4j.stream.MimeConfig',
+  'org.apache.jcp.xml.dsig.internal.dom.DOMDigestMethod',
+  'org.apache.jcp.xml.dsig.internal.dom.DOMKeyInfo',
+  'org.apache.jcp.xml.dsig.internal.dom.DOMReference',
+  'org.apache.jcp.xml.dsig.internal.dom.DOMSignedInfo',
+  'org.apache.log.Hierarchy',
+  'org.apache.log.Logger',
+  'org.apache.pdfbox.tools.imageio.ImageIOUtil',
+  'org.apache.sis.internal.util.CheckedArrayList',
+  'org.apache.sis.internal.util.CheckedHashSet',
+  'org.apache.sis.metadata.iso.DefaultMetadata',
+  'org.apache.sis.metadata.iso.DefaultMetadataScope',
+  'org.apache.sis.metadata.iso.constraint.DefaultLegalConstraints',
+  'org.apache.sis.metadata.iso.extent.DefaultGeographicBoundingBox',
+  'org.apache.sis.metadata.iso.extent.DefaultGeographicDescription',
+  'org.apache.sis.metadata.iso.identification.DefaultDataIdentification',
+  'org.apache.sis.storage.DataStore',
+  'org.apache.sis.storage.DataStores',
+  'org.apache.sis.util.collection.CodeListSet',
+  'org.apache.tools.ant.BuildException',
+  'org.apache.tools.ant.FileScanner',
+  'org.apache.tools.ant.Project',
+  'org.apache.tools.ant.taskdefs.Jar',
+  'org.apache.tools.ant.taskdefs.Javac',
+  'org.apache.tools.ant.taskdefs.MatchingTask',
+  'org.apache.tools.ant.types.FileSet',
+  'org.apache.tools.ant.types.Path$PathElement',
+  'org.apache.tools.ant.types.Path',
+  'org.apache.tools.ant.types.Reference',
+  'org.apache.uima.UIMAFramework',
+  'org.apache.uima.analysis_engine.AnalysisEngine',
+  'org.apache.uima.cas.Type',
+  'org.apache.uima.cas.impl.XCASSerializer',
+  'org.apache.uima.cas.impl.XmiCasSerializer',
+  'org.apache.uima.cas.impl.XmiSerializationSharedData',
+  'org.apache.uima.fit.util.JCasUtil',
+  'org.apache.uima.jcas.JCas',
+  'org.apache.uima.jcas.cas.FSArray',
+  'org.apache.uima.util.XMLInputSource',
+  'org.apache.uima.util.XMLParser',
+  'org.apache.uima.util.XmlCasSerializer',
+  'org.apache.xml.security.Init',
+  'org.apache.xml.security.c14n.Canonicalizer',
+  'org.apache.xml.security.utils.Base64',
+  'org.etsi.uri.x01903.v13.AnyType',
+  'org.etsi.uri.x01903.v13.ClaimedRolesListType',
+  'org.etsi.uri.x01903.v13.CounterSignatureType',
+  'org.etsi.uri.x01903.v13.DataObjectFormatType$Factory',
+  'org.etsi.uri.x01903.v13.DataObjectFormatType',
+  'org.etsi.uri.x01903.v13.IdentifierType',
+  'org.etsi.uri.x01903.v13.IncludeType',
+  'org.etsi.uri.x01903.v13.ObjectIdentifierType',
+  'org.etsi.uri.x01903.v13.OtherCertStatusRefsType',
+  'org.etsi.uri.x01903.v13.OtherCertStatusValuesType',
+  'org.etsi.uri.x01903.v13.ReferenceInfoType',
+  'org.etsi.uri.x01903.v13.SigPolicyQualifiersListType',
+  'org.etsi.uri.x01903.v13.SignaturePolicyIdType',
+  'org.etsi.uri.x01903.v13.SignatureProductionPlaceType',
+  'org.etsi.uri.x01903.v13.SignedDataObjectPropertiesType',
+  'org.etsi.uri.x01903.v13.SignerRoleType',
+  'org.etsi.uri.x01903.v13.UnsignedDataObjectPropertiesType',
+  'org.etsi.uri.x01903.v13.impl.CRLRefsTypeImpl$1CRLRefList',
+  'org.etsi.uri.x01903.v13.impl.CRLValuesTypeImpl$1EncapsulatedCRLValueList',
+  'org.etsi.uri.x01903.v13.impl.CertIDListTypeImpl$1CertList',
+  'org.etsi.uri.x01903.v13.impl.CertificateValuesTypeImpl$1EncapsulatedX509CertificateList',
+  'org.etsi.uri.x01903.v13.impl.CertificateValuesTypeImpl$1OtherCertificateList',
+  'org.etsi.uri.x01903.v13.impl.GenericTimeStampTypeImpl$1EncapsulatedTimeStampList',
+  'org.etsi.uri.x01903.v13.impl.GenericTimeStampTypeImpl$1IncludeList',
+  'org.etsi.uri.x01903.v13.impl.GenericTimeStampTypeImpl$1ReferenceInfoList',
+  'org.etsi.uri.x01903.v13.impl.GenericTimeStampTypeImpl$1XMLTimeStampList',
+  'org.etsi.uri.x01903.v13.impl.OCSPRefsTypeImpl$1OCSPRefList',
+  'org.etsi.uri.x01903.v13.impl.OCSPValuesTypeImpl$1EncapsulatedOCSPValueList',
+  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1ArchiveTimeStampList',
+  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1AttrAuthoritiesCertValuesList',
+  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1AttributeCertificateRefsList',
+  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1AttributeRevocationRefsList',
+  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1AttributeRevocationValuesList',
+  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1CertificateValuesList',
+  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1CompleteCertificateRefsList',
+  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1CompleteRevocationRefsList',
+  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1CounterSignatureList',
+  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1RefsOnlyTimeStampList',
+  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1RevocationValuesList',
+  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1SigAndRefsTimeStampList',
+  'org.etsi.uri.x01903.v13.impl.UnsignedSignaturePropertiesTypeImpl$1SignatureTimeStampList',
+  'org.etsi.uri.x01903.v14.ValidationDataType$Factory',
+  'org.etsi.uri.x01903.v14.ValidationDataType',
+  'org.json.JSONArray',
+  'org.json.JSONObject',
+  'org.json.XML',
+  'org.json.simple.JSONArray',
+  'org.json.simple.JSONObject',
+  'org.json.simple.parser.JSONParser',
+  'org.junit.Test',
+  'org.junit.internal.TextListener',
+  'org.junit.runner.JUnitCore',
   'org.junit.runner.Result',
-  'org.objectweb.asm.AnnotationVisitor', 
-  'org.objectweb.asm.Attribute', 
-  'org.objectweb.asm.ClassReader', 
-  'org.objectweb.asm.ClassVisitor', 
-  'org.objectweb.asm.FieldVisitor', 
-  'org.objectweb.asm.MethodVisitor', 
-  'org.objectweb.asm.Type', 
-  'org.opengis.metadata.Identifier', 
-  'org.opengis.metadata.citation.Address', 
-  'org.opengis.metadata.citation.Citation', 
-  'org.opengis.metadata.citation.CitationDate', 
-  'org.opengis.metadata.citation.Contact', 
-  'org.opengis.metadata.citation.DateType', 
-  'org.opengis.metadata.citation.OnLineFunction', 
-  'org.opengis.metadata.citation.OnlineResource', 
-  'org.opengis.metadata.citation.ResponsibleParty', 
-  'org.opengis.metadata.citation.Role', 
-  'org.opengis.metadata.constraint.Restriction', 
-  'org.opengis.metadata.distribution.DigitalTransferOptions', 
-  'org.opengis.metadata.distribution.Distribution', 
-  'org.opengis.metadata.distribution.Distributor', 
-  'org.opengis.metadata.distribution.Format', 
-  'org.opengis.metadata.extent.Extent', 
-  'org.opengis.metadata.identification.Identification', 
-  'org.opengis.metadata.identification.KeywordType', 
-  'org.opengis.metadata.identification.Keywords', 
-  'org.opengis.metadata.identification.Progress', 
-  'org.opengis.metadata.identification.TopicCategory', 
-  'org.opengis.metadata.maintenance.ScopeCode', 
-  'org.opengis.util.InternationalString', 
+  'org.objectweb.asm.AnnotationVisitor',
+  'org.objectweb.asm.Attribute',
+  'org.objectweb.asm.ClassReader',
+  'org.objectweb.asm.ClassVisitor',
+  'org.objectweb.asm.FieldVisitor',
+  'org.objectweb.asm.MethodVisitor',
+  'org.objectweb.asm.Type',
+  'org.opengis.metadata.Identifier',
+  'org.opengis.metadata.citation.Address',
+  'org.opengis.metadata.citation.Citation',
+  'org.opengis.metadata.citation.CitationDate',
+  'org.opengis.metadata.citation.Contact',
+  'org.opengis.metadata.citation.DateType',
+  'org.opengis.metadata.citation.OnLineFunction',
+  'org.opengis.metadata.citation.OnlineResource',
+  'org.opengis.metadata.citation.ResponsibleParty',
+  'org.opengis.metadata.citation.Role',
+  'org.opengis.metadata.constraint.Restriction',
+  'org.opengis.metadata.distribution.DigitalTransferOptions',
+  'org.opengis.metadata.distribution.Distribution',
+  'org.opengis.metadata.distribution.Distributor',
+  'org.opengis.metadata.distribution.Format',
+  'org.opengis.metadata.extent.Extent',
+  'org.opengis.metadata.identification.Identification',
+  'org.opengis.metadata.identification.KeywordType',
+  'org.opengis.metadata.identification.Keywords',
+  'org.opengis.metadata.identification.Progress',
+  'org.opengis.metadata.identification.TopicCategory',
+  'org.opengis.metadata.maintenance.ScopeCode',
+  'org.opengis.util.InternationalString',
 
   // Missing openxml schema classes are explained by the fact we use the smaller jar:
-  // "The full jar of all of the schemas is ooxml-schemas-xx.jar, and it is currently around 15mb. 
-  //  The smaller poi-ooxml-schemas jar is only about 4mb. 
+  // "The full jar of all of the schemas is ooxml-schemas-xx.jar, and it is currently around 15mb.
+  //  The smaller poi-ooxml-schemas jar is only about 4mb.
   //  This latter jar file only contains the typically used parts though."
   // http://poi.apache.org/faq.html#faq-N10025
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTArea3DChart', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTAreaChart', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTAxisUnit', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTBar3DChart', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTBarChart', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTBubbleChart', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTChartLines', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTDLbls', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTDPt', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTDTable', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTDateAx', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTDispBlanksAs', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTDispUnits', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTDoughnutChart', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTErrBars', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTExtensionList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTExternalData', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTFirstSliceAng', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTGrouping', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTLblAlgn', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTLblOffset', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTLegendEntry', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTLine3DChart', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTMarkerSize', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTMultiLvlStrRef', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTOfPieChart', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTPie3DChart', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTPivotFmts', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTPivotSource', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTProtection', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTRadarChart', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTRelId', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTSerAx', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTSkip', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTStockChart', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTStyle', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTSurface', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTSurface3DChart', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTSurfaceChart', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTTextLanguageID', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTTrendline', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTUpDownBars', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.CTView3D', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.STPageSetupOrientation', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTLegendImpl$1LegendEntryList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTLineChartImpl$1AxIdList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTLineChartImpl$1SerList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTLineSerImpl$1DPtList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTLineSerImpl$1TrendlineList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTNumDataImpl$1PtList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPieChartImpl$1SerList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPieSerImpl$1DPtList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1Area3DChartList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1AreaChartList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1Bar3DChartList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1BarChartList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1BubbleChartList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1CatAxList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1DateAxList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1DoughnutChartList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1Line3DChartList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1LineChartList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1OfPieChartList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1Pie3DChartList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1PieChartList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1RadarChartList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1ScatterChartList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1SerAxList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1StockChartList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1Surface3DChartList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1SurfaceChartList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1ValAxList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTScatterChartImpl$1AxIdList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTScatterChartImpl$1SerList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTScatterSerImpl$1DPtList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTScatterSerImpl$1ErrBarsList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTScatterSerImpl$1TrendlineList', 
-  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTStrDataImpl$1PtList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTAlphaBiLevelEffect', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTAlphaCeilingEffect', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTAlphaFloorEffect', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTAlphaInverseEffect', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTAlphaModulateEffect', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTAlphaReplaceEffect', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTAngle', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTAudioCD', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTAudioFile', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTBiLevelEffect', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTBlurEffect', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTCell3D', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTColorChangeEffect', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTColorReplaceEffect', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTColorSchemeList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTComplementTransform', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTConnectionSite', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTConnectorLocking', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTCustomColorList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTDashStopList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTDuotoneEffect', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTEffectContainer', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTEmbeddedWAVAudioFile', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTFillOverlayEffect', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTFlatText', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTGammaTransform', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTGlowEffect', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTGrayscaleEffect', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTGrayscaleTransform', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTGroupFillProperties', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTGroupLocking', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTHSLEffect', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTInnerShadowEffect', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTInverseGammaTransform', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTInverseTransform', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTLineJoinBevel', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTLuminanceEffect', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTObjectStyleDefaults', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTPath2DArcTo', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTPatternFillProperties', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTPolarAdjustHandle', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTPositiveFixedAngle', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTPresetShadowEffect', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTPresetTextShape', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTQuickTimeFile', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTReflectionEffect', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTScene3D', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTShape3D', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTShapeLocking', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTSoftEdgesEffect', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTSupplementalFont', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTTableBackgroundStyle', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTTablePartStyle', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTTextBlipBullet', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTTextBulletColorFollowText', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTTextBulletSizeFollowText', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTTextBulletTypefaceFollowText', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTTextUnderlineFillFollowText', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTTextUnderlineFillGroupWrapper', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTTextUnderlineLineFollowText', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTTileInfoProperties', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTTintEffect', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTVideoFile', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.CTXYAdjustHandle', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.STBlackWhiteMode', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.STBlipCompression', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.STFixedAngle', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.STGuid', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.STPanose', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.STPathFillMode', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.STRectAlignment', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.STTextColumnCount', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.STTextNonNegativePoint', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.STTextTabAlignType', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.STTileFlipMode', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTAdjustHandleListImpl$1AhPolarList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTAdjustHandleListImpl$1AhXYList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBackgroundFillStyleListImpl$1BlipFillList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBackgroundFillStyleListImpl$1GradFillList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBackgroundFillStyleListImpl$1GrpFillList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBackgroundFillStyleListImpl$1NoFillList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBackgroundFillStyleListImpl$1PattFillList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBackgroundFillStyleListImpl$1SolidFillList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1AlphaBiLevelList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1AlphaCeilingList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1AlphaFloorList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1AlphaInvList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1AlphaModFixList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1AlphaModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1AlphaReplList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1BiLevelList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1BlurList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1ClrChangeList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1ClrReplList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1DuotoneList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1FillOverlayList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1GraysclList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1HslList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1LumList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1TintList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTConnectionSiteListImpl$1CxnList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTEffectStyleListImpl$1EffectStyleList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTFillStyleListImpl$1BlipFillList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTFillStyleListImpl$1GradFillList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTFillStyleListImpl$1GrpFillList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTFillStyleListImpl$1NoFillList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTFillStyleListImpl$1PattFillList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTFillStyleListImpl$1SolidFillList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTFontCollectionImpl$1FontList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTGeomGuideListImpl$1GdList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTGradientStopListImpl$1GsList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1AlphaList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1AlphaModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1AlphaOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1BlueList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1BlueModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1BlueOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1CompList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1GammaList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1GrayList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1GreenList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1GreenModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1GreenOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1HueList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1HueModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1HueOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1InvGammaList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1InvList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1LumList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1LumModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1LumOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1RedList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1RedModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1RedOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1SatList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1SatModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1SatOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1ShadeList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1TintList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTLineStyleListImpl$1LnList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTOfficeArtExtensionListImpl$1ExtList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPath2DCubicBezierToImpl$1PtList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPath2DImpl$1ArcToList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPath2DImpl$1CloseList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPath2DImpl$1CubicBezToList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPath2DImpl$1LnToList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPath2DImpl$1MoveToList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPath2DImpl$1QuadBezToList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPath2DListImpl$1PathList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1AlphaList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1AlphaModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1AlphaOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1BlueList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1BlueModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1BlueOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1CompList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1GammaList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1GrayList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1GreenList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1GreenModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1GreenOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1HueList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1HueModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1HueOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1InvGammaList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1InvList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1LumList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1LumModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1LumOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1RedList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1RedModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1RedOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1SatList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1SatModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1SatOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1ShadeList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1TintList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1AlphaList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1AlphaModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1AlphaOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1BlueList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1BlueModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1BlueOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1CompList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1GammaList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1GrayList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1GreenList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1GreenModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1GreenOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1HueList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1HueModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1HueOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1InvGammaList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1InvList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1LumList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1LumModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1LumOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1RedList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1RedModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1RedOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1SatList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1SatModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1SatOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1ShadeList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1TintList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1AlphaList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1AlphaModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1AlphaOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1BlueList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1BlueModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1BlueOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1CompList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1GammaList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1GrayList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1GreenList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1GreenModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1GreenOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1HueList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1HueModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1HueOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1InvGammaList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1InvList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1LumList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1LumModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1LumOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1RedList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1RedModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1RedOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1SatList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1SatModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1SatOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1ShadeList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1TintList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1AlphaList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1AlphaModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1AlphaOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1BlueList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1BlueModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1BlueOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1CompList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1GammaList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1GrayList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1GreenList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1GreenModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1GreenOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1HueList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1HueModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1HueOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1InvGammaList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1InvList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1LumList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1LumModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1LumOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1RedList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1RedModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1RedOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1SatList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1SatModList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1SatOffList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1ShadeList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1TintList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTTableGridImpl$1GridColList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTTableImpl$1TrList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTTableRowImpl$1TcList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTTableStyleListImpl$1TblStyleList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTTextBodyImpl$1PList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTTextParagraphImpl$1BrList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTTextParagraphImpl$1FldList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTTextParagraphImpl$1RList', 
-  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTTextTabStopListImpl$1TabList', 
-  'org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.CTAbsoluteAnchor', 
-  'org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.impl.CTDrawingImpl$1AbsoluteAnchorList', 
-  'org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.impl.CTDrawingImpl$1OneCellAnchorList', 
-  'org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.impl.CTDrawingImpl$1TwoCellAnchorList', 
-  'org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.impl.CTGroupShapeImpl$1CxnSpList', 
-  'org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.impl.CTGroupShapeImpl$1GraphicFrameList', 
-  'org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.impl.CTGroupShapeImpl$1GrpSpList', 
-  'org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.impl.CTGroupShapeImpl$1PicList', 
-  'org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.impl.CTGroupShapeImpl$1SpList', 
-  'org.openxmlformats.schemas.drawingml.x2006.wordprocessingDrawing.CTEffectExtent', 
-  'org.openxmlformats.schemas.drawingml.x2006.wordprocessingDrawing.CTPosH', 
-  'org.openxmlformats.schemas.drawingml.x2006.wordprocessingDrawing.CTPosV', 
-  'org.openxmlformats.schemas.drawingml.x2006.wordprocessingDrawing.CTWrapNone', 
-  'org.openxmlformats.schemas.drawingml.x2006.wordprocessingDrawing.CTWrapSquare', 
-  'org.openxmlformats.schemas.drawingml.x2006.wordprocessingDrawing.CTWrapThrough', 
-  'org.openxmlformats.schemas.drawingml.x2006.wordprocessingDrawing.CTWrapTight', 
-  'org.openxmlformats.schemas.drawingml.x2006.wordprocessingDrawing.CTWrapTopBottom', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.CTArray', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.CTCf', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.CTEmpty', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.CTNull', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.CTVstream', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.STCy', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.STError', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.STVectorBaseType', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1BoolList', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1BstrList', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1CfList', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1ClsidList', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1CyList', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1DateList', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1ErrorList', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1FiletimeList', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1I1List', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1I2List', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1I4List', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1I8List', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1LpstrList', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1LpwstrList', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1R4List', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1R8List', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1Ui1List', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1Ui2List', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1Ui4List', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1Ui8List', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1VariantList', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2BoolList', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2BstrList', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2ClsidList', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2CyList', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2DateList', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2ErrorList', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2FiletimeList', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2I1List', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2I2List', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2I4List', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2I8List', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2LpstrList', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2LpwstrList', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2R4List', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2R8List', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2Ui1List', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2Ui2List', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2Ui4List', 
-  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2Ui8List', 
-  'org.openxmlformats.schemas.officeDocument.x2006.math.CTAcc', 
-  'org.openxmlformats.schemas.officeDocument.x2006.math.CTBar', 
-  'org.openxmlformats.schemas.officeDocument.x2006.math.CTBorderBox', 
-  'org.openxmlformats.schemas.officeDocument.x2006.math.CTBox', 
-  'org.openxmlformats.schemas.officeDocument.x2006.math.CTD', 
-  'org.openxmlformats.schemas.officeDocument.x2006.math.CTEqArr', 
-  'org.openxmlformats.schemas.officeDocument.x2006.math.CTF', 
-  'org.openxmlformats.schemas.officeDocument.x2006.math.CTFunc', 
-  'org.openxmlformats.schemas.officeDocument.x2006.math.CTGroupChr', 
-  'org.openxmlformats.schemas.officeDocument.x2006.math.CTLimLow', 
-  'org.openxmlformats.schemas.officeDocument.x2006.math.CTLimUpp', 
-  'org.openxmlformats.schemas.officeDocument.x2006.math.CTM', 
-  'org.openxmlformats.schemas.officeDocument.x2006.math.CTMathPr', 
-  'org.openxmlformats.schemas.officeDocument.x2006.math.CTNary', 
-  'org.openxmlformats.schemas.officeDocument.x2006.math.CTOMath', 
-  'org.openxmlformats.schemas.officeDocument.x2006.math.CTOMathPara', 
-  'org.openxmlformats.schemas.officeDocument.x2006.math.CTPhant', 
-  'org.openxmlformats.schemas.officeDocument.x2006.math.CTR', 
-  'org.openxmlformats.schemas.officeDocument.x2006.math.CTRad', 
-  'org.openxmlformats.schemas.officeDocument.x2006.math.CTSPre', 
-  'org.openxmlformats.schemas.officeDocument.x2006.math.CTSSub', 
-  'org.openxmlformats.schemas.officeDocument.x2006.math.CTSSubSup', 
-  'org.openxmlformats.schemas.officeDocument.x2006.math.CTSSup', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.CTControlList', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.CTCustomShowList', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.CTCustomerData', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.CTEmbeddedFontList', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.CTExtensionList', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.CTExtensionListModify', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.CTHandoutMasterIdList', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.CTHeaderFooter', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.CTKinsoku', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.CTModifyVerifier', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.CTPhotoAlbum', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.CTSlideLayoutIdList', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.CTSlideTiming', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.CTSlideTransition', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.CTSmartTags', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.STBookmarkIdSeed', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.STDirection', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.STIndex', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.STPlaceholderSize', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.STSlideSizeType', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.impl.CTCommentAuthorListImpl$1CmAuthorList', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.impl.CTCommentListImpl$1CmList', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.impl.CTCustomerDataListImpl$1CustDataList', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.impl.CTGroupShapeImpl$1GraphicFrameList', 
-  'org.openxmlformats.schemas.presentationml.x2006.main.impl.CTGroupShapeImpl$1PicList', 
-  'org.openxmlformats.schemas.schemaLibrary.x2006.main.CTSchemaLibrary', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTAutoSortScope', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTBoolean', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTCacheHierarchies', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTCalculatedItems', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTCalculatedMembers', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTCellStyles', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTCellWatches', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTChartFormats', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTChartsheetPr', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTChartsheetProtection', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTChartsheetViews', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTColHierarchiesUsage', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTColItems', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTColors', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTConditionalFormats', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTConsolidation', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTControls', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTCsPageSetup', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTCustomChartsheetViews', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTCustomProperties', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTCustomSheetViews', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTCustomWorkbookViews', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTDataBinding', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTDataConsolidate', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTDateTime', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTDdeLink', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTDimensions', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTError', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTExtensionList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTExternalSheetDataSet', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTFieldGroup', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTFileRecoveryPr', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTFileSharing', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTFileVersion', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTFilterColumn', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTFormats', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTFunctionGroups', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTGradientFill', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTIgnoredErrors', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTMeasureDimensionMaps', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTMeasureGroups', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTMissing', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTNumber', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTOleLink', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTOleObjects', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTOleSize', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPCDKPIs', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPhoneticRun', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPivotFilters', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPivotHierarchies', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPivotSelection', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTProtectedRanges', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTRecord', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTRowHierarchiesUsage', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTRowItems', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTScenarios', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTSheetBackgroundPicture', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTSmartTagPr', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTSmartTagTypes', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTSmartTags', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTSortState', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTString', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTTableFormula', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTTableStyleInfo', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTTableStyles', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTTupleCache', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTWebPublishItems', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTWebPublishObjects', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTWebPublishing', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTX', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STCellSpans', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STDataValidationImeMode', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STFieldSortType', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STGuid', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STObjects', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STPhoneticAlignment', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STPhoneticType', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STPrintError', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STRefMode', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STSheetViewType', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STShowDataAs', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STTableType', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STTimePeriod', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STTotalsRowFunction', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STUpdateLinks', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STVisibility', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTAuthorsImpl$1AuthorList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTAuthorsImpl$2AuthorList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTAutoFilterImpl$1FilterColumnList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTBookViewsImpl$1WorkbookViewList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTBordersImpl$1BorderList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTCacheFieldImpl$1MpMapList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTCacheFieldsImpl$1CacheFieldList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTCalcChainImpl$1CList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTCellStyleXfsImpl$1XfList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTCellXfsImpl$1XfList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTCfRuleImpl$1FormulaList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTCfRuleImpl$2FormulaList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTColFieldsImpl$1FieldList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTColorScaleImpl$1CfvoList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTColorScaleImpl$1ColorList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTColsImpl$1ColList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTCommentListImpl$1CommentList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTConditionalFormattingImpl$1CfRuleList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTDataBarImpl$1CfvoList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTDataValidationsImpl$1DataValidationList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTDxfsImpl$1DxfList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTExternalDefinedNamesImpl$1DefinedNameList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTExternalReferencesImpl$1ExternalReferenceList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTExternalSheetNamesImpl$1SheetNameList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFillsImpl$1FillList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1BList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1CharsetList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1ColorList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1CondenseList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1ExtendList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1FamilyList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1IList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1NameList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1OutlineList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1SchemeList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1ShadowList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1StrikeList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1SzList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1UList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1VertAlignList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontsImpl$1FontList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTHyperlinksImpl$1HyperlinkList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTIconSetImpl$1CfvoList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTItemsImpl$1ItemList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTMapInfoImpl$1MapList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTMapInfoImpl$1SchemaList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTMergeCellsImpl$1MergeCellList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTNumFmtsImpl$1NumFmtList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTPageBreakImpl$1BrkList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTPageFieldsImpl$1PageFieldList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTPivotCacheRecordsImpl$1RList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTPivotCachesImpl$1PivotCacheList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTPivotFieldsImpl$1PivotFieldList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1BList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1CharsetList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1ColorList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1CondenseList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1ExtendList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1FamilyList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1IList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1OutlineList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1RFontList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1SchemeList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1ShadowList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1StrikeList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1SzList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1UList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1VertAlignList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRowFieldsImpl$1FieldList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRowImpl$1CList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRstImpl$1RList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRstImpl$1RPhList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSharedItemsImpl$1BList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSharedItemsImpl$1DList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSharedItemsImpl$1EList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSharedItemsImpl$1MList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSharedItemsImpl$1NList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSharedItemsImpl$1SList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSheetDataImpl$1RowList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSheetViewImpl$1PivotSelectionList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSheetViewImpl$1SelectionList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSheetViewsImpl$1SheetViewList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSingleXmlCellsImpl$1SingleXmlCellList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSstImpl$1SiList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTTableColumnsImpl$1TableColumnList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTTablePartsImpl$1TablePartList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTWorkbookImpl$1FileRecoveryPrList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTWorksheetImpl$1ColsList', 
-  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTWorksheetImpl$1ConditionalFormattingList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTAltChunk', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTAttr', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTBackground', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTCaptions', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTCellMergeTrackChange', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTCharacterSpacing', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTCnf', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTColorSchemeMapping', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTColumns', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTCompat', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTControl', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTCustomXmlBlock', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTCustomXmlCell', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTCustomXmlRow', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTCustomXmlRun', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTDataBinding', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTDocGrid', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTDocRsids', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTDocType', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTDocVars', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTEastAsianLayout', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTEdnDocProps', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTEdnProps', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTEm', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTFFDDList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTFFHelpText', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTFFName', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTFFStatusText', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTFFTextInput', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTFitText', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTFramePr', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTFtnDocProps', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTFtnProps', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTHighlight', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTKinsoku', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTLevelSuffix', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTLineNumber', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTLock', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTLongHexNumber', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTLvlLegacy', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTMacroName', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTMailMerge', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTMultiLevelType', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTNumPicBullet', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTPPrChange', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTPageBorders', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTPageMar', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTPageNumber', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTPageSz', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTPaperSource', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTParaRPrChange', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTPerm', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTPermStart', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTPlaceholder', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTProof', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTRPrChange', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTReadingModeInkLockDown', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTRuby', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTSaveThroughXslt', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTSdtComboBox', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTSdtDate', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTSdtDropDownList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTSdtRow', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTSdtText', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTSectPrChange', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTSectType', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTShapeDefaults', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTShortHexNumber', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTSignedTwipsMeasure', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTSmartTagType', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTblGridChange', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTblLayoutType', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTblOverlap', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTblPPr', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTblPrChange', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTblPrExChange', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTblStylePr', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTcMar', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTcPrChange', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTextDirection', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTextEffect', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTextScale', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTextboxTightWrap', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTrPrChange', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTrackChangeNumbering', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTrackChangesView', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTwipsMeasure', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTView', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTWriteProtection', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTWritingStyle', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STDateTime', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STDisplacedByCustomXml', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STHeightRule', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STHint', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STPTabAlignment', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STPTabLeader', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STPTabRelativeTo', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STProofErr', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STShortHexNumber', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STThemeColor', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STUcharHexNumber', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STZoom', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTAbstractNumImpl$1LvlList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1AltChunkList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1BookmarkEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1BookmarkStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1CommentRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1CommentRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1CustomXmlDelRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1CustomXmlDelRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1CustomXmlInsRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1CustomXmlInsRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1CustomXmlList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1CustomXmlMoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1CustomXmlMoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1CustomXmlMoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1CustomXmlMoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1DelList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1InsList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1MoveFromList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1MoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1MoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1MoveToList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1MoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1MoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1OMathList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1OMathParaList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1PList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1PermEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1PermStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1ProofErrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1SdtList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1TblList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1AltChunkList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1BookmarkEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1BookmarkStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1CommentRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1CommentRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1CustomXmlDelRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1CustomXmlDelRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1CustomXmlInsRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1CustomXmlInsRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1CustomXmlList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1CustomXmlMoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1CustomXmlMoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1CustomXmlMoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1CustomXmlMoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1DelList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1InsList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1MoveFromList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1MoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1MoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1MoveToList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1MoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1MoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1OMathList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1OMathParaList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1PList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1PermEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1PermStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1ProofErrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1SdtList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1TblList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentsImpl$1CommentList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTDrawingImpl$1AnchorList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTDrawingImpl$1InlineList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTEndnotesImpl$1EndnoteList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFFDataImpl$1CalcOnExitList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFFDataImpl$1DdListList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFFDataImpl$1EnabledList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFFDataImpl$1EntryMacroList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFFDataImpl$1ExitMacroList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFFDataImpl$1HelpTextList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFFDataImpl$1NameList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFFDataImpl$1StatusTextList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFFDataImpl$1TextInputList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFootnotesImpl$1FootnoteList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1AltChunkList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1BookmarkEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1BookmarkStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1CommentRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1CommentRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1CustomXmlDelRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1CustomXmlDelRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1CustomXmlInsRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1CustomXmlInsRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1CustomXmlList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1CustomXmlMoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1CustomXmlMoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1CustomXmlMoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1CustomXmlMoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1DelList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1InsList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1MoveFromList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1MoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1MoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1MoveToList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1MoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1MoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1OMathList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1OMathParaList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1PList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1PermEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1PermStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1ProofErrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1SdtList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1TblList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1AltChunkList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1BookmarkEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1BookmarkStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1CommentRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1CommentRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1CustomXmlDelRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1CustomXmlDelRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1CustomXmlInsRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1CustomXmlInsRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1CustomXmlList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1CustomXmlMoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1CustomXmlMoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1CustomXmlMoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1CustomXmlMoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1DelList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1InsList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1MoveFromList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1MoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1MoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1MoveToList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1MoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1MoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1OMathList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1OMathParaList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1PList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1PermEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1PermStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1ProofErrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1SdtList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1TblList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1BookmarkEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1BookmarkStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1CommentRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1CommentRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1CustomXmlDelRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1CustomXmlDelRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1CustomXmlInsRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1CustomXmlInsRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1CustomXmlList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1CustomXmlMoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1CustomXmlMoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1CustomXmlMoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1CustomXmlMoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1DelList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1FldSimpleList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1HyperlinkList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1InsList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1MoveFromList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1MoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1MoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1MoveToList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1MoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1MoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1OMathList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1OMathParaList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1PermEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1PermStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1ProofErrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1RList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1SdtList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1SmartTagList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1SubDocList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTLatentStylesImpl$1LsdExceptionList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTNumImpl$1LvlOverrideList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTNumberingImpl$1AbstractNumList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTNumberingImpl$1NumList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTNumberingImpl$1NumPicBulletList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1BookmarkEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1BookmarkStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CommentRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CommentRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CustomXmlDelRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CustomXmlDelRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CustomXmlInsRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CustomXmlInsRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CustomXmlList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CustomXmlMoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CustomXmlMoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CustomXmlMoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CustomXmlMoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1DelList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1FldSimpleList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1HyperlinkList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1InsList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1MoveFromList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1MoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1MoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1MoveToList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1MoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1MoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1OMathList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1OMathParaList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1PermEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1PermStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1ProofErrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1RList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1SdtList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1SmartTagList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1SubDocList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1AnnotationRefList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1BrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1CommentReferenceList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1ContinuationSeparatorList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1CrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1DayLongList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1DayShortList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1DelInstrTextList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1DelTextList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1DrawingList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1EndnoteRefList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1EndnoteReferenceList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1FldCharList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1FootnoteRefList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1FootnoteReferenceList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1InstrTextList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1LastRenderedPageBreakList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1MonthLongList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1MonthShortList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1NoBreakHyphenList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1ObjectList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1PgNumList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1PictList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1PtabList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1RubyList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1SeparatorList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1SoftHyphenList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1SymList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1TList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1TabList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1YearLongList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1YearShortList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1BookmarkEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1BookmarkStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1CommentRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1CommentRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1CustomXmlDelRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1CustomXmlDelRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1CustomXmlInsRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1CustomXmlInsRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1CustomXmlList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1CustomXmlMoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1CustomXmlMoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1CustomXmlMoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1CustomXmlMoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1DelList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1InsList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1MoveFromList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1MoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1MoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1MoveToList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1MoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1MoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1OMathList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1OMathParaList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1PermEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1PermStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1ProofErrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1SdtList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1TcList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1AccList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1BarList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1BookmarkEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1BookmarkStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1BorderBoxList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1BoxList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1CommentRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1CommentRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1CustomXmlDelRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1CustomXmlDelRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1CustomXmlInsRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1CustomXmlInsRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1CustomXmlList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1CustomXmlMoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1CustomXmlMoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1CustomXmlMoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1CustomXmlMoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1DList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1DelList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1EqArrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1FList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1FuncList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1GroupChrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1InsList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1LimLowList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1LimUppList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1MList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1MoveFromList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1MoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1MoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1MoveToList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1MoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1MoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1NaryList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1OMathList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1OMathParaList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1PermEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1PermStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1PhantList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1ProofErrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1R2List', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1RList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1RadList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1SPreList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1SSubList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1SSubSupList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1SSupList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1SdtList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1SmartTagList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1BookmarkEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1BookmarkStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1CommentRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1CommentRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1CustomXmlDelRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1CustomXmlDelRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1CustomXmlInsRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1CustomXmlInsRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1CustomXmlList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1CustomXmlMoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1CustomXmlMoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1CustomXmlMoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1CustomXmlMoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1DelList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1InsList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1MoveFromList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1MoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1MoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1MoveToList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1MoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1MoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1OMathList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1OMathParaList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1PList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1PermEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1PermStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1ProofErrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1SdtList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1TblList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1BookmarkEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1BookmarkStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1CommentRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1CommentRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1CustomXmlDelRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1CustomXmlDelRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1CustomXmlInsRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1CustomXmlInsRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1CustomXmlList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1CustomXmlMoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1CustomXmlMoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1CustomXmlMoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1CustomXmlMoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1DelList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1InsList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1MoveFromList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1MoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1MoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1MoveToList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1MoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1MoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1OMathList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1OMathParaList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1PermEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1PermStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1ProofErrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1SdtList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1TcList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1BookmarkEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1BookmarkStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1CommentRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1CommentRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1CustomXmlDelRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1CustomXmlDelRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1CustomXmlInsRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1CustomXmlInsRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1CustomXmlList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1CustomXmlMoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1CustomXmlMoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1CustomXmlMoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1CustomXmlMoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1DelList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1FldSimpleList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1HyperlinkList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1InsList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1MoveFromList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1MoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1MoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1MoveToList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1MoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1MoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1OMathList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1OMathParaList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1PermEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1PermStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1ProofErrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1RList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1SdtList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1SmartTagList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1SubDocList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtEndPrImpl$1RPrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1AliasList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1BibliographyList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1CitationList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1ComboBoxList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1DataBindingList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1DateList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1DocPartListList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1DocPartObjList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1DropDownListList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1EquationList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1GroupList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1IdList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1LockList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1PictureList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1PlaceholderList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1RPrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1RichTextList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1ShowingPlcHdrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1TagList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1TemporaryList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1TextList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSectPrImpl$1FooterReferenceList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSectPrImpl$1HeaderReferenceList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSettingsImpl$1ActiveWritingStyleList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSettingsImpl$1AttachedSchemaList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSettingsImpl$1SmartTagTypeList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1BookmarkEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1BookmarkStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1CommentRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1CommentRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1CustomXmlDelRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1CustomXmlDelRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1CustomXmlInsRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1CustomXmlInsRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1CustomXmlList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1CustomXmlMoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1CustomXmlMoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1CustomXmlMoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1CustomXmlMoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1DelList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1FldSimpleList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1HyperlinkList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1InsList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1MoveFromList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1MoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1MoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1MoveToList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1MoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1MoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1OMathList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1OMathParaList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1PermEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1PermStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1ProofErrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1RList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1SdtList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1SmartTagList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1SubDocList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagPrImpl$1AttrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1BookmarkEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1BookmarkStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1CommentRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1CommentRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1CustomXmlDelRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1CustomXmlDelRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1CustomXmlInsRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1CustomXmlInsRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1CustomXmlList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1CustomXmlMoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1CustomXmlMoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1CustomXmlMoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1CustomXmlMoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1DelList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1FldSimpleList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1HyperlinkList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1InsList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1MoveFromList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1MoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1MoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1MoveToList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1MoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1MoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1OMathList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1OMathParaList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1PermEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1PermStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1ProofErrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1RList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1SdtList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1SmartTagList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1SubDocList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTStyleImpl$1TblStylePrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTStylesImpl$1StyleList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTabsImpl$1TabList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblGridBaseImpl$1GridColList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1BookmarkEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1BookmarkStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1CommentRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1CommentRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1CustomXmlDelRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1CustomXmlDelRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1CustomXmlInsRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1CustomXmlInsRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1CustomXmlList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1CustomXmlMoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1CustomXmlMoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1CustomXmlMoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1CustomXmlMoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1DelList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1InsList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1MoveFromList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1MoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1MoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1MoveToList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1MoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1MoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1OMathList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1OMathParaList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1PermEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1PermStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1ProofErrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1SdtList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1TrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1AltChunkList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1BookmarkEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1BookmarkStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1CommentRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1CommentRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1CustomXmlDelRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1CustomXmlDelRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1CustomXmlInsRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1CustomXmlInsRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1CustomXmlList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1CustomXmlMoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1CustomXmlMoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1CustomXmlMoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1CustomXmlMoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1DelList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1InsList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1MoveFromList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1MoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1MoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1MoveToList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1MoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1MoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1OMathList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1OMathParaList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1PList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1PermEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1PermStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1ProofErrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1SdtList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1TblList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1CantSplitList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1CnfStyleList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1DivIdList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1GridAfterList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1GridBeforeList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1HiddenList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1JcList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1TblCellSpacingList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1TblHeaderList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1TrHeightList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1WAfterList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1WBeforeList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1AltChunkList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1BookmarkEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1BookmarkStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1CommentRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1CommentRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1CustomXmlDelRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1CustomXmlDelRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1CustomXmlInsRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1CustomXmlInsRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1CustomXmlList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1CustomXmlMoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1CustomXmlMoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1CustomXmlMoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1CustomXmlMoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1DelList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1InsList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1MoveFromList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1MoveFromRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1MoveFromRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1MoveToList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1MoveToRangeEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1MoveToRangeStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1OMathList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1OMathParaList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1PList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1PermEndList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1PermStartList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1ProofErrList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1SdtList', 
-  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1TblList', 
-  'org.osgi.framework.BundleActivator', 
-  'org.osgi.framework.BundleContext', 
-  'org.osgi.framework.ServiceReference', 
-  'org.osgi.framework.ServiceRegistration', 
-  'org.osgi.util.tracker.ServiceTracker', 
-  'org.osgi.util.tracker.ServiceTrackerCustomizer', 
-  'org.sqlite.SQLiteConfig', 
-  'org.tukaani.xz.ARMOptions', 
-  'org.tukaani.xz.ARMThumbOptions', 
-  'org.tukaani.xz.DeltaOptions', 
-  'org.tukaani.xz.FilterOptions', 
-  'org.tukaani.xz.FinishableWrapperOutputStream', 
-  'org.tukaani.xz.IA64Options', 
-  'org.tukaani.xz.LZMA2InputStream', 
-  'org.tukaani.xz.LZMA2Options', 
-  'org.tukaani.xz.LZMAInputStream', 
-  'org.tukaani.xz.PowerPCOptions', 
-  'org.tukaani.xz.SPARCOptions', 
-  'org.tukaani.xz.SingleXZInputStream', 
-  'org.tukaani.xz.UnsupportedOptionsException', 
-  'org.tukaani.xz.X86Options', 
-  'org.tukaani.xz.XZ', 
-  'org.tukaani.xz.XZInputStream', 
-  'org.tukaani.xz.XZOutputStream', 
-  'org.w3.x2000.x09.xmldsig.KeyInfoType', 
-  'org.w3.x2000.x09.xmldsig.SignatureMethodType', 
-  'org.w3.x2000.x09.xmldsig.SignatureValueType', 
-  'org.w3.x2000.x09.xmldsig.TransformsType', 
-  'org.w3.x2000.x09.xmldsig.impl.SignatureTypeImpl$1ObjectList', 
-  'org.w3.x2000.x09.xmldsig.impl.SignedInfoTypeImpl$1ReferenceList', 
-  'org.w3.x2000.x09.xmldsig.impl.TransformTypeImpl$1XPathList', 
-  'org.w3.x2000.x09.xmldsig.impl.TransformTypeImpl$2XPathList', 
-  'schemasMicrosoftComOfficeExcel.STCF', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1Accel2List', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1AccelList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1AnchorList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1AutoFillList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1AutoLineList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1AutoPictList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1AutoScaleList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1CFList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1CameraList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1CancelList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1CheckedList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1ColHiddenList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1ColoredList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1ColumnList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1DDEList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1DefaultList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1DefaultSizeList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1DisabledList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1DismissList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1DropLinesList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1DropStyleList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1DxList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1FirstButtonList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1FmlaGroupList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1FmlaLinkList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1FmlaMacroList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1FmlaPictList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1FmlaRangeList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1FmlaTxbxList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1HelpList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1HorizList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1IncList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1JustLastXList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1LCTList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1ListItemList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1LockTextList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1LockedList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1MapOCXList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1MaxList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1MinList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1MoveWithCellsList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1MultiLineList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1MultiSelList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1NoThreeD2List', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1NoThreeDList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1PageList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1PrintObjectList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1RecalcAlwaysList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1RowHiddenList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1RowList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1ScriptExtendedList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1ScriptLanguageList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1ScriptLocationList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1ScriptTextList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1SecretEditList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1SelList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1SelTypeList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1SizeWithCellsList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1TextHAlignList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1TextVAlignList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1UIObjList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1VScrollList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1VTEditList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1ValList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1ValidIdsList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1VisibleList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$1WidthMinList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2Accel2List', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2AccelList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2AnchorList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2AutoFillList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2AutoLineList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2AutoPictList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2AutoScaleList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2CFList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2CameraList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2CancelList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2CheckedList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2ColHiddenList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2ColoredList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2ColumnList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2DDEList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2DefaultList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2DefaultSizeList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2DisabledList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2DismissList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2DropLinesList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2DropStyleList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2DxList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2FirstButtonList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2FmlaGroupList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2FmlaLinkList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2FmlaMacroList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2FmlaPictList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2FmlaRangeList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2FmlaTxbxList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2HelpList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2HorizList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2IncList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2JustLastXList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2LCTList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2ListItemList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2LockTextList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2LockedList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2MapOCXList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2MaxList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2MinList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2MoveWithCellsList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2MultiLineList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2MultiSelList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2NoThreeD2List', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2NoThreeDList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2PageList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2PrintObjectList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2RecalcAlwaysList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2RowHiddenList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2RowList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2ScriptExtendedList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2ScriptLanguageList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2ScriptLocationList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2ScriptTextList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2SecretEditList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2SelList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2SelTypeList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2SizeWithCellsList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2TextHAlignList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2TextVAlignList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2UIObjList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2VScrollList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2VTEditList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2ValList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2ValidIdsList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2VisibleList', 
-  'schemasMicrosoftComOfficeExcel.impl.CTClientDataImpl$2WidthMinList', 
-  'schemasMicrosoftComOfficeOffice.CTCallout', 
-  'schemasMicrosoftComOfficeOffice.CTClipPath', 
-  'schemasMicrosoftComOfficeOffice.CTComplex', 
-  'schemasMicrosoftComOfficeOffice.CTDiagram', 
-  'schemasMicrosoftComOfficeOffice.CTExtrusion', 
-  'schemasMicrosoftComOfficeOffice.CTFill', 
-  'schemasMicrosoftComOfficeOffice.CTInk', 
-  'schemasMicrosoftComOfficeOffice.CTRegroupTable', 
-  'schemasMicrosoftComOfficeOffice.CTRules', 
-  'schemasMicrosoftComOfficeOffice.CTSignatureLine', 
-  'schemasMicrosoftComOfficeOffice.CTSkew', 
-  'schemasMicrosoftComOfficeOffice.CTStrokeChild', 
-  'schemasMicrosoftComOfficeOffice.STBWMode', 
-  'schemasMicrosoftComOfficeOffice.STConnectorType', 
-  'schemasMicrosoftComOfficeOffice.STHrAlign', 
-  'schemasMicrosoftComOfficeOffice.STRelationshipId', 
-  'schemasMicrosoftComOfficeOffice.STTrueFalse', 
-  'schemasMicrosoftComOfficeOffice.STTrueFalseBlank', 
-  'schemasMicrosoftComOfficePowerpoint.CTEmpty', 
-  'schemasMicrosoftComOfficePowerpoint.CTRel', 
-  'schemasMicrosoftComOfficeWord.CTAnchorLock', 
-  'schemasMicrosoftComOfficeWord.CTBorder', 
-  'schemasMicrosoftComOfficeWord.CTWrap', 
-  'schemasMicrosoftComVml.CTArc', 
-  'schemasMicrosoftComVml.CTCurve', 
-  'schemasMicrosoftComVml.CTImage', 
-  'schemasMicrosoftComVml.CTImageData', 
-  'schemasMicrosoftComVml.CTLine', 
-  'schemasMicrosoftComVml.CTOval', 
-  'schemasMicrosoftComVml.CTPolyLine', 
-  'schemasMicrosoftComVml.CTRect', 
-  'schemasMicrosoftComVml.CTRoundRect', 
-  'schemasMicrosoftComVml.STEditAs', 
-  'schemasMicrosoftComVml.STFillMethod', 
-  'schemasMicrosoftComVml.STFillType', 
-  'schemasMicrosoftComVml.STImageAspect', 
-  'schemasMicrosoftComVml.STShadowType', 
-  'schemasMicrosoftComVml.STStrokeArrowLength', 
-  'schemasMicrosoftComVml.STStrokeArrowType', 
-  'schemasMicrosoftComVml.STStrokeArrowWidth', 
-  'schemasMicrosoftComVml.STStrokeEndCap', 
-  'schemasMicrosoftComVml.STStrokeLineStyle', 
-  'schemasMicrosoftComVml.STTrueFalseBlank', 
-  'schemasMicrosoftComVml.impl.CTFormulasImpl$1FList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1AnchorlockList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1ArcList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1BorderbottomList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1BorderleftList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1BorderrightList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1BordertopList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1CalloutList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1ClientDataList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1ClippathList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1CurveList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1DiagramList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1ExtrusionList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1FillList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1FormulasList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1GroupList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1HandlesList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1ImageList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1ImagedataList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1LineList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1LockList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1OvalList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1PathList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1PolylineList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1RectList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1RoundrectList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1ShadowList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1ShapeList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1ShapetypeList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1SignaturelineList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1SkewList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1StrokeList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1TextboxList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1TextdataList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1TextpathList', 
-  'schemasMicrosoftComVml.impl.CTGroupImpl$1WrapList', 
-  'schemasMicrosoftComVml.impl.CTHandlesImpl$1HList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1AnchorlockList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1BorderbottomList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1BorderleftList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1BorderrightList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1BordertopList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1CalloutList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1ClippathList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1ExtrusionList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1FillList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1FormulasList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1HandlesList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1ImagedataList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1InkList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1IscommentList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1LockList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1PathList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1ShadowList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1SignaturelineList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1SkewList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1StrokeList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1TextboxList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1TextdataList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1TextpathList', 
-  'schemasMicrosoftComVml.impl.CTShapeImpl$1WrapList', 
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1AnchorlockList', 
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1BorderbottomList', 
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1BorderleftList', 
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1BorderrightList', 
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1BordertopList', 
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1CalloutList', 
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1ClientDataList', 
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1ClippathList', 
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1ExtrusionList', 
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1FillList', 
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1FormulasList', 
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1HandlesList', 
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1ImagedataList', 
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1LockList', 
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1PathList', 
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1ShadowList', 
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1SignaturelineList', 
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1SkewList', 
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1StrokeList', 
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1TextboxList', 
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1TextdataList', 
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1TextpathList', 
-  'schemasMicrosoftComVml.impl.CTShapetypeImpl$1WrapList', 
-  'ucar.ma2.DataType', 
-  'ucar.nc2.Attribute', 
-  'ucar.nc2.Dimension', 
-  'ucar.nc2.Group', 
-  'ucar.nc2.NetcdfFile', 
-  'ucar.nc2.Variable', 
-  'ucar.nc2.dataset.NetcdfDataset',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTArea3DChart',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTAreaChart',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTAxisUnit',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTBar3DChart',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTBarChart',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTBubbleChart',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTChartLines',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTDLbls',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTDPt',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTDTable',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTDateAx',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTDispBlanksAs',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTDispUnits',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTDoughnutChart',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTErrBars',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTExtensionList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTExternalData',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTFirstSliceAng',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTGrouping',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTLblAlgn',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTLblOffset',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTLegendEntry',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTLine3DChart',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTMarkerSize',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTMultiLvlStrRef',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTOfPieChart',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTPie3DChart',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTPivotFmts',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTPivotSource',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTProtection',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTRadarChart',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTRelId',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTSerAx',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTSkip',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTStockChart',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTStyle',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTSurface',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTSurface3DChart',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTSurfaceChart',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTTextLanguageID',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTTrendline',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTUpDownBars',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.CTView3D',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.STPageSetupOrientation',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTLegendImpl$1LegendEntryList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTLineChartImpl$1AxIdList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTLineChartImpl$1SerList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTLineSerImpl$1DPtList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTLineSerImpl$1TrendlineList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTNumDataImpl$1PtList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPieChartImpl$1SerList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPieSerImpl$1DPtList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1Area3DChartList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1AreaChartList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1Bar3DChartList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1BarChartList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1BubbleChartList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1CatAxList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1DateAxList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1DoughnutChartList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1Line3DChartList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1LineChartList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1OfPieChartList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1Pie3DChartList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1PieChartList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1RadarChartList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1ScatterChartList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1SerAxList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1StockChartList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1Surface3DChartList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1SurfaceChartList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTPlotAreaImpl$1ValAxList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTScatterChartImpl$1AxIdList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTScatterChartImpl$1SerList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTScatterSerImpl$1DPtList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTScatterSerImpl$1ErrBarsList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTScatterSerImpl$1TrendlineList',
+  'org.openxmlformats.schemas.drawingml.x2006.chart.impl.CTStrDataImpl$1PtList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTAlphaBiLevelEffect',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTAlphaCeilingEffect',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTAlphaFloorEffect',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTAlphaInverseEffect',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTAlphaModulateEffect',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTAlphaReplaceEffect',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTAngle',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTAudioCD',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTAudioFile',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTBiLevelEffect',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTBlurEffect',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTCell3D',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTColorChangeEffect',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTColorReplaceEffect',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTColorSchemeList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTComplementTransform',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTConnectionSite',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTConnectorLocking',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTCustomColorList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTDashStopList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTDuotoneEffect',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTEffectContainer',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTEmbeddedWAVAudioFile',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTFillOverlayEffect',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTFlatText',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTGammaTransform',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTGlowEffect',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTGrayscaleEffect',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTGrayscaleTransform',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTGroupFillProperties',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTGroupLocking',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTHSLEffect',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTInnerShadowEffect',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTInverseGammaTransform',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTInverseTransform',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTLineJoinBevel',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTLuminanceEffect',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTObjectStyleDefaults',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTPath2DArcTo',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTPatternFillProperties',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTPolarAdjustHandle',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTPositiveFixedAngle',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTPresetShadowEffect',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTPresetTextShape',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTQuickTimeFile',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTReflectionEffect',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTScene3D',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTShape3D',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTShapeLocking',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTSoftEdgesEffect',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTSupplementalFont',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTTableBackgroundStyle',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTTablePartStyle',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTTextBlipBullet',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTTextBulletColorFollowText',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTTextBulletSizeFollowText',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTTextBulletTypefaceFollowText',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTTextUnderlineFillFollowText',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTTextUnderlineFillGroupWrapper',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTTextUnderlineLineFollowText',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTTileInfoProperties',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTTintEffect',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTVideoFile',
+  'org.openxmlformats.schemas.drawingml.x2006.main.CTXYAdjustHandle',
+  'org.openxmlformats.schemas.drawingml.x2006.main.STBlackWhiteMode',
+  'org.openxmlformats.schemas.drawingml.x2006.main.STBlipCompression',
+  'org.openxmlformats.schemas.drawingml.x2006.main.STFixedAngle',
+  'org.openxmlformats.schemas.drawingml.x2006.main.STGuid',
+  'org.openxmlformats.schemas.drawingml.x2006.main.STPanose',
+  'org.openxmlformats.schemas.drawingml.x2006.main.STPathFillMode',
+  'org.openxmlformats.schemas.drawingml.x2006.main.STRectAlignment',
+  'org.openxmlformats.schemas.drawingml.x2006.main.STTextColumnCount',
+  'org.openxmlformats.schemas.drawingml.x2006.main.STTextNonNegativePoint',
+  'org.openxmlformats.schemas.drawingml.x2006.main.STTextTabAlignType',
+  'org.openxmlformats.schemas.drawingml.x2006.main.STTileFlipMode',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTAdjustHandleListImpl$1AhPolarList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTAdjustHandleListImpl$1AhXYList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBackgroundFillStyleListImpl$1BlipFillList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBackgroundFillStyleListImpl$1GradFillList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBackgroundFillStyleListImpl$1GrpFillList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBackgroundFillStyleListImpl$1NoFillList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBackgroundFillStyleListImpl$1PattFillList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBackgroundFillStyleListImpl$1SolidFillList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1AlphaBiLevelList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1AlphaCeilingList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1AlphaFloorList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1AlphaInvList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1AlphaModFixList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1AlphaModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1AlphaReplList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1BiLevelList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1BlurList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1ClrChangeList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1ClrReplList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1DuotoneList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1FillOverlayList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1GraysclList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1HslList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1LumList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTBlipImpl$1TintList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTConnectionSiteListImpl$1CxnList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTEffectStyleListImpl$1EffectStyleList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTFillStyleListImpl$1BlipFillList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTFillStyleListImpl$1GradFillList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTFillStyleListImpl$1GrpFillList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTFillStyleListImpl$1NoFillList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTFillStyleListImpl$1PattFillList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTFillStyleListImpl$1SolidFillList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTFontCollectionImpl$1FontList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTGeomGuideListImpl$1GdList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTGradientStopListImpl$1GsList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1AlphaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1AlphaModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1AlphaOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1BlueList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1BlueModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1BlueOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1CompList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1GammaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1GrayList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1GreenList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1GreenModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1GreenOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1HueList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1HueModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1HueOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1InvGammaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1InvList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1LumList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1LumModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1LumOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1RedList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1RedModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1RedOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1SatList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1SatModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1SatOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1ShadeList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTHslColorImpl$1TintList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTLineStyleListImpl$1LnList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTOfficeArtExtensionListImpl$1ExtList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPath2DCubicBezierToImpl$1PtList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPath2DImpl$1ArcToList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPath2DImpl$1CloseList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPath2DImpl$1CubicBezToList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPath2DImpl$1LnToList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPath2DImpl$1MoveToList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPath2DImpl$1QuadBezToList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPath2DListImpl$1PathList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1AlphaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1AlphaModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1AlphaOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1BlueList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1BlueModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1BlueOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1CompList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1GammaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1GrayList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1GreenList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1GreenModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1GreenOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1HueList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1HueModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1HueOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1InvGammaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1InvList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1LumList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1LumModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1LumOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1RedList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1RedModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1RedOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1SatList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1SatModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1SatOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1ShadeList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTPresetColorImpl$1TintList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1AlphaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1AlphaModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1AlphaOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1BlueList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1BlueModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1BlueOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1CompList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1GammaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1GrayList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1GreenList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1GreenModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1GreenOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1HueList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1HueModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1HueOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1InvGammaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1InvList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1LumList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1LumModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1LumOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1RedList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1RedModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1RedOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1SatList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1SatModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1SatOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1ShadeList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSRgbColorImpl$1TintList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1AlphaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1AlphaModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1AlphaOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1BlueList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1BlueModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1BlueOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1CompList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1GammaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1GrayList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1GreenList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1GreenModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1GreenOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1HueList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1HueModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1HueOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1InvGammaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1InvList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1LumList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1LumModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1LumOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1RedList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1RedModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1RedOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1SatList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1SatModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1SatOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1ShadeList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSchemeColorImpl$1TintList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1AlphaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1AlphaModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1AlphaOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1BlueList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1BlueModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1BlueOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1CompList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1GammaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1GrayList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1GreenList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1GreenModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1GreenOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1HueList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1HueModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1HueOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1InvGammaList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1InvList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1LumList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1LumModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1LumOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1RedList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1RedModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1RedOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1SatList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1SatModList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1SatOffList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1ShadeList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTSystemColorImpl$1TintList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTTableGridImpl$1GridColList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTTableImpl$1TrList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTTableRowImpl$1TcList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTTableStyleListImpl$1TblStyleList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTTextBodyImpl$1PList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTTextParagraphImpl$1BrList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTTextParagraphImpl$1FldList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTTextParagraphImpl$1RList',
+  'org.openxmlformats.schemas.drawingml.x2006.main.impl.CTTextTabStopListImpl$1TabList',
+  'org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.CTAbsoluteAnchor',
+  'org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.impl.CTDrawingImpl$1AbsoluteAnchorList',
+  'org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.impl.CTDrawingImpl$1OneCellAnchorList',
+  'org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.impl.CTDrawingImpl$1TwoCellAnchorList',
+  'org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.impl.CTGroupShapeImpl$1CxnSpList',
+  'org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.impl.CTGroupShapeImpl$1GraphicFrameList',
+  'org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.impl.CTGroupShapeImpl$1GrpSpList',
+  'org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.impl.CTGroupShapeImpl$1PicList',
+  'org.openxmlformats.schemas.drawingml.x2006.spreadsheetDrawing.impl.CTGroupShapeImpl$1SpList',
+  'org.openxmlformats.schemas.drawingml.x2006.wordprocessingDrawing.CTEffectExtent',
+  'org.openxmlformats.schemas.drawingml.x2006.wordprocessingDrawing.CTPosH',
+  'org.openxmlformats.schemas.drawingml.x2006.wordprocessingDrawing.CTPosV',
+  'org.openxmlformats.schemas.drawingml.x2006.wordprocessingDrawing.CTWrapNone',
+  'org.openxmlformats.schemas.drawingml.x2006.wordprocessingDrawing.CTWrapSquare',
+  'org.openxmlformats.schemas.drawingml.x2006.wordprocessingDrawing.CTWrapThrough',
+  'org.openxmlformats.schemas.drawingml.x2006.wordprocessingDrawing.CTWrapTight',
+  'org.openxmlformats.schemas.drawingml.x2006.wordprocessingDrawing.CTWrapTopBottom',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.CTArray',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.CTCf',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.CTEmpty',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.CTNull',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.CTVstream',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.STCy',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.STError',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.STVectorBaseType',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1BoolList',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1BstrList',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1CfList',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1ClsidList',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1CyList',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1DateList',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1ErrorList',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1FiletimeList',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1I1List',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1I2List',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1I4List',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1I8List',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1LpstrList',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1LpwstrList',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1R4List',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1R8List',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1Ui1List',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1Ui2List',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1Ui4List',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1Ui8List',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$1VariantList',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2BoolList',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2BstrList',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2ClsidList',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2CyList',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2DateList',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2ErrorList',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2FiletimeList',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2I1List',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2I2List',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2I4List',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2I8List',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2LpstrList',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2LpwstrList',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2R4List',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2R8List',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2Ui1List',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2Ui2List',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2Ui4List',
+  'org.openxmlformats.schemas.officeDocument.x2006.docPropsVTypes.impl.CTVectorImpl$2Ui8List',
+  'org.openxmlformats.schemas.officeDocument.x2006.math.CTAcc',
+  'org.openxmlformats.schemas.officeDocument.x2006.math.CTBar',
+  'org.openxmlformats.schemas.officeDocument.x2006.math.CTBorderBox',
+  'org.openxmlformats.schemas.officeDocument.x2006.math.CTBox',
+  'org.openxmlformats.schemas.officeDocument.x2006.math.CTD',
+  'org.openxmlformats.schemas.officeDocument.x2006.math.CTEqArr',
+  'org.openxmlformats.schemas.officeDocument.x2006.math.CTF',
+  'org.openxmlformats.schemas.officeDocument.x2006.math.CTFunc',
+  'org.openxmlformats.schemas.officeDocument.x2006.math.CTGroupChr',
+  'org.openxmlformats.schemas.officeDocument.x2006.math.CTLimLow',
+  'org.openxmlformats.schemas.officeDocument.x2006.math.CTLimUpp',
+  'org.openxmlformats.schemas.officeDocument.x2006.math.CTM',
+  'org.openxmlformats.schemas.officeDocument.x2006.math.CTMathPr',
+  'org.openxmlformats.schemas.officeDocument.x2006.math.CTNary',
+  'org.openxmlformats.schemas.officeDocument.x2006.math.CTOMath',
+  'org.openxmlformats.schemas.officeDocument.x2006.math.CTOMathPara',
+  'org.openxmlformats.schemas.officeDocument.x2006.math.CTPhant',
+  'org.openxmlformats.schemas.officeDocument.x2006.math.CTR',
+  'org.openxmlformats.schemas.officeDocument.x2006.math.CTRad',
+  'org.openxmlformats.schemas.officeDocument.x2006.math.CTSPre',
+  'org.openxmlformats.schemas.officeDocument.x2006.math.CTSSub',
+  'org.openxmlformats.schemas.officeDocument.x2006.math.CTSSubSup',
+  'org.openxmlformats.schemas.officeDocument.x2006.math.CTSSup',
+  'org.openxmlformats.schemas.presentationml.x2006.main.CTControlList',
+  'org.openxmlformats.schemas.presentationml.x2006.main.CTCustomShowList',
+  'org.openxmlformats.schemas.presentationml.x2006.main.CTCustomerData',
+  'org.openxmlformats.schemas.presentationml.x2006.main.CTEmbeddedFontList',
+  'org.openxmlformats.schemas.presentationml.x2006.main.CTExtensionList',
+  'org.openxmlformats.schemas.presentationml.x2006.main.CTExtensionListModify',
+  'org.openxmlformats.schemas.presentationml.x2006.main.CTHandoutMasterIdList',
+  'org.openxmlformats.schemas.presentationml.x2006.main.CTHeaderFooter',
+  'org.openxmlformats.schemas.presentationml.x2006.main.CTKinsoku',
+  'org.openxmlformats.schemas.presentationml.x2006.main.CTModifyVerifier',
+  'org.openxmlformats.schemas.presentationml.x2006.main.CTPhotoAlbum',
+  'org.openxmlformats.schemas.presentationml.x2006.main.CTSlideLayoutIdList',
+  'org.openxmlformats.schemas.presentationml.x2006.main.CTSlideTiming',
+  'org.openxmlformats.schemas.presentationml.x2006.main.CTSlideTransition',
+  'org.openxmlformats.schemas.presentationml.x2006.main.CTSmartTags',
+  'org.openxmlformats.schemas.presentationml.x2006.main.STBookmarkIdSeed',
+  'org.openxmlformats.schemas.presentationml.x2006.main.STDirection',
+  'org.openxmlformats.schemas.presentationml.x2006.main.STIndex',
+  'org.openxmlformats.schemas.presentationml.x2006.main.STPlaceholderSize',
+  'org.openxmlformats.schemas.presentationml.x2006.main.STSlideSizeType',
+  'org.openxmlformats.schemas.presentationml.x2006.main.impl.CTCommentAuthorListImpl$1CmAuthorList',
+  'org.openxmlformats.schemas.presentationml.x2006.main.impl.CTCommentListImpl$1CmList',
+  'org.openxmlformats.schemas.presentationml.x2006.main.impl.CTCustomerDataListImpl$1CustDataList',
+  'org.openxmlformats.schemas.schemaLibrary.x2006.main.CTSchemaLibrary',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTAutoSortScope',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTBoolean',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTCacheHierarchies',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTCalculatedItems',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTCalculatedMembers',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTCellStyles',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTCellWatches',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTChartFormats',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTChartsheetPr',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTChartsheetProtection',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTChartsheetViews',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTColHierarchiesUsage',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTColItems',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTColors',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTConditionalFormats',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTConsolidation',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTControls',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTCsPageSetup',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTCustomChartsheetViews',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTCustomProperties',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTCustomSheetViews',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTCustomWorkbookViews',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTDataBinding',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTDataConsolidate',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTDateTime',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTDdeLink',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTDimensions',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTError',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTExtensionList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTExternalSheetDataSet',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTFieldGroup',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTFileRecoveryPr',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTFileSharing',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTFileVersion',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTFilterColumn',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTFormats',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTFunctionGroups',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTGradientFill',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTMeasureDimensionMaps',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTMeasureGroups',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTMissing',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTNumber',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTOleLink',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTOleObjects',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTOleSize',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPCDKPIs',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPhoneticRun',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPivotFilters',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPivotHierarchies',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTPivotSelection',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTProtectedRanges',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTRecord',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTRowHierarchiesUsage',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTRowItems',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTScenarios',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTSheetBackgroundPicture',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTSmartTagPr',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTSmartTagTypes',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTSmartTags',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTSortState',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTString',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTTableFormula',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTTableStyles',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTTupleCache',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTWebPublishItems',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTWebPublishObjects',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTWebPublishing',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.CTX',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STCellSpans',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STDataValidationImeMode',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STFieldSortType',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STGuid',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STObjects',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STPhoneticAlignment',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STPhoneticType',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STPrintError',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STRefMode',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STSheetViewType',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STShowDataAs',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STTableType',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STTimePeriod',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STTotalsRowFunction',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STUpdateLinks',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.STVisibility',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTAuthorsImpl$1AuthorList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTAuthorsImpl$2AuthorList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTAutoFilterImpl$1FilterColumnList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTBookViewsImpl$1WorkbookViewList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTBordersImpl$1BorderList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTCacheFieldImpl$1MpMapList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTCacheFieldsImpl$1CacheFieldList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTCellStyleXfsImpl$1XfList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTCellXfsImpl$1XfList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTCfRuleImpl$1FormulaList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTCfRuleImpl$2FormulaList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTColFieldsImpl$1FieldList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTColorScaleImpl$1CfvoList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTColorScaleImpl$1ColorList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTCommentListImpl$1CommentList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTConditionalFormattingImpl$1CfRuleList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTDataBarImpl$1CfvoList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTDataValidationsImpl$1DataValidationList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTDxfsImpl$1DxfList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTExternalDefinedNamesImpl$1DefinedNameList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTExternalReferencesImpl$1ExternalReferenceList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTExternalSheetNamesImpl$1SheetNameList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFillsImpl$1FillList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1BList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1CharsetList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1ColorList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1CondenseList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1ExtendList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1FamilyList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1IList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1NameList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1OutlineList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1SchemeList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1ShadowList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1StrikeList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1SzList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1UList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontImpl$1VertAlignList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTFontsImpl$1FontList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTHyperlinksImpl$1HyperlinkList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTIconSetImpl$1CfvoList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTItemsImpl$1ItemList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTMapInfoImpl$1MapList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTMapInfoImpl$1SchemaList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTMergeCellsImpl$1MergeCellList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTNumFmtsImpl$1NumFmtList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTPageBreakImpl$1BrkList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTPageFieldsImpl$1PageFieldList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTPivotCacheRecordsImpl$1RList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTPivotCachesImpl$1PivotCacheList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTPivotFieldsImpl$1PivotFieldList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1BList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1CharsetList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1ColorList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1CondenseList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1ExtendList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1FamilyList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1IList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1OutlineList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1RFontList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1SchemeList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1ShadowList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1StrikeList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1SzList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1UList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRPrEltImpl$1VertAlignList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRowFieldsImpl$1FieldList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRowImpl$1CList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRstImpl$1RList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTRstImpl$1RPhList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSharedItemsImpl$1BList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSharedItemsImpl$1DList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSharedItemsImpl$1EList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSharedItemsImpl$1MList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSharedItemsImpl$1NList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSharedItemsImpl$1SList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSheetDataImpl$1RowList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSheetViewImpl$1PivotSelectionList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSheetViewImpl$1SelectionList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSheetViewsImpl$1SheetViewList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSingleXmlCellsImpl$1SingleXmlCellList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSstImpl$1SiList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTTableColumnsImpl$1TableColumnList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTTablePartsImpl$1TablePartList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTWorkbookImpl$1FileRecoveryPrList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTWorksheetImpl$1ColsList',
+  'org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTWorksheetImpl$1ConditionalFormattingList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTAltChunk',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTAttr',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTBackground',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTCaptions',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTCellMergeTrackChange',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTCharacterSpacing',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTCnf',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTColorSchemeMapping',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTColumns',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTCompat',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTControl',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTCustomXmlBlock',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTCustomXmlCell',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTCustomXmlRow',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTCustomXmlRun',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTDataBinding',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTDocGrid',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTDocRsids',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTDocType',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTDocVars',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTEastAsianLayout',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTEdnDocProps',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTEdnProps',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTEm',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTFFDDList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTFFHelpText',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTFFName',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTFFStatusText',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTFFTextInput',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTFitText',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTFramePr',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTFtnDocProps',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTFtnProps',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTKinsoku',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTLevelSuffix',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTLineNumber',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTLock',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTLongHexNumber',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTLvlLegacy',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTMacroName',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTMailMerge',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTMultiLevelType',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTNumPicBullet',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTPPrChange',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTPageBorders',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTPageMar',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTPageNumber',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTPageSz',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTPaperSource',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTParaRPrChange',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTPerm',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTPermStart',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTPlaceholder',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTProof',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTRPrChange',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTReadingModeInkLockDown',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTRuby',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTSaveThroughXslt',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTSdtComboBox',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTSdtDate',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTSdtDropDownList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTSdtRow',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTSdtText',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTSectPrChange',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTSectType',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTShapeDefaults',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTShortHexNumber',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTSignedTwipsMeasure',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTSmartTagType',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTblGridChange',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTblLayoutType',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTblOverlap',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTblPPr',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTblPrChange',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTblPrExChange',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTblStylePr',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTcMar',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTcPrChange',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTextDirection',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTextEffect',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTextScale',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTextboxTightWrap',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTrPrChange',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTrackChangeNumbering',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTrackChangesView',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTTwipsMeasure',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTView',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTWriteProtection',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.CTWritingStyle',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STDateTime',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STDisplacedByCustomXml',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STHeightRule',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STHint',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STPTabAlignment',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STPTabLeader',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STPTabRelativeTo',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STProofErr',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STShortHexNumber',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STThemeColor',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STUcharHexNumber',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.STZoom',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTAbstractNumImpl$1LvlList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1AltChunkList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1BookmarkEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1BookmarkStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1CommentRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1CommentRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1CustomXmlDelRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1CustomXmlDelRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1CustomXmlInsRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1CustomXmlInsRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1CustomXmlList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1CustomXmlMoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1CustomXmlMoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1CustomXmlMoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1CustomXmlMoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1DelList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1InsList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1MoveFromList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1MoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1MoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1MoveToList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1MoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1MoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1OMathList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1OMathParaList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1PList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1PermEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1PermStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1ProofErrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1SdtList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTBodyImpl$1TblList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1AltChunkList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1BookmarkEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1BookmarkStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1CommentRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1CommentRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1CustomXmlDelRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1CustomXmlDelRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1CustomXmlInsRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1CustomXmlInsRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1CustomXmlList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1CustomXmlMoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1CustomXmlMoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1CustomXmlMoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1CustomXmlMoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1DelList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1InsList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1MoveFromList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1MoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1MoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1MoveToList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1MoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1MoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1OMathList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1OMathParaList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1PList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1PermEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1PermStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1ProofErrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1SdtList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentImpl$1TblList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTCommentsImpl$1CommentList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTDrawingImpl$1AnchorList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTDrawingImpl$1InlineList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTEndnotesImpl$1EndnoteList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFFDataImpl$1CalcOnExitList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFFDataImpl$1DdListList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFFDataImpl$1EnabledList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFFDataImpl$1EntryMacroList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFFDataImpl$1ExitMacroList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFFDataImpl$1HelpTextList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFFDataImpl$1NameList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFFDataImpl$1StatusTextList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFFDataImpl$1TextInputList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFootnotesImpl$1FootnoteList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1AltChunkList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1BookmarkEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1BookmarkStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1CommentRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1CommentRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1CustomXmlDelRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1CustomXmlDelRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1CustomXmlInsRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1CustomXmlInsRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1CustomXmlList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1CustomXmlMoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1CustomXmlMoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1CustomXmlMoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1CustomXmlMoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1DelList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1InsList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1MoveFromList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1MoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1MoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1MoveToList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1MoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1MoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1OMathList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1OMathParaList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1PList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1PermEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1PermStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1ProofErrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1SdtList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTFtnEdnImpl$1TblList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1AltChunkList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1BookmarkEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1BookmarkStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1CommentRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1CommentRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1CustomXmlDelRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1CustomXmlDelRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1CustomXmlInsRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1CustomXmlInsRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1CustomXmlList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1CustomXmlMoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1CustomXmlMoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1CustomXmlMoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1CustomXmlMoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1DelList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1InsList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1MoveFromList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1MoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1MoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1MoveToList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1MoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1MoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1OMathList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1OMathParaList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1PList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1PermEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1PermStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1ProofErrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1SdtList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHdrFtrImpl$1TblList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1BookmarkEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1BookmarkStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1CommentRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1CommentRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1CustomXmlDelRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1CustomXmlDelRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1CustomXmlInsRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1CustomXmlInsRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1CustomXmlList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1CustomXmlMoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1CustomXmlMoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1CustomXmlMoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1CustomXmlMoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1DelList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1FldSimpleList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1HyperlinkList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1InsList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1MoveFromList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1MoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1MoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1MoveToList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1MoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1MoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1OMathList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1OMathParaList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1PermEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1PermStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1ProofErrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1RList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1SdtList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1SmartTagList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTHyperlinkImpl$1SubDocList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTLatentStylesImpl$1LsdExceptionList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTNumImpl$1LvlOverrideList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTNumberingImpl$1AbstractNumList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTNumberingImpl$1NumList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTNumberingImpl$1NumPicBulletList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1BookmarkEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CommentRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CommentRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CustomXmlDelRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CustomXmlDelRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CustomXmlInsRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CustomXmlInsRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CustomXmlList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CustomXmlMoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CustomXmlMoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CustomXmlMoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1CustomXmlMoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1DelList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1FldSimpleList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1HyperlinkList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1InsList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1MoveFromList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1MoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1MoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1MoveToList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1MoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1MoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1OMathList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1OMathParaList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1PermEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1PermStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1ProofErrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1RList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1SdtList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1SmartTagList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTPImpl$1SubDocList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1AnnotationRefList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1BrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1CommentReferenceList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1ContinuationSeparatorList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1CrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1DayLongList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1DayShortList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1DelInstrTextList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1DelTextList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1DrawingList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1EndnoteRefList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1EndnoteReferenceList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1FldCharList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1FootnoteRefList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1FootnoteReferenceList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1InstrTextList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1LastRenderedPageBreakList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1MonthLongList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1MonthShortList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1NoBreakHyphenList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1ObjectList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1PgNumList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1PictList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1PtabList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1RubyList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1SeparatorList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1SoftHyphenList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1SymList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1TList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1TabList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1YearLongList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRImpl$1YearShortList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1BookmarkEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1BookmarkStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1CommentRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1CommentRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1CustomXmlDelRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1CustomXmlDelRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1CustomXmlInsRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1CustomXmlInsRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1CustomXmlList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1CustomXmlMoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1CustomXmlMoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1CustomXmlMoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1CustomXmlMoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1DelList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1InsList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1MoveFromList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1MoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1MoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1MoveToList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1MoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1MoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1OMathList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1OMathParaList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1PermEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1PermStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1ProofErrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1SdtList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRowImpl$1TcList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1AccList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1BarList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1BookmarkEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1BookmarkStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1BorderBoxList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1BoxList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1CommentRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1CommentRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1CustomXmlDelRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1CustomXmlDelRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1CustomXmlInsRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1CustomXmlInsRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1CustomXmlList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1CustomXmlMoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1CustomXmlMoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1CustomXmlMoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1CustomXmlMoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1DList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1DelList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1EqArrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1FList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1FuncList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1GroupChrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1InsList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1LimLowList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1LimUppList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1MList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1MoveFromList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1MoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1MoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1MoveToList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1MoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1MoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1NaryList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1OMathList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1OMathParaList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1PermEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1PermStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1PhantList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1ProofErrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1R2List',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1RList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1RadList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1SPreList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1SSubList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1SSubSupList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1SSupList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1SdtList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTRunTrackChangeImpl$1SmartTagList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1BookmarkEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1BookmarkStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1CommentRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1CommentRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1CustomXmlDelRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1CustomXmlDelRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1CustomXmlInsRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1CustomXmlInsRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1CustomXmlList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1CustomXmlMoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1CustomXmlMoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1CustomXmlMoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1CustomXmlMoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1DelList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1InsList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1MoveFromList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1MoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1MoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1MoveToList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1MoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1MoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1OMathList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1OMathParaList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1PList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1PermEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1PermStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1ProofErrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1SdtList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentBlockImpl$1TblList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1BookmarkEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1BookmarkStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1CommentRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1CommentRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1CustomXmlDelRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1CustomXmlDelRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1CustomXmlInsRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1CustomXmlInsRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1CustomXmlList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1CustomXmlMoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1CustomXmlMoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1CustomXmlMoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1CustomXmlMoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1DelList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1InsList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1MoveFromList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1MoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1MoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1MoveToList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1MoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1MoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1OMathList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1OMathParaList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1PermEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1PermStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1ProofErrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1SdtList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentCellImpl$1TcList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1BookmarkEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1BookmarkStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1CommentRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1CommentRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1CustomXmlDelRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1CustomXmlDelRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1CustomXmlInsRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1CustomXmlInsRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1CustomXmlList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1CustomXmlMoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1CustomXmlMoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1CustomXmlMoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1CustomXmlMoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1DelList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1FldSimpleList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1HyperlinkList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1InsList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1MoveFromList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1MoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1MoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1MoveToList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1MoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1MoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1OMathList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1OMathParaList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1PermEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1PermStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1ProofErrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1RList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1SdtList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1SmartTagList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtContentRunImpl$1SubDocList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtEndPrImpl$1RPrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1AliasList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1BibliographyList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1CitationList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1ComboBoxList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1DataBindingList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1DateList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1DocPartListList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1DocPartObjList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1DropDownListList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1EquationList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1GroupList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1IdList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1LockList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1PictureList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1PlaceholderList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1RPrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1RichTextList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1ShowingPlcHdrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1TagList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1TemporaryList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSdtPrImpl$1TextList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSectPrImpl$1FooterReferenceList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSectPrImpl$1HeaderReferenceList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSettingsImpl$1ActiveWritingStyleList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSettingsImpl$1AttachedSchemaList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSettingsImpl$1SmartTagTypeList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1BookmarkEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1BookmarkStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1CommentRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1CommentRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1CustomXmlDelRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1CustomXmlDelRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1CustomXmlInsRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1CustomXmlInsRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1CustomXmlList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1CustomXmlMoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1CustomXmlMoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1CustomXmlMoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1CustomXmlMoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1DelList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1FldSimpleList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1HyperlinkList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1InsList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1MoveFromList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1MoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1MoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1MoveToList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1MoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1MoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1OMathList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1OMathParaList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1PermEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1PermStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1ProofErrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1RList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1SdtList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1SmartTagList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSimpleFieldImpl$1SubDocList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagPrImpl$1AttrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1BookmarkEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1BookmarkStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1CommentRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1CommentRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1CustomXmlDelRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1CustomXmlDelRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1CustomXmlInsRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1CustomXmlInsRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1CustomXmlList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1CustomXmlMoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1CustomXmlMoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1CustomXmlMoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1CustomXmlMoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1DelList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1FldSimpleList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1HyperlinkList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1InsList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1MoveFromList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1MoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1MoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1MoveToList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1MoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1MoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1OMathList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1OMathParaList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1PermEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1PermStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1ProofErrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1RList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1SdtList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1SmartTagList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTSmartTagRunImpl$1SubDocList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTStyleImpl$1TblStylePrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTStylesImpl$1StyleList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTabsImpl$1TabList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblGridBaseImpl$1GridColList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1BookmarkEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1BookmarkStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1CommentRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1CommentRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1CustomXmlDelRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1CustomXmlDelRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1CustomXmlInsRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1CustomXmlInsRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1CustomXmlList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1CustomXmlMoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1CustomXmlMoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1CustomXmlMoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1CustomXmlMoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1DelList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1InsList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1MoveFromList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1MoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1MoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1MoveToList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1MoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1MoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1OMathList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1OMathParaList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1PermEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1PermStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1ProofErrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1SdtList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTblImpl$1TrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1AltChunkList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1BookmarkEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1BookmarkStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1CommentRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1CommentRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1CustomXmlDelRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1CustomXmlDelRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1CustomXmlInsRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1CustomXmlInsRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1CustomXmlList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1CustomXmlMoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1CustomXmlMoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1CustomXmlMoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1CustomXmlMoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1DelList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1InsList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1MoveFromList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1MoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1MoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1MoveToList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1MoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1MoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1OMathList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1OMathParaList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1PList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1PermEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1PermStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1ProofErrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1SdtList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTcImpl$1TblList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1CantSplitList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1CnfStyleList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1DivIdList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1GridAfterList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1GridBeforeList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1HiddenList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1JcList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1TblCellSpacingList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1TblHeaderList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1TrHeightList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1WAfterList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTrPrBaseImpl$1WBeforeList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1AltChunkList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1BookmarkEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1BookmarkStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1CommentRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1CommentRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1CustomXmlDelRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1CustomXmlDelRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1CustomXmlInsRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1CustomXmlInsRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1CustomXmlList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1CustomXmlMoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1CustomXmlMoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1CustomXmlMoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1CustomXmlMoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1DelList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1InsList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1MoveFromList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1MoveFromRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1MoveFromRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1MoveToList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1MoveToRangeEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1MoveToRangeStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1OMathList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1OMathParaList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1PList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1PermEndList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1PermStartList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1ProofErrList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1SdtList',
+  'org.openxmlformats.schemas.wordprocessingml.x2006.main.impl.CTTxbxContentImpl$1TblList',
+  'org.osgi.framework.BundleActivator',
+  'org.osgi.framework.BundleContext',
+  'org.osgi.framework.ServiceReference',
+  'org.osgi.framework.ServiceRegistration',
+  'org.osgi.util.tracker.ServiceTracker',
+  'org.osgi.util.tracker.ServiceTrackerCustomizer',
+  'org.slf4j.Logger',
+  'org.slf4j.LoggerFactory',
+  'org.sqlite.SQLiteConfig',
+  'org.tukaani.xz.ARMOptions',
+  'org.tukaani.xz.ARMThumbOptions',
+  'org.tukaani.xz.DeltaOptions',
+  'org.tukaani.xz.FilterOptions',
+  'org.tukaani.xz.FinishableWrapperOutputStream',
+  'org.tukaani.xz.IA64Options',
+  'org.tukaani.xz.LZMA2InputStream',
+  'org.tukaani.xz.LZMA2Options',
+  'org.tukaani.xz.LZMAInputStream',
+  'org.tukaani.xz.PowerPCOptions',
+  'org.tukaani.xz.SPARCOptions',
+  'org.tukaani.xz.SingleXZInputStream',
+  'org.tukaani.xz.UnsupportedOptionsException',
+  'org.tukaani.xz.X86Options',
+  'org.tukaani.xz.XZ',
+  'org.tukaani.xz.XZInputStream',
+  'org.tukaani.xz.XZOutputStream',
+  'org.w3.x2000.x09.xmldsig.KeyInfoType',
+  'org.w3.x2000.x09.xmldsig.SignatureMethodType',
+  'org.w3.x2000.x09.xmldsig.SignatureValueType',
+  'org.w3.x2000.x09.xmldsig.TransformsType',
+  'org.w3.x2000.x09.xmldsig.impl.SignatureTypeImpl$1ObjectList',
+  'org.w3.x2000.x09.xmldsig.impl.SignedInfoTypeImpl$1ReferenceList',
+  'org.w3.x2000.x09.xmldsig.impl.TransformTypeImpl$1XPathList',
+  'org.w3.x2000.x09.xmldsig.impl.TransformTypeImpl$2XPathList',
+  'ucar.ma2.DataType',
+  'ucar.nc2.Attribute',
+  'ucar.nc2.Dimension',
+  'ucar.nc2.Group',
+  'ucar.nc2.NetcdfFile',
+  'ucar.nc2.Variable',
+  'ucar.nc2.dataset.NetcdfDataset'
 ]
diff --git a/plugins/mapper-attachments/licenses/bcmail-jdk15on-1.52.jar.sha1 b/plugins/mapper-attachments/licenses/bcmail-jdk15on-1.52.jar.sha1
deleted file mode 100644
index de084c948f4..00000000000
--- a/plugins/mapper-attachments/licenses/bcmail-jdk15on-1.52.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4995a870400e1554d1c7ed2afcb5d198fae12db9
diff --git a/plugins/mapper-attachments/licenses/bcmail-jdk15on-1.54.jar.sha1 b/plugins/mapper-attachments/licenses/bcmail-jdk15on-1.54.jar.sha1
new file mode 100644
index 00000000000..79da45c5c42
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/bcmail-jdk15on-1.54.jar.sha1
@@ -0,0 +1 @@
+9d9b5432b4b29ef4a853223bc6e19379ef116cca
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/bcpkix-jdk15on-1.52.jar.sha1 b/plugins/mapper-attachments/licenses/bcpkix-jdk15on-1.52.jar.sha1
deleted file mode 100644
index 489ceeaaf36..00000000000
--- a/plugins/mapper-attachments/licenses/bcpkix-jdk15on-1.52.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b8ffac2bbc6626f86909589c8cc63637cc936504
diff --git a/plugins/mapper-attachments/licenses/bcpkix-jdk15on-1.54.jar.sha1 b/plugins/mapper-attachments/licenses/bcpkix-jdk15on-1.54.jar.sha1
new file mode 100644
index 00000000000..2d0c3cf4e27
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/bcpkix-jdk15on-1.54.jar.sha1
@@ -0,0 +1 @@
+b11bfee99bb11eea344de6e4a07fe89212c55c02
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/bcprov-jdk15on-1.52.jar.sha1 b/plugins/mapper-attachments/licenses/bcprov-jdk15on-1.52.jar.sha1
deleted file mode 100644
index 14ecc1be40b..00000000000
--- a/plugins/mapper-attachments/licenses/bcprov-jdk15on-1.52.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-88a941faf9819d371e3174b5ed56a3f3f7d73269
diff --git a/plugins/mapper-attachments/licenses/bcprov-jdk15on-1.54.jar.sha1 b/plugins/mapper-attachments/licenses/bcprov-jdk15on-1.54.jar.sha1
new file mode 100644
index 00000000000..fcda646b42a
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/bcprov-jdk15on-1.54.jar.sha1
@@ -0,0 +1 @@
+1acdedeb89f1d950d67b73d481eb7736df65eedb
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/fontbox-1.8.10.jar.sha1 b/plugins/mapper-attachments/licenses/fontbox-1.8.10.jar.sha1
deleted file mode 100644
index ce7f9f5d49c..00000000000
--- a/plugins/mapper-attachments/licenses/fontbox-1.8.10.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-41776c7713e3f3a1ce688bd96459fc597298c340
diff --git a/plugins/mapper-attachments/licenses/fontbox-2.0.1.jar.sha1 b/plugins/mapper-attachments/licenses/fontbox-2.0.1.jar.sha1
new file mode 100644
index 00000000000..0668199b242
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/fontbox-2.0.1.jar.sha1
@@ -0,0 +1 @@
+b9d4f0993e015f3f1ce0be9e7300cf62dd7a7f15
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/jempbox-1.8.10.jar.sha1 b/plugins/mapper-attachments/licenses/jempbox-1.8.10.jar.sha1
deleted file mode 100644
index 5a7b1997208..00000000000
--- a/plugins/mapper-attachments/licenses/jempbox-1.8.10.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-40df4e4ca884aadc20b82d5abd0a3679774c55a6
diff --git a/plugins/mapper-attachments/licenses/jempbox-1.8.12.jar.sha1 b/plugins/mapper-attachments/licenses/jempbox-1.8.12.jar.sha1
new file mode 100644
index 00000000000..0e3dcf4573b
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/jempbox-1.8.12.jar.sha1
@@ -0,0 +1 @@
+426450c573c19f6f2c751a7a52c11931b712c9f6
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/pdfbox-1.8.10.jar.sha1 b/plugins/mapper-attachments/licenses/pdfbox-1.8.10.jar.sha1
deleted file mode 100644
index 98ce1f9d98c..00000000000
--- a/plugins/mapper-attachments/licenses/pdfbox-1.8.10.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-bc5d1254495be36d0a3b3d6c35f88d05200b9311
diff --git a/plugins/mapper-attachments/licenses/pdfbox-2.0.1.jar.sha1 b/plugins/mapper-attachments/licenses/pdfbox-2.0.1.jar.sha1
new file mode 100644
index 00000000000..1014db34044
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/pdfbox-2.0.1.jar.sha1
@@ -0,0 +1 @@
+dbc69649118b7eff278f228c070a40ee559e1f62
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/poi-3.13.jar.sha1 b/plugins/mapper-attachments/licenses/poi-3.13.jar.sha1
deleted file mode 100644
index 09063c1e5e0..00000000000
--- a/plugins/mapper-attachments/licenses/poi-3.13.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0f59f504ba8c521e61e25f417ec652fd485010f3
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/poi-3.15-beta1.jar.sha1 b/plugins/mapper-attachments/licenses/poi-3.15-beta1.jar.sha1
new file mode 100644
index 00000000000..6049604dd97
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/poi-3.15-beta1.jar.sha1
@@ -0,0 +1 @@
+048bb8326b81323631d9ceb4236cfbd382e56da2
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/poi-ooxml-3.13.jar.sha1 b/plugins/mapper-attachments/licenses/poi-ooxml-3.13.jar.sha1
deleted file mode 100644
index 16784299855..00000000000
--- a/plugins/mapper-attachments/licenses/poi-ooxml-3.13.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c364a8f5422d613e3a56db3b4b889f2989d7ee73
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/poi-ooxml-3.15-beta1.jar.sha1 b/plugins/mapper-attachments/licenses/poi-ooxml-3.15-beta1.jar.sha1
new file mode 100644
index 00000000000..c3cf49d9246
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/poi-ooxml-3.15-beta1.jar.sha1
@@ -0,0 +1 @@
+81085a47fdf0d74d473d605c6b3784e26731842e
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/poi-ooxml-schemas-3.13.jar.sha1 b/plugins/mapper-attachments/licenses/poi-ooxml-schemas-3.13.jar.sha1
deleted file mode 100644
index b5a3a05c489..00000000000
--- a/plugins/mapper-attachments/licenses/poi-ooxml-schemas-3.13.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-56fb0b9f3ffc3d7f7fc9b59e17b5fa2c3ab921e7
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/poi-ooxml-schemas-3.15-beta1.jar.sha1 b/plugins/mapper-attachments/licenses/poi-ooxml-schemas-3.15-beta1.jar.sha1
new file mode 100644
index 00000000000..afd3b676d08
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/poi-ooxml-schemas-3.15-beta1.jar.sha1
@@ -0,0 +1 @@
+f8bc979ad79908a99483337f1ca2edf78558ac20
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/poi-scratchpad-3.13.jar.sha1 b/plugins/mapper-attachments/licenses/poi-scratchpad-3.13.jar.sha1
deleted file mode 100644
index cc61780e2a5..00000000000
--- a/plugins/mapper-attachments/licenses/poi-scratchpad-3.13.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-09d763275e6c7fa05d47e2581606748669e88c55
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/poi-scratchpad-3.15-beta1.jar.sha1 b/plugins/mapper-attachments/licenses/poi-scratchpad-3.15-beta1.jar.sha1
new file mode 100644
index 00000000000..7056a9fa49e
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/poi-scratchpad-3.15-beta1.jar.sha1
@@ -0,0 +1 @@
+f4e276aaf97a60a1156388c9e38069122b7ea914
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/tika-core-1.11.jar.sha1 b/plugins/mapper-attachments/licenses/tika-core-1.11.jar.sha1
deleted file mode 100644
index a6dfd778a9c..00000000000
--- a/plugins/mapper-attachments/licenses/tika-core-1.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d37a6b9080c8361e47b2050f69833fd61501ede9
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/tika-core-1.13.jar.sha1 b/plugins/mapper-attachments/licenses/tika-core-1.13.jar.sha1
new file mode 100644
index 00000000000..cfc36a450bd
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/tika-core-1.13.jar.sha1
@@ -0,0 +1 @@
+1305c798d41d1d7bbf12cb7c0ca184c98eed25ad
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/tika-parsers-1.11.jar.sha1 b/plugins/mapper-attachments/licenses/tika-parsers-1.11.jar.sha1
deleted file mode 100644
index fbbd59efaf9..00000000000
--- a/plugins/mapper-attachments/licenses/tika-parsers-1.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-355dc05d842ed223fc682da472229473ba706d68
\ No newline at end of file
diff --git a/plugins/mapper-attachments/licenses/tika-parsers-1.13.jar.sha1 b/plugins/mapper-attachments/licenses/tika-parsers-1.13.jar.sha1
new file mode 100644
index 00000000000..7fb2755d545
--- /dev/null
+++ b/plugins/mapper-attachments/licenses/tika-parsers-1.13.jar.sha1
@@ -0,0 +1 @@
+374fde67b9d35f785534b0e6c4953533c31bab5f
\ No newline at end of file
diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle
index a083309891e..a7b8d8233eb 100644
--- a/plugins/repository-s3/build.gradle
+++ b/plugins/repository-s3/build.gradle
@@ -18,7 +18,7 @@
  */
 
 esplugin {
-  description 'The S3 repository plugin adds S3 repositories.'
+  description 'The S3 repository plugin adds S3 repositories'
   classname 'org.elasticsearch.plugin.repository.s3.S3RepositoryPlugin'
 }
 
@@ -36,11 +36,16 @@ dependencies {
   compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile "com.fasterxml.jackson.core:jackson-databind:2.5.3"
   compile "com.fasterxml.jackson.core:jackson-annotations:2.5.0"
+
+  // HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
+  // and whitelist this hack in JarHell 
+  compile 'javax.xml.bind:jaxb-api:2.2.2'
 }
 
 dependencyLicenses {
   mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk'
   mapping from: /jackson-.*/, to: 'jackson'
+  mapping from: /jaxb-.*/, to: 'jaxb'
 }
 
 test {
@@ -60,4 +65,107 @@ thirdPartyAudit.excludes = [
   'org.apache.avalon.framework.logger.Logger', 
   'org.apache.log.Hierarchy', 
   'org.apache.log.Logger',
+
+  // jarhell with jdk (intentionally, because jaxb was removed from default modules in java 9)
+  'javax.xml.bind.Binder',
+  'javax.xml.bind.ContextFinder$1',
+  'javax.xml.bind.ContextFinder',
+  'javax.xml.bind.DataBindingException',
+  'javax.xml.bind.DatatypeConverter',
+  'javax.xml.bind.DatatypeConverterImpl$CalendarFormatter',
+  'javax.xml.bind.DatatypeConverterImpl',
+  'javax.xml.bind.DatatypeConverterInterface',
+  'javax.xml.bind.Element',
+  'javax.xml.bind.GetPropertyAction',
+  'javax.xml.bind.JAXB$Cache',
+  'javax.xml.bind.JAXB',
+  'javax.xml.bind.JAXBContext',
+  'javax.xml.bind.JAXBElement$GlobalScope',
+  'javax.xml.bind.JAXBElement',
+  'javax.xml.bind.JAXBException',
+  'javax.xml.bind.JAXBIntrospector',
+  'javax.xml.bind.JAXBPermission',
+  'javax.xml.bind.MarshalException',
+  'javax.xml.bind.Marshaller$Listener',
+  'javax.xml.bind.Marshaller',
+  'javax.xml.bind.Messages',
+  'javax.xml.bind.NotIdentifiableEvent',
+  'javax.xml.bind.ParseConversionEvent',
+  'javax.xml.bind.PrintConversionEvent',
+  'javax.xml.bind.PropertyException',
+  'javax.xml.bind.SchemaOutputResolver',
+  'javax.xml.bind.TypeConstraintException',
+  'javax.xml.bind.UnmarshalException',
+  'javax.xml.bind.Unmarshaller$Listener',
+  'javax.xml.bind.Unmarshaller',
+  'javax.xml.bind.UnmarshallerHandler',
+  'javax.xml.bind.ValidationEvent',
+  'javax.xml.bind.ValidationEventHandler',
+  'javax.xml.bind.ValidationEventLocator',
+  'javax.xml.bind.ValidationException',
+  'javax.xml.bind.Validator',
+  'javax.xml.bind.WhiteSpaceProcessor',
+  'javax.xml.bind.annotation.DomHandler',
+  'javax.xml.bind.annotation.W3CDomHandler',
+  'javax.xml.bind.annotation.XmlAccessOrder',
+  'javax.xml.bind.annotation.XmlAccessType',
+  'javax.xml.bind.annotation.XmlAccessorOrder',
+  'javax.xml.bind.annotation.XmlAccessorType',
+  'javax.xml.bind.annotation.XmlAnyAttribute',
+  'javax.xml.bind.annotation.XmlAnyElement',
+  'javax.xml.bind.annotation.XmlAttachmentRef',
+  'javax.xml.bind.annotation.XmlAttribute',
+  'javax.xml.bind.annotation.XmlElement$DEFAULT',
+  'javax.xml.bind.annotation.XmlElement',
+  'javax.xml.bind.annotation.XmlElementDecl$GLOBAL',
+  'javax.xml.bind.annotation.XmlElementDecl',
+  'javax.xml.bind.annotation.XmlElementRef$DEFAULT',
+  'javax.xml.bind.annotation.XmlElementRef',
+  'javax.xml.bind.annotation.XmlElementRefs',
+  'javax.xml.bind.annotation.XmlElementWrapper',
+  'javax.xml.bind.annotation.XmlElements',
+  'javax.xml.bind.annotation.XmlEnum',
+  'javax.xml.bind.annotation.XmlEnumValue',
+  'javax.xml.bind.annotation.XmlID',
+  'javax.xml.bind.annotation.XmlIDREF',
+  'javax.xml.bind.annotation.XmlInlineBinaryData',
+  'javax.xml.bind.annotation.XmlList',
+  'javax.xml.bind.annotation.XmlMimeType',
+  'javax.xml.bind.annotation.XmlMixed',
+  'javax.xml.bind.annotation.XmlNs',
+  'javax.xml.bind.annotation.XmlNsForm',
+  'javax.xml.bind.annotation.XmlRegistry',
+  'javax.xml.bind.annotation.XmlRootElement',
+  'javax.xml.bind.annotation.XmlSchema',
+  'javax.xml.bind.annotation.XmlSchemaType$DEFAULT',
+  'javax.xml.bind.annotation.XmlSchemaType',
+  'javax.xml.bind.annotation.XmlSchemaTypes',
+  'javax.xml.bind.annotation.XmlSeeAlso',
+  'javax.xml.bind.annotation.XmlTransient',
+  'javax.xml.bind.annotation.XmlType$DEFAULT',
+  'javax.xml.bind.annotation.XmlType',
+  'javax.xml.bind.annotation.XmlValue',
+  'javax.xml.bind.annotation.adapters.CollapsedStringAdapter',
+  'javax.xml.bind.annotation.adapters.HexBinaryAdapter',
+  'javax.xml.bind.annotation.adapters.NormalizedStringAdapter',
+  'javax.xml.bind.annotation.adapters.XmlAdapter',
+  'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter$DEFAULT',
+  'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter',
+  'javax.xml.bind.annotation.adapters.XmlJavaTypeAdapters',
+  'javax.xml.bind.attachment.AttachmentMarshaller',
+  'javax.xml.bind.attachment.AttachmentUnmarshaller',
+  'javax.xml.bind.helpers.AbstractMarshallerImpl',
+  'javax.xml.bind.helpers.AbstractUnmarshallerImpl',
+  'javax.xml.bind.helpers.DefaultValidationEventHandler',
+  'javax.xml.bind.helpers.Messages',
+  'javax.xml.bind.helpers.NotIdentifiableEventImpl',
+  'javax.xml.bind.helpers.ParseConversionEventImpl',
+  'javax.xml.bind.helpers.PrintConversionEventImpl',
+  'javax.xml.bind.helpers.ValidationEventImpl',
+  'javax.xml.bind.helpers.ValidationEventLocatorImpl',
+  'javax.xml.bind.util.JAXBResult',
+  'javax.xml.bind.util.JAXBSource$1',
+  'javax.xml.bind.util.JAXBSource',
+  'javax.xml.bind.util.Messages',
+  'javax.xml.bind.util.ValidationEventCollector'
 ]
diff --git a/plugins/repository-s3/licenses/jaxb-LICENSE.txt b/plugins/repository-s3/licenses/jaxb-LICENSE.txt
new file mode 100644
index 00000000000..a3e62b08787
--- /dev/null
+++ b/plugins/repository-s3/licenses/jaxb-LICENSE.txt
@@ -0,0 +1,705 @@
+COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0
+
+1. Definitions.
+
+   1.1. Contributor. means each individual or  entity  that  creates  or
+contributes to the creation of Modifications.
+
+   1.2. Contributor Version.  means  the  combination  of  the  Original
+Software,  prior  Modifications  used by a Contributor (if any), and the
+Modifications made by that particular Contributor.
+
+   1.3. Covered Software.  means  (a)  the  Original  Software,  or  (b)
+Modifications,  or  (c)  the  combination  of  files containing Original
+Software with files containing Modifications,  in  each  case  including
+portions thereof.
+
+   1.4. Executable. means the Covered Software in any  form  other  than
+Source Code.
+
+   1.5. Initial Developer. means the individual  or  entity  that  first
+makes Original Software available under this License.
+
+   1.6. Larger Work. means a work which  combines  Covered  Software  or
+portions thereof with code not governed by the terms of this License.
+
+   1.7. License. means this document.
+
+   1.8. Licensable. means having the right  to  grant,  to  the  maximum
+extent   possible,   whether  at  the  time  of  the  initial  grant  or
+subsequently acquired, any and all of the rights conveyed herein.
+
+   1.9. Modifications. means the Source Code and Executable form of  any
+of the following:
+
+        A. Any file that results from an addition to, deletion  from  or
+modification  of  the contents of a file containing Original Software or
+previous Modifications;
+
+        B. Any new file that contains any part of the Original  Software
+or previous Modification; or
+
+        C. Any new file that is contributed or otherwise made  available
+under the terms of this License.
+
+   1.10. Original Software. means the Source Code and Executable form of
+computer software code that is originally released under this License.
+
+   1.11.  Patent  Claims.  means  any  patent  claim(s),  now  owned  or
+hereafter  acquired,  including without limitation, method, process, and
+apparatus claims, in any patent Licensable by grantor.
+
+   1.12. Source Code. means (a) the common  form  of  computer  software
+code  in  which  modifications are made and (b) associated documentation
+included in or with such code.
+
+   1.13. You.  (or  .Your.)  means  an  individual  or  a  legal  entity
+exercising  rights  under,  and complying with all of the terms of, this
+License. For legal entities, .You. includes any entity  which  controls,
+is  controlled  by, or is under common control with You. For purposes of
+this definition, .control. means (a) the power, direct or  indirect,  to
+cause the direction or management of such entity, whether by contract or
+otherwise,  or  (b)  ownership  of  more than fifty percent (50%) of the
+outstanding shares or beneficial ownership of such entity.
+
+2. License Grants.
+
+      2.1. The Initial Developer Grant.
+
+      Conditioned upon  Your  compliance  with  Section  3.1  below  and
+subject  to  third  party  intellectual  property  claims,  the  Initial
+Developer hereby grants You a  world-wide,  royalty-free,  non-exclusive
+license:
+
+         (a) under intellectual property rights (other  than  patent  or
+trademark)  Licensable  by Initial Developer, to use, reproduce, modify,
+display, perform, sublicense and distribute the  Original  Software  (or
+portions  thereof),  with  or without Modifications, and/or as part of a
+Larger Work; and
+
+         (b) under Patent Claims  infringed  by  the  making,  using  or
+selling  of  Original Software, to make, have made, use, practice, sell,
+and offer for sale, and/or otherwise dispose of  the  Original  Software
+(or portions thereof).
+
+        (c)  The  licenses  granted  in  Sections  2.1(a)  and  (b)  are
+effective  on  the date Initial Developer first distributes or otherwise
+makes the Original Software available to a third party under  the  terms
+of this License.
+
+        (d) Notwithstanding Section 2.1(b) above, no patent  license  is
+granted: (1) for code that You delete from the Original Software, or (2)
+for  infringements  caused  by:  (i)  the  modification  of the Original
+Software, or (ii) the combination of the Original  Software  with  other
+software or devices.
+
+    2.2. Contributor Grant.
+
+    Conditioned upon Your compliance with Section 3.1 below and  subject
+to  third  party  intellectual  property claims, each Contributor hereby
+grants You a world-wide, royalty-free, non-exclusive license:
+
+        (a) under intellectual property rights  (other  than  patent  or
+trademark) Licensable by Contributor to use, reproduce, modify, display,
+perform,  sublicense  and  distribute  the Modifications created by such
+Contributor (or portions thereof), either on an unmodified  basis,  with
+other Modifications, as Covered Software and/or as part of a Larger Work;
+and
+
+        (b) under Patent Claims  infringed  by  the  making,  using,  or
+selling of Modifications made by that Contributor either alone and/or in
+combination   with   its   Contributor  Version  (or  portions  of  such
+combination), to make, use, sell, offer  for  sale,  have  made,  and/or
+otherwise  dispose  of:  (1)  Modifications made by that Contributor (or
+portions thereof); and (2) the combination of Modifications made by that
+Contributor  with  its  Contributor  Version  (or   portions   of   such
+combination).
+
+        (c) The licenses granted  in  Sections  2.2(a)  and  2.2(b)  are
+effective  on  the date Contributor first distributes or otherwise makes
+the Modifications available to a third party.
+
+        (d) Notwithstanding Section 2.2(b) above, no patent  license  is
+granted:  (1)  for  any  code  that  Contributor  has  deleted  from the
+Contributor Version; (2) for infringements caused by:  (i)  third  party
+modifications  of  Contributor  Version,  or  (ii)  the  combination  of
+Modifications made by that Contributor with other  software  (except  as
+part  of  the Contributor Version) or other devices; or (3) under Patent
+Claims infringed by Covered Software in  the  absence  of  Modifications
+made by that Contributor.
+
+3. Distribution Obligations.
+
+      3.1. Availability of Source Code.
+      Any  Covered  Software  that  You  distribute  or  otherwise  make
+available  in Executable form must also be made available in Source Code
+form and that Source Code form must be distributed only under the  terms
+of this License. You must include a copy of this License with every copy
+of  the  Source  Code  form  of  the  Covered Software You distribute or
+otherwise make available. You must inform recipients of any such Covered
+Software in Executable form as to  how  they  can  obtain  such  Covered
+Software  in  Source  Code  form  in a reasonable manner on or through a
+medium customarily used for software exchange.
+
+      3.2. Modifications.
+      The Modifications that You create or to which You  contribute  are
+governed  by  the  terms of this License. You represent that You believe
+Your  Modifications  are  Your  original  creation(s)  and/or  You  have
+sufficient rights to grant the rights conveyed by this License.
+
+      3.3. Required Notices.
+      You must include a notice  in  each  of  Your  Modifications  that
+identifies  You  as  the  Contributor  of  the Modification. You may not
+remove or alter any copyright, patent  or  trademark  notices  contained
+within  the  Covered  Software,  or  any  notices  of  licensing  or any
+descriptive text giving attribution to any Contributor  or  the  Initial
+Developer.
+
+      3.4. Application of Additional Terms.
+      You may not offer or impose any terms on any Covered  Software  in
+Source Code form that alters or restricts the applicable version of this
+License  or  the  recipients. rights hereunder. You may choose to offer,
+and to charge a fee  for,  warranty,  support,  indemnity  or  liability
+obligations  to one or more recipients of Covered Software. However, you
+may do so only on Your own behalf, and not  on  behalf  of  the  Initial
+Developer or any Contributor. You must make it absolutely clear that any
+such  warranty, support, indemnity or liability obligation is offered by
+You alone, and You hereby agree to indemnify the Initial  Developer  and
+every Contributor for any liability incurred by the Initial Developer or
+such  Contributor  as  a  result  of  warranty,  support,  indemnity  or
+liability terms You offer.
+
+      3.5. Distribution of Executable Versions.
+      You may distribute the Executable form  of  the  Covered  Software
+under  the terms of this License or under the terms of a license of Your
+choice, which may contain terms different from  this  License,  provided
+that  You  are in compliance with the terms of this License and that the
+license for the Executable form does not attempt to limit or  alter  the
+recipient.s  rights in the Source Code form from the rights set forth in
+this License. If You distribute the Covered Software in Executable  form
+under  a  different  license, You must make it absolutely clear that any
+terms which differ from this License are offered by You  alone,  not  by
+the  Initial Developer or Contributor. You hereby agree to indemnify the
+Initial Developer and every Contributor for any  liability  incurred  by
+the  Initial Developer or such Contributor as a result of any such terms
+You offer.
+
+      3.6. Larger Works.
+      You may create a Larger Work by combining  Covered  Software  with
+other  code not governed by the terms of this License and distribute the
+Larger Work as a single product. In such a case, You must make sure  the
+requirements of this License are fulfilled for the Covered Software.
+
+4. Versions of the License.
+
+      4.1. New Versions.
+      Sun Microsystems, Inc. is the  initial  license  steward  and  may
+publish  revised  and/or new versions of this License from time to time.
+Each version will be given a distinguishing version  number.  Except  as
+provided  in  Section 4.3, no one other than the license steward has the
+right to modify this License.
+
+      4.2. Effect of New Versions.
+      You may always continue to use, distribute or otherwise  make  the
+Covered Software available under the terms of the version of the License
+under which You originally received the Covered Software. If the Initial
+Developer includes a notice in the Original Software prohibiting it from
+being  distributed  or  otherwise  made  available  under any subsequent
+version of the  License,  You  must  distribute  and  make  the  Covered
+Software  available  under the terms of the version of the License under
+which You originally received the Covered Software. Otherwise,  You  may
+also  choose  to  use, distribute or otherwise make the Covered Software
+available under the terms of  any  subsequent  version  of  the  License
+published by the license steward.
+
+      4.3. Modified Versions.
+      When You are an Initial Developer and You want  to  create  a  new
+license  for  Your  Original Software, You may create and use a modified
+version of this License if You: (a) rename the license  and  remove  any
+references  to  the name of the license steward (except to note that the
+license differs from this License); and (b) otherwise make it clear that
+the license contains terms which differ from this License.
+
+5. DISCLAIMER OF WARRANTY.
+
+   COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN .AS IS.  BASIS,
+WITHOUT  WARRANTY  OF  ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
+WITHOUT LIMITATION, WARRANTIES THAT THE  COVERED  SOFTWARE  IS  FREE  OF
+DEFECTS,  MERCHANTABLE,  FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING.
+THE ENTIRE RISK AS  TO  THE  QUALITY  AND  PERFORMANCE  OF  THE  COVERED
+SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY
+RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME
+THE  COST  OF  ANY  NECESSARY  SERVICING,  REPAIR  OR  CORRECTION.  THIS
+DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO
+USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER  EXCEPT  UNDER  THIS
+DISCLAIMER.
+
+6. TERMINATION.
+
+      6.1. This License and the rights granted hereunder will  terminate
+automatically  if  You fail to comply with terms herein and fail to cure
+such breach within 30 days of becoming aware of the  breach.  Provisions
+which,  by their nature, must remain in effect beyond the termination of
+this License shall survive.
+
+      6.2.  If  You  assert  a  patent  infringement  claim   (excluding
+declaratory judgment actions) against Initial Developer or a Contributor
+(the Initial Developer or Contributor against whom You assert such claim
+is  referred to as .Participant.) alleging that the Participant Software
+(meaning the Contributor Version where the Participant is a  Contributor
+or the Original Software where the Participant is the Initial Developer)
+directly  or  indirectly  infringes  any patent, then any and all rights
+granted directly or indirectly to You by such Participant,  the  Initial
+Developer  (if  the  Initial  Developer  is not the Participant) and all
+Contributors under Sections 2.1 and/or 2.2 of this License  shall,  upon
+60   days   notice   from   Participant   terminate   prospectively  and
+automatically at the expiration of such 60 day notice period, unless  if
+within  such  60  day period You withdraw Your claim with respect to the
+Participant Software against such  Participant  either  unilaterally  or
+pursuant to a written agreement with Participant.
+
+      6.3. In the event of termination under Sections 6.1 or 6.2  above,
+all  end  user  licenses  that  have  been validly granted by You or any
+distributor hereunder prior to termination (excluding  licenses  granted
+to You by any distributor) shall survive termination.
+
+7. LIMITATION OF LIABILITY.
+
+   UNDER NO CIRCUMSTANCES  AND  UNDER  NO  LEGAL  THEORY,  WHETHER  TORT
+(INCLUDING  NEGLIGENCE),  CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL
+DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE,
+OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR  ANY
+INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER
+INCLUDING,  WITHOUT  LIMITATION,  DAMAGES  FOR  LOST  PROFITS,  LOSS  OF
+GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL
+OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL  HAVE  BEEN
+INFORMED  OF  THE  POSSIBILITY  OF  SUCH  DAMAGES.  THIS  LIMITATION  OF
+LIABILITY SHALL NOT APPLY TO LIABILITY  FOR  DEATH  OR  PERSONAL  INJURY
+RESULTING  FROM  SUCH  PARTY.S  NEGLIGENCE  TO THE EXTENT APPLICABLE LAW
+PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION
+OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS  EXCLUSION
+AND LIMITATION MAY NOT APPLY TO YOU.
+
+8. U.S. GOVERNMENT END USERS.
+
+   The Covered Software is a .commercial item,. as that term is  defined
+in  48  C.F.R.  2.101  (Oct.  1995),  consisting of .commercial computer
+software. (as that term is defined at 48  C.F.R.  º  252.227-7014(a)(1))
+and  .commercial computer software documentation. as such terms are used
+in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R.  12.212  and
+48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all U.S. Government
+End  Users  acquire  Covered  Software  with only those rights set forth
+herein. This U.S. Government Rights clause is in lieu of, and supersedes,
+any other FAR,  DFAR,  or  other  clause  or  provision  that  addresses
+Government rights in computer software under this License.
+
+9. MISCELLANEOUS.
+
+   This License represents the  complete  agreement  concerning  subject
+matter  hereof.  If  any  provision  of  this  License  is  held  to  be
+unenforceable, such provision shall  be  reformed  only  to  the  extent
+necessary  to make it enforceable. This License shall be governed by the
+law of the jurisdiction specified  in  a  notice  contained  within  the
+Original Software (except to the extent applicable law, if any, provides
+otherwise),  excluding  such  jurisdiction.s conflict-of-law provisions.
+Any litigation  relating  to  this  License  shall  be  subject  to  the
+jurisdiction  of  the  courts  located  in  the  jurisdiction  and venue
+specified in a notice contained within the Original Software,  with  the
+losing party responsible for costs, including, without limitation, court
+costs  and  reasonable  attorneys. fees and expenses. The application of
+the United Nations Convention on Contracts for the International Sale of
+Goods is expressly excluded. Any law or regulation which  provides  that
+the  language of a contract shall be construed against the drafter shall
+not apply to this License. You agree that You alone are responsible  for
+compliance with the United States export administration regulations (and
+the  export control laws and regulation of any other countries) when You
+use, distribute or otherwise make available any Covered Software.
+
+10. RESPONSIBILITY FOR CLAIMS.
+
+   As between Initial Developer and  the  Contributors,  each  party  is
+responsible  for claims and damages arising, directly or indirectly, out
+of its utilization of rights under this License and You  agree  to  work
+with   Initial   Developer   and   Contributors   to   distribute   such
+responsibility on an equitable basis.  Nothing  herein  is  intended  or
+shall be deemed to constitute any admission of liability.
+
+   NOTICE  PURSUANT  TO  SECTION  9  OF  THE  COMMON   DEVELOPMENT   AND
+DISTRIBUTION LICENSE (CDDL)
+
+   The code released under the CDDL shall be governed by the laws of the
+State  of  California  (excluding   conflict-of-law   provisions).   Any
+litigation relating to this License shall be subject to the jurisdiction
+of  the  Federal  Courts  of the Northern District of California and the
+state courts of the State of California, with venue lying in Santa Clara
+County, California.
+
+
+The GNU General Public License (GPL) Version 2, June 1991
+
+
+Copyright (C) 1989, 1991 Free Software Foundation, Inc. 59 Temple Place,
+Suite 330, Boston, MA 02111-1307 USA
+
+Everyone is permitted to copy and distribute  verbatim  copies  of  this
+license document, but changing it is not allowed.
+
+Preamble
+
+The licenses for most software are designed to take away your freedom to
+share and change it. By contrast, the  GNU  General  Public  License  is
+intended to guarantee your freedom to share and change free software--to
+make  sure  the  software is free for all its users. This General Public
+License applies to most of the Free Software Foundation's  software  and
+to  any other program whose authors commit to using it. (Some other Free
+Software Foundation software is  covered  by  the  GNU  Library  General
+Public License instead.) You can apply it to your programs, too.
+
+When we speak of free software, we are referring to freedom, not  price.
+Our  General Public Licenses are designed to make sure that you have the
+freedom to distribute copies of  free  software  (and  charge  for  this
+service  if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces  of  it  in  new
+free programs; and that you know you can do these things.
+
+To protect your rights, we need to make restrictions that forbid  anyone
+to  deny  you  these rights or to ask you to surrender the rights. These
+restrictions translate  to  certain  responsibilities  for  you  if  you
+distribute copies of the software, or if you modify it.
+
+For example, if you distribute copies of such a program, whether  gratis
+or for a fee, you must give the recipients all the rights that you have.
+You  must  make sure that they, too, receive or can get the source code.
+And you must show them these terms so they know their rights.
+
+We protect your rights with two steps: (1) copyright the  software,  and
+(2)  offer  you  this  license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+Also, for each author's protection and ours, we  want  to  make  certain
+that  everyone  understands  that  there  is  no  warranty for this free
+software. If the software is modified by someone else and passed on,  we
+want  its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the  original
+authors' reputations.
+
+Finally, any free program is threatened constantly by software  patents.
+We  wish  to avoid the danger that redistributors of a free program will
+individually obtain  patent  licenses,  in  effect  making  the  program
+proprietary. To prevent this, we have made it clear that any patent must
+be licensed for everyone's free use or not licensed at all.
+
+The  precise  terms  and  conditions  for  copying,   distribution   and
+modification follow.
+
+
+TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+0. This License applies to any program or other work  which  contains  a
+notice placed by the copyright holder saying it may be distributed under
+the  terms  of this General Public License. The "Program", below, refers
+to any such program or work, and a "work based  on  the  Program"  means
+either  the  Program or any derivative work under copyright law: that is
+to say, a work containing  the  Program  or  a  portion  of  it,  either
+verbatim  or with modifications and/or translated into another language.
+(Hereinafter, translation is included without  limitation  in  the  term
+"modification".) Each licensee is addressed as "you".
+
+Activities other than copying, distribution  and  modification  are  not
+covered  by this License; they are outside its scope. The act of running
+the Program is not restricted,  and  the  output  from  the  Program  is
+covered  only  if  its  contents  constitute a work based on the Program
+(independent of having been made by running the Program).  Whether  that
+is true depends on what the Program does.
+
+1. You may copy and distribute verbatim copies of the  Program's  source
+code  as  you receive it, in any medium, provided that you conspicuously
+and appropriately publish on each copy an appropriate  copyright  notice
+and  disclaimer  of  warranty; keep intact all the notices that refer to
+this License and to the absence of any  warranty;  and  give  any  other
+recipients of the Program a copy of this License along with the Program.
+
+You may charge a fee for the physical act of transferring  a  copy,  and
+you may at your option offer warranty protection in exchange for a fee.
+
+2. You may modify your copy or copies of the Program or any  portion  of
+it,  thus  forming  a work based on the Program, and copy and distribute
+such modifications or work under the terms of Section 1 above,  provided
+that you also meet all of these conditions:
+
+   a) You must cause the  modified  files  to  carry  prominent  notices
+stating that you changed the files and the date of any change.
+
+   b) You must cause any work that you distribute or  publish,  that  in
+whole  or  in  part  contains or is derived from the Program or any part
+thereof, to be licensed as a whole at no charge  to  all  third  parties
+under the terms of this License.
+
+   c) If the modified program normally reads commands interactively when
+run, you must cause it, when started running for such interactive use in
+the most ordinary way, to print or display an announcement including  an
+appropriate  copyright notice and a notice that there is no warranty (or
+else,  saying  that  you  provide  a  warranty)  and  that   users   may
+redistribute  the  program  under these conditions, and telling the user
+how to view a copy of this License. (Exception: if the Program itself is
+interactive but does not normally print such an announcement, your  work
+based on the Program is not required to print an announcement.)
+
+These  requirements  apply  to  the  modified  work  as  a   whole.   If
+identifiable sections of that work are not derived from the Program, and
+can   be   reasonably  considered  independent  and  separate  works  in
+themselves, then this License, and its terms,  do  not  apply  to  those
+sections  when  you  distribute  them  as  separate  works. But when you
+distribute the same sections as part of a whole which is a work based on
+the Program, the distribution of the whole must be on the terms of  this
+License,  whose  permissions  for  other  licensees extend to the entire
+whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim  rights  or  contest
+your  rights  to  work written entirely by you; rather, the intent is to
+exercise  the  right  to  control  the  distribution  of  derivative  or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on  the  Program
+with  the Program (or with a work based on the Program) on a volume of a
+storage or distribution medium does not bring the other work  under  the
+scope of this License.
+
+3. You may copy and distribute the Program (or a work based on it, under
+Section 2) in object code or executable form under the terms of Sections
+1 and 2 above provided that you also do one of the following:
+
+   a) Accompany it  with  the  complete  corresponding  machine-readable
+source code, which must be distributed under the terms of Sections 1 and
+2 above on a medium customarily used for software interchange; or,
+
+   b) Accompany it with a written offer, valid for at least three years,
+to give any third party,  for  a  charge  no  more  than  your  cost  of
+physically  performing  source distribution, a complete machine-readable
+copy of the corresponding source code, to be distributed under the terms
+of Sections 1 and 2 above on a  medium  customarily  used  for  software
+interchange; or,
+
+   c) Accompany it with the information you received as to the offer  to
+distribute  corresponding source code. (This alternative is allowed only
+for noncommercial distribution and only if you received the  program  in
+object  code  or  executable  form  with  such  an offer, in accord with
+Subsection b above.)
+
+The source code for a work means the preferred  form  of  the  work  for
+making modifications to it. For an executable work, complete source code
+means  all  the  source  code  for  all  modules  it  contains, plus any
+associated interface definition files, plus the scripts used to  control
+compilation  and  installation  of the executable. However, as a special
+exception, the source code distributed need not include anything that is
+normally distributed (in either source or binary form)  with  the  major
+components  (compiler,  kernel,  and  so  on) of the operating system on
+which the executable runs, unless that component itself accompanies  the
+executable.
+
+If distribution of executable or object code is made by offering  access
+to copy from a designated place, then offering equivalent access to copy
+the source code from the same place counts as distribution of the source
+code,  even  though  third  parties are not compelled to copy the source
+along with the object code.
+
+4. You may not copy,  modify,  sublicense,  or  distribute  the  Program
+except  as  expressly provided under this License. Any attempt otherwise
+to copy, modify, sublicense or distribute the Program is void, and  will
+automatically terminate your rights under this License. However, parties
+who  have  received  copies, or rights, from you under this License will
+not have their licenses terminated so long as  such  parties  remain  in
+full compliance.
+
+5. You are not required to accept  this  License,  since  you  have  not
+signed  it.  However,  nothing  else  grants you permission to modify or
+distribute the Program  or  its  derivative  works.  These  actions  are
+prohibited  by  law  if  you  do  not accept this License. Therefore, by
+modifying or distributing the Program (or any work based on the Program),
+you indicate your acceptance of this License to do so, and all its terms
+and conditions for copying, distributing or  modifying  the  Program  or
+works based on it.
+
+6. Each time you redistribute the Program (or  any  work  based  on  the
+Program),  the  recipient  automatically  receives  a  license  from the
+original licensor to copy, distribute or modify the Program  subject  to
+these  terms and conditions. You may not impose any further restrictions
+on the recipients' exercise of the rights granted herein.  You  are  not
+responsible for enforcing compliance by third parties to this License.
+
+7. If, as a consequence of a court  judgment  or  allegation  of  patent
+infringement  or  for  any  other reason (not limited to patent issues),
+conditions are imposed on you (whether  by  court  order,  agreement  or
+otherwise)  that  contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot distribute
+so as to satisfy simultaneously your obligations under this License  and
+any  other  pertinent  obligations,  then  as  a consequence you may not
+distribute the Program at all. For example, if a  patent  license  would
+not  permit  royalty-free redistribution of the Program by all those who
+receive copies directly or indirectly through you, then the only way you
+could satisfy both it and this License would be to refrain entirely from
+distribution of the Program.
+
+If any portion of this section is held invalid  or  unenforceable  under
+any  particular  circumstance, the balance of the section is intended to
+apply and the  section  as  a  whole  is  intended  to  apply  in  other
+circumstances.
+
+It is not the purpose of this section to  induce  you  to  infringe  any
+patents  or  other  property  right claims or to contest validity of any
+such claims; this  section  has  the  sole  purpose  of  protecting  the
+integrity of the free software distribution system, which is implemented
+by   public   license   practices.   Many   people  have  made  generous
+contributions to the wide range of  software  distributed  through  that
+system in reliance on consistent application of that system; it is up to
+the  author/donor  to  decide  if  he  or  she  is willing to distribute
+software through any other system and  a  licensee  cannot  impose  that
+choice.
+
+This section is intended to make thoroughly clear what is believed to be
+a consequence of the rest of this License.
+
+8. If the distribution and/or  use  of  the  Program  is  restricted  in
+certain  countries  either  by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License  may
+add  an  explicit  geographical  distribution limitation excluding those
+countries, so that distribution is permitted only in or among  countries
+not   thus  excluded.  In  such  case,  this  License  incorporates  the
+limitation as if written in the body of this License.
+
+9. The Free Software Foundation may publish revised and/or new  versions
+of  the General Public License from time to time. Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If  the  Program
+specifies  a version number of this License which applies to it and "any
+later  version",  you  have  the  option  of  following  the  terms  and
+conditions  either  of that version or of any later version published by
+the Free Software Foundation. If the Program does not specify a  version
+number of this License, you may choose any version ever published by the
+Free Software Foundation.
+
+10. If you wish to incorporate parts of  the  Program  into  other  free
+programs  whose  distribution  conditions  are  different,  write to the
+author to ask for permission. For software which is copyrighted  by  the
+Free  Software  Foundation,  write  to  the Free Software Foundation; we
+sometimes make exceptions for this. Our decision will be guided  by  the
+two  goals  of preserving the free status of all derivatives of our free
+software and of promoting the sharing and reuse of software generally.
+
+NO WARRANTY
+
+11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT  WHEN
+OTHERWISE  STATED  IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM  "AS  IS"  WITHOUT  WARRANTY  OF  ANY  KIND,  EITHER
+EXPRESSED  OR  IMPLIED,  INCLUDING,  BUT  NOT  LIMITED  TO,  THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE
+ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.
+SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY
+SERVICING, REPAIR OR CORRECTION.
+
+12. IN NO EVENT UNLESS REQUIRED  BY  APPLICABLE  LAW  OR  AGREED  TO  IN
+WRITING  WILL  ANY  COPYRIGHT  HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR
+DAMAGES, INCLUDING ANY GENERAL,  SPECIAL,  INCIDENTAL  OR  CONSEQUENTIAL
+DAMAGES  ARISING  OUT  OF  THE  USE  OR  INABILITY  TO  USE  THE PROGRAM
+(INCLUDING BUT NOT LIMITED TO  LOSS  OF  DATA  OR  DATA  BEING  RENDERED
+INACCURATE  OR  LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF
+THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER  OR
+OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+END OF TERMS AND CONDITIONS
+
+
+How to Apply These Terms to Your New Programs
+
+If you develop a new program, and you want it  to  be  of  the  greatest
+possible  use  to the public, the best way to achieve this is to make it
+free software which everyone can redistribute  and  change  under  these
+terms.
+
+To do so, attach the following notices to the program. It is  safest  to
+attach  them to the start of each source file to most effectively convey
+the exclusion of warranty; and  each  file  should  have  at  least  the
+"copyright" line and a pointer to where the full notice is found.
+
+   One line to give the program's name and a brief idea of what it does.
+
+   Copyright (C)
+
+   This program is free software; you can redistribute it and/or  modify
+it under the terms of the GNU General Public License as published by the
+Free  Software  Foundation; either version 2 of the License, or (at your
+option) any later version.
+
+   This program is distributed in the hope that it will be  useful,  but
+WITHOUT   ANY   WARRANTY;   without   even   the   implied  warranty  of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+Public License for more details.
+
+   You should have received a copy of the  GNU  General  Public  License
+along  with this program; if not, write to the Free Software Foundation,
+Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice  like  this
+when it starts in an interactive mode:
+
+   Gnomovision version 69, Copyright (C) year name of author
+   Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show
+w'. This is free software, and you are welcome to redistribute it  under
+certain conditions; type `show c' for details.
+
+The hypothetical  commands  `show  w'  and  `show  c'  should  show  the
+appropriate parts of the General Public License. Of course, the commands
+you  use  may be called something other than `show w' and `show c'; they
+could even be mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or  your
+school,  if  any,  to  sign a "copyright disclaimer" for the program, if
+necessary. Here is a sample; alter the names:
+
+   Yoyodyne, Inc.,  hereby  disclaims  all  copyright  interest  in  the
+program `Gnomovision' (which makes passes at compilers) written by James
+Hacker.
+
+   signature of Ty Coon, 1 April 1989
+   Ty Coon, President of Vice
+
+This General Public License does not permit incorporating  your  program
+into  proprietary programs. If your program is a subroutine library, you
+may consider it more useful to permit linking  proprietary  applications
+with  the  library.  If this is what you want to do, use the GNU Library
+General Public License instead of this License.
+
+
+"CLASSPATH" EXCEPTION TO THE GPL VERSION 2
+
+Certain source files distributed by Sun Microsystems, Inc.  are  subject
+to  the following clarification and special exception to the GPL Version
+2, but only where Sun has expressly included in  the  particular  source
+file's header the words
+
+"Sun designates this particular  file  as  subject  to  the  "Classpath"
+exception  as  provided by Sun in the License file that accompanied this
+code."
+
+Linking this library statically or dynamically  with  other  modules  is
+making  a  combined  work  based  on  this  library. Thus, the terms and
+conditions of the GNU General Public License Version 2 cover  the  whole
+combination.
+
+As a special exception, the copyright holders of this library  give  you
+permission  to  link this library with independent modules to produce an
+executable, regardless of the license terms of these independent modules,
+and to copy and distribute the resulting executable under terms of  your
+choice, provided that you also meet, for each linked independent module,
+the  terms and conditions of the license of that module.? An independent
+module is a module which is not derived from or based on this  library.?
+If  you  modify  this  library,  you  may  extend this exception to your
+version of the library, but you are not obligated to do so.? If  you  do
+not wish to do so, delete this exception statement from your version.
+
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/jaxb-NOTICE.txt b/plugins/repository-s3/licenses/jaxb-NOTICE.txt
new file mode 100644
index 00000000000..8d1c8b69c3f
--- /dev/null
+++ b/plugins/repository-s3/licenses/jaxb-NOTICE.txt
@@ -0,0 +1 @@
+ 
diff --git a/plugins/repository-s3/licenses/jaxb-api-2.2.2.jar.sha1 b/plugins/repository-s3/licenses/jaxb-api-2.2.2.jar.sha1
new file mode 100644
index 00000000000..a37e1872389
--- /dev/null
+++ b/plugins/repository-s3/licenses/jaxb-api-2.2.2.jar.sha1
@@ -0,0 +1 @@
+aeb3021ca93dde265796d82015beecdcff95bf09
\ No newline at end of file
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index 247a340d8c4..98b17c2323e 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -1968,7 +1968,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
         assert INSTANCE == null;
         if (isSuiteScopedTest(targetClass)) {
             // note we need to do this this way to make sure this is reproducible
-            INSTANCE = (ESIntegTestCase) targetClass.newInstance();
+            INSTANCE = (ESIntegTestCase) targetClass.getConstructor().newInstance();
             boolean success = false;
             try {
                 INSTANCE.printTestMessage("setup");
diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java b/test/framework/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java
index b9643dda7c7..4e0623cd134 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/test/SuiteScopeClusterIT.java
@@ -48,7 +48,7 @@ public class SuiteScopeClusterIT extends ESIntegTestCase {
                 SEQUENCE[i] = randomLong();
             }
         } else {
-            assertEquals(CLUSTER_SEED, new Long(cluster().seed()));
+            assertEquals(CLUSTER_SEED, Long.valueOf(cluster().seed()));
             for (int i = 0; i < SEQUENCE.length; i++) {
                 assertThat(SEQUENCE[i], equalTo(randomLong()));
             }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/TestScopeClusterIT.java b/test/framework/src/test/java/org/elasticsearch/test/test/TestScopeClusterIT.java
index 8c3c18454cf..8ef38305018 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/test/TestScopeClusterIT.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/test/TestScopeClusterIT.java
@@ -42,7 +42,7 @@ public class TestScopeClusterIT extends ESIntegTestCase {
                 SEQUENCE[i] = randomLong();
             }
         } else {
-            assertEquals(CLUSTER_SEED, new Long(cluster().seed()));
+            assertEquals(CLUSTER_SEED, Long.valueOf(cluster().seed()));
             for (int i = 0; i < SEQUENCE.length; i++) {
                 assertThat(SEQUENCE[i], equalTo(randomLong()));
             }

From b26f5711d6de423d8805c961ea99115ed9ea13f9 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Sat, 21 May 2016 13:17:42 -0700
Subject: [PATCH 0405/1311] Fix log4j buggy java version detection

---
 .../main/java/org/apache/log4j/Java9Hack.java | 32 +++++++++++++++++++
 .../common/logging/LogConfigurator.java       |  6 ++++
 2 files changed, 38 insertions(+)
 create mode 100644 core/src/main/java/org/apache/log4j/Java9Hack.java

diff --git a/core/src/main/java/org/apache/log4j/Java9Hack.java b/core/src/main/java/org/apache/log4j/Java9Hack.java
new file mode 100644
index 00000000000..76f10361565
--- /dev/null
+++ b/core/src/main/java/org/apache/log4j/Java9Hack.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.log4j;
+
+import org.apache.log4j.helpers.ThreadLocalMap;
+
+/**
+ * TODO: document this hack
+ */
+public class Java9Hack {
+
+    public static void fixLog4j() {
+        MDC.mdc.tlm = new ThreadLocalMap();
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java b/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
index eba89c2e02a..e903c963023 100644
--- a/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
+++ b/core/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java
@@ -19,7 +19,9 @@
 
 package org.elasticsearch.common.logging;
 
+import org.apache.log4j.Java9Hack;
 import org.apache.log4j.PropertyConfigurator;
+import org.apache.lucene.util.Constants;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.bootstrap.BootstrapInfo;
 import org.elasticsearch.common.settings.Settings;
@@ -87,6 +89,10 @@ public class LogConfigurator {
         replacements.put("ttcc", "org.apache.log4j.TTCCLayout");
         replacements.put("xml", "org.apache.log4j.XMLLayout");
         REPLACEMENTS = unmodifiableMap(replacements);
+
+        if (Constants.JRE_IS_MINIMUM_JAVA9) {
+            Java9Hack.fixLog4j();
+        }
     }
 
     private static boolean loaded;

From 41a5c0cfa1bec6ab42ca51f22eaab1c5cc7ac019 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Sat, 21 May 2016 13:41:38 -0700
Subject: [PATCH 0406/1311] Force java9 log4j hack in testing

---
 .../main/java/org/apache/log4j/Java9Hack.java |  6 +++-
 .../java/org/apache/log4j/Java9HackTests.java | 28 +++++++++++++++++++
 .../logging/LoggingConfigurationTests.java    |  1 +
 .../bootstrap/BootstrapForTesting.java        |  6 ++++
 4 files changed, 40 insertions(+), 1 deletion(-)
 create mode 100644 core/src/test/java/org/apache/log4j/Java9HackTests.java

diff --git a/core/src/main/java/org/apache/log4j/Java9Hack.java b/core/src/main/java/org/apache/log4j/Java9Hack.java
index 76f10361565..39ee01f7b9f 100644
--- a/core/src/main/java/org/apache/log4j/Java9Hack.java
+++ b/core/src/main/java/org/apache/log4j/Java9Hack.java
@@ -27,6 +27,10 @@ import org.apache.log4j.helpers.ThreadLocalMap;
 public class Java9Hack {
 
     public static void fixLog4j() {
-        MDC.mdc.tlm = new ThreadLocalMap();
+        System.out.println("Fixing log4j");
+        if (MDC.mdc.tlm == null) {
+            System.out.println("FOR REAL");
+            MDC.mdc.tlm = new ThreadLocalMap();
+        }
     }
 }
diff --git a/core/src/test/java/org/apache/log4j/Java9HackTests.java b/core/src/test/java/org/apache/log4j/Java9HackTests.java
new file mode 100644
index 00000000000..e917f1d3060
--- /dev/null
+++ b/core/src/test/java/org/apache/log4j/Java9HackTests.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.log4j;
+
+import org.elasticsearch.test.ESTestCase;
+
+public class Java9HackTests extends ESTestCase {
+    public void testJava9Hack() {
+        assertNotNull(MDC.mdc.tlm != null);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/logging/LoggingConfigurationTests.java b/core/src/test/java/org/elasticsearch/common/logging/LoggingConfigurationTests.java
index 5c812cca0a7..fabace237b2 100644
--- a/core/src/test/java/org/elasticsearch/common/logging/LoggingConfigurationTests.java
+++ b/core/src/test/java/org/elasticsearch/common/logging/LoggingConfigurationTests.java
@@ -27,6 +27,7 @@ import java.util.Arrays;
 
 import org.apache.log4j.Appender;
 import org.apache.log4j.Logger;
+import org.apache.log4j.MDC;
 import org.elasticsearch.cli.MockTerminal;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
diff --git a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
index 68eb0420b39..5b79721948e 100644
--- a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
+++ b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
@@ -20,6 +20,8 @@
 package org.elasticsearch.bootstrap;
 
 import com.carrotsearch.randomizedtesting.RandomizedRunner;
+import org.apache.log4j.Java9Hack;
+import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.SecureSM;
 import org.elasticsearch.common.Strings;
@@ -89,6 +91,10 @@ public class BootstrapForTesting {
             throw new RuntimeException("found jar hell in test classpath", e);
         }
 
+        if (Constants.JRE_IS_MINIMUM_JAVA9) {
+            Java9Hack.fixLog4j();
+        }
+
         // install security manager if requested
         if (systemPropertyAsBoolean("tests.security.manager", true)) {
             try {

From ba14aca2185ef578280f8b05516794423d9a24e0 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Sat, 21 May 2016 11:02:41 -0400
Subject: [PATCH 0407/1311] Refactor property placeholder use of env. vars

This commit is a slight refactoring of the use of environment variables
in replacing property placeholders. In commit
115f983827c0c29652bd444c072b658b76651317 the constructor for
Settings.Builder was made package visible to provide a hook for tests to
mock obtaining environment variables. But we do not need to go that far
and can instead provide a small hook for this for tests without opening
up the constructor. Thus, in this commit we refactor
Settings.Builder#replacePropertyPlaceholders to a package-visible method
that accepts a function providing environment variables by names. The
public-visible method just delegates to this method passing in
System::getenv and tests can use the package-visible method to mock the
behavior they need without relying on external environment variables.
---
 .../common/settings/Settings.java             | 55 +++++++++----------
 .../common/settings/SettingsTests.java        | 10 +---
 2 files changed, 29 insertions(+), 36 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/settings/Settings.java b/core/src/main/java/org/elasticsearch/common/settings/Settings.java
index 2ceafc376c5..15554e5ccaa 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/Settings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/Settings.java
@@ -602,8 +602,7 @@ public final class Settings implements ToXContent {
 
         private final Map<String, String> map = new LinkedHashMap<>();
 
-        // visible for testing
-        Builder() {
+        private Builder() {
 
         }
 
@@ -961,33 +960,38 @@ public final class Settings implements ToXContent {
          * another setting already set on this builder.
          */
         public Builder replacePropertyPlaceholders() {
+            return replacePropertyPlaceholders(System::getenv);
+        }
+
+        // visible for testing
+        Builder replacePropertyPlaceholders(Function<String, String> getenv) {
             PropertyPlaceholder propertyPlaceholder = new PropertyPlaceholder("${", "}", false);
             PropertyPlaceholder.PlaceholderResolver placeholderResolver = new PropertyPlaceholder.PlaceholderResolver() {
-                    @Override
-                    public String resolvePlaceholder(String placeholderName) {
-                        final String value = getenv(placeholderName);
-                        if (value != null) {
-                            return value;
-                        }
-                        return map.get(placeholderName);
+                @Override
+                public String resolvePlaceholder(String placeholderName) {
+                    final String value = getenv.apply(placeholderName);
+                    if (value != null) {
+                        return value;
                     }
+                    return map.get(placeholderName);
+                }
 
-                    @Override
-                    public boolean shouldIgnoreMissing(String placeholderName) {
-                        if (placeholderName.startsWith("prompt.")) {
-                            return true;
-                        }
-                        return false;
-                    }
-
-                    @Override
-                    public boolean shouldRemoveMissingPlaceholder(String placeholderName) {
-                        if (placeholderName.startsWith("prompt.")) {
-                            return false;
-                        }
+                @Override
+                public boolean shouldIgnoreMissing(String placeholderName) {
+                    if (placeholderName.startsWith("prompt.")) {
                         return true;
                     }
-                };
+                    return false;
+                }
+
+                @Override
+                public boolean shouldRemoveMissingPlaceholder(String placeholderName) {
+                    if (placeholderName.startsWith("prompt.")) {
+                        return false;
+                    }
+                    return true;
+                }
+            };
             for (Map.Entry<String, String> entry : new HashMap<>(map).entrySet()) {
                 String value = propertyPlaceholder.replacePlaceholders(entry.getKey(), entry.getValue(), placeholderResolver);
                 // if the values exists and has length, we should maintain it  in the map
@@ -1001,11 +1005,6 @@ public final class Settings implements ToXContent {
             return this;
         }
 
-        // visible for testing
-        String getenv(String placeholderName) {
-            return System.getenv(placeholderName);
-        }
-
         /**
          * Checks that all settings in the builder start with the specified prefix.
          *
diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
index 5b3a698d328..346c5bc60de 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/SettingsTests.java
@@ -63,15 +63,9 @@ public class SettingsTests extends ESTestCase {
 
     public void testReplacePropertiesPlaceholderByEnvironmentVariables() {
         final String hostname = randomAsciiOfLength(16);
-        final Settings.Builder builder = new Settings.Builder() {
-            @Override
-            protected String getenv(String placeholderName) {
-                return "HOSTNAME".equals(placeholderName) ? hostname : null;
-            }
-        };
-        final Settings implicitEnvSettings = builder
+        final Settings implicitEnvSettings = Settings.builder()
             .put("setting1", "${HOSTNAME}")
-            .replacePropertyPlaceholders()
+            .replacePropertyPlaceholders(name -> "HOSTNAME".equals(name) ? hostname : null)
             .build();
         assertThat(implicitEnvSettings.get("setting1"), equalTo(hostname));
     }

From f01f15d3b8592db4210f725f0c37baff4a554a35 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Sat, 21 May 2016 14:14:12 -0700
Subject: [PATCH 0408/1311] Document the hack

---
 core/src/main/java/org/apache/log4j/Java9Hack.java | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/apache/log4j/Java9Hack.java b/core/src/main/java/org/apache/log4j/Java9Hack.java
index 39ee01f7b9f..831cf5b35ae 100644
--- a/core/src/main/java/org/apache/log4j/Java9Hack.java
+++ b/core/src/main/java/org/apache/log4j/Java9Hack.java
@@ -22,14 +22,15 @@ package org.apache.log4j;
 import org.apache.log4j.helpers.ThreadLocalMap;
 
 /**
- * TODO: document this hack
+ * Log4j 1.2 MDC breaks because it parses java.version incorrectly (does not handle new java9 versioning).
+ *
+ * This hack fixes up the pkg private members as if it had detected the java version correctly.
  */
 public class Java9Hack {
 
     public static void fixLog4j() {
-        System.out.println("Fixing log4j");
         if (MDC.mdc.tlm == null) {
-            System.out.println("FOR REAL");
+            MDC.mdc.java1 = false;
             MDC.mdc.tlm = new ThreadLocalMap();
         }
     }

From 7cf758943eb37ed5f57b9c37abb91246ea274670 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Sat, 21 May 2016 22:58:55 -0700
Subject: [PATCH 0409/1311] Fix git info lookup to use correct dir where git
 exists

---
 build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index 3d52d4ab279..c843069f43e 100644
--- a/build.gradle
+++ b/build.gradle
@@ -81,7 +81,7 @@ subprojects {
     nexus {
       String buildSnapshot = System.getProperty('build.snapshot', 'true')
       if (buildSnapshot == 'false') {
-        Repository repo = new RepositoryBuilder().findGitDir(new File('.')).build()
+        Repository repo = new RepositoryBuilder().findGitDir(project.rootDir).build()
         String shortHash = repo.resolve('HEAD')?.name?.substring(0,7)
         repositoryUrl = project.hasProperty('build.repository') ? project.property('build.repository') : "file://${System.getenv('HOME')}/elasticsearch-releases/${version}-${shortHash}/"
       }

From cf5490358080107dd94feb6af2cfff5d8b59d056 Mon Sep 17 00:00:00 2001
From: "G. Richard Bellamy" <rbellamy@terradatum.com>
Date: Sat, 21 May 2016 18:27:38 -0700
Subject: [PATCH 0410/1311] Support full range of Java Long for epoch DateTime

Remove the arbitrary limit on epoch_millis and epoch_seconds of 13 and 10
characters, respectively. Instead allow any character combination that can
be converted to a Java Long.

Update the docs to reflect this change.
---
 .../java/org/elasticsearch/common/joda/Joda.java     | 12 +++---------
 .../common/joda/DateMathParserTests.java             |  2 ++
 .../org/elasticsearch/deps/joda/SimpleJodaTests.java | 10 ++++++++--
 docs/reference/mapping/params/format.asciidoc        | 10 ++++------
 4 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/joda/Joda.java b/core/src/main/java/org/elasticsearch/common/joda/Joda.java
index b65a248c21f..cffea836ac2 100644
--- a/core/src/main/java/org/elasticsearch/common/joda/Joda.java
+++ b/core/src/main/java/org/elasticsearch/common/joda/Joda.java
@@ -321,20 +321,15 @@ public class Joda {
 
     public static class EpochTimeParser implements DateTimeParser {
 
-        private static final Pattern MILLI_SECOND_PRECISION_PATTERN = Pattern.compile("^-?\\d{1,13}$");
-        private static final Pattern SECOND_PRECISION_PATTERN = Pattern.compile("^-?\\d{1,10}$");
-
         private final boolean hasMilliSecondPrecision;
-        private final Pattern pattern;
 
         public EpochTimeParser(boolean hasMilliSecondPrecision) {
             this.hasMilliSecondPrecision = hasMilliSecondPrecision;
-            this.pattern = hasMilliSecondPrecision ? MILLI_SECOND_PRECISION_PATTERN : SECOND_PRECISION_PATTERN;
         }
 
         @Override
         public int estimateParsedLength() {
-            return hasMilliSecondPrecision ? 13 : 10;
+            return hasMilliSecondPrecision ? 19 : 16;
         }
 
         @Override
@@ -344,8 +339,7 @@ public class Joda {
 
             if ((isPositive && isTooLong) ||
                 // timestamps have to have UTC timezone
-                bucket.getZone() != DateTimeZone.UTC ||
-                pattern.matcher(text).matches() == false) {
+                bucket.getZone() != DateTimeZone.UTC) {
                 return -1;
             }
 
@@ -378,7 +372,7 @@ public class Joda {
 
         @Override
         public int estimatePrintedLength() {
-            return hasMilliSecondPrecision ? 13 : 10;
+            return hasMilliSecondPrecision ? 19 : 16;
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/common/joda/DateMathParserTests.java b/core/src/test/java/org/elasticsearch/common/joda/DateMathParserTests.java
index 96a4a3fdf35..b1d0abc96cf 100644
--- a/core/src/test/java/org/elasticsearch/common/joda/DateMathParserTests.java
+++ b/core/src/test/java/org/elasticsearch/common/joda/DateMathParserTests.java
@@ -196,6 +196,8 @@ public class DateMathParserTests extends ESTestCase {
 
     public void testTimestamps() {
         assertDateMathEquals("1418248078000", "2014-12-10T21:47:58.000");
+        assertDateMathEquals("32484216259000", "2999-05-20T17:24:19.000");
+        assertDateMathEquals("253382837059000", "9999-05-20T17:24:19.000");
 
         // datemath still works on timestamps
         assertDateMathEquals("1418248078000||/m", "2014-12-10T21:47:00.000");
diff --git a/core/src/test/java/org/elasticsearch/deps/joda/SimpleJodaTests.java b/core/src/test/java/org/elasticsearch/deps/joda/SimpleJodaTests.java
index c4775b6cbf2..595394dbfa9 100644
--- a/core/src/test/java/org/elasticsearch/deps/joda/SimpleJodaTests.java
+++ b/core/src/test/java/org/elasticsearch/deps/joda/SimpleJodaTests.java
@@ -289,16 +289,18 @@ public class SimpleJodaTests extends ESTestCase {
             formatter.parser().parseDateTime("-100000000");
             formatter.parser().parseDateTime("-999999999999");
             formatter.parser().parseDateTime("-1234567890123");
+            formatter.parser().parseDateTime("-1234567890123456789");
         } else {
             formatter.parser().parseDateTime("-100000000");
             formatter.parser().parseDateTime("-1234567890");
+            formatter.parser().parseDateTime("-1234567890123456");
         }
     }
 
     public void testForInvalidDatesInEpochSecond() {
         FormatDateTimeFormatter formatter = Joda.forPattern("epoch_second");
         try {
-            formatter.parser().parseDateTime(randomFrom("invalid date", "12345678901", "12345678901234"));
+            formatter.parser().parseDateTime(randomFrom("invalid date", "12345678901234567", "12345678901234567890"));
             fail("Expected IllegalArgumentException");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), containsString("Invalid format"));
@@ -308,7 +310,7 @@ public class SimpleJodaTests extends ESTestCase {
     public void testForInvalidDatesInEpochMillis() {
         FormatDateTimeFormatter formatter = Joda.forPattern("epoch_millis");
         try {
-            formatter.parser().parseDateTime(randomFrom("invalid date", "12345678901234"));
+            formatter.parser().parseDateTime(randomFrom("invalid date", "12345678901234567890"));
             fail("Expected IllegalArgumentException");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), containsString("Invalid format"));
@@ -350,6 +352,8 @@ public class SimpleJodaTests extends ESTestCase {
         assertThat(dateTime.getMillis(), is(1234567890456L));
         dateTime = formatter.parser().parseDateTime("1234567890789");
         assertThat(dateTime.getMillis(), is(1234567890789L));
+        dateTime = formatter.parser().parseDateTime("1234567890123456789");
+        assertThat(dateTime.getMillis(), is(1234567890123456789L));
 
         FormatDateTimeFormatter secondsFormatter = Joda.forPattern("epoch_second");
         DateTime secondsDateTime = secondsFormatter.parser().parseDateTime("1234567890");
@@ -358,6 +362,8 @@ public class SimpleJodaTests extends ESTestCase {
         assertThat(secondsDateTime.getMillis(), is(1234567890000L));
         secondsDateTime = secondsFormatter.parser().parseDateTime("1234567890");
         assertThat(secondsDateTime.getMillis(), is(1234567890000L));
+        secondsDateTime = secondsFormatter.parser().parseDateTime("1234567890123456");
+        assertThat(secondsDateTime.getMillis(), is(1234567890123456000L));
     }
 
     public void testThatDefaultFormatterChecksForCorrectYearLength() throws Exception {
diff --git a/docs/reference/mapping/params/format.asciidoc b/docs/reference/mapping/params/format.asciidoc
index 38cb0ea624f..0553c302b54 100644
--- a/docs/reference/mapping/params/format.asciidoc
+++ b/docs/reference/mapping/params/format.asciidoc
@@ -57,16 +57,14 @@ The following tables lists all the defaults ISO formats supported:
 `epoch_millis`::
 
     A formatter for the number of milliseconds since the epoch. Note, that
-    this timestamp allows a max length of 13 chars, so only dates between 1653
-    and 2286 are supported. You should use a different date formatter in
-    that case.
+    this timestamp is subject to the limits of a Java `Long.MIN_VALUE` and
+    `Long.MAX_VALUE`.
 
 `epoch_second`::
 
     A formatter for the number of seconds since the epoch. Note, that this
-    timestamp allows a max length of 10 chars, so only dates between 1653 and
-    2286 are supported. You should use a different date formatter in that
-    case.
+    timestamp is subject to the limits of a Java `Long.MIN_VALUE` and `Long.
+    MAX_VALUE` divided by 1000 (the number of milliseconds in a second).
 
 [[strict-date-time]]`date_optional_time` or `strict_date_optional_time`::
 

From c1a09291237ae6f8979aba7ae99ef1e098bf37a8 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Fri, 20 May 2016 17:28:00 +0200
Subject: [PATCH 0411/1311] percolator: Add support dor MatchNoDocsQuery in
 query terms extract service

Before the query extraction would have been aborted and the percolator query would be marked as unknown.
This resulted in a situation that these queries always need to be evaluated by the memory index at search time.
By adding support for this query many more percolator query candidate hits can skip the expensive memory index verification step. For example the `match` query parser returns a MatchNoDocsQuery if the query terms are removed by text analysis (lets query text only contained stop words).
---
 .../percolator/ExtractQueryTermsService.java  | 20 +++++++--------
 .../ExtractQueryTermsServiceTests.java        | 18 +++++++++++++
 .../query-dsl/percolate-query.asciidoc        | 25 ++++++++++++++++++-
 3 files changed, 52 insertions(+), 11 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/percolator/ExtractQueryTermsService.java b/core/src/main/java/org/elasticsearch/index/percolator/ExtractQueryTermsService.java
index 2f48a0c6439..28082a4bf83 100644
--- a/core/src/main/java/org/elasticsearch/index/percolator/ExtractQueryTermsService.java
+++ b/core/src/main/java/org/elasticsearch/index/percolator/ExtractQueryTermsService.java
@@ -37,19 +37,16 @@ import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.spans.FieldMaskingSpanQuery;
-import org.apache.lucene.search.spans.SpanContainingQuery;
 import org.apache.lucene.search.spans.SpanFirstQuery;
-import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
 import org.apache.lucene.search.spans.SpanNearQuery;
 import org.apache.lucene.search.spans.SpanNotQuery;
 import org.apache.lucene.search.spans.SpanOrQuery;
 import org.apache.lucene.search.spans.SpanQuery;
 import org.apache.lucene.search.spans.SpanTermQuery;
-import org.apache.lucene.search.spans.SpanWithinQuery;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.common.logging.LoggerMessageFormat;
+import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.index.mapper.ParseContext;
 
 import java.io.IOException;
@@ -72,11 +69,11 @@ public final class ExtractQueryTermsService {
 
     /**
      * Extracts all terms from the specified query and adds it to the specified document.
-     *  @param query The query to extract terms from
-     * @param document The document to add the extracted terms to
-     * @param queryTermsFieldField The field in the document holding the extracted terms
-     * @param unknownQueryField The field used to mark a document that not all query terms could be extracted. For example
-     *                          the query contained an unsupported query (e.g. WildcardQuery).
+     * @param query                 The query to extract terms from
+     * @param document              The document to add the extracted terms to
+     * @param queryTermsFieldField  The field in the document holding the extracted terms
+     * @param unknownQueryField     The field used to mark a document that not all query terms could be extracted.
+     *                              For example the query contained an unsupported query (e.g. WildcardQuery).
      * @param fieldType The field type for the query metadata field
      */
     public static void extractQueryTerms(Query query, ParseContext.Document document, String queryTermsFieldField, String unknownQueryField, FieldType fieldType) {
@@ -106,7 +103,10 @@ public final class ExtractQueryTermsService {
      * an UnsupportedQueryException is thrown.
      */
     static Set<Term> extractQueryTerms(Query query) {
-        if (query instanceof TermQuery) {
+        if (query instanceof MatchNoDocsQuery) {
+            // no terms to extract as this query matches no docs
+            return Collections.emptySet();
+        } else if (query instanceof TermQuery) {
             return Collections.singleton(((TermQuery) query).getTerm());
         } else if (query instanceof TermsQuery) {
             Set<Term> terms = new HashSet<>();
diff --git a/core/src/test/java/org/elasticsearch/index/percolator/ExtractQueryTermsServiceTests.java b/core/src/test/java/org/elasticsearch/index/percolator/ExtractQueryTermsServiceTests.java
index 73f2da8c6d7..45a1386d912 100644
--- a/core/src/test/java/org/elasticsearch/index/percolator/ExtractQueryTermsServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/index/percolator/ExtractQueryTermsServiceTests.java
@@ -41,6 +41,7 @@ import org.apache.lucene.search.spans.SpanNotQuery;
 import org.apache.lucene.search.spans.SpanOrQuery;
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.test.ESTestCase;
 
@@ -296,6 +297,23 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         assertTermsEqual(terms, spanTermQuery1.getTerm());
     }
 
+    public void testExtractQueryMetadata_matchNoDocsQuery() {
+        Set<Term> terms = ExtractQueryTermsService.extractQueryTerms(new MatchNoDocsQuery("sometimes there is no reason at all"));
+        assertEquals(0, terms.size());
+
+        BooleanQuery.Builder bq = new BooleanQuery.Builder();
+        bq.add(new TermQuery(new Term("field", "value")), BooleanClause.Occur.MUST);
+        bq.add(new MatchNoDocsQuery("sometimes there is no reason at all"), BooleanClause.Occur.MUST);
+        terms = ExtractQueryTermsService.extractQueryTerms(bq.build());
+        assertEquals(0, terms.size());
+
+        bq = new BooleanQuery.Builder();
+        bq.add(new TermQuery(new Term("field", "value")), BooleanClause.Occur.SHOULD);
+        bq.add(new MatchNoDocsQuery("sometimes there is no reason at all"), BooleanClause.Occur.SHOULD);
+        terms = ExtractQueryTermsService.extractQueryTerms(bq.build());
+        assertTermsEqual(terms, new Term("field", "value"));
+    }
+
     public void testExtractQueryMetadata_unsupportedQuery() {
         TermRangeQuery termRangeQuery = new TermRangeQuery("_field", null, null, true, false);
 
diff --git a/docs/reference/query-dsl/percolate-query.asciidoc b/docs/reference/query-dsl/percolate-query.asciidoc
index 48ba500c98d..2ae03b4d9a5 100644
--- a/docs/reference/query-dsl/percolate-query.asciidoc
+++ b/docs/reference/query-dsl/percolate-query.asciidoc
@@ -333,4 +333,27 @@ of the query gets stored, but also the query's terms are analyzed and stored int
 At search time, the document specified in the request gets parsed into a Lucene document and is stored in a in-memory
 temporary Lucene index. This in-memory index can just hold this one document and it is optimized for that. After this
 a special query is build based on the terms in the in-memory index that select candidate percolator queries based on
-their indexed query terms. These queries are then evaluated by the in-memory index if they actually match.
\ No newline at end of file
+their indexed query terms. These queries are then evaluated by the in-memory index if they actually match.
+
+The selecting of candidate percolator queries matches is an important performance optimization during the execution
+of the `percolate` query as it can significantly reduce the number of candidate matches the in-memory index need to
+evaluate. The reason the `percolate` query can do this is because during indexing of the percolator queries the query
+terms are being extracted and indexed with the percolator query. Unfortunately the percolator cannot extract terms from
+all queries (for example the `wildcard` or `geo_shape` query) and as a result of that in certain cases the percolator
+can't do the selecting optimization (for example if an unsupported query is defined in a required clause of a boolean query
+or the unsupported query is the only query in the percolator document).  These queries are marked by the percolator and
+can be found by running the following search:
+
+[source,js]
+--------------------------------------------------
+curl -XGET "http://localhost:9200/_search" -d'
+{
+  "query": {
+    "term" : {
+      "query.unknown_query" : ""
+    }
+  }
+}'
+--------------------------------------------------
+
+NOTE: The above example assumes that there is a `query` field of type `percolator` in the mappings.
\ No newline at end of file

From e714a04c67beeb429f7a19941453f4f47410b486 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Sun, 22 May 2016 22:50:31 +0200
Subject: [PATCH 0412/1311] docs: fix typo

---
 docs/reference/query-dsl/percolate-query.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/query-dsl/percolate-query.asciidoc b/docs/reference/query-dsl/percolate-query.asciidoc
index 2ae03b4d9a5..447a4d3189c 100644
--- a/docs/reference/query-dsl/percolate-query.asciidoc
+++ b/docs/reference/query-dsl/percolate-query.asciidoc
@@ -336,7 +336,7 @@ a special query is build based on the terms in the in-memory index that select c
 their indexed query terms. These queries are then evaluated by the in-memory index if they actually match.
 
 The selecting of candidate percolator queries matches is an important performance optimization during the execution
-of the `percolate` query as it can significantly reduce the number of candidate matches the in-memory index need to
+of the `percolate` query as it can significantly reduce the number of candidate matches the in-memory index needs to
 evaluate. The reason the `percolate` query can do this is because during indexing of the percolator queries the query
 terms are being extracted and indexed with the percolator query. Unfortunately the percolator cannot extract terms from
 all queries (for example the `wildcard` or `geo_shape` query) and as a result of that in certain cases the percolator

From c706c46aec892e6239656ca6c06e495ad43b4a95 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Sun, 22 May 2016 23:11:37 +0200
Subject: [PATCH 0413/1311] Update to forbiddenapis 2.1

---
 buildSrc/build.gradle                                          | 2 +-
 .../org/elasticsearch/gradle/precommit/PrecommitTasks.groovy   | 3 +--
 .../elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy  | 3 +--
 3 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
index b286124bfee..623fdab3e3e 100644
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -84,7 +84,7 @@ dependencies {
   compile 'com.netflix.nebula:gradle-info-plugin:3.0.3'
   compile 'org.eclipse.jgit:org.eclipse.jgit:3.2.0.201312181205-r'
   compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
-  compile 'de.thetaphi:forbiddenapis:2.0'
+  compile 'de.thetaphi:forbiddenapis:2.1'
   compile 'com.bmuschko:gradle-nexus-plugin:2.3.1'
   compile 'org.apache.rat:apache-rat:0.11'
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
index 427d3191dc5..48a4d7c26dc 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
@@ -62,9 +62,8 @@ class PrecommitTasks {
     private static Task configureForbiddenApis(Project project) {
         project.pluginManager.apply(ForbiddenApisPlugin.class)
         project.forbiddenApis {
-            internalRuntimeForbidden = true
             failOnUnsupportedJava = false
-            bundledSignatures = ['jdk-unsafe', 'jdk-deprecated', 'jdk-system-out']
+            bundledSignatures = ['jdk-unsafe', 'jdk-deprecated', 'jdk-non-portable', 'jdk-system-out']
             signaturesURLs = [getClass().getResource('/forbidden/jdk-signatures.txt'),
                               getClass().getResource('/forbidden/es-all-signatures.txt')]
             suppressAnnotations = ['**.SuppressForbidden']
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy
index 3ff5a06ad42..076a564f84a 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy
@@ -203,8 +203,7 @@ public class ThirdPartyAuditTask extends AntTask {
         Set<String> sheistySet = getSheistyClasses(tmpDir.toPath());
 
         try {
-            ant.thirdPartyAudit(internalRuntimeForbidden: false,
-                            failOnUnsupportedJava: false,
+            ant.thirdPartyAudit(failOnUnsupportedJava: false,
                             failOnMissingClasses: false,
                             signaturesFile: new File(getClass().getResource('/forbidden/third-party-audit.txt').toURI()),
                             classpath: classpath.asPath) {

From cb2cfdd9c0476bfbe5e1c13a4b3eb84ebb61bb6d Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Thu, 19 May 2016 18:38:35 +0200
Subject: [PATCH 0414/1311] Speed up named queries. #18470

Named queries have a performance bug when they are used with expensive queries
that need to perform a lot of work up-front like fuzzy or range queries
(including with points). The reason is that they currently re-create the weight
and scorer for every hit. Instead we should create weights exactly once and
use a single Scorer for all documents that are on the same segment.
---
 .../MatchedQueriesFetchSubPhase.java          | 91 +++++++++++--------
 1 file changed, 53 insertions(+), 38 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/fetch/matchedqueries/MatchedQueriesFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/matchedqueries/MatchedQueriesFetchSubPhase.java
index 6adb01ad10c..983e131215d 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/matchedqueries/MatchedQueriesFetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/matchedqueries/MatchedQueriesFetchSubPhase.java
@@ -18,11 +18,14 @@
  */
 package org.elasticsearch.search.fetch.matchedqueries;
 
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.ReaderUtil;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.Bits;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.internal.InternalSearchHit;
@@ -31,6 +34,8 @@ import org.elasticsearch.search.internal.SearchContext.Lifetime;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -48,58 +53,68 @@ public class MatchedQueriesFetchSubPhase implements FetchSubPhase {
 
     @Override
     public boolean hitsExecutionNeeded(SearchContext context) {
-        return false;
+        return true; // we short-circuit in hitsExecute
     }
 
     @Override
     public void hitsExecute(SearchContext context, InternalSearchHit[] hits) {
-    }
+        if (hits.length == 0) {
+            return;
+        }
+        hits = hits.clone(); // don't modify the incoming hits
+        Arrays.sort(hits, (a, b) -> Integer.compare(a.docId(), b.docId()));
+        @SuppressWarnings("unchecked")
+        List<String>[] matchedQueries = new List[hits.length];
+        for (int i = 0; i < matchedQueries.length; ++i) {
+            matchedQueries[i] = new ArrayList<>();
+        }
 
-    @Override
-    public boolean hitExecutionNeeded(SearchContext context) {
-        return !context.parsedQuery().namedFilters().isEmpty()
-                || (context.parsedPostFilter() !=null && !context.parsedPostFilter().namedFilters().isEmpty());
-    }
-
-    @Override
-    public void hitExecute(SearchContext context, HitContext hitContext) {
-        List<String> matchedQueries = new ArrayList<>(2);
+        Map<String, Query> namedQueries = new HashMap<>(context.parsedQuery().namedFilters());
+        if (context.parsedPostFilter() != null) {
+            namedQueries.putAll(context.parsedPostFilter().namedFilters());
+        }
 
         try {
-            addMatchedQueries(hitContext, context.parsedQuery().namedFilters(), matchedQueries);
-
-            if (context.parsedPostFilter() != null) {
-                addMatchedQueries(hitContext, context.parsedPostFilter().namedFilters(), matchedQueries);
+            for (Map.Entry<String, Query> entry : namedQueries.entrySet()) {
+                String name = entry.getKey();
+                Query query = entry.getValue();
+                int readerIndex = -1;
+                int docBase = -1;
+                Weight weight = context.searcher().createNormalizedWeight(query, false);
+                Bits matchingDocs = null;
+                for (int i = 0; i < hits.length; ++i) {
+                    InternalSearchHit hit = hits[i];
+                    int hitReaderIndex = ReaderUtil.subIndex(hit.docId(), context.searcher().getIndexReader().leaves());
+                    if (readerIndex != hitReaderIndex) {
+                        readerIndex = hitReaderIndex;
+                        LeafReaderContext ctx = context.searcher().getIndexReader().leaves().get(readerIndex);
+                        docBase = ctx.docBase;
+                        // scorers can be costly to create, so reuse them across docs of the same segment
+                        Scorer scorer = weight.scorer(ctx);
+                        matchingDocs = Lucene.asSequentialAccessBits(ctx.reader().maxDoc(), scorer);
+                    }
+                    if (matchingDocs.get(hit.docId() - docBase)) {
+                        matchedQueries[i].add(name);
+                    }
+                }
+            }
+            for (int i = 0; i < hits.length; ++i) {
+                hits[i].matchedQueries(matchedQueries[i].toArray(new String[0]));
             }
         } catch (IOException e) {
             throw ExceptionsHelper.convertToElastic(e);
         } finally {
             SearchContext.current().clearReleasables(Lifetime.COLLECTION);
         }
-
-        hitContext.hit().matchedQueries(matchedQueries.toArray(new String[matchedQueries.size()]));
     }
 
-    private void addMatchedQueries(HitContext hitContext, Map<String, Query> namedQueries, List<String> matchedQueries) throws IOException {
-        for (Map.Entry<String, Query> entry : namedQueries.entrySet()) {
-            String name = entry.getKey();
-            Query filter = entry.getValue();
+    @Override
+    public boolean hitExecutionNeeded(SearchContext context) {
+        return false;
+    }
 
-            final Weight weight = hitContext.topLevelSearcher().createNormalizedWeight(filter, false);
-            final Scorer scorer = weight.scorer(hitContext.readerContext());
-            if (scorer == null) {
-                continue;
-            }
-            final TwoPhaseIterator twoPhase = scorer.twoPhaseIterator();
-            if (twoPhase == null) {
-                if (scorer.iterator().advance(hitContext.docId()) == hitContext.docId()) {
-                    matchedQueries.add(name);
-                }
-            } else {
-                if (twoPhase.approximation().advance(hitContext.docId()) == hitContext.docId() && twoPhase.matches()) {
-                    matchedQueries.add(name);
-                }
-            }
-        }
+    @Override
+    public void hitExecute(SearchContext context, HitContext hitContext) {
+        // we do everything in hitsExecute
     }
 }

From 238d390637db61406514ca2de983a871af9000a1 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Fri, 20 May 2016 12:31:03 +0200
Subject: [PATCH 0415/1311] Fixes for _only_nodes preferences:   * Handle
 multiple attributes/name (coma separated).   * Shuffle the nodes that match
 the preferences.

Fix #12546
Fix #12700
---
 .../routing/IndexShardRoutingTable.java       | 26 ++++---
 .../cluster/routing/OperationRouting.java     |  4 +-
 .../structure/RoutingIteratorTests.java       | 18 +++++
 .../search/preference/SearchPreferenceIT.java | 71 ++++++++++++++++++-
 4 files changed, 104 insertions(+), 15 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
index 53b094bc34b..a1e891bce3d 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
@@ -22,6 +22,7 @@ package org.elasticsearch.cluster.routing;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Randomness;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -331,15 +332,13 @@ public class IndexShardRoutingTable implements Iterable<ShardRouting> {
 
     public ShardIterator onlyNodeActiveInitializingShardsIt(String nodeId) {
         ArrayList<ShardRouting> ordered = new ArrayList<>(activeShards.size() + allInitializingShards.size());
-        // fill it in a randomized fashion
-        for (int i = 0; i < activeShards.size(); i++) {
-            ShardRouting shardRouting = activeShards.get(i);
+        int seed = shuffler.nextSeed();
+        for (ShardRouting shardRouting : shuffler.shuffle(activeShards, seed)) {
             if (nodeId.equals(shardRouting.currentNodeId())) {
                 ordered.add(shardRouting);
             }
         }
-        for (int i = 0; i < allInitializingShards.size(); i++) {
-            ShardRouting shardRouting = allInitializingShards.get(i);
+        for (ShardRouting shardRouting : shuffler.shuffle(allInitializingShards, seed)) {
             if (nodeId.equals(shardRouting.currentNodeId())) {
                 ordered.add(shardRouting);
             }
@@ -347,26 +346,31 @@ public class IndexShardRoutingTable implements Iterable<ShardRouting> {
         return new PlainShardIterator(shardId, ordered);
     }
 
+    public ShardIterator onlyNodeSelectorActiveInitializingShardsIt(String nodeAttributes, DiscoveryNodes discoveryNodes) {
+        return onlyNodeSelectorActiveInitializingShardsIt(new String[] {nodeAttributes}, discoveryNodes);
+    }
+
     /**
      * Returns shards based on nodeAttributes given  such as node name , node attribute, node IP
      * Supports node specifications in cluster API
      */
-    public ShardIterator onlyNodeSelectorActiveInitializingShardsIt(String nodeAttribute, DiscoveryNodes discoveryNodes) {
+    public ShardIterator onlyNodeSelectorActiveInitializingShardsIt(String[] nodeAttributes, DiscoveryNodes discoveryNodes) {
         ArrayList<ShardRouting> ordered = new ArrayList<>(activeShards.size() + allInitializingShards.size());
-        Set<String> selectedNodes = Sets.newHashSet(discoveryNodes.resolveNodesIds(nodeAttribute));
-
-        for (ShardRouting shardRouting : activeShards) {
+        Set<String> selectedNodes = Sets.newHashSet(discoveryNodes.resolveNodesIds(nodeAttributes));
+        int seed = shuffler.nextSeed();
+        for (ShardRouting shardRouting : shuffler.shuffle(activeShards, seed)) {
             if (selectedNodes.contains(shardRouting.currentNodeId())) {
                 ordered.add(shardRouting);
             }
         }
-        for (ShardRouting shardRouting : allInitializingShards) {
+        for (ShardRouting shardRouting : shuffler.shuffle(allInitializingShards, seed)) {
             if (selectedNodes.contains(shardRouting.currentNodeId())) {
                 ordered.add(shardRouting);
             }
         }
         if (ordered.isEmpty()) {
-            throw new IllegalArgumentException("No data node with criteria [" + nodeAttribute + "] found");
+            throw new IllegalArgumentException("no data nodes with critera(s) " +
+                Strings.arrayToCommaDelimitedString(nodeAttributes) + "] found for shard:" + shardId());
         }
         return new PlainShardIterator(shardId, ordered);
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
index 70246026894..3fb6f55a919 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
@@ -177,8 +177,8 @@ public class OperationRouting extends AbstractComponent {
                     ensureNodeIdExists(nodes, nodeId);
                     return indexShard.onlyNodeActiveInitializingShardsIt(nodeId);
                 case ONLY_NODES:
-                    String nodeAttribute = preference.substring(Preference.ONLY_NODES.type().length() + 1);
-                    return indexShard.onlyNodeSelectorActiveInitializingShardsIt(nodeAttribute, nodes);
+                    String nodeAttributes = preference.substring(Preference.ONLY_NODES.type().length() + 1);
+                    return indexShard.onlyNodeSelectorActiveInitializingShardsIt(nodeAttributes.split(","), nodes);
                 default:
                     throw new IllegalArgumentException("unknown preference [" + preferenceType + "]");
             }
diff --git a/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java b/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java
index 255693669c1..c9dd0e26361 100644
--- a/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java
@@ -320,6 +320,24 @@ public class RoutingIteratorTests extends ESAllocationTestCase {
         assertThat(shardsIterator.size(), equalTo(1));
         assertThat(shardsIterator.nextOrNull().currentNodeId(),equalTo("node2"));
 
+        shardsIterator = clusterState.routingTable().index("test").shard(0)
+            .onlyNodeSelectorActiveInitializingShardsIt(new String[] {"disk:eph*","disk:ebs"},clusterState.nodes());
+        assertThat(shardsIterator.size(), equalTo(2));
+        assertThat(shardsIterator.nextOrNull().currentNodeId(),equalTo("node2"));
+        assertThat(shardsIterator.nextOrNull().currentNodeId(),equalTo("node1"));
+
+        shardsIterator = clusterState.routingTable().index("test").shard(0)
+            .onlyNodeSelectorActiveInitializingShardsIt(new String[] {"disk:*", "invalid_name"},clusterState.nodes());
+        assertThat(shardsIterator.size(), equalTo(2));
+        assertThat(shardsIterator.nextOrNull().currentNodeId(),equalTo("node2"));
+        assertThat(shardsIterator.nextOrNull().currentNodeId(),equalTo("node1"));
+
+        shardsIterator = clusterState.routingTable().index("test").shard(0)
+            .onlyNodeSelectorActiveInitializingShardsIt(new String[] {"disk:*", "disk:*"},clusterState.nodes());
+        assertThat(shardsIterator.size(), equalTo(2));
+        assertThat(shardsIterator.nextOrNull().currentNodeId(),equalTo("node2"));
+        assertThat(shardsIterator.nextOrNull().currentNodeId(),equalTo("node1"));
+
         try {
             shardsIterator = clusterState.routingTable().index("test").shard(0).onlyNodeSelectorActiveInitializingShardsIt("welma", clusterState.nodes());
             fail("should have raised illegalArgumentException");
diff --git a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java
index 654712d928e..8bfadeb9443 100644
--- a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java
+++ b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java
@@ -19,22 +19,30 @@
 
 package org.elasticsearch.search.preference;
 
+import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
+import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
+import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Set;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 @ESIntegTestCase.ClusterScope(minNumDataNodes = 2)
 public class SearchPreferenceIT extends ESIntegTestCase {
@@ -155,4 +163,63 @@ public class SearchPreferenceIT extends ESIntegTestCase {
             assertThat(e.getMessage(), is("No data node with id[DOES-NOT-EXIST] found"));
         }
     }
+
+    public void testNodesOnlyRandom() throws Exception {
+        assertAcked(prepareCreate("test").setSettings(
+            //this test needs at least a replica to make sure two consecutive searches go to two different copies of the same data
+            Settings.builder().put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(1, maximumNumberOfReplicas()))));
+        ensureGreen();
+        client().prepareIndex("test", "type1").setSource("field1", "value1").execute().actionGet();
+        refresh();
+
+        final Client client = internalCluster().smartClient();
+        SearchRequestBuilder request = client.prepareSearch("test")
+            .setQuery(matchAllQuery()).setPreference("_only_nodes:*,nodes*"); // multiple wildchar  to cover multi-param usecase
+        assertSearchOnRandomNodes(request);
+
+        request = client.prepareSearch("test")
+            .setQuery(matchAllQuery()).setPreference("_only_nodes:*");
+        assertSearchOnRandomNodes(request);
+
+        ArrayList<String> allNodeIds = new ArrayList<>();
+        ArrayList<String> allNodeNames = new ArrayList<>();
+        ArrayList<String> allNodeHosts = new ArrayList<>();
+        NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().execute().actionGet();
+        for (NodeStats node : nodeStats.getNodes()) {
+            allNodeIds.add(node.getNode().getId());
+            allNodeNames.add(node.getNode().getName());
+            allNodeHosts.add(node.getHostname());
+        }
+
+        String node_expr = "_only_nodes:" + Strings.arrayToCommaDelimitedString(allNodeIds.toArray());
+        request = client.prepareSearch("test").setQuery(matchAllQuery()).setPreference(node_expr);
+        assertSearchOnRandomNodes(request);
+
+        node_expr = "_only_nodes:" + Strings.arrayToCommaDelimitedString(allNodeNames.toArray());
+        request = client.prepareSearch("test").setQuery(matchAllQuery()).setPreference(node_expr);
+        assertSearchOnRandomNodes(request);
+
+        node_expr = "_only_nodes:" + Strings.arrayToCommaDelimitedString(allNodeHosts.toArray());
+        request = client.prepareSearch("test").setQuery(matchAllQuery()).setPreference(node_expr);
+        assertSearchOnRandomNodes(request);
+
+        node_expr = "_only_nodes:" + Strings.arrayToCommaDelimitedString(allNodeHosts.toArray());
+        request = client.prepareSearch("test").setQuery(matchAllQuery()).setPreference(node_expr);
+        assertSearchOnRandomNodes(request);
+
+        // Mix of valid and invalid nodes
+        node_expr = "_only_nodes:*,invalidnode";
+        request = client.prepareSearch("test").setQuery(matchAllQuery()).setPreference(node_expr);
+        assertSearchOnRandomNodes(request);
+    }
+
+    private void assertSearchOnRandomNodes(SearchRequestBuilder request) {
+        Set<String> hitNodes = new HashSet<>();
+        for (int i = 0; i < 2; i++) {
+            SearchResponse searchResponse = request.execute().actionGet();
+            assertThat(searchResponse.getHits().getHits().length, greaterThan(0));
+            hitNodes.add(searchResponse.getHits().getAt(0).shard().nodeId());
+        }
+        assertThat(hitNodes.size(), greaterThan(1));
+    }
 }

From c5a9edf1c7707359d90e29504ef6ff7f86ad548f Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Tue, 17 May 2016 14:34:56 +0200
Subject: [PATCH 0416/1311] Add `Character.MODIFIER_SYMBOL` to the list of
 symbol categories. #18402

Closes #18388
---
 .../index/analysis/CharMatcher.java           |  1 +
 .../index/analysis/CharMatcherTests.java      | 71 +++++++++++++++++++
 2 files changed, 72 insertions(+)
 create mode 100644 core/src/test/java/org/elasticsearch/index/analysis/CharMatcherTests.java

diff --git a/core/src/main/java/org/elasticsearch/index/analysis/CharMatcher.java b/core/src/main/java/org/elasticsearch/index/analysis/CharMatcher.java
index 5584cc54df2..90c6ad3f3b4 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/CharMatcher.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/CharMatcher.java
@@ -88,6 +88,7 @@ public interface CharMatcher {
                 case Character.CURRENCY_SYMBOL:
                 case Character.MATH_SYMBOL:
                 case Character.OTHER_SYMBOL:
+                case Character.MODIFIER_SYMBOL:
                     return true;
                  default:
                      return false;
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/CharMatcherTests.java b/core/src/test/java/org/elasticsearch/index/analysis/CharMatcherTests.java
new file mode 100644
index 00000000000..31f80a66cda
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/analysis/CharMatcherTests.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.elasticsearch.test.ESTestCase;
+
+public class CharMatcherTests extends ESTestCase {
+
+    public void testLetter() {
+        assertTrue(CharMatcher.Basic.LETTER.isTokenChar('a')); // category Ll
+        assertTrue(CharMatcher.Basic.LETTER.isTokenChar('é')); // category Ll
+        assertTrue(CharMatcher.Basic.LETTER.isTokenChar('A')); // category Lu
+        assertTrue(CharMatcher.Basic.LETTER.isTokenChar('Ã…')); // category Lu
+        assertTrue(CharMatcher.Basic.LETTER.isTokenChar('Ê°')); // category Lm
+        assertTrue(CharMatcher.Basic.LETTER.isTokenChar('ª')); // category Lo
+        assertTrue(CharMatcher.Basic.LETTER.isTokenChar('Ç…')); // category Lt
+        assertFalse(CharMatcher.Basic.LETTER.isTokenChar(' '));
+        assertFalse(CharMatcher.Basic.LETTER.isTokenChar('0'));
+        assertFalse(CharMatcher.Basic.LETTER.isTokenChar('!'));
+    }
+
+    public void testSpace() {
+        assertTrue(CharMatcher.Basic.WHITESPACE.isTokenChar(' '));
+        assertTrue(CharMatcher.Basic.WHITESPACE.isTokenChar('\t'));
+        assertFalse(CharMatcher.Basic.WHITESPACE.isTokenChar('\u00A0')); // nbsp
+    }
+
+    public void testNumber() {
+        assertTrue(CharMatcher.Basic.DIGIT.isTokenChar('1'));
+        assertTrue(CharMatcher.Basic.DIGIT.isTokenChar('Ù¡')); // ARABIC-INDIC DIGIT ONE
+        assertFalse(CharMatcher.Basic.DIGIT.isTokenChar(','));
+        assertFalse(CharMatcher.Basic.DIGIT.isTokenChar('a'));
+    }
+
+    public void testSymbol() {
+        assertTrue(CharMatcher.Basic.SYMBOL.isTokenChar('$')); // category Sc
+        assertTrue(CharMatcher.Basic.SYMBOL.isTokenChar('+')); // category Sm
+        assertTrue(CharMatcher.Basic.SYMBOL.isTokenChar('`')); // category Sm
+        assertTrue(CharMatcher.Basic.SYMBOL.isTokenChar('^')); // category Sk
+        assertTrue(CharMatcher.Basic.SYMBOL.isTokenChar('¦')); // category Sc
+        assertFalse(CharMatcher.Basic.SYMBOL.isTokenChar(' '));
+    }
+
+    public void testPunctuation() {
+        assertTrue(CharMatcher.Basic.PUNCTUATION.isTokenChar('(')); // category Ps
+        assertTrue(CharMatcher.Basic.PUNCTUATION.isTokenChar(')')); // category Pe
+        assertTrue(CharMatcher.Basic.PUNCTUATION.isTokenChar('_')); // category Pc
+        assertTrue(CharMatcher.Basic.PUNCTUATION.isTokenChar('!')); // category Po
+        assertTrue(CharMatcher.Basic.PUNCTUATION.isTokenChar('-')); // category Pd
+        assertTrue(CharMatcher.Basic.PUNCTUATION.isTokenChar('«')); // category Pi
+        assertTrue(CharMatcher.Basic.PUNCTUATION.isTokenChar('»')); // category Pf
+        assertFalse(CharMatcher.Basic.PUNCTUATION.isTokenChar(' '));
+    }
+}

From b0b503035ab35e8c2d2c4dcdc936e76b509c6ff3 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Fri, 20 May 2016 21:11:15 +0200
Subject: [PATCH 0417/1311] Update reindex cancel tests

---
 .../AbstractAsyncBulkByScrollAction.java      |  11 +-
 .../reindex/UpdateByQueryRequestBuilder.java  |   5 +
 .../reindex/AsyncBulkByScrollActionTests.java |   3 +-
 .../index/reindex/CancelTestUtils.java        | 166 ------------
 .../index/reindex/CancelTests.java            | 240 ++++++++++++++++++
 .../reindex/DeleteByQueryCancelTests.java     | 184 --------------
 .../index/reindex/ReindexCancelTests.java     |  52 ----
 .../reindex/UpdateByQueryCancelTests.java     |  53 ----
 8 files changed, 254 insertions(+), 460 deletions(-)
 delete mode 100644 modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTestUtils.java
 create mode 100644 modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
 delete mode 100644 modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryCancelTests.java
 delete mode 100644 modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexCancelTests.java
 delete mode 100644 modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryCancelTests.java

diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
index ee2f5484737..699c6340095 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
@@ -259,10 +259,6 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
      * Processes bulk responses, accounting for failures.
      */
     void onBulkResponse(BulkResponse response) {
-        if (task.isCancelled()) {
-            finishHim(null);
-            return;
-        }
         try {
             List<Failure> failures = new ArrayList<Failure>();
             Set<String> destinationIndicesThisBatch = new HashSet<>();
@@ -291,6 +287,12 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
                 // Track the indexes we've seen so we can refresh them if requested
                 destinationIndicesThisBatch.add(item.getIndex());
             }
+
+            if (task.isCancelled()) {
+                finishHim(null);
+                return;
+            }
+
             addDestinationIndices(destinationIndicesThisBatch);
 
             if (false == failures.isEmpty()) {
@@ -303,6 +305,7 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
                 startNormalTermination(emptyList(), emptyList(), false);
                 return;
             }
+
             startNextScroll(response.getItems().length);
         } catch (Throwable t) {
             finishHim(t);
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java
index 8985f439fba..311e138fb1f 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/UpdateByQueryRequestBuilder.java
@@ -48,4 +48,9 @@ public class UpdateByQueryRequestBuilder extends
         request.setAbortOnVersionConflict(abortOnVersionConflict);
         return this;
     }
+
+    public UpdateByQueryRequestBuilder setPipeline(String pipeline) {
+        request.setPipeline(pipeline);
+        return this;
+    }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
index d78fcfe69ca..9ce505bacd3 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
@@ -585,7 +585,8 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
 
     public void testCancelBeforeOnBulkResponse() throws Exception {
         // We bail so early we don't need to pass in a half way valid response.
-        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.onBulkResponse(null));
+        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) ->
+                action.onBulkResponse(new BulkResponse(new BulkItemResponse[0], 0)));
     }
 
     public void testCancelBeforeStartNextScroll() throws Exception {
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTestUtils.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTestUtils.java
deleted file mode 100644
index fc6af19e6b3..00000000000
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTestUtils.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.reindex;
-
-import org.elasticsearch.action.ListenableActionFuture;
-import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
-import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
-import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo;
-import org.elasticsearch.index.reindex.BulkByScrollTask.Status;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.script.ExecutableScript;
-import org.elasticsearch.script.NativeScriptFactory;
-import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptModule;
-import org.elasticsearch.script.ScriptService.ScriptType;
-import org.elasticsearch.test.ESIntegTestCase;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.BrokenBarrierException;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import static java.util.Collections.emptyMap;
-import static org.elasticsearch.test.ESIntegTestCase.client;
-import static org.hamcrest.Matchers.empty;
-import static org.hamcrest.Matchers.hasSize;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertThat;
-
-/**
- * Utilities for testing reindex and update-by-query cancellation. This whole class isn't thread safe. Luckily we run our tests in separate
- * jvms.
- */
-public class CancelTestUtils {
-    public static Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(ReindexPlugin.class, StickyScriptPlugin.class);
-    }
-
-    private static final CyclicBarrier barrier = new CyclicBarrier(2);
-
-    public static <Request extends AbstractBulkIndexByScrollRequest<Request>,
-                    Builder extends AbstractBulkIndexByScrollRequestBuilder<Request, Builder>>
-            BulkIndexByScrollResponse testCancel(ESIntegTestCase test, Builder request, String actionToCancel) throws Exception {
-
-        test.indexRandom(true, client().prepareIndex("source", "test", "1").setSource("foo", "a"),
-                client().prepareIndex("source", "test", "2").setSource("foo", "a"));
-
-        request.source("source").script(new Script("sticky", ScriptType.INLINE, "native", emptyMap()));
-        request.source().setSize(1);
-        ListenableActionFuture<BulkIndexByScrollResponse> response = request.execute();
-
-        // Wait until the script is on the first document.
-        barrier.await(30, TimeUnit.SECONDS);
-
-        // Let just one document through.
-        barrier.await(30, TimeUnit.SECONDS);
-
-        // Wait until the script is on the second document.
-        barrier.await(30, TimeUnit.SECONDS);
-
-        // Status should show running
-        ListTasksResponse tasksList = client().admin().cluster().prepareListTasks().setActions(actionToCancel).setDetailed(true).get();
-        assertThat(tasksList.getNodeFailures(), empty());
-        assertThat(tasksList.getTaskFailures(), empty());
-        assertThat(tasksList.getTasks(), hasSize(1));
-        BulkByScrollTask.Status status = (Status) tasksList.getTasks().get(0).getStatus();
-        assertNull(status.getReasonCancelled());
-
-        // Cancel the request while the script is running. This will prevent the request from being sent at all.
-        List<TaskInfo> cancelledTasks = client().admin().cluster().prepareCancelTasks().setActions(actionToCancel).get().getTasks();
-        assertThat(cancelledTasks, hasSize(1));
-
-        // The status should now show canceled. The request will still be in the list because the script is still blocked.
-        tasksList = client().admin().cluster().prepareListTasks().setActions(actionToCancel).setDetailed(true).get();
-        assertThat(tasksList.getNodeFailures(), empty());
-        assertThat(tasksList.getTaskFailures(), empty());
-        assertThat(tasksList.getTasks(), hasSize(1));
-        status = (Status) tasksList.getTasks().get(0).getStatus();
-        assertEquals(CancelTasksRequest.DEFAULT_REASON, status.getReasonCancelled());
-
-        // Now let the next document through. It won't be sent because the request is cancelled but we need to unblock the script.
-        barrier.await();
-
-        // Now we can just wait on the request and make sure it was actually cancelled half way through.
-        return response.get();
-    }
-
-    public static class StickyScriptPlugin extends Plugin {
-        @Override
-        public String name() {
-            return "sticky-script";
-        }
-
-        @Override
-        public String description() {
-            return "installs a script that \"sticks\" when it runs for testing reindex";
-        }
-
-        public void onModule(ScriptModule module) {
-            module.registerScript("sticky", StickyScriptFactory.class);
-        }
-    }
-
-    public static class StickyScriptFactory implements NativeScriptFactory {
-        @Override
-        public ExecutableScript newScript(Map<String, Object> params) {
-            return new ExecutableScript() {
-                private Map<String, Object> source;
-                @Override
-                @SuppressWarnings("unchecked") // Safe because _ctx always has this shape
-                public void setNextVar(String name, Object value) {
-                    if ("ctx".equals(name)) {
-                        Map<String, Object> ctx = (Map<String, Object>) value;
-                        source = (Map<String, Object>) ctx.get("_source");
-                    } else {
-                        throw new IllegalArgumentException("Unexpected var:  " + name);
-                    }
-                }
-
-                @Override
-                public Object run() {
-                    try {
-                        // Tell the test we've started a document.
-                        barrier.await(30, TimeUnit.SECONDS);
-
-                        // Wait for the test to tell us to proceed.
-                        barrier.await(30, TimeUnit.SECONDS);
-
-                        // Make some change to the source so that update-by-query tests can make sure only one document was changed.
-                        source.put("giraffes", "giraffes");
-                        return null;
-                    } catch (InterruptedException | BrokenBarrierException | TimeoutException e) {
-                        throw new RuntimeException(e);
-                    }
-                }
-            };
-        }
-
-        @Override
-        public boolean needsScores() {
-            return false;
-        }
-    }
-}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
new file mode 100644
index 00000000000..5a778d0b0fa
--- /dev/null
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
@@ -0,0 +1,240 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.ListenableActionFuture;
+import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo;
+import org.elasticsearch.action.ingest.DeletePipelineRequest;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.index.IndexModule;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.engine.Engine.Operation.Origin;
+import org.elasticsearch.index.shard.IndexingOperationListener;
+import org.elasticsearch.plugins.Plugin;
+import org.junit.BeforeClass;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.emptyIterable;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+
+/**
+ * Test that you can actually cancel a reindex/update-by-query/delete-by-query request and all the plumbing works. Doesn't test all of the
+ * different cancellation places - that is the responsibility of {@link AsyncBulkByScrollActionTests} which have more precise control to
+ * simulate failures but do not exercise important portion of the stack like transport and task management.
+ */
+public class CancelTests extends ReindexTestCase {
+
+    protected static final String INDEX = "reindex-cancel-index";
+    protected static final String TYPE = "reindex-cancel-type";
+
+    private static final int MIN_OPERATIONS = 2;
+    private static final int BLOCKING_OPERATIONS = 1;
+
+    // Semaphore used to allow & block indexing operations during the test
+    private static final Semaphore ALLOWED_OPERATIONS = new Semaphore(0);
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        Collection<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
+        plugins.add(ReindexCancellationPlugin.class);
+        return plugins;
+    }
+
+    @BeforeClass
+    public static void clearAllowedOperations() {
+        ALLOWED_OPERATIONS.drainPermits();
+    }
+
+    /**
+     * Executes the cancellation test
+     */
+    private void testCancel(String action,
+                            AbstractBulkByScrollRequestBuilder<?, ? extends BulkIndexByScrollResponse, ?> builder,
+                            CancelAssertion assertion) throws Exception {
+
+        createIndex(INDEX);
+
+        // Total number of documents created for this test (~10 per primary shard)
+        int numDocs = getNumShards(INDEX).numPrimaries * 10;
+        ALLOWED_OPERATIONS.release(numDocs);
+
+        indexRandom(true, false, true, IntStream.range(0, numDocs)
+                .mapToObj(i -> client().prepareIndex(INDEX, TYPE, String.valueOf(i)).setSource("n", i))
+                .collect(Collectors.toList()));
+
+        // Checks that the all documents have been indexed and correctly counted
+        assertHitCount(client().prepareSearch(INDEX).setSize(0).get(), numDocs);
+        assertThat(ALLOWED_OPERATIONS.drainPermits(), equalTo(0));
+
+        // Scroll 1 by 1 so that cancellation is easier to control
+        builder.source().setSize(1);
+
+        // Allow a random number of the documents minus 1
+        // to be modified by the reindex action
+        int numModifiedDocs = randomIntBetween(MIN_OPERATIONS, numDocs);
+        ALLOWED_OPERATIONS.release(numModifiedDocs - BLOCKING_OPERATIONS);
+
+        // Now execute the reindex action...
+        ListenableActionFuture<? extends BulkIndexByScrollResponse> future = builder.execute();
+
+        // ... and waits for the indexing operation listeners to block
+        awaitBusy(() -> ALLOWED_OPERATIONS.hasQueuedThreads() && ALLOWED_OPERATIONS.availablePermits() == 0);
+
+        // Status should show the task running
+        ListTasksResponse tasksList = client().admin().cluster().prepareListTasks().setActions(action).setDetailed(true).get();
+        assertThat(tasksList.getNodeFailures(), empty());
+        assertThat(tasksList.getTaskFailures(), empty());
+        assertThat(tasksList.getTasks(), hasSize(1));
+        BulkByScrollTask.Status status = (BulkByScrollTask.Status) tasksList.getTasks().get(0).getStatus();
+        assertNull(status.getReasonCancelled());
+
+        // Cancel the request while the reindex action is blocked by the indexing operation listeners.
+        // This will prevent further requests from being sent.
+        List<TaskInfo> cancelledTasks = client().admin().cluster().prepareCancelTasks().setActions(action).get().getTasks();
+        assertThat(cancelledTasks, hasSize(1));
+
+        // The status should now show canceled. The request will still be in the list because it is still blocked.
+        tasksList = client().admin().cluster().prepareListTasks().setActions(action).setDetailed(true).get();
+        assertThat(tasksList.getNodeFailures(), empty());
+        assertThat(tasksList.getTaskFailures(), empty());
+        assertThat(tasksList.getTasks(), hasSize(1));
+        status = (BulkByScrollTask.Status) tasksList.getTasks().get(0).getStatus();
+        assertEquals(CancelTasksRequest.DEFAULT_REASON, status.getReasonCancelled());
+
+        // Unblock the last operation
+        ALLOWED_OPERATIONS.release(BLOCKING_OPERATIONS);
+
+        // Checks that no more operations are executed
+        assertBusy(() -> ALLOWED_OPERATIONS.availablePermits() == 0 && ALLOWED_OPERATIONS.getQueueLength() == 0);
+
+        // And check the status of the response
+        BulkIndexByScrollResponse response = future.get();
+        assertThat(response.getReasonCancelled(), equalTo("by user request"));
+        assertThat(response.getIndexingFailures(), emptyIterable());
+        assertThat(response.getSearchFailures(), emptyIterable());
+
+        flushAndRefresh(INDEX);
+        assertion.assertThat(response, numDocs, numModifiedDocs);
+    }
+
+    public void testReindexCancel() throws Exception {
+        testCancel(ReindexAction.NAME, reindex().source(INDEX).destination("dest", TYPE), (response, total, modified) -> {
+            assertThat(response, matcher().created(modified).reasonCancelled(equalTo("by user request")));
+
+            refresh("dest");
+            assertHitCount(client().prepareSearch("dest").setTypes(TYPE).setSize(0).get(), modified);
+        });
+    }
+
+    public void testUpdateByQueryCancel() throws Exception {
+        BytesReference pipeline = new BytesArray("{\n" +
+                "  \"description\" : \"sets updated to true\",\n" +
+                "  \"processors\" : [ {\n" +
+                "      \"set\" : {\n" +
+                "        \"field\": \"updated\",\n" +
+                "        \"value\": true" +
+                "      }\n" +
+                "  } ]\n" +
+                "}");
+        assertAcked(client().admin().cluster().preparePutPipeline("set-foo", pipeline).get());
+
+        testCancel(UpdateByQueryAction.NAME, updateByQuery().setPipeline("set-foo").source(INDEX), (response, total, modified) -> {
+            assertThat(response, matcher().updated(modified).reasonCancelled(equalTo("by user request")));
+            assertHitCount(client().prepareSearch(INDEX).setSize(0).setQuery(termQuery("updated", true)).get(), modified);
+        });
+
+        assertAcked(client().admin().cluster().deletePipeline(new DeletePipelineRequest("set-foo")).get());
+    }
+
+    public void testDeleteByQueryCancel() throws Exception {
+        testCancel(DeleteByQueryAction.NAME, deleteByQuery().source(INDEX), (response, total, modified) -> {
+            assertThat(response, matcher().deleted(modified).reasonCancelled(equalTo("by user request")));
+            assertHitCount(client().prepareSearch(INDEX).setSize(0).get(), total - modified);
+        });
+    }
+
+    /**
+     * {@link CancelAssertion} is used to check the result of the cancel test.
+     */
+    private interface CancelAssertion {
+        void assertThat(BulkIndexByScrollResponse response, int total, int modified);
+    }
+
+    public static class ReindexCancellationPlugin extends Plugin {
+
+        @Override
+        public String name() {
+            return "reindex-cancellation";
+        }
+
+        @Override
+        public String description() {
+            return "See " + CancelTests.class.getName() + " documentation";
+        }
+
+        @Override
+        public void onIndexModule(IndexModule indexModule) {
+            indexModule.addIndexOperationListener(new BlockingDeleteListener());
+        }
+    }
+
+    public static class BlockingDeleteListener implements IndexingOperationListener {
+
+        @Override
+        public Engine.Index preIndex(Engine.Index index) {
+            return preCheck(index, index.type());
+        }
+
+        @Override
+        public Engine.Delete preDelete(Engine.Delete delete) {
+            return preCheck(delete, delete.type());
+        }
+
+        private <T extends Engine.Operation> T preCheck(T operation, String type) {
+            if ((TYPE.equals(type) == false) || (operation.origin() != Origin.PRIMARY)) {
+                return operation;
+            }
+
+            try {
+                if (ALLOWED_OPERATIONS.tryAcquire(30, TimeUnit.SECONDS)) {
+                    return operation;
+                }
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+            throw new IllegalStateException("Something went wrong");
+        }
+    }
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryCancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryCancelTests.java
deleted file mode 100644
index 6007b646429..00000000000
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/DeleteByQueryCancelTests.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.reindex;
-
-import org.elasticsearch.action.ListenableActionFuture;
-import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
-import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
-import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo;
-import org.elasticsearch.common.util.concurrent.CountDown;
-import org.elasticsearch.index.IndexModule;
-import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.shard.IndexingOperationListener;
-import org.elasticsearch.plugins.Plugin;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.concurrent.BrokenBarrierException;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.hamcrest.Matchers.empty;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.lessThanOrEqualTo;
-
-/**
- * Tests that you can actually cancel a delete-by-query request and all the plumbing works. Doesn't test all of the different cancellation
- * places - that is the responsibility of {@link AsyncBulkByScrollActionTests} which have more precise control to simulate failures but do
- * not exercise important portion of the stack like transport and task management.
- */
-public class DeleteByQueryCancelTests extends ReindexTestCase {
-
-    private static final String INDEX = "test-delete-by-query";
-    private static final String TYPE = "test";
-
-    private static final int MAX_DELETIONS = 10;
-    private static final CyclicBarrier barrier = new CyclicBarrier(2);
-
-    @Override
-    protected int numberOfShards() {
-        // Only 1 shard and no replica so that test execution
-        // can be easily controlled within a {@link IndexingOperationListener#preDelete}
-        return 1;
-    }
-
-    @Override
-    protected int numberOfReplicas() {
-        // Only 1 shard and no replica so that test execution
-        // can be easily controlled within a {@link IndexingOperationListener#preDelete}
-        return 0;
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        Collection<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
-        plugins.add(DeleteByQueryCancellationPlugin.class);
-        return plugins;
-    }
-
-    public void testCancel() throws Exception {
-        createIndex(INDEX);
-
-        int totalNumShards = getNumShards(INDEX).totalNumShards;
-
-        // Number of documents to be deleted in this test
-        final int nbDocsToDelete = totalNumShards * MAX_DELETIONS;
-
-        // Total number of documents that will be created in this test
-        final int nbDocs = nbDocsToDelete * randomIntBetween(1, 5);
-        for (int i = 0; i < nbDocs; i++) {
-            indexRandom(false, client().prepareIndex(INDEX, TYPE, String.valueOf(i)).setSource("n", i));
-        }
-
-        refresh(INDEX);
-        assertHitCount(client().prepareSearch(INDEX).setSize(0).get(), nbDocs);
-
-        // Executes the delete by query; each shard will block after MAX_DELETIONS
-        DeleteByQueryRequestBuilder deleteByQuery = deleteByQuery().source("_all");
-        deleteByQuery.source().setSize(1);
-
-        ListenableActionFuture<BulkIndexByScrollResponse> future = deleteByQuery.execute();
-
-        // Waits for the indexing operation listener to block
-        barrier.await(30, TimeUnit.SECONDS);
-
-        // Status should show running
-        ListTasksResponse tasksList = client().admin().cluster().prepareListTasks()
-                .setActions(DeleteByQueryAction.NAME).setDetailed(true).get();
-        assertThat(tasksList.getNodeFailures(), empty());
-        assertThat(tasksList.getTaskFailures(), empty());
-        assertThat(tasksList.getTasks(), hasSize(1));
-        BulkByScrollTask.Status status = (BulkByScrollTask.Status) tasksList.getTasks().get(0).getStatus();
-        assertNull(status.getReasonCancelled());
-
-        // Cancel the request while the deletions are blocked. This will prevent further deletions requests from being sent.
-        List<TaskInfo> cancelledTasks = client().admin().cluster().prepareCancelTasks()
-                .setActions(DeleteByQueryAction.NAME).get().getTasks();
-        assertThat(cancelledTasks, hasSize(1));
-
-        // The status should now show canceled. The request will still be in the list because the script is still blocked.
-        tasksList = client().admin().cluster().prepareListTasks().setActions(DeleteByQueryAction.NAME).setDetailed(true).get();
-        assertThat(tasksList.getNodeFailures(), empty());
-        assertThat(tasksList.getTaskFailures(), empty());
-        assertThat(tasksList.getTasks(), hasSize(1));
-        status = (BulkByScrollTask.Status) tasksList.getTasks().get(0).getStatus();
-        assertEquals(CancelTasksRequest.DEFAULT_REASON, status.getReasonCancelled());
-
-        // Now unblock the listener so that it can proceed
-        barrier.await();
-
-        // And check the status of the response
-        BulkIndexByScrollResponse response = future.get();
-        assertThat(response, matcher()
-                .deleted(lessThanOrEqualTo((long) MAX_DELETIONS)).batches(MAX_DELETIONS).reasonCancelled(equalTo("by user request")));
-    }
-
-
-    public static class DeleteByQueryCancellationPlugin extends Plugin {
-
-        @Override
-        public String name() {
-            return "delete-by-query-cancellation";
-        }
-
-        @Override
-        public String description() {
-            return "See " + DeleteByQueryCancellationPlugin.class.getName();
-        }
-
-        @Override
-        public void onIndexModule(IndexModule indexModule) {
-            indexModule.addIndexOperationListener(new BlockingDeleteListener());
-        }
-    }
-
-    /**
-     * A {@link IndexingOperationListener} that allows a given number of documents to be deleted
-     * and then blocks until it is notified to proceed.
-     */
-    public static class BlockingDeleteListener implements IndexingOperationListener {
-
-        private final CountDown blockAfter = new CountDown(MAX_DELETIONS);
-
-        @Override
-        public Engine.Delete preDelete(Engine.Delete delete) {
-            if (blockAfter.isCountedDown() || (TYPE.equals(delete.type()) == false)) {
-                return delete;
-            }
-
-            if (blockAfter.countDown()) {
-                try {
-                    // Tell the test we've deleted enough documents.
-                    barrier.await(30, TimeUnit.SECONDS);
-
-                    // Wait for the test to tell us to proceed.
-                    barrier.await(30, TimeUnit.SECONDS);
-                } catch (InterruptedException | BrokenBarrierException | TimeoutException e) {
-                    throw new RuntimeException(e);
-                }
-            }
-            return delete;
-        }
-    }
-}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexCancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexCancelTests.java
deleted file mode 100644
index f9dde6045a6..00000000000
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexCancelTests.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.reindex;
-
-import org.elasticsearch.plugins.Plugin;
-
-import java.util.Collection;
-
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.hamcrest.Matchers.equalTo;
-
-/**
- * Tests that you can actually cancel a reindex request and all the plumbing works. Doesn't test all of the different cancellation places -
- * that is the responsibility of {@link AsyncBulkByScrollActionTests} which have more precise control to simulate failures but do not
- * exercise important portion of the stack like transport and task management.
- */
-public class ReindexCancelTests extends ReindexTestCase {
-    public void testCancel() throws Exception {
-        BulkIndexByScrollResponse response = CancelTestUtils.testCancel(this, reindex().destination("dest", "test"), ReindexAction.NAME);
-
-        assertThat(response, matcher().created(1).reasonCancelled(equalTo("by user request")));
-        refresh("dest");
-        assertHitCount(client().prepareSearch("dest").setSize(0).get(), 1);
-    }
-
-    @Override
-    protected int numberOfShards() {
-        return 1;
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return CancelTestUtils.nodePlugins();
-    }
-}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryCancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryCancelTests.java
deleted file mode 100644
index 4cb859c0017..00000000000
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryCancelTests.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.reindex;
-
-import org.elasticsearch.plugins.Plugin;
-
-import java.util.Collection;
-
-import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.hamcrest.Matchers.equalTo;
-
-/**
- * Tests that you can actually cancel an update-by-query request and all the plumbing works. Doesn't test all of the different cancellation
- * places - that is the responsibility of {@link AsyncBulkByScrollActionTests} which have more precise control to simulate failures but do
- * not exercise important portion of the stack like transport and task management.
- */
-public class UpdateByQueryCancelTests extends ReindexTestCase {
-    public void testCancel() throws Exception {
-        BulkIndexByScrollResponse response = CancelTestUtils.testCancel(this, updateByQuery(), UpdateByQueryAction.NAME);
-
-        assertThat(response, matcher().updated(1).reasonCancelled(equalTo("by user request")));
-        refresh("source");
-        assertHitCount(client().prepareSearch("source").setSize(0).setQuery(matchQuery("giraffes", "giraffes")).get(), 1);
-    }
-
-    @Override
-    protected int numberOfShards() {
-        return 1;
-    }
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return CancelTestUtils.nodePlugins();
-    }
-}

From e7eb664c78dce2451dbeb55db0c0b761a65b295b Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Thu, 19 May 2016 14:53:22 +0200
Subject: [PATCH 0418/1311] Change BlobPath.buildAsString() method

---
 .../common/blobstore/BlobPath.java            | 15 ++++---
 .../common/blobstore/BlobPathTests.java       | 39 +++++++++++++++++++
 .../snapshots/mockstore/MockRepository.java   |  2 +-
 .../azure/blobstore/AzureBlobContainer.java   |  6 +--
 .../cloud/azure/blobstore/AzureBlobStore.java |  6 +--
 .../gcs/GoogleCloudStorageBlobContainer.java  |  8 +---
 .../gcs/GoogleCloudStorageBlobStore.java      |  7 +---
 .../cloud/aws/blobstore/S3BlobContainer.java  |  6 +--
 .../cloud/aws/blobstore/S3BlobStore.java      |  6 +--
 9 files changed, 53 insertions(+), 42 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/common/blobstore/BlobPathTests.java

diff --git a/core/src/main/java/org/elasticsearch/common/blobstore/BlobPath.java b/core/src/main/java/org/elasticsearch/common/blobstore/BlobPath.java
index 7636097e288..858486d282c 100644
--- a/core/src/main/java/org/elasticsearch/common/blobstore/BlobPath.java
+++ b/core/src/main/java/org/elasticsearch/common/blobstore/BlobPath.java
@@ -30,6 +30,8 @@ import java.util.List;
  */
 public class BlobPath implements Iterable<String> {
 
+    private static final String SEPARATOR = "/";
+
     private final List<String> paths;
 
     public BlobPath() {
@@ -60,15 +62,12 @@ public class BlobPath implements Iterable<String> {
         return new BlobPath(Collections.unmodifiableList(paths));
     }
 
-    public String buildAsString(String separator) {
-        StringBuilder sb = new StringBuilder();
-        for (int i = 0; i < paths.size(); i++) {
-            sb.append(paths.get(i));
-            if (i < (paths.size() - 1)) {
-                sb.append(separator);
-            }
+    public String buildAsString() {
+        String p = String.join(SEPARATOR, paths);
+        if (p.isEmpty()) {
+            return p;
         }
-        return sb.toString();
+        return p + SEPARATOR;
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/common/blobstore/BlobPathTests.java b/core/src/test/java/org/elasticsearch/common/blobstore/BlobPathTests.java
new file mode 100644
index 00000000000..09225914644
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/blobstore/BlobPathTests.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.blobstore;
+
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.is;
+
+public class BlobPathTests extends ESTestCase {
+
+    public void testBuildAsString() {
+        BlobPath path = new BlobPath();
+        assertThat(path.buildAsString(), is(""));
+
+        path = path.add("a");
+        assertThat(path.buildAsString(), is("a/"));
+
+        path = path.add("b").add("c");
+        assertThat(path.buildAsString(), is("a/b/c/"));
+
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java b/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
index cbd9bc0e96f..7df44738076 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
@@ -231,7 +231,7 @@ public class MockRepository extends FsRepository {
 
             private boolean shouldFail(String blobName, double probability) {
                 if (probability > 0.0) {
-                    String path = path().add(blobName).buildAsString("/") + "/" + randomPrefix;
+                    String path = path().add(blobName).buildAsString() + randomPrefix;
                     path += "/" + incrementAndGet(path);
                     logger.info("checking [{}] [{}]", path, Math.abs(hashCode(path)) < Integer.MAX_VALUE * probability);
                     return Math.abs(hashCode(path)) < Integer.MAX_VALUE * probability;
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
index cf25e5d8b77..23f1b12c211 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
@@ -52,11 +52,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
     public AzureBlobContainer(String repositoryName, BlobPath path, AzureBlobStore blobStore) {
         super(path);
         this.blobStore = blobStore;
-        String keyPath = path.buildAsString("/");
-        if (!keyPath.isEmpty()) {
-            keyPath = keyPath + "/";
-        }
-        this.keyPath = keyPath;
+        this.keyPath = path.buildAsString();
         this.repositoryName = repositoryName;
     }
 
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
index 85baf00b909..2809b8588f1 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
@@ -85,11 +85,7 @@ public class AzureBlobStore extends AbstractComponent implements BlobStore {
 
     @Override
     public void delete(BlobPath path) {
-        String keyPath = path.buildAsString("/");
-        if (!keyPath.isEmpty()) {
-            keyPath = keyPath + "/";
-        }
-
+        String keyPath = path.buildAsString();
         try {
             this.client.deleteFiles(this.accountName, this.locMode, container, keyPath);
         } catch (URISyntaxException | StorageException e) {
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobContainer.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobContainer.java
index d8117180ce3..0387187eef9 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobContainer.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobContainer.java
@@ -42,13 +42,7 @@ public class GoogleCloudStorageBlobContainer extends AbstractBlobContainer {
     GoogleCloudStorageBlobContainer(BlobPath path, GoogleCloudStorageBlobStore blobStore) {
         super(path);
         this.blobStore = blobStore;
-
-        String keyPath = path.buildAsString("/");
-        // TODO Move this keyPath logic to the buildAsString() method
-        if (!keyPath.isEmpty()) {
-            keyPath = keyPath + "/";
-        }
-        this.path = keyPath;
+        this.path = path.buildAsString();
     }
 
     @Override
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStore.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStore.java
index 7bf79494440..6ff5aa41819 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStore.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/common/blobstore/gcs/GoogleCloudStorageBlobStore.java
@@ -88,12 +88,7 @@ public class GoogleCloudStorageBlobStore extends AbstractComponent implements Bl
 
     @Override
     public void delete(BlobPath path) throws IOException {
-        String keyPath = path.buildAsString("/");
-        // TODO Move this keyPath logic to the buildAsString() method
-        if (!keyPath.isEmpty()) {
-            keyPath = keyPath + "/";
-        }
-        deleteBlobsByPrefix(keyPath);
+        deleteBlobsByPrefix(path.buildAsString());
     }
 
     @Override
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java
index 4861ccc202b..42df840ce40 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java
@@ -54,11 +54,7 @@ public class S3BlobContainer extends AbstractBlobContainer {
     public S3BlobContainer(BlobPath path, S3BlobStore blobStore) {
         super(path);
         this.blobStore = blobStore;
-        String keyPath = path.buildAsString("/");
-        if (!keyPath.isEmpty()) {
-            keyPath = keyPath + "/";
-        }
-        this.keyPath = keyPath;
+        this.keyPath = path.buildAsString();
     }
 
     @Override
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java
index 650d71f62ad..991e5f9707f 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStore.java
@@ -145,11 +145,7 @@ public class S3BlobStore extends AbstractComponent implements BlobStore {
             if (prevListing != null) {
                 list = client.listNextBatchOfObjects(prevListing);
             } else {
-                String keyPath = path.buildAsString("/");
-                if (!keyPath.isEmpty()) {
-                    keyPath = keyPath + "/";
-                }
-                list = client.listObjects(bucket, keyPath);
+                list = client.listObjects(bucket, path.buildAsString());
                 multiObjectDeleteRequest = new DeleteObjectsRequest(list.getBucketName());
             }
             for (S3ObjectSummary summary : list.getObjectSummaries()) {

From 459916f5ddb597204c428c35d239d45812a78205 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Tue, 17 May 2016 17:16:23 +0200
Subject: [PATCH 0419/1311] Remove custom Base64 implementation. #18413

This replaces o.e.common.Base64 with java.util.Base64.
---
 .../action/search/ParsedScrollId.java         |   11 +-
 .../SearchDfsQueryAndFetchAsyncAction.java    |    2 +-
 .../SearchDfsQueryThenFetchAsyncAction.java   |    2 +-
 .../SearchQueryAndFetchAsyncAction.java       |    2 +-
 .../SearchQueryThenFetchAsyncAction.java      |    2 +-
 .../action/search/TransportSearchHelper.java  |   97 +-
 .../java/org/elasticsearch/common/Base64.java | 1621 -----------------
 .../common/RandomBasedUUIDGenerator.java      |   11 +-
 .../common/TimeBasedUUIDGenerator.java        |   14 +-
 .../index/engine/CommitStats.java             |    6 +-
 .../elasticsearch/index/engine/Engine.java    |    9 +-
 .../index/mapper/core/BinaryFieldMapper.java  |    9 +-
 .../ingest/core/IngestDocument.java           |   12 +-
 .../org/elasticsearch/common/Base64Tests.java |   57 -
 .../index/engine/InternalEngineTests.java     |    4 +-
 .../search/scroll/SearchScrollIT.java         |   34 +-
 .../messy/tests/SearchFieldsTests.java        |    5 +-
 .../TransportDeleteByQueryActionTests.java    |    2 +-
 .../attachment/AttachmentProcessorTests.java  |    4 +-
 .../MultifieldAttachmentMapperTests.java      |    6 +-
 20 files changed, 73 insertions(+), 1837 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/common/Base64.java
 delete mode 100644 core/src/test/java/org/elasticsearch/common/Base64Tests.java

diff --git a/core/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java b/core/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java
index ee0ba95b2b1..2ddb35e1357 100644
--- a/core/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java
+++ b/core/src/main/java/org/elasticsearch/action/search/ParsedScrollId.java
@@ -19,8 +19,6 @@
 
 package org.elasticsearch.action.search;
 
-import java.util.Map;
-
 /**
  *
  */
@@ -36,13 +34,10 @@ class ParsedScrollId {
 
     private final ScrollIdForNode[] context;
 
-    private final Map<String, String> attributes;
-
-    public ParsedScrollId(String source, String type, ScrollIdForNode[] context, Map<String, String> attributes) {
+    public ParsedScrollId(String source, String type, ScrollIdForNode[] context) {
         this.source = source;
         this.type = type;
         this.context = context;
-        this.attributes = attributes;
     }
 
     public String getSource() {
@@ -56,8 +51,4 @@ class ParsedScrollId {
     public ScrollIdForNode[] getContext() {
         return context;
     }
-
-    public Map<String, String> getAttributes() {
-        return this.attributes;
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryAndFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryAndFetchAsyncAction.java
index f7cb72b22e9..7cf608559ce 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryAndFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryAndFetchAsyncAction.java
@@ -123,7 +123,7 @@ class SearchDfsQueryAndFetchAsyncAction extends AbstractSearchAsyncAction<DfsSea
                     queryFetchResults);
                 String scrollId = null;
                 if (request.scroll() != null) {
-                    scrollId = TransportSearchHelper.buildScrollId(request.searchType(), firstResults, null);
+                    scrollId = TransportSearchHelper.buildScrollId(request.searchType(), firstResults);
                 }
                 listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps, successfulOps.get(),
                     buildTookInMillis(), buildShardFailures()));
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
index c5f320f1b33..954630b9295 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
@@ -200,7 +200,7 @@ class SearchDfsQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<DfsSe
                     fetchResults);
                 String scrollId = null;
                 if (request.scroll() != null) {
-                    scrollId = TransportSearchHelper.buildScrollId(request.searchType(), firstResults, null);
+                    scrollId = TransportSearchHelper.buildScrollId(request.searchType(), firstResults);
                 }
                 listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps, successfulOps.get(),
                     buildTookInMillis(), buildShardFailures()));
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchQueryAndFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchQueryAndFetchAsyncAction.java
index 1b338847762..ee955f2c894 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchQueryAndFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchQueryAndFetchAsyncAction.java
@@ -66,7 +66,7 @@ class SearchQueryAndFetchAsyncAction extends AbstractSearchAsyncAction<QueryFetc
                     firstResults);
                 String scrollId = null;
                 if (request.scroll() != null) {
-                    scrollId = TransportSearchHelper.buildScrollId(request.searchType(), firstResults, null);
+                    scrollId = TransportSearchHelper.buildScrollId(request.searchType(), firstResults);
                 }
                 listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps, successfulOps.get(),
                     buildTookInMillis(), buildShardFailures()));
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
index 3feb40411f9..59a6f790e2c 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
@@ -133,7 +133,7 @@ class SearchQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<QuerySea
                     fetchResults);
                 String scrollId = null;
                 if (request.scroll() != null) {
-                    scrollId = TransportSearchHelper.buildScrollId(request.searchType(), firstResults, null);
+                    scrollId = TransportSearchHelper.buildScrollId(request.searchType(), firstResults);
                 }
                 listener.onResponse(new SearchResponse(internalResponse, scrollId, expectedSuccessfulOps,
                     successfulOps.get(), buildTookInMillis(), buildShardFailures()));
diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java b/core/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
index 8da195b739b..db42527b125 100644
--- a/core/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
+++ b/core/src/main/java/org/elasticsearch/action/search/TransportSearchHelper.java
@@ -19,21 +19,16 @@
 
 package org.elasticsearch.action.search;
 
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.CharsRefBuilder;
+import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.store.RAMOutputStream;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.common.Base64;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.internal.InternalScrollSearchRequest;
 import org.elasticsearch.search.internal.ShardSearchTransportRequest;
 
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static java.util.Collections.emptyMap;
+import java.util.Base64;
 
 /**
  *
@@ -49,79 +44,49 @@ final class TransportSearchHelper {
         return new InternalScrollSearchRequest(request, id);
     }
 
-    static String buildScrollId(SearchType searchType, AtomicArray<? extends SearchPhaseResult> searchPhaseResults,
-                                @Nullable Map<String, String> attributes) throws IOException {
+    static String buildScrollId(SearchType searchType, AtomicArray<? extends SearchPhaseResult> searchPhaseResults) throws IOException {
         if (searchType == SearchType.DFS_QUERY_THEN_FETCH || searchType == SearchType.QUERY_THEN_FETCH) {
-            return buildScrollId(ParsedScrollId.QUERY_THEN_FETCH_TYPE, searchPhaseResults, attributes);
+            return buildScrollId(ParsedScrollId.QUERY_THEN_FETCH_TYPE, searchPhaseResults);
         } else if (searchType == SearchType.QUERY_AND_FETCH || searchType == SearchType.DFS_QUERY_AND_FETCH) {
-            return buildScrollId(ParsedScrollId.QUERY_AND_FETCH_TYPE, searchPhaseResults, attributes);
+            return buildScrollId(ParsedScrollId.QUERY_AND_FETCH_TYPE, searchPhaseResults);
         } else {
             throw new IllegalStateException("search_type [" + searchType + "] not supported");
         }
     }
 
-    static String buildScrollId(String type, AtomicArray<? extends SearchPhaseResult> searchPhaseResults,
-                                @Nullable Map<String, String> attributes) throws IOException {
-        StringBuilder sb = new StringBuilder().append(type).append(';');
-        sb.append(searchPhaseResults.asList().size()).append(';');
-        for (AtomicArray.Entry<? extends SearchPhaseResult> entry : searchPhaseResults.asList()) {
-            SearchPhaseResult searchPhaseResult = entry.value;
-            sb.append(searchPhaseResult.id()).append(':').append(searchPhaseResult.shardTarget().nodeId()).append(';');
-        }
-        if (attributes == null) {
-            sb.append("0;");
-        } else {
-            sb.append(attributes.size()).append(";");
-            for (Map.Entry<String, String> entry : attributes.entrySet()) {
-                sb.append(entry.getKey()).append(':').append(entry.getValue()).append(';');
+    static String buildScrollId(String type, AtomicArray<? extends SearchPhaseResult> searchPhaseResults) throws IOException {
+        try (RAMOutputStream out = new RAMOutputStream()) {
+            out.writeString(type);
+            out.writeVInt(searchPhaseResults.asList().size());
+            for (AtomicArray.Entry<? extends SearchPhaseResult> entry : searchPhaseResults.asList()) {
+                SearchPhaseResult searchPhaseResult = entry.value;
+                out.writeLong(searchPhaseResult.id());
+                out.writeString(searchPhaseResult.shardTarget().nodeId());
             }
+            byte[] bytes = new byte[(int) out.getFilePointer()];
+            out.writeTo(bytes, 0);
+            return Base64.getUrlEncoder().encodeToString(bytes);
         }
-        BytesRef bytesRef = new BytesRef(sb);
-        return Base64.encodeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length, Base64.URL_SAFE);
     }
 
     static ParsedScrollId parseScrollId(String scrollId) {
-        CharsRefBuilder spare = new CharsRefBuilder();
         try {
-            byte[] decode = Base64.decode(scrollId, Base64.URL_SAFE);
-            spare.copyUTF8Bytes(decode, 0, decode.length);
+            byte[] bytes = Base64.getUrlDecoder().decode(scrollId);
+            ByteArrayDataInput in = new ByteArrayDataInput(bytes);
+            String type = in.readString();
+            ScrollIdForNode[] context = new ScrollIdForNode[in.readVInt()];
+            for (int i = 0; i < context.length; ++i) {
+                long id = in.readLong();
+                String target = in.readString();
+                context[i] = new ScrollIdForNode(target, id);
+            }
+            if (in.getPosition() != bytes.length) {
+                throw new IllegalArgumentException("Not all bytes were read");
+            }
+            return new ParsedScrollId(scrollId, type, context);
         } catch (Exception e) {
-            throw new IllegalArgumentException("Failed to decode scrollId", e);
+            throw new IllegalArgumentException("Cannot parse scroll id", e);
         }
-        String[] elements = spare.get().toString().split(";");
-        if (elements.length < 2) {
-            throw new IllegalArgumentException("Malformed scrollId [" + scrollId + "]");
-        }
-
-        int index = 0;
-        String type = elements[index++];
-        int contextSize = Integer.parseInt(elements[index++]);
-        if (elements.length < contextSize + 2) {
-            throw new IllegalArgumentException("Malformed scrollId [" + scrollId + "]");
-        }
-
-        ScrollIdForNode[] context = new ScrollIdForNode[contextSize];
-        for (int i = 0; i < contextSize; i++) {
-            String element = elements[index++];
-            int sep = element.indexOf(':');
-            if (sep == -1) {
-                throw new IllegalArgumentException("Malformed scrollId [" + scrollId + "]");
-            }
-            context[i] = new ScrollIdForNode(element.substring(sep + 1), Long.parseLong(element.substring(0, sep)));
-        }
-        Map<String, String> attributes;
-        int attributesSize = Integer.parseInt(elements[index++]);
-        if (attributesSize == 0) {
-            attributes = emptyMap();
-        } else {
-            attributes = new HashMap<>(attributesSize);
-            for (int i = 0; i < attributesSize; i++) {
-                String element = elements[index++];
-                int sep = element.indexOf(':');
-                attributes.put(element.substring(0, sep), element.substring(sep + 1));
-            }
-        }
-        return new ParsedScrollId(scrollId, type, context, attributes);
     }
 
     private TransportSearchHelper() {
diff --git a/core/src/main/java/org/elasticsearch/common/Base64.java b/core/src/main/java/org/elasticsearch/common/Base64.java
deleted file mode 100644
index fa499a55d4d..00000000000
--- a/core/src/main/java/org/elasticsearch/common/Base64.java
+++ /dev/null
@@ -1,1621 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.common;
-
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.Locale;
-import java.util.Objects;
-
-/**
- * <p>Encodes and decodes to and from Base64 notation.</p>
- * <p>Homepage: <a href="http://iharder.net/base64">http://iharder.net/base64</a>.</p>
- * <p>
- * Example:
- * <p>
- * <code>String encoded = Base64.encode( myByteArray );</code>
- * <br>
- * <code>byte[] myByteArray = Base64.decode( encoded );</code>
- * <p>
- * The <tt>options</tt> parameter, which appears in a few places, is used to pass
- * several pieces of information to the encoder. In the "higher level" methods such as
- * encodeBytes( bytes, options ) the options parameter can be used to indicate such
- * things as first gzipping the bytes before encoding them, not inserting linefeeds,
- * and encoding using the URL-safe and Ordered dialects.
- * <p>
- * Note, according to <a href="http://www.faqs.org/rfcs/rfc3548.html">RFC3548</a>,
- * Section 2.1, implementations should not add line feeds unless explicitly told
- * to do so. I've got Base64 set to this behavior now, although earlier versions
- * broke lines by default.
- * <p>
- * The constants defined in Base64 can be OR-ed together to combine options, so you
- * might make a call like this:
- * <p>
- * <code>String encoded = Base64.encodeBytes( mybytes, Base64.GZIP | Base64.DO_BREAK_LINES );</code>
- * <p>to compress the data before encoding it and then making the output have newline characters.
- * <p>Also...
- * <code>String encoded = Base64.encodeBytes( crazyString.getBytes() );</code>
- * <p>
- * Change Log:
- * <ul>
- * <li>v2.3.7 - Fixed subtle bug when base 64 input stream contained the
- * value 01111111, which is an invalid base 64 character but should not
- * throw an ArrayIndexOutOfBoundsException either. Led to discovery of
- * mishandling (or potential for better handling) of other bad input
- * characters. You should now get an IOException if you try decoding
- * something that has bad characters in it.</li>
- * <li>v2.3.6 - Fixed bug when breaking lines and the final byte of the encoded
- * string ended in the last column; the buffer was not properly shrunk and
- * contained an extra (null) byte that made it into the string.</li>
- * <li>v2.3.5 - Fixed bug in {@code #encodeFromFile} where estimated buffer size
- * was wrong for files of size 31, 34, and 37 bytes.</li>
- * <li>v2.3.4 - Fixed bug when working with gzipped streams whereby flushing
- * the Base64.OutputStream closed the Base64 encoding (by padding with equals
- * signs) too soon. Also added an option to suppress the automatic decoding
- * of gzipped streams. Also added experimental support for specifying a
- * class loader when using the
- * {@code #decodeToObject(java.lang.String, int, java.lang.ClassLoader)}
- * method.</li>
- * <li>v2.3.3 - Changed default char encoding to US-ASCII which reduces the internal Java
- * footprint with its CharEncoders and so forth. Fixed some javadocs that were
- * inconsistent. Removed imports and specified things like java.io.IOException
- * explicitly inline.</li>
- * <li>v2.3.2 - Reduced memory footprint! Finally refined the "guessing" of how big the
- * final encoded data will be so that the code doesn't have to create two output
- * arrays: an oversized initial one and then a final, exact-sized one. Big win
- * when using the {@link #encodeBytesToBytes(byte[])} family of methods (and not
- * using the gzip options which uses a different mechanism with streams and stuff).</li>
- * <li>v2.3.1 - Added {@link #encodeBytesToBytes(byte[], int, int, int)} and some
- * similar helper methods to be more efficient with memory by not returning a
- * String but just a byte array.</li>
- * <li>v2.3 - <strong>This is not a drop-in replacement!</strong> This is two years of comments
- * and bug fixes queued up and finally executed. Thanks to everyone who sent
- * me stuff, and I'm sorry I wasn't able to distribute your fixes to everyone else.
- * Much bad coding was cleaned up including throwing exceptions where necessary
- * instead of returning null values or something similar. Here are some changes
- * that may affect you:
- * <ul>
- * <li><em>Does not break lines, by default.</em> This is to keep in compliance with
- * <a href="http://www.faqs.org/rfcs/rfc3548.html">RFC3548</a>.</li>
- * <li><em>Throws exceptions instead of returning null values.</em> Because some operations
- * (especially those that may permit the GZIP option) use IO streams, there
- * is a possibility of an java.io.IOException being thrown. After some discussion and
- * thought, I've changed the behavior of the methods to throw java.io.IOExceptions
- * rather than return null if ever there's an error. I think this is more
- * appropriate, though it will require some changes to your code. Sorry,
- * it should have been done this way to begin with.</li>
- * <li><em>Removed all references to System.out, System.err, and the like.</em>
- * Shame on me. All I can say is sorry they were ever there.</li>
- * <li><em>Throws NullPointerExceptions and IllegalArgumentExceptions</em> as needed
- * such as when passed arrays are null or offsets are invalid.</li>
- * <li>Cleaned up as much javadoc as I could to avoid any javadoc warnings.
- * This was especially annoying before for people who were thorough in their
- * own projects and then had gobs of javadoc warnings on this file.</li>
- * </ul>
- * <li>v2.2.1 - Fixed bug using URL_SAFE and ORDERED encodings. Fixed bug
- * when using very small files (~&lt; 40 bytes).</li>
- * <li>v2.2 - Added some helper methods for encoding/decoding directly from
- * one file to the next. Also added a main() method to support command line
- * encoding/decoding from one file to the next. Also added these Base64 dialects:
- * <ol>
- * <li>The default is RFC3548 format.</li>
- * <li>Calling Base64.setFormat(Base64.BASE64_FORMAT.URLSAFE_FORMAT) generates
- * URL and file name friendly format as described in Section 4 of RFC3548.
- * http://www.faqs.org/rfcs/rfc3548.html</li>
- * <li>Calling Base64.setFormat(Base64.BASE64_FORMAT.ORDERED_FORMAT) generates
- * URL and file name friendly format that preserves lexical ordering as described
- * in http://www.faqs.org/qa/rfcc-1940.html</li>
- * </ol>
- * Special thanks to Jim Kellerman at <a href="http://www.powerset.com/">http://www.powerset.com/</a>
- * for contributing the new Base64 dialects.
- * </li>
- * <li>v2.1 - Cleaned up javadoc comments and unused variables and methods. Added
- * some convenience methods for reading and writing to and from files.</li>
- * <li>v2.0.2 - Now specifies UTF-8 encoding in places where the code fails on systems
- * with other encodings (like EBCDIC).</li>
- * <li>v2.0.1 - Fixed an error when decoding a single byte, that is, when the
- * encoded data was a single byte.</li>
- * <li>v2.0 - I got rid of methods that used booleans to set options.
- * Now everything is more consolidated and cleaner. The code now detects
- * when data that's being decoded is gzip-compressed and will decompress it
- * automatically. Generally things are cleaner. You'll probably have to
- * change some method calls that you were making to support the new
- * options format (<tt>int</tt>s that you "OR" together).</li>
- * <li>v1.5.1 - Fixed bug when decompressing and decoding to a
- * byte[] using <tt>decode( String s, boolean gzipCompressed )</tt>.
- * Added the ability to "suspend" encoding in the Output Stream so
- * you can turn on and off the encoding if you need to embed base64
- * data in an otherwise "normal" stream (like an XML file).</li>
- * <li>v1.5 - Output stream pases on flush() command but doesn't do anything itself.
- * This helps when using GZIP streams.
- * Added the ability to GZip-compress objects before encoding them.</li>
- * <li>v1.4 - Added helper methods to read/write files.</li>
- * <li>v1.3.6 - Fixed OutputStream.flush() so that 'position' is reset.</li>
- * <li>v1.3.5 - Added flag to turn on and off line breaks. Fixed bug in input stream
- * where last buffer being read, if not completely full, was not returned.</li>
- * <li>v1.3.4 - Fixed when "improperly padded stream" error was thrown at the wrong time.</li>
- * <li>v1.3.3 - Fixed I/O streams which were totally messed up.</li>
- * </ul>
- * <p>
- * I am placing this code in the Public Domain. Do with it as you will.
- * This software comes with no guarantees or warranties but with
- * plenty of well-wishing instead!
- * Please visit <a href="http://iharder.net/base64">http://iharder.net/base64</a>
- * periodically to check for updates or to contribute improvements.
- *
- * @author Robert Harder
- * @author rob@iharder.net
- * @version 2.3.7
- */
-public final class Base64 {
-
-/* ********  P U B L I C   F I E L D S  ******** */
-
-
-    /**
-     * No options specified. Value is zero.
-     */
-    public final static int NO_OPTIONS = 0;
-
-    /**
-     * Specify encoding in first bit. Value is one.
-     */
-    public final static int ENCODE = 1;
-
-
-    /**
-     * Specify decoding in first bit. Value is zero.
-     */
-    public final static int DECODE = 0;
-
-
-    /**
-     * Specify that data should be gzip-compressed in second bit. Value is two.
-     */
-    public final static int GZIP = 2;
-
-    /**
-     * Specify that gzipped data should <em>not</em> be automatically gunzipped.
-     */
-    public final static int DONT_GUNZIP = 4;
-
-
-    /**
-     * Do break lines when encoding. Value is 8.
-     */
-    public final static int DO_BREAK_LINES = 8;
-
-    /**
-     * Encode using Base64-like encoding that is URL- and Filename-safe as described
-     * in Section 4 of RFC3548:
-     * <a href="http://www.faqs.org/rfcs/rfc3548.html">http://www.faqs.org/rfcs/rfc3548.html</a>.
-     * It is important to note that data encoded this way is <em>not</em> officially valid Base64,
-     * or at the very least should not be called Base64 without also specifying that is
-     * was encoded using the URL- and Filename-safe dialect.
-     */
-    public final static int URL_SAFE = 16;
-
-
-    /**
-     * Encode using the special "ordered" dialect of Base64 described here:
-     * <a href="http://www.faqs.org/qa/rfcc-1940.html">http://www.faqs.org/qa/rfcc-1940.html</a>.
-     */
-    public final static int ORDERED = 32;
-
-
-/* ********  P R I V A T E   F I E L D S  ******** */
-
-
-    /**
-     * Maximum line length (76) of Base64 output.
-     */
-    private final static int MAX_LINE_LENGTH = 76;
-
-
-    /**
-     * The equals sign (=) as a byte.
-     */
-    private final static byte EQUALS_SIGN = (byte) '=';
-
-
-    /**
-     * The new line character (\n) as a byte.
-     */
-    private final static byte NEW_LINE = (byte) '\n';
-
-
-    /**
-     * Preferred encoding.
-     */
-    public final static Charset PREFERRED_ENCODING = Charset.forName("US-ASCII");
-
-
-    private final static byte WHITE_SPACE_ENC = -5; // Indicates white space in encoding
-    private final static byte EQUALS_SIGN_ENC = -1; // Indicates equals sign in encoding
-
-
-/* ********  S T A N D A R D   B A S E 6 4   A L P H A B E T  ******** */
-
-    /**
-     * The 64 valid Base64 values.
-     */
-    /* Host platform me be something funny like EBCDIC, so we hardcode these values. */
-    private final static byte[] _STANDARD_ALPHABET = {
-            (byte) 'A', (byte) 'B', (byte) 'C', (byte) 'D', (byte) 'E', (byte) 'F', (byte) 'G',
-            (byte) 'H', (byte) 'I', (byte) 'J', (byte) 'K', (byte) 'L', (byte) 'M', (byte) 'N',
-            (byte) 'O', (byte) 'P', (byte) 'Q', (byte) 'R', (byte) 'S', (byte) 'T', (byte) 'U',
-            (byte) 'V', (byte) 'W', (byte) 'X', (byte) 'Y', (byte) 'Z',
-            (byte) 'a', (byte) 'b', (byte) 'c', (byte) 'd', (byte) 'e', (byte) 'f', (byte) 'g',
-            (byte) 'h', (byte) 'i', (byte) 'j', (byte) 'k', (byte) 'l', (byte) 'm', (byte) 'n',
-            (byte) 'o', (byte) 'p', (byte) 'q', (byte) 'r', (byte) 's', (byte) 't', (byte) 'u',
-            (byte) 'v', (byte) 'w', (byte) 'x', (byte) 'y', (byte) 'z',
-            (byte) '0', (byte) '1', (byte) '2', (byte) '3', (byte) '4', (byte) '5',
-            (byte) '6', (byte) '7', (byte) '8', (byte) '9', (byte) '+', (byte) '/'
-    };
-
-
-    /**
-     * Translates a Base64 value to either its 6-bit reconstruction value
-     * or a negative number indicating some other meaning.
-     */
-    private final static byte[] _STANDARD_DECODABET = {
-            -9, -9, -9, -9, -9, -9, -9, -9, -9,                 // Decimal  0 -  8
-            -5, -5,                                      // Whitespace: Tab and Linefeed
-            -9, -9,                                      // Decimal 11 - 12
-            -5,                                         // Whitespace: Carriage Return
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 14 - 26
-            -9, -9, -9, -9, -9,                             // Decimal 27 - 31
-            -5,                                         // Whitespace: Space
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,              // Decimal 33 - 42
-            62,                                         // Plus sign at decimal 43
-            -9, -9, -9,                                   // Decimal 44 - 46
-            63,                                         // Slash at decimal 47
-            52, 53, 54, 55, 56, 57, 58, 59, 60, 61,              // Numbers zero through nine
-            -9, -9, -9,                                   // Decimal 58 - 60
-            -1,                                         // Equals sign at decimal 61
-            -9, -9, -9,                                      // Decimal 62 - 64
-            0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,            // Letters 'A' through 'N'
-            14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,        // Letters 'O' through 'Z'
-            -9, -9, -9, -9, -9, -9,                          // Decimal 91 - 96
-            26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38,     // Letters 'a' through 'm'
-            39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51,     // Letters 'n' through 'z'
-            -9, -9, -9, -9, -9                              // Decimal 123 - 127
-            , -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,       // Decimal 128 - 139
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 140 - 152
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 153 - 165
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 166 - 178
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 179 - 191
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 192 - 204
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 205 - 217
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 218 - 230
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 231 - 243
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9         // Decimal 244 - 255
-    };
-
-
-/* ********  U R L   S A F E   B A S E 6 4   A L P H A B E T  ******** */
-
-    /**
-     * Used in the URL- and Filename-safe dialect described in Section 4 of RFC3548:
-     * <a href="http://www.faqs.org/rfcs/rfc3548.html">http://www.faqs.org/rfcs/rfc3548.html</a>.
-     * Notice that the last two bytes become "hyphen" and "underscore" instead of "plus" and "slash."
-     */
-    private final static byte[] _URL_SAFE_ALPHABET = {
-            (byte) 'A', (byte) 'B', (byte) 'C', (byte) 'D', (byte) 'E', (byte) 'F', (byte) 'G',
-            (byte) 'H', (byte) 'I', (byte) 'J', (byte) 'K', (byte) 'L', (byte) 'M', (byte) 'N',
-            (byte) 'O', (byte) 'P', (byte) 'Q', (byte) 'R', (byte) 'S', (byte) 'T', (byte) 'U',
-            (byte) 'V', (byte) 'W', (byte) 'X', (byte) 'Y', (byte) 'Z',
-            (byte) 'a', (byte) 'b', (byte) 'c', (byte) 'd', (byte) 'e', (byte) 'f', (byte) 'g',
-            (byte) 'h', (byte) 'i', (byte) 'j', (byte) 'k', (byte) 'l', (byte) 'm', (byte) 'n',
-            (byte) 'o', (byte) 'p', (byte) 'q', (byte) 'r', (byte) 's', (byte) 't', (byte) 'u',
-            (byte) 'v', (byte) 'w', (byte) 'x', (byte) 'y', (byte) 'z',
-            (byte) '0', (byte) '1', (byte) '2', (byte) '3', (byte) '4', (byte) '5',
-            (byte) '6', (byte) '7', (byte) '8', (byte) '9', (byte) '-', (byte) '_'
-    };
-
-    /**
-     * Used in decoding URL- and Filename-safe dialects of Base64.
-     */
-    private final static byte[] _URL_SAFE_DECODABET = {
-            -9, -9, -9, -9, -9, -9, -9, -9, -9,                 // Decimal  0 -  8
-            -5, -5,                                      // Whitespace: Tab and Linefeed
-            -9, -9,                                      // Decimal 11 - 12
-            -5,                                         // Whitespace: Carriage Return
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 14 - 26
-            -9, -9, -9, -9, -9,                             // Decimal 27 - 31
-            -5,                                         // Whitespace: Space
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,              // Decimal 33 - 42
-            -9,                                         // Plus sign at decimal 43
-            -9,                                         // Decimal 44
-            62,                                         // Minus sign at decimal 45
-            -9,                                         // Decimal 46
-            -9,                                         // Slash at decimal 47
-            52, 53, 54, 55, 56, 57, 58, 59, 60, 61,              // Numbers zero through nine
-            -9, -9, -9,                                   // Decimal 58 - 60
-            -1,                                         // Equals sign at decimal 61
-            -9, -9, -9,                                   // Decimal 62 - 64
-            0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13,            // Letters 'A' through 'N'
-            14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,        // Letters 'O' through 'Z'
-            -9, -9, -9, -9,                                // Decimal 91 - 94
-            63,                                         // Underscore at decimal 95
-            -9,                                         // Decimal 96
-            26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38,     // Letters 'a' through 'm'
-            39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51,     // Letters 'n' through 'z'
-            -9, -9, -9, -9, -9                              // Decimal 123 - 127
-            , -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 128 - 139
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 140 - 152
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 153 - 165
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 166 - 178
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 179 - 191
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 192 - 204
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 205 - 217
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 218 - 230
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 231 - 243
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9         // Decimal 244 - 255
-    };
-
-
-/* ********  O R D E R E D   B A S E 6 4   A L P H A B E T  ******** */
-
-    /**
-     * I don't get the point of this technique, but someone requested it,
-     * and it is described here:
-     * <a href="http://www.faqs.org/qa/rfcc-1940.html">http://www.faqs.org/qa/rfcc-1940.html</a>.
-     */
-    private final static byte[] _ORDERED_ALPHABET = {
-            (byte) '-',
-            (byte) '0', (byte) '1', (byte) '2', (byte) '3', (byte) '4',
-            (byte) '5', (byte) '6', (byte) '7', (byte) '8', (byte) '9',
-            (byte) 'A', (byte) 'B', (byte) 'C', (byte) 'D', (byte) 'E', (byte) 'F', (byte) 'G',
-            (byte) 'H', (byte) 'I', (byte) 'J', (byte) 'K', (byte) 'L', (byte) 'M', (byte) 'N',
-            (byte) 'O', (byte) 'P', (byte) 'Q', (byte) 'R', (byte) 'S', (byte) 'T', (byte) 'U',
-            (byte) 'V', (byte) 'W', (byte) 'X', (byte) 'Y', (byte) 'Z',
-            (byte) '_',
-            (byte) 'a', (byte) 'b', (byte) 'c', (byte) 'd', (byte) 'e', (byte) 'f', (byte) 'g',
-            (byte) 'h', (byte) 'i', (byte) 'j', (byte) 'k', (byte) 'l', (byte) 'm', (byte) 'n',
-            (byte) 'o', (byte) 'p', (byte) 'q', (byte) 'r', (byte) 's', (byte) 't', (byte) 'u',
-            (byte) 'v', (byte) 'w', (byte) 'x', (byte) 'y', (byte) 'z'
-    };
-
-    /**
-     * Used in decoding the "ordered" dialect of Base64.
-     */
-    private final static byte[] _ORDERED_DECODABET = {
-            -9, -9, -9, -9, -9, -9, -9, -9, -9,                 // Decimal  0 -  8
-            -5, -5,                                      // Whitespace: Tab and Linefeed
-            -9, -9,                                      // Decimal 11 - 12
-            -5,                                         // Whitespace: Carriage Return
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 14 - 26
-            -9, -9, -9, -9, -9,                             // Decimal 27 - 31
-            -5,                                         // Whitespace: Space
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,              // Decimal 33 - 42
-            -9,                                         // Plus sign at decimal 43
-            -9,                                         // Decimal 44
-            0,                                          // Minus sign at decimal 45
-            -9,                                         // Decimal 46
-            -9,                                         // Slash at decimal 47
-            1, 2, 3, 4, 5, 6, 7, 8, 9, 10,                       // Numbers zero through nine
-            -9, -9, -9,                                   // Decimal 58 - 60
-            -1,                                         // Equals sign at decimal 61
-            -9, -9, -9,                                   // Decimal 62 - 64
-            11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,     // Letters 'A' through 'M'
-            24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36,     // Letters 'N' through 'Z'
-            -9, -9, -9, -9,                                // Decimal 91 - 94
-            37,                                         // Underscore at decimal 95
-            -9,                                         // Decimal 96
-            38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50,     // Letters 'a' through 'm'
-            51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,     // Letters 'n' through 'z'
-            -9, -9, -9, -9, -9                                 // Decimal 123 - 127
-            , -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 128 - 139
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 140 - 152
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 153 - 165
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 166 - 178
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 179 - 191
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 192 - 204
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 205 - 217
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 218 - 230
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9,     // Decimal 231 - 243
-            -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9, -9         // Decimal 244 - 255
-    };
-
-
-/* ********  D E T E R M I N E   W H I C H   A L H A B E T  ******** */
-
-
-    /**
-     * Returns one of the _SOMETHING_ALPHABET byte arrays depending on
-     * the options specified.
-     * It's possible, though silly, to specify ORDERED <b>and</b> URLSAFE
-     * in which case one of them will be picked, though there is
-     * no guarantee as to which one will be picked.
-     */
-    private final static byte[] getAlphabet(int options) {
-        if ((options & URL_SAFE) == URL_SAFE) {
-            return _URL_SAFE_ALPHABET;
-        } else if ((options & ORDERED) == ORDERED) {
-            return _ORDERED_ALPHABET;
-        } else {
-            return _STANDARD_ALPHABET;
-        }
-    }    // end getAlphabet
-
-
-    /**
-     * Returns one of the _SOMETHING_DECODABET byte arrays depending on
-     * the options specified.
-     * It's possible, though silly, to specify ORDERED and URL_SAFE
-     * in which case one of them will be picked, though there is
-     * no guarantee as to which one will be picked.
-     */
-    private final static byte[] getDecodabet(int options) {
-        if ((options & URL_SAFE) == URL_SAFE) {
-            return _URL_SAFE_DECODABET;
-        } else if ((options & ORDERED) == ORDERED) {
-            return _ORDERED_DECODABET;
-        } else {
-            return _STANDARD_DECODABET;
-        }
-    }    // end getAlphabet
-
-
-    /**
-     * Defeats instantiation.
-     */
-    private Base64() {
-    }
-
-
-/* ********  E N C O D I N G   M E T H O D S  ******** */
-
-
-    /**
-     * Encodes up to the first three bytes of array <var>threeBytes</var>
-     * and returns a four-byte array in Base64 notation.
-     * The actual number of significant bytes in your array is
-     * given by <var>numSigBytes</var>.
-     * The array <var>threeBytes</var> needs only be as big as
-     * <var>numSigBytes</var>.
-     * Code can reuse a byte array by passing a four-byte array as <var>b4</var>.
-     *
-     * @param b4          A reusable byte array to reduce array instantiation
-     * @param threeBytes  the array to convert
-     * @param numSigBytes the number of significant bytes in your array
-     * @return four byte array in Base64 notation.
-     * @since 1.5.1
-     */
-    private static byte[] encode3to4(byte[] b4, byte[] threeBytes, int numSigBytes, int options) {
-        encode3to4(threeBytes, 0, numSigBytes, b4, 0, options);
-        return b4;
-    }   // end encode3to4
-
-
-    /**
-     * <p>Encodes up to three bytes of the array <var>source</var>
-     * and writes the resulting four Base64 bytes to <var>destination</var>.
-     * The source and destination arrays can be manipulated
-     * anywhere along their length by specifying
-     * <var>srcOffset</var> and <var>destOffset</var>.
-     * This method does not check to make sure your arrays
-     * are large enough to accommodate <var>srcOffset</var> + 3 for
-     * the <var>source</var> array or <var>destOffset</var> + 4 for
-     * the <var>destination</var> array.
-     * The actual number of significant bytes in your array is
-     * given by <var>numSigBytes</var>.</p>
-     * <p>This is the lowest level of the encoding methods with
-     * all possible parameters.</p>
-     *
-     * @param source      the array to convert
-     * @param srcOffset   the index where conversion begins
-     * @param numSigBytes the number of significant bytes in your array
-     * @param destination the array to hold the conversion
-     * @param destOffset  the index where output will be put
-     * @return the <var>destination</var> array
-     * @since 1.3
-     */
-    private static byte[] encode3to4(
-            byte[] source, int srcOffset, int numSigBytes,
-            byte[] destination, int destOffset, int options) {
-
-        byte[] ALPHABET = getAlphabet(options);
-
-        //           1         2         3
-        // 01234567890123456789012345678901 Bit position
-        // --------000000001111111122222222 Array position from threeBytes
-        // --------|    ||    ||    ||    | Six bit groups to index ALPHABET
-        //          >>18  >>12  >> 6  >> 0  Right shift necessary
-        //                0x3f  0x3f  0x3f  Additional AND
-
-        // Create buffer with zero-padding if there are only one or two
-        // significant bytes passed in the array.
-        // We have to shift left 24 in order to flush out the 1's that appear
-        // when Java treats a value as negative that is cast from a byte to an int.
-        int inBuff = (numSigBytes > 0 ? ((source[srcOffset] << 24) >>> 8) : 0)
-                | (numSigBytes > 1 ? ((source[srcOffset + 1] << 24) >>> 16) : 0)
-                | (numSigBytes > 2 ? ((source[srcOffset + 2] << 24) >>> 24) : 0);
-
-        switch (numSigBytes) {
-            case 3:
-                destination[destOffset] = ALPHABET[(inBuff >>> 18)];
-                destination[destOffset + 1] = ALPHABET[(inBuff >>> 12) & 0x3f];
-                destination[destOffset + 2] = ALPHABET[(inBuff >>> 6) & 0x3f];
-                destination[destOffset + 3] = ALPHABET[(inBuff) & 0x3f];
-                return destination;
-
-            case 2:
-                destination[destOffset] = ALPHABET[(inBuff >>> 18)];
-                destination[destOffset + 1] = ALPHABET[(inBuff >>> 12) & 0x3f];
-                destination[destOffset + 2] = ALPHABET[(inBuff >>> 6) & 0x3f];
-                destination[destOffset + 3] = EQUALS_SIGN;
-                return destination;
-
-            case 1:
-                destination[destOffset] = ALPHABET[(inBuff >>> 18)];
-                destination[destOffset + 1] = ALPHABET[(inBuff >>> 12) & 0x3f];
-                destination[destOffset + 2] = EQUALS_SIGN;
-                destination[destOffset + 3] = EQUALS_SIGN;
-                return destination;
-
-            default:
-                return destination;
-        }   // end switch
-    }   // end encode3to4
-
-
-    /**
-     * Performs Base64 encoding on the <code>raw</code> ByteBuffer,
-     * writing it to the <code>encoded</code> ByteBuffer.
-     * This is an experimental feature. Currently it does not
-     * pass along any options (such as {@link #DO_BREAK_LINES}
-     * or {@link #GZIP}.
-     *
-     * @param raw     input buffer
-     * @param encoded output buffer
-     * @since 2.3
-     */
-    public static void encode(java.nio.ByteBuffer raw, java.nio.ByteBuffer encoded) {
-        byte[] raw3 = new byte[3];
-        byte[] enc4 = new byte[4];
-
-        while (raw.hasRemaining()) {
-            int rem = Math.min(3, raw.remaining());
-            raw.get(raw3, 0, rem);
-            Base64.encode3to4(enc4, raw3, rem, Base64.NO_OPTIONS);
-            encoded.put(enc4);
-        }   // end input remaining
-    }
-
-
-    /**
-     * Performs Base64 encoding on the <code>raw</code> ByteBuffer,
-     * writing it to the <code>encoded</code> CharBuffer.
-     * This is an experimental feature. Currently it does not
-     * pass along any options (such as {@link #DO_BREAK_LINES}
-     * or {@link #GZIP}.
-     *
-     * @param raw     input buffer
-     * @param encoded output buffer
-     * @since 2.3
-     */
-    public static void encode(java.nio.ByteBuffer raw, java.nio.CharBuffer encoded) {
-        byte[] raw3 = new byte[3];
-        byte[] enc4 = new byte[4];
-
-        while (raw.hasRemaining()) {
-            int rem = Math.min(3, raw.remaining());
-            raw.get(raw3, 0, rem);
-            Base64.encode3to4(enc4, raw3, rem, Base64.NO_OPTIONS);
-            for (int i = 0; i < 4; i++) {
-                encoded.put((char) (enc4[i] & 0xFF));
-            }
-        }   // end input remaining
-    }
-
-
-
-    /**
-     * Encodes a byte array into Base64 notation.
-     * Does not GZip-compress data.
-     *
-     * @param source The data to convert
-     * @return The data in Base64-encoded form
-     * @throws NullPointerException if source array is null
-     * @since 1.4
-     */
-    public static String encodeBytes(byte[] source) {
-        // Since we're not going to have the GZIP encoding turned on,
-        // we're not going to have an java.io.IOException thrown, so
-        // we should not force the user to have to catch it.
-        String encoded = null;
-        try {
-            encoded = encodeBytes(source, 0, source.length, NO_OPTIONS);
-        } catch (java.io.IOException ex) {
-            // not sure why this was an assertion before, running with assertions disabled would mean swallowing this exception
-            throw new IllegalStateException(ex);
-        }   // end catch
-        assert encoded != null;
-        return encoded;
-    }   // end encodeBytes
-
-
-    /**
-     * Encodes a byte array into Base64 notation.
-     * <p>
-     * Example options:<pre>
-     *   GZIP: gzip-compresses object before encoding it.
-     *   DO_BREAK_LINES: break lines at 76 characters
-     *     <i>Note: Technically, this makes your encoding non-compliant.</i>
-     * </pre>
-     * <p>
-     * Example: <code>encodeBytes( myData, Base64.GZIP )</code> or
-     * <p>
-     * Example: <code>encodeBytes( myData, Base64.GZIP | Base64.DO_BREAK_LINES )</code>
-     * <p>As of v 2.3, if there is an error with the GZIP stream,
-     * the method will throw an java.io.IOException. <b>This is new to v2.3!</b>
-     * In earlier versions, it just returned a null value, but
-     * in retrospect that's a pretty poor way to handle it.</p>
-     *
-     * @param source  The data to convert
-     * @param options Specified options
-     * @return The Base64-encoded data as a String
-     * @throws java.io.IOException  if there is an error
-     * @throws NullPointerException if source array is null
-     * @see Base64#GZIP
-     * @see Base64#DO_BREAK_LINES
-     * @since 2.0
-     */
-    public static String encodeBytes(byte[] source, int options) throws java.io.IOException {
-        return encodeBytes(source, 0, source.length, options);
-    }   // end encodeBytes
-
-    /**
-     * Encodes a byte array into Base64 notation.
-     * Does not GZip-compress data.
-     * <p>
-     * As of v 2.3, if there is an error,
-     * the method will throw an java.io.IOException. <b>This is new to v2.3!</b>
-     * In earlier versions, it just returned a null value, but
-     * in retrospect that's a pretty poor way to handle it.</p>
-     *
-     * @param source The data to convert
-     * @param off    Offset in array where conversion should begin
-     * @param len    Length of data to convert
-     * @return The Base64-encoded data as a String
-     * @throws NullPointerException     if source array is null
-     * @throws IllegalArgumentException if source array, offset, or length are invalid
-     * @since 1.4
-     */
-    public static String encodeBytes(byte[] source, int off, int len) {
-        // Since we're not going to have the GZIP encoding turned on,
-        // we're not going to have an java.io.IOException thrown, so
-        // we should not force the user to have to catch it.
-        String encoded = null;
-        try {
-            encoded = encodeBytes(source, off, len, NO_OPTIONS);
-        } catch (java.io.IOException ex) {
-            throw new IllegalStateException(ex);
-        }   // end catch
-        assert encoded != null;
-        return encoded;
-    }   // end encodeBytes
-
-
-    /**
-     * Encodes a byte array into Base64 notation.
-     * <p>
-     * Example options:<pre>
-     *   GZIP: gzip-compresses object before encoding it.
-     *   DO_BREAK_LINES: break lines at 76 characters
-     *     <i>Note: Technically, this makes your encoding non-compliant.</i>
-     * </pre>
-     * <p>
-     * Example: <code>encodeBytes( myData, Base64.GZIP )</code> or
-     * <p>
-     * Example: <code>encodeBytes( myData, Base64.GZIP | Base64.DO_BREAK_LINES )</code>
-     * <p>
-     * As of v 2.3, if there is an error with the GZIP stream,
-     * the method will throw an java.io.IOException. <b>This is new to v2.3!</b>
-     * In earlier versions, it just returned a null value, but
-     * in retrospect that's a pretty poor way to handle it.
-     *
-     * @param source  The data to convert
-     * @param off     Offset in array where conversion should begin
-     * @param len     Length of data to convert
-     * @param options Specified options
-     * @return The Base64-encoded data as a String
-     * @throws java.io.IOException      if there is an error
-     * @throws NullPointerException     if source array is null
-     * @throws IllegalArgumentException if source array, offset, or length are invalid
-     * @see Base64#GZIP
-     * @see Base64#DO_BREAK_LINES
-     * @since 2.0
-     */
-    public static String encodeBytes(byte[] source, int off, int len, int options) throws java.io.IOException {
-        byte[] encoded = encodeBytesToBytes(source, off, len, options);
-
-        // Return value according to relevant encoding.
-        return new String(encoded, PREFERRED_ENCODING);
-
-    }   // end encodeBytes
-
-
-    /**
-     * Similar to {@link #encodeBytes(byte[])} but returns
-     * a byte array instead of instantiating a String. This is more efficient
-     * if you're working with I/O streams and have large data sets to encode.
-     *
-     * @param source The data to convert
-     * @return The Base64-encoded data as a byte[] (of ASCII characters)
-     * @throws NullPointerException if source array is null
-     * @since 2.3.1
-     */
-    public static byte[] encodeBytesToBytes(byte[] source) {
-        byte[] encoded = null;
-        try {
-            encoded = encodeBytesToBytes(source, 0, source.length, Base64.NO_OPTIONS);
-        } catch (java.io.IOException ex) {
-            throw new IllegalStateException("IOExceptions only come from GZipping, which is turned off: ", ex);
-        }
-        return encoded;
-    }
-
-
-    /**
-     * Similar to {@link #encodeBytes(byte[], int, int, int)} but returns
-     * a byte array instead of instantiating a String. This is more efficient
-     * if you're working with I/O streams and have large data sets to encode.
-     *
-     * @param source  The data to convert
-     * @param off     Offset in array where conversion should begin
-     * @param len     Length of data to convert
-     * @param options Specified options
-     * @return The Base64-encoded data as a String
-     * @throws java.io.IOException      if there is an error
-     * @throws NullPointerException     if source array is null
-     * @throws IllegalArgumentException if source array, offset, or length are invalid
-     * @see Base64#GZIP
-     * @see Base64#DO_BREAK_LINES
-     * @since 2.3.1
-     */
-    public static byte[] encodeBytesToBytes(byte[] source, int off, int len, int options) throws java.io.IOException {
-        Objects.requireNonNull(source, "Cannot serialize a null array.");
-
-        if (off < 0) {
-            throw new IllegalArgumentException("Cannot have negative offset: " + off);
-        }   // end if: off < 0
-
-        if (len < 0) {
-            throw new IllegalArgumentException("Cannot have length offset: " + len);
-        }   // end if: len < 0
-
-        if (off + len > source.length) {
-            throw new IllegalArgumentException(
-                    String.format(Locale.ROOT, "Cannot have offset of %d and length of %d with array of length %d", off, len, source.length));
-        }   // end if: off < 0
-
-        // Compress?
-        if ((options & GZIP) != 0) {
-            return encodeCompressedBytes(source, off, len, options);
-        }   // end if: compress
-
-        // Else, don't compress. Better not to use streams at all then.
-        else {
-            return encodeNonCompressedBytes(source, off, len, options);
-        }   // end else: don't compress
-
-    }   // end encodeBytesToBytes
-
-    private static byte[] encodeNonCompressedBytes(byte[] source, int off, int len, int options) {
-        boolean breakLines = (options & DO_BREAK_LINES) != 0;
-
-        //int    len43   = len * 4 / 3;
-        //byte[] outBuff = new byte[   ( len43 )                      // Main 4:3
-        //                           + ( (len % 3) > 0 ? 4 : 0 )      // Account for padding
-        //                           + (breakLines ? ( len43 / MAX_LINE_LENGTH ) : 0) ]; // New lines
-        // Try to determine more precisely how big the array needs to be.
-        // If we get it right, we don't have to do an array copy, and
-        // we save a bunch of memory.
-        int encLen = (len / 3) * 4 + (len % 3 > 0 ? 4 : 0); // Bytes needed for actual encoding
-        if (breakLines) {
-            encLen += encLen / MAX_LINE_LENGTH; // Plus extra newline characters
-        }
-        byte[] outBuff = new byte[encLen];
-
-
-        int d = 0;
-        int e = 0;
-        int len2 = len - 2;
-        int lineLength = 0;
-        for (; d < len2; d += 3, e += 4) {
-            encode3to4(source, d + off, 3, outBuff, e, options);
-
-            lineLength += 4;
-            if (breakLines && lineLength >= MAX_LINE_LENGTH) {
-                outBuff[e + 4] = NEW_LINE;
-                e++;
-                lineLength = 0;
-            }   // end if: end of line
-        }   // en dfor: each piece of array
-
-        if (d < len) {
-            encode3to4(source, d + off, len - d, outBuff, e, options);
-            e += 4;
-        }   // end if: some padding needed
-
-
-        // Only resize array if we didn't guess it right.
-        if (e <= outBuff.length - 1) {
-            // If breaking lines and the last byte falls right at
-            // the line length (76 bytes per line), there will be
-            // one extra byte, and the array will need to be resized.
-            // Not too bad of an estimate on array size, I'd say.
-            byte[] finalOut = new byte[e];
-            System.arraycopy(outBuff, 0, finalOut, 0, e);
-            //System.err.println("Having to resize array from " + outBuff.length + " to " + e );
-            return finalOut;
-        } else {
-            //System.err.println("No need to resize array.");
-            return outBuff;
-        }
-    }
-
-    private static byte[] encodeCompressedBytes(byte[] source, int off, int len, int options) throws IOException {
-        java.io.ByteArrayOutputStream baos = null;
-        java.util.zip.GZIPOutputStream gzos = null;
-        OutputStream b64os = null;
-
-        try {
-            // GZip -> Base64 -> ByteArray
-            baos = new java.io.ByteArrayOutputStream();
-            b64os = new OutputStream(baos, ENCODE | options);
-            gzos = new java.util.zip.GZIPOutputStream(b64os);
-
-            gzos.write(source, off, len);
-            gzos.close();
-        }   // end try
-        catch (IOException e) {
-            // Catch it and then throw it immediately so that
-            // the finally{} block is called for cleanup.
-            throw e;
-        }   // end catch
-        finally {
-            try {
-                gzos.close();
-            } catch (Exception e) {
-            }
-            try {
-                b64os.close();
-            } catch (Exception e) {
-            }
-            try {
-                baos.close();
-            } catch (Exception e) {
-            }
-        }   // end finally
-
-        return baos.toByteArray();
-    }
-
-
-/* ********  D E C O D I N G   M E T H O D S  ******** */
-
-
-    /**
-     * Decodes four bytes from array <var>source</var>
-     * and writes the resulting bytes (up to three of them)
-     * to <var>destination</var>.
-     * The source and destination arrays can be manipulated
-     * anywhere along their length by specifying
-     * <var>srcOffset</var> and <var>destOffset</var>.
-     * This method does not check to make sure your arrays
-     * are large enough to accommodate <var>srcOffset</var> + 4 for
-     * the <var>source</var> array or <var>destOffset</var> + 3 for
-     * the <var>destination</var> array.
-     * This method returns the actual number of bytes that
-     * were converted from the Base64 encoding.
-     * <p>This is the lowest level of the decoding methods with
-     * all possible parameters.</p>
-     *
-     * @param source      the array to convert
-     * @param srcOffset   the index where conversion begins
-     * @param destination the array to hold the conversion
-     * @param destOffset  the index where output will be put
-     * @param options     alphabet type is pulled from this (standard, url-safe, ordered)
-     * @return the number of decoded bytes converted
-     * @throws NullPointerException     if source or destination arrays are null
-     * @throws IllegalArgumentException if srcOffset or destOffset are invalid
-     *                                  or there is not enough room in the array.
-     * @since 1.3
-     */
-    private static int decode4to3(byte[] source, int srcOffset, byte[] destination, int destOffset, int options) {
-        // Lots of error checking and exception throwing
-        Objects.requireNonNull(source, "Source array was null.");
-        Objects.requireNonNull(destination, "Destination array was null.");
-        if (srcOffset < 0 || srcOffset + 3 >= source.length) {
-            throw new IllegalArgumentException(String.format(Locale.ROOT,
-                    "Source array with length %d cannot have offset of %d and still process four bytes.", source.length, srcOffset));
-        }   // end if
-        if (destOffset < 0 || destOffset + 2 >= destination.length) {
-            throw new IllegalArgumentException(String.format(Locale.ROOT,
-                    "Destination array with length %d cannot have offset of %d and still store three bytes.", destination.length, destOffset));
-        }   // end if
-
-        byte[] DECODABET = getDecodabet(options);
-
-
-        // Two ways to do the same thing. Don't know which way I like best.
-        //int outBuff =   ( ( DECODABET[ source[ srcOffset    ] ] << 24 ) >>>  6 )
-        //              | ( ( DECODABET[ source[ srcOffset + 1] ] << 24 ) >>> 12 );
-        int outBuff = ((DECODABET[source[srcOffset]] & 0xFF) << 18)
-                | ((DECODABET[source[srcOffset + 1]] & 0xFF) << 12);
-
-        destination[destOffset] = (byte) (outBuff >>> 16);
-
-        // Example: Dk==
-        if (source[srcOffset + 2] == EQUALS_SIGN) {
-            return 1;
-        }
-
-        outBuff |= ((DECODABET[source[srcOffset + 2]] & 0xFF) << 6);
-        destination[destOffset + 1] = (byte) (outBuff >>> 8);
-
-        // Example: DkL=
-        if (source[srcOffset + 3] == EQUALS_SIGN) {
-            return 2;
-        }
-
-        outBuff |= ((DECODABET[source[srcOffset + 3]] & 0xFF));
-        destination[destOffset + 2] = (byte) (outBuff);
-
-        // Example: DkLE
-        return 3;
-    }
-
-
-    /**
-     * Low-level access to decoding ASCII characters in
-     * the form of a byte array. <strong>Ignores GUNZIP option, if
-     * it's set.</strong> This is not generally a recommended method,
-     * although it is used internally as part of the decoding process.
-     * Special case: if len = 0, an empty array is returned. Still,
-     * if you need more speed and reduced memory footprint (and aren't
-     * gzipping), consider this method.
-     *
-     * @param source The Base64 encoded data
-     * @return decoded data
-     * @since 2.3.1
-     */
-    public static byte[] decode(byte[] source)
-            throws java.io.IOException {
-        byte[] decoded = null;
-//        try {
-        decoded = decode(source, 0, source.length, Base64.NO_OPTIONS);
-//        } catch( java.io.IOException ex ) {
-//            assert false : "IOExceptions only come from GZipping, which is turned off: " + ex.getMessage();
-//        }
-        return decoded;
-    }
-
-
-    /**
-     * Low-level access to decoding ASCII characters in
-     * the form of a byte array. <strong>Ignores GUNZIP option, if
-     * it's set.</strong> This is not generally a recommended method,
-     * although it is used internally as part of the decoding process.
-     * Special case: if len = 0, an empty array is returned. Still,
-     * if you need more speed and reduced memory footprint (and aren't
-     * gzipping), consider this method.
-     *
-     * @param source  The Base64 encoded data
-     * @param off     The offset of where to begin decoding
-     * @param len     The length of characters to decode
-     * @param options Can specify options such as alphabet type to use
-     * @return decoded data
-     * @throws java.io.IOException If bogus characters exist in source data
-     * @since 1.3
-     */
-    public static byte[] decode(byte[] source, int off, int len, int options) throws java.io.IOException {
-        // Lots of error checking and exception throwing
-        Objects.requireNonNull(source, "Cannot decode null source array.");
-        if (off < 0 || off + len > source.length) {
-            throw new IllegalArgumentException(String.format(Locale.ROOT,
-                    "Source array with length %d cannot have offset of %d and process %d bytes.", source.length, off, len));
-        }   // end if
-
-        if (len == 0) {
-            return new byte[0];
-        } else if (len < 4) {
-            throw new IllegalArgumentException(
-                    "Base64-encoded string must have at least four characters, but length specified was " + len);
-        }   // end if
-
-        byte[] DECODABET = getDecodabet(options);
-
-        int len34 = len * 3 / 4;       // Estimate on array size
-        byte[] outBuff = new byte[len34]; // Upper limit on size of output
-
-        int outBuffPosn = decode(source, off, len, options, DECODABET, outBuff);
-
-        byte[] out = new byte[outBuffPosn];
-        System.arraycopy(outBuff, 0, out, 0, outBuffPosn);
-        return out;
-    }   // end decode
-
-    private static int decode(byte[] source, int off, int len, int options, byte[] DECODABET, byte[] outBuff) throws IOException {
-        int outBuffPosn = 0;             // Keep track of where we're writing
-        byte[] b4 = new byte[4];     // Four byte buffer from source, eliminating white space
-        int b4Posn = 0;               // Keep track of four byte input buffer
-        for (int i = off; i < off + len; i++) {  // Loop through source
-
-            byte sbiDecode = DECODABET[source[i] & 0xFF];
-
-            // White space, Equals sign, or legit Base64 character
-            // Note the values such as -5 and -9 in the
-            // DECODABETs at the top of the file.
-            if (sbiDecode >= WHITE_SPACE_ENC) {
-                if (sbiDecode >= EQUALS_SIGN_ENC) {
-                    b4[b4Posn++] = source[i];         // Save non-whitespace
-                    if (b4Posn > 3) {                  // Time to decode?
-                        outBuffPosn += decode4to3(b4, 0, outBuff, outBuffPosn, options);
-                        b4Posn = 0;
-
-                        // If that was the equals sign, break out of 'for' loop
-                        if (source[i] == EQUALS_SIGN) {
-                            // check if the equals sign is somewhere in between
-                            if (i+1 < len + off) {
-                              throw new IOException(String.format(Locale.ROOT,
-                                      "Found equals sign at position %d of the base64 string, not at the end", i));
-                            }
-                            break;
-                        }   // end if: equals sign
-                    }   // end if: quartet built
-                    else {
-                      if (source[i] == EQUALS_SIGN && len + off > i && source[i+1] != EQUALS_SIGN) {
-                        throw new IOException(String.format(Locale.ROOT,
-                                "Found equals sign at position %d of the base64 string, not at the end", i));
-                      } // enf if: equals sign and next character not as well
-                    } // end else:
-                }   // end if: equals sign or better
-            }   // end if: white space, equals sign or better
-            else {
-                // There's a bad input character in the Base64 stream.
-                throw new IOException(String.format(Locale.ROOT,
-                        "Bad Base64 input character decimal %d in array position %d", ((int) source[i]) & 0xFF, i));
-            }   // end else:
-        }   // each input character
-        return outBuffPosn;
-    }
-
-
-    /**
-     * Decodes data from Base64 notation, automatically
-     * detecting gzip-compressed data and decompressing it.
-     *
-     * @param s the string to decode
-     * @return the decoded data
-     * @throws java.io.IOException If there is a problem
-     * @since 1.4
-     */
-    public static byte[] decode(String s) throws java.io.IOException {
-        return decode(s, NO_OPTIONS);
-    }
-
-
-    /**
-     * Decodes data from Base64 notation, automatically
-     * detecting gzip-compressed data and decompressing it.
-     *
-     * @param s       the string to decode
-     * @param options encode options such as URL_SAFE
-     * @return the decoded data
-     * @throws java.io.IOException  if there is an error
-     * @throws NullPointerException if <tt>s</tt> is null
-     * @since 1.4
-     */
-    public static byte[] decode(String s, int options) throws java.io.IOException {
-
-        if (s == null) {
-            throw new NullPointerException("Input string was null.");
-        }   // end if
-
-        byte[] bytes = s.getBytes(PREFERRED_ENCODING);
-        //</change>
-
-        // Decode
-        bytes = decode(bytes, 0, bytes.length, options);
-
-        // Check to see if it's gzip-compressed
-        // GZIP Magic Two-Byte Number: 0x8b1f (35615)
-        boolean dontGunzip = (options & DONT_GUNZIP) != 0;
-        if ((bytes != null) && (bytes.length >= 4) && (!dontGunzip)) {
-
-            int head = ((int) bytes[0] & 0xff) | ((bytes[1] << 8) & 0xff00);
-            if (java.util.zip.GZIPInputStream.GZIP_MAGIC == head) {
-                java.io.ByteArrayInputStream bais = null;
-                java.util.zip.GZIPInputStream gzis = null;
-                java.io.ByteArrayOutputStream baos = null;
-                byte[] buffer = new byte[2048];
-                int length = 0;
-
-                try {
-                    baos = new java.io.ByteArrayOutputStream();
-                    bais = new java.io.ByteArrayInputStream(bytes);
-                    gzis = new java.util.zip.GZIPInputStream(bais);
-
-                    while ((length = gzis.read(buffer)) >= 0) {
-                        baos.write(buffer, 0, length);
-                    }   // end while: reading input
-
-                    // No error? Get new bytes.
-                    bytes = baos.toByteArray();
-
-                }   // end try
-                catch (java.io.IOException e) {
-                    // e.printStackTrace();
-                    // Just return originally-decoded bytes
-                }   // end catch
-                finally {
-                    try {
-                        baos.close();
-                    } catch (Exception e) {
-                    }
-                    try {
-                        gzis.close();
-                    } catch (Exception e) {
-                    }
-                    try {
-                        bais.close();
-                    } catch (Exception e) {
-                    }
-                }   // end finally
-
-            }   // end if: gzipped
-        }   // end if: bytes.length >= 2
-
-        return bytes;
-    }   // end decode
-
-
-
-    /* ********  I N N E R   C L A S S   I N P U T S T R E A M  ******** */
-
-
-    /**
-     * A {@link Base64.InputStream} will read data from another
-     * <tt>java.io.InputStream</tt>, given in the constructor,
-     * and encode/decode to/from Base64 notation on the fly.
-     *
-     * @see Base64
-     * @since 1.3
-     */
-    public static class InputStream extends java.io.FilterInputStream {
-
-        private boolean encode;         // Encoding or decoding
-        private int position;       // Current position in the buffer
-        private byte[] buffer;         // Small buffer holding converted data
-        private int bufferLength;   // Length of buffer (3 or 4)
-        private int numSigBytes;    // Number of meaningful bytes in the buffer
-        private int lineLength;
-        private boolean breakLines;     // Break lines at less than 80 characters
-        private int options;        // Record options used to create the stream.
-        private byte[] decodabet;      // Local copies to avoid extra method calls
-
-
-        /**
-         * Constructs a {@link Base64.InputStream} in DECODE mode.
-         *
-         * @param in the <tt>java.io.InputStream</tt> from which to read data.
-         * @since 1.3
-         */
-        public InputStream(java.io.InputStream in) {
-            this(in, DECODE);
-        }   // end constructor
-
-
-        /**
-         * Constructs a {@link Base64.InputStream} in
-         * either ENCODE or DECODE mode.
-         * <p>
-         * Valid options:<pre>
-         *   ENCODE or DECODE: Encode or Decode as data is read.
-         *   DO_BREAK_LINES: break lines at 76 characters
-         *     (only meaningful when encoding)
-         * </pre>
-         * <p>
-         * Example: <code>new Base64.InputStream( in, Base64.DECODE )</code>
-         *
-         * @param in      the <tt>java.io.InputStream</tt> from which to read data.
-         * @param options Specified options
-         * @see Base64#ENCODE
-         * @see Base64#DECODE
-         * @see Base64#DO_BREAK_LINES
-         * @since 2.0
-         */
-        public InputStream(java.io.InputStream in, int options) {
-
-            super(in);
-            this.options = options; // Record for later
-            this.breakLines = (options & DO_BREAK_LINES) > 0;
-            this.encode = (options & ENCODE) > 0;
-            this.bufferLength = encode ? 4 : 3;
-            this.buffer = new byte[bufferLength];
-            this.position = -1;
-            this.lineLength = 0;
-            this.decodabet = getDecodabet(options);
-        }   // end constructor
-
-        /**
-         * Reads enough of the input stream to convert
-         * to/from Base64 and returns the next byte.
-         *
-         * @return next byte
-         * @since 1.3
-         */
-        @Override
-        public int read() throws java.io.IOException {
-
-            // Do we need to get data?
-            if (position < 0) {
-                if (encode) {
-                    byte[] b3 = new byte[3];
-                    int numBinaryBytes = 0;
-                    for (int i = 0; i < 3; i++) {
-                        int b = in.read();
-
-                        // If end of stream, b is -1.
-                        if (b >= 0) {
-                            b3[i] = (byte) b;
-                            numBinaryBytes++;
-                        } else {
-                            break; // out of for loop
-                        }   // end else: end of stream
-
-                    }   // end for: each needed input byte
-
-                    if (numBinaryBytes > 0) {
-                        encode3to4(b3, 0, numBinaryBytes, buffer, 0, options);
-                        position = 0;
-                        numSigBytes = 4;
-                    }   // end if: got data
-                    else {
-                        return -1;  // Must be end of stream
-                    }   // end else
-                }   // end if: encoding
-
-                // Else decoding
-                else {
-                    byte[] b4 = new byte[4];
-                    int i = 0;
-                    for (i = 0; i < 4; i++) {
-                        // Read four "meaningful" bytes:
-                        int b = 0;
-                        do {
-                            b = in.read();
-                        }
-                        while (b >= 0 && decodabet[b & 0x7f] <= WHITE_SPACE_ENC);
-
-                        if (b < 0) {
-                            break; // Reads a -1 if end of stream
-                        }   // end if: end of stream
-
-                        b4[i] = (byte) b;
-                    }   // end for: each needed input byte
-
-                    if (i == 4) {
-                        numSigBytes = decode4to3(b4, 0, buffer, 0, options);
-                        position = 0;
-                    }   // end if: got four characters
-                    else if (i == 0) {
-                        return -1;
-                    }   // end else if: also padded correctly
-                    else {
-                        // Must have broken out from above.
-                        throw new java.io.IOException("Improperly padded Base64 input.");
-                    }   // end
-
-                }   // end else: decode
-            }   // end else: get data
-
-            // Got data?
-            if (position >= 0) {
-                // End of relevant data?
-                if ( /*!encode &&*/ position >= numSigBytes) {
-                    return -1;
-                }   // end if: got data
-
-                if (encode && breakLines && lineLength >= MAX_LINE_LENGTH) {
-                    lineLength = 0;
-                    return '\n';
-                }   // end if
-                else {
-                    lineLength++;   // This isn't important when decoding
-                    // but throwing an extra "if" seems
-                    // just as wasteful.
-
-                    int b = buffer[position++];
-
-                    if (position >= bufferLength) {
-                        position = -1;
-                    }   // end if: end
-
-                    return b & 0xFF; // This is how you "cast" a byte that's
-                    // intended to be unsigned.
-                }   // end else
-            }   // end if: position >= 0
-
-            // Else error
-            else {
-                throw new java.io.IOException("Error in Base64 code reading stream.");
-            }   // end else
-        }   // end read
-
-
-        /**
-         * Calls {@link #read()} repeatedly until the end of stream
-         * is reached or <var>len</var> bytes are read.
-         * Returns number of bytes read into array or -1 if
-         * end of stream is encountered.
-         *
-         * @param dest array to hold values
-         * @param off  offset for array
-         * @param len  max number of bytes to read into array
-         * @return bytes read into array or -1 if end of stream is encountered.
-         * @since 1.3
-         */
-        @Override
-        public int read(byte[] dest, int off, int len)
-                throws java.io.IOException {
-            int i;
-            int b;
-            for (i = 0; i < len; i++) {
-                b = read();
-
-                if (b >= 0) {
-                    dest[off + i] = (byte) b;
-                } else if (i == 0) {
-                    return -1;
-                } else {
-                    break; // Out of 'for' loop
-                } // Out of 'for' loop
-            }   // end for: each byte read
-            return i;
-        }   // end read
-
-    }   // end inner class InputStream
-
-
-    /* ********  I N N E R   C L A S S   O U T P U T S T R E A M  ******** */
-
-
-    /**
-     * A {@link Base64.OutputStream} will write data to another
-     * <tt>java.io.OutputStream</tt>, given in the constructor,
-     * and encode/decode to/from Base64 notation on the fly.
-     *
-     * @see Base64
-     * @since 1.3
-     */
-    public static class OutputStream extends java.io.FilterOutputStream {
-
-        private boolean encode;
-        private int position;
-        private byte[] buffer;
-        private int bufferLength;
-        private int lineLength;
-        private boolean breakLines;
-        private byte[] b4;         // Scratch used in a few places
-        private boolean suspendEncoding;
-        private int options;    // Record for later
-        private byte[] decodabet;  // Local copies to avoid extra method calls
-
-        /**
-         * Constructs a {@link Base64.OutputStream} in ENCODE mode.
-         *
-         * @param out the <tt>java.io.OutputStream</tt> to which data will be written.
-         * @since 1.3
-         */
-        public OutputStream(java.io.OutputStream out) {
-            this(out, ENCODE);
-        }   // end constructor
-
-
-        /**
-         * Constructs a {@link Base64.OutputStream} in
-         * either ENCODE or DECODE mode.
-         * <p>
-         * Valid options:<pre>
-         *   ENCODE or DECODE: Encode or Decode as data is read.
-         *   DO_BREAK_LINES: don't break lines at 76 characters
-         *     (only meaningful when encoding)
-         * </pre>
-         * <p>
-         * Example: <code>new Base64.OutputStream( out, Base64.ENCODE )</code>
-         *
-         * @param out     the <tt>java.io.OutputStream</tt> to which data will be written.
-         * @param options Specified options.
-         * @see Base64#ENCODE
-         * @see Base64#DECODE
-         * @see Base64#DO_BREAK_LINES
-         * @since 1.3
-         */
-        public OutputStream(java.io.OutputStream out, int options) {
-            super(out);
-            this.breakLines = (options & DO_BREAK_LINES) != 0;
-            this.encode = (options & ENCODE) != 0;
-            this.bufferLength = encode ? 3 : 4;
-            this.buffer = new byte[bufferLength];
-            this.position = 0;
-            this.lineLength = 0;
-            this.suspendEncoding = false;
-            this.b4 = new byte[4];
-            this.options = options;
-            this.decodabet = getDecodabet(options);
-        }   // end constructor
-
-
-        /**
-         * Writes the byte to the output stream after
-         * converting to/from Base64 notation.
-         * When encoding, bytes are buffered three
-         * at a time before the output stream actually
-         * gets a write() call.
-         * When decoding, bytes are buffered four
-         * at a time.
-         *
-         * @param theByte the byte to write
-         * @since 1.3
-         */
-        @Override
-        public void write(int theByte)
-                throws java.io.IOException {
-            // Encoding suspended?
-            if (suspendEncoding) {
-                this.out.write(theByte);
-                return;
-            }   // end if: suspended
-
-            // Encode?
-            if (encode) {
-                buffer[position++] = (byte) theByte;
-                if (position >= bufferLength) { // Enough to encode.
-
-                    this.out.write(encode3to4(b4, buffer, bufferLength, options));
-
-                    lineLength += 4;
-                    if (breakLines && lineLength >= MAX_LINE_LENGTH) {
-                        this.out.write(NEW_LINE);
-                        lineLength = 0;
-                    }   // end if: end of line
-
-                    position = 0;
-                }   // end if: enough to output
-            }   // end if: encoding
-
-            // Else, Decoding
-            else {
-                // Meaningful Base64 character?
-                if (decodabet[theByte & 0x7f] > WHITE_SPACE_ENC) {
-                    buffer[position++] = (byte) theByte;
-                    if (position >= bufferLength) { // Enough to output.
-
-                        int len = Base64.decode4to3(buffer, 0, b4, 0, options);
-                        out.write(b4, 0, len);
-                        position = 0;
-                    }   // end if: enough to output
-                }   // end if: meaningful base64 character
-                else if (decodabet[theByte & 0x7f] != WHITE_SPACE_ENC) {
-                    throw new java.io.IOException("Invalid character in Base64 data.");
-                }   // end else: not white space either
-            }   // end else: decoding
-        }   // end write
-
-
-        /**
-         * Calls {@link #write(int)} repeatedly until <var>len</var>
-         * bytes are written.
-         *
-         * @param theBytes array from which to read bytes
-         * @param off      offset for array
-         * @param len      max number of bytes to read into array
-         * @since 1.3
-         */
-        @Override
-        public void write(byte[] theBytes, int off, int len)
-                throws java.io.IOException {
-            // Encoding suspended?
-            if (suspendEncoding) {
-                this.out.write(theBytes, off, len);
-                return;
-            }   // end if: suspended
-
-            for (int i = 0; i < len; i++) {
-                write(theBytes[off + i]);
-            }   // end for: each byte written
-
-        }   // end write
-
-
-        /**
-         * Method added by PHIL. [Thanks, PHIL. -Rob]
-         * This pads the buffer without closing the stream.
-         *
-         * @throws java.io.IOException if there's an error.
-         */
-        public void flushBase64() throws java.io.IOException {
-            if (position > 0) {
-                if (encode) {
-                    out.write(encode3to4(b4, buffer, position, options));
-                    position = 0;
-                }   // end if: encoding
-                else {
-                    throw new java.io.IOException("Base64 input not properly padded.");
-                }   // end else: decoding
-            }   // end if: buffer partially full
-
-        }   // end flush
-
-
-        /**
-         * Flushes and closes (I think, in the superclass) the stream.
-         *
-         * @since 1.3
-         */
-        @Override
-        public void close() throws java.io.IOException {
-            // 1. Ensure that pending characters are written
-            flushBase64();
-
-            // 2. Actually close the stream
-            // Base class both flushes and closes.
-            super.close();
-
-            buffer = null;
-            out = null;
-        }   // end close
-
-
-        /**
-         * Suspends encoding of the stream.
-         * May be helpful if you need to embed a piece of
-         * base64-encoded data in a stream.
-         *
-         * @throws java.io.IOException if there's an error flushing
-         * @since 1.5.1
-         */
-        public void suspendEncoding() throws java.io.IOException {
-            flushBase64();
-            this.suspendEncoding = true;
-        }   // end suspendEncoding
-
-
-        /**
-         * Resumes encoding of the stream.
-         * May be helpful if you need to embed a piece of
-         * base64-encoded data in a stream.
-         *
-         * @since 1.5.1
-         */
-        public void resumeEncoding() {
-            this.suspendEncoding = false;
-        }   // end resumeEncoding
-
-
-    }   // end inner class OutputStream
-
-
-}   // end class Base64
diff --git a/core/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java b/core/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java
index 9a3c35f3527..9f5e5f34a1b 100644
--- a/core/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java
+++ b/core/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java
@@ -21,6 +21,7 @@ package org.elasticsearch.common;
 
 
 import java.io.IOException;
+import java.util.Base64;
 import java.util.Random;
 
 class RandomBasedUUIDGenerator implements UUIDGenerator {
@@ -54,14 +55,6 @@ class RandomBasedUUIDGenerator implements UUIDGenerator {
          * We set only the MSB of the variant*/
         randomBytes[8] &= 0x3f;  /* clear the 2 most significant bits */
         randomBytes[8] |= 0x80;  /* set the variant (MSB is set)*/
-        try {
-            byte[] encoded = Base64.encodeBytesToBytes(randomBytes, 0, randomBytes.length, Base64.URL_SAFE);
-            // we know the bytes are 16, and not a multi of 3, so remove the 2 padding chars that are added
-            assert encoded[encoded.length - 1] == '=';
-            assert encoded[encoded.length - 2] == '=';
-            return new String(encoded, 0, encoded.length - 2, Base64.PREFERRED_ENCODING);
-        } catch (IOException e) {
-            throw new IllegalStateException("should not be thrown");
-        }
+        return Base64.getUrlEncoder().withoutPadding().encodeToString(randomBytes);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/TimeBasedUUIDGenerator.java b/core/src/main/java/org/elasticsearch/common/TimeBasedUUIDGenerator.java
index d1a22a17cda..8d507ae7f22 100644
--- a/core/src/main/java/org/elasticsearch/common/TimeBasedUUIDGenerator.java
+++ b/core/src/main/java/org/elasticsearch/common/TimeBasedUUIDGenerator.java
@@ -19,8 +19,7 @@
 
 package org.elasticsearch.common;
 
-
-import java.io.IOException;
+import java.util.Base64;
 import java.util.concurrent.atomic.AtomicInteger;
 
 /** These are essentially flake ids (http://boundary.com/blog/2012/01/12/flake-a-decentralized-k-ordered-unique-id-generator-in-erlang) but
@@ -80,15 +79,6 @@ class TimeBasedUUIDGenerator implements UUIDGenerator {
 
         assert 9 + SECURE_MUNGED_ADDRESS.length == uuidBytes.length;
 
-        byte[] encoded;
-        try {
-            encoded = Base64.encodeBytesToBytes(uuidBytes, 0, uuidBytes.length, Base64.URL_SAFE);
-        } catch (IOException e) {
-            throw new IllegalStateException("should not be thrown", e);
-        }
-
-        // We are a multiple of 3 bytes so we should not see any padding:
-        assert encoded[encoded.length - 1] != '=';
-        return new String(encoded, 0, encoded.length, Base64.PREFERRED_ENCODING);
+        return Base64.getUrlEncoder().withoutPadding().encodeToString(uuidBytes);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/engine/CommitStats.java b/core/src/main/java/org/elasticsearch/index/engine/CommitStats.java
index 3656dcbfb96..48fb8a80eeb 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/CommitStats.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/CommitStats.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.index.engine;
 
 import org.apache.lucene.index.SegmentInfos;
-import org.elasticsearch.common.Base64;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -29,6 +28,7 @@ import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.Base64;
 import java.util.Map;
 
 /** a class the returns dynamic information with respect to the last commit point of this shard */
@@ -44,9 +44,7 @@ public final class CommitStats implements Streamable, ToXContent {
         userData = MapBuilder.<String, String>newMapBuilder().putAll(segmentInfos.getUserData()).immutableMap();
         // lucene calls the current generation, last generation.
         generation = segmentInfos.getLastGeneration();
-        if (segmentInfos.getId() != null) { // id is only written starting with Lucene 5.0
-            id = Base64.encodeBytes(segmentInfos.getId());
-        }
+        id = Base64.getEncoder().encodeToString(segmentInfos.getId());
         numDocs = Lucene.getNumDocs(segmentInfos);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/engine/Engine.java b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
index 69aee4ac680..d5fedcf0343 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -28,22 +28,18 @@ import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.SegmentCommitInfo;
-import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.index.SegmentReader;
 import org.apache.lucene.index.SnapshotDeletionPolicy;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
 import org.apache.lucene.search.SearcherManager;
-import org.apache.lucene.search.join.BitSetProducer;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.Accountables;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.common.Base64;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -65,7 +61,6 @@ import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.merge.MergeStats;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.index.shard.TranslogRecoveryPerformer;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
 
@@ -74,7 +69,7 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.nio.file.NoSuchFileException;
 import java.util.Arrays;
-import java.util.Collection;
+import java.util.Base64;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
@@ -1092,7 +1087,7 @@ public abstract class Engine implements Closeable {
 
         @Override
         public String toString() {
-            return Base64.encodeBytes(id);
+            return Base64.getEncoder().encodeToString(id);
         }
 
         public boolean idsEqual(byte[] id) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
index 1ca67a8cbc0..ff6d943b72a 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java
@@ -26,8 +26,6 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.store.ByteArrayDataOutput;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.Base64;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
@@ -45,6 +43,7 @@ import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.QueryShardException;
 
 import java.io.IOException;
+import java.util.Base64;
 import java.util.List;
 import java.util.Map;
 
@@ -124,11 +123,7 @@ public class BinaryFieldMapper extends FieldMapper {
             } else if (value instanceof byte[]) {
                 bytes = new BytesArray((byte[]) value);
             } else {
-                try {
-                    bytes = new BytesArray(Base64.decode(value.toString()));
-                } catch (IOException e) {
-                    throw new ElasticsearchParseException("failed to convert bytes", e);
-                }
+                bytes = new BytesArray(Base64.getDecoder().decode(value.toString()));
             }
             return bytes;
         }
diff --git a/core/src/main/java/org/elasticsearch/ingest/core/IngestDocument.java b/core/src/main/java/org/elasticsearch/ingest/core/IngestDocument.java
index 8be307d776c..8b1f02607d8 100644
--- a/core/src/main/java/org/elasticsearch/ingest/core/IngestDocument.java
+++ b/core/src/main/java/org/elasticsearch/ingest/core/IngestDocument.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.ingest.core;
 
-import org.elasticsearch.common.Base64;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.index.mapper.internal.IdFieldMapper;
 import org.elasticsearch.index.mapper.internal.IndexFieldMapper;
@@ -30,11 +29,11 @@ import org.elasticsearch.index.mapper.internal.TTLFieldMapper;
 import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
 import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
 
-import java.io.IOException;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Base64;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
@@ -43,8 +42,6 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.TimeZone;
 
-import static java.nio.charset.StandardCharsets.UTF_8;
-
 /**
  * Represents a single document being captured before indexing and holds the source and metadata (like id, type and index).
  */
@@ -144,11 +141,7 @@ public final class IngestDocument {
         if (object instanceof byte[]) {
             return (byte[]) object;
         } else if (object instanceof String) {
-            try {
-                return Base64.decode(object.toString().getBytes(UTF_8));
-            } catch (IOException e) {
-                throw new IllegalArgumentException("Could not base64 decode path [ " + path + "]", e);
-            }
+            return Base64.getDecoder().decode(object.toString());
         } else {
             throw new IllegalArgumentException("Content field [" + path + "] of unknown type [" + object.getClass().getName() +
                 "], must be string or byte array");
@@ -464,7 +457,6 @@ public final class IngestDocument {
 
     private static void appendValues(List<Object> list, Object value) {
         if (value instanceof List) {
-            @SuppressWarnings("unchecked")
             List<?> valueList = (List<?>) value;
             valueList.stream().forEach(list::add);
         } else {
diff --git a/core/src/test/java/org/elasticsearch/common/Base64Tests.java b/core/src/test/java/org/elasticsearch/common/Base64Tests.java
deleted file mode 100644
index 74691c0b739..00000000000
--- a/core/src/test/java/org/elasticsearch/common/Base64Tests.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.common;
-
-import org.elasticsearch.test.ESTestCase;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.Locale;
-
-import static org.hamcrest.Matchers.is;
-
-/**
- *
- */
-public class Base64Tests extends ESTestCase {
-    // issue #6334
-    public void testBase64DecodeWithExtraCharactersAfterPadding() throws Exception {
-        String plain = randomAsciiOfLengthBetween(1, 20) + ":" + randomAsciiOfLengthBetween(1, 20);
-        String encoded = Base64.encodeBytes(plain.getBytes(StandardCharsets.UTF_8));
-        assertValidBase64(encoded, plain);
-
-        // lets append some trash here, if the encoded string has been padded
-        char lastChar = encoded.charAt(encoded.length() - 1);
-        if (lastChar == '=') {
-            assertInvalidBase64(encoded + randomAsciiOfLength(3));
-        }
-    }
-
-    private void assertValidBase64(String base64, String expected) throws IOException {
-        String decoded = new String(Base64.decode(base64.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8);
-        assertThat(decoded, is(expected));
-    }
-
-    private void assertInvalidBase64(String base64) {
-        try {
-            Base64.decode(base64.getBytes(StandardCharsets.UTF_8));
-            fail(String.format(Locale.ROOT, "Expected IOException to be thrown for string %s (len %d)", base64, base64.length()));
-        } catch (IOException e) {}
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 0e20c22e119..6a84b62fabd 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -54,7 +54,6 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.support.TransportActions;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.Base64;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -111,6 +110,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Base64;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
@@ -829,7 +829,7 @@ public class InternalEngineTests extends ESTestCase {
             engine.index(new Engine.Index(newUid("1"), doc));
             Engine.CommitId commitID = engine.flush();
             assertThat(commitID, equalTo(new Engine.CommitId(store.readLastCommittedSegmentsInfo().getId())));
-            byte[] wrongBytes = Base64.decode(commitID.toString());
+            byte[] wrongBytes = Base64.getDecoder().decode(commitID.toString());
             wrongBytes[0] = (byte) ~wrongBytes[0];
             Engine.CommitId wrongId = new Engine.CommitId(wrongBytes);
             assertEquals("should fail to sync flush with wrong id (but no docs)", engine.syncFlush(syncId + "1", wrongId),
diff --git a/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java b/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
index ec9e72639d2..9ee9e0841cf 100644
--- a/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
+++ b/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
@@ -31,7 +31,6 @@ import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -318,7 +317,7 @@ public class SearchScrollIT extends ESIntegTestCase {
     public void testClearNonExistentScrollId() throws Exception {
         createIndex("idx");
         ClearScrollResponse response = client().prepareClearScroll()
-                .addScrollId("cXVlcnlUaGVuRmV0Y2g7MzsyOlpBRC1qOUhrUjhhZ0NtQWUxU2FuWlE7MjpRcjRaNEJ2R1JZV1VEMW02ZGF1LW5ROzI6S0xUal9lZDRTd3lWNUhUU2VSb01CQTswOw==")
+                .addScrollId("DnF1ZXJ5VGhlbkZldGNoAwAAAAAAAAABFnRtLWMyRzBqUUQyNk1uM0xDTjJ4S0EAAAAAAAAAARYzNkhxbWFTYVFVNmgxTGQyYUZVYV9nAAAAAAAAAAEWdVcxNWZmRGZSVFN2V0xMUGF2NGx1Zw==")
                 .get();
         // Whether we actually clear a scroll, we can't know, since that information isn't serialized in the
         // free search context response, which is returned from each node we want to clear a particular scroll.
@@ -330,24 +329,19 @@ public class SearchScrollIT extends ESIntegTestCase {
 
     public void testClearIllegalScrollId() throws Exception {
         createIndex("idx");
-        try {
-            client().prepareClearScroll().addScrollId("c2Nhbjs2OzM0NDg1ODpzRlBLc0FXNlNyNm5JWUc1").get();
-            fail();
-        } catch (IllegalArgumentException e) {
-        }
-        try {
-            // Fails during base64 decoding (Base64-encoded string must have at least four characters)
-            client().prepareClearScroll().addScrollId("a").get();
-            fail();
-        } catch (IllegalArgumentException e) {
-        }
-        try {
-            client().prepareClearScroll().addScrollId("abcabc").get();
-            fail();
-            // if running without -ea this will also throw ElasticsearchIllegalArgumentException
-        } catch (UncategorizedExecutionException e) {
-            assertThat(e.getRootCause(), instanceOf(AssertionError.class));
-        }
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> client().prepareClearScroll().addScrollId("c2Nhbjs2OzM0NDg1ODpzRlBLc0FXNlNyNm5JWUc1").get());
+        assertEquals("Cannot parse scroll id", e.getMessage());
+
+        e = expectThrows(IllegalArgumentException.class,
+                // Fails during base64 decoding (Base64-encoded string must have at least four characters)
+                () ->  client().prepareClearScroll().addScrollId("a").get());
+        assertEquals("Cannot parse scroll id", e.getMessage());
+
+        e = expectThrows(IllegalArgumentException.class,
+                // Other invalid base64
+                () ->  client().prepareClearScroll().addScrollId("abcabc").get());
+        assertEquals("Cannot parse scroll id", e.getMessage());
     }
 
     public void testSimpleScrollQueryThenFetchClearAllScrollIds() throws Exception {
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
index 00710dc2b4d..8cecbf2c787 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
@@ -22,7 +22,6 @@ package org.elasticsearch.messy.tests;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.common.Base64;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -45,8 +44,10 @@ import org.elasticsearch.test.ESIntegTestCase;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Base64;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
@@ -413,7 +414,7 @@ public class SearchFieldsTests extends ESIntegTestCase {
                 .field("double_field", 6.0d)
                 .field("date_field", Joda.forPattern("dateOptionalTime").printer().print(new DateTime(2012, 3, 22, 0, 0, DateTimeZone.UTC)))
                 .field("boolean_field", true)
-                .field("binary_field", Base64.encodeBytes("testing text".getBytes("UTF8")))
+                .field("binary_field", Base64.getEncoder().encodeToString("testing text".getBytes("UTF-8")))
                 .endObject()).execute().actionGet();
 
         client().admin().indices().prepareRefresh().execute().actionGet();
diff --git a/plugins/delete-by-query/src/test/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryActionTests.java b/plugins/delete-by-query/src/test/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryActionTests.java
index 980ee76c2ce..3867c8be494 100644
--- a/plugins/delete-by-query/src/test/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryActionTests.java
+++ b/plugins/delete-by-query/src/test/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryActionTests.java
@@ -99,7 +99,7 @@ public class TransportDeleteByQueryActionTests extends ESSingleNodeTestCase {
         newAsyncAction(delete, listener).executeScroll("123");
         waitForCompletion("scroll request should fail on malformed scroll id", listener);
 
-        assertFailure(listener, "Failed to decode scrollId");
+        assertFailure(listener, "Cannot parse scroll id");
         assertSearchContextsClosed();
     }
 
diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java
index 945c5d34790..56e72be351e 100644
--- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java
+++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java
@@ -21,7 +21,6 @@ package org.elasticsearch.ingest.attachment;
 
 import org.apache.commons.io.IOUtils;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.Base64;
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.ingest.core.IngestDocument;
 import org.elasticsearch.test.ESTestCase;
@@ -30,6 +29,7 @@ import org.junit.Before;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
+import java.util.Base64;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -209,7 +209,7 @@ public class AttachmentProcessorTests extends ESTestCase {
         String path = "/org/elasticsearch/ingest/attachment/test/sample-files/" + filename;
         try (InputStream is = AttachmentProcessorTests.class.getResourceAsStream(path)) {
             byte bytes[] = IOUtils.toByteArray(is);
-            return Base64.encodeBytes(bytes);
+            return Base64.getEncoder().encodeToString(bytes);
         }
     }
 }
diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MultifieldAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MultifieldAttachmentMapperTests.java
index f701c3282cc..a5f87f57ac0 100644
--- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MultifieldAttachmentMapperTests.java
+++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MultifieldAttachmentMapperTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.mapper.attachments;
 
-import org.elasticsearch.common.Base64;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -33,6 +32,7 @@ import org.elasticsearch.index.mapper.core.TextFieldMapper;
 import org.junit.Before;
 
 import java.nio.charset.StandardCharsets;
+import java.util.Base64;
 
 import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
 import static org.hamcrest.Matchers.instanceOf;
@@ -84,7 +84,7 @@ public class MultifieldAttachmentMapperTests extends AttachmentUnitTestCase {
         String originalText = "This is an elasticsearch mapper attachment test.";
         String forcedName = "dummyname.txt";
 
-        String bytes = Base64.encodeBytes(originalText.getBytes(StandardCharsets.ISO_8859_1));
+        String bytes = Base64.getEncoder().encodeToString(originalText.getBytes(StandardCharsets.ISO_8859_1));
 
         MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper());
 
@@ -150,7 +150,7 @@ public class MultifieldAttachmentMapperTests extends AttachmentUnitTestCase {
         String forcedLanguage = randomAsciiOfLength(20);
         String forcedContentType = randomAsciiOfLength(20);
 
-        String bytes = Base64.encodeBytes(originalText.getBytes(StandardCharsets.ISO_8859_1));
+        String bytes = Base64.getEncoder().encodeToString(originalText.getBytes(StandardCharsets.ISO_8859_1));
 
         MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(),
             Settings.builder().put(AttachmentMapper.INDEX_ATTACHMENT_DETECT_LANGUAGE_SETTING.getKey(), true).build(),

From d2afe759a709cc9bd96860efca04f414ee277a29 Mon Sep 17 00:00:00 2001
From: Luca Cavanna <javanna@users.noreply.github.com>
Date: Mon, 23 May 2016 12:17:42 +0200
Subject: [PATCH 0420/1311] prevent registration of duplicated rest spec
 (#18504)

Rather than having one win against the other, reject duplicated apis. Also enforce the convention that see the api name have the same name as the name of the rest spec file that defines it.
---
 .../test/rest/parser/RestTestSuiteParser.java |  15 +-
 .../elasticsearch/test/rest/spec/RestApi.java |   8 +-
 .../test/rest/spec/RestApiParser.java         | 206 +++++++++---------
 .../test/rest/spec/RestSpec.java              |  20 +-
 .../rest/test/RestApiParserFailingTests.java  |   4 +-
 .../test/rest/test/RestApiParserTests.java    |   6 +-
 6 files changed, 134 insertions(+), 125 deletions(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java
index fa9c5cf099a..d3f93939c2e 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java
@@ -18,6 +18,11 @@
  */
 package org.elasticsearch.test.rest.parser;
 
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+import org.elasticsearch.test.rest.section.RestTestSuite;
+import org.elasticsearch.test.rest.section.TestSection;
+
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
@@ -25,11 +30,6 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.yaml.YamlXContent;
-import org.elasticsearch.test.rest.section.RestTestSuite;
-import org.elasticsearch.test.rest.section.TestSection;
-
 /**
  * Parser for a complete test suite (yaml file)
  */
@@ -57,14 +57,11 @@ public class RestTestSuiteParser implements RestTestFragmentParser<RestTestSuite
             }
         }
 
-        XContentParser parser = YamlXContent.yamlXContent.createParser(Files.newInputStream(file));
-        try {
+        try (XContentParser parser = YamlXContent.yamlXContent.createParser(Files.newInputStream(file))) {
             RestTestSuiteParseContext testParseContext = new RestTestSuiteParseContext(api, filename, parser);
             return parse(testParseContext);
         } catch(Exception e) {
             throw new RestTestParseException("Error parsing " + api + "/" + filename, e);
-        } finally {
-            parser.close();
         }
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/spec/RestApi.java b/test/framework/src/main/java/org/elasticsearch/test/rest/spec/RestApi.java
index 60c39b64411..86bd806eedf 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/spec/RestApi.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/spec/RestApi.java
@@ -32,6 +32,7 @@ import java.util.Set;
  */
 public class RestApi {
 
+    private final String location;
     private final String name;
     private List<String> methods = new ArrayList<>();
     private List<String> paths = new ArrayList<>();
@@ -43,7 +44,8 @@ public class RestApi {
         NOT_SUPPORTED, OPTIONAL, REQUIRED
     }
 
-    RestApi(String name) {
+    RestApi(String location, String name) {
+        this.location = location;
         this.name = name;
     }
 
@@ -51,6 +53,10 @@ public class RestApi {
         return name;
     }
 
+    public String getLocation() {
+        return location;
+    }
+
     public List<String> getMethods() {
         return methods;
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/spec/RestApiParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/spec/RestApiParser.java
index 0328e4c87d8..95fe132471a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/spec/RestApiParser.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/spec/RestApiParser.java
@@ -27,113 +27,107 @@ import java.io.IOException;
  */
 public class RestApiParser {
 
-    public RestApi parse(XContentParser parser) throws IOException {
+    public RestApi parse(String location, XContentParser parser) throws IOException {
 
-        try {
-            while ( parser.nextToken() != XContentParser.Token.FIELD_NAME ) {
-                //move to first field name
-            }
-
-            RestApi restApi = new RestApi(parser.currentName());
-
-            int level = -1;
-            while (parser.nextToken() != XContentParser.Token.END_OBJECT || level >= 0) {
-
-                if (parser.currentToken() == XContentParser.Token.FIELD_NAME) {
-                    if ("methods".equals(parser.currentName())) {
-                        parser.nextToken();
-                        while (parser.nextToken() == XContentParser.Token.VALUE_STRING) {
-                            restApi.addMethod(parser.text());
-                        }
-                    }
-
-                    if ("url".equals(parser.currentName())) {
-                        String currentFieldName = "url";
-                        int innerLevel = -1;
-                        while(parser.nextToken() != XContentParser.Token.END_OBJECT || innerLevel >= 0) {
-                            if (parser.currentToken() == XContentParser.Token.FIELD_NAME) {
-                                currentFieldName = parser.currentName();
-                            }
-
-                            if (parser.currentToken() == XContentParser.Token.START_ARRAY && "paths".equals(currentFieldName)) {
-                                while (parser.nextToken() == XContentParser.Token.VALUE_STRING) {
-                                    restApi.addPath(parser.text());
-                                }
-                            }
-
-                            if (parser.currentToken() == XContentParser.Token.START_OBJECT && "parts".equals(currentFieldName)) {
-                                while (parser.nextToken() == XContentParser.Token.FIELD_NAME) {
-                                    restApi.addPathPart(parser.currentName());
-                                    parser.nextToken();
-                                    if (parser.currentToken() != XContentParser.Token.START_OBJECT) {
-                                        throw new IOException("Expected parts field in rest api definition to contain an object");
-                                    }
-                                    parser.skipChildren();
-                                }
-                            }
-
-                            if (parser.currentToken() == XContentParser.Token.START_OBJECT && "params".equals(currentFieldName)) {
-                                while (parser.nextToken() == XContentParser.Token.FIELD_NAME) {
-                                    restApi.addParam(parser.currentName());
-                                    parser.nextToken();
-                                    if (parser.currentToken() != XContentParser.Token.START_OBJECT) {
-                                        throw new IOException("Expected params field in rest api definition to contain an object");
-                                    }
-                                    parser.skipChildren();
-                                }
-                            }
-
-                            if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
-                                innerLevel++;
-                            }
-                            if (parser.currentToken() == XContentParser.Token.END_OBJECT) {
-                                innerLevel--;
-                            }
-                        }
-                    }
-
-                    if ("body".equals(parser.currentName())) {
-                        parser.nextToken();
-                        if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
-                            boolean requiredFound = false;
-                            while(parser.nextToken() != XContentParser.Token.END_OBJECT) {
-                                if (parser.currentToken() == XContentParser.Token.FIELD_NAME) {
-                                    if ("required".equals(parser.currentName())) {
-                                        requiredFound = true;
-                                        parser.nextToken();
-                                        if (parser.booleanValue()) {
-                                            restApi.setBodyRequired();
-                                        } else {
-                                            restApi.setBodyOptional();
-                                        }
-                                    }
-                                }
-                            }
-                            if (!requiredFound) {
-                                restApi.setBodyOptional();
-                            }
-                        }
-                    }
-                }
-
-                if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
-                    level++;
-                }
-                if (parser.currentToken() == XContentParser.Token.END_OBJECT) {
-                    level--;
-                }
-
-            }
-
-            parser.nextToken();
-            assert parser.currentToken() == XContentParser.Token.END_OBJECT : "Expected [END_OBJECT] but was ["  + parser.currentToken() +"]";
-            parser.nextToken();
-
-            return restApi;
-
-        } finally {
-            parser.close();
+        while ( parser.nextToken() != XContentParser.Token.FIELD_NAME ) {
+            //move to first field name
         }
-    }
 
+        RestApi restApi = new RestApi(location, parser.currentName());
+
+        int level = -1;
+        while (parser.nextToken() != XContentParser.Token.END_OBJECT || level >= 0) {
+
+            if (parser.currentToken() == XContentParser.Token.FIELD_NAME) {
+                if ("methods".equals(parser.currentName())) {
+                    parser.nextToken();
+                    while (parser.nextToken() == XContentParser.Token.VALUE_STRING) {
+                        restApi.addMethod(parser.text());
+                    }
+                }
+
+                if ("url".equals(parser.currentName())) {
+                    String currentFieldName = "url";
+                    int innerLevel = -1;
+                    while(parser.nextToken() != XContentParser.Token.END_OBJECT || innerLevel >= 0) {
+                        if (parser.currentToken() == XContentParser.Token.FIELD_NAME) {
+                            currentFieldName = parser.currentName();
+                        }
+
+                        if (parser.currentToken() == XContentParser.Token.START_ARRAY && "paths".equals(currentFieldName)) {
+                            while (parser.nextToken() == XContentParser.Token.VALUE_STRING) {
+                                restApi.addPath(parser.text());
+                            }
+                        }
+
+                        if (parser.currentToken() == XContentParser.Token.START_OBJECT && "parts".equals(currentFieldName)) {
+                            while (parser.nextToken() == XContentParser.Token.FIELD_NAME) {
+                                restApi.addPathPart(parser.currentName());
+                                parser.nextToken();
+                                if (parser.currentToken() != XContentParser.Token.START_OBJECT) {
+                                    throw new IOException("Expected parts field in rest api definition to contain an object");
+                                }
+                                parser.skipChildren();
+                            }
+                        }
+
+                        if (parser.currentToken() == XContentParser.Token.START_OBJECT && "params".equals(currentFieldName)) {
+                            while (parser.nextToken() == XContentParser.Token.FIELD_NAME) {
+                                restApi.addParam(parser.currentName());
+                                parser.nextToken();
+                                if (parser.currentToken() != XContentParser.Token.START_OBJECT) {
+                                    throw new IOException("Expected params field in rest api definition to contain an object");
+                                }
+                                parser.skipChildren();
+                            }
+                        }
+
+                        if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
+                            innerLevel++;
+                        }
+                        if (parser.currentToken() == XContentParser.Token.END_OBJECT) {
+                            innerLevel--;
+                        }
+                    }
+                }
+
+                if ("body".equals(parser.currentName())) {
+                    parser.nextToken();
+                    if (parser.currentToken() != XContentParser.Token.VALUE_NULL) {
+                        boolean requiredFound = false;
+                        while(parser.nextToken() != XContentParser.Token.END_OBJECT) {
+                            if (parser.currentToken() == XContentParser.Token.FIELD_NAME) {
+                                if ("required".equals(parser.currentName())) {
+                                    requiredFound = true;
+                                    parser.nextToken();
+                                    if (parser.booleanValue()) {
+                                        restApi.setBodyRequired();
+                                    } else {
+                                        restApi.setBodyOptional();
+                                    }
+                                }
+                            }
+                        }
+                        if (!requiredFound) {
+                            restApi.setBodyOptional();
+                        }
+                    }
+                }
+            }
+
+            if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
+                level++;
+            }
+            if (parser.currentToken() == XContentParser.Token.END_OBJECT) {
+                level--;
+            }
+
+        }
+
+        parser.nextToken();
+        assert parser.currentToken() == XContentParser.Token.END_OBJECT : "Expected [END_OBJECT] but was ["  + parser.currentToken() +"]";
+        parser.nextToken();
+
+        return restApi;
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/spec/RestSpec.java b/test/framework/src/main/java/org/elasticsearch/test/rest/spec/RestSpec.java
index 2f154728b98..106ff5176c7 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/spec/RestSpec.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/spec/RestSpec.java
@@ -41,7 +41,11 @@ public class RestSpec {
     }
 
     void addApi(RestApi restApi) {
-        restApiMap.put(restApi.getName(), restApi);
+        RestApi previous = restApiMap.putIfAbsent(restApi.getName(), restApi);
+        if (previous != null) {
+            throw new IllegalArgumentException("cannot register api [" + restApi.getName() + "] found in [" + restApi.getLocation() + "]. "
+                    + "api with same name was already found in [" + previous.getLocation() + "]");
+        }
     }
 
     public RestApi getApi(String api) {
@@ -57,12 +61,20 @@ public class RestSpec {
      */
     public static RestSpec parseFrom(FileSystem fileSystem, String optionalPathPrefix, String... paths) throws IOException {
         RestSpec restSpec = new RestSpec();
+        RestApiParser restApiParser = new RestApiParser();
         for (String path : paths) {
             for (Path jsonFile : FileUtils.findJsonSpec(fileSystem, optionalPathPrefix, path)) {
                 try (InputStream stream = Files.newInputStream(jsonFile)) {
-                    XContentParser parser = JsonXContent.jsonXContent.createParser(stream);
-                    RestApi restApi = new RestApiParser().parse(parser);
-                    restSpec.addApi(restApi);
+                    try (XContentParser parser = JsonXContent.jsonXContent.createParser(stream)) {
+                        RestApi restApi = restApiParser.parse(jsonFile.toString(), parser);
+                        String filename = jsonFile.getFileName().toString();
+                        String expectedApiName = filename.substring(0, filename.lastIndexOf('.'));
+                        if (restApi.getName().equals(expectedApiName) == false) {
+                            throw new IllegalArgumentException("found api [" + restApi.getName() + "] in [" + jsonFile.toString() + "]. " +
+                                    "Each api is expected to have the same name as the file that defines it.");
+                        }
+                        restSpec.addApi(restApi);
+                    }
                 } catch (Throwable ex) {
                     throw new IOException("Can't parse rest spec file: [" + jsonFile + "]", ex);
                 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/test/RestApiParserFailingTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/test/RestApiParserFailingTests.java
index e2f321c81c5..0cd8ee31398 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/test/RestApiParserFailingTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/test/RestApiParserFailingTests.java
@@ -28,7 +28,7 @@ import java.io.IOException;
 import static org.hamcrest.Matchers.containsString;
 
 /**
- *
+ * These tests are not part of {@link RestApiParserTests} because the tested failures don't allow to consume the whole yaml stream
  */
 public class RestApiParserFailingTests extends ESTestCase {
     public void testBrokenSpecShouldThrowUsefulExceptionWhenParsingFailsOnParams() throws Exception {
@@ -42,7 +42,7 @@ public class RestApiParserFailingTests extends ESTestCase {
     private void parseAndExpectFailure(String brokenJson, String expectedErrorMessage) throws Exception {
         XContentParser parser = JsonXContent.jsonXContent.createParser(brokenJson);
         try {
-            new RestApiParser().parse(parser);
+            new RestApiParser().parse("location", parser);
             fail("Expected to fail parsing but did not happen");
         } catch (IOException e) {
             assertThat(e.getMessage(), containsString(expectedErrorMessage));
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/test/RestApiParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/test/RestApiParserTests.java
index 262b155c668..d884b327f71 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/test/RestApiParserTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/test/RestApiParserTests.java
@@ -29,7 +29,7 @@ import static org.hamcrest.Matchers.notNullValue;
 public class RestApiParserTests extends AbstractParserTestCase {
     public void testParseRestSpecIndexApi() throws Exception {
         parser = JsonXContent.jsonXContent.createParser(REST_SPEC_INDEX_API);
-        RestApi restApi = new RestApiParser().parse(parser);
+        RestApi restApi = new RestApiParser().parse("location", parser);
 
         assertThat(restApi, notNullValue());
         assertThat(restApi.getName(), equalTo("index"));
@@ -51,7 +51,7 @@ public class RestApiParserTests extends AbstractParserTestCase {
 
     public void testParseRestSpecGetTemplateApi() throws Exception {
         parser = JsonXContent.jsonXContent.createParser(REST_SPEC_GET_TEMPLATE_API);
-        RestApi restApi = new RestApiParser().parse(parser);
+        RestApi restApi = new RestApiParser().parse("location", parser);
         assertThat(restApi, notNullValue());
         assertThat(restApi.getName(), equalTo("indices.get_template"));
         assertThat(restApi.getMethods().size(), equalTo(1));
@@ -68,7 +68,7 @@ public class RestApiParserTests extends AbstractParserTestCase {
 
     public void testParseRestSpecCountApi() throws Exception {
         parser = JsonXContent.jsonXContent.createParser(REST_SPEC_COUNT_API);
-        RestApi restApi = new RestApiParser().parse(parser);
+        RestApi restApi = new RestApiParser().parse("location", parser);
         assertThat(restApi, notNullValue());
         assertThat(restApi.getName(), equalTo("count"));
         assertThat(restApi.getMethods().size(), equalTo(2));

From 8040ed0c16718bc063e1ba64cdc32b4c9d70f7b9 Mon Sep 17 00:00:00 2001
From: Lee Hinman <lee@writequit.org>
Date: Tue, 3 May 2016 11:32:15 -0600
Subject: [PATCH 0421/1311] Add whether the shard state fetch is pending to the
 allocation explain API

If the shard state fetch is still pending, this will now return a
message like:

```json
{
  "shard" : {
    "index" : "i",
    "index_uuid" : "de1W1374T4qgvUP4a9Ieaw",
    "id" : 0,
    "primary" : false
  },
  "assigned" : false,
  "shard_state_fetch_pending": true,
  "unassigned_info" : {
    "reason" : "INDEX_CREATED",
    "at" : "2016-04-26T16:34:53.227Z"
  },
  "allocation_delay_ms" : 0,
  "remaining_delay_ms" : 0,
  "nodes" : {
    "z-CbkiELT-SoWT91HIszLA" : {
      "node_name" : "Brain Cell",
      "node_attributes" : {
        "testattr" : "test"
      },
      "store" : {
        "shard_copy" : "NONE"
      },
      "final_decision" : "NO",
      "final_explanation" : "the shard state fetch is pending",
      "weight" : 5.0,
      "decisions" : [ ]
    }
  }
}
```

Adds the `shard_state_fetch_pending` field and uses the state to
influence the final decision and final explanation.

Relates to #17372
---
 .../ClusterAllocationExplanation.java         | 13 ++-
 ...ansportClusterAllocationExplainAction.java | 35 ++++++--
 .../gateway/GatewayAllocator.java             | 18 ++++
 .../ClusterAllocationExplainIT.java           |  1 +
 .../ClusterAllocationExplainTests.java        |  2 +
 .../ClusterAllocationExplanationTests.java    | 82 +++++++++----------
 .../cluster/allocation-explain.asciidoc       | 35 ++++----
 7 files changed, 120 insertions(+), 66 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
index dbabe681c7a..9ca0efd6c56 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
@@ -42,15 +42,18 @@ public final class ClusterAllocationExplanation implements ToXContent, Writeable
 
     private final ShardId shard;
     private final boolean primary;
+    private final boolean hasPendingAsyncFetch;
     private final String assignedNodeId;
     private final UnassignedInfo unassignedInfo;
     private final long remainingDelayMillis;
     private final Map<DiscoveryNode, NodeExplanation> nodeExplanations;
 
     public ClusterAllocationExplanation(ShardId shard, boolean primary, @Nullable String assignedNodeId, long remainingDelayMillis,
-                                        @Nullable UnassignedInfo unassignedInfo, Map<DiscoveryNode, NodeExplanation> nodeExplanations) {
+                                        @Nullable UnassignedInfo unassignedInfo, boolean hasPendingAsyncFetch,
+                                        Map<DiscoveryNode, NodeExplanation> nodeExplanations) {
         this.shard = shard;
         this.primary = primary;
+        this.hasPendingAsyncFetch = hasPendingAsyncFetch;
         this.assignedNodeId = assignedNodeId;
         this.unassignedInfo = unassignedInfo;
         this.remainingDelayMillis = remainingDelayMillis;
@@ -60,6 +63,7 @@ public final class ClusterAllocationExplanation implements ToXContent, Writeable
     public ClusterAllocationExplanation(StreamInput in) throws IOException {
         this.shard = ShardId.readShardId(in);
         this.primary = in.readBoolean();
+        this.hasPendingAsyncFetch = in.readBoolean();
         this.assignedNodeId = in.readOptionalString();
         this.unassignedInfo = in.readOptionalWriteable(UnassignedInfo::new);
         this.remainingDelayMillis = in.readVLong();
@@ -77,6 +81,7 @@ public final class ClusterAllocationExplanation implements ToXContent, Writeable
     public void writeTo(StreamOutput out) throws IOException {
         this.getShard().writeTo(out);
         out.writeBoolean(this.isPrimary());
+        out.writeBoolean(this.isStillFetchingShardData());
         out.writeOptionalString(this.getAssignedNodeId());
         out.writeOptionalWriteable(this.getUnassignedInfo());
         out.writeVLong(remainingDelayMillis);
@@ -97,6 +102,11 @@ public final class ClusterAllocationExplanation implements ToXContent, Writeable
         return this.primary;
     }
 
+    /** Return turn if shard data is still being fetched for the allocation */
+    public boolean isStillFetchingShardData() {
+        return this.hasPendingAsyncFetch;
+    }
+
     /** Return turn if the shard is assigned to a node */
     public boolean isAssigned() {
         return this.assignedNodeId != null;
@@ -138,6 +148,7 @@ public final class ClusterAllocationExplanation implements ToXContent, Writeable
             if (assignedNodeId != null) {
                 builder.field("assigned_node_id", this.assignedNodeId);
             }
+            builder.field("shard_state_fetch_pending", this.hasPendingAsyncFetch);
             // If we have unassigned info, show that
             if (unassignedInfo != null) {
                 unassignedInfo.toXContent(builder, params);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
index da97f492f7a..9dda8787372 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
@@ -50,6 +50,7 @@ import org.elasticsearch.common.collect.ImmutableOpenIntMap;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.gateway.GatewayAllocator;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -69,19 +70,22 @@ public class TransportClusterAllocationExplainAction
     private final AllocationDeciders allocationDeciders;
     private final ShardsAllocator shardAllocator;
     private final TransportIndicesShardStoresAction shardStoresAction;
+    private final GatewayAllocator gatewayAllocator;
 
     @Inject
     public TransportClusterAllocationExplainAction(Settings settings, TransportService transportService, ClusterService clusterService,
                                                    ThreadPool threadPool, ActionFilters actionFilters,
                                                    IndexNameExpressionResolver indexNameExpressionResolver,
                                                    ClusterInfoService clusterInfoService, AllocationDeciders allocationDeciders,
-                                                   ShardsAllocator shardAllocator, TransportIndicesShardStoresAction shardStoresAction) {
+                                                   ShardsAllocator shardAllocator, TransportIndicesShardStoresAction shardStoresAction,
+                                                   GatewayAllocator gatewayAllocator) {
         super(settings, ClusterAllocationExplainAction.NAME, transportService, clusterService, threadPool, actionFilters,
                 indexNameExpressionResolver, ClusterAllocationExplainRequest::new);
         this.clusterInfoService = clusterInfoService;
         this.allocationDeciders = allocationDeciders;
         this.shardAllocator = shardAllocator;
         this.shardStoresAction = shardStoresAction;
+        this.gatewayAllocator = gatewayAllocator;
     }
 
     @Override
@@ -130,7 +134,8 @@ public class TransportClusterAllocationExplainAction
                                                            Float nodeWeight,
                                                            IndicesShardStoresResponse.StoreStatus storeStatus,
                                                            String assignedNodeId,
-                                                           Set<String> activeAllocationIds) {
+                                                           Set<String> activeAllocationIds,
+                                                           boolean hasPendingAsyncFetch) {
         final ClusterAllocationExplanation.FinalDecision finalDecision;
         final ClusterAllocationExplanation.StoreCopy storeCopy;
         final String finalExplanation;
@@ -161,6 +166,19 @@ public class TransportClusterAllocationExplainAction
         if (node.getId().equals(assignedNodeId)) {
             finalDecision = ClusterAllocationExplanation.FinalDecision.ALREADY_ASSIGNED;
             finalExplanation = "the shard is already assigned to this node";
+        } else if (hasPendingAsyncFetch &&
+                shard.primary() == false &&
+                shard.unassigned() &&
+                shard.allocatedPostIndexCreate(indexMetaData) &&
+                nodeDecision.type() != Decision.Type.YES) {
+            finalExplanation = "the shard cannot be assigned because allocation deciders return a " + nodeDecision.type().name() +
+                    " decision and the shard's state is still being fetched";
+            finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
+        } else if (hasPendingAsyncFetch &&
+                shard.unassigned() &&
+                shard.allocatedPostIndexCreate(indexMetaData)) {
+            finalExplanation = "the shard's state is still being fetched so it cannot be allocated";
+            finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
         } else if (shard.primary() && shard.unassigned() && shard.allocatedPostIndexCreate(indexMetaData) &&
                 storeCopy == ClusterAllocationExplanation.StoreCopy.STALE) {
             finalExplanation = "the copy of the shard is stale, allocation ids do not match";
@@ -180,6 +198,7 @@ public class TransportClusterAllocationExplainAction
                 finalDecision = ClusterAllocationExplanation.FinalDecision.NO;
                 finalExplanation = "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision";
             } else {
+                // TODO: handle throttling decision better here
                 finalDecision = ClusterAllocationExplanation.FinalDecision.YES;
                 if (storeCopy == ClusterAllocationExplanation.StoreCopy.AVAILABLE) {
                     finalExplanation = "the shard can be assigned and the node contains a valid copy of the shard data";
@@ -198,7 +217,8 @@ public class TransportClusterAllocationExplainAction
      */
     public static ClusterAllocationExplanation explainShard(ShardRouting shard, RoutingAllocation allocation, RoutingNodes routingNodes,
                                                             boolean includeYesDecisions, ShardsAllocator shardAllocator,
-                                                            List<IndicesShardStoresResponse.StoreStatus> shardStores) {
+                                                            List<IndicesShardStoresResponse.StoreStatus> shardStores,
+                                                            GatewayAllocator gatewayAllocator) {
         // don't short circuit deciders, we want a full explanation
         allocation.debugDecision(true);
         // get the existing unassigned info if available
@@ -238,11 +258,12 @@ public class TransportClusterAllocationExplainAction
             Float weight = weights.get(node);
             IndicesShardStoresResponse.StoreStatus storeStatus = nodeToStatus.get(node);
             NodeExplanation nodeExplanation = calculateNodeExplanation(shard, indexMetaData, node, decision, weight,
-                    storeStatus, shard.currentNodeId(), indexMetaData.activeAllocationIds(shard.getId()));
+                    storeStatus, shard.currentNodeId(), indexMetaData.activeAllocationIds(shard.getId()),
+                    allocation.hasPendingAsyncFetch());
             explanations.put(node, nodeExplanation);
         }
-        return new ClusterAllocationExplanation(shard.shardId(), shard.primary(),
-                shard.currentNodeId(), remainingDelayMillis, ui, explanations);
+        return new ClusterAllocationExplanation(shard.shardId(), shard.primary(), shard.currentNodeId(),
+                remainingDelayMillis, ui, gatewayAllocator.hasFetchPending(shard.shardId(), shard.primary()), explanations);
     }
 
     @Override
@@ -297,7 +318,7 @@ public class TransportClusterAllocationExplainAction
                         shardStoreResponse.getStoreStatuses().get(shardRouting.getIndexName());
                 List<IndicesShardStoresResponse.StoreStatus> shardStoreStatus = shardStatuses.get(shardRouting.id());
                 ClusterAllocationExplanation cae = explainShard(shardRouting, allocation, routingNodes,
-                        request.includeYesDecisions(), shardAllocator, shardStoreStatus);
+                        request.includeYesDecisions(), shardAllocator, shardStoreStatus, gatewayAllocator);
                 listener.onResponse(new ClusterAllocationExplainResponse(cae));
             }
 
diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java b/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
index 15cd0e2bf63..f074a3ec09c 100644
--- a/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
+++ b/core/src/main/java/org/elasticsearch/gateway/GatewayAllocator.java
@@ -62,6 +62,24 @@ public class GatewayAllocator extends AbstractComponent {
         this.replicaShardAllocator = new InternalReplicaShardAllocator(settings, storeAction);
     }
 
+    /**
+     * Returns true if the given shard has an async fetch pending
+     */
+    public boolean hasFetchPending(ShardId shardId, boolean primary) {
+        if (primary) {
+            AsyncShardFetch<TransportNodesListGatewayStartedShards.NodeGatewayStartedShards> fetch = asyncFetchStarted.get(shardId);
+            if (fetch != null) {
+                return fetch.getNumberOfInFlightFetches() > 0;
+            }
+        } else {
+            AsyncShardFetch<TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> fetch = asyncFetchStore.get(shardId);
+            if (fetch != null) {
+                return fetch.getNumberOfInFlightFetches() > 0;
+            }
+        }
+        return false;
+    }
+
     public void setReallocation(final ClusterService clusterService, final RoutingService routingService) {
         this.routingService = routingService;
         clusterService.add(new ClusterStateListener() {
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
index 628e26bc4cf..9b19a34b2ff 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
@@ -120,6 +120,7 @@ public final class ClusterAllocationExplainIT extends ESIntegTestCase {
         assertThat(cae.getShard().getIndexName(), equalTo("only-foo"));
         assertFalse(cae.isPrimary());
         assertFalse(cae.isAssigned());
+        assertFalse(cae.isStillFetchingShardData());
         assertThat(UnassignedInfo.Reason.INDEX_CREATED, equalTo(cae.getUnassignedInfo().getReason()));
         assertThat("expecting no remaining delay: " + cae.getRemainingDelayMillis(), cae.getRemainingDelayMillis(), equalTo(0L));
 
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java
index db130238443..d5cefc6d1f3 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java
@@ -42,6 +42,7 @@ public final class ClusterAllocationExplainTests extends ESSingleNodeTestCase {
         assertEquals(0, cae.getShard().getId());
         assertEquals(false, cae.isPrimary());
         assertNull(cae.getAssignedNodeId());
+        assertFalse(cae.isStillFetchingShardData());
         assertNotNull(cae.getUnassignedInfo());
         NodeExplanation explanation = cae.getNodeExplanations().values().iterator().next();
         ClusterAllocationExplanation.FinalDecision fd = explanation.getFinalDecision();
@@ -68,6 +69,7 @@ public final class ClusterAllocationExplainTests extends ESSingleNodeTestCase {
         assertEquals("test", cae.getShard().getIndexName());
         assertEquals(0, cae.getShard().getId());
         assertEquals(true, cae.isPrimary());
+        assertFalse(cae.isStillFetchingShardData());
         assertNotNull("shard should have assigned node id", cae.getAssignedNodeId());
         assertNull("assigned shard should not have unassigned info", cae.getUnassignedInfo());
         explanation = cae.getNodeExplanations().values().iterator().next();
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
index 1f6df76c82d..511c547f233 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
@@ -81,26 +81,6 @@ public final class ClusterAllocationExplanationTests extends ESTestCase {
     }
 
 
-    private NodeExplanation makeNodeExplanation(boolean primary, boolean isAssigned, boolean hasErr, boolean hasActiveId) {
-        Float nodeWeight = randomFloat();
-        Exception e = hasErr ? new ElasticsearchException("stuff's broke, yo") : null;
-        IndicesShardStoresResponse.StoreStatus storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
-                IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, e);
-        String assignedNodeId;
-        if (isAssigned) {
-            assignedNodeId = "node-0";
-        } else {
-            assignedNodeId = "node-9";
-        }
-        Set<String> activeAllocationIds = new HashSet<>();
-        if (hasActiveId) {
-            activeAllocationIds.add("eggplant");
-        }
-
-        return TransportClusterAllocationExplainAction.calculateNodeExplanation(primary ? primaryShard : replicaShard,
-                indexMetaData, node, noDecision, nodeWeight, storeStatus, assignedNodeId, activeAllocationIds);
-    }
-
     private void assertExplanations(NodeExplanation ne, String finalExplanation, ClusterAllocationExplanation.FinalDecision finalDecision,
                                     ClusterAllocationExplanation.StoreCopy storeCopy) {
         assertEquals(finalExplanation, ne.getFinalExplanation());
@@ -117,71 +97,89 @@ public final class ClusterAllocationExplanationTests extends ESTestCase {
         ShardRouting primaryStartedShard = ShardRouting.newUnassigned(new ShardId(i, 0), null, true,
                 new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, "foo"));
         assertTrue(primaryStartedShard.allocatedPostIndexCreate(indexMetaData));
+        ShardRouting replicaStartedShard = ShardRouting.newUnassigned(new ShardId(i, 0), null, false,
+                new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, "foo"));
+        assertTrue(replicaStartedShard.allocatedPostIndexCreate(indexMetaData));
 
         IndicesShardStoresResponse.StoreStatus storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
                 IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, e);
         NodeExplanation ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node,
-                yesDecision, nodeWeight, storeStatus, "", activeAllocationIds);
+                yesDecision, nodeWeight, storeStatus, "", activeAllocationIds, false);
         assertExplanations(ne, "the copy of the shard cannot be read",
                 ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.IO_ERROR);
 
         ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node, yesDecision, nodeWeight,
-                null, "", activeAllocationIds);
+                null, "", activeAllocationIds, false);
         assertExplanations(ne, "the shard can be assigned",
                 ClusterAllocationExplanation.FinalDecision.YES, ClusterAllocationExplanation.StoreCopy.NONE);
 
         ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryStartedShard, indexMetaData, node, yesDecision,
-                nodeWeight, null, "", activeAllocationIds);
+                nodeWeight, null, "", activeAllocationIds, false);
         assertExplanations(ne, "there is no copy of the shard available",
                 ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.NONE);
 
         ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node, noDecision, nodeWeight,
-                null, "", activeAllocationIds);
+                null, "", activeAllocationIds, false);
         assertExplanations(ne, "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
                 ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.NONE);
 
         storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
                 IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null);
         ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node, noDecision, nodeWeight,
-                storeStatus, "", activeAllocationIds);
+                storeStatus, "", activeAllocationIds, false);
         assertExplanations(ne, "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
                 ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.AVAILABLE);
 
         storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
                 IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, corruptE);
         ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node, yesDecision, nodeWeight,
-                storeStatus, "", activeAllocationIds);
+                storeStatus, "", activeAllocationIds, false);
         assertExplanations(ne, "the copy of the shard is corrupt",
                 ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.CORRUPT);
 
         storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "banana",
                 IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null);
         ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node, yesDecision, nodeWeight,
-                storeStatus, "", activeAllocationIds);
+                storeStatus, "", activeAllocationIds, false);
         assertExplanations(ne, "the shard can be assigned",
                 ClusterAllocationExplanation.FinalDecision.YES, ClusterAllocationExplanation.StoreCopy.STALE);
 
         storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "banana",
                 IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null);
         ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryStartedShard, indexMetaData, node, yesDecision,
-                nodeWeight, storeStatus, "", activeAllocationIds);
+                nodeWeight, storeStatus, "", activeAllocationIds, false);
         assertExplanations(ne, "the copy of the shard is stale, allocation ids do not match",
                 ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.STALE);
 
         storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
                 IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null);
         ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node, yesDecision, nodeWeight,
-                storeStatus, "node-0", activeAllocationIds);
+                storeStatus, "node-0", activeAllocationIds, false);
         assertExplanations(ne, "the shard is already assigned to this node",
                 ClusterAllocationExplanation.FinalDecision.ALREADY_ASSIGNED, ClusterAllocationExplanation.StoreCopy.AVAILABLE);
 
         storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
                 IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null);
         ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node, yesDecision, nodeWeight,
-                storeStatus, "", activeAllocationIds);
+                storeStatus, "", activeAllocationIds, false);
         assertExplanations(ne, "the shard can be assigned and the node contains a valid copy of the shard data",
                 ClusterAllocationExplanation.FinalDecision.YES, ClusterAllocationExplanation.StoreCopy.AVAILABLE);
-}
+
+        storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
+                IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null);
+        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryStartedShard, indexMetaData, node, yesDecision,
+                nodeWeight, storeStatus, "", activeAllocationIds, true);
+        assertExplanations(ne, "the shard's state is still being fetched so it cannot be allocated",
+                ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.AVAILABLE);
+
+        storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
+                IndicesShardStoresResponse.StoreStatus.AllocationStatus.REPLICA, null);
+        ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(replicaStartedShard, indexMetaData, node, noDecision,
+                nodeWeight, storeStatus, "", activeAllocationIds, true);
+        assertExplanations(ne, "the shard cannot be assigned because allocation deciders return a NO " +
+                        "decision and the shard's state is still being fetched",
+                ClusterAllocationExplanation.FinalDecision.NO, ClusterAllocationExplanation.StoreCopy.AVAILABLE);
+    }
 
     public void testDecisionEquality() {
         Decision.Multi d = new Decision.Multi();
@@ -206,10 +204,10 @@ public final class ClusterAllocationExplanationTests extends ESTestCase {
         IndicesShardStoresResponse.StoreStatus storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
                 IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null);
         NodeExplanation ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node,
-                yesDecision, nodeWeight, storeStatus, "", activeAllocationIds);
+                yesDecision, nodeWeight, storeStatus, "", activeAllocationIds, false);
         nodeExplanations.put(ne.getNode(), ne);
         ClusterAllocationExplanation cae = new ClusterAllocationExplanation(shard, true,
-                "assignedNode", remainingDelay, null, nodeExplanations);
+                "assignedNode", remainingDelay, null, false, nodeExplanations);
         BytesStreamOutput out = new BytesStreamOutput();
         cae.writeTo(out);
         StreamInput in = StreamInput.wrap(out.bytes());
@@ -243,21 +241,21 @@ public final class ClusterAllocationExplanationTests extends ESTestCase {
         IndicesShardStoresResponse.StoreStatus storeStatus = new IndicesShardStoresResponse.StoreStatus(node, 42, "eggplant",
                 IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, new ElasticsearchException("stuff's broke, yo"));
         NodeExplanation ne = TransportClusterAllocationExplainAction.calculateNodeExplanation(primaryShard, indexMetaData, node,
-                d, nodeWeight, storeStatus, "node-0", allocationIds);
+                d, nodeWeight, storeStatus, "node-0", allocationIds, false);
         Map<DiscoveryNode, NodeExplanation> nodeExplanations = new HashMap<>(1);
         nodeExplanations.put(ne.getNode(), ne);
         ClusterAllocationExplanation cae = new ClusterAllocationExplanation(shardId, true,
-                "assignedNode", remainingDelay, null, nodeExplanations);
+                "assignedNode", remainingDelay, null, false, nodeExplanations);
         XContentBuilder builder = XContentFactory.jsonBuilder();
         cae.toXContent(builder, ToXContent.EMPTY_PARAMS);
         assertEquals("{\"shard\":{\"index\":\"foo\",\"index_uuid\":\"uuid\",\"id\":0,\"primary\":true},\"assigned\":true," +
-                        "\"assigned_node_id\":\"assignedNode\",\"nodes\":{\"node-0\":{\"node_name\":\"\",\"node_attribute" +
-                        "s\":{},\"store\":{\"shard_copy\":\"IO_ERROR\",\"store_exception\":\"ElasticsearchException[stuff" +
-                        "'s broke, yo]\"},\"final_decision\":\"ALREADY_ASSIGNED\",\"final_explanation\":\"the shard is al" +
-                        "ready assigned to this node\",\"weight\":1.5,\"decisions\":[{\"decider\":\"no label\",\"decision" +
-                        "\":\"NO\",\"explanation\":\"because I said no\"},{\"decider\":\"yes label\",\"decision\":\"YES\"" +
-                        ",\"explanation\":\"yes please\"},{\"decider\":\"throttle label\",\"decision\":\"THROTTLE\",\"exp" +
-                        "lanation\":\"wait a sec\"}]}}}",
+                        "\"assigned_node_id\":\"assignedNode\",\"shard_state_fetch_pending\":false,\"nodes\":{\"node-0\":" +
+                        "{\"node_name\":\"\",\"node_attributes\":{},\"store\":{\"shard_copy\":\"IO_ERROR\",\"store_except" +
+                        "ion\":\"ElasticsearchException[stuff's broke, yo]\"},\"final_decision\":\"ALREADY_ASSIGNED\",\"f" +
+                        "inal_explanation\":\"the shard is already assigned to this node\",\"weight\":1.5,\"decisions\":[" +
+                        "{\"decider\":\"no label\",\"decision\":\"NO\",\"explanation\":\"because I said no\"},{\"decider" +
+                        "\":\"yes label\",\"decision\":\"YES\",\"explanation\":\"yes please\"},{\"decider\":\"throttle la" +
+                        "bel\",\"decision\":\"THROTTLE\",\"explanation\":\"wait a sec\"}]}}}",
                 builder.string());
     }
 }
diff --git a/docs/reference/cluster/allocation-explain.asciidoc b/docs/reference/cluster/allocation-explain.asciidoc
index 7a8a3e4a8ce..3b07b40d9e7 100644
--- a/docs/reference/cluster/allocation-explain.asciidoc
+++ b/docs/reference/cluster/allocation-explain.asciidoc
@@ -30,25 +30,26 @@ The response looks like:
     "primary" : false
   },
   "assigned" : false,                             <1>
+  "shard_state_fetch_pending": false,             <2>
   "unassigned_info" : {
-    "reason" : "INDEX_CREATED",                   <2>
+    "reason" : "INDEX_CREATED",                   <3>
     "at" : "2016-03-22T20:04:23.620Z"
   },
-  "allocation_delay_ms" : 0,                      <3>
-  "remaining_delay_ms" : 0,                       <4>
+  "allocation_delay_ms" : 0,                      <4>
+  "remaining_delay_ms" : 0,                       <5>
   "nodes" : {
     "V-Spi0AyRZ6ZvKbaI3691w" : {
       "node_name" : "node1",
-      "node_attributes" : {                       <5>
+      "node_attributes" : {                       <6>
         "bar" : "baz"
       },
       "store" : {
-        "shard_copy" : "NONE"                     <6>
+        "shard_copy" : "NONE"                     <7>
       },
-      "final_decision" : "NO",                    <7>
+      "final_decision" : "NO",                    <8>
       "final_explanation" : "the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
-      "weight" : 0.06666675,                      <8>
-      "decisions" : [ {                           <9>
+      "weight" : 0.06666675,                      <9>
+      "decisions" : [ {                           <10>
         "decider" : "filter",
         "decision" : "NO",
         "explanation" : "node does not match index include filters [foo:\"bar\"]"
@@ -91,14 +92,15 @@ The response looks like:
 }
 --------------------------------------------------
 <1> Whether the shard is assigned or unassigned
-<2> Reason for the shard originally becoming unassigned
-<3> Configured delay before the shard can be allocated
-<4> Remaining delay before the shard can be allocated
-<5> User-added attributes the node has
-<6> The shard copy information for this node and error (if applicable)
-<7> Final decision and explanation of whether the shard can be allocated to this node
-<8> Weight for how much the allocator would like to allocate the shard to this node
-<9> List of node decisions factoring into final decision about the shard
+<2> Whether information about the shard is still being fetched
+<3> Reason for the shard originally becoming unassigned
+<4> Configured delay before the shard can be allocated
+<5> Remaining delay before the shard can be allocated
+<6> User-added attributes the node has
+<7> The shard copy information for this node and error (if applicable)
+<8> Final decision and explanation of whether the shard can be allocated to this node
+<9> Weight for how much the allocator would like to allocate the shard to this node
+<10> List of node decisions factoring into final decision about the shard
 
 For a shard that is already assigned, the output looks similar to:
 
@@ -113,6 +115,7 @@ For a shard that is already assigned, the output looks similar to:
   },
   "assigned" : true,
   "assigned_node_id" : "Qc6VL8c5RWaw1qXZ0Rg57g",      <1>
+  "shard_state_fetch_pending": false,
   "allocation_delay_ms" : 0,
   "remaining_delay_ms" : 0,
   "nodes" : {

From 555de4256e137f0829af810c53549e6f0a22b392 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 23 May 2016 14:26:38 -0400
Subject: [PATCH 0422/1311] Remove stale transport mode instructions from docs

This commit removes some stale instructions for setting the transport
mode via an environment variable from the testing docs.
---
 TESTING.asciidoc | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/TESTING.asciidoc b/TESTING.asciidoc
index af46c2e567b..50953f16378 100644
--- a/TESTING.asciidoc
+++ b/TESTING.asciidoc
@@ -32,12 +32,6 @@ Use local transport (default since 1.3):
 -Des.node.mode=local
 -------------------------------------
 
-Alternatively, you can set the `ES_TEST_LOCAL` environment variable:
-
--------------------------------------
-export ES_TEST_LOCAL=true && gradle test
--------------------------------------
-
 === Running Elasticsearch from a checkout
 
 In order to run Elasticsearch from source without building a package, you can

From b7817a630640215df04a78d395e3f4e0c5b58ad4 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 18 May 2016 16:33:09 -0400
Subject: [PATCH 0423/1311] [reindex] Retry the retry test if it didn't cause
 retries

The retry test has failed a couple of times in CI because it wasn't able
to cause any retries. Putting it in a bash `while` loop shows that it
eventually does fail that way. The seed "4F6477A9C999CA20" seems especially
good at failing to get retries. It doesn't fail all the time, but more
than most.

This adds a retry to each test case, retrying a maximum of 10 times or
until it causes the retries. I've seen it fail to get retries 7 times
in a row but not go beyond that. Retrying doesn't seem to really hurt
the test runtime all that much. Most of the time is in the startup
cost.

Failing CI build that triggered this:
https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+master+periodic/852/console
---
 .../index/reindex/RetryTests.java             | 64 +++++++++++++++----
 1 file changed, 53 insertions(+), 11 deletions(-)

diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
index ff821724316..dce0fb09ce7 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
@@ -26,6 +26,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.bulk.Retry;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.plugins.Plugin;
@@ -54,6 +55,10 @@ public class RetryTests extends ReindexTestCase {
      * Enough docs that the requests will likely step on each other.
      */
     private static final int DOC_COUNT = 200;
+    /**
+     * Maximum number of times to attempt the test case before bailing with a failure if we don't see both a bulk and a search retry.
+     */
+    private static final int ATTEMPTS = 10;
 
     /**
      * Lower the queue sizes to be small enough that both bulk and searches will time out and have to be retried.
@@ -85,19 +90,59 @@ public class RetryTests extends ReindexTestCase {
     }
 
     public void testReindex() throws Exception {
-        setupSourceIndex("source");
-        testCase(true, i -> reindex().source("source").destination("dest" + i));
+        testCase(true, () -> setupSourceIndex("source"), i -> reindex().source("source").destination("dest" + i));
     }
 
     public void testUpdateByQuery() throws Exception {
-        for (int i = 0; i < CONCURRENT; i++) {
-            setupSourceIndex("source" + i);
-        }
-        testCase(false, i -> updateByQuery().source("source" + i));
+        Runnable setup = () -> {
+            for (int i = 0; i < CONCURRENT; i++) {
+                setupSourceIndex("source" + i);
+            }
+        };
+        testCase(false, setup, i -> updateByQuery().source("source" + i));
     }
 
-    private void testCase(boolean expectCreated, IntFunction<AbstractBulkIndexByScrollRequestBuilder<?, ?>> requestBuilder)
+    /**
+     * Repeatedly attempts to cause bulk and search retries, failing if any of the requests fail and if after 5 attempts it isn't able to
+     * cause at least one of both types of retries across all attempts.
+     *
+     * @param expectCreated should the number of effected documents be created (true) or updated (false)
+     * @param setup called before every request attempt to create the test data
+     * @param requestBuilder called to build each parallel request.
+     */
+    private void testCase(boolean expectCreated, Runnable setup, IntFunction<AbstractBulkIndexByScrollRequestBuilder<?, ?>> requestBuilder)
             throws Exception {
+        long bulkRetries = 0;
+        long searchRetries = 0;
+        int attempt = 0;
+        while (attempt < ATTEMPTS) {
+            client().admin().indices().prepareDelete("*").get();
+            setup.run();
+            Tuple<Long, Long> result = attemptTestCase(expectCreated, requestBuilder);
+            bulkRetries += result.v1();
+            searchRetries += result.v2();
+            attempt += 1;
+            if (bulkRetries == 0) {
+                logger.warn("Didn't get any bulk retries after {} attempts", attempt);
+                continue;
+            }
+            if (searchRetries == 0) {
+                logger.warn("Didn't get any search retries after {} attempts", attempt);
+                continue;
+            }
+            break;
+        }
+        // We expect at least one retry or this test isn't very useful
+        assertThat(bulkRetries, greaterThan(0L));
+        assertThat(searchRetries, greaterThan(0L));
+    }
+
+    /**
+     * Attempts to cause retries for the requests provided by requestBuilder, failing if the requests fail. Returns a tuple of (number of
+     * bulk retries, number of search retries).
+     */
+    private Tuple<Long, Long> attemptTestCase(boolean expectCreated,
+            IntFunction<AbstractBulkIndexByScrollRequestBuilder<?, ?>> requestBuilder) throws Exception {
         List<ListenableActionFuture<BulkIndexByScrollResponse>> futures = new ArrayList<>(CONCURRENT);
         for (int i = 0; i < CONCURRENT; i++) {
             AbstractBulkIndexByScrollRequestBuilder<?, ?> request = requestBuilder.apply(i);
@@ -128,10 +173,7 @@ public class RetryTests extends ReindexTestCase {
             bulkRetries += response.getBulkRetries();
             searchRetries += response.getSearchRetries();
         }
-
-        // We expect at least one retry or this test isn't very useful
-        assertThat(bulkRetries, greaterThan(0L));
-        assertThat(searchRetries, greaterThan(0L));
+        return new Tuple<>(bulkRetries, searchRetries);
     }
 
     private void setupSourceIndex(String name) {

From 62ac719a94a6ec56b84109deea7773e65c83493f Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 20 May 2016 12:10:48 -0400
Subject: [PATCH 0424/1311] Rerwite RetryTests to very carefully block the
 executors

This reproduces every time. No more randomness! Hurray!
---
 .../AbstractAsyncBulkByScrollAction.java      |   5 +-
 .../index/reindex/ReindexTestCase.java        |   2 +-
 .../index/reindex/RetryTests.java             | 240 ++++++++----------
 3 files changed, 114 insertions(+), 133 deletions(-)

diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
index 699c6340095..0921abcb677 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
@@ -519,8 +519,9 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
             public void onFailure(Throwable e) {
                 if (ExceptionsHelper.unwrap(e, EsRejectedExecutionException.class) != null) {
                     if (retries.hasNext()) {
-                        logger.trace("retrying rejected search", e);
-                        threadPool.schedule(retries.next(), ThreadPool.Names.SAME, this);
+                        TimeValue delay = retries.next();
+                        logger.trace("retrying rejected search after [{}]", e, delay);
+                        threadPool.schedule(delay, ThreadPool.Names.SAME, this);
                         task.countSearchRetry();
                     } else {
                         logger.warn("giving up on search because we retried {} times without success", e, retries);
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java
index f4a777a1973..97d0c05083e 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexTestCase.java
@@ -51,7 +51,7 @@ public abstract class ReindexTestCase extends ESIntegTestCase {
         return RethrottleAction.INSTANCE.newRequestBuilder(client());
     }
 
-    protected static BulkIndexByScrollResponseMatcher matcher() {
+    public static BulkIndexByScrollResponseMatcher matcher() {
         return new BulkIndexByScrollResponseMatcher();
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
index dce0fb09ce7..21b935ef530 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
@@ -21,179 +21,159 @@ package org.elasticsearch.index.reindex;
 
 import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
 import org.elasticsearch.action.ListenableActionFuture;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.action.bulk.BackoffPolicy;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.bulk.Retry;
-import org.elasticsearch.cluster.health.ClusterHealthStatus;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.search.MockSearchService;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.junit.After;
+import org.junit.Before;
 
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
-import java.util.function.IntFunction;
+import java.util.concurrent.CyclicBarrier;
 
-import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.index.reindex.ReindexTestCase.matcher;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.hasSize;
 
 /**
  * Integration test for retry behavior. Useful because retrying relies on the way that the rest of Elasticsearch throws exceptions and unit
  * tests won't verify that.
  */
-@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/18456")
-public class RetryTests extends ReindexTestCase {
-    /**
-     * The number of concurrent requests to test.
-     */
-    private static final int CONCURRENT = 12;
-    /**
-     * Enough docs that the requests will likely step on each other.
-     */
-    private static final int DOC_COUNT = 200;
-    /**
-     * Maximum number of times to attempt the test case before bailing with a failure if we don't see both a bulk and a search retry.
-     */
-    private static final int ATTEMPTS = 10;
+public class RetryTests extends ESSingleNodeTestCase {
+    private static final int DOC_COUNT = 20;
+
+    private List<CyclicBarrier> blockedExecutors = new ArrayList<>();
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(ReindexPlugin.class);
+    }
 
     /**
      * Lower the queue sizes to be small enough that both bulk and searches will time out and have to be retried.
      */
     @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        Settings.Builder settings = Settings.builder().put(super.nodeSettings(nodeOrdinal));
-        settings.put("threadpool.bulk.queue_size", 1);
+    protected Settings nodeSettings() {
+        Settings.Builder settings = Settings.builder().put(super.nodeSettings());
+        // Use pools of size 1 so we can block them
         settings.put("threadpool.bulk.size", 1);
-        settings.put("threadpool.search.queue_size", 1);
         settings.put("threadpool.search.size", 1);
+        // Use queues of size 1 because size 0 is broken and because search requests need the queue to function
+        settings.put("threadpool.bulk.queue_size", 1);
+        settings.put("threadpool.search.queue_size", 1);
         return settings.build();
     }
 
-    /**
-     * Disable search context leak detection because we expect leaks when there is an {@link EsRejectedExecutionException} queueing the
-     * reduce phase.
-     */
-    @Override
-    protected Collection<Class<? extends Plugin>> getMockPlugins() {
-        List<Class<? extends Plugin>> mockPlugins = new ArrayList<>();
-        for (Class<? extends Plugin> plugin: super.getMockPlugins()) {
-            if (plugin.equals(MockSearchService.TestPlugin.class)) {
-                continue;
-            }
-            mockPlugins.add(plugin);
+    @Before
+    public void setupSourceIndex() throws Exception {
+        createIndex("source");
+        // Build the test data. Don't use indexRandom because that won't work consistently with such small thread pools.
+        BulkRequestBuilder bulk = client().prepareBulk();
+        for (int i = 0; i < DOC_COUNT; i++) {
+            bulk.add(client().prepareIndex("source", "test").setSource("foo", "bar " + i));
+        }
+        Retry retry = Retry.on(EsRejectedExecutionException.class).policy(BackoffPolicy.exponentialBackoff());
+        BulkResponse response = retry.withSyncBackoff(client(), bulk.request());
+        assertFalse(response.buildFailureMessage(), response.hasFailures());
+        client().admin().indices().prepareRefresh("source").get();
+    }
+
+    @After
+    public void forceUnblockAllExecutors() {
+        for (CyclicBarrier barrier: blockedExecutors) {
+            barrier.reset();
         }
-        return mockPlugins;
     }
 
     public void testReindex() throws Exception {
-        testCase(true, () -> setupSourceIndex("source"), i -> reindex().source("source").destination("dest" + i));
+        testCase(ReindexAction.NAME, ReindexAction.INSTANCE.newRequestBuilder(client()).source("source").destination("dest"),
+                matcher().created(DOC_COUNT));
     }
 
     public void testUpdateByQuery() throws Exception {
-        Runnable setup = () -> {
-            for (int i = 0; i < CONCURRENT; i++) {
-                setupSourceIndex("source" + i);
-            }
-        };
-        testCase(false, setup, i -> updateByQuery().source("source" + i));
+        testCase(UpdateByQueryAction.NAME, UpdateByQueryAction.INSTANCE.newRequestBuilder(client()).source("source"),
+                matcher().updated(DOC_COUNT));
     }
 
-    /**
-     * Repeatedly attempts to cause bulk and search retries, failing if any of the requests fail and if after 5 attempts it isn't able to
-     * cause at least one of both types of retries across all attempts.
-     *
-     * @param expectCreated should the number of effected documents be created (true) or updated (false)
-     * @param setup called before every request attempt to create the test data
-     * @param requestBuilder called to build each parallel request.
-     */
-    private void testCase(boolean expectCreated, Runnable setup, IntFunction<AbstractBulkIndexByScrollRequestBuilder<?, ?>> requestBuilder)
+    private void testCase(String action, AbstractBulkIndexByScrollRequestBuilder<?, ?> request, BulkIndexByScrollResponseMatcher matcher)
             throws Exception {
-        long bulkRetries = 0;
-        long searchRetries = 0;
-        int attempt = 0;
-        while (attempt < ATTEMPTS) {
-            client().admin().indices().prepareDelete("*").get();
-            setup.run();
-            Tuple<Long, Long> result = attemptTestCase(expectCreated, requestBuilder);
-            bulkRetries += result.v1();
-            searchRetries += result.v2();
-            attempt += 1;
-            if (bulkRetries == 0) {
-                logger.warn("Didn't get any bulk retries after {} attempts", attempt);
-                continue;
-            }
-            if (searchRetries == 0) {
-                logger.warn("Didn't get any search retries after {} attempts", attempt);
-                continue;
-            }
-            break;
-        }
-        // We expect at least one retry or this test isn't very useful
-        assertThat(bulkRetries, greaterThan(0L));
-        assertThat(searchRetries, greaterThan(0L));
+        logger.info("Blocking search");
+        CyclicBarrier initialSearchBlock = blockExecutor(ThreadPool.Names.SEARCH);
+
+        // Make sure we use more than one batch so we have to scroll
+        request.source().setSize(DOC_COUNT / randomIntBetween(2, 10));
+
+        logger.info("Starting request");
+        ListenableActionFuture<BulkIndexByScrollResponse> responseListener = request.execute();
+
+        logger.info("Waiting for search rejections on the initial search");
+        assertBusy(() -> assertThat(taskStatus(action).getSearchRetries(), greaterThan(0L)));
+
+        logger.info("Blocking bulk and unblocking search so we start to get bulk rejections");
+        CyclicBarrier bulkBlock = blockExecutor(ThreadPool.Names.BULK);
+        initialSearchBlock.await();
+
+        logger.info("Waiting for bulk rejections");
+        assertBusy(() -> assertThat(taskStatus(action).getBulkRetries(), greaterThan(0L)));
+
+        // Keep a copy of the current number of search rejections so we can assert that we get more when we block the scroll
+        long initialSearchRejections = taskStatus(action).getSearchRetries();
+
+        logger.info("Blocking search and unblocking bulk so we should get search rejections for the scroll");
+        CyclicBarrier scrollBlock = blockExecutor(ThreadPool.Names.SEARCH);
+        bulkBlock.await();
+
+        logger.info("Waiting for search rejections for the scroll");
+        assertBusy(() -> assertThat(taskStatus(action).getSearchRetries(), greaterThan(initialSearchRejections)));
+
+        logger.info("Unblocking the scroll");
+        scrollBlock.await();
+
+        logger.info("Waiting for the request to finish");
+        BulkIndexByScrollResponse response = responseListener.get();
+        assertThat(response, matcher);
+        assertThat(response.getBulkRetries(), greaterThan(0L));
+        assertThat(response.getSearchRetries(), greaterThan(initialSearchRejections));
     }
 
     /**
-     * Attempts to cause retries for the requests provided by requestBuilder, failing if the requests fail. Returns a tuple of (number of
-     * bulk retries, number of search retries).
+     * Blocks the named executor by getting its only thread running a task blocked on a CyclicBarrier and fills the queue with a noop task.
+     * So requests to use this queue should get {@link EsRejectedExecutionException}s.
      */
-    private Tuple<Long, Long> attemptTestCase(boolean expectCreated,
-            IntFunction<AbstractBulkIndexByScrollRequestBuilder<?, ?>> requestBuilder) throws Exception {
-        List<ListenableActionFuture<BulkIndexByScrollResponse>> futures = new ArrayList<>(CONCURRENT);
-        for (int i = 0; i < CONCURRENT; i++) {
-            AbstractBulkIndexByScrollRequestBuilder<?, ?> request = requestBuilder.apply(i);
-            // Make sure we use more than one batch so we get the full reindex behavior
-            request.source().setSize(DOC_COUNT / randomIntBetween(2, 10));
-            // Use a low, random initial wait so we are unlikely collide with others retrying.
-            request.setRetryBackoffInitialTime(timeValueMillis(randomIntBetween(10, 300)));
-            futures.add(request.execute());
-        }
-
-        // Finish all the requests
-        List<BulkIndexByScrollResponse> responses = new ArrayList<>(CONCURRENT);
-        for (ListenableActionFuture<BulkIndexByScrollResponse> future : futures) {
-            responses.add(future.get());
-        }
-
-        // Now check them
-        long bulkRetries = 0;
-        long searchRetries = 0;
-        BulkIndexByScrollResponseMatcher matcher = matcher();
-        if (expectCreated) {
-            matcher.created(DOC_COUNT);
-        } else {
-            matcher.updated(DOC_COUNT);
-        }
-        for (BulkIndexByScrollResponse response : responses) {
-            assertThat(response, matcher);
-            bulkRetries += response.getBulkRetries();
-            searchRetries += response.getSearchRetries();
-        }
-        return new Tuple<>(bulkRetries, searchRetries);
+    private CyclicBarrier blockExecutor(String name) throws Exception {
+        ThreadPool threadPool = getInstanceFromNode(ThreadPool.class);
+        CyclicBarrier barrier = new CyclicBarrier(2);
+        logger.info("Blocking the [{}] executor", name);
+        threadPool.executor(name).execute(() -> {
+            try {
+                threadPool.executor(name).execute(() -> {});
+                barrier.await();
+                logger.info("Blocked the [{}] executor", name);
+                barrier.await();
+                logger.info("Ublocking the [{}] executor", name);
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        });
+        barrier.await();
+        blockedExecutors.add(barrier);
+        return barrier;
     }
 
-    private void setupSourceIndex(String name) {
-        try {
-            // Build the test index with a single shard so we can be sure that a search request *can* complete with the one thread
-            assertAcked(client().admin().indices().prepareCreate(name).setSettings(
-                    "index.number_of_shards", 1,
-                    "index.number_of_replicas", 0).get());
-            waitForRelocation(ClusterHealthStatus.GREEN);
-            // Build the test data. Don't use indexRandom because that won't work consistently with such small thread pools.
-            BulkRequestBuilder bulk = client().prepareBulk();
-            for (int i = 0; i < DOC_COUNT; i++) {
-                bulk.add(client().prepareIndex(name, "test").setSource("foo", "bar " + i));
-            }
-            Retry retry = Retry.on(EsRejectedExecutionException.class).policy(BackoffPolicy.exponentialBackoff());
-            BulkResponse response = retry.withSyncBackoff(client(), bulk.request());
-            assertFalse(response.buildFailureMessage(), response.hasFailures());
-            refresh(name);
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
+    /**
+     * Fetch the status for a task of type "action". Fails if there aren't exactly one of that type of task running.
+     */
+    private BulkByScrollTask.Status taskStatus(String action) {
+        ListTasksResponse response = client().admin().cluster().prepareListTasks().setActions(action).setDetailed(true).get();
+        assertThat(response.getTasks(), hasSize(1));
+        return (BulkByScrollTask.Status) response.getTasks().get(0).getStatus();
     }
 }

From f63d1255d16ecba1a5fb8839fc93be61a9212d93 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 23 May 2016 14:47:22 -0400
Subject: [PATCH 0425/1311] Cleanup settings and system properties entanglement

This commit cleans up some additional places where system properties
were being used to pass settings to Elasticsearch.

Relates #18524
---
 dev-tools/create_bwc_index.py                      | 14 +++++++-------
 .../java/org/elasticsearch/test/ExternalNode.java  | 10 +++++-----
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/dev-tools/create_bwc_index.py b/dev-tools/create_bwc_index.py
index 361934908ec..f73bc58554b 100644
--- a/dev-tools/create_bwc_index.py
+++ b/dev-tools/create_bwc_index.py
@@ -137,13 +137,13 @@ def start_node(version, release_dir, data_dir, repo_dir, tcp_port=DEFAULT_TRANSP
     
   cmd = [
     os.path.join(release_dir, 'bin/elasticsearch'),
-    '-Des.path.data=%s' % data_dir,
-    '-Des.path.logs=logs',
-    '-Des.cluster.name=%s' % cluster_name,
-    '-Des.network.host=localhost',
-    '-Des.transport.tcp.port=%s' % tcp_port,
-    '-Des.http.port=%s' % http_port,
-    '-Des.path.repo=%s' % repo_dir
+    '-Epath.data=%s' % data_dir,
+    '-Epath.logs=logs',
+    '-Ecluster.name=%s' % cluster_name,
+    '-Enetwork.host=localhost',
+    '-Etransport.tcp.port=%s' % tcp_port,
+    '-Ehttp.port=%s' % http_port,
+    '-Epath.repo=%s' % repo_dir
   ]
   if version.startswith('0.') or version.startswith('1.0.0.Beta') :
     cmd.append('-f') # version before 1.0 start in background automatically
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java b/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java
index efdd6bad90f..c08c2793eaa 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java
@@ -99,8 +99,8 @@ final class ExternalNode implements Closeable {
         } else {
             params.add("bin/elasticsearch.bat");
         }
-        params.add("-Des.cluster.name=" + clusterName);
-        params.add("-Des.node.name=" + nodeName);
+        params.add("-Ecluster.name=" + clusterName);
+        params.add("-Enode.name=" + nodeName);
         Settings.Builder externaNodeSettingsBuilder = Settings.builder();
         for (Map.Entry<String, String> entry : settings.getAsMap().entrySet()) {
             switch (entry.getKey()) {
@@ -121,11 +121,11 @@ final class ExternalNode implements Closeable {
         }
         this.externalNodeSettings = externaNodeSettingsBuilder.put(REQUIRED_SETTINGS).build();
         for (Map.Entry<String, String> entry : externalNodeSettings.getAsMap().entrySet()) {
-            params.add("-Des." + entry.getKey() + "=" + entry.getValue());
+            params.add("-E" + entry.getKey() + "=" + entry.getValue());
         }
 
-        params.add("-Des.path.home=" + PathUtils.get(".").toAbsolutePath());
-        params.add("-Des.path.conf=" + path + "/config");
+        params.add("-Epath.home=" + PathUtils.get(".").toAbsolutePath());
+        params.add("-Epath.conf=" + path + "/config");
 
         ProcessBuilder builder = new ProcessBuilder(params);
         builder.directory(path.toFile());

From 2d4db1c88d94df9753d438767f94964568d10fa6 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 23 May 2016 14:55:25 -0400
Subject: [PATCH 0426/1311] Remove travis configuration file

This commit removes the Travis configuration file as it's out-of-date.

Relates #18525
---
 .travis.yml | 10 ----------
 1 file changed, 10 deletions(-)
 delete mode 100644 .travis.yml

diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 1a4e5b58f33..00000000000
--- a/.travis.yml
+++ /dev/null
@@ -1,10 +0,0 @@
-language: java
-jdk:
-  - openjdk7
-
-env:
-  - ES_TEST_LOCAL=true
-  - ES_TEST_LOCAL=false
-
-notifications:
-  email: false

From ceff71fa9832051b6aa9b633271e6005d9b655a6 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Mon, 23 May 2016 12:28:54 -0700
Subject: [PATCH 0427/1311] Build: Use -release with java 9 instead of
 -source/-target

This uses the new -release flag for javac which does not require setting
the bootclasspath as -source/-target did.
---
 build.gradle                                        |  8 ++++++++
 .../org/elasticsearch/gradle/BuildPlugin.groovy     | 13 ++++++++-----
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/build.gradle b/build.gradle
index c843069f43e..bf4b5a3b415 100644
--- a/build.gradle
+++ b/build.gradle
@@ -144,6 +144,14 @@ subprojects {
       // see https://discuss.gradle.org/t/add-custom-javadoc-option-that-does-not-take-an-argument/5959
       javadoc.options.encoding='UTF8'
       javadoc.options.addStringOption('Xdoclint:all,-missing', '-quiet')
+      /*
+      TODO: building javadocs with java 9 b118 is currently broken with weird errors, so
+      for now this is commented out...try again with the next ea build...
+      javadoc.executable = new File(project.javaHome, 'bin/javadoc')
+      if (project.javaVersion == JavaVersion.VERSION_1_9) {
+        // TODO: remove this hack! gradle should be passing this...
+        javadoc.options.addStringOption('source', '8')
+      }*/
     }
   }
 
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index 455f16d123a..168ceca1d23 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -143,7 +143,7 @@ class BuildPlugin implements Plugin<Project> {
             }
 
             project.rootProject.ext.javaHome = javaHome
-            project.rootProject.ext.javaVersion = javaVersion
+            project.rootProject.ext.javaVersion = javaVersionEnum
             project.rootProject.ext.buildChecksDone = true
         }
         project.targetCompatibility = minimumJava
@@ -387,10 +387,13 @@ class BuildPlugin implements Plugin<Project> {
                 options.encoding = 'UTF-8'
                 //options.incremental = true
 
-                // gradle ignores target/source compatibility when it is "unnecessary", but since to compile with
-                // java 9, gradle is running in java 8, it incorrectly thinks it is unnecessary
-                assert minimumJava == JavaVersion.VERSION_1_8
-                options.compilerArgs << '-target' << '1.8' << '-source' << '1.8'
+                if (project.javaVersion == JavaVersion.VERSION_1_9) {
+                    // hack until gradle supports java 9's new "-release" arg
+                    assert minimumJava == JavaVersion.VERSION_1_8
+                    options.compilerArgs << '-release' << '8'
+                    project.sourceCompatibility = null
+                    project.targetCompatibility = null
+                }
             }
         }
     }

From ef435d00994e85c2ea5d65558a50024a7cbcf6de Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Mon, 23 May 2016 13:44:59 -0700
Subject: [PATCH 0428/1311] Remove unnecessary boxing and use of deprecated
 Double ctors

---
 .../elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java | 2 +-
 .../index/query/functionscore/FunctionScoreTests.java           | 2 +-
 .../search/aggregations/bucket/DateScriptMocks.java             | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java
index 7afbecad4d3..ea22461243c 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java
@@ -340,7 +340,7 @@ public class GeoDistanceRangeQueryBuilder extends AbstractQueryBuilder<GeoDistan
                 fromValue = geoDistance.normalize(fromValue, DistanceUnit.DEFAULT);
             }
         } else {
-            fromValue = new Double(0);
+            fromValue = 0.0;
         }
 
         if (to != null) {
diff --git a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java
index 51cdf5581bc..ba80aca31e9 100644
--- a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreTests.java
@@ -243,7 +243,7 @@ public class FunctionScoreTests extends ESTestCase {
 
     private static final ScoreFunction RANDOM_SCORE_FUNCTION = new RandomScoreFunction(0, 0, new IndexFieldDataStub());
     private static final ScoreFunction FIELD_VALUE_FACTOR_FUNCTION = new FieldValueFactorFunction("test", 1,
-            FieldValueFactorFunction.Modifier.LN, new Double(1), null);
+            FieldValueFactorFunction.Modifier.LN, 1.0, null);
     private static final ScoreFunction GAUSS_DECAY_FUNCTION = new DecayFunctionBuilder.NumericFieldDataScoreFunction(0, 1, 0.1, 0,
             GaussDecayFunctionBuilder.GAUSS_DECAY_FUNCTION, new IndexNumericFieldDataStub(), MultiValueMode.MAX);
     private static final ScoreFunction EXP_DECAY_FUNCTION = new DecayFunctionBuilder.NumericFieldDataScoreFunction(0, 1, 0.1, 0,
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateScriptMocks.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateScriptMocks.java
index 50e95605f27..f97b163c24c 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateScriptMocks.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateScriptMocks.java
@@ -122,7 +122,7 @@ public class DateScriptMocks {
 
         @Override
         public double runAsDouble() {
-            return new DateTime(new Double((double) vars.get("_value")).longValue(), DateTimeZone.UTC).plusMonths(1).getMillis();
+            return new DateTime(Double.valueOf((double) vars.get("_value")).longValue(), DateTimeZone.UTC).plusMonths(1).getMillis();
         }
 
         @Override

From c7b45b2cc7b9f115a9a942d68693c8b411cf578a Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Mon, 23 May 2016 16:17:43 -0700
Subject: [PATCH 0429/1311] Tests: Remove unnecessary Callable variant of
 assertBusy

The assertBusy method currently has both a Runnable and Callable
version. This has caused confusion with type inference and lambdas
sometimes, in particular with java 9. This change removes the callable
version as nothing was actually using it.
---
 .../index/reindex/CancelTests.java            |  2 +-
 .../org/elasticsearch/test/ESTestCase.java    | 20 +++++--------------
 2 files changed, 6 insertions(+), 16 deletions(-)

diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
index 5a778d0b0fa..e2cc5b43f0a 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
@@ -137,7 +137,7 @@ public class CancelTests extends ReindexTestCase {
         ALLOWED_OPERATIONS.release(BLOCKING_OPERATIONS);
 
         // Checks that no more operations are executed
-        assertBusy(() -> ALLOWED_OPERATIONS.availablePermits() == 0 && ALLOWED_OPERATIONS.getQueueLength() == 0);
+        assertBusy(() -> assertTrue(ALLOWED_OPERATIONS.availablePermits() == 0 && ALLOWED_OPERATIONS.getQueueLength() == 0));
 
         // And check the status of the response
         BulkIndexByScrollResponse response = future.get();
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 67d93c6887a..6ed9a746f46 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -443,24 +443,13 @@ public abstract class ESTestCase extends LuceneTestCase {
      * Runs the code block for 10 seconds waiting for no assertion to trip.
      */
     public static void assertBusy(Runnable codeBlock) throws Exception {
-        assertBusy(Executors.callable(codeBlock), 10, TimeUnit.SECONDS);
-    }
-
-    public static void assertBusy(Runnable codeBlock, long maxWaitTime, TimeUnit unit) throws Exception {
-        assertBusy(Executors.callable(codeBlock), maxWaitTime, unit);
-    }
-
-    /**
-     * Runs the code block for 10 seconds waiting for no assertion to trip.
-     */
-    public static <V> V assertBusy(Callable<V> codeBlock) throws Exception {
-        return assertBusy(codeBlock, 10, TimeUnit.SECONDS);
+        assertBusy(codeBlock, 10, TimeUnit.SECONDS);
     }
 
     /**
      * Runs the code block for the provided interval, waiting for no assertions to trip.
      */
-    public static <V> V assertBusy(Callable<V> codeBlock, long maxWaitTime, TimeUnit unit) throws Exception {
+    public static void assertBusy(Runnable codeBlock, long maxWaitTime, TimeUnit unit) throws Exception {
         long maxTimeInMillis = TimeUnit.MILLISECONDS.convert(maxWaitTime, unit);
         long iterations = Math.max(Math.round(Math.log10(maxTimeInMillis) / Math.log10(2)), 1);
         long timeInMillis = 1;
@@ -468,7 +457,8 @@ public abstract class ESTestCase extends LuceneTestCase {
         List<AssertionError> failures = new ArrayList<>();
         for (int i = 0; i < iterations; i++) {
             try {
-                return codeBlock.call();
+                codeBlock.run();
+                return;
             } catch (AssertionError e) {
                 failures.add(e);
             }
@@ -479,7 +469,7 @@ public abstract class ESTestCase extends LuceneTestCase {
         timeInMillis = maxTimeInMillis - sum;
         Thread.sleep(Math.max(timeInMillis, 0));
         try {
-            return codeBlock.call();
+            codeBlock.run();
         } catch (AssertionError e) {
             for (AssertionError failure : failures) {
                 e.addSuppressed(failure);

From a56a9cb8186e4c2d9dd19a17d79e7b98fbde2ccd Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 23 May 2016 19:35:28 -0400
Subject: [PATCH 0430/1311] improve painless whitelist coverage of java api

---
 .../elasticsearch/painless/Definition.java    |  256 ++--
 .../elasticsearch/painless/node/LCall.java    |    2 +-
 .../elasticsearch/painless/node/LField.java   |   14 +-
 .../org/elasticsearch/painless/definition.txt |  375 ------
 .../org/elasticsearch/painless/java.lang.txt  | 1079 ++++++++++++++++
 .../org/elasticsearch/painless/java.math.txt  |  149 +++
 .../org/elasticsearch/painless/java.text.txt  |  486 +++++++
 .../painless/java.util.function.txt           |  232 ++++
 .../painless/java.util.stream.txt             |  273 ++++
 .../org/elasticsearch/painless/java.util.txt  | 1148 +++++++++++++++++
 .../painless/org.elasticsearch.txt            |  127 ++
 .../elasticsearch/painless/BasicAPITests.java |    7 +
 12 files changed, 3620 insertions(+), 528 deletions(-)
 delete mode 100644 modules/lang-painless/src/main/resources/org/elasticsearch/painless/definition.txt
 create mode 100644 modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
 create mode 100644 modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.math.txt
 create mode 100644 modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.text.txt
 create mode 100644 modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.function.txt
 create mode 100644 modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.stream.txt
 create mode 100644 modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt
 create mode 100644 modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 05c12c30239..db9330174cc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -38,7 +38,14 @@ import java.util.Objects;
  */
 public final class Definition {
     
-    private static final String DEFINITION_FILE = "definition.txt";
+    private static final List<String> DEFINITION_FILES = Collections.unmodifiableList(
+        Arrays.asList("org.elasticsearch.txt", 
+                      "java.lang.txt",
+                      "java.math.txt",
+                      "java.text.txt",
+                      "java.util.txt",
+                      "java.util.function.txt",
+                      "java.util.stream.txt"));
 
     private static final Definition INSTANCE = new Definition();
     
@@ -158,15 +165,13 @@ public final class Definition {
         public final Struct owner;
         public final List<Type> arguments;
         public final org.objectweb.asm.commons.Method method;
-        public final java.lang.reflect.Constructor<?> reflect;
 
-        private Constructor(final String name, final Struct owner, final List<Type> arguments,
-                            final org.objectweb.asm.commons.Method method, final java.lang.reflect.Constructor<?> reflect) {
+        private Constructor(String name, Struct owner, List<Type> arguments,
+                            org.objectweb.asm.commons.Method method) {
             this.name = name;
             this.owner = owner;
             this.arguments = Collections.unmodifiableList(arguments);
             this.method = method;
-            this.reflect = reflect;
         }
     }
 
@@ -176,18 +181,17 @@ public final class Definition {
         public final Type rtn;
         public final List<Type> arguments;
         public final org.objectweb.asm.commons.Method method;
-        public final java.lang.reflect.Method reflect;
+        public final int modifiers;
         public final MethodHandle handle;
 
-        private Method(final String name, final Struct owner, final Type rtn, final List<Type> arguments,
-                       final org.objectweb.asm.commons.Method method, final java.lang.reflect.Method reflect,
-                       final MethodHandle handle) {
+        private Method(String name, Struct owner, Type rtn, List<Type> arguments,
+                       org.objectweb.asm.commons.Method method, int modifiers, MethodHandle handle) {
             this.name = name;
             this.owner = owner;
             this.rtn = rtn;
             this.arguments = Collections.unmodifiableList(arguments);
             this.method = method;
-            this.reflect = reflect;
+            this.modifiers = modifiers;
             this.handle = handle;
         }
     }
@@ -196,16 +200,17 @@ public final class Definition {
         public final String name;
         public final Struct owner;
         public final Type type;
-        public final java.lang.reflect.Field reflect;
-        public final MethodHandle getter;
-        public final MethodHandle setter;
+        public final String javaName;
+        public final int modifiers;
+        private final MethodHandle getter;
+        private final MethodHandle setter;
 
-        private Field(final String name, final Struct owner, final Type type,
-                      final java.lang.reflect.Field reflect, final MethodHandle getter, final MethodHandle setter) {
+        private Field(String name, String javaName, Struct owner, Type type, int modifiers, MethodHandle getter, MethodHandle setter) {
             this.name = name;
+            this.javaName = javaName;
             this.owner = owner;
             this.type = type;
-            this.reflect = reflect;
+            this.modifiers = modifiers;
             this.getter = getter;
             this.setter = setter;
         }
@@ -420,102 +425,106 @@ public final class Definition {
     /** adds classes from definition. returns hierarchy */
     private Map<String,List<String>> addStructs() {
         final Map<String,List<String>> hierarchy = new HashMap<>();
-        int currentLine = -1;
-        try {
-            try (InputStream stream = Definition.class.getResourceAsStream(DEFINITION_FILE);
-                    LineNumberReader reader = new LineNumberReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
-                String line = null;
-                while ((line = reader.readLine()) != null) {
-                    currentLine = reader.getLineNumber();
-                    line = line.trim();
-                    if (line.length() == 0 || line.charAt(0) == '#') {
-                        continue;
-                    }
-                    if (line.startsWith("class ")) {
-                        String elements[] = line.split("\u0020");
-                        assert elements[2].equals("->");
-                        if (elements.length == 7) {
-                            hierarchy.put(elements[1], Arrays.asList(elements[5].split(",")));
-                        } else {
-                            assert elements.length == 5;
+        for (String file : DEFINITION_FILES) {
+            int currentLine = -1;
+            try {
+                try (InputStream stream = Definition.class.getResourceAsStream(file);
+                        LineNumberReader reader = new LineNumberReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
+                    String line = null;
+                    while ((line = reader.readLine()) != null) {
+                        currentLine = reader.getLineNumber();
+                        line = line.trim();
+                        if (line.length() == 0 || line.charAt(0) == '#') {
+                            continue;
                         }
-                        String className = elements[1];
-                        String javaPeer = elements[3];
-                        final Class<?> javaClazz;
-                        switch (javaPeer) {
-                            case "void":
-                                javaClazz = void.class;
-                                break;
-                            case "boolean":
-                                javaClazz = boolean.class;
-                                break;
-                            case "byte":
-                                javaClazz = byte.class;
-                                break;
-                            case "short":
-                                javaClazz = short.class;
-                                break;
-                            case "char":
-                                javaClazz = char.class;
-                                break;
-                            case "int":
-                                javaClazz = int.class;
-                                break;
-                            case "long":
-                                javaClazz = long.class;
-                                break;
-                            case "float":
-                                javaClazz = float.class;
-                                break;
-                            case "double":
-                                javaClazz = double.class;
-                                break;
-                            default:
-                                javaClazz = Class.forName(javaPeer);
-                                break;
+                        if (line.startsWith("class ")) {
+                            String elements[] = line.split("\u0020");
+                            assert elements[2].equals("->");
+                            if (elements.length == 7) {
+                                hierarchy.put(elements[1], Arrays.asList(elements[5].split(",")));
+                            } else {
+                                assert elements.length == 5;
+                            }
+                            String className = elements[1];
+                            String javaPeer = elements[3];
+                            final Class<?> javaClazz;
+                            switch (javaPeer) {
+                                case "void":
+                                    javaClazz = void.class;
+                                    break;
+                                case "boolean":
+                                    javaClazz = boolean.class;
+                                    break;
+                                case "byte":
+                                    javaClazz = byte.class;
+                                    break;
+                                case "short":
+                                    javaClazz = short.class;
+                                    break;
+                                case "char":
+                                    javaClazz = char.class;
+                                    break;
+                                case "int":
+                                    javaClazz = int.class;
+                                    break;
+                                case "long":
+                                    javaClazz = long.class;
+                                    break;
+                                case "float":
+                                    javaClazz = float.class;
+                                    break;
+                                case "double":
+                                    javaClazz = double.class;
+                                    break;
+                                default:
+                                    javaClazz = Class.forName(javaPeer);
+                                    break;
+                            }
+                            addStruct(className, javaClazz);
                         }
-                        addStruct(className, javaClazz);
                     }
                 }
+            } catch (Exception e) {
+                throw new RuntimeException("syntax error in " + file + ", line: " + currentLine, e);
             }
-        } catch (Exception e) {
-            throw new RuntimeException("syntax error in definition line: " + currentLine, e);
         }
         return hierarchy;
     }
 
     /** adds class methods/fields/ctors */
     private void addElements() {
-        int currentLine = -1;
-        try {
-            try (InputStream stream = Definition.class.getResourceAsStream(DEFINITION_FILE);
-                 LineNumberReader reader = new LineNumberReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
-                String line = null;
-                String currentClass = null;
-                while ((line = reader.readLine()) != null) {
-                    currentLine = reader.getLineNumber();
-                    line = line.trim();
-                    if (line.length() == 0 || line.charAt(0) == '#') {
-                        continue;
-                    } else if (line.startsWith("class ")) {
-                        assert currentClass == null;
-                        currentClass = line.split("\u0020")[1];
-                    } else if (line.equals("}")) {
-                        assert currentClass != null;
-                        currentClass = null;
-                    } else {
-                        assert currentClass != null;
-                        addSignature(currentClass, line);
+        for (String file : DEFINITION_FILES) {
+            int currentLine = -1;
+            try {
+                try (InputStream stream = Definition.class.getResourceAsStream(file);
+                        LineNumberReader reader = new LineNumberReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
+                    String line = null;
+                    String currentClass = null;
+                    while ((line = reader.readLine()) != null) {
+                        currentLine = reader.getLineNumber();
+                        line = line.trim();
+                        if (line.length() == 0 || line.charAt(0) == '#') {
+                            continue;
+                        } else if (line.startsWith("class ")) {
+                            assert currentClass == null;
+                            currentClass = line.split("\u0020")[1];
+                        } else if (line.equals("}")) {
+                            assert currentClass != null;
+                            currentClass = null;
+                        } else {
+                            assert currentClass != null;
+                            addSignature(currentClass, line);
+                        }
                     }
                 }
+            } catch (Exception e) {
+                throw new RuntimeException("syntax error in " + file + ", line: " + currentLine, e);
             }
-        } catch (Exception e) {
-            throw new RuntimeException("syntax error in definition line: " + currentLine, e);
         }
     }
 
     private final void addStruct(final String name, final Class<?> clazz) {
-        if (!name.matches("^[_a-zA-Z][<>,_a-zA-Z0-9]*$")) {
+        if (!name.matches("^[_a-zA-Z][\\.,_a-zA-Z0-9]*$")) {
             throw new IllegalArgumentException("Invalid struct name [" + name + "].");
         }
 
@@ -575,7 +584,7 @@ public final class Definition {
         }
 
         final org.objectweb.asm.commons.Method asm = org.objectweb.asm.commons.Method.getMethod(reflect);
-        final Constructor constructor = new Constructor(name, owner, Arrays.asList(args), asm, reflect);
+        final Constructor constructor = new Constructor(name, owner, Arrays.asList(args), asm);
 
         owner.constructors.put(methodKey, constructor);
     }
@@ -697,8 +706,8 @@ public final class Definition {
                 " with arguments " + Arrays.toString(classes) + ".");
         }
 
-        final Method method = new Method(name, owner, rtn, Arrays.asList(args), asm, reflect, handle);
         final int modifiers = reflect.getModifiers();
+        final Method method = new Method(name, owner, rtn, Arrays.asList(args), asm, modifiers, handle);
 
         if (java.lang.reflect.Modifier.isStatic(modifiers)) {
             owner.staticMethods.put(methodKey, method);
@@ -751,7 +760,7 @@ public final class Definition {
                 " not found for class [" + owner.clazz.getName() + "].");
         }
 
-        final Field field = new Field(name, owner, type, reflect, getter, setter);
+        final Field field = new Field(name, reflect.getName(), owner, type, modifiers, getter, setter);
 
         if (isStatic) {
             // require that all static fields are static final
@@ -766,7 +775,7 @@ public final class Definition {
         }
     }
 
-    private final void copyStruct(final String struct, List<String> children) {
+    private void copyStruct(String struct, List<String> children) {
         final Struct owner = structsMap.get(struct);
 
         if (owner == null) {
@@ -776,7 +785,7 @@ public final class Definition {
         for (int count = 0; count < children.size(); ++count) {
             final Struct child = structsMap.get(children.get(count));
 
-            if (struct == null) {
+            if (child == null) {
                 throw new IllegalArgumentException("Child struct [" + children.get(count) + "]" +
                     " not defined for copy to owner struct [" + owner.name + "].");
             }
@@ -786,62 +795,19 @@ public final class Definition {
                     " is not a super type of owner struct [" + owner.name + "] in copy.");
             }
 
-            final boolean object = child.clazz.equals(Object.class) &&
-                java.lang.reflect.Modifier.isInterface(owner.clazz.getModifiers());
-
             for (Map.Entry<MethodKey,Method> kvPair : child.methods.entrySet()) {
                 MethodKey methodKey = kvPair.getKey();
                 Method method = kvPair.getValue();
                 if (owner.methods.get(methodKey) == null) {
-                    final Class<?> clazz = object ? Object.class : owner.clazz;
-
-                    java.lang.reflect.Method reflect;
-                    MethodHandle handle;
-
-                    try {
-                        reflect = clazz.getMethod(method.method.getName(), method.reflect.getParameterTypes());
-                    } catch (final NoSuchMethodException exception) {
-                        throw new IllegalArgumentException("Method [" + method.method.getName() + "] not found for" +
-                            " class [" + owner.clazz.getName() + "] with arguments " +
-                            Arrays.toString(method.reflect.getParameterTypes()) + ".");
-                    }
-
-                    try {
-                        handle = MethodHandles.publicLookup().in(owner.clazz).unreflect(reflect);
-                    } catch (final IllegalAccessException exception) {
-                        throw new IllegalArgumentException("Method [" + method.method.getName() + "] not found for" +
-                            " class [" + owner.clazz.getName() + "] with arguments " +
-                            Arrays.toString(method.reflect.getParameterTypes()) + ".");
-                    }
-
                     owner.methods.put(methodKey,
-                        new Method(method.name, owner, method.rtn, method.arguments, method.method, reflect, handle));
+                        new Method(method.name, owner, method.rtn, method.arguments, method.method, method.modifiers, method.handle));
                 }
             }
 
-            for (final Field field : child.members.values()) {
+            for (Field field : child.members.values()) {
                 if (owner.members.get(field.name) == null) {
-                    java.lang.reflect.Field reflect;
-                    MethodHandle getter;
-                    MethodHandle setter;
-
-                    try {
-                        reflect = owner.clazz.getField(field.reflect.getName());
-                    } catch (final NoSuchFieldException exception) {
-                        throw new IllegalArgumentException("Field [" + field.reflect.getName() + "]" +
-                            " not found for class [" + owner.clazz.getName() + "].");
-                    }
-
-                    try {
-                        getter = MethodHandles.publicLookup().unreflectGetter(reflect);
-                        setter = MethodHandles.publicLookup().unreflectSetter(reflect);
-                    } catch (final IllegalAccessException exception) {
-                        throw new IllegalArgumentException("Getter/Setter [" + field.name + "]" +
-                            " not found for class [" + owner.clazz.getName() + "].");
-                    }
-
                     owner.members.put(field.name,
-                        new Field(field.name, owner, field.type, reflect, getter, setter));
+                        new Field(field.name, field.javaName, owner, field.type, field.modifiers, field.getter, field.setter));
                 }
             }
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
index fdb6612e1f4..ea75215c5c8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
@@ -94,7 +94,7 @@ public final class LCall extends ALink {
             argument.write(adapter);
         }
 
-        if (java.lang.reflect.Modifier.isStatic(method.reflect.getModifiers())) {
+        if (java.lang.reflect.Modifier.isStatic(method.modifiers)) {
             adapter.invokeStatic(method.owner.type, method.method);
         } else if (java.lang.reflect.Modifier.isInterface(method.owner.clazz.getModifiers())) {
             adapter.invokeInterface(method.owner.type, method.method);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
index 44ba33acdda..0cf3b9a6e94 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
@@ -62,7 +62,7 @@ public final class LField extends ALink {
         field = statik ? struct.staticMembers.get(value) : struct.members.get(value);
 
         if (field != null) {
-            if (store && java.lang.reflect.Modifier.isFinal(field.reflect.getModifiers())) {
+            if (store && java.lang.reflect.Modifier.isFinal(field.modifiers)) {
                 throw new IllegalArgumentException(error(
                     "Cannot write to read-only field [" + value + "] for type [" + struct.name + "]."));
             }
@@ -104,19 +104,19 @@ public final class LField extends ALink {
 
     @Override
     void load(MethodWriter adapter) {
-        if (java.lang.reflect.Modifier.isStatic(field.reflect.getModifiers())) {
-            adapter.getStatic(field.owner.type, field.reflect.getName(), field.type.type);
+        if (java.lang.reflect.Modifier.isStatic(field.modifiers)) {
+            adapter.getStatic(field.owner.type, field.javaName, field.type.type);
         } else {
-            adapter.getField(field.owner.type, field.reflect.getName(), field.type.type);
+            adapter.getField(field.owner.type, field.javaName, field.type.type);
         }
     }
 
     @Override
     void store(MethodWriter adapter) {
-        if (java.lang.reflect.Modifier.isStatic(field.reflect.getModifiers())) {
-            adapter.putStatic(field.owner.type, field.reflect.getName(), field.type.type);
+        if (java.lang.reflect.Modifier.isStatic(field.modifiers)) {
+            adapter.putStatic(field.owner.type, field.javaName, field.type.type);
         } else {
-            adapter.putField(field.owner.type, field.reflect.getName(), field.type.type);
+            adapter.putField(field.owner.type, field.javaName, field.type.type);
         }
     }
 }
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/definition.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/definition.txt
deleted file mode 100644
index 379fadb9865..00000000000
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/definition.txt
+++ /dev/null
@@ -1,375 +0,0 @@
-#
-# Licensed to Elasticsearch under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-#
-# Painless definition file. This defines the hierarchy of classes,
-# what methods and fields they have, etc.
-#
-
-# primitive types
-
-class void -> void {
-}
-
-class boolean -> boolean {
-}
-
-class byte -> byte {
-}
-
-class short -> short {
-}
-
-class char -> char {
-}
-
-class int -> int {
-}
-
-class long -> long {
-}
-
-class float -> float {
-}
-
-class double -> double {
-}
-
-# basic JDK classes
-
-class Object -> java.lang.Object {
-  boolean equals(Object)
-  int hashCode()
-  String toString()
-}
-
-class def -> java.lang.Object {
-  boolean equals(Object)
-  int hashCode()
-  String toString()
-}
-
-class Void -> java.lang.Void extends Object {
-}
-
-class Boolean -> java.lang.Boolean extends Object {
-  Boolean TRUE
-  Boolean FALSE
-  int compare(boolean,boolean)
-  boolean parseBoolean(String)
-  Boolean valueOf(boolean)
-  boolean booleanValue()
-  int compareTo(Boolean)
-} 
-
-class Byte -> java.lang.Byte extends Number,Object {
-  byte MIN_VALUE
-  byte MAX_VALUE
-  int compare(byte,byte)
-  int compareTo(Byte)
-  byte parseByte(String)
-  Byte valueOf(byte)
-}
-
-class Short -> java.lang.Short extends Number,Object {
-  short MIN_VALUE
-  short MAX_VALUE
-  int compare(short,short)
-  int compareTo(Short)
-  short parseShort(String)
-  Short valueOf(short)
-}
-
-class Character -> java.lang.Character extends Object {
-  char MIN_VALUE
-  char MAX_VALUE
-  int charCount(int)
-  char charValue()
-  int compare(char,char)
-  int compareTo(Character)
-  int digit(int,int)
-  char forDigit(int,int)
-  String getName(int)
-  int getNumericValue(int)
-  boolean isAlphabetic(int)
-  boolean isDefined(int)
-  boolean isDigit(int)
-  boolean isIdeographic(int)
-  boolean isLetter(int)
-  boolean isLetterOrDigit(int)
-  boolean isLowerCase(int)
-  boolean isMirrored(int)
-  boolean isSpaceChar(int)
-  boolean isTitleCase(int)
-  boolean isUpperCase(int)
-  boolean isWhitespace(int)
-  Character valueOf(char)
-}
-
-class Integer -> java.lang.Integer extends Number,Object {
-  int MIN_VALUE
-  int MAX_VALUE
-  int compare(int,int)
-  int compareTo(Integer)
-  int min(int,int)
-  int max(int,int)
-  int parseInt(String)
-  int signum(int)
-  String toHexString(int)
-  Integer valueOf(int)
-}
-
-class Long -> java.lang.Long extends Number,Object {
-  long MIN_VALUE
-  long MAX_VALUE
-  int compare(long,long)
-  int compareTo(Long)
-  long min(long,long)
-  long max(long,long)
-  long parseLong(String)
-  int signum(long)
-  String toHexString(long)
-  Long valueOf(long)
-}
-
-class Float -> java.lang.Float extends Number,Object {
-  float MIN_VALUE
-  float MAX_VALUE
-  int compare(float,float)
-  int compareTo(Float)
-  float min(float,float)
-  float max(float,float)
-  float parseFloat(String)
-  String toHexString(float)
-  Float valueOf(float)
-}
-
-class Double -> java.lang.Double extends Number,Object {
-  double MIN_VALUE
-  double MAX_VALUE
-  int compare(double,double)
-  int compareTo(Double)
-  double min(double,double)
-  double max(double,double)
-  double parseDouble(String)
-  String toHexString(double)
-  Double valueOf(double)
-}
-
-class Number -> java.lang.Number extends Object {
-  byte byteValue()
-  short shortValue()
-  int intValue()
-  long longValue()
-  float floatValue()
-  double doubleValue()
-}
-
-class CharSequence -> java.lang.CharSequence extends Object {
-  char charAt(int)
-  int length()
-}
-
-class String -> java.lang.String extends CharSequence,Object {
-  String <init>()
-  int codePointAt(int)
-  int compareTo(String)
-  String concat(String)
-  boolean endsWith(String)
-  int indexOf(String)
-  int indexOf(String,int)
-  boolean isEmpty()
-  String replace(CharSequence,CharSequence)
-  boolean startsWith(String)
-  String substring(int,int)
-  char[] toCharArray()
-  String trim()
-}
-
-class Math -> java.lang.Math {
-  double E
-  double PI
-  double abs(double)
-  double acos(double)
-  double asin(double)
-  double atan(double)
-  double atan2(double,double)
-  double cbrt(double)
-  double ceil(double)
-  double cos(double)
-  double cosh(double)
-  double exp(double)
-  double expm1(double)
-  double floor(double)
-  double hypot(double,double)
-  double log(double)
-  double log10(double)
-  double log1p(double)
-  double max(double,double)
-  double min(double,double)
-  double pow(double,double)
-  double random()
-  double rint(double)
-  long round(double)
-  double sin(double)
-  double sinh(double)
-  double sqrt(double)
-  double tan(double)
-  double tanh(double)
-  double toDegrees(double)
-  double toRadians(double)
-}
-
-class Iterator -> java.util.Iterator extends Object {
-  boolean hasNext()
-  def next()
-  void remove()
-}
-
-class Collection -> java.util.Collection extends Object {
-  boolean add(def)
-  void clear()
-  boolean contains(def)
-  boolean isEmpty()
-  Iterator iterator()
-  boolean remove(def)
-  int size()
-}
-
-class List -> java.util.List extends Collection,Object {
-  def set(int,def)
-  def get(int)
-  def remove(int)
-  int getLength/size()
-}
-
-class ArrayList -> java.util.ArrayList extends List,Collection,Object {
-  ArrayList <init>()
-}
-
-class Set -> java.util.Set extends Collection,Object {
-}
-
-class HashSet -> java.util.HashSet extends Set,Collection,Object {
-  HashSet <init>()
-}
-
-class Map -> java.util.Map extends Object {
-  def put(def,def)
-  def get(def)
-  def remove(def)
-  boolean isEmpty()
-  int size()
-  boolean containsKey(def)
-  Set keySet()
-  Collection values()
-}
-
-class HashMap -> java.util.HashMap extends Map,Object {
-  HashMap <init>()
-}
-
-class Exception -> java.lang.Exception extends Object {
-  String getMessage()
-}
-
-class ArithmeticException -> java.lang.ArithmeticException extends Exception,Object {
-  ArithmeticException <init>()
-}
-
-class IllegalArgumentException -> java.lang.IllegalArgumentException extends Exception,Object {
-  IllegalArgumentException <init>()
-}
-
-class IllegalStateException -> java.lang.IllegalStateException extends Exception,Object {
-  IllegalStateException <init>()
-}
-
-class NumberFormatException -> java.lang.NumberFormatException extends Exception,Object {
-  NumberFormatException <init>()
-}
-
-# ES Scripting API
-
-class GeoPoint -> org.elasticsearch.common.geo.GeoPoint extends Object {
-  double getLat()
-  double getLon()
-}
-
-class Strings -> org.elasticsearch.index.fielddata.ScriptDocValues$Strings extends List,Collection,Object {
-  String getValue()
-  List getValues()
-}
-
-class Longs -> org.elasticsearch.index.fielddata.ScriptDocValues$Longs extends List,Collection,Object {
-  long getValue()
-  List getValues()
-}
-
-class Doubles -> org.elasticsearch.index.fielddata.ScriptDocValues$Doubles extends List,Collection,Object {
-  double getValue()
-  List getValues()
-}
-
-class GeoPoints -> org.elasticsearch.index.fielddata.ScriptDocValues$GeoPoints extends List,Collection,Object {
-  GeoPoint getValue()
-  List getValues()
-  double getLat()
-  double getLon()
-  double[] getLats()
-  double[] getLons()
-
-  # geo distance functions... so many...
-  double factorDistance(double,double)
-  double factorDistanceWithDefault(double,double,double)
-  double factorDistance02(double,double)
-  double factorDistance13(double,double)
-  double arcDistance(double,double)
-  double arcDistanceWithDefault(double,double,double)
-  double arcDistanceInKm(double,double)
-  double arcDistanceInKmWithDefault(double,double,double)
-  double arcDistanceInMiles(double,double)
-  double arcDistanceInMilesWithDefault(double,double,double)
-  double distance(double,double)
-  double distanceWithDefault(double,double,double)
-  double distanceInKm(double,double)
-  double distanceInKmWithDefault(double,double,double)
-  double distanceInMiles(double,double)
-  double distanceInMilesWithDefault(double,double,double)
-  double geohashDistance(String)
-  double geohashDistanceInKm(String)
-  double geohashDistanceInMiles(String)
-}
-
-# for testing.
-# currently FeatureTest exposes overloaded constructor, field load store, and overloaded static methods
-class FeatureTest -> org.elasticsearch.painless.FeatureTest extends Object {
-  FeatureTest <init>()
-  FeatureTest <init>(int,int)
-  int getX()
-  int getY()
-  void setX(int)
-  void setY(int)
-  boolean overloadedStatic()
-  boolean overloadedStatic(boolean)
-}
-
-# currently needed internally
-class Executable -> org.elasticsearch.painless.Executable {
-}
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
new file mode 100644
index 00000000000..01694b60db0
--- /dev/null
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
@@ -0,0 +1,1079 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#
+# Painless definition file. This defines the hierarchy of classes,
+# what methods and fields they have, etc.
+#
+
+#### Interfaces
+
+class Appendable -> java.lang.Appendable {
+  # append(char/CharSequence): skipped. left to subclasses (e.g. StringBuilder).
+  Appendable append(CharSequence,int,int)
+}
+
+# AutoCloseable: i/o
+
+class CharSequence -> java.lang.CharSequence {
+  char charAt(int)
+  IntStream chars()
+  IntStream codePoints()
+  int length()
+  CharSequence subSequence(int,int)
+  String toString()
+}
+
+# Cloneable: add clone() to subclasses directly.
+
+class Comparable -> java.lang.Comparable {
+  int compareTo(def)
+}
+
+class Iterable -> java.lang.Iterable {
+  void forEach(Consumer)
+  Iterator iterator()
+  Spliterator spliterator()
+}
+
+# Readable: i/o
+# Runnable: threads.
+# Thread.UncaughtExceptionHandler: threads.
+
+#### Classes
+
+class Boolean -> java.lang.Boolean extends Comparable,Object {
+  Boolean TRUE
+  Boolean FALSE
+  boolean booleanValue()
+  int compare(boolean,boolean)
+  int hashCode(boolean)
+  boolean logicalAnd(boolean,boolean)
+  boolean logicalOr(boolean,boolean)
+  boolean logicalXor(boolean,boolean)
+  boolean parseBoolean(String)
+  String toString(boolean)
+  Boolean valueOf(boolean)
+}
+
+class Byte -> java.lang.Byte extends Number,Comparable,Object {
+  int BYTES
+  byte MAX_VALUE
+  byte MIN_VALUE
+  int SIZE
+  int compare(byte,byte)
+  Byte decode(String)
+  int hashCode(byte)
+  byte parseByte(String)
+  byte parseByte(String,int)
+  String toString(byte)
+  int toUnsignedInt(byte)
+  long toUnsignedLong(byte)
+  Byte valueOf(byte)
+  Byte valueOf(String,int)
+}
+
+class Character -> java.lang.Character extends Comparable,Object {
+  int BYTES
+  byte COMBINING_SPACING_MARK
+  byte CONNECTOR_PUNCTUATION
+  byte CONTROL
+  byte CURRENCY_SYMBOL
+  byte DASH_PUNCTUATION
+  byte DECIMAL_DIGIT_NUMBER
+  byte DIRECTIONALITY_ARABIC_NUMBER
+  byte DIRECTIONALITY_BOUNDARY_NEUTRAL
+  byte DIRECTIONALITY_COMMON_NUMBER_SEPARATOR
+  byte DIRECTIONALITY_EUROPEAN_NUMBER
+  byte DIRECTIONALITY_EUROPEAN_NUMBER_SEPARATOR
+  byte DIRECTIONALITY_EUROPEAN_NUMBER_TERMINATOR
+  byte DIRECTIONALITY_LEFT_TO_RIGHT
+  byte DIRECTIONALITY_LEFT_TO_RIGHT_EMBEDDING
+  byte DIRECTIONALITY_LEFT_TO_RIGHT_OVERRIDE
+  byte DIRECTIONALITY_NONSPACING_MARK
+  byte DIRECTIONALITY_OTHER_NEUTRALS
+  byte DIRECTIONALITY_PARAGRAPH_SEPARATOR
+  byte DIRECTIONALITY_POP_DIRECTIONAL_FORMAT
+  byte DIRECTIONALITY_RIGHT_TO_LEFT
+  byte DIRECTIONALITY_RIGHT_TO_LEFT_ARABIC
+  byte DIRECTIONALITY_RIGHT_TO_LEFT_EMBEDDING
+  byte DIRECTIONALITY_RIGHT_TO_LEFT_OVERRIDE
+  byte DIRECTIONALITY_SEGMENT_SEPARATOR
+  byte DIRECTIONALITY_UNDEFINED
+  byte DIRECTIONALITY_WHITESPACE
+  byte ENCLOSING_MARK
+  byte END_PUNCTUATION
+  byte FINAL_QUOTE_PUNCTUATION
+  byte FORMAT
+  byte INITIAL_QUOTE_PUNCTUATION
+  byte LETTER_NUMBER
+  byte LINE_SEPARATOR
+  byte LOWERCASE_LETTER
+  byte MATH_SYMBOL
+  int MAX_CODE_POINT
+  char MAX_HIGH_SURROGATE
+  char MAX_LOW_SURROGATE
+  int MAX_RADIX
+  char MAX_SURROGATE
+  char MAX_VALUE
+  char MIN_CODE_POINT
+  char MIN_HIGH_SURROGATE
+  char MIN_LOW_SURROGATE
+  int MIN_RADIX
+  int MIN_SUPPLEMENTARY_CODE_POINT
+  char MIN_SURROGATE
+  char MIN_VALUE
+  byte MODIFIER_LETTER
+  byte MODIFIER_SYMBOL
+  byte NON_SPACING_MARK
+  byte OTHER_LETTER
+  byte OTHER_NUMBER
+  byte OTHER_PUNCTUATION
+  byte OTHER_SYMBOL
+  byte PARAGRAPH_SEPARATOR
+  byte PRIVATE_USE
+  int SIZE
+  byte SPACE_SEPARATOR
+  byte START_PUNCTUATION
+  byte SURROGATE
+  byte TITLECASE_LETTER
+  byte UNASSIGNED
+  byte UPPERCASE_LETTER
+  int charCount(int)
+  char charValue()
+  int codePointAt(char[],int,int)
+  int codePointAt(CharSequence,int)
+  int codePointBefore(char[],int,int)
+  int codePointBefore(CharSequence,int)
+  int codePointCount(CharSequence,int,int)
+  int compare(char,char)
+  int digit(int,int)
+  char forDigit(int,int)
+  byte getDirectionality(int)
+  String getName(int)
+  int getNumericValue(int)
+  int getType(int)
+  int hashCode(char)
+  char highSurrogate(int)
+  boolean isAlphabetic(int)
+  boolean isBmpCodePoint(int)
+  boolean isDefined(int)
+  boolean isDigit(int)
+  boolean isHighSurrogate(char)
+  boolean isIdentifierIgnorable(int)
+  boolean isIdeographic(int)
+  boolean isISOControl(int)
+  boolean isJavaIdentifierPart(int)
+  boolean isJavaIdentifierStart(int)
+  boolean isLetter(int)
+  boolean isLetterOrDigit(int)
+  boolean isLowerCase(int)
+  boolean isMirrored(int)
+  boolean isSpaceChar(int)
+  boolean isSupplementaryCodePoint(int)
+  boolean isSurrogate(char)
+  boolean isSurrogatePair(char,char)
+  boolean isTitleCase(int)
+  boolean isUnicodeIdentifierPart(int)
+  boolean isUnicodeIdentifierStart(int)
+  boolean isUpperCase(int)
+  boolean isValidCodePoint(int)
+  boolean isWhitespace(int)
+  char lowSurrogate(int)
+  int offsetByCodePoints(char[],int,int,int,int)
+  int offsetByCodePoints(CharSequence,int,int)
+  char reverseBytes(char)
+  char[] toChars(int)
+  int toChars(int,char[],int)
+  int toCodePoint(char,char)
+  char toLowerCase(char)
+  String toString(char)
+  char toTitleCase(char)
+  char toUpperCase(char)
+  Character valueOf(char)
+}
+
+class Character.Subset -> java.lang.Character$Subset extends Object {
+}
+
+class Character.UnicodeBlock -> java.lang.Character$UnicodeBlock extends Character.Subset,Object {
+  Character.UnicodeBlock AEGEAN_NUMBERS
+  Character.UnicodeBlock ALCHEMICAL_SYMBOLS
+  Character.UnicodeBlock ALPHABETIC_PRESENTATION_FORMS
+  Character.UnicodeBlock ANCIENT_GREEK_MUSICAL_NOTATION
+  Character.UnicodeBlock ANCIENT_GREEK_NUMBERS
+  Character.UnicodeBlock ANCIENT_SYMBOLS
+  Character.UnicodeBlock ARABIC
+  Character.UnicodeBlock ARABIC_EXTENDED_A
+  Character.UnicodeBlock ARABIC_MATHEMATICAL_ALPHABETIC_SYMBOLS
+  Character.UnicodeBlock ARABIC_PRESENTATION_FORMS_A
+  Character.UnicodeBlock ARABIC_PRESENTATION_FORMS_B
+  Character.UnicodeBlock ARABIC_SUPPLEMENT
+  Character.UnicodeBlock ARMENIAN
+  Character.UnicodeBlock ARROWS
+  Character.UnicodeBlock AVESTAN
+  Character.UnicodeBlock BALINESE
+  Character.UnicodeBlock BAMUM
+  Character.UnicodeBlock BAMUM_SUPPLEMENT
+  Character.UnicodeBlock BASIC_LATIN
+  Character.UnicodeBlock BATAK
+  Character.UnicodeBlock BENGALI
+  Character.UnicodeBlock BLOCK_ELEMENTS
+  Character.UnicodeBlock BOPOMOFO
+  Character.UnicodeBlock BOPOMOFO_EXTENDED
+  Character.UnicodeBlock BOX_DRAWING
+  Character.UnicodeBlock BRAHMI
+  Character.UnicodeBlock BRAILLE_PATTERNS
+  Character.UnicodeBlock BUGINESE
+  Character.UnicodeBlock BUHID
+  Character.UnicodeBlock BYZANTINE_MUSICAL_SYMBOLS
+  Character.UnicodeBlock CARIAN
+  Character.UnicodeBlock CHAKMA
+  Character.UnicodeBlock CHAM
+  Character.UnicodeBlock CHEROKEE
+  Character.UnicodeBlock CJK_COMPATIBILITY
+  Character.UnicodeBlock CJK_COMPATIBILITY_FORMS
+  Character.UnicodeBlock CJK_COMPATIBILITY_IDEOGRAPHS
+  Character.UnicodeBlock CJK_COMPATIBILITY_IDEOGRAPHS_SUPPLEMENT
+  Character.UnicodeBlock CJK_RADICALS_SUPPLEMENT
+  Character.UnicodeBlock CJK_STROKES
+  Character.UnicodeBlock CJK_SYMBOLS_AND_PUNCTUATION
+  Character.UnicodeBlock CJK_UNIFIED_IDEOGRAPHS
+  Character.UnicodeBlock CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A
+  Character.UnicodeBlock CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B
+  Character.UnicodeBlock CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C
+  Character.UnicodeBlock CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D
+  Character.UnicodeBlock COMBINING_DIACRITICAL_MARKS
+  Character.UnicodeBlock COMBINING_DIACRITICAL_MARKS_SUPPLEMENT
+  Character.UnicodeBlock COMBINING_HALF_MARKS
+  Character.UnicodeBlock COMBINING_MARKS_FOR_SYMBOLS
+  Character.UnicodeBlock COMMON_INDIC_NUMBER_FORMS
+  Character.UnicodeBlock CONTROL_PICTURES
+  Character.UnicodeBlock COPTIC
+  Character.UnicodeBlock COUNTING_ROD_NUMERALS
+  Character.UnicodeBlock CUNEIFORM
+  Character.UnicodeBlock CUNEIFORM_NUMBERS_AND_PUNCTUATION
+  Character.UnicodeBlock CURRENCY_SYMBOLS
+  Character.UnicodeBlock CYPRIOT_SYLLABARY
+  Character.UnicodeBlock CYRILLIC
+  Character.UnicodeBlock CYRILLIC_EXTENDED_A
+  Character.UnicodeBlock CYRILLIC_EXTENDED_B
+  Character.UnicodeBlock CYRILLIC_SUPPLEMENTARY
+  Character.UnicodeBlock DESERET
+  Character.UnicodeBlock DEVANAGARI
+  Character.UnicodeBlock DEVANAGARI_EXTENDED
+  Character.UnicodeBlock DINGBATS
+  Character.UnicodeBlock DOMINO_TILES
+  Character.UnicodeBlock EGYPTIAN_HIEROGLYPHS
+  Character.UnicodeBlock EMOTICONS
+  Character.UnicodeBlock ENCLOSED_ALPHANUMERIC_SUPPLEMENT
+  Character.UnicodeBlock ENCLOSED_ALPHANUMERICS
+  Character.UnicodeBlock ENCLOSED_CJK_LETTERS_AND_MONTHS
+  Character.UnicodeBlock ENCLOSED_IDEOGRAPHIC_SUPPLEMENT
+  Character.UnicodeBlock ETHIOPIC
+  Character.UnicodeBlock ETHIOPIC_EXTENDED
+  Character.UnicodeBlock ETHIOPIC_EXTENDED_A
+  Character.UnicodeBlock ETHIOPIC_SUPPLEMENT
+  Character.UnicodeBlock GENERAL_PUNCTUATION
+  Character.UnicodeBlock GEOMETRIC_SHAPES
+  Character.UnicodeBlock GEORGIAN
+  Character.UnicodeBlock GEORGIAN_SUPPLEMENT
+  Character.UnicodeBlock GLAGOLITIC
+  Character.UnicodeBlock GOTHIC
+  Character.UnicodeBlock GREEK
+  Character.UnicodeBlock GREEK_EXTENDED
+  Character.UnicodeBlock GUJARATI
+  Character.UnicodeBlock GURMUKHI
+  Character.UnicodeBlock HALFWIDTH_AND_FULLWIDTH_FORMS
+  Character.UnicodeBlock HANGUL_COMPATIBILITY_JAMO
+  Character.UnicodeBlock HANGUL_JAMO
+  Character.UnicodeBlock HANGUL_JAMO_EXTENDED_A
+  Character.UnicodeBlock HANGUL_JAMO_EXTENDED_B
+  Character.UnicodeBlock HANGUL_SYLLABLES
+  Character.UnicodeBlock HANUNOO
+  Character.UnicodeBlock HEBREW
+  Character.UnicodeBlock HIGH_PRIVATE_USE_SURROGATES
+  Character.UnicodeBlock HIGH_SURROGATES
+  Character.UnicodeBlock HIRAGANA
+  Character.UnicodeBlock IDEOGRAPHIC_DESCRIPTION_CHARACTERS
+  Character.UnicodeBlock IMPERIAL_ARAMAIC
+  Character.UnicodeBlock INSCRIPTIONAL_PAHLAVI
+  Character.UnicodeBlock INSCRIPTIONAL_PARTHIAN
+  Character.UnicodeBlock IPA_EXTENSIONS
+  Character.UnicodeBlock JAVANESE
+  Character.UnicodeBlock KAITHI
+  Character.UnicodeBlock KANA_SUPPLEMENT
+  Character.UnicodeBlock KANBUN
+  Character.UnicodeBlock KANGXI_RADICALS
+  Character.UnicodeBlock KANNADA
+  Character.UnicodeBlock KATAKANA
+  Character.UnicodeBlock KATAKANA_PHONETIC_EXTENSIONS
+  Character.UnicodeBlock KAYAH_LI
+  Character.UnicodeBlock KHAROSHTHI
+  Character.UnicodeBlock KHMER
+  Character.UnicodeBlock KHMER_SYMBOLS
+  Character.UnicodeBlock LAO
+  Character.UnicodeBlock LATIN_1_SUPPLEMENT
+  Character.UnicodeBlock LATIN_EXTENDED_A
+  Character.UnicodeBlock LATIN_EXTENDED_ADDITIONAL
+  Character.UnicodeBlock LATIN_EXTENDED_B
+  Character.UnicodeBlock LATIN_EXTENDED_C
+  Character.UnicodeBlock LATIN_EXTENDED_D
+  Character.UnicodeBlock LEPCHA
+  Character.UnicodeBlock LETTERLIKE_SYMBOLS
+  Character.UnicodeBlock LIMBU
+  Character.UnicodeBlock LINEAR_B_IDEOGRAMS
+  Character.UnicodeBlock LINEAR_B_SYLLABARY
+  Character.UnicodeBlock LISU
+  Character.UnicodeBlock LOW_SURROGATES
+  Character.UnicodeBlock LYCIAN
+  Character.UnicodeBlock LYDIAN
+  Character.UnicodeBlock MAHJONG_TILES
+  Character.UnicodeBlock MALAYALAM
+  Character.UnicodeBlock MANDAIC
+  Character.UnicodeBlock MATHEMATICAL_ALPHANUMERIC_SYMBOLS
+  Character.UnicodeBlock MATHEMATICAL_OPERATORS
+  Character.UnicodeBlock MEETEI_MAYEK
+  Character.UnicodeBlock MEETEI_MAYEK_EXTENSIONS
+  Character.UnicodeBlock MEROITIC_CURSIVE
+  Character.UnicodeBlock MEROITIC_HIEROGLYPHS
+  Character.UnicodeBlock MIAO
+  Character.UnicodeBlock MISCELLANEOUS_MATHEMATICAL_SYMBOLS_A
+  Character.UnicodeBlock MISCELLANEOUS_MATHEMATICAL_SYMBOLS_B
+  Character.UnicodeBlock MISCELLANEOUS_SYMBOLS
+  Character.UnicodeBlock MISCELLANEOUS_SYMBOLS_AND_ARROWS
+  Character.UnicodeBlock MISCELLANEOUS_SYMBOLS_AND_PICTOGRAPHS
+  Character.UnicodeBlock MISCELLANEOUS_TECHNICAL
+  Character.UnicodeBlock MODIFIER_TONE_LETTERS
+  Character.UnicodeBlock MONGOLIAN
+  Character.UnicodeBlock MUSICAL_SYMBOLS
+  Character.UnicodeBlock MYANMAR
+  Character.UnicodeBlock MYANMAR_EXTENDED_A
+  Character.UnicodeBlock NEW_TAI_LUE
+  Character.UnicodeBlock NKO
+  Character.UnicodeBlock NUMBER_FORMS
+  Character.UnicodeBlock OGHAM
+  Character.UnicodeBlock OL_CHIKI
+  Character.UnicodeBlock OLD_ITALIC
+  Character.UnicodeBlock OLD_PERSIAN
+  Character.UnicodeBlock OLD_SOUTH_ARABIAN
+  Character.UnicodeBlock OLD_TURKIC
+  Character.UnicodeBlock OPTICAL_CHARACTER_RECOGNITION
+  Character.UnicodeBlock ORIYA
+  Character.UnicodeBlock OSMANYA
+  Character.UnicodeBlock PHAGS_PA
+  Character.UnicodeBlock PHAISTOS_DISC
+  Character.UnicodeBlock PHOENICIAN
+  Character.UnicodeBlock PHONETIC_EXTENSIONS
+  Character.UnicodeBlock PHONETIC_EXTENSIONS_SUPPLEMENT
+  Character.UnicodeBlock PLAYING_CARDS
+  Character.UnicodeBlock PRIVATE_USE_AREA
+  Character.UnicodeBlock REJANG
+  Character.UnicodeBlock RUMI_NUMERAL_SYMBOLS
+  Character.UnicodeBlock RUNIC
+  Character.UnicodeBlock SAMARITAN
+  Character.UnicodeBlock SAURASHTRA
+  Character.UnicodeBlock SHARADA
+  Character.UnicodeBlock SHAVIAN
+  Character.UnicodeBlock SINHALA
+  Character.UnicodeBlock SMALL_FORM_VARIANTS
+  Character.UnicodeBlock SORA_SOMPENG
+  Character.UnicodeBlock SPACING_MODIFIER_LETTERS
+  Character.UnicodeBlock SPECIALS
+  Character.UnicodeBlock SUNDANESE
+  Character.UnicodeBlock SUNDANESE_SUPPLEMENT
+  Character.UnicodeBlock SUPERSCRIPTS_AND_SUBSCRIPTS
+  Character.UnicodeBlock SUPPLEMENTAL_ARROWS_A
+  Character.UnicodeBlock SUPPLEMENTAL_ARROWS_B
+  Character.UnicodeBlock SUPPLEMENTAL_MATHEMATICAL_OPERATORS
+  Character.UnicodeBlock SUPPLEMENTAL_PUNCTUATION
+  Character.UnicodeBlock SUPPLEMENTARY_PRIVATE_USE_AREA_A
+  Character.UnicodeBlock SUPPLEMENTARY_PRIVATE_USE_AREA_B
+  Character.UnicodeBlock SYLOTI_NAGRI
+  Character.UnicodeBlock SYRIAC
+  Character.UnicodeBlock TAGALOG
+  Character.UnicodeBlock TAGBANWA
+  Character.UnicodeBlock TAGS
+  Character.UnicodeBlock TAI_LE
+  Character.UnicodeBlock TAI_THAM
+  Character.UnicodeBlock TAI_VIET
+  Character.UnicodeBlock TAI_XUAN_JING_SYMBOLS
+  Character.UnicodeBlock TAKRI
+  Character.UnicodeBlock TAMIL
+  Character.UnicodeBlock TELUGU
+  Character.UnicodeBlock THAANA
+  Character.UnicodeBlock THAI
+  Character.UnicodeBlock TIBETAN
+  Character.UnicodeBlock TIFINAGH
+  Character.UnicodeBlock TRANSPORT_AND_MAP_SYMBOLS
+  Character.UnicodeBlock UGARITIC
+  Character.UnicodeBlock UNIFIED_CANADIAN_ABORIGINAL_SYLLABICS
+  Character.UnicodeBlock UNIFIED_CANADIAN_ABORIGINAL_SYLLABICS_EXTENDED
+  Character.UnicodeBlock VAI
+  Character.UnicodeBlock VARIATION_SELECTORS
+  Character.UnicodeBlock VARIATION_SELECTORS_SUPPLEMENT
+  Character.UnicodeBlock VEDIC_EXTENSIONS
+  Character.UnicodeBlock VERTICAL_FORMS
+  Character.UnicodeBlock YI_RADICALS
+  Character.UnicodeBlock YI_SYLLABLES
+  Character.UnicodeBlock YIJING_HEXAGRAM_SYMBOLS
+  Character.UnicodeBlock forName(String)
+  Character.UnicodeBlock of(int)
+}
+
+# Class: skipped for obvious reasons
+# ClassLoader: ...
+# ClassValue: ...
+# Compiler: ...
+
+class Double -> java.lang.Double extends Number,Comparable,Object {
+  int BYTES
+  int MAX_EXPONENT
+  double MAX_VALUE
+  int MIN_EXPONENT
+  double MIN_NORMAL
+  double MIN_VALUE
+  double NaN
+  double NEGATIVE_INFINITY
+  double POSITIVE_INFINITY
+  int SIZE
+  int compare(double,double)
+  long doubleToLongBits(double)
+  long doubleToRawLongBits(double)
+  int hashCode(double)
+  boolean isFinite(double)
+  boolean isInfinite()
+  boolean isInfinite(double)
+  boolean isNaN()
+  boolean isNaN(double)
+  double longBitsToDouble(long)
+  double max(double,double)
+  double min(double,double)
+  double parseDouble(String)
+  double sum(double,double)
+  String toHexString(double)
+  String toString(double)
+  Double valueOf(double)
+}
+
+class Enum -> java.lang.Enum extends Comparable,Object {
+  String name();
+  int ordinal();
+}
+
+class Float -> java.lang.Float extends Number,Comparable,Object {
+  int BYTES
+  int MAX_EXPONENT
+  float MAX_VALUE
+  int MIN_EXPONENT
+  float MIN_NORMAL
+  float MIN_VALUE
+  float NaN
+  float NEGATIVE_INFINITY
+  float POSITIVE_INFINITY
+  int SIZE
+  int compare(float,float)
+  int floatToIntBits(float)
+  int floatToRawIntBits(float)
+  int hashCode(float)
+  float intBitsToFloat(int)
+  boolean isFinite(float)
+  boolean isInfinite()
+  boolean isInfinite(float)
+  boolean isNaN()
+  boolean isNaN(float)
+  float max(float,float)
+  float min(float,float)
+  float parseFloat(String)
+  float sum(float,float)
+  String toHexString(float)
+  String toString(float)
+  Float valueOf(float)
+}
+
+# InheritableThreadLocal: threads
+
+class Integer -> java.lang.Integer extends Number,Comparable,Object {
+  int BYTES
+  int MAX_VALUE
+  int MIN_VALUE
+  int SIZE
+  int bitCount(int)
+  int compare(int,int)
+  int compareUnsigned(int,int)
+  Integer decode(String)
+  int divideUnsigned(int,int)
+  int hashCode(int)
+  int highestOneBit(int)
+  int lowestOneBit(int)
+  int max(int,int)
+  int min(int,int)
+  int numberOfLeadingZeros(int)
+  int numberOfTrailingZeros(int)
+  int parseInt(String)
+  int parseInt(String,int)
+  int parseUnsignedInt(String)
+  int parseUnsignedInt(String,int)
+  int remainderUnsigned(int,int)
+  int reverse(int)
+  int reverseBytes(int)
+  int rotateLeft(int,int)
+  int rotateRight(int,int)
+  int signum(int)
+  String toBinaryString(int)
+  String toHexString(int)
+  String toOctalString(int)
+  String toString(int)
+  String toString(int,int)
+  long toUnsignedLong(int)
+  String toUnsignedString(int)
+  String toUnsignedString(int,int)
+  Integer valueOf(int)
+  Integer valueOf(String,int)
+}
+
+class Long -> java.lang.Long extends Number,Comparable,Object {
+  int BYTES
+  long MAX_VALUE
+  long MIN_VALUE
+  int SIZE
+  int bitCount(long)
+  int compare(long,long)
+  int compareUnsigned(long,long)
+  Long decode(String)
+  long divideUnsigned(long,long)
+  int hashCode(long)
+  long highestOneBit(long)
+  long lowestOneBit(long)
+  long max(long,long)
+  long min(long,long)
+  int numberOfLeadingZeros(long)
+  int numberOfTrailingZeros(long)
+  long parseLong(String)
+  long parseLong(String,int)
+  long parseUnsignedLong(String)
+  long parseUnsignedLong(String,int)
+  long remainderUnsigned(long,long)
+  long reverse(long)
+  long reverseBytes(long)
+  long rotateLeft(long,int)
+  long rotateRight(long,int)
+  int signum(long)
+  long sum(long,long)
+  String toBinaryString(long)
+  String toHexString(long)
+  String toOctalString(long)
+  String toString(long)
+  String toString(long,int)
+  String toUnsignedString(long)
+  String toUnsignedString(long,int)
+  Long valueOf(long)
+  Long valueOf(String,int)
+}
+
+class Math -> java.lang.Math {
+  double E
+  double PI
+  double abs(double)
+  double acos(double)
+  double asin(double)
+  double atan(double)
+  double atan2(double,double)
+  double cbrt(double)
+  double ceil(double)
+  double copySign(double,double)
+  double cos(double)
+  double cosh(double)
+  double exp(double)
+  double expm1(double)
+  double floor(double)
+  double hypot(double,double)
+  double IEEEremainder(double,double)
+  double log(double)
+  double log10(double)
+  double log1p(double)
+  double max(double,double)
+  double min(double,double)
+  double nextAfter(double,double)
+  double nextDown(double)
+  double nextUp(double)
+  double pow(double,double)
+  double random()
+  double rint(double)
+  long round(double)
+  double scalb(double,int)
+  double signum(double)
+  double sin(double)
+  double sinh(double)
+  double sqrt(double)
+  double tan(double)
+  double tanh(double)
+  double toDegrees(double)
+  double toRadians(double)
+  double ulp(double)
+}
+
+class Number -> java.lang.Number extends Object {
+  byte byteValue()
+  short shortValue()
+  int intValue()
+  long longValue()
+  float floatValue()
+  double doubleValue()
+}
+
+class Object -> java.lang.Object {
+  boolean equals(Object)
+  int hashCode()
+  String toString()
+}
+
+# Package: skipped
+# Process: skipped
+# ProcessBuilder: skipped
+# ProcessBuilder.Redirect: skipped
+# Runtime: skipped
+# RuntimePermission: skipped
+# SecurityManger: skipped
+
+class Short -> java.lang.Short extends Number,Comparable,Object {
+  int BYTES
+  short MAX_VALUE
+  short MIN_VALUE
+  int SIZE
+  int compare(short,short)
+  Short decode(String)
+  int hashCode(short)
+  short parseShort(String)
+  short parseShort(String,int)
+  short reverseBytes(short)
+  String toString(short)
+  int toUnsignedInt(short)
+  long toUnsignedLong(short)
+  Short valueOf(short)
+  Short valueOf(String,int)
+}
+
+class StackTraceElement -> java.lang.StackTraceElement extends Object {
+  StackTraceElement <init>(String,String,String,int)
+  String getClassName()
+  String getFileName()
+  int getLineNumber()
+  String getMethodName()
+  boolean isNativeMethod()
+}
+
+class StrictMath -> java.lang.StrictMath {
+  double E
+  double PI
+  double abs(double)
+  double acos(double)
+  double asin(double)
+  double atan(double)
+  double atan2(double,double)
+  double cbrt(double)
+  double ceil(double)
+  double copySign(double,double)
+  double cos(double)
+  double cosh(double)
+  double exp(double)
+  double expm1(double)
+  double floor(double)
+  double hypot(double,double)
+  double IEEEremainder(double,double)
+  double log(double)
+  double log10(double)
+  double log1p(double)
+  double max(double,double)
+  double min(double,double)
+  double nextAfter(double,double)
+  double nextDown(double)
+  double nextUp(double)
+  double pow(double,double)
+  double random()
+  double rint(double)
+  long round(double)
+  double scalb(double,int)
+  double signum(double)
+  double sin(double)
+  double sinh(double)
+  double sqrt(double)
+  double tan(double)
+  double tanh(double)
+  double toDegrees(double)
+  double toRadians(double)
+  double ulp(double)
+}
+
+class String -> java.lang.String extends CharSequence,Comparable,Object {
+  String <init>()
+  int codePointAt(int)
+  int codePointBefore(int)
+  int codePointCount(int,int)
+  int compareToIgnoreCase(String)
+  String concat(String)
+  boolean contains(CharSequence)
+  boolean contentEquals(CharSequence)
+  String copyValueOf(char[])
+  String copyValueOf(char[],int,int)
+  boolean endsWith(String)
+  boolean equalsIgnoreCase(String)
+  String format(Locale,String,def[])
+  String format(String,def[])
+  void getChars(int,int,char[],int)
+  int indexOf(String)
+  int indexOf(String,int)
+  boolean isEmpty()
+  String join(CharSequence,Iterable)
+  int lastIndexOf(String)
+  int lastIndexOf(String,int)
+  int offsetByCodePoints(int,int)
+  boolean regionMatches(boolean,int,String,int,int)
+  boolean regionMatches(int,String,int,int)
+  String replace(CharSequence,CharSequence)
+  boolean startsWith(String)
+  boolean startsWith(String,int)
+  String substring(int)
+  String substring(int,int)
+  char[] toCharArray()
+  String toLowerCase()
+  String toLowerCase(Locale)
+  String toUpperCase()
+  String toUpperCase(Locale)
+  String trim()
+  String valueOf(def)
+}
+
+class StringBuffer -> java.lang.StringBuffer extends CharSequence,Appendable,Object {
+  StringBuffer <init>()
+  StringBuffer <init>(CharSequence)
+  StringBuffer append(def)
+  StringBuffer append(CharSequence,int,int)
+  StringBuffer appendCodePoint(int)
+  int capacity()
+  int codePointAt(int)
+  int codePointBefore(int)
+  int codePointCount(int,int)
+  StringBuffer delete(int,int)
+  StringBuffer deleteCharAt(int)
+  void getChars(int,int,char[],int)
+  int indexOf(String)
+  int indexOf(String,int)
+  StringBuffer insert(int,def)
+  int lastIndexOf(String)
+  int lastIndexOf(String,int)
+  int offsetByCodePoints(int,int)
+  StringBuffer replace(int,int,String)
+  StringBuffer reverse()
+  void setCharAt(int,char)
+  void setLength(int)
+  String substring(int)
+  String substring(int,int)
+}
+
+class StringBuilder -> java.lang.StringBuilder extends CharSequence,Appendable,Object {
+  StringBuilder <init>()
+  StringBuilder <init>(CharSequence)
+  StringBuilder append(def)
+  StringBuilder append(CharSequence,int,int)
+  StringBuilder appendCodePoint(int)
+  int capacity()
+  int codePointAt(int)
+  int codePointBefore(int)
+  int codePointCount(int,int)
+  StringBuilder delete(int,int)
+  StringBuilder deleteCharAt(int)
+  void getChars(int,int,char[],int)
+  int indexOf(String)
+  int indexOf(String,int)
+  StringBuilder insert(int,def)
+  int lastIndexOf(String)
+  int lastIndexOf(String,int)
+  int offsetByCodePoints(int,int)
+  StringBuilder replace(int,int,String)
+  StringBuilder reverse()
+  void setCharAt(int,char)
+  void setLength(int)
+  String substring(int)
+  String substring(int,int)
+}
+
+class System -> java.lang.System extends Object {
+  void arraycopy(Object,int,Object,int,int)
+  long currentTimeMillis()
+  long nanoTime()
+}
+
+# Thread: skipped
+# ThreadGroup: skipped
+# ThreadLocal: skipped
+# Throwable: skipped (reserved for painless, users can only catch Exceptions)
+
+class Void -> java.lang.Void extends Object {
+}
+
+#### Enums
+
+class Character.UnicodeScript -> java.lang.Character$UnicodeScript extends Enum,Object {
+  Character.UnicodeScript ARABIC
+  Character.UnicodeScript ARMENIAN
+  Character.UnicodeScript AVESTAN
+  Character.UnicodeScript BALINESE
+  Character.UnicodeScript BAMUM
+  Character.UnicodeScript BATAK
+  Character.UnicodeScript BENGALI
+  Character.UnicodeScript BOPOMOFO
+  Character.UnicodeScript BRAHMI
+  Character.UnicodeScript BRAILLE
+  Character.UnicodeScript BUGINESE
+  Character.UnicodeScript BUHID
+  Character.UnicodeScript CANADIAN_ABORIGINAL
+  Character.UnicodeScript CARIAN
+  Character.UnicodeScript CHAKMA
+  Character.UnicodeScript CHAM
+  Character.UnicodeScript CHEROKEE
+  Character.UnicodeScript COMMON
+  Character.UnicodeScript COPTIC
+  Character.UnicodeScript CUNEIFORM
+  Character.UnicodeScript CYPRIOT
+  Character.UnicodeScript CYRILLIC
+  Character.UnicodeScript DESERET
+  Character.UnicodeScript DEVANAGARI
+  Character.UnicodeScript EGYPTIAN_HIEROGLYPHS
+  Character.UnicodeScript ETHIOPIC
+  Character.UnicodeScript GEORGIAN
+  Character.UnicodeScript GLAGOLITIC
+  Character.UnicodeScript GOTHIC
+  Character.UnicodeScript GREEK
+  Character.UnicodeScript GUJARATI
+  Character.UnicodeScript GURMUKHI
+  Character.UnicodeScript HAN
+  Character.UnicodeScript HANGUL
+  Character.UnicodeScript HANUNOO
+  Character.UnicodeScript HEBREW
+  Character.UnicodeScript HIRAGANA
+  Character.UnicodeScript IMPERIAL_ARAMAIC
+  Character.UnicodeScript INHERITED
+  Character.UnicodeScript INSCRIPTIONAL_PAHLAVI
+  Character.UnicodeScript INSCRIPTIONAL_PARTHIAN
+  Character.UnicodeScript JAVANESE
+  Character.UnicodeScript KAITHI
+  Character.UnicodeScript KANNADA
+  Character.UnicodeScript KATAKANA
+  Character.UnicodeScript KAYAH_LI
+  Character.UnicodeScript KHAROSHTHI
+  Character.UnicodeScript KHMER
+  Character.UnicodeScript LAO
+  Character.UnicodeScript LATIN
+  Character.UnicodeScript LEPCHA
+  Character.UnicodeScript LIMBU
+  Character.UnicodeScript LINEAR_B
+  Character.UnicodeScript LISU
+  Character.UnicodeScript LYCIAN
+  Character.UnicodeScript LYDIAN
+  Character.UnicodeScript MALAYALAM
+  Character.UnicodeScript MANDAIC
+  Character.UnicodeScript MEETEI_MAYEK
+  Character.UnicodeScript MEROITIC_CURSIVE
+  Character.UnicodeScript MEROITIC_HIEROGLYPHS
+  Character.UnicodeScript MIAO
+  Character.UnicodeScript MONGOLIAN
+  Character.UnicodeScript MYANMAR
+  Character.UnicodeScript NEW_TAI_LUE
+  Character.UnicodeScript NKO
+  Character.UnicodeScript OGHAM
+  Character.UnicodeScript OL_CHIKI
+  Character.UnicodeScript OLD_ITALIC
+  Character.UnicodeScript OLD_PERSIAN
+  Character.UnicodeScript OLD_SOUTH_ARABIAN
+  Character.UnicodeScript OLD_TURKIC
+  Character.UnicodeScript ORIYA
+  Character.UnicodeScript OSMANYA
+  Character.UnicodeScript PHAGS_PA
+  Character.UnicodeScript PHOENICIAN
+  Character.UnicodeScript REJANG
+  Character.UnicodeScript RUNIC
+  Character.UnicodeScript SAMARITAN
+  Character.UnicodeScript SAURASHTRA
+  Character.UnicodeScript SHARADA
+  Character.UnicodeScript SHAVIAN
+  Character.UnicodeScript SINHALA
+  Character.UnicodeScript SORA_SOMPENG
+  Character.UnicodeScript SUNDANESE
+  Character.UnicodeScript SYLOTI_NAGRI
+  Character.UnicodeScript SYRIAC
+  Character.UnicodeScript TAGALOG
+  Character.UnicodeScript TAGBANWA
+  Character.UnicodeScript TAI_LE
+  Character.UnicodeScript TAI_THAM
+  Character.UnicodeScript TAI_VIET
+  Character.UnicodeScript TAKRI
+  Character.UnicodeScript TAMIL
+  Character.UnicodeScript TELUGU
+  Character.UnicodeScript THAANA
+  Character.UnicodeScript THAI
+  Character.UnicodeScript TIBETAN
+  Character.UnicodeScript TIFINAGH
+  Character.UnicodeScript UGARITIC
+  Character.UnicodeScript UNKNOWN
+  Character.UnicodeScript VAI
+  Character.UnicodeScript YI
+  Character.UnicodeScript forName(String)
+  Character.UnicodeScript of(int)
+  Character.UnicodeScript valueOf(String)
+  Character.UnicodeScript[] values()
+}
+
+#### Exceptions
+
+class ArithmeticException -> java.lang.ArithmeticException extends RuntimeException,Exception,Object {
+  ArithmeticException <init>()
+  ArithmeticException <init>(String)
+}
+
+class ArrayIndexOutOfBoundsException -> java.lang.ArrayIndexOutOfBoundsException extends IndexOutOfBoundsException,RuntimeException,Exception,Object {
+  ArrayIndexOutOfBoundsException <init>()
+  ArrayIndexOutOfBoundsException <init>(String)
+}
+
+class ArrayStoreException -> java.lang.ArrayStoreException extends RuntimeException,Exception,Object {
+  ArrayStoreException <init>()
+  ArrayStoreException <init>(String)
+}
+
+class ClassCastException -> java.lang.ClassCastException extends RuntimeException,Exception,Object {
+  ClassCastException <init>()
+  ClassCastException <init>(String)
+}
+
+class ClassNotFoundException -> java.lang.ClassNotFoundException extends ReflectiveOperationException,Exception,Object {
+  ClassNotFoundException <init>()
+  ClassNotFoundException <init>(String)
+}
+
+class CloneNotSupportedException -> java.lang.CloneNotSupportedException extends Exception,Object {
+  CloneNotSupportedException <init>()
+  CloneNotSupportedException <init>(String)
+}
+
+class EnumConstantNotPresentException -> java.lang.EnumConstantNotPresentException extends RuntimeException,Exception,Object {
+  String constantName()
+}
+
+class Exception -> java.lang.Exception extends Object {
+  Exception <init>()
+  Exception <init>(String)
+  String getLocalizedMessage()
+  String getMessage()
+  StackTraceElement[] getStackTrace()
+}
+
+class IllegalAccessException -> java.lang.IllegalAccessException extends ReflectiveOperationException,Exception,Object {
+  IllegalAccessException <init>()
+  IllegalAccessException <init>(String)
+}
+
+class IllegalArgumentException -> java.lang.IllegalArgumentException extends RuntimeException,Exception,Object {
+  IllegalArgumentException <init>()
+  IllegalArgumentException <init>(String)
+}
+
+class IllegalMonitorStateException -> java.lang.IllegalMonitorStateException extends RuntimeException,Exception,Object {
+  IllegalMonitorStateException <init>()
+  IllegalMonitorStateException <init>(String)
+}
+
+class IllegalStateException -> java.lang.IllegalStateException extends RuntimeException,Exception,Object {
+  IllegalStateException <init>()
+  IllegalStateException <init>(String)
+}
+
+class IllegalThreadStateException -> java.lang.IllegalThreadStateException extends IllegalArgumentException,RuntimeException,Exception,Object {
+  IllegalThreadStateException <init>()
+  IllegalThreadStateException <init>(String)
+}
+
+class IndexOutOfBoundsException -> java.lang.IndexOutOfBoundsException extends RuntimeException,Exception,Object {
+  IndexOutOfBoundsException <init>()
+  IndexOutOfBoundsException <init>(String)
+}
+
+class InstantiationException -> java.lang.InstantiationException extends ReflectiveOperationException,Exception,Object {
+  InstantiationException <init>()
+  InstantiationException <init>(String)
+}
+
+class InterruptedException -> java.lang.InterruptedException extends Exception,Object {
+  InterruptedException <init>()
+  InterruptedException <init>(String)
+}
+
+class NegativeArraySizeException -> java.lang.NegativeArraySizeException extends RuntimeException,Exception,Object {
+  NegativeArraySizeException <init>()
+  NegativeArraySizeException <init>(String)
+}
+
+class NoSuchFieldException -> java.lang.NoSuchFieldException extends ReflectiveOperationException,Exception,Object {
+  NoSuchFieldException <init>()
+  NoSuchFieldException <init>(String)
+}
+
+class NoSuchMethodException -> java.lang.NoSuchMethodException extends ReflectiveOperationException,Exception,Object {
+  NoSuchMethodException <init>()
+  NoSuchMethodException <init>(String)
+}
+
+class NullPointerException -> java.lang.NullPointerException extends RuntimeException,Exception,Object {
+  NullPointerException <init>()
+  NullPointerException <init>(String)
+}
+
+class NumberFormatException -> java.lang.NumberFormatException extends RuntimeException,Exception,Object {
+  NumberFormatException <init>()
+  NumberFormatException <init>(String)
+}
+
+class ReflectiveOperationException -> java.lang.ReflectiveOperationException extends Exception,Object {
+  ReflectiveOperationException <init>()
+  ReflectiveOperationException <init>(String)
+}
+
+class RuntimeException -> java.lang.RuntimeException extends Exception,Object {
+  RuntimeException <init>()
+}
+
+class SecurityException -> java.lang.SecurityException extends RuntimeException,Exception,Object {
+  SecurityException <init>()
+  SecurityException <init>(String)
+}
+
+class StringIndexOutOfBoundsException -> java.lang.StringIndexOutOfBoundsException extends IndexOutOfBoundsException,RuntimeException,Exception,Object {
+  StringIndexOutOfBoundsException <init>()
+  StringIndexOutOfBoundsException <init>(String)
+}
+
+class TypeNotPresentException -> java.lang.TypeNotPresentException extends RuntimeException,Exception,Object {
+  String typeName()
+}
+
+class UnsupportedOperationException -> java.lang.UnsupportedOperationException extends RuntimeException,Exception,Object {
+  UnsupportedOperationException <init>()
+  UnsupportedOperationException <init>(String)
+}
+
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.math.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.math.txt
new file mode 100644
index 00000000000..42680f8a428
--- /dev/null
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.math.txt
@@ -0,0 +1,149 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#
+# Painless definition file. This defines the hierarchy of classes,
+# what methods and fields they have, etc.
+#
+
+#### Classes
+
+class BigDecimal -> java.math.BigDecimal extends Number,Comparable,Object {
+  BigDecimal ONE
+  BigDecimal TEN
+  BigDecimal ZERO
+  BigDecimal <init>(String)
+  BigDecimal <init>(String,MathContext)
+  BigDecimal abs()
+  BigDecimal abs(MathContext)
+  BigDecimal add(BigDecimal)
+  BigDecimal add(BigDecimal,MathContext)
+  byte byteValueExact()
+  BigDecimal divide(BigDecimal)
+  BigDecimal divide(BigDecimal,MathContext)
+  BigDecimal[] divideAndRemainder(BigDecimal)
+  BigDecimal[] divideAndRemainder(BigDecimal,MathContext)
+  BigDecimal divideToIntegralValue(BigDecimal)
+  BigDecimal divideToIntegralValue(BigDecimal,MathContext)
+  int intValueExact()
+  long longValueExact()
+  BigDecimal max(BigDecimal)
+  BigDecimal min(BigDecimal)
+  BigDecimal movePointLeft(int)
+  BigDecimal movePointRight(int)
+  BigDecimal multiply(BigDecimal)
+  BigDecimal multiply(BigDecimal,MathContext)
+  BigDecimal negate()
+  BigDecimal negate(MathContext)
+  BigDecimal plus()
+  BigDecimal plus(MathContext)
+  BigDecimal pow(int)
+  BigDecimal pow(int,MathContext)
+  int precision()
+  BigDecimal remainder(BigDecimal)
+  BigDecimal remainder(BigDecimal,MathContext)
+  BigDecimal round(MathContext)
+  int scale()
+  BigDecimal scaleByPowerOfTen(int)
+  BigDecimal setScale(int)
+  BigDecimal setScale(int,RoundingMode)
+  short shortValueExact()
+  int signum()
+  BigDecimal stripTrailingZeros()
+  BigDecimal subtract(BigDecimal)
+  BigDecimal subtract(BigDecimal,MathContext)
+  BigInteger toBigInteger()
+  BigInteger toBigIntegerExact()
+  String toEngineeringString()
+  String toPlainString()
+  BigDecimal ulp()
+  BigDecimal valueOf(double)
+}
+
+class BigInteger -> java.math.BigInteger extends Number,Comparable,Object {
+  BigInteger ONE
+  BigInteger TEN
+  BigInteger ZERO
+  BigInteger <init>(String)
+  BigInteger <init>(String,int)
+  BigInteger abs()
+  BigInteger add(BigInteger)
+  BigInteger and(BigInteger)
+  BigInteger andNot(BigInteger)
+  int bitCount()
+  int bitLength()
+  byte byteValueExact()
+  BigInteger clearBit(int)
+  BigInteger divide(BigInteger)
+  BigInteger[] divideAndRemainder(BigInteger)
+  BigInteger flipBit(int)
+  BigInteger gcd(BigInteger)
+  int getLowestSetBit()
+  int intValueExact()
+  long longValueExact()
+  BigInteger max(BigInteger)
+  BigInteger min(BigInteger)
+  BigInteger mod(BigInteger)
+  BigInteger modInverse(BigInteger)
+  BigInteger modPow(BigInteger,BigInteger)
+  BigInteger multiply(BigInteger)
+  BigInteger negate()
+  BigInteger not()
+  BigInteger or(BigInteger)
+  BigInteger pow(int)
+  BigInteger remainder(BigInteger)
+  BigInteger setBit(int)
+  BigInteger shiftLeft(int)
+  BigInteger shiftRight(int)
+  short shortValueExact()
+  int signum()
+  BigInteger subtract(BigInteger)
+  boolean testBit(int)
+  byte[] toByteArray()
+  String toString(int)
+  BigInteger valueOf(long)
+  BigInteger xor(BigInteger)
+}
+
+class MathContext -> java.math.MathContext extends Object {
+  MathContext DECIMAL128
+  MathContext DECIMAL32
+  MathContext DECIMAL64
+  MathContext UNLIMITED
+  MathContext <init>(int)
+  MathContext <init>(int,RoundingMode)
+  int getPrecision()
+  RoundingMode getRoundingMode()
+}
+
+#### Enums
+
+class RoundingMode -> java.math.RoundingMode extends Enum,Object {
+  RoundingMode CEILING
+  RoundingMode DOWN
+  RoundingMode FLOOR
+  RoundingMode HALF_DOWN
+  RoundingMode HALF_EVEN
+  RoundingMode HALF_UP
+  RoundingMode UNNECESSARY
+  RoundingMode UP
+  RoundingMode valueOf(String)
+  RoundingMode[] values()
+}
+
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.text.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.text.txt
new file mode 100644
index 00000000000..eac725cfb94
--- /dev/null
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.text.txt
@@ -0,0 +1,486 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#
+# Painless definition file. This defines the hierarchy of classes,
+# what methods and fields they have, etc.
+#
+
+#### Interfaces
+
+class AttributedCharacterIterator -> java.text.AttributedCharacterIterator extends CharacterIterator {
+  Set getAllAttributeKeys()
+  def getAttribute(AttributedCharacterIterator.Attribute)
+  Map getAttributes()
+  int getRunLimit()
+  int getRunLimit(Set)
+  int getRunStart()
+  int getRunStart(Set)
+}
+
+class CharacterIterator -> java.text.CharacterIterator {
+  char DONE
+  def clone()
+  char current()
+  char first()
+  int getBeginIndex()
+  int getEndIndex()
+  int getIndex()
+  char last()
+  char next()
+  char previous()
+  char setIndex(int)
+}
+
+#### Classes
+
+class Annotation -> java.text.Annotation extends Object {
+  Annotation <init>(Object)
+  def getValue()
+}
+
+class AttributedCharacterIterator.Attribute -> java.text.AttributedCharacterIterator$Attribute extends Object {
+  AttributedCharacterIterator.Attribute INPUT_METHOD_SEGMENT
+  AttributedCharacterIterator.Attribute LANGUAGE
+  AttributedCharacterIterator.Attribute READING
+}
+
+class AttributedString -> java.text.AttributedString extends Object {
+  AttributedString <init>(String)
+  AttributedString <init>(String,Map)
+  void addAttribute(AttributedCharacterIterator.Attribute,Object)
+  void addAttribute(AttributedCharacterIterator.Attribute,Object,int,int)
+  void addAttributes(Map,int,int)
+  AttributedCharacterIterator getIterator()
+  AttributedCharacterIterator getIterator(AttributedCharacterIterator.Attribute[])
+  AttributedCharacterIterator getIterator(AttributedCharacterIterator.Attribute[],int,int)
+}
+
+class Bidi -> java.text.Bidi extends Object {
+  int DIRECTION_DEFAULT_LEFT_TO_RIGHT
+  int DIRECTION_DEFAULT_RIGHT_TO_LEFT
+  int DIRECTION_LEFT_TO_RIGHT
+  int DIRECTION_RIGHT_TO_LEFT
+  Bidi <init>(AttributedCharacterIterator)
+  Bidi <init>(char[],int,byte[],int,int,int)
+  Bidi <init>(String,int)
+  boolean baseIsLeftToRight()
+  Bidi createLineBidi(int,int)
+  int getBaseLevel()
+  int getLength()
+  int getLevelAt(int)
+  int getRunCount()
+  int getRunLevel(int)
+  int getRunLimit(int)
+  int getRunStart(int)
+  boolean isLeftToRight()
+  boolean isMixed()
+  boolean isRightToLeft()
+  void reorderVisually(byte[],int,Object[],int,int)
+  boolean requiresBidi(char[],int,int)
+}
+
+class BreakIterator -> java.text.BreakIterator extend Object {
+  int DONE
+  def clone()
+  int current()
+  int first()
+  int following(int)
+  Locale[] getAvailableLocales()
+  BreakIterator getCharacterInstance()
+  BreakIterator getCharacterInstance(Locale)
+  BreakIterator getLineInstance()
+  BreakIterator getLineInstance(Locale)
+  BreakIterator getSentenceInstance()
+  BreakIterator getSentenceInstance(Locale)
+  CharacterIterator getText()
+  BreakIterator getWordInstance()
+  BreakIterator getWordInstance(Locale)
+  boolean isBoundary(int)
+  int last()
+  int next()
+  int next(int)
+  int preceding(int)
+  int previous()
+  void setText(String)
+}
+
+class ChoiceFormat -> java.text.ChoiceFormat extends NumberFormat,Format,Object {
+  ChoiceFormat <init>(double[],String[])
+  ChoiceFormat <init>(String)
+  void applyPattern(String)
+  def[] getFormats()
+  double[] getLimits()
+  double nextDouble(double)
+  double nextDouble(double,boolean)
+  double previousDouble(double)
+  void setChoices(double[],String[])
+  String toPattern()
+}
+
+class CollationElementIterator -> java.text.CollationElementIterator extends Object {
+  int NULLORDER
+  int getMaxExpansion(int)
+  int getOffset()
+  int next()
+  int previous()
+  int primaryOrder(int)
+  void reset()
+  short secondaryOrder(int)
+  void setOffset(int)
+  void setText(String)
+  short tertiaryOrder(int)
+}
+
+class CollationKey -> java.text.CollationKey extends Comparable,Object {
+  String getSourceString()
+  byte[] toByteArray()
+}
+
+class Collator -> java.text.Collator extends Comparator,Object {
+  int CANONICAL_DECOMPOSITION
+  int FULL_DECOMPOSITION
+  int IDENTICAL
+  int NO_DECOMPOSITION
+  int PRIMARY
+  int SECONDARY
+  int TERTIARY
+  def clone()
+  boolean equals(String,String)
+  Locale[] getAvailableLocales()
+  CollationKey getCollationKey(String)
+  int getDecomposition()
+  Collator getInstance()
+  Collator getInstance(Locale)
+  int getStrength()
+  void setDecomposition(int)
+  void setStrength(int)
+}
+
+class DateFormat -> java.text.DateFormat extends Format,Object {
+  int AM_PM_FIELD
+  int DATE_FIELD
+  int DAY_OF_WEEK_FIELD
+  int DAY_OF_WEEK_IN_MONTH_FIELD
+  int DAY_OF_YEAR_FIELD
+  int DEFAULT
+  int ERA_FIELD
+  int FULL
+  int HOUR_OF_DAY0_FIELD
+  int HOUR_OF_DAY1_FIELD
+  int HOUR0_FIELD
+  int HOUR1_FIELD
+  int LONG
+  int MEDIUM
+  int MILLISECOND_FIELD
+  int MINUTE_FIELD
+  int MONTH_FIELD
+  int SECOND_FIELD
+  int SHORT
+  int TIMEZONE_FIELD
+  int WEEK_OF_MONTH_FIELD
+  int WEEK_OF_YEAR_FIELD
+  int YEAR_FIELD
+  Locale[] getAvailableLocales()
+  Calendar getCalendar()
+  DateFormat getDateInstance()
+  DateFormat getDateInstance(int)
+  DateFormat getDateInstance(int,Locale)
+  DateFormat getDateTimeInstance()
+  DateFormat getDateTimeInstance(int,int)
+  DateFormat getDateTimeInstance(int,int,Locale)
+  DateFormat getInstance()
+  NumberFormat getNumberFormat()
+  DateFormat getTimeInstance()
+  DateFormat getTimeInstance(int)
+  DateFormat getTimeInstance(int,Locale)
+  TimeZone getTimeZone()
+  boolean isLenient()
+  Date parse(String)
+  Date parse(String,ParsePosition)
+  void setCalendar(Calendar)
+  void setLenient(boolean)
+  void setNumberFormat(NumberFormat)
+  void setTimeZone(TimeZone)
+}
+
+class DateFormat.Field -> java.text.DateFormat$Field extends Format.Field,AttributedCharacterIterator.Attribute,Object {
+  DateFormat.Field AM_PM
+  DateFormat.Field DAY_OF_MONTH
+  DateFormat.Field DAY_OF_WEEK
+  DateFormat.Field DAY_OF_WEEK_IN_MONTH
+  DateFormat.Field DAY_OF_YEAR
+  DateFormat.Field ERA
+  DateFormat.Field HOUR_OF_DAY0
+  DateFormat.Field HOUR_OF_DAY1
+  DateFormat.Field HOUR0
+  DateFormat.Field HOUR1
+  DateFormat.Field MILLISECOND
+  DateFormat.Field MINUTE
+  DateFormat.Field MONTH
+  DateFormat.Field SECOND
+  DateFormat.Field TIME_ZONE
+  DateFormat.Field WEEK_OF_MONTH
+  DateFormat.Field WEEK_OF_YEAR
+  DateFormat.Field YEAR
+  int getCalendarField()
+  DateFormat.Field ofCalendarField(int)
+}
+
+class DateFormatSymbols -> java.text.DateFormatSymbols extends Object {
+  DateFormatSymbols <init>()
+  DateFormatSymbols <init>(Locale)
+  def clone()
+  String[] getAmPmStrings()
+  Locale[] getAvailableLocales()
+  String[] getEras()
+  DateFormatSymbols getInstance()
+  DateFormatSymbols getInstance(Locale)
+  String getLocalPatternChars()
+  String[] getMonths()
+  String[] getShortMonths()
+  String[] getShortWeekdays()
+  String[] getWeekdays()
+  String[][] getZoneStrings()
+  int hashCode()
+  void setAmPmStrings(String[])
+  void setEras(String[])
+  void setLocalPatternChars(String)
+  void setMonths(String[])
+  void setShortMonths(String[])
+  void setShortWeekdays(String[])
+  void setWeekdays(String[])
+  void setZoneStrings(String[][])
+}
+
+class DecimalFormat -> java.text.DecimalFormat extends NumberFormat,Format,Object {
+  DecimalFormat <init>()
+  DecimalFormat <init>(String)
+  DecimalFormat <init>(String,DecimalFormatSymbols)
+  void applyLocalizedPattern(String)
+  void applyPattern(String)
+  DecimalFormatSymbols getDecimalFormatSymbols()
+  int getGroupingSize()
+  int getMultiplier()
+  String getNegativePrefix()
+  String getNegativeSuffix()
+  String getPositivePrefix()
+  String getPositiveSuffix()
+  boolean isDecimalSeparatorAlwaysShown()
+  boolean isParseBigDecimal()
+  void setDecimalFormatSymbols(DecimalFormatSymbols)
+  void setDecimalSeparatorAlwaysShown(boolean)
+  void setGroupingSize(int)
+  void setMultiplier(int)
+  void setNegativePrefix(String)
+  void setNegativeSuffix(String)
+  void setPositivePrefix(String)
+  void setPositiveSuffix(String)
+  void setParseBigDecimal(boolean)
+  String toLocalizedPattern()
+  String toPattern()
+}
+
+class DecimalFormatSymbols -> java.text.DecimalFormatSymbols extends Object {
+  DecimalFormatSymbols <init>()
+  DecimalFormatSymbols <init>(Locale)
+  def clone()
+  Locale[] getAvailableLocales()
+  Currency getCurrency()
+  String getCurrencySymbol()
+  char getDecimalSeparator()
+  char getDigit()
+  String getExponentSeparator()
+  char getGroupingSeparator()
+  String getInfinity()
+  DecimalFormatSymbols getInstance()
+  DecimalFormatSymbols getInstance(Locale)
+  String getInternationalCurrencySymbol()
+  char getMinusSign()
+  char getMonetaryDecimalSeparator()
+  String getNaN()
+  char getPatternSeparator()
+  char getPercent()
+  char getPerMill()
+  char getZeroDigit()
+  void setCurrency(Currency)
+  void setCurrencySymbol(String)
+  void setDecimalSeparator(char)
+  void setDigit(char)
+  void setExponentSeparator(String)
+  void setGroupingSeparator(char)
+  void setInfinity(String)
+  void setInternationalCurrencySymbol(String)
+  void setMinusSign(char)
+  void setMonetaryDecimalSeparator(char)
+  void setNaN(String)
+  void setPatternSeparator(char)
+  void setPercent(char)
+  void setPerMill(char)
+  void setZeroDigit(char)
+}
+
+class FieldPosition -> java.text.FieldPosition extends Object {
+  FieldPosition <init>(int)
+  FieldPosition <init>(Format.Field,int)
+  int getBeginIndex()
+  int getEndIndex()
+  int getField()
+  Format.Field getFieldAttribute()
+  void setBeginIndex(int)
+  void setEndIndex(int)
+}
+
+class Format -> java.text.Format extends Object {
+  def clone()
+  String format(Object)
+  StringBuffer format(Object,StringBuffer,FieldPosition)
+  AttributedCharacterIterator formatToCharacterIterator(Object)
+  Object parseObject(String)
+  Object parseObject(String,ParsePosition)
+}
+
+class Format.Field -> java.text.Format$Field extends AttributedCharacterIterator.Attribute,Object {
+}
+
+class MessageFormat -> java.text.MessageFormat extends Format,Object {
+  void applyPattern(String)
+  String format(String,Object[])
+  Format[] getFormats()
+  Format[] getFormatsByArgumentIndex()
+  Locale getLocale()
+  Object[] parse(String)
+  Object[] parse(String,ParsePosition)
+  void setFormat(int,Format)
+  void setFormatByArgumentIndex(int,Format)
+  void setFormats(Format[])
+  void setFormatsByArgumentIndex(Format[])
+  void setLocale(Locale)
+  String toPattern()
+}
+
+class MessageFormat.Field -> java.text.MessageFormat$Field extends Format.Field,AttributedCharacterIterator.Attribute,Object {
+  MessageFormat.Field ARGUMENT
+}
+
+class Normalizer -> java.text.Normalizer extends Object {
+  boolean isNormalized(CharSequence,Normalizer.Form)
+  String normalize(CharSequence,Normalizer.Form)
+}
+
+class NumberFormat -> java.text.NumberFormat extends Format,Object {
+  int FRACTION_FIELD
+  int INTEGER_FIELD
+  Locale[] getAvailableLocales()
+  Currency getCurrency()
+  NumberFormat getCurrencyInstance()
+  NumberFormat getCurrencyInstance(Locale)
+  NumberFormat getInstance()
+  NumberFormat getInstance(Locale)
+  NumberFormat getIntegerInstance()
+  NumberFormat getIntegerInstance(Locale)
+  int getMaximumFractionDigits()
+  int getMaximumIntegerDigits()
+  int getMinimumFractionDigits()
+  int getMinimumIntegerDigits()
+  NumberFormat getNumberInstance()
+  NumberFormat getNumberInstance(Locale)
+  NumberFormat getPercentInstance()
+  NumberFormat getPercentInstance(Locale)
+  RoundingMode getRoundingMode()
+  boolean isGroupingUsed()
+  boolean isParseIntegerOnly()
+  Number parse(String)
+  Number parse(String,ParsePosition)
+  void setCurrency(Currency)
+  void setGroupingUsed(boolean)
+  void setMaximumFractionDigits(int)
+  void setMaximumIntegerDigits(int)
+  void setMinimumFractionDigits(int)
+  void setMinimumIntegerDigits(int)
+  void setParseIntegerOnly(boolean)
+  void setRoundingMode(RoundingMode)
+}
+
+class NumberFormat.Field -> java.text.NumberFormat$Field extends Format.Field,AttributedCharacterIterator.Attribute,Object {
+  NumberFormat.Field CURRENCY
+  NumberFormat.Field DECIMAL_SEPARATOR
+  NumberFormat.Field EXPONENT
+  NumberFormat.Field EXPONENT_SIGN
+  NumberFormat.Field EXPONENT_SYMBOL
+  NumberFormat.Field FRACTION
+  NumberFormat.Field GROUPING_SEPARATOR
+  NumberFormat.Field INTEGER
+  NumberFormat.Field PERCENT
+  NumberFormat.Field PERMILLE
+  NumberFormat.Field SIGN
+}
+
+class ParsePosition -> java.text.ParsePosition extends Object {
+  ParsePosition <init>(int)
+  int getErrorIndex()
+  int getIndex()
+  void setErrorIndex(int)
+  void setIndex(int)
+}
+
+class RuleBasedCollator -> java.text.RuleBasedCollator extends Collator,Comparator,Object {
+  RuleBasedCollator <init>(String)
+  CollationElementIterator getCollationElementIterator(String)
+  String getRules()
+}
+
+class SimpleDateFormat -> java.text.SimpleDateFormat extends DateFormat,Format,Object {
+  SimpleDateFormat <init>()
+  SimpleDateFormat <init>(String)
+  SimpleDateFormat <init>(String,Locale)
+  void applyLocalizedPattern(String)
+  void applyPattern(String)
+  Date get2DigitYearStart()
+  DateFormatSymbols getDateFormatSymbols()
+  void setDateFormatSymbols(DateFormatSymbols)
+  void set2DigitYearStart(Date)
+  String toLocalizedPattern()
+  String toPattern()
+}
+
+class StringCharacterIterator -> java.text.StringCharacterIterator extends CharacterIterator,Object {
+  StringCharacterIterator <init>(String)
+  StringCharacterIterator <init>(String,int)
+  StringCharacterIterator <init>(String,int,int,int)
+  void setText(String)
+}
+
+#### Enums
+
+class Normalizer.Form -> java.text.Normalizer$Form extends Enum,Object {
+  Normalizer.Form NFC
+  Normalizer.Form NFD
+  Normalizer.Form NFKC
+  Normalizer.Form NFKD
+  Normalizer.Form valueOf(String)
+  Normalizer.Form[] values()
+}
+
+#### Exceptions
+
+class ParseException -> java.text.ParseException extends Exception,Object {
+  ParseException <init>(String,int)
+  int getErrorOffset()
+}
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.function.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.function.txt
new file mode 100644
index 00000000000..969a8d6fb46
--- /dev/null
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.function.txt
@@ -0,0 +1,232 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#
+# Painless definition file. This defines the hierarchy of classes,
+# what methods and fields they have, etc.
+#
+
+#### Interfaces
+
+class BiConsumer -> java.util.function.BiConsumer {
+  void accept(def,def)
+  BiConsumer andThen(BiConsumer)
+}
+
+class BiFunction -> java.util.function.BiFunction {
+  BiFunction andThen(Function)
+  def apply(def,def)
+}
+
+class BinaryOperator -> java.util.function.BinaryOperator extends BiFunction {
+  BinaryOperator maxBy(Comparator)
+  BinaryOperator minBy(Comparator)
+}
+
+class BiPredicate -> java.util.function.BiPredicate {
+  BiPredicate and(BiPredicate)
+  BiPredicate negate()
+  BiPredicate or(BiPredicate)
+  boolean test(def,def)
+}
+
+class BooleanSupplier -> java.util.function.BooleanSupplier {
+  boolean getAsBoolean()
+}
+
+class Consumer -> java.util.function.Consumer {
+  void accept(def)
+  Consumer andThen(Consumer)
+}
+
+class DoubleBinaryOperator -> java.util.function.DoubleBinaryOperator {
+  double applyAsDouble(double,double)
+}
+
+class DoubleConsumer -> java.util.function.DoubleConsumer {
+  void accept(double)
+  DoubleConsumer andThen(DoubleConsumer)
+}
+
+class DoubleFunction -> java.util.function.DoubleFunction {
+  def apply(double)
+}
+
+class DoublePredicate -> java.util.function.DoublePredicate {
+  DoublePredicate and(DoublePredicate)
+  DoublePredicate negate()
+  DoublePredicate or(DoublePredicate)
+  boolean test(double)
+}
+
+class DoubleSupplier -> java.util.function.DoubleSupplier {
+  double getAsDouble()
+}
+
+class DoubleToIntFunction -> java.util.function.DoubleToIntFunction {
+  int applyAsInt(double)
+}
+
+class DoubleToLongFunction -> java.util.function.DoubleToLongFunction {
+  long applyAsLong(double)
+}
+
+class DoubleUnaryOperator -> java.util.function.DoubleUnaryOperator {
+  DoubleUnaryOperator andThen(DoubleUnaryOperator)
+  double applyAsDouble(double)
+  DoubleUnaryOperator compose(DoubleUnaryOperator)
+  DoubleUnaryOperator identity()
+}
+
+class Function -> java.util.function.Function {
+  Function andThen(Function)
+  def apply(def)
+  Function compose(Function)
+  Function identity()
+}
+
+class IntBinaryOperator -> java.util.function.IntBinaryOperator {
+  int applyAsInt(int,int)
+}
+
+class IntConsumer -> java.util.function.IntConsumer {
+  void accept(int)
+  IntConsumer andThen(IntConsumer)
+}
+
+class IntFunction -> java.util.function.IntFunction {
+  def apply(int)
+}
+
+class IntPredicate -> java.util.function.IntPredicate {
+  IntPredicate and(IntPredicate)
+  IntPredicate negate()
+  IntPredicate or(IntPredicate)
+  boolean test(int)
+}
+
+class IntSupplier -> java.util.function.IntSupplier {
+  int getAsInt()
+}
+
+class IntToDoubleFunction -> java.util.function.IntToDoubleFunction {
+  double applyAsDouble(int)
+}
+
+class IntToLongFunction -> java.util.function.IntToLongFunction {
+  long applyAsLong(int)
+}
+
+class IntUnaryOperator -> java.util.function.IntUnaryOperator {
+  IntUnaryOperator andThen(IntUnaryOperator)
+  int applyAsInt(int)
+  IntUnaryOperator compose(IntUnaryOperator)
+  IntUnaryOperator identity()
+}
+
+class LongBinaryOperator -> java.util.function.LongBinaryOperator {
+  long applyAsLong(long,long)
+}
+
+class LongConsumer -> java.util.function.LongConsumer {
+  void accept(long)
+  LongConsumer andThen(LongConsumer)
+}
+
+class LongFunction -> java.util.function.LongFunction {
+  def apply(long)
+}
+
+class LongPredicate -> java.util.function.LongPredicate {
+  LongPredicate and(LongPredicate)
+  LongPredicate negate()
+  LongPredicate or(LongPredicate)
+  boolean test(long)
+}
+
+class LongSupplier -> java.util.function.LongSupplier {
+  long getAsLong()
+}
+
+class LongToDoubleFunction -> java.util.function.LongToDoubleFunction {
+  double applyAsDouble(long)
+}
+
+class LongToIntFunction -> java.util.function.LongToIntFunction {
+  int applyAsInt(long)
+}
+
+class LongUnaryOperator -> java.util.function.LongUnaryOperator {
+  LongUnaryOperator andThen(LongUnaryOperator)
+  long applyAsLong(long)
+  LongUnaryOperator compose(LongUnaryOperator)
+  LongUnaryOperator identity()
+}
+
+class ObjDoubleConsumer -> java.util.function.ObjDoubleConsumer {
+  void accept(def,double)
+}
+
+class ObjIntConsumer -> java.util.function.ObjIntConsumer {
+  void accept(def,int)
+}
+
+class ObjLongConsumer -> java.util.function.ObjLongConsumer {
+  void accept(def,long)
+}
+
+class Predicate -> java.util.function.Predicate {
+  Predicate and(Predicate)
+  Predicate isEqual(def)
+  Predicate negate()
+  Predicate or(Predicate)
+  boolean test(def)
+}
+
+class Supplier -> java.util.function.Supplier {
+  def get()
+}
+
+class ToDoubleBiFunction -> java.util.function.ToDoubleBiFunction {
+  double applyAsDouble(def,def)
+}
+
+class ToDoubleFunction -> java.util.function.ToDoubleFunction {
+  double applyAsDouble(def)
+}
+
+class ToIntBiFunction -> java.util.function.ToIntBiFunction {
+  int applyAsInt(def,def)
+}
+
+class ToIntFunction -> java.util.function.ToIntFunction {
+  int applyAsInt(def)
+}
+
+class ToLongBiFunction -> java.util.function.ToLongBiFunction {
+  long applyAsLong(def,def)
+}
+
+class ToLongFunction -> java.util.function.ToLongFunction {
+  long applyAsLong(def)
+}
+
+class UnaryOperator -> java.util.function.UnaryOperator extends Function {
+  UnaryOperator identity()
+}
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.stream.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.stream.txt
new file mode 100644
index 00000000000..d24cf8c0424
--- /dev/null
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.stream.txt
@@ -0,0 +1,273 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#
+# Painless definition file. This defines the hierarchy of classes,
+# what methods and fields they have, etc.
+#
+
+#### Interfaces
+
+class BaseStream -> java.util.stream.BaseStream {
+  void close()
+  boolean isParallel()
+  Iterator iterator()
+  BaseStream sequential()
+  Spliterator spliterator()
+  BaseStream unordered()
+}
+
+class Collector -> java.util.stream.Collector {
+  BiConsumer accumulator()
+  Set characteristics()
+  BinaryOperator combiner()
+  Function finisher()
+  Collector of(Supplier,BiConsumer,BinaryOperator,Function,Collector.Characteristics[])
+  Collector of(Supplier,BiConsumer,BinaryOperator,Collector.Characteristics[])
+  Supplier supplier()
+}
+
+class DoubleStream -> java.util.stream.DoubleStream extends BaseStream {
+  boolean allMatch(DoublePredicate)
+  boolean anyMatch(DoublePredicate)
+  OptionalDouble average()
+  Stream boxed()
+  DoubleStream.Builder builder()
+  def collect(Supplier,ObjDoubleConsumer,BiConsumer)
+  DoubleStream concat(DoubleStream,DoubleStream)
+  long count()
+  DoubleStream distinct()
+  DoubleStream empty()
+  DoubleStream filter(DoublePredicate)
+  OptionalDouble findAny()
+  OptionalDouble findFirst()
+  DoubleStream flatMap(DoubleFunction)
+  void forEach(DoubleConsumer)
+  void forEachOrdered(DoubleConsumer)
+  PrimitiveIterator.OfDouble iterator()
+  DoubleStream limit(long)
+  DoubleStream map(DoubleUnaryOperator)
+  IntStream mapToInt(DoubleToIntFunction)
+  LongStream mapToLong(DoubleToLongFunction)
+  Stream mapToObj(DoubleFunction)
+  OptionalDouble max()
+  OptionalDouble min()
+  boolean noneMatch(DoublePredicate)
+  DoubleStream of(double[])
+  DoubleStream peek(DoubleConsumer)
+  OptionalDouble reduce(DoubleBinaryOperator)
+  double reduce(double,DoubleBinaryOperator)
+  DoubleStream sequential()
+  DoubleStream skip(long)
+  DoubleStream sorted()
+  Spliterator.OfDouble spliterator()
+  double sum()
+  DoubleSummaryStatistics summaryStatistics()
+  double[] toArray()
+}
+
+class DoubleStream.Builder -> java.util.stream.DoubleStream$Builder extends DoubleConsumer {
+  DoubleStream.Builder add(double)
+  DoubleStream build()
+}
+
+class IntStream -> java.util.stream.IntStream extends BaseStream {
+  boolean allMatch(IntPredicate)
+  boolean anyMatch(IntPredicate)
+  DoubleStream asDoubleStream()
+  LongStream asLongStream()
+  OptionalDouble average()
+  Stream boxed()
+  IntStream.Builder builder()
+  def collect(Supplier,ObjIntConsumer,BiConsumer)
+  IntStream concat(IntStream,IntStream)
+  long count()
+  IntStream distinct()
+  IntStream empty()
+  IntStream filter(IntPredicate)
+  OptionalInt findAny()
+  OptionalInt findFirst()
+  IntStream flatMap(IntFunction)
+  void forEach(IntConsumer)
+  void forEachOrdered(IntConsumer)
+  PrimitiveIterator.OfInt iterator()
+  IntStream limit(long)
+  IntStream map(IntUnaryOperator)
+  DoubleStream mapToDouble(IntToDoubleFunction)
+  LongStream mapToLong(IntToLongFunction)
+  Stream mapToObj(IntFunction)
+  OptionalInt max()
+  OptionalInt min()
+  boolean noneMatch(IntPredicate)
+  IntStream of(int[])
+  IntStream peek(IntConsumer)
+  IntStream range(int,int)
+  IntStream rangeClosed(int,int)
+  OptionalInt reduce(IntBinaryOperator)
+  int reduce(int,IntBinaryOperator)
+  IntStream sequential()
+  IntStream skip(long)
+  IntStream sorted()
+  Spliterator.OfInt spliterator()
+  int sum()
+  IntSummaryStatistics summaryStatistics()
+  int[] toArray()
+}
+
+class IntStream.Builder -> java.util.stream.IntStream$Builder extends IntConsumer {
+  IntStream.Builder add(int)
+  IntStream build()
+}
+
+class LongStream -> java.util.stream.LongStream extends BaseStream {
+  boolean allMatch(LongPredicate)
+  boolean anyMatch(LongPredicate)
+  DoubleStream asDoubleStream()
+  OptionalDouble average()
+  Stream boxed()
+  LongStream.Builder builder()
+  def collect(Supplier,ObjLongConsumer,BiConsumer)
+  LongStream concat(LongStream,LongStream)
+  long count()
+  LongStream distinct()
+  LongStream empty()
+  LongStream filter(LongPredicate)
+  OptionalLong findAny()
+  OptionalLong findFirst()
+  LongStream flatMap(LongFunction)
+  void forEach(LongConsumer)
+  void forEachOrdered(LongConsumer)
+  PrimitiveIterator.OfLong iterator()
+  LongStream limit(long)
+  LongStream map(LongUnaryOperator)
+  DoubleStream mapToDouble(LongToDoubleFunction)
+  IntStream mapToInt(LongToIntFunction)
+  Stream mapToObj(LongFunction)
+  OptionalLong max()
+  OptionalLong min()
+  boolean noneMatch(LongPredicate)
+  LongStream of(long[])
+  LongStream peek(LongConsumer)
+  LongStream range(long,long)
+  LongStream rangeClosed(long,long)
+  OptionalLong reduce(LongBinaryOperator)
+  long reduce(long,LongBinaryOperator)
+  LongStream sequential()
+  LongStream skip(long)
+  LongStream sorted()
+  Spliterator.OfLong spliterator()
+  long sum()
+  LongSummaryStatistics summaryStatistics()
+  long[] toArray()
+}
+
+class LongStream.Builder -> java.util.stream.LongStream$Builder extends LongConsumer {
+  LongStream.Builder add(long)
+  LongStream build()
+}
+
+class Stream -> java.util.stream.Stream extends BaseStream {
+  boolean allMatch(Predicate)
+  boolean anyMatch(Predicate)
+  Stream.Builder builder()
+  def collect(Collector)
+  def collect(Supplier,BiConsumer,BiConsumer)
+  Stream concat(Stream,Stream)
+  long count()
+  Stream distinct()
+  Stream empty()
+  Stream filter(Predicate)
+  Optional findAny()
+  Optional findFirst()
+  Stream flatMap(Function)
+  DoubleStream flatMapToDouble(Function)
+  IntStream flatMapToInt(Function)
+  LongStream flatMapToLong(Function)
+  void forEach(Consumer)
+  void forEachOrdered(Consumer)
+  Stream limit(long)
+  Stream map(Function)
+  DoubleStream mapToDouble(ToDoubleFunction)
+  IntStream mapToInt(ToIntFunction)
+  LongStream mapToLong(ToLongFunction)
+  Optional max(Comparator)
+  Optional min(Comparator)
+  boolean noneMatch(Predicate)
+  Stream of(def[])
+  Stream peek(Consumer)
+  Optional reduce(BinaryOperator)
+  def reduce(def,BinaryOperator)
+  def reduce(def,BiFunction,BinaryOperator)
+  Stream skip(long)
+  Stream sorted()
+  Stream sorted(Comparator)
+  def[] toArray()
+  def[] toArray(IntFunction)
+}
+
+class Stream.Builder -> java.util.stream.Stream$Builder extends Consumer {
+  Stream.Builder add(def)
+  Stream build()
+}
+
+#### Classes
+
+class Collectors -> java.util.stream.Collectors extends Object {
+  Collector averagingDouble(ToDoubleFunction)
+  Collector averagingInt(ToIntFunction)
+  Collector averagingLong(ToLongFunction)
+  Collector collectingAndThen(Collector,Function)
+  Collector counting()
+  Collector groupingBy(Function)
+  Collector groupingBy(Function,Collector)
+  Collector groupingBy(Function,Supplier,Collector)
+  Collector joining()
+  Collector joining(CharSequence)
+  Collector joining(CharSequence,CharSequence,CharSequence)
+  Collector mapping(Function,Collector)
+  Collector maxBy(Comparator)
+  Collector minBy(Comparator)
+  Collector partitioningBy(Predicate)
+  Collector partitioningBy(Predicate,Collector)
+  Collector reducing(BinaryOperator)
+  Collector reducing(def,BinaryOperator)
+  Collector reducing(def,Function,BinaryOperator)
+  Collector summarizingDouble(ToDoubleFunction)
+  Collector summarizingInt(ToIntFunction)
+  Collector summarizingLong(ToLongFunction)
+  Collector summingDouble(ToDoubleFunction)
+  Collector summingInt(ToIntFunction)
+  Collector summingLong(ToLongFunction)
+  Collector toCollection(Supplier)
+  Collector toList()
+  Collector toMap(Function,Function)
+  Collector toMap(Function,Function,BinaryOperator)
+  Collector toMap(Function,Function,BinaryOperator,Supplier)
+  Collector toSet()
+}
+
+#### Enums
+
+class Collector.Characteristics -> java.util.stream.Collector$Characteristics extends Enum,Object {
+  Collector.Characteristics CONCURRENT
+  Collector.Characteristics IDENTITY_FINISH
+  Collector.Characteristics UNORDERED
+  Collector.Characteristics valueOf(String)
+  Collector.Characteristics[] values()
+}
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt
new file mode 100644
index 00000000000..9be890f15d5
--- /dev/null
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt
@@ -0,0 +1,1148 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#
+# Painless definition file. This defines the hierarchy of classes,
+# what methods and fields they have, etc.
+#
+
+#### Interfaces
+
+class Collection -> java.util.Collection extends Iterable {
+  boolean add(def)
+  boolean addAll(Collection)
+  void clear()
+  boolean contains(def)
+  boolean containsAll(Collection)
+  boolean isEmpty()
+  boolean removeAll(Collection)
+  boolean removeIf(Predicate)
+  boolean retainAll(Collection)
+  int size()
+  Spliterator spliterator()
+  Stream stream()
+  def[] toArray()
+  def[] toArray(def[])
+}
+
+class Comparator -> java.util.Comparator {
+  int compare(def,def)
+  Comparator comparing(Function)
+  Comparator comparing(Function,Comparator)
+  Comparator comparingDouble(ToDoubleFunction)
+  Comparator comparingInt(ToIntFunction)
+  Comparator comparingLong(ToLongFunction)
+  boolean equals(Object)
+  Comparator naturalOrder()
+  Comparator nullsFirst(Comparator)
+  Comparator nullsLast(Comparator)
+  Comparator reversed()
+  Comparator reverseOrder()
+  Comparator thenComparing(Comparator)
+  Comparator thenComparing(Function,Comparator)
+  Comparator thenComparingDouble(ToDoubleFunction)
+  Comparator thenComparingInt(ToIntFunction)
+  Comparator thenComparingLong(ToLongFunction)
+}
+
+class Deque -> java.util.Deque extends Queue,Collection,Iterable {
+  void addFirst(def)
+  void addLast(def)
+  Iterator descendingIterator()
+  def getFirst()
+  def getLast()
+  boolean offerFirst(def)
+  boolean offerLast(def)
+  def peekFirst()
+  def peekLast()
+  def pollFirst()
+  def pollLast()
+  def pop()
+  void push(def)
+  boolean remove(def)
+  def removeFirst()
+  boolean removeFirstOccurrence(def)
+  def removeLast()
+  boolean removeLastOccurrence(def)
+}
+
+class Enumeration -> java.util.Enumeration {
+  boolean hasMoreElements()
+  def nextElement()
+}
+
+class EventListener -> java.util.EventListener {
+}
+
+class Formattable -> java.util.Formattable {
+  void formatTo(Formatter,int,int,int)
+}
+
+class Iterator -> java.util.Iterator {
+  void forEachRemaining(Consumer)
+  boolean hasNext()
+  def next()
+  void remove()
+}
+
+class List -> java.util.List extends Collection,Iterable {
+  void add(int,def)
+  boolean addAll(int,Collection)
+  boolean equals(Object)
+  def get(int)
+  int hashCode()
+  int indexOf(def)
+  int lastIndexOf(def)
+  ListIterator listIterator()
+  ListIterator listIterator(int)
+  def remove(int)
+  void replaceAll(UnaryOperator)
+  def set(int,def)
+  # TODO: wtf?
+  int getLength/size()
+  void sort(Comparator)
+  List subList(int,int)
+}
+
+class ListIterator -> java.util.ListIterator extends Iterator {
+  void add(def)
+  boolean hasPrevious()
+  int nextIndex()
+  int previousIndex()
+  void set(def)
+}
+
+class Map -> java.util.Map {
+  void clear()
+  def compute(def,BiFunction)
+  def computeIfAbsent(def,Function)
+  def computeIfPresent(def,BiFunction)
+  boolean containsKey(def)
+  boolean containsValue(def)
+  Set entrySet()
+  boolean equals(Object)
+  void forEach(BiConsumer)
+  def get(def)
+  def getOrDefault(def,def)
+  boolean isEmpty()
+  Set keySet()
+  def merge(def,def,BiFunction)
+  def put(def,def)
+  void putAll(Map)
+  def putIfAbsent(def,def)
+  def remove(def)
+  boolean remove(def,def)
+  def replace(def,def)
+  boolean replace(def,def,def)
+  void replaceAll(BiFunction)
+  int size()
+  Collection values()
+}
+
+class Map.Entry -> java.util.Map$Entry {
+  Comparator comparingByKey()
+  Comparator comparingByKey(Comparator)
+  Comparator comparingByValue()
+  Comparator comparingByValue(Comparator)
+  boolean equals(Object)
+  def getKey()
+  def getValue()
+  int hashCode()
+  def setValue(def)
+}
+
+class NavigableMap -> java.util.NavigableMap extends SortedMap,Map {
+  Map.Entry ceilingEntry(def)
+  def ceilingKey(def)
+  NavigableSet descendingKeySet()
+  NavigableMap descendingMap()
+  Map.Entry firstEntry()
+  Map.Entry floorEntry(def)
+  def floorKey(def)
+  NavigableMap headMap(def,boolean)
+  Map.Entry higherEntry(def)
+  def higherKey(def)
+  Map.Entry lastEntry()
+  Map.Entry lowerEntry(def)
+  NavigableSet navigableKeySet()
+  Map.Entry pollFirstEntry()
+  Map.Entry pollLastEntry()
+  NavigableMap subMap(def,boolean,def,boolean)
+  NavigableMap tailMap(def,boolean)
+}
+
+class NavigableSet -> java.util.NavigableSet extends SortedSet,Set,Collection,Iterable {
+  def ceiling(def)
+  Iterator descendingIterator()
+  NavigableSet descendingSet()
+  def floor(def)
+  NavigableSet headSet(def,boolean)
+  def higher(def)
+  def lower(def)
+  def pollFirst()
+  def pollLast()
+  NavigableSet subSet(def,boolean,def,boolean)
+  NavigableSet tailSet(def,boolean)
+}
+
+class Observer -> java.util.Observer {
+  void update(Observable,Object)
+}
+
+class PrimitiveIterator -> java.util.PrimitiveIterator extends Iterator {
+  void forEachRemaining(def)
+}
+
+class PrimitiveIterator.OfDouble -> java.util.PrimitiveIterator$OfDouble extends PrimitiveIterator,Iterator {
+  Double next()
+  double nextDouble()
+}
+
+class PrimitiveIterator.OfInt -> java.util.PrimitiveIterator$OfInt extends PrimitiveIterator,Iterator {
+  Integer next()
+  int nextInt()
+}
+
+class PrimitiveIterator.OfLong -> java.util.PrimitiveIterator$OfLong extends PrimitiveIterator,Iterator {
+  Long next()
+  long nextLong()
+}
+
+class Spliterator -> java.util.Spliterator {
+  int CONCURRENT
+  int DISTINCT
+  int IMMUTABLE
+  int NONNULL
+  int ORDERED
+  int SIZED
+  int SORTED
+  int SUBSIZED
+  int characteristics()
+  long estimateSize()
+  void forEachRemaining(Consumer)
+  Comparator getComparator()
+  long getExactSizeIfKnown()
+  boolean hasCharacteristics(int)
+  boolean tryAdvance(Consumer)
+  Spliterator trySplit()
+}
+
+class Spliterator.OfPrimitive -> java.util.Spliterator$OfPrimitive extends Spliterator {
+  void forEachRemaining(def)
+  boolean tryAdvance(def)
+  Spliterator.OfPrimitive trySplit()
+}
+
+class Spliterator.OfDouble -> java.util.Spliterator$OfDouble extends Spliterator.OfPrimitive,Spliterator {
+  Spliterator.OfDouble trySplit()
+}
+
+class Spliterator.OfInt -> java.util.Spliterator$OfInt extends Spliterator.OfPrimitive,Spliterator {
+  Spliterator.OfInt trySplit()
+}
+
+class Spliterator.OfLong -> java.util.Spliterator$OfLong extends Spliterator.OfPrimitive,Spliterator {
+  Spliterator.OfLong trySplit()
+}
+
+class Queue -> java.util.Queue extends Collection,Iterable {
+  def element()
+  boolean offer(def)
+  def peek()
+  def poll()
+  def remove()
+}
+
+class RandomAccess -> java.util.RandomAccess {
+}
+
+class Set -> java.util.Set extends Collection,Iterable {
+  boolean equals(Object)
+  int hashCode()
+  boolean remove(def)
+}
+
+class SortedMap -> java.util.SortedMap extends Map {
+  Comparator comparator()
+  def firstKey()
+  SortedMap headMap(def)
+  def lastKey()
+  SortedMap subMap(def,def)
+  SortedMap tailMap(def)
+}
+
+class SortedSet -> java.util.SortedSet extends Set,Collection,Iterable {
+  Comparator comparator()
+  def first()
+  SortedSet headSet(def)
+  def last()
+  SortedSet subSet(def,def)
+  SortedSet tailSet(def)
+}
+
+#### Classes
+
+class AbstractCollection -> java.util.AbstractCollection extends Collection,Iterable,Object {
+}
+
+class AbstractList -> java.util.AbstractList extends AbstractCollection,List,Collection,Iterable,Object {
+}
+
+class AbstractMap -> java.util.AbstractMap extends Map,Object {
+}
+
+class AbstractMap.SimpleEntry -> java.util.AbstractMap$SimpleEntry extends Map.Entry,Object {
+  AbstractMap.SimpleEntry <init>(def,def)
+  AbstractMap.SimpleEntry <init>(Map.Entry)
+}
+
+class AbstractMap.SimpleImmutableEntry -> java.util.AbstractMap$SimpleImmutableEntry extends Map.Entry,Object {
+  AbstractMap.SimpleImmutableEntry <init>(def,def)
+  AbstractMap.SimpleImmutableEntry <init>(Map.Entry)
+}
+
+class AbstractQueue -> java.util.AbstractQueue extends AbstractCollection,Queue,Collection,Iterable,Object {
+}
+
+class AbstractSequentialList -> java.util.AbstractSequentialList extends AbstractList,AbstractCollection,List,Collection,Iterable,Object {
+}
+
+class AbstractSet -> java.util.AbstractSet extends AbstractCollection,Set,Collection,Iterable,Object {
+}
+
+class ArrayDeque -> java.util.ArrayDeque extends AbstractCollection,Deque,Queue,Collection,Iterable,Object {
+  ArrayDeque <init>()
+  ArrayDeque <init>(Collection)
+  ArrayDeque clone()
+}
+
+class ArrayList -> java.util.ArrayList extends AbstractList,AbstractCollection,List,RandomAccess,Collection,Iterable,Object {
+  ArrayList <init>()
+  ArrayList <init>(Collection)
+  def clone()
+  void trimToSize()
+}
+
+class Arrays -> java.util.Arrays extends Object {
+  List asList(Object[])
+  boolean deepEquals(Object[],Object[])
+  int deepHashCode(Object[])
+  String deepToString(Object[])
+}
+
+class Base64 -> java.util.Base64 extends Object {
+  Base64.Decoder getDecoder()
+  Base64.Encoder getEncoder()
+  Base64.Decoder getMimeDecoder()
+  Base64.Encoder getMimeEncoder()
+  Base64.Encoder getMimeEncoder(int,byte[])
+  Base64.Decoder getUrlDecoder()
+  Base64.Encoder getUrlEncoder()
+}
+
+class Base64.Decoder -> java.util.Base64$Decoder extends Object {
+  int decode(byte[],byte[])
+  byte[] decode(String)
+}
+
+class Base64.Encoder -> java.util.Base64$Encoder extends Object {
+  int encode(byte[],byte[])
+  String encodeToString(byte[])
+  Base64.Encoder withoutPadding()
+}
+
+class BitSet -> java.util.BitSet extends Object {
+  BitSet <init>()
+  BitSet <init>(int)
+  void and(BitSet)
+  void andNot(BitSet)
+  int cardinality()
+  void clear()
+  void clear(int)
+  void clear(int,int)
+  def clone()
+  void flip(int)
+  void flip(int,int)
+  boolean intersects(BitSet)
+  boolean isEmpty()
+  int length()
+  int nextClearBit(int)
+  int nextSetBit(int)
+  void or(BitSet)
+  int previousClearBit(int)
+  int previousSetBit(int)
+  void set(int)
+  void set(int,int)
+  void set(int,int,boolean)
+  int size()
+  byte[] toByteArray()
+  long[] toLongArray()
+  BitSet valueOf(long[])
+  void xor(BitSet)
+}
+
+class Calendar -> java.util.Calendar extends Comparable,Object {
+  int ALL_STYLES
+  int AM
+  int AM_PM
+  int APRIL
+  int AUGUST
+  int DATE
+  int DAY_OF_MONTH
+  int DAY_OF_WEEK
+  int DAY_OF_WEEK_IN_MONTH
+  int DAY_OF_YEAR
+  int DECEMBER
+  int DST_OFFSET
+  int ERA
+  int FEBRUARY
+  int FIELD_COUNT
+  int FRIDAY
+  int HOUR
+  int HOUR_OF_DAY
+  int JANUARY
+  int JULY
+  int JUNE
+  int LONG
+  int LONG_FORMAT
+  int LONG_STANDALONE
+  int MARCH
+  int MAY
+  int MILLISECOND
+  int MINUTE
+  int MONDAY
+  int MONTH
+  int NARROW_FORMAT
+  int NARROW_STANDALONE
+  int NOVEMBER
+  int OCTOBER
+  int PM
+  int SATURDAY
+  int SECOND
+  int SEPTEMBER
+  int SHORT
+  int SHORT_FORMAT
+  int SHORT_STANDALONE
+  int SUNDAY
+  int THURSDAY
+  int TUESDAY
+  int UNDECIMBER
+  int WEDNESDAY
+  int WEEK_OF_MONTH
+  int WEEK_OF_YEAR
+  int YEAR
+  int ZONE_OFFSET
+  void add(int,int)
+  boolean after(Object)
+  boolean before(Object)
+  void clear()
+  void clear(int)
+  def clone()
+  int get(int)
+  int getActualMaximum(int)
+  int getActualMinimum(int)
+  Set getAvailableCalendarTypes()
+  Locale[] getAvailableLocales()
+  String getCalendarType()
+  String getDisplayName(int,int,Locale)
+  Map getDisplayNames(int,int,Locale)
+  int getFirstDayOfWeek()
+  int getGreatestMinimum(int)
+  Calendar getInstance()
+  Calendar getInstance(TimeZone)
+  Calendar getInstance(TimeZone,Locale)
+  int getLeastMaximum(int)
+  int getMaximum(int)
+  int getMinimalDaysInFirstWeek()
+  int getMinimum(int)
+  Date getTime()
+  long getTimeInMillis()
+  TimeZone getTimeZone()
+  int getWeeksInWeekYear()
+  int getWeekYear()
+  boolean isLenient()
+  boolean isSet(int)
+  boolean isWeekDateSupported()
+  void roll(int,int)
+  void set(int,int)
+  void set(int,int,int)
+  void set(int,int,int,int,int)
+  void set(int,int,int,int,int,int)
+  void setFirstDayOfWeek(int)
+  void setLenient(boolean)
+  void setMinimalDaysInFirstWeek(int)
+  void setTime(Date)
+  void setTimeInMillis(long)
+  void setTimeZone(TimeZone)
+  void setWeekDate(int,int,int)
+}
+
+class Calendar.Builder -> java.util.Calendar$Builder extends Object {
+  Calendar.Builder <init>()
+  Calendar build()
+  Calendar.Builder set(int,int)
+  Calendar.Builder setCalendarType(String)
+  Calendar.Builder setDate(int,int,int)
+  Calendar.Builder setFields(int[])
+  Calendar.Builder setInstant(long)
+  Calendar.Builder setLenient(boolean)
+  Calendar.Builder setLocale(Locale)
+  Calendar.Builder setTimeOfDay(int,int,int)
+  Calendar.Builder setTimeOfDay(int,int,int,int)
+  Calendar.Builder setTimeZone(TimeZone)
+  Calendar.Builder setWeekDate(int,int,int)
+  Calendar.Builder setWeekDefinition(int,int)
+}
+
+class Collections -> java.util.Collections extends Object {
+  List EMPTY_LIST
+  Map EMPTY_MAP
+  Set EMPTY_SET
+  boolean addAll(Collection,def[])
+  Queue asLifoQueue(Deque)
+  int binarySearch(List,def)
+  int binarySearch(List,def,Comparator)
+  void copy(List,List)
+  boolean disjoint(Collection,Collection)
+  Enumeration emptyEnumeration()
+  Iterator emptyIterator()
+  List emptyList()
+  ListIterator emptyListIterator()
+  Map emptyMap()
+  NavigableMap emptyNavigableMap()
+  NavigableSet emptyNavigableSet()
+  Set emptySet()
+  SortedMap emptySortedMap()
+  SortedSet emptySortedSet()
+  Enumeration enumeration(Collection)
+  void fill(List,def)
+  int frequency(Collection,def)
+  int indexOfSubList(List,List)
+  int lastIndexOfSubList(List,List)
+  ArrayList list(Enumeration)
+  def max(Collection)
+  def max(Collection,Comparator)
+  def min(Collection)
+  def min(Collection,Comparator)
+  List nCopies(int,def)
+  Set newSetFromMap(Map)
+  boolean replaceAll(List,def,def)
+  void reverse(List)
+  Comparator reverseOrder()
+  Comparator reverseOrder(Comparator)
+  void rotate(List,int)
+  void shuffle(List)
+  void shuffle(List,Random)
+  Set singleton(def)
+  List singletonList(def)
+  Map singletonMap(def,def)
+  void sort(List)
+  void sort(List,Comparator)
+  void swap(List,int,int)
+  Collection unmodifiableCollection(Collection)
+  List unmodifiableList(List)
+  Map unmodifiableMap(Map)
+  NavigableMap unmodifiableNavigableMap(NavigableMap)
+  NavigableSet unmodifiableNavigableSet(NavigableSet)
+  Set unmodifiableSet(Set)
+  SortedMap unmodifiableSortedMap(SortedMap)
+  SortedSet unmodifiableSortedSet(SortedSet)
+}
+
+class Currency -> java.util.Currency extends Object {
+  Set getAvailableCurrencies()
+  String getCurrencyCode()
+  int getDefaultFractionDigits()
+  String getDisplayName()
+  String getDisplayName(Locale)
+  Currency getInstance(String)
+  int getNumericCode()
+  String getSymbol()
+  String getSymbol(Locale)
+}
+
+class Date -> java.util.Date extends Comparable,Object {
+  Date <init>()
+  Date <init>(long)
+  boolean after(Date)
+  boolean before(Date)
+  def clone()
+  long getTime()
+  void setTime(long)
+}
+
+class Dictionary -> java.util.Dictionary extends Object {
+  Enumeration elements()
+  def get(def)
+  boolean isEmpty()
+  Enumeration keys()
+  def put(def,def)
+  def remove(def)
+  int size()
+}
+
+class DoubleSummaryStatistics -> java.util.DoubleSummaryStatistics extends DoubleConsumer,Object {
+  DoubleSummaryStatistics <init>()
+  void combine(DoubleSummaryStatistics)
+  double getAverage()
+  long getCount()
+  double getMax()
+  double getMin()
+  double getSum()
+}
+
+class EventListenerProxy -> java.util.EventListenerProxy extends EventListener,Object {
+  EventListener getListener()
+}
+
+class EventObject -> java.util.EventObject extends Object {
+  EventObject <init>(Object)
+  Object getSource()
+}
+
+class FormattableFlags -> java.util.FormattableFlags extends Object {
+  int ALTERNATE
+  int LEFT_JUSTIFY
+  int UPPERCASE
+}
+
+class Formatter -> java.util.Formatter extends Object {
+  Formatter <init>()
+  Formatter <init>(Appendable)
+  Formatter <init>(Appendable,Locale)
+  Formatter format(Locale,String,def[])
+  Formatter format(String,def[])
+  Locale locale()
+  Appendable out()
+}
+
+class GregorianCalendar -> java.util.GregorianCalendar extends Calendar,Comparable,Object {
+  int AD
+  int BC
+  GregorianCalendar <init>()
+  GregorianCalendar <init>(int,int,int)
+  GregorianCalendar <init>(int,int,int,int,int)
+  GregorianCalendar <init>(int,int,int,int,int,int)
+  GregorianCalendar <init>(TimeZone)
+  GregorianCalendar <init>(TimeZone,Locale)
+  Date getGregorianChange()
+  boolean isLeapYear(int)
+  void setGregorianChange(Date)
+}
+
+class HashMap -> java.util.HashMap extends AbstractMap,Map,Object {
+  HashMap <init>()
+  HashMap <init>(Map)
+  def clone()
+}
+
+class HashSet -> java.util.HashSet extends AbstractSet,Set,Collection,Iterable,Object {
+  HashSet <init>()
+  HashSet <init>(Collection)
+  def clone()
+}
+
+class Hashtable -> java.util.Hashtable extends Dictionary,Map,Object {
+  Hashtable <init>()
+  Hashtable <init>(Map)
+  def clone()
+}
+
+class IdentityHashMap -> java.util.IdentityHashMap extends AbstractMap,Map,Object {
+  IdentityHashMap <init>()
+  IdentityHashMap <init>(Map)
+  def clone()
+}
+
+class IntSummaryStatistics -> java.util.IntSummaryStatistics extends IntConsumer,Object {
+  IntSummaryStatistics <init>()
+  void combine(IntSummaryStatistics)
+  double getAverage()
+  long getCount()
+  int getMax()
+  int getMin()
+  long getSum()
+}
+
+class LinkedHashMap -> java.util.LinkedHashMap extends HashMap,AbstractMap,Map,Object {
+  LinkedHashMap <init>()
+  LinkedHashMap <init>(Map)
+}
+
+class LinkedHashSet -> java.util.LinkedHashSet extends HashSet,AbstractSet,Set,AbstractCollection,Collection,Iterable,Object {
+  LinkedHashSet <init>()
+  LinkedHashSet <init>(Collection)
+}
+
+class LinkedList -> java.util.LinkedList extends AbstractSequentialList,AbstractList,List,Deque,Queue,AbstractCollection,Collection,Iterable,Object {
+  LinkedList <init>()
+  LinkedList <init>(Collection)
+  def clone()
+}
+
+class Locale -> java.util.Locale extends Object {
+  Locale CANADA
+  Locale CANADA_FRENCH
+  Locale CHINA
+  Locale CHINESE
+  Locale ENGLISH
+  Locale FRANCE
+  Locale FRENCH
+  Locale GERMAN
+  Locale GERMANY
+  Locale ITALIAN
+  Locale ITALY
+  Locale JAPAN
+  Locale JAPANESE
+  Locale KOREA
+  Locale KOREAN
+  Locale PRC
+  char PRIVATE_USE_EXTENSION
+  Locale ROOT
+  Locale SIMPLIFIED_CHINESE
+  Locale TAIWAN
+  Locale TRADITIONAL_CHINESE
+  Locale UK
+  char UNICODE_LOCALE_EXTENSION
+  Locale US
+  Locale <init>(String)
+  Locale <init>(String,String)
+  Locale <init>(String,String,String)
+  def clone()
+  List filter(List,Collection)
+  List filterTags(List,Collection)
+  Locale forLanguageTag(String)
+  Locale[] getAvailableLocales()
+  String getCountry()
+  Locale getDefault()
+  Locale getDefault(Locale.Category)
+  String getDisplayCountry()
+  String getDisplayCountry(Locale)
+  String getDisplayLanguage()
+  String getDisplayLanguage(Locale)
+  String getDisplayName()
+  String getDisplayName(Locale)
+  String getDisplayScript()
+  String getDisplayScript(Locale)
+  String getDisplayVariant()
+  String getDisplayVariant(Locale)
+  String getExtension(char)
+  Set getExtensionKeys()
+  String getISO3Country()
+  String getISO3Language()
+  String[] getISOCountries()
+  String[] getISOLanguages()
+  String getLanguage()
+  String getScript()
+  Set getUnicodeLocaleAttributes()
+  Set getUnicodeLocaleKeys()
+  String getUnicodeLocaleType(String)
+  String getVariant()
+  boolean hasExtensions()
+  Locale lookup(List,Collection)
+  String lookupTag(List,Collection)
+  Locale stripExtensions()
+  String toLanguageTag()
+}
+
+class Locale.Builder -> java.util.Locale$Builder extends Object {
+  Locale.Builder <init>()
+  Locale.Builder addUnicodeLocaleAttribute(String)
+  Locale build()
+  Locale.Builder clear()
+  Locale.Builder clearExtensions()
+  Locale.Builder removeUnicodeLocaleAttribute(String)
+  Locale.Builder setExtension(char,String)
+  Locale.Builder setLanguage(String)
+  Locale.Builder setLanguageTag(String)
+  Locale.Builder setLocale(Locale)
+  Locale.Builder setRegion(String)
+  Locale.Builder setScript(String)
+  Locale.Builder setUnicodeLocaleKeyword(String,String)
+  Locale.Builder setVariant(String)
+}
+
+class Locale.LanguageRange -> java.util.Locale$LanguageRange extends Object {
+  double MAX_WEIGHT
+  double MIN_WEIGHT
+  Locale.LanguageRange <init>(String)
+  Locale.LanguageRange <init>(String,double)
+  String getRange()
+  double getWeight()
+  List mapEquivalents(List,Map)
+  List parse(String)
+  List parse(String,Map)
+}
+
+class LongSummaryStatistics -> java.util.LongSummaryStatistics extends LongConsumer,Object {
+  LongSummaryStatistics <init>()
+  void combine(LongSummaryStatistics)
+  double getAverage()
+  long getCount()
+  long getMax()
+  long getMin()
+  long getSum()
+}
+
+class Objects -> java.util.Objects extends Object {
+  int compare(def,def,Comparator)
+  boolean deepEquals(Object,Object)
+  boolean equals(Object,Object)
+  int hash(Object[])
+  int hashCode(Object)
+  boolean isNull(Object)
+  boolean nonNull(Object)
+  def requireNonNull(def)
+  def requireNonNull(def,String)
+  String toString(Object)
+  String toString(Object,String)
+}
+
+class Observable -> java.util.Observable extends Object {
+  Observable <init>()
+  void addObserver(Observer)
+  int countObservers()
+  void deleteObserver(Observer)
+  void deleteObservers()
+  boolean hasChanged()
+  void notifyObservers()
+  void notifyObservers(Object)
+}
+
+class Optional -> java.util.Optional extends Object {
+  Optional empty()
+  Optional filter(Predicate)
+  Optional flatMap(Function)
+  def get()
+  void ifPresent(Consumer)
+  boolean isPresent()
+  Optional map(Function)
+  Optional of(def)
+  Optional ofNullable(def)
+  def orElse(def)
+  def orElseGet(Supplier)
+  def orElseThrow(Supplier)
+}
+
+class OptionalDouble -> java.util.OptionalDouble extends Object {
+  OptionalDouble empty()
+  double getAsDouble()
+  void ifPresent(DoubleConsumer)
+  boolean isPresent()
+  OptionalDouble of(double)
+  double orElse(double)
+  double orElseGet(DoubleSupplier)
+  double orElseThrow(Supplier)
+}
+
+class OptionalInt -> java.util.OptionalInt extends Object {
+  OptionalInt empty()
+  int getAsInt()
+  void ifPresent(IntConsumer)
+  boolean isPresent()
+  OptionalInt of(int)
+  int orElse(int)
+  int orElseGet(IntSupplier)
+  int orElseThrow(Supplier)
+}
+
+class OptionalLong -> java.util.OptionalLong extends Object {
+  OptionalLong empty()
+  long getAsLong()
+  void ifPresent(LongConsumer)
+  boolean isPresent()
+  OptionalLong of(long)
+  long orElse(long)
+  long orElseGet(LongSupplier)
+  long orElseThrow(Supplier)
+}
+
+class PriorityQueue -> java.util.PriorityQueue extends AbstractQueue,Queue,AbstractCollection,Collection,Iterable,Object {
+  PriorityQueue <init>()
+  PriorityQueue <init>(Comparator)
+}
+
+class Random -> java.util.Random extends Object {
+  Random <init>()
+  Random <init>(long)
+  DoubleStream doubles(long)
+  DoubleStream doubles(long,double,double)
+  IntStream ints(long)
+  IntStream ints(long,int,int)
+  LongStream longs(long)
+  LongStream longs(long,long,long)
+  boolean nextBoolean()
+  void nextBytes(byte[])
+  double nextDouble()
+  float nextFloat()
+  double nextGaussian()
+  int nextInt()
+  int nextInt(int)
+  long nextLong()
+  void setSeed(long)
+}
+
+class SimpleTimeZone -> java.util.SimpleTimeZone extends TimeZone,Object {
+  int STANDARD_TIME
+  int UTC_TIME
+  int WALL_TIME
+  SimpleTimeZone <init>(int,String)
+  SimpleTimeZone <init>(int,String,int,int,int,int,int,int,int,int)
+  SimpleTimeZone <init>(int,String,int,int,int,int,int,int,int,int,int)
+  SimpleTimeZone <init>(int,String,int,int,int,int,int,int,int,int,int,int,int)
+  int getDSTSavings()
+  void setDSTSavings(int)
+  void setEndRule(int,int,int)
+  void setEndRule(int,int,int,int)
+  void setEndRule(int,int,int,int,boolean)
+  void setStartRule(int,int,int)
+  void setStartRule(int,int,int,int)
+  void setStartRule(int,int,int,int,boolean)
+  void setStartYear(int)
+}
+
+class Spliterators -> java.util.Spliterators extends Object {
+  Spliterator.OfDouble emptyDoubleSpliterator()
+  Spliterator.OfInt emptyIntSpliterator()
+  Spliterator.OfLong emptyLongSpliterator()
+  Spliterator emptySpliterator()
+  Iterator iterator(Spliterator)
+  Spliterator spliterator(Collection,int)
+  Spliterator spliterator(Iterator,long,int)
+  Spliterator spliteratorUnknownSize(Iterator,int)
+}
+
+class Stack -> java.util.Stack extends Vector,AbstractList,List,AbstractCollection,Collection,Iterable,RandomAccess,Object {
+  Stack <init>()
+  def push(def)
+  def pop()
+  def peek()
+  boolean empty()
+  int search(def)
+}
+
+class StringJoiner -> java.util.StringJoiner extends Object {
+  StringJoiner <init>(CharSequence)
+  StringJoiner <init>(CharSequence,CharSequence,CharSequence)
+  StringJoiner add(CharSequence)
+  int length()
+  StringJoiner merge(StringJoiner)
+  StringJoiner setEmptyValue(CharSequence)
+}
+
+class StringTokenizer -> java.util.StringTokenizer extends Enumeration,Object {
+  StringTokenizer <init>(String)
+  StringTokenizer <init>(String,String)
+  StringTokenizer <init>(String,String,boolean)
+  int countTokens()
+  boolean hasMoreTokens()
+  String nextToken()
+  String nextToken(String)
+}
+
+class TimeZone -> java.util.TimeZone extends Object {
+  int LONG
+  int SHORT
+  def clone()
+  String[] getAvailableIDs()
+  String[] getAvailableIDs(int)
+  TimeZone getDefault()
+  String getDisplayName()
+  String getDisplayName(boolean,int)
+  String getDisplayName(boolean,int,Locale)
+  String getDisplayName(Locale)
+  int getDSTSavings()
+  String getID()
+  int getOffset(int,int,int,int,int,int)
+  int getOffset(long)
+  int getRawOffset()
+  TimeZone getTimeZone(String)
+  boolean hasSameRules(TimeZone)
+  boolean inDaylightTime(Date)
+  boolean observesDaylightTime()
+  boolean useDaylightTime()
+}
+
+class TreeMap -> java.util.TreeMap extends AbstractMap,NavigableMap,SortedMap,Map,Object {
+  TreeMap <init>()
+  TreeMap <init>(Comparator)
+  def clone()
+}
+
+class TreeSet -> java.util.TreeSet extends AbstractSet,NavigableSet,SortedSet,Set,AbstractCollection,Collection,Iterable,Object {
+  TreeSet <init>()
+  TreeSet <init>(Comparator)
+  def clone()
+}
+
+class UUID -> java.util.UUID extends Comparable,Object {
+  UUID <init>(long,long)
+  int clockSequence()
+  UUID fromString(String)
+  long getLeastSignificantBits()
+  long getMostSignificantBits()
+  UUID nameUUIDFromBytes(byte[])
+  long node()
+  long timestamp()
+  int variant()
+  int version()
+}
+
+class Vector -> java.util.Vector extends AbstractList,List,AbstractCollection,Collection,Iterable,RandomAccess,Object {
+  Vector <init>()
+  Vector <init>(Collection)
+  void addElement(def)
+  void copyInto(Object[])
+  def elementAt(int)
+  Enumeration elements()
+  def firstElement()
+  void insertElementAt(def,int)
+  def lastElement()
+  int lastIndexOf(def,int)
+  void removeAllElements()
+  boolean removeElement(def)
+  void removeElementAt(int)
+  void setElementAt(def,int)
+  def clone()
+}
+
+#### Enums
+
+class Formatter.BigDecimalLayoutForm -> java.util.Formatter$BigDecimalLayoutForm extends Enum,Comparable,Object {
+  Formatter.BigDecimalLayoutForm DECIMAL_FLOAT
+  Formatter.BigDecimalLayoutForm SCIENTIFIC
+}
+
+class Locale.Category -> java.util.Locale$Category extends Enum,Comparable,Object {
+  Locale.Category DISPLAY
+  Locale.Category FORMAT
+  Locale.Category valueOf(String)
+  Locale.Category[] values()
+}
+
+class Locale.FilteringMode -> java.util.Locale$FilteringMode extends Enum,Comparable,Object {
+  Locale.FilteringMode AUTOSELECT_FILTERING
+  Locale.FilteringMode EXTENDED_FILTERING
+  Locale.FilteringMode IGNORE_EXTENDED_RANGES
+  Locale.FilteringMode MAP_EXTENDED_RANGES
+  Locale.FilteringMode REJECT_EXTENDED_RANGES
+  Locale.FilteringMode valueOf(String)
+  Locale.FilteringMode[] values()
+}
+
+#### Exceptions
+
+class ConcurrentModificationException -> java.util.ConcurrentModificationException extends RuntimeException,Exception {
+  ConcurrentModificationException <init>()
+  ConcurrentModificationException <init>(String)
+}
+
+class DuplicateFormatFlagsException -> java.util.DuplicateFormatFlagsException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception {
+  DuplicateFormatFlagsException <init>(String)
+  String getFlags()
+}
+
+class EmptyStackException -> java.util.EmptyStackException extends RuntimeException,Exception {
+  EmptyStackException <init>()
+}
+
+class FormatFlagsConversionMismatchException -> java.util.FormatFlagsConversionMismatchException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception {
+  FormatFlagsConversionMismatchException <init>(String,char)
+  char getConversion()
+  String getFlags()
+}
+
+class FormatterClosedException -> java.util.FormatterClosedException extends IllegalStateException,RuntimeException,Exception {
+  FormatterClosedException <init>()
+}
+
+class IllegalFormatCodePointException -> java.util.IllegalFormatCodePointException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception {
+  IllegalFormatCodePointException <init>(int)
+  int getCodePoint()
+}
+
+class IllegalFormatConversionException -> java.util.IllegalFormatConversionException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception {
+  char getConversion()
+}
+
+class IllegalFormatException -> java.util.IllegalFormatException extends IllegalArgumentException,RuntimeException,Exception {
+}
+
+class IllegalFormatFlagsException -> java.util.IllegalFormatFlagsException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception {
+  IllegalFormatFlagsException <init>(String)
+  String getFlags()
+}
+
+class IllegalFormatPrecisionException -> java.util.IllegalFormatPrecisionException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception {
+  IllegalFormatPrecisionException <init>(int)
+  int getPrecision()
+}
+
+class IllegalFormatWidthException -> java.util.IllegalFormatWidthException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception {
+  IllegalFormatWidthException <init>(int)
+  int getWidth()
+}
+
+class IllformedLocaleException -> java.util.IllformedLocaleException extends RuntimeException,Exception {
+  IllformedLocaleException <init>()
+  IllformedLocaleException <init>(String)
+  IllformedLocaleException <init>(String,int)
+  int getErrorIndex()
+}
+
+class InputMismatchException -> java.util.InputMismatchException extends NoSuchElementException,RuntimeException,Exception {
+  InputMismatchException <init>()
+  InputMismatchException <init>(String)
+}
+
+class MissingFormatArgumentException -> java.util.MissingFormatArgumentException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception {
+  MissingFormatArgumentException <init>(String)
+  String getFormatSpecifier()
+}
+
+class MissingFormatWidthException -> java.util.MissingFormatWidthException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception {
+  MissingFormatWidthException <init>(String)
+  String getFormatSpecifier()
+}
+
+class MissingResourceException -> java.util.MissingResourceException extends RuntimeException,Exception {
+  MissingResourceException <init>(String,String,String)
+  String getClassName()
+  String getKey()
+}
+
+class NoSuchElementException -> java.util.NoSuchElementException extends RuntimeException,Exception {
+  NoSuchElementException <init>()
+  NoSuchElementException <init>(String)
+}
+
+class TooManyListenersException -> java.util.TooManyListenersException extends Exception {
+  TooManyListenersException <init>()
+  TooManyListenersException <init>(String)
+}
+
+class UnknownFormatConversionException -> java.util.UnknownFormatConversionException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception {
+  UnknownFormatConversionException <init>(String)
+  String getConversion()
+}
+
+class UnknownFormatFlagsException -> java.util.UnknownFormatFlagsException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception {
+  UnknownFormatFlagsException <init>(String)
+  String getFlags()
+}
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt
new file mode 100644
index 00000000000..a4af7c318ec
--- /dev/null
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt
@@ -0,0 +1,127 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#
+# Painless definition file. This defines the hierarchy of classes,
+# what methods and fields they have, etc.
+#
+
+#### Primitive types
+
+class void -> void {
+}
+
+class boolean -> boolean {
+}
+
+class byte -> byte {
+}
+
+class short -> short {
+}
+
+class char -> char {
+}
+
+class int -> int {
+}
+
+class long -> long {
+}
+
+class float -> float {
+}
+
+class double -> double {
+}
+
+class def -> java.lang.Object {
+  boolean equals(Object)
+  int hashCode()
+  String toString()
+}
+
+#### ES Scripting API
+
+class GeoPoint -> org.elasticsearch.common.geo.GeoPoint extends Object {
+  double getLat()
+  double getLon()
+}
+
+class Strings -> org.elasticsearch.index.fielddata.ScriptDocValues$Strings extends List,Collection,Iterable,Object {
+  String getValue()
+  List getValues()
+}
+
+class Longs -> org.elasticsearch.index.fielddata.ScriptDocValues$Longs extends List,Collection,Iterable,Object {
+  long getValue()
+  List getValues()
+}
+
+class Doubles -> org.elasticsearch.index.fielddata.ScriptDocValues$Doubles extends List,Collection,Iterable,Object {
+  double getValue()
+  List getValues()
+}
+
+class GeoPoints -> org.elasticsearch.index.fielddata.ScriptDocValues$GeoPoints extends List,Collection,Iterable,Object {
+  GeoPoint getValue()
+  List getValues()
+  double getLat()
+  double getLon()
+  double[] getLats()
+  double[] getLons()
+
+  # geo distance functions... so many...
+  double factorDistance(double,double)
+  double factorDistanceWithDefault(double,double,double)
+  double factorDistance02(double,double)
+  double factorDistance13(double,double)
+  double arcDistance(double,double)
+  double arcDistanceWithDefault(double,double,double)
+  double arcDistanceInKm(double,double)
+  double arcDistanceInKmWithDefault(double,double,double)
+  double arcDistanceInMiles(double,double)
+  double arcDistanceInMilesWithDefault(double,double,double)
+  double distance(double,double)
+  double distanceWithDefault(double,double,double)
+  double distanceInKm(double,double)
+  double distanceInKmWithDefault(double,double,double)
+  double distanceInMiles(double,double)
+  double distanceInMilesWithDefault(double,double,double)
+  double geohashDistance(String)
+  double geohashDistanceInKm(String)
+  double geohashDistanceInMiles(String)
+}
+
+# for testing.
+# currently FeatureTest exposes overloaded constructor, field load store, and overloaded static methods
+class FeatureTest -> org.elasticsearch.painless.FeatureTest extends Object {
+  FeatureTest <init>()
+  FeatureTest <init>(int,int)
+  int getX()
+  int getY()
+  void setX(int)
+  void setY(int)
+  boolean overloadedStatic()
+  boolean overloadedStatic(boolean)
+}
+
+# currently needed internally
+class Executable -> org.elasticsearch.painless.Executable {
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
index 338e3f00113..043b614cadd 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
@@ -19,6 +19,9 @@
 
 package org.elasticsearch.painless;
 
+import java.util.Arrays;
+import java.util.Collections;
+
 public class BasicAPITests extends ScriptTestCase {
 
     public void testListIterator() {
@@ -90,4 +93,8 @@ public class BasicAPITests extends ScriptTestCase {
         assertBytecodeExists("def x = 1F", "INVOKESTATIC java/lang/Float.valueOf (F)Ljava/lang/Float;");
         assertBytecodeExists("def x = 1D", "INVOKESTATIC java/lang/Double.valueOf (D)Ljava/lang/Double;");
     }
+    
+    void testStream() {
+        assertEquals(11, exec("params.list.stream().sum()", Collections.singletonMap("list", Arrays.asList(1,2,3,5))));
+    }
 }

From b49635539d929d6cb5ee56153e2f95872fe4bb16 Mon Sep 17 00:00:00 2001
From: Chris Earle <chris.earle@elastic.co>
Date: Mon, 23 May 2016 13:19:25 -0400
Subject: [PATCH 0431/1311] Remove support for -Des.* system properties in
 integration tests

This now requires that system properties passed to Gradle must be in the form of "-Dtests.es.*" instead of
"-Des.*". It then chops off "tests.es." and passes that as a "-E" property to Elasticsearch.

Also changed system properties:

- `tests.logger.level` became `tests.es.logger.level`
- `node.mode` became `tests.es.node.mode`
- `node.local` became `tests.es.node.local`
---
 TESTING.asciidoc                                  |  8 ++++----
 .../org/elasticsearch/gradle/test/NodeInfo.groovy |  8 ++------
 .../elasticsearch/test/InternalTestCluster.java   | 15 +++++++--------
 .../junit/listeners/ReproduceInfoPrinter.java     |  4 ++--
 4 files changed, 15 insertions(+), 20 deletions(-)

diff --git a/TESTING.asciidoc b/TESTING.asciidoc
index 50953f16378..ce81f97548f 100644
--- a/TESTING.asciidoc
+++ b/TESTING.asciidoc
@@ -18,18 +18,18 @@ gradle assemble
 
 == Other test options
 
-To disable and enable network transport, set the `Des.node.mode`.
+To disable and enable network transport, set the `tests.es.node.mode` system property.
 
 Use network transport:
 
 ------------------------------------
--Des.node.mode=network
+-Dtests.es.node.mode=network
 ------------------------------------
 
 Use local transport (default since 1.3):
 
 -------------------------------------
--Des.node.mode=local
+-Dtests.es.node.mode=local
 -------------------------------------
 
 === Running Elasticsearch from a checkout
@@ -195,7 +195,7 @@ gradle test -Dtests.timeoutSuite=5000! ...
 Change the logging level of ES (not gradle)
 
 --------------------------------
-gradle test -Dtests.logger.level=DEBUG
+gradle test -Dtests.es.logger.level=DEBUG
 --------------------------------
 
 Print all the logging output from the test runs to the commandline
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
index cd0475d2aea..5d9961a0425 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy
@@ -130,17 +130,13 @@ class NodeInfo {
 
         env = [ 'JAVA_HOME' : project.javaHome ]
         args.addAll("-E", "node.portsfile=true")
-        String loggerLevel = System.getProperty("tests.logger.level")
-        if (loggerLevel != null) {
-            args.addAll("-E", "logger.level=${loggerLevel}")
-        }
         String collectedSystemProperties = config.systemProperties.collect { key, value -> "-D${key}=${value}" }.join(" ")
         String esJavaOpts = config.jvmArgs.isEmpty() ? collectedSystemProperties : collectedSystemProperties + " " + config.jvmArgs
         env.put('ES_JAVA_OPTS', esJavaOpts)
         for (Map.Entry<String, String> property : System.properties.entrySet()) {
-            if (property.getKey().startsWith('es.')) {
+            if (property.key.startsWith('tests.es.')) {
                 args.add("-E")
-                args.add("${property.getKey()}=${property.getValue()}")
+                args.add("${property.key.substring('tests.es.'.size())}=${property.value}")
             }
         }
         env.put('ES_JVM_OPTIONS', new File(confDir, 'jvm.options'))
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index 995ca5480f3..69f811b7d07 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -80,7 +80,6 @@ import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.recovery.RecoverySettings;
 import org.elasticsearch.node.MockNode;
 import org.elasticsearch.node.Node;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
 import org.elasticsearch.node.service.NodeService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptService;
@@ -293,8 +292,8 @@ public final class InternalTestCluster extends TestCluster {
         builder.put("http.port", HTTP_BASE_PORT + "-" + (HTTP_BASE_PORT + PORTS_PER_CLUSTER));
         builder.put(Node.NODE_MODE_SETTING.getKey(), nodeMode);
         builder.put("http.pipelining", enableHttpPipelining);
-        if (Strings.hasLength(System.getProperty("tests.logger.level"))) {
-            builder.put("logger.level", System.getProperty("tests.logger.level"));
+        if (Strings.hasLength(System.getProperty("tests.es.logger.level"))) {
+            builder.put("logger.level", System.getProperty("tests.es.logger.level"));
         }
         if (Strings.hasLength(System.getProperty("es.logger.prefix"))) {
             builder.put("logger.prefix", System.getProperty("es.logger.prefix"));
@@ -318,14 +317,14 @@ public final class InternalTestCluster extends TestCluster {
 
     public static String configuredNodeMode() {
         Builder builder = Settings.builder();
-        if (Strings.isEmpty(System.getProperty("node.mode")) && Strings.isEmpty(System.getProperty("node.local"))) {
+        if (Strings.isEmpty(System.getProperty("tests.es.node.mode")) && Strings.isEmpty(System.getProperty("tests.node.local"))) {
             return "local"; // default if nothing is specified
         }
-        if (Strings.hasLength(System.getProperty("node.mode"))) {
-            builder.put(Node.NODE_MODE_SETTING.getKey(), System.getProperty("node.mode"));
+        if (Strings.hasLength(System.getProperty("tests.es.node.mode"))) {
+            builder.put(Node.NODE_MODE_SETTING.getKey(), System.getProperty("tests.es.node.mode"));
         }
-        if (Strings.hasLength(System.getProperty("node.local"))) {
-            builder.put(Node.NODE_LOCAL_SETTING.getKey(), System.getProperty("node.local"));
+        if (Strings.hasLength(System.getProperty("tests.es.node.local"))) {
+            builder.put(Node.NODE_LOCAL_SETTING.getKey(), System.getProperty("tests.es.node.local"));
         }
         if (DiscoveryNode.isLocalNode(builder.build())) {
             return "local";
diff --git a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java
index 75bc916ab94..2bdec79f3a0 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/junit/listeners/ReproduceInfoPrinter.java
@@ -137,10 +137,10 @@ public class ReproduceInfoPrinter extends RunListener {
         }
 
         public ReproduceErrorMessageBuilder appendESProperties() {
-            appendProperties("tests.logger.level");
+            appendProperties("tests.es.logger.level");
             if (inVerifyPhase()) {
                 // these properties only make sense for integration tests
-                appendProperties("node.mode", "node.local", TESTS_CLUSTER,
+                appendProperties("tests.es.node.mode", "tests.es.node.local", TESTS_CLUSTER,
                     ESIntegTestCase.TESTS_ENABLE_MOCK_MODULES);
             }
             appendProperties("tests.assertion.disabled", "tests.security.manager", "tests.nightly", "tests.jvms",

From 1424f09c02927d9caada7f6ea45285b411d88d12 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Mon, 23 May 2016 16:36:01 -0700
Subject: [PATCH 0432/1311] Remove Painless grammar ambiguities.

Closes #18531
---
 modules/lang-painless/ant.xml                 |   14 +-
 .../src/main/antlr/PainlessLexer.g4           |   22 +-
 .../src/main/antlr/PainlessLexer.tokens       |  137 +
 .../src/main/antlr/PainlessParser.g4          |  161 +-
 .../src/main/antlr/PainlessParser.tokens      |  137 +
 .../org/elasticsearch/painless/Analyzer.java  |    5 +-
 .../org/elasticsearch/painless/Compiler.java  |   22 +-
 .../painless/CompilerSettings.java            |   28 +
 .../elasticsearch/painless/Definition.java    |   54 +-
 .../painless/PainlessScriptEngineService.java |    6 +
 .../org/elasticsearch/painless/Variables.java |   62 +-
 .../painless/antlr/PainlessLexer.java         |  373 +--
 .../painless/antlr/PainlessParser.java        | 2700 ++++++++---------
 .../antlr/PainlessParserBaseVisitor.java      |  108 +-
 .../painless/antlr/PainlessParserVisitor.java |  209 +-
 .../elasticsearch/painless/antlr/Walker.java  |  863 +++---
 .../painless/node/AExpression.java            |   18 +-
 .../elasticsearch/painless/node/ALink.java    |   10 +-
 .../elasticsearch/painless/node/ANode.java    |   20 +-
 .../painless/node/AStatement.java             |    6 +-
 .../elasticsearch/painless/node/EBinary.java  |   70 +-
 .../elasticsearch/painless/node/EBool.java    |   62 +-
 .../elasticsearch/painless/node/EBoolean.java |    4 +-
 .../elasticsearch/painless/node/ECast.java    |   12 +-
 .../elasticsearch/painless/node/EChain.java   |   94 +-
 .../elasticsearch/painless/node/EComp.java    |  130 +-
 .../painless/node/EConditional.java           |   22 +-
 .../painless/node/EConstant.java              |   32 +-
 .../elasticsearch/painless/node/EDecimal.java |   10 +-
 .../painless/node/EExplicit.java              |    8 +-
 .../elasticsearch/painless/node/ENull.java    |    8 +-
 .../elasticsearch/painless/node/ENumeric.java |   18 +-
 .../elasticsearch/painless/node/EUnary.java   |   58 +-
 .../painless/node/LArrayLength.java           |   12 +-
 .../elasticsearch/painless/node/LBrace.java   |   24 +-
 .../elasticsearch/painless/node/LCall.java    |   35 +-
 .../elasticsearch/painless/node/LCast.java    |   16 +-
 .../painless/node/LDefArray.java              |   21 +-
 .../elasticsearch/painless/node/LDefCall.java |   22 +-
 .../painless/node/LDefField.java              |   18 +-
 .../elasticsearch/painless/node/LField.java   |   51 +-
 .../painless/node/LListShortcut.java          |   24 +-
 .../painless/node/LMapShortcut.java           |   24 +-
 .../painless/node/LNewArray.java              |   26 +-
 .../elasticsearch/painless/node/LNewObj.java  |   28 +-
 .../painless/node/LShortcut.java              |   29 +-
 .../elasticsearch/painless/node/LStatic.java  |   69 +
 .../elasticsearch/painless/node/LString.java  |   14 +-
 .../painless/node/LVariable.java              |   43 +-
 .../elasticsearch/painless/node/SBlock.java   |   12 +-
 .../elasticsearch/painless/node/SBreak.java   |   11 +-
 .../painless/node/{STrap.java => SCatch.java} |   41 +-
 .../painless/node/SContinue.java              |   11 +-
 .../painless/node/SDeclBlock.java             |   12 +-
 .../painless/node/SDeclaration.java           |   59 +-
 .../org/elasticsearch/painless/node/SDo.java  |   37 +-
 .../painless/node/SExpression.java            |   15 +-
 .../org/elasticsearch/painless/node/SFor.java |   52 +-
 .../org/elasticsearch/painless/node/SIf.java  |   84 +
 .../elasticsearch/painless/node/SIfElse.java  |   77 +-
 .../elasticsearch/painless/node/SReturn.java  |   13 +-
 .../elasticsearch/painless/node/SSource.java  |   20 +-
 .../elasticsearch/painless/node/SThrow.java   |   13 +-
 .../org/elasticsearch/painless/node/STry.java |   69 +-
 .../elasticsearch/painless/node/SWhile.java   |   39 +-
 .../painless/node/package-info.java           |    4 +-
 .../painless/NoSemiColonTests.java            |    2 +-
 .../painless/ScriptTestCase.java              |    5 +-
 .../painless/WhenThingsGoWrongTests.java      |    6 +-
 .../painless/antlr/ParserTests.java           |   81 +
 70 files changed, 3519 insertions(+), 3083 deletions(-)
 create mode 100644 modules/lang-painless/src/main/antlr/PainlessLexer.tokens
 create mode 100644 modules/lang-painless/src/main/antlr/PainlessParser.tokens
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java
 rename modules/lang-painless/src/main/java/org/elasticsearch/painless/node/{STrap.java => SCatch.java} (68%)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
 create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/antlr/ParserTests.java

diff --git a/modules/lang-painless/ant.xml b/modules/lang-painless/ant.xml
index 1d2bdb678e8..9d3cf9ece91 100644
--- a/modules/lang-painless/ant.xml
+++ b/modules/lang-painless/ant.xml
@@ -42,10 +42,16 @@
   <macrodef name="regen-delete">
     <attribute name="grammar" />
     <sequential>
+      <local name="grammar.path"/>
+      <property name="grammar.path" location="src/main/antlr"/>
+      <!-- delete token files so files will be generated -->
+      <delete dir="${grammar.path}" includes="@{grammar}*.tokens"/>
       <local name="output.path"/>
       <patternset id="grammar.@{grammar}.patternset">
         <include name="@{grammar}Lexer.java" />
         <include name="@{grammar}Parser.java" />
+        <include name="@{grammar}Lexer.tokens" />
+        <include name="@{grammar}Parser.tokens" />
         <include name="@{grammar}ParserVisitor.java" />
         <include name="@{grammar}ParserBaseVisitor.java" />
       </patternset>
@@ -70,6 +76,7 @@
         <sysproperty key="user.language" value="en"/>
         <sysproperty key="user.country" value="US"/>
         <sysproperty key="user.variant" value=""/>
+        <arg value="-Werror"/>
         <arg value="-package"/>
         <arg value="org.elasticsearch.painless.antlr"/>
         <arg value="-o"/>
@@ -92,6 +99,7 @@
         <sysproperty key="user.language" value="en"/>
         <sysproperty key="user.country" value="US"/>
         <sysproperty key="user.variant" value=""/>
+        <arg value="-Werror"/>
         <arg value="-package"/>
         <arg value="org.elasticsearch.painless.antlr"/>
         <arg value="-no-listener"/>
@@ -121,9 +129,9 @@
       <fileset id="grammar.fileset" dir="${output.path}">
         <patternset refid="grammar.@{grammar}.patternset"/>
       </fileset>
-      <!-- remove files that are not needed to compile or at runtime -->
-      <delete dir="${grammar.path}" includes="@{grammar}*.tokens"/>
-      <delete dir="${output.path}" includes="@{grammar}*.tokens"/>
+      <!-- moves token files to grammar directory for use with IDE's -->
+      <move file="${output.path}/@{grammar}Lexer.tokens" todir="${grammar.path}"/>
+      <move file="${output.path}/@{grammar}Parser.tokens" todir="${grammar.path}"/>
       <!-- make the generated classes package private -->
       <replaceregexp match="public ((interface|class) \Q@{grammar}\E\w+)" replace="\1" encoding="UTF-8">
         <fileset refid="grammar.fileset"/>
diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.g4 b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
index 97c79845043..2588a646dcb 100644
--- a/modules/lang-painless/src/main/antlr/PainlessLexer.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
@@ -19,6 +19,10 @@
 
 lexer grammar PainlessLexer;
 
+@header {
+import org.elasticsearch.painless.Definition;
+}
+
 WS: [ \t\n\r]+ -> skip;
 COMMENT: ( '//' .*? [\n\r] | '/*' .*? '*/' ) -> skip;
 
@@ -28,7 +32,7 @@ LBRACE:    '[';
 RBRACE:    ']';
 LP:        '(';
 RP:        ')';
-DOT:       '.' -> mode(EXT);
+DOT:       '.' -> mode(AFTER_DOT);
 COMMA:     ',';
 SEMICOLON: ';';
 IF:        'if';
@@ -88,7 +92,7 @@ AUSH:   '>>>=';
 OCTAL: '0' [0-7]+ [lL]?;
 HEX: '0' [xX] [0-9a-fA-F]+ [lL]?;
 INTEGER: ( '0' | [1-9] [0-9]* ) [lLfFdD]?;
-DECIMAL: ( '0' | [1-9] [0-9]* ) DOT [0-9]* ( [eE] [+\-]? [0-9]+ )? [fF]?;
+DECIMAL: ( '0' | [1-9] [0-9]* ) (DOT [0-9]+)? ( [eE] [+\-]? [0-9]+ )? [fF]?;
 
 STRING: ( '"' ( '\\"' | '\\\\' | ~[\\"] )*? '"' ) | ( '\'' ( '\\\'' | '\\\\' | ~[\\"] )*? '\'' );
 
@@ -97,8 +101,16 @@ FALSE: 'false';
 
 NULL: 'null';
 
+// The predicate here allows us to remove ambiguities when
+// dealing with types versus identifiers.  We check against
+// the current whitelist to determine whether a token is a type
+// or not.  Note this works by processing one character at a time
+// and the rule is added or removed as this happens.  This is also known
+// as "the lexer hack."  See (https://en.wikipedia.org/wiki/The_lexer_hack).
+TYPE: ID ( DOT ID )* { Definition.isSimpleType(getText()) }?;
 ID: [_a-zA-Z] [_a-zA-Z0-9]*;
 
-mode EXT;
-EXTINTEGER: ( '0' | [1-9] [0-9]* ) -> mode(DEFAULT_MODE);
-EXTID: [_a-zA-Z] [_a-zA-Z0-9]* -> mode(DEFAULT_MODE);
+mode AFTER_DOT;
+
+DOTINTEGER: ( '0' | [1-9] [0-9]* )                        -> mode(DEFAULT_MODE);
+DOTID: [_a-z] [_a-zA-Z0-9]*                               -> mode(DEFAULT_MODE);
diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.tokens b/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
new file mode 100644
index 00000000000..d42e157e964
--- /dev/null
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
@@ -0,0 +1,137 @@
+WS=1
+COMMENT=2
+LBRACK=3
+RBRACK=4
+LBRACE=5
+RBRACE=6
+LP=7
+RP=8
+DOT=9
+COMMA=10
+SEMICOLON=11
+IF=12
+ELSE=13
+WHILE=14
+DO=15
+FOR=16
+CONTINUE=17
+BREAK=18
+RETURN=19
+NEW=20
+TRY=21
+CATCH=22
+THROW=23
+BOOLNOT=24
+BWNOT=25
+MUL=26
+DIV=27
+REM=28
+ADD=29
+SUB=30
+LSH=31
+RSH=32
+USH=33
+LT=34
+LTE=35
+GT=36
+GTE=37
+EQ=38
+EQR=39
+NE=40
+NER=41
+BWAND=42
+XOR=43
+BWOR=44
+BOOLAND=45
+BOOLOR=46
+COND=47
+COLON=48
+INCR=49
+DECR=50
+ASSIGN=51
+AADD=52
+ASUB=53
+AMUL=54
+ADIV=55
+AREM=56
+AAND=57
+AXOR=58
+AOR=59
+ALSH=60
+ARSH=61
+AUSH=62
+OCTAL=63
+HEX=64
+INTEGER=65
+DECIMAL=66
+STRING=67
+TRUE=68
+FALSE=69
+NULL=70
+TYPE=71
+ID=72
+DOTINTEGER=73
+DOTID=74
+'{'=3
+'}'=4
+'['=5
+']'=6
+'('=7
+')'=8
+'.'=9
+','=10
+';'=11
+'if'=12
+'else'=13
+'while'=14
+'do'=15
+'for'=16
+'continue'=17
+'break'=18
+'return'=19
+'new'=20
+'try'=21
+'catch'=22
+'throw'=23
+'!'=24
+'~'=25
+'*'=26
+'/'=27
+'%'=28
+'+'=29
+'-'=30
+'<<'=31
+'>>'=32
+'>>>'=33
+'<'=34
+'<='=35
+'>'=36
+'>='=37
+'=='=38
+'==='=39
+'!='=40
+'!=='=41
+'&'=42
+'^'=43
+'|'=44
+'&&'=45
+'||'=46
+'?'=47
+':'=48
+'++'=49
+'--'=50
+'='=51
+'+='=52
+'-='=53
+'*='=54
+'/='=55
+'%='=56
+'&='=57
+'^='=58
+'|='=59
+'<<='=60
+'>>='=61
+'>>>='=62
+'true'=68
+'false'=69
+'null'=70
diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index b36dec5afbb..12de27cefd3 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -22,35 +22,37 @@ parser grammar PainlessParser;
 options { tokenVocab=PainlessLexer; }
 
 source
-    : statement+ EOF
+    : statement* EOF
     ;
 
+// Note we use a predicate on the if/else case here to prevent the
+// "dangling-else" ambiguity by forcing the 'else' token to be consumed
+// as soon as one is found.  See (https://en.wikipedia.org/wiki/Dangling_else).
 statement
-    : IF LP expression RP block ( ELSE block )?                                              # if
-    | WHILE LP expression RP ( block | empty )                                               # while
-    | DO block WHILE LP expression RP ( SEMICOLON | EOF )                                    # do
-    | FOR LP initializer? SEMICOLON expression? SEMICOLON afterthought? RP ( block | empty ) # for
-    | declaration ( SEMICOLON | EOF )                                                        # decl
-    | CONTINUE ( SEMICOLON | EOF )                                                           # continue
-    | BREAK ( SEMICOLON | EOF )                                                              # break
-    | RETURN expression ( SEMICOLON | EOF )                                                  # return
-    | TRY block trap+                                                                        # try
-    | THROW expression ( SEMICOLON | EOF )                                                   # throw
-    | expression ( SEMICOLON | EOF )                                                         # expr
+    : IF LP expression RP trailer ( ELSE trailer | { _input.LA(1) != ELSE }? )                 # if
+    | WHILE LP expression RP ( trailer | empty )                                               # while
+    | DO block WHILE LP expression RP delimiter                                                # do
+    | FOR LP initializer? SEMICOLON expression? SEMICOLON afterthought? RP ( trailer | empty ) # for
+    | declaration delimiter                                                                    # decl
+    | CONTINUE delimiter                                                                       # continue
+    | BREAK delimiter                                                                          # break
+    | RETURN expression delimiter                                                              # return
+    | TRY block trap+                                                                          # try
+    | THROW expression delimiter                                                               # throw
+    | expression delimiter                                                                     # expr
+    ;
+
+trailer
+    : block
+    | statement
     ;
 
 block
-    : LBRACK statement+ RBRACK # multiple
-    | statement                # single
+    : LBRACK statement* RBRACK
     ;
 
 empty
-    : emptyscope
-    | SEMICOLON
-    ;
-
-emptyscope
-    : LBRACK RBRACK
+    : SEMICOLON
     ;
 
 initializer
@@ -63,75 +65,96 @@ afterthought
     ;
 
 declaration
-    : decltype declvar ( COMMA declvar )*
+    : decltype declvar (COMMA declvar)*
     ;
 
 decltype
-    : identifier (LBRACE RBRACE)*
+    : TYPE (LBRACE RBRACE)*
     ;
 
 declvar
-    : identifier ( ASSIGN expression )?
+    : ID ( ASSIGN expression )?
     ;
 
 trap
-    : CATCH LP ( identifier identifier ) RP ( block | emptyscope )
+    : CATCH LP TYPE ID RP block
     ;
 
-identifier
-    : ID generic?
+delimiter
+    : SEMICOLON
+    | EOF
     ;
 
-generic
-    : LT identifier ( COMMA identifier )* GT
+// Note we return the boolean s.  This is returned as true
+// if secondaries (postfixes) are allowed, otherwise, false.
+// This prevents illegal secondaries from being appended to
+// expressions using precedence that aren't variable/method chains.
+expression returns [boolean s = true]
+    :               u = unary[false]                                       { $s = $u.s; }           # single
+    |               expression ( MUL | DIV | REM ) expression              { $s = false; }          # binary
+    |               expression ( ADD | SUB ) expression                    { $s = false; }          # binary
+    |               expression ( LSH | RSH | USH ) expression              { $s = false; }          # binary
+    |               expression ( LT | LTE | GT | GTE ) expression          { $s = false; }          # comp
+    |               expression ( EQ | EQR | NE | NER ) expression          { $s = false; }          # comp
+    |               expression BWAND expression                            { $s = false; }          # binary
+    |               expression XOR expression                              { $s = false; }          # binary
+    |               expression BWOR expression                             { $s = false; }          # binary
+    |               expression BOOLAND expression                          { $s = false; }          # bool
+    |               expression BOOLOR expression                           { $s = false; }          # bool
+    | <assoc=right> expression COND e0 = expression COLON e1 = expression  { $s = $e0.s && $e1.s; } # conditional
+    // TODO: Should we allow crazy syntax like (x = 5).call()?
+    //       Other crazy syntaxes work, but this one requires
+    //       a complete restructure of the rules as EChain isn't
+    //       designed to handle more postfixes after an assignment.
+    | <assoc=right> chain[true] ( ASSIGN | AADD | ASUB | AMUL |
+                                  ADIV   | AREM | AAND | AXOR |
+                                  AOR    | ALSH | ARSH | AUSH ) expression { $s = false; }         # assignment
     ;
 
-expression
-    :               LP expression RP                                    # precedence
-    |               ( OCTAL | HEX | INTEGER | DECIMAL )                 # numeric
-    |               TRUE                                                # true
-    |               FALSE                                               # false
-    |               NULL                                                # null
-    | <assoc=right> chain ( INCR | DECR )                               # postinc
-    | <assoc=right> ( INCR | DECR ) chain                               # preinc
-    |               chain                                               # read
-    | <assoc=right> ( BOOLNOT | BWNOT | ADD | SUB ) expression          # unary
-    | <assoc=right> LP decltype RP expression                           # cast
-    |               expression ( MUL | DIV | REM ) expression           # binary
-    |               expression ( ADD | SUB ) expression                 # binary
-    |               expression ( LSH | RSH | USH ) expression           # binary
-    |               expression ( LT | LTE | GT | GTE ) expression       # comp
-    |               expression ( EQ | EQR | NE | NER ) expression       # comp
-    |               expression BWAND expression                         # binary
-    |               expression XOR expression                           # binary
-    |               expression BWOR expression                          # binary
-    |               expression BOOLAND expression                       # bool
-    |               expression BOOLOR expression                        # bool
-    | <assoc=right> expression COND expression COLON expression         # conditional
-    | <assoc=right> chain ( ASSIGN | AADD | ASUB | AMUL | ADIV
-                                      | AREM | AAND | AXOR | AOR
-                                      | ALSH | ARSH | AUSH ) expression # assignment
+// Note we take in the boolean c.  This is used to indicate
+// whether or not this rule was called when we are already
+// processing a variable/method chain.  This prevents the chain
+// from being applied to rules where it wouldn't be allowed.
+unary[boolean c] returns [boolean s = true]
+    : { !$c }? ( INCR | DECR ) chain[true]                                  # pre
+    | { !$c }? chain[true] (INCR | DECR )                                   # post
+    | { !$c }? chain[false]                                                 # read
+    | { !$c }? ( OCTAL | HEX | INTEGER | DECIMAL )          { $s = false; } # numeric
+    | { !$c }? TRUE                                         { $s = false; } # true
+    | { !$c }? FALSE                                        { $s = false; } # false
+    | { !$c }? NULL                                         { $s = false; } # null
+    | { !$c }? ( BOOLNOT | BWNOT | ADD | SUB ) unary[false]                 # operator
+    |          LP decltype RP unary[$c]                                     # cast
     ;
 
-chain
-    : linkprec
-    | linkcast
-    | linkvar
-    | linknew
-    | linkstring
+chain[boolean c]
+    : p = primary[$c] secondary[$p.s]*                             # dynamic
+    | decltype dot secondary[true]*                                # static
+    | NEW TYPE (LBRACE expression RBRACE)+ (dot secondary[true]*)? # newarray
     ;
 
-linkprec:   LP ( linkprec | linkcast | linkvar | linknew | linkstring ) RP ( linkdot | linkbrace )?;
-linkcast:   LP decltype RP ( linkprec | linkcast | linkvar | linknew | linkstring );
-linkbrace:  LBRACE expression RBRACE ( linkdot | linkbrace )?;
-linkdot:    DOT ( linkcall | linkfield );
-linkcall:   EXTID arguments ( linkdot | linkbrace )?;
-linkvar:    identifier ( linkdot | linkbrace )?;
-linkfield:  ( EXTID | EXTINTEGER ) ( linkdot | linkbrace )?;
-linknew:    NEW identifier ( ( arguments linkdot? ) | ( ( LBRACE expression RBRACE )+ linkdot? ) );
-linkstring: STRING (linkdot | linkbrace )?;
+primary[boolean c] returns [boolean s = true]
+    : { !$c }? LP e = expression RP { $s = $e.s; } # exprprec
+    | { $c }?  LP unary[true] RP                   # chainprec
+    |          STRING                              # string
+    |          ID                                  # variable
+    |          NEW TYPE arguments                  # newobject
+    ;
+
+secondary[boolean s]
+    : { $s }? dot
+    | { $s }? brace
+    ;
+
+dot
+    : DOT DOTID arguments        # callinvoke
+    | DOT ( DOTID | DOTINTEGER ) # fieldaccess
+    ;
+
+brace
+    : LBRACE expression RBRACE # braceaccess
+    ;
 
 arguments
     : ( LP ( expression ( COMMA expression )* )? RP )
     ;
-
diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.tokens b/modules/lang-painless/src/main/antlr/PainlessParser.tokens
new file mode 100644
index 00000000000..d42e157e964
--- /dev/null
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.tokens
@@ -0,0 +1,137 @@
+WS=1
+COMMENT=2
+LBRACK=3
+RBRACK=4
+LBRACE=5
+RBRACE=6
+LP=7
+RP=8
+DOT=9
+COMMA=10
+SEMICOLON=11
+IF=12
+ELSE=13
+WHILE=14
+DO=15
+FOR=16
+CONTINUE=17
+BREAK=18
+RETURN=19
+NEW=20
+TRY=21
+CATCH=22
+THROW=23
+BOOLNOT=24
+BWNOT=25
+MUL=26
+DIV=27
+REM=28
+ADD=29
+SUB=30
+LSH=31
+RSH=32
+USH=33
+LT=34
+LTE=35
+GT=36
+GTE=37
+EQ=38
+EQR=39
+NE=40
+NER=41
+BWAND=42
+XOR=43
+BWOR=44
+BOOLAND=45
+BOOLOR=46
+COND=47
+COLON=48
+INCR=49
+DECR=50
+ASSIGN=51
+AADD=52
+ASUB=53
+AMUL=54
+ADIV=55
+AREM=56
+AAND=57
+AXOR=58
+AOR=59
+ALSH=60
+ARSH=61
+AUSH=62
+OCTAL=63
+HEX=64
+INTEGER=65
+DECIMAL=66
+STRING=67
+TRUE=68
+FALSE=69
+NULL=70
+TYPE=71
+ID=72
+DOTINTEGER=73
+DOTID=74
+'{'=3
+'}'=4
+'['=5
+']'=6
+'('=7
+')'=8
+'.'=9
+','=10
+';'=11
+'if'=12
+'else'=13
+'while'=14
+'do'=15
+'for'=16
+'continue'=17
+'break'=18
+'return'=19
+'new'=20
+'try'=21
+'catch'=22
+'throw'=23
+'!'=24
+'~'=25
+'*'=26
+'/'=27
+'%'=28
+'+'=29
+'-'=30
+'<<'=31
+'>>'=32
+'>>>'=33
+'<'=34
+'<='=35
+'>'=36
+'>='=37
+'=='=38
+'==='=39
+'!='=40
+'!=='=41
+'&'=42
+'^'=43
+'|'=44
+'&&'=45
+'||'=46
+'?'=47
+':'=48
+'++'=49
+'--'=50
+'='=51
+'+='=52
+'-='=53
+'*='=54
+'/='=55
+'%='=56
+'&='=57
+'^='=58
+'|='=59
+'<<='=60
+'>>='=61
+'>>>='=62
+'true'=68
+'false'=69
+'null'=70
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
index e81e828c875..a06bcdf9840 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
@@ -26,9 +26,8 @@ import org.elasticsearch.painless.node.SSource;
  * Runs the analysis phase of compilation using the Painless AST.
  */
 final class Analyzer {
-    static Variables analyze(final CompilerSettings settings,
-                                    final Reserved shortcut, final SSource root) {
-        final Variables variables = new Variables(settings, shortcut);
+    static Variables analyze(Reserved shortcut, SSource root) {
+        Variables variables = new Variables(shortcut);
         root.analyze(variables);
 
         return variables;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
index 16ad355177f..666f28bb406 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
@@ -57,7 +57,7 @@ final class Compiler {
         try {
             // Setup the code privileges.
             CODESOURCE = new CodeSource(new URL("file:" + BootstrapInfo.UNTRUSTED_CODEBASE), (Certificate[]) null);
-        } catch (final MalformedURLException impossible) {
+        } catch (MalformedURLException impossible) {
             throw new RuntimeException(impossible);
         }
     }
@@ -69,7 +69,7 @@ final class Compiler {
         /**
          * @param parent The parent ClassLoader.
          */
-        Loader(final ClassLoader parent) {
+        Loader(ClassLoader parent) {
             super(parent);
         }
 
@@ -79,7 +79,7 @@ final class Compiler {
          * @param bytes The generated byte code.
          * @return A Class object extending {@link Executable}.
          */
-        Class<? extends Executable> define(final String name, final byte[] bytes) {
+        Class<? extends Executable> define(String name, byte[] bytes) {
             return defineClass(name, bytes, 0, bytes.length, CODESOURCE).asSubclass(Executable.class);
         }
     }
@@ -92,7 +92,7 @@ final class Compiler {
      * @param settings The CompilerSettings to be used during the compilation.
      * @return An {@link Executable} Painless script.
      */
-    static Executable compile(final Loader loader, final String name, final String source, final CompilerSettings settings) {
+    static Executable compile(Loader loader, String name, String source, CompilerSettings settings) {
         byte[] bytes = compile(name, source, settings);
 
         return createExecutable(loader, name, source, bytes);
@@ -111,9 +111,9 @@ final class Compiler {
                 " plugin if a script longer than this length is a requirement.");
         }
 
-        final Reserved reserved = new Reserved();
-        final SSource root = Walker.buildPainlessTree(source, reserved, settings);
-        final Variables variables = Analyzer.analyze(settings, reserved, root);
+        Reserved reserved = new Reserved();
+        SSource root = Walker.buildPainlessTree(source, reserved, settings);
+        Variables variables = Analyzer.analyze(reserved, root);
 
         return Writer.write(settings, name, source, variables, root);
     }
@@ -126,13 +126,13 @@ final class Compiler {
      * @param bytes The ASM generated byte code to define the class with.
      * @return A Painless {@link Executable} script.
      */
-    private static Executable createExecutable(final Loader loader, final String name, final String source, final byte[] bytes) {
+    private static Executable createExecutable(Loader loader, String name, String source, byte[] bytes) {
         try {
-            final Class<? extends Executable> clazz = loader.define(CLASS_NAME, bytes);
-            final java.lang.reflect.Constructor<? extends Executable> constructor = clazz.getConstructor(String.class, String.class);
+            Class<? extends Executable> clazz = loader.define(CLASS_NAME, bytes);
+            java.lang.reflect.Constructor<? extends Executable> constructor = clazz.getConstructor(String.class, String.class);
 
             return constructor.newInstance(name, source);
-        } catch (final Exception exception) { // Catch everything to let the user know this is something caused internally.
+        } catch (Exception exception) { // Catch everything to let the user know this is something caused internally.
             throw new IllegalStateException(
                     "An internal error occurred attempting to define the script [" + name + "].", exception);
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java
index 4cafe32bb56..79b75b32b98 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java
@@ -29,11 +29,22 @@ public final class CompilerSettings {
      */
     public static final String MAX_LOOP_COUNTER = "max_loop_counter";
 
+    /**
+     * Constant to be used for enabling additional internal compilation checks (slower).
+     */
+    public static final String PICKY = "picky";
+
     /**
      * The maximum number of statements allowed to be run in a loop.
      */
     private int maxLoopCounter = 10000;
 
+    /**
+     * Whether to throw exception on ambiguity or other internal parsing issues. This option
+     * makes things slower too, it is only for debugging.
+     */
+    private boolean picky = false;
+
     /**
      * Returns the value for the cumulative total number of statements that can be made in all loops
      * in a script before an exception is thrown.  This attempts to prevent infinite loops.  Note if
@@ -50,4 +61,21 @@ public final class CompilerSettings {
     public final void setMaxLoopCounter(int max) {
         this.maxLoopCounter = max;
     }
+
+    /**
+     * Returns true if the compiler should be picky. This means it runs slower and enables additional
+     * runtime checks, throwing an exception if there are ambiguities in the grammar or other low level
+     * parsing problems.
+     */
+    public boolean isPicky() {
+      return picky;
+    }
+
+    /**
+     * Set to true if compilation should be picky.
+     * @see #isPicky
+     */
+    public void setPicky(boolean picky) {
+      this.picky = picky;
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 05c12c30239..1aa8d199761 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -37,11 +37,11 @@ import java.util.Objects;
  * methods and fields during at both compile-time and runtime.
  */
 public final class Definition {
-    
+
     private static final String DEFINITION_FILE = "definition.txt";
 
     private static final Definition INSTANCE = new Definition();
-    
+
     /** Some native types as constants: */
     public static final Type VOID_TYPE = getType("void");
     public static final Type BOOLEAN_TYPE = getType("boolean");
@@ -290,7 +290,7 @@ public final class Definition {
             staticMembers = new HashMap<>();
             members = new HashMap<>();
         }
-        
+
         private Struct(final Struct struct) {
             name = struct.name;
             clazz = struct.clazz;
@@ -374,6 +374,22 @@ public final class Definition {
         }
     }
 
+    /** Returns whether or not a non-array type exists. */
+    public static boolean isSimpleType(final String name) {
+        return INSTANCE.structsMap.containsKey(name);
+    }
+
+    /** Returns whether or not a type exists without an exception. */
+    public static boolean isType(final String name) {
+        try {
+            INSTANCE.getTypeInternal(name);
+
+            return true;
+        } catch (IllegalArgumentException exception) {
+            return false;
+        }
+    }
+
     /** Gets the type given by its name */
     public static Type getType(final String name) {
         return INSTANCE.getTypeInternal(name);
@@ -383,13 +399,13 @@ public final class Definition {
     public static Type getType(final Struct struct, final int dimensions) {
         return INSTANCE.getTypeInternal(struct, dimensions);
     }
-    
+
     public static RuntimeClass getRuntimeClass(Class<?> clazz) {
         return INSTANCE.runtimeMap.get(clazz);
     }
-    
+
     // INTERNAL IMPLEMENTATION:
-    
+
     private final Map<Class<?>, RuntimeClass> runtimeMap;
     private final Map<String, Struct> structsMap;
     private final Map<String, Type> simpleTypesMap;
@@ -898,15 +914,17 @@ public final class Definition {
         runtimeMap.put(struct.clazz, new RuntimeClass(methods, getters, setters));
     }
 
-    private Type getTypeInternal(final String name) {
+    private Type getTypeInternal(String name) {
         // simple types (e.g. 0 array dimensions) are a simple hash lookup for speed
         Type simple = simpleTypesMap.get(name);
+
         if (simple != null) {
             return simple;
         }
-        final int dimensions = getDimensions(name);
-        final String structstr = dimensions == 0 ? name : name.substring(0, name.indexOf('['));
-        final Struct struct = structsMap.get(structstr);
+
+        int dimensions = getDimensions(name);
+        String structstr = dimensions == 0 ? name : name.substring(0, name.indexOf('['));
+        Struct struct = structsMap.get(structstr);
 
         if (struct == null) {
             throw new IllegalArgumentException("The struct with name [" + name + "] has not been defined.");
@@ -915,29 +933,29 @@ public final class Definition {
         return getTypeInternal(struct, dimensions);
     }
 
-    private Type getTypeInternal(final Struct struct, final int dimensions) {
+    private Type getTypeInternal(Struct struct, int dimensions) {
         String name = struct.name;
         org.objectweb.asm.Type type = struct.type;
         Class<?> clazz = struct.clazz;
         Sort sort;
 
         if (dimensions > 0) {
-            final StringBuilder builder = new StringBuilder(name);
-            final char[] brackets = new char[dimensions];
+            StringBuilder builder = new StringBuilder(name);
+            char[] brackets = new char[dimensions];
 
             for (int count = 0; count < dimensions; ++count) {
                 builder.append("[]");
                 brackets[count] = '[';
             }
 
-            final String descriptor = new String(brackets) + struct.type.getDescriptor();
+            String descriptor = new String(brackets) + struct.type.getDescriptor();
 
             name = builder.toString();
             type = org.objectweb.asm.Type.getType(descriptor);
 
             try {
                 clazz = Class.forName(type.getInternalName().replace('/', '.'));
-            } catch (final ClassNotFoundException exception) {
+            } catch (ClassNotFoundException exception) {
                 throw new IllegalArgumentException("The class [" + type.getInternalName() + "]" +
                     " could not be found to create type [" + name + "].");
             }
@@ -948,7 +966,7 @@ public final class Definition {
         } else {
             sort = Sort.OBJECT;
 
-            for (final Sort value : Sort.values()) {
+            for (Sort value : Sort.values()) {
                 if (value.clazz == null) {
                     continue;
                 }
@@ -964,12 +982,12 @@ public final class Definition {
         return new Type(name, dimensions, struct, clazz, type, sort);
     }
 
-    private int getDimensions(final String name) {
+    private int getDimensions(String name) {
         int dimensions = 0;
         int index = name.indexOf('[');
 
         if (index != -1) {
-            final int length = name.length();
+            int length = name.length();
 
             while (index < length) {
                 if (name.charAt(index) == '[' && ++index < length && name.charAt(index++) == ']') {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
index 72a657cd7f0..f9ee949e3d4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
@@ -121,6 +121,12 @@ public final class PainlessScriptEngineService extends AbstractComponent impleme
                 compilerSettings.setMaxLoopCounter(Integer.parseInt(value));
             }
 
+            value = copy.remove(CompilerSettings.PICKY);
+
+            if (value != null) {
+                compilerSettings.setPicky(Boolean.parseBoolean(value));
+            }
+
             if (!copy.isEmpty()) {
                 throw new IllegalArgumentException("Unrecognized compile-time parameter(s): " + copy);
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
index 28a7416d800..4443b1dc0f8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
@@ -50,7 +50,7 @@ public final class Variables {
         boolean ctx = false;
         boolean loop = false;
 
-        public void markReserved(final String name) {
+        public void markReserved(String name) {
             if (SCORE.equals(name)) {
                 score = true;
             } else if (CTX.equals(name)) {
@@ -58,7 +58,7 @@ public final class Variables {
             }
         }
 
-        public boolean isReserved(final String name) {
+        public boolean isReserved(String name) {
             return name.equals(THIS) || name.equals(PARAMS) || name.equals(SCORER) || name.equals(DOC) ||
                 name.equals(VALUE) || name.equals(SCORE) || name.equals(CTX) || name.equals(LOOP);
          }
@@ -77,7 +77,7 @@ public final class Variables {
 
         public boolean read = false;
 
-        private Variable(final String location, final String name, final Type type, final int slot, final boolean readonly) {
+        private Variable(String location, String name, Type type, int slot, boolean readonly) {
             this.location = location;
             this.name = name;
             this.type = type;
@@ -91,7 +91,7 @@ public final class Variables {
     private final Deque<Integer> scopes = new ArrayDeque<>();
     private final Deque<Variable> variables = new ArrayDeque<>();
 
-    public Variables(final CompilerSettings settings, final Reserved reserved) {
+    public Variables(Reserved reserved) {
         this.reserved = reserved;
 
         incrementScope();
@@ -99,35 +99,35 @@ public final class Variables {
         // Method variables.
 
         // This reference.  Internal use only.
-        addVariable("[" + Reserved.THIS + "]"  , "Executable", Reserved.THIS  , true, true);
+        addVariable("[" + Reserved.THIS + "]", Definition.getType("Executable"), Reserved.THIS, true, true);
 
-        // Input map of variables passed to the script.  TODO: Rename to 'params' since that will be its use.
-        addVariable("[" + Reserved.PARAMS + "]", "Map", Reserved.PARAMS, true, true);
+        // Input map of variables passed to the script.
+        addVariable("[" + Reserved.PARAMS + "]", Definition.getType("Map"), Reserved.PARAMS, true, true);
 
         // Scorer parameter passed to the script.  Internal use only.
-        addVariable("[" + Reserved.SCORER + "]", "def", Reserved.SCORER, true, true);
+        addVariable("[" + Reserved.SCORER + "]", Definition.DEF_TYPE, Reserved.SCORER, true, true);
 
         // Doc parameter passed to the script. TODO: Currently working as a Map, we can do better?
-        addVariable("[" + Reserved.DOC + "]"   , "Map", Reserved.DOC   , true, true);
+        addVariable("[" + Reserved.DOC + "]", Definition.getType("Map"), Reserved.DOC, true, true);
 
         // Aggregation _value parameter passed to the script.
-        addVariable("[" + Reserved.VALUE + "]" , "def", Reserved.VALUE , true, true);
+        addVariable("[" + Reserved.VALUE + "]", Definition.DEF_TYPE, Reserved.VALUE, true, true);
 
         // Shortcut variables.
 
         // Document's score as a read-only double.
         if (reserved.score) {
-            addVariable("[" + Reserved.SCORE + "]", "double", Reserved.SCORE, true, true);
+            addVariable("[" + Reserved.SCORE + "]", Definition.DOUBLE_TYPE, Reserved.SCORE, true, true);
         }
 
         // The ctx map set by executable scripts as a read-only map.
         if (reserved.ctx) {
-            addVariable("[" + Reserved.CTX + "]", "Map", Reserved.CTX, true, true);
+            addVariable("[" + Reserved.CTX + "]", Definition.getType("Map"), Reserved.CTX, true, true);
         }
 
         // Loop counter to catch infinite loops.  Internal use only.
-        if (reserved.loop && settings.getMaxLoopCounter() > 0) {
-            addVariable("[" + Reserved.LOOP + "]", "int", Reserved.LOOP, true, true);
+        if (reserved.loop) {
+            addVariable("[" + Reserved.LOOP + "]", Definition.INT_TYPE, Reserved.LOOP, true, true);
         }
     }
 
@@ -139,7 +139,7 @@ public final class Variables {
         int remove = scopes.pop();
 
         while (remove > 0) {
-            final Variable variable = variables.pop();
+             Variable variable = variables.pop();
 
             if (variable.read) {
                 throw new IllegalArgumentException("Error [" + variable.location + "]: Variable [" + variable.name + "] never used.");
@@ -149,11 +149,11 @@ public final class Variables {
         }
     }
 
-    public Variable getVariable(final String location, final String name) {
-        final Iterator<Variable> itr = variables.iterator();
+    public Variable getVariable(String location, String name) {
+         Iterator<Variable> itr = variables.iterator();
 
         while (itr.hasNext()) {
-            final Variable variable = itr.next();
+             Variable variable = itr.next();
 
             if (variable.name.equals(name)) {
                 return variable;
@@ -167,8 +167,7 @@ public final class Variables {
         return null;
     }
 
-    public Variable addVariable(final String location, final String typestr, final String name,
-                                final boolean readonly, final boolean reserved) {
+    public Variable addVariable(String location, Type type, String name, boolean readonly, boolean reserved) {
         if (!reserved && this.reserved.isReserved(name)) {
             throw new IllegalArgumentException("Error " + location + ": Variable name [" + name + "] is reserved.");
         }
@@ -177,38 +176,23 @@ public final class Variables {
             throw new IllegalArgumentException("Error " + location + ": Variable name [" + name + "] already defined.");
         }
 
-        final Type type;
-
-        try {
-            type = Definition.getType(typestr);
-        } catch (final IllegalArgumentException exception) {
-            throw new IllegalArgumentException("Error " + location + ": Not a type [" + typestr + "].");
-        }
-
-        boolean legal = !name.contains("<");
-
         try {
             Definition.getType(name);
-            legal = false;
-        } catch (final IllegalArgumentException exception) {
+        } catch (IllegalArgumentException exception) {
             // Do nothing.
         }
 
-        if (!legal) {
-            throw new IllegalArgumentException("Error " + location + ": Variable name [" + name + "] cannot be a type.");
-        }
-
-        final Variable previous = variables.peekFirst();
+        Variable previous = variables.peekFirst();
         int slot = 0;
 
         if (previous != null) {
             slot = previous.slot + previous.type.type.getSize();
         }
 
-        final Variable variable = new Variable(location, name, type, slot, readonly);
+        Variable variable = new Variable(location, name, type, slot, readonly);
         variables.push(variable);
 
-        final int update = scopes.pop() + 1;
+        int update = scopes.pop() + 1;
         scopes.push(update);
 
         return variable;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
index 8cb9a605868..b85217f9490 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
@@ -1,5 +1,8 @@
 // ANTLR GENERATED CODE: DO NOT EDIT
 package org.elasticsearch.painless.antlr;
+
+import org.elasticsearch.painless.Definition;
+
 import org.antlr.v4.runtime.Lexer;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.Token;
@@ -25,10 +28,10 @@ class PainlessLexer extends Lexer {
     BWOR=44, BOOLAND=45, BOOLOR=46, COND=47, COLON=48, INCR=49, DECR=50, ASSIGN=51, 
     AADD=52, ASUB=53, AMUL=54, ADIV=55, AREM=56, AAND=57, AXOR=58, AOR=59, 
     ALSH=60, ARSH=61, AUSH=62, OCTAL=63, HEX=64, INTEGER=65, DECIMAL=66, STRING=67, 
-    TRUE=68, FALSE=69, NULL=70, ID=71, EXTINTEGER=72, EXTID=73;
-  public static final int EXT = 1;
+    TRUE=68, FALSE=69, NULL=70, TYPE=71, ID=72, DOTINTEGER=73, DOTID=74;
+  public static final int AFTER_DOT = 1;
   public static String[] modeNames = {
-    "DEFAULT_MODE", "EXT"
+    "DEFAULT_MODE", "AFTER_DOT"
   };
 
   public static final String[] ruleNames = {
@@ -39,8 +42,8 @@ class PainlessLexer extends Lexer {
     "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", "BOOLAND", "BOOLOR", 
     "COND", "COLON", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", 
     "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", 
-    "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "ID", "EXTINTEGER", 
-    "EXTID"
+    "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "TYPE", "ID", 
+    "DOTINTEGER", "DOTID"
   };
 
   private static final String[] _LITERAL_NAMES = {
@@ -61,8 +64,8 @@ class PainlessLexer extends Lexer {
     "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", "BOOLAND", "BOOLOR", 
     "COND", "COLON", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", 
     "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", 
-    "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "ID", "EXTINTEGER", 
-    "EXTID"
+    "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "TYPE", "ID", 
+    "DOTINTEGER", "DOTID"
   };
   public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
 
@@ -118,8 +121,24 @@ class PainlessLexer extends Lexer {
   @Override
   public ATN getATN() { return _ATN; }
 
+  @Override
+  public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) {
+    switch (ruleIndex) {
+    case 70:
+      return TYPE_sempred((RuleContext)_localctx, predIndex);
+    }
+    return true;
+  }
+  private boolean TYPE_sempred(RuleContext _localctx, int predIndex) {
+    switch (predIndex) {
+    case 0:
+      return  Definition.isSimpleType(getText()) ;
+    }
+    return true;
+  }
+
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2K\u01fb\b\1\b\1\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2L\u0209\b\1\b\1\4"+
     "\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n"+
     "\4\13\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22"+
     "\t\22\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31"+
@@ -128,172 +147,178 @@ class PainlessLexer extends Lexer {
     "+\4,\t,\4-\t-\4.\t.\4/\t/\4\60\t\60\4\61\t\61\4\62\t\62\4\63\t\63\4\64"+
     "\t\64\4\65\t\65\4\66\t\66\4\67\t\67\48\t8\49\t9\4:\t:\4;\t;\4<\t<\4=\t"+
     "=\4>\t>\4?\t?\4@\t@\4A\tA\4B\tB\4C\tC\4D\tD\4E\tE\4F\tF\4G\tG\4H\tH\4"+
-    "I\tI\4J\tJ\3\2\6\2\u0098\n\2\r\2\16\2\u0099\3\2\3\2\3\3\3\3\3\3\3\3\7"+
-    "\3\u00a2\n\3\f\3\16\3\u00a5\13\3\3\3\3\3\3\3\3\3\3\3\7\3\u00ac\n\3\f\3"+
-    "\16\3\u00af\13\3\3\3\3\3\5\3\u00b3\n\3\3\3\3\3\3\4\3\4\3\5\3\5\3\6\3\6"+
-    "\3\7\3\7\3\b\3\b\3\t\3\t\3\n\3\n\3\n\3\n\3\13\3\13\3\f\3\f\3\r\3\r\3\r"+
-    "\3\16\3\16\3\16\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3\17\3\20\3\20\3\20"+
-    "\3\21\3\21\3\21\3\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\23"+
-    "\3\23\3\23\3\23\3\23\3\23\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\25\3\25"+
-    "\3\25\3\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\27\3\27\3\27\3\30\3\30"+
-    "\3\30\3\30\3\30\3\30\3\31\3\31\3\32\3\32\3\33\3\33\3\34\3\34\3\35\3\35"+
-    "\3\36\3\36\3\37\3\37\3 \3 \3 \3!\3!\3!\3\"\3\"\3\"\3\"\3#\3#\3$\3$\3$"+
-    "\3%\3%\3&\3&\3&\3\'\3\'\3\'\3(\3(\3(\3(\3)\3)\3)\3*\3*\3*\3*\3+\3+\3,"+
-    "\3,\3-\3-\3.\3.\3.\3/\3/\3/\3\60\3\60\3\61\3\61\3\62\3\62\3\62\3\63\3"+
-    "\63\3\63\3\64\3\64\3\65\3\65\3\65\3\66\3\66\3\66\3\67\3\67\3\67\38\38"+
-    "\38\39\39\39\3:\3:\3:\3;\3;\3;\3<\3<\3<\3=\3=\3=\3=\3>\3>\3>\3>\3?\3?"+
-    "\3?\3?\3?\3@\3@\6@\u0179\n@\r@\16@\u017a\3@\5@\u017e\n@\3A\3A\3A\6A\u0183"+
-    "\nA\rA\16A\u0184\3A\5A\u0188\nA\3B\3B\3B\7B\u018d\nB\fB\16B\u0190\13B"+
-    "\5B\u0192\nB\3B\5B\u0195\nB\3C\3C\3C\7C\u019a\nC\fC\16C\u019d\13C\5C\u019f"+
-    "\nC\3C\3C\7C\u01a3\nC\fC\16C\u01a6\13C\3C\3C\5C\u01aa\nC\3C\6C\u01ad\n"+
-    "C\rC\16C\u01ae\5C\u01b1\nC\3C\5C\u01b4\nC\3D\3D\3D\3D\3D\3D\7D\u01bc\n"+
-    "D\fD\16D\u01bf\13D\3D\3D\3D\3D\3D\3D\3D\7D\u01c8\nD\fD\16D\u01cb\13D\3"+
-    "D\5D\u01ce\nD\3E\3E\3E\3E\3E\3F\3F\3F\3F\3F\3F\3G\3G\3G\3G\3G\3H\3H\7"+
-    "H\u01e2\nH\fH\16H\u01e5\13H\3I\3I\3I\7I\u01ea\nI\fI\16I\u01ed\13I\5I\u01ef"+
-    "\nI\3I\3I\3J\3J\7J\u01f5\nJ\fJ\16J\u01f8\13J\3J\3J\6\u00a3\u00ad\u01bd"+
-    "\u01c9\2K\4\3\6\4\b\5\n\6\f\7\16\b\20\t\22\n\24\13\26\f\30\r\32\16\34"+
-    "\17\36\20 \21\"\22$\23&\24(\25*\26,\27.\30\60\31\62\32\64\33\66\348\35"+
-    ":\36<\37> @!B\"D#F$H%J&L\'N(P)R*T+V,X-Z.\\/^\60`\61b\62d\63f\64h\65j\66"+
-    "l\67n8p9r:t;v<x=z>|?~@\u0080A\u0082B\u0084C\u0086D\u0088E\u008aF\u008c"+
-    "G\u008eH\u0090I\u0092J\u0094K\4\2\3\21\5\2\13\f\17\17\"\"\4\2\f\f\17\17"+
-    "\3\2\629\4\2NNnn\4\2ZZzz\5\2\62;CHch\3\2\63;\3\2\62;\b\2FFHHNNffhhnn\4"+
-    "\2GGgg\4\2--//\4\2HHhh\4\2$$^^\5\2C\\aac|\6\2\62;C\\aac|\u0216\2\4\3\2"+
-    "\2\2\2\6\3\2\2\2\2\b\3\2\2\2\2\n\3\2\2\2\2\f\3\2\2\2\2\16\3\2\2\2\2\20"+
-    "\3\2\2\2\2\22\3\2\2\2\2\24\3\2\2\2\2\26\3\2\2\2\2\30\3\2\2\2\2\32\3\2"+
-    "\2\2\2\34\3\2\2\2\2\36\3\2\2\2\2 \3\2\2\2\2\"\3\2\2\2\2$\3\2\2\2\2&\3"+
-    "\2\2\2\2(\3\2\2\2\2*\3\2\2\2\2,\3\2\2\2\2.\3\2\2\2\2\60\3\2\2\2\2\62\3"+
-    "\2\2\2\2\64\3\2\2\2\2\66\3\2\2\2\28\3\2\2\2\2:\3\2\2\2\2<\3\2\2\2\2>\3"+
-    "\2\2\2\2@\3\2\2\2\2B\3\2\2\2\2D\3\2\2\2\2F\3\2\2\2\2H\3\2\2\2\2J\3\2\2"+
-    "\2\2L\3\2\2\2\2N\3\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2T\3\2\2\2\2V\3\2\2\2\2"+
-    "X\3\2\2\2\2Z\3\2\2\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3\2\2\2\2b\3\2\2\2\2d\3"+
-    "\2\2\2\2f\3\2\2\2\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2\2\2n\3\2\2\2\2p\3\2\2"+
-    "\2\2r\3\2\2\2\2t\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2z\3\2\2\2\2|\3\2\2\2\2"+
-    "~\3\2\2\2\2\u0080\3\2\2\2\2\u0082\3\2\2\2\2\u0084\3\2\2\2\2\u0086\3\2"+
-    "\2\2\2\u0088\3\2\2\2\2\u008a\3\2\2\2\2\u008c\3\2\2\2\2\u008e\3\2\2\2\2"+
-    "\u0090\3\2\2\2\3\u0092\3\2\2\2\3\u0094\3\2\2\2\4\u0097\3\2\2\2\6\u00b2"+
-    "\3\2\2\2\b\u00b6\3\2\2\2\n\u00b8\3\2\2\2\f\u00ba\3\2\2\2\16\u00bc\3\2"+
-    "\2\2\20\u00be\3\2\2\2\22\u00c0\3\2\2\2\24\u00c2\3\2\2\2\26\u00c6\3\2\2"+
-    "\2\30\u00c8\3\2\2\2\32\u00ca\3\2\2\2\34\u00cd\3\2\2\2\36\u00d2\3\2\2\2"+
-    " \u00d8\3\2\2\2\"\u00db\3\2\2\2$\u00df\3\2\2\2&\u00e8\3\2\2\2(\u00ee\3"+
-    "\2\2\2*\u00f5\3\2\2\2,\u00f9\3\2\2\2.\u00fd\3\2\2\2\60\u0103\3\2\2\2\62"+
-    "\u0109\3\2\2\2\64\u010b\3\2\2\2\66\u010d\3\2\2\28\u010f\3\2\2\2:\u0111"+
-    "\3\2\2\2<\u0113\3\2\2\2>\u0115\3\2\2\2@\u0117\3\2\2\2B\u011a\3\2\2\2D"+
-    "\u011d\3\2\2\2F\u0121\3\2\2\2H\u0123\3\2\2\2J\u0126\3\2\2\2L\u0128\3\2"+
-    "\2\2N\u012b\3\2\2\2P\u012e\3\2\2\2R\u0132\3\2\2\2T\u0135\3\2\2\2V\u0139"+
-    "\3\2\2\2X\u013b\3\2\2\2Z\u013d\3\2\2\2\\\u013f\3\2\2\2^\u0142\3\2\2\2"+
-    "`\u0145\3\2\2\2b\u0147\3\2\2\2d\u0149\3\2\2\2f\u014c\3\2\2\2h\u014f\3"+
-    "\2\2\2j\u0151\3\2\2\2l\u0154\3\2\2\2n\u0157\3\2\2\2p\u015a\3\2\2\2r\u015d"+
-    "\3\2\2\2t\u0160\3\2\2\2v\u0163\3\2\2\2x\u0166\3\2\2\2z\u0169\3\2\2\2|"+
-    "\u016d\3\2\2\2~\u0171\3\2\2\2\u0080\u0176\3\2\2\2\u0082\u017f\3\2\2\2"+
-    "\u0084\u0191\3\2\2\2\u0086\u019e\3\2\2\2\u0088\u01cd\3\2\2\2\u008a\u01cf"+
-    "\3\2\2\2\u008c\u01d4\3\2\2\2\u008e\u01da\3\2\2\2\u0090\u01df\3\2\2\2\u0092"+
-    "\u01ee\3\2\2\2\u0094\u01f2\3\2\2\2\u0096\u0098\t\2\2\2\u0097\u0096\3\2"+
-    "\2\2\u0098\u0099\3\2\2\2\u0099\u0097\3\2\2\2\u0099\u009a\3\2\2\2\u009a"+
-    "\u009b\3\2\2\2\u009b\u009c\b\2\2\2\u009c\5\3\2\2\2\u009d\u009e\7\61\2"+
-    "\2\u009e\u009f\7\61\2\2\u009f\u00a3\3\2\2\2\u00a0\u00a2\13\2\2\2\u00a1"+
-    "\u00a0\3\2\2\2\u00a2\u00a5\3\2\2\2\u00a3\u00a4\3\2\2\2\u00a3\u00a1\3\2"+
-    "\2\2\u00a4\u00a6\3\2\2\2\u00a5\u00a3\3\2\2\2\u00a6\u00b3\t\3\2\2\u00a7"+
-    "\u00a8\7\61\2\2\u00a8\u00a9\7,\2\2\u00a9\u00ad\3\2\2\2\u00aa\u00ac\13"+
-    "\2\2\2\u00ab\u00aa\3\2\2\2\u00ac\u00af\3\2\2\2\u00ad\u00ae\3\2\2\2\u00ad"+
-    "\u00ab\3\2\2\2\u00ae\u00b0\3\2\2\2\u00af\u00ad\3\2\2\2\u00b0\u00b1\7,"+
-    "\2\2\u00b1\u00b3\7\61\2\2\u00b2\u009d\3\2\2\2\u00b2\u00a7\3\2\2\2\u00b3"+
-    "\u00b4\3\2\2\2\u00b4\u00b5\b\3\2\2\u00b5\7\3\2\2\2\u00b6\u00b7\7}\2\2"+
-    "\u00b7\t\3\2\2\2\u00b8\u00b9\7\177\2\2\u00b9\13\3\2\2\2\u00ba\u00bb\7"+
-    "]\2\2\u00bb\r\3\2\2\2\u00bc\u00bd\7_\2\2\u00bd\17\3\2\2\2\u00be\u00bf"+
-    "\7*\2\2\u00bf\21\3\2\2\2\u00c0\u00c1\7+\2\2\u00c1\23\3\2\2\2\u00c2\u00c3"+
-    "\7\60\2\2\u00c3\u00c4\3\2\2\2\u00c4\u00c5\b\n\3\2\u00c5\25\3\2\2\2\u00c6"+
-    "\u00c7\7.\2\2\u00c7\27\3\2\2\2\u00c8\u00c9\7=\2\2\u00c9\31\3\2\2\2\u00ca"+
-    "\u00cb\7k\2\2\u00cb\u00cc\7h\2\2\u00cc\33\3\2\2\2\u00cd\u00ce\7g\2\2\u00ce"+
-    "\u00cf\7n\2\2\u00cf\u00d0\7u\2\2\u00d0\u00d1\7g\2\2\u00d1\35\3\2\2\2\u00d2"+
-    "\u00d3\7y\2\2\u00d3\u00d4\7j\2\2\u00d4\u00d5\7k\2\2\u00d5\u00d6\7n\2\2"+
-    "\u00d6\u00d7\7g\2\2\u00d7\37\3\2\2\2\u00d8\u00d9\7f\2\2\u00d9\u00da\7"+
-    "q\2\2\u00da!\3\2\2\2\u00db\u00dc\7h\2\2\u00dc\u00dd\7q\2\2\u00dd\u00de"+
-    "\7t\2\2\u00de#\3\2\2\2\u00df\u00e0\7e\2\2\u00e0\u00e1\7q\2\2\u00e1\u00e2"+
-    "\7p\2\2\u00e2\u00e3\7v\2\2\u00e3\u00e4\7k\2\2\u00e4\u00e5\7p\2\2\u00e5"+
-    "\u00e6\7w\2\2\u00e6\u00e7\7g\2\2\u00e7%\3\2\2\2\u00e8\u00e9\7d\2\2\u00e9"+
-    "\u00ea\7t\2\2\u00ea\u00eb\7g\2\2\u00eb\u00ec\7c\2\2\u00ec\u00ed\7m\2\2"+
-    "\u00ed\'\3\2\2\2\u00ee\u00ef\7t\2\2\u00ef\u00f0\7g\2\2\u00f0\u00f1\7v"+
-    "\2\2\u00f1\u00f2\7w\2\2\u00f2\u00f3\7t\2\2\u00f3\u00f4\7p\2\2\u00f4)\3"+
-    "\2\2\2\u00f5\u00f6\7p\2\2\u00f6\u00f7\7g\2\2\u00f7\u00f8\7y\2\2\u00f8"+
-    "+\3\2\2\2\u00f9\u00fa\7v\2\2\u00fa\u00fb\7t\2\2\u00fb\u00fc\7{\2\2\u00fc"+
-    "-\3\2\2\2\u00fd\u00fe\7e\2\2\u00fe\u00ff\7c\2\2\u00ff\u0100\7v\2\2\u0100"+
-    "\u0101\7e\2\2\u0101\u0102\7j\2\2\u0102/\3\2\2\2\u0103\u0104\7v\2\2\u0104"+
-    "\u0105\7j\2\2\u0105\u0106\7t\2\2\u0106\u0107\7q\2\2\u0107\u0108\7y\2\2"+
-    "\u0108\61\3\2\2\2\u0109\u010a\7#\2\2\u010a\63\3\2\2\2\u010b\u010c\7\u0080"+
-    "\2\2\u010c\65\3\2\2\2\u010d\u010e\7,\2\2\u010e\67\3\2\2\2\u010f\u0110"+
-    "\7\61\2\2\u01109\3\2\2\2\u0111\u0112\7\'\2\2\u0112;\3\2\2\2\u0113\u0114"+
-    "\7-\2\2\u0114=\3\2\2\2\u0115\u0116\7/\2\2\u0116?\3\2\2\2\u0117\u0118\7"+
-    ">\2\2\u0118\u0119\7>\2\2\u0119A\3\2\2\2\u011a\u011b\7@\2\2\u011b\u011c"+
-    "\7@\2\2\u011cC\3\2\2\2\u011d\u011e\7@\2\2\u011e\u011f\7@\2\2\u011f\u0120"+
-    "\7@\2\2\u0120E\3\2\2\2\u0121\u0122\7>\2\2\u0122G\3\2\2\2\u0123\u0124\7"+
-    ">\2\2\u0124\u0125\7?\2\2\u0125I\3\2\2\2\u0126\u0127\7@\2\2\u0127K\3\2"+
-    "\2\2\u0128\u0129\7@\2\2\u0129\u012a\7?\2\2\u012aM\3\2\2\2\u012b\u012c"+
-    "\7?\2\2\u012c\u012d\7?\2\2\u012dO\3\2\2\2\u012e\u012f\7?\2\2\u012f\u0130"+
-    "\7?\2\2\u0130\u0131\7?\2\2\u0131Q\3\2\2\2\u0132\u0133\7#\2\2\u0133\u0134"+
-    "\7?\2\2\u0134S\3\2\2\2\u0135\u0136\7#\2\2\u0136\u0137\7?\2\2\u0137\u0138"+
-    "\7?\2\2\u0138U\3\2\2\2\u0139\u013a\7(\2\2\u013aW\3\2\2\2\u013b\u013c\7"+
-    "`\2\2\u013cY\3\2\2\2\u013d\u013e\7~\2\2\u013e[\3\2\2\2\u013f\u0140\7("+
-    "\2\2\u0140\u0141\7(\2\2\u0141]\3\2\2\2\u0142\u0143\7~\2\2\u0143\u0144"+
-    "\7~\2\2\u0144_\3\2\2\2\u0145\u0146\7A\2\2\u0146a\3\2\2\2\u0147\u0148\7"+
-    "<\2\2\u0148c\3\2\2\2\u0149\u014a\7-\2\2\u014a\u014b\7-\2\2\u014be\3\2"+
-    "\2\2\u014c\u014d\7/\2\2\u014d\u014e\7/\2\2\u014eg\3\2\2\2\u014f\u0150"+
-    "\7?\2\2\u0150i\3\2\2\2\u0151\u0152\7-\2\2\u0152\u0153\7?\2\2\u0153k\3"+
-    "\2\2\2\u0154\u0155\7/\2\2\u0155\u0156\7?\2\2\u0156m\3\2\2\2\u0157\u0158"+
-    "\7,\2\2\u0158\u0159\7?\2\2\u0159o\3\2\2\2\u015a\u015b\7\61\2\2\u015b\u015c"+
-    "\7?\2\2\u015cq\3\2\2\2\u015d\u015e\7\'\2\2\u015e\u015f\7?\2\2\u015fs\3"+
-    "\2\2\2\u0160\u0161\7(\2\2\u0161\u0162\7?\2\2\u0162u\3\2\2\2\u0163\u0164"+
-    "\7`\2\2\u0164\u0165\7?\2\2\u0165w\3\2\2\2\u0166\u0167\7~\2\2\u0167\u0168"+
-    "\7?\2\2\u0168y\3\2\2\2\u0169\u016a\7>\2\2\u016a\u016b\7>\2\2\u016b\u016c"+
-    "\7?\2\2\u016c{\3\2\2\2\u016d\u016e\7@\2\2\u016e\u016f\7@\2\2\u016f\u0170"+
-    "\7?\2\2\u0170}\3\2\2\2\u0171\u0172\7@\2\2\u0172\u0173\7@\2\2\u0173\u0174"+
-    "\7@\2\2\u0174\u0175\7?\2\2\u0175\177\3\2\2\2\u0176\u0178\7\62\2\2\u0177"+
-    "\u0179\t\4\2\2\u0178\u0177\3\2\2\2\u0179\u017a\3\2\2\2\u017a\u0178\3\2"+
-    "\2\2\u017a\u017b\3\2\2\2\u017b\u017d\3\2\2\2\u017c\u017e\t\5\2\2\u017d"+
-    "\u017c\3\2\2\2\u017d\u017e\3\2\2\2\u017e\u0081\3\2\2\2\u017f\u0180\7\62"+
-    "\2\2\u0180\u0182\t\6\2\2\u0181\u0183\t\7\2\2\u0182\u0181\3\2\2\2\u0183"+
-    "\u0184\3\2\2\2\u0184\u0182\3\2\2\2\u0184\u0185\3\2\2\2\u0185\u0187\3\2"+
-    "\2\2\u0186\u0188\t\5\2\2\u0187\u0186\3\2\2\2\u0187\u0188\3\2\2\2\u0188"+
-    "\u0083\3\2\2\2\u0189\u0192\7\62\2\2\u018a\u018e\t\b\2\2\u018b\u018d\t"+
-    "\t\2\2\u018c\u018b\3\2\2\2\u018d\u0190\3\2\2\2\u018e\u018c\3\2\2\2\u018e"+
-    "\u018f\3\2\2\2\u018f\u0192\3\2\2\2\u0190\u018e\3\2\2\2\u0191\u0189\3\2"+
-    "\2\2\u0191\u018a\3\2\2\2\u0192\u0194\3\2\2\2\u0193\u0195\t\n\2\2\u0194"+
-    "\u0193\3\2\2\2\u0194\u0195\3\2\2\2\u0195\u0085\3\2\2\2\u0196\u019f\7\62"+
-    "\2\2\u0197\u019b\t\b\2\2\u0198\u019a\t\t\2\2\u0199\u0198\3\2\2\2\u019a"+
-    "\u019d\3\2\2\2\u019b\u0199\3\2\2\2\u019b\u019c\3\2\2\2\u019c\u019f\3\2"+
-    "\2\2\u019d\u019b\3\2\2\2\u019e\u0196\3\2\2\2\u019e\u0197\3\2\2\2\u019f"+
-    "\u01a0\3\2\2\2\u01a0\u01a4\5\24\n\2\u01a1\u01a3\t\t\2\2\u01a2\u01a1\3"+
-    "\2\2\2\u01a3\u01a6\3\2\2\2\u01a4\u01a2\3\2\2\2\u01a4\u01a5\3\2\2\2\u01a5"+
-    "\u01b0\3\2\2\2\u01a6\u01a4\3\2\2\2\u01a7\u01a9\t\13\2\2\u01a8\u01aa\t"+
-    "\f\2\2\u01a9\u01a8\3\2\2\2\u01a9\u01aa\3\2\2\2\u01aa\u01ac\3\2\2\2\u01ab"+
-    "\u01ad\t\t\2\2\u01ac\u01ab\3\2\2\2\u01ad\u01ae\3\2\2\2\u01ae\u01ac\3\2"+
-    "\2\2\u01ae\u01af\3\2\2\2\u01af\u01b1\3\2\2\2\u01b0\u01a7\3\2\2\2\u01b0"+
-    "\u01b1\3\2\2\2\u01b1\u01b3\3\2\2\2\u01b2\u01b4\t\r\2\2\u01b3\u01b2\3\2"+
-    "\2\2\u01b3\u01b4\3\2\2\2\u01b4\u0087\3\2\2\2\u01b5\u01bd\7$\2\2\u01b6"+
-    "\u01b7\7^\2\2\u01b7\u01bc\7$\2\2\u01b8\u01b9\7^\2\2\u01b9\u01bc\7^\2\2"+
-    "\u01ba\u01bc\n\16\2\2\u01bb\u01b6\3\2\2\2\u01bb\u01b8\3\2\2\2\u01bb\u01ba"+
-    "\3\2\2\2\u01bc\u01bf\3\2\2\2\u01bd\u01be\3\2\2\2\u01bd\u01bb\3\2\2\2\u01be"+
-    "\u01c0\3\2\2\2\u01bf\u01bd\3\2\2\2\u01c0\u01ce\7$\2\2\u01c1\u01c9\7)\2"+
-    "\2\u01c2\u01c3\7^\2\2\u01c3\u01c8\7)\2\2\u01c4\u01c5\7^\2\2\u01c5\u01c8"+
-    "\7^\2\2\u01c6\u01c8\n\16\2\2\u01c7\u01c2\3\2\2\2\u01c7\u01c4\3\2\2\2\u01c7"+
-    "\u01c6\3\2\2\2\u01c8\u01cb\3\2\2\2\u01c9\u01ca\3\2\2\2\u01c9\u01c7\3\2"+
-    "\2\2\u01ca\u01cc\3\2\2\2\u01cb\u01c9\3\2\2\2\u01cc\u01ce\7)\2\2\u01cd"+
-    "\u01b5\3\2\2\2\u01cd\u01c1\3\2\2\2\u01ce\u0089\3\2\2\2\u01cf\u01d0\7v"+
-    "\2\2\u01d0\u01d1\7t\2\2\u01d1\u01d2\7w\2\2\u01d2\u01d3\7g\2\2\u01d3\u008b"+
-    "\3\2\2\2\u01d4\u01d5\7h\2\2\u01d5\u01d6\7c\2\2\u01d6\u01d7\7n\2\2\u01d7"+
-    "\u01d8\7u\2\2\u01d8\u01d9\7g\2\2\u01d9\u008d\3\2\2\2\u01da\u01db\7p\2"+
-    "\2\u01db\u01dc\7w\2\2\u01dc\u01dd\7n\2\2\u01dd\u01de\7n\2\2\u01de\u008f"+
-    "\3\2\2\2\u01df\u01e3\t\17\2\2\u01e0\u01e2\t\20\2\2\u01e1\u01e0\3\2\2\2"+
-    "\u01e2\u01e5\3\2\2\2\u01e3\u01e1\3\2\2\2\u01e3\u01e4\3\2\2\2\u01e4\u0091"+
-    "\3\2\2\2\u01e5\u01e3\3\2\2\2\u01e6\u01ef\7\62\2\2\u01e7\u01eb\t\b\2\2"+
-    "\u01e8\u01ea\t\t\2\2\u01e9\u01e8\3\2\2\2\u01ea\u01ed\3\2\2\2\u01eb\u01e9"+
-    "\3\2\2\2\u01eb\u01ec\3\2\2\2\u01ec\u01ef\3\2\2\2\u01ed\u01eb\3\2\2\2\u01ee"+
-    "\u01e6\3\2\2\2\u01ee\u01e7\3\2\2\2\u01ef\u01f0\3\2\2\2\u01f0\u01f1\bI"+
-    "\4\2\u01f1\u0093\3\2\2\2\u01f2\u01f6\t\17\2\2\u01f3\u01f5\t\20\2\2\u01f4"+
-    "\u01f3\3\2\2\2\u01f5\u01f8\3\2\2\2\u01f6\u01f4\3\2\2\2\u01f6\u01f7\3\2"+
-    "\2\2\u01f7\u01f9\3\2\2\2\u01f8\u01f6\3\2\2\2\u01f9\u01fa\bJ\4\2\u01fa"+
-    "\u0095\3\2\2\2\37\2\3\u0099\u00a3\u00ad\u00b2\u017a\u017d\u0184\u0187"+
-    "\u018e\u0191\u0194\u019b\u019e\u01a4\u01a9\u01ae\u01b0\u01b3\u01bb\u01bd"+
-    "\u01c7\u01c9\u01cd\u01e3\u01eb\u01ee\u01f6\5\b\2\2\4\3\2\4\2\2";
+    "I\tI\4J\tJ\4K\tK\3\2\6\2\u009a\n\2\r\2\16\2\u009b\3\2\3\2\3\3\3\3\3\3"+
+    "\3\3\7\3\u00a4\n\3\f\3\16\3\u00a7\13\3\3\3\3\3\3\3\3\3\3\3\7\3\u00ae\n"+
+    "\3\f\3\16\3\u00b1\13\3\3\3\3\3\5\3\u00b5\n\3\3\3\3\3\3\4\3\4\3\5\3\5\3"+
+    "\6\3\6\3\7\3\7\3\b\3\b\3\t\3\t\3\n\3\n\3\n\3\n\3\13\3\13\3\f\3\f\3\r\3"+
+    "\r\3\r\3\16\3\16\3\16\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3\17\3\20\3\20"+
+    "\3\20\3\21\3\21\3\21\3\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22"+
+    "\3\23\3\23\3\23\3\23\3\23\3\23\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\25"+
+    "\3\25\3\25\3\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\27\3\27\3\27\3\30"+
+    "\3\30\3\30\3\30\3\30\3\30\3\31\3\31\3\32\3\32\3\33\3\33\3\34\3\34\3\35"+
+    "\3\35\3\36\3\36\3\37\3\37\3 \3 \3 \3!\3!\3!\3\"\3\"\3\"\3\"\3#\3#\3$\3"+
+    "$\3$\3%\3%\3&\3&\3&\3\'\3\'\3\'\3(\3(\3(\3(\3)\3)\3)\3*\3*\3*\3*\3+\3"+
+    "+\3,\3,\3-\3-\3.\3.\3.\3/\3/\3/\3\60\3\60\3\61\3\61\3\62\3\62\3\62\3\63"+
+    "\3\63\3\63\3\64\3\64\3\65\3\65\3\65\3\66\3\66\3\66\3\67\3\67\3\67\38\3"+
+    "8\38\39\39\39\3:\3:\3:\3;\3;\3;\3<\3<\3<\3=\3=\3=\3=\3>\3>\3>\3>\3?\3"+
+    "?\3?\3?\3?\3@\3@\6@\u017b\n@\r@\16@\u017c\3@\5@\u0180\n@\3A\3A\3A\6A\u0185"+
+    "\nA\rA\16A\u0186\3A\5A\u018a\nA\3B\3B\3B\7B\u018f\nB\fB\16B\u0192\13B"+
+    "\5B\u0194\nB\3B\5B\u0197\nB\3C\3C\3C\7C\u019c\nC\fC\16C\u019f\13C\5C\u01a1"+
+    "\nC\3C\3C\6C\u01a5\nC\rC\16C\u01a6\5C\u01a9\nC\3C\3C\5C\u01ad\nC\3C\6"+
+    "C\u01b0\nC\rC\16C\u01b1\5C\u01b4\nC\3C\5C\u01b7\nC\3D\3D\3D\3D\3D\3D\7"+
+    "D\u01bf\nD\fD\16D\u01c2\13D\3D\3D\3D\3D\3D\3D\3D\7D\u01cb\nD\fD\16D\u01ce"+
+    "\13D\3D\5D\u01d1\nD\3E\3E\3E\3E\3E\3F\3F\3F\3F\3F\3F\3G\3G\3G\3G\3G\3"+
+    "H\3H\3H\3H\7H\u01e7\nH\fH\16H\u01ea\13H\3H\3H\3I\3I\7I\u01f0\nI\fI\16"+
+    "I\u01f3\13I\3J\3J\3J\7J\u01f8\nJ\fJ\16J\u01fb\13J\5J\u01fd\nJ\3J\3J\3"+
+    "K\3K\7K\u0203\nK\fK\16K\u0206\13K\3K\3K\6\u00a5\u00af\u01c0\u01cc\2L\4"+
+    "\3\6\4\b\5\n\6\f\7\16\b\20\t\22\n\24\13\26\f\30\r\32\16\34\17\36\20 \21"+
+    "\"\22$\23&\24(\25*\26,\27.\30\60\31\62\32\64\33\66\348\35:\36<\37> @!"+
+    "B\"D#F$H%J&L\'N(P)R*T+V,X-Z.\\/^\60`\61b\62d\63f\64h\65j\66l\67n8p9r:"+
+    "t;v<x=z>|?~@\u0080A\u0082B\u0084C\u0086D\u0088E\u008aF\u008cG\u008eH\u0090"+
+    "I\u0092J\u0094K\u0096L\4\2\3\22\5\2\13\f\17\17\"\"\4\2\f\f\17\17\3\2\62"+
+    "9\4\2NNnn\4\2ZZzz\5\2\62;CHch\3\2\63;\3\2\62;\b\2FFHHNNffhhnn\4\2GGgg"+
+    "\4\2--//\4\2HHhh\4\2$$^^\5\2C\\aac|\6\2\62;C\\aac|\4\2aac|\u0226\2\4\3"+
+    "\2\2\2\2\6\3\2\2\2\2\b\3\2\2\2\2\n\3\2\2\2\2\f\3\2\2\2\2\16\3\2\2\2\2"+
+    "\20\3\2\2\2\2\22\3\2\2\2\2\24\3\2\2\2\2\26\3\2\2\2\2\30\3\2\2\2\2\32\3"+
+    "\2\2\2\2\34\3\2\2\2\2\36\3\2\2\2\2 \3\2\2\2\2\"\3\2\2\2\2$\3\2\2\2\2&"+
+    "\3\2\2\2\2(\3\2\2\2\2*\3\2\2\2\2,\3\2\2\2\2.\3\2\2\2\2\60\3\2\2\2\2\62"+
+    "\3\2\2\2\2\64\3\2\2\2\2\66\3\2\2\2\28\3\2\2\2\2:\3\2\2\2\2<\3\2\2\2\2"+
+    ">\3\2\2\2\2@\3\2\2\2\2B\3\2\2\2\2D\3\2\2\2\2F\3\2\2\2\2H\3\2\2\2\2J\3"+
+    "\2\2\2\2L\3\2\2\2\2N\3\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2T\3\2\2\2\2V\3\2\2"+
+    "\2\2X\3\2\2\2\2Z\3\2\2\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3\2\2\2\2b\3\2\2\2"+
+    "\2d\3\2\2\2\2f\3\2\2\2\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2\2\2n\3\2\2\2\2p"+
+    "\3\2\2\2\2r\3\2\2\2\2t\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2z\3\2\2\2\2|\3\2"+
+    "\2\2\2~\3\2\2\2\2\u0080\3\2\2\2\2\u0082\3\2\2\2\2\u0084\3\2\2\2\2\u0086"+
+    "\3\2\2\2\2\u0088\3\2\2\2\2\u008a\3\2\2\2\2\u008c\3\2\2\2\2\u008e\3\2\2"+
+    "\2\2\u0090\3\2\2\2\2\u0092\3\2\2\2\3\u0094\3\2\2\2\3\u0096\3\2\2\2\4\u0099"+
+    "\3\2\2\2\6\u00b4\3\2\2\2\b\u00b8\3\2\2\2\n\u00ba\3\2\2\2\f\u00bc\3\2\2"+
+    "\2\16\u00be\3\2\2\2\20\u00c0\3\2\2\2\22\u00c2\3\2\2\2\24\u00c4\3\2\2\2"+
+    "\26\u00c8\3\2\2\2\30\u00ca\3\2\2\2\32\u00cc\3\2\2\2\34\u00cf\3\2\2\2\36"+
+    "\u00d4\3\2\2\2 \u00da\3\2\2\2\"\u00dd\3\2\2\2$\u00e1\3\2\2\2&\u00ea\3"+
+    "\2\2\2(\u00f0\3\2\2\2*\u00f7\3\2\2\2,\u00fb\3\2\2\2.\u00ff\3\2\2\2\60"+
+    "\u0105\3\2\2\2\62\u010b\3\2\2\2\64\u010d\3\2\2\2\66\u010f\3\2\2\28\u0111"+
+    "\3\2\2\2:\u0113\3\2\2\2<\u0115\3\2\2\2>\u0117\3\2\2\2@\u0119\3\2\2\2B"+
+    "\u011c\3\2\2\2D\u011f\3\2\2\2F\u0123\3\2\2\2H\u0125\3\2\2\2J\u0128\3\2"+
+    "\2\2L\u012a\3\2\2\2N\u012d\3\2\2\2P\u0130\3\2\2\2R\u0134\3\2\2\2T\u0137"+
+    "\3\2\2\2V\u013b\3\2\2\2X\u013d\3\2\2\2Z\u013f\3\2\2\2\\\u0141\3\2\2\2"+
+    "^\u0144\3\2\2\2`\u0147\3\2\2\2b\u0149\3\2\2\2d\u014b\3\2\2\2f\u014e\3"+
+    "\2\2\2h\u0151\3\2\2\2j\u0153\3\2\2\2l\u0156\3\2\2\2n\u0159\3\2\2\2p\u015c"+
+    "\3\2\2\2r\u015f\3\2\2\2t\u0162\3\2\2\2v\u0165\3\2\2\2x\u0168\3\2\2\2z"+
+    "\u016b\3\2\2\2|\u016f\3\2\2\2~\u0173\3\2\2\2\u0080\u0178\3\2\2\2\u0082"+
+    "\u0181\3\2\2\2\u0084\u0193\3\2\2\2\u0086\u01a0\3\2\2\2\u0088\u01d0\3\2"+
+    "\2\2\u008a\u01d2\3\2\2\2\u008c\u01d7\3\2\2\2\u008e\u01dd\3\2\2\2\u0090"+
+    "\u01e2\3\2\2\2\u0092\u01ed\3\2\2\2\u0094\u01fc\3\2\2\2\u0096\u0200\3\2"+
+    "\2\2\u0098\u009a\t\2\2\2\u0099\u0098\3\2\2\2\u009a\u009b\3\2\2\2\u009b"+
+    "\u0099\3\2\2\2\u009b\u009c\3\2\2\2\u009c\u009d\3\2\2\2\u009d\u009e\b\2"+
+    "\2\2\u009e\5\3\2\2\2\u009f\u00a0\7\61\2\2\u00a0\u00a1\7\61\2\2\u00a1\u00a5"+
+    "\3\2\2\2\u00a2\u00a4\13\2\2\2\u00a3\u00a2\3\2\2\2\u00a4\u00a7\3\2\2\2"+
+    "\u00a5\u00a6\3\2\2\2\u00a5\u00a3\3\2\2\2\u00a6\u00a8\3\2\2\2\u00a7\u00a5"+
+    "\3\2\2\2\u00a8\u00b5\t\3\2\2\u00a9\u00aa\7\61\2\2\u00aa\u00ab\7,\2\2\u00ab"+
+    "\u00af\3\2\2\2\u00ac\u00ae\13\2\2\2\u00ad\u00ac\3\2\2\2\u00ae\u00b1\3"+
+    "\2\2\2\u00af\u00b0\3\2\2\2\u00af\u00ad\3\2\2\2\u00b0\u00b2\3\2\2\2\u00b1"+
+    "\u00af\3\2\2\2\u00b2\u00b3\7,\2\2\u00b3\u00b5\7\61\2\2\u00b4\u009f\3\2"+
+    "\2\2\u00b4\u00a9\3\2\2\2\u00b5\u00b6\3\2\2\2\u00b6\u00b7\b\3\2\2\u00b7"+
+    "\7\3\2\2\2\u00b8\u00b9\7}\2\2\u00b9\t\3\2\2\2\u00ba\u00bb\7\177\2\2\u00bb"+
+    "\13\3\2\2\2\u00bc\u00bd\7]\2\2\u00bd\r\3\2\2\2\u00be\u00bf\7_\2\2\u00bf"+
+    "\17\3\2\2\2\u00c0\u00c1\7*\2\2\u00c1\21\3\2\2\2\u00c2\u00c3\7+\2\2\u00c3"+
+    "\23\3\2\2\2\u00c4\u00c5\7\60\2\2\u00c5\u00c6\3\2\2\2\u00c6\u00c7\b\n\3"+
+    "\2\u00c7\25\3\2\2\2\u00c8\u00c9\7.\2\2\u00c9\27\3\2\2\2\u00ca\u00cb\7"+
+    "=\2\2\u00cb\31\3\2\2\2\u00cc\u00cd\7k\2\2\u00cd\u00ce\7h\2\2\u00ce\33"+
+    "\3\2\2\2\u00cf\u00d0\7g\2\2\u00d0\u00d1\7n\2\2\u00d1\u00d2\7u\2\2\u00d2"+
+    "\u00d3\7g\2\2\u00d3\35\3\2\2\2\u00d4\u00d5\7y\2\2\u00d5\u00d6\7j\2\2\u00d6"+
+    "\u00d7\7k\2\2\u00d7\u00d8\7n\2\2\u00d8\u00d9\7g\2\2\u00d9\37\3\2\2\2\u00da"+
+    "\u00db\7f\2\2\u00db\u00dc\7q\2\2\u00dc!\3\2\2\2\u00dd\u00de\7h\2\2\u00de"+
+    "\u00df\7q\2\2\u00df\u00e0\7t\2\2\u00e0#\3\2\2\2\u00e1\u00e2\7e\2\2\u00e2"+
+    "\u00e3\7q\2\2\u00e3\u00e4\7p\2\2\u00e4\u00e5\7v\2\2\u00e5\u00e6\7k\2\2"+
+    "\u00e6\u00e7\7p\2\2\u00e7\u00e8\7w\2\2\u00e8\u00e9\7g\2\2\u00e9%\3\2\2"+
+    "\2\u00ea\u00eb\7d\2\2\u00eb\u00ec\7t\2\2\u00ec\u00ed\7g\2\2\u00ed\u00ee"+
+    "\7c\2\2\u00ee\u00ef\7m\2\2\u00ef\'\3\2\2\2\u00f0\u00f1\7t\2\2\u00f1\u00f2"+
+    "\7g\2\2\u00f2\u00f3\7v\2\2\u00f3\u00f4\7w\2\2\u00f4\u00f5\7t\2\2\u00f5"+
+    "\u00f6\7p\2\2\u00f6)\3\2\2\2\u00f7\u00f8\7p\2\2\u00f8\u00f9\7g\2\2\u00f9"+
+    "\u00fa\7y\2\2\u00fa+\3\2\2\2\u00fb\u00fc\7v\2\2\u00fc\u00fd\7t\2\2\u00fd"+
+    "\u00fe\7{\2\2\u00fe-\3\2\2\2\u00ff\u0100\7e\2\2\u0100\u0101\7c\2\2\u0101"+
+    "\u0102\7v\2\2\u0102\u0103\7e\2\2\u0103\u0104\7j\2\2\u0104/\3\2\2\2\u0105"+
+    "\u0106\7v\2\2\u0106\u0107\7j\2\2\u0107\u0108\7t\2\2\u0108\u0109\7q\2\2"+
+    "\u0109\u010a\7y\2\2\u010a\61\3\2\2\2\u010b\u010c\7#\2\2\u010c\63\3\2\2"+
+    "\2\u010d\u010e\7\u0080\2\2\u010e\65\3\2\2\2\u010f\u0110\7,\2\2\u0110\67"+
+    "\3\2\2\2\u0111\u0112\7\61\2\2\u01129\3\2\2\2\u0113\u0114\7\'\2\2\u0114"+
+    ";\3\2\2\2\u0115\u0116\7-\2\2\u0116=\3\2\2\2\u0117\u0118\7/\2\2\u0118?"+
+    "\3\2\2\2\u0119\u011a\7>\2\2\u011a\u011b\7>\2\2\u011bA\3\2\2\2\u011c\u011d"+
+    "\7@\2\2\u011d\u011e\7@\2\2\u011eC\3\2\2\2\u011f\u0120\7@\2\2\u0120\u0121"+
+    "\7@\2\2\u0121\u0122\7@\2\2\u0122E\3\2\2\2\u0123\u0124\7>\2\2\u0124G\3"+
+    "\2\2\2\u0125\u0126\7>\2\2\u0126\u0127\7?\2\2\u0127I\3\2\2\2\u0128\u0129"+
+    "\7@\2\2\u0129K\3\2\2\2\u012a\u012b\7@\2\2\u012b\u012c\7?\2\2\u012cM\3"+
+    "\2\2\2\u012d\u012e\7?\2\2\u012e\u012f\7?\2\2\u012fO\3\2\2\2\u0130\u0131"+
+    "\7?\2\2\u0131\u0132\7?\2\2\u0132\u0133\7?\2\2\u0133Q\3\2\2\2\u0134\u0135"+
+    "\7#\2\2\u0135\u0136\7?\2\2\u0136S\3\2\2\2\u0137\u0138\7#\2\2\u0138\u0139"+
+    "\7?\2\2\u0139\u013a\7?\2\2\u013aU\3\2\2\2\u013b\u013c\7(\2\2\u013cW\3"+
+    "\2\2\2\u013d\u013e\7`\2\2\u013eY\3\2\2\2\u013f\u0140\7~\2\2\u0140[\3\2"+
+    "\2\2\u0141\u0142\7(\2\2\u0142\u0143\7(\2\2\u0143]\3\2\2\2\u0144\u0145"+
+    "\7~\2\2\u0145\u0146\7~\2\2\u0146_\3\2\2\2\u0147\u0148\7A\2\2\u0148a\3"+
+    "\2\2\2\u0149\u014a\7<\2\2\u014ac\3\2\2\2\u014b\u014c\7-\2\2\u014c\u014d"+
+    "\7-\2\2\u014de\3\2\2\2\u014e\u014f\7/\2\2\u014f\u0150\7/\2\2\u0150g\3"+
+    "\2\2\2\u0151\u0152\7?\2\2\u0152i\3\2\2\2\u0153\u0154\7-\2\2\u0154\u0155"+
+    "\7?\2\2\u0155k\3\2\2\2\u0156\u0157\7/\2\2\u0157\u0158\7?\2\2\u0158m\3"+
+    "\2\2\2\u0159\u015a\7,\2\2\u015a\u015b\7?\2\2\u015bo\3\2\2\2\u015c\u015d"+
+    "\7\61\2\2\u015d\u015e\7?\2\2\u015eq\3\2\2\2\u015f\u0160\7\'\2\2\u0160"+
+    "\u0161\7?\2\2\u0161s\3\2\2\2\u0162\u0163\7(\2\2\u0163\u0164\7?\2\2\u0164"+
+    "u\3\2\2\2\u0165\u0166\7`\2\2\u0166\u0167\7?\2\2\u0167w\3\2\2\2\u0168\u0169"+
+    "\7~\2\2\u0169\u016a\7?\2\2\u016ay\3\2\2\2\u016b\u016c\7>\2\2\u016c\u016d"+
+    "\7>\2\2\u016d\u016e\7?\2\2\u016e{\3\2\2\2\u016f\u0170\7@\2\2\u0170\u0171"+
+    "\7@\2\2\u0171\u0172\7?\2\2\u0172}\3\2\2\2\u0173\u0174\7@\2\2\u0174\u0175"+
+    "\7@\2\2\u0175\u0176\7@\2\2\u0176\u0177\7?\2\2\u0177\177\3\2\2\2\u0178"+
+    "\u017a\7\62\2\2\u0179\u017b\t\4\2\2\u017a\u0179\3\2\2\2\u017b\u017c\3"+
+    "\2\2\2\u017c\u017a\3\2\2\2\u017c\u017d\3\2\2\2\u017d\u017f\3\2\2\2\u017e"+
+    "\u0180\t\5\2\2\u017f\u017e\3\2\2\2\u017f\u0180\3\2\2\2\u0180\u0081\3\2"+
+    "\2\2\u0181\u0182\7\62\2\2\u0182\u0184\t\6\2\2\u0183\u0185\t\7\2\2\u0184"+
+    "\u0183\3\2\2\2\u0185\u0186\3\2\2\2\u0186\u0184\3\2\2\2\u0186\u0187\3\2"+
+    "\2\2\u0187\u0189\3\2\2\2\u0188\u018a\t\5\2\2\u0189\u0188\3\2\2\2\u0189"+
+    "\u018a\3\2\2\2\u018a\u0083\3\2\2\2\u018b\u0194\7\62\2\2\u018c\u0190\t"+
+    "\b\2\2\u018d\u018f\t\t\2\2\u018e\u018d\3\2\2\2\u018f\u0192\3\2\2\2\u0190"+
+    "\u018e\3\2\2\2\u0190\u0191\3\2\2\2\u0191\u0194\3\2\2\2\u0192\u0190\3\2"+
+    "\2\2\u0193\u018b\3\2\2\2\u0193\u018c\3\2\2\2\u0194\u0196\3\2\2\2\u0195"+
+    "\u0197\t\n\2\2\u0196\u0195\3\2\2\2\u0196\u0197\3\2\2\2\u0197\u0085\3\2"+
+    "\2\2\u0198\u01a1\7\62\2\2\u0199\u019d\t\b\2\2\u019a\u019c\t\t\2\2\u019b"+
+    "\u019a\3\2\2\2\u019c\u019f\3\2\2\2\u019d\u019b\3\2\2\2\u019d\u019e\3\2"+
+    "\2\2\u019e\u01a1\3\2\2\2\u019f\u019d\3\2\2\2\u01a0\u0198\3\2\2\2\u01a0"+
+    "\u0199\3\2\2\2\u01a1\u01a8\3\2\2\2\u01a2\u01a4\5\24\n\2\u01a3\u01a5\t"+
+    "\t\2\2\u01a4\u01a3\3\2\2\2\u01a5\u01a6\3\2\2\2\u01a6\u01a4\3\2\2\2\u01a6"+
+    "\u01a7\3\2\2\2\u01a7\u01a9\3\2\2\2\u01a8\u01a2\3\2\2\2\u01a8\u01a9\3\2"+
+    "\2\2\u01a9\u01b3\3\2\2\2\u01aa\u01ac\t\13\2\2\u01ab\u01ad\t\f\2\2\u01ac"+
+    "\u01ab\3\2\2\2\u01ac\u01ad\3\2\2\2\u01ad\u01af\3\2\2\2\u01ae\u01b0\t\t"+
+    "\2\2\u01af\u01ae\3\2\2\2\u01b0\u01b1\3\2\2\2\u01b1\u01af\3\2\2\2\u01b1"+
+    "\u01b2\3\2\2\2\u01b2\u01b4\3\2\2\2\u01b3\u01aa\3\2\2\2\u01b3\u01b4\3\2"+
+    "\2\2\u01b4\u01b6\3\2\2\2\u01b5\u01b7\t\r\2\2\u01b6\u01b5\3\2\2\2\u01b6"+
+    "\u01b7\3\2\2\2\u01b7\u0087\3\2\2\2\u01b8\u01c0\7$\2\2\u01b9\u01ba\7^\2"+
+    "\2\u01ba\u01bf\7$\2\2\u01bb\u01bc\7^\2\2\u01bc\u01bf\7^\2\2\u01bd\u01bf"+
+    "\n\16\2\2\u01be\u01b9\3\2\2\2\u01be\u01bb\3\2\2\2\u01be\u01bd\3\2\2\2"+
+    "\u01bf\u01c2\3\2\2\2\u01c0\u01c1\3\2\2\2\u01c0\u01be\3\2\2\2\u01c1\u01c3"+
+    "\3\2\2\2\u01c2\u01c0\3\2\2\2\u01c3\u01d1\7$\2\2\u01c4\u01cc\7)\2\2\u01c5"+
+    "\u01c6\7^\2\2\u01c6\u01cb\7)\2\2\u01c7\u01c8\7^\2\2\u01c8\u01cb\7^\2\2"+
+    "\u01c9\u01cb\n\16\2\2\u01ca\u01c5\3\2\2\2\u01ca\u01c7\3\2\2\2\u01ca\u01c9"+
+    "\3\2\2\2\u01cb\u01ce\3\2\2\2\u01cc\u01cd\3\2\2\2\u01cc\u01ca\3\2\2\2\u01cd"+
+    "\u01cf\3\2\2\2\u01ce\u01cc\3\2\2\2\u01cf\u01d1\7)\2\2\u01d0\u01b8\3\2"+
+    "\2\2\u01d0\u01c4\3\2\2\2\u01d1\u0089\3\2\2\2\u01d2\u01d3\7v\2\2\u01d3"+
+    "\u01d4\7t\2\2\u01d4\u01d5\7w\2\2\u01d5\u01d6\7g\2\2\u01d6\u008b\3\2\2"+
+    "\2\u01d7\u01d8\7h\2\2\u01d8\u01d9\7c\2\2\u01d9\u01da\7n\2\2\u01da\u01db"+
+    "\7u\2\2\u01db\u01dc\7g\2\2\u01dc\u008d\3\2\2\2\u01dd\u01de\7p\2\2\u01de"+
+    "\u01df\7w\2\2\u01df\u01e0\7n\2\2\u01e0\u01e1\7n\2\2\u01e1\u008f\3\2\2"+
+    "\2\u01e2\u01e8\5\u0092I\2\u01e3\u01e4\5\24\n\2\u01e4\u01e5\5\u0092I\2"+
+    "\u01e5\u01e7\3\2\2\2\u01e6\u01e3\3\2\2\2\u01e7\u01ea\3\2\2\2\u01e8\u01e6"+
+    "\3\2\2\2\u01e8\u01e9\3\2\2\2\u01e9\u01eb\3\2\2\2\u01ea\u01e8\3\2\2\2\u01eb"+
+    "\u01ec\6H\2\2\u01ec\u0091\3\2\2\2\u01ed\u01f1\t\17\2\2\u01ee\u01f0\t\20"+
+    "\2\2\u01ef\u01ee\3\2\2\2\u01f0\u01f3\3\2\2\2\u01f1\u01ef\3\2\2\2\u01f1"+
+    "\u01f2\3\2\2\2\u01f2\u0093\3\2\2\2\u01f3\u01f1\3\2\2\2\u01f4\u01fd\7\62"+
+    "\2\2\u01f5\u01f9\t\b\2\2\u01f6\u01f8\t\t\2\2\u01f7\u01f6\3\2\2\2\u01f8"+
+    "\u01fb\3\2\2\2\u01f9\u01f7\3\2\2\2\u01f9\u01fa\3\2\2\2\u01fa\u01fd\3\2"+
+    "\2\2\u01fb\u01f9\3\2\2\2\u01fc\u01f4\3\2\2\2\u01fc\u01f5\3\2\2\2\u01fd"+
+    "\u01fe\3\2\2\2\u01fe\u01ff\bJ\4\2\u01ff\u0095\3\2\2\2\u0200\u0204\t\21"+
+    "\2\2\u0201\u0203\t\20\2\2\u0202\u0201\3\2\2\2\u0203\u0206\3\2\2\2\u0204"+
+    "\u0202\3\2\2\2\u0204\u0205\3\2\2\2\u0205\u0207\3\2\2\2\u0206\u0204\3\2"+
+    "\2\2\u0207\u0208\bK\4\2\u0208\u0097\3\2\2\2!\2\3\u009b\u00a5\u00af\u00b4"+
+    "\u017c\u017f\u0186\u0189\u0190\u0193\u0196\u019d\u01a0\u01a6\u01a8\u01ac"+
+    "\u01b1\u01b3\u01b6\u01be\u01c0\u01ca\u01cc\u01d0\u01e8\u01f1\u01f9\u01fc"+
+    "\u0204\5\b\2\2\4\3\2\4\2\2";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
index c8e76ee4bd9..11b38dc9742 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
@@ -25,20 +25,17 @@ class PainlessParser extends Parser {
     BWOR=44, BOOLAND=45, BOOLOR=46, COND=47, COLON=48, INCR=49, DECR=50, ASSIGN=51, 
     AADD=52, ASUB=53, AMUL=54, ADIV=55, AREM=56, AAND=57, AXOR=58, AOR=59, 
     ALSH=60, ARSH=61, AUSH=62, OCTAL=63, HEX=64, INTEGER=65, DECIMAL=66, STRING=67, 
-    TRUE=68, FALSE=69, NULL=70, ID=71, EXTINTEGER=72, EXTID=73;
+    TRUE=68, FALSE=69, NULL=70, TYPE=71, ID=72, DOTINTEGER=73, DOTID=74;
   public static final int
-    RULE_source = 0, RULE_statement = 1, RULE_block = 2, RULE_empty = 3, RULE_emptyscope = 4, 
-    RULE_initializer = 5, RULE_afterthought = 6, RULE_declaration = 7, RULE_decltype = 8, 
-    RULE_declvar = 9, RULE_trap = 10, RULE_identifier = 11, RULE_generic = 12, 
-    RULE_expression = 13, RULE_chain = 14, RULE_linkprec = 15, RULE_linkcast = 16, 
-    RULE_linkbrace = 17, RULE_linkdot = 18, RULE_linkcall = 19, RULE_linkvar = 20, 
-    RULE_linkfield = 21, RULE_linknew = 22, RULE_linkstring = 23, RULE_arguments = 24;
+    RULE_source = 0, RULE_statement = 1, RULE_trailer = 2, RULE_block = 3, 
+    RULE_empty = 4, RULE_initializer = 5, RULE_afterthought = 6, RULE_declaration = 7, 
+    RULE_decltype = 8, RULE_declvar = 9, RULE_trap = 10, RULE_delimiter = 11, 
+    RULE_expression = 12, RULE_unary = 13, RULE_chain = 14, RULE_primary = 15, 
+    RULE_secondary = 16, RULE_dot = 17, RULE_brace = 18, RULE_arguments = 19;
   public static final String[] ruleNames = {
-    "source", "statement", "block", "empty", "emptyscope", "initializer", 
-    "afterthought", "declaration", "decltype", "declvar", "trap", "identifier", 
-    "generic", "expression", "chain", "linkprec", "linkcast", "linkbrace", 
-    "linkdot", "linkcall", "linkvar", "linkfield", "linknew", "linkstring", 
-    "arguments"
+    "source", "statement", "trailer", "block", "empty", "initializer", "afterthought", 
+    "declaration", "decltype", "declvar", "trap", "delimiter", "expression", 
+    "unary", "chain", "primary", "secondary", "dot", "brace", "arguments"
   };
 
   private static final String[] _LITERAL_NAMES = {
@@ -59,8 +56,8 @@ class PainlessParser extends Parser {
     "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", "BOOLAND", "BOOLOR", 
     "COND", "COLON", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", 
     "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", 
-    "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "ID", "EXTINTEGER", 
-    "EXTID"
+    "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "TYPE", "ID", 
+    "DOTINTEGER", "DOTID"
   };
   public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
 
@@ -133,25 +130,27 @@ class PainlessParser extends Parser {
   public final SourceContext source() throws RecognitionException {
     SourceContext _localctx = new SourceContext(_ctx, getState());
     enterRule(_localctx, 0, RULE_source);
-    int _la;
     try {
+      int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(51); 
+      setState(43);
       _errHandler.sync(this);
-      _la = _input.LA(1);
-      do {
-        {
-        {
-        setState(50);
-        statement();
+      _alt = getInterpreter().adaptivePredict(_input,0,_ctx);
+      while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
+        if ( _alt==1 ) {
+          {
+          {
+          setState(40);
+          statement();
+          }
+          } 
         }
-        }
-        setState(53); 
+        setState(45);
         _errHandler.sync(this);
-        _la = _input.LA(1);
-      } while ( (((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << IF) | (1L << WHILE) | (1L << DO) | (1L << FOR) | (1L << CONTINUE) | (1L << BREAK) | (1L << RETURN) | (1L << NEW) | (1L << TRY) | (1L << THROW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0) );
-      setState(55);
+        _alt = getInterpreter().adaptivePredict(_input,0,_ctx);
+      }
+      setState(46);
       match(EOF);
       }
     }
@@ -181,8 +180,9 @@ class PainlessParser extends Parser {
     public DeclarationContext declaration() {
       return getRuleContext(DeclarationContext.class,0);
     }
-    public TerminalNode SEMICOLON() { return getToken(PainlessParser.SEMICOLON, 0); }
-    public TerminalNode EOF() { return getToken(PainlessParser.EOF, 0); }
+    public DelimiterContext delimiter() {
+      return getRuleContext(DelimiterContext.class,0);
+    }
     public DeclContext(StatementContext ctx) { copyFrom(ctx); }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
@@ -192,8 +192,9 @@ class PainlessParser extends Parser {
   }
   public static class BreakContext extends StatementContext {
     public TerminalNode BREAK() { return getToken(PainlessParser.BREAK, 0); }
-    public TerminalNode SEMICOLON() { return getToken(PainlessParser.SEMICOLON, 0); }
-    public TerminalNode EOF() { return getToken(PainlessParser.EOF, 0); }
+    public DelimiterContext delimiter() {
+      return getRuleContext(DelimiterContext.class,0);
+    }
     public BreakContext(StatementContext ctx) { copyFrom(ctx); }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
@@ -206,8 +207,9 @@ class PainlessParser extends Parser {
     public ExpressionContext expression() {
       return getRuleContext(ExpressionContext.class,0);
     }
-    public TerminalNode SEMICOLON() { return getToken(PainlessParser.SEMICOLON, 0); }
-    public TerminalNode EOF() { return getToken(PainlessParser.EOF, 0); }
+    public DelimiterContext delimiter() {
+      return getRuleContext(DelimiterContext.class,0);
+    }
     public ThrowContext(StatementContext ctx) { copyFrom(ctx); }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
@@ -217,8 +219,9 @@ class PainlessParser extends Parser {
   }
   public static class ContinueContext extends StatementContext {
     public TerminalNode CONTINUE() { return getToken(PainlessParser.CONTINUE, 0); }
-    public TerminalNode SEMICOLON() { return getToken(PainlessParser.SEMICOLON, 0); }
-    public TerminalNode EOF() { return getToken(PainlessParser.EOF, 0); }
+    public DelimiterContext delimiter() {
+      return getRuleContext(DelimiterContext.class,0);
+    }
     public ContinueContext(StatementContext ctx) { copyFrom(ctx); }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
@@ -234,8 +237,8 @@ class PainlessParser extends Parser {
       return getToken(PainlessParser.SEMICOLON, i);
     }
     public TerminalNode RP() { return getToken(PainlessParser.RP, 0); }
-    public BlockContext block() {
-      return getRuleContext(BlockContext.class,0);
+    public TrailerContext trailer() {
+      return getRuleContext(TrailerContext.class,0);
     }
     public EmptyContext empty() {
       return getRuleContext(EmptyContext.class,0);
@@ -278,8 +281,9 @@ class PainlessParser extends Parser {
     public ExpressionContext expression() {
       return getRuleContext(ExpressionContext.class,0);
     }
-    public TerminalNode SEMICOLON() { return getToken(PainlessParser.SEMICOLON, 0); }
-    public TerminalNode EOF() { return getToken(PainlessParser.EOF, 0); }
+    public DelimiterContext delimiter() {
+      return getRuleContext(DelimiterContext.class,0);
+    }
     public ExprContext(StatementContext ctx) { copyFrom(ctx); }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
@@ -298,8 +302,9 @@ class PainlessParser extends Parser {
       return getRuleContext(ExpressionContext.class,0);
     }
     public TerminalNode RP() { return getToken(PainlessParser.RP, 0); }
-    public TerminalNode SEMICOLON() { return getToken(PainlessParser.SEMICOLON, 0); }
-    public TerminalNode EOF() { return getToken(PainlessParser.EOF, 0); }
+    public DelimiterContext delimiter() {
+      return getRuleContext(DelimiterContext.class,0);
+    }
     public DoContext(StatementContext ctx) { copyFrom(ctx); }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
@@ -314,8 +319,8 @@ class PainlessParser extends Parser {
       return getRuleContext(ExpressionContext.class,0);
     }
     public TerminalNode RP() { return getToken(PainlessParser.RP, 0); }
-    public BlockContext block() {
-      return getRuleContext(BlockContext.class,0);
+    public TrailerContext trailer() {
+      return getRuleContext(TrailerContext.class,0);
     }
     public EmptyContext empty() {
       return getRuleContext(EmptyContext.class,0);
@@ -334,11 +339,11 @@ class PainlessParser extends Parser {
       return getRuleContext(ExpressionContext.class,0);
     }
     public TerminalNode RP() { return getToken(PainlessParser.RP, 0); }
-    public List<BlockContext> block() {
-      return getRuleContexts(BlockContext.class);
+    public List<TrailerContext> trailer() {
+      return getRuleContexts(TrailerContext.class);
     }
-    public BlockContext block(int i) {
-      return getRuleContext(BlockContext.class,i);
+    public TrailerContext trailer(int i) {
+      return getRuleContext(TrailerContext.class,i);
     }
     public TerminalNode ELSE() { return getToken(PainlessParser.ELSE, 0); }
     public IfContext(StatementContext ctx) { copyFrom(ctx); }
@@ -353,8 +358,9 @@ class PainlessParser extends Parser {
     public ExpressionContext expression() {
       return getRuleContext(ExpressionContext.class,0);
     }
-    public TerminalNode SEMICOLON() { return getToken(PainlessParser.SEMICOLON, 0); }
-    public TerminalNode EOF() { return getToken(PainlessParser.EOF, 0); }
+    public DelimiterContext delimiter() {
+      return getRuleContext(DelimiterContext.class,0);
+    }
     public ReturnContext(StatementContext ctx) { copyFrom(ctx); }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
@@ -366,33 +372,38 @@ class PainlessParser extends Parser {
   public final StatementContext statement() throws RecognitionException {
     StatementContext _localctx = new StatementContext(_ctx, getState());
     enterRule(_localctx, 2, RULE_statement);
-    int _la;
     try {
       int _alt;
-      setState(125);
+      setState(117);
       switch ( getInterpreter().adaptivePredict(_input,8,_ctx) ) {
       case 1:
         _localctx = new IfContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(57);
+        setState(48);
         match(IF);
-        setState(58);
+        setState(49);
         match(LP);
-        setState(59);
+        setState(50);
         expression(0);
-        setState(60);
+        setState(51);
         match(RP);
-        setState(61);
-        block();
-        setState(64);
+        setState(52);
+        trailer();
+        setState(56);
         switch ( getInterpreter().adaptivePredict(_input,1,_ctx) ) {
         case 1:
           {
-          setState(62);
+          setState(53);
           match(ELSE);
-          setState(63);
-          block();
+          setState(54);
+          trailer();
+          }
+          break;
+        case 2:
+          {
+          setState(55);
+          if (!( _input.LA(1) != ELSE )) throw new FailedPredicateException(this, " _input.LA(1) != ELSE ");
           }
           break;
         }
@@ -402,25 +413,25 @@ class PainlessParser extends Parser {
         _localctx = new WhileContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(66);
+        setState(58);
         match(WHILE);
-        setState(67);
+        setState(59);
         match(LP);
-        setState(68);
+        setState(60);
         expression(0);
-        setState(69);
+        setState(61);
         match(RP);
-        setState(72);
+        setState(64);
         switch ( getInterpreter().adaptivePredict(_input,2,_ctx) ) {
         case 1:
           {
-          setState(70);
-          block();
+          setState(62);
+          trailer();
           }
           break;
         case 2:
           {
-          setState(71);
+          setState(63);
           empty();
           }
           break;
@@ -431,79 +442,74 @@ class PainlessParser extends Parser {
         _localctx = new DoContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(74);
+        setState(66);
         match(DO);
-        setState(75);
+        setState(67);
         block();
-        setState(76);
+        setState(68);
         match(WHILE);
-        setState(77);
+        setState(69);
         match(LP);
-        setState(78);
+        setState(70);
         expression(0);
-        setState(79);
+        setState(71);
         match(RP);
-        setState(80);
-        _la = _input.LA(1);
-        if ( !(_la==EOF || _la==SEMICOLON) ) {
-        _errHandler.recoverInline(this);
-        } else {
-          consume();
-        }
+        setState(72);
+        delimiter();
         }
         break;
       case 4:
         _localctx = new ForContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(82);
+        setState(74);
         match(FOR);
-        setState(83);
+        setState(75);
         match(LP);
-        setState(85);
-        _la = _input.LA(1);
-        if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0)) {
+        setState(77);
+        switch ( getInterpreter().adaptivePredict(_input,3,_ctx) ) {
+        case 1:
           {
-          setState(84);
+          setState(76);
           initializer();
           }
+          break;
         }
-
-        setState(87);
+        setState(79);
         match(SEMICOLON);
-        setState(89);
-        _la = _input.LA(1);
-        if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0)) {
+        setState(81);
+        switch ( getInterpreter().adaptivePredict(_input,4,_ctx) ) {
+        case 1:
           {
-          setState(88);
+          setState(80);
           expression(0);
           }
+          break;
         }
-
-        setState(91);
+        setState(83);
         match(SEMICOLON);
-        setState(93);
-        _la = _input.LA(1);
-        if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0)) {
+        setState(85);
+        switch ( getInterpreter().adaptivePredict(_input,5,_ctx) ) {
+        case 1:
           {
-          setState(92);
+          setState(84);
           afterthought();
           }
+          break;
         }
-
-        setState(95);
+        setState(87);
         match(RP);
-        setState(98);
+        setState(90);
         switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) {
         case 1:
           {
-          setState(96);
-          block();
+          setState(88);
+          trailer();
           }
           break;
         case 2:
           {
-          setState(97);
+          setState(89);
           empty();
           }
           break;
@@ -514,73 +520,53 @@ class PainlessParser extends Parser {
         _localctx = new DeclContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(100);
+        setState(92);
         declaration();
-        setState(101);
-        _la = _input.LA(1);
-        if ( !(_la==EOF || _la==SEMICOLON) ) {
-        _errHandler.recoverInline(this);
-        } else {
-          consume();
-        }
+        setState(93);
+        delimiter();
         }
         break;
       case 6:
         _localctx = new ContinueContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(103);
+        setState(95);
         match(CONTINUE);
-        setState(104);
-        _la = _input.LA(1);
-        if ( !(_la==EOF || _la==SEMICOLON) ) {
-        _errHandler.recoverInline(this);
-        } else {
-          consume();
-        }
+        setState(96);
+        delimiter();
         }
         break;
       case 7:
         _localctx = new BreakContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(105);
+        setState(97);
         match(BREAK);
-        setState(106);
-        _la = _input.LA(1);
-        if ( !(_la==EOF || _la==SEMICOLON) ) {
-        _errHandler.recoverInline(this);
-        } else {
-          consume();
-        }
+        setState(98);
+        delimiter();
         }
         break;
       case 8:
         _localctx = new ReturnContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(107);
+        setState(99);
         match(RETURN);
-        setState(108);
+        setState(100);
         expression(0);
-        setState(109);
-        _la = _input.LA(1);
-        if ( !(_la==EOF || _la==SEMICOLON) ) {
-        _errHandler.recoverInline(this);
-        } else {
-          consume();
-        }
+        setState(101);
+        delimiter();
         }
         break;
       case 9:
         _localctx = new TryContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(111);
+        setState(103);
         match(TRY);
-        setState(112);
+        setState(104);
         block();
-        setState(114); 
+        setState(106); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -588,7 +574,7 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(113);
+            setState(105);
             trap();
             }
             }
@@ -596,7 +582,7 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(116); 
+          setState(108); 
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,7,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
@@ -606,32 +592,73 @@ class PainlessParser extends Parser {
         _localctx = new ThrowContext(_localctx);
         enterOuterAlt(_localctx, 10);
         {
-        setState(118);
+        setState(110);
         match(THROW);
-        setState(119);
+        setState(111);
         expression(0);
-        setState(120);
-        _la = _input.LA(1);
-        if ( !(_la==EOF || _la==SEMICOLON) ) {
-        _errHandler.recoverInline(this);
-        } else {
-          consume();
-        }
+        setState(112);
+        delimiter();
         }
         break;
       case 11:
         _localctx = new ExprContext(_localctx);
         enterOuterAlt(_localctx, 11);
         {
-        setState(122);
+        setState(114);
         expression(0);
-        setState(123);
-        _la = _input.LA(1);
-        if ( !(_la==EOF || _la==SEMICOLON) ) {
-        _errHandler.recoverInline(this);
-        } else {
-          consume();
+        setState(115);
+        delimiter();
         }
+        break;
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
+  public static class TrailerContext extends ParserRuleContext {
+    public BlockContext block() {
+      return getRuleContext(BlockContext.class,0);
+    }
+    public StatementContext statement() {
+      return getRuleContext(StatementContext.class,0);
+    }
+    public TrailerContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_trailer; }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitTrailer(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final TrailerContext trailer() throws RecognitionException {
+    TrailerContext _localctx = new TrailerContext(_ctx, getState());
+    enterRule(_localctx, 4, RULE_trailer);
+    try {
+      setState(121);
+      switch ( getInterpreter().adaptivePredict(_input,9,_ctx) ) {
+      case 1:
+        enterOuterAlt(_localctx, 1);
+        {
+        setState(119);
+        block();
+        }
+        break;
+      case 2:
+        enterOuterAlt(_localctx, 2);
+        {
+        setState(120);
+        statement();
         }
         break;
       }
@@ -648,28 +675,6 @@ class PainlessParser extends Parser {
   }
 
   public static class BlockContext extends ParserRuleContext {
-    public BlockContext(ParserRuleContext parent, int invokingState) {
-      super(parent, invokingState);
-    }
-    @Override public int getRuleIndex() { return RULE_block; }
-   
-    public BlockContext() { }
-    public void copyFrom(BlockContext ctx) {
-      super.copyFrom(ctx);
-    }
-  }
-  public static class SingleContext extends BlockContext {
-    public StatementContext statement() {
-      return getRuleContext(StatementContext.class,0);
-    }
-    public SingleContext(BlockContext ctx) { copyFrom(ctx); }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitSingle(this);
-      else return visitor.visitChildren(this);
-    }
-  }
-  public static class MultipleContext extends BlockContext {
     public TerminalNode LBRACK() { return getToken(PainlessParser.LBRACK, 0); }
     public TerminalNode RBRACK() { return getToken(PainlessParser.RBRACK, 0); }
     public List<StatementContext> statement() {
@@ -678,80 +683,44 @@ class PainlessParser extends Parser {
     public StatementContext statement(int i) {
       return getRuleContext(StatementContext.class,i);
     }
-    public MultipleContext(BlockContext ctx) { copyFrom(ctx); }
+    public BlockContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_block; }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitMultiple(this);
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitBlock(this);
       else return visitor.visitChildren(this);
     }
   }
 
   public final BlockContext block() throws RecognitionException {
     BlockContext _localctx = new BlockContext(_ctx, getState());
-    enterRule(_localctx, 4, RULE_block);
-    int _la;
+    enterRule(_localctx, 6, RULE_block);
     try {
-      setState(136);
-      switch (_input.LA(1)) {
-      case LBRACK:
-        _localctx = new MultipleContext(_localctx);
-        enterOuterAlt(_localctx, 1);
-        {
-        setState(127);
-        match(LBRACK);
-        setState(129); 
-        _errHandler.sync(this);
-        _la = _input.LA(1);
-        do {
+      int _alt;
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(123);
+      match(LBRACK);
+      setState(127);
+      _errHandler.sync(this);
+      _alt = getInterpreter().adaptivePredict(_input,10,_ctx);
+      while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
+        if ( _alt==1 ) {
           {
           {
-          setState(128);
+          setState(124);
           statement();
           }
-          }
-          setState(131); 
-          _errHandler.sync(this);
-          _la = _input.LA(1);
-        } while ( (((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << IF) | (1L << WHILE) | (1L << DO) | (1L << FOR) | (1L << CONTINUE) | (1L << BREAK) | (1L << RETURN) | (1L << NEW) | (1L << TRY) | (1L << THROW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0) );
-        setState(133);
-        match(RBRACK);
+          } 
         }
-        break;
-      case LP:
-      case IF:
-      case WHILE:
-      case DO:
-      case FOR:
-      case CONTINUE:
-      case BREAK:
-      case RETURN:
-      case NEW:
-      case TRY:
-      case THROW:
-      case BOOLNOT:
-      case BWNOT:
-      case ADD:
-      case SUB:
-      case INCR:
-      case DECR:
-      case OCTAL:
-      case HEX:
-      case INTEGER:
-      case DECIMAL:
-      case STRING:
-      case TRUE:
-      case FALSE:
-      case NULL:
-      case ID:
-        _localctx = new SingleContext(_localctx);
-        enterOuterAlt(_localctx, 2);
-        {
-        setState(135);
-        statement();
-        }
-        break;
-      default:
-        throw new NoViableAltException(this);
+        setState(129);
+        _errHandler.sync(this);
+        _alt = getInterpreter().adaptivePredict(_input,10,_ctx);
+      }
+      setState(130);
+      match(RBRACK);
       }
     }
     catch (RecognitionException re) {
@@ -766,9 +735,6 @@ class PainlessParser extends Parser {
   }
 
   public static class EmptyContext extends ParserRuleContext {
-    public EmptyscopeContext emptyscope() {
-      return getRuleContext(EmptyscopeContext.class,0);
-    }
     public TerminalNode SEMICOLON() { return getToken(PainlessParser.SEMICOLON, 0); }
     public EmptyContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
@@ -783,63 +749,12 @@ class PainlessParser extends Parser {
 
   public final EmptyContext empty() throws RecognitionException {
     EmptyContext _localctx = new EmptyContext(_ctx, getState());
-    enterRule(_localctx, 6, RULE_empty);
-    try {
-      setState(140);
-      switch (_input.LA(1)) {
-      case LBRACK:
-        enterOuterAlt(_localctx, 1);
-        {
-        setState(138);
-        emptyscope();
-        }
-        break;
-      case SEMICOLON:
-        enterOuterAlt(_localctx, 2);
-        {
-        setState(139);
-        match(SEMICOLON);
-        }
-        break;
-      default:
-        throw new NoViableAltException(this);
-      }
-    }
-    catch (RecognitionException re) {
-      _localctx.exception = re;
-      _errHandler.reportError(this, re);
-      _errHandler.recover(this, re);
-    }
-    finally {
-      exitRule();
-    }
-    return _localctx;
-  }
-
-  public static class EmptyscopeContext extends ParserRuleContext {
-    public TerminalNode LBRACK() { return getToken(PainlessParser.LBRACK, 0); }
-    public TerminalNode RBRACK() { return getToken(PainlessParser.RBRACK, 0); }
-    public EmptyscopeContext(ParserRuleContext parent, int invokingState) {
-      super(parent, invokingState);
-    }
-    @Override public int getRuleIndex() { return RULE_emptyscope; }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitEmptyscope(this);
-      else return visitor.visitChildren(this);
-    }
-  }
-
-  public final EmptyscopeContext emptyscope() throws RecognitionException {
-    EmptyscopeContext _localctx = new EmptyscopeContext(_ctx, getState());
-    enterRule(_localctx, 8, RULE_emptyscope);
+    enterRule(_localctx, 8, RULE_empty);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(142);
-      match(LBRACK);
-      setState(143);
-      match(RBRACK);
+      setState(132);
+      match(SEMICOLON);
       }
     }
     catch (RecognitionException re) {
@@ -875,19 +790,19 @@ class PainlessParser extends Parser {
     InitializerContext _localctx = new InitializerContext(_ctx, getState());
     enterRule(_localctx, 10, RULE_initializer);
     try {
-      setState(147);
-      switch ( getInterpreter().adaptivePredict(_input,12,_ctx) ) {
+      setState(136);
+      switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(145);
+        setState(134);
         declaration();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(146);
+        setState(135);
         expression(0);
         }
         break;
@@ -925,7 +840,7 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(149);
+      setState(138);
       expression(0);
       }
     }
@@ -972,23 +887,23 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(151);
+      setState(140);
       decltype();
-      setState(152);
+      setState(141);
       declvar();
-      setState(157);
+      setState(146);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==COMMA) {
         {
         {
-        setState(153);
+        setState(142);
         match(COMMA);
-        setState(154);
+        setState(143);
         declvar();
         }
         }
-        setState(159);
+        setState(148);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
@@ -1006,9 +921,7 @@ class PainlessParser extends Parser {
   }
 
   public static class DecltypeContext extends ParserRuleContext {
-    public IdentifierContext identifier() {
-      return getRuleContext(IdentifierContext.class,0);
-    }
+    public TerminalNode TYPE() { return getToken(PainlessParser.TYPE, 0); }
     public List<TerminalNode> LBRACE() { return getTokens(PainlessParser.LBRACE); }
     public TerminalNode LBRACE(int i) {
       return getToken(PainlessParser.LBRACE, i);
@@ -1035,21 +948,21 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(160);
-      identifier();
-      setState(165);
+      setState(149);
+      match(TYPE);
+      setState(154);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==LBRACE) {
         {
         {
-        setState(161);
+        setState(150);
         match(LBRACE);
-        setState(162);
+        setState(151);
         match(RBRACE);
         }
         }
-        setState(167);
+        setState(156);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
@@ -1067,9 +980,7 @@ class PainlessParser extends Parser {
   }
 
   public static class DeclvarContext extends ParserRuleContext {
-    public IdentifierContext identifier() {
-      return getRuleContext(IdentifierContext.class,0);
-    }
+    public TerminalNode ID() { return getToken(PainlessParser.ID, 0); }
     public TerminalNode ASSIGN() { return getToken(PainlessParser.ASSIGN, 0); }
     public ExpressionContext expression() {
       return getRuleContext(ExpressionContext.class,0);
@@ -1092,15 +1003,15 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(168);
-      identifier();
-      setState(171);
+      setState(157);
+      match(ID);
+      setState(160);
       _la = _input.LA(1);
       if (_la==ASSIGN) {
         {
-        setState(169);
+        setState(158);
         match(ASSIGN);
-        setState(170);
+        setState(159);
         expression(0);
         }
       }
@@ -1121,19 +1032,12 @@ class PainlessParser extends Parser {
   public static class TrapContext extends ParserRuleContext {
     public TerminalNode CATCH() { return getToken(PainlessParser.CATCH, 0); }
     public TerminalNode LP() { return getToken(PainlessParser.LP, 0); }
+    public TerminalNode TYPE() { return getToken(PainlessParser.TYPE, 0); }
+    public TerminalNode ID() { return getToken(PainlessParser.ID, 0); }
     public TerminalNode RP() { return getToken(PainlessParser.RP, 0); }
-    public List<IdentifierContext> identifier() {
-      return getRuleContexts(IdentifierContext.class);
-    }
-    public IdentifierContext identifier(int i) {
-      return getRuleContext(IdentifierContext.class,i);
-    }
     public BlockContext block() {
       return getRuleContext(BlockContext.class,0);
     }
-    public EmptyscopeContext emptyscope() {
-      return getRuleContext(EmptyscopeContext.class,0);
-    }
     public TrapContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
     }
@@ -1151,79 +1055,18 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(173);
+      setState(162);
       match(CATCH);
-      setState(174);
+      setState(163);
       match(LP);
-      {
-      setState(175);
-      identifier();
-      setState(176);
-      identifier();
-      }
-      setState(178);
-      match(RP);
-      setState(181);
-      switch ( getInterpreter().adaptivePredict(_input,16,_ctx) ) {
-      case 1:
-        {
-        setState(179);
-        block();
-        }
-        break;
-      case 2:
-        {
-        setState(180);
-        emptyscope();
-        }
-        break;
-      }
-      }
-    }
-    catch (RecognitionException re) {
-      _localctx.exception = re;
-      _errHandler.reportError(this, re);
-      _errHandler.recover(this, re);
-    }
-    finally {
-      exitRule();
-    }
-    return _localctx;
-  }
-
-  public static class IdentifierContext extends ParserRuleContext {
-    public TerminalNode ID() { return getToken(PainlessParser.ID, 0); }
-    public GenericContext generic() {
-      return getRuleContext(GenericContext.class,0);
-    }
-    public IdentifierContext(ParserRuleContext parent, int invokingState) {
-      super(parent, invokingState);
-    }
-    @Override public int getRuleIndex() { return RULE_identifier; }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitIdentifier(this);
-      else return visitor.visitChildren(this);
-    }
-  }
-
-  public final IdentifierContext identifier() throws RecognitionException {
-    IdentifierContext _localctx = new IdentifierContext(_ctx, getState());
-    enterRule(_localctx, 22, RULE_identifier);
-    try {
-      enterOuterAlt(_localctx, 1);
-      {
-      setState(183);
+      setState(164);
+      match(TYPE);
+      setState(165);
       match(ID);
-      setState(185);
-      switch ( getInterpreter().adaptivePredict(_input,17,_ctx) ) {
-      case 1:
-        {
-        setState(184);
-        generic();
-        }
-        break;
-      }
+      setState(166);
+      match(RP);
+      setState(167);
+      block();
       }
     }
     catch (RecognitionException re) {
@@ -1237,59 +1080,34 @@ class PainlessParser extends Parser {
     return _localctx;
   }
 
-  public static class GenericContext extends ParserRuleContext {
-    public TerminalNode LT() { return getToken(PainlessParser.LT, 0); }
-    public List<IdentifierContext> identifier() {
-      return getRuleContexts(IdentifierContext.class);
-    }
-    public IdentifierContext identifier(int i) {
-      return getRuleContext(IdentifierContext.class,i);
-    }
-    public TerminalNode GT() { return getToken(PainlessParser.GT, 0); }
-    public List<TerminalNode> COMMA() { return getTokens(PainlessParser.COMMA); }
-    public TerminalNode COMMA(int i) {
-      return getToken(PainlessParser.COMMA, i);
-    }
-    public GenericContext(ParserRuleContext parent, int invokingState) {
+  public static class DelimiterContext extends ParserRuleContext {
+    public TerminalNode SEMICOLON() { return getToken(PainlessParser.SEMICOLON, 0); }
+    public TerminalNode EOF() { return getToken(PainlessParser.EOF, 0); }
+    public DelimiterContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
     }
-    @Override public int getRuleIndex() { return RULE_generic; }
+    @Override public int getRuleIndex() { return RULE_delimiter; }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitGeneric(this);
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitDelimiter(this);
       else return visitor.visitChildren(this);
     }
   }
 
-  public final GenericContext generic() throws RecognitionException {
-    GenericContext _localctx = new GenericContext(_ctx, getState());
-    enterRule(_localctx, 24, RULE_generic);
+  public final DelimiterContext delimiter() throws RecognitionException {
+    DelimiterContext _localctx = new DelimiterContext(_ctx, getState());
+    enterRule(_localctx, 22, RULE_delimiter);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(187);
-      match(LT);
-      setState(188);
-      identifier();
-      setState(193);
-      _errHandler.sync(this);
+      setState(169);
       _la = _input.LA(1);
-      while (_la==COMMA) {
-        {
-        {
-        setState(189);
-        match(COMMA);
-        setState(190);
-        identifier();
-        }
-        }
-        setState(195);
-        _errHandler.sync(this);
-        _la = _input.LA(1);
+      if ( !(_la==EOF || _la==SEMICOLON) ) {
+      _errHandler.recoverInline(this);
+      } else {
+        consume();
       }
-      setState(196);
-      match(GT);
       }
     }
     catch (RecognitionException re) {
@@ -1304,6 +1122,7 @@ class PainlessParser extends Parser {
   }
 
   public static class ExpressionContext extends ParserRuleContext {
+    public boolean s =  true;
     public ExpressionContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
     }
@@ -1312,6 +1131,19 @@ class PainlessParser extends Parser {
     public ExpressionContext() { }
     public void copyFrom(ExpressionContext ctx) {
       super.copyFrom(ctx);
+      this.s = ctx.s;
+    }
+  }
+  public static class SingleContext extends ExpressionContext {
+    public UnaryContext u;
+    public UnaryContext unary() {
+      return getRuleContext(UnaryContext.class,0);
+    }
+    public SingleContext(ExpressionContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitSingle(this);
+      else return visitor.visitChildren(this);
     }
   }
   public static class CompContext extends ExpressionContext {
@@ -1336,17 +1168,6 @@ class PainlessParser extends Parser {
       else return visitor.visitChildren(this);
     }
   }
-  public static class ReadContext extends ExpressionContext {
-    public ChainContext chain() {
-      return getRuleContext(ChainContext.class,0);
-    }
-    public ReadContext(ExpressionContext ctx) { copyFrom(ctx); }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitRead(this);
-      else return visitor.visitChildren(this);
-    }
-  }
   public static class BoolContext extends ExpressionContext {
     public List<ExpressionContext> expression() {
       return getRuleContexts(ExpressionContext.class);
@@ -1364,6 +1185,8 @@ class PainlessParser extends Parser {
     }
   }
   public static class ConditionalContext extends ExpressionContext {
+    public ExpressionContext e0;
+    public ExpressionContext e1;
     public List<ExpressionContext> expression() {
       return getRuleContexts(ExpressionContext.class);
     }
@@ -1405,106 +1228,6 @@ class PainlessParser extends Parser {
       else return visitor.visitChildren(this);
     }
   }
-  public static class FalseContext extends ExpressionContext {
-    public TerminalNode FALSE() { return getToken(PainlessParser.FALSE, 0); }
-    public FalseContext(ExpressionContext ctx) { copyFrom(ctx); }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitFalse(this);
-      else return visitor.visitChildren(this);
-    }
-  }
-  public static class NumericContext extends ExpressionContext {
-    public TerminalNode OCTAL() { return getToken(PainlessParser.OCTAL, 0); }
-    public TerminalNode HEX() { return getToken(PainlessParser.HEX, 0); }
-    public TerminalNode INTEGER() { return getToken(PainlessParser.INTEGER, 0); }
-    public TerminalNode DECIMAL() { return getToken(PainlessParser.DECIMAL, 0); }
-    public NumericContext(ExpressionContext ctx) { copyFrom(ctx); }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitNumeric(this);
-      else return visitor.visitChildren(this);
-    }
-  }
-  public static class UnaryContext extends ExpressionContext {
-    public ExpressionContext expression() {
-      return getRuleContext(ExpressionContext.class,0);
-    }
-    public TerminalNode BOOLNOT() { return getToken(PainlessParser.BOOLNOT, 0); }
-    public TerminalNode BWNOT() { return getToken(PainlessParser.BWNOT, 0); }
-    public TerminalNode ADD() { return getToken(PainlessParser.ADD, 0); }
-    public TerminalNode SUB() { return getToken(PainlessParser.SUB, 0); }
-    public UnaryContext(ExpressionContext ctx) { copyFrom(ctx); }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitUnary(this);
-      else return visitor.visitChildren(this);
-    }
-  }
-  public static class PrecedenceContext extends ExpressionContext {
-    public TerminalNode LP() { return getToken(PainlessParser.LP, 0); }
-    public ExpressionContext expression() {
-      return getRuleContext(ExpressionContext.class,0);
-    }
-    public TerminalNode RP() { return getToken(PainlessParser.RP, 0); }
-    public PrecedenceContext(ExpressionContext ctx) { copyFrom(ctx); }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitPrecedence(this);
-      else return visitor.visitChildren(this);
-    }
-  }
-  public static class PreincContext extends ExpressionContext {
-    public ChainContext chain() {
-      return getRuleContext(ChainContext.class,0);
-    }
-    public TerminalNode INCR() { return getToken(PainlessParser.INCR, 0); }
-    public TerminalNode DECR() { return getToken(PainlessParser.DECR, 0); }
-    public PreincContext(ExpressionContext ctx) { copyFrom(ctx); }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitPreinc(this);
-      else return visitor.visitChildren(this);
-    }
-  }
-  public static class PostincContext extends ExpressionContext {
-    public ChainContext chain() {
-      return getRuleContext(ChainContext.class,0);
-    }
-    public TerminalNode INCR() { return getToken(PainlessParser.INCR, 0); }
-    public TerminalNode DECR() { return getToken(PainlessParser.DECR, 0); }
-    public PostincContext(ExpressionContext ctx) { copyFrom(ctx); }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitPostinc(this);
-      else return visitor.visitChildren(this);
-    }
-  }
-  public static class CastContext extends ExpressionContext {
-    public TerminalNode LP() { return getToken(PainlessParser.LP, 0); }
-    public DecltypeContext decltype() {
-      return getRuleContext(DecltypeContext.class,0);
-    }
-    public TerminalNode RP() { return getToken(PainlessParser.RP, 0); }
-    public ExpressionContext expression() {
-      return getRuleContext(ExpressionContext.class,0);
-    }
-    public CastContext(ExpressionContext ctx) { copyFrom(ctx); }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitCast(this);
-      else return visitor.visitChildren(this);
-    }
-  }
-  public static class NullContext extends ExpressionContext {
-    public TerminalNode NULL() { return getToken(PainlessParser.NULL, 0); }
-    public NullContext(ExpressionContext ctx) { copyFrom(ctx); }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitNull(this);
-      else return visitor.visitChildren(this);
-    }
-  }
   public static class BinaryContext extends ExpressionContext {
     public List<ExpressionContext> expression() {
       return getRuleContexts(ExpressionContext.class);
@@ -1530,15 +1253,6 @@ class PainlessParser extends Parser {
       else return visitor.visitChildren(this);
     }
   }
-  public static class TrueContext extends ExpressionContext {
-    public TerminalNode TRUE() { return getToken(PainlessParser.TRUE, 0); }
-    public TrueContext(ExpressionContext ctx) { copyFrom(ctx); }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitTrue(this);
-      else return visitor.visitChildren(this);
-    }
-  }
 
   public final ExpressionContext expression() throws RecognitionException {
     return expression(0);
@@ -1549,339 +1263,235 @@ class PainlessParser extends Parser {
     int _parentState = getState();
     ExpressionContext _localctx = new ExpressionContext(_ctx, _parentState);
     ExpressionContext _prevctx = _localctx;
-    int _startState = 26;
-    enterRecursionRule(_localctx, 26, RULE_expression, _p);
+    int _startState = 24;
+    enterRecursionRule(_localctx, 24, RULE_expression, _p);
     int _la;
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(224);
-      switch ( getInterpreter().adaptivePredict(_input,19,_ctx) ) {
+      setState(180);
+      switch ( getInterpreter().adaptivePredict(_input,15,_ctx) ) {
       case 1:
-        {
-        _localctx = new UnaryContext(_localctx);
-        _ctx = _localctx;
-        _prevctx = _localctx;
-
-        setState(199);
-        _la = _input.LA(1);
-        if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB))) != 0)) ) {
-        _errHandler.recoverInline(this);
-        } else {
-          consume();
-        }
-        setState(200);
-        expression(14);
-        }
-        break;
-      case 2:
-        {
-        _localctx = new CastContext(_localctx);
-        _ctx = _localctx;
-        _prevctx = _localctx;
-        setState(201);
-        match(LP);
-        setState(202);
-        decltype();
-        setState(203);
-        match(RP);
-        setState(204);
-        expression(13);
-        }
-        break;
-      case 3:
         {
         _localctx = new AssignmentContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(206);
-        chain();
-        setState(207);
+
+        setState(172);
+        chain(true);
+        setState(173);
         _la = _input.LA(1);
         if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ASSIGN) | (1L << AADD) | (1L << ASUB) | (1L << AMUL) | (1L << ADIV) | (1L << AREM) | (1L << AAND) | (1L << AXOR) | (1L << AOR) | (1L << ALSH) | (1L << ARSH) | (1L << AUSH))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(208);
+        setState(174);
         expression(1);
+         ((AssignmentContext)_localctx).s =  false; 
         }
         break;
-      case 4:
+      case 2:
         {
-        _localctx = new PrecedenceContext(_localctx);
+        _localctx = new SingleContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(210);
-        match(LP);
-        setState(211);
-        expression(0);
-        setState(212);
-        match(RP);
-        }
-        break;
-      case 5:
-        {
-        _localctx = new NumericContext(_localctx);
-        _ctx = _localctx;
-        _prevctx = _localctx;
-        setState(214);
-        _la = _input.LA(1);
-        if ( !(((((_la - 63)) & ~0x3f) == 0 && ((1L << (_la - 63)) & ((1L << (OCTAL - 63)) | (1L << (HEX - 63)) | (1L << (INTEGER - 63)) | (1L << (DECIMAL - 63)))) != 0)) ) {
-        _errHandler.recoverInline(this);
-        } else {
-          consume();
-        }
-        }
-        break;
-      case 6:
-        {
-        _localctx = new TrueContext(_localctx);
-        _ctx = _localctx;
-        _prevctx = _localctx;
-        setState(215);
-        match(TRUE);
-        }
-        break;
-      case 7:
-        {
-        _localctx = new FalseContext(_localctx);
-        _ctx = _localctx;
-        _prevctx = _localctx;
-        setState(216);
-        match(FALSE);
-        }
-        break;
-      case 8:
-        {
-        _localctx = new NullContext(_localctx);
-        _ctx = _localctx;
-        _prevctx = _localctx;
-        setState(217);
-        match(NULL);
-        }
-        break;
-      case 9:
-        {
-        _localctx = new PostincContext(_localctx);
-        _ctx = _localctx;
-        _prevctx = _localctx;
-        setState(218);
-        chain();
-        setState(219);
-        _la = _input.LA(1);
-        if ( !(_la==INCR || _la==DECR) ) {
-        _errHandler.recoverInline(this);
-        } else {
-          consume();
-        }
-        }
-        break;
-      case 10:
-        {
-        _localctx = new PreincContext(_localctx);
-        _ctx = _localctx;
-        _prevctx = _localctx;
-        setState(221);
-        _la = _input.LA(1);
-        if ( !(_la==INCR || _la==DECR) ) {
-        _errHandler.recoverInline(this);
-        } else {
-          consume();
-        }
-        setState(222);
-        chain();
-        }
-        break;
-      case 11:
-        {
-        _localctx = new ReadContext(_localctx);
-        _ctx = _localctx;
-        _prevctx = _localctx;
-        setState(223);
-        chain();
+        setState(177);
+        ((SingleContext)_localctx).u = unary(false);
+         ((SingleContext)_localctx).s =  ((SingleContext)_localctx).u.s; 
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(264);
+      setState(241);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,17,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(262);
-          switch ( getInterpreter().adaptivePredict(_input,20,_ctx) ) {
+          setState(239);
+          switch ( getInterpreter().adaptivePredict(_input,16,_ctx) ) {
           case 1:
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(226);
+            setState(182);
             if (!(precpred(_ctx, 12))) throw new FailedPredicateException(this, "precpred(_ctx, 12)");
-            setState(227);
+            setState(183);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << MUL) | (1L << DIV) | (1L << REM))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(228);
+            setState(184);
             expression(13);
+             ((BinaryContext)_localctx).s =  false; 
             }
             break;
           case 2:
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(229);
+            setState(187);
             if (!(precpred(_ctx, 11))) throw new FailedPredicateException(this, "precpred(_ctx, 11)");
-            setState(230);
+            setState(188);
             _la = _input.LA(1);
             if ( !(_la==ADD || _la==SUB) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(231);
+            setState(189);
             expression(12);
+             ((BinaryContext)_localctx).s =  false; 
             }
             break;
           case 3:
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(232);
+            setState(192);
             if (!(precpred(_ctx, 10))) throw new FailedPredicateException(this, "precpred(_ctx, 10)");
-            setState(233);
+            setState(193);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LSH) | (1L << RSH) | (1L << USH))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(234);
+            setState(194);
             expression(11);
+             ((BinaryContext)_localctx).s =  false; 
             }
             break;
           case 4:
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(235);
+            setState(197);
             if (!(precpred(_ctx, 9))) throw new FailedPredicateException(this, "precpred(_ctx, 9)");
-            setState(236);
+            setState(198);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LT) | (1L << LTE) | (1L << GT) | (1L << GTE))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(237);
+            setState(199);
             expression(10);
+             ((CompContext)_localctx).s =  false; 
             }
             break;
           case 5:
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(238);
+            setState(202);
             if (!(precpred(_ctx, 8))) throw new FailedPredicateException(this, "precpred(_ctx, 8)");
-            setState(239);
+            setState(203);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << EQ) | (1L << EQR) | (1L << NE) | (1L << NER))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(240);
+            setState(204);
             expression(9);
+             ((CompContext)_localctx).s =  false; 
             }
             break;
           case 6:
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(241);
+            setState(207);
             if (!(precpred(_ctx, 7))) throw new FailedPredicateException(this, "precpred(_ctx, 7)");
-            setState(242);
+            setState(208);
             match(BWAND);
-            setState(243);
+            setState(209);
             expression(8);
+             ((BinaryContext)_localctx).s =  false; 
             }
             break;
           case 7:
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(244);
+            setState(212);
             if (!(precpred(_ctx, 6))) throw new FailedPredicateException(this, "precpred(_ctx, 6)");
-            setState(245);
+            setState(213);
             match(XOR);
-            setState(246);
+            setState(214);
             expression(7);
+             ((BinaryContext)_localctx).s =  false; 
             }
             break;
           case 8:
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(247);
+            setState(217);
             if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)");
-            setState(248);
+            setState(218);
             match(BWOR);
-            setState(249);
+            setState(219);
             expression(6);
+             ((BinaryContext)_localctx).s =  false; 
             }
             break;
           case 9:
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(250);
+            setState(222);
             if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)");
-            setState(251);
+            setState(223);
             match(BOOLAND);
-            setState(252);
+            setState(224);
             expression(5);
+             ((BoolContext)_localctx).s =  false; 
             }
             break;
           case 10:
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(253);
+            setState(227);
             if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)");
-            setState(254);
+            setState(228);
             match(BOOLOR);
-            setState(255);
+            setState(229);
             expression(4);
+             ((BoolContext)_localctx).s =  false; 
             }
             break;
           case 11:
             {
             _localctx = new ConditionalContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(256);
+            setState(232);
             if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
-            setState(257);
+            setState(233);
             match(COND);
-            setState(258);
-            expression(0);
-            setState(259);
+            setState(234);
+            ((ConditionalContext)_localctx).e0 = expression(0);
+            setState(235);
             match(COLON);
-            setState(260);
-            expression(2);
+            setState(236);
+            ((ConditionalContext)_localctx).e1 = expression(2);
+             ((ConditionalContext)_localctx).s =  ((ConditionalContext)_localctx).e0.s && ((ConditionalContext)_localctx).e1.s; 
             }
             break;
           }
           } 
         }
-        setState(266);
+        setState(243);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,17,_ctx);
       }
       }
     }
@@ -1896,270 +1506,262 @@ class PainlessParser extends Parser {
     return _localctx;
   }
 
-  public static class ChainContext extends ParserRuleContext {
-    public LinkprecContext linkprec() {
-      return getRuleContext(LinkprecContext.class,0);
-    }
-    public LinkcastContext linkcast() {
-      return getRuleContext(LinkcastContext.class,0);
-    }
-    public LinkvarContext linkvar() {
-      return getRuleContext(LinkvarContext.class,0);
-    }
-    public LinknewContext linknew() {
-      return getRuleContext(LinknewContext.class,0);
-    }
-    public LinkstringContext linkstring() {
-      return getRuleContext(LinkstringContext.class,0);
-    }
-    public ChainContext(ParserRuleContext parent, int invokingState) {
+  public static class UnaryContext extends ParserRuleContext {
+    public boolean c;
+    public boolean s =  true;
+    public UnaryContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); }
+    public UnaryContext(ParserRuleContext parent, int invokingState, boolean c) {
       super(parent, invokingState);
+      this.c = c;
     }
-    @Override public int getRuleIndex() { return RULE_chain; }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitChain(this);
-      else return visitor.visitChildren(this);
+    @Override public int getRuleIndex() { return RULE_unary; }
+   
+    public UnaryContext() { }
+    public void copyFrom(UnaryContext ctx) {
+      super.copyFrom(ctx);
+      this.c = ctx.c;
+      this.s = ctx.s;
     }
   }
-
-  public final ChainContext chain() throws RecognitionException {
-    ChainContext _localctx = new ChainContext(_ctx, getState());
-    enterRule(_localctx, 28, RULE_chain);
-    try {
-      setState(272);
-      switch ( getInterpreter().adaptivePredict(_input,22,_ctx) ) {
-      case 1:
-        enterOuterAlt(_localctx, 1);
-        {
-        setState(267);
-        linkprec();
-        }
-        break;
-      case 2:
-        enterOuterAlt(_localctx, 2);
-        {
-        setState(268);
-        linkcast();
-        }
-        break;
-      case 3:
-        enterOuterAlt(_localctx, 3);
-        {
-        setState(269);
-        linkvar();
-        }
-        break;
-      case 4:
-        enterOuterAlt(_localctx, 4);
-        {
-        setState(270);
-        linknew();
-        }
-        break;
-      case 5:
-        enterOuterAlt(_localctx, 5);
-        {
-        setState(271);
-        linkstring();
-        }
-        break;
-      }
-    }
-    catch (RecognitionException re) {
-      _localctx.exception = re;
-      _errHandler.reportError(this, re);
-      _errHandler.recover(this, re);
-    }
-    finally {
-      exitRule();
-    }
-    return _localctx;
-  }
-
-  public static class LinkprecContext extends ParserRuleContext {
-    public TerminalNode LP() { return getToken(PainlessParser.LP, 0); }
-    public TerminalNode RP() { return getToken(PainlessParser.RP, 0); }
-    public LinkprecContext linkprec() {
-      return getRuleContext(LinkprecContext.class,0);
-    }
-    public LinkcastContext linkcast() {
-      return getRuleContext(LinkcastContext.class,0);
-    }
-    public LinkvarContext linkvar() {
-      return getRuleContext(LinkvarContext.class,0);
-    }
-    public LinknewContext linknew() {
-      return getRuleContext(LinknewContext.class,0);
-    }
-    public LinkstringContext linkstring() {
-      return getRuleContext(LinkstringContext.class,0);
-    }
-    public LinkdotContext linkdot() {
-      return getRuleContext(LinkdotContext.class,0);
-    }
-    public LinkbraceContext linkbrace() {
-      return getRuleContext(LinkbraceContext.class,0);
-    }
-    public LinkprecContext(ParserRuleContext parent, int invokingState) {
-      super(parent, invokingState);
-    }
-    @Override public int getRuleIndex() { return RULE_linkprec; }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitLinkprec(this);
-      else return visitor.visitChildren(this);
-    }
-  }
-
-  public final LinkprecContext linkprec() throws RecognitionException {
-    LinkprecContext _localctx = new LinkprecContext(_ctx, getState());
-    enterRule(_localctx, 30, RULE_linkprec);
-    try {
-      enterOuterAlt(_localctx, 1);
-      {
-      setState(274);
-      match(LP);
-      setState(280);
-      switch ( getInterpreter().adaptivePredict(_input,23,_ctx) ) {
-      case 1:
-        {
-        setState(275);
-        linkprec();
-        }
-        break;
-      case 2:
-        {
-        setState(276);
-        linkcast();
-        }
-        break;
-      case 3:
-        {
-        setState(277);
-        linkvar();
-        }
-        break;
-      case 4:
-        {
-        setState(278);
-        linknew();
-        }
-        break;
-      case 5:
-        {
-        setState(279);
-        linkstring();
-        }
-        break;
-      }
-      setState(282);
-      match(RP);
-      setState(285);
-      switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) {
-      case 1:
-        {
-        setState(283);
-        linkdot();
-        }
-        break;
-      case 2:
-        {
-        setState(284);
-        linkbrace();
-        }
-        break;
-      }
-      }
-    }
-    catch (RecognitionException re) {
-      _localctx.exception = re;
-      _errHandler.reportError(this, re);
-      _errHandler.recover(this, re);
-    }
-    finally {
-      exitRule();
-    }
-    return _localctx;
-  }
-
-  public static class LinkcastContext extends ParserRuleContext {
+  public static class CastContext extends UnaryContext {
     public TerminalNode LP() { return getToken(PainlessParser.LP, 0); }
     public DecltypeContext decltype() {
       return getRuleContext(DecltypeContext.class,0);
     }
     public TerminalNode RP() { return getToken(PainlessParser.RP, 0); }
-    public LinkprecContext linkprec() {
-      return getRuleContext(LinkprecContext.class,0);
+    public UnaryContext unary() {
+      return getRuleContext(UnaryContext.class,0);
     }
-    public LinkcastContext linkcast() {
-      return getRuleContext(LinkcastContext.class,0);
-    }
-    public LinkvarContext linkvar() {
-      return getRuleContext(LinkvarContext.class,0);
-    }
-    public LinknewContext linknew() {
-      return getRuleContext(LinknewContext.class,0);
-    }
-    public LinkstringContext linkstring() {
-      return getRuleContext(LinkstringContext.class,0);
-    }
-    public LinkcastContext(ParserRuleContext parent, int invokingState) {
-      super(parent, invokingState);
-    }
-    @Override public int getRuleIndex() { return RULE_linkcast; }
+    public CastContext(UnaryContext ctx) { copyFrom(ctx); }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitLinkcast(this);
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitCast(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+  public static class PreContext extends UnaryContext {
+    public ChainContext chain() {
+      return getRuleContext(ChainContext.class,0);
+    }
+    public TerminalNode INCR() { return getToken(PainlessParser.INCR, 0); }
+    public TerminalNode DECR() { return getToken(PainlessParser.DECR, 0); }
+    public PreContext(UnaryContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitPre(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+  public static class ReadContext extends UnaryContext {
+    public ChainContext chain() {
+      return getRuleContext(ChainContext.class,0);
+    }
+    public ReadContext(UnaryContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitRead(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+  public static class PostContext extends UnaryContext {
+    public ChainContext chain() {
+      return getRuleContext(ChainContext.class,0);
+    }
+    public TerminalNode INCR() { return getToken(PainlessParser.INCR, 0); }
+    public TerminalNode DECR() { return getToken(PainlessParser.DECR, 0); }
+    public PostContext(UnaryContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitPost(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+  public static class NullContext extends UnaryContext {
+    public TerminalNode NULL() { return getToken(PainlessParser.NULL, 0); }
+    public NullContext(UnaryContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitNull(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+  public static class TrueContext extends UnaryContext {
+    public TerminalNode TRUE() { return getToken(PainlessParser.TRUE, 0); }
+    public TrueContext(UnaryContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitTrue(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+  public static class FalseContext extends UnaryContext {
+    public TerminalNode FALSE() { return getToken(PainlessParser.FALSE, 0); }
+    public FalseContext(UnaryContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitFalse(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+  public static class NumericContext extends UnaryContext {
+    public TerminalNode OCTAL() { return getToken(PainlessParser.OCTAL, 0); }
+    public TerminalNode HEX() { return getToken(PainlessParser.HEX, 0); }
+    public TerminalNode INTEGER() { return getToken(PainlessParser.INTEGER, 0); }
+    public TerminalNode DECIMAL() { return getToken(PainlessParser.DECIMAL, 0); }
+    public NumericContext(UnaryContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitNumeric(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+  public static class OperatorContext extends UnaryContext {
+    public UnaryContext unary() {
+      return getRuleContext(UnaryContext.class,0);
+    }
+    public TerminalNode BOOLNOT() { return getToken(PainlessParser.BOOLNOT, 0); }
+    public TerminalNode BWNOT() { return getToken(PainlessParser.BWNOT, 0); }
+    public TerminalNode ADD() { return getToken(PainlessParser.ADD, 0); }
+    public TerminalNode SUB() { return getToken(PainlessParser.SUB, 0); }
+    public OperatorContext(UnaryContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitOperator(this);
       else return visitor.visitChildren(this);
     }
   }
 
-  public final LinkcastContext linkcast() throws RecognitionException {
-    LinkcastContext _localctx = new LinkcastContext(_ctx, getState());
-    enterRule(_localctx, 32, RULE_linkcast);
+  public final UnaryContext unary(boolean c) throws RecognitionException {
+    UnaryContext _localctx = new UnaryContext(_ctx, getState(), c);
+    enterRule(_localctx, 26, RULE_unary);
+    int _la;
     try {
-      enterOuterAlt(_localctx, 1);
-      {
-      setState(287);
-      match(LP);
-      setState(288);
-      decltype();
-      setState(289);
-      match(RP);
-      setState(295);
-      switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) {
+      setState(273);
+      switch ( getInterpreter().adaptivePredict(_input,18,_ctx) ) {
       case 1:
+        _localctx = new PreContext(_localctx);
+        enterOuterAlt(_localctx, 1);
         {
-        setState(290);
-        linkprec();
+        setState(244);
+        if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
+        setState(245);
+        _la = _input.LA(1);
+        if ( !(_la==INCR || _la==DECR) ) {
+        _errHandler.recoverInline(this);
+        } else {
+          consume();
+        }
+        setState(246);
+        chain(true);
         }
         break;
       case 2:
+        _localctx = new PostContext(_localctx);
+        enterOuterAlt(_localctx, 2);
         {
-        setState(291);
-        linkcast();
+        setState(247);
+        if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
+        setState(248);
+        chain(true);
+        setState(249);
+        _la = _input.LA(1);
+        if ( !(_la==INCR || _la==DECR) ) {
+        _errHandler.recoverInline(this);
+        } else {
+          consume();
+        }
         }
         break;
       case 3:
+        _localctx = new ReadContext(_localctx);
+        enterOuterAlt(_localctx, 3);
         {
-        setState(292);
-        linkvar();
+        setState(251);
+        if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
+        setState(252);
+        chain(false);
         }
         break;
       case 4:
+        _localctx = new NumericContext(_localctx);
+        enterOuterAlt(_localctx, 4);
         {
-        setState(293);
-        linknew();
+        setState(253);
+        if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
+        setState(254);
+        _la = _input.LA(1);
+        if ( !(((((_la - 63)) & ~0x3f) == 0 && ((1L << (_la - 63)) & ((1L << (OCTAL - 63)) | (1L << (HEX - 63)) | (1L << (INTEGER - 63)) | (1L << (DECIMAL - 63)))) != 0)) ) {
+        _errHandler.recoverInline(this);
+        } else {
+          consume();
+        }
+         ((NumericContext)_localctx).s =  false; 
         }
         break;
       case 5:
+        _localctx = new TrueContext(_localctx);
+        enterOuterAlt(_localctx, 5);
         {
-        setState(294);
-        linkstring();
+        setState(256);
+        if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
+        setState(257);
+        match(TRUE);
+         ((TrueContext)_localctx).s =  false; 
+        }
+        break;
+      case 6:
+        _localctx = new FalseContext(_localctx);
+        enterOuterAlt(_localctx, 6);
+        {
+        setState(259);
+        if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
+        setState(260);
+        match(FALSE);
+         ((FalseContext)_localctx).s =  false; 
+        }
+        break;
+      case 7:
+        _localctx = new NullContext(_localctx);
+        enterOuterAlt(_localctx, 7);
+        {
+        setState(262);
+        if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
+        setState(263);
+        match(NULL);
+         ((NullContext)_localctx).s =  false; 
+        }
+        break;
+      case 8:
+        _localctx = new OperatorContext(_localctx);
+        enterOuterAlt(_localctx, 8);
+        {
+        setState(265);
+        if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
+        setState(266);
+        _la = _input.LA(1);
+        if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB))) != 0)) ) {
+        _errHandler.recoverInline(this);
+        } else {
+          consume();
+        }
+        setState(267);
+        unary(false);
+        }
+        break;
+      case 9:
+        _localctx = new CastContext(_localctx);
+        enterOuterAlt(_localctx, 9);
+        {
+        setState(268);
+        match(LP);
+        setState(269);
+        decltype();
+        setState(270);
+        match(RP);
+        setState(271);
+        unary(_localctx.c);
         }
         break;
-      }
       }
     }
     catch (RecognitionException re) {
@@ -2173,314 +1775,62 @@ class PainlessParser extends Parser {
     return _localctx;
   }
 
-  public static class LinkbraceContext extends ParserRuleContext {
-    public TerminalNode LBRACE() { return getToken(PainlessParser.LBRACE, 0); }
-    public ExpressionContext expression() {
-      return getRuleContext(ExpressionContext.class,0);
-    }
-    public TerminalNode RBRACE() { return getToken(PainlessParser.RBRACE, 0); }
-    public LinkdotContext linkdot() {
-      return getRuleContext(LinkdotContext.class,0);
-    }
-    public LinkbraceContext linkbrace() {
-      return getRuleContext(LinkbraceContext.class,0);
-    }
-    public LinkbraceContext(ParserRuleContext parent, int invokingState) {
+  public static class ChainContext extends ParserRuleContext {
+    public boolean c;
+    public ChainContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); }
+    public ChainContext(ParserRuleContext parent, int invokingState, boolean c) {
       super(parent, invokingState);
+      this.c = c;
     }
-    @Override public int getRuleIndex() { return RULE_linkbrace; }
+    @Override public int getRuleIndex() { return RULE_chain; }
+   
+    public ChainContext() { }
+    public void copyFrom(ChainContext ctx) {
+      super.copyFrom(ctx);
+      this.c = ctx.c;
+    }
+  }
+  public static class StaticContext extends ChainContext {
+    public DecltypeContext decltype() {
+      return getRuleContext(DecltypeContext.class,0);
+    }
+    public DotContext dot() {
+      return getRuleContext(DotContext.class,0);
+    }
+    public List<SecondaryContext> secondary() {
+      return getRuleContexts(SecondaryContext.class);
+    }
+    public SecondaryContext secondary(int i) {
+      return getRuleContext(SecondaryContext.class,i);
+    }
+    public StaticContext(ChainContext ctx) { copyFrom(ctx); }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitLinkbrace(this);
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitStatic(this);
       else return visitor.visitChildren(this);
     }
   }
-
-  public final LinkbraceContext linkbrace() throws RecognitionException {
-    LinkbraceContext _localctx = new LinkbraceContext(_ctx, getState());
-    enterRule(_localctx, 34, RULE_linkbrace);
-    try {
-      enterOuterAlt(_localctx, 1);
-      {
-      setState(297);
-      match(LBRACE);
-      setState(298);
-      expression(0);
-      setState(299);
-      match(RBRACE);
-      setState(302);
-      switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) {
-      case 1:
-        {
-        setState(300);
-        linkdot();
-        }
-        break;
-      case 2:
-        {
-        setState(301);
-        linkbrace();
-        }
-        break;
-      }
-      }
+  public static class DynamicContext extends ChainContext {
+    public PrimaryContext p;
+    public PrimaryContext primary() {
+      return getRuleContext(PrimaryContext.class,0);
     }
-    catch (RecognitionException re) {
-      _localctx.exception = re;
-      _errHandler.reportError(this, re);
-      _errHandler.recover(this, re);
+    public List<SecondaryContext> secondary() {
+      return getRuleContexts(SecondaryContext.class);
     }
-    finally {
-      exitRule();
+    public SecondaryContext secondary(int i) {
+      return getRuleContext(SecondaryContext.class,i);
     }
-    return _localctx;
-  }
-
-  public static class LinkdotContext extends ParserRuleContext {
-    public TerminalNode DOT() { return getToken(PainlessParser.DOT, 0); }
-    public LinkcallContext linkcall() {
-      return getRuleContext(LinkcallContext.class,0);
-    }
-    public LinkfieldContext linkfield() {
-      return getRuleContext(LinkfieldContext.class,0);
-    }
-    public LinkdotContext(ParserRuleContext parent, int invokingState) {
-      super(parent, invokingState);
-    }
-    @Override public int getRuleIndex() { return RULE_linkdot; }
+    public DynamicContext(ChainContext ctx) { copyFrom(ctx); }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitLinkdot(this);
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitDynamic(this);
       else return visitor.visitChildren(this);
     }
   }
-
-  public final LinkdotContext linkdot() throws RecognitionException {
-    LinkdotContext _localctx = new LinkdotContext(_ctx, getState());
-    enterRule(_localctx, 36, RULE_linkdot);
-    try {
-      enterOuterAlt(_localctx, 1);
-      {
-      setState(304);
-      match(DOT);
-      setState(307);
-      switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) {
-      case 1:
-        {
-        setState(305);
-        linkcall();
-        }
-        break;
-      case 2:
-        {
-        setState(306);
-        linkfield();
-        }
-        break;
-      }
-      }
-    }
-    catch (RecognitionException re) {
-      _localctx.exception = re;
-      _errHandler.reportError(this, re);
-      _errHandler.recover(this, re);
-    }
-    finally {
-      exitRule();
-    }
-    return _localctx;
-  }
-
-  public static class LinkcallContext extends ParserRuleContext {
-    public TerminalNode EXTID() { return getToken(PainlessParser.EXTID, 0); }
-    public ArgumentsContext arguments() {
-      return getRuleContext(ArgumentsContext.class,0);
-    }
-    public LinkdotContext linkdot() {
-      return getRuleContext(LinkdotContext.class,0);
-    }
-    public LinkbraceContext linkbrace() {
-      return getRuleContext(LinkbraceContext.class,0);
-    }
-    public LinkcallContext(ParserRuleContext parent, int invokingState) {
-      super(parent, invokingState);
-    }
-    @Override public int getRuleIndex() { return RULE_linkcall; }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitLinkcall(this);
-      else return visitor.visitChildren(this);
-    }
-  }
-
-  public final LinkcallContext linkcall() throws RecognitionException {
-    LinkcallContext _localctx = new LinkcallContext(_ctx, getState());
-    enterRule(_localctx, 38, RULE_linkcall);
-    try {
-      enterOuterAlt(_localctx, 1);
-      {
-      setState(309);
-      match(EXTID);
-      setState(310);
-      arguments();
-      setState(313);
-      switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) {
-      case 1:
-        {
-        setState(311);
-        linkdot();
-        }
-        break;
-      case 2:
-        {
-        setState(312);
-        linkbrace();
-        }
-        break;
-      }
-      }
-    }
-    catch (RecognitionException re) {
-      _localctx.exception = re;
-      _errHandler.reportError(this, re);
-      _errHandler.recover(this, re);
-    }
-    finally {
-      exitRule();
-    }
-    return _localctx;
-  }
-
-  public static class LinkvarContext extends ParserRuleContext {
-    public IdentifierContext identifier() {
-      return getRuleContext(IdentifierContext.class,0);
-    }
-    public LinkdotContext linkdot() {
-      return getRuleContext(LinkdotContext.class,0);
-    }
-    public LinkbraceContext linkbrace() {
-      return getRuleContext(LinkbraceContext.class,0);
-    }
-    public LinkvarContext(ParserRuleContext parent, int invokingState) {
-      super(parent, invokingState);
-    }
-    @Override public int getRuleIndex() { return RULE_linkvar; }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitLinkvar(this);
-      else return visitor.visitChildren(this);
-    }
-  }
-
-  public final LinkvarContext linkvar() throws RecognitionException {
-    LinkvarContext _localctx = new LinkvarContext(_ctx, getState());
-    enterRule(_localctx, 40, RULE_linkvar);
-    try {
-      enterOuterAlt(_localctx, 1);
-      {
-      setState(315);
-      identifier();
-      setState(318);
-      switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
-      case 1:
-        {
-        setState(316);
-        linkdot();
-        }
-        break;
-      case 2:
-        {
-        setState(317);
-        linkbrace();
-        }
-        break;
-      }
-      }
-    }
-    catch (RecognitionException re) {
-      _localctx.exception = re;
-      _errHandler.reportError(this, re);
-      _errHandler.recover(this, re);
-    }
-    finally {
-      exitRule();
-    }
-    return _localctx;
-  }
-
-  public static class LinkfieldContext extends ParserRuleContext {
-    public TerminalNode EXTID() { return getToken(PainlessParser.EXTID, 0); }
-    public TerminalNode EXTINTEGER() { return getToken(PainlessParser.EXTINTEGER, 0); }
-    public LinkdotContext linkdot() {
-      return getRuleContext(LinkdotContext.class,0);
-    }
-    public LinkbraceContext linkbrace() {
-      return getRuleContext(LinkbraceContext.class,0);
-    }
-    public LinkfieldContext(ParserRuleContext parent, int invokingState) {
-      super(parent, invokingState);
-    }
-    @Override public int getRuleIndex() { return RULE_linkfield; }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitLinkfield(this);
-      else return visitor.visitChildren(this);
-    }
-  }
-
-  public final LinkfieldContext linkfield() throws RecognitionException {
-    LinkfieldContext _localctx = new LinkfieldContext(_ctx, getState());
-    enterRule(_localctx, 42, RULE_linkfield);
-    int _la;
-    try {
-      enterOuterAlt(_localctx, 1);
-      {
-      setState(320);
-      _la = _input.LA(1);
-      if ( !(_la==EXTINTEGER || _la==EXTID) ) {
-      _errHandler.recoverInline(this);
-      } else {
-        consume();
-      }
-      setState(323);
-      switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
-      case 1:
-        {
-        setState(321);
-        linkdot();
-        }
-        break;
-      case 2:
-        {
-        setState(322);
-        linkbrace();
-        }
-        break;
-      }
-      }
-    }
-    catch (RecognitionException re) {
-      _localctx.exception = re;
-      _errHandler.reportError(this, re);
-      _errHandler.recover(this, re);
-    }
-    finally {
-      exitRule();
-    }
-    return _localctx;
-  }
-
-  public static class LinknewContext extends ParserRuleContext {
+  public static class NewarrayContext extends ChainContext {
     public TerminalNode NEW() { return getToken(PainlessParser.NEW, 0); }
-    public IdentifierContext identifier() {
-      return getRuleContext(IdentifierContext.class,0);
-    }
-    public ArgumentsContext arguments() {
-      return getRuleContext(ArgumentsContext.class,0);
-    }
-    public LinkdotContext linkdot() {
-      return getRuleContext(LinkdotContext.class,0);
-    }
+    public TerminalNode TYPE() { return getToken(PainlessParser.TYPE, 0); }
     public List<TerminalNode> LBRACE() { return getTokens(PainlessParser.LBRACE); }
     public TerminalNode LBRACE(int i) {
       return getToken(PainlessParser.LBRACE, i);
@@ -2495,51 +1845,89 @@ class PainlessParser extends Parser {
     public TerminalNode RBRACE(int i) {
       return getToken(PainlessParser.RBRACE, i);
     }
-    public LinknewContext(ParserRuleContext parent, int invokingState) {
-      super(parent, invokingState);
+    public DotContext dot() {
+      return getRuleContext(DotContext.class,0);
     }
-    @Override public int getRuleIndex() { return RULE_linknew; }
+    public List<SecondaryContext> secondary() {
+      return getRuleContexts(SecondaryContext.class);
+    }
+    public SecondaryContext secondary(int i) {
+      return getRuleContext(SecondaryContext.class,i);
+    }
+    public NewarrayContext(ChainContext ctx) { copyFrom(ctx); }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitLinknew(this);
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitNewarray(this);
       else return visitor.visitChildren(this);
     }
   }
 
-  public final LinknewContext linknew() throws RecognitionException {
-    LinknewContext _localctx = new LinknewContext(_ctx, getState());
-    enterRule(_localctx, 44, RULE_linknew);
+  public final ChainContext chain(boolean c) throws RecognitionException {
+    ChainContext _localctx = new ChainContext(_ctx, getState(), c);
+    enterRule(_localctx, 28, RULE_chain);
     try {
       int _alt;
-      enterOuterAlt(_localctx, 1);
-      {
-      setState(325);
-      match(NEW);
-      setState(326);
-      identifier();
-      setState(342);
-      switch (_input.LA(1)) {
-      case LP:
+      setState(309);
+      switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) {
+      case 1:
+        _localctx = new DynamicContext(_localctx);
+        enterOuterAlt(_localctx, 1);
         {
-        {
-        setState(327);
-        arguments();
-        setState(329);
-        switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) {
-        case 1:
-          {
-          setState(328);
-          linkdot();
+        setState(275);
+        ((DynamicContext)_localctx).p = primary(_localctx.c);
+        setState(279);
+        _errHandler.sync(this);
+        _alt = getInterpreter().adaptivePredict(_input,19,_ctx);
+        while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
+          if ( _alt==1 ) {
+            {
+            {
+            setState(276);
+            secondary(((DynamicContext)_localctx).p.s);
+            }
+            } 
           }
-          break;
-        }
+          setState(281);
+          _errHandler.sync(this);
+          _alt = getInterpreter().adaptivePredict(_input,19,_ctx);
         }
         }
         break;
-      case LBRACE:
+      case 2:
+        _localctx = new StaticContext(_localctx);
+        enterOuterAlt(_localctx, 2);
         {
+        setState(282);
+        decltype();
+        setState(283);
+        dot();
+        setState(287);
+        _errHandler.sync(this);
+        _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
+        while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
+          if ( _alt==1 ) {
+            {
+            {
+            setState(284);
+            secondary(true);
+            }
+            } 
+          }
+          setState(289);
+          _errHandler.sync(this);
+          _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
+        }
+        }
+        break;
+      case 3:
+        _localctx = new NewarrayContext(_localctx);
+        enterOuterAlt(_localctx, 3);
         {
-        setState(335); 
+        setState(290);
+        match(NEW);
+        setState(291);
+        match(TYPE);
+        setState(296); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -2547,11 +1935,11 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(331);
+            setState(292);
             match(LBRACE);
-            setState(332);
+            setState(293);
             expression(0);
-            setState(333);
+            setState(294);
             match(RBRACE);
             }
             }
@@ -2559,25 +1947,37 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(337); 
+          setState(298); 
           _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,32,_ctx);
+          _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
-        setState(340);
-        switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) {
+        setState(307);
+        switch ( getInterpreter().adaptivePredict(_input,23,_ctx) ) {
         case 1:
           {
-          setState(339);
-          linkdot();
+          setState(300);
+          dot();
+          setState(304);
+          _errHandler.sync(this);
+          _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
+          while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
+            if ( _alt==1 ) {
+              {
+              {
+              setState(301);
+              secondary(true);
+              }
+              } 
+            }
+            setState(306);
+            _errHandler.sync(this);
+            _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
+          }
           }
           break;
         }
         }
-        }
         break;
-      default:
-        throw new NoViableAltException(this);
-      }
       }
     }
     catch (RecognitionException re) {
@@ -2591,48 +1991,337 @@ class PainlessParser extends Parser {
     return _localctx;
   }
 
-  public static class LinkstringContext extends ParserRuleContext {
-    public TerminalNode STRING() { return getToken(PainlessParser.STRING, 0); }
-    public LinkdotContext linkdot() {
-      return getRuleContext(LinkdotContext.class,0);
-    }
-    public LinkbraceContext linkbrace() {
-      return getRuleContext(LinkbraceContext.class,0);
-    }
-    public LinkstringContext(ParserRuleContext parent, int invokingState) {
+  public static class PrimaryContext extends ParserRuleContext {
+    public boolean c;
+    public boolean s =  true;
+    public PrimaryContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); }
+    public PrimaryContext(ParserRuleContext parent, int invokingState, boolean c) {
       super(parent, invokingState);
+      this.c = c;
     }
-    @Override public int getRuleIndex() { return RULE_linkstring; }
+    @Override public int getRuleIndex() { return RULE_primary; }
+   
+    public PrimaryContext() { }
+    public void copyFrom(PrimaryContext ctx) {
+      super.copyFrom(ctx);
+      this.c = ctx.c;
+      this.s = ctx.s;
+    }
+  }
+  public static class StringContext extends PrimaryContext {
+    public TerminalNode STRING() { return getToken(PainlessParser.STRING, 0); }
+    public StringContext(PrimaryContext ctx) { copyFrom(ctx); }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitLinkstring(this);
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitString(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+  public static class VariableContext extends PrimaryContext {
+    public TerminalNode ID() { return getToken(PainlessParser.ID, 0); }
+    public VariableContext(PrimaryContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitVariable(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+  public static class ExprprecContext extends PrimaryContext {
+    public ExpressionContext e;
+    public TerminalNode LP() { return getToken(PainlessParser.LP, 0); }
+    public TerminalNode RP() { return getToken(PainlessParser.RP, 0); }
+    public ExpressionContext expression() {
+      return getRuleContext(ExpressionContext.class,0);
+    }
+    public ExprprecContext(PrimaryContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitExprprec(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+  public static class NewobjectContext extends PrimaryContext {
+    public TerminalNode NEW() { return getToken(PainlessParser.NEW, 0); }
+    public TerminalNode TYPE() { return getToken(PainlessParser.TYPE, 0); }
+    public ArgumentsContext arguments() {
+      return getRuleContext(ArgumentsContext.class,0);
+    }
+    public NewobjectContext(PrimaryContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitNewobject(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+  public static class ChainprecContext extends PrimaryContext {
+    public TerminalNode LP() { return getToken(PainlessParser.LP, 0); }
+    public UnaryContext unary() {
+      return getRuleContext(UnaryContext.class,0);
+    }
+    public TerminalNode RP() { return getToken(PainlessParser.RP, 0); }
+    public ChainprecContext(PrimaryContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitChainprec(this);
       else return visitor.visitChildren(this);
     }
   }
 
-  public final LinkstringContext linkstring() throws RecognitionException {
-    LinkstringContext _localctx = new LinkstringContext(_ctx, getState());
-    enterRule(_localctx, 46, RULE_linkstring);
+  public final PrimaryContext primary(boolean c) throws RecognitionException {
+    PrimaryContext _localctx = new PrimaryContext(_ctx, getState(), c);
+    enterRule(_localctx, 30, RULE_primary);
     try {
-      enterOuterAlt(_localctx, 1);
-      {
-      setState(344);
-      match(STRING);
-      setState(347);
-      switch ( getInterpreter().adaptivePredict(_input,35,_ctx) ) {
+      setState(327);
+      switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) {
       case 1:
+        _localctx = new ExprprecContext(_localctx);
+        enterOuterAlt(_localctx, 1);
         {
-        setState(345);
-        linkdot();
+        setState(311);
+        if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
+        setState(312);
+        match(LP);
+        setState(313);
+        ((ExprprecContext)_localctx).e = expression(0);
+        setState(314);
+        match(RP);
+         ((ExprprecContext)_localctx).s =  ((ExprprecContext)_localctx).e.s; 
         }
         break;
       case 2:
+        _localctx = new ChainprecContext(_localctx);
+        enterOuterAlt(_localctx, 2);
         {
-        setState(346);
-        linkbrace();
+        setState(317);
+        if (!( _localctx.c )) throw new FailedPredicateException(this, " $c ");
+        setState(318);
+        match(LP);
+        setState(319);
+        unary(true);
+        setState(320);
+        match(RP);
+        }
+        break;
+      case 3:
+        _localctx = new StringContext(_localctx);
+        enterOuterAlt(_localctx, 3);
+        {
+        setState(322);
+        match(STRING);
+        }
+        break;
+      case 4:
+        _localctx = new VariableContext(_localctx);
+        enterOuterAlt(_localctx, 4);
+        {
+        setState(323);
+        match(ID);
+        }
+        break;
+      case 5:
+        _localctx = new NewobjectContext(_localctx);
+        enterOuterAlt(_localctx, 5);
+        {
+        setState(324);
+        match(NEW);
+        setState(325);
+        match(TYPE);
+        setState(326);
+        arguments();
         }
         break;
       }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
+  public static class SecondaryContext extends ParserRuleContext {
+    public boolean s;
+    public DotContext dot() {
+      return getRuleContext(DotContext.class,0);
+    }
+    public BraceContext brace() {
+      return getRuleContext(BraceContext.class,0);
+    }
+    public SecondaryContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); }
+    public SecondaryContext(ParserRuleContext parent, int invokingState, boolean s) {
+      super(parent, invokingState);
+      this.s = s;
+    }
+    @Override public int getRuleIndex() { return RULE_secondary; }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitSecondary(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final SecondaryContext secondary(boolean s) throws RecognitionException {
+    SecondaryContext _localctx = new SecondaryContext(_ctx, getState(), s);
+    enterRule(_localctx, 32, RULE_secondary);
+    try {
+      setState(333);
+      switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) {
+      case 1:
+        enterOuterAlt(_localctx, 1);
+        {
+        setState(329);
+        if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
+        setState(330);
+        dot();
+        }
+        break;
+      case 2:
+        enterOuterAlt(_localctx, 2);
+        {
+        setState(331);
+        if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
+        setState(332);
+        brace();
+        }
+        break;
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
+  public static class DotContext extends ParserRuleContext {
+    public DotContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_dot; }
+   
+    public DotContext() { }
+    public void copyFrom(DotContext ctx) {
+      super.copyFrom(ctx);
+    }
+  }
+  public static class CallinvokeContext extends DotContext {
+    public TerminalNode DOT() { return getToken(PainlessParser.DOT, 0); }
+    public TerminalNode DOTID() { return getToken(PainlessParser.DOTID, 0); }
+    public ArgumentsContext arguments() {
+      return getRuleContext(ArgumentsContext.class,0);
+    }
+    public CallinvokeContext(DotContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitCallinvoke(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+  public static class FieldaccessContext extends DotContext {
+    public TerminalNode DOT() { return getToken(PainlessParser.DOT, 0); }
+    public TerminalNode DOTID() { return getToken(PainlessParser.DOTID, 0); }
+    public TerminalNode DOTINTEGER() { return getToken(PainlessParser.DOTINTEGER, 0); }
+    public FieldaccessContext(DotContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitFieldaccess(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final DotContext dot() throws RecognitionException {
+    DotContext _localctx = new DotContext(_ctx, getState());
+    enterRule(_localctx, 34, RULE_dot);
+    int _la;
+    try {
+      setState(340);
+      switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) {
+      case 1:
+        _localctx = new CallinvokeContext(_localctx);
+        enterOuterAlt(_localctx, 1);
+        {
+        setState(335);
+        match(DOT);
+        setState(336);
+        match(DOTID);
+        setState(337);
+        arguments();
+        }
+        break;
+      case 2:
+        _localctx = new FieldaccessContext(_localctx);
+        enterOuterAlt(_localctx, 2);
+        {
+        setState(338);
+        match(DOT);
+        setState(339);
+        _la = _input.LA(1);
+        if ( !(_la==DOTINTEGER || _la==DOTID) ) {
+        _errHandler.recoverInline(this);
+        } else {
+          consume();
+        }
+        }
+        break;
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
+  public static class BraceContext extends ParserRuleContext {
+    public BraceContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_brace; }
+   
+    public BraceContext() { }
+    public void copyFrom(BraceContext ctx) {
+      super.copyFrom(ctx);
+    }
+  }
+  public static class BraceaccessContext extends BraceContext {
+    public TerminalNode LBRACE() { return getToken(PainlessParser.LBRACE, 0); }
+    public ExpressionContext expression() {
+      return getRuleContext(ExpressionContext.class,0);
+    }
+    public TerminalNode RBRACE() { return getToken(PainlessParser.RBRACE, 0); }
+    public BraceaccessContext(BraceContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitBraceaccess(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final BraceContext brace() throws RecognitionException {
+    BraceContext _localctx = new BraceContext(_ctx, getState());
+    enterRule(_localctx, 36, RULE_brace);
+    try {
+      _localctx = new BraceaccessContext(_localctx);
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(342);
+      match(LBRACE);
+      setState(343);
+      expression(0);
+      setState(344);
+      match(RBRACE);
       }
     }
     catch (RecognitionException re) {
@@ -2672,40 +2361,40 @@ class PainlessParser extends Parser {
 
   public final ArgumentsContext arguments() throws RecognitionException {
     ArgumentsContext _localctx = new ArgumentsContext(_ctx, getState());
-    enterRule(_localctx, 48, RULE_arguments);
+    enterRule(_localctx, 38, RULE_arguments);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
       {
-      setState(349);
+      setState(346);
       match(LP);
-      setState(358);
-      _la = _input.LA(1);
-      if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LP) | (1L << NEW) | (1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB) | (1L << INCR) | (1L << DECR) | (1L << OCTAL))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)) | (1L << (STRING - 64)) | (1L << (TRUE - 64)) | (1L << (FALSE - 64)) | (1L << (NULL - 64)) | (1L << (ID - 64)))) != 0)) {
+      setState(355);
+      switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
+      case 1:
         {
-        setState(350);
+        setState(347);
         expression(0);
-        setState(355);
+        setState(352);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(351);
+          setState(348);
           match(COMMA);
-          setState(352);
+          setState(349);
           expression(0);
           }
           }
-          setState(357);
+          setState(354);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
+        break;
       }
-
-      setState(360);
+      setState(357);
       match(RP);
       }
       }
@@ -2723,179 +2412,224 @@ class PainlessParser extends Parser {
 
   public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) {
     switch (ruleIndex) {
-    case 13:
+    case 1:
+      return statement_sempred((StatementContext)_localctx, predIndex);
+    case 12:
       return expression_sempred((ExpressionContext)_localctx, predIndex);
+    case 13:
+      return unary_sempred((UnaryContext)_localctx, predIndex);
+    case 15:
+      return primary_sempred((PrimaryContext)_localctx, predIndex);
+    case 16:
+      return secondary_sempred((SecondaryContext)_localctx, predIndex);
+    }
+    return true;
+  }
+  private boolean statement_sempred(StatementContext _localctx, int predIndex) {
+    switch (predIndex) {
+    case 0:
+      return  _input.LA(1) != ELSE ;
     }
     return true;
   }
   private boolean expression_sempred(ExpressionContext _localctx, int predIndex) {
     switch (predIndex) {
-    case 0:
-      return precpred(_ctx, 12);
     case 1:
-      return precpred(_ctx, 11);
+      return precpred(_ctx, 12);
     case 2:
-      return precpred(_ctx, 10);
+      return precpred(_ctx, 11);
     case 3:
-      return precpred(_ctx, 9);
+      return precpred(_ctx, 10);
     case 4:
-      return precpred(_ctx, 8);
+      return precpred(_ctx, 9);
     case 5:
-      return precpred(_ctx, 7);
+      return precpred(_ctx, 8);
     case 6:
-      return precpred(_ctx, 6);
+      return precpred(_ctx, 7);
     case 7:
-      return precpred(_ctx, 5);
+      return precpred(_ctx, 6);
     case 8:
-      return precpred(_ctx, 4);
+      return precpred(_ctx, 5);
     case 9:
-      return precpred(_ctx, 3);
+      return precpred(_ctx, 4);
     case 10:
+      return precpred(_ctx, 3);
+    case 11:
       return precpred(_ctx, 2);
     }
     return true;
   }
+  private boolean unary_sempred(UnaryContext _localctx, int predIndex) {
+    switch (predIndex) {
+    case 12:
+      return  !_localctx.c ;
+    case 13:
+      return  !_localctx.c ;
+    case 14:
+      return  !_localctx.c ;
+    case 15:
+      return  !_localctx.c ;
+    case 16:
+      return  !_localctx.c ;
+    case 17:
+      return  !_localctx.c ;
+    case 18:
+      return  !_localctx.c ;
+    case 19:
+      return  !_localctx.c ;
+    }
+    return true;
+  }
+  private boolean primary_sempred(PrimaryContext _localctx, int predIndex) {
+    switch (predIndex) {
+    case 20:
+      return  !_localctx.c ;
+    case 21:
+      return  _localctx.c ;
+    }
+    return true;
+  }
+  private boolean secondary_sempred(SecondaryContext _localctx, int predIndex) {
+    switch (predIndex) {
+    case 22:
+      return  _localctx.s ;
+    case 23:
+      return  _localctx.s ;
+    }
+    return true;
+  }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3K\u016d\4\2\t\2\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3L\u016a\4\2\t\2\4"+
     "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
     "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
-    "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+
-    "\4\32\t\32\3\2\6\2\66\n\2\r\2\16\2\67\3\2\3\2\3\3\3\3\3\3\3\3\3\3\3\3"+
-    "\3\3\5\3C\n\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3K\n\3\3\3\3\3\3\3\3\3\3\3\3\3"+
-    "\3\3\3\3\3\3\3\3\3\3\5\3X\n\3\3\3\3\3\5\3\\\n\3\3\3\3\3\5\3`\n\3\3\3\3"+
-    "\3\3\3\5\3e\n\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3"+
-    "\3\6\3u\n\3\r\3\16\3v\3\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3\u0080\n\3\3\4\3"+
-    "\4\6\4\u0084\n\4\r\4\16\4\u0085\3\4\3\4\3\4\5\4\u008b\n\4\3\5\3\5\5\5"+
-    "\u008f\n\5\3\6\3\6\3\6\3\7\3\7\5\7\u0096\n\7\3\b\3\b\3\t\3\t\3\t\3\t\7"+
-    "\t\u009e\n\t\f\t\16\t\u00a1\13\t\3\n\3\n\3\n\7\n\u00a6\n\n\f\n\16\n\u00a9"+
-    "\13\n\3\13\3\13\3\13\5\13\u00ae\n\13\3\f\3\f\3\f\3\f\3\f\3\f\3\f\3\f\5"+
-    "\f\u00b8\n\f\3\r\3\r\5\r\u00bc\n\r\3\16\3\16\3\16\3\16\7\16\u00c2\n\16"+
-    "\f\16\16\16\u00c5\13\16\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3"+
-    "\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3"+
-    "\17\3\17\3\17\3\17\3\17\5\17\u00e3\n\17\3\17\3\17\3\17\3\17\3\17\3\17"+
+    "\4\23\t\23\4\24\t\24\4\25\t\25\3\2\7\2,\n\2\f\2\16\2/\13\2\3\2\3\2\3\3"+
+    "\3\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3;\n\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3C\n\3"+
+    "\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3P\n\3\3\3\3\3\5\3T\n\3"+
+    "\3\3\3\3\5\3X\n\3\3\3\3\3\3\3\5\3]\n\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3"+
+    "\3\3\3\3\3\3\3\3\3\3\3\3\6\3m\n\3\r\3\16\3n\3\3\3\3\3\3\3\3\3\3\3\3\3"+
+    "\3\5\3x\n\3\3\4\3\4\5\4|\n\4\3\5\3\5\7\5\u0080\n\5\f\5\16\5\u0083\13\5"+
+    "\3\5\3\5\3\6\3\6\3\7\3\7\5\7\u008b\n\7\3\b\3\b\3\t\3\t\3\t\3\t\7\t\u0093"+
+    "\n\t\f\t\16\t\u0096\13\t\3\n\3\n\3\n\7\n\u009b\n\n\f\n\16\n\u009e\13\n"+
+    "\3\13\3\13\3\13\5\13\u00a3\n\13\3\f\3\f\3\f\3\f\3\f\3\f\3\f\3\r\3\r\3"+
+    "\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\5\16\u00b7\n\16\3\16\3\16"+
+    "\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16"+
+    "\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16"+
+    "\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16"+
+    "\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\7\16"+
+    "\u00f2\n\16\f\16\16\16\u00f5\13\16\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
     "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
-    "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
-    "\3\17\3\17\7\17\u0109\n\17\f\17\16\17\u010c\13\17\3\20\3\20\3\20\3\20"+
-    "\3\20\5\20\u0113\n\20\3\21\3\21\3\21\3\21\3\21\3\21\5\21\u011b\n\21\3"+
-    "\21\3\21\3\21\5\21\u0120\n\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22"+
-    "\5\22\u012a\n\22\3\23\3\23\3\23\3\23\3\23\5\23\u0131\n\23\3\24\3\24\3"+
-    "\24\5\24\u0136\n\24\3\25\3\25\3\25\3\25\5\25\u013c\n\25\3\26\3\26\3\26"+
-    "\5\26\u0141\n\26\3\27\3\27\3\27\5\27\u0146\n\27\3\30\3\30\3\30\3\30\5"+
-    "\30\u014c\n\30\3\30\3\30\3\30\3\30\6\30\u0152\n\30\r\30\16\30\u0153\3"+
-    "\30\5\30\u0157\n\30\5\30\u0159\n\30\3\31\3\31\3\31\5\31\u015e\n\31\3\32"+
-    "\3\32\3\32\3\32\7\32\u0164\n\32\f\32\16\32\u0167\13\32\5\32\u0169\n\32"+
-    "\3\32\3\32\3\32\2\3\34\33\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&"+
-    "(*,.\60\62\2\r\3\3\r\r\4\2\32\33\37 \3\2\65@\3\2AD\3\2\63\64\3\2\34\36"+
-    "\3\2\37 \3\2!#\3\2$\'\3\2(+\3\2JK\u01a3\2\65\3\2\2\2\4\177\3\2\2\2\6\u008a"+
-    "\3\2\2\2\b\u008e\3\2\2\2\n\u0090\3\2\2\2\f\u0095\3\2\2\2\16\u0097\3\2"+
-    "\2\2\20\u0099\3\2\2\2\22\u00a2\3\2\2\2\24\u00aa\3\2\2\2\26\u00af\3\2\2"+
-    "\2\30\u00b9\3\2\2\2\32\u00bd\3\2\2\2\34\u00e2\3\2\2\2\36\u0112\3\2\2\2"+
-    " \u0114\3\2\2\2\"\u0121\3\2\2\2$\u012b\3\2\2\2&\u0132\3\2\2\2(\u0137\3"+
-    "\2\2\2*\u013d\3\2\2\2,\u0142\3\2\2\2.\u0147\3\2\2\2\60\u015a\3\2\2\2\62"+
-    "\u015f\3\2\2\2\64\66\5\4\3\2\65\64\3\2\2\2\66\67\3\2\2\2\67\65\3\2\2\2"+
-    "\678\3\2\2\289\3\2\2\29:\7\2\2\3:\3\3\2\2\2;<\7\16\2\2<=\7\t\2\2=>\5\34"+
-    "\17\2>?\7\n\2\2?B\5\6\4\2@A\7\17\2\2AC\5\6\4\2B@\3\2\2\2BC\3\2\2\2C\u0080"+
-    "\3\2\2\2DE\7\20\2\2EF\7\t\2\2FG\5\34\17\2GJ\7\n\2\2HK\5\6\4\2IK\5\b\5"+
-    "\2JH\3\2\2\2JI\3\2\2\2K\u0080\3\2\2\2LM\7\21\2\2MN\5\6\4\2NO\7\20\2\2"+
-    "OP\7\t\2\2PQ\5\34\17\2QR\7\n\2\2RS\t\2\2\2S\u0080\3\2\2\2TU\7\22\2\2U"+
-    "W\7\t\2\2VX\5\f\7\2WV\3\2\2\2WX\3\2\2\2XY\3\2\2\2Y[\7\r\2\2Z\\\5\34\17"+
-    "\2[Z\3\2\2\2[\\\3\2\2\2\\]\3\2\2\2]_\7\r\2\2^`\5\16\b\2_^\3\2\2\2_`\3"+
-    "\2\2\2`a\3\2\2\2ad\7\n\2\2be\5\6\4\2ce\5\b\5\2db\3\2\2\2dc\3\2\2\2e\u0080"+
-    "\3\2\2\2fg\5\20\t\2gh\t\2\2\2h\u0080\3\2\2\2ij\7\23\2\2j\u0080\t\2\2\2"+
-    "kl\7\24\2\2l\u0080\t\2\2\2mn\7\25\2\2no\5\34\17\2op\t\2\2\2p\u0080\3\2"+
-    "\2\2qr\7\27\2\2rt\5\6\4\2su\5\26\f\2ts\3\2\2\2uv\3\2\2\2vt\3\2\2\2vw\3"+
-    "\2\2\2w\u0080\3\2\2\2xy\7\31\2\2yz\5\34\17\2z{\t\2\2\2{\u0080\3\2\2\2"+
-    "|}\5\34\17\2}~\t\2\2\2~\u0080\3\2\2\2\177;\3\2\2\2\177D\3\2\2\2\177L\3"+
-    "\2\2\2\177T\3\2\2\2\177f\3\2\2\2\177i\3\2\2\2\177k\3\2\2\2\177m\3\2\2"+
-    "\2\177q\3\2\2\2\177x\3\2\2\2\177|\3\2\2\2\u0080\5\3\2\2\2\u0081\u0083"+
-    "\7\5\2\2\u0082\u0084\5\4\3\2\u0083\u0082\3\2\2\2\u0084\u0085\3\2\2\2\u0085"+
-    "\u0083\3\2\2\2\u0085\u0086\3\2\2\2\u0086\u0087\3\2\2\2\u0087\u0088\7\6"+
-    "\2\2\u0088\u008b\3\2\2\2\u0089\u008b\5\4\3\2\u008a\u0081\3\2\2\2\u008a"+
-    "\u0089\3\2\2\2\u008b\7\3\2\2\2\u008c\u008f\5\n\6\2\u008d\u008f\7\r\2\2"+
-    "\u008e\u008c\3\2\2\2\u008e\u008d\3\2\2\2\u008f\t\3\2\2\2\u0090\u0091\7"+
-    "\5\2\2\u0091\u0092\7\6\2\2\u0092\13\3\2\2\2\u0093\u0096\5\20\t\2\u0094"+
-    "\u0096\5\34\17\2\u0095\u0093\3\2\2\2\u0095\u0094\3\2\2\2\u0096\r\3\2\2"+
-    "\2\u0097\u0098\5\34\17\2\u0098\17\3\2\2\2\u0099\u009a\5\22\n\2\u009a\u009f"+
-    "\5\24\13\2\u009b\u009c\7\f\2\2\u009c\u009e\5\24\13\2\u009d\u009b\3\2\2"+
-    "\2\u009e\u00a1\3\2\2\2\u009f\u009d\3\2\2\2\u009f\u00a0\3\2\2\2\u00a0\21"+
-    "\3\2\2\2\u00a1\u009f\3\2\2\2\u00a2\u00a7\5\30\r\2\u00a3\u00a4\7\7\2\2"+
-    "\u00a4\u00a6\7\b\2\2\u00a5\u00a3\3\2\2\2\u00a6\u00a9\3\2\2\2\u00a7\u00a5"+
-    "\3\2\2\2\u00a7\u00a8\3\2\2\2\u00a8\23\3\2\2\2\u00a9\u00a7\3\2\2\2\u00aa"+
-    "\u00ad\5\30\r\2\u00ab\u00ac\7\65\2\2\u00ac\u00ae\5\34\17\2\u00ad\u00ab"+
-    "\3\2\2\2\u00ad\u00ae\3\2\2\2\u00ae\25\3\2\2\2\u00af\u00b0\7\30\2\2\u00b0"+
-    "\u00b1\7\t\2\2\u00b1\u00b2\5\30\r\2\u00b2\u00b3\5\30\r\2\u00b3\u00b4\3"+
-    "\2\2\2\u00b4\u00b7\7\n\2\2\u00b5\u00b8\5\6\4\2\u00b6\u00b8\5\n\6\2\u00b7"+
-    "\u00b5\3\2\2\2\u00b7\u00b6\3\2\2\2\u00b8\27\3\2\2\2\u00b9\u00bb\7I\2\2"+
-    "\u00ba\u00bc\5\32\16\2\u00bb\u00ba\3\2\2\2\u00bb\u00bc\3\2\2\2\u00bc\31"+
-    "\3\2\2\2\u00bd\u00be\7$\2\2\u00be\u00c3\5\30\r\2\u00bf\u00c0\7\f\2\2\u00c0"+
-    "\u00c2\5\30\r\2\u00c1\u00bf\3\2\2\2\u00c2\u00c5\3\2\2\2\u00c3\u00c1\3"+
-    "\2\2\2\u00c3\u00c4\3\2\2\2\u00c4\u00c6\3\2\2\2\u00c5\u00c3\3\2\2\2\u00c6"+
-    "\u00c7\7&\2\2\u00c7\33\3\2\2\2\u00c8\u00c9\b\17\1\2\u00c9\u00ca\t\3\2"+
-    "\2\u00ca\u00e3\5\34\17\20\u00cb\u00cc\7\t\2\2\u00cc\u00cd\5\22\n\2\u00cd"+
-    "\u00ce\7\n\2\2\u00ce\u00cf\5\34\17\17\u00cf\u00e3\3\2\2\2\u00d0\u00d1"+
-    "\5\36\20\2\u00d1\u00d2\t\4\2\2\u00d2\u00d3\5\34\17\3\u00d3\u00e3\3\2\2"+
-    "\2\u00d4\u00d5\7\t\2\2\u00d5\u00d6\5\34\17\2\u00d6\u00d7\7\n\2\2\u00d7"+
-    "\u00e3\3\2\2\2\u00d8\u00e3\t\5\2\2\u00d9\u00e3\7F\2\2\u00da\u00e3\7G\2"+
-    "\2\u00db\u00e3\7H\2\2\u00dc\u00dd\5\36\20\2\u00dd\u00de\t\6\2\2\u00de"+
-    "\u00e3\3\2\2\2\u00df\u00e0\t\6\2\2\u00e0\u00e3\5\36\20\2\u00e1\u00e3\5"+
-    "\36\20\2\u00e2\u00c8\3\2\2\2\u00e2\u00cb\3\2\2\2\u00e2\u00d0\3\2\2\2\u00e2"+
-    "\u00d4\3\2\2\2\u00e2\u00d8\3\2\2\2\u00e2\u00d9\3\2\2\2\u00e2\u00da\3\2"+
-    "\2\2\u00e2\u00db\3\2\2\2\u00e2\u00dc\3\2\2\2\u00e2\u00df\3\2\2\2\u00e2"+
-    "\u00e1\3\2\2\2\u00e3\u010a\3\2\2\2\u00e4\u00e5\f\16\2\2\u00e5\u00e6\t"+
-    "\7\2\2\u00e6\u0109\5\34\17\17\u00e7\u00e8\f\r\2\2\u00e8\u00e9\t\b\2\2"+
-    "\u00e9\u0109\5\34\17\16\u00ea\u00eb\f\f\2\2\u00eb\u00ec\t\t\2\2\u00ec"+
-    "\u0109\5\34\17\r\u00ed\u00ee\f\13\2\2\u00ee\u00ef\t\n\2\2\u00ef\u0109"+
-    "\5\34\17\f\u00f0\u00f1\f\n\2\2\u00f1\u00f2\t\13\2\2\u00f2\u0109\5\34\17"+
-    "\13\u00f3\u00f4\f\t\2\2\u00f4\u00f5\7,\2\2\u00f5\u0109\5\34\17\n\u00f6"+
-    "\u00f7\f\b\2\2\u00f7\u00f8\7-\2\2\u00f8\u0109\5\34\17\t\u00f9\u00fa\f"+
-    "\7\2\2\u00fa\u00fb\7.\2\2\u00fb\u0109\5\34\17\b\u00fc\u00fd\f\6\2\2\u00fd"+
-    "\u00fe\7/\2\2\u00fe\u0109\5\34\17\7\u00ff\u0100\f\5\2\2\u0100\u0101\7"+
-    "\60\2\2\u0101\u0109\5\34\17\6\u0102\u0103\f\4\2\2\u0103\u0104\7\61\2\2"+
-    "\u0104\u0105\5\34\17\2\u0105\u0106\7\62\2\2\u0106\u0107\5\34\17\4\u0107"+
-    "\u0109\3\2\2\2\u0108\u00e4\3\2\2\2\u0108\u00e7\3\2\2\2\u0108\u00ea\3\2"+
-    "\2\2\u0108\u00ed\3\2\2\2\u0108\u00f0\3\2\2\2\u0108\u00f3\3\2\2\2\u0108"+
-    "\u00f6\3\2\2\2\u0108\u00f9\3\2\2\2\u0108\u00fc\3\2\2\2\u0108\u00ff\3\2"+
-    "\2\2\u0108\u0102\3\2\2\2\u0109\u010c\3\2\2\2\u010a\u0108\3\2\2\2\u010a"+
-    "\u010b\3\2\2\2\u010b\35\3\2\2\2\u010c\u010a\3\2\2\2\u010d\u0113\5 \21"+
-    "\2\u010e\u0113\5\"\22\2\u010f\u0113\5*\26\2\u0110\u0113\5.\30\2\u0111"+
-    "\u0113\5\60\31\2\u0112\u010d\3\2\2\2\u0112\u010e\3\2\2\2\u0112\u010f\3"+
-    "\2\2\2\u0112\u0110\3\2\2\2\u0112\u0111\3\2\2\2\u0113\37\3\2\2\2\u0114"+
-    "\u011a\7\t\2\2\u0115\u011b\5 \21\2\u0116\u011b\5\"\22\2\u0117\u011b\5"+
-    "*\26\2\u0118\u011b\5.\30\2\u0119\u011b\5\60\31\2\u011a\u0115\3\2\2\2\u011a"+
-    "\u0116\3\2\2\2\u011a\u0117\3\2\2\2\u011a\u0118\3\2\2\2\u011a\u0119\3\2"+
-    "\2\2\u011b\u011c\3\2\2\2\u011c\u011f\7\n\2\2\u011d\u0120\5&\24\2\u011e"+
-    "\u0120\5$\23\2\u011f\u011d\3\2\2\2\u011f\u011e\3\2\2\2\u011f\u0120\3\2"+
-    "\2\2\u0120!\3\2\2\2\u0121\u0122\7\t\2\2\u0122\u0123\5\22\n\2\u0123\u0129"+
-    "\7\n\2\2\u0124\u012a\5 \21\2\u0125\u012a\5\"\22\2\u0126\u012a\5*\26\2"+
-    "\u0127\u012a\5.\30\2\u0128\u012a\5\60\31\2\u0129\u0124\3\2\2\2\u0129\u0125"+
-    "\3\2\2\2\u0129\u0126\3\2\2\2\u0129\u0127\3\2\2\2\u0129\u0128\3\2\2\2\u012a"+
-    "#\3\2\2\2\u012b\u012c\7\7\2\2\u012c\u012d\5\34\17\2\u012d\u0130\7\b\2"+
-    "\2\u012e\u0131\5&\24\2\u012f\u0131\5$\23\2\u0130\u012e\3\2\2\2\u0130\u012f"+
-    "\3\2\2\2\u0130\u0131\3\2\2\2\u0131%\3\2\2\2\u0132\u0135\7\13\2\2\u0133"+
-    "\u0136\5(\25\2\u0134\u0136\5,\27\2\u0135\u0133\3\2\2\2\u0135\u0134\3\2"+
-    "\2\2\u0136\'\3\2\2\2\u0137\u0138\7K\2\2\u0138\u013b\5\62\32\2\u0139\u013c"+
-    "\5&\24\2\u013a\u013c\5$\23\2\u013b\u0139\3\2\2\2\u013b\u013a\3\2\2\2\u013b"+
-    "\u013c\3\2\2\2\u013c)\3\2\2\2\u013d\u0140\5\30\r\2\u013e\u0141\5&\24\2"+
-    "\u013f\u0141\5$\23\2\u0140\u013e\3\2\2\2\u0140\u013f\3\2\2\2\u0140\u0141"+
-    "\3\2\2\2\u0141+\3\2\2\2\u0142\u0145\t\f\2\2\u0143\u0146\5&\24\2\u0144"+
-    "\u0146\5$\23\2\u0145\u0143\3\2\2\2\u0145\u0144\3\2\2\2\u0145\u0146\3\2"+
-    "\2\2\u0146-\3\2\2\2\u0147\u0148\7\26\2\2\u0148\u0158\5\30\r\2\u0149\u014b"+
-    "\5\62\32\2\u014a\u014c\5&\24\2\u014b\u014a\3\2\2\2\u014b\u014c\3\2\2\2"+
-    "\u014c\u0159\3\2\2\2\u014d\u014e\7\7\2\2\u014e\u014f\5\34\17\2\u014f\u0150"+
-    "\7\b\2\2\u0150\u0152\3\2\2\2\u0151\u014d\3\2\2\2\u0152\u0153\3\2\2\2\u0153"+
-    "\u0151\3\2\2\2\u0153\u0154\3\2\2\2\u0154\u0156\3\2\2\2\u0155\u0157\5&"+
-    "\24\2\u0156\u0155\3\2\2\2\u0156\u0157\3\2\2\2\u0157\u0159\3\2\2\2\u0158"+
-    "\u0149\3\2\2\2\u0158\u0151\3\2\2\2\u0159/\3\2\2\2\u015a\u015d\7E\2\2\u015b"+
-    "\u015e\5&\24\2\u015c\u015e\5$\23\2\u015d\u015b\3\2\2\2\u015d\u015c\3\2"+
-    "\2\2\u015d\u015e\3\2\2\2\u015e\61\3\2\2\2\u015f\u0168\7\t\2\2\u0160\u0165"+
-    "\5\34\17\2\u0161\u0162\7\f\2\2\u0162\u0164\5\34\17\2\u0163\u0161\3\2\2"+
-    "\2\u0164\u0167\3\2\2\2\u0165\u0163\3\2\2\2\u0165\u0166\3\2\2\2\u0166\u0169"+
-    "\3\2\2\2\u0167\u0165\3\2\2\2\u0168\u0160\3\2\2\2\u0168\u0169\3\2\2\2\u0169"+
-    "\u016a\3\2\2\2\u016a\u016b\7\n\2\2\u016b\63\3\2\2\2(\67BJW[_dv\177\u0085"+
-    "\u008a\u008e\u0095\u009f\u00a7\u00ad\u00b7\u00bb\u00c3\u00e2\u0108\u010a"+
-    "\u0112\u011a\u011f\u0129\u0130\u0135\u013b\u0140\u0145\u014b\u0153\u0156"+
-    "\u0158\u015d\u0165\u0168";
+    "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\5\17\u0114\n\17\3\20\3\20\7\20"+
+    "\u0118\n\20\f\20\16\20\u011b\13\20\3\20\3\20\3\20\7\20\u0120\n\20\f\20"+
+    "\16\20\u0123\13\20\3\20\3\20\3\20\3\20\3\20\3\20\6\20\u012b\n\20\r\20"+
+    "\16\20\u012c\3\20\3\20\7\20\u0131\n\20\f\20\16\20\u0134\13\20\5\20\u0136"+
+    "\n\20\5\20\u0138\n\20\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
+    "\3\21\3\21\3\21\3\21\3\21\3\21\5\21\u014a\n\21\3\22\3\22\3\22\3\22\5\22"+
+    "\u0150\n\22\3\23\3\23\3\23\3\23\3\23\5\23\u0157\n\23\3\24\3\24\3\24\3"+
+    "\24\3\25\3\25\3\25\3\25\7\25\u0161\n\25\f\25\16\25\u0164\13\25\5\25\u0166"+
+    "\n\25\3\25\3\25\3\25\2\3\32\26\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36"+
+    " \"$&(\2\r\3\3\r\r\3\2\65@\3\2\34\36\3\2\37 \3\2!#\3\2$\'\3\2(+\3\2\63"+
+    "\64\3\2AD\4\2\32\33\37 \3\2KL\u0190\2-\3\2\2\2\4w\3\2\2\2\6{\3\2\2\2\b"+
+    "}\3\2\2\2\n\u0086\3\2\2\2\f\u008a\3\2\2\2\16\u008c\3\2\2\2\20\u008e\3"+
+    "\2\2\2\22\u0097\3\2\2\2\24\u009f\3\2\2\2\26\u00a4\3\2\2\2\30\u00ab\3\2"+
+    "\2\2\32\u00b6\3\2\2\2\34\u0113\3\2\2\2\36\u0137\3\2\2\2 \u0149\3\2\2\2"+
+    "\"\u014f\3\2\2\2$\u0156\3\2\2\2&\u0158\3\2\2\2(\u015c\3\2\2\2*,\5\4\3"+
+    "\2+*\3\2\2\2,/\3\2\2\2-+\3\2\2\2-.\3\2\2\2.\60\3\2\2\2/-\3\2\2\2\60\61"+
+    "\7\2\2\3\61\3\3\2\2\2\62\63\7\16\2\2\63\64\7\t\2\2\64\65\5\32\16\2\65"+
+    "\66\7\n\2\2\66:\5\6\4\2\678\7\17\2\28;\5\6\4\29;\6\3\2\2:\67\3\2\2\2:"+
+    "9\3\2\2\2;x\3\2\2\2<=\7\20\2\2=>\7\t\2\2>?\5\32\16\2?B\7\n\2\2@C\5\6\4"+
+    "\2AC\5\n\6\2B@\3\2\2\2BA\3\2\2\2Cx\3\2\2\2DE\7\21\2\2EF\5\b\5\2FG\7\20"+
+    "\2\2GH\7\t\2\2HI\5\32\16\2IJ\7\n\2\2JK\5\30\r\2Kx\3\2\2\2LM\7\22\2\2M"+
+    "O\7\t\2\2NP\5\f\7\2ON\3\2\2\2OP\3\2\2\2PQ\3\2\2\2QS\7\r\2\2RT\5\32\16"+
+    "\2SR\3\2\2\2ST\3\2\2\2TU\3\2\2\2UW\7\r\2\2VX\5\16\b\2WV\3\2\2\2WX\3\2"+
+    "\2\2XY\3\2\2\2Y\\\7\n\2\2Z]\5\6\4\2[]\5\n\6\2\\Z\3\2\2\2\\[\3\2\2\2]x"+
+    "\3\2\2\2^_\5\20\t\2_`\5\30\r\2`x\3\2\2\2ab\7\23\2\2bx\5\30\r\2cd\7\24"+
+    "\2\2dx\5\30\r\2ef\7\25\2\2fg\5\32\16\2gh\5\30\r\2hx\3\2\2\2ij\7\27\2\2"+
+    "jl\5\b\5\2km\5\26\f\2lk\3\2\2\2mn\3\2\2\2nl\3\2\2\2no\3\2\2\2ox\3\2\2"+
+    "\2pq\7\31\2\2qr\5\32\16\2rs\5\30\r\2sx\3\2\2\2tu\5\32\16\2uv\5\30\r\2"+
+    "vx\3\2\2\2w\62\3\2\2\2w<\3\2\2\2wD\3\2\2\2wL\3\2\2\2w^\3\2\2\2wa\3\2\2"+
+    "\2wc\3\2\2\2we\3\2\2\2wi\3\2\2\2wp\3\2\2\2wt\3\2\2\2x\5\3\2\2\2y|\5\b"+
+    "\5\2z|\5\4\3\2{y\3\2\2\2{z\3\2\2\2|\7\3\2\2\2}\u0081\7\5\2\2~\u0080\5"+
+    "\4\3\2\177~\3\2\2\2\u0080\u0083\3\2\2\2\u0081\177\3\2\2\2\u0081\u0082"+
+    "\3\2\2\2\u0082\u0084\3\2\2\2\u0083\u0081\3\2\2\2\u0084\u0085\7\6\2\2\u0085"+
+    "\t\3\2\2\2\u0086\u0087\7\r\2\2\u0087\13\3\2\2\2\u0088\u008b\5\20\t\2\u0089"+
+    "\u008b\5\32\16\2\u008a\u0088\3\2\2\2\u008a\u0089\3\2\2\2\u008b\r\3\2\2"+
+    "\2\u008c\u008d\5\32\16\2\u008d\17\3\2\2\2\u008e\u008f\5\22\n\2\u008f\u0094"+
+    "\5\24\13\2\u0090\u0091\7\f\2\2\u0091\u0093\5\24\13\2\u0092\u0090\3\2\2"+
+    "\2\u0093\u0096\3\2\2\2\u0094\u0092\3\2\2\2\u0094\u0095\3\2\2\2\u0095\21"+
+    "\3\2\2\2\u0096\u0094\3\2\2\2\u0097\u009c\7I\2\2\u0098\u0099\7\7\2\2\u0099"+
+    "\u009b\7\b\2\2\u009a\u0098\3\2\2\2\u009b\u009e\3\2\2\2\u009c\u009a\3\2"+
+    "\2\2\u009c\u009d\3\2\2\2\u009d\23\3\2\2\2\u009e\u009c\3\2\2\2\u009f\u00a2"+
+    "\7J\2\2\u00a0\u00a1\7\65\2\2\u00a1\u00a3\5\32\16\2\u00a2\u00a0\3\2\2\2"+
+    "\u00a2\u00a3\3\2\2\2\u00a3\25\3\2\2\2\u00a4\u00a5\7\30\2\2\u00a5\u00a6"+
+    "\7\t\2\2\u00a6\u00a7\7I\2\2\u00a7\u00a8\7J\2\2\u00a8\u00a9\7\n\2\2\u00a9"+
+    "\u00aa\5\b\5\2\u00aa\27\3\2\2\2\u00ab\u00ac\t\2\2\2\u00ac\31\3\2\2\2\u00ad"+
+    "\u00ae\b\16\1\2\u00ae\u00af\5\36\20\2\u00af\u00b0\t\3\2\2\u00b0\u00b1"+
+    "\5\32\16\3\u00b1\u00b2\b\16\1\2\u00b2\u00b7\3\2\2\2\u00b3\u00b4\5\34\17"+
+    "\2\u00b4\u00b5\b\16\1\2\u00b5\u00b7\3\2\2\2\u00b6\u00ad\3\2\2\2\u00b6"+
+    "\u00b3\3\2\2\2\u00b7\u00f3\3\2\2\2\u00b8\u00b9\f\16\2\2\u00b9\u00ba\t"+
+    "\4\2\2\u00ba\u00bb\5\32\16\17\u00bb\u00bc\b\16\1\2\u00bc\u00f2\3\2\2\2"+
+    "\u00bd\u00be\f\r\2\2\u00be\u00bf\t\5\2\2\u00bf\u00c0\5\32\16\16\u00c0"+
+    "\u00c1\b\16\1\2\u00c1\u00f2\3\2\2\2\u00c2\u00c3\f\f\2\2\u00c3\u00c4\t"+
+    "\6\2\2\u00c4\u00c5\5\32\16\r\u00c5\u00c6\b\16\1\2\u00c6\u00f2\3\2\2\2"+
+    "\u00c7\u00c8\f\13\2\2\u00c8\u00c9\t\7\2\2\u00c9\u00ca\5\32\16\f\u00ca"+
+    "\u00cb\b\16\1\2\u00cb\u00f2\3\2\2\2\u00cc\u00cd\f\n\2\2\u00cd\u00ce\t"+
+    "\b\2\2\u00ce\u00cf\5\32\16\13\u00cf\u00d0\b\16\1\2\u00d0\u00f2\3\2\2\2"+
+    "\u00d1\u00d2\f\t\2\2\u00d2\u00d3\7,\2\2\u00d3\u00d4\5\32\16\n\u00d4\u00d5"+
+    "\b\16\1\2\u00d5\u00f2\3\2\2\2\u00d6\u00d7\f\b\2\2\u00d7\u00d8\7-\2\2\u00d8"+
+    "\u00d9\5\32\16\t\u00d9\u00da\b\16\1\2\u00da\u00f2\3\2\2\2\u00db\u00dc"+
+    "\f\7\2\2\u00dc\u00dd\7.\2\2\u00dd\u00de\5\32\16\b\u00de\u00df\b\16\1\2"+
+    "\u00df\u00f2\3\2\2\2\u00e0\u00e1\f\6\2\2\u00e1\u00e2\7/\2\2\u00e2\u00e3"+
+    "\5\32\16\7\u00e3\u00e4\b\16\1\2\u00e4\u00f2\3\2\2\2\u00e5\u00e6\f\5\2"+
+    "\2\u00e6\u00e7\7\60\2\2\u00e7\u00e8\5\32\16\6\u00e8\u00e9\b\16\1\2\u00e9"+
+    "\u00f2\3\2\2\2\u00ea\u00eb\f\4\2\2\u00eb\u00ec\7\61\2\2\u00ec\u00ed\5"+
+    "\32\16\2\u00ed\u00ee\7\62\2\2\u00ee\u00ef\5\32\16\4\u00ef\u00f0\b\16\1"+
+    "\2\u00f0\u00f2\3\2\2\2\u00f1\u00b8\3\2\2\2\u00f1\u00bd\3\2\2\2\u00f1\u00c2"+
+    "\3\2\2\2\u00f1\u00c7\3\2\2\2\u00f1\u00cc\3\2\2\2\u00f1\u00d1\3\2\2\2\u00f1"+
+    "\u00d6\3\2\2\2\u00f1\u00db\3\2\2\2\u00f1\u00e0\3\2\2\2\u00f1\u00e5\3\2"+
+    "\2\2\u00f1\u00ea\3\2\2\2\u00f2\u00f5\3\2\2\2\u00f3\u00f1\3\2\2\2\u00f3"+
+    "\u00f4\3\2\2\2\u00f4\33\3\2\2\2\u00f5\u00f3\3\2\2\2\u00f6\u00f7\6\17\16"+
+    "\3\u00f7\u00f8\t\t\2\2\u00f8\u0114\5\36\20\2\u00f9\u00fa\6\17\17\3\u00fa"+
+    "\u00fb\5\36\20\2\u00fb\u00fc\t\t\2\2\u00fc\u0114\3\2\2\2\u00fd\u00fe\6"+
+    "\17\20\3\u00fe\u0114\5\36\20\2\u00ff\u0100\6\17\21\3\u0100\u0101\t\n\2"+
+    "\2\u0101\u0114\b\17\1\2\u0102\u0103\6\17\22\3\u0103\u0104\7F\2\2\u0104"+
+    "\u0114\b\17\1\2\u0105\u0106\6\17\23\3\u0106\u0107\7G\2\2\u0107\u0114\b"+
+    "\17\1\2\u0108\u0109\6\17\24\3\u0109\u010a\7H\2\2\u010a\u0114\b\17\1\2"+
+    "\u010b\u010c\6\17\25\3\u010c\u010d\t\13\2\2\u010d\u0114\5\34\17\2\u010e"+
+    "\u010f\7\t\2\2\u010f\u0110\5\22\n\2\u0110\u0111\7\n\2\2\u0111\u0112\5"+
+    "\34\17\2\u0112\u0114\3\2\2\2\u0113\u00f6\3\2\2\2\u0113\u00f9\3\2\2\2\u0113"+
+    "\u00fd\3\2\2\2\u0113\u00ff\3\2\2\2\u0113\u0102\3\2\2\2\u0113\u0105\3\2"+
+    "\2\2\u0113\u0108\3\2\2\2\u0113\u010b\3\2\2\2\u0113\u010e\3\2\2\2\u0114"+
+    "\35\3\2\2\2\u0115\u0119\5 \21\2\u0116\u0118\5\"\22\2\u0117\u0116\3\2\2"+
+    "\2\u0118\u011b\3\2\2\2\u0119\u0117\3\2\2\2\u0119\u011a\3\2\2\2\u011a\u0138"+
+    "\3\2\2\2\u011b\u0119\3\2\2\2\u011c\u011d\5\22\n\2\u011d\u0121\5$\23\2"+
+    "\u011e\u0120\5\"\22\2\u011f\u011e\3\2\2\2\u0120\u0123\3\2\2\2\u0121\u011f"+
+    "\3\2\2\2\u0121\u0122\3\2\2\2\u0122\u0138\3\2\2\2\u0123\u0121\3\2\2\2\u0124"+
+    "\u0125\7\26\2\2\u0125\u012a\7I\2\2\u0126\u0127\7\7\2\2\u0127\u0128\5\32"+
+    "\16\2\u0128\u0129\7\b\2\2\u0129\u012b\3\2\2\2\u012a\u0126\3\2\2\2\u012b"+
+    "\u012c\3\2\2\2\u012c\u012a\3\2\2\2\u012c\u012d\3\2\2\2\u012d\u0135\3\2"+
+    "\2\2\u012e\u0132\5$\23\2\u012f\u0131\5\"\22\2\u0130\u012f\3\2\2\2\u0131"+
+    "\u0134\3\2\2\2\u0132\u0130\3\2\2\2\u0132\u0133\3\2\2\2\u0133\u0136\3\2"+
+    "\2\2\u0134\u0132\3\2\2\2\u0135\u012e\3\2\2\2\u0135\u0136\3\2\2\2\u0136"+
+    "\u0138\3\2\2\2\u0137\u0115\3\2\2\2\u0137\u011c\3\2\2\2\u0137\u0124\3\2"+
+    "\2\2\u0138\37\3\2\2\2\u0139\u013a\6\21\26\3\u013a\u013b\7\t\2\2\u013b"+
+    "\u013c\5\32\16\2\u013c\u013d\7\n\2\2\u013d\u013e\b\21\1\2\u013e\u014a"+
+    "\3\2\2\2\u013f\u0140\6\21\27\3\u0140\u0141\7\t\2\2\u0141\u0142\5\34\17"+
+    "\2\u0142\u0143\7\n\2\2\u0143\u014a\3\2\2\2\u0144\u014a\7E\2\2\u0145\u014a"+
+    "\7J\2\2\u0146\u0147\7\26\2\2\u0147\u0148\7I\2\2\u0148\u014a\5(\25\2\u0149"+
+    "\u0139\3\2\2\2\u0149\u013f\3\2\2\2\u0149\u0144\3\2\2\2\u0149\u0145\3\2"+
+    "\2\2\u0149\u0146\3\2\2\2\u014a!\3\2\2\2\u014b\u014c\6\22\30\3\u014c\u0150"+
+    "\5$\23\2\u014d\u014e\6\22\31\3\u014e\u0150\5&\24\2\u014f\u014b\3\2\2\2"+
+    "\u014f\u014d\3\2\2\2\u0150#\3\2\2\2\u0151\u0152\7\13\2\2\u0152\u0153\7"+
+    "L\2\2\u0153\u0157\5(\25\2\u0154\u0155\7\13\2\2\u0155\u0157\t\f\2\2\u0156"+
+    "\u0151\3\2\2\2\u0156\u0154\3\2\2\2\u0157%\3\2\2\2\u0158\u0159\7\7\2\2"+
+    "\u0159\u015a\5\32\16\2\u015a\u015b\7\b\2\2\u015b\'\3\2\2\2\u015c\u0165"+
+    "\7\t\2\2\u015d\u0162\5\32\16\2\u015e\u015f\7\f\2\2\u015f\u0161\5\32\16"+
+    "\2\u0160\u015e\3\2\2\2\u0161\u0164\3\2\2\2\u0162\u0160\3\2\2\2\u0162\u0163"+
+    "\3\2\2\2\u0163\u0166\3\2\2\2\u0164\u0162\3\2\2\2\u0165\u015d\3\2\2\2\u0165"+
+    "\u0166\3\2\2\2\u0166\u0167\3\2\2\2\u0167\u0168\7\n\2\2\u0168)\3\2\2\2"+
+    " -:BOSW\\nw{\u0081\u008a\u0094\u009c\u00a2\u00b6\u00f1\u00f3\u0113\u0119"+
+    "\u0121\u012c\u0132\u0135\u0137\u0149\u014f\u0156\u0162\u0165";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
index dad6e419e25..7d5ed4c9e6f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
@@ -101,14 +101,14 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitMultiple(PainlessParser.MultipleContext ctx) { return visitChildren(ctx); }
+  @Override public T visitTrailer(PainlessParser.TrailerContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitSingle(PainlessParser.SingleContext ctx) { return visitChildren(ctx); }
+  @Override public T visitBlock(PainlessParser.BlockContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
@@ -116,13 +116,6 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitEmpty(PainlessParser.EmptyContext ctx) { return visitChildren(ctx); }
-  /**
-   * {@inheritDoc}
-   *
-   * <p>The default implementation returns the result of calling
-   * {@link #visitChildren} on {@code ctx}.</p>
-   */
-  @Override public T visitEmptyscope(PainlessParser.EmptyscopeContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
@@ -171,14 +164,14 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitIdentifier(PainlessParser.IdentifierContext ctx) { return visitChildren(ctx); }
+  @Override public T visitDelimiter(PainlessParser.DelimiterContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitGeneric(PainlessParser.GenericContext ctx) { return visitChildren(ctx); }
+  @Override public T visitSingle(PainlessParser.SingleContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
@@ -186,13 +179,6 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitComp(PainlessParser.CompContext ctx) { return visitChildren(ctx); }
-  /**
-   * {@inheritDoc}
-   *
-   * <p>The default implementation returns the result of calling
-   * {@link #visitChildren} on {@code ctx}.</p>
-   */
-  @Override public T visitRead(PainlessParser.ReadContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
@@ -220,7 +206,28 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitFalse(PainlessParser.FalseContext ctx) { return visitChildren(ctx); }
+  @Override public T visitBinary(PainlessParser.BinaryContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitPre(PainlessParser.PreContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitPost(PainlessParser.PostContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitRead(PainlessParser.ReadContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
@@ -234,35 +241,14 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitUnary(PainlessParser.UnaryContext ctx) { return visitChildren(ctx); }
+  @Override public T visitTrue(PainlessParser.TrueContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitPrecedence(PainlessParser.PrecedenceContext ctx) { return visitChildren(ctx); }
-  /**
-   * {@inheritDoc}
-   *
-   * <p>The default implementation returns the result of calling
-   * {@link #visitChildren} on {@code ctx}.</p>
-   */
-  @Override public T visitPreinc(PainlessParser.PreincContext ctx) { return visitChildren(ctx); }
-  /**
-   * {@inheritDoc}
-   *
-   * <p>The default implementation returns the result of calling
-   * {@link #visitChildren} on {@code ctx}.</p>
-   */
-  @Override public T visitPostinc(PainlessParser.PostincContext ctx) { return visitChildren(ctx); }
-  /**
-   * {@inheritDoc}
-   *
-   * <p>The default implementation returns the result of calling
-   * {@link #visitChildren} on {@code ctx}.</p>
-   */
-  @Override public T visitCast(PainlessParser.CastContext ctx) { return visitChildren(ctx); }
+  @Override public T visitFalse(PainlessParser.FalseContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
@@ -276,84 +262,98 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitBinary(PainlessParser.BinaryContext ctx) { return visitChildren(ctx); }
+  @Override public T visitOperator(PainlessParser.OperatorContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitTrue(PainlessParser.TrueContext ctx) { return visitChildren(ctx); }
+  @Override public T visitCast(PainlessParser.CastContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitChain(PainlessParser.ChainContext ctx) { return visitChildren(ctx); }
+  @Override public T visitDynamic(PainlessParser.DynamicContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitLinkprec(PainlessParser.LinkprecContext ctx) { return visitChildren(ctx); }
+  @Override public T visitStatic(PainlessParser.StaticContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitLinkcast(PainlessParser.LinkcastContext ctx) { return visitChildren(ctx); }
+  @Override public T visitNewarray(PainlessParser.NewarrayContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitLinkbrace(PainlessParser.LinkbraceContext ctx) { return visitChildren(ctx); }
+  @Override public T visitExprprec(PainlessParser.ExprprecContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitLinkdot(PainlessParser.LinkdotContext ctx) { return visitChildren(ctx); }
+  @Override public T visitChainprec(PainlessParser.ChainprecContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitLinkcall(PainlessParser.LinkcallContext ctx) { return visitChildren(ctx); }
+  @Override public T visitString(PainlessParser.StringContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitLinkvar(PainlessParser.LinkvarContext ctx) { return visitChildren(ctx); }
+  @Override public T visitVariable(PainlessParser.VariableContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitLinkfield(PainlessParser.LinkfieldContext ctx) { return visitChildren(ctx); }
+  @Override public T visitNewobject(PainlessParser.NewobjectContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitLinknew(PainlessParser.LinknewContext ctx) { return visitChildren(ctx); }
+  @Override public T visitSecondary(PainlessParser.SecondaryContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
    * <p>The default implementation returns the result of calling
    * {@link #visitChildren} on {@code ctx}.</p>
    */
-  @Override public T visitLinkstring(PainlessParser.LinkstringContext ctx) { return visitChildren(ctx); }
+  @Override public T visitCallinvoke(PainlessParser.CallinvokeContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitFieldaccess(PainlessParser.FieldaccessContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitBraceaccess(PainlessParser.BraceaccessContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
index 7a3b594ab15..ab1254e1a91 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
@@ -94,31 +94,23 @@ interface PainlessParserVisitor<T> extends ParseTreeVisitor<T> {
    */
   T visitExpr(PainlessParser.ExprContext ctx);
   /**
-   * Visit a parse tree produced by the {@code multiple}
-   * labeled alternative in {@link PainlessParser#block}.
+   * Visit a parse tree produced by {@link PainlessParser#trailer}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitMultiple(PainlessParser.MultipleContext ctx);
+  T visitTrailer(PainlessParser.TrailerContext ctx);
   /**
-   * Visit a parse tree produced by the {@code single}
-   * labeled alternative in {@link PainlessParser#block}.
+   * Visit a parse tree produced by {@link PainlessParser#block}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitSingle(PainlessParser.SingleContext ctx);
+  T visitBlock(PainlessParser.BlockContext ctx);
   /**
    * Visit a parse tree produced by {@link PainlessParser#empty}.
    * @param ctx the parse tree
    * @return the visitor result
    */
   T visitEmpty(PainlessParser.EmptyContext ctx);
-  /**
-   * Visit a parse tree produced by {@link PainlessParser#emptyscope}.
-   * @param ctx the parse tree
-   * @return the visitor result
-   */
-  T visitEmptyscope(PainlessParser.EmptyscopeContext ctx);
   /**
    * Visit a parse tree produced by {@link PainlessParser#initializer}.
    * @param ctx the parse tree
@@ -156,17 +148,18 @@ interface PainlessParserVisitor<T> extends ParseTreeVisitor<T> {
    */
   T visitTrap(PainlessParser.TrapContext ctx);
   /**
-   * Visit a parse tree produced by {@link PainlessParser#identifier}.
+   * Visit a parse tree produced by {@link PainlessParser#delimiter}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitIdentifier(PainlessParser.IdentifierContext ctx);
+  T visitDelimiter(PainlessParser.DelimiterContext ctx);
   /**
-   * Visit a parse tree produced by {@link PainlessParser#generic}.
+   * Visit a parse tree produced by the {@code single}
+   * labeled alternative in {@link PainlessParser#expression}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitGeneric(PainlessParser.GenericContext ctx);
+  T visitSingle(PainlessParser.SingleContext ctx);
   /**
    * Visit a parse tree produced by the {@code comp}
    * labeled alternative in {@link PainlessParser#expression}.
@@ -174,13 +167,6 @@ interface PainlessParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitComp(PainlessParser.CompContext ctx);
-  /**
-   * Visit a parse tree produced by the {@code read}
-   * labeled alternative in {@link PainlessParser#expression}.
-   * @param ctx the parse tree
-   * @return the visitor result
-   */
-  T visitRead(PainlessParser.ReadContext ctx);
   /**
    * Visit a parse tree produced by the {@code bool}
    * labeled alternative in {@link PainlessParser#expression}.
@@ -202,62 +188,6 @@ interface PainlessParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitAssignment(PainlessParser.AssignmentContext ctx);
-  /**
-   * Visit a parse tree produced by the {@code false}
-   * labeled alternative in {@link PainlessParser#expression}.
-   * @param ctx the parse tree
-   * @return the visitor result
-   */
-  T visitFalse(PainlessParser.FalseContext ctx);
-  /**
-   * Visit a parse tree produced by the {@code numeric}
-   * labeled alternative in {@link PainlessParser#expression}.
-   * @param ctx the parse tree
-   * @return the visitor result
-   */
-  T visitNumeric(PainlessParser.NumericContext ctx);
-  /**
-   * Visit a parse tree produced by the {@code unary}
-   * labeled alternative in {@link PainlessParser#expression}.
-   * @param ctx the parse tree
-   * @return the visitor result
-   */
-  T visitUnary(PainlessParser.UnaryContext ctx);
-  /**
-   * Visit a parse tree produced by the {@code precedence}
-   * labeled alternative in {@link PainlessParser#expression}.
-   * @param ctx the parse tree
-   * @return the visitor result
-   */
-  T visitPrecedence(PainlessParser.PrecedenceContext ctx);
-  /**
-   * Visit a parse tree produced by the {@code preinc}
-   * labeled alternative in {@link PainlessParser#expression}.
-   * @param ctx the parse tree
-   * @return the visitor result
-   */
-  T visitPreinc(PainlessParser.PreincContext ctx);
-  /**
-   * Visit a parse tree produced by the {@code postinc}
-   * labeled alternative in {@link PainlessParser#expression}.
-   * @param ctx the parse tree
-   * @return the visitor result
-   */
-  T visitPostinc(PainlessParser.PostincContext ctx);
-  /**
-   * Visit a parse tree produced by the {@code cast}
-   * labeled alternative in {@link PainlessParser#expression}.
-   * @param ctx the parse tree
-   * @return the visitor result
-   */
-  T visitCast(PainlessParser.CastContext ctx);
-  /**
-   * Visit a parse tree produced by the {@code null}
-   * labeled alternative in {@link PainlessParser#expression}.
-   * @param ctx the parse tree
-   * @return the visitor result
-   */
-  T visitNull(PainlessParser.NullContext ctx);
   /**
    * Visit a parse tree produced by the {@code binary}
    * labeled alternative in {@link PainlessParser#expression}.
@@ -265,73 +195,152 @@ interface PainlessParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitBinary(PainlessParser.BinaryContext ctx);
+  /**
+   * Visit a parse tree produced by the {@code pre}
+   * labeled alternative in {@link PainlessParser#unary}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitPre(PainlessParser.PreContext ctx);
+  /**
+   * Visit a parse tree produced by the {@code post}
+   * labeled alternative in {@link PainlessParser#unary}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitPost(PainlessParser.PostContext ctx);
+  /**
+   * Visit a parse tree produced by the {@code read}
+   * labeled alternative in {@link PainlessParser#unary}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitRead(PainlessParser.ReadContext ctx);
+  /**
+   * Visit a parse tree produced by the {@code numeric}
+   * labeled alternative in {@link PainlessParser#unary}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitNumeric(PainlessParser.NumericContext ctx);
   /**
    * Visit a parse tree produced by the {@code true}
-   * labeled alternative in {@link PainlessParser#expression}.
+   * labeled alternative in {@link PainlessParser#unary}.
    * @param ctx the parse tree
    * @return the visitor result
    */
   T visitTrue(PainlessParser.TrueContext ctx);
   /**
-   * Visit a parse tree produced by {@link PainlessParser#chain}.
+   * Visit a parse tree produced by the {@code false}
+   * labeled alternative in {@link PainlessParser#unary}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitChain(PainlessParser.ChainContext ctx);
+  T visitFalse(PainlessParser.FalseContext ctx);
   /**
-   * Visit a parse tree produced by {@link PainlessParser#linkprec}.
+   * Visit a parse tree produced by the {@code null}
+   * labeled alternative in {@link PainlessParser#unary}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitLinkprec(PainlessParser.LinkprecContext ctx);
+  T visitNull(PainlessParser.NullContext ctx);
   /**
-   * Visit a parse tree produced by {@link PainlessParser#linkcast}.
+   * Visit a parse tree produced by the {@code operator}
+   * labeled alternative in {@link PainlessParser#unary}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitLinkcast(PainlessParser.LinkcastContext ctx);
+  T visitOperator(PainlessParser.OperatorContext ctx);
   /**
-   * Visit a parse tree produced by {@link PainlessParser#linkbrace}.
+   * Visit a parse tree produced by the {@code cast}
+   * labeled alternative in {@link PainlessParser#unary}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitLinkbrace(PainlessParser.LinkbraceContext ctx);
+  T visitCast(PainlessParser.CastContext ctx);
   /**
-   * Visit a parse tree produced by {@link PainlessParser#linkdot}.
+   * Visit a parse tree produced by the {@code dynamic}
+   * labeled alternative in {@link PainlessParser#chain}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitLinkdot(PainlessParser.LinkdotContext ctx);
+  T visitDynamic(PainlessParser.DynamicContext ctx);
   /**
-   * Visit a parse tree produced by {@link PainlessParser#linkcall}.
+   * Visit a parse tree produced by the {@code static}
+   * labeled alternative in {@link PainlessParser#chain}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitLinkcall(PainlessParser.LinkcallContext ctx);
+  T visitStatic(PainlessParser.StaticContext ctx);
   /**
-   * Visit a parse tree produced by {@link PainlessParser#linkvar}.
+   * Visit a parse tree produced by the {@code newarray}
+   * labeled alternative in {@link PainlessParser#chain}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitLinkvar(PainlessParser.LinkvarContext ctx);
+  T visitNewarray(PainlessParser.NewarrayContext ctx);
   /**
-   * Visit a parse tree produced by {@link PainlessParser#linkfield}.
+   * Visit a parse tree produced by the {@code exprprec}
+   * labeled alternative in {@link PainlessParser#primary}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitLinkfield(PainlessParser.LinkfieldContext ctx);
+  T visitExprprec(PainlessParser.ExprprecContext ctx);
   /**
-   * Visit a parse tree produced by {@link PainlessParser#linknew}.
+   * Visit a parse tree produced by the {@code chainprec}
+   * labeled alternative in {@link PainlessParser#primary}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitLinknew(PainlessParser.LinknewContext ctx);
+  T visitChainprec(PainlessParser.ChainprecContext ctx);
   /**
-   * Visit a parse tree produced by {@link PainlessParser#linkstring}.
+   * Visit a parse tree produced by the {@code string}
+   * labeled alternative in {@link PainlessParser#primary}.
    * @param ctx the parse tree
    * @return the visitor result
    */
-  T visitLinkstring(PainlessParser.LinkstringContext ctx);
+  T visitString(PainlessParser.StringContext ctx);
+  /**
+   * Visit a parse tree produced by the {@code variable}
+   * labeled alternative in {@link PainlessParser#primary}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitVariable(PainlessParser.VariableContext ctx);
+  /**
+   * Visit a parse tree produced by the {@code newobject}
+   * labeled alternative in {@link PainlessParser#primary}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitNewobject(PainlessParser.NewobjectContext ctx);
+  /**
+   * Visit a parse tree produced by {@link PainlessParser#secondary}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitSecondary(PainlessParser.SecondaryContext ctx);
+  /**
+   * Visit a parse tree produced by the {@code callinvoke}
+   * labeled alternative in {@link PainlessParser#dot}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitCallinvoke(PainlessParser.CallinvokeContext ctx);
+  /**
+   * Visit a parse tree produced by the {@code fieldaccess}
+   * labeled alternative in {@link PainlessParser#dot}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitFieldaccess(PainlessParser.FieldaccessContext ctx);
+  /**
+   * Visit a parse tree produced by the {@code braceaccess}
+   * labeled alternative in {@link PainlessParser#brace}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitBraceaccess(PainlessParser.BraceaccessContext ctx);
   /**
    * Visit a parse tree produced by {@link PainlessParser#arguments}.
    * @param ctx the parse tree
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index 9ecf6754248..3669ab24ecc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -20,8 +20,13 @@
 package org.elasticsearch.painless.antlr;
 
 import org.antlr.v4.runtime.ANTLRInputStream;
+import org.antlr.v4.runtime.BaseErrorListener;
 import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.DiagnosticErrorListener;
 import org.antlr.v4.runtime.ParserRuleContext;
+import org.antlr.v4.runtime.RecognitionException;
+import org.antlr.v4.runtime.Recognizer;
+import org.antlr.v4.runtime.atn.PredictionMode;
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Variables.Reserved;
@@ -29,51 +34,54 @@ import org.elasticsearch.painless.antlr.PainlessParser.AfterthoughtContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ArgumentsContext;
 import org.elasticsearch.painless.antlr.PainlessParser.AssignmentContext;
 import org.elasticsearch.painless.antlr.PainlessParser.BinaryContext;
+import org.elasticsearch.painless.antlr.PainlessParser.BlockContext;
+import org.elasticsearch.painless.antlr.PainlessParser.BoolContext;
+import org.elasticsearch.painless.antlr.PainlessParser.BraceaccessContext;
 import org.elasticsearch.painless.antlr.PainlessParser.BreakContext;
+import org.elasticsearch.painless.antlr.PainlessParser.CallinvokeContext;
 import org.elasticsearch.painless.antlr.PainlessParser.CastContext;
+import org.elasticsearch.painless.antlr.PainlessParser.ChainprecContext;
+import org.elasticsearch.painless.antlr.PainlessParser.CompContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ConditionalContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ContinueContext;
 import org.elasticsearch.painless.antlr.PainlessParser.DeclContext;
 import org.elasticsearch.painless.antlr.PainlessParser.DeclarationContext;
 import org.elasticsearch.painless.antlr.PainlessParser.DecltypeContext;
 import org.elasticsearch.painless.antlr.PainlessParser.DeclvarContext;
+import org.elasticsearch.painless.antlr.PainlessParser.DelimiterContext;
 import org.elasticsearch.painless.antlr.PainlessParser.DoContext;
+import org.elasticsearch.painless.antlr.PainlessParser.DynamicContext;
 import org.elasticsearch.painless.antlr.PainlessParser.EmptyContext;
-import org.elasticsearch.painless.antlr.PainlessParser.EmptyscopeContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ExprContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ExpressionContext;
-import org.elasticsearch.painless.antlr.PainlessParser.LinkbraceContext;
-import org.elasticsearch.painless.antlr.PainlessParser.LinkcallContext;
-import org.elasticsearch.painless.antlr.PainlessParser.LinkcastContext;
-import org.elasticsearch.painless.antlr.PainlessParser.LinkdotContext;
-import org.elasticsearch.painless.antlr.PainlessParser.ReadContext;
-import org.elasticsearch.painless.antlr.PainlessParser.LinkfieldContext;
-import org.elasticsearch.painless.antlr.PainlessParser.LinknewContext;
-import org.elasticsearch.painless.antlr.PainlessParser.LinkprecContext;
-import org.elasticsearch.painless.antlr.PainlessParser.ChainContext;
-import org.elasticsearch.painless.antlr.PainlessParser.LinkstringContext;
-import org.elasticsearch.painless.antlr.PainlessParser.LinkvarContext;
+import org.elasticsearch.painless.antlr.PainlessParser.ExprprecContext;
 import org.elasticsearch.painless.antlr.PainlessParser.FalseContext;
+import org.elasticsearch.painless.antlr.PainlessParser.FieldaccessContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ForContext;
-import org.elasticsearch.painless.antlr.PainlessParser.GenericContext;
-import org.elasticsearch.painless.antlr.PainlessParser.IdentifierContext;
 import org.elasticsearch.painless.antlr.PainlessParser.IfContext;
 import org.elasticsearch.painless.antlr.PainlessParser.InitializerContext;
-import org.elasticsearch.painless.antlr.PainlessParser.MultipleContext;
+import org.elasticsearch.painless.antlr.PainlessParser.NewarrayContext;
+import org.elasticsearch.painless.antlr.PainlessParser.NewobjectContext;
 import org.elasticsearch.painless.antlr.PainlessParser.NullContext;
 import org.elasticsearch.painless.antlr.PainlessParser.NumericContext;
-import org.elasticsearch.painless.antlr.PainlessParser.PostincContext;
-import org.elasticsearch.painless.antlr.PainlessParser.PrecedenceContext;
-import org.elasticsearch.painless.antlr.PainlessParser.PreincContext;
+import org.elasticsearch.painless.antlr.PainlessParser.OperatorContext;
+import org.elasticsearch.painless.antlr.PainlessParser.PostContext;
+import org.elasticsearch.painless.antlr.PainlessParser.PreContext;
+import org.elasticsearch.painless.antlr.PainlessParser.ReadContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ReturnContext;
+import org.elasticsearch.painless.antlr.PainlessParser.SecondaryContext;
 import org.elasticsearch.painless.antlr.PainlessParser.SingleContext;
 import org.elasticsearch.painless.antlr.PainlessParser.SourceContext;
 import org.elasticsearch.painless.antlr.PainlessParser.StatementContext;
+import org.elasticsearch.painless.antlr.PainlessParser.StaticContext;
+import org.elasticsearch.painless.antlr.PainlessParser.StringContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ThrowContext;
+import org.elasticsearch.painless.antlr.PainlessParser.TrailerContext;
 import org.elasticsearch.painless.antlr.PainlessParser.TrapContext;
 import org.elasticsearch.painless.antlr.PainlessParser.TrueContext;
 import org.elasticsearch.painless.antlr.PainlessParser.TryContext;
 import org.elasticsearch.painless.antlr.PainlessParser.UnaryContext;
+import org.elasticsearch.painless.antlr.PainlessParser.VariableContext;
 import org.elasticsearch.painless.antlr.PainlessParser.WhileContext;
 import org.elasticsearch.painless.node.AExpression;
 import org.elasticsearch.painless.node.ALink;
@@ -82,11 +90,11 @@ import org.elasticsearch.painless.node.AStatement;
 import org.elasticsearch.painless.node.EBinary;
 import org.elasticsearch.painless.node.EBool;
 import org.elasticsearch.painless.node.EBoolean;
-import org.elasticsearch.painless.node.EExplicit;
 import org.elasticsearch.painless.node.EChain;
 import org.elasticsearch.painless.node.EComp;
 import org.elasticsearch.painless.node.EConditional;
 import org.elasticsearch.painless.node.EDecimal;
+import org.elasticsearch.painless.node.EExplicit;
 import org.elasticsearch.painless.node.ENull;
 import org.elasticsearch.painless.node.ENumeric;
 import org.elasticsearch.painless.node.EUnary;
@@ -96,21 +104,23 @@ import org.elasticsearch.painless.node.LCast;
 import org.elasticsearch.painless.node.LField;
 import org.elasticsearch.painless.node.LNewArray;
 import org.elasticsearch.painless.node.LNewObj;
+import org.elasticsearch.painless.node.LStatic;
 import org.elasticsearch.painless.node.LString;
 import org.elasticsearch.painless.node.LVariable;
 import org.elasticsearch.painless.node.SBlock;
 import org.elasticsearch.painless.node.SBreak;
+import org.elasticsearch.painless.node.SCatch;
 import org.elasticsearch.painless.node.SContinue;
 import org.elasticsearch.painless.node.SDeclBlock;
 import org.elasticsearch.painless.node.SDeclaration;
 import org.elasticsearch.painless.node.SDo;
 import org.elasticsearch.painless.node.SExpression;
 import org.elasticsearch.painless.node.SFor;
+import org.elasticsearch.painless.node.SIf;
 import org.elasticsearch.painless.node.SIfElse;
 import org.elasticsearch.painless.node.SReturn;
 import org.elasticsearch.painless.node.SSource;
 import org.elasticsearch.painless.node.SThrow;
-import org.elasticsearch.painless.node.STrap;
 import org.elasticsearch.painless.node.STry;
 import org.elasticsearch.painless.node.SWhile;
 
@@ -120,7 +130,7 @@ import java.util.List;
 /**
  * Converts the ANTLR tree to a Painless tree.
  */
-public final class Walker extends PainlessParserBaseVisitor<ANode> {
+public final class Walker extends PainlessParserBaseVisitor<Object> {
 
     public static SSource buildPainlessTree(String source, Reserved reserved, CompilerSettings settings) {
         return new Walker(source, reserved, settings).source;
@@ -136,329 +146,292 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
         this.source = (SSource)visit(buildAntlrTree(source));
     }
 
-    private SourceContext buildAntlrTree(final String source) {
-        final ANTLRInputStream stream = new ANTLRInputStream(source);
-        final PainlessLexer lexer = new ErrorHandlingLexer(stream);
-        final PainlessParser parser = new PainlessParser(new CommonTokenStream(lexer));
-        final ParserErrorStrategy strategy = new ParserErrorStrategy();
+    private SourceContext buildAntlrTree(String source) {
+        ANTLRInputStream stream = new ANTLRInputStream(source);
+        PainlessLexer lexer = new ErrorHandlingLexer(stream);
+        PainlessParser parser = new PainlessParser(new CommonTokenStream(lexer));
+        ParserErrorStrategy strategy = new ParserErrorStrategy();
 
         lexer.removeErrorListeners();
         parser.removeErrorListeners();
+
+        if (settings.isPicky()) {
+            setupPicky(parser);
+        }
+
         parser.setErrorHandler(strategy);
 
         return parser.source();
     }
 
-    private int line(final ParserRuleContext ctx) {
+    private void setupPicky(PainlessParser parser) {
+        // Diagnostic listener invokes syntaxError on other listeners for ambiguity issues,
+        parser.addErrorListener(new DiagnosticErrorListener(true));
+        // a second listener to fail the test when the above happens.
+        parser.addErrorListener(new BaseErrorListener() {
+            @Override
+            public void syntaxError(final Recognizer<?,?> recognizer, final Object offendingSymbol, final int line,
+                                    final int charPositionInLine, final String msg, final RecognitionException e) {
+                throw new AssertionError("line: " + line + ", offset: " + charPositionInLine +
+                    ", symbol:" + offendingSymbol + " " + msg);
+            }
+        });
+
+        // Enable exact ambiguity detection (costly). we enable exact since its the default for
+        // DiagnosticErrorListener, life is too short to think about what 'inexact ambiguity' might mean.
+        parser.getInterpreter().setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION);
+    }
+
+    private int line(ParserRuleContext ctx) {
         return ctx.getStart().getLine();
     }
 
-    private String location(final ParserRuleContext ctx) {
+    private int offset(ParserRuleContext ctx) {
+        return ctx.getStart().getStartIndex();
+    }
+
+    private String location(ParserRuleContext ctx) {
         return "[ " + ctx.getStart().getLine() + " : " + ctx.getStart().getCharPositionInLine() + " ]";
     }
 
     @Override
-    public ANode visitSource(final SourceContext ctx) {
-        final List<AStatement> statements = new ArrayList<>();
+    public Object visitSource(SourceContext ctx) {
+        List<AStatement> statements = new ArrayList<>();
 
-        for (final StatementContext statement : ctx.statement()) {
+        for (StatementContext statement : ctx.statement()) {
             statements.add((AStatement)visit(statement));
         }
 
-        return new SSource(line(ctx), location(ctx), statements);
+        return new SSource(line(ctx), offset(ctx), location(ctx), statements);
     }
 
     @Override
-    public ANode visitIf(final IfContext ctx) {
-        final AExpression condition = (AExpression)visit(ctx.expression());
-        final AStatement ifblock = (AStatement)visit(ctx.block(0));
-        final AStatement elseblock = ctx.block(1) == null ? null : (AStatement)visit(ctx.block(1));
+    public Object visitIf(IfContext ctx) {
+        AExpression expression = (AExpression)visitExpression(ctx.expression());
+        SBlock ifblock = (SBlock)visit(ctx.trailer(0));
 
-        return new SIfElse(line(ctx), location(ctx), condition, ifblock, elseblock);
+        if (ctx.trailer().size() > 1) {
+            SBlock elseblock = (SBlock)visit(ctx.trailer(1));
+
+            return new SIfElse(line(ctx), offset(ctx), location(ctx), expression, ifblock, elseblock);
+        } else {
+            return new SIf(line(ctx), offset(ctx), location(ctx), expression, ifblock);
+        }
     }
 
     @Override
-    public ANode visitWhile(final WhileContext ctx) {
-        final AExpression condition = (AExpression)visit(ctx.expression());
-        final AStatement block = ctx.block() == null ? null : (AStatement)visit(ctx.block());
+    public Object visitWhile(WhileContext ctx) {
+        if (settings.getMaxLoopCounter() > 0) {
+            reserved.usesLoop();
+        }
 
-        reserved.usesLoop();
+        AExpression expression = (AExpression)visitExpression(ctx.expression());
 
-        return new SWhile(line(ctx), location(ctx), condition, block, settings.getMaxLoopCounter());
+        if (ctx.trailer() != null) {
+            SBlock block = (SBlock)visit(ctx.trailer());
+
+            return new SWhile(line(ctx), offset(ctx), location(ctx), settings.getMaxLoopCounter(), expression, block);
+        } else if (ctx.empty() != null) {
+            return new SWhile(line(ctx), offset(ctx), location(ctx), settings.getMaxLoopCounter(), expression, null);
+        } else {
+            throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
+        }
     }
 
     @Override
-    public ANode visitDo(final DoContext ctx) {
-        final AStatement block = ctx.block() == null ? null : (AStatement)visit(ctx.block());
-        final AExpression condition = (AExpression)visit(ctx.expression());
+    public Object visitDo(DoContext ctx) {
+        if (settings.getMaxLoopCounter() > 0) {
+            reserved.usesLoop();
+        }
 
-        reserved.usesLoop();
+        AExpression expression = (AExpression)visitExpression(ctx.expression());
+        SBlock block = (SBlock)visit(ctx.block());
 
-        return new SDo(line(ctx), location(ctx), block, condition, settings.getMaxLoopCounter());
+        return new SDo(line(ctx), offset(ctx), location(ctx), settings.getMaxLoopCounter(), block, expression);
     }
 
     @Override
-    public ANode visitFor(final ForContext ctx) {
-        final ANode intializer = ctx.initializer() == null ? null : visit(ctx.initializer());
-        final AExpression condition = ctx.expression() == null ? null : (AExpression)visit(ctx.expression());
-        final AExpression afterthought = ctx.afterthought() == null ? null : (AExpression)visit(ctx.afterthought());
-        final AStatement block = ctx.block() == null ? null : (AStatement)visit(ctx.block());
+    public Object visitFor(ForContext ctx) {
+        if (settings.getMaxLoopCounter() > 0) {
+            reserved.usesLoop();
+        }
 
-        reserved.usesLoop();
+        ANode initializer = ctx.initializer() == null ? null : (ANode)visit(ctx.initializer());
+        AExpression expression = ctx.expression() == null ? null : (AExpression)visitExpression(ctx.expression());
+        AExpression afterthought = ctx.afterthought() == null ? null : (AExpression)visit(ctx.afterthought());
 
-        return new SFor(line(ctx), location(ctx), intializer, condition, afterthought, block, settings.getMaxLoopCounter());
+        if (ctx.trailer() != null) {
+            SBlock block = (SBlock)visit(ctx.trailer());
+
+            return new SFor(line(ctx), offset(ctx), location(ctx),
+                settings.getMaxLoopCounter(), initializer, expression, afterthought, block);
+        } else if (ctx.empty() != null) {
+            return new SFor(line(ctx), offset(ctx), location(ctx),
+                settings.getMaxLoopCounter(), initializer, expression, afterthought, null);
+        } else {
+            throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
+        }
     }
 
     @Override
-    public ANode visitDecl(final DeclContext ctx) {
+    public Object visitDecl(DeclContext ctx) {
         return visit(ctx.declaration());
     }
 
     @Override
-    public ANode visitContinue(final ContinueContext ctx) {
-        return new SContinue(line(ctx), location(ctx));
+    public Object visitContinue(ContinueContext ctx) {
+        return new SContinue(line(ctx), offset(ctx), location(ctx));
     }
 
     @Override
-    public ANode visitBreak(final BreakContext ctx) {
-        return new SBreak(line(ctx), location(ctx));
+    public Object visitBreak(BreakContext ctx) {
+        return new SBreak(line(ctx), offset(ctx), location(ctx));
     }
 
     @Override
-    public ANode visitReturn(final ReturnContext ctx) {
-        final AExpression expression = (AExpression)visit(ctx.expression());
+    public Object visitReturn(ReturnContext ctx) {
+        AExpression expression = (AExpression)visitExpression(ctx.expression());
 
-        return new SReturn(line(ctx), location(ctx), expression);
+        return new SReturn(line(ctx), offset(ctx), location(ctx), expression);
     }
 
     @Override
-    public ANode visitTry(final TryContext ctx) {
-        final AStatement block = (AStatement)visit(ctx.block());
-        final List<STrap> traps = new ArrayList<>();
+    public Object visitTry(TryContext ctx) {
+        SBlock block = (SBlock)visit(ctx.block());
+        List<SCatch> catches = new ArrayList<>();
 
-        for (final TrapContext trap : ctx.trap()) {
-            traps.add((STrap)visit(trap));
+        for (TrapContext trap : ctx.trap()) {
+            catches.add((SCatch)visit(trap));
         }
 
-        return new STry(line(ctx), location(ctx), block, traps);
+        return new STry(line(ctx), offset(ctx), location(ctx), block, catches);
     }
 
     @Override
-    public ANode visitThrow(final ThrowContext ctx) {
-        final AExpression expression = (AExpression)visit(ctx.expression());
+    public Object visitThrow(ThrowContext ctx) {
+        AExpression expression = (AExpression)visitExpression(ctx.expression());
 
-        return new SThrow(line(ctx), location(ctx), expression);
+        return new SThrow(line(ctx), offset(ctx), location(ctx), expression);
     }
 
     @Override
-    public ANode visitExpr(final ExprContext ctx) {
-        final AExpression expression = (AExpression)visit(ctx.expression());
+    public Object visitExpr(ExprContext ctx) {
+        AExpression expression = (AExpression)visitExpression(ctx.expression());
 
-        return new SExpression(line(ctx), location(ctx), expression);
+        return new SExpression(line(ctx), offset(ctx), location(ctx), expression);
     }
 
     @Override
-    public ANode visitMultiple(final MultipleContext ctx) {
-        final List<AStatement> statements = new ArrayList<>();
+    public Object visitTrailer(TrailerContext ctx) {
+        if (ctx.block() != null) {
+            return visit(ctx.block());
+        } else if (ctx.statement() != null) {
+            List<AStatement> statements = new ArrayList<>();
+            statements.add((AStatement)visit(ctx.statement()));
 
-        for (final StatementContext statement : ctx.statement()) {
-            statements.add((AStatement)visit(statement));
+            return new SBlock(line(ctx), offset(ctx), location(ctx), statements);
+        } else {
+            throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
         }
-
-        return new SBlock(line(ctx), location(ctx), statements);
     }
 
     @Override
-    public ANode visitSingle(final SingleContext ctx) {
-        final List<AStatement> statements = new ArrayList<>();
-        statements.add((AStatement)visit(ctx.statement()));
+    public Object visitBlock(BlockContext ctx) {
+        if (ctx.statement().isEmpty()) {
+            return null;
+        } else {
+            List<AStatement> statements = new ArrayList<>();
 
-        return new SBlock(line(ctx), location(ctx), statements);
+            for (StatementContext statement : ctx.statement()) {
+                statements.add((AStatement)visit(statement));
+            }
+
+            return new SBlock(line(ctx), offset(ctx), location(ctx), statements);
+        }
     }
 
     @Override
-    public ANode visitEmpty(final EmptyContext ctx) {
-        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    public Object visitEmpty(EmptyContext ctx) {
+        throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
     }
 
     @Override
-    public ANode visitEmptyscope(final EmptyscopeContext ctx) {
-        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
-    }
-
-    @Override
-    public ANode visitInitializer(final InitializerContext ctx) {
+    public Object visitInitializer(InitializerContext ctx) {
         if (ctx.declaration() != null) {
             return visit(ctx.declaration());
         } else if (ctx.expression() != null) {
-            return visit(ctx.expression());
-        }
-
-        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
-    }
-
-    @Override
-    public ANode visitAfterthought(final AfterthoughtContext ctx) {
-        return visit(ctx.expression());
-    }
-
-    @Override
-    public ANode visitDeclaration(final DeclarationContext ctx) {
-        final String type = ctx.decltype().getText();
-        final List<SDeclaration> declarations = new ArrayList<>();
-
-        for (final DeclvarContext declvar : ctx.declvar()) {
-            final String name = declvar.identifier().getText();
-            final AExpression expression = declvar.expression() == null ? null : (AExpression)visit(declvar.expression());
-            declarations.add(new SDeclaration(line(ctx), location(ctx), type, name, expression));
-        }
-
-        return new SDeclBlock(line(ctx), location(ctx), declarations);
-    }
-
-    @Override
-    public ANode visitDecltype(final DecltypeContext ctx) {
-        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
-    }
-
-    @Override
-    public ANode visitDeclvar(final DeclvarContext ctx) {
-        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
-    }
-
-    @Override
-    public ANode visitTrap(final TrapContext ctx) {
-        final String type = ctx.identifier(0).getText();
-        final String name = ctx.identifier(1).getText();
-        final AStatement block = ctx.block() == null ? null : (AStatement)visit(ctx.block());
-
-        return new STrap(line(ctx), location(ctx), type, name, block);
-    }
-
-    @Override
-    public ANode visitIdentifier(final IdentifierContext ctx) {
-        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
-    }
-
-    @Override
-    public ANode visitGeneric(final GenericContext ctx) {
-        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
-    }
-
-    @Override
-    public ANode visitPrecedence(final PrecedenceContext ctx) {
-        return visit(ctx.expression());
-    }
-
-    @Override
-    public ANode visitNumeric(final NumericContext ctx) {
-        final boolean negate = ctx.parent instanceof UnaryContext && ((UnaryContext)ctx.parent).SUB() != null;
-
-        if (ctx.DECIMAL() != null) {
-            return new EDecimal(line(ctx), location(ctx), (negate ? "-" : "") + ctx.DECIMAL().getText());
-        } else if (ctx.HEX() != null) {
-            return new ENumeric(line(ctx), location(ctx), (negate ? "-" : "") + ctx.HEX().getText().substring(2), 16);
-        } else if (ctx.INTEGER() != null) {
-            return new ENumeric(line(ctx), location(ctx), (negate ? "-" : "") + ctx.INTEGER().getText(), 10);
-        } else if (ctx.OCTAL() != null) {
-            return new ENumeric(line(ctx), location(ctx), (negate ? "-" : "") + ctx.OCTAL().getText().substring(1), 8);
+            return visitExpression(ctx.expression());
         } else {
-            throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+            throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
         }
     }
 
     @Override
-    public ANode visitTrue(final TrueContext ctx) {
-        return new EBoolean(line(ctx), location(ctx), true);
+    public Object visitAfterthought(AfterthoughtContext ctx) {
+        return visitExpression(ctx.expression());
     }
 
     @Override
-    public ANode visitFalse(FalseContext ctx) {
-        return new EBoolean(line(ctx), location(ctx), false);
+    public Object visitDeclaration(DeclarationContext ctx) {
+        String type = ctx.decltype().getText();
+        List<SDeclaration> declarations = new ArrayList<>();
+
+        for (DeclvarContext declvar : ctx.declvar()) {
+            String name = declvar.ID().getText();
+            AExpression expression = declvar.expression() == null ? null : (AExpression)visitExpression(declvar.expression());
+
+            declarations.add(new SDeclaration(line(ctx), offset(ctx), location(ctx), type, name, expression));
+        }
+
+        return new SDeclBlock(line(ctx), offset(ctx), location(ctx), declarations);
     }
 
     @Override
-    public ANode visitNull(final NullContext ctx) {
-        return new ENull(line(ctx), location(ctx));
+    public Object visitDecltype(DecltypeContext ctx) {
+        throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
     }
 
     @Override
-    public ANode visitPostinc(final PostincContext ctx) {
-        final List<ALink> links = new ArrayList<>();
-        final Operation operation;
+    public Object visitDeclvar(DeclvarContext ctx) {
+        throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
+    }
 
-        visitChain(ctx.chain(), links);
+    @Override
+    public Object visitTrap(TrapContext ctx) {
+        String type = ctx.TYPE().getText();
+        String name = ctx.ID().getText();
+        SBlock block = (SBlock)visit(ctx.block());
 
-        if (ctx.INCR() != null) {
-            operation = Operation.INCR;
-        } else if (ctx.DECR() != null) {
-            operation = Operation.DECR;
+        return new SCatch(line(ctx), offset(ctx), location(ctx), type, name, block);
+    }
+
+    @Override
+    public Object visitDelimiter(DelimiterContext ctx) {
+        throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
+    }
+
+    private Object visitExpression(ExpressionContext ctx) {
+        Object expression = visit(ctx);
+
+        if (expression instanceof List) {
+            @SuppressWarnings("unchecked")
+            List<ALink> links = (List<ALink>)expression;
+
+            return new EChain(line(ctx), offset(ctx), location(ctx), links, false, false, null, null);
         } else {
-            throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
-        }
-
-        return new EChain(line(ctx), location(ctx), links, false, true, operation, null);
-    }
-
-    @Override
-    public ANode visitPreinc(final PreincContext ctx) {
-        final List<ALink> links = new ArrayList<>();
-        final Operation operation;
-
-        visitChain(ctx.chain(), links);
-
-        if (ctx.INCR() != null) {
-            operation = Operation.INCR;
-        } else if (ctx.DECR() != null) {
-            operation = Operation.DECR;
-        } else {
-            throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
-        }
-
-        return new EChain(line(ctx), location(ctx), links, true, false, operation, null);
-    }
-
-    @Override
-    public ANode visitRead(final ReadContext ctx) {
-        final List<ALink> links = new ArrayList<>();
-
-        visitChain(ctx.chain(), links);
-
-        return new EChain(line(ctx), location(ctx), links, false, false, null, null);
-    }
-
-    @Override
-    public ANode visitUnary(final UnaryContext ctx) {
-        if (ctx.SUB() != null && ctx.expression() instanceof NumericContext) {
-            return visit(ctx.expression());
-        } else {
-            final Operation operation;
-
-            if (ctx.BOOLNOT() != null) {
-                operation = Operation.NOT;
-            } else if (ctx.BWNOT() != null) {
-                operation = Operation.BWNOT;
-            } else if (ctx.ADD() != null) {
-                operation = Operation.ADD;
-            } else if (ctx.SUB() != null) {
-                operation = Operation.SUB;
-            } else {
-                throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
-            }
-
-            return new EUnary(line(ctx), location(ctx), operation, (AExpression)visit(ctx.expression()));
+            return expression;
         }
     }
 
     @Override
-    public ANode visitCast(final CastContext ctx) {
-        return new EExplicit(line(ctx), location(ctx), ctx.decltype().getText(), (AExpression)visit(ctx.expression()));
+    public Object visitSingle(SingleContext ctx) {
+        return visit(ctx.unary());
     }
 
     @Override
-    public ANode visitBinary(final BinaryContext ctx) {
-        final AExpression left = (AExpression)visit(ctx.expression(0));
-        final AExpression right = (AExpression)visit(ctx.expression(1));
+    public Object visitBinary(BinaryContext ctx) {
+        AExpression left = (AExpression)visitExpression(ctx.expression(0));
+        AExpression right = (AExpression)visitExpression(ctx.expression(1));
         final Operation operation;
 
         if (ctx.MUL() != null) {
@@ -487,13 +460,13 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
             throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
         }
 
-        return new EBinary(line(ctx), location(ctx), operation, left, right);
+        return new EBinary(line(ctx), offset(ctx), location(ctx), operation, left, right);
     }
 
     @Override
-    public ANode visitComp(PainlessParser.CompContext ctx) {
-        final AExpression left = (AExpression)visit(ctx.expression(0));
-        final AExpression right = (AExpression)visit(ctx.expression(1));
+    public Object visitComp(CompContext ctx) {
+        AExpression left = (AExpression)visitExpression(ctx.expression(0));
+        AExpression right = (AExpression)visitExpression(ctx.expression(1));
         final Operation operation;
 
         if (ctx.LT() != null) {
@@ -516,13 +489,13 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
             throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
         }
 
-        return new EComp(line(ctx), location(ctx), operation, left, right);
+        return new EComp(line(ctx), offset(ctx), location(ctx), operation, left, right);
     }
 
     @Override
-    public ANode visitBool(PainlessParser.BoolContext ctx) {
-        final AExpression left = (AExpression)visit(ctx.expression(0));
-        final AExpression right = (AExpression)visit(ctx.expression(1));
+    public Object visitBool(BoolContext ctx) {
+        AExpression left = (AExpression)visitExpression(ctx.expression(0));
+        AExpression right = (AExpression)visitExpression(ctx.expression(1));
         final Operation operation;
 
         if (ctx.BOOLAND() != null) {
@@ -533,27 +506,27 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
             throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
         }
 
-        return new EBool(line(ctx), location(ctx), operation, left, right);
-    }
-
-
-    @Override
-    public ANode visitConditional(final ConditionalContext ctx) {
-        final AExpression condition = (AExpression)visit(ctx.expression(0));
-        final AExpression left = (AExpression)visit(ctx.expression(1));
-        final AExpression right = (AExpression)visit(ctx.expression(2));
-
-        return new EConditional(line(ctx), location(ctx), condition, left, right);
+        return new EBool(line(ctx), offset(ctx), location(ctx), operation, left, right);
     }
 
     @Override
-    public ANode visitAssignment(final AssignmentContext ctx) {
-        final List<ALink> links = new ArrayList<>();
+    public Object visitConditional(ConditionalContext ctx) {
+        AExpression condition = (AExpression)visitExpression(ctx.expression(0));
+        AExpression left = (AExpression)visitExpression(ctx.expression(1));
+        AExpression right = (AExpression)visitExpression(ctx.expression(2));
+
+        return new EConditional(line(ctx), offset(ctx), location(ctx), condition, left, right);
+    }
+
+    @Override
+    public Object visitAssignment(AssignmentContext ctx) {
+        @SuppressWarnings("unchecked")
+        List<ALink> links = (List<ALink>)visit(ctx.chain());
         final Operation operation;
 
-        visitChain(ctx.chain(), links);
-
-        if (ctx.AMUL() != null) {
+        if (ctx.ASSIGN() != null) {
+            operation = null;
+        } else if (ctx.AMUL() != null) {
             operation = Operation.MUL;
         } else if (ctx.ADIV() != null) {
             operation = Operation.DIV;
@@ -576,222 +549,290 @@ public final class Walker extends PainlessParserBaseVisitor<ANode> {
         } else if (ctx.AOR() != null) {
             operation = Operation.BWOR;
         } else {
-            operation = null;
+            throw new IllegalStateException("Error " + location(ctx) + ": Illegal tree structure.");
         }
 
-        return new EChain(line(ctx), location(ctx), links, false, false, operation, (AExpression)visit(ctx.expression()));
+        AExpression expression = (AExpression)visitExpression(ctx.expression());
+
+        return new EChain(line(ctx), offset(ctx), location(ctx), links, false, false, operation, expression);
     }
 
-    private void visitChain(final ChainContext ctx, final List<ALink> links) {
-        if (ctx.linkprec() != null) {
-            visitLinkprec(ctx.linkprec(), links);
-        } else if (ctx.linkcast() != null) {
-            visitLinkcast(ctx.linkcast(), links);
-        } else if (ctx.linkvar() != null) {
-            visitLinkvar(ctx.linkvar(), links);
-        } else if (ctx.linknew() != null) {
-            visitLinknew(ctx.linknew(), links);
-        } else if (ctx.linkstring() != null) {
-            visitLinkstring(ctx.linkstring(), links);
+    private Object visitUnary(UnaryContext ctx) {
+        Object expression = visit(ctx);
+
+        if (expression instanceof List) {
+            @SuppressWarnings("unchecked")
+            List<ALink> links = (List<ALink>)expression;
+
+            return new EChain(line(ctx), offset(ctx), location(ctx), links, false, false, null, null);
         } else {
-            throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+            return expression;
         }
     }
 
     @Override
-    public ANode visitChain(final ChainContext ctx) {
-        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
-    }
+    public Object visitPre(PreContext ctx) {
+        @SuppressWarnings("unchecked")
+        List<ALink> links = (List<ALink>)visit(ctx.chain());
+        final Operation operation;
 
-    private void visitLinkprec(final LinkprecContext ctx, final List<ALink> links) {
-        if (ctx.linkprec() != null) {
-            visitLinkprec(ctx.linkprec(), links);
-        } else if (ctx.linkcast() != null) {
-            visitLinkcast(ctx.linkcast(), links);
-        } else if (ctx.linkvar() != null) {
-            visitLinkvar(ctx.linkvar(), links);
-        } else if (ctx.linknew() != null) {
-            visitLinknew(ctx.linknew(), links);
-        } else if (ctx.linkstring() != null) {
-            visitLinkstring(ctx.linkstring(), links);
+        if (ctx.INCR() != null) {
+            operation = Operation.INCR;
+        } else if (ctx.DECR() != null) {
+            operation = Operation.DECR;
         } else {
-            throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+            throw new IllegalStateException("Error " + location(ctx) + ": Illegal tree structure.");
         }
 
-        if (ctx.linkbrace() != null) {
-            visitLinkbrace(ctx.linkbrace(), links);
-        } else if (ctx.linkdot() != null) {
-            visitLinkdot(ctx.linkdot(), links);
-        }
+        return new EChain(line(ctx), offset(ctx), location(ctx), links, true, false, operation, null);
     }
 
     @Override
-    public ANode visitLinkprec(final LinkprecContext ctx) {
-        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
-    }
+    public Object visitPost(PostContext ctx) {
+        @SuppressWarnings("unchecked")
+        List<ALink> links = (List<ALink>)visit(ctx.chain());
+        final Operation operation;
 
-    private void visitLinkcast(final LinkcastContext ctx, final List<ALink> links) {
-        if (ctx.linkprec() != null) {
-            visitLinkprec(ctx.linkprec(), links);
-        } else if (ctx.linkcast() != null) {
-            visitLinkcast(ctx.linkcast(), links);
-        } else if (ctx.linkvar() != null) {
-            visitLinkvar(ctx.linkvar(), links);
-        } else if (ctx.linknew() != null) {
-            visitLinknew(ctx.linknew(), links);
-        } else if (ctx.linkstring() != null) {
-            visitLinkstring(ctx.linkstring(), links);
+        if (ctx.INCR() != null) {
+            operation = Operation.INCR;
+        } else if (ctx.DECR() != null) {
+            operation = Operation.DECR;
         } else {
-            throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+            throw new IllegalStateException("Error " + location(ctx) + ": Illegal tree structure.");
         }
 
-        links.add(new LCast(line(ctx), location(ctx), ctx.decltype().getText()));
+        return new EChain(line(ctx), offset(ctx), location(ctx), links, false, true, operation, null);
     }
 
     @Override
-    public ANode visitLinkcast(final LinkcastContext ctx) {
-        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    public Object visitRead(ReadContext ctx) {
+        return visit(ctx.chain());
     }
 
-    private void visitLinkbrace(final LinkbraceContext ctx, final List<ALink> links) {
-        links.add(new LBrace(line(ctx), location(ctx), (AExpression)visit(ctx.expression())));
+    @Override
+    public Object visitNumeric(NumericContext ctx) {
+        final boolean negate = ctx.parent instanceof OperatorContext && ((OperatorContext)ctx.parent).SUB() != null;
 
-        if (ctx.linkbrace() != null) {
-            visitLinkbrace(ctx.linkbrace(), links);
-        } else if (ctx.linkdot() != null) {
-            visitLinkdot(ctx.linkdot(), links);
+        if (ctx.DECIMAL() != null) {
+            return new EDecimal(line(ctx), offset(ctx), location(ctx), (negate ? "-" : "") + ctx.DECIMAL().getText());
+        } else if (ctx.HEX() != null) {
+            return new ENumeric(line(ctx), offset(ctx), location(ctx), (negate ? "-" : "") + ctx.HEX().getText().substring(2), 16);
+        } else if (ctx.INTEGER() != null) {
+            return new ENumeric(line(ctx), offset(ctx), location(ctx), (negate ? "-" : "") + ctx.INTEGER().getText(), 10);
+        } else if (ctx.OCTAL() != null) {
+            return new ENumeric(line(ctx), offset(ctx), location(ctx), (negate ? "-" : "") + ctx.OCTAL().getText().substring(1), 8);
+        } else {
+            throw new IllegalStateException("Error " + location(ctx) + ": Illegal tree structure.");
         }
     }
 
     @Override
-    public ANode visitLinkbrace(final LinkbraceContext ctx) {
-        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    public Object visitTrue(TrueContext ctx) {
+        return new EBoolean(line(ctx), offset(ctx), location(ctx), true);
     }
 
-    private void visitLinkdot(final LinkdotContext ctx, final List<ALink> links) {
-        if (ctx.linkcall() != null) {
-            visitLinkcall(ctx.linkcall(), links);
-        } else if (ctx.linkfield() != null) {
-            visitLinkfield(ctx.linkfield(), links);
+    @Override
+    public Object visitFalse(FalseContext ctx) {
+        return new EBoolean(line(ctx), offset(ctx), location(ctx), false);
+    }
+
+    @Override
+    public Object visitNull(NullContext ctx) {
+        return new ENull(line(ctx), offset(ctx), location(ctx));
+    }
+
+    @Override
+    public Object visitOperator(OperatorContext ctx) {
+        if (ctx.SUB() != null && ctx.unary() instanceof NumericContext) {
+            return visit(ctx.unary());
+        } else {
+            AExpression expression = (AExpression)visitUnary(ctx.unary());
+            final Operation operation;
+
+            if (ctx.BOOLNOT() != null) {
+                operation = Operation.NOT;
+            } else if (ctx.BWNOT() != null) {
+                operation = Operation.BWNOT;
+            } else if (ctx.ADD() != null) {
+                operation = Operation.ADD;
+            } else if (ctx.SUB() != null) {
+                operation = Operation.SUB;
+            } else {
+                throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
+            }
+
+            return new EUnary(line(ctx), offset(ctx), location(ctx), operation, expression);
         }
     }
 
     @Override
-    public ANode visitLinkdot(final LinkdotContext ctx) {
-        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
-    }
+    public Object visitCast(CastContext ctx) {
+        String type = ctx.decltype().getText();
+        Object child = visit(ctx.unary());
 
-    private void visitLinkcall(final LinkcallContext ctx, final List<ALink> links) {
-        final List<AExpression> arguments = new ArrayList<>();
+        if (child instanceof List) {
+            @SuppressWarnings("unchecked")
+            List<ALink> links = (List<ALink>)child;
+            links.add(new LCast(line(ctx), offset(ctx), location(ctx), type));
 
-        for (final ExpressionContext expression : ctx.arguments().expression()) {
-            arguments.add((AExpression)visit(expression));
-        }
-
-        links.add(new LCall(line(ctx), location(ctx), ctx.EXTID().getText(), arguments));
-
-        if (ctx.linkbrace() != null) {
-            visitLinkbrace(ctx.linkbrace(), links);
-        } else if (ctx.linkdot() != null) {
-            visitLinkdot(ctx.linkdot(), links);
+            return links;
+        } else {
+            return new EExplicit(line(ctx), offset(ctx), location(ctx), type, (AExpression)child);
         }
     }
 
     @Override
-    public ANode visitLinkcall(final LinkcallContext ctx) {
-        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    public Object visitDynamic(DynamicContext ctx) {
+        Object child = visit(ctx.primary());
+
+        if (child instanceof List) {
+            @SuppressWarnings("unchecked")
+            List<ALink> links = (List<ALink>)child;
+
+            for (SecondaryContext secondary : ctx.secondary()) {
+                links.add((ALink)visit(secondary));
+            }
+
+            return links;
+        } else if (!ctx.secondary().isEmpty()) {
+            throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
+        } else {
+            return child;
+        }
     }
 
-    private void visitLinkvar(final LinkvarContext ctx, final List<ALink> links) {
-        final String name = ctx.identifier().getText();
+    @Override
+    public Object visitStatic(StaticContext ctx) {
+        String type = ctx.decltype().getText();
+        List<ALink> links = new ArrayList<>();
+
+        links.add(new LStatic(line(ctx), offset(ctx), location(ctx), type));
+        links.add((ALink)visit(ctx.dot()));
+
+        for (SecondaryContext secondary : ctx.secondary()) {
+            links.add((ALink)visit(secondary));
+        }
+
+        return links;
+    }
+
+    @Override
+    public Object visitNewarray(NewarrayContext ctx) {
+        String type = ctx.TYPE().getText();
+        List<AExpression> expressions = new ArrayList<>();
+
+        for (ExpressionContext expression : ctx.expression()) {
+            expressions.add((AExpression)visitExpression(expression));
+        }
+
+        List<ALink> links = new ArrayList<>();
+        links.add(new LNewArray(line(ctx), offset(ctx), location(ctx), type, expressions));
+
+        if (ctx.dot() != null) {
+            links.add((ALink)visit(ctx.dot()));
+
+            for (SecondaryContext secondary : ctx.secondary()) {
+                links.add((ALink)visit(secondary));
+            }
+        } else if (!ctx.secondary().isEmpty()) {
+            throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
+        }
+
+        return links;
+    }
+
+    @Override
+    public Object visitExprprec(ExprprecContext ctx) {
+        return visit(ctx.expression());
+    }
+
+    @Override
+    public Object visitChainprec(ChainprecContext ctx) {
+        return visit(ctx.unary());
+    }
+
+    @Override
+    public Object visitString(StringContext ctx) {
+        String string = ctx.STRING().getText().substring(1, ctx.STRING().getText().length() - 1);
+        List<ALink> links = new ArrayList<>();
+        links.add(new LString(line(ctx), offset(ctx), location(ctx), string));
+
+        return links;
+    }
+
+    @Override
+    public Object visitVariable(VariableContext ctx) {
+        String name = ctx.ID().getText();
+        List<ALink> links = new ArrayList<>();
+        links.add(new LVariable(line(ctx), offset(ctx), location(ctx), name));
 
         reserved.markReserved(name);
 
-        links.add(new LVariable(line(ctx), location(ctx), name));
+        return links;
+    }
 
-        if (ctx.linkbrace() != null) {
-            visitLinkbrace(ctx.linkbrace(), links);
-        } else if (ctx.linkdot() != null) {
-            visitLinkdot(ctx.linkdot(), links);
+    @Override
+    public Object visitNewobject(NewobjectContext ctx) {
+        String type = ctx.TYPE().getText();
+        List<AExpression> arguments = new ArrayList<>();
+
+        for (ExpressionContext expression : ctx.arguments().expression()) {
+            arguments.add((AExpression)visitExpression(expression));
+        }
+
+        List<ALink> links = new ArrayList<>();
+        links.add(new LNewObj(line(ctx), offset(ctx), location(ctx), type, arguments));
+
+        return links;
+    }
+
+    @Override
+    public Object visitSecondary(SecondaryContext ctx) {
+        if (ctx.dot() != null) {
+            return visit(ctx.dot());
+        } else if (ctx.brace() != null) {
+            return visit(ctx.brace());
+        } else {
+            throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
         }
     }
 
     @Override
-    public ANode visitLinkvar(final LinkvarContext ctx) {
-        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    public Object visitCallinvoke(CallinvokeContext ctx) {
+        String name = ctx.DOTID().getText();
+        List<AExpression> arguments = new ArrayList<>();
+
+        for (ExpressionContext expression : ctx.arguments().expression()) {
+            arguments.add((AExpression)visitExpression(expression));
+        }
+
+        return new LCall(line(ctx), offset(ctx), location(ctx), name, arguments);
     }
 
-    private void visitLinkfield(final LinkfieldContext ctx, final List<ALink> links) {
+    @Override
+    public Object visitFieldaccess(FieldaccessContext ctx) {
         final String value;
 
-        if (ctx.EXTID() != null) {
-            value = ctx.EXTID().getText();
-        } else if (ctx.EXTINTEGER() != null) {
-            value = ctx.EXTINTEGER().getText();
+        if (ctx.DOTID() != null) {
+            value = ctx.DOTID().getText();
+        } else if (ctx.DOTINTEGER() != null) {
+            value = ctx.DOTINTEGER().getText();
         } else {
-            throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+            throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
         }
 
-        links.add(new LField(line(ctx), location(ctx), value));
-
-        if (ctx.linkbrace() != null) {
-            visitLinkbrace(ctx.linkbrace(), links);
-        } else if (ctx.linkdot() != null) {
-            visitLinkdot(ctx.linkdot(), links);
-        }
+        return new LField(line(ctx), offset(ctx), location(ctx), value);
     }
 
     @Override
-    public ANode visitLinkfield(final LinkfieldContext ctx) {
-        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
-    }
+    public Object visitBraceaccess(BraceaccessContext ctx) {
+        AExpression expression = (AExpression)visitExpression(ctx.expression());
 
-    private void visitLinknew(final LinknewContext ctx, final List<ALink> links) {
-        final List<AExpression> arguments = new ArrayList<>();
-
-        if (ctx.arguments() != null) {
-            for (final ExpressionContext expression : ctx.arguments().expression()) {
-                arguments.add((AExpression)visit(expression));
-            }
-
-            links.add(new LNewObj(line(ctx), location(ctx), ctx.identifier().getText(), arguments));
-        } else if (ctx.expression().size() > 0) {
-            for (final ExpressionContext expression : ctx.expression()) {
-                arguments.add((AExpression)visit(expression));
-            }
-
-            links.add(new LNewArray(line(ctx), location(ctx), ctx.identifier().getText(), arguments));
-        } else {
-            throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
-        }
-
-        if (ctx.linkdot() != null) {
-            visitLinkdot(ctx.linkdot(), links);
-        }
+        return new LBrace(line(ctx), offset(ctx), location(ctx), expression);
     }
 
     @Override
-    public ANode visitLinknew(final LinknewContext ctx) {
-        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
-    }
-
-    private void visitLinkstring(final LinkstringContext ctx, final List<ALink> links) {
-        links.add(new LString(line(ctx), location(ctx), ctx.STRING().getText().substring(1, ctx.STRING().getText().length() - 1)));
-
-        if (ctx.linkbrace() != null) {
-            visitLinkbrace(ctx.linkbrace(), links);
-        } else if (ctx.linkdot() != null) {
-            visitLinkdot(ctx.linkdot(), links);
-        }
-    }
-
-    @Override
-    public ANode visitLinkstring(final LinkstringContext ctx) {
-        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
-    }
-
-    @Override
-    public ANode visitArguments(final ArgumentsContext ctx) {
-        throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+    public Object visitArguments(ArgumentsContext ctx) {
+        throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
index 54eb8091e8e..c46181817e2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
@@ -98,8 +98,8 @@ public abstract class AExpression extends ANode {
      */
     protected Label fals = null;
 
-    public AExpression(final int line, final String location) {
-        super(line, location);
+    public AExpression(int line, int offset, String location) {
+        super(line, offset, location);
     }
 
     /**
@@ -110,7 +110,7 @@ public abstract class AExpression extends ANode {
     /**
      * Writes ASM based on the data collected during the analysis phase.
      */
-    abstract void write(MethodWriter adapter);
+    abstract void write(MethodWriter writer);
 
     /**
      * Inserts {@link ECast} nodes into the tree for implicit casts.  Also replaces
@@ -124,7 +124,7 @@ public abstract class AExpression extends ANode {
             if (constant == null || this instanceof EConstant) {
                 return this;
             } else {
-                final EConstant econstant = new EConstant(line, location, constant);
+                final EConstant econstant = new EConstant(line, offset, location, constant);
                 econstant.analyze(variables);
 
                 if (!expected.equals(econstant.actual)) {
@@ -135,7 +135,7 @@ public abstract class AExpression extends ANode {
             }
         } else {
             if (constant == null) {
-                final ECast ecast = new ECast(line, location, this, cast);
+                final ECast ecast = new ECast(line, offset, location, this, cast);
                 ecast.statement = statement;
                 ecast.actual = expected;
                 ecast.isNull = isNull;
@@ -145,7 +145,7 @@ public abstract class AExpression extends ANode {
                 if (expected.sort.constant) {
                     constant = AnalyzerCaster.constCast(location, constant, cast);
 
-                    final EConstant econstant = new EConstant(line, location, constant);
+                    final EConstant econstant = new EConstant(line, offset, location, constant);
                     econstant.analyze(variables);
 
                     if (!expected.equals(econstant.actual)) {
@@ -154,19 +154,19 @@ public abstract class AExpression extends ANode {
 
                     return econstant;
                 } else if (this instanceof EConstant) {
-                    final ECast ecast = new ECast(line, location, this, cast);
+                    final ECast ecast = new ECast(line, offset, location, this, cast);
                     ecast.actual = expected;
 
                     return ecast;
                 } else {
-                    final EConstant econstant = new EConstant(line, location, constant);
+                    final EConstant econstant = new EConstant(line, offset, location, constant);
                     econstant.analyze(variables);
 
                     if (!actual.equals(econstant.actual)) {
                         throw new IllegalStateException(error("Illegal tree structure."));
                     }
 
-                    final ECast ecast = new ECast(line, location, econstant, cast);
+                    final ECast ecast = new ECast(line, offset, location, econstant, cast);
                     ecast.actual = expected;
 
                     return ecast;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
index e134e7fbcb9..d68f662f913 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
@@ -73,8 +73,8 @@ public abstract class ALink extends ANode {
      */
     String string = null;
 
-    ALink(int line, String location, int size) {
-        super(line, location);
+    ALink(int line, int offset, String location, int size) {
+        super(line, offset, location);
 
         this.size = size;
     }
@@ -90,17 +90,17 @@ public abstract class ALink extends ANode {
     /**
      * Write values before a load/store occurs such as an array index.
      */
-    abstract void write(MethodWriter adapter);
+    abstract void write(MethodWriter writer);
 
     /**
      * Write a load for the specific link type.
      */
-    abstract void load(MethodWriter adapter);
+    abstract void load(MethodWriter writer);
 
     /**
      * Write a store for the specific link type.
      */
-    abstract void store(MethodWriter adapter);
+    abstract void store(MethodWriter writer);
 
     /**
      * Used to copy link data from one to another during analysis in the case of replacement.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
index a61b002dfca..414f6afe038 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
@@ -28,32 +28,38 @@ import org.objectweb.asm.Label;
 public abstract class ANode {
 
     /**
-     * The line number in the original source used for debug messages.
+     * The line number in the original source used for debugging and errors.
      */
     final int line;
 
+    /**
+     * The character offset in the original source used for debugging and errors.
+     */
+    final int offset;
+
     /**
      * The location in the original source to be printed in error messages.
      */
     final String location;
 
-    ANode(final int line, final String location) {
+    ANode(int line, int offset, String location) {
         this.line = line;
+        this.offset = offset;
         this.location = location;
     }
 
     public String error(final String message) {
         return "Error " + location  + ": " + message;
     }
-    
-    /** 
+
+    /**
      * Writes line number information
      * <p>
      * Currently we emit line number data for for leaf S-nodes
      */
-    void writeDebugInfo(MethodWriter adapter) {
+    void writeDebugInfo(MethodWriter writer) {
         Label label = new Label();
-        adapter.visitLabel(label);
-        adapter.visitLineNumber(line, label);
+        writer.visitLabel(label);
+        writer.visitLineNumber(line, label);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
index ebc4a166268..80d24b4cab3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
@@ -107,8 +107,8 @@ public abstract class AStatement extends ANode {
      */
     Label brake = null;
 
-    AStatement(int line, String location) {
-        super(line, location);
+    AStatement(int line, int offset, String location) {
+        super(line, offset, location);
     }
 
     /**
@@ -119,5 +119,5 @@ public abstract class AStatement extends ANode {
     /**
      * Writes ASM based on the data collected during the analysis phase.
      */
-    abstract void write(MethodWriter adapter);
+    abstract void write(MethodWriter writer);
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index ee990f60b61..0c083111d66 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -38,8 +38,8 @@ public final class EBinary extends AExpression {
 
     boolean cat = false;
 
-    public EBinary(int line, String location, Operation operation, AExpression left, AExpression right) {
-        super(line, location);
+    public EBinary(int line, int offset, String location, Operation operation, AExpression left, AExpression right) {
+        super(line, offset, location);
 
         this.operation = operation;
         this.left = left;
@@ -79,7 +79,7 @@ public final class EBinary extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+        Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply multiply [*] to types " +
@@ -93,7 +93,7 @@ public final class EBinary extends AExpression {
         right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
-            final Sort sort = promote.sort;
+            Sort sort = promote.sort;
 
             if (sort == Sort.INT) {
                 constant = (int)left.constant * (int)right.constant;
@@ -115,7 +115,7 @@ public final class EBinary extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+        Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply divide [/] to types " +
@@ -129,7 +129,7 @@ public final class EBinary extends AExpression {
         right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
-            final Sort sort = promote.sort;
+            Sort sort = promote.sort;
 
             if (sort == Sort.INT) {
                 constant = (int)left.constant / (int)right.constant;
@@ -151,7 +151,7 @@ public final class EBinary extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+        Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply remainder [%] to types " +
@@ -165,7 +165,7 @@ public final class EBinary extends AExpression {
         right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
-            final Sort sort = promote.sort;
+            Sort sort = promote.sort;
 
             if (sort == Sort.INT) {
                 constant = (int)left.constant % (int)right.constant;
@@ -187,14 +187,14 @@ public final class EBinary extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        final Type promote = AnalyzerCaster.promoteAdd(left.actual, right.actual);
+        Type promote = AnalyzerCaster.promoteAdd(left.actual, right.actual);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply add [+] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        final Sort sort = promote.sort;
+        Sort sort = promote.sort;
 
         if (sort == Sort.STRING) {
             left.expected = left.actual;
@@ -239,7 +239,7 @@ public final class EBinary extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+        Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply subtract [-] to types " +
@@ -253,7 +253,7 @@ public final class EBinary extends AExpression {
         right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
-            final Sort sort = promote.sort;
+            Sort sort = promote.sort;
 
             if (sort == Sort.INT) {
                 constant = (int)left.constant - (int)right.constant;
@@ -275,7 +275,7 @@ public final class EBinary extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, false);
+        Type promote = AnalyzerCaster.promoteNumeric(left.actual, false);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply left shift [<<] to types " +
@@ -290,7 +290,7 @@ public final class EBinary extends AExpression {
         right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
-            final Sort sort = promote.sort;
+            Sort sort = promote.sort;
 
             if (sort == Sort.INT) {
                 constant = (int)left.constant << (int)right.constant;
@@ -308,7 +308,7 @@ public final class EBinary extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, false);
+        Type promote = AnalyzerCaster.promoteNumeric(left.actual, false);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply right shift [>>] to types " +
@@ -323,7 +323,7 @@ public final class EBinary extends AExpression {
         right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
-            final Sort sort = promote.sort;
+            Sort sort = promote.sort;
 
             if (sort == Sort.INT) {
                 constant = (int)left.constant >> (int)right.constant;
@@ -341,7 +341,7 @@ public final class EBinary extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, false);
+        Type promote = AnalyzerCaster.promoteNumeric(left.actual, false);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply unsigned shift [>>>] to types " +
@@ -356,7 +356,7 @@ public final class EBinary extends AExpression {
         right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
-            final Sort sort = promote.sort;
+            Sort sort = promote.sort;
 
             if (sort == Sort.INT) {
                 constant = (int)left.constant >>> (int)right.constant;
@@ -374,7 +374,7 @@ public final class EBinary extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false);
+        Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply and [&] to types " +
@@ -388,7 +388,7 @@ public final class EBinary extends AExpression {
         right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
-            final Sort sort = promote.sort;
+            Sort sort = promote.sort;
 
             if (sort == Sort.INT) {
                 constant = (int)left.constant & (int)right.constant;
@@ -406,7 +406,7 @@ public final class EBinary extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        final Type promote = AnalyzerCaster.promoteXor(left.actual, right.actual);
+        Type promote = AnalyzerCaster.promoteXor(left.actual, right.actual);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply xor [^] to types " +
@@ -420,7 +420,7 @@ public final class EBinary extends AExpression {
         right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
-            final Sort sort = promote.sort;
+            Sort sort = promote.sort;
 
             if (sort == Sort.BOOL) {
                 constant = (boolean)left.constant ^ (boolean)right.constant;
@@ -440,7 +440,7 @@ public final class EBinary extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false);
+        Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply or [|] to types " +
@@ -454,7 +454,7 @@ public final class EBinary extends AExpression {
         right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
-            final Sort sort = promote.sort;
+            Sort sort = promote.sort;
 
             if (sort == Sort.INT) {
                 constant = (int)left.constant | (int)right.constant;
@@ -469,34 +469,34 @@ public final class EBinary extends AExpression {
     }
 
     @Override
-    void write(MethodWriter adapter) {
+    void write(MethodWriter writer) {
         if (actual.sort == Sort.STRING && operation == Operation.ADD) {
             if (!cat) {
-                adapter.writeNewStrings();
+                writer.writeNewStrings();
             }
 
-            left.write(adapter);
+            left.write(writer);
 
             if (!(left instanceof EBinary) || ((EBinary)left).operation != Operation.ADD || left.actual.sort != Sort.STRING) {
-                adapter.writeAppendStrings(left.actual);
+                writer.writeAppendStrings(left.actual);
             }
 
-            right.write(adapter);
+            right.write(writer);
 
             if (!(right instanceof EBinary) || ((EBinary)right).operation != Operation.ADD || right.actual.sort != Sort.STRING) {
-                adapter.writeAppendStrings(right.actual);
+                writer.writeAppendStrings(right.actual);
             }
 
             if (!cat) {
-                adapter.writeToStrings();
+                writer.writeToStrings();
             }
         } else {
-            left.write(adapter);
-            right.write(adapter);
+            left.write(writer);
+            right.write(writer);
 
-            adapter.writeBinaryInstruction(location, actual, operation);
+            writer.writeBinaryInstruction(location, actual, operation);
         }
 
-        adapter.writeBranch(tru, fals);
+        writer.writeBranch(tru, fals);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
index 27cd3018ded..eda3e364795 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
@@ -34,8 +34,8 @@ public final class EBool extends AExpression {
     AExpression left;
     AExpression right;
 
-    public EBool(int line, String location, Operation operation, AExpression left, AExpression right) {
-        super(line, location);
+    public EBool(int line, int offset, String location, Operation operation, AExpression left, AExpression right) {
+        super(line, offset, location);
 
         this.operation = operation;
         this.left = left;
@@ -66,70 +66,70 @@ public final class EBool extends AExpression {
     }
 
     @Override
-    void write(MethodWriter adapter) {
+    void write(MethodWriter writer) {
         if (tru != null || fals != null) {
             if (operation == Operation.AND) {
-                final Label localfals = fals == null ? new Label() : fals;
+                Label localfals = fals == null ? new Label() : fals;
 
                 left.fals = localfals;
                 right.tru = tru;
                 right.fals = fals;
 
-                left.write(adapter);
-                right.write(adapter);
+                left.write(writer);
+                right.write(writer);
 
                 if (fals == null) {
-                    adapter.mark(localfals);
+                    writer.mark(localfals);
                 }
             } else if (operation == Operation.OR) {
-                final Label localtru = tru == null ? new Label() : tru;
+                Label localtru = tru == null ? new Label() : tru;
 
                 left.tru = localtru;
                 right.tru = tru;
                 right.fals = fals;
 
-                left.write(adapter);
-                right.write(adapter);
+                left.write(writer);
+                right.write(writer);
 
                 if (tru == null) {
-                    adapter.mark(localtru);
+                    writer.mark(localtru);
                 }
             } else {
                 throw new IllegalStateException(error("Illegal tree structure."));
             }
         } else {
             if (operation == Operation.AND) {
-                final Label localfals = new Label();
-                final Label end = new Label();
+                Label localfals = new Label();
+                Label end = new Label();
 
                 left.fals = localfals;
                 right.fals = localfals;
 
-                left.write(adapter);
-                right.write(adapter);
+                left.write(writer);
+                right.write(writer);
 
-                adapter.push(true);
-                adapter.goTo(end);
-                adapter.mark(localfals);
-                adapter.push(false);
-                adapter.mark(end);
+                writer.push(true);
+                writer.goTo(end);
+                writer.mark(localfals);
+                writer.push(false);
+                writer.mark(end);
             } else if (operation == Operation.OR) {
-                final Label localtru = new Label();
-                final Label localfals = new Label();
-                final Label end = new Label();
+                Label localtru = new Label();
+                Label localfals = new Label();
+                Label end = new Label();
 
                 left.tru = localtru;
                 right.fals = localfals;
 
-                left.write(adapter);
-                right.write(adapter);
+                left.write(writer);
+                right.write(writer);
 
-                adapter.mark(localtru);
-                adapter.push(true);
-                adapter.goTo(end);
-                adapter.mark(localfals);
-                adapter.push(false);
-                adapter.mark(end);
+                writer.mark(localtru);
+                writer.push(true);
+                writer.goTo(end);
+                writer.mark(localfals);
+                writer.push(false);
+                writer.mark(end);
             } else {
                 throw new IllegalStateException(error("Illegal tree structure."));
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
index 27d7bb9a626..a1585e13dbc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
@@ -28,8 +28,8 @@ import org.elasticsearch.painless.MethodWriter;
  */
 public final class EBoolean extends AExpression {
 
-    public EBoolean(int line, String location, boolean constant) {
-        super(line, location);
+    public EBoolean(int line, int offset, String location, boolean constant) {
+        super(line, offset, location);
 
         this.constant = constant;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
index 7a8c4a29b60..e2fe5cb6d10 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
@@ -34,8 +34,8 @@ final class ECast extends AExpression {
 
     Cast cast = null;
 
-    ECast(int line, String location, AExpression child, Cast cast) {
-        super(line, location);
+    ECast(int line, int offset, String location, AExpression child, Cast cast) {
+        super(line, offset, location);
 
         this.type = null;
         this.child = child;
@@ -49,9 +49,9 @@ final class ECast extends AExpression {
     }
 
     @Override
-    void write(MethodWriter adapter) {
-        child.write(adapter);
-        adapter.writeCast(cast);
-        adapter.writeBranch(tru, fals);
+    void write(MethodWriter writer) {
+        child.write(writer);
+        writer.writeCast(cast);
+        writer.writeBranch(tru, fals);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index 31bff2dd66d..2de1a5b320e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -46,9 +46,9 @@ public final class EChain extends AExpression {
     Cast there = null;
     Cast back = null;
 
-    public EChain(int line, String location, List<ALink> links,
+    public EChain(int line, int offset, String location, List<ALink> links,
                   boolean pre, boolean post, Operation operation, AExpression expression) {
-        super(line, location);
+        super(line, offset, location);
 
         this.links = links;
         this.pre = pre;
@@ -76,7 +76,7 @@ public final class EChain extends AExpression {
         int index = 0;
 
         while (index < links.size()) {
-            final ALink current = links.get(index);
+            ALink current = links.get(index);
 
             if (previous != null) {
                 current.before = previous.after;
@@ -91,7 +91,7 @@ public final class EChain extends AExpression {
                 current.store = expression != null || pre || post;
             }
 
-            final ALink analyzed = current.analyze(variables);
+            ALink analyzed = current.analyze(variables);
 
             if (analyzed == null) {
                 links.remove(index);
@@ -111,7 +111,7 @@ public final class EChain extends AExpression {
     }
 
     private void analyzeIncrDecr() {
-        final ALink last = links.get(links.size() - 1);
+        ALink last = links.get(links.size() - 1);
 
         if (pre && post) {
             throw new IllegalStateException(error("Illegal tree structure."));
@@ -120,29 +120,29 @@ public final class EChain extends AExpression {
                 throw new IllegalStateException(error("Illegal tree structure."));
             }
 
-            final Sort sort = last.after.sort;
+            Sort sort = last.after.sort;
 
             if (operation == Operation.INCR) {
                 if (sort == Sort.DOUBLE) {
-                    expression = new EConstant(line, location, 1D);
+                    expression = new EConstant(line, offset, location, 1D);
                 } else if (sort == Sort.FLOAT) {
-                    expression = new EConstant(line, location, 1F);
+                    expression = new EConstant(line, offset, location, 1F);
                 } else if (sort == Sort.LONG) {
-                    expression = new EConstant(line, location, 1L);
+                    expression = new EConstant(line, offset, location, 1L);
                 } else {
-                    expression = new EConstant(line, location, 1);
+                    expression = new EConstant(line, offset, location, 1);
                 }
 
                 operation = Operation.ADD;
             } else if (operation == Operation.DECR) {
                 if (sort == Sort.DOUBLE) {
-                    expression = new EConstant(line, location, 1D);
+                    expression = new EConstant(line, offset, location, 1D);
                 } else if (sort == Sort.FLOAT) {
-                    expression = new EConstant(line, location, 1F);
+                    expression = new EConstant(line, offset, location, 1F);
                 } else if (sort == Sort.LONG) {
-                    expression = new EConstant(line, location, 1L);
+                    expression = new EConstant(line, offset, location, 1L);
                 } else {
-                    expression = new EConstant(line, location, 1);
+                    expression = new EConstant(line, offset, location, 1);
                 }
 
                 operation = Operation.SUB;
@@ -153,7 +153,7 @@ public final class EChain extends AExpression {
     }
 
     private void analyzeCompound(Variables variables) {
-        final ALink last = links.get(links.size() - 1);
+        ALink last = links.get(links.size() - 1);
 
         expression.analyze(variables);
 
@@ -214,9 +214,9 @@ public final class EChain extends AExpression {
     }
 
     private void analyzeWrite(Variables variables) {
-        final ALink last = links.get(links.size() - 1);
+        ALink last = links.get(links.size() - 1);
 
-        // If the store node is a DEF node, we remove the cast to DEF from the expression
+        // If the store node is a def node, we remove the cast to def from the expression
         // and promote the real type to it:
         if (last instanceof IDefLink) {
             expression.analyze(variables);
@@ -234,9 +234,9 @@ public final class EChain extends AExpression {
     }
 
     private void analyzeRead() {
-        final ALink last = links.get(links.size() - 1);
+        ALink last = links.get(links.size() - 1);
 
-        // If the load node is a DEF node, we adapt its after type to use _this_ expected output type:
+        // If the load node is a def node, we adapt its after type to use _this_ expected output type:
         if (last instanceof IDefLink && this.expected != null) {
             last.after = this.expected;
         }
@@ -247,70 +247,70 @@ public final class EChain extends AExpression {
     }
 
     @Override
-    void write(MethodWriter adapter) {
+    void write(MethodWriter writer) {
         if (cat) {
-            adapter.writeNewStrings();
+            writer.writeNewStrings();
         }
 
-        final ALink last = links.get(links.size() - 1);
+        ALink last = links.get(links.size() - 1);
 
-        for (final ALink link : links) {
-            link.write(adapter);
+        for (ALink link : links) {
+            link.write(writer);
 
             if (link == last && link.store) {
                 if (cat) {
-                    adapter.writeDup(link.size, 1);
-                    link.load(adapter);
-                    adapter.writeAppendStrings(link.after);
+                    writer.writeDup(link.size, 1);
+                    link.load(writer);
+                    writer.writeAppendStrings(link.after);
 
-                    expression.write(adapter);
+                    expression.write(writer);
 
                     if (!(expression instanceof EBinary) ||
                         ((EBinary)expression).operation != Operation.ADD || expression.actual.sort != Sort.STRING) {
-                        adapter.writeAppendStrings(expression.actual);
+                        writer.writeAppendStrings(expression.actual);
                     }
 
-                    adapter.writeToStrings();
-                    adapter.writeCast(back);
+                    writer.writeToStrings();
+                    writer.writeCast(back);
 
                     if (link.load) {
-                        adapter.writeDup(link.after.sort.size, link.size);
+                        writer.writeDup(link.after.sort.size, link.size);
                     }
 
-                    link.store(adapter);
+                    link.store(writer);
                 } else if (operation != null) {
-                    adapter.writeDup(link.size, 0);
-                    link.load(adapter);
+                    writer.writeDup(link.size, 0);
+                    link.load(writer);
 
                     if (link.load && post) {
-                        adapter.writeDup(link.after.sort.size, link.size);
+                        writer.writeDup(link.after.sort.size, link.size);
                     }
 
-                    adapter.writeCast(there);
-                    expression.write(adapter);
-                    adapter.writeBinaryInstruction(location, promote, operation);
+                    writer.writeCast(there);
+                    expression.write(writer);
+                    writer.writeBinaryInstruction(location, promote, operation);
 
-                    adapter.writeCast(back);
+                    writer.writeCast(back);
 
                     if (link.load && !post) {
-                        adapter.writeDup(link.after.sort.size, link.size);
+                        writer.writeDup(link.after.sort.size, link.size);
                     }
 
-                    link.store(adapter);
+                    link.store(writer);
                 } else {
-                    expression.write(adapter);
+                    expression.write(writer);
 
                     if (link.load) {
-                        adapter.writeDup(link.after.sort.size, link.size);
+                        writer.writeDup(link.after.sort.size, link.size);
                     }
 
-                    link.store(adapter);
+                    link.store(writer);
                 }
             } else {
-                link.load(adapter);
+                link.load(writer);
             }
         }
 
-        adapter.writeBranch(tru, fals);
+        writer.writeBranch(tru, fals);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index 08fed075335..4521c76d911 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -46,8 +46,8 @@ public final class EComp extends AExpression {
     AExpression left;
     AExpression right;
 
-    public EComp(int line, String location, Operation operation, AExpression left, AExpression right) {
-        super(line, location);
+    public EComp(int line, int offset, String location, Operation operation, AExpression left, AExpression right) {
+        super(line, offset, location);
 
         this.operation = operation;
         this.left = left;
@@ -81,7 +81,7 @@ public final class EComp extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        final Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual);
+        Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply equals [==] to types " +
@@ -99,7 +99,7 @@ public final class EComp extends AExpression {
         }
 
         if ((left.constant != null || left.isNull) && (right.constant != null || right.isNull)) {
-            final Sort sort = promote.sort;
+            Sort sort = promote.sort;
 
             if (sort == Sort.BOOL) {
                 constant = (boolean)left.constant == (boolean)right.constant;
@@ -127,7 +127,7 @@ public final class EComp extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        final Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual);
+        Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply reference equals [===] to types " +
@@ -145,7 +145,7 @@ public final class EComp extends AExpression {
         }
 
         if ((left.constant != null || left.isNull) && (right.constant != null || right.isNull)) {
-            final Sort sort = promote.sort;
+            Sort sort = promote.sort;
 
             if (sort == Sort.BOOL) {
                 constant = (boolean)left.constant == (boolean)right.constant;
@@ -169,7 +169,7 @@ public final class EComp extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        final Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual);
+        Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply not equals [!=] to types " +
@@ -187,7 +187,7 @@ public final class EComp extends AExpression {
         }
 
         if ((left.constant != null || left.isNull) && (right.constant != null || right.isNull)) {
-            final Sort sort = promote.sort;
+            Sort sort = promote.sort;
 
             if (sort == Sort.BOOL) {
                 constant = (boolean)left.constant != (boolean)right.constant;
@@ -215,7 +215,7 @@ public final class EComp extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        final Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual);
+        Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply reference not equals [!==] to types " +
@@ -233,7 +233,7 @@ public final class EComp extends AExpression {
         }
 
         if ((left.constant != null || left.isNull) && (right.constant != null || right.isNull)) {
-            final Sort sort = promote.sort;
+            Sort sort = promote.sort;
 
             if (sort == Sort.BOOL) {
                 constant = (boolean)left.constant != (boolean)right.constant;
@@ -257,7 +257,7 @@ public final class EComp extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+        Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply greater than or equals [>=] to types " +
@@ -271,7 +271,7 @@ public final class EComp extends AExpression {
         right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
-            final Sort sort = promote.sort;
+            Sort sort = promote.sort;
 
             if (sort == Sort.INT) {
                 constant = (int)left.constant >= (int)right.constant;
@@ -293,7 +293,7 @@ public final class EComp extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+        Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply greater than [>] to types " +
@@ -307,7 +307,7 @@ public final class EComp extends AExpression {
         right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
-            final Sort sort = promote.sort;
+            Sort sort = promote.sort;
 
             if (sort == Sort.INT) {
                 constant = (int)left.constant > (int)right.constant;
@@ -329,7 +329,7 @@ public final class EComp extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+        Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply less than or equals [<=] to types " +
@@ -343,7 +343,7 @@ public final class EComp extends AExpression {
         right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
-            final Sort sort = promote.sort;
+            Sort sort = promote.sort;
 
             if (sort == Sort.INT) {
                 constant = (int)left.constant <= (int)right.constant;
@@ -365,7 +365,7 @@ public final class EComp extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+        Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply less than [>=] to types " +
@@ -379,7 +379,7 @@ public final class EComp extends AExpression {
         right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
-            final Sort sort = promote.sort;
+            Sort sort = promote.sort;
 
             if (sort == Sort.INT) {
                 constant = (int)left.constant < (int)right.constant;
@@ -398,28 +398,28 @@ public final class EComp extends AExpression {
     }
 
     @Override
-    void write(MethodWriter adapter) {
-        final boolean branch = tru != null || fals != null;
-        final org.objectweb.asm.Type rtype = right.actual.type;
-        final Sort rsort = right.actual.sort;
+    void write(MethodWriter writer) {
+        boolean branch = tru != null || fals != null;
+        org.objectweb.asm.Type rtype = right.actual.type;
+        Sort rsort = right.actual.sort;
 
-        left.write(adapter);
+        left.write(writer);
 
         if (!right.isNull) {
-            right.write(adapter);
+            right.write(writer);
         }
 
-        final Label jump = tru != null ? tru : fals != null ? fals : new Label();
-        final Label end = new Label();
+        Label jump = tru != null ? tru : fals != null ? fals : new Label();
+        Label end = new Label();
 
-        final boolean eq = (operation == Operation.EQ || operation == Operation.EQR) && (tru != null || fals == null) ||
+        boolean eq = (operation == Operation.EQ || operation == Operation.EQR) && (tru != null || fals == null) ||
             (operation == Operation.NE || operation == Operation.NER) && fals != null;
-        final boolean ne = (operation == Operation.NE || operation == Operation.NER) && (tru != null || fals == null) ||
+        boolean ne = (operation == Operation.NE || operation == Operation.NER) && (tru != null || fals == null) ||
             (operation == Operation.EQ || operation == Operation.EQR) && fals != null;
-        final boolean lt  = operation == Operation.LT  && (tru != null || fals == null) || operation == Operation.GTE && fals != null;
-        final boolean lte = operation == Operation.LTE && (tru != null || fals == null) || operation == Operation.GT  && fals != null;
-        final boolean gt  = operation == Operation.GT  && (tru != null || fals == null) || operation == Operation.LTE && fals != null;
-        final boolean gte = operation == Operation.GTE && (tru != null || fals == null) || operation == Operation.LT  && fals != null;
+        boolean lt  = operation == Operation.LT  && (tru != null || fals == null) || operation == Operation.GTE && fals != null;
+        boolean lte = operation == Operation.LTE && (tru != null || fals == null) || operation == Operation.GT  && fals != null;
+        boolean gt  = operation == Operation.GT  && (tru != null || fals == null) || operation == Operation.LTE && fals != null;
+        boolean gte = operation == Operation.GTE && (tru != null || fals == null) || operation == Operation.LT  && fals != null;
 
         boolean writejump = true;
 
@@ -430,8 +430,8 @@ public final class EComp extends AExpression {
             case CHAR:
                 throw new IllegalStateException(error("Illegal tree structure."));
             case BOOL:
-                if      (eq) adapter.ifZCmp(MethodWriter.EQ, jump);
-                else if (ne) adapter.ifZCmp(MethodWriter.NE, jump);
+                if      (eq) writer.ifZCmp(MethodWriter.EQ, jump);
+                else if (ne) writer.ifZCmp(MethodWriter.NE, jump);
                 else {
                     throw new IllegalStateException(error("Illegal tree structure."));
                 }
@@ -441,12 +441,12 @@ public final class EComp extends AExpression {
             case LONG:
             case FLOAT:
             case DOUBLE:
-                if      (eq)  adapter.ifCmp(rtype, MethodWriter.EQ, jump);
-                else if (ne)  adapter.ifCmp(rtype, MethodWriter.NE, jump);
-                else if (lt)  adapter.ifCmp(rtype, MethodWriter.LT, jump);
-                else if (lte) adapter.ifCmp(rtype, MethodWriter.LE, jump);
-                else if (gt)  adapter.ifCmp(rtype, MethodWriter.GT, jump);
-                else if (gte) adapter.ifCmp(rtype, MethodWriter.GE, jump);
+                if      (eq)  writer.ifCmp(rtype, MethodWriter.EQ, jump);
+                else if (ne)  writer.ifCmp(rtype, MethodWriter.NE, jump);
+                else if (lt)  writer.ifCmp(rtype, MethodWriter.LT, jump);
+                else if (lte) writer.ifCmp(rtype, MethodWriter.LE, jump);
+                else if (gt)  writer.ifCmp(rtype, MethodWriter.GT, jump);
+                else if (gte) writer.ifCmp(rtype, MethodWriter.GE, jump);
                 else {
                     throw new IllegalStateException(error("Illegal tree structure."));
                 }
@@ -455,66 +455,66 @@ public final class EComp extends AExpression {
             case DEF:
                 if (eq) {
                     if (right.isNull) {
-                        adapter.ifNull(jump);
+                        writer.ifNull(jump);
                     } else if (!left.isNull && (operation == Operation.EQ || operation == Operation.NE)) {
-                        adapter.invokeStatic(DEF_UTIL_TYPE, DEF_EQ_CALL);
+                        writer.invokeStatic(DEF_UTIL_TYPE, DEF_EQ_CALL);
                         writejump = false;
                     } else {
-                        adapter.ifCmp(rtype, MethodWriter.EQ, jump);
+                        writer.ifCmp(rtype, MethodWriter.EQ, jump);
                     }
                 } else if (ne) {
                     if (right.isNull) {
-                        adapter.ifNonNull(jump);
+                        writer.ifNonNull(jump);
                     } else if (!left.isNull && (operation == Operation.EQ || operation == Operation.NE)) {
-                        adapter.invokeStatic(DEF_UTIL_TYPE, DEF_EQ_CALL);
-                        adapter.ifZCmp(MethodWriter.EQ, jump);
+                        writer.invokeStatic(DEF_UTIL_TYPE, DEF_EQ_CALL);
+                        writer.ifZCmp(MethodWriter.EQ, jump);
                     } else {
-                        adapter.ifCmp(rtype, MethodWriter.NE, jump);
+                        writer.ifCmp(rtype, MethodWriter.NE, jump);
                     }
                 } else if (lt) {
-                    adapter.invokeStatic(DEF_UTIL_TYPE, DEF_LT_CALL);
+                    writer.invokeStatic(DEF_UTIL_TYPE, DEF_LT_CALL);
                     writejump = false;
                 } else if (lte) {
-                    adapter.invokeStatic(DEF_UTIL_TYPE, DEF_LTE_CALL);
+                    writer.invokeStatic(DEF_UTIL_TYPE, DEF_LTE_CALL);
                     writejump = false;
                 } else if (gt) {
-                    adapter.invokeStatic(DEF_UTIL_TYPE, DEF_GT_CALL);
+                    writer.invokeStatic(DEF_UTIL_TYPE, DEF_GT_CALL);
                     writejump = false;
                 } else if (gte) {
-                    adapter.invokeStatic(DEF_UTIL_TYPE, DEF_GTE_CALL);
+                    writer.invokeStatic(DEF_UTIL_TYPE, DEF_GTE_CALL);
                     writejump = false;
                 } else {
                     throw new IllegalStateException(error("Illegal tree structure."));
                 }
 
                 if (branch && !writejump) {
-                    adapter.ifZCmp(MethodWriter.NE, jump);
+                    writer.ifZCmp(MethodWriter.NE, jump);
                 }
 
                 break;
             default:
                 if (eq) {
                     if (right.isNull) {
-                        adapter.ifNull(jump);
+                        writer.ifNull(jump);
                     } else if (operation == Operation.EQ || operation == Operation.NE) {
-                        adapter.invokeStatic(UTILITY_TYPE, CHECKEQUALS);
+                        writer.invokeStatic(UTILITY_TYPE, CHECKEQUALS);
 
                         if (branch) {
-                            adapter.ifZCmp(MethodWriter.NE, jump);
+                            writer.ifZCmp(MethodWriter.NE, jump);
                         }
 
                         writejump = false;
                     } else {
-                        adapter.ifCmp(rtype, MethodWriter.EQ, jump);
+                        writer.ifCmp(rtype, MethodWriter.EQ, jump);
                     }
                 } else if (ne) {
                     if (right.isNull) {
-                        adapter.ifNonNull(jump);
+                        writer.ifNonNull(jump);
                     } else if (operation == Operation.EQ || operation == Operation.NE) {
-                        adapter.invokeStatic(UTILITY_TYPE, CHECKEQUALS);
-                        adapter.ifZCmp(MethodWriter.EQ, jump);
+                        writer.invokeStatic(UTILITY_TYPE, CHECKEQUALS);
+                        writer.ifZCmp(MethodWriter.EQ, jump);
                     } else {
-                        adapter.ifCmp(rtype, MethodWriter.NE, jump);
+                        writer.ifCmp(rtype, MethodWriter.NE, jump);
                     }
                 } else {
                     throw new IllegalStateException(error("Illegal tree structure."));
@@ -522,11 +522,11 @@ public final class EComp extends AExpression {
         }
 
         if (!branch && writejump) {
-            adapter.push(false);
-            adapter.goTo(end);
-            adapter.mark(jump);
-            adapter.push(true);
-            adapter.mark(end);
+            writer.push(false);
+            writer.goTo(end);
+            writer.mark(jump);
+            writer.push(true);
+            writer.mark(end);
         }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
index 8f9d0038586..a13176e56b7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
@@ -35,8 +35,8 @@ public final class EConditional extends AExpression {
     AExpression left;
     AExpression right;
 
-    public EConditional(int line, String location, AExpression condition, AExpression left, AExpression right) {
-        super(line, location);
+    public EConditional(int line, int offset, String location, AExpression condition, AExpression left, AExpression right) {
+        super(line, offset, location);
 
         this.condition = condition;
         this.left = left;
@@ -77,19 +77,19 @@ public final class EConditional extends AExpression {
     }
 
     @Override
-    void write(MethodWriter adapter) {
-        final Label localfals = new Label();
-        final Label end = new Label();
+    void write(MethodWriter writer) {
+        Label localfals = new Label();
+        Label end = new Label();
 
         condition.fals = localfals;
         left.tru = right.tru = tru;
         left.fals = right.fals = fals;
 
-        condition.write(adapter);
-        left.write(adapter);
-        adapter.goTo(end);
-        adapter.mark(localfals);
-        right.write(adapter);
-        adapter.mark(end);
+        condition.write(writer);
+        left.write(writer);
+        writer.goTo(end);
+        writer.mark(localfals);
+        right.write(writer);
+        writer.mark(end);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
index ac18b849162..b45e3c33bc2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
@@ -30,8 +30,8 @@ import org.elasticsearch.painless.MethodWriter;
  */
 final class EConstant extends AExpression {
 
-    EConstant(int line, String location, Object constant) {
-        super(line, location);
+    EConstant(int line, int offset, String location, Object constant) {
+        super(line, offset, location);
 
         this.constant = constant;
     }
@@ -62,25 +62,25 @@ final class EConstant extends AExpression {
     }
 
     @Override
-    void write(MethodWriter adapter) {
-        final Sort sort = actual.sort;
+    void write(MethodWriter writer) {
+        Sort sort = actual.sort;
 
         switch (sort) {
-            case STRING: adapter.push((String)constant);  break;
-            case DOUBLE: adapter.push((double)constant);  break;
-            case FLOAT:  adapter.push((float)constant);   break;
-            case LONG:   adapter.push((long)constant);    break;
-            case INT:    adapter.push((int)constant);     break;
-            case CHAR:   adapter.push((char)constant);    break;
-            case SHORT:  adapter.push((short)constant);   break;
-            case BYTE:   adapter.push((byte)constant);    break;
+            case STRING: writer.push((String)constant);  break;
+            case DOUBLE: writer.push((double)constant);  break;
+            case FLOAT:  writer.push((float)constant);   break;
+            case LONG:   writer.push((long)constant);    break;
+            case INT:    writer.push((int)constant);     break;
+            case CHAR:   writer.push((char)constant);    break;
+            case SHORT:  writer.push((short)constant);   break;
+            case BYTE:   writer.push((byte)constant);    break;
             case BOOL:
                 if (tru != null && (boolean)constant) {
-                    adapter.goTo(tru);
+                    writer.goTo(tru);
                 } else if (fals != null && !(boolean)constant) {
-                    adapter.goTo(fals);
+                    writer.goTo(fals);
                 } else if (tru == null && fals == null) {
-                    adapter.push((boolean)constant);
+                    writer.push((boolean)constant);
                 }
 
                 break;
@@ -89,7 +89,7 @@ final class EConstant extends AExpression {
         }
 
         if (sort != Sort.BOOL) {
-            adapter.writeBranch(tru, fals);
+            writer.writeBranch(tru, fals);
         }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
index 85e857da3c4..034a4ec97f9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
@@ -30,8 +30,8 @@ public final class EDecimal extends AExpression {
 
     final String value;
 
-    public EDecimal(int line, String location, String value) {
-        super(line, location);
+    public EDecimal(int line, int offset, String location, String value) {
+        super(line, offset, location);
 
         this.value = value;
     }
@@ -42,21 +42,21 @@ public final class EDecimal extends AExpression {
             try {
                 constant = Float.parseFloat(value.substring(0, value.length() - 1));
                 actual = Definition.FLOAT_TYPE;
-            } catch (final NumberFormatException exception) {
+            } catch (NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid float constant [" + value + "]."));
             }
         } else {
             try {
                 constant = Double.parseDouble(value);
                 actual = Definition.DOUBLE_TYPE;
-            } catch (final NumberFormatException exception) {
+            } catch (NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid double constant [" + value + "]."));
             }
         }
     }
 
     @Override
-    void write(MethodWriter adapter) {
+    void write(MethodWriter writer) {
         throw new IllegalArgumentException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
index 7a8c9dadeca..643dc2d4141 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
@@ -31,8 +31,8 @@ public final class EExplicit extends AExpression {
     final String type;
     AExpression child;
 
-    public EExplicit(int line, String location, String type, AExpression child) {
-        super(line, location);
+    public EExplicit(int line, int offset, String location, String type, AExpression child) {
+        super(line, offset, location);
 
         this.type = type;
         this.child = child;
@@ -42,7 +42,7 @@ public final class EExplicit extends AExpression {
     void analyze(Variables variables) {
         try {
             actual = Definition.getType(this.type);
-        } catch (final IllegalArgumentException exception) {
+        } catch (IllegalArgumentException exception) {
             throw new IllegalArgumentException(error("Not a type [" + this.type + "]."));
         }
 
@@ -53,7 +53,7 @@ public final class EExplicit extends AExpression {
     }
 
     @Override
-    void write(MethodWriter adapter) {
+    void write(MethodWriter writer) {
         throw new IllegalArgumentException(error("Illegal tree structure."));
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
index 3a8005b20d6..2bb4f4ba585 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
@@ -29,8 +29,8 @@ import org.elasticsearch.painless.MethodWriter;
  */
 public final class ENull extends AExpression {
 
-    public ENull(int line, String location) {
-        super(line, location);
+    public ENull(int line, int offset, String location) {
+        super(line, offset, location);
     }
 
     @Override
@@ -49,7 +49,7 @@ public final class ENull extends AExpression {
     }
 
     @Override
-    void write(MethodWriter adapter) {
-        adapter.visitInsn(Opcodes.ACONST_NULL);
+    void write(MethodWriter writer) {
+        writer.visitInsn(Opcodes.ACONST_NULL);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
index ee70bb77d44..0fed456dd64 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
@@ -32,8 +32,8 @@ public final class ENumeric extends AExpression {
     final String value;
     int radix;
 
-    public ENumeric(int line, String location, String value, int radix) {
-        super(line, location);
+    public ENumeric(int line, int offset, String location, String value, int radix) {
+        super(line, offset, location);
 
         this.value = value;
         this.radix = radix;
@@ -49,7 +49,7 @@ public final class ENumeric extends AExpression {
             try {
                 constant = Double.parseDouble(value.substring(0, value.length() - 1));
                 actual = Definition.DOUBLE_TYPE;
-            } catch (final NumberFormatException exception) {
+            } catch (NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid double constant [" + value + "]."));
             }
         } else if (value.endsWith("f") || value.endsWith("F")) {
@@ -60,20 +60,20 @@ public final class ENumeric extends AExpression {
             try {
                 constant = Float.parseFloat(value.substring(0, value.length() - 1));
                 actual = Definition.FLOAT_TYPE;
-            } catch (final NumberFormatException exception) {
+            } catch (NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid float constant [" + value + "]."));
             }
         } else if (value.endsWith("l") || value.endsWith("L")) {
             try {
                 constant = Long.parseLong(value.substring(0, value.length() - 1), radix);
                 actual = Definition.LONG_TYPE;
-            } catch (final NumberFormatException exception) {
+            } catch (NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid long constant [" + value + "]."));
             }
         } else {
             try {
-                final Sort sort = expected == null ? Sort.INT : expected.sort;
-                final int integer = Integer.parseInt(value, radix);
+                Sort sort = expected == null ? Sort.INT : expected.sort;
+                int integer = Integer.parseInt(value, radix);
 
                 if (sort == Sort.BYTE && integer >= Byte.MIN_VALUE && integer <= Byte.MAX_VALUE) {
                     constant = (byte)integer;
@@ -88,14 +88,14 @@ public final class ENumeric extends AExpression {
                     constant = integer;
                     actual = Definition.INT_TYPE;
                 }
-            } catch (final NumberFormatException exception) {
+            } catch (NumberFormatException exception) {
                 throw new IllegalArgumentException(error("Invalid int constant [" + value + "]."));
             }
         }
     }
 
     @Override
-    void write(MethodWriter adapter) {
+    void write(MethodWriter writer) {
         throw new IllegalArgumentException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index d62fa2c8ebf..8b02522fca9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -37,11 +37,11 @@ import static org.elasticsearch.painless.WriterConstants.DEF_UTIL_TYPE;
  */
 public final class EUnary extends AExpression {
 
-    Operation operation;
+    final Operation operation;
     AExpression child;
 
-    public EUnary(int line, String location, Operation operation, AExpression child) {
-        super(line, location);
+    public EUnary(int line, int offset, String location, Operation operation, AExpression child) {
+        super(line, offset, location);
 
         this.operation = operation;
         this.child = child;
@@ -77,7 +77,7 @@ public final class EUnary extends AExpression {
     void analyzeBWNot(Variables variables) {
         child.analyze(variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(child.actual, false);
+        Type promote = AnalyzerCaster.promoteNumeric(child.actual, false);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply not [~] to type [" + child.actual.name + "]."));
@@ -87,7 +87,7 @@ public final class EUnary extends AExpression {
         child = child.cast(variables);
 
         if (child.constant != null) {
-            final Sort sort = promote.sort;
+            Sort sort = promote.sort;
 
             if (sort == Sort.INT) {
                 constant = ~(int)child.constant;
@@ -104,7 +104,7 @@ public final class EUnary extends AExpression {
     void analyzerAdd(Variables variables) {
         child.analyze(variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(child.actual, true);
+        Type promote = AnalyzerCaster.promoteNumeric(child.actual, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply positive [+] to type [" + child.actual.name + "]."));
@@ -114,7 +114,7 @@ public final class EUnary extends AExpression {
         child = child.cast(variables);
 
         if (child.constant != null) {
-            final Sort sort = promote.sort;
+            Sort sort = promote.sort;
 
             if (sort == Sort.INT) {
                 constant = +(int)child.constant;
@@ -135,7 +135,7 @@ public final class EUnary extends AExpression {
     void analyzerSub(Variables variables) {
         child.analyze(variables);
 
-        final Type promote = AnalyzerCaster.promoteNumeric(child.actual, true);
+        Type promote = AnalyzerCaster.promoteNumeric(child.actual, true);
 
         if (promote == null) {
             throw new ClassCastException(error("Cannot apply negative [-] to type [" + child.actual.name + "]."));
@@ -145,7 +145,7 @@ public final class EUnary extends AExpression {
         child = child.cast(variables);
 
         if (child.constant != null) {
-            final Sort sort = promote.sort;
+            Sort sort = promote.sort;
 
             if (sort == Sort.INT) {
                 constant = -(int)child.constant;
@@ -164,56 +164,56 @@ public final class EUnary extends AExpression {
     }
 
     @Override
-    void write(MethodWriter adapter) {
+    void write(MethodWriter writer) {
         if (operation == Operation.NOT) {
             if (tru == null && fals == null) {
-                final Label localfals = new Label();
-                final Label end = new Label();
+                Label localfals = new Label();
+                Label end = new Label();
 
                 child.fals = localfals;
-                child.write(adapter);
+                child.write(writer);
 
-                adapter.push(false);
-                adapter.goTo(end);
-                adapter.mark(localfals);
-                adapter.push(true);
-                adapter.mark(end);
+                writer.push(false);
+                writer.goTo(end);
+                writer.mark(localfals);
+                writer.push(true);
+                writer.mark(end);
             } else {
                 child.tru = fals;
                 child.fals = tru;
-                child.write(adapter);
+                child.write(writer);
             }
         } else {
-            final org.objectweb.asm.Type type = actual.type;
-            final Sort sort = actual.sort;
+            org.objectweb.asm.Type type = actual.type;
+            Sort sort = actual.sort;
 
-            child.write(adapter);
+            child.write(writer);
 
             if (operation == Operation.BWNOT) {
                 if (sort == Sort.DEF) {
-                    adapter.invokeStatic(DEF_UTIL_TYPE, DEF_NOT_CALL);
+                    writer.invokeStatic(DEF_UTIL_TYPE, DEF_NOT_CALL);
                 } else {
                     if (sort == Sort.INT) {
-                        adapter.push(-1);
+                        writer.push(-1);
                     } else if (sort == Sort.LONG) {
-                        adapter.push(-1L);
+                        writer.push(-1L);
                     } else {
                         throw new IllegalStateException(error("Illegal tree structure."));
                     }
 
-                    adapter.math(MethodWriter.XOR, type);
+                    writer.math(MethodWriter.XOR, type);
                 }
             } else if (operation == Operation.SUB) {
                 if (sort == Sort.DEF) {
-                    adapter.invokeStatic(DEF_UTIL_TYPE, DEF_NEG_CALL);
+                    writer.invokeStatic(DEF_UTIL_TYPE, DEF_NEG_CALL);
                 } else {
-                    adapter.math(MethodWriter.NEG, type);
+                    writer.math(MethodWriter.NEG, type);
                 }
             } else if (operation != Operation.ADD) {
                 throw new IllegalStateException(error("Illegal tree structure."));
             }
 
-            adapter.writeBranch(tru, fals);
+            writer.writeBranch(tru, fals);
         }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
index 4a6df96073e..cea5c629b14 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
@@ -30,8 +30,8 @@ public final class LArrayLength extends ALink {
 
     final String value;
 
-    LArrayLength(int line, String location, String value) {
-        super(line, location, -1);
+    LArrayLength(int line, int offset, String location, String value) {
+        super(line, offset, location, -1);
 
         this.value = value;
     }
@@ -54,17 +54,17 @@ public final class LArrayLength extends ALink {
     }
 
     @Override
-    void write(MethodWriter adapter) {
+    void write(MethodWriter writer) {
         // Do nothing.
     }
 
     @Override
-    void load(MethodWriter adapter) {
-        adapter.arrayLength();
+    void load(MethodWriter writer) {
+        writer.arrayLength();
     }
 
     @Override
-    void store(MethodWriter adapter) {
+    void store(MethodWriter writer) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
index 95cc02602f6..4411913ea07 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
@@ -34,8 +34,8 @@ public final class LBrace extends ALink {
 
     AExpression index;
 
-    public LBrace(int line, String location, AExpression index) {
-        super(line, location, 2);
+    public LBrace(int line, int offset, String location, AExpression index) {
+        super(line, offset, location, 2);
 
         this.index = index;
     }
@@ -43,7 +43,7 @@ public final class LBrace extends ALink {
     @Override
     ALink analyze(Variables variables) {
         if (before == null) {
-            throw new IllegalStateException(error("Illegal tree structure."));
+            throw new IllegalArgumentException(error("Illegal array access made without target."));
         }
 
         final Sort sort = before.sort;
@@ -57,29 +57,29 @@ public final class LBrace extends ALink {
 
             return this;
         } else if (sort == Sort.DEF) {
-            return new LDefArray(line, location, index).copy(this).analyze(variables);
+            return new LDefArray(line, offset, location, index).copy(this).analyze(variables);
         } else if (Map.class.isAssignableFrom(before.clazz)) {
-            return new LMapShortcut(line, location, index).copy(this).analyze(variables);
+            return new LMapShortcut(line, offset, location, index).copy(this).analyze(variables);
         } else if (List.class.isAssignableFrom(before.clazz)) {
-            return new LListShortcut(line, location, index).copy(this).analyze(variables);
+            return new LListShortcut(line, offset, location, index).copy(this).analyze(variables);
         }
 
         throw new IllegalArgumentException(error("Illegal array access on type [" + before.name + "]."));
     }
 
     @Override
-    void write(MethodWriter adapter) {
-        index.write(adapter);
+    void write(MethodWriter writer) {
+        index.write(writer);
     }
 
     @Override
-    void load(MethodWriter adapter) {
-        adapter.arrayLoad(after.type);
+    void load(MethodWriter writer) {
+        writer.arrayLoad(after.type);
     }
 
     @Override
-    void store(MethodWriter adapter) {
-        adapter.arrayStore(after.type);
+    void store(MethodWriter writer) {
+        writer.arrayStore(after.type);
     }
 
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
index fdb6612e1f4..29b57d8d3ef 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
@@ -21,6 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Method;
+import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Struct;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
@@ -37,8 +38,8 @@ public final class LCall extends ALink {
 
     Method method = null;
 
-    public LCall(int line, String location, String name, List<AExpression> arguments) {
-        super(line, location, -1);
+    public LCall(int line, int offset, String location, String name, List<AExpression> arguments) {
+        super(line, offset, location, -1);
 
         this.name = name;
         this.arguments = arguments;
@@ -47,20 +48,20 @@ public final class LCall extends ALink {
     @Override
     ALink analyze(Variables variables) {
         if (before == null) {
-            throw new IllegalStateException(error("Illegal tree structure."));
-        } else if (before.sort == Definition.Sort.ARRAY) {
+            throw new IllegalArgumentException(error("Illegal call [" + name + "] made without target."));
+        } else if (before.sort == Sort.ARRAY) {
             throw new IllegalArgumentException(error("Illegal call [" + name + "] on array type."));
         } else if (store) {
             throw new IllegalArgumentException(error("Cannot assign a value to a call [" + name + "]."));
         }
 
         Definition.MethodKey methodKey = new Definition.MethodKey(name, arguments.size());
-        final Struct struct = before.struct;
+        Struct struct = before.struct;
         method = statik ? struct.staticMethods.get(methodKey) : struct.methods.get(methodKey);
 
         if (method != null) {
             for (int argument = 0; argument < arguments.size(); ++argument) {
-                final AExpression expression = arguments.get(argument);
+                AExpression expression = arguments.get(argument);
 
                 expression.expected = method.arguments.get(argument);
                 expression.internal = true;
@@ -72,8 +73,8 @@ public final class LCall extends ALink {
             after = method.rtn;
 
             return this;
-        } else if (before.sort == Definition.Sort.DEF) {
-            final ALink link = new LDefCall(line, location, name, arguments);
+        } else if (before.sort == Sort.DEF) {
+            ALink link = new LDefCall(line, offset, location, name, arguments);
             link.copy(this);
 
             return link.analyze(variables);
@@ -84,31 +85,31 @@ public final class LCall extends ALink {
     }
 
     @Override
-    void write(MethodWriter adapter) {
+    void write(MethodWriter writer) {
         // Do nothing.
     }
 
     @Override
-    void load(MethodWriter adapter) {
-        for (final AExpression argument : arguments) {
-            argument.write(adapter);
+    void load(MethodWriter writer) {
+        for (AExpression argument : arguments) {
+            argument.write(writer);
         }
 
         if (java.lang.reflect.Modifier.isStatic(method.reflect.getModifiers())) {
-            adapter.invokeStatic(method.owner.type, method.method);
+            writer.invokeStatic(method.owner.type, method.method);
         } else if (java.lang.reflect.Modifier.isInterface(method.owner.clazz.getModifiers())) {
-            adapter.invokeInterface(method.owner.type, method.method);
+            writer.invokeInterface(method.owner.type, method.method);
         } else {
-            adapter.invokeVirtual(method.owner.type, method.method);
+            writer.invokeVirtual(method.owner.type, method.method);
         }
 
         if (!method.rtn.clazz.equals(method.handle.type().returnType())) {
-            adapter.checkCast(method.rtn.type);
+            writer.checkCast(method.rtn.type);
         }
     }
 
     @Override
-    void store(MethodWriter adapter) {
+    void store(MethodWriter writer) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
index 8e917c5b99e..06dee04fca6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
@@ -34,8 +34,8 @@ public final class LCast extends ALink {
 
     Cast cast = null;
 
-    public LCast(int line, String location, String type) {
-        super(line, location, -1);
+    public LCast(int line, int offset, String location, String type) {
+        super(line, offset, location, -1);
 
         this.type = type;
     }
@@ -43,14 +43,14 @@ public final class LCast extends ALink {
     @Override
     ALink analyze(Variables variables) {
         if (before == null) {
-            throw new IllegalStateException(error("Illegal tree structure."));
+            throw new IllegalStateException(error("Illegal cast without a target."));
         } else if (store) {
             throw new IllegalArgumentException(error("Cannot assign a value to a cast."));
         }
 
         try {
             after = Definition.getType(type);
-        } catch (final IllegalArgumentException exception) {
+        } catch (IllegalArgumentException exception) {
             throw new IllegalArgumentException(error("Not a type [" + type + "]."));
         }
 
@@ -60,17 +60,17 @@ public final class LCast extends ALink {
     }
 
     @Override
-    void write(MethodWriter adapter) {
-        adapter.writeCast(cast);
+    void write(MethodWriter writer) {
+        writer.writeCast(cast);
     }
 
     @Override
-    void load(MethodWriter adapter) {
+    void load(MethodWriter writer) {
         // Do nothing.
     }
 
     @Override
-    void store(MethodWriter adapter) {
+    void store(MethodWriter writer) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index 1ef12026e65..452a0732cdb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -34,8 +34,8 @@ final class LDefArray extends ALink implements IDefLink {
 
     AExpression index;
 
-    LDefArray(int line, String location, AExpression index) {
-        super(line, location, 2);
+    LDefArray(int line, int offset, String location, AExpression index) {
+        super(line, offset, location, 2);
 
         this.index = index;
     }
@@ -52,20 +52,19 @@ final class LDefArray extends ALink implements IDefLink {
     }
 
     @Override
-    void write(MethodWriter adapter) {
-        index.write(adapter);
+    void write(MethodWriter writer) {
+        index.write(writer);
     }
 
     @Override
-    void load(MethodWriter adapter) {
-        final String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type, index.actual.type);
-        adapter.invokeDynamic("arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_LOAD);
+    void load(MethodWriter writer) {
+        String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type, index.actual.type);
+        writer.invokeDynamic("arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ARRAY_LOAD);
     }
 
     @Override
-    void store(MethodWriter adapter) {
-        final String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type,
-            index.actual.type, after.type);
-        adapter.invokeDynamic("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_STORE);
+    void store(MethodWriter writer) {
+        String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, index.actual.type, after.type);
+        writer.invokeDynamic("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ARRAY_STORE);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index a9d1cb2b892..8ddde1cbed4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -36,8 +36,8 @@ final class LDefCall extends ALink implements IDefLink {
     final String name;
     final List<AExpression> arguments;
 
-    LDefCall(int line, String location, String name, List<AExpression> arguments) {
-        super(line, location, -1);
+    LDefCall(int line, int offset, String location, String name, List<AExpression> arguments) {
+        super(line, offset, location, -1);
 
         this.name = name;
         this.arguments = arguments;
@@ -46,7 +46,7 @@ final class LDefCall extends ALink implements IDefLink {
     @Override
     ALink analyze(Variables variables) {
         for (int argument = 0; argument < arguments.size(); ++argument) {
-            final AExpression expression = arguments.get(argument);
+            AExpression expression = arguments.get(argument);
 
             expression.internal = true;
             expression.analyze(variables);
@@ -61,34 +61,32 @@ final class LDefCall extends ALink implements IDefLink {
     }
 
     @Override
-    void write(MethodWriter adapter) {
+    void write(MethodWriter writer) {
         // Do nothing.
     }
 
     @Override
-    void load(MethodWriter adapter) {
-        final StringBuilder signature = new StringBuilder();
+    void load(MethodWriter writer) {
+        StringBuilder signature = new StringBuilder();
 
         signature.append('(');
         // first parameter is the receiver, we never know its type: always Object
         signature.append(Definition.DEF_TYPE.type.getDescriptor());
 
-        // TODO: remove our explicit conversions and feed more type information for return value,
-        // it can avoid some unnecessary boxing etc.
-        for (final AExpression argument : arguments) {
+        for (AExpression argument : arguments) {
             signature.append(argument.actual.type.getDescriptor());
-            argument.write(adapter);
+            argument.write(writer);
         }
 
         signature.append(')');
         // return value
         signature.append(after.type.getDescriptor());
 
-        adapter.invokeDynamic(name, signature.toString(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.METHOD_CALL);
+        writer.invokeDynamic(name, signature.toString(), DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.METHOD_CALL);
     }
 
     @Override
-    void store(MethodWriter adapter) {
+    void store(MethodWriter writer) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index ee0cf990c5b..b0a9d4dd0d9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -34,8 +34,8 @@ final class LDefField extends ALink implements IDefLink {
 
     final String value;
 
-    LDefField(int line, String location, String value) {
-        super(line, location, 1);
+    LDefField(int line, int offset, String location, String value) {
+        super(line, offset, location, 1);
 
         this.value = value;
     }
@@ -49,19 +49,19 @@ final class LDefField extends ALink implements IDefLink {
     }
 
     @Override
-    void write(MethodWriter adapter) {
+    void write(MethodWriter writer) {
         // Do nothing.
     }
 
     @Override
-    void load(MethodWriter adapter) {
-        final String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type);
-        adapter.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.LOAD);
+    void load(MethodWriter writer) {
+        String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type);
+        writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.LOAD);
     }
 
     @Override
-    void store(MethodWriter adapter) {
-        final String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, after.type);
-        adapter.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.STORE);
+    void store(MethodWriter writer) {
+        String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, after.type);
+        writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.STORE);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
index 44ba33acdda..9fbe543838f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
@@ -38,8 +38,8 @@ public final class LField extends ALink {
 
     Field field;
 
-    public LField(int line, String location, String value) {
-        super(line, location, 1);
+    public LField(int line, int offset, String location, String value) {
+        super(line, offset, location, 1);
 
         this.value = value;
     }
@@ -47,18 +47,18 @@ public final class LField extends ALink {
     @Override
     ALink analyze(Variables variables) {
         if (before == null) {
-            throw new IllegalStateException(error("Illegal tree structure."));
+            throw new IllegalArgumentException(error("Illegal field [" + value + "] access made without target."));
         }
 
-        final Sort sort = before.sort;
+        Sort sort = before.sort;
 
         if (sort == Sort.ARRAY) {
-            return new LArrayLength(line, location, value).copy(this).analyze(variables);
+            return new LArrayLength(line, offset, location, value).copy(this).analyze(variables);
         } else if (sort == Sort.DEF) {
-            return new LDefField(line, location, value).copy(this).analyze(variables);
+            return new LDefField(line, offset, location, value).copy(this).analyze(variables);
         }
 
-        final Struct struct = before.struct;
+        Struct struct = before.struct;
         field = statik ? struct.staticMembers.get(value) : struct.members.get(value);
 
         if (field != null) {
@@ -71,25 +71,26 @@ public final class LField extends ALink {
 
             return this;
         } else {
-            // TODO: improve this: the isXXX case seems missing???
-            final boolean shortcut =
-                struct.methods.containsKey(new Definition.MethodKey("get" + 
-                        Character.toUpperCase(value.charAt(0)) + value.substring(1), 0)) ||
-                struct.methods.containsKey(new Definition.MethodKey("set" + 
-                        Character.toUpperCase(value.charAt(0)) + value.substring(1), 1));
+            boolean shortcut =
+                struct.methods.containsKey(new Definition.MethodKey("get" +
+                    Character.toUpperCase(value.charAt(0)) + value.substring(1), 0)) ||
+                struct.methods.containsKey(new Definition.MethodKey("is" +
+                    Character.toUpperCase(value.charAt(0)) + value.substring(1), 0)) ||
+                struct.methods.containsKey(new Definition.MethodKey("set" +
+                    Character.toUpperCase(value.charAt(0)) + value.substring(1), 1));
 
             if (shortcut) {
-                return new LShortcut(line, location, value).copy(this).analyze(variables);
+                return new LShortcut(line, offset, location, value).copy(this).analyze(variables);
             } else {
-                final EConstant index = new EConstant(line, location, value);
+                EConstant index = new EConstant(line, offset, location, value);
                 index.analyze(variables);
 
                 if (Map.class.isAssignableFrom(before.clazz)) {
-                    return new LMapShortcut(line, location, index).copy(this).analyze(variables);
+                    return new LMapShortcut(line, offset, location, index).copy(this).analyze(variables);
                 }
-                
+
                 if (List.class.isAssignableFrom(before.clazz)) {
-                    return new LListShortcut(line, location, index).copy(this).analyze(variables);
+                    return new LListShortcut(line, offset, location, index).copy(this).analyze(variables);
                 }
             }
         }
@@ -98,25 +99,25 @@ public final class LField extends ALink {
     }
 
     @Override
-    void write(MethodWriter adapter) {
+    void write(MethodWriter writer) {
         // Do nothing.
     }
 
     @Override
-    void load(MethodWriter adapter) {
+    void load(MethodWriter writer) {
         if (java.lang.reflect.Modifier.isStatic(field.reflect.getModifiers())) {
-            adapter.getStatic(field.owner.type, field.reflect.getName(), field.type.type);
+            writer.getStatic(field.owner.type, field.reflect.getName(), field.type.type);
         } else {
-            adapter.getField(field.owner.type, field.reflect.getName(), field.type.type);
+            writer.getField(field.owner.type, field.reflect.getName(), field.type.type);
         }
     }
 
     @Override
-    void store(MethodWriter adapter) {
+    void store(MethodWriter writer) {
         if (java.lang.reflect.Modifier.isStatic(field.reflect.getModifiers())) {
-            adapter.putStatic(field.owner.type, field.reflect.getName(), field.type.type);
+            writer.putStatic(field.owner.type, field.reflect.getName(), field.type.type);
         } else {
-            adapter.putField(field.owner.type, field.reflect.getName(), field.type.type);
+            writer.putField(field.owner.type, field.reflect.getName(), field.type.type);
         }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
index 52f82c9d1ca..f95c24f8592 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
@@ -34,8 +34,8 @@ final class LListShortcut extends ALink {
     Method getter;
     Method setter;
 
-    LListShortcut(int line, String location, AExpression index) {
-        super(line, location, 2);
+    LListShortcut(int line, int offset, String location, AExpression index) {
+        super(line, offset, location, 2);
 
         this.index = index;
     }
@@ -73,31 +73,31 @@ final class LListShortcut extends ALink {
     }
 
     @Override
-    void write(MethodWriter adapter) {
-        index.write(adapter);
+    void write(MethodWriter writer) {
+        index.write(writer);
     }
 
     @Override
-    void load(MethodWriter adapter) {
+    void load(MethodWriter writer) {
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
-            adapter.invokeInterface(getter.owner.type, getter.method);
+            writer.invokeInterface(getter.owner.type, getter.method);
         } else {
-            adapter.invokeVirtual(getter.owner.type, getter.method);
+            writer.invokeVirtual(getter.owner.type, getter.method);
         }
 
         if (!getter.rtn.clazz.equals(getter.handle.type().returnType())) {
-            adapter.checkCast(getter.rtn.type);
+            writer.checkCast(getter.rtn.type);
         }
     }
 
     @Override
-    void store(MethodWriter adapter) {
+    void store(MethodWriter writer) {
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
-            adapter.invokeInterface(setter.owner.type, setter.method);
+            writer.invokeInterface(setter.owner.type, setter.method);
         } else {
-            adapter.invokeVirtual(setter.owner.type, setter.method);
+            writer.invokeVirtual(setter.owner.type, setter.method);
         }
 
-        adapter.writePop(setter.rtn.sort.size);
+        writer.writePop(setter.rtn.sort.size);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
index 9fbc39d72b3..afe93ba1048 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
@@ -34,8 +34,8 @@ final class LMapShortcut extends ALink {
     Method getter;
     Method setter;
 
-    LMapShortcut(int line, String location, AExpression index) {
-        super(line, location, 2);
+    LMapShortcut(int line, int offset, String location, AExpression index) {
+        super(line, offset, location, 2);
 
         this.index = index;
     }
@@ -72,31 +72,31 @@ final class LMapShortcut extends ALink {
     }
 
     @Override
-    void write(MethodWriter adapter) {
-        index.write(adapter);
+    void write(MethodWriter writer) {
+        index.write(writer);
     }
 
     @Override
-    void load(MethodWriter adapter) {
+    void load(MethodWriter writer) {
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
-            adapter.invokeInterface(getter.owner.type, getter.method);
+            writer.invokeInterface(getter.owner.type, getter.method);
         } else {
-            adapter.invokeVirtual(getter.owner.type, getter.method);
+            writer.invokeVirtual(getter.owner.type, getter.method);
         }
 
         if (!getter.rtn.clazz.equals(getter.handle.type().returnType())) {
-            adapter.checkCast(getter.rtn.type);
+            writer.checkCast(getter.rtn.type);
         }
     }
 
     @Override
-    void store(MethodWriter adapter) {
+    void store(MethodWriter writer) {
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
-            adapter.invokeInterface(setter.owner.type, setter.method);
+            writer.invokeInterface(setter.owner.type, setter.method);
         } else {
-            adapter.invokeVirtual(setter.owner.type, setter.method);
+            writer.invokeVirtual(setter.owner.type, setter.method);
         }
 
-        adapter.writePop(setter.rtn.sort.size);
+        writer.writePop(setter.rtn.sort.size);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
index da88b45e72d..ffe6168f812 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
@@ -34,8 +34,8 @@ public final class LNewArray extends ALink {
     final String type;
     final List<AExpression> arguments;
 
-    public LNewArray(int line, String location, String type, List<AExpression> arguments) {
-        super(line, location, -1);
+    public LNewArray(int line, int offset, String location, String type, List<AExpression> arguments) {
+        super(line, offset, location, -1);
 
         this.type = type;
         this.arguments = arguments;
@@ -44,23 +44,23 @@ public final class LNewArray extends ALink {
     @Override
     ALink analyze(Variables variables) {
         if (before != null) {
-            throw new IllegalStateException(error("Illegal tree structure."));
+            throw new IllegalArgumentException(error("Cannot create a new array with a target already defined."));
         } else if (store) {
             throw new IllegalArgumentException(error("Cannot assign a value to a new array."));
         } else if (!load) {
-            throw new IllegalArgumentException(error("A newly created array must be assigned."));
+            throw new IllegalArgumentException(error("A newly created array must be read."));
         }
 
         final Type type;
 
         try {
             type = Definition.getType(this.type);
-        } catch (final IllegalArgumentException exception) {
+        } catch (IllegalArgumentException exception) {
             throw new IllegalArgumentException(error("Not a type [" + this.type + "]."));
         }
 
         for (int argument = 0; argument < arguments.size(); ++argument) {
-            final AExpression expression = arguments.get(argument);
+            AExpression expression = arguments.get(argument);
 
             expression.expected = Definition.INT_TYPE;
             expression.analyze(variables);
@@ -73,25 +73,25 @@ public final class LNewArray extends ALink {
     }
 
     @Override
-    void write(MethodWriter adapter) {
+    void write(MethodWriter writer) {
         // Do nothing.
     }
 
     @Override
-    void load(MethodWriter adapter) {
-        for (final AExpression argument : arguments) {
-            argument.write(adapter);
+    void load(MethodWriter writer) {
+        for (AExpression argument : arguments) {
+            argument.write(writer);
         }
 
         if (arguments.size() > 1) {
-            adapter.visitMultiANewArrayInsn(after.type.getDescriptor(), after.type.getDimensions());
+            writer.visitMultiANewArrayInsn(after.type.getDescriptor(), after.type.getDimensions());
         } else {
-            adapter.newArray(Definition.getType(after.struct, 0).type);
+            writer.newArray(Definition.getType(after.struct, 0).type);
         }
     }
 
     @Override
-    void store(MethodWriter adapter) {
+    void store(MethodWriter writer) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
index 28209e96f2b..dc8b711ee8c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
@@ -38,8 +38,8 @@ public final class LNewObj extends ALink {
 
     Constructor constructor;
 
-    public LNewObj(int line, String location, String type, List<AExpression> arguments) {
-        super(line, location, -1);
+    public LNewObj(int line, int offset, String location, String type, List<AExpression> arguments) {
+        super(line, offset, location, -1);
 
         this.type = type;
         this.arguments = arguments;
@@ -48,7 +48,7 @@ public final class LNewObj extends ALink {
     @Override
     ALink analyze(Variables variables) {
         if (before != null) {
-            throw new IllegalStateException(error("Illegal tree structure"));
+            throw new IllegalArgumentException(error("Illegal new call with a target already defined."));
         } else if (store) {
             throw new IllegalArgumentException(error("Cannot assign a value to a new call."));
         }
@@ -57,15 +57,15 @@ public final class LNewObj extends ALink {
 
         try {
             type = Definition.getType(this.type);
-        } catch (final IllegalArgumentException exception) {
+        } catch (IllegalArgumentException exception) {
             throw new IllegalArgumentException(error("Not a type [" + this.type + "]."));
         }
 
-        final Struct struct = type.struct;
+        Struct struct = type.struct;
         constructor = struct.constructors.get(new Definition.MethodKey("new", arguments.size()));
 
         if (constructor != null) {
-            final Type[] types = new Type[constructor.arguments.size()];
+            Type[] types = new Type[constructor.arguments.size()];
             constructor.arguments.toArray(types);
 
             if (constructor.arguments.size() != arguments.size()) {
@@ -74,7 +74,7 @@ public final class LNewObj extends ALink {
             }
 
             for (int argument = 0; argument < arguments.size(); ++argument) {
-                final AExpression expression = arguments.get(argument);
+                AExpression expression = arguments.get(argument);
 
                 expression.expected = types[argument];
                 expression.internal = true;
@@ -92,27 +92,27 @@ public final class LNewObj extends ALink {
     }
 
     @Override
-    void write(MethodWriter adapter) {
+    void write(MethodWriter writer) {
         // Do nothing.
     }
 
     @Override
-    void load(MethodWriter adapter) {
-        adapter.newInstance(after.type);
+    void load(MethodWriter writer) {
+        writer.newInstance(after.type);
 
         if (load) {
-            adapter.dup();
+            writer.dup();
         }
 
         for (AExpression argument : arguments) {
-            argument.write(adapter);
+            argument.write(writer);
         }
 
-        adapter.invokeConstructor(constructor.owner.type, constructor.method);
+        writer.invokeConstructor(constructor.owner.type, constructor.method);
     }
 
     @Override
-    void store(MethodWriter adapter) {
+    void store(MethodWriter writer) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
index 85a7d70c46e..c6aafd47bff 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
@@ -36,17 +36,22 @@ final class LShortcut extends ALink {
     Method getter = null;
     Method setter = null;
 
-    LShortcut(int line, String location, String value) {
-        super(line, location, 1);
+    LShortcut(int line, int offset, String location, String value) {
+        super(line, offset, location, 1);
 
         this.value = value;
     }
 
     @Override
     ALink analyze(Variables variables) {
-        final Struct struct = before.struct;
+        Struct struct = before.struct;
 
         getter = struct.methods.get(new Definition.MethodKey("get" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0));
+
+        if (getter == null) {
+            getter = struct.methods.get(new Definition.MethodKey("is" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0));
+        }
+
         setter = struct.methods.get(new Definition.MethodKey("set" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 1));
 
         if (getter != null && (getter.rtn.sort == Sort.VOID || !getter.arguments.isEmpty())) {
@@ -73,31 +78,31 @@ final class LShortcut extends ALink {
     }
 
     @Override
-    void write(MethodWriter adapter) {
+    void write(MethodWriter writer) {
         // Do nothing.
     }
 
     @Override
-    void load(MethodWriter adapter) {
+    void load(MethodWriter writer) {
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
-            adapter.invokeInterface(getter.owner.type, getter.method);
+            writer.invokeInterface(getter.owner.type, getter.method);
         } else {
-            adapter.invokeVirtual(getter.owner.type, getter.method);
+            writer.invokeVirtual(getter.owner.type, getter.method);
         }
 
         if (!getter.rtn.clazz.equals(getter.handle.type().returnType())) {
-            adapter.checkCast(getter.rtn.type);
+            writer.checkCast(getter.rtn.type);
         }
     }
 
     @Override
-    void store(MethodWriter adapter) {
+    void store(MethodWriter writer) {
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
-            adapter.invokeInterface(setter.owner.type, setter.method);
+            writer.invokeInterface(setter.owner.type, setter.method);
         } else {
-            adapter.invokeVirtual(setter.owner.type, setter.method);
+            writer.invokeVirtual(setter.owner.type, setter.method);
         }
 
-        adapter.writePop(setter.rtn.sort.size);
+        writer.writePop(setter.rtn.sort.size);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java
new file mode 100644
index 00000000000..8919b75dc27
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.Variables;
+
+/**
+ * Represents a static type target.
+ */
+public final class LStatic extends ALink {
+
+    final String type;
+
+    public LStatic(int line, int offset, String location, String type) {
+        super(line, offset, location, 0);
+
+        this.type = type;
+    }
+
+    @Override
+    ALink analyze(Variables variables) {
+        if (before != null) {
+            throw new IllegalArgumentException(error("Illegal static type [" + type + "] after target already defined."));
+        }
+
+        try {
+            after = Definition.getType(type);
+            statik = true;
+        } catch (IllegalArgumentException exception) {
+            throw new IllegalArgumentException(error("Not a type [" + type + "]."));
+        }
+
+        return this;
+    }
+
+    @Override
+    void write(MethodWriter writer) {
+        throw new IllegalStateException(error("Illegal tree structure."));
+    }
+
+    @Override
+    void load(MethodWriter writer) {
+        throw new IllegalStateException(error("Illegal tree structure."));
+    }
+
+    @Override
+    void store(MethodWriter writer) {
+        throw new IllegalStateException(error("Illegal tree structure."));
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
index 07d76d42676..446750a22bb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
@@ -28,8 +28,8 @@ import org.elasticsearch.painless.MethodWriter;
  */
 public final class LString extends ALink {
 
-    public LString(int line, String location, String string) {
-        super(line, location, -1);
+    public LString(int line, int offset, String location, String string) {
+        super(line, offset, location, -1);
 
         this.string = string;
     }
@@ -37,7 +37,7 @@ public final class LString extends ALink {
     @Override
     ALink analyze(Variables variables) {
         if (before != null) {
-            throw new IllegalStateException("Illegal tree structure.");
+            throw new IllegalArgumentException(error("Illegal String constant [" + string + "]."));
         } else if (store) {
             throw new IllegalArgumentException(error("Cannot write to read-only String constant [" + string + "]."));
         } else if (!load) {
@@ -50,17 +50,17 @@ public final class LString extends ALink {
     }
 
     @Override
-    void write(MethodWriter adapter) {
+    void write(MethodWriter writer) {
         // Do nothing.
     }
 
     @Override
-    void load(MethodWriter adapter) {
-        adapter.push(string);
+    void load(MethodWriter writer) {
+        writer.push(string);
     }
 
     @Override
-    void store(MethodWriter adapter) {
+    void store(MethodWriter writer) {
         throw new IllegalStateException(error("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
index 85a2048bdc6..3a555d93c87 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
@@ -19,12 +19,10 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.Definition;
-import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.Variables.Variable;
 import org.objectweb.asm.Opcodes;
-import org.elasticsearch.painless.MethodWriter;
 
 /**
  * Represents a variable load/store.
@@ -35,8 +33,8 @@ public final class LVariable extends ALink {
 
     int slot;
 
-    public LVariable(int line, String location, String name) {
-        super(line, location, 0);
+    public LVariable(int line, int offset, String location, String name) {
+        super(line, offset, location, 0);
 
         this.name = name;
     }
@@ -44,46 +42,33 @@ public final class LVariable extends ALink {
     @Override
     ALink analyze(Variables variables) {
         if (before != null) {
-            throw new IllegalStateException(error("Illegal tree structure."));
+            throw new IllegalArgumentException(error("Illegal variable [" + name + "] access with target already defined."));
         }
 
-        Type type = null;
+        Variable variable = variables.getVariable(location, name);
 
-        try {
-            type = Definition.getType(name);
-        } catch (final IllegalArgumentException exception) {
-            // Do nothing.
+        if (store && variable.readonly) {
+            throw new IllegalArgumentException(error("Variable [" + variable.name + "] is read-only."));
         }
 
-        if (type != null) {
-            statik = true;
-            after = type;
-        } else {
-            final Variable variable = variables.getVariable(location, name);
-
-            if (store && variable.readonly) {
-                throw new IllegalArgumentException(error("Variable [" + variable.name + "] is read-only."));
-            }
-
-            slot = variable.slot;
-            after = variable.type;
-        }
+        slot = variable.slot;
+        after = variable.type;
 
         return this;
     }
 
     @Override
-    void write(MethodWriter adapter) {
+    void write(MethodWriter writer) {
         // Do nothing.
     }
 
     @Override
-    void load(MethodWriter adapter) {
-        adapter.visitVarInsn(after.type.getOpcode(Opcodes.ILOAD), slot);
+    void load(MethodWriter writer) {
+        writer.visitVarInsn(after.type.getOpcode(Opcodes.ILOAD), slot);
     }
 
     @Override
-    void store(MethodWriter adapter) {
-        adapter.visitVarInsn(after.type.getOpcode(Opcodes.ISTORE), slot);
+    void store(MethodWriter writer) {
+        writer.visitVarInsn(after.type.getOpcode(Opcodes.ISTORE), slot);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
index a925c411a2a..0f05243f994 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
@@ -32,14 +32,18 @@ public final class SBlock extends AStatement {
 
     final List<AStatement> statements;
 
-    public SBlock(int line, String location, List<AStatement> statements) {
-        super(line, location);
+    public SBlock(int line, int offset, String location, List<AStatement> statements) {
+        super(line, offset, location);
 
         this.statements = Collections.unmodifiableList(statements);
     }
 
     @Override
     void analyze(Variables variables) {
+        if (statements == null || statements.isEmpty()) {
+            throw new IllegalArgumentException(error("A block must contain at least one statement."));
+        }
+
         final AStatement last = statements.get(statements.size() - 1);
 
         for (AStatement statement : statements) {
@@ -63,11 +67,11 @@ public final class SBlock extends AStatement {
     }
 
     @Override
-    void write(MethodWriter adapter) {
+    void write(MethodWriter writer) {
         for (AStatement statement : statements) {
             statement.continu = continu;
             statement.brake = brake;
-            statement.write(adapter);
+            statement.write(writer);
         }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
index fb12a2b452e..727cd4b929c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
@@ -27,8 +27,8 @@ import org.elasticsearch.painless.MethodWriter;
  */
 public final class SBreak extends AStatement {
 
-    public SBreak(int line, String location) {
-        super(line, location);
+    public SBreak(int line, int offset, String location) {
+        super(line, offset, location);
     }
 
     @Override
@@ -44,8 +44,9 @@ public final class SBreak extends AStatement {
     }
 
     @Override
-    void write(MethodWriter adapter) {
-        writeDebugInfo(adapter);
-        adapter.goTo(brake);
+    void write(MethodWriter writer) {
+        writeDebugInfo(writer);
+
+        writer.goTo(brake);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
similarity index 68%
rename from modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java
rename to modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
index 9a791b1615d..b3b32d741cd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.painless.node;
 
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.Variables.Variable;
 import org.objectweb.asm.Label;
@@ -28,11 +30,11 @@ import org.elasticsearch.painless.MethodWriter;
 /**
  * Represents a catch block as part of a try-catch block.
  */
-public final class STrap extends AStatement {
+public final class SCatch extends AStatement {
 
     final String type;
     final String name;
-    final AStatement block;
+    final SBlock block;
 
     Variable variable;
 
@@ -40,8 +42,8 @@ public final class STrap extends AStatement {
     Label end;
     Label exception;
 
-    public STrap(int line, String location, String type, String name, AStatement block) {
-        super(line, location);
+    public SCatch(int line, int offset, String location, String type, String name, SBlock block) {
+        super(line, offset, location);
 
         this.type = type;
         this.name = name;
@@ -50,12 +52,20 @@ public final class STrap extends AStatement {
 
     @Override
     void analyze(Variables variables) {
-        variable = variables.addVariable(location, type, name, true, false);
+        final Type type;
 
-        if (!Exception.class.isAssignableFrom(variable.type.clazz)) {
-            throw new ClassCastException(error("Not an exception type [" + variable.type.name + "]."));
+        try {
+            type = Definition.getType(this.type);
+        } catch (IllegalArgumentException exception) {
+            throw new IllegalArgumentException(error("Not a type [" + this.type + "]."));
         }
 
+        if (!Exception.class.isAssignableFrom(type.clazz)) {
+            throw new ClassCastException(error("Not an exception type [" + this.type + "]."));
+        }
+
+        variable = variables.addVariable(location, type, name, true, false);
+
         if (block != null) {
             block.lastSource = lastSource;
             block.inLoop = inLoop;
@@ -73,23 +83,24 @@ public final class STrap extends AStatement {
     }
 
     @Override
-    void write(MethodWriter adapter) {
-        writeDebugInfo(adapter);
-        final Label jump = new Label();
+    void write(MethodWriter writer) {
+        writeDebugInfo(writer);
 
-        adapter.mark(jump);
-        adapter.visitVarInsn(variable.type.type.getOpcode(Opcodes.ISTORE), variable.slot);
+        Label jump = new Label();
+
+        writer.mark(jump);
+        writer.visitVarInsn(variable.type.type.getOpcode(Opcodes.ISTORE), variable.slot);
 
         if (block != null) {
             block.continu = continu;
             block.brake = brake;
-            block.write(adapter);
+            block.write(writer);
         }
 
-        adapter.visitTryCatchBlock(begin, end, jump, variable.type.type.getInternalName());
+        writer.visitTryCatchBlock(begin, end, jump, variable.type.type.getInternalName());
 
         if (exception != null && !block.allEscape) {
-            adapter.goTo(exception);
+            writer.goTo(exception);
         }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
index 01ca3f5697d..bfea104f23d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
@@ -27,8 +27,8 @@ import org.elasticsearch.painless.MethodWriter;
  */
 public final class SContinue extends AStatement {
 
-    public SContinue(int line, String location) {
-        super(line, location);
+    public SContinue(int line, int offset, String location) {
+        super(line, offset, location);
     }
 
     @Override
@@ -47,8 +47,9 @@ public final class SContinue extends AStatement {
     }
 
     @Override
-    void write(MethodWriter adapter) {
-        writeDebugInfo(adapter);
-        adapter.goTo(continu);
+    void write(MethodWriter writer) {
+        writeDebugInfo(writer);
+
+        writer.goTo(continu);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
index 917dd9b8e5a..ddfe54c22eb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
@@ -32,15 +32,15 @@ public final class SDeclBlock extends AStatement {
 
     final List<SDeclaration> declarations;
 
-    public SDeclBlock(int line, String location, List<SDeclaration> declarations) {
-        super(line, location);
+    public SDeclBlock(int line, int offset, String location, List<SDeclaration> declarations) {
+        super(line, offset, location);
 
         this.declarations = Collections.unmodifiableList(declarations);
     }
 
     @Override
     void analyze(Variables variables) {
-        for (final SDeclaration declaration : declarations) {
+        for (SDeclaration declaration : declarations) {
             declaration.analyze(variables);
         }
 
@@ -48,9 +48,9 @@ public final class SDeclBlock extends AStatement {
     }
 
     @Override
-    void write(MethodWriter adapter) {
-        for (final SDeclaration declaration : declarations) {
-            declaration.write(adapter);
+    void write(MethodWriter writer) {
+        for (SDeclaration declaration : declarations) {
+            declaration.write(writer);
         }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
index 0e366dfc19b..246fb992836 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
@@ -19,7 +19,8 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.Variables.Variable;
 import org.objectweb.asm.Opcodes;
@@ -36,8 +37,8 @@ public final class SDeclaration extends AStatement {
 
     Variable variable;
 
-    public SDeclaration(int line, String location, String type, String name, AExpression expression) {
-        super(line, location);
+    public SDeclaration(int line, int offset, String location, String type, String name, AExpression expression) {
+        super(line, offset, location);
 
         this.type = type;
         this.name = name;
@@ -46,40 +47,44 @@ public final class SDeclaration extends AStatement {
 
     @Override
     void analyze(Variables variables) {
-        variable = variables.addVariable(location, type, name, false, false);
+        final Type type;
+
+        try {
+            type = Definition.getType(this.type);
+        } catch (IllegalArgumentException exception) {
+            throw new IllegalArgumentException(error("Not a type [" + this.type + "]."));
+        }
 
         if (expression != null) {
-            expression.expected = variable.type;
+            expression.expected = type;
             expression.analyze(variables);
             expression = expression.cast(variables);
         }
+
+        variable = variables.addVariable(location, type, name, false, false);
     }
 
     @Override
-    void write(MethodWriter adapter) {
-        writeDebugInfo(adapter);
-        final org.objectweb.asm.Type type = variable.type.type;
-        final Sort sort = variable.type.sort;
+    void write(MethodWriter writer) {
+        writeDebugInfo(writer);
 
-        final boolean initialize = expression == null;
-
-        if (!initialize) {
-            expression.write(adapter);
+        if (expression == null) {
+            switch (variable.type.sort) {
+                case VOID:   throw new IllegalStateException(error("Illegal tree structure."));
+                case BOOL:
+                case BYTE:
+                case SHORT:
+                case CHAR:
+                case INT:    writer.push(0);    break;
+                case LONG:   writer.push(0L);   break;
+                case FLOAT:  writer.push(0.0F); break;
+                case DOUBLE: writer.push(0.0);  break;
+                default:     writer.visitInsn(Opcodes.ACONST_NULL);
+            }
+        } else {
+            expression.write(writer);
         }
 
-        switch (sort) {
-            case VOID:   throw new IllegalStateException(error("Illegal tree structure."));
-            case BOOL:
-            case BYTE:
-            case SHORT:
-            case CHAR:
-            case INT:    if (initialize) adapter.push(0);    break;
-            case LONG:   if (initialize) adapter.push(0L);   break;
-            case FLOAT:  if (initialize) adapter.push(0.0F); break;
-            case DOUBLE: if (initialize) adapter.push(0.0);  break;
-            default:     if (initialize) adapter.visitInsn(Opcodes.ACONST_NULL);
-        }
-
-        adapter.visitVarInsn(type.getOpcode(Opcodes.ISTORE), variable.slot);
+        writer.visitVarInsn(variable.type.type.getOpcode(Opcodes.ISTORE), variable.slot);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
index 6c8785a923a..3fbc3b9fafc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -29,12 +29,12 @@ import org.elasticsearch.painless.MethodWriter;
  */
 public final class SDo extends AStatement {
 
-    final AStatement block;
-    AExpression condition;
     final int maxLoopCounter;
+    final SBlock block;
+    AExpression condition;
 
-    public SDo(int line, String location, AStatement block, AExpression condition, int maxLoopCounter) {
-        super(line, location);
+    public SDo(int line, int offset, String location, int maxLoopCounter, SBlock block, AExpression condition) {
+        super(line, offset, location);
 
         this.condition = condition;
         this.block = block;
@@ -45,6 +45,10 @@ public final class SDo extends AStatement {
     void analyze(Variables variables) {
         variables.incrementScope();
 
+        if (block == null) {
+            throw new IllegalArgumentException(error("Extraneous do while loop."));
+        }
+
         block.beginLoop = true;
         block.inLoop = true;
 
@@ -81,26 +85,27 @@ public final class SDo extends AStatement {
     }
 
     @Override
-    void write(MethodWriter adapter) {
-        writeDebugInfo(adapter);
-        final Label start = new Label();
-        final Label begin = new Label();
-        final Label end = new Label();
+    void write(MethodWriter writer) {
+        writeDebugInfo(writer);
 
-        adapter.mark(start);
+        Label start = new Label();
+        Label begin = new Label();
+        Label end = new Label();
+
+        writer.mark(start);
 
         block.continu = begin;
         block.brake = end;
-        block.write(adapter);
+        block.write(writer);
 
-        adapter.mark(begin);
+        writer.mark(begin);
 
         condition.fals = end;
-        condition.write(adapter);
+        condition.write(writer);
 
-        adapter.writeLoopCounter(loopCounterSlot, Math.max(1, block.statementCount));
+        writer.writeLoopCounter(loopCounterSlot, Math.max(1, block.statementCount));
 
-        adapter.goTo(start);
-        adapter.mark(end);
+        writer.goTo(start);
+        writer.mark(end);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index 9d2bc6385a5..a9f47fb6522 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -31,8 +31,8 @@ public final class SExpression extends AStatement {
 
     AExpression expression;
 
-    public SExpression(int line, String location, AExpression expression) {
-        super(line, location);
+    public SExpression(int line, int offset, String location, AExpression expression) {
+        super(line, offset, location);
 
         this.expression = expression;
     }
@@ -59,14 +59,15 @@ public final class SExpression extends AStatement {
     }
 
     @Override
-    void write(MethodWriter adapter) {
-        writeDebugInfo(adapter);
-        expression.write(adapter);
+    void write(MethodWriter writer) {
+        writeDebugInfo(writer);
+
+        expression.write(writer);
 
         if (methodEscape) {
-            adapter.returnValue();
+            writer.returnValue();
         } else {
-            adapter.writePop(expression.expected.sort.size);
+            writer.writePop(expression.expected.sort.size);
         }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index 56073e6e8db..5fb1845e551 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -29,15 +29,15 @@ import org.elasticsearch.painless.MethodWriter;
  */
 public final class SFor extends AStatement {
 
+    final int maxLoopCounter;
     ANode initializer;
     AExpression condition;
     AExpression afterthought;
-    final AStatement block;
-    final int maxLoopCounter;
+    final SBlock block;
 
-    public SFor(int line, String location,
-                ANode initializer, AExpression condition, AExpression afterthought, AStatement block, int maxLoopCounter) {
-        super(line, location);
+    public SFor(int line, int offset, String location, int maxLoopCounter,
+                ANode initializer, AExpression condition, AExpression afterthought, SBlock block) {
+        super(line, offset, location);
 
         this.initializer = initializer;
         this.condition = condition;
@@ -56,7 +56,7 @@ public final class SFor extends AStatement {
             if (initializer instanceof SDeclBlock) {
                 ((SDeclBlock)initializer).analyze(variables);
             } else if (initializer instanceof AExpression) {
-                final AExpression initializer = (AExpression)this.initializer;
+                AExpression initializer = (AExpression)this.initializer;
 
                 initializer.read = false;
                 initializer.analyze(variables);
@@ -70,7 +70,6 @@ public final class SFor extends AStatement {
         }
 
         if (condition != null) {
-
             condition.expected = Definition.BOOLEAN_TYPE;
             condition.analyze(variables);
             condition = condition.cast(variables);
@@ -99,8 +98,6 @@ public final class SFor extends AStatement {
             }
         }
 
-        int count = 1;
-
         if (block != null) {
             block.beginLoop = true;
             block.inLoop = true;
@@ -116,7 +113,7 @@ public final class SFor extends AStatement {
                 allEscape = true;
             }
 
-            block.statementCount = Math.max(count, block.statementCount);
+            block.statementCount = Math.max(1, block.statementCount);
         }
 
         statementCount = 1;
@@ -129,26 +126,27 @@ public final class SFor extends AStatement {
     }
 
     @Override
-    void write(MethodWriter adapter) {
-        writeDebugInfo(adapter);
-        final Label start = new Label();
-        final Label begin = afterthought == null ? start : new Label();
-        final Label end = new Label();
+    void write(MethodWriter writer) {
+        writeDebugInfo(writer);
+
+        Label start = new Label();
+        Label begin = afterthought == null ? start : new Label();
+        Label end = new Label();
 
         if (initializer instanceof SDeclBlock) {
-            ((SDeclBlock)initializer).write(adapter);
+            ((SDeclBlock)initializer).write(writer);
         } else if (initializer instanceof AExpression) {
             AExpression initializer = (AExpression)this.initializer;
 
-            initializer.write(adapter);
-            adapter.writePop(initializer.expected.sort.size);
+            initializer.write(writer);
+            writer.writePop(initializer.expected.sort.size);
         }
 
-        adapter.mark(start);
+        writer.mark(start);
 
         if (condition != null) {
             condition.fals = end;
-            condition.write(adapter);
+            condition.write(writer);
         }
 
         boolean allEscape = false;
@@ -162,21 +160,21 @@ public final class SFor extends AStatement {
                 ++statementCount;
             }
 
-            adapter.writeLoopCounter(loopCounterSlot, statementCount);
-            block.write(adapter);
+            writer.writeLoopCounter(loopCounterSlot, statementCount);
+            block.write(writer);
         } else {
-            adapter.writeLoopCounter(loopCounterSlot, 1);
+            writer.writeLoopCounter(loopCounterSlot, 1);
         }
 
         if (afterthought != null) {
-            adapter.mark(begin);
-            afterthought.write(adapter);
+            writer.mark(begin);
+            afterthought.write(writer);
         }
 
         if (afterthought != null || !allEscape) {
-            adapter.goTo(start);
+            writer.goTo(start);
         }
 
-        adapter.mark(end);
+        writer.mark(end);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
new file mode 100644
index 00000000000..180b5023811
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.Label;
+import org.elasticsearch.painless.MethodWriter;
+
+/**
+ * Represents an if block.
+ */
+public final class SIf extends AStatement {
+
+    AExpression condition;
+    final SBlock ifblock;
+
+    public SIf(int line, int offset, String location, AExpression condition, SBlock ifblock) {
+        super(line, offset, location);
+
+        this.condition = condition;
+        this.ifblock = ifblock;
+    }
+
+    @Override
+    void analyze(Variables variables) {
+        condition.expected = Definition.BOOLEAN_TYPE;
+        condition.analyze(variables);
+        condition = condition.cast(variables);
+
+        if (condition.constant != null) {
+            throw new IllegalArgumentException(error("Extraneous if statement."));
+        }
+
+        if (ifblock == null) {
+            throw new IllegalArgumentException(error("Extraneous if statement."));
+        }
+
+        ifblock.lastSource = lastSource;
+        ifblock.inLoop = inLoop;
+        ifblock.lastLoop = lastLoop;
+
+        variables.incrementScope();
+        ifblock.analyze(variables);
+        variables.decrementScope();
+
+        anyContinue = ifblock.anyContinue;
+        anyBreak = ifblock.anyBreak;
+        statementCount = ifblock.statementCount;
+    }
+
+    @Override
+    void write(MethodWriter writer) {
+        writeDebugInfo(writer);
+
+        Label fals = new Label();
+
+        condition.fals = fals;
+        condition.write(writer);
+
+        ifblock.continu = continu;
+        ifblock.brake = brake;
+        ifblock.write(writer);
+
+        writer.mark(fals);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
index 560ce44e415..217584f32ab 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
@@ -30,11 +30,11 @@ import org.elasticsearch.painless.MethodWriter;
 public final class SIfElse extends AStatement {
 
     AExpression condition;
-    final AStatement ifblock;
-    final AStatement elseblock;
+    final SBlock ifblock;
+    final SBlock elseblock;
 
-    public SIfElse(int line, String location, AExpression condition, AStatement ifblock, AStatement elseblock) {
-        super(line, location);
+    public SIfElse(int line, int offset, String location, AExpression condition, SBlock ifblock, SBlock elseblock) {
+        super(line, offset, location);
 
         this.condition = condition;
         this.ifblock = ifblock;
@@ -51,6 +51,10 @@ public final class SIfElse extends AStatement {
             throw new IllegalArgumentException(error("Extraneous if statement."));
         }
 
+        if (ifblock == null) {
+            throw new IllegalArgumentException(error("Extraneous if statement."));
+        }
+
         ifblock.lastSource = lastSource;
         ifblock.inLoop = inLoop;
         ifblock.lastLoop = lastLoop;
@@ -63,49 +67,50 @@ public final class SIfElse extends AStatement {
         anyBreak = ifblock.anyBreak;
         statementCount = ifblock.statementCount;
 
-        if (elseblock != null) {
-            elseblock.lastSource = lastSource;
-            elseblock.inLoop = inLoop;
-            elseblock.lastLoop = lastLoop;
-
-            variables.incrementScope();
-            elseblock.analyze(variables);
-            variables.decrementScope();
-
-            methodEscape = ifblock.methodEscape && elseblock.methodEscape;
-            loopEscape = ifblock.loopEscape && elseblock.loopEscape;
-            allEscape = ifblock.allEscape && elseblock.allEscape;
-            anyContinue |= elseblock.anyContinue;
-            anyBreak |= elseblock.anyBreak;
-            statementCount = Math.max(ifblock.statementCount, elseblock.statementCount);
+        if (elseblock == null) {
+            throw new IllegalArgumentException(error("Extraneous else statement."));
         }
+
+        elseblock.lastSource = lastSource;
+        elseblock.inLoop = inLoop;
+        elseblock.lastLoop = lastLoop;
+
+        variables.incrementScope();
+        elseblock.analyze(variables);
+        variables.decrementScope();
+
+        methodEscape = ifblock.methodEscape && elseblock.methodEscape;
+        loopEscape = ifblock.loopEscape && elseblock.loopEscape;
+        allEscape = ifblock.allEscape && elseblock.allEscape;
+        anyContinue |= elseblock.anyContinue;
+        anyBreak |= elseblock.anyBreak;
+        statementCount = Math.max(ifblock.statementCount, elseblock.statementCount);
     }
 
     @Override
-    void write(MethodWriter adapter) {
-        writeDebugInfo(adapter);
-        final Label end = new Label();
-        final Label fals = elseblock != null ? new Label() : end;
+    void write(MethodWriter writer) {
+        writeDebugInfo(writer);
+
+        Label end = new Label();
+        Label fals = elseblock != null ? new Label() : end;
 
         condition.fals = fals;
-        condition.write(adapter);
+        condition.write(writer);
 
         ifblock.continu = continu;
         ifblock.brake = brake;
-        ifblock.write(adapter);
+        ifblock.write(writer);
 
-        if (elseblock != null) {
-            if (!ifblock.allEscape) {
-                adapter.goTo(end);
-            }
-
-            adapter.mark(fals);
-
-            elseblock.continu = continu;
-            elseblock.brake = brake;
-            elseblock.write(adapter);
+        if (!ifblock.allEscape) {
+            writer.goTo(end);
         }
 
-        adapter.mark(end);
+        writer.mark(fals);
+
+        elseblock.continu = continu;
+        elseblock.brake = brake;
+        elseblock.write(writer);
+
+        writer.mark(end);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
index ecdf5c7b69a..2f342a5b7b9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -30,8 +30,8 @@ public final class SReturn extends AStatement {
 
     AExpression expression;
 
-    public SReturn(int line, String location, AExpression expression) {
-        super(line, location);
+    public SReturn(int line, int offset, String location, AExpression expression) {
+        super(line, offset, location);
 
         this.expression = expression;
     }
@@ -51,9 +51,10 @@ public final class SReturn extends AStatement {
     }
 
     @Override
-    void write(MethodWriter adapter) {
-        writeDebugInfo(adapter);
-        expression.write(adapter);
-        adapter.returnValue();
+    void write(MethodWriter writer) {
+        writeDebugInfo(writer);
+
+        expression.write(writer);
+        writer.returnValue();
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index 0932ae6e2f0..899d54225dc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -33,19 +33,23 @@ public final class SSource extends AStatement {
 
     final List<AStatement> statements;
 
-    public SSource(int line, String location, List<AStatement> statements) {
-        super(line, location);
+    public SSource(int line, int offset, String location, List<AStatement> statements) {
+        super(line, offset, location);
 
         this.statements = Collections.unmodifiableList(statements);
     }
 
     @Override
     public void analyze(Variables variables) {
+        if (statements == null || statements.isEmpty()) {
+            throw new IllegalArgumentException(error("Cannot generate an empty script."));
+        }
+
         variables.incrementScope();
 
         final AStatement last = statements.get(statements.size() - 1);
 
-        for (final AStatement statement : statements) {
+        for (AStatement statement : statements) {
             if (allEscape) {
                 throw new IllegalArgumentException(error("Unreachable statement."));
             }
@@ -61,14 +65,14 @@ public final class SSource extends AStatement {
     }
 
     @Override
-    public void write(MethodWriter adapter) {
-        for (final AStatement statement : statements) {
-            statement.write(adapter);
+    public void write(MethodWriter writer) {
+        for (AStatement statement : statements) {
+            statement.write(writer);
         }
 
         if (!methodEscape) {
-            adapter.visitInsn(Opcodes.ACONST_NULL);
-            adapter.returnValue();
+            writer.visitInsn(Opcodes.ACONST_NULL);
+            writer.returnValue();
         }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
index a0bfa12e267..d002f1f9dad 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
@@ -30,8 +30,8 @@ public final class SThrow extends AStatement {
 
     AExpression expression;
 
-    public SThrow(int line, String location, AExpression expression) {
-        super(line, location);
+    public SThrow(int line, int offset, String location, AExpression expression) {
+        super(line, offset, location);
 
         this.expression = expression;
     }
@@ -49,9 +49,10 @@ public final class SThrow extends AStatement {
     }
 
     @Override
-    void write(MethodWriter adapter) {
-        writeDebugInfo(adapter);
-        expression.write(adapter);
-        adapter.throwException();
+    void write(MethodWriter writer) {
+        writeDebugInfo(writer);
+
+        expression.write(writer);
+        writer.throwException();
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
index 4f97d5835cc..a4ef00ef146 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
@@ -31,18 +31,22 @@ import java.util.List;
  */
 public final class STry extends AStatement {
 
-    final AStatement block;
-    final List<STrap> traps;
+    final SBlock block;
+    final List<SCatch> catches;
 
-    public STry(int line, String location, AStatement block, List<STrap> traps) {
-        super(line, location);
+    public STry(int line, int offset, String location, SBlock block, List<SCatch> traps) {
+        super(line, offset, location);
 
         this.block = block;
-        this.traps = Collections.unmodifiableList(traps);
+        this.catches = Collections.unmodifiableList(traps);
     }
 
     @Override
     void analyze(Variables variables) {
+        if (block == null) {
+            throw new IllegalArgumentException(error("Extraneous try statement."));
+        }
+
         block.lastSource = lastSource;
         block.inLoop = inLoop;
         block.lastLoop = lastLoop;
@@ -59,55 +63,56 @@ public final class STry extends AStatement {
 
         int statementCount = 0;
 
-        for (final STrap trap : traps) {
-            trap.lastSource = lastSource;
-            trap.inLoop = inLoop;
-            trap.lastLoop = lastLoop;
+        for (SCatch catc : catches) {
+            catc.lastSource = lastSource;
+            catc.inLoop = inLoop;
+            catc.lastLoop = lastLoop;
 
             variables.incrementScope();
-            trap.analyze(variables);
+            catc.analyze(variables);
             variables.decrementScope();
 
-            methodEscape &= trap.methodEscape;
-            loopEscape &= trap.loopEscape;
-            allEscape &= trap.allEscape;
-            anyContinue |= trap.anyContinue;
-            anyBreak |= trap.anyBreak;
+            methodEscape &= catc.methodEscape;
+            loopEscape &= catc.loopEscape;
+            allEscape &= catc.allEscape;
+            anyContinue |= catc.anyContinue;
+            anyBreak |= catc.anyBreak;
 
-            statementCount = Math.max(statementCount, trap.statementCount);
+            statementCount = Math.max(statementCount, catc.statementCount);
         }
 
         this.statementCount = block.statementCount + statementCount;
     }
 
     @Override
-    void write(MethodWriter adapter) {
-        writeDebugInfo(adapter);
-        final Label begin = new Label();
-        final Label end = new Label();
-        final Label exception = new Label();
+    void write(MethodWriter writer) {
+        writeDebugInfo(writer);
 
-        adapter.mark(begin);
+        Label begin = new Label();
+        Label end = new Label();
+        Label exception = new Label();
+
+        writer.mark(begin);
 
         block.continu = continu;
         block.brake = brake;
-        block.write(adapter);
+        block.write(writer);
 
         if (!block.allEscape) {
-            adapter.goTo(exception);
+            writer.goTo(exception);
         }
 
-        adapter.mark(end);
+        writer.mark(end);
 
-        for (final STrap trap : traps) {
-            trap.begin = begin;
-            trap.end = end;
-            trap.exception = traps.size() > 1 ? exception : null;
-            trap.write(adapter);
+        for (SCatch catc : catches) {
+            catc.begin = begin;
+            catc.end = end;
+            catc.exception = catches.size() > 1 ? exception : null;
+            catc.write(writer);
         }
 
-        if (!block.allEscape || traps.size() > 1) {
-            adapter.mark(exception);
+        if (!block.allEscape || catches.size() > 1) {
+            writer.mark(exception);
         }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
index 48f59c55189..322ae110f66 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -29,16 +29,16 @@ import org.elasticsearch.painless.MethodWriter;
  */
 public final class SWhile extends AStatement {
 
-    AExpression condition;
-    final AStatement block;
     final int maxLoopCounter;
+    AExpression condition;
+    final SBlock block;
 
-    public SWhile(int line, String location, AExpression condition, AStatement block, int maxLoopCounter) {
-        super(line, location);
+    public SWhile(int line, int offset, String location, int maxLoopCounter, AExpression condition, SBlock block) {
+        super(line, offset, location);
 
+        this.maxLoopCounter = maxLoopCounter;
         this.condition = condition;
         this.block = block;
-        this.maxLoopCounter = maxLoopCounter;
     }
 
     @Override
@@ -63,8 +63,6 @@ public final class SWhile extends AStatement {
             }
         }
 
-        int count = 1;
-
         if (block != null) {
             block.beginLoop = true;
             block.inLoop = true;
@@ -72,7 +70,7 @@ public final class SWhile extends AStatement {
             block.analyze(variables);
 
             if (block.loopEscape && !block.anyContinue) {
-                throw new IllegalArgumentException(error("Extranous while loop."));
+                throw new IllegalArgumentException(error("Extraneous while loop."));
             }
 
             if (continuous && !block.anyBreak) {
@@ -80,7 +78,7 @@ public final class SWhile extends AStatement {
                 allEscape = true;
             }
 
-            block.statementCount = Math.max(count, block.statementCount);
+            block.statementCount = Math.max(1, block.statementCount);
         }
 
         statementCount = 1;
@@ -93,30 +91,31 @@ public final class SWhile extends AStatement {
     }
 
     @Override
-    void write(MethodWriter adapter) {
-        writeDebugInfo(adapter);
-        final Label begin = new Label();
-        final Label end = new Label();
+    void write(MethodWriter writer) {
+        writeDebugInfo(writer);
 
-        adapter.mark(begin);
+        Label begin = new Label();
+        Label end = new Label();
+
+        writer.mark(begin);
 
         condition.fals = end;
-        condition.write(adapter);
+        condition.write(writer);
 
         if (block != null) {
-            adapter.writeLoopCounter(loopCounterSlot, Math.max(1, block.statementCount));
+            writer.writeLoopCounter(loopCounterSlot, Math.max(1, block.statementCount));
 
             block.continu = begin;
             block.brake = end;
-            block.write(adapter);
+            block.write(writer);
         } else {
-            adapter.writeLoopCounter(loopCounterSlot, 1);
+            writer.writeLoopCounter(loopCounterSlot, 1);
         }
 
         if (block == null || !block.allEscape) {
-            adapter.goTo(begin);
+            writer.goTo(begin);
         }
 
-        adapter.mark(end);
+        writer.mark(end);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
index 0d4e993b39b..47aa868d6dc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
@@ -59,21 +59,23 @@
  * {@link org.elasticsearch.painless.node.LNewArray} - Represents an array instantiation.
  * {@link org.elasticsearch.painless.node.LNewObj} - Respresents and object instantiation.
  * {@link org.elasticsearch.painless.node.LShortcut} - Represents a field load/store shortcut.  (Internal only.)
+ * {@link org.elasticsearch.painless.node.LStatic} - Represents a static type target.
  * {@link org.elasticsearch.painless.node.LString} - Represents a string constant.
  * {@link org.elasticsearch.painless.node.LVariable} - Represents a variable load/store.
  * {@link org.elasticsearch.painless.node.SBlock} - Represents a set of statements as a branch of control-flow.
  * {@link org.elasticsearch.painless.node.SBreak} - Represents a break statement.
+ * {@link org.elasticsearch.painless.node.SCatch} - Represents a catch block as part of a try-catch block.
  * {@link org.elasticsearch.painless.node.SContinue} - Represents a continue statement.
  * {@link org.elasticsearch.painless.node.SDeclaration} - Represents a single variable declaration.
  * {@link org.elasticsearch.painless.node.SDeclBlock} - Represents a series of declarations.
  * {@link org.elasticsearch.painless.node.SDo} - Represents a do-while loop.
  * {@link org.elasticsearch.painless.node.SExpression} - Represents the top-level node for an expression as a statement.
  * {@link org.elasticsearch.painless.node.SFor} - Represents a for loop.
+ * {@link org.elasticsearch.painless.node.SIf} - Represents an if block.
  * {@link org.elasticsearch.painless.node.SIfElse} - Represents an if/else block.
  * {@link org.elasticsearch.painless.node.SReturn} - Represents a return statement.
  * {@link org.elasticsearch.painless.node.SSource} - The root of all Painless trees.  Contains a series of statements.
  * {@link org.elasticsearch.painless.node.SThrow} - Represents a throw statement.
- * {@link org.elasticsearch.painless.node.STrap} - Represents a catch block as part of a try-catch block.
  * {@link org.elasticsearch.painless.node.STry} - Represents the try block as part of a try-catch block.
  * {@link org.elasticsearch.painless.node.SWhile} - Represents a while loop.
  * <p>
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NoSemiColonTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NoSemiColonTests.java
index e4af7a2d166..c57a756f109 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NoSemiColonTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NoSemiColonTests.java
@@ -59,7 +59,7 @@ public class NoSemiColonTests extends ScriptTestCase {
         assertEquals(String[][].class, exec("String[][] a = new String[1][2]; return a").getClass());
         assertEquals(Map[][][].class, exec("Map[][][] a = new Map[1][2][3]; return a").getClass());
     }
-    
+
     public void testExpression() {
         assertEquals(10, exec("10"));
         assertEquals(10, exec("5 + 5"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
index c3ce127034e..d95fa3897da 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
@@ -26,6 +26,7 @@ import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Map;
 
 /**
@@ -48,7 +49,9 @@ public abstract class ScriptTestCase extends ESTestCase {
 
     /** Compiles and returns the result of {@code script} with access to {@code vars} */
     public Object exec(String script, Map<String, Object> vars) {
-        return exec(script, vars, Collections.emptyMap());
+        Map<String,String> compilerSettings = new HashMap<>();
+        compilerSettings.put(CompilerSettings.PICKY, "true");
+        return exec(script, vars, compilerSettings);
     }
 
     /** Compiles and returns the result of {@code script} with access to {@code vars} and compile-time parameters */
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
index 15ae321534e..9cdce7583f5 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
@@ -52,13 +52,13 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
                  "return y.isEmpty();");
         });
         assertEquals(3, exception.getStackTrace()[0].getLineNumber());
-        
+
         // trigger NPE at line 4 in script (inside conditional)
         exception = expectThrows(NullPointerException.class, () -> {
             exec("String x = null;\n" +
                  "boolean y = false;\n" +
                  "if (!y) {\n" +
-                 "  y = x.isEmpty();\n" + 
+                 "  y = x.isEmpty();\n" +
                  "}\n" +
                  "return y;");
         });
@@ -133,7 +133,7 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
             exec("try { int x; } catch (PainlessError error) {}");
             fail("should have hit ParseException");
         });
-        assertTrue(parseException.getMessage().contains("Not a type [PainlessError]."));
+        assertTrue(parseException.getMessage().contains("unexpected token ['PainlessError']"));
     }
 
     public void testLoopLimits() {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/antlr/ParserTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/antlr/ParserTests.java
new file mode 100644
index 00000000000..aedecfc9c91
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/antlr/ParserTests.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.antlr;
+
+import org.antlr.v4.runtime.ANTLRInputStream;
+import org.antlr.v4.runtime.BaseErrorListener;
+import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.DiagnosticErrorListener;
+import org.antlr.v4.runtime.RecognitionException;
+import org.antlr.v4.runtime.Recognizer;
+import org.antlr.v4.runtime.atn.PredictionMode;
+import org.elasticsearch.painless.antlr.PainlessParser.SourceContext;
+import org.elasticsearch.painless.ScriptTestCase;
+
+import java.text.ParseException;
+
+public class ParserTests extends ScriptTestCase {
+    private static class TestException extends RuntimeException {
+        TestException(String msg) {
+            super(msg);
+        }
+    }
+
+    private SourceContext buildAntlrTree(String source) {
+        ANTLRInputStream stream = new ANTLRInputStream(source);
+        PainlessLexer lexer = new ErrorHandlingLexer(stream);
+        PainlessParser parser = new PainlessParser(new CommonTokenStream(lexer));
+        ParserErrorStrategy strategy = new ParserErrorStrategy();
+
+        lexer.removeErrorListeners();
+        parser.removeErrorListeners();
+
+        // Diagnostic listener invokes syntaxError on other listeners for ambiguity issues,
+        parser.addErrorListener(new DiagnosticErrorListener(true));
+        // a second listener to fail the test when the above happens.
+        parser.addErrorListener(new BaseErrorListener() {
+            @Override
+            public void syntaxError(final Recognizer<?, ?> recognizer, final Object offendingSymbol, final int line,
+                                    final int charPositionInLine, final String msg, final RecognitionException e) {
+                throw new TestException("line: " + line + ", offset: " + charPositionInLine +
+                    ", symbol:" + offendingSymbol + " " + msg);
+            }
+        });
+
+        // Enable exact ambiguity detection (costly). we enable exact since its the default for
+        // DiagnosticErrorListener, life is too short to think about what 'inexact ambiguity' might mean.
+        parser.getInterpreter().setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION);
+        parser.setErrorHandler(strategy);
+
+        return parser.source();
+    }
+
+    public void testIllegalSecondary() {
+        //TODO: Need way more corner case tests.
+        Exception exception = expectThrows(TestException.class, () -> buildAntlrTree("(x = 5).y"));
+        assertTrue(exception.getMessage().contains("no viable alternative"));
+        exception = expectThrows(TestException.class, () -> buildAntlrTree("((x = 5).y = 2).z;"));
+        assertTrue(exception.getMessage().contains("no viable alternative"));
+        exception = expectThrows(TestException.class, () -> buildAntlrTree("(2 + 2).z"));
+        assertTrue(exception.getMessage().contains("no viable alternative"));
+        exception = expectThrows(RuntimeException.class, () -> buildAntlrTree("((Map)x.-x)"));
+        assertTrue(exception.getMessage().contains("unexpected character"));
+    }
+}

From a5d0ccf50b4197931a801c687cb62278d9aa5e3f Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 23 May 2016 23:36:40 -0400
Subject: [PATCH 0433/1311] painless: add trap tests

---
 .../org/elasticsearch/painless/java.lang.txt  |  1 +
 .../elasticsearch/painless/TryCatchTests.java | 58 +++++++++++++++++++
 2 files changed, 59 insertions(+)
 create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/TryCatchTests.java

diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
index 01694b60db0..755c8d44c0f 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
@@ -1056,6 +1056,7 @@ class ReflectiveOperationException -> java.lang.ReflectiveOperationException ext
 
 class RuntimeException -> java.lang.RuntimeException extends Exception,Object {
   RuntimeException <init>()
+  RuntimeException <init>(String)
 }
 
 class SecurityException -> java.lang.SecurityException extends RuntimeException,Exception,Object {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/TryCatchTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/TryCatchTests.java
new file mode 100644
index 00000000000..07439f7f42b
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/TryCatchTests.java
@@ -0,0 +1,58 @@
+package org.elasticsearch.painless;
+
+import java.util.Collections;
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/** tests for throw/try/catch in painless */
+public class TryCatchTests extends ScriptTestCase {
+
+    /** throws an exception */
+    public void testThrow() {
+        RuntimeException exception = expectThrows(RuntimeException.class, () -> {
+            exec("throw new RuntimeException('test')");
+        });
+        assertEquals("test", exception.getMessage());
+    }
+    
+    /** catches the exact exception */
+    public void testCatch() {
+        assertEquals(1, exec("try { if (params.param == 'true') throw new RuntimeException('test'); } " + 
+                             "catch (RuntimeException e) { return 1; } return 2;", 
+                              Collections.singletonMap("param", "true")));
+    }
+    
+    /** catches superclass of the exception */
+    public void testCatchSuperclass() {
+        assertEquals(1, exec("try { if (params.param == 'true') throw new RuntimeException('test'); } " + 
+                             "catch (Exception e) { return 1; } return 2;", 
+                              Collections.singletonMap("param", "true")));
+    }
+    
+    /** tries to catch a different type of exception */
+    public void testNoCatch() {
+        RuntimeException exception = expectThrows(RuntimeException.class, () -> {
+           exec("try { if (params.param == 'true') throw new RuntimeException('test'); } " + 
+                "catch (ArithmeticException e) { return 1; } return 2;", 
+                Collections.singletonMap("param", "true"));
+        });
+        assertEquals("test", exception.getMessage());
+    }
+}

From 27cc2fe4dc8d4bcdae8f1553c6adc8a5f2205059 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Fri, 20 May 2016 23:53:51 +0200
Subject: [PATCH 0434/1311] Moved the percolator from core to its own module

Significant changes:
* AbstractQueryTestCase has moved to the test framework module, in order for query builder tests in modules and plugins
* Added support to AbstractQueryTestCase to register plugins
* Lift the restriction that only one percolator could be added per index. This validation existed in MapperService, but because the percolator moved to a module it could no longer exist there. Instead of bringing it back it was removed. This validation existed since the percolator cache only supported one percolator query per document, since the percolator cache has been removed this restriction could removed as well.
* While moving percolator tests to the new module, also removed a couple of tests for the deprecated percolate and mpercolate api. These APIs are now sugar  APIs for bwc and rediect to the searvh and msearvh APIs. Some tests were still testing as if percolate and mpercolate API did the percolation, but this no longer the case and these tests could be removed.
---
 .../resources/checkstyle_suppressions.xml     |  28 +-
 .../elasticsearch/ElasticsearchException.java |   3 -
 .../elasticsearch/action/ActionModule.java    |   6 -
 .../java/org/elasticsearch/client/Client.java |  36 --
 .../client/support/AbstractClient.java        |  38 --
 .../metadata/MetaDataMappingService.java      |   1 -
 .../common/network/NetworkModule.java         |   7 -
 .../common/settings/IndexScopedSettings.java  |   2 -
 .../index/mapper/MapperService.java           |  28 +-
 .../index/query/QueryBuilders.java            |  18 -
 .../index/query/QueryShardContext.java        |   7 +-
 .../elasticsearch/indices/IndicesModule.java  |   3 -
 .../elasticsearch/search/SearchModule.java    |   5 +-
 .../ExceptionSerializationTests.java          |   2 +-
 .../node/tasks/CancellableTasksTests.java     |   2 +-
 .../node/tasks/TaskManagerTestCase.java       |   4 +-
 .../bulk/TransportBulkActionTookTests.java    |   2 +-
 .../TransportBroadcastByNodeActionTests.java  |   4 +-
 .../TransportMasterNodeActionTests.java       |   4 +-
 .../nodes/TransportNodesActionTests.java      |   4 +-
 .../BroadcastReplicationTests.java            |   4 +-
 .../TransportReplicationActionTests.java      |   5 +-
 ...ortInstanceSingleOperationActionTests.java |   4 +-
 .../elasticsearch/cluster/NoMasterNodeIT.java |  20 -
 .../action/shard/ShardStateActionTests.java   |   4 +-
 .../health/ClusterStateHealthTests.java       |   2 +-
 .../cluster/routing/RoutingServiceTests.java  |   4 +-
 .../cluster/service/ClusterServiceTests.java  |   2 +-
 .../discovery/ZenFaultDetectionTests.java     |   4 +-
 .../zen/NodeJoinControllerTests.java          |   4 +-
 .../elasticsearch/index/IndexModuleTests.java |   4 +-
 .../mapper/DynamicMappingDisabledTests.java   |   2 +-
 .../query/AbstractTermQueryTestCase.java      |   1 +
 .../index/query/BoolQueryBuilderTests.java    |   1 +
 .../query/BoostingQueryBuilderTests.java      |   1 +
 .../query/CommonTermsQueryBuilderTests.java   |  33 +-
 .../query/ConstantScoreQueryBuilderTests.java |   1 +
 .../index/query/DisMaxQueryBuilderTests.java  |   1 +
 .../index/query/ExistsQueryBuilderTests.java  |   1 +
 .../FieldMaskingSpanQueryBuilderTests.java    |  29 +-
 .../index/query/FuzzyQueryBuilderTests.java   |  29 +-
 .../GeoBoundingBoxQueryBuilderTests.java      |   1 +
 .../query/GeoDistanceQueryBuilderTests.java   |   1 +
 .../query/GeoDistanceRangeQueryTests.java     |   1 +
 .../query/GeoPolygonQueryBuilderTests.java    |   1 +
 .../query/GeoShapeQueryBuilderTests.java      |   1 +
 .../query/GeohashCellQueryBuilderTests.java   |   1 +
 .../query/HasChildQueryBuilderTests.java      |   7 +-
 .../query/HasParentQueryBuilderTests.java     |   7 +-
 .../index/query/IdsQueryBuilderTests.java     |  13 +-
 .../index/query/IndicesQueryBuilderTests.java |  43 +-
 .../query/MatchAllQueryBuilderTests.java      |  11 +-
 .../query/MatchNoneQueryBuilderTests.java     |   9 +-
 .../MatchPhrasePrefixQueryBuilderTests.java   |   2 +
 .../query/MatchPhraseQueryBuilderTests.java   |   1 +
 .../index/query/MatchQueryBuilderTests.java   |   1 +
 .../query/MoreLikeThisQueryBuilderTests.java  |  47 +-
 .../query/MultiMatchQueryBuilderTests.java    |   1 +
 .../index/query/NestedQueryBuilderTests.java  |   5 +-
 .../query/ParentIdQueryBuilderTests.java      |  31 +-
 .../index/query/PrefixQueryBuilderTests.java  |   3 +-
 .../query/QueryStringQueryBuilderTests.java   |   1 +
 .../index/query/RandomQueryBuilder.java       |   1 +
 .../index/query/RangeQueryBuilderTests.java   |   1 +
 .../index/query/RegexpQueryBuilderTests.java  |  20 +-
 .../index/query/ScriptQueryBuilderTests.java  |  21 +-
 .../query/SimpleQueryStringBuilderTests.java  |   1 +
 .../SpanContainingQueryBuilderTests.java      |   1 +
 .../query/SpanFirstQueryBuilderTests.java     |  27 +-
 .../query/SpanMultiTermQueryBuilderTests.java |  25 +-
 .../query/SpanNearQueryBuilderTests.java      |   1 +
 .../index/query/SpanNotQueryBuilderTests.java |   1 +
 .../index/query/SpanOrQueryBuilderTests.java  |   1 +
 .../query/SpanWithinQueryBuilderTests.java    |   1 +
 .../query/TemplateQueryBuilderTests.java      |   9 +-
 .../index/query/TermsQueryBuilderTests.java   |   2 +-
 .../index/query/TypeQueryBuilderTests.java    |   1 +
 .../query/WildcardQueryBuilderTests.java      |   1 +
 .../index/query/WrapperQueryBuilderTests.java |   1 +
 .../FunctionScoreQueryBuilderTests.java       |  34 +-
 .../indices/IndicesOptionsIntegrationIT.java  |  61 +--
 .../indices/store/IndicesStoreTests.java      |   2 +-
 .../search/SearchModuleTests.java             |   1 -
 .../aggregations/AggregatorParsingTests.java  |   6 +-
 .../aggregations/BaseAggregationTestCase.java |   6 +-
 .../BasePipelineAggregationTestCase.java      |   6 +-
 .../aggregations/metrics/TopHitsTests.java    |   2 +-
 .../builder/SearchSourceBuilderTests.java     |   6 +-
 .../suggest/CompletionSuggestSearch2xIT.java  |  36 --
 .../suggest/CompletionSuggestSearchIT.java    |  37 --
 .../ContextAndHeaderTransportIT.java          |  21 -
 .../query-dsl/percolate-query.asciidoc        |  21 +-
 .../query-dsl/special-queries.asciidoc        |   3 +
 .../migration/migrate_5_0/percolator.asciidoc |  59 ++-
 .../percolator/build.gradle                   |  26 +-
 .../percolator/ExtractQueryTermsService.java  |  11 +-
 .../percolator}/MultiPercolateAction.java     |   2 +-
 .../percolator}/MultiPercolateRequest.java    |   5 +-
 .../MultiPercolateRequestBuilder.java         |   5 +-
 .../percolator}/MultiPercolateResponse.java   |   5 +-
 .../percolator}/PercolateAction.java          |   2 +-
 .../percolator}/PercolateQuery.java           |   3 +-
 .../percolator}/PercolateQueryBuilder.java    |  15 +-
 .../percolator}/PercolateRequest.java         |   9 +-
 .../percolator}/PercolateRequestBuilder.java  |   5 +-
 .../percolator}/PercolateResponse.java        |   5 +-
 .../percolator}/PercolateSourceBuilder.java   |   5 +-
 .../percolator/PercolatorFieldMapper.java     |  44 +-
 .../PercolatorHighlightSubFetchPhase.java     |   5 +-
 .../percolator/PercolatorPlugin.java          |  80 ++++
 .../percolator}/RestMultiPercolateAction.java |  15 +-
 .../percolator}/RestPercolateAction.java      |  20 +-
 .../TransportMultiPercolateAction.java        |   2 +-
 .../percolator}/TransportPercolateAction.java |   3 +-
 .../ExtractQueryTermsServiceTests.java        |   2 +-
 .../percolator/MultiPercolatorIT.java         | 108 ++---
 .../MultiPercolatorRequestTests.java          |   8 +-
 .../PercolateQueryBuilderTests.java           |  33 +-
 .../percolator}/PercolateQueryTests.java      |   7 +-
 .../percolator/PercolatorAggregationsIT.java  |  29 +-
 ...PercolatorBackwardsCompatibilityTests.java |  26 +-
 .../PercolatorFieldMapperTests.java           |  33 +-
 ...PercolatorHighlightSubFetchPhaseTests.java |   3 +-
 .../percolator/PercolatorIT.java              | 192 ++++----
 .../percolator/PercolatorQuerySearchIT.java   |  70 +--
 .../percolator/PercolatorRestIT.java          |  39 ++
 .../percolator/PercolatorTestUtil.java        |  24 +-
 .../indices/percolator/bwc_index_2.0.0.zip    | Bin
 .../percolator}/mpercolate1.json              |   0
 .../percolator}/mpercolate2.json              |   0
 .../test/mpercolate/10_basic.yaml             |   0
 .../rest-api-spec/test/percolate/15_new.yaml  |   0
 .../test/percolate/16_existing_doc.yaml       |   0
 .../test/percolate/17_empty.yaml              |   0
 .../percolate/18_highligh_with_query.yaml     |   0
 .../test/percolate/19_nested.yaml             |   0
 settings.gradle                               |   1 +
 .../test}/AbstractQueryTestCase.java          | 447 +++++++++---------
 .../test}/ClusterServiceUtils.java            |   3 +-
 .../test/IndexSettingsModule.java             |  10 +
 .../hamcrest/ElasticsearchAssertions.java     |  21 -
 141 files changed, 1160 insertions(+), 1127 deletions(-)
 rename core/src/main/java/org/elasticsearch/index/percolator/PercolatorException.java => modules/percolator/build.gradle (58%)
 rename {core/src/main/java/org/elasticsearch/index => modules/percolator/src/main/java/org/elasticsearch}/percolator/ExtractQueryTermsService.java (96%)
 rename {core/src/main/java/org/elasticsearch/action/percolate => modules/percolator/src/main/java/org/elasticsearch/percolator}/MultiPercolateAction.java (97%)
 rename {core/src/main/java/org/elasticsearch/action/percolate => modules/percolator/src/main/java/org/elasticsearch/percolator}/MultiPercolateRequest.java (99%)
 rename {core/src/main/java/org/elasticsearch/action/percolate => modules/percolator/src/main/java/org/elasticsearch/percolator}/MultiPercolateRequestBuilder.java (94%)
 rename {core/src/main/java/org/elasticsearch/action/percolate => modules/percolator/src/main/java/org/elasticsearch/percolator}/MultiPercolateResponse.java (97%)
 rename {core/src/main/java/org/elasticsearch/action/percolate => modules/percolator/src/main/java/org/elasticsearch/percolator}/PercolateAction.java (97%)
 rename {core/src/main/java/org/elasticsearch/index/query => modules/percolator/src/main/java/org/elasticsearch/percolator}/PercolateQuery.java (99%)
 rename {core/src/main/java/org/elasticsearch/index/query => modules/percolator/src/main/java/org/elasticsearch/percolator}/PercolateQueryBuilder.java (97%)
 rename {core/src/main/java/org/elasticsearch/action/percolate => modules/percolator/src/main/java/org/elasticsearch/percolator}/PercolateRequest.java (97%)
 rename {core/src/main/java/org/elasticsearch/action/percolate => modules/percolator/src/main/java/org/elasticsearch/percolator}/PercolateRequestBuilder.java (98%)
 rename {core/src/main/java/org/elasticsearch/action/percolate => modules/percolator/src/main/java/org/elasticsearch/percolator}/PercolateResponse.java (98%)
 rename {core/src/main/java/org/elasticsearch/action/percolate => modules/percolator/src/main/java/org/elasticsearch/percolator}/PercolateSourceBuilder.java (98%)
 rename {core/src/main/java/org/elasticsearch/index => modules/percolator/src/main/java/org/elasticsearch}/percolator/PercolatorFieldMapper.java (87%)
 rename {core/src/main/java/org/elasticsearch/index => modules/percolator/src/main/java/org/elasticsearch}/percolator/PercolatorHighlightSubFetchPhase.java (97%)
 create mode 100644 modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
 rename {core/src/main/java/org/elasticsearch/rest/action/percolate => modules/percolator/src/main/java/org/elasticsearch/percolator}/RestMultiPercolateAction.java (89%)
 rename {core/src/main/java/org/elasticsearch/rest/action/percolate => modules/percolator/src/main/java/org/elasticsearch/percolator}/RestPercolateAction.java (93%)
 rename {core/src/main/java/org/elasticsearch/action/percolate => modules/percolator/src/main/java/org/elasticsearch/percolator}/TransportMultiPercolateAction.java (99%)
 rename {core/src/main/java/org/elasticsearch/action/percolate => modules/percolator/src/main/java/org/elasticsearch/percolator}/TransportPercolateAction.java (99%)
 rename {core/src/test/java/org/elasticsearch/index => modules/percolator/src/test/java/org/elasticsearch}/percolator/ExtractQueryTermsServiceTests.java (99%)
 rename {core => modules/percolator}/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java (83%)
 rename {core/src/test/java/org/elasticsearch/action/percolate => modules/percolator/src/test/java/org/elasticsearch/percolator}/MultiPercolatorRequestTests.java (98%)
 rename {core/src/test/java/org/elasticsearch/index/query => modules/percolator/src/test/java/org/elasticsearch/percolator}/PercolateQueryBuilderTests.java (88%)
 rename {core/src/test/java/org/elasticsearch/index/query => modules/percolator/src/test/java/org/elasticsearch/percolator}/PercolateQueryTests.java (98%)
 rename {core => modules/percolator}/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java (92%)
 rename core/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityIT.java => modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java (86%)
 rename {core/src/test/java/org/elasticsearch/index => modules/percolator/src/test/java/org/elasticsearch}/percolator/PercolatorFieldMapperTests.java (91%)
 rename {core/src/test/java/org/elasticsearch/index => modules/percolator/src/test/java/org/elasticsearch}/percolator/PercolatorHighlightSubFetchPhaseTests.java (97%)
 rename {core => modules/percolator}/src/test/java/org/elasticsearch/percolator/PercolatorIT.java (93%)
 rename {core/src/test/java/org/elasticsearch/search => modules/percolator/src/test/java/org/elasticsearch}/percolator/PercolatorQuerySearchIT.java (89%)
 create mode 100644 modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorRestIT.java
 rename {test/framework/src/main => modules/percolator/src/test}/java/org/elasticsearch/percolator/PercolatorTestUtil.java (57%)
 rename {core => modules/percolator}/src/test/resources/indices/percolator/bwc_index_2.0.0.zip (100%)
 rename {core/src/test/resources/org/elasticsearch/action/percolate => modules/percolator/src/test/resources/org/elasticsearch/percolator}/mpercolate1.json (100%)
 rename {core/src/test/resources/org/elasticsearch/action/percolate => modules/percolator/src/test/resources/org/elasticsearch/percolator}/mpercolate2.json (100%)
 rename {rest-api-spec/src/main => modules/percolator/src/test}/resources/rest-api-spec/test/mpercolate/10_basic.yaml (100%)
 rename {rest-api-spec/src/main => modules/percolator/src/test}/resources/rest-api-spec/test/percolate/15_new.yaml (100%)
 rename {rest-api-spec/src/main => modules/percolator/src/test}/resources/rest-api-spec/test/percolate/16_existing_doc.yaml (100%)
 rename {rest-api-spec/src/main => modules/percolator/src/test}/resources/rest-api-spec/test/percolate/17_empty.yaml (100%)
 rename {rest-api-spec/src/main => modules/percolator/src/test}/resources/rest-api-spec/test/percolate/18_highligh_with_query.yaml (100%)
 rename {rest-api-spec/src/main => modules/percolator/src/test}/resources/rest-api-spec/test/percolate/19_nested.yaml (100%)
 rename {core/src/test/java/org/elasticsearch/index/query => test/framework/src/main/java/org/elasticsearch/test}/AbstractQueryTestCase.java (73%)
 rename {core/src/test/java/org/elasticsearch/cluster/service => test/framework/src/main/java/org/elasticsearch/test}/ClusterServiceUtils.java (98%)

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 5c776083279..4c467039117 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -179,12 +179,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]SimulatePipelineRequest.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]SimulatePipelineRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]SimulatePipelineTransportAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]percolate[/\\]MultiPercolateRequest.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]percolate[/\\]MultiPercolateRequestBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]percolate[/\\]PercolateShardResponse.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]percolate[/\\]TransportMultiPercolateAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]percolate[/\\]TransportPercolateAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]percolate[/\\]TransportShardMultiPercolateAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]MultiSearchRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]SearchPhaseExecutionException.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]SearchResponse.java" checks="LineLength" />
@@ -453,9 +447,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]object[/\\]ObjectMapper.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]object[/\\]RootObjectMapper.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]merge[/\\]MergeStats.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]percolator[/\\]ExtractQueryTermsService.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]percolator[/\\]PercolatorFieldMapper.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]percolator[/\\]PercolatorQueriesRegistry.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]AbstractQueryBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]MatchQueryParser.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]QueryBuilders.java" checks="LineLength" />
@@ -520,7 +511,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]JvmStats.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]node[/\\]Node.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]node[/\\]internal[/\\]InternalSettingsPreparer.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolatorQuery.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]DummyPluginInfo.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]PluginsService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]RemovePluginCommand.java" checks="LineLength" />
@@ -566,7 +556,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestThreadPoolAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]get[/\\]RestMultiGetAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]index[/\\]RestIndexAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]percolate[/\\]RestPercolateAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]script[/\\]RestDeleteIndexedScriptAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]script[/\\]RestPutIndexedScriptAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]search[/\\]RestClearScrollAction.java" checks="LineLength" />
@@ -745,7 +734,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]SimulatePipelineRequestParsingTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]SimulatePipelineResponseTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ingest[/\\]WriteableIngestDocumentTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]percolate[/\\]MultiPercolatorRequestTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]MultiSearchRequestTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]search[/\\]SearchRequestBuilderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]support[/\\]AutoCreateIndexTests.java" checks="LineLength" />
@@ -981,8 +969,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]typelevels[/\\]ParseDocumentTypeLevelsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]update[/\\]UpdateMappingOnClusterIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]update[/\\]UpdateMappingTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]percolator[/\\]PercolatorFieldMapperTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]AbstractQueryTestCase.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]BoolQueryBuilderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]BoostingQueryBuilderTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]CommonTermsQueryBuilderTests.java" checks="LineLength" />
@@ -1071,9 +1057,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]os[/\\]OsProbeTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]nodesinfo[/\\]NodeInfoStreamingTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]options[/\\]detailederrors[/\\]DetailedErrorsEnabledIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]MultiPercolatorIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolatorIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolatorQueryTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]PluginInfoTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]PluginsServiceTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]recovery[/\\]FullRollingRestartIT.java" checks="LineLength" />
@@ -1222,6 +1205,16 @@
   <suppress files="modules[/\\]lang-mustache[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]package-info.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-mustache[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]mustache[/\\]MustacheScriptEngineTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-mustache[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]mustache[/\\]MustacheTests.java" checks="LineLength" />
+  <suppress files="modules[/\\]percolator[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]MultiPercolateRequest.java" checks="LineLength" />
+  <suppress files="modules[/\\]percolator[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]MultiPercolateRequestBuilder.java" checks="LineLength" />
+  <suppress files="modules[/\\]percolator[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolateShardResponse.java" checks="LineLength" />
+  <suppress files="modules[/\\]percolator[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]TransportMultiPercolateAction.java" checks="LineLength" />
+  <suppress files="modules[/\\]percolator[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]TransportPercolateAction.java" checks="LineLength" />
+  <suppress files="modules[/\\]percolator[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]TransportShardMultiPercolateAction.java" checks="LineLength" />
+  <suppress files="modules[/\\]percolator[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]RestPercolateAction.java" checks="LineLength" />
+  <suppress files="modules[/\\]percolator[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]MultiPercolatorIT.java" checks="LineLength" />
+  <suppress files="modules[/\\]percolator[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolatorIT.java" checks="LineLength" />
+  <suppress files="modules[/\\]percolator[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]MultiPercolatorRequestTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]analysis-icu[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]IcuCollationTokenFilterFactory.java" checks="LineLength" />
   <suppress files="plugins[/\\]analysis-icu[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]IcuFoldingTokenFilterFactory.java" checks="LineLength" />
   <suppress files="plugins[/\\]analysis-icu[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]IcuNormalizerTokenFilterFactory.java" checks="LineLength" />
@@ -1309,6 +1302,7 @@
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]cli[/\\]CliToolTestCase.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]MockBigArrays.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]script[/\\]NativeSignificanceScoreScriptWithParams.java" checks="LineLength" />
+  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]AbstractQueryTestCase.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]BackgroundIndexer.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]CompositeTestCluster.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]CorruptionUtils.java" checks="LineLength" />
diff --git a/core/src/main/java/org/elasticsearch/ElasticsearchException.java b/core/src/main/java/org/elasticsearch/ElasticsearchException.java
index b242811b7be..1faf6cd4237 100644
--- a/core/src/main/java/org/elasticsearch/ElasticsearchException.java
+++ b/core/src/main/java/org/elasticsearch/ElasticsearchException.java
@@ -21,7 +21,6 @@ package org.elasticsearch;
 
 import org.elasticsearch.action.support.replication.ReplicationOperation;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -681,8 +680,6 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
                 org.elasticsearch.index.shard.IndexShardRecoveryException::new, 106),
         REPOSITORY_MISSING_EXCEPTION(org.elasticsearch.repositories.RepositoryMissingException.class,
                 org.elasticsearch.repositories.RepositoryMissingException::new, 107),
-        PERCOLATOR_EXCEPTION(org.elasticsearch.index.percolator.PercolatorException.class,
-                org.elasticsearch.index.percolator.PercolatorException::new, 108),
         DOCUMENT_SOURCE_MISSING_EXCEPTION(org.elasticsearch.index.engine.DocumentSourceMissingException.class,
                 org.elasticsearch.index.engine.DocumentSourceMissingException::new, 109),
         FLUSH_NOT_ALLOWED_ENGINE_EXCEPTION(org.elasticsearch.index.engine.FlushNotAllowedEngineException.class,
diff --git a/core/src/main/java/org/elasticsearch/action/ActionModule.java b/core/src/main/java/org/elasticsearch/action/ActionModule.java
index 3e93f699645..bab3dcb2ed2 100644
--- a/core/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/core/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -165,10 +165,6 @@ import org.elasticsearch.action.ingest.SimulatePipelineAction;
 import org.elasticsearch.action.ingest.SimulatePipelineTransportAction;
 import org.elasticsearch.action.main.MainAction;
 import org.elasticsearch.action.main.TransportMainAction;
-import org.elasticsearch.action.percolate.MultiPercolateAction;
-import org.elasticsearch.action.percolate.PercolateAction;
-import org.elasticsearch.action.percolate.TransportMultiPercolateAction;
-import org.elasticsearch.action.percolate.TransportPercolateAction;
 import org.elasticsearch.action.search.ClearScrollAction;
 import org.elasticsearch.action.search.MultiSearchAction;
 import org.elasticsearch.action.search.SearchAction;
@@ -332,8 +328,6 @@ public class ActionModule extends AbstractModule {
         registerAction(SearchAction.INSTANCE, TransportSearchAction.class);
         registerAction(SearchScrollAction.INSTANCE, TransportSearchScrollAction.class);
         registerAction(MultiSearchAction.INSTANCE, TransportMultiSearchAction.class);
-        registerAction(PercolateAction.INSTANCE, TransportPercolateAction.class);
-        registerAction(MultiPercolateAction.INSTANCE, TransportMultiPercolateAction.class);
         registerAction(ExplainAction.INSTANCE, TransportExplainAction.class);
         registerAction(ClearScrollAction.INSTANCE, TransportClearScrollAction.class);
         registerAction(RecoveryAction.INSTANCE, TransportRecoveryAction.class);
diff --git a/core/src/main/java/org/elasticsearch/client/Client.java b/core/src/main/java/org/elasticsearch/client/Client.java
index 47e8e43f37b..0cf22d7a2c4 100644
--- a/core/src/main/java/org/elasticsearch/client/Client.java
+++ b/core/src/main/java/org/elasticsearch/client/Client.java
@@ -42,12 +42,6 @@ import org.elasticsearch.action.get.MultiGetResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.action.percolate.MultiPercolateRequest;
-import org.elasticsearch.action.percolate.MultiPercolateRequestBuilder;
-import org.elasticsearch.action.percolate.MultiPercolateResponse;
-import org.elasticsearch.action.percolate.PercolateRequest;
-import org.elasticsearch.action.percolate.PercolateRequestBuilder;
-import org.elasticsearch.action.percolate.PercolateResponse;
 import org.elasticsearch.action.search.ClearScrollRequest;
 import org.elasticsearch.action.search.ClearScrollRequestBuilder;
 import org.elasticsearch.action.search.ClearScrollResponse;
@@ -419,36 +413,6 @@ public interface Client extends ElasticsearchClient, Releasable {
      */
     MultiTermVectorsRequestBuilder prepareMultiTermVectors();
 
-    /**
-     * Percolates a request returning the matches documents.
-     */
-    ActionFuture<PercolateResponse> percolate(PercolateRequest request);
-
-    /**
-     * Percolates a request returning the matches documents.
-     */
-    void percolate(PercolateRequest request, ActionListener<PercolateResponse> listener);
-
-    /**
-     * Percolates a request returning the matches documents.
-     */
-    PercolateRequestBuilder preparePercolate();
-
-    /**
-     * Performs multiple percolate requests.
-     */
-    ActionFuture<MultiPercolateResponse> multiPercolate(MultiPercolateRequest request);
-
-    /**
-     * Performs multiple percolate requests.
-     */
-    void multiPercolate(MultiPercolateRequest request, ActionListener<MultiPercolateResponse> listener);
-
-    /**
-     * Performs multiple percolate requests.
-     */
-    MultiPercolateRequestBuilder prepareMultiPercolate();
-
     /**
      * Computes a score explanation for the specified request.
      *
diff --git a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
index 6083422862c..e36bc4b8d77 100644
--- a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
+++ b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
@@ -295,14 +295,6 @@ import org.elasticsearch.action.ingest.SimulatePipelineRequest;
 import org.elasticsearch.action.ingest.SimulatePipelineRequestBuilder;
 import org.elasticsearch.action.ingest.SimulatePipelineResponse;
 import org.elasticsearch.action.ingest.WritePipelineResponse;
-import org.elasticsearch.action.percolate.MultiPercolateAction;
-import org.elasticsearch.action.percolate.MultiPercolateRequest;
-import org.elasticsearch.action.percolate.MultiPercolateRequestBuilder;
-import org.elasticsearch.action.percolate.MultiPercolateResponse;
-import org.elasticsearch.action.percolate.PercolateAction;
-import org.elasticsearch.action.percolate.PercolateRequest;
-import org.elasticsearch.action.percolate.PercolateRequestBuilder;
-import org.elasticsearch.action.percolate.PercolateResponse;
 import org.elasticsearch.action.search.ClearScrollAction;
 import org.elasticsearch.action.search.ClearScrollRequest;
 import org.elasticsearch.action.search.ClearScrollRequestBuilder;
@@ -623,36 +615,6 @@ public abstract class AbstractClient extends AbstractComponent implements Client
         return new MultiTermVectorsRequestBuilder(this, MultiTermVectorsAction.INSTANCE);
     }
 
-    @Override
-    public ActionFuture<PercolateResponse> percolate(final PercolateRequest request) {
-        return execute(PercolateAction.INSTANCE, request);
-    }
-
-    @Override
-    public void percolate(final PercolateRequest request, final ActionListener<PercolateResponse> listener) {
-        execute(PercolateAction.INSTANCE, request, listener);
-    }
-
-    @Override
-    public PercolateRequestBuilder preparePercolate() {
-        return new PercolateRequestBuilder(this, PercolateAction.INSTANCE);
-    }
-
-    @Override
-    public MultiPercolateRequestBuilder prepareMultiPercolate() {
-        return new MultiPercolateRequestBuilder(this, MultiPercolateAction.INSTANCE);
-    }
-
-    @Override
-    public void multiPercolate(MultiPercolateRequest request, ActionListener<MultiPercolateResponse> listener) {
-        execute(MultiPercolateAction.INSTANCE, request, listener);
-    }
-
-    @Override
-    public ActionFuture<MultiPercolateResponse> multiPercolate(MultiPercolateRequest request) {
-        return execute(MultiPercolateAction.INSTANCE, request);
-    }
-
     @Override
     public ExplainRequestBuilder prepareExplain(String index, String type, String id) {
         return new ExplainRequestBuilder(this, ExplainAction.INSTANCE, index, type, id);
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
index d3b5e7ecbad..a41d02d28fa 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
@@ -42,7 +42,6 @@ import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.percolator.PercolatorFieldMapper;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.InvalidTypeNameException;
 
diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
index c0b650e555d..eebd8544bf8 100644
--- a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
+++ b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
@@ -20,9 +20,7 @@
 package org.elasticsearch.common.network;
 
 import java.util.Arrays;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
 import org.elasticsearch.action.support.replication.ReplicationTask;
 import org.elasticsearch.client.transport.TransportClientNodesService;
@@ -36,7 +34,6 @@ import org.elasticsearch.cluster.routing.allocation.command.AllocationCommandReg
 import org.elasticsearch.cluster.routing.allocation.command.CancelAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -137,8 +134,6 @@ import org.elasticsearch.rest.action.ingest.RestGetPipelineAction;
 import org.elasticsearch.rest.action.ingest.RestPutPipelineAction;
 import org.elasticsearch.rest.action.ingest.RestSimulatePipelineAction;
 import org.elasticsearch.rest.action.main.RestMainAction;
-import org.elasticsearch.rest.action.percolate.RestMultiPercolateAction;
-import org.elasticsearch.rest.action.percolate.RestPercolateAction;
 import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestDeleteStoredScriptAction;
 import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestGetStoredScriptAction;
 import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestPutStoredScriptAction;
@@ -250,8 +245,6 @@ public class NetworkModule extends AbstractModule {
         RestMultiTermVectorsAction.class,
         RestBulkAction.class,
         RestUpdateAction.class,
-        RestPercolateAction.class,
-        RestMultiPercolateAction.class,
 
         RestSearchAction.class,
         RestSearchScrollAction.class,
diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
index 203d1db76b3..027100b3469 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
@@ -36,7 +36,6 @@ import org.elasticsearch.index.engine.EngineConfig;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.percolator.PercolatorFieldMapper;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.index.store.FsDirectoryService;
 import org.elasticsearch.index.store.IndexStore;
@@ -127,7 +126,6 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         FieldMapper.IGNORE_MALFORMED_SETTING,
         FieldMapper.COERCE_SETTING,
         Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING,
-        PercolatorFieldMapper.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING,
         MapperService.INDEX_MAPPER_DYNAMIC_SETTING,
         MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING,
         MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING,
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
index 0635a532a58..adb87929b84 100755
--- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
@@ -23,7 +23,6 @@ import com.carrotsearch.hppc.ObjectHashSet;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;
-import org.apache.lucene.document.FieldType;
 import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.compress.CompressedXContent;
@@ -35,15 +34,12 @@ import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.mapper.Mapper.BuilderContext;
 import org.elasticsearch.index.mapper.object.ObjectMapper;
-import org.elasticsearch.index.percolator.PercolatorFieldMapper;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.InvalidTypeNameException;
 import org.elasticsearch.indices.TypeMissingException;
 import org.elasticsearch.indices.mapper.MapperRegistry;
-import org.elasticsearch.script.ScriptService;
 
-import java.io.Closeable;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -97,6 +93,8 @@ public class MapperService extends AbstractIndexComponent {
             "_uid", "_id", "_type", "_all", "_parent", "_routing", "_index",
             "_size", "_timestamp", "_ttl"
     );
+    @Deprecated
+    public static final String PERCOLATOR_LEGACY_TYPE_NAME = ".percolator";
 
     private final AnalysisService analysisService;
 
@@ -269,7 +267,6 @@ public class MapperService extends AbstractIndexComponent {
             checkNestedFieldsLimit(fullPathObjectMappers);
             checkTotalFieldsLimit(objectMappers.size() + fieldMappers.size());
             checkDepthLimit(fullPathObjectMappers.keySet());
-            checkPercolatorFieldLimit(fieldTypes);
         }
 
         Set<String> parentTypes = this.parentTypes;
@@ -321,7 +318,7 @@ public class MapperService extends AbstractIndexComponent {
     private boolean typeNameStartsWithIllegalDot(DocumentMapper mapper) {
         boolean legacyIndex = getIndexSettings().getIndexVersionCreated().before(Version.V_5_0_0_alpha1);
         if (legacyIndex) {
-            return mapper.type().startsWith(".") && !PercolatorFieldMapper.LEGACY_TYPE_NAME.equals(mapper.type());
+            return mapper.type().startsWith(".") && !PERCOLATOR_LEGACY_TYPE_NAME.equals(mapper.type());
         } else {
             return mapper.type().startsWith(".");
         }
@@ -432,25 +429,6 @@ public class MapperService extends AbstractIndexComponent {
         }
     }
 
-    /**
-     * We only allow upto 1 percolator field per index.
-     *
-     * Reasoning here is that the PercolatorQueryCache only supports a single document having a percolator query.
-     * Also specifying multiple queries per document feels like an anti pattern
-     */
-    private void checkPercolatorFieldLimit(Iterable<MappedFieldType> fieldTypes) {
-        List<String> percolatorFieldTypes = new ArrayList<>();
-        for (MappedFieldType fieldType : fieldTypes) {
-            if (fieldType instanceof PercolatorFieldMapper.PercolatorFieldType) {
-                percolatorFieldTypes.add(fieldType.name());
-            }
-        }
-        if (percolatorFieldTypes.size() > 1) {
-            throw new IllegalArgumentException("Up to one percolator field type is allowed per index, " +
-                    "found the following percolator fields [" + percolatorFieldTypes + "]");
-        }
-    }
-
     public DocumentMapper parse(String mappingType, CompressedXContent mappingSource, boolean applyDefault) throws MapperParsingException {
         return documentParser.parse(mappingType, mappingSource, applyDefault ? defaultMappingSource : null);
     }
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java b/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
index 756be4c3032..d048f73381e 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
@@ -839,24 +839,6 @@ public abstract class QueryBuilders {
         return new ExistsQueryBuilder(name);
     }
 
-    public static PercolateQueryBuilder percolateQuery(String queryField, String documentType, BytesReference document) {
-        return new PercolateQueryBuilder(queryField, documentType, document);
-    }
-
-    public static PercolateQueryBuilder percolateQuery(String queryField, String documentType, String indexedDocumentIndex,
-                                                       String indexedDocumentType, String indexedDocumentId) {
-        return new PercolateQueryBuilder(queryField, documentType, indexedDocumentIndex, indexedDocumentType, indexedDocumentId,
-                null, null, null);
-    }
-
-    public static PercolateQueryBuilder percolateQuery(String queryField, String documentType, String indexedDocumentIndex,
-                                                       String indexedDocumentType, String indexedDocumentId,
-                                                       String indexedDocumentRouting, String indexedDocumentPreference,
-                                                       Long indexedDocumentVersion) {
-        return new PercolateQueryBuilder(queryField, documentType, indexedDocumentIndex, indexedDocumentType, indexedDocumentId,
-                indexedDocumentRouting, indexedDocumentPreference, indexedDocumentVersion);
-    }
-
     private QueryBuilders() {
 
     }
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
index 3cdabb74d7e..56a75a69138 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
@@ -171,7 +171,12 @@ public class QueryShardContext extends QueryRewriteContext {
         return isFilter;
     }
 
-    void setIsFilter(boolean isFilter) {
+    /**
+     * Public for testing only!
+     *
+     * Sets whether we are currently parsing a filter or a query
+     */
+    public void setIsFilter(boolean isFilter) {
         this.isFilter = isFilter;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
index 1ff198c6cbf..8cedea931c0 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
@@ -34,7 +34,6 @@ import org.elasticsearch.index.mapper.core.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.core.StringFieldMapper;
 import org.elasticsearch.index.mapper.core.TextFieldMapper;
 import org.elasticsearch.index.mapper.core.TokenCountFieldMapper;
-import org.elasticsearch.index.mapper.core.LegacyTokenCountFieldMapper;
 import org.elasticsearch.index.mapper.core.NumberFieldMapper;
 import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
 import org.elasticsearch.index.mapper.geo.GeoShapeFieldMapper;
@@ -52,7 +51,6 @@ import org.elasticsearch.index.mapper.internal.UidFieldMapper;
 import org.elasticsearch.index.mapper.internal.VersionFieldMapper;
 import org.elasticsearch.index.mapper.ip.IpFieldMapper;
 import org.elasticsearch.index.mapper.object.ObjectMapper;
-import org.elasticsearch.index.percolator.PercolatorFieldMapper;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.flush.SyncedFlushService;
 import org.elasticsearch.indices.mapper.MapperRegistry;
@@ -98,7 +96,6 @@ public class IndicesModule extends AbstractModule {
         registerMapper(ObjectMapper.NESTED_CONTENT_TYPE, new ObjectMapper.TypeParser());
         registerMapper(CompletionFieldMapper.CONTENT_TYPE, new CompletionFieldMapper.TypeParser());
         registerMapper(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser());
-        registerMapper(PercolatorFieldMapper.CONTENT_TYPE, new PercolatorFieldMapper.TypeParser());
 
         if (ShapesAvailability.JTS_AVAILABLE && ShapesAvailability.SPATIAL4J_AVAILABLE) {
             registerMapper(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser());
diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java
index e76597c256c..f601edd50a4 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -32,7 +32,6 @@ import org.elasticsearch.common.lucene.search.function.ScoreFunction;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ParseFieldRegistry;
-import org.elasticsearch.index.percolator.PercolatorHighlightSubFetchPhase;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.BoostingQueryBuilder;
 import org.elasticsearch.index.query.CommonTermsQueryBuilder;
@@ -61,7 +60,6 @@ import org.elasticsearch.index.query.MoreLikeThisQueryBuilder;
 import org.elasticsearch.index.query.MultiMatchQueryBuilder;
 import org.elasticsearch.index.query.NestedQueryBuilder;
 import org.elasticsearch.index.query.ParentIdQueryBuilder;
-import org.elasticsearch.index.query.PercolateQueryBuilder;
 import org.elasticsearch.index.query.PrefixQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParser;
@@ -469,7 +467,6 @@ public class SearchModule extends AbstractModule {
         fetchSubPhaseMultibinder.addBinding().to(MatchedQueriesFetchSubPhase.class);
         fetchSubPhaseMultibinder.addBinding().to(HighlightPhase.class);
         fetchSubPhaseMultibinder.addBinding().to(ParentFieldSubFetchPhase.class);
-        fetchSubPhaseMultibinder.addBinding().to(PercolatorHighlightSubFetchPhase.class);
         for (Class<? extends FetchSubPhase> clazz : fetchSubPhases) {
             fetchSubPhaseMultibinder.addBinding().to(clazz);
         }
@@ -697,7 +694,7 @@ public class SearchModule extends AbstractModule {
         registerQuery(ExistsQueryBuilder::new, ExistsQueryBuilder::fromXContent, ExistsQueryBuilder.QUERY_NAME_FIELD);
         registerQuery(MatchNoneQueryBuilder::new, MatchNoneQueryBuilder::fromXContent, MatchNoneQueryBuilder.QUERY_NAME_FIELD);
         registerQuery(ParentIdQueryBuilder::new, ParentIdQueryBuilder::fromXContent, ParentIdQueryBuilder.QUERY_NAME_FIELD);
-        registerQuery(PercolateQueryBuilder::new, PercolateQueryBuilder::fromXContent, PercolateQueryBuilder.QUERY_NAME_FIELD);
+
         if (ShapesAvailability.JTS_AVAILABLE && ShapesAvailability.SPATIAL4J_AVAILABLE) {
             registerQuery(GeoShapeQueryBuilder::new, GeoShapeQueryBuilder::fromXContent, GeoShapeQueryBuilder.QUERY_NAME_FIELD);
         }
diff --git a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
index 79edf0f5fff..b28758ed941 100644
--- a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
+++ b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
@@ -743,7 +743,7 @@ public class ExceptionSerializationTests extends ESTestCase {
         ids.put(105, org.elasticsearch.cluster.routing.RoutingException.class);
         ids.put(106, org.elasticsearch.index.shard.IndexShardRecoveryException.class);
         ids.put(107, org.elasticsearch.repositories.RepositoryMissingException.class);
-        ids.put(108, org.elasticsearch.index.percolator.PercolatorException.class);
+        ids.put(108, null);
         ids.put(109, org.elasticsearch.index.engine.DocumentSourceMissingException.class);
         ids.put(110, org.elasticsearch.index.engine.FlushNotAllowedEngineException.class);
         ids.put(111, org.elasticsearch.common.settings.NoClassSettingsException.class);
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
index 9af2bb07417..c35a9e91314 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
@@ -50,7 +50,7 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicReference;
 
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
+import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
index ffc19aa0dcd..f04c758ef9a 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
@@ -57,8 +57,8 @@ import java.util.function.Supplier;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
+import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.test.ClusterServiceUtils.setState;
 
 /**
  * The test case for unit testing task manager and related transport actions
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
index db1d09b3aa7..5131ddebca7 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
@@ -51,7 +51,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.LongSupplier;
 
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
 import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
diff --git a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
index 23b0df27480..0fac744625f 100644
--- a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
@@ -78,8 +78,8 @@ import java.util.function.Supplier;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
+import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.object.HasToString.hasToString;
diff --git a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
index ccdb13f710a..b92ba64c2fb 100644
--- a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
@@ -59,8 +59,8 @@ import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
+import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
diff --git a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
index a3335d87fd6..a7289124704 100644
--- a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
@@ -55,8 +55,8 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReferenceArray;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
+import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.mockito.Mockito.mock;
 
 public class TransportNodesActionTests extends ESTestCase {
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
index 5253097818e..6175f822b6a 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
@@ -65,8 +65,8 @@ import java.util.concurrent.TimeUnit;
 import static org.elasticsearch.action.support.replication.ClusterStateCreationUtils.state;
 import static org.elasticsearch.action.support.replication.ClusterStateCreationUtils.stateWithAssignedPrimariesAndOneReplica;
 import static org.elasticsearch.action.support.replication.ClusterStateCreationUtils.stateWithNoShard;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
+import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
index d88960ea9ac..2e81ec712eb 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
@@ -43,7 +43,6 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -86,8 +85,8 @@ import java.util.stream.Collectors;
 
 import static org.elasticsearch.action.support.replication.ClusterStateCreationUtils.state;
 import static org.elasticsearch.action.support.replication.ClusterStateCreationUtils.stateWithActivePrimary;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
+import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.equalTo;
diff --git a/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
index 86165461c84..10132af1b1d 100644
--- a/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
@@ -60,8 +60,8 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
+import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.core.IsEqual.equalTo;
 
 public class TransportInstanceSingleOperationActionTests extends ESTestCase {
diff --git a/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java b/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java
index 99c46f1fc2e..84410a92c83 100644
--- a/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/NoMasterNodeIT.java
@@ -23,7 +23,6 @@ import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.action.percolate.PercolateSourceBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -40,9 +39,6 @@ import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 
-import java.util.HashMap;
-
-import static org.elasticsearch.action.percolate.PercolateSourceBuilder.docBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertExists;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
@@ -101,22 +97,6 @@ public class NoMasterNodeIT extends ESIntegTestCase {
                 ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE
         );
 
-        PercolateSourceBuilder percolateSource = new PercolateSourceBuilder();
-        percolateSource.setDoc(docBuilder().setDoc(new HashMap()));
-        assertThrows(client().preparePercolate()
-                        .setIndices("test").setDocumentType("type1")
-                        .setSource(percolateSource),
-                ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE
-        );
-
-        percolateSource = new PercolateSourceBuilder();
-        percolateSource.setDoc(docBuilder().setDoc(new HashMap()));
-        assertThrows(client().preparePercolate()
-                        .setIndices("no_index").setDocumentType("type1")
-                        .setSource(percolateSource),
-                ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE
-        );
-
 
         assertThrows(client().admin().indices().prepareAnalyze("test", "this is a test"),
                 ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE
diff --git a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
index fabee819a11..261da3e2bb8 100644
--- a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
@@ -56,8 +56,8 @@ import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.LongConsumer;
 
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
+import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.Matchers.is;
diff --git a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
index 298175c41ee..93c33a97cdc 100644
--- a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
@@ -54,7 +54,7 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
 import static org.hamcrest.CoreMatchers.allOf;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java
index eb62653978f..68027312581 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java
@@ -43,8 +43,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import static java.util.Collections.singleton;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
+import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.equalTo;
 
 /**
diff --git a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
index 7303d5a67df..81fc9d3752d 100644
--- a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
@@ -64,7 +64,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
+import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
diff --git a/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java b/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
index 1108e8335e9..d3da4513d95 100644
--- a/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
@@ -58,8 +58,8 @@ import java.util.concurrent.TimeUnit;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static java.util.Collections.singleton;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
+import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
index 3a81b905eb5..b1e3ae157a7 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
@@ -65,8 +65,8 @@ import java.util.concurrent.atomic.AtomicReference;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static java.util.Collections.shuffle;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
+import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
index 7beeaaee0a6..49e5d0c4595 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
@@ -36,7 +36,7 @@ import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.cluster.service.ClusterServiceUtils;
+import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -88,8 +88,6 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
 
-import static java.util.Collections.emptyMap;
-
 public class IndexModuleTests extends ESTestCase {
     private Index index;
     private Settings settings;
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
index 7f9c4a3bbfe..26399218829 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
@@ -46,7 +46,7 @@ import java.util.Collections;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
 import static org.hamcrest.CoreMatchers.instanceOf;
 
 public class DynamicMappingDisabledTests extends ESSingleNodeTestCase {
diff --git a/core/src/test/java/org/elasticsearch/index/query/AbstractTermQueryTestCase.java b/core/src/test/java/org/elasticsearch/index/query/AbstractTermQueryTestCase.java
index 0fe35530234..a2eec493c9c 100644
--- a/core/src/test/java/org/elasticsearch/index/query/AbstractTermQueryTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/query/AbstractTermQueryTestCase.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.query;
 
 import com.fasterxml.jackson.core.io.JsonStringEncoder;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.util.HashMap;
 import java.util.Map;
diff --git a/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
index a49d410fdd6..15180b9d989 100644
--- a/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
@@ -28,6 +28,7 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
diff --git a/core/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java
index ecdadeca923..34c15d63577 100644
--- a/core/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.query;
 
 import org.apache.lucene.queries.BoostingQuery;
 import org.apache.lucene.search.Query;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java
index d7a48195343..0b293cd0157 100644
--- a/core/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.query;
 
 import org.apache.lucene.queries.ExtendedCommonTermsQuery;
 import org.apache.lucene.search.Query;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 
@@ -106,21 +107,21 @@ public class CommonTermsQueryBuilderTests extends AbstractQueryTestCase<CommonTe
 
     public void testFromJson() throws IOException {
         String query =
-                "{\n" + 
-                "  \"common\" : {\n" + 
-                "    \"body\" : {\n" + 
-                "      \"query\" : \"nelly the elephant not as a cartoon\",\n" + 
-                "      \"disable_coord\" : true,\n" + 
-                "      \"high_freq_operator\" : \"AND\",\n" + 
-                "      \"low_freq_operator\" : \"OR\",\n" + 
-                "      \"cutoff_frequency\" : 0.001,\n" + 
-                "      \"minimum_should_match\" : {\n" + 
-                "        \"low_freq\" : \"2\",\n" + 
-                "        \"high_freq\" : \"3\"\n" + 
-                "      },\n" + 
-                "      \"boost\" : 42.0\n" + 
-                "    }\n" + 
-                "  }\n" + 
+                "{\n" +
+                "  \"common\" : {\n" +
+                "    \"body\" : {\n" +
+                "      \"query\" : \"nelly the elephant not as a cartoon\",\n" +
+                "      \"disable_coord\" : true,\n" +
+                "      \"high_freq_operator\" : \"AND\",\n" +
+                "      \"low_freq_operator\" : \"OR\",\n" +
+                "      \"cutoff_frequency\" : 0.001,\n" +
+                "      \"minimum_should_match\" : {\n" +
+                "        \"low_freq\" : \"2\",\n" +
+                "        \"high_freq\" : \"3\"\n" +
+                "      },\n" +
+                "      \"boost\" : 42.0\n" +
+                "    }\n" +
+                "  }\n" +
                 "}";
 
         CommonTermsQueryBuilder queryBuilder = (CommonTermsQueryBuilder) parseQuery(query);
@@ -132,7 +133,7 @@ public class CommonTermsQueryBuilderTests extends AbstractQueryTestCase<CommonTe
         assertEquals(query, Operator.AND, queryBuilder.highFreqOperator());
         assertEquals(query, "nelly the elephant not as a cartoon", queryBuilder.value());
     }
-    
+
     public void testNoTermsFromQueryString() throws IOException {
         CommonTermsQueryBuilder builder = new CommonTermsQueryBuilder(STRING_FIELD_NAME, "");
         QueryShardContext context = createShardContext();
diff --git a/core/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java
index 8d2b30c9ba8..c24dd25b987 100644
--- a/core/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java
@@ -22,6 +22,7 @@ package org.elasticsearch.index.query;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
index b77abd280a7..6d2348660c6 100644
--- a/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
@@ -24,6 +24,7 @@ import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 import java.util.Collection;
diff --git a/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java
index d46cce420e1..89de6b102ac 100644
--- a/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/ExistsQueryBuilderTests.java
@@ -25,6 +25,7 @@ import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 import java.util.Collection;
diff --git a/core/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java
index f46f2982091..1939f0e6b44 100644
--- a/core/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilderTests.java
@@ -22,6 +22,7 @@ package org.elasticsearch.index.query;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.spans.FieldMaskingSpanQuery;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 
@@ -77,20 +78,20 @@ public class FieldMaskingSpanQueryBuilderTests extends AbstractQueryTestCase<Fie
 
     public void testFromJson() throws IOException {
         String json =
-                "{\n" + 
-                "  \"field_masking_span\" : {\n" + 
-                "    \"query\" : {\n" + 
-                "      \"span_term\" : {\n" + 
-                "        \"value\" : {\n" + 
-                "          \"value\" : 0.5,\n" + 
-                "          \"boost\" : 0.23\n" + 
-                "        }\n" + 
-                "      }\n" + 
-                "    },\n" + 
-                "    \"field\" : \"mapped_geo_shape\",\n" + 
-                "    \"boost\" : 42.0,\n" + 
-                "    \"_name\" : \"KPI\"\n" + 
-                "  }\n" + 
+                "{\n" +
+                "  \"field_masking_span\" : {\n" +
+                "    \"query\" : {\n" +
+                "      \"span_term\" : {\n" +
+                "        \"value\" : {\n" +
+                "          \"value\" : 0.5,\n" +
+                "          \"boost\" : 0.23\n" +
+                "        }\n" +
+                "      }\n" +
+                "    },\n" +
+                "    \"field\" : \"mapped_geo_shape\",\n" +
+                "    \"boost\" : 42.0,\n" +
+                "    \"_name\" : \"KPI\"\n" +
+                "  }\n" +
                 "}";
 
         FieldMaskingSpanQueryBuilder parsed = (FieldMaskingSpanQueryBuilder) parseQuery(json);
diff --git a/core/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java
index a1633ab16e7..92583403638 100644
--- a/core/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/FuzzyQueryBuilderTests.java
@@ -19,21 +19,16 @@
 
 package org.elasticsearch.index.query;
 
-import org.apache.lucene.document.IntPoint;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.FuzzyQuery;
-import org.apache.lucene.search.LegacyNumericRangeQuery;
-import org.apache.lucene.search.PointRangeQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.Version;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
 
-import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -145,17 +140,17 @@ public class FuzzyQueryBuilderTests extends AbstractQueryTestCase<FuzzyQueryBuil
 
     public void testFromJson() throws IOException {
         String json =
-                "{\n" + 
-                "  \"fuzzy\" : {\n" + 
-                "    \"user\" : {\n" + 
-                "      \"value\" : \"ki\",\n" + 
-                "      \"fuzziness\" : \"2\",\n" + 
-                "      \"prefix_length\" : 0,\n" + 
-                "      \"max_expansions\" : 100,\n" + 
-                "      \"transpositions\" : false,\n" + 
-                "      \"boost\" : 42.0\n" + 
-                "    }\n" + 
-                "  }\n" + 
+                "{\n" +
+                "  \"fuzzy\" : {\n" +
+                "    \"user\" : {\n" +
+                "      \"value\" : \"ki\",\n" +
+                "      \"fuzziness\" : \"2\",\n" +
+                "      \"prefix_length\" : 0,\n" +
+                "      \"max_expansions\" : 100,\n" +
+                "      \"transpositions\" : false,\n" +
+                "      \"boost\" : 42.0\n" +
+                "    }\n" +
+                "  }\n" +
                 "}";
         FuzzyQueryBuilder parsed = (FuzzyQueryBuilder) parseQuery(json);
         checkGeneratedJson(json, parsed);
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java
index 4c6b26d6848..48b0ed7d4ce 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilderTests.java
@@ -32,6 +32,7 @@ import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.search.geo.InMemoryGeoBoundingBoxQuery;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.test.geo.RandomShapeGenerator;
 import org.locationtech.spatial4j.io.GeohashUtils;
 import org.locationtech.spatial4j.shape.Rectangle;
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java
index b57dc61ee4a..7780d218b52 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java
@@ -28,6 +28,7 @@ import org.elasticsearch.common.geo.GeoDistance;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.index.search.geo.GeoDistanceRangeQuery;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.test.geo.RandomShapeGenerator;
 import org.locationtech.spatial4j.shape.Point;
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java
index 3b3ffa18cab..a14bf7e0f31 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java
@@ -32,6 +32,7 @@ import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.search.geo.GeoDistanceRangeQuery;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.test.geo.RandomGeoGenerator;
 
 import java.io.IOException;
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
index 9313b4278b0..b41d8efafdf 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
@@ -32,6 +32,7 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.search.geo.GeoPolygonQuery;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.test.geo.RandomShapeGenerator;
 import org.elasticsearch.test.geo.RandomShapeGenerator.ShapeType;
 import org.locationtech.spatial4j.shape.jts.JtsGeometry;
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
index aaf804e7703..940dbb3242c 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoShapeQueryBuilderTests.java
@@ -38,6 +38,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.get.GetResult;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.test.geo.RandomShapeGenerator;
 import org.elasticsearch.test.geo.RandomShapeGenerator.ShapeType;
 import org.junit.After;
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeohashCellQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeohashCellQueryBuilderTests.java
index 1a7f4e362ac..208b0479209 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeohashCellQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeohashCellQueryBuilderTests.java
@@ -29,6 +29,7 @@ import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.index.mapper.geo.BaseGeoPointFieldMapper;
 import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
 import org.elasticsearch.index.query.GeohashCellQuery.Builder;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.test.geo.RandomShapeGenerator;
 import org.locationtech.spatial4j.shape.Point;
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java
index 2df15592f45..1c6a99ca099 100644
--- a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java
@@ -48,6 +48,8 @@ import org.elasticsearch.search.fetch.innerhits.InnerHitsContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.test.AbstractQueryTestCase;
+import org.junit.Before;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
@@ -67,10 +69,9 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase<HasChildQue
 
     private static String similarity;
 
-    @BeforeClass
-    public static void before() throws Exception {
+    @Override
+    protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
         similarity = randomFrom("classic", "BM25");
-        MapperService mapperService = createShardContext().getMapperService();
         mapperService.merge(PARENT_TYPE, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(PARENT_TYPE,
                 STRING_FIELD_NAME, "type=text",
                 STRING_FIELD_NAME_2, "type=keyword",
diff --git a/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java
index d1b0c6ab222..7b700cbfc7f 100644
--- a/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java
@@ -38,6 +38,8 @@ import org.elasticsearch.search.fetch.innerhits.InnerHitsContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.test.AbstractQueryTestCase;
+import org.junit.Before;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
@@ -53,9 +55,8 @@ public class HasParentQueryBuilderTests extends AbstractQueryTestCase<HasParentQ
     protected static final String PARENT_TYPE = "parent";
     protected static final String CHILD_TYPE = "child";
 
-    @BeforeClass
-    public static void beforeClass() throws Exception {
-        MapperService mapperService = createShardContext().getMapperService();
+    @Override
+    protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
         mapperService.merge(PARENT_TYPE, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(PARENT_TYPE,
                 STRING_FIELD_NAME, "type=text",
                 STRING_FIELD_NAME_2, "type=keyword",
diff --git a/core/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java
index c7cfa4e0e1d..42f4aaf56aa 100644
--- a/core/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java
@@ -25,6 +25,7 @@ import org.apache.lucene.search.Query;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -153,12 +154,12 @@ public class IdsQueryBuilderTests extends AbstractQueryTestCase<IdsQueryBuilder>
 
     public void testFromJson() throws IOException {
         String json =
-                "{\n" + 
-                "  \"ids\" : {\n" + 
-                "    \"type\" : [ \"my_type\" ],\n" + 
-                "    \"values\" : [ \"1\", \"100\", \"4\" ],\n" + 
-                "    \"boost\" : 1.0\n" + 
-                "  }\n" + 
+                "{\n" +
+                "  \"ids\" : {\n" +
+                "    \"type\" : [ \"my_type\" ],\n" +
+                "    \"values\" : [ \"1\", \"100\", \"4\" ],\n" +
+                "    \"boost\" : 1.0\n" +
+                "  }\n" +
                 "}";
         IdsQueryBuilder parsed = (IdsQueryBuilder) parseQuery(json);
         checkGeneratedJson(json, parsed);
diff --git a/core/src/test/java/org/elasticsearch/index/query/IndicesQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/IndicesQueryBuilderTests.java
index c234016c465..12527a927c4 100644
--- a/core/src/test/java/org/elasticsearch/index/query/IndicesQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/IndicesQueryBuilderTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.Query;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 
@@ -77,27 +78,27 @@ public class IndicesQueryBuilderTests extends AbstractQueryTestCase<IndicesQuery
 
     public void testFromJson() throws IOException {
         String json =
-                "{\n" + 
-                "  \"indices\" : {\n" + 
-                "    \"indices\" : [ \"index1\", \"index2\" ],\n" + 
-                "    \"query\" : {\n" + 
-                "      \"term\" : {\n" + 
-                "        \"tag\" : {\n" + 
-                "          \"value\" : \"wow\",\n" + 
-                "          \"boost\" : 1.0\n" + 
-                "        }\n" + 
-                "      }\n" + 
-                "    },\n" + 
-                "    \"no_match_query\" : {\n" + 
-                "      \"term\" : {\n" + 
-                "        \"tag\" : {\n" + 
-                "          \"value\" : \"kow\",\n" + 
-                "          \"boost\" : 1.0\n" + 
-                "        }\n" + 
-                "      }\n" + 
-                "    },\n" + 
-                "    \"boost\" : 1.0\n" + 
-                "  }\n" + 
+                "{\n" +
+                "  \"indices\" : {\n" +
+                "    \"indices\" : [ \"index1\", \"index2\" ],\n" +
+                "    \"query\" : {\n" +
+                "      \"term\" : {\n" +
+                "        \"tag\" : {\n" +
+                "          \"value\" : \"wow\",\n" +
+                "          \"boost\" : 1.0\n" +
+                "        }\n" +
+                "      }\n" +
+                "    },\n" +
+                "    \"no_match_query\" : {\n" +
+                "      \"term\" : {\n" +
+                "        \"tag\" : {\n" +
+                "          \"value\" : \"kow\",\n" +
+                "          \"boost\" : 1.0\n" +
+                "        }\n" +
+                "      }\n" +
+                "    },\n" +
+                "    \"boost\" : 1.0\n" +
+                "  }\n" +
                 "}";
         IndicesQueryBuilder parsed = (IndicesQueryBuilder) parseQuery(json);
         checkGeneratedJson(json, parsed);
diff --git a/core/src/test/java/org/elasticsearch/index/query/MatchAllQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MatchAllQueryBuilderTests.java
index a5aea5eb7aa..ef8a4ffc28d 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MatchAllQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MatchAllQueryBuilderTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -52,13 +53,13 @@ public class MatchAllQueryBuilderTests extends AbstractQueryTestCase<MatchAllQue
 
     public void testFromJson() throws IOException {
         String json =
-                "{\n" + 
-                "  \"match_all\" : {\n" + 
-                "    \"boost\" : 1.2\n" + 
-                "  }\n" + 
+                "{\n" +
+                "  \"match_all\" : {\n" +
+                "    \"boost\" : 1.2\n" +
+                "  }\n" +
                 "}";
         MatchAllQueryBuilder parsed = (MatchAllQueryBuilder) parseQuery(json);
         checkGeneratedJson(json, parsed);
         assertEquals(json, 1.2, parsed.boost(), 0.0001);
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/test/java/org/elasticsearch/index/query/MatchNoneQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MatchNoneQueryBuilderTests.java
index e2ebbd8864d..4806ec368a4 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MatchNoneQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MatchNoneQueryBuilderTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 
@@ -40,10 +41,10 @@ public class MatchNoneQueryBuilderTests extends AbstractQueryTestCase<MatchNoneQ
 
     public void testFromJson() throws IOException {
         String json =
-                "{\n" + 
-                "  \"match_none\" : {\n" + 
-                "    \"boost\" : 1.2\n" + 
-                "  }\n" + 
+                "{\n" +
+                "  \"match_none\" : {\n" +
+                "    \"boost\" : 1.2\n" +
+                "  }\n" +
                 "}";
         MatchNoneQueryBuilder parsed = (MatchNoneQueryBuilder) parseQuery(json);
         checkGeneratedJson(json, parsed);
diff --git a/core/src/test/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilderTests.java
index bb6f332c901..f5b2690db64 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilderTests.java
@@ -25,6 +25,8 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
+import org.elasticsearch.test.AbstractQueryTestCase;
+
 import java.io.IOException;
 import static org.hamcrest.CoreMatchers.either;
 import static org.hamcrest.CoreMatchers.instanceOf;
diff --git a/core/src/test/java/org/elasticsearch/index/query/MatchPhraseQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MatchPhraseQueryBuilderTests.java
index dc5a8d58389..d20bf655e2c 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MatchPhraseQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MatchPhraseQueryBuilderTests.java
@@ -25,6 +25,7 @@ import org.apache.lucene.search.PointRangeQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
index c5e01c1d0ff..6a9a07c59dd 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MatchQueryBuilderTests.java
@@ -36,6 +36,7 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.search.MatchQuery;
 import org.elasticsearch.index.search.MatchQuery.Type;
 import org.elasticsearch.index.search.MatchQuery.ZeroTermsQuery;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.hamcrest.Matcher;
 
 import java.io.IOException;
diff --git a/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
index 31a11e36b0a..e4b0a6893ac 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
@@ -41,6 +41,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.query.MoreLikeThisQueryBuilder.Item;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -304,30 +305,30 @@ public class MoreLikeThisQueryBuilderTests extends AbstractQueryTestCase<MoreLik
 
     public void testFromJson() throws IOException {
         String json =
-                "{\n" + 
+                "{\n" +
                 "  \"more_like_this\" : {\n" +
-                "    \"fields\" : [ \"title\", \"description\" ],\n" + 
-                "    \"like\" : [ \"and potentially some more text here as well\", {\n" + 
-                "      \"_index\" : \"imdb\",\n" + 
-                "      \"_type\" : \"movies\",\n" + 
-                "      \"_id\" : \"1\"\n" + 
-                "    }, {\n" + 
-                "      \"_index\" : \"imdb\",\n" + 
-                "      \"_type\" : \"movies\",\n" + 
-                "      \"_id\" : \"2\"\n" + 
-                "    } ],\n" + 
-                "    \"max_query_terms\" : 12,\n" + 
-                "    \"min_term_freq\" : 1,\n" + 
-                "    \"min_doc_freq\" : 5,\n" + 
-                "    \"max_doc_freq\" : 2147483647,\n" + 
-                "    \"min_word_length\" : 0,\n" + 
-                "    \"max_word_length\" : 0,\n" + 
-                "    \"minimum_should_match\" : \"30%\",\n" + 
-                "    \"boost_terms\" : 0.0,\n" + 
-                "    \"include\" : false,\n" + 
-                "    \"fail_on_unsupported_field\" : true,\n" + 
-                "    \"boost\" : 1.0\n" + 
-                "  }\n" + 
+                "    \"fields\" : [ \"title\", \"description\" ],\n" +
+                "    \"like\" : [ \"and potentially some more text here as well\", {\n" +
+                "      \"_index\" : \"imdb\",\n" +
+                "      \"_type\" : \"movies\",\n" +
+                "      \"_id\" : \"1\"\n" +
+                "    }, {\n" +
+                "      \"_index\" : \"imdb\",\n" +
+                "      \"_type\" : \"movies\",\n" +
+                "      \"_id\" : \"2\"\n" +
+                "    } ],\n" +
+                "    \"max_query_terms\" : 12,\n" +
+                "    \"min_term_freq\" : 1,\n" +
+                "    \"min_doc_freq\" : 5,\n" +
+                "    \"max_doc_freq\" : 2147483647,\n" +
+                "    \"min_word_length\" : 0,\n" +
+                "    \"max_word_length\" : 0,\n" +
+                "    \"minimum_should_match\" : \"30%\",\n" +
+                "    \"boost_terms\" : 0.0,\n" +
+                "    \"include\" : false,\n" +
+                "    \"fail_on_unsupported_field\" : true,\n" +
+                "    \"boost\" : 1.0\n" +
+                "  }\n" +
                 "}";
 
         MoreLikeThisQueryBuilder parsed = (MoreLikeThisQueryBuilder) parseQuery(json);
diff --git a/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
index f81b14b5b87..e2222002ce0 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
@@ -37,6 +37,7 @@ import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
 import org.elasticsearch.index.query.MultiMatchQueryBuilder.Type;
 import org.elasticsearch.index.search.MatchQuery;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 import java.util.HashMap;
diff --git a/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java
index 23d595107b4..04bbadb02c2 100644
--- a/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java
@@ -36,6 +36,7 @@ import org.elasticsearch.search.fetch.innerhits.InnerHitsContext;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -49,9 +50,7 @@ import static org.hamcrest.CoreMatchers.notNullValue;
 public class NestedQueryBuilderTests extends AbstractQueryTestCase<NestedQueryBuilder> {
 
     @Override
-    public void setUp() throws Exception {
-        super.setUp();
-        MapperService mapperService = createShardContext().getMapperService();
+    protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
         mapperService.merge("nested_doc", new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef("nested_doc",
                 STRING_FIELD_NAME, "type=text",
                 INT_FIELD_NAME, "type=integer",
diff --git a/core/src/test/java/org/elasticsearch/index/query/ParentIdQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/ParentIdQueryBuilderTests.java
index 070fb08dc67..89d0829e012 100644
--- a/core/src/test/java/org/elasticsearch/index/query/ParentIdQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/ParentIdQueryBuilderTests.java
@@ -28,6 +28,7 @@ import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.hamcrest.Matchers;
 
 import java.io.IOException;
@@ -42,25 +43,23 @@ public class ParentIdQueryBuilderTests extends AbstractQueryTestCase<ParentIdQue
     protected static final String CHILD_TYPE = "child";
 
     @Override
-    public void setUp() throws Exception {
-        super.setUp();
-        MapperService mapperService = createShardContext().getMapperService();
+    protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
         mapperService.merge(PARENT_TYPE, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(PARENT_TYPE,
-            STRING_FIELD_NAME, "type=text",
-            INT_FIELD_NAME, "type=integer",
-            DOUBLE_FIELD_NAME, "type=double",
-            BOOLEAN_FIELD_NAME, "type=boolean",
-            DATE_FIELD_NAME, "type=date",
-            OBJECT_FIELD_NAME, "type=object"
+                STRING_FIELD_NAME, "type=text",
+                INT_FIELD_NAME, "type=integer",
+                DOUBLE_FIELD_NAME, "type=double",
+                BOOLEAN_FIELD_NAME, "type=boolean",
+                DATE_FIELD_NAME, "type=date",
+                OBJECT_FIELD_NAME, "type=object"
         ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
         mapperService.merge(CHILD_TYPE, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(CHILD_TYPE,
-            "_parent", "type=" + PARENT_TYPE,
-            STRING_FIELD_NAME, "type=text",
-            INT_FIELD_NAME, "type=integer",
-            DOUBLE_FIELD_NAME, "type=double",
-            BOOLEAN_FIELD_NAME, "type=boolean",
-            DATE_FIELD_NAME, "type=date",
-            OBJECT_FIELD_NAME, "type=object"
+                "_parent", "type=" + PARENT_TYPE,
+                STRING_FIELD_NAME, "type=text",
+                INT_FIELD_NAME, "type=integer",
+                DOUBLE_FIELD_NAME, "type=double",
+                BOOLEAN_FIELD_NAME, "type=boolean",
+                DATE_FIELD_NAME, "type=date",
+                OBJECT_FIELD_NAME, "type=object"
         ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
index d49e33e91cb..aba6bec096b 100644
--- a/core/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/PrefixQueryBuilderTests.java
@@ -23,12 +23,11 @@ import org.apache.lucene.index.Term;
 import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
-import java.util.Arrays;
 
 import static org.elasticsearch.index.query.QueryBuilders.prefixQuery;
-import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
index cf16456b2bf..8a3552cf8f2 100644
--- a/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
@@ -38,6 +38,7 @@ import org.apache.lucene.search.MultiTermQuery;
 import org.apache.lucene.util.automaton.TooComplexToDeterminizeException;
 import org.elasticsearch.common.lucene.all.AllTermQuery;
 import org.elasticsearch.common.unit.Fuzziness;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.hamcrest.Matchers;
 import org.joda.time.DateTimeZone;
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java b/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java
index 23ef8045cc1..cf4921b7a43 100644
--- a/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java
+++ b/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.query;
 
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.util.Random;
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
index dd97e7a68eb..b30ff61f6f8 100644
--- a/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/RangeQueryBuilderTests.java
@@ -30,6 +30,7 @@ import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType.Relation;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.chrono.ISOChronology;
diff --git a/core/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java
index 777d8a6800b..64a62fb766f 100644
--- a/core/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/RegexpQueryBuilderTests.java
@@ -21,12 +21,12 @@ package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.RegexpQuery;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
-import static org.elasticsearch.index.query.QueryBuilders.prefixQuery;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -85,15 +85,15 @@ public class RegexpQueryBuilderTests extends AbstractQueryTestCase<RegexpQueryBu
 
     public void testFromJson() throws IOException {
         String json =
-                "{\n" + 
-                "  \"regexp\" : {\n" + 
-                "    \"name.first\" : {\n" + 
-                "      \"value\" : \"s.*y\",\n" + 
-                "      \"flags_value\" : 7,\n" + 
-                "      \"max_determinized_states\" : 20000,\n" + 
-                "      \"boost\" : 1.0\n" + 
-                "    }\n" + 
-                "  }\n" + 
+                "{\n" +
+                "  \"regexp\" : {\n" +
+                "    \"name.first\" : {\n" +
+                "      \"value\" : \"s.*y\",\n" +
+                "      \"flags_value\" : 7,\n" +
+                "      \"max_determinized_states\" : 20000,\n" +
+                "      \"boost\" : 1.0\n" +
+                "    }\n" +
+                "  }\n" +
                 "}";
 
         RegexpQueryBuilder parsed = (RegexpQueryBuilder) parseQuery(json);
diff --git a/core/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java
index 57bc7fd55d8..427ce45f9cd 100644
--- a/core/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/ScriptQueryBuilderTests.java
@@ -23,6 +23,7 @@ import org.apache.lucene.search.Query;
 import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -49,16 +50,16 @@ public class ScriptQueryBuilderTests extends AbstractQueryTestCase<ScriptQueryBu
 
     public void testFromJson() throws IOException {
         String json =
-                "{\n" + 
-                "  \"script\" : {\n" + 
-                "    \"script\" : {\n" + 
-                "      \"inline\" : \"5\",\n" + 
-                "      \"lang\" : \"mockscript\",\n" + 
-                "      \"params\" : { }\n" + 
-                "    },\n" + 
-                "    \"boost\" : 1.0,\n" + 
-                "    \"_name\" : \"PcKdEyPOmR\"\n" + 
-                "  }\n" + 
+                "{\n" +
+                "  \"script\" : {\n" +
+                "    \"script\" : {\n" +
+                "      \"inline\" : \"5\",\n" +
+                "      \"lang\" : \"mockscript\",\n" +
+                "      \"params\" : { }\n" +
+                "    },\n" +
+                "    \"boost\" : 1.0,\n" +
+                "    \"_name\" : \"PcKdEyPOmR\"\n" +
+                "  }\n" +
                 "}";
 
         ScriptQueryBuilder parsed = (ScriptQueryBuilder) parseQuery(json);
diff --git a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
index 940d15bb0f3..9168b489eb2 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
@@ -28,6 +28,7 @@ import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 import java.util.HashMap;
diff --git a/core/src/test/java/org/elasticsearch/index/query/SpanContainingQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SpanContainingQueryBuilderTests.java
index 999d85c6307..96b89fa521c 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SpanContainingQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SpanContainingQueryBuilderTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.spans.SpanContainingQuery;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/SpanFirstQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SpanFirstQueryBuilderTests.java
index 05d2fce5368..a0279a7942e 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SpanFirstQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SpanFirstQueryBuilderTests.java
@@ -24,6 +24,7 @@ import org.apache.lucene.search.spans.SpanFirstQuery;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 
@@ -81,19 +82,19 @@ public class SpanFirstQueryBuilderTests extends AbstractQueryTestCase<SpanFirstQ
 
     public void testFromJson() throws IOException {
         String json =
-                "{\n" + 
-                "  \"span_first\" : {\n" + 
-                "    \"match\" : {\n" + 
-                "      \"span_term\" : {\n" + 
-                "        \"user\" : {\n" + 
-                "          \"value\" : \"kimchy\",\n" + 
-                "          \"boost\" : 1.0\n" + 
-                "        }\n" + 
-                "      }\n" + 
-                "    },\n" + 
-                "    \"end\" : 3,\n" + 
-                "    \"boost\" : 1.0\n" + 
-                "  }\n" + 
+                "{\n" +
+                "  \"span_first\" : {\n" +
+                "    \"match\" : {\n" +
+                "      \"span_term\" : {\n" +
+                "        \"user\" : {\n" +
+                "          \"value\" : \"kimchy\",\n" +
+                "          \"boost\" : 1.0\n" +
+                "        }\n" +
+                "      }\n" +
+                "    },\n" +
+                "    \"end\" : 3,\n" +
+                "    \"boost\" : 1.0\n" +
+                "  }\n" +
                 "}";
 
         SpanFirstQueryBuilder parsed = (SpanFirstQueryBuilder) parseQuery(json);
diff --git a/core/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java
index e9ac69001ef..81d7598df1d 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilderTests.java
@@ -25,6 +25,7 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.search.spans.SpanBoostQuery;
 import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
 import org.apache.lucene.search.spans.SpanQuery;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 
@@ -90,18 +91,18 @@ public class SpanMultiTermQueryBuilderTests extends AbstractQueryTestCase<SpanMu
 
     public void testFromJson() throws IOException {
         String json =
-                "{\n" + 
-                "  \"span_multi\" : {\n" + 
-                "    \"match\" : {\n" + 
-                "      \"prefix\" : {\n" + 
-                "        \"user\" : {\n" + 
-                "          \"value\" : \"ki\",\n" + 
-                "          \"boost\" : 1.08\n" + 
-                "        }\n" + 
-                "      }\n" + 
-                "    },\n" + 
-                "    \"boost\" : 1.0\n" + 
-                "  }\n" + 
+                "{\n" +
+                "  \"span_multi\" : {\n" +
+                "    \"match\" : {\n" +
+                "      \"prefix\" : {\n" +
+                "        \"user\" : {\n" +
+                "          \"value\" : \"ki\",\n" +
+                "          \"boost\" : 1.08\n" +
+                "        }\n" +
+                "      }\n" +
+                "    },\n" +
+                "    \"boost\" : 1.0\n" +
+                "  }\n" +
                 "}";
 
         SpanMultiTermQueryBuilder parsed = (SpanMultiTermQueryBuilder) parseQuery(json);
diff --git a/core/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java
index cbad5a60aa3..f2f04d04bbf 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SpanNearQueryBuilderTests.java
@@ -24,6 +24,7 @@ import org.apache.lucene.search.spans.SpanNearQuery;
 import org.apache.lucene.search.spans.SpanQuery;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 import java.util.Iterator;
diff --git a/core/src/test/java/org/elasticsearch/index/query/SpanNotQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SpanNotQueryBuilderTests.java
index e536dec0598..ccfda9596f3 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SpanNotQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SpanNotQueryBuilderTests.java
@@ -24,6 +24,7 @@ import org.apache.lucene.search.spans.SpanNotQuery;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java
index 1622210b731..183bab7a818 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SpanOrQueryBuilderTests.java
@@ -22,6 +22,7 @@ package org.elasticsearch.index.query;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.spans.SpanOrQuery;
 import org.apache.lucene.search.spans.SpanQuery;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 import java.util.Iterator;
diff --git a/core/src/test/java/org/elasticsearch/index/query/SpanWithinQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/SpanWithinQueryBuilderTests.java
index 3b7723be000..a05a2a1af81 100644
--- a/core/src/test/java/org/elasticsearch/index/query/SpanWithinQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/SpanWithinQueryBuilderTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.spans.SpanWithinQuery;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/TemplateQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/TemplateQueryBuilderTests.java
index ee6621bef5b..d0548fd2780 100644
--- a/core/src/test/java/org/elasticsearch/index/query/TemplateQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/TemplateQueryBuilderTests.java
@@ -28,7 +28,8 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.script.Script.ScriptParseException;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.script.Template;
-import org.junit.BeforeClass;
+import org.elasticsearch.test.AbstractQueryTestCase;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -40,10 +41,10 @@ public class TemplateQueryBuilderTests extends AbstractQueryTestCase<TemplateQue
     /**
      * The query type all template tests will be based on.
      */
-    private static QueryBuilder templateBase;
+    private QueryBuilder templateBase;
 
-    @BeforeClass
-    public static void setupClass() {
+    @Before
+    public void before() {
         templateBase = RandomQueryBuilder.createQuery(random());
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
index 2dd509c159b..a8bdeb69362 100644
--- a/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
@@ -36,6 +36,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.indices.TermsLookup;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.hamcrest.Matchers;
 import org.junit.Before;
 
@@ -45,7 +46,6 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.index.query.QueryBuilders.prefixQuery;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
diff --git a/core/src/test/java/org/elasticsearch/index/query/TypeQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/TypeQueryBuilderTests.java
index db9a4d8e7cd..a3265d06d70 100644
--- a/core/src/test/java/org/elasticsearch/index/query/TypeQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/TypeQueryBuilderTests.java
@@ -23,6 +23,7 @@ import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
index 9ad777b80e4..34a006bf8e3 100644
--- a/core/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/WildcardQueryBuilderTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.WildcardQuery;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/WrapperQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/WrapperQueryBuilderTests.java
index e2a0d551919..4612aedb2c3 100644
--- a/core/src/test/java/org/elasticsearch/index/query/WrapperQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/WrapperQueryBuilderTests.java
@@ -26,6 +26,7 @@ import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
diff --git a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
index 0cbd80dc76c..52632edcf47 100644
--- a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
@@ -36,8 +36,11 @@ import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 import org.elasticsearch.common.lucene.search.function.WeightFactorFunction;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
-import org.elasticsearch.index.query.AbstractQueryTestCase;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
@@ -53,9 +56,11 @@ import org.elasticsearch.search.MultiValueMode;
 import org.hamcrest.Matcher;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
+import org.junit.Before;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
 
@@ -74,10 +79,10 @@ import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.nullValue;
 
 public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase<FunctionScoreQueryBuilder> {
-    @BeforeClass
-    public static void registerTestRandomScoreFunction() {
-        getSearchModule().registerScoreFunction(RandomScoreFunctionBuilderWithFixedSeed::new,
-                RandomScoreFunctionBuilderWithFixedSeed::fromXContent, RandomScoreFunctionBuilderWithFixedSeed.FUNCTION_NAME_FIELD);
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return Collections.singleton(TestPlugin.class);
     }
 
     @Override
@@ -731,4 +736,23 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase<Functi
             return replacement;
         }
     }
+
+    public static class TestPlugin extends Plugin {
+
+        @Override
+        public String name() {
+            return "test-plugin";
+        }
+
+        @Override
+        public String description() {
+            return "Adds random function with fixed seed";
+        }
+
+        public void onModule(SearchModule module) {
+            module.registerScoreFunction(RandomScoreFunctionBuilderWithFixedSeed::new,
+                    RandomScoreFunctionBuilderWithFixedSeed::fromXContent, RandomScoreFunctionBuilderWithFixedSeed.FUNCTION_NAME_FIELD);
+        }
+
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
index 3be16393e2a..e0446fe329b 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
@@ -36,10 +36,6 @@ import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequestBui
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder;
-import org.elasticsearch.action.percolate.MultiPercolateRequestBuilder;
-import org.elasticsearch.action.percolate.MultiPercolateResponse;
-import org.elasticsearch.action.percolate.PercolateRequestBuilder;
-import org.elasticsearch.action.percolate.PercolateSourceBuilder;
 import org.elasticsearch.action.search.MultiSearchRequestBuilder;
 import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.action.search.SearchRequestBuilder;
@@ -57,7 +53,6 @@ import org.elasticsearch.test.ESIntegTestCase;
 import java.util.Collection;
 import java.util.function.Function;
 
-import static org.elasticsearch.action.percolate.PercolateSourceBuilder.docBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
@@ -75,7 +70,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
     }
 
     public void testSpecifiedIndexUnavailableMultipleIndices() throws Exception {
-        assertAcked(prepareCreate("test1").addMapping("query", "query", "type=percolator"));
+        assertAcked(prepareCreate("test1"));
         ensureYellow();
 
         // Verify defaults
@@ -90,8 +85,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(validateQuery("test1", "test2"), true);
         verify(aliasExists("test1", "test2"), true);
         verify(typesExists("test1", "test2"), true);
-        verify(percolate("test1", "test2"), true);
-        verify(mpercolate(null, "test1", "test2"), false);
         verify(getAliases("test1", "test2"), true);
         verify(getFieldMapping("test1", "test2"), true);
         verify(getMapping("test1", "test2"), true);
@@ -109,8 +102,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(validateQuery("test1", "test2").setIndicesOptions(options), true);
         verify(aliasExists("test1", "test2").setIndicesOptions(options), true);
         verify(typesExists("test1", "test2").setIndicesOptions(options), true);
-        verify(percolate("test1", "test2").setIndicesOptions(options), true);
-        verify(mpercolate(options, "test1", "test2").setIndicesOptions(options), false);
         verify(getAliases("test1", "test2").setIndicesOptions(options), true);
         verify(getFieldMapping("test1", "test2").setIndicesOptions(options), true);
         verify(getMapping("test1", "test2").setIndicesOptions(options), true);
@@ -128,15 +119,13 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(validateQuery("test1", "test2").setIndicesOptions(options), false);
         verify(aliasExists("test1", "test2").setIndicesOptions(options), false);
         verify(typesExists("test1", "test2").setIndicesOptions(options), false);
-        verify(percolate("test1", "test2").setIndicesOptions(options), false);
-        verify(mpercolate(options, "test1", "test2").setIndicesOptions(options), false);
         verify(getAliases("test1", "test2").setIndicesOptions(options), false);
         verify(getFieldMapping("test1", "test2").setIndicesOptions(options), false);
         verify(getMapping("test1", "test2").setIndicesOptions(options), false);
         verify(getSettings("test1", "test2").setIndicesOptions(options), false);
 
         options = IndicesOptions.strictExpandOpen();
-        assertAcked(prepareCreate("test2").addMapping("query", "query", "type=percolator"));
+        assertAcked(prepareCreate("test2"));
         ensureYellow();
         verify(search("test1", "test2").setIndicesOptions(options), false);
         verify(msearch(options, "test1", "test2").setIndicesOptions(options), false);
@@ -149,8 +138,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(validateQuery("test1", "test2").setIndicesOptions(options), false);
         verify(aliasExists("test1", "test2").setIndicesOptions(options), false);
         verify(typesExists("test1", "test2").setIndicesOptions(options), false);
-        verify(percolate("test1", "test2").setIndicesOptions(options), false);
-        verify(mpercolate(options, "test1", "test2").setIndicesOptions(options), false);
         verify(getAliases("test1", "test2").setIndicesOptions(options), false);
         verify(getFieldMapping("test1", "test2").setIndicesOptions(options), false);
         verify(getMapping("test1", "test2").setIndicesOptions(options), false);
@@ -158,7 +145,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
     }
 
     public void testSpecifiedIndexUnavailableSingleIndexThatIsClosed() throws Exception {
-        assertAcked(prepareCreate("test1").addMapping("query", "query", "type=percolator"));
+        assertAcked(prepareCreate("test1"));
         // we need to wait until all shards are allocated since recovery from
         // gateway will fail unless the majority of the replicas was allocated
         // pre-closing. with lots of replicas this will fail.
@@ -178,8 +165,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(validateQuery("test1").setIndicesOptions(options), true);
         verify(aliasExists("test1").setIndicesOptions(options), true);
         verify(typesExists("test1").setIndicesOptions(options), true);
-        verify(percolate("test1").setIndicesOptions(options), true);
-        verify(mpercolate(options, "test1").setIndicesOptions(options), true);
         verify(getAliases("test1").setIndicesOptions(options), true);
         verify(getFieldMapping("test1").setIndicesOptions(options), true);
         verify(getMapping("test1").setIndicesOptions(options), true);
@@ -197,8 +182,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(validateQuery("test1").setIndicesOptions(options), false);
         verify(aliasExists("test1").setIndicesOptions(options), false);
         verify(typesExists("test1").setIndicesOptions(options), false);
-        verify(percolate("test1").setIndicesOptions(options), false);
-        verify(mpercolate(options, "test1").setIndicesOptions(options), false);
         verify(getAliases("test1").setIndicesOptions(options), false);
         verify(getFieldMapping("test1").setIndicesOptions(options), false);
         verify(getMapping("test1").setIndicesOptions(options), false);
@@ -219,8 +202,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(validateQuery("test1").setIndicesOptions(options), false);
         verify(aliasExists("test1").setIndicesOptions(options), false);
         verify(typesExists("test1").setIndicesOptions(options), false);
-        verify(percolate("test1").setIndicesOptions(options), false);
-        verify(mpercolate(options, "test1").setIndicesOptions(options), false);
         verify(getAliases("test1").setIndicesOptions(options), false);
         verify(getFieldMapping("test1").setIndicesOptions(options), false);
         verify(getMapping("test1").setIndicesOptions(options), false);
@@ -240,7 +221,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(validateQuery("test1").setIndicesOptions(options), true);
         verify(aliasExists("test1").setIndicesOptions(options), true);
         verify(typesExists("test1").setIndicesOptions(options), true);
-        verify(percolate("test1").setIndicesOptions(options), true);
         verify(getAliases("test1").setIndicesOptions(options), true);
         verify(getFieldMapping("test1").setIndicesOptions(options), true);
         verify(getMapping("test1").setIndicesOptions(options), true);
@@ -258,13 +238,12 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(validateQuery("test1").setIndicesOptions(options), false);
         verify(aliasExists("test1").setIndicesOptions(options), false);
         verify(typesExists("test1").setIndicesOptions(options), false);
-        verify(percolate("test1").setIndicesOptions(options), false);
         verify(getAliases("test1").setIndicesOptions(options), false);
         verify(getFieldMapping("test1").setIndicesOptions(options), false);
         verify(getMapping("test1").setIndicesOptions(options), false);
         verify(getSettings("test1").setIndicesOptions(options), false);
 
-        assertAcked(prepareCreate("test1").addMapping("query", "query", "type=percolator"));
+        assertAcked(prepareCreate("test1"));
         ensureYellow();
 
         options = IndicesOptions.strictExpandOpenAndForbidClosed();
@@ -279,7 +258,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(validateQuery("test1").setIndicesOptions(options), false);
         verify(aliasExists("test1").setIndicesOptions(options), false);
         verify(typesExists("test1").setIndicesOptions(options), false);
-        verify(percolate("test1").setIndicesOptions(options), false);
         verify(getAliases("test1").setIndicesOptions(options), false);
         verify(getFieldMapping("test1").setIndicesOptions(options), false);
         verify(getMapping("test1").setIndicesOptions(options), false);
@@ -330,8 +308,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(validateQuery(indices), true);
         verify(aliasExists(indices), false);
         verify(typesExists(indices), false);
-        verify(percolate(indices), false);
-        verify(mpercolate(null, indices), false);
         verify(getAliases(indices), false);
         verify(getFieldMapping(indices), false);
         verify(getMapping(indices), false);
@@ -350,14 +326,12 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(validateQuery(indices).setIndicesOptions(options), false);
         verify(aliasExists(indices).setIndicesOptions(options), false);
         verify(typesExists(indices).setIndicesOptions(options), false);
-        verify(percolate(indices).setIndicesOptions(options), false);
-        verify(mpercolate(options, indices), false);
         verify(getAliases(indices).setIndicesOptions(options), false);
         verify(getFieldMapping(indices).setIndicesOptions(options), false);
         verify(getMapping(indices).setIndicesOptions(options), false);
         verify(getSettings(indices).setIndicesOptions(options), false);
 
-        assertAcked(prepareCreate("foobar").addMapping("query", "query", "type=percolator"));
+        assertAcked(prepareCreate("foobar"));
         client().prepareIndex("foobar", "type", "1").setSource("k", "v").setRefresh(true).execute().actionGet();
 
         // Verify defaults for wildcards, with one wildcard expression and one existing index
@@ -373,8 +347,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(validateQuery(indices), false);
         verify(aliasExists(indices), false);
         verify(typesExists(indices), false);
-        verify(percolate(indices), false);
-        verify(mpercolate(null, indices), false);
         verify(getAliases(indices), false);
         verify(getFieldMapping(indices), false);
         verify(getMapping(indices), false);
@@ -393,8 +365,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(validateQuery(indices), true);
         verify(aliasExists(indices), false);
         verify(typesExists(indices), false);
-        verify(percolate(indices), false);
-        verify(mpercolate(null, indices), false);
         verify(getAliases(indices), false);
         verify(getFieldMapping(indices), false);
         verify(getMapping(indices), false);
@@ -413,8 +383,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(validateQuery(indices).setIndicesOptions(options), false);
         verify(aliasExists(indices).setIndicesOptions(options), false);
         verify(typesExists(indices).setIndicesOptions(options), false);
-        verify(percolate(indices).setIndicesOptions(options), false);
-        verify(mpercolate(options, indices), false);
         verify(getAliases(indices).setIndicesOptions(options), false);
         verify(getFieldMapping(indices).setIndicesOptions(options), false);
         verify(getMapping(indices).setIndicesOptions(options), false);
@@ -724,20 +692,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         return client().admin().indices().prepareTypesExists(indices).setTypes("dummy");
     }
 
-    private static PercolateRequestBuilder percolate(String... indices) {
-        return client().preparePercolate().setIndices(indices)
-                .setSource(new PercolateSourceBuilder().setDoc(docBuilder().setDoc("k", "v")))
-                .setDocumentType("type");
-    }
-
-    private static MultiPercolateRequestBuilder mpercolate(IndicesOptions options, String... indices) {
-        MultiPercolateRequestBuilder builder = client().prepareMultiPercolate();
-        if (options != null) {
-            builder.setIndicesOptions(options);
-        }
-        return builder.add(percolate(indices));
-    }
-
     private static GetAliasesRequestBuilder getAliases(String... indices) {
         return client().admin().indices().prepareGetAliases("dummy").addIndices(indices);
     }
@@ -776,11 +730,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
                 assertThat(multiSearchResponse.getResponses().length, equalTo(1));
                 assertThat(multiSearchResponse.getResponses()[0].isFailure(), is(true));
                 assertThat(multiSearchResponse.getResponses()[0].getResponse(), nullValue());
-            } else if (requestBuilder instanceof MultiPercolateRequestBuilder) {
-                MultiPercolateResponse multiPercolateResponse = ((MultiPercolateRequestBuilder) requestBuilder).get();
-                assertThat(multiPercolateResponse.getItems().length, equalTo(1));
-                assertThat(multiPercolateResponse.getItems()[0].isFailure(), is(true));
-                assertThat(multiPercolateResponse.getItems()[0].getResponse(), nullValue());
             } else {
                 try {
                     requestBuilder.get();
diff --git a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
index cf28de64b87..f877ae6629b 100644
--- a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
@@ -50,7 +50,7 @@ import java.util.concurrent.TimeUnit;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static org.elasticsearch.Version.CURRENT;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
 import static org.elasticsearch.test.VersionUtils.randomVersion;
 
 /**
diff --git a/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java b/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java
index 0c56234e48d..384e181c899 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java
@@ -143,7 +143,6 @@ public class SearchModuleTests extends ModuleTestCase {
             "multi_match",
             "nested",
             "parent_id",
-            "percolate",
             "prefix",
             "query_string",
             "range",
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
index 60185dbae28..7d21fbaf2a1 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
@@ -40,7 +40,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.EnvironmentModule;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.query.AbstractQueryTestCase;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
@@ -74,8 +74,8 @@ import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
+import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.containsString;
 
 public class AggregatorParsingTests extends ESTestCase {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
index 8df37e25f76..e7933fb1d0c 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
@@ -44,7 +44,7 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.EnvironmentModule;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.query.AbstractQueryTestCase;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
@@ -75,8 +75,8 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
+import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.equalTo;
 
 public abstract class BaseAggregationTestCase<AB extends AggregationBuilder<AB>> extends ESTestCase {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
index c20344f35fc..8fdc3de325d 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
@@ -44,7 +44,7 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.EnvironmentModule;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.query.AbstractQueryTestCase;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
@@ -76,8 +76,8 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
+import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.equalTo;
 
 public abstract class BasePipelineAggregationTestCase<AF extends PipelineAggregatorBuilder> extends ESTestCase {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java
index c79ab04e492..ab42d076a23 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java
@@ -21,7 +21,7 @@ package org.elasticsearch.search.aggregations.metrics;
 
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.AbstractQueryTestCase;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.AggregationInitializationException;
diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index 661643b0e8b..f8236f66e48 100644
--- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -48,7 +48,7 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.EnvironmentModule;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.query.AbstractQueryTestCase;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.index.query.EmptyQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryParseContext;
@@ -97,8 +97,8 @@ import java.util.List;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
+import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
+import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.equalTo;
 
 public class SearchSourceBuilderTests extends ESTestCase {
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearch2xIT.java b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearch2xIT.java
index d09c8f172df..812e43918d9 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearch2xIT.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearch2xIT.java
@@ -29,10 +29,8 @@ import org.elasticsearch.action.admin.indices.segments.IndexShardSegments;
 import org.elasticsearch.action.admin.indices.segments.ShardSegments;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.action.percolate.PercolateResponse;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.Fuzziness;
@@ -65,7 +63,6 @@ import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful;
 import static org.hamcrest.Matchers.containsString;
@@ -114,39 +111,6 @@ public class CompletionSuggestSearch2xIT extends ESIntegTestCase {
         assertSuggestionsNotInOrder("t", "The Prodigy", "Turbonegro", "Turbonegro Get it on", "The Prodigy Firestarter");
     }
 
-    public void testSuggestFieldWithPercolateApi() throws Exception {
-        createIndexAndMapping(completionMappingBuilder);
-        String[][] inputs = {{"Foo Fighters"}, {"Foo Fighters"}, {"Foo Fighters"}, {"Foo Fighters"},
-            {"Generator", "Foo Fighters Generator"}, {"Learn to Fly", "Foo Fighters Learn to Fly"},
-            {"The Prodigy"}, {"The Prodigy"}, {"The Prodigy"}, {"Firestarter", "The Prodigy Firestarter"},
-            {"Turbonegro"}, {"Turbonegro"}, {"Get it on", "Turbonegro Get it on"}}; // work with frequencies
-        for (int i = 0; i < inputs.length; i++) {
-            XContentBuilder source = jsonBuilder()
-                .startObject().startObject(FIELD)
-                .startArray("input");
-            for (String input : inputs[i]) {
-                source.value(input);
-            }
-            source.endArray()
-                .endObject()
-                .endObject();
-            client().prepareIndex(INDEX, TYPE, "" + i)
-                .setSource(source).execute().actionGet();
-        }
-
-        client().prepareIndex(INDEX, ".percolator", "4")
-            .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
-            .execute().actionGet();
-
-        refresh();
-
-        PercolateResponse response = client().preparePercolate().setIndices(INDEX).setDocumentType(TYPE)
-            .setGetRequest(Requests.getRequest(INDEX).type(TYPE).id("1"))
-            .execute().actionGet();
-        assertThat(response.getCount(), equalTo(1L));
-
-    }
-
     public void testBasicPrefixSuggestion() throws Exception {
         completionMappingBuilder.payloads(true);
         createIndexAndMapping(completionMappingBuilder);
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java
index a625fae7b4e..9ef3c898cab 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java
@@ -31,17 +31,14 @@ import org.elasticsearch.action.admin.indices.segments.IndexShardSegments;
 import org.elasticsearch.action.admin.indices.segments.ShardSegments;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.action.percolate.PercolateResponse;
 import org.elasticsearch.action.search.ReduceSearchPhaseException;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.percolator.PercolatorFieldMapper;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
@@ -71,7 +68,6 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.common.util.CollectionUtils.iterableAsArrayList;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
@@ -394,39 +390,6 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
         }
     }
 
-    public void testSuggestFieldWithPercolateApi() throws Exception {
-        createIndexAndMapping(completionMappingBuilder);
-        String[][] inputs = {{"Foo Fighters"}, {"Foo Fighters"}, {"Foo Fighters"}, {"Foo Fighters"},
-                {"Generator", "Foo Fighters Generator"}, {"Learn to Fly", "Foo Fighters Learn to Fly"},
-                {"The Prodigy"}, {"The Prodigy"}, {"The Prodigy"}, {"Firestarter", "The Prodigy Firestarter"},
-                {"Turbonegro"}, {"Turbonegro"}, {"Get it on", "Turbonegro Get it on"}}; // work with frequencies
-        for (int i = 0; i < inputs.length; i++) {
-            XContentBuilder source = jsonBuilder()
-                    .startObject().startObject(FIELD)
-                    .startArray("input");
-            for (String input : inputs[i]) {
-                source.value(input);
-            }
-            source.endArray()
-                    .endObject()
-                    .endObject();
-            client().prepareIndex(INDEX, TYPE, "" + i)
-                    .setSource(source).execute().actionGet();
-        }
-        client().admin().indices().preparePutMapping(INDEX).setType("query").setSource("query", "type=percolator").get();
-
-        client().prepareIndex(INDEX, "query", "4")
-                .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
-                .execute().actionGet();
-
-        refresh();
-
-        PercolateResponse response = client().preparePercolate().setIndices(INDEX).setDocumentType(TYPE)
-                .setGetRequest(Requests.getRequest(INDEX).type(TYPE).id("1"))
-                .execute().actionGet();
-        assertThat(response.getCount(), equalTo(1L));
-    }
-
     public void testThatWeightsAreWorking() throws Exception {
         createIndexAndMapping(completionMappingBuilder);
 
diff --git a/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java b/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
index 5c3f0999ea2..eb13de34858 100644
--- a/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
+++ b/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
@@ -28,7 +28,6 @@ import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.percolate.PercolateResponse;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.ActionFilter;
@@ -217,26 +216,6 @@ public class ContextAndHeaderTransportIT extends ESIntegTestCase {
         assertRequestsContainHeader(MultiTermVectorsRequest.class);
     }
 
-    public void testThatPercolatingExistingDocumentGetRequestContainsContextAndHeaders() throws Exception {
-        Client client = transportClient();
-        client.admin().indices().preparePutMapping(lookupIndex).setType("query").setSource("query", "type=percolator").get();
-        client.prepareIndex(lookupIndex, "query", "1")
-            .setSource(jsonBuilder().startObject()
-                        .startObject("query").startObject("match").field("name", "star wars").endObject().endObject()
-                    .endObject())
-            .get();
-        client.prepareIndex(lookupIndex, "type", "1")
-            .setSource(jsonBuilder().startObject().field("name", "Star Wars - The new republic").endObject())
-            .get();
-        client.admin().indices().prepareRefresh(lookupIndex).get();
-
-        GetRequest getRequest = client.prepareGet(lookupIndex, "type", "1").request();
-        PercolateResponse response = client.preparePercolate().setDocumentType("type").setGetRequest(getRequest).get();
-        assertThat(response.getCount(), is(1L));
-
-        assertGetRequestsContainHeaders();
-    }
-
     public void testThatRelevantHttpHeadersBecomeRequestHeaders() throws Exception {
         String releventHeaderName = "relevant_" + randomHeaderKey;
         for (RestController restController : internalCluster().getDataNodeInstances(RestController.class)) {
diff --git a/docs/java-api/query-dsl/percolate-query.asciidoc b/docs/java-api/query-dsl/percolate-query.asciidoc
index 60e03e9f32e..186d707379e 100644
--- a/docs/java-api/query-dsl/percolate-query.asciidoc
+++ b/docs/java-api/query-dsl/percolate-query.asciidoc
@@ -1,8 +1,25 @@
 [[java-query-percolate-query]]
 ==== Percolate query
 
-See:
- * {ref}/query-dsl-percolate-query.html[Percolate Query]
+See: {ref}/query-dsl-percolate-query.html[Percolate Query]
+
+In order to use the `percolate` query from the Java API your
+the percolator module dependency should be on the classpath and
+the transport client should be loaded with the percolator plugin:
+
+[source,java]
+--------------------------------------------------
+TransportClient transportClient = TransportClient.builder()
+        .settings(Settings.builder().put("node.name", "node"))
+        .addPlugin(PercolatorPlugin.class)
+        .build();
+transportClient.addTransportAddress(
+        new InetSocketTransportAddress(new InetSocketAddress(InetAddresses.forString("127.0.0.1"), 9300))
+);
+--------------------------------------------------
+
+Before the `percolate` query can be used an `percolator` mapping should be added and
+a document containing a percolator query should be indexed:
 
 [source,java]
 --------------------------------------------------
diff --git a/docs/java-api/query-dsl/special-queries.asciidoc b/docs/java-api/query-dsl/special-queries.asciidoc
index 64645685ca5..31db47ce636 100644
--- a/docs/java-api/query-dsl/special-queries.asciidoc
+++ b/docs/java-api/query-dsl/special-queries.asciidoc
@@ -20,6 +20,9 @@ final query to execute.
 This query allows a script to act as a filter.  Also see the
 <<java-query-dsl-function-score-query,`function_score` query>>.
 
+<<java-query-percolate-query,`percolate` query>>::
+
+This query finds percolator queries based on documents.
 
 include::mlt-query.asciidoc[]
 
diff --git a/docs/reference/migration/migrate_5_0/percolator.asciidoc b/docs/reference/migration/migrate_5_0/percolator.asciidoc
index d3b1ccb97a3..ae2057bddfb 100644
--- a/docs/reference/migration/migrate_5_0/percolator.asciidoc
+++ b/docs/reference/migration/migrate_5_0/percolator.asciidoc
@@ -46,4 +46,61 @@ the existing document.
 
 ==== Percolate Stats
 
-The percolate stats have been removed. This is because the percolator no longer caches the percolator queries.
\ No newline at end of file
+The percolate stats have been removed. This is because the percolator no longer caches the percolator queries.
+
+==== Java client
+
+The percolator is no longer part of the core elasticsearch dependency. It has moved to the percolator module.
+Therefor when using the percolator feature from the Java client the new percolator module should also be on the
+classpath. Also the transport client should load the percolator module as plugin:
+
+[source,java]
+--------------------------------------------------
+TransportClient transportClient = TransportClient.builder()
+        .settings(Settings.builder().put("node.name", "node"))
+        .addPlugin(PercolatorPlugin.class)
+        .build();
+transportClient.addTransportAddress(
+        new InetSocketTransportAddress(new InetSocketAddress(InetAddresses.forString("127.0.0.1"), 9300))
+);
+--------------------------------------------------
+
+The percolator and multi percolate related methods from the `Client` interface have been removed. These APIs have been
+deprecated and it is recommended to use the `percolate` query in either the search or multi search APIs. However the
+percolate and multi percolate APIs can still be used from the Java client.
+
+Using percolate request:
+
+[source,java]
+--------------------------------------------------
+PercolateRequest request = new PercolateRequest();
+// set stuff and then execute:
+PercolateResponse response = transportClient.execute(PercolateAction.INSTANCE, request).actionGet();
+--------------------------------------------------
+
+Using percolate request builder:
+
+[source,java]
+--------------------------------------------------
+PercolateRequestBuilder builder = new PercolateRequestBuilder(transportClient, PercolateAction.INSTANCE);
+// set stuff and then execute:
+PercolateResponse response = builder.get();
+--------------------------------------------------
+
+Using multi percolate request:
+
+[source,java]
+--------------------------------------------------
+MultiPercolateRequest request = new MultiPercolateRequest();
+// set stuff and then execute:
+MultiPercolateResponse response = transportClient.execute(MultiPercolateAction.INSTANCE, request).get();
+--------------------------------------------------
+
+Using multi percolate request builder:
+
+[source,java]
+--------------------------------------------------
+MultiPercolateRequestBuilder builder = new MultiPercolateRequestBuilder(transportClient, MultiPercolateAction.INSTANCE);
+// set stuff and then execute:
+MultiPercolateResponse response = builder.get();
+--------------------------------------------------
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorException.java b/modules/percolator/build.gradle
similarity index 58%
rename from core/src/main/java/org/elasticsearch/index/percolator/PercolatorException.java
rename to modules/percolator/build.gradle
index 3813679d81c..4e55fe25521 100644
--- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorException.java
+++ b/modules/percolator/build.gradle
@@ -16,25 +16,11 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.index.percolator;
 
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.index.Index;
-
-import java.io.IOException;
-
-/**
- * Exception during indexing a percolator query.
- */
-public class PercolatorException extends ElasticsearchException {
-
-    public PercolatorException(Index index, String msg, Throwable cause) {
-        super(msg, cause);
-        setIndex(index);
-    }
-
-    public PercolatorException(StreamInput in) throws IOException{
-        super(in);
-    }
+esplugin {
+    description 'Percolator module adds capability to index queries and query these queries by specifying documents'
+    classname 'org.elasticsearch.percolator.PercolatorPlugin'
 }
+
+compileJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes"
+compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes"
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/ExtractQueryTermsService.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java
similarity index 96%
rename from core/src/main/java/org/elasticsearch/index/percolator/ExtractQueryTermsService.java
rename to modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java
index 28082a4bf83..f0c61b74e13 100644
--- a/core/src/main/java/org/elasticsearch/index/percolator/ExtractQueryTermsService.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.index.percolator;
+package org.elasticsearch.percolator;
 
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
@@ -76,7 +76,8 @@ public final class ExtractQueryTermsService {
      *                              For example the query contained an unsupported query (e.g. WildcardQuery).
      * @param fieldType The field type for the query metadata field
      */
-    public static void extractQueryTerms(Query query, ParseContext.Document document, String queryTermsFieldField, String unknownQueryField, FieldType fieldType) {
+    public static void extractQueryTerms(Query query, ParseContext.Document document, String queryTermsFieldField,
+                                         String unknownQueryField, FieldType fieldType) {
         Set<Term> queryTerms;
         try {
             queryTerms = extractQueryTerms(query);
@@ -145,7 +146,8 @@ public final class ExtractQueryTermsService {
                 for (BooleanClause clause : clauses) {
                     if (clause.isRequired() == false) {
                         // skip must_not clauses, we don't need to remember the things that do *not* match...
-                        // skip should clauses, this bq has must clauses, so we don't need to remember should clauses, since they are completely optional.
+                        // skip should clauses, this bq has must clauses, so we don't need to remember should clauses,
+                        // since they are completely optional.
                         continue;
                     }
 
@@ -234,7 +236,8 @@ public final class ExtractQueryTermsService {
     /**
      * Creates a boolean query with a should clause for each term on all fields of the specified index reader.
      */
-    public static Query createQueryTermsQuery(IndexReader indexReader, String queryMetadataField, String unknownQueryField) throws IOException {
+    public static Query createQueryTermsQuery(IndexReader indexReader, String queryMetadataField,
+                                              String unknownQueryField) throws IOException {
         Objects.requireNonNull(queryMetadataField);
         Objects.requireNonNull(unknownQueryField);
 
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/MultiPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateAction.java
similarity index 97%
rename from core/src/main/java/org/elasticsearch/action/percolate/MultiPercolateAction.java
rename to modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateAction.java
index 2b9538ae054..f80e1c45494 100644
--- a/core/src/main/java/org/elasticsearch/action/percolate/MultiPercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateAction.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.action.percolate;
+package org.elasticsearch.percolator;
 
 import org.elasticsearch.action.Action;
 import org.elasticsearch.client.ElasticsearchClient;
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/MultiPercolateRequest.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateRequest.java
similarity index 99%
rename from core/src/main/java/org/elasticsearch/action/percolate/MultiPercolateRequest.java
rename to modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateRequest.java
index ac49fed763a..a18c12bb769 100644
--- a/core/src/main/java/org/elasticsearch/action/percolate/MultiPercolateRequest.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateRequest.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.action.percolate;
+package org.elasticsearch.percolator;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.ActionRequest;
@@ -44,7 +44,10 @@ import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeSt
 
 /**
  * A multi percolate request that encapsulates multiple {@link PercolateRequest} instances in a single api call.
+ *
+ * @deprecated Instead use multi search API with {@link PercolateQueryBuilder}
  */
+@Deprecated
 public class MultiPercolateRequest extends ActionRequest<MultiPercolateRequest> implements CompositeIndicesRequest {
 
     private String[] indices;
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/MultiPercolateRequestBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateRequestBuilder.java
similarity index 94%
rename from core/src/main/java/org/elasticsearch/action/percolate/MultiPercolateRequestBuilder.java
rename to modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateRequestBuilder.java
index df36f9ff7f4..8613b8b07bd 100644
--- a/core/src/main/java/org/elasticsearch/action/percolate/MultiPercolateRequestBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateRequestBuilder.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.action.percolate;
+package org.elasticsearch.percolator;
 
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -24,7 +24,10 @@ import org.elasticsearch.client.ElasticsearchClient;
 
 /**
  * A builder for to ease the use of defining a {@link MultiPercolateRequest} instance.
+ *
+ * @deprecated Instead use multi search API with {@link PercolateQueryBuilder}
  */
+@Deprecated
 public class MultiPercolateRequestBuilder extends ActionRequestBuilder<MultiPercolateRequest, MultiPercolateResponse, MultiPercolateRequestBuilder> {
 
     public MultiPercolateRequestBuilder(ElasticsearchClient client, MultiPercolateAction action) {
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/MultiPercolateResponse.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateResponse.java
similarity index 97%
rename from core/src/main/java/org/elasticsearch/action/percolate/MultiPercolateResponse.java
rename to modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateResponse.java
index 3c626678e74..c05c0097c90 100644
--- a/core/src/main/java/org/elasticsearch/action/percolate/MultiPercolateResponse.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateResponse.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.action.percolate;
+package org.elasticsearch.percolator;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionResponse;
@@ -36,7 +36,10 @@ import java.util.Iterator;
  *
  * Each item represents the response of a percolator request and the order of the items is in the same order as the
  * percolator requests were defined in the multi percolate request.
+ *
+ * @deprecated Instead use multi search API with {@link PercolateQueryBuilder}
  */
+@Deprecated
 public class MultiPercolateResponse extends ActionResponse implements Iterable<MultiPercolateResponse.Item>, ToXContent {
 
     private Item[] items;
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/PercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateAction.java
similarity index 97%
rename from core/src/main/java/org/elasticsearch/action/percolate/PercolateAction.java
rename to modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateAction.java
index 412f4e3ed62..94f5a0f04df 100644
--- a/core/src/main/java/org/elasticsearch/action/percolate/PercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateAction.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.action.percolate;
+package org.elasticsearch.percolator;
 
 import org.elasticsearch.action.Action;
 import org.elasticsearch.client.ElasticsearchClient;
diff --git a/core/src/main/java/org/elasticsearch/index/query/PercolateQuery.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
similarity index 99%
rename from core/src/main/java/org/elasticsearch/index/query/PercolateQuery.java
rename to modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
index 41448e14ef7..2f3108d6298 100644
--- a/core/src/main/java/org/elasticsearch/index/query/PercolateQuery.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.query;
+package org.elasticsearch.percolator;
 
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
@@ -34,7 +34,6 @@ import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.Accountable;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.index.percolator.ExtractQueryTermsService;
 
 import java.io.IOException;
 import java.util.Objects;
diff --git a/core/src/main/java/org/elasticsearch/index/query/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
similarity index 97%
rename from core/src/main/java/org/elasticsearch/index/query/PercolateQueryBuilder.java
rename to modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
index 59be8d7a2d0..627cb09da19 100644
--- a/core/src/main/java/org/elasticsearch/index/query/PercolateQueryBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.query;
+package org.elasticsearch.percolator;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;
@@ -66,14 +66,19 @@ import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
 import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
-import org.elasticsearch.index.percolator.PercolatorFieldMapper;
+import org.elasticsearch.index.query.AbstractQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.QueryShardException;
 
 import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 
 import static org.elasticsearch.index.mapper.SourceToParse.source;
-import static org.elasticsearch.index.percolator.PercolatorFieldMapper.parseQuery;
+import static org.elasticsearch.percolator.PercolatorFieldMapper.parseQuery;
 
 public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBuilder> {
 
@@ -154,7 +159,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
     /**
      * Read from a stream.
      */
-    public PercolateQueryBuilder(StreamInput in) throws IOException {
+    PercolateQueryBuilder(StreamInput in) throws IOException {
         super(in);
         field = in.readString();
         documentType = in.readString();
@@ -420,7 +425,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
             builder.extractQueryTermsQuery(pft.getExtractedTermsField(), pft.getUnknownQueryFieldName());
             return builder.build();
         } else {
-            Query percolateTypeQuery = new TermQuery(new Term(TypeFieldMapper.NAME, PercolatorFieldMapper.LEGACY_TYPE_NAME));
+            Query percolateTypeQuery = new TermQuery(new Term(TypeFieldMapper.NAME, MapperService.PERCOLATOR_LEGACY_TYPE_NAME));
             PercolateQuery.Builder builder = new PercolateQuery.Builder(
                     documentType, createLegacyStore(context, mapUnmappedFieldsAsString), document, docSearcher
             );
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequest.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequest.java
similarity index 97%
rename from core/src/main/java/org/elasticsearch/action/percolate/PercolateRequest.java
rename to modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequest.java
index c9887cba03f..4509ab3cb50 100644
--- a/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequest.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequest.java
@@ -16,16 +16,14 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.action.percolate;
+package org.elasticsearch.percolator;
 
 import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.CompositeIndicesRequest;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.action.support.broadcast.BroadcastRequest;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -36,15 +34,16 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
 /**
  * A request to execute a percolate operation.
+ *
+ * @deprecated Instead use search API with {@link PercolateQueryBuilder}
  */
+@Deprecated
 public class PercolateRequest extends ActionRequest<PercolateRequest> implements IndicesRequest.Replaceable {
 
     protected String[] indices;
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequestBuilder.java
similarity index 98%
rename from core/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java
rename to modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequestBuilder.java
index 9490abd0b68..7d42382e76e 100644
--- a/core/src/main/java/org/elasticsearch/action/percolate/PercolateRequestBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequestBuilder.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.action.percolate;
+package org.elasticsearch.percolator;
 
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.get.GetRequest;
@@ -36,7 +36,10 @@ import java.util.Map;
 
 /**
  * A builder the easy to use of defining a percolate request.
+ *
+ * @deprecated Instead use search API with {@link PercolateQueryBuilder}
  */
+@Deprecated
 public class PercolateRequestBuilder extends ActionRequestBuilder<PercolateRequest, PercolateResponse, PercolateRequestBuilder> {
 
     private PercolateSourceBuilder sourceBuilder;
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/PercolateResponse.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateResponse.java
similarity index 98%
rename from core/src/main/java/org/elasticsearch/action/percolate/PercolateResponse.java
rename to modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateResponse.java
index 110ed37342f..622c27fca92 100644
--- a/core/src/main/java/org/elasticsearch/action/percolate/PercolateResponse.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateResponse.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.action.percolate;
+package org.elasticsearch.percolator;
 
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
@@ -41,7 +41,10 @@ import java.util.Map;
 
 /**
  * Encapsulates the response of a percolator request.
+ *
+ * @deprecated Instead use search API with {@link PercolateQueryBuilder}
  */
+@Deprecated
 public class PercolateResponse extends BroadcastResponse implements Iterable<PercolateResponse.Match>, ToXContent {
 
     public static final Match[] EMPTY = new Match[0];
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateSourceBuilder.java
similarity index 98%
rename from core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java
rename to modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateSourceBuilder.java
index 5c69d3be50b..88e709447f7 100644
--- a/core/src/main/java/org/elasticsearch/action/percolate/PercolateSourceBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateSourceBuilder.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.action.percolate;
+package org.elasticsearch.percolator;
 
 import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.action.support.ToXContentToBytes;
@@ -44,7 +44,10 @@ import java.util.Map;
 
 /**
  * Builder to create the percolate request body.
+ *
+ * @deprecated Instead use search API with {@link PercolateQueryBuilder}
  */
+@Deprecated
 public class PercolateSourceBuilder extends ToXContentToBytes {
 
     private DocBuilder docBuilder;
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
similarity index 87%
rename from core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
rename to modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
index decb91e960a..bf5c4c0ae8b 100644
--- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorFieldMapper.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.index.percolator;
+package org.elasticsearch.percolator;
 
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.DocValuesType;
@@ -38,7 +38,6 @@ import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.core.BinaryFieldMapper;
 import org.elasticsearch.index.mapper.core.KeywordFieldMapper;
-import org.elasticsearch.index.query.PercolateQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
@@ -56,8 +55,6 @@ public class PercolatorFieldMapper extends FieldMapper {
     public final static XContentType QUERY_BUILDER_CONTENT_TYPE = XContentType.SMILE;
     public final static Setting<Boolean> INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING =
             Setting.boolSetting("index.percolator.map_unmapped_fields_as_string", false, Setting.Property.IndexScope);
-    @Deprecated
-    public static final String LEGACY_TYPE_NAME = ".percolator";
     public static final String CONTENT_TYPE = "percolator";
     private static final PercolatorFieldType FIELD_TYPE = new PercolatorFieldType();
 
@@ -181,20 +178,34 @@ public class PercolatorFieldMapper extends FieldMapper {
         this.mapUnmappedFieldAsString = INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING.get(indexSettings);
     }
 
+    @Override
+    public FieldMapper updateFieldType(Map<String, MappedFieldType> fullNameToFieldType) {
+        PercolatorFieldMapper updated = (PercolatorFieldMapper) super.updateFieldType(fullNameToFieldType);
+        KeywordFieldMapper queryTermsUpdated = (KeywordFieldMapper) queryTermsField.updateFieldType(fullNameToFieldType);
+        KeywordFieldMapper unknownQueryUpdated = (KeywordFieldMapper) unknownQueryField.updateFieldType(fullNameToFieldType);
+        BinaryFieldMapper queryBuilderUpdated = (BinaryFieldMapper) queryBuilderField.updateFieldType(fullNameToFieldType);
+
+        if (updated == this || queryTermsUpdated == queryTermsField || unknownQueryUpdated == unknownQueryField
+                || queryBuilderUpdated == queryBuilderField) {
+            return this;
+        }
+        if (updated == this) {
+            updated = (PercolatorFieldMapper) updated.clone();
+        }
+        updated.queryTermsField = queryTermsUpdated;
+        updated.unknownQueryField = unknownQueryUpdated;
+        updated.queryBuilderField = queryBuilderUpdated;
+        return updated;
+    }
+
     @Override
     public Mapper parse(ParseContext context) throws IOException {
         QueryShardContext queryShardContext = new QueryShardContext(this.queryShardContext);
-        DocumentMapper documentMapper = queryShardContext.getMapperService().documentMapper(context.sourceToParse().type());
-        for (FieldMapper fieldMapper : documentMapper.mappers()) {
-            if (fieldMapper instanceof PercolatorFieldMapper) {
-                PercolatorFieldType fieldType = (PercolatorFieldType) fieldMapper.fieldType();
-                if (context.doc().getField(fieldType.getQueryBuilderFieldName()) != null) {
-                    // If a percolator query has been defined in an array object then multiple percolator queries
-                    // could be provided. In order to prevent this we fail if we try to parse more than one query
-                    // for the current document.
-                    throw new IllegalArgumentException("a document can only contain one percolator query");
-                }
-            }
+        if (context.doc().getField(queryBuilderField.name()) != null) {
+            // If a percolator query has been defined in an array object then multiple percolator queries
+            // could be provided. In order to prevent this we fail if we try to parse more than one query
+            // for the current document.
+            throw new IllegalArgumentException("a document can only contain one percolator query");
         }
 
         XContentParser parser = context.parser();
@@ -210,7 +221,8 @@ public class PercolatorFieldMapper extends FieldMapper {
         }
 
         Query query = toQuery(queryShardContext, mapUnmappedFieldAsString, queryBuilder);
-        ExtractQueryTermsService.extractQueryTerms(query, context.doc(), queryTermsField.name(), unknownQueryField.name(), queryTermsField.fieldType());
+        ExtractQueryTermsService.extractQueryTerms(query, context.doc(), queryTermsField.name(), unknownQueryField.name(),
+                queryTermsField.fieldType());
         return null;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorHighlightSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
similarity index 97%
rename from core/src/main/java/org/elasticsearch/index/percolator/PercolatorHighlightSubFetchPhase.java
rename to modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
index 2c231593860..41a43d39328 100644
--- a/core/src/main/java/org/elasticsearch/index/percolator/PercolatorHighlightSubFetchPhase.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.percolator;
+package org.elasticsearch.percolator;
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.ReaderUtil;
@@ -31,7 +31,6 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.index.query.ParsedQuery;
-import org.elasticsearch.index.query.PercolateQuery;
 import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.highlight.HighlightPhase;
@@ -46,7 +45,7 @@ import java.util.List;
 import java.util.Map;
 
 // Highlighting in the case of the percolate query is a bit different, because the PercolateQuery itself doesn't get highlighted,
-// but the source of the PercolateQuery gets highlighted by each hit with type '.percolator' (percolator queries).
+// but the source of the PercolateQuery gets highlighted by each hit containing a query.
 public class PercolatorHighlightSubFetchPhase implements FetchSubPhase {
 
     private final HighlightPhase highlightPhase;
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
new file mode 100644
index 00000000000..963f14041b0
--- /dev/null
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.percolator;
+
+import org.elasticsearch.action.ActionModule;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsModule;
+import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.SearchModule;
+
+public class PercolatorPlugin extends Plugin {
+
+    public static final String NAME = "percolator";
+
+    private final boolean transportClientMode;
+
+    public PercolatorPlugin(Settings settings) {
+        this.transportClientMode = transportClientMode(settings);
+    }
+
+    @Override
+    public String name() {
+        return NAME;
+    }
+
+    @Override
+    public String description() {
+        return "Percolator module adds capability to index queries and query these queries by specifying documents";
+    }
+
+    public void onModule(ActionModule module) {
+        module.registerAction(PercolateAction.INSTANCE, TransportPercolateAction.class);
+        module.registerAction(MultiPercolateAction.INSTANCE, TransportMultiPercolateAction.class);
+    }
+
+    public void onModule(NetworkModule module) {
+        if (transportClientMode == false) {
+            module.registerRestHandler(RestPercolateAction.class);
+            module.registerRestHandler(RestMultiPercolateAction.class);
+        }
+    }
+
+    public void onModule(IndicesModule module) {
+        module.registerMapper(PercolatorFieldMapper.CONTENT_TYPE, new PercolatorFieldMapper.TypeParser());
+    }
+
+    public void onModule(SearchModule module) {
+        module.registerQuery(PercolateQueryBuilder::new, PercolateQueryBuilder::fromXContent, PercolateQueryBuilder.QUERY_NAME_FIELD);
+        module.registerFetchSubPhase(PercolatorHighlightSubFetchPhase.class);
+    }
+
+    public void onModule(SettingsModule module) {
+        module.registerSetting(PercolatorFieldMapper.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING);
+    }
+
+    static boolean transportClientMode(Settings settings) {
+        return TransportClient.CLIENT_TYPE.equals(settings.get(Client.CLIENT_TYPE_SETTING_S.getKey()));
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/rest/action/percolate/RestMultiPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java
similarity index 89%
rename from core/src/main/java/org/elasticsearch/rest/action/percolate/RestMultiPercolateAction.java
rename to modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java
index 6a12ff1438b..a2902a9a7c2 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/percolate/RestMultiPercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java
@@ -16,10 +16,8 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.rest.action.percolate;
+package org.elasticsearch.percolator;
 
-import org.elasticsearch.action.percolate.MultiPercolateRequest;
-import org.elasticsearch.action.percolate.MultiPercolateResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.Strings;
@@ -35,16 +33,16 @@ import org.elasticsearch.rest.action.support.RestToXContentListener;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
-/**
- *
- */
 public class RestMultiPercolateAction extends BaseRestHandler {
 
     private final boolean allowExplicitIndex;
+    private final TransportMultiPercolateAction action;
 
     @Inject
-    public RestMultiPercolateAction(Settings settings, RestController controller, Client client) {
+    public RestMultiPercolateAction(Settings settings, RestController controller, Client client,
+                                    TransportMultiPercolateAction action) {
         super(settings, client);
+        this.action = action;
         controller.registerHandler(POST, "/_mpercolate", this);
         controller.registerHandler(POST, "/{index}/_mpercolate", this);
         controller.registerHandler(POST, "/{index}/{type}/_mpercolate", this);
@@ -63,8 +61,7 @@ public class RestMultiPercolateAction extends BaseRestHandler {
         multiPercolateRequest.indices(Strings.splitStringByCommaToArray(restRequest.param("index")));
         multiPercolateRequest.documentType(restRequest.param("type"));
         multiPercolateRequest.add(RestActions.getRestContent(restRequest), allowExplicitIndex);
-
-        client.multiPercolate(multiPercolateRequest, new RestToXContentListener<MultiPercolateResponse>(restChannel));
+        action.execute(multiPercolateRequest, new RestToXContentListener<MultiPercolateResponse>(restChannel));
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/percolate/RestPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java
similarity index 93%
rename from core/src/main/java/org/elasticsearch/rest/action/percolate/RestPercolateAction.java
rename to modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java
index 91ecc9dbd6d..b752cc55f6c 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/percolate/RestPercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java
@@ -16,11 +16,9 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.rest.action.percolate;
+package org.elasticsearch.percolator;
 
 import org.elasticsearch.action.get.GetRequest;
-import org.elasticsearch.action.percolate.PercolateRequest;
-import org.elasticsearch.action.percolate.PercolateResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.Strings;
@@ -37,14 +35,14 @@ import org.elasticsearch.rest.action.support.RestToXContentListener;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
-/**
- *
- */
 public class RestPercolateAction extends BaseRestHandler {
 
+    private final TransportPercolateAction action;
+
     @Inject
-    public RestPercolateAction(Settings settings, RestController controller, Client client) {
+    public RestPercolateAction(Settings settings, RestController controller, Client client, TransportPercolateAction action) {
         super(settings, client);
+        this.action = action;
         controller.registerHandler(GET, "/{index}/{type}/_percolate", this);
         controller.registerHandler(POST, "/{index}/{type}/_percolate", this);
 
@@ -69,7 +67,7 @@ public class RestPercolateAction extends BaseRestHandler {
         percolateRequest.source(RestActions.getRestContent(restRequest));
 
         percolateRequest.indicesOptions(IndicesOptions.fromRequest(restRequest, percolateRequest.indicesOptions()));
-        executePercolate(percolateRequest, restChannel, client);
+        executePercolate(percolateRequest, restChannel);
     }
 
     void parseExistingDocPercolate(PercolateRequest percolateRequest, RestRequest restRequest, RestChannel restChannel, final Client client) {
@@ -93,11 +91,11 @@ public class RestPercolateAction extends BaseRestHandler {
         percolateRequest.source(RestActions.getRestContent(restRequest));
 
         percolateRequest.indicesOptions(IndicesOptions.fromRequest(restRequest, percolateRequest.indicesOptions()));
-        executePercolate(percolateRequest, restChannel, client);
+        executePercolate(percolateRequest, restChannel);
     }
 
-    void executePercolate(final PercolateRequest percolateRequest, final RestChannel restChannel, final Client client) {
-        client.percolate(percolateRequest, new RestToXContentListener<PercolateResponse>(restChannel));
+    void executePercolate(final PercolateRequest percolateRequest, final RestChannel restChannel) {
+        action.execute(percolateRequest, new RestToXContentListener<>(restChannel));
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/TransportMultiPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java
similarity index 99%
rename from core/src/main/java/org/elasticsearch/action/percolate/TransportMultiPercolateAction.java
rename to modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java
index bf0d79d884e..2b9f77ab82a 100644
--- a/core/src/main/java/org/elasticsearch/action/percolate/TransportMultiPercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.action.percolate;
+package org.elasticsearch.percolator;
 
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
diff --git a/core/src/main/java/org/elasticsearch/action/percolate/TransportPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java
similarity index 99%
rename from core/src/main/java/org/elasticsearch/action/percolate/TransportPercolateAction.java
rename to modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java
index b80589df936..f3ee9230eaa 100644
--- a/core/src/main/java/org/elasticsearch/action/percolate/TransportPercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.action.percolate;
+package org.elasticsearch.percolator;
 
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.ActionListener;
@@ -42,7 +42,6 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
-import org.elasticsearch.index.query.PercolateQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryParseContext;
diff --git a/core/src/test/java/org/elasticsearch/index/percolator/ExtractQueryTermsServiceTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java
similarity index 99%
rename from core/src/test/java/org/elasticsearch/index/percolator/ExtractQueryTermsServiceTests.java
rename to modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java
index 45a1386d912..4245c99ba18 100644
--- a/core/src/test/java/org/elasticsearch/index/percolator/ExtractQueryTermsServiceTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.index.percolator;
+package org.elasticsearch.percolator;
 
 import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
 import org.apache.lucene.document.FieldType;
diff --git a/core/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java
similarity index 83%
rename from core/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java
rename to modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java
index 6ec9ab24960..efa6ecee59a 100644
--- a/core/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java
@@ -19,20 +19,20 @@
 package org.elasticsearch.percolator;
 
 import org.apache.lucene.search.join.ScoreMode;
-import org.elasticsearch.action.percolate.MultiPercolateRequestBuilder;
-import org.elasticsearch.action.percolate.MultiPercolateResponse;
-import org.elasticsearch.action.percolate.PercolateSourceBuilder;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.query.Operator;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
 
-import static org.elasticsearch.action.percolate.PercolateSourceBuilder.docBuilder;
+import static org.elasticsearch.percolator.PercolateSourceBuilder.docBuilder;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.common.xcontent.XContentFactory.smileBuilder;
 import static org.elasticsearch.common.xcontent.XContentFactory.yamlBuilder;
@@ -42,7 +42,9 @@ import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
 import static org.elasticsearch.percolator.PercolatorTestUtil.convertFromTextArray;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertMatchCount;
+import static org.elasticsearch.percolator.PercolatorTestUtil.assertMatchCount;
+import static org.elasticsearch.percolator.PercolatorTestUtil.preparePercolate;
+import static org.elasticsearch.percolator.PercolatorTestUtil.prepareMultiPercolate;
 import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
 import static org.hamcrest.Matchers.arrayWithSize;
@@ -58,6 +60,16 @@ public class MultiPercolatorIT extends ESIntegTestCase {
     private final static String INDEX_NAME = "queries";
     private final static String TYPE_NAME = "query";
 
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singleton(PercolatorPlugin.class);
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
+        return Collections.singleton(PercolatorPlugin.class);
+    }
+
     public void testBasics() throws Exception {
         assertAcked(prepareCreate(INDEX_NAME)
                 .addMapping(TYPE_NAME, "query", "type=percolator")
@@ -82,20 +94,20 @@ public class MultiPercolatorIT extends ESIntegTestCase {
                 .execute().actionGet();
         refresh();
 
-        MultiPercolateResponse response = client().prepareMultiPercolate()
-                .add(client().preparePercolate()
+        MultiPercolateResponse response = prepareMultiPercolate(client())
+                .add(preparePercolate(client())
                         .setIndices(INDEX_NAME).setDocumentType("type")
                         .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "b").endObject())))
-                .add(client().preparePercolate()
+                .add(preparePercolate(client())
                         .setIndices(INDEX_NAME).setDocumentType("type")
                         .setPercolateDoc(docBuilder().setDoc(yamlBuilder().startObject().field("field1", "c").endObject())))
-                .add(client().preparePercolate()
+                .add(preparePercolate(client())
                         .setIndices(INDEX_NAME).setDocumentType("type")
                         .setPercolateDoc(docBuilder().setDoc(smileBuilder().startObject().field("field1", "b c").endObject())))
-                .add(client().preparePercolate()
+                .add(preparePercolate(client())
                         .setIndices(INDEX_NAME).setDocumentType("type")
                         .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "d").endObject())))
-                .add(client().preparePercolate() // non existing doc, so error element
+                .add(preparePercolate(client()) // non existing doc, so error element
                         .setIndices(INDEX_NAME).setDocumentType("type")
                         .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("5")))
                 .execute().actionGet();
@@ -158,24 +170,24 @@ public class MultiPercolatorIT extends ESIntegTestCase {
                 .execute().actionGet();
         refresh();
 
-        MultiPercolateResponse response = client().prepareMultiPercolate()
-                .add(client().preparePercolate()
+        MultiPercolateResponse response = prepareMultiPercolate(client())
+                .add(preparePercolate(client())
                         .setIndices(INDEX_NAME).setDocumentType("type")
                         .setRouting("a")
                         .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "b").endObject())))
-                .add(client().preparePercolate()
+                .add(preparePercolate(client())
                         .setIndices(INDEX_NAME).setDocumentType("type")
                         .setRouting("a")
                         .setPercolateDoc(docBuilder().setDoc(yamlBuilder().startObject().field("field1", "c").endObject())))
-                .add(client().preparePercolate()
+                .add(preparePercolate(client())
                         .setIndices(INDEX_NAME).setDocumentType("type")
                         .setRouting("a")
                         .setPercolateDoc(docBuilder().setDoc(smileBuilder().startObject().field("field1", "b c").endObject())))
-                .add(client().preparePercolate()
+                .add(preparePercolate(client())
                         .setIndices(INDEX_NAME).setDocumentType("type")
                         .setRouting("a")
                         .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "d").endObject())))
-                .add(client().preparePercolate() // non existing doc, so error element
+                .add(preparePercolate(client()) // non existing doc, so error element
                         .setIndices(INDEX_NAME).setDocumentType("type")
                         .setRouting("a")
                         .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("5")))
@@ -227,11 +239,11 @@ public class MultiPercolatorIT extends ESIntegTestCase {
                 .execute().actionGet();
         refresh();
 
-        MultiPercolateRequestBuilder builder = client().prepareMultiPercolate();
+        MultiPercolateRequestBuilder builder = prepareMultiPercolate(client());
         int numPercolateRequest = randomIntBetween(50, 100);
         for (int i = 0; i < numPercolateRequest; i++) {
             builder.add(
-                    client().preparePercolate()
+                    preparePercolate(client())
                             .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("1"))
                             .setIndices(INDEX_NAME).setDocumentType("type")
                             .setSize(numQueries)
@@ -247,10 +259,10 @@ public class MultiPercolatorIT extends ESIntegTestCase {
         }
 
         // Non existing doc
-        builder = client().prepareMultiPercolate();
+        builder = prepareMultiPercolate(client());
         for (int i = 0; i < numPercolateRequest; i++) {
             builder.add(
-                    client().preparePercolate()
+                    preparePercolate(client())
                             .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("2"))
                             .setIndices(INDEX_NAME).setDocumentType("type").setSize(numQueries)
 
@@ -266,16 +278,16 @@ public class MultiPercolatorIT extends ESIntegTestCase {
         }
 
         // One existing doc
-        builder = client().prepareMultiPercolate();
+        builder = prepareMultiPercolate(client());
         for (int i = 0; i < numPercolateRequest; i++) {
             builder.add(
-                    client().preparePercolate()
+                    preparePercolate(client())
                             .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("2"))
                             .setIndices(INDEX_NAME).setDocumentType("type").setSize(numQueries)
             );
         }
         builder.add(
-                client().preparePercolate()
+                preparePercolate(client())
                         .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("1"))
                         .setIndices(INDEX_NAME).setDocumentType("type").setSize(numQueries)
         );
@@ -300,11 +312,11 @@ public class MultiPercolatorIT extends ESIntegTestCase {
         }
         refresh();
 
-        MultiPercolateRequestBuilder builder = client().prepareMultiPercolate();
+        MultiPercolateRequestBuilder builder = prepareMultiPercolate(client());
         int numPercolateRequest = randomIntBetween(50, 100);
         for (int i = 0; i < numPercolateRequest; i++) {
             builder.add(
-                    client().preparePercolate()
+                    preparePercolate(client())
                             .setIndices(INDEX_NAME).setDocumentType("type")
                             .setSize(numQueries)
                             .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field", "a").endObject())));
@@ -319,10 +331,10 @@ public class MultiPercolatorIT extends ESIntegTestCase {
         }
 
         // All illegal json
-        builder = client().prepareMultiPercolate();
+        builder = prepareMultiPercolate(client());
         for (int i = 0; i < numPercolateRequest; i++) {
             builder.add(
-                    client().preparePercolate()
+                    preparePercolate(client())
                             .setIndices(INDEX_NAME).setDocumentType("type")
                             .setSource("illegal json"));
         }
@@ -335,15 +347,15 @@ public class MultiPercolatorIT extends ESIntegTestCase {
         }
 
         // one valid request
-        builder = client().prepareMultiPercolate();
+        builder = prepareMultiPercolate(client());
         for (int i = 0; i < numPercolateRequest; i++) {
             builder.add(
-                    client().preparePercolate()
+                    preparePercolate(client())
                             .setIndices(INDEX_NAME).setDocumentType("type")
                             .setSource("illegal json"));
         }
         builder.add(
-                client().preparePercolate()
+                preparePercolate(client())
                         .setSize(numQueries)
                         .setIndices(INDEX_NAME).setDocumentType("type")
                         .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field", "a").endObject())));
@@ -357,43 +369,15 @@ public class MultiPercolatorIT extends ESIntegTestCase {
 
     public void testNestedMultiPercolation() throws IOException {
         initNestedIndexAndPercolation();
-        MultiPercolateRequestBuilder mpercolate= client().prepareMultiPercolate();
-        mpercolate.add(client().preparePercolate().setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc(getNotMatchingNestedDoc())).setIndices(INDEX_NAME).setDocumentType("company"));
-        mpercolate.add(client().preparePercolate().setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc(getMatchingNestedDoc())).setIndices(INDEX_NAME).setDocumentType("company"));
+        MultiPercolateRequestBuilder mpercolate= prepareMultiPercolate(client());
+        mpercolate.add(preparePercolate(client()).setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc(getNotMatchingNestedDoc())).setIndices(INDEX_NAME).setDocumentType("company"));
+        mpercolate.add(preparePercolate(client()).setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc(getMatchingNestedDoc())).setIndices(INDEX_NAME).setDocumentType("company"));
         MultiPercolateResponse response = mpercolate.get();
         assertEquals(response.getItems()[0].getResponse().getMatches().length, 0);
         assertEquals(response.getItems()[1].getResponse().getMatches().length, 1);
         assertEquals(response.getItems()[1].getResponse().getMatches()[0].getId().string(), "Q");
     }
 
-    public void testStartTimeIsPropagatedToShardRequests() throws Exception {
-        // See: https://github.com/elastic/elasticsearch/issues/15908
-        internalCluster().ensureAtLeastNumDataNodes(2);
-        client().admin().indices().prepareCreate(INDEX_NAME)
-            .setSettings(Settings.builder()
-                .put("index.number_of_shards", 1)
-                .put("index.number_of_replicas", 1)
-            )
-            .addMapping("type", "date_field", "type=date,format=strict_date_optional_time||epoch_millis")
-            .addMapping(TYPE_NAME, "query", "type=percolator")
-            .get();
-        ensureGreen();
-
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "1")
-            .setSource(jsonBuilder().startObject().field("query", rangeQuery("date_field").lt("now+90d")).endObject())
-            .setRefresh(true)
-            .get();
-
-        for (int i = 0; i < 32; i++) {
-            MultiPercolateResponse response = client().prepareMultiPercolate()
-                .add(client().preparePercolate().setDocumentType("type").setIndices(INDEX_NAME)
-                    .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("date_field", "2015-07-21T10:28:01-07:00")))
-                .get();
-            assertThat(response.getItems()[0].getResponse().getCount(), equalTo(1L));
-            assertThat(response.getItems()[0].getResponse().getMatches()[0].getId().string(), equalTo("1"));
-        }
-    }
-
     void initNestedIndexAndPercolation() throws IOException {
         XContentBuilder mapping = XContentFactory.jsonBuilder();
         mapping.startObject().startObject("properties").startObject("companyname").field("type", "text").endObject()
diff --git a/core/src/test/java/org/elasticsearch/action/percolate/MultiPercolatorRequestTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorRequestTests.java
similarity index 98%
rename from core/src/test/java/org/elasticsearch/action/percolate/MultiPercolatorRequestTests.java
rename to modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorRequestTests.java
index 16251463d57..6f64dc63189 100644
--- a/core/src/test/java/org/elasticsearch/action/percolate/MultiPercolatorRequestTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorRequestTests.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.action.percolate;
+package org.elasticsearch.percolator;
 
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.collect.MapBuilder;
@@ -30,11 +30,9 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
-/**
- */
 public class MultiPercolatorRequestTests extends ESTestCase {
     public void testParseBulkRequests() throws Exception {
-        byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/percolate/mpercolate1.json");
+        byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/percolator/mpercolate1.json");
         MultiPercolateRequest request = new MultiPercolateRequest().add(data, 0, data.length);
 
         assertThat(request.requests().size(), equalTo(8));
@@ -150,7 +148,7 @@ public class MultiPercolatorRequestTests extends ESTestCase {
     }
 
     public void testParseBulkRequestsDefaults() throws Exception {
-        byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/percolate/mpercolate2.json");
+        byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/percolator/mpercolate2.json");
         MultiPercolateRequest request = new MultiPercolateRequest();
         request.indices("my-index1").documentType("my-type1").indicesOptions(IndicesOptions.lenientExpandOpen());
         request.add(data, 0, data.length);
diff --git a/core/src/test/java/org/elasticsearch/index/query/PercolateQueryBuilderTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
similarity index 88%
rename from core/src/test/java/org/elasticsearch/index/query/PercolateQueryBuilderTests.java
rename to modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
index 92a9369aed5..c090ebf6dbf 100644
--- a/core/src/test/java/org/elasticsearch/index/query/PercolateQueryBuilderTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.query;
+package org.elasticsearch.percolator;
 
 import com.fasterxml.jackson.core.JsonParseException;
 import org.apache.lucene.search.Query;
@@ -34,12 +34,17 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.AbstractQueryTestCase;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.script.Script;
 import org.hamcrest.Matchers;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.Collections;
 
 import static org.hamcrest.Matchers.containsString;
@@ -62,11 +67,15 @@ public class PercolateQueryBuilderTests extends AbstractQueryTestCase<PercolateQ
 
     boolean indexedDocumentExists = true;
 
-    @BeforeClass
-    public static void before() throws Exception {
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return Collections.singleton(PercolatorPlugin.class);
+    }
+
+    @Override
+    protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
         queryField = randomAsciiOfLength(4);
         docType = randomAsciiOfLength(4);
-        MapperService mapperService = createShardContext().getMapperService();
         mapperService.merge("query_type", new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef("query_type",
                 queryField, "type=percolator"
         ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
@@ -182,38 +191,38 @@ public class PercolateQueryBuilderTests extends AbstractQueryTestCase<PercolateQ
 
     public void testRequiredParameters() {
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
-            QueryBuilders.percolateQuery(null, null, new BytesArray("{}"));
+            new PercolateQueryBuilder(null, null, new BytesArray("{}"));
         });
         assertThat(e.getMessage(), equalTo("[field] is a required argument"));
 
-        e = expectThrows(IllegalArgumentException.class, () -> QueryBuilders.percolateQuery("_field", null, new BytesArray("{}")));
+        e = expectThrows(IllegalArgumentException.class, () -> new PercolateQueryBuilder("_field", null, new BytesArray("{}")));
         assertThat(e.getMessage(), equalTo("[document_type] is a required argument"));
 
-        e = expectThrows(IllegalArgumentException.class, () -> QueryBuilders.percolateQuery("_field", "_document_type", null));
+        e = expectThrows(IllegalArgumentException.class, () -> new PercolateQueryBuilder("_field", "_document_type", null));
         assertThat(e.getMessage(), equalTo("[document] is a required argument"));
 
         e = expectThrows(IllegalArgumentException.class, () -> {
-            QueryBuilders.percolateQuery(null, null, "_index", "_type", "_id", null, null, null);
+            new PercolateQueryBuilder(null, null, "_index", "_type", "_id", null, null, null);
         });
         assertThat(e.getMessage(), equalTo("[field] is a required argument"));
 
         e = expectThrows(IllegalArgumentException.class, () -> {
-            QueryBuilders.percolateQuery("_field", null, "_index", "_type", "_id", null, null, null);
+            new PercolateQueryBuilder("_field", null, "_index", "_type", "_id", null, null, null);
         });
         assertThat(e.getMessage(), equalTo("[document_type] is a required argument"));
 
         e = expectThrows(IllegalArgumentException.class, () -> {
-            QueryBuilders.percolateQuery("_field", "_document_type", null, "_type", "_id", null, null, null);
+            new PercolateQueryBuilder("_field", "_document_type", null, "_type", "_id", null, null, null);
         });
         assertThat(e.getMessage(), equalTo("[index] is a required argument"));
 
         e = expectThrows(IllegalArgumentException.class, () -> {
-            QueryBuilders.percolateQuery("_field", "_document_type", "_index", null, "_id", null, null, null);
+            new PercolateQueryBuilder("_field", "_document_type", "_index", null, "_id", null, null, null);
         });
         assertThat(e.getMessage(), equalTo("[type] is a required argument"));
 
         e = expectThrows(IllegalArgumentException.class, () -> {
-            QueryBuilders.percolateQuery("_field", "_document_type", "_index", "_type", null, null, null, null);
+            new PercolateQueryBuilder("_field", "_document_type", "_index", "_type", null, null, null, null);
         });
         assertThat(e.getMessage(), equalTo("[id] is a required argument"));
     }
diff --git a/core/src/test/java/org/elasticsearch/index/query/PercolateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java
similarity index 98%
rename from core/src/test/java/org/elasticsearch/index/query/PercolateQueryTests.java
rename to modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java
index 8dde535e3cc..07959db1ff1 100644
--- a/core/src/test/java/org/elasticsearch/index/query/PercolateQueryTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.query;
+package org.elasticsearch.percolator;
 
 import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
 import org.apache.lucene.document.Field;
@@ -52,11 +52,10 @@ import org.apache.lucene.search.spans.SpanOrQuery;
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.mapper.internal.UidFieldMapper;
-import org.elasticsearch.index.percolator.ExtractQueryTermsService;
-import org.elasticsearch.index.percolator.PercolatorFieldMapper;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.After;
 import org.junit.Before;
@@ -324,7 +323,7 @@ public class PercolateQueryTests extends ESTestCase {
         ParseContext.Document document = new ParseContext.Document();
         ExtractQueryTermsService.extractQueryTerms(query, document, EXTRACTED_TERMS_FIELD_NAME, UNKNOWN_QUERY_FIELD_NAME,
                 EXTRACTED_TERMS_FIELD_TYPE);
-        document.add(new StoredField(UidFieldMapper.NAME, Uid.createUid(PercolatorFieldMapper.LEGACY_TYPE_NAME, id)));
+        document.add(new StoredField(UidFieldMapper.NAME, Uid.createUid(MapperService.PERCOLATOR_LEGACY_TYPE_NAME, id)));
         assert extraFields.length % 2 == 0;
         for (int i = 0; i < extraFields.length; i++) {
             document.add(new StringField(extraFields[i], extraFields[++i], Field.Store.NO));
diff --git a/core/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java
similarity index 92%
rename from core/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java
rename to modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java
index 167110f0333..c8f79d3b977 100644
--- a/core/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java
@@ -18,11 +18,9 @@
  */
 package org.elasticsearch.percolator;
 
-import org.elasticsearch.action.percolate.PercolateRequestBuilder;
-import org.elasticsearch.action.percolate.PercolateResponse;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.percolator.PercolatorFieldMapper;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.Aggregations;
@@ -34,14 +32,17 @@ import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.InternalBuck
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 
-import static org.elasticsearch.action.percolate.PercolateSourceBuilder.docBuilder;
+import static org.elasticsearch.percolator.PercolateSourceBuilder.docBuilder;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertMatchCount;
+import static org.elasticsearch.percolator.PercolatorTestUtil.assertMatchCount;
+import static org.elasticsearch.percolator.PercolatorTestUtil.preparePercolate;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.equalTo;
@@ -55,6 +56,16 @@ public class PercolatorAggregationsIT extends ESIntegTestCase {
     private final static String INDEX_NAME = "queries";
     private final static String TYPE_NAME = "query";
 
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singleton(PercolatorPlugin.class);
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
+        return Collections.singleton(PercolatorPlugin.class);
+    }
+
     // Just test the integration with facets and aggregations, not the facet and aggregation functionality!
     public void testAggregations() throws Exception {
         assertAcked(prepareCreate(INDEX_NAME)
@@ -83,7 +94,7 @@ public class PercolatorAggregationsIT extends ESIntegTestCase {
 
         for (int i = 0; i < numQueries; i++) {
             String value = values[i % numUniqueQueries];
-            PercolateRequestBuilder percolateRequestBuilder = client().preparePercolate()
+            PercolateRequestBuilder percolateRequestBuilder = preparePercolate(client())
                     .setIndices(INDEX_NAME)
                     .setDocumentType("type")
                     .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", value).endObject()))
@@ -150,7 +161,7 @@ public class PercolatorAggregationsIT extends ESIntegTestCase {
 
         for (int i = 0; i < numQueries; i++) {
             String value = values[i % numUniqueQueries];
-            PercolateRequestBuilder percolateRequestBuilder = client().preparePercolate()
+            PercolateRequestBuilder percolateRequestBuilder = preparePercolate(client())
                     .setIndices(INDEX_NAME)
                     .setDocumentType("type")
                     .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", value).endObject()))
@@ -204,7 +215,7 @@ public class PercolatorAggregationsIT extends ESIntegTestCase {
                 .addMapping(TYPE_NAME, "query", "type=percolator")
                 .execute().actionGet();
         ensureGreen();
-        PercolateRequestBuilder percolateRequestBuilder = client().preparePercolate().setIndices(INDEX_NAME).setDocumentType("type")
+        PercolateRequestBuilder percolateRequestBuilder = preparePercolate(client()).setIndices(INDEX_NAME).setDocumentType("type")
                 .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "value").endObject()))
                 .addAggregation(AggregationBuilders.significantTerms("a").field("field2"));
         PercolateResponse response = percolateRequestBuilder.get();
@@ -232,7 +243,7 @@ public class PercolatorAggregationsIT extends ESIntegTestCase {
 
         for (int i = 0; i < numQueries; i++) {
             String value = "value0";
-            PercolateRequestBuilder percolateRequestBuilder = client().preparePercolate()
+            PercolateRequestBuilder percolateRequestBuilder = preparePercolate(client())
                     .setIndices(INDEX_NAME)
                     .setDocumentType("type")
                     .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", value).endObject()))
diff --git a/core/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java
similarity index 86%
rename from core/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityIT.java
rename to modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java
index fbad26a5b6b..1071ffe759b 100644
--- a/core/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java
@@ -21,34 +21,47 @@ package org.elasticsearch.percolator;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestUtil;
 import org.elasticsearch.Version;
-import org.elasticsearch.action.percolate.PercolateResponse;
-import org.elasticsearch.action.percolate.PercolateSourceBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Collection;
+import java.util.Collections;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.elasticsearch.percolator.PercolatorTestUtil.preparePercolate;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0)
 @LuceneTestCase.SuppressFileSystems("ExtrasFS")
-public class PercolatorBackwardsCompatibilityIT extends ESIntegTestCase {
+// Can'r run as IT as the test cluster is immutable and this test adds nodes during the test
+public class PercolatorBackwardsCompatibilityTests extends ESIntegTestCase {
 
     private final static String INDEX_NAME = "percolator_index";
 
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singleton(PercolatorPlugin.class);
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
+        return Collections.singleton(PercolatorPlugin.class);
+    }
+
     public void testOldPercolatorIndex() throws Exception {
         setupNode();
 
@@ -76,7 +89,7 @@ public class PercolatorBackwardsCompatibilityIT extends ESIntegTestCase {
         assertThat(searchResponse.getHits().getAt(2).id(), equalTo("3"));
 
         // verify percolate response
-        PercolateResponse percolateResponse = client().preparePercolate()
+        PercolateResponse percolateResponse = preparePercolate(client())
             .setIndices(INDEX_NAME)
             .setDocumentType("message")
             .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("message", "the quick brown fox jumps over the lazy dog"))
@@ -93,7 +106,7 @@ public class PercolatorBackwardsCompatibilityIT extends ESIntegTestCase {
             .get();
         refresh();
 
-        percolateResponse = client().preparePercolate()
+        percolateResponse = preparePercolate(client())
             .setIndices(INDEX_NAME)
             .setDocumentType("message")
             .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("message", "the quick brown fox jumps over the lazy dog"))
@@ -109,7 +122,8 @@ public class PercolatorBackwardsCompatibilityIT extends ESIntegTestCase {
     private void setupNode() throws Exception {
         Path dataDir = createTempDir();
         Path clusterDir = Files.createDirectory(dataDir.resolve(cluster().getClusterName()));
-        try (InputStream stream = PercolatorBackwardsCompatibilityIT.class.getResourceAsStream("/indices/percolator/bwc_index_2.0.0.zip")) {
+        try (InputStream stream = PercolatorBackwardsCompatibilityTests.class.
+                getResourceAsStream("/indices/percolator/bwc_index_2.0.0.zip")) {
             TestUtil.unzip(stream, clusterDir);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/index/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
similarity index 91%
rename from core/src/test/java/org/elasticsearch/index/percolator/PercolatorFieldMapperTests.java
rename to modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
index 8faad63c0a4..bcb9ed8d40d 100644
--- a/core/src/test/java/org/elasticsearch/index/percolator/PercolatorFieldMapperTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.percolator;
+package org.elasticsearch.percolator;
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.compress.CompressedXContent;
@@ -32,10 +32,13 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.indices.TermsLookup;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
@@ -58,6 +61,11 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
     private MapperService mapperService;
     private PercolatorFieldMapper.PercolatorFieldType fieldType;
 
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return Collections.singleton(PercolatorPlugin.class);
+    }
+
     @Before
     public void init() throws Exception {
         indexService = createIndex("test", Settings.EMPTY);
@@ -119,7 +127,8 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
         // (it can't use shard data for rewriting purposes, because percolator queries run on MemoryIndex)
 
         for (QueryBuilder query : queries) {
-            ParsedDocument doc = mapperService.documentMapper(typeName).parse("test", typeName, "1", XContentFactory.jsonBuilder().startObject()
+            ParsedDocument doc = mapperService.documentMapper(typeName).parse("test", typeName, "1",
+                    XContentFactory.jsonBuilder().startObject()
                     .field(fieldName, query)
                     .endObject().bytes());
             BytesRef qbSource = doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName())[0].binaryValue();
@@ -191,11 +200,21 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
                     .startObject("query_field2").field("type", "percolator").endObject()
                 .endObject()
                 .endObject().endObject().string();
-        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> {
-            mapperService.merge(typeName, new CompressedXContent(percolatorMapper), MapperService.MergeReason.MAPPING_UPDATE, true);
-        });
-        assertThat(exception.getMessage(), equalTo("Up to one percolator field type is allowed per index, " +
-                "found the following percolator fields [[query_field1, query_field2]]"));
+        mapperService.merge(typeName, new CompressedXContent(percolatorMapper), MapperService.MergeReason.MAPPING_UPDATE, true);
+
+        QueryBuilder queryBuilder = matchQuery("field", "value");
+        ParsedDocument doc = mapperService.documentMapper(typeName).parse("test", typeName, "1",
+                jsonBuilder().startObject()
+                        .field("query_field1", queryBuilder)
+                        .field("query_field2", queryBuilder)
+                        .endObject().bytes()
+        );
+        assertThat(doc.rootDoc().getFields().size(), equalTo(22)); // also includes all other meta fields
+        BytesRef queryBuilderAsBytes = doc.rootDoc().getField("query_field1.query_builder_field").binaryValue();
+        assertQueryBuilder(queryBuilderAsBytes, queryBuilder);
+
+        queryBuilderAsBytes = doc.rootDoc().getField("query_field2.query_builder_field").binaryValue();
+        assertQueryBuilder(queryBuilderAsBytes, queryBuilder);
     }
 
     // percolator field can be nested under an object field, but only one query can be specified per document
diff --git a/core/src/test/java/org/elasticsearch/index/percolator/PercolatorHighlightSubFetchPhaseTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
similarity index 97%
rename from core/src/test/java/org/elasticsearch/index/percolator/PercolatorHighlightSubFetchPhaseTests.java
rename to modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
index fb130d5dde0..3f7aaafc105 100644
--- a/core/src/test/java/org/elasticsearch/index/percolator/PercolatorHighlightSubFetchPhaseTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.index.percolator;
+package org.elasticsearch.percolator;
 
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
@@ -25,7 +25,6 @@ import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.index.query.PercolateQuery;
 import org.elasticsearch.search.highlight.SearchContextHighlight;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.ESTestCase;
diff --git a/core/src/test/java/org/elasticsearch/percolator/PercolatorIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
similarity index 93%
rename from core/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
rename to modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
index c489025a943..e4b318f5d39 100644
--- a/core/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
@@ -24,9 +24,6 @@ import org.apache.lucene.search.join.ScoreMode;
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
-import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
-import org.elasticsearch.action.percolate.PercolateResponse;
-import org.elasticsearch.action.percolate.PercolateSourceBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Requests;
@@ -43,13 +40,15 @@ import org.elasticsearch.index.query.Operator;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.query.functionscore.WeightBuilder;
-import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.highlight.HighlightBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -60,7 +59,7 @@ import java.util.NavigableSet;
 import java.util.Set;
 import java.util.TreeSet;
 
-import static org.elasticsearch.action.percolate.PercolateSourceBuilder.docBuilder;
+import static org.elasticsearch.percolator.PercolateSourceBuilder.docBuilder;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.common.xcontent.XContentFactory.smileBuilder;
 import static org.elasticsearch.common.xcontent.XContentFactory.yamlBuilder;
@@ -71,20 +70,19 @@ import static org.elasticsearch.index.query.QueryBuilders.geoShapeQuery;
 import static org.elasticsearch.index.query.QueryBuilders.hasChildQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
-import static org.elasticsearch.index.query.QueryBuilders.nestedQuery;
 import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.fieldValueFactorFunction;
 import static org.elasticsearch.percolator.PercolatorTestUtil.convertFromTextArray;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertMatchCount;
+import static org.elasticsearch.percolator.PercolatorTestUtil.assertMatchCount;
+import static org.elasticsearch.percolator.PercolatorTestUtil.preparePercolate;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
 import static org.hamcrest.Matchers.arrayWithSize;
-import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.emptyArray;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -98,6 +96,16 @@ public class PercolatorIT extends ESIntegTestCase {
     private final static String INDEX_NAME = "queries";
     private final static String TYPE_NAME = "query";
 
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singleton(PercolatorPlugin.class);
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
+        return Collections.singleton(PercolatorPlugin.class);
+    }
+
     public void testSimple1() throws Exception {
         client().admin().indices().prepareCreate(INDEX_NAME).addMapping(TYPE_NAME, "query", "type=percolator").get();
         ensureGreen();
@@ -124,7 +132,7 @@ public class PercolatorIT extends ESIntegTestCase {
         refresh();
 
         logger.info("--> Percolate doc with field1=b");
-        PercolateResponse response = client().preparePercolate()
+        PercolateResponse response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType(TYPE_NAME)
                 .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "b").endObject()))
                 .execute().actionGet();
@@ -133,7 +141,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "4"));
 
         logger.info("--> Percolate doc with field1=c");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType(TYPE_NAME)
                 .setPercolateDoc(docBuilder().setDoc(yamlBuilder().startObject().field("field1", "c").endObject()))
                 .execute().actionGet();
@@ -142,7 +150,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("2", "4"));
 
         logger.info("--> Percolate doc with field1=b c");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType(TYPE_NAME)
                 .setPercolateDoc(docBuilder().setDoc(smileBuilder().startObject().field("field1", "b c").endObject()))
                 .execute().actionGet();
@@ -151,7 +159,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "2", "3", "4"));
 
         logger.info("--> Percolate doc with field1=d");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType(TYPE_NAME)
                 .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "d").endObject()))
                 .execute().actionGet();
@@ -161,7 +169,7 @@ public class PercolatorIT extends ESIntegTestCase {
 
         logger.info("--> Percolate non existing doc");
         try {
-            client().preparePercolate()
+            preparePercolate(client())
                     .setIndices(INDEX_NAME).setDocumentType(TYPE_NAME)
                     .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("5"))
                     .execute().actionGet();
@@ -184,7 +192,7 @@ public class PercolatorIT extends ESIntegTestCase {
                 .field("field2", "value")
                 .endObject().endObject();
 
-        PercolateResponse response = client().preparePercolate().setSource(doc)
+        PercolateResponse response = preparePercolate(client()).setSource(doc)
                 .setIndices(INDEX_NAME).setDocumentType(TYPE_NAME)
                 .execute().actionGet();
         assertMatchCount(response, 0L);
@@ -196,7 +204,7 @@ public class PercolatorIT extends ESIntegTestCase {
                 .execute().actionGet();
         refresh();
 
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType(TYPE_NAME)
                 .setSource(doc).execute().actionGet();
         assertMatchCount(response, 1L);
@@ -209,7 +217,7 @@ public class PercolatorIT extends ESIntegTestCase {
                 .execute().actionGet();
         refresh();
 
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type1")
                 .setSource(doc)
                 .execute().actionGet();
@@ -220,7 +228,7 @@ public class PercolatorIT extends ESIntegTestCase {
 
         client().prepareDelete(INDEX_NAME, TYPE_NAME, "test2").execute().actionGet();
         refresh();
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type1")
                 .setSource(doc).execute().actionGet();
         assertMatchCount(response, 1L);
@@ -246,7 +254,7 @@ public class PercolatorIT extends ESIntegTestCase {
         refresh();
 
         logger.info("--> Percolate doc with no routing");
-        PercolateResponse response = client().preparePercolate()
+        PercolateResponse response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type")
                 .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().startObject("doc").field("field1", "value").endObject().endObject()))
                 .setSize(100)
@@ -255,7 +263,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(response.getMatches(), arrayWithSize(100));
 
         logger.info("--> Percolate doc with routing=0");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type")
                 .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().startObject("doc").field("field1", "value").endObject().endObject()))
                         .setSize(100)
@@ -265,7 +273,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(response.getMatches(), arrayWithSize(50));
 
         logger.info("--> Percolate doc with routing=1");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type")
                 .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().startObject("doc").field("field1", "value").endObject().endObject()))
                 .setSize(100)
@@ -334,7 +342,7 @@ public class PercolatorIT extends ESIntegTestCase {
                 .execute().actionGet();
         refresh();
 
-        PercolateResponse percolate = client().preparePercolate()
+        PercolateResponse percolate = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("doc")
                 .setSource(jsonBuilder().startObject()
                         .startObject("doc").field("filingcategory", "s").endObject()
@@ -368,7 +376,7 @@ public class PercolatorIT extends ESIntegTestCase {
 
 
         for (int i = 0; i < 10; i++) {
-            PercolateResponse percolate = client().preparePercolate()
+            PercolateResponse percolate = preparePercolate(client())
                     .setIndices(INDEX_NAME).setDocumentType("type1")
                     .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value1").endObject().endObject())
                     .execute().actionGet();
@@ -377,7 +385,7 @@ public class PercolatorIT extends ESIntegTestCase {
         }
 
         for (int i = 0; i < 10; i++) {
-            PercolateResponse percolate = client().preparePercolate()
+            PercolateResponse percolate = preparePercolate(client())
                     .setIndices(INDEX_NAME).setDocumentType("type1")
                     .setPreference("_local")
                     .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value1").endObject().endObject())
@@ -421,7 +429,7 @@ public class PercolatorIT extends ESIntegTestCase {
                 .setRefresh(true)
                 .execute().actionGet();
 
-        PercolateResponse percolate = client().preparePercolate()
+        PercolateResponse percolate = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type1")
                 .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value1").endObject().endObject())
                 .execute().actionGet();
@@ -429,7 +437,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(percolate.getMatches(), arrayWithSize(1));
         assertThat(convertFromTextArray(percolate.getMatches(), INDEX_NAME), arrayContaining("kuku"));
 
-        percolate = client().preparePercolate()
+        percolate = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type1")
                 .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value2").endObject().endObject())
                 .execute().actionGet();
@@ -456,7 +464,7 @@ public class PercolatorIT extends ESIntegTestCase {
                 .setRefresh(true)
                 .execute().actionGet();
 
-        PercolateResponse percolate = client().preparePercolate()
+        PercolateResponse percolate = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type1")
                 .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value1").endObject().endObject())
                 .execute().actionGet();
@@ -473,7 +481,7 @@ public class PercolatorIT extends ESIntegTestCase {
                 .setRefresh(true)
                 .execute().actionGet();
 
-        percolate = client().preparePercolate()
+        percolate = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type1")
                 .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value2").endObject().endObject())
                 .execute().actionGet();
@@ -493,7 +501,7 @@ public class PercolatorIT extends ESIntegTestCase {
         PercolateSourceBuilder sourceBuilder = new PercolateSourceBuilder()
                 .setDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "value2").endObject()))
                 .setQueryBuilder(termQuery("color", "red"));
-        percolate = client().preparePercolate()
+        percolate = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type1")
                 .setSource(sourceBuilder)
                 .execute().actionGet();
@@ -504,7 +512,7 @@ public class PercolatorIT extends ESIntegTestCase {
         logger.info("--> deleting query 1");
         client().prepareDelete(INDEX_NAME, TYPE_NAME, "kuku").setRefresh(true).execute().actionGet();
 
-        percolate = client().preparePercolate()
+        percolate = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type1")
                 .setSource(jsonBuilder().startObject().startObject("doc").startObject("type1")
                         .field("field1", "value1")
@@ -545,7 +553,7 @@ public class PercolatorIT extends ESIntegTestCase {
         refresh();
 
         logger.info("--> Percolate existing doc with id 1");
-        PercolateResponse response = client().preparePercolate()
+        PercolateResponse response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type")
                 .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("1"))
                 .execute().actionGet();
@@ -554,7 +562,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "4"));
 
         logger.info("--> Percolate existing doc with id 2");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type")
                 .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("2"))
                 .execute().actionGet();
@@ -563,7 +571,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("2", "4"));
 
         logger.info("--> Percolate existing doc with id 3");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type")
                 .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("3"))
                 .execute().actionGet();
@@ -572,7 +580,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "2", "3", "4"));
 
         logger.info("--> Percolate existing doc with id 4");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type")
                 .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("4"))
                 .execute().actionGet();
@@ -612,7 +620,7 @@ public class PercolatorIT extends ESIntegTestCase {
         refresh();
 
         logger.info("--> Percolate existing doc with id 1");
-        PercolateResponse response = client().preparePercolate()
+        PercolateResponse response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type")
                 .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("1").routing("4"))
                 .execute().actionGet();
@@ -621,7 +629,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "4"));
 
         logger.info("--> Percolate existing doc with id 2");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type")
                 .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("2").routing("3"))
                 .execute().actionGet();
@@ -630,7 +638,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("2", "4"));
 
         logger.info("--> Percolate existing doc with id 3");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type")
                 .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("3").routing("2"))
                 .execute().actionGet();
@@ -639,7 +647,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "2", "3", "4"));
 
         logger.info("--> Percolate existing doc with id 4");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type")
                 .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("4").routing("1"))
                 .execute().actionGet();
@@ -679,7 +687,7 @@ public class PercolatorIT extends ESIntegTestCase {
         refresh();
 
         logger.info("--> Percolate existing doc with id 2 and version 1");
-        PercolateResponse response = client().preparePercolate()
+        PercolateResponse response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type")
                 .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("2").version(1L))
                 .execute().actionGet();
@@ -689,7 +697,7 @@ public class PercolatorIT extends ESIntegTestCase {
 
         logger.info("--> Percolate existing doc with id 2 and version 2");
         try {
-            client().preparePercolate()
+            preparePercolate(client())
                     .setIndices(INDEX_NAME).setDocumentType("type")
                     .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("2").version(2L))
                     .execute().actionGet();
@@ -701,7 +709,7 @@ public class PercolatorIT extends ESIntegTestCase {
         client().prepareIndex(INDEX_NAME, "type", "2").setSource("field1", "c").execute().actionGet();
 
         logger.info("--> Percolate existing doc with id 2 and version 2");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type")
                 .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("2").version(2L))
                 .execute().actionGet();
@@ -725,7 +733,7 @@ public class PercolatorIT extends ESIntegTestCase {
         refresh();
 
         logger.info("--> Percolate doc to index test1");
-        PercolateResponse response = client().preparePercolate()
+        PercolateResponse response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type")
                 .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value").endObject().endObject())
                 .execute().actionGet();
@@ -733,7 +741,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(response.getMatches(), arrayWithSize(5));
 
         logger.info("--> Percolate doc to index test2");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME + "2").setDocumentType("type")
                 .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value").endObject().endObject())
                 .execute().actionGet();
@@ -741,7 +749,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(response.getMatches(), arrayWithSize(5));
 
         logger.info("--> Percolate doc to index test1 and test2");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME, INDEX_NAME + "2").setDocumentType("type")
                 .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value").endObject().endObject())
                 .execute().actionGet();
@@ -749,7 +757,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(response.getMatches(), arrayWithSize(10));
 
         logger.info("--> Percolate doc to index test2 and test3, with ignore missing");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME , INDEX_NAME + "3").setDocumentType("type")
                 .setIndicesOptions(IndicesOptions.lenientExpandOpen())
                 .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value").endObject().endObject())
@@ -767,7 +775,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertTrue(aliasesResponse.isAcknowledged());
 
         logger.info("--> Percolate doc to my-alias1");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices("my-alias1").setDocumentType("type")
                 .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value").endObject().endObject())
                 .execute().actionGet();
@@ -778,7 +786,7 @@ public class PercolatorIT extends ESIntegTestCase {
         }
 
         logger.info("--> Percolate doc to my-alias2");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices("my-alias2").setDocumentType("type")
                 .setSource(jsonBuilder().startObject().startObject("doc").field("field1", "value").endObject().endObject())
                 .execute().actionGet();
@@ -808,7 +816,7 @@ public class PercolatorIT extends ESIntegTestCase {
         // Specifying only the document to percolate and no filter, sorting or aggs, the queries are retrieved from
         // memory directly. Otherwise we need to retrieve those queries from lucene to be able to execute filters,
         // aggregations and sorting on top of them. So this test a different code execution path.
-        PercolateResponse response = client().preparePercolate()
+        PercolateResponse response = preparePercolate(client())
                 .setIndices("a")
                 .setDocumentType("my-type")
                 .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("{}"))
@@ -817,7 +825,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(response.getCount(), equalTo(1L));
         assertThat(response.getMatches()[0].getId().string(), equalTo("1"));
 
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices("b")
                 .setDocumentType("my-type")
                 .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("{}"))
@@ -827,7 +835,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(response.getMatches()[0].getId().string(), equalTo("2"));
 
 
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices("c")
                 .setDocumentType("my-type")
                 .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("{}"))
@@ -836,7 +844,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(response.getCount(), equalTo(0L));
 
         // Testing that the alias filter and the filter specified while percolating are both taken into account.
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices("a")
                 .setDocumentType("my-type")
                 .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("{}"))
@@ -846,7 +854,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(response.getCount(), equalTo(1L));
         assertThat(response.getMatches()[0].getId().string(), equalTo("1"));
 
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices("b")
                 .setDocumentType("my-type")
                 .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("{}"))
@@ -857,7 +865,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(response.getMatches()[0].getId().string(), equalTo("2"));
 
 
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices("c")
                 .setDocumentType("my-type")
                 .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("{}"))
@@ -895,7 +903,7 @@ public class PercolatorIT extends ESIntegTestCase {
         refresh();
 
         logger.info("--> Count percolate doc with field1=b");
-        PercolateResponse response = client().preparePercolate()
+        PercolateResponse response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type").setOnlyCount(true)
                 .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "b").endObject()))
                 .execute().actionGet();
@@ -903,7 +911,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(response.getMatches(), nullValue());
 
         logger.info("--> Count percolate doc with field1=c");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type").setOnlyCount(true)
                 .setPercolateDoc(docBuilder().setDoc(yamlBuilder().startObject().field("field1", "c").endObject()))
                 .execute().actionGet();
@@ -911,7 +919,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(response.getMatches(), nullValue());
 
         logger.info("--> Count percolate doc with field1=b c");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type").setOnlyCount(true)
                 .setPercolateDoc(docBuilder().setDoc(smileBuilder().startObject().field("field1", "b c").endObject()))
                 .execute().actionGet();
@@ -919,7 +927,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(response.getMatches(), nullValue());
 
         logger.info("--> Count percolate doc with field1=d");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type").setOnlyCount(true)
                 .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "d").endObject()))
                 .execute().actionGet();
@@ -928,7 +936,7 @@ public class PercolatorIT extends ESIntegTestCase {
 
         logger.info("--> Count percolate non existing doc");
         try {
-            client().preparePercolate()
+            preparePercolate(client())
                     .setIndices(INDEX_NAME).setDocumentType("type").setOnlyCount(true)
                     .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("5"))
                     .execute().actionGet();
@@ -969,7 +977,7 @@ public class PercolatorIT extends ESIntegTestCase {
         refresh();
 
         logger.info("--> Count percolate existing doc with id 1");
-        PercolateResponse response = client().preparePercolate()
+        PercolateResponse response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type").setOnlyCount(true)
                 .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("1"))
                 .execute().actionGet();
@@ -977,7 +985,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(response.getMatches(), nullValue());
 
         logger.info("--> Count percolate existing doc with id 2");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type").setOnlyCount(true)
                 .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("2"))
                 .execute().actionGet();
@@ -985,7 +993,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(response.getMatches(), nullValue());
 
         logger.info("--> Count percolate existing doc with id 3");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type").setOnlyCount(true)
                 .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("3"))
                 .execute().actionGet();
@@ -993,7 +1001,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(response.getMatches(), nullValue());
 
         logger.info("--> Count percolate existing doc with id 4");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type").setOnlyCount(true)
                 .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("4"))
                 .execute().actionGet();
@@ -1021,7 +1029,7 @@ public class PercolatorIT extends ESIntegTestCase {
         refresh();
 
         boolean onlyCount = randomBoolean();
-        PercolateResponse response = client().preparePercolate()
+        PercolateResponse response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("my-type")
                 .setOnlyCount(onlyCount)
                 .setPercolateDoc(docBuilder().setDoc("field", "value"))
@@ -1033,7 +1041,7 @@ public class PercolatorIT extends ESIntegTestCase {
         }
 
         int size = randomIntBetween(0, (int) totalQueries - 1);
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("my-type")
                 .setOnlyCount(onlyCount)
                 .setPercolateDoc(docBuilder().setDoc("field", "value"))
@@ -1050,7 +1058,7 @@ public class PercolatorIT extends ESIntegTestCase {
         int runs = randomIntBetween(3, 16);
         for (int i = 0; i < runs; i++) {
             onlyCount = randomBoolean();
-            response = client().preparePercolate()
+            response = preparePercolate(client())
                     .setIndices(INDEX_NAME).setDocumentType("my-type")
                     .setOnlyCount(onlyCount)
                     .setPercolateDoc(docBuilder().setDoc("field", "value"))
@@ -1065,7 +1073,7 @@ public class PercolatorIT extends ESIntegTestCase {
 
         for (int i = 0; i < runs; i++) {
             onlyCount = randomBoolean();
-            response = client().preparePercolate()
+            response = preparePercolate(client())
                     .setIndices(INDEX_NAME).setDocumentType("my-type")
                     .setOnlyCount(onlyCount)
                     .setPercolateDoc(docBuilder().setDoc("field", "value"))
@@ -1081,7 +1089,7 @@ public class PercolatorIT extends ESIntegTestCase {
         for (int i = 0; i < runs; i++) {
             onlyCount = randomBoolean();
             size = randomIntBetween(0, (int) numQueriesPerLevel - 1);
-            response = client().preparePercolate()
+            response = preparePercolate(client())
                     .setIndices(INDEX_NAME).setDocumentType("my-type")
                     .setOnlyCount(onlyCount)
                     .setSize(size)
@@ -1124,7 +1132,7 @@ public class PercolatorIT extends ESIntegTestCase {
         int runs = randomInt(27);
         for (int i = 0; i < runs; i++) {
             int size = randomIntBetween(1, 50);
-            PercolateResponse response = client().preparePercolate().setIndices(INDEX_NAME).setDocumentType("my-type")
+            PercolateResponse response = preparePercolate(client()).setIndices(INDEX_NAME).setDocumentType("my-type")
                     .setScore(true)
                     .setSize(size)
                     .setPercolateDoc(docBuilder().setDoc("field", "value"))
@@ -1141,7 +1149,7 @@ public class PercolatorIT extends ESIntegTestCase {
         // Sort the queries by the score
         for (int i = 0; i < runs; i++) {
             int size = randomIntBetween(1, 10);
-            PercolateResponse response = client().preparePercolate().setIndices(INDEX_NAME).setDocumentType("my-type")
+            PercolateResponse response = preparePercolate(client()).setIndices(INDEX_NAME).setDocumentType("my-type")
                     .setSortByScore(true)
                     .setSize(size)
                     .setPercolateDoc(docBuilder().setDoc("field", "value"))
@@ -1164,7 +1172,7 @@ public class PercolatorIT extends ESIntegTestCase {
             int value = usedValues.get(randomInt(usedValues.size() - 1));
             NavigableSet<Integer> levels = controlMap.get(value);
             int size = randomIntBetween(1, levels.size());
-            PercolateResponse response = client().preparePercolate().setIndices(INDEX_NAME).setDocumentType("my-type")
+            PercolateResponse response = preparePercolate(client()).setIndices(INDEX_NAME).setDocumentType("my-type")
                     .setSortByScore(true)
                     .setSize(size)
                     .setPercolateDoc(docBuilder().setDoc("field", "value"))
@@ -1200,7 +1208,7 @@ public class PercolatorIT extends ESIntegTestCase {
                 .execute().actionGet();
         refresh();
 
-        PercolateResponse response = client().preparePercolate().setIndices(INDEX_NAME).setDocumentType("my-type")
+        PercolateResponse response = preparePercolate(client()).setIndices(INDEX_NAME).setDocumentType("my-type")
                 .setSortByScore(true)
                 .setSize(2)
                 .setPercolateDoc(docBuilder().setDoc("field", "value"))
@@ -1219,7 +1227,7 @@ public class PercolatorIT extends ESIntegTestCase {
                 .get();
         ensureGreen();
 
-        PercolateResponse response = client().preparePercolate().setIndices(INDEX_NAME).setDocumentType("my-type")
+        PercolateResponse response = preparePercolate(client()).setIndices(INDEX_NAME).setDocumentType("my-type")
                 .setSortByScore(true)
                 .setSize(2)
                 .setPercolateDoc(docBuilder().setDoc("field", "value"))
@@ -1260,7 +1268,7 @@ public class PercolatorIT extends ESIntegTestCase {
         refresh();
 
         logger.info("--> Percolate doc with field1=The quick brown fox jumps over the lazy dog");
-        PercolateResponse response = client().preparePercolate()
+        PercolateResponse response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type")
                 .setSize(5)
                 .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "The quick brown fox jumps over the lazy dog").endObject()))
@@ -1280,7 +1288,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(matches[4].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown <em>fox</em> jumps over the lazy dog"));
 
         logger.info("--> Query percolate doc with field1=The quick brown fox jumps over the lazy dog");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type")
                 .setSize(5)
                 .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "The quick brown fox jumps over the lazy dog").endObject()))
@@ -1306,7 +1314,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(matches[4].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown <em>fox</em> jumps over the lazy dog"));
 
         logger.info("--> Query percolate with score for doc with field1=The quick brown fox jumps over the lazy dog");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type")
                 .setSize(5)
                 .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "The quick brown fox jumps over the lazy dog").endObject()))
@@ -1338,7 +1346,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(matches[4].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown <em>fox</em> jumps over the lazy dog"));
 
         logger.info("--> Top percolate for doc with field1=The quick brown fox jumps over the lazy dog");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type")
                 .setSize(5)
                 .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "The quick brown fox jumps over the lazy dog").endObject()))
@@ -1370,7 +1378,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(matches[4].getHighlightFields().get("field1").fragments()[0].string(), equalTo("The quick brown <em>fox</em> jumps over the lazy dog"));
 
         logger.info("--> Top percolate for doc with field1=The quick brown fox jumps over the lazy dog");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type")
                 .setSize(5)
                 .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "The quick brown fox jumps over the lazy dog").endObject()))
@@ -1408,7 +1416,7 @@ public class PercolatorIT extends ESIntegTestCase {
         refresh();
 
         logger.info("--> Top percolate for doc with field1=The quick brown fox jumps over the lazy dog");
-        response = client().preparePercolate()
+        response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type")
                 .setSize(5)
                 .setGetRequest(Requests.getRequest(INDEX_NAME).type("type").id("1"))
@@ -1457,7 +1465,7 @@ public class PercolatorIT extends ESIntegTestCase {
                 .execute().actionGet();
         refresh();
 
-        PercolateResponse percolate = client().preparePercolate()
+        PercolateResponse percolate = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("doc")
                 .setSource(jsonBuilder().startObject()
                         .startObject("doc").field("message", "A new bonsai tree ").endObject()
@@ -1469,9 +1477,9 @@ public class PercolatorIT extends ESIntegTestCase {
 
     public void testNestedPercolation() throws IOException {
         initNestedIndexAndPercolation();
-        PercolateResponse response = client().preparePercolate().setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc(getNotMatchingNestedDoc())).setIndices(INDEX_NAME).setDocumentType("company").get();
+        PercolateResponse response = preparePercolate(client()).setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc(getNotMatchingNestedDoc())).setIndices(INDEX_NAME).setDocumentType("company").get();
         assertEquals(response.getMatches().length, 0);
-        response = client().preparePercolate().setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc(getMatchingNestedDoc())).setIndices(INDEX_NAME).setDocumentType("company").get();
+        response = preparePercolate(client()).setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc(getMatchingNestedDoc())).setIndices(INDEX_NAME).setDocumentType("company").get();
         assertEquals(response.getMatches().length, 1);
         assertEquals(response.getMatches()[0].getId().string(), "Q");
     }
@@ -1481,7 +1489,7 @@ public class PercolatorIT extends ESIntegTestCase {
         XContentBuilder doc = jsonBuilder();
         doc.startObject();
         doc.field("some_unnested_field", "value");
-        PercolateResponse response = client().preparePercolate().setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc(doc)).setIndices(INDEX_NAME).setDocumentType("company").get();
+        PercolateResponse response = preparePercolate(client()).setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc(doc)).setIndices(INDEX_NAME).setDocumentType("company").get();
         assertNoFailures(response);
     }
 
@@ -1490,9 +1498,9 @@ public class PercolatorIT extends ESIntegTestCase {
         client().prepareIndex(INDEX_NAME, "company", "notmatching").setSource(getNotMatchingNestedDoc()).get();
         client().prepareIndex(INDEX_NAME, "company", "matching").setSource(getMatchingNestedDoc()).get();
         refresh();
-        PercolateResponse response = client().preparePercolate().setGetRequest(Requests.getRequest(INDEX_NAME).type("company").id("notmatching")).setDocumentType("company").setIndices(INDEX_NAME).get();
+        PercolateResponse response = preparePercolate(client()).setGetRequest(Requests.getRequest(INDEX_NAME).type("company").id("notmatching")).setDocumentType("company").setIndices(INDEX_NAME).get();
         assertEquals(response.getMatches().length, 0);
-        response = client().preparePercolate().setGetRequest(Requests.getRequest(INDEX_NAME).type("company").id("matching")).setDocumentType("company").setIndices(INDEX_NAME).get();
+        response = preparePercolate(client()).setGetRequest(Requests.getRequest(INDEX_NAME).type("company").id("matching")).setDocumentType("company").setIndices(INDEX_NAME).get();
         assertEquals(response.getMatches().length, 1);
         assertEquals(response.getMatches()[0].getId().string(), "Q");
     }
@@ -1511,7 +1519,7 @@ public class PercolatorIT extends ESIntegTestCase {
                 .get();
         refresh();
 
-        PercolateResponse response = client().preparePercolate()
+        PercolateResponse response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type")
                 .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field", "value").endObject()))
                 .setPercolateQuery(QueryBuilders.matchAllQuery())
@@ -1562,7 +1570,7 @@ public class PercolatorIT extends ESIntegTestCase {
         refresh();
 
         logger.info("--> Percolate doc with field1=b");
-        PercolateResponse response = client().preparePercolate()
+        PercolateResponse response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("my-type")
                 .setPercolateDoc(docBuilder().setDoc("timestamp", System.currentTimeMillis()))
                 .get();
@@ -1722,7 +1730,7 @@ public class PercolatorIT extends ESIntegTestCase {
         client().prepareIndex(INDEX_NAME, TYPE_NAME).setSource(q5).setId("q5").get();
         client().prepareIndex(INDEX_NAME, TYPE_NAME).setSource(q6).setId("q6").get();
         refresh();
-        PercolateResponse response = client().preparePercolate()
+        PercolateResponse response = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("doc")
                 .setPercolateDoc(docBuilder().setDoc(doc))
                 .get();
@@ -1735,12 +1743,12 @@ public class PercolatorIT extends ESIntegTestCase {
             assertTrue(expectedIds.remove(match.getId().string()));
         }
         assertTrue(expectedIds.isEmpty());
-        response = client().preparePercolate().setOnlyCount(true)
+        response = preparePercolate(client()).setOnlyCount(true)
                 .setIndices(INDEX_NAME).setDocumentType("doc")
                 .setPercolateDoc(docBuilder().setDoc(doc))
                 .get();
         assertMatchCount(response, 3L);
-        response = client().preparePercolate().setScore(randomBoolean()).setSortByScore(randomBoolean()).setOnlyCount(randomBoolean()).setSize(10).setPercolateQuery(QueryBuilders.termQuery("text", "foo"))
+        response = preparePercolate(client()).setScore(randomBoolean()).setSortByScore(randomBoolean()).setOnlyCount(randomBoolean()).setSize(10).setPercolateQuery(QueryBuilders.termQuery("text", "foo"))
                 .setIndices(INDEX_NAME).setDocumentType("doc")
                 .setPercolateDoc(docBuilder().setDoc(doc))
                 .get();
@@ -1759,7 +1767,7 @@ public class PercolatorIT extends ESIntegTestCase {
                 .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "value")).endObject()).get();
         refresh();
         logger.info("--> Percolate doc with field1=value");
-        PercolateResponse response1 = client().preparePercolate()
+        PercolateResponse response1 = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type")
                 .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "value").endObject()))
                 .execute().actionGet();
@@ -1781,7 +1789,7 @@ public class PercolatorIT extends ESIntegTestCase {
             .get();
         refresh();
 
-        PercolateResponse response1 = client().preparePercolate()
+        PercolateResponse response1 = preparePercolate(client())
             .setIndices(INDEX_NAME).setDocumentType("type")
             .setPercolateDoc(docBuilder().setDoc(jsonBuilder().startObject()
                 .startObject("location")
@@ -1817,7 +1825,7 @@ public class PercolatorIT extends ESIntegTestCase {
         refresh();
 
         // Just percolating a document that has a _parent field in its mapping should just work:
-        PercolateResponse response = client().preparePercolate()
+        PercolateResponse response = preparePercolate(client())
                 .setDocumentType("parent")
                 .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("field", "value"))
                 .get();
@@ -1832,7 +1840,7 @@ public class PercolatorIT extends ESIntegTestCase {
                 .get();
         refresh();
 
-        PercolateResponse response = client().preparePercolate()
+        PercolateResponse response = preparePercolate(client())
                 .setIndices(INDEX_NAME)
                 .setDocumentType("type")
                 .setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc("{}"))
diff --git a/core/src/test/java/org/elasticsearch/search/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
similarity index 89%
rename from core/src/test/java/org/elasticsearch/search/percolator/PercolatorQuerySearchIT.java
rename to modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
index 067d84d5e00..aba81575b37 100644
--- a/core/src/test/java/org/elasticsearch/search/percolator/PercolatorQuerySearchIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.search.percolator;
+package org.elasticsearch.percolator;
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -24,17 +24,20 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.query.MultiMatchQueryBuilder;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.highlight.HighlightBuilder;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
+import java.util.Collection;
+import java.util.Collections;
+
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
 import static org.elasticsearch.index.query.QueryBuilders.commonTermsQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery;
-import static org.elasticsearch.index.query.QueryBuilders.percolateQuery;
 import static org.elasticsearch.index.query.QueryBuilders.spanNearQuery;
 import static org.elasticsearch.index.query.QueryBuilders.spanNotQuery;
 import static org.elasticsearch.index.query.QueryBuilders.spanTermQuery;
@@ -46,6 +49,11 @@ import static org.hamcrest.Matchers.startsWith;
 
 public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
 
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return Collections.singleton(PercolatorPlugin.class);
+    }
+
     public void testPercolatorQuery() throws Exception {
         createIndex("test", client().admin().indices().prepareCreate("test")
                 .addMapping("type", "field1", "type=keyword", "field2", "type=keyword")
@@ -68,7 +76,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
         BytesReference source = jsonBuilder().startObject().endObject().bytes();
         logger.info("percolating empty doc");
         SearchResponse response = client().prepareSearch()
-            .setQuery(percolateQuery("query", "type", source))
+            .setQuery(new PercolateQueryBuilder("query", "type", source))
             .get();
         assertHitCount(response, 1);
         assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
@@ -76,7 +84,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
         source = jsonBuilder().startObject().field("field1", "value").endObject().bytes();
         logger.info("percolating doc with 1 field");
         response = client().prepareSearch()
-            .setQuery(percolateQuery("query", "type", source))
+            .setQuery(new PercolateQueryBuilder("query", "type", source))
             .addSort("_uid", SortOrder.ASC)
             .get();
         assertHitCount(response, 2);
@@ -86,7 +94,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
         source = jsonBuilder().startObject().field("field1", "value").field("field2", "value").endObject().bytes();
         logger.info("percolating doc with 2 fields");
         response = client().prepareSearch()
-            .setQuery(percolateQuery("query", "type", source))
+            .setQuery(new PercolateQueryBuilder("query", "type", source))
             .addSort("_uid", SortOrder.ASC)
             .get();
         assertHitCount(response, 3);
@@ -120,14 +128,14 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
 
         logger.info("percolating empty doc");
         SearchResponse response = client().prepareSearch()
-                .setQuery(percolateQuery("query", "type", "test", "type", "1"))
+                .setQuery(new PercolateQueryBuilder("query", "type", "test", "type", "1", null, null, null))
                 .get();
         assertHitCount(response, 1);
         assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
 
         logger.info("percolating doc with 1 field");
         response = client().prepareSearch()
-                .setQuery(percolateQuery("query", "type", "test", "type", "2"))
+                .setQuery(new PercolateQueryBuilder("query", "type", "test", "type", "2", null, null, null))
                 .addSort("_uid", SortOrder.ASC)
                 .get();
         assertHitCount(response, 2);
@@ -136,7 +144,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
 
         logger.info("percolating doc with 2 fields");
         response = client().prepareSearch()
-                .setQuery(percolateQuery("query", "type", "test", "type", "3"))
+                .setQuery(new PercolateQueryBuilder("query", "type", "test", "type", "3", null, null, null))
                 .addSort("_uid", SortOrder.ASC)
                 .get();
         assertHitCount(response, 3);
@@ -203,7 +211,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
                 .field("field2", "the quick brown fox falls down into the well")
                 .endObject().bytes();
         SearchResponse response = client().prepareSearch()
-                .setQuery(percolateQuery("query", "type", source))
+                .setQuery(new PercolateQueryBuilder("query", "type", source))
                 .addSort("_uid", SortOrder.ASC)
                 .get();
         assertHitCount(response, 4);
@@ -250,7 +258,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
                 .field("field1", "The quick brown fox jumps over the lazy dog")
                 .endObject().bytes();
         SearchResponse searchResponse = client().prepareSearch()
-                .setQuery(percolateQuery("query", "type", document))
+                .setQuery(new PercolateQueryBuilder("query", "type", document))
                 .highlighter(new HighlightBuilder().field("field1"))
                 .addSort("_uid", SortOrder.ASC)
                 .get();
@@ -284,34 +292,30 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
         client().admin().indices().prepareRefresh().get();
 
         SearchResponse response = client().prepareSearch().setQuery(
-                percolateQuery("query", "type", new BytesArray("{\"field\" : [\"brown\", \"fox\"]}"))
+                new PercolateQueryBuilder("query", "type", new BytesArray("{\"field\" : [\"brown\", \"fox\"]}"))
         ).get();
         assertHitCount(response, 1);
         assertThat(response.getHits().getAt(0).getId(), equalTo("2"));
     }
 
 
-    public void testIllegalMappings() throws Exception {
+    public void testManyPercolatorFields() throws Exception {
         String queryFieldName = randomAsciiOfLength(8);
-        MapperParsingException e = expectThrows(MapperParsingException.class, () -> {
-            createIndex("test", client().admin().indices().prepareCreate("test")
-                    .addMapping("doc_type", "field", "type=keyword")
-                    .addMapping("query_type1", queryFieldName, "type=percolator")
-                    .addMapping("query_type2", queryFieldName, "type=percolator", "second_query_field", "type=percolator")
-                    .addMapping("query_type3", jsonBuilder().startObject().startObject("query_type3").startObject("properties")
-                            .startObject("object_field")
-                            .field("type", "object")
-                            .startObject("properties")
-                            .startObject(queryFieldName)
-                            .field("type", "percolator")
-                            .endObject()
-                            .endObject()
-                            .endObject()
-                            .endObject().endObject())
-            );
-        });
-        assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
-        assertThat(e.getCause().getMessage(), startsWith("Up to one percolator field type is allowed per index"));
+        createIndex("test", client().admin().indices().prepareCreate("test")
+                .addMapping("doc_type", "field", "type=keyword")
+                .addMapping("query_type1", queryFieldName, "type=percolator")
+                .addMapping("query_type2", queryFieldName, "type=percolator", "second_query_field", "type=percolator")
+                .addMapping("query_type3", jsonBuilder().startObject().startObject("query_type3").startObject("properties")
+                        .startObject("object_field")
+                        .field("type", "object")
+                        .startObject("properties")
+                        .startObject(queryFieldName)
+                        .field("type", "percolator")
+                        .endObject()
+                        .endObject()
+                        .endObject()
+                        .endObject().endObject())
+        );
     }
 
     public void testWithMultiplePercolatorFields() throws Exception {
@@ -346,7 +350,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
 
         BytesReference source = jsonBuilder().startObject().field("field", "value").endObject().bytes();
         SearchResponse response = client().prepareSearch()
-                .setQuery(percolateQuery(queryFieldName, "doc_type", source))
+                .setQuery(new PercolateQueryBuilder(queryFieldName, "doc_type", source))
                 .setIndices("test1")
                 .get();
         assertHitCount(response, 1);
@@ -355,7 +359,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
         assertThat(response.getHits().getAt(0).index(), equalTo("test1"));
 
         response = client().prepareSearch()
-                .setQuery(percolateQuery("object_field." + queryFieldName, "doc_type", source))
+                .setQuery(new PercolateQueryBuilder("object_field." + queryFieldName, "doc_type", source))
                 .setIndices("test2")
                 .get();
         assertHitCount(response, 1);
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorRestIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorRestIT.java
new file mode 100644
index 00000000000..9f4eff64995
--- /dev/null
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorRestIT.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.percolator;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.RestTestCandidate;
+import org.elasticsearch.test.rest.parser.RestTestParseException;
+
+import java.io.IOException;
+
+public class PercolatorRestIT extends ESRestTestCase {
+    public PercolatorRestIT(@Name("yaml") RestTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws IOException, RestTestParseException {
+        return ESRestTestCase.createParameters(0, 1);
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/percolator/PercolatorTestUtil.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorTestUtil.java
similarity index 57%
rename from test/framework/src/main/java/org/elasticsearch/percolator/PercolatorTestUtil.java
rename to modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorTestUtil.java
index 671d7b8fd3b..32ea4c23ad4 100644
--- a/test/framework/src/main/java/org/elasticsearch/percolator/PercolatorTestUtil.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorTestUtil.java
@@ -19,13 +19,33 @@
 
 package org.elasticsearch.percolator;
 
-import org.elasticsearch.action.percolate.PercolateResponse;
+import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.Strings;
 import org.junit.Assert;
 
+import static org.hamcrest.Matchers.greaterThan;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertVersionSerializable;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.formatShardStatus;
+
 /** Static method pulled out of PercolatorIT, used by other tests */
 public class PercolatorTestUtil extends Assert {
-    
+
+    public static PercolateRequestBuilder preparePercolate(ElasticsearchClient client) {
+        return new PercolateRequestBuilder(client, PercolateAction.INSTANCE);
+    }
+
+    public static MultiPercolateRequestBuilder prepareMultiPercolate(ElasticsearchClient client) {
+        return new MultiPercolateRequestBuilder(client, MultiPercolateAction.INSTANCE);
+    }
+
+    public static void assertMatchCount(PercolateResponse percolateResponse, long expectedHitCount) {
+        if (percolateResponse.getCount() != expectedHitCount) {
+            fail("Count is " + percolateResponse.getCount() + " but " + expectedHitCount + " was expected. " +
+                    formatShardStatus(percolateResponse));
+        }
+        assertVersionSerializable(percolateResponse);
+    }
+
     public static String[] convertFromTextArray(PercolateResponse.Match[] matches, String index) {
         if (matches.length == 0) {
             return Strings.EMPTY_ARRAY;
diff --git a/core/src/test/resources/indices/percolator/bwc_index_2.0.0.zip b/modules/percolator/src/test/resources/indices/percolator/bwc_index_2.0.0.zip
similarity index 100%
rename from core/src/test/resources/indices/percolator/bwc_index_2.0.0.zip
rename to modules/percolator/src/test/resources/indices/percolator/bwc_index_2.0.0.zip
diff --git a/core/src/test/resources/org/elasticsearch/action/percolate/mpercolate1.json b/modules/percolator/src/test/resources/org/elasticsearch/percolator/mpercolate1.json
similarity index 100%
rename from core/src/test/resources/org/elasticsearch/action/percolate/mpercolate1.json
rename to modules/percolator/src/test/resources/org/elasticsearch/percolator/mpercolate1.json
diff --git a/core/src/test/resources/org/elasticsearch/action/percolate/mpercolate2.json b/modules/percolator/src/test/resources/org/elasticsearch/percolator/mpercolate2.json
similarity index 100%
rename from core/src/test/resources/org/elasticsearch/action/percolate/mpercolate2.json
rename to modules/percolator/src/test/resources/org/elasticsearch/percolator/mpercolate2.json
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/mpercolate/10_basic.yaml b/modules/percolator/src/test/resources/rest-api-spec/test/mpercolate/10_basic.yaml
similarity index 100%
rename from rest-api-spec/src/main/resources/rest-api-spec/test/mpercolate/10_basic.yaml
rename to modules/percolator/src/test/resources/rest-api-spec/test/mpercolate/10_basic.yaml
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/percolate/15_new.yaml b/modules/percolator/src/test/resources/rest-api-spec/test/percolate/15_new.yaml
similarity index 100%
rename from rest-api-spec/src/main/resources/rest-api-spec/test/percolate/15_new.yaml
rename to modules/percolator/src/test/resources/rest-api-spec/test/percolate/15_new.yaml
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/percolate/16_existing_doc.yaml b/modules/percolator/src/test/resources/rest-api-spec/test/percolate/16_existing_doc.yaml
similarity index 100%
rename from rest-api-spec/src/main/resources/rest-api-spec/test/percolate/16_existing_doc.yaml
rename to modules/percolator/src/test/resources/rest-api-spec/test/percolate/16_existing_doc.yaml
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/percolate/17_empty.yaml b/modules/percolator/src/test/resources/rest-api-spec/test/percolate/17_empty.yaml
similarity index 100%
rename from rest-api-spec/src/main/resources/rest-api-spec/test/percolate/17_empty.yaml
rename to modules/percolator/src/test/resources/rest-api-spec/test/percolate/17_empty.yaml
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/percolate/18_highligh_with_query.yaml b/modules/percolator/src/test/resources/rest-api-spec/test/percolate/18_highligh_with_query.yaml
similarity index 100%
rename from rest-api-spec/src/main/resources/rest-api-spec/test/percolate/18_highligh_with_query.yaml
rename to modules/percolator/src/test/resources/rest-api-spec/test/percolate/18_highligh_with_query.yaml
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/percolate/19_nested.yaml b/modules/percolator/src/test/resources/rest-api-spec/test/percolate/19_nested.yaml
similarity index 100%
rename from rest-api-spec/src/main/resources/rest-api-spec/test/percolate/19_nested.yaml
rename to modules/percolator/src/test/resources/rest-api-spec/test/percolate/19_nested.yaml
diff --git a/settings.gradle b/settings.gradle
index 88217a9dde5..ded500ebe56 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -20,6 +20,7 @@ List projects = [
   'modules:lang-mustache',
   'modules:lang-painless',
   'modules:reindex',
+  'modules:percolator',
   'plugins:analysis-icu',
   'plugins:analysis-kuromoji',
   'plugins:analysis-phonetic',
diff --git a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
similarity index 73%
rename from core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
rename to test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
index b49488bd7c2..c27617a7287 100644
--- a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
@@ -17,11 +17,23 @@
  * under the License.
  */
 
-package org.elasticsearch.index.query;
+package org.elasticsearch.test;
 
 import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
 import com.fasterxml.jackson.core.JsonParseException;
 import com.fasterxml.jackson.core.io.JsonStringEncoder;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.index.query.AbstractQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.node.internal.InternalSettingsPreparer;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.PluginsModule;
+import org.elasticsearch.plugins.PluginsService;
 
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.Query;
@@ -95,11 +107,6 @@ import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptSettings;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.IndexSettingsModule;
-import org.elasticsearch.test.InternalSettingsPlugin;
-import org.elasticsearch.test.TestSearchContext;
-import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPoolModule;
 import org.joda.time.DateTime;
@@ -107,13 +114,14 @@ import org.joda.time.DateTimeZone;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
-import org.junit.BeforeClass;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.lang.reflect.InvocationHandler;
 import java.lang.reflect.Method;
 import java.lang.reflect.Proxy;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -122,8 +130,6 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.createClusterService;
-import static org.elasticsearch.cluster.service.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;
@@ -133,7 +139,7 @@ import static org.hamcrest.Matchers.not;
 public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>> extends ESTestCase {
 
     private static final GeohashGenerator geohashGenerator = new GeohashGenerator();
-    protected static final String STRING_FIELD_NAME = "mapped_string";
+    public static final String STRING_FIELD_NAME = "mapped_string";
     protected static final String STRING_FIELD_NAME_2 = "mapped_string_2";
     protected static final String INT_FIELD_NAME = "mapped_int";
     protected static final String DOUBLE_FIELD_NAME = "mapped_double";
@@ -149,209 +155,42 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, GEO_POINT_FIELD_NAME};
     private static final int NUMBER_OF_TESTQUERIES = 20;
 
-    private static Injector injector;
-    private static IndicesQueriesRegistry indicesQueriesRegistry;
-    private static IndexFieldDataService indexFieldDataService;
+    private static ServiceHolder serviceHolder;
     private static int queryNameId = 0;
-    private static SearchModule searchModule;
-
-    protected static IndexFieldDataService indexFieldDataService() {
-        return indexFieldDataService;
-    }
-
-    private static Index index;
-
-    protected static Index getIndex() {
-        return index;
-    }
-
-    private static Version indexVersionCreated;
-
-    protected static Version getIndexVersionCreated() {
-        return indexVersionCreated;
-    }
-
-    private static String[] currentTypes;
-
-    protected static String[] getCurrentTypes() {
-        return currentTypes;
-    }
-
-    protected static SearchModule getSearchModule() {
-        return searchModule;
-    }
-
-    private static NamedWriteableRegistry namedWriteableRegistry;
-
     private static String[] randomTypes;
-    private static ClientInvocationHandler clientInvocationHandler = new ClientInvocationHandler();
-    private static IndexSettings idxSettings;
-    private static SimilarityService similarityService;
-    private static MapperService mapperService;
-    private static BitsetFilterCache bitsetFilterCache;
-    private static ScriptService scriptService;
 
-    /**
-     * Setup for the whole base test class.
-     */
-    @BeforeClass
-    public static void init() throws IOException {
-        // we have to prefer CURRENT since with the range of versions we support it's rather unlikely to get the current actually.
-        indexVersionCreated = randomBoolean() ? Version.CURRENT
-                : VersionUtils.randomVersionBetween(random(), Version.V_2_0_0_beta1, Version.CURRENT);
-        Settings settings = Settings.builder()
-                .put("node.name", AbstractQueryTestCase.class.toString())
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-                .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
-                .build();
-        Settings indexSettings = Settings.builder()
-                .put(ParseFieldMatcher.PARSE_STRICT, true)
-                .put(IndexMetaData.SETTING_VERSION_CREATED, indexVersionCreated).build();
-        final ThreadPool threadPool = new ThreadPool(settings);
-        index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
-        idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings);
-        ClusterService clusterService = createClusterService(threadPool);
-        setState(clusterService, new ClusterState.Builder(clusterService.state()).metaData(new MetaData.Builder().put(
-                new IndexMetaData.Builder(index.getName()).settings(indexSettings).numberOfShards(1).numberOfReplicas(0))));
+    protected Index getIndex() {
+        return serviceHolder.index;
+    }
 
-        SettingsModule settingsModule = new SettingsModule(settings);
-        settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
-        final Client proxy = (Client) Proxy.newProxyInstance(
-                Client.class.getClassLoader(),
-                new Class[]{Client.class},
-                clientInvocationHandler);
-        namedWriteableRegistry = new NamedWriteableRegistry();
-        ScriptModule scriptModule = new ScriptModule() {
-            @Override
-            protected void configure() {
-                Settings settings = Settings.builder()
-                        .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-                        // no file watching, so we don't need a ResourceWatcherService
-                        .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
-                        .build();
-                MockScriptEngine mockScriptEngine = new MockScriptEngine();
-                Multibinder<ScriptEngineService> multibinder = Multibinder.newSetBinder(binder(), ScriptEngineService.class);
-                multibinder.addBinding().toInstance(mockScriptEngine);
-                Set<ScriptEngineService> engines = new HashSet<>();
-                engines.add(mockScriptEngine);
-                List<ScriptContext.Plugin> customContexts = new ArrayList<>();
-                ScriptEngineRegistry scriptEngineRegistry =
-                        new ScriptEngineRegistry(Collections
-                                .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
-                                                                                                 MockScriptEngine.NAME, true)));
-                bind(ScriptEngineRegistry.class).toInstance(scriptEngineRegistry);
-                ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts);
-                bind(ScriptContextRegistry.class).toInstance(scriptContextRegistry);
-                ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
-                bind(ScriptSettings.class).toInstance(scriptSettings);
-                try {
-                    ScriptService scriptService = new ScriptService(settings, new Environment(settings), engines, null,
-                            scriptEngineRegistry, scriptContextRegistry, scriptSettings);
-                    bind(ScriptService.class).toInstance(scriptService);
-                } catch (IOException e) {
-                    throw new IllegalStateException("error while binding ScriptService", e);
-                }
-            }
-        };
-        scriptModule.prepareSettings(settingsModule);
-        searchModule = new SearchModule(settings, namedWriteableRegistry) {
-            @Override
-            protected void configureSearch() {
-                // Skip me
-            }
-        };
-        injector = new ModulesBuilder().add(
-                new EnvironmentModule(new Environment(settings)),
-                settingsModule,
-                new ThreadPoolModule(threadPool),
-                new IndicesModule() {
-                    @Override
-                    public void configure() {
-                        // skip services
-                        bindMapperExtension();
-                    }
-                },
-                scriptModule,
-                new IndexSettingsModule(index, indexSettings),
-                searchModule,
-                new AbstractModule() {
-                    @Override
-                    protected void configure() {
-                        bind(Client.class).toInstance(proxy);
-                        bind(ClusterService.class).toProvider(Providers.of(clusterService));
-                        bind(CircuitBreakerService.class).to(NoneCircuitBreakerService.class);
-                        bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry);
-                    }
-                }
-        ).createInjector();
-        AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
-        scriptService = injector.getInstance(ScriptService.class);
-        similarityService = new SimilarityService(idxSettings, Collections.emptyMap());
-        MapperRegistry mapperRegistry = injector.getInstance(MapperRegistry.class);
-        mapperService = new MapperService(idxSettings, analysisService, similarityService, mapperRegistry,
-                () -> createShardContext());
-        IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() {
-        });
-        indexFieldDataService = new IndexFieldDataService(idxSettings, indicesFieldDataCache,
-                injector.getInstance(CircuitBreakerService.class), mapperService);
-        bitsetFilterCache = new BitsetFilterCache(idxSettings, new BitsetFilterCache.Listener() {
-            @Override
-            public void onCache(ShardId shardId, Accountable accountable) {
+    protected Version getIndexVersionCreated() {
+        return serviceHolder.indexVersionCreated;
+    }
 
-            }
+    protected String[] getCurrentTypes() {
+        return serviceHolder.currentTypes;
+    }
 
-            @Override
-            public void onRemoval(ShardId shardId, Accountable accountable) {
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return Collections.emptyList();
+    }
 
-            }
-        });
-        indicesQueriesRegistry = injector.getInstance(IndicesQueriesRegistry.class);
-        //create some random type with some default field, those types will stick around for all of the subclasses
-        currentTypes = new String[randomIntBetween(0, 5)];
-        for (int i = 0; i < currentTypes.length; i++) {
-            String type = randomAsciiOfLengthBetween(1, 10);
-            mapperService.merge(type, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(type,
-                    STRING_FIELD_NAME, "type=text",
-                    STRING_FIELD_NAME_2, "type=keyword",
-                    INT_FIELD_NAME, "type=integer",
-                    DOUBLE_FIELD_NAME, "type=double",
-                    BOOLEAN_FIELD_NAME, "type=boolean",
-                    DATE_FIELD_NAME, "type=date",
-                    OBJECT_FIELD_NAME, "type=object",
-                    GEO_POINT_FIELD_NAME, GEO_POINT_FIELD_MAPPING,
-                    GEO_SHAPE_FIELD_NAME, "type=geo_shape"
-            ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
-            // also add mappings for two inner field in the object field
-            mapperService.merge(type, new CompressedXContent("{\"properties\":{\"" + OBJECT_FIELD_NAME + "\":{\"type\":\"object\","
-                            + "\"properties\":{\"" + DATE_FIELD_NAME + "\":{\"type\":\"date\"},\"" + INT_FIELD_NAME + "\":{\"type\":\"integer\"}}}}}"),
-                    MapperService.MergeReason.MAPPING_UPDATE, false);
-            currentTypes[i] = type;
-        }
-        namedWriteableRegistry = injector.getInstance(NamedWriteableRegistry.class);
+    protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
     }
 
     @AfterClass
     public static void afterClass() throws Exception {
-        injector.getInstance(ClusterService.class).close();
-        terminate(injector.getInstance(ThreadPool.class));
-        injector = null;
-        index = null;
-        currentTypes = null;
-        namedWriteableRegistry = null;
-        randomTypes = null;
-        indicesQueriesRegistry = null;
-        indexFieldDataService = null;
-        searchModule = null;
-        idxSettings = null;
-        similarityService = null;
-        mapperService = null;
-        bitsetFilterCache = null;
-        scriptService = null;
+        IOUtils.close(serviceHolder);
+        serviceHolder = null;
     }
 
     @Before
-    public void beforeTest() {
-        clientInvocationHandler.delegate = this;
+    public void beforeTest() throws IOException {
+        if (serviceHolder == null) {
+            serviceHolder = new ServiceHolder(getPlugins(), this);
+        }
+
+        serviceHolder.clientInvocationHandler.delegate = this;
         //set some random types to be queried as part the search request, before each test
         randomTypes = getRandomTypes();
     }
@@ -360,12 +199,12 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
         TestSearchContext testSearchContext = new TestSearchContext(context) {
             @Override
             public MapperService mapperService() {
-                return mapperService; // need to build / parse inner hits sort fields
+                return serviceHolder.mapperService; // need to build / parse inner hits sort fields
             }
 
             @Override
             public IndexFieldDataService fieldData() {
-                return indexFieldDataService(); // need to build / parse inner hits sort fields
+                return serviceHolder.indexFieldDataService; // need to build / parse inner hits sort fields
             }
         };
         testSearchContext.getQueryShardContext().setTypes(types);
@@ -374,11 +213,11 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
 
     @After
     public void afterTest() {
-        clientInvocationHandler.delegate = null;
+        serviceHolder.clientInvocationHandler.delegate = null;
         SearchContext.removeCurrent();
     }
 
-    protected final QB createTestQueryBuilder() {
+    public final QB createTestQueryBuilder() {
         QB query = doCreateTestQueryBuilder();
         //we should not set boost and query name for queries that don't parse it
         if (supportsBoostAndQueryName()) {
@@ -683,7 +522,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
     protected <QB extends QueryBuilder> QB assertSerialization(QB testQuery) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             output.writeNamedWriteable(testQuery);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
+            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), serviceHolder.namedWriteableRegistry)) {
                 QueryBuilder deserializedQuery = in.readNamedWriteable(QueryBuilder.class);
                 assertEquals(testQuery, deserializedQuery);
                 assertEquals(testQuery.hashCode(), deserializedQuery.hashCode());
@@ -732,7 +571,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
     protected QB copyQuery(QB query) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             output.writeNamedWriteable(query);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
+            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), serviceHolder.namedWriteableRegistry)) {
                 return (QB) in.readNamedWriteable(QueryBuilder.class);
             }
         }
@@ -742,17 +581,14 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
      * @return a new {@link QueryShardContext} based on the base test index and queryParserService
      */
     protected static QueryShardContext createShardContext() {
-        ClusterState state = ClusterState.builder(new ClusterName("_name")).build();
-        Client client = injector.getInstance(Client.class);
-        return new QueryShardContext(idxSettings, bitsetFilterCache, indexFieldDataService, mapperService, similarityService,
-                scriptService, indicesQueriesRegistry, client, null, state);
+        return serviceHolder.createShardContext();
     }
 
     /**
      * @return a new {@link QueryParseContext} based on the base test index and queryParserService
      */
     protected static QueryParseContext createParseContext(XContentParser parser, ParseFieldMatcher matcher) {
-        QueryParseContext queryParseContext = new QueryParseContext(indicesQueriesRegistry, parser, matcher);
+        QueryParseContext queryParseContext = new QueryParseContext(serviceHolder.indicesQueriesRegistry, parser, matcher);
         return queryParseContext;
     }
 
@@ -805,7 +641,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
      */
     protected String getRandomFieldName() {
         // if no type is set then return a random field name
-        if (currentTypes == null || currentTypes.length == 0 || randomBoolean()) {
+        if (serviceHolder.currentTypes == null || serviceHolder.currentTypes.length == 0 || randomBoolean()) {
             return randomAsciiOfLengthBetween(1, 10);
         }
         return randomFrom(MAPPED_LEAF_FIELD_NAMES);
@@ -838,11 +674,11 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
 
     protected String[] getRandomTypes() {
         String[] types;
-        if (currentTypes.length > 0 && randomBoolean()) {
-            int numberOfQueryTypes = randomIntBetween(1, currentTypes.length);
+        if (serviceHolder.currentTypes.length > 0 && randomBoolean()) {
+            int numberOfQueryTypes = randomIntBetween(1, serviceHolder.currentTypes.length);
             types = new String[numberOfQueryTypes];
             for (int i = 0; i < numberOfQueryTypes; i++) {
-                types[i] = randomFrom(currentTypes);
+                types[i] = randomFrom(serviceHolder.currentTypes);
             }
         } else {
             if (randomBoolean()) {
@@ -855,7 +691,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
     }
 
     protected String getRandomType() {
-        return (currentTypes.length == 0) ? MetaData.ALL : randomFrom(currentTypes);
+        return (serviceHolder.currentTypes.length == 0) ? MetaData.ALL : randomFrom(serviceHolder.currentTypes);
     }
 
     public static String randomGeohash(int minPrecision, int maxPrecision) {
@@ -1006,4 +842,185 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
         return query;
     }
 
+    static class ServiceHolder implements Closeable {
+
+        private final Injector injector;
+        private final IndicesQueriesRegistry indicesQueriesRegistry;
+        private final IndexFieldDataService indexFieldDataService;
+        private final SearchModule searchModule;
+        private final Index index;
+        private final Version indexVersionCreated;
+        private final String[] currentTypes;
+        private final NamedWriteableRegistry namedWriteableRegistry;
+        private final ClientInvocationHandler clientInvocationHandler = new ClientInvocationHandler();
+        private final IndexSettings idxSettings;
+        private final SimilarityService similarityService;
+        private final MapperService mapperService;
+        private final BitsetFilterCache bitsetFilterCache;
+        private final ScriptService scriptService;
+
+        public ServiceHolder(Collection<Class<? extends Plugin>> plugins, AbstractQueryTestCase<?> testCase) throws IOException {
+            // we have to prefer CURRENT since with the range of versions we support it's rather unlikely to get the current actually.
+            indexVersionCreated = randomBoolean() ? Version.CURRENT
+                    : VersionUtils.randomVersionBetween(random(), Version.V_2_0_0_beta1, Version.CURRENT);
+            Settings settings = Settings.builder()
+                    .put("node.name", AbstractQueryTestCase.class.toString())
+                    .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+                    .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
+                    .build();
+            Settings indexSettings = Settings.builder()
+                    .put(ParseFieldMatcher.PARSE_STRICT, true)
+                    .put(IndexMetaData.SETTING_VERSION_CREATED, indexVersionCreated).build();
+            final ThreadPool threadPool = new ThreadPool(settings);
+            index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
+            ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool);
+            ClusterServiceUtils.setState(clusterService, new ClusterState.Builder(clusterService.state()).metaData(new MetaData.Builder().put(
+                    new IndexMetaData.Builder(index.getName()).settings(indexSettings).numberOfShards(1).numberOfReplicas(0))));
+            Environment env = InternalSettingsPreparer.prepareEnvironment(settings, null);
+            PluginsService pluginsService =new PluginsService(settings, env.modulesFile(), env.pluginsFile(), plugins);
+
+            SettingsModule settingsModule = new SettingsModule(settings);
+            settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
+            final Client proxy = (Client) Proxy.newProxyInstance(
+                    Client.class.getClassLoader(),
+                    new Class[]{Client.class},
+                    clientInvocationHandler);
+            NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
+            ScriptModule scriptModule = new ScriptModule() {
+                @Override
+                protected void configure() {
+                    Settings settings = Settings.builder()
+                            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+                            // no file watching, so we don't need a ResourceWatcherService
+                            .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
+                            .build();
+                    MockScriptEngine mockScriptEngine = new MockScriptEngine();
+                    Multibinder<ScriptEngineService> multibinder = Multibinder.newSetBinder(binder(), ScriptEngineService.class);
+                    multibinder.addBinding().toInstance(mockScriptEngine);
+                    Set<ScriptEngineService> engines = new HashSet<>();
+                    engines.add(mockScriptEngine);
+                    List<ScriptContext.Plugin> customContexts = new ArrayList<>();
+                    ScriptEngineRegistry scriptEngineRegistry =
+                            new ScriptEngineRegistry(Collections
+                                    .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
+                                            MockScriptEngine.NAME, true)));
+                    bind(ScriptEngineRegistry.class).toInstance(scriptEngineRegistry);
+                    ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts);
+                    bind(ScriptContextRegistry.class).toInstance(scriptContextRegistry);
+                    ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
+                    bind(ScriptSettings.class).toInstance(scriptSettings);
+                    try {
+                        ScriptService scriptService = new ScriptService(settings, new Environment(settings), engines, null,
+                                scriptEngineRegistry, scriptContextRegistry, scriptSettings);
+                        bind(ScriptService.class).toInstance(scriptService);
+                    } catch (IOException e) {
+                        throw new IllegalStateException("error while binding ScriptService", e);
+                    }
+                }
+            };
+            scriptModule.prepareSettings(settingsModule);
+            searchModule = new SearchModule(settings, namedWriteableRegistry) {
+                @Override
+                protected void configureSearch() {
+                    // Skip me
+                }
+            };
+            ModulesBuilder modulesBuilder = new ModulesBuilder();
+            for (Module pluginModule : pluginsService.nodeModules()) {
+                modulesBuilder.add(pluginModule);
+            }
+            modulesBuilder.add(new PluginsModule(pluginsService));
+            modulesBuilder.add(
+                    new EnvironmentModule(new Environment(settings)),
+                    settingsModule,
+                    new ThreadPoolModule(threadPool),
+                    new IndicesModule() {
+                        @Override
+                        public void configure() {
+                            // skip services
+                            bindMapperExtension();
+                        }
+                    },
+                    scriptModule,
+                    new IndexSettingsModule(index, indexSettings),
+                    searchModule,
+                    new AbstractModule() {
+                        @Override
+                        protected void configure() {
+                            bind(Client.class).toInstance(proxy);
+                            bind(ClusterService.class).toProvider(Providers.of(clusterService));
+                            bind(CircuitBreakerService.class).to(NoneCircuitBreakerService.class);
+                            bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry);
+                        }
+                    }
+            );
+            pluginsService.processModules(modulesBuilder);
+            injector = modulesBuilder.createInjector();
+            IndexScopedSettings indexScopedSettings = injector.getInstance(IndexScopedSettings.class);
+            idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings, indexScopedSettings);
+            AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
+            scriptService = injector.getInstance(ScriptService.class);
+            similarityService = new SimilarityService(idxSettings, Collections.emptyMap());
+            MapperRegistry mapperRegistry = injector.getInstance(MapperRegistry.class);
+            mapperService = new MapperService(idxSettings, analysisService, similarityService, mapperRegistry,
+                    () -> createShardContext());
+            IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() {
+            });
+            indexFieldDataService = new IndexFieldDataService(idxSettings, indicesFieldDataCache,
+                    injector.getInstance(CircuitBreakerService.class), mapperService);
+            bitsetFilterCache = new BitsetFilterCache(idxSettings, new BitsetFilterCache.Listener() {
+                @Override
+                public void onCache(ShardId shardId, Accountable accountable) {
+
+                }
+
+                @Override
+                public void onRemoval(ShardId shardId, Accountable accountable) {
+
+                }
+            });
+            indicesQueriesRegistry = injector.getInstance(IndicesQueriesRegistry.class);
+            //create some random type with some default field, those types will stick around for all of the subclasses
+            currentTypes = new String[randomIntBetween(0, 5)];
+            for (int i = 0; i < currentTypes.length; i++) {
+                String type = randomAsciiOfLengthBetween(1, 10);
+                mapperService.merge(type, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(type,
+                        STRING_FIELD_NAME, "type=text",
+                        STRING_FIELD_NAME_2, "type=keyword",
+                        INT_FIELD_NAME, "type=integer",
+                        DOUBLE_FIELD_NAME, "type=double",
+                        BOOLEAN_FIELD_NAME, "type=boolean",
+                        DATE_FIELD_NAME, "type=date",
+                        OBJECT_FIELD_NAME, "type=object",
+                        GEO_POINT_FIELD_NAME, GEO_POINT_FIELD_MAPPING,
+                        GEO_SHAPE_FIELD_NAME, "type=geo_shape"
+                ).string()), MapperService.MergeReason.MAPPING_UPDATE, false);
+                // also add mappings for two inner field in the object field
+                mapperService.merge(type, new CompressedXContent("{\"properties\":{\"" + OBJECT_FIELD_NAME + "\":{\"type\":\"object\","
+                                + "\"properties\":{\"" + DATE_FIELD_NAME + "\":{\"type\":\"date\"},\"" + INT_FIELD_NAME + "\":{\"type\":\"integer\"}}}}}"),
+                        MapperService.MergeReason.MAPPING_UPDATE, false);
+                currentTypes[i] = type;
+            }
+            testCase.initializeAdditionalMappings(mapperService);
+            this.namedWriteableRegistry = injector.getInstance(NamedWriteableRegistry.class);
+        }
+
+        public void close() throws IOException {
+            injector.getInstance(ClusterService.class).close();
+            try {
+                terminate(injector.getInstance(ThreadPool.class));
+            } catch (InterruptedException e) {
+                IOUtils.reThrow(e);
+            }
+        }
+
+        QueryShardContext createShardContext() {
+            ClusterState state = ClusterState.builder(new ClusterName("_name")).build();
+            Client client = injector.getInstance(Client.class);
+            return new QueryShardContext(idxSettings, bitsetFilterCache, indexFieldDataService, mapperService, similarityService,
+                    scriptService, indicesQueriesRegistry, client, null, state);
+        }
+
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceUtils.java b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
similarity index 98%
rename from core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceUtils.java
rename to test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
index e3f132b18fb..3e542231001 100644
--- a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.cluster.service;
+package org.elasticsearch.test;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
@@ -27,6 +27,7 @@ import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.NodeConnectionsService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.DummyTransportAddress;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/IndexSettingsModule.java b/test/framework/src/main/java/org/elasticsearch/test/IndexSettingsModule.java
index ad365d42850..789eb693f7f 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/IndexSettingsModule.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/IndexSettingsModule.java
@@ -66,6 +66,16 @@ public class IndexSettingsModule extends AbstractModule {
         return new IndexSettings(metaData, Settings.EMPTY, (idx) -> Regex.simpleMatch(idx, metaData.getIndex().getName()), new IndexScopedSettings(Settings.EMPTY, settingSet));
     }
 
+    public static IndexSettings newIndexSettings(Index index, Settings settings, IndexScopedSettings indexScopedSettings) {
+        Settings build = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(settings)
+                .build();
+        IndexMetaData metaData = IndexMetaData.builder(index.getName()).settings(build).build();
+        return new IndexSettings(metaData, Settings.EMPTY, (idx) -> Regex.simpleMatch(idx, metaData.getIndex().getName()), indexScopedSettings);
+    }
+
     public static IndexSettings newIndexSettings(final IndexMetaData indexMetaData, Setting<?>... setting) {
         Set<Setting<?>> settingSet = new HashSet<>(IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
         if (setting.length > 0) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
index c0809290177..d10c113b590 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
@@ -29,15 +29,12 @@ import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
-import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
 import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.action.percolate.PercolateResponse;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
@@ -57,7 +54,6 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.plugins.PluginInfo;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchModule;
@@ -68,7 +64,6 @@ import org.elasticsearch.test.rest.client.http.HttpResponse;
 import org.hamcrest.CoreMatchers;
 import org.hamcrest.Matcher;
 import org.hamcrest.Matchers;
-import org.junit.Assert;
 
 import java.io.IOException;
 import java.lang.reflect.Constructor;
@@ -81,9 +76,6 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Set;
-import java.util.function.Function;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
 
 import static org.apache.lucene.util.LuceneTestCase.random;
 import static org.elasticsearch.test.VersionUtils.randomVersion;
@@ -253,13 +245,6 @@ public class ElasticsearchAssertions {
         assertVersionSerializable(countResponse);
     }
 
-    public static void assertMatchCount(PercolateResponse percolateResponse, long expectedHitCount) {
-        if (percolateResponse.getCount() != expectedHitCount) {
-            fail("Count is " + percolateResponse.getCount() + " but " + expectedHitCount + " was expected. " + formatShardStatus(percolateResponse));
-        }
-        assertVersionSerializable(percolateResponse);
-    }
-
     public static void assertExists(GetResponse response) {
         String message = String.format(Locale.ROOT, "Expected %s/%s/%s to exist, but does not", response.getIndex(), response.getType(), response.getId());
         assertThat(message, response.isExists(), is(true));
@@ -334,12 +319,6 @@ public class ElasticsearchAssertions {
         }
     }
 
-    public static void assertFailures(PercolateResponse percolateResponse) {
-        assertThat("Expected at least one shard failure, got none",
-            percolateResponse.getShardFailures().length, greaterThan(0));
-        assertVersionSerializable(percolateResponse);
-    }
-
     public static void assertNoFailures(BroadcastResponse response) {
         assertThat("Unexpected ShardFailures: " + Arrays.toString(response.getShardFailures()), response.getFailedShards(), equalTo(0));
         assertVersionSerializable(response);

From 4c02e97bcdeedf518cfe35a4c02a0c7bad884591 Mon Sep 17 00:00:00 2001
From: Isabel Drost-Fromm <isabel.drostfromm@elasticsearch.com>
Date: Tue, 24 May 2016 11:58:43 +0200
Subject: [PATCH 0435/1311] Add back doc execution to query dsl.

Relates to #18211

This reverts commit 20aafb1196192d4f9f7faea8ce9a36b278e501a1.
---
 docs/reference/query-dsl/bool-query.asciidoc  |   6 +-
 .../query-dsl/boosting-query.asciidoc         |  28 ++-
 .../query-dsl/common-terms-query.asciidoc     | 226 ++++++++++--------
 .../query-dsl/constant-score-query.asciidoc   |  14 +-
 .../query-dsl/dis-max-query.asciidoc          |  26 +-
 .../reference/query-dsl/exists-query.asciidoc |  20 +-
 .../query-dsl/function-score-query.asciidoc   | 114 +++++----
 docs/reference/query-dsl/fuzzy-query.asciidoc |  25 +-
 .../query-dsl/geo-bounding-box-query.asciidoc | 214 ++++++++++-------
 .../query-dsl/geo-distance-query.asciidoc     | 133 +++++++----
 .../geo-distance-range-query.asciidoc         |  26 +-
 .../query-dsl/geo-polygon-query.asciidoc      | 104 ++++----
 .../query-dsl/geo-shape-query.asciidoc        |  40 ++--
 .../query-dsl/geohash-cell-query.asciidoc     |  31 ++-
 .../query-dsl/has-child-query.asciidoc        |  54 +++--
 .../query-dsl/has-parent-query.asciidoc       |  34 ++-
 docs/reference/query-dsl/ids-query.asciidoc   |  10 +-
 .../query-dsl/indices-query.asciidoc          |  14 +-
 .../query-dsl/match-all-query.asciidoc        |  24 +-
 .../match-phrase-prefix-query.asciidoc        |  22 +-
 .../query-dsl/match-phrase-query.asciidoc     |  20 +-
 docs/reference/query-dsl/match-query.asciidoc |  47 ++--
 docs/reference/query-dsl/mlt-query.asciidoc   | 142 ++++++-----
 .../query-dsl/multi-match-query.asciidoc      | 212 ++++++++++------
 .../reference/query-dsl/nested-query.asciidoc |  39 +--
 .../query-dsl/parent-id-query.asciidoc        |  14 +-
 .../query-dsl/percolate-query.asciidoc        |  91 ++++---
 .../reference/query-dsl/prefix-query.asciidoc |  15 +-
 .../query-dsl/query-string-query.asciidoc     |  64 +++--
 .../query-dsl/query_filter_context.asciidoc   |   3 +-
 docs/reference/query-dsl/range-query.asciidoc |  54 +++--
 .../reference/query-dsl/regexp-query.asciidoc |  47 ++--
 .../reference/query-dsl/script-query.asciidoc |  40 ++--
 .../simple-query-string-query.asciidoc        |  24 +-
 .../query-dsl/span-containing-query.asciidoc  |  28 ++-
 .../query-dsl/span-first-query.asciidoc       |  14 +-
 .../query-dsl/span-multi-term-query.asciidoc  |  20 +-
 .../query-dsl/span-near-query.asciidoc        |  20 +-
 .../query-dsl/span-not-query.asciidoc         |  28 ++-
 .../query-dsl/span-or-query.asciidoc          |  16 +-
 .../query-dsl/span-term-query.asciidoc        |  18 +-
 .../query-dsl/span-within-query.asciidoc      |  28 ++-
 .../query-dsl/template-query.asciidoc         |  15 +-
 docs/reference/query-dsl/terms-query.asciidoc |  54 +++--
 docs/reference/query-dsl/type-query.asciidoc  |   8 +-
 .../query-dsl/wildcard-query.asciidoc         |  18 +-
 46 files changed, 1391 insertions(+), 853 deletions(-)

diff --git a/docs/reference/query-dsl/bool-query.asciidoc b/docs/reference/query-dsl/bool-query.asciidoc
index 8fcd66129cd..2a289910c43 100644
--- a/docs/reference/query-dsl/bool-query.asciidoc
+++ b/docs/reference/query-dsl/bool-query.asciidoc
@@ -81,7 +81,7 @@ all documents where the `status` field contains the term `active`.
 This first query assigns a score of `0` to all documents, as no scoring
 query has been specified:
 
-[source,json]
+[source,js]
 ---------------------------------
 GET _search
 {
@@ -101,7 +101,7 @@ GET _search
 This `bool` query has a `match_all` query, which assigns a score of `1.0` to
 all documents.
 
-[source,json]
+[source,js]
 ---------------------------------
 GET _search
 {
@@ -125,7 +125,7 @@ This `constant_score` query behaves in exactly the same way as the second exampl
 The `constant_score` query assigns a score of `1.0` to all documents matched
 by the filter.
 
-[source,json]
+[source,js]
 ---------------------------------
 GET _search
 {
diff --git a/docs/reference/query-dsl/boosting-query.asciidoc b/docs/reference/query-dsl/boosting-query.asciidoc
index 969b3bbedfe..5bb07392ab7 100644
--- a/docs/reference/query-dsl/boosting-query.asciidoc
+++ b/docs/reference/query-dsl/boosting-query.asciidoc
@@ -8,19 +8,23 @@ overall score.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "boosting" : {
-        "positive" : {
-            "term" : {
-                "field1" : "value1"
-            }
-        },
-        "negative" : {
-            "term" : {
-                "field2" : "value2"
-            }
-        },
-        "negative_boost" : 0.2
+    "query": {
+        "boosting" : {
+            "positive" : {
+                "term" : {
+                    "field1" : "value1"
+                }
+            },
+            "negative" : {
+                 "term" : {
+                     "field2" : "value2"
+                }
+            },
+            "negative_boost" : 0.2
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/query-dsl/common-terms-query.asciidoc b/docs/reference/query-dsl/common-terms-query.asciidoc
index a956c33c1ee..fcc4ace2ec6 100644
--- a/docs/reference/query-dsl/common-terms-query.asciidoc
+++ b/docs/reference/query-dsl/common-terms-query.asciidoc
@@ -70,15 +70,19 @@ In this example, words that have a document frequency greater than 0.1%
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "common": {
-    "body": {
-      "query": "this is bonsai cool",
-      "cutoff_frequency": 0.001
+    "query": {
+        "common": {
+            "body": {
+                "query": "this is bonsai cool",
+                    "cutoff_frequency": 0.001
+            }
+        }
     }
-  }
 }
 --------------------------------------------------
+// CONSOLE
 
 The number of terms which should match can be controlled with the
 <<query-dsl-minimum-should-match,`minimum_should_match`>>
@@ -90,36 +94,44 @@ all terms required:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "common": {
-    "body": {
-      "query": "nelly the elephant as a cartoon",
-      "cutoff_frequency": 0.001,
-      "low_freq_operator": "and"
+    "query": {
+        "common": {
+            "body": {
+                "query": "nelly the elephant as a cartoon",
+                    "cutoff_frequency": 0.001,
+                    "low_freq_operator": "and"
+            }
+        }
     }
-  }
 }
 --------------------------------------------------
+// CONSOLE
 
 which is roughly equivalent to:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "bool": {
-    "must": [
-      { "term": { "body": "nelly"}},
-      { "term": { "body": "elephant"}},
-      { "term": { "body": "cartoon"}}
-    ],
-    "should": [
-      { "term": { "body": "the"}}
-      { "term": { "body": "as"}}
-      { "term": { "body": "a"}}
-    ]
-  }
+    "query": {
+        "bool": {
+            "must": [
+            { "term": { "body": "nelly"}},
+            { "term": { "body": "elephant"}},
+            { "term": { "body": "cartoon"}}
+            ],
+            "should": [
+            { "term": { "body": "the"}},
+            { "term": { "body": "as"}},
+            { "term": { "body": "a"}}
+            ]
+        }
+    }
 }
 --------------------------------------------------
+// CONSOLE
 
 Alternatively use
 <<query-dsl-minimum-should-match,`minimum_should_match`>>
@@ -128,41 +140,49 @@ must be present, for instance:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "common": {
-    "body": {
-      "query": "nelly the elephant as a cartoon",
-      "cutoff_frequency": 0.001,
-      "minimum_should_match": 2
+    "query": {
+        "common": {
+            "body": {
+                "query": "nelly the elephant as a cartoon",
+                "cutoff_frequency": 0.001,
+                "minimum_should_match": 2
+            }
+        }
     }
-  }
 }
 --------------------------------------------------
+// CONSOLE
 
 which is roughly equivalent to:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "bool": {
-    "must": {
-      "bool": {
-        "should": [
-          { "term": { "body": "nelly"}},
-          { "term": { "body": "elephant"}},
-          { "term": { "body": "cartoon"}}
-        ],
-        "minimum_should_match": 2
-      }
-    },
-    "should": [
-      { "term": { "body": "the"}}
-      { "term": { "body": "as"}}
-      { "term": { "body": "a"}}
-    ]
-  }
+    "query": {
+        "bool": {
+            "must": {
+                "bool": {
+                    "should": [
+                    { "term": { "body": "nelly"}},
+                    { "term": { "body": "elephant"}},
+                    { "term": { "body": "cartoon"}}
+                    ],
+                    "minimum_should_match": 2
+                }
+            },
+            "should": [
+                { "term": { "body": "the"}},
+                { "term": { "body": "as"}},
+                { "term": { "body": "a"}}
+                ]
+        }
+    }
 }
 --------------------------------------------------
+// CONSOLE
 
 minimum_should_match
 
@@ -174,50 +194,58 @@ additional parameters (note the change in structure):
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "common": {
-    "body": {
-      "query": "nelly the elephant not as a cartoon",
-      "cutoff_frequency": 0.001,
-      "minimum_should_match": {
-          "low_freq" : 2,
-          "high_freq" : 3
-       }
+    "query": {
+        "common": {
+            "body": {
+                "query": "nelly the elephant not as a cartoon",
+                    "cutoff_frequency": 0.001,
+                    "minimum_should_match": {
+                        "low_freq" : 2,
+                        "high_freq" : 3
+                    }
+            }
+        }
     }
-  }
 }
 --------------------------------------------------
+// CONSOLE
 
 which is roughly equivalent to:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "bool": {
-    "must": {
-      "bool": {
-        "should": [
-          { "term": { "body": "nelly"}},
-          { "term": { "body": "elephant"}},
-          { "term": { "body": "cartoon"}}
-        ],
-        "minimum_should_match": 2
-      }
-    },
-    "should": {
-      "bool": {
-        "should": [
-          { "term": { "body": "the"}},
-          { "term": { "body": "not"}},
-          { "term": { "body": "as"}},
-          { "term": { "body": "a"}}
-        ],
-        "minimum_should_match": 3
-      }
+    "query": {
+        "bool": {
+            "must": {
+                "bool": {
+                    "should": [
+                    { "term": { "body": "nelly"}},
+                    { "term": { "body": "elephant"}},
+                    { "term": { "body": "cartoon"}}
+                    ],
+                    "minimum_should_match": 2
+                }
+            },
+            "should": {
+                "bool": {
+                    "should": [
+                    { "term": { "body": "the"}},
+                    { "term": { "body": "not"}},
+                    { "term": { "body": "as"}},
+                    { "term": { "body": "a"}}
+                    ],
+                    "minimum_should_match": 3
+                }
+            }
+        }
     }
-  }
 }
 --------------------------------------------------
+// CONSOLE
 
 In this case it means the high frequency terms have only an impact on
 relevance when there are at least three of them. But the most
@@ -227,36 +255,44 @@ for high frequency terms is when there are only high frequency terms:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "common": {
-    "body": {
-      "query": "how not to be",
-      "cutoff_frequency": 0.001,
-      "minimum_should_match": {
-          "low_freq" : 2,
-          "high_freq" : 3
-       }
+    "query": {
+        "common": {
+            "body": {
+                "query": "how not to be",
+                    "cutoff_frequency": 0.001,
+                    "minimum_should_match": {
+                        "low_freq" : 2,
+                        "high_freq" : 3
+                    }
+            }
+        }
     }
-  }
 }
 --------------------------------------------------
+// CONSOLE
 
 which is roughly equivalent to:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "bool": {
-    "should": [
-      { "term": { "body": "how"}},
-      { "term": { "body": "not"}},
-      { "term": { "body": "to"}},
-      { "term": { "body": "be"}}
-    ],
-    "minimum_should_match": "3<50%"
-  }
+    "query": {
+        "bool": {
+            "should": [
+            { "term": { "body": "how"}},
+            { "term": { "body": "not"}},
+            { "term": { "body": "to"}},
+            { "term": { "body": "be"}}
+            ],
+            "minimum_should_match": "3<50%"
+        }
+    }
 }
 --------------------------------------------------
+// CONSOLE
 
 The high frequency generated query is then slightly less restrictive
 than with an `AND`.
diff --git a/docs/reference/query-dsl/constant-score-query.asciidoc b/docs/reference/query-dsl/constant-score-query.asciidoc
index 8e76ac13ff5..bced9fc9fbe 100644
--- a/docs/reference/query-dsl/constant-score-query.asciidoc
+++ b/docs/reference/query-dsl/constant-score-query.asciidoc
@@ -7,12 +7,16 @@ filter. Maps to Lucene `ConstantScoreQuery`.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "constant_score" : {
-        "filter" : {
-            "term" : { "user" : "kimchy"}
-        },
-        "boost" : 1.2
+    "query": {
+        "constant_score" : {
+            "filter" : {
+                "term" : { "user" : "kimchy"}
+            },
+            "boost" : 1.2
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/query-dsl/dis-max-query.asciidoc b/docs/reference/query-dsl/dis-max-query.asciidoc
index 2938c8db8ea..2f82f2294b3 100644
--- a/docs/reference/query-dsl/dis-max-query.asciidoc
+++ b/docs/reference/query-dsl/dis-max-query.asciidoc
@@ -27,18 +27,22 @@ This query maps to Lucene `DisjunctionMaxQuery`.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "dis_max" : {
-        "tie_breaker" : 0.7,
-        "boost" : 1.2,
-        "queries" : [
-            {
-                "term" : { "age" : 34 }
-            },
-            {
-                "term" : { "age" : 35 }
-            }
-        ]
+    "query": {
+        "dis_max" : {
+            "tie_breaker" : 0.7,
+                "boost" : 1.2,
+                "queries" : [
+                {
+                    "term" : { "age" : 34 }
+                },
+                {
+                    "term" : { "age" : 35 }
+                }
+                ]
+        }
     }
 }    
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/query-dsl/exists-query.asciidoc b/docs/reference/query-dsl/exists-query.asciidoc
index b484d47f4b6..4971219366f 100644
--- a/docs/reference/query-dsl/exists-query.asciidoc
+++ b/docs/reference/query-dsl/exists-query.asciidoc
@@ -5,10 +5,14 @@ Returns documents that have at least one non-`null` value in the original field:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "exists" : { "field" : "user" }
+    "query": {
+        "exists" : { "field" : "user" }
+    }
 }
 --------------------------------------------------
+// CONSOLE
 
 For instance, these documents would all match the above query:
 
@@ -77,14 +81,20 @@ clause as follows:
 
 [source,js]
 --------------------------------------------------
-"bool": {
-    "must_not": {
-        "exists": {
-            "field": "user"
+GET /_search
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "exists": {
+                    "field": "user"
+                }
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 This query returns documents that have no value in the user field.
 
diff --git a/docs/reference/query-dsl/function-score-query.asciidoc b/docs/reference/query-dsl/function-score-query.asciidoc
index 5f92d8c172c..e7e4b8f5877 100644
--- a/docs/reference/query-dsl/function-score-query.asciidoc
+++ b/docs/reference/query-dsl/function-score-query.asciidoc
@@ -14,13 +14,20 @@ by the query.
 
 [source,js]
 --------------------------------------------------
-"function_score": {
-    "query": {},
-    "boost": "boost for the whole query",
-    "FUNCTION": {}, <1>
-    "boost_mode":"(multiply|replace|...)"
+GET /_search
+{
+    "query": {
+        "function_score": {
+            "query": {},
+            "boost": "5",
+            "random_score": {}, <1>
+            "boost_mode":"multiply"
+        }
+    }
 }
 --------------------------------------------------
+// CONSOLE
+
 <1> See <<score-functions>> for a list of supported functions.
 
 Furthermore, several functions can be combined. In this case one can
@@ -29,30 +36,35 @@ given filtering query
 
 [source,js]
 --------------------------------------------------
-"function_score": {
-    "query": {},
-    "boost": "boost for the whole query",
-    "functions": [
-        {
-            "filter": {},
-            "FUNCTION": {}, <1>
-            "weight": number
-        },
-        {
-            "FUNCTION": {} <1>
-        },
-        {
-            "filter": {},
-            "weight": number
+GET /_search
+{
+    "query": {
+        "function_score": {
+          "query": {},
+          "boost": "5", <1>
+          "functions": [
+              {
+                  "filter": {},
+                  "random_score": {}, <2>
+                  "weight": 23
+              },
+              {
+                  "filter": {},
+                  "weight": 42 
+              }
+          ],
+          "max_boost": 42,
+          "score_mode": "max",
+          "boost_mode": "multiply",
+          "min_score" : 42
         }
-    ],
-    "max_boost": number,
-    "score_mode": "(multiply|max|...)",
-    "boost_mode": "(multiply|replace|...)",
-    "min_score" : number
+    }
 }
 --------------------------------------------------
-<1> See <<score-functions>> for a list of supported functions.
+// CONSOLE
+
+<1> Boost for the whole query.
+<2> See <<score-functions>> for a list of supported functions.
 
 NOTE: The scores produced by the filtering query of each function do not matter.
 
@@ -459,36 +471,36 @@ the request would look like this:
 
 [source,js]
 --------------------------------------------------
-GET _search
+GET /_search
 {
-  "query": {
-    "function_score": {
-      "functions": [
-        {
-          "gauss": {
-            "price": {
-              "origin": "0",
-              "scale": "20"
+    "query": {
+        "function_score": {
+          "functions": [
+            {
+              "gauss": {
+                "price": {
+                  "origin": "0",
+                  "scale": "20"
+                }
+              }
+            },
+            {
+              "gauss": {
+                "location": {
+                  "origin": "11, 12",
+                  "scale": "2km"
+                }
+              }
             }
-          }
-        },
-        {
-          "gauss": {
-            "location": {
-              "origin": "11, 12",
-              "scale": "2km"
+          ],
+          "query": {
+            "match": {
+              "properties": "balcony"
             }
-          }
+          },
+          "score_mode": "multiply"
         }
-      ],
-      "query": {
-        "match": {
-          "properties": "balcony"
-        }
-      },
-      "score_mode": "multiply"
     }
-  }
 }
 --------------------------------------------------
 // CONSOLE
diff --git a/docs/reference/query-dsl/fuzzy-query.asciidoc b/docs/reference/query-dsl/fuzzy-query.asciidoc
index 48c9c45c0ff..f320e81b579 100644
--- a/docs/reference/query-dsl/fuzzy-query.asciidoc
+++ b/docs/reference/query-dsl/fuzzy-query.asciidoc
@@ -16,27 +16,35 @@ Here is a simple example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "fuzzy" : { "user" : "ki" }
+    "query": {
+       "fuzzy" : { "user" : "ki" }
+    }
 }
 --------------------------------------------------
+// CONSOLE
 
 Or with more advanced settings:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "fuzzy" : {
-        "user" : {
-            "value" :         "ki",
-            "boost" :         1.0,
-            "fuzziness" :     2,
-            "prefix_length" : 0,
-            "max_expansions": 100
+    "query": {
+        "fuzzy" : {
+            "user" : {
+                "value" :         "ki",
+                    "boost" :         1.0,
+                    "fuzziness" :     2,
+                    "prefix_length" : 0,
+                    "max_expansions": 100
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ===== Parameters
@@ -62,3 +70,4 @@ WARNING: This query can be very heavy if `prefix_length` is set to `0` and if
 `max_expansions` is set to a high number. It could result in every term in the
 index being examined!
 
+
diff --git a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
index 00120a98fc9..40debb57105 100644
--- a/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
+++ b/docs/reference/query-dsl/geo-bounding-box-query.asciidoc
@@ -6,6 +6,24 @@ bounding box. Assuming the following indexed document:
 
 [source,js]
 --------------------------------------------------
+PUT /my_locations
+{
+    "mappings": {
+        "location": {
+            "properties": {
+                "pin": {
+                    "properties": {
+                        "location": {
+                            "type": "geo_point"
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+PUT /my_locations/location/1
 {
     "pin" : {
         "location" : {
@@ -15,27 +33,32 @@ bounding box. Assuming the following indexed document:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TESTSETUP
 
 Then the following simple query can be executed with a
 `geo_bounding_box` filter:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_bounding_box" : {
-                "pin.location" : {
-                    "top_left" : {
-                        "lat" : 40.73,
-                        "lon" : -74.1
-                    },
-                    "bottom_right" : {
-                        "lat" : 40.01,
-                        "lon" : -71.12
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_bounding_box" : {
+                    "pin.location" : {
+                        "top_left" : {
+                            "lat" : 40.73,
+                            "lon" : -74.1
+                        },
+                        "bottom_right" : {
+                            "lat" : 40.01,
+                            "lon" : -71.12
+                        }
                     }
                 }
             }
@@ -43,6 +66,7 @@ Then the following simple query can be executed with a
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== Query Options
@@ -75,21 +99,24 @@ representation of the geo point, the filter can accept it as well:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_bounding_box" : {
-                "pin.location" : {
-                    "top_left" : {
-                        "lat" : 40.73,
-                        "lon" : -74.1
-                    },
-                    "bottom_right" : {
-                        "lat" : 40.01,
-                        "lon" : -71.12
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_bounding_box" : {
+                    "pin.location" : {
+                        "top_left" : {
+                            "lat" : 40.73,
+                            "lon" : -74.1
+                        },
+                        "bottom_right" : {
+                            "lat" : 40.01,
+                            "lon" : -71.12
+                        }
                     }
                 }
             }
@@ -97,6 +124,7 @@ representation of the geo point, the filter can accept it as well:
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ===== Lat Lon As Array
@@ -106,22 +134,26 @@ conform with http://geojson.org/[GeoJSON].
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_bounding_box" : {
-                "pin.location" : {
-                    "top_left" : [-74.1, 40.73],
-                    "bottom_right" : [-71.12, 40.01]
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_bounding_box" : {
+                    "pin.location" : {
+                        "top_left" : [-74.1, 40.73],
+                        "bottom_right" : [-71.12, 40.01]
+                    }
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ===== Lat Lon As String
@@ -130,44 +162,52 @@ Format in `lat,lon`.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_bounding_box" : {
-                "pin.location" : {
-                    "top_left" : "40.73, -74.1",
-                    "bottom_right" : "40.01, -71.12"
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_bounding_box" : {
+                    "pin.location" : {
+                        "top_left" : "40.73, -74.1",
+                        "bottom_right" : "40.01, -71.12"
+                    }
                 }
             }
-        }
     }
 }
+}
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ===== Geohash
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_bounding_box" : {
-                "pin.location" : {
-                    "top_left" : "dr5r9ydj2y73",
-                    "bottom_right" : "drj7teegpus6"
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_bounding_box" : {
+                    "pin.location" : {
+                        "top_left" : "dr5r9ydj2y73",
+                        "bottom_right" : "drj7teegpus6"
+                    }
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== Vertices
@@ -181,24 +221,28 @@ values separately.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_bounding_box" : {
-                "pin.location" : {
-                    "top" : 40.73,
-                    "left" : -74.1,
-                    "bottom" : 40.01,
-                    "right" : -71.12
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_bounding_box" : {
+                    "pin.location" : {
+                        "top" : 40.73,
+                        "left" : -74.1,
+                        "bottom" : 40.01,
+                        "right" : -71.12
+                    }
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 
 [float]
@@ -227,29 +271,33 @@ are not supported. Here is an example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_bounding_box" : {
-                "pin.location" : {
-                    "top_left" : {
-                        "lat" : 40.73,
-                        "lon" : -74.1
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_bounding_box" : {
+                    "pin.location" : {
+                        "top_left" : {
+                            "lat" : 40.73,
+                            "lon" : -74.1
+                        },
+                        "bottom_right" : {
+                            "lat" : 40.10,
+                            "lon" : -71.12
+                        }
                     },
-                    "bottom_right" : {
-                        "lat" : 40.10,
-                        "lon" : -71.12
-                    }
-                },
-                "type" : "indexed"
+                    "type" : "indexed"
+                }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== Ignore Unmapped
diff --git a/docs/reference/query-dsl/geo-distance-query.asciidoc b/docs/reference/query-dsl/geo-distance-query.asciidoc
index c29391e5b57..5e2f0ce43c7 100644
--- a/docs/reference/query-dsl/geo-distance-query.asciidoc
+++ b/docs/reference/query-dsl/geo-distance-query.asciidoc
@@ -2,10 +2,29 @@
 === Geo Distance Query
 
 Filters documents that include only hits that exists within a specific
-distance from a geo point. Assuming the following indexed json:
+distance from a geo point. Assuming the following mapping and indexed
+document:
 
 [source,js]
 --------------------------------------------------
+PUT /my_locations
+{
+    "mappings": {
+        "location": {
+            "properties": {
+                "pin": {
+                    "properties": {
+                        "location": {
+                            "type": "geo_point"
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+PUT /my_locations/location/1
 {
     "pin" : {
         "location" : {
@@ -15,29 +34,36 @@ distance from a geo point. Assuming the following indexed json:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TESTSETUP
+
 
 Then the following simple query can be executed with a `geo_distance`
 filter:
 
 [source,js]
 --------------------------------------------------
+GET /my_locations/location/_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_distance" : {
-                "distance" : "200km",
-                "pin.location" : {
-                    "lat" : 40,
-                    "lon" : -70
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_distance" : {
+                    "distance" : "200km",
+                    "pin.location" : {
+                        "lat" : 40,
+                        "lon" : -70
+                    }
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== Accepted Formats
@@ -50,23 +76,27 @@ representation of the geo point, the filter can accept it as well:
 
 [source,js]
 --------------------------------------------------
+GET /my_locations/location/_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_distance" : {
-                "distance" : "12km",
-                "pin.location" : {
-                    "lat" : 40,
-                    "lon" : -70
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_distance" : {
+                    "distance" : "12km",
+                    "pin.location" : {
+                        "lat" : 40,
+                        "lon" : -70
+                    }
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ===== Lat Lon As Array
@@ -76,20 +106,25 @@ conform with http://geojson.org/[GeoJSON].
 
 [source,js]
 --------------------------------------------------
+GET /my_locations/location/_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_distance" : {
-                "distance" : "12km",
-                "pin.location" : [-70, 40]
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_distance" : {
+                    "distance" : "12km",
+                    "pin.location" : [-70, 40]
+                }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
+
 
 [float]
 ===== Lat Lon As String
@@ -98,40 +133,48 @@ Format in `lat,lon`.
 
 [source,js]
 --------------------------------------------------
+GET /my_locations/location/_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_distance" : {
-                "distance" : "12km",
-                "pin.location" : "40,-70"
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_distance" : {
+                    "distance" : "12km",
+                    "pin.location" : "40,-70"
+                }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ===== Geohash
 
 [source,js]
 --------------------------------------------------
+GET /my_locations/location/_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_distance" : {
-                "distance" : "12km",
-                "pin.location" : "drm3btev3e86"
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_distance" : {
+                    "distance" : "12km",
+                    "pin.location" : "drm3btev3e86"
+                }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== Options
diff --git a/docs/reference/query-dsl/geo-distance-range-query.asciidoc b/docs/reference/query-dsl/geo-distance-range-query.asciidoc
index abf557d68c7..c54c50638bc 100644
--- a/docs/reference/query-dsl/geo-distance-range-query.asciidoc
+++ b/docs/reference/query-dsl/geo-distance-range-query.asciidoc
@@ -5,24 +5,28 @@ Filters documents that exists within a range from a specific point:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_distance_range" : {
-                "from" : "200km",
-                "to" : "400km",
-                "pin.location" : {
-                    "lat" : 40,
-                    "lon" : -70
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_distance_range" : {
+                    "from" : "200km",
+                    "to" : "400km",
+                    "pin.location" : {
+                        "lat" : 40,
+                        "lon" : -70
+                    }
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Supports the same point location parameter and query options as the
 <<query-dsl-geo-distance-query,geo_distance>>
diff --git a/docs/reference/query-dsl/geo-polygon-query.asciidoc b/docs/reference/query-dsl/geo-polygon-query.asciidoc
index 5717c5dc924..35278fda4a5 100644
--- a/docs/reference/query-dsl/geo-polygon-query.asciidoc
+++ b/docs/reference/query-dsl/geo-polygon-query.asciidoc
@@ -6,25 +6,29 @@ points. Here is an example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "query" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_polygon" : {
-                "person.location" : {
-                    "points" : [
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_polygon" : {
+                    "person.location" : {
+                        "points" : [
                         {"lat" : 40, "lon" : -70},
                         {"lat" : 30, "lon" : -80},
                         {"lat" : 20, "lon" : -90}
-                    ]
+                        ]
+                    }
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== Query Options
@@ -53,25 +57,29 @@ conform with http://geojson.org/[GeoJSON].
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_polygon" : {
-                "person.location" : {
-                    "points" : [
-                        [-70, 40],
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_polygon" : {
+                    "person.location" : {
+                        "points" : [
+                            [-70, 40],
                         [-80, 30],
-                        [-90, 20]
-                    ]
+                            [-90, 20]
+                        ]
+                    }
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ===== Lat Lon as String
@@ -80,50 +88,58 @@ Format in `lat,lon`.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_polygon" : {
-                "person.location" : {
-                    "points" : [
-                        "40, -70",
-                        "30, -80",
-                        "20, -90"
-                    ]
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+               "geo_polygon" : {
+                    "person.location" : {
+                        "points" : [
+                            "40, -70",
+                            "30, -80",
+                            "20, -90"
+                        ]
+                    }
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ===== Geohash
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geo_polygon" : {
-                "person.location" : {
-                    "points" : [
-                        "drn5x1g8cu2y",
-                        "30, -80",
-                        "20, -90"
-                    ]
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+               "geo_polygon" : {
+                    "person.location" : {
+                        "points" : [
+                            "drn5x1g8cu2y",
+                            "30, -80",
+                            "20, -90"
+                        ]
+                    }
                 }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== geo_point Type
diff --git a/docs/reference/query-dsl/geo-shape-query.asciidoc b/docs/reference/query-dsl/geo-shape-query.asciidoc
index a892080dda4..12203061336 100644
--- a/docs/reference/query-dsl/geo-shape-query.asciidoc
+++ b/docs/reference/query-dsl/geo-shape-query.asciidoc
@@ -26,10 +26,10 @@ Given a document that looks like this:
 --------------------------------------------------
 {
     "name": "Wind & Wetter, Berlin, Germany",
-    "location": {
-        "type": "Point",
-        "coordinates": [13.400544, 52.530286]
-    }
+        "location": {
+            "type": "Point",
+            "coordinates": [13.400544, 52.530286]
+        }
 }
 --------------------------------------------------
 
@@ -38,6 +38,7 @@ The following query will find the point using the Elasticsearch's
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
     "query":{
         "bool": {
@@ -59,6 +60,7 @@ The following query will find the point using the Elasticsearch's
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 ==== Pre-Indexed Shape
 
@@ -81,26 +83,30 @@ shape:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool": {
-        "must": {
-            "match_all": {}
-        },
-        "filter": {
-            "geo_shape": {
-                "location": {
-                    "indexed_shape": {
-                        "id": "DEU",
-                        "type": "countries",
-                        "index": "shapes",
-                        "path": "location"
+    "query": {
+        "bool": {
+            "must": {
+                "match_all": {}
+            },
+                "filter": {
+                    "geo_shape": {
+                        "location": {
+                            "indexed_shape": {
+                                "id": "DEU",
+                                "type": "countries",
+                                "index": "shapes",
+                                "path": "location"
+                            }
+                        }
                     }
                 }
-            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 ==== Spatial Relations
 
diff --git a/docs/reference/query-dsl/geohash-cell-query.asciidoc b/docs/reference/query-dsl/geohash-cell-query.asciidoc
index 2dd701fd958..27e6319bc71 100644
--- a/docs/reference/query-dsl/geohash-cell-query.asciidoc
+++ b/docs/reference/query-dsl/geohash-cell-query.asciidoc
@@ -13,6 +13,7 @@ setting the `geohash_prefix` option:
 
 [source,js]
 --------------------------------------------------
+PUT /my_index
 {
     "mappings" : {
         "location": {
@@ -28,6 +29,8 @@ setting the `geohash_prefix` option:
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TESTSETUP
 
 The geohash cell can defined by all formats of `geo_points`. If such a cell is
 defined by a latitude and longitude pair the size of the cell needs to be
@@ -42,24 +45,28 @@ next to the given cell.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "bool" : {
-        "must" : {
-            "match_all" : {}
-        },
-        "filter" : {
-            "geohash_cell": {
-                "pin": {
-                    "lat": 13.4080,
-                    "lon": 52.5186
-                },
-                "precision": 3,
-                "neighbors": true
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geohash_cell": {
+                    "pin": {
+                        "lat": 13.4080,
+                        "lon": 52.5186
+                    },
+                    "precision": 3,
+                    "neighbors": true
+                }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== Ignore Unmapped
diff --git a/docs/reference/query-dsl/has-child-query.asciidoc b/docs/reference/query-dsl/has-child-query.asciidoc
index bfadc33c06c..3b8352fd4e0 100644
--- a/docs/reference/query-dsl/has-child-query.asciidoc
+++ b/docs/reference/query-dsl/has-child-query.asciidoc
@@ -7,17 +7,21 @@ an example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "has_child" : {
-        "type" : "blog_tag",
-        "query" : {
-            "term" : {
-                "tag" : "something"
-            }
+    "query": {
+        "has_child" : {
+            "type" : "blog_tag",
+                "query" : {
+                    "term" : {
+                        "tag" : "something"
+                    }
+                }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== Scoring capabilities
@@ -32,18 +36,22 @@ inside the `has_child` query:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "has_child" : {
-        "type" : "blog_tag",
-        "score_mode" : "min",
-        "query" : {
-            "term" : {
-                "tag" : "something"
-            }
+    "query": {
+        "has_child" : {
+            "type" : "blog_tag",
+                "score_mode" : "min",
+                "query" : {
+                    "term" : {
+                        "tag" : "something"
+                    }
+                }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== Min/Max Children
@@ -54,20 +62,24 @@ a match:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "has_child" : {
-        "type" : "blog_tag",
-        "score_mode" : "min",
-        "min_children": 2, <1>
-        "max_children": 10, <1>
-        "query" : {
-            "term" : {
-                "tag" : "something"
+    "query": {
+        "has_child" : {
+            "type" : "blog_tag",
+            "score_mode" : "min",
+            "min_children": 2, <1>
+            "max_children": 10, <1>
+            "query" : {
+                "term" : {
+                    "tag" : "something"
+                }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 <1> Both `min_children` and `max_children` are optional.
 
 The  `min_children` and `max_children` parameters can be combined with
diff --git a/docs/reference/query-dsl/has-parent-query.asciidoc b/docs/reference/query-dsl/has-parent-query.asciidoc
index 9896be6a0a5..202bcaac432 100644
--- a/docs/reference/query-dsl/has-parent-query.asciidoc
+++ b/docs/reference/query-dsl/has-parent-query.asciidoc
@@ -9,17 +9,21 @@ in the same manner as the `has_child` query.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "has_parent" : {
-        "parent_type" : "blog",
-        "query" : {
-            "term" : {
-                "tag" : "something"
-            }
+    "query": {
+        "has_parent" : {
+            "parent_type" : "blog",
+                "query" : {
+                    "term" : {
+                        "tag" : "something"
+                    }
+                }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== Scoring capabilities
@@ -34,18 +38,22 @@ matching parent document. The score mode can be specified with the
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "has_parent" : {
-        "parent_type" : "blog",
-        "score" : true,
-        "query" : {
-            "term" : {
-                "tag" : "something"
-            }
+    "query": {
+        "has_parent" : {
+            "parent_type" : "blog",
+                "score" : true,
+                "query" : {
+                    "term" : {
+                        "tag" : "something"
+                    }
+                }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== Ignore Unmapped
diff --git a/docs/reference/query-dsl/ids-query.asciidoc b/docs/reference/query-dsl/ids-query.asciidoc
index 7d08243a78f..09541ce51d3 100644
--- a/docs/reference/query-dsl/ids-query.asciidoc
+++ b/docs/reference/query-dsl/ids-query.asciidoc
@@ -6,13 +6,17 @@ uses the <<mapping-uid-field,_uid>> field.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "ids" : {
-        "type" : "my_type",
-        "values" : ["1", "4", "100"]
+    "query": {
+        "ids" : {
+            "type" : "my_type",
+            "values" : ["1", "4", "100"]
+        }
     }
 }    
 --------------------------------------------------
+// CONSOLE
 
 The `type` is optional and can be omitted, and can also accept an array
 of values. If no type is specified, all types defined in the index mapping are tried.
diff --git a/docs/reference/query-dsl/indices-query.asciidoc b/docs/reference/query-dsl/indices-query.asciidoc
index e3b604b7a39..8f2f958086e 100644
--- a/docs/reference/query-dsl/indices-query.asciidoc
+++ b/docs/reference/query-dsl/indices-query.asciidoc
@@ -9,18 +9,18 @@ on the list, the alternative `no_match_query` is executed.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "indices" : {
-        "indices" : ["index1", "index2"],
-        "query" : {
-            "term" : { "tag" : "wow" }
-        },
-        "no_match_query" : {
-            "term" : { "tag" : "kow" }
+    "query": {
+        "indices" : {
+            "indices" : ["index1", "index2"],
+            "query" : { "term" : { "tag" : "wow" } },
+            "no_match_query" : { "term" : { "tag" : "kow" } }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE 
 
 You can use the `index` field to provide a single index.
 
diff --git a/docs/reference/query-dsl/match-all-query.asciidoc b/docs/reference/query-dsl/match-all-query.asciidoc
index d46b08f9e55..6e448828676 100644
--- a/docs/reference/query-dsl/match-all-query.asciidoc
+++ b/docs/reference/query-dsl/match-all-query.asciidoc
@@ -6,15 +6,27 @@ of `1.0`.
 
 [source,js]
 --------------------------------------------------
-{ "match_all": {} }
+GET /_search
+{ 
+    "query": {
+        "match_all": {}
+    }
+}
 --------------------------------------------------
+// CONSOLE
 
 The `_score` can be changed with the `boost` parameter:
 
 [source,js]
 --------------------------------------------------
-{ "match_all": { "boost" : 1.2 }}
+GET /_search
+{
+    "query": {
+        "match_all": { "boost" : 1.2 }
+    }
+}
 --------------------------------------------------
+// CONSOLE
 
 [[query-dsl-match-none-query]]
 [float]
@@ -24,5 +36,11 @@ This is the inverse of the `match_all` query, which matches no documents.
 
 [source,js]
 --------------------------------------------------
-{ "match_none": {} }
+GET /_search
+{
+    "query": {
+        "match_none": {} 
+    }
+}
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/query-dsl/match-phrase-prefix-query.asciidoc b/docs/reference/query-dsl/match-phrase-prefix-query.asciidoc
index b715bf8bd6d..e7e0f618652 100644
--- a/docs/reference/query-dsl/match-phrase-prefix-query.asciidoc
+++ b/docs/reference/query-dsl/match-phrase-prefix-query.asciidoc
@@ -6,12 +6,16 @@ allows for prefix matches on the last term in the text. For example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "match_phrase_prefix" : {
-        "message" : "quick brown f"
+    "query": {
+        "match_phrase_prefix" : {
+            "message" : "quick brown f"
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 It accepts the same parameters as the phrase type. In addition, it also
 accepts a `max_expansions` parameter (default `50`) that can control to how
@@ -21,15 +25,19 @@ example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "match_phrase_prefix" : {
-        "message" : {
-            "query" : "quick brown f",
-            "max_expansions" : 10
+    "query": {
+        "match_phrase_prefix" : {
+            "message" : {
+                "query" : "quick brown f",
+                "max_expansions" : 10
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [IMPORTANT]
 ===================================================
@@ -53,4 +61,4 @@ For better solutions for _search-as-you-type_ see the
 <<search-suggesters-completion,completion suggester>> and
 {guide}/_index_time_search_as_you_type.html[Index-Time Search-as-You-Type].
 
-===================================================
\ No newline at end of file
+===================================================
diff --git a/docs/reference/query-dsl/match-phrase-query.asciidoc b/docs/reference/query-dsl/match-phrase-query.asciidoc
index 105866be12a..943d0e84d36 100644
--- a/docs/reference/query-dsl/match-phrase-query.asciidoc
+++ b/docs/reference/query-dsl/match-phrase-query.asciidoc
@@ -6,12 +6,16 @@ out of the analyzed text. For example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "match_phrase" : {
-        "message" : "this is a test"
+    "query": {
+        "match_phrase" : {
+            "message" : "this is a test"
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 A phrase query matches terms up to a configurable `slop`
 (which defaults to 0) in any order. Transposed terms have a slop of 2.
@@ -22,12 +26,16 @@ definition, or the default search analyzer, for example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "match_phrase" : {
-        "message" : {
-            "query" : "this is a test",
-            "analyzer" : "my_analyzer"
+    "query": {
+        "match_phrase" : {
+            "message" : {
+                "query" : "this is a test",
+                "analyzer" : "my_analyzer"
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/query-dsl/match-query.asciidoc b/docs/reference/query-dsl/match-query.asciidoc
index 3dbb60d1ca2..c0081f1de9a 100644
--- a/docs/reference/query-dsl/match-query.asciidoc
+++ b/docs/reference/query-dsl/match-query.asciidoc
@@ -7,12 +7,16 @@ them, and constructs a query. For example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "match" : {
-        "message" : "this is a test"
+    "query": {
+        "match" : {
+            "message" : "this is a test"
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Note, `message` is the name of a field, you can substitute the name of
 any field (including `_all`) instead.
@@ -57,15 +61,19 @@ change in structure, `message` is the field name):
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "match" : {
-        "message" : {
-            "query" : "this is a test",
-            "operator" : "and"
+    "query": {
+        "match" : {
+            "message" : {
+                "query" : "this is a test",
+                "operator" : "and"
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [[query-dsl-match-query-zero]]
 ===== Zero terms query
@@ -76,16 +84,20 @@ change that the `zero_terms_query` option can be used, which accepts
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "match" : {
-        "message" : {
-            "query" : "to be or not to be",
-            "operator" : "and",
-            "zero_terms_query": "all"
+    "query": {
+        "match" : {
+            "message" : {
+                "query" : "to be or not to be",
+                "operator" : "and",
+                "zero_terms_query": "all"
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [[query-dsl-match-query-cutoff]]
 ===== Cutoff frequency
@@ -113,16 +125,19 @@ Here is an example showing a query composed of stopwords exclusively:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "match" : {
-        "message" : {
-            "query" : "to be or not to be",
-            "cutoff_frequency" : 0.001
+    "query": {
+        "match" : {
+            "message" : {
+                "query" : "to be or not to be",
+                "cutoff_frequency" : 0.001
+            }
         }
     }
 }
 --------------------------------------------------
-
+// CONSOLE
 
 IMPORTANT: The `cutoff_frequency` option operates on a per-shard-level. This means
 that when trying it out on test indexes with low document numbers you
diff --git a/docs/reference/query-dsl/mlt-query.asciidoc b/docs/reference/query-dsl/mlt-query.asciidoc
index dc525c2d311..b132b49f234 100644
--- a/docs/reference/query-dsl/mlt-query.asciidoc
+++ b/docs/reference/query-dsl/mlt-query.asciidoc
@@ -15,15 +15,19 @@ fields, limiting the number of selected terms to 12.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "more_like_this" : {
-        "fields" : ["title", "description"],
-        "like" : "Once upon a time",
-        "min_term_freq" : 1,
-        "max_query_terms" : 12
+    "query": {
+        "more_like_this" : {
+            "fields" : ["title", "description"],
+            "like" : "Once upon a time",
+            "min_term_freq" : 1,
+            "max_query_terms" : 12
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 A more complicated use case consists of mixing texts with documents already
 existing in the index. In this case, the syntax to specify a document is
@@ -31,27 +35,31 @@ similar to the one used in the <<docs-multi-get,Multi GET API>>.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "more_like_this" : {
-        "fields" : ["title", "description"],
-        "like" : [
-        {
-            "_index" : "imdb",
-            "_type" : "movies",
-            "_id" : "1"
-        },
-        {
-            "_index" : "imdb",
-            "_type" : "movies",
-            "_id" : "2"
-        },
-        "and potentially some more text here as well"
-        ],
-        "min_term_freq" : 1,
-        "max_query_terms" : 12
+    "query": {
+        "more_like_this" : {
+            "fields" : ["title", "description"],
+            "like" : [
+            {
+                "_index" : "imdb",
+                "_type" : "movies",
+                "_id" : "1"
+            },
+            {
+                "_index" : "imdb",
+                "_type" : "movies",
+                "_id" : "2"
+            },
+            "and potentially some more text here as well"
+            ],
+            "min_term_freq" : 1,
+            "max_query_terms" : 12
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Finally, users can mix some texts, a chosen set of documents but also provide
 documents not necessarily present in the index. To provide documents not
@@ -59,32 +67,36 @@ present in the index, the syntax is similar to <<docs-termvectors-artificial-doc
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "more_like_this" : {
-        "fields" : ["name.first", "name.last"],
-        "like" : [
-        {
-            "_index" : "marvel",
-            "_type" : "quotes",
-            "doc" : {
-                "name": {
-                    "first": "Ben",
-                    "last": "Grimm"
-                },
-                "tweet": "You got no idea what I'd... what I'd give to be invisible."
-              }
-        },
-        {
-            "_index" : "marvel",
-            "_type" : "quotes",
-            "_id" : "2"
+    "query": {
+        "more_like_this" : {
+            "fields" : ["name.first", "name.last"],
+            "like" : [
+            {
+                "_index" : "marvel",
+                "_type" : "quotes",
+                "doc" : {
+                    "name": {
+                        "first": "Ben",
+                        "last": "Grimm"
+                    },
+                    "tweet": "You got no idea what I'd... what I'd give to be invisible."
+                  }
+            },
+            {
+                "_index" : "marvel",
+                "_type" : "quotes",
+                "_id" : "2"
+            }
+            ],
+            "min_term_freq" : 1,
+            "max_query_terms" : 12
         }
-        ],
-        "min_term_freq" : 1,
-        "max_query_terms" : 12
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 ==== How it Works
 
@@ -111,32 +123,34 @@ default, but there will be no speed up on analysis for these fields.
 
 [source,js]
 --------------------------------------------------
-curl -s -XPUT 'http://localhost:9200/imdb/' -d '{
-  "mappings": {
-    "movies": {
-      "properties": {
-        "title": {
-          "type": "text",
-          "term_vector": "yes"
-         },
-         "description": {
-          "type": "text"
-        },
-        "tags": {
-          "type": "text",
-          "fields" : {
-            "raw": {
-              "type" : "text",
-              "analyzer": "keyword",
-              "term_vector" : "yes"
+PUT /imdb
+{
+    "mappings": {
+        "movies": {
+            "properties": {
+                "title": {
+                    "type": "text",
+                    "term_vector": "yes"
+                },
+                "description": {
+                    "type": "text"
+                },
+                "tags": {
+                    "type": "text",
+                    "fields" : {
+                        "raw": {
+                            "type" : "text",
+                            "analyzer": "keyword",
+                            "term_vector" : "yes"
+                        }
+                    }
+                }
             }
-          }
         }
-      }
     }
-  }
 }
 --------------------------------------------------
+// CONSOLE
 
 ==== Parameters
 
diff --git a/docs/reference/query-dsl/multi-match-query.asciidoc b/docs/reference/query-dsl/multi-match-query.asciidoc
index 5c958eeaf81..068156ff70f 100644
--- a/docs/reference/query-dsl/multi-match-query.asciidoc
+++ b/docs/reference/query-dsl/multi-match-query.asciidoc
@@ -6,13 +6,17 @@ to allow multi-field queries:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "multi_match" : {
-    "query":    "this is a test", <1>
-    "fields": [ "subject", "message" ] <2>
+  "query": {
+    "multi_match" : {
+      "query":    "this is a test", <1>
+      "fields": [ "subject", "message" ] <2>
+    }
   }
 }
 --------------------------------------------------
+// CONSOLE
 <1> The query string.
 <2> The fields to be queried.
 
@@ -23,26 +27,35 @@ Fields can be specified with wildcards, eg:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "multi_match" : {
-    "query":    "Will Smith",
-    "fields": [ "title", "*_name" ] <1>
+  "query": {
+    "multi_match" : {
+      "query":    "Will Smith",
+      "fields": [ "title", "*_name" ] <1>
+    }
   }
 }
 --------------------------------------------------
+// CONSOLE
 <1> Query the `title`, `first_name` and `last_name` fields.
 
 Individual fields can be boosted with the caret (`^`) notation:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "multi_match" : {
-    "query" : "this is a test",
-    "fields" : [ "subject^3", "message" ] <1>
+  "query": {
+    "multi_match" : {
+      "query" : "this is a test",
+      "fields" : [ "subject^3", "message" ] <1>
+    }
   }
 }
 --------------------------------------------------
+// CONSOLE
+
 <1> The `subject` field is three times as important as the `message` field.
 
 [[multi-match-types]]
@@ -82,30 +95,38 @@ find the single best matching field.  For instance, this query:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "multi_match" : {
-    "query":      "brown fox",
-    "type":       "best_fields",
-    "fields":     [ "subject", "message" ],
-    "tie_breaker": 0.3
+  "query": {
+    "multi_match" : {
+      "query":      "brown fox",
+      "type":       "best_fields",
+      "fields":     [ "subject", "message" ],
+      "tie_breaker": 0.3
+    }
   }
 }
 --------------------------------------------------
+// CONSOLE
 
 would be executed as:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "dis_max": {
-    "queries": [
-      { "match": { "subject": "brown fox" }},
-      { "match": { "message": "brown fox" }}
-    ],
-    "tie_breaker": 0.3
+  "query": {
+    "dis_max": {
+      "queries": [
+        { "match": { "subject": "brown fox" }},
+        { "match": { "message": "brown fox" }}
+      ],
+      "tie_breaker": 0.3
+    }
   }
 }
 --------------------------------------------------
+// CONSOLE
 
 Normally the `best_fields` type uses the score of the *single* best matching
 field, but if `tie_breaker` is specified, then it calculates the score as
@@ -132,15 +153,20 @@ Take this query for example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "multi_match" : {
-    "query":      "Will Smith",
-    "type":       "best_fields",
-    "fields":     [ "first_name", "last_name" ],
-    "operator":   "and" <1>
+  "query": {
+    "multi_match" : {
+      "query":      "Will Smith",
+      "type":       "best_fields",
+      "fields":     [ "first_name", "last_name" ],
+      "operator":   "and" <1>
+    }
   }
 }
 --------------------------------------------------
+// CONSOLE
+
 <1> All terms must be present.
 
 This query is executed as:
@@ -170,29 +196,37 @@ This query:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "multi_match" : {
-    "query":      "quick brown fox",
-    "type":       "most_fields",
-    "fields":     [ "title", "title.original", "title.shingles" ]
+  "query": {
+    "multi_match" : {
+      "query":      "quick brown fox",
+      "type":       "most_fields",
+      "fields":     [ "title", "title.original", "title.shingles" ]
+    }
   }
 }
 --------------------------------------------------
+// CONSOLE
 
 would be executed as:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "bool": {
-    "should": [
-      { "match": { "title":          "quick brown fox" }},
-      { "match": { "title.original": "quick brown fox" }},
-      { "match": { "title.shingles": "quick brown fox" }}
-    ]
+  "query": {
+    "bool": {
+      "should": [
+        { "match": { "title":          "quick brown fox" }},
+        { "match": { "title.original": "quick brown fox" }},
+        { "match": { "title.shingles": "quick brown fox" }}
+      ]
+    }
   }
 }
 --------------------------------------------------
+// CONSOLE
 
 The score from each `match` clause is added together, then divided by the
 number of `match` clauses.
@@ -212,28 +246,36 @@ but they use a `match_phrase` or `match_phrase_prefix` query instead of a
 This query:
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "multi_match" : {
-    "query":      "quick brown f",
-    "type":       "phrase_prefix",
-    "fields":     [ "subject", "message" ]
+  "query": {
+    "multi_match" : {
+      "query":      "quick brown f",
+      "type":       "phrase_prefix",
+      "fields":     [ "subject", "message" ]
+    }
   }
 }
 --------------------------------------------------
+// CONSOLE
 
 would be executed as:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "dis_max": {
-    "queries": [
-      { "match_phrase_prefix": { "subject": "quick brown f" }},
-      { "match_phrase_prefix": { "message": "quick brown f" }}
-    ]
+  "query": {
+    "dis_max": {
+      "queries": [
+        { "match_phrase_prefix": { "subject": "quick brown f" }},
+        { "match_phrase_prefix": { "message": "quick brown f" }}
+      ]
+    }
   }
 }
 --------------------------------------------------
+// CONSOLE
 
 Also, accepts `analyzer`, `boost`, `slop` and `zero_terms_query`  as explained
 in <<query-dsl-match-query>>.  Type `phrase_prefix` additionally accepts
@@ -288,15 +330,19 @@ A query like:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "multi_match" : {
-    "query":      "Will Smith",
-    "type":       "cross_fields",
-    "fields":     [ "first_name", "last_name" ],
-    "operator":   "and"
+  "query": {
+    "multi_match" : {
+      "query":      "Will Smith",
+      "type":       "cross_fields",
+      "fields":     [ "first_name", "last_name" ],
+      "operator":   "and"
+    }
   }
 }
 --------------------------------------------------
+// CONSOLE
 
 is executed as:
 
@@ -344,17 +390,21 @@ both use an `edge_ngram` analyzer, this query:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "multi_match" : {
-    "query":      "Jon",
-    "type":       "cross_fields",
-    "fields":     [
+  "query": {
+    "multi_match" : {
+      "query":      "Jon",
+      "type":       "cross_fields",
+      "fields":     [
         "first", "first.edge",
         "last",  "last.edge"
-    ]
+      ]
+    }
   }
 }
 --------------------------------------------------
+// CONSOLE
 
 would be executed as:
 
@@ -379,28 +429,33 @@ parameter to just one of them:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
+  "query": {
     "bool": {
-        "should": [
-            {
-              "multi_match" : {
-                "query":      "Will Smith",
-                "type":       "cross_fields",
-                "fields":     [ "first", "last" ],
-                "minimum_should_match": "50%" <1>
-              }
-            },
-            {
-              "multi_match" : {
-                "query":      "Will Smith",
-                "type":       "cross_fields",
-                "fields":     [ "*.edge" ]
-              }
-            }
-        ]
+      "should": [
+        {
+          "multi_match" : {
+            "query":      "Will Smith",
+            "type":       "cross_fields",
+            "fields":     [ "first", "last" ],
+            "minimum_should_match": "50%" <1>
+          }
+        },
+        {
+          "multi_match" : {
+            "query":      "Will Smith",
+            "type":       "cross_fields",
+            "fields":     [ "*.edge" ]
+          }
+        }
+      ]
     }
+  }
 }
 --------------------------------------------------
+// CONSOLE 
+
 <1> Either `will` or `smith` must be present in either of the `first`
     or `last` fields
 
@@ -409,15 +464,20 @@ parameter in the query.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-  "multi_match" : {
-    "query":      "Jon",
-    "type":       "cross_fields",
-    "analyzer":   "standard", <1>
-    "fields":     [ "first", "last", "*.edge" ]
+  "query": {
+   "multi_match" : {
+      "query":      "Jon",
+      "type":       "cross_fields",
+      "analyzer":   "standard", <1>
+      "fields":     [ "first", "last", "*.edge" ]
+    }
   }
 }
 --------------------------------------------------
+// CONSOLE
+
 <1> Use the `standard` analyzer for all fields.
 
 which will be executed as:
diff --git a/docs/reference/query-dsl/nested-query.asciidoc b/docs/reference/query-dsl/nested-query.asciidoc
index 0b861509c0e..06720552bde 100644
--- a/docs/reference/query-dsl/nested-query.asciidoc
+++ b/docs/reference/query-dsl/nested-query.asciidoc
@@ -10,40 +10,45 @@ will work with:
 
 [source,js]
 --------------------------------------------------
+PUT /my_index
 {
-    "type1" : {
-        "properties" : {
-            "obj1" : {
-                "type" : "nested"
+    "mappings": {
+        "type1" : {
+            "properties" : {
+                "obj1" : {
+                    "type" : "nested"
+                }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
+// TESTSETUP
 
 And here is a sample nested query usage:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "nested" : {
-        "path" : "obj1",
-        "score_mode" : "avg",
-        "query" : {
-            "bool" : {
-                "must" : [
-                    {
-                        "match" : {"obj1.name" : "blue"}
-                    },
-                    {
-                        "range" : {"obj1.count" : {"gt" : 5}}
-                    }
-                ]
+    "query": {
+        "nested" : {
+            "path" : "obj1",
+            "score_mode" : "avg",
+            "query" : {
+                "bool" : {
+                    "must" : [
+                    { "match" : {"obj1.name" : "blue"} },
+                    { "range" : {"obj1.count" : {"gt" : 5}} }
+                    ]
+                }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 The query `path` points to the nested object path, and the `query`
 includes the query that will run on the nested docs matching the
diff --git a/docs/reference/query-dsl/parent-id-query.asciidoc b/docs/reference/query-dsl/parent-id-query.asciidoc
index a29073dafa9..8ea07a6d0b7 100644
--- a/docs/reference/query-dsl/parent-id-query.asciidoc
+++ b/docs/reference/query-dsl/parent-id-query.asciidoc
@@ -54,17 +54,21 @@ better as it does not need to do a join:
 
 [source,js]
 --------------------------------------------------
+GET /my_index/_search
 {
-  "has_parent": {
-    "type": "blog",
-    "query": {
-      "term": {
-        "_id": "1"
+  "query": {
+    "has_parent": {
+      "type": "blog_post",
+        "query": {
+          "term": {
+            "_id": "1"
+        }
       }
     }
   }
 }
 --------------------------------------------------
+// CONSOLE
 
 ==== Parameters
 
diff --git a/docs/reference/query-dsl/percolate-query.asciidoc b/docs/reference/query-dsl/percolate-query.asciidoc
index 447a4d3189c..896fa7ec3ee 100644
--- a/docs/reference/query-dsl/percolate-query.asciidoc
+++ b/docs/reference/query-dsl/percolate-query.asciidoc
@@ -13,26 +13,27 @@ Create an index with two mappings:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT "http://localhost:9200/my-index" -d'
+PUT /my-index
 {
-  "mappings": {
-    "doctype": {
-      "properties": {
-        "message": {
-          "type": "string"
+    "mappings": {
+        "doctype": {
+            "properties": {
+                "message": {
+                    "type": "string"
+                }
+            }
+        },
+        "queries": {
+            "properties": {
+                "query": {
+                    "type": "percolator"
+                }
+            }
         }
-      }
-    },
-    "queries": {
-      "properties": {
-        "query": {
-          "type": "percolator"
-        }
-      }
     }
-  }
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
 The `doctype` mapping is the mapping used to preprocess
 the document defined in the `percolator` query before it
@@ -50,20 +51,24 @@ Register a query in the percolator:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT 'localhost:9200/my-index/queries/1' -d '{
+PUT /my-index/queries/1
+{
     "query" : {
         "match" : {
             "message" : "bonsai tree"
         }
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 Match a document to the registered percolator queries:
 
 [source,js]
 --------------------------------------------------
-curl -XGET 'localhost:9200/my-index/_search' -d '{
+GET /my-index/_search
+{
     "query" : {
         "percolate" : {
             "field" : "query",
@@ -73,8 +78,10 @@ curl -XGET 'localhost:9200/my-index/_search' -d '{
             }
         }
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 The above request will yield the following response:
 
@@ -151,12 +158,13 @@ Index the document we want to percolate:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT "http://localhost:9200/my-index/message/1" -d'
+PUT /my-index/message/1
 {
   "message" : "A new bonsai tree in the office"
-}'
+}
 --------------------------------------------------
-
+// CONSOLE
+// TEST[continued]
 Index response:
 
 [source,js]
@@ -179,7 +187,7 @@ Percolating an existing document, using the index response as basis to build to
 
 [source,js]
 --------------------------------------------------
-curl -XGET "http://localhost:9200/my-index/_search" -d'
+GET /my-index/_search
 {
     "query" : {
         "percolate" : {
@@ -191,8 +199,10 @@ curl -XGET "http://localhost:9200/my-index/_search" -d'
             "version" : 1 <1>
         }
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 <1> The version is optional, but useful in certain cases. We can then ensure that we are try to percolate
 the document we just have indexed. A change may be made after we have indexed, and if that is the
@@ -216,35 +226,39 @@ Save a query:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT "http://localhost:9200/my-index/queries/1" -d'
+PUT /my-index/queries/1
 {
     "query" : {
         "match" : {
             "message" : "brown fox"
         }
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 Save another query:
 
 [source,js]
 --------------------------------------------------
-curl -XPUT "http://localhost:9200/my-index/queries/2" -d'
+PUT /my-index/queries/2
 {
     "query" : {
         "match" : {
             "message" : "lazy dog"
         }
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 Execute a search request with the `percolate` query and highlighting enabled:
 
 [source,js]
 --------------------------------------------------
-curl -XGET "http://localhost:9200/my-index/_search" -d'
+GET /my-index/_search
 {
     "query" : {
         "percolate" : {
@@ -260,8 +274,10 @@ curl -XGET "http://localhost:9200/my-index/_search" -d'
         "message": {}
       }
     }
-}'
+}
 --------------------------------------------------
+// CONSOLE
+// TEST[continued]
 
 This will yield the following response.
 
@@ -344,16 +360,19 @@ can't do the selecting optimization (for example if an unsupported query is defi
 or the unsupported query is the only query in the percolator document).  These queries are marked by the percolator and
 can be found by running the following search:
 
+
 [source,js]
---------------------------------------------------
-curl -XGET "http://localhost:9200/_search" -d'
+---------------------------------------------------
+GET /_search
 {
   "query": {
     "term" : {
       "query.unknown_query" : ""
     }
   }
-}'
---------------------------------------------------
+}
+---------------------------------------------------
+// CONSOLE
 
-NOTE: The above example assumes that there is a `query` field of type `percolator` in the mappings.
\ No newline at end of file
+NOTE: The above example assumes that there is a `query` field of type
+`percolator` in the mappings.
diff --git a/docs/reference/query-dsl/prefix-query.asciidoc b/docs/reference/query-dsl/prefix-query.asciidoc
index cf26e850ad8..d2b75d10e5f 100644
--- a/docs/reference/query-dsl/prefix-query.asciidoc
+++ b/docs/reference/query-dsl/prefix-query.asciidoc
@@ -8,28 +8,37 @@ that starts with `ki`:
 
 [source,js]
 --------------------------------------------------
-{
+GET /_search
+{ "query": {
     "prefix" : { "user" : "ki" }
+  }
 }
 --------------------------------------------------
+// CONSOLE
 
 A boost can also be associated with the query:
 
 [source,js]
 --------------------------------------------------
-{
+GET /_search
+{ "query": {
     "prefix" : { "user" :  { "value" : "ki", "boost" : 2.0 } }
+  }
 }
 --------------------------------------------------
+// CONSOLE
 
 Or :
 
 [source,js]
 --------------------------------------------------
-{
+GET /_search
+{ "query": {
     "prefix" : { "user" :  { "prefix" : "ki", "boost" : 2.0 } }
+  }
 }
 --------------------------------------------------
+// CONSOLE
 
 This multi term query allows you to control how it gets rewritten using the
 <<query-dsl-multi-term-rewrite,rewrite>>
diff --git a/docs/reference/query-dsl/query-string-query.asciidoc b/docs/reference/query-dsl/query-string-query.asciidoc
index 9ad68ad4f73..60477d6e28a 100644
--- a/docs/reference/query-dsl/query-string-query.asciidoc
+++ b/docs/reference/query-dsl/query-string-query.asciidoc
@@ -6,13 +6,17 @@ an example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query_string" : {
-        "default_field" : "content",
-        "query" : "this AND that OR thus"
+    "query": {
+        "query_string" : {
+            "default_field" : "content",
+            "query" : "this AND that OR thus"
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 The `query_string` top level parameters include:
 
@@ -113,25 +117,33 @@ For example, the following query
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query_string" : {
-        "fields" : ["content", "name"],
-        "query" : "this AND that"
+    "query": {
+        "query_string" : {
+            "fields" : ["content", "name"],
+            "query" : "this AND that"
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 matches the same words as
 
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query_string": {
-      "query": "(content:this OR name:this) AND (content:that OR name:that)"
+    "query": {
+        "query_string": {
+            "query": "(content:this OR name:this) AND (content:that OR name:that)"
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Since several queries are generated from the individual search terms,
 combining them can be automatically done using either a `dis_max` query or a
@@ -140,14 +152,18 @@ notation):
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query_string" : {
-        "fields" : ["content", "name^5"],
-        "query" : "this AND that OR thus",
-        "use_dis_max" : true
+    "query": {
+        "query_string" : {
+            "fields" : ["content", "name^5"],
+            "query" : "this AND that OR thus",
+            "use_dis_max" : true
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Simple wildcard can also be used to search "within" specific inner
 elements of the document. For example, if we have a `city` object with
@@ -156,14 +172,18 @@ search on all "city" fields:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query_string" : {
-        "fields" : ["city.*"],
-        "query" : "this AND that OR thus",
-        "use_dis_max" : true
+    "query": {
+        "query_string" : {
+            "fields" : ["city.*"],
+            "query" : "this AND that OR thus",
+            "use_dis_max" : true
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Another option is to provide the wildcard fields search in the query
 string itself (properly escaping the `*` sign), for example:
@@ -188,13 +208,17 @@ introduced fields included). For example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "query_string" : {
-        "fields" : ["content", "name.*^5"],
-        "query" : "this AND that OR thus",
-        "use_dis_max" : true
+    "query": {
+        "query_string" : {
+            "fields" : ["content", "name.*^5"],
+            "query" : "this AND that OR thus",
+            "use_dis_max" : true
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 include::query-string-syntax.asciidoc[]
diff --git a/docs/reference/query-dsl/query_filter_context.asciidoc b/docs/reference/query-dsl/query_filter_context.asciidoc
index 6fadc36aa73..79f8c4bd960 100644
--- a/docs/reference/query-dsl/query_filter_context.asciidoc
+++ b/docs/reference/query-dsl/query_filter_context.asciidoc
@@ -47,7 +47,7 @@ conditions are met:
 
 [source,js]
 ------------------------------------
-GET _search
+GET /_search
 {
   "query": { <1>
     "bool": { <2>
@@ -63,6 +63,7 @@ GET _search
   }
 }
 ------------------------------------
+// CONSOLE
 <1> The `query` parameter indicates query context.
 <2> The `bool` and two `match` clauses are used in query context,
     which means that they are used to score how well each document
diff --git a/docs/reference/query-dsl/range-query.asciidoc b/docs/reference/query-dsl/range-query.asciidoc
index eaf1dca8181..a0005ff3ff2 100644
--- a/docs/reference/query-dsl/range-query.asciidoc
+++ b/docs/reference/query-dsl/range-query.asciidoc
@@ -9,16 +9,20 @@ a `NumericRangeQuery`. The following example returns all documents where
 
 [source,js]
 --------------------------------------------------
+GET _search
 {
-    "range" : {
-        "age" : {
-            "gte" : 10,
-            "lte" : 20,
-            "boost" : 2.0
+    "query": {
+        "range" : {
+            "age" : {
+                "gte" : 10,
+                "lte" : 20,
+                "boost" : 2.0
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE 
 
 The `range` query accepts the following parameters:
 
@@ -38,15 +42,19 @@ specified using <<date-math>>:
 
 [source,js]
 --------------------------------------------------
+GET _search
 {
-    "range" : {
-        "date" : {
-            "gte" : "now-1d/d",
-            "lt" :  "now/d"
+    "query": {
+        "range" : {
+            "date" : {
+                "gte" : "now-1d/d",
+                "lt" :  "now/d"
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 ===== Date math and rounding
 
@@ -86,16 +94,20 @@ passing the `format` parameter to the `range` query:
 
 [source,js]
 --------------------------------------------------
+GET _search
 {
-    "range" : {
-        "born" : {
-            "gte": "01/01/2012",
-            "lte": "2013",
-            "format": "dd/MM/yyyy||yyyy"
+    "query": {
+        "range" : {
+            "born" : {
+                "gte": "01/01/2012",
+                "lte": "2013",
+                "format": "dd/MM/yyyy||yyyy"
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE 
 
 ===== Time zone in range queries
 
@@ -105,15 +117,19 @@ accepts it), or it can be specified as the `time_zone` parameter:
 
 [source,js]
 --------------------------------------------------
+GET _search
 {
-    "range" : {
-        "timestamp" : {
-            "gte": "2015-01-01 00:00:00", <1>
-            "lte": "now", <2>
-            "time_zone": "+01:00"
+    "query": {
+        "range" : {
+            "timestamp" : {
+                "gte": "2015-01-01 00:00:00", <1>
+                "lte": "now", <2>
+                "time_zone": "+01:00"
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 <1> This date will be converted to `2014-12-31T23:00:00 UTC`.
 <2> `now` is not affected by the `time_zone` parameter (dates must be stored as UTC).
diff --git a/docs/reference/query-dsl/regexp-query.asciidoc b/docs/reference/query-dsl/regexp-query.asciidoc
index 692caf43480..2d57f1d7dd0 100644
--- a/docs/reference/query-dsl/regexp-query.asciidoc
+++ b/docs/reference/query-dsl/regexp-query.asciidoc
@@ -15,40 +15,52 @@ matchers like `.*?+` will mostly lower performance.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "regexp":{
-        "name.first": "s.*y"
+    "query": {
+        "regexp":{
+            "name.first": "s.*y"
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Boosting is also supported
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "regexp":{
-        "name.first":{
-            "value":"s.*y",
-            "boost":1.2
+    "query": {
+        "regexp":{
+            "name.first":{
+                "value":"s.*y",
+                "boost":1.2
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 You can also use special flags
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "regexp":{
-        "name.first": {
-            "value": "s.*y",
-            "flags" : "INTERSECTION|COMPLEMENT|EMPTY"
+    "query": {
+        "regexp":{
+            "name.first": {
+                "value": "s.*y",
+                "flags" : "INTERSECTION|COMPLEMENT|EMPTY"
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 Possible flags are `ALL` (default), `ANYSTRING`, `COMPLEMENT`,
 `EMPTY`, `INTERSECTION`, `INTERVAL`, or `NONE`. Please check the
@@ -64,16 +76,19 @@ this limit to allow more complex regular expressions to execute.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "regexp":{
-        "name.first": {
-            "value": "s.*y",
-            "flags" : "INTERSECTION|COMPLEMENT|EMPTY",
-	    "max_determinized_states": 20000
+    "query": {
+        "regexp":{
+            "name.first": {
+                "value": "s.*y",
+                "flags" : "INTERSECTION|COMPLEMENT|EMPTY",
+                "max_determinized_states": 20000
+            }
         }
     }
 }
 --------------------------------------------------
-
+// CONSOLE
 
 include::regexp-syntax.asciidoc[]
diff --git a/docs/reference/query-dsl/script-query.asciidoc b/docs/reference/query-dsl/script-query.asciidoc
index 223460f723d..ee06a1b64bd 100644
--- a/docs/reference/query-dsl/script-query.asciidoc
+++ b/docs/reference/query-dsl/script-query.asciidoc
@@ -7,17 +7,20 @@ context, for example:
 
 [source,js]
 ----------------------------------------------
-"bool" : {
-    "must" : {
-        ...
-    }, 
-    "filter" : {
-        "script" : {
-            "script" : "doc['num1'].value > 1"
+GET /_search
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "script" : {
+                    "script" : "doc['num1'].value > 1"
+                }
+            }
         }
     }
 }
 ----------------------------------------------
+// CONSOLE
 
 [float]
 ==== Custom Parameters
@@ -28,20 +31,23 @@ to use the ability to pass parameters to the script itself, for example:
 
 [source,js]
 ----------------------------------------------
-"bool" : {
-    "must" : {
-        ...
-    }, 
-    "filter" : {
-        "script" : {
-            "script" : {
-                "inline" : "doc['num1'].value > param1"
-                "params" : {
-                    "param1" : 5
+GET /_search
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "script" : {
+                    "script" : {
+                        "inline" : "doc['num1'].value > param1",
+                        "params" : {
+                            "param1" : 5
+                        }
+                    }
                 }
             }
         }
     }
 }
 ----------------------------------------------
+// CONSOLE
 
diff --git a/docs/reference/query-dsl/simple-query-string-query.asciidoc b/docs/reference/query-dsl/simple-query-string-query.asciidoc
index 572c0661fa4..796f2517fea 100644
--- a/docs/reference/query-dsl/simple-query-string-query.asciidoc
+++ b/docs/reference/query-dsl/simple-query-string-query.asciidoc
@@ -8,15 +8,19 @@ an example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
+  "query": {
     "simple_query_string" : {
         "query": "\"fried eggs\" +(eggplant | potato) -frittata",
         "analyzer": "snowball",
         "fields": ["body^5","_all"],
         "default_operator": "and"
     }
+  }
 }
 --------------------------------------------------
+// CONSOLE
 
 The `simple_query_string` top level parameters include:
 
@@ -94,13 +98,17 @@ introduced fields included). For example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "simple_query_string" : {
-        "fields" : ["content", "name.*^5"],
-        "query" : "foo bar baz"
+    "query": {
+        "simple_query_string" : {
+            "fields" : ["content", "name.*^5"],
+            "query" : "foo bar baz"
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 [float]
 ==== Flags
@@ -110,13 +118,17 @@ should be enabled. It is specified as a `|`-delimited string with the
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "simple_query_string" : {
-        "query" : "foo | bar + baz*",
-        "flags" : "OR|AND|PREFIX"
+    "query": {
+        "simple_query_string" : {
+            "query" : "foo | bar + baz*",
+            "flags" : "OR|AND|PREFIX"
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 The available flags are: `ALL`, `NONE`, `AND`, `OR`, `NOT`, `PREFIX`, `PHRASE`,
 `PRECEDENCE`, `ESCAPE`, `WHITESPACE`, `FUZZY`, `NEAR`, and `SLOP`.
diff --git a/docs/reference/query-dsl/span-containing-query.asciidoc b/docs/reference/query-dsl/span-containing-query.asciidoc
index 965bf855b6f..638c6999233 100644
--- a/docs/reference/query-dsl/span-containing-query.asciidoc
+++ b/docs/reference/query-dsl/span-containing-query.asciidoc
@@ -6,24 +6,28 @@ query maps to Lucene `SpanContainingQuery`. Here is an example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "span_containing" : {
-        "little" : {
-            "span_term" : { "field1" : "foo" }
-        },
-        "big" : {
-            "span_near" : { 
-                "clauses" : [
-                    { "span_term" : { "field1" : "bar" } },
-                    { "span_term" : { "field1" : "baz" } }
-                ],
-                "slop" : 5,
-                "in_order" : true
+    "query": {
+        "span_containing" : {
+            "little" : {
+                "span_term" : { "field1" : "foo" }
+            },
+            "big" : {
+                "span_near" : { 
+                    "clauses" : [
+                        { "span_term" : { "field1" : "bar" } },
+                        { "span_term" : { "field1" : "baz" } }
+                    ],
+                    "slop" : 5,
+                    "in_order" : true
+                }
             }
         }
     }
 }
 --------------------------------------------------
+//  CONSOLE
 
 The `big` and `little` clauses can be any span type query. Matching
 spans from `big` that contain matches from `little` are returned.
diff --git a/docs/reference/query-dsl/span-first-query.asciidoc b/docs/reference/query-dsl/span-first-query.asciidoc
index 74fe7ff88ba..dba7932661d 100644
--- a/docs/reference/query-dsl/span-first-query.asciidoc
+++ b/docs/reference/query-dsl/span-first-query.asciidoc
@@ -6,15 +6,19 @@ to Lucene `SpanFirstQuery`. Here is an example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "span_first" : {
-        "match" : {
-            "span_term" : { "user" : "kimchy" }
-        },
-        "end" : 3
+    "query": {
+        "span_first" : {
+            "match" : {
+                "span_term" : { "user" : "kimchy" }
+            },
+            "end" : 3
+        }
     }
 }    
 --------------------------------------------------
+// CONSOLE
 
 The `match` clause can be any other span type query. The `end` controls
 the maximum end position permitted in a match.
diff --git a/docs/reference/query-dsl/span-multi-term-query.asciidoc b/docs/reference/query-dsl/span-multi-term-query.asciidoc
index af3da5cf7dd..3d1672753d2 100644
--- a/docs/reference/query-dsl/span-multi-term-query.asciidoc
+++ b/docs/reference/query-dsl/span-multi-term-query.asciidoc
@@ -7,24 +7,32 @@ it can be nested. Example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "span_multi":{
-        "match":{
-            "prefix" : { "user" :  { "value" : "ki" } }
+    "query": {
+        "span_multi":{
+            "match":{
+                "prefix" : { "user" :  { "value" : "ki" } }
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 A boost can also be associated with the query:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "span_multi":{
-        "match":{
-            "prefix" : { "user" :  { "value" : "ki", "boost" : 1.08 } }
+    "query": {
+        "span_multi":{
+            "match":{
+                "prefix" : { "user" :  { "value" : "ki", "boost" : 1.08 } }
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/query-dsl/span-near-query.asciidoc b/docs/reference/query-dsl/span-near-query.asciidoc
index 258bc3b51ad..e69be783e3d 100644
--- a/docs/reference/query-dsl/span-near-query.asciidoc
+++ b/docs/reference/query-dsl/span-near-query.asciidoc
@@ -8,18 +8,22 @@ matches are required to be in-order. The span near query maps to Lucene
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "span_near" : {
-        "clauses" : [
-            { "span_term" : { "field" : "value1" } },
-            { "span_term" : { "field" : "value2" } },
-            { "span_term" : { "field" : "value3" } }
-        ],
-        "slop" : 12,
-        "in_order" : false
+    "query": {
+        "span_near" : {
+            "clauses" : [
+                { "span_term" : { "field" : "value1" } },
+                { "span_term" : { "field" : "value2" } },
+                { "span_term" : { "field" : "value3" } }
+            ],
+            "slop" : 12,
+            "in_order" : false
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 The `clauses` element is a list of one or more other span type queries
 and the `slop` controls the maximum number of intervening unmatched
diff --git a/docs/reference/query-dsl/span-not-query.asciidoc b/docs/reference/query-dsl/span-not-query.asciidoc
index 73186985b0d..5a648bd4b0e 100644
--- a/docs/reference/query-dsl/span-not-query.asciidoc
+++ b/docs/reference/query-dsl/span-not-query.asciidoc
@@ -6,24 +6,28 @@ query maps to Lucene `SpanNotQuery`. Here is an example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "span_not" : {
-        "include" : {
-            "span_term" : { "field1" : "hoya" }
-        },
-        "exclude" : {
-            "span_near" : { 
-                "clauses" : [
-                    { "span_term" : { "field1" : "la" } },
-                    { "span_term" : { "field1" : "hoya" } }
-                ],
-                "slop" : 0,
-                "in_order" : true
+    "query": {
+        "span_not" : {
+            "include" : {
+                "span_term" : { "field1" : "hoya" }
+            },
+            "exclude" : {
+                "span_near" : { 
+                    "clauses" : [
+                        { "span_term" : { "field1" : "la" } },
+                        { "span_term" : { "field1" : "hoya" } }
+                    ],
+                    "slop" : 0,
+                    "in_order" : true
+                }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 The `include` and `exclude` clauses can be any span type query. The
 `include` clause is the span query whose matches are filtered, and the
diff --git a/docs/reference/query-dsl/span-or-query.asciidoc b/docs/reference/query-dsl/span-or-query.asciidoc
index 72a4ce8724b..470935d6f5c 100644
--- a/docs/reference/query-dsl/span-or-query.asciidoc
+++ b/docs/reference/query-dsl/span-or-query.asciidoc
@@ -6,15 +6,19 @@ Matches the union of its span clauses. The span or query maps to Lucene
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "span_or" : {
-        "clauses" : [
-            { "span_term" : { "field" : "value1" } },
-            { "span_term" : { "field" : "value2" } },
-            { "span_term" : { "field" : "value3" } }
-        ]
+    "query": {
+        "span_or" : {
+            "clauses" : [
+                { "span_term" : { "field" : "value1" } },
+                { "span_term" : { "field" : "value2" } },
+                { "span_term" : { "field" : "value3" } }
+            ]
+        }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 The `clauses` element is a list of one or more other span type queries.
diff --git a/docs/reference/query-dsl/span-term-query.asciidoc b/docs/reference/query-dsl/span-term-query.asciidoc
index 9de86d48684..1b12a3c35f7 100644
--- a/docs/reference/query-dsl/span-term-query.asciidoc
+++ b/docs/reference/query-dsl/span-term-query.asciidoc
@@ -6,25 +6,37 @@ Matches spans containing a term. The span term query maps to Lucene
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "span_term" : { "user" : "kimchy" }
+    "query": {
+        "span_term" : { "user" : "kimchy" }
+    }
 }    
 --------------------------------------------------
+// CONSOLE
 
 A boost can also be associated with the query:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "span_term" : { "user" : { "value" : "kimchy", "boost" : 2.0 } }
+    "query": {
+       "span_term" : { "user" : { "value" : "kimchy", "boost" : 2.0 } }
+    }
 }    
 --------------------------------------------------
+// CONSOLE
 
 Or :
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "span_term" : { "user" : { "term" : "kimchy", "boost" : 2.0 } }
+    "query": {
+        "span_term" : { "user" : { "term" : "kimchy", "boost" : 2.0 } }
+    }
 }    
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/query-dsl/span-within-query.asciidoc b/docs/reference/query-dsl/span-within-query.asciidoc
index dc5c4bbfdfd..b70835c4134 100644
--- a/docs/reference/query-dsl/span-within-query.asciidoc
+++ b/docs/reference/query-dsl/span-within-query.asciidoc
@@ -6,24 +6,28 @@ query maps to Lucene `SpanWithinQuery`. Here is an example:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "span_within" : {
-        "little" : {
-            "span_term" : { "field1" : "foo" }
-        },
-        "big" : {
-            "span_near" : { 
-                "clauses" : [
-                    { "span_term" : { "field1" : "bar" } },
-                    { "span_term" : { "field1" : "baz" } }
-                ],
-                "slop" : 5,
-                "in_order" : true
+    "query": {
+        "span_within" : {
+            "little" : {
+                "span_term" : { "field1" : "foo" }
+            },
+            "big" : {
+                "span_near" : { 
+                    "clauses" : [
+                        { "span_term" : { "field1" : "bar" } },
+                        { "span_term" : { "field1" : "baz" } }
+                    ],
+                    "slop" : 5,
+                    "in_order" : true
+                }
             }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 The `big` and `little` clauses can be any span type query. Matching
 spans from `little` that are enclosed within `big` are returned.
diff --git a/docs/reference/query-dsl/template-query.asciidoc b/docs/reference/query-dsl/template-query.asciidoc
index 35365e6f9b2..f66dbe56d3e 100644
--- a/docs/reference/query-dsl/template-query.asciidoc
+++ b/docs/reference/query-dsl/template-query.asciidoc
@@ -19,8 +19,8 @@ GET /_search
         }
     }
 }
-
 ------------------------------------------
+// CONSOLE
 
 The above request is translated into:
 
@@ -34,8 +34,8 @@ GET /_search
         }
     }
 }
-
 ------------------------------------------
+// CONSOLE
 
 Alternatively passing the template as an escaped string works as well:
 
@@ -53,6 +53,8 @@ GET /_search
     }
 }
 ------------------------------------------
+// CONSOLE
+
 <1> New line characters (`\n`) should be escaped as `\\n` or removed,
     and quotes (`"`) should be escaped as `\\"`.
 
@@ -77,6 +79,8 @@ GET /_search
     }
 }
 ------------------------------------------
+// CONSOLE
+
 <1> Name of the query template in `config/scripts/`, i.e., `my_template.mustache`.
 
 Alternatively, you can register a query template in the cluster state with:
@@ -85,9 +89,10 @@ Alternatively, you can register a query template in the cluster state with:
 ------------------------------------------
 PUT /_search/template/my_template
 {
-    "template": { "match": { "text": "{{query_string}}" }},
+    "template": { "match": { "text": "{{query_string}}" }}
 }
 ------------------------------------------
+// CONSOLE
 
 and refer to it in the `template` query with the `id` parameter:
 
@@ -106,9 +111,13 @@ GET /_search
     }
 }
 ------------------------------------------
+// CONSOLE
+// TEST[continued]
+
 <1> Name of the query template in `config/scripts/`, i.e., `my_template.mustache`.
 
 
 There is also a dedicated `template` endpoint, allows you to template an entire search request.
 Please see <<search-template>> for more details.
 
+
diff --git a/docs/reference/query-dsl/terms-query.asciidoc b/docs/reference/query-dsl/terms-query.asciidoc
index 08df1e6fece..e00c18bb56f 100644
--- a/docs/reference/query-dsl/terms-query.asciidoc
+++ b/docs/reference/query-dsl/terms-query.asciidoc
@@ -6,14 +6,18 @@ Filters documents that have fields that match any of the provided terms
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "constant_score" : {
-        "filter" : {
-            "terms" : { "user" : ["kimchy", "elasticsearch"]}
+    "query": {
+        "constant_score" : {
+            "filter" : {
+                "terms" : { "user" : ["kimchy", "elasticsearch"]}
+            }
         }
     }
 }
 --------------------------------------------------
+// CONSOLE
 
 The `terms` query is also aliased with `in` as the filter name for
 simpler usage deprecated[5.0.0,use `terms` instead].
@@ -63,33 +67,37 @@ possible, reducing the need for networking.
 
 [float]
 ===== Terms lookup twitter example
+At first we index the information for user with id 2, specifically, its
+followers, than index a tweet from user with id 1. Finally we search on
+all the tweets that match the followers of user 2.
 
 [source,js]
 --------------------------------------------------
-# index the information for user with id 2, specifically, its followers
-curl -XPUT localhost:9200/users/user/2 -d '{
-   "followers" : ["1", "3"]
-}'
+PUT /users/user/2
+{
+    "followers" : ["1", "3"]
+}
 
-# index a tweet, from user with id 1
-curl -XPUT localhost:9200/tweets/tweet/1 -d '{
-   "user" : "1"
-}'
+PUT /tweets/tweet/1
+{
+    "user" : "1"
+}
 
-# search on all the tweets that match the followers of user 2
-curl -XGET localhost:9200/tweets/_search -d '{
-  "query" : {
-    "terms" : {
-      "user" : {
-        "index" : "users",
-        "type" : "user",
-        "id" : "2",
-        "path" : "followers"
-      }
+GET /tweets/_search
+{
+    "query" : {
+        "terms" : {
+            "user" : {
+                "index" : "users",
+                "type" : "user",
+                "id" : "2",
+                "path" : "followers"
+            }
+        }
     }
-  }
-}'
+}
 --------------------------------------------------
+// CONSOLE
 
 The structure of the external terms document can also include array of
 inner objects, for example:
diff --git a/docs/reference/query-dsl/type-query.asciidoc b/docs/reference/query-dsl/type-query.asciidoc
index d3ef1a30fb1..05e909bc366 100644
--- a/docs/reference/query-dsl/type-query.asciidoc
+++ b/docs/reference/query-dsl/type-query.asciidoc
@@ -5,9 +5,13 @@ Filters documents matching the provided document / mapping type.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "type" : {
-        "value" : "my_type"
+    "query": {
+        "type" : {
+            "value" : "my_type"
+        }
     }
 }    
 --------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/query-dsl/wildcard-query.asciidoc b/docs/reference/query-dsl/wildcard-query.asciidoc
index d72dbec2481..ad82c029d8c 100644
--- a/docs/reference/query-dsl/wildcard-query.asciidoc
+++ b/docs/reference/query-dsl/wildcard-query.asciidoc
@@ -11,28 +11,40 @@ query maps to Lucene `WildcardQuery`.
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "wildcard" : { "user" : "ki*y" }
+    "query": {
+        "wildcard" : { "user" : "ki*y" }
+    }
 }
 --------------------------------------------------
+// CONSOLE
 
 A boost can also be associated with the query:
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "wildcard" : { "user" : { "value" : "ki*y", "boost" : 2.0 } }
+    "query": {
+        "wildcard" : { "user" : { "value" : "ki*y", "boost" : 2.0 } }
+    }
 }
 --------------------------------------------------
+// CONSOLE
 
 Or :
 
 [source,js]
 --------------------------------------------------
+GET /_search
 {
-    "wildcard" : { "user" : { "wildcard" : "ki*y", "boost" : 2.0 } }
+    "query": {
+        "wildcard" : { "user" : { "wildcard" : "ki*y", "boost" : 2.0 } }
+    }
 }
 --------------------------------------------------
+// CONSOLE
 
 This multi term query allows to control how it gets rewritten using the
 <<query-dsl-multi-term-rewrite,rewrite>>

From 1f011f9deaa27a8634bf75d2417bf1c2551d811e Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Mon, 23 May 2016 13:39:36 +0200
Subject: [PATCH 0436/1311] Remove Delete-By-Query plugin

closes #18469
---
 .../resources/checkstyle_suppressions.xml     |   7 -
 .../plugins/InstallPluginCommand.java         |   1 -
 dev-tools/smoke_test_rc.py                    |   1 -
 docs/plugins/api.asciidoc                     |  13 -
 docs/plugins/delete-by-query.asciidoc         | 270 -----------
 docs/plugins/redirects.asciidoc               |   7 +
 docs/reference/docs/delete.asciidoc           |   4 +-
 .../migration/migrate_5_0/plugins.asciidoc    |   5 +
 .../migration/migrate_5_0/rest.asciidoc       |   5 +
 plugins/delete-by-query/build.gradle          |  24 -
 .../deletebyquery/DeleteByQueryAction.java    |  43 --
 .../deletebyquery/DeleteByQueryRequest.java   | 244 ----------
 .../DeleteByQueryRequestBuilder.java          | 103 ----
 .../deletebyquery/DeleteByQueryResponse.java  | 201 --------
 .../IndexDeleteByQueryResponse.java           | 155 ------
 .../TransportDeleteByQueryAction.java         | 347 -------------
 .../deletebyquery/DeleteByQueryPlugin.java    |  51 --
 .../RestDeleteByQueryAction.java              |  77 ---
 .../IndexDeleteByQueryResponseTests.java      | 162 -------
 .../TransportDeleteByQueryActionTests.java    | 458 ------------------
 .../deletebyquery/DeleteByQueryTests.java     | 446 -----------------
 .../test/rest/DeleteByQueryRestIT.java        |  41 --
 .../rest-api-spec/api/delete_by_query.json    |  66 ---
 .../test/delete_by_query/10_basic.yaml        |  58 ---
 .../scripts/module_and_plugin_test_cases.bash |  10 +-
 .../rest-api-spec/api/delete_by_query.json    |   2 +-
 settings.gradle                               |   1 -
 27 files changed, 21 insertions(+), 2781 deletions(-)
 delete mode 100644 docs/plugins/delete-by-query.asciidoc
 delete mode 100644 plugins/delete-by-query/build.gradle
 delete mode 100644 plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryAction.java
 delete mode 100644 plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java
 delete mode 100644 plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequestBuilder.java
 delete mode 100644 plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryResponse.java
 delete mode 100644 plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/IndexDeleteByQueryResponse.java
 delete mode 100644 plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryAction.java
 delete mode 100644 plugins/delete-by-query/src/main/java/org/elasticsearch/plugin/deletebyquery/DeleteByQueryPlugin.java
 delete mode 100644 plugins/delete-by-query/src/main/java/org/elasticsearch/rest/action/deletebyquery/RestDeleteByQueryAction.java
 delete mode 100644 plugins/delete-by-query/src/test/java/org/elasticsearch/action/deletebyquery/IndexDeleteByQueryResponseTests.java
 delete mode 100644 plugins/delete-by-query/src/test/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryActionTests.java
 delete mode 100644 plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/DeleteByQueryTests.java
 delete mode 100644 plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/test/rest/DeleteByQueryRestIT.java
 delete mode 100644 plugins/delete-by-query/src/test/resources/rest-api-spec/api/delete_by_query.json
 delete mode 100644 plugins/delete-by-query/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 4c467039117..90b8aa8423f 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -1225,13 +1225,6 @@
   <suppress files="plugins[/\\]analysis-phonetic[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]PhoneticTokenFilterFactory.java" checks="LineLength" />
   <suppress files="plugins[/\\]analysis-smartcn[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]SimpleSmartChineseAnalysisTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]analysis-stempel[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]PolishAnalysisTests.java" checks="LineLength" />
-  <suppress files="plugins[/\\]delete-by-query[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]deletebyquery[/\\]DeleteByQueryRequest.java" checks="LineLength" />
-  <suppress files="plugins[/\\]delete-by-query[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]deletebyquery[/\\]DeleteByQueryRequestBuilder.java" checks="LineLength" />
-  <suppress files="plugins[/\\]delete-by-query[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]deletebyquery[/\\]DeleteByQueryResponse.java" checks="LineLength" />
-  <suppress files="plugins[/\\]delete-by-query[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]deletebyquery[/\\]TransportDeleteByQueryAction.java" checks="LineLength" />
-  <suppress files="plugins[/\\]delete-by-query[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]deletebyquery[/\\]IndexDeleteByQueryResponseTests.java" checks="LineLength" />
-  <suppress files="plugins[/\\]delete-by-query[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]deletebyquery[/\\]TransportDeleteByQueryActionTests.java" checks="LineLength" />
-  <suppress files="plugins[/\\]delete-by-query[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugin[/\\]deletebyquery[/\\]DeleteByQueryTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]discovery-azure[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]AbstractAzureTestCase.java" checks="LineLength" />
   <suppress files="plugins[/\\]discovery-azure[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]azure[/\\]AzureMinimumMasterNodesTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]discovery-azure[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]azure[/\\]AzureSimpleTests.java" checks="LineLength" />
diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
index 645d07bfb64..1b98128b85e 100644
--- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
@@ -118,7 +118,6 @@ class InstallPluginCommand extends SettingCommand {
             "analysis-phonetic",
             "analysis-smartcn",
             "analysis-stempel",
-            "delete-by-query",
             "discovery-azure",
             "discovery-ec2",
             "discovery-gce",
diff --git a/dev-tools/smoke_test_rc.py b/dev-tools/smoke_test_rc.py
index ac5e9afec47..63153b45f1b 100644
--- a/dev-tools/smoke_test_rc.py
+++ b/dev-tools/smoke_test_rc.py
@@ -63,7 +63,6 @@ DEFAULT_PLUGINS = ["analysis-icu",
                    "analysis-phonetic",
                    "analysis-smartcn",
                    "analysis-stempel",
-                   "delete-by-query",
                    "discovery-azure",
                    "discovery-ec2",
                    "discovery-gce",
diff --git a/docs/plugins/api.asciidoc b/docs/plugins/api.asciidoc
index fc5ad3ced5e..54edcbc7f0e 100644
--- a/docs/plugins/api.asciidoc
+++ b/docs/plugins/api.asciidoc
@@ -3,18 +3,6 @@
 
 API extension plugins add new functionality to Elasticsearch by adding new APIs or features, usually to do with search or mapping.
 
-[float]
-=== Core API extension plugins
-
-The core API extension plugins are:
-
-<<plugins-delete-by-query,Delete by Query>>::
-
-The delete by query plugin adds support for deleting all of the documents
-(from one or more indices) which match the specified query. It is a
-replacement for the problematic _delete-by-query_ functionality which has been
-removed from Elasticsearch core.
-
 [float]
 === Community contributed API extension plugins
 
@@ -46,4 +34,3 @@ A number of plugins have been contributed by our community:
   http://mahout.apache.org/[Mahout] Collaboration filtering (by hadashiA)
 * https://github.com/jurgc11/es-change-feed-plugin[WebSocket Change Feed Plugin] (by ForgeRock/Chris Clifton)
 
-include::delete-by-query.asciidoc[]
diff --git a/docs/plugins/delete-by-query.asciidoc b/docs/plugins/delete-by-query.asciidoc
deleted file mode 100644
index aebc0dd7433..00000000000
--- a/docs/plugins/delete-by-query.asciidoc
+++ /dev/null
@@ -1,270 +0,0 @@
-[[plugins-delete-by-query]]
-=== Delete By Query Plugin
-
-The delete-by-query plugin adds support for deleting all of the documents
-(from one or more indices) which match the specified query. It is a
-replacement for the problematic _delete-by-query_ functionality which has been
-removed from Elasticsearch core.
-
-Internally, it uses {ref}/search-request-scroll.html[Scroll]
-and {ref}/docs-bulk.html[Bulk] APIs to delete documents in an efficient and
-safe manner. It is slower than the old _delete-by-query_ functionality, but
-fixes the problems with the previous implementation.
-
-To understand more about why we removed delete-by-query from core and about
-the semantics of the new implementation, see
-<<delete-by-query-plugin-reason>>.
-
-[TIP]
-============================================
-Queries which match large numbers of documents may run for a long time,
-as every document has to be deleted individually.  Don't use _delete-by-query_
-to clean out all or most documents in an index.  Rather create a new index and
-perhaps reindex the documents you want to keep.
-============================================
-
-[float]
-==== Installation
-
-This plugin can be installed using the plugin manager:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install delete-by-query
-----------------------------------------------------------------
-
-The plugin must be installed on every node in the cluster, and each node must
-be restarted after installation.
-
-[float]
-==== Removal
-
-The plugin can be removed with the following command:
-
-[source,sh]
-----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove delete-by-query
-----------------------------------------------------------------
-
-The node must be stopped before removing the plugin.
-
-[[delete-by-query-usage]]
-==== Using Delete-by-Query
-
-The query can either be provided using a simple query string as
-a parameter:
-
-[source,shell]
---------------------------------------------------
-DELETE /twitter/tweet/_query?q=user:kimchy
---------------------------------------------------
-// CONSOLE
-
-or using the {ref}/query-dsl.html[Query DSL] defined within the request body:
-
-[source,js]
---------------------------------------------------
-DELETE /twitter/tweet/_query
-{
-  "query": { <1>
-    "term": {
-      "user": "kimchy"
-    }
-  }
-}
---------------------------------------------------
-// CONSOLE
-
-<1> The query must be passed as a value to the `query` key, in the same way as
-the {ref}/search-search.html[search api].
-
-Both of the above examples end up doing the same thing, which is to delete all
-tweets from the twitter index for the user `kimchy`.
-
-Delete-by-query supports deletion across
-{ref}/search-search.html#search-multi-index-type[multiple indices and multiple types].
-
-[float]
-=== Query-string parameters
-
-The following query string parameters are supported:
-
-`q`::
-
-Instead of using the {ref}/query-dsl.html[Query DSL] to pass a `query` in the request
-body, you can use the `q` query string parameter to  specify a query using
-{ref}/query-dsl-query-string-query.html#query-string-syntax[`query_string` syntax].
-In this case, the following additional parameters are supported: `df`,
-`analyzer`, `default_operator`,  `lowercase_expanded_terms`,
-`analyze_wildcard` and `lenient`.
-See {ref}/search-uri-request.html[URI search request] for details.
-
-`size`::
-
-The number of hits returned by the {ref}/search-request-scroll.html[scroll]
-request.  Defaults to 10.  May also be specified in the request body.
-
-`timeout`::
-
-The maximum execution time of the delete by query process. Once expired, no
-more documents will be deleted.
-
-`routing`::
-
-A comma separated list of routing values to control which shards the delete by
-query request should be executed on.
-
-When using the `q` parameter, the following additional parameters are
-supported (as explained in {ref}/search-uri-request.html[URI search request]): `df`, `analyzer`,
-`default_operator`.
-
-
-[float]
-=== Response body
-
-The JSON response looks like this:
-
-[source,js]
---------------------------------------------------
-{
-  "took" : 639,
-  "timed_out" : false,
-  "_indices" : {
-    "_all" : {
-      "found" : 5901,
-      "deleted" : 5901,
-      "missing" : 0,
-      "failed" : 0
-    },
-    "twitter" : {
-      "found" : 5901,
-      "deleted" : 5901,
-      "missing" : 0,
-      "failed" : 0
-    }
-  },
-  "failures" : [ ]
-}
---------------------------------------------------
-
-Internally, the query is used to execute an initial
-{ref}/search-request-scroll.html[scroll] request. As hits are
-pulled from the scroll API, they are passed to the {ref}/docs-bulk.html[Bulk
-API] for deletion.
-
-IMPORTANT: Delete by query will only delete the version of the document that
-was visible to search at the time the request was executed.  Any documents
-that have been reindexed or updated during execution will not be deleted.
-
-Since documents can be updated or deleted by external operations during the
-_scroll-bulk_ process, the plugin keeps track of different counters for
-each index, with the totals displayed under the `_all` index.  The counters
-are as follows:
-
-`found`::
-
-The number of documents matching the query for the given index.
-
-`deleted`::
-
-The number of documents successfully deleted for the given index.
-
-`missing`::
-
-The number of documents that were missing when the plugin tried to delete
-them. Missing documents were present when the original query was run, but have
-already been deleted by another process.
-
-`failed`::
-
-The number of documents that failed to be deleted for the given index. A
-document may fail to be deleted if it has been updated to a new version by
-another process, or if the shard containing the document has gone missing due
-to hardware failure, for example.
-
-[[delete-by-query-plugin-reason]]
-==== Why Delete-By-Query is a plugin
-
-The old delete-by-query API in Elasticsearch 1.x was fast but problematic. We
-decided to remove the feature from Elasticsearch for these reasons:
-
-Forward compatibility::
-
-    The old implementation wrote a delete-by-query request, including the
-    query, to the transaction log.  This meant that, when upgrading to a new
-    version, old unsupported queries which cannot be executed might exist in
-    the translog, thus causing data corruption.
-
-Consistency and correctness::
-
-    The old implementation executed the query and deleted all matching docs on
-    the primary first.  It then repeated this procedure on each replica shard.
-    There was no guarantee that the queries on the primary and the replicas
-    matched the same document, so it was quite possible to end up with
-    different documents on each shard copy.
-
-Resiliency::
-
-    The old implementation could cause out-of-memory exceptions, merge storms,
-    and dramatic slow downs if used incorrectly.
-
-[float]
-=== New delete-by-query implementation
-
-The new implementation, provided by this plugin, is built internally
-using  {ref}/search-request-scroll.html[scroll] to return
-the document IDs and versions of all the documents that need to be deleted.
-It then uses  the {ref}/docs-bulk.html[`bulk` API] to do the actual deletion.
-
-This can have performance as well as visibility implications. Delete-by-query
-now has the following semantics:
-
-non-atomic::
-
-    A delete-by-query may fail at any time while some documents matching the
-    query have already been deleted.
-
-try-once::
-
-    A delete-by-query may fail at any time and will not retry it's execution.
-    All retry logic is left to the user.
-
-syntactic sugar::
-
-    A delete-by-query is equivalent to a scroll search ordered by `_doc` and
-    corresponding bulk-deletes by ID.
-
-point-in-time::
-
-    A delete-by-query will only delete the documents that are visible at the
-    point in time the delete-by-query was started, equivalent to the
-    scan/scroll API.
-
-consistent::
-
-    A delete-by-query will yield consistent results across all replicas of a
-    shard.
-
-forward-compatible::
-
-    A delete-by-query will only send IDs to the shards as deletes such that no
-    queries are stored in the transaction logs that might not be supported in
-    the future.
-
-visibility::
-
-    The effect of a delete-by-query request will not be visible to search
-    until the user refreshes the index, or the index is refreshed
-    automatically.
-
-The new implementation suffers from two issues, which is why we decided to
-move the functionality to a plugin instead of replacing the feautre in core:
-
-* It is not as fast as the previous implementation. For most use cases, this
-  difference should not be noticeable but users running delete-by-query on
-  many matching documents may be affected.
-
-* There is currently no way to monitor or cancel a running delete-by-query
-  request, except for the `timeout` parameter.
-
-We have plans to solve both of these issues in a later version of Elasticsearch.
diff --git a/docs/plugins/redirects.asciidoc b/docs/plugins/redirects.asciidoc
index caf2008e521..c8cf10c6319 100644
--- a/docs/plugins/redirects.asciidoc
+++ b/docs/plugins/redirects.asciidoc
@@ -33,6 +33,13 @@ The `cloud-azure` plugin has been split into two separate plugins:
 
 The `cloud-gce` plugin has been renamed to <<discovery-gce>> (`discovery-gce`).
 
+[role="exclude",id="plugins-delete-by-query"]
+=== Delete-By-Query plugin removed
+
+The Delete-By-Query plugin has been removed in favor of a new {ref}/docs-delete-by-query.html[Delete By Query API]
+implementation in core.
+
+
 
 
 
diff --git a/docs/reference/docs/delete.asciidoc b/docs/reference/docs/delete.asciidoc
index f36ffe7abf9..175c07d005e 100644
--- a/docs/reference/docs/delete.asciidoc
+++ b/docs/reference/docs/delete.asciidoc
@@ -70,8 +70,8 @@ setting the routing parameter.
 
 Note that deleting a parent document does not automatically delete its
 children. One way of deleting all child documents given a parent's id is
-to use the `delete-by-query` plugin to perform a delete on the child
-index with the automatically generated (and indexed)
+to use the <<docs-delete-by-query,Delete By Query API>> to perform a
+ index with the automatically generated (and indexed)
 field _parent, which is in the format parent_type#parent_id.
 
 [float]
diff --git a/docs/reference/migration/migrate_5_0/plugins.asciidoc b/docs/reference/migration/migrate_5_0/plugins.asciidoc
index 1ed45b9b596..ae1113caa48 100644
--- a/docs/reference/migration/migrate_5_0/plugins.asciidoc
+++ b/docs/reference/migration/migrate_5_0/plugins.asciidoc
@@ -95,6 +95,11 @@ cloud:
 
 Cloud GCE plugin has been renamed to {plugins}/discovery-gce.html[Discovery GCE plugin].
 
+==== Delete-By-Query plugin removed
+
+The Delete-By-Query plugin has been removed in favor of a new <<docs-delete-by-query,Delete By Query API>>
+implementation in core. It now supports throttling, retries and cancellation but no longer supports timeouts.
+Instead use the <<docs-delete-by-query-cancel-task-api,cancel API>> to cancel deletes that run too long.
 
 ==== Mapper Attachments plugin deprecated
 
diff --git a/docs/reference/migration/migrate_5_0/rest.asciidoc b/docs/reference/migration/migrate_5_0/rest.asciidoc
index 200d76e45ca..23928551f77 100644
--- a/docs/reference/migration/migrate_5_0/rest.asciidoc
+++ b/docs/reference/migration/migrate_5_0/rest.asciidoc
@@ -57,3 +57,8 @@ removed in Elasticsearch 6.0.0.
 
 The deprecated `filters`/`token_filters`/`char_filters` parameter has been
 renamed `filter`/`token_filter`/`char_filter`.
+
+==== `DELETE /_query` endpoint removed
+
+The `DELETE /_query` endpoint provided by the Delete-By-Query plugin has been
+removed and replaced by the <<docs-delete-by-query,Delete By Query API>>.
diff --git a/plugins/delete-by-query/build.gradle b/plugins/delete-by-query/build.gradle
deleted file mode 100644
index 2a5d00519e2..00000000000
--- a/plugins/delete-by-query/build.gradle
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-esplugin {
-  description 'The Delete By Query plugin allows to delete documents in Elasticsearch with a single query.'
-  classname 'org.elasticsearch.plugin.deletebyquery.DeleteByQueryPlugin'
-}
-
diff --git a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryAction.java b/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryAction.java
deleted file mode 100644
index d77da6e6d3b..00000000000
--- a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryAction.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.deletebyquery;
-
-import org.elasticsearch.action.Action;
-import org.elasticsearch.client.ElasticsearchClient;
-
-public class DeleteByQueryAction extends Action<DeleteByQueryRequest, DeleteByQueryResponse, DeleteByQueryRequestBuilder> {
-
-    public static final DeleteByQueryAction INSTANCE = new DeleteByQueryAction();
-    public static final String NAME = "indices:data/write/delete/by_query";
-
-    private DeleteByQueryAction() {
-        super(NAME);
-    }
-
-    @Override
-    public DeleteByQueryResponse newResponse() {
-        return new DeleteByQueryResponse();
-    }
-
-    @Override
-    public DeleteByQueryRequestBuilder newRequestBuilder(ElasticsearchClient client) {
-        return new DeleteByQueryRequestBuilder(client, this);
-    }
-}
diff --git a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java b/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java
deleted file mode 100644
index 682fec46c3b..00000000000
--- a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequest.java
+++ /dev/null
@@ -1,244 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.deletebyquery;
-
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.IndicesRequest;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.search.Scroll;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-import static org.elasticsearch.action.ValidateActions.addValidationError;
-import static org.elasticsearch.search.Scroll.readScroll;
-
-/**
- * Creates a new {@link DeleteByQueryRequest}. Delete-by-query is since elasticsearch 2.0.0 moved into a plugin
- * and is not part of elasticsearch core. In contrast to the previous, in-core, implementation delete-by-query now
- * uses scan/scroll and the returned IDs do delete all documents matching the query. This can have performance
- * as well as visibility implications. Delete-by-query now has the following semantics:
- * <ul>
- *     <li>it's <tt>non-actomic</tt>, a delete-by-query may fail at any time while some documents matching the query have already been deleted</li>
- *     <li>it's <tt>try-once</tt>, a delete-by-query may fail at any time and will not retry it's execution. All retry logic is left to the user</li>
- *     <li>it's <tt>syntactic sugar</tt>, a delete-by-query is equivalent to a scan/scroll search and corresponding bulk-deletes by ID</li>
- *     <li>it's executed on a <tt>point-in-time</tt> snapshot, a delete-by-query will only delete the documents that are visible at the point in time the delete-by-query was started, equivalent to the scan/scroll API</li>
- *     <li>it's <tt>consistent</tt>, a delete-by-query will yield consistent results across all replicas of a shard</li>
- *     <li>it's <tt>forward-compativle</tt>, a delete-by-query will only send IDs to the shards as deletes such that no queries are stored in the transaction logs that might not be supported in the future.</li>
- *     <li>it's results won't be visible until the user refreshes the index.</li>
- * </ul>
- *
- * The main reason why delete-by-query is now extracted as a plugin are:
- * <ul>
- *     <li><tt>forward-compatibility</tt>, the previous implementation was prone to store unsupported queries in the transaction logs which is equvalent to data-loss</li>
- *     <li><tt>consistency &amp; correctness</tt>, the previous implementation was prone to produce different results on a shards replica which can essentially result in a corrupted index</li>
- *     <li><tt>resiliency</tt>, the previous implementation could cause OOM errors, merge-storms and dramatic slowdowns if used incorrectly</li>
- * </ul>
- *
- * While delete-by-query is a very useful feature, it's implementation is very tricky in system that is based on per-document modifications. The move towards
- * a plugin based solution was mainly done to minimize the risk of cluster failures or corrupted indices which where easily possible wiht the previous implementation.
- * Users that rely delete by query should install the plugin in oder to use this functionality.
- */
-public class DeleteByQueryRequest extends ActionRequest<DeleteByQueryRequest> implements IndicesRequest.Replaceable {
-
-    private String[] indices = Strings.EMPTY_ARRAY;
-    private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, false, true, false);
-
-    private String[] types = Strings.EMPTY_ARRAY;
-
-    private QueryBuilder query;
-
-    private String routing;
-
-    private int size = 0;
-
-    private Scroll scroll = new Scroll(TimeValue.timeValueMinutes(10));
-
-    private TimeValue timeout;
-
-    public DeleteByQueryRequest() {
-    }
-
-    /**
-     * Constructs a new delete by query request to run against the provided indices. No indices means
-     * it will run against all indices.
-     */
-    public DeleteByQueryRequest(String... indices) {
-        this.indices = indices;
-    }
-
-    @Override
-    public ActionRequestValidationException validate() {
-        ActionRequestValidationException validationException = null;
-        if (query == null) {
-            validationException = addValidationError("source is missing", validationException);
-        }
-        return validationException;
-    }
-
-    @Override
-    public String[] indices() {
-        return this.indices;
-    }
-
-    @Override
-    public DeleteByQueryRequest indices(String... indices) {
-        this.indices = indices;
-        return this;
-    }
-
-    @Override
-    public IndicesOptions indicesOptions() {
-        return indicesOptions;
-    }
-
-    public DeleteByQueryRequest indicesOptions(IndicesOptions indicesOptions) {
-        if (indicesOptions == null) {
-            throw new IllegalArgumentException("IndicesOptions must not be null");
-        }
-        this.indicesOptions = indicesOptions;
-        return this;
-    }
-
-    public String[] types() {
-        return this.types;
-    }
-
-    public DeleteByQueryRequest types(String... types) {
-        this.types = types;
-        return this;
-    }
-
-    public QueryBuilder query() {
-        return query;
-    }
-
-    public DeleteByQueryRequest query(QueryBuilder queryBuilder) {
-        this.query = queryBuilder;
-        return this;
-    }
-
-    public String routing() {
-        return this.routing;
-    }
-
-    public DeleteByQueryRequest routing(String routing) {
-        this.routing = routing;
-        return this;
-    }
-
-    public DeleteByQueryRequest routing(String... routings) {
-        this.routing = Strings.arrayToCommaDelimitedString(routings);
-        return this;
-    }
-
-    public DeleteByQueryRequest size(int size) {
-        if (size < 0) {
-            throw new IllegalArgumentException("size must be greater than zero");
-        }
-        this.size = size;
-        return this;
-    }
-
-    public int size() {
-        return size;
-    }
-
-
-    public Scroll scroll() {
-        return scroll;
-    }
-
-    public DeleteByQueryRequest scroll(Scroll scroll) {
-        this.scroll = scroll;
-        return this;
-    }
-
-    public DeleteByQueryRequest scroll(TimeValue keepAlive) {
-        return scroll(new Scroll(keepAlive));
-    }
-
-    public DeleteByQueryRequest scroll(String keepAlive) {
-        return scroll(new Scroll(TimeValue.parseTimeValue(keepAlive, null, getClass().getSimpleName() + ".keepAlive")));
-    }
-
-    public TimeValue timeout() {
-        return timeout;
-    }
-
-    public DeleteByQueryRequest timeout(TimeValue timeout) {
-        if (timeout == null) {
-            throw new IllegalArgumentException("timeout must not be null");
-        }
-        this.timeout = timeout;
-        return this;
-    }
-
-    public DeleteByQueryRequest timeout(String timeout) {
-        timeout(TimeValue.parseTimeValue(timeout, null, getClass().getSimpleName() + ".timeout"));
-        return this;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        indices = in.readStringArray();
-        indicesOptions = IndicesOptions.readIndicesOptions(in);
-        types = in.readStringArray();
-        query = in.readNamedWriteable(QueryBuilder.class);
-        routing = in.readOptionalString();
-        size = in.readVInt();
-        if (in.readBoolean()) {
-            scroll = readScroll(in);
-        }
-        if (in.readBoolean()) {
-            timeout = TimeValue.readTimeValue(in);
-        }
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeStringArray(indices);
-        indicesOptions.writeIndicesOptions(out);
-        out.writeStringArray(types);
-        out.writeNamedWriteable(query);
-        out.writeOptionalString(routing);
-        out.writeVInt(size);
-        out.writeOptionalStreamable(scroll);
-        out.writeOptionalStreamable(timeout);
-    }
-
-    @Override
-    public String toString() {
-        return "delete-by-query indices:" + Arrays.toString(indices) +
-                ", types:" + Arrays.toString(types) +
-                ", size:" + size +
-                ", timeout:" + timeout +
-                ", routing:" + routing +
-                ", query:" + query;
-    }
-}
diff --git a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequestBuilder.java b/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequestBuilder.java
deleted file mode 100644
index dc5ba3a15fb..00000000000
--- a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryRequestBuilder.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.deletebyquery;
-
-import org.elasticsearch.action.ActionRequestBuilder;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.index.query.QueryBuilder;
-
-/**
- * Creates a new {@link DeleteByQueryRequestBuilder}
- * @see DeleteByQueryRequest
- */
-public class DeleteByQueryRequestBuilder extends ActionRequestBuilder<DeleteByQueryRequest, DeleteByQueryResponse, DeleteByQueryRequestBuilder> {
-
-    public DeleteByQueryRequestBuilder(ElasticsearchClient client, DeleteByQueryAction action) {
-        super(client, action, new DeleteByQueryRequest());
-    }
-
-    public DeleteByQueryRequestBuilder setIndices(String... indices) {
-        request.indices(indices);
-        return this;
-    }
-
-    /**
-     * Specifies what type of requested indices to ignore and wildcard indices expressions.
-     * <p>
-     * For example indices that don't exist.
-     */
-    public DeleteByQueryRequestBuilder setIndicesOptions(IndicesOptions options) {
-        request.indicesOptions(options);
-        return this;
-    }
-
-    /**
-     * The query used to delete documents.
-     *
-     * @see org.elasticsearch.index.query.QueryBuilders
-     */
-    public DeleteByQueryRequestBuilder setQuery(QueryBuilder queryBuilder) {
-        request.query(queryBuilder);
-        return this;
-    }
-
-    /**
-     * A comma separated list of routing values to control the shards the action will be executed on.
-     */
-    public DeleteByQueryRequestBuilder setRouting(String routing) {
-        request.routing(routing);
-        return this;
-    }
-
-    /**
-     * The routing values to control the shards that the action will be executed on.
-     */
-    public DeleteByQueryRequestBuilder setRouting(String... routing) {
-        request.routing(routing);
-        return this;
-    }
-
-    /**
-     * An optional timeout to control how long the delete by query is allowed to take.
-     */
-    public DeleteByQueryRequestBuilder setTimeout(TimeValue timeout) {
-        request.timeout(timeout);
-        return this;
-    }
-
-    /**
-     * An optional timeout to control how long the delete by query is allowed to take.
-     */
-    public DeleteByQueryRequestBuilder setTimeout(String timeout) {
-        request.timeout(timeout);
-        return this;
-    }
-
-    /**
-     * The types of documents the query will run against. Defaults to all types.
-     */
-    public DeleteByQueryRequestBuilder setTypes(String... types) {
-        request.types(types);
-        return this;
-    }
-
-}
diff --git a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryResponse.java b/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryResponse.java
deleted file mode 100644
index 80fae396a25..00000000000
--- a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/DeleteByQueryResponse.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.deletebyquery;
-
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.action.ShardOperationFailedException;
-import org.elasticsearch.action.search.ShardSearchFailure;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-
-import java.io.IOException;
-
-import static org.elasticsearch.action.search.ShardSearchFailure.readShardSearchFailure;
-
-/**
- * Delete by query response
- * @see DeleteByQueryRequest
- */
-public class DeleteByQueryResponse extends ActionResponse implements ToXContent {
-
-    private long tookInMillis;
-    private boolean timedOut = false;
-
-    private long found;
-    private long deleted;
-    private long missing;
-    private long failed;
-
-    private IndexDeleteByQueryResponse[] indices = IndexDeleteByQueryResponse.EMPTY_ARRAY;
-    private ShardOperationFailedException[] shardFailures = ShardSearchFailure.EMPTY_ARRAY;
-
-    DeleteByQueryResponse() {
-    }
-
-    DeleteByQueryResponse(long tookInMillis, boolean timedOut, long found, long deleted, long missing, long failed, IndexDeleteByQueryResponse[] indices, ShardOperationFailedException[] shardFailures) {
-        this.tookInMillis = tookInMillis;
-        this.timedOut = timedOut;
-        this.found = found;
-        this.deleted = deleted;
-        this.missing = missing;
-        this.failed = failed;
-        this.indices = indices;
-        this.shardFailures = shardFailures;
-    }
-
-    /**
-     * The responses from all the different indices.
-     */
-    public IndexDeleteByQueryResponse[] getIndices() {
-        return indices;
-    }
-
-    /**
-     * The response of a specific index.
-     */
-    public IndexDeleteByQueryResponse getIndex(String index) {
-        if (index == null) {
-            return null;
-        }
-        for (IndexDeleteByQueryResponse i : indices) {
-            if (index.equals(i.getIndex())) {
-                return i;
-            }
-        }
-        return null;
-    }
-
-    public TimeValue getTook() {
-        return new TimeValue(tookInMillis);
-    }
-
-    public long getTookInMillis() {
-        return tookInMillis;
-    }
-
-    public boolean isTimedOut() {
-        return this.timedOut;
-    }
-
-    public long getTotalFound() {
-        return found;
-    }
-
-    public long getTotalDeleted() {
-        return deleted;
-    }
-
-    public long getTotalMissing() {
-        return missing;
-    }
-
-    public long getTotalFailed() {
-        return failed;
-    }
-
-    public ShardOperationFailedException[] getShardFailures() {
-        return shardFailures;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        tookInMillis = in.readVLong();
-        timedOut = in.readBoolean();
-        found = in.readVLong();
-        deleted = in.readVLong();
-        missing = in.readVLong();
-        failed = in.readVLong();
-
-        int size = in.readVInt();
-        indices = new IndexDeleteByQueryResponse[size];
-        for (int i = 0; i < size; i++) {
-            IndexDeleteByQueryResponse index = new IndexDeleteByQueryResponse();
-            index.readFrom(in);
-            indices[i] = index;
-        }
-
-        size = in.readVInt();
-        if (size == 0) {
-            shardFailures = ShardSearchFailure.EMPTY_ARRAY;
-        } else {
-            shardFailures = new ShardSearchFailure[size];
-            for (int i = 0; i < shardFailures.length; i++) {
-                shardFailures[i] = readShardSearchFailure(in);
-            }
-        }
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeVLong(tookInMillis);
-        out.writeBoolean(timedOut);
-        out.writeVLong(found);
-        out.writeVLong(deleted);
-        out.writeVLong(missing);
-        out.writeVLong(failed);
-
-        out.writeVInt(indices.length);
-        for (IndexDeleteByQueryResponse indexResponse : indices) {
-            indexResponse.writeTo(out);
-        }
-
-        out.writeVInt(shardFailures.length);
-        for (ShardOperationFailedException shardSearchFailure : shardFailures) {
-            shardSearchFailure.writeTo(out);
-        }
-    }
-
-    static final class Fields {
-        static final String TOOK = "took";
-        static final String TIMED_OUT = "timed_out";
-        static final String INDICES = "_indices";
-        static final String FAILURES = "failures";
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(Fields.TOOK, tookInMillis);
-        builder.field(Fields.TIMED_OUT, timedOut);
-
-        builder.startObject(Fields.INDICES);
-        IndexDeleteByQueryResponse all = new IndexDeleteByQueryResponse("_all", found, deleted, missing, failed);
-        all.toXContent(builder, params);
-        for (IndexDeleteByQueryResponse indexResponse : indices) {
-            indexResponse.toXContent(builder, params);
-        }
-        builder.endObject();
-
-        builder.startArray(Fields.FAILURES);
-        if (shardFailures != null) {
-            for (ShardOperationFailedException shardFailure : shardFailures) {
-                builder.startObject();
-                shardFailure.toXContent(builder, params);
-                builder.endObject();
-            }
-        }
-        builder.endArray();
-        return builder;
-    }
-}
diff --git a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/IndexDeleteByQueryResponse.java b/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/IndexDeleteByQueryResponse.java
deleted file mode 100644
index 78fca1ef5f1..00000000000
--- a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/IndexDeleteByQueryResponse.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.deletebyquery;
-
-import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-
-import java.io.IOException;
-
-/**
- * Delete by query response executed on a specific index.
- */
-public class IndexDeleteByQueryResponse extends ActionResponse implements ToXContent {
-
-    public static final IndexDeleteByQueryResponse[] EMPTY_ARRAY = new IndexDeleteByQueryResponse[0];
-
-    private String index;
-
-    private long found = 0L;
-    private long deleted = 0L;
-    private long missing = 0L;
-    private long failed = 0L;
-
-    IndexDeleteByQueryResponse() {
-    }
-
-    IndexDeleteByQueryResponse(String index) {
-        this.index = index;
-    }
-
-    /**
-     * Instantiates an IndexDeleteByQueryResponse with given values for counters. Counters should not be negative.
-     */
-    public IndexDeleteByQueryResponse(String index, long found, long deleted, long missing, long failed) {
-        this(index);
-        incrementFound(found);
-        incrementDeleted(deleted);
-        incrementMissing(missing);
-        incrementFailed(failed);
-    }
-
-    public String getIndex() {
-        return this.index;
-    }
-
-    public long getFound() {
-        return found;
-    }
-
-    public void incrementFound() {
-        incrementFound(1L);
-    }
-
-    public void incrementFound(long delta) {
-        assert (found + delta >= 0) : "counter 'found' cannot be negative";
-        this.found = found + delta;
-    }
-
-    public long getDeleted() {
-        return deleted;
-    }
-
-    public void incrementDeleted() {
-        incrementDeleted(1L);
-    }
-
-    public void incrementDeleted(long delta) {
-        assert (deleted + delta >= 0) : "counter 'deleted' cannot be negative";
-        this.deleted = deleted + delta;
-    }
-
-    public long getMissing() {
-        return missing;
-    }
-
-    public void incrementMissing() {
-        incrementMissing(1L);
-    }
-
-    public void incrementMissing(long delta) {
-        assert (missing + delta >= 0) : "counter 'missing' cannot be negative";
-        this.missing = missing + delta;
-    }
-
-    public long getFailed() {
-        return failed;
-    }
-
-    public void incrementFailed() {
-        incrementFailed(1L);
-    }
-
-    public void incrementFailed(long delta) {
-        assert (failed + delta >= 0) : "counter 'failed' cannot be negative";
-        this.failed = failed + delta;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        index = in.readString();
-        found = in.readVLong();
-        deleted = in.readVLong();
-        missing = in.readVLong();
-        failed = in.readVLong();
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        out.writeString(index);
-        out.writeVLong(found);
-        out.writeVLong(deleted);
-        out.writeVLong(missing);
-        out.writeVLong(failed);
-    }
-
-    static final class Fields {
-        static final String FOUND = "found";
-        static final String DELETED = "deleted";
-        static final String MISSING = "missing";
-        static final String FAILED = "failed";
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(index);
-        builder.field(Fields.FOUND, found);
-        builder.field(Fields.DELETED, deleted);
-        builder.field(Fields.MISSING, missing);
-        builder.field(Fields.FAILED, failed);
-        builder.endObject();
-        return builder;
-    }
-}
diff --git a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryAction.java b/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryAction.java
deleted file mode 100644
index f4127c4e532..00000000000
--- a/plugins/delete-by-query/src/main/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryAction.java
+++ /dev/null
@@ -1,347 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.deletebyquery;
-
-import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ShardOperationFailedException;
-import org.elasticsearch.action.bulk.BulkItemResponse;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.delete.DeleteResponse;
-import org.elasticsearch.action.search.ClearScrollRequest;
-import org.elasticsearch.action.search.ClearScrollResponse;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchScrollRequest;
-import org.elasticsearch.action.search.ShardSearchFailure;
-import org.elasticsearch.action.search.TransportSearchAction;
-import org.elasticsearch.action.search.TransportSearchScrollAction;
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.CollectionUtils;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.SearchHitField;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportService;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * Delete-By-Query implementation that uses efficient scrolling and bulks deletions to delete large set of documents.
- */
-public class TransportDeleteByQueryAction extends HandledTransportAction<DeleteByQueryRequest, DeleteByQueryResponse> {
-
-    private final TransportSearchAction searchAction;
-    private final TransportSearchScrollAction scrollAction;
-    private final Client client;
-
-    @Inject
-    public TransportDeleteByQueryAction(Settings settings, ThreadPool threadPool, Client client,
-                                           TransportSearchAction transportSearchAction,
-                                           TransportSearchScrollAction transportSearchScrollAction,
-                                           TransportService transportService, ActionFilters actionFilters,
-                                           IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, DeleteByQueryAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, DeleteByQueryRequest::new);
-        this.searchAction = transportSearchAction;
-        this.scrollAction = transportSearchScrollAction;
-        this.client = client;
-    }
-
-    @Override
-    protected void doExecute(DeleteByQueryRequest request, ActionListener<DeleteByQueryResponse> listener) {
-        new AsyncDeleteByQueryAction(request, listener).start();
-    }
-
-    class AsyncDeleteByQueryAction {
-
-        private final DeleteByQueryRequest request;
-        private final ActionListener<DeleteByQueryResponse> listener;
-
-        private final long startTime;
-        private final AtomicBoolean timedOut;
-        private final AtomicLong total;
-
-        private volatile ShardOperationFailedException[] shardFailures;
-        private final Map<String, IndexDeleteByQueryResponse> results;
-
-        AsyncDeleteByQueryAction(DeleteByQueryRequest request, ActionListener<DeleteByQueryResponse> listener) {
-            this.request = request;
-            this.listener = listener;
-            this.startTime = threadPool.estimatedTimeInMillis();
-            this.timedOut = new AtomicBoolean(false);
-            this.total = new AtomicLong(0L);
-            this.shardFailures = ShardSearchFailure.EMPTY_ARRAY;
-            this.results = new HashMap<>();
-        }
-
-        public void start() {
-            executeScan();
-        }
-
-        void executeScan() {
-            try {
-                final SearchRequest scanRequest = new SearchRequest()
-                        .indices(request.indices())
-                        .types(request.types())
-                        .indicesOptions(request.indicesOptions())
-                        .scroll(request.scroll());
-                if (request.routing() != null) {
-                    scanRequest.routing(request.routing());
-                }
-
-                List<String> fields = new ArrayList<>();
-                fields.add("_routing");
-                fields.add("_parent");
-                SearchSourceBuilder source = new SearchSourceBuilder()
-                        .query(request.query())
-                        .fields(fields)
-                        .sort("_doc") // important for performance
-                        .fetchSource(false)
-                        .version(true);
-                if (request.size() > 0) {
-                    source.size(request.size());
-                }
-                if (request.timeout() != null) {
-                    source.timeout(request.timeout());
-                }
-                scanRequest.source(source);
-
-                logger.trace("executing scan request");
-                searchAction.execute(scanRequest, new ActionListener<SearchResponse>() {
-                    @Override
-                    public void onResponse(SearchResponse searchResponse) {
-                        long hits = searchResponse.getHits().getTotalHits();
-                        logger.trace("first request executed: found [{}] document(s) to delete", hits);
-                        total.set(hits);
-                        deleteHits(null, searchResponse);
-                    }
-
-                    @Override
-                    public void onFailure(Throwable e) {
-                        listener.onFailure(e);
-                    }
-                });
-            } catch (Throwable t) {
-                logger.error("unable to execute the initial scan request of delete by query", t);
-                listener.onFailure(t);
-            }
-        }
-
-        void executeScroll(final String scrollId) {
-            try {
-                logger.trace("executing scroll request [{}]", scrollId);
-                scrollAction.execute(new SearchScrollRequest().scrollId(scrollId).scroll(request.scroll()), new ActionListener<SearchResponse>() {
-                    @Override
-                    public void onResponse(SearchResponse scrollResponse) {
-                        deleteHits(scrollId, scrollResponse);
-                    }
-
-                    @Override
-                    public void onFailure(Throwable e) {
-                        logger.error("scroll request [{}] failed, scrolling document(s) is stopped", e, scrollId);
-                        finishHim(scrollId, hasTimedOut(), e);
-                    }
-                });
-            } catch (Throwable t) {
-                logger.error("unable to execute scroll request [{}]", t, scrollId);
-                finishHim(scrollId, false, t);
-            }
-        }
-
-        void deleteHits(String scrollId, SearchResponse scrollResponse) {
-            final SearchHit[] docs = scrollResponse.getHits().getHits();
-            final String nextScrollId = scrollResponse.getScrollId();
-            addShardFailures(scrollResponse.getShardFailures());
-
-            if (logger.isTraceEnabled()) {
-                logger.trace("scroll request [{}] executed: [{}] document(s) returned", scrollId, docs.length);
-            }
-
-            if ((docs.length == 0) || (nextScrollId == null)) {
-                logger.trace("scrolling documents terminated");
-                // if scrollId is null we are on the first request - just pass the nextScrollId which sill be non-null if the query matched no docs
-                finishHim(scrollId == null ? nextScrollId : scrollId, false, null);
-                return;
-            }
-
-            if (hasTimedOut()) {
-                logger.trace("scrolling documents timed out");
-                // if scrollId is null we are on the first request - just pass the nextScrollId which sill be non-null if the query matched no docs
-                finishHim(scrollId == null ? nextScrollId : scrollId, true, null);
-                return;
-            }
-
-            // Delete the scrolled documents using the Bulk API
-            BulkRequest bulkRequest = new BulkRequest();
-            for (SearchHit doc : docs) {
-                DeleteRequest delete = new DeleteRequest().index(doc.index()).type(doc.type()).id(doc.id()).version(doc.version());
-                SearchHitField routing = doc.field("_routing");
-                if (routing != null) {
-                    delete.routing((String) routing.value());
-                }
-                SearchHitField parent = doc.field("_parent");
-                if (parent != null) {
-                    delete.parent((String) parent.value());
-                }
-                bulkRequest.add(delete);
-            }
-
-            logger.trace("executing bulk request with [{}] deletions", bulkRequest.numberOfActions());
-            client.bulk(bulkRequest, new ActionListener<BulkResponse>() {
-                @Override
-                public void onResponse(BulkResponse bulkResponse) {
-                    onBulkResponse(nextScrollId, bulkResponse);
-                }
-
-                @Override
-                public void onFailure(Throwable e) {
-                    onBulkFailure(nextScrollId, docs, e);
-                }
-            });
-        }
-
-        void onBulkResponse(String scrollId, BulkResponse bulkResponse) {
-            try {
-                for (BulkItemResponse item : bulkResponse.getItems()) {
-                    IndexDeleteByQueryResponse indexCounter = results.get(item.getIndex());
-                    if (indexCounter == null) {
-                        indexCounter = new IndexDeleteByQueryResponse(item.getIndex());
-                    }
-                    indexCounter.incrementFound();
-                    if (item.isFailed()) {
-                        indexCounter.incrementFailed();
-                    } else {
-                        DeleteResponse delete = item.getResponse();
-                        if (delete.isFound()) {
-                            indexCounter.incrementDeleted();
-                        } else {
-                            indexCounter.incrementMissing();
-                        }
-                    }
-                    results.put(item.getIndex(), indexCounter);
-                }
-
-                logger.trace("scrolling next batch of document(s) with scroll id [{}]", scrollId);
-                executeScroll(scrollId);
-            } catch (Throwable t) {
-                logger.error("unable to process bulk response", t);
-                finishHim(scrollId, false, t);
-            }
-        }
-
-        void onBulkFailure(String scrollId, SearchHit[] docs, Throwable failure) {
-            try {
-                logger.trace("execution of scroll request failed: {}", failure.getMessage());
-                for (SearchHit doc : docs) {
-                    IndexDeleteByQueryResponse indexCounter = results.get(doc.index());
-                    if (indexCounter == null) {
-                        indexCounter = new IndexDeleteByQueryResponse(doc.index());
-                    }
-                    indexCounter.incrementFound();
-                    indexCounter.incrementFailed();
-                    results.put(doc.getIndex(), indexCounter);
-                }
-
-                logger.trace("scrolling document terminated due to scroll request failure [{}]", scrollId);
-                finishHim(scrollId, hasTimedOut(), failure);
-            } catch (Throwable t) {
-                logger.error("unable to process bulk failure", t);
-                finishHim(scrollId, false, t);
-            }
-        }
-
-        void finishHim(final String scrollId, boolean scrollTimedOut, Throwable failure) {
-            try {
-                if (scrollTimedOut) {
-                    logger.trace("delete-by-query response marked as timed out");
-                    timedOut.set(true);
-                }
-
-                if (Strings.hasText(scrollId)) {
-                    ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
-                    clearScrollRequest.addScrollId(scrollId);
-                    client.clearScroll(clearScrollRequest, new ActionListener<ClearScrollResponse>() {
-                        @Override
-                        public void onResponse(ClearScrollResponse clearScrollResponse) {
-                            logger.trace("scroll id [{}] cleared", scrollId);
-                        }
-
-                        @Override
-                        public void onFailure(Throwable e) {
-                            logger.warn("unable to clear scroll id [{}]: {}", scrollId, e.getMessage());
-                        }
-                    });
-                }
-
-                if (failure != null) {
-                    logger.trace("scrolling document(s) terminated with failures: {}", failure.getMessage());
-                    listener.onFailure(failure);
-                } else {
-                    logger.trace("scrolling document(s) terminated with success");
-                    listener.onResponse(buildResponse());
-                }
-            } catch (Throwable t) {
-                listener.onFailure(t);
-            }
-        }
-
-        boolean hasTimedOut() {
-            return request.timeout() != null && (threadPool.estimatedTimeInMillis() >= (startTime + request.timeout().millis()));
-        }
-
-        void addShardFailures(ShardOperationFailedException[] failures) {
-            if (!CollectionUtils.isEmpty(failures)) {
-                ShardOperationFailedException[] duplicates = new ShardOperationFailedException[shardFailures.length + failures.length];
-                System.arraycopy(shardFailures, 0, duplicates, 0, shardFailures.length);
-                System.arraycopy(failures, 0, duplicates, shardFailures.length, failures.length);
-                shardFailures = ExceptionsHelper.groupBy(duplicates);
-            }
-        }
-
-        protected DeleteByQueryResponse buildResponse() {
-            long took = threadPool.estimatedTimeInMillis() - startTime;
-            long deleted = 0;
-            long missing = 0;
-            long failed = 0;
-
-            // Calculates the total number  deleted/failed/missing documents
-            for (IndexDeleteByQueryResponse result : results.values()) {
-                deleted = deleted + result.getDeleted();
-                missing = missing + result.getMissing();
-                failed = failed + result.getFailed();
-            }
-            IndexDeleteByQueryResponse[] indices = results.values().toArray(new IndexDeleteByQueryResponse[results.size()]);
-            return new DeleteByQueryResponse(took, timedOut.get(), total.get(), deleted, missing, failed, indices, shardFailures);
-        }
-    }
-}
diff --git a/plugins/delete-by-query/src/main/java/org/elasticsearch/plugin/deletebyquery/DeleteByQueryPlugin.java b/plugins/delete-by-query/src/main/java/org/elasticsearch/plugin/deletebyquery/DeleteByQueryPlugin.java
deleted file mode 100644
index 8395223f669..00000000000
--- a/plugins/delete-by-query/src/main/java/org/elasticsearch/plugin/deletebyquery/DeleteByQueryPlugin.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.plugin.deletebyquery;
-
-import org.elasticsearch.action.ActionModule;
-import org.elasticsearch.action.deletebyquery.DeleteByQueryAction;
-import org.elasticsearch.action.deletebyquery.TransportDeleteByQueryAction;
-import org.elasticsearch.common.network.NetworkModule;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.rest.action.deletebyquery.RestDeleteByQueryAction;
-
-public class DeleteByQueryPlugin extends Plugin {
-
-    public static final String NAME = "delete-by-query";
-
-    @Override
-    public String name() {
-        return NAME;
-    }
-
-    @Override
-    public String description() {
-        return "Elasticsearch Delete-By-Query Plugin";
-    }
-
-    public void onModule(ActionModule actionModule) {
-        actionModule.registerAction(DeleteByQueryAction.INSTANCE, TransportDeleteByQueryAction.class);
-    }
-
-    public void onModule(NetworkModule module) {
-        module.registerRestHandler(RestDeleteByQueryAction.class);
-    }
-
-}
diff --git a/plugins/delete-by-query/src/main/java/org/elasticsearch/rest/action/deletebyquery/RestDeleteByQueryAction.java b/plugins/delete-by-query/src/main/java/org/elasticsearch/rest/action/deletebyquery/RestDeleteByQueryAction.java
deleted file mode 100644
index 2b537d1cf8a..00000000000
--- a/plugins/delete-by-query/src/main/java/org/elasticsearch/rest/action/deletebyquery/RestDeleteByQueryAction.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.rest.action.deletebyquery;
-
-import org.elasticsearch.action.deletebyquery.DeleteByQueryRequest;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
-import org.elasticsearch.rest.RestController;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.action.support.RestActions;
-import org.elasticsearch.rest.action.support.RestToXContentListener;
-
-import java.io.IOException;
-
-import static org.elasticsearch.action.deletebyquery.DeleteByQueryAction.INSTANCE;
-import static org.elasticsearch.rest.RestRequest.Method.DELETE;
-
-/**
- * @see DeleteByQueryRequest
- */
-public class RestDeleteByQueryAction extends BaseRestHandler {
-
-    private IndicesQueriesRegistry indicesQueriesRegistry;
-
-    @Inject
-    public RestDeleteByQueryAction(Settings settings, RestController controller, Client client,
-            IndicesQueriesRegistry indicesQueriesRegistry) {
-        super(settings, client);
-        this.indicesQueriesRegistry = indicesQueriesRegistry;
-        controller.registerHandler(DELETE, "/{index}/_query", this);
-        controller.registerHandler(DELETE, "/{index}/{type}/_query", this);
-    }
-
-    @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws IOException {
-        DeleteByQueryRequest delete = new DeleteByQueryRequest(Strings.splitStringByCommaToArray(request.param("index")));
-        delete.indicesOptions(IndicesOptions.fromRequest(request, delete.indicesOptions()));
-        delete.routing(request.param("routing"));
-        if (request.hasParam("timeout")) {
-            delete.timeout(request.paramAsTime("timeout", null));
-        }
-        if (RestActions.hasBodyContent(request)) {
-            delete.query(RestActions.getQueryContent(RestActions.getRestContent(request), indicesQueriesRegistry, parseFieldMatcher));
-        } else {
-            QueryBuilder queryBuilder = RestActions.urlParamsToQueryBuilder(request);
-            if (queryBuilder != null) {
-                delete.query(queryBuilder);
-            }
-        }
-        delete.types(Strings.splitStringByCommaToArray(request.param("type")));
-        client.execute(INSTANCE, delete, new RestToXContentListener<>(channel));
-    }
-}
diff --git a/plugins/delete-by-query/src/test/java/org/elasticsearch/action/deletebyquery/IndexDeleteByQueryResponseTests.java b/plugins/delete-by-query/src/test/java/org/elasticsearch/action/deletebyquery/IndexDeleteByQueryResponseTests.java
deleted file mode 100644
index ea814b44f5d..00000000000
--- a/plugins/delete-by-query/src/test/java/org/elasticsearch/action/deletebyquery/IndexDeleteByQueryResponseTests.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.deletebyquery;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.VersionUtils;
-
-import static org.hamcrest.Matchers.equalTo;
-
-public class IndexDeleteByQueryResponseTests extends ESTestCase {
-    public void testIncrements() {
-        String indexName = randomAsciiOfLength(5);
-
-        // Use randomInt to prevent range overflow
-        long found = Math.abs(randomInt());
-        long deleted = Math.abs(randomInt());
-        long missing = Math.abs(randomInt());
-        long failed = Math.abs(randomInt());
-
-        IndexDeleteByQueryResponse response = new IndexDeleteByQueryResponse(indexName, found, deleted, missing, failed);
-        assertThat(response.getIndex(), equalTo(indexName));
-        assertThat(response.getFound(), equalTo(found));
-        assertThat(response.getDeleted(), equalTo(deleted));
-        assertThat(response.getMissing(), equalTo(missing));
-        assertThat(response.getFailed(), equalTo(failed));
-
-        response.incrementFound();
-        response.incrementDeleted();
-        response.incrementMissing();
-        response.incrementFailed();
-
-        assertThat(response.getFound(), equalTo(found + 1));
-        assertThat(response.getDeleted(), equalTo(deleted + 1));
-        assertThat(response.getMissing(), equalTo(missing + 1));
-        assertThat(response.getFailed(), equalTo(failed + 1));
-
-        // Use randomInt to prevent range overflow
-        long inc = randomIntBetween(0, 1000);
-        response.incrementFound(inc);
-        response.incrementDeleted(inc);
-        response.incrementMissing(inc);
-        response.incrementFailed(inc);
-
-        assertThat(response.getFound(), equalTo(found + 1 + inc));
-        assertThat(response.getDeleted(), equalTo(deleted + 1 + inc));
-        assertThat(response.getMissing(), equalTo(missing + 1 + inc));
-        assertThat(response.getFailed(), equalTo(failed + 1 + inc));
-    }
-
-    public void testNegativeCounters() {
-        assumeTrue("assertions must be enable for this test to pass", assertionsEnabled());
-        try {
-            new IndexDeleteByQueryResponse("index", -1L, 0L, 0L, 0L);
-            fail("should have thrown an assertion error concerning the negative counter");
-        } catch (AssertionError e) {
-            assertThat("message contains error about a negative counter: " + e.getMessage(),
-                    e.getMessage().contains("counter 'found' cannot be negative"), equalTo(true));
-        }
-
-        try {
-            new IndexDeleteByQueryResponse("index", 0L, -1L, 0L, 0L);
-            fail("should have thrown an assertion error concerning the negative counter");
-        } catch (AssertionError e) {
-            assertThat("message contains error about a negative counter: " + e.getMessage(),
-                    e.getMessage().contains("counter 'deleted' cannot be negative"), equalTo(true));
-        }
-
-        try {
-            new IndexDeleteByQueryResponse("index", 0L, 0L, -1L, 0L);
-            fail("should have thrown an assertion error concerning the negative counter");
-        } catch (AssertionError e) {
-            assertThat("message contains error about a negative counter: " + e.getMessage(),
-                    e.getMessage().contains("counter 'missing' cannot be negative"), equalTo(true));
-        }
-
-        try {
-            new IndexDeleteByQueryResponse("index", 0L, 0L, 0L, -1L);
-            fail("should have thrown an assertion error concerning the negative counter");
-        } catch (AssertionError e) {
-            assertThat("message contains error about a negative counter: " + e.getMessage(),
-                    e.getMessage().contains("counter 'failed' cannot be negative"), equalTo(true));
-        }
-    }
-
-    public void testNegativeIncrements() {
-        assumeTrue("assertions must be enable for this test to pass", assertionsEnabled());
-        try {
-            IndexDeleteByQueryResponse response = new IndexDeleteByQueryResponse();
-            response.incrementFound(-10L);
-            fail("should have thrown an assertion error concerning the negative counter");
-        } catch (AssertionError e) {
-            assertThat("message contains error about a negative counter: " + e.getMessage(),
-                    e.getMessage().contains("counter 'found' cannot be negative"), equalTo(true));
-        }
-
-        try {
-            IndexDeleteByQueryResponse response = new IndexDeleteByQueryResponse();
-            response.incrementDeleted(-10L);
-            fail("should have thrown an assertion error concerning the negative counter");
-        } catch (AssertionError e) {
-            assertThat("message contains error about a negative counter: " + e.getMessage(),
-                    e.getMessage().contains("counter 'deleted' cannot be negative"), equalTo(true));
-        }
-
-        try {
-            IndexDeleteByQueryResponse response = new IndexDeleteByQueryResponse();
-            response.incrementMissing(-10L);
-            fail("should have thrown an assertion error concerning the negative counter");
-        } catch (AssertionError e) {
-            assertThat("message contains error about a negative counter: " + e.getMessage(),
-                    e.getMessage().contains("counter 'missing' cannot be negative"), equalTo(true));
-        }
-
-        try {
-            IndexDeleteByQueryResponse response = new IndexDeleteByQueryResponse();
-            response.incrementFailed(-1L);
-            fail("should have thrown an assertion error concerning the negative counter");
-        } catch (AssertionError e) {
-            assertThat("message contains error about a negative counter: " + e.getMessage(),
-                    e.getMessage().contains("counter 'failed' cannot be negative"), equalTo(true));
-        }
-    }
-
-    public void testSerialization() throws Exception {
-        IndexDeleteByQueryResponse response = new IndexDeleteByQueryResponse(randomAsciiOfLength(5), Math.abs(randomLong()), Math.abs(randomLong()), Math.abs(randomLong()), Math.abs(randomLong()));
-        Version testVersion = VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.CURRENT);
-        BytesStreamOutput output = new BytesStreamOutput();
-        output.setVersion(testVersion);
-        response.writeTo(output);
-
-        StreamInput streamInput = StreamInput.wrap(output.bytes());
-        streamInput.setVersion(testVersion);
-        IndexDeleteByQueryResponse deserializedResponse = new IndexDeleteByQueryResponse();
-        deserializedResponse.readFrom(streamInput);
-
-        assertThat(deserializedResponse.getIndex(), equalTo(response.getIndex()));
-        assertThat(deserializedResponse.getFound(), equalTo(response.getFound()));
-        assertThat(deserializedResponse.getDeleted(), equalTo(response.getDeleted()));
-        assertThat(deserializedResponse.getMissing(), equalTo(response.getMissing()));
-        assertThat(deserializedResponse.getFailed(), equalTo(response.getFailed()));
-    }
-}
diff --git a/plugins/delete-by-query/src/test/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryActionTests.java b/plugins/delete-by-query/src/test/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryActionTests.java
deleted file mode 100644
index 3867c8be494..00000000000
--- a/plugins/delete-by-query/src/test/java/org/elasticsearch/action/deletebyquery/TransportDeleteByQueryActionTests.java
+++ /dev/null
@@ -1,458 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.deletebyquery;
-
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
-import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
-import org.elasticsearch.action.bulk.BulkItemResponse;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.delete.DeleteResponse;
-import org.elasticsearch.action.search.ClearScrollResponse;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.text.Text;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.CollectionUtils;
-import org.elasticsearch.common.util.concurrent.CountDown;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.internal.InternalSearchHit;
-import org.elasticsearch.test.ESSingleNodeTestCase;
-
-import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
-import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-
-public class TransportDeleteByQueryActionTests extends ESSingleNodeTestCase {
-    public void testExecuteScanFailsOnMissingIndex() {
-        DeleteByQueryRequest delete = new DeleteByQueryRequest().indices(new String[]{"none"});
-        TestActionListener listener = new TestActionListener();
-
-        newAsyncAction(delete, listener).executeScan();
-        waitForCompletion("scan request should fail on missing index", listener);
-
-        assertFailure(listener, "no such index");
-        assertSearchContextsClosed();
-    }
-
-    public void testExecuteScan() {
-        createIndex("test");
-        final int numDocs = randomIntBetween(1, 200);
-        for (int i = 1; i <= numDocs; i++) {
-            client().prepareIndex("test", "type").setSource("num", i).get();
-        }
-        client().admin().indices().prepareRefresh("test").get();
-        assertHitCount(client().prepareSearch("test").setSize(0).get(), numDocs);
-
-        final long limit = randomIntBetween(0, numDocs);
-        DeleteByQueryRequest delete = new DeleteByQueryRequest().indices(new String[]{"test"}).query(boolQuery().must(rangeQuery("num").lte(limit)));
-        TestActionListener listener = new TestActionListener();
-
-        newAsyncAction(delete, listener).executeScan();
-        waitForCompletion("scan request should return the exact number of documents", listener);
-
-        assertNoFailures(listener);
-        DeleteByQueryResponse response = listener.getResponse();
-        assertNotNull(response);
-        assertThat(response.getTotalFound(), equalTo(limit));
-        assertThat(response.getTotalDeleted(), equalTo(limit));
-        assertSearchContextsClosed();
-    }
-
-    public void testExecuteScrollFailsOnMissingScrollId() {
-        DeleteByQueryRequest delete = new DeleteByQueryRequest();
-        TestActionListener listener = new TestActionListener();
-
-        newAsyncAction(delete, listener).executeScroll(null);
-        waitForCompletion("scroll request should fail on missing scroll id", listener);
-
-        assertFailure(listener, "scrollId is missing");
-        assertSearchContextsClosed();
-    }
-
-    public void testExecuteScrollFailsOnMalformedScrollId() {
-        DeleteByQueryRequest delete = new DeleteByQueryRequest();
-        TestActionListener listener = new TestActionListener();
-
-        newAsyncAction(delete, listener).executeScroll("123");
-        waitForCompletion("scroll request should fail on malformed scroll id", listener);
-
-        assertFailure(listener, "Cannot parse scroll id");
-        assertSearchContextsClosed();
-    }
-
-    public void testExecuteScrollFailsOnExpiredScrollId() {
-        final long numDocs = randomIntBetween(1, 100);
-        for (int i = 1; i <= numDocs; i++) {
-            client().prepareIndex("test", "type").setSource("num", i).get();
-        }
-        client().admin().indices().prepareRefresh("test").get();
-        assertHitCount(client().prepareSearch("test").setSize(0).get(), numDocs);
-
-        SearchResponse searchResponse = client().prepareSearch("test").setScroll(TimeValue.timeValueSeconds(10)).get();
-        assertThat(searchResponse.getHits().getTotalHits(), equalTo(numDocs));
-
-        String scrollId = searchResponse.getScrollId();
-        assertTrue(Strings.hasText(scrollId));
-
-        ClearScrollResponse clearScrollResponse = client().prepareClearScroll().addScrollId(scrollId).get();
-        assertTrue(clearScrollResponse.isSucceeded());
-
-        DeleteByQueryRequest delete = new DeleteByQueryRequest().indices(new String[]{"test"});
-        TestActionListener listener = new TestActionListener();
-
-        newAsyncAction(delete, listener).executeScroll(searchResponse.getScrollId());
-        waitForCompletion("scroll request returns zero documents on expired scroll id", listener);
-
-        assertNotNull(listener.getError());
-        assertThrowableContains(listener.getError(), "No search context found");
-        assertSearchContextsClosed();
-    }
-
-    public void testExecuteScrollTimedOut() throws InterruptedException {
-        client().prepareIndex("test", "type", "1").setSource("num", "1").get();
-        client().prepareIndex("test", "type", "2").setSource("num", "1").get();
-        client().admin().indices().prepareRefresh("test").get();
-
-        SearchResponse searchResponse = client().prepareSearch("test").setSize(1).setScroll(TimeValue.timeValueSeconds(10)).get();
-        String scrollId = searchResponse.getScrollId();
-        assertTrue(Strings.hasText(scrollId));
-
-        DeleteByQueryRequest delete = new DeleteByQueryRequest().indices(new String[]{"test"}).timeout(TimeValue.timeValueSeconds(1));
-        TestActionListener listener = new TestActionListener();
-
-        final TransportDeleteByQueryAction.AsyncDeleteByQueryAction async = newAsyncAction(delete, listener);
-        // Wait until the action timed out
-        awaitBusy(() -> async.hasTimedOut());
-
-        async.executeScroll(searchResponse.getScrollId());
-        waitForCompletion("scroll request returns zero documents on expired scroll id", listener);
-
-        assertNull(listener.getError());
-        assertTrue(listener.getResponse().isTimedOut());
-        assertThat(listener.getResponse().getTotalDeleted(), equalTo(0L));
-        assertSearchContextsClosed();
-    }
-
-    public void testExecuteScrollNoDocuments() {
-        createIndex("test");
-        SearchResponse searchResponse = client().prepareSearch("test").setScroll(TimeValue.timeValueSeconds(10)).get();
-        String scrollId = searchResponse.getScrollId();
-        assertTrue(Strings.hasText(scrollId));
-
-        DeleteByQueryRequest delete = new DeleteByQueryRequest().indices(new String[]{"test"});
-        TestActionListener listener = new TestActionListener();
-
-        newAsyncAction(delete, listener).executeScroll(searchResponse.getScrollId());
-        waitForCompletion("scroll request returns zero documents", listener);
-
-        assertNull(listener.getError());
-        assertFalse(listener.getResponse().isTimedOut());
-        assertThat(listener.getResponse().getTotalFound(), equalTo(0L));
-        assertThat(listener.getResponse().getTotalDeleted(), equalTo(0L));
-        assertSearchContextsClosed();
-    }
-
-    public void testExecuteScroll() {
-        final int numDocs = randomIntBetween(1, 100);
-        for (int i = 1; i <= numDocs; i++) {
-            client().prepareIndex("test", "type").setSource("num", i).get();
-        }
-        client().admin().indices().prepareRefresh("test").get();
-        assertHitCount(client().prepareSearch("test").setSize(0).get(), numDocs);
-
-        final long limit = randomIntBetween(0, numDocs);
-
-        SearchResponse searchResponse = client().prepareSearch("test")
-                .setScroll(TimeValue.timeValueSeconds(10))
-                .setQuery(boolQuery().must(rangeQuery("num").lte(limit)))
-                .fields("_routing", "_parent")
-                .setFetchSource(false)
-                .setVersion(true)
-                .get();
-
-        String scrollId = searchResponse.getScrollId();
-        assertTrue(Strings.hasText(scrollId));
-        assertThat(searchResponse.getHits().getTotalHits(), equalTo(limit));
-
-        DeleteByQueryRequest delete = new DeleteByQueryRequest().indices(new String[]{"test"}).size(100).query(boolQuery().must(rangeQuery("num").lte(limit)));
-        TestActionListener listener = new TestActionListener();
-
-        newAsyncAction(delete, listener).executeScroll(searchResponse.getScrollId());
-        waitForCompletion("scroll request should return all documents", listener);
-
-        assertNull(listener.getError());
-        assertFalse(listener.getResponse().isTimedOut());
-        // docs that have been returned on the 1st page have been skipped
-        final long expectedDeleted = Math.max(0, limit - searchResponse.getHits().hits().length);
-        assertThat(listener.getResponse().getTotalDeleted(), equalTo(expectedDeleted));
-        assertSearchContextsClosed();
-    }
-
-    public void testOnBulkResponse() {
-        final int nbItems = randomIntBetween(0, 20);
-        long deleted = 0;
-        long missing = 0;
-        long failed = 0;
-
-        BulkItemResponse[] items = new BulkItemResponse[nbItems];
-        for (int i = 0; i < nbItems; i++) {
-            if (randomBoolean()) {
-                boolean delete = true;
-                if (rarely()) {
-                    delete = false;
-                    missing++;
-                } else {
-                    deleted++;
-                }
-                items[i] = new BulkItemResponse(i, "delete", new DeleteResponse(new ShardId("test", "_na_", 0), "type", String.valueOf(i), 1, delete));
-            } else {
-                items[i] = new BulkItemResponse(i, "delete", new BulkItemResponse.Failure("test", "type", String.valueOf(i), new Throwable("item failed")));
-                failed++;
-            }
-        }
-
-        // We just need a valid scroll id
-        createIndex("test");
-        SearchResponse searchResponse = client().prepareSearch().setScroll(TimeValue.timeValueSeconds(10)).get();
-        String scrollId = searchResponse.getScrollId();
-        assertTrue(Strings.hasText(scrollId));
-
-        try {
-            DeleteByQueryRequest delete = new DeleteByQueryRequest();
-            TestActionListener listener = new TestActionListener();
-
-            newAsyncAction(delete, listener).onBulkResponse(scrollId, new BulkResponse(items, 0L));
-            waitForCompletion("waiting for bulk response to complete", listener);
-
-            assertNoFailures(listener);
-            assertThat(listener.getResponse().getTotalDeleted(), equalTo(deleted));
-            assertThat(listener.getResponse().getTotalFailed(), equalTo(failed));
-            assertThat(listener.getResponse().getTotalMissing(), equalTo(missing));
-        } finally {
-            client().prepareClearScroll().addScrollId(scrollId).get();
-        }
-    }
-
-    public void testOnBulkResponseMultipleIndices() {
-        final int nbIndices = randomIntBetween(2, 5);
-
-        // Holds counters for the total + all indices
-        final long[] found = new long[1 + nbIndices];
-        final long[] deleted = new long[1 + nbIndices];
-        final long[] missing = new long[1 + nbIndices];
-        final long[] failed = new long[1 + nbIndices];
-
-        final int nbItems = randomIntBetween(0, 100);
-        found[0] = nbItems;
-
-        BulkItemResponse[] items = new BulkItemResponse[nbItems];
-        for (int i = 0; i < nbItems; i++) {
-            int index = randomIntBetween(1, nbIndices);
-            found[index] = found[index] + 1;
-
-            if (randomBoolean()) {
-                boolean delete = true;
-                if (rarely()) {
-                    delete = false;
-                    missing[0] = missing[0] + 1;
-                    missing[index] = missing[index] + 1;
-                } else {
-                    deleted[0] = deleted[0] + 1;
-                    deleted[index] = deleted[index] + 1;
-                }
-                items[i] = new BulkItemResponse(i, "delete", new DeleteResponse(new ShardId("test-" + index, "_na_", 0), "type", String.valueOf(i), 1, delete));
-            } else {
-                items[i] = new BulkItemResponse(i, "delete", new BulkItemResponse.Failure("test-" + index, "type", String.valueOf(i), new Throwable("item failed")));
-                failed[0] = failed[0] + 1;
-                failed[index] = failed[index] + 1;
-            }
-        }
-
-        // We just need a valid scroll id
-        createIndex("test");
-        SearchResponse searchResponse = client().prepareSearch().setScroll(TimeValue.timeValueSeconds(10)).get();
-        String scrollId = searchResponse.getScrollId();
-        assertTrue(Strings.hasText(scrollId));
-
-        try {
-            DeleteByQueryRequest delete = new DeleteByQueryRequest();
-            TestActionListener listener = new TestActionListener();
-
-            newAsyncAction(delete, listener).onBulkResponse(scrollId, new BulkResponse(items, 0L));
-            waitForCompletion("waiting for bulk response to complete", listener);
-
-            assertNoFailures(listener);
-            assertThat(listener.getResponse().getTotalDeleted(), equalTo(deleted[0]));
-            assertThat(listener.getResponse().getTotalFailed(), equalTo(failed[0]));
-            assertThat(listener.getResponse().getTotalMissing(), equalTo(missing[0]));
-
-            for (int i = 1; i <= nbIndices; i++) {
-                IndexDeleteByQueryResponse indexResponse = listener.getResponse().getIndex("test-" + i);
-                if (found[i] >= 1) {
-                    assertNotNull(indexResponse);
-                    assertThat(indexResponse.getFound(), equalTo(found[i]));
-                    assertThat(indexResponse.getDeleted(), equalTo(deleted[i]));
-                    assertThat(indexResponse.getFailed(), equalTo(failed[i]));
-                    assertThat(indexResponse.getMissing(), equalTo(missing[i]));
-                } else {
-                    assertNull(indexResponse);
-                }
-            }
-        } finally {
-            client().prepareClearScroll().addScrollId(scrollId).get();
-        }
-    }
-
-    public void testOnBulkFailureNoDocuments() {
-        DeleteByQueryRequest delete = new DeleteByQueryRequest();
-        TestActionListener listener = new TestActionListener();
-
-        newAsyncAction(delete, listener).onBulkFailure(null, new SearchHit[0], new Throwable("This is a bulk failure"));
-        waitForCompletion("waiting for bulk failure to complete", listener);
-
-        assertFailure(listener, "This is a bulk failure");
-    }
-
-    public void testOnBulkFailure() {
-        final int nbDocs = randomIntBetween(0, 20);
-        SearchHit[] docs = new SearchHit[nbDocs];
-        for (int i = 0; i < nbDocs; i++) {
-            InternalSearchHit doc = new InternalSearchHit(randomInt(), String.valueOf(i), new Text("type"), null);
-            doc.shard(new SearchShardTarget("node", new Index("test", "_na_"), randomInt()));
-            docs[i] = doc;
-        }
-
-        DeleteByQueryRequest delete = new DeleteByQueryRequest();
-        TestActionListener listener = new TestActionListener();
-
-        TransportDeleteByQueryAction.AsyncDeleteByQueryAction async = newAsyncAction(delete, listener);
-        async.onBulkFailure(null, docs, new Throwable("This is a bulk failure"));
-        waitForCompletion("waiting for bulk failure to complete", listener);
-        assertFailure(listener, "This is a bulk failure");
-
-        DeleteByQueryResponse response = async.buildResponse();
-        assertThat(response.getTotalFailed(), equalTo((long) nbDocs));
-        assertThat(response.getTotalDeleted(), equalTo(0L));
-    }
-
-    public void testFinishHim() {
-        TestActionListener listener = new TestActionListener();
-        newAsyncAction(new DeleteByQueryRequest(), listener).finishHim(null, false, null);
-        waitForCompletion("waiting for finishHim to complete with success", listener);
-        assertNoFailures(listener);
-        assertNotNull(listener.getResponse());
-        assertFalse(listener.getResponse().isTimedOut());
-
-        listener = new TestActionListener();
-        newAsyncAction(new DeleteByQueryRequest(), listener).finishHim(null, true, null);
-        waitForCompletion("waiting for finishHim to complete with timed out = true", listener);
-        assertNoFailures(listener);
-        assertNotNull(listener.getResponse());
-        assertTrue(listener.getResponse().isTimedOut());
-
-        listener = new TestActionListener();
-        newAsyncAction(new DeleteByQueryRequest(), listener).finishHim(null, false, new Throwable("Fake error"));
-        waitForCompletion("waiting for finishHim to complete with error", listener);
-        assertFailure(listener, "Fake error");
-        assertNull(listener.getResponse());
-    }
-
-    private TransportDeleteByQueryAction.AsyncDeleteByQueryAction newAsyncAction(DeleteByQueryRequest request, TestActionListener listener) {
-        TransportDeleteByQueryAction action = getInstanceFromNode(TransportDeleteByQueryAction.class);
-        assertNotNull(action);
-        return action.new AsyncDeleteByQueryAction(request, listener);
-    }
-
-    private void waitForCompletion(String testName, final TestActionListener listener) {
-        logger.info(" --> waiting for delete-by-query [{}] to complete", testName);
-        try {
-            awaitBusy(() -> listener.isTerminated());
-        } catch (InterruptedException e) {
-            fail("exception when waiting for delete-by-query [" + testName + "] to complete: " + e.getMessage());
-            logger.error("exception when waiting for delete-by-query [{}] to complete", e, testName);
-        }
-    }
-
-    private void assertFailure(TestActionListener listener, String expectedMessage) {
-        Throwable t = listener.getError();
-        assertNotNull(t);
-        assertTrue(Strings.hasText(expectedMessage));
-        assertTrue("error message should contain [" + expectedMessage + "] but got [" + t.getMessage() + "]", t.getMessage().contains(expectedMessage));
-    }
-
-    private void assertNoFailures(TestActionListener listener) {
-        assertNull(listener.getError());
-        assertTrue(CollectionUtils.isEmpty(listener.getResponse().getShardFailures()));
-    }
-
-    private void assertSearchContextsClosed() {
-        NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).get();
-        for (NodeStats nodeStat : nodesStats.getNodes()){
-            assertThat(nodeStat.getIndices().getSearch().getOpenContexts(), equalTo(0L));
-        }
-    }
-
-    private void assertThrowableContains(Throwable t, String expectedFailure) {
-        assertThat(t.toString(), containsString(expectedFailure));
-    }
-
-    private class TestActionListener implements ActionListener<DeleteByQueryResponse> {
-        private final CountDown count = new CountDown(1);
-
-        private DeleteByQueryResponse response;
-        private Throwable error;
-
-        @Override
-        public void onResponse(DeleteByQueryResponse response) {
-            try {
-                this.response = response;
-            } finally {
-                count.countDown();
-            }
-        }
-
-        @Override
-        public void onFailure(Throwable e) {
-            try {
-                this.error = e;
-            } finally {
-                count.countDown();
-            }
-        }
-
-        public boolean isTerminated() {
-            return count.isCountedDown();
-        }
-
-        public DeleteByQueryResponse getResponse() {
-            return response;
-        }
-
-        public Throwable getError() {
-            return error;
-        }
-    }
-}
diff --git a/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/DeleteByQueryTests.java b/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/DeleteByQueryTests.java
deleted file mode 100644
index 1245c83444c..00000000000
--- a/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/DeleteByQueryTests.java
+++ /dev/null
@@ -1,446 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.plugin.deletebyquery;
-
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
-import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
-import org.elasticsearch.action.admin.indices.alias.Alias;
-import org.elasticsearch.action.deletebyquery.DeleteByQueryAction;
-import org.elasticsearch.action.deletebyquery.DeleteByQueryRequestBuilder;
-import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse;
-import org.elasticsearch.action.deletebyquery.IndexDeleteByQueryResponse;
-import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.IndexNotFoundException;
-import org.elasticsearch.index.query.MatchQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-
-import java.util.Collection;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
-
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.nullValue;
-
-@ClusterScope(scope = SUITE, transportClientRatio = 0)
-public class DeleteByQueryTests extends ESIntegTestCase {
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return pluginList(DeleteByQueryPlugin.class);
-    }
-
-    public void testDeleteByQueryWithNoSource() {
-        try {
-            newDeleteByQuery().get();
-            fail("should have thrown a validation exception because of the missing source");
-        } catch (ActionRequestValidationException e) {
-            assertThat(e.getMessage(), containsString("source is missing"));
-        }
-    }
-
-    public void testDeleteByQueryWithNoIndices() throws Exception {
-        DeleteByQueryRequestBuilder delete = newDeleteByQuery().setQuery(QueryBuilders.matchAllQuery());
-        delete.setIndicesOptions(IndicesOptions.fromOptions(false, true, true, false));
-        assertDBQResponse(delete.get(), 0L, 0L, 0L, 0L);
-        assertSearchContextsClosed();
-    }
-
-    public void testDeleteByQueryWithOneIndex() throws Exception {
-        final long docs = randomIntBetween(1, 50);
-        for (int i = 0; i < docs; i++) {
-            index("test", "test", String.valueOf(i), "fields1", 1);
-        }
-        refresh();
-        assertHitCount(client().prepareSearch("test").setSize(0).get(), docs);
-
-        DeleteByQueryRequestBuilder delete = newDeleteByQuery().setIndices("t*").setQuery(QueryBuilders.matchAllQuery());
-        assertDBQResponse(delete.get(), docs, docs, 0L, 0L);
-        refresh();
-        assertHitCount(client().prepareSearch("test").setSize(0).get(), 0);
-        assertSearchContextsClosed();
-    }
-
-    public void testDeleteByQueryWithMultipleIndices() throws Exception {
-        final int indices = randomIntBetween(2, 5);
-        final int docs = randomIntBetween(2, 10) * 2;
-        long[] candidates = new long[indices];
-
-        for (int i = 0; i < indices; i++) {
-            // number of documents to be deleted with the upcoming delete-by-query
-            // (this number differs for each index)
-            candidates[i] = randomIntBetween(1, docs);
-
-            for (int j = 0; j < docs; j++) {
-                boolean candidate = (j < candidates[i]);
-                index("test-" + i, "test", String.valueOf(j), "candidate", candidate);
-            }
-        }
-
-        // total number of expected deletions
-        long deletions = 0;
-        for (long i : candidates) {
-            deletions = deletions + i;
-        }
-        refresh();
-
-        assertHitCount(client().prepareSearch().setSize(0).get(), docs * indices);
-        for (int i = 0; i < indices; i++) {
-            assertHitCount(client().prepareSearch("test-" + i).setSize(0).get(), docs);
-        }
-
-        // Deletes all the documents with candidate=true
-        DeleteByQueryResponse response = newDeleteByQuery().setIndices("test-*").setQuery(QueryBuilders.termQuery("candidate", true)).get();
-        refresh();
-
-        // Checks that the DBQ response returns the expected number of deletions
-        assertDBQResponse(response, deletions, deletions, 0L, 0L);
-        assertNotNull(response.getIndices());
-        assertThat(response.getIndices().length, equalTo(indices));
-
-        for (int i = 0; i < indices; i++) {
-            String indexName = "test-" + i;
-            IndexDeleteByQueryResponse indexResponse = response.getIndex(indexName);
-            assertThat(indexResponse.getFound(), equalTo(candidates[i]));
-            assertThat(indexResponse.getDeleted(), equalTo(candidates[i]));
-            assertThat(indexResponse.getFailed(), equalTo(0L));
-            assertThat(indexResponse.getMissing(), equalTo(0L));
-            assertThat(indexResponse.getIndex(), equalTo(indexName));
-            long remaining = docs - candidates[i];
-            assertHitCount(client().prepareSearch(indexName).setSize(0).get(), remaining);
-        }
-
-        assertHitCount(client().prepareSearch().setSize(0).get(), (indices * docs) - deletions);
-        assertSearchContextsClosed();
-    }
-
-    public void testDeleteByQueryWithMissingIndex() throws Exception {
-        client().prepareIndex("test", "test")
-                .setSource(jsonBuilder().startObject().field("field1", 1).endObject())
-                .setRefresh(true)
-                .get();
-        assertHitCount(client().prepareSearch().setSize(0).get(), 1);
-
-        DeleteByQueryRequestBuilder delete = newDeleteByQuery().setIndices("test", "missing").setQuery(QueryBuilders.matchAllQuery());
-        try {
-            delete.get();
-            fail("should have thrown an exception because of a missing index");
-        } catch (IndexNotFoundException e) {
-            // Ok
-        }
-
-        delete.setIndicesOptions(IndicesOptions.lenientExpandOpen());
-        assertDBQResponse(delete.get(), 1L, 1L, 0L, 0L);
-        refresh();
-        assertHitCount(client().prepareSearch("test").setSize(0).get(), 0);
-        assertSearchContextsClosed();
-    }
-
-    public void testDeleteByQueryWithTypes() throws Exception {
-        final long docs = randomIntBetween(1, 50);
-        for (int i = 0; i < docs; i++) {
-            index(randomFrom("test1", "test2", "test3"), "type1", String.valueOf(i), "foo", "bar");
-            index(randomFrom("test1", "test2", "test3"), "type2", String.valueOf(i), "foo", "bar");
-        }
-        refresh();
-        assertHitCount(client().prepareSearch().setSize(0).get(), docs * 2);
-        assertHitCount(client().prepareSearch().setSize(0).setTypes("type1").get(), docs);
-        assertHitCount(client().prepareSearch().setSize(0).setTypes("type2").get(), docs);
-
-        DeleteByQueryRequestBuilder delete = newDeleteByQuery().setTypes("type1").setQuery(QueryBuilders.matchAllQuery());
-        assertDBQResponse(delete.get(), docs, docs, 0L, 0L);
-        refresh();
-
-        assertHitCount(client().prepareSearch().setSize(0).get(), docs);
-        assertHitCount(client().prepareSearch().setSize(0).setTypes("type1").get(), 0);
-        assertHitCount(client().prepareSearch().setSize(0).setTypes("type2").get(), docs);
-        assertSearchContextsClosed();
-    }
-
-    public void testDeleteByQueryWithRouting() throws Exception {
-        assertAcked(prepareCreate("test").setSettings("number_of_shards", 2));
-        ensureGreen("test");
-
-        final int docs = randomIntBetween(2, 10);
-        logger.info("--> indexing [{}] documents with routing", docs);
-        for (int i = 0; i < docs; i++) {
-            client().prepareIndex("test", "test", String.valueOf(i)).setRouting(String.valueOf(i)).setSource("field1", 1).get();
-        }
-        refresh();
-
-        logger.info("--> counting documents with no routing, should be equal to [{}]", docs);
-        assertHitCount(client().prepareSearch().setSize(0).get(), docs);
-
-        String routing = String.valueOf(randomIntBetween(2, docs));
-
-        logger.info("--> counting documents with routing [{}]", routing);
-        long expected = client().prepareSearch().setSize(0).setRouting(routing).get().getHits().totalHits();
-
-        logger.info("--> delete all documents with routing [{}] with a delete-by-query", routing);
-        DeleteByQueryRequestBuilder delete = newDeleteByQuery().setRouting(routing).setQuery(QueryBuilders.matchAllQuery());
-        assertDBQResponse(delete.get(), expected, expected, 0L, 0L);
-        refresh();
-
-        assertHitCount(client().prepareSearch().setSize(0).get(), docs - expected);
-        assertSearchContextsClosed();
-    }
-
-    public void testDeleteByFieldQuery() throws Exception {
-        assertAcked(prepareCreate("test").addAlias(new Alias("alias")));
-
-        int numDocs = scaledRandomIntBetween(10, 100);
-        for (int i = 0; i < numDocs; i++) {
-            client().prepareIndex("test", "test", Integer.toString(i))
-                    .setRouting(randomAsciiOfLengthBetween(1, 5))
-                    .setSource("foo", "bar").get();
-        }
-        refresh();
-
-        int n = between(0, numDocs - 1);
-        assertHitCount(client().prepareSearch("test").setSize(0).setQuery(QueryBuilders.matchQuery("_id", Integer.toString(n))).get(), 1);
-        assertHitCount(client().prepareSearch("test").setSize(0).setQuery(QueryBuilders.matchAllQuery()).get(), numDocs);
-
-        DeleteByQueryRequestBuilder delete = newDeleteByQuery().setIndices("alias").setQuery(QueryBuilders.matchQuery("_id", Integer.toString(n)));
-        assertDBQResponse(delete.get(), 1L, 1L, 0L, 0L);
-        refresh();
-        assertHitCount(client().prepareSearch("test").setSize(0).setQuery(QueryBuilders.matchAllQuery()).get(), numDocs - 1);
-        assertSearchContextsClosed();
-    }
-
-    public void testDeleteByQueryWithDateMath() throws Exception {
-        index("test", "type", "1", "d", "2013-01-01");
-        ensureGreen();
-        refresh();
-        assertHitCount(client().prepareSearch("test").setSize(0).get(), 1);
-
-        DeleteByQueryRequestBuilder delete = newDeleteByQuery().setIndices("test").setQuery(QueryBuilders.rangeQuery("d").to("now-1h"));
-        assertDBQResponse(delete.get(), 1L, 1L, 0L, 0L);
-        refresh();
-        assertHitCount(client().prepareSearch("test").setSize(0).get(), 0);
-        assertSearchContextsClosed();
-    }
-
-    public void testDeleteByTermQuery() throws Exception {
-        createIndex("test");
-        ensureGreen();
-
-        int numDocs = scaledRandomIntBetween(10, 50);
-        IndexRequestBuilder[] indexRequestBuilders = new IndexRequestBuilder[numDocs + 1];
-        for (int i = 0; i < numDocs; i++) {
-            indexRequestBuilders[i] = client().prepareIndex("test", "test", Integer.toString(i)).setSource("field", "value");
-        }
-        indexRequestBuilders[numDocs] = client().prepareIndex("test", "test", Integer.toString(numDocs)).setSource("field", "other_value");
-        indexRandom(true, indexRequestBuilders);
-
-        SearchResponse searchResponse = client().prepareSearch("test").get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo((long) numDocs + 1));
-
-        DeleteByQueryResponse delete = newDeleteByQuery().setIndices("test").setQuery(QueryBuilders.termQuery("field", "value")).get();
-        assertDBQResponse(delete, numDocs, numDocs, 0L, 0L);
-
-        refresh();
-        searchResponse = client().prepareSearch("test").get();
-        assertNoFailures(searchResponse);
-        assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
-        assertSearchContextsClosed();
-    }
-
-    public void testConcurrentDeleteByQueriesOnDifferentDocs() throws Throwable {
-        createIndex("test");
-        ensureGreen();
-
-        final Thread[] threads =  new Thread[scaledRandomIntBetween(2, 5)];
-        final long docs = randomIntBetween(1, 50);
-        for (int i = 0; i < docs; i++) {
-            for (int j = 0; j < threads.length; j++) {
-                index("test", "test", String.valueOf(i * 10 + j), "field", j);
-            }
-        }
-        refresh();
-        assertHitCount(client().prepareSearch("test").setSize(0).get(), docs * threads.length);
-
-        final CountDownLatch start = new CountDownLatch(1);
-        final AtomicReference<Throwable> exceptionHolder = new AtomicReference<>();
-
-        for (int i = 0; i < threads.length; i++) {
-            final int threadNum = i;
-            assertHitCount(client().prepareSearch("test").setSize(0).setQuery(QueryBuilders.termQuery("field", threadNum)).get(), docs);
-
-            Runnable r =  new Runnable() {
-                @Override
-                public void run() {
-                    try {
-                        start.await();
-
-                        DeleteByQueryResponse rsp = newDeleteByQuery().setQuery(QueryBuilders.termQuery("field", threadNum)).get();
-                        assertDBQResponse(rsp, docs, docs, 0L, 0L);
-                    } catch (InterruptedException e) {
-                        Thread.currentThread().interrupt();
-                    } catch (Throwable e) {
-                        exceptionHolder.set(e);
-                        Thread.currentThread().interrupt();
-                    }
-                }
-            };
-            threads[i] = new Thread(r);
-            threads[i].start();
-        }
-
-        start.countDown();
-        for (Thread thread : threads) {
-            thread.join();
-        }
-
-        Throwable assertionError = exceptionHolder.get();
-        if (assertionError != null) {
-            throw assertionError;
-        }
-
-        refresh();
-        for (int i = 0; i < threads.length; i++) {
-            assertHitCount(client().prepareSearch("test").setSize(0).setQuery(QueryBuilders.termQuery("field", i)).get(), 0);
-        }
-        assertSearchContextsClosed();
-    }
-
-    public void testConcurrentDeleteByQueriesOnSameDocs() throws Throwable {
-        assertAcked(prepareCreate("test").setSettings(Settings.builder().put("index.refresh_interval", -1)));
-        ensureGreen();
-
-        final long docs = randomIntBetween(50, 100);
-        for (int i = 0; i < docs; i++) {
-            index("test", "test", String.valueOf(i), "foo", "bar");
-        }
-        refresh();
-        assertHitCount(client().prepareSearch("test").setSize(0).get(), docs);
-
-        final Thread[] threads =  new Thread[scaledRandomIntBetween(2, 9)];
-
-        final CountDownLatch start = new CountDownLatch(1);
-        final AtomicReference<Throwable> exceptionHolder = new AtomicReference<>();
-
-        final MatchQueryBuilder query = QueryBuilders.matchQuery("foo", "bar");
-        final AtomicLong deleted = new AtomicLong(0);
-
-        for (int i = 0; i < threads.length; i++) {
-            assertHitCount(client().prepareSearch("test").setSize(0).setQuery(query).get(), docs);
-
-            Runnable r =  new Runnable() {
-                @Override
-                public void run() {
-                    try {
-                        start.await();
-                        DeleteByQueryResponse rsp = newDeleteByQuery().setQuery(query).get();
-                        deleted.addAndGet(rsp.getTotalDeleted());
-
-                        assertThat(rsp.getTotalFound(), equalTo(docs));
-                    } catch (InterruptedException e) {
-                        Thread.currentThread().interrupt();
-                    } catch (Throwable e) {
-                        exceptionHolder.set(e);
-                        Thread.currentThread().interrupt();
-                    }
-                }
-            };
-            threads[i] = new Thread(r);
-            threads[i].start();
-        }
-
-        start.countDown();
-        for (Thread thread : threads) {
-            thread.join();
-        }
-        refresh();
-
-        Throwable assertionError = exceptionHolder.get();
-        if (assertionError != null) {
-            throw assertionError;
-        }
-        assertHitCount(client().prepareSearch("test").setSize(0).get(), 0L);
-        assertThat(deleted.get(), equalTo(docs));
-        assertSearchContextsClosed();
-    }
-
-    public void testDeleteByQueryOnReadOnlyIndex() throws Exception {
-        createIndex("test");
-        ensureGreen();
-
-        final long docs = randomIntBetween(1, 50);
-        for (int i = 0; i < docs; i++) {
-            index("test", "test", String.valueOf(i), "field", 1);
-        }
-        refresh();
-        assertHitCount(client().prepareSearch("test").setSize(0).get(), docs);
-
-        try {
-            enableIndexBlock("test", IndexMetaData.SETTING_READ_ONLY);
-            DeleteByQueryResponse rsp = newDeleteByQuery().setQuery(QueryBuilders.matchAllQuery()).get();
-            assertDBQResponse(rsp, docs, 0L, docs, 0L);
-        } finally {
-            disableIndexBlock("test", IndexMetaData.SETTING_READ_ONLY);
-        }
-
-        assertHitCount(client().prepareSearch("test").setSize(0).get(), docs);
-        assertSearchContextsClosed();
-    }
-
-    private DeleteByQueryRequestBuilder newDeleteByQuery() {
-        return new DeleteByQueryRequestBuilder(client(), DeleteByQueryAction.INSTANCE);
-    }
-
-    private void assertDBQResponse(DeleteByQueryResponse response, long found, long deleted, long failed, long missing) {
-        assertNotNull(response);
-        assertThat(response.isTimedOut(), equalTo(false));
-        assertThat(response.getShardFailures().length, equalTo(0));
-        assertThat(response.getTotalFound(), equalTo(found));
-        assertThat(response.getTotalDeleted(), equalTo(deleted));
-        assertThat(response.getTotalFailed(), equalTo(failed));
-        assertThat(response.getTotalMissing(), equalTo(missing));
-    }
-
-    private void assertSearchContextsClosed() throws Exception {
-        // The scroll id (and thus the underlying search context) is cleared in
-        // an async manner in TransportDeleteByQueryAction. so we need to use
-        // assertBusy() here to wait for the search context to be released.
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).get();
-                for (NodeStats nodeStat : nodesStats.getNodes()){
-                    assertThat(nodeStat.getIndices().getSearch().getOpenContexts(), equalTo(0L));
-                }
-            }
-        });
-    }
-}
diff --git a/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/test/rest/DeleteByQueryRestIT.java b/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/test/rest/DeleteByQueryRestIT.java
deleted file mode 100644
index 9674d541354..00000000000
--- a/plugins/delete-by-query/src/test/java/org/elasticsearch/plugin/deletebyquery/test/rest/DeleteByQueryRestIT.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.plugin.deletebyquery.test.rest;
-
-import com.carrotsearch.randomizedtesting.annotations.Name;
-import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import org.elasticsearch.test.rest.ESRestTestCase;
-import org.elasticsearch.test.rest.RestTestCandidate;
-import org.elasticsearch.test.rest.parser.RestTestParseException;
-
-import java.io.IOException;
-
-public class DeleteByQueryRestIT extends ESRestTestCase {
-
-    public DeleteByQueryRestIT(@Name("yaml") RestTestCandidate testCandidate) {
-        super(testCandidate);
-    }
-
-    @ParametersFactory
-    public static Iterable<Object[]> parameters() throws IOException, RestTestParseException {
-        return ESRestTestCase.createParameters(0, 1);
-    }
-}
-
diff --git a/plugins/delete-by-query/src/test/resources/rest-api-spec/api/delete_by_query.json b/plugins/delete-by-query/src/test/resources/rest-api-spec/api/delete_by_query.json
deleted file mode 100644
index d8d4446a1b2..00000000000
--- a/plugins/delete-by-query/src/test/resources/rest-api-spec/api/delete_by_query.json
+++ /dev/null
@@ -1,66 +0,0 @@
-{
-  "delete_by_query": {
-    "documentation": "https://www.elastic.co/guide/en/elasticsearch/plugins/master/plugins-delete-by-query.html",
-    "methods": ["DELETE"],
-    "url": {
-      "path": "/{index}/_query",
-      "paths": ["/{index}/_query", "/{index}/{type}/_query"],
-      "parts": {
-        "index": {
-          "type" : "list",
-          "required": true,
-          "description" : "A comma-separated list of indices to restrict the operation; use `_all` to perform the operation on all indices"
-        },
-        "type": {
-          "type" : "list",
-          "description" : "A comma-separated list of types to restrict the operation"
-        }
-      },
-      "params": {
-        "analyzer": {
-          "type" : "string",
-          "description" : "The analyzer to use for the query string"
-        },
-        "default_operator": {
-          "type" : "enum",
-          "options" : ["AND","OR"],
-          "default" : "OR",
-          "description" : "The default operator for query string query (AND or OR)"
-        },
-        "df": {
-          "type" : "string",
-          "description" : "The field to use as default where no field prefix is given in the query string"
-        },
-        "ignore_unavailable": {
-          "type" : "boolean",
-          "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
-        },
-        "allow_no_indices": {
-          "type" : "boolean",
-          "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
-        },
-        "expand_wildcards": {
-          "type" : "enum",
-          "options" : ["open","closed","none","all"],
-          "default" : "open",
-          "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both."
-        },
-        "q": {
-          "type" : "string",
-          "description" : "Query in the Lucene query string syntax"
-        },
-        "routing": {
-          "type" : "string",
-          "description" : "Specific routing value"
-        },
-        "timeout": {
-          "type" : "time",
-          "description" : "Explicit operation timeout"
-        }
-      }
-    },
-    "body": {
-      "description" : "A query to restrict the operation specified with the Query DSL"
-    }
-  }
-}
diff --git a/plugins/delete-by-query/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml b/plugins/delete-by-query/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml
deleted file mode 100644
index 124122a44bf..00000000000
--- a/plugins/delete-by-query/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml
+++ /dev/null
@@ -1,58 +0,0 @@
-setup:
-  - do:
-      index:
-          index:  test_1
-          type:   test
-          id:     1
-          body:   { foo: bar }
-
-  - do:
-      index:
-          index:  test_1
-          type:   test
-          id:     2
-          body:   { foo: baz }
-
-  - do:
-      index:
-          index:  test_1
-          type:   test
-          id:     3
-          body:   { foo: foo }
-
-  - do:
-      indices.refresh: {}
-
----
-"Basic delete_by_query":
-  - skip:
-      version: all
-      reason: "Test muted because of a REST test namespace conflict, see https://github.com/elastic/elasticsearch/issues/18469"
-
-  - do:
-      delete_by_query:
-        index:  test_1
-        body:
-          query:
-            match:
-              foo: bar
-
-  - do:
-      indices.refresh: {}
-
-  - do:
-      count:
-        index: test_1
-
-  - match:  { count:     2  }
-
----
-"Delete_by_query body without query element":
-  - do:
-      catch: request
-      delete_by_query:
-        index:  test_1
-        body:
-          match:
-            foo: bar
-
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
index 362b5d60341..a5b18cf65e0 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
@@ -103,7 +103,7 @@ fi
         echo "CONF_FILE=$CONF_FILE" >> /etc/sysconfig/elasticsearch;
     fi
 
-    run_elasticsearch_service 1 -Edefault.config="$CONF_FILE"
+    run_elasticsearch_service 1 -Ees.default.config="$CONF_FILE"
 
     # remove settings again otherwise cleaning up before next testrun will fail
     if is_dpkg ; then
@@ -208,10 +208,6 @@ fi
     install_and_check_plugin discovery gce google-api-client-*.jar
 }
 
-@test "[$GROUP] install delete by query plugin" {
-    install_and_check_plugin - delete-by-query
-}
-
 @test "[$GROUP] install discovery-azure plugin" {
     install_and_check_plugin discovery azure azure-core-*.jar
 }
@@ -347,10 +343,6 @@ fi
     remove_plugin discovery-gce
 }
 
-@test "[$GROUP] remove delete by query plugin" {
-    remove_plugin delete-by-query
-}
-
 @test "[$GROUP] remove discovery-azure plugin" {
     remove_plugin discovery-azure
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
index 834133138b2..d1f10dbfbfc 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
@@ -1,6 +1,6 @@
 {
   "delete_by_query": {
-    "documentation": "https://www.elastic.co/guide/en/elasticsearch/plugins/master/plugins-delete-by-query.html",
+    "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-delete-by-query.html",
     "methods": ["POST"],
     "url": {
       "path": "/{index}/_delete_by_query",
diff --git a/settings.gradle b/settings.gradle
index ded500ebe56..86673589e10 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -26,7 +26,6 @@ List projects = [
   'plugins:analysis-phonetic',
   'plugins:analysis-smartcn',
   'plugins:analysis-stempel',
-  'plugins:delete-by-query',
   'plugins:discovery-azure',
   'plugins:discovery-ec2',
   'plugins:discovery-gce',

From 9c9bea925874b4dc9a666c316b588b01c86bddce Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Tue, 24 May 2016 16:46:05 +0200
Subject: [PATCH 0437/1311] Set version to 5.0.0-alpha3 (#18550)

* Set version to 5.0.0-alpha3

* Updated version in qa/backwards tests too
---
 buildSrc/version.properties                            |  2 +-
 core/src/main/java/org/elasticsearch/Version.java      | 10 +++++-----
 .../index/mapper/internal/IndexFieldMapper.java        |  2 +-
 qa/backwards-5.0/build.gradle                          |  2 +-
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index b6e64a3c263..6669abb90b3 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,4 +1,4 @@
-elasticsearch     = 5.0.0
+elasticsearch     = 5.0.0-alpha3
 lucene            = 6.0.0
 
 # optional dependencies
diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java
index 9cc526d8f97..b2b4e9c296d 100644
--- a/core/src/main/java/org/elasticsearch/Version.java
+++ b/core/src/main/java/org/elasticsearch/Version.java
@@ -75,9 +75,9 @@ public class Version {
     public static final Version V_5_0_0_alpha1 = new Version(V_5_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_6_0_0);
     public static final int V_5_0_0_alpha2_ID = 5000002;
     public static final Version V_5_0_0_alpha2 = new Version(V_5_0_0_alpha2_ID, org.apache.lucene.util.Version.LUCENE_6_0_0);
-    public static final int V_5_0_0_ID = 5000099;
-    public static final Version V_5_0_0 = new Version(V_5_0_0_ID, org.apache.lucene.util.Version.LUCENE_6_0_0);
-    public static final Version CURRENT = V_5_0_0;
+    public static final int V_5_0_0_alpha3_ID = 5000003;
+    public static final Version V_5_0_0_alpha3 = new Version(V_5_0_0_alpha3_ID, org.apache.lucene.util.Version.LUCENE_6_0_0);
+    public static final Version CURRENT = V_5_0_0_alpha3;
 
     static {
         assert CURRENT.luceneVersion.equals(org.apache.lucene.util.Version.LATEST) : "Version must be upgraded to ["
@@ -90,8 +90,8 @@ public class Version {
 
     public static Version fromId(int id) {
         switch (id) {
-            case V_5_0_0_ID:
-                return V_5_0_0;
+            case V_5_0_0_alpha3_ID:
+                return V_5_0_0_alpha3;
             case V_5_0_0_alpha2_ID:
                 return V_5_0_0_alpha2;
             case V_5_0_0_alpha1_ID:
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
index 408641fb67e..f56b04c92aa 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java
@@ -85,7 +85,7 @@ public class IndexFieldMapper extends MetadataFieldMapper {
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
         public MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
-            if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0)) {
+            if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0_alpha3)) {
                 throw new MapperParsingException(NAME + " is not configurable");
             }
             return new Builder(parserContext.mapperService().fullName(NAME));
diff --git a/qa/backwards-5.0/build.gradle b/qa/backwards-5.0/build.gradle
index 93d361c989c..70889ad5009 100644
--- a/qa/backwards-5.0/build.gradle
+++ b/qa/backwards-5.0/build.gradle
@@ -18,6 +18,6 @@ integTest {
   cluster {
     numNodes = 2
     numBwcNodes = 1
-    bwcVersion = "5.0.0-SNAPSHOT" // this is the same as the current version until we released the first RC
+    bwcVersion = "5.0.0-alpha3-SNAPSHOT" // this is the same as the current version until we released the first RC
   }
 }

From 72eb621bcecafd391e300944c19ec2842e13b940 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 12 May 2016 12:43:01 -0400
Subject: [PATCH 0438/1311] Docs: Replace [source,json] with [source,js]

The syntax highlighter only supports [source,js].

Also adds a check to the rest test generator that runs during
the build that'll fail the build if it sees `[source,json]`.
---
 .../doc/RestTestsFromSnippetsTask.groovy       |  4 ++++
 docs/plugins/analysis-icu.asciidoc             | 18 +++++++++---------
 docs/plugins/analysis-kuromoji.asciidoc        | 16 ++++++++--------
 docs/plugins/analysis-phonetic.asciidoc        |  2 +-
 docs/plugins/lang-javascript.asciidoc          |  6 +++---
 docs/plugins/lang-python.asciidoc              |  6 +++---
 docs/plugins/repository-azure.asciidoc         |  2 +-
 docs/plugins/repository-s3.asciidoc            |  2 +-
 docs/plugins/store-smb.asciidoc                |  2 +-
 .../allocation/prioritization.asciidoc         |  4 ++--
 .../migration/migrate_5_0/mapping.asciidoc     |  4 ++--
 .../modules/scripting/security.asciidoc        |  6 +++---
 12 files changed, 38 insertions(+), 34 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
index ef8c8e280ed..9f840df36e1 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
@@ -87,6 +87,10 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
          * calls buildTest to actually build the test.
          */
         void handleSnippet(Snippet snippet) {
+            if (snippet.language == 'json') {
+                throw new InvalidUserDataException(
+                        "$snippet: Use `js` instead of `json`.")
+            }
             if (snippet.testSetup) {
                 setup(snippet)
                 return
diff --git a/docs/plugins/analysis-icu.asciidoc b/docs/plugins/analysis-icu.asciidoc
index 19b4fdb07cd..815e6285235 100644
--- a/docs/plugins/analysis-icu.asciidoc
+++ b/docs/plugins/analysis-icu.asciidoc
@@ -48,7 +48,7 @@ convert `nfc` to `nfd` or `nfkc` to `nfkd` respectively:
 Here are two examples, the default usage and a customised character filter:
 
 
-[source,json]
+[source,js]
 --------------------------------------------------
 PUT icu_sample
 {
@@ -96,7 +96,7 @@ but adds better support for some Asian languages by using a dictionary-based
 approach to identify words in Thai, Lao, Chinese, Japanese, and Korean, and
 using custom rules to break Myanmar and Khmer text into syllables.
 
-[source,json]
+[source,js]
 --------------------------------------------------
 PUT icu_sample
 {
@@ -137,7 +137,7 @@ As a demonstration of how the rule files can be used, save the following user fi
 
 Then create an analyzer to use this rule file as follows:
 
-[source,json]
+[source,js]
 --------------------------------------------------
 PUT icu_sample
 {
@@ -167,7 +167,7 @@ POST icu_sample/_analyze?analyzer=my_analyzer&text=Elasticsearch. Wow!
 
 The above `analyze` request returns the following:
 
-[source,json]
+[source,js]
 --------------------------------------------------
 # Result
 {
@@ -198,7 +198,7 @@ You should probably prefer the <<analysis-icu-normalization-charfilter,Normaliza
 
 Here are two examples, the default usage and a customised token filter:
 
-[source,json]
+[source,js]
 --------------------------------------------------
 PUT icu_sample
 {
@@ -244,7 +244,7 @@ Case folding of Unicode characters based on `UTR#30`, like the
 on steroids. It registers itself as the `icu_folding` token filter and is
 available to all indices:
 
-[source,json]
+[source,js]
 --------------------------------------------------
 PUT icu_sample
 {
@@ -278,7 +278,7 @@ to note that both upper and lowercase forms should be specified, and that
 these filtered character are not lowercased which is why we add the
 `lowercase` filter as well:
 
-[source,json]
+[source,js]
 --------------------------------------------------
 PUT icu_sample
 {
@@ -319,7 +319,7 @@ which is a best-effort attempt at language-neutral sorting.
 Below is an example of how to set up a field for sorting German names in
 ``phonebook'' order:
 
-[source,json]
+[source,js]
 --------------------------------------------------
 PUT /my_index
 {
@@ -452,7 +452,7 @@ rulesets are not yet supported.
 
 For example:
 
-[source,json]
+[source,js]
 --------------------------------------------------
 PUT icu_sample
 {
diff --git a/docs/plugins/analysis-kuromoji.asciidoc b/docs/plugins/analysis-kuromoji.asciidoc
index 2d679de07f4..f29d13c2867 100644
--- a/docs/plugins/analysis-kuromoji.asciidoc
+++ b/docs/plugins/analysis-kuromoji.asciidoc
@@ -146,7 +146,7 @@ If both parameters are used, the largest number of both is applied.
 
 Then create an analyzer as follows:
 
-[source,json]
+[source,js]
 --------------------------------------------------
 PUT kuromoji_sample
 {
@@ -178,7 +178,7 @@ POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=�京スカイツリー
 
 The above `analyze` request returns the following:
 
-[source,json]
+[source,js]
 --------------------------------------------------
 # Result
 {
@@ -204,7 +204,7 @@ The above `analyze` request returns the following:
 The `kuromoji_baseform` token filter replaces terms with their
 BaseFormAttribute. This acts as a lemmatizer for verbs and adjectives.
 
-[source,json]
+[source,js]
 --------------------------------------------------
 PUT kuromoji_sample
 {
@@ -253,7 +253,7 @@ part-of-speech tags. It accepts the following setting:
     An array of part-of-speech tags that should be removed. It defaults to the
     `stoptags.txt` file embedded in the `lucene-analyzer-kuromoji.jar`.
 
-[source,json]
+[source,js]
 --------------------------------------------------
 PUT kuromoji_sample
 {
@@ -322,7 +322,7 @@ to `true`. The default when defining a custom `kuromoji_readingform`, however,
 is `false`.  The only reason to use the custom form is if you need the
 katakana reading form:
 
-[source,json]
+[source,js]
 --------------------------------------------------
 PUT kuromoji_sample
 {
@@ -379,7 +379,7 @@ This token filter accepts the following setting:
     is `4`).
 
 
-[source,json]
+[source,js]
 --------------------------------------------------
 PUT kuromoji_sample
 {
@@ -425,7 +425,7 @@ the predefined `_japanese_` stopwords list.  If you want to use a different
 predefined list, then use the
 {ref}/analysis-stop-tokenfilter.html[`stop` token filter] instead.
 
-[source,json]
+[source,js]
 --------------------------------------------------
 PUT kuromoji_sample
 {
@@ -480,7 +480,7 @@ The above request returns:
 The `kuromoji_number` token filter normalizes Japanese numbers (kansūji)
 to regular Arabic decimal numbers in half-width characters.
 
-[source,json]
+[source,js]
 --------------------------------------------------
 PUT kuromoji_sample
 {
diff --git a/docs/plugins/analysis-phonetic.asciidoc b/docs/plugins/analysis-phonetic.asciidoc
index a0055ef5d69..f37547d9f75 100644
--- a/docs/plugins/analysis-phonetic.asciidoc
+++ b/docs/plugins/analysis-phonetic.asciidoc
@@ -50,7 +50,7 @@ The `phonetic` token filter takes the following settings:
     token. Accepts `true` (default) and `false`.  Not supported by
     `beidermorse` encoding.
 
-[source,json]
+[source,js]
 --------------------------------------------------
 PUT phonetic_sample
 {
diff --git a/docs/plugins/lang-javascript.asciidoc b/docs/plugins/lang-javascript.asciidoc
index f5887d53e9c..bfee18a0a89 100644
--- a/docs/plugins/lang-javascript.asciidoc
+++ b/docs/plugins/lang-javascript.asciidoc
@@ -51,7 +51,7 @@ See <<lang-javascript-file>> for a safer option.
 If you have enabled {ref}/modules-scripting-security.html#enable-dynamic-scripting[inline scripts],
 you can use JavaScript as follows:
 
-[source,json]
+[source,js]
 ----
 DELETE test
 
@@ -94,7 +94,7 @@ See <<lang-javascript-file>> for a safer option.
 If you have enabled {ref}/modules-scripting-security.html#enable-dynamic-scripting[stored scripts],
 you can use JavaScript as follows:
 
-[source,json]
+[source,js]
 ----
 DELETE test
 
@@ -155,7 +155,7 @@ doc["num"].value * factor
 
 then use the script as follows:
 
-[source,json]
+[source,js]
 ----
 DELETE test
 
diff --git a/docs/plugins/lang-python.asciidoc b/docs/plugins/lang-python.asciidoc
index 070ed3eb6a9..af2fc8a3a39 100644
--- a/docs/plugins/lang-python.asciidoc
+++ b/docs/plugins/lang-python.asciidoc
@@ -50,7 +50,7 @@ See <<lang-python-file>> for a safer option.
 If you have enabled {ref}/modules-scripting-security.html#enable-dynamic-scripting[inline scripts],
 you can use Python as follows:
 
-[source,json]
+[source,js]
 ----
 DELETE test
 
@@ -93,7 +93,7 @@ See <<lang-python-file>> for a safer option.
 If you have enabled {ref}/modules-scripting-security.html#enable-dynamic-scripting[stored scripts],
 you can use Python as follows:
 
-[source,json]
+[source,js]
 ----
 DELETE test
 
@@ -154,7 +154,7 @@ doc["num"].value * factor
 
 then use the script as follows:
 
-[source,json]
+[source,js]
 ----
 DELETE test
 
diff --git a/docs/plugins/repository-azure.asciidoc b/docs/plugins/repository-azure.asciidoc
index dd2ae068eb6..f338728a35d 100644
--- a/docs/plugins/repository-azure.asciidoc
+++ b/docs/plugins/repository-azure.asciidoc
@@ -129,7 +129,7 @@ The Azure repository supports following settings:
 
 Some examples, using scripts:
 
-[source,json]
+[source,js]
 ----
 # The simpliest one
 PUT _snapshot/my_backup1
diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc
index 61ae79180dd..82485b6d9d0 100644
--- a/docs/plugins/repository-s3.asciidoc
+++ b/docs/plugins/repository-s3.asciidoc
@@ -137,7 +137,7 @@ use `S3SignerType`, which is Signature Version 2.
 
 The S3 repository is using S3 to store snapshots. The S3 repository can be created using the following command:
 
-[source,json]
+[source,js]
 ----
 PUT _snapshot/my_s3_repository
 {
diff --git a/docs/plugins/store-smb.asciidoc b/docs/plugins/store-smb.asciidoc
index cf0094173d1..731894ae0a8 100644
--- a/docs/plugins/store-smb.asciidoc
+++ b/docs/plugins/store-smb.asciidoc
@@ -68,7 +68,7 @@ Note that setting will be applied for newly created indices.
 
 It can also be set on a per-index basis at index creation time:
 
-[source,json]
+[source,js]
 ----
 PUT my_index
 {
diff --git a/docs/reference/index-modules/allocation/prioritization.asciidoc b/docs/reference/index-modules/allocation/prioritization.asciidoc
index eecc98abcb0..92051cc4dbc 100644
--- a/docs/reference/index-modules/allocation/prioritization.asciidoc
+++ b/docs/reference/index-modules/allocation/prioritization.asciidoc
@@ -13,7 +13,7 @@ This means that, by default, newer indices will be recovered before older indice
 Use the per-index dynamically updateable `index.priority` setting to customise
 the index prioritization order.  For instance:
 
-[source,json]
+[source,js]
 ------------------------------
 PUT index_1
 
@@ -45,7 +45,7 @@ In the above example:
 This setting accepts an integer, and can be updated on a live index with the
 <<indices-update-settings,update index settings API>>:
 
-[source,json]
+[source,js]
 ------------------------------
 PUT index_4/_settings
 {
diff --git a/docs/reference/migration/migrate_5_0/mapping.asciidoc b/docs/reference/migration/migrate_5_0/mapping.asciidoc
index 233ef84ae09..84dea729ec1 100644
--- a/docs/reference/migration/migrate_5_0/mapping.asciidoc
+++ b/docs/reference/migration/migrate_5_0/mapping.asciidoc
@@ -20,7 +20,7 @@ values.  For backwards compatibility purposes, during the 5.x series:
 
 String mappings now have the following default mappings:
 
-[source,json]
+[source,js]
 ---------------
 {
   "type": "text",
@@ -135,7 +135,7 @@ will reject this option.
 Core types no longer support the object notation, which was used to provide
 per document boosts as follows:
 
-[source,json]
+[source,js]
 ---------------
 {
   "value": "field_value",
diff --git a/docs/reference/modules/scripting/security.asciidoc b/docs/reference/modules/scripting/security.asciidoc
index 89065d86b5f..4d35b0da250 100644
--- a/docs/reference/modules/scripting/security.asciidoc
+++ b/docs/reference/modules/scripting/security.asciidoc
@@ -165,7 +165,7 @@ https://github.com/elastic/elasticsearch/blob/{branch}/core/src/main/java/org/el
 In a script, attempting to load a class that does not appear in the whitelist
 _may_ result in a `ClassNotFoundException`, for instance this script:
 
-[source,json]
+[source,js]
 ------------------------------
 GET _search
 {
@@ -179,7 +179,7 @@ GET _search
 
 will return the following exception:
 
-[source,json]
+[source,js]
 ------------------------------
 {
   "reason": {
@@ -207,7 +207,7 @@ use(groovy.time.TimeCategory); new Date(123456789).format('HH')
 
 Returns the following exception:
 
-[source,json]
+[source,js]
 ------------------------------
 {
   "reason": {

From f210605af82ab0642b6294c08163c81b10e84969 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 24 May 2016 11:26:36 -0400
Subject: [PATCH 0439/1311] Add test for parsing fractional seconds

This commit adds a test that ensures that strings containing a
fractional number of seconds are correctly parsed into milliseconds.

Relates #18548
---
 .../java/org/elasticsearch/common/unit/TimeValueTests.java     | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
index 20568826d40..3cd68bea038 100644
--- a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
+++ b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
@@ -83,6 +83,9 @@ public class TimeValueTests extends ESTestCase {
         assertEquals(new TimeValue(10, TimeUnit.SECONDS),
                      TimeValue.parseTimeValue("10S", null, "test"));
 
+        assertEquals(new TimeValue(100, TimeUnit.MILLISECONDS),
+                    TimeValue.parseTimeValue("0.1s", null, "test"));
+
         assertEquals(new TimeValue(10, TimeUnit.MINUTES),
                      TimeValue.parseTimeValue("10 m", null, "test"));
         assertEquals(new TimeValue(10, TimeUnit.MINUTES),

From a93f578bf681f416f7f5f2c9040bd03c266e0508 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 15 Apr 2016 17:32:25 -0400
Subject: [PATCH 0440/1311] Move parsing of allocation commands into REST

Port them to the ObjectParser.

Don't let plugins register custom allocation commands
---
 .../resources/checkstyle_suppressions.xml     |   2 -
 .../reroute/ClusterRerouteRequest.java        |  61 ++---
 .../reroute/ClusterRerouteRequestBuilder.java |  15 +-
 .../AbstractAllocateAllocationCommand.java    |  19 ++
 .../AllocateReplicaAllocationCommand.java     |   2 -
 .../allocation/command/AllocationCommand.java |   5 +-
 .../command/AllocationCommands.java           |  35 ++-
 .../command/BasePrimaryAllocationCommand.java |  14 ++
 .../command/CancelAllocationCommand.java      |  20 ++
 .../command/MoveAllocationCommand.java        |  20 ++
 .../common/network/NetworkModule.java         |  20 +-
 .../reroute/RestClusterRerouteAction.java     |  61 +++--
 .../reroute/ClusterRerouteRequestTests.java   | 214 ++++++++++++++++++
 .../cluster/reroute/ClusterRerouteTests.java  |   2 +-
 .../migration/migrate_5_0/allocation.asciidoc |   5 +
 15 files changed, 405 insertions(+), 90 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 90b8aa8423f..63133dd851b 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -37,8 +37,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]repositories[/\\]put[/\\]TransportPutRepositoryAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]repositories[/\\]verify[/\\]TransportVerifyRepositoryAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]repositories[/\\]verify[/\\]VerifyRepositoryRequestBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]reroute[/\\]ClusterRerouteRequest.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]reroute[/\\]ClusterRerouteRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]reroute[/\\]TransportClusterRerouteAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]settings[/\\]ClusterUpdateSettingsAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]settings[/\\]ClusterUpdateSettingsRequestBuilder.java" checks="LineLength" />
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequest.java
index dcc45ab21b9..4ec729bbc53 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequest.java
@@ -19,20 +19,15 @@
 
 package org.elasticsearch.action.admin.cluster.reroute;
 
-import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommand;
-import org.elasticsearch.cluster.routing.allocation.command.AllocationCommandRegistry;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.Objects;
 
 /**
  * Request to submit cluster reroute allocation commands
@@ -109,8 +104,8 @@ public class ClusterRerouteRequest extends AcknowledgedRequest<ClusterRerouteReq
     /**
      * Set the allocation commands to execute.
      */
-    public ClusterRerouteRequest commands(AllocationCommand... commands) {
-        this.commands = new AllocationCommands(commands);
+    public ClusterRerouteRequest commands(AllocationCommands commands) {
+        this.commands = commands;
         return this;
     }
 
@@ -121,35 +116,6 @@ public class ClusterRerouteRequest extends AcknowledgedRequest<ClusterRerouteReq
         return commands;
     }
 
-    /**
-     * Sets the source for the request.
-     */
-    public ClusterRerouteRequest source(BytesReference source, AllocationCommandRegistry registry, ParseFieldMatcher parseFieldMatcher)
-            throws Exception {
-        try (XContentParser parser = XContentHelper.createParser(source)) {
-            XContentParser.Token token;
-            String currentFieldName = null;
-            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                if (token == XContentParser.Token.FIELD_NAME) {
-                    currentFieldName = parser.currentName();
-                } else if (token == XContentParser.Token.START_ARRAY) {
-                    if ("commands".equals(currentFieldName)) {
-                        this.commands = AllocationCommands.fromXContent(parser, parseFieldMatcher, registry);
-                    } else {
-                        throw new ElasticsearchParseException("failed to parse reroute request, got start array with wrong field name [{}]", currentFieldName);
-                    }
-                } else if (token.isValue()) {
-                    if ("dry_run".equals(currentFieldName) || "dryRun".equals(currentFieldName)) {
-                        dryRun = parser.booleanValue();
-                    } else {
-                        throw new ElasticsearchParseException("failed to parse reroute request, got value with wrong field name [{}]", currentFieldName);
-                    }
-                }
-            }
-        }
-        return this;
-    }
-
     @Override
     public ActionRequestValidationException validate() {
         return null;
@@ -174,4 +140,25 @@ public class ClusterRerouteRequest extends AcknowledgedRequest<ClusterRerouteReq
         out.writeBoolean(retryFailed);
         writeTimeout(out);
     }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        ClusterRerouteRequest other = (ClusterRerouteRequest) obj;
+        // Override equals and hashCode for testing
+        return Objects.equals(commands, other.commands) &&
+                Objects.equals(dryRun, other.dryRun) &&
+                Objects.equals(explain, other.explain) &&
+                Objects.equals(timeout, other.timeout) &&
+                Objects.equals(retryFailed, other.retryFailed) &&
+                Objects.equals(masterNodeTimeout, other.masterNodeTimeout);
+    }
+
+    @Override
+    public int hashCode() {
+        // Override equals and hashCode for testing
+        return Objects.hash(commands, dryRun, explain, timeout, retryFailed, masterNodeTimeout);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestBuilder.java
index e5d130cf812..3a7ffbeb44d 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestBuilder.java
@@ -22,13 +22,12 @@ package org.elasticsearch.action.admin.cluster.reroute;
 import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommand;
-import org.elasticsearch.common.bytes.BytesReference;
 
 /**
  * Builder for a cluster reroute request
  */
-public class ClusterRerouteRequestBuilder extends AcknowledgedRequestBuilder<ClusterRerouteRequest, ClusterRerouteResponse, ClusterRerouteRequestBuilder> {
-
+public class ClusterRerouteRequestBuilder
+        extends AcknowledgedRequestBuilder<ClusterRerouteRequest, ClusterRerouteResponse, ClusterRerouteRequestBuilder> {
     public ClusterRerouteRequestBuilder(ElasticsearchClient client, ClusterRerouteAction action) {
         super(client, action, new ClusterRerouteRequest());
     }
@@ -68,12 +67,4 @@ public class ClusterRerouteRequestBuilder extends AcknowledgedRequestBuilder<Clu
         request.setRetryFailed(retryFailed);
         return this;
     }
-
-    /**
-     * Sets the commands for the request to execute.
-     */
-    public ClusterRerouteRequestBuilder setCommands(AllocationCommand... commands) throws Exception {
-        request.commands(commands);
-        return this;
-    }
-}
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java
index 20918159619..0fb27a80154 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java
@@ -38,6 +38,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.Objects;
 import java.util.function.Consumer;
 import java.util.function.Function;
 
@@ -228,4 +229,22 @@ public abstract class AbstractAllocateAllocationCommand implements AllocationCom
 
     protected void extraXContent(XContentBuilder builder) throws IOException {
     }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        AbstractAllocateAllocationCommand other = (AbstractAllocateAllocationCommand) obj;
+        // Override equals and hashCode for testing
+        return Objects.equals(index, other.index) &&
+                Objects.equals(shardId, other.shardId) &&
+                Objects.equals(node, other.node);
+    }
+
+    @Override
+    public int hashCode() {
+        // Override equals and hashCode for testing
+        return Objects.hash(index, shardId, node);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateReplicaAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateReplicaAllocationCommand.java
index b651580ea74..8c47deee66f 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateReplicaAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateReplicaAllocationCommand.java
@@ -136,6 +136,4 @@ public class AllocateReplicaAllocationCommand extends AbstractAllocateAllocation
         initializeUnassignedShard(allocation, routingNodes, routingNode, shardRouting);
         return new RerouteExplanation(this, decision);
     }
-
-
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommand.java
index 736018531fa..92c1ffa9921 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommand.java
@@ -22,13 +22,16 @@ package org.elasticsearch.cluster.routing.allocation.command;
 import org.elasticsearch.cluster.routing.allocation.RerouteExplanation;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 
 /**
- * This interface defines the basic methods of commands for allocation
+ * A command to move shards in some way.
+ *
+ * Commands are registered in {@link NetworkModule}.
  */
 public interface AllocationCommand extends NamedWriteable, ToXContent {
     interface Parser<T extends AllocationCommand> {
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java
index ca0eab6e33b..10ba3f55944 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocationCommands.java
@@ -20,12 +20,12 @@
 package org.elasticsearch.cluster.routing.allocation.command;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.RoutingExplanations;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 
@@ -33,12 +33,13 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * A simple {@link AllocationCommand} composite managing several
  * {@link AllocationCommand} implementations
  */
-public class AllocationCommands {
+public class AllocationCommands extends ToXContentToBytes {
     private final List<AllocationCommand> commands = new ArrayList<>();
 
     /**
@@ -171,21 +172,31 @@ public class AllocationCommands {
         return commands;
     }
 
-    /**
-     * Writes {@link AllocationCommands} to a {@link XContentBuilder}
-     *
-     * @param commands {@link AllocationCommands} to write
-     * @param builder {@link XContentBuilder} to use
-     * @param params Parameters to use for building
-     * @throws IOException if something bad happens while building the content
-     */
-    public static void toXContent(AllocationCommands commands, XContentBuilder builder, ToXContent.Params params) throws IOException {
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startArray("commands");
-        for (AllocationCommand command : commands.commands) {
+        for (AllocationCommand command : commands) {
             builder.startObject();
             builder.field(command.name(), command);
             builder.endObject();
         }
         builder.endArray();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        AllocationCommands other = (AllocationCommands) obj;
+        // Override equals and hashCode for testing
+        return Objects.equals(commands, other.commands);
+    }
+
+    @Override
+    public int hashCode() {
+        // Override equals and hashCode for testing
+        return Objects.hashCode(commands);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/BasePrimaryAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/BasePrimaryAllocationCommand.java
index 0013061e8ea..2eb3af9d4f9 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/BasePrimaryAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/BasePrimaryAllocationCommand.java
@@ -83,4 +83,18 @@ public abstract class BasePrimaryAllocationCommand extends AbstractAllocateAlloc
     protected void extraXContent(XContentBuilder builder) throws IOException {
         builder.field(ACCEPT_DATA_LOSS_FIELD, acceptDataLoss);
     }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (false == super.equals(obj)) {
+            return false;
+        }
+        BasePrimaryAllocationCommand other = (BasePrimaryAllocationCommand) obj;
+        return acceptDataLoss == other.acceptDataLoss;
+    }
+
+    @Override
+    public int hashCode() {
+        return 31 * super.hashCode() + Boolean.hashCode(acceptDataLoss);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java
index 60b0842b273..9adb3e30708 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java
@@ -35,6 +35,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.Objects;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
 
@@ -240,4 +241,23 @@ public class CancelAllocationCommand implements AllocationCommand {
         }
         return new CancelAllocationCommand(index, shardId, nodeId, allowPrimary);
     }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        CancelAllocationCommand other = (CancelAllocationCommand) obj;
+        // Override equals and hashCode for testing
+        return Objects.equals(index, other.index) &&
+                Objects.equals(shardId, other.shardId) &&
+                Objects.equals(node, other.node) &&
+                Objects.equals(allowPrimary, other.allowPrimary);
+    }
+
+    @Override
+    public int hashCode() {
+        // Override equals and hashCode for testing
+        return Objects.hash(index, shardId, node, allowPrimary);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/MoveAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/MoveAllocationCommand.java
index a2e1a54e515..69bd8f0eeca 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/MoveAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/MoveAllocationCommand.java
@@ -34,6 +34,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.Objects;
 
 /**
  * A command that moves a shard from a specific node to another node.<br>
@@ -195,4 +196,23 @@ public class MoveAllocationCommand implements AllocationCommand {
         }
         return new MoveAllocationCommand(index, shardId, fromNode, toNode);
     }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        MoveAllocationCommand other = (MoveAllocationCommand) obj;
+        // Override equals and hashCode for testing
+        return Objects.equals(index, other.index) &&
+                Objects.equals(shardId, other.shardId) &&
+                Objects.equals(fromNode, other.fromNode) &&
+                Objects.equals(toNode, other.toNode);
+    }
+
+    @Override
+    public int hashCode() {
+        // Override equals and hashCode for testing
+        return Objects.hash(index, shardId, fromNode, toNode);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
index eebd8544bf8..201e5297511 100644
--- a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
+++ b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
@@ -19,9 +19,6 @@
 
 package org.elasticsearch.common.network;
 
-import java.util.Arrays;
-import java.util.List;
-
 import org.elasticsearch.action.support.replication.ReplicationTask;
 import org.elasticsearch.client.transport.TransportClientNodesService;
 import org.elasticsearch.client.transport.support.TransportProxyClient;
@@ -68,6 +65,12 @@ import org.elasticsearch.rest.action.admin.cluster.snapshots.restore.RestRestore
 import org.elasticsearch.rest.action.admin.cluster.snapshots.status.RestSnapshotsStatusAction;
 import org.elasticsearch.rest.action.admin.cluster.state.RestClusterStateAction;
 import org.elasticsearch.rest.action.admin.cluster.stats.RestClusterStatsAction;
+import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestDeleteSearchTemplateAction;
+import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestDeleteStoredScriptAction;
+import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestGetSearchTemplateAction;
+import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestGetStoredScriptAction;
+import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestPutSearchTemplateAction;
+import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestPutStoredScriptAction;
 import org.elasticsearch.rest.action.admin.cluster.tasks.RestPendingClusterTasksAction;
 import org.elasticsearch.rest.action.admin.indices.alias.RestIndicesAliasesAction;
 import org.elasticsearch.rest.action.admin.indices.alias.delete.RestIndexDeleteAliasesAction;
@@ -134,17 +137,11 @@ import org.elasticsearch.rest.action.ingest.RestGetPipelineAction;
 import org.elasticsearch.rest.action.ingest.RestPutPipelineAction;
 import org.elasticsearch.rest.action.ingest.RestSimulatePipelineAction;
 import org.elasticsearch.rest.action.main.RestMainAction;
-import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestDeleteStoredScriptAction;
-import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestGetStoredScriptAction;
-import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestPutStoredScriptAction;
 import org.elasticsearch.rest.action.search.RestClearScrollAction;
 import org.elasticsearch.rest.action.search.RestMultiSearchAction;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.rest.action.search.RestSearchScrollAction;
 import org.elasticsearch.rest.action.suggest.RestSuggestAction;
-import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestDeleteSearchTemplateAction;
-import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestGetSearchTemplateAction;
-import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestPutSearchTemplateAction;
 import org.elasticsearch.rest.action.termvectors.RestMultiTermVectorsAction;
 import org.elasticsearch.rest.action.termvectors.RestTermVectorsAction;
 import org.elasticsearch.rest.action.update.RestUpdateAction;
@@ -154,6 +151,9 @@ import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.local.LocalTransport;
 import org.elasticsearch.transport.netty.NettyTransport;
 
+import java.util.Arrays;
+import java.util.List;
+
 /**
  * A module to handle registering and binding all network related classes.
  */
@@ -397,7 +397,7 @@ public class NetworkModule extends AbstractModule {
      * @param commandName the names under which the command should be parsed. The {@link ParseField#getPreferredName()} is special because
      *        it is the name under which the command's reader is registered.
      */
-    public <T extends AllocationCommand> void registerAllocationCommand(Writeable.Reader<T> reader, AllocationCommand.Parser<T> parser,
+    private <T extends AllocationCommand> void registerAllocationCommand(Writeable.Reader<T> reader, AllocationCommand.Parser<T> parser,
             ParseField commandName) {
         allocationCommandRegistry.register(parser, commandName);
         namedWriteableRegistry.register(AllocationCommand.class, commandName.getPreferredName(), reader);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/reroute/RestClusterRerouteAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/reroute/RestClusterRerouteAction.java
index b34c6726c09..a06466137a5 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/reroute/RestClusterRerouteAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/reroute/RestClusterRerouteAction.java
@@ -25,12 +25,20 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommandRegistry;
+import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
@@ -43,12 +51,17 @@ import java.util.EnumSet;
 /**
  */
 public class RestClusterRerouteAction extends BaseRestHandler {
+    private static final ObjectParser<ClusterRerouteRequest, ParseContext> PARSER = new ObjectParser<>("cluster_reroute");
+    static {
+        PARSER.declareField((p, v, c) -> v.commands(AllocationCommands.fromXContent(p, c.getParseFieldMatcher(), c.registry)),
+                new ParseField("commands"), ValueType.OBJECT_ARRAY);
+        PARSER.declareBoolean(ClusterRerouteRequest::dryRun, new ParseField("dry_run"));
+    }
 
-    private final SettingsFilter settingsFilter;
-
-    private static String DEFAULT_METRICS = Strings
+    private static final String DEFAULT_METRICS = Strings
             .arrayToCommaDelimitedString(EnumSet.complementOf(EnumSet.of(ClusterState.Metric.METADATA)).toArray());
 
+    private final SettingsFilter settingsFilter;
     private final AllocationCommandRegistry registry;
 
     @Inject
@@ -62,16 +75,7 @@ public class RestClusterRerouteAction extends BaseRestHandler {
 
     @Override
     public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws Exception {
-        final ClusterRerouteRequest clusterRerouteRequest = Requests.clusterRerouteRequest();
-        clusterRerouteRequest.dryRun(request.paramAsBoolean("dry_run", clusterRerouteRequest.dryRun()));
-        clusterRerouteRequest.setRetryFailed(request.paramAsBoolean("retry_failed", clusterRerouteRequest.isRetryFailed()));
-        clusterRerouteRequest.explain(request.paramAsBoolean("explain", clusterRerouteRequest.explain()));
-        clusterRerouteRequest.timeout(request.paramAsTime("timeout", clusterRerouteRequest.timeout()));
-        clusterRerouteRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterRerouteRequest.masterNodeTimeout()));
-        if (request.hasContent()) {
-            clusterRerouteRequest.source(request.content(), registry, parseFieldMatcher);
-        }
-
+        ClusterRerouteRequest clusterRerouteRequest = createRequest(request, registry, parseFieldMatcher);
         client.admin().cluster().reroute(clusterRerouteRequest, new AcknowledgedRestListener<ClusterRerouteResponse>(channel) {
             @Override
             protected void addCustomFields(XContentBuilder builder, ClusterRerouteResponse response) throws IOException {
@@ -90,4 +94,35 @@ public class RestClusterRerouteAction extends BaseRestHandler {
             }
         });
     }
+
+    public static ClusterRerouteRequest createRequest(RestRequest request, AllocationCommandRegistry registry,
+            ParseFieldMatcher parseFieldMatcher) throws IOException {
+        ClusterRerouteRequest clusterRerouteRequest = Requests.clusterRerouteRequest();
+        clusterRerouteRequest.dryRun(request.paramAsBoolean("dry_run", clusterRerouteRequest.dryRun()));
+        clusterRerouteRequest.explain(request.paramAsBoolean("explain", clusterRerouteRequest.explain()));
+        clusterRerouteRequest.timeout(request.paramAsTime("timeout", clusterRerouteRequest.timeout()));
+        clusterRerouteRequest.setRetryFailed(request.paramAsBoolean("retry_failed", clusterRerouteRequest.isRetryFailed()));
+        clusterRerouteRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterRerouteRequest.masterNodeTimeout()));
+        if (request.hasContent()) {
+            try (XContentParser parser = XContentHelper.createParser(request.content())) {
+                PARSER.parse(parser, clusterRerouteRequest, new ParseContext(registry, parseFieldMatcher));
+            }
+        }
+        return clusterRerouteRequest;
+    }
+
+    private static class ParseContext implements ParseFieldMatcherSupplier {
+        private final AllocationCommandRegistry registry;
+        private final ParseFieldMatcher parseFieldMatcher;
+
+        private ParseContext(AllocationCommandRegistry registry, ParseFieldMatcher parseFieldMatcher) {
+            this.registry = registry;
+            this.parseFieldMatcher = parseFieldMatcher;
+        }
+
+        @Override
+        public ParseFieldMatcher getParseFieldMatcher() {
+            return parseFieldMatcher;
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java
new file mode 100644
index 00000000000..06ca3a4f869
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.reroute;
+
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.action.support.master.MasterNodeRequest;
+import org.elasticsearch.cluster.routing.allocation.command.AllocateEmptyPrimaryAllocationCommand;
+import org.elasticsearch.cluster.routing.allocation.command.AllocateReplicaAllocationCommand;
+import org.elasticsearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand;
+import org.elasticsearch.cluster.routing.allocation.command.AllocationCommand;
+import org.elasticsearch.cluster.routing.allocation.command.AllocationCommandRegistry;
+import org.elasticsearch.cluster.routing.allocation.command.CancelAllocationCommand;
+import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.admin.cluster.reroute.RestClusterRerouteAction;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.FakeRestRequest;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.unmodifiableList;
+import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
+
+/**
+ * Test for serialization and parsing of {@link ClusterRerouteRequest} and its commands. See the superclass for, well, everything.
+ */
+public class ClusterRerouteRequestTests extends ESTestCase {
+    private static final int ROUNDS = 30;
+    private final List<Supplier<AllocationCommand>> RANDOM_COMMAND_GENERATORS = unmodifiableList(Arrays.asList(
+            () -> new AllocateReplicaAllocationCommand(randomAsciiOfLengthBetween(2, 10), between(0, 1000),
+                    randomAsciiOfLengthBetween(2, 10)),
+            () -> new AllocateEmptyPrimaryAllocationCommand(randomAsciiOfLengthBetween(2, 10), between(0, 1000),
+                    randomAsciiOfLengthBetween(2, 10), randomBoolean()),
+            () -> new AllocateStalePrimaryAllocationCommand(randomAsciiOfLengthBetween(2, 10), between(0, 1000),
+                    randomAsciiOfLengthBetween(2, 10), randomBoolean()),
+            () -> new CancelAllocationCommand(randomAsciiOfLengthBetween(2, 10), between(0, 1000),
+                    randomAsciiOfLengthBetween(2, 10), randomBoolean()),
+            () -> new MoveAllocationCommand(randomAsciiOfLengthBetween(2, 10), between(0, 1000),
+                    randomAsciiOfLengthBetween(2, 10), randomAsciiOfLengthBetween(2, 10))));
+    private final NamedWriteableRegistry namedWriteableRegistry;
+    private final AllocationCommandRegistry allocationCommandRegistry;
+
+    public ClusterRerouteRequestTests() {
+        namedWriteableRegistry = new NamedWriteableRegistry();
+        allocationCommandRegistry = new NetworkModule(null, null, true, namedWriteableRegistry).getAllocationCommandRegistry();
+    }
+
+    private ClusterRerouteRequest randomRequest() {
+        ClusterRerouteRequest request = new ClusterRerouteRequest();
+        int commands = between(0, 10);
+        for (int i = 0; i < commands; i++) {
+            request.add(randomFrom(RANDOM_COMMAND_GENERATORS).get());
+        }
+        request.dryRun(randomBoolean());
+        request.explain(randomBoolean());
+        request.setRetryFailed(randomBoolean());
+        return request;
+    }
+
+    public void testEqualsAndHashCode() {
+        for (int round = 0; round < ROUNDS; round++) {
+            ClusterRerouteRequest request = randomRequest();
+            assertEquals(request, request);
+            assertEquals(request.hashCode(), request.hashCode());
+
+            ClusterRerouteRequest copy = new ClusterRerouteRequest()
+                    .add(request.getCommands().commands().toArray(new AllocationCommand[0]));
+            copy.dryRun(request.dryRun()).explain(request.explain()).timeout(request.timeout()).setRetryFailed(request.isRetryFailed());
+            copy.masterNodeTimeout(request.masterNodeTimeout());
+            assertEquals(request, copy);
+            assertEquals(copy, request); // Commutative
+            assertEquals(request.hashCode(), copy.hashCode());
+
+            // Changing dryRun makes requests not equal
+            copy.dryRun(!copy.dryRun());
+            assertNotEquals(request, copy);
+            assertNotEquals(request.hashCode(), copy.hashCode());
+            copy.dryRun(!copy.dryRun());
+            assertEquals(request, copy);
+            assertEquals(request.hashCode(), copy.hashCode());
+
+            // Changing explain makes requests not equal
+            copy.explain(!copy.explain());
+            assertNotEquals(request, copy);
+            assertNotEquals(request.hashCode(), copy.hashCode());
+            copy.explain(!copy.explain());
+            assertEquals(request, copy);
+            assertEquals(request.hashCode(), copy.hashCode());
+
+            // Changing timeout makes requests not equal
+            copy.timeout(timeValueMillis(request.timeout().millis() + 1));
+            assertNotEquals(request, copy);
+            assertNotEquals(request.hashCode(), copy.hashCode());
+            copy.timeout(request.timeout());
+            assertEquals(request, copy);
+            assertEquals(request.hashCode(), copy.hashCode());
+
+            // Changing masterNodeTime makes requests not equal
+            copy.masterNodeTimeout(timeValueMillis(request.masterNodeTimeout().millis() + 1));
+            assertNotEquals(request, copy);
+            assertNotEquals(request.hashCode(), copy.hashCode());
+            copy.masterNodeTimeout(request.masterNodeTimeout());
+            assertEquals(request, copy);
+            assertEquals(request.hashCode(), copy.hashCode());
+
+            // Changing commands makes requests not equal
+            copy.add(randomFrom(RANDOM_COMMAND_GENERATORS).get());
+            assertNotEquals(request, copy);
+            // Can't check hashCode because we can't be sure that changing commands changes the hashCode. It usually does but might not.
+        }
+    }
+
+    public void testSerialization() throws IOException {
+        for (int round = 0; round < ROUNDS; round++) {
+            ClusterRerouteRequest request = randomRequest();
+            ClusterRerouteRequest copy = roundTripThroughBytes(request);
+            assertEquals(request, copy);
+            assertEquals(request.hashCode(), copy.hashCode());
+            assertNotSame(request, copy);
+        }
+    }
+
+    public void testParsing() throws IOException {
+        for (int round = 0; round < ROUNDS; round++) {
+            ClusterRerouteRequest request = randomRequest();
+            ClusterRerouteRequest copy = roundTripThroughRestRequest(request);
+            assertEquals(request, copy);
+            assertEquals(request.hashCode(), copy.hashCode());
+            assertNotSame(request, copy);
+        }
+    }
+
+    private ClusterRerouteRequest roundTripThroughBytes(ClusterRerouteRequest original) throws IOException {
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            original.writeTo(output);
+            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
+                ClusterRerouteRequest copy = new ClusterRerouteRequest();
+                copy.readFrom(in);
+                return copy;
+            }
+        }
+    }
+
+    private ClusterRerouteRequest roundTripThroughRestRequest(ClusterRerouteRequest original) throws IOException {
+        RestRequest restRequest = toRestRequest(original);
+        return RestClusterRerouteAction.createRequest(restRequest, allocationCommandRegistry, ParseFieldMatcher.STRICT);
+    }
+
+    private static RestRequest toRestRequest(ClusterRerouteRequest original) throws IOException {
+        Map<String, String> params = new HashMap<>();
+        XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+        boolean hasBody = false;
+        if (randomBoolean()) {
+            builder.prettyPrint();
+        }
+        builder.startObject();
+        if (randomBoolean()) {
+            params.put("dry_run", Boolean.toString(original.dryRun()));
+        } else {
+            hasBody = true;
+            builder.field("dry_run", original.dryRun());
+        }
+        params.put("explain", Boolean.toString(original.explain()));
+        if (false == original.timeout().equals(AcknowledgedRequest.DEFAULT_ACK_TIMEOUT) || randomBoolean()) {
+            params.put("timeout", original.timeout().toString());
+        }
+        if (original.isRetryFailed() || randomBoolean()) {
+            params.put("retry_failed", Boolean.toString(original.isRetryFailed()));
+        }
+        if (false == original.masterNodeTimeout().equals(MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT) || randomBoolean()) {
+            params.put("master_timeout", original.masterNodeTimeout().toString());
+        }
+        if (original.getCommands() != null) {
+            hasBody = true;
+            original.getCommands().toXContent(builder, ToXContent.EMPTY_PARAMS);
+        }
+        builder.endObject();
+
+        return new FakeRestRequest(emptyMap(), params, hasBody ? builder.bytes() : null);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
index 927f572487c..c11bf0b9ca6 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
@@ -59,7 +59,7 @@ public class ClusterRerouteTests extends ESAllocationTestCase {
         req.setRetryFailed(randomBoolean());
         req.dryRun(randomBoolean());
         req.explain(randomBoolean());
-        req.commands(new AllocateEmptyPrimaryAllocationCommand("foo", 1, "bar", randomBoolean()));
+        req.add(new AllocateEmptyPrimaryAllocationCommand("foo", 1, "bar", randomBoolean()));
         req.timeout(TimeValue.timeValueMillis(randomIntBetween(0, 100)));
         BytesStreamOutput out = new BytesStreamOutput();
         req.writeTo(out);
diff --git a/docs/reference/migration/migrate_5_0/allocation.asciidoc b/docs/reference/migration/migrate_5_0/allocation.asciidoc
index 1e095831381..d7affd8ff95 100644
--- a/docs/reference/migration/migrate_5_0/allocation.asciidoc
+++ b/docs/reference/migration/migrate_5_0/allocation.asciidoc
@@ -39,6 +39,11 @@ command corresponds to the old `allocate` command  with `allow_primary` set to
 false. The new `allocate_empty_primary` command corresponds to the old
 `allocate` command with `allow_primary` set to true.
 
+==== Custom Reroute Commands
+
+Elasticsearch no longer supports plugins registering custom allocation
+commands. It was unused and hopefully unneeded.
+
 ==== `index.shared_filesystem.recover_on_any_node` changes
 
 The behavior of `index.shared_filesystem.recover_on_any_node: true` has been

From 5e8127050971296247b14dac7fe3af6d5686c016 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 23 May 2016 14:45:46 -0400
Subject: [PATCH 0441/1311] Add retry test case for delete-by-query

Tests that we retry failed searches, scrolls, and bulks.
---
 .../org/elasticsearch/index/reindex/RetryTests.java    | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
index 21b935ef530..478f7707dfd 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
 import org.elasticsearch.action.ListenableActionFuture;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.action.bulk.BackoffPolicy;
@@ -103,8 +102,13 @@ public class RetryTests extends ESSingleNodeTestCase {
                 matcher().updated(DOC_COUNT));
     }
 
-    private void testCase(String action, AbstractBulkIndexByScrollRequestBuilder<?, ?> request, BulkIndexByScrollResponseMatcher matcher)
-            throws Exception {
+    public void testDeleteByQuery() throws Exception {
+        testCase(DeleteByQueryAction.NAME, DeleteByQueryAction.INSTANCE.newRequestBuilder(client()).source("source"),
+                matcher().deleted(DOC_COUNT));
+    }
+
+    private void testCase(String action, AbstractBulkByScrollRequestBuilder<?, BulkIndexByScrollResponse, ?> request,
+            BulkIndexByScrollResponseMatcher matcher) throws Exception {
         logger.info("Blocking search");
         CyclicBarrier initialSearchBlock = blockExecutor(ThreadPool.Names.SEARCH);
 

From 2a2730405e9f13d65a5bf58115c2640a9430f62e Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 24 May 2016 12:15:52 -0400
Subject: [PATCH 0442/1311] Add wait for yellow to doc snippet so it runs
 cleanly

Found by http://build-us-00.elastic.co/job/es_core_master_window-2008/3866/console
---
 docs/reference/query-dsl/nested-query.asciidoc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/reference/query-dsl/nested-query.asciidoc b/docs/reference/query-dsl/nested-query.asciidoc
index 06720552bde..6e990e07f91 100644
--- a/docs/reference/query-dsl/nested-query.asciidoc
+++ b/docs/reference/query-dsl/nested-query.asciidoc
@@ -22,6 +22,8 @@ PUT /my_index
         }
     }
 }
+
+GET _cluster/health?wait_for_status=yellow
 --------------------------------------------------
 // CONSOLE
 // TESTSETUP

From bef1c8511d3f99aa681c49396bf9c707f2e7d8a9 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 24 May 2016 13:24:30 -0400
Subject: [PATCH 0443/1311] s/tests.logger.level/tests.es.logger.level/

This is a leftover spot that wasn't changed. It was breaking
ClusterSettingsIT#ClusterSettingsIT because that test expected
the test's log level to default to the default logger level for
the nodes.
---
 test/framework/src/main/resources/log4j.properties | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/framework/src/main/resources/log4j.properties b/test/framework/src/main/resources/log4j.properties
index 11a864df0f3..87d4560f72f 100644
--- a/test/framework/src/main/resources/log4j.properties
+++ b/test/framework/src/main/resources/log4j.properties
@@ -1,5 +1,5 @@
-tests.logger.level=INFO
-log4j.rootLogger=${tests.logger.level}, out
+tests.es.logger.level=INFO
+log4j.rootLogger=${tests.es.logger.level}, out
 
 log4j.logger.org.apache.http=INFO, out
 log4j.additivity.org.apache.http=false

From b3a8c54928f85fd5511efb0bb8bef085e0e09dad Mon Sep 17 00:00:00 2001
From: Britta Weber <britta.weber@elasticsearch.com>
Date: Tue, 24 May 2016 19:59:50 +0200
Subject: [PATCH 0444/1311] Test for deadlock when relocating and indexing
 concurrently

see #18553

Reproduces with

gradle :core:integTest -Dtests.class=org.elasticsearch.recovery.RelocationIT -Dtests.method="testIndexAndRelocateConcurrently"  -Dtests.iters=20 -Dtests.failfast=true -Dtests.logger.level=DEBUG -Dtests.seed=9BE7064B819064FA:C44B70F31707D081 -Dtests.timeoutSuite=60000!
---
 .../elasticsearch/recovery/RelocationIT.java  | 61 +++++++++++++++++++
 1 file changed, 61 insertions(+)

diff --git a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
index 7783f9cbf83..bc9909b21c1 100644
--- a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
+++ b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
@@ -22,6 +22,7 @@ package org.elasticsearch.recovery;
 import com.carrotsearch.hppc.IntHashSet;
 import com.carrotsearch.hppc.procedures.IntProcedure;
 import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.util.English;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
@@ -52,6 +53,7 @@ import org.elasticsearch.test.BackgroundIndexer;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
+import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.MockIndexEventListener;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
@@ -71,12 +73,15 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.startsWith;
@@ -428,6 +433,62 @@ public class RelocationIT extends ESIntegTestCase {
         }
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/18553")
+    public void testIndexAndRelocateConcurrently() throws ExecutionException, InterruptedException {
+        Settings blueSetting = Settings.builder().put("node.attr.color", "blue").build();
+        InternalTestCluster.Async<List<String>> blueFuture = internalCluster().startNodesAsync(blueSetting, blueSetting);
+        Settings redSetting = Settings.builder().put("node.attr.color", "red").build();
+        InternalTestCluster.Async<java.util.List<String>> redFuture = internalCluster().startNodesAsync(redSetting, redSetting);
+        blueFuture.get();
+        redFuture.get();
+        logger.info("blue nodes: {}", blueFuture.get());
+        logger.info("red nodes: {}", redFuture.get());
+        ensureStableCluster(4);
+
+        assertAcked(prepareCreate("test").setSettings(Settings.builder()
+            .put("index.routing.allocation.exclude.color", "blue")
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
+            .put(indexSettings())));
+        ensureYellow();
+        assertAllShardsOnNodes("test", redFuture.get().toArray(new String[2]));
+        int numDocs = randomIntBetween(100, 150);
+        ArrayList<String> ids = new ArrayList<>();
+        logger.info(" --> indexing [{}] docs", numDocs);
+        IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs];
+        for (int i = 0; i < numDocs; i++) {
+            String id = randomRealisticUnicodeOfLength(10) + String.valueOf(i);
+            ids.add(id);
+            docs[i] = client().prepareIndex("test", "type1", id).setSource("field1", English.intToEnglish(i));
+        }
+        indexRandom(true, docs);
+        SearchResponse countResponse = client().prepareSearch("test").get();
+        assertHitCount(countResponse, numDocs);
+
+        logger.info(" --> moving index to new nodes");
+        Settings build = Settings.builder().put("index.routing.allocation.exclude.color", "red")
+            .put("index.routing.allocation.include.color", "blue").build();
+        client().admin().indices().prepareUpdateSettings("test").setSettings(build).execute().actionGet();
+
+        // index while relocating
+        logger.info(" --> indexing [{}] more docs", numDocs);
+        for (int i = 0; i < numDocs; i++) {
+            String id = randomRealisticUnicodeOfLength(10) + String.valueOf(numDocs + i);
+            ids.add(id);
+            docs[i] = client().prepareIndex("test", "type1", id).setSource("field1", English.intToEnglish(numDocs + i));
+        }
+        indexRandom(true, docs);
+        numDocs *= 2;
+
+        logger.info(" --> waiting for relocation to complete", numDocs);
+        ensureGreen("test");// move all shards to the new node (it waits on relocation)
+        final int numIters = randomIntBetween(10, 20);
+        for (int i = 0; i < numIters; i++) {
+            SearchResponse afterRelocation = client().prepareSearch().setSize(ids.size()).get();
+            assertNoFailures(afterRelocation);
+            assertSearchHits(afterRelocation, ids.toArray(new String[ids.size()]));
+        }
+    }
+
     class RecoveryCorruption extends MockTransportService.DelegateTransport {
 
         private final CountDownLatch corruptionCount;

From d505c3d5ba9087c64010b93e5ed8cd46e4e1fc24 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Tue, 24 May 2016 12:11:13 -0700
Subject: [PATCH 0445/1311] Build: Use dash for deb package version and
 underscore for rpm

This change tweaks the metadata version associated with the deb and rpm
packages. For rpm, dashes are allowed in versions, so we can use the
version as it already exists. For rpm, dashes are not allowed, so this
uses underscores instead. This only affects prerelease versions (eg
alpha/beta/rc), and usually someone doesn't specify the version, so the
inconsistency doesn't matter that much.
---
 distribution/build.gradle     |  4 ----
 distribution/deb/build.gradle |  2 ++
 distribution/rpm/build.gradle | 10 ++--------
 3 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/distribution/build.gradle b/distribution/build.gradle
index 0df2f24fef0..12bb17b807f 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -302,10 +302,6 @@ configure(subprojects.findAll { ['deb', 'rpm'].contains(it.name) }) {
       '''.stripIndent().replace('\n', ' ').trim()
     url 'https://www.elastic.co/'
 
-    /* The version of the package can't contain -SNAPSHOT so we rip it off if
-      we see it. We'll add it back on to the file name though. */
-    version project.version.replace('-SNAPSHOT', '')
-
     // signing setup
     if (project.hasProperty('signing.password') && System.getProperty('build.snapshot', 'true') == 'false') {
       signingKeyId = project.hasProperty('signing.keyId') ? project.property('signing.keyId') : 'D88E42B4'
diff --git a/distribution/deb/build.gradle b/distribution/deb/build.gradle
index a3d42a2c042..89d33597888 100644
--- a/distribution/deb/build.gradle
+++ b/distribution/deb/build.gradle
@@ -22,6 +22,8 @@ task buildDeb(type: Deb) {
   baseName 'elasticsearch' // this is what pom generation uses for artifactId
   // Follow elasticsearch's deb file naming convention
   archiveName "${packageName}-${project.version}.deb"
+  version = project.version
+
   packageGroup 'web'
   requires 'libc6'
   requires 'adduser'
diff --git a/distribution/rpm/build.gradle b/distribution/rpm/build.gradle
index 72ed58e52c3..185f558ff02 100644
--- a/distribution/rpm/build.gradle
+++ b/distribution/rpm/build.gradle
@@ -25,14 +25,8 @@ task buildRpm(type: Rpm) {
   packageGroup 'Application/Internet'
   prefix '/usr'
   packager 'Elasticsearch'
-  if (version.contains('~')) {
-    def tokenized = version.tokenize('~')
-    version tokenized[0]
-    release tokenized[1]
-  } else {
-    version version
-    release '1'
-  }
+  version = project.version.replace('-', '_')
+  release = '1'
   arch 'NOARCH'
   os 'LINUX'
   license '2009'

From 105aee08b39abeafc275c46c4fe128e2325d6792 Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Tue, 24 May 2016 15:25:11 -0400
Subject: [PATCH 0446/1311] Removes multiple toXContent entry points for
 SnapshotInfo

SnapshotInfo had a toXContent and an externalToXContent, the former for
writing snapshot info to the snapshot blob and the latter for writing the
snapshot info to the APIs. This commit unifies writing x-content to one
method, toXContent, which distinguishes which format to write the
snapshot info in based on the Params parameter.  In addition, it makes
use of the already existing snapshot specific params found in the
BlobStoreFormat.

Closes #18494
---
 .../create/CreateSnapshotResponse.java        |  2 +-
 .../snapshots/get/GetSnapshotsResponse.java   |  2 +-
 .../blobstore/BlobStoreFormat.java            |  3 +
 .../elasticsearch/snapshots/SnapshotInfo.java | 63 ++++++++++---------
 4 files changed, 39 insertions(+), 31 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
index d49be2d294e..efc2fbeb5b5 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/CreateSnapshotResponse.java
@@ -85,7 +85,7 @@ public class CreateSnapshotResponse extends ActionResponse implements ToXContent
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         if (snapshotInfo != null) {
             builder.field("snapshot");
-            snapshotInfo.toExternalXContent(builder, params);
+            snapshotInfo.toXContent(builder, params);
         } else {
             builder.field("accepted", true);
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
index 7c4b255ff7c..ec996e6d366 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
@@ -78,7 +78,7 @@ public class GetSnapshotsResponse extends ActionResponse implements ToXContent {
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
         builder.startArray("snapshots");
         for (SnapshotInfo snapshotInfo : snapshots) {
-            snapshotInfo.toExternalXContent(builder, params);
+            snapshotInfo.toXContent(builder, params);
         }
         builder.endArray();
         return builder;
diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreFormat.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreFormat.java
index 2061c235e63..23d390dfcfe 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreFormat.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreFormat.java
@@ -26,6 +26,7 @@ import org.elasticsearch.common.xcontent.FromXContentBuilder;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.snapshots.SnapshotInfo;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -51,6 +52,8 @@ public abstract class BlobStoreFormat<T extends ToXContent> {
         // when metadata is serialized certain elements of the metadata shouldn't be included into snapshot
         // exclusion of these elements is done by setting MetaData.CONTEXT_MODE_PARAM to MetaData.CONTEXT_MODE_SNAPSHOT
         snapshotOnlyParams.put(MetaData.CONTEXT_MODE_PARAM, MetaData.CONTEXT_MODE_SNAPSHOT);
+        // serialize SnapshotInfo using the SNAPSHOT mode
+        snapshotOnlyParams.put(SnapshotInfo.CONTEXT_MODE_PARAM, SnapshotInfo.CONTEXT_MODE_SNAPSHOT);
         SNAPSHOT_ONLY_FORMAT_PARAMS = new ToXContent.MapParams(snapshotOnlyParams);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
index 871e765cfd0..2b8ea8ace31 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
@@ -44,6 +44,8 @@ import java.util.List;
 public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent, FromXContentBuilder<SnapshotInfo>, Writeable {
 
     public static final SnapshotInfo PROTO = new SnapshotInfo("", Collections.emptyList(), 0);
+    public static final String CONTEXT_MODE_PARAM = "context_mode";
+    public static final String CONTEXT_MODE_SNAPSHOT = "SNAPSHOT";
     private static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("strictDateOptionalTime");
     private static final String SNAPSHOT = "snapshot";
     private static final String INDICES = "indices";
@@ -294,37 +296,12 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
 
     @Override
     public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
-        builder.startObject(SNAPSHOT);
-        builder.field(NAME, name);
-        builder.field(VERSION_ID, version.id);
-        builder.startArray(INDICES);
-        for (String index : indices) {
-            builder.value(index);
+        // write snapshot info to repository snapshot blob format
+        if (CONTEXT_MODE_SNAPSHOT.equals(params.param(CONTEXT_MODE_PARAM))) {
+            return toXContentSnapshot(builder, params);
         }
-        builder.endArray();
-        builder.field(STATE, state);
-        if (reason != null) {
-            builder.field(REASON, reason);
-        }
-        builder.field(START_TIME, startTime);
-        builder.field(END_TIME, endTime);
-        builder.field(TOTAL_SHARDS, totalShards);
-        builder.field(SUCCESSFUL_SHARDS, successfulShards);
-        builder.startArray(FAILURES);
-        for (SnapshotShardFailure shardFailure : shardFailures) {
-            builder.startObject();
-            shardFailure.toXContent(builder, params);
-            builder.endObject();
-        }
-        builder.endArray();
-        builder.endObject();
-        return builder;
-    }
 
-    /**
-     * Produces the external X-content that is delivered through the REST layer.
-     */
-    public XContentBuilder toExternalXContent(final XContentBuilder builder, final ToXContent.Params params) throws IOException {
+        // write snapshot info for the API and any other situations
         builder.startObject();
         builder.field(SNAPSHOT, name);
         builder.field(VERSION_ID, version.id);
@@ -363,6 +340,34 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
         return builder;
     }
 
+    private XContentBuilder toXContentSnapshot(final XContentBuilder builder, final ToXContent.Params params) throws IOException {
+        builder.startObject(SNAPSHOT);
+        builder.field(NAME, name);
+        builder.field(VERSION_ID, version.id);
+        builder.startArray(INDICES);
+        for (String index : indices) {
+            builder.value(index);
+        }
+        builder.endArray();
+        builder.field(STATE, state);
+        if (reason != null) {
+            builder.field(REASON, reason);
+        }
+        builder.field(START_TIME, startTime);
+        builder.field(END_TIME, endTime);
+        builder.field(TOTAL_SHARDS, totalShards);
+        builder.field(SUCCESSFUL_SHARDS, successfulShards);
+        builder.startArray(FAILURES);
+        for (SnapshotShardFailure shardFailure : shardFailures) {
+            builder.startObject();
+            shardFailure.toXContent(builder, params);
+            builder.endObject();
+        }
+        builder.endArray();
+        builder.endObject();
+        return builder;
+    }
+
     @Override
     public SnapshotInfo fromXContent(final XContentParser parser, final ParseFieldMatcher matcher) throws IOException {
         return fromXContent(parser);

From 568c26a76c5a6ea35054a34a8caf9a28211f39d0 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 24 May 2016 16:55:55 -0400
Subject: [PATCH 0447/1311] Log OS and JVM on startup

This commit adds the OS and JVM to the initial logline on startup.

Relates #18557
---
 .../main/java/org/elasticsearch/node/Node.java   | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index d543d116c8f..ec222210b4e 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.node;
 
+import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Build;
 import org.elasticsearch.ElasticsearchException;
@@ -168,7 +169,20 @@ public class Node implements Closeable {
 
         ESLogger logger = Loggers.getLogger(Node.class, NODE_NAME_SETTING.get(tmpSettings));
         final String displayVersion = version + (Build.CURRENT.isSnapshot() ? "-SNAPSHOT" : "");
-        logger.info("version[{}], pid[{}], build[{}/{}]", displayVersion, JvmInfo.jvmInfo().pid(), Build.CURRENT.shortHash(), Build.CURRENT.date());
+        final JvmInfo jvmInfo = JvmInfo.jvmInfo();
+        logger.info(
+            "version[{}], pid[{}], build[{}/{}], OS[{}/{}/{}], JVM[{}/{}/{}/{}]",
+            displayVersion,
+            jvmInfo.pid(),
+            Build.CURRENT.shortHash(),
+            Build.CURRENT.date(),
+            Constants.OS_NAME,
+            Constants.OS_VERSION,
+            Constants.OS_ARCH,
+            Constants.JVM_VENDOR,
+            Constants.JVM_NAME,
+            Constants.JAVA_VERSION,
+            Constants.JVM_VERSION);
 
         logger.info("initializing ...");
 

From 84dfa360b14c0809a236df0601d7a47b5aa4598f Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 24 May 2016 17:01:35 -0400
Subject: [PATCH 0448/1311] Fix list of poi dependencies for ingest-attachment

This commit fixes the list of poi dependencies for the ingest-attachment
plugin after they were upgraded in
1d40c4bbc1f8c3414cfc64a30e08ab097aed19d4.
---
 .../packaging/scripts/module_and_plugin_test_cases.bash       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
index a5b18cf65e0..e20959a82e1 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
@@ -217,10 +217,10 @@ fi
 }
 
 @test "[$GROUP] install ingest-attachment plugin" {
-    # we specify the version on the poi-3.13.jar so that the test does
+    # we specify the version on the poi-3.15-beta1.jar so that the test does
     # not spuriously pass if the jar is missing but the other poi jars
     # are present
-    install_and_check_plugin ingest attachment bcprov-jdk15on-*.jar tika-core-*.jar pdfbox-*.jar poi-3.13.jar
+    install_and_check_plugin ingest attachment bcprov-jdk15on-*.jar tika-core-*.jar pdfbox-*.jar poi-3.15-beta1.jar poi-ooxml-3.15-beta1.jar poi-ooxml-schemas-*.jar poi-scratchpad-*.jar
 }
 
 @test "[$GROUP] install ingest-geoip plugin" {

From 0fa67e1538e189ff1b6b46c602ccf0cc6ff22bd1 Mon Sep 17 00:00:00 2001
From: Tal Levy <JubBoy333@gmail.com>
Date: Tue, 24 May 2016 14:25:40 -0700
Subject: [PATCH 0449/1311] Expose underlying processor to blame for thrown
 exception within CompoundProcessor (#18342)

Fixes #17823
---
 .../ingest/core/CompoundProcessor.java        | 44 ++++++++---
 .../ingest/SimulateExecutionServiceTests.java |  6 +-
 .../elasticsearch/ingest/IngestClientIT.java  |  4 +-
 .../ingest/PipelineExecutionServiceTests.java |  5 +-
 .../ingest/core/CompoundProcessorTests.java   | 76 +++++++++++++++++--
 .../TrackingResultProcessorTests.java         | 10 ++-
 6 files changed, 119 insertions(+), 26 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/ingest/core/CompoundProcessor.java b/core/src/main/java/org/elasticsearch/ingest/core/CompoundProcessor.java
index 16b3aa10a22..fb10f8a1927 100644
--- a/core/src/main/java/org/elasticsearch/ingest/core/CompoundProcessor.java
+++ b/core/src/main/java/org/elasticsearch/ingest/core/CompoundProcessor.java
@@ -20,14 +20,13 @@
 
 package org.elasticsearch.ingest.core;
 
-import org.elasticsearch.common.util.iterable.Iterables;
+import org.elasticsearch.ElasticsearchException;
 
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.stream.Collectors;
 
 /**
@@ -94,30 +93,38 @@ public class CompoundProcessor implements Processor {
             try {
                 processor.execute(ingestDocument);
             } catch (Exception e) {
+                ElasticsearchException compoundProcessorException = newCompoundProcessorException(e, processor.getType(), processor.getTag());
                 if (onFailureProcessors.isEmpty()) {
-                    throw e;
+                    throw compoundProcessorException;
                 } else {
-                    executeOnFailure(ingestDocument, e, processor.getType(), processor.getTag());
+                    executeOnFailure(ingestDocument, compoundProcessorException);
                 }
-                break;
             }
         }
     }
 
-    void executeOnFailure(IngestDocument ingestDocument, Exception cause, String failedProcessorType, String failedProcessorTag) throws Exception {
+    void executeOnFailure(IngestDocument ingestDocument, ElasticsearchException exception) throws Exception {
         try {
-            putFailureMetadata(ingestDocument, cause, failedProcessorType, failedProcessorTag);
+            putFailureMetadata(ingestDocument, exception);
             for (Processor processor : onFailureProcessors) {
-                processor.execute(ingestDocument);
+                try {
+                    processor.execute(ingestDocument);
+                } catch (Exception e) {
+                    throw newCompoundProcessorException(e, processor.getType(), processor.getTag());
+                }
             }
         } finally {
             removeFailureMetadata(ingestDocument);
         }
     }
 
-    private void putFailureMetadata(IngestDocument ingestDocument, Exception cause, String failedProcessorType, String failedProcessorTag) {
+    private void putFailureMetadata(IngestDocument ingestDocument, ElasticsearchException cause) {
+        List<String> processorTypeHeader = cause.getHeader("processor_type");
+        List<String> processorTagHeader = cause.getHeader("processor_tag");
+        String failedProcessorType = (processorTypeHeader != null) ? processorTypeHeader.get(0) : null;
+        String failedProcessorTag = (processorTagHeader != null) ? processorTagHeader.get(0) : null;
         Map<String, String> ingestMetadata = ingestDocument.getIngestMetadata();
-        ingestMetadata.put(ON_FAILURE_MESSAGE_FIELD, cause.getMessage());
+        ingestMetadata.put(ON_FAILURE_MESSAGE_FIELD, cause.getRootCause().getMessage());
         ingestMetadata.put(ON_FAILURE_PROCESSOR_TYPE_FIELD, failedProcessorType);
         ingestMetadata.put(ON_FAILURE_PROCESSOR_TAG_FIELD, failedProcessorTag);
     }
@@ -128,4 +135,21 @@ public class CompoundProcessor implements Processor {
         ingestMetadata.remove(ON_FAILURE_PROCESSOR_TYPE_FIELD);
         ingestMetadata.remove(ON_FAILURE_PROCESSOR_TAG_FIELD);
     }
+
+    private ElasticsearchException newCompoundProcessorException(Exception e, String processorType, String processorTag) {
+        if (e instanceof ElasticsearchException && ((ElasticsearchException)e).getHeader("processor_type") != null) {
+            return (ElasticsearchException) e;
+        }
+
+        ElasticsearchException exception = new ElasticsearchException(new IllegalArgumentException(e));
+
+        if (processorType != null) {
+            exception.addHeader("processor_type", processorType);
+        }
+        if (processorTag != null) {
+            exception.addHeader("processor_tag", processorTag);
+        }
+
+        return exception;
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
index 56e67bb4ad2..30c29f8db5d 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.ingest;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.ingest.TestProcessor;
@@ -167,7 +168,8 @@ public class SimulateExecutionServiceTests extends ESTestCase {
         SimulateDocumentBaseResult simulateDocumentBaseResult = (SimulateDocumentBaseResult) actualItemResponse;
         assertThat(simulateDocumentBaseResult.getIngestDocument(), nullValue());
         assertThat(simulateDocumentBaseResult.getFailure(), instanceOf(RuntimeException.class));
-        RuntimeException runtimeException = (RuntimeException) simulateDocumentBaseResult.getFailure();
-        assertThat(runtimeException.getMessage(), equalTo("processor failed"));
+        Exception exception = simulateDocumentBaseResult.getFailure();
+        assertThat(exception, instanceOf(ElasticsearchException.class));
+        assertThat(exception.getMessage(), equalTo("java.lang.IllegalArgumentException: java.lang.RuntimeException: processor failed"));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java b/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java
index e5f20499435..111133a0521 100644
--- a/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java
+++ b/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.ingest;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.bulk.BulkItemResponse;
@@ -154,7 +155,8 @@ public class IngestClientIT extends ESIntegTestCase {
             BulkItemResponse itemResponse = response.getItems()[i];
             if (i % 2 == 0) {
                 BulkItemResponse.Failure failure = itemResponse.getFailure();
-                assertThat(failure.getMessage(), equalTo("java.lang.IllegalArgumentException: test processor failed"));
+                ElasticsearchException compoundProcessorException = (ElasticsearchException) failure.getCause();
+                assertThat(compoundProcessorException.getRootCause().getMessage(), equalTo("test processor failed"));
             } else {
                 IndexResponse indexResponse = itemResponse.getResponse();
                 assertThat("Expected a successful response but found failure [" + itemResponse.getFailure() + "].",
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
index 3c0de328c8c..254057d2ede 100644
--- a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.ingest;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
@@ -188,6 +189,8 @@ public class PipelineExecutionServiceTests extends ESTestCase {
 
     public void testExecuteSuccessWithOnFailure() throws Exception {
         Processor processor = mock(Processor.class);
+        when(processor.getType()).thenReturn("mock_processor_type");
+        when(processor.getTag()).thenReturn("mock_processor_tag");
         Processor onFailureProcessor = mock(Processor.class);
         CompoundProcessor compoundProcessor = new CompoundProcessor(Collections.singletonList(processor), Collections.singletonList(new CompoundProcessor(onFailureProcessor)));
         when(store.get("_id")).thenReturn(new Pipeline("_id", "_description", compoundProcessor));
@@ -198,7 +201,7 @@ public class PipelineExecutionServiceTests extends ESTestCase {
         @SuppressWarnings("unchecked")
         Consumer<Boolean> completionHandler = mock(Consumer.class);
         executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler);
-        verify(failureHandler, never()).accept(any(RuntimeException.class));
+        verify(failureHandler, never()).accept(any(ElasticsearchException.class));
         verify(completionHandler, times(1)).accept(true);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/ingest/core/CompoundProcessorTests.java b/core/src/test/java/org/elasticsearch/ingest/core/CompoundProcessorTests.java
index 7bc8922af41..b4ee7eca07c 100644
--- a/core/src/test/java/org/elasticsearch/ingest/core/CompoundProcessorTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/core/CompoundProcessorTests.java
@@ -19,21 +19,17 @@
 
 package org.elasticsearch.ingest.core;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ingest.TestProcessor;
-import org.elasticsearch.ingest.TestTemplateService;
-import org.elasticsearch.ingest.processor.AppendProcessor;
-import org.elasticsearch.ingest.processor.SetProcessor;
-import org.elasticsearch.ingest.processor.SplitProcessor;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
 
 public class CompoundProcessorTests extends ESTestCase {
@@ -70,8 +66,8 @@ public class CompoundProcessorTests extends ESTestCase {
         try {
             compoundProcessor.execute(ingestDocument);
             fail("should throw exception");
-        } catch (Exception e) {
-            assertThat(e.getMessage(), equalTo("error"));
+        } catch (ElasticsearchException e) {
+            assertThat(e.getRootCause().getMessage(), equalTo("error"));
         }
         assertThat(processor.getInvokedCounter(), equalTo(1));
     }
@@ -117,4 +113,68 @@ public class CompoundProcessorTests extends ESTestCase {
         assertThat(processorToFail.getInvokedCounter(), equalTo(1));
         assertThat(lastProcessor.getInvokedCounter(), equalTo(1));
     }
+
+    public void testCompoundProcessorExceptionFailWithoutOnFailure() throws Exception {
+        TestProcessor firstProcessor = new TestProcessor("id1", "first", ingestDocument -> {throw new RuntimeException("error");});
+        TestProcessor secondProcessor = new TestProcessor("id3", "second", ingestDocument -> {
+            Map<String, String> ingestMetadata = ingestDocument.getIngestMetadata();
+            assertThat(ingestMetadata.entrySet(), hasSize(3));
+            assertThat(ingestMetadata.get(CompoundProcessor.ON_FAILURE_MESSAGE_FIELD), equalTo("error"));
+            assertThat(ingestMetadata.get(CompoundProcessor.ON_FAILURE_PROCESSOR_TYPE_FIELD), equalTo("first"));
+            assertThat(ingestMetadata.get(CompoundProcessor.ON_FAILURE_PROCESSOR_TAG_FIELD), equalTo("id1"));
+        });
+
+        CompoundProcessor failCompoundProcessor = new CompoundProcessor(firstProcessor);
+
+        CompoundProcessor compoundProcessor = new CompoundProcessor(Collections.singletonList(failCompoundProcessor),
+            Collections.singletonList(secondProcessor));
+        compoundProcessor.execute(ingestDocument);
+
+        assertThat(firstProcessor.getInvokedCounter(), equalTo(1));
+        assertThat(secondProcessor.getInvokedCounter(), equalTo(1));
+    }
+
+    public void testCompoundProcessorExceptionFail() throws Exception {
+        TestProcessor firstProcessor = new TestProcessor("id1", "first", ingestDocument -> {throw new RuntimeException("error");});
+        TestProcessor failProcessor = new TestProcessor("tag_fail", "fail", ingestDocument -> {throw new RuntimeException("custom error message");});
+        TestProcessor secondProcessor = new TestProcessor("id3", "second", ingestDocument -> {
+            Map<String, String> ingestMetadata = ingestDocument.getIngestMetadata();
+            assertThat(ingestMetadata.entrySet(), hasSize(3));
+            assertThat(ingestMetadata.get(CompoundProcessor.ON_FAILURE_MESSAGE_FIELD), equalTo("custom error message"));
+            assertThat(ingestMetadata.get(CompoundProcessor.ON_FAILURE_PROCESSOR_TYPE_FIELD), equalTo("fail"));
+            assertThat(ingestMetadata.get(CompoundProcessor.ON_FAILURE_PROCESSOR_TAG_FIELD), equalTo("tag_fail"));
+        });
+
+        CompoundProcessor failCompoundProcessor = new CompoundProcessor(Collections.singletonList(firstProcessor),
+            Collections.singletonList(failProcessor));
+
+        CompoundProcessor compoundProcessor = new CompoundProcessor(Collections.singletonList(failCompoundProcessor),
+            Collections.singletonList(secondProcessor));
+        compoundProcessor.execute(ingestDocument);
+
+        assertThat(firstProcessor.getInvokedCounter(), equalTo(1));
+        assertThat(secondProcessor.getInvokedCounter(), equalTo(1));
+    }
+
+    public void testCompoundProcessorExceptionFailInOnFailure() throws Exception {
+        TestProcessor firstProcessor = new TestProcessor("id1", "first", ingestDocument -> {throw new RuntimeException("error");});
+        TestProcessor failProcessor = new TestProcessor("tag_fail", "fail", ingestDocument -> {throw new RuntimeException("custom error message");});
+        TestProcessor secondProcessor = new TestProcessor("id3", "second", ingestDocument -> {
+            Map<String, String> ingestMetadata = ingestDocument.getIngestMetadata();
+            assertThat(ingestMetadata.entrySet(), hasSize(3));
+            assertThat(ingestMetadata.get(CompoundProcessor.ON_FAILURE_MESSAGE_FIELD), equalTo("custom error message"));
+            assertThat(ingestMetadata.get(CompoundProcessor.ON_FAILURE_PROCESSOR_TYPE_FIELD), equalTo("fail"));
+            assertThat(ingestMetadata.get(CompoundProcessor.ON_FAILURE_PROCESSOR_TAG_FIELD), equalTo("tag_fail"));
+        });
+
+        CompoundProcessor failCompoundProcessor = new CompoundProcessor(Collections.singletonList(firstProcessor),
+            Collections.singletonList(new CompoundProcessor(failProcessor)));
+
+        CompoundProcessor compoundProcessor = new CompoundProcessor(Collections.singletonList(failCompoundProcessor),
+            Collections.singletonList(secondProcessor));
+        compoundProcessor.execute(ingestDocument);
+
+        assertThat(firstProcessor.getInvokedCounter(), equalTo(1));
+        assertThat(secondProcessor.getInvokedCounter(), equalTo(1));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/TrackingResultProcessorTests.java b/core/src/test/java/org/elasticsearch/ingest/processor/TrackingResultProcessorTests.java
index e53eec56cf1..2b53a9d08bc 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/TrackingResultProcessorTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/processor/TrackingResultProcessorTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.ingest.processor;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ingest.SimulateProcessorResult;
 import org.elasticsearch.ingest.TestProcessor;
 import org.elasticsearch.ingest.core.CompoundProcessor;
@@ -73,8 +74,9 @@ public class TrackingResultProcessorTests extends ESTestCase {
 
         try {
             trackingProcessor.execute(ingestDocument);
-        } catch (Exception e) {
-            assertThat(e.getMessage(), equalTo(exception.getMessage()));
+            fail("processor should throw exception");
+        } catch (ElasticsearchException e) {
+            assertThat(e.getRootCause().getMessage(), equalTo(exception.getMessage()));
         }
 
         SimulateProcessorResult expectedFirstResult = new SimulateProcessorResult(testProcessor.getTag(), ingestDocument);
@@ -121,8 +123,8 @@ public class TrackingResultProcessorTests extends ESTestCase {
 
         metadata = resultList.get(3).getIngestDocument().getIngestMetadata();
         assertThat(metadata.get(ON_FAILURE_MESSAGE_FIELD), equalTo("fail"));
-        assertThat(metadata.get(ON_FAILURE_PROCESSOR_TYPE_FIELD), equalTo("compound"));
-        assertThat(metadata.get(ON_FAILURE_PROCESSOR_TAG_FIELD), equalTo("CompoundProcessor-fail-success-success-fail"));
+        assertThat(metadata.get(ON_FAILURE_PROCESSOR_TYPE_FIELD), equalTo("test"));
+        assertThat(metadata.get(ON_FAILURE_PROCESSOR_TAG_FIELD), equalTo("fail"));
         assertThat(resultList.get(3).getFailure(), nullValue());
         assertThat(resultList.get(3).getProcessorTag(), equalTo(expectedSuccessResult.getProcessorTag()));
     }

From 7b98d5e0cb743ea9f97a3dcd95eaa0ae2d2b7d33 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 24 May 2016 19:44:57 -0400
Subject: [PATCH 0450/1311] Fix plugin installation command

This commit fixes the plugin installation command produced by the
release candidate driver.
---
 dev-tools/prepare_release_candidate.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev-tools/prepare_release_candidate.py b/dev-tools/prepare_release_candidate.py
index be5660c60ae..84c10f4b9e4 100644
--- a/dev-tools/prepare_release_candidate.py
+++ b/dev-tools/prepare_release_candidate.py
@@ -56,7 +56,7 @@ The packages may be downloaded from the following URLs:
 
 Plugins can be installed as follows:
 
-    bin/elasticsearch-plugin -Des.plugins.staging=true install cloud-aws
+    ES_JAVA_OPTS="-Des.plugins.staging=true" bin/elasticsearch-plugin install cloud-aws
 
 The same goes for the x-plugins:
 

From e7243ed83dff31c72f4cb255de275344d4ca0c2e Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 24 May 2016 19:48:53 -0400
Subject: [PATCH 0451/1311] Fix smoke test RC plugin installation

This commit fixes the smoke test RC plugin installation command to set
the es.plugins.staging system properly correctly.
---
 dev-tools/smoke_test_rc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev-tools/smoke_test_rc.py b/dev-tools/smoke_test_rc.py
index 63153b45f1b..32269e319bb 100644
--- a/dev-tools/smoke_test_rc.py
+++ b/dev-tools/smoke_test_rc.py
@@ -191,7 +191,7 @@ def smoke_test_release(release, files, expected_hash, plugins):
     plugin_names = {}
     for plugin  in plugins:
       print('     Install plugin [%s]' % (plugin))
-      run('%s; %s -Des.plugins.staging=true %s %s' % (java_exe(), es_plugin_path, 'install -b', plugin))
+      run('%s; export ES_JAVA_OPTS="-Des.plugins.staging=true"; %s %s %s' % (java_exe(), es_plugin_path, 'install -b', plugin))
       plugin_names[plugin] = True
     if 'x-pack' in plugin_names:
       headers = { 'Authorization' : 'Basic %s' % base64.b64encode(b"es_admin:foobar").decode("UTF-8") }

From 82ae65bbb519bca00228146632c6790f27c077f5 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Tue, 24 May 2016 19:53:50 -0700
Subject: [PATCH 0452/1311] Tests: Make score tests unit tests

This change simply removes the need for single node tests from score
tests.
---
 .../elasticsearch/painless/ScoreTests.java    | 93 +++++++++++--------
 .../painless/ScriptTestCase.java              | 13 ++-
 .../painless/WhenThingsGoWrongTests.java      |  2 +-
 3 files changed, 65 insertions(+), 43 deletions(-)

diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScoreTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScoreTests.java
index d02d2957ab5..deaabfcebc6 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScoreTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScoreTests.java
@@ -19,50 +19,65 @@
 
 package org.elasticsearch.painless;
 
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.test.ESSingleNodeTestCase;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.concurrent.atomic.AtomicBoolean;
 
-import java.util.Collection;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Scorer;
 
-public class ScoreTests extends ESSingleNodeTestCase {
-    @Override
-    protected Collection<Class<? extends Plugin>> getPlugins() {
-        return pluginList(PainlessPlugin.class);
+public class ScoreTests extends ScriptTestCase {
+
+    /** Most of a dummy scorer impl that requires overriding just score(). */
+    abstract class MockScorer extends Scorer {
+        MockScorer() {
+            super(null);
+        }
+        @Override
+        public int docID() {
+            return 0;
+        }
+        @Override
+        public int freq() throws IOException {
+            throw new UnsupportedOperationException();
+        }
+        @Override
+        public DocIdSetIterator iterator() {
+            throw new UnsupportedOperationException();
+        }
     }
 
-    public void testScore() {
-        createIndex("test", Settings.EMPTY, "type", "t", "type=text");
-        ensureGreen("test");
+    public void testScoreWorks() {
+        assertEquals(2.5, exec("_score", Collections.emptyMap(), Collections.emptyMap(),
+            new MockScorer() {
+                @Override
+                public float score() throws IOException {
+                    return 2.5f;
+                }
+            }));
+    }
 
-        client().prepareIndex("test", "type", "1").setSource("t", "a").get();
-        client().prepareIndex("test", "type", "2").setSource("t", "a a b").get();
-        client().prepareIndex("test", "type", "3").setSource("t", "a a a b c").get();
-        client().prepareIndex("test", "type", "4").setSource("t", "a b c d").get();
-        client().prepareIndex("test", "type", "5").setSource("t", "a a b c d e").get();
-        client().admin().indices().prepareRefresh("test").get();
+    public void testScoreNotUsed() {
+        assertEquals(3.5, exec("3.5", Collections.emptyMap(), Collections.emptyMap(),
+            new MockScorer() {
+                @Override
+                public float score() throws IOException {
+                    throw new AssertionError("score() should not be called");
+                }
+            }));
+    }
 
-        final Script script = new Script("_score + 1", ScriptService.ScriptType.INLINE, "painless", null);
-
-        final SearchResponse sr = client().prepareSearch("test").setQuery(
-            QueryBuilders.functionScoreQuery(QueryBuilders.matchQuery("t", "a"),
-                ScoreFunctionBuilders.scriptFunction(script))).get();
-        final SearchHit[] hits = sr.getHits().getHits();
-
-        for (final SearchHit hit : hits) {
-            assertTrue(hit.score() > 0.9999F && hit.score() < 2.0001F);
-        }
-
-        assertEquals("1", hits[0].getId());
-        assertEquals("3", hits[1].getId());
-        assertEquals("2", hits[2].getId());
-        assertEquals("5", hits[3].getId());
-        assertEquals("4", hits[4].getId());
+    public void testScoreCached() {
+        assertEquals(9.0, exec("_score + _score", Collections.emptyMap(), Collections.emptyMap(),
+            new MockScorer() {
+                private boolean used = false;
+                @Override
+                public float score() throws IOException {
+                    if (used == false) {
+                        return 4.5f;
+                    }
+                    throw new AssertionError("score() should not be called twice");
+                }
+            }));
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
index d95fa3897da..27558dc745e 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
@@ -19,8 +19,11 @@
 
 package org.elasticsearch.painless;
 
+import org.apache.lucene.search.Scorer;
+import org.elasticsearch.common.lucene.ScorerAware;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.CompiledScript;
+import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
@@ -51,14 +54,18 @@ public abstract class ScriptTestCase extends ESTestCase {
     public Object exec(String script, Map<String, Object> vars) {
         Map<String,String> compilerSettings = new HashMap<>();
         compilerSettings.put(CompilerSettings.PICKY, "true");
-        return exec(script, vars, compilerSettings);
+        return exec(script, vars, compilerSettings, null);
     }
 
     /** Compiles and returns the result of {@code script} with access to {@code vars} and compile-time parameters */
-    public Object exec(String script, Map<String, Object> vars, Map<String,String> compileParams) {
+    public Object exec(String script, Map<String, Object> vars, Map<String,String> compileParams, Scorer scorer) {
         Object object = scriptEngine.compile(null, script, compileParams);
         CompiledScript compiled = new CompiledScript(ScriptService.ScriptType.INLINE, getTestName(), "painless", object);
-        return scriptEngine.executable(compiled, vars).run();
+        ExecutableScript executableScript = scriptEngine.executable(compiled, vars);
+        if (scorer != null) {
+            ((ScorerAware)executableScript).setScorer(scorer);
+        }
+        return executableScript.run();
     }
 
     /**
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
index 9cdce7583f5..c2b40df8c9d 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
@@ -77,7 +77,7 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
 
     public void testBogusParameter() {
         IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
-            exec("return 5;", null, Collections.singletonMap("bogusParameterKey", "bogusParameterValue"));
+            exec("return 5;", null, Collections.singletonMap("bogusParameterKey", "bogusParameterValue"), null);
         });
         assertTrue(expected.getMessage().contains("Unrecognized compile-time parameter"));
     }

From b40628d4e8876dbbf50a0a5fdbebea736d804f3e Mon Sep 17 00:00:00 2001
From: Tal Levy <JubBoy333@gmail.com>
Date: Tue, 24 May 2016 22:32:50 -0700
Subject: [PATCH 0453/1311] fix simulate spec test for new exception handling
 (#18564)

---
 .../main/resources/rest-api-spec/test/ingest/40_simulate.yaml   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/40_simulate.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/40_simulate.yaml
index 060752c1ab1..bc1b444ab54 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/40_simulate.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/40_simulate.yaml
@@ -341,7 +341,7 @@
             ]
           }
   - length: { docs: 2 }
-  - match: { docs.0.error.type: "illegal_argument_exception" }
+  - match: { docs.0.error.type: "exception" }
   - match: { docs.1.doc._source.foo: "BAR" }
   - length: { docs.1.doc._ingest: 1 }
   - is_true: docs.1.doc._ingest.timestamp

From bf41ac8bf2590564c6f8f45226b7760ece75fbae Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Thu, 19 May 2016 17:13:35 +0200
Subject: [PATCH 0454/1311] Makes DeleteByQueryRequest implements
 CompositeIndicesRequest

---
 .../index/reindex/DeleteByQueryRequest.java      | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
index 327459bd339..da56a5dd100 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequest.java
@@ -20,7 +20,9 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.support.IndicesOptions;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
@@ -41,7 +43,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
  *     <li>it's results won't be visible until the index is refreshed.</li>
  * </ul>
  */
-public class DeleteByQueryRequest extends AbstractBulkByScrollRequest<DeleteByQueryRequest> {
+public class DeleteByQueryRequest extends AbstractBulkByScrollRequest<DeleteByQueryRequest> implements IndicesRequest {
 
     public DeleteByQueryRequest() {
     }
@@ -76,4 +78,16 @@ public class DeleteByQueryRequest extends AbstractBulkByScrollRequest<DeleteByQu
         searchToString(b);
         return b.toString();
     }
+
+    @Override
+    public String[] indices() {
+        assert getSearchRequest() != null;
+        return getSearchRequest().indices();
+    }
+
+    @Override
+    public IndicesOptions indicesOptions() {
+        assert getSearchRequest() != null;
+        return getSearchRequest().indicesOptions();
+    }
 }

From dee34c916ca6b281df2b70b15052500dfa6710bf Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Wed, 25 May 2016 10:02:14 +0200
Subject: [PATCH 0455/1311] Expand wildcards to closed indices in /_cat/indices
 (#18545)

Closed indices are already displayed when no indices are explicitly selected. This commit ensures that closed indices are also shown when wildcard filtering is used. It also addresses another issue that is caused by the fact that the cat action is based internally on 3 different cluster states (one when we query the cluster state to get all indices, one when we query cluster health, and one when we query indices stats). We currently fail the cat request when the user specifies a concrete index as parameter that does not exist. The implementation works as intended in that regard. It checks this not only for the first cluster state request, but also the subsequent indices stats one. This means that if the index is deleted before the cat action has queried the indices stats, it rightfully fails. In case the user provides wildcards (or no parameter at all), however, we fail the indices stats as we pass the resolved concrete indices to the indices stats request and fail to distinguish whether these indices have been resolved by wildcards or explicitly requested by the user. This means that if an index has been deleted before the indices stats request gets to execute, we fail the overall cat request. The fix is to let the indices stats request do the resolving again and not pass the concrete indices.

Closes #16419
Closes #17395
---
 .../rest/action/cat/RestIndicesAction.java    | 30 ++++++++++++++-----
 .../test/cat.indices/10_basic.yaml            | 23 +++++++++++++-
 2 files changed, 44 insertions(+), 9 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
index e266efa4357..ac8cefcc5aa 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
@@ -77,27 +77,41 @@ public class RestIndicesAction extends AbstractCatAction {
         clusterStateRequest.clear().indices(indices).metaData(true);
         clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
         clusterStateRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterStateRequest.masterNodeTimeout()));
+        final IndicesOptions strictExpandIndicesOptions = IndicesOptions.strictExpand();
+        clusterStateRequest.indicesOptions(strictExpandIndicesOptions);
 
         client.admin().cluster().state(clusterStateRequest, new RestActionListener<ClusterStateResponse>(channel) {
             @Override
             public void processResponse(final ClusterStateResponse clusterStateResponse) {
-                ClusterState state = clusterStateResponse.getState();
-                final IndicesOptions concreteIndicesOptions = IndicesOptions.fromOptions(false, true, true, true);
-                final String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, concreteIndicesOptions, indices);
-                final String[] openIndices = indexNameExpressionResolver.concreteIndexNames(state, IndicesOptions.lenientExpandOpen(), indices);
-                ClusterHealthRequest clusterHealthRequest = Requests.clusterHealthRequest(openIndices);
+                final ClusterState state = clusterStateResponse.getState();
+                final String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, strictExpandIndicesOptions, indices);
+                // concreteIndices should contain exactly the indices in state.metaData() that were selected by clusterStateRequest using
+                // IndicesOptions.strictExpand(). We select the indices again here so that they can be displayed in the resulting table
+                // in the requesting order.
+                assert concreteIndices.length == state.metaData().getIndices().size();
+
+                // Indices that were successfully resolved during the cluster state request might be deleted when the subsequent cluster
+                // health and indices stats requests execute. We have to distinguish two cases:
+                // 1) the deleted index was explicitly passed as parameter to the /_cat/indices request. In this case we want the subsequent
+                //    requests to fail.
+                // 2) the deleted index was resolved as part of a wildcard or _all. In this case, we want the subsequent requests not to
+                //    fail on the deleted index (as we want to ignore wildcards that cannot be resolved).
+                // This behavior can be ensured by letting the cluster health and indices stats requests re-resolve the index names with the
+                // same indices options that we used for the initial cluster state request (strictExpand). Unfortunately cluster health
+                // requests hard-code their indices options and the best we can do is apply strictExpand to the indices stats request.
+                ClusterHealthRequest clusterHealthRequest = Requests.clusterHealthRequest(indices);
                 clusterHealthRequest.local(request.paramAsBoolean("local", clusterHealthRequest.local()));
                 client.admin().cluster().health(clusterHealthRequest, new RestActionListener<ClusterHealthResponse>(channel) {
                     @Override
                     public void processResponse(final ClusterHealthResponse clusterHealthResponse) {
                         IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest();
-                        indicesStatsRequest.indices(concreteIndices);
-                        indicesStatsRequest.indicesOptions(concreteIndicesOptions);
+                        indicesStatsRequest.indices(indices);
+                        indicesStatsRequest.indicesOptions(strictExpandIndicesOptions);
                         indicesStatsRequest.all();
                         client.admin().indices().stats(indicesStatsRequest, new RestResponseListener<IndicesStatsResponse>(channel) {
                             @Override
                             public RestResponse buildResponse(IndicesStatsResponse indicesStatsResponse) throws Exception {
-                                Table tab = buildTable(request, concreteIndices, clusterHealthResponse, indicesStatsResponse, clusterStateResponse.getState().metaData());
+                                Table tab = buildTable(request, concreteIndices, clusterHealthResponse, indicesStatsResponse, state.metaData());
                                 return RestTable.buildResponse(tab, channel);
                             }
                         });
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yaml
index a5a67d1a557..51f8fe9ed4c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yaml
@@ -24,7 +24,7 @@
   - match:
       $body: |
                /^(green  \s+
-                  (open|close) \s+
+                  open   \s+
                   index1 \s+
                   1      \s+
                   0      \s+
@@ -49,3 +49,24 @@
                   (\d\d\d\d\-\d\d\-\d\dT\d\d:\d\d:\d\d.\d\d\dZ) \s*
                 )
                 $/
+  - do:
+      indices.close:
+        index: index1
+
+  - do:
+      cat.indices:
+        index: index*
+
+  - match:
+      $body: |
+               /^(       \s+
+                  close  \s+
+                  index1 \s+
+                         \s+
+                         \s+
+                         \s+
+                         \s+
+                         \s+
+                         \s*
+                )
+                $/

From c4d3bf472bc5eaa18a1f1ffac727110a33f89b42 Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Wed, 25 May 2016 11:02:11 +0200
Subject: [PATCH 0456/1311] Fix comment and rename blob_container to
 blobContainer

---
 .../storage/AzureStorageServiceImpl.java      | 45 ++++++++++---------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
index ce5a06fdefa..09dbc6520a1 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
@@ -138,8 +138,8 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
     public boolean doesContainerExist(String account, LocationMode mode, String container) {
         try {
             CloudBlobClient client = this.getSelectedClient(account, mode);
-            CloudBlobContainer blob_container = client.getContainerReference(container);
-            return blob_container.exists();
+            CloudBlobContainer blobContainer = client.getContainerReference(container);
+            return blobContainer.exists();
         } catch (Exception e) {
             logger.error("can not access container [{}]", container);
         }
@@ -149,25 +149,25 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
     @Override
     public void removeContainer(String account, LocationMode mode, String container) throws URISyntaxException, StorageException {
         CloudBlobClient client = this.getSelectedClient(account, mode);
-        CloudBlobContainer blob_container = client.getContainerReference(container);
+        CloudBlobContainer blobContainer = client.getContainerReference(container);
         // TODO Should we set some timeout and retry options?
         /*
         BlobRequestOptions options = new BlobRequestOptions();
         options.setTimeoutIntervalInMs(1000);
         options.setRetryPolicyFactory(new RetryNoRetry());
-        blob_container.deleteIfExists(options, null);
+        blobContainer.deleteIfExists(options, null);
         */
         logger.trace("removing container [{}]", container);
-        blob_container.deleteIfExists();
+        blobContainer.deleteIfExists();
     }
 
     @Override
     public void createContainer(String account, LocationMode mode, String container) throws URISyntaxException, StorageException {
         try {
             CloudBlobClient client = this.getSelectedClient(account, mode);
-            CloudBlobContainer blob_container = client.getContainerReference(container);
+            CloudBlobContainer blobContainer = client.getContainerReference(container);
             logger.trace("creating container [{}]", container);
-            blob_container.createIfNotExists();
+            blobContainer.createIfNotExists();
         } catch (IllegalArgumentException e) {
             logger.trace("fails creating container [{}]", e, container);
             throw new RepositoryException(container, e.getMessage());
@@ -180,10 +180,10 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
 
         // Container name must be lower case.
         CloudBlobClient client = this.getSelectedClient(account, mode);
-        CloudBlobContainer blob_container = client.getContainerReference(container);
-        if (blob_container.exists()) {
+        CloudBlobContainer blobContainer = client.getContainerReference(container);
+        if (blobContainer.exists()) {
             // We list the blobs using a flat blob listing mode
-            for (ListBlobItem blobItem : blob_container.listBlobs(path, true)) {
+            for (ListBlobItem blobItem : blobContainer.listBlobs(path, true)) {
                 String blobName = blobNameFromUri(blobItem.getUri());
                 logger.trace("removing blob [{}] full URI was [{}]", blobName, blobItem.getUri());
                 deleteBlob(account, mode, container, blobName);
@@ -201,9 +201,10 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
         String path = uri.getPath();
 
         // We remove the container name from the path
-        // The 3 magic number cames from the fact we have // in the first part of the URI (protocol)
-        // Then a / after the server address
-        // And we finally split after the container/
+        // The 3 magic number cames from the fact if path is /container/path/to/myfile
+        // First occurrence is empty "/"
+        // Second occurrence is "container
+        // Last part contains "path/to/myfile" which is what we want to get
         String[] splits = path.split("/", 3);
 
         // We return the remaining end of the string
@@ -214,9 +215,9 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
     public boolean blobExists(String account, LocationMode mode, String container, String blob) throws URISyntaxException, StorageException {
         // Container name must be lower case.
         CloudBlobClient client = this.getSelectedClient(account, mode);
-        CloudBlobContainer blob_container = client.getContainerReference(container);
-        if (blob_container.exists()) {
-            CloudBlockBlob azureBlob = blob_container.getBlockBlobReference(blob);
+        CloudBlobContainer blobContainer = client.getContainerReference(container);
+        if (blobContainer.exists()) {
+            CloudBlockBlob azureBlob = blobContainer.getBlockBlobReference(blob);
             return azureBlob.exists();
         }
 
@@ -229,10 +230,10 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
 
         // Container name must be lower case.
         CloudBlobClient client = this.getSelectedClient(account, mode);
-        CloudBlobContainer blob_container = client.getContainerReference(container);
-        if (blob_container.exists()) {
+        CloudBlobContainer blobContainer = client.getContainerReference(container);
+        if (blobContainer.exists()) {
             logger.trace("container [{}]: blob [{}] found. removing.", container, blob);
-            CloudBlockBlob azureBlob = blob_container.getBlockBlobReference(blob);
+            CloudBlockBlob azureBlob = blobContainer.getBlockBlobReference(blob);
             azureBlob.delete();
         }
     }
@@ -292,10 +293,10 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
         logger.debug("moveBlob container [{}], sourceBlob [{}], targetBlob [{}]", container, sourceBlob, targetBlob);
 
         CloudBlobClient client = this.getSelectedClient(account, mode);
-        CloudBlobContainer blob_container = client.getContainerReference(container);
-        CloudBlockBlob blobSource = blob_container.getBlockBlobReference(sourceBlob);
+        CloudBlobContainer blobContainer = client.getContainerReference(container);
+        CloudBlockBlob blobSource = blobContainer.getBlockBlobReference(sourceBlob);
         if (blobSource.exists()) {
-            CloudBlockBlob blobTarget = blob_container.getBlockBlobReference(targetBlob);
+            CloudBlockBlob blobTarget = blobContainer.getBlockBlobReference(targetBlob);
             blobTarget.startCopy(blobSource);
             blobSource.delete();
             logger.debug("moveBlob container [{}], sourceBlob [{}], targetBlob [{}] -> done", container, sourceBlob, targetBlob);

From eab311320437ddf44e3742dc5cbd70556eb9463e Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Wed, 25 May 2016 11:51:28 +0200
Subject: [PATCH 0457/1311] Drop 1.x BWC and cut over to Writeable for
 Translog.Operation (#18565)

We still maintain BWC for the translog operations back to 1.1 which is not
supported in the current version anyway. This commit drops the bwc and moves
the operations to the Writeable interface enforcing immutability.
---
 .../index/translog/Translog.java              | 185 ++++++++----------
 1 file changed, 77 insertions(+), 108 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/translog/Translog.java b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
index 93f9db67b35..2946bc9c447 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/Translog.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
@@ -33,7 +33,7 @@ import org.elasticsearch.common.bytes.ReleasablePagedBytesReference;
 import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.util.BigArrays;
@@ -712,7 +712,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
      * A generic interface representing an operation performed on the transaction log.
      * Each is associated with a type.
      */
-    public interface Operation extends Streamable {
+    public interface Operation extends Writeable {
         enum Type {
             @Deprecated
             CREATE((byte) 1),
@@ -721,7 +721,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
 
             private final byte id;
 
-            private Type(byte id) {
+            Type(byte id) {
                 this.id = id;
             }
 
@@ -749,6 +749,33 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
 
         Source getSource();
 
+        /**
+         * Reads the type and the operation from the given stream. The operatino must be written with
+         * {@link #writeType(Operation, StreamOutput)}
+         */
+        static Operation readType(StreamInput input) throws IOException {
+            Translog.Operation.Type type = Translog.Operation.Type.fromId(input.readByte());
+            switch (type) {
+                case CREATE:
+                    // the deserialization logic in Index was identical to that of Create when create was deprecated
+                    return new Index(input);
+                case DELETE:
+                    return new Translog.Delete(input);
+                case INDEX:
+                    return new Index(input);
+                default:
+                    throw new IOException("No type for [" + type + "]");
+            }
+        }
+
+        /**
+         * Writes the type and translog operation to the given stream
+         */
+        static void writeType(Translog.Operation operation, StreamOutput output) throws IOException {
+            output.writeByte(operation.opType().id());
+            operation.writeTo(output);
+        }
+
     }
 
     public static class Source {
@@ -768,19 +795,30 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
     }
 
     public static class Index implements Operation {
-        public static final int SERIALIZATION_FORMAT = 6;
+        public static final int SERIALIZATION_FORMAT = 6; // since 2.0-beta1 and 1.1
+        private final String id;
+        private final String type;
+        private final long version;
+        private final VersionType versionType;
+        private final BytesReference source;
+        private final String routing;
+        private final String parent;
+        private final long timestamp;
+        private final long ttl;
 
-        private String id;
-        private String type;
-        private long version = Versions.MATCH_ANY;
-        private VersionType versionType = VersionType.INTERNAL;
-        private BytesReference source;
-        private String routing;
-        private String parent;
-        private long timestamp;
-        private long ttl;
-
-        public Index() {
+        public Index(StreamInput in) throws IOException {
+            final int format = in.readVInt(); // SERIALIZATION_FORMAT
+            assert format == SERIALIZATION_FORMAT : "format was: " + format;
+            id = in.readString();
+            type = in.readString();
+            source = in.readBytesReference();
+            routing = in.readOptionalString();
+            parent = in.readOptionalString();
+            this.version = in.readLong();
+            this.timestamp = in.readLong();
+            this.ttl = in.readLong();
+            this.versionType = VersionType.fromValue(in.readByte());
+            assert versionType.validateVersionForWrites(this.version);
         }
 
         public Index(Engine.Index index) {
@@ -799,6 +837,12 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
             this.type = type;
             this.id = id;
             this.source = new BytesArray(source);
+            version = Versions.MATCH_ANY;
+            versionType = VersionType.INTERNAL;
+            routing = null;
+            parent = null;
+            timestamp = 0;
+            ttl = 0;
         }
 
         @Override
@@ -852,60 +896,14 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
             return new Source(source, routing, parent, timestamp, ttl);
         }
 
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            int version = in.readVInt(); // version
-            id = in.readString();
-            type = in.readString();
-            source = in.readBytesReference();
-            try {
-                if (version >= 1) {
-                    if (in.readBoolean()) {
-                        routing = in.readString();
-                    }
-                }
-                if (version >= 2) {
-                    if (in.readBoolean()) {
-                        parent = in.readString();
-                    }
-                }
-                if (version >= 3) {
-                    this.version = in.readLong();
-                }
-                if (version >= 4) {
-                    this.timestamp = in.readLong();
-                }
-                if (version >= 5) {
-                    this.ttl = in.readLong();
-                }
-                if (version >= 6) {
-                    this.versionType = VersionType.fromValue(in.readByte());
-                }
-            } catch (Exception e) {
-                throw new ElasticsearchException("failed to read [" + type + "][" + id + "]", e);
-            }
-
-            assert versionType.validateVersionForWrites(version);
-        }
-
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeVInt(SERIALIZATION_FORMAT);
             out.writeString(id);
             out.writeString(type);
             out.writeBytesReference(source);
-            if (routing == null) {
-                out.writeBoolean(false);
-            } else {
-                out.writeBoolean(true);
-                out.writeString(routing);
-            }
-            if (parent == null) {
-                out.writeBoolean(false);
-            } else {
-                out.writeBoolean(true);
-                out.writeString(parent);
-            }
+            out.writeOptionalString(routing);
+            out.writeOptionalString(parent);
             out.writeLong(version);
             out.writeLong(timestamp);
             out.writeLong(ttl);
@@ -963,23 +961,29 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
     }
 
     public static class Delete implements Operation {
-        public static final int SERIALIZATION_FORMAT = 2;
+        public static final int SERIALIZATION_FORMAT = 2; // since 2.0-beta1 and 1.1
 
-        private Term uid;
-        private long version = Versions.MATCH_ANY;
-        private VersionType versionType = VersionType.INTERNAL;
+        private final Term uid;
+        private final long version;
+        private final VersionType versionType;
 
-        public Delete() {
+        public Delete(StreamInput in) throws IOException {
+            final int format = in.readVInt();// SERIALIZATION_FORMAT
+            assert format == SERIALIZATION_FORMAT : "format was: " + format;
+            uid = new Term(in.readString(), in.readString());
+            this.version = in.readLong();
+            this.versionType = VersionType.fromValue(in.readByte());
+            assert versionType.validateVersionForWrites(this.version);
         }
 
         public Delete(Engine.Delete delete) {
-            this(delete.uid());
+            this.uid = delete.uid();
             this.version = delete.version();
             this.versionType = delete.versionType();
         }
 
         public Delete(Term uid) {
-            this.uid = uid;
+            this(uid, Versions.MATCH_ANY, VersionType.INTERNAL);
         }
 
         public Delete(Term uid, long version, VersionType versionType) {
@@ -1015,20 +1019,6 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
             throw new IllegalStateException("trying to read doc source from delete operation");
         }
 
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            int version = in.readVInt(); // version
-            uid = new Term(in.readString(), in.readString());
-            if (version >= 1) {
-                this.version = in.readLong();
-            }
-            if (version >= 2) {
-                this.versionType = VersionType.fromValue(in.readByte());
-            }
-            assert versionType.validateVersionForWrites(version);
-
-        }
-
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeVInt(SERIALIZATION_FORMAT);
@@ -1107,7 +1097,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
     }
 
     static Translog.Operation readOperation(BufferedChecksumStreamInput in) throws IOException {
-        Translog.Operation operation;
+        final Translog.Operation operation;
         try {
             final int opSize = in.readInt();
             if (opSize < 4) { // 4byte for the checksum
@@ -1124,9 +1114,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
                 verifyChecksum(in);
                 in.reset();
             }
-            Translog.Operation.Type type = Translog.Operation.Type.fromId(in.readByte());
-            operation = newOperationFromType(type);
-            operation.readFrom(in);
+            operation = Translog.Operation.readType(in);
             verifyChecksum(in);
         } catch (EOFException e) {
             throw new TruncatedTranslogException("reached premature end of file, translog is truncated", e);
@@ -1169,30 +1157,11 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
         // because closing it closes the underlying stream, which we don't
         // want to do here.
         out.resetDigest();
-        out.writeByte(op.opType().id());
-        op.writeTo(out);
+        Translog.Operation.writeType(op, out);
         long checksum = out.getChecksum();
         out.writeInt((int) checksum);
     }
 
-    /**
-     * Returns a new empty translog operation for the given {@link Translog.Operation.Type}
-     */
-    static Translog.Operation newOperationFromType(Translog.Operation.Type type) throws IOException {
-        switch (type) {
-            case CREATE:
-                // the deserialization logic in Index was identical to that of Create when create was deprecated
-                return new Index();
-            case DELETE:
-                return new Translog.Delete();
-            case INDEX:
-                return new Index();
-            default:
-                throw new IOException("No type for [" + type + "]");
-        }
-    }
-
-
     @Override
     public void prepareCommit() throws IOException {
         try (ReleasableLock lock = writeLock.acquire()) {

From de6b4f35b18f3a6bf64e84d8f8e0d6ac729f252a Mon Sep 17 00:00:00 2001
From: Chris Earle <chris.earle@elastic.co>
Date: Mon, 23 May 2016 19:46:31 -0400
Subject: [PATCH 0458/1311] Remove inaccurate Javadoc on Setting constructor

The 'Setting' constructor has some outdated Javadoc that suggested that it would automatically apply 'Property.NodeScope' if no scope is supplied, but no scope is added in that case.
---
 .../java/org/elasticsearch/common/settings/Setting.java     | 6 +++---
 .../org/elasticsearch/common/settings/SettingTests.java     | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/settings/Setting.java b/core/src/main/java/org/elasticsearch/common/settings/Setting.java
index 1be1fa8f3f4..da32468acc3 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/Setting.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/Setting.java
@@ -122,7 +122,7 @@ public class Setting<T> extends ToXContentToBytes {
         this.defaultValue = defaultValue;
         this.parser = parser;
         if (properties == null) {
-            throw new IllegalArgumentException("properties can not be null for setting [" + key + "]");
+            throw new IllegalArgumentException("properties cannot be null for setting [" + key + "]");
         }
         if (properties.length == 0) {
             this.properties = EMPTY_PROPERTIES;
@@ -132,7 +132,7 @@ public class Setting<T> extends ToXContentToBytes {
     }
 
     /**
-     * Creates a new Setting instance. When no scope is provided, we default to {@link Property#NodeScope}.
+     * Creates a new Setting instance
      * @param key the settings key for this setting.
      * @param defaultValue a default value function that returns the default values string representation.
      * @param parser a parser that parses the string rep into a complex datatype.
@@ -165,7 +165,7 @@ public class Setting<T> extends ToXContentToBytes {
     }
 
     /**
-     * Creates a new Setting instance. When no scope is provided, we default to {@link Property#NodeScope}.
+     * Creates a new Setting instance
      * @param key the settings key for this setting.
      * @param fallbackSetting a setting who's value to fallback on if this setting is not defined
      * @param parser a parser that parses the string rep into a complex datatype.
diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java
index 78319297a98..b1504672f4e 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/SettingTests.java
@@ -475,7 +475,7 @@ public class SettingTests extends ESTestCase {
             Setting.simpleString("foo.bar", (Property[]) null);
             fail();
         } catch (IllegalArgumentException ex) {
-            assertThat(ex.getMessage(), containsString("properties can not be null for setting"));
+            assertThat(ex.getMessage(), containsString("properties cannot be null for setting"));
         }
     }
 }

From bdee8c263206880924cf841fb6be0d5c338a60f5 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Tue, 24 May 2016 13:37:32 +0200
Subject: [PATCH 0459/1311] Disable XContent auto closing of object and arrays

---
 .../node/tasks/list/ListTasksResponse.java    |   1 +
 .../common/xcontent/XContentBuilder.java      |   2 +-
 .../common/xcontent/cbor/CborXContent.java    |   3 +
 .../common/xcontent/json/JsonXContent.java    |   2 +
 .../xcontent/json/JsonXContentGenerator.java  |   4 +
 .../common/xcontent/smile/SmileXContent.java  |   3 +
 .../fieldstats/RestFieldStatsAction.java      |   2 +
 .../elasticsearch/script/ScriptMetaData.java  |   1 +
 .../common/xcontent/BaseXContentTestCase.java |  25 ++++
 .../builder/XContentBuilderTests.java         |  24 ++++
 .../org/elasticsearch/get/GetActionIT.java    |   2 +-
 .../index/mapper/DocumentParserTests.java     |   6 +-
 .../index/mapper/DynamicMappingTests.java     |  49 ++++----
 .../mapper/boost/FieldLevelBoostTests.java    |  55 +++++----
 .../mapper/compound/CompoundTypesTests.java   |   3 +
 .../copyto/CopyToMapperIntegrationIT.java     |   3 +-
 .../mapper/multifield/MultiFieldTests.java    |   1 +
 .../mapper/nested/NestedMappingTests.java     |  14 +--
 .../mapper/simple/SimpleMapperTests.java      |   2 +-
 ...gFieldMapperPositionIncrementGapTests.java |   2 +
 .../timestamp/TimestampMappingTests.java      |   6 +-
 .../index/mapper/ttl/TTLMappingTests.java     |   4 +-
 .../update/UpdateMappingOnClusterIT.java      |   2 +-
 .../query/MoreLikeThisQueryBuilderTests.java  |   1 +
 .../index/similarity/SimilarityTests.java     |   1 +
 .../aggregations/AggregatorParsingTests.java  | 110 +++++++++---------
 .../aggregations/bucket/ParentIdAggIT.java    |   1 +
 .../search/highlight/HighlighterSearchIT.java |   3 +-
 .../search/morelikethis/MoreLikeThisIT.java   |  17 +--
 .../search/query/SearchQueryIT.java           |   3 +-
 .../suggest/ContextSuggestSearch2xIT.java     |   2 +
 .../completion/GeoContextMappingTests.java    |  31 ++---
 .../similarity/SimilarityIT.java              |   1 +
 .../timestamp/SimpleTimestampIT.java          |   2 +-
 .../org/elasticsearch/ttl/SimpleTTLIT.java    |   2 +-
 .../messy/tests/EquivalenceTests.java         |  38 +++---
 .../messy/tests/SearchFieldsTests.java        |  35 ++++--
 .../percolator/MultiPercolatorIT.java         |   2 +-
 .../percolator/PercolatorIT.java              |   1 +
 .../percolator/PercolatorQuerySearchIT.java   |  48 ++++----
 .../SimpleAttachmentMapperTests.java          |   1 +
 .../index/mapper/size/SizeMappingIT.java      |   2 +-
 42 files changed, 308 insertions(+), 209 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
index 446ae3affb7..bad4001e211 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
@@ -192,6 +192,7 @@ public class ListTasksResponse extends BaseTasksResponse implements ToXContent {
                 builder.endObject();
                 builder.endObject();
             }
+            builder.endObject();
         } else if ("parents".equals(groupBy)) {
             builder.startObject("tasks");
             for (TaskGroup group : getTaskGroups()) {
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
index 3fa182e3727..2891ca2608c 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
@@ -781,7 +781,7 @@ public final class XContentBuilder implements BytesStream, Releasable {
         try {
             generator.close();
         } catch (IOException e) {
-            // ignore
+            throw new IllegalStateException("failed to close the XContentBuilder", e);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java
index bc2bc81a2c0..1670810511a 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.common.xcontent.cbor;
 
 import com.fasterxml.jackson.core.JsonEncoding;
+import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.dataformat.cbor.CBORFactory;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -50,6 +51,8 @@ public class CborXContent implements XContent {
     static {
         cborFactory = new CBORFactory();
         cborFactory.configure(CBORFactory.Feature.FAIL_ON_SYMBOL_HASH_OVERFLOW, false); // this trips on many mappings now...
+        // Do not automatically close unclosed objects/arrays in com.fasterxml.jackson.dataformat.cbor.CBORGenerator#close() method
+        cborFactory.configure(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT, false);
         cborXContent = new CborXContent();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java
index f991a462f3b..13cacc41538 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java
@@ -73,6 +73,8 @@ public class JsonXContent implements XContent {
         jsonFactory.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, true);
         jsonFactory.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
         jsonFactory.configure(JsonFactory.Feature.FAIL_ON_SYMBOL_HASH_OVERFLOW, false); // this trips on many mappings now...
+        // Do not automatically close unclosed objects/arrays in com.fasterxml.jackson.core.json.UTF8JsonGenerator#close() method
+        jsonFactory.configure(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT, false);
         jsonXContent = new JsonXContent();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java
index e74ab01bd6a..462288f688a 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java
@@ -389,6 +389,10 @@ public class JsonXContentGenerator implements XContentGenerator {
         if (generator.isClosed()) {
             return;
         }
+        JsonStreamContext context = generator.getOutputContext();
+        if ((context != null) && (context.inRoot() ==  false)) {
+            throw new IOException("unclosed object or array found");
+        }
         if (writeLineFeedAtEnd) {
             flush();
             generator.writeRaw(LF);
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java
index fc12e40f5a3..3a104056b09 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.common.xcontent.smile;
 
 import com.fasterxml.jackson.core.JsonEncoding;
+import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.dataformat.smile.SmileFactory;
 import com.fasterxml.jackson.dataformat.smile.SmileGenerator;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -51,6 +52,8 @@ public class SmileXContent implements XContent {
         smileFactory = new SmileFactory();
         smileFactory.configure(SmileGenerator.Feature.ENCODE_BINARY_AS_7BIT, false); // for now, this is an overhead, might make sense for web sockets
         smileFactory.configure(SmileFactory.Feature.FAIL_ON_SYMBOL_HASH_OVERFLOW, false); // this trips on many mappings now...
+        // Do not automatically close unclosed objects/arrays in com.fasterxml.jackson.dataformat.smile.SmileGenerator#close() method
+        smileFactory.configure(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT, false);
         smileXContent = new SmileXContent();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/fieldstats/RestFieldStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/fieldstats/RestFieldStatsAction.java
index c2dc3220ec8..744a76e7352 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/fieldstats/RestFieldStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/fieldstats/RestFieldStatsAction.java
@@ -99,7 +99,9 @@ public class RestFieldStatsAction extends BaseRestHandler {
                     for (Map.Entry<String, String> entry : response.getConflicts().entrySet()) {
                         builder.field(entry.getKey(), entry.getValue());
                     }
+                    builder.endObject();
                 }
+                builder.endObject();
                 return new BytesRestResponse(RestStatus.OK, builder);
             }
         });
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java b/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java
index 84b4da4b818..5c0f8cee04c 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java
@@ -94,6 +94,7 @@ public final class ScriptMetaData implements MetaData.Custom {
                     // because the parsers current location is already beyond the beginning we need to add a START_OBJECT:
                     builder.startObject();
                     builder.copyCurrentStructure(parser);
+                    builder.endObject();
                     break;
             }
             return builder.string();
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java b/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java
index 1b917263b79..b4ddd4ab589 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java
@@ -27,6 +27,8 @@ import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 
+import static org.hamcrest.Matchers.equalTo;
+
 public abstract class BaseXContentTestCase extends ESTestCase {
 
     public abstract XContentType xcontentType();
@@ -41,6 +43,29 @@ public abstract class BaseXContentTestCase extends ESTestCase {
         assertEquals(xcontentType(), XContentFactory.xContentType(data));
     }
 
+    public void testMissingEndObject() throws IOException {
+        IOException e = expectThrows(IOException.class, () -> {
+            ByteArrayOutputStream os = new ByteArrayOutputStream();
+            try (XContentGenerator generator = xcontentType().xContent().createGenerator(os)) {
+                generator.writeStartObject();
+                generator.writeFieldName("foo");
+                generator.writeNumber(2L);
+            }
+        });
+        assertEquals(e.getMessage(), "unclosed object or array found");
+    }
+
+    public void testMissingEndArray() throws IOException {
+        IOException e = expectThrows(IOException.class, () -> {
+            ByteArrayOutputStream os = new ByteArrayOutputStream();
+            try (XContentGenerator generator = xcontentType().xContent().createGenerator(os)) {
+                generator.writeStartArray();
+                generator.writeNumber(2L);
+            }
+        });
+        assertEquals(e.getMessage(), "unclosed object or array found");
+    }
+
     public void testRawField() throws Exception {
         for (boolean useStream : new boolean[] {false, true}) {
             for (XContentType xcontentType : XContentType.values()) {
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java
index 42695959bbe..34944e713bd 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java
@@ -363,4 +363,28 @@ public class XContentBuilderTests extends ESTestCase {
             assertThat(e.getMessage(), equalTo("field name cannot be null"));
         }
     }
+
+    public void testMissingEndObject() throws IOException {
+        IllegalStateException e = expectThrows(IllegalStateException.class, () -> {
+            try (XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()))) {
+                builder.startObject();
+                builder.field("foo", true);
+            }
+        });
+        assertThat(e.getMessage(), equalTo("failed to close the XContentBuilder"));
+        assertThat(e.getCause().getMessage(), equalTo("unclosed object or array found"));
+    }
+
+    public void testMissingEndArray() throws IOException {
+        IllegalStateException e = expectThrows(IllegalStateException.class, () -> {
+            try (XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()))) {
+                builder.startObject();
+                builder.startArray("foo");
+                builder.value(0);
+                builder.value(1);
+            }
+        });
+        assertThat(e.getMessage(), equalTo("failed to close the XContentBuilder"));
+        assertThat(e.getCause().getMessage(), equalTo("unclosed object or array found"));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/get/GetActionIT.java b/core/src/test/java/org/elasticsearch/get/GetActionIT.java
index 88c70b3dc3e..8c4b699cbff 100644
--- a/core/src/test/java/org/elasticsearch/get/GetActionIT.java
+++ b/core/src/test/java/org/elasticsearch/get/GetActionIT.java
@@ -627,7 +627,7 @@ public class GetActionIT extends ESIntegTestCase {
                                     .startObject("field4").field("type", "text").field("store", true)
                                 .endObject().endObject()
                             .endObject().endObject()
-                        .endObject().endObject()
+                        .endObject().endObject().endObject()
                         .endObject().endObject().endObject()));
 
         BytesReference source = jsonBuilder().startObject()
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
index 197ac9ec55e..058b1bd5360 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java
@@ -109,7 +109,7 @@ public class DocumentParserTests extends ESSingleNodeTestCase {
                     .field("type", "object")
                     .field("dynamic", true)
                     .startObject("properties")
-            .endObject().endObject().endObject().endObject().string();
+            .endObject().endObject().endObject().endObject().endObject().string();
         DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
         BytesReference bytes = XContentFactory.jsonBuilder()
             .startObject().startObject("foo")
@@ -129,7 +129,7 @@ public class DocumentParserTests extends ESSingleNodeTestCase {
             .field("type", "object")
             .field("dynamic", true)
             .startObject("properties")
-            .endObject().endObject().endObject().endObject().string();
+            .endObject().endObject().endObject().endObject().endObject().string();
         DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
         BytesReference bytes = XContentFactory.jsonBuilder()
             .startObject().startObject("foo").startObject("bar")
@@ -148,7 +148,7 @@ public class DocumentParserTests extends ESSingleNodeTestCase {
             .startObject("foo")
             .field("type", "object")
             .startObject("properties")
-            .endObject().endObject().endObject().endObject().string();
+            .endObject().endObject().endObject().endObject().endObject().string();
         DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping));
         BytesReference bytes = XContentFactory.jsonBuilder()
             .startObject().startObject("foo")
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
index af96bc503c0..c44ea296f73 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
@@ -65,6 +65,7 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
                 .startObject()
                 .field("field1", "value1")
                 .field("field2", "value2")
+                .endObject()
                 .bytes());
 
         assertThat(doc.rootDoc().get("field1"), equalTo("value1"));
@@ -85,6 +86,7 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
                 .startObject()
                 .field("field1", "value1")
                 .field("field2", "value2")
+                .endObject()
                 .bytes());
 
         assertThat(doc.rootDoc().get("field1"), equalTo("value1"));
@@ -102,27 +104,21 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
-        try {
-            defaultMapper.parse("test", "type", "1", jsonBuilder()
-                    .startObject()
-                    .field("field1", "value1")
-                    .field("field2", "value2")
-                    .bytes());
-            fail();
-        } catch (StrictDynamicMappingException e) {
-            // all is well
-        }
+        StrictDynamicMappingException e = expectThrows(StrictDynamicMappingException.class, () -> defaultMapper.parse("test", "type", "1", jsonBuilder()
+                .startObject()
+                .field("field1", "value1")
+                .field("field2", "value2")
+                .endObject()
+                .bytes()));
+        assertThat(e.getMessage(), equalTo("mapping set to strict, dynamic introduction of [field2] within [type] is not allowed"));
 
-        try {
-            defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
+        e = expectThrows(StrictDynamicMappingException.class, () -> defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
                     .startObject()
                     .field("field1", "value1")
                     .field("field2", (String) null)
-                    .bytes());
-            fail();
-        } catch (StrictDynamicMappingException e) {
-            // all is well
-        }
+                    .endObject()
+                    .bytes()));
+        assertThat(e.getMessage(), equalTo("mapping set to strict, dynamic introduction of [field2] within [type] is not allowed"));
     }
 
     public void testDynamicFalseWithInnerObjectButDynamicSetOnRoot() throws IOException {
@@ -142,6 +138,7 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
                 .field("field1", "value1")
                 .field("field2", "value2")
                 .endObject()
+                .endObject()
                 .bytes());
 
         assertThat(doc.rootDoc().get("obj1.field1"), equalTo("value1"));
@@ -160,17 +157,15 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
 
         DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
-        try {
+        StrictDynamicMappingException e = expectThrows(StrictDynamicMappingException.class, () ->
             defaultMapper.parse("test", "type", "1", jsonBuilder()
                     .startObject().startObject("obj1")
                     .field("field1", "value1")
                     .field("field2", "value2")
                     .endObject()
-                    .bytes());
-            fail();
-        } catch (StrictDynamicMappingException e) {
-            // all is well
-        }
+                    .endObject()
+                    .bytes()));
+        assertThat(e.getMessage(), equalTo("mapping set to strict, dynamic introduction of [field2] within [obj1] is not allowed"));
     }
 
     public void testDynamicMappingOnEmptyString() throws Exception {
@@ -220,7 +215,7 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
         DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("foo").field("type", "text").endObject().endObject()
-                .endObject().string();
+                .endObject().endObject().string();
 
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
         Mapper update = parse(mapper, parser, XContentFactory.jsonBuilder().startObject().field("foo", "bar").endObject());
@@ -263,7 +258,7 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
         // every new field introduction runs in linear time with the total number of fields
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("properties").startObject("foo").field("type", "text").endObject().endObject()
-                .endObject().string();
+                .endObject().endObject().string();
 
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
         assertEquals(mapping, serialize(mapper));
@@ -283,7 +278,7 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
                         .endObject()
                     .endObject()
                 .endObject()
-                .endObject().endObject().string(), serialize(update));
+                .endObject().endObject().endObject().string(), serialize(update));
     }
 
     public void testIntroduceTwoFields() throws Exception {
@@ -318,7 +313,7 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
                         .endObject()
                     .endObject()
                 .endObject()
-                .endObject().endObject().string(), serialize(update));
+                .endObject().endObject().endObject().string(), serialize(update));
     }
 
     public void testObject() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/boost/FieldLevelBoostTests.java b/core/src/test/java/org/elasticsearch/index/mapper/boost/FieldLevelBoostTests.java
index 90121e66ea8..9405a23eb03 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/boost/FieldLevelBoostTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/boost/FieldLevelBoostTests.java
@@ -37,6 +37,7 @@ import java.util.Collection;
 
 import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
 
 /**
  */
@@ -59,6 +60,7 @@ public class FieldLevelBoostTests extends ESSingleNodeTestCase {
                 .startObject("float_field").field("type", "float").startObject("norms").field("enabled", true).endObject().endObject()
                 .startObject("long_field").field("type", "long").startObject("norms").field("enabled", true).endObject().endObject()
                 .startObject("short_field").field("type", "short").startObject("norms").field("enabled", true).endObject().endObject()
+                .endObject().endObject().endObject()
                 .string();
 
         DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("person", new CompressedXContent(mapping));
@@ -71,6 +73,7 @@ public class FieldLevelBoostTests extends ESSingleNodeTestCase {
                 .startObject("float_field").field("boost", 7.0).field("value", 40.0).endObject()
                 .startObject("long_field").field("boost", 8.0).field("value", 50).endObject()
                 .startObject("short_field").field("boost", 9.0).field("value", 60).endObject()
+                .endObject()
                 .bytes();
         Document doc = docMapper.parse("test", "person", "1", json).rootDoc();
 
@@ -109,6 +112,7 @@ public class FieldLevelBoostTests extends ESSingleNodeTestCase {
             .startObject("float_field").field("type", "float").field("boost", "7.0").endObject()
             .startObject("long_field").field("type", "long").field("boost", "8.0").endObject()
             .startObject("short_field").field("type", "short").field("boost", "9.0").endObject()
+            .endObject().endObject().endObject()
             .string();
 
         {
@@ -122,6 +126,7 @@ public class FieldLevelBoostTests extends ESSingleNodeTestCase {
                 .field("float_field", 40.0)
                 .field("long_field", 50)
                 .field("short_field", 60)
+                .endObject()
                 .bytes();
             Document doc = docMapper.parse("test", "person", "1", json).rootDoc();
 
@@ -161,6 +166,7 @@ public class FieldLevelBoostTests extends ESSingleNodeTestCase {
                 .field("float_field", 40.0)
                 .field("long_field", 50)
                 .field("short_field", 60)
+                .endObject()
                 .bytes();
             Document doc = docMapper.parse("test", "person", "1", json).rootDoc();
 
@@ -200,79 +206,80 @@ public class FieldLevelBoostTests extends ESSingleNodeTestCase {
                 .startObject("float_field").field("type", "float").startObject("norms").field("enabled", true).endObject().endObject()
                 .startObject("long_field").field("type", "long").startObject("norms").field("enabled", true).endObject().endObject()
                 .startObject("short_field").field("type", "short").startObject("norms").field("enabled", true).endObject().endObject()
+                .endObject().endObject().endObject()
                 .string();
 
         DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("person", new CompressedXContent(mapping));
         try {
             docMapper.parse("test", "person", "1", XContentFactory.jsonBuilder().startObject()
                     .startObject("str_field").field("foo", "bar")
-                    .endObject().bytes()).rootDoc();
+                    .endObject().endObject().bytes()).rootDoc();
             fail();
-        } catch (MapperParsingException ex) {
-            // Expected
+        } catch (Exception ex) {
+            assertThat(ex, instanceOf(MapperParsingException.class));
         }
 
         try {
             docMapper.parse("test", "person", "1", XContentFactory.jsonBuilder().startObject()
                     .startObject("int_field").field("foo", "bar")
-                    .endObject().bytes()).rootDoc();
+                    .endObject().endObject().bytes()).rootDoc();
             fail();
-        } catch (MapperParsingException ex) {
-            // Expected
+        } catch (Exception ex) {
+            assertThat(ex, instanceOf(MapperParsingException.class));
         }
 
         try {
             docMapper.parse("test", "person", "1", XContentFactory.jsonBuilder().startObject()
                     .startObject("byte_field").field("foo", "bar")
-                    .endObject().bytes()).rootDoc();
+                    .endObject().endObject().bytes()).rootDoc();
             fail();
-        } catch (MapperParsingException ex) {
-            // Expected
+        } catch (Exception ex) {
+            assertThat(ex, instanceOf(MapperParsingException.class));
         }
 
         try {
             docMapper.parse("test", "person", "1", XContentFactory.jsonBuilder().startObject()
                     .startObject("date_field").field("foo", "bar")
-                    .endObject().bytes()).rootDoc();
+                    .endObject().endObject().bytes()).rootDoc();
             fail();
-        } catch (MapperParsingException ex) {
-            // Expected
+        } catch (Exception ex) {
+            assertThat(ex, instanceOf(MapperParsingException.class));
         }
 
         try {
             docMapper.parse("test", "person", "1", XContentFactory.jsonBuilder().startObject()
                     .startObject("double_field").field("foo", "bar")
-                    .endObject().bytes()).rootDoc();
+                    .endObject().endObject().bytes()).rootDoc();
             fail();
-        } catch (MapperParsingException ex) {
-            // Expected
+        } catch (Exception ex) {
+            assertThat(ex, instanceOf(MapperParsingException.class));
         }
 
         try {
             docMapper.parse("test", "person", "1", XContentFactory.jsonBuilder().startObject()
                     .startObject("float_field").field("foo", "bar")
-                    .endObject().bytes()).rootDoc();
+                    .endObject().endObject().bytes()).rootDoc();
             fail();
-        } catch (MapperParsingException ex) {
-            // Expected
+        } catch (Exception ex) {
+            assertThat(ex, instanceOf(MapperParsingException.class));
         }
 
         try {
             docMapper.parse("test", "person", "1", XContentFactory.jsonBuilder().startObject()
                     .startObject("long_field").field("foo", "bar")
-                    .endObject().bytes()).rootDoc();
+                    .endObject().endObject().bytes()).rootDoc();
             fail();
-        } catch (MapperParsingException ex) {
-            // Expected
+        } catch (Exception ex) {
+            assertThat(ex, instanceOf(MapperParsingException.class));
         }
 
         try {
             docMapper.parse("test", "person", "1", XContentFactory.jsonBuilder().startObject()
                     .startObject("short_field").field("foo", "bar")
-                    .endObject().bytes()).rootDoc();
+                    .endObject().endObject().bytes()).rootDoc();
             fail();
-        } catch (MapperParsingException ex) {
-            // Expected
+        } catch (Exception ex) {
+            assertThat(ex, instanceOf(MapperParsingException.class));
         }
 
     }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/compound/CompoundTypesTests.java b/core/src/test/java/org/elasticsearch/index/mapper/compound/CompoundTypesTests.java
index ea66eaa4c5c..79ede4a80cb 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/compound/CompoundTypesTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/compound/CompoundTypesTests.java
@@ -57,6 +57,7 @@ public class CompoundTypesTests extends ESSingleNodeTestCase {
                 .startObject()
                 .field("field1", "value1")
                 .field("field2", "value2")
+                .endObject()
                 .bytes());
 
         assertThat(doc.rootDoc().get("field1"), equalTo("value1"));
@@ -67,6 +68,7 @@ public class CompoundTypesTests extends ESSingleNodeTestCase {
                 .startObject()
                 .startObject("field1").field("value", "value1").field("boost", 2.0f).endObject()
                 .field("field2", "value2")
+                .endObject()
                 .bytes());
 
         assertThat(doc.rootDoc().get("field1"), equalTo("value1"));
@@ -77,6 +79,7 @@ public class CompoundTypesTests extends ESSingleNodeTestCase {
                 .startObject()
                 .field("field1", "value1")
                 .field("field2", "value2")
+                .endObject()
                 .bytes());
 
         assertThat(doc.rootDoc().get("field1"), equalTo("value1"));
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/copyto/CopyToMapperIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/mapper/copyto/CopyToMapperIntegrationIT.java
index fa98f2a45c4..a65aae4ab20 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/copyto/CopyToMapperIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/copyto/CopyToMapperIntegrationIT.java
@@ -106,7 +106,8 @@ public class CopyToMapperIntegrationIT extends ESIntegTestCase {
                 .field("copy_to", "{name}_raw").endObject()
                 .endObject().endObject()
 
-                .endArray();
+                .endArray()
+                .endObject().endObject();
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldTests.java b/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldTests.java
index 70af77513ab..f08e94d243e 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldTests.java
@@ -241,6 +241,7 @@ public class MultiFieldTests extends ESSingleNodeTestCase {
                 .endObject()
                 .endObject()
                 .endObject()
+                .endObject()
                 .endObject();
 
         MapperService mapperService = createIndex("test").mapperService();
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/nested/NestedMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/nested/NestedMappingTests.java
index 1b5245624ce..ddae147a802 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/nested/NestedMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/nested/NestedMappingTests.java
@@ -117,7 +117,7 @@ public class NestedMappingTests extends ESSingleNodeTestCase {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
                 .startObject("nested1").field("type", "nested").startObject("properties")
                 .startObject("nested2").field("type", "nested")
-                .endObject().endObject()
+                .endObject().endObject().endObject()
                 .endObject().endObject().endObject().string();
 
         DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
@@ -168,7 +168,7 @@ public class NestedMappingTests extends ESSingleNodeTestCase {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
                 .startObject("nested1").field("type", "nested").startObject("properties")
                 .startObject("nested2").field("type", "nested").field("include_in_parent", true)
-                .endObject().endObject()
+                .endObject().endObject().endObject()
                 .endObject().endObject().endObject().string();
 
         DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
@@ -219,7 +219,7 @@ public class NestedMappingTests extends ESSingleNodeTestCase {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
                 .startObject("nested1").field("type", "nested").field("include_in_parent", true).startObject("properties")
                 .startObject("nested2").field("type", "nested").field("include_in_parent", true)
-                .endObject().endObject()
+                .endObject().endObject().endObject()
                 .endObject().endObject().endObject().string();
 
         DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
@@ -270,7 +270,7 @@ public class NestedMappingTests extends ESSingleNodeTestCase {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
                 .startObject("nested1").field("type", "nested").startObject("properties")
                 .startObject("nested2").field("type", "nested").field("include_in_root", true)
-                .endObject().endObject()
+                .endObject().endObject().endObject()
                 .endObject().endObject().endObject().string();
 
         DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
@@ -321,7 +321,7 @@ public class NestedMappingTests extends ESSingleNodeTestCase {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
                 .startObject("nested1").field("type", "nested").field("dynamic", "strict").startObject("properties")
                 .startObject("field1").field("type", "text")
-                .endObject().endObject()
+                .endObject().endObject().endObject()
                 .endObject().endObject().endObject().string();
 
         DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
@@ -355,7 +355,7 @@ public class NestedMappingTests extends ESSingleNodeTestCase {
                 return XContentFactory.jsonBuilder().startObject().startObject(type).startObject("properties")
                     .startObject("nested1").field("type", "nested").startObject("properties")
                     .startObject("nested2").field("type", "nested")
-                    .endObject().endObject()
+                    .endObject().endObject().endObject()
                     .endObject().endObject().endObject().string();
             } catch (IOException e) {
                 throw new UncheckedIOException(e);
@@ -390,7 +390,7 @@ public class NestedMappingTests extends ESSingleNodeTestCase {
         mapperService.merge("type2", new CompressedXContent(mapping.apply("type2")), MergeReason.MAPPING_UPDATE, false);
         // adding new fields from different type is not ok
         String mapping2 = XContentFactory.jsonBuilder().startObject().startObject("type3").startObject("properties").startObject("nested3")
-            .field("type", "nested").startObject("properties").endObject().endObject().endObject().endObject().string();
+            .field("type", "nested").startObject("properties").endObject().endObject().endObject().endObject().endObject().string();
         try {
             mapperService.merge("type3", new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE, false);
             fail("Expected IllegalArgumentException");
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/simple/SimpleMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/simple/SimpleMapperTests.java
index c6e6d69662a..0f9733c44bb 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/simple/SimpleMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/simple/SimpleMapperTests.java
@@ -125,7 +125,7 @@ public class SimpleMapperTests extends ESSingleNodeTestCase {
         DocumentMapperParser mapperParser = indexService.mapperService().documentMapperParser();
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
             .startObject("foo.bar").field("type", "text").endObject()
-            .endObject().endObject().string();
+            .endObject().endObject().endObject().string();
         try {
             mapperParser.parse("type", new CompressedXContent(mapping));
             fail("Mapping parse should have failed");
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/string/StringFieldMapperPositionIncrementGapTests.java b/core/src/test/java/org/elasticsearch/index/mapper/string/StringFieldMapperPositionIncrementGapTests.java
index 680f0732f1f..91a926805bf 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/string/StringFieldMapperPositionIncrementGapTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/string/StringFieldMapperPositionIncrementGapTests.java
@@ -136,6 +136,7 @@ public class StringFieldMapperPositionIncrementGapTests extends ESSingleNodeTest
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("properties").startObject("string");
         mapping.field("type", "string");
         mapping.field("position_increment_gap", positionIncrementGap);
+        mapping.endObject().endObject().endObject();
         client().admin().indices().prepareCreate("test")
                 .setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0).build())
                 .addMapping("test", mapping)
@@ -151,6 +152,7 @@ public class StringFieldMapperPositionIncrementGapTests extends ESSingleNodeTest
         XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("properties").startObject("string");
         mapping.field("type", "string");
         mapping.field("analyzer", analyzer);
+        mapping.endObject().endObject().endObject();
         client().admin().indices().prepareCreate("test")
                 .addMapping("test", mapping)
                 .setSettings(settings)
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
index a0794325bdf..14d86290db4 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
@@ -69,7 +69,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
     }
 
     public void testSimpleDisabled() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().string();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
         DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         BytesReference source = XContentFactory.jsonBuilder()
                 .startObject()
@@ -104,8 +104,8 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
             version = randomVersion(random());
         } while (version.before(Version.V_2_0_0_beta1));
         for (String mapping : Arrays.asList(
-                XContentFactory.jsonBuilder().startObject().startObject("type").endObject().string(),
-                XContentFactory.jsonBuilder().startObject().startObject("type").startObject("_timestamp").endObject().endObject().string())) {
+                XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(),
+                XContentFactory.jsonBuilder().startObject().startObject("type").startObject("_timestamp").endObject().endObject().endObject().string())) {
             DocumentMapper docMapper = createIndex("test", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build()).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
             assertThat(docMapper.timestampFieldMapper().enabled(), equalTo(TimestampFieldMapper.Defaults.ENABLED.enabled));
             assertThat(docMapper.timestampFieldMapper().fieldType().stored(), equalTo(version.onOrAfter(Version.V_2_0_0_beta1)));
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java
index 2d28b699944..fe057045882 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java
@@ -44,7 +44,7 @@ import static org.hamcrest.Matchers.notNullValue;
 
 public class TTLMappingTests extends ESSingleNodeTestCase {
     public void testSimpleDisabled() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().string();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
         DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         BytesReference source = XContentFactory.jsonBuilder()
                 .startObject()
@@ -74,7 +74,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
     }
 
     public void testDefaultValues() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().string();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
         DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         assertThat(docMapper.TTLFieldMapper().enabled(), equalTo(TTLFieldMapper.Defaults.ENABLED_STATE.enabled));
         assertThat(docMapper.TTLFieldMapper().fieldType().stored(), equalTo(TTLFieldMapper.Defaults.TTL_FIELD_TYPE.stored()));
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java
index 600f84b5f5f..ed20cdb7e35 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java
@@ -125,7 +125,7 @@ public class UpdateMappingOnClusterIT extends ESIntegTestCase {
     }
 
     public void testDocValuesInvalidMappingOnUpdate() throws Exception {
-        String mapping = jsonBuilder().startObject().startObject(TYPE).startObject("properties").startObject("text").field("type", "text").endObject().endObject().endObject().string();
+        String mapping = jsonBuilder().startObject().startObject(TYPE).startObject("properties").startObject("text").field("type", "text").endObject().endObject().endObject().endObject().string();
         prepareCreate(INDEX).addMapping(TYPE, mapping).get();
         String mappingUpdate = jsonBuilder().startObject().startObject(TYPE).startObject("_all").startObject("fielddata").field("format", "doc_values").endObject().endObject().endObject().endObject().string();
         GetMappingsResponse mappingsBeforeUpdateResponse = client().admin().indices().prepareGetMappings(INDEX).addTypes(TYPE).get();
diff --git a/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
index e4b0a6893ac..91a42d70809 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
@@ -121,6 +121,7 @@ public class MoreLikeThisQueryBuilderTests extends AbstractQueryTestCase<MoreLik
             for (String field : randomFields) {
                 doc.field(field, randomAsciiOfLength(10));
             }
+            doc.endObject();
         } catch (IOException e) {
             throw new ElasticsearchException("Unable to generate random artificial doc!");
         }
diff --git a/core/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java b/core/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java
index 1c7391f670b..653eea4196c 100644
--- a/core/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java
+++ b/core/src/test/java/org/elasticsearch/index/similarity/SimilarityTests.java
@@ -240,6 +240,7 @@ public class SimilarityTests extends ESSingleNodeTestCase {
             .field("type", "text")
             .endObject()
             .endObject()
+            .endObject()
             .endObject().string();
         Settings settings = Settings.builder()
             .put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_2_2_0))
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
index 7d21fbaf2a1..8801b3da45e 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
@@ -225,25 +225,26 @@ public class AggregatorParsingTests extends ESTestCase {
     public void testTwoAggs() throws Exception {
         String source = JsonXContent.contentBuilder()
                 .startObject()
-                .startObject("by_date")
-                .startObject("date_histogram")
-                .field("field", "timestamp")
-                .field("interval", "month")
-                .endObject()
-                .startObject("aggs")
-                .startObject("tag_count")
-                .startObject("cardinality")
-                .field("field", "tag")
-                .endObject()
-                .endObject()
-                .endObject()
-                .startObject("aggs") // 2nd "aggs": illegal
-                .startObject("tag_count2")
-                .startObject("cardinality")
-                .field("field", "tag")
-                .endObject()
-                .endObject()
-                .endObject()
+                    .startObject("by_date")
+                        .startObject("date_histogram")
+                            .field("field", "timestamp")
+                            .field("interval", "month")
+                        .endObject()
+                        .startObject("aggs")
+                            .startObject("tag_count")
+                                .startObject("cardinality")
+                                    .field("field", "tag")
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                        .startObject("aggs") // 2nd "aggs": illegal
+                            .startObject("tag_count2")
+                                .startObject("cardinality")
+                                    .field("field", "tag")
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject()
                 .endObject().string();
         try {
             XContentParser parser = XContentFactory.xContent(source).createParser(source);
@@ -274,14 +275,15 @@ public class AggregatorParsingTests extends ESTestCase {
 
         String source = JsonXContent.contentBuilder()
                 .startObject()
-                .startObject(name)
-                .startObject("filter")
-                .startObject("range")
-                .startObject("stock")
-                .field("gt", 0)
-                .endObject()
-                .endObject()
-                .endObject()
+                    .startObject(name)
+                        .startObject("filter")
+                            .startObject("range")
+                                .startObject("stock")
+                                    .field("gt", 0)
+                                .endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject()
                 .endObject().string();
         try {
             XContentParser parser = XContentFactory.xContent(source).createParser(source);
@@ -323,19 +325,20 @@ public class AggregatorParsingTests extends ESTestCase {
     public void testMissingName() throws Exception {
         String source = JsonXContent.contentBuilder()
                 .startObject()
-                .startObject("by_date")
-                .startObject("date_histogram")
-                .field("field", "timestamp")
-                .field("interval", "month")
-                .endObject()
-                .startObject("aggs")
-                // the aggregation name is missing
-                //.startObject("tag_count")
-                .startObject("cardinality")
-                .field("field", "tag")
-                .endObject()
-                //.endObject()
-                .endObject()
+                    .startObject("by_date")
+                        .startObject("date_histogram")
+                            .field("field", "timestamp")
+                            .field("interval", "month")
+                        .endObject()
+                        .startObject("aggs")
+                            // the aggregation name is missing
+                            //.startObject("tag_count")
+                            .startObject("cardinality")
+                                .field("field", "tag")
+                            .endObject()
+                            //.endObject()
+                        .endObject()
+                    .endObject()
                 .endObject().string();
         try {
             XContentParser parser = XContentFactory.xContent(source).createParser(source);
@@ -351,19 +354,20 @@ public class AggregatorParsingTests extends ESTestCase {
     public void testMissingType() throws Exception {
         String source = JsonXContent.contentBuilder()
                 .startObject()
-                .startObject("by_date")
-                .startObject("date_histogram")
-                .field("field", "timestamp")
-                .field("interval", "month")
-                .endObject()
-                .startObject("aggs")
-                .startObject("tag_count")
-                // the aggregation type is missing
-                //.startObject("cardinality")
-                .field("field", "tag")
-                //.endObject()
-                .endObject()
-                .endObject()
+                    .startObject("by_date")
+                        .startObject("date_histogram")
+                            .field("field", "timestamp")
+                            .field("interval", "month")
+                        .endObject()
+                        .startObject("aggs")
+                            .startObject("tag_count")
+                                // the aggregation type is missing
+                                //.startObject("cardinality")
+                                .field("field", "tag")
+                                //.endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject()
                 .endObject().string();
         try {
             XContentParser parser = XContentFactory.xContent(source).createParser(source);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ParentIdAggIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ParentIdAggIT.java
index 0c01825d7e5..4ee01301b8a 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ParentIdAggIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ParentIdAggIT.java
@@ -40,6 +40,7 @@ public class ParentIdAggIT extends ESIntegTestCase {
                 .startObject("_parent")
                 .field("type", "parenttype")
                 .endObject()
+                .endObject()
                 .endObject();
         assertAcked(prepareCreate("testidx").addMapping("childtype", mapping));
         client().prepareIndex("testidx", "childtype").setSource(jsonBuilder().startObject().field("num", 1).endObject()).setParent("p1").get();
diff --git a/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java b/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java
index ab4d7655aa6..876c8b53d87 100644
--- a/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java
@@ -199,6 +199,7 @@ public class HighlighterSearchIT extends ESIntegTestCase {
                         .field("type", "text")
                         .endObject()
                         .endObject()
+                        .endObject()
                         .endObject())
                 .setSettings(Settings.builder()
                         .put(indexSettings())
@@ -828,7 +829,7 @@ public class HighlighterSearchIT extends ESIntegTestCase {
                             .endObject()
                         .endObject()
                     .endObject()
-                .endObject()));
+                .endObject().endObject().endObject()));
         ensureGreen();
 
         index("test", "type1", "1",
diff --git a/core/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java b/core/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java
index f6277658230..0b89e56dd10 100644
--- a/core/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java
+++ b/core/src/test/java/org/elasticsearch/search/morelikethis/MoreLikeThisIT.java
@@ -179,7 +179,7 @@ public class MoreLikeThisIT extends ESIntegTestCase {
                 .endObject().endObject().string();
         client().admin().indices().prepareCreate("foo").addMapping("bar", mapping).execute().actionGet();
         client().prepareIndex("foo", "bar", "1")
-                .setSource(jsonBuilder().startObject().startObject("foo").field("bar", "boz").endObject())
+                .setSource(jsonBuilder().startObject().startObject("foo").field("bar", "boz").endObject().endObject())
                 .execute().actionGet();
         client().admin().indices().prepareRefresh("foo").execute().actionGet();
         assertThat(ensureGreen(), equalTo(ClusterHealthStatus.GREEN));
@@ -204,7 +204,7 @@ public class MoreLikeThisIT extends ESIntegTestCase {
         ensureGreen();
 
         client().prepareIndex("foo", "bar", "1")
-                .setSource(jsonBuilder().startObject().startObject("foo").field("bar", "boz").endObject())
+                .setSource(jsonBuilder().startObject().startObject("foo").field("bar", "boz").endObject().endObject())
                 .setRouting("2")
                 .execute().actionGet();
         client().admin().indices().prepareRefresh("foo").execute().actionGet();
@@ -227,7 +227,7 @@ public class MoreLikeThisIT extends ESIntegTestCase {
         ensureGreen();
 
         client().prepareIndex("foo", "bar", "1")
-                .setSource(jsonBuilder().startObject().startObject("foo").field("bar", "boz").endObject())
+                .setSource(jsonBuilder().startObject().startObject("foo").field("bar", "boz").endObject().endObject())
                 .setRouting("4000")
                 .execute().actionGet();
         client().admin().indices().prepareRefresh("foo").execute().actionGet();
@@ -522,17 +522,6 @@ public class MoreLikeThisIT extends ESIntegTestCase {
         assertSearchResponse(response);
         assertHitCount(response, 0);
 
-        logger.info("Checking when document is malformed ...");
-        XContentBuilder malformedDoc = jsonBuilder().startObject();
-        mltQuery = moreLikeThisQuery(null, new Item[] {new Item("test", "type1", malformedDoc)})
-                .minTermFreq(0)
-                .minDocFreq(0)
-                .minimumShouldMatch("0%");
-        response = client().prepareSearch("test").setTypes("type1")
-                .setQuery(mltQuery).get();
-        assertSearchResponse(response);
-        assertHitCount(response, 0);
-
         logger.info("Checking the document matches otherwise ...");
         XContentBuilder normalDoc = jsonBuilder()
                 .startObject()
diff --git a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
index 882226afe63..15ee5822486 100644
--- a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
@@ -1711,7 +1711,8 @@ public class SearchQueryIT extends ESIntegTestCase {
     public void testAllDisabledButQueried() throws Exception {
         createIndex("myindex");
         assertAcked(client().admin().indices().preparePutMapping("myindex").setType("mytype").setSource(
-                jsonBuilder().startObject().startObject("mytype").startObject("_all").field("enabled", false)));
+                jsonBuilder().startObject().startObject("mytype").startObject("_all").field("enabled", false)
+                .endObject().endObject().endObject()));
         client().prepareIndex("myindex", "mytype").setId("1").setSource("bar", "foo").setRefresh(true).get();
         SearchResponse response = client().prepareSearch("myindex").setQuery(matchQuery("_all", "foo")).get();
         assertNoFailures(response);
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/ContextSuggestSearch2xIT.java b/core/src/test/java/org/elasticsearch/search/suggest/ContextSuggestSearch2xIT.java
index ec74c3e753d..9152fe61847 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/ContextSuggestSearch2xIT.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/ContextSuggestSearch2xIT.java
@@ -193,6 +193,7 @@ public class ContextSuggestSearch2xIT extends ESIntegTestCase {
             .field("type", "geo")
             .field("precision", precision)
             .endObject()
+            .endObject()
             .endObject().endObject()
             .endObject().endObject();
 
@@ -212,6 +213,7 @@ public class ContextSuggestSearch2xIT extends ESIntegTestCase {
             .field("type", "geo")
             .field("precision", precision)
             .endObject()
+            .endObject()
             .endObject().endObject()
             .endObject().endObject();
         assertAcked(client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(mapping.string()).get());
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java
index 02a3d94fc9a..33624ed8002 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/GeoContextMappingTests.java
@@ -142,21 +142,22 @@ public class GeoContextMappingTests extends ESSingleNodeTestCase {
         MappedFieldType completionFieldType = fieldMapper.fieldType();
         ParsedDocument parsedDocument = defaultMapper.parse("test", "type1", "1", jsonBuilder()
                 .startObject()
-                .startObject("completion")
-                .field("input", "suggestion5", "suggestion6", "suggestion7")
-                .startObject("contexts")
-                .startArray("ctx")
-                .startObject()
-                .field("lat", 43.6624803)
-                .field("lon", -79.3863353)
-                .endObject()
-                .startObject()
-                .field("lat", 43.6624718)
-                .field("lon", -79.3873227)
-                .endObject()
-                .endArray()
-                .endObject()
-                .field("weight", 5)
+                    .startObject("completion")
+                        .field("input", "suggestion5", "suggestion6", "suggestion7")
+                        .startObject("contexts")
+                            .startArray("ctx")
+                                .startObject()
+                                    .field("lat", 43.6624803)
+                                    .field("lon", -79.3863353)
+                                .endObject()
+                                .startObject()
+                                    .field("lat", 43.6624718)
+                                    .field("lon", -79.3873227)
+                                .endObject()
+                            .endArray()
+                        .endObject()
+                        .field("weight", 5)
+                    .endObject()
                 .endObject()
                 .bytes());
         IndexableField[] fields = parsedDocument.rootDoc().getFields(completionFieldType.name());
diff --git a/core/src/test/java/org/elasticsearch/similarity/SimilarityIT.java b/core/src/test/java/org/elasticsearch/similarity/SimilarityIT.java
index 399aed57526..a123f7133b3 100644
--- a/core/src/test/java/org/elasticsearch/similarity/SimilarityIT.java
+++ b/core/src/test/java/org/elasticsearch/similarity/SimilarityIT.java
@@ -47,6 +47,7 @@ public class SimilarityIT extends ESIntegTestCase {
                                 .startObject("field2")
                                     .field("similarity", "classic")
                                     .field("type", "text")
+                                .endObject()
                             .endObject()
                         .endObject()
                     .endObject())
diff --git a/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java b/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java
index 6742d1b0bfb..21408e7b1df 100644
--- a/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java
+++ b/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java
@@ -102,7 +102,7 @@ public class SimpleTimestampIT extends ESIntegTestCase {
         assertTimestampMappingEnabled(index, type, true);
 
         // update some field in the mapping
-        XContentBuilder updateMappingBuilder = jsonBuilder().startObject().startObject("properties").startObject("otherField").field("type", "text").endObject().endObject();
+        XContentBuilder updateMappingBuilder = jsonBuilder().startObject().startObject("properties").startObject("otherField").field("type", "text").endObject().endObject().endObject();
         PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping(index).setType(type).setSource(updateMappingBuilder).get();
         assertAcked(putMappingResponse);
 
diff --git a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
index 57e668678e8..f216f998305 100644
--- a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
+++ b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
@@ -214,7 +214,7 @@ public class SimpleTTLIT extends ESIntegTestCase {
         assertTTLMappingEnabled(index, type);
 
         // update some field in the mapping
-        XContentBuilder updateMappingBuilder = jsonBuilder().startObject().startObject("properties").startObject("otherField").field("type", "text").endObject().endObject();
+        XContentBuilder updateMappingBuilder = jsonBuilder().startObject().startObject("properties").startObject("otherField").field("type", "text").endObject().endObject().endObject();
         PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping(index).setType(type).setSource(updateMappingBuilder).get();
         assertAcked(putMappingResponse);
 
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java
index deed43abf12..1bca59dc426 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java
@@ -184,25 +184,27 @@ public class EquivalenceTests extends ESIntegTestCase {
         final IntHashSet valuesSet = new IntHashSet();
         cluster().wipeIndices("idx");
         prepareCreate("idx")
-                .addMapping("type", jsonBuilder().startObject()
-                        .startObject("type")
-                        .startObject("properties")
-                        .startObject("string_values")
-                        .field("type", "keyword")
-                        .startObject("fields")
-                            .startObject("doc_values")
-                                .field("type", "keyword")
-                                .field("index", false)
+                .addMapping("type", jsonBuilder()
+                        .startObject()
+                            .startObject("type")
+                                .startObject("properties")
+                                    .startObject("string_values")
+                                        .field("type", "keyword")
+                                        .startObject("fields")
+                                            .startObject("doc_values")
+                                                .field("type", "keyword")
+                                                .field("index", false)
+                                            .endObject()
+                                        .endObject()
+                                    .endObject()
+                                    .startObject("long_values")
+                                        .field("type", "long")
+                                    .endObject()
+                                    .startObject("double_values")
+                                        .field("type", "double")
+                                    .endObject()
+                                .endObject()
                             .endObject()
-                        .endObject()
-                        .endObject()
-                        .startObject("long_values")
-                        .field("type", "long")
-                        .endObject()
-                        .startObject("double_values")
-                        .field("type", "double")
-                        .endObject()
-                        .endObject()
                         .endObject()).execute().actionGet();
 
         List<IndexRequestBuilder> indexingRequests = new ArrayList<>();
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
index 8cecbf2c787..4e3accfdc06 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
@@ -484,15 +484,32 @@ public class SearchFieldsTests extends ESIntegTestCase {
     public void testGetFieldsComplexField() throws Exception {
         client().admin().indices().prepareCreate("my-index")
                 .setSettings(Settings.builder().put("index.refresh_interval", -1))
-                .addMapping("my-type2", jsonBuilder().startObject().startObject("my-type2").startObject("properties")
-                        .startObject("field1").field("type", "object").startObject("properties")
-                        .startObject("field2").field("type", "object").startObject("properties")
-                        .startObject("field3").field("type", "object").startObject("properties")
-                        .startObject("field4").field("type", "text").field("store", true)
-                        .endObject().endObject()
-                        .endObject().endObject()
-                        .endObject().endObject()
-                        .endObject().endObject().endObject())
+                .addMapping("my-type2", jsonBuilder()
+                        .startObject()
+                            .startObject("my-type2")
+                                .startObject("properties")
+                                    .startObject("field1")
+                                        .field("type", "object")
+                                        .startObject("properties")
+                                            .startObject("field2")
+                                                .field("type", "object")
+                                                .startObject("properties")
+                                                    .startObject("field3")
+                                                        .field("type", "object")
+                                                        .startObject("properties")
+                                                            .startObject("field4")
+                                                                .field("type", "text")
+                                                                .field("store", true)
+                                                            .endObject()
+                                                        .endObject()
+                                                    .endObject()
+                                                .endObject()
+                                            .endObject()
+                                        .endObject()
+                                    .endObject()
+                                .endObject()
+                            .endObject()
+                        .endObject())
                 .get();
 
         BytesReference source = jsonBuilder().startObject()
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java
index efa6ecee59a..c45af9c089b 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java
@@ -235,7 +235,7 @@ public class MultiPercolatorIT extends ESIntegTestCase {
         }
 
         client().prepareIndex(INDEX_NAME, "type", "1")
-                .setSource(jsonBuilder().startObject().field("field", "a"))
+                .setSource(jsonBuilder().startObject().field("field", "a").endObject())
                 .execute().actionGet();
         refresh();
 
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
index e4b318f5d39..ed9b65130a9 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
@@ -1489,6 +1489,7 @@ public class PercolatorIT extends ESIntegTestCase {
         XContentBuilder doc = jsonBuilder();
         doc.startObject();
         doc.field("some_unnested_field", "value");
+        doc.endObject();
         PercolateResponse response = preparePercolate(client()).setPercolateDoc(new PercolateSourceBuilder.DocBuilder().setDoc(doc)).setIndices(INDEX_NAME).setDocumentType("company").get();
         assertNoFailures(response);
     }
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
index aba81575b37..a277549f48e 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
@@ -61,32 +61,32 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
         );
 
         client().prepareIndex("test", "queries", "1")
-            .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
-            .get();
+                .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
+                .get();
         client().prepareIndex("test", "queries", "2")
-            .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "value")).endObject())
-            .get();
+                .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "value")).endObject())
+                .get();
         client().prepareIndex("test", "queries", "3")
-            .setSource(jsonBuilder().startObject().field("query", boolQuery()
-                .must(matchQuery("field1", "value"))
-                .must(matchQuery("field2", "value"))
-            ).endObject()).get();
+                .setSource(jsonBuilder().startObject().field("query", boolQuery()
+                        .must(matchQuery("field1", "value"))
+                        .must(matchQuery("field2", "value"))
+                ).endObject()).get();
         client().admin().indices().prepareRefresh().get();
 
         BytesReference source = jsonBuilder().startObject().endObject().bytes();
         logger.info("percolating empty doc");
         SearchResponse response = client().prepareSearch()
-            .setQuery(new PercolateQueryBuilder("query", "type", source))
-            .get();
+                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .get();
         assertHitCount(response, 1);
         assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
 
         source = jsonBuilder().startObject().field("field1", "value").endObject().bytes();
         logger.info("percolating doc with 1 field");
         response = client().prepareSearch()
-            .setQuery(new PercolateQueryBuilder("query", "type", source))
-            .addSort("_uid", SortOrder.ASC)
-            .get();
+                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .addSort("_uid", SortOrder.ASC)
+                .get();
         assertHitCount(response, 2);
         assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
         assertThat(response.getHits().getAt(1).getId(), equalTo("2"));
@@ -94,9 +94,9 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
         source = jsonBuilder().startObject().field("field1", "value").field("field2", "value").endObject().bytes();
         logger.info("percolating doc with 2 fields");
         response = client().prepareSearch()
-            .setQuery(new PercolateQueryBuilder("query", "type", source))
-            .addSort("_uid", SortOrder.ASC)
-            .get();
+                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .addSort("_uid", SortOrder.ASC)
+                .get();
         assertHitCount(response, 3);
         assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
         assertThat(response.getHits().getAt(1).getId(), equalTo("2"));
@@ -314,6 +314,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
                         .endObject()
                         .endObject()
                         .endObject()
+                        .endObject()
                         .endObject().endObject())
         );
     }
@@ -327,12 +328,13 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
                 .addMapping("doc_type", "field", "type=keyword")
                 .addMapping("query_type", jsonBuilder().startObject().startObject("query_type").startObject("properties")
                         .startObject("object_field")
-                            .field("type", "object")
-                            .startObject("properties")
-                                .startObject(queryFieldName)
-                                    .field("type", "percolator")
-                                .endObject()
-                            .endObject()
+                        .field("type", "object")
+                        .startObject("properties")
+                        .startObject(queryFieldName)
+                        .field("type", "percolator")
+                        .endObject()
+                        .endObject()
+                        .endObject()
                         .endObject()
                         .endObject().endObject())
         );
@@ -344,7 +346,7 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
         client().prepareIndex("test2", "query_type", "1")
                 .setSource(jsonBuilder().startObject().startObject("object_field")
                         .field(queryFieldName, matchQuery("field", "value"))
-                    .endObject().endObject())
+                        .endObject().endObject())
                 .get();
         client().admin().indices().prepareRefresh().get();
 
diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java
index fd5f480700c..8c86800e52b 100644
--- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java
+++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java
@@ -113,6 +113,7 @@ public class SimpleAttachmentMapperTests extends AttachmentUnitTestCase {
                 .endObject()
                 .endObject()
                 .endObject()
+                .endObject()
                 .endObject();
 
         byte[] mapping = mappingBuilder.bytes().toBytes();
diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java
index 996ac964a38..b6c341c0601 100644
--- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java
+++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java
@@ -56,7 +56,7 @@ public class SizeMappingIT extends ESIntegTestCase {
         assertSizeMappingEnabled(index, type, true);
 
         // update some field in the mapping
-        XContentBuilder updateMappingBuilder = jsonBuilder().startObject().startObject("properties").startObject("otherField").field("type", "text").endObject().endObject();
+        XContentBuilder updateMappingBuilder = jsonBuilder().startObject().startObject("properties").startObject("otherField").field("type", "text").endObject().endObject().endObject();
         PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping(index).setType(type).setSource(updateMappingBuilder).get();
         assertAcked(putMappingResponse);
 

From 1c59c7e34981dd48da8fb79cf2037e15f79fcb3a Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Wed, 25 May 2016 16:49:08 +0200
Subject: [PATCH 0460/1311] Log warning if minimum_master_nodes is set to less
 than a quorum of master-eligible nodes (#15625)

The setting minimum_master_nodes is crucial to avoid split brains in a cluster. In order to avoid data loss, it should always be configured to at least a quorum (majority) of master-eligible nodes.

This commit adds a warning to the logs on the master node if the value is set to less than quorum of master-eligible nodes.
---
 .../discovery/zen/NodeJoinController.java     |  7 +++++-
 .../discovery/zen/ZenDiscovery.java           |  3 ++-
 .../zen/elect/ElectMasterService.java         | 22 +++++++++++++++++++
 .../zen/NodeJoinControllerTests.java          |  2 ++
 4 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
index 39a3f51ac32..ee96335c2b2 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
@@ -33,6 +33,7 @@ import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.discovery.DiscoverySettings;
+import org.elasticsearch.discovery.zen.elect.ElectMasterService;
 import org.elasticsearch.discovery.zen.membership.MembershipAction;
 
 import java.util.ArrayList;
@@ -53,6 +54,7 @@ public class NodeJoinController extends AbstractComponent {
 
     final ClusterService clusterService;
     final RoutingService routingService;
+    final ElectMasterService electMaster;
     final DiscoverySettings discoverySettings;
     final AtomicBoolean accumulateJoins = new AtomicBoolean(false);
 
@@ -62,10 +64,11 @@ public class NodeJoinController extends AbstractComponent {
 
     protected final Map<DiscoveryNode, List<MembershipAction.JoinCallback>> pendingJoinRequests = new HashMap<>();
 
-    public NodeJoinController(ClusterService clusterService, RoutingService routingService, DiscoverySettings discoverySettings, Settings settings) {
+    public NodeJoinController(ClusterService clusterService, RoutingService routingService, ElectMasterService electMaster, DiscoverySettings discoverySettings, Settings settings) {
         super(settings);
         this.clusterService = clusterService;
         this.routingService = routingService;
+        this.electMaster = electMaster;
         this.discoverySettings = discoverySettings;
     }
 
@@ -449,6 +452,8 @@ public class NodeJoinController extends AbstractComponent {
                     logger.error("unexpected error during [{}]", e, source);
                 }
             }
+
+            NodeJoinController.this.electMaster.logMinimumMasterNodesWarningIfNecessary(oldState, newState);
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
index 0e36c72e854..9f0b194241b 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
@@ -215,7 +215,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
         nodesFD.setLocalNode(clusterService.localNode());
         joinThreadControl.start();
         pingService.start();
-        this.nodeJoinController = new NodeJoinController(clusterService, routingService, discoverySettings, settings);
+        this.nodeJoinController = new NodeJoinController(clusterService, routingService, electMaster, discoverySettings, settings);
     }
 
     @Override
@@ -617,6 +617,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
 
             @Override
             public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                electMaster.logMinimumMasterNodesWarningIfNecessary(oldState, newState);
             }
         });
     }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/elect/ElectMasterService.java b/core/src/main/java/org/elasticsearch/discovery/zen/elect/ElectMasterService.java
index 2870037eeb7..5ff79f112d2 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/elect/ElectMasterService.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/elect/ElectMasterService.java
@@ -22,6 +22,7 @@ package org.elasticsearch.discovery.zen.elect;
 import com.carrotsearch.hppc.ObjectContainer;
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
@@ -80,6 +81,27 @@ public class ElectMasterService extends AbstractComponent {
         return count >= minimumMasterNodes;
     }
 
+    public boolean hasTooManyMasterNodes(Iterable<DiscoveryNode> nodes) {
+        int count = 0;
+        for (DiscoveryNode node : nodes) {
+            if (node.isMasterNode()) {
+                count++;
+            }
+        }
+        return count > 1 && minimumMasterNodes <= count / 2;
+    }
+
+    public void logMinimumMasterNodesWarningIfNecessary(ClusterState oldState, ClusterState newState) {
+        // check if min_master_nodes setting is too low and log warning
+        if (hasTooManyMasterNodes(oldState.nodes()) == false && hasTooManyMasterNodes(newState.nodes())) {
+            logger.warn("value for setting \""
+                    + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()
+                    + "\" is too low. This can result in data loss! Please set it to at least a quorum of master-eligible nodes "
+                    + "(current value: [{}], total number of master-eligible nodes used for publishing in this round: [{}])",
+                minimumMasterNodes(), newState.getNodes().getMasterNodes().size());
+        }
+    }
+
     /**
      * Returns the given nodes sorted by likelihood of being elected as master, most likely first.
      * Non-master nodes are not removed but are rather put in the end
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
index b1e3ae157a7..9013d99678d 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
@@ -39,6 +39,7 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.BaseFuture;
 import org.elasticsearch.discovery.DiscoverySettings;
+import org.elasticsearch.discovery.zen.elect.ElectMasterService;
 import org.elasticsearch.discovery.zen.membership.MembershipAction;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.junit.annotations.TestLogging;
@@ -99,6 +100,7 @@ public class NodeJoinControllerTests extends ESTestCase {
         setState(clusterService, ClusterState.builder(clusterService.state()).nodes(
                 DiscoveryNodes.builder(initialNodes).masterNodeId(localNode.getId())));
         nodeJoinController = new NodeJoinController(clusterService, new NoopRoutingService(Settings.EMPTY),
+                new ElectMasterService(Settings.EMPTY, Version.CURRENT),
                 new DiscoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)),
                 Settings.EMPTY);
     }

From 6d62f337021a6d47bbdc7940031a0e7f1e742106 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Mon, 9 May 2016 20:23:42 +0200
Subject: [PATCH 0461/1311] Make doc_values accessible for _type

`doc_values` for _type field are created but any attempt to load them throws an IAE.

This PR re-enables `doc_values` loading for _type, it also enables `fielddata` loading for indices created between 2.0 and 2.1 since doc_values were disabled during that period.

It also restores the old docs that gives example on how to sort or aggregate on _type field.
---
 .../mapper/internal/TypeFieldMapper.java      | 62 ++++++++++++++++++-
 .../mapper/internal/TypeFieldMapperTests.java |  6 ++
 .../mapper/internal/TypeFieldTypeTests.java   | 12 ++++
 .../mapping/fields/type-field.asciidoc        | 25 +++++++-
 4 files changed, 101 insertions(+), 4 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
index 7aee0aefad8..f960ecaa977 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
@@ -35,12 +35,16 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
+import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.StringFieldType;
+import org.elasticsearch.index.mapper.core.TextFieldMapper;
 import org.elasticsearch.index.query.QueryShardContext;
 
 import java.io.IOException;
@@ -87,12 +91,29 @@ public class TypeFieldMapper extends MetadataFieldMapper {
     }
 
     static final class TypeFieldType extends StringFieldType {
+        private boolean fielddata;
 
         public TypeFieldType() {
+            this.fielddata = false;
         }
 
         protected TypeFieldType(TypeFieldType ref) {
             super(ref);
+            this.fielddata = ref.fielddata;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (super.equals(o) == false) {
+                return false;
+            }
+            TypeFieldType that = (TypeFieldType) o;
+            return fielddata == that.fielddata;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(super.hashCode(), fielddata);
         }
 
         @Override
@@ -105,6 +126,29 @@ public class TypeFieldMapper extends MetadataFieldMapper {
             return CONTENT_TYPE;
         }
 
+        public boolean fielddata() {
+            return fielddata;
+        }
+
+        public void setFielddata(boolean fielddata) {
+            checkIfFrozen();
+            this.fielddata = fielddata;
+        }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder() {
+            if (hasDocValues()) {
+                return new DocValuesIndexFieldData.Builder();
+            }
+            assert indexOptions() != IndexOptions.NONE;
+            if (fielddata) {
+                return new PagedBytesIndexFieldData.Builder(TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY,
+                    TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY,
+                    TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE);
+            }
+            return super.fielddataBuilder();
+        }
+
         @Override
         public Query termQuery(Object value, @Nullable QueryShardContext context) {
             if (indexOptions() == IndexOptions.NONE) {
@@ -112,6 +156,19 @@ public class TypeFieldMapper extends MetadataFieldMapper {
             }
             return new TypeQuery(indexedValueForSearch(value));
         }
+
+        @Override
+        public void checkCompatibility(MappedFieldType other,
+                                       List<String> conflicts, boolean strict) {
+            super.checkCompatibility(other, conflicts, strict);
+            TypeFieldType otherType = (TypeFieldType) other;
+            if (strict) {
+                if (fielddata() != otherType.fielddata()) {
+                    conflicts.add("mapper [" + name() + "] is used by multiple types. Set update_all_types to true to update [fielddata] "
+                        + "across all types.");
+                }
+            }
+        }
     }
 
     public static class TypeQuery extends Query {
@@ -169,7 +226,10 @@ public class TypeFieldMapper extends MetadataFieldMapper {
     private static MappedFieldType defaultFieldType(Settings indexSettings) {
         MappedFieldType defaultFieldType = Defaults.FIELD_TYPE.clone();
         Version indexCreated = Version.indexCreated(indexSettings);
-        if (indexCreated.onOrAfter(Version.V_2_1_0)) {
+        if (indexCreated.before(Version.V_2_1_0)) {
+            // enables fielddata loading, doc values was disabled on _type between 2.0 and 2.1.
+            ((TypeFieldType) defaultFieldType).setFielddata(true);
+        } else {
             defaultFieldType.setHasDocValues(true);
         }
         return defaultFieldType;
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldMapperTests.java
index 77e7b46537e..bd3aaf84a4f 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldMapperTests.java
@@ -24,6 +24,8 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
+import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
@@ -31,6 +33,8 @@ import org.elasticsearch.test.InternalSettingsPlugin;
 
 import java.util.Collection;
 
+import static org.hamcrest.Matchers.instanceOf;
+
 public class TypeFieldMapperTests extends ESSingleNodeTestCase {
 
     @Override
@@ -44,6 +48,7 @@ public class TypeFieldMapperTests extends ESSingleNodeTestCase {
         DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         TypeFieldMapper typeMapper = docMapper.metadataMapper(TypeFieldMapper.class);
         assertTrue(typeMapper.fieldType().hasDocValues());
+        assertThat(typeMapper.fieldType().fielddataBuilder(), instanceOf(DocValuesIndexFieldData.Builder.class));
     }
 
     public void testDocValuesPre21() throws Exception {
@@ -54,5 +59,6 @@ public class TypeFieldMapperTests extends ESSingleNodeTestCase {
         DocumentMapper docMapper = createIndex("test", bwcSettings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         TypeFieldMapper typeMapper = docMapper.metadataMapper(TypeFieldMapper.class);
         assertFalse(typeMapper.fieldType().hasDocValues());
+        assertThat(typeMapper.fieldType().fielddataBuilder(), instanceOf(PagedBytesIndexFieldData.Builder.class));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldTypeTests.java
index 91216983b70..8f87ca9d683 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldTypeTests.java
@@ -37,6 +37,7 @@ import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.index.mapper.FieldTypeTestCase;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.junit.Before;
 
 public class TypeFieldTypeTests extends FieldTypeTestCase {
     @Override
@@ -44,6 +45,17 @@ public class TypeFieldTypeTests extends FieldTypeTestCase {
         return new TypeFieldMapper.TypeFieldType();
     }
 
+    @Before
+    public void setupProperties() {
+        addModifier(new Modifier("fielddata", true) {
+            @Override
+            public void modify(MappedFieldType ft) {
+                TypeFieldMapper.TypeFieldType tft = (TypeFieldMapper.TypeFieldType) ft;
+                tft.setFielddata(tft.fielddata() == false);
+            }
+        });
+    }
+
     public void testTermQuery() throws Exception {
         Directory dir = newDirectory();
         IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
diff --git a/docs/reference/mapping/fields/type-field.asciidoc b/docs/reference/mapping/fields/type-field.asciidoc
index ae283e44569..703ee9930d2 100644
--- a/docs/reference/mapping/fields/type-field.asciidoc
+++ b/docs/reference/mapping/fields/type-field.asciidoc
@@ -5,7 +5,8 @@ Each document indexed is associated with a <<mapping-type-field,`_type`>> (see
 <<mapping-type>>) and an <<mapping-id-field,`_id`>>.  The `_type` field is
 indexed in order to make searching by type name fast.
 
-The value of the `_type` field is accessible in queries and scripts:
+The value of the `_type` field is accessible in queries, aggregations,
+scripts, and when sorting:
 
 [source,js]
 --------------------------
@@ -27,9 +28,24 @@ GET my_index/type_*/_search
       "_type": [ "type_1", "type_2" ] <1>
     }
   },
+  "aggs": {
+    "types": {
+      "terms": {
+        "field": "_type", <2>
+        "size": 10
+      }
+    }
+  },
+  "sort": [
+    {
+      "_type": { <3>
+        "order": "desc"
+      }
+    }
+  ],
   "script_fields": {
     "type": {
-      "script": "doc['_type']" <2>
+      "script": "doc['_type']" <4>
     }
   }
 }
@@ -38,4 +54,7 @@ GET my_index/type_*/_search
 // CONSOLE
 
 <1> Querying on the `_type` field
-<2> Accessing the `_type` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
+<2> Aggregating on the `_type` field
+<3> Sorting on the `_type` field
+<4> Accessing the `_type` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
+

From eefcafd8317a1cf7ce8ef17be336c90bc1732289 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Wed, 25 May 2016 10:34:21 -0700
Subject: [PATCH 0462/1311] Add method reference stub to Painless.

---
 .../src/main/antlr/PainlessLexer.g4           |   3 +-
 .../src/main/antlr/PainlessLexer.tokens       |  88 +-
 .../src/main/antlr/PainlessParser.g4          |  11 +-
 .../src/main/antlr/PainlessParser.tokens      |  88 +-
 .../painless/antlr/PainlessLexer.java         | 381 ++++----
 .../painless/antlr/PainlessParser.java        | 901 ++++++++++--------
 .../antlr/PainlessParserBaseVisitor.java      |  14 +
 .../painless/antlr/PainlessParserVisitor.java |  12 +
 .../elasticsearch/painless/antlr/Walker.java  |  41 +-
 .../painless/node/EFunctionRef.java           |  48 +
 .../painless/node/package-info.java           |   1 +
 .../painless/FunctionRefTests.java            |  30 +
 .../elasticsearch/painless/ScoreTests.java    |   7 +-
 13 files changed, 932 insertions(+), 693 deletions(-)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
 create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java

diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.g4 b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
index 2588a646dcb..6e34d35111d 100644
--- a/modules/lang-painless/src/main/antlr/PainlessLexer.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
@@ -73,6 +73,7 @@ BOOLAND: '&&';
 BOOLOR:  '||';
 COND:    '?';
 COLON:   ':';
+REF:     '::';
 INCR:    '++';
 DECR:    '--';
 
@@ -113,4 +114,4 @@ ID: [_a-zA-Z] [_a-zA-Z0-9]*;
 mode AFTER_DOT;
 
 DOTINTEGER: ( '0' | [1-9] [0-9]* )                        -> mode(DEFAULT_MODE);
-DOTID: [_a-z] [_a-zA-Z0-9]*                               -> mode(DEFAULT_MODE);
+DOTID: [_a-zA-Z] [_a-zA-Z0-9]*                            -> mode(DEFAULT_MODE);
diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.tokens b/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
index d42e157e964..567fb8fdddd 100644
--- a/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
@@ -46,32 +46,33 @@ BOOLAND=45
 BOOLOR=46
 COND=47
 COLON=48
-INCR=49
-DECR=50
-ASSIGN=51
-AADD=52
-ASUB=53
-AMUL=54
-ADIV=55
-AREM=56
-AAND=57
-AXOR=58
-AOR=59
-ALSH=60
-ARSH=61
-AUSH=62
-OCTAL=63
-HEX=64
-INTEGER=65
-DECIMAL=66
-STRING=67
-TRUE=68
-FALSE=69
-NULL=70
-TYPE=71
-ID=72
-DOTINTEGER=73
-DOTID=74
+REF=49
+INCR=50
+DECR=51
+ASSIGN=52
+AADD=53
+ASUB=54
+AMUL=55
+ADIV=56
+AREM=57
+AAND=58
+AXOR=59
+AOR=60
+ALSH=61
+ARSH=62
+AUSH=63
+OCTAL=64
+HEX=65
+INTEGER=66
+DECIMAL=67
+STRING=68
+TRUE=69
+FALSE=70
+NULL=71
+TYPE=72
+ID=73
+DOTINTEGER=74
+DOTID=75
 '{'=3
 '}'=4
 '['=5
@@ -118,20 +119,21 @@ DOTID=74
 '||'=46
 '?'=47
 ':'=48
-'++'=49
-'--'=50
-'='=51
-'+='=52
-'-='=53
-'*='=54
-'/='=55
-'%='=56
-'&='=57
-'^='=58
-'|='=59
-'<<='=60
-'>>='=61
-'>>>='=62
-'true'=68
-'false'=69
-'null'=70
+'::'=49
+'++'=50
+'--'=51
+'='=52
+'+='=53
+'-='=54
+'*='=55
+'/='=56
+'%='=57
+'&='=58
+'^='=59
+'|='=60
+'<<='=61
+'>>='=62
+'>>>='=63
+'true'=69
+'false'=70
+'null'=71
diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index 12de27cefd3..5125e7b29a6 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -72,6 +72,10 @@ decltype
     : TYPE (LBRACE RBRACE)*
     ;
 
+funcref
+    : TYPE REF ID
+    ;
+
 declvar
     : ID ( ASSIGN expression )?
     ;
@@ -156,5 +160,10 @@ brace
     ;
 
 arguments
-    : ( LP ( expression ( COMMA expression )* )? RP )
+    : ( LP ( argument ( COMMA argument )* )? RP )
+    ;
+
+argument
+    : expression
+    | funcref
     ;
diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.tokens b/modules/lang-painless/src/main/antlr/PainlessParser.tokens
index d42e157e964..567fb8fdddd 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.tokens
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.tokens
@@ -46,32 +46,33 @@ BOOLAND=45
 BOOLOR=46
 COND=47
 COLON=48
-INCR=49
-DECR=50
-ASSIGN=51
-AADD=52
-ASUB=53
-AMUL=54
-ADIV=55
-AREM=56
-AAND=57
-AXOR=58
-AOR=59
-ALSH=60
-ARSH=61
-AUSH=62
-OCTAL=63
-HEX=64
-INTEGER=65
-DECIMAL=66
-STRING=67
-TRUE=68
-FALSE=69
-NULL=70
-TYPE=71
-ID=72
-DOTINTEGER=73
-DOTID=74
+REF=49
+INCR=50
+DECR=51
+ASSIGN=52
+AADD=53
+ASUB=54
+AMUL=55
+ADIV=56
+AREM=57
+AAND=58
+AXOR=59
+AOR=60
+ALSH=61
+ARSH=62
+AUSH=63
+OCTAL=64
+HEX=65
+INTEGER=66
+DECIMAL=67
+STRING=68
+TRUE=69
+FALSE=70
+NULL=71
+TYPE=72
+ID=73
+DOTINTEGER=74
+DOTID=75
 '{'=3
 '}'=4
 '['=5
@@ -118,20 +119,21 @@ DOTID=74
 '||'=46
 '?'=47
 ':'=48
-'++'=49
-'--'=50
-'='=51
-'+='=52
-'-='=53
-'*='=54
-'/='=55
-'%='=56
-'&='=57
-'^='=58
-'|='=59
-'<<='=60
-'>>='=61
-'>>>='=62
-'true'=68
-'false'=69
-'null'=70
+'::'=49
+'++'=50
+'--'=51
+'='=52
+'+='=53
+'-='=54
+'*='=55
+'/='=56
+'%='=57
+'&='=58
+'^='=59
+'|='=60
+'<<='=61
+'>>='=62
+'>>>='=63
+'true'=69
+'false'=70
+'null'=71
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
index b85217f9490..9eb5730abc0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
@@ -25,10 +25,11 @@ class PainlessLexer extends Lexer {
     BREAK=18, RETURN=19, NEW=20, TRY=21, CATCH=22, THROW=23, BOOLNOT=24, BWNOT=25, 
     MUL=26, DIV=27, REM=28, ADD=29, SUB=30, LSH=31, RSH=32, USH=33, LT=34, 
     LTE=35, GT=36, GTE=37, EQ=38, EQR=39, NE=40, NER=41, BWAND=42, XOR=43, 
-    BWOR=44, BOOLAND=45, BOOLOR=46, COND=47, COLON=48, INCR=49, DECR=50, ASSIGN=51, 
-    AADD=52, ASUB=53, AMUL=54, ADIV=55, AREM=56, AAND=57, AXOR=58, AOR=59, 
-    ALSH=60, ARSH=61, AUSH=62, OCTAL=63, HEX=64, INTEGER=65, DECIMAL=66, STRING=67, 
-    TRUE=68, FALSE=69, NULL=70, TYPE=71, ID=72, DOTINTEGER=73, DOTID=74;
+    BWOR=44, BOOLAND=45, BOOLOR=46, COND=47, COLON=48, REF=49, INCR=50, DECR=51, 
+    ASSIGN=52, AADD=53, ASUB=54, AMUL=55, ADIV=56, AREM=57, AAND=58, AXOR=59, 
+    AOR=60, ALSH=61, ARSH=62, AUSH=63, OCTAL=64, HEX=65, INTEGER=66, DECIMAL=67, 
+    STRING=68, TRUE=69, FALSE=70, NULL=71, TYPE=72, ID=73, DOTINTEGER=74, 
+    DOTID=75;
   public static final int AFTER_DOT = 1;
   public static String[] modeNames = {
     "DEFAULT_MODE", "AFTER_DOT"
@@ -40,10 +41,10 @@ class PainlessLexer extends Lexer {
     "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "BOOLNOT", "BWNOT", 
     "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", "LTE", "GT", 
     "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", "BOOLAND", "BOOLOR", 
-    "COND", "COLON", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", 
-    "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", 
-    "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "TYPE", "ID", 
-    "DOTINTEGER", "DOTID"
+    "COND", "COLON", "REF", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", 
+    "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", 
+    "HEX", "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "TYPE", 
+    "ID", "DOTINTEGER", "DOTID"
   };
 
   private static final String[] _LITERAL_NAMES = {
@@ -52,9 +53,9 @@ class PainlessLexer extends Lexer {
     "'return'", "'new'", "'try'", "'catch'", "'throw'", "'!'", "'~'", "'*'", 
     "'/'", "'%'", "'+'", "'-'", "'<<'", "'>>'", "'>>>'", "'<'", "'<='", "'>'", 
     "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", "'^'", "'|'", "'&&'", 
-    "'||'", "'?'", "':'", "'++'", "'--'", "'='", "'+='", "'-='", "'*='", "'/='", 
-    "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", "'>>>='", null, null, 
-    null, null, null, "'true'", "'false'", "'null'"
+    "'||'", "'?'", "':'", "'::'", "'++'", "'--'", "'='", "'+='", "'-='", "'*='", 
+    "'/='", "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", "'>>>='", null, 
+    null, null, null, null, "'true'", "'false'", "'null'"
   };
   private static final String[] _SYMBOLIC_NAMES = {
     null, "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", 
@@ -62,10 +63,10 @@ class PainlessLexer extends Lexer {
     "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "BOOLNOT", "BWNOT", 
     "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", "LTE", "GT", 
     "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", "BOOLAND", "BOOLOR", 
-    "COND", "COLON", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", 
-    "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", 
-    "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "TYPE", "ID", 
-    "DOTINTEGER", "DOTID"
+    "COND", "COLON", "REF", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", 
+    "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", 
+    "HEX", "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "TYPE", 
+    "ID", "DOTINTEGER", "DOTID"
   };
   public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
 
@@ -124,7 +125,7 @@ class PainlessLexer extends Lexer {
   @Override
   public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) {
     switch (ruleIndex) {
-    case 70:
+    case 71:
       return TYPE_sempred((RuleContext)_localctx, predIndex);
     }
     return true;
@@ -138,7 +139,7 @@ class PainlessLexer extends Lexer {
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2L\u0209\b\1\b\1\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2M\u020e\b\1\b\1\4"+
     "\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n"+
     "\4\13\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22"+
     "\t\22\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31"+
@@ -147,178 +148,180 @@ class PainlessLexer extends Lexer {
     "+\4,\t,\4-\t-\4.\t.\4/\t/\4\60\t\60\4\61\t\61\4\62\t\62\4\63\t\63\4\64"+
     "\t\64\4\65\t\65\4\66\t\66\4\67\t\67\48\t8\49\t9\4:\t:\4;\t;\4<\t<\4=\t"+
     "=\4>\t>\4?\t?\4@\t@\4A\tA\4B\tB\4C\tC\4D\tD\4E\tE\4F\tF\4G\tG\4H\tH\4"+
-    "I\tI\4J\tJ\4K\tK\3\2\6\2\u009a\n\2\r\2\16\2\u009b\3\2\3\2\3\3\3\3\3\3"+
-    "\3\3\7\3\u00a4\n\3\f\3\16\3\u00a7\13\3\3\3\3\3\3\3\3\3\3\3\7\3\u00ae\n"+
-    "\3\f\3\16\3\u00b1\13\3\3\3\3\3\5\3\u00b5\n\3\3\3\3\3\3\4\3\4\3\5\3\5\3"+
-    "\6\3\6\3\7\3\7\3\b\3\b\3\t\3\t\3\n\3\n\3\n\3\n\3\13\3\13\3\f\3\f\3\r\3"+
-    "\r\3\r\3\16\3\16\3\16\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3\17\3\20\3\20"+
-    "\3\20\3\21\3\21\3\21\3\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22"+
-    "\3\23\3\23\3\23\3\23\3\23\3\23\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\25"+
-    "\3\25\3\25\3\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\27\3\27\3\27\3\30"+
-    "\3\30\3\30\3\30\3\30\3\30\3\31\3\31\3\32\3\32\3\33\3\33\3\34\3\34\3\35"+
-    "\3\35\3\36\3\36\3\37\3\37\3 \3 \3 \3!\3!\3!\3\"\3\"\3\"\3\"\3#\3#\3$\3"+
-    "$\3$\3%\3%\3&\3&\3&\3\'\3\'\3\'\3(\3(\3(\3(\3)\3)\3)\3*\3*\3*\3*\3+\3"+
-    "+\3,\3,\3-\3-\3.\3.\3.\3/\3/\3/\3\60\3\60\3\61\3\61\3\62\3\62\3\62\3\63"+
-    "\3\63\3\63\3\64\3\64\3\65\3\65\3\65\3\66\3\66\3\66\3\67\3\67\3\67\38\3"+
-    "8\38\39\39\39\3:\3:\3:\3;\3;\3;\3<\3<\3<\3=\3=\3=\3=\3>\3>\3>\3>\3?\3"+
-    "?\3?\3?\3?\3@\3@\6@\u017b\n@\r@\16@\u017c\3@\5@\u0180\n@\3A\3A\3A\6A\u0185"+
-    "\nA\rA\16A\u0186\3A\5A\u018a\nA\3B\3B\3B\7B\u018f\nB\fB\16B\u0192\13B"+
-    "\5B\u0194\nB\3B\5B\u0197\nB\3C\3C\3C\7C\u019c\nC\fC\16C\u019f\13C\5C\u01a1"+
-    "\nC\3C\3C\6C\u01a5\nC\rC\16C\u01a6\5C\u01a9\nC\3C\3C\5C\u01ad\nC\3C\6"+
-    "C\u01b0\nC\rC\16C\u01b1\5C\u01b4\nC\3C\5C\u01b7\nC\3D\3D\3D\3D\3D\3D\7"+
-    "D\u01bf\nD\fD\16D\u01c2\13D\3D\3D\3D\3D\3D\3D\3D\7D\u01cb\nD\fD\16D\u01ce"+
-    "\13D\3D\5D\u01d1\nD\3E\3E\3E\3E\3E\3F\3F\3F\3F\3F\3F\3G\3G\3G\3G\3G\3"+
-    "H\3H\3H\3H\7H\u01e7\nH\fH\16H\u01ea\13H\3H\3H\3I\3I\7I\u01f0\nI\fI\16"+
-    "I\u01f3\13I\3J\3J\3J\7J\u01f8\nJ\fJ\16J\u01fb\13J\5J\u01fd\nJ\3J\3J\3"+
-    "K\3K\7K\u0203\nK\fK\16K\u0206\13K\3K\3K\6\u00a5\u00af\u01c0\u01cc\2L\4"+
-    "\3\6\4\b\5\n\6\f\7\16\b\20\t\22\n\24\13\26\f\30\r\32\16\34\17\36\20 \21"+
-    "\"\22$\23&\24(\25*\26,\27.\30\60\31\62\32\64\33\66\348\35:\36<\37> @!"+
-    "B\"D#F$H%J&L\'N(P)R*T+V,X-Z.\\/^\60`\61b\62d\63f\64h\65j\66l\67n8p9r:"+
-    "t;v<x=z>|?~@\u0080A\u0082B\u0084C\u0086D\u0088E\u008aF\u008cG\u008eH\u0090"+
-    "I\u0092J\u0094K\u0096L\4\2\3\22\5\2\13\f\17\17\"\"\4\2\f\f\17\17\3\2\62"+
-    "9\4\2NNnn\4\2ZZzz\5\2\62;CHch\3\2\63;\3\2\62;\b\2FFHHNNffhhnn\4\2GGgg"+
-    "\4\2--//\4\2HHhh\4\2$$^^\5\2C\\aac|\6\2\62;C\\aac|\4\2aac|\u0226\2\4\3"+
-    "\2\2\2\2\6\3\2\2\2\2\b\3\2\2\2\2\n\3\2\2\2\2\f\3\2\2\2\2\16\3\2\2\2\2"+
-    "\20\3\2\2\2\2\22\3\2\2\2\2\24\3\2\2\2\2\26\3\2\2\2\2\30\3\2\2\2\2\32\3"+
-    "\2\2\2\2\34\3\2\2\2\2\36\3\2\2\2\2 \3\2\2\2\2\"\3\2\2\2\2$\3\2\2\2\2&"+
-    "\3\2\2\2\2(\3\2\2\2\2*\3\2\2\2\2,\3\2\2\2\2.\3\2\2\2\2\60\3\2\2\2\2\62"+
-    "\3\2\2\2\2\64\3\2\2\2\2\66\3\2\2\2\28\3\2\2\2\2:\3\2\2\2\2<\3\2\2\2\2"+
-    ">\3\2\2\2\2@\3\2\2\2\2B\3\2\2\2\2D\3\2\2\2\2F\3\2\2\2\2H\3\2\2\2\2J\3"+
-    "\2\2\2\2L\3\2\2\2\2N\3\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2T\3\2\2\2\2V\3\2\2"+
-    "\2\2X\3\2\2\2\2Z\3\2\2\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3\2\2\2\2b\3\2\2\2"+
-    "\2d\3\2\2\2\2f\3\2\2\2\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2\2\2n\3\2\2\2\2p"+
-    "\3\2\2\2\2r\3\2\2\2\2t\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2z\3\2\2\2\2|\3\2"+
-    "\2\2\2~\3\2\2\2\2\u0080\3\2\2\2\2\u0082\3\2\2\2\2\u0084\3\2\2\2\2\u0086"+
-    "\3\2\2\2\2\u0088\3\2\2\2\2\u008a\3\2\2\2\2\u008c\3\2\2\2\2\u008e\3\2\2"+
-    "\2\2\u0090\3\2\2\2\2\u0092\3\2\2\2\3\u0094\3\2\2\2\3\u0096\3\2\2\2\4\u0099"+
-    "\3\2\2\2\6\u00b4\3\2\2\2\b\u00b8\3\2\2\2\n\u00ba\3\2\2\2\f\u00bc\3\2\2"+
-    "\2\16\u00be\3\2\2\2\20\u00c0\3\2\2\2\22\u00c2\3\2\2\2\24\u00c4\3\2\2\2"+
-    "\26\u00c8\3\2\2\2\30\u00ca\3\2\2\2\32\u00cc\3\2\2\2\34\u00cf\3\2\2\2\36"+
-    "\u00d4\3\2\2\2 \u00da\3\2\2\2\"\u00dd\3\2\2\2$\u00e1\3\2\2\2&\u00ea\3"+
-    "\2\2\2(\u00f0\3\2\2\2*\u00f7\3\2\2\2,\u00fb\3\2\2\2.\u00ff\3\2\2\2\60"+
-    "\u0105\3\2\2\2\62\u010b\3\2\2\2\64\u010d\3\2\2\2\66\u010f\3\2\2\28\u0111"+
-    "\3\2\2\2:\u0113\3\2\2\2<\u0115\3\2\2\2>\u0117\3\2\2\2@\u0119\3\2\2\2B"+
-    "\u011c\3\2\2\2D\u011f\3\2\2\2F\u0123\3\2\2\2H\u0125\3\2\2\2J\u0128\3\2"+
-    "\2\2L\u012a\3\2\2\2N\u012d\3\2\2\2P\u0130\3\2\2\2R\u0134\3\2\2\2T\u0137"+
-    "\3\2\2\2V\u013b\3\2\2\2X\u013d\3\2\2\2Z\u013f\3\2\2\2\\\u0141\3\2\2\2"+
-    "^\u0144\3\2\2\2`\u0147\3\2\2\2b\u0149\3\2\2\2d\u014b\3\2\2\2f\u014e\3"+
-    "\2\2\2h\u0151\3\2\2\2j\u0153\3\2\2\2l\u0156\3\2\2\2n\u0159\3\2\2\2p\u015c"+
-    "\3\2\2\2r\u015f\3\2\2\2t\u0162\3\2\2\2v\u0165\3\2\2\2x\u0168\3\2\2\2z"+
-    "\u016b\3\2\2\2|\u016f\3\2\2\2~\u0173\3\2\2\2\u0080\u0178\3\2\2\2\u0082"+
-    "\u0181\3\2\2\2\u0084\u0193\3\2\2\2\u0086\u01a0\3\2\2\2\u0088\u01d0\3\2"+
-    "\2\2\u008a\u01d2\3\2\2\2\u008c\u01d7\3\2\2\2\u008e\u01dd\3\2\2\2\u0090"+
-    "\u01e2\3\2\2\2\u0092\u01ed\3\2\2\2\u0094\u01fc\3\2\2\2\u0096\u0200\3\2"+
-    "\2\2\u0098\u009a\t\2\2\2\u0099\u0098\3\2\2\2\u009a\u009b\3\2\2\2\u009b"+
-    "\u0099\3\2\2\2\u009b\u009c\3\2\2\2\u009c\u009d\3\2\2\2\u009d\u009e\b\2"+
-    "\2\2\u009e\5\3\2\2\2\u009f\u00a0\7\61\2\2\u00a0\u00a1\7\61\2\2\u00a1\u00a5"+
-    "\3\2\2\2\u00a2\u00a4\13\2\2\2\u00a3\u00a2\3\2\2\2\u00a4\u00a7\3\2\2\2"+
-    "\u00a5\u00a6\3\2\2\2\u00a5\u00a3\3\2\2\2\u00a6\u00a8\3\2\2\2\u00a7\u00a5"+
-    "\3\2\2\2\u00a8\u00b5\t\3\2\2\u00a9\u00aa\7\61\2\2\u00aa\u00ab\7,\2\2\u00ab"+
-    "\u00af\3\2\2\2\u00ac\u00ae\13\2\2\2\u00ad\u00ac\3\2\2\2\u00ae\u00b1\3"+
-    "\2\2\2\u00af\u00b0\3\2\2\2\u00af\u00ad\3\2\2\2\u00b0\u00b2\3\2\2\2\u00b1"+
-    "\u00af\3\2\2\2\u00b2\u00b3\7,\2\2\u00b3\u00b5\7\61\2\2\u00b4\u009f\3\2"+
-    "\2\2\u00b4\u00a9\3\2\2\2\u00b5\u00b6\3\2\2\2\u00b6\u00b7\b\3\2\2\u00b7"+
-    "\7\3\2\2\2\u00b8\u00b9\7}\2\2\u00b9\t\3\2\2\2\u00ba\u00bb\7\177\2\2\u00bb"+
-    "\13\3\2\2\2\u00bc\u00bd\7]\2\2\u00bd\r\3\2\2\2\u00be\u00bf\7_\2\2\u00bf"+
-    "\17\3\2\2\2\u00c0\u00c1\7*\2\2\u00c1\21\3\2\2\2\u00c2\u00c3\7+\2\2\u00c3"+
-    "\23\3\2\2\2\u00c4\u00c5\7\60\2\2\u00c5\u00c6\3\2\2\2\u00c6\u00c7\b\n\3"+
-    "\2\u00c7\25\3\2\2\2\u00c8\u00c9\7.\2\2\u00c9\27\3\2\2\2\u00ca\u00cb\7"+
-    "=\2\2\u00cb\31\3\2\2\2\u00cc\u00cd\7k\2\2\u00cd\u00ce\7h\2\2\u00ce\33"+
-    "\3\2\2\2\u00cf\u00d0\7g\2\2\u00d0\u00d1\7n\2\2\u00d1\u00d2\7u\2\2\u00d2"+
-    "\u00d3\7g\2\2\u00d3\35\3\2\2\2\u00d4\u00d5\7y\2\2\u00d5\u00d6\7j\2\2\u00d6"+
-    "\u00d7\7k\2\2\u00d7\u00d8\7n\2\2\u00d8\u00d9\7g\2\2\u00d9\37\3\2\2\2\u00da"+
-    "\u00db\7f\2\2\u00db\u00dc\7q\2\2\u00dc!\3\2\2\2\u00dd\u00de\7h\2\2\u00de"+
-    "\u00df\7q\2\2\u00df\u00e0\7t\2\2\u00e0#\3\2\2\2\u00e1\u00e2\7e\2\2\u00e2"+
-    "\u00e3\7q\2\2\u00e3\u00e4\7p\2\2\u00e4\u00e5\7v\2\2\u00e5\u00e6\7k\2\2"+
-    "\u00e6\u00e7\7p\2\2\u00e7\u00e8\7w\2\2\u00e8\u00e9\7g\2\2\u00e9%\3\2\2"+
-    "\2\u00ea\u00eb\7d\2\2\u00eb\u00ec\7t\2\2\u00ec\u00ed\7g\2\2\u00ed\u00ee"+
-    "\7c\2\2\u00ee\u00ef\7m\2\2\u00ef\'\3\2\2\2\u00f0\u00f1\7t\2\2\u00f1\u00f2"+
-    "\7g\2\2\u00f2\u00f3\7v\2\2\u00f3\u00f4\7w\2\2\u00f4\u00f5\7t\2\2\u00f5"+
-    "\u00f6\7p\2\2\u00f6)\3\2\2\2\u00f7\u00f8\7p\2\2\u00f8\u00f9\7g\2\2\u00f9"+
-    "\u00fa\7y\2\2\u00fa+\3\2\2\2\u00fb\u00fc\7v\2\2\u00fc\u00fd\7t\2\2\u00fd"+
-    "\u00fe\7{\2\2\u00fe-\3\2\2\2\u00ff\u0100\7e\2\2\u0100\u0101\7c\2\2\u0101"+
-    "\u0102\7v\2\2\u0102\u0103\7e\2\2\u0103\u0104\7j\2\2\u0104/\3\2\2\2\u0105"+
-    "\u0106\7v\2\2\u0106\u0107\7j\2\2\u0107\u0108\7t\2\2\u0108\u0109\7q\2\2"+
-    "\u0109\u010a\7y\2\2\u010a\61\3\2\2\2\u010b\u010c\7#\2\2\u010c\63\3\2\2"+
-    "\2\u010d\u010e\7\u0080\2\2\u010e\65\3\2\2\2\u010f\u0110\7,\2\2\u0110\67"+
-    "\3\2\2\2\u0111\u0112\7\61\2\2\u01129\3\2\2\2\u0113\u0114\7\'\2\2\u0114"+
-    ";\3\2\2\2\u0115\u0116\7-\2\2\u0116=\3\2\2\2\u0117\u0118\7/\2\2\u0118?"+
-    "\3\2\2\2\u0119\u011a\7>\2\2\u011a\u011b\7>\2\2\u011bA\3\2\2\2\u011c\u011d"+
-    "\7@\2\2\u011d\u011e\7@\2\2\u011eC\3\2\2\2\u011f\u0120\7@\2\2\u0120\u0121"+
-    "\7@\2\2\u0121\u0122\7@\2\2\u0122E\3\2\2\2\u0123\u0124\7>\2\2\u0124G\3"+
-    "\2\2\2\u0125\u0126\7>\2\2\u0126\u0127\7?\2\2\u0127I\3\2\2\2\u0128\u0129"+
-    "\7@\2\2\u0129K\3\2\2\2\u012a\u012b\7@\2\2\u012b\u012c\7?\2\2\u012cM\3"+
-    "\2\2\2\u012d\u012e\7?\2\2\u012e\u012f\7?\2\2\u012fO\3\2\2\2\u0130\u0131"+
-    "\7?\2\2\u0131\u0132\7?\2\2\u0132\u0133\7?\2\2\u0133Q\3\2\2\2\u0134\u0135"+
-    "\7#\2\2\u0135\u0136\7?\2\2\u0136S\3\2\2\2\u0137\u0138\7#\2\2\u0138\u0139"+
-    "\7?\2\2\u0139\u013a\7?\2\2\u013aU\3\2\2\2\u013b\u013c\7(\2\2\u013cW\3"+
-    "\2\2\2\u013d\u013e\7`\2\2\u013eY\3\2\2\2\u013f\u0140\7~\2\2\u0140[\3\2"+
-    "\2\2\u0141\u0142\7(\2\2\u0142\u0143\7(\2\2\u0143]\3\2\2\2\u0144\u0145"+
-    "\7~\2\2\u0145\u0146\7~\2\2\u0146_\3\2\2\2\u0147\u0148\7A\2\2\u0148a\3"+
-    "\2\2\2\u0149\u014a\7<\2\2\u014ac\3\2\2\2\u014b\u014c\7-\2\2\u014c\u014d"+
-    "\7-\2\2\u014de\3\2\2\2\u014e\u014f\7/\2\2\u014f\u0150\7/\2\2\u0150g\3"+
-    "\2\2\2\u0151\u0152\7?\2\2\u0152i\3\2\2\2\u0153\u0154\7-\2\2\u0154\u0155"+
-    "\7?\2\2\u0155k\3\2\2\2\u0156\u0157\7/\2\2\u0157\u0158\7?\2\2\u0158m\3"+
-    "\2\2\2\u0159\u015a\7,\2\2\u015a\u015b\7?\2\2\u015bo\3\2\2\2\u015c\u015d"+
-    "\7\61\2\2\u015d\u015e\7?\2\2\u015eq\3\2\2\2\u015f\u0160\7\'\2\2\u0160"+
-    "\u0161\7?\2\2\u0161s\3\2\2\2\u0162\u0163\7(\2\2\u0163\u0164\7?\2\2\u0164"+
-    "u\3\2\2\2\u0165\u0166\7`\2\2\u0166\u0167\7?\2\2\u0167w\3\2\2\2\u0168\u0169"+
-    "\7~\2\2\u0169\u016a\7?\2\2\u016ay\3\2\2\2\u016b\u016c\7>\2\2\u016c\u016d"+
-    "\7>\2\2\u016d\u016e\7?\2\2\u016e{\3\2\2\2\u016f\u0170\7@\2\2\u0170\u0171"+
-    "\7@\2\2\u0171\u0172\7?\2\2\u0172}\3\2\2\2\u0173\u0174\7@\2\2\u0174\u0175"+
-    "\7@\2\2\u0175\u0176\7@\2\2\u0176\u0177\7?\2\2\u0177\177\3\2\2\2\u0178"+
-    "\u017a\7\62\2\2\u0179\u017b\t\4\2\2\u017a\u0179\3\2\2\2\u017b\u017c\3"+
-    "\2\2\2\u017c\u017a\3\2\2\2\u017c\u017d\3\2\2\2\u017d\u017f\3\2\2\2\u017e"+
-    "\u0180\t\5\2\2\u017f\u017e\3\2\2\2\u017f\u0180\3\2\2\2\u0180\u0081\3\2"+
-    "\2\2\u0181\u0182\7\62\2\2\u0182\u0184\t\6\2\2\u0183\u0185\t\7\2\2\u0184"+
-    "\u0183\3\2\2\2\u0185\u0186\3\2\2\2\u0186\u0184\3\2\2\2\u0186\u0187\3\2"+
-    "\2\2\u0187\u0189\3\2\2\2\u0188\u018a\t\5\2\2\u0189\u0188\3\2\2\2\u0189"+
-    "\u018a\3\2\2\2\u018a\u0083\3\2\2\2\u018b\u0194\7\62\2\2\u018c\u0190\t"+
-    "\b\2\2\u018d\u018f\t\t\2\2\u018e\u018d\3\2\2\2\u018f\u0192\3\2\2\2\u0190"+
-    "\u018e\3\2\2\2\u0190\u0191\3\2\2\2\u0191\u0194\3\2\2\2\u0192\u0190\3\2"+
-    "\2\2\u0193\u018b\3\2\2\2\u0193\u018c\3\2\2\2\u0194\u0196\3\2\2\2\u0195"+
-    "\u0197\t\n\2\2\u0196\u0195\3\2\2\2\u0196\u0197\3\2\2\2\u0197\u0085\3\2"+
-    "\2\2\u0198\u01a1\7\62\2\2\u0199\u019d\t\b\2\2\u019a\u019c\t\t\2\2\u019b"+
-    "\u019a\3\2\2\2\u019c\u019f\3\2\2\2\u019d\u019b\3\2\2\2\u019d\u019e\3\2"+
-    "\2\2\u019e\u01a1\3\2\2\2\u019f\u019d\3\2\2\2\u01a0\u0198\3\2\2\2\u01a0"+
-    "\u0199\3\2\2\2\u01a1\u01a8\3\2\2\2\u01a2\u01a4\5\24\n\2\u01a3\u01a5\t"+
-    "\t\2\2\u01a4\u01a3\3\2\2\2\u01a5\u01a6\3\2\2\2\u01a6\u01a4\3\2\2\2\u01a6"+
-    "\u01a7\3\2\2\2\u01a7\u01a9\3\2\2\2\u01a8\u01a2\3\2\2\2\u01a8\u01a9\3\2"+
-    "\2\2\u01a9\u01b3\3\2\2\2\u01aa\u01ac\t\13\2\2\u01ab\u01ad\t\f\2\2\u01ac"+
-    "\u01ab\3\2\2\2\u01ac\u01ad\3\2\2\2\u01ad\u01af\3\2\2\2\u01ae\u01b0\t\t"+
-    "\2\2\u01af\u01ae\3\2\2\2\u01b0\u01b1\3\2\2\2\u01b1\u01af\3\2\2\2\u01b1"+
-    "\u01b2\3\2\2\2\u01b2\u01b4\3\2\2\2\u01b3\u01aa\3\2\2\2\u01b3\u01b4\3\2"+
-    "\2\2\u01b4\u01b6\3\2\2\2\u01b5\u01b7\t\r\2\2\u01b6\u01b5\3\2\2\2\u01b6"+
-    "\u01b7\3\2\2\2\u01b7\u0087\3\2\2\2\u01b8\u01c0\7$\2\2\u01b9\u01ba\7^\2"+
-    "\2\u01ba\u01bf\7$\2\2\u01bb\u01bc\7^\2\2\u01bc\u01bf\7^\2\2\u01bd\u01bf"+
-    "\n\16\2\2\u01be\u01b9\3\2\2\2\u01be\u01bb\3\2\2\2\u01be\u01bd\3\2\2\2"+
-    "\u01bf\u01c2\3\2\2\2\u01c0\u01c1\3\2\2\2\u01c0\u01be\3\2\2\2\u01c1\u01c3"+
-    "\3\2\2\2\u01c2\u01c0\3\2\2\2\u01c3\u01d1\7$\2\2\u01c4\u01cc\7)\2\2\u01c5"+
-    "\u01c6\7^\2\2\u01c6\u01cb\7)\2\2\u01c7\u01c8\7^\2\2\u01c8\u01cb\7^\2\2"+
-    "\u01c9\u01cb\n\16\2\2\u01ca\u01c5\3\2\2\2\u01ca\u01c7\3\2\2\2\u01ca\u01c9"+
-    "\3\2\2\2\u01cb\u01ce\3\2\2\2\u01cc\u01cd\3\2\2\2\u01cc\u01ca\3\2\2\2\u01cd"+
-    "\u01cf\3\2\2\2\u01ce\u01cc\3\2\2\2\u01cf\u01d1\7)\2\2\u01d0\u01b8\3\2"+
-    "\2\2\u01d0\u01c4\3\2\2\2\u01d1\u0089\3\2\2\2\u01d2\u01d3\7v\2\2\u01d3"+
-    "\u01d4\7t\2\2\u01d4\u01d5\7w\2\2\u01d5\u01d6\7g\2\2\u01d6\u008b\3\2\2"+
-    "\2\u01d7\u01d8\7h\2\2\u01d8\u01d9\7c\2\2\u01d9\u01da\7n\2\2\u01da\u01db"+
-    "\7u\2\2\u01db\u01dc\7g\2\2\u01dc\u008d\3\2\2\2\u01dd\u01de\7p\2\2\u01de"+
-    "\u01df\7w\2\2\u01df\u01e0\7n\2\2\u01e0\u01e1\7n\2\2\u01e1\u008f\3\2\2"+
-    "\2\u01e2\u01e8\5\u0092I\2\u01e3\u01e4\5\24\n\2\u01e4\u01e5\5\u0092I\2"+
-    "\u01e5\u01e7\3\2\2\2\u01e6\u01e3\3\2\2\2\u01e7\u01ea\3\2\2\2\u01e8\u01e6"+
-    "\3\2\2\2\u01e8\u01e9\3\2\2\2\u01e9\u01eb\3\2\2\2\u01ea\u01e8\3\2\2\2\u01eb"+
-    "\u01ec\6H\2\2\u01ec\u0091\3\2\2\2\u01ed\u01f1\t\17\2\2\u01ee\u01f0\t\20"+
-    "\2\2\u01ef\u01ee\3\2\2\2\u01f0\u01f3\3\2\2\2\u01f1\u01ef\3\2\2\2\u01f1"+
-    "\u01f2\3\2\2\2\u01f2\u0093\3\2\2\2\u01f3\u01f1\3\2\2\2\u01f4\u01fd\7\62"+
-    "\2\2\u01f5\u01f9\t\b\2\2\u01f6\u01f8\t\t\2\2\u01f7\u01f6\3\2\2\2\u01f8"+
-    "\u01fb\3\2\2\2\u01f9\u01f7\3\2\2\2\u01f9\u01fa\3\2\2\2\u01fa\u01fd\3\2"+
-    "\2\2\u01fb\u01f9\3\2\2\2\u01fc\u01f4\3\2\2\2\u01fc\u01f5\3\2\2\2\u01fd"+
-    "\u01fe\3\2\2\2\u01fe\u01ff\bJ\4\2\u01ff\u0095\3\2\2\2\u0200\u0204\t\21"+
-    "\2\2\u0201\u0203\t\20\2\2\u0202\u0201\3\2\2\2\u0203\u0206\3\2\2\2\u0204"+
-    "\u0202\3\2\2\2\u0204\u0205\3\2\2\2\u0205\u0207\3\2\2\2\u0206\u0204\3\2"+
-    "\2\2\u0207\u0208\bK\4\2\u0208\u0097\3\2\2\2!\2\3\u009b\u00a5\u00af\u00b4"+
-    "\u017c\u017f\u0186\u0189\u0190\u0193\u0196\u019d\u01a0\u01a6\u01a8\u01ac"+
-    "\u01b1\u01b3\u01b6\u01be\u01c0\u01ca\u01cc\u01d0\u01e8\u01f1\u01f9\u01fc"+
-    "\u0204\5\b\2\2\4\3\2\4\2\2";
+    "I\tI\4J\tJ\4K\tK\4L\tL\3\2\6\2\u009c\n\2\r\2\16\2\u009d\3\2\3\2\3\3\3"+
+    "\3\3\3\3\3\7\3\u00a6\n\3\f\3\16\3\u00a9\13\3\3\3\3\3\3\3\3\3\3\3\7\3\u00b0"+
+    "\n\3\f\3\16\3\u00b3\13\3\3\3\3\3\5\3\u00b7\n\3\3\3\3\3\3\4\3\4\3\5\3\5"+
+    "\3\6\3\6\3\7\3\7\3\b\3\b\3\t\3\t\3\n\3\n\3\n\3\n\3\13\3\13\3\f\3\f\3\r"+
+    "\3\r\3\r\3\16\3\16\3\16\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3\17\3\20\3"+
+    "\20\3\20\3\21\3\21\3\21\3\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3"+
+    "\22\3\23\3\23\3\23\3\23\3\23\3\23\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3"+
+    "\25\3\25\3\25\3\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\27\3\27\3\27\3"+
+    "\30\3\30\3\30\3\30\3\30\3\30\3\31\3\31\3\32\3\32\3\33\3\33\3\34\3\34\3"+
+    "\35\3\35\3\36\3\36\3\37\3\37\3 \3 \3 \3!\3!\3!\3\"\3\"\3\"\3\"\3#\3#\3"+
+    "$\3$\3$\3%\3%\3&\3&\3&\3\'\3\'\3\'\3(\3(\3(\3(\3)\3)\3)\3*\3*\3*\3*\3"+
+    "+\3+\3,\3,\3-\3-\3.\3.\3.\3/\3/\3/\3\60\3\60\3\61\3\61\3\62\3\62\3\62"+
+    "\3\63\3\63\3\63\3\64\3\64\3\64\3\65\3\65\3\66\3\66\3\66\3\67\3\67\3\67"+
+    "\38\38\38\39\39\39\3:\3:\3:\3;\3;\3;\3<\3<\3<\3=\3=\3=\3>\3>\3>\3>\3?"+
+    "\3?\3?\3?\3@\3@\3@\3@\3@\3A\3A\6A\u0180\nA\rA\16A\u0181\3A\5A\u0185\n"+
+    "A\3B\3B\3B\6B\u018a\nB\rB\16B\u018b\3B\5B\u018f\nB\3C\3C\3C\7C\u0194\n"+
+    "C\fC\16C\u0197\13C\5C\u0199\nC\3C\5C\u019c\nC\3D\3D\3D\7D\u01a1\nD\fD"+
+    "\16D\u01a4\13D\5D\u01a6\nD\3D\3D\6D\u01aa\nD\rD\16D\u01ab\5D\u01ae\nD"+
+    "\3D\3D\5D\u01b2\nD\3D\6D\u01b5\nD\rD\16D\u01b6\5D\u01b9\nD\3D\5D\u01bc"+
+    "\nD\3E\3E\3E\3E\3E\3E\7E\u01c4\nE\fE\16E\u01c7\13E\3E\3E\3E\3E\3E\3E\3"+
+    "E\7E\u01d0\nE\fE\16E\u01d3\13E\3E\5E\u01d6\nE\3F\3F\3F\3F\3F\3G\3G\3G"+
+    "\3G\3G\3G\3H\3H\3H\3H\3H\3I\3I\3I\3I\7I\u01ec\nI\fI\16I\u01ef\13I\3I\3"+
+    "I\3J\3J\7J\u01f5\nJ\fJ\16J\u01f8\13J\3K\3K\3K\7K\u01fd\nK\fK\16K\u0200"+
+    "\13K\5K\u0202\nK\3K\3K\3L\3L\7L\u0208\nL\fL\16L\u020b\13L\3L\3L\6\u00a7"+
+    "\u00b1\u01c5\u01d1\2M\4\3\6\4\b\5\n\6\f\7\16\b\20\t\22\n\24\13\26\f\30"+
+    "\r\32\16\34\17\36\20 \21\"\22$\23&\24(\25*\26,\27.\30\60\31\62\32\64\33"+
+    "\66\348\35:\36<\37> @!B\"D#F$H%J&L\'N(P)R*T+V,X-Z.\\/^\60`\61b\62d\63"+
+    "f\64h\65j\66l\67n8p9r:t;v<x=z>|?~@\u0080A\u0082B\u0084C\u0086D\u0088E"+
+    "\u008aF\u008cG\u008eH\u0090I\u0092J\u0094K\u0096L\u0098M\4\2\3\21\5\2"+
+    "\13\f\17\17\"\"\4\2\f\f\17\17\3\2\629\4\2NNnn\4\2ZZzz\5\2\62;CHch\3\2"+
+    "\63;\3\2\62;\b\2FFHHNNffhhnn\4\2GGgg\4\2--//\4\2HHhh\4\2$$^^\5\2C\\aa"+
+    "c|\6\2\62;C\\aac|\u022b\2\4\3\2\2\2\2\6\3\2\2\2\2\b\3\2\2\2\2\n\3\2\2"+
+    "\2\2\f\3\2\2\2\2\16\3\2\2\2\2\20\3\2\2\2\2\22\3\2\2\2\2\24\3\2\2\2\2\26"+
+    "\3\2\2\2\2\30\3\2\2\2\2\32\3\2\2\2\2\34\3\2\2\2\2\36\3\2\2\2\2 \3\2\2"+
+    "\2\2\"\3\2\2\2\2$\3\2\2\2\2&\3\2\2\2\2(\3\2\2\2\2*\3\2\2\2\2,\3\2\2\2"+
+    "\2.\3\2\2\2\2\60\3\2\2\2\2\62\3\2\2\2\2\64\3\2\2\2\2\66\3\2\2\2\28\3\2"+
+    "\2\2\2:\3\2\2\2\2<\3\2\2\2\2>\3\2\2\2\2@\3\2\2\2\2B\3\2\2\2\2D\3\2\2\2"+
+    "\2F\3\2\2\2\2H\3\2\2\2\2J\3\2\2\2\2L\3\2\2\2\2N\3\2\2\2\2P\3\2\2\2\2R"+
+    "\3\2\2\2\2T\3\2\2\2\2V\3\2\2\2\2X\3\2\2\2\2Z\3\2\2\2\2\\\3\2\2\2\2^\3"+
+    "\2\2\2\2`\3\2\2\2\2b\3\2\2\2\2d\3\2\2\2\2f\3\2\2\2\2h\3\2\2\2\2j\3\2\2"+
+    "\2\2l\3\2\2\2\2n\3\2\2\2\2p\3\2\2\2\2r\3\2\2\2\2t\3\2\2\2\2v\3\2\2\2\2"+
+    "x\3\2\2\2\2z\3\2\2\2\2|\3\2\2\2\2~\3\2\2\2\2\u0080\3\2\2\2\2\u0082\3\2"+
+    "\2\2\2\u0084\3\2\2\2\2\u0086\3\2\2\2\2\u0088\3\2\2\2\2\u008a\3\2\2\2\2"+
+    "\u008c\3\2\2\2\2\u008e\3\2\2\2\2\u0090\3\2\2\2\2\u0092\3\2\2\2\2\u0094"+
+    "\3\2\2\2\3\u0096\3\2\2\2\3\u0098\3\2\2\2\4\u009b\3\2\2\2\6\u00b6\3\2\2"+
+    "\2\b\u00ba\3\2\2\2\n\u00bc\3\2\2\2\f\u00be\3\2\2\2\16\u00c0\3\2\2\2\20"+
+    "\u00c2\3\2\2\2\22\u00c4\3\2\2\2\24\u00c6\3\2\2\2\26\u00ca\3\2\2\2\30\u00cc"+
+    "\3\2\2\2\32\u00ce\3\2\2\2\34\u00d1\3\2\2\2\36\u00d6\3\2\2\2 \u00dc\3\2"+
+    "\2\2\"\u00df\3\2\2\2$\u00e3\3\2\2\2&\u00ec\3\2\2\2(\u00f2\3\2\2\2*\u00f9"+
+    "\3\2\2\2,\u00fd\3\2\2\2.\u0101\3\2\2\2\60\u0107\3\2\2\2\62\u010d\3\2\2"+
+    "\2\64\u010f\3\2\2\2\66\u0111\3\2\2\28\u0113\3\2\2\2:\u0115\3\2\2\2<\u0117"+
+    "\3\2\2\2>\u0119\3\2\2\2@\u011b\3\2\2\2B\u011e\3\2\2\2D\u0121\3\2\2\2F"+
+    "\u0125\3\2\2\2H\u0127\3\2\2\2J\u012a\3\2\2\2L\u012c\3\2\2\2N\u012f\3\2"+
+    "\2\2P\u0132\3\2\2\2R\u0136\3\2\2\2T\u0139\3\2\2\2V\u013d\3\2\2\2X\u013f"+
+    "\3\2\2\2Z\u0141\3\2\2\2\\\u0143\3\2\2\2^\u0146\3\2\2\2`\u0149\3\2\2\2"+
+    "b\u014b\3\2\2\2d\u014d\3\2\2\2f\u0150\3\2\2\2h\u0153\3\2\2\2j\u0156\3"+
+    "\2\2\2l\u0158\3\2\2\2n\u015b\3\2\2\2p\u015e\3\2\2\2r\u0161\3\2\2\2t\u0164"+
+    "\3\2\2\2v\u0167\3\2\2\2x\u016a\3\2\2\2z\u016d\3\2\2\2|\u0170\3\2\2\2~"+
+    "\u0174\3\2\2\2\u0080\u0178\3\2\2\2\u0082\u017d\3\2\2\2\u0084\u0186\3\2"+
+    "\2\2\u0086\u0198\3\2\2\2\u0088\u01a5\3\2\2\2\u008a\u01d5\3\2\2\2\u008c"+
+    "\u01d7\3\2\2\2\u008e\u01dc\3\2\2\2\u0090\u01e2\3\2\2\2\u0092\u01e7\3\2"+
+    "\2\2\u0094\u01f2\3\2\2\2\u0096\u0201\3\2\2\2\u0098\u0205\3\2\2\2\u009a"+
+    "\u009c\t\2\2\2\u009b\u009a\3\2\2\2\u009c\u009d\3\2\2\2\u009d\u009b\3\2"+
+    "\2\2\u009d\u009e\3\2\2\2\u009e\u009f\3\2\2\2\u009f\u00a0\b\2\2\2\u00a0"+
+    "\5\3\2\2\2\u00a1\u00a2\7\61\2\2\u00a2\u00a3\7\61\2\2\u00a3\u00a7\3\2\2"+
+    "\2\u00a4\u00a6\13\2\2\2\u00a5\u00a4\3\2\2\2\u00a6\u00a9\3\2\2\2\u00a7"+
+    "\u00a8\3\2\2\2\u00a7\u00a5\3\2\2\2\u00a8\u00aa\3\2\2\2\u00a9\u00a7\3\2"+
+    "\2\2\u00aa\u00b7\t\3\2\2\u00ab\u00ac\7\61\2\2\u00ac\u00ad\7,\2\2\u00ad"+
+    "\u00b1\3\2\2\2\u00ae\u00b0\13\2\2\2\u00af\u00ae\3\2\2\2\u00b0\u00b3\3"+
+    "\2\2\2\u00b1\u00b2\3\2\2\2\u00b1\u00af\3\2\2\2\u00b2\u00b4\3\2\2\2\u00b3"+
+    "\u00b1\3\2\2\2\u00b4\u00b5\7,\2\2\u00b5\u00b7\7\61\2\2\u00b6\u00a1\3\2"+
+    "\2\2\u00b6\u00ab\3\2\2\2\u00b7\u00b8\3\2\2\2\u00b8\u00b9\b\3\2\2\u00b9"+
+    "\7\3\2\2\2\u00ba\u00bb\7}\2\2\u00bb\t\3\2\2\2\u00bc\u00bd\7\177\2\2\u00bd"+
+    "\13\3\2\2\2\u00be\u00bf\7]\2\2\u00bf\r\3\2\2\2\u00c0\u00c1\7_\2\2\u00c1"+
+    "\17\3\2\2\2\u00c2\u00c3\7*\2\2\u00c3\21\3\2\2\2\u00c4\u00c5\7+\2\2\u00c5"+
+    "\23\3\2\2\2\u00c6\u00c7\7\60\2\2\u00c7\u00c8\3\2\2\2\u00c8\u00c9\b\n\3"+
+    "\2\u00c9\25\3\2\2\2\u00ca\u00cb\7.\2\2\u00cb\27\3\2\2\2\u00cc\u00cd\7"+
+    "=\2\2\u00cd\31\3\2\2\2\u00ce\u00cf\7k\2\2\u00cf\u00d0\7h\2\2\u00d0\33"+
+    "\3\2\2\2\u00d1\u00d2\7g\2\2\u00d2\u00d3\7n\2\2\u00d3\u00d4\7u\2\2\u00d4"+
+    "\u00d5\7g\2\2\u00d5\35\3\2\2\2\u00d6\u00d7\7y\2\2\u00d7\u00d8\7j\2\2\u00d8"+
+    "\u00d9\7k\2\2\u00d9\u00da\7n\2\2\u00da\u00db\7g\2\2\u00db\37\3\2\2\2\u00dc"+
+    "\u00dd\7f\2\2\u00dd\u00de\7q\2\2\u00de!\3\2\2\2\u00df\u00e0\7h\2\2\u00e0"+
+    "\u00e1\7q\2\2\u00e1\u00e2\7t\2\2\u00e2#\3\2\2\2\u00e3\u00e4\7e\2\2\u00e4"+
+    "\u00e5\7q\2\2\u00e5\u00e6\7p\2\2\u00e6\u00e7\7v\2\2\u00e7\u00e8\7k\2\2"+
+    "\u00e8\u00e9\7p\2\2\u00e9\u00ea\7w\2\2\u00ea\u00eb\7g\2\2\u00eb%\3\2\2"+
+    "\2\u00ec\u00ed\7d\2\2\u00ed\u00ee\7t\2\2\u00ee\u00ef\7g\2\2\u00ef\u00f0"+
+    "\7c\2\2\u00f0\u00f1\7m\2\2\u00f1\'\3\2\2\2\u00f2\u00f3\7t\2\2\u00f3\u00f4"+
+    "\7g\2\2\u00f4\u00f5\7v\2\2\u00f5\u00f6\7w\2\2\u00f6\u00f7\7t\2\2\u00f7"+
+    "\u00f8\7p\2\2\u00f8)\3\2\2\2\u00f9\u00fa\7p\2\2\u00fa\u00fb\7g\2\2\u00fb"+
+    "\u00fc\7y\2\2\u00fc+\3\2\2\2\u00fd\u00fe\7v\2\2\u00fe\u00ff\7t\2\2\u00ff"+
+    "\u0100\7{\2\2\u0100-\3\2\2\2\u0101\u0102\7e\2\2\u0102\u0103\7c\2\2\u0103"+
+    "\u0104\7v\2\2\u0104\u0105\7e\2\2\u0105\u0106\7j\2\2\u0106/\3\2\2\2\u0107"+
+    "\u0108\7v\2\2\u0108\u0109\7j\2\2\u0109\u010a\7t\2\2\u010a\u010b\7q\2\2"+
+    "\u010b\u010c\7y\2\2\u010c\61\3\2\2\2\u010d\u010e\7#\2\2\u010e\63\3\2\2"+
+    "\2\u010f\u0110\7\u0080\2\2\u0110\65\3\2\2\2\u0111\u0112\7,\2\2\u0112\67"+
+    "\3\2\2\2\u0113\u0114\7\61\2\2\u01149\3\2\2\2\u0115\u0116\7\'\2\2\u0116"+
+    ";\3\2\2\2\u0117\u0118\7-\2\2\u0118=\3\2\2\2\u0119\u011a\7/\2\2\u011a?"+
+    "\3\2\2\2\u011b\u011c\7>\2\2\u011c\u011d\7>\2\2\u011dA\3\2\2\2\u011e\u011f"+
+    "\7@\2\2\u011f\u0120\7@\2\2\u0120C\3\2\2\2\u0121\u0122\7@\2\2\u0122\u0123"+
+    "\7@\2\2\u0123\u0124\7@\2\2\u0124E\3\2\2\2\u0125\u0126\7>\2\2\u0126G\3"+
+    "\2\2\2\u0127\u0128\7>\2\2\u0128\u0129\7?\2\2\u0129I\3\2\2\2\u012a\u012b"+
+    "\7@\2\2\u012bK\3\2\2\2\u012c\u012d\7@\2\2\u012d\u012e\7?\2\2\u012eM\3"+
+    "\2\2\2\u012f\u0130\7?\2\2\u0130\u0131\7?\2\2\u0131O\3\2\2\2\u0132\u0133"+
+    "\7?\2\2\u0133\u0134\7?\2\2\u0134\u0135\7?\2\2\u0135Q\3\2\2\2\u0136\u0137"+
+    "\7#\2\2\u0137\u0138\7?\2\2\u0138S\3\2\2\2\u0139\u013a\7#\2\2\u013a\u013b"+
+    "\7?\2\2\u013b\u013c\7?\2\2\u013cU\3\2\2\2\u013d\u013e\7(\2\2\u013eW\3"+
+    "\2\2\2\u013f\u0140\7`\2\2\u0140Y\3\2\2\2\u0141\u0142\7~\2\2\u0142[\3\2"+
+    "\2\2\u0143\u0144\7(\2\2\u0144\u0145\7(\2\2\u0145]\3\2\2\2\u0146\u0147"+
+    "\7~\2\2\u0147\u0148\7~\2\2\u0148_\3\2\2\2\u0149\u014a\7A\2\2\u014aa\3"+
+    "\2\2\2\u014b\u014c\7<\2\2\u014cc\3\2\2\2\u014d\u014e\7<\2\2\u014e\u014f"+
+    "\7<\2\2\u014fe\3\2\2\2\u0150\u0151\7-\2\2\u0151\u0152\7-\2\2\u0152g\3"+
+    "\2\2\2\u0153\u0154\7/\2\2\u0154\u0155\7/\2\2\u0155i\3\2\2\2\u0156\u0157"+
+    "\7?\2\2\u0157k\3\2\2\2\u0158\u0159\7-\2\2\u0159\u015a\7?\2\2\u015am\3"+
+    "\2\2\2\u015b\u015c\7/\2\2\u015c\u015d\7?\2\2\u015do\3\2\2\2\u015e\u015f"+
+    "\7,\2\2\u015f\u0160\7?\2\2\u0160q\3\2\2\2\u0161\u0162\7\61\2\2\u0162\u0163"+
+    "\7?\2\2\u0163s\3\2\2\2\u0164\u0165\7\'\2\2\u0165\u0166\7?\2\2\u0166u\3"+
+    "\2\2\2\u0167\u0168\7(\2\2\u0168\u0169\7?\2\2\u0169w\3\2\2\2\u016a\u016b"+
+    "\7`\2\2\u016b\u016c\7?\2\2\u016cy\3\2\2\2\u016d\u016e\7~\2\2\u016e\u016f"+
+    "\7?\2\2\u016f{\3\2\2\2\u0170\u0171\7>\2\2\u0171\u0172\7>\2\2\u0172\u0173"+
+    "\7?\2\2\u0173}\3\2\2\2\u0174\u0175\7@\2\2\u0175\u0176\7@\2\2\u0176\u0177"+
+    "\7?\2\2\u0177\177\3\2\2\2\u0178\u0179\7@\2\2\u0179\u017a\7@\2\2\u017a"+
+    "\u017b\7@\2\2\u017b\u017c\7?\2\2\u017c\u0081\3\2\2\2\u017d\u017f\7\62"+
+    "\2\2\u017e\u0180\t\4\2\2\u017f\u017e\3\2\2\2\u0180\u0181\3\2\2\2\u0181"+
+    "\u017f\3\2\2\2\u0181\u0182\3\2\2\2\u0182\u0184\3\2\2\2\u0183\u0185\t\5"+
+    "\2\2\u0184\u0183\3\2\2\2\u0184\u0185\3\2\2\2\u0185\u0083\3\2\2\2\u0186"+
+    "\u0187\7\62\2\2\u0187\u0189\t\6\2\2\u0188\u018a\t\7\2\2\u0189\u0188\3"+
+    "\2\2\2\u018a\u018b\3\2\2\2\u018b\u0189\3\2\2\2\u018b\u018c\3\2\2\2\u018c"+
+    "\u018e\3\2\2\2\u018d\u018f\t\5\2\2\u018e\u018d\3\2\2\2\u018e\u018f\3\2"+
+    "\2\2\u018f\u0085\3\2\2\2\u0190\u0199\7\62\2\2\u0191\u0195\t\b\2\2\u0192"+
+    "\u0194\t\t\2\2\u0193\u0192\3\2\2\2\u0194\u0197\3\2\2\2\u0195\u0193\3\2"+
+    "\2\2\u0195\u0196\3\2\2\2\u0196\u0199\3\2\2\2\u0197\u0195\3\2\2\2\u0198"+
+    "\u0190\3\2\2\2\u0198\u0191\3\2\2\2\u0199\u019b\3\2\2\2\u019a\u019c\t\n"+
+    "\2\2\u019b\u019a\3\2\2\2\u019b\u019c\3\2\2\2\u019c\u0087\3\2\2\2\u019d"+
+    "\u01a6\7\62\2\2\u019e\u01a2\t\b\2\2\u019f\u01a1\t\t\2\2\u01a0\u019f\3"+
+    "\2\2\2\u01a1\u01a4\3\2\2\2\u01a2\u01a0\3\2\2\2\u01a2\u01a3\3\2\2\2\u01a3"+
+    "\u01a6\3\2\2\2\u01a4\u01a2\3\2\2\2\u01a5\u019d\3\2\2\2\u01a5\u019e\3\2"+
+    "\2\2\u01a6\u01ad\3\2\2\2\u01a7\u01a9\5\24\n\2\u01a8\u01aa\t\t\2\2\u01a9"+
+    "\u01a8\3\2\2\2\u01aa\u01ab\3\2\2\2\u01ab\u01a9\3\2\2\2\u01ab\u01ac\3\2"+
+    "\2\2\u01ac\u01ae\3\2\2\2\u01ad\u01a7\3\2\2\2\u01ad\u01ae\3\2\2\2\u01ae"+
+    "\u01b8\3\2\2\2\u01af\u01b1\t\13\2\2\u01b0\u01b2\t\f\2\2\u01b1\u01b0\3"+
+    "\2\2\2\u01b1\u01b2\3\2\2\2\u01b2\u01b4\3\2\2\2\u01b3\u01b5\t\t\2\2\u01b4"+
+    "\u01b3\3\2\2\2\u01b5\u01b6\3\2\2\2\u01b6\u01b4\3\2\2\2\u01b6\u01b7\3\2"+
+    "\2\2\u01b7\u01b9\3\2\2\2\u01b8\u01af\3\2\2\2\u01b8\u01b9\3\2\2\2\u01b9"+
+    "\u01bb\3\2\2\2\u01ba\u01bc\t\r\2\2\u01bb\u01ba\3\2\2\2\u01bb\u01bc\3\2"+
+    "\2\2\u01bc\u0089\3\2\2\2\u01bd\u01c5\7$\2\2\u01be\u01bf\7^\2\2\u01bf\u01c4"+
+    "\7$\2\2\u01c0\u01c1\7^\2\2\u01c1\u01c4\7^\2\2\u01c2\u01c4\n\16\2\2\u01c3"+
+    "\u01be\3\2\2\2\u01c3\u01c0\3\2\2\2\u01c3\u01c2\3\2\2\2\u01c4\u01c7\3\2"+
+    "\2\2\u01c5\u01c6\3\2\2\2\u01c5\u01c3\3\2\2\2\u01c6\u01c8\3\2\2\2\u01c7"+
+    "\u01c5\3\2\2\2\u01c8\u01d6\7$\2\2\u01c9\u01d1\7)\2\2\u01ca\u01cb\7^\2"+
+    "\2\u01cb\u01d0\7)\2\2\u01cc\u01cd\7^\2\2\u01cd\u01d0\7^\2\2\u01ce\u01d0"+
+    "\n\16\2\2\u01cf\u01ca\3\2\2\2\u01cf\u01cc\3\2\2\2\u01cf\u01ce\3\2\2\2"+
+    "\u01d0\u01d3\3\2\2\2\u01d1\u01d2\3\2\2\2\u01d1\u01cf\3\2\2\2\u01d2\u01d4"+
+    "\3\2\2\2\u01d3\u01d1\3\2\2\2\u01d4\u01d6\7)\2\2\u01d5\u01bd\3\2\2\2\u01d5"+
+    "\u01c9\3\2\2\2\u01d6\u008b\3\2\2\2\u01d7\u01d8\7v\2\2\u01d8\u01d9\7t\2"+
+    "\2\u01d9\u01da\7w\2\2\u01da\u01db\7g\2\2\u01db\u008d\3\2\2\2\u01dc\u01dd"+
+    "\7h\2\2\u01dd\u01de\7c\2\2\u01de\u01df\7n\2\2\u01df\u01e0\7u\2\2\u01e0"+
+    "\u01e1\7g\2\2\u01e1\u008f\3\2\2\2\u01e2\u01e3\7p\2\2\u01e3\u01e4\7w\2"+
+    "\2\u01e4\u01e5\7n\2\2\u01e5\u01e6\7n\2\2\u01e6\u0091\3\2\2\2\u01e7\u01ed"+
+    "\5\u0094J\2\u01e8\u01e9\5\24\n\2\u01e9\u01ea\5\u0094J\2\u01ea\u01ec\3"+
+    "\2\2\2\u01eb\u01e8\3\2\2\2\u01ec\u01ef\3\2\2\2\u01ed\u01eb\3\2\2\2\u01ed"+
+    "\u01ee\3\2\2\2\u01ee\u01f0\3\2\2\2\u01ef\u01ed\3\2\2\2\u01f0\u01f1\6I"+
+    "\2\2\u01f1\u0093\3\2\2\2\u01f2\u01f6\t\17\2\2\u01f3\u01f5\t\20\2\2\u01f4"+
+    "\u01f3\3\2\2\2\u01f5\u01f8\3\2\2\2\u01f6\u01f4\3\2\2\2\u01f6\u01f7\3\2"+
+    "\2\2\u01f7\u0095\3\2\2\2\u01f8\u01f6\3\2\2\2\u01f9\u0202\7\62\2\2\u01fa"+
+    "\u01fe\t\b\2\2\u01fb\u01fd\t\t\2\2\u01fc\u01fb\3\2\2\2\u01fd\u0200\3\2"+
+    "\2\2\u01fe\u01fc\3\2\2\2\u01fe\u01ff\3\2\2\2\u01ff\u0202\3\2\2\2\u0200"+
+    "\u01fe\3\2\2\2\u0201\u01f9\3\2\2\2\u0201\u01fa\3\2\2\2\u0202\u0203\3\2"+
+    "\2\2\u0203\u0204\bK\4\2\u0204\u0097\3\2\2\2\u0205\u0209\t\17\2\2\u0206"+
+    "\u0208\t\20\2\2\u0207\u0206\3\2\2\2\u0208\u020b\3\2\2\2\u0209\u0207\3"+
+    "\2\2\2\u0209\u020a\3\2\2\2\u020a\u020c\3\2\2\2\u020b\u0209\3\2\2\2\u020c"+
+    "\u020d\bL\4\2\u020d\u0099\3\2\2\2!\2\3\u009d\u00a7\u00b1\u00b6\u0181\u0184"+
+    "\u018b\u018e\u0195\u0198\u019b\u01a2\u01a5\u01ab\u01ad\u01b1\u01b6\u01b8"+
+    "\u01bb\u01c3\u01c5\u01cf\u01d1\u01d5\u01ed\u01f6\u01fe\u0201\u0209\5\b"+
+    "\2\2\4\3\2\4\2\2";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
index 11b38dc9742..98eb16732fd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
@@ -22,20 +22,23 @@ class PainlessParser extends Parser {
     BREAK=18, RETURN=19, NEW=20, TRY=21, CATCH=22, THROW=23, BOOLNOT=24, BWNOT=25, 
     MUL=26, DIV=27, REM=28, ADD=29, SUB=30, LSH=31, RSH=32, USH=33, LT=34, 
     LTE=35, GT=36, GTE=37, EQ=38, EQR=39, NE=40, NER=41, BWAND=42, XOR=43, 
-    BWOR=44, BOOLAND=45, BOOLOR=46, COND=47, COLON=48, INCR=49, DECR=50, ASSIGN=51, 
-    AADD=52, ASUB=53, AMUL=54, ADIV=55, AREM=56, AAND=57, AXOR=58, AOR=59, 
-    ALSH=60, ARSH=61, AUSH=62, OCTAL=63, HEX=64, INTEGER=65, DECIMAL=66, STRING=67, 
-    TRUE=68, FALSE=69, NULL=70, TYPE=71, ID=72, DOTINTEGER=73, DOTID=74;
+    BWOR=44, BOOLAND=45, BOOLOR=46, COND=47, COLON=48, REF=49, INCR=50, DECR=51, 
+    ASSIGN=52, AADD=53, ASUB=54, AMUL=55, ADIV=56, AREM=57, AAND=58, AXOR=59, 
+    AOR=60, ALSH=61, ARSH=62, AUSH=63, OCTAL=64, HEX=65, INTEGER=66, DECIMAL=67, 
+    STRING=68, TRUE=69, FALSE=70, NULL=71, TYPE=72, ID=73, DOTINTEGER=74, 
+    DOTID=75;
   public static final int
     RULE_source = 0, RULE_statement = 1, RULE_trailer = 2, RULE_block = 3, 
     RULE_empty = 4, RULE_initializer = 5, RULE_afterthought = 6, RULE_declaration = 7, 
-    RULE_decltype = 8, RULE_declvar = 9, RULE_trap = 10, RULE_delimiter = 11, 
-    RULE_expression = 12, RULE_unary = 13, RULE_chain = 14, RULE_primary = 15, 
-    RULE_secondary = 16, RULE_dot = 17, RULE_brace = 18, RULE_arguments = 19;
+    RULE_decltype = 8, RULE_funcref = 9, RULE_declvar = 10, RULE_trap = 11, 
+    RULE_delimiter = 12, RULE_expression = 13, RULE_unary = 14, RULE_chain = 15, 
+    RULE_primary = 16, RULE_secondary = 17, RULE_dot = 18, RULE_brace = 19, 
+    RULE_arguments = 20, RULE_argument = 21;
   public static final String[] ruleNames = {
     "source", "statement", "trailer", "block", "empty", "initializer", "afterthought", 
-    "declaration", "decltype", "declvar", "trap", "delimiter", "expression", 
-    "unary", "chain", "primary", "secondary", "dot", "brace", "arguments"
+    "declaration", "decltype", "funcref", "declvar", "trap", "delimiter", 
+    "expression", "unary", "chain", "primary", "secondary", "dot", "brace", 
+    "arguments", "argument"
   };
 
   private static final String[] _LITERAL_NAMES = {
@@ -44,9 +47,9 @@ class PainlessParser extends Parser {
     "'return'", "'new'", "'try'", "'catch'", "'throw'", "'!'", "'~'", "'*'", 
     "'/'", "'%'", "'+'", "'-'", "'<<'", "'>>'", "'>>>'", "'<'", "'<='", "'>'", 
     "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", "'^'", "'|'", "'&&'", 
-    "'||'", "'?'", "':'", "'++'", "'--'", "'='", "'+='", "'-='", "'*='", "'/='", 
-    "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", "'>>>='", null, null, 
-    null, null, null, "'true'", "'false'", "'null'"
+    "'||'", "'?'", "':'", "'::'", "'++'", "'--'", "'='", "'+='", "'-='", "'*='", 
+    "'/='", "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", "'>>>='", null, 
+    null, null, null, null, "'true'", "'false'", "'null'"
   };
   private static final String[] _SYMBOLIC_NAMES = {
     null, "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", 
@@ -54,10 +57,10 @@ class PainlessParser extends Parser {
     "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "BOOLNOT", "BWNOT", 
     "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", "LTE", "GT", 
     "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", "BOOLAND", "BOOLOR", 
-    "COND", "COLON", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", 
-    "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", 
-    "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "TYPE", "ID", 
-    "DOTINTEGER", "DOTID"
+    "COND", "COLON", "REF", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", 
+    "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", 
+    "HEX", "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "TYPE", 
+    "ID", "DOTINTEGER", "DOTID"
   };
   public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
 
@@ -134,23 +137,23 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(43);
+      setState(47);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,0,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(40);
+          setState(44);
           statement();
           }
           } 
         }
-        setState(45);
+        setState(49);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,0,_ctx);
       }
-      setState(46);
+      setState(50);
       match(EOF);
       }
     }
@@ -374,35 +377,35 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 2, RULE_statement);
     try {
       int _alt;
-      setState(117);
+      setState(121);
       switch ( getInterpreter().adaptivePredict(_input,8,_ctx) ) {
       case 1:
         _localctx = new IfContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(48);
-        match(IF);
-        setState(49);
-        match(LP);
-        setState(50);
-        expression(0);
-        setState(51);
-        match(RP);
         setState(52);
-        trailer();
+        match(IF);
+        setState(53);
+        match(LP);
+        setState(54);
+        expression(0);
+        setState(55);
+        match(RP);
         setState(56);
+        trailer();
+        setState(60);
         switch ( getInterpreter().adaptivePredict(_input,1,_ctx) ) {
         case 1:
           {
-          setState(53);
+          setState(57);
           match(ELSE);
-          setState(54);
+          setState(58);
           trailer();
           }
           break;
         case 2:
           {
-          setState(55);
+          setState(59);
           if (!( _input.LA(1) != ELSE )) throw new FailedPredicateException(this, " _input.LA(1) != ELSE ");
           }
           break;
@@ -413,25 +416,25 @@ class PainlessParser extends Parser {
         _localctx = new WhileContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(58);
+        setState(62);
         match(WHILE);
-        setState(59);
+        setState(63);
         match(LP);
-        setState(60);
-        expression(0);
-        setState(61);
-        match(RP);
         setState(64);
+        expression(0);
+        setState(65);
+        match(RP);
+        setState(68);
         switch ( getInterpreter().adaptivePredict(_input,2,_ctx) ) {
         case 1:
           {
-          setState(62);
+          setState(66);
           trailer();
           }
           break;
         case 2:
           {
-          setState(63);
+          setState(67);
           empty();
           }
           break;
@@ -442,19 +445,19 @@ class PainlessParser extends Parser {
         _localctx = new DoContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(66);
-        match(DO);
-        setState(67);
-        block();
-        setState(68);
-        match(WHILE);
-        setState(69);
-        match(LP);
         setState(70);
-        expression(0);
+        match(DO);
         setState(71);
-        match(RP);
+        block();
         setState(72);
+        match(WHILE);
+        setState(73);
+        match(LP);
+        setState(74);
+        expression(0);
+        setState(75);
+        match(RP);
+        setState(76);
         delimiter();
         }
         break;
@@ -462,54 +465,54 @@ class PainlessParser extends Parser {
         _localctx = new ForContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(74);
+        setState(78);
         match(FOR);
-        setState(75);
+        setState(79);
         match(LP);
-        setState(77);
+        setState(81);
         switch ( getInterpreter().adaptivePredict(_input,3,_ctx) ) {
-        case 1:
-          {
-          setState(76);
-          initializer();
-          }
-          break;
-        }
-        setState(79);
-        match(SEMICOLON);
-        setState(81);
-        switch ( getInterpreter().adaptivePredict(_input,4,_ctx) ) {
         case 1:
           {
           setState(80);
-          expression(0);
+          initializer();
           }
           break;
         }
         setState(83);
         match(SEMICOLON);
         setState(85);
-        switch ( getInterpreter().adaptivePredict(_input,5,_ctx) ) {
+        switch ( getInterpreter().adaptivePredict(_input,4,_ctx) ) {
         case 1:
           {
           setState(84);
-          afterthought();
+          expression(0);
           }
           break;
         }
         setState(87);
-        match(RP);
-        setState(90);
-        switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) {
+        match(SEMICOLON);
+        setState(89);
+        switch ( getInterpreter().adaptivePredict(_input,5,_ctx) ) {
         case 1:
           {
           setState(88);
+          afterthought();
+          }
+          break;
+        }
+        setState(91);
+        match(RP);
+        setState(94);
+        switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) {
+        case 1:
+          {
+          setState(92);
           trailer();
           }
           break;
         case 2:
           {
-          setState(89);
+          setState(93);
           empty();
           }
           break;
@@ -520,9 +523,9 @@ class PainlessParser extends Parser {
         _localctx = new DeclContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(92);
+        setState(96);
         declaration();
-        setState(93);
+        setState(97);
         delimiter();
         }
         break;
@@ -530,9 +533,9 @@ class PainlessParser extends Parser {
         _localctx = new ContinueContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(95);
+        setState(99);
         match(CONTINUE);
-        setState(96);
+        setState(100);
         delimiter();
         }
         break;
@@ -540,9 +543,9 @@ class PainlessParser extends Parser {
         _localctx = new BreakContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(97);
+        setState(101);
         match(BREAK);
-        setState(98);
+        setState(102);
         delimiter();
         }
         break;
@@ -550,11 +553,11 @@ class PainlessParser extends Parser {
         _localctx = new ReturnContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(99);
+        setState(103);
         match(RETURN);
-        setState(100);
+        setState(104);
         expression(0);
-        setState(101);
+        setState(105);
         delimiter();
         }
         break;
@@ -562,11 +565,11 @@ class PainlessParser extends Parser {
         _localctx = new TryContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(103);
+        setState(107);
         match(TRY);
-        setState(104);
+        setState(108);
         block();
-        setState(106); 
+        setState(110); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -574,7 +577,7 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(105);
+            setState(109);
             trap();
             }
             }
@@ -582,7 +585,7 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(108); 
+          setState(112); 
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,7,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
@@ -592,11 +595,11 @@ class PainlessParser extends Parser {
         _localctx = new ThrowContext(_localctx);
         enterOuterAlt(_localctx, 10);
         {
-        setState(110);
+        setState(114);
         match(THROW);
-        setState(111);
+        setState(115);
         expression(0);
-        setState(112);
+        setState(116);
         delimiter();
         }
         break;
@@ -604,9 +607,9 @@ class PainlessParser extends Parser {
         _localctx = new ExprContext(_localctx);
         enterOuterAlt(_localctx, 11);
         {
-        setState(114);
+        setState(118);
         expression(0);
-        setState(115);
+        setState(119);
         delimiter();
         }
         break;
@@ -645,19 +648,19 @@ class PainlessParser extends Parser {
     TrailerContext _localctx = new TrailerContext(_ctx, getState());
     enterRule(_localctx, 4, RULE_trailer);
     try {
-      setState(121);
+      setState(125);
       switch ( getInterpreter().adaptivePredict(_input,9,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(119);
+        setState(123);
         block();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(120);
+        setState(124);
         statement();
         }
         break;
@@ -701,25 +704,25 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(123);
-      match(LBRACK);
       setState(127);
+      match(LBRACK);
+      setState(131);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,10,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(124);
+          setState(128);
           statement();
           }
           } 
         }
-        setState(129);
+        setState(133);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,10,_ctx);
       }
-      setState(130);
+      setState(134);
       match(RBRACK);
       }
     }
@@ -753,7 +756,7 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(132);
+      setState(136);
       match(SEMICOLON);
       }
     }
@@ -790,19 +793,19 @@ class PainlessParser extends Parser {
     InitializerContext _localctx = new InitializerContext(_ctx, getState());
     enterRule(_localctx, 10, RULE_initializer);
     try {
-      setState(136);
+      setState(140);
       switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(134);
+        setState(138);
         declaration();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(135);
+        setState(139);
         expression(0);
         }
         break;
@@ -840,7 +843,7 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(138);
+      setState(142);
       expression(0);
       }
     }
@@ -887,23 +890,23 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(140);
+      setState(144);
       decltype();
-      setState(141);
+      setState(145);
       declvar();
-      setState(146);
+      setState(150);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==COMMA) {
         {
         {
-        setState(142);
+        setState(146);
         match(COMMA);
-        setState(143);
+        setState(147);
         declvar();
         }
         }
-        setState(148);
+        setState(152);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
@@ -948,21 +951,21 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(149);
+      setState(153);
       match(TYPE);
-      setState(154);
+      setState(158);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==LBRACE) {
         {
         {
-        setState(150);
+        setState(154);
         match(LBRACE);
-        setState(151);
+        setState(155);
         match(RBRACE);
         }
         }
-        setState(156);
+        setState(160);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
@@ -979,6 +982,46 @@ class PainlessParser extends Parser {
     return _localctx;
   }
 
+  public static class FuncrefContext extends ParserRuleContext {
+    public TerminalNode TYPE() { return getToken(PainlessParser.TYPE, 0); }
+    public TerminalNode REF() { return getToken(PainlessParser.REF, 0); }
+    public TerminalNode ID() { return getToken(PainlessParser.ID, 0); }
+    public FuncrefContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_funcref; }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitFuncref(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final FuncrefContext funcref() throws RecognitionException {
+    FuncrefContext _localctx = new FuncrefContext(_ctx, getState());
+    enterRule(_localctx, 18, RULE_funcref);
+    try {
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(161);
+      match(TYPE);
+      setState(162);
+      match(REF);
+      setState(163);
+      match(ID);
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
   public static class DeclvarContext extends ParserRuleContext {
     public TerminalNode ID() { return getToken(PainlessParser.ID, 0); }
     public TerminalNode ASSIGN() { return getToken(PainlessParser.ASSIGN, 0); }
@@ -998,20 +1041,20 @@ class PainlessParser extends Parser {
 
   public final DeclvarContext declvar() throws RecognitionException {
     DeclvarContext _localctx = new DeclvarContext(_ctx, getState());
-    enterRule(_localctx, 18, RULE_declvar);
+    enterRule(_localctx, 20, RULE_declvar);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(157);
+      setState(165);
       match(ID);
-      setState(160);
+      setState(168);
       _la = _input.LA(1);
       if (_la==ASSIGN) {
         {
-        setState(158);
+        setState(166);
         match(ASSIGN);
-        setState(159);
+        setState(167);
         expression(0);
         }
       }
@@ -1051,21 +1094,21 @@ class PainlessParser extends Parser {
 
   public final TrapContext trap() throws RecognitionException {
     TrapContext _localctx = new TrapContext(_ctx, getState());
-    enterRule(_localctx, 20, RULE_trap);
+    enterRule(_localctx, 22, RULE_trap);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(162);
+      setState(170);
       match(CATCH);
-      setState(163);
+      setState(171);
       match(LP);
-      setState(164);
+      setState(172);
       match(TYPE);
-      setState(165);
+      setState(173);
       match(ID);
-      setState(166);
+      setState(174);
       match(RP);
-      setState(167);
+      setState(175);
       block();
       }
     }
@@ -1096,12 +1139,12 @@ class PainlessParser extends Parser {
 
   public final DelimiterContext delimiter() throws RecognitionException {
     DelimiterContext _localctx = new DelimiterContext(_ctx, getState());
-    enterRule(_localctx, 22, RULE_delimiter);
+    enterRule(_localctx, 24, RULE_delimiter);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(169);
+      setState(177);
       _la = _input.LA(1);
       if ( !(_la==EOF || _la==SEMICOLON) ) {
       _errHandler.recoverInline(this);
@@ -1263,14 +1306,14 @@ class PainlessParser extends Parser {
     int _parentState = getState();
     ExpressionContext _localctx = new ExpressionContext(_ctx, _parentState);
     ExpressionContext _prevctx = _localctx;
-    int _startState = 24;
-    enterRecursionRule(_localctx, 24, RULE_expression, _p);
+    int _startState = 26;
+    enterRecursionRule(_localctx, 26, RULE_expression, _p);
     int _la;
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(180);
+      setState(188);
       switch ( getInterpreter().adaptivePredict(_input,15,_ctx) ) {
       case 1:
         {
@@ -1278,16 +1321,16 @@ class PainlessParser extends Parser {
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(172);
+        setState(180);
         chain(true);
-        setState(173);
+        setState(181);
         _la = _input.LA(1);
         if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ASSIGN) | (1L << AADD) | (1L << ASUB) | (1L << AMUL) | (1L << ADIV) | (1L << AREM) | (1L << AAND) | (1L << AXOR) | (1L << AOR) | (1L << ALSH) | (1L << ARSH) | (1L << AUSH))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(174);
+        setState(182);
         expression(1);
          ((AssignmentContext)_localctx).s =  false; 
         }
@@ -1297,14 +1340,14 @@ class PainlessParser extends Parser {
         _localctx = new SingleContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(177);
+        setState(185);
         ((SingleContext)_localctx).u = unary(false);
          ((SingleContext)_localctx).s =  ((SingleContext)_localctx).u.s; 
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(241);
+      setState(249);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,17,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -1312,22 +1355,22 @@ class PainlessParser extends Parser {
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(239);
+          setState(247);
           switch ( getInterpreter().adaptivePredict(_input,16,_ctx) ) {
           case 1:
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(182);
+            setState(190);
             if (!(precpred(_ctx, 12))) throw new FailedPredicateException(this, "precpred(_ctx, 12)");
-            setState(183);
+            setState(191);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << MUL) | (1L << DIV) | (1L << REM))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(184);
+            setState(192);
             expression(13);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1336,16 +1379,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(187);
+            setState(195);
             if (!(precpred(_ctx, 11))) throw new FailedPredicateException(this, "precpred(_ctx, 11)");
-            setState(188);
+            setState(196);
             _la = _input.LA(1);
             if ( !(_la==ADD || _la==SUB) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(189);
+            setState(197);
             expression(12);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1354,16 +1397,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(192);
+            setState(200);
             if (!(precpred(_ctx, 10))) throw new FailedPredicateException(this, "precpred(_ctx, 10)");
-            setState(193);
+            setState(201);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LSH) | (1L << RSH) | (1L << USH))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(194);
+            setState(202);
             expression(11);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1372,16 +1415,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(197);
+            setState(205);
             if (!(precpred(_ctx, 9))) throw new FailedPredicateException(this, "precpred(_ctx, 9)");
-            setState(198);
+            setState(206);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LT) | (1L << LTE) | (1L << GT) | (1L << GTE))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(199);
+            setState(207);
             expression(10);
              ((CompContext)_localctx).s =  false; 
             }
@@ -1390,16 +1433,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(202);
+            setState(210);
             if (!(precpred(_ctx, 8))) throw new FailedPredicateException(this, "precpred(_ctx, 8)");
-            setState(203);
+            setState(211);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << EQ) | (1L << EQR) | (1L << NE) | (1L << NER))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(204);
+            setState(212);
             expression(9);
              ((CompContext)_localctx).s =  false; 
             }
@@ -1408,11 +1451,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(207);
+            setState(215);
             if (!(precpred(_ctx, 7))) throw new FailedPredicateException(this, "precpred(_ctx, 7)");
-            setState(208);
+            setState(216);
             match(BWAND);
-            setState(209);
+            setState(217);
             expression(8);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1421,11 +1464,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(212);
+            setState(220);
             if (!(precpred(_ctx, 6))) throw new FailedPredicateException(this, "precpred(_ctx, 6)");
-            setState(213);
+            setState(221);
             match(XOR);
-            setState(214);
+            setState(222);
             expression(7);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1434,11 +1477,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(217);
+            setState(225);
             if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)");
-            setState(218);
+            setState(226);
             match(BWOR);
-            setState(219);
+            setState(227);
             expression(6);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1447,11 +1490,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(222);
+            setState(230);
             if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)");
-            setState(223);
+            setState(231);
             match(BOOLAND);
-            setState(224);
+            setState(232);
             expression(5);
              ((BoolContext)_localctx).s =  false; 
             }
@@ -1460,11 +1503,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(227);
+            setState(235);
             if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)");
-            setState(228);
+            setState(236);
             match(BOOLOR);
-            setState(229);
+            setState(237);
             expression(4);
              ((BoolContext)_localctx).s =  false; 
             }
@@ -1473,15 +1516,15 @@ class PainlessParser extends Parser {
             {
             _localctx = new ConditionalContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(232);
+            setState(240);
             if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
-            setState(233);
+            setState(241);
             match(COND);
-            setState(234);
+            setState(242);
             ((ConditionalContext)_localctx).e0 = expression(0);
-            setState(235);
+            setState(243);
             match(COLON);
-            setState(236);
+            setState(244);
             ((ConditionalContext)_localctx).e1 = expression(2);
              ((ConditionalContext)_localctx).s =  ((ConditionalContext)_localctx).e0.s && ((ConditionalContext)_localctx).e1.s; 
             }
@@ -1489,7 +1532,7 @@ class PainlessParser extends Parser {
           }
           } 
         }
-        setState(243);
+        setState(251);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,17,_ctx);
       }
@@ -1633,25 +1676,25 @@ class PainlessParser extends Parser {
 
   public final UnaryContext unary(boolean c) throws RecognitionException {
     UnaryContext _localctx = new UnaryContext(_ctx, getState(), c);
-    enterRule(_localctx, 26, RULE_unary);
+    enterRule(_localctx, 28, RULE_unary);
     int _la;
     try {
-      setState(273);
+      setState(281);
       switch ( getInterpreter().adaptivePredict(_input,18,_ctx) ) {
       case 1:
         _localctx = new PreContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(244);
+        setState(252);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(245);
+        setState(253);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(246);
+        setState(254);
         chain(true);
         }
         break;
@@ -1659,11 +1702,11 @@ class PainlessParser extends Parser {
         _localctx = new PostContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(247);
+        setState(255);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(248);
+        setState(256);
         chain(true);
-        setState(249);
+        setState(257);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
@@ -1676,9 +1719,9 @@ class PainlessParser extends Parser {
         _localctx = new ReadContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(251);
+        setState(259);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(252);
+        setState(260);
         chain(false);
         }
         break;
@@ -1686,11 +1729,11 @@ class PainlessParser extends Parser {
         _localctx = new NumericContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(253);
+        setState(261);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(254);
+        setState(262);
         _la = _input.LA(1);
-        if ( !(((((_la - 63)) & ~0x3f) == 0 && ((1L << (_la - 63)) & ((1L << (OCTAL - 63)) | (1L << (HEX - 63)) | (1L << (INTEGER - 63)) | (1L << (DECIMAL - 63)))) != 0)) ) {
+        if ( !(((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (OCTAL - 64)) | (1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
@@ -1702,9 +1745,9 @@ class PainlessParser extends Parser {
         _localctx = new TrueContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(256);
+        setState(264);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(257);
+        setState(265);
         match(TRUE);
          ((TrueContext)_localctx).s =  false; 
         }
@@ -1713,9 +1756,9 @@ class PainlessParser extends Parser {
         _localctx = new FalseContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(259);
+        setState(267);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(260);
+        setState(268);
         match(FALSE);
          ((FalseContext)_localctx).s =  false; 
         }
@@ -1724,9 +1767,9 @@ class PainlessParser extends Parser {
         _localctx = new NullContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(262);
+        setState(270);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(263);
+        setState(271);
         match(NULL);
          ((NullContext)_localctx).s =  false; 
         }
@@ -1735,16 +1778,16 @@ class PainlessParser extends Parser {
         _localctx = new OperatorContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(265);
+        setState(273);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(266);
+        setState(274);
         _la = _input.LA(1);
         if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(267);
+        setState(275);
         unary(false);
         }
         break;
@@ -1752,13 +1795,13 @@ class PainlessParser extends Parser {
         _localctx = new CastContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(268);
+        setState(276);
         match(LP);
-        setState(269);
+        setState(277);
         decltype();
-        setState(270);
+        setState(278);
         match(RP);
-        setState(271);
+        setState(279);
         unary(_localctx.c);
         }
         break;
@@ -1864,30 +1907,30 @@ class PainlessParser extends Parser {
 
   public final ChainContext chain(boolean c) throws RecognitionException {
     ChainContext _localctx = new ChainContext(_ctx, getState(), c);
-    enterRule(_localctx, 28, RULE_chain);
+    enterRule(_localctx, 30, RULE_chain);
     try {
       int _alt;
-      setState(309);
+      setState(317);
       switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) {
       case 1:
         _localctx = new DynamicContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(275);
+        setState(283);
         ((DynamicContext)_localctx).p = primary(_localctx.c);
-        setState(279);
+        setState(287);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,19,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(276);
+            setState(284);
             secondary(((DynamicContext)_localctx).p.s);
             }
             } 
           }
-          setState(281);
+          setState(289);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,19,_ctx);
         }
@@ -1897,23 +1940,23 @@ class PainlessParser extends Parser {
         _localctx = new StaticContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(282);
+        setState(290);
         decltype();
-        setState(283);
+        setState(291);
         dot();
-        setState(287);
+        setState(295);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(284);
+            setState(292);
             secondary(true);
             }
             } 
           }
-          setState(289);
+          setState(297);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
         }
@@ -1923,11 +1966,11 @@ class PainlessParser extends Parser {
         _localctx = new NewarrayContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(290);
+        setState(298);
         match(NEW);
-        setState(291);
+        setState(299);
         match(TYPE);
-        setState(296); 
+        setState(304); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -1935,11 +1978,11 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(292);
+            setState(300);
             match(LBRACE);
-            setState(293);
+            setState(301);
             expression(0);
-            setState(294);
+            setState(302);
             match(RBRACE);
             }
             }
@@ -1947,29 +1990,29 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(298); 
+          setState(306); 
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
-        setState(307);
+        setState(315);
         switch ( getInterpreter().adaptivePredict(_input,23,_ctx) ) {
         case 1:
           {
-          setState(300);
+          setState(308);
           dot();
-          setState(304);
+          setState(312);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
           while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
             if ( _alt==1 ) {
               {
               {
-              setState(301);
+              setState(309);
               secondary(true);
               }
               } 
             }
-            setState(306);
+            setState(314);
             _errHandler.sync(this);
             _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
           }
@@ -2069,21 +2112,21 @@ class PainlessParser extends Parser {
 
   public final PrimaryContext primary(boolean c) throws RecognitionException {
     PrimaryContext _localctx = new PrimaryContext(_ctx, getState(), c);
-    enterRule(_localctx, 30, RULE_primary);
+    enterRule(_localctx, 32, RULE_primary);
     try {
-      setState(327);
+      setState(335);
       switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) {
       case 1:
         _localctx = new ExprprecContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(311);
+        setState(319);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(312);
+        setState(320);
         match(LP);
-        setState(313);
+        setState(321);
         ((ExprprecContext)_localctx).e = expression(0);
-        setState(314);
+        setState(322);
         match(RP);
          ((ExprprecContext)_localctx).s =  ((ExprprecContext)_localctx).e.s; 
         }
@@ -2092,13 +2135,13 @@ class PainlessParser extends Parser {
         _localctx = new ChainprecContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(317);
+        setState(325);
         if (!( _localctx.c )) throw new FailedPredicateException(this, " $c ");
-        setState(318);
+        setState(326);
         match(LP);
-        setState(319);
+        setState(327);
         unary(true);
-        setState(320);
+        setState(328);
         match(RP);
         }
         break;
@@ -2106,7 +2149,7 @@ class PainlessParser extends Parser {
         _localctx = new StringContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(322);
+        setState(330);
         match(STRING);
         }
         break;
@@ -2114,7 +2157,7 @@ class PainlessParser extends Parser {
         _localctx = new VariableContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(323);
+        setState(331);
         match(ID);
         }
         break;
@@ -2122,11 +2165,11 @@ class PainlessParser extends Parser {
         _localctx = new NewobjectContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(324);
+        setState(332);
         match(NEW);
-        setState(325);
+        setState(333);
         match(TYPE);
-        setState(326);
+        setState(334);
         arguments();
         }
         break;
@@ -2166,25 +2209,25 @@ class PainlessParser extends Parser {
 
   public final SecondaryContext secondary(boolean s) throws RecognitionException {
     SecondaryContext _localctx = new SecondaryContext(_ctx, getState(), s);
-    enterRule(_localctx, 32, RULE_secondary);
+    enterRule(_localctx, 34, RULE_secondary);
     try {
-      setState(333);
+      setState(341);
       switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(329);
+        setState(337);
         if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
-        setState(330);
+        setState(338);
         dot();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(331);
+        setState(339);
         if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
-        setState(332);
+        setState(340);
         brace();
         }
         break;
@@ -2239,20 +2282,20 @@ class PainlessParser extends Parser {
 
   public final DotContext dot() throws RecognitionException {
     DotContext _localctx = new DotContext(_ctx, getState());
-    enterRule(_localctx, 34, RULE_dot);
+    enterRule(_localctx, 36, RULE_dot);
     int _la;
     try {
-      setState(340);
+      setState(348);
       switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) {
       case 1:
         _localctx = new CallinvokeContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(335);
+        setState(343);
         match(DOT);
-        setState(336);
+        setState(344);
         match(DOTID);
-        setState(337);
+        setState(345);
         arguments();
         }
         break;
@@ -2260,9 +2303,9 @@ class PainlessParser extends Parser {
         _localctx = new FieldaccessContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(338);
+        setState(346);
         match(DOT);
-        setState(339);
+        setState(347);
         _la = _input.LA(1);
         if ( !(_la==DOTINTEGER || _la==DOTID) ) {
         _errHandler.recoverInline(this);
@@ -2311,16 +2354,16 @@ class PainlessParser extends Parser {
 
   public final BraceContext brace() throws RecognitionException {
     BraceContext _localctx = new BraceContext(_ctx, getState());
-    enterRule(_localctx, 36, RULE_brace);
+    enterRule(_localctx, 38, RULE_brace);
     try {
       _localctx = new BraceaccessContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(342);
+      setState(350);
       match(LBRACE);
-      setState(343);
+      setState(351);
       expression(0);
-      setState(344);
+      setState(352);
       match(RBRACE);
       }
     }
@@ -2338,11 +2381,11 @@ class PainlessParser extends Parser {
   public static class ArgumentsContext extends ParserRuleContext {
     public TerminalNode LP() { return getToken(PainlessParser.LP, 0); }
     public TerminalNode RP() { return getToken(PainlessParser.RP, 0); }
-    public List<ExpressionContext> expression() {
-      return getRuleContexts(ExpressionContext.class);
+    public List<ArgumentContext> argument() {
+      return getRuleContexts(ArgumentContext.class);
     }
-    public ExpressionContext expression(int i) {
-      return getRuleContext(ExpressionContext.class,i);
+    public ArgumentContext argument(int i) {
+      return getRuleContext(ArgumentContext.class,i);
     }
     public List<TerminalNode> COMMA() { return getTokens(PainlessParser.COMMA); }
     public TerminalNode COMMA(int i) {
@@ -2361,40 +2404,40 @@ class PainlessParser extends Parser {
 
   public final ArgumentsContext arguments() throws RecognitionException {
     ArgumentsContext _localctx = new ArgumentsContext(_ctx, getState());
-    enterRule(_localctx, 38, RULE_arguments);
+    enterRule(_localctx, 40, RULE_arguments);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
       {
-      setState(346);
+      setState(354);
       match(LP);
-      setState(355);
+      setState(363);
       switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
       case 1:
         {
-        setState(347);
-        expression(0);
-        setState(352);
+        setState(355);
+        argument();
+        setState(360);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(348);
+          setState(356);
           match(COMMA);
-          setState(349);
-          expression(0);
+          setState(357);
+          argument();
           }
           }
-          setState(354);
+          setState(362);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
         break;
       }
-      setState(357);
+      setState(365);
       match(RP);
       }
       }
@@ -2410,17 +2453,68 @@ class PainlessParser extends Parser {
     return _localctx;
   }
 
+  public static class ArgumentContext extends ParserRuleContext {
+    public ExpressionContext expression() {
+      return getRuleContext(ExpressionContext.class,0);
+    }
+    public FuncrefContext funcref() {
+      return getRuleContext(FuncrefContext.class,0);
+    }
+    public ArgumentContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_argument; }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitArgument(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final ArgumentContext argument() throws RecognitionException {
+    ArgumentContext _localctx = new ArgumentContext(_ctx, getState());
+    enterRule(_localctx, 42, RULE_argument);
+    try {
+      setState(369);
+      switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
+      case 1:
+        enterOuterAlt(_localctx, 1);
+        {
+        setState(367);
+        expression(0);
+        }
+        break;
+      case 2:
+        enterOuterAlt(_localctx, 2);
+        {
+        setState(368);
+        funcref();
+        }
+        break;
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
   public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) {
     switch (ruleIndex) {
     case 1:
       return statement_sempred((StatementContext)_localctx, predIndex);
-    case 12:
-      return expression_sempred((ExpressionContext)_localctx, predIndex);
     case 13:
+      return expression_sempred((ExpressionContext)_localctx, predIndex);
+    case 14:
       return unary_sempred((UnaryContext)_localctx, predIndex);
-    case 15:
-      return primary_sempred((PrimaryContext)_localctx, predIndex);
     case 16:
+      return primary_sempred((PrimaryContext)_localctx, predIndex);
+    case 17:
       return secondary_sempred((SecondaryContext)_localctx, predIndex);
     }
     return true;
@@ -2500,136 +2594,141 @@ class PainlessParser extends Parser {
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3L\u016a\4\2\t\2\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3M\u0176\4\2\t\2\4"+
     "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
     "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
-    "\4\23\t\23\4\24\t\24\4\25\t\25\3\2\7\2,\n\2\f\2\16\2/\13\2\3\2\3\2\3\3"+
-    "\3\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3;\n\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3C\n\3"+
-    "\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3P\n\3\3\3\3\3\5\3T\n\3"+
-    "\3\3\3\3\5\3X\n\3\3\3\3\3\3\3\5\3]\n\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3"+
-    "\3\3\3\3\3\3\3\3\3\3\3\3\6\3m\n\3\r\3\16\3n\3\3\3\3\3\3\3\3\3\3\3\3\3"+
-    "\3\5\3x\n\3\3\4\3\4\5\4|\n\4\3\5\3\5\7\5\u0080\n\5\f\5\16\5\u0083\13\5"+
-    "\3\5\3\5\3\6\3\6\3\7\3\7\5\7\u008b\n\7\3\b\3\b\3\t\3\t\3\t\3\t\7\t\u0093"+
-    "\n\t\f\t\16\t\u0096\13\t\3\n\3\n\3\n\7\n\u009b\n\n\f\n\16\n\u009e\13\n"+
-    "\3\13\3\13\3\13\5\13\u00a3\n\13\3\f\3\f\3\f\3\f\3\f\3\f\3\f\3\r\3\r\3"+
-    "\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\5\16\u00b7\n\16\3\16\3\16"+
-    "\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16"+
-    "\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16"+
-    "\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16"+
-    "\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\16\7\16"+
-    "\u00f2\n\16\f\16\16\16\u00f5\13\16\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
+    "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\3\2\7\2\60\n\2\f\2"+
+    "\16\2\63\13\2\3\2\3\2\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3?\n\3\3\3\3\3"+
+    "\3\3\3\3\3\3\3\3\5\3G\n\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3"+
+    "\5\3T\n\3\3\3\3\3\5\3X\n\3\3\3\3\3\5\3\\\n\3\3\3\3\3\3\3\5\3a\n\3\3\3"+
+    "\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\6\3q\n\3\r\3\16\3"+
+    "r\3\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3|\n\3\3\4\3\4\5\4\u0080\n\4\3\5\3\5\7"+
+    "\5\u0084\n\5\f\5\16\5\u0087\13\5\3\5\3\5\3\6\3\6\3\7\3\7\5\7\u008f\n\7"+
+    "\3\b\3\b\3\t\3\t\3\t\3\t\7\t\u0097\n\t\f\t\16\t\u009a\13\t\3\n\3\n\3\n"+
+    "\7\n\u009f\n\n\f\n\16\n\u00a2\13\n\3\13\3\13\3\13\3\13\3\f\3\f\3\f\5\f"+
+    "\u00ab\n\f\3\r\3\r\3\r\3\r\3\r\3\r\3\r\3\16\3\16\3\17\3\17\3\17\3\17\3"+
+    "\17\3\17\3\17\3\17\3\17\5\17\u00bf\n\17\3\17\3\17\3\17\3\17\3\17\3\17"+
     "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
-    "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\5\17\u0114\n\17\3\20\3\20\7\20"+
-    "\u0118\n\20\f\20\16\20\u011b\13\20\3\20\3\20\3\20\7\20\u0120\n\20\f\20"+
-    "\16\20\u0123\13\20\3\20\3\20\3\20\3\20\3\20\3\20\6\20\u012b\n\20\r\20"+
-    "\16\20\u012c\3\20\3\20\7\20\u0131\n\20\f\20\16\20\u0134\13\20\5\20\u0136"+
-    "\n\20\5\20\u0138\n\20\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
-    "\3\21\3\21\3\21\3\21\3\21\3\21\5\21\u014a\n\21\3\22\3\22\3\22\3\22\5\22"+
-    "\u0150\n\22\3\23\3\23\3\23\3\23\3\23\5\23\u0157\n\23\3\24\3\24\3\24\3"+
-    "\24\3\25\3\25\3\25\3\25\7\25\u0161\n\25\f\25\16\25\u0164\13\25\5\25\u0166"+
-    "\n\25\3\25\3\25\3\25\2\3\32\26\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36"+
-    " \"$&(\2\r\3\3\r\r\3\2\65@\3\2\34\36\3\2\37 \3\2!#\3\2$\'\3\2(+\3\2\63"+
-    "\64\3\2AD\4\2\32\33\37 \3\2KL\u0190\2-\3\2\2\2\4w\3\2\2\2\6{\3\2\2\2\b"+
-    "}\3\2\2\2\n\u0086\3\2\2\2\f\u008a\3\2\2\2\16\u008c\3\2\2\2\20\u008e\3"+
-    "\2\2\2\22\u0097\3\2\2\2\24\u009f\3\2\2\2\26\u00a4\3\2\2\2\30\u00ab\3\2"+
-    "\2\2\32\u00b6\3\2\2\2\34\u0113\3\2\2\2\36\u0137\3\2\2\2 \u0149\3\2\2\2"+
-    "\"\u014f\3\2\2\2$\u0156\3\2\2\2&\u0158\3\2\2\2(\u015c\3\2\2\2*,\5\4\3"+
-    "\2+*\3\2\2\2,/\3\2\2\2-+\3\2\2\2-.\3\2\2\2.\60\3\2\2\2/-\3\2\2\2\60\61"+
-    "\7\2\2\3\61\3\3\2\2\2\62\63\7\16\2\2\63\64\7\t\2\2\64\65\5\32\16\2\65"+
-    "\66\7\n\2\2\66:\5\6\4\2\678\7\17\2\28;\5\6\4\29;\6\3\2\2:\67\3\2\2\2:"+
-    "9\3\2\2\2;x\3\2\2\2<=\7\20\2\2=>\7\t\2\2>?\5\32\16\2?B\7\n\2\2@C\5\6\4"+
-    "\2AC\5\n\6\2B@\3\2\2\2BA\3\2\2\2Cx\3\2\2\2DE\7\21\2\2EF\5\b\5\2FG\7\20"+
-    "\2\2GH\7\t\2\2HI\5\32\16\2IJ\7\n\2\2JK\5\30\r\2Kx\3\2\2\2LM\7\22\2\2M"+
-    "O\7\t\2\2NP\5\f\7\2ON\3\2\2\2OP\3\2\2\2PQ\3\2\2\2QS\7\r\2\2RT\5\32\16"+
-    "\2SR\3\2\2\2ST\3\2\2\2TU\3\2\2\2UW\7\r\2\2VX\5\16\b\2WV\3\2\2\2WX\3\2"+
-    "\2\2XY\3\2\2\2Y\\\7\n\2\2Z]\5\6\4\2[]\5\n\6\2\\Z\3\2\2\2\\[\3\2\2\2]x"+
-    "\3\2\2\2^_\5\20\t\2_`\5\30\r\2`x\3\2\2\2ab\7\23\2\2bx\5\30\r\2cd\7\24"+
-    "\2\2dx\5\30\r\2ef\7\25\2\2fg\5\32\16\2gh\5\30\r\2hx\3\2\2\2ij\7\27\2\2"+
-    "jl\5\b\5\2km\5\26\f\2lk\3\2\2\2mn\3\2\2\2nl\3\2\2\2no\3\2\2\2ox\3\2\2"+
-    "\2pq\7\31\2\2qr\5\32\16\2rs\5\30\r\2sx\3\2\2\2tu\5\32\16\2uv\5\30\r\2"+
-    "vx\3\2\2\2w\62\3\2\2\2w<\3\2\2\2wD\3\2\2\2wL\3\2\2\2w^\3\2\2\2wa\3\2\2"+
-    "\2wc\3\2\2\2we\3\2\2\2wi\3\2\2\2wp\3\2\2\2wt\3\2\2\2x\5\3\2\2\2y|\5\b"+
-    "\5\2z|\5\4\3\2{y\3\2\2\2{z\3\2\2\2|\7\3\2\2\2}\u0081\7\5\2\2~\u0080\5"+
-    "\4\3\2\177~\3\2\2\2\u0080\u0083\3\2\2\2\u0081\177\3\2\2\2\u0081\u0082"+
-    "\3\2\2\2\u0082\u0084\3\2\2\2\u0083\u0081\3\2\2\2\u0084\u0085\7\6\2\2\u0085"+
-    "\t\3\2\2\2\u0086\u0087\7\r\2\2\u0087\13\3\2\2\2\u0088\u008b\5\20\t\2\u0089"+
-    "\u008b\5\32\16\2\u008a\u0088\3\2\2\2\u008a\u0089\3\2\2\2\u008b\r\3\2\2"+
-    "\2\u008c\u008d\5\32\16\2\u008d\17\3\2\2\2\u008e\u008f\5\22\n\2\u008f\u0094"+
-    "\5\24\13\2\u0090\u0091\7\f\2\2\u0091\u0093\5\24\13\2\u0092\u0090\3\2\2"+
-    "\2\u0093\u0096\3\2\2\2\u0094\u0092\3\2\2\2\u0094\u0095\3\2\2\2\u0095\21"+
-    "\3\2\2\2\u0096\u0094\3\2\2\2\u0097\u009c\7I\2\2\u0098\u0099\7\7\2\2\u0099"+
-    "\u009b\7\b\2\2\u009a\u0098\3\2\2\2\u009b\u009e\3\2\2\2\u009c\u009a\3\2"+
-    "\2\2\u009c\u009d\3\2\2\2\u009d\23\3\2\2\2\u009e\u009c\3\2\2\2\u009f\u00a2"+
-    "\7J\2\2\u00a0\u00a1\7\65\2\2\u00a1\u00a3\5\32\16\2\u00a2\u00a0\3\2\2\2"+
-    "\u00a2\u00a3\3\2\2\2\u00a3\25\3\2\2\2\u00a4\u00a5\7\30\2\2\u00a5\u00a6"+
-    "\7\t\2\2\u00a6\u00a7\7I\2\2\u00a7\u00a8\7J\2\2\u00a8\u00a9\7\n\2\2\u00a9"+
-    "\u00aa\5\b\5\2\u00aa\27\3\2\2\2\u00ab\u00ac\t\2\2\2\u00ac\31\3\2\2\2\u00ad"+
-    "\u00ae\b\16\1\2\u00ae\u00af\5\36\20\2\u00af\u00b0\t\3\2\2\u00b0\u00b1"+
-    "\5\32\16\3\u00b1\u00b2\b\16\1\2\u00b2\u00b7\3\2\2\2\u00b3\u00b4\5\34\17"+
-    "\2\u00b4\u00b5\b\16\1\2\u00b5\u00b7\3\2\2\2\u00b6\u00ad\3\2\2\2\u00b6"+
-    "\u00b3\3\2\2\2\u00b7\u00f3\3\2\2\2\u00b8\u00b9\f\16\2\2\u00b9\u00ba\t"+
-    "\4\2\2\u00ba\u00bb\5\32\16\17\u00bb\u00bc\b\16\1\2\u00bc\u00f2\3\2\2\2"+
-    "\u00bd\u00be\f\r\2\2\u00be\u00bf\t\5\2\2\u00bf\u00c0\5\32\16\16\u00c0"+
-    "\u00c1\b\16\1\2\u00c1\u00f2\3\2\2\2\u00c2\u00c3\f\f\2\2\u00c3\u00c4\t"+
-    "\6\2\2\u00c4\u00c5\5\32\16\r\u00c5\u00c6\b\16\1\2\u00c6\u00f2\3\2\2\2"+
-    "\u00c7\u00c8\f\13\2\2\u00c8\u00c9\t\7\2\2\u00c9\u00ca\5\32\16\f\u00ca"+
-    "\u00cb\b\16\1\2\u00cb\u00f2\3\2\2\2\u00cc\u00cd\f\n\2\2\u00cd\u00ce\t"+
-    "\b\2\2\u00ce\u00cf\5\32\16\13\u00cf\u00d0\b\16\1\2\u00d0\u00f2\3\2\2\2"+
-    "\u00d1\u00d2\f\t\2\2\u00d2\u00d3\7,\2\2\u00d3\u00d4\5\32\16\n\u00d4\u00d5"+
-    "\b\16\1\2\u00d5\u00f2\3\2\2\2\u00d6\u00d7\f\b\2\2\u00d7\u00d8\7-\2\2\u00d8"+
-    "\u00d9\5\32\16\t\u00d9\u00da\b\16\1\2\u00da\u00f2\3\2\2\2\u00db\u00dc"+
-    "\f\7\2\2\u00dc\u00dd\7.\2\2\u00dd\u00de\5\32\16\b\u00de\u00df\b\16\1\2"+
-    "\u00df\u00f2\3\2\2\2\u00e0\u00e1\f\6\2\2\u00e1\u00e2\7/\2\2\u00e2\u00e3"+
-    "\5\32\16\7\u00e3\u00e4\b\16\1\2\u00e4\u00f2\3\2\2\2\u00e5\u00e6\f\5\2"+
-    "\2\u00e6\u00e7\7\60\2\2\u00e7\u00e8\5\32\16\6\u00e8\u00e9\b\16\1\2\u00e9"+
-    "\u00f2\3\2\2\2\u00ea\u00eb\f\4\2\2\u00eb\u00ec\7\61\2\2\u00ec\u00ed\5"+
-    "\32\16\2\u00ed\u00ee\7\62\2\2\u00ee\u00ef\5\32\16\4\u00ef\u00f0\b\16\1"+
-    "\2\u00f0\u00f2\3\2\2\2\u00f1\u00b8\3\2\2\2\u00f1\u00bd\3\2\2\2\u00f1\u00c2"+
-    "\3\2\2\2\u00f1\u00c7\3\2\2\2\u00f1\u00cc\3\2\2\2\u00f1\u00d1\3\2\2\2\u00f1"+
-    "\u00d6\3\2\2\2\u00f1\u00db\3\2\2\2\u00f1\u00e0\3\2\2\2\u00f1\u00e5\3\2"+
-    "\2\2\u00f1\u00ea\3\2\2\2\u00f2\u00f5\3\2\2\2\u00f3\u00f1\3\2\2\2\u00f3"+
-    "\u00f4\3\2\2\2\u00f4\33\3\2\2\2\u00f5\u00f3\3\2\2\2\u00f6\u00f7\6\17\16"+
-    "\3\u00f7\u00f8\t\t\2\2\u00f8\u0114\5\36\20\2\u00f9\u00fa\6\17\17\3\u00fa"+
-    "\u00fb\5\36\20\2\u00fb\u00fc\t\t\2\2\u00fc\u0114\3\2\2\2\u00fd\u00fe\6"+
-    "\17\20\3\u00fe\u0114\5\36\20\2\u00ff\u0100\6\17\21\3\u0100\u0101\t\n\2"+
-    "\2\u0101\u0114\b\17\1\2\u0102\u0103\6\17\22\3\u0103\u0104\7F\2\2\u0104"+
-    "\u0114\b\17\1\2\u0105\u0106\6\17\23\3\u0106\u0107\7G\2\2\u0107\u0114\b"+
-    "\17\1\2\u0108\u0109\6\17\24\3\u0109\u010a\7H\2\2\u010a\u0114\b\17\1\2"+
-    "\u010b\u010c\6\17\25\3\u010c\u010d\t\13\2\2\u010d\u0114\5\34\17\2\u010e"+
-    "\u010f\7\t\2\2\u010f\u0110\5\22\n\2\u0110\u0111\7\n\2\2\u0111\u0112\5"+
-    "\34\17\2\u0112\u0114\3\2\2\2\u0113\u00f6\3\2\2\2\u0113\u00f9\3\2\2\2\u0113"+
-    "\u00fd\3\2\2\2\u0113\u00ff\3\2\2\2\u0113\u0102\3\2\2\2\u0113\u0105\3\2"+
-    "\2\2\u0113\u0108\3\2\2\2\u0113\u010b\3\2\2\2\u0113\u010e\3\2\2\2\u0114"+
-    "\35\3\2\2\2\u0115\u0119\5 \21\2\u0116\u0118\5\"\22\2\u0117\u0116\3\2\2"+
-    "\2\u0118\u011b\3\2\2\2\u0119\u0117\3\2\2\2\u0119\u011a\3\2\2\2\u011a\u0138"+
-    "\3\2\2\2\u011b\u0119\3\2\2\2\u011c\u011d\5\22\n\2\u011d\u0121\5$\23\2"+
-    "\u011e\u0120\5\"\22\2\u011f\u011e\3\2\2\2\u0120\u0123\3\2\2\2\u0121\u011f"+
-    "\3\2\2\2\u0121\u0122\3\2\2\2\u0122\u0138\3\2\2\2\u0123\u0121\3\2\2\2\u0124"+
-    "\u0125\7\26\2\2\u0125\u012a\7I\2\2\u0126\u0127\7\7\2\2\u0127\u0128\5\32"+
-    "\16\2\u0128\u0129\7\b\2\2\u0129\u012b\3\2\2\2\u012a\u0126\3\2\2\2\u012b"+
-    "\u012c\3\2\2\2\u012c\u012a\3\2\2\2\u012c\u012d\3\2\2\2\u012d\u0135\3\2"+
-    "\2\2\u012e\u0132\5$\23\2\u012f\u0131\5\"\22\2\u0130\u012f\3\2\2\2\u0131"+
-    "\u0134\3\2\2\2\u0132\u0130\3\2\2\2\u0132\u0133\3\2\2\2\u0133\u0136\3\2"+
-    "\2\2\u0134\u0132\3\2\2\2\u0135\u012e\3\2\2\2\u0135\u0136\3\2\2\2\u0136"+
-    "\u0138\3\2\2\2\u0137\u0115\3\2\2\2\u0137\u011c\3\2\2\2\u0137\u0124\3\2"+
-    "\2\2\u0138\37\3\2\2\2\u0139\u013a\6\21\26\3\u013a\u013b\7\t\2\2\u013b"+
-    "\u013c\5\32\16\2\u013c\u013d\7\n\2\2\u013d\u013e\b\21\1\2\u013e\u014a"+
-    "\3\2\2\2\u013f\u0140\6\21\27\3\u0140\u0141\7\t\2\2\u0141\u0142\5\34\17"+
-    "\2\u0142\u0143\7\n\2\2\u0143\u014a\3\2\2\2\u0144\u014a\7E\2\2\u0145\u014a"+
-    "\7J\2\2\u0146\u0147\7\26\2\2\u0147\u0148\7I\2\2\u0148\u014a\5(\25\2\u0149"+
-    "\u0139\3\2\2\2\u0149\u013f\3\2\2\2\u0149\u0144\3\2\2\2\u0149\u0145\3\2"+
-    "\2\2\u0149\u0146\3\2\2\2\u014a!\3\2\2\2\u014b\u014c\6\22\30\3\u014c\u0150"+
-    "\5$\23\2\u014d\u014e\6\22\31\3\u014e\u0150\5&\24\2\u014f\u014b\3\2\2\2"+
-    "\u014f\u014d\3\2\2\2\u0150#\3\2\2\2\u0151\u0152\7\13\2\2\u0152\u0153\7"+
-    "L\2\2\u0153\u0157\5(\25\2\u0154\u0155\7\13\2\2\u0155\u0157\t\f\2\2\u0156"+
-    "\u0151\3\2\2\2\u0156\u0154\3\2\2\2\u0157%\3\2\2\2\u0158\u0159\7\7\2\2"+
-    "\u0159\u015a\5\32\16\2\u015a\u015b\7\b\2\2\u015b\'\3\2\2\2\u015c\u0165"+
-    "\7\t\2\2\u015d\u0162\5\32\16\2\u015e\u015f\7\f\2\2\u015f\u0161\5\32\16"+
-    "\2\u0160\u015e\3\2\2\2\u0161\u0164\3\2\2\2\u0162\u0160\3\2\2\2\u0162\u0163"+
-    "\3\2\2\2\u0163\u0166\3\2\2\2\u0164\u0162\3\2\2\2\u0165\u015d\3\2\2\2\u0165"+
-    "\u0166\3\2\2\2\u0166\u0167\3\2\2\2\u0167\u0168\7\n\2\2\u0168)\3\2\2\2"+
-    " -:BOSW\\nw{\u0081\u008a\u0094\u009c\u00a2\u00b6\u00f1\u00f3\u0113\u0119"+
-    "\u0121\u012c\u0132\u0135\u0137\u0149\u014f\u0156\u0162\u0165";
+    "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
+    "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
+    "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\7\17\u00fa\n\17\f\17\16"+
+    "\17\u00fd\13\17\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
+    "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
+    "\3\20\3\20\3\20\3\20\5\20\u011c\n\20\3\21\3\21\7\21\u0120\n\21\f\21\16"+
+    "\21\u0123\13\21\3\21\3\21\3\21\7\21\u0128\n\21\f\21\16\21\u012b\13\21"+
+    "\3\21\3\21\3\21\3\21\3\21\3\21\6\21\u0133\n\21\r\21\16\21\u0134\3\21\3"+
+    "\21\7\21\u0139\n\21\f\21\16\21\u013c\13\21\5\21\u013e\n\21\5\21\u0140"+
+    "\n\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22"+
+    "\3\22\3\22\3\22\5\22\u0152\n\22\3\23\3\23\3\23\3\23\5\23\u0158\n\23\3"+
+    "\24\3\24\3\24\3\24\3\24\5\24\u015f\n\24\3\25\3\25\3\25\3\25\3\26\3\26"+
+    "\3\26\3\26\7\26\u0169\n\26\f\26\16\26\u016c\13\26\5\26\u016e\n\26\3\26"+
+    "\3\26\3\27\3\27\5\27\u0174\n\27\3\27\2\3\34\30\2\4\6\b\n\f\16\20\22\24"+
+    "\26\30\32\34\36 \"$&(*,\2\r\3\3\r\r\3\2\66A\3\2\34\36\3\2\37 \3\2!#\3"+
+    "\2$\'\3\2(+\3\2\64\65\3\2BE\4\2\32\33\37 \3\2LM\u019b\2\61\3\2\2\2\4{"+
+    "\3\2\2\2\6\177\3\2\2\2\b\u0081\3\2\2\2\n\u008a\3\2\2\2\f\u008e\3\2\2\2"+
+    "\16\u0090\3\2\2\2\20\u0092\3\2\2\2\22\u009b\3\2\2\2\24\u00a3\3\2\2\2\26"+
+    "\u00a7\3\2\2\2\30\u00ac\3\2\2\2\32\u00b3\3\2\2\2\34\u00be\3\2\2\2\36\u011b"+
+    "\3\2\2\2 \u013f\3\2\2\2\"\u0151\3\2\2\2$\u0157\3\2\2\2&\u015e\3\2\2\2"+
+    "(\u0160\3\2\2\2*\u0164\3\2\2\2,\u0173\3\2\2\2.\60\5\4\3\2/.\3\2\2\2\60"+
+    "\63\3\2\2\2\61/\3\2\2\2\61\62\3\2\2\2\62\64\3\2\2\2\63\61\3\2\2\2\64\65"+
+    "\7\2\2\3\65\3\3\2\2\2\66\67\7\16\2\2\678\7\t\2\289\5\34\17\29:\7\n\2\2"+
+    ":>\5\6\4\2;<\7\17\2\2<?\5\6\4\2=?\6\3\2\2>;\3\2\2\2>=\3\2\2\2?|\3\2\2"+
+    "\2@A\7\20\2\2AB\7\t\2\2BC\5\34\17\2CF\7\n\2\2DG\5\6\4\2EG\5\n\6\2FD\3"+
+    "\2\2\2FE\3\2\2\2G|\3\2\2\2HI\7\21\2\2IJ\5\b\5\2JK\7\20\2\2KL\7\t\2\2L"+
+    "M\5\34\17\2MN\7\n\2\2NO\5\32\16\2O|\3\2\2\2PQ\7\22\2\2QS\7\t\2\2RT\5\f"+
+    "\7\2SR\3\2\2\2ST\3\2\2\2TU\3\2\2\2UW\7\r\2\2VX\5\34\17\2WV\3\2\2\2WX\3"+
+    "\2\2\2XY\3\2\2\2Y[\7\r\2\2Z\\\5\16\b\2[Z\3\2\2\2[\\\3\2\2\2\\]\3\2\2\2"+
+    "]`\7\n\2\2^a\5\6\4\2_a\5\n\6\2`^\3\2\2\2`_\3\2\2\2a|\3\2\2\2bc\5\20\t"+
+    "\2cd\5\32\16\2d|\3\2\2\2ef\7\23\2\2f|\5\32\16\2gh\7\24\2\2h|\5\32\16\2"+
+    "ij\7\25\2\2jk\5\34\17\2kl\5\32\16\2l|\3\2\2\2mn\7\27\2\2np\5\b\5\2oq\5"+
+    "\30\r\2po\3\2\2\2qr\3\2\2\2rp\3\2\2\2rs\3\2\2\2s|\3\2\2\2tu\7\31\2\2u"+
+    "v\5\34\17\2vw\5\32\16\2w|\3\2\2\2xy\5\34\17\2yz\5\32\16\2z|\3\2\2\2{\66"+
+    "\3\2\2\2{@\3\2\2\2{H\3\2\2\2{P\3\2\2\2{b\3\2\2\2{e\3\2\2\2{g\3\2\2\2{"+
+    "i\3\2\2\2{m\3\2\2\2{t\3\2\2\2{x\3\2\2\2|\5\3\2\2\2}\u0080\5\b\5\2~\u0080"+
+    "\5\4\3\2\177}\3\2\2\2\177~\3\2\2\2\u0080\7\3\2\2\2\u0081\u0085\7\5\2\2"+
+    "\u0082\u0084\5\4\3\2\u0083\u0082\3\2\2\2\u0084\u0087\3\2\2\2\u0085\u0083"+
+    "\3\2\2\2\u0085\u0086\3\2\2\2\u0086\u0088\3\2\2\2\u0087\u0085\3\2\2\2\u0088"+
+    "\u0089\7\6\2\2\u0089\t\3\2\2\2\u008a\u008b\7\r\2\2\u008b\13\3\2\2\2\u008c"+
+    "\u008f\5\20\t\2\u008d\u008f\5\34\17\2\u008e\u008c\3\2\2\2\u008e\u008d"+
+    "\3\2\2\2\u008f\r\3\2\2\2\u0090\u0091\5\34\17\2\u0091\17\3\2\2\2\u0092"+
+    "\u0093\5\22\n\2\u0093\u0098\5\26\f\2\u0094\u0095\7\f\2\2\u0095\u0097\5"+
+    "\26\f\2\u0096\u0094\3\2\2\2\u0097\u009a\3\2\2\2\u0098\u0096\3\2\2\2\u0098"+
+    "\u0099\3\2\2\2\u0099\21\3\2\2\2\u009a\u0098\3\2\2\2\u009b\u00a0\7J\2\2"+
+    "\u009c\u009d\7\7\2\2\u009d\u009f\7\b\2\2\u009e\u009c\3\2\2\2\u009f\u00a2"+
+    "\3\2\2\2\u00a0\u009e\3\2\2\2\u00a0\u00a1\3\2\2\2\u00a1\23\3\2\2\2\u00a2"+
+    "\u00a0\3\2\2\2\u00a3\u00a4\7J\2\2\u00a4\u00a5\7\63\2\2\u00a5\u00a6\7K"+
+    "\2\2\u00a6\25\3\2\2\2\u00a7\u00aa\7K\2\2\u00a8\u00a9\7\66\2\2\u00a9\u00ab"+
+    "\5\34\17\2\u00aa\u00a8\3\2\2\2\u00aa\u00ab\3\2\2\2\u00ab\27\3\2\2\2\u00ac"+
+    "\u00ad\7\30\2\2\u00ad\u00ae\7\t\2\2\u00ae\u00af\7J\2\2\u00af\u00b0\7K"+
+    "\2\2\u00b0\u00b1\7\n\2\2\u00b1\u00b2\5\b\5\2\u00b2\31\3\2\2\2\u00b3\u00b4"+
+    "\t\2\2\2\u00b4\33\3\2\2\2\u00b5\u00b6\b\17\1\2\u00b6\u00b7\5 \21\2\u00b7"+
+    "\u00b8\t\3\2\2\u00b8\u00b9\5\34\17\3\u00b9\u00ba\b\17\1\2\u00ba\u00bf"+
+    "\3\2\2\2\u00bb\u00bc\5\36\20\2\u00bc\u00bd\b\17\1\2\u00bd\u00bf\3\2\2"+
+    "\2\u00be\u00b5\3\2\2\2\u00be\u00bb\3\2\2\2\u00bf\u00fb\3\2\2\2\u00c0\u00c1"+
+    "\f\16\2\2\u00c1\u00c2\t\4\2\2\u00c2\u00c3\5\34\17\17\u00c3\u00c4\b\17"+
+    "\1\2\u00c4\u00fa\3\2\2\2\u00c5\u00c6\f\r\2\2\u00c6\u00c7\t\5\2\2\u00c7"+
+    "\u00c8\5\34\17\16\u00c8\u00c9\b\17\1\2\u00c9\u00fa\3\2\2\2\u00ca\u00cb"+
+    "\f\f\2\2\u00cb\u00cc\t\6\2\2\u00cc\u00cd\5\34\17\r\u00cd\u00ce\b\17\1"+
+    "\2\u00ce\u00fa\3\2\2\2\u00cf\u00d0\f\13\2\2\u00d0\u00d1\t\7\2\2\u00d1"+
+    "\u00d2\5\34\17\f\u00d2\u00d3\b\17\1\2\u00d3\u00fa\3\2\2\2\u00d4\u00d5"+
+    "\f\n\2\2\u00d5\u00d6\t\b\2\2\u00d6\u00d7\5\34\17\13\u00d7\u00d8\b\17\1"+
+    "\2\u00d8\u00fa\3\2\2\2\u00d9\u00da\f\t\2\2\u00da\u00db\7,\2\2\u00db\u00dc"+
+    "\5\34\17\n\u00dc\u00dd\b\17\1\2\u00dd\u00fa\3\2\2\2\u00de\u00df\f\b\2"+
+    "\2\u00df\u00e0\7-\2\2\u00e0\u00e1\5\34\17\t\u00e1\u00e2\b\17\1\2\u00e2"+
+    "\u00fa\3\2\2\2\u00e3\u00e4\f\7\2\2\u00e4\u00e5\7.\2\2\u00e5\u00e6\5\34"+
+    "\17\b\u00e6\u00e7\b\17\1\2\u00e7\u00fa\3\2\2\2\u00e8\u00e9\f\6\2\2\u00e9"+
+    "\u00ea\7/\2\2\u00ea\u00eb\5\34\17\7\u00eb\u00ec\b\17\1\2\u00ec\u00fa\3"+
+    "\2\2\2\u00ed\u00ee\f\5\2\2\u00ee\u00ef\7\60\2\2\u00ef\u00f0\5\34\17\6"+
+    "\u00f0\u00f1\b\17\1\2\u00f1\u00fa\3\2\2\2\u00f2\u00f3\f\4\2\2\u00f3\u00f4"+
+    "\7\61\2\2\u00f4\u00f5\5\34\17\2\u00f5\u00f6\7\62\2\2\u00f6\u00f7\5\34"+
+    "\17\4\u00f7\u00f8\b\17\1\2\u00f8\u00fa\3\2\2\2\u00f9\u00c0\3\2\2\2\u00f9"+
+    "\u00c5\3\2\2\2\u00f9\u00ca\3\2\2\2\u00f9\u00cf\3\2\2\2\u00f9\u00d4\3\2"+
+    "\2\2\u00f9\u00d9\3\2\2\2\u00f9\u00de\3\2\2\2\u00f9\u00e3\3\2\2\2\u00f9"+
+    "\u00e8\3\2\2\2\u00f9\u00ed\3\2\2\2\u00f9\u00f2\3\2\2\2\u00fa\u00fd\3\2"+
+    "\2\2\u00fb\u00f9\3\2\2\2\u00fb\u00fc\3\2\2\2\u00fc\35\3\2\2\2\u00fd\u00fb"+
+    "\3\2\2\2\u00fe\u00ff\6\20\16\3\u00ff\u0100\t\t\2\2\u0100\u011c\5 \21\2"+
+    "\u0101\u0102\6\20\17\3\u0102\u0103\5 \21\2\u0103\u0104\t\t\2\2\u0104\u011c"+
+    "\3\2\2\2\u0105\u0106\6\20\20\3\u0106\u011c\5 \21\2\u0107\u0108\6\20\21"+
+    "\3\u0108\u0109\t\n\2\2\u0109\u011c\b\20\1\2\u010a\u010b\6\20\22\3\u010b"+
+    "\u010c\7G\2\2\u010c\u011c\b\20\1\2\u010d\u010e\6\20\23\3\u010e\u010f\7"+
+    "H\2\2\u010f\u011c\b\20\1\2\u0110\u0111\6\20\24\3\u0111\u0112\7I\2\2\u0112"+
+    "\u011c\b\20\1\2\u0113\u0114\6\20\25\3\u0114\u0115\t\13\2\2\u0115\u011c"+
+    "\5\36\20\2\u0116\u0117\7\t\2\2\u0117\u0118\5\22\n\2\u0118\u0119\7\n\2"+
+    "\2\u0119\u011a\5\36\20\2\u011a\u011c\3\2\2\2\u011b\u00fe\3\2\2\2\u011b"+
+    "\u0101\3\2\2\2\u011b\u0105\3\2\2\2\u011b\u0107\3\2\2\2\u011b\u010a\3\2"+
+    "\2\2\u011b\u010d\3\2\2\2\u011b\u0110\3\2\2\2\u011b\u0113\3\2\2\2\u011b"+
+    "\u0116\3\2\2\2\u011c\37\3\2\2\2\u011d\u0121\5\"\22\2\u011e\u0120\5$\23"+
+    "\2\u011f\u011e\3\2\2\2\u0120\u0123\3\2\2\2\u0121\u011f\3\2\2\2\u0121\u0122"+
+    "\3\2\2\2\u0122\u0140\3\2\2\2\u0123\u0121\3\2\2\2\u0124\u0125\5\22\n\2"+
+    "\u0125\u0129\5&\24\2\u0126\u0128\5$\23\2\u0127\u0126\3\2\2\2\u0128\u012b"+
+    "\3\2\2\2\u0129\u0127\3\2\2\2\u0129\u012a\3\2\2\2\u012a\u0140\3\2\2\2\u012b"+
+    "\u0129\3\2\2\2\u012c\u012d\7\26\2\2\u012d\u0132\7J\2\2\u012e\u012f\7\7"+
+    "\2\2\u012f\u0130\5\34\17\2\u0130\u0131\7\b\2\2\u0131\u0133\3\2\2\2\u0132"+
+    "\u012e\3\2\2\2\u0133\u0134\3\2\2\2\u0134\u0132\3\2\2\2\u0134\u0135\3\2"+
+    "\2\2\u0135\u013d\3\2\2\2\u0136\u013a\5&\24\2\u0137\u0139\5$\23\2\u0138"+
+    "\u0137\3\2\2\2\u0139\u013c\3\2\2\2\u013a\u0138\3\2\2\2\u013a\u013b\3\2"+
+    "\2\2\u013b\u013e\3\2\2\2\u013c\u013a\3\2\2\2\u013d\u0136\3\2\2\2\u013d"+
+    "\u013e\3\2\2\2\u013e\u0140\3\2\2\2\u013f\u011d\3\2\2\2\u013f\u0124\3\2"+
+    "\2\2\u013f\u012c\3\2\2\2\u0140!\3\2\2\2\u0141\u0142\6\22\26\3\u0142\u0143"+
+    "\7\t\2\2\u0143\u0144\5\34\17\2\u0144\u0145\7\n\2\2\u0145\u0146\b\22\1"+
+    "\2\u0146\u0152\3\2\2\2\u0147\u0148\6\22\27\3\u0148\u0149\7\t\2\2\u0149"+
+    "\u014a\5\36\20\2\u014a\u014b\7\n\2\2\u014b\u0152\3\2\2\2\u014c\u0152\7"+
+    "F\2\2\u014d\u0152\7K\2\2\u014e\u014f\7\26\2\2\u014f\u0150\7J\2\2\u0150"+
+    "\u0152\5*\26\2\u0151\u0141\3\2\2\2\u0151\u0147\3\2\2\2\u0151\u014c\3\2"+
+    "\2\2\u0151\u014d\3\2\2\2\u0151\u014e\3\2\2\2\u0152#\3\2\2\2\u0153\u0154"+
+    "\6\23\30\3\u0154\u0158\5&\24\2\u0155\u0156\6\23\31\3\u0156\u0158\5(\25"+
+    "\2\u0157\u0153\3\2\2\2\u0157\u0155\3\2\2\2\u0158%\3\2\2\2\u0159\u015a"+
+    "\7\13\2\2\u015a\u015b\7M\2\2\u015b\u015f\5*\26\2\u015c\u015d\7\13\2\2"+
+    "\u015d\u015f\t\f\2\2\u015e\u0159\3\2\2\2\u015e\u015c\3\2\2\2\u015f\'\3"+
+    "\2\2\2\u0160\u0161\7\7\2\2\u0161\u0162\5\34\17\2\u0162\u0163\7\b\2\2\u0163"+
+    ")\3\2\2\2\u0164\u016d\7\t\2\2\u0165\u016a\5,\27\2\u0166\u0167\7\f\2\2"+
+    "\u0167\u0169\5,\27\2\u0168\u0166\3\2\2\2\u0169\u016c\3\2\2\2\u016a\u0168"+
+    "\3\2\2\2\u016a\u016b\3\2\2\2\u016b\u016e\3\2\2\2\u016c\u016a\3\2\2\2\u016d"+
+    "\u0165\3\2\2\2\u016d\u016e\3\2\2\2\u016e\u016f\3\2\2\2\u016f\u0170\7\n"+
+    "\2\2\u0170+\3\2\2\2\u0171\u0174\5\34\17\2\u0172\u0174\5\24\13\2\u0173"+
+    "\u0171\3\2\2\2\u0173\u0172\3\2\2\2\u0174-\3\2\2\2!\61>FSW[`r{\177\u0085"+
+    "\u008e\u0098\u00a0\u00aa\u00be\u00f9\u00fb\u011b\u0121\u0129\u0134\u013a"+
+    "\u013d\u013f\u0151\u0157\u015e\u016a\u016d\u0173";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
index 7d5ed4c9e6f..b4429fc0841 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
@@ -144,6 +144,13 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitDecltype(PainlessParser.DecltypeContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitFuncref(PainlessParser.FuncrefContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
@@ -361,4 +368,11 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitArguments(PainlessParser.ArgumentsContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitArgument(PainlessParser.ArgumentContext ctx) { return visitChildren(ctx); }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
index ab1254e1a91..6dc511cb4e9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
@@ -135,6 +135,12 @@ interface PainlessParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitDecltype(PainlessParser.DecltypeContext ctx);
+  /**
+   * Visit a parse tree produced by {@link PainlessParser#funcref}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitFuncref(PainlessParser.FuncrefContext ctx);
   /**
    * Visit a parse tree produced by {@link PainlessParser#declvar}.
    * @param ctx the parse tree
@@ -347,4 +353,10 @@ interface PainlessParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitArguments(PainlessParser.ArgumentsContext ctx);
+  /**
+   * Visit a parse tree produced by {@link PainlessParser#argument}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitArgument(PainlessParser.ArgumentContext ctx);
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index 3669ab24ecc..9d32d78e053 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -31,6 +31,7 @@ import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Variables.Reserved;
 import org.elasticsearch.painless.antlr.PainlessParser.AfterthoughtContext;
+import org.elasticsearch.painless.antlr.PainlessParser.ArgumentContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ArgumentsContext;
 import org.elasticsearch.painless.antlr.PainlessParser.AssignmentContext;
 import org.elasticsearch.painless.antlr.PainlessParser.BinaryContext;
@@ -58,6 +59,7 @@ import org.elasticsearch.painless.antlr.PainlessParser.ExprprecContext;
 import org.elasticsearch.painless.antlr.PainlessParser.FalseContext;
 import org.elasticsearch.painless.antlr.PainlessParser.FieldaccessContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ForContext;
+import org.elasticsearch.painless.antlr.PainlessParser.FuncrefContext;
 import org.elasticsearch.painless.antlr.PainlessParser.IfContext;
 import org.elasticsearch.painless.antlr.PainlessParser.InitializerContext;
 import org.elasticsearch.painless.antlr.PainlessParser.NewarrayContext;
@@ -95,6 +97,7 @@ import org.elasticsearch.painless.node.EComp;
 import org.elasticsearch.painless.node.EConditional;
 import org.elasticsearch.painless.node.EDecimal;
 import org.elasticsearch.painless.node.EExplicit;
+import org.elasticsearch.painless.node.EFunctionRef;
 import org.elasticsearch.painless.node.ENull;
 import org.elasticsearch.painless.node.ENumeric;
 import org.elasticsearch.painless.node.EUnary;
@@ -391,6 +394,11 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
     }
 
+    @Override
+    public Object visitFuncref(FuncrefContext ctx) {
+        return new EFunctionRef(line(ctx), offset(ctx), location(ctx), ctx.TYPE().getText(), ctx.ID().getText());
+    }
+
     @Override
     public Object visitDeclvar(DeclvarContext ctx) {
         throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
@@ -774,11 +782,8 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
     @Override
     public Object visitNewobject(NewobjectContext ctx) {
         String type = ctx.TYPE().getText();
-        List<AExpression> arguments = new ArrayList<>();
-
-        for (ExpressionContext expression : ctx.arguments().expression()) {
-            arguments.add((AExpression)visitExpression(expression));
-        }
+        @SuppressWarnings("unchecked")
+        List<AExpression> arguments = (List<AExpression>)visit(ctx.arguments());
 
         List<ALink> links = new ArrayList<>();
         links.add(new LNewObj(line(ctx), offset(ctx), location(ctx), type, arguments));
@@ -800,11 +805,8 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
     @Override
     public Object visitCallinvoke(CallinvokeContext ctx) {
         String name = ctx.DOTID().getText();
-        List<AExpression> arguments = new ArrayList<>();
-
-        for (ExpressionContext expression : ctx.arguments().expression()) {
-            arguments.add((AExpression)visitExpression(expression));
-        }
+        @SuppressWarnings("unchecked")
+        List<AExpression> arguments = (List<AExpression>)visit(ctx.arguments());
 
         return new LCall(line(ctx), offset(ctx), location(ctx), name, arguments);
     }
@@ -833,6 +835,23 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
 
     @Override
     public Object visitArguments(ArgumentsContext ctx) {
-        throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
+        List<AExpression> arguments = new ArrayList<>();
+
+        for (ArgumentContext argument : ctx.argument()) {
+            arguments.add((AExpression)visit(argument));
+        }
+
+        return arguments;
+    }
+
+    @Override
+    public Object visitArgument(ArgumentContext ctx) {
+        if (ctx.expression() != null) {
+            return visitExpression(ctx.expression());
+        } else if (ctx.funcref() != null) {
+            return visit(ctx.funcref());
+        } else {
+            throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
+        }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
new file mode 100644
index 00000000000..77eb6a00e44
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.Variables;
+
+/**
+ * Represents a function reference.
+ */
+public class EFunctionRef extends AExpression {
+    public String type;
+    public String call;
+
+    public EFunctionRef(int line, int offset, String location, String type, String call) {
+        super(line, offset, location);
+
+        this.type = type;
+        this.call = call;
+    }
+
+    @Override
+    void analyze(Variables variables) {
+        throw new UnsupportedOperationException(error("Function references [" + type + "::" + call + "] are not currently supported."));
+    }
+
+    @Override
+    void write(MethodWriter writer) {
+        throw new IllegalStateException(error("Illegal tree structure."));
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
index 47aa868d6dc..51825f41d70 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
@@ -42,6 +42,7 @@
  * {@link org.elasticsearch.painless.node.EConstant} - Respresents a constant.  (Internal only.)
  * {@link org.elasticsearch.painless.node.EDecimal} - Respresents a decimal constant.
  * {@link org.elasticsearch.painless.node.EExplicit} - Represents an explicit cast.
+ * {@link org.elasticsearch.painless.node.EFunctionRef} - Represents a function reference.
  * {@link org.elasticsearch.painless.node.ENull} - Represents a null constant.
  * {@link org.elasticsearch.painless.node.ENumeric} - Respresents a non-decimal numeric constant.
  * {@link org.elasticsearch.painless.node.EUnary} - Represents a unary math expression.
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
new file mode 100644
index 00000000000..f612a6c84ed
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+public class FunctionRefTests extends ScriptTestCase {
+    public void testUnsupported() {
+        expectThrows(UnsupportedOperationException.class, () -> {
+           exec("DoubleStream.Builder builder = DoubleStream.builder();" +
+               "builder.add(2.0); builder.add(1.0); builder.add(3.0);" +
+               "builder.build().reduce(Double::unsupported);");
+        });
+    }
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScoreTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScoreTests.java
index deaabfcebc6..63947ced79a 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScoreTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScoreTests.java
@@ -19,13 +19,12 @@
 
 package org.elasticsearch.painless;
 
-import java.io.IOException;
-import java.util.Collections;
-import java.util.concurrent.atomic.AtomicBoolean;
-
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Scorer;
 
+import java.io.IOException;
+import java.util.Collections;
+
 public class ScoreTests extends ScriptTestCase {
 
     /** Most of a dummy scorer impl that requires overriding just score(). */

From 19b3ec88af95166fd654ce2f8d62a1f9b473ba8f Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Wed, 25 May 2016 11:41:40 -0700
Subject: [PATCH 0463/1311] Build: ignore deprecation warnings

We currently fail on any deprecations found during the build. However,
this includes things deprecated within ES, which adds a heavy burden in
order to deprecate apis (requring to add suppressions to all internal
callers of the API).

This change adds `-deprecation` to xlint. We should consider in the
future having a task to "see" what deprecated apis we currently use for
analysis.
---
 .../src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index 168ceca1d23..c4eb76164fe 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -378,7 +378,7 @@ class BuildPlugin implements Plugin<Project> {
                  * -serial because we don't use java serialization.
                  */
                 // don't even think about passing args with -J-xxx, oracle will ask you to submit a bug report :)
-                options.compilerArgs << '-Werror' << '-Xlint:all,-path,-serial,-options' << '-Xdoclint:all' << '-Xdoclint:-missing'
+                options.compilerArgs << '-Werror' << '-Xlint:all,-path,-serial,-options,-deprecation' << '-Xdoclint:all' << '-Xdoclint:-missing'
                 // compile with compact 3 profile by default
                 // NOTE: this is just a compile time check: does not replace testing with a compact3 JRE
                 if (project.compactProfile != 'full') {

From 6dcfcf0bf14568b98b478506b76037309015584a Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Wed, 25 May 2016 12:08:57 -0700
Subject: [PATCH 0464/1311] Build: Remove buildSrc as a project in IDEs

We have have a virtual build-tools project which pulls in all the same
IDE settings we have for other projects, so there is no longer a need
for hacking buildSrc into our IDEs.
---
 build.gradle | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/build.gradle b/build.gradle
index bf4b5a3b415..3f0895a4e75 100644
--- a/build.gradle
+++ b/build.gradle
@@ -269,13 +269,6 @@ tasks.idea.doLast {
 if (System.getProperty('idea.active') != null && ideaMarker.exists() == false) {
   throw new GradleException('You must run gradle idea from the root of elasticsearch before importing into IntelliJ')
 }
-// add buildSrc itself as a groovy project
-task buildSrcIdea(type: GradleBuild) {
-  buildFile = 'buildSrc/build.gradle'
-  tasks = ['cleanIdea', 'ideaModule']
-}
-tasks.idea.dependsOn(buildSrcIdea)
-
 
 // eclipse configuration
 allprojects {
@@ -318,13 +311,6 @@ allprojects {
   tasks.eclipse.dependsOn(cleanEclipse, copyEclipseSettings)
 }
 
-// add buildSrc itself as a groovy project
-task buildSrcEclipse(type: GradleBuild) {
-  buildFile = 'buildSrc/build.gradle'
-  tasks = ['cleanEclipse', 'eclipse']
-}
-tasks.eclipse.dependsOn(buildSrcEclipse)
-
 // we need to add the same --debug-jvm option as
 // the real RunTask has, so we can pass it through
 class Run extends DefaultTask {

From edfbdf2748f7abe0bcccd00893c37bb007dfc036 Mon Sep 17 00:00:00 2001
From: Tal Levy <JubBoy333@gmail.com>
Date: Wed, 25 May 2016 12:20:39 -0700
Subject: [PATCH 0465/1311] add ability to specify multiple grok patterns
 (#18074)

- now you can specify a list of grok patterns to match your field with
and the first one to successfully match wins.
- only non-null captures will be inserted into your matched document.

Fixes #17903.
---
 docs/reference/ingest/ingest-node.asciidoc    |  7 +-
 .../ingest/grok/GrokProcessor.java            | 81 ++++++++++++---
 .../grok/GrokProcessorFactoryTests.java       | 56 +++++------
 .../ingest/grok/GrokProcessorTests.java       | 98 +++++++++++++------
 .../test/ingest_grok/20_grok.yaml             | 46 ++++++++-
 .../test/ingest/20_combine_processors.yaml    |  4 +-
 6 files changed, 206 insertions(+), 86 deletions(-)

diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc
index 3dcf719f382..54a10f766fd 100644
--- a/docs/reference/ingest/ingest-node.asciidoc
+++ b/docs/reference/ingest/ingest-node.asciidoc
@@ -1038,8 +1038,9 @@ Grok expression.
 |======
 | Name                   | Required  | Default             | Description
 | `field`                | yes       | -                   | The field to use for grok expression parsing
-| `pattern`              | yes       | -                   | The grok expression to match and extract named captures with
+| `patterns`             | yes       | -                   | An ordered list of grok expression to match and extract named captures with. Returns on the first expression in the list that matches.
 | `pattern_definitions`  | no        | -                   | A map of pattern-name and pattern tuples defining custom patterns to be used by the current processor. Patterns matching existing names will override the pre-existing definition.
+| `trace_match`          | no        | false               | when true, `_ingest._grok_match_index` will be inserted into your matched document's metadata with the index into the pattern found in `patterns` that matched.
 |======
 
 Here is an example of using the provided patterns to extract out and name structured fields from a string field in
@@ -1069,7 +1070,7 @@ Here is an example pipeline for processing the above document by using Grok:
     {
       "grok": {
         "field": "message",
-        "pattern": "%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}"
+        "patterns": ["%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}"]
       }
     }
   ]
@@ -1107,7 +1108,7 @@ Here is an example of a pipeline specifying custom pattern definitions:
     {
       "grok": {
         "field": "message",
-        "pattern": "my %{FAVORITE_DOG:dog} is colored %{RGB:color}"
+        "patterns": ["my %{FAVORITE_DOG:dog} is colored %{RGB:color}"]
         "pattern_definitions" : {
           "FAVORITE_DOG" : "beagle",
           "RGB" : "RED|GREEN|BLUE"
diff --git a/modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/GrokProcessor.java b/modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/GrokProcessor.java
index 9237821baba..6a10c8db058 100644
--- a/modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/GrokProcessor.java
+++ b/modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/GrokProcessor.java
@@ -25,31 +25,50 @@ import org.elasticsearch.ingest.core.ConfigurationUtils;
 import org.elasticsearch.ingest.core.IngestDocument;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 import static org.elasticsearch.ingest.core.ConfigurationUtils.newConfigurationException;
 
 public final class GrokProcessor extends AbstractProcessor {
 
     public static final String TYPE = "grok";
+    private static final String PATTERN_MATCH_KEY = "_ingest._grok_match_index";
 
     private final String matchField;
     private final Grok grok;
+    private final boolean traceMatch;
 
-    public GrokProcessor(String tag, Grok grok, String matchField) {
+    public GrokProcessor(String tag, Map<String, String> patternBank, List<String> matchPatterns, String matchField) {
+        this(tag, patternBank, matchPatterns, matchField, false);
+    }
+
+    public GrokProcessor(String tag, Map<String, String> patternBank, List<String> matchPatterns, String matchField, boolean traceMatch) {
         super(tag);
         this.matchField = matchField;
-        this.grok = grok;
+        this.grok = new Grok(patternBank, combinePatterns(matchPatterns, traceMatch));
+        this.traceMatch = traceMatch;
     }
 
     @Override
     public void execute(IngestDocument ingestDocument) throws Exception {
         String fieldValue = ingestDocument.getFieldValue(matchField, String.class);
         Map<String, Object> matches = grok.captures(fieldValue);
-        if (matches != null) {
-            matches.forEach((k, v) -> ingestDocument.setFieldValue(k, v));
-        } else {
-            throw new IllegalArgumentException("Grok expression does not match field value: [" + fieldValue + "]");
+        if (matches == null) {
+            throw new IllegalArgumentException("Provided Grok expressions do not match field value: [" + fieldValue + "]");
+        }
+
+        matches.entrySet().stream()
+            .filter((e) -> Objects.nonNull(e.getValue()))
+            .forEach((e) -> ingestDocument.setFieldValue(e.getKey(), e.getValue()));
+
+        if (traceMatch) {
+            @SuppressWarnings("unchecked")
+            HashMap<String, String> matchMap = (HashMap<String, String>) ingestDocument.getFieldValue(PATTERN_MATCH_KEY, Object.class);
+            matchMap.keySet().stream().findFirst().ifPresent((index) -> {
+                ingestDocument.setFieldValue(PATTERN_MATCH_KEY, index);
+            });
         }
     }
 
@@ -58,12 +77,41 @@ public final class GrokProcessor extends AbstractProcessor {
         return TYPE;
     }
 
+    public Grok getGrok() {
+        return grok;
+    }
+
     String getMatchField() {
         return matchField;
     }
 
-    Grok getGrok() {
-        return grok;
+    static String combinePatterns(List<String> patterns, boolean traceMatch) {
+        String combinedPattern;
+        if (patterns.size() > 1) {
+            if (traceMatch) {
+                combinedPattern = "";
+                for (int i = 0; i < patterns.size(); i++) {
+                    String valueWrap = "(?<" + PATTERN_MATCH_KEY + "." + i + ">" + patterns.get(i) + ")";
+                    if (combinedPattern.equals("")) {
+                        combinedPattern = valueWrap;
+                    } else {
+                        combinedPattern = combinedPattern + "|" + valueWrap;
+                    }
+                }
+            } else {
+                combinedPattern = patterns.stream().reduce("", (prefix, value) -> {
+                    if (prefix.equals("")) {
+                        return "(?:" + value + ")";
+                    } else {
+                        return prefix + "|" + "(?:" + value + ")";
+                    }
+                });
+            }
+        }  else {
+            combinedPattern = patterns.get(0);
+        }
+
+        return combinedPattern;
     }
 
     public final static class Factory extends AbstractProcessorFactory<GrokProcessor> {
@@ -77,22 +125,25 @@ public final class GrokProcessor extends AbstractProcessor {
         @Override
         public GrokProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
             String matchField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
-            String matchPattern = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "pattern");
+            List<String> matchPatterns = ConfigurationUtils.readList(TYPE, processorTag, config, "patterns");
+            boolean traceMatch = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "trace_match", false);
+
+            if (matchPatterns.isEmpty()) {
+                throw newConfigurationException(TYPE, processorTag, "patterns", "List of patterns must not be empty");
+            }
             Map<String, String> customPatternBank = ConfigurationUtils.readOptionalMap(TYPE, processorTag, config, "pattern_definitions");
             Map<String, String> patternBank = new HashMap<>(builtinPatterns);
             if (customPatternBank != null) {
                 patternBank.putAll(customPatternBank);
             }
 
-            Grok grok;
             try {
-                grok = new Grok(patternBank, matchPattern);
+                return new GrokProcessor(processorTag, patternBank, matchPatterns, matchField, traceMatch);
             } catch (Exception e) {
-                throw newConfigurationException(TYPE, processorTag, "pattern", "Invalid regex pattern. " + e.getMessage());
+                throw newConfigurationException(TYPE, processorTag, "patterns",
+                    "Invalid regex pattern found in: " + matchPatterns + ". " + e.getMessage());
             }
-            return new GrokProcessor(processorTag, grok, matchField);
+
         }
-
     }
-
 }
diff --git a/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokProcessorFactoryTests.java b/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokProcessorFactoryTests.java
index 3880d389c52..ec3b908aac0 100644
--- a/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokProcessorFactoryTests.java
+++ b/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokProcessorFactoryTests.java
@@ -37,7 +37,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-        config.put("pattern", "(?<foo>\\w+)");
+        config.put("patterns", Collections.singletonList("(?<foo>\\w+)"));
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
         GrokProcessor processor = factory.create(config);
@@ -49,27 +49,26 @@ public class GrokProcessorFactoryTests extends ESTestCase {
     public void testBuildMissingField() throws Exception {
         GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap());
         Map<String, Object> config = new HashMap<>();
-        config.put("pattern", "(?<foo>\\w+)");
-        try {
-            factory.create(config);
-            fail("should fail");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), equalTo("[field] required property is missing"));
-
-        }
+        config.put("patterns", Collections.singletonList("(?<foo>\\w+)"));
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        assertThat(e.getMessage(), equalTo("[field] required property is missing"));
     }
 
-    public void testBuildMissingPattern() throws Exception {
+    public void testBuildMissingPatterns() throws Exception {
         GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap());
         Map<String, Object> config = new HashMap<>();
         config.put("field", "foo");
-        try {
-            factory.create(config);
-            fail("should fail");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), equalTo("[pattern] required property is missing"));
-        }
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        assertThat(e.getMessage(), equalTo("[patterns] required property is missing"));
+    }
 
+    public void testBuildEmptyPatternsList() throws Exception {
+        GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap());
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "foo");
+        config.put("patterns", Collections.emptyList());
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        assertThat(e.getMessage(), equalTo("[patterns] List of patterns must not be empty"));
     }
 
     public void testCreateWithCustomPatterns() throws Exception {
@@ -77,7 +76,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-        config.put("pattern", "%{MY_PATTERN:name}!");
+        config.put("patterns", Collections.singletonList("%{MY_PATTERN:name}!"));
         config.put("pattern_definitions", Collections.singletonMap("MY_PATTERN", "foo"));
         GrokProcessor processor = factory.create(config);
         assertThat(processor.getMatchField(), equalTo("_field"));
@@ -89,28 +88,19 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap());
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-        config.put("pattern", "[");
-        try {
-            factory.create(config);
-            fail("should fail");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), equalTo("[pattern] Invalid regex pattern. premature end of char-class"));
-        }
-
+        config.put("patterns", Collections.singletonList("["));
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        assertThat(e.getMessage(), equalTo("[patterns] Invalid regex pattern found in: [[]. premature end of char-class"));
     }
 
     public void testCreateWithInvalidPatternDefinition() throws Exception {
         GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap());
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-        config.put("pattern", "%{MY_PATTERN:name}!");
+        config.put("patterns", Collections.singletonList("%{MY_PATTERN:name}!"));
         config.put("pattern_definitions", Collections.singletonMap("MY_PATTERN", "["));
-        try {
-            factory.create(config);
-            fail("should fail");
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), equalTo("[pattern] Invalid regex pattern. premature end of char-class"));
-        }
-
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        assertThat(e.getMessage(),
+            equalTo("[patterns] Invalid regex pattern found in: [%{MY_PATTERN:name}!]. premature end of char-class"));
     }
 }
diff --git a/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokProcessorTests.java b/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokProcessorTests.java
index 840cf954022..82354a943ae 100644
--- a/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokProcessorTests.java
+++ b/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokProcessorTests.java
@@ -21,14 +21,15 @@ package org.elasticsearch.ingest.grok;
 
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.grok.Grok;
-import org.elasticsearch.ingest.grok.GrokProcessor;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
 
 
 public class GrokProcessorTests extends ESTestCase {
@@ -37,8 +38,8 @@ public class GrokProcessorTests extends ESTestCase {
         String fieldName = RandomDocumentPicks.randomFieldName(random());
         IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         doc.setFieldValue(fieldName, "1");
-        Grok grok = new Grok(Collections.singletonMap("ONE", "1"), "%{ONE:one}");
-        GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), grok, fieldName);
+        GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
+            Collections.singletonList("%{ONE:one}"), fieldName);
         processor.execute(doc);
         assertThat(doc.getFieldValue("one", String.class), equalTo("1"));
     }
@@ -47,14 +48,10 @@ public class GrokProcessorTests extends ESTestCase {
         String fieldName = RandomDocumentPicks.randomFieldName(random());
         IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         doc.setFieldValue(fieldName, "23");
-        Grok grok = new Grok(Collections.singletonMap("ONE", "1"), "%{ONE:one}");
-        GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), grok, fieldName);
-        try {
-            processor.execute(doc);
-            fail();
-        } catch (Exception e) {
-            assertThat(e.getMessage(), equalTo("Grok expression does not match field value: [23]"));
-        }
+        GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
+            Collections.singletonList("%{ONE:one}"), fieldName);
+        Exception e = expectThrows(Exception.class, () -> processor.execute(doc));
+        assertThat(e.getMessage(), equalTo("Provided Grok expressions do not match field value: [23]"));
     }
 
     public void testMatchWithoutCaptures() throws Exception {
@@ -62,8 +59,8 @@ public class GrokProcessorTests extends ESTestCase {
         IngestDocument originalDoc = new IngestDocument(new HashMap<>(), new HashMap<>());
         originalDoc.setFieldValue(fieldName, fieldName);
         IngestDocument doc = new IngestDocument(originalDoc);
-        Grok grok = new Grok(Collections.emptyMap(), fieldName);
-        GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), grok, fieldName);
+        GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.emptyMap(),
+            Collections.singletonList(fieldName), fieldName);
         processor.execute(doc);
         assertThat(doc, equalTo(originalDoc));
     }
@@ -72,26 +69,67 @@ public class GrokProcessorTests extends ESTestCase {
         String fieldName = RandomDocumentPicks.randomFieldName(random());
         IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         doc.setFieldValue(fieldName, 1);
-        Grok grok = new Grok(Collections.singletonMap("ONE", "1"), "%{ONE:one}");
-        GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), grok, fieldName);
-        try {
-            processor.execute(doc);
-            fail();
-        } catch (Exception e) {
-            assertThat(e.getMessage(), equalTo("field [" + fieldName + "] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
-        }
+        GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
+            Collections.singletonList("%{ONE:one}"), fieldName);
+        Exception e = expectThrows(Exception.class, () -> processor.execute(doc));
+        assertThat(e.getMessage(), equalTo("field [" + fieldName + "] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
     }
 
     public void testMissingField() {
         String fieldName = "foo.bar";
         IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
-        Grok grok = new Grok(Collections.singletonMap("ONE", "1"), "%{ONE:one}");
-        GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), grok, fieldName);
-        try {
-            processor.execute(doc);
-            fail();
-        } catch (Exception e) {
-            assertThat(e.getMessage(), equalTo("field [foo] not present as part of path [foo.bar]"));
-        }
+        GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), Collections.singletonMap("ONE", "1"),
+            Collections.singletonList("%{ONE:one}"), fieldName);
+        Exception e = expectThrows(Exception.class, () -> processor.execute(doc));
+        assertThat(e.getMessage(), equalTo("field [foo] not present as part of path [foo.bar]"));
+    }
+
+    public void testMultiplePatternsWithMatchReturn() throws Exception {
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        doc.setFieldValue(fieldName, "2");
+        Map<String, String> patternBank = new HashMap<>();
+        patternBank.put("ONE", "1");
+        patternBank.put("TWO", "2");
+        patternBank.put("THREE", "3");
+        GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), patternBank,
+            Arrays.asList("%{ONE:one}", "%{TWO:two}", "%{THREE:three}"), fieldName);
+        processor.execute(doc);
+        assertThat(doc.hasField("one"), equalTo(false));
+        assertThat(doc.getFieldValue("two", String.class), equalTo("2"));
+        assertThat(doc.hasField("three"), equalTo(false));
+    }
+
+    public void testSetMetadata() throws Exception {
+        String fieldName = RandomDocumentPicks.randomFieldName(random());
+        IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        doc.setFieldValue(fieldName, "abc23");
+        Map<String, String> patternBank = new HashMap<>();
+        patternBank.put("ONE", "1");
+        patternBank.put("TWO", "2");
+        patternBank.put("THREE", "3");
+        GrokProcessor processor = new GrokProcessor(randomAsciiOfLength(10), patternBank,
+            Arrays.asList("%{ONE:one}", "%{TWO:two}", "%{THREE:three}"), fieldName, true);
+        processor.execute(doc);
+        assertThat(doc.hasField("one"), equalTo(false));
+        assertThat(doc.getFieldValue("two", String.class), equalTo("2"));
+        assertThat(doc.hasField("three"), equalTo(false));
+        assertThat(doc.getFieldValue("_ingest._grok_match_index", String.class), equalTo("1"));
+    }
+
+    public void testCombinedPatterns() {
+        String combined;
+        combined = GrokProcessor.combinePatterns(Arrays.asList(""), false);
+        assertThat(combined, equalTo(""));
+        combined = GrokProcessor.combinePatterns(Arrays.asList(""), true);
+        assertThat(combined, equalTo(""));
+        combined = GrokProcessor.combinePatterns(Arrays.asList("foo"), false);
+        assertThat(combined, equalTo("foo"));
+        combined = GrokProcessor.combinePatterns(Arrays.asList("foo"), true);
+        assertThat(combined, equalTo("foo"));
+        combined = GrokProcessor.combinePatterns(Arrays.asList("foo", "bar"), false);
+        assertThat(combined, equalTo("(?:foo)|(?:bar)"));
+        combined = GrokProcessor.combinePatterns(Arrays.asList("foo", "bar"), true);
+        assertThat(combined, equalTo("(?<_ingest._grok_match_index.0>foo)|(?<_ingest._grok_match_index.1>bar)"));
     }
 }
diff --git a/modules/ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok/20_grok.yaml b/modules/ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok/20_grok.yaml
index f88136d8a79..a51f5a102b0 100644
--- a/modules/ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok/20_grok.yaml
+++ b/modules/ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok/20_grok.yaml
@@ -10,7 +10,7 @@
               {
                 "grok" : {
                   "field" : "field1",
-                  "pattern" : "%{NUMBER:val:float} %{NUMBER:status:int} <%{WORD:msg}>"
+                  "patterns" : ["%{NUMBER:val:float} %{NUMBER:status:int} <%{WORD:msg}>"]
                 }
               }
             ]
@@ -46,7 +46,7 @@
               {
                 "grok" : {
                   "field" : "field1",
-                  "pattern" : "<%{MY_PATTERN:msg}>",
+                  "patterns" : ["<%{MY_PATTERN:msg}>"],
                   "pattern_definitions" : {
                     "MY_PATTERN" : "foo"
                   }
@@ -83,7 +83,7 @@
               {
                 "grok" : {
                   "field" : "field1",
-                  "pattern" : "<%{NUMBER:msg}>",
+                  "patterns" : ["<%{NUMBER:msg}>"],
                   "pattern_definitions" : {
                     "NUMBER" : "foo"
                   }
@@ -107,3 +107,43 @@
         type: test
         id: 1
   - match: { _source.msg: "foo" }
+
+---
+"Test simulate with grok debugging enabled":
+  - do:
+      ingest.simulate:
+        body: >
+          {
+            "pipeline": {
+              "description": "_description",
+              "processors": [
+                {
+                  "grok" : {
+                    "field" : "field",
+                    "patterns" : ["%{ONE:one}", "%{TWO:two}"],
+                    "pattern_definitions" : {
+                      "ONE" : "1",
+                      "TWO" : "2"
+                    },
+                    "trace_match" : true
+                  }
+                }
+              ]
+            },
+            "docs": [
+              {
+                "_index": "index",
+                "_type": "type",
+                "_id": "id",
+                "_source": {
+                  "field": "abc2xyz"
+                }
+              }
+            ]
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._source.field: "abc2xyz" }
+  - match: { docs.0.doc._source.two: "2" }
+  - length: { docs.0.doc._ingest: 2 }
+  - match: { docs.0.doc._ingest._grok_match_index: "1" }
+  - is_true: docs.0.doc._ingest.timestamp
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/20_combine_processors.yaml b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/20_combine_processors.yaml
index 8c09ffb3101..45bda3146a6 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/20_combine_processors.yaml
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/20_combine_processors.yaml
@@ -9,7 +9,7 @@
               {
                 "grok" : {
                   "field" : "log",
-                  "pattern": "%{COMBINEDAPACHELOG}"
+                  "patterns": ["%{COMBINEDAPACHELOG}"]
                 }
               },
               {
@@ -55,7 +55,7 @@
         index: test
         type: test
         id: 1
-  - length: { _source: 14 }
+  - length: { _source: 13 }
   - match: { _source.request: "/presentations/logstash-scale11x/images/ahhh___rage_face_by_samusmmx-d5g5zap.png" }
   - match: { _source.agent: "\"Mozilla/5.0 (Linux; Android 4.2.2; VS980 4G Build/JDQ39B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.135 Mobile Safari/537.36\"" }
   - match: { _source.auth: "-" }

From 478877edf7f6c052efcc644f7cc3e278d3515690 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Wed, 25 May 2016 12:36:31 -0700
Subject: [PATCH 0466/1311] Plugins: Changing staging property to be the hash
 instead of a boolean

With the unified release, we will have staged releases based on a
unified hash (hash of all the hashes), so using the elasticsearch hash
for plugins staging will no longer work. This change makes the
`es.plugins.staging` property take the staging hash it should use.
---
 .../org/elasticsearch/plugins/InstallPluginCommand.java   | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
index 1b98128b85e..6ac37942cf5 100644
--- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
@@ -25,7 +25,6 @@ import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Build;
 import org.elasticsearch.Version;
 import org.elasticsearch.bootstrap.JarHell;
-import org.elasticsearch.cli.Command;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.SettingCommand;
 import org.elasticsearch.cli.Terminal;
@@ -101,7 +100,7 @@ import static org.elasticsearch.common.util.set.Sets.newHashSet;
  */
 class InstallPluginCommand extends SettingCommand {
 
-    private static final String PROPERTY_SUPPORT_STAGING_URLS = "es.plugins.staging";
+    private static final String PROPERTY_STAGING_ID = "es.plugins.staging";
 
     // TODO: make this a resource file generated by gradle
     static final Set<String> MODULES = unmodifiableSet(newHashSet(
@@ -209,12 +208,13 @@ class InstallPluginCommand extends SettingCommand {
         if (OFFICIAL_PLUGINS.contains(pluginId)) {
             final String version = Version.CURRENT.toString();
             final String url;
-            if (System.getProperty(PROPERTY_SUPPORT_STAGING_URLS, "false").equals("true")) {
+            final String stagingHash = System.getProperty(PROPERTY_STAGING_ID);
+            if (stagingHash != null) {
                 url = String.format(
                         Locale.ROOT,
                         "https://download.elastic.co/elasticsearch/staging/%1$s-%2$s/org/elasticsearch/plugin/%3$s/%1$s/%3$s-%1$s.zip",
                         version,
-                        Build.CURRENT.shortHash(),
+                        stagingHash,
                         pluginId);
             } else {
                 url = String.format(

From a9e7bdc54c88043d33f066c3926f048dae16be10 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Wed, 25 May 2016 13:31:04 -0700
Subject: [PATCH 0467/1311] Plugins: use resource files for list of modules and
 plugins

This adds modules.txt and plugins.txt to the core jar resource files,
which the install plugin command statically loads, in place of the
previously hardcoded lists (which have often gone out of date).
---
 core/build.gradle                             | 30 ++++++
 .../plugins/InstallPluginCommand.java         | 99 ++++++++++---------
 2 files changed, 81 insertions(+), 48 deletions(-)

diff --git a/core/build.gradle b/core/build.gradle
index a549f3260fa..16d21f66ef8 100644
--- a/core/build.gradle
+++ b/core/build.gradle
@@ -121,6 +121,36 @@ forbiddenPatterns {
   exclude '**/org/elasticsearch/cluster/routing/shard_routes.txt'
 }
 
+task generateModulesList {
+  List<String> modules = project(':modules').subprojects.collect { it.name }
+  File modulesFile = new File(buildDir, 'generated-resources/modules.txt')
+  processResources.from(modulesFile) 
+  inputs.property('modules', modules) 
+  outputs.file(modulesFile)
+  doLast {
+    modulesFile.parentFile.mkdirs()
+    modulesFile.setText(modules.join('\n'), 'UTF-8')
+  }
+}
+
+task generatePluginsList {
+  List<String> plugins = project(':plugins').subprojects
+    .findAll { it.name.contains('example') == false }
+    .collect { it.name }
+  File pluginsFile = new File(buildDir, 'generated-resources/plugins.txt')
+  processResources.from(pluginsFile) 
+  inputs.property('plugins', plugins) 
+  outputs.file(pluginsFile)
+  doLast {
+    pluginsFile.parentFile.mkdirs()
+    pluginsFile.setText(plugins.join('\n'), 'UTF-8')
+  }
+}
+
+processResources {
+  dependsOn generateModulesList, generatePluginsList
+}
+
 thirdPartyAudit.excludes = [
   // uses internal java api: sun.security.x509 (X509CertInfo, X509CertImpl, X500Name)
   'org.jboss.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator',
diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
index 1b98128b85e..a0465a07f2f 100644
--- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
@@ -19,23 +19,6 @@
 
 package org.elasticsearch.plugins;
 
-import joptsimple.OptionSet;
-import joptsimple.OptionSpec;
-import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.Build;
-import org.elasticsearch.Version;
-import org.elasticsearch.bootstrap.JarHell;
-import org.elasticsearch.cli.Command;
-import org.elasticsearch.cli.ExitCodes;
-import org.elasticsearch.cli.SettingCommand;
-import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.cli.UserError;
-import org.elasticsearch.common.hash.MessageDigests;
-import org.elasticsearch.common.io.FileSystemUtils;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
-
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
@@ -62,12 +45,28 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.TreeSet;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.Build;
+import org.elasticsearch.Version;
+import org.elasticsearch.bootstrap.JarHell;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.SettingCommand;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserError;
+import org.elasticsearch.common.hash.MessageDigests;
+import org.elasticsearch.common.io.FileSystemUtils;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.internal.InternalSettingsPreparer;
+
 import static java.util.Collections.unmodifiableSet;
 import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE;
-import static org.elasticsearch.common.util.set.Sets.newHashSet;
 
 /**
  * A command for the plugin cli to install a plugin into elasticsearch.
@@ -103,37 +102,41 @@ class InstallPluginCommand extends SettingCommand {
 
     private static final String PROPERTY_SUPPORT_STAGING_URLS = "es.plugins.staging";
 
-    // TODO: make this a resource file generated by gradle
-    static final Set<String> MODULES = unmodifiableSet(newHashSet(
-            "ingest-grok",
-            "lang-expression",
-            "lang-groovy",
-            "lang-painless",
-            "reindex"));
+    /** The builtin modules, which are plugins, but cannot be installed or removed. */
+    static final Set<String> MODULES;
+    static {
+        try (InputStream stream = InstallPluginCommand.class.getResourceAsStream("/modules.txt");
+             BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
+            Set<String> modules = new HashSet<>();
+            String line = reader.readLine();
+            while (line != null) {
+                modules.add(line.trim());
+                line = reader.readLine();
+            }
+            MODULES = Collections.unmodifiableSet(modules);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
 
-    // TODO: make this a resource file generated by gradle
-    static final Set<String> OFFICIAL_PLUGINS = unmodifiableSet(new LinkedHashSet<>(Arrays.asList(
-            "analysis-icu",
-            "analysis-kuromoji",
-            "analysis-phonetic",
-            "analysis-smartcn",
-            "analysis-stempel",
-            "discovery-azure",
-            "discovery-ec2",
-            "discovery-gce",
-            "ingest-attachment",
-            "ingest-geoip",
-            "lang-javascript",
-            "lang-python",
-            "mapper-attachments",
-            "mapper-murmur3",
-            "mapper-size",
-            "repository-azure",
-            "repository-gcs",
-            "repository-hdfs",
-            "repository-s3",
-            "store-smb",
-            "x-pack")));
+    /** The official plugins that can be installed simply by name. */
+    static final Set<String> OFFICIAL_PLUGINS;
+    static {
+        try (InputStream stream = InstallPluginCommand.class.getResourceAsStream("/plugins.txt");
+             BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
+            Set<String> plugins = new TreeSet<>(); // use tree set to get sorting for help command
+            String line = reader.readLine();
+            while (line != null) {
+                plugins.add(line.trim());
+                line = reader.readLine();
+            }
+            plugins.add("x-pack");
+            OFFICIAL_PLUGINS = Collections.unmodifiableSet(plugins);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+
+    }
 
     private final OptionSpec<Void> batchOption;
     private final OptionSpec<String> arguments;

From 9d39b0584536a3275d164e084ffe53ed0e35c09e Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 25 May 2016 17:15:36 -0400
Subject: [PATCH 0468/1311] Remove deprecation suppression

Failing the build on deprecation warnings was removed in
19b3ec88af95166fd654ce2f8d62a1f9b473ba8f. This commit removes the
suppressed deprecation warnings so that their use is surfaced in the
build now.

Relates #18582
---
 core/src/main/java/org/elasticsearch/Version.java               | 1 -
 .../elasticsearch/index/analysis/StopTokenFilterFactory.java    | 1 -
 .../java/org/elasticsearch/index/store/FsDirectoryService.java  | 1 -
 core/src/main/java/org/elasticsearch/index/store/Store.java     | 1 -
 .../elasticsearch/index/query/QueryDSLDocumentationTests.java   | 1 -
 .../org/elasticsearch/search/highlight/HighlighterSearchIT.java | 1 -
 .../elasticsearch/search/matchedqueries/MatchedQueriesIT.java   | 1 -
 .../search/profile/query/RandomQueryGenerator.java              | 2 --
 .../test/java/org/elasticsearch/search/query/SearchQueryIT.java | 1 -
 .../org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java | 1 -
 .../java/org/elasticsearch/validate/SimpleValidateQueryIT.java  | 1 -
 .../index/analysis/IcuCollationTokenFilterFactory.java          | 2 --
 .../org/elasticsearch/mapper/attachments/AttachmentMapper.java  | 1 -
 .../java/org/elasticsearch/common/io/PathUtilsForTesting.java   | 1 -
 .../main/java/org/elasticsearch/test/InternalTestCluster.java   | 1 -
 .../org/elasticsearch/test/store/MockFSDirectoryService.java    | 1 -
 16 files changed, 18 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java
index b2b4e9c296d..c220c297388 100644
--- a/core/src/main/java/org/elasticsearch/Version.java
+++ b/core/src/main/java/org/elasticsearch/Version.java
@@ -32,7 +32,6 @@ import java.io.IOException;
 
 /**
  */
-@SuppressWarnings("deprecation")
 public class Version {
     /*
      * The logic for ID is: XXYYZZAA, where XX is major version, YY is minor version, ZZ is revision, and AA is alpha/beta/rc indicator AA
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/StopTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/StopTokenFilterFactory.java
index 322fcea452f..e724f064bb4 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/StopTokenFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/StopTokenFilterFactory.java
@@ -34,7 +34,6 @@ import java.util.Set;
 /**
  *
  */
-@SuppressWarnings("deprecation")
 public class StopTokenFilterFactory extends AbstractTokenFilterFactory {
 
     private final CharArraySet stopWords;
diff --git a/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java b/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java
index d41697cd14c..5ad5c232f2e 100644
--- a/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java
+++ b/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java
@@ -129,7 +129,6 @@ public class FsDirectoryService extends DirectoryService implements StoreRateLim
         throw new IllegalArgumentException("No directory found for type [" + storeType + "]");
     }
 
-    @SuppressWarnings("deprecation")
     private static boolean isDefault(String storeType) {
         return IndexModule.Type.DEFAULT.match(storeType);
     }
diff --git a/core/src/main/java/org/elasticsearch/index/store/Store.java b/core/src/main/java/org/elasticsearch/index/store/Store.java
index d3aa93b6434..bb162880384 100644
--- a/core/src/main/java/org/elasticsearch/index/store/Store.java
+++ b/core/src/main/java/org/elasticsearch/index/store/Store.java
@@ -821,7 +821,6 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
                 final SegmentInfos segmentCommitInfos = Store.readSegmentsInfo(commit, directory);
                 numDocs = Lucene.getNumDocs(segmentCommitInfos);
                 commitUserDataBuilder.putAll(segmentCommitInfos.getUserData());
-                @SuppressWarnings("deprecation")
                 Version maxVersion = segmentCommitInfos.getMinSegmentLuceneVersion(); // we don't know which version was used to write so we take the max version.
                 for (SegmentCommitInfo info : segmentCommitInfos) {
                     final Version version = info.info.getVersion();
diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryDSLDocumentationTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryDSLDocumentationTests.java
index d0f2236ce06..1184c5b3823 100644
--- a/core/src/test/java/org/elasticsearch/index/query/QueryDSLDocumentationTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/QueryDSLDocumentationTests.java
@@ -138,7 +138,6 @@ public class QueryDSLDocumentationTests extends ESTestCase {
         functionScoreQuery(functions);
     }
 
-    @SuppressWarnings("deprecation") // fuzzy queries will be removed in 4.0
     public void testFuzzy() {
         fuzzyQuery("name", "kimchy");
     }
diff --git a/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java b/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java
index 876c8b53d87..8d7cdef4ca8 100644
--- a/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/highlight/HighlighterSearchIT.java
@@ -2269,7 +2269,6 @@ public class HighlighterSearchIT extends ESIntegTestCase {
 
     }
 
-    @SuppressWarnings("deprecation") // fuzzy queries will be removed in 4.0
     public void testPostingsHighlighterFuzzyQuery() throws Exception {
         assertAcked(prepareCreate("test").addMapping("type1", type1PostingsffsetsMapping()));
         ensureGreen();
diff --git a/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java b/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java
index 00fa879f08f..ef7a4ecc7ce 100644
--- a/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java
+++ b/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java
@@ -252,7 +252,6 @@ public class MatchedQueriesIT extends ESIntegTestCase {
         }
     }
 
-    @SuppressWarnings("deprecation") // fuzzy queries will be removed in 4.0
     public void testFuzzyQuerySupportsName() {
         createIndex("test1");
         ensureGreen();
diff --git a/core/src/test/java/org/elasticsearch/search/profile/query/RandomQueryGenerator.java b/core/src/test/java/org/elasticsearch/search/profile/query/RandomQueryGenerator.java
index 7fb896e1b2f..d40f85aa04e 100644
--- a/core/src/test/java/org/elasticsearch/search/profile/query/RandomQueryGenerator.java
+++ b/core/src/test/java/org/elasticsearch/search/profile/query/RandomQueryGenerator.java
@@ -72,7 +72,6 @@ public class RandomQueryGenerator {
         }
     }
 
-    @SuppressWarnings("deprecation") // fuzzy queries will be removed in 4.0
     private static QueryBuilder randomTerminalQuery(List<String> stringFields, List<String> numericFields, int numDocs) {
         switch (randomIntBetween(0,6)) {
             case 0:
@@ -196,7 +195,6 @@ public class RandomQueryGenerator {
         return q;
     }
 
-    @SuppressWarnings("deprecation") // fuzzy queries will be removed in 4.0
     @Deprecated
     private static QueryBuilder randomFuzzyQuery(List<String> fields) {
 
diff --git a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
index 15ee5822486..d7d4a3fd188 100644
--- a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
@@ -1440,7 +1440,6 @@ public class SearchQueryIT extends ESIntegTestCase {
         assertHitCount(searchResponse, 3L);
     }
 
-    @SuppressWarnings("deprecation") // fuzzy queries will be removed in 4.0
     public void testSpanMultiTermQuery() throws IOException {
         createIndex("test");
 
diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java
index d40158129dc..49bd306c1d8 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderIT.java
@@ -185,7 +185,6 @@ public class GeoDistanceSortBuilderIT extends ESIntegTestCase {
         builder.endObject();
     }
 
-    @SuppressWarnings("deprecation")
     public void testManyToManyGeoPointsWithDifferentFormats() throws ExecutionException, InterruptedException, IOException {
         /**   q     d1       d2
          * |4  o|   x    |   x
diff --git a/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java b/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java
index 2b163a4d7ea..79ee4389527 100644
--- a/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java
+++ b/core/src/test/java/org/elasticsearch/validate/SimpleValidateQueryIT.java
@@ -212,7 +212,6 @@ public class SimpleValidateQueryIT extends ESIntegTestCase {
         assertThat(validateQueryResponse.getQueryExplanation().get(0).getExplanation(), containsString("field:\"foo (one* two*)\""));
     }
 
-    @SuppressWarnings("deprecation") // fuzzy queries will be removed in 4.0
     public void testExplainWithRewriteValidateQuery() throws Exception {
         client().admin().indices().prepareCreate("test")
                 .addMapping("type1", "field", "type=text,analyzer=whitespace")
diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuCollationTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuCollationTokenFilterFactory.java
index a8b71f3f2f9..091189f361a 100644
--- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuCollationTokenFilterFactory.java
+++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuCollationTokenFilterFactory.java
@@ -46,7 +46,6 @@ public class IcuCollationTokenFilterFactory extends AbstractTokenFilterFactory {
 
     private final Collator collator;
 
-    @SuppressWarnings("deprecation") // Intentionally sets deprecated options for backwards compatibility
     public IcuCollationTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
 
@@ -167,7 +166,6 @@ public class IcuCollationTokenFilterFactory extends AbstractTokenFilterFactory {
     }
 
     @Override
-    @SuppressWarnings("deprecation") // Constructs a deprecated filter for backwards compatibility
     public TokenStream create(TokenStream tokenStream) {
         return new ICUCollationKeyFilter(tokenStream, collator);
     }
diff --git a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java
index 09667733251..79174e54c62 100644
--- a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java
+++ b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java
@@ -427,7 +427,6 @@ public class AttachmentMapper extends FieldMapper {
     }
 
     @Override
-    @SuppressWarnings("deprecation") // https://github.com/elastic/elasticsearch/issues/15843
     public Mapper parse(ParseContext context) throws IOException {
         byte[] content = null;
         String contentType = null;
diff --git a/test/framework/src/main/java/org/elasticsearch/common/io/PathUtilsForTesting.java b/test/framework/src/main/java/org/elasticsearch/common/io/PathUtilsForTesting.java
index 36f766b8067..e576d177066 100644
--- a/test/framework/src/main/java/org/elasticsearch/common/io/PathUtilsForTesting.java
+++ b/test/framework/src/main/java/org/elasticsearch/common/io/PathUtilsForTesting.java
@@ -29,7 +29,6 @@ import java.nio.file.FileSystem;
 public class PathUtilsForTesting {
 
     /** Sets a new default filesystem for testing */
-    @SuppressWarnings("deprecation") // https://github.com/elastic/elasticsearch/issues/15845
     public static void setup() {
         installMock(LuceneTestCase.getBaseTempDirForTestClass().getFileSystem());
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index 69f811b7d07..aa1d0aa29ef 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -1033,7 +1033,6 @@ public final class InternalTestCluster extends TestCluster {
         }
     }
 
-    @SuppressWarnings("deprecation") // https://github.com/elastic/elasticsearch/issues/15844
     private void randomlyResetClients() throws IOException {
         // only reset the clients on nightly tests, it causes heavy load...
         if (RandomizedTest.isNightly() && rarely(random)) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java
index 7d17746d54f..7b1c3fd936b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java
@@ -113,7 +113,6 @@ public class MockFSDirectoryService extends FsDirectoryService {
         throw new UnsupportedOperationException();
     }
 
-    @SuppressWarnings("deprecation") // https://github.com/elastic/elasticsearch/issues/15846
     public static void checkIndex(ESLogger logger, Store store, ShardId shardId) {
         if (store.tryIncRef()) {
             logger.info("start check index");

From 5e7b8d7788b659d6ae41ad8c41efb794739b521a Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Wed, 25 May 2016 14:21:33 -0700
Subject: [PATCH 0469/1311] Add test for official plugins list being sorted

---
 .../plugins/InstallPluginCommandTests.java    | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
index 22b2ef39a88..14910ffcdeb 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
@@ -36,8 +36,10 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.PosixPermissionsResetter;
 import org.junit.After;
 
+import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.StringReader;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
@@ -532,6 +534,29 @@ public class InstallPluginCommandTests extends ESTestCase {
         assertTrue(e.getMessage(), e.getMessage().contains("resolving outside of plugin directory"));
     }
 
+    public void testOfficialPluginsHelpSorted() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        new InstallPluginCommand().main(new String[] { "--help" }, terminal);
+        try (BufferedReader reader = new BufferedReader(new StringReader(terminal.getOutput()))) {
+            String line = reader.readLine();
+
+            // first find the beginning of our list of official plugins
+            while (line.endsWith("may be installed by name:") == false) {
+                line = reader.readLine();
+            }
+
+            // now check each line compares greater than the last, until we reach an empty line
+            String prev = reader.readLine();
+            line = reader.readLine();
+            while (line != null && line.trim().isEmpty() == false) {
+                assertTrue(prev + " < " + line, prev.compareTo(line) < 0);
+                prev = line;
+                line = reader.readLine();
+            }
+        }
+        terminal.getOutput();
+    }
+
     // TODO: test batch flag?
     // TODO: test checksum (need maven/official below)
     // TODO: test maven, official, and staging install...need tests with fixtures...

From 45adab0cb82a96ce1a80c69355cbfda66986117a Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Wed, 25 May 2016 14:23:57 -0700
Subject: [PATCH 0470/1311] Add test that x-pack is in official plugins list

---
 .../org/elasticsearch/plugins/InstallPluginCommand.java     | 1 -
 .../elasticsearch/plugins/InstallPluginCommandTests.java    | 6 ++++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
index a0465a07f2f..18a63fa1b44 100644
--- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
@@ -135,7 +135,6 @@ class InstallPluginCommand extends SettingCommand {
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
-
     }
 
     private final OptionSpec<Void> batchOption;
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
index 14910ffcdeb..f23b3f3d987 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
@@ -557,6 +557,12 @@ public class InstallPluginCommandTests extends ESTestCase {
         terminal.getOutput();
     }
 
+    public void testOfficialPluginsIncludesXpack() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        new InstallPluginCommand().main(new String[] { "--help" }, terminal);
+        assertTrue(terminal.getOutput(), terminal.getOutput().contains("x-pack"));
+    }
+
     // TODO: test batch flag?
     // TODO: test checksum (need maven/official below)
     // TODO: test maven, official, and staging install...need tests with fixtures...

From c951e03edc47931db3a29a666169c3c2e413eae8 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Wed, 25 May 2016 14:38:10 -0700
Subject: [PATCH 0471/1311] Remove unused line in official plugins test.

---
 .../org/elasticsearch/plugins/InstallPluginCommandTests.java     | 1 -
 1 file changed, 1 deletion(-)

diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
index f23b3f3d987..2ca46b4effc 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
@@ -554,7 +554,6 @@ public class InstallPluginCommandTests extends ESTestCase {
                 line = reader.readLine();
             }
         }
-        terminal.getOutput();
     }
 
     public void testOfficialPluginsIncludesXpack() throws Exception {

From 63549ec6f862232dd3898b7a6dee929eb7584e6b Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Wed, 25 May 2016 14:44:56 -0700
Subject: [PATCH 0472/1311] Update usage of es.staging.plugins sysprop to pass
 build hash

---
 dev-tools/smoke_test_rc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev-tools/smoke_test_rc.py b/dev-tools/smoke_test_rc.py
index 32269e319bb..259081ddc10 100644
--- a/dev-tools/smoke_test_rc.py
+++ b/dev-tools/smoke_test_rc.py
@@ -191,7 +191,7 @@ def smoke_test_release(release, files, expected_hash, plugins):
     plugin_names = {}
     for plugin  in plugins:
       print('     Install plugin [%s]' % (plugin))
-      run('%s; export ES_JAVA_OPTS="-Des.plugins.staging=true"; %s %s %s' % (java_exe(), es_plugin_path, 'install -b', plugin))
+      run('%s; export ES_JAVA_OPTS="-Des.plugins.staging=%s"; %s %s %s' % (java_exe(), expected_hash, es_plugin_path, 'install -b', plugin))
       plugin_names[plugin] = True
     if 'x-pack' in plugin_names:
       headers = { 'Authorization' : 'Basic %s' % base64.b64encode(b"es_admin:foobar").decode("UTF-8") }

From cad959b9804c0161b7304f6498b5040bac92a0a4 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Thu, 26 May 2016 10:07:38 +0200
Subject: [PATCH 0473/1311] Validate parameters of native sig score scripts so
 that we know which ones are not set.

---
 .../script/NativeSignificanceScoreScriptNoParams.java    | 1 +
 .../script/NativeSignificanceScoreScriptWithParams.java  | 3 +++
 .../search/aggregations/bucket/script/TestScript.java    | 9 +++++++++
 3 files changed, 13 insertions(+)

diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptNoParams.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptNoParams.java
index 313734cc9cc..e770f98b2d5 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptNoParams.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptNoParams.java
@@ -47,6 +47,7 @@ public class NativeSignificanceScoreScriptNoParams extends TestScript {
 
     @Override
     public Object run() {
+        checkParams();
         return _subset_freq.longValue() + _subset_size.longValue() + _superset_freq.longValue() + _superset_size.longValue();
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptWithParams.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptWithParams.java
index 6b248fe56fa..e4fda62c29d 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptWithParams.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptWithParams.java
@@ -24,6 +24,7 @@ import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.NativeScriptFactory;
 
 import java.util.Map;
+import java.util.Objects;
 
 public class NativeSignificanceScoreScriptWithParams extends TestScript {
 
@@ -49,6 +50,8 @@ public class NativeSignificanceScoreScriptWithParams extends TestScript {
 
     @Override
     public Object run() {
+        checkParams();
+        Objects.requireNonNull(factor, "factor");
         return factor * (_subset_freq.longValue() + _subset_size.longValue() + _superset_freq.longValue() + _superset_size.longValue()) / factor;
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/TestScript.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/TestScript.java
index 3060d7af818..13c1340de31 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/TestScript.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/TestScript.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.search.aggregations.bucket.script;
 
+import java.util.Objects;
+
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.ScriptHeuristic;
 
@@ -48,6 +50,13 @@ public abstract class TestScript implements ExecutableScript{
         }
     }
 
+    protected final void checkParams() {
+        Objects.requireNonNull(_subset_freq, "_subset_freq");
+        Objects.requireNonNull(_subset_size, "_subset_size");
+        Objects.requireNonNull(_superset_freq, "_superset_freq");
+        Objects.requireNonNull(_superset_size, "_superset_size");
+    }
+
     @Override
     public Double unwrap(Object value) {
         return ((Number) value).doubleValue();

From 31b0777c911a87b7ebae0efd7d5b14d1e06f3879 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Thu, 26 May 2016 13:39:55 +0200
Subject: [PATCH 0474/1311] Simplify delayed shard allocation (#18351)

This commit simplifies the delayed shard allocation implementation by assigning clear responsibilities to the various components that are affected by delayed shard allocation:

- UnassignedInfo gets a boolean flag delayed which determines whether assignment of the shard should be delayed. The flag gets persisted in the cluster state and is thus available across nodes, i.e. each node knows whether a shard was delayed-unassigned in a specific cluster state. Before, nodes other than the current master were unaware of that information.
- This flag is initially set as true if the shard becomes unassigned due to a node leaving and the index setting index.unassigned.node_left.delayed_timeout being strictly positive. From then on, unassigned shards can only transition from delayed to non-delayed, never in the other direction.
- The reroute step is in charge of removing the delay marker (comparing timestamp when node left to current timestamp).
- A dedicated service DelayedAllocationService, reacting to cluster change events, has the responsibility to schedule reroutes to remove the delay marker.

Closes #18293
---
 .../ClusterAllocationExplanation.java         |  18 +-
 ...ansportClusterAllocationExplainAction.java |  13 +-
 .../elasticsearch/cluster/ClusterModule.java  |   2 +
 .../routing/DelayedAllocationService.java     | 225 ++++++++
 .../cluster/routing/RoutingService.java       |  61 +--
 .../cluster/routing/ShardRouting.java         |   1 +
 .../cluster/routing/UnassignedInfo.java       | 153 ++----
 .../routing/allocation/AllocationService.java | 112 ++--
 .../allocation/FailedRerouteAllocation.java   |   4 +-
 .../routing/allocation/RoutingAllocation.java |   6 +-
 .../allocation/StartedRerouteAllocation.java  |   4 +-
 ...AllocateEmptyPrimaryAllocationCommand.java |   2 +-
 .../cluster/service/ClusterService.java       |   2 +-
 .../discovery/zen/NodeJoinController.java     |  14 +-
 .../discovery/zen/ZenDiscovery.java           |  12 +-
 .../gateway/ReplicaShardAllocator.java        |  15 +-
 .../ClusterAllocationExplanationTests.java    |  17 +-
 .../cluster/routing/DelayedAllocationIT.java  |   9 +-
 .../DelayedAllocationServiceTests.java        | 510 ++++++++++++++++++
 .../cluster/routing/RoutingServiceTests.java  | 195 +------
 .../cluster/routing/UnassignedInfoTests.java  |  68 +--
 .../zen/NodeJoinControllerTests.java          |   2 +-
 .../gateway/ReplicaShardAllocatorTests.java   |  10 +-
 .../test/ESAllocationTestCase.java            |   9 +-
 24 files changed, 923 insertions(+), 541 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/cluster/routing/DelayedAllocationService.java
 create mode 100644 core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
index 9ca0efd6c56..e007929faf2 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java
@@ -45,17 +45,19 @@ public final class ClusterAllocationExplanation implements ToXContent, Writeable
     private final boolean hasPendingAsyncFetch;
     private final String assignedNodeId;
     private final UnassignedInfo unassignedInfo;
+    private final long allocationDelayMillis;
     private final long remainingDelayMillis;
     private final Map<DiscoveryNode, NodeExplanation> nodeExplanations;
 
-    public ClusterAllocationExplanation(ShardId shard, boolean primary, @Nullable String assignedNodeId, long remainingDelayMillis,
-                                        @Nullable UnassignedInfo unassignedInfo, boolean hasPendingAsyncFetch,
+    public ClusterAllocationExplanation(ShardId shard, boolean primary, @Nullable String assignedNodeId, long allocationDelayMillis,
+                                        long remainingDelayMillis, @Nullable UnassignedInfo unassignedInfo, boolean hasPendingAsyncFetch,
                                         Map<DiscoveryNode, NodeExplanation> nodeExplanations) {
         this.shard = shard;
         this.primary = primary;
         this.hasPendingAsyncFetch = hasPendingAsyncFetch;
         this.assignedNodeId = assignedNodeId;
         this.unassignedInfo = unassignedInfo;
+        this.allocationDelayMillis = allocationDelayMillis;
         this.remainingDelayMillis = remainingDelayMillis;
         this.nodeExplanations = nodeExplanations;
     }
@@ -66,6 +68,7 @@ public final class ClusterAllocationExplanation implements ToXContent, Writeable
         this.hasPendingAsyncFetch = in.readBoolean();
         this.assignedNodeId = in.readOptionalString();
         this.unassignedInfo = in.readOptionalWriteable(UnassignedInfo::new);
+        this.allocationDelayMillis = in.readVLong();
         this.remainingDelayMillis = in.readVLong();
 
         int mapSize = in.readVInt();
@@ -84,6 +87,7 @@ public final class ClusterAllocationExplanation implements ToXContent, Writeable
         out.writeBoolean(this.isStillFetchingShardData());
         out.writeOptionalString(this.getAssignedNodeId());
         out.writeOptionalWriteable(this.getUnassignedInfo());
+        out.writeVLong(allocationDelayMillis);
         out.writeVLong(remainingDelayMillis);
 
         out.writeVInt(this.nodeExplanations.size());
@@ -124,7 +128,12 @@ public final class ClusterAllocationExplanation implements ToXContent, Writeable
         return this.unassignedInfo;
     }
 
-    /** Return the remaining allocation delay for this shard in millisocends */
+    /** Return the configured delay before the shard can be allocated in milliseconds */
+    public long getAllocationDelayMillis() {
+        return this.allocationDelayMillis;
+    }
+
+    /** Return the remaining allocation delay for this shard in milliseconds */
     public long getRemainingDelayMillis() {
         return this.remainingDelayMillis;
     }
@@ -152,8 +161,7 @@ public final class ClusterAllocationExplanation implements ToXContent, Writeable
             // If we have unassigned info, show that
             if (unassignedInfo != null) {
                 unassignedInfo.toXContent(builder, params);
-                long delay = unassignedInfo.getLastComputedLeftDelayNanos();
-                builder.timeValueField("allocation_delay_in_millis", "allocation_delay", TimeValue.timeValueNanos(delay));
+                builder.timeValueField("allocation_delay_in_millis", "allocation_delay", TimeValue.timeValueMillis(allocationDelayMillis));
                 builder.timeValueField("remaining_delay_in_millis", "remaining_delay", TimeValue.timeValueMillis(remainingDelayMillis));
             }
             builder.startObject("nodes");
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
index 46a4d1795ba..c5e1f6cc2a1 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
@@ -59,6 +59,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING;
+
 /**
  * The {@code TransportClusterAllocationExplainAction} is responsible for actually executing the explanation of a shard's allocation on the
  * master node in the cluster.
@@ -237,9 +239,9 @@ public class TransportClusterAllocationExplainAction
         long remainingDelayMillis = 0;
         final MetaData metadata = allocation.metaData();
         final IndexMetaData indexMetaData = metadata.index(shard.index());
-        if (ui != null) {
-            final Settings indexSettings = indexMetaData.getSettings();
-            long remainingDelayNanos = ui.getRemainingDelay(System.nanoTime(), metadata.settings(), indexSettings);
+        long allocationDelayMillis = INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.get(indexMetaData.getSettings()).getMillis();
+        if (ui != null && ui.isDelayed()) {
+            long remainingDelayNanos = ui.getRemainingDelay(System.nanoTime(), indexMetaData.getSettings());
             remainingDelayMillis = TimeValue.timeValueNanos(remainingDelayNanos).millis();
         }
 
@@ -262,8 +264,9 @@ public class TransportClusterAllocationExplainAction
                     allocation.hasPendingAsyncFetch());
             explanations.put(node, nodeExplanation);
         }
-        return new ClusterAllocationExplanation(shard.shardId(), shard.primary(), shard.currentNodeId(),
-                remainingDelayMillis, ui, gatewayAllocator.hasFetchPending(shard.shardId(), shard.primary()), explanations);
+        return new ClusterAllocationExplanation(shard.shardId(), shard.primary(),
+            shard.currentNodeId(), allocationDelayMillis, remainingDelayMillis, ui,
+            gatewayAllocator.hasFetchPending(shard.shardId(), shard.primary()), explanations);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index a02e399ac0c..707521b2cfe 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -33,6 +33,7 @@ import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService;
 import org.elasticsearch.cluster.metadata.MetaDataMappingService;
 import org.elasticsearch.cluster.metadata.MetaDataUpdateSettingsService;
 import org.elasticsearch.cluster.node.DiscoveryNodeService;
+import org.elasticsearch.cluster.routing.DelayedAllocationService;
 import org.elasticsearch.cluster.routing.OperationRouting;
 import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
@@ -151,6 +152,7 @@ public class ClusterModule extends AbstractModule {
         bind(MetaDataIndexTemplateService.class).asEagerSingleton();
         bind(IndexNameExpressionResolver.class).asEagerSingleton();
         bind(RoutingService.class).asEagerSingleton();
+        bind(DelayedAllocationService.class).asEagerSingleton();
         bind(ShardStateAction.class).asEagerSingleton();
         bind(NodeIndexDeletedAction.class).asEagerSingleton();
         bind(NodeMappingRefreshAction.class).asEagerSingleton();
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/DelayedAllocationService.java b/core/src/main/java/org/elasticsearch/cluster/routing/DelayedAllocationService.java
new file mode 100644
index 00000000000..e79884cf9c9
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/DelayedAllocationService.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing;
+
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.FutureUtils;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * The {@link DelayedAllocationService} listens to cluster state changes and checks
+ * if there are unassigned shards with delayed allocation (unassigned shards that have
+ * the delay marker). These are shards that have become unassigned due to a node leaving
+ * and which were assigned the delay marker based on the index delay setting
+ * {@link UnassignedInfo#INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING}
+ * (see {@link AllocationService#deassociateDeadNodes(RoutingAllocation)}).
+ * This class is responsible for choosing the next (closest) delay expiration of a
+ * delayed shard to schedule a reroute to remove the delay marker.
+ * The actual removal of the delay marker happens in
+ * {@link AllocationService#removeDelayMarkers(RoutingAllocation)}, triggering yet
+ * another cluster change event.
+ */
+public class DelayedAllocationService extends AbstractLifecycleComponent<DelayedAllocationService> implements ClusterStateListener {
+
+    static final String CLUSTER_UPDATE_TASK_SOURCE = "delayed_allocation_reroute";
+
+    final ThreadPool threadPool;
+    private final ClusterService clusterService;
+    private final AllocationService allocationService;
+
+    AtomicReference<DelayedRerouteTask> delayedRerouteTask = new AtomicReference<>(); // package private to access from tests
+
+    /**
+     * represents a delayed scheduling of the reroute action that can be cancelled.
+     */
+    class DelayedRerouteTask extends ClusterStateUpdateTask {
+        final TimeValue nextDelay; // delay until submitting the reroute command
+        final long baseTimestampNanos; // timestamp (in nanos) upon which delay was calculated
+        volatile ScheduledFuture future;
+        final AtomicBoolean cancelScheduling = new AtomicBoolean();
+
+        DelayedRerouteTask(TimeValue nextDelay, long baseTimestampNanos) {
+            this.nextDelay = nextDelay;
+            this.baseTimestampNanos = baseTimestampNanos;
+        }
+
+        public long scheduledTimeToRunInNanos() {
+            return baseTimestampNanos + nextDelay.nanos();
+        }
+
+        public void cancelScheduling() {
+            cancelScheduling.set(true);
+            FutureUtils.cancel(future);
+            removeIfSameTask(this);
+        }
+
+        public void schedule() {
+            future = threadPool.schedule(nextDelay, ThreadPool.Names.SAME, new AbstractRunnable() {
+                @Override
+                protected void doRun() throws Exception {
+                    if (cancelScheduling.get()) {
+                        return;
+                    }
+                    clusterService.submitStateUpdateTask(CLUSTER_UPDATE_TASK_SOURCE, DelayedRerouteTask.this);
+                }
+
+                @Override
+                public void onFailure(Throwable t) {
+                    logger.warn("failed to submit schedule/execute reroute post unassigned shard", t);
+                    removeIfSameTask(DelayedRerouteTask.this);
+                }
+            });
+        }
+
+        @Override
+        public ClusterState execute(ClusterState currentState) throws Exception {
+            removeIfSameTask(this);
+            RoutingAllocation.Result routingResult = allocationService.reroute(currentState, "assign delayed unassigned shards");
+            if (routingResult.changed()) {
+                return ClusterState.builder(currentState).routingResult(routingResult).build();
+            } else {
+                return currentState;
+            }
+        }
+
+        @Override
+        public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+            if (oldState == newState) {
+                // no state changed, check when we should remove the delay flag from the shards the next time.
+                // if cluster state changed, we can leave the scheduling of the next delay up to the clusterChangedEvent
+                // this should not be needed, but we want to be extra safe here
+                scheduleIfNeeded(currentNanoTime(), newState);
+            }
+        }
+
+        @Override
+        public void onFailure(String source, Throwable t) {
+            removeIfSameTask(this);
+            logger.warn("failed to schedule/execute reroute post unassigned shard", t);
+        }
+    }
+
+    @Inject
+    public DelayedAllocationService(Settings settings, ThreadPool threadPool, ClusterService clusterService,
+                                    AllocationService allocationService) {
+        super(settings);
+        this.threadPool = threadPool;
+        this.clusterService = clusterService;
+        this.allocationService = allocationService;
+        clusterService.addFirst(this);
+    }
+
+    @Override
+    protected void doStart() {
+    }
+
+    @Override
+    protected void doStop() {
+    }
+
+    @Override
+    protected void doClose() {
+        clusterService.remove(this);
+        removeTaskAndCancel();
+    }
+
+    /** override this to control time based decisions during delayed allocation */
+    protected long currentNanoTime() {
+        return System.nanoTime();
+    }
+
+    @Override
+    public void clusterChanged(ClusterChangedEvent event) {
+        long currentNanoTime = currentNanoTime();
+        if (event.state().nodes().isLocalNodeElectedMaster()) {
+            scheduleIfNeeded(currentNanoTime, event.state());
+        }
+    }
+
+    private void removeTaskAndCancel() {
+        DelayedRerouteTask existingTask = delayedRerouteTask.getAndSet(null);
+        if (existingTask != null) {
+            logger.trace("cancelling existing delayed reroute task");
+            existingTask.cancelScheduling();
+        }
+    }
+
+    private void removeIfSameTask(DelayedRerouteTask expectedTask) {
+        delayedRerouteTask.compareAndSet(expectedTask, null);
+    }
+
+    /**
+     * Figure out if an existing scheduled reroute is good enough or whether we need to cancel and reschedule.
+     */
+    private void scheduleIfNeeded(long currentNanoTime, ClusterState state) {
+        assertClusterStateThread();
+        long nextDelayNanos = UnassignedInfo.findNextDelayedAllocation(currentNanoTime, state);
+        if (nextDelayNanos < 0) {
+            logger.trace("no need to schedule reroute - no delayed unassigned shards");
+            removeTaskAndCancel();
+        } else {
+            TimeValue nextDelay = TimeValue.timeValueNanos(nextDelayNanos);
+            final boolean earlierRerouteNeeded;
+            DelayedRerouteTask existingTask = delayedRerouteTask.get();
+            DelayedRerouteTask newTask = new DelayedRerouteTask(nextDelay, currentNanoTime);
+            if (existingTask == null) {
+                earlierRerouteNeeded = true;
+            } else if (newTask.scheduledTimeToRunInNanos() < existingTask.scheduledTimeToRunInNanos()) {
+                // we need an earlier delayed reroute
+                logger.trace("cancelling existing delayed reroute task as delayed reroute has to happen [{}] earlier",
+                    TimeValue.timeValueNanos(existingTask.scheduledTimeToRunInNanos() - newTask.scheduledTimeToRunInNanos()));
+                existingTask.cancelScheduling();
+                earlierRerouteNeeded = true;
+            } else {
+                earlierRerouteNeeded = false;
+            }
+
+            if (earlierRerouteNeeded) {
+                logger.info("scheduling reroute for delayed shards in [{}] ({} delayed shards)", nextDelay,
+                    UnassignedInfo.getNumberOfDelayedUnassigned(state));
+                DelayedRerouteTask currentTask = delayedRerouteTask.getAndSet(newTask);
+                assert existingTask == currentTask || currentTask == null;
+                newTask.schedule();
+            } else {
+                logger.trace("no need to reschedule delayed reroute - currently scheduled delayed reroute in [{}] is enough", nextDelay);
+            }
+        }
+    }
+
+    // protected so that it can be overridden (and disabled) by unit tests
+    protected void assertClusterStateThread() {
+        ClusterService.assertClusterStateThread();
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
index 1ebd4699d1a..78e7e15d389 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
@@ -21,7 +21,6 @@ package org.elasticsearch.cluster.routing;
 
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
@@ -30,12 +29,7 @@ import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.common.util.concurrent.FutureUtils;
-import org.elasticsearch.threadpool.ThreadPool;
 
-import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
@@ -50,27 +44,20 @@ import java.util.concurrent.atomic.AtomicBoolean;
  * actions.
  * </p>
  */
-public class RoutingService extends AbstractLifecycleComponent<RoutingService> implements ClusterStateListener {
+public class RoutingService extends AbstractLifecycleComponent<RoutingService> {
 
     private static final String CLUSTER_UPDATE_TASK_SOURCE = "cluster_reroute";
 
-    final ThreadPool threadPool;
     private final ClusterService clusterService;
     private final AllocationService allocationService;
 
     private AtomicBoolean rerouting = new AtomicBoolean();
-    private volatile long minDelaySettingAtLastSchedulingNanos = Long.MAX_VALUE;
-    private volatile ScheduledFuture registeredNextDelayFuture;
 
     @Inject
-    public RoutingService(Settings settings, ThreadPool threadPool, ClusterService clusterService, AllocationService allocationService) {
+    public RoutingService(Settings settings, ClusterService clusterService, AllocationService allocationService) {
         super(settings);
-        this.threadPool = threadPool;
         this.clusterService = clusterService;
         this.allocationService = allocationService;
-        if (clusterService != null) {
-            clusterService.addFirst(this);
-        }
     }
 
     @Override
@@ -83,8 +70,6 @@ public class RoutingService extends AbstractLifecycleComponent<RoutingService> i
 
     @Override
     protected void doClose() {
-        FutureUtils.cancel(registeredNextDelayFuture);
-        clusterService.remove(this);
     }
 
     public AllocationService getAllocationService() {
@@ -98,48 +83,6 @@ public class RoutingService extends AbstractLifecycleComponent<RoutingService> i
         performReroute(reason);
     }
 
-    @Override
-    public void clusterChanged(ClusterChangedEvent event) {
-        if (event.state().nodes().isLocalNodeElectedMaster()) {
-            // Figure out if an existing scheduled reroute is good enough or whether we need to cancel and reschedule.
-            // If the minimum of the currently relevant delay settings is larger than something we scheduled in the past,
-            // we are guaranteed that the planned schedule will happen before any of the current shard delays are expired.
-            long minDelaySetting = UnassignedInfo.findSmallestDelayedAllocationSettingNanos(settings, event.state());
-            if (minDelaySetting <= 0) {
-                logger.trace("no need to schedule reroute - no delayed unassigned shards, minDelaySetting [{}], scheduled [{}]", minDelaySetting, minDelaySettingAtLastSchedulingNanos);
-                minDelaySettingAtLastSchedulingNanos = Long.MAX_VALUE;
-                FutureUtils.cancel(registeredNextDelayFuture);
-            } else if (minDelaySetting < minDelaySettingAtLastSchedulingNanos) {
-                FutureUtils.cancel(registeredNextDelayFuture);
-                minDelaySettingAtLastSchedulingNanos = minDelaySetting;
-                TimeValue nextDelay = TimeValue.timeValueNanos(UnassignedInfo.findNextDelayedAllocationIn(event.state()));
-                assert nextDelay.nanos() > 0 : "next delay must be non 0 as minDelaySetting is [" + minDelaySetting + "]";
-                logger.info("delaying allocation for [{}] unassigned shards, next check in [{}]",
-                        UnassignedInfo.getNumberOfDelayedUnassigned(event.state()), nextDelay);
-                registeredNextDelayFuture = threadPool.schedule(nextDelay, ThreadPool.Names.SAME, new AbstractRunnable() {
-                    @Override
-                    protected void doRun() throws Exception {
-                        minDelaySettingAtLastSchedulingNanos = Long.MAX_VALUE;
-                        reroute("assign delayed unassigned shards");
-                    }
-
-                    @Override
-                    public void onFailure(Throwable t) {
-                        logger.warn("failed to schedule/execute reroute post unassigned shard", t);
-                        minDelaySettingAtLastSchedulingNanos = Long.MAX_VALUE;
-                    }
-                });
-            } else {
-                logger.trace("no need to schedule reroute - current schedule reroute is enough. minDelaySetting [{}], scheduled [{}]", minDelaySetting, minDelaySettingAtLastSchedulingNanos);
-            }
-        }
-    }
-
-    // visible for testing
-    long getMinDelaySettingAtLastSchedulingNanos() {
-        return this.minDelaySettingAtLastSchedulingNanos;
-    }
-
     // visible for testing
     protected void performReroute(String reason) {
         try {
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
index 60bb455dfe6..810f8c183da 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
@@ -316,6 +316,7 @@ public final class ShardRouting implements Writeable, ToXContent {
 
     public ShardRouting updateUnassignedInfo(UnassignedInfo unassignedInfo) {
         assert this.unassignedInfo != null : "can only update unassign info if they are already set";
+        assert this.unassignedInfo.isDelayed() || (unassignedInfo.isDelayed() == false) : "cannot transition from non-delayed to delayed";
         return new ShardRouting(shardId, currentNodeId, relocatingNodeId, restoreSource, primary, state,
             unassignedInfo, allocationId, expectedShardSize);
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java b/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
index bc44cd1701c..5d6333972b3 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
@@ -22,6 +22,7 @@ package org.elasticsearch.cluster.routing;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -43,10 +44,9 @@ import java.io.IOException;
 public final class UnassignedInfo implements ToXContent, Writeable {
 
     public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("dateOptionalTime");
-    private static final TimeValue DEFAULT_DELAYED_NODE_LEFT_TIMEOUT = TimeValue.timeValueMinutes(1);
 
     public static final Setting<TimeValue> INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING =
-        Setting.timeSetting("index.unassigned.node_left.delayed_timeout", DEFAULT_DELAYED_NODE_LEFT_TIMEOUT, Property.Dynamic,
+        Setting.timeSetting("index.unassigned.node_left.delayed_timeout", TimeValue.timeValueMinutes(1), Property.Dynamic,
             Property.IndexScope);
     /**
      * Reason why the shard is in unassigned state.
@@ -112,19 +112,19 @@ public final class UnassignedInfo implements ToXContent, Writeable {
     private final Reason reason;
     private final long unassignedTimeMillis; // used for display and log messages, in milliseconds
     private final long unassignedTimeNanos; // in nanoseconds, used to calculate delay for delayed shard allocation
-    private final long lastComputedLeftDelayNanos; // how long to delay shard allocation, not serialized (always positive, 0 means no delay)
+    private final boolean delayed; // if allocation of this shard is delayed due to INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING
     private final String message;
     private final Throwable failure;
     private final int failedAllocations;
 
     /**
-     * creates an UnassingedInfo object based **current** time
+     * creates an UnassignedInfo object based on **current** time
      *
      * @param reason  the cause for making this shard unassigned. See {@link Reason} for more information.
      * @param message more information about cause.
      **/
     public UnassignedInfo(Reason reason, String message) {
-        this(reason, message, null, reason == Reason.ALLOCATION_FAILED ? 1 : 0, System.nanoTime(), System.currentTimeMillis());
+        this(reason, message, null, reason == Reason.ALLOCATION_FAILED ? 1 : 0, System.nanoTime(), System.currentTimeMillis(), false);
     }
 
     /**
@@ -133,28 +133,21 @@ public final class UnassignedInfo implements ToXContent, Writeable {
      * @param failure              the shard level failure that caused this shard to be unassigned, if exists.
      * @param unassignedTimeNanos  the time to use as the base for any delayed re-assignment calculation
      * @param unassignedTimeMillis the time of unassignment used to display to in our reporting.
+     * @param delayed              if allocation of this shard is delayed due to INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.
      */
-    public UnassignedInfo(Reason reason, @Nullable String message, @Nullable Throwable failure, int failedAllocations, long unassignedTimeNanos, long unassignedTimeMillis) {
+    public UnassignedInfo(Reason reason, @Nullable String message, @Nullable Throwable failure, int failedAllocations,
+        long unassignedTimeNanos, long unassignedTimeMillis, boolean delayed) {
         this.reason = reason;
         this.unassignedTimeMillis = unassignedTimeMillis;
         this.unassignedTimeNanos = unassignedTimeNanos;
-        this.lastComputedLeftDelayNanos = 0L;
+        this.delayed = delayed;
         this.message = message;
         this.failure = failure;
         this.failedAllocations = failedAllocations;
-        assert (failedAllocations > 0) == (reason == Reason.ALLOCATION_FAILED):
+        assert (failedAllocations > 0) == (reason == Reason.ALLOCATION_FAILED) :
             "failedAllocations: " + failedAllocations + " for reason " + reason;
         assert !(message == null && failure != null) : "provide a message if a failure exception is provided";
-    }
-
-    public UnassignedInfo(UnassignedInfo unassignedInfo, long newComputedLeftDelayNanos) {
-        this.reason = unassignedInfo.reason;
-        this.unassignedTimeMillis = unassignedInfo.unassignedTimeMillis;
-        this.unassignedTimeNanos = unassignedInfo.unassignedTimeNanos;
-        this.lastComputedLeftDelayNanos = newComputedLeftDelayNanos;
-        this.message = unassignedInfo.message;
-        this.failure = unassignedInfo.failure;
-        this.failedAllocations = unassignedInfo.failedAllocations;
+        assert !(delayed && reason != Reason.NODE_LEFT) : "shard can only be delayed if it is unassigned due to a node leaving";
     }
 
     public UnassignedInfo(StreamInput in) throws IOException {
@@ -163,7 +156,7 @@ public final class UnassignedInfo implements ToXContent, Writeable {
         // As System.nanoTime() cannot be compared across different JVMs, reset it to now.
         // This means that in master fail-over situations, elapsed delay time is forgotten.
         this.unassignedTimeNanos = System.nanoTime();
-        this.lastComputedLeftDelayNanos = 0L;
+        this.delayed = in.readBoolean();
         this.message = in.readOptionalString();
         this.failure = in.readThrowable();
         this.failedAllocations = in.readVInt();
@@ -173,6 +166,7 @@ public final class UnassignedInfo implements ToXContent, Writeable {
         out.writeByte((byte) reason.ordinal());
         out.writeLong(unassignedTimeMillis);
         // Do not serialize unassignedTimeNanos as System.nanoTime() cannot be compared across different JVMs
+        out.writeBoolean(delayed);
         out.writeOptionalString(message);
         out.writeThrowable(failure);
         out.writeVInt(failedAllocations);
@@ -185,7 +179,16 @@ public final class UnassignedInfo implements ToXContent, Writeable {
     /**
      * Returns the number of previously failed allocations of this shard.
      */
-    public int getNumFailedAllocations() { return failedAllocations; }
+    public int getNumFailedAllocations() {
+        return failedAllocations;
+    }
+
+    /**
+     * Returns true if allocation of this shard is delayed due to {@link #INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING}
+     */
+    public boolean isDelayed() {
+        return delayed;
+    }
 
     /**
      * The reason why the shard is unassigned.
@@ -239,50 +242,16 @@ public final class UnassignedInfo implements ToXContent, Writeable {
     }
 
     /**
-     * The allocation delay value in nano seconds associated with the index (defaulting to node settings if not set).
-     */
-    public long getAllocationDelayTimeoutSettingNanos(Settings settings, Settings indexSettings) {
-        if (reason != Reason.NODE_LEFT) {
-            return 0;
-        }
-        TimeValue delayTimeout = INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.get(indexSettings, settings);
-        return Math.max(0L, delayTimeout.nanos());
-    }
-
-    /**
-     * The delay in nanoseconds until this unassigned shard can be reassigned. This value is cached and might be slightly out-of-date.
-     * See also the {@link #updateDelay(long, Settings, Settings)} method.
-     */
-    public long getLastComputedLeftDelayNanos() {
-        return lastComputedLeftDelayNanos;
-    }
-
-    /**
-     * Calculates the delay left based on current time (in nanoseconds) and index/node settings.
+     * Calculates the delay left based on current time (in nanoseconds) and the delay defined by the index settings.
+     * Only relevant if shard is effectively delayed (see {@link #isDelayed()})
+     * Returns 0 if delay is negative
      *
      * @return calculated delay in nanoseconds
      */
-    public long getRemainingDelay(final long nanoTimeNow, final Settings settings, final Settings indexSettings) {
-        final long delayTimeoutNanos = getAllocationDelayTimeoutSettingNanos(settings, indexSettings);
-        if (delayTimeoutNanos == 0L) {
-            return 0L;
-        } else {
-            assert nanoTimeNow >= unassignedTimeNanos;
-            return Math.max(0L, delayTimeoutNanos - (nanoTimeNow - unassignedTimeNanos));
-        }
-    }
-
-    /**
-     * Creates new UnassignedInfo object if delay needs updating.
-     *
-     * @return new Unassigned with updated delay, or this if no change in delay
-     */
-    public UnassignedInfo updateDelay(final long nanoTimeNow, final Settings settings, final Settings indexSettings) {
-        final long newComputedLeftDelayNanos = getRemainingDelay(nanoTimeNow, settings, indexSettings);
-        if (lastComputedLeftDelayNanos == newComputedLeftDelayNanos) {
-            return this;
-        }
-        return new UnassignedInfo(this, newComputedLeftDelayNanos);
+    public long getRemainingDelay(final long nanoTimeNow, final Settings indexSettings) {
+        long delayTimeoutNanos = INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.get(indexSettings).nanos();
+        assert nanoTimeNow >= unassignedTimeNanos;
+        return Math.max(0L, delayTimeoutNanos - (nanoTimeNow - unassignedTimeNanos));
     }
 
     /**
@@ -291,49 +260,34 @@ public final class UnassignedInfo implements ToXContent, Writeable {
     public static int getNumberOfDelayedUnassigned(ClusterState state) {
         int count = 0;
         for (ShardRouting shard : state.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED)) {
-            if (shard.primary() == false) {
-                long delay = shard.unassignedInfo().getLastComputedLeftDelayNanos();
-                if (delay > 0) {
-                    count++;
-                }
+            if (shard.unassignedInfo().isDelayed()) {
+                count++;
             }
         }
         return count;
     }
 
     /**
-     * Finds the smallest delay expiration setting in nanos of all unassigned shards that are still delayed. Returns 0 if there are none.
+     * Finds the next (closest) delay expiration of an delayed shard in nanoseconds based on current time.
+     * Returns 0 if delay is negative.
+     * Returns -1 if no delayed shard is found.
      */
-    public static long findSmallestDelayedAllocationSettingNanos(Settings settings, ClusterState state) {
-        long minDelaySetting = Long.MAX_VALUE;
-        for (ShardRouting shard : state.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED)) {
-            if (shard.primary() == false) {
-                IndexMetaData indexMetaData = state.metaData().index(shard.getIndexName());
-                boolean delayed = shard.unassignedInfo().getLastComputedLeftDelayNanos() > 0;
-                long delayTimeoutSetting = shard.unassignedInfo().getAllocationDelayTimeoutSettingNanos(settings, indexMetaData.getSettings());
-                if (delayed && delayTimeoutSetting > 0 && delayTimeoutSetting < minDelaySetting) {
-                    minDelaySetting = delayTimeoutSetting;
+    public static long findNextDelayedAllocation(long currentNanoTime, ClusterState state) {
+        MetaData metaData = state.metaData();
+        RoutingTable routingTable = state.routingTable();
+        long nextDelayNanos = Long.MAX_VALUE;
+        for (ShardRouting shard : routingTable.shardsWithState(ShardRoutingState.UNASSIGNED)) {
+            UnassignedInfo unassignedInfo = shard.unassignedInfo();
+            if (unassignedInfo.isDelayed()) {
+                Settings indexSettings = metaData.index(shard.index()).getSettings();
+                // calculate next time to schedule
+                final long newComputedLeftDelayNanos = unassignedInfo.getRemainingDelay(currentNanoTime, indexSettings);
+                if (newComputedLeftDelayNanos < nextDelayNanos) {
+                    nextDelayNanos = newComputedLeftDelayNanos;
                 }
             }
         }
-        return minDelaySetting == Long.MAX_VALUE ? 0L : minDelaySetting;
-    }
-
-
-    /**
-     * Finds the next (closest) delay expiration of an unassigned shard in nanoseconds. Returns 0 if there are none.
-     */
-    public static long findNextDelayedAllocationIn(ClusterState state) {
-        long nextDelay = Long.MAX_VALUE;
-        for (ShardRouting shard : state.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED)) {
-            if (shard.primary() == false) {
-                long nextShardDelay = shard.unassignedInfo().getLastComputedLeftDelayNanos();
-                if (nextShardDelay > 0 && nextShardDelay < nextDelay) {
-                    nextDelay = nextShardDelay;
-                }
-            }
-        }
-        return nextDelay == Long.MAX_VALUE ? 0L : nextDelay;
+        return nextDelayNanos == Long.MAX_VALUE ? -1L : nextDelayNanos;
     }
 
     public String shortSummary() {
@@ -343,6 +297,7 @@ public final class UnassignedInfo implements ToXContent, Writeable {
         if (failedAllocations >  0) {
             sb.append(", failed_attempts[").append(failedAllocations).append("]");
         }
+        sb.append(", delayed=").append(delayed);
         String details = getDetails();
 
         if (details != null) {
@@ -364,6 +319,7 @@ public final class UnassignedInfo implements ToXContent, Writeable {
         if (failedAllocations >  0) {
             builder.field("failed_attempts", failedAllocations);
         }
+        builder.field("delayed", delayed);
         String details = getDetails();
         if (details != null) {
             builder.field("details", details);
@@ -386,6 +342,12 @@ public final class UnassignedInfo implements ToXContent, Writeable {
         if (unassignedTimeMillis != that.unassignedTimeMillis) {
             return false;
         }
+        if (delayed != that.delayed) {
+            return false;
+        }
+        if (failedAllocations != that.failedAllocations) {
+            return false;
+        }
         if (reason != that.reason) {
             return false;
         }
@@ -393,12 +355,13 @@ public final class UnassignedInfo implements ToXContent, Writeable {
             return false;
         }
         return !(failure != null ? !failure.equals(that.failure) : that.failure != null);
-
     }
 
     @Override
     public int hashCode() {
         int result = reason != null ? reason.hashCode() : 0;
+        result = 31 * result + Boolean.hashCode(delayed);
+        result = 31 * result + Integer.hashCode(failedAllocations);
         result = 31 * result + Long.hashCode(unassignedTimeMillis);
         result = 31 * result + (message != null ? message.hashCode() : 0);
         result = 31 * result + (failure != null ? failure.hashCode() : 0);
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
index d59113675d8..7dd9b1f8fec 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
@@ -53,6 +53,8 @@ import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING;
+
 
 /**
  * This service manages the node allocation of a cluster. For this reason the
@@ -90,7 +92,7 @@ public class AllocationService extends AbstractComponent {
         RoutingNodes routingNodes = getMutableRoutingNodes(clusterState);
         // shuffle the unassigned nodes, just so we won't have things like poison failed shards
         routingNodes.unassigned().shuffle();
-        StartedRerouteAllocation allocation = new StartedRerouteAllocation(allocationDeciders, routingNodes, clusterState, startedShards, clusterInfoService.getClusterInfo());
+        StartedRerouteAllocation allocation = new StartedRerouteAllocation(allocationDeciders, routingNodes, clusterState, startedShards, clusterInfoService.getClusterInfo(), currentNanoTime());
         boolean changed = applyStartedShards(routingNodes, startedShards);
         if (!changed) {
             return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData());
@@ -99,28 +101,27 @@ public class AllocationService extends AbstractComponent {
         if (withReroute) {
             reroute(allocation);
         }
-        final RoutingAllocation.Result result = buildChangedResult(clusterState.metaData(), clusterState.routingTable(), routingNodes);
-
         String startedShardsAsString = firstListElementsToCommaDelimitedString(startedShards, s -> s.shardId().toString());
-        logClusterHealthStateChange(
-                new ClusterStateHealth(clusterState),
-                new ClusterStateHealth(clusterState.metaData(), result.routingTable()),
-                "shards started [" + startedShardsAsString + "] ..."
-        );
-        return result;
+        return buildResultAndLogHealthChange(allocation, "shards started [" + startedShardsAsString + "] ...");
+    }
+
+    protected RoutingAllocation.Result buildResultAndLogHealthChange(RoutingAllocation allocation, String reason) {
+        return buildResultAndLogHealthChange(allocation, reason, new RoutingExplanations());
 
     }
 
-    protected RoutingAllocation.Result buildChangedResult(MetaData oldMetaData, RoutingTable oldRoutingTable, RoutingNodes newRoutingNodes) {
-        return buildChangedResult(oldMetaData, oldRoutingTable, newRoutingNodes, new RoutingExplanations());
-
-    }
-
-    protected RoutingAllocation.Result buildChangedResult(MetaData oldMetaData, RoutingTable oldRoutingTable, RoutingNodes newRoutingNodes,
-                                                          RoutingExplanations explanations) {
+    protected RoutingAllocation.Result buildResultAndLogHealthChange(RoutingAllocation allocation, String reason, RoutingExplanations explanations) {
+        MetaData oldMetaData = allocation.metaData();
+        RoutingTable oldRoutingTable = allocation.routingTable();
+        RoutingNodes newRoutingNodes = allocation.routingNodes();
         final RoutingTable newRoutingTable = new RoutingTable.Builder().updateNodes(newRoutingNodes).build();
         MetaData newMetaData = updateMetaDataWithRoutingTable(oldMetaData, oldRoutingTable, newRoutingTable);
         assert newRoutingTable.validate(newMetaData); // validates the routing table is coherent with the cluster state metadata
+        logClusterHealthStateChange(
+            new ClusterStateHealth(allocation.metaData(), allocation.routingTable()),
+            new ClusterStateHealth(newMetaData, newRoutingTable),
+            reason
+        );
         return new RoutingAllocation.Result(true, newRoutingTable, newMetaData, explanations);
     }
 
@@ -216,7 +217,8 @@ public class AllocationService extends AbstractComponent {
         RoutingNodes routingNodes = getMutableRoutingNodes(clusterState);
         // shuffle the unassigned nodes, just so we won't have things like poison failed shards
         routingNodes.unassigned().shuffle();
-        FailedRerouteAllocation allocation = new FailedRerouteAllocation(allocationDeciders, routingNodes, clusterState, failedShards, clusterInfoService.getClusterInfo());
+        long currentNanoTime = currentNanoTime();
+        FailedRerouteAllocation allocation = new FailedRerouteAllocation(allocationDeciders, routingNodes, clusterState, failedShards, clusterInfoService.getClusterInfo(), currentNanoTime);
         boolean changed = false;
         // as failing primaries also fail associated replicas, we fail replicas first here so that their nodes are added to ignore list
         List<FailedRerouteAllocation.FailedShard> orderedFailedShards = new ArrayList<>(failedShards);
@@ -225,21 +227,38 @@ public class AllocationService extends AbstractComponent {
             UnassignedInfo unassignedInfo = failedShard.shard.unassignedInfo();
             final int failedAllocations = unassignedInfo != null ? unassignedInfo.getNumFailedAllocations() : 0;
             changed |= applyFailedShard(allocation, failedShard.shard, true, new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, failedShard.message, failedShard.failure,
-                    failedAllocations + 1, System.nanoTime(), System.currentTimeMillis()));
+                    failedAllocations + 1, currentNanoTime, System.currentTimeMillis(), false));
         }
         if (!changed) {
             return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData());
         }
         gatewayAllocator.applyFailedShards(allocation);
         reroute(allocation);
-        final RoutingAllocation.Result result = buildChangedResult(clusterState.metaData(), clusterState.routingTable(), routingNodes);
         String failedShardsAsString = firstListElementsToCommaDelimitedString(failedShards, s -> s.shard.shardId().toString());
-        logClusterHealthStateChange(
-                new ClusterStateHealth(clusterState),
-                new ClusterStateHealth(clusterState.getMetaData(), result.routingTable()),
-                "shards failed [" + failedShardsAsString + "] ..."
-        );
-        return result;
+        return buildResultAndLogHealthChange(allocation, "shards failed [" + failedShardsAsString + "] ...");
+    }
+
+    /**
+     * Removes delay markers from unassigned shards based on current time stamp. Returns true if markers were removed.
+     */
+    private boolean removeDelayMarkers(RoutingAllocation allocation) {
+        final RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator = allocation.routingNodes().unassigned().iterator();
+        final MetaData metaData = allocation.metaData();
+        boolean changed = false;
+        while (unassignedIterator.hasNext()) {
+            ShardRouting shardRouting = unassignedIterator.next();
+            UnassignedInfo unassignedInfo = shardRouting.unassignedInfo();
+            if (unassignedInfo.isDelayed()) {
+                final long newComputedLeftDelayNanos = unassignedInfo.getRemainingDelay(allocation.getCurrentNanoTime(),
+                    metaData.getIndexSafe(shardRouting.index()).getSettings());
+                if (newComputedLeftDelayNanos == 0) {
+                    changed = true;
+                    unassignedIterator.updateUnassignedInfo(new UnassignedInfo(unassignedInfo.getReason(), unassignedInfo.getMessage(), unassignedInfo.getFailure(),
+                        unassignedInfo.getNumFailedAllocations(), unassignedInfo.getUnassignedTimeInNanos(), unassignedInfo.getUnassignedTimeInMillis(), false));
+                }
+            }
+        }
+        return changed;
     }
 
     /**
@@ -276,13 +295,7 @@ public class AllocationService extends AbstractComponent {
         // the assumption is that commands will move / act on shards (or fail through exceptions)
         // so, there will always be shard "movements", so no need to check on reroute
         reroute(allocation);
-        RoutingAllocation.Result result = buildChangedResult(clusterState.metaData(), clusterState.routingTable(), routingNodes, explanations);
-        logClusterHealthStateChange(
-                new ClusterStateHealth(clusterState),
-                new ClusterStateHealth(clusterState.getMetaData(), result.routingTable()),
-                "reroute commands"
-        );
-        return result;
+        return buildResultAndLogHealthChange(allocation, "reroute commands", explanations);
     }
 
 
@@ -310,13 +323,7 @@ public class AllocationService extends AbstractComponent {
         if (!reroute(allocation)) {
             return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData());
         }
-        RoutingAllocation.Result result = buildChangedResult(clusterState.metaData(), clusterState.routingTable(), routingNodes);
-        logClusterHealthStateChange(
-                new ClusterStateHealth(clusterState),
-                new ClusterStateHealth(clusterState.getMetaData(), result.routingTable()),
-                reason
-        );
-        return result;
+        return buildResultAndLogHealthChange(allocation, reason);
     }
 
     private void logClusterHealthStateChange(ClusterStateHealth previousStateHealth, ClusterStateHealth newStateHealth, String reason) {
@@ -341,8 +348,7 @@ public class AllocationService extends AbstractComponent {
 
         // now allocate all the unassigned to available nodes
         if (allocation.routingNodes().unassigned().size() > 0) {
-            updateLeftDelayOfUnassignedShards(allocation, settings);
-
+            changed |= removeDelayMarkers(allocation);
             changed |= gatewayAllocator.allocateUnassigned(allocation);
         }
 
@@ -351,22 +357,6 @@ public class AllocationService extends AbstractComponent {
         return changed;
     }
 
-    // public for testing
-    public static void updateLeftDelayOfUnassignedShards(RoutingAllocation allocation, Settings settings) {
-        final RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator = allocation.routingNodes().unassigned().iterator();
-        final MetaData metaData = allocation.metaData();
-        while (unassignedIterator.hasNext()) {
-            ShardRouting shardRouting = unassignedIterator.next();
-            final IndexMetaData indexMetaData = metaData.getIndexSafe(shardRouting.index());
-            UnassignedInfo previousUnassignedInfo = shardRouting.unassignedInfo();
-            UnassignedInfo updatedUnassignedInfo = previousUnassignedInfo.updateDelay(allocation.getCurrentNanoTime(), settings,
-                indexMetaData.getSettings());
-            if (updatedUnassignedInfo != previousUnassignedInfo) { // reference equality!
-                unassignedIterator.updateUnassignedInfo(updatedUnassignedInfo);
-            }
-        }
-    }
-
     private boolean electPrimariesAndUnassignedDanglingReplicas(RoutingAllocation allocation) {
         boolean changed = false;
         final RoutingNodes routingNodes = allocation.routingNodes();
@@ -436,8 +426,10 @@ public class AllocationService extends AbstractComponent {
             changed = true;
             // now, go over all the shards routing on the node, and fail them
             for (ShardRouting shardRouting : node.copyShards()) {
-                UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, "node_left[" + node.nodeId() + "]", null,
-                        0, allocation.getCurrentNanoTime(), System.currentTimeMillis());
+                final IndexMetaData indexMetaData = allocation.metaData().getIndexSafe(shardRouting.index());
+                boolean delayed = INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.get(indexMetaData.getSettings()).nanos() > 0;
+                UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, "node_left[" + node.nodeId() + "]",
+                    null, 0, allocation.getCurrentNanoTime(), System.currentTimeMillis(), delayed);
                 applyFailedShard(allocation, shardRouting, false, unassignedInfo);
             }
             // its a dead node, remove it, note, its important to remove it *after* we apply failed shard
@@ -458,7 +450,7 @@ public class AllocationService extends AbstractComponent {
         for (ShardRouting routing : replicas) {
             changed |= applyFailedShard(allocation, routing, false,
                     new UnassignedInfo(UnassignedInfo.Reason.PRIMARY_FAILED, "primary failed while replica initializing",
-                            null, 0, allocation.getCurrentNanoTime(), System.currentTimeMillis()));
+                            null, 0, allocation.getCurrentNanoTime(), System.currentTimeMillis(), false));
         }
         return changed;
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java
index ef2e42eed76..b1b0dfce1fe 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java
@@ -57,8 +57,8 @@ public class FailedRerouteAllocation extends RoutingAllocation {
 
     private final List<FailedShard> failedShards;
 
-    public FailedRerouteAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, ClusterState clusterState, List<FailedShard> failedShards, ClusterInfo clusterInfo) {
-        super(deciders, routingNodes, clusterState, clusterInfo, System.nanoTime(), false);
+    public FailedRerouteAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, ClusterState clusterState, List<FailedShard> failedShards, ClusterInfo clusterInfo, long currentNanoTime) {
+        super(deciders, routingNodes, clusterState, clusterInfo, currentNanoTime, false);
         this.failedShards = failedShards;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
index 0df8074e14c..584b141913f 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
@@ -56,7 +56,7 @@ public class RoutingAllocation {
 
         private final MetaData metaData;
 
-        private RoutingExplanations explanations = new RoutingExplanations();
+        private final RoutingExplanations explanations;
 
         /**
          * Creates a new {@link RoutingAllocation.Result}
@@ -65,9 +65,7 @@ public class RoutingAllocation {
          * @param metaData the {@link MetaData} this Result references
          */
         public Result(boolean changed, RoutingTable routingTable, MetaData metaData) {
-            this.changed = changed;
-            this.routingTable = routingTable;
-            this.metaData = metaData;
+            this(changed, routingTable, metaData, new RoutingExplanations());
         }
 
         /**
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StartedRerouteAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StartedRerouteAllocation.java
index 0f55ab4fda1..4d1ac1408a2 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StartedRerouteAllocation.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StartedRerouteAllocation.java
@@ -35,8 +35,8 @@ public class StartedRerouteAllocation extends RoutingAllocation {
 
     private final List<? extends ShardRouting> startedShards;
 
-    public StartedRerouteAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, ClusterState clusterState, List<? extends ShardRouting> startedShards, ClusterInfo clusterInfo) {
-        super(deciders, routingNodes, clusterState, clusterInfo, System.nanoTime(), false);
+    public StartedRerouteAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, ClusterState clusterState, List<? extends ShardRouting> startedShards, ClusterInfo clusterInfo, long currentNanoTime) {
+        super(deciders, routingNodes, clusterState, clusterInfo, currentNanoTime, false);
         this.startedShards = startedShards;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java
index c80afde3086..08be17a8e98 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java
@@ -125,7 +125,7 @@ public class AllocateEmptyPrimaryAllocationCommand extends BasePrimaryAllocation
             // we need to move the unassigned info back to treat it as if it was index creation
             unassignedInfoToUpdate = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED,
                 "force empty allocation from previous reason " + shardRouting.unassignedInfo().getReason() + ", " + shardRouting.unassignedInfo().getMessage(),
-                shardRouting.unassignedInfo().getFailure(), 0, System.nanoTime(), System.currentTimeMillis());
+                shardRouting.unassignedInfo().getFailure(), 0, System.nanoTime(), System.currentTimeMillis(), false);
         }
 
         initializeUnassignedShard(allocation, routingNodes, routingNode, shardRouting, unassignedInfoToUpdate);
diff --git a/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
index 15c2b5c3939..ac4830fe0ec 100644
--- a/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
@@ -455,7 +455,7 @@ public class ClusterService extends AbstractLifecycleComponent<ClusterService> {
     }
 
     /** asserts that the current thread is the cluster state update thread */
-    public boolean assertClusterStateThread() {
+    public static boolean assertClusterStateThread() {
         assert Thread.currentThread().getName().contains(ClusterService.UPDATE_THREAD_NAME) :
                 "not called from the cluster state update thread";
         return true;
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
index ee96335c2b2..fdf084639d1 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
@@ -89,7 +89,7 @@ public class NodeJoinController extends AbstractComponent {
         assert accumulateJoins.get() : "waitToBeElectedAsMaster is called we are not accumulating joins";
 
         final CountDownLatch done = new CountDownLatch(1);
-        final ElectionContext newContext = new ElectionContext(callback, requiredMasterJoins, clusterService) {
+        final ElectionContext newContext = new ElectionContext(callback, requiredMasterJoins) {
             @Override
             void onClose() {
                 if (electionContext.compareAndSet(this, null)) {
@@ -307,16 +307,14 @@ public class NodeJoinController extends AbstractComponent {
     static abstract class ElectionContext implements ElectionCallback {
         private final ElectionCallback callback;
         private final int requiredMasterJoins;
-        private final ClusterService clusterService;
 
         /** set to true after enough joins have been seen and a cluster update task is submitted to become master */
         final AtomicBoolean pendingSetAsMasterTask = new AtomicBoolean();
         final AtomicBoolean closed = new AtomicBoolean();
 
-        ElectionContext(ElectionCallback callback, int requiredMasterJoins, ClusterService clusterService) {
+        ElectionContext(ElectionCallback callback, int requiredMasterJoins) {
             this.callback = callback;
             this.requiredMasterJoins = requiredMasterJoins;
-            this.clusterService = clusterService;
         }
 
         abstract void onClose();
@@ -324,7 +322,7 @@ public class NodeJoinController extends AbstractComponent {
         @Override
         public void onElectedAsMaster(ClusterState state) {
             assert pendingSetAsMasterTask.get() : "onElectedAsMaster called but pendingSetAsMasterTask is not set";
-            assertClusterStateThread();
+            ClusterService.assertClusterStateThread();
             assert state.nodes().isLocalNodeElectedMaster() : "onElectedAsMaster called but local node is not master";
             if (closed.compareAndSet(false, true)) {
                 try {
@@ -337,7 +335,7 @@ public class NodeJoinController extends AbstractComponent {
 
         @Override
         public void onFailure(Throwable t) {
-            assertClusterStateThread();
+            ClusterService.assertClusterStateThread();
             if (closed.compareAndSet(false, true)) {
                 try {
                     onClose();
@@ -346,10 +344,6 @@ public class NodeJoinController extends AbstractComponent {
                 }
             }
         }
-
-        private void assertClusterStateThread() {
-            assert clusterService instanceof ClusterService == false || ((ClusterService) clusterService).assertClusterStateThread();
-        }
     }
 
 
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
index 9f0b194241b..4d2b9e7c5e7 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
@@ -1145,14 +1145,14 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
 
         /** cleans any running joining thread and calls {@link #rejoin} */
         public ClusterState stopRunningThreadAndRejoin(ClusterState clusterState, String reason) {
-            assertClusterStateThread();
+            ClusterService.assertClusterStateThread();
             currentJoinThread.set(null);
             return rejoin(clusterState, reason);
         }
 
         /** starts a new joining thread if there is no currently active one and join thread controlling is started */
         public void startNewThreadIfNotRunning() {
-            assertClusterStateThread();
+            ClusterService.assertClusterStateThread();
             if (joinThreadActive()) {
                 return;
             }
@@ -1185,7 +1185,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
          * If the given thread is not the currently running join thread, the command is ignored.
          */
         public void markThreadAsDoneAndStartNew(Thread joinThread) {
-            assertClusterStateThread();
+            ClusterService.assertClusterStateThread();
             if (!markThreadAsDone(joinThread)) {
                 return;
             }
@@ -1194,7 +1194,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
 
         /** marks the given joinThread as completed. Returns false if the supplied thread is not the currently active join thread */
         public boolean markThreadAsDone(Thread joinThread) {
-            assertClusterStateThread();
+            ClusterService.assertClusterStateThread();
             return currentJoinThread.compareAndSet(joinThread, null);
         }
 
@@ -1210,9 +1210,5 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
             running.set(true);
         }
 
-        private void assertClusterStateThread() {
-            assert clusterService instanceof ClusterService == false || ((ClusterService) clusterService).assertClusterStateThread();
-        }
-
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
index 8b6e425c26a..cf8a54dfc64 100644
--- a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
+++ b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
@@ -108,7 +108,7 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
                                 currentNode, nodeWithHighestMatch);
                         it.moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.REALLOCATED_REPLICA,
                                 "existing allocation of replica to [" + currentNode + "] cancelled, sync id match found on node [" + nodeWithHighestMatch + "]",
-                                null, 0, allocation.getCurrentNanoTime(), System.currentTimeMillis()));
+                                null, 0, allocation.getCurrentNanoTime(), System.currentTimeMillis(), false));
                         changed = true;
                     }
                 }
@@ -179,7 +179,7 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
                 }
             } else if (matchingNodes.hasAnyData() == false) {
                 // if we didn't manage to find *any* data (regardless of matching sizes), check if the allocation of the replica shard needs to be delayed
-                changed |= ignoreUnassignedIfDelayed(unassignedIterator, shard);
+                ignoreUnassignedIfDelayed(unassignedIterator, shard);
             }
         }
         return changed;
@@ -195,21 +195,16 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
      *
      * @param unassignedIterator iterator over unassigned shards
      * @param shard the shard which might be delayed
-     * @return true iff allocation is delayed for this shard
      */
-    public boolean ignoreUnassignedIfDelayed(RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator, ShardRouting shard) {
-        // calculate delay and store it in UnassignedInfo to be used by RoutingService
-        long delay = shard.unassignedInfo().getLastComputedLeftDelayNanos();
-        if (delay > 0) {
-            logger.debug("[{}][{}]: delaying allocation of [{}] for [{}]", shard.index(), shard.id(), shard, TimeValue.timeValueNanos(delay));
+    public void ignoreUnassignedIfDelayed(RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator, ShardRouting shard) {
+        if (shard.unassignedInfo().isDelayed()) {
+            logger.debug("{}: allocation of [{}] is delayed", shard.shardId(), shard);
             /**
              * mark it as changed, since we want to kick a publishing to schedule future allocation,
              * see {@link org.elasticsearch.cluster.routing.RoutingService#clusterChanged(ClusterChangedEvent)}).
              */
             unassignedIterator.removeAndIgnore();
-            return true;
         }
-        return false;
     }
 
     /**
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
index 511c547f233..d0e8ef14d01 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
@@ -21,19 +21,18 @@ package org.elasticsearch.action.admin.cluster.allocation;
 
 import org.apache.lucene.index.CorruptIndexException;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.ShardRoutingHelper;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -43,10 +42,8 @@ import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -61,13 +58,11 @@ public final class ClusterAllocationExplanationTests extends ESTestCase {
     private Index i = new Index("foo", "uuid");
     private ShardRouting primaryShard = ShardRouting.newUnassigned(new ShardId(i, 0), null, true,
             new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
-    private ShardRouting replicaShard = ShardRouting.newUnassigned(new ShardId(i, 0), null, false,
-            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
     private IndexMetaData indexMetaData = IndexMetaData.builder("foo")
             .settings(Settings.builder()
                     .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                     .put(IndexMetaData.SETTING_INDEX_UUID, "uuid"))
-            .putActiveAllocationIds(0, new HashSet<String>(Arrays.asList("aid1", "aid2")))
+            .putActiveAllocationIds(0, Sets.newHashSet("aid1", "aid2"))
             .numberOfShards(1)
             .numberOfReplicas(1)
             .build();
@@ -80,7 +75,6 @@ public final class ClusterAllocationExplanationTests extends ESTestCase {
         noDecision.add(Decision.single(Decision.Type.NO, "no label", "no thanks"));
     }
 
-
     private void assertExplanations(NodeExplanation ne, String finalExplanation, ClusterAllocationExplanation.FinalDecision finalDecision,
                                     ClusterAllocationExplanation.StoreCopy storeCopy) {
         assertEquals(finalExplanation, ne.getFinalExplanation());
@@ -195,6 +189,7 @@ public final class ClusterAllocationExplanationTests extends ESTestCase {
 
     public void testExplanationSerialization() throws Exception {
         ShardId shard = new ShardId("test", "uuid", 0);
+        long allocationDelay = randomIntBetween(0, 500);
         long remainingDelay = randomIntBetween(0, 500);
         Map<DiscoveryNode, NodeExplanation> nodeExplanations = new HashMap<>(1);
         Float nodeWeight = randomFloat();
@@ -207,7 +202,7 @@ public final class ClusterAllocationExplanationTests extends ESTestCase {
                 yesDecision, nodeWeight, storeStatus, "", activeAllocationIds, false);
         nodeExplanations.put(ne.getNode(), ne);
         ClusterAllocationExplanation cae = new ClusterAllocationExplanation(shard, true,
-                "assignedNode", remainingDelay, null, false, nodeExplanations);
+                "assignedNode", allocationDelay, remainingDelay, null, false, nodeExplanations);
         BytesStreamOutput out = new BytesStreamOutput();
         cae.writeTo(out);
         StreamInput in = StreamInput.wrap(out.bytes());
@@ -217,6 +212,7 @@ public final class ClusterAllocationExplanationTests extends ESTestCase {
         assertTrue(cae2.isAssigned());
         assertEquals("assignedNode", cae2.getAssignedNodeId());
         assertNull(cae2.getUnassignedInfo());
+        assertEquals(allocationDelay, cae2.getAllocationDelayMillis());
         assertEquals(remainingDelay, cae2.getRemainingDelayMillis());
         for (Map.Entry<DiscoveryNode, NodeExplanation> entry : cae2.getNodeExplanations().entrySet()) {
             DiscoveryNode node = entry.getKey();
@@ -230,7 +226,6 @@ public final class ClusterAllocationExplanationTests extends ESTestCase {
 
     public void testExplanationToXContent() throws Exception {
         ShardId shardId = new ShardId("foo", "uuid", 0);
-        long remainingDelay = 42;
         Decision.Multi d = new Decision.Multi();
         d.add(Decision.single(Decision.Type.NO, "no label", "because I said no"));
         d.add(Decision.single(Decision.Type.YES, "yes label", "yes please"));
@@ -245,7 +240,7 @@ public final class ClusterAllocationExplanationTests extends ESTestCase {
         Map<DiscoveryNode, NodeExplanation> nodeExplanations = new HashMap<>(1);
         nodeExplanations.put(ne.getNode(), ne);
         ClusterAllocationExplanation cae = new ClusterAllocationExplanation(shardId, true,
-                "assignedNode", remainingDelay, null, false, nodeExplanations);
+                "assignedNode", 42, 42, null, false, nodeExplanations);
         XContentBuilder builder = XContentFactory.jsonBuilder();
         cae.toXContent(builder, ToXContent.EMPTY_PARAMS);
         assertEquals("{\"shard\":{\"index\":\"foo\",\"index_uuid\":\"uuid\",\"id\":0,\"primary\":true},\"assigned\":true," +
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java
index eda6bca80f7..88f09a55c2e 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java
@@ -108,8 +108,6 @@ public class DelayedAllocationIT extends ESIntegTestCase {
      * allocation to a very small value, it kicks the allocation of the unassigned shard
      * even though the node it was hosted on will not come back.
      */
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/18293")
-    @TestLogging("_root:DEBUG,cluster.routing:TRACE")
     public void testDelayedAllocationChangeWithSettingTo100ms() throws Exception {
         internalCluster().startNodesAsync(3).get();
         prepareCreate("test").setSettings(Settings.builder()
@@ -145,12 +143,7 @@ public class DelayedAllocationIT extends ESIntegTestCase {
         ensureGreen("test");
         indexRandomData();
         internalCluster().stopRandomNode(InternalTestCluster.nameFilter(findNodeWithShard()));
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                assertThat(client().admin().cluster().prepareState().all().get().getState().getRoutingNodes().unassigned().size() > 0, equalTo(true));
-            }
-        });
+        assertBusy(() -> assertThat(client().admin().cluster().prepareState().all().get().getState().getRoutingNodes().unassigned().size() > 0, equalTo(true)));
         assertThat(client().admin().cluster().prepareHealth().get().getDelayedUnassignedShards(), equalTo(1));
         assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), TimeValue.timeValueMillis(0))).get());
         ensureGreen("test");
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
new file mode 100644
index 00000000000..be8e85c0fa8
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
@@ -0,0 +1,510 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.routing;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterInfoService;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.gateway.GatewayAllocator;
+import org.elasticsearch.test.ESAllocationTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static java.util.Collections.singleton;
+import static org.elasticsearch.cluster.routing.DelayedAllocationService.CLUSTER_UPDATE_TASK_SOURCE;
+import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
+import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
+import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
+import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
+import static org.hamcrest.Matchers.equalTo;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+/**
+ */
+public class DelayedAllocationServiceTests extends ESAllocationTestCase {
+
+    private TestDelayAllocationService delayedAllocationService;
+    private MockAllocationService allocationService;
+    private ClusterService clusterService;
+    private ThreadPool threadPool;
+
+    @Before
+    public void createDelayedAllocationService() {
+        threadPool = new ThreadPool(getTestName());
+        clusterService = mock(ClusterService.class);
+        allocationService = createAllocationService(Settings.EMPTY, new DelayedShardsMockGatewayAllocator());
+        delayedAllocationService = new TestDelayAllocationService(Settings.EMPTY, threadPool, clusterService, allocationService);
+        verify(clusterService).addFirst(delayedAllocationService);
+    }
+
+    @After
+    public void shutdownThreadPool() throws Exception {
+        terminate(threadPool);
+    }
+
+    public void testNoDelayedUnassigned() throws Exception {
+        MetaData metaData = MetaData.builder()
+            .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)
+                .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "0"))
+                .numberOfShards(1).numberOfReplicas(1))
+            .build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metaData(metaData)
+            .routingTable(RoutingTable.builder().addAsNew(metaData.index("test")).build()).build();
+        clusterState = ClusterState.builder(clusterState)
+            .nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")).localNodeId("node1").masterNodeId("node1"))
+            .build();
+        clusterState = ClusterState.builder(clusterState).routingResult(allocationService.reroute(clusterState, "reroute")).build();
+        // starting primaries
+        clusterState = ClusterState.builder(clusterState)
+            .routingResult(allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)))
+            .build();
+        // starting replicas
+        clusterState = ClusterState.builder(clusterState)
+            .routingResult(allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)))
+            .build();
+        assertThat(clusterState.getRoutingNodes().unassigned().size() > 0, equalTo(false));
+        ClusterState prevState = clusterState;
+        // remove node2 and reroute
+        DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterState.nodes()).remove("node2");
+        boolean nodeAvailableForAllocation = randomBoolean();
+        if (nodeAvailableForAllocation) {
+            nodes.put(newNode("node3"));
+        }
+        clusterState = ClusterState.builder(clusterState).nodes(nodes).build();
+        clusterState = ClusterState.builder(clusterState).routingResult(allocationService.reroute(clusterState, "reroute")).build();
+        ClusterState newState = clusterState;
+        List<ShardRouting> unassignedShards = newState.getRoutingTable().shardsWithState(ShardRoutingState.UNASSIGNED);
+        if (nodeAvailableForAllocation) {
+            assertThat(unassignedShards.size(), equalTo(0));
+        } else {
+            assertThat(unassignedShards.size(), equalTo(1));
+            assertThat(unassignedShards.get(0).unassignedInfo().isDelayed(), equalTo(false));
+        }
+
+        delayedAllocationService.clusterChanged(new ClusterChangedEvent("test", newState, prevState));
+        verifyNoMoreInteractions(clusterService);
+        assertNull(delayedAllocationService.delayedRerouteTask.get());
+    }
+
+    public void testDelayedUnassignedScheduleReroute() throws Exception {
+        TimeValue delaySetting = timeValueMillis(100);
+        MetaData metaData = MetaData.builder()
+            .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)
+                .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), delaySetting))
+                .numberOfShards(1).numberOfReplicas(1))
+            .build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metaData(metaData)
+            .routingTable(RoutingTable.builder().addAsNew(metaData.index("test")).build()).build();
+        clusterState = ClusterState.builder(clusterState)
+            .nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")).localNodeId("node1").masterNodeId("node1"))
+            .build();
+        final long baseTimestampNanos = System.nanoTime();
+        allocationService.setNanoTimeOverride(baseTimestampNanos);
+        clusterState = ClusterState.builder(clusterState).routingResult(allocationService.reroute(clusterState, "reroute")).build();
+        // starting primaries
+        clusterState = ClusterState.builder(clusterState)
+            .routingResult(allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)))
+            .build();
+        // starting replicas
+        clusterState = ClusterState.builder(clusterState)
+            .routingResult(allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)))
+            .build();
+        assertFalse("no shards should be unassigned", clusterState.getRoutingNodes().unassigned().size() > 0);
+        String nodeId = null;
+        final List<ShardRouting> allShards = clusterState.getRoutingNodes().routingTable().allShards("test");
+        // we need to find the node with the replica otherwise we will not reroute
+        for (ShardRouting shardRouting : allShards) {
+            if (shardRouting.primary() == false) {
+                nodeId = shardRouting.currentNodeId();
+                break;
+            }
+        }
+        assertNotNull(nodeId);
+
+        // remove node that has replica and reroute
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove(nodeId)).build();
+        clusterState = ClusterState.builder(clusterState).routingResult(allocationService.reroute(clusterState, "reroute")).build();
+        ClusterState stateWithDelayedShard = clusterState;
+        // make sure the replica is marked as delayed (i.e. not reallocated)
+        assertEquals(1, UnassignedInfo.getNumberOfDelayedUnassigned(stateWithDelayedShard));
+        ShardRouting delayedShard = stateWithDelayedShard.getRoutingNodes().unassigned().iterator().next();
+        assertEquals(baseTimestampNanos, delayedShard.unassignedInfo().getUnassignedTimeInNanos());
+
+        // mock ClusterService.submitStateUpdateTask() method
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicReference<ClusterStateUpdateTask> clusterStateUpdateTask = new AtomicReference<>();
+        doAnswer(invocationOnMock -> {
+            clusterStateUpdateTask.set((ClusterStateUpdateTask)invocationOnMock.getArguments()[1]);
+            latch.countDown();
+            return null;
+        }).when(clusterService).submitStateUpdateTask(eq(CLUSTER_UPDATE_TASK_SOURCE), any(ClusterStateUpdateTask.class));
+        assertNull(delayedAllocationService.delayedRerouteTask.get());
+        long delayUntilClusterChangeEvent = TimeValue.timeValueNanos(randomInt((int)delaySetting.nanos() - 1)).nanos();
+        long clusterChangeEventTimestampNanos = baseTimestampNanos + delayUntilClusterChangeEvent;
+        delayedAllocationService.setNanoTimeOverride(clusterChangeEventTimestampNanos);
+        delayedAllocationService.clusterChanged(new ClusterChangedEvent("fake node left", stateWithDelayedShard, clusterState));
+
+        // check that delayed reroute task was created and registered with the proper settings
+        DelayedAllocationService.DelayedRerouteTask delayedRerouteTask = delayedAllocationService.delayedRerouteTask.get();
+        assertNotNull(delayedRerouteTask);
+        assertFalse(delayedRerouteTask.cancelScheduling.get());
+        assertThat(delayedRerouteTask.baseTimestampNanos, equalTo(clusterChangeEventTimestampNanos));
+        assertThat(delayedRerouteTask.nextDelay.nanos(),
+            equalTo(delaySetting.nanos() - (clusterChangeEventTimestampNanos - baseTimestampNanos)));
+
+        // check that submitStateUpdateTask() was invoked on the cluster service mock
+        assertTrue(latch.await(30, TimeUnit.SECONDS));
+        verify(clusterService).submitStateUpdateTask(eq(CLUSTER_UPDATE_TASK_SOURCE), eq(clusterStateUpdateTask.get()));
+
+        // advance the time on the allocation service to a timestamp that happened after the delayed scheduling
+        long nanoTimeForReroute = clusterChangeEventTimestampNanos + delaySetting.nanos() + timeValueMillis(randomInt(200)).nanos();
+        allocationService.setNanoTimeOverride(nanoTimeForReroute);
+        // apply cluster state
+        ClusterState stateWithRemovedDelay = clusterStateUpdateTask.get().execute(stateWithDelayedShard);
+        // check that shard is not delayed anymore
+        assertEquals(0, UnassignedInfo.getNumberOfDelayedUnassigned(stateWithRemovedDelay));
+        // check that task is now removed
+        assertNull(delayedAllocationService.delayedRerouteTask.get());
+
+        // simulate calling listener (cluster change event)
+        delayedAllocationService.setNanoTimeOverride(nanoTimeForReroute + timeValueMillis(randomInt(200)).nanos());
+        delayedAllocationService.clusterChanged(
+            new ClusterChangedEvent(CLUSTER_UPDATE_TASK_SOURCE, stateWithRemovedDelay, stateWithDelayedShard));
+        // check that no new task is scheduled
+        assertNull(delayedAllocationService.delayedRerouteTask.get());
+        // check that no further cluster state update was submitted
+        verifyNoMoreInteractions(clusterService);
+    }
+
+    /**
+     * This tests that a new delayed reroute is scheduled right after a delayed reroute was run
+     */
+    public void testDelayedUnassignedScheduleRerouteAfterDelayedReroute() throws Exception {
+        TimeValue shortDelaySetting = timeValueMillis(100);
+        TimeValue longDelaySetting = TimeValue.timeValueSeconds(1);
+        MetaData metaData = MetaData.builder()
+            .put(IndexMetaData.builder("short_delay")
+                .settings(settings(Version.CURRENT).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), shortDelaySetting))
+                .numberOfShards(1).numberOfReplicas(1))
+            .put(IndexMetaData.builder("long_delay")
+                .settings(settings(Version.CURRENT).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), longDelaySetting))
+                .numberOfShards(1).numberOfReplicas(1))
+            .build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData)
+            .routingTable(RoutingTable.builder().addAsNew(metaData.index("short_delay")).addAsNew(metaData.index("long_delay")).build())
+            .nodes(DiscoveryNodes.builder()
+                .put(newNode("node0", singleton(DiscoveryNode.Role.MASTER))).localNodeId("node0").masterNodeId("node0")
+                .put(newNode("node1")).put(newNode("node2")).put(newNode("node3")).put(newNode("node4"))).build();
+        // allocate shards
+        clusterState = ClusterState.builder(clusterState).routingResult(allocationService.reroute(clusterState, "reroute")).build();
+        // start primaries
+        clusterState = ClusterState.builder(clusterState)
+            .routingResult(allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)))
+            .build();
+        // start replicas
+        clusterState = ClusterState.builder(clusterState)
+            .routingResult(allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)))
+            .build();
+        assertThat("all shards should be started", clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(4));
+
+        // find replica of short_delay
+        ShardRouting shortDelayReplica = null;
+        for (ShardRouting shardRouting : clusterState.getRoutingNodes().routingTable().allShards("short_delay")) {
+            if (shardRouting.primary() == false) {
+                shortDelayReplica = shardRouting;
+                break;
+            }
+        }
+        assertNotNull(shortDelayReplica);
+
+        // find replica of long_delay
+        ShardRouting longDelayReplica = null;
+        for (ShardRouting shardRouting : clusterState.getRoutingNodes().routingTable().allShards("long_delay")) {
+            if (shardRouting.primary() == false) {
+                longDelayReplica = shardRouting;
+                break;
+            }
+        }
+        assertNotNull(longDelayReplica);
+
+        final long baseTimestampNanos = System.nanoTime();
+
+        // remove node of shortDelayReplica and node of longDelayReplica and reroute
+        ClusterState clusterStateBeforeNodeLeft = clusterState;
+        clusterState = ClusterState.builder(clusterState)
+            .nodes(DiscoveryNodes.builder(clusterState.nodes())
+                .remove(shortDelayReplica.currentNodeId())
+                .remove(longDelayReplica.currentNodeId()))
+            .build();
+        // make sure both replicas are marked as delayed (i.e. not reallocated)
+        allocationService.setNanoTimeOverride(baseTimestampNanos);
+        clusterState = ClusterState.builder(clusterState).routingResult(allocationService.reroute(clusterState, "reroute")).build();
+        final ClusterState stateWithDelayedShards = clusterState;
+        assertEquals(2, UnassignedInfo.getNumberOfDelayedUnassigned(stateWithDelayedShards));
+        RoutingNodes.UnassignedShards.UnassignedIterator iter = stateWithDelayedShards.getRoutingNodes().unassigned().iterator();
+        assertEquals(baseTimestampNanos, iter.next().unassignedInfo().getUnassignedTimeInNanos());
+        assertEquals(baseTimestampNanos, iter.next().unassignedInfo().getUnassignedTimeInNanos());
+
+        // mock ClusterService.submitStateUpdateTask() method
+        CountDownLatch latch1 = new CountDownLatch(1);
+        AtomicReference<ClusterStateUpdateTask> clusterStateUpdateTask1 = new AtomicReference<>();
+        doAnswer(invocationOnMock -> {
+            clusterStateUpdateTask1.set((ClusterStateUpdateTask)invocationOnMock.getArguments()[1]);
+            latch1.countDown();
+            return null;
+        }).when(clusterService).submitStateUpdateTask(eq(CLUSTER_UPDATE_TASK_SOURCE), any(ClusterStateUpdateTask.class));
+        assertNull(delayedAllocationService.delayedRerouteTask.get());
+        long delayUntilClusterChangeEvent = TimeValue.timeValueNanos(randomInt((int)shortDelaySetting.nanos() - 1)).nanos();
+        long clusterChangeEventTimestampNanos = baseTimestampNanos + delayUntilClusterChangeEvent;
+        delayedAllocationService.setNanoTimeOverride(clusterChangeEventTimestampNanos);
+        delayedAllocationService.clusterChanged(
+            new ClusterChangedEvent("fake node left", stateWithDelayedShards, clusterStateBeforeNodeLeft));
+
+        // check that delayed reroute task was created and registered with the proper settings
+        DelayedAllocationService.DelayedRerouteTask firstDelayedRerouteTask = delayedAllocationService.delayedRerouteTask.get();
+        assertNotNull(firstDelayedRerouteTask);
+        assertFalse(firstDelayedRerouteTask.cancelScheduling.get());
+        assertThat(firstDelayedRerouteTask.baseTimestampNanos, equalTo(clusterChangeEventTimestampNanos));
+        assertThat(firstDelayedRerouteTask.nextDelay.nanos(),
+            equalTo(UnassignedInfo.findNextDelayedAllocation(clusterChangeEventTimestampNanos, stateWithDelayedShards)));
+        assertThat(firstDelayedRerouteTask.nextDelay.nanos(),
+            equalTo(shortDelaySetting.nanos() - (clusterChangeEventTimestampNanos - baseTimestampNanos)));
+
+        // check that submitStateUpdateTask() was invoked on the cluster service mock
+        assertTrue(latch1.await(30, TimeUnit.SECONDS));
+        verify(clusterService).submitStateUpdateTask(eq(CLUSTER_UPDATE_TASK_SOURCE), eq(clusterStateUpdateTask1.get()));
+
+        // advance the time on the allocation service to a timestamp that happened after the delayed scheduling
+        long nanoTimeForReroute = clusterChangeEventTimestampNanos + shortDelaySetting.nanos() + timeValueMillis(randomInt(50)).nanos();
+        allocationService.setNanoTimeOverride(nanoTimeForReroute);
+        // apply cluster state
+        ClusterState stateWithOnlyOneDelayedShard = clusterStateUpdateTask1.get().execute(stateWithDelayedShards);
+        // check that shard is not delayed anymore
+        assertEquals(1, UnassignedInfo.getNumberOfDelayedUnassigned(stateWithOnlyOneDelayedShard));
+        // check that task is now removed
+        assertNull(delayedAllocationService.delayedRerouteTask.get());
+
+        // mock ClusterService.submitStateUpdateTask() method again
+        CountDownLatch latch2 = new CountDownLatch(1);
+        AtomicReference<ClusterStateUpdateTask> clusterStateUpdateTask2 = new AtomicReference<>();
+        doAnswer(invocationOnMock -> {
+            clusterStateUpdateTask2.set((ClusterStateUpdateTask)invocationOnMock.getArguments()[1]);
+            latch2.countDown();
+            return null;
+        }).when(clusterService).submitStateUpdateTask(eq(CLUSTER_UPDATE_TASK_SOURCE), any(ClusterStateUpdateTask.class));
+        // simulate calling listener (cluster change event)
+        delayUntilClusterChangeEvent = timeValueMillis(randomInt(50)).nanos();
+        clusterChangeEventTimestampNanos = nanoTimeForReroute + delayUntilClusterChangeEvent;
+        delayedAllocationService.setNanoTimeOverride(clusterChangeEventTimestampNanos);
+        delayedAllocationService.clusterChanged(
+            new ClusterChangedEvent(CLUSTER_UPDATE_TASK_SOURCE, stateWithOnlyOneDelayedShard, stateWithDelayedShards));
+
+        // check that new delayed reroute task was created and registered with the proper settings
+        DelayedAllocationService.DelayedRerouteTask secondDelayedRerouteTask = delayedAllocationService.delayedRerouteTask.get();
+        assertNotNull(secondDelayedRerouteTask);
+        assertFalse(secondDelayedRerouteTask.cancelScheduling.get());
+        assertThat(secondDelayedRerouteTask.baseTimestampNanos, equalTo(clusterChangeEventTimestampNanos));
+        assertThat(secondDelayedRerouteTask.nextDelay.nanos(),
+            equalTo(UnassignedInfo.findNextDelayedAllocation(clusterChangeEventTimestampNanos, stateWithOnlyOneDelayedShard)));
+        assertThat(secondDelayedRerouteTask.nextDelay.nanos(),
+            equalTo(longDelaySetting.nanos() - (clusterChangeEventTimestampNanos - baseTimestampNanos)));
+
+        // check that submitStateUpdateTask() was invoked on the cluster service mock
+        assertTrue(latch2.await(30, TimeUnit.SECONDS));
+        verify(clusterService).submitStateUpdateTask(eq(CLUSTER_UPDATE_TASK_SOURCE), eq(clusterStateUpdateTask2.get()));
+
+        // advance the time on the allocation service to a timestamp that happened after the delayed scheduling
+        nanoTimeForReroute = clusterChangeEventTimestampNanos + longDelaySetting.nanos() + timeValueMillis(randomInt(50)).nanos();
+        allocationService.setNanoTimeOverride(nanoTimeForReroute);
+        // apply cluster state
+        ClusterState stateWithNoDelayedShards = clusterStateUpdateTask2.get().execute(stateWithOnlyOneDelayedShard);
+        // check that shard is not delayed anymore
+        assertEquals(0, UnassignedInfo.getNumberOfDelayedUnassigned(stateWithNoDelayedShards));
+        // check that task is now removed
+        assertNull(delayedAllocationService.delayedRerouteTask.get());
+
+        // simulate calling listener (cluster change event)
+        delayedAllocationService.setNanoTimeOverride(nanoTimeForReroute + timeValueMillis(randomInt(50)).nanos());
+        delayedAllocationService.clusterChanged(
+            new ClusterChangedEvent(CLUSTER_UPDATE_TASK_SOURCE, stateWithNoDelayedShards, stateWithOnlyOneDelayedShard));
+        // check that no new task is scheduled
+        assertNull(delayedAllocationService.delayedRerouteTask.get());
+        // check that no further cluster state update was submitted
+        verifyNoMoreInteractions(clusterService);
+    }
+
+    public void testDelayedUnassignedScheduleRerouteRescheduledOnShorterDelay() throws Exception {
+        TimeValue delaySetting = timeValueSeconds(30);
+        TimeValue shorterDelaySetting = timeValueMillis(100);
+        MetaData metaData = MetaData.builder()
+            .put(IndexMetaData.builder("foo").settings(settings(Version.CURRENT)
+                .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), delaySetting))
+                .numberOfShards(1).numberOfReplicas(1))
+            .put(IndexMetaData.builder("bar").settings(settings(Version.CURRENT)
+                .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), shorterDelaySetting))
+                .numberOfShards(1).numberOfReplicas(1))
+            .build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metaData(metaData)
+            .routingTable(RoutingTable.builder()
+                .addAsNew(metaData.index("foo"))
+                .addAsNew(metaData.index("bar"))
+                .build()).build();
+        clusterState = ClusterState.builder(clusterState)
+            .nodes(DiscoveryNodes.builder()
+                .put(newNode("node1")).put(newNode("node2")).put(newNode("node3")).put(newNode("node4"))
+                .localNodeId("node1").masterNodeId("node1"))
+            .build();
+        final long nodeLeftTimestampNanos = System.nanoTime();
+        allocationService.setNanoTimeOverride(nodeLeftTimestampNanos);
+        clusterState = ClusterState.builder(clusterState).routingResult(allocationService.reroute(clusterState, "reroute")).build();
+        // starting primaries
+        clusterState = ClusterState.builder(clusterState)
+            .routingResult(allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)))
+            .build();
+        // starting replicas
+        clusterState = ClusterState.builder(clusterState)
+            .routingResult(allocationService.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)))
+            .build();
+        assertFalse("no shards should be unassigned", clusterState.getRoutingNodes().unassigned().size() > 0);
+        String nodeIdOfFooReplica = null;
+        for (ShardRouting shardRouting : clusterState.getRoutingNodes().routingTable().allShards("foo")) {
+            if (shardRouting.primary() == false) {
+                nodeIdOfFooReplica = shardRouting.currentNodeId();
+                break;
+            }
+        }
+        assertNotNull(nodeIdOfFooReplica);
+
+        // remove node that has replica and reroute
+        clusterState = ClusterState.builder(clusterState).nodes(
+            DiscoveryNodes.builder(clusterState.nodes()).remove(nodeIdOfFooReplica)).build();
+        clusterState = ClusterState.builder(clusterState).routingResult(allocationService.reroute(clusterState, "fake node left")).build();
+        ClusterState stateWithDelayedShard = clusterState;
+        // make sure the replica is marked as delayed (i.e. not reallocated)
+        assertEquals(1, UnassignedInfo.getNumberOfDelayedUnassigned(stateWithDelayedShard));
+        ShardRouting delayedShard = stateWithDelayedShard.getRoutingNodes().unassigned().iterator().next();
+        assertEquals(nodeLeftTimestampNanos, delayedShard.unassignedInfo().getUnassignedTimeInNanos());
+
+        assertNull(delayedAllocationService.delayedRerouteTask.get());
+        long delayUntilClusterChangeEvent = TimeValue.timeValueNanos(randomInt((int)shorterDelaySetting.nanos() - 1)).nanos();
+        long clusterChangeEventTimestampNanos = nodeLeftTimestampNanos + delayUntilClusterChangeEvent;
+        delayedAllocationService.setNanoTimeOverride(clusterChangeEventTimestampNanos);
+        delayedAllocationService.clusterChanged(new ClusterChangedEvent("fake node left", stateWithDelayedShard, clusterState));
+
+        // check that delayed reroute task was created and registered with the proper settings
+        DelayedAllocationService.DelayedRerouteTask delayedRerouteTask = delayedAllocationService.delayedRerouteTask.get();
+        assertNotNull(delayedRerouteTask);
+        assertFalse(delayedRerouteTask.cancelScheduling.get());
+        assertThat(delayedRerouteTask.baseTimestampNanos, equalTo(clusterChangeEventTimestampNanos));
+        assertThat(delayedRerouteTask.nextDelay.nanos(),
+            equalTo(delaySetting.nanos() - (clusterChangeEventTimestampNanos - nodeLeftTimestampNanos)));
+
+        if (randomBoolean()) {
+            // update settings with shorter delay
+            ClusterState stateWithShorterDelay = ClusterState.builder(stateWithDelayedShard).metaData(MetaData.builder(
+                stateWithDelayedShard.metaData()).updateSettings(Settings.builder().put(
+                UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), shorterDelaySetting).build(), "foo")).build();
+            delayedAllocationService.setNanoTimeOverride(clusterChangeEventTimestampNanos);
+            delayedAllocationService.clusterChanged(
+                new ClusterChangedEvent("apply shorter delay", stateWithShorterDelay, stateWithDelayedShard));
+        } else {
+            // node leaves with replica shard of index bar that has shorter delay
+            String nodeIdOfBarReplica = null;
+            for (ShardRouting shardRouting : stateWithDelayedShard.getRoutingNodes().routingTable().allShards("bar")) {
+                if (shardRouting.primary() == false) {
+                    nodeIdOfBarReplica = shardRouting.currentNodeId();
+                    break;
+                }
+            }
+            assertNotNull(nodeIdOfBarReplica);
+
+            // remove node that has replica and reroute
+            clusterState = ClusterState.builder(stateWithDelayedShard).nodes(
+                DiscoveryNodes.builder(stateWithDelayedShard.nodes()).remove(nodeIdOfBarReplica)).build();
+            ClusterState stateWithShorterDelay = ClusterState.builder(clusterState).routingResult(
+                allocationService.reroute(clusterState, "fake node left")).build();
+            delayedAllocationService.setNanoTimeOverride(clusterChangeEventTimestampNanos);
+            delayedAllocationService.clusterChanged(
+                new ClusterChangedEvent("fake node left", stateWithShorterDelay, stateWithDelayedShard));
+        }
+
+        // check that delayed reroute task was replaced by shorter reroute task
+        DelayedAllocationService.DelayedRerouteTask shorterDelayedRerouteTask = delayedAllocationService.delayedRerouteTask.get();
+        assertNotNull(shorterDelayedRerouteTask);
+        assertNotEquals(shorterDelayedRerouteTask, delayedRerouteTask);
+        assertTrue(delayedRerouteTask.cancelScheduling.get()); // existing task was cancelled
+        assertFalse(shorterDelayedRerouteTask.cancelScheduling.get());
+        assertThat(delayedRerouteTask.baseTimestampNanos, equalTo(clusterChangeEventTimestampNanos));
+        assertThat(shorterDelayedRerouteTask.nextDelay.nanos(),
+            equalTo(shorterDelaySetting.nanos() - (clusterChangeEventTimestampNanos - nodeLeftTimestampNanos)));
+    }
+
+    private static class TestDelayAllocationService extends DelayedAllocationService {
+        private volatile long nanoTimeOverride = -1L;
+
+        public TestDelayAllocationService(Settings settings, ThreadPool threadPool, ClusterService clusterService,
+                                          AllocationService allocationService) {
+            super(settings, threadPool, clusterService, allocationService);
+        }
+
+        @Override
+        protected void assertClusterStateThread() {
+            // do not check this in the unit tests
+        }
+
+        public void setNanoTimeOverride(long nanoTime) {
+            this.nanoTimeOverride = nanoTime;
+        }
+
+        @Override
+        protected long currentNanoTime() {
+            return nanoTimeOverride == -1L ? super.currentNanoTime() : nanoTimeOverride;
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java
index 68027312581..4006ed0e1db 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java
@@ -19,32 +19,12 @@
 
 package org.elasticsearch.cluster.routing;
 
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.AllocationService;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESAllocationTestCase;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.junit.After;
 import org.junit.Before;
 
-import java.util.List;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import static java.util.Collections.singleton;
-import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
-import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
-import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
-import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.equalTo;
 
 /**
@@ -58,191 +38,18 @@ public class RoutingServiceTests extends ESAllocationTestCase {
         routingService = new TestRoutingService();
     }
 
-    @After
-    public void shutdownRoutingService() throws Exception {
-        routingService.shutdown();
-    }
-
     public void testReroute() {
         assertThat(routingService.hasReroutedAndClear(), equalTo(false));
         routingService.reroute("test");
         assertThat(routingService.hasReroutedAndClear(), equalTo(true));
     }
 
-    public void testNoDelayedUnassigned() throws Exception {
-        AllocationService allocation = createAllocationService(Settings.EMPTY, new DelayedShardsMockGatewayAllocator());
-        MetaData metaData = MetaData.builder()
-                .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "0"))
-                        .numberOfShards(1).numberOfReplicas(1))
-                .build();
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
-                .metaData(metaData)
-                .routingTable(RoutingTable.builder().addAsNew(metaData.index("test")).build()).build();
-        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")).localNodeId("node1").masterNodeId("node1")).build();
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build();
-        // starting primaries
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build();
-        // starting replicas
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build();
-        assertThat(clusterState.getRoutingNodes().unassigned().size() > 0, equalTo(false));
-        // remove node2 and reroute
-        ClusterState prevState = clusterState;
-        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node2")).build();
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build();
-        ClusterState newState = clusterState;
-
-        assertThat(routingService.getMinDelaySettingAtLastSchedulingNanos(), equalTo(Long.MAX_VALUE));
-        routingService.clusterChanged(new ClusterChangedEvent("test", newState, prevState));
-        assertThat(routingService.getMinDelaySettingAtLastSchedulingNanos(), equalTo(Long.MAX_VALUE));
-        assertThat(routingService.hasReroutedAndClear(), equalTo(false));
-    }
-
-    public void testDelayedUnassignedScheduleReroute() throws Exception {
-        MockAllocationService allocation = createAllocationService(Settings.EMPTY, new DelayedShardsMockGatewayAllocator());
-        MetaData metaData = MetaData.builder()
-                .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms"))
-                        .numberOfShards(1).numberOfReplicas(1))
-                .build();
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
-                .metaData(metaData)
-                .routingTable(RoutingTable.builder().addAsNew(metaData.index("test")).build()).build();
-        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")).localNodeId("node1").masterNodeId("node1")).build();
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build();
-        // starting primaries
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build();
-        // starting replicas
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build();
-        assertFalse("no shards should be unassigned", clusterState.getRoutingNodes().unassigned().size() > 0);
-        String nodeId = null;
-        final List<ShardRouting> allShards = clusterState.getRoutingNodes().routingTable().allShards("test");
-        // we need to find the node with the replica otherwise we will not reroute
-        for (ShardRouting shardRouting : allShards) {
-            if (shardRouting.primary() == false) {
-                nodeId = shardRouting.currentNodeId();
-                break;
-            }
-        }
-        assertNotNull(nodeId);
-
-        // remove nodeId and reroute
-        ClusterState prevState = clusterState;
-        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove(nodeId)).build();
-        // make sure the replica is marked as delayed (i.e. not reallocated)
-        clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build();
-        assertEquals(1, clusterState.getRoutingNodes().unassigned().size());
-
-        ClusterState newState = clusterState;
-        routingService.clusterChanged(new ClusterChangedEvent("test", newState, prevState));
-        assertBusy(() -> assertTrue("routing service should have run a reroute", routingService.hasReroutedAndClear()));
-        // verify the registration has been reset
-        assertThat(routingService.getMinDelaySettingAtLastSchedulingNanos(), equalTo(Long.MAX_VALUE));
-    }
-
-    /**
-     * This tests that a new delayed reroute is scheduled right after a delayed reroute was run
-     */
-    public void testDelayedUnassignedScheduleRerouteAfterDelayedReroute() throws Exception {
-        final ThreadPool testThreadPool = new ThreadPool(getTestName());
-        ClusterService clusterService = createClusterService(testThreadPool);
-        try {
-            MockAllocationService allocation = createAllocationService(Settings.EMPTY, new DelayedShardsMockGatewayAllocator());
-            MetaData metaData = MetaData.builder()
-                    .put(IndexMetaData.builder("short_delay").settings(settings(Version.CURRENT).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms"))
-                            .numberOfShards(1).numberOfReplicas(1))
-                    .put(IndexMetaData.builder("long_delay").settings(settings(Version.CURRENT).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "10s"))
-                            .numberOfShards(1).numberOfReplicas(1))
-                    .build();
-            ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData)
-                    .routingTable(RoutingTable.builder().addAsNew(metaData.index("short_delay")).addAsNew(metaData.index("long_delay")).build())
-                    .nodes(DiscoveryNodes.builder()
-                            .put(newNode("node0", singleton(DiscoveryNode.Role.MASTER))).localNodeId("node0").masterNodeId("node0")
-                            .put(newNode("node1")).put(newNode("node2")).put(newNode("node3")).put(newNode("node4"))).build();
-            // allocate shards
-            clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build();
-            // start primaries
-            clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build();
-            // start replicas
-            clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build();
-            assertThat("all shards should be started", clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(4));
-
-            // find replica of short_delay
-            ShardRouting shortDelayReplica = null;
-            for (ShardRouting shardRouting : clusterState.getRoutingNodes().routingTable().allShards("short_delay")) {
-                if (shardRouting.primary() == false) {
-                    shortDelayReplica = shardRouting;
-                    break;
-                }
-            }
-            assertNotNull(shortDelayReplica);
-
-            // find replica of long_delay
-            ShardRouting longDelayReplica = null;
-            for (ShardRouting shardRouting : clusterState.getRoutingNodes().routingTable().allShards("long_delay")) {
-                if (shardRouting.primary() == false) {
-                    longDelayReplica = shardRouting;
-                    break;
-                }
-            }
-            assertNotNull(longDelayReplica);
-
-            final long baseTime = System.nanoTime();
-
-            // remove node of shortDelayReplica and node of longDelayReplica and reroute
-            ClusterState prevState = clusterState;
-            clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove(shortDelayReplica.currentNodeId()).remove(longDelayReplica.currentNodeId())).build();
-            // make sure both replicas are marked as delayed (i.e. not reallocated)
-            allocation.setNanoTimeOverride(baseTime);
-            clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build();
-
-            // check that shortDelayReplica and longDelayReplica have been marked unassigned
-            RoutingNodes.UnassignedShards unassigned = clusterState.getRoutingNodes().unassigned();
-            assertEquals(2, unassigned.size());
-            // update shortDelayReplica and longDelayReplica variables with new shard routing
-            ShardRouting shortDelayUnassignedReplica = null;
-            ShardRouting longDelayUnassignedReplica = null;
-            for (ShardRouting shr : unassigned) {
-                if (shr.getIndexName().equals("short_delay")) {
-                    shortDelayUnassignedReplica = shr;
-                } else {
-                    longDelayUnassignedReplica = shr;
-                }
-            }
-            assertTrue(shortDelayReplica.isSameShard(shortDelayUnassignedReplica));
-            assertTrue(longDelayReplica.isSameShard(longDelayUnassignedReplica));
-
-            // manually trigger a clusterChanged event on routingService
-            ClusterState newState = clusterState;
-            setState(clusterService, newState);
-            // create routing service, also registers listener on cluster service
-            RoutingService routingService = new RoutingService(Settings.EMPTY, testThreadPool, clusterService, allocation);
-            routingService.start(); // just so performReroute does not prematurely return
-            // next (delayed) reroute should only delay longDelayReplica/longDelayUnassignedReplica, simulate that we are now 1 second after shards became unassigned
-            allocation.setNanoTimeOverride(baseTime + TimeValue.timeValueSeconds(1).nanos());
-            // register listener on cluster state so we know when cluster state has been changed
-            CountDownLatch latch = new CountDownLatch(1);
-            clusterService.addLast(event -> latch.countDown());
-            // instead of clusterService calling clusterChanged, we call it directly here
-            routingService.clusterChanged(new ClusterChangedEvent("test", newState, prevState));
-            // cluster service should have updated state and called routingService with clusterChanged
-            latch.await();
-            // verify the registration has been set to the delay of longDelayReplica/longDelayUnassignedReplica
-            assertThat(routingService.getMinDelaySettingAtLastSchedulingNanos(), equalTo(TimeValue.timeValueSeconds(10).nanos()));
-        } finally {
-            clusterService.stop();
-            terminate(testThreadPool);
-        }
-    }
-
     private class TestRoutingService extends RoutingService {
 
         private AtomicBoolean rerouted = new AtomicBoolean();
 
         public TestRoutingService() {
-            super(Settings.EMPTY, new ThreadPool(getTestName()), null, null);
-        }
-
-        void shutdown() throws Exception {
-            terminate(threadPool);
+            super(Settings.EMPTY, null, null);
         }
 
         public boolean hasReroutedAndClear() {
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
index f46224570b0..a0ef14ee98d 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
@@ -38,7 +38,6 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESAllocationTestCase;
 
 import java.util.Collections;
-import java.util.EnumSet;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
@@ -75,7 +74,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
     public void testSerialization() throws Exception {
         UnassignedInfo.Reason reason = RandomPicks.randomFrom(random(), UnassignedInfo.Reason.values());
         UnassignedInfo meta = reason == UnassignedInfo.Reason.ALLOCATION_FAILED ?
-            new UnassignedInfo(reason, randomBoolean() ? randomAsciiOfLength(4) : null, null, randomIntBetween(1, 100), System.nanoTime(), System.currentTimeMillis()):
+            new UnassignedInfo(reason, randomBoolean() ? randomAsciiOfLength(4) : null, null, randomIntBetween(1, 100), System.nanoTime(), System.currentTimeMillis(), false):
             new UnassignedInfo(reason, randomBoolean() ? randomAsciiOfLength(4) : null);
         BytesStreamOutput out = new BytesStreamOutput();
         meta.writeTo(out);
@@ -262,59 +261,20 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
     /**
      * Verifies that delayed allocation calculation are correct.
      */
-    public void testUnassignedDelayedOnlyOnNodeLeft() throws Exception {
-        UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, null);
-        unassignedInfo = unassignedInfo.updateDelay(unassignedInfo.getUnassignedTimeInNanos() + 1, // add 1 tick delay
-                Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "10h").build(), Settings.EMPTY);
-        long delay = unassignedInfo.getLastComputedLeftDelayNanos();
-        long cachedDelay = unassignedInfo.getLastComputedLeftDelayNanos();
-        assertThat(delay, equalTo(cachedDelay));
-        assertThat(delay, equalTo(TimeValue.timeValueHours(10).nanos() - 1));
-    }
-
-    /**
-     * Verifies that delayed allocation is only computed when the reason is NODE_LEFT.
-     */
-    public void testUnassignedDelayOnlyNodeLeftNonNodeLeftReason() throws Exception {
-        EnumSet<UnassignedInfo.Reason> reasons = EnumSet.allOf(UnassignedInfo.Reason.class);
-        reasons.remove(UnassignedInfo.Reason.NODE_LEFT);
-        UnassignedInfo.Reason reason = RandomPicks.randomFrom(random(), reasons);
-        UnassignedInfo unassignedInfo = reason == UnassignedInfo.Reason.ALLOCATION_FAILED ?
-            new UnassignedInfo(reason, null, null, 1, System.nanoTime(), System.currentTimeMillis()):
-            new UnassignedInfo(reason, null);
-        unassignedInfo = unassignedInfo.updateDelay(unassignedInfo.getUnassignedTimeInNanos() + 1, // add 1 tick delay
-                Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "10h").build(), Settings.EMPTY);
-        long delay = unassignedInfo.getLastComputedLeftDelayNanos();
-        assertThat(delay, equalTo(0L));
-        delay = unassignedInfo.getLastComputedLeftDelayNanos();
-        assertThat(delay, equalTo(0L));
-    }
-
-    /**
-     * Verifies that delayed allocation calculation are correct.
-     */
-    public void testLeftDelayCalculation() throws Exception {
+    public void testRemainingDelayCalculation() throws Exception {
         final long baseTime = System.nanoTime();
-        UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, "test", null, 0, baseTime, System.currentTimeMillis());
+        UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, "test", null, 0, baseTime, System.currentTimeMillis(), randomBoolean());
         final long totalDelayNanos = TimeValue.timeValueMillis(10).nanos();
-        final Settings settings = Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), TimeValue.timeValueNanos(totalDelayNanos)).build();
-        unassignedInfo = unassignedInfo.updateDelay(baseTime, settings, Settings.EMPTY);
-        long delay = unassignedInfo.getLastComputedLeftDelayNanos();
+        final Settings indexSettings = Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), TimeValue.timeValueNanos(totalDelayNanos)).build();
+        long delay = unassignedInfo.getRemainingDelay(baseTime, indexSettings);
         assertThat(delay, equalTo(totalDelayNanos));
-        assertThat(delay, equalTo(unassignedInfo.getLastComputedLeftDelayNanos()));
         long delta1 = randomIntBetween(1, (int) (totalDelayNanos - 1));
-        unassignedInfo = unassignedInfo.updateDelay(baseTime + delta1, settings, Settings.EMPTY);
-        delay = unassignedInfo.getLastComputedLeftDelayNanos();
+        delay = unassignedInfo.getRemainingDelay(baseTime + delta1, indexSettings);
         assertThat(delay, equalTo(totalDelayNanos - delta1));
-        assertThat(delay, equalTo(unassignedInfo.getLastComputedLeftDelayNanos()));
-        unassignedInfo = unassignedInfo.updateDelay(baseTime + totalDelayNanos, settings, Settings.EMPTY);
-        delay = unassignedInfo.getLastComputedLeftDelayNanos();
+        delay = unassignedInfo.getRemainingDelay(baseTime + totalDelayNanos, indexSettings);
         assertThat(delay, equalTo(0L));
-        assertThat(delay, equalTo(unassignedInfo.getLastComputedLeftDelayNanos()));
-        unassignedInfo = unassignedInfo.updateDelay(baseTime + totalDelayNanos + randomIntBetween(1, 20), settings, Settings.EMPTY);
-        delay = unassignedInfo.getLastComputedLeftDelayNanos();
+        delay = unassignedInfo.getRemainingDelay(baseTime + totalDelayNanos + randomIntBetween(1, 20), indexSettings);
         assertThat(delay, equalTo(0L));
-        assertThat(delay, equalTo(unassignedInfo.getLastComputedLeftDelayNanos()));
     }
 
 
@@ -344,8 +304,6 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
 
     public void testFindNextDelayedAllocation() {
         MockAllocationService allocation = createAllocationService(Settings.EMPTY, new DelayedShardsMockGatewayAllocator());
-        final long baseTime = System.nanoTime();
-        allocation.setNanoTimeOverride(baseTime);
         final TimeValue delayTest1 = TimeValue.timeValueMillis(randomIntBetween(1, 200));
         final TimeValue delayTest2 = TimeValue.timeValueMillis(randomIntBetween(1, 200));
         final long expectMinDelaySettingsNanos = Math.min(delayTest1.nanos(), delayTest2.nanos());
@@ -366,20 +324,18 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
         clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build();
         assertThat(clusterState.getRoutingNodes().unassigned().size() > 0, equalTo(false));
         // remove node2 and reroute
+        final long baseTime = System.nanoTime();
+        allocation.setNanoTimeOverride(baseTime);
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node2")).build();
         clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build();
 
-        final long delta = randomBoolean() ? 0 : randomInt((int) expectMinDelaySettingsNanos);
+        final long delta = randomBoolean() ? 0 : randomInt((int) expectMinDelaySettingsNanos - 1);
 
         if (delta > 0) {
             allocation.setNanoTimeOverride(baseTime + delta);
             clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "time moved")).build();
         }
 
-        long minDelaySetting = UnassignedInfo.findSmallestDelayedAllocationSettingNanos(Settings.EMPTY, clusterState);
-        assertThat(minDelaySetting, equalTo(expectMinDelaySettingsNanos));
-
-        long nextDelay = UnassignedInfo.findNextDelayedAllocationIn(clusterState);
-        assertThat(nextDelay, equalTo(expectMinDelaySettingsNanos - delta));
+        assertThat(UnassignedInfo.findNextDelayedAllocation(baseTime + delta, clusterState), equalTo(expectMinDelaySettingsNanos - delta));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
index 9013d99678d..651eab421ad 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
@@ -521,7 +521,7 @@ public class NodeJoinControllerTests extends ESTestCase {
     static class NoopRoutingService extends RoutingService {
 
         public NoopRoutingService(Settings settings) {
-            super(settings, null, null, new NoopAllocationService(settings));
+            super(settings, null, new NoopAllocationService(settings));
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
index 20eb6286813..82004448658 100644
--- a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
@@ -233,16 +233,14 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
             // we sometime return empty list of files, make sure we test this as well
             testAllocator.addData(node2, false, null);
         }
-        AllocationService.updateLeftDelayOfUnassignedShards(allocation, Settings.EMPTY);
         boolean changed = testAllocator.allocateUnassigned(allocation);
-        assertThat(changed, equalTo(true));
+        assertThat(changed, equalTo(false));
         assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1));
         assertThat(allocation.routingNodes().unassigned().ignored().get(0).shardId(), equalTo(shardId));
 
         allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders(),
                 Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), TimeValue.timeValueHours(1)).build(), UnassignedInfo.Reason.NODE_LEFT);
         testAllocator.addData(node2, false, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"));
-        AllocationService.updateLeftDelayOfUnassignedShards(allocation, Settings.EMPTY);
         changed = testAllocator.allocateUnassigned(allocation);
         assertThat(changed, equalTo(true));
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1));
@@ -290,11 +288,15 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
                     .numberOfShards(1).numberOfReplicas(1)
                     .putActiveAllocationIds(0, Sets.newHashSet(primaryShard.allocationId().getId())))
             .build();
+        // mark shard as delayed if reason is NODE_LEFT
+        boolean delayed = reason == UnassignedInfo.Reason.NODE_LEFT &&
+            UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.get(settings).nanos() > 0;
         RoutingTable routingTable = RoutingTable.builder()
                 .add(IndexRoutingTable.builder(shardId.getIndex())
                                 .addIndexShard(new IndexShardRoutingTable.Builder(shardId)
                                         .addShard(primaryShard)
-                                        .addShard(ShardRouting.newUnassigned(shardId, null, false, new UnassignedInfo(reason, null)))
+                                        .addShard(ShardRouting.newUnassigned(shardId, null, false,
+                                            new UnassignedInfo(reason, null, null, 0, System.nanoTime(), System.currentTimeMillis(), delayed)))
                                         .build())
                 )
                 .build();
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java
index d7d5df54240..7cbb7e819c4 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java
@@ -196,7 +196,7 @@ public abstract class ESAllocationTestCase extends ESTestCase {
     /** A lock {@link AllocationService} allowing tests to override time */
     protected static class MockAllocationService extends AllocationService {
 
-        private Long nanoTimeOverride = null;
+        private volatile long nanoTimeOverride = -1L;
 
         public MockAllocationService(Settings settings, AllocationDeciders allocationDeciders, GatewayAllocator gatewayAllocator,
                                      ShardsAllocator shardsAllocator, ClusterInfoService clusterInfoService) {
@@ -209,7 +209,7 @@ public abstract class ESAllocationTestCase extends ESTestCase {
 
         @Override
         protected long currentNanoTime() {
-            return nanoTimeOverride == null ? super.currentNanoTime() : nanoTimeOverride;
+            return nanoTimeOverride == -1L ? super.currentNanoTime() : nanoTimeOverride;
         }
     }
 
@@ -238,16 +238,15 @@ public abstract class ESAllocationTestCase extends ESTestCase {
         @Override
         public boolean allocateUnassigned(RoutingAllocation allocation) {
             final RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator = allocation.routingNodes().unassigned().iterator();
-            boolean changed = false;
             while (unassignedIterator.hasNext()) {
                 ShardRouting shard = unassignedIterator.next();
                 IndexMetaData indexMetaData = allocation.metaData().index(shard.getIndexName());
                 if (shard.primary() || shard.allocatedPostIndexCreate(indexMetaData) == false) {
                     continue;
                 }
-                changed |= replicaShardAllocator.ignoreUnassignedIfDelayed(unassignedIterator, shard);
+                replicaShardAllocator.ignoreUnassignedIfDelayed(unassignedIterator, shard);
             }
-            return changed;
+            return false;
         }
     }
 }

From dbe0b421404efdae849e851fba2dade23ad4924e Mon Sep 17 00:00:00 2001
From: Mike McCandless <mikemccand@apache.org>
Date: Thu, 26 May 2016 09:40:22 -0400
Subject: [PATCH 0475/1311] Document the hard limits from #15585 on index and
 bulk thread pool sizes

---
 docs/reference/modules/threadpool.asciidoc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/reference/modules/threadpool.asciidoc b/docs/reference/modules/threadpool.asciidoc
index fe9900b21f7..65069e25940 100644
--- a/docs/reference/modules/threadpool.asciidoc
+++ b/docs/reference/modules/threadpool.asciidoc
@@ -16,7 +16,8 @@ There are several thread pools, but the important ones include:
 `index`::
     For index/delete operations. Thread pool type is `fixed`
     with a size of `# of available processors`,
-    queue_size of `200`.
+    queue_size of `200`.  The maximum size for this pool
+    is `# of available processors`.
 
 `search`::
     For count/search/suggest operations. Thread pool type is `fixed`
@@ -31,7 +32,8 @@ There are several thread pools, but the important ones include:
 `bulk`::
     For bulk operations. Thread pool type is `fixed`
     with a size of `# of available processors`,
-    queue_size of `50`.
+    queue_size of `50`.  The maximum size for this pool
+    is `# of available processors`.
 
 `percolate`::
     For percolate operations. Thread pool type is `fixed`

From 0f529e10a8ff4931cb12cb564ea7bf202e5acfaf Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 26 May 2016 09:18:20 -0400
Subject: [PATCH 0476/1311] Fix plugin command name in remove plugin command

This commit fixes the name of the plugin command that is output when a
user attempts to remove a plugin that does not exist.
---
 .../java/org/elasticsearch/plugins/RemovePluginCommand.java     | 2 +-
 .../org/elasticsearch/plugins/RemovePluginCommandTests.java     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java
index af48c1d8207..8279cc5c51f 100644
--- a/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java
@@ -71,7 +71,7 @@ class RemovePluginCommand extends SettingCommand {
 
         Path pluginDir = env.pluginsFile().resolve(pluginName);
         if (Files.exists(pluginDir) == false) {
-            throw new UserError(ExitCodes.USAGE, "Plugin " + pluginName + " not found. Run 'plugin list' to get list of installed plugins.");
+            throw new UserError(ExitCodes.USAGE, "plugin " + pluginName + " not found; run 'elasticsearch-plugin list' to get list of installed plugins");
         }
 
         List<Path> pluginPaths = new ArrayList<>();
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java
index 6528bbc0911..3a4639fa839 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java
@@ -73,7 +73,7 @@ public class RemovePluginCommandTests extends ESTestCase {
 
     public void testMissing() throws Exception {
         UserError e = expectThrows(UserError.class, () -> removePlugin("dne", home));
-        assertTrue(e.getMessage(), e.getMessage().contains("Plugin dne not found"));
+        assertTrue(e.getMessage(), e.getMessage().contains("plugin dne not found"));
         assertRemoveCleaned(env);
     }
 

From d29844e59748c76475d7c434e987fffebf0f853a Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 26 May 2016 09:37:51 -0400
Subject: [PATCH 0477/1311] Remove custom plugins path

This commit removes the ability to specify a custom plugins
path. Instead, the plugins path will always be a subdirectory called
"plugins" off of the home directory.
---
 .../org/elasticsearch/bootstrap/Security.java |  2 +-
 .../common/settings/ClusterSettings.java      |  1 -
 .../org/elasticsearch/env/Environment.java    |  7 +------
 .../org/elasticsearch/tribe/TribeService.java |  3 ---
 .../tribe/TribeServiceTests.java              |  2 --
 docs/plugins/plugin-script.asciidoc           | 13 +-----------
 .../migration/migrate_5_0/plugins.asciidoc    |  5 +++++
 docs/reference/modules/tribe.asciidoc         |  1 -
 docs/reference/setup/install/deb.asciidoc     |  1 -
 docs/reference/setup/install/rpm.asciidoc     |  1 -
 docs/reference/setup/install/windows.asciidoc |  1 -
 .../setup/install/zip-targz.asciidoc          |  1 -
 .../bootstrap/EvilSecurityTests.java          |  1 -
 .../scripts/module_and_plugin_test_cases.bash | 21 -------------------
 14 files changed, 8 insertions(+), 52 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Security.java b/core/src/main/java/org/elasticsearch/bootstrap/Security.java
index 4437097bb35..e1c6d320df6 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Security.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Security.java
@@ -244,7 +244,7 @@ final class Security {
         addPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.binFile(), "read,readlink");
         addPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.libFile(), "read,readlink");
         addPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.modulesFile(), "read,readlink");
-        addPath(policy, Environment.PATH_PLUGINS_SETTING.getKey(), environment.pluginsFile(), "read,readlink");
+        addPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.pluginsFile(), "read,readlink");
         addPath(policy, Environment.PATH_CONF_SETTING.getKey(), environment.configFile(), "read,readlink");
         addPath(policy, Environment.PATH_SCRIPTS_SETTING.getKey(), environment.scriptsFile(), "read,readlink");
         // read-write dirs
diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index 36ee01484e6..b29e7604a98 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -332,7 +332,6 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     Environment.PATH_DATA_SETTING,
                     Environment.PATH_HOME_SETTING,
                     Environment.PATH_LOGS_SETTING,
-                    Environment.PATH_PLUGINS_SETTING,
                     Environment.PATH_REPO_SETTING,
                     Environment.PATH_SCRIPTS_SETTING,
                     Environment.PATH_SHARED_DATA_SETTING,
diff --git a/core/src/main/java/org/elasticsearch/env/Environment.java b/core/src/main/java/org/elasticsearch/env/Environment.java
index e022ce6ad2f..c44db111a5a 100644
--- a/core/src/main/java/org/elasticsearch/env/Environment.java
+++ b/core/src/main/java/org/elasticsearch/env/Environment.java
@@ -53,7 +53,6 @@ public class Environment {
     public static final Setting<List<String>> PATH_DATA_SETTING =
         Setting.listSetting("path.data", Collections.emptyList(), Function.identity(), Property.NodeScope);
     public static final Setting<String> PATH_LOGS_SETTING = Setting.simpleString("path.logs", Property.NodeScope);
-    public static final Setting<String> PATH_PLUGINS_SETTING = Setting.simpleString("path.plugins", Property.NodeScope);
     public static final Setting<List<String>> PATH_REPO_SETTING =
         Setting.listSetting("path.repo", Collections.emptyList(), Function.identity(), Property.NodeScope);
     public static final Setting<String> PATH_SHARED_DATA_SETTING = Setting.simpleString("path.shared_data", Property.NodeScope);
@@ -128,11 +127,7 @@ public class Environment {
             scriptsFile = configFile.resolve("scripts");
         }
 
-        if (PATH_PLUGINS_SETTING.exists(settings)) {
-            pluginsFile = PathUtils.get(cleanPath(PATH_PLUGINS_SETTING.get(settings)));
-        } else {
-            pluginsFile = homeFile.resolve("plugins");
-        }
+        pluginsFile = homeFile.resolve("plugins");
 
         List<String> dataPaths = PATH_DATA_SETTING.get(settings);
         if (dataPaths.isEmpty() == false) {
diff --git a/core/src/main/java/org/elasticsearch/tribe/TribeService.java b/core/src/main/java/org/elasticsearch/tribe/TribeService.java
index b66f428367c..50be6e39b07 100644
--- a/core/src/main/java/org/elasticsearch/tribe/TribeService.java
+++ b/core/src/main/java/org/elasticsearch/tribe/TribeService.java
@@ -220,9 +220,6 @@ public class TribeService extends AbstractLifecycleComponent<TribeService> {
         if (Environment.PATH_CONF_SETTING.exists(globalSettings)) {
             sb.put(Environment.PATH_CONF_SETTING.getKey(), Environment.PATH_CONF_SETTING.get(globalSettings));
         }
-        if (Environment.PATH_PLUGINS_SETTING.exists(globalSettings)) {
-            sb.put(Environment.PATH_PLUGINS_SETTING.getKey(), Environment.PATH_PLUGINS_SETTING.get(globalSettings));
-        }
         if (Environment.PATH_LOGS_SETTING.exists(globalSettings)) {
             sb.put(Environment.PATH_LOGS_SETTING.getKey(), Environment.PATH_LOGS_SETTING.get(globalSettings));
         }
diff --git a/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java b/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java
index daeb80d194c..2bbedd8784b 100644
--- a/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java
@@ -43,13 +43,11 @@ public class TribeServiceTests extends ESTestCase {
             .put("node.name", "nodename")
             .put("path.home", "some/path")
             .put("path.conf", "conf/path")
-            .put("path.plugins", "plugins/path")
             .put("path.scripts", "scripts/path")
             .put("path.logs", "logs/path").build();
         Settings clientSettings = TribeService.buildClientSettings("tribe1", globalSettings, Settings.EMPTY);
         assertEquals("some/path", clientSettings.get("path.home"));
         assertEquals("conf/path", clientSettings.get("path.conf"));
-        assertEquals("plugins/path", clientSettings.get("path.plugins"));
         assertEquals("scripts/path", clientSettings.get("path.scripts"));
         assertEquals("logs/path", clientSettings.get("path.logs"));
 
diff --git a/docs/plugins/plugin-script.asciidoc b/docs/plugins/plugin-script.asciidoc
index 08ad129f22f..1e21288e39c 100644
--- a/docs/plugins/plugin-script.asciidoc
+++ b/docs/plugins/plugin-script.asciidoc
@@ -180,18 +180,7 @@ set ES_JAVA_OPTS="-DproxyHost=host_name -DproxyPort=port_number"
 bin/elasticsearch-plugin install analysis-icu
 -----------------------------------
 
-=== Settings related to plugins
-
-[float]
-=== Custom plugins directory
-
-The `plugins` directory can be changed from the default by adding the
-following to the `elasticsearch.yml` config file:
-
-[source,yml]
----------------------
-path.plugins: /path/to/custom/plugins/dir
----------------------
+=== Plugins directory
 
 The default location of the `plugins` directory depends on which package you install:
 
diff --git a/docs/reference/migration/migrate_5_0/plugins.asciidoc b/docs/reference/migration/migrate_5_0/plugins.asciidoc
index ae1113caa48..1578ae074df 100644
--- a/docs/reference/migration/migrate_5_0/plugins.asciidoc
+++ b/docs/reference/migration/migrate_5_0/plugins.asciidoc
@@ -112,3 +112,8 @@ Previously, Java system properties could be passed to the plugin
 command by passing `-D` style arguments directly to the plugin script.
 This is no longer permitted and such system properties must be passed
 via ES_JAVA_OPTS.
+
+==== Custom plugins path
+
+The ability to specify a custom plugins path via `path.plugins` has
+been removed.
diff --git a/docs/reference/modules/tribe.asciidoc b/docs/reference/modules/tribe.asciidoc
index 74c383eaa9e..a1240a28f84 100644
--- a/docs/reference/modules/tribe.asciidoc
+++ b/docs/reference/modules/tribe.asciidoc
@@ -87,7 +87,6 @@ configuration options are passed down from the tribe node to each node client:
 * `transport.publish_host`
 * `path.home`
 * `path.conf`
-* `path.plugins`
 * `path.logs`
 * `path.scripts`
 * `shield.*`
diff --git a/docs/reference/setup/install/deb.asciidoc b/docs/reference/setup/install/deb.asciidoc
index 0feba56b0b6..e02d0f84a1d 100644
--- a/docs/reference/setup/install/deb.asciidoc
+++ b/docs/reference/setup/install/deb.asciidoc
@@ -174,7 +174,6 @@ locations for a Debian-based system:
 | plugins
   | Plugin files location. Each plugin will be contained in a subdirectory.
   | /usr/share/elasticsearch/plugins
-  | path.plugins
 
 | repo
   | Shared file system repository locations. Can hold multiple locations. A file system repository can be placed in to any subdirectory of any directory specified here.
diff --git a/docs/reference/setup/install/rpm.asciidoc b/docs/reference/setup/install/rpm.asciidoc
index 46a5c8739eb..d7f216670dc 100644
--- a/docs/reference/setup/install/rpm.asciidoc
+++ b/docs/reference/setup/install/rpm.asciidoc
@@ -160,7 +160,6 @@ locations for an RPM-based system:
 | plugins
   | Plugin files location. Each plugin will be contained in a subdirectory.
   | /usr/share/elasticsearch/plugins
-  | path.plugins
 
 | repo
   | Shared file system repository locations. Can hold multiple locations. A file system repository can be placed in to any subdirectory of any directory specified here.
diff --git a/docs/reference/setup/install/windows.asciidoc b/docs/reference/setup/install/windows.asciidoc
index 0d2e8bf04f6..edf91e08890 100644
--- a/docs/reference/setup/install/windows.asciidoc
+++ b/docs/reference/setup/install/windows.asciidoc
@@ -228,7 +228,6 @@ directory so that you do not delete important data later on.
 | plugins
   | Plugin files location. Each plugin will be contained in a subdirectory.
   | %ES_HOME%\plugins
-  | path.plugins
 
 | repo
   | Shared file system repository locations. Can hold multiple locations. A file system repository can be placed in to any subdirectory of any directory specified here.
diff --git a/docs/reference/setup/install/zip-targz.asciidoc b/docs/reference/setup/install/zip-targz.asciidoc
index 7fc41a0f3f8..680d698976d 100644
--- a/docs/reference/setup/install/zip-targz.asciidoc
+++ b/docs/reference/setup/install/zip-targz.asciidoc
@@ -147,7 +147,6 @@ directory so that you do not delete important data later on.
 | plugins
   | Plugin files location. Each plugin will be contained in a subdirectory.
   | $ES_HOME/plugins
-  | path.plugins
 
 | repo
   | Shared file system repository locations. Can hold multiple locations. A file system repository can be placed in to any subdirectory of any directory specified here.
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java
index c213f18f333..df54cf87fa4 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java
@@ -77,7 +77,6 @@ public class EvilSecurityTests extends ESTestCase {
         settingsBuilder.put(Environment.PATH_HOME_SETTING.getKey(), esHome.resolve("home").toString());
         settingsBuilder.put(Environment.PATH_CONF_SETTING.getKey(), esHome.resolve("conf").toString());
         settingsBuilder.put(Environment.PATH_SCRIPTS_SETTING.getKey(), esHome.resolve("scripts").toString());
-        settingsBuilder.put(Environment.PATH_PLUGINS_SETTING.getKey(), esHome.resolve("plugins").toString());
         settingsBuilder.putArray(Environment.PATH_DATA_SETTING.getKey(), esHome.resolve("data1").toString(), esHome.resolve("data2").toString());
         settingsBuilder.put(Environment.PATH_SHARED_DATA_SETTING.getKey(), esHome.resolve("custom").toString());
         settingsBuilder.put(Environment.PATH_LOGS_SETTING.getKey(), esHome.resolve("logs").toString());
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
index e20959a82e1..19d9f69ece0 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
@@ -113,27 +113,6 @@ fi
     fi
 }
 
-@test "[$GROUP] install jvm-example plugin with a custom path.plugins" {
-    # Clean up after the last time this test was run
-    rm -rf /tmp/plugins.*
-
-    local oldPlugins="$ESPLUGINS"
-    export ESPLUGINS=$(mktemp -d -t 'plugins.XXXX')
-
-    # Modify the path.plugins setting in configuration file
-    echo "path.plugins: $ESPLUGINS" >> "$ESCONFIG/elasticsearch.yml"
-    chown -R elasticsearch:elasticsearch "$ESPLUGINS"
-
-    install_jvm_example
-    start_elasticsearch_service
-    # check that configuration was actually picked up
-    curl -s localhost:9200/_cat/configured_example | sed 's/ *$//' > /tmp/installed
-    echo "foo" > /tmp/expected
-    diff /tmp/installed /tmp/expected
-    stop_elasticsearch_service
-    remove_jvm_example
-}
-
 @test "[$GROUP] install jvm-example plugin with a custom CONFIG_DIR" {
     # Clean up after the last time we ran this test
     rm -rf /tmp/config.*

From f03780711712b08b459f1d5d23b70c34e8e4b35f Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Thu, 26 May 2016 11:43:29 -0400
Subject: [PATCH 0478/1311] replace ScriptException with a better one

---
 .../elasticsearch/ElasticsearchException.java |  41 +------
 .../search/function/ScriptScoreFunction.java  |   4 +-
 .../xcontent/json/JsonXContentGenerator.java  |   2 +-
 .../script/GeneralScriptException.java        |  51 ++++++++
 .../elasticsearch/script/ScriptException.java | 116 ++++++++++++++++--
 .../elasticsearch/script/ScriptService.java   |  15 ++-
 .../org/elasticsearch/ESExceptionTests.java   |  24 +++-
 .../ExceptionSerializationTests.java          |   3 +-
 .../function/ScriptScoreFunctionTests.java    |   4 +-
 .../script/ScriptContextTests.java            |   4 +-
 .../script/ScriptExceptionTests.java          |  99 +++++++++++++++
 .../script/ScriptServiceTests.java            |   2 +-
 .../ExpressionExecutableScript.java           |  10 +-
 .../ExpressionScriptEngineService.java        |  65 ++++++----
 .../expression/ExpressionSearchScript.java    |   6 +-
 .../script/expression/ExpressionTests.java    |  56 +++++----
 .../expression/MoreExpressionTests.java       |  10 +-
 .../groovy/GroovyScriptEngineService.java     |  10 +-
 .../script/groovy/GroovySecurityTests.java    |   4 +-
 .../mustache/MustacheScriptEngineService.java |   4 +-
 .../org/elasticsearch/painless/Compiler.java  |  46 +++----
 .../elasticsearch/painless/Executable.java    |  23 +++-
 .../elasticsearch/painless/MethodWriter.java  |  32 ++++-
 .../elasticsearch/painless/ScriptImpl.java    |  61 ++++++++-
 .../org/elasticsearch/painless/Writer.java    |  13 +-
 .../painless/WriterConstants.java             |   3 +-
 .../elasticsearch/painless/antlr/Walker.java  |   2 +-
 .../elasticsearch/painless/node/ANode.java    |  14 ---
 .../elasticsearch/painless/node/EBinary.java  |   1 +
 .../elasticsearch/painless/node/ECast.java    |   1 +
 .../elasticsearch/painless/node/EChain.java   |   4 +
 .../elasticsearch/painless/node/EComp.java    |   1 +
 .../painless/node/EConditional.java           |   1 +
 .../elasticsearch/painless/node/EUnary.java   |   1 +
 .../painless/node/LArrayLength.java           |   1 +
 .../elasticsearch/painless/node/LBrace.java   |   2 +
 .../elasticsearch/painless/node/LCall.java    |   1 +
 .../elasticsearch/painless/node/LCast.java    |   1 +
 .../painless/node/LDefArray.java              |   2 +
 .../elasticsearch/painless/node/LDefCall.java |   1 +
 .../painless/node/LDefField.java              |   2 +
 .../elasticsearch/painless/node/LField.java   |   2 +
 .../painless/node/LListShortcut.java          |   2 +
 .../painless/node/LMapShortcut.java           |   2 +
 .../painless/node/LNewArray.java              |   1 +
 .../elasticsearch/painless/node/LNewObj.java  |   1 +
 .../painless/node/LShortcut.java              |   2 +
 .../elasticsearch/painless/node/SBreak.java   |   2 -
 .../elasticsearch/painless/node/SCatch.java   |   3 +-
 .../painless/node/SContinue.java              |   2 -
 .../painless/node/SDeclaration.java           |   3 +-
 .../org/elasticsearch/painless/node/SDo.java  |   5 +-
 .../painless/node/SExpression.java            |   3 +-
 .../org/elasticsearch/painless/node/SFor.java |   7 +-
 .../org/elasticsearch/painless/node/SIf.java  |   3 +-
 .../elasticsearch/painless/node/SIfElse.java  |   3 +-
 .../elasticsearch/painless/node/SReturn.java  |   3 +-
 .../elasticsearch/painless/node/SThrow.java   |   3 +-
 .../org/elasticsearch/painless/node/STry.java |   3 +-
 .../elasticsearch/painless/node/SWhile.java   |   7 +-
 .../painless/CompoundAssignmentTests.java     |  25 ++--
 .../painless/DefOperationTests.java           |   8 +-
 .../painless/DefOptimizationTests.java        |   2 +-
 .../elasticsearch/painless/DivisionTests.java |  28 ++---
 .../elasticsearch/painless/OverloadTests.java |   2 +-
 .../painless/RemainderTests.java              |  14 +--
 .../painless/ScriptTestCase.java              |  27 +++-
 .../elasticsearch/painless/StringTests.java   |  32 ++---
 .../elasticsearch/painless/TryCatchTests.java |   4 +-
 .../painless/WhenThingsGoWrongTests.java      |  49 ++++----
 .../test/reindex/40_search_failures.yaml      |   2 +-
 .../update_by_query/40_search_failure.yaml    |   2 +-
 72 files changed, 688 insertions(+), 307 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/script/GeneralScriptException.java
 create mode 100644 core/src/test/java/org/elasticsearch/script/ScriptExceptionTests.java

diff --git a/core/src/main/java/org/elasticsearch/ElasticsearchException.java b/core/src/main/java/org/elasticsearch/ElasticsearchException.java
index 1faf6cd4237..ad5a2e79cd1 100644
--- a/core/src/main/java/org/elasticsearch/ElasticsearchException.java
+++ b/core/src/main/java/org/elasticsearch/ElasticsearchException.java
@@ -200,41 +200,6 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
         return rootCause;
     }
 
-    /**
-     * Check whether this exception contains an exception of the given type:
-     * either it is of the given class itself or it contains a nested cause
-     * of the given type.
-     *
-     * @param exType the exception type to look for
-     * @return whether there is a nested exception of the specified type
-     */
-    public boolean contains(Class<? extends Throwable> exType) {
-        if (exType == null) {
-            return false;
-        }
-        if (exType.isInstance(this)) {
-            return true;
-        }
-        Throwable cause = getCause();
-        if (cause == this) {
-            return false;
-        }
-        if (cause instanceof ElasticsearchException) {
-            return ((ElasticsearchException) cause).contains(exType);
-        } else {
-            while (cause != null) {
-                if (exType.isInstance(cause)) {
-                    return true;
-                }
-                if (cause.getCause() == cause) {
-                    break;
-                }
-                cause = cause.getCause();
-            }
-            return false;
-        }
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(this.getMessage());
@@ -531,7 +496,8 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
                 org.elasticsearch.index.shard.IndexShardStartedException::new, 23),
         SEARCH_CONTEXT_MISSING_EXCEPTION(org.elasticsearch.search.SearchContextMissingException.class,
                 org.elasticsearch.search.SearchContextMissingException::new, 24),
-        SCRIPT_EXCEPTION(org.elasticsearch.script.ScriptException.class, org.elasticsearch.script.ScriptException::new, 25),
+        GENERAL_SCRIPT_EXCEPTION(org.elasticsearch.script.GeneralScriptException.class, 
+                org.elasticsearch.script.GeneralScriptException::new, 25),
         BATCH_OPERATION_EXCEPTION(org.elasticsearch.index.shard.TranslogRecoveryPerformer.BatchOperationException.class,
                 org.elasticsearch.index.shard.TranslogRecoveryPerformer.BatchOperationException::new, 26),
         SNAPSHOT_CREATION_EXCEPTION(org.elasticsearch.snapshots.SnapshotCreationException.class,
@@ -741,7 +707,8 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
         QUERY_SHARD_EXCEPTION(org.elasticsearch.index.query.QueryShardException.class,
                 org.elasticsearch.index.query.QueryShardException::new, 141),
         NO_LONGER_PRIMARY_SHARD_EXCEPTION(ShardStateAction.NoLongerPrimaryShardException.class,
-                ShardStateAction.NoLongerPrimaryShardException::new, 142);
+                ShardStateAction.NoLongerPrimaryShardException::new, 142),
+        SCRIPT_EXCEPTION(org.elasticsearch.script.ScriptException.class, org.elasticsearch.script.ScriptException::new, 143);
 
 
         final Class<? extends ElasticsearchException> exceptionClass;
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java
index 82454f3dd62..47b85250735 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunction.java
@@ -26,7 +26,7 @@ import org.apache.lucene.search.Scorer;
 import org.elasticsearch.script.ExplainableSearchScript;
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptException;
+import org.elasticsearch.script.GeneralScriptException;
 import org.elasticsearch.script.SearchScript;
 
 import java.io.IOException;
@@ -87,7 +87,7 @@ public class ScriptScoreFunction extends ScoreFunction {
                 scorer.score = subQueryScore;
                 double result = leafScript.runAsDouble();
                 if (Double.isNaN(result)) {
-                    throw new ScriptException("script_score returned NaN");
+                    throw new GeneralScriptException("script_score returned NaN");
                 }
                 return result;
             }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java
index 462288f688a..56baf30c9e0 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java
@@ -100,7 +100,7 @@ public class JsonXContentGenerator implements XContentGenerator {
 
     @Override
     public final void usePrettyPrint() {
-        generator.setPrettyPrinter(new DefaultPrettyPrinter().withObjectIndenter(INDENTER));
+        generator.setPrettyPrinter(new DefaultPrettyPrinter().withObjectIndenter(INDENTER).withArrayIndenter(INDENTER));
         prettyPrint = true;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/script/GeneralScriptException.java b/core/src/main/java/org/elasticsearch/script/GeneralScriptException.java
new file mode 100644
index 00000000000..72820622772
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/script/GeneralScriptException.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.io.stream.StreamInput;
+
+import java.io.IOException;
+
+/**
+ * Simple exception class from a script.
+ * <p>
+ * Use of this exception should generally be avoided, it doesn't provide
+ * much context or structure to users trying to debug scripting when
+ * things go wrong.
+ * @deprecated Use ScriptException for exceptions from the scripting engine,
+ *             otherwise use a more appropriate exception (e.g. if thrown
+ *             from various abstractions)
+ */
+@Deprecated
+public class GeneralScriptException extends ElasticsearchException {
+
+    public GeneralScriptException(String msg) {
+        super(msg);
+    }
+
+    public GeneralScriptException(String msg, Throwable cause) {
+        super(msg, cause);
+    }
+
+    public GeneralScriptException(StreamInput in) throws IOException{
+        super(in);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptException.java b/core/src/main/java/org/elasticsearch/script/ScriptException.java
index 3cc3560da7c..475091f9f6d 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptException.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptException.java
@@ -1,3 +1,5 @@
+package org.elasticsearch.script;
+
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -17,27 +19,117 @@
  * under the License.
  */
 
-package org.elasticsearch.script;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.io.stream.StreamInput;
-
-import java.io.IOException;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
 
 /**
- *
+ * Exception from a scripting engine.
+ * <p>
+ * A ScriptException has the following components:
+ * <ul>
+ *   <li>{@code message}: A short and simple summary of what happened, such as "compile error".
+ *   <li>{@code cause}: The underlying cause of the exception.
+ *   <li>{@code scriptStack}: An implementation-specific "stacktrace" for the error in the script.
+ *   <li>{@code script}: Identifier for which script failed.
+ *   <li>{@code lang}: Scripting engine language, such as "painless"
+ * </ul>
  */
+@SuppressWarnings("serial")
 public class ScriptException extends ElasticsearchException {
-
-    public ScriptException(String msg) {
-        super(msg);
+    private final List<String> scriptStack;
+    private final String script;
+    private final String lang;
+    
+    /**
+     * Create a new ScriptException.
+     * @param message A short and simple summary of what happened, such as "compile error". 
+     *                Must not be {@code null}. 
+     * @param cause The underlying cause of the exception. Must not be {@code null}.
+     * @param scriptStack An implementation-specific "stacktrace" for the error in the script.
+     *                Must not be {@code null}, but can be empty (though this should be avoided if possible).
+     * @param script Identifier for which script failed. Must not be {@code null}.
+     * @param lang Scripting engine language, such as "painless". Must not be {@code null}.
+     * @throws NullPointerException if any parameters are {@code null}.
+     */
+    public ScriptException(String message, Throwable cause, List<String> scriptStack, String script, String lang) {
+        super(Objects.requireNonNull(message), Objects.requireNonNull(cause));
+        this.scriptStack = Collections.unmodifiableList(Objects.requireNonNull(scriptStack));
+        this.script = Objects.requireNonNull(script);
+        this.lang = Objects.requireNonNull(lang);
     }
 
-    public ScriptException(String msg, Throwable cause) {
-        super(msg, cause);
-    }
-
-    public ScriptException(StreamInput in) throws IOException{
+    /**
+     * Deserializes a ScriptException from a {@code StreamInput}
+     */
+    public ScriptException(StreamInput in) throws IOException {
         super(in);
+        scriptStack = Arrays.asList(in.readStringArray());
+        script = in.readString();
+        lang = in.readString();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeStringArray(scriptStack.toArray(new String[0]));
+        out.writeString(script);
+        out.writeString(lang);
+    }
+    
+    @Override
+    protected void innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        super.innerToXContent(builder, params);
+        builder.field("script_stack", scriptStack);
+        builder.field("script", script);
+        builder.field("lang", lang);
+    }
+
+    /**
+     * Returns the stacktrace for the error in the script.
+     * @return a read-only list of frames, which may be empty.
+     */
+    public List<String> getScriptStack() {
+        return scriptStack;
+    }
+    
+    /**
+     * Returns the identifier for which script.
+     * @return script's name or source text that identifies the script.
+     */
+    public String getScript() {
+        return script;
+    }
+    
+    /**
+     * Returns the language of the script.
+     * @return the {@code lang} parameter of the scripting engine.
+     */
+    public String getLang() {
+        return lang;
+    }
+
+    /** 
+     * Returns a JSON version of this exception for debugging.
+     */
+    public String toJsonString() {
+        try {
+            XContentBuilder json = XContentFactory.jsonBuilder().prettyPrint();
+            json.startObject();
+            toXContent(json, ToXContent.EMPTY_PARAMS);
+            json.endObject();
+            return json.string();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java
index cf0ba1ca941..f5ee456c119 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java
@@ -31,7 +31,6 @@ import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
@@ -236,7 +235,7 @@ public class ScriptService extends AbstractComponent implements Closeable {
 
         ScriptEngineService scriptEngineService = getScriptEngineServiceForLang(lang);
         if (canExecuteScript(lang, scriptEngineService, script.getType(), scriptContext) == false) {
-            throw new ScriptException("scripts of type [" + script.getType() + "], operation [" + scriptContext.getKey() + "] and lang [" + lang + "] are disabled");
+            throw new IllegalStateException("scripts of type [" + script.getType() + "], operation [" + scriptContext.getKey() + "] and lang [" + lang + "] are disabled");
         }
 
         // TODO: fix this through some API or something, that's wrong
@@ -244,7 +243,7 @@ public class ScriptService extends AbstractComponent implements Closeable {
         boolean expression = "expression".equals(script.getLang());
         boolean notSupported = scriptContext.getKey().equals(ScriptContext.Standard.UPDATE.getKey());
         if (expression && notSupported) {
-            throw new ScriptException("scripts of type [" + script.getType() + "]," +
+            throw new UnsupportedOperationException("scripts of type [" + script.getType() + "]," +
                     " operation [" + scriptContext.getKey() + "] and lang [" + lang + "] are not supported");
         }
 
@@ -307,7 +306,11 @@ public class ScriptService extends AbstractComponent implements Closeable {
                 String actualName = (type == ScriptType.INLINE) ? null : name;
                 compiledScript = new CompiledScript(type, name, lang, scriptEngineService.compile(actualName, code, params));
             } catch (Exception exception) {
-                throw new ScriptException("Failed to compile " + type + " script [" + name + "] using lang [" + lang + "]", exception);
+                // TODO: remove this try-catch completely, when all script engines have good exceptions!
+                if (exception instanceof ScriptException) {
+                    throw exception; // its already good!
+                }
+                throw new GeneralScriptException("Failed to compile " + type + " script [" + name + "] using lang [" + lang + "]", exception);
             }
 
             //Since the cache key is the script content itself we don't need to
@@ -365,6 +368,10 @@ public class ScriptService extends AbstractComponent implements Closeable {
                                 template.getScript(), scriptLang);
                     }
                 } catch (Exception e) {
+                    // TODO: remove this when all script engines have good exceptions!
+                    if (e instanceof ScriptException) {
+                        throw e; // its already good!
+                    }
                     throw new IllegalArgumentException("Unable to parse [" + template.getScript() +
                             "] lang [" + scriptLang + "]", e);
                 }
diff --git a/core/src/test/java/org/elasticsearch/ESExceptionTests.java b/core/src/test/java/org/elasticsearch/ESExceptionTests.java
index 55b8c3ed4ec..9ac8782110e 100644
--- a/core/src/test/java/org/elasticsearch/ESExceptionTests.java
+++ b/core/src/test/java/org/elasticsearch/ESExceptionTests.java
@@ -179,12 +179,32 @@ public class ESExceptionTests extends ESTestCase {
         }
     }
 
+    /**
+     * Check whether this exception contains an exception of the given type:
+     * either it is of the given class itself or it contains a nested cause
+     * of the given type.
+     *
+     * @param exType the exception type to look for
+     * @return whether there is a nested exception of the specified type
+     */
+    private boolean contains(Throwable t, Class<? extends Throwable> exType) {
+        if (exType == null) {
+            return false;
+        }
+        for (Throwable cause = t; t != null; t = t.getCause()) {
+            if (exType.isInstance(cause)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     public void testGetRootCause() {
         Exception root = new RuntimeException("foobar");
         ElasticsearchException exception = new ElasticsearchException("foo", new ElasticsearchException("bar", new IllegalArgumentException("index is closed", root)));
         assertEquals(root, exception.getRootCause());
-        assertTrue(exception.contains(RuntimeException.class));
-        assertFalse(exception.contains(EOFException.class));
+        assertTrue(contains(exception, RuntimeException.class));
+        assertFalse(contains(exception, EOFException.class));
     }
 
     public void testToString() {
diff --git a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
index b28758ed941..96f9cbb3522 100644
--- a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
+++ b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
@@ -663,7 +663,7 @@ public class ExceptionSerializationTests extends ESTestCase {
         ids.put(22, null); // was CreateFailedEngineException
         ids.put(23, org.elasticsearch.index.shard.IndexShardStartedException.class);
         ids.put(24, org.elasticsearch.search.SearchContextMissingException.class);
-        ids.put(25, org.elasticsearch.script.ScriptException.class);
+        ids.put(25, org.elasticsearch.script.GeneralScriptException.class);
         ids.put(26, org.elasticsearch.index.shard.TranslogRecoveryPerformer.BatchOperationException.class);
         ids.put(27, org.elasticsearch.snapshots.SnapshotCreationException.class);
         ids.put(28, org.elasticsearch.index.engine.DeleteFailedEngineException.class);
@@ -778,6 +778,7 @@ public class ExceptionSerializationTests extends ESTestCase {
         ids.put(140, org.elasticsearch.discovery.Discovery.FailedToCommitClusterStateException.class);
         ids.put(141, org.elasticsearch.index.query.QueryShardException.class);
         ids.put(142, ShardStateAction.NoLongerPrimaryShardException.class);
+        ids.put(143, org.elasticsearch.script.ScriptException.class);
 
         Map<Class<? extends ElasticsearchException>, Integer> reverse = new HashMap<>();
         for (Map.Entry<Integer, Class<? extends ElasticsearchException>> entry : ids.entrySet()) {
diff --git a/core/src/test/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunctionTests.java b/core/src/test/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunctionTests.java
index 861476b9d47..21fbf43a133 100644
--- a/core/src/test/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunctionTests.java
+++ b/core/src/test/java/org/elasticsearch/common/lucene/search/function/ScriptScoreFunctionTests.java
@@ -23,7 +23,7 @@ import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.script.AbstractDoubleSearchScript;
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptException;
+import org.elasticsearch.script.GeneralScriptException;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.test.ESTestCase;
 
@@ -57,7 +57,7 @@ public class ScriptScoreFunctionTests extends ESTestCase {
             }
         });
         LeafScoreFunction leafScoreFunction = scoreFunction.getLeafScoreFunction(null);
-        ScriptException expected = expectThrows(ScriptException.class, () -> {
+        GeneralScriptException expected = expectThrows(GeneralScriptException.class, () -> {
             leafScoreFunction.score(randomInt(), randomFloat());
         });
         assertTrue(expected.getMessage().contains("returned NaN"));
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
index 715694fe890..bbfdb720814 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
@@ -64,7 +64,7 @@ public class ScriptContextTests extends ESTestCase {
                 Script script = new Script("1", scriptType, MockScriptEngine.NAME, null);
                 scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_globally_disabled_op"), Collections.emptyMap(), null);
                 fail("script compilation should have been rejected");
-            } catch (ScriptException e) {
+            } catch (IllegalStateException e) {
                 assertThat(e.getMessage(), containsString("scripts of type [" + scriptType + "], operation [" + PLUGIN_NAME + "_custom_globally_disabled_op] and lang [" + MockScriptEngine.NAME + "] are disabled"));
             }
         }
@@ -76,7 +76,7 @@ public class ScriptContextTests extends ESTestCase {
         try {
             scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_exp_disabled_op"), Collections.emptyMap(), null);
             fail("script compilation should have been rejected");
-        } catch (ScriptException e) {
+        } catch (IllegalStateException e) {
             assertTrue(e.getMessage(), e.getMessage().contains("scripts of type [inline], operation [" + PLUGIN_NAME + "_custom_exp_disabled_op] and lang [" + MockScriptEngine.NAME + "] are disabled"));
         }
 
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptExceptionTests.java b/core/src/test/java/org/elasticsearch/script/ScriptExceptionTests.java
new file mode 100644
index 00000000000..929c14e45e2
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/script/ScriptExceptionTests.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+import org.elasticsearch.common.io.stream.DataOutputStreamOutput;
+import org.elasticsearch.common.io.stream.InputStreamStreamInput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+/** Simple tests for {@link ScriptException} */
+public class ScriptExceptionTests extends ESTestCase {
+    
+    /** ensure we can round trip in serialization */
+    public void testRoundTrip() throws IOException {
+        ScriptException e = new ScriptException("messageData", new Exception("causeData"), Arrays.asList("stack1", "stack2"), 
+                                                "sourceData", "langData");
+        
+        ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+        StreamOutput output = new DataOutputStreamOutput(new DataOutputStream(bytes));
+        e.writeTo(output);
+        output.close();
+        
+        StreamInput input = new InputStreamStreamInput(new ByteArrayInputStream(bytes.toByteArray()));
+        ScriptException e2 = new ScriptException(input);
+        input.close();
+        
+        assertEquals(e.getMessage(), e2.getMessage());
+        assertEquals(e.getScriptStack(), e2.getScriptStack());
+        assertEquals(e.getScript(), e2.getScript());
+        assertEquals(e.getLang(), e2.getLang());
+    }
+    
+    /** Test that our elements are present in the json output */
+    public void testJsonOutput() {
+        ScriptException e = new ScriptException("messageData", new Exception("causeData"), Arrays.asList("stack1", "stack2"), 
+                                                "sourceData", "langData");
+        String json = e.toJsonString();
+        assertTrue(json.contains(e.getMessage()));
+        assertTrue(json.contains(e.getCause().getMessage()));
+        assertTrue(json.contains("stack1"));
+        assertTrue(json.contains("stack2"));
+        assertTrue(json.contains(e.getScript()));
+        assertTrue(json.contains(e.getLang()));
+    }
+
+    /** ensure the script stack is immutable */
+    public void testImmutableStack() {
+        ScriptException e = new ScriptException("a", new Exception(), Arrays.asList("element1", "element2"), "a", "b");
+        List<String> stack = e.getScriptStack();
+        expectThrows(UnsupportedOperationException.class, () -> {
+            stack.add("no");
+        });
+    }
+
+    /** ensure no parameters can be null */
+    public void testNoLeniency() {
+        expectThrows(NullPointerException.class, () -> {
+           new ScriptException(null, new Exception(), Collections.emptyList(), "a", "b");
+        });
+        expectThrows(NullPointerException.class, () -> {
+            new ScriptException("test", null, Collections.emptyList(), "a", "b");
+        });
+        expectThrows(NullPointerException.class, () -> {
+            new ScriptException("test", new Exception(), null, "a", "b");
+        });
+        expectThrows(NullPointerException.class, () -> {
+            new ScriptException("test", new Exception(), Collections.emptyList(), null, "b");
+        });
+        expectThrows(NullPointerException.class, () -> {
+            new ScriptException("test", new Exception(), Collections.emptyList(), "a", null);
+        });
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
index 890ffccc514..c81c8b10a80 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
@@ -481,7 +481,7 @@ public class ScriptServiceTests extends ESTestCase {
         try {
             scriptService.compile(new Script(script, scriptType, lang, null), scriptContext, Collections.emptyMap(), emptyClusterState());
             fail("compile should have been rejected for lang [" + lang + "], script_type [" + scriptType + "], scripted_op [" + scriptContext + "]");
-        } catch(ScriptException e) {
+        } catch(IllegalStateException e) {
             //all good
         }
     }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java
index 3f8bbf7ad13..85a82ac79da 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionExecutableScript.java
@@ -25,7 +25,7 @@ import java.util.Map;
 import org.apache.lucene.expressions.Expression;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
-import org.elasticsearch.script.ScriptException;
+import org.elasticsearch.script.GeneralScriptException;
 
 /**
  * A bridge to evaluate an {@link Expression} against a map of variables in the context
@@ -45,7 +45,7 @@ public class ExpressionExecutableScript implements ExecutableScript {
         int functionValuesLength = expression.variables.length;
 
         if (vars.size() != functionValuesLength) {
-            throw new ScriptException("Error using " + compiledScript + ". " +
+            throw new GeneralScriptException("Error using " + compiledScript + ". " +
                     "The number of variables in an executable expression script [" +
                     functionValuesLength + "] must match the number of variables in the variable map" +
                     " [" + vars.size() + "].");
@@ -72,12 +72,12 @@ public class ExpressionExecutableScript implements ExecutableScript {
                 double doubleValue = ((Number)value).doubleValue();
                 functionValuesMap.get(name).setValue(doubleValue);
             } else {
-                throw new ScriptException("Error using " + compiledScript + ". " +
+                throw new GeneralScriptException("Error using " + compiledScript + ". " +
                         "Executable expressions scripts can only process numbers." +
                         "  The variable [" + name + "] is not a number.");
             }
         } else {
-            throw new ScriptException("Error using " + compiledScript + ". " +
+            throw new GeneralScriptException("Error using " + compiledScript + ". " +
                     "The variable [" + name + "] does not exist in the executable expressions script.");
         }
     }
@@ -87,7 +87,7 @@ public class ExpressionExecutableScript implements ExecutableScript {
         try {
             return ((Expression) compiledScript.compiled()).evaluate(NO_DOCUMENT, functionValuesArray);
         } catch (Exception exception) {
-            throw new ScriptException("Error evaluating " + compiledScript, exception);
+            throw new GeneralScriptException("Error evaluating " + compiledScript, exception);
         }
     }
 }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
index b06590a56e2..5c4ff3ca94d 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
@@ -50,7 +50,7 @@ import java.security.AccessControlContext;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.text.ParseException;
-import java.util.Collections;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
@@ -107,7 +107,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                     // NOTE: validation is delayed to allow runtime vars, and we don't have access to per index stuff here
                     return JavascriptCompiler.compile(scriptSource, JavascriptCompiler.DEFAULT_FUNCTIONS, loader);
                 } catch (ParseException e) {
-                    throw new ScriptException("Failed to parse expression: " + scriptSource, e);
+                    throw convertToScriptException("compile error", scriptSource, scriptSource, e);
                 }
             }
         });
@@ -115,15 +115,15 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
 
     @Override
     public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, @Nullable Map<String, Object> vars) {
-        try {
-            Expression expr = (Expression)compiledScript.compiled();
-            MapperService mapper = lookup.doc().mapperService();
-            // NOTE: if we need to do anything complicated with bindings in the future, we can just extend Bindings,
-            // instead of complicating SimpleBindings (which should stay simple)
-            SimpleBindings bindings = new SimpleBindings();
-            ReplaceableConstValueSource specialValue = null;
-
-            for (String variable : expr.variables) {
+        Expression expr = (Expression)compiledScript.compiled();
+        MapperService mapper = lookup.doc().mapperService();
+        // NOTE: if we need to do anything complicated with bindings in the future, we can just extend Bindings,
+        // instead of complicating SimpleBindings (which should stay simple)
+        SimpleBindings bindings = new SimpleBindings();
+        ReplaceableConstValueSource specialValue = null;
+        
+        for (String variable : expr.variables) {
+            try {
                 if (variable.equals("_score")) {
                     bindings.add(new SortField("_score", SortField.Type.SCORE));
                 } else if (variable.equals("_value")) {
@@ -141,7 +141,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                     if (value instanceof Number) {
                         bindings.add(variable, new DoubleConstValueSource(((Number) value).doubleValue()));
                     } else {
-                        throw new ScriptException("Parameter [" + variable + "] must be a numeric type");
+                        throw new ParseException("Parameter [" + variable + "] must be a numeric type", 0);
                     }
 
                 } else {
@@ -150,10 +150,10 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                     String variablename = "value"; // .value is the default for doc['field'], its optional.
                     VariableContext[] parts = VariableContext.parse(variable);
                     if (parts[0].text.equals("doc") == false) {
-                        throw new ScriptException("Unknown variable [" + parts[0].text + "] in expression");
+                        throw new ParseException("Unknown variable [" + parts[0].text + "]", 0);
                     }
                     if (parts.length < 2 || parts[1].type != VariableContext.Type.STR_INDEX) {
-                        throw new ScriptException("Variable 'doc' in expression must be used with a specific field like: doc['myfield']");
+                        throw new ParseException("Variable 'doc' must be used with a specific field like: doc['myfield']", 3);
                     } else {
                         fieldname = parts[1].text;
                     }
@@ -163,17 +163,17 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                         } else if (parts[2].type == VariableContext.Type.MEMBER) {
                             variablename = parts[2].text;
                         } else {
-                            throw new ScriptException("Only member variables or member methods may be accessed on a field when not accessing the field directly");
+                            throw new IllegalArgumentException("Only member variables or member methods may be accessed on a field when not accessing the field directly");
                         }
                     }
                     if (parts.length > 3) {
-                        throw new ScriptException("Variable [" + variable + "] does not follow an allowed format of either doc['field'] or doc['field'].method()");
+                        throw new IllegalArgumentException("Variable [" + variable + "] does not follow an allowed format of either doc['field'] or doc['field'].method()");
                     }
 
                     MappedFieldType fieldType = mapper.fullName(fieldname);
 
                     if (fieldType == null) {
-                        throw new ScriptException("Field [" + fieldname + "] used in expression does not exist in mappings");
+                        throw new ParseException("Field [" + fieldname + "] does not exist in mappings", 5);
                     }
 
                     IndexFieldData<?> fieldData = lookup.doc().fieldDataService().getForField(fieldType);
@@ -205,18 +205,37 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                             valueSource = NumericField.getMethod(fieldData, fieldname, methodname);
                         }
                     } else {
-                        throw new ScriptException("Field [" + fieldname + "] used in expression must be numeric, date, or geopoint");
+                        throw new ParseException("Field [" + fieldname + "] must be numeric, date, or geopoint", 5);
                     }
                     
                     bindings.add(variable, valueSource);
                 }
+            } catch (Exception e) {
+                // we defer "binding" of variables until here: give context for that variable
+                throw convertToScriptException("link error", expr.sourceText, variable, e);
             }
-
-            final boolean needsScores = expr.getSortField(bindings, false).needsScores();
-            return new ExpressionSearchScript(compiledScript, bindings, specialValue, needsScores);
-        } catch (Exception exception) {
-            throw new ScriptException("Error during search with " + compiledScript, exception);
         }
+        
+        final boolean needsScores = expr.getSortField(bindings, false).needsScores();
+        return new ExpressionSearchScript(compiledScript, bindings, specialValue, needsScores);
+    }
+    
+    /**
+     * converts a ParseException at compile-time or link-time to a ScriptException
+     */
+    private ScriptException convertToScriptException(String message, String source, String portion, Throwable cause) {
+        List<String> stack = new ArrayList<>();
+        stack.add(portion);
+        StringBuilder pointer = new StringBuilder();
+        if (cause instanceof ParseException) {
+            int offset = ((ParseException) cause).getErrorOffset();
+            for (int i = 0; i < offset; i++) {
+                pointer.append(' ');
+            }
+        }
+        pointer.append("^---- HERE");
+        stack.add(pointer.toString());
+        throw new ScriptException(message, cause, stack, source, NAME);
     }
 
     @Override
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java
index edc6a7c3924..af89450613c 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionSearchScript.java
@@ -33,7 +33,7 @@ import org.apache.lucene.search.Scorer;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.LeafSearchScript;
-import org.elasticsearch.script.ScriptException;
+import org.elasticsearch.script.GeneralScriptException;
 import org.elasticsearch.script.SearchScript;
 
 /**
@@ -73,7 +73,7 @@ class ExpressionSearchScript implements SearchScript {
                 try {
                     return values.doubleVal(docid);
                 } catch (Exception exception) {
-                    throw new ScriptException("Error evaluating " + compiledScript, exception);
+                    throw new GeneralScriptException("Error evaluating " + compiledScript, exception);
                 }
             }
 
@@ -114,7 +114,7 @@ class ExpressionSearchScript implements SearchScript {
                     if (value instanceof Number) {
                         specialValue.setValue(((Number)value).doubleValue());
                     } else {
-                        throw new ScriptException("Cannot use expression with text variable using " + compiledScript);
+                        throw new GeneralScriptException("Cannot use expression with text variable using " + compiledScript);
                     }
                 }
             }
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java
index 209718124b6..2b74aaa9f1e 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTests.java
@@ -22,36 +22,50 @@ package org.elasticsearch.script.expression;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.script.CompiledScript;
+import org.elasticsearch.script.ScriptException;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
+import java.text.ParseException;
 import java.util.Collections;
 
 public class ExpressionTests extends ESSingleNodeTestCase {
-
-    public void testNeedsScores() {
+    ExpressionScriptEngineService service;
+    SearchLookup lookup;
+    
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
         IndexService index = createIndex("test", Settings.EMPTY, "type", "d", "type=double");
-
-        ExpressionScriptEngineService service = new ExpressionScriptEngineService(Settings.EMPTY);
-        SearchLookup lookup = new SearchLookup(index.mapperService(), index.fieldData(), null);
-
-        Object compiled = service.compile(null, "1.2", Collections.emptyMap());
-        SearchScript ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "expression", compiled), lookup, Collections.<String, Object>emptyMap());
-        assertFalse(ss.needsScores());
-
-        compiled = service.compile(null, "doc['d'].value", Collections.emptyMap());
-        ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "expression", compiled), lookup, Collections.<String, Object>emptyMap());
-        assertFalse(ss.needsScores());
-
-        compiled = service.compile(null, "1/_score", Collections.emptyMap());
-        ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "expression", compiled), lookup, Collections.<String, Object>emptyMap());
-        assertTrue(ss.needsScores());
-
-        compiled = service.compile(null, "doc['d'].value * _score", Collections.emptyMap());
-        ss = service.search(new CompiledScript(ScriptType.INLINE, "randomName", "expression", compiled), lookup, Collections.<String, Object>emptyMap());
-        assertTrue(ss.needsScores());
+        service = new ExpressionScriptEngineService(Settings.EMPTY);
+        lookup = new SearchLookup(index.mapperService(), index.fieldData(), null);
+    }
+    
+    private SearchScript compile(String expression) {
+        Object compiled = service.compile(null, expression, Collections.emptyMap());
+        return service.search(new CompiledScript(ScriptType.INLINE, "randomName", "expression", compiled), lookup, Collections.<String, Object>emptyMap());
     }
 
+    public void testNeedsScores() {
+        assertFalse(compile("1.2").needsScores());
+        assertFalse(compile("doc['d'].value").needsScores());
+        assertTrue(compile("1/_score").needsScores());
+        assertTrue(compile("doc['d'].value * _score").needsScores());
+    }
+    
+    public void testCompileError() {
+        ScriptException e = expectThrows(ScriptException.class, () -> {
+            compile("doc['d'].value * *@#)(@$*@#$ + 4");
+        });
+        assertTrue(e.getCause() instanceof ParseException);
+    }
+    
+    public void testLinkError() {
+        ScriptException e = expectThrows(ScriptException.class, () -> {
+            compile("doc['e'].value * 5");
+        });
+        assertTrue(e.getCause() instanceof ParseException);
+    }
 }
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
index f023ec9f38a..3f2466e52e9 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
@@ -44,7 +44,7 @@ import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptException;
+import org.elasticsearch.script.GeneralScriptException;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
@@ -324,7 +324,7 @@ public class MoreExpressionTests extends ESIntegTestCase {
             assertThat(e.toString() + "should have contained ScriptException",
                     e.toString().contains("ScriptException"), equalTo(true));
             assertThat(e.toString() + "should have contained compilation failure",
-                    e.toString().contains("Failed to parse expression"), equalTo(true));
+                    e.toString().contains("compile error"), equalTo(true));
         }
     }
 
@@ -494,7 +494,7 @@ public class MoreExpressionTests extends ESIntegTestCase {
             ees = new ExpressionExecutableScript(compiledScript, vars);
             ees.run();
             fail("An incorrect number of variables were allowed to be used in an expression.");
-        } catch (ScriptException se) {
+        } catch (GeneralScriptException se) {
             message = se.getMessage();
             assertThat(message + " should have contained number of variables", message.contains("number of variables"), equalTo(true));
         }
@@ -507,7 +507,7 @@ public class MoreExpressionTests extends ESIntegTestCase {
             ees = new ExpressionExecutableScript(compiledScript, vars);
             ees.run();
             fail("A variable was allowed to be set that does not exist in the expression.");
-        } catch (ScriptException se) {
+        } catch (GeneralScriptException se) {
             message = se.getMessage();
             assertThat(message + " should have contained does not exist in", message.contains("does not exist in"), equalTo(true));
         }
@@ -520,7 +520,7 @@ public class MoreExpressionTests extends ESIntegTestCase {
             ees = new ExpressionExecutableScript(compiledScript, vars);
             ees.run();
             fail("A non-number was allowed to be use in the expression.");
-        } catch (ScriptException se) {
+        } catch (GeneralScriptException se) {
             message = se.getMessage();
             assertThat(message + " should have contained process numbers", message.contains("process numbers"), equalTo(true));
         }
diff --git a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
index 5d06196a83b..ef4f6fd76b6 100644
--- a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
+++ b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
@@ -50,7 +50,7 @@ import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.ScoreAccessor;
 import org.elasticsearch.script.ScriptEngineService;
-import org.elasticsearch.script.ScriptException;
+import org.elasticsearch.script.GeneralScriptException;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.lookup.LeafSearchLookup;
 import org.elasticsearch.search.lookup.SearchLookup;
@@ -193,7 +193,7 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
             if (logger.isTraceEnabled()) {
                 logger.trace("exception compiling Groovy script:", e);
             }
-            throw new ScriptException("failed to compile groovy script", e);
+            throw new GeneralScriptException("failed to compile groovy script", e);
         }
     }
 
@@ -219,7 +219,7 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
             }
             return new GroovyScript(compiledScript, createScript(compiledScript.compiled(), allVars), this.logger);
         } catch (ReflectiveOperationException e) {
-            throw new ScriptException("failed to build executable " + compiledScript, e);
+            throw new GeneralScriptException("failed to build executable " + compiledScript, e);
         }
     }
 
@@ -239,7 +239,7 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
                 try {
                     scriptObject = createScript(compiledScript.compiled(), allVars);
                 } catch (ReflectiveOperationException e) {
-                    throw new ScriptException("failed to build search " + compiledScript, e);
+                    throw new GeneralScriptException("failed to build search " + compiledScript, e);
                 }
                 return new GroovyScript(compiledScript, scriptObject, leafLookup, logger);
             }
@@ -312,7 +312,7 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
                 if (logger.isTraceEnabled()) {
                     logger.trace("failed to run {}", e, compiledScript);
                 }
-                throw new ScriptException("failed to run " + compiledScript, e);
+                throw new GeneralScriptException("failed to run " + compiledScript, e);
             }
         }
 
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java
index e0e439aa6e8..7bdfe5a205e 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java
@@ -24,7 +24,7 @@ import org.apache.lucene.util.Constants;
 import org.codehaus.groovy.control.MultipleCompilationErrorsException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.CompiledScript;
-import org.elasticsearch.script.ScriptException;
+import org.elasticsearch.script.GeneralScriptException;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
 
@@ -157,7 +157,7 @@ public class GroovySecurityTests extends ESTestCase {
         try {
             doTest(script);
             fail("did not get expected exception");
-        } catch (ScriptException expected) {
+        } catch (GeneralScriptException expected) {
             Throwable cause = expected.getCause();
             assertNotNull(cause);
             if (exceptionClass.isAssignableFrom(cause.getClass()) == false) {
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
index b93eaaa9b52..d565f66b8e2 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
@@ -31,7 +31,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.ScriptEngineService;
-import org.elasticsearch.script.ScriptException;
+import org.elasticsearch.script.GeneralScriptException;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.lookup.SearchLookup;
 
@@ -185,7 +185,7 @@ public final class MustacheScriptEngineService extends AbstractComponent impleme
                 });
             } catch (Exception e) {
                 logger.error("Error running {}", e, template);
-                throw new ScriptException("Error running " + template, e);
+                throw new GeneralScriptException("Error running " + template, e);
             }
             return result.bytes();
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
index 666f28bb406..36d41056013 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
@@ -29,6 +29,7 @@ import java.net.URL;
 import java.security.CodeSource;
 import java.security.SecureClassLoader;
 import java.security.cert.Certificate;
+import java.util.BitSet;
 
 import static org.elasticsearch.painless.WriterConstants.CLASS_NAME;
 
@@ -93,9 +94,28 @@ final class Compiler {
      * @return An {@link Executable} Painless script.
      */
     static Executable compile(Loader loader, String name, String source, CompilerSettings settings) {
-        byte[] bytes = compile(name, source, settings);
+        if (source.length() > MAXIMUM_SOURCE_LENGTH) {
+            throw new IllegalArgumentException("Scripts may be no longer than " + MAXIMUM_SOURCE_LENGTH +
+                " characters.  The passed in script is " + source.length() + " characters.  Consider using a" +
+                " plugin if a script longer than this length is a requirement.");
+        }
 
-        return createExecutable(loader, name, source, bytes);
+        Reserved reserved = new Reserved();
+        SSource root = Walker.buildPainlessTree(source, reserved, settings);
+        Variables variables = Analyzer.analyze(reserved, root);
+        BitSet expressions = new BitSet(source.length());
+
+        byte[] bytes = Writer.write(settings, name, source, variables, root, expressions);
+        try {
+            Class<? extends Executable> clazz = loader.define(CLASS_NAME, bytes);
+            java.lang.reflect.Constructor<? extends Executable> constructor = 
+                    clazz.getConstructor(String.class, String.class, BitSet.class);
+
+            return constructor.newInstance(name, source, expressions);
+        } catch (Exception exception) { // Catch everything to let the user know this is something caused internally.
+            throw new IllegalStateException(
+                    "An internal error occurred attempting to define the script [" + name + "].", exception);
+        }
     }
 
     /**
@@ -115,27 +135,7 @@ final class Compiler {
         SSource root = Walker.buildPainlessTree(source, reserved, settings);
         Variables variables = Analyzer.analyze(reserved, root);
 
-        return Writer.write(settings, name, source, variables, root);
-    }
-
-    /**
-     * Generates an {@link Executable} that can run a Painless script.
-     * @param loader The {@link Loader} to define the script's class file.
-     * @param name The name of the script.
-     * @param source The source text of the script.
-     * @param bytes The ASM generated byte code to define the class with.
-     * @return A Painless {@link Executable} script.
-     */
-    private static Executable createExecutable(Loader loader, String name, String source, byte[] bytes) {
-        try {
-            Class<? extends Executable> clazz = loader.define(CLASS_NAME, bytes);
-            java.lang.reflect.Constructor<? extends Executable> constructor = clazz.getConstructor(String.class, String.class);
-
-            return constructor.newInstance(name, source);
-        } catch (Exception exception) { // Catch everything to let the user know this is something caused internally.
-            throw new IllegalStateException(
-                    "An internal error occurred attempting to define the script [" + name + "].", exception);
-        }
+        return Writer.write(settings, name, source, variables, root, new BitSet(source.length()));
     }
 
     /**
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java
index 7681ae8232a..c6e38374aa7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Executable.java
@@ -22,6 +22,7 @@ package org.elasticsearch.painless;
 import org.apache.lucene.search.Scorer;
 import org.elasticsearch.search.lookup.LeafDocLookup;
 
+import java.util.BitSet;
 import java.util.Map;
 
 /**
@@ -31,10 +32,12 @@ public abstract class Executable {
 
     private final String name;
     private final String source;
+    private final BitSet statements;
 
-    public Executable(final String name, final String source) {
+    public Executable(String name, String source, BitSet statements) {
         this.name = name;
         this.source = source;
+        this.statements = statements;
     }
 
     public String getName() {
@@ -45,6 +48,24 @@ public abstract class Executable {
         return source;
     }
 
+    /** 
+     * Finds the start of the first statement boundary that is
+     * on or before {@code offset}. If one is not found, {@code -1}
+     * is returned.
+     */
+    public int getPreviousStatement(int offset) {
+        return statements.previousSetBit(offset);
+    }
+    
+    /** 
+     * Finds the start of the first statement boundary that is
+     * after {@code offset}. If one is not found, {@code -1}
+     * is returned.
+     */
+    public int getNextStatement(int offset) {
+        return statements.nextSetBit(offset+1);
+    }
+
     public abstract Object execute(
         final Map<String, Object> params, final Scorer scorer, final LeafDocLookup doc, final Object value);
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index 90c02b7e801..7c11c5b15bf 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -30,6 +30,7 @@ import org.objectweb.asm.commons.Method;
 
 import java.util.ArrayDeque;
 import java.util.ArrayList;
+import java.util.BitSet;
 import java.util.Deque;
 import java.util.List;
 
@@ -86,13 +87,15 @@ import static org.elasticsearch.painless.WriterConstants.UTILITY_TYPE;
  * shared by the nodes of the Painless tree.
  */
 public final class MethodWriter extends GeneratorAdapter {
+    private final BitSet statements;
 
     private final Deque<List<org.objectweb.asm.Type>> stringConcatArgs = (INDY_STRING_CONCAT_BOOTSTRAP_HANDLE == null) ?
             null : new ArrayDeque<>();
 
-    MethodWriter(int access, Method method, org.objectweb.asm.Type[] exceptions, ClassVisitor cv) {
+    MethodWriter(int access, Method method, org.objectweb.asm.Type[] exceptions, ClassVisitor cv, BitSet statements) {
         super(Opcodes.ASM5, cv.visitMethod(access, method.getName(), method.getDescriptor(), null, getInternalNames(exceptions)),
                 access, method.getName(), method.getDescriptor());
+        this.statements = statements;
     }
 
     private static String[] getInternalNames(final org.objectweb.asm.Type[] types) {
@@ -106,8 +109,33 @@ public final class MethodWriter extends GeneratorAdapter {
         return names;
     }
 
-    public void writeLoopCounter(final int slot, final int count) {
+    /** 
+     * Marks a new statement boundary. 
+     * <p>
+     * This is invoked for each statement boundary (leaf {@code S*} nodes).
+     */
+    public void writeStatementOffset(int offset) {
+        // ensure we don't have duplicate stuff going in here. can catch bugs
+        // (e.g. nodes get assigned wrong offsets by antlr walker)
+        assert statements.get(offset) == false;
+        statements.set(offset);
+    }
+
+    /** 
+     * Encodes the offset into the line number table as {@code offset + 1}.
+     * <p>
+     * This is invoked before instructions that can hit exceptions.
+     */
+    public void writeDebugInfo(int offset) {
+        // TODO: maybe track these in bitsets too? this is trickier...
+        Label label = new Label();
+        visitLabel(label);
+        visitLineNumber(offset + 1, label);
+    }
+
+    public void writeLoopCounter(int slot, int count, int offset) {
         if (slot > -1) {
+            writeDebugInfo(offset);
             final Label end = new Label();
 
             iinc(slot, -count);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
index 04d5a8d3d60..83ae7b664c1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
@@ -22,10 +22,13 @@ package org.elasticsearch.painless;
 import org.apache.lucene.search.Scorer;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.LeafSearchScript;
+import org.elasticsearch.script.ScriptException;
 import org.elasticsearch.search.lookup.LeafDocLookup;
 import org.elasticsearch.search.lookup.LeafSearchLookup;
 
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -114,7 +117,63 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
      */
     @Override
     public Object run() {
-        return executable.execute(variables, scorer, doc, aggregationValue);
+        try {
+            return executable.execute(variables, scorer, doc, aggregationValue);
+        } catch (PainlessError | Exception t) {
+            throw convertToScriptException(t);
+        }
+    }
+    
+    private ScriptException convertToScriptException(Throwable t) {
+        // create a script stack: this is just the script portion
+        List<String> scriptStack = new ArrayList<>();
+        for (StackTraceElement element : t.getStackTrace()) {
+            if (WriterConstants.CLASS_NAME.equals(element.getClassName())) {
+                // found the script portion
+                int offset = element.getLineNumber();
+                if (offset == -1) {
+                    scriptStack.add("<<< unknown portion of script >>>");
+                } else {
+                    offset--; // offset is 1 based, line numbers must be!
+                    int startOffset = executable.getPreviousStatement(offset);
+                    if (startOffset == -1) {
+                        assert false; // should never happen unless we hit exc in ctor prologue...
+                        startOffset = 0;
+                    }
+                    int endOffset = executable.getNextStatement(startOffset);
+                    if (endOffset == -1) {
+                        endOffset = executable.getSource().length();
+                    }
+                    // TODO: if this is still too long, truncate and use ellipses
+                    String snippet = executable.getSource().substring(startOffset, endOffset);
+                    scriptStack.add(snippet);
+                    StringBuilder pointer = new StringBuilder();
+                    for (int i = startOffset; i < offset; i++) {
+                        pointer.append(' ');
+                    }
+                    pointer.append("^---- HERE");
+                    scriptStack.add(pointer.toString());
+                }
+                break;
+            // but filter our own internal stacks (e.g. indy bootstrap)
+            } else if (!shouldFilter(element)) {
+                scriptStack.add(element.toString());
+            }
+        }
+        // build a name for the script:
+        final String name;
+        if (PainlessScriptEngineService.INLINE_NAME.equals(executable.getName())) {
+            name = executable.getSource();
+        } else {
+            name = executable.getName();
+        }
+        throw new ScriptException("runtime error", t, scriptStack, name, PainlessScriptEngineService.NAME);
+    }
+    
+    /** returns true for methods that are part of the runtime */
+    private static boolean shouldFilter(StackTraceElement element) {
+        return element.getClassName().startsWith("org.elasticsearch.painless.") ||
+               element.getClassName().startsWith("java.lang.invoke.");
     }
 
     /**
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
index 6e2d0e1431b..362d81f2479 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
@@ -33,15 +33,15 @@ import static org.elasticsearch.painless.WriterConstants.EXECUTE;
 import static org.elasticsearch.painless.WriterConstants.MAP_GET;
 import static org.elasticsearch.painless.WriterConstants.MAP_TYPE;
 
+import java.util.BitSet;
+
 /**
  * Runs the writing phase of compilation using the Painless AST.
  */
 final class Writer {
 
-    static byte[] write(final CompilerSettings settings,
-                               String name, final String source, final Variables variables, final SSource root) {
-        final Writer writer = new Writer(settings, name, source, variables, root);
-
+    static byte[] write(CompilerSettings settings, String name, String source, Variables variables, SSource root, BitSet expressions) {
+        Writer writer = new Writer(settings, name, source, variables, root, expressions);
         return writer.getBytes();
     }
 
@@ -54,8 +54,7 @@ final class Writer {
     private final ClassWriter writer;
     private final MethodWriter adapter;
 
-    private Writer(final CompilerSettings settings,
-                     String name, final String source, final Variables variables, final SSource root) {
+    private Writer(CompilerSettings settings, String name, String source, Variables variables, SSource root, BitSet expressions) {
         this.settings = settings;
         this.scriptName = name;
         this.source = source;
@@ -67,7 +66,7 @@ final class Writer {
         writeBegin();
         writeConstructor();
 
-        adapter = new MethodWriter(Opcodes.ACC_PUBLIC, EXECUTE, null, writer);
+        adapter = new MethodWriter(Opcodes.ACC_PUBLIC, EXECUTE, null, writer, expressions);
 
         writeExecute();
         writeEnd();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index 410c06e6fd7..a4804cde434 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -29,6 +29,7 @@ import org.objectweb.asm.commons.Method;
 import java.lang.invoke.CallSite;
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodType;
+import java.util.BitSet;
 import java.util.Map;
 
 /**
@@ -42,7 +43,7 @@ public final class WriterConstants {
     public final static String CLASS_NAME      = BASE_CLASS_NAME + "$Script";
     public final static Type CLASS_TYPE        = Type.getObjectType(CLASS_NAME.replace('.', '/'));
 
-    public final static Method CONSTRUCTOR = getAsmMethod(void.class, "<init>", String.class, String.class);
+    public final static Method CONSTRUCTOR = getAsmMethod(void.class, "<init>", String.class, String.class, BitSet.class);
     public final static Method EXECUTE     =
         getAsmMethod(Object.class, "execute", Map.class, Scorer.class, LeafDocLookup.class, Object.class);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index 3669ab24ecc..d42cb769495 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -380,7 +380,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
             String name = declvar.ID().getText();
             AExpression expression = declvar.expression() == null ? null : (AExpression)visitExpression(declvar.expression());
 
-            declarations.add(new SDeclaration(line(ctx), offset(ctx), location(ctx), type, name, expression));
+            declarations.add(new SDeclaration(line(declvar), offset(declvar), location(declvar), type, name, expression));
         }
 
         return new SDeclBlock(line(ctx), offset(ctx), location(ctx), declarations);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
index 414f6afe038..a27f6748086 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
@@ -19,9 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.MethodWriter;
-import org.objectweb.asm.Label;
-
 /**
  * The superclass for all other nodes.
  */
@@ -51,15 +48,4 @@ public abstract class ANode {
     public String error(final String message) {
         return "Error " + location  + ": " + message;
     }
-
-    /**
-     * Writes line number information
-     * <p>
-     * Currently we emit line number data for for leaf S-nodes
-     */
-    void writeDebugInfo(MethodWriter writer) {
-        Label label = new Label();
-        writer.visitLabel(label);
-        writer.visitLineNumber(line, label);
-    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 0c083111d66..23179a9ed0d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -470,6 +470,7 @@ public final class EBinary extends AExpression {
 
     @Override
     void write(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         if (actual.sort == Sort.STRING && operation == Operation.ADD) {
             if (!cat) {
                 writer.writeNewStrings();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
index e2fe5cb6d10..96e07511485 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
@@ -51,6 +51,7 @@ final class ECast extends AExpression {
     @Override
     void write(MethodWriter writer) {
         child.write(writer);
+        writer.writeDebugInfo(offset);
         writer.writeCast(cast);
         writer.writeBranch(tru, fals);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index 2de1a5b320e..affd1fe78c0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -248,6 +248,10 @@ public final class EChain extends AExpression {
 
     @Override
     void write(MethodWriter writer) {
+        if (cat) {
+            writer.writeDebugInfo(offset);
+        }
+
         if (cat) {
             writer.writeNewStrings();
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index 4521c76d911..5bc6e4ee400 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -399,6 +399,7 @@ public final class EComp extends AExpression {
 
     @Override
     void write(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         boolean branch = tru != null || fals != null;
         org.objectweb.asm.Type rtype = right.actual.type;
         Sort rsort = right.actual.sort;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
index a13176e56b7..d7f7ab7e4b6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
@@ -78,6 +78,7 @@ public final class EConditional extends AExpression {
 
     @Override
     void write(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         Label localfals = new Label();
         Label end = new Label();
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index 8b02522fca9..2122174442d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -165,6 +165,7 @@ public final class EUnary extends AExpression {
 
     @Override
     void write(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         if (operation == Operation.NOT) {
             if (tru == null && fals == null) {
                 Label localfals = new Label();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
index cea5c629b14..b8e2fc7739b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
@@ -60,6 +60,7 @@ public final class LArrayLength extends ALink {
 
     @Override
     void load(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         writer.arrayLength();
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
index 4411913ea07..02e8ceacd9f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
@@ -74,11 +74,13 @@ public final class LBrace extends ALink {
 
     @Override
     void load(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         writer.arrayLoad(after.type);
     }
 
     @Override
     void store(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         writer.arrayStore(after.type);
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
index fdb369468bc..30775e3a6e7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
@@ -91,6 +91,7 @@ public final class LCall extends ALink {
 
     @Override
     void load(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         for (AExpression argument : arguments) {
             argument.write(writer);
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
index 06dee04fca6..bce3f213a71 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
@@ -61,6 +61,7 @@ public final class LCast extends ALink {
 
     @Override
     void write(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         writer.writeCast(cast);
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index 452a0732cdb..b00eafe3f9f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -58,12 +58,14 @@ final class LDefArray extends ALink implements IDefLink {
 
     @Override
     void load(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type, index.actual.type);
         writer.invokeDynamic("arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ARRAY_LOAD);
     }
 
     @Override
     void store(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, index.actual.type, after.type);
         writer.invokeDynamic("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ARRAY_STORE);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index 8ddde1cbed4..0292ac3c589 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -67,6 +67,7 @@ final class LDefCall extends ALink implements IDefLink {
 
     @Override
     void load(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         StringBuilder signature = new StringBuilder();
 
         signature.append('(');
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index b0a9d4dd0d9..e762de9e862 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -55,12 +55,14 @@ final class LDefField extends ALink implements IDefLink {
 
     @Override
     void load(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type);
         writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.LOAD);
     }
 
     @Override
     void store(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, after.type);
         writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.STORE);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
index ce986413375..88173ad3f65 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
@@ -105,6 +105,7 @@ public final class LField extends ALink {
 
     @Override
     void load(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         if (java.lang.reflect.Modifier.isStatic(field.modifiers)) {
             writer.getStatic(field.owner.type, field.javaName, field.type.type);
         } else {
@@ -114,6 +115,7 @@ public final class LField extends ALink {
 
     @Override
     void store(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         if (java.lang.reflect.Modifier.isStatic(field.modifiers)) {
             writer.putStatic(field.owner.type, field.javaName, field.type.type);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
index f95c24f8592..b628fee5f37 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
@@ -79,6 +79,7 @@ final class LListShortcut extends ALink {
 
     @Override
     void load(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
             writer.invokeInterface(getter.owner.type, getter.method);
         } else {
@@ -92,6 +93,7 @@ final class LListShortcut extends ALink {
 
     @Override
     void store(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
             writer.invokeInterface(setter.owner.type, setter.method);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
index afe93ba1048..a500a6673fc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
@@ -78,6 +78,7 @@ final class LMapShortcut extends ALink {
 
     @Override
     void load(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
             writer.invokeInterface(getter.owner.type, getter.method);
         } else {
@@ -91,6 +92,7 @@ final class LMapShortcut extends ALink {
 
     @Override
     void store(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
             writer.invokeInterface(setter.owner.type, setter.method);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
index ffe6168f812..c555f04ab95 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
@@ -79,6 +79,7 @@ public final class LNewArray extends ALink {
 
     @Override
     void load(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         for (AExpression argument : arguments) {
             argument.write(writer);
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
index dc8b711ee8c..44738450962 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
@@ -98,6 +98,7 @@ public final class LNewObj extends ALink {
 
     @Override
     void load(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         writer.newInstance(after.type);
 
         if (load) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
index c6aafd47bff..7eb44156e55 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
@@ -84,6 +84,7 @@ final class LShortcut extends ALink {
 
     @Override
     void load(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
             writer.invokeInterface(getter.owner.type, getter.method);
         } else {
@@ -97,6 +98,7 @@ final class LShortcut extends ALink {
 
     @Override
     void store(MethodWriter writer) {
+        writer.writeDebugInfo(offset);
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
             writer.invokeInterface(setter.owner.type, setter.method);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
index 727cd4b929c..4cd1decb26b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
@@ -45,8 +45,6 @@ public final class SBreak extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writeDebugInfo(writer);
-
         writer.goTo(brake);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
index b3b32d741cd..3fb8c8b4680 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
@@ -84,8 +84,7 @@ public final class SCatch extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writeDebugInfo(writer);
-
+        writer.writeStatementOffset(offset);
         Label jump = new Label();
 
         writer.mark(jump);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
index bfea104f23d..18ce5c81231 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
@@ -48,8 +48,6 @@ public final class SContinue extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writeDebugInfo(writer);
-
         writer.goTo(continu);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
index 246fb992836..526d90dca26 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
@@ -66,8 +66,7 @@ public final class SDeclaration extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writeDebugInfo(writer);
-
+        writer.writeStatementOffset(offset);
         if (expression == null) {
             switch (variable.type.sort) {
                 case VOID:   throw new IllegalStateException(error("Illegal tree structure."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
index 3fbc3b9fafc..3b854ea4ebf 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -86,8 +86,7 @@ public final class SDo extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writeDebugInfo(writer);
-
+        writer.writeStatementOffset(offset);
         Label start = new Label();
         Label begin = new Label();
         Label end = new Label();
@@ -103,7 +102,7 @@ public final class SDo extends AStatement {
         condition.fals = end;
         condition.write(writer);
 
-        writer.writeLoopCounter(loopCounterSlot, Math.max(1, block.statementCount));
+        writer.writeLoopCounter(loopCounterSlot, Math.max(1, block.statementCount), offset);
 
         writer.goTo(start);
         writer.mark(end);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index a9f47fb6522..37f1cbeb24c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -60,8 +60,7 @@ public final class SExpression extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writeDebugInfo(writer);
-
+        writer.writeStatementOffset(offset);
         expression.write(writer);
 
         if (methodEscape) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index 5fb1845e551..ac8e721a1f6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -127,8 +127,7 @@ public final class SFor extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writeDebugInfo(writer);
-
+        writer.writeStatementOffset(offset);
         Label start = new Label();
         Label begin = afterthought == null ? start : new Label();
         Label end = new Label();
@@ -160,10 +159,10 @@ public final class SFor extends AStatement {
                 ++statementCount;
             }
 
-            writer.writeLoopCounter(loopCounterSlot, statementCount);
+            writer.writeLoopCounter(loopCounterSlot, statementCount, offset);
             block.write(writer);
         } else {
-            writer.writeLoopCounter(loopCounterSlot, 1);
+            writer.writeLoopCounter(loopCounterSlot, 1, offset);
         }
 
         if (afterthought != null) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
index 180b5023811..1f498215478 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
@@ -68,8 +68,7 @@ public final class SIf extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writeDebugInfo(writer);
-
+        writer.writeStatementOffset(offset);
         Label fals = new Label();
 
         condition.fals = fals;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
index 217584f32ab..d8fd5c0756e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
@@ -89,8 +89,7 @@ public final class SIfElse extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writeDebugInfo(writer);
-
+        writer.writeStatementOffset(offset);
         Label end = new Label();
         Label fals = elseblock != null ? new Label() : end;
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
index 2f342a5b7b9..d6cc5598263 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -52,8 +52,7 @@ public final class SReturn extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writeDebugInfo(writer);
-
+        writer.writeStatementOffset(offset);
         expression.write(writer);
         writer.returnValue();
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
index d002f1f9dad..7c65aafbc56 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
@@ -50,8 +50,7 @@ public final class SThrow extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writeDebugInfo(writer);
-
+        writer.writeStatementOffset(offset);
         expression.write(writer);
         writer.throwException();
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
index a4ef00ef146..0fdb70bdc82 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
@@ -86,8 +86,7 @@ public final class STry extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writeDebugInfo(writer);
-
+        writer.writeStatementOffset(offset);
         Label begin = new Label();
         Label end = new Label();
         Label exception = new Label();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
index 322ae110f66..43ac824dac7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -92,8 +92,7 @@ public final class SWhile extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writeDebugInfo(writer);
-
+        writer.writeStatementOffset(offset);
         Label begin = new Label();
         Label end = new Label();
 
@@ -103,13 +102,13 @@ public final class SWhile extends AStatement {
         condition.write(writer);
 
         if (block != null) {
-            writer.writeLoopCounter(loopCounterSlot, Math.max(1, block.statementCount));
+            writer.writeLoopCounter(loopCounterSlot, Math.max(1, block.statementCount), offset);
 
             block.continu = begin;
             block.brake = end;
             block.write(writer);
         } else {
-            writer.writeLoopCounter(loopCounterSlot, 1);
+            writer.writeLoopCounter(loopCounterSlot, 1, offset);
         }
 
         if (block == null || !block.allEscape) {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/CompoundAssignmentTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/CompoundAssignmentTests.java
index 03593116538..73a92edf668 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/CompoundAssignmentTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/CompoundAssignmentTests.java
@@ -120,34 +120,29 @@ public class CompoundAssignmentTests extends ScriptTestCase {
 
     public void testDivisionByZero() {
         // byte
-        try {
+        expectScriptThrows(ArithmeticException.class, () -> {
             exec("byte x = 1; x /= 0; return x;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
+        });
 
         // short
-        try {
+        expectScriptThrows(ArithmeticException.class, () -> {
             exec("short x = 1; x /= 0; return x;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
+        });
 
         // char
-        try {
+        expectScriptThrows(ArithmeticException.class, () -> {
             exec("char x = 1; x /= 0; return x;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
+        });
 
         // int
-        try {
+        expectScriptThrows(ArithmeticException.class, () -> {
             exec("int x = 1; x /= 0; return x;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
+        });
 
         // long
-        try {
+        expectScriptThrows(ArithmeticException.class, () -> {
             exec("long x = 1; x /= 0; return x;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {}
+        });
     }
 
     public void testRemainder() {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
index 9f171a96889..ca611760f07 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
@@ -21,10 +21,14 @@ package org.elasticsearch.painless;
 
 public class DefOperationTests extends ScriptTestCase {
     public void testIllegalCast() {
-        Exception exception = expectThrows(ClassCastException.class, () -> exec("def x = 1.0; int y = x; return y;"));
+        Exception exception = expectScriptThrows(ClassCastException.class, () -> { 
+            exec("def x = 1.0; int y = x; return y;");
+        });
         assertTrue(exception.getMessage().contains("cannot be cast"));
 
-        exception = expectThrows(ClassCastException.class, () -> exec("def x = (short)1; byte y = x; return y;"));
+        exception = expectScriptThrows(ClassCastException.class, () -> { 
+            exec("def x = (short)1; byte y = x; return y;");
+        });
         assertTrue(exception.getMessage().contains("cannot be cast"));
     }
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
index 72d08b677b9..f9771862335 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
@@ -173,7 +173,7 @@ public class DefOptimizationTests extends ScriptTestCase {
         final String script = "int x;\ndef y = new HashMap();\ny['double'] = 1.0;\nx = y.get('double');\n";
         assertBytecodeExists(script, "INVOKEDYNAMIC get(Ljava/lang/Object;Ljava/lang/String;)I");
 
-        final Exception exception = expectThrows(ClassCastException.class, () -> {
+        final Exception exception = expectScriptThrows(ClassCastException.class, () -> {
             exec(script);
         });
         assertTrue(exception.getMessage().contains("Cannot cast java.lang.Double to java.lang.Integer"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java
index 99a48d1ee9d..61046ab59f4 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java
@@ -114,34 +114,22 @@ public class DivisionTests extends ScriptTestCase {
     }
 
     public void testDivideByZero() throws Exception {
-        try {
+        expectScriptThrows(ArithmeticException.class, () -> {
             exec("int x = 1; int y = 0; return x / y;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {
-            // divide by zero
-        }
+        });
 
-        try {
+        expectScriptThrows(ArithmeticException.class, () -> {
             exec("long x = 1L; long y = 0L; return x / y;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {
-            // divide by zero
-        }
+        });
     }
 
     public void testDivideByZeroConst() throws Exception {
-        try {
+        expectThrows(ArithmeticException.class, () -> {
             exec("return 1/0;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {
-            // divide by zero
-        }
+        });
 
-        try {
+        expectThrows(ArithmeticException.class, () -> {
             exec("return 1L/0L;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {
-            // divide by zero
-        }
+        });
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java
index 3e0e4a5fc71..6662aa95fbd 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java
@@ -34,7 +34,7 @@ public class OverloadTests extends ScriptTestCase {
     public void testMethodDynamic() {
         assertEquals(2, exec("def x = 'abc123abc'; return x.indexOf('c');"));
         assertEquals(8, exec("def x = 'abc123abc'; return x.indexOf('c', 3);"));
-        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("def x = 'abc123abc'; return x.indexOf('c', 3, 'bogus');");
         });
         assertTrue(expected.getMessage().contains("dynamic method [indexOf] with signature [(String,int,String)"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java
index 836405a9829..b60de9233df 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java
@@ -114,19 +114,13 @@ public class RemainderTests extends ScriptTestCase {
     }
 
     public void testDivideByZero() throws Exception {
-        try {
+        expectScriptThrows(ArithmeticException.class, () -> {
             exec("int x = 1; int y = 0; return x % y;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {
-            // divide by zero
-        }
+        });
 
-        try {
+        expectScriptThrows(ArithmeticException.class, () -> {
             exec("long x = 1L; long y = 0L; return x % y;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {
-            // divide by zero
-        }
+        });
     }
 
     public void testDivideByZeroConst() throws Exception {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
index 27558dc745e..b4e416ca614 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
@@ -24,11 +24,13 @@ import org.elasticsearch.common.lucene.ScorerAware;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.ScriptException;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
-import java.util.Collections;
+import junit.framework.AssertionFailedError;
+
 import java.util.HashMap;
 import java.util.Map;
 
@@ -76,4 +78,27 @@ public abstract class ScriptTestCase extends ESTestCase {
         final String asm = Debugger.toString(script);
         assertTrue("bytecode not found", asm.contains(bytecode));
     }
+    
+    /** Checks a specific exception class is thrown (boxed inside ScriptException) and returns it. */
+    public static <T extends Throwable> T expectScriptThrows(Class<T> expectedType, ThrowingRunnable runnable) {
+        try {
+            runnable.run();
+        } catch (Throwable e) {
+            if (e instanceof ScriptException) {
+                e = e.getCause();
+                if (expectedType.isInstance(e)) {
+                    return expectedType.cast(e);
+                }
+            } else {
+                AssertionFailedError assertion = new AssertionFailedError("Expected boxed ScriptException");
+                assertion.initCause(e);
+                throw assertion;
+            }
+            AssertionFailedError assertion = new AssertionFailedError("Unexpected exception type, expected " 
+                                                                      + expectedType.getSimpleName());
+            assertion.initCause(e);
+            throw assertion;
+        }
+        throw new AssertionFailedError("Expected exception " + expectedType.getSimpleName());
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
index b06199cf903..12d275de6e5 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
@@ -162,32 +162,24 @@ public class StringTests extends ScriptTestCase {
         assertEquals('c', exec("String s = \"c\"; (char)s"));
         assertEquals('c', exec("String s = 'c'; (char)s"));
 
-        try {
+        ClassCastException expected = expectScriptThrows(ClassCastException.class, () -> {
             assertEquals("cc", exec("return (String)(char)\"cc\""));
-            fail();
-        } catch (final ClassCastException cce) {
-            assertTrue(cce.getMessage().contains("Cannot cast [String] with length greater than one to [char]."));
-        }
+        });
+        assertTrue(expected.getMessage().contains("Cannot cast [String] with length greater than one to [char]."));
 
-        try {
+        expected = expectScriptThrows(ClassCastException.class, () -> {
             assertEquals("cc", exec("return (String)(char)'cc'"));
-            fail();
-        } catch (final ClassCastException cce) {
-            assertTrue(cce.getMessage().contains("Cannot cast [String] with length greater than one to [char]."));
-        }
+        });
+        assertTrue(expected.getMessage().contains("Cannot cast [String] with length greater than one to [char]."));
 
-        try {
+        expected = expectScriptThrows(ClassCastException.class, () -> {
             assertEquals('c', exec("String s = \"cc\"; (char)s"));
-            fail();
-        } catch (final ClassCastException cce) {
-            assertTrue(cce.getMessage().contains("Cannot cast [String] with length greater than one to [char]."));
-        }
+        });
+        assertTrue(expected.getMessage().contains("Cannot cast [String] with length greater than one to [char]."));
 
-        try {
+        expected = expectScriptThrows(ClassCastException.class, () -> {
             assertEquals('c', exec("String s = 'cc'; (char)s"));
-            fail();
-        } catch (final ClassCastException cce) {
-            assertTrue(cce.getMessage().contains("Cannot cast [String] with length greater than one to [char]."));
-        }
+        });
+        assertTrue(expected.getMessage().contains("Cannot cast [String] with length greater than one to [char]."));
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/TryCatchTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/TryCatchTests.java
index 07439f7f42b..bf78ee0afa5 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/TryCatchTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/TryCatchTests.java
@@ -26,7 +26,7 @@ public class TryCatchTests extends ScriptTestCase {
 
     /** throws an exception */
     public void testThrow() {
-        RuntimeException exception = expectThrows(RuntimeException.class, () -> {
+        RuntimeException exception = expectScriptThrows(RuntimeException.class, () -> {
             exec("throw new RuntimeException('test')");
         });
         assertEquals("test", exception.getMessage());
@@ -48,7 +48,7 @@ public class TryCatchTests extends ScriptTestCase {
     
     /** tries to catch a different type of exception */
     public void testNoCatch() {
-        RuntimeException exception = expectThrows(RuntimeException.class, () -> {
+        RuntimeException exception = expectScriptThrows(RuntimeException.class, () -> {
            exec("try { if (params.param == 'true') throw new RuntimeException('test'); } " + 
                 "catch (ArithmeticException e) { return 1; } return 2;", 
                 Collections.singletonMap("param", "true"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
index c2b40df8c9d..961c3561c48 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
@@ -25,36 +25,40 @@ import java.util.Collections;
 
 public class WhenThingsGoWrongTests extends ScriptTestCase {
     public void testNullPointer() {
-        expectThrows(NullPointerException.class, () -> {
+        expectScriptThrows(NullPointerException.class, () -> {
             exec("int x = params['missing']; return x;");
         });
     }
 
+    /** test "line numbers" in the bytecode, which are really 1-based offsets */
     public void testLineNumbers() {
         // trigger NPE at line 1 of the script
-        NullPointerException exception = expectThrows(NullPointerException.class, () -> {
+        NullPointerException exception = expectScriptThrows(NullPointerException.class, () -> {
             exec("String x = null; boolean y = x.isEmpty();\n" +
                  "return y;");
         });
-        assertEquals(1, exception.getStackTrace()[0].getLineNumber());
+        // null deref at x.isEmpty(), the '.' is offset 30 (+1)
+        assertEquals(30 + 1, exception.getStackTrace()[0].getLineNumber());
 
         // trigger NPE at line 2 of the script
-        exception = expectThrows(NullPointerException.class, () -> {
+        exception = expectScriptThrows(NullPointerException.class, () -> {
             exec("String x = null;\n" +
                  "return x.isEmpty();");
         });
-        assertEquals(2, exception.getStackTrace()[0].getLineNumber());
+        // null deref at x.isEmpty(), the '.' is offset 25 (+1)
+        assertEquals(25 + 1, exception.getStackTrace()[0].getLineNumber());
 
         // trigger NPE at line 3 of the script
-        exception = expectThrows(NullPointerException.class, () -> {
+        exception = expectScriptThrows(NullPointerException.class, () -> {
             exec("String x = null;\n" +
                  "String y = x;\n" +
                  "return y.isEmpty();");
         });
-        assertEquals(3, exception.getStackTrace()[0].getLineNumber());
+        // null deref at y.isEmpty(), the '.' is offset 39 (+1)
+        assertEquals(39 + 1, exception.getStackTrace()[0].getLineNumber());
 
         // trigger NPE at line 4 in script (inside conditional)
-        exception = expectThrows(NullPointerException.class, () -> {
+        exception = expectScriptThrows(NullPointerException.class, () -> {
             exec("String x = null;\n" +
                  "boolean y = false;\n" +
                  "if (!y) {\n" +
@@ -62,7 +66,8 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
                  "}\n" +
                  "return y;");
         });
-        assertEquals(4, exception.getStackTrace()[0].getLineNumber());
+        // null deref at x.isEmpty(), the '.' is offset 53 (+1)
+        assertEquals(53 + 1, exception.getStackTrace()[0].getLineNumber());
     }
 
     public void testInvalidShift() {
@@ -83,46 +88,46 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
     }
 
     public void testInfiniteLoops() {
-        PainlessError expected = expectThrows(PainlessError.class, () -> {
+        PainlessError expected = expectScriptThrows(PainlessError.class, () -> {
             exec("boolean x = true; while (x) {}");
         });
         assertTrue(expected.getMessage().contains(
                    "The maximum number of statements that can be executed in a loop has been reached."));
 
-        expected = expectThrows(PainlessError.class, () -> {
+        expected = expectScriptThrows(PainlessError.class, () -> {
             exec("while (true) {int y = 5;}");
         });
         assertTrue(expected.getMessage().contains(
                    "The maximum number of statements that can be executed in a loop has been reached."));
 
-        expected = expectThrows(PainlessError.class, () -> {
+        expected = expectScriptThrows(PainlessError.class, () -> {
             exec("while (true) { boolean x = true; while (x) {} }");
         });
         assertTrue(expected.getMessage().contains(
                    "The maximum number of statements that can be executed in a loop has been reached."));
 
-        expected = expectThrows(PainlessError.class, () -> {
+        expected = expectScriptThrows(PainlessError.class, () -> {
             exec("while (true) { boolean x = false; while (x) {} }");
             fail("should have hit PainlessError");
         });
         assertTrue(expected.getMessage().contains(
                    "The maximum number of statements that can be executed in a loop has been reached."));
 
-        expected = expectThrows(PainlessError.class, () -> {
+        expected = expectScriptThrows(PainlessError.class, () -> {
             exec("boolean x = true; for (;x;) {}");
             fail("should have hit PainlessError");
         });
         assertTrue(expected.getMessage().contains(
                    "The maximum number of statements that can be executed in a loop has been reached."));
 
-        expected = expectThrows(PainlessError.class, () -> {
+        expected = expectScriptThrows(PainlessError.class, () -> {
             exec("for (;;) {int x = 5;}");
             fail("should have hit PainlessError");
         });
         assertTrue(expected.getMessage().contains(
                    "The maximum number of statements that can be executed in a loop has been reached."));
 
-        expected = expectThrows(PainlessError.class, () -> {
+        expected = expectScriptThrows(PainlessError.class, () -> {
             exec("def x = true; do {int y = 5;} while (x)");
             fail("should have hit PainlessError");
         });
@@ -140,7 +145,7 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
         // right below limit: ok
         exec("for (int x = 0; x < 9999; ++x) {}");
 
-        PainlessError expected = expectThrows(PainlessError.class, () -> {
+        PainlessError expected = expectScriptThrows(PainlessError.class, () -> {
             exec("for (int x = 0; x < 10000; ++x) {}");
         });
         assertTrue(expected.getMessage().contains(
@@ -163,32 +168,32 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
     }
 
     public void testIllegalDynamicMethod() {
-        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("def x = 'test'; return x.getClass().toString()");
         });
         assertTrue(expected.getMessage().contains("Unable to find dynamic method"));
     }
 
     public void testDynamicNPE() {
-        expectThrows(NullPointerException.class, () -> {
+        expectScriptThrows(NullPointerException.class, () -> {
             exec("def x = null; return x.toString()");
         });
     }
 
     public void testDynamicWrongArgs() {
-        expectThrows(WrongMethodTypeException.class, () -> {
+        expectScriptThrows(WrongMethodTypeException.class, () -> {
             exec("def x = new ArrayList(); return x.get('bogus');");
         });
     }
 
     public void testDynamicArrayWrongIndex() {
-        expectThrows(WrongMethodTypeException.class, () -> {
+        expectScriptThrows(WrongMethodTypeException.class, () -> {
             exec("def x = new long[1]; x[0]=1; return x['bogus'];");
         });
     }
 
     public void testDynamicListWrongIndex() {
-        expectThrows(WrongMethodTypeException.class, () -> {
+        expectScriptThrows(WrongMethodTypeException.class, () -> {
             exec("def x = new ArrayList(); x.add('foo'); return x['bogus'];");
         });
     }
diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml b/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml
index 2c649220b75..c43374e7de2 100644
--- a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml
+++ b/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml
@@ -27,7 +27,7 @@
   - is_true: failures.0.shard
   - match: {failures.0.index:  source}
   - is_true: failures.0.node
-  - match: {failures.0.reason.type:   script_exception}
+  - match: {failures.0.reason.type:   general_script_exception}
   - match: {failures.0.reason.reason: "failed to run inline script [1/0] using lang [groovy]"}
   - match: {failures.0.reason.caused_by.type:   arithmetic_exception}
   - match: {failures.0.reason.caused_by.reason: Division by zero}
diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yaml b/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yaml
index 748e5cd15e0..3aecca73332 100644
--- a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yaml
+++ b/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yaml
@@ -23,7 +23,7 @@
   - is_true: failures.0.shard
   - match: {failures.0.index:  source}
   - is_true: failures.0.node
-  - match: {failures.0.reason.type:   script_exception}
+  - match: {failures.0.reason.type:   general_script_exception}
   - match: {failures.0.reason.reason: "failed to run inline script [1/0] using lang [groovy]"}
   - match: {failures.0.reason.caused_by.type:   arithmetic_exception}
   - match: {failures.0.reason.caused_by.reason: Division by zero}

From a8a38c282a413171285d662d1acf086298dc256d Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Thu, 26 May 2016 08:53:19 -0700
Subject: [PATCH 0479/1311] Remove extra mostly duplicate readme file

It looks like the readme was duplicated when plugins were merged back
into the repo. We removed all these extra files from the plugins, this
removes the remaining duplicate from core.

closes #18597
---
 core/README.textile | 235 --------------------------------------------
 1 file changed, 235 deletions(-)
 delete mode 100644 core/README.textile

diff --git a/core/README.textile b/core/README.textile
deleted file mode 100644
index daaf5ecb70e..00000000000
--- a/core/README.textile
+++ /dev/null
@@ -1,235 +0,0 @@
-h1. Elasticsearch
-
-h2. A Distributed RESTful Search Engine
-
-h3. "https://www.elastic.co/products/elasticsearch":https://www.elastic.co/products/elasticsearch
-
-Elasticsearch is a distributed RESTful search engine built for the cloud. Features include:
-
-* Distributed and Highly Available Search Engine.
-** Each index is fully sharded with a configurable number of shards.
-** Each shard can have one or more replicas.
-** Read / Search operations performed on either one of the replica shard.
-* Multi Tenant with Multi Types.
-** Support for more than one index.
-** Support for more than one type per index.
-** Index level configuration (number of shards, index storage, ...).
-* Various set of APIs
-** HTTP RESTful API
-** Native Java API.
-** All APIs perform automatic node operation rerouting.
-* Document oriented
-** No need for upfront schema definition.
-** Schema can be defined per type for customization of the indexing process.
-* Reliable, Asynchronous Write Behind for long term persistency.
-* (Near) Real Time Search.
-* Built on top of Lucene
-** Each shard is a fully functional Lucene index
-** All the power of Lucene easily exposed through simple configuration / plugins.
-* Per operation consistency
-** Single document level operations are atomic, consistent, isolated and durable.
-* Open Source under the Apache License, version 2 ("ALv2")
-
-h2. Getting Started
-
-First of all, DON'T PANIC. It will take 5 minutes to get the gist of what Elasticsearch is all about.
-
-h3. Requirements
-
-You need to have a recent version of Java installed. See the "Setup":http://www.elastic.co/guide/en/elasticsearch/reference/current/setup.html#jvm-version page for more information.
-
-h3. Installation
-
-* "Download":https://www.elastic.co/downloads/elasticsearch and unzip the Elasticsearch official distribution.
-* Run @bin/elasticsearch@ on unix, or @bin\elasticsearch.bat@ on windows.
-* Run @curl -X GET http://localhost:9200/@.
-* Start more servers ...
-
-h3. Indexing
-
-Let's try and index some twitter like information. First, let's create a twitter user, and add some tweets (the @twitter@ index will be created automatically):
-
-<pre>
-curl -XPUT 'http://localhost:9200/twitter/user/kimchy' -d '{ "name" : "Shay Banon" }'
-
-curl -XPUT 'http://localhost:9200/twitter/tweet/1' -d '
-{
-    "user": "kimchy",
-    "postDate": "2009-11-15T13:12:00",
-    "message": "Trying out Elasticsearch, so far so good?"
-}'
-
-curl -XPUT 'http://localhost:9200/twitter/tweet/2' -d '
-{
-    "user": "kimchy",
-    "postDate": "2009-11-15T14:12:12",
-    "message": "Another tweet, will it be indexed?"
-}'
-</pre>
-
-Now, let's see if the information was added by GETting it:
-
-<pre>
-curl -XGET 'http://localhost:9200/twitter/user/kimchy?pretty=true'
-curl -XGET 'http://localhost:9200/twitter/tweet/1?pretty=true'
-curl -XGET 'http://localhost:9200/twitter/tweet/2?pretty=true'
-</pre>
-
-h3. Searching
-
-Mmm search..., shouldn't it be elastic?
-Let's find all the tweets that @kimchy@ posted:
-
-<pre>
-curl -XGET 'http://localhost:9200/twitter/tweet/_search?q=user:kimchy&pretty=true'
-</pre>
-
-We can also use the JSON query language Elasticsearch provides instead of a query string:
-
-<pre>
-curl -XGET 'http://localhost:9200/twitter/tweet/_search?pretty=true' -d '
-{
-    "query" : {
-        "match" : { "user": "kimchy" }
-    }
-}'
-</pre>
-
-Just for kicks, let's get all the documents stored (we should see the user as well):
-
-<pre>
-curl -XGET 'http://localhost:9200/twitter/_search?pretty=true' -d '
-{
-    "query" : {
-        "matchAll" : {}
-    }
-}'
-</pre>
-
-We can also do range search (the @postDate@ was automatically identified as date)
-
-<pre>
-curl -XGET 'http://localhost:9200/twitter/_search?pretty=true' -d '
-{
-    "query" : {
-        "range" : {
-            "postDate" : { "from" : "2009-11-15T13:00:00", "to" : "2009-11-15T14:00:00" }
-        }
-    }
-}'
-</pre>
-
-There are many more options to perform search, after all, it's a search product no? All the familiar Lucene queries are available through the JSON query language, or through the query parser.
-
-h3. Multi Tenant - Indices and Types
-
-Maan, that twitter index might get big (in this case, index size == valuation). Let's see if we can structure our twitter system a bit differently in order to support such large amounts of data.
-
-Elasticsearch supports multiple indices, as well as multiple types per index. In the previous example we used an index called @twitter@, with two types, @user@ and @tweet@.
-
-Another way to define our simple twitter system is to have a different index per user (note, though that each index has an overhead). Here is the indexing curl's in this case:
-
-<pre>
-curl -XPUT 'http://localhost:9200/kimchy/info/1' -d '{ "name" : "Shay Banon" }'
-
-curl -XPUT 'http://localhost:9200/kimchy/tweet/1' -d '
-{
-    "user": "kimchy",
-    "postDate": "2009-11-15T13:12:00",
-    "message": "Trying out Elasticsearch, so far so good?"
-}'
-
-curl -XPUT 'http://localhost:9200/kimchy/tweet/2' -d '
-{
-    "user": "kimchy",
-    "postDate": "2009-11-15T14:12:12",
-    "message": "Another tweet, will it be indexed?"
-}'
-</pre>
-
-The above will index information into the @kimchy@ index, with two types, @info@ and @tweet@. Each user will get his own special index.
-
-Complete control on the index level is allowed. As an example, in the above case, we would want to change from the default 5 shards with 1 replica per index, to only 1 shard with 1 replica per index (== per twitter user). Here is how this can be done (the configuration can be in yaml as well):
-
-<pre>
-curl -XPUT http://localhost:9200/another_user/ -d '
-{
-    "index" : {
-        "numberOfShards" : 1,
-        "numberOfReplicas" : 1
-    }
-}'
-</pre>
-
-Search (and similar operations) are multi index aware. This means that we can easily search on more than one
-index (twitter user), for example:
-
-<pre>
-curl -XGET 'http://localhost:9200/kimchy,another_user/_search?pretty=true' -d '
-{
-    "query" : {
-        "matchAll" : {}
-    }
-}'
-</pre>
-
-Or on all the indices:
-
-<pre>
-curl -XGET 'http://localhost:9200/_search?pretty=true' -d '
-{
-    "query" : {
-        "matchAll" : {}
-    }
-}'
-</pre>
-
-{One liner teaser}: And the cool part about that? You can easily search on multiple twitter users (indices), with different boost levels per user (index), making social search so much simpler (results from my friends rank higher than results from friends of my friends).
-
-h3. Distributed, Highly Available
-
-Let's face it, things will fail....
-
-Elasticsearch is a highly available and distributed search engine. Each index is broken down into shards, and each shard can have one or more replica. By default, an index is created with 5 shards and 1 replica per shard (5/1). There are many topologies that can be used, including 1/10 (improve search performance), or 20/1 (improve indexing performance, with search executed in a map reduce fashion across shards).
-
-In order to play with the distributed nature of Elasticsearch, simply bring more nodes up and shut down nodes. The system will continue to serve requests (make sure you use the correct http port) with the latest data indexed.
-
-h3. Where to go from here?
-
-We have just covered a very small portion of what Elasticsearch is all about. For more information, please refer to the "elastic.co":http://www.elastic.co/products/elasticsearch website.
-
-h3. Building from Source
-
-Elasticsearch uses "Maven":http://maven.apache.org for its build system.
-
-In order to create a distribution, simply run the @mvn clean package
--DskipTests@ command in the cloned directory.
-
-The distribution will be created under @target/releases@.
-
-See the "TESTING":TESTING.asciidoc file for more information about
-running the Elasticsearch test suite.
-
-h3. Upgrading to Elasticsearch 1.x?
-
-In order to ensure a smooth upgrade process from earlier versions of Elasticsearch (< 1.0.0), it is recommended to perform a full cluster restart. Please see the "setup reference":https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html for more details on the upgrade process.
-
-h1. License
-
-<pre>
-This software is licensed under the Apache License, version 2 ("ALv2"), quoted below.
-
-Copyright 2009-2016 Elasticsearch <https://www.elastic.co>
-
-Licensed under the Apache License, Version 2.0 (the "License"); you may not
-use this file except in compliance with the License. You may obtain a copy of
-the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-License for the specific language governing permissions and limitations under
-the License.
-</pre>

From 4735b5a5fa407d7c30af3cec18d445ff80427f6e Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Thu, 26 May 2016 09:19:38 -0700
Subject: [PATCH 0480/1311] Build: Use license, notice and readme files from
 root for distributions

The distributions had their own copies of these extra files, which was a
carry over from maven. This change removes the duplicate files and
copies them from the root of the project.

closes #18597
---
 distribution/build.gradle                     |  38 ++-
 distribution/src/main/resources/LICENSE.txt   | 202 ---------------
 distribution/src/main/resources/NOTICE.txt    |   5 -
 .../src/main/resources/README.textile         | 235 ------------------
 4 files changed, 25 insertions(+), 455 deletions(-)
 delete mode 100644 distribution/src/main/resources/LICENSE.txt
 delete mode 100644 distribution/src/main/resources/NOTICE.txt
 delete mode 100644 distribution/src/main/resources/README.textile

diff --git a/distribution/build.gradle b/distribution/build.gradle
index 12bb17b807f..b4c183d9bd1 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -150,15 +150,21 @@ subprojects {
       MavenFilteringHack.filter(it, expansions)
     }
 
-    commonFiles = copySpec {
-      // everything except windows files, and config is separate
-      from '../src/main/resources'
-      exclude 'bin/*.bat'
-      exclude 'bin/*.exe'
-      exclude 'config/**'
-      filesMatching('bin/*') { it.setMode(0755) }
+    binFiles = copySpec {
+      // everything except windows files
+      from '../src/main/resources/bin'
+      exclude '*.bat'
+      exclude '*.exe'
+      eachFile { it.setMode(0755) }
       MavenFilteringHack.filter(it, expansions)
     }
+
+    commonFiles = copySpec {
+      from rootProject.projectDir
+      include 'NOTICE.txt'
+      include 'LICENSE.txt'
+      include 'README.textile'
+    }
   }
 
    /*****************************************************************************
@@ -187,14 +193,17 @@ configure(subprojects.findAll { ['zip', 'tar', 'integ-test-zip'].contains(it.nam
       into('config') {
         with configFiles
       }
-      with copySpec {
-        with commonFiles
-        from('../src/main/resources') {
-          include 'bin/*.bat'
-          filter(FixCrLfFilter, eol: FixCrLfFilter.CrLf.newInstance('crlf'))
+      into('bin') {
+        with copySpec {
+          with binFiles
+          from('../src/main/resources') {
+            include 'bin/*.bat'
+            filter(FixCrLfFilter, eol: FixCrLfFilter.CrLf.newInstance('crlf'))
+          }
+          MavenFilteringHack.filter(it, expansions)
         }
-        MavenFilteringHack.filter(it, expansions)
       }
+      with commonFiles
       from('../src/main/resources') {
         include 'bin/*.exe'
       }
@@ -330,6 +339,9 @@ configure(subprojects.findAll { ['deb', 'rpm'].contains(it.name) }) {
     permissionGroup 'root'
     with libFiles
     with modulesFiles
+    into('bin') {
+      with binFiles
+    }
     with copySpec {
       with commonFiles
       if (project.name == 'deb') {
diff --git a/distribution/src/main/resources/LICENSE.txt b/distribution/src/main/resources/LICENSE.txt
deleted file mode 100644
index d6456956733..00000000000
--- a/distribution/src/main/resources/LICENSE.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
diff --git a/distribution/src/main/resources/NOTICE.txt b/distribution/src/main/resources/NOTICE.txt
deleted file mode 100644
index c99b9581931..00000000000
--- a/distribution/src/main/resources/NOTICE.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-Elasticsearch
-Copyright 2009-2016 Elasticsearch
-
-This product includes software developed by The Apache Software
-Foundation (http://www.apache.org/).
diff --git a/distribution/src/main/resources/README.textile b/distribution/src/main/resources/README.textile
deleted file mode 100644
index c19041cec22..00000000000
--- a/distribution/src/main/resources/README.textile
+++ /dev/null
@@ -1,235 +0,0 @@
-h1. Elasticsearch
-
-h2. A Distributed RESTful Search Engine
-
-h3. "https://www.elastic.co/products/elasticsearch":https://www.elastic.co/products/elasticsearch
-
-Elasticsearch is a distributed RESTful search engine built for the cloud. Features include:
-
-* Distributed and Highly Available Search Engine.
-** Each index is fully sharded with a configurable number of shards.
-** Each shard can have one or more replicas.
-** Read / Search operations performed on either one of the replica shard.
-* Multi Tenant with Multi Types.
-** Support for more than one index.
-** Support for more than one type per index.
-** Index level configuration (number of shards, index storage, ...).
-* Various set of APIs
-** HTTP RESTful API
-** Native Java API.
-** All APIs perform automatic node operation rerouting.
-* Document oriented
-** No need for upfront schema definition.
-** Schema can be defined per type for customization of the indexing process.
-* Reliable, Asynchronous Write Behind for long term persistency.
-* (Near) Real Time Search.
-* Built on top of Lucene
-** Each shard is a fully functional Lucene index
-** All the power of Lucene easily exposed through simple configuration / plugins.
-* Per operation consistency
-** Single document level operations are atomic, consistent, isolated and durable.
-* Open Source under the Apache License, version 2 ("ALv2")
-
-h2. Getting Started
-
-First of all, DON'T PANIC. It will take 5 minutes to get the gist of what Elasticsearch is all about.
-
-h3. Requirements
-
-You need to have a recent version of Java installed. See the "Setup":http://www.elastic.co/guide/en/elasticsearch/reference/current/setup.html#jvm-version page for more information.
-
-h3. Installation
-
-* "Download":https://www.elastic.co/downloads/elasticsearch and unzip the Elasticsearch official distribution.
-* Run @bin/elasticsearch@ on unix, or @bin\elasticsearch.bat@ on windows.
-* Run @curl -X GET http://127.0.0.1:9200/@.
-* Start more servers ...
-
-h3. Indexing
-
-Let's try and index some twitter like information. First, let's create a twitter user, and add some tweets (the @twitter@ index will be created automatically):
-
-<pre>
-curl -XPUT 'http://127.0.0.1:9200/twitter/user/kimchy' -d '{ "name" : "Shay Banon" }'
-
-curl -XPUT 'http://127.0.0.1:9200/twitter/tweet/1' -d '
-{
-    "user": "kimchy",
-    "postDate": "2009-11-15T13:12:00",
-    "message": "Trying out Elasticsearch, so far so good?"
-}'
-
-curl -XPUT 'http://127.0.0.1:9200/twitter/tweet/2' -d '
-{
-    "user": "kimchy",
-    "postDate": "2009-11-15T14:12:12",
-    "message": "Another tweet, will it be indexed?"
-}'
-</pre>
-
-Now, let's see if the information was added by GETting it:
-
-<pre>
-curl -XGET 'http://127.0.0.1:9200/twitter/user/kimchy?pretty=true'
-curl -XGET 'http://127.0.0.1:9200/twitter/tweet/1?pretty=true'
-curl -XGET 'http://127.0.0.1:9200/twitter/tweet/2?pretty=true'
-</pre>
-
-h3. Searching
-
-Mmm search..., shouldn't it be elastic?
-Let's find all the tweets that @kimchy@ posted:
-
-<pre>
-curl -XGET 'http://127.0.0.1:9200/twitter/tweet/_search?q=user:kimchy&pretty=true'
-</pre>
-
-We can also use the JSON query language Elasticsearch provides instead of a query string:
-
-<pre>
-curl -XGET 'http://127.0.0.1:9200/twitter/tweet/_search?pretty=true' -d '
-{
-    "query" : {
-        "match" : { "user": "kimchy" }
-    }
-}'
-</pre>
-
-Just for kicks, let's get all the documents stored (we should see the user as well):
-
-<pre>
-curl -XGET 'http://127.0.0.1:9200/twitter/_search?pretty=true' -d '
-{
-    "query" : {
-        "matchAll" : {}
-    }
-}'
-</pre>
-
-We can also do range search (the @postDate@ was automatically identified as date)
-
-<pre>
-curl -XGET 'http://127.0.0.1:9200/twitter/_search?pretty=true' -d '
-{
-    "query" : {
-        "range" : {
-            "postDate" : { "from" : "2009-11-15T13:00:00", "to" : "2009-11-15T14:00:00" }
-        }
-    }
-}'
-</pre>
-
-There are many more options to perform search, after all, it's a search product no? All the familiar Lucene queries are available through the JSON query language, or through the query parser.
-
-h3. Multi Tenant - Indices and Types
-
-Maan, that twitter index might get big (in this case, index size == valuation). Let's see if we can structure our twitter system a bit differently in order to support such large amounts of data.
-
-Elasticsearch supports multiple indices, as well as multiple types per index. In the previous example we used an index called @twitter@, with two types, @user@ and @tweet@.
-
-Another way to define our simple twitter system is to have a different index per user (note, though that each index has an overhead). Here is the indexing curl's in this case:
-
-<pre>
-curl -XPUT 'http://127.0.0.1:9200/kimchy/info/1' -d '{ "name" : "Shay Banon" }'
-
-curl -XPUT 'http://127.0.0.1:9200/kimchy/tweet/1' -d '
-{
-    "user": "kimchy",
-    "postDate": "2009-11-15T13:12:00",
-    "message": "Trying out Elasticsearch, so far so good?"
-}'
-
-curl -XPUT 'http://127.0.0.1:9200/kimchy/tweet/2' -d '
-{
-    "user": "kimchy",
-    "postDate": "2009-11-15T14:12:12",
-    "message": "Another tweet, will it be indexed?"
-}'
-</pre>
-
-The above will index information into the @kimchy@ index, with two types, @info@ and @tweet@. Each user will get his own special index.
-
-Complete control on the index level is allowed. As an example, in the above case, we would want to change from the default 5 shards with 1 replica per index, to only 1 shard with 1 replica per index (== per twitter user). Here is how this can be done (the configuration can be in yaml as well):
-
-<pre>
-curl -XPUT http://127.0.0.1:9200/another_user/ -d '
-{
-    "index" : {
-        "numberOfShards" : 1,
-        "numberOfReplicas" : 1
-    }
-}'
-</pre>
-
-Search (and similar operations) are multi index aware. This means that we can easily search on more than one
-index (twitter user), for example:
-
-<pre>
-curl -XGET 'http://127.0.0.1:9200/kimchy,another_user/_search?pretty=true' -d '
-{
-    "query" : {
-        "matchAll" : {}
-    }
-}'
-</pre>
-
-Or on all the indices:
-
-<pre>
-curl -XGET 'http://127.0.0.1:9200/_search?pretty=true' -d '
-{
-    "query" : {
-        "matchAll" : {}
-    }
-}'
-</pre>
-
-{One liner teaser}: And the cool part about that? You can easily search on multiple twitter users (indices), with different boost levels per user (index), making social search so much simpler (results from my friends rank higher than results from friends of my friends).
-
-h3. Distributed, Highly Available
-
-Let's face it, things will fail....
-
-Elasticsearch is a highly available and distributed search engine. Each index is broken down into shards, and each shard can have one or more replica. By default, an index is created with 5 shards and 1 replica per shard (5/1). There are many topologies that can be used, including 1/10 (improve search performance), or 20/1 (improve indexing performance, with search executed in a map reduce fashion across shards).
-
-In order to play with the distributed nature of Elasticsearch, simply bring more nodes up and shut down nodes. The system will continue to serve requests (make sure you use the correct http port) with the latest data indexed.
-
-h3. Where to go from here?
-
-We have just covered a very small portion of what Elasticsearch is all about. For more information, please refer to the "elastic.co":http://www.elastic.co/products/elasticsearch website.
-
-h3. Building from Source
-
-Elasticsearch uses "Maven":http://maven.apache.org for its build system.
-
-In order to create a distribution, simply run the @mvn clean package
--DskipTests@ command in the cloned directory.
-
-The distribution will be created under @target/releases@.
-
-See the "TESTING":TESTING.asciidoc file for more information about
-running the Elasticsearch test suite.
-
-h3. Upgrading to Elasticsearch 1.x?
-
-In order to ensure a smooth upgrade process from earlier versions of Elasticsearch (< 1.0.0), it is recommended to perform a full cluster restart. Please see the "setup reference":https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html for more details on the upgrade process.
-
-h1. License
-
-<pre>
-This software is licensed under the Apache License, version 2 ("ALv2"), quoted below.
-
-Copyright 2009-2016 Elasticsearch <https://www.elastic.co>
-
-Licensed under the Apache License, Version 2.0 (the "License"); you may not
-use this file except in compliance with the License. You may obtain a copy of
-the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-License for the specific language governing permissions and limitations under
-the License.
-</pre>

From f98ca5310cc1e8f0b2f1fd478f90f6599abe15c4 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Thu, 26 May 2016 19:00:26 +0200
Subject: [PATCH 0481/1311] Fix ReplicaShardAllocatorTests when unassigned
 reason is ALLOCATION_FAILED

When mocking unassigned shards which have failed with reason ALLOCATION_FAILED we
have to ensure that the failed allocation counter is strictly positive.
---
 .../org/elasticsearch/gateway/ReplicaShardAllocatorTests.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
index 82004448658..38dc2f9e632 100644
--- a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
@@ -291,12 +291,14 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
         // mark shard as delayed if reason is NODE_LEFT
         boolean delayed = reason == UnassignedInfo.Reason.NODE_LEFT &&
             UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.get(settings).nanos() > 0;
+        int failedAllocations = reason == UnassignedInfo.Reason.ALLOCATION_FAILED ? 1 : 0;
         RoutingTable routingTable = RoutingTable.builder()
                 .add(IndexRoutingTable.builder(shardId.getIndex())
                                 .addIndexShard(new IndexShardRoutingTable.Builder(shardId)
                                         .addShard(primaryShard)
                                         .addShard(ShardRouting.newUnassigned(shardId, null, false,
-                                            new UnassignedInfo(reason, null, null, 0, System.nanoTime(), System.currentTimeMillis(), delayed)))
+                                            new UnassignedInfo(reason, null, null, failedAllocations, System.nanoTime(),
+                                                System.currentTimeMillis(), delayed)))
                                         .build())
                 )
                 .build();

From f16f65741e426b0e58bdf67673aad474cdd5bc7e Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 26 May 2016 14:10:32 -0400
Subject: [PATCH 0482/1311] Fix when plugins directory is symlink

This commit fixes an issue with the plugins directory being a symbolic
link. Namely, the install plugins command attempts to always create the
plugins directory just in case it does not exist. The JDK method used
here guarantees that the directory is created, and an exception is not
thrown if the directory could not be created because it already
exists. The problem is that this JDK method does not respect symlinks so
its internal existence checks fails, it proceeds to attempt to create
the directory, but the directory creation fails because the symlink
exists. This is documented as being not an issue. We work around this by
checking if there is a symlink where we expect the plugins directory to
be, and only attempt to create if not. We add a unit test that plugin
installation to a symlinked plugins directory works as expected.
---
 .../plugins/InstallPluginCommand.java         |  4 +++-
 .../scripts/module_and_plugin_test_cases.bash | 22 +++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
index 95fc2373766..9e17eec2e93 100644
--- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
@@ -304,7 +304,9 @@ class InstallPluginCommand extends SettingCommand {
 
                 // be on the safe side: do not rely on that directories are always extracted
                 // before their children (although this makes sense, but is it guaranteed?)
-                Files.createDirectories(targetFile.getParent());
+                if (!Files.isSymbolicLink(targetFile.getParent())) {
+                    Files.createDirectories(targetFile.getParent());
+                }
                 if (entry.isDirectory() == false) {
                     try (OutputStream out = Files.newOutputStream(targetFile)) {
                         int len;
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
index 19d9f69ece0..32118fda389 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
@@ -113,6 +113,28 @@ fi
     fi
 }
 
+@test "[$GROUP] install jvm-example plugin with a symlinked plugins path" {
+    # Clean up after the last time this test was run
+    rm -rf /tmp/plugins.*
+    rm -rf /tmp/old_plugins.*
+
+    rm -rf "$ESPLUGINS"
+    local es_plugins=$(mktemp -d -t 'plugins.XXXX')
+    chown -R elasticsearch:elasticsearch "$es_plugins"
+    ln -s "$es_plugins" "$ESPLUGINS"
+
+    install_jvm_example
+    start_elasticsearch_service
+    # check that symlinked plugin was actually picked up
+    curl -s localhost:9200/_cat/configured_example | sed 's/ *$//' > /tmp/installed
+    echo "foo" > /tmp/expected
+    diff /tmp/installed /tmp/expected
+    stop_elasticsearch_service
+    remove_jvm_example
+
+    unlink "$ESPLUGINS"
+}
+
 @test "[$GROUP] install jvm-example plugin with a custom CONFIG_DIR" {
     # Clean up after the last time we ran this test
     rm -rf /tmp/config.*

From 76ca4af5612f81b7914ac13c00f54d9b1c1782c4 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Thu, 26 May 2016 15:03:13 -0400
Subject: [PATCH 0483/1311] move instanceof to catch block

---
 .../org/elasticsearch/script/ScriptService.java    | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java
index f5ee456c119..646ad5e25d6 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java
@@ -305,11 +305,10 @@ public class ScriptService extends AbstractComponent implements Closeable {
                 // for the inline case, then its anonymous: null.
                 String actualName = (type == ScriptType.INLINE) ? null : name;
                 compiledScript = new CompiledScript(type, name, lang, scriptEngineService.compile(actualName, code, params));
-            } catch (Exception exception) {
+            } catch (ScriptException good) {
                 // TODO: remove this try-catch completely, when all script engines have good exceptions!
-                if (exception instanceof ScriptException) {
-                    throw exception; // its already good!
-                }
+                throw good; // its already good
+            } catch (Exception exception) {
                 throw new GeneralScriptException("Failed to compile " + type + " script [" + name + "] using lang [" + lang + "]", exception);
             }
 
@@ -367,11 +366,10 @@ public class ScriptService extends AbstractComponent implements Closeable {
                                 "skipping compile of script [{}], lang [{}] as all scripted operations are disabled for indexed scripts",
                                 template.getScript(), scriptLang);
                     }
-                } catch (Exception e) {
+                } catch (ScriptException good) {
                     // TODO: remove this when all script engines have good exceptions!
-                    if (e instanceof ScriptException) {
-                        throw e; // its already good!
-                    }
+                    throw good; // its already good!
+                } catch (Exception e) {
                     throw new IllegalArgumentException("Unable to parse [" + template.getScript() +
                             "] lang [" + scriptLang + "]", e);
                 }

From 7c3715e009c89dbdfa8585b43f8b4919c2f76794 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 26 May 2016 15:02:27 -0400
Subject: [PATCH 0484/1311] Remove a leftover "es." prefix from docs

This commit removes a leftover usage of the "es." in the CLI syntax in
the zip/targz docs.
---
 docs/reference/setup/install/zip-targz.asciidoc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/reference/setup/install/zip-targz.asciidoc b/docs/reference/setup/install/zip-targz.asciidoc
index 680d698976d..16a79d000e7 100644
--- a/docs/reference/setup/install/zip-targz.asciidoc
+++ b/docs/reference/setup/install/zip-targz.asciidoc
@@ -88,8 +88,7 @@ file by default.  The format of this config file is explained in
 <<settings>>.
 
 Any settings that can be specified in the config file can also be specified on
-the command line, using the `-E` syntax, and prepending `es.` to the setting
-name, as follows:
+the command line, using the `-E` syntax as follows:
 
 [source,sh]
 --------------------------------------------

From cde83e363aac2195c2dc0f81bbe348bcfe8b4152 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Thu, 26 May 2016 13:16:22 -0700
Subject: [PATCH 0485/1311] Add an update script equivalent test.

---
 .../elasticsearch/painless/BasicAPITests.java | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
index 043b614cadd..46609a76173 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
@@ -21,6 +21,8 @@ package org.elasticsearch.painless;
 
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
 
 public class BasicAPITests extends ScriptTestCase {
 
@@ -55,6 +57,21 @@ public class BasicAPITests extends ScriptTestCase {
         assertEquals(5, exec("def x = new HashMap(); x['abc'] = 5; return x['abc'];"));
     }
 
+    /** Test loads and stores with update script equivalent */
+    public void testUpdateMapLoadStore() {
+        Map<String, Object> load = new HashMap<>();
+        Map<String, Object> _source = new HashMap<>();
+        Map<String, Object> ctx = new HashMap<>();
+        Map<String, Object> params = new HashMap<>();
+
+        load.put("load5", "testvalue");
+        _source.put("load", load);
+        ctx.put("_source", _source);
+        params.put("ctx", ctx);
+
+        assertEquals("testvalue", exec("ctx._source['load'].5 = ctx._source['load'].remove('load5')", params));
+    }
+
     /** Test loads and stores with a list */
     public void testListLoadStore() {
         assertEquals(5, exec("def x = new ArrayList(); x.add(3); x.0 = 5; return x.0;"));
@@ -93,7 +110,7 @@ public class BasicAPITests extends ScriptTestCase {
         assertBytecodeExists("def x = 1F", "INVOKESTATIC java/lang/Float.valueOf (F)Ljava/lang/Float;");
         assertBytecodeExists("def x = 1D", "INVOKESTATIC java/lang/Double.valueOf (D)Ljava/lang/Double;");
     }
-    
+
     void testStream() {
         assertEquals(11, exec("params.list.stream().sum()", Collections.singletonMap("list", Arrays.asList(1,2,3,5))));
     }

From fb763c1e8e31176398cbd07942f9adf9edc3f937 Mon Sep 17 00:00:00 2001
From: Igor Motov <igor@motovs.org>
Date: Thu, 19 May 2016 09:14:34 -0400
Subject: [PATCH 0486/1311] Add ability to store results for long running tasks

The results of the tasks are stored in a special index .results
---
 .../elasticsearch/action/ActionRequest.java   |   4 +
 .../action/support/TransportAction.java       |  39 ++++-
 .../elasticsearch/cluster/ClusterModule.java  |   2 +
 .../java/org/elasticsearch/node/Node.java     |   2 +
 .../java/org/elasticsearch/tasks/Task.java    |  15 ++
 .../org/elasticsearch/tasks/TaskManager.java  |  81 ++++++++-
 .../org/elasticsearch/tasks/TaskResult.java   |  90 ++++++++++
 .../tasks/TaskResultsService.java             | 158 ++++++++++++++++++
 .../tasks/task-results-index-mapping.json     |  50 ++++++
 .../admin/cluster/node/tasks/TasksIT.java     |  87 ++++++++++
 .../cluster/node/tasks/TestTaskPlugin.java    |  96 +++++++++--
 11 files changed, 608 insertions(+), 16 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/tasks/TaskResult.java
 create mode 100644 core/src/main/java/org/elasticsearch/tasks/TaskResultsService.java
 create mode 100644 core/src/main/resources/org/elasticsearch/tasks/task-results-index-mapping.json

diff --git a/core/src/main/java/org/elasticsearch/action/ActionRequest.java b/core/src/main/java/org/elasticsearch/action/ActionRequest.java
index 7955855bc0d..dac3b4f2a21 100644
--- a/core/src/main/java/org/elasticsearch/action/ActionRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/ActionRequest.java
@@ -39,6 +39,10 @@ public abstract class ActionRequest<Request extends ActionRequest<Request>> exte
 
     public abstract ActionRequestValidationException validate();
 
+    public boolean getShouldPersistResult() {
+        return false;
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
diff --git a/core/src/main/java/org/elasticsearch/action/support/TransportAction.java b/core/src/main/java/org/elasticsearch/action/support/TransportAction.java
index 79dbf85db65..75542784997 100644
--- a/core/src/main/java/org/elasticsearch/action/support/TransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/TransportAction.java
@@ -133,6 +133,10 @@ public abstract class TransportAction<Request extends ActionRequest<Request>, Re
             return;
         }
 
+        if (task != null && request.getShouldPersistResult()) {
+            listener = new PersistentActionListener<>(taskManager, task, listener);
+        }
+
         if (filters.length == 0) {
             try {
                 doExecute(task, request, listener);
@@ -171,7 +175,7 @@ public abstract class TransportAction<Request extends ActionRequest<Request>, Re
                 if (i < this.action.filters.length) {
                     this.action.filters[i].apply(task, actionName, request, listener, this);
                 } else if (i == this.action.filters.length) {
-                    this.action.doExecute(task, request, new FilteredActionListener<Response>(actionName, listener,
+                    this.action.doExecute(task, request, new FilteredActionListener<>(actionName, listener,
                             new ResponseFilterChain<>(this.action.filters, logger)));
                 } else {
                     listener.onFailure(new IllegalStateException("proceed was called too many times"));
@@ -246,4 +250,37 @@ public abstract class TransportAction<Request extends ActionRequest<Request>, Re
             listener.onFailure(e);
         }
     }
+
+    /**
+     * Wrapper for an action listener that persists the result at the end of the execution
+     */
+    private static class PersistentActionListener<Response extends ActionResponse> implements ActionListener<Response> {
+        private final ActionListener<Response> delegate;
+        private final Task task;
+        private final TaskManager taskManager;
+
+        private  PersistentActionListener(TaskManager taskManager, Task task, ActionListener<Response> delegate) {
+            this.taskManager = taskManager;
+            this.task = task;
+            this.delegate = delegate;
+        }
+
+        @Override
+        public void onResponse(Response response) {
+            try {
+                taskManager.persistResult(task, response, delegate);
+            } catch (Throwable e) {
+                delegate.onFailure(e);
+            }
+        }
+
+        @Override
+        public void onFailure(Throwable e) {
+            try {
+                taskManager.persistResult(task, e, delegate);
+            } catch (Throwable e1) {
+                delegate.onFailure(e1);
+            }
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index 707521b2cfe..15ed58346e7 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -64,6 +64,7 @@ import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.ExtensionPoint;
 import org.elasticsearch.gateway.GatewayAllocator;
+import org.elasticsearch.tasks.TaskResultsService;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -157,5 +158,6 @@ public class ClusterModule extends AbstractModule {
         bind(NodeIndexDeletedAction.class).asEagerSingleton();
         bind(NodeMappingRefreshAction.class).asEagerSingleton();
         bind(MappingUpdatedAction.class).asEagerSingleton();
+        bind(TaskResultsService.class).asEagerSingleton();
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index ec222210b4e..2b7651de7b6 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -97,6 +97,7 @@ import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.snapshots.SnapshotShardsService;
 import org.elasticsearch.snapshots.SnapshotsService;
+import org.elasticsearch.tasks.TaskResultsService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPoolModule;
 import org.elasticsearch.transport.TransportService;
@@ -332,6 +333,7 @@ public class Node implements Closeable {
 
         // Start the transport service now so the publish address will be added to the local disco node in ClusterService
         TransportService transportService = injector.getInstance(TransportService.class);
+        transportService.getTaskManager().setTaskResultsService(injector.getInstance(TaskResultsService.class));
         transportService.start();
 
         validateNodeBeforeAcceptingRequests(settings, transportService.boundAddress());
diff --git a/core/src/main/java/org/elasticsearch/tasks/Task.java b/core/src/main/java/org/elasticsearch/tasks/Task.java
index ab07874c9c2..9c3f4cc842d 100644
--- a/core/src/main/java/org/elasticsearch/tasks/Task.java
+++ b/core/src/main/java/org/elasticsearch/tasks/Task.java
@@ -20,11 +20,14 @@
 
 package org.elasticsearch.tasks;
 
+import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.xcontent.ToXContent;
 
+import java.io.IOException;
+
 /**
  * Current task information
  */
@@ -132,4 +135,16 @@ public class Task {
     }
 
     public interface Status extends ToXContent, NamedWriteable {}
+
+    public TaskResult result(DiscoveryNode node, Throwable error) throws IOException {
+        return new TaskResult(taskInfo(node, true), error);
+    }
+
+    public TaskResult result(DiscoveryNode node, ActionResponse response) throws IOException {
+        if (response instanceof ToXContent) {
+            return new TaskResult(taskInfo(node, true), (ToXContent) response);
+        } else {
+            throw new IllegalStateException("response has to implement ToXContent for persistence");
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskManager.java b/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
index e7b0cb0068e..c44eb65301a 100644
--- a/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
+++ b/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
@@ -19,8 +19,12 @@
 
 package org.elasticsearch.tasks;
 
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterStateListener;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
@@ -28,6 +32,7 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.ConcurrentMapLong;
 import org.elasticsearch.transport.TransportRequest;
 
+import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -52,11 +57,18 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen
 
     private final Map<TaskId, String> banedParents = new ConcurrentHashMap<>();
 
+    private TaskResultsService taskResultsService;
+
+    private DiscoveryNodes lastDiscoveryNodes = DiscoveryNodes.EMPTY_NODES;
+
     public TaskManager(Settings settings) {
         super(settings);
     }
 
-    private DiscoveryNodes lastDiscoveryNodes = DiscoveryNodes.EMPTY_NODES;
+    public void setTaskResultsService(TaskResultsService taskResultsService) {
+        assert this.taskResultsService == null;
+        this.taskResultsService = taskResultsService;
+    }
 
     /**
      * Registers a task without parent task
@@ -130,6 +142,72 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen
         }
     }
 
+    /**
+     * Stores the task failure
+     */
+    public <Response extends  ActionResponse> void persistResult(Task task, Throwable error, ActionListener<Response> listener) {
+        DiscoveryNode localNode = lastDiscoveryNodes.getLocalNode();
+        if (localNode == null) {
+            // too early to persist anything, shouldn't really be here - just pass the error along
+            listener.onFailure(error);
+            return;
+        }
+        final TaskResult taskResult;
+        try {
+            taskResult = task.result(localNode, error);
+        } catch (IOException ex) {
+            logger.warn("couldn't persist error {}", ex, ExceptionsHelper.detailedMessage(error));
+            listener.onFailure(ex);
+            return;
+        }
+        taskResultsService.persist(taskResult, new ActionListener<Void>() {
+            @Override
+            public void onResponse(Void aVoid) {
+                listener.onFailure(error);
+            }
+
+            @Override
+            public void onFailure(Throwable e) {
+                logger.warn("couldn't persist error {}", e, ExceptionsHelper.detailedMessage(error));
+                listener.onFailure(e);
+            }
+        });
+    }
+
+    /**
+     * Stores the task result
+     */
+    public <Response extends  ActionResponse> void persistResult(Task task, Response response, ActionListener<Response> listener) {
+        DiscoveryNode localNode = lastDiscoveryNodes.getLocalNode();
+        if (localNode == null) {
+            // too early to persist anything, shouldn't really be here - just pass the response along
+            logger.warn("couldn't persist response {}, the node didn't join the cluster yet", response);
+            listener.onResponse(response);
+            return;
+        }
+        final TaskResult taskResult;
+        try {
+            taskResult = task.result(localNode, response);
+        } catch (IOException ex) {
+            logger.warn("couldn't persist response {}", ex, response);
+            listener.onFailure(ex);
+            return;
+        }
+
+        taskResultsService.persist(taskResult, new ActionListener<Void>() {
+            @Override
+            public void onResponse(Void aVoid) {
+                listener.onResponse(response);
+            }
+
+            @Override
+            public void onFailure(Throwable e) {
+                logger.warn("couldn't persist response {}", e, response);
+                listener.onFailure(e);
+            }
+        });
+    }
+
     /**
      * Returns the list of currently running tasks on the node
      */
@@ -223,6 +301,7 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen
 
     @Override
     public void clusterChanged(ClusterChangedEvent event) {
+        lastDiscoveryNodes = event.state().getNodes();
         if (event.nodesRemoved()) {
             synchronized (banedParents) {
                 lastDiscoveryNodes = event.state().getNodes();
diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskResult.java b/core/src/main/java/org/elasticsearch/tasks/TaskResult.java
new file mode 100644
index 00000000000..a33a7842c27
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/tasks/TaskResult.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.tasks;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo;
+import org.elasticsearch.client.Requests;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+
+import java.io.IOException;
+
+/**
+ * Represents the result or failure of a running task
+ */
+public class TaskResult {
+
+    private final BytesReference result;
+
+    private final TaskId taskId;
+
+    public TaskResult(TaskInfo taskInfo, Throwable e) throws IOException {
+        ToXContent.Params params = ToXContent.EMPTY_PARAMS;
+        XContentBuilder builder = XContentFactory.contentBuilder(Requests.INDEX_CONTENT_TYPE);
+        builder.startObject();
+        {
+            builder.startObject("task");
+            {
+                taskInfo.toXContent(builder, params);
+            }
+            builder.endObject();
+            builder.startObject("error");
+            {
+                ElasticsearchException.toXContent(builder, params, e);
+            }
+            builder.endObject();
+        }
+        builder.endObject();
+        result = builder.bytes();
+        taskId = taskInfo.getTaskId();
+    }
+
+    public TaskResult(TaskInfo taskInfo, ToXContent toXContent) throws IOException {
+        ToXContent.Params params = ToXContent.EMPTY_PARAMS;
+        XContentBuilder builder = XContentFactory.contentBuilder(Requests.INDEX_CONTENT_TYPE);
+        builder.startObject();
+        {
+            builder.startObject("task");
+            {
+                taskInfo.toXContent(builder, params);
+            }
+            builder.endObject();
+            builder.startObject("result");
+            {
+                toXContent.toXContent(builder, params);
+            }
+            builder.endObject();
+        }
+        builder.endObject();
+        result = builder.bytes();
+        taskId = taskInfo.getTaskId();
+    }
+
+    public TaskId getTaskId() {
+        return taskId;
+    }
+
+    public BytesReference getResult() {
+        return result;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskResultsService.java b/core/src/main/java/org/elasticsearch/tasks/TaskResultsService.java
new file mode 100644
index 00000000000..c0f63868d3d
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/tasks/TaskResultsService.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.tasks;
+
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.indices.IndexAlreadyExistsException;
+
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+
+/**
+ * Service that can persist task results
+ */
+public class TaskResultsService extends AbstractComponent {
+
+    public static final String TASK_RESULT_INDEX = ".results";
+
+    public static final String TASK_RESULT_TYPE = "result";
+
+    public static final String TASK_RESULT_INDEX_MAPPING_FILE = "task-results-index-mapping.json";
+
+    private final Client client;
+
+    private final ClusterService clusterService;
+
+    private final TransportCreateIndexAction createIndexAction;
+
+    @Inject
+    public TaskResultsService(Settings settings, Client client, ClusterService clusterService,
+                              TransportCreateIndexAction createIndexAction) {
+        super(settings);
+        this.client = client;
+        this.clusterService = clusterService;
+        this.createIndexAction = createIndexAction;
+    }
+
+    public void persist(TaskResult taskResult, ActionListener<Void> listener) {
+
+        ClusterState state = clusterService.state();
+
+        if (state.routingTable().hasIndex(TASK_RESULT_INDEX) == false) {
+            CreateIndexRequest createIndexRequest = new CreateIndexRequest();
+            createIndexRequest.settings(taskResultIndexSettings());
+            createIndexRequest.index(TASK_RESULT_INDEX);
+            createIndexRequest.mapping(TASK_RESULT_TYPE, taskResultIndexMapping());
+            createIndexRequest.cause("auto(task api)");
+
+            createIndexAction.execute(null, createIndexRequest, new ActionListener<CreateIndexResponse>() {
+                @Override
+                public void onResponse(CreateIndexResponse result) {
+                    doPersist(taskResult, listener);
+                }
+
+                @Override
+                public void onFailure(Throwable e) {
+                    if (ExceptionsHelper.unwrapCause(e) instanceof IndexAlreadyExistsException) {
+                        // we have the index, do it
+                        try {
+                            doPersist(taskResult, listener);
+                        } catch (Throwable e1) {
+                            listener.onFailure(e1);
+                        }
+                    } else {
+                        listener.onFailure(e);
+                    }
+                }
+            });
+        } else {
+            IndexMetaData metaData = state.getMetaData().index(TASK_RESULT_INDEX);
+            if (metaData.getMappings().containsKey(TASK_RESULT_TYPE) == false) {
+                // The index already exists but doesn't have our mapping
+                client.admin().indices().preparePutMapping(TASK_RESULT_INDEX).setType(TASK_RESULT_TYPE).setSource(taskResultIndexMapping())
+                    .execute(new ActionListener<PutMappingResponse>() {
+                                 @Override
+                                 public void onResponse(PutMappingResponse putMappingResponse) {
+                                     doPersist(taskResult, listener);
+                                 }
+
+                                 @Override
+                                 public void onFailure(Throwable e) {
+                                     listener.onFailure(e);
+                                 }
+                             }
+                    );
+            } else {
+                doPersist(taskResult, listener);
+            }
+        }
+    }
+
+
+    private void doPersist(TaskResult taskResult, ActionListener<Void> listener) {
+        client.prepareIndex(TASK_RESULT_INDEX, TASK_RESULT_TYPE, taskResult.getTaskId().toString()).setSource(taskResult.getResult())
+            .execute(new ActionListener<IndexResponse>() {
+                @Override
+                public void onResponse(IndexResponse indexResponse) {
+                    listener.onResponse(null);
+                }
+
+                @Override
+                public void onFailure(Throwable e) {
+                    listener.onFailure(e);
+                }
+            });
+
+    }
+
+    private Settings taskResultIndexSettings() {
+        return Settings.builder()
+            .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
+            .put(IndexMetaData.INDEX_AUTO_EXPAND_REPLICAS_SETTING.getKey(), "0-1")
+            .put(IndexMetaData.SETTING_PRIORITY, Integer.MAX_VALUE)
+            .build();
+    }
+
+    public String taskResultIndexMapping() {
+        try (InputStream is = getClass().getResourceAsStream(TASK_RESULT_INDEX_MAPPING_FILE)) {
+            ByteArrayOutputStream out = new ByteArrayOutputStream();
+            Streams.copy(is, out);
+            return out.toString(IOUtils.UTF_8);
+        } catch (Exception e) {
+            logger.error("failed to create tasks results index template [{}]", e, TASK_RESULT_INDEX_MAPPING_FILE);
+            throw new IllegalStateException("failed to create tasks results index template [" + TASK_RESULT_INDEX_MAPPING_FILE + "]", e);
+        }
+
+    }
+}
diff --git a/core/src/main/resources/org/elasticsearch/tasks/task-results-index-mapping.json b/core/src/main/resources/org/elasticsearch/tasks/task-results-index-mapping.json
new file mode 100644
index 00000000000..0310418bb43
--- /dev/null
+++ b/core/src/main/resources/org/elasticsearch/tasks/task-results-index-mapping.json
@@ -0,0 +1,50 @@
+{
+  "result" : {
+    "dynamic" : "strict",
+    "properties" : {
+      "task" : {
+        "properties": {
+          "action": {
+            "type": "keyword"
+          },
+          "cancellable": {
+            "type": "boolean"
+          },
+          "id": {
+            "type": "long"
+          },
+          "parent_id": {
+            "type": "keyword"
+          },
+          "node": {
+            "type": "keyword"
+          },
+          "running_time_in_nanos": {
+            "type": "long"
+          },
+          "start_time_in_millis": {
+            "type": "long"
+          },
+          "type": {
+            "type": "keyword"
+          },
+          "status": {
+            "type" : "object",
+            "enabled" : false
+          },
+          "description": {
+            "type": "string"
+          }
+        }
+      },
+      "result" : {
+        "type" : "object",
+        "enabled" : false
+      },
+      "error" : {
+        "type" : "object",
+        "enabled" : false
+      }
+    }
+  }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
index cfa937f7dc1..5b9427a4b75 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
@@ -32,14 +32,19 @@ import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeAction;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryAction;
 import org.elasticsearch.action.bulk.BulkAction;
 import org.elasticsearch.action.fieldstats.FieldStatsAction;
+import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.index.IndexAction;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskResultsService;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.tasks.MockTaskManager;
 import org.elasticsearch.test.tasks.MockTaskManagerListener;
@@ -62,6 +67,9 @@ import java.util.function.Function;
 
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.emptyCollectionOf;
@@ -476,6 +484,85 @@ public class TasksIT extends ESIntegTestCase {
         assertThat(response.getTasks().size(), greaterThanOrEqualTo(1));
     }
 
+    public void testTaskResultPersistence() throws Exception {
+        // Randomly create an empty index to make sure the type is created automatically
+        if (randomBoolean()) {
+            logger.info("creating an empty results index with custom settings");
+            assertAcked(client().admin().indices().prepareCreate(TaskResultsService.TASK_RESULT_INDEX));
+        }
+
+        registerTaskManageListeners(TestTaskPlugin.TestTaskAction.NAME);  // we need this to get task id of the process
+
+        // Start non-blocking test task
+        TestTaskPlugin.TestTaskAction.INSTANCE.newRequestBuilder(client()).setShouldPersistResult(true).setShouldBlock(false).get();
+
+        List<TaskInfo> events = findEvents(TestTaskPlugin.TestTaskAction.NAME, Tuple::v1);
+
+        assertEquals(1, events.size());
+        TaskInfo taskInfo = events.get(0);
+        String taskId = taskInfo.getTaskId().toString();
+
+        GetResponse resultDoc = client().prepareGet(TaskResultsService.TASK_RESULT_INDEX, TaskResultsService.TASK_RESULT_TYPE, taskId)
+            .get();
+        assertTrue(resultDoc.isExists());
+
+        Map<String, Object> source = resultDoc.getSource();
+        Map<String, Object> task = (Map<String, Object>) source.get("task");
+        assertEquals(taskInfo.getNode().getId(), task.get("node"));
+        assertEquals(taskInfo.getAction(), task.get("action"));
+        assertEquals(Long.toString(taskInfo.getId()), task.get("id").toString());
+
+        Map<String, Object> result = (Map<String, Object>) source.get("result");
+        assertEquals("0", result.get("failure_count").toString());
+
+        assertNoFailures(client().admin().indices().prepareRefresh(TaskResultsService.TASK_RESULT_INDEX).get());
+
+        SearchResponse searchResponse = client().prepareSearch(TaskResultsService.TASK_RESULT_INDEX)
+            .setTypes(TaskResultsService.TASK_RESULT_TYPE)
+            .setSource(SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("task.action", taskInfo.getAction())))
+            .get();
+
+        assertEquals(1L, searchResponse.getHits().totalHits());
+
+        searchResponse = client().prepareSearch(TaskResultsService.TASK_RESULT_INDEX).setTypes(TaskResultsService.TASK_RESULT_TYPE)
+            .setSource(SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("task.node", taskInfo.getNode().getId()))).get();
+
+        assertEquals(1L, searchResponse.getHits().totalHits());
+    }
+
+    public void testTaskFailurePersistence() throws Exception {
+        registerTaskManageListeners(TestTaskPlugin.TestTaskAction.NAME);  // we need this to get task id of the process
+
+        // Start non-blocking test task that should fail
+        assertThrows(
+            TestTaskPlugin.TestTaskAction.INSTANCE.newRequestBuilder(client())
+                .setShouldFail(true)
+                .setShouldPersistResult(true)
+                .setShouldBlock(false),
+            IllegalStateException.class
+        );
+
+        List<TaskInfo> events = findEvents(TestTaskPlugin.TestTaskAction.NAME, Tuple::v1);
+        assertEquals(1, events.size());
+        TaskInfo failedTaskInfo = events.get(0);
+        String failedTaskId = failedTaskInfo.getTaskId().toString();
+
+        GetResponse failedResultDoc = client()
+            .prepareGet(TaskResultsService.TASK_RESULT_INDEX, TaskResultsService.TASK_RESULT_TYPE, failedTaskId)
+            .get();
+        assertTrue(failedResultDoc.isExists());
+
+        Map<String, Object> source = failedResultDoc.getSource();
+        Map<String, Object> task = (Map<String, Object>) source.get("task");
+        assertEquals(failedTaskInfo.getNode().getId(), task.get("node"));
+        assertEquals(failedTaskInfo.getAction(), task.get("action"));
+        assertEquals(Long.toString(failedTaskInfo.getId()), task.get("id").toString());
+
+        Map<String, Object> error = (Map<String, Object>) source.get("error");
+        assertEquals("Simulating operation failure", error.get("reason"));
+        assertEquals("illegal_state_exception", error.get("type"));
+    }
+
     @Override
     public void tearDown() throws Exception {
         for (Map.Entry<Tuple<String, String>, RecordingTaskManagerListener> entry : listeners.entrySet()) {
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
index ec05ff9c984..09f3d49e0a7 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
@@ -29,6 +29,7 @@ import org.elasticsearch.action.support.nodes.BaseNodeRequest;
 import org.elasticsearch.action.support.nodes.BaseNodeResponse;
 import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
+import org.elasticsearch.action.support.nodes.NodesOperationRequestBuilder;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.action.support.tasks.BaseTasksRequest;
 import org.elasticsearch.action.support.tasks.BaseTasksResponse;
@@ -44,6 +45,8 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
@@ -108,7 +111,7 @@ public class TestTaskPlugin extends Plugin {
         }
     }
 
-    public static class NodesResponse extends BaseNodesResponse<NodeResponse> {
+    public static class NodesResponse extends BaseNodesResponse<NodeResponse> implements ToXContent {
 
         NodesResponse() {
 
@@ -128,23 +131,31 @@ public class TestTaskPlugin extends Plugin {
             out.writeStreamableList(nodes);
         }
 
-        public int failureCount() {
+        public int getFailureCount() {
             return failures().size();
         }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.field("failure_count", getFailureCount());
+            return builder;
+        }
     }
 
     public static class NodeRequest extends BaseNodeRequest {
         protected String requestName;
         protected String nodeId;
+        protected boolean shouldBlock;
 
         public NodeRequest() {
             super();
         }
 
-        public NodeRequest(NodesRequest request, String nodeId) {
+        public NodeRequest(NodesRequest request, String nodeId, boolean shouldBlock) {
             super(nodeId);
             requestName = request.requestName;
             this.nodeId = nodeId;
+            this.shouldBlock = shouldBlock;
         }
 
         @Override
@@ -152,6 +163,7 @@ public class TestTaskPlugin extends Plugin {
             super.readFrom(in);
             requestName = in.readString();
             nodeId = in.readString();
+            shouldBlock = in.readBoolean();
         }
 
         @Override
@@ -159,6 +171,7 @@ public class TestTaskPlugin extends Plugin {
             super.writeTo(out);
             out.writeString(requestName);
             out.writeString(nodeId);
+            out.writeBoolean(shouldBlock);
         }
 
         @Override
@@ -174,6 +187,9 @@ public class TestTaskPlugin extends Plugin {
 
     public static class NodesRequest extends BaseNodesRequest<NodesRequest> {
         private String requestName;
+        private boolean shouldPersistResult = false;
+        private boolean shouldBlock = true;
+        private boolean shouldFail = false;
 
         NodesRequest() {
             super();
@@ -184,16 +200,47 @@ public class TestTaskPlugin extends Plugin {
             this.requestName = requestName;
         }
 
+        public void setShouldPersistResult(boolean shouldPersistResult) {
+            this.shouldPersistResult = shouldPersistResult;
+        }
+
+        @Override
+        public boolean getShouldPersistResult() {
+            return shouldPersistResult;
+        }
+
+        public void setShouldBlock(boolean shouldBlock) {
+            this.shouldBlock = shouldBlock;
+        }
+
+        public boolean getShouldBlock() {
+            return shouldBlock;
+        }
+
+        public void setShouldFail(boolean shouldFail) {
+            this.shouldFail = shouldFail;
+        }
+
+        public boolean getShouldFail() {
+            return shouldFail;
+        }
+
         @Override
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
             requestName = in.readString();
+            shouldPersistResult = in.readBoolean();
+            shouldBlock = in.readBoolean();
+            shouldFail = in.readBoolean();
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             out.writeString(requestName);
+            out.writeBoolean(shouldPersistResult);
+            out.writeBoolean(shouldBlock);
+            out.writeBoolean(shouldFail);
         }
 
         @Override
@@ -219,6 +266,9 @@ public class TestTaskPlugin extends Plugin {
 
         @Override
         protected NodesResponse newResponse(NodesRequest request, List<NodeResponse> responses, List<FailedNodeException> failures) {
+            if (request.getShouldFail()) {
+                throw new IllegalStateException("Simulating operation failure");
+            }
             return new NodesResponse(clusterName, responses, failures);
         }
 
@@ -235,7 +285,7 @@ public class TestTaskPlugin extends Plugin {
 
         @Override
         protected NodeRequest newNodeRequest(String nodeId, NodesRequest request) {
-            return new NodeRequest(request, nodeId);
+            return new NodeRequest(request, nodeId, request.getShouldBlock());
         }
 
         @Override
@@ -251,15 +301,17 @@ public class TestTaskPlugin extends Plugin {
         @Override
         protected NodeResponse nodeOperation(NodeRequest request, Task task) {
             logger.info("Test task started on the node {}", clusterService.localNode());
-            try {
-                awaitBusy(() -> {
-                    if (((CancellableTask) task).isCancelled()) {
-                        throw new RuntimeException("Cancelled!");
-                    }
-                    return ((TestTask) task).isBlocked() == false;
-                });
-            } catch (InterruptedException ex) {
-                Thread.currentThread().interrupt();
+            if (request.shouldBlock) {
+                try {
+                    awaitBusy(() -> {
+                        if (((CancellableTask) task).isCancelled()) {
+                            throw new RuntimeException("Cancelled!");
+                        }
+                        return ((TestTask) task).isBlocked() == false;
+                    });
+                } catch (InterruptedException ex) {
+                    Thread.currentThread().interrupt();
+                }
             }
             logger.info("Test task finished on the node {}", clusterService.localNode());
             return new NodeResponse(clusterService.localNode());
@@ -296,11 +348,27 @@ public class TestTaskPlugin extends Plugin {
         }
     }
 
-    public static class NodesRequestBuilder extends ActionRequestBuilder<NodesRequest, NodesResponse, NodesRequestBuilder> {
+    public static class NodesRequestBuilder extends NodesOperationRequestBuilder<NodesRequest, NodesResponse, NodesRequestBuilder> {
 
         protected NodesRequestBuilder(ElasticsearchClient client, Action<NodesRequest, NodesResponse, NodesRequestBuilder> action) {
             super(client, action, new NodesRequest("test"));
         }
+
+
+        public NodesRequestBuilder setShouldPersistResult(boolean shouldPersistResult) {
+            request().setShouldPersistResult(shouldPersistResult);
+            return this;
+        }
+
+        public NodesRequestBuilder setShouldBlock(boolean shouldBlock) {
+            request().setShouldBlock(shouldBlock);
+            return this;
+        }
+
+        public NodesRequestBuilder setShouldFail(boolean shouldFail) {
+            request().setShouldFail(shouldFail);
+            return this;
+        }
     }
 
 

From 318a4e3ef6e4594619cbe3adcc41494fa23fa569 Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Fri, 27 May 2016 08:44:20 +0200
Subject: [PATCH 0487/1311] Introduce dedicated master nodes in testing
 infrastructure (#18514)

This PR changes the InternalTestCluster to support dedicated master nodes. The creation of dedicated master nodes can be controlled using a new `supportsMasterNodes` parameter to the ClusterScope annotation. If set to true (the default), dedicated master nodes will randomly be used. If set to false,  no master nodes will be created and data nodes will also be allowed to become masters. If active, test runs will either have 1 or 3 masternodes
---
 .../admin/cluster/node/tasks/TasksIT.java     |   8 +-
 .../cluster/node/tasks/TestTaskPlugin.java    |   2 +-
 .../admin/cluster/stats/ClusterStatsIT.java   |  31 ++-
 .../transport/TransportClientRetryIT.java     |   3 +-
 .../cluster/settings/SettingsFilteringIT.java |   2 +-
 .../cluster/shards/ClusterSearchShardsIT.java |   2 +-
 .../netty/NettyHttpRequestSizeLimitIT.java    |   2 +-
 .../http/netty/NettyPipeliningDisabledIT.java |   2 +-
 .../http/netty/NettyPipeliningEnabledIT.java  |   2 +-
 .../index/SettingsListenerIT.java             |  10 +-
 .../index/engine/InternalEngineMergeIT.java   |   2 +-
 .../index/store/CorruptedFileIT.java          | 165 ++++++------
 .../indices/stats/IndexStatsIT.java           |  20 +-
 .../DetailedErrorsDisabledIT.java             |   2 +-
 .../DetailedErrorsEnabledIT.java              |   3 +-
 .../plugins/ResponseHeaderPluginIT.java       |   4 +-
 .../org/elasticsearch/rest/CorsNotSetIT.java  |   2 +-
 .../org/elasticsearch/rest/CorsRegexIT.java   |   2 +-
 .../search/fetch/FetchSubPhasePluginIT.java   |   2 +-
 .../functionscore/ExplainableScriptIT.java    |   2 +-
 .../functionscore/FunctionScorePluginIT.java  |   2 +-
 .../highlight/CustomHighlighterSearchIT.java  |   2 +-
 .../AbstractSnapshotIntegTestCase.java        |   3 +-
 .../SharedClusterSnapshotRestoreIT.java       |   1 +
 .../transport/netty/NettyTransportIT.java     |   2 +-
 .../NettyTransportMultiPortIntegrationIT.java |   2 +-
 .../java/org/elasticsearch/tribe/TribeIT.java |   2 +-
 .../org/elasticsearch/ttl/SimpleTTLIT.java    |   2 +-
 .../testing/testing-framework.asciidoc        |  10 +-
 .../AzureDiscoveryClusterFormationTests.java  |   4 +-
 .../Ec2DiscoveryClusterFormationTests.java    |   2 +-
 .../discovery/gce/GceDiscoverTests.java       |   2 +-
 ...AzureSnapshotRestoreServiceIntegTests.java |   1 +
 .../azure/AzureSnapshotRestoreTests.java      |   2 +-
 .../elasticsearch/test/ESIntegTestCase.java   |  24 +-
 .../test/InternalTestCluster.java             | 240 ++++++++++++------
 .../test/test/InternalTestClusterTests.java   |  18 +-
 37 files changed, 348 insertions(+), 239 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
index 5b9427a4b75..270059b09dc 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
@@ -371,7 +371,7 @@ public class TasksIT extends ESIntegTestCase {
         logger.info("--> started test tasks");
 
         // Wait for the task to start on all nodes
-        assertBusy(() -> assertEquals(internalCluster().numDataAndMasterNodes(),
+        assertBusy(() -> assertEquals(internalCluster().size(),
             client().admin().cluster().prepareListTasks().setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").get().getTasks().size()));
 
         logger.info("--> cancelling the main test task");
@@ -391,7 +391,7 @@ public class TasksIT extends ESIntegTestCase {
         ListenableActionFuture<TestTaskPlugin.NodesResponse> future = TestTaskPlugin.TestTaskAction.INSTANCE.newRequestBuilder(client())
                 .execute();
         // Wait for the task to start on all nodes
-        assertBusy(() -> assertEquals(internalCluster().numDataAndMasterNodes(),
+        assertBusy(() -> assertEquals(internalCluster().size(),
             client().admin().cluster().prepareListTasks().setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").get().getTasks().size()));
 
         TestTaskPlugin.UnblockTestTasksAction.INSTANCE.newRequestBuilder(client()).get();
@@ -409,7 +409,7 @@ public class TasksIT extends ESIntegTestCase {
         ListenableActionFuture<ListTasksResponse> waitResponseFuture;
         try {
             // Wait for the task to start on all nodes
-            assertBusy(() -> assertEquals(internalCluster().numDataAndMasterNodes(), client().admin().cluster().prepareListTasks()
+            assertBusy(() -> assertEquals(internalCluster().size(), client().admin().cluster().prepareListTasks()
                     .setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").get().getTasks().size()));
 
             // Spin up a request to wait for that task to finish
@@ -437,7 +437,7 @@ public class TasksIT extends ESIntegTestCase {
                 .execute();
         try {
             // Wait for the task to start on all nodes
-            assertBusy(() -> assertEquals(internalCluster().numDataAndMasterNodes(), client().admin().cluster().prepareListTasks()
+            assertBusy(() -> assertEquals(internalCluster().size(), client().admin().cluster().prepareListTasks()
                     .setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").get().getTasks().size()));
 
             // Spin up a request that should wait for those tasks to finish
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
index 09f3d49e0a7..9874bdd44b1 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
@@ -276,7 +276,7 @@ public class TestTaskPlugin extends Plugin {
         protected String[] filterNodeIds(DiscoveryNodes nodes, String[] nodesIds) {
             List<String> list = new ArrayList<>();
             for (String node : nodesIds) {
-                if (nodes.getDataNodes().containsKey(node)) {
+                if (nodes.nodeExists(node)) {
                     list.add(node);
                 }
             }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java
index 1be7c626502..0a63dd46095 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIT.java
@@ -27,7 +27,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.index.store.Store;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@@ -37,12 +36,12 @@ import org.hamcrest.Matchers;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ExecutionException;
 
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
-@ClusterScope(scope = Scope.SUITE, numDataNodes = 1, numClientNodes = 0)
+@ClusterScope(scope = Scope.TEST, numDataNodes = 0)
 public class ClusterStatsIT extends ESIntegTestCase {
 
     private void assertCounts(ClusterStatsNodes.Counts counts, int total, Map<String, Integer> roles) {
@@ -58,12 +57,14 @@ public class ClusterStatsIT extends ESIntegTestCase {
 
     public void testNodeCounts() {
         int total = 1;
+        internalCluster().startNode();
         Map<String, Integer> expectedCounts = new HashMap<>();
         expectedCounts.put(DiscoveryNode.Role.DATA.getRoleName(), 1);
         expectedCounts.put(DiscoveryNode.Role.MASTER.getRoleName(), 1);
         expectedCounts.put(DiscoveryNode.Role.INGEST.getRoleName(), 1);
         expectedCounts.put(ClusterStatsNodes.Counts.COORDINATING_ONLY, 0);
         int numNodes = randomIntBetween(1, 5);
+
         ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get();
         assertCounts(response.getNodesStats().getCounts(), total, expectedCounts);
 
@@ -112,7 +113,9 @@ public class ClusterStatsIT extends ESIntegTestCase {
         assertThat(stats.getReplication(), Matchers.equalTo(replicationFactor));
     }
 
-    public void testIndicesShardStats() {
+    public void testIndicesShardStats() throws ExecutionException, InterruptedException {
+        internalCluster().startNode();
+        ensureGreen();
         ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get();
         assertThat(response.getStatus(), Matchers.equalTo(ClusterHealthStatus.GREEN));
 
@@ -155,10 +158,8 @@ public class ClusterStatsIT extends ESIntegTestCase {
 
     }
 
-    public void testValuesSmokeScreen() throws IOException {
-        internalCluster().ensureAtMostNumDataNodes(5);
-        internalCluster().ensureAtLeastNumDataNodes(1);
-        assertAcked(prepareCreate("test1").setSettings(Settings.builder().put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), 0).build()));
+    public void testValuesSmokeScreen() throws IOException, ExecutionException, InterruptedException {
+        internalCluster().startNodesAsync(randomIntBetween(1, 3)).get();
         index("test1", "type", "1", "f", "f");
         /*
          * Ensure at least one shard is allocated otherwise the FS stats might
@@ -188,9 +189,6 @@ public class ClusterStatsIT extends ESIntegTestCase {
     }
 
     public void testAllocatedProcessors() throws Exception {
-        // stop all other nodes
-        internalCluster().ensureAtMostNumDataNodes(0);
-
         // start one node with 7 processors.
         internalCluster().startNodesAsync(Settings.builder().put(EsExecutors.PROCESSORS_SETTING.getKey(), 7).build()).get();
         waitForNodes(1);
@@ -200,14 +198,15 @@ public class ClusterStatsIT extends ESIntegTestCase {
     }
 
     public void testClusterStatusWhenStateNotRecovered() throws Exception {
-        // stop all other nodes
-        internalCluster().ensureAtMostNumDataNodes(0);
-
-        internalCluster().startNode(Settings.builder().put("gateway.recover_after_nodes", 2).build());
+        internalCluster().startMasterOnlyNode(Settings.builder().put("gateway.recover_after_nodes", 2).build());
         ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get();
         assertThat(response.getStatus(), equalTo(ClusterHealthStatus.RED));
 
-        internalCluster().ensureAtLeastNumDataNodes(3);
+        if (randomBoolean()) {
+            internalCluster().startMasterOnlyNode(Settings.EMPTY);
+        } else {
+            internalCluster().startDataOnlyNode(Settings.EMPTY);
+        }
         // wait for the cluster status to settle
         ensureGreen();
         response = client().admin().cluster().prepareClusterStats().get();
diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientRetryIT.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientRetryIT.java
index 4ec1f66df57..ed9136851b4 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientRetryIT.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientRetryIT.java
@@ -29,7 +29,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.node.Node;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
@@ -41,7 +40,7 @@ import java.util.concurrent.ExecutionException;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
-@ClusterScope(scope = Scope.TEST, numClientNodes = 0)
+@ClusterScope(scope = Scope.TEST, numClientNodes = 0, supportsDedicatedMasters = false)
 public class TransportClientRetryIT extends ESIntegTestCase {
     public void testRetry() throws IOException, ExecutionException, InterruptedException {
         Iterable<TransportService> instances = internalCluster().getInstances(TransportService.class);
diff --git a/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java b/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java
index e61bbc5f719..194b82620de 100644
--- a/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java
@@ -37,7 +37,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
 
-@ClusterScope(scope = SUITE, numDataNodes = 1)
+@ClusterScope(scope = SUITE, supportsDedicatedMasters = false, numDataNodes = 1)
 public class SettingsFilteringIT extends ESIntegTestCase {
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java b/core/src/test/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java
index 84cd23a7f09..5e8a99d82fe 100644
--- a/core/src/test/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/shards/ClusterSearchShardsIT.java
@@ -43,7 +43,7 @@ public class ClusterSearchShardsIT extends ESIntegTestCase {
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
-        switch(nodeOrdinal) {
+        switch(nodeOrdinal % 2) {
         case 1:
             return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put("node.attr.tag", "B").build();
         case 0:
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java
index eeda96743df..ba6b4438aaa 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java
@@ -41,7 +41,7 @@ import static org.hamcrest.Matchers.hasSize;
 /**
  *
  */
-@ClusterScope(scope = Scope.TEST, numDataNodes = 1)
+@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1)
 public class NettyHttpRequestSizeLimitIT extends ESIntegTestCase {
     private static final ByteSizeValue LIMIT = new ByteSizeValue(2, ByteSizeUnit.KB);
 
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningDisabledIT.java b/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningDisabledIT.java
index f048b5526d5..576456c0647 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningDisabledIT.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningDisabledIT.java
@@ -40,7 +40,7 @@ import static org.hamcrest.Matchers.hasSize;
 /**
  *
  */
-@ClusterScope(scope = Scope.TEST, numDataNodes = 1)
+@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1)
 public class NettyPipeliningDisabledIT extends ESIntegTestCase {
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningEnabledIT.java b/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningEnabledIT.java
index fd1c493ac36..7ca714ece93 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningEnabledIT.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyPipeliningEnabledIT.java
@@ -36,7 +36,7 @@ import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
 
 
-@ClusterScope(scope = Scope.TEST, numDataNodes = 1)
+@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1)
 public class NettyPipeliningEnabledIT extends ESIntegTestCase {
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
diff --git a/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java b/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java
index e9e8dcfc007..07afef1d8ae 100644
--- a/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java
+++ b/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java
@@ -34,7 +34,7 @@ import java.util.Collections;
 import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 
-@ClusterScope(scope = SUITE, numDataNodes = 1, numClientNodes = 0)
+@ClusterScope(scope = SUITE, supportsDedicatedMasters = false, numDataNodes = 1, numClientNodes = 0)
 public class SettingsListenerIT extends ESIntegTestCase {
 
     @Override
@@ -109,13 +109,13 @@ public class SettingsListenerIT extends ESIntegTestCase {
                 .put("index.test.new.setting", 21)
                 .build()).get());
 
-        for (SettingsTestingService instance : internalCluster().getInstances(SettingsTestingService.class)) {
+        for (SettingsTestingService instance : internalCluster().getDataNodeInstances(SettingsTestingService.class)) {
             assertEquals(21, instance.value);
         }
 
         client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder()
                 .put("index.test.new.setting", 42)).get();
-        for (SettingsTestingService instance : internalCluster().getInstances(SettingsTestingService.class)) {
+        for (SettingsTestingService instance : internalCluster().getDataNodeInstances(SettingsTestingService.class)) {
             assertEquals(42, instance.value);
         }
 
@@ -123,14 +123,14 @@ public class SettingsListenerIT extends ESIntegTestCase {
                 .put("index.test.new.setting", 21)
                 .build()).get());
 
-        for (SettingsTestingService instance : internalCluster().getInstances(SettingsTestingService.class)) {
+        for (SettingsTestingService instance : internalCluster().getDataNodeInstances(SettingsTestingService.class)) {
             assertEquals(42, instance.value);
         }
 
         client().admin().indices().prepareUpdateSettings("other").setSettings(Settings.builder()
                 .put("index.test.new.setting", 84)).get();
 
-        for (SettingsTestingService instance : internalCluster().getInstances(SettingsTestingService.class)) {
+        for (SettingsTestingService instance : internalCluster().getDataNodeInstances(SettingsTestingService.class)) {
             assertEquals(42, instance.value);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java
index cf56f41c83a..110e34d59a9 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineMergeIT.java
@@ -37,7 +37,7 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 
-@ClusterScope(numDataNodes = 1, scope = Scope.SUITE)
+@ClusterScope(supportsDedicatedMasters = false, numDataNodes = 1, scope = Scope.SUITE)
 public class InternalEngineMergeIT extends ESIntegTestCase {
 
     @TestLogging("_root:DEBUG")
diff --git a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
index df8736b44ef..c3323919c82 100644
--- a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
+++ b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.index.store;
 
+import com.carrotsearch.hppc.cursors.IntObjectCursor;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import org.apache.lucene.index.CheckIndex;
 import org.apache.lucene.index.IndexFileNames;
@@ -26,6 +27,7 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.Requests;
@@ -48,6 +50,7 @@ import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.gateway.PrimaryShardAllocator;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
@@ -65,7 +68,6 @@ import org.elasticsearch.snapshots.SnapshotState;
 import org.elasticsearch.test.CorruptionUtils;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
-import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.MockIndexEventListener;
 import org.elasticsearch.test.store.MockFSIndexStore;
 import org.elasticsearch.test.transport.MockTransportService;
@@ -84,9 +86,7 @@ import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.concurrent.CopyOnWriteArrayList;
@@ -113,13 +113,13 @@ public class CorruptedFileIT extends ESIntegTestCase {
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
-                // we really need local GW here since this also checks for corruption etc.
-                // and we need to make sure primaries are not just trashed if we don't have replicas
-                .put(super.nodeSettings(nodeOrdinal))
-                        // speed up recoveries
-                .put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING.getKey(), 5)
-                .put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING.getKey(), 5)
-                .build();
+            // we really need local GW here since this also checks for corruption etc.
+            // and we need to make sure primaries are not just trashed if we don't have replicas
+            .put(super.nodeSettings(nodeOrdinal))
+            // speed up recoveries
+            .put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING.getKey(), 5)
+            .put(ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING.getKey(), 5)
+            .build();
     }
 
     @Override
@@ -142,11 +142,11 @@ public class CorruptedFileIT extends ESIntegTestCase {
         assertThat(cluster().numDataNodes(), greaterThanOrEqualTo(3));
 
         assertAcked(prepareCreate("test").setSettings(Settings.builder()
-                        .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
-                        .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "1")
-                        .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)
-                        .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) // no checkindex - we corrupt shards on purpose
-                        .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)) // no translog based flush - it might change the .liv / segments.N files
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "1")
+            .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)
+            .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) // no checkindex - we corrupt shards on purpose
+            .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)) // no translog based flush - it might change the .liv / segments.N files
         ));
         ensureGreen();
         disableAllocation("test");
@@ -171,9 +171,9 @@ public class CorruptedFileIT extends ESIntegTestCase {
         Settings build = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "2").build();
         client().admin().indices().prepareUpdateSettings("test").setSettings(build).get();
         ClusterHealthResponse health = client().admin().cluster()
-                .health(Requests.clusterHealthRequest("test").waitForGreenStatus()
-                        .timeout("5m") // sometimes due to cluster rebalacing and random settings default timeout is just not enough.
-                        .waitForRelocatingShards(0)).actionGet();
+            .health(Requests.clusterHealthRequest("test").waitForGreenStatus()
+                .timeout("5m") // sometimes due to cluster rebalacing and random settings default timeout is just not enough.
+                .waitForRelocatingShards(0)).actionGet();
         if (health.isTimedOut()) {
             logger.info("cluster state:\n{}\n{}", client().admin().cluster().prepareState().get().getState().prettyPrint(), client().admin().cluster().preparePendingClusterTasks().get().prettyPrint());
             assertThat("timed out waiting for green state", health.isTimedOut(), equalTo(false));
@@ -247,10 +247,10 @@ public class CorruptedFileIT extends ESIntegTestCase {
         internalCluster().ensureAtLeastNumDataNodes(2);
 
         assertAcked(prepareCreate("test").setSettings(Settings.builder()
-                        .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
-                        .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)
-                        .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) // no checkindex - we corrupt shards on purpose
-                        .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)) // no translog based flush - it might change the .liv / segments.N files
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
+            .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)
+            .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) // no checkindex - we corrupt shards on purpose
+            .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)) // no translog based flush - it might change the .liv / segments.N files
         ));
         ensureGreen();
         IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs];
@@ -273,12 +273,12 @@ public class CorruptedFileIT extends ESIntegTestCase {
         client().admin().cluster().prepareReroute().get();
 
         boolean didClusterTurnRed = awaitBusy(() -> {
-                ClusterHealthStatus test = client().admin().cluster()
-                        .health(Requests.clusterHealthRequest("test")).actionGet().getStatus();
-                return test == ClusterHealthStatus.RED;
+            ClusterHealthStatus test = client().admin().cluster()
+                .health(Requests.clusterHealthRequest("test")).actionGet().getStatus();
+            return test == ClusterHealthStatus.RED;
         }, 5, TimeUnit.MINUTES);// sometimes on slow nodes the replication / recovery is just dead slow
         final ClusterHealthResponse response = client().admin().cluster()
-                .health(Requests.clusterHealthRequest("test")).get();
+            .health(Requests.clusterHealthRequest("test")).get();
         if (response.getStatus() != ClusterHealthStatus.RED) {
             logger.info("Cluster turned red in busy loop: {}", didClusterTurnRed);
             logger.info("cluster state:\n{}\n{}", client().admin().cluster().prepareState().get().getState().prettyPrint(), client().admin().cluster().preparePendingClusterTasks().get().prettyPrint());
@@ -328,10 +328,10 @@ public class CorruptedFileIT extends ESIntegTestCase {
         NodeStats primariesNode = dataNodeStats.get(0);
         NodeStats unluckyNode = dataNodeStats.get(1);
         assertAcked(prepareCreate("test").setSettings(Settings.builder()
-                        .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
-                        .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
-                        .put("index.routing.allocation.include._name", primariesNode.getNode().getName())
-                        .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put("index.routing.allocation.include._name", primariesNode.getNode().getName())
+            .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE)
 
         ));
         ensureGreen(); // allocated with empty commit
@@ -356,8 +356,8 @@ public class CorruptedFileIT extends ESIntegTestCase {
         }
 
         Settings build = Settings.builder()
-                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "1")
-                .put("index.routing.allocation.include._name", primariesNode.getNode().getName() + "," + unluckyNode.getNode().getName()).build();
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "1")
+            .put("index.routing.allocation.include._name", primariesNode.getNode().getName() + "," + unluckyNode.getNode().getName()).build();
         client().admin().indices().prepareUpdateSettings("test").setSettings(build).get();
         client().admin().cluster().prepareReroute().get();
         hasCorrupted.await();
@@ -390,12 +390,12 @@ public class CorruptedFileIT extends ESIntegTestCase {
 
 
         assertAcked(prepareCreate("test").setSettings(Settings.builder()
-                        .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
-                        .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, between(1, 4)) // don't go crazy here it must recovery fast
-                                // This does corrupt files on the replica, so we can't check:
-                        .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false)
-                        .put("index.routing.allocation.include._name", primariesNode.getNode().getName())
-                        .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, between(1, 4)) // don't go crazy here it must recovery fast
+            // This does corrupt files on the replica, so we can't check:
+            .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false)
+            .put("index.routing.allocation.include._name", primariesNode.getNode().getName())
+            .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE)
         ));
         ensureGreen();
         IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs];
@@ -432,12 +432,12 @@ public class CorruptedFileIT extends ESIntegTestCase {
         }
 
         Settings build = Settings.builder()
-                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "1")
-                .put("index.routing.allocation.include._name", "*").build();
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "1")
+            .put("index.routing.allocation.include._name", "*").build();
         client().admin().indices().prepareUpdateSettings("test").setSettings(build).get();
         client().admin().cluster().prepareReroute().get();
         ClusterHealthResponse actionGet = client().admin().cluster()
-                .health(Requests.clusterHealthRequest("test").waitForGreenStatus()).actionGet();
+            .health(Requests.clusterHealthRequest("test").waitForGreenStatus()).actionGet();
         if (actionGet.isTimedOut()) {
             logger.info("ensureGreen timed out, cluster state:\n{}\n{}", client().admin().cluster().prepareState().get().getState().prettyPrint(), client().admin().cluster().preparePendingClusterTasks().get().prettyPrint());
             assertThat("timed out waiting for green state", actionGet.isTimedOut(), equalTo(false));
@@ -525,11 +525,12 @@ public class CorruptedFileIT extends ESIntegTestCase {
         internalCluster().ensureAtLeastNumDataNodes(2);
 
         assertAcked(prepareCreate("test").setSettings(Settings.builder()
-                        .put(PrimaryShardAllocator.INDEX_RECOVERY_INITIAL_SHARDS_SETTING.getKey(), "one")
-                        .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, cluster().numDataNodes() - 1)
-                        .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)
-                        .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) // no checkindex - we corrupt shards on purpose
-                        .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.PB)) // no translog based flush - it might change the .liv / segments.N files
+            .put(PrimaryShardAllocator.INDEX_RECOVERY_INITIAL_SHARDS_SETTING.getKey(), "one")
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, cluster().numDataNodes() - 1)
+            .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)
+            .put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false) // no checkindex - we corrupt shards on purpose
+            .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(),
+                new ByteSizeValue(1, ByteSizeUnit.PB)) // no translog based flush - it might change the .liv / segments.N files
         ));
         ensureGreen();
         IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs];
@@ -543,22 +544,41 @@ public class CorruptedFileIT extends ESIntegTestCase {
         SearchResponse countResponse = client().prepareSearch().setSize(0).get();
         assertHitCount(countResponse, numDocs);
 
-        final Map<String, List<Path>> filesToCorrupt = findFilesToCorruptForReplica();
-        internalCluster().fullRestart(new InternalTestCluster.RestartCallback() {
-            @Override
-            public Settings onNodeStopped(String nodeName) throws Exception {
-                List<Path> paths = filesToCorrupt.get(nodeName);
-                if (paths != null) {
-                    for (Path path : paths) {
+        // disable allocations of replicas post restart (the restart will change replicas to primaries, so we have
+        // to capture replicas post restart)
+        assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(
+            Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), "primaries")
+        ));
+
+        internalCluster().fullRestart();
+
+        ensureYellow();
+
+        final Index index = resolveIndex("test");
+
+        final IndicesShardStoresResponse stores = client().admin().indices().prepareShardStores(index.getName()).get();
+
+
+        for (IntObjectCursor<List<IndicesShardStoresResponse.StoreStatus>> shards :
+            stores.getStoreStatuses().get(index.getName())) {
+            for (IndicesShardStoresResponse.StoreStatus store : shards.value) {
+                final ShardId shardId = new ShardId(index, shards.key);
+                if (store.getAllocationStatus().equals(IndicesShardStoresResponse.StoreStatus.AllocationStatus.UNUSED)) {
+                    for (Path path : findFilesToCorruptOnNode(store.getNode().getName(), shardId)) {
                         try (OutputStream os = Files.newOutputStream(path)) {
                             os.write(0);
                         }
-                        logger.info("corrupting file {} on node {}", path, nodeName);
+                        logger.info("corrupting file {} on node {}", path, store.getNode().getName());
                     }
                 }
-                return null;
             }
-        });
+        }
+
+        // enable allocation
+        assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(
+            Settings.builder().putNull(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey())
+        ));
+
         ensureGreen();
     }
 
@@ -568,34 +588,21 @@ public class CorruptedFileIT extends ESIntegTestCase {
         return shardIterators.size();
     }
 
-    private Map<String, List<Path>> findFilesToCorruptForReplica() throws IOException {
-        Map<String, List<Path>> filesToNodes = new HashMap<>();
-        ClusterState state = client().admin().cluster().prepareState().get().getState();
-        Index test = state.metaData().index("test").getIndex();
-        for (ShardRouting shardRouting : state.getRoutingTable().allShards("test")) {
-            if (shardRouting.primary()) {
-                continue;
-            }
-            assertTrue(shardRouting.assignedToNode());
-            NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(shardRouting.currentNodeId()).setFs(true).get();
-            NodeStats nodeStats = nodeStatses.getNodes().get(0);
-            List<Path> files = new ArrayList<>();
-            filesToNodes.put(nodeStats.getNode().getName(), files);
-            for (FsInfo.Path info : nodeStats.getFs()) {
-                String path = info.getPath();
-                Path file = PathUtils.get(path).resolve("indices").resolve(test.getUUID()).resolve(Integer.toString(shardRouting.getId())).resolve("index");
-                if (Files.exists(file)) { // multi data path might only have one path in use
-                    try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) {
-                        for (Path item : stream) {
-                            if (item.getFileName().toString().startsWith("segments_")) {
-                                files.add(item);
-                            }
+    private List<Path> findFilesToCorruptOnNode(final String nodeName, final ShardId shardId) throws IOException {
+        List<Path> files = new ArrayList<>();
+        for (Path path : internalCluster().getInstance(NodeEnvironment.class, nodeName).availableShardPaths(shardId)) {
+            path = path.resolve("index");
+            if (Files.exists(path)) { // multi data path might only have one path in use
+                try (DirectoryStream<Path> stream = Files.newDirectoryStream(path)) {
+                    for (Path item : stream) {
+                        if (item.getFileName().toString().startsWith("segments_")) {
+                            files.add(item);
                         }
                     }
                 }
             }
         }
-        return filesToNodes;
+        return files;
     }
 
     private ShardRouting corruptRandomPrimaryFile() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
index 92c3260aeb0..66aed327f70 100644
--- a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
@@ -39,16 +39,16 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.MergePolicyConfig;
+import org.elasticsearch.index.MergeSchedulerConfig;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.cache.query.QueryCacheStats;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.MergePolicyConfig;
-import org.elasticsearch.index.MergeSchedulerConfig;
 import org.elasticsearch.index.store.IndexStore;
 import org.elasticsearch.index.translog.Translog;
-import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.IndicesRequestCache;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
@@ -103,7 +103,7 @@ public class IndexStatsIT extends ESIntegTestCase {
         client().prepareIndex("test", "type", "2").setSource("field", "value2", "field2", "value2").execute().actionGet();
         client().admin().indices().prepareRefresh().execute().actionGet();
 
-        NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
+        NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true).execute().actionGet();
         assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L));
         IndicesStatsResponse indicesStats = client().admin().indices().prepareStats("test").clear().setFieldData(true).execute().actionGet();
         assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0L));
@@ -112,7 +112,7 @@ public class IndexStatsIT extends ESIntegTestCase {
         client().prepareSearch().addSort("field", SortOrder.ASC).execute().actionGet();
         client().prepareSearch().addSort("field", SortOrder.ASC).execute().actionGet();
 
-        nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
+        nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true).execute().actionGet();
         assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
         indicesStats = client().admin().indices().prepareStats("test").clear().setFieldData(true).execute().actionGet();
         assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
@@ -122,7 +122,7 @@ public class IndexStatsIT extends ESIntegTestCase {
         client().prepareSearch().addSort("field2", SortOrder.ASC).execute().actionGet();
 
         // now check the per field stats
-        nodesStats = client().admin().cluster().prepareNodesStats().setIndices(new CommonStatsFlags().set(CommonStatsFlags.Flag.FieldData, true).fieldDataFields("*")).execute().actionGet();
+        nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(new CommonStatsFlags().set(CommonStatsFlags.Flag.FieldData, true).fieldDataFields("*")).execute().actionGet();
         assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
         assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getFields().get("field") + nodesStats.getNodes().get(1).getIndices().getFieldData().getFields().get("field"), greaterThan(0L));
         assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getFields().get("field") + nodesStats.getNodes().get(1).getIndices().getFieldData().getFields().get("field"), lessThan(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes()));
@@ -133,7 +133,7 @@ public class IndexStatsIT extends ESIntegTestCase {
         assertThat(indicesStats.getTotal().getFieldData().getFields().get("field"), lessThan(indicesStats.getTotal().getFieldData().getMemorySizeInBytes()));
 
         client().admin().indices().prepareClearCache().setFieldDataCache(true).execute().actionGet();
-        nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
+        nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true).execute().actionGet();
         assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L));
         indicesStats = client().admin().indices().prepareStats("test").clear().setFieldData(true).execute().actionGet();
         assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0L));
@@ -150,7 +150,7 @@ public class IndexStatsIT extends ESIntegTestCase {
         client().prepareIndex("test", "type", "2").setSource("field", "value2").execute().actionGet();
         client().admin().indices().prepareRefresh().execute().actionGet();
 
-        NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true)
+        NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true)
                 .execute().actionGet();
         assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L));
         assertThat(nodesStats.getNodes().get(0).getIndices().getQueryCache().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getQueryCache().getMemorySizeInBytes(), equalTo(0L));
@@ -171,7 +171,7 @@ public class IndexStatsIT extends ESIntegTestCase {
                 .addSort("field", SortOrder.ASC)
                 .execute().actionGet();
 
-        nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true)
+        nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true)
                 .execute().actionGet();
         assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
         assertThat(nodesStats.getNodes().get(0).getIndices().getQueryCache().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getQueryCache().getMemorySizeInBytes(), greaterThan(0L));
@@ -184,7 +184,7 @@ public class IndexStatsIT extends ESIntegTestCase {
 
         client().admin().indices().prepareClearCache().execute().actionGet();
         Thread.sleep(100); // Make sure the filter cache entries have been removed...
-        nodesStats = client().admin().cluster().prepareNodesStats().setIndices(true)
+        nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true)
                 .execute().actionGet();
         assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L));
         assertThat(nodesStats.getNodes().get(0).getIndices().getQueryCache().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getQueryCache().getMemorySizeInBytes(), equalTo(0L));
diff --git a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java
index f839cd2583f..683ae71a112 100644
--- a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java
+++ b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java
@@ -36,7 +36,7 @@ import static org.hamcrest.Matchers.is;
 /**
  * Tests that when disabling detailed errors, a request with the error_trace parameter returns a HTTP 400
  */
-@ClusterScope(scope = Scope.TEST, numDataNodes = 1)
+@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1)
 public class DetailedErrorsDisabledIT extends ESIntegTestCase {
     // Build our cluster settings
     @Override
diff --git a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java
index 6b9d78e5468..d98db83dddc 100644
--- a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java
+++ b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java
@@ -23,7 +23,6 @@ import org.apache.http.impl.client.HttpClients;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.http.HttpServerTransport;
-import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
@@ -37,7 +36,7 @@ import static org.hamcrest.Matchers.not;
 /**
  * Tests that by default the error_trace parameter can be used to show stacktraces
  */
-@ClusterScope(scope = Scope.TEST, numDataNodes = 1)
+@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1)
 public class DetailedErrorsEnabledIT extends ESIntegTestCase {
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
diff --git a/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java b/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
index 85d83e69b75..ac6d8fddd89 100644
--- a/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
+++ b/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
@@ -35,7 +35,7 @@ import static org.hamcrest.Matchers.equalTo;
 /**
  * Test a rest action that sets special response headers
  */
-@ClusterScope(scope = Scope.SUITE, numDataNodes = 1)
+@ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1)
 public class ResponseHeaderPluginIT extends ESIntegTestCase {
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
@@ -60,4 +60,4 @@ public class ResponseHeaderPluginIT extends ESIntegTestCase {
         assertThat(authResponse, hasStatus(OK));
         assertThat(authResponse.getHeaders().get("Secret"), equalTo("granted"));
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java b/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java
index 9e043c660f4..dfdd88d1987 100644
--- a/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java
+++ b/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java
@@ -32,7 +32,7 @@ import static org.hamcrest.Matchers.not;
 /**
  *
  */
-@ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 1)
+@ClusterScope(scope = ESIntegTestCase.Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1)
 public class CorsNotSetIT extends ESIntegTestCase {
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java b/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java
index 837b0523072..18351dcb294 100644
--- a/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java
+++ b/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java
@@ -39,7 +39,7 @@ import static org.hamcrest.Matchers.not;
 /**
  * Test CORS where the allow origin value is a regular expression.
  */
-@ClusterScope(scope = Scope.SUITE, numDataNodes = 1)
+@ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1)
 public class CorsRegexIT extends ESIntegTestCase {
 
     protected static final ESLogger logger = Loggers.getLogger(CorsRegexIT.class);
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
index ea76dbf7d6c..cf8830aa7ef 100644
--- a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
+++ b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
@@ -58,7 +58,7 @@ import static org.hamcrest.CoreMatchers.equalTo;
 /**
  *
  */
-@ClusterScope(scope = Scope.SUITE, numDataNodes = 1)
+@ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1)
 public class FetchSubPhasePluginIT extends ESIntegTestCase {
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java
index 489c6bf908f..9999f96e0b1 100644
--- a/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java
+++ b/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java
@@ -56,7 +56,7 @@ import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
-@ClusterScope(scope = Scope.SUITE, numDataNodes = 1)
+@ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1)
 public class ExplainableScriptIT extends ESIntegTestCase {
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java
index 2d3c593c895..7fdeca4680e 100644
--- a/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java
+++ b/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java
@@ -53,7 +53,7 @@ import static org.hamcrest.Matchers.equalTo;
 /**
  *
  */
-@ClusterScope(scope = Scope.SUITE, numDataNodes = 1)
+@ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1)
 public class FunctionScorePluginIT extends ESIntegTestCase {
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
diff --git a/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterSearchIT.java b/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterSearchIT.java
index 07045cc9176..4b5153b2417 100644
--- a/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterSearchIT.java
@@ -37,7 +37,7 @@ import static org.hamcrest.Matchers.equalTo;
 /**
  *
  */
-@ClusterScope(scope = Scope.SUITE, numDataNodes = 1)
+@ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1)
 public class CustomHighlighterSearchIT extends ESIntegTestCase {
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
index 7e0bb173e40..102173626f3 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
@@ -71,7 +71,8 @@ public abstract class AbstractSnapshotIntegTestCase extends ESIntegTestCase {
 
     public static long getFailureCount(String repository) {
         long failureCount = 0;
-        for (RepositoriesService repositoriesService : internalCluster().getDataNodeInstances(RepositoriesService.class)) {
+        for (RepositoriesService repositoriesService :
+            internalCluster().getDataOrMasterNodeInstances(RepositoriesService.class)) {
             MockRepository mockRepository = (MockRepository) repositoriesService.repository(repository);
             failureCount += mockRepository.getFailureCount();
         }
diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
index 7ca30132a4a..e9d10d31a04 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -563,6 +563,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
                 }
             }
         } catch (Exception ex) {
+            logger.info("--> caught a top level exception, asserting what's expected", ex);
             assertThat(getFailureCount("test-repo"), greaterThan(0L));
             assertThat(ExceptionsHelper.detailedMessage(ex), containsString("IOException"));
         }
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java
index 10e20928376..f01a4ca5e92 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java
@@ -53,7 +53,7 @@ import static org.hamcrest.Matchers.is;
 /**
  *
  */
-@ClusterScope(scope = Scope.TEST, numDataNodes = 1)
+@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1)
 public class NettyTransportIT extends ESIntegTestCase {
     // static so we can use it in anonymous classes
     private static String channelProfileName = null;
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortIntegrationIT.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortIntegrationIT.java
index 7f673e00fee..0189b73799a 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortIntegrationIT.java
@@ -47,7 +47,7 @@ import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
-@ClusterScope(scope = Scope.SUITE, numDataNodes = 1, numClientNodes = 0)
+@ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1, numClientNodes = 0)
 public class NettyTransportMultiPortIntegrationIT extends ESIntegTestCase {
 
     private static int randomPort = -1;
diff --git a/core/src/test/java/org/elasticsearch/tribe/TribeIT.java b/core/src/test/java/org/elasticsearch/tribe/TribeIT.java
index 52827491fe4..6ea466d3cb3 100644
--- a/core/src/test/java/org/elasticsearch/tribe/TribeIT.java
+++ b/core/src/test/java/org/elasticsearch/tribe/TribeIT.java
@@ -94,7 +94,7 @@ public class TribeIT extends ESIntegTestCase {
             }
 
         };
-        cluster2 = new InternalTestCluster(InternalTestCluster.configuredNodeMode(), randomLong(), createTempDir(), 2, 2,
+        cluster2 = new InternalTestCluster(InternalTestCluster.configuredNodeMode(), randomLong(), createTempDir(), true, 2, 2,
                 UUIDs.randomBase64UUID(random()), nodeConfigurationSource, 0, false, SECOND_CLUSTER_NODE_PREFIX, Collections.emptyList(), Function.identity());
 
         cluster2.beforeTest(random(), 0.1);
diff --git a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
index f216f998305..f6a74faac39 100644
--- a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
+++ b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
@@ -49,7 +49,7 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
-@ClusterScope(scope= Scope.SUITE, numDataNodes = 1)
+@ClusterScope(scope= Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1)
 public class SimpleTTLIT extends ESIntegTestCase {
 
     static private final long PURGE_INTERVAL = 200;
diff --git a/docs/reference/testing/testing-framework.asciidoc b/docs/reference/testing/testing-framework.asciidoc
index e0b27733441..fe7daf8cca4 100644
--- a/docs/reference/testing/testing-framework.asciidoc
+++ b/docs/reference/testing/testing-framework.asciidoc
@@ -148,13 +148,19 @@ You can use the `@ClusterScope` annotation at class level to configure this beha
 
 [source,java]
 -----------------------------------------
-@ClusterScope(scope=TEST, numNodes=1)
+@ClusterScope(scope=TEST, numDataNodes=1)
 public class CustomSuggesterSearchTests extends ESIntegTestCase {
   // ... tests go here
 }
 -----------------------------------------
 
-The above sample configures the test to use a new cluster for each test method. The default scope is `SUITE` (one cluster for all test methods in the test). The `numNodes` settings allows you to only start a certain number of nodes, which can speed up test execution, as starting a new node is a costly and time consuming operation and might not be needed for this test.
+The above sample configures the test to use a new cluster for each test method. The default scope is `SUITE` (one cluster for all
+test methods in the test). The `numDataNodes` settings allows you to only start a certain number of data nodes, which can speed up test
+execution, as starting a new node is a costly and time consuming operation and might not be needed for this test.
+
+By default, the testing infrastructure will randomly start dedicated master nodes. If you want to disable dedicated masters
+you can set `supportsDedicatedMasters=false` in a similar fashion to the `numDataNodes` setting. If dedicated master nodes are not used,
+data nodes will be allowed to become masters as well.
 
 
 [[changing-node-configuration]]
diff --git a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryClusterFormationTests.java b/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryClusterFormationTests.java
index 6c2bec3ba43..489f7043b83 100644
--- a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryClusterFormationTests.java
+++ b/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryClusterFormationTests.java
@@ -277,9 +277,9 @@ public class AzureDiscoveryClusterFormationTests extends ESIntegTestCase {
 
     public void testJoin() throws ExecutionException, InterruptedException {
         // only wait for the cluster to form
-        assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(2)).get());
+        ensureClusterSizeConsistency();
         // add one more node and wait for it to join
         internalCluster().startDataOnlyNodeAsync().get();
-        assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(3)).get());
+        ensureClusterSizeConsistency();
     }
 }
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java
index 6d1104ce6e2..0271f522943 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java
@@ -57,7 +57,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTi
 import static org.hamcrest.Matchers.equalTo;
 
 @ESIntegTestCase.SuppressLocalMode
-@ESIntegTestCase.ClusterScope(numDataNodes = 2, numClientNodes = 0)
+@ESIntegTestCase.ClusterScope(supportsDedicatedMasters = false, numDataNodes = 2, numClientNodes = 0)
 @SuppressForbidden(reason = "use http server")
 // TODO this should be a IT but currently all ITs in this project run against a real cluster
 public class Ec2DiscoveryClusterFormationTests extends ESIntegTestCase {
diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java
index dbedbe1a6a9..f6a850d1d23 100644
--- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java
+++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java
@@ -57,7 +57,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTi
 
 
 @ESIntegTestCase.SuppressLocalMode
-@ESIntegTestCase.ClusterScope(numDataNodes = 2, numClientNodes = 0)
+@ESIntegTestCase.ClusterScope(supportsDedicatedMasters = false, numDataNodes = 2, numClientNodes = 0)
 @SuppressForbidden(reason = "use http server")
 // TODO this should be a IT but currently all ITs in this project run against a real cluster
 public class GceDiscoverTests extends ESIntegTestCase {
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceIntegTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceIntegTests.java
index 2c50deafb59..9ac15cae5a2 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceIntegTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreServiceIntegTests.java
@@ -36,6 +36,7 @@ import static org.hamcrest.Matchers.greaterThan;
 
 @ESIntegTestCase.ClusterScope(
         scope = ESIntegTestCase.Scope.SUITE,
+        supportsDedicatedMasters = false,
         numDataNodes = 1,
         numClientNodes = 0,
         transportClientRatio = 0.0)
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java
index 79b41a355b7..c062e765c32 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java
@@ -62,7 +62,7 @@ import static org.hamcrest.Matchers.greaterThan;
  */
 @ClusterScope(
         scope = ESIntegTestCase.Scope.SUITE,
-        numDataNodes = 1,
+        supportsDedicatedMasters = false, numDataNodes = 1,
         transportClientRatio = 0.0)
 public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyIntegTestCase {
     private String getRepositoryPath() {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index 8f62617ee7b..2a3eecf4cc8 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -108,8 +108,8 @@ import org.elasticsearch.index.codec.CodecService;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
 import org.elasticsearch.index.translog.Translog;
-import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.IndicesRequestCache;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.store.IndicesStore;
 import org.elasticsearch.node.NodeMocksPlugin;
 import org.elasticsearch.plugins.Plugin;
@@ -1026,7 +1026,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
         logger.info("memory: {}", XContentHelper.toString(client().admin().cluster().prepareNodesStats().clear().setJvm(true).get()));
     }
 
-    void ensureClusterSizeConsistency() {
+    protected void ensureClusterSizeConsistency() {
         if (cluster() != null) { // if static init fails the cluster can be null
             logger.trace("Check consistency for [{}] nodes", cluster().size());
             assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(cluster().size())).get());
@@ -1478,17 +1478,24 @@ public abstract class ESIntegTestCase extends ESTestCase {
         int numDataNodes() default -1;
 
         /**
-         * Returns the minimum number of nodes in the cluster. Default is <tt>-1</tt>.
+         * Returns the minimum number of data nodes in the cluster. Default is <tt>-1</tt>.
          * Ignored when {@link ClusterScope#numDataNodes()} is set.
          */
         int minNumDataNodes() default -1;
 
         /**
-         * Returns the maximum number of nodes in the cluster.  Default is <tt>-1</tt>.
+         * Returns the maximum number of data nodes in the cluster.  Default is <tt>-1</tt>.
          * Ignored when {@link ClusterScope#numDataNodes()} is set.
          */
         int maxNumDataNodes() default -1;
 
+        /**
+         * Indicates whether the cluster can have dedicated master nodes. If <tt>false</tt> means data nodes will serve as master nodes
+         * and there will be no dedicated master (and data) nodes. Default is <tt>true</tt> which means
+         * dedicated master nodes will be randomly used.
+         */
+        boolean supportsDedicatedMasters() default true;
+
         /**
          * Returns the number of client nodes in the cluster. Default is {@link InternalTestCluster#DEFAULT_NUM_CLIENT_NODES}, a
          * negative value means that the number of client nodes will be randomized.
@@ -1571,7 +1578,6 @@ public abstract class ESIntegTestCase extends ESTestCase {
         return getAnnotation(clazz.getSuperclass(), annotationClass);
     }
 
-
     private Scope getCurrentClusterScope() {
         return getCurrentClusterScope(this.getClass());
     }
@@ -1582,6 +1588,11 @@ public abstract class ESIntegTestCase extends ESTestCase {
         return annotation == null ? Scope.SUITE : annotation.scope();
     }
 
+    private boolean getSupportsDedicatedMasters() {
+        ClusterScope annotation = getAnnotation(this.getClass(), ClusterScope.class);
+        return annotation == null ? true : annotation.supportsDedicatedMasters();
+    }
+
     private int getNumDataNodes() {
         ClusterScope annotation = getAnnotation(this.getClass(), ClusterScope.class);
         return annotation == null ? -1 : annotation.numDataNodes();
@@ -1717,6 +1728,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
             }
         };
 
+        boolean supportsDedicatedMasters = getSupportsDedicatedMasters();
         int numDataNodes = getNumDataNodes();
         int minNumDataNodes;
         int maxNumDataNodes;
@@ -1739,7 +1751,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
 
         Collection<Class<? extends Plugin>> mockPlugins = getMockPlugins();
 
-        return new InternalTestCluster(nodeMode, seed, createTempDir(), minNumDataNodes, maxNumDataNodes,
+        return new InternalTestCluster(nodeMode, seed, createTempDir(), supportsDedicatedMasters, minNumDataNodes, maxNumDataNodes,
                 InternalTestCluster.clusterName(scope.name(), seed) + "-cluster", nodeConfigurationSource, getNumClientNodes(),
                 InternalTestCluster.DEFAULT_ENABLE_HTTP_PIPELINING, nodePrefix, mockPlugins, getClientWrapper());
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index aa1d0aa29ef..c67ffc07a2b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -161,7 +161,9 @@ public final class InternalTestCluster extends TestCluster {
 
     private static final int JVM_ORDINAL = Integer.parseInt(System.getProperty(SysGlobals.CHILDVM_SYSPROP_JVM_ID, "0"));
 
-    /** a per-JVM unique offset to be used for calculating unique port ranges. */
+    /**
+     * a per-JVM unique offset to be used for calculating unique port ranges.
+     */
     public static final int JVM_BASE_PORT_OFFSET = PORTS_PER_JVM * (JVM_ORDINAL + 1);
 
     private static final AtomicInteger clusterOrdinal = new AtomicInteger();
@@ -171,6 +173,9 @@ public final class InternalTestCluster extends TestCluster {
     public final int HTTP_BASE_PORT = GLOBAL_HTTP_BASE_PORT + CLUSTER_BASE_PORT_OFFSET;
 
 
+    static final int DEFAULT_LOW_NUM_MASTER_NODES = 1;
+    static final int DEFAULT_HIGH_NUM_MASTER_NODES = 3;
+
     static final int DEFAULT_MIN_NUM_DATA_NODES = 1;
     static final int DEFAULT_MAX_NUM_DATA_NODES = TEST_NIGHTLY ? 6 : 3;
 
@@ -198,9 +203,13 @@ public final class InternalTestCluster extends TestCluster {
      * fully shared cluster to be more reproducible */
     private final long[] sharedNodesSeeds;
 
-    private final int numSharedAllRolesNodes;
 
-    private final int numShareCoordOnlyNodes;
+    // if set to 0, data nodes will also assume the master role
+    private final int numSharedDedicatedMasterNodes;
+
+    private final int numSharedDataNodes;
+
+    private final int numSharedCoordOnlyNodes;
 
     private final NodeConfigurationSource nodeConfigurationSource;
 
@@ -219,6 +228,7 @@ public final class InternalTestCluster extends TestCluster {
     private Function<Client, Client> clientWrapper;
 
     public InternalTestCluster(String nodeMode, long clusterSeed, Path baseDir,
+                               boolean randomlyAddDedicatedMasters,
                                int minNumDataNodes, int maxNumDataNodes, String clusterName, NodeConfigurationSource nodeConfigurationSource, int numClientNodes,
                                boolean enableHttpPipelining, String nodePrefix, Collection<Class<? extends Plugin>> mockPlugins, Function<Client, Client> clientWrapper) {
         super(clusterSeed);
@@ -239,39 +249,47 @@ public final class InternalTestCluster extends TestCluster {
 
         Random random = new Random(clusterSeed);
 
-        this.numSharedAllRolesNodes = RandomInts.randomIntBetween(random, minNumDataNodes, maxNumDataNodes);
-        assert this.numSharedAllRolesNodes >= 0;
+        boolean useDedicatedMasterNodes = randomlyAddDedicatedMasters ? random.nextBoolean() : false;
 
-        //for now all shared data nodes are also master eligible
-        if (numSharedAllRolesNodes == 0) {
-            this.numShareCoordOnlyNodes = 0;
+        this.numSharedDataNodes = RandomInts.randomIntBetween(random, minNumDataNodes, maxNumDataNodes);
+        assert this.numSharedDataNodes >= 0;
+
+        if (numSharedDataNodes == 0) {
+            this.numSharedCoordOnlyNodes = 0;
+            this.numSharedDedicatedMasterNodes = 0;
         } else {
-            if (numClientNodes < 0) {
-                this.numShareCoordOnlyNodes = RandomInts.randomIntBetween(random, DEFAULT_MIN_NUM_CLIENT_NODES, DEFAULT_MAX_NUM_CLIENT_NODES);
+            if (useDedicatedMasterNodes) {
+                if (random.nextBoolean()) {
+                    // use a dedicated master, but only low number to reduce overhead to tests
+                    this.numSharedDedicatedMasterNodes = DEFAULT_LOW_NUM_MASTER_NODES;
+                } else {
+                    this.numSharedDedicatedMasterNodes = DEFAULT_HIGH_NUM_MASTER_NODES;
+                }
             } else {
-                this.numShareCoordOnlyNodes = numClientNodes;
+                this.numSharedDedicatedMasterNodes = 0;
+            }
+            if (numClientNodes < 0) {
+                this.numSharedCoordOnlyNodes = RandomInts.randomIntBetween(random, DEFAULT_MIN_NUM_CLIENT_NODES, DEFAULT_MAX_NUM_CLIENT_NODES);
+            } else {
+                this.numSharedCoordOnlyNodes = numClientNodes;
             }
         }
-        assert this.numShareCoordOnlyNodes >= 0;
+        assert this.numSharedCoordOnlyNodes >= 0;
 
         this.nodePrefix = nodePrefix;
 
         assert nodePrefix != null;
         this.mockPlugins = mockPlugins;
 
-        /*
-         *  TODO
-         *  - we might want start some master only nodes?
-         *  - we could add a flag that returns a client to the master all the time?
-         *  - we could add a flag that never returns a client to the master
-         *  - along those lines use a dedicated node that is master eligible and let all other nodes be only data nodes
-         */
-        sharedNodesSeeds = new long[numSharedAllRolesNodes + numShareCoordOnlyNodes];
+        sharedNodesSeeds = new long[numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes];
         for (int i = 0; i < sharedNodesSeeds.length; i++) {
             sharedNodesSeeds[i] = random.nextLong();
         }
 
-        logger.info("Setup InternalTestCluster [{}] with seed [{}] using [{}] data nodes and [{}] client nodes", clusterName, SeedUtils.formatSeed(clusterSeed), numSharedAllRolesNodes, numShareCoordOnlyNodes);
+        logger.info("Setup InternalTestCluster [{}] with seed [{}] using [{}] dedicated masters, " +
+                "[{}] (data) nodes and [{}] coord only nodes",
+            clusterName, SeedUtils.formatSeed(clusterSeed),
+            numSharedDedicatedMasterNodes, numSharedDataNodes, numSharedCoordOnlyNodes);
         this.nodeConfigurationSource = nodeConfigurationSource;
         Builder builder = Settings.builder();
         if (random.nextInt(5) == 0) { // sometimes set this
@@ -348,7 +366,16 @@ public final class InternalTestCluster extends TestCluster {
 
     private Settings getSettings(int nodeOrdinal, long nodeSeed, Settings others) {
         Builder builder = Settings.builder().put(defaultSettings)
-                .put(getRandomNodeSettings(nodeSeed));
+            .put(getRandomNodeSettings(nodeSeed));
+        Settings interimSettings = builder.build();
+        final String dataSuffix = getRoleSuffix(interimSettings);
+        if (dataSuffix.isEmpty() == false) {
+            // to make sure that a master node will not pick up on the data folder of a data only node
+            // once restarted we append the role suffix to each path.
+            String[] dataPath = Environment.PATH_DATA_SETTING.get(interimSettings).stream()
+                .map(path -> path + dataSuffix).toArray(String[]::new);
+            builder.putArray(Environment.PATH_DATA_SETTING.getKey(), dataPath);
+        }
         Settings settings = nodeConfigurationSource.nodeSettings(nodeOrdinal);
         if (settings != null) {
             if (settings.get(ClusterName.CLUSTER_NAME_SETTING.getKey()) != null) {
@@ -506,7 +533,11 @@ public final class InternalTestCluster extends TestCluster {
             int size = numDataNodes();
             for (int i = size; i < n; i++) {
                 logger.info("increasing cluster size from {} to {}", size, n);
-                asyncs.add(startNodeAsync());
+                if (numSharedDedicatedMasterNodes > 0) {
+                    asyncs.add(startDataOnlyNodeAsync());
+                } else {
+                    asyncs.add(startNodeAsync());
+                }
             }
         }
         try {
@@ -535,10 +566,10 @@ public final class InternalTestCluster extends TestCluster {
         }
         // prevent killing the master if possible and client nodes
         final Stream<NodeAndClient> collection =
-                n == 0 ? nodes.values().stream() : nodes.values().stream().filter(new DataNodePredicate().and(new MasterNodePredicate(getMasterName()).negate()));
+            n == 0 ? nodes.values().stream() : nodes.values().stream().filter(new DataNodePredicate().and(new MasterNodePredicate(getMasterName()).negate()));
         final Iterator<NodeAndClient> values = collection.iterator();
 
-        logger.info("changing cluster size from {} to {}, {} data nodes", size(), n + numShareCoordOnlyNodes, n);
+        logger.info("changing cluster size from {} to {}, {} data nodes", size(), n + numSharedCoordOnlyNodes, n);
         Set<NodeAndClient> nodesToRemove = new HashSet<>();
         int numNodesAndClients = 0;
         while (values.hasNext() && numNodesAndClients++ < size - n) {
@@ -557,33 +588,59 @@ public final class InternalTestCluster extends TestCluster {
 
     private NodeAndClient buildNode(Settings settings, Version version) {
         int ord = nextNodeId.getAndIncrement();
-        return buildNode(ord, random.nextLong(), settings, version);
+        return buildNode(ord, random.nextLong(), settings, version, false);
     }
 
     private NodeAndClient buildNode() {
         int ord = nextNodeId.getAndIncrement();
-        return buildNode(ord, random.nextLong(), null, Version.CURRENT);
+        return buildNode(ord, random.nextLong(), null, Version.CURRENT, false);
     }
 
-    private NodeAndClient buildNode(int nodeId, long seed, Settings settings, Version version) {
+    private NodeAndClient buildNode(int nodeId, long seed, Settings settings, Version version, boolean reuseExisting) {
         assert Thread.holdsLock(this);
         ensureOpen();
         settings = getSettings(nodeId, seed, settings);
         Collection<Class<? extends Plugin>> plugins = getPlugins();
-        String name = buildNodeName(nodeId);
-        assert !nodes.containsKey(name);
+        String name = buildNodeName(nodeId, settings);
+        if (reuseExisting && nodes.containsKey(name)) {
+            return nodes.get(name);
+        } else {
+            assert reuseExisting == true || nodes.containsKey(name) == false :
+                "node name [" + name + "] already exists but not allowed to use it";
+        }
         Settings finalSettings = Settings.builder()
-                .put(Environment.PATH_HOME_SETTING.getKey(), baseDir) // allow overriding path.home
-                .put(settings)
-                .put("node.name", name)
-                .put(DiscoveryNodeService.NODE_ID_SEED_SETTING.getKey(), seed)
-                .build();
+            .put(Environment.PATH_HOME_SETTING.getKey(), baseDir) // allow overriding path.home
+            .put(settings)
+            .put("node.name", name)
+            .put(DiscoveryNodeService.NODE_ID_SEED_SETTING.getKey(), seed)
+            .build();
         MockNode node = new MockNode(finalSettings, version, plugins);
         return new NodeAndClient(name, node);
     }
 
-    private String buildNodeName(int id) {
-        return nodePrefix + id;
+    private String buildNodeName(int id, Settings settings) {
+        String prefix = nodePrefix;
+        prefix = prefix + getRoleSuffix(settings);
+        return prefix + id;
+    }
+
+    /**
+     * returns a suffix string based on the node role. If no explicit role is defined, the suffix will be empty
+     */
+    private String getRoleSuffix(Settings settings) {
+        String suffix = "";
+        if (Node.NODE_MASTER_SETTING.exists(settings) && Node.NODE_MASTER_SETTING.get(settings)) {
+            suffix = suffix + DiscoveryNode.Role.MASTER.getAbbreviation();
+        }
+        if (Node.NODE_DATA_SETTING.exists(settings) && Node.NODE_DATA_SETTING.get(settings)) {
+            suffix = suffix + DiscoveryNode.Role.DATA.getAbbreviation();
+        }
+        if (Node.NODE_MASTER_SETTING.exists(settings) && Node.NODE_MASTER_SETTING.get(settings) == false &&
+            Node.NODE_DATA_SETTING.exists(settings) && Node.NODE_DATA_SETTING.get(settings) == false
+            ) {
+            suffix = suffix + "c";
+        }
+        return suffix;
     }
 
     /**
@@ -873,14 +930,14 @@ public final class InternalTestCluster extends TestCluster {
             TransportAddress addr = node.injector().getInstance(TransportService.class).boundAddress().publishAddress();
             Settings nodeSettings = node.settings();
             Builder builder = Settings.builder()
-                    .put("client.transport.nodes_sampler_interval", "1s")
-                    .put(Environment.PATH_HOME_SETTING.getKey(), baseDir)
-                    .put("node.name", TRANSPORT_CLIENT_PREFIX + node.settings().get("node.name"))
-                    .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), clusterName).put("client.transport.sniff", sniff)
-                    .put(Node.NODE_MODE_SETTING.getKey(), Node.NODE_MODE_SETTING.exists(nodeSettings) ? Node.NODE_MODE_SETTING.get(nodeSettings) : nodeMode)
-                    .put("logger.prefix", nodeSettings.get("logger.prefix", ""))
-                    .put("logger.level", nodeSettings.get("logger.level", "INFO"))
-                    .put(settings);
+                .put("client.transport.nodes_sampler_interval", "1s")
+                .put(Environment.PATH_HOME_SETTING.getKey(), baseDir)
+                .put("node.name", TRANSPORT_CLIENT_PREFIX + node.settings().get("node.name"))
+                .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), clusterName).put("client.transport.sniff", sniff)
+                .put(Node.NODE_MODE_SETTING.getKey(), Node.NODE_MODE_SETTING.exists(nodeSettings) ? Node.NODE_MODE_SETTING.get(nodeSettings) : nodeMode)
+                .put("logger.prefix", nodeSettings.get("logger.prefix", ""))
+                .put("logger.level", nodeSettings.get("logger.level", "INFO"))
+                .put(settings);
 
             if (Node.NODE_LOCAL_SETTING.exists(nodeSettings)) {
                 builder.put(Node.NODE_LOCAL_SETTING.getKey(), Node.NODE_LOCAL_SETTING.get(nodeSettings));
@@ -924,39 +981,35 @@ public final class InternalTestCluster extends TestCluster {
 
 
         Set<NodeAndClient> sharedNodes = new HashSet<>();
-        assert sharedNodesSeeds.length == numSharedAllRolesNodes + numShareCoordOnlyNodes;
-        boolean changed = false;
-        for (int i = 0; i < numSharedAllRolesNodes; i++) {
-            String buildNodeName = buildNodeName(i);
-            NodeAndClient nodeAndClient = nodes.get(buildNodeName);
-            if (nodeAndClient == null) {
-                changed = true;
-                nodeAndClient = buildNode(i, sharedNodesSeeds[i], null, Version.CURRENT);
-                nodeAndClient.node.start();
-                logger.info("Start Shared Node [{}] not shared", nodeAndClient.name);
-            }
+        assert sharedNodesSeeds.length == numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes;
+        for (int i = 0; i < numSharedDedicatedMasterNodes; i++) {
+            final Settings.Builder settings = Settings.builder();
+            settings.put(Node.NODE_MASTER_SETTING.getKey(), true).build();
+            settings.put(Node.NODE_DATA_SETTING.getKey(), false).build();
+            NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), Version.CURRENT, true);
+            nodeAndClient.node().start();
             sharedNodes.add(nodeAndClient);
         }
-        for (int i = numSharedAllRolesNodes; i < numSharedAllRolesNodes + numShareCoordOnlyNodes; i++) {
-            String buildNodeName = buildNodeName(i);
-            NodeAndClient nodeAndClient = nodes.get(buildNodeName);
-            if (nodeAndClient == null) {
-                changed = true;
-                Builder clientSettingsBuilder = Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false)
-                    .put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_INGEST_SETTING.getKey(), false);
-                nodeAndClient = buildNode(i, sharedNodesSeeds[i], clientSettingsBuilder.build(), Version.CURRENT);
-                nodeAndClient.node.start();
-                logger.info("Start Shared Node [{}] not shared", nodeAndClient.name);
+        for (int i = numSharedDedicatedMasterNodes; i < numSharedDedicatedMasterNodes + numSharedDataNodes; i++) {
+            final Settings.Builder settings = Settings.builder();
+            if (numSharedDedicatedMasterNodes > 0) {
+                // if we don't have dedicated master nodes, keep things default
+                settings.put(Node.NODE_MASTER_SETTING.getKey(), false).build();
+                settings.put(Node.NODE_DATA_SETTING.getKey(), true).build();
             }
+            NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), Version.CURRENT, true);
+            nodeAndClient.node().start();
             sharedNodes.add(nodeAndClient);
         }
-        if (!changed && sharedNodes.size() == nodes.size()) {
-            logger.debug("Cluster is consistent - moving out - nodes: [{}] nextNodeId: [{}] numSharedNodes: [{}]", nodes.keySet(), nextNodeId.get(), sharedNodesSeeds.length);
-            if (size() > 0) {
-                client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(sharedNodesSeeds.length)).get();
-            }
-            return; // we are consistent - return
+        for (int i = numSharedDedicatedMasterNodes + numSharedDataNodes;
+             i < numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes; i++) {
+            final Builder settings = Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false)
+                .put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_INGEST_SETTING.getKey(), false);
+            NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), Version.CURRENT, true);
+            nodeAndClient.node().start();
+            sharedNodes.add(nodeAndClient);
         }
+
         for (NodeAndClient nodeAndClient : sharedNodes) {
             nodes.remove(nodeAndClient.name);
         }
@@ -1092,6 +1145,14 @@ public final class InternalTestCluster extends TestCluster {
         return getInstances(clazz, new DataNodePredicate());
     }
 
+    /**
+     * Returns an Iterable to all instances for the given class &gt;T&lt; across all data and master nodes
+     * in the cluster.
+     */
+    public synchronized <T> Iterable<T> getDataOrMasterNodeInstances(Class<T> clazz) {
+        return getInstances(clazz, new DataOrMasterNodePredicate());
+    }
+
     private synchronized <T> Iterable<T> getInstances(Class<T> clazz, Predicate<NodeAndClient> predicate) {
         Iterable<NodeAndClient> filteredNodes = nodes.values().stream().filter(predicate)::iterator;
         List<T> instances = new ArrayList<>();
@@ -1296,7 +1357,19 @@ public final class InternalTestCluster extends TestCluster {
                 }
                 nodeAndClient.closeNode();
             }
-            for (NodeAndClient nodeAndClient : nodes.values()) {
+
+            // starting master nodes first, for now so restart will be quick. If we'll start
+            // the data nodes first, they will wait for 30s for a master
+            List<DiscoveryNode> discoveryNodes = new ArrayList<>();
+            for (ClusterService clusterService : getInstances(ClusterService.class)) {
+                discoveryNodes.add(clusterService.localNode());
+            }
+
+            discoveryNodes.sort((n1, n2) -> Boolean.compare(n1.isMasterNode() == false, n2.isMasterNode() == false));
+
+
+            for (DiscoveryNode node : discoveryNodes) {
+                NodeAndClient nodeAndClient = nodes.get(node.getName());
                 logger.info("Starting node [{}] ", nodeAndClient.name);
                 if (activeDisruptionScheme != null) {
                     activeDisruptionScheme.removeFromNode(nodeAndClient.name, this);
@@ -1375,11 +1448,11 @@ public final class InternalTestCluster extends TestCluster {
     private synchronized Set<String> nRandomDataNodes(int numNodes) {
         assert size() >= numNodes;
         Map<String, NodeAndClient> dataNodes =
-                nodes
-                        .entrySet()
-                        .stream()
-                        .filter(new EntryNodePredicate(new DataNodePredicate()))
-                        .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+            nodes
+                .entrySet()
+                .stream()
+                .filter(new EntryNodePredicate(new DataNodePredicate()))
+                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
         final HashSet<String> set = new HashSet<>();
         final Iterator<String> iterator = dataNodes.keySet().iterator();
         for (int i = 0; i < numNodes; i++) {
@@ -1632,10 +1705,10 @@ public final class InternalTestCluster extends TestCluster {
 
     private synchronized Collection<NodeAndClient> filterNodes(Map<String, InternalTestCluster.NodeAndClient> map, Predicate<NodeAndClient> predicate) {
         return map
-                .values()
-                .stream()
-                .filter(predicate)
-                .collect(Collectors.toCollection(ArrayList::new));
+            .values()
+            .stream()
+            .filter(predicate)
+            .collect(Collectors.toCollection(ArrayList::new));
     }
 
     private static final class DataNodePredicate implements Predicate<NodeAndClient> {
@@ -1649,7 +1722,7 @@ public final class InternalTestCluster extends TestCluster {
         @Override
         public boolean test(NodeAndClient nodeAndClient) {
             return DiscoveryNode.isDataNode(nodeAndClient.node.settings()) ||
-                    DiscoveryNode.isMasterNode(nodeAndClient.node.settings());
+                DiscoveryNode.isMasterNode(nodeAndClient.node.settings());
         }
     }
 
@@ -1887,6 +1960,7 @@ public final class InternalTestCluster extends TestCluster {
 
     /**
      * Simple interface that allows to wait for an async operation to finish
+     *
      * @param <T> the result of the async execution
      */
     public interface Async<T> {
diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java
index 2c7e35fa4ea..ae774220ccf 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java
@@ -49,6 +49,7 @@ public class InternalTestClusterTests extends ESTestCase {
 
     public void testInitializiationIsConsistent() {
         long clusterSeed = randomLong();
+        boolean masterNodes = randomBoolean();
         int minNumDataNodes = randomIntBetween(0, 9);
         int maxNumDataNodes = randomIntBetween(minNumDataNodes, 10);
         String clusterName = randomRealisticUnicodeOfCodepointLengthBetween(1, 10);
@@ -58,8 +59,12 @@ public class InternalTestClusterTests extends ESTestCase {
         String nodePrefix = randomRealisticUnicodeOfCodepointLengthBetween(1, 10);
 
         Path baseDir = createTempDir();
-        InternalTestCluster cluster0 = new InternalTestCluster("local", clusterSeed, baseDir, minNumDataNodes, maxNumDataNodes, clusterName, nodeConfigurationSource, numClientNodes, enableHttpPipelining, nodePrefix, Collections.emptyList(), Function.identity());
-        InternalTestCluster cluster1 = new InternalTestCluster("local", clusterSeed, baseDir, minNumDataNodes, maxNumDataNodes, clusterName, nodeConfigurationSource, numClientNodes, enableHttpPipelining, nodePrefix, Collections.emptyList(), Function.identity());
+        InternalTestCluster cluster0 = new InternalTestCluster("local", clusterSeed, baseDir, masterNodes,
+            minNumDataNodes, maxNumDataNodes, clusterName, nodeConfigurationSource, numClientNodes,
+            enableHttpPipelining, nodePrefix, Collections.emptyList(), Function.identity());
+        InternalTestCluster cluster1 = new InternalTestCluster("local", clusterSeed, baseDir, masterNodes,
+            minNumDataNodes, maxNumDataNodes, clusterName, nodeConfigurationSource, numClientNodes,
+            enableHttpPipelining, nodePrefix, Collections.emptyList(), Function.identity());
         // TODO: this is not ideal - we should have a way to make sure ports are initialized in the same way
         assertClusters(cluster0, cluster1, false);
 
@@ -101,6 +106,7 @@ public class InternalTestClusterTests extends ESTestCase {
 
     public void testBeforeTest() throws Exception {
         long clusterSeed = randomLong();
+        boolean masterNodes = randomBoolean();
         int minNumDataNodes = randomIntBetween(0, 3);
         int maxNumDataNodes = randomIntBetween(minNumDataNodes, 4);
         final String clusterName1 = "shared1";//clusterName("shared1", clusterSeed);
@@ -115,8 +121,12 @@ public class InternalTestClusterTests extends ESTestCase {
         String nodePrefix = "foobar";
 
         Path baseDir = createTempDir();
-        InternalTestCluster cluster0 = new InternalTestCluster("local", clusterSeed, baseDir, minNumDataNodes, maxNumDataNodes, clusterName1, nodeConfigurationSource, numClientNodes, enableHttpPipelining, nodePrefix, Collections.emptyList(), Function.identity());
-        InternalTestCluster cluster1 = new InternalTestCluster("local", clusterSeed, baseDir, minNumDataNodes, maxNumDataNodes, clusterName2, nodeConfigurationSource, numClientNodes, enableHttpPipelining, nodePrefix, Collections.emptyList(), Function.identity());
+        InternalTestCluster cluster0 = new InternalTestCluster("local", clusterSeed, baseDir, masterNodes,
+            minNumDataNodes, maxNumDataNodes, clusterName1, nodeConfigurationSource, numClientNodes,
+            enableHttpPipelining, nodePrefix, Collections.emptyList(), Function.identity());
+        InternalTestCluster cluster1 = new InternalTestCluster("local", clusterSeed, baseDir, masterNodes,
+            minNumDataNodes, maxNumDataNodes, clusterName2, nodeConfigurationSource, numClientNodes,
+            enableHttpPipelining, nodePrefix, Collections.emptyList(), Function.identity());
 
         assertClusters(cluster0, cluster1, false);
         long seed = randomLong();

From a445654123b783469b0a8afd90f669dfcd8e9204 Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Fri, 27 May 2016 10:13:16 +0200
Subject: [PATCH 0488/1311] Fix after review

* changes `throttle_retries` to `use_throttle_retries`
* removes registering of all individual repository settings when the plugin starts. Not needed
* adds more comment about deprecated method in AWS SDK we need to implement though in a Delegate class within our tests
---
 docs/plugins/repository-s3.asciidoc           |  5 +++--
 plugins/repository-s3/build.gradle            |  4 +++-
 .../elasticsearch/cloud/aws/AwsS3Service.java |  2 +-
 .../cloud/aws/InternalAwsS3Service.java       |  8 ++++----
 .../repository/s3/S3RepositoryPlugin.java     | 19 +------------------
 .../repositories/s3/S3Repository.java         | 16 ++++++++--------
 .../cloud/aws/TestAwsS3Service.java           |  4 ++--
 7 files changed, 22 insertions(+), 36 deletions(-)

diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc
index b4b34ee6059..fc07e6ae65a 100644
--- a/docs/plugins/repository-s3.asciidoc
+++ b/docs/plugins/repository-s3.asciidoc
@@ -218,7 +218,7 @@ The following settings are supported:
 
     Number of retries in case of S3 errors. Defaults to `3`.
 
-`throttle_retries`::
+`use_throttle_retries`::
 
     Set to `true` if you want to throttle retries. Defaults to AWS SDK default value (`false`).
 
@@ -243,7 +243,8 @@ The following settings are supported:
     different classes, see http://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html[AWS Storage Classes Guide]
 
 Note that you can define S3 repository settings for all S3 repositories in `elasticsearch.yml` configuration file.
-They are all prefixed with `repositories.s3.`.
+They are all prefixed with `repositories.s3.`. For example, you can define compression for all S3 repositories
+by setting `repositories.s3.compress: true` in `elasticsearch.yml`.
 
 The S3 repositories use the same credentials as the rest of the AWS services
 provided by this plugin (`discovery`). See <<repository-s3-usage>> for details.
diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle
index d01ebd29c12..8ff92a0a20b 100644
--- a/plugins/repository-s3/build.gradle
+++ b/plugins/repository-s3/build.gradle
@@ -57,5 +57,7 @@ thirdPartyAudit.excludes = [
   'org.apache.log.Logger',
 ]
 
-// AWS SDK is exposing some deprecated methods which we call using a delegate
+// AWS SDK is exposing some deprecated methods which we call using a delegate:
+// * setObjectRedirectLocation(String bucketName, String key, String newRedirectLocation)
+// * changeObjectStorageClass(String bucketName, String key, StorageClass newStorageClass)
 compileTestJava.options.compilerArgs << "-Xlint:-deprecation"
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
index 59a1dbff45f..17e5de37b7c 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
@@ -154,5 +154,5 @@ public interface AwsS3Service extends LifecycleComponent<AwsS3Service> {
     }
 
     AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries,
-                    boolean throttleRetries);
+                    boolean useThrottleRetries);
 }
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
index 887e8ba61d9..352aa196b06 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
@@ -58,7 +58,7 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
 
     @Override
     public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries,
-                                        boolean throttleRetries) {
+                                        boolean useThrottleRetries) {
         if (Strings.isNullOrEmpty(endpoint)) {
             // We need to set the endpoint based on the region
             if (region != null) {
@@ -70,11 +70,11 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
             }
         }
 
-        return getClient(endpoint, protocol, account, key, maxRetries, throttleRetries);
+        return getClient(endpoint, protocol, account, key, maxRetries, useThrottleRetries);
     }
 
     private synchronized AmazonS3 getClient(String endpoint, Protocol protocol, String account, String key, Integer maxRetries,
-                                            boolean throttleRetries) {
+                                            boolean useThrottleRetries) {
         Tuple<String, String> clientDescriptor = new Tuple<>(endpoint, account);
         AmazonS3Client client = clients.get(clientDescriptor);
         if (client != null) {
@@ -104,7 +104,7 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
             // If not explicitly set, default to 3 with exponential backoff policy
             clientConfiguration.setMaxErrorRetry(maxRetries);
         }
-        clientConfiguration.setUseThrottleRetries(throttleRetries);
+        clientConfiguration.setUseThrottleRetries(useThrottleRetries);
 
         // #155: we might have 3rd party users using older S3 API version
         String awsSigner = CLOUD_S3.SIGNER_SETTING.get(settings);
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
index 9fd9b0d21c7..7922174f40e 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
@@ -128,24 +128,7 @@ public class S3RepositoryPlugin extends Plugin {
         settingsModule.registerSetting(S3Repository.Repositories.STORAGE_CLASS_SETTING);
         settingsModule.registerSetting(S3Repository.Repositories.CANNED_ACL_SETTING);
         settingsModule.registerSetting(S3Repository.Repositories.BASE_PATH_SETTING);
-        settingsModule.registerSetting(S3Repository.Repositories.THROTTLE_RETRIES_SETTING);
-
-        // Register S3 single repository settings
-        settingsModule.registerSetting(S3Repository.Repository.KEY_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.SECRET_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.BUCKET_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.ENDPOINT_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.PROTOCOL_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.REGION_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.SERVER_SIDE_ENCRYPTION_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.BUFFER_SIZE_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.MAX_RETRIES_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.CHUNK_SIZE_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.COMPRESS_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.STORAGE_CLASS_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.CANNED_ACL_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.BASE_PATH_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.THROTTLE_RETRIES_SETTING);
+        settingsModule.registerSetting(S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING);
     }
 
     /**
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
index c959f6e53fe..9954c2006bb 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
@@ -117,9 +117,9 @@ public class S3Repository extends BlobStoreRepository {
          */
         Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("repositories.s3.max_retries", 3, Property.NodeScope);
         /**
-         * repositories.s3.throttle_retries: Set to `true` if you want to throttle retries. Defaults to AWS SDK default value (`false`).
+         * repositories.s3.use_throttle_retries: Set to `true` if you want to throttle retries. Defaults to AWS SDK default value (`false`).
          */
-        Setting<Boolean> THROTTLE_RETRIES_SETTING = Setting.boolSetting("repositories.s3.throttle_retries",
+        Setting<Boolean> USE_THROTTLE_RETRIES_SETTING = Setting.boolSetting("repositories.s3.use_throttle_retries",
             ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope);
         /**
          * repositories.s3.chunk_size: Big files can be broken down into chunks during snapshotting if needed. Defaults to 1g.
@@ -203,10 +203,10 @@ public class S3Repository extends BlobStoreRepository {
          */
         Setting<Integer> MAX_RETRIES_SETTING = Setting.intSetting("max_retries", 3, Property.NodeScope);
         /**
-         * throttle_retries
-         * @see  Repositories#THROTTLE_RETRIES_SETTING
+         * use_throttle_retries
+         * @see  Repositories#USE_THROTTLE_RETRIES_SETTING
          */
-        Setting<Boolean> THROTTLE_RETRIES_SETTING = Setting.boolSetting("throttle_retries",
+        Setting<Boolean> USE_THROTTLE_RETRIES_SETTING = Setting.boolSetting("use_throttle_retries",
             ClientConfiguration.DEFAULT_THROTTLE_RETRIES, Property.NodeScope);
         /**
          * chunk_size
@@ -274,7 +274,7 @@ public class S3Repository extends BlobStoreRepository {
         boolean serverSideEncryption = getValue(repositorySettings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING);
         ByteSizeValue bufferSize = getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING);
         Integer maxRetries = getValue(repositorySettings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING);
-        boolean throttleRetries = getValue(repositorySettings, Repository.THROTTLE_RETRIES_SETTING, Repositories.THROTTLE_RETRIES_SETTING);
+        boolean useThrottleRetries = getValue(repositorySettings, Repository.USE_THROTTLE_RETRIES_SETTING, Repositories.USE_THROTTLE_RETRIES_SETTING);
         this.chunkSize = getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING);
         this.compress = getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING);
 
@@ -290,13 +290,13 @@ public class S3Repository extends BlobStoreRepository {
 
         logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], " +
             "buffer_size [{}], max_retries [{}], throttle_retries [{}], cannedACL [{}], storageClass [{}]",
-            bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, throttleRetries, cannedACL,
+            bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, useThrottleRetries, cannedACL,
             storageClass);
 
         String key = getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING);
         String secret = getValue(repositorySettings, Repository.SECRET_SETTING, Repositories.SECRET_SETTING);
 
-        blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, key, secret, maxRetries, throttleRetries),
+        blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, key, secret, maxRetries, useThrottleRetries),
                 bucket, region, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
 
         String basePath = getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING);
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
index 1a3ce418c0b..d8ddd3e7162 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
@@ -52,8 +52,8 @@ public class TestAwsS3Service extends InternalAwsS3Service {
 
     @Override
     public synchronized AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key, Integer maxRetries,
-                                        boolean throttleRetries) {
-        return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries, throttleRetries));
+                                        boolean useThrottleRetries) {
+        return cachedWrapper(super.client(endpoint, protocol, region, account, key, maxRetries, useThrottleRetries));
     }
 
     private AmazonS3 cachedWrapper(AmazonS3 client) {

From 82713bab6da38dfb2878786b85070485ce3998a7 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 27 May 2016 06:03:35 -0400
Subject: [PATCH 0489/1311] Add bootstrap check docs

This commit adds documentation for the bootstrap checks and provides
either links or inline guidance for setting the necessary settings to
pass the bootstrap checks.

Relates #18605
---
 docs/reference/setup.asciidoc                 |   2 +
 .../reference/setup/bootstrap-checks.asciidoc | 147 ++++++++++++++++++
 2 files changed, 149 insertions(+)
 create mode 100644 docs/reference/setup/bootstrap-checks.asciidoc

diff --git a/docs/reference/setup.asciidoc b/docs/reference/setup.asciidoc
index 80d2a71fef0..a883b0cc543 100644
--- a/docs/reference/setup.asciidoc
+++ b/docs/reference/setup.asciidoc
@@ -42,6 +42,8 @@ include::setup/configuration.asciidoc[]
 
 include::setup/important-settings.asciidoc[]
 
+include::setup/bootstrap-checks.asciidoc[]
+
 include::setup/sysconfig.asciidoc[]
 
 include::setup/upgrade.asciidoc[]
diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc
new file mode 100644
index 00000000000..dd1d2e63342
--- /dev/null
+++ b/docs/reference/setup/bootstrap-checks.asciidoc
@@ -0,0 +1,147 @@
+[[bootstrap-checks]] == Bootstrap Checks
+
+Collectively, we have a lot of experience with users suffering
+unexpected issues because they have not configured
+<<important-setting,important settings>>. In previous versions of
+Elasticsearch, misconfiguration of some of these settings were logged
+as warnings. Understandably, users sometimes miss these log messages.
+To ensure that these settings receive the attention that they deserve,
+Elasticsearch has bootstrap checks upon startup.
+
+These bootstrap checks inspect a variety of Elasticsearch and system
+settings and compare them to values that are safe for the operation of
+Elasticsearch. If Elasticsearch is in development mode, any bootstrap
+checks that fail appear as warnings in the Elasticsearch log. If
+Elasticsearch is in production mode, any bootstrap checks that fail will
+cause Elasticsearch to refuse to start.
+
+=== Development vs. production mode
+
+By default, Elasticsearch binds and publishes to `localhost`. This is
+fine for downloading and playing with Elasticsearch, and everyday
+development but it's useless for production systems. For a production
+installation to be reachable, it must either bind or publish to an
+external interface. Thus, we consider Elasticsearch to be in development
+mode if it does not bind nor publish to an external interface (the
+default), and is otherwise in production mode if it does bind or publish
+to an external interface.
+
+=== Heap size check
+
+If a JVM is started with unequal initial and max heap size, it can be
+prone to pauses as the JVM heap is resized during system usage. To avoid
+these resize pauses, it's best to start the JVM with the initial heap
+size equal to the maximum heap size. Additionally, if
+<<bootstrap.mlockall>> is enabled, the JVM will lock the initial size of
+the heap on startup. If the initial heap size is not equal to the
+maximum heap size, after a resize it will not be the case that all of
+the JVM heap is locked in memory. To pass the heap size check, you must
+configure the [[heap-size,heap size]].
+
+=== File descriptor check
+
+File descriptors are a Unix construct for tracking open "files". In Unix
+though, [everything is a
+file](https://en.wikipedia.org/wiki/Everything_is_a_file). For example,
+"files" could be a physical file, a virtual file (e.g.,
+``/proc/loadavg`), or network sockets. Elasticsearch requires lots file
+descriptors (e.g., every shard is composed of multiple segments and
+other files, plus connections to other nodes, etc.). This bootstrap
+check is enforced on OS X and Linux. To pass the file descriptor check,
+you might have to configure [[file-descriptors,file descriptors]].
+
+=== Memory lock check
+
+When the JVM does a major garbage collection it touches every page of
+the heap. If any of those pages are swapped out to disk they will have
+to be swapped back in to memory. That causes lots of disk thrashing that
+Elasticsearch would much rather use to service requests. There are
+several ways to configure a system to disallow swapping. One way is by
+requesting the JVM to lock the heap in memory through `mlockall` (Unix)
+or virtual lock (Windows). This is done via the Elasticsearch setting
+<<bootstrap.mlockall,`bootstrap.mlockall`>>. However, there are cases
+where this setting can be passed to Elasticsearch but Elasticsearch is
+not able to lock the heap (e.g., if the `elasticsearch` user does not
+have `memlock unlimited`). The memory lock check verifies that *if* the
+`bootstrap.mlockall` setting is enabled, that the JVM was successfully
+able to lock the heap. To pass the memory lock check, you might have to
+configure <<mlockall,`mlockall`>>.
+
+=== Minimum master nodes check
+
+Elasticsearch uses a single master for managing cluster state but
+enables there to be multiple master-eligible nodes for
+high-availability. In the case of a partition, master-eligible nodes on
+each side of the partition might be elected as the acting master without
+knowing that there is a master on the side of the partition. This can
+lead to divergent cluster states potentially leading to data loss when
+the partition is healed. This is the notion of a split brain and it is
+the worst thing that can happen to an Elasticsearch cluster. But by
+configuring <<minimum_master_nodes>> to be equal to a quorum of
+master-eligible nodes, it is not possible for the cluster to suffer from
+split brain because during a network partition there can be at most one
+side of the partition that contains a quorum of master nodes. The
+minimum master nodes check enforces that you've set
+<<minimum_master_nodes>>. To pass the minimum master nodes check, you
+must configure <<minimum_master_nodes>>.
+
+NOTE: The minimum master nodes check does not enforce that you've
+configured <<minimum_master_nodes>> correctly, only that you have it
+configured. Elasticsearch does log a warning message if it detects that
+<<minimum_master_nodes>> is incorrectly configured based on the number
+of master-eligible nodes visible in the cluster state. Future versions
+of Elasticsearch will contain stricter enforcement of
+<<minimum_master_nodes>>.
+
+=== Maximum number of threads check
+
+Elasticsearch executes requests by breaking the request down into stages
+and handing those stages off to different thread pool executors. There
+are different <<modules-threadpool,thread pool executors>> for a variety
+of tasks within Elasticsearch. Thus, Elasticsearch needs the ability to
+create a lot of threads. The maximum number of threads check ensures
+that the Elasticsearch process has the rights to create enough threads
+under normal use. This check is enforced only on Linux. If you are on
+Linux, to pass the maximum number of threads check, you must configure
+your system to allow the Elasticsearch process the ability to create at
+least 2048 threads. This can be done via `/etc/security/limits.conf`
+using the `nproc` setting (note that you might have to increase the
+limits for the `root` user too).
+
+[[max-size-virtual-memory-check]]
+=== Maximum size virtual memory check
+
+Elasticsearch and Lucene use `mmap` to great effect to map portions of
+an index into the Elasticsearch address space. This keeps certain index
+data off the JVM heap but in memory for blazing fast access. For this to
+be effective, the Elasticsearch should have unlimited address space. The
+maximum size virtual memory check enforces that the Elasticsearch
+process has unlimited address space and is enforced only on Linux. To
+pass the maximum size virtual memory check, you must configure your
+system to allow the Elasticsearch process the ability to have unlimited
+address space. This can be done via `/etc/security/limits.conf` using
+the `as` setting to `unlimited` (note that you might have to increaes
+the limits for the `root` user too).
+
+=== Maximum map count check
+
+Continuing from the previous <<max-size-virtual-memory-check,point>>, to
+use `mmap` effectively, Elasticsearch also requires the ability to
+create many memory-mapped areas. The maximum map count check checks that
+the kernel allows a process to have at least 262,144 memory-mapped areas
+and is enforced on Linux only. To pass the maximum map count check, you
+must configure `vm.max_map_count` via `sysctl` to be at least `262144`.
+
+=== Client JVM check
+
+There are two different JVMs provided by OpenJDK-derived JVMs: the
+client JVM and the server JVM. These JVMs use different compilers for
+producing executable machine code from Java bytecode. The client JVM is
+tuned for startup time and memory footprint while the server JVM is
+tuned for maximizing performance. The difference in performance between
+the two VMs can be substantial. The client JVM check ensures that
+Elasticsearch is not running inside the client JVM. To pass the client
+JVM check, you must start Elasticsearch with the server VM. On modern
+systems and operating systems, the server VM is the
+default. Additionally, Elasticsearch is configured by default to force
+the server VM.

From cebbf0de4181b8d49887ec6391b27d7896a30e90 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 27 May 2016 06:04:11 -0400
Subject: [PATCH 0490/1311] Do not replay into translog on local recovery

When performing a local recovery, the engine replays operations
recovered from the translog into the translog again. These duplicated
operations are eventually cleared after successful recovery on flush,
but there is no need to play these operations into the translog at
all. This commit modifies the local recovery process so as to not replay
these operations into the translog.

Relates #18547
---
 .../elasticsearch/index/engine/Engine.java    | 32 +++++++++++++++--
 .../index/engine/InternalEngine.java          | 24 ++++++++-----
 .../elasticsearch/index/shard/IndexShard.java |  2 +-
 .../index/shard/InternalIndexingStats.java    | 12 +++----
 .../shard/TranslogRecoveryPerformer.java      | 12 +++----
 .../index/translog/Translog.java              |  2 +-
 .../index/translog/TranslogSnapshot.java      |  6 ++--
 .../indices/IndexingMemoryController.java     |  6 +---
 .../index/engine/InternalEngineTests.java     | 36 ++++++++++++++++++-
 9 files changed, 98 insertions(+), 34 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/engine/Engine.java b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
index d5fedcf0343..eeda4b468d1 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -773,10 +773,15 @@ public abstract class Engine implements Closeable {
             this.startTime = startTime;
         }
 
-        public static enum Origin {
+        public enum Origin {
             PRIMARY,
             REPLICA,
-            RECOVERY
+            PEER_RECOVERY,
+            LOCAL_TRANSLOG_RECOVERY;
+
+            public boolean isRecovery() {
+                return this == PEER_RECOVERY || this == LOCAL_TRANSLOG_RECOVERY;
+            }
         }
 
         public Origin origin() {
@@ -803,6 +808,16 @@ public abstract class Engine implements Closeable {
             return this.location;
         }
 
+        public int sizeInBytes() {
+            if (location != null) {
+                return location.size;
+            } else {
+                return estimatedSizeInBytes();
+            }
+        }
+
+        protected abstract int estimatedSizeInBytes();
+
         public VersionType versionType() {
             return this.versionType;
         }
@@ -884,9 +899,16 @@ public abstract class Engine implements Closeable {
         public BytesReference source() {
             return this.doc.source();
         }
+
+        @Override
+        protected int estimatedSizeInBytes() {
+            return (id().length() + type().length()) * 2 + source().length() + 12;
+        }
+
     }
 
     public static class Delete extends Operation {
+
         private final String type;
         private final String id;
         private boolean found;
@@ -922,6 +944,12 @@ public abstract class Engine implements Closeable {
         public boolean found() {
             return this.found;
         }
+
+        @Override
+        protected int estimatedSizeInBytes() {
+            return (uid().field().length() + uid().text().length()) * 2 + 20;
+        }
+
     }
 
     public static class Get {
diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index 7489b3b7a93..41363bdace0 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -336,7 +336,7 @@ public class InternalEngine extends Engine {
         final boolean created;
         try (ReleasableLock lock = readLock.acquire()) {
             ensureOpen();
-            if (index.origin() == Operation.Origin.RECOVERY) {
+            if (index.origin().isRecovery()) {
                 // Don't throttle recovery operations
                 created = innerIndex(index);
             } else {
@@ -371,7 +371,7 @@ public class InternalEngine extends Engine {
 
             long expectedVersion = index.version();
             if (isVersionConflictForWrites(index, currentVersion, deleted, expectedVersion)) {
-                if (index.origin() != Operation.Origin.RECOVERY) {
+                if (!index.origin().isRecovery()) {
                     throw new VersionConflictEngineException(shardId, index.type(), index.id(),
                         index.versionType().explainConflictForWrites(currentVersion, expectedVersion, deleted));
                 }
@@ -389,10 +389,13 @@ public class InternalEngine extends Engine {
             } else {
                 created = update(index, versionValue, indexWriter);
             }
-            Translog.Location translogLocation = translog.add(new Translog.Index(index));
 
-            versionMap.putUnderLock(index.uid().bytes(), new VersionValue(updatedVersion, translogLocation));
-            index.setTranslogLocation(translogLocation);
+            if (index.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) {
+                final Translog.Location translogLocation = translog.add(new Translog.Index(index));
+                index.setTranslogLocation(translogLocation);
+                versionMap.putUnderLock(index.uid().bytes(), new VersionValue(updatedVersion, index.getTranslogLocation()));
+            }
+
             return created;
         }
     }
@@ -467,7 +470,7 @@ public class InternalEngine extends Engine {
             long updatedVersion;
             long expectedVersion = delete.version();
             if (delete.versionType().isVersionConflictForWrites(currentVersion, expectedVersion, deleted)) {
-                if (delete.origin() == Operation.Origin.RECOVERY) {
+                if (delete.origin().isRecovery()) {
                     return;
                 } else {
                     throw new VersionConflictEngineException(shardId, delete.type(), delete.id(),
@@ -489,9 +492,12 @@ public class InternalEngine extends Engine {
             }
 
             delete.updateVersion(updatedVersion, found);
-            Translog.Location translogLocation = translog.add(new Translog.Delete(delete));
-            versionMap.putUnderLock(delete.uid().bytes(), new DeleteVersionValue(updatedVersion, engineConfig.getThreadPool().estimatedTimeInMillis(), translogLocation));
-            delete.setTranslogLocation(translogLocation);
+
+            if (delete.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) {
+                final Translog.Location translogLocation = translog.add(new Translog.Delete(delete));
+                delete.setTranslogLocation(translogLocation);
+                versionMap.putUnderLock(delete.uid().bytes(), new DeleteVersionValue(updatedVersion, engineConfig.getThreadPool().estimatedTimeInMillis(), delete.getTranslogLocation()));
+            }
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 2cadc1ad24b..28ecf290466 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -1011,7 +1011,7 @@ public class IndexShard extends AbstractIndexShardComponent {
             if (writeAllowedStatesForPrimary.contains(state) == false) {
                 throw new IllegalIndexShardStateException(shardId, state, "operation only allowed when shard state is one of " + writeAllowedStatesForPrimary + ", origin [" + origin + "]");
             }
-        } else if (origin == Engine.Operation.Origin.RECOVERY) {
+        } else if (origin.isRecovery()) {
             if (state != IndexShardState.RECOVERING) {
                 throw new IllegalIndexShardStateException(shardId, state, "operation only allowed when recovering, origin [" + origin + "]");
             }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java b/core/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java
index 1faee2f6510..b85ca7e6a9d 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java
@@ -66,7 +66,7 @@ final class InternalIndexingStats implements IndexingOperationListener {
 
     @Override
     public Engine.Index preIndex(Engine.Index operation) {
-        if (operation.origin() != Engine.Operation.Origin.RECOVERY) {
+        if (!operation.origin().isRecovery()) {
             totalStats.indexCurrent.inc();
             typeStats(operation.type()).indexCurrent.inc();
         }
@@ -75,7 +75,7 @@ final class InternalIndexingStats implements IndexingOperationListener {
 
     @Override
     public void postIndex(Engine.Index index, boolean created) {
-        if (index.origin() != Engine.Operation.Origin.RECOVERY) {
+        if (!index.origin().isRecovery()) {
             long took = index.endTime() - index.startTime();
             totalStats.indexMetric.inc(took);
             totalStats.indexCurrent.dec();
@@ -87,7 +87,7 @@ final class InternalIndexingStats implements IndexingOperationListener {
 
     @Override
     public void postIndex(Engine.Index index, Throwable ex) {
-        if (index.origin() != Engine.Operation.Origin.RECOVERY) {
+        if (!index.origin().isRecovery()) {
             totalStats.indexCurrent.dec();
             typeStats(index.type()).indexCurrent.dec();
             totalStats.indexFailed.inc();
@@ -97,7 +97,7 @@ final class InternalIndexingStats implements IndexingOperationListener {
 
     @Override
     public Engine.Delete preDelete(Engine.Delete delete) {
-        if (delete.origin() != Engine.Operation.Origin.RECOVERY) {
+        if (!delete.origin().isRecovery()) {
             totalStats.deleteCurrent.inc();
             typeStats(delete.type()).deleteCurrent.inc();
         }
@@ -107,7 +107,7 @@ final class InternalIndexingStats implements IndexingOperationListener {
 
     @Override
     public void postDelete(Engine.Delete delete) {
-        if (delete.origin() != Engine.Operation.Origin.RECOVERY) {
+        if (!delete.origin().isRecovery()) {
             long took = delete.endTime() - delete.startTime();
             totalStats.deleteMetric.inc(took);
             totalStats.deleteCurrent.dec();
@@ -119,7 +119,7 @@ final class InternalIndexingStats implements IndexingOperationListener {
 
     @Override
     public void postDelete(Engine.Delete delete, Throwable ex) {
-        if (delete.origin() != Engine.Operation.Origin.RECOVERY) {
+        if (!delete.origin().isRecovery()) {
             totalStats.deleteCurrent.dec();
             typeStats(delete.type()).deleteCurrent.dec();
         }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java
index f3903fb6f9d..54cfa80926c 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java
@@ -68,7 +68,7 @@ public class TranslogRecoveryPerformer {
         int numOps = 0;
         try {
             for (Translog.Operation operation : operations) {
-                performRecoveryOperation(engine, operation, false);
+                performRecoveryOperation(engine, operation, false, Engine.Operation.Origin.PEER_RECOVERY);
                 numOps++;
             }
             engine.getTranslog().sync();
@@ -83,7 +83,7 @@ public class TranslogRecoveryPerformer {
         int opsRecovered = 0;
         while ((operation = snapshot.next()) != null) {
             try {
-                performRecoveryOperation(engine, operation, true);
+                performRecoveryOperation(engine, operation, true, Engine.Operation.Origin.LOCAL_TRANSLOG_RECOVERY);
                 opsRecovered++;
             } catch (ElasticsearchException e) {
                 if (e.status() == RestStatus.BAD_REQUEST) {
@@ -146,14 +146,14 @@ public class TranslogRecoveryPerformer {
      *                            cause a {@link MapperException} to be thrown if an update
      *                            is encountered.
      */
-    public void performRecoveryOperation(Engine engine, Translog.Operation operation, boolean allowMappingUpdates) {
+    private void performRecoveryOperation(Engine engine, Translog.Operation operation, boolean allowMappingUpdates, Engine.Operation.Origin origin) {
         try {
             switch (operation.opType()) {
                 case INDEX:
                     Translog.Index index = (Translog.Index) operation;
                     Engine.Index engineIndex = IndexShard.prepareIndex(docMapper(index.type()), source(shardId.getIndexName(), index.type(), index.id(), index.source())
-                                    .routing(index.routing()).parent(index.parent()).timestamp(index.timestamp()).ttl(index.ttl()),
-                            index.version(), index.versionType().versionTypeForReplicationAndRecovery(), Engine.Operation.Origin.RECOVERY);
+                            .routing(index.routing()).parent(index.parent()).timestamp(index.timestamp()).ttl(index.ttl()),
+                        index.version(), index.versionType().versionTypeForReplicationAndRecovery(), origin);
                     maybeAddMappingUpdate(engineIndex.type(), engineIndex.parsedDoc().dynamicMappingsUpdate(), engineIndex.id(), allowMappingUpdates);
                     if (logger.isTraceEnabled()) {
                         logger.trace("[translog] recover [index] op of [{}][{}]", index.type(), index.id());
@@ -167,7 +167,7 @@ public class TranslogRecoveryPerformer {
                         logger.trace("[translog] recover [delete] op of [{}][{}]", uid.type(), uid.id());
                     }
                     final Engine.Delete engineDelete = new Engine.Delete(uid.type(), uid.id(), delete.uid(), delete.version(),
-                            delete.versionType().versionTypeForReplicationAndRecovery(), Engine.Operation.Origin.RECOVERY, System.nanoTime(), false);
+                        delete.versionType().versionTypeForReplicationAndRecovery(), origin, System.nanoTime(), false);
                     delete(engine, engineDelete);
                     break;
                 default:
diff --git a/core/src/main/java/org/elasticsearch/index/translog/Translog.java b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
index 2946bc9c447..b66c82d4083 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/Translog.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
@@ -928,7 +928,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
                     type.equals(index.type) == false ||
                     versionType != index.versionType ||
                     source.equals(index.source) == false) {
-                return false;
+                    return false;
             }
             if (routing != null ? !routing.equals(index.routing) : index.routing != null) {
                 return false;
diff --git a/core/src/main/java/org/elasticsearch/index/translog/TranslogSnapshot.java b/core/src/main/java/org/elasticsearch/index/translog/TranslogSnapshot.java
index 10f381f8eba..f33ec1bd607 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/TranslogSnapshot.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/TranslogSnapshot.java
@@ -27,6 +27,7 @@ import java.nio.channels.FileChannel;
 import java.nio.file.Path;
 
 public class TranslogSnapshot extends BaseTranslogReader implements Translog.Snapshot {
+
     private final int totalOperations;
     protected final long length;
 
@@ -35,7 +36,6 @@ public class TranslogSnapshot extends BaseTranslogReader implements Translog.Sna
     private int readOperations;
     private BufferedChecksumStreamInput reuse;
 
-
     /**
      * Create a snapshot of translog file channel. The length parameter should be consistent with totalOperations and point
      * at the end of the last operation in this snapshot.
@@ -73,7 +73,6 @@ public class TranslogSnapshot extends BaseTranslogReader implements Translog.Sna
         return op;
     }
 
-
     public long sizeInBytes() {
         return length;
     }
@@ -101,4 +100,5 @@ public class TranslogSnapshot extends BaseTranslogReader implements Translog.Sna
                 ", reusableBuffer=" + reusableBuffer +
                 '}';
     }
-}
\ No newline at end of file
+
+}
diff --git a/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java b/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
index 2649ed81589..27877614300 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
@@ -219,11 +219,7 @@ public class IndexingMemoryController extends AbstractComponent implements Index
     }
 
     private void recordOperationBytes(Engine.Operation op) {
-        Translog.Location loc = op.getTranslogLocation();
-        // This can be null on (harmless) version conflict during recovery:
-        if (loc != null) {
-            bytesWritten(loc.size);
-        }
+        bytesWritten(op.sizeInBytes());
     }
 
     private static final class ShardAndBytesUsed implements Comparable<ShardAndBytesUsed> {
diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 6a84b62fabd..44f0b4a7263 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.index.engine;
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
-
 import org.apache.log4j.AppenderSkeleton;
 import org.apache.log4j.Level;
 import org.apache.log4j.LogManager;
@@ -592,6 +591,41 @@ public class InternalEngineTests extends ESTestCase {
         engine.flush();
     }
 
+    public void testTranslogRecoveryDoesNotReplayIntoTranslog() throws IOException {
+        final int docs = randomIntBetween(1, 32);
+        Engine initialEngine = null;
+        try {
+            initialEngine = engine;
+            for (int i = 0; i < docs; i++) {
+                final String id = Integer.toString(i);
+                final ParsedDocument doc = testParsedDocument(id, id, "test", null, -1, -1, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+                initialEngine.index(new Engine.Index(newUid(id), doc));
+            }
+        } finally {
+            IOUtils.close(initialEngine);
+        }
+
+        Engine recoveringEngine = null;
+        try {
+            final AtomicBoolean flushed = new AtomicBoolean();
+            recoveringEngine = new InternalEngine(copy(engine.config(), EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG)) {
+                @Override
+                public CommitId flush(boolean force, boolean waitIfOngoing) throws EngineException {
+                    assertThat(getTranslog().totalOperations(), equalTo(docs));
+                    final CommitId commitId = super.flush(force, waitIfOngoing);
+                    flushed.set(true);
+                    return commitId;
+                }
+            };
+
+            assertThat(recoveringEngine.getTranslog().totalOperations(), equalTo(docs));
+            recoveringEngine.recoverFromTranslog();
+            assertTrue(flushed.get());
+        } finally {
+            IOUtils.close(recoveringEngine);
+        }
+    }
+
     public void testConcurrentGetAndFlush() throws Exception {
         ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
         engine.index(new Engine.Index(newUid("1"), doc));

From 8c6cbf0305156351a4d56950f96f7ff953d288eb Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Thu, 26 May 2016 16:57:50 +0200
Subject: [PATCH 0491/1311] Update the minimum required version of gradle to
 2.13

It came out with improvements around idea integration and language levels. This will make it possible to have the upcoming java client as a new project compiled against java 7 and have idea working on the right language level.
---
 build.gradle                                                    | 1 -
 .../src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/build.gradle b/build.gradle
index 3f0895a4e75..b52d299ad94 100644
--- a/build.gradle
+++ b/build.gradle
@@ -257,7 +257,6 @@ allprojects {
 
 idea {
   project {
-    languageLevel = org.elasticsearch.gradle.BuildPlugin.minimumJava.toString()
     vcs = 'Git'
   }
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index c4eb76164fe..bde51ba0d43 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -113,7 +113,7 @@ class BuildPlugin implements Plugin<Project> {
             }
 
             // enforce gradle version
-            GradleVersion minGradle = GradleVersion.version('2.8')
+            GradleVersion minGradle = GradleVersion.version('2.13')
             if (GradleVersion.current() < minGradle) {
                 throw new GradleException("${minGradle} or above is required to build elasticsearch")
             }

From 0e563b3fab3c5867183996beeb1456399d579fc5 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Thu, 26 May 2016 17:02:56 +0200
Subject: [PATCH 0492/1311] Don't assume that we always compile for java 1.8 or
 1.9

This change makes it possible to compile a separate project with e.g. targetCompatibility 1.7. Adds specific options (compact profile) only when compiling for >= 1.8.
---
 .../org/elasticsearch/gradle/BuildPlugin.groovy      | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index bde51ba0d43..8d77e7a9a34 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -372,6 +372,13 @@ class BuildPlugin implements Plugin<Project> {
                 options.fork = true
                 options.forkOptions.executable = new File(project.javaHome, 'bin/javac')
                 options.forkOptions.memoryMaximumSize = "1g"
+                if (project.targetCompatibility >= JavaVersion.VERSION_1_8) {
+                    // compile with compact 3 profile by default
+                    // NOTE: this is just a compile time check: does not replace testing with a compact3 JRE
+                    if (project.compactProfile != 'full') {
+                        options.compilerArgs << '-profile' << project.compactProfile
+                    }
+                }
                 /*
                  * -path because gradle will send in paths that don't always exist.
                  * -missing because we have tons of missing @returns and @param.
@@ -379,11 +386,6 @@ class BuildPlugin implements Plugin<Project> {
                  */
                 // don't even think about passing args with -J-xxx, oracle will ask you to submit a bug report :)
                 options.compilerArgs << '-Werror' << '-Xlint:all,-path,-serial,-options,-deprecation' << '-Xdoclint:all' << '-Xdoclint:-missing'
-                // compile with compact 3 profile by default
-                // NOTE: this is just a compile time check: does not replace testing with a compact3 JRE
-                if (project.compactProfile != 'full') {
-                    options.compilerArgs << '-profile' << project.compactProfile
-                }
                 options.encoding = 'UTF-8'
                 //options.incremental = true
 

From 63622aa6b650ef7af684fe905208913985a92192 Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Fri, 27 May 2016 12:48:05 +0200
Subject: [PATCH 0493/1311] Fix log use_throttle_retries

---
 .../java/org/elasticsearch/repositories/s3/S3Repository.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
index 9954c2006bb..fe184a06aef 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
@@ -289,7 +289,7 @@ public class S3Repository extends BlobStoreRepository {
         String cannedACL = getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING);
 
         logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], " +
-            "buffer_size [{}], max_retries [{}], throttle_retries [{}], cannedACL [{}], storageClass [{}]",
+            "buffer_size [{}], max_retries [{}], use_throttle_retries [{}], cannedACL [{}], storageClass [{}]",
             bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, useThrottleRetries, cannedACL,
             storageClass);
 

From 0b965167fd43a807d9dcaa43e876a5fac0ccf0cb Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 27 May 2016 13:16:43 +0200
Subject: [PATCH 0494/1311] Revert "Update the minimum required version of
 gradle to 2.13"

This reverts commit 8c6cbf0305156351a4d56950f96f7ff953d288eb. We need to update all our CI machines first, then get this change back in.
---
 build.gradle                                                    | 1 +
 .../src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index b52d299ad94..3f0895a4e75 100644
--- a/build.gradle
+++ b/build.gradle
@@ -257,6 +257,7 @@ allprojects {
 
 idea {
   project {
+    languageLevel = org.elasticsearch.gradle.BuildPlugin.minimumJava.toString()
     vcs = 'Git'
   }
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index 8d77e7a9a34..256aa70fdca 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -113,7 +113,7 @@ class BuildPlugin implements Plugin<Project> {
             }
 
             // enforce gradle version
-            GradleVersion minGradle = GradleVersion.version('2.13')
+            GradleVersion minGradle = GradleVersion.version('2.8')
             if (GradleVersion.current() < minGradle) {
                 throw new GradleException("${minGradle} or above is required to build elasticsearch")
             }

From 0e9f3addd294bc5dd36674fd597f3bb5f8c8206c Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Tue, 24 May 2016 22:47:28 +0200
Subject: [PATCH 0495/1311] Nested inner hits shouldn't use relative paths

Like on other places in the query dsl the full field name should be used.
Before this change this wasn't the case for nested inner hits when source filtering was used.
Highlighting has a workaround, which is now removed as the source of nested inner hits can only be refered by the full name.

Closes #16653
---
 .../search/fetch/FetchPhase.java              | 36 +++++++++++--------
 .../search/fetch/FetchSubPhase.java           | 19 ----------
 .../search/highlight/HighlightUtils.java      |  2 +-
 .../SourceScoreOrderFragmentsBuilder.java     |  2 +-
 .../SourceSimpleFragmentsBuilder.java         |  2 +-
 .../aggregations/metrics/TopHitsIT.java       |  5 +--
 .../search/innerhits/InnerHitsIT.java         | 31 ++++++++++++++++
 .../migration/migrate_5_0/search.asciidoc     |  3 ++
 8 files changed, 62 insertions(+), 38 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
index 32326aa1579..1b08df76743 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
@@ -239,14 +239,18 @@ public class FetchPhase implements SearchPhase {
             Tuple<XContentType, Map<String, Object>> tuple = XContentHelper.convertToMap(source, true);
             Map<String, Object> sourceAsMap = tuple.v2();
 
-            List<Map<String, Object>> nestedParsedSource;
-            SearchHit.NestedIdentity nested = nestedIdentity;
-            do {
-                Object extractedValue = XContentMapValues.extractValue(nested.getField().string(), sourceAsMap);
-                if (extractedValue == null) {
-                    // The nested objects may not exist in the _source, because it was filtered because of _source filtering
-                    break;
-                } else if (extractedValue instanceof List) {
+            // Isolate the nested json array object that matches with nested hit and wrap it back into the same json
+            // structure with the nested json array object being the actual content. The latter is important, so that
+            // features like source filtering and highlighting work consistent regardless of whether the field points
+            // to a json object array for consistency reasons on how we refer to fields
+            Map<String, Object> nestedSourceAsMap = new HashMap<>();
+            Map<String, Object> current = nestedSourceAsMap;
+            for (SearchHit.NestedIdentity nested = nestedIdentity; nested != null; nested = nested.getChild()) {
+                String nestedPath = nested.getField().string();
+                current.put(nestedPath, new HashMap<>());
+                Object extractedValue = XContentMapValues.extractValue(nestedPath, sourceAsMap);
+                List<Map<String, Object>> nestedParsedSource;
+                if (extractedValue instanceof List) {
                     // nested field has an array value in the _source
                     nestedParsedSource = (List<Map<String, Object>>) extractedValue;
                 } else if (extractedValue instanceof Map) {
@@ -256,18 +260,22 @@ public class FetchPhase implements SearchPhase {
                     throw new IllegalStateException("extracted source isn't an object or an array");
                 }
                 sourceAsMap = nestedParsedSource.get(nested.getOffset());
-                nested = nested.getChild();
-            } while (nested != null);
-
-            context.lookup().source().setSource(sourceAsMap);
+                if (nested.getChild() == null) {
+                    current.put(nestedPath, sourceAsMap);
+                } else {
+                    Map<String, Object> next = new HashMap<>();
+                    current.put(nestedPath, next);
+                    current = next;
+                }
+            }
+            context.lookup().source().setSource(nestedSourceAsMap);
             XContentType contentType = tuple.v1();
             BytesReference nestedSource = contentBuilder(contentType).map(sourceAsMap).bytes();
             context.lookup().source().setSource(nestedSource);
             context.lookup().source().setSourceContentType(contentType);
         }
 
-        InternalSearchHit searchHit = new InternalSearchHit(nestedTopDocId, rootFieldsVisitor.uid().id(), documentMapper.typeText(), nestedIdentity, searchFields);
-        return searchHit;
+        return new InternalSearchHit(nestedTopDocId, rootFieldsVisitor.uid().id(), documentMapper.typeText(), nestedIdentity, searchFields);
     }
 
     private Map<String, SearchHitField> getSearchFields(SearchContext context, int nestedSubDocId, Set<String> fieldNames, List<String> fieldNamePatterns, LeafReaderContext subReaderContext) {
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java
index 58d6ff36a2f..5ba75b84fcc 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java
@@ -81,25 +81,6 @@ public interface FetchSubPhase {
             return cache;
         }
 
-        public String getSourcePath(String sourcePath) {
-            SearchHit.NestedIdentity nested = hit().getNestedIdentity();
-            if (nested != null) {
-                // in case of nested we need to figure out what is the _source field from the perspective
-                // of the nested hit it self. The nested _source is isolated and the root and potentially parent objects
-                // are gone
-                StringBuilder nestedPath = new StringBuilder();
-                for (; nested != null; nested = nested.getChild()) {
-                    nestedPath.append(nested.getField());
-                }
-
-                assert sourcePath.startsWith(nestedPath.toString());
-                int startIndex = nestedPath.length() + 1; // the path until the deepest nested object + '.'
-                return sourcePath.substring(startIndex);
-            } else {
-                return sourcePath;
-            }
-        }
-
     }
 
     default Map<String, ? extends SearchParseElement> parseElements() {
diff --git a/core/src/main/java/org/elasticsearch/search/highlight/HighlightUtils.java b/core/src/main/java/org/elasticsearch/search/highlight/HighlightUtils.java
index d4095c14ec0..16f5b94fdc5 100644
--- a/core/src/main/java/org/elasticsearch/search/highlight/HighlightUtils.java
+++ b/core/src/main/java/org/elasticsearch/search/highlight/HighlightUtils.java
@@ -58,7 +58,7 @@ public final class HighlightUtils {
         } else {
             SourceLookup sourceLookup = searchContext.lookup().source();
             sourceLookup.setSegmentAndDocument(hitContext.readerContext(), hitContext.docId());
-            textsToHighlight = sourceLookup.extractRawValues(hitContext.getSourcePath(mapper.fieldType().name()));
+            textsToHighlight = sourceLookup.extractRawValues(mapper.fieldType().name());
         }
         assert textsToHighlight != null;
         return textsToHighlight;
diff --git a/core/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceScoreOrderFragmentsBuilder.java b/core/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceScoreOrderFragmentsBuilder.java
index 2d226aabf94..8de58f062a7 100644
--- a/core/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceScoreOrderFragmentsBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceScoreOrderFragmentsBuilder.java
@@ -59,7 +59,7 @@ public class SourceScoreOrderFragmentsBuilder extends ScoreOrderFragmentsBuilder
         SourceLookup sourceLookup = searchContext.lookup().source();
         sourceLookup.setSegmentAndDocument((LeafReaderContext) reader.getContext(), docId);
 
-        List<Object> values = sourceLookup.extractRawValues(hitContext.getSourcePath(mapper.fieldType().name()));
+        List<Object> values = sourceLookup.extractRawValues(mapper.fieldType().name());
         Field[] fields = new Field[values.size()];
         for (int i = 0; i < values.size(); i++) {
             fields[i] = new Field(mapper.fieldType().name(), values.get(i).toString(), TextField.TYPE_NOT_STORED);
diff --git a/core/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceSimpleFragmentsBuilder.java b/core/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceSimpleFragmentsBuilder.java
index b80c239376e..b9fbd929f42 100644
--- a/core/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceSimpleFragmentsBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceSimpleFragmentsBuilder.java
@@ -55,7 +55,7 @@ public class SourceSimpleFragmentsBuilder extends SimpleFragmentsBuilder {
         SourceLookup sourceLookup = searchContext.lookup().source();
         sourceLookup.setSegmentAndDocument((LeafReaderContext) reader.getContext(), docId);
 
-        List<Object> values = sourceLookup.extractRawValues(hitContext.getSourcePath(mapper.fieldType().name()));
+        List<Object> values = sourceLookup.extractRawValues(mapper.fieldType().name());
         if (values.isEmpty()) {
             return EMPTY_FIELDS;
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
index 2cd1fed7fbe..84455c231aa 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
@@ -26,6 +26,7 @@ import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptEngine;
@@ -853,7 +854,7 @@ public class TopHitsIT extends ESIntegTestCase {
                         nested("to-comments", "comments").subAggregation(
                                 topHits("top-comments").size(1).highlighter(new HighlightBuilder().field(hlField)).explain(true)
                                                 .fieldDataField("comments.user")
-                                        .scriptField("script", new Script("5", ScriptService.ScriptType.INLINE, MockScriptEngine.NAME, Collections.emptyMap())).fetchSource("message", null)
+                                        .scriptField("script", new Script("5", ScriptService.ScriptType.INLINE, MockScriptEngine.NAME, Collections.emptyMap())).fetchSource("comments.message", null)
                                         .version(true).sort("comments.date", SortOrder.ASC))).get();
         assertHitCount(searchResponse, 2);
         Nested nested = searchResponse.getAggregations().get("to-comments");
@@ -888,7 +889,7 @@ public class TopHitsIT extends ESIntegTestCase {
         assertThat(field.getValue().toString(), equalTo("5"));
 
         assertThat(searchHit.sourceAsMap().size(), equalTo(1));
-        assertThat(searchHit.sourceAsMap().get("message").toString(), equalTo("some comment"));
+        assertThat(XContentMapValues.extractValue("comments.message", searchHit.sourceAsMap()), equalTo("some comment"));
     }
 
     public void testTopHitsInNested() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java b/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
index 8d8d3194bee..0d1ba43659c 100644
--- a/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
@@ -26,6 +26,7 @@ import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.InnerHitBuilder;
 import org.elasticsearch.plugins.Plugin;
@@ -34,6 +35,7 @@ import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.fetch.source.FetchSourceContext;
 import org.elasticsearch.search.highlight.HighlightBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.SortBuilders;
@@ -47,6 +49,7 @@ import java.util.List;
 import java.util.Locale;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
 import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery;
 import static org.elasticsearch.index.query.QueryBuilders.hasChildQuery;
@@ -928,4 +931,32 @@ public class InnerHitsIT extends ESIntegTestCase {
         assertHitCount(response, 1);
     }
 
+    public void testNestedSourceFiltering() throws Exception {
+        assertAcked(prepareCreate("index1").addMapping("message", "comments", "type=nested"));
+        client().prepareIndex("index1", "message", "1").setSource(jsonBuilder().startObject()
+                .field("message", "quick brown fox")
+                .startArray("comments")
+                .startObject().field("message", "fox eat quick").endObject()
+                .startObject().field("message", "fox ate rabbit x y z").endObject()
+                .startObject().field("message", "rabbit got away").endObject()
+                .endArray()
+                .endObject()).get();
+        refresh();
+
+        // the field name (comments.message) used for source filtering should be the same as when using that field for
+        // other features (like in the query dsl or aggs) in order for consistency:
+        SearchResponse response = client().prepareSearch()
+                .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.None)
+                .innerHit(new InnerHitBuilder().setFetchSourceContext(new FetchSourceContext("comments.message"))))
+                .get();
+        assertNoFailures(response);
+        assertHitCount(response, 1);
+
+        assertThat(response.getHits().getAt(0).getInnerHits().get("comments").totalHits(), equalTo(2L));
+        assertThat(extractValue("comments.message", response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).sourceAsMap()),
+                equalTo("fox eat quick"));
+        assertThat(extractValue("comments.message", response.getHits().getAt(0).getInnerHits().get("comments").getAt(1).sourceAsMap()),
+                equalTo("fox ate rabbit x y z"));
+    }
+
 }
diff --git a/docs/reference/migration/migrate_5_0/search.asciidoc b/docs/reference/migration/migrate_5_0/search.asciidoc
index bb450f9a819..62384afa60a 100644
--- a/docs/reference/migration/migrate_5_0/search.asciidoc
+++ b/docs/reference/migration/migrate_5_0/search.asciidoc
@@ -177,6 +177,9 @@ The `coerce` and `ignore_malformed` parameters were deprecated in favour of `val
 `has_child` and `has_parent` queries. Use cases previously only possible with top level inner hits can now be done
 with inner hits defined inside the query dsl.
 
+* Source filtering for inner hits inside nested queries requires full field names instead of relative field names.
+This is now consistent for source filtering on other places in the search API.
+
 ==== Query Profiler
 
 In the response for profiling queries, the `query_type` has been renamed to `type` and `lucene` has been renamed to

From 1a298c2e6f05ea140d300efa4e2312d924ffc4fb Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 27 May 2016 14:40:38 +0200
Subject: [PATCH 0496/1311] Revert "Revert "Update the minimum required version
 of gradle to 2.13""

This reverts commit 0b965167fd43a807d9dcaa43e876a5fac0ccf0cb.
All the CI machines have been updated to gradle 2.13, we should be ok upgrading the required minimum version.
---
 build.gradle                                                    | 1 -
 .../src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/build.gradle b/build.gradle
index 3f0895a4e75..b52d299ad94 100644
--- a/build.gradle
+++ b/build.gradle
@@ -257,7 +257,6 @@ allprojects {
 
 idea {
   project {
-    languageLevel = org.elasticsearch.gradle.BuildPlugin.minimumJava.toString()
     vcs = 'Git'
   }
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index 256aa70fdca..8d77e7a9a34 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -113,7 +113,7 @@ class BuildPlugin implements Plugin<Project> {
             }
 
             // enforce gradle version
-            GradleVersion minGradle = GradleVersion.version('2.8')
+            GradleVersion minGradle = GradleVersion.version('2.13')
             if (GradleVersion.current() < minGradle) {
                 throw new GradleException("${minGradle} or above is required to build elasticsearch")
             }

From 41710f1028bd744d14fdddfdcc50e17f460953fb Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 27 May 2016 08:51:19 -0400
Subject: [PATCH 0497/1311] Upgrade joda-time to 2.9.4

This commit upgrades joda-time to version 2.9.4 to integrate a bug fix
there into Elasticsearch.

Relates #18609
---
 core/build.gradle                                           | 2 +-
 distribution/licenses/joda-time-2.8.2.jar.sha1              | 1 -
 distribution/licenses/joda-time-2.9.4.jar.sha1              | 1 +
 .../elasticsearch/script/groovy/GroovySecurityTests.java    | 6 ------
 4 files changed, 2 insertions(+), 8 deletions(-)
 delete mode 100644 distribution/licenses/joda-time-2.8.2.jar.sha1
 create mode 100644 distribution/licenses/joda-time-2.9.4.jar.sha1

diff --git a/core/build.gradle b/core/build.gradle
index 16d21f66ef8..9137ce68824 100644
--- a/core/build.gradle
+++ b/core/build.gradle
@@ -63,7 +63,7 @@ dependencies {
   compile 'com.carrotsearch:hppc:0.7.1'
 
   // time handling, remove with java 8 time
-  compile 'joda-time:joda-time:2.8.2'
+  compile 'joda-time:joda-time:2.9.4'
   // joda 2.0 moved to using volatile fields for datetime
   // When updating to a new version, make sure to update our copy of BaseDateTime
   compile 'org.joda:joda-convert:1.2'
diff --git a/distribution/licenses/joda-time-2.8.2.jar.sha1 b/distribution/licenses/joda-time-2.8.2.jar.sha1
deleted file mode 100644
index cf1e6a935c7..00000000000
--- a/distribution/licenses/joda-time-2.8.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d27c24204c5e507b16fec01006b3d0f1ec42aed4
diff --git a/distribution/licenses/joda-time-2.9.4.jar.sha1 b/distribution/licenses/joda-time-2.9.4.jar.sha1
new file mode 100644
index 00000000000..e9ea891bfee
--- /dev/null
+++ b/distribution/licenses/joda-time-2.9.4.jar.sha1
@@ -0,0 +1 @@
+1c295b462f16702ebe720bbb08f62e1ba80da41b
\ No newline at end of file
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java
index 7bdfe5a205e..51aad9b2fba 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java
@@ -44,12 +44,6 @@ public class GroovySecurityTests extends ESTestCase {
 
     private GroovyScriptEngineService se;
 
-    static {
-        // ensure we load all the timezones in the parent classloader with all permissions
-        // relates to https://github.com/elastic/elasticsearch/issues/14524
-        org.joda.time.DateTimeZone.getDefault();
-    }
-
     @Override
     public void setUp() throws Exception {
         super.setUp();

From 4439a86cb8364742e977ea324bb174bcf7f51f06 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Wed, 25 May 2016 15:31:59 +0200
Subject: [PATCH 0498/1311] Improve TimeUnitRounding for edge cases and DST
 transitions

Our current testing for TimeUnitRoundings rounding() and nextRoundingValue()
methods that are used especially for date histograms lacked proper randomization
for time zones. We only did randomized tests for fixed offset time zones
(e.g. +01:00, -05:00) but didn't account for real world time zones with
DST transitions.

Adding those tests revealed a couple of problems with our current rounding logic.
In some cases, usually happening around those transitions, rounding a value down
could land on a value that itself is not a proper rounded value. Also sometimes
the nextRoundingValue would not line up properly with the rounded value of all
dates in the next unit interval.

This change improves the current rounding logic in TimeUnitRounding in two ways:
it makes sure that calling round(date) always returns a date that when rounded
again won't change (making round() idempotent) by handling special cases happening
during dst transitions by performing a second rounding. It also changes the
nextRoundingValue() method to itself rely on the round method to make sure we
always return rounded values for the unit interval boundaries.

Also adding tests for randomized TimeUnitRounding that assert important basic
properties the rounding values should have. For better understanding and readability
a few of the pathological edge cases are also added as a special test case.
---
 .../common/rounding/DateTimeUnit.java         |  37 ++--
 .../common/rounding/TimeZoneRounding.java     |  44 ++--
 .../DerivativePipelineAggregatorBuilder.java  |  15 +-
 .../common/rounding/DateTimeUnitTests.java    |  21 +-
 .../rounding/TimeZoneRoundingTests.java       | 189 +++++++++++++++---
 5 files changed, 210 insertions(+), 96 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/rounding/DateTimeUnit.java b/core/src/main/java/org/elasticsearch/common/rounding/DateTimeUnit.java
index bd0c5506859..375c1a27212 100644
--- a/core/src/main/java/org/elasticsearch/common/rounding/DateTimeUnit.java
+++ b/core/src/main/java/org/elasticsearch/common/rounding/DateTimeUnit.java
@@ -21,45 +21,42 @@ package org.elasticsearch.common.rounding;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.joda.Joda;
 import org.joda.time.DateTimeField;
+import org.joda.time.DateTimeZone;
 import org.joda.time.chrono.ISOChronology;
 
+import java.util.function.Function;
+
 /**
  *
  */
 public enum DateTimeUnit {
 
-    WEEK_OF_WEEKYEAR(   (byte) 1, ISOChronology.getInstanceUTC().weekOfWeekyear()),
-    YEAR_OF_CENTURY(    (byte) 2, ISOChronology.getInstanceUTC().yearOfCentury()),
-    QUARTER(            (byte) 3, Joda.QuarterOfYear.getField(ISOChronology.getInstanceUTC())),
-    MONTH_OF_YEAR(      (byte) 4, ISOChronology.getInstanceUTC().monthOfYear()),
-    DAY_OF_MONTH(       (byte) 5, ISOChronology.getInstanceUTC().dayOfMonth()),
-    HOUR_OF_DAY(        (byte) 6, ISOChronology.getInstanceUTC().hourOfDay()),
-    MINUTES_OF_HOUR(    (byte) 7, ISOChronology.getInstanceUTC().minuteOfHour()),
-    SECOND_OF_MINUTE(   (byte) 8, ISOChronology.getInstanceUTC().secondOfMinute());
+    WEEK_OF_WEEKYEAR(   (byte) 1, tz -> ISOChronology.getInstance(tz).weekOfWeekyear()),
+    YEAR_OF_CENTURY(    (byte) 2, tz -> ISOChronology.getInstance(tz).yearOfCentury()),
+    QUARTER(            (byte) 3, tz -> Joda.QuarterOfYear.getField(ISOChronology.getInstance(tz))),
+    MONTH_OF_YEAR(      (byte) 4, tz -> ISOChronology.getInstance(tz).monthOfYear()),
+    DAY_OF_MONTH(       (byte) 5, tz -> ISOChronology.getInstance(tz).dayOfMonth()),
+    HOUR_OF_DAY(        (byte) 6, tz -> ISOChronology.getInstance(tz).hourOfDay()),
+    MINUTES_OF_HOUR(    (byte) 7, tz -> ISOChronology.getInstance(tz).minuteOfHour()),
+    SECOND_OF_MINUTE(   (byte) 8, tz -> ISOChronology.getInstance(tz).secondOfMinute());
 
     private final byte id;
-    private final DateTimeField field;
+    private final Function<DateTimeZone, DateTimeField> fieldFunction;
 
-    private DateTimeUnit(byte id, DateTimeField field) {
+    private DateTimeUnit(byte id, Function<DateTimeZone, DateTimeField> fieldFunction) {
         this.id = id;
-        this.field = field;
+        this.fieldFunction = fieldFunction;
     }
 
     public byte id() {
         return id;
     }
 
-    public DateTimeField field() {
-        return field;
-    }
-
     /**
-     * @param unit the {@link DateTimeUnit} to check
-     * @return true if the unit is a day or longer
+     * @return the {@link DateTimeField} for the provided {@link DateTimeZone} for this time unit
      */
-    public static boolean isDayOrLonger(DateTimeUnit unit) {
-        return (unit == DateTimeUnit.HOUR_OF_DAY || unit == DateTimeUnit.MINUTES_OF_HOUR
-                || unit == DateTimeUnit.SECOND_OF_MINUTE) == false;
+    public DateTimeField field(DateTimeZone tz) {
+        return fieldFunction.apply(tz);
     }
 
     public static DateTimeUnit resolve(byte id) {
diff --git a/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java b/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java
index e0ffb89c8b0..5bb868ed253 100644
--- a/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java
+++ b/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java
@@ -25,7 +25,6 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
 import org.joda.time.DateTimeField;
 import org.joda.time.DateTimeZone;
-import org.joda.time.DurationField;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -108,7 +107,6 @@ public abstract class TimeZoneRounding extends Rounding {
 
         private DateTimeUnit unit;
         private DateTimeField field;
-        private DurationField durationField;
         private DateTimeZone timeZone;
 
         TimeUnitRounding() { // for serialization
@@ -116,8 +114,7 @@ public abstract class TimeZoneRounding extends Rounding {
 
         TimeUnitRounding(DateTimeUnit unit, DateTimeZone timeZone) {
             this.unit = unit;
-            this.field = unit.field();
-            this.durationField = field.getDurationField();
+            this.field = unit.field(timeZone);
             this.timeZone = timeZone;
         }
 
@@ -128,10 +125,16 @@ public abstract class TimeZoneRounding extends Rounding {
 
         @Override
         public long roundKey(long utcMillis) {
-            long timeLocal = utcMillis;
-            timeLocal = timeZone.convertUTCToLocal(utcMillis);
-            long rounded = field.roundFloor(timeLocal);
-            return timeZone.convertLocalToUTC(rounded, false, utcMillis);
+            long rounded = field.roundFloor(utcMillis);
+            if (timeZone.isFixed() == false && timeZone.getOffset(utcMillis) != timeZone.getOffset(rounded)) {
+                // in this case, we crossed a time zone transition. In some edge cases this will
+                // result in a value that is not a rounded value itself. We need to round again
+                // to make sure. This will have no affect in cases where 'rounded' was already a proper
+                // rounded value
+                rounded = field.roundFloor(rounded);
+            }
+            assert rounded == field.roundFloor(rounded);
+            return rounded;
         }
 
         @Override
@@ -141,24 +144,22 @@ public abstract class TimeZoneRounding extends Rounding {
         }
 
         @Override
-        public long nextRoundingValue(long time) {
-            if (DateTimeUnit.isDayOrLonger(unit)) {
-                time = timeZone.convertUTCToLocal(time);
-            }
-            long next = durationField.add(time, 1);
-            if (DateTimeUnit.isDayOrLonger(unit)) {
-                return timeZone.convertLocalToUTC(next, false);
-            } else {
-                return next;
+        public long nextRoundingValue(long utcMillis) {
+            long floor = roundKey(utcMillis);
+            // add one unit and round to get to next rounded value
+            long next = roundKey(field.add(floor, 1));
+            if (next == floor) {
+                // in rare case we need to add more than one unit
+                next = roundKey(field.add(floor, 2));
             }
+            return next;
         }
 
         @Override
         public void readFrom(StreamInput in) throws IOException {
             unit = DateTimeUnit.resolve(in.readByte());
-            field = unit.field();
-            durationField = field.getDurationField();
             timeZone = DateTimeZone.forID(in.readString());
+            field = unit.field(timeZone);
         }
 
         @Override
@@ -184,6 +185,11 @@ public abstract class TimeZoneRounding extends Rounding {
             return Objects.equals(unit, other.unit)
                     && Objects.equals(timeZone, other.timeZone);
         }
+
+        @Override
+        public String toString() {
+            return "[" + timeZone + "][" + unit +"]";
+        }
     }
 
     static class TimeIntervalRounding extends TimeZoneRounding {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregatorBuilder.java
index ea75dee3d14..ac970720330 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregatorBuilder.java
@@ -19,12 +19,6 @@
 
 package org.elasticsearch.search.aggregations.pipeline.derivative;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -42,6 +36,13 @@ import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInter
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
+import org.joda.time.DateTimeZone;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
 
 public class DerivativePipelineAggregatorBuilder extends PipelineAggregatorBuilder<DerivativePipelineAggregatorBuilder> {
     public static final String NAME = DerivativePipelineAggregator.TYPE.name();
@@ -142,7 +143,7 @@ public class DerivativePipelineAggregatorBuilder extends PipelineAggregatorBuild
         if (units != null) {
             DateTimeUnit dateTimeUnit = DateHistogramAggregatorFactory.DATE_FIELD_UNITS.get(units);
             if (dateTimeUnit != null) {
-                xAxisUnits = dateTimeUnit.field().getDurationField().getUnitMillis();
+                xAxisUnits = dateTimeUnit.field(DateTimeZone.UTC).getDurationField().getUnitMillis();
             } else {
                 TimeValue timeValue = TimeValue.parseTimeValue(units, null, getClass().getSimpleName() + ".unit");
                 if (timeValue != null) {
diff --git a/core/src/test/java/org/elasticsearch/common/rounding/DateTimeUnitTests.java b/core/src/test/java/org/elasticsearch/common/rounding/DateTimeUnitTests.java
index 79ef6929645..6c92d3d1f9a 100644
--- a/core/src/test/java/org/elasticsearch/common/rounding/DateTimeUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/common/rounding/DateTimeUnitTests.java
@@ -20,14 +20,14 @@ package org.elasticsearch.common.rounding;
 
 import org.elasticsearch.test.ESTestCase;
 
-import static org.elasticsearch.common.rounding.DateTimeUnit.WEEK_OF_WEEKYEAR;
-import static org.elasticsearch.common.rounding.DateTimeUnit.YEAR_OF_CENTURY;
-import static org.elasticsearch.common.rounding.DateTimeUnit.QUARTER;
-import static org.elasticsearch.common.rounding.DateTimeUnit.MONTH_OF_YEAR;
 import static org.elasticsearch.common.rounding.DateTimeUnit.DAY_OF_MONTH;
 import static org.elasticsearch.common.rounding.DateTimeUnit.HOUR_OF_DAY;
 import static org.elasticsearch.common.rounding.DateTimeUnit.MINUTES_OF_HOUR;
+import static org.elasticsearch.common.rounding.DateTimeUnit.MONTH_OF_YEAR;
+import static org.elasticsearch.common.rounding.DateTimeUnit.QUARTER;
 import static org.elasticsearch.common.rounding.DateTimeUnit.SECOND_OF_MINUTE;
+import static org.elasticsearch.common.rounding.DateTimeUnit.WEEK_OF_WEEKYEAR;
+import static org.elasticsearch.common.rounding.DateTimeUnit.YEAR_OF_CENTURY;
 
 public class DateTimeUnitTests extends ESTestCase {
 
@@ -59,17 +59,4 @@ public class DateTimeUnitTests extends ESTestCase {
         assertEquals(8, SECOND_OF_MINUTE.id());
         assertEquals(SECOND_OF_MINUTE, DateTimeUnit.resolve((byte) 8));
     }
-
-    public void testIsDayOrLonger() {
-        for (DateTimeUnit unit : DateTimeUnit.values()) {
-            if (DateTimeUnit.isDayOrLonger(unit)) {
-                assertTrue(unit == DAY_OF_MONTH ||
-                        unit == MONTH_OF_YEAR ||
-                        unit == QUARTER ||
-                        unit == YEAR_OF_CENTURY ||
-                        unit == WEEK_OF_WEEKYEAR);
-            }
-        }
-    }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java b/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
index 61710b726d4..af7b53712c1 100644
--- a/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
+++ b/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
@@ -19,9 +19,12 @@
 
 package org.elasticsearch.common.rounding;
 
+import org.elasticsearch.common.rounding.TimeZoneRounding.TimeIntervalRounding;
+import org.elasticsearch.common.rounding.TimeZoneRounding.TimeUnitRounding;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESTestCase;
 import org.joda.time.DateTime;
+import org.joda.time.DateTimeConstants;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.ISODateTimeFormat;
 
@@ -29,6 +32,7 @@ import java.util.concurrent.TimeUnit;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
 /**
@@ -208,50 +212,54 @@ public class TimeZoneRoundingTests extends ESTestCase {
     }
 
     /**
-     * randomized test on TimeUnitRounding with random time units and time zone offsets
+     * Randomized test on TimeUnitRounding.
+     * Test uses random {@link DateTimeUnit} and {@link DateTimeZone} and often (50% of the time) chooses
+     * test dates that are exactly on or close to offset changes (e.g. DST) in the chosen time zone.
+     *
+     * It rounds the test date down and up and performs various checks on the rounding unit interval that is
+     * defined by this. Assumptions tested are described in {@link #assertInterval(long, long, long, TimeZoneRounding)}
      */
     public void testTimeZoneRoundingRandom() {
         for (int i = 0; i < 1000; ++i) {
             DateTimeUnit timeUnit = randomTimeUnit();
-            TimeZoneRounding rounding;
-            int timezoneOffset = randomIntBetween(-23, 23);
-            rounding = new TimeZoneRounding.TimeUnitRounding(timeUnit, DateTimeZone.forOffsetHours(timezoneOffset));
-            long date = Math.abs(randomLong() % ((long) 10e11));
+            DateTimeZone timezone = randomDateTimeZone();
+            TimeZoneRounding rounding = new TimeZoneRounding.TimeUnitRounding(timeUnit, timezone);
+            long date = Math.abs(randomLong() % (2 * (long) 10e11)); // 1970-01-01T00:00:00Z - 2033-05-18T05:33:20.000+02:00
+            long unitMillis = timeUnit.field(timezone).getDurationField().getUnitMillis();
+            if (randomBoolean()) {
+                nastyDate(date, timezone, unitMillis);
+            }
             final long roundedDate = rounding.round(date);
             final long nextRoundingValue = rounding.nextRoundingValue(roundedDate);
-            assertThat("Rounding should be idempotent", roundedDate, equalTo(rounding.round(roundedDate)));
-            assertThat("Rounded value smaller or equal than unrounded, regardless of timezone", roundedDate, lessThanOrEqualTo(date));
-            assertThat("NextRounding value should be greater than date", nextRoundingValue, greaterThan(roundedDate));
-            assertThat("NextRounding value should be a rounded date", nextRoundingValue, equalTo(rounding.round(nextRoundingValue)));
+
+            assertInterval(roundedDate, date, nextRoundingValue, rounding);
+
+            // check correct unit interval width for units smaller than a day, they should be fixed size except for transitions
+            if (unitMillis <= DateTimeConstants.MILLIS_PER_DAY) {
+                // if the interval defined didn't cross timezone offset transition, it should cover unitMillis width
+                if (timezone.getOffset(roundedDate - 1) == timezone.getOffset(nextRoundingValue + 1)) {
+                    assertThat("unit interval width not as expected for [" + timeUnit + "], [" + timezone + "] at "
+                            + new DateTime(roundedDate), nextRoundingValue - roundedDate, equalTo(unitMillis));
+                }
+            }
         }
     }
 
     /**
-     * Test that nextRoundingValue() for hour rounding (and smaller) is equally spaced (see #18326)
-     * Start at a random date in a random time zone, then find the next zone offset transition (if any).
-     * From there, check that when we advance by using rounding#nextRoundingValue(), we always advance by the same
-     * amount of milliseconds.
+     * To be even more nasty, go to a transition in the selected time zone.
+     * In one third of the cases stay there, otherwise go half a unit back or forth
      */
-    public void testSubHourNextRoundingEquallySpaced() {
-        DateTimeUnit unit = randomFrom(new DateTimeUnit[] { DateTimeUnit.HOUR_OF_DAY, DateTimeUnit.MINUTES_OF_HOUR,
-                DateTimeUnit.SECOND_OF_MINUTE });
-        DateTimeZone timezone = randomDateTimeZone();
-        TimeZoneRounding rounding = new TimeZoneRounding.TimeUnitRounding(unit, timezone);
-        // move the random date to transition for timezones that have offset change due to dst transition
-        long nextTransition = timezone.nextTransition(Math.abs(randomLong() % ((long) 10e11)));
-        final long millisPerUnit = unit.field().getDurationField().getUnitMillis();
-        // start ten units before transition
-        long roundedDate = rounding.round(nextTransition - (10 * millisPerUnit));
-        while (roundedDate < nextTransition + 10 * millisPerUnit) {
-            long delta = rounding.nextRoundingValue(roundedDate) - roundedDate;
-            assertEquals("Difference between rounded values not equally spaced for [" + unit.name() + "], [" + timezone + "] at "
-                    + new DateTime(roundedDate), millisPerUnit, delta);
-            roundedDate = rounding.nextRoundingValue(roundedDate);
+    private static long nastyDate(long initialDate, DateTimeZone timezone, long unitMillis) {
+        long date = timezone.nextTransition(initialDate);
+        if (randomBoolean()) {
+            return date + (randomLong() % unitMillis);  // positive and negative offset possible
+        } else {
+            return date;
         }
     }
 
     /**
-     * randomized test on TimeIntervalRounding with random interval and time zone offsets
+     * randomized test on {@link TimeIntervalRounding} with random interval and time zone offsets
      */
     public void testIntervalRoundingRandom() {
         for (int i = 0; i < 1000; ++i) {
@@ -324,20 +332,135 @@ public class TimeZoneRoundingTests extends ESTestCase {
         }
     }
 
-    private DateTimeUnit randomTimeUnit() {
+    public void testEdgeCasesTransition() {
+        {
+            // standard +/-1 hour DST transition, CET
+            DateTimeUnit timeUnit = DateTimeUnit.HOUR_OF_DAY;
+            DateTimeZone timezone = DateTimeZone.forID("CET");
+            TimeZoneRounding rounding = new TimeZoneRounding.TimeUnitRounding(timeUnit, timezone);
+
+            // 29 Mar 2015 - Daylight Saving Time Started
+            // at 02:00:00 clocks were turned forward 1 hour to 03:00:00
+            assertInterval(time("2015-03-29T00:00:00.000+01:00"), time("2015-03-29T01:00:00.000+01:00"), rounding, 60);
+            assertInterval(time("2015-03-29T01:00:00.000+01:00"), time("2015-03-29T03:00:00.000+02:00"), rounding, 60);
+            assertInterval(time("2015-03-29T03:00:00.000+02:00"), time("2015-03-29T04:00:00.000+02:00"), rounding, 60);
+
+            // 25 Oct 2015 - Daylight Saving Time Ended
+            // at 03:00:00 clocks were turned backward 1 hour to 02:00:00
+            assertInterval(time("2015-10-25T01:00:00.000+02:00"), time("2015-10-25T02:00:00.000+02:00"), rounding, 60);
+            assertInterval(time("2015-10-25T02:00:00.000+02:00"), time("2015-10-25T02:00:00.000+01:00"), rounding, 60);
+            assertInterval(time("2015-10-25T02:00:00.000+01:00"), time("2015-10-25T03:00:00.000+01:00"), rounding, 60);
+        }
+
+        {
+            // time zone "Asia/Kathmandu"
+            // 1 Jan 1986 - Time Zone Change (IST → NPT), at 00:00:00 clocks were turned forward 00:15 minutes
+            //
+            // hour rounding is stable before 1985-12-31T23:00:00.000 and after 1986-01-01T01:00:00.000+05:45
+            // the interval between is 105 minutes long because the hour after transition starts at 00:15
+            // which is not a round value for hourly rounding
+            DateTimeUnit timeUnit = DateTimeUnit.HOUR_OF_DAY;
+            DateTimeZone timezone = DateTimeZone.forID("Asia/Kathmandu");
+            TimeZoneRounding rounding = new TimeZoneRounding.TimeUnitRounding(timeUnit, timezone);
+
+            assertInterval(time("1985-12-31T22:00:00.000+05:30"), time("1985-12-31T23:00:00.000+05:30"), rounding, 60);
+            assertInterval(time("1985-12-31T23:00:00.000+05:30"), time("1986-01-01T01:00:00.000+05:45"), rounding, 105);
+            assertInterval(time("1986-01-01T01:00:00.000+05:45"), time("1986-01-01T02:00:00.000+05:45"), rounding, 60);
+        }
+
+        {
+            // time zone "Australia/Lord_Howe"
+            // 3 Mar 1991 - Daylight Saving Time Ended
+            // at 02:00:00 clocks were turned backward 0:30 hours to Sunday, 3 March 1991, 01:30:00
+            DateTimeUnit timeUnit = DateTimeUnit.HOUR_OF_DAY;
+            DateTimeZone timezone = DateTimeZone.forID("Australia/Lord_Howe");
+            TimeZoneRounding rounding = new TimeZoneRounding.TimeUnitRounding(timeUnit, timezone);
+
+            assertInterval(time("1991-03-03T00:00:00.000+11:00"), time("1991-03-03T01:00:00.000+11:00"), rounding, 60);
+            assertInterval(time("1991-03-03T01:00:00.000+11:00"), time("1991-03-03T02:00:00.000+10:30"), rounding, 90);
+            assertInterval(time("1991-03-03T02:00:00.000+10:30"), time("1991-03-03T03:00:00.000+10:30"), rounding, 60);
+
+            // 27 Oct 1991 - Daylight Saving Time Started
+            // at 02:00:00 clocks were turned forward 0:30 hours to 02:30:00
+            assertInterval(time("1991-10-27T00:00:00.000+10:30"), time("1991-10-27T01:00:00.000+10:30"), rounding, 60);
+            // the interval containing the switch time is 90 minutes long
+            assertInterval(time("1991-10-27T01:00:00.000+10:30"), time("1991-10-27T03:00:00.000+11:00"), rounding, 90);
+            assertInterval(time("1991-10-27T03:00:00.000+11:00"), time("1991-10-27T04:00:00.000+11:00"), rounding, 60);
+        }
+
+        {
+            // time zone "Pacific/Chatham"
+            // 5 Apr 2015 - Daylight Saving Time Ended
+            // at 03:45:00 clocks were turned backward 1 hour to 02:45:00
+            DateTimeUnit timeUnit = DateTimeUnit.HOUR_OF_DAY;
+            DateTimeZone timezone = DateTimeZone.forID("Pacific/Chatham");
+            TimeZoneRounding rounding = new TimeZoneRounding.TimeUnitRounding(timeUnit, timezone);
+
+            assertInterval(time("2015-04-05T02:00:00.000+13:45"), time("2015-04-05T03:00:00.000+13:45"), rounding, 60);
+            assertInterval(time("2015-04-05T03:00:00.000+13:45"), time("2015-04-05T03:00:00.000+12:45"), rounding, 60);
+            assertInterval(time("2015-04-05T03:00:00.000+12:45"), time("2015-04-05T04:00:00.000+12:45"), rounding, 60);
+
+            // 27 Sep 2015 - Daylight Saving Time Started
+            // at 02:45:00 clocks were turned forward 1 hour to 03:45:00
+
+            assertInterval(time("2015-09-27T01:00:00.000+12:45"), time("2015-09-27T02:00:00.000+12:45"), rounding, 60);
+            assertInterval(time("2015-09-27T02:00:00.000+12:45"), time("2015-09-27T04:00:00.000+13:45"), rounding, 60);
+            assertInterval(time("2015-09-27T04:00:00.000+13:45"), time("2015-09-27T05:00:00.000+13:45"), rounding, 60);
+        }
+    }
+
+    private static void assertInterval(long rounded, long nextRoundingValue, TimeZoneRounding rounding, int minutes) {
+        assertInterval(rounded, dateBetween(rounded, nextRoundingValue), nextRoundingValue, rounding);
+        assertEquals(DateTimeConstants.MILLIS_PER_MINUTE * minutes, nextRoundingValue - rounded);
+    }
+
+    /**
+     * perform a number on assertions and checks on {@link TimeUnitRounding} intervals
+     * @param rounded the expected low end of the rounding interval
+     * @param unrounded a date in the interval to be checked for rounding
+     * @param nextRoundingValue the expected upper end of the rounding interval
+     * @param rounding the rounding instance
+     */
+    private static void assertInterval(long rounded, long unrounded, long nextRoundingValue, TimeZoneRounding rounding) {
+        assert rounded <= unrounded && unrounded <= nextRoundingValue;
+        assertThat("rounding should be idempotent " + rounding, rounded, equalTo(rounding.round(rounded)));
+        assertThat("rounded value smaller or equal than unrounded" + rounding, rounded, lessThanOrEqualTo(unrounded));
+        assertThat("values less than rounded should round further down" + rounding, rounding.round(rounded - 1), lessThan(rounded));
+        assertThat("nextRounding value should be greater than date" + rounding, nextRoundingValue, greaterThan(unrounded));
+        assertThat("nextRounding value should be a rounded date" + rounding, nextRoundingValue, equalTo(rounding.round(nextRoundingValue)));
+        assertThat("values above nextRounding should round down there" + rounding, rounding.round(nextRoundingValue + 1),
+                equalTo(nextRoundingValue));
+
+        long dateBetween = dateBetween(rounded, nextRoundingValue);
+        assertThat("dateBetween should round down to roundedDate" + rounding, rounding.round(dateBetween), equalTo(rounded));
+        assertThat("dateBetween should round up to nextRoundingValue" + rounding, rounding.nextRoundingValue(dateBetween),
+                equalTo(nextRoundingValue));
+    }
+
+    private static long dateBetween(long lower, long upper) {
+        long dateBetween = lower + Math.abs((randomLong() % (upper - lower)));
+        assert lower <= dateBetween && dateBetween <= upper;
+        return dateBetween;
+    }
+
+    private static DateTimeUnit randomTimeUnit() {
         byte id = (byte) randomIntBetween(1, 8);
         return DateTimeUnit.resolve(id);
     }
 
-    private String toUTCDateString(long time) {
+    private static String toUTCDateString(long time) {
         return new DateTime(time, DateTimeZone.UTC).toString();
     }
 
-    private long utc(String time) {
+    private static long utc(String time) {
         return time(time, DateTimeZone.UTC);
     }
 
-    private long time(String time, DateTimeZone zone) {
+    private static long time(String time) {
+        return ISODateTimeFormat.dateOptionalTimeParser().parseMillis(time);
+    }
+
+    private static long time(String time, DateTimeZone zone) {
         return ISODateTimeFormat.dateOptionalTimeParser().withZone(zone).parseMillis(time);
     }
 }

From 29c970958c9fadc293666ea87225ea3b3d96363e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Fri, 27 May 2016 12:59:26 +0200
Subject: [PATCH 0499/1311] Adding tests for derivatives on date histogram
 aggregation with time zones

---
 .../pipeline/DateDerivativeIT.java            | 152 +++++++++++++++++-
 1 file changed, 148 insertions(+), 4 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java
index d1495cc7321..abda95dd253 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DateDerivativeIT.java
@@ -30,6 +30,7 @@ import org.elasticsearch.search.aggregations.metrics.sum.Sum;
 import org.elasticsearch.search.aggregations.pipeline.derivative.Derivative;
 import org.elasticsearch.search.aggregations.support.AggregationPath;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.hamcrest.Matcher;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.DateTimeFormat;
@@ -54,6 +55,11 @@ import static org.hamcrest.core.IsNull.nullValue;
 @ESIntegTestCase.SuiteScopeTestCase
 public class DateDerivativeIT extends ESIntegTestCase {
 
+    // some index names used during these tests
+    private static final String IDX_DST_START = "idx_dst_start";
+    private static final String IDX_DST_END = "idx_dst_end";
+    private static final String IDX_DST_KATHMANDU = "idx_dst_kathmandu";
+
     private DateTime date(int month, int day) {
         return new DateTime(2012, month, day, 0, 0, DateTimeZone.UTC);
     }
@@ -66,10 +72,9 @@ public class DateDerivativeIT extends ESIntegTestCase {
         return DateTimeFormat.forPattern(pattern).print(date);
     }
 
-    private IndexRequestBuilder indexDoc(String idx, DateTime date, int value) throws Exception {
+    private static IndexRequestBuilder indexDoc(String idx, DateTime date, int value) throws Exception {
         return client().prepareIndex(idx, "type").setSource(
-                jsonBuilder().startObject().field("date", date).field("value", value).startArray("dates").value(date)
-                        .value(date.plusMonths(1).plusDays(1)).endArray().endObject());
+                jsonBuilder().startObject().field("date", date).field("value", value).endObject());
     }
 
     private IndexRequestBuilder indexDoc(int month, int day, int value) throws Exception {
@@ -101,7 +106,7 @@ public class DateDerivativeIT extends ESIntegTestCase {
 
     @After
     public void afterEachTest() throws IOException {
-        internalCluster().wipeIndices("idx2");
+        internalCluster().wipeIndices(IDX_DST_START, IDX_DST_END, IDX_DST_KATHMANDU);
     }
 
     public void testSingleValuedField() throws Exception {
@@ -191,6 +196,145 @@ public class DateDerivativeIT extends ESIntegTestCase {
         assertThat(docCountDeriv.normalizedValue(), closeTo(1d / 29d, 0.00001));
     }
 
+    /**
+     * Do a derivative on a date histogram with time zone CET at DST start
+     */
+    public void testSingleValuedFieldNormalised_timeZone_CET_DstStart() throws Exception {
+        createIndex(IDX_DST_START);
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+
+        DateTimeZone timezone = DateTimeZone.forID("CET");
+        addNTimes(1, IDX_DST_START, new DateTime("2012-03-24T01:00:00", timezone), builders);
+        addNTimes(2, IDX_DST_START, new DateTime("2012-03-25T01:00:00", timezone), builders); // day with dst shift, only 23h long
+        addNTimes(3, IDX_DST_START, new DateTime("2012-03-26T01:00:00", timezone), builders);
+        addNTimes(4, IDX_DST_START, new DateTime("2012-03-27T01:00:00", timezone), builders);
+        indexRandom(true, builders);
+        ensureSearchable();
+
+        SearchResponse response = client()
+                .prepareSearch(IDX_DST_START)
+                .addAggregation(dateHistogram("histo").field("date").dateHistogramInterval(DateHistogramInterval.DAY)
+                        .timeZone(timezone).minDocCount(0)
+                        .subAggregation(derivative("deriv", "_count").unit(DateHistogramInterval.HOUR)))
+                .execute()
+                .actionGet();
+
+        assertSearchResponse(response);
+
+        InternalHistogram deriv = response.getAggregations().get("histo");
+        assertThat(deriv, notNullValue());
+        assertThat(deriv.getName(), equalTo("histo"));
+        List<? extends Bucket> buckets = deriv.getBuckets();
+        assertThat(buckets.size(), equalTo(4));
+
+        assertBucket(buckets.get(0), new DateTime("2012-03-24", timezone).toDateTime(DateTimeZone.UTC), 1L, nullValue(), null, null);
+        assertBucket(buckets.get(1), new DateTime("2012-03-25", timezone).toDateTime(DateTimeZone.UTC), 2L, notNullValue(), 1d, 1d / 24d);
+        // the following is normalized using a 23h bucket width
+        assertBucket(buckets.get(2), new DateTime("2012-03-26", timezone).toDateTime(DateTimeZone.UTC), 3L, notNullValue(), 1d, 1d / 23d);
+        assertBucket(buckets.get(3), new DateTime("2012-03-27", timezone).toDateTime(DateTimeZone.UTC), 4L, notNullValue(), 1d, 1d / 24d);
+    }
+
+    /**
+     * Do a derivative on a date histogram with time zone CET at DST end
+     */
+    public void testSingleValuedFieldNormalised_timeZone_CET_DstEnd() throws Exception {
+        createIndex(IDX_DST_END);
+        DateTimeZone timezone = DateTimeZone.forID("CET");
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+
+        addNTimes(1, IDX_DST_END, new DateTime("2012-10-27T01:00:00", timezone), builders);
+        addNTimes(2, IDX_DST_END, new DateTime("2012-10-28T01:00:00", timezone), builders); // day with dst shift -1h, 25h long
+        addNTimes(3, IDX_DST_END, new DateTime("2012-10-29T01:00:00", timezone), builders);
+        addNTimes(4, IDX_DST_END, new DateTime("2012-10-30T01:00:00", timezone), builders);
+        indexRandom(true, builders);
+        ensureSearchable();
+
+        SearchResponse response = client()
+                .prepareSearch(IDX_DST_END)
+                .addAggregation(dateHistogram("histo").field("date").dateHistogramInterval(DateHistogramInterval.DAY)
+                        .timeZone(timezone).minDocCount(0)
+                        .subAggregation(derivative("deriv", "_count").unit(DateHistogramInterval.HOUR)))
+                .execute()
+                .actionGet();
+
+        assertSearchResponse(response);
+
+        InternalHistogram deriv = response.getAggregations().get("histo");
+        assertThat(deriv, notNullValue());
+        assertThat(deriv.getName(), equalTo("histo"));
+        List<? extends Bucket> buckets = deriv.getBuckets();
+        assertThat(buckets.size(), equalTo(4));
+
+        assertBucket(buckets.get(0), new DateTime("2012-10-27", timezone).toDateTime(DateTimeZone.UTC), 1L, nullValue(), null, null);
+        assertBucket(buckets.get(1), new DateTime("2012-10-28", timezone).toDateTime(DateTimeZone.UTC), 2L, notNullValue(), 1d, 1d / 24d);
+        // the following is normalized using a 25h bucket width
+        assertBucket(buckets.get(2), new DateTime("2012-10-29", timezone).toDateTime(DateTimeZone.UTC), 3L, notNullValue(), 1d, 1d / 25d);
+        assertBucket(buckets.get(3), new DateTime("2012-10-30", timezone).toDateTime(DateTimeZone.UTC), 4L, notNullValue(), 1d, 1d / 24d);
+    }
+
+    /**
+     * also check for time zone shifts that are not one hour, e.g.
+     * "Asia/Kathmandu, 1 Jan 1986 - Time Zone Change (IST → NPT), at 00:00:00 clocks were turned forward 00:15 minutes
+     */
+    public void testSingleValuedFieldNormalised_timeZone_AsiaKathmandu() throws Exception {
+        createIndex(IDX_DST_KATHMANDU);
+        DateTimeZone timezone = DateTimeZone.forID("Asia/Kathmandu");
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+
+        addNTimes(1, IDX_DST_KATHMANDU, new DateTime("1985-12-31T22:30:00", timezone), builders);
+        // the shift happens during the next bucket, which includes the 45min that do not start on the full hour
+        addNTimes(2, IDX_DST_KATHMANDU, new DateTime("1985-12-31T23:30:00", timezone), builders);
+        addNTimes(3, IDX_DST_KATHMANDU, new DateTime("1986-01-01T01:30:00", timezone), builders);
+        addNTimes(4, IDX_DST_KATHMANDU, new DateTime("1986-01-01T02:30:00", timezone), builders);
+        indexRandom(true, builders);
+        ensureSearchable();
+
+        SearchResponse response = client()
+                .prepareSearch(IDX_DST_KATHMANDU)
+                .addAggregation(dateHistogram("histo").field("date").dateHistogramInterval(DateHistogramInterval.HOUR)
+                        .timeZone(timezone).minDocCount(0)
+                        .subAggregation(derivative("deriv", "_count").unit(DateHistogramInterval.MINUTE)))
+                .execute()
+                .actionGet();
+
+        assertSearchResponse(response);
+
+        InternalHistogram deriv = response.getAggregations().get("histo");
+        assertThat(deriv, notNullValue());
+        assertThat(deriv.getName(), equalTo("histo"));
+        List<? extends Bucket> buckets = deriv.getBuckets();
+        assertThat(buckets.size(), equalTo(4));
+
+        assertBucket(buckets.get(0), new DateTime("1985-12-31T22:00:00", timezone).toDateTime(DateTimeZone.UTC), 1L, nullValue(), null,
+                null);
+        assertBucket(buckets.get(1), new DateTime("1985-12-31T23:00:00", timezone).toDateTime(DateTimeZone.UTC), 2L, notNullValue(), 1d,
+                1d / 60d);
+        // the following is normalized using a 105min bucket width
+        assertBucket(buckets.get(2), new DateTime("1986-01-01T01:00:00", timezone).toDateTime(DateTimeZone.UTC), 3L, notNullValue(), 1d,
+                1d / 105d);
+        assertBucket(buckets.get(3), new DateTime("1986-01-01T02:00:00", timezone).toDateTime(DateTimeZone.UTC), 4L, notNullValue(), 1d,
+                1d / 60d);
+    }
+
+    private static void addNTimes(int amount, String index, DateTime dateTime, List<IndexRequestBuilder> builders) throws Exception {
+        for (int i = 0; i < amount; i++) {
+            builders.add(indexDoc(index, dateTime, 1));
+        }
+    }
+
+    private static void assertBucket(Histogram.Bucket bucket, DateTime expectedKey, long expectedDocCount,
+            Matcher<Object> derivativeMatcher, Double derivative, Double normalizedDerivative) {
+        assertThat(bucket, notNullValue());
+        assertThat((DateTime) bucket.getKey(), equalTo(expectedKey));
+        assertThat(bucket.getDocCount(), equalTo(expectedDocCount));
+        Derivative docCountDeriv = bucket.getAggregations().get("deriv");
+        assertThat(docCountDeriv, derivativeMatcher);
+        if (docCountDeriv != null) {
+            assertThat(docCountDeriv.value(), closeTo(derivative, 0.00001));
+            assertThat(docCountDeriv.normalizedValue(), closeTo(normalizedDerivative, 0.00001));
+        }
+    }
+
     public void testSingleValuedFieldWithSubAggregation() throws Exception {
         SearchResponse response = client()
                 .prepareSearch("idx")

From 123e40726e4c73414c3bc1905909691526107760 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 27 May 2016 09:19:49 -0400
Subject: [PATCH 0500/1311] Fix bootstrap check docs

This commit fixes some incorrect links in the bootstrap check docs.
---
 docs/reference/setup/bootstrap-checks.asciidoc | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc
index dd1d2e63342..8a5c215f3d2 100644
--- a/docs/reference/setup/bootstrap-checks.asciidoc
+++ b/docs/reference/setup/bootstrap-checks.asciidoc
@@ -1,8 +1,9 @@
-[[bootstrap-checks]] == Bootstrap Checks
+[[bootstrap-checks]]
+== Bootstrap Checks
 
 Collectively, we have a lot of experience with users suffering
 unexpected issues because they have not configured
-<<important-setting,important settings>>. In previous versions of
+<<important-settings,important settings>>. In previous versions of
 Elasticsearch, misconfiguration of some of these settings were logged
 as warnings. Understandably, users sometimes miss these log messages.
 To ensure that these settings receive the attention that they deserve,
@@ -36,7 +37,7 @@ size equal to the maximum heap size. Additionally, if
 the heap on startup. If the initial heap size is not equal to the
 maximum heap size, after a resize it will not be the case that all of
 the JVM heap is locked in memory. To pass the heap size check, you must
-configure the [[heap-size,heap size]].
+configure the <<heap-size,heap size>>.
 
 === File descriptor check
 
@@ -48,7 +49,7 @@ file](https://en.wikipedia.org/wiki/Everything_is_a_file). For example,
 descriptors (e.g., every shard is composed of multiple segments and
 other files, plus connections to other nodes, etc.). This bootstrap
 check is enforced on OS X and Linux. To pass the file descriptor check,
-you might have to configure [[file-descriptors,file descriptors]].
+you might have to configure <<file-descriptors,file descriptors>>.
 
 === Memory lock check
 

From 2b47a2643c1272ede3890f0c3bfb10e3f0b650e8 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Fri, 27 May 2016 15:28:57 +0200
Subject: [PATCH 0501/1311] Only fail relocation target shard if failing source
 shard is a primary (#18574)

If the relocation source fails during the relocation of a shard from one node to another, the
relocation target is currently failed as well. For replica shards this is not necessary,
however, as the actual shard recovery of the relocation target is done via the primary shard.
---
 .../cluster/routing/RoutingNodes.java         | 27 ++++++++++-----
 .../cluster/routing/ShardRouting.java         | 14 ++++++++
 .../routing/allocation/AllocationService.java | 27 ++++++++++-----
 .../command/CancelAllocationCommand.java      | 20 ++++++++---
 .../allocation/AllocationCommandsTests.java   | 34 +++++++++++++++++--
 .../indices/recovery/IndexRecoveryIT.java     | 22 ++++++++++++
 6 files changed, 119 insertions(+), 25 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
index cb86c898487..159dfd99d1f 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
@@ -23,12 +23,9 @@ import com.carrotsearch.hppc.ObjectIntHashMap;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.block.ClusterBlocks;
-import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Randomness;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
@@ -808,12 +805,10 @@ public class RoutingNodes implements Iterable<RoutingNode> {
         Map<Index, Integer> indicesAndShards = new HashMap<>();
         for (RoutingNode node : routingNodes) {
             for (ShardRouting shard : node) {
-                if (!shard.active() && shard.relocatingNodeId() == null) {
-                    if (!shard.relocating()) {
-                        inactiveShardCount++;
-                        if (shard.primary()) {
-                            inactivePrimaryCount++;
-                        }
+                if (shard.initializing() && shard.relocatingNodeId() == null) {
+                    inactiveShardCount++;
+                    if (shard.primary()) {
+                        inactivePrimaryCount++;
                     }
                 }
                 if (shard.relocating()) {
@@ -986,6 +981,20 @@ public class RoutingNodes implements Iterable<RoutingNode> {
             return unassigned;
         }
 
+        /**
+         * Removes relocation source of an initializing non-primary shard. This allows the replica shard to continue recovery from
+         * the primary even though its non-primary relocation source has failed.
+         */
+        public ShardRouting removeRelocationSource() {
+            assert shard.isRelocationTarget();
+            ensureMutable();
+            ShardRouting relocationMarkerRemoved = shard.removeRelocationSource();
+            updateAssigned(shard, relocationMarkerRemoved);
+            inactiveShardCount++; // relocation targets are not counted as inactive shards whereas initializing shards are
+            Recoveries.getOrAdd(recoveriesPerNode, shard.relocatingNodeId()).addOutgoing(-1);
+            return relocationMarkerRemoved;
+        }
+
         public ShardRouting current() {
             return shard;
         }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
index 810f8c183da..34a86d554ed 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java
@@ -371,6 +371,20 @@ public final class ShardRouting implements Writeable, ToXContent {
             null, AllocationId.cancelRelocation(allocationId), UNAVAILABLE_EXPECTED_SHARD_SIZE);
     }
 
+    /**
+     * Removes relocation source of a non-primary shard. The shard state must be <code>INITIALIZING</code>.
+     * This allows the non-primary shard to continue recovery from the primary even though its non-primary
+     * relocation source has failed.
+     */
+    public ShardRouting removeRelocationSource() {
+        assert primary == false : this;
+        assert state == ShardRoutingState.INITIALIZING : this;
+        assert assignedToNode() : this;
+        assert relocatingNodeId != null : this;
+        return new ShardRouting(shardId, currentNodeId, null, restoreSource, primary, state, unassignedInfo,
+            AllocationId.finishRelocation(allocationId), expectedShardSize);
+    }
+
     /**
      * Moves the shard from started to initializing
      */
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
index 7dd9b1f8fec..83f7ac9f2af 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
@@ -576,13 +576,11 @@ public class AllocationService extends AbstractComponent {
                 }
             }
         } else {
-            // The fail shard is the main copy of the current shard routing. Any
-            // relocation will be cancelled (and the target shard removed as well)
-            // and the shard copy needs to be marked as unassigned
+            // The fail shard is the main copy of the current shard routing.
 
+            boolean addAsUnassigned = true;
             if (failedShard.relocatingNodeId() != null) {
-                // handle relocation source shards.  we need to find the target initializing shard that is recovering, and remove it...
-                assert failedShard.initializing() == false; // should have been dealt with and returned
+                // now, find the shard that is initializing on the target node
                 assert failedShard.relocating();
 
                 RoutingNodes.RoutingNodeIterator initializingNode = routingNodes.routingNodeIter(failedShard.relocatingNodeId());
@@ -590,14 +588,25 @@ public class AllocationService extends AbstractComponent {
                     while (initializingNode.hasNext()) {
                         ShardRouting shardRouting = initializingNode.next();
                         if (shardRouting.isRelocationTargetOf(failedShard)) {
-                            logger.trace("{} is removed due to the failure of the source shard", shardRouting);
-                            initializingNode.remove();
+                            if (failedShard.primary()) {
+                                logger.trace("{} is removed due to the failure of the source shard", shardRouting);
+                                // cancel and remove target shard
+                                initializingNode.remove();
+                            } else {
+                                logger.trace("{}, relocation source failed, mark as initializing without relocation source", shardRouting);
+                                // promote to initializing shard without relocation source and ensure that removed relocation source
+                                // is not added back as unassigned shard
+                                initializingNode.removeRelocationSource();
+                                addAsUnassigned = false;
+                            }
+                            break;
                         }
                     }
                 }
             }
-
-            matchedNode.moveToUnassigned(unassignedInfo);
+            if (addAsUnassigned) {
+                matchedNode.moveToUnassigned(unassignedInfo);
+            }
         }
         assert matchedNode.isRemoved() : "failedShard " + failedShard + " was matched but wasn't removed";
         return true;
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java
index 9adb3e30708..0aaacaf89ff 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java
@@ -146,7 +146,6 @@ public class CancelAllocationCommand implements AllocationCommand {
                     }
                 } else if (shardRouting.relocating()) {
 
-                    // the shard is relocating to another node, cancel the recovery on the other node, and deallocate this one
                     if (!allowPrimary && shardRouting.primary()) {
                         // can't cancel a primary shard being initialized
                         if (explain) {
@@ -156,17 +155,30 @@ public class CancelAllocationCommand implements AllocationCommand {
                         throw new IllegalArgumentException("[cancel_allocation] can't cancel " + shardId + " on node " +
                                 discoNode + ", shard is primary and initializing its state");
                     }
-                    shardRouting = it.moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.REROUTE_CANCELLED, null));
-                    // now, go and find the shard that is initializing on the target node, and cancel it as well...
+                    it.remove();
+                    boolean addAsUnassigned = true;
+                    // now, find the shard that is initializing on the target node
                     RoutingNodes.RoutingNodeIterator initializingNode = allocation.routingNodes().routingNodeIter(shardRouting.relocatingNodeId());
                     if (initializingNode != null) {
                         while (initializingNode.hasNext()) {
                             ShardRouting initializingShardRouting = initializingNode.next();
                             if (initializingShardRouting.isRelocationTargetOf(shardRouting)) {
-                                initializingNode.remove();
+                                if (shardRouting.primary()) {
+                                    // cancel and remove target shard
+                                    initializingNode.remove();
+                                } else {
+                                    // promote to initializing shard without relocation source and ensure that removed relocation source
+                                    // is not added back as unassigned shard
+                                    initializingNode.removeRelocationSource();
+                                    addAsUnassigned = false;
+                                }
+                                break;
                             }
                         }
                     }
+                    if (addAsUnassigned) {
+                        it.moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.REROUTE_CANCELLED, null));
+                    }
                 }
             } else {
                 // the shard is not relocating, its either started, or initializing, just cancel it and move on...
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
index b63692e0d2a..0d637e451f0 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
@@ -59,6 +59,7 @@ import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
 import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
 
 /**
  */
@@ -381,15 +382,42 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
         assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
         assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).size(), equalTo(1));
 
+        logger.info("--> move the replica shard again");
+        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", 0, "node2", "node3")), false, false);
+        clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
+        assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(RELOCATING).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState(INITIALIZING).size(), equalTo(1));
+
+        logger.info("--> cancel the source replica shard");
+        rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new CancelAllocationCommand("test", 0, "node2", false)), false, false);
+        clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
+        assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(0));
+        assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState(INITIALIZING).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState(INITIALIZING).get(0).relocatingNodeId(), nullValue());
+
+        logger.info("--> start the former target replica shard");
+        rerouteResult = allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
+        clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
+        assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node1").shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(0));
+        assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState(STARTED).size(), equalTo(1));
+
 
         logger.info("--> cancel the primary allocation (with allow_primary set to true)");
         rerouteResult = allocation.reroute(clusterState, new AllocationCommands(new CancelAllocationCommand("test", 0, "node1", true)), false, false);
         clusterState = ClusterState.builder(clusterState).routingTable(rerouteResult.routingTable()).build();
         assertThat(rerouteResult.changed(), equalTo(true));
-        logger.error(clusterState.prettyPrint());
-        assertThat(clusterState.getRoutingNodes().node("node2").shardsWithState(STARTED).iterator().next().primary(), equalTo(true));
+        assertThat(clusterState.getRoutingNodes().node("node3").shardsWithState(STARTED).iterator().next().primary(), equalTo(true));
         assertThat(clusterState.getRoutingNodes().node("node1").size(), equalTo(0));
-        assertThat(clusterState.getRoutingNodes().node("node3").size(), equalTo(0));
+        assertThat(clusterState.getRoutingNodes().node("node2").size(), equalTo(0));
     }
 
     public void testSerialization() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
index 77cffc20ae1..625cb559761 100644
--- a/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java
@@ -408,6 +408,28 @@ public class IndexRecoveryIT extends ESIntegTestCase {
         assertOnGoingRecoveryState(nodeCRecoveryStates.get(0), 0, Type.REPLICA, nodeB, nodeC, false);
         validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
 
+        if (randomBoolean()) {
+            // shutdown node with relocation source of replica shard and check if recovery continues
+            internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeA));
+            ensureStableCluster(2);
+
+            response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
+            recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
+
+            nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
+            assertThat(nodeARecoveryStates.size(), equalTo(0));
+            nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
+            assertThat(nodeBRecoveryStates.size(), equalTo(1));
+            nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
+            assertThat(nodeCRecoveryStates.size(), equalTo(1));
+
+            assertRecoveryState(nodeBRecoveryStates.get(0), 0, Type.PRIMARY_RELOCATION, Stage.DONE, nodeA, nodeB, false);
+            validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
+
+            assertOnGoingRecoveryState(nodeCRecoveryStates.get(0), 0, Type.REPLICA, nodeB, nodeC, false);
+            validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
+        }
+
         logger.info("--> speeding up recoveries");
         restoreRecoverySpeed();
         ensureGreen();

From 98fd45dc0282e0c81e0fad2d2a536f13dc68ac85 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Fri, 27 May 2016 15:31:40 +0200
Subject: [PATCH 0502/1311] Move DocStats under Engine to get more accurate
 numbers (#18587)

Today we pull doc stats from an index reader which might not reflect reality.
IndexWriter might have merged all deletes away but due to a missing refresh
the stats are completely off. This change pulls doc stats from the IndexWriter
directly instead of relying on refreshes to run regularly. Note: Buffered deletes
are still not visible until the segments are flushed to disk.
---
 .../elasticsearch/index/engine/Engine.java    | 11 +++++++
 .../index/engine/InternalEngine.java          |  8 +++++
 .../elasticsearch/index/shard/IndexShard.java |  6 ++--
 .../index/engine/InternalEngineTests.java     | 28 +++++++++++++++++
 .../index/engine/ShadowEngineTests.java       | 31 +++++++++++++++++++
 5 files changed, 81 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/engine/Engine.java b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
index eeda4b468d1..76f9a5f8a81 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -60,6 +60,7 @@ import org.elasticsearch.index.mapper.ParseContext.Document;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.merge.MergeStats;
+import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
@@ -1160,6 +1161,16 @@ public abstract class Engine implements Closeable {
         return this.lastWriteNanos;
     }
 
+    /**
+     * Returns the engines current document statistics
+     */
+    public DocsStats getDocStats() {
+        try (Engine.Searcher searcher = acquireSearcher("doc_stats")) {
+            IndexReader reader = searcher.reader();
+            return new DocsStats(reader.numDocs(), reader.numDeletedDocs());
+        }
+    }
+
     /**
      * Called for each new opened engine searcher to warm new segments
      * @see EngineConfig#getWarmer()
diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index 41363bdace0..b8ad093486c 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -56,6 +56,7 @@ import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.merge.MergeStats;
 import org.elasticsearch.index.merge.OnGoingMerge;
+import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.shard.ElasticsearchMergePolicy;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.TranslogRecoveryPerformer;
@@ -1132,4 +1133,11 @@ public class InternalEngine extends Engine {
     public MergeStats getMergeStats() {
         return mergeScheduler.stats();
     }
+
+    @Override
+    public DocsStats getDocStats() {
+        final int numDocs = indexWriter.numDocs();
+        final int maxDoc = indexWriter.maxDoc();
+        return new DocsStats(numDocs, maxDoc-numDocs);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 28ecf290466..d4a147d475b 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -614,9 +614,9 @@ public class IndexShard extends AbstractIndexShardComponent {
     }
 
     public DocsStats docStats() {
-        try (Engine.Searcher searcher = acquireSearcher("doc_stats")) {
-            return new DocsStats(searcher.reader().numDocs(), searcher.reader().numDeletedDocs());
-        }
+        readAllowed();
+        final Engine engine = getEngine();
+        return engine.getDocStats();
     }
 
     /**
diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 44f0b4a7263..8956c54f0ee 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -81,6 +81,7 @@ import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
 import org.elasticsearch.index.mapper.internal.UidFieldMapper;
 import org.elasticsearch.index.mapper.object.RootObjectMapper;
+import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.shard.IndexSearcherWrapper;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardUtils;
@@ -2144,4 +2145,31 @@ public class InternalEngineTests extends ESTestCase {
             }
         }
     }
+
+    public void testDocStats() throws IOException {
+        final int numDocs = randomIntBetween(1, 10);
+        for (int i = 0; i < numDocs; i++) {
+            ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null);
+            Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(i)), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime());
+            engine.index(firstIndexRequest);
+            assertThat(firstIndexRequest.version(), equalTo(1L));
+        }
+        DocsStats docStats = engine.getDocStats();
+        assertEquals(numDocs, docStats.getCount());
+        assertEquals(0, docStats.getDeleted());
+        engine.forceMerge(randomBoolean(), 1, false, false, false);
+
+        ParsedDocument doc = testParsedDocument(Integer.toString(0), Integer.toString(0), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null);
+        Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(0)), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime());
+        engine.index(firstIndexRequest);
+        assertThat(firstIndexRequest.version(), equalTo(2L));
+        engine.flush(); // flush - buffered deletes are not counted
+        docStats = engine.getDocStats();
+        assertEquals(1, docStats.getDeleted());
+        assertEquals(numDocs, docStats.getCount());
+        engine.forceMerge(randomBoolean(), 1, false, false, false);
+        docStats = engine.getDocStats();
+        assertEquals(0, docStats.getDeleted());
+        assertEquals(numDocs, docStats.getCount());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
index 8a5dcea5d6d..8595f235c0e 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
@@ -43,16 +43,19 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.codec.CodecService;
 import org.elasticsearch.index.mapper.Mapping;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
 import org.elasticsearch.index.mapper.internal.UidFieldMapper;
+import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardUtils;
 import org.elasticsearch.index.store.DirectoryService;
@@ -75,6 +78,7 @@ import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.elasticsearch.index.engine.Engine.Operation.Origin.PRIMARY;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasKey;
@@ -980,4 +984,31 @@ public class ShadowEngineTests extends ESTestCase {
             // all good
         }
     }
+
+    public void testDocStats() throws IOException {
+        final int numDocs = randomIntBetween(1, 10);
+        for (int i = 0; i < numDocs; i++) {
+            ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null);
+            Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(i)), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime());
+            primaryEngine.index(firstIndexRequest);
+            assertThat(firstIndexRequest.version(), equalTo(1L));
+        }
+        DocsStats docStats = primaryEngine.getDocStats();
+        assertEquals(numDocs, docStats.getCount());
+        assertEquals(0, docStats.getDeleted());
+
+        docStats = replicaEngine.getDocStats();
+        assertEquals(0, docStats.getCount());
+        assertEquals(0, docStats.getDeleted());
+        primaryEngine.flush();
+
+        docStats = replicaEngine.getDocStats();
+        assertEquals(0, docStats.getCount());
+        assertEquals(0, docStats.getDeleted());
+        replicaEngine.refresh("test");
+        docStats = replicaEngine.getDocStats();
+        assertEquals(numDocs, docStats.getCount());
+        assertEquals(0, docStats.getDeleted());
+        primaryEngine.forceMerge(randomBoolean(), 1, false, false, false);
+    }
 }

From df7b3c6e82a77608851019f0236fb7769751e367 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 27 May 2016 09:43:19 -0400
Subject: [PATCH 0503/1311] Fix translog replay multiple operations same doc

Modifying the translog replay to not replay again into the translog
introduced a bug for the case of multiple operations for the same
doc. Namely, since we were no longer updating the version map for each
operation, the second operation for a doc would be treated as a creation
instead of as an update. This commit fixes this bug by placing these
operations into version map. This commit includes a failing test case.

Relates #18611
---
 .../index/engine/InternalEngine.java          | 12 ++++++
 .../index/engine/VersionValue.java            |  5 ++-
 .../index/engine/InternalEngineTests.java     | 37 +++++++++++++++++++
 3 files changed, 52 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index b8ad093486c..cf96fb06358 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -395,6 +395,12 @@ public class InternalEngine extends Engine {
                 final Translog.Location translogLocation = translog.add(new Translog.Index(index));
                 index.setTranslogLocation(translogLocation);
                 versionMap.putUnderLock(index.uid().bytes(), new VersionValue(updatedVersion, index.getTranslogLocation()));
+            } else {
+                // we do not replay in to the translog, so there is no
+                // translog location; that is okay because real-time
+                // gets are not possible during recovery and we will
+                // flush when the recovery is complete
+                versionMap.putUnderLock(index.uid().bytes(), new VersionValue(updatedVersion, null));
             }
 
             return created;
@@ -498,6 +504,12 @@ public class InternalEngine extends Engine {
                 final Translog.Location translogLocation = translog.add(new Translog.Delete(delete));
                 delete.setTranslogLocation(translogLocation);
                 versionMap.putUnderLock(delete.uid().bytes(), new DeleteVersionValue(updatedVersion, engineConfig.getThreadPool().estimatedTimeInMillis(), delete.getTranslogLocation()));
+            } else {
+                // we do not replay in to the translog, so there is no
+                // translog location; that is okay because real-time
+                // gets are not possible during recovery and we will
+                // flush when the recovery is complete
+                versionMap.putUnderLock(delete.uid().bytes(), new DeleteVersionValue(updatedVersion, engineConfig.getThreadPool().estimatedTimeInMillis(), null));
             }
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/index/engine/VersionValue.java b/core/src/main/java/org/elasticsearch/index/engine/VersionValue.java
index 6b780c2a6a3..20116a47b1d 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/VersionValue.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/VersionValue.java
@@ -54,9 +54,10 @@ class VersionValue implements Accountable {
 
     @Override
     public long ramBytesUsed() {
-        return RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + Long.BYTES + RamUsageEstimator.NUM_BYTES_OBJECT_REF + translogLocation.ramBytesUsed();
+        return RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + Long.BYTES + RamUsageEstimator.NUM_BYTES_OBJECT_REF +
+            (translogLocation != null ? translogLocation.size : 0);
     }
-    
+
     @Override
     public Collection<Accountable> getChildResources() {
         return Collections.emptyList();
diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 8956c54f0ee..4246e075b4c 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -44,6 +44,7 @@ import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TotalHitCountCollector;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MockDirectoryWrapper;
@@ -592,6 +593,42 @@ public class InternalEngineTests extends ESTestCase {
         engine.flush();
     }
 
+    public void testTranslogMultipleOperationsSameDocument() throws IOException {
+        final int ops = randomIntBetween(1, 32);
+        Engine initialEngine;
+        final List<Engine.Operation> operations = new ArrayList<>();
+        try {
+            initialEngine = engine;
+            for (int i = 0; i < ops; i++) {
+                final ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
+                if (randomBoolean()) {
+                    final Engine.Index operation = new Engine.Index(newUid("test#1"), doc, i, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime());
+                    operations.add(operation);
+                    initialEngine.index(operation);
+                } else {
+                    final Engine.Delete operation = new Engine.Delete("test", "1", newUid("test#1"), i, VersionType.EXTERNAL, Engine.Operation.Origin.PRIMARY, System.nanoTime(), false);
+                    operations.add(operation);
+                    initialEngine.delete(operation);
+                }
+            }
+        } finally {
+            IOUtils.close(engine);
+        }
+
+        Engine recoveringEngine = null;
+        try {
+            recoveringEngine = new InternalEngine(copy(engine.config(), EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG));
+            recoveringEngine.recoverFromTranslog();
+            try (Engine.Searcher searcher = recoveringEngine.acquireSearcher("test")) {
+                final TotalHitCountCollector collector = new TotalHitCountCollector();
+                searcher.searcher().search(new MatchAllDocsQuery(), collector);
+                assertThat(collector.getTotalHits(), equalTo(operations.get(operations.size() - 1) instanceof Engine.Delete ? 0 : 1));
+            }
+        } finally {
+            IOUtils.close(recoveringEngine);
+        }
+    }
+
     public void testTranslogRecoveryDoesNotReplayIntoTranslog() throws IOException {
         final int docs = randomIntBetween(1, 32);
         Engine initialEngine = null;

From 8ff0077b52d11f2e41ac8b10deaeafd54ddce353 Mon Sep 17 00:00:00 2001
From: Alexander Kazakov <alexshadow007@gmail.com>
Date: Fri, 27 May 2016 17:44:58 +0300
Subject: [PATCH 0504/1311] Update repository-gcs.asciidoc

Fixed link
---
 docs/plugins/repository-gcs.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/plugins/repository-gcs.asciidoc b/docs/plugins/repository-gcs.asciidoc
index bed78b4cbbf..ea2cb99ee9b 100644
--- a/docs/plugins/repository-gcs.asciidoc
+++ b/docs/plugins/repository-gcs.asciidoc
@@ -157,7 +157,7 @@ The service account used to access the bucket must have the "Writer" access to t
 
 1. Connect to the https://console.cloud.google.com/[Google Cloud Platform Console]
 2. Select your project
-3. Got to the [https://console.cloud.google.com/storage/browser]Storage Browser
+3. Got to the https://console.cloud.google.com/storage/browser[Storage Browser]
 4. Select the bucket and "Edit bucket permission"
 5. The service account must be configured as a "User" with "Writer" access
 

From 46162a40e70949d35c35272d2fc0e487a5c2010b Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 27 May 2016 10:57:23 -0400
Subject: [PATCH 0505/1311] Additional bootstrap check doc fixes

This commit fixes some additional poorly-formatted internal and external
links in the bootstrap check docs.
---
 .../reference/setup/bootstrap-checks.asciidoc | 54 ++++++++++---------
 1 file changed, 29 insertions(+), 25 deletions(-)

diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc
index 8a5c215f3d2..bfa43727d28 100644
--- a/docs/reference/setup/bootstrap-checks.asciidoc
+++ b/docs/reference/setup/bootstrap-checks.asciidoc
@@ -33,23 +33,24 @@ If a JVM is started with unequal initial and max heap size, it can be
 prone to pauses as the JVM heap is resized during system usage. To avoid
 these resize pauses, it's best to start the JVM with the initial heap
 size equal to the maximum heap size. Additionally, if
-<<bootstrap.mlockall>> is enabled, the JVM will lock the initial size of
-the heap on startup. If the initial heap size is not equal to the
-maximum heap size, after a resize it will not be the case that all of
-the JVM heap is locked in memory. To pass the heap size check, you must
-configure the <<heap-size,heap size>>.
+<<bootstrap.mlockall,`bootstrap.mlockall`>> is enabled, the JVM will
+lock the initial size of the heap on startup. If the initial heap size
+is not equal to the maximum heap size, after a resize it will not be the
+case that all of the JVM heap is locked in memory. To pass the heap size
+check, you must configure the <<heap-size,heap size>>.
+
 
 === File descriptor check
 
 File descriptors are a Unix construct for tracking open "files". In Unix
-though, [everything is a
-file](https://en.wikipedia.org/wiki/Everything_is_a_file). For example,
-"files" could be a physical file, a virtual file (e.g.,
-``/proc/loadavg`), or network sockets. Elasticsearch requires lots file
-descriptors (e.g., every shard is composed of multiple segments and
-other files, plus connections to other nodes, etc.). This bootstrap
-check is enforced on OS X and Linux. To pass the file descriptor check,
-you might have to configure <<file-descriptors,file descriptors>>.
+though, https://en.wikipedia.org/wiki/Everything_is_a_file[everything is
+a file]. For example, "files" could be a physical file, a virtual file
+(e.g., ``/proc/loadavg`), or network sockets. Elasticsearch requires
+lots file descriptors (e.g., every shard is composed of multiple
+segments and other files, plus connections to other nodes, etc.). This
+bootstrap check is enforced on OS X and Linux. To pass the file
+descriptor check, you might have to configure <<file-descriptors,file
+descriptors>>.
 
 === Memory lock check
 
@@ -78,21 +79,24 @@ knowing that there is a master on the side of the partition. This can
 lead to divergent cluster states potentially leading to data loss when
 the partition is healed. This is the notion of a split brain and it is
 the worst thing that can happen to an Elasticsearch cluster. But by
-configuring <<minimum_master_nodes>> to be equal to a quorum of
-master-eligible nodes, it is not possible for the cluster to suffer from
-split brain because during a network partition there can be at most one
-side of the partition that contains a quorum of master nodes. The
-minimum master nodes check enforces that you've set
+configuring
+<<minimum_master_nodes,`discovery.zen.minimum_master_nodes`>> to be
+equal to a quorum of master-eligible nodes, it is not possible for the
+cluster to suffer from split brain because during a network partition
+there can be at most one side of the partition that contains a quorum of
+master nodes. The minimum master nodes check enforces that you've set
 <<minimum_master_nodes>>. To pass the minimum master nodes check, you
-must configure <<minimum_master_nodes>>.
+must configure
+<<minimum_master_nodes,`discovery.zen.minimum_master_nodes`>>.
 
 NOTE: The minimum master nodes check does not enforce that you've
-configured <<minimum_master_nodes>> correctly, only that you have it
-configured. Elasticsearch does log a warning message if it detects that
-<<minimum_master_nodes>> is incorrectly configured based on the number
-of master-eligible nodes visible in the cluster state. Future versions
-of Elasticsearch will contain stricter enforcement of
-<<minimum_master_nodes>>.
+configured <<minimum_master_nodes,`discovery.zen.minimum_master_nodes`>>
+correctly, only that you have it configured. Elasticsearch does log a
+warning message if it detects that <<minimum_master_nodes>> is
+incorrectly configured based on the number of master-eligible nodes
+visible in the cluster state. Future versions of Elasticsearch will
+contain stricter enforcement of
+<<minimum_master_nodes,`discovery.zen.minimum_master_nodes`>>.
 
 === Maximum number of threads check
 

From fbd3f8df2b62cc6548d24771209dba486f0597f8 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Fri, 27 May 2016 14:36:00 +0200
Subject: [PATCH 0506/1311] percolator: Don't bail query term extraction when a
 must clause contains an unsupported query, if there is at least one other
 must clause in a boolean query that can be extracted.

---
 .../percolator/ExtractQueryTermsService.java  | 14 ++-
 .../ExtractQueryTermsServiceTests.java        | 96 ++++++++++++-------
 2 files changed, 72 insertions(+), 38 deletions(-)

diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java
index f0c61b74e13..f9f1aa575b7 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java
@@ -116,7 +116,7 @@ public final class ExtractQueryTermsService {
             for (BytesRef term = iterator.next(); term != null; term = iterator.next()) {
                 terms.add(new Term(iterator.field(), term));
             }
-            return  terms;
+            return terms;
         } else if (query instanceof PhraseQuery) {
             Term[] terms = ((PhraseQuery) query).getTerms();
             if (terms.length == 0) {
@@ -142,6 +142,7 @@ public final class ExtractQueryTermsService {
                 }
             }
             if (hasRequiredClauses) {
+                UnsupportedQueryException uqe = null;
                 Set<Term> bestClause = null;
                 for (BooleanClause clause : clauses) {
                     if (clause.isRequired() == false) {
@@ -151,12 +152,21 @@ public final class ExtractQueryTermsService {
                         continue;
                     }
 
-                    Set<Term> temp = extractQueryTerms(clause.getQuery());
+                    Set<Term> temp;
+                    try {
+                        temp = extractQueryTerms(clause.getQuery());
+                    } catch (UnsupportedQueryException e) {
+                        uqe = e;
+                        continue;
+                    }
                     bestClause = selectTermListWithTheLongestShortestTerm(temp, bestClause);
                 }
                 if (bestClause != null) {
                     return bestClause;
                 } else {
+                    if (uqe != null) {
+                        throw uqe;
+                    }
                     return Collections.emptySet();
                 }
             } else {
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java
index 4245c99ba18..444e47d90ce 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java
@@ -53,6 +53,10 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import static org.elasticsearch.percolator.ExtractQueryTermsService.UnsupportedQueryException;
+import static org.elasticsearch.percolator.ExtractQueryTermsService.extractQueryTerms;
+import static org.elasticsearch.percolator.ExtractQueryTermsService.createQueryTermsQuery;
+import static org.elasticsearch.percolator.ExtractQueryTermsService.selectTermListWithTheLongestShortestTerm;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.sameInstance;
 
@@ -76,7 +80,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         bq.add(termQuery2, BooleanClause.Occur.SHOULD);
 
         ParseContext.Document document = new ParseContext.Document();
-        ExtractQueryTermsService.extractQueryTerms(bq.build(), document, QUERY_TERMS_FIELD, UNKNOWN_QUERY_FIELD, QUERY_TERMS_FIELD_TYPE);
+        extractQueryTerms(bq.build(), document, QUERY_TERMS_FIELD, UNKNOWN_QUERY_FIELD, QUERY_TERMS_FIELD_TYPE);
         Collections.sort(document.getFields(), (field1, field2) -> field1.binaryValue().compareTo(field2.binaryValue()));
         assertThat(document.getFields().size(), equalTo(2));
         assertThat(document.getFields().get(0).name(), equalTo(QUERY_TERMS_FIELD));
@@ -94,7 +98,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
 
         TermRangeQuery query = new TermRangeQuery("field1", new BytesRef("a"), new BytesRef("z"), true, true);
         ParseContext.Document document = new ParseContext.Document();
-        ExtractQueryTermsService.extractQueryTerms(query, document, QUERY_TERMS_FIELD, UNKNOWN_QUERY_FIELD, QUERY_TERMS_FIELD_TYPE);
+        extractQueryTerms(query, document, QUERY_TERMS_FIELD, UNKNOWN_QUERY_FIELD, QUERY_TERMS_FIELD_TYPE);
         assertThat(document.getFields().size(), equalTo(1));
         assertThat(document.getFields().get(0).name(), equalTo(UNKNOWN_QUERY_FIELD));
         assertThat(document.getFields().get(0).binaryValue().utf8ToString(), equalTo(""));
@@ -102,7 +106,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
 
     public void testExtractQueryMetadata_termQuery() {
         TermQuery termQuery = new TermQuery(new Term("_field", "_term"));
-        List<Term> terms = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(termQuery));
+        List<Term> terms = new ArrayList<>(extractQueryTerms(termQuery));
         assertThat(terms.size(), equalTo(1));
         assertThat(terms.get(0).field(), equalTo(termQuery.getTerm().field()));
         assertThat(terms.get(0).bytes(), equalTo(termQuery.getTerm().bytes()));
@@ -110,7 +114,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
 
     public void testExtractQueryMetadata_termsQuery() {
         TermsQuery termsQuery = new TermsQuery("_field", new BytesRef("_term1"), new BytesRef("_term2"));
-        List<Term> terms = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(termsQuery));
+        List<Term> terms = new ArrayList<>(extractQueryTerms(termsQuery));
         Collections.sort(terms);
         assertThat(terms.size(), equalTo(2));
         assertThat(terms.get(0).field(), equalTo("_field"));
@@ -120,7 +124,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
 
         // test with different fields
         termsQuery = new TermsQuery(new Term("_field1", "_term1"), new Term("_field2", "_term2"));
-        terms = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(termsQuery));
+        terms = new ArrayList<>(extractQueryTerms(termsQuery));
         Collections.sort(terms);
         assertThat(terms.size(), equalTo(2));
         assertThat(terms.get(0).field(), equalTo("_field1"));
@@ -131,7 +135,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
 
     public void testExtractQueryMetadata_phraseQuery() {
         PhraseQuery phraseQuery = new PhraseQuery("_field", "_term1", "term2");
-        List<Term> terms = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(phraseQuery));
+        List<Term> terms = new ArrayList<>(extractQueryTerms(phraseQuery));
         assertThat(terms.size(), equalTo(1));
         assertThat(terms.get(0).field(), equalTo(phraseQuery.getTerms()[0].field()));
         assertThat(terms.get(0).bytes(), equalTo(phraseQuery.getTerms()[0].bytes()));
@@ -152,7 +156,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         builder.add(subBuilder.build(), BooleanClause.Occur.SHOULD);
 
         BooleanQuery booleanQuery = builder.build();
-        List<Term> terms = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(booleanQuery));
+        List<Term> terms = new ArrayList<>(extractQueryTerms(booleanQuery));
         Collections.sort(terms);
         assertThat(terms.size(), equalTo(3));
         assertThat(terms.get(0).field(), equalTo(termQuery1.getTerm().field()));
@@ -178,7 +182,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         builder.add(subBuilder.build(), BooleanClause.Occur.SHOULD);
 
         BooleanQuery booleanQuery = builder.build();
-        List<Term> terms = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(booleanQuery));
+        List<Term> terms = new ArrayList<>(extractQueryTerms(booleanQuery));
         Collections.sort(terms);
         assertThat(terms.size(), equalTo(4));
         assertThat(terms.get(0).field(), equalTo(termQuery1.getTerm().field()));
@@ -199,7 +203,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         builder.add(phraseQuery, BooleanClause.Occur.SHOULD);
 
         BooleanQuery booleanQuery = builder.build();
-        List<Term> terms = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(booleanQuery));
+        List<Term> terms = new ArrayList<>(extractQueryTerms(booleanQuery));
         assertThat(terms.size(), equalTo(1));
         assertThat(terms.get(0).field(), equalTo(phraseQuery.getTerms()[0].field()));
         assertThat(terms.get(0).bytes(), equalTo(phraseQuery.getTerms()[0].bytes()));
@@ -208,7 +212,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
     public void testExtractQueryMetadata_constantScoreQuery() {
         TermQuery termQuery1 = new TermQuery(new Term("_field", "_term"));
         ConstantScoreQuery constantScoreQuery = new ConstantScoreQuery(termQuery1);
-        List<Term> terms = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(constantScoreQuery));
+        List<Term> terms = new ArrayList<>(extractQueryTerms(constantScoreQuery));
         assertThat(terms.size(), equalTo(1));
         assertThat(terms.get(0).field(), equalTo(termQuery1.getTerm().field()));
         assertThat(terms.get(0).bytes(), equalTo(termQuery1.getTerm().bytes()));
@@ -217,7 +221,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
     public void testExtractQueryMetadata_boostQuery() {
         TermQuery termQuery1 = new TermQuery(new Term("_field", "_term"));
         BoostQuery constantScoreQuery = new BoostQuery(termQuery1, 1f);
-        List<Term> terms = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(constantScoreQuery));
+        List<Term> terms = new ArrayList<>(extractQueryTerms(constantScoreQuery));
         assertThat(terms.size(), equalTo(1));
         assertThat(terms.get(0).field(), equalTo(termQuery1.getTerm().field()));
         assertThat(terms.get(0).bytes(), equalTo(termQuery1.getTerm().bytes()));
@@ -227,7 +231,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         CommonTermsQuery commonTermsQuery = new CommonTermsQuery(BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, 100);
         commonTermsQuery.add(new Term("_field", "_term1"));
         commonTermsQuery.add(new Term("_field", "_term2"));
-        List<Term> terms = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(commonTermsQuery));
+        List<Term> terms = new ArrayList<>(extractQueryTerms(commonTermsQuery));
         Collections.sort(terms);
         assertThat(terms.size(), equalTo(2));
         assertThat(terms.get(0).field(), equalTo("_field"));
@@ -239,7 +243,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
     public void testExtractQueryMetadata_blendedTermQuery() {
         Term[] terms = new Term[]{new Term("_field", "_term1"), new Term("_field", "_term2")};
         BlendedTermQuery commonTermsQuery = BlendedTermQuery.booleanBlendedQuery(terms, false);
-        List<Term> result = new ArrayList<>(ExtractQueryTermsService.extractQueryTerms(commonTermsQuery));
+        List<Term> result = new ArrayList<>(extractQueryTerms(commonTermsQuery));
         Collections.sort(result);
         assertThat(result.size(), equalTo(2));
         assertThat(result.get(0).field(), equalTo("_field"));
@@ -261,7 +265,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         // 4) FieldMaskingSpanQuery is a tricky query so we shouldn't optimize this
 
         SpanTermQuery spanTermQuery1 = new SpanTermQuery(new Term("_field", "_short_term"));
-        Set<Term> terms = ExtractQueryTermsService.extractQueryTerms(spanTermQuery1);
+        Set<Term> terms = extractQueryTerms(spanTermQuery1);
         assertTermsEqual(terms, spanTermQuery1.getTerm());
     }
 
@@ -270,7 +274,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         SpanTermQuery spanTermQuery2 = new SpanTermQuery(new Term("_field", "_very_long_term"));
         SpanNearQuery spanNearQuery = new SpanNearQuery.Builder("_field", true)
                 .addClause(spanTermQuery1).addClause(spanTermQuery2).build();
-        Set<Term> terms = ExtractQueryTermsService.extractQueryTerms(spanNearQuery);
+        Set<Term> terms = extractQueryTerms(spanNearQuery);
         assertTermsEqual(terms, spanTermQuery2.getTerm());
     }
 
@@ -278,14 +282,14 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         SpanTermQuery spanTermQuery1 = new SpanTermQuery(new Term("_field", "_short_term"));
         SpanTermQuery spanTermQuery2 = new SpanTermQuery(new Term("_field", "_very_long_term"));
         SpanOrQuery spanOrQuery = new SpanOrQuery(spanTermQuery1, spanTermQuery2);
-        Set<Term> terms = ExtractQueryTermsService.extractQueryTerms(spanOrQuery);
+        Set<Term> terms = extractQueryTerms(spanOrQuery);
         assertTermsEqual(terms, spanTermQuery1.getTerm(), spanTermQuery2.getTerm());
     }
 
     public void testExtractQueryMetadata_spanFirstQuery() {
         SpanTermQuery spanTermQuery1 = new SpanTermQuery(new Term("_field", "_short_term"));
         SpanFirstQuery spanFirstQuery = new SpanFirstQuery(spanTermQuery1, 20);
-        Set<Term> terms = ExtractQueryTermsService.extractQueryTerms(spanFirstQuery);
+        Set<Term> terms = extractQueryTerms(spanFirstQuery);
         assertTermsEqual(terms, spanTermQuery1.getTerm());
     }
 
@@ -293,36 +297,31 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         SpanTermQuery spanTermQuery1 = new SpanTermQuery(new Term("_field", "_short_term"));
         SpanTermQuery spanTermQuery2 = new SpanTermQuery(new Term("_field", "_very_long_term"));
         SpanNotQuery spanNotQuery = new SpanNotQuery(spanTermQuery1, spanTermQuery2);
-        Set<Term> terms = ExtractQueryTermsService.extractQueryTerms(spanNotQuery);
+        Set<Term> terms = extractQueryTerms(spanNotQuery);
         assertTermsEqual(terms, spanTermQuery1.getTerm());
     }
 
     public void testExtractQueryMetadata_matchNoDocsQuery() {
-        Set<Term> terms = ExtractQueryTermsService.extractQueryTerms(new MatchNoDocsQuery("sometimes there is no reason at all"));
+        Set<Term> terms = extractQueryTerms(new MatchNoDocsQuery("sometimes there is no reason at all"));
         assertEquals(0, terms.size());
 
         BooleanQuery.Builder bq = new BooleanQuery.Builder();
         bq.add(new TermQuery(new Term("field", "value")), BooleanClause.Occur.MUST);
         bq.add(new MatchNoDocsQuery("sometimes there is no reason at all"), BooleanClause.Occur.MUST);
-        terms = ExtractQueryTermsService.extractQueryTerms(bq.build());
+        terms = extractQueryTerms(bq.build());
         assertEquals(0, terms.size());
 
         bq = new BooleanQuery.Builder();
         bq.add(new TermQuery(new Term("field", "value")), BooleanClause.Occur.SHOULD);
         bq.add(new MatchNoDocsQuery("sometimes there is no reason at all"), BooleanClause.Occur.SHOULD);
-        terms = ExtractQueryTermsService.extractQueryTerms(bq.build());
+        terms = extractQueryTerms(bq.build());
         assertTermsEqual(terms, new Term("field", "value"));
     }
 
     public void testExtractQueryMetadata_unsupportedQuery() {
         TermRangeQuery termRangeQuery = new TermRangeQuery("_field", null, null, true, false);
-
-        try {
-            ExtractQueryTermsService.extractQueryTerms(termRangeQuery);
-            fail("UnsupportedQueryException expected");
-        } catch (ExtractQueryTermsService.UnsupportedQueryException e) {
-            assertThat(e.getUnsupportedQuery(), sameInstance(termRangeQuery));
-        }
+        UnsupportedQueryException e = expectThrows(UnsupportedQueryException.class, () -> extractQueryTerms(termRangeQuery));
+        assertThat(e.getUnsupportedQuery(), sameInstance(termRangeQuery));
 
         TermQuery termQuery1 = new TermQuery(new Term("_field", "_term"));
         BooleanQuery.Builder builder = new BooleanQuery.Builder();
@@ -330,12 +329,37 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         builder.add(termRangeQuery, BooleanClause.Occur.SHOULD);
         BooleanQuery bq = builder.build();
 
-        try {
-            ExtractQueryTermsService.extractQueryTerms(bq);
-            fail("UnsupportedQueryException expected");
-        } catch (ExtractQueryTermsService.UnsupportedQueryException e) {
-            assertThat(e.getUnsupportedQuery(), sameInstance(termRangeQuery));
-        }
+        e = expectThrows(UnsupportedQueryException.class, () -> extractQueryTerms(bq));
+        assertThat(e.getUnsupportedQuery(), sameInstance(termRangeQuery));
+    }
+
+    public void testExtractQueryMetadata_unsupportedQueryInBoolQueryWithMustClauses() {
+        TermRangeQuery unsupportedQuery = new TermRangeQuery("_field", null, null, true, false);
+
+        TermQuery termQuery1 = new TermQuery(new Term("_field", "_term"));
+        BooleanQuery.Builder builder = new BooleanQuery.Builder();
+        builder.add(termQuery1, BooleanClause.Occur.MUST);
+        builder.add(unsupportedQuery, BooleanClause.Occur.MUST);
+        BooleanQuery bq1 = builder.build();
+
+        Set<Term> terms = extractQueryTerms(bq1);
+        assertTermsEqual(terms, termQuery1.getTerm());
+
+        TermQuery termQuery2 = new TermQuery(new Term("_field", "_longer_term"));
+        builder = new BooleanQuery.Builder();
+        builder.add(termQuery1, BooleanClause.Occur.MUST);
+        builder.add(termQuery2, BooleanClause.Occur.MUST);
+        builder.add(unsupportedQuery, BooleanClause.Occur.MUST);
+        bq1 = builder.build();
+        terms = extractQueryTerms(bq1);
+        assertTermsEqual(terms, termQuery2.getTerm());
+
+        builder = new BooleanQuery.Builder();
+        builder.add(unsupportedQuery, BooleanClause.Occur.MUST);
+        builder.add(unsupportedQuery, BooleanClause.Occur.MUST);
+        BooleanQuery bq2 = builder.build();
+        UnsupportedQueryException e = expectThrows(UnsupportedQueryException.class, () -> extractQueryTerms(bq2));
+        assertThat(e.getUnsupportedQuery(), sameInstance(unsupportedQuery));
     }
 
     public void testCreateQueryMetadataQuery() throws Exception {
@@ -347,7 +371,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
 
         IndexReader indexReader = memoryIndex.createSearcher().getIndexReader();
         TermsQuery query = (TermsQuery)
-                ExtractQueryTermsService.createQueryTermsQuery(indexReader, QUERY_TERMS_FIELD, UNKNOWN_QUERY_FIELD);
+                createQueryTermsQuery(indexReader, QUERY_TERMS_FIELD, UNKNOWN_QUERY_FIELD);
 
         PrefixCodedTerms terms = query.getTermData();
         assertThat(terms.size(), equalTo(15L));
@@ -390,7 +414,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
             sumTermLength -= length;
         }
 
-        Set<Term> result = ExtractQueryTermsService.selectTermListWithTheLongestShortestTerm(terms1, terms2);
+        Set<Term> result = selectTermListWithTheLongestShortestTerm(terms1, terms2);
         Set<Term> expected = shortestTerms1Length >= shortestTerms2Length ? terms1 : terms2;
         assertThat(result, sameInstance(expected));
     }

From aa532854489566122cf7d34f097f6178d072600a Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Fri, 27 May 2016 15:07:27 -0400
Subject: [PATCH 0507/1311] add java.time packages to painless whitelist

---
 .../elasticsearch/painless/Definition.java    |   5 +
 .../painless/java.time.chrono.txt             | 336 +++++++++
 .../painless/java.time.format.txt             | 177 +++++
 .../painless/java.time.temporal.txt           | 225 ++++++
 .../org/elasticsearch/painless/java.time.txt  | 643 ++++++++++++++++++
 .../elasticsearch/painless/java.time.zone.txt |  93 +++
 .../org/elasticsearch/painless/java.util.txt  |   4 +
 7 files changed, 1483 insertions(+)
 create mode 100644 modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.chrono.txt
 create mode 100644 modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.format.txt
 create mode 100644 modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.temporal.txt
 create mode 100644 modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.txt
 create mode 100644 modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.zone.txt

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index ac8e7b89b79..0923cffc59d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -43,6 +43,11 @@ public final class Definition {
                       "java.lang.txt",
                       "java.math.txt",
                       "java.text.txt",
+                      "java.time.txt",
+                      "java.time.chrono.txt",
+                      "java.time.format.txt",
+                      "java.time.temporal.txt",
+                      "java.time.zone.txt",
                       "java.util.txt",
                       "java.util.function.txt",
                       "java.util.stream.txt"));
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.chrono.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.chrono.txt
new file mode 100644
index 00000000000..8dd0478605e
--- /dev/null
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.chrono.txt
@@ -0,0 +1,336 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#
+# Painless definition file. This defines the hierarchy of classes,
+# what methods and fields they have, etc.
+#
+
+#### Interfaces
+
+class ChronoLocalDate -> java.time.chrono.ChronoLocalDate extends Comparable,Temporal,TemporalAccessor,TemporalAdjuster {
+  ChronoLocalDateTime atTime(LocalTime)
+  boolean equals(Object)
+  String format(DateTimeFormatter)
+  ChronoLocalDate from(TemporalAccessor)
+  Chronology getChronology()
+  Era getEra()
+  int hashCode()
+  boolean isAfter(ChronoLocalDate)
+  boolean isBefore(ChronoLocalDate)
+  boolean isEqual(ChronoLocalDate)
+  boolean isLeapYear()
+  int lengthOfMonth()
+  int lengthOfYear()
+  ChronoLocalDate minus(TemporalAmount)
+  ChronoLocalDate minus(long,TemporalUnit)
+  ChronoLocalDate plus(TemporalAmount)
+  ChronoLocalDate plus(long,TemporalUnit)
+  Comparator timeLineOrder()
+  long toEpochDay()
+  String toString()
+  ChronoPeriod until(ChronoLocalDate)
+  ChronoLocalDate with(TemporalAdjuster)
+  ChronoLocalDate with(TemporalField,long)
+}
+
+class ChronoLocalDateTime -> java.time.chrono.ChronoLocalDateTime extends Comparable,Temporal,TemporalAccessor,TemporalAdjuster {
+  ChronoZonedDateTime atZone(ZoneId)
+  boolean equals(Object)
+  String format(DateTimeFormatter)
+  ChronoLocalDateTime from(TemporalAccessor)
+  int hashCode()
+  boolean isAfter(ChronoLocalDateTime)
+  boolean isBefore(ChronoLocalDateTime)
+  boolean isEqual(ChronoLocalDateTime)
+  Chronology getChronology()
+  ChronoLocalDateTime minus(TemporalAmount)
+  ChronoLocalDateTime minus(long,TemporalUnit)
+  ChronoLocalDateTime plus(TemporalAmount)
+  ChronoLocalDateTime plus(long,TemporalUnit)
+  Comparator timeLineOrder()
+  long toEpochSecond(ZoneOffset)
+  Instant toInstant(ZoneOffset)
+  ChronoLocalDate toLocalDate()
+  LocalTime toLocalTime()
+  String toString()
+  ChronoLocalDateTime with(TemporalAdjuster)
+  ChronoLocalDateTime with(TemporalField,long)
+}
+
+class Chronology -> java.time.chrono.Chronology extends Comparable {
+  ChronoLocalDate date(TemporalAccessor)
+  ChronoLocalDate date(Era,int,int,int)
+  ChronoLocalDate date(int,int,int)
+  ChronoLocalDate dateEpochDay(long)
+  ChronoLocalDate dateNow()
+  ChronoLocalDate dateNow(ZoneId)
+  ChronoLocalDate dateYearDay(Era,int,int)
+  ChronoLocalDate dateYearDay(int,int)
+  boolean equals(Object)
+  Era eraOf(int)
+  List eras()
+  Chronology from(TemporalAccessor)
+  Set getAvailableChronologies()
+  String getDisplayName(TextStyle,Locale)
+  String getId()
+  String getCalendarType()
+  int hashCode()
+  boolean isLeapYear(long)
+  ChronoLocalDateTime localDateTime(TemporalAccessor)
+  Chronology of(String)
+  Chronology ofLocale(Locale)
+  ChronoPeriod period(int,int,int)
+  int prolepticYear(Era,int)
+  ValueRange range(ChronoField)
+  ChronoLocalDate resolveDate(Map,ResolverStyle)
+  String toString()
+  ChronoZonedDateTime zonedDateTime(TemporalAccessor)
+  ChronoZonedDateTime zonedDateTime(Instant,ZoneId)
+}
+
+class ChronoPeriod -> java.time.chrono.ChronoPeriod extends TemporalAmount {
+  ChronoPeriod between(ChronoLocalDate,ChronoLocalDate)
+  boolean equals(Object)
+  Chronology getChronology()
+  List getUnits()
+  int hashCode()
+  boolean isNegative()
+  boolean isZero()
+  ChronoPeriod minus(TemporalAmount)
+  ChronoPeriod multipliedBy(int)
+  ChronoPeriod negated()
+  ChronoPeriod normalized()
+  ChronoPeriod plus(TemporalAmount)
+  String toString()
+}
+
+class ChronoZonedDateTime -> java.time.chrono.ChronoZonedDateTime extends Comparable,Temporal,TemporalAccessor {
+  boolean equals(Object)
+  String format(DateTimeFormatter)
+  ChronoZonedDateTime from(TemporalAccessor)
+  Chronology getChronology()
+  ZoneOffset getOffset()
+  ZoneId getZone()
+  int hashCode()
+  boolean isBefore(ChronoZonedDateTime)
+  boolean isAfter(ChronoZonedDateTime)
+  boolean isEqual(ChronoZonedDateTime)
+  ChronoZonedDateTime minus(TemporalAmount)
+  ChronoZonedDateTime minus(long,TemporalUnit)
+  ChronoZonedDateTime plus(TemporalAmount)
+  ChronoZonedDateTime plus(long,TemporalUnit)
+  Comparator timeLineOrder()
+  Instant toInstant()
+  long toEpochSecond()
+  ChronoLocalDate toLocalDate()
+  LocalTime toLocalTime()
+  ChronoLocalDateTime toLocalDateTime()
+  String toString()
+  ChronoZonedDateTime with(TemporalAdjuster)
+  ChronoZonedDateTime with(TemporalField,long)
+  ChronoZonedDateTime withEarlierOffsetAtOverlap()
+  ChronoZonedDateTime withLaterOffsetAtOverlap()
+  ChronoZonedDateTime withZoneSameLocal(ZoneId)
+  ChronoZonedDateTime withZoneSameInstant(ZoneId)
+}
+
+class Era -> java.time.chrono.Era extends TemporalAccessor,TemporalAdjuster {
+  String getDisplayName(TextStyle,Locale)
+  int getValue()
+}
+
+#### Classes
+
+class AbstractChronology -> java.time.chrono.Chronology extends Comparable,Chronology,Object {
+}
+
+class HijrahChronology -> java.time.chrono.HijrahChronology extends AbstractChronology,Comparable,Chronology,Object {
+  HijrahChronology INSTANCE
+  HijrahDate date(TemporalAccessor)
+  HijrahDate date(int,int,int)
+  HijrahDate date(Era,int,int,int)
+  HijrahDate dateEpochDay(long)
+  HijrahDate dateNow()
+  HijrahDate dateNow(ZoneId)
+  HijrahDate dateYearDay(int,int)
+  HijrahDate dateYearDay(Era,int,int)
+  HijrahEra eraOf(int)
+}
+
+class HijrahDate -> java.time.chrono.HijrahDate extends Comparable,ChronoLocalDate,Temporal,TemporalAccessor,TemporalAdjuster,Object {
+  HijrahDate now()
+  HijrahDate now(ZoneId)
+  HijrahDate of(int,int,int)
+  HijrahDate from(TemporalAccessor)
+  HijrahChronology getChronology()
+  HijrahEra getEra()
+  HijrahDate with(TemporalField,long)
+  HijrahDate with(TemporalAdjuster)
+  HijrahDate withVariant(HijrahChronology)
+  HijrahDate plus(TemporalAmount)
+  HijrahDate minus(TemporalAmount)
+  HijrahDate plus(long,TemporalUnit)
+  HijrahDate minus(long,TemporalUnit)
+}
+
+class IsoChronology -> java.time.chrono.IsoChronology extends AbstractChronology,Comparable,Chronology,Object {
+  IsoChronology INSTANCE
+  LocalDate date(TemporalAccessor)
+  LocalDate date(int,int,int)
+  LocalDate date(Era,int,int,int)
+  LocalDate dateEpochDay(long)
+  LocalDate dateNow()
+  LocalDate dateNow(ZoneId)
+  LocalDate dateYearDay(int,int)
+  LocalDate dateYearDay(Era,int,int)
+  IsoEra eraOf(int)
+}
+
+class JapaneseChronology -> java.time.chrono.JapaneseChronology extends AbstractChronology,Comparable,Chronology,Object {
+  JapaneseChronology INSTANCE
+  JapaneseDate date(TemporalAccessor)
+  JapaneseDate date(int,int,int)
+  JapaneseDate date(Era,int,int,int)
+  JapaneseDate dateEpochDay(long)
+  JapaneseDate dateNow()
+  JapaneseDate dateNow(ZoneId)
+  JapaneseDate dateYearDay(int,int)
+  JapaneseDate dateYearDay(Era,int,int)
+  JapaneseEra eraOf(int)
+}
+
+class JapaneseDate -> java.time.chrono.JapaneseDate extends Comparable,ChronoLocalDate,Temporal,TemporalAccessor,TemporalAdjuster,Object {
+  JapaneseDate now()
+  JapaneseDate now(ZoneId)
+  JapaneseDate of(int,int,int)
+  JapaneseDate from(TemporalAccessor)
+  JapaneseChronology getChronology()
+  JapaneseEra getEra()
+  JapaneseDate with(TemporalField,long)
+  JapaneseDate with(TemporalAdjuster)
+  JapaneseDate plus(TemporalAmount)
+  JapaneseDate minus(TemporalAmount)
+  JapaneseDate plus(long,TemporalUnit)
+  JapaneseDate minus(long,TemporalUnit)
+}
+
+class JapaneseEra -> java.time.chrono.JapaneseEra extends Era,TemporalAccessor,TemporalAdjuster,Object {
+  JapaneseEra HEISEI
+  JapaneseEra MEIJI
+  JapaneseEra SHOWA
+  JapaneseEra TAISHO
+  int getValue()
+  JapaneseEra of(int)
+  JapaneseEra valueOf(String)
+  JapaneseEra[] values()
+}
+
+class MinguoChronology -> java.time.chrono.MinguoChronology extends AbstractChronology,Comparable,Chronology,Object {
+  MinguoChronology INSTANCE
+  MinguoDate date(TemporalAccessor)
+  MinguoDate date(int,int,int)
+  MinguoDate date(Era,int,int,int)
+  MinguoDate dateEpochDay(long)
+  MinguoDate dateNow()
+  MinguoDate dateNow(ZoneId)
+  MinguoDate dateYearDay(int,int)
+  MinguoDate dateYearDay(Era,int,int)
+  MinguoEra eraOf(int)
+}
+
+class MinguoDate -> java.time.chrono.MinguoDate extends Comparable,ChronoLocalDate,Temporal,TemporalAccessor,TemporalAdjuster,Object {
+  MinguoDate now()
+  MinguoDate now(ZoneId)
+  MinguoDate of(int,int,int)
+  MinguoDate from(TemporalAccessor)
+  MinguoChronology getChronology()
+  MinguoEra getEra()
+  MinguoDate with(TemporalField,long)
+  MinguoDate with(TemporalAdjuster)
+  MinguoDate plus(TemporalAmount)
+  MinguoDate minus(TemporalAmount)
+  MinguoDate plus(long,TemporalUnit)
+  MinguoDate minus(long,TemporalUnit)
+}
+
+class ThaiBuddhistChronology -> java.time.chrono.ThaiBuddhistChronology extends AbstractChronology,Comparable,Chronology,Object {
+  ThaiBuddhistChronology INSTANCE
+  ThaiBuddhistDate date(TemporalAccessor)
+  ThaiBuddhistDate date(int,int,int)
+  ThaiBuddhistDate date(Era,int,int,int)
+  ThaiBuddhistDate dateEpochDay(long)
+  ThaiBuddhistDate dateNow()
+  ThaiBuddhistDate dateNow(ZoneId)
+  ThaiBuddhistDate dateYearDay(int,int)
+  ThaiBuddhistDate dateYearDay(Era,int,int)
+  ThaiBuddhistEra eraOf(int)
+}
+
+class ThaiBuddhistDate -> java.time.chrono.ThaiBuddhistDate extends Comparable,ChronoLocalDate,Temporal,TemporalAccessor,TemporalAdjuster,Object {
+  ThaiBuddhistDate now()
+  ThaiBuddhistDate now(ZoneId)
+  ThaiBuddhistDate of(int,int,int)
+  ThaiBuddhistDate from(TemporalAccessor)
+  ThaiBuddhistChronology getChronology()
+  ThaiBuddhistEra getEra()
+  ThaiBuddhistDate with(TemporalField,long)
+  ThaiBuddhistDate with(TemporalAdjuster)
+  ThaiBuddhistDate plus(TemporalAmount)
+  ThaiBuddhistDate minus(TemporalAmount)
+  ThaiBuddhistDate plus(long,TemporalUnit)
+  ThaiBuddhistDate minus(long,TemporalUnit)
+}
+
+#### Enums
+
+class HijrahEra -> java.time.chrono.HijrahEra extends Enum,Comparable,Era,TemporalAccessor,TemporalAdjuster,Object {
+  HijrahEra AH
+  int getValue()
+  HijrahEra of(int)
+  HijrahEra valueOf(String)
+  HijrahEra[] values()
+}
+
+class IsoEra -> java.time.chrono.IsoEra extends Enum,Comparable,Era,TemporalAccessor,TemporalAdjuster,Object {
+  IsoEra BCE
+  IsoEra CE
+  int getValue()
+  IsoEra of(int)
+  IsoEra valueOf(String)
+  IsoEra[] values()
+}
+
+class MinguoEra -> java.time.chrono.MinguoEra extends Enum,Comparable,Era,TemporalAccessor,TemporalAdjuster,Object {
+  MinguoEra BEFORE_ROC
+  MinguoEra ROC
+  int getValue()
+  MinguoEra of(int)
+  MinguoEra valueOf(String)
+  MinguoEra[] values()
+}
+
+class ThaiBuddhistEra -> java.time.chrono.ThaiBuddhistEra extends Enum,Comparable,Era,TemporalAccessor,TemporalAdjuster,Object {
+  ThaiBuddhistEra BE
+  ThaiBuddhistEra BEFORE_BE
+  int getValue()
+  ThaiBuddhistEra of(int)
+  ThaiBuddhistEra valueOf(String)
+  ThaiBuddhistEra[] values()
+}
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.format.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.format.txt
new file mode 100644
index 00000000000..20831c4b6b4
--- /dev/null
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.format.txt
@@ -0,0 +1,177 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#
+# Painless definition file. This defines the hierarchy of classes,
+# what methods and fields they have, etc.
+#
+
+#### Classes
+
+class DateTimeFormatter -> java.time.format.DateTimeFormatter extends Object {
+  DateTimeFormatter BASIC_ISO_DATE
+  DateTimeFormatter ISO_DATE
+  DateTimeFormatter ISO_DATE_TIME
+  DateTimeFormatter ISO_INSTANT
+  DateTimeFormatter ISO_LOCAL_DATE
+  DateTimeFormatter ISO_LOCAL_DATE_TIME
+  DateTimeFormatter ISO_LOCAL_TIME
+  DateTimeFormatter ISO_OFFSET_DATE
+  DateTimeFormatter ISO_OFFSET_DATE_TIME
+  DateTimeFormatter ISO_OFFSET_TIME
+  DateTimeFormatter ISO_ORDINAL_DATE
+  DateTimeFormatter ISO_TIME
+  DateTimeFormatter ISO_WEEK_DATE
+  DateTimeFormatter ISO_ZONED_DATE_TIME
+  DateTimeFormatter RFC_1123_DATE_TIME
+  String format(TemporalAccessor)
+  void formatTo(TemporalAccessor,Appendable)
+  Chronology getChronology()
+  DecimalStyle getDecimalStyle()
+  Locale getLocale()
+  Set getResolverFields()
+  ResolverStyle getResolverStyle()
+  ZoneId getZone()
+  DateTimeFormatter ofLocalizedDate(FormatStyle)
+  DateTimeFormatter ofLocalizedDateTime(FormatStyle)
+  DateTimeFormatter ofLocalizedDateTime(FormatStyle,FormatStyle)
+  DateTimeFormatter ofLocalizedTime(FormatStyle)
+  DateTimeFormatter ofPattern(String)
+  DateTimeFormatter ofPattern(String,Locale)
+  TemporalAccessor parse(CharSequence)
+  def parse(CharSequence,TemporalQuery)
+  TemporalAccessor parseBest(CharSequence,TemporalQuery[])
+  TemporalQuery parsedExcessDays()
+  TemporalQuery parsedLeapSecond()
+  TemporalAccessor parseUnresolved(CharSequence,ParsePosition)
+  Format toFormat()
+  Format toFormat(TemporalQuery)
+  DateTimeFormatter withChronology(Chronology)
+  DateTimeFormatter withDecimalStyle(DecimalStyle)
+  DateTimeFormatter withLocale(Locale)
+  DateTimeFormatter withResolverFields(Set)
+  DateTimeFormatter withResolverStyle(ResolverStyle)
+  DateTimeFormatter withZone(ZoneId)
+}
+
+class DateTimeFormatterBuilder -> java.time.format.DateTimeFormatterBuilder extends Object {
+  DateTimeFormatterBuilder <init>()
+  DateTimeFormatterBuilder append(DateTimeFormatter)
+  DateTimeFormatterBuilder appendChronologyId()
+  DateTimeFormatterBuilder appendChronologyText(TextStyle)
+  DateTimeFormatterBuilder appendFraction(TemporalField,int,int,boolean)
+  DateTimeFormatterBuilder appendInstant()
+  DateTimeFormatterBuilder appendInstant(int)
+  DateTimeFormatterBuilder appendLiteral(String)
+  DateTimeFormatterBuilder appendLocalized(FormatStyle,FormatStyle)
+  DateTimeFormatterBuilder appendLocalizedOffset(TextStyle)
+  DateTimeFormatterBuilder appendOffset(String,String)
+  DateTimeFormatterBuilder appendOffsetId()
+  DateTimeFormatterBuilder appendOptional(DateTimeFormatter)
+  DateTimeFormatterBuilder appendPattern(String)
+  DateTimeFormatterBuilder appendText(TemporalField)
+  DateTimeFormatterBuilder appendText(TemporalField,TextStyle)
+  DateTimeFormatterBuilder appendValue(TemporalField)
+  DateTimeFormatterBuilder appendValue(TemporalField,int)
+  DateTimeFormatterBuilder appendValue(TemporalField,int,int,SignStyle)
+  DateTimeFormatterBuilder appendValueReduced(TemporalField,int,int,int)
+  DateTimeFormatterBuilder appendZoneId()
+  DateTimeFormatterBuilder appendZoneOrOffsetId()
+  DateTimeFormatterBuilder appendZoneRegionId()
+  DateTimeFormatterBuilder appendZoneText(TextStyle)
+  DateTimeFormatterBuilder appendZoneText(TextStyle,Set)
+  String getLocalizedDateTimePattern(FormatStyle,FormatStyle,Chronology,Locale)
+  DateTimeFormatterBuilder optionalEnd()
+  DateTimeFormatterBuilder optionalStart()
+  DateTimeFormatterBuilder padNext(int)
+  DateTimeFormatterBuilder padNext(int,char)
+  DateTimeFormatterBuilder parseCaseInsensitive()
+  DateTimeFormatterBuilder parseCaseSensitive()
+  DateTimeFormatterBuilder parseDefaulting(TemporalField,long)
+  DateTimeFormatterBuilder parseLenient()
+  DateTimeFormatterBuilder parseStrict()
+  DateTimeFormatter toFormatter()
+  DateTimeFormatter toFormatter(Locale)
+}
+
+class DecimalStyle -> java.time.format.DecimalStyle extends Object {
+  DecimalStyle STANDARD
+  Set getAvailableLocales()
+  char getDecimalSeparator()
+  char getNegativeSign()
+  char getPositiveSign()
+  char getZeroDigit()
+  DecimalStyle of(Locale)
+  DecimalStyle ofDefaultLocale()
+  DecimalStyle withDecimalSeparator(char)
+  DecimalStyle withNegativeSign(char)
+  DecimalStyle withPositiveSign(char)
+  DecimalStyle withZeroDigit(char)
+}
+
+#### Enums
+
+class FormatStyle -> java.time.format.FormatStyle extends Enum,Comparable,Object {
+  FormatStyle FULL
+  FormatStyle LONG
+  FormatStyle MEDIUM
+  FormatStyle SHORT
+  FormatStyle valueOf(String)
+  FormatStyle[] values()
+}
+
+class ResolverStyle -> java.time.format.ResolverStyle extends Enum,Comparable,Object {
+  ResolverStyle LENIENT
+  ResolverStyle SMART
+  ResolverStyle STRICT
+  ResolverStyle valueOf(String)
+  ResolverStyle[] values()
+}
+
+class SignStyle -> java.time.format.SignStyle extends Enum,Comparable,Object {
+  SignStyle ALWAYS
+  SignStyle EXCEEDS_PAD
+  SignStyle NEVER
+  SignStyle NORMAL
+  SignStyle NOT_NEGATIVE
+  SignStyle valueOf(String)
+  SignStyle[] values()
+}
+
+class TextStyle -> java.time.format.TextStyle extends Enum,Comparable,Object {
+  TextStyle FULL
+  TextStyle FULL_STANDALONE
+  TextStyle NARROW
+  TextStyle NARROW_STANDALONE
+  TextStyle SHORT
+  TextStyle SHORT_STANDALONE
+  TextStyle asNormal()
+  TextStyle asStandalone()
+  boolean isStandalone()
+  TextStyle valueOf(String)
+  TextStyle[] values()
+}
+
+#### Exceptions
+
+class DateTimeParseException -> java.time.format.DateTimeParseException extends DateTimeException,RuntimeException,Exception,Object {
+  DateTimeParseException <init>(String,CharSequence,int)
+  int getErrorIndex()
+  String getParsedString()
+}
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.temporal.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.temporal.txt
new file mode 100644
index 00000000000..9094dab6ba1
--- /dev/null
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.temporal.txt
@@ -0,0 +1,225 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#
+# Painless definition file. This defines the hierarchy of classes,
+# what methods and fields they have, etc.
+#
+
+#### Interfaces
+
+class Temporal -> java.time.temporal.Temporal extends TemporalAccessor {
+  Temporal minus(long,TemporalUnit)
+  Temporal minus(TemporalAmount)
+  Temporal plus(long,TemporalUnit)
+  Temporal plus(TemporalAmount)
+  long until(Temporal,TemporalUnit)
+  Temporal with(TemporalAdjuster)
+  Temporal with(TemporalField,long)
+}
+
+class TemporalAccessor -> java.time.temporal.TemporalAccessor {
+  int get(TemporalField)
+  long getLong(TemporalField)
+  boolean isSupported(TemporalField)
+  def query(TemporalQuery)
+  ValueRange range(TemporalField)
+}
+
+class TemporalAdjuster -> java.time.temporal.TemporalAdjuster {
+  Temporal adjustInto(Temporal)
+}
+
+class TemporalAmount -> java.time.temporal.TemporalAmount {
+  Temporal addTo(Temporal)
+  long get(TemporalUnit)
+  List getUnits()
+  Temporal subtractFrom(Temporal)
+}
+
+class TemporalField -> java.time.temporal.TemporalField {
+  Temporal adjustInto(Temporal,long)
+  TemporalUnit getBaseUnit()
+  String getDisplayName(Locale)
+  long getFrom(TemporalAccessor)
+  TemporalUnit getRangeUnit()
+  boolean isDateBased()
+  boolean isSupportedBy(TemporalAccessor)
+  boolean isTimeBased()
+  ValueRange range()
+  ValueRange rangeRefinedBy(TemporalAccessor)
+  TemporalAccessor resolve(Map,TemporalAccessor,ResolverStyle)
+  String toString()
+}
+
+class TemporalQuery -> java.time.temporal.TemporalQuery {
+  def queryFrom(TemporalAccessor)
+}
+
+class TemporalUnit -> java.time.temporal.TemporalUnit {
+  Temporal addTo(Temporal,long)
+  long between(Temporal,Temporal)
+  Duration getDuration()
+  boolean isDateBased()
+  boolean isDurationEstimated()
+  boolean isSupportedBy(Temporal)
+  boolean isTimeBased()
+  String toString()
+}
+
+#### Classes
+
+class IsoFields -> java.time.temporal.IsoFields extends Object {
+  TemporalField DAY_OF_QUARTER
+  TemporalField QUARTER_OF_YEAR
+  TemporalUnit QUARTER_YEARS
+  TemporalField WEEK_BASED_YEAR
+  TemporalUnit WEEK_BASED_YEARS
+  TemporalField WEEK_OF_WEEK_BASED_YEAR
+}
+
+class JulianFields -> java.time.temporal.JulianFields extends Object {
+  TemporalField JULIAN_DAY
+  TemporalField MODIFIED_JULIAN_DAY
+  TemporalField RATA_DIE
+}
+
+class TemporalAdjusters -> java.time.temporal.TemporalAdjusters extends Object {
+  TemporalAdjuster dayOfWeekInMonth(int,DayOfWeek)
+  TemporalAdjuster firstDayOfMonth()
+  TemporalAdjuster firstDayOfNextMonth()
+  TemporalAdjuster firstDayOfNextYear()
+  TemporalAdjuster firstDayOfYear()
+  TemporalAdjuster firstInMonth(DayOfWeek)
+  TemporalAdjuster lastDayOfMonth()
+  TemporalAdjuster lastDayOfYear()
+  TemporalAdjuster lastInMonth(DayOfWeek)
+  TemporalAdjuster next(DayOfWeek)
+  TemporalAdjuster nextOrSame(DayOfWeek)
+  TemporalAdjuster ofDateAdjuster(UnaryOperator)
+  TemporalAdjuster previous(DayOfWeek)
+  TemporalAdjuster previousOrSame(DayOfWeek)
+}
+
+class TemporalQueries -> java.time.temporal.TemporalQueries extends Object {
+  TemporalQuery chronology()
+  TemporalQuery localDate()
+  TemporalQuery localTime()
+  TemporalQuery offset()
+  TemporalQuery precision()
+  TemporalQuery zone()
+  TemporalQuery zoneId()
+}
+
+class ValueRange -> java.time.temporal.ValueRange extends Object {
+  int checkValidIntValue(long,TemporalField)
+  long checkValidValue(long,TemporalField)
+  long getLargestMinimum()
+  long getMaximum()
+  long getMinimum()
+  long getSmallestMaximum()
+  boolean isFixed()
+  boolean isIntValue()
+  boolean isValidIntValue(long)
+  boolean isValidValue(long)
+  ValueRange of(long,long)
+  ValueRange of(long,long,long)
+  ValueRange of(long,long,long,long)
+}
+
+class WeekFields -> java.time.temporal.WeekFields extends Object {
+  WeekFields ISO
+  WeekFields SUNDAY_START
+  TemporalUnit WEEK_BASED_YEARS
+  TemporalField dayOfWeek()
+  DayOfWeek getFirstDayOfWeek()
+  int getMinimalDaysInFirstWeek()
+  WeekFields of(DayOfWeek,int)
+  WeekFields of(Locale)
+  TemporalField weekBasedYear()
+  TemporalField weekOfMonth()
+  TemporalField weekOfWeekBasedYear()
+  TemporalField weekOfYear()
+}
+
+#### Enums
+
+class ChronoField -> java.time.temporal.ChronoField extends Enum,Comparable,TemporalField,Object {
+  ChronoField ALIGNED_DAY_OF_WEEK_IN_MONTH
+  ChronoField ALIGNED_DAY_OF_WEEK_IN_YEAR
+  ChronoField ALIGNED_WEEK_OF_MONTH
+  ChronoField ALIGNED_WEEK_OF_YEAR
+  ChronoField AMPM_OF_DAY
+  ChronoField CLOCK_HOUR_OF_AMPM
+  ChronoField CLOCK_HOUR_OF_DAY
+  ChronoField DAY_OF_MONTH
+  ChronoField DAY_OF_WEEK
+  ChronoField DAY_OF_YEAR
+  ChronoField EPOCH_DAY
+  ChronoField ERA
+  ChronoField HOUR_OF_AMPM
+  ChronoField HOUR_OF_DAY
+  ChronoField INSTANT_SECONDS
+  ChronoField MICRO_OF_DAY
+  ChronoField MICRO_OF_SECOND
+  ChronoField MILLI_OF_DAY
+  ChronoField MILLI_OF_SECOND
+  ChronoField MINUTE_OF_DAY
+  ChronoField MINUTE_OF_HOUR
+  ChronoField MONTH_OF_YEAR
+  ChronoField NANO_OF_DAY
+  ChronoField NANO_OF_SECOND
+  ChronoField OFFSET_SECONDS
+  ChronoField PROLEPTIC_MONTH
+  ChronoField SECOND_OF_DAY
+  ChronoField SECOND_OF_MINUTE
+  ChronoField YEAR
+  ChronoField YEAR_OF_ERA
+  int checkValidIntValue(long)
+  long checkValidValue(long)
+  ChronoField valueOf(String)
+  ChronoField[] values()
+}
+
+class ChronoUnit -> java.time.temporal.ChronoUnit extends Enum,Comparable,TemporalUnit,Object {
+  ChronoUnit CENTURIES
+  ChronoUnit DAYS
+  ChronoUnit DECADES
+  ChronoUnit ERAS
+  ChronoUnit FOREVER
+  ChronoUnit HALF_DAYS
+  ChronoUnit HOURS
+  ChronoUnit MICROS
+  ChronoUnit MILLENNIA
+  ChronoUnit MILLIS
+  ChronoUnit MINUTES
+  ChronoUnit MONTHS
+  ChronoUnit NANOS
+  ChronoUnit SECONDS
+  ChronoUnit WEEKS
+  ChronoUnit YEARS
+  ChronoUnit valueOf(String)
+  ChronoUnit[] values()
+}
+
+#### Exceptions
+
+class UnsupportedTemporalTypeException -> java.time.temporal.UnsupportedTemporalTypeException extends DateTimeException,RuntimeException,Exception,Object {
+  UnsupportedTemporalTypeException <init>(String)
+}
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.txt
new file mode 100644
index 00000000000..1b7d19a6c9c
--- /dev/null
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.txt
@@ -0,0 +1,643 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#
+# Painless definition file. This defines the hierarchy of classes,
+# what methods and fields they have, etc.
+#
+
+#### Classes
+
+class Clock -> java.time.Clock extends Object {
+  Clock fixed(Instant,ZoneId)
+  ZoneId getZone()
+  Instant instant()
+  long millis()
+  Clock offset(Clock,Duration)
+  Clock system(ZoneId)
+  Clock systemDefaultZone()
+  Clock systemUTC()
+  Clock tick(Clock,Duration)
+  Clock tickMinutes(ZoneId)
+  Clock tickSeconds(ZoneId)
+  Clock withZone(ZoneId)
+}
+
+class Duration -> java.time.Duration extends Comparable,TemporalAmount,Object {
+  Duration ZERO
+  Duration abs()
+  Duration between(Temporal,Temporal)
+  Duration dividedBy(long)
+  Duration from(TemporalAmount)
+  int getNano()
+  long getSeconds()
+  boolean isNegative()
+  boolean isZero()
+  Duration minus(Duration)
+  Duration minus(long,TemporalUnit)
+  Duration minusDays(long)
+  Duration minusHours(long)
+  Duration minusMinutes(long)
+  Duration minusSeconds(long)
+  Duration minusMillis(long)
+  Duration minusNanos(long)
+  Duration multipliedBy(long)
+  Duration negated()
+  Duration of(long,TemporalUnit)
+  Duration ofDays(long)
+  Duration ofHours(long)
+  Duration ofMillis(long) 
+  Duration ofMinutes(long)
+  Duration ofNanos(long)
+  Duration ofSeconds(long)
+  Duration ofSeconds(long,long)
+  Duration parse(CharSequence)
+  Duration plus(Duration)
+  Duration plus(long,TemporalUnit)
+  Duration plusDays(long)
+  Duration plusHours(long)
+  Duration plusMinutes(long)
+  Duration plusSeconds(long)
+  Duration plusMillis(long)
+  Duration plusNanos(long)
+  long toDays()
+  long toHours()
+  long toMinutes()
+  long toMillis()
+  long toNanos()
+  Duration withSeconds(long)
+  Duration withNanos(int)
+}
+
+class Instant -> java.time.Instant extends Comparable,Temporal,TemporalAccessor,TemporalAdjuster,Object {
+  Instant EPOCH
+  Instant MAX
+  Instant MIN
+  OffsetDateTime atOffset(ZoneOffset)
+  ZonedDateTime atZone(ZoneId)
+  Instant from(TemporalAccessor)
+  long getEpochSecond()
+  int getNano()
+  boolean isAfter(Instant)
+  boolean isBefore(Instant)
+  Instant minus(TemporalAmount)
+  Instant minus(long,TemporalUnit)
+  Instant minusMillis(long)
+  Instant minusNanos(long)
+  Instant minusSeconds(long)
+  Instant now()
+  Instant now(Clock)
+  Instant ofEpochSecond(long)
+  Instant ofEpochSecond(long,long)
+  Instant ofEpochMilli(long)
+  Instant parse(CharSequence)
+  Instant plus(TemporalAmount)
+  Instant plus(long,TemporalUnit)
+  Instant plusMillis(long)
+  Instant plusNanos(long)
+  Instant plusSeconds(long)
+  long toEpochMilli()
+  Instant truncatedTo(TemporalUnit)
+  Instant with(TemporalAdjuster)
+  Instant with(TemporalField,long)
+}
+
+class LocalDate -> java.time.LocalDate extends Comparable,ChronoLocalDate,Temporal,TemporalAccessor,TemporalAdjuster,Object {
+  LocalDate MAX
+  LocalDate MIN
+  LocalDateTime atStartOfDay()
+  ZonedDateTime atStartOfDay(ZoneId)
+  LocalDateTime atTime(LocalTime)
+  LocalDateTime atTime(int,int)
+  LocalDateTime atTime(int,int,int)
+  LocalDateTime atTime(int,int,int,int)
+  LocalDate from(TemporalAccessor)
+  IsoChronology getChronology()
+  int getDayOfMonth()
+  DayOfWeek getDayOfWeek()
+  int getDayOfYear()
+  Month getMonth()
+  int getMonthValue()
+  int getYear()
+  LocalDate minus(TemporalAmount)
+  LocalDate minus(long,TemporalUnit)
+  LocalDate minusYears(long)
+  LocalDate minusMonths(long)
+  LocalDate minusWeeks(long)
+  LocalDate minusDays(long)
+  LocalDate now()
+  LocalDate now(ZoneId)
+  LocalDate of(int,int,int)
+  LocalDate ofYearDay(int,int)
+  LocalDate ofEpochDay(long)
+  LocalDate parse(CharSequence)
+  LocalDate parse(CharSequence,DateTimeFormatter)
+  LocalDate plus(TemporalAmount)
+  LocalDate plus(long,TemporalUnit)
+  LocalDate plusYears(long)
+  LocalDate plusMonths(long)
+  LocalDate plusWeeks(long)
+  LocalDate plusDays(long)
+  LocalDate with(TemporalAdjuster)
+  LocalDate with(TemporalField,long)
+  LocalDate withDayOfMonth(int)
+  LocalDate withDayOfYear(int)
+  LocalDate withMonth(int)
+  LocalDate withYear(int)
+}
+
+class LocalDateTime -> java.time.LocalDateTime extends Comparable,ChronoLocalDateTime,Temporal,TemporalAccessor,TemporalAdjuster,Object {
+  LocalDateTime MIN
+  LocalDateTime MAX
+  OffsetDateTime atOffset(ZoneOffset)
+  ZonedDateTime atZone(ZoneId)
+  LocalDateTime from(TemporalAccessor)
+  int getDayOfMonth()
+  DayOfWeek getDayOfWeek()
+  int getDayOfYear()
+  int getHour()
+  int getMinute()
+  Month getMonth()
+  int getMonthValue()
+  int getNano()
+  int getSecond()
+  int getYear()
+  LocalDateTime minus(TemporalAmount)
+  LocalDateTime minus(long,TemporalUnit)
+  LocalDateTime minusDays(long)
+  LocalDateTime minusHours(long)
+  LocalDateTime minusMinutes(long)
+  LocalDateTime minusMonths(long)
+  LocalDateTime minusNanos(long)
+  LocalDateTime minusSeconds(long)
+  LocalDateTime minusWeeks(long)
+  LocalDateTime minusYears(long)
+  LocalDateTime now()
+  LocalDateTime now(ZoneId)
+  LocalDateTime of(LocalDate,LocalTime)
+  LocalDateTime of(int,int,int,int,int)
+  LocalDateTime of(int,int,int,int,int,int)
+  LocalDateTime of(int,int,int,int,int,int,int)
+  LocalDateTime ofInstant(Instant,ZoneId)
+  LocalDateTime ofEpochSecond(long,int,ZoneOffset)
+  LocalDateTime parse(CharSequence)
+  LocalDateTime parse(CharSequence,DateTimeFormatter)
+  LocalDateTime plus(TemporalAmount)
+  LocalDateTime plus(long,TemporalUnit)
+  LocalDateTime plusDays(long)
+  LocalDateTime plusHours(long)
+  LocalDateTime plusMinutes(long)
+  LocalDateTime plusMonths(long)
+  LocalDateTime plusNanos(long)
+  LocalDateTime plusSeconds(long)
+  LocalDateTime plusWeeks(long)
+  LocalDateTime plusYears(long)
+  LocalDateTime truncatedTo(TemporalUnit)
+  LocalDateTime with(TemporalAdjuster)
+  LocalDateTime with(TemporalField,long)
+  LocalDateTime withDayOfMonth(int)
+  LocalDateTime withDayOfYear(int)
+  LocalDateTime withHour(int)
+  LocalDateTime withMinute(int)
+  LocalDateTime withMonth(int)
+  LocalDateTime withSecond(int)
+  LocalDateTime withYear(int)
+}
+
+class LocalTime -> java.time.LocalTime extends Comparable,Temporal,TemporalAccessor,TemporalAdjuster,Object {
+  LocalTime MAX
+  LocalTime MIDNIGHT
+  LocalTime MIN
+  LocalTime NOON
+  LocalDateTime atDate(LocalDate)
+  OffsetTime atOffset(ZoneOffset)
+  String format(DateTimeFormatter)
+  LocalTime from(TemporalAccessor)
+  int getHour()
+  int getMinute()
+  int getNano()
+  int getSecond()
+  boolean isAfter(LocalTime)
+  boolean isBefore(LocalTime)
+  LocalTime minus(TemporalAmount)
+  LocalTime minus(long,TemporalUnit)
+  LocalTime minusHours(long)
+  LocalTime minusMinutes(long)
+  LocalTime minusNanos(long)
+  LocalTime minusSeconds(long)
+  LocalTime now()
+  LocalTime now(ZoneId)
+  LocalTime of(int,int)
+  LocalTime of(int,int,int)
+  LocalTime of(int,int,int,int)
+  LocalTime ofNanoOfDay(long)
+  LocalTime ofSecondOfDay(long)
+  LocalTime parse(CharSequence)
+  LocalTime parse(CharSequence,DateTimeFormatter)
+  LocalTime plus(TemporalAmount)
+  LocalTime plus(long,TemporalUnit)
+  LocalTime plusHours(long)
+  LocalTime plusMinutes(long)
+  LocalTime plusNanos(long)
+  LocalTime plusSeconds(long)
+  long toNanoOfDay()
+  int toSecondOfDay()
+  LocalTime truncatedTo(TemporalUnit)
+  LocalTime with(TemporalAdjuster)
+  LocalTime with(TemporalField,long)
+  LocalTime withHour(int)
+  LocalTime withMinute(int)
+  LocalTime withNano(int)
+  LocalTime withSecond(int)
+}
+
+class MonthDay -> java.time.MonthDay extends Comparable,TemporalAccessor,TemporalAdjuster,Object {
+  LocalDate atYear(int)
+  String format(DateTimeFormatter)
+  MonthDay from(TemporalAccessor)
+  int getMonthValue()
+  Month getMonth()
+  int getDayOfMonth()
+  boolean isAfter(MonthDay)
+  boolean isBefore(MonthDay)
+  boolean isValidYear(int)
+  MonthDay now()
+  MonthDay now(ZoneId)
+  MonthDay of(int,int)
+  MonthDay parse(CharSequence)   
+  MonthDay parse(CharSequence,DateTimeFormatter)
+  MonthDay with(Month)
+  MonthDay withDayOfMonth(int)
+  MonthDay withMonth(int)
+}
+
+class OffsetDateTime -> java.time.OffsetDateTime extends Comparable,Temporal,TemporalAccessor,TemporalAdjuster,Object {
+  OffsetDateTime MAX
+  OffsetDateTime MIN
+  ZonedDateTime atZoneSameInstant(ZoneId)
+  ZonedDateTime atZoneSimilarLocal(ZoneId)
+  String format(DateTimeFormatter)
+  OffsetDateTime from(TemporalAccessor)
+  int getDayOfMonth()
+  DayOfWeek getDayOfWeek()
+  int getDayOfYear()
+  int getHour()
+  int getMinute()
+  Month getMonth()
+  int getMonthValue()
+  int getNano()
+  ZoneOffset getOffset()
+  int getSecond()
+  int getYear()
+  boolean isAfter(OffsetDateTime)
+  boolean isBefore(OffsetDateTime)
+  boolean isEqual(OffsetDateTime)
+  OffsetDateTime minus(TemporalAmount)
+  OffsetDateTime minus(long,TemporalUnit)
+  OffsetDateTime minusYears(long)
+  OffsetDateTime minusMonths(long)
+  OffsetDateTime minusWeeks(long)
+  OffsetDateTime minusDays(long)
+  OffsetDateTime minusHours(long)
+  OffsetDateTime minusMinutes(long)
+  OffsetDateTime minusSeconds(long)
+  OffsetDateTime minusNanos(long)
+  OffsetDateTime now()
+  OffsetDateTime now(ZoneId)
+  OffsetDateTime of(LocalDate,LocalTime,ZoneOffset)
+  OffsetDateTime of(LocalDateTime,ZoneOffset)
+  OffsetDateTime of(int,int,int,int,int,int,int,ZoneOffset)
+  OffsetDateTime ofInstant(Instant,ZoneId)
+  OffsetDateTime parse(CharSequence)
+  OffsetDateTime parse(CharSequence,DateTimeFormatter)
+  OffsetDateTime plus(TemporalAmount)
+  OffsetDateTime plus(long,TemporalUnit)
+  OffsetDateTime plusYears(long)
+  OffsetDateTime plusMonths(long)
+  OffsetDateTime plusWeeks(long)
+  OffsetDateTime plusDays(long)
+  OffsetDateTime plusHours(long)
+  OffsetDateTime plusMinutes(long)
+  OffsetDateTime plusSeconds(long)
+  OffsetDateTime plusNanos(long)
+  Comparator timeLineOrder()
+  long toEpochSecond()
+  Instant toInstant()
+  LocalDate toLocalDate()
+  LocalDateTime toLocalDateTime()
+  LocalTime toLocalTime()
+  OffsetTime toOffsetTime()
+  ZonedDateTime toZonedDateTime()
+  OffsetDateTime truncatedTo(TemporalUnit)
+  OffsetDateTime with(TemporalAdjuster)
+  OffsetDateTime with(TemporalField,long)
+  OffsetDateTime withDayOfMonth(int)
+  OffsetDateTime withDayOfYear(int)
+  OffsetDateTime withHour(int)
+  OffsetDateTime withMinute(int)
+  OffsetDateTime withMonth(int)
+  OffsetDateTime withNano(int)
+  OffsetDateTime withSecond(int)
+  OffsetDateTime withYear(int)
+  OffsetDateTime withOffsetSameLocal(ZoneOffset)
+  OffsetDateTime withOffsetSameInstant(ZoneOffset)
+}
+
+class OffsetTime -> java.time.OffsetTime extends Comparable,Temporal,TemporalAccessor,TemporalAdjuster,Object {
+  OffsetTime MAX
+  OffsetTime MIN
+  String format(DateTimeFormatter)
+  OffsetTime from(TemporalAccessor)
+  int getHour()
+  int getMinute()
+  ZoneOffset getOffset()
+  int getSecond()
+  int getNano()
+  boolean isAfter(OffsetTime)
+  boolean isBefore(OffsetTime)
+  boolean isEqual(OffsetTime)
+  OffsetTime now()
+  OffsetTime now(ZoneId)
+  OffsetTime of(LocalTime,ZoneOffset)
+  OffsetTime of(int,int,int,int,ZoneOffset)
+  OffsetTime ofInstant(Instant,ZoneId)
+  OffsetTime plus(TemporalAmount)
+  OffsetTime plus(long,TemporalUnit)
+  OffsetTime plusHours(long)
+  OffsetTime plusMinutes(long)
+  OffsetTime plusSeconds(long)
+  OffsetTime plusNanos(long)
+  OffsetTime minus(TemporalAmount)
+  OffsetTime minus(long,TemporalUnit)
+  OffsetTime minusHours(long)
+  OffsetTime minusMinutes(long)
+  OffsetTime minusSeconds(long)
+  OffsetTime minusNanos(long)
+  OffsetTime parse(CharSequence)
+  OffsetTime parse(CharSequence,DateTimeFormatter)
+  LocalTime toLocalTime()
+  OffsetTime truncatedTo(TemporalUnit)
+  OffsetTime with(TemporalAdjuster)
+  OffsetTime with(TemporalField,long)
+  OffsetTime withHour(int)
+  OffsetTime withMinute(int)
+  OffsetTime withNano(int)
+  OffsetTime withOffsetSameLocal(ZoneOffset)
+  OffsetTime withOffsetSameInstant(ZoneOffset)
+  OffsetTime withSecond(int)
+}
+
+class Period -> java.time.Period extends ChronoPeriod,TemporalAmount,Object {
+  Period ZERO
+  Period between(LocalDate,LocalDate)
+  Period from(TemporalAmount)
+  IsoChronology getChronology()
+  int getDays()
+  int getMonths()
+  int getYears()
+  Period of(int,int,int)
+  Period ofYears(int)
+  Period ofMonths(int)
+  Period ofWeeks(int)
+  Period ofDays(int)
+  Period parse(CharSequence)
+  Period plus(TemporalAmount)
+  Period plusYears(long)
+  Period plusMonths(long)
+  Period plusDays(long)
+  Period minus(TemporalAmount)
+  Period minusYears(long)
+  Period minusMonths(long)
+  Period minusDays(long)
+  Period multipliedBy(int)
+  Period negated()
+  Period normalized()
+  long toTotalMonths()
+  Period withDays(int)
+  Period withMonths(int)
+  Period withYears(int)
+}
+
+class Year -> java.time.Year extends Comparable,Temporal,TemporalAccessor,TemporalAdjuster,Object {
+  int MAX_VALUE
+  int MIN_VALUE
+  LocalDate atDay(int)
+  YearMonth atMonth(int)
+  LocalDate atMonthDay(MonthDay)
+  String format(DateTimeFormatter)
+  Year from(TemporalAccessor)
+  int getValue()
+  boolean isAfter(Year)
+  boolean isLeap()
+  boolean isLeap(long)
+  boolean isValidMonthDay(MonthDay)
+  int length()
+  Year minus(TemporalAmount)
+  Year minus(long,TemporalUnit)
+  Year minusYears(long)
+  Year now()
+  Year now(ZoneId)
+  Year of(int)
+  Year parse(CharSequence)
+  Year parse(CharSequence,DateTimeFormatter)
+  Year plus(TemporalAmount)
+  Year plus(long,TemporalUnit)
+  Year plusYears(long)
+  Year with(TemporalAdjuster)
+  Year with(TemporalField,long)
+}
+
+class YearMonth -> java.time.YearMonth extends Comparable,Temporal,TemporalAccessor,TemporalAdjuster,Object {
+  LocalDate atDay(int)
+  LocalDate atEndOfMonth()
+  String format(DateTimeFormatter)
+  YearMonth from(TemporalAccessor)
+  Month getMonth()
+  int getMonthValue()
+  int getYear()
+  boolean isAfter(YearMonth)
+  boolean isBefore(YearMonth)
+  boolean isLeapYear()
+  boolean isValidDay(int)
+  int lengthOfMonth()
+  int lengthOfYear()
+  YearMonth minus(TemporalAmount)
+  YearMonth minus(long,TemporalUnit)
+  YearMonth minusYears(long)
+  YearMonth minusMonths(long)
+  YearMonth now()
+  YearMonth now(ZoneId)
+  YearMonth of(int,int)
+  YearMonth parse(CharSequence)
+  YearMonth parse(CharSequence,DateTimeFormatter)
+  YearMonth plus(TemporalAmount)
+  YearMonth plus(long,TemporalUnit)
+  YearMonth plusYears(long)
+  YearMonth plusMonths(long)
+  YearMonth with(TemporalAdjuster)
+  YearMonth with(TemporalField,long)
+  YearMonth withYear(int)
+  YearMonth withMonth(int)
+}
+
+class ZonedDateTime -> java.time.ZonedDateTime extends Comparable,ChronoZonedDateTime,Temporal,TemporalAccessor,Object {
+  int getDayOfMonth()
+  DayOfWeek getDayOfWeek()
+  int getDayOfYear()
+  int getHour()
+  LocalDate toLocalDate()
+  LocalDateTime toLocalDateTime()
+  int getMinute()
+  Month getMonth()
+  int getMonthValue()
+  int getNano()
+  int getSecond()
+  int getYear()
+  ZonedDateTime from(TemporalAccessor)
+  ZonedDateTime minus(TemporalAmount)
+  ZonedDateTime minus(long,TemporalUnit)
+  ZonedDateTime minusYears(long)
+  ZonedDateTime minusMonths(long)
+  ZonedDateTime minusWeeks(long)
+  ZonedDateTime minusDays(long)
+  ZonedDateTime minusHours(long)
+  ZonedDateTime minusMinutes(long)
+  ZonedDateTime minusSeconds(long)
+  ZonedDateTime minusNanos(long)
+  ZonedDateTime now()
+  ZonedDateTime now(ZoneId)
+  ZonedDateTime of(LocalDate,LocalTime,ZoneId)
+  ZonedDateTime of(LocalDateTime,ZoneId)
+  ZonedDateTime of(int,int,int,int,int,int,int,ZoneId)
+  ZonedDateTime ofInstant(Instant,ZoneId)
+  ZonedDateTime ofInstant(LocalDateTime,ZoneOffset,ZoneId)
+  ZonedDateTime ofLocal(LocalDateTime,ZoneId,ZoneOffset)
+  ZonedDateTime ofStrict(LocalDateTime,ZoneOffset,ZoneId)
+  ZonedDateTime parse(CharSequence)
+  ZonedDateTime parse(CharSequence,DateTimeFormatter)
+  ZonedDateTime plus(TemporalAmount)
+  ZonedDateTime plus(long,TemporalUnit)
+  ZonedDateTime plusDays(long)
+  ZonedDateTime plusHours(long)
+  ZonedDateTime plusMinutes(long)
+  ZonedDateTime plusMonths(long)
+  ZonedDateTime plusNanos(long)
+  ZonedDateTime plusSeconds(long)
+  ZonedDateTime plusWeeks(long)
+  ZonedDateTime plusYears(long)
+  OffsetDateTime toOffsetDateTime()
+  ZonedDateTime truncatedTo(TemporalUnit)
+  ZonedDateTime with(TemporalAdjuster)
+  ZonedDateTime with(TemporalField,long)
+  ZonedDateTime withDayOfMonth(int)
+  ZonedDateTime withDayOfYear(int)
+  ZonedDateTime withEarlierOffsetAtOverlap()
+  ZonedDateTime withFixedOffsetZone()
+  ZonedDateTime withHour(int)
+  ZonedDateTime withLaterOffsetAtOverlap()
+  ZonedDateTime withMinute(int)
+  ZonedDateTime withMonth(int)
+  ZonedDateTime withNano(int)
+  ZonedDateTime withSecond(int)
+  ZonedDateTime withYear(int)
+  ZonedDateTime withZoneSameLocal(ZoneId)
+  ZonedDateTime withZoneSameInstant(ZoneId)
+}
+
+class ZoneId -> java.time.ZoneId extends Object {
+  Map SHORT_IDS
+  Set getAvailableZoneIds()
+  ZoneId of(String)
+  ZoneId of(String,Map)
+  ZoneId ofOffset(String,ZoneOffset)
+  ZoneId from(TemporalAccessor)
+  String getId()
+  String getDisplayName(TextStyle,Locale)
+  ZoneId normalized()
+  ZoneId systemDefault()
+  ZoneRules getRules()
+}
+
+class ZoneOffset -> java.time.ZoneOffset extends ZoneId,Object {
+  ZoneOffset MAX
+  ZoneOffset MIN
+  ZoneOffset UTC
+  ZoneOffset from(TemporalAccessor)
+  int getTotalSeconds()
+  ZoneOffset of(String)
+  ZoneOffset ofHours(int)
+  ZoneOffset ofHoursMinutes(int,int)
+  ZoneOffset ofHoursMinutesSeconds(int,int,int)
+  ZoneOffset ofTotalSeconds(int)
+}
+
+#### Enums
+
+class DayOfWeek -> java.time.DayOfWeek extends Enum,Comparable,TemporalAccessor,TemporalAdjuster,Object {
+  DayOfWeek FRIDAY
+  DayOfWeek MONDAY
+  DayOfWeek SATURDAY
+  DayOfWeek SUNDAY
+  DayOfWeek THURSDAY
+  DayOfWeek TUESDAY
+  DayOfWeek WEDNESDAY
+  DayOfWeek of(int)
+  DayOfWeek from(TemporalAccessor)
+  int getValue()
+  String getDisplayName(TextStyle,Locale)
+  DayOfWeek minus(long)
+  DayOfWeek plus(long)
+  DayOfWeek valueOf(String)
+  DayOfWeek[] values()
+}
+
+class Month -> java.time.Month extends Enum,Comparable,TemporalAccessor,TemporalAdjuster,Object {
+  Month APRIL
+  Month AUGUST
+  Month DECEMBER
+  Month FEBRUARY
+  Month JANUARY
+  Month JULY
+  Month JUNE
+  Month MARCH
+  Month MAY
+  Month NOVEMBER
+  Month OCTOBER
+  Month SEPTEMBER
+  Month from(TemporalAccessor)
+  int firstDayOfYear(boolean)
+  Month firstMonthOfQuarter()
+  int getValue()
+  String getDisplayName(TextStyle,Locale)
+  int length(boolean)
+  int maxLength()
+  int minLength()
+  Month minus(long)
+  Month of(int)
+  Month plus(long)
+  Month valueOf(String)
+  Month[] values()
+}
+
+#### Exceptions
+
+class DateTimeException -> java.time.DateTimeException extends RuntimeException,Exception,Object {
+  DateTimeException <init>(String)
+}
+
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.zone.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.zone.txt
new file mode 100644
index 00000000000..036e2a734f9
--- /dev/null
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.zone.txt
@@ -0,0 +1,93 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#
+# Painless definition file. This defines the hierarchy of classes,
+# what methods and fields they have, etc.
+#
+
+#### Classes
+
+class ZoneOffsetTransition -> java.time.zone.ZoneOffsetTransition extends Comparable,Object {
+  LocalDateTime getDateTimeAfter()
+  LocalDateTime getDateTimeBefore()
+  Duration getDuration()
+  Instant getInstant()
+  ZoneOffset getOffsetAfter()
+  ZoneOffset getOffsetBefore()
+  boolean isGap()
+  boolean isOverlap()
+  boolean isValidOffset(ZoneOffset)
+  ZoneOffsetTransition of(LocalDateTime,ZoneOffset,ZoneOffset)
+  long toEpochSecond()
+}
+
+class ZoneOffsetTransitionRule -> java.time.zone.ZoneOffsetTransitionRule extends Object {
+  ZoneOffsetTransition createTransition(int)
+  int getDayOfMonthIndicator()
+  DayOfWeek getDayOfWeek()
+  LocalTime getLocalTime()
+  Month getMonth()
+  ZoneOffset getOffsetAfter()
+  ZoneOffset getOffsetBefore()
+  ZoneOffset getStandardOffset()
+  ZoneOffsetTransitionRule.TimeDefinition getTimeDefinition()
+  boolean isMidnightEndOfDay()
+  ZoneOffsetTransitionRule of(Month,int,DayOfWeek,LocalTime,boolean,ZoneOffsetTransitionRule.TimeDefinition,ZoneOffset,ZoneOffset,ZoneOffset)
+}
+
+class ZoneRules -> java.time.zone.ZoneRules extends Object {
+  Duration getDaylightSavings(Instant)
+  ZoneOffset getOffset(Instant)
+  ZoneOffset getStandardOffset(Instant)
+  ZoneOffsetTransition getTransition(LocalDateTime)
+  List getTransitionRules()
+  List getTransitions()
+  List getValidOffsets(LocalDateTime)
+  boolean isDaylightSavings(Instant)
+  boolean isFixedOffset()
+  boolean isValidOffset(LocalDateTime,ZoneOffset)
+  ZoneOffsetTransition nextTransition(Instant)
+  ZoneRules of(ZoneOffset)
+  ZoneRules of(ZoneOffset,ZoneOffset,List,List,List)
+  ZoneOffsetTransition previousTransition(Instant)
+}
+
+class ZoneRulesProvider -> java.time.zone.ZoneRulesProvider extends Object {
+  Set getAvailableZoneIds()
+  ZoneRules getRules(String,boolean)
+  NavigableMap getVersions(String)
+}
+
+#### Enums
+
+class ZoneOffsetTransitionRule.TimeDefinition -> java.time.zone.ZoneOffsetTransitionRule$TimeDefinition extends Enum,Comparable,Object {
+  ZoneOffsetTransitionRule.TimeDefinition STANDARD
+  ZoneOffsetTransitionRule.TimeDefinition UTC
+  ZoneOffsetTransitionRule.TimeDefinition WALL
+  LocalDateTime createDateTime(LocalDateTime,ZoneOffset,ZoneOffset)
+  ZoneOffsetTransitionRule.TimeDefinition valueOf(String)
+  ZoneOffsetTransitionRule.TimeDefinition[] values()
+}
+
+#### Exceptions
+
+class ZoneRulesException -> java.time.zone.ZoneRulesException extends DateTimeException,RuntimeException,Exception,Object {
+  ZoneRulesException <init>(String)
+}
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt
index 9be890f15d5..ffa80da79c1 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt
@@ -491,6 +491,7 @@ class Calendar -> java.util.Calendar extends Comparable,Object {
   void setTimeInMillis(long)
   void setTimeZone(TimeZone)
   void setWeekDate(int,int,int)
+  Instant toInstant()
 }
 
 class Calendar.Builder -> java.util.Calendar$Builder extends Object {
@@ -583,6 +584,7 @@ class Date -> java.util.Date extends Comparable,Object {
   boolean after(Date)
   boolean before(Date)
   def clone()
+  Date from(Instant)
   long getTime()
   void setTime(long)
 }
@@ -976,6 +978,8 @@ class TimeZone -> java.util.TimeZone extends Object {
   boolean hasSameRules(TimeZone)
   boolean inDaylightTime(Date)
   boolean observesDaylightTime()
+  void setRawOffset(int)
+  ZoneId toZoneId()
   boolean useDaylightTime()
 }
 

From 2d8030f09fd0ef71c3511a162db01d27a725161a Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Fri, 27 May 2016 13:50:38 -0700
Subject: [PATCH 0508/1311] Added foreach in grammar.

---
 .../src/main/antlr/PainlessParser.g4          |   1 +
 .../painless/antlr/PainlessParser.java        | 697 ++++++++++--------
 .../antlr/PainlessParserBaseVisitor.java      |   7 +
 .../painless/antlr/PainlessParserVisitor.java |   7 +
 4 files changed, 388 insertions(+), 324 deletions(-)

diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index 5125e7b29a6..dad8db62117 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -33,6 +33,7 @@ statement
     | WHILE LP expression RP ( trailer | empty )                                               # while
     | DO block WHILE LP expression RP delimiter                                                # do
     | FOR LP initializer? SEMICOLON expression? SEMICOLON afterthought? RP ( trailer | empty ) # for
+    | FOR LP decltype ID COLON expression RP trailer                                           # each
     | declaration delimiter                                                                    # decl
     | CONTINUE delimiter                                                                       # continue
     | BREAK delimiter                                                                          # break
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
index 98eb16732fd..16dfb031707 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
@@ -356,6 +356,28 @@ class PainlessParser extends Parser {
       else return visitor.visitChildren(this);
     }
   }
+  public static class EachContext extends StatementContext {
+    public TerminalNode FOR() { return getToken(PainlessParser.FOR, 0); }
+    public TerminalNode LP() { return getToken(PainlessParser.LP, 0); }
+    public DecltypeContext decltype() {
+      return getRuleContext(DecltypeContext.class,0);
+    }
+    public TerminalNode ID() { return getToken(PainlessParser.ID, 0); }
+    public TerminalNode COLON() { return getToken(PainlessParser.COLON, 0); }
+    public ExpressionContext expression() {
+      return getRuleContext(ExpressionContext.class,0);
+    }
+    public TerminalNode RP() { return getToken(PainlessParser.RP, 0); }
+    public TrailerContext trailer() {
+      return getRuleContext(TrailerContext.class,0);
+    }
+    public EachContext(StatementContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitEach(this);
+      else return visitor.visitChildren(this);
+    }
+  }
   public static class ReturnContext extends StatementContext {
     public TerminalNode RETURN() { return getToken(PainlessParser.RETURN, 0); }
     public ExpressionContext expression() {
@@ -377,7 +399,7 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 2, RULE_statement);
     try {
       int _alt;
-      setState(121);
+      setState(130);
       switch ( getInterpreter().adaptivePredict(_input,8,_ctx) ) {
       case 1:
         _localctx = new IfContext(_localctx);
@@ -520,56 +542,78 @@ class PainlessParser extends Parser {
         }
         break;
       case 5:
-        _localctx = new DeclContext(_localctx);
+        _localctx = new EachContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
         setState(96);
-        declaration();
+        match(FOR);
         setState(97);
-        delimiter();
+        match(LP);
+        setState(98);
+        decltype();
+        setState(99);
+        match(ID);
+        setState(100);
+        match(COLON);
+        setState(101);
+        expression(0);
+        setState(102);
+        match(RP);
+        setState(103);
+        trailer();
         }
         break;
       case 6:
-        _localctx = new ContinueContext(_localctx);
+        _localctx = new DeclContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(99);
-        match(CONTINUE);
-        setState(100);
+        setState(105);
+        declaration();
+        setState(106);
         delimiter();
         }
         break;
       case 7:
-        _localctx = new BreakContext(_localctx);
+        _localctx = new ContinueContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(101);
-        match(BREAK);
-        setState(102);
+        setState(108);
+        match(CONTINUE);
+        setState(109);
         delimiter();
         }
         break;
       case 8:
-        _localctx = new ReturnContext(_localctx);
+        _localctx = new BreakContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(103);
-        match(RETURN);
-        setState(104);
-        expression(0);
-        setState(105);
+        setState(110);
+        match(BREAK);
+        setState(111);
         delimiter();
         }
         break;
       case 9:
-        _localctx = new TryContext(_localctx);
+        _localctx = new ReturnContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(107);
+        setState(112);
+        match(RETURN);
+        setState(113);
+        expression(0);
+        setState(114);
+        delimiter();
+        }
+        break;
+      case 10:
+        _localctx = new TryContext(_localctx);
+        enterOuterAlt(_localctx, 10);
+        {
+        setState(116);
         match(TRY);
-        setState(108);
+        setState(117);
         block();
-        setState(110); 
+        setState(119); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -577,7 +621,7 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(109);
+            setState(118);
             trap();
             }
             }
@@ -585,31 +629,31 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(112); 
+          setState(121); 
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,7,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
         }
         break;
-      case 10:
+      case 11:
         _localctx = new ThrowContext(_localctx);
-        enterOuterAlt(_localctx, 10);
+        enterOuterAlt(_localctx, 11);
         {
-        setState(114);
+        setState(123);
         match(THROW);
-        setState(115);
+        setState(124);
         expression(0);
-        setState(116);
+        setState(125);
         delimiter();
         }
         break;
-      case 11:
+      case 12:
         _localctx = new ExprContext(_localctx);
-        enterOuterAlt(_localctx, 11);
+        enterOuterAlt(_localctx, 12);
         {
-        setState(118);
+        setState(127);
         expression(0);
-        setState(119);
+        setState(128);
         delimiter();
         }
         break;
@@ -648,19 +692,19 @@ class PainlessParser extends Parser {
     TrailerContext _localctx = new TrailerContext(_ctx, getState());
     enterRule(_localctx, 4, RULE_trailer);
     try {
-      setState(125);
+      setState(134);
       switch ( getInterpreter().adaptivePredict(_input,9,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(123);
+        setState(132);
         block();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(124);
+        setState(133);
         statement();
         }
         break;
@@ -704,25 +748,25 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(127);
+      setState(136);
       match(LBRACK);
-      setState(131);
+      setState(140);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,10,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(128);
+          setState(137);
           statement();
           }
           } 
         }
-        setState(133);
+        setState(142);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,10,_ctx);
       }
-      setState(134);
+      setState(143);
       match(RBRACK);
       }
     }
@@ -756,7 +800,7 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(136);
+      setState(145);
       match(SEMICOLON);
       }
     }
@@ -793,19 +837,19 @@ class PainlessParser extends Parser {
     InitializerContext _localctx = new InitializerContext(_ctx, getState());
     enterRule(_localctx, 10, RULE_initializer);
     try {
-      setState(140);
+      setState(149);
       switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(138);
+        setState(147);
         declaration();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(139);
+        setState(148);
         expression(0);
         }
         break;
@@ -843,7 +887,7 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(142);
+      setState(151);
       expression(0);
       }
     }
@@ -890,23 +934,23 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(144);
+      setState(153);
       decltype();
-      setState(145);
+      setState(154);
       declvar();
-      setState(150);
+      setState(159);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==COMMA) {
         {
         {
-        setState(146);
+        setState(155);
         match(COMMA);
-        setState(147);
+        setState(156);
         declvar();
         }
         }
-        setState(152);
+        setState(161);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
@@ -951,21 +995,21 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(153);
+      setState(162);
       match(TYPE);
-      setState(158);
+      setState(167);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==LBRACE) {
         {
         {
-        setState(154);
+        setState(163);
         match(LBRACE);
-        setState(155);
+        setState(164);
         match(RBRACE);
         }
         }
-        setState(160);
+        setState(169);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
@@ -1003,11 +1047,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(161);
+      setState(170);
       match(TYPE);
-      setState(162);
+      setState(171);
       match(REF);
-      setState(163);
+      setState(172);
       match(ID);
       }
     }
@@ -1046,15 +1090,15 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(165);
+      setState(174);
       match(ID);
-      setState(168);
+      setState(177);
       _la = _input.LA(1);
       if (_la==ASSIGN) {
         {
-        setState(166);
+        setState(175);
         match(ASSIGN);
-        setState(167);
+        setState(176);
         expression(0);
         }
       }
@@ -1098,17 +1142,17 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(170);
+      setState(179);
       match(CATCH);
-      setState(171);
+      setState(180);
       match(LP);
-      setState(172);
+      setState(181);
       match(TYPE);
-      setState(173);
+      setState(182);
       match(ID);
-      setState(174);
+      setState(183);
       match(RP);
-      setState(175);
+      setState(184);
       block();
       }
     }
@@ -1144,7 +1188,7 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(177);
+      setState(186);
       _la = _input.LA(1);
       if ( !(_la==EOF || _la==SEMICOLON) ) {
       _errHandler.recoverInline(this);
@@ -1313,7 +1357,7 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(188);
+      setState(197);
       switch ( getInterpreter().adaptivePredict(_input,15,_ctx) ) {
       case 1:
         {
@@ -1321,16 +1365,16 @@ class PainlessParser extends Parser {
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(180);
+        setState(189);
         chain(true);
-        setState(181);
+        setState(190);
         _la = _input.LA(1);
         if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ASSIGN) | (1L << AADD) | (1L << ASUB) | (1L << AMUL) | (1L << ADIV) | (1L << AREM) | (1L << AAND) | (1L << AXOR) | (1L << AOR) | (1L << ALSH) | (1L << ARSH) | (1L << AUSH))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(182);
+        setState(191);
         expression(1);
          ((AssignmentContext)_localctx).s =  false; 
         }
@@ -1340,14 +1384,14 @@ class PainlessParser extends Parser {
         _localctx = new SingleContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(185);
+        setState(194);
         ((SingleContext)_localctx).u = unary(false);
          ((SingleContext)_localctx).s =  ((SingleContext)_localctx).u.s; 
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(249);
+      setState(258);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,17,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -1355,22 +1399,22 @@ class PainlessParser extends Parser {
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(247);
+          setState(256);
           switch ( getInterpreter().adaptivePredict(_input,16,_ctx) ) {
           case 1:
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(190);
+            setState(199);
             if (!(precpred(_ctx, 12))) throw new FailedPredicateException(this, "precpred(_ctx, 12)");
-            setState(191);
+            setState(200);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << MUL) | (1L << DIV) | (1L << REM))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(192);
+            setState(201);
             expression(13);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1379,16 +1423,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(195);
+            setState(204);
             if (!(precpred(_ctx, 11))) throw new FailedPredicateException(this, "precpred(_ctx, 11)");
-            setState(196);
+            setState(205);
             _la = _input.LA(1);
             if ( !(_la==ADD || _la==SUB) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(197);
+            setState(206);
             expression(12);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1397,16 +1441,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(200);
+            setState(209);
             if (!(precpred(_ctx, 10))) throw new FailedPredicateException(this, "precpred(_ctx, 10)");
-            setState(201);
+            setState(210);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LSH) | (1L << RSH) | (1L << USH))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(202);
+            setState(211);
             expression(11);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1415,16 +1459,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(205);
+            setState(214);
             if (!(precpred(_ctx, 9))) throw new FailedPredicateException(this, "precpred(_ctx, 9)");
-            setState(206);
+            setState(215);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LT) | (1L << LTE) | (1L << GT) | (1L << GTE))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(207);
+            setState(216);
             expression(10);
              ((CompContext)_localctx).s =  false; 
             }
@@ -1433,16 +1477,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(210);
+            setState(219);
             if (!(precpred(_ctx, 8))) throw new FailedPredicateException(this, "precpred(_ctx, 8)");
-            setState(211);
+            setState(220);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << EQ) | (1L << EQR) | (1L << NE) | (1L << NER))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(212);
+            setState(221);
             expression(9);
              ((CompContext)_localctx).s =  false; 
             }
@@ -1451,11 +1495,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(215);
+            setState(224);
             if (!(precpred(_ctx, 7))) throw new FailedPredicateException(this, "precpred(_ctx, 7)");
-            setState(216);
+            setState(225);
             match(BWAND);
-            setState(217);
+            setState(226);
             expression(8);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1464,11 +1508,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(220);
+            setState(229);
             if (!(precpred(_ctx, 6))) throw new FailedPredicateException(this, "precpred(_ctx, 6)");
-            setState(221);
+            setState(230);
             match(XOR);
-            setState(222);
+            setState(231);
             expression(7);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1477,11 +1521,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(225);
+            setState(234);
             if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)");
-            setState(226);
+            setState(235);
             match(BWOR);
-            setState(227);
+            setState(236);
             expression(6);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1490,11 +1534,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(230);
+            setState(239);
             if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)");
-            setState(231);
+            setState(240);
             match(BOOLAND);
-            setState(232);
+            setState(241);
             expression(5);
              ((BoolContext)_localctx).s =  false; 
             }
@@ -1503,11 +1547,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(235);
+            setState(244);
             if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)");
-            setState(236);
+            setState(245);
             match(BOOLOR);
-            setState(237);
+            setState(246);
             expression(4);
              ((BoolContext)_localctx).s =  false; 
             }
@@ -1516,15 +1560,15 @@ class PainlessParser extends Parser {
             {
             _localctx = new ConditionalContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(240);
+            setState(249);
             if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
-            setState(241);
+            setState(250);
             match(COND);
-            setState(242);
+            setState(251);
             ((ConditionalContext)_localctx).e0 = expression(0);
-            setState(243);
+            setState(252);
             match(COLON);
-            setState(244);
+            setState(253);
             ((ConditionalContext)_localctx).e1 = expression(2);
              ((ConditionalContext)_localctx).s =  ((ConditionalContext)_localctx).e0.s && ((ConditionalContext)_localctx).e1.s; 
             }
@@ -1532,7 +1576,7 @@ class PainlessParser extends Parser {
           }
           } 
         }
-        setState(251);
+        setState(260);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,17,_ctx);
       }
@@ -1679,22 +1723,22 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 28, RULE_unary);
     int _la;
     try {
-      setState(281);
+      setState(290);
       switch ( getInterpreter().adaptivePredict(_input,18,_ctx) ) {
       case 1:
         _localctx = new PreContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(252);
+        setState(261);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(253);
+        setState(262);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(254);
+        setState(263);
         chain(true);
         }
         break;
@@ -1702,11 +1746,11 @@ class PainlessParser extends Parser {
         _localctx = new PostContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(255);
+        setState(264);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(256);
+        setState(265);
         chain(true);
-        setState(257);
+        setState(266);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
@@ -1719,9 +1763,9 @@ class PainlessParser extends Parser {
         _localctx = new ReadContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(259);
+        setState(268);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(260);
+        setState(269);
         chain(false);
         }
         break;
@@ -1729,9 +1773,9 @@ class PainlessParser extends Parser {
         _localctx = new NumericContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(261);
+        setState(270);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(262);
+        setState(271);
         _la = _input.LA(1);
         if ( !(((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (OCTAL - 64)) | (1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)))) != 0)) ) {
         _errHandler.recoverInline(this);
@@ -1745,9 +1789,9 @@ class PainlessParser extends Parser {
         _localctx = new TrueContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(264);
+        setState(273);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(265);
+        setState(274);
         match(TRUE);
          ((TrueContext)_localctx).s =  false; 
         }
@@ -1756,9 +1800,9 @@ class PainlessParser extends Parser {
         _localctx = new FalseContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(267);
+        setState(276);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(268);
+        setState(277);
         match(FALSE);
          ((FalseContext)_localctx).s =  false; 
         }
@@ -1767,9 +1811,9 @@ class PainlessParser extends Parser {
         _localctx = new NullContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(270);
+        setState(279);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(271);
+        setState(280);
         match(NULL);
          ((NullContext)_localctx).s =  false; 
         }
@@ -1778,16 +1822,16 @@ class PainlessParser extends Parser {
         _localctx = new OperatorContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(273);
+        setState(282);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(274);
+        setState(283);
         _la = _input.LA(1);
         if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(275);
+        setState(284);
         unary(false);
         }
         break;
@@ -1795,13 +1839,13 @@ class PainlessParser extends Parser {
         _localctx = new CastContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(276);
+        setState(285);
         match(LP);
-        setState(277);
+        setState(286);
         decltype();
-        setState(278);
+        setState(287);
         match(RP);
-        setState(279);
+        setState(288);
         unary(_localctx.c);
         }
         break;
@@ -1910,27 +1954,27 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 30, RULE_chain);
     try {
       int _alt;
-      setState(317);
+      setState(326);
       switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) {
       case 1:
         _localctx = new DynamicContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(283);
+        setState(292);
         ((DynamicContext)_localctx).p = primary(_localctx.c);
-        setState(287);
+        setState(296);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,19,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(284);
+            setState(293);
             secondary(((DynamicContext)_localctx).p.s);
             }
             } 
           }
-          setState(289);
+          setState(298);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,19,_ctx);
         }
@@ -1940,23 +1984,23 @@ class PainlessParser extends Parser {
         _localctx = new StaticContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(290);
+        setState(299);
         decltype();
-        setState(291);
+        setState(300);
         dot();
-        setState(295);
+        setState(304);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(292);
+            setState(301);
             secondary(true);
             }
             } 
           }
-          setState(297);
+          setState(306);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
         }
@@ -1966,11 +2010,11 @@ class PainlessParser extends Parser {
         _localctx = new NewarrayContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(298);
+        setState(307);
         match(NEW);
-        setState(299);
+        setState(308);
         match(TYPE);
-        setState(304); 
+        setState(313); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -1978,11 +2022,11 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(300);
+            setState(309);
             match(LBRACE);
-            setState(301);
+            setState(310);
             expression(0);
-            setState(302);
+            setState(311);
             match(RBRACE);
             }
             }
@@ -1990,29 +2034,29 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(306); 
+          setState(315); 
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
-        setState(315);
+        setState(324);
         switch ( getInterpreter().adaptivePredict(_input,23,_ctx) ) {
         case 1:
           {
-          setState(308);
+          setState(317);
           dot();
-          setState(312);
+          setState(321);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
           while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
             if ( _alt==1 ) {
               {
               {
-              setState(309);
+              setState(318);
               secondary(true);
               }
               } 
             }
-            setState(314);
+            setState(323);
             _errHandler.sync(this);
             _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
           }
@@ -2114,19 +2158,19 @@ class PainlessParser extends Parser {
     PrimaryContext _localctx = new PrimaryContext(_ctx, getState(), c);
     enterRule(_localctx, 32, RULE_primary);
     try {
-      setState(335);
+      setState(344);
       switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) {
       case 1:
         _localctx = new ExprprecContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(319);
+        setState(328);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(320);
+        setState(329);
         match(LP);
-        setState(321);
+        setState(330);
         ((ExprprecContext)_localctx).e = expression(0);
-        setState(322);
+        setState(331);
         match(RP);
          ((ExprprecContext)_localctx).s =  ((ExprprecContext)_localctx).e.s; 
         }
@@ -2135,13 +2179,13 @@ class PainlessParser extends Parser {
         _localctx = new ChainprecContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(325);
+        setState(334);
         if (!( _localctx.c )) throw new FailedPredicateException(this, " $c ");
-        setState(326);
+        setState(335);
         match(LP);
-        setState(327);
+        setState(336);
         unary(true);
-        setState(328);
+        setState(337);
         match(RP);
         }
         break;
@@ -2149,7 +2193,7 @@ class PainlessParser extends Parser {
         _localctx = new StringContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(330);
+        setState(339);
         match(STRING);
         }
         break;
@@ -2157,7 +2201,7 @@ class PainlessParser extends Parser {
         _localctx = new VariableContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(331);
+        setState(340);
         match(ID);
         }
         break;
@@ -2165,11 +2209,11 @@ class PainlessParser extends Parser {
         _localctx = new NewobjectContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(332);
+        setState(341);
         match(NEW);
-        setState(333);
+        setState(342);
         match(TYPE);
-        setState(334);
+        setState(343);
         arguments();
         }
         break;
@@ -2211,23 +2255,23 @@ class PainlessParser extends Parser {
     SecondaryContext _localctx = new SecondaryContext(_ctx, getState(), s);
     enterRule(_localctx, 34, RULE_secondary);
     try {
-      setState(341);
+      setState(350);
       switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(337);
+        setState(346);
         if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
-        setState(338);
+        setState(347);
         dot();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(339);
+        setState(348);
         if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
-        setState(340);
+        setState(349);
         brace();
         }
         break;
@@ -2285,17 +2329,17 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 36, RULE_dot);
     int _la;
     try {
-      setState(348);
+      setState(357);
       switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) {
       case 1:
         _localctx = new CallinvokeContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(343);
+        setState(352);
         match(DOT);
-        setState(344);
+        setState(353);
         match(DOTID);
-        setState(345);
+        setState(354);
         arguments();
         }
         break;
@@ -2303,9 +2347,9 @@ class PainlessParser extends Parser {
         _localctx = new FieldaccessContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(346);
+        setState(355);
         match(DOT);
-        setState(347);
+        setState(356);
         _la = _input.LA(1);
         if ( !(_la==DOTINTEGER || _la==DOTID) ) {
         _errHandler.recoverInline(this);
@@ -2359,11 +2403,11 @@ class PainlessParser extends Parser {
       _localctx = new BraceaccessContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(350);
+      setState(359);
       match(LBRACE);
-      setState(351);
+      setState(360);
       expression(0);
-      setState(352);
+      setState(361);
       match(RBRACE);
       }
     }
@@ -2410,34 +2454,34 @@ class PainlessParser extends Parser {
       enterOuterAlt(_localctx, 1);
       {
       {
-      setState(354);
-      match(LP);
       setState(363);
+      match(LP);
+      setState(372);
       switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
       case 1:
         {
-        setState(355);
+        setState(364);
         argument();
-        setState(360);
+        setState(369);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(356);
+          setState(365);
           match(COMMA);
-          setState(357);
+          setState(366);
           argument();
           }
           }
-          setState(362);
+          setState(371);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
         break;
       }
-      setState(365);
+      setState(374);
       match(RP);
       }
       }
@@ -2475,19 +2519,19 @@ class PainlessParser extends Parser {
     ArgumentContext _localctx = new ArgumentContext(_ctx, getState());
     enterRule(_localctx, 42, RULE_argument);
     try {
-      setState(369);
+      setState(378);
       switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(367);
+        setState(376);
         expression(0);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(368);
+        setState(377);
         funcref();
         }
         break;
@@ -2594,141 +2638,146 @@ class PainlessParser extends Parser {
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3M\u0176\4\2\t\2\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3M\u017f\4\2\t\2\4"+
     "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
     "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
     "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\3\2\7\2\60\n\2\f\2"+
     "\16\2\63\13\2\3\2\3\2\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3?\n\3\3\3\3\3"+
     "\3\3\3\3\3\3\3\3\5\3G\n\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3"+
     "\5\3T\n\3\3\3\3\3\5\3X\n\3\3\3\3\3\5\3\\\n\3\3\3\3\3\3\3\5\3a\n\3\3\3"+
-    "\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\6\3q\n\3\r\3\16\3"+
-    "r\3\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3|\n\3\3\4\3\4\5\4\u0080\n\4\3\5\3\5\7"+
-    "\5\u0084\n\5\f\5\16\5\u0087\13\5\3\5\3\5\3\6\3\6\3\7\3\7\5\7\u008f\n\7"+
-    "\3\b\3\b\3\t\3\t\3\t\3\t\7\t\u0097\n\t\f\t\16\t\u009a\13\t\3\n\3\n\3\n"+
-    "\7\n\u009f\n\n\f\n\16\n\u00a2\13\n\3\13\3\13\3\13\3\13\3\f\3\f\3\f\5\f"+
-    "\u00ab\n\f\3\r\3\r\3\r\3\r\3\r\3\r\3\r\3\16\3\16\3\17\3\17\3\17\3\17\3"+
-    "\17\3\17\3\17\3\17\3\17\5\17\u00bf\n\17\3\17\3\17\3\17\3\17\3\17\3\17"+
+    "\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3"+
+    "\3\3\3\3\3\3\3\3\3\6\3z\n\3\r\3\16\3{\3\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3"+
+    "\u0085\n\3\3\4\3\4\5\4\u0089\n\4\3\5\3\5\7\5\u008d\n\5\f\5\16\5\u0090"+
+    "\13\5\3\5\3\5\3\6\3\6\3\7\3\7\5\7\u0098\n\7\3\b\3\b\3\t\3\t\3\t\3\t\7"+
+    "\t\u00a0\n\t\f\t\16\t\u00a3\13\t\3\n\3\n\3\n\7\n\u00a8\n\n\f\n\16\n\u00ab"+
+    "\13\n\3\13\3\13\3\13\3\13\3\f\3\f\3\f\5\f\u00b4\n\f\3\r\3\r\3\r\3\r\3"+
+    "\r\3\r\3\r\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\5\17"+
+    "\u00c8\n\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
     "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
     "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
     "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
-    "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\7\17\u00fa\n\17\f\17\16"+
-    "\17\u00fd\13\17\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
+    "\3\17\3\17\3\17\7\17\u0103\n\17\f\17\16\17\u0106\13\17\3\20\3\20\3\20"+
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
-    "\3\20\3\20\3\20\3\20\5\20\u011c\n\20\3\21\3\21\7\21\u0120\n\21\f\21\16"+
-    "\21\u0123\13\21\3\21\3\21\3\21\7\21\u0128\n\21\f\21\16\21\u012b\13\21"+
-    "\3\21\3\21\3\21\3\21\3\21\3\21\6\21\u0133\n\21\r\21\16\21\u0134\3\21\3"+
-    "\21\7\21\u0139\n\21\f\21\16\21\u013c\13\21\5\21\u013e\n\21\5\21\u0140"+
-    "\n\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22"+
-    "\3\22\3\22\3\22\5\22\u0152\n\22\3\23\3\23\3\23\3\23\5\23\u0158\n\23\3"+
-    "\24\3\24\3\24\3\24\3\24\5\24\u015f\n\24\3\25\3\25\3\25\3\25\3\26\3\26"+
-    "\3\26\3\26\7\26\u0169\n\26\f\26\16\26\u016c\13\26\5\26\u016e\n\26\3\26"+
-    "\3\26\3\27\3\27\5\27\u0174\n\27\3\27\2\3\34\30\2\4\6\b\n\f\16\20\22\24"+
-    "\26\30\32\34\36 \"$&(*,\2\r\3\3\r\r\3\2\66A\3\2\34\36\3\2\37 \3\2!#\3"+
-    "\2$\'\3\2(+\3\2\64\65\3\2BE\4\2\32\33\37 \3\2LM\u019b\2\61\3\2\2\2\4{"+
-    "\3\2\2\2\6\177\3\2\2\2\b\u0081\3\2\2\2\n\u008a\3\2\2\2\f\u008e\3\2\2\2"+
-    "\16\u0090\3\2\2\2\20\u0092\3\2\2\2\22\u009b\3\2\2\2\24\u00a3\3\2\2\2\26"+
-    "\u00a7\3\2\2\2\30\u00ac\3\2\2\2\32\u00b3\3\2\2\2\34\u00be\3\2\2\2\36\u011b"+
-    "\3\2\2\2 \u013f\3\2\2\2\"\u0151\3\2\2\2$\u0157\3\2\2\2&\u015e\3\2\2\2"+
-    "(\u0160\3\2\2\2*\u0164\3\2\2\2,\u0173\3\2\2\2.\60\5\4\3\2/.\3\2\2\2\60"+
-    "\63\3\2\2\2\61/\3\2\2\2\61\62\3\2\2\2\62\64\3\2\2\2\63\61\3\2\2\2\64\65"+
-    "\7\2\2\3\65\3\3\2\2\2\66\67\7\16\2\2\678\7\t\2\289\5\34\17\29:\7\n\2\2"+
-    ":>\5\6\4\2;<\7\17\2\2<?\5\6\4\2=?\6\3\2\2>;\3\2\2\2>=\3\2\2\2?|\3\2\2"+
-    "\2@A\7\20\2\2AB\7\t\2\2BC\5\34\17\2CF\7\n\2\2DG\5\6\4\2EG\5\n\6\2FD\3"+
-    "\2\2\2FE\3\2\2\2G|\3\2\2\2HI\7\21\2\2IJ\5\b\5\2JK\7\20\2\2KL\7\t\2\2L"+
-    "M\5\34\17\2MN\7\n\2\2NO\5\32\16\2O|\3\2\2\2PQ\7\22\2\2QS\7\t\2\2RT\5\f"+
-    "\7\2SR\3\2\2\2ST\3\2\2\2TU\3\2\2\2UW\7\r\2\2VX\5\34\17\2WV\3\2\2\2WX\3"+
-    "\2\2\2XY\3\2\2\2Y[\7\r\2\2Z\\\5\16\b\2[Z\3\2\2\2[\\\3\2\2\2\\]\3\2\2\2"+
-    "]`\7\n\2\2^a\5\6\4\2_a\5\n\6\2`^\3\2\2\2`_\3\2\2\2a|\3\2\2\2bc\5\20\t"+
-    "\2cd\5\32\16\2d|\3\2\2\2ef\7\23\2\2f|\5\32\16\2gh\7\24\2\2h|\5\32\16\2"+
-    "ij\7\25\2\2jk\5\34\17\2kl\5\32\16\2l|\3\2\2\2mn\7\27\2\2np\5\b\5\2oq\5"+
-    "\30\r\2po\3\2\2\2qr\3\2\2\2rp\3\2\2\2rs\3\2\2\2s|\3\2\2\2tu\7\31\2\2u"+
-    "v\5\34\17\2vw\5\32\16\2w|\3\2\2\2xy\5\34\17\2yz\5\32\16\2z|\3\2\2\2{\66"+
-    "\3\2\2\2{@\3\2\2\2{H\3\2\2\2{P\3\2\2\2{b\3\2\2\2{e\3\2\2\2{g\3\2\2\2{"+
-    "i\3\2\2\2{m\3\2\2\2{t\3\2\2\2{x\3\2\2\2|\5\3\2\2\2}\u0080\5\b\5\2~\u0080"+
-    "\5\4\3\2\177}\3\2\2\2\177~\3\2\2\2\u0080\7\3\2\2\2\u0081\u0085\7\5\2\2"+
-    "\u0082\u0084\5\4\3\2\u0083\u0082\3\2\2\2\u0084\u0087\3\2\2\2\u0085\u0083"+
-    "\3\2\2\2\u0085\u0086\3\2\2\2\u0086\u0088\3\2\2\2\u0087\u0085\3\2\2\2\u0088"+
-    "\u0089\7\6\2\2\u0089\t\3\2\2\2\u008a\u008b\7\r\2\2\u008b\13\3\2\2\2\u008c"+
-    "\u008f\5\20\t\2\u008d\u008f\5\34\17\2\u008e\u008c\3\2\2\2\u008e\u008d"+
-    "\3\2\2\2\u008f\r\3\2\2\2\u0090\u0091\5\34\17\2\u0091\17\3\2\2\2\u0092"+
-    "\u0093\5\22\n\2\u0093\u0098\5\26\f\2\u0094\u0095\7\f\2\2\u0095\u0097\5"+
-    "\26\f\2\u0096\u0094\3\2\2\2\u0097\u009a\3\2\2\2\u0098\u0096\3\2\2\2\u0098"+
-    "\u0099\3\2\2\2\u0099\21\3\2\2\2\u009a\u0098\3\2\2\2\u009b\u00a0\7J\2\2"+
-    "\u009c\u009d\7\7\2\2\u009d\u009f\7\b\2\2\u009e\u009c\3\2\2\2\u009f\u00a2"+
-    "\3\2\2\2\u00a0\u009e\3\2\2\2\u00a0\u00a1\3\2\2\2\u00a1\23\3\2\2\2\u00a2"+
-    "\u00a0\3\2\2\2\u00a3\u00a4\7J\2\2\u00a4\u00a5\7\63\2\2\u00a5\u00a6\7K"+
-    "\2\2\u00a6\25\3\2\2\2\u00a7\u00aa\7K\2\2\u00a8\u00a9\7\66\2\2\u00a9\u00ab"+
-    "\5\34\17\2\u00aa\u00a8\3\2\2\2\u00aa\u00ab\3\2\2\2\u00ab\27\3\2\2\2\u00ac"+
-    "\u00ad\7\30\2\2\u00ad\u00ae\7\t\2\2\u00ae\u00af\7J\2\2\u00af\u00b0\7K"+
-    "\2\2\u00b0\u00b1\7\n\2\2\u00b1\u00b2\5\b\5\2\u00b2\31\3\2\2\2\u00b3\u00b4"+
-    "\t\2\2\2\u00b4\33\3\2\2\2\u00b5\u00b6\b\17\1\2\u00b6\u00b7\5 \21\2\u00b7"+
-    "\u00b8\t\3\2\2\u00b8\u00b9\5\34\17\3\u00b9\u00ba\b\17\1\2\u00ba\u00bf"+
-    "\3\2\2\2\u00bb\u00bc\5\36\20\2\u00bc\u00bd\b\17\1\2\u00bd\u00bf\3\2\2"+
-    "\2\u00be\u00b5\3\2\2\2\u00be\u00bb\3\2\2\2\u00bf\u00fb\3\2\2\2\u00c0\u00c1"+
-    "\f\16\2\2\u00c1\u00c2\t\4\2\2\u00c2\u00c3\5\34\17\17\u00c3\u00c4\b\17"+
-    "\1\2\u00c4\u00fa\3\2\2\2\u00c5\u00c6\f\r\2\2\u00c6\u00c7\t\5\2\2\u00c7"+
-    "\u00c8\5\34\17\16\u00c8\u00c9\b\17\1\2\u00c9\u00fa\3\2\2\2\u00ca\u00cb"+
-    "\f\f\2\2\u00cb\u00cc\t\6\2\2\u00cc\u00cd\5\34\17\r\u00cd\u00ce\b\17\1"+
-    "\2\u00ce\u00fa\3\2\2\2\u00cf\u00d0\f\13\2\2\u00d0\u00d1\t\7\2\2\u00d1"+
-    "\u00d2\5\34\17\f\u00d2\u00d3\b\17\1\2\u00d3\u00fa\3\2\2\2\u00d4\u00d5"+
-    "\f\n\2\2\u00d5\u00d6\t\b\2\2\u00d6\u00d7\5\34\17\13\u00d7\u00d8\b\17\1"+
-    "\2\u00d8\u00fa\3\2\2\2\u00d9\u00da\f\t\2\2\u00da\u00db\7,\2\2\u00db\u00dc"+
-    "\5\34\17\n\u00dc\u00dd\b\17\1\2\u00dd\u00fa\3\2\2\2\u00de\u00df\f\b\2"+
-    "\2\u00df\u00e0\7-\2\2\u00e0\u00e1\5\34\17\t\u00e1\u00e2\b\17\1\2\u00e2"+
-    "\u00fa\3\2\2\2\u00e3\u00e4\f\7\2\2\u00e4\u00e5\7.\2\2\u00e5\u00e6\5\34"+
-    "\17\b\u00e6\u00e7\b\17\1\2\u00e7\u00fa\3\2\2\2\u00e8\u00e9\f\6\2\2\u00e9"+
-    "\u00ea\7/\2\2\u00ea\u00eb\5\34\17\7\u00eb\u00ec\b\17\1\2\u00ec\u00fa\3"+
-    "\2\2\2\u00ed\u00ee\f\5\2\2\u00ee\u00ef\7\60\2\2\u00ef\u00f0\5\34\17\6"+
-    "\u00f0\u00f1\b\17\1\2\u00f1\u00fa\3\2\2\2\u00f2\u00f3\f\4\2\2\u00f3\u00f4"+
-    "\7\61\2\2\u00f4\u00f5\5\34\17\2\u00f5\u00f6\7\62\2\2\u00f6\u00f7\5\34"+
-    "\17\4\u00f7\u00f8\b\17\1\2\u00f8\u00fa\3\2\2\2\u00f9\u00c0\3\2\2\2\u00f9"+
-    "\u00c5\3\2\2\2\u00f9\u00ca\3\2\2\2\u00f9\u00cf\3\2\2\2\u00f9\u00d4\3\2"+
-    "\2\2\u00f9\u00d9\3\2\2\2\u00f9\u00de\3\2\2\2\u00f9\u00e3\3\2\2\2\u00f9"+
-    "\u00e8\3\2\2\2\u00f9\u00ed\3\2\2\2\u00f9\u00f2\3\2\2\2\u00fa\u00fd\3\2"+
-    "\2\2\u00fb\u00f9\3\2\2\2\u00fb\u00fc\3\2\2\2\u00fc\35\3\2\2\2\u00fd\u00fb"+
-    "\3\2\2\2\u00fe\u00ff\6\20\16\3\u00ff\u0100\t\t\2\2\u0100\u011c\5 \21\2"+
-    "\u0101\u0102\6\20\17\3\u0102\u0103\5 \21\2\u0103\u0104\t\t\2\2\u0104\u011c"+
-    "\3\2\2\2\u0105\u0106\6\20\20\3\u0106\u011c\5 \21\2\u0107\u0108\6\20\21"+
-    "\3\u0108\u0109\t\n\2\2\u0109\u011c\b\20\1\2\u010a\u010b\6\20\22\3\u010b"+
-    "\u010c\7G\2\2\u010c\u011c\b\20\1\2\u010d\u010e\6\20\23\3\u010e\u010f\7"+
-    "H\2\2\u010f\u011c\b\20\1\2\u0110\u0111\6\20\24\3\u0111\u0112\7I\2\2\u0112"+
-    "\u011c\b\20\1\2\u0113\u0114\6\20\25\3\u0114\u0115\t\13\2\2\u0115\u011c"+
-    "\5\36\20\2\u0116\u0117\7\t\2\2\u0117\u0118\5\22\n\2\u0118\u0119\7\n\2"+
-    "\2\u0119\u011a\5\36\20\2\u011a\u011c\3\2\2\2\u011b\u00fe\3\2\2\2\u011b"+
-    "\u0101\3\2\2\2\u011b\u0105\3\2\2\2\u011b\u0107\3\2\2\2\u011b\u010a\3\2"+
-    "\2\2\u011b\u010d\3\2\2\2\u011b\u0110\3\2\2\2\u011b\u0113\3\2\2\2\u011b"+
-    "\u0116\3\2\2\2\u011c\37\3\2\2\2\u011d\u0121\5\"\22\2\u011e\u0120\5$\23"+
-    "\2\u011f\u011e\3\2\2\2\u0120\u0123\3\2\2\2\u0121\u011f\3\2\2\2\u0121\u0122"+
-    "\3\2\2\2\u0122\u0140\3\2\2\2\u0123\u0121\3\2\2\2\u0124\u0125\5\22\n\2"+
-    "\u0125\u0129\5&\24\2\u0126\u0128\5$\23\2\u0127\u0126\3\2\2\2\u0128\u012b"+
-    "\3\2\2\2\u0129\u0127\3\2\2\2\u0129\u012a\3\2\2\2\u012a\u0140\3\2\2\2\u012b"+
-    "\u0129\3\2\2\2\u012c\u012d\7\26\2\2\u012d\u0132\7J\2\2\u012e\u012f\7\7"+
-    "\2\2\u012f\u0130\5\34\17\2\u0130\u0131\7\b\2\2\u0131\u0133\3\2\2\2\u0132"+
-    "\u012e\3\2\2\2\u0133\u0134\3\2\2\2\u0134\u0132\3\2\2\2\u0134\u0135\3\2"+
-    "\2\2\u0135\u013d\3\2\2\2\u0136\u013a\5&\24\2\u0137\u0139\5$\23\2\u0138"+
-    "\u0137\3\2\2\2\u0139\u013c\3\2\2\2\u013a\u0138\3\2\2\2\u013a\u013b\3\2"+
-    "\2\2\u013b\u013e\3\2\2\2\u013c\u013a\3\2\2\2\u013d\u0136\3\2\2\2\u013d"+
-    "\u013e\3\2\2\2\u013e\u0140\3\2\2\2\u013f\u011d\3\2\2\2\u013f\u0124\3\2"+
-    "\2\2\u013f\u012c\3\2\2\2\u0140!\3\2\2\2\u0141\u0142\6\22\26\3\u0142\u0143"+
-    "\7\t\2\2\u0143\u0144\5\34\17\2\u0144\u0145\7\n\2\2\u0145\u0146\b\22\1"+
-    "\2\u0146\u0152\3\2\2\2\u0147\u0148\6\22\27\3\u0148\u0149\7\t\2\2\u0149"+
-    "\u014a\5\36\20\2\u014a\u014b\7\n\2\2\u014b\u0152\3\2\2\2\u014c\u0152\7"+
-    "F\2\2\u014d\u0152\7K\2\2\u014e\u014f\7\26\2\2\u014f\u0150\7J\2\2\u0150"+
-    "\u0152\5*\26\2\u0151\u0141\3\2\2\2\u0151\u0147\3\2\2\2\u0151\u014c\3\2"+
-    "\2\2\u0151\u014d\3\2\2\2\u0151\u014e\3\2\2\2\u0152#\3\2\2\2\u0153\u0154"+
-    "\6\23\30\3\u0154\u0158\5&\24\2\u0155\u0156\6\23\31\3\u0156\u0158\5(\25"+
-    "\2\u0157\u0153\3\2\2\2\u0157\u0155\3\2\2\2\u0158%\3\2\2\2\u0159\u015a"+
-    "\7\13\2\2\u015a\u015b\7M\2\2\u015b\u015f\5*\26\2\u015c\u015d\7\13\2\2"+
-    "\u015d\u015f\t\f\2\2\u015e\u0159\3\2\2\2\u015e\u015c\3\2\2\2\u015f\'\3"+
-    "\2\2\2\u0160\u0161\7\7\2\2\u0161\u0162\5\34\17\2\u0162\u0163\7\b\2\2\u0163"+
-    ")\3\2\2\2\u0164\u016d\7\t\2\2\u0165\u016a\5,\27\2\u0166\u0167\7\f\2\2"+
-    "\u0167\u0169\5,\27\2\u0168\u0166\3\2\2\2\u0169\u016c\3\2\2\2\u016a\u0168"+
-    "\3\2\2\2\u016a\u016b\3\2\2\2\u016b\u016e\3\2\2\2\u016c\u016a\3\2\2\2\u016d"+
-    "\u0165\3\2\2\2\u016d\u016e\3\2\2\2\u016e\u016f\3\2\2\2\u016f\u0170\7\n"+
-    "\2\2\u0170+\3\2\2\2\u0171\u0174\5\34\17\2\u0172\u0174\5\24\13\2\u0173"+
-    "\u0171\3\2\2\2\u0173\u0172\3\2\2\2\u0174-\3\2\2\2!\61>FSW[`r{\177\u0085"+
-    "\u008e\u0098\u00a0\u00aa\u00be\u00f9\u00fb\u011b\u0121\u0129\u0134\u013a"+
-    "\u013d\u013f\u0151\u0157\u015e\u016a\u016d\u0173";
+    "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\5\20\u0125"+
+    "\n\20\3\21\3\21\7\21\u0129\n\21\f\21\16\21\u012c\13\21\3\21\3\21\3\21"+
+    "\7\21\u0131\n\21\f\21\16\21\u0134\13\21\3\21\3\21\3\21\3\21\3\21\3\21"+
+    "\6\21\u013c\n\21\r\21\16\21\u013d\3\21\3\21\7\21\u0142\n\21\f\21\16\21"+
+    "\u0145\13\21\5\21\u0147\n\21\5\21\u0149\n\21\3\22\3\22\3\22\3\22\3\22"+
+    "\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\5\22\u015b\n\22"+
+    "\3\23\3\23\3\23\3\23\5\23\u0161\n\23\3\24\3\24\3\24\3\24\3\24\5\24\u0168"+
+    "\n\24\3\25\3\25\3\25\3\25\3\26\3\26\3\26\3\26\7\26\u0172\n\26\f\26\16"+
+    "\26\u0175\13\26\5\26\u0177\n\26\3\26\3\26\3\27\3\27\5\27\u017d\n\27\3"+
+    "\27\2\3\34\30\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,\2\r\3\3\r"+
+    "\r\3\2\66A\3\2\34\36\3\2\37 \3\2!#\3\2$\'\3\2(+\3\2\64\65\3\2BE\4\2\32"+
+    "\33\37 \3\2LM\u01a5\2\61\3\2\2\2\4\u0084\3\2\2\2\6\u0088\3\2\2\2\b\u008a"+
+    "\3\2\2\2\n\u0093\3\2\2\2\f\u0097\3\2\2\2\16\u0099\3\2\2\2\20\u009b\3\2"+
+    "\2\2\22\u00a4\3\2\2\2\24\u00ac\3\2\2\2\26\u00b0\3\2\2\2\30\u00b5\3\2\2"+
+    "\2\32\u00bc\3\2\2\2\34\u00c7\3\2\2\2\36\u0124\3\2\2\2 \u0148\3\2\2\2\""+
+    "\u015a\3\2\2\2$\u0160\3\2\2\2&\u0167\3\2\2\2(\u0169\3\2\2\2*\u016d\3\2"+
+    "\2\2,\u017c\3\2\2\2.\60\5\4\3\2/.\3\2\2\2\60\63\3\2\2\2\61/\3\2\2\2\61"+
+    "\62\3\2\2\2\62\64\3\2\2\2\63\61\3\2\2\2\64\65\7\2\2\3\65\3\3\2\2\2\66"+
+    "\67\7\16\2\2\678\7\t\2\289\5\34\17\29:\7\n\2\2:>\5\6\4\2;<\7\17\2\2<?"+
+    "\5\6\4\2=?\6\3\2\2>;\3\2\2\2>=\3\2\2\2?\u0085\3\2\2\2@A\7\20\2\2AB\7\t"+
+    "\2\2BC\5\34\17\2CF\7\n\2\2DG\5\6\4\2EG\5\n\6\2FD\3\2\2\2FE\3\2\2\2G\u0085"+
+    "\3\2\2\2HI\7\21\2\2IJ\5\b\5\2JK\7\20\2\2KL\7\t\2\2LM\5\34\17\2MN\7\n\2"+
+    "\2NO\5\32\16\2O\u0085\3\2\2\2PQ\7\22\2\2QS\7\t\2\2RT\5\f\7\2SR\3\2\2\2"+
+    "ST\3\2\2\2TU\3\2\2\2UW\7\r\2\2VX\5\34\17\2WV\3\2\2\2WX\3\2\2\2XY\3\2\2"+
+    "\2Y[\7\r\2\2Z\\\5\16\b\2[Z\3\2\2\2[\\\3\2\2\2\\]\3\2\2\2]`\7\n\2\2^a\5"+
+    "\6\4\2_a\5\n\6\2`^\3\2\2\2`_\3\2\2\2a\u0085\3\2\2\2bc\7\22\2\2cd\7\t\2"+
+    "\2de\5\22\n\2ef\7K\2\2fg\7\62\2\2gh\5\34\17\2hi\7\n\2\2ij\5\6\4\2j\u0085"+
+    "\3\2\2\2kl\5\20\t\2lm\5\32\16\2m\u0085\3\2\2\2no\7\23\2\2o\u0085\5\32"+
+    "\16\2pq\7\24\2\2q\u0085\5\32\16\2rs\7\25\2\2st\5\34\17\2tu\5\32\16\2u"+
+    "\u0085\3\2\2\2vw\7\27\2\2wy\5\b\5\2xz\5\30\r\2yx\3\2\2\2z{\3\2\2\2{y\3"+
+    "\2\2\2{|\3\2\2\2|\u0085\3\2\2\2}~\7\31\2\2~\177\5\34\17\2\177\u0080\5"+
+    "\32\16\2\u0080\u0085\3\2\2\2\u0081\u0082\5\34\17\2\u0082\u0083\5\32\16"+
+    "\2\u0083\u0085\3\2\2\2\u0084\66\3\2\2\2\u0084@\3\2\2\2\u0084H\3\2\2\2"+
+    "\u0084P\3\2\2\2\u0084b\3\2\2\2\u0084k\3\2\2\2\u0084n\3\2\2\2\u0084p\3"+
+    "\2\2\2\u0084r\3\2\2\2\u0084v\3\2\2\2\u0084}\3\2\2\2\u0084\u0081\3\2\2"+
+    "\2\u0085\5\3\2\2\2\u0086\u0089\5\b\5\2\u0087\u0089\5\4\3\2\u0088\u0086"+
+    "\3\2\2\2\u0088\u0087\3\2\2\2\u0089\7\3\2\2\2\u008a\u008e\7\5\2\2\u008b"+
+    "\u008d\5\4\3\2\u008c\u008b\3\2\2\2\u008d\u0090\3\2\2\2\u008e\u008c\3\2"+
+    "\2\2\u008e\u008f\3\2\2\2\u008f\u0091\3\2\2\2\u0090\u008e\3\2\2\2\u0091"+
+    "\u0092\7\6\2\2\u0092\t\3\2\2\2\u0093\u0094\7\r\2\2\u0094\13\3\2\2\2\u0095"+
+    "\u0098\5\20\t\2\u0096\u0098\5\34\17\2\u0097\u0095\3\2\2\2\u0097\u0096"+
+    "\3\2\2\2\u0098\r\3\2\2\2\u0099\u009a\5\34\17\2\u009a\17\3\2\2\2\u009b"+
+    "\u009c\5\22\n\2\u009c\u00a1\5\26\f\2\u009d\u009e\7\f\2\2\u009e\u00a0\5"+
+    "\26\f\2\u009f\u009d\3\2\2\2\u00a0\u00a3\3\2\2\2\u00a1\u009f\3\2\2\2\u00a1"+
+    "\u00a2\3\2\2\2\u00a2\21\3\2\2\2\u00a3\u00a1\3\2\2\2\u00a4\u00a9\7J\2\2"+
+    "\u00a5\u00a6\7\7\2\2\u00a6\u00a8\7\b\2\2\u00a7\u00a5\3\2\2\2\u00a8\u00ab"+
+    "\3\2\2\2\u00a9\u00a7\3\2\2\2\u00a9\u00aa\3\2\2\2\u00aa\23\3\2\2\2\u00ab"+
+    "\u00a9\3\2\2\2\u00ac\u00ad\7J\2\2\u00ad\u00ae\7\63\2\2\u00ae\u00af\7K"+
+    "\2\2\u00af\25\3\2\2\2\u00b0\u00b3\7K\2\2\u00b1\u00b2\7\66\2\2\u00b2\u00b4"+
+    "\5\34\17\2\u00b3\u00b1\3\2\2\2\u00b3\u00b4\3\2\2\2\u00b4\27\3\2\2\2\u00b5"+
+    "\u00b6\7\30\2\2\u00b6\u00b7\7\t\2\2\u00b7\u00b8\7J\2\2\u00b8\u00b9\7K"+
+    "\2\2\u00b9\u00ba\7\n\2\2\u00ba\u00bb\5\b\5\2\u00bb\31\3\2\2\2\u00bc\u00bd"+
+    "\t\2\2\2\u00bd\33\3\2\2\2\u00be\u00bf\b\17\1\2\u00bf\u00c0\5 \21\2\u00c0"+
+    "\u00c1\t\3\2\2\u00c1\u00c2\5\34\17\3\u00c2\u00c3\b\17\1\2\u00c3\u00c8"+
+    "\3\2\2\2\u00c4\u00c5\5\36\20\2\u00c5\u00c6\b\17\1\2\u00c6\u00c8\3\2\2"+
+    "\2\u00c7\u00be\3\2\2\2\u00c7\u00c4\3\2\2\2\u00c8\u0104\3\2\2\2\u00c9\u00ca"+
+    "\f\16\2\2\u00ca\u00cb\t\4\2\2\u00cb\u00cc\5\34\17\17\u00cc\u00cd\b\17"+
+    "\1\2\u00cd\u0103\3\2\2\2\u00ce\u00cf\f\r\2\2\u00cf\u00d0\t\5\2\2\u00d0"+
+    "\u00d1\5\34\17\16\u00d1\u00d2\b\17\1\2\u00d2\u0103\3\2\2\2\u00d3\u00d4"+
+    "\f\f\2\2\u00d4\u00d5\t\6\2\2\u00d5\u00d6\5\34\17\r\u00d6\u00d7\b\17\1"+
+    "\2\u00d7\u0103\3\2\2\2\u00d8\u00d9\f\13\2\2\u00d9\u00da\t\7\2\2\u00da"+
+    "\u00db\5\34\17\f\u00db\u00dc\b\17\1\2\u00dc\u0103\3\2\2\2\u00dd\u00de"+
+    "\f\n\2\2\u00de\u00df\t\b\2\2\u00df\u00e0\5\34\17\13\u00e0\u00e1\b\17\1"+
+    "\2\u00e1\u0103\3\2\2\2\u00e2\u00e3\f\t\2\2\u00e3\u00e4\7,\2\2\u00e4\u00e5"+
+    "\5\34\17\n\u00e5\u00e6\b\17\1\2\u00e6\u0103\3\2\2\2\u00e7\u00e8\f\b\2"+
+    "\2\u00e8\u00e9\7-\2\2\u00e9\u00ea\5\34\17\t\u00ea\u00eb\b\17\1\2\u00eb"+
+    "\u0103\3\2\2\2\u00ec\u00ed\f\7\2\2\u00ed\u00ee\7.\2\2\u00ee\u00ef\5\34"+
+    "\17\b\u00ef\u00f0\b\17\1\2\u00f0\u0103\3\2\2\2\u00f1\u00f2\f\6\2\2\u00f2"+
+    "\u00f3\7/\2\2\u00f3\u00f4\5\34\17\7\u00f4\u00f5\b\17\1\2\u00f5\u0103\3"+
+    "\2\2\2\u00f6\u00f7\f\5\2\2\u00f7\u00f8\7\60\2\2\u00f8\u00f9\5\34\17\6"+
+    "\u00f9\u00fa\b\17\1\2\u00fa\u0103\3\2\2\2\u00fb\u00fc\f\4\2\2\u00fc\u00fd"+
+    "\7\61\2\2\u00fd\u00fe\5\34\17\2\u00fe\u00ff\7\62\2\2\u00ff\u0100\5\34"+
+    "\17\4\u0100\u0101\b\17\1\2\u0101\u0103\3\2\2\2\u0102\u00c9\3\2\2\2\u0102"+
+    "\u00ce\3\2\2\2\u0102\u00d3\3\2\2\2\u0102\u00d8\3\2\2\2\u0102\u00dd\3\2"+
+    "\2\2\u0102\u00e2\3\2\2\2\u0102\u00e7\3\2\2\2\u0102\u00ec\3\2\2\2\u0102"+
+    "\u00f1\3\2\2\2\u0102\u00f6\3\2\2\2\u0102\u00fb\3\2\2\2\u0103\u0106\3\2"+
+    "\2\2\u0104\u0102\3\2\2\2\u0104\u0105\3\2\2\2\u0105\35\3\2\2\2\u0106\u0104"+
+    "\3\2\2\2\u0107\u0108\6\20\16\3\u0108\u0109\t\t\2\2\u0109\u0125\5 \21\2"+
+    "\u010a\u010b\6\20\17\3\u010b\u010c\5 \21\2\u010c\u010d\t\t\2\2\u010d\u0125"+
+    "\3\2\2\2\u010e\u010f\6\20\20\3\u010f\u0125\5 \21\2\u0110\u0111\6\20\21"+
+    "\3\u0111\u0112\t\n\2\2\u0112\u0125\b\20\1\2\u0113\u0114\6\20\22\3\u0114"+
+    "\u0115\7G\2\2\u0115\u0125\b\20\1\2\u0116\u0117\6\20\23\3\u0117\u0118\7"+
+    "H\2\2\u0118\u0125\b\20\1\2\u0119\u011a\6\20\24\3\u011a\u011b\7I\2\2\u011b"+
+    "\u0125\b\20\1\2\u011c\u011d\6\20\25\3\u011d\u011e\t\13\2\2\u011e\u0125"+
+    "\5\36\20\2\u011f\u0120\7\t\2\2\u0120\u0121\5\22\n\2\u0121\u0122\7\n\2"+
+    "\2\u0122\u0123\5\36\20\2\u0123\u0125\3\2\2\2\u0124\u0107\3\2\2\2\u0124"+
+    "\u010a\3\2\2\2\u0124\u010e\3\2\2\2\u0124\u0110\3\2\2\2\u0124\u0113\3\2"+
+    "\2\2\u0124\u0116\3\2\2\2\u0124\u0119\3\2\2\2\u0124\u011c\3\2\2\2\u0124"+
+    "\u011f\3\2\2\2\u0125\37\3\2\2\2\u0126\u012a\5\"\22\2\u0127\u0129\5$\23"+
+    "\2\u0128\u0127\3\2\2\2\u0129\u012c\3\2\2\2\u012a\u0128\3\2\2\2\u012a\u012b"+
+    "\3\2\2\2\u012b\u0149\3\2\2\2\u012c\u012a\3\2\2\2\u012d\u012e\5\22\n\2"+
+    "\u012e\u0132\5&\24\2\u012f\u0131\5$\23\2\u0130\u012f\3\2\2\2\u0131\u0134"+
+    "\3\2\2\2\u0132\u0130\3\2\2\2\u0132\u0133\3\2\2\2\u0133\u0149\3\2\2\2\u0134"+
+    "\u0132\3\2\2\2\u0135\u0136\7\26\2\2\u0136\u013b\7J\2\2\u0137\u0138\7\7"+
+    "\2\2\u0138\u0139\5\34\17\2\u0139\u013a\7\b\2\2\u013a\u013c\3\2\2\2\u013b"+
+    "\u0137\3\2\2\2\u013c\u013d\3\2\2\2\u013d\u013b\3\2\2\2\u013d\u013e\3\2"+
+    "\2\2\u013e\u0146\3\2\2\2\u013f\u0143\5&\24\2\u0140\u0142\5$\23\2\u0141"+
+    "\u0140\3\2\2\2\u0142\u0145\3\2\2\2\u0143\u0141\3\2\2\2\u0143\u0144\3\2"+
+    "\2\2\u0144\u0147\3\2\2\2\u0145\u0143\3\2\2\2\u0146\u013f\3\2\2\2\u0146"+
+    "\u0147\3\2\2\2\u0147\u0149\3\2\2\2\u0148\u0126\3\2\2\2\u0148\u012d\3\2"+
+    "\2\2\u0148\u0135\3\2\2\2\u0149!\3\2\2\2\u014a\u014b\6\22\26\3\u014b\u014c"+
+    "\7\t\2\2\u014c\u014d\5\34\17\2\u014d\u014e\7\n\2\2\u014e\u014f\b\22\1"+
+    "\2\u014f\u015b\3\2\2\2\u0150\u0151\6\22\27\3\u0151\u0152\7\t\2\2\u0152"+
+    "\u0153\5\36\20\2\u0153\u0154\7\n\2\2\u0154\u015b\3\2\2\2\u0155\u015b\7"+
+    "F\2\2\u0156\u015b\7K\2\2\u0157\u0158\7\26\2\2\u0158\u0159\7J\2\2\u0159"+
+    "\u015b\5*\26\2\u015a\u014a\3\2\2\2\u015a\u0150\3\2\2\2\u015a\u0155\3\2"+
+    "\2\2\u015a\u0156\3\2\2\2\u015a\u0157\3\2\2\2\u015b#\3\2\2\2\u015c\u015d"+
+    "\6\23\30\3\u015d\u0161\5&\24\2\u015e\u015f\6\23\31\3\u015f\u0161\5(\25"+
+    "\2\u0160\u015c\3\2\2\2\u0160\u015e\3\2\2\2\u0161%\3\2\2\2\u0162\u0163"+
+    "\7\13\2\2\u0163\u0164\7M\2\2\u0164\u0168\5*\26\2\u0165\u0166\7\13\2\2"+
+    "\u0166\u0168\t\f\2\2\u0167\u0162\3\2\2\2\u0167\u0165\3\2\2\2\u0168\'\3"+
+    "\2\2\2\u0169\u016a\7\7\2\2\u016a\u016b\5\34\17\2\u016b\u016c\7\b\2\2\u016c"+
+    ")\3\2\2\2\u016d\u0176\7\t\2\2\u016e\u0173\5,\27\2\u016f\u0170\7\f\2\2"+
+    "\u0170\u0172\5,\27\2\u0171\u016f\3\2\2\2\u0172\u0175\3\2\2\2\u0173\u0171"+
+    "\3\2\2\2\u0173\u0174\3\2\2\2\u0174\u0177\3\2\2\2\u0175\u0173\3\2\2\2\u0176"+
+    "\u016e\3\2\2\2\u0176\u0177\3\2\2\2\u0177\u0178\3\2\2\2\u0178\u0179\7\n"+
+    "\2\2\u0179+\3\2\2\2\u017a\u017d\5\34\17\2\u017b\u017d\5\24\13\2\u017c"+
+    "\u017a\3\2\2\2\u017c\u017b\3\2\2\2\u017d-\3\2\2\2!\61>FSW[`{\u0084\u0088"+
+    "\u008e\u0097\u00a1\u00a9\u00b3\u00c7\u0102\u0104\u0124\u012a\u0132\u013d"+
+    "\u0143\u0146\u0148\u015a\u0160\u0167\u0173\u0176\u017c";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
index b4429fc0841..f116f087c5c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
@@ -46,6 +46,13 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitFor(PainlessParser.ForContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitEach(PainlessParser.EachContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
index 6dc511cb4e9..f0943743ef8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
@@ -44,6 +44,13 @@ interface PainlessParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitFor(PainlessParser.ForContext ctx);
+  /**
+   * Visit a parse tree produced by the {@code each}
+   * labeled alternative in {@link PainlessParser#statement}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitEach(PainlessParser.EachContext ctx);
   /**
    * Visit a parse tree produced by the {@code decl}
    * labeled alternative in {@link PainlessParser#statement}.

From f5be0982d9c8ec61c1c8189987dc643ba9f8571e Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Fri, 27 May 2016 17:41:31 -0700
Subject: [PATCH 0509/1311] Added initial infrastructure to allow for each to
 be made.

---
 .../src/main/antlr/PainlessLexer.g4           |  4 +
 .../org/elasticsearch/painless/Compiler.java  |  7 +-
 .../elasticsearch/painless/ScriptImpl.java    | 10 +-
 .../painless/node/AExpression.java            | 52 ++++++++--
 .../painless/node/AStatement.java             |  5 +-
 .../elasticsearch/painless/node/EBinary.java  |  1 +
 .../elasticsearch/painless/node/EChain.java   | 94 +++++++++++++------
 .../elasticsearch/painless/node/EComp.java    |  1 +
 .../painless/node/EConditional.java           |  1 +
 .../elasticsearch/painless/node/EUnary.java   |  1 +
 .../elasticsearch/painless/node/LBrace.java   |  1 -
 .../elasticsearch/painless/node/LCall.java    |  1 +
 .../painless/node/LDefArray.java              |  2 +
 .../elasticsearch/painless/node/LDefCall.java |  1 +
 .../painless/node/LDefField.java              |  2 +
 .../elasticsearch/painless/node/LField.java   |  2 +
 .../painless/node/LListShortcut.java          |  2 +
 .../painless/node/LMapShortcut.java           |  2 +
 .../painless/node/LNewArray.java              |  1 +
 .../painless/node/LShortcut.java              |  2 +
 .../elasticsearch/painless/node/SBlock.java   | 21 +++--
 .../elasticsearch/painless/node/SBreak.java   |  4 +-
 .../elasticsearch/painless/node/SCatch.java   |  9 +-
 .../painless/node/SContinue.java              |  4 +-
 .../painless/node/SDeclBlock.java             | 15 +--
 .../painless/node/SDeclaration.java           |  5 +-
 .../org/elasticsearch/painless/node/SDo.java  |  9 +-
 .../painless/node/SExpression.java            |  4 +-
 .../org/elasticsearch/painless/node/SFor.java | 13 ++-
 .../org/elasticsearch/painless/node/SIf.java  |  9 +-
 .../elasticsearch/painless/node/SIfElse.java  | 13 ++-
 .../elasticsearch/painless/node/SReturn.java  |  4 +-
 .../elasticsearch/painless/node/SSource.java  | 15 +--
 .../elasticsearch/painless/node/SThrow.java   |  4 +-
 .../org/elasticsearch/painless/node/STry.java | 20 ++--
 .../elasticsearch/painless/node/SWhile.java   |  9 +-
 36 files changed, 250 insertions(+), 100 deletions(-)

diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.g4 b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
index 6e34d35111d..4ca5778ad3c 100644
--- a/modules/lang-painless/src/main/antlr/PainlessLexer.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
@@ -32,6 +32,10 @@ LBRACE:    '[';
 RBRACE:    ']';
 LP:        '(';
 RP:        ')';
+// We switch modes after a dot to ensure there are not conflicts
+// between shortcuts and decimal values.  Without the mode switch
+// shortcuts such as id.0.0 will fail because 0.0 will be interpreted
+// as a decimal value instead of two individual list-style shortcuts.
 DOT:       '.' -> mode(AFTER_DOT);
 COMMA:     ',';
 SEMICOLON: ';';
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
index 36d41056013..961c5ab8c2e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
@@ -104,17 +104,16 @@ final class Compiler {
         SSource root = Walker.buildPainlessTree(source, reserved, settings);
         Variables variables = Analyzer.analyze(reserved, root);
         BitSet expressions = new BitSet(source.length());
-
         byte[] bytes = Writer.write(settings, name, source, variables, root, expressions);
+
         try {
             Class<? extends Executable> clazz = loader.define(CLASS_NAME, bytes);
-            java.lang.reflect.Constructor<? extends Executable> constructor = 
+            java.lang.reflect.Constructor<? extends Executable> constructor =
                     clazz.getConstructor(String.class, String.class, BitSet.class);
 
             return constructor.newInstance(name, source, expressions);
         } catch (Exception exception) { // Catch everything to let the user know this is something caused internally.
-            throw new IllegalStateException(
-                    "An internal error occurred attempting to define the script [" + name + "].", exception);
+            throw new IllegalStateException("An internal error occurred attempting to define the script [" + name + "].", exception);
         }
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
index 83ae7b664c1..c601bfc2064 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
@@ -123,7 +123,13 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
             throw convertToScriptException(t);
         }
     }
-    
+
+    /**
+     * Adds stack trace and other useful information to exceptiosn thrown
+     * from a Painless script.
+     * @param t The throwable to build an exception around.
+     * @return The generated ScriptException.
+     */
     private ScriptException convertToScriptException(Throwable t) {
         // create a script stack: this is just the script portion
         List<String> scriptStack = new ArrayList<>();
@@ -169,7 +175,7 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
         }
         throw new ScriptException("runtime error", t, scriptStack, name, PainlessScriptEngineService.NAME);
     }
-    
+
     /** returns true for methods that are part of the runtime */
     private static boolean shouldFilter(StackTraceElement element) {
         return element.getClassName().startsWith("org.elasticsearch.painless.") ||
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
index c46181817e2..f82e1c2da81 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
@@ -122,9 +122,19 @@ public abstract class AExpression extends ANode {
 
         if (cast == null) {
             if (constant == null || this instanceof EConstant) {
+                // For the case where a cast is not required and a constant is not set
+                // or the node is already an EConstant no changes are required to the tree.
+
                 return this;
             } else {
-                final EConstant econstant = new EConstant(line, offset, location, constant);
+                // For the case where a cast is not required but a
+                // constant is set, an EConstant replaces this node
+                // with the constant copied from this node.  Note that
+                // for constants output data does not need to be copied
+                // from this node because the output data for the EConstant
+                // will already be the same.
+
+                EConstant econstant = new EConstant(line, offset, location, constant);
                 econstant.analyze(variables);
 
                 if (!expected.equals(econstant.actual)) {
@@ -135,7 +145,12 @@ public abstract class AExpression extends ANode {
             }
         } else {
             if (constant == null) {
-                final ECast ecast = new ECast(line, offset, location, this, cast);
+                // For the case where a cast is required and a constant is not set.
+                // Modify the tree to add an ECast between this node and its parent.
+                // The output data from this node is copied to the ECast for
+                // further reads done by the parent.
+
+                ECast ecast = new ECast(line, offset, location, this, cast);
                 ecast.statement = statement;
                 ecast.actual = expected;
                 ecast.isNull = isNull;
@@ -143,9 +158,17 @@ public abstract class AExpression extends ANode {
                 return ecast;
             } else {
                 if (expected.sort.constant) {
+                    // For the case where a cast is required, a constant is set,
+                    // and the constant can be immediately cast to the expected type.
+                    // An EConstant replaces this node with the constant cast appropriately
+                    // from the constant value defined by this node.  Note that
+                    // for constants output data does not need to be copied
+                    // from this node because the output data for the EConstant
+                    // will already be the same.
+
                     constant = AnalyzerCaster.constCast(location, constant, cast);
 
-                    final EConstant econstant = new EConstant(line, offset, location, constant);
+                    EConstant econstant = new EConstant(line, offset, location, constant);
                     econstant.analyze(variables);
 
                     if (!expected.equals(econstant.actual)) {
@@ -154,19 +177,36 @@ public abstract class AExpression extends ANode {
 
                     return econstant;
                 } else if (this instanceof EConstant) {
-                    final ECast ecast = new ECast(line, offset, location, this, cast);
+                    // For the case where a cast is required, a constant is set,
+                    // the constant cannot be immediately cast to the expected type,
+                    // and this node is already an EConstant.  Modify the tree to add
+                    // an ECast between this node and its parent.  Note that
+                    // for constants output data does not need to be copied
+                    // from this node because the output data for the EConstant
+                    // will already be the same.
+
+                    ECast ecast = new ECast(line, offset, location, this, cast);
                     ecast.actual = expected;
 
                     return ecast;
                 } else {
-                    final EConstant econstant = new EConstant(line, offset, location, constant);
+                    // For the case where a cast is required, a constant is set,
+                    // the constant cannot be immediately cast to the expected type,
+                    // and this node is not an EConstant.  Replace this node with
+                    // an Econstant node copying the constant from this node.
+                    // Modify the tree to add an ECast between the EConstant node
+                    // and its parent.  Note that for constants output data does not
+                    // need to be copied from this node because the output data for
+                    // the EConstant will already be the same.
+
+                    EConstant econstant = new EConstant(line, offset, location, constant);
                     econstant.analyze(variables);
 
                     if (!actual.equals(econstant.actual)) {
                         throw new IllegalStateException(error("Illegal tree structure."));
                     }
 
-                    final ECast ecast = new ECast(line, offset, location, econstant, cast);
+                    ECast ecast = new ECast(line, offset, location, econstant, cast);
                     ecast.actual = expected;
 
                     return ecast;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
index 80d24b4cab3..276a2b43865 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
@@ -113,8 +113,11 @@ public abstract class AStatement extends ANode {
 
     /**
      * Checks for errors and collects data for the writing phase.
+     * @return The new child node for the parent node calling this method.
+     * Possibly returns a different {@link AStatement} node if a type is
+     * def or if a different specialization is used. Otherwise, returns itself.
      */
-    abstract void analyze(Variables variables);
+    abstract AStatement analyze(Variables variables);
 
     /**
      * Writes ASM based on the data collected during the analysis phase.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 23179a9ed0d..92b565e34df 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -471,6 +471,7 @@ public final class EBinary extends AExpression {
     @Override
     void write(MethodWriter writer) {
         writer.writeDebugInfo(offset);
+
         if (actual.sort == Sort.STRING && operation == Operation.ADD) {
             if (!cat) {
                 writer.writeNewStrings();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index affd1fe78c0..af111b99995 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -246,75 +246,115 @@ public final class EChain extends AExpression {
         actual = last.after;
     }
 
+    /**
+     * Handles writing byte code for variable/method chains for all given possibilities
+     * including String concatenation, compound assignment, regular assignment, and simple
+     * reads.  Includes proper duplication for chained assignments and assignments that are
+     * also read from.
+     *
+     * Example given 'x[0] += 5;' where x is an array of shorts and x[0] is 1.
+     * Note this example has two links -- x (LVariable) and [0] (LBrace).
+     * The following steps occur:
+     * 1. call link{x}.write(...) -- no op [...]
+     * 2. call link{x}.load(...) -- loads the address of the x array onto the stack [..., address(x)]
+     * 3. call writer.dup(...) -- dup's the address of the x array onto the stack for later use with store [..., address(x), address(x)]
+     * 4. call link{[0]}.write(...) -- load the array index value of the constant int 0 onto the stack [..., address(x), address(x), int(0)]
+     * 5. call link{[0]}.load(...) -- load the short value from x[0] onto the stack [..., address(x), short(1)]
+     * 6. call writer.writeCast(there) -- casts the short on the stack to an int so it can be added with the rhs [..., address(x), int(1)]
+     * 7. call expression.write(...) -- puts the expression's value of the constant int 5 onto the stack [..., address(x), int(1), int(5)]
+     * 8. call writer.writeBinaryInstruction(operation) -- writes the int addition instruction [..., address(x), int(6)]
+     * 9. call writer.writeCast(back) -- convert the value on the stack back into a short [..., address(x), short(6)]
+     * 10. call link{[0]}.store(...) -- store the value on the stack into the 0th index of the array x [...]
+     */
     @Override
     void write(MethodWriter writer) {
+        // For the case where the chain represents a String concatenation
+        // we must first write debug information, and then depending on the
+        // Java version write a StringBuilder or track types going onto the
+        // stack.  This must be done before the links in the chain are read
+        // because we need the StringBuilder to be placed on the stack
+        // ahead of any potential concatenation arguments.
         if (cat) {
             writer.writeDebugInfo(offset);
-        }
-
-        if (cat) {
             writer.writeNewStrings();
         }
 
         ALink last = links.get(links.size() - 1);
 
+        // Go through all the links in the chain first calling write
+        // and then load, except for the final link which may be a store.
+        // See individual links for more information on what each of the
+        // write, load, and store methods do.
         for (ALink link : links) {
-            link.write(writer);
+            link.write(writer); // call the write method on the link to prepare for a load/store operation
 
             if (link == last && link.store) {
                 if (cat) {
-                    writer.writeDup(link.size, 1);
-                    link.load(writer);
-                    writer.writeAppendStrings(link.after);
+                    // Handle the case where we are doing a compound assignment
+                    // representing a String concatenation.
 
-                    expression.write(writer);
+                    writer.writeDup(link.size, 1);         // dup the StringBuilder
+                    link.load(writer);                     // read the current link's value
+                    writer.writeAppendStrings(link.after); // append the link's value using the StringBuilder
+
+                    expression.write(writer); // write the bytecode for the rhs expression
 
                     if (!(expression instanceof EBinary) ||
                         ((EBinary)expression).operation != Operation.ADD || expression.actual.sort != Sort.STRING) {
-                        writer.writeAppendStrings(expression.actual);
+                        writer.writeAppendStrings(expression.actual); // append the expression's value unless its also a concatenation
                     }
 
-                    writer.writeToStrings();
-                    writer.writeCast(back);
+                    writer.writeToStrings(); // put the value of the StringBuilder on the stack
+                    writer.writeCast(back);  // if necessary, cast the String to the lhs actual type
 
                     if (link.load) {
-                        writer.writeDup(link.after.sort.size, link.size);
+                        writer.writeDup(link.after.sort.size, link.size); // if this link is also read from dup the value onto the stack
                     }
 
-                    link.store(writer);
+                    link.store(writer); // store the link's value from the stack in its respective variable/field/array
                 } else if (operation != null) {
-                    writer.writeDup(link.size, 0);
-                    link.load(writer);
+                    // Handle the case where we are doing a compound assignment that
+                    // does not represent a String concatenation.
+
+                    writer.writeDup(link.size, 0); // if necessary, dup the previous link's value to be both loaded from and stored to
+                    link.load(writer);             // load the current link's value
 
                     if (link.load && post) {
-                        writer.writeDup(link.after.sort.size, link.size);
+                        writer.writeDup(link.after.sort.size, link.size); // dup the value if the link is also
+                                                                          // read from and is a post increment
                     }
 
-                    writer.writeCast(there);
-                    expression.write(writer);
-                    writer.writeBinaryInstruction(location, promote, operation);
+                    writer.writeCast(there);                                     // if necessary cast the current link's value
+                                                                                 // to the promotion type between the lhs and rhs types
+                    expression.write(writer);                                    // write the bytecode for the rhs expression
+                    writer.writeBinaryInstruction(location, promote, operation); // write the operation instruction for compound assignment
 
-                    writer.writeCast(back);
+                    writer.writeCast(back); // if necessary cast the promotion type value back to the link's type
 
                     if (link.load && !post) {
-                        writer.writeDup(link.after.sort.size, link.size);
+                        writer.writeDup(link.after.sort.size, link.size); // dup the value if the link is also
+                                                                          // read from and is not a post increment
                     }
 
-                    link.store(writer);
+                    link.store(writer); // store the link's value from the stack in its respective variable/field/array
                 } else {
-                    expression.write(writer);
+                    // Handle the case for a simple write.
+
+                    expression.write(writer); // write the bytecode for the rhs expression
 
                     if (link.load) {
-                        writer.writeDup(link.after.sort.size, link.size);
+                        writer.writeDup(link.after.sort.size, link.size); // dup the value if the link is also read from
                     }
 
-                    link.store(writer);
+                    link.store(writer); // store the link's value from the stack in its respective variable/field/array
                 }
             } else {
-                link.load(writer);
+                // Handle the case for a simple read.
+
+                link.load(writer); // read the link's value onto the stack
             }
         }
 
-        writer.writeBranch(tru, fals);
+        writer.writeBranch(tru, fals); // if this is a branch node, write the bytecode to make an appropiate jump
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index 5bc6e4ee400..8e681d6b3c3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -400,6 +400,7 @@ public final class EComp extends AExpression {
     @Override
     void write(MethodWriter writer) {
         writer.writeDebugInfo(offset);
+
         boolean branch = tru != null || fals != null;
         org.objectweb.asm.Type rtype = right.actual.type;
         Sort rsort = right.actual.sort;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
index d7f7ab7e4b6..aadcedd42ce 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
@@ -79,6 +79,7 @@ public final class EConditional extends AExpression {
     @Override
     void write(MethodWriter writer) {
         writer.writeDebugInfo(offset);
+
         Label localfals = new Label();
         Label end = new Label();
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index 2122174442d..63e80d77cb4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -166,6 +166,7 @@ public final class EUnary extends AExpression {
     @Override
     void write(MethodWriter writer) {
         writer.writeDebugInfo(offset);
+
         if (operation == Operation.NOT) {
             if (tru == null && fals == null) {
                 Label localfals = new Label();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
index 02e8ceacd9f..cef45eed0a7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
@@ -83,5 +83,4 @@ public final class LBrace extends ALink {
         writer.writeDebugInfo(offset);
         writer.arrayStore(after.type);
     }
-
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
index 30775e3a6e7..a4e05c38316 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
@@ -92,6 +92,7 @@ public final class LCall extends ALink {
     @Override
     void load(MethodWriter writer) {
         writer.writeDebugInfo(offset);
+
         for (AExpression argument : arguments) {
             argument.write(writer);
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index b00eafe3f9f..27bce1d6533 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -59,6 +59,7 @@ final class LDefArray extends ALink implements IDefLink {
     @Override
     void load(MethodWriter writer) {
         writer.writeDebugInfo(offset);
+
         String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type, index.actual.type);
         writer.invokeDynamic("arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ARRAY_LOAD);
     }
@@ -66,6 +67,7 @@ final class LDefArray extends ALink implements IDefLink {
     @Override
     void store(MethodWriter writer) {
         writer.writeDebugInfo(offset);
+
         String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, index.actual.type, after.type);
         writer.invokeDynamic("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ARRAY_STORE);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index 0292ac3c589..bf7ea1778df 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -68,6 +68,7 @@ final class LDefCall extends ALink implements IDefLink {
     @Override
     void load(MethodWriter writer) {
         writer.writeDebugInfo(offset);
+
         StringBuilder signature = new StringBuilder();
 
         signature.append('(');
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index e762de9e862..b9937d735ec 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -56,6 +56,7 @@ final class LDefField extends ALink implements IDefLink {
     @Override
     void load(MethodWriter writer) {
         writer.writeDebugInfo(offset);
+
         String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type);
         writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.LOAD);
     }
@@ -63,6 +64,7 @@ final class LDefField extends ALink implements IDefLink {
     @Override
     void store(MethodWriter writer) {
         writer.writeDebugInfo(offset);
+
         String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, after.type);
         writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.STORE);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
index 88173ad3f65..50f67d0d7f6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
@@ -106,6 +106,7 @@ public final class LField extends ALink {
     @Override
     void load(MethodWriter writer) {
         writer.writeDebugInfo(offset);
+
         if (java.lang.reflect.Modifier.isStatic(field.modifiers)) {
             writer.getStatic(field.owner.type, field.javaName, field.type.type);
         } else {
@@ -116,6 +117,7 @@ public final class LField extends ALink {
     @Override
     void store(MethodWriter writer) {
         writer.writeDebugInfo(offset);
+
         if (java.lang.reflect.Modifier.isStatic(field.modifiers)) {
             writer.putStatic(field.owner.type, field.javaName, field.type.type);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
index b628fee5f37..357c9478254 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
@@ -80,6 +80,7 @@ final class LListShortcut extends ALink {
     @Override
     void load(MethodWriter writer) {
         writer.writeDebugInfo(offset);
+
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
             writer.invokeInterface(getter.owner.type, getter.method);
         } else {
@@ -94,6 +95,7 @@ final class LListShortcut extends ALink {
     @Override
     void store(MethodWriter writer) {
         writer.writeDebugInfo(offset);
+
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
             writer.invokeInterface(setter.owner.type, setter.method);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
index a500a6673fc..5580ef3d01f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
@@ -79,6 +79,7 @@ final class LMapShortcut extends ALink {
     @Override
     void load(MethodWriter writer) {
         writer.writeDebugInfo(offset);
+
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
             writer.invokeInterface(getter.owner.type, getter.method);
         } else {
@@ -93,6 +94,7 @@ final class LMapShortcut extends ALink {
     @Override
     void store(MethodWriter writer) {
         writer.writeDebugInfo(offset);
+
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
             writer.invokeInterface(setter.owner.type, setter.method);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
index c555f04ab95..21340caac61 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
@@ -80,6 +80,7 @@ public final class LNewArray extends ALink {
     @Override
     void load(MethodWriter writer) {
         writer.writeDebugInfo(offset);
+
         for (AExpression argument : arguments) {
             argument.write(writer);
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
index 7eb44156e55..42128ad86a6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
@@ -85,6 +85,7 @@ final class LShortcut extends ALink {
     @Override
     void load(MethodWriter writer) {
         writer.writeDebugInfo(offset);
+
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
             writer.invokeInterface(getter.owner.type, getter.method);
         } else {
@@ -99,6 +100,7 @@ final class LShortcut extends ALink {
     @Override
     void store(MethodWriter writer) {
         writer.writeDebugInfo(offset);
+
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
             writer.invokeInterface(setter.owner.type, setter.method);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
index 0f05243f994..1790ebbbfc1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
@@ -22,7 +22,6 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
 
-import java.util.Collections;
 import java.util.List;
 
 /**
@@ -35,27 +34,27 @@ public final class SBlock extends AStatement {
     public SBlock(int line, int offset, String location, List<AStatement> statements) {
         super(line, offset, location);
 
-        this.statements = Collections.unmodifiableList(statements);
+        this.statements = statements;
     }
 
     @Override
-    void analyze(Variables variables) {
+    AStatement analyze(Variables variables) {
         if (statements == null || statements.isEmpty()) {
             throw new IllegalArgumentException(error("A block must contain at least one statement."));
         }
 
-        final AStatement last = statements.get(statements.size() - 1);
-
-        for (AStatement statement : statements) {
+        for (int index = 0; index < statements.size(); ++index) {
             if (allEscape) {
                 throw new IllegalArgumentException(error("Unreachable statement."));
             }
 
-            statement.inLoop = inLoop;
-            statement.lastSource = lastSource && statement == last;
-            statement.lastLoop = (beginLoop || lastLoop) && statement == last;
+            AStatement statement = statements.get(index);
 
-            statement.analyze(variables);
+            statement.inLoop = inLoop;
+            statement.lastSource = lastSource && index == statements.size() - 1;
+            statement.lastLoop = (beginLoop || lastLoop) && index == statements.size() - 1;
+
+            statements.set(index, statement.analyze(variables));
 
             methodEscape = statement.methodEscape;
             loopEscape = statement.loopEscape;
@@ -64,6 +63,8 @@ public final class SBlock extends AStatement {
             anyBreak |= statement.anyBreak;
             statementCount += statement.statementCount;
         }
+
+        return this;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
index 4cd1decb26b..14a27bcbaac 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
@@ -32,7 +32,7 @@ public final class SBreak extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
+    AStatement analyze(Variables variables) {
         if (!inLoop) {
             throw new IllegalArgumentException(error("Break statement outside of a loop."));
         }
@@ -41,6 +41,8 @@ public final class SBreak extends AStatement {
         allEscape = true;
         anyBreak = true;
         statementCount = 1;
+
+        return this;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
index 3fb8c8b4680..a35ca88d7f6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
@@ -34,7 +34,7 @@ public final class SCatch extends AStatement {
 
     final String type;
     final String name;
-    final SBlock block;
+    AStatement block;
 
     Variable variable;
 
@@ -51,7 +51,7 @@ public final class SCatch extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
+    AStatement analyze(Variables variables) {
         final Type type;
 
         try {
@@ -71,7 +71,7 @@ public final class SCatch extends AStatement {
             block.inLoop = inLoop;
             block.lastLoop = lastLoop;
 
-            block.analyze(variables);
+            block = block.analyze(variables);
 
             methodEscape = block.methodEscape;
             loopEscape = block.loopEscape;
@@ -80,11 +80,14 @@ public final class SCatch extends AStatement {
             anyBreak = block.anyBreak;
             statementCount = block.statementCount;
         }
+
+        return this;
     }
 
     @Override
     void write(MethodWriter writer) {
         writer.writeStatementOffset(offset);
+
         Label jump = new Label();
 
         writer.mark(jump);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
index 18ce5c81231..c5b486db942 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
@@ -32,7 +32,7 @@ public final class SContinue extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
+    AStatement analyze(Variables variables) {
         if (!inLoop) {
             throw new IllegalArgumentException(error("Continue statement outside of a loop."));
         }
@@ -44,6 +44,8 @@ public final class SContinue extends AStatement {
         allEscape = true;
         anyContinue = true;
         statementCount = 1;
+
+        return this;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
index ddfe54c22eb..214d7beb576 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
@@ -22,7 +22,6 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
 
-import java.util.Collections;
 import java.util.List;
 
 /**
@@ -35,21 +34,25 @@ public final class SDeclBlock extends AStatement {
     public SDeclBlock(int line, int offset, String location, List<SDeclaration> declarations) {
         super(line, offset, location);
 
-        this.declarations = Collections.unmodifiableList(declarations);
+        this.declarations = declarations;
     }
 
     @Override
-    void analyze(Variables variables) {
-        for (SDeclaration declaration : declarations) {
-            declaration.analyze(variables);
+    AStatement analyze(Variables variables) {
+        for (int index = 0; index < declarations.size(); ++index) {
+            AStatement declaration = declarations.get(index);
+
+            declarations.set(index, (SDeclaration)declaration.analyze(variables));
         }
 
         statementCount = declarations.size();
+
+        return this;
     }
 
     @Override
     void write(MethodWriter writer) {
-        for (SDeclaration declaration : declarations) {
+        for (AStatement declaration : declarations) {
             declaration.write(writer);
         }
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
index 526d90dca26..6dc0a1ad78a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
@@ -46,7 +46,7 @@ public final class SDeclaration extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
+    AStatement analyze(Variables variables) {
         final Type type;
 
         try {
@@ -62,11 +62,14 @@ public final class SDeclaration extends AStatement {
         }
 
         variable = variables.addVariable(location, type, name, false, false);
+
+        return this;
     }
 
     @Override
     void write(MethodWriter writer) {
         writer.writeStatementOffset(offset);
+
         if (expression == null) {
             switch (variable.type.sort) {
                 case VOID:   throw new IllegalStateException(error("Illegal tree structure."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
index 3b854ea4ebf..efd287669c7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -30,7 +30,7 @@ import org.elasticsearch.painless.MethodWriter;
 public final class SDo extends AStatement {
 
     final int maxLoopCounter;
-    final SBlock block;
+    AStatement block;
     AExpression condition;
 
     public SDo(int line, int offset, String location, int maxLoopCounter, SBlock block, AExpression condition) {
@@ -42,7 +42,7 @@ public final class SDo extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
+    AStatement analyze(Variables variables) {
         variables.incrementScope();
 
         if (block == null) {
@@ -52,7 +52,7 @@ public final class SDo extends AStatement {
         block.beginLoop = true;
         block.inLoop = true;
 
-        block.analyze(variables);
+        block = block.analyze(variables);
 
         if (block.loopEscape && !block.anyContinue) {
             throw new IllegalArgumentException(error("Extraneous do while loop."));
@@ -82,11 +82,14 @@ public final class SDo extends AStatement {
         }
 
         variables.decrementScope();
+
+        return this;
     }
 
     @Override
     void write(MethodWriter writer) {
         writer.writeStatementOffset(offset);
+
         Label start = new Label();
         Label begin = new Label();
         Label end = new Label();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index 37f1cbeb24c..d15a57422b6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -38,7 +38,7 @@ public final class SExpression extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
+    AStatement analyze(Variables variables) {
         expression.read = lastSource;
         expression.analyze(variables);
 
@@ -56,6 +56,8 @@ public final class SExpression extends AStatement {
         loopEscape = rtn;
         allEscape = rtn;
         statementCount = 1;
+
+        return this;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index ac8e721a1f6..1bb18bea6c8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -33,7 +33,7 @@ public final class SFor extends AStatement {
     ANode initializer;
     AExpression condition;
     AExpression afterthought;
-    final SBlock block;
+    AStatement block;
 
     public SFor(int line, int offset, String location, int maxLoopCounter,
                 ANode initializer, AExpression condition, AExpression afterthought, SBlock block) {
@@ -47,14 +47,14 @@ public final class SFor extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
+    AStatement analyze(Variables variables) {
         variables.incrementScope();
 
         boolean continuous = false;
 
         if (initializer != null) {
-            if (initializer instanceof SDeclBlock) {
-                ((SDeclBlock)initializer).analyze(variables);
+            if (initializer instanceof AStatement) {
+                initializer = ((AStatement)initializer).analyze(variables);
             } else if (initializer instanceof AExpression) {
                 AExpression initializer = (AExpression)this.initializer;
 
@@ -102,7 +102,7 @@ public final class SFor extends AStatement {
             block.beginLoop = true;
             block.inLoop = true;
 
-            block.analyze(variables);
+            block = block.analyze(variables);
 
             if (block.loopEscape && !block.anyContinue) {
                 throw new IllegalArgumentException(error("Extraneous for loop."));
@@ -123,11 +123,14 @@ public final class SFor extends AStatement {
         }
 
         variables.decrementScope();
+
+        return this;
     }
 
     @Override
     void write(MethodWriter writer) {
         writer.writeStatementOffset(offset);
+
         Label start = new Label();
         Label begin = afterthought == null ? start : new Label();
         Label end = new Label();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
index 1f498215478..46751e8b449 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
@@ -30,7 +30,7 @@ import org.elasticsearch.painless.MethodWriter;
 public final class SIf extends AStatement {
 
     AExpression condition;
-    final SBlock ifblock;
+    AStatement ifblock;
 
     public SIf(int line, int offset, String location, AExpression condition, SBlock ifblock) {
         super(line, offset, location);
@@ -40,7 +40,7 @@ public final class SIf extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
+    AStatement analyze(Variables variables) {
         condition.expected = Definition.BOOLEAN_TYPE;
         condition.analyze(variables);
         condition = condition.cast(variables);
@@ -58,17 +58,20 @@ public final class SIf extends AStatement {
         ifblock.lastLoop = lastLoop;
 
         variables.incrementScope();
-        ifblock.analyze(variables);
+        ifblock = ifblock.analyze(variables);
         variables.decrementScope();
 
         anyContinue = ifblock.anyContinue;
         anyBreak = ifblock.anyBreak;
         statementCount = ifblock.statementCount;
+
+        return this;
     }
 
     @Override
     void write(MethodWriter writer) {
         writer.writeStatementOffset(offset);
+
         Label fals = new Label();
 
         condition.fals = fals;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
index d8fd5c0756e..f484819c4a2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
@@ -30,8 +30,8 @@ import org.elasticsearch.painless.MethodWriter;
 public final class SIfElse extends AStatement {
 
     AExpression condition;
-    final SBlock ifblock;
-    final SBlock elseblock;
+    AStatement ifblock;
+    AStatement elseblock;
 
     public SIfElse(int line, int offset, String location, AExpression condition, SBlock ifblock, SBlock elseblock) {
         super(line, offset, location);
@@ -42,7 +42,7 @@ public final class SIfElse extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
+    AStatement analyze(Variables variables) {
         condition.expected = Definition.BOOLEAN_TYPE;
         condition.analyze(variables);
         condition = condition.cast(variables);
@@ -60,7 +60,7 @@ public final class SIfElse extends AStatement {
         ifblock.lastLoop = lastLoop;
 
         variables.incrementScope();
-        ifblock.analyze(variables);
+        ifblock = ifblock.analyze(variables);
         variables.decrementScope();
 
         anyContinue = ifblock.anyContinue;
@@ -76,7 +76,7 @@ public final class SIfElse extends AStatement {
         elseblock.lastLoop = lastLoop;
 
         variables.incrementScope();
-        elseblock.analyze(variables);
+        elseblock = elseblock.analyze(variables);
         variables.decrementScope();
 
         methodEscape = ifblock.methodEscape && elseblock.methodEscape;
@@ -85,11 +85,14 @@ public final class SIfElse extends AStatement {
         anyContinue |= elseblock.anyContinue;
         anyBreak |= elseblock.anyBreak;
         statementCount = Math.max(ifblock.statementCount, elseblock.statementCount);
+
+        return this;
     }
 
     @Override
     void write(MethodWriter writer) {
         writer.writeStatementOffset(offset);
+
         Label end = new Label();
         Label fals = elseblock != null ? new Label() : end;
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
index d6cc5598263..4af533cdf84 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -37,7 +37,7 @@ public final class SReturn extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
+    AStatement analyze(Variables variables) {
         expression.expected = Definition.OBJECT_TYPE;
         expression.internal = true;
         expression.analyze(variables);
@@ -48,6 +48,8 @@ public final class SReturn extends AStatement {
         allEscape = true;
 
         statementCount = 1;
+
+        return this;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index 899d54225dc..c1ac248b976 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -23,7 +23,6 @@ import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Opcodes;
 import org.elasticsearch.painless.MethodWriter;
 
-import java.util.Collections;
 import java.util.List;
 
 /**
@@ -36,32 +35,34 @@ public final class SSource extends AStatement {
     public SSource(int line, int offset, String location, List<AStatement> statements) {
         super(line, offset, location);
 
-        this.statements = Collections.unmodifiableList(statements);
+        this.statements = statements;
     }
 
     @Override
-    public void analyze(Variables variables) {
+    public AStatement analyze(Variables variables) {
         if (statements == null || statements.isEmpty()) {
             throw new IllegalArgumentException(error("Cannot generate an empty script."));
         }
 
         variables.incrementScope();
 
-        final AStatement last = statements.get(statements.size() - 1);
+        for (int index = 0; index < statements.size(); ++index) {
+            AStatement statement = statements.get(index);
 
-        for (AStatement statement : statements) {
             if (allEscape) {
                 throw new IllegalArgumentException(error("Unreachable statement."));
             }
 
-            statement.lastSource = statement == last;
-            statement.analyze(variables);
+            statement.lastSource = index == statements.size() - 1;
+            statements.set(index, statement.analyze(variables));
 
             methodEscape = statement.methodEscape;
             allEscape = statement.allEscape;
         }
 
         variables.decrementScope();
+
+        return this;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
index 7c65aafbc56..fb0a260e3c9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
@@ -37,7 +37,7 @@ public final class SThrow extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
+    AStatement analyze(Variables variables) {
         expression.expected = Definition.EXCEPTION_TYPE;
         expression.analyze(variables);
         expression = expression.cast(variables);
@@ -46,6 +46,8 @@ public final class SThrow extends AStatement {
         loopEscape = true;
         allEscape = true;
         statementCount = 1;
+
+        return this;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
index 0fdb70bdc82..41c5612625e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
@@ -23,7 +23,6 @@ import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.MethodWriter;
 
-import java.util.Collections;
 import java.util.List;
 
 /**
@@ -31,18 +30,18 @@ import java.util.List;
  */
 public final class STry extends AStatement {
 
-    final SBlock block;
+    AStatement block;
     final List<SCatch> catches;
 
-    public STry(int line, int offset, String location, SBlock block, List<SCatch> traps) {
+    public STry(int line, int offset, String location, SBlock block, List<SCatch> catches) {
         super(line, offset, location);
 
         this.block = block;
-        this.catches = Collections.unmodifiableList(traps);
+        this.catches = catches;
     }
 
     @Override
-    void analyze(Variables variables) {
+    AStatement analyze(Variables variables) {
         if (block == null) {
             throw new IllegalArgumentException(error("Extraneous try statement."));
         }
@@ -52,7 +51,7 @@ public final class STry extends AStatement {
         block.lastLoop = lastLoop;
 
         variables.incrementScope();
-        block.analyze(variables);
+        block = block.analyze(variables);
         variables.decrementScope();
 
         methodEscape = block.methodEscape;
@@ -63,13 +62,15 @@ public final class STry extends AStatement {
 
         int statementCount = 0;
 
-        for (SCatch catc : catches) {
+        for (int index = 0; index < catches.size(); ++index) {
+            SCatch catc = catches.get(index);
+
             catc.lastSource = lastSource;
             catc.inLoop = inLoop;
             catc.lastLoop = lastLoop;
 
             variables.incrementScope();
-            catc.analyze(variables);
+            catches.set(index, (SCatch)catc.analyze(variables));
             variables.decrementScope();
 
             methodEscape &= catc.methodEscape;
@@ -82,11 +83,14 @@ public final class STry extends AStatement {
         }
 
         this.statementCount = block.statementCount + statementCount;
+
+        return this;
     }
 
     @Override
     void write(MethodWriter writer) {
         writer.writeStatementOffset(offset);
+
         Label begin = new Label();
         Label end = new Label();
         Label exception = new Label();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
index 43ac824dac7..3103b1b46d1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -31,7 +31,7 @@ public final class SWhile extends AStatement {
 
     final int maxLoopCounter;
     AExpression condition;
-    final SBlock block;
+    AStatement block;
 
     public SWhile(int line, int offset, String location, int maxLoopCounter, AExpression condition, SBlock block) {
         super(line, offset, location);
@@ -42,7 +42,7 @@ public final class SWhile extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
+    AStatement analyze(Variables variables) {
         variables.incrementScope();
 
         condition.expected = Definition.BOOLEAN_TYPE;
@@ -67,7 +67,7 @@ public final class SWhile extends AStatement {
             block.beginLoop = true;
             block.inLoop = true;
 
-            block.analyze(variables);
+            block = block.analyze(variables);
 
             if (block.loopEscape && !block.anyContinue) {
                 throw new IllegalArgumentException(error("Extraneous while loop."));
@@ -88,11 +88,14 @@ public final class SWhile extends AStatement {
         }
 
         variables.decrementScope();
+
+        return this;
     }
 
     @Override
     void write(MethodWriter writer) {
         writer.writeStatementOffset(offset);
+
         Label begin = new Label();
         Label end = new Label();
 

From 32e9a879b426ae72d9f4d9c6968a813823bd1272 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Sat, 28 May 2016 08:14:49 +0200
Subject: [PATCH 0510/1311] Use at least 2 docs to see deleted docs

Closes #18623
---
 .../org/elasticsearch/index/engine/InternalEngineTests.java     | 2 +-
 .../java/org/elasticsearch/index/engine/ShadowEngineTests.java  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 4246e075b4c..51df3ee0386 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -2184,7 +2184,7 @@ public class InternalEngineTests extends ESTestCase {
     }
 
     public void testDocStats() throws IOException {
-        final int numDocs = randomIntBetween(1, 10);
+        final int numDocs = randomIntBetween(2, 10); // at least 2 documents otherwise we don't see any deletes below
         for (int i = 0; i < numDocs; i++) {
             ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null);
             Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(i)), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime());
diff --git a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
index 8595f235c0e..f71f2d72019 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
@@ -986,7 +986,7 @@ public class ShadowEngineTests extends ESTestCase {
     }
 
     public void testDocStats() throws IOException {
-        final int numDocs = randomIntBetween(1, 10);
+        final int numDocs = randomIntBetween(2, 10); // at least 2 documents otherwise we don't see any deletes below
         for (int i = 0; i < numDocs; i++) {
             ParsedDocument doc = testParsedDocument(Integer.toString(i), Integer.toString(i), "test", null, -1, -1, testDocument(), new BytesArray("{}"), null);
             Engine.Index firstIndexRequest = new Engine.Index(newUid(Integer.toString(i)), doc, Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime());

From b2f2d38ebb399a812a0f85f9f83ab9c2c9f69c08 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Sun, 29 May 2016 11:50:36 +0200
Subject: [PATCH 0511/1311] Update configuring.asciidoc

Minor asciidoc changes
---
 docs/reference/setup/sysconfig/configuring.asciidoc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/reference/setup/sysconfig/configuring.asciidoc b/docs/reference/setup/sysconfig/configuring.asciidoc
index 0f8f0725103..84d0f832141 100644
--- a/docs/reference/setup/sysconfig/configuring.asciidoc
+++ b/docs/reference/setup/sysconfig/configuring.asciidoc
@@ -105,10 +105,10 @@ LimitMEMLOCK=infinity
 ==== Setting JVM system properties
 
 The preferred method of setting Java Virtual Machine options (including
-system properties and JVM flags) is via the jvm.options configuration
-file. The default location of this file is config/jvm.options (when
+system properties and JVM flags) is via the `jvm.options` configuration
+file. The default location of this file is `config/jvm.options` (when
 installing from the tar or zip distributions) and
-/etc/elasticsearch/jvm.options (when installing from the Debian or RPM
+`/etc/elasticsearch/jvm.options` (when installing from the Debian or RPM
 packages). This file contains a line-delimited list of JVM arguments,
 which must begin with `-`. You can add custom JVM flags to this file and
 check this configuration into your version control system.

From e35bd115816cdc24f6f73cffd3b8cfff32be898b Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Sun, 29 May 2016 11:56:02 +0200
Subject: [PATCH 0512/1311] Update bootstrap-checks.asciidoc

Fixed asciidoc
---
 docs/reference/setup/bootstrap-checks.asciidoc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc
index bfa43727d28..3fe4e33a7eb 100644
--- a/docs/reference/setup/bootstrap-checks.asciidoc
+++ b/docs/reference/setup/bootstrap-checks.asciidoc
@@ -16,6 +16,7 @@ checks that fail appear as warnings in the Elasticsearch log. If
 Elasticsearch is in production mode, any bootstrap checks that fail will
 cause Elasticsearch to refuse to start.
 
+[float]
 === Development vs. production mode
 
 By default, Elasticsearch binds and publishes to `localhost`. This is
@@ -45,7 +46,7 @@ check, you must configure the <<heap-size,heap size>>.
 File descriptors are a Unix construct for tracking open "files". In Unix
 though, https://en.wikipedia.org/wiki/Everything_is_a_file[everything is
 a file]. For example, "files" could be a physical file, a virtual file
-(e.g., ``/proc/loadavg`), or network sockets. Elasticsearch requires
+(e.g., `/proc/loadavg`), or network sockets. Elasticsearch requires
 lots file descriptors (e.g., every shard is composed of multiple
 segments and other files, plus connections to other nodes, etc.). This
 bootstrap check is enforced on OS X and Linux. To pass the file

From 37a3588c37f8b2ccb6d5916336692f91a3a7de22 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Sun, 29 May 2016 07:59:51 -0400
Subject: [PATCH 0513/1311] Fix min. master nodes links in boostrap check docs

This commit fixes two links to the minimum master nodes configuration
section of the docs in the bootstrap check docs.
---
 docs/reference/setup/bootstrap-checks.asciidoc | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc
index 3fe4e33a7eb..fea711a52f0 100644
--- a/docs/reference/setup/bootstrap-checks.asciidoc
+++ b/docs/reference/setup/bootstrap-checks.asciidoc
@@ -86,14 +86,15 @@ equal to a quorum of master-eligible nodes, it is not possible for the
 cluster to suffer from split brain because during a network partition
 there can be at most one side of the partition that contains a quorum of
 master nodes. The minimum master nodes check enforces that you've set
-<<minimum_master_nodes>>. To pass the minimum master nodes check, you
-must configure
+<<minimum_master_nodes,`discovery.zen.minimum_master_nodes`>>. To pass
+the minimum master nodes check, you must configure
 <<minimum_master_nodes,`discovery.zen.minimum_master_nodes`>>.
 
 NOTE: The minimum master nodes check does not enforce that you've
 configured <<minimum_master_nodes,`discovery.zen.minimum_master_nodes`>>
 correctly, only that you have it configured. Elasticsearch does log a
-warning message if it detects that <<minimum_master_nodes>> is
+warning message if it detects that
+<<minimum_master_nodes,`discovery.zen.minimum_master_nodes`>> is
 incorrectly configured based on the number of master-eligible nodes
 visible in the cluster state. Future versions of Elasticsearch will
 contain stricter enforcement of

From 16307eee6f4737afd8bb7f480150a537dd1b0c18 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Mon, 30 May 2016 09:19:35 +0200
Subject: [PATCH 0514/1311] Remove extra bin/ directory in bin folder

Recent changes adds an extra bin/ directory that contains Windows bat files in the normal Bin folder of elasticsearch. For exemple the script elasticsearch.bat is located in bin/bin/elasticsearch.bat instead of bin/elasticsearch.bat.
---
 distribution/build.gradle | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distribution/build.gradle b/distribution/build.gradle
index b4c183d9bd1..540796fc5a3 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -196,8 +196,8 @@ configure(subprojects.findAll { ['zip', 'tar', 'integ-test-zip'].contains(it.nam
       into('bin') {
         with copySpec {
           with binFiles
-          from('../src/main/resources') {
-            include 'bin/*.bat'
+          from('../src/main/resources/bin') {
+            include '*.bat'
             filter(FixCrLfFilter, eol: FixCrLfFilter.CrLf.newInstance('crlf'))
           }
           MavenFilteringHack.filter(it, expansions)

From 148fd3f8912fc2ea9e51398aa1de485dc207497c Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Mon, 30 May 2016 11:57:42 +0200
Subject: [PATCH 0515/1311] [TEST] mute testDeleteIndexStore test

Issue: https://github.com/elastic/elasticsearch/issues/18558
---
 .../test/java/org/elasticsearch/indices/IndicesServiceTests.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
index 76f7a30e078..2abf07164bf 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
@@ -105,6 +105,7 @@ public class IndicesServiceTests extends ESSingleNodeTestCase {
             indicesService.canDeleteShardContent(notAllocated, test.getIndexSettings()));
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/18558")
     public void testDeleteIndexStore() throws Exception {
         IndicesService indicesService = getIndicesService();
         IndexService test = createIndex("test");

From fc1696822fa2444a89e8ddcff145a8ed4d8217a3 Mon Sep 17 00:00:00 2001
From: Britta Weber <brwe@users.noreply.github.com>
Date: Mon, 30 May 2016 15:26:20 +0200
Subject: [PATCH 0516/1311] Fix filtering of node ids for TransportNodesAction
 (#18634)

* Fix filtering of node ids for TransportNodesAction

Don't mix up member variables  with local variables
in constructor.

closes #18618
---
 .../support/nodes/TransportNodesAction.java   |  6 +-
 .../nodes/TransportNodesActionTests.java      | 64 ++++++++++++++++---
 2 files changed, 58 insertions(+), 12 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
index 39d6bf767e1..2b72ef8b31b 100644
--- a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
@@ -176,12 +176,10 @@ public abstract class TransportNodesAction<NodesRequest extends BaseNodesRequest
             this.request = request;
             this.listener = listener;
             ClusterState clusterState = clusterService.state();
-            String[] nodesIds = resolveNodes(request, clusterState);
-            this.nodesIds = filterNodeIds(clusterState.nodes(), nodesIds);
-            ImmutableOpenMap<String, DiscoveryNode> nodes = clusterState.nodes().getNodes();
+            nodesIds = filterNodeIds(clusterState.nodes(), resolveNodes(request, clusterState));
             this.nodes = new DiscoveryNode[nodesIds.length];
             for (int i = 0; i < nodesIds.length; i++) {
-                this.nodes[i] = nodes.get(nodesIds[i]);
+                this.nodes[i] = clusterState.nodes().get(nodesIds[i]);
             }
             this.responses = new AtomicReferenceArray<>(this.nodesIds.length);
         }
diff --git a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
index a7289124704..c53adb08ce2 100644
--- a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
@@ -66,9 +66,10 @@ public class TransportNodesActionTests extends ESTestCase {
 
     private ClusterService clusterService;
     private CapturingTransport transport;
-    private TestTransportNodesAction action;
+    private TransportService transportService;
 
     public void testRequestIsSentToEachNode() throws Exception {
+        TransportNodesAction action = getTestTransportNodesAction();
         TestNodesRequest request = new TestNodesRequest();
         PlainActionFuture<TestNodesResponse> listener = new PlainActionFuture<>();
         action.new AsyncAction(null, request, listener).start();
@@ -79,6 +80,7 @@ public class TransportNodesActionTests extends ESTestCase {
     }
 
     public void testNodesSelectors() {
+        TransportNodesAction action = getTestTransportNodesAction();
         int numSelectors = randomIntBetween(1, 5);
         Set<String> nodeSelectors = new HashSet<>();
         for (int i = 0; i < numSelectors; i++) {
@@ -98,10 +100,12 @@ public class TransportNodesActionTests extends ESTestCase {
     }
 
     public void testNewResponseNullArray() {
+        TransportNodesAction action = getTestTransportNodesAction();
         expectThrows(NullPointerException.class, () -> action.newResponse(new TestNodesRequest(), null));
     }
 
     public void testNewResponse() {
+        TestTransportNodesAction action = getTestTransportNodesAction();
         TestNodesRequest request = new TestNodesRequest();
         List<TestNodeResponse> expectedNodeResponses = mockList(TestNodeResponse.class, randomIntBetween(0, 2));
         expectedNodeResponses.add(new TestNodeResponse());
@@ -125,6 +129,19 @@ public class TransportNodesActionTests extends ESTestCase {
         assertTrue(failures.containsAll(response.failures()));
     }
 
+    public void testFiltering() throws Exception {
+        TransportNodesAction action = getFilteringTestTransportNodesAction(transportService);
+        TestNodesRequest request = new TestNodesRequest();
+        PlainActionFuture<TestNodesResponse> listener = new PlainActionFuture<>();
+        action.new AsyncAction(null, request, listener).start();
+        Map<String, List<CapturingTransport.CapturedRequest>> capturedRequests = transport.getCapturedRequestsByTargetNodeAndClear();
+        // check requests were only sent to data nodes
+        for (String nodeTarget : capturedRequests.keySet()) {
+            assertTrue(clusterService.state().nodes().get(nodeTarget).isDataNode());
+        }
+        assertEquals(clusterService.state().nodes().getDataNodes().size(), capturedRequests.size());
+    }
+
     private <T> List<T> mockList(Class<T> clazz, int size) {
         List<T> failures = new ArrayList<>(size);
         for (int i = 0; i < size; ++i) {
@@ -160,7 +177,7 @@ public class TransportNodesActionTests extends ESTestCase {
         super.setUp();
         transport = new CapturingTransport();
         clusterService = createClusterService(THREAD_POOL);
-        final TransportService transportService = new TransportService(transport, THREAD_POOL, clusterService.state().getClusterName());
+        transportService = new TransportService(transport, THREAD_POOL, clusterService.state().getClusterName());
         transportService.start();
         transportService.acceptIncomingRequests();
         int numNodes = randomIntBetween(3, 10);
@@ -182,7 +199,17 @@ public class TransportNodesActionTests extends ESTestCase {
         stateBuilder.nodes(discoBuilder);
         ClusterState clusterState = stateBuilder.build();
         setState(clusterService, clusterState);
-        action = new TestTransportNodesAction(
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        super.tearDown();
+        clusterService.close();
+        transport.close();
+    }
+
+    public TestTransportNodesAction getTestTransportNodesAction() {
+        return new TestTransportNodesAction(
                 Settings.EMPTY,
                 THREAD_POOL,
                 clusterService,
@@ -194,11 +221,17 @@ public class TransportNodesActionTests extends ESTestCase {
         );
     }
 
-    @After
-    public void tearDown() throws Exception {
-        super.tearDown();
-        clusterService.close();
-        transport.close();
+    public FilteringTestTransportNodesAction getFilteringTestTransportNodesAction(TransportService transportService) {
+        return new FilteringTestTransportNodesAction(
+            Settings.EMPTY,
+            THREAD_POOL,
+            clusterService,
+            transportService,
+            new ActionFilters(Collections.emptySet()),
+            TestNodesRequest::new,
+            TestNodeRequest::new,
+            ThreadPool.Names.SAME
+        );
     }
 
     private static DiscoveryNode newNode(int nodeId, Map<String, String> attributes, Set<DiscoveryNode.Role> roles) {
@@ -243,6 +276,21 @@ public class TransportNodesActionTests extends ESTestCase {
         }
     }
 
+    private static class FilteringTestTransportNodesAction
+        extends TestTransportNodesAction {
+
+        FilteringTestTransportNodesAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService
+            transportService, ActionFilters actionFilters, Supplier<TestNodesRequest> request,
+                                          Supplier<TestNodeRequest> nodeRequest, String nodeExecutor) {
+            super(settings, threadPool, clusterService, transportService, actionFilters, request, nodeRequest, nodeExecutor);
+        }
+
+        @Override
+        protected String[] filterNodeIds(DiscoveryNodes nodes, String[] nodesIds) {
+            return nodes.getDataNodes().keys().toArray(String.class);
+        }
+    }
+
     private static class TestNodesRequest extends BaseNodesRequest<TestNodesRequest> {
         TestNodesRequest(String... nodesIds) {
             super(nodesIds);

From a1c4086a172b67fa8f19423b91a29d2ed467f00b Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Mon, 30 May 2016 16:55:44 +0200
Subject: [PATCH 0517/1311] No need to inject the ClusterService into
 IndicesService (#18639)

It's already available where it's needed.
---
 .../main/java/org/elasticsearch/indices/IndicesService.java  | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index 6b1d24e47f1..188e3608cb3 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -133,7 +133,6 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
     private final TimeValue shardsClosedTimeout;
     private final AnalysisRegistry analysisRegistry;
     private final IndicesQueriesRegistry indicesQueriesRegistry;
-    private final ClusterService clusterService;
     private final IndexNameExpressionResolver indexNameExpressionResolver;
     private final IndexScopedSettings indexScopeSetting;
     private final IndicesFieldDataCache indicesFieldDataCache;
@@ -162,7 +161,7 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
     public IndicesService(Settings settings, PluginsService pluginsService, NodeEnvironment nodeEnv,
                           ClusterSettings clusterSettings, AnalysisRegistry analysisRegistry,
                           IndicesQueriesRegistry indicesQueriesRegistry, IndexNameExpressionResolver indexNameExpressionResolver,
-                          ClusterService clusterService, MapperRegistry mapperRegistry, NamedWriteableRegistry namedWriteableRegistry,
+                          MapperRegistry mapperRegistry, NamedWriteableRegistry namedWriteableRegistry,
                           ThreadPool threadPool, IndexScopedSettings indexScopedSettings, CircuitBreakerService circuitBreakerService,
                           MetaStateService metaStateService) {
         super(settings);
@@ -173,7 +172,6 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
         this.indexStoreConfig = new IndexStoreConfig(settings);
         this.analysisRegistry = analysisRegistry;
         this.indicesQueriesRegistry = indicesQueriesRegistry;
-        this.clusterService = clusterService;
         this.indexNameExpressionResolver = indexNameExpressionResolver;
         this.indicesRequestCache = new IndicesRequestCache(settings);
         this.indicesQueryCache = new IndicesQueryCache(settings);
@@ -379,6 +377,7 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
      */
     private synchronized IndexService createIndexService(final String reason, final NodeServicesProvider nodeServicesProvider, IndexMetaData indexMetaData, IndicesQueryCache indicesQueryCache, IndicesFieldDataCache indicesFieldDataCache, List<IndexEventListener> builtInListeners, IndexingOperationListener... indexingOperationListeners) throws IOException {
         final Index index = indexMetaData.getIndex();
+        final ClusterService clusterService = nodeServicesProvider.getClusterService();
         final Predicate<String> indexNameMatcher = (indexExpression) -> indexNameExpressionResolver.matchesIndex(index.getName(), indexExpression, clusterService.state());
         final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, indexNameMatcher, indexScopeSetting);
         logger.debug("creating Index [{}], shards [{}]/[{}{}] - reason [{}]",

From 04cae88ff4547ee63f8012783b7146f3d846064f Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 30 May 2016 10:46:37 -0400
Subject: [PATCH 0518/1311] Do not use Lucene SuppressForbidden

Lucene SuppressForbidden is marked lucene.internal and should not be
used outside of Lucene. This commit removes the uses of this class
within Elasticsearch. Instead,
org.elasticsearch.common.SuppressForbidden should be used, which was
already the case in most places.
---
 .../main/java/org/elasticsearch/bootstrap/BootstrapCheck.java  | 2 +-
 .../main/java/org/elasticsearch/common/logging/Loggers.java    | 2 +-
 core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java   | 2 +-
 .../org/elasticsearch/plugins/InstallPluginCommandTests.java   | 3 +--
 4 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
index 925940a3903..0ee5099dc69 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.bootstrap;
 
 import org.apache.lucene.util.Constants;
-import org.apache.lucene.util.SuppressForbidden;
+import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
diff --git a/core/src/main/java/org/elasticsearch/common/logging/Loggers.java b/core/src/main/java/org/elasticsearch/common/logging/Loggers.java
index a195ab450b5..674522589ae 100644
--- a/core/src/main/java/org/elasticsearch/common/logging/Loggers.java
+++ b/core/src/main/java/org/elasticsearch/common/logging/Loggers.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.common.logging;
 
-import org.apache.lucene.util.SuppressForbidden;
 import org.elasticsearch.common.Classes;
+import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
diff --git a/core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java b/core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
index 5ee2232068f..c6b4f080b91 100644
--- a/core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
+++ b/core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.monitor.os;
 
 import org.apache.lucene.util.Constants;
-import org.apache.lucene.util.SuppressForbidden;
+import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.monitor.Probes;
 
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
index 2ca46b4effc..5c766018f13 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
@@ -23,10 +23,10 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.google.common.jimfs.Configuration;
 import com.google.common.jimfs.Jimfs;
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.SuppressForbidden;
 import org.elasticsearch.Version;
 import org.elasticsearch.cli.MockTerminal;
 import org.elasticsearch.cli.UserError;
+import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.io.PathUtilsForTesting;
@@ -63,7 +63,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
-import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;

From f74a78940cadf3c809bc0e23fb6fd71b55bec6f1 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Mon, 30 May 2016 17:22:03 +0200
Subject: [PATCH 0519/1311] Improve phrase suggest test speed (#18633)

There is no reason to read the entire marvel hero file to test the features,
it might take several seconds to do so which is unnecessary.
This commit also splits SearchSuggestTests into core and modules/mustache
also add @Nighlty to forbidden API to make sure we don't use it since they won't run in CI these days.
---
 .../resources/checkstyle_suppressions.xml     |    1 -
 .../forbidden/es-test-signatures.txt          |    2 +
 .../search/basic/SearchWhileRelocatingIT.java |    9 -
 .../search/suggest/SuggestSearchIT.java       | 1135 +++++++++++++++++
 .../phrase/NoisyChannelSpellCheckerTests.java |   69 +-
 .../messy/tests/SuggestSearchTests.java       | 1090 +---------------
 6 files changed, 1223 insertions(+), 1083 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 63133dd851b..6432ccb3ddd 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -1197,7 +1197,6 @@
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]groovy[/\\]GroovySecurityTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-mustache[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]mustache[/\\]MustachePlugin.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-mustache[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]RenderSearchTemplateTests.java" checks="LineLength" />
-  <suppress files="modules[/\\]lang-mustache[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]SuggestSearchTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-mustache[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]TemplateQueryParserTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-mustache[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]TemplateQueryTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-mustache[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]package-info.java" checks="LineLength" />
diff --git a/buildSrc/src/main/resources/forbidden/es-test-signatures.txt b/buildSrc/src/main/resources/forbidden/es-test-signatures.txt
index bd6744ee05f..08e591e1cfa 100644
--- a/buildSrc/src/main/resources/forbidden/es-test-signatures.txt
+++ b/buildSrc/src/main/resources/forbidden/es-test-signatures.txt
@@ -21,5 +21,7 @@ com.carrotsearch.randomizedtesting.annotations.Repeat @ Don't commit hardcoded r
 org.apache.lucene.codecs.Codec#setDefault(org.apache.lucene.codecs.Codec) @ Use the SuppressCodecs("*") annotation instead
 org.apache.lucene.util.LuceneTestCase$Slow @ Don't write slow tests
 org.junit.Ignore @ Use AwaitsFix instead
+org.apache.lucene.util.LuceneTestCase$Nightly @ We don't run nightly tests at this point!
+com.carrotsearch.randomizedtesting.annotations.Nightly @ We don't run nightly tests at this point!
 
 org.junit.Test @defaultMessage Just name your test method testFooBar
diff --git a/core/src/test/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java b/core/src/test/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java
index a7648e18cd6..98ae3241dbb 100644
--- a/core/src/test/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java
+++ b/core/src/test/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java
@@ -43,15 +43,6 @@ import static org.hamcrest.Matchers.is;
 
 @ESIntegTestCase.ClusterScope(minNumDataNodes = 2)
 public class SearchWhileRelocatingIT extends ESIntegTestCase {
-    @Nightly
-    public void testSearchAndRelocateConcurrently0Replicas() throws Exception {
-        testSearchAndRelocateConcurrently(0);
-    }
-
-    @Nightly
-    public void testSearchAndRelocateConcurrently1Replicas() throws Exception {
-        testSearchAndRelocateConcurrently(1);
-    }
 
     public void testSearchAndRelocateConcurrentlyRanodmReplicas() throws Exception {
         testSearchAndRelocateConcurrently(randomIntBetween(0, 1));
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java
new file mode 100644
index 00000000000..09129072177
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/suggest/SuggestSearchIT.java
@@ -0,0 +1,1135 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.suggest;
+
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.ReduceSearchPhaseException;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.search.suggest.phrase.DirectCandidateGeneratorBuilder;
+import org.elasticsearch.search.suggest.phrase.Laplace;
+import org.elasticsearch.search.suggest.phrase.LinearInterpolation;
+import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
+import org.elasticsearch.search.suggest.phrase.StupidBackoff;
+import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
+import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SuggestMode;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.ExecutionException;
+
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
+import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
+import static org.elasticsearch.search.suggest.SuggestBuilders.phraseSuggestion;
+import static org.elasticsearch.search.suggest.SuggestBuilders.termSuggestion;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestion;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionSize;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.endsWith;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.nullValue;
+
+/**
+ * Integration tests for term and phrase suggestions.  Many of these tests many requests that vary only slightly from one another.  Where
+ * possible these tests should declare for the first request, make the request, modify the configuration for the next request, make that
+ * request, modify again, request again, etc.  This makes it very obvious what changes between requests.
+ */
+public class SuggestSearchIT extends ESIntegTestCase {
+
+    // see #3196
+    public void testSuggestAcrossMultipleIndices() throws IOException {
+        createIndex("test");
+        ensureGreen();
+
+        index("test", "type1", "1", "text", "abcd");
+        index("test", "type1", "2", "text", "aacd");
+        index("test", "type1", "3", "text", "abbd");
+        index("test", "type1", "4", "text", "abcc");
+        refresh();
+
+        TermSuggestionBuilder termSuggest = termSuggestion("text")
+                .suggestMode(SuggestMode.ALWAYS) // Always, otherwise the results can vary between requests.
+                .text("abcd");
+        logger.info("--> run suggestions with one index");
+        searchSuggest("test", termSuggest);
+        createIndex("test_1");
+        ensureGreen();
+
+        index("test_1", "type1", "1", "text", "ab cd");
+        index("test_1", "type1", "2", "text", "aa cd");
+        index("test_1", "type1", "3", "text", "ab bd");
+        index("test_1", "type1", "4", "text", "ab cc");
+        refresh();
+        termSuggest = termSuggestion("text")
+                .suggestMode(SuggestMode.ALWAYS) // Always, otherwise the results can vary between requests.
+                .text("ab cd")
+                .minWordLength(1);
+        logger.info("--> run suggestions with two indices");
+        searchSuggest("test", termSuggest);
+
+
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
+                .startObject("properties")
+                .startObject("text").field("type", "text").field("analyzer", "keyword").endObject()
+                .endObject()
+                .endObject().endObject();
+        assertAcked(prepareCreate("test_2").addMapping("type1", mapping));
+        ensureGreen();
+
+        index("test_2", "type1", "1", "text", "ab cd");
+        index("test_2", "type1", "2", "text", "aa cd");
+        index("test_2", "type1", "3", "text", "ab bd");
+        index("test_2", "type1", "4", "text", "ab cc");
+        index("test_2", "type1", "1", "text", "abcd");
+        index("test_2", "type1", "2", "text", "aacd");
+        index("test_2", "type1", "3", "text", "abbd");
+        index("test_2", "type1", "4", "text", "abcc");
+        refresh();
+
+        termSuggest = termSuggestion("text")
+                .suggestMode(SuggestMode.ALWAYS) // Always, otherwise the results can vary between requests.
+                .text("ab cd")
+                .minWordLength(1);
+        logger.info("--> run suggestions with three indices");
+        try {
+            searchSuggest("test", termSuggest);
+            fail(" can not suggest across multiple indices with different analysis chains");
+        } catch (ReduceSearchPhaseException ex) {
+            assertThat(ex.getCause(), instanceOf(IllegalStateException.class));
+            assertThat(ex.getCause().getMessage(),
+                    anyOf(endsWith("Suggest entries have different sizes actual [1] expected [2]"),
+                            endsWith("Suggest entries have different sizes actual [2] expected [1]")));
+        } catch (IllegalStateException ex) {
+            assertThat(ex.getMessage(), anyOf(endsWith("Suggest entries have different sizes actual [1] expected [2]"),
+                    endsWith("Suggest entries have different sizes actual [2] expected [1]")));
+        }
+
+
+        termSuggest = termSuggestion("text")
+                .suggestMode(SuggestMode.ALWAYS) // Always, otherwise the results can vary between requests.
+                .text("ABCD")
+                .minWordLength(1);
+        logger.info("--> run suggestions with four indices");
+        try {
+            searchSuggest("test", termSuggest);
+            fail(" can not suggest across multiple indices with different analysis chains");
+        } catch (ReduceSearchPhaseException ex) {
+            assertThat(ex.getCause(), instanceOf(IllegalStateException.class));
+            assertThat(ex.getCause().getMessage(), anyOf(endsWith("Suggest entries have different text actual [ABCD] expected [abcd]"),
+                    endsWith("Suggest entries have different text actual [abcd] expected [ABCD]")));
+        } catch (IllegalStateException ex) {
+            assertThat(ex.getMessage(), anyOf(endsWith("Suggest entries have different text actual [ABCD] expected [abcd]"),
+                    endsWith("Suggest entries have different text actual [abcd] expected [ABCD]")));
+        }
+    }
+
+    // see #3037
+    public void testSuggestModes() throws IOException {
+        CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder()
+                .put(SETTING_NUMBER_OF_SHARDS, 1)
+                .put(SETTING_NUMBER_OF_REPLICAS, 0)
+                .put("index.analysis.analyzer.biword.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.biword.filter", "shingler", "lowercase")
+                .put("index.analysis.filter.shingler.type", "shingle")
+                .put("index.analysis.filter.shingler.min_shingle_size", 2)
+                .put("index.analysis.filter.shingler.max_shingle_size", 3));
+
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
+                .startObject("properties")
+                .startObject("name")
+                    .field("type", "text")
+                    .startObject("fields")
+                        .startObject("shingled")
+                            .field("type", "text")
+                            .field("analyzer", "biword")
+                            .field("search_analyzer", "standard")
+                        .endObject()
+                    .endObject()
+                .endObject()
+                .endObject()
+                .endObject().endObject();
+        assertAcked(builder.addMapping("type1", mapping));
+        ensureGreen();
+
+
+        index("test", "type1", "1", "name", "I like iced tea");
+        index("test", "type1", "2", "name", "I like tea.");
+        index("test", "type1", "3", "name", "I like ice cream.");
+        refresh();
+
+        DirectCandidateGeneratorBuilder generator = candidateGenerator("name").prefixLength(0).minWordLength(0).suggestMode("always")
+            .maxEdits(2);
+        PhraseSuggestionBuilder phraseSuggestion = phraseSuggestion("name.shingled")
+                .addCandidateGenerator(generator)
+                .gramSize(3);
+        Suggest searchSuggest = searchSuggest("ice tea", "did_you_mean", phraseSuggestion);
+        assertSuggestion(searchSuggest, 0, "did_you_mean", "iced tea");
+
+        generator.suggestMode(null);
+        searchSuggest = searchSuggest( "ice tea", "did_you_mean", phraseSuggestion);
+        assertSuggestionSize(searchSuggest, 0, 0, "did_you_mean");
+    }
+
+    /**
+     * Creates a new {@link DirectCandidateGeneratorBuilder}
+     *
+     * @param field
+     *            the field this candidate generator operates on.
+     */
+    private DirectCandidateGeneratorBuilder candidateGenerator(String field) {
+        return new DirectCandidateGeneratorBuilder(field);
+    }
+
+    // see #2729
+    public void testSizeOneShard() throws Exception {
+        prepareCreate("test").setSettings(
+                SETTING_NUMBER_OF_SHARDS, 1,
+                SETTING_NUMBER_OF_REPLICAS, 0).get();
+        ensureGreen();
+
+        for (int i = 0; i < 15; i++) {
+            index("test", "type1", Integer.toString(i), "text", "abc" + i);
+        }
+        refresh();
+
+        SearchResponse search = client().prepareSearch().setQuery(matchQuery("text", "spellchecker")).get();
+        assertThat("didn't ask for suggestions but got some", search.getSuggest(), nullValue());
+
+        TermSuggestionBuilder termSuggestion = termSuggestion("text")
+                .suggestMode(SuggestMode.ALWAYS) // Always, otherwise the results can vary between requests.
+                .text("abcd")
+                .size(10);
+        Suggest suggest = searchSuggest("test", termSuggestion);
+        assertSuggestion(suggest, 0, "test", 10, "abc0");
+
+        termSuggestion.text("abcd").shardSize(5);
+        suggest = searchSuggest("test", termSuggestion);
+        assertSuggestion(suggest, 0, "test", 5, "abc0");
+    }
+
+    public void testUnmappedField() throws IOException, InterruptedException, ExecutionException {
+        CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder()
+                .put(indexSettings())
+                .put("index.analysis.analyzer.biword.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.biword.filter", "shingler", "lowercase")
+                .put("index.analysis.filter.shingler.type", "shingle")
+                .put("index.analysis.filter.shingler.min_shingle_size", 2)
+                .put("index.analysis.filter.shingler.max_shingle_size", 3));
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
+                .startObject("properties")
+                .startObject("name")
+                    .field("type", "text")
+                    .startObject("fields")
+                        .startObject("shingled")
+                            .field("type", "text")
+                            .field("analyzer", "biword")
+                            .field("search_analyzer", "standard")
+                        .endObject()
+                    .endObject()
+                .endObject()
+                .endObject()
+                .endObject().endObject();
+        assertAcked(builder.addMapping("type1", mapping));
+        ensureGreen();
+
+        indexRandom(true, client().prepareIndex("test", "type1").setSource("name", "I like iced tea"),
+        client().prepareIndex("test", "type1").setSource("name", "I like tea."),
+        client().prepareIndex("test", "type1").setSource("name", "I like ice cream."));
+        refresh();
+
+        PhraseSuggestionBuilder phraseSuggestion = phraseSuggestion("name.shingled")
+                .addCandidateGenerator(candidateGenerator("name").prefixLength(0).minWordLength(0).suggestMode("always").maxEdits(2))
+                .gramSize(3);
+        Suggest searchSuggest = searchSuggest("ice tea", "did_you_mean", phraseSuggestion);
+        assertSuggestion(searchSuggest, 0, 0, "did_you_mean", "iced tea");
+
+        phraseSuggestion = phraseSuggestion("nosuchField")
+                .addCandidateGenerator(candidateGenerator("name").prefixLength(0).minWordLength(0).suggestMode("always").maxEdits(2))
+                .gramSize(3);
+        {
+            SearchRequestBuilder searchBuilder = client().prepareSearch().setSize(0);
+            searchBuilder.suggest(new SuggestBuilder().setGlobalText("tetsting sugestion").addSuggestion("did_you_mean", phraseSuggestion));
+            assertThrows(searchBuilder, SearchPhaseExecutionException.class);
+        }
+        {
+            SearchRequestBuilder searchBuilder = client().prepareSearch().setSize(0);
+            searchBuilder.suggest(new SuggestBuilder().setGlobalText("tetsting sugestion").addSuggestion("did_you_mean", phraseSuggestion));
+            assertThrows(searchBuilder, SearchPhaseExecutionException.class);
+        }
+    }
+
+    public void testSimple() throws Exception {
+        createIndex("test");
+        ensureGreen();
+
+        index("test", "type1", "1", "text", "abcd");
+        index("test", "type1", "2", "text", "aacd");
+        index("test", "type1", "3", "text", "abbd");
+        index("test", "type1", "4", "text", "abcc");
+        refresh();
+
+        SearchResponse search = client().prepareSearch().setQuery(matchQuery("text", "spellcecker")).get();
+        assertThat("didn't ask for suggestions but got some", search.getSuggest(), nullValue());
+
+        TermSuggestionBuilder termSuggest = termSuggestion("text")
+                .suggestMode(SuggestMode.ALWAYS) // Always, otherwise the results can vary between requests.
+                .text("abcd");
+        Suggest suggest = searchSuggest("test", termSuggest);
+        assertSuggestion(suggest, 0, "test", "aacd", "abbd", "abcc");
+        assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
+
+        suggest = searchSuggest("test", termSuggest);
+        assertSuggestion(suggest, 0, "test", "aacd","abbd", "abcc");
+        assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
+    }
+
+    public void testEmpty() throws Exception {
+        createIndex("test");
+        ensureGreen();
+
+        index("test", "type1", "1", "text", "bar");
+        refresh();
+
+        TermSuggestionBuilder termSuggest = termSuggestion("text")
+                .suggestMode(SuggestMode.ALWAYS) // Always, otherwise the results can vary between requests.
+                .text("abcd");
+        Suggest suggest = searchSuggest("test", termSuggest);
+        assertSuggestionSize(suggest, 0, 0, "test");
+        assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
+
+        suggest = searchSuggest("test", termSuggest);
+        assertSuggestionSize(suggest, 0, 0, "test");
+        assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
+    }
+
+    public void testWithMultipleCommands() throws Exception {
+        createIndex("test");
+        ensureGreen();
+
+        index("test", "typ1", "1", "field1", "prefix_abcd", "field2", "prefix_efgh");
+        index("test", "typ1", "2", "field1", "prefix_aacd", "field2", "prefix_eeeh");
+        index("test", "typ1", "3", "field1", "prefix_abbd", "field2", "prefix_efff");
+        index("test", "typ1", "4", "field1", "prefix_abcc", "field2", "prefix_eggg");
+        refresh();
+
+        Map<String, SuggestionBuilder<?>> suggestions = new HashMap<>();
+        suggestions.put("size1", termSuggestion("field1")
+                .size(1).text("prefix_abcd").maxTermFreq(10).prefixLength(1).minDocFreq(0)
+                .suggestMode(SuggestMode.ALWAYS));
+        suggestions.put("field2", termSuggestion("field2")
+                .text("prefix_eeeh prefix_efgh")
+                .maxTermFreq(10).minDocFreq(0).suggestMode(SuggestMode.ALWAYS));
+        suggestions.put("accuracy", termSuggestion("field2")
+                .text("prefix_efgh").accuracy(1f)
+                .maxTermFreq(10).minDocFreq(0).suggestMode(SuggestMode.ALWAYS));
+        Suggest suggest = searchSuggest(null, 0, suggestions);
+        assertSuggestion(suggest, 0, "size1", "prefix_aacd");
+        assertThat(suggest.getSuggestion("field2").getEntries().get(0).getText().string(), equalTo("prefix_eeeh"));
+        assertSuggestion(suggest, 0, "field2", "prefix_efgh");
+        assertThat(suggest.getSuggestion("field2").getEntries().get(1).getText().string(), equalTo("prefix_efgh"));
+        assertSuggestion(suggest, 1, "field2", "prefix_eeeh", "prefix_efff", "prefix_eggg");
+        assertSuggestionSize(suggest, 0, 0, "accuracy");
+    }
+
+    public void testSizeAndSort() throws Exception {
+        createIndex("test");
+        ensureGreen();
+
+        Map<String, Integer> termsAndDocCount = new HashMap<>();
+        termsAndDocCount.put("prefix_aaad", 20);
+        termsAndDocCount.put("prefix_abbb", 18);
+        termsAndDocCount.put("prefix_aaca", 16);
+        termsAndDocCount.put("prefix_abba", 14);
+        termsAndDocCount.put("prefix_accc", 12);
+        termsAndDocCount.put("prefix_addd", 10);
+        termsAndDocCount.put("prefix_abaa", 8);
+        termsAndDocCount.put("prefix_dbca", 6);
+        termsAndDocCount.put("prefix_cbad", 4);
+        termsAndDocCount.put("prefix_aacd", 1);
+        termsAndDocCount.put("prefix_abcc", 1);
+        termsAndDocCount.put("prefix_accd", 1);
+
+        for (Entry<String, Integer> entry : termsAndDocCount.entrySet()) {
+            for (int i = 0; i < entry.getValue(); i++) {
+                index("test", "type1", entry.getKey() + i, "field1", entry.getKey());
+            }
+        }
+        refresh();
+
+        Map<String, SuggestionBuilder<?>> suggestions = new HashMap<>();
+        suggestions.put("size3SortScoreFirst", termSuggestion("field1")
+                .size(3).minDocFreq(0).suggestMode(SuggestMode.ALWAYS));
+        suggestions.put("size10SortScoreFirst", termSuggestion("field1")
+                .size(10).minDocFreq(0).suggestMode(SuggestMode.ALWAYS).shardSize(50));
+        suggestions.put("size3SortScoreFirstMaxEdits1", termSuggestion("field1")
+                .maxEdits(1)
+                .size(10).minDocFreq(0).suggestMode(SuggestMode.ALWAYS));
+        suggestions.put("size10SortFrequencyFirst", termSuggestion("field1")
+                .size(10).sort(SortBy.FREQUENCY).shardSize(1000)
+                .minDocFreq(0).suggestMode(SuggestMode.ALWAYS));
+        Suggest suggest = searchSuggest("prefix_abcd", 0, suggestions);
+
+        // The commented out assertions fail sometimes because suggestions are based off of shard frequencies instead of index frequencies.
+        assertSuggestion(suggest, 0, "size3SortScoreFirst", "prefix_aacd", "prefix_abcc", "prefix_accd");
+        assertSuggestion(suggest, 0, "size10SortScoreFirst", 10, "prefix_aacd", "prefix_abcc", "prefix_accd" /*, "prefix_aaad" */);
+        assertSuggestion(suggest, 0, "size3SortScoreFirstMaxEdits1", "prefix_aacd", "prefix_abcc", "prefix_accd");
+        assertSuggestion(suggest, 0, "size10SortFrequencyFirst", "prefix_aaad", "prefix_abbb", "prefix_aaca", "prefix_abba",
+                "prefix_accc", "prefix_addd", "prefix_abaa", "prefix_dbca", "prefix_cbad", "prefix_aacd");
+
+        // assertThat(suggest.get(3).getSuggestedWords().get("prefix_abcd").get(4).getTerm(), equalTo("prefix_abcc"));
+        // assertThat(suggest.get(3).getSuggestedWords().get("prefix_abcd").get(4).getTerm(), equalTo("prefix_accd"));
+    }
+
+    // see #2817
+    public void testStopwordsOnlyPhraseSuggest() throws IOException {
+        assertAcked(prepareCreate("test").addMapping("typ1", "body", "type=text,analyzer=stopwd").setSettings(
+                Settings.builder()
+                        .put("index.analysis.analyzer.stopwd.tokenizer", "whitespace")
+                        .putArray("index.analysis.analyzer.stopwd.filter", "stop")
+        ));
+        ensureGreen();
+        index("test", "typ1", "1", "body", "this is a test");
+        refresh();
+
+        Suggest searchSuggest = searchSuggest( "a an the", "simple_phrase",
+                phraseSuggestion("body").gramSize(1)
+                        .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always"))
+                        .size(1));
+        assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
+    }
+
+    public void testPrefixLength() throws IOException {
+        CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder()
+                .put(SETTING_NUMBER_OF_SHARDS, 1)
+                .put("index.analysis.analyzer.reverse.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.reverse.filter", "lowercase", "reverse")
+                .put("index.analysis.analyzer.body.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.body.filter", "lowercase")
+                .put("index.analysis.analyzer.bigram.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.bigram.filter", "my_shingle", "lowercase")
+                .put("index.analysis.filter.my_shingle.type", "shingle")
+                .put("index.analysis.filter.my_shingle.output_unigrams", false)
+                .put("index.analysis.filter.my_shingle.min_shingle_size", 2)
+                .put("index.analysis.filter.my_shingle.max_shingle_size", 2));
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
+                .startObject("_all").field("store", true).field("term_vector", "with_positions_offsets").endObject()
+                .startObject("properties")
+                .startObject("body").field("type", "text").field("analyzer", "body").endObject()
+                .startObject("body_reverse").field("type", "text").field("analyzer", "reverse").endObject()
+                .startObject("bigram").field("type", "text").field("analyzer", "bigram").endObject()
+                .endObject()
+                .endObject().endObject();
+        assertAcked(builder.addMapping("type1", mapping));
+        ensureGreen();
+
+        index("test", "type1", "1", "body", "hello world");
+        index("test", "type1", "2", "body", "hello world");
+        index("test", "type1", "3", "body", "hello words");
+        refresh();
+
+        Suggest searchSuggest = searchSuggest( "hello word", "simple_phrase",
+                phraseSuggestion("body")
+                        .addCandidateGenerator(candidateGenerator("body").prefixLength(4).minWordLength(1).suggestMode("always"))
+                        .size(1).confidence(1.0f));
+        assertSuggestion(searchSuggest, 0, "simple_phrase", "hello words");
+
+        searchSuggest = searchSuggest( "hello word", "simple_phrase",
+                phraseSuggestion("body")
+                        .addCandidateGenerator(candidateGenerator("body").prefixLength(2).minWordLength(1).suggestMode("always"))
+                        .size(1).confidence(1.0f));
+        assertSuggestion(searchSuggest, 0, "simple_phrase", "hello world");
+    }
+
+    public void testBasicPhraseSuggest() throws IOException, URISyntaxException {
+        CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder()
+                .put(indexSettings())
+                .put("index.analysis.analyzer.reverse.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.reverse.filter", "lowercase", "reverse")
+                .put("index.analysis.analyzer.body.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.body.filter", "lowercase")
+                .put("index.analysis.analyzer.bigram.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.bigram.filter", "my_shingle", "lowercase")
+                .put("index.analysis.filter.my_shingle.type", "shingle")
+                .put("index.analysis.filter.my_shingle.output_unigrams", false)
+                .put("index.analysis.filter.my_shingle.min_shingle_size", 2)
+                .put("index.analysis.filter.my_shingle.max_shingle_size", 2)
+                .put("index.number_of_shards", 1));
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
+                    .startObject("_all")
+                        .field("store", true)
+                        .field("term_vector", "with_positions_offsets")
+                    .endObject()
+                    .startObject("properties")
+                        .startObject("body").
+                            field("type", "text").
+                            field("analyzer", "body")
+                        .endObject()
+                        .startObject("body_reverse").
+                            field("type", "text").
+                            field("analyzer", "reverse")
+                         .endObject()
+                         .startObject("bigram").
+                             field("type", "text").
+                             field("analyzer", "bigram")
+                         .endObject()
+                     .endObject()
+                .endObject().endObject();
+        assertAcked(builder.addMapping("type1", mapping));
+        ensureGreen();
+
+        String[] strings = new String[]{
+            "Arthur, King of the Britons",
+            "Sir Lancelot the Brave",
+            "Patsy, Arthur's Servant",
+            "Sir Robin the Not-Quite-So-Brave-as-Sir-Lancelot",
+            "Sir Bedevere the Wise",
+            "Sir Galahad the Pure",
+            "Miss Islington, the Witch",
+            "Zoot",
+            "Leader of Robin's Minstrels",
+            "Old Crone",
+            "Frank, the Historian",
+            "Frank's Wife",
+            "Dr. Piglet",
+            "Dr. Winston",
+            "Sir Robin (Stand-in)",
+            "Knight Who Says Ni",
+            "Police sergeant who stops the film",
+        };
+        for (String line : strings) {
+            index("test", "type1", line, "body", line, "body_reverse", line, "bigram", line);
+        }
+        refresh();
+
+        PhraseSuggestionBuilder phraseSuggest = phraseSuggestion("bigram").gramSize(2).analyzer("body")
+                .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always"))
+                .size(1);
+        Suggest searchSuggest = searchSuggest( "Frank's Wise", "simple_phrase", phraseSuggest);
+        assertSuggestion(searchSuggest, 0, "simple_phrase", "frank's wife");
+        assertThat(searchSuggest.getSuggestion("simple_phrase").getEntries().get(0).getText().string(), equalTo("Frank's Wise"));
+
+        phraseSuggest.realWordErrorLikelihood(0.95f);
+        searchSuggest = searchSuggest( "Artur, Kinh of the Britons", "simple_phrase", phraseSuggest);
+        assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
+        // Check the "text" field this one time.
+        assertThat(searchSuggest.getSuggestion("simple_phrase").getEntries().get(0).getText().string(),
+            equalTo("Artur, Kinh of the Britons"));
+
+        // Ask for highlighting
+        phraseSuggest.highlight("<em>", "</em>");
+        searchSuggest = searchSuggest( "Artur, King of the Britns", "simple_phrase", phraseSuggest);
+        assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
+        assertThat(searchSuggest.getSuggestion("simple_phrase").getEntries().get(0).getOptions().get(0).getHighlighted().string(),
+            equalTo("<em>arthur</em> king of the <em>britons</em>"));
+
+        // pass in a correct phrase
+        phraseSuggest.highlight(null, null).confidence(0f).size(1).maxErrors(0.5f);
+        searchSuggest = searchSuggest( "Arthur, King of the Britons", "simple_phrase", phraseSuggest);
+        assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
+
+        // pass in a correct phrase - set confidence to 2
+        phraseSuggest.confidence(2f);
+        searchSuggest = searchSuggest( "Arthur, King of the Britons", "simple_phrase", phraseSuggest);
+        assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
+
+        // pass in a correct phrase - set confidence to 0.99
+        phraseSuggest.confidence(0.99f);
+        searchSuggest = searchSuggest( "Arthur, King of the Britons", "simple_phrase", phraseSuggest);
+        assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
+
+        //test reverse suggestions with pre & post filter
+        phraseSuggest
+            .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always"))
+            .addCandidateGenerator(candidateGenerator("body_reverse").minWordLength(1).suggestMode("always").preFilter("reverse")
+                .postFilter("reverse"));
+        searchSuggest = searchSuggest( "Artur, Ging of the Britons",  "simple_phrase", phraseSuggest);
+        assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
+
+        // set all mass to trigrams (not indexed)
+        phraseSuggest.clearCandidateGenerators()
+            .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always"))
+            .smoothingModel(new LinearInterpolation(1,0,0));
+        searchSuggest = searchSuggest( "Artur, King of the Britns", "simple_phrase", phraseSuggest);
+        assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
+
+        // set all mass to bigrams
+        phraseSuggest.smoothingModel(new LinearInterpolation(0,1,0));
+        searchSuggest =  searchSuggest( "Artur, King of the Britns", "simple_phrase", phraseSuggest);
+        assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
+
+        // distribute mass
+        phraseSuggest.smoothingModel(new LinearInterpolation(0.4,0.4,0.2));
+        searchSuggest = searchSuggest( "Artur, King of the Britns", "simple_phrase", phraseSuggest);
+        assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
+
+        searchSuggest = searchSuggest( "Frank's Wise", "simple_phrase", phraseSuggest);
+        assertSuggestion(searchSuggest, 0, "simple_phrase", "frank's wife");
+
+        // try all smoothing methods
+        phraseSuggest.smoothingModel(new LinearInterpolation(0.4,0.4,0.2));
+        searchSuggest = searchSuggest( "Artur, King of the Britns", "simple_phrase", phraseSuggest);
+        assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
+
+        phraseSuggest.smoothingModel(new Laplace(0.2));
+        searchSuggest = searchSuggest( "Artur, King of the Britns",  "simple_phrase", phraseSuggest);
+        assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
+
+        phraseSuggest.smoothingModel(new StupidBackoff(0.1));
+        searchSuggest = searchSuggest( "Artur, King of the Britns",  "simple_phrase",phraseSuggest);
+        assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
+
+        // check tokenLimit
+        phraseSuggest.smoothingModel(null).tokenLimit(4);
+        searchSuggest = searchSuggest( "Artur, King of the Britns", "simple_phrase", phraseSuggest);
+        assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
+
+        phraseSuggest.tokenLimit(15).smoothingModel(new StupidBackoff(0.1));
+        searchSuggest = searchSuggest( "Sir Bedever the Wife Sir Bedever the Wife Sir Bedever the Wife", "simple_phrase", phraseSuggest);
+        assertSuggestion(searchSuggest, 0, "simple_phrase", "sir bedevere the wise sir bedevere the wise sir bedevere the wise");
+        // Check the name this time because we're repeating it which is funky
+        assertThat(searchSuggest.getSuggestion("simple_phrase").getEntries().get(0).getText().string(),
+            equalTo("Sir Bedever the Wife Sir Bedever the Wife Sir Bedever the Wife"));
+    }
+
+    public void testSizeParam() throws IOException {
+        CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder()
+                .put(SETTING_NUMBER_OF_SHARDS, 1)
+                .put("index.analysis.analyzer.reverse.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.reverse.filter", "lowercase", "reverse")
+                .put("index.analysis.analyzer.body.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.body.filter", "lowercase")
+                .put("index.analysis.analyzer.bigram.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.bigram.filter", "my_shingle", "lowercase")
+                .put("index.analysis.filter.my_shingle.type", "shingle")
+                .put("index.analysis.filter.my_shingle.output_unigrams", false)
+                .put("index.analysis.filter.my_shingle.min_shingle_size", 2)
+                .put("index.analysis.filter.my_shingle.max_shingle_size", 2));
+
+        XContentBuilder mapping = XContentFactory.jsonBuilder()
+                .startObject()
+                    .startObject("type1")
+                        .startObject("_all")
+                            .field("store", true)
+                            .field("term_vector", "with_positions_offsets")
+                        .endObject()
+                        .startObject("properties")
+                            .startObject("body")
+                                .field("type", "text")
+                                .field("analyzer", "body")
+                            .endObject()
+                         .startObject("body_reverse")
+                             .field("type", "text")
+                             .field("analyzer", "reverse")
+                         .endObject()
+                         .startObject("bigram")
+                             .field("type", "text")
+                             .field("analyzer", "bigram")
+                         .endObject()
+                     .endObject()
+                 .endObject()
+             .endObject();
+        assertAcked(builder.addMapping("type1", mapping));
+        ensureGreen();
+
+        String line = "xorr the god jewel";
+        index("test", "type1", "1", "body", line, "body_reverse", line, "bigram", line);
+        line = "I got it this time";
+        index("test", "type1", "2", "body", line, "body_reverse", line, "bigram", line);
+        refresh();
+
+        PhraseSuggestionBuilder phraseSuggestion = phraseSuggestion("bigram")
+                .realWordErrorLikelihood(0.95f)
+                .gramSize(2)
+                .analyzer("body")
+                .addCandidateGenerator(candidateGenerator("body").minWordLength(1).prefixLength(1).suggestMode("always").size(1)
+                    .accuracy(0.1f))
+                .smoothingModel(new StupidBackoff(0.1))
+                .maxErrors(1.0f)
+                .size(5);
+        Suggest searchSuggest = searchSuggest("Xorr the Gut-Jewel", "simple_phrase", phraseSuggestion);
+        assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
+
+        // we allow a size of 2 now on the shard generator level so "god" will be found since it's LD2
+        phraseSuggestion.clearCandidateGenerators()
+                .addCandidateGenerator(candidateGenerator("body").minWordLength(1).prefixLength(1).suggestMode("always").size(2)
+                    .accuracy(0.1f));
+        searchSuggest = searchSuggest( "Xorr the Gut-Jewel", "simple_phrase", phraseSuggestion);
+        assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
+    }
+
+    public void testPhraseBoundaryCases() throws IOException, URISyntaxException {
+        CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder()
+                .put(indexSettings()).put(SETTING_NUMBER_OF_SHARDS, 1) // to get reliable statistics we should put this all into one shard
+                .put("index.analysis.analyzer.body.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.body.filter", "lowercase")
+                .put("index.analysis.analyzer.bigram.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.bigram.filter", "my_shingle", "lowercase")
+                .put("index.analysis.analyzer.ngram.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.ngram.filter", "my_shingle2", "lowercase")
+                .put("index.analysis.analyzer.myDefAnalyzer.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.myDefAnalyzer.filter", "shingle", "lowercase")
+                .put("index.analysis.filter.my_shingle.type", "shingle")
+                .put("index.analysis.filter.my_shingle.output_unigrams", false)
+                .put("index.analysis.filter.my_shingle.min_shingle_size", 2)
+                .put("index.analysis.filter.my_shingle.max_shingle_size", 2)
+                .put("index.analysis.filter.my_shingle2.type", "shingle")
+                .put("index.analysis.filter.my_shingle2.output_unigrams", true)
+                .put("index.analysis.filter.my_shingle2.min_shingle_size", 2)
+                .put("index.analysis.filter.my_shingle2.max_shingle_size", 2));
+
+        XContentBuilder mapping = XContentFactory.jsonBuilder()
+                    .startObject().startObject("type1")
+                    .startObject("_all").field("store", true).field("term_vector", "with_positions_offsets").endObject()
+                .startObject("properties")
+                .startObject("body").field("type", "text").field("analyzer", "body").endObject()
+                .startObject("bigram").field("type", "text").field("analyzer", "bigram").endObject()
+                .startObject("ngram").field("type", "text").field("analyzer", "ngram").endObject()
+                .endObject()
+                .endObject().endObject();
+        assertAcked(builder.addMapping("type1", mapping));
+        ensureGreen();
+
+        String[] strings = new String[]{
+            "Xorr the God-Jewel",
+            "Grog the God-Crusher",
+            "Xorn",
+            "Walter Newell",
+            "Wanda Maximoff",
+            "Captain America",
+            "American Ace",
+            "Wundarr the Aquarian",
+            "Will o' the Wisp",
+            "Xemnu the Titan"
+        };
+        for (String line : strings) {
+            index("test", "type1", line, "body", line, "bigram", line, "ngram", line);
+        }
+        refresh();
+
+        NumShards numShards = getNumShards("test");
+
+        // Lets make sure some things throw exceptions
+        PhraseSuggestionBuilder phraseSuggestion = phraseSuggestion("bigram")
+                .analyzer("body")
+                .addCandidateGenerator(candidateGenerator("does_not_exist").minWordLength(1).suggestMode("always"))
+                .realWordErrorLikelihood(0.95f)
+                .maxErrors(0.5f)
+                .size(1);
+        phraseSuggestion.clearCandidateGenerators().analyzer(null);
+        try {
+            searchSuggest("xor the got-jewel", numShards.numPrimaries, Collections.singletonMap("simple_phrase", phraseSuggestion));
+            fail("analyzer does only produce ngrams");
+        } catch (SearchPhaseExecutionException e) {
+        }
+
+        phraseSuggestion.analyzer("bigram");
+        try {
+            searchSuggest("xor the got-jewel", numShards.numPrimaries, Collections.singletonMap("simple_phrase", phraseSuggestion));
+            fail("analyzer does only produce ngrams");
+        } catch (SearchPhaseExecutionException e) {
+        }
+
+        // Now we'll make sure some things don't
+        phraseSuggestion.forceUnigrams(false);
+        searchSuggest( "xor the got-jewel", 0, Collections.singletonMap("simple_phrase", phraseSuggestion));
+
+        // Field doesn't produce unigrams but the analyzer does
+        phraseSuggestion.forceUnigrams(true).analyzer("ngram");
+        searchSuggest( "xor the got-jewel", 0, Collections.singletonMap("simple_phrase", phraseSuggestion));
+
+        phraseSuggestion = phraseSuggestion("ngram")
+                .analyzer("myDefAnalyzer")
+                .forceUnigrams(true)
+                .realWordErrorLikelihood(0.95f)
+                .maxErrors(0.5f)
+                .size(1)
+                .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always"));
+        Suggest suggest = searchSuggest( "xor the got-jewel", 0, Collections.singletonMap("simple_phrase", phraseSuggestion));
+
+        // "xorr the god jewel" and and "xorn the god jewel" have identical scores (we are only using unigrams to score), so we tie break by
+        // earlier term (xorn):
+        assertSuggestion(suggest, 0, "simple_phrase", "xorn the god jewel");
+
+        phraseSuggestion.analyzer(null);
+        suggest = searchSuggest( "xor the got-jewel", 0, Collections.singletonMap("simple_phrase", phraseSuggestion));
+
+        // In this case xorr has a better score than xorn because we set the field back to the default (my_shingle2) analyzer, so the
+        // probability that the term is not in the dictionary but is NOT a misspelling is relatively high in this case compared to the
+        // others that have no n-gram with the other terms in the phrase :) you can set this realWorldErrorLikelyhood
+        assertSuggestion(suggest, 0, "simple_phrase", "xorr the god jewel");
+    }
+
+    public void testDifferentShardSize() throws Exception {
+        createIndex("test");
+        ensureGreen();
+        indexRandom(true, client().prepareIndex("test", "type1", "1").setSource("field1", "foobar1").setRouting("1"),
+                client().prepareIndex("test", "type1", "2").setSource("field1", "foobar2").setRouting("2"),
+                client().prepareIndex("test", "type1", "3").setSource("field1", "foobar3").setRouting("3"));
+
+        Suggest suggest = searchSuggest( "foobar", "simple",
+                termSuggestion("field1")
+                        .size(10).minDocFreq(0).suggestMode(SuggestMode.ALWAYS));
+        ElasticsearchAssertions.assertSuggestionSize(suggest, 0, 3, "simple");
+    }
+
+    // see #3469
+    public void testShardFailures() throws IOException, InterruptedException {
+        CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder()
+                .put(indexSettings())
+                .put("index.analysis.analyzer.suggest.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.suggest.filter", "standard", "lowercase", "shingler")
+                .put("index.analysis.filter.shingler.type", "shingle")
+                .put("index.analysis.filter.shingler.min_shingle_size", 2)
+                .put("index.analysis.filter.shingler.max_shingle_size", 5)
+                .put("index.analysis.filter.shingler.output_unigrams", true));
+
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type2")
+                .startObject("properties")
+                    .startObject("name")
+                        .field("type", "text")
+                        .field("analyzer", "suggest")
+                    .endObject()
+                .endObject()
+                .endObject().endObject();
+        assertAcked(builder.addMapping("type2", mapping));
+        ensureGreen();
+
+        index("test", "type2", "1", "foo", "bar");
+        index("test", "type2", "2", "foo", "bar");
+        index("test", "type2", "3", "foo", "bar");
+        index("test", "type2", "4", "foo", "bar");
+        index("test", "type2", "5", "foo", "bar");
+        index("test", "type2", "1", "name", "Just testing the suggestions api");
+        index("test", "type2", "2", "name", "An other title about equal length");
+        // Note that the last document has to have about the same length as the other or cutoff rechecking will remove the useful suggestion
+        refresh();
+
+        // When searching on a shard with a non existing mapping, we should fail
+        SearchRequestBuilder request = client().prepareSearch().setSize(0)
+                .suggest(
+                        new SuggestBuilder().setGlobalText("tetsting sugestion").addSuggestion("did_you_mean",
+                                phraseSuggestion("fielddoesnotexist").maxErrors(5.0f)));
+        assertThrows(request, SearchPhaseExecutionException.class);
+
+        // When searching on a shard which does not hold yet any document of an existing type, we should not fail
+        SearchResponse searchResponse = client().prepareSearch().setSize(0)
+                .suggest(
+                        new SuggestBuilder().setGlobalText("tetsting sugestion").addSuggestion("did_you_mean",
+                                phraseSuggestion("name").maxErrors(5.0f)))
+            .get();
+        ElasticsearchAssertions.assertNoFailures(searchResponse);
+        ElasticsearchAssertions.assertSuggestion(searchResponse.getSuggest(), 0, 0, "did_you_mean", "testing suggestions");
+    }
+
+    // see #3469
+    public void testEmptyShards() throws IOException, InterruptedException {
+        XContentBuilder mappingBuilder = XContentFactory.jsonBuilder().
+                startObject().
+                    startObject("type1").
+                        startObject("properties").
+                            startObject("name").
+                                field("type", "text").
+                                field("analyzer", "suggest").
+                            endObject().
+                        endObject().
+                    endObject().
+                endObject();
+        assertAcked(prepareCreate("test").setSettings(Settings.builder()
+                .put(indexSettings())
+                .put("index.analysis.analyzer.suggest.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.suggest.filter", "standard", "lowercase", "shingler")
+                .put("index.analysis.filter.shingler.type", "shingle")
+                .put("index.analysis.filter.shingler.min_shingle_size", 2)
+                .put("index.analysis.filter.shingler.max_shingle_size", 5)
+                .put("index.analysis.filter.shingler.output_unigrams", true)).addMapping("type1", mappingBuilder));
+        ensureGreen();
+
+        index("test", "type2", "1", "foo", "bar");
+        index("test", "type2", "2", "foo", "bar");
+        index("test", "type1", "1", "name", "Just testing the suggestions api");
+        index("test", "type1", "2", "name", "An other title about equal length");
+        refresh();
+
+        SearchResponse searchResponse = client().prepareSearch()
+                .setSize(0)
+                .suggest(
+                        new SuggestBuilder().setGlobalText("tetsting sugestion").addSuggestion("did_you_mean",
+                                phraseSuggestion("name").maxErrors(5.0f)))
+                .get();
+
+        assertNoFailures(searchResponse);
+        assertSuggestion(searchResponse.getSuggest(), 0, 0, "did_you_mean", "testing suggestions");
+    }
+
+    /**
+     * Searching for a rare phrase shouldn't provide any suggestions if confidence &gt; 1.  This was possible before we rechecked the cutoff
+     * score during the reduce phase.  Failures don't occur every time - maybe two out of five tries but we don't repeat it to save time.
+     */
+    public void testSearchForRarePhrase() throws IOException {
+        // If there isn't enough chaf per shard then shards can become unbalanced, making the cutoff recheck this is testing do more harm
+        // then good.
+        int chafPerShard = 100;
+
+        CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder()
+                .put(indexSettings())
+                .put("index.analysis.analyzer.body.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.body.filter", "lowercase", "my_shingle")
+                .put("index.analysis.filter.my_shingle.type", "shingle")
+                .put("index.analysis.filter.my_shingle.output_unigrams", true)
+                .put("index.analysis.filter.my_shingle.min_shingle_size", 2)
+                .put("index.analysis.filter.my_shingle.max_shingle_size", 2));
+
+        XContentBuilder mapping = XContentFactory.jsonBuilder()
+                .startObject()
+                    .startObject("type1")
+                        .startObject("_all")
+                            .field("store", true)
+                            .field("term_vector", "with_positions_offsets")
+                        .endObject()
+                        .startObject("properties")
+                            .startObject("body")
+                                .field("type", "text")
+                                .field("analyzer", "body")
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                .endObject();
+        assertAcked(builder.addMapping("type1", mapping));
+        ensureGreen();
+
+        NumShards test = getNumShards("test");
+
+        List<String> phrases = new ArrayList<>();
+        Collections.addAll(phrases, "nobel prize", "noble gases", "somethingelse prize", "pride and joy", "notes are fun");
+        for (int i = 0; i < 8; i++) {
+            phrases.add("noble somethingelse" + i);
+        }
+        for (int i = 0; i < test.numPrimaries * chafPerShard; i++) {
+            phrases.add("chaff" + i);
+        }
+        for (String phrase: phrases) {
+            index("test", "type1", phrase, "body", phrase);
+        }
+        refresh();
+
+        Suggest searchSuggest = searchSuggest("nobel prize", "simple_phrase", phraseSuggestion("body")
+                .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f))
+                .confidence(2f)
+                .maxErrors(5f)
+                .size(1));
+        assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
+
+        searchSuggest = searchSuggest("noble prize", "simple_phrase", phraseSuggestion("body")
+                .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f))
+                .confidence(2f)
+                .maxErrors(5f)
+                .size(1));
+        assertSuggestion(searchSuggest, 0, 0, "simple_phrase", "nobel prize");
+    }
+
+    public void testSuggestWithManyCandidates() throws InterruptedException, ExecutionException, IOException {
+        CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder()
+                .put(indexSettings())
+                .put(SETTING_NUMBER_OF_SHARDS, 1) // A single shard will help to keep the tests repeatable.
+                .put("index.analysis.analyzer.text.tokenizer", "standard")
+                .putArray("index.analysis.analyzer.text.filter", "lowercase", "my_shingle")
+                .put("index.analysis.filter.my_shingle.type", "shingle")
+                .put("index.analysis.filter.my_shingle.output_unigrams", true)
+                .put("index.analysis.filter.my_shingle.min_shingle_size", 2)
+                .put("index.analysis.filter.my_shingle.max_shingle_size", 3));
+
+        XContentBuilder mapping = XContentFactory.jsonBuilder()
+                .startObject()
+                    .startObject("type1")
+                        .startObject("properties")
+                            .startObject("title")
+                                .field("type", "text")
+                                .field("analyzer", "text")
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                .endObject();
+        assertAcked(builder.addMapping("type1", mapping));
+        ensureGreen();
+
+        List<String> titles = new ArrayList<>();
+
+        // We're going to be searching for:
+        //   united states house of representatives elections in washington 2006
+        // But we need to make sure we generate a ton of suggestions so we add a bunch of candidates.
+        // Many of these candidates are drawn from page names on English Wikipedia.
+
+        // Tons of different options very near the exact query term
+        titles.add("United States House of Representatives Elections in Washington 1789");
+        for (int year = 1790; year < 2014; year+= 2) {
+            titles.add("United States House of Representatives Elections in Washington " + year);
+        }
+        // Six of these are near enough to be viable suggestions, just not the top one
+
+        // But we can't stop there!  Titles that are just a year are pretty common so lets just add one per year
+        // since 0.  Why not?
+        for (int year = 0; year < 2015; year++) {
+            titles.add(Integer.toString(year));
+        }
+        // That ought to provide more less good candidates for the last term
+
+        // Now remove or add plural copies of every term we can
+        titles.add("State");
+        titles.add("Houses of Parliament");
+        titles.add("Representative Government");
+        titles.add("Election");
+
+        // Now some possessive
+        titles.add("Washington's Birthday");
+
+        // And some conjugation
+        titles.add("Unified Modeling Language");
+        titles.add("Unite Against Fascism");
+        titles.add("Stated Income Tax");
+        titles.add("Media organizations housed within colleges");
+
+        // And other stuff
+        titles.add("Untied shoelaces");
+        titles.add("Unit circle");
+        titles.add("Untitled");
+        titles.add("Unicef");
+        titles.add("Unrated");
+        titles.add("UniRed");
+        titles.add("Jalan Uniten–Dengkil"); // Highway in Malaysia
+        titles.add("UNITAS");
+        titles.add("UNITER");
+        titles.add("Un-Led-Ed");
+        titles.add("STATS LLC");
+        titles.add("Staples");
+        titles.add("Skates");
+        titles.add("Statues of the Liberators");
+        titles.add("Staten Island");
+        titles.add("Statens Museum for Kunst");
+        titles.add("Hause"); // The last name or the German word, whichever.
+        titles.add("Hose");
+        titles.add("Hoses");
+        titles.add("Howse Peak");
+        titles.add("The Hoose-Gow");
+        titles.add("Hooser");
+        titles.add("Electron");
+        titles.add("Electors");
+        titles.add("Evictions");
+        titles.add("Coronal mass ejection");
+        titles.add("Wasington"); // A film?
+        titles.add("Warrington"); // A town in England
+        titles.add("Waddington"); // Lots of places have this name
+        titles.add("Watlington"); // Ditto
+        titles.add("Waplington"); // Yup, also a town
+        titles.add("Washing of the Spears"); // Book
+
+        for (char c = 'A'; c <= 'Z'; c++) {
+            // Can't forget lists, glorious lists!
+            titles.add("List of former members of the United States House of Representatives (" + c + ")");
+
+            // Lots of people are named Washington <Middle Initial>. LastName
+            titles.add("Washington " + c + ". Lastname");
+
+            // Lets just add some more to be evil
+            titles.add("United " + c);
+            titles.add("States " + c);
+            titles.add("House " + c);
+            titles.add("Elections " + c);
+            titles.add("2006 " + c);
+            titles.add(c + " United");
+            titles.add(c + " States");
+            titles.add(c + " House");
+            titles.add(c + " Elections");
+            titles.add(c + " 2006");
+        }
+
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+        for (String title: titles) {
+            builders.add(client().prepareIndex("test", "type1").setSource("title", title));
+        }
+
+        indexRandom(true, builders);
+
+        PhraseSuggestionBuilder suggest = phraseSuggestion("title")
+                .addCandidateGenerator(candidateGenerator("title")
+                        .suggestMode("always")
+                        .maxTermFreq(.99f)
+                        .size(1000) // Setting a silly high size helps of generate a larger list of candidates for testing.
+                        .maxInspections(1000) // This too
+                )
+                .confidence(0f)
+                .maxErrors(2f)
+                .shardSize(30000)
+                .size(30000);
+        Suggest searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", suggest);
+        assertSuggestion(searchSuggest, 0, 0, "title", "united states house of representatives elections in washington 2006");
+        assertSuggestionSize(searchSuggest, 0, 25480, "title");  // Just to prove that we've run through a ton of options
+
+        suggest.size(1);
+        long start = System.currentTimeMillis();
+        searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", suggest);
+        long total = System.currentTimeMillis() - start;
+        assertSuggestion(searchSuggest, 0, 0, "title", "united states house of representatives elections in washington 2006");
+        // assertThat(total, lessThan(1000L)); // Takes many seconds without fix - just for debugging
+    }
+
+    protected Suggest searchSuggest(String name, SuggestionBuilder<?> suggestion) {
+        return searchSuggest(null, name, suggestion);
+    }
+
+    protected Suggest searchSuggest(String suggestText, String name, SuggestionBuilder<?> suggestion) {
+        Map<String, SuggestionBuilder<?>> map = new HashMap<>();
+        map.put(name, suggestion);
+        return searchSuggest(suggestText, 0, map);
+    }
+
+    protected Suggest searchSuggest(String suggestText, int expectShardsFailed, Map<String, SuggestionBuilder<?>> suggestions) {
+        SearchRequestBuilder builder = client().prepareSearch().setSize(0);
+        SuggestBuilder suggestBuilder = new SuggestBuilder();
+        if (suggestText != null) {
+            suggestBuilder.setGlobalText(suggestText);
+        }
+        for (Entry<String, SuggestionBuilder<?>> suggestion : suggestions.entrySet()) {
+            suggestBuilder.addSuggestion(suggestion.getKey(), suggestion.getValue());
+        }
+        builder.suggest(suggestBuilder);
+        SearchResponse actionGet = builder.execute().actionGet();
+        assertThat(Arrays.toString(actionGet.getShardFailures()), actionGet.getFailedShards(), equalTo(expectShardsFailed));
+        return actionGet.getSuggest();
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellCheckerTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellCheckerTests.java
index a7d6e8b920a..0640bf0d6f0 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellCheckerTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellCheckerTests.java
@@ -59,7 +59,7 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase {
     private final BytesRef preTag = new BytesRef("<em>");
     private final BytesRef postTag = new BytesRef("</em>");
 
-    public void testMarvelHeros() throws IOException {
+    public void testNgram() throws IOException {
         RAMDirectory dir = new RAMDirectory();
         Map<String, Analyzer> mapping = new HashMap<>();
         mapping.put("body_ngram", new Analyzer() {
@@ -87,9 +87,23 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase {
 
         IndexWriterConfig conf = new IndexWriterConfig(wrapper);
         IndexWriter writer = new IndexWriter(dir, conf);
-        BufferedReader reader = new BufferedReader(new InputStreamReader(NoisyChannelSpellCheckerTests.class.getResourceAsStream("/config/names.txt"), StandardCharsets.UTF_8));
-        String line = null;
-        while ((line = reader.readLine()) != null) {
+        String[] strings = new String[]{
+            "Xorr the God-Jewel",
+            "Grog the God-Crusher",
+            "Xorn",
+            "Walter Newell",
+            "Wanda Maximoff",
+            "Captain America",
+            "American Ace",
+            "USA Hero",
+            "Wundarr the Aquarian",
+            "Will o' the Wisp",
+            "Xemnu the Titan",
+            "Fantastic Four",
+            "Quasar",
+            "Quasar II"
+        };
+        for (String line : strings) {
             Document doc = new Document();
             doc.add(new Field("body", line, TextField.TYPE_NOT_STORED));
             doc.add(new Field("body_ngram", line, TextField.TYPE_NOT_STORED));
@@ -161,7 +175,7 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase {
                 TokenFilter filter = new LowerCaseFilter(t);
                 try {
                     SolrSynonymParser parser = new SolrSynonymParser(true, false, new WhitespaceAnalyzer());
-                    parser.parse(new StringReader("usa => usa, america, american\nursa => usa, america, american"));
+                    parser.parse(new StringReader("usa => usa, america, american"));
                     filter = new SynonymFilter(filter, parser.build(), true);
                 } catch (Exception e) {
                     throw new RuntimeException(e);
@@ -191,7 +205,7 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase {
         assertThat(corrections[0].join(space, preTag, postTag).utf8ToString(), equalTo("captain <em>america</em>"));
     }
 
-    public void testMarvelHerosMultiGenerator() throws IOException {
+    public void testMultiGenerator() throws IOException {
         RAMDirectory dir = new RAMDirectory();
         Map<String, Analyzer> mapping = new HashMap<>();
         mapping.put("body_ngram", new Analyzer() {
@@ -228,9 +242,22 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase {
 
         IndexWriterConfig conf = new IndexWriterConfig(wrapper);
         IndexWriter writer = new IndexWriter(dir, conf);
-        BufferedReader reader = new BufferedReader(new InputStreamReader(NoisyChannelSpellCheckerTests.class.getResourceAsStream("/config/names.txt"), StandardCharsets.UTF_8));
-        String line = null;
-        while ((line = reader.readLine()) != null) {
+        String[] strings = new String[]{
+            "Xorr the God-Jewel",
+            "Grog the God-Crusher",
+            "Xorn",
+            "Walter Newell",
+            "Wanda Maximoff",
+            "Captain America",
+            "American Ace",
+            "Wundarr the Aquarian",
+            "Will o' the Wisp",
+            "Xemnu the Titan",
+            "Fantastic Four",
+            "Quasar",
+            "Quasar II"
+        };
+        for (String line : strings) {
             Document doc = new Document();
             doc.add(new Field("body", line, TextField.TYPE_NOT_STORED));
             doc.add(new Field("body_reverse", line, TextField.TYPE_NOT_STORED));
@@ -284,7 +311,7 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase {
         assertThat(corrections[0].join(new BytesRef(" ")).utf8ToString(), equalTo("quasar ii"));
     }
 
-    public void testMarvelHerosTrigram() throws IOException {
+    public void testTrigram() throws IOException {
         RAMDirectory dir = new RAMDirectory();
         Map<String, Analyzer> mapping = new HashMap<>();
         mapping.put("body_ngram", new Analyzer() {
@@ -312,9 +339,23 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase {
 
         IndexWriterConfig conf = new IndexWriterConfig(wrapper);
         IndexWriter writer = new IndexWriter(dir, conf);
-        BufferedReader reader = new BufferedReader(new InputStreamReader(NoisyChannelSpellCheckerTests.class.getResourceAsStream("/config/names.txt"), StandardCharsets.UTF_8));
-        String line = null;
-        while ((line = reader.readLine()) != null) {
+        String[] strings = new String[]{
+            "Xorr the God-Jewel",
+            "Grog the God-Crusher",
+            "Xorn",
+            "Walter Newell",
+            "Wanda Maximoff",
+            "Captain America",
+            "American Ace",
+            "USA Hero",
+            "Wundarr the Aquarian",
+            "Will o' the Wisp",
+            "Xemnu the Titan",
+            "Fantastic Four",
+            "Quasar",
+            "Quasar II"
+        };
+        for (String line : strings) {
             Document doc = new Document();
             doc.add(new Field("body", line, TextField.TYPE_NOT_STORED));
             doc.add(new Field("body_ngram", line, TextField.TYPE_NOT_STORED));
@@ -370,7 +411,7 @@ public class NoisyChannelSpellCheckerTests extends ESTestCase {
                 TokenFilter filter = new LowerCaseFilter(t);
                 try {
                     SolrSynonymParser parser = new SolrSynonymParser(true, false, new WhitespaceAnalyzer());
-                    parser.parse(new StringReader("usa => usa, america, american\nursa => usa, america, american"));
+                    parser.parse(new StringReader("usa => usa, america, american"));
                     filter = new SynonymFilter(filter, parser.build(), true);
                 } catch (Exception e) {
                     throw new RuntimeException(e);
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java
index ccb8614d87f..bac598ba282 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/SuggestSearchTests.java
@@ -20,27 +20,24 @@
 package org.elasticsearch.messy.tests;
 
 
-import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
-import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
-import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
-import static org.elasticsearch.search.suggest.SuggestBuilders.phraseSuggestion;
-import static org.elasticsearch.search.suggest.SuggestBuilders.termSuggestion;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestion;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionPhraseCollateMatchExists;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionSize;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
-import static org.hamcrest.Matchers.anyOf;
-import static org.hamcrest.Matchers.endsWith;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.nullValue;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.mustache.MustachePlugin;
+import org.elasticsearch.search.suggest.Suggest;
+import org.elasticsearch.search.suggest.SuggestBuilder;
+import org.elasticsearch.search.suggest.SuggestionBuilder;
+import org.elasticsearch.search.suggest.phrase.DirectCandidateGeneratorBuilder;
+import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
+import org.elasticsearch.test.ESIntegTestCase;
 
 import java.io.IOException;
-import java.net.URISyntaxException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -51,32 +48,12 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.concurrent.ExecutionException;
 
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
-import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.action.search.ReduceSearchPhaseException;
-import org.elasticsearch.action.search.SearchPhaseExecutionException;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.common.io.PathUtils;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.script.mustache.MustachePlugin;
-import org.elasticsearch.search.suggest.SortBy;
-import org.elasticsearch.search.suggest.Suggest;
-import org.elasticsearch.search.suggest.SuggestBuilder;
-import org.elasticsearch.search.suggest.SuggestionBuilder;
-import org.elasticsearch.search.suggest.phrase.DirectCandidateGeneratorBuilder;
-import org.elasticsearch.search.suggest.phrase.Laplace;
-import org.elasticsearch.search.suggest.phrase.LinearInterpolation;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
-import org.elasticsearch.search.suggest.phrase.StupidBackoff;
-import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
-import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SuggestMode;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
+import static org.elasticsearch.search.suggest.SuggestBuilders.phraseSuggestion;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionPhraseCollateMatchExists;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestionSize;
+import static org.hamcrest.Matchers.equalTo;
 
 /**
  * Integration tests for term and phrase suggestions.  Many of these tests many requests that vary only slightly from one another.  Where
@@ -90,1015 +67,6 @@ public class SuggestSearchTests extends ESIntegTestCase {
         return Collections.singleton(MustachePlugin.class);
     }
 
-    // see #3196
-    public void testSuggestAcrossMultipleIndices() throws IOException {
-        createIndex("test");
-        ensureGreen();
-
-        index("test", "type1", "1", "text", "abcd");
-        index("test", "type1", "2", "text", "aacd");
-        index("test", "type1", "3", "text", "abbd");
-        index("test", "type1", "4", "text", "abcc");
-        refresh();
-
-        TermSuggestionBuilder termSuggest = termSuggestion("text")
-                .suggestMode(TermSuggestionBuilder.SuggestMode.ALWAYS) // Always, otherwise the results can vary between requests.
-                .text("abcd");
-        logger.info("--> run suggestions with one index");
-        searchSuggest("test", termSuggest);
-        createIndex("test_1");
-        ensureGreen();
-
-        index("test_1", "type1", "1", "text", "ab cd");
-        index("test_1", "type1", "2", "text", "aa cd");
-        index("test_1", "type1", "3", "text", "ab bd");
-        index("test_1", "type1", "4", "text", "ab cc");
-        refresh();
-        termSuggest = termSuggestion("text")
-                .suggestMode(SuggestMode.ALWAYS) // Always, otherwise the results can vary between requests.
-                .text("ab cd")
-                .minWordLength(1);
-        logger.info("--> run suggestions with two indices");
-        searchSuggest("test", termSuggest);
-
-
-        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
-                .startObject("properties")
-                .startObject("text").field("type", "text").field("analyzer", "keyword").endObject()
-                .endObject()
-                .endObject().endObject();
-        assertAcked(prepareCreate("test_2").addMapping("type1", mapping));
-        ensureGreen();
-
-        index("test_2", "type1", "1", "text", "ab cd");
-        index("test_2", "type1", "2", "text", "aa cd");
-        index("test_2", "type1", "3", "text", "ab bd");
-        index("test_2", "type1", "4", "text", "ab cc");
-        index("test_2", "type1", "1", "text", "abcd");
-        index("test_2", "type1", "2", "text", "aacd");
-        index("test_2", "type1", "3", "text", "abbd");
-        index("test_2", "type1", "4", "text", "abcc");
-        refresh();
-
-        termSuggest = termSuggestion("text")
-                .suggestMode(SuggestMode.ALWAYS) // Always, otherwise the results can vary between requests.
-                .text("ab cd")
-                .minWordLength(1);
-        logger.info("--> run suggestions with three indices");
-        try {
-            searchSuggest("test", termSuggest);
-            fail(" can not suggest across multiple indices with different analysis chains");
-        } catch (ReduceSearchPhaseException ex) {
-            assertThat(ex.getCause(), instanceOf(IllegalStateException.class));
-            assertThat(ex.getCause().getMessage(),
-                    anyOf(endsWith("Suggest entries have different sizes actual [1] expected [2]"),
-                            endsWith("Suggest entries have different sizes actual [2] expected [1]")));
-        } catch (IllegalStateException ex) {
-            assertThat(ex.getMessage(), anyOf(endsWith("Suggest entries have different sizes actual [1] expected [2]"),
-                    endsWith("Suggest entries have different sizes actual [2] expected [1]")));
-        }
-
-
-        termSuggest = termSuggestion("text")
-                .suggestMode(SuggestMode.ALWAYS) // Always, otherwise the results can vary between requests.
-                .text("ABCD")
-                .minWordLength(1);
-        logger.info("--> run suggestions with four indices");
-        try {
-            searchSuggest("test", termSuggest);
-            fail(" can not suggest across multiple indices with different analysis chains");
-        } catch (ReduceSearchPhaseException ex) {
-            assertThat(ex.getCause(), instanceOf(IllegalStateException.class));
-            assertThat(ex.getCause().getMessage(), anyOf(endsWith("Suggest entries have different text actual [ABCD] expected [abcd]"),
-                    endsWith("Suggest entries have different text actual [abcd] expected [ABCD]")));
-        } catch (IllegalStateException ex) {
-            assertThat(ex.getMessage(), anyOf(endsWith("Suggest entries have different text actual [ABCD] expected [abcd]"),
-                    endsWith("Suggest entries have different text actual [abcd] expected [ABCD]")));
-        }
-    }
-
-    // see #3037
-    public void testSuggestModes() throws IOException {
-        CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder()
-                .put(SETTING_NUMBER_OF_SHARDS, 1)
-                .put(SETTING_NUMBER_OF_REPLICAS, 0)
-                .put("index.analysis.analyzer.biword.tokenizer", "standard")
-                .putArray("index.analysis.analyzer.biword.filter", "shingler", "lowercase")
-                .put("index.analysis.filter.shingler.type", "shingle")
-                .put("index.analysis.filter.shingler.min_shingle_size", 2)
-                .put("index.analysis.filter.shingler.max_shingle_size", 3));
-
-        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
-                .startObject("properties")
-                .startObject("name")
-                    .field("type", "text")
-                    .startObject("fields")
-                        .startObject("shingled")
-                            .field("type", "text")
-                            .field("analyzer", "biword")
-                            .field("search_analyzer", "standard")
-                        .endObject()
-                    .endObject()
-                .endObject()
-                .endObject()
-                .endObject().endObject();
-        assertAcked(builder.addMapping("type1", mapping));
-        ensureGreen();
-
-
-        index("test", "type1", "1", "name", "I like iced tea");
-        index("test", "type1", "2", "name", "I like tea.");
-        index("test", "type1", "3", "name", "I like ice cream.");
-        refresh();
-
-        DirectCandidateGeneratorBuilder generator = candidateGenerator("name").prefixLength(0).minWordLength(0).suggestMode("always").maxEdits(2);
-        PhraseSuggestionBuilder phraseSuggestion = phraseSuggestion("name.shingled")
-                .addCandidateGenerator(generator)
-                .gramSize(3);
-        Suggest searchSuggest = searchSuggest("ice tea", "did_you_mean", phraseSuggestion);
-        assertSuggestion(searchSuggest, 0, "did_you_mean", "iced tea");
-
-        generator.suggestMode(null);
-        searchSuggest = searchSuggest( "ice tea", "did_you_mean", phraseSuggestion);
-        assertSuggestionSize(searchSuggest, 0, 0, "did_you_mean");
-    }
-
-    /**
-     * Creates a new {@link DirectCandidateGeneratorBuilder}
-     *
-     * @param field
-     *            the field this candidate generator operates on.
-     */
-    private DirectCandidateGeneratorBuilder candidateGenerator(String field) {
-        return new DirectCandidateGeneratorBuilder(field);
-    }
-
-    // see #2729
-    public void testSizeOneShard() throws Exception {
-        prepareCreate("test").setSettings(
-                SETTING_NUMBER_OF_SHARDS, 1,
-                SETTING_NUMBER_OF_REPLICAS, 0).get();
-        ensureGreen();
-
-        for (int i = 0; i < 15; i++) {
-            index("test", "type1", Integer.toString(i), "text", "abc" + i);
-        }
-        refresh();
-
-        SearchResponse search = client().prepareSearch().setQuery(matchQuery("text", "spellchecker")).get();
-        assertThat("didn't ask for suggestions but got some", search.getSuggest(), nullValue());
-
-        TermSuggestionBuilder termSuggestion = termSuggestion("text")
-                .suggestMode(SuggestMode.ALWAYS) // Always, otherwise the results can vary between requests.
-                .text("abcd")
-                .size(10);
-        Suggest suggest = searchSuggest("test", termSuggestion);
-        assertSuggestion(suggest, 0, "test", 10, "abc0");
-
-        termSuggestion.text("abcd").shardSize(5);
-        suggest = searchSuggest("test", termSuggestion);
-        assertSuggestion(suggest, 0, "test", 5, "abc0");
-    }
-
-    public void testUnmappedField() throws IOException, InterruptedException, ExecutionException {
-        CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder()
-                .put(indexSettings())
-                .put("index.analysis.analyzer.biword.tokenizer", "standard")
-                .putArray("index.analysis.analyzer.biword.filter", "shingler", "lowercase")
-                .put("index.analysis.filter.shingler.type", "shingle")
-                .put("index.analysis.filter.shingler.min_shingle_size", 2)
-                .put("index.analysis.filter.shingler.max_shingle_size", 3));
-        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
-                .startObject("properties")
-                .startObject("name")
-                    .field("type", "text")
-                    .startObject("fields")
-                        .startObject("shingled")
-                            .field("type", "text")
-                            .field("analyzer", "biword")
-                            .field("search_analyzer", "standard")
-                        .endObject()
-                    .endObject()
-                .endObject()
-                .endObject()
-                .endObject().endObject();
-        assertAcked(builder.addMapping("type1", mapping));
-        ensureGreen();
-
-        indexRandom(true, client().prepareIndex("test", "type1").setSource("name", "I like iced tea"),
-        client().prepareIndex("test", "type1").setSource("name", "I like tea."),
-        client().prepareIndex("test", "type1").setSource("name", "I like ice cream."));
-        refresh();
-
-        PhraseSuggestionBuilder phraseSuggestion = phraseSuggestion("name.shingled")
-                .addCandidateGenerator(candidateGenerator("name").prefixLength(0).minWordLength(0).suggestMode("always").maxEdits(2))
-                .gramSize(3);
-        Suggest searchSuggest = searchSuggest("ice tea", "did_you_mean", phraseSuggestion);
-        assertSuggestion(searchSuggest, 0, 0, "did_you_mean", "iced tea");
-
-        phraseSuggestion = phraseSuggestion("nosuchField")
-                .addCandidateGenerator(candidateGenerator("name").prefixLength(0).minWordLength(0).suggestMode("always").maxEdits(2))
-                .gramSize(3);
-        {
-            SearchRequestBuilder searchBuilder = client().prepareSearch().setSize(0);
-            searchBuilder.suggest(new SuggestBuilder().setGlobalText("tetsting sugestion").addSuggestion("did_you_mean", phraseSuggestion));
-            assertThrows(searchBuilder, SearchPhaseExecutionException.class);
-        }
-        {
-            SearchRequestBuilder searchBuilder = client().prepareSearch().setSize(0);
-            searchBuilder.suggest(new SuggestBuilder().setGlobalText("tetsting sugestion").addSuggestion("did_you_mean", phraseSuggestion));
-            assertThrows(searchBuilder, SearchPhaseExecutionException.class);
-        }
-    }
-
-    public void testSimple() throws Exception {
-        createIndex("test");
-        ensureGreen();
-
-        index("test", "type1", "1", "text", "abcd");
-        index("test", "type1", "2", "text", "aacd");
-        index("test", "type1", "3", "text", "abbd");
-        index("test", "type1", "4", "text", "abcc");
-        refresh();
-
-        SearchResponse search = client().prepareSearch().setQuery(matchQuery("text", "spellcecker")).get();
-        assertThat("didn't ask for suggestions but got some", search.getSuggest(), nullValue());
-
-        TermSuggestionBuilder termSuggest = termSuggestion("text")
-                .suggestMode(SuggestMode.ALWAYS) // Always, otherwise the results can vary between requests.
-                .text("abcd");
-        Suggest suggest = searchSuggest("test", termSuggest);
-        assertSuggestion(suggest, 0, "test", "aacd", "abbd", "abcc");
-        assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
-
-        suggest = searchSuggest("test", termSuggest);
-        assertSuggestion(suggest, 0, "test", "aacd","abbd", "abcc");
-        assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
-    }
-
-    public void testEmpty() throws Exception {
-        createIndex("test");
-        ensureGreen();
-
-        index("test", "type1", "1", "text", "bar");
-        refresh();
-
-        TermSuggestionBuilder termSuggest = termSuggestion("text")
-                .suggestMode(SuggestMode.ALWAYS) // Always, otherwise the results can vary between requests.
-                .text("abcd");
-        Suggest suggest = searchSuggest("test", termSuggest);
-        assertSuggestionSize(suggest, 0, 0, "test");
-        assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
-
-        suggest = searchSuggest("test", termSuggest);
-        assertSuggestionSize(suggest, 0, 0, "test");
-        assertThat(suggest.getSuggestion("test").getEntries().get(0).getText().string(), equalTo("abcd"));
-    }
-
-    public void testWithMultipleCommands() throws Exception {
-        createIndex("test");
-        ensureGreen();
-
-        index("test", "typ1", "1", "field1", "prefix_abcd", "field2", "prefix_efgh");
-        index("test", "typ1", "2", "field1", "prefix_aacd", "field2", "prefix_eeeh");
-        index("test", "typ1", "3", "field1", "prefix_abbd", "field2", "prefix_efff");
-        index("test", "typ1", "4", "field1", "prefix_abcc", "field2", "prefix_eggg");
-        refresh();
-
-        Map<String, SuggestionBuilder<?>> suggestions = new HashMap<>();
-        suggestions.put("size1", termSuggestion("field1")
-                .size(1).text("prefix_abcd").maxTermFreq(10).prefixLength(1).minDocFreq(0)
-                .suggestMode(SuggestMode.ALWAYS));
-        suggestions.put("field2", termSuggestion("field2")
-                .text("prefix_eeeh prefix_efgh")
-                .maxTermFreq(10).minDocFreq(0).suggestMode(SuggestMode.ALWAYS));
-        suggestions.put("accuracy", termSuggestion("field2")
-                .text("prefix_efgh").accuracy(1f)
-                .maxTermFreq(10).minDocFreq(0).suggestMode(SuggestMode.ALWAYS));
-        Suggest suggest = searchSuggest(null, 0, suggestions);
-        assertSuggestion(suggest, 0, "size1", "prefix_aacd");
-        assertThat(suggest.getSuggestion("field2").getEntries().get(0).getText().string(), equalTo("prefix_eeeh"));
-        assertSuggestion(suggest, 0, "field2", "prefix_efgh");
-        assertThat(suggest.getSuggestion("field2").getEntries().get(1).getText().string(), equalTo("prefix_efgh"));
-        assertSuggestion(suggest, 1, "field2", "prefix_eeeh", "prefix_efff", "prefix_eggg");
-        assertSuggestionSize(suggest, 0, 0, "accuracy");
-    }
-
-    public void testSizeAndSort() throws Exception {
-        createIndex("test");
-        ensureGreen();
-
-        Map<String, Integer> termsAndDocCount = new HashMap<>();
-        termsAndDocCount.put("prefix_aaad", 20);
-        termsAndDocCount.put("prefix_abbb", 18);
-        termsAndDocCount.put("prefix_aaca", 16);
-        termsAndDocCount.put("prefix_abba", 14);
-        termsAndDocCount.put("prefix_accc", 12);
-        termsAndDocCount.put("prefix_addd", 10);
-        termsAndDocCount.put("prefix_abaa", 8);
-        termsAndDocCount.put("prefix_dbca", 6);
-        termsAndDocCount.put("prefix_cbad", 4);
-        termsAndDocCount.put("prefix_aacd", 1);
-        termsAndDocCount.put("prefix_abcc", 1);
-        termsAndDocCount.put("prefix_accd", 1);
-
-        for (Map.Entry<String, Integer> entry : termsAndDocCount.entrySet()) {
-            for (int i = 0; i < entry.getValue(); i++) {
-                index("test", "type1", entry.getKey() + i, "field1", entry.getKey());
-            }
-        }
-        refresh();
-
-        Map<String, SuggestionBuilder<?>> suggestions = new HashMap<>();
-        suggestions.put("size3SortScoreFirst", termSuggestion("field1")
-                .size(3).minDocFreq(0).suggestMode(SuggestMode.ALWAYS));
-        suggestions.put("size10SortScoreFirst", termSuggestion("field1")
-                .size(10).minDocFreq(0).suggestMode(SuggestMode.ALWAYS).shardSize(50));
-        suggestions.put("size3SortScoreFirstMaxEdits1", termSuggestion("field1")
-                .maxEdits(1)
-                .size(10).minDocFreq(0).suggestMode(SuggestMode.ALWAYS));
-        suggestions.put("size10SortFrequencyFirst", termSuggestion("field1")
-                .size(10).sort(SortBy.FREQUENCY).shardSize(1000)
-                .minDocFreq(0).suggestMode(SuggestMode.ALWAYS));
-        Suggest suggest = searchSuggest("prefix_abcd", 0, suggestions);
-
-        // The commented out assertions fail sometimes because suggestions are based off of shard frequencies instead of index frequencies.
-        assertSuggestion(suggest, 0, "size3SortScoreFirst", "prefix_aacd", "prefix_abcc", "prefix_accd");
-        assertSuggestion(suggest, 0, "size10SortScoreFirst", 10, "prefix_aacd", "prefix_abcc", "prefix_accd" /*, "prefix_aaad" */);
-        assertSuggestion(suggest, 0, "size3SortScoreFirstMaxEdits1", "prefix_aacd", "prefix_abcc", "prefix_accd");
-        assertSuggestion(suggest, 0, "size10SortFrequencyFirst", "prefix_aaad", "prefix_abbb", "prefix_aaca", "prefix_abba",
-                "prefix_accc", "prefix_addd", "prefix_abaa", "prefix_dbca", "prefix_cbad", "prefix_aacd");
-
-        // assertThat(suggest.get(3).getSuggestedWords().get("prefix_abcd").get(4).getTerm(), equalTo("prefix_abcc"));
-        // assertThat(suggest.get(3).getSuggestedWords().get("prefix_abcd").get(4).getTerm(), equalTo("prefix_accd"));
-    }
-
-    // see #2817
-    public void testStopwordsOnlyPhraseSuggest() throws IOException {
-        assertAcked(prepareCreate("test").addMapping("typ1", "body", "type=text,analyzer=stopwd").setSettings(
-                Settings.builder()
-                        .put("index.analysis.analyzer.stopwd.tokenizer", "whitespace")
-                        .putArray("index.analysis.analyzer.stopwd.filter", "stop")
-        ));
-        ensureGreen();
-        index("test", "typ1", "1", "body", "this is a test");
-        refresh();
-
-        Suggest searchSuggest = searchSuggest( "a an the", "simple_phrase",
-                phraseSuggestion("body").gramSize(1)
-                        .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always"))
-                        .size(1));
-        assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
-    }
-
-    public void testPrefixLength() throws IOException {
-        CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder()
-                .put(SETTING_NUMBER_OF_SHARDS, 1)
-                .put("index.analysis.analyzer.reverse.tokenizer", "standard")
-                .putArray("index.analysis.analyzer.reverse.filter", "lowercase", "reverse")
-                .put("index.analysis.analyzer.body.tokenizer", "standard")
-                .putArray("index.analysis.analyzer.body.filter", "lowercase")
-                .put("index.analysis.analyzer.bigram.tokenizer", "standard")
-                .putArray("index.analysis.analyzer.bigram.filter", "my_shingle", "lowercase")
-                .put("index.analysis.filter.my_shingle.type", "shingle")
-                .put("index.analysis.filter.my_shingle.output_unigrams", false)
-                .put("index.analysis.filter.my_shingle.min_shingle_size", 2)
-                .put("index.analysis.filter.my_shingle.max_shingle_size", 2));
-        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
-                .startObject("_all").field("store", true).field("term_vector", "with_positions_offsets").endObject()
-                .startObject("properties")
-                .startObject("body").field("type", "text").field("analyzer", "body").endObject()
-                .startObject("body_reverse").field("type", "text").field("analyzer", "reverse").endObject()
-                .startObject("bigram").field("type", "text").field("analyzer", "bigram").endObject()
-                .endObject()
-                .endObject().endObject();
-        assertAcked(builder.addMapping("type1", mapping));
-        ensureGreen();
-
-        index("test", "type1", "1", "body", "hello world");
-        index("test", "type1", "2", "body", "hello world");
-        index("test", "type1", "3", "body", "hello words");
-        refresh();
-
-        Suggest searchSuggest = searchSuggest( "hello word", "simple_phrase",
-                phraseSuggestion("body")
-                        .addCandidateGenerator(candidateGenerator("body").prefixLength(4).minWordLength(1).suggestMode("always"))
-                        .size(1).confidence(1.0f));
-        assertSuggestion(searchSuggest, 0, "simple_phrase", "hello words");
-
-        searchSuggest = searchSuggest( "hello word", "simple_phrase",
-                phraseSuggestion("body")
-                        .addCandidateGenerator(candidateGenerator("body").prefixLength(2).minWordLength(1).suggestMode("always"))
-                        .size(1).confidence(1.0f));
-        assertSuggestion(searchSuggest, 0, "simple_phrase", "hello world");
-    }
-
-    @Nightly
-    public void testMarvelHerosPhraseSuggest() throws IOException, URISyntaxException {
-        CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder()
-                .put(indexSettings())
-                .put("index.analysis.analyzer.reverse.tokenizer", "standard")
-                .putArray("index.analysis.analyzer.reverse.filter", "lowercase", "reverse")
-                .put("index.analysis.analyzer.body.tokenizer", "standard")
-                .putArray("index.analysis.analyzer.body.filter", "lowercase")
-                .put("index.analysis.analyzer.bigram.tokenizer", "standard")
-                .putArray("index.analysis.analyzer.bigram.filter", "my_shingle", "lowercase")
-                .put("index.analysis.filter.my_shingle.type", "shingle")
-                .put("index.analysis.filter.my_shingle.output_unigrams", false)
-                .put("index.analysis.filter.my_shingle.min_shingle_size", 2)
-                .put("index.analysis.filter.my_shingle.max_shingle_size", 2));
-        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
-                    .startObject("_all")
-                        .field("store", true)
-                        .field("term_vector", "with_positions_offsets")
-                    .endObject()
-                    .startObject("properties")
-                        .startObject("body").
-                            field("type", "text").
-                            field("analyzer", "body")
-                        .endObject()
-                        .startObject("body_reverse").
-                            field("type", "text").
-                            field("analyzer", "reverse")
-                         .endObject()
-                         .startObject("bigram").
-                             field("type", "text").
-                             field("analyzer", "bigram")
-                         .endObject()
-                     .endObject()
-                .endObject().endObject();
-        assertAcked(builder.addMapping("type1", mapping));
-        ensureGreen();
-
-        for (String line : readMarvelHeroNames()) {
-            index("test", "type1", line, "body", line, "body_reverse", line, "bigram", line);
-        }
-        refresh();
-
-        PhraseSuggestionBuilder phraseSuggest = phraseSuggestion("bigram").gramSize(2).analyzer("body")
-                .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always"))
-                .size(1);
-        Suggest searchSuggest = searchSuggest( "american ame", "simple_phrase", phraseSuggest);
-        assertSuggestion(searchSuggest, 0, "simple_phrase", "american ace");
-        assertThat(searchSuggest.getSuggestion("simple_phrase").getEntries().get(0).getText().string(), equalTo("american ame"));
-
-        phraseSuggest.realWordErrorLikelihood(0.95f);
-        searchSuggest = searchSuggest( "Xor the Got-Jewel", "simple_phrase", phraseSuggest);
-        assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
-        // Check the "text" field this one time.
-        assertThat(searchSuggest.getSuggestion("simple_phrase").getEntries().get(0).getText().string(), equalTo("Xor the Got-Jewel"));
-
-        // Ask for highlighting
-        phraseSuggest.highlight("<em>", "</em>");
-        searchSuggest = searchSuggest( "Xor the Got-Jewel", "simple_phrase", phraseSuggest);
-        assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
-        assertThat(searchSuggest.getSuggestion("simple_phrase").getEntries().get(0).getOptions().get(0).getHighlighted().string(), equalTo("<em>xorr</em> the <em>god</em> jewel"));
-
-        // pass in a correct phrase
-        phraseSuggest.highlight(null, null).confidence(0f).size(1).maxErrors(0.5f);
-        searchSuggest = searchSuggest( "Xorr the God-Jewel", "simple_phrase", phraseSuggest);
-        assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
-
-        // pass in a correct phrase - set confidence to 2
-        phraseSuggest.confidence(2f);
-        searchSuggest = searchSuggest( "Xorr the God-Jewel", "simple_phrase", phraseSuggest);
-        assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
-
-        // pass in a correct phrase - set confidence to 0.99
-        phraseSuggest.confidence(0.99f);
-        searchSuggest = searchSuggest( "Xorr the God-Jewel", "simple_phrase", phraseSuggest);
-        assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
-
-        //test reverse suggestions with pre & post filter
-        phraseSuggest
-            .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always"))
-            .addCandidateGenerator(candidateGenerator("body_reverse").minWordLength(1).suggestMode("always").preFilter("reverse").postFilter("reverse"));
-        searchSuggest = searchSuggest( "xor the yod-Jewel",  "simple_phrase", phraseSuggest);
-        assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
-
-        // set all mass to trigrams (not indexed)
-        phraseSuggest.clearCandidateGenerators()
-            .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always"))
-            .smoothingModel(new LinearInterpolation(1,0,0));
-        searchSuggest = searchSuggest( "Xor the Got-Jewel", "simple_phrase", phraseSuggest);
-        assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
-
-        // set all mass to bigrams
-        phraseSuggest.smoothingModel(new LinearInterpolation(0,1,0));
-        searchSuggest =  searchSuggest( "Xor the Got-Jewel", "simple_phrase", phraseSuggest);
-        assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
-
-        // distribute mass
-        phraseSuggest.smoothingModel(new LinearInterpolation(0.4,0.4,0.2));
-        searchSuggest = searchSuggest( "Xor the Got-Jewel", "simple_phrase", phraseSuggest);
-        assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
-
-        searchSuggest = searchSuggest( "american ame", "simple_phrase", phraseSuggest);
-        assertSuggestion(searchSuggest, 0, "simple_phrase", "american ace");
-
-        // try all smoothing methods
-        phraseSuggest.smoothingModel(new LinearInterpolation(0.4,0.4,0.2));
-        searchSuggest = searchSuggest( "Xor the Got-Jewel", "simple_phrase", phraseSuggest);
-        assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
-
-        phraseSuggest.smoothingModel(new Laplace(0.2));
-        searchSuggest = searchSuggest( "Xor the Got-Jewel",  "simple_phrase", phraseSuggest);
-        assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
-
-        phraseSuggest.smoothingModel(new StupidBackoff(0.1));
-        searchSuggest = searchSuggest( "Xor the Got-Jewel",  "simple_phrase",phraseSuggest);
-        assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
-
-        // check tokenLimit
-        phraseSuggest.smoothingModel(null).tokenLimit(4);
-        searchSuggest = searchSuggest( "Xor the Got-Jewel", "simple_phrase", phraseSuggest);
-        assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
-
-        phraseSuggest.tokenLimit(15).smoothingModel(new StupidBackoff(0.1));
-        searchSuggest = searchSuggest( "Xor the Got-Jewel Xor the Got-Jewel Xor the Got-Jewel", "simple_phrase", phraseSuggest);
-        assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel xorr the god jewel xorr the god jewel");
-        // Check the name this time because we're repeating it which is funky
-        assertThat(searchSuggest.getSuggestion("simple_phrase").getEntries().get(0).getText().string(), equalTo("Xor the Got-Jewel Xor the Got-Jewel Xor the Got-Jewel"));
-    }
-
-    private List<String> readMarvelHeroNames() throws IOException, URISyntaxException {
-        return Files.readAllLines(PathUtils.get(Suggest.class.getResource("/config/names.txt").toURI()), StandardCharsets.UTF_8);
-    }
-
-    public void testSizeParam() throws IOException {
-        CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder()
-                .put(SETTING_NUMBER_OF_SHARDS, 1)
-                .put("index.analysis.analyzer.reverse.tokenizer", "standard")
-                .putArray("index.analysis.analyzer.reverse.filter", "lowercase", "reverse")
-                .put("index.analysis.analyzer.body.tokenizer", "standard")
-                .putArray("index.analysis.analyzer.body.filter", "lowercase")
-                .put("index.analysis.analyzer.bigram.tokenizer", "standard")
-                .putArray("index.analysis.analyzer.bigram.filter", "my_shingle", "lowercase")
-                .put("index.analysis.filter.my_shingle.type", "shingle")
-                .put("index.analysis.filter.my_shingle.output_unigrams", false)
-                .put("index.analysis.filter.my_shingle.min_shingle_size", 2)
-                .put("index.analysis.filter.my_shingle.max_shingle_size", 2));
-
-        XContentBuilder mapping = XContentFactory.jsonBuilder()
-                .startObject()
-                    .startObject("type1")
-                        .startObject("_all")
-                            .field("store", true)
-                            .field("term_vector", "with_positions_offsets")
-                        .endObject()
-                        .startObject("properties")
-                            .startObject("body")
-                                .field("type", "text")
-                                .field("analyzer", "body")
-                            .endObject()
-                         .startObject("body_reverse")
-                             .field("type", "text")
-                             .field("analyzer", "reverse")
-                         .endObject()
-                         .startObject("bigram")
-                             .field("type", "text")
-                             .field("analyzer", "bigram")
-                         .endObject()
-                     .endObject()
-                 .endObject()
-             .endObject();
-        assertAcked(builder.addMapping("type1", mapping));
-        ensureGreen();
-
-        String line = "xorr the god jewel";
-        index("test", "type1", "1", "body", line, "body_reverse", line, "bigram", line);
-        line = "I got it this time";
-        index("test", "type1", "2", "body", line, "body_reverse", line, "bigram", line);
-        refresh();
-
-        PhraseSuggestionBuilder phraseSuggestion = phraseSuggestion("bigram")
-                .realWordErrorLikelihood(0.95f)
-                .gramSize(2)
-                .analyzer("body")
-                .addCandidateGenerator(candidateGenerator("body").minWordLength(1).prefixLength(1).suggestMode("always").size(1).accuracy(0.1f))
-                .smoothingModel(new StupidBackoff(0.1))
-                .maxErrors(1.0f)
-                .size(5);
-        Suggest searchSuggest = searchSuggest("Xorr the Gut-Jewel", "simple_phrase", phraseSuggestion);
-        assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
-
-        // we allow a size of 2 now on the shard generator level so "god" will be found since it's LD2
-        phraseSuggestion.clearCandidateGenerators()
-                .addCandidateGenerator(candidateGenerator("body").minWordLength(1).prefixLength(1).suggestMode("always").size(2).accuracy(0.1f));
-        searchSuggest = searchSuggest( "Xorr the Gut-Jewel", "simple_phrase", phraseSuggestion);
-        assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
-    }
-
-    @Nightly
-    public void testPhraseBoundaryCases() throws IOException, URISyntaxException {
-        CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder()
-                .put(indexSettings()).put(SETTING_NUMBER_OF_SHARDS, 1) // to get reliable statistics we should put this all into one shard
-                .put("index.analysis.analyzer.body.tokenizer", "standard")
-                .putArray("index.analysis.analyzer.body.filter", "lowercase")
-                .put("index.analysis.analyzer.bigram.tokenizer", "standard")
-                .putArray("index.analysis.analyzer.bigram.filter", "my_shingle", "lowercase")
-                .put("index.analysis.analyzer.ngram.tokenizer", "standard")
-                .putArray("index.analysis.analyzer.ngram.filter", "my_shingle2", "lowercase")
-                .put("index.analysis.analyzer.myDefAnalyzer.tokenizer", "standard")
-                .putArray("index.analysis.analyzer.myDefAnalyzer.filter", "shingle", "lowercase")
-                .put("index.analysis.filter.my_shingle.type", "shingle")
-                .put("index.analysis.filter.my_shingle.output_unigrams", false)
-                .put("index.analysis.filter.my_shingle.min_shingle_size", 2)
-                .put("index.analysis.filter.my_shingle.max_shingle_size", 2)
-                .put("index.analysis.filter.my_shingle2.type", "shingle")
-                .put("index.analysis.filter.my_shingle2.output_unigrams", true)
-                .put("index.analysis.filter.my_shingle2.min_shingle_size", 2)
-                .put("index.analysis.filter.my_shingle2.max_shingle_size", 2));
-
-        XContentBuilder mapping = XContentFactory.jsonBuilder()
-                    .startObject().startObject("type1")
-                    .startObject("_all").field("store", true).field("term_vector", "with_positions_offsets").endObject()
-                .startObject("properties")
-                .startObject("body").field("type", "text").field("analyzer", "body").endObject()
-                .startObject("bigram").field("type", "text").field("analyzer", "bigram").endObject()
-                .startObject("ngram").field("type", "text").field("analyzer", "ngram").endObject()
-                .endObject()
-                .endObject().endObject();
-        assertAcked(builder.addMapping("type1", mapping));
-        ensureGreen();
-
-        for (String line : readMarvelHeroNames()) {
-            index("test", "type1", line, "body", line, "bigram", line, "ngram", line);
-        }
-        refresh();
-
-        NumShards numShards = getNumShards("test");
-
-        // Lets make sure some things throw exceptions
-        PhraseSuggestionBuilder phraseSuggestion = phraseSuggestion("bigram")
-                .analyzer("body")
-                .addCandidateGenerator(candidateGenerator("does_not_exist").minWordLength(1).suggestMode("always"))
-                .realWordErrorLikelihood(0.95f)
-                .maxErrors(0.5f)
-                .size(1);
-        Map<String, SuggestionBuilder<?>> suggestion = new HashMap<>();
-        suggestion.put("simple_phrase", phraseSuggestion);
-        try {
-            searchSuggest("Xor the Got-Jewel", numShards.numPrimaries, suggestion);
-            fail("field does not exists");
-        } catch (SearchPhaseExecutionException e) {}
-
-        phraseSuggestion.clearCandidateGenerators().analyzer(null);
-        try {
-            searchSuggest("Xor the Got-Jewel", numShards.numPrimaries, suggestion);
-            fail("analyzer does only produce ngrams");
-        } catch (SearchPhaseExecutionException e) {
-        }
-
-        phraseSuggestion.analyzer("bigram");
-        try {
-            searchSuggest("Xor the Got-Jewel", numShards.numPrimaries, suggestion);
-            fail("analyzer does only produce ngrams");
-        } catch (SearchPhaseExecutionException e) {
-        }
-
-        // Now we'll make sure some things don't
-        phraseSuggestion.forceUnigrams(false);
-        searchSuggest( "Xor the Got-Jewel", 0, suggestion);
-
-        // Field doesn't produce unigrams but the analyzer does
-        phraseSuggestion.forceUnigrams(true).analyzer("ngram");
-        searchSuggest( "Xor the Got-Jewel", 0, suggestion);
-
-        phraseSuggestion = phraseSuggestion("ngram")
-                .analyzer("myDefAnalyzer")
-                .forceUnigrams(true)
-                .realWordErrorLikelihood(0.95f)
-                .maxErrors(0.5f)
-                .size(1)
-                .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always"));
-        Suggest suggest = searchSuggest( "Xor the Got-Jewel", 0, suggestion);
-
-        // "xorr the god jewel" and and "xorn the god jewel" have identical scores (we are only using unigrams to score), so we tie break by
-        // earlier term (xorn):
-        assertSuggestion(suggest, 0, "simple_phrase", "xorn the god jewel");
-
-        phraseSuggestion.analyzer(null);
-        suggest = searchSuggest( "Xor the Got-Jewel", 0, suggestion);
-
-        // In this case xorr has a better score than xorn because we set the field back to the default (my_shingle2) analyzer, so the
-        // probability that the term is not in the dictionary but is NOT a misspelling is relatively high in this case compared to the
-        // others that have no n-gram with the other terms in the phrase :) you can set this realWorldErrorLikelyhood
-        assertSuggestion(suggest, 0, "simple_phrase", "xorr the god jewel");
-    }
-
-    public void testDifferentShardSize() throws Exception {
-        createIndex("test");
-        ensureGreen();
-        indexRandom(true, client().prepareIndex("test", "type1", "1").setSource("field1", "foobar1").setRouting("1"),
-                client().prepareIndex("test", "type1", "2").setSource("field1", "foobar2").setRouting("2"),
-                client().prepareIndex("test", "type1", "3").setSource("field1", "foobar3").setRouting("3"));
-
-        Suggest suggest = searchSuggest( "foobar", "simple",
-                termSuggestion("field1")
-                        .size(10).minDocFreq(0).suggestMode(SuggestMode.ALWAYS));
-        ElasticsearchAssertions.assertSuggestionSize(suggest, 0, 3, "simple");
-    }
-
-    // see #3469
-    public void testShardFailures() throws IOException, InterruptedException {
-        CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder()
-                .put(indexSettings())
-                .put("index.analysis.analyzer.suggest.tokenizer", "standard")
-                .putArray("index.analysis.analyzer.suggest.filter", "standard", "lowercase", "shingler")
-                .put("index.analysis.filter.shingler.type", "shingle")
-                .put("index.analysis.filter.shingler.min_shingle_size", 2)
-                .put("index.analysis.filter.shingler.max_shingle_size", 5)
-                .put("index.analysis.filter.shingler.output_unigrams", true));
-
-        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type2")
-                .startObject("properties")
-                    .startObject("name")
-                        .field("type", "text")
-                        .field("analyzer", "suggest")
-                    .endObject()
-                .endObject()
-                .endObject().endObject();
-        assertAcked(builder.addMapping("type2", mapping));
-        ensureGreen();
-
-        index("test", "type2", "1", "foo", "bar");
-        index("test", "type2", "2", "foo", "bar");
-        index("test", "type2", "3", "foo", "bar");
-        index("test", "type2", "4", "foo", "bar");
-        index("test", "type2", "5", "foo", "bar");
-        index("test", "type2", "1", "name", "Just testing the suggestions api");
-        index("test", "type2", "2", "name", "An other title about equal length");
-        // Note that the last document has to have about the same length as the other or cutoff rechecking will remove the useful suggestion.
-        refresh();
-
-        // When searching on a shard with a non existing mapping, we should fail
-        SearchRequestBuilder request = client().prepareSearch().setSize(0)
-                .suggest(
-                        new SuggestBuilder().setGlobalText("tetsting sugestion").addSuggestion("did_you_mean",
-                                phraseSuggestion("fielddoesnotexist").maxErrors(5.0f)));
-        assertThrows(request, SearchPhaseExecutionException.class);
-
-        // When searching on a shard which does not hold yet any document of an existing type, we should not fail
-        SearchResponse searchResponse = client().prepareSearch().setSize(0)
-                .suggest(
-                        new SuggestBuilder().setGlobalText("tetsting sugestion").addSuggestion("did_you_mean",
-                                phraseSuggestion("name").maxErrors(5.0f)))
-            .get();
-        ElasticsearchAssertions.assertNoFailures(searchResponse);
-        ElasticsearchAssertions.assertSuggestion(searchResponse.getSuggest(), 0, 0, "did_you_mean", "testing suggestions");
-    }
-
-    // see #3469
-    public void testEmptyShards() throws IOException, InterruptedException {
-        XContentBuilder mappingBuilder = XContentFactory.jsonBuilder().
-                startObject().
-                    startObject("type1").
-                        startObject("properties").
-                            startObject("name").
-                                field("type", "text").
-                                field("analyzer", "suggest").
-                            endObject().
-                        endObject().
-                    endObject().
-                endObject();
-        assertAcked(prepareCreate("test").setSettings(Settings.builder()
-                .put(indexSettings())
-                .put("index.analysis.analyzer.suggest.tokenizer", "standard")
-                .putArray("index.analysis.analyzer.suggest.filter", "standard", "lowercase", "shingler")
-                .put("index.analysis.filter.shingler.type", "shingle")
-                .put("index.analysis.filter.shingler.min_shingle_size", 2)
-                .put("index.analysis.filter.shingler.max_shingle_size", 5)
-                .put("index.analysis.filter.shingler.output_unigrams", true)).addMapping("type1", mappingBuilder));
-        ensureGreen();
-
-        index("test", "type2", "1", "foo", "bar");
-        index("test", "type2", "2", "foo", "bar");
-        index("test", "type1", "1", "name", "Just testing the suggestions api");
-        index("test", "type1", "2", "name", "An other title about equal length");
-        refresh();
-
-        SearchResponse searchResponse = client().prepareSearch()
-                .setSize(0)
-                .suggest(
-                        new SuggestBuilder().setGlobalText("tetsting sugestion").addSuggestion("did_you_mean",
-                                phraseSuggestion("name").maxErrors(5.0f)))
-                .get();
-
-        assertNoFailures(searchResponse);
-        assertSuggestion(searchResponse.getSuggest(), 0, 0, "did_you_mean", "testing suggestions");
-    }
-
-    /**
-     * Searching for a rare phrase shouldn't provide any suggestions if confidence &gt; 1.  This was possible before we rechecked the cutoff
-     * score during the reduce phase.  Failures don't occur every time - maybe two out of five tries but we don't repeat it to save time.
-     */
-    public void testSearchForRarePhrase() throws IOException {
-        // If there isn't enough chaf per shard then shards can become unbalanced, making the cutoff recheck this is testing do more harm then good.
-        int chafPerShard = 100;
-
-        CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder()
-                .put(indexSettings())
-                .put("index.analysis.analyzer.body.tokenizer", "standard")
-                .putArray("index.analysis.analyzer.body.filter", "lowercase", "my_shingle")
-                .put("index.analysis.filter.my_shingle.type", "shingle")
-                .put("index.analysis.filter.my_shingle.output_unigrams", true)
-                .put("index.analysis.filter.my_shingle.min_shingle_size", 2)
-                .put("index.analysis.filter.my_shingle.max_shingle_size", 2));
-
-        XContentBuilder mapping = XContentFactory.jsonBuilder()
-                .startObject()
-                    .startObject("type1")
-                        .startObject("_all")
-                            .field("store", true)
-                            .field("term_vector", "with_positions_offsets")
-                        .endObject()
-                        .startObject("properties")
-                            .startObject("body")
-                                .field("type", "text")
-                                .field("analyzer", "body")
-                            .endObject()
-                        .endObject()
-                    .endObject()
-                .endObject();
-        assertAcked(builder.addMapping("type1", mapping));
-        ensureGreen();
-
-        NumShards test = getNumShards("test");
-
-        List<String> phrases = new ArrayList<>();
-        Collections.addAll(phrases, "nobel prize", "noble gases", "somethingelse prize", "pride and joy", "notes are fun");
-        for (int i = 0; i < 8; i++) {
-            phrases.add("noble somethingelse" + i);
-        }
-        for (int i = 0; i < test.numPrimaries * chafPerShard; i++) {
-            phrases.add("chaff" + i);
-        }
-        for (String phrase: phrases) {
-            index("test", "type1", phrase, "body", phrase);
-        }
-        refresh();
-
-        Suggest searchSuggest = searchSuggest("nobel prize", "simple_phrase", phraseSuggestion("body")
-                .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f))
-                .confidence(2f)
-                .maxErrors(5f)
-                .size(1));
-        assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
-
-        searchSuggest = searchSuggest("noble prize", "simple_phrase", phraseSuggestion("body")
-                .addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f))
-                .confidence(2f)
-                .maxErrors(5f)
-                .size(1));
-        assertSuggestion(searchSuggest, 0, 0, "simple_phrase", "nobel prize");
-    }
-
-    @Nightly
-    public void testSuggestWithManyCandidates() throws InterruptedException, ExecutionException, IOException {
-        CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder()
-                .put(indexSettings())
-                .put(SETTING_NUMBER_OF_SHARDS, 1) // A single shard will help to keep the tests repeatable.
-                .put("index.analysis.analyzer.text.tokenizer", "standard")
-                .putArray("index.analysis.analyzer.text.filter", "lowercase", "my_shingle")
-                .put("index.analysis.filter.my_shingle.type", "shingle")
-                .put("index.analysis.filter.my_shingle.output_unigrams", true)
-                .put("index.analysis.filter.my_shingle.min_shingle_size", 2)
-                .put("index.analysis.filter.my_shingle.max_shingle_size", 3));
-
-        XContentBuilder mapping = XContentFactory.jsonBuilder()
-                .startObject()
-                    .startObject("type1")
-                        .startObject("properties")
-                            .startObject("title")
-                                .field("type", "text")
-                                .field("analyzer", "text")
-                            .endObject()
-                        .endObject()
-                    .endObject()
-                .endObject();
-        assertAcked(builder.addMapping("type1", mapping));
-        ensureGreen();
-
-        List<String> titles = new ArrayList<>();
-
-        // We're going to be searching for:
-        //   united states house of representatives elections in washington 2006
-        // But we need to make sure we generate a ton of suggestions so we add a bunch of candidates.
-        // Many of these candidates are drawn from page names on English Wikipedia.
-
-        // Tons of different options very near the exact query term
-        titles.add("United States House of Representatives Elections in Washington 1789");
-        for (int year = 1790; year < 2014; year+= 2) {
-            titles.add("United States House of Representatives Elections in Washington " + year);
-        }
-        // Six of these are near enough to be viable suggestions, just not the top one
-
-        // But we can't stop there!  Titles that are just a year are pretty common so lets just add one per year
-        // since 0.  Why not?
-        for (int year = 0; year < 2015; year++) {
-            titles.add(Integer.toString(year));
-        }
-        // That ought to provide more less good candidates for the last term
-
-        // Now remove or add plural copies of every term we can
-        titles.add("State");
-        titles.add("Houses of Parliament");
-        titles.add("Representative Government");
-        titles.add("Election");
-
-        // Now some possessive
-        titles.add("Washington's Birthday");
-
-        // And some conjugation
-        titles.add("Unified Modeling Language");
-        titles.add("Unite Against Fascism");
-        titles.add("Stated Income Tax");
-        titles.add("Media organizations housed within colleges");
-
-        // And other stuff
-        titles.add("Untied shoelaces");
-        titles.add("Unit circle");
-        titles.add("Untitled");
-        titles.add("Unicef");
-        titles.add("Unrated");
-        titles.add("UniRed");
-        titles.add("Jalan Uniten–Dengkil"); // Highway in Malaysia
-        titles.add("UNITAS");
-        titles.add("UNITER");
-        titles.add("Un-Led-Ed");
-        titles.add("STATS LLC");
-        titles.add("Staples");
-        titles.add("Skates");
-        titles.add("Statues of the Liberators");
-        titles.add("Staten Island");
-        titles.add("Statens Museum for Kunst");
-        titles.add("Hause"); // The last name or the German word, whichever.
-        titles.add("Hose");
-        titles.add("Hoses");
-        titles.add("Howse Peak");
-        titles.add("The Hoose-Gow");
-        titles.add("Hooser");
-        titles.add("Electron");
-        titles.add("Electors");
-        titles.add("Evictions");
-        titles.add("Coronal mass ejection");
-        titles.add("Wasington"); // A film?
-        titles.add("Warrington"); // A town in England
-        titles.add("Waddington"); // Lots of places have this name
-        titles.add("Watlington"); // Ditto
-        titles.add("Waplington"); // Yup, also a town
-        titles.add("Washing of the Spears"); // Book
-
-        for (char c = 'A'; c <= 'Z'; c++) {
-            // Can't forget lists, glorious lists!
-            titles.add("List of former members of the United States House of Representatives (" + c + ")");
-
-            // Lots of people are named Washington <Middle Initial>. LastName
-            titles.add("Washington " + c + ". Lastname");
-
-            // Lets just add some more to be evil
-            titles.add("United " + c);
-            titles.add("States " + c);
-            titles.add("House " + c);
-            titles.add("Elections " + c);
-            titles.add("2006 " + c);
-            titles.add(c + " United");
-            titles.add(c + " States");
-            titles.add(c + " House");
-            titles.add(c + " Elections");
-            titles.add(c + " 2006");
-        }
-
-        List<IndexRequestBuilder> builders = new ArrayList<>();
-        for (String title: titles) {
-            builders.add(client().prepareIndex("test", "type1").setSource("title", title));
-        }
-        indexRandom(true, builders);
-
-        PhraseSuggestionBuilder suggest = phraseSuggestion("title")
-                .addCandidateGenerator(candidateGenerator("title")
-                        .suggestMode("always")
-                        .maxTermFreq(.99f)
-                        .size(1000) // Setting a silly high size helps of generate a larger list of candidates for testing.
-                        .maxInspections(1000) // This too
-                )
-                .confidence(0f)
-                .maxErrors(2f)
-                .shardSize(30000)
-                .size(30000);
-        Suggest searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", suggest);
-        assertSuggestion(searchSuggest, 0, 0, "title", "united states house of representatives elections in washington 2006");
-        assertSuggestionSize(searchSuggest, 0, 25480, "title");  // Just to prove that we've run through a ton of options
-
-        suggest.size(1);
-        long start = System.currentTimeMillis();
-        searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", suggest);
-        long total = System.currentTimeMillis() - start;
-        assertSuggestion(searchSuggest, 0, 0, "title", "united states house of representatives elections in washington 2006");
-        // assertThat(total, lessThan(1000L)); // Takes many seconds without fix - just for debugging
-    }
-
     public void testPhraseSuggesterCollate() throws InterruptedException, ExecutionException, IOException {
         CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder()
                 .put(indexSettings())
@@ -1141,7 +109,7 @@ public class SuggestSearchTests extends ESIntegTestCase {
 
         // suggest without collate
         PhraseSuggestionBuilder suggest = phraseSuggestion("title")
-                .addCandidateGenerator(candidateGenerator("title")
+                .addCandidateGenerator(new DirectCandidateGeneratorBuilder("title")
                         .suggestMode("always")
                         .maxTermFreq(.99f)
                         .size(10)
@@ -1201,7 +169,8 @@ public class SuggestSearchTests extends ESIntegTestCase {
                 .string();
 
         PhraseSuggestionBuilder filteredFilterSuggest = suggest.collateQuery(filterStringAsFilter);
-        searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", filteredFilterSuggest);
+        searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title",
+            filteredFilterSuggest);
         assertSuggestionSize(searchSuggest, 0, 2, "title");
 
         // collate suggest with bad query
@@ -1245,12 +214,15 @@ public class SuggestSearchTests extends ESIntegTestCase {
         params.put("query_field", "title");
 
         PhraseSuggestionBuilder phraseSuggestWithParams = suggest.collateQuery(collateWithParams).collateParams(params);
-        searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", phraseSuggestWithParams);
+        searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title",
+            phraseSuggestWithParams);
         assertSuggestionSize(searchSuggest, 0, 2, "title");
 
         // collate query request with prune set to true
-        PhraseSuggestionBuilder phraseSuggestWithParamsAndReturn = suggest.collateQuery(collateWithParams).collateParams(params).collatePrune(true);
-        searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title", phraseSuggestWithParamsAndReturn);
+        PhraseSuggestionBuilder phraseSuggestWithParamsAndReturn = suggest.collateQuery(collateWithParams).collateParams(params)
+            .collatePrune(true);
+        searchSuggest = searchSuggest("united states house of representatives elections in washington 2006", "title",
+            phraseSuggestWithParamsAndReturn);
         assertSuggestionSize(searchSuggest, 0, 10, "title");
         assertSuggestionPhraseCollateMatchExists(searchSuggest, "title", 2);
     }

From 38bee27b112b62cfd227a86eb330a0349958f190 Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Mon, 30 May 2016 21:37:34 +0200
Subject: [PATCH 0520/1311] testCorruptionOnNetworkLayerFinalizingRecovery
 should keep on trying to assign replicas

In rare cases it can reach the limit set by index.allocation.max_retries
---
 .../test/java/org/elasticsearch/index/store/CorruptedFileIT.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
index c3323919c82..0000a6d65dd 100644
--- a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
+++ b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
@@ -332,6 +332,7 @@ public class CorruptedFileIT extends ESIntegTestCase {
             .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
             .put("index.routing.allocation.include._name", primariesNode.getNode().getName())
             .put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE)
+            .put("index.allocation.max_retries", Integer.MAX_VALUE) // keep on retrying
 
         ));
         ensureGreen(); // allocated with empty commit

From 454de6a8f21a5dceb6bca2fd8201a247c8af6a9d Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Mon, 30 May 2016 21:27:38 -0700
Subject: [PATCH 0521/1311] Tests: Remove unnecessary evil jarhell tests

We have 3 evil tests for jarhell. They have been failing in java 9
because of how evil they are. The first checks the leniency we add for
jarhell in the jdk itself. This is unecessary, since if the leniency
wasn't there, we would already be failing all jarhell checks. The second
is checking the compile version is compatible with the jdk. This is
simpler since we don't need to fake the java version: we know 1.7 should
be compatibile with both java 8 and 9, so we can use that as a constant.
Finally the last test checks if the java version system property is
broken. This is simply something we should not check, we have to trust
that java specifies it correctly, and again, if it was broken, all
jarhell checks would be broken.
---
 .../elasticsearch/bootstrap/JarHellTests.java |  11 ++
 .../bootstrap/EvilJarHellTests.java           | 100 ------------------
 2 files changed, 11 insertions(+), 100 deletions(-)
 delete mode 100644 qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilJarHellTests.java

diff --git a/core/src/test/java/org/elasticsearch/bootstrap/JarHellTests.java b/core/src/test/java/org/elasticsearch/bootstrap/JarHellTests.java
index 984b9846b15..bb3ef29176e 100644
--- a/core/src/test/java/org/elasticsearch/bootstrap/JarHellTests.java
+++ b/core/src/test/java/org/elasticsearch/bootstrap/JarHellTests.java
@@ -182,6 +182,17 @@ public class JarHellTests extends ESTestCase {
         }
     }
 
+    public void testRequiredJDKVersionIsOK() throws Exception {
+        Path dir = createTempDir();
+        Manifest manifest = new Manifest();
+        Attributes attributes = manifest.getMainAttributes();
+        attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0.0");
+        attributes.put(new Attributes.Name("X-Compile-Target-JDK"), "1.7");
+        URL[] jars = {makeJar(dir, "foo.jar", manifest, "Foo.class")};
+
+        JarHell.checkJarHell(jars);
+    }
+
     /** make sure if a plugin is compiled against the same ES version, it works */
     public void testGoodESVersionInJar() throws Exception {
         Path dir = createTempDir();
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilJarHellTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilJarHellTests.java
deleted file mode 100644
index 25bf3ac48f9..00000000000
--- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilJarHellTests.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.bootstrap;
-
-import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.test.ESTestCase;
-
-import java.io.IOException;
-import java.net.URL;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardOpenOption;
-import java.util.jar.Attributes;
-import java.util.jar.JarOutputStream;
-import java.util.jar.Manifest;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
-
-/** Tests for Jarhell that change very important system properties... very evil! */
-@SuppressForbidden(reason  = "modifies system properties intentionally")
-public class EvilJarHellTests extends ESTestCase {
-
-    URL makeJar(Path dir, String name, Manifest manifest, String... files) throws IOException {
-        Path jarpath = dir.resolve(name);
-        ZipOutputStream out;
-        if (manifest == null) {
-            out = new JarOutputStream(Files.newOutputStream(jarpath, StandardOpenOption.CREATE));
-        } else {
-            out = new JarOutputStream(Files.newOutputStream(jarpath, StandardOpenOption.CREATE), manifest);
-        }
-        for (String file : files) {
-            out.putNextEntry(new ZipEntry(file));
-        }
-        out.close();
-        return jarpath.toUri().toURL();
-    }
-
-    public void testBootclasspathLeniency() throws Exception {
-        Path dir = createTempDir();
-        String previousJavaHome = System.getProperty("java.home");
-        System.setProperty("java.home", dir.toString());
-        URL[] jars = {makeJar(dir, "foo.jar", null, "DuplicateClass.class"), makeJar(dir, "bar.jar", null, "DuplicateClass.class")};
-        try {
-            JarHell.checkJarHell(jars);
-        } finally {
-            System.setProperty("java.home", previousJavaHome);
-        }
-    }
-
-    public void testRequiredJDKVersionIsOK() throws Exception {
-        Path dir = createTempDir();
-        String previousJavaVersion = System.getProperty("java.specification.version");
-        System.setProperty("java.specification.version", "1.7");
-
-        Manifest manifest = new Manifest();
-        Attributes attributes = manifest.getMainAttributes();
-        attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0.0");
-        attributes.put(new Attributes.Name("X-Compile-Target-JDK"), "1.7");
-        URL[] jars = {makeJar(dir, "foo.jar", manifest, "Foo.class")};
-        try {
-            JarHell.checkJarHell(jars);
-        } finally {
-            System.setProperty("java.specification.version", previousJavaVersion);
-        }
-    }
-
-    public void testBadJDKVersionProperty() throws Exception {
-        Path dir = createTempDir();
-        String previousJavaVersion = System.getProperty("java.specification.version");
-        System.setProperty("java.specification.version", "bogus");
-
-        Manifest manifest = new Manifest();
-        Attributes attributes = manifest.getMainAttributes();
-        attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0.0");
-        attributes.put(new Attributes.Name("X-Compile-Target-JDK"), "1.7");
-        URL[] jars = {makeJar(dir, "foo.jar", manifest, "Foo.class")};
-        try {
-            JarHell.checkJarHell(jars);
-        } finally {
-            System.setProperty("java.specification.version", previousJavaVersion);
-        }
-    }
-}

From 9570feaf8de61c737d48272b60dc1c7788d2d656 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Mon, 30 May 2016 16:29:25 +0200
Subject: [PATCH 0522/1311] Process dynamic templates in order. #18638

When calling `findTemplateBuilder(context, currentFieldName, "text", null)`,
elasticsearch ignores all templates that have a `match_mapping_type` set since
no dynamic type is provided (the last parameter, which is null in that case).
So this should only be called _last_. Otherwise, if a path-based template
matches, it will have precedence over all type-based templates.

Closes #18625
---
 .../index/mapper/DocumentParser.java          | 10 -------
 .../index/mapper/object/RootObjectMapper.java |  2 +-
 .../index/mapper/DynamicMappingTests.java     | 29 +++++++++++++++++++
 3 files changed, 30 insertions(+), 11 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
index 536e6ecbca5..41f8963b593 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
@@ -676,16 +676,6 @@ final class DocumentParser {
 
     private static Mapper.Builder<?,?> createBuilderFromDynamicValue(final ParseContext context, XContentParser.Token token, String currentFieldName) throws IOException {
         if (token == XContentParser.Token.VALUE_STRING) {
-            // do a quick test to see if its fits a dynamic template, if so, use it.
-            // we need to do it here so we can handle things like attachment templates, where calling
-            // text (to see if its a date) causes the binary value to be cleared
-            {
-                Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, "text", null);
-                if (builder != null) {
-                    return builder;
-                }
-            }
-
             if (context.root().dateDetection()) {
                 String text = context.parser().text();
                 // a safe check since "1" gets parsed as well
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/object/RootObjectMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/object/RootObjectMapper.java
index 7c1c28b68d5..c92b6502730 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/object/RootObjectMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/object/RootObjectMapper.java
@@ -278,7 +278,7 @@ public class RootObjectMapper extends ObjectMapper {
         return typeParser.parse(name, dynamicTemplate.mappingForName(name, dynamicType), parserContext);
     }
 
-    public DynamicTemplate findTemplate(ContentPath path, String name, String matchType) {
+    private DynamicTemplate findTemplate(ContentPath path, String name, String matchType) {
         for (DynamicTemplate dynamicTemplate : dynamicTemplates) {
             if (dynamicTemplate.match(path, name, matchType)) {
                 return dynamicTemplate;
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
index c44ea296f73..76ee2efa7ea 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
@@ -34,6 +34,7 @@ import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.core.BooleanFieldMapper;
 import org.elasticsearch.index.mapper.core.BooleanFieldMapper.BooleanFieldType;
 import org.elasticsearch.index.mapper.core.DateFieldMapper;
+import org.elasticsearch.index.mapper.core.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.core.DateFieldMapper.DateFieldType;
 import org.elasticsearch.index.mapper.core.NumberFieldMapper;
 import org.elasticsearch.index.mapper.core.NumberFieldMapper.NumberFieldType;
@@ -645,4 +646,32 @@ public class DynamicMappingTests extends ESSingleNodeTestCase {
         mapper = defaultMapper.mappers().smartNameFieldMapper("s_double");
         assertThat(mapper, instanceOf(TextFieldMapper.class));
     }
+
+    public void testDynamicTemplateOrder() throws IOException {
+        // https://github.com/elastic/elasticsearch/issues/18625
+        // elasticsearch used to apply templates that do not have a match_mapping_type first
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startArray("dynamic_templates")
+                    .startObject()
+                        .startObject("type-based")
+                            .field("match_mapping_type", "string")
+                            .startObject("mapping")
+                                .field("type", "keyword")
+                            .endObject()
+                        .endObject()
+                    .endObject()
+                    .startObject()
+                    .startObject("path-based")
+                        .field("path_match", "foo")
+                        .startObject("mapping")
+                            .field("type", "long")
+                        .endObject()
+                    .endObject()
+                .endObject()
+                .endArray()
+                .endObject().endObject();
+        IndexService index = createIndex("test", Settings.EMPTY, "type", mapping);
+        client().prepareIndex("test", "type", "1").setSource("foo", "abc").get();
+        assertThat(index.mapperService().fullName("foo"), instanceOf(KeywordFieldMapper.KeywordFieldType.class));
+    }
 }

From 502a775a7c938aeda566566fa22416c5260cd858 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Tue, 31 May 2016 10:41:44 +0200
Subject: [PATCH 0523/1311] Add primitive to shrink an index into a single
 shard (#18270)

This adds a low level primitive operations to shrink an existing
index into a new index with a single shard. This primitive expects
all shards of the source index to allocated on a single node. Once the target index is initializing on the shrink node it takes a snapshot of the source index shards and copies all files into the target indices data folder. An [optimization](https://issues.apache.org/jira/browse/LUCENE-7300) coming in Lucene 6.1 will also allow for optional constant time copy if hard-links are supported by the filesystem. All mappings are merged into the new indexes metadata once the snapshots have been taken on the merge node.

To shrink an existing index all shards must be moved to a single node (one instance of each shard) and the index must be read-only:

```BASH
$ curl -XPUT 'http://localhost:9200/logs/_settings' -d '{
    "settings" : {
        "index.routing.allocation.require._name" : "shrink_node_name",
        "index.blocks.write" : true
    }
}
```
once all shards are started on the shrink node. the new index can be created via:

```BASH
$ curl -XPUT 'http://localhost:9200/logs/_shrink/logs_single_shard' -d '{
    "settings" : {
        "index.codec" : "best_compression",
        "index.number_of_replicas" : 1
    }
}'
```

This API will perform all needed check before the new index is created and selects the shrink node based on the allocation of the source index. This call returns immediately, to monitor shrink progress the recovery API should be used since all copy operations are reflected in the recovery API with byte copy progress etc.

The shrink operation does not modify the source index, if a shrink operation should
be canceled or if the shrink failed, the target index can simply be deleted and
all resources are released.
---
 .../elasticsearch/action/ActionModule.java    |   3 +
 .../elasticsearch/action/IndicesRequest.java  |   2 +-
 .../CreateIndexClusterStateUpdateRequest.java |  13 +-
 .../indices/create/CreateIndexResponse.java   |   4 +-
 .../admin/indices/shrink/ShrinkAction.java    |  45 +++++
 .../admin/indices/shrink/ShrinkRequest.java   | 107 ++++++++++
 .../indices/shrink/ShrinkRequestBuilder.java  |  47 +++++
 .../admin/indices/shrink/ShrinkResponse.java  |  31 +++
 .../indices/shrink/TransportShrinkAction.java | 162 ++++++++++++++++
 .../org/elasticsearch/bootstrap/ESPolicy.java |   6 +-
 .../client/IndicesAdminClient.java            |  18 ++
 .../client/support/AbstractClient.java        |  20 ++
 .../cluster/metadata/IndexMetaData.java       |   8 +
 .../metadata/MetaDataCreateIndexService.java  |  94 ++++++++-
 .../cluster/routing/RoutingNode.java          |   2 +
 .../common/network/NetworkModule.java         |   2 +
 .../elasticsearch/index/shard/IndexShard.java | 109 ++++++++---
 .../index/shard/LocalShardSnapshot.java       | 138 +++++++++++++
 .../index/shard/StoreRecovery.java            | 182 ++++++++++++++++--
 .../org/elasticsearch/index/store/Store.java  |   3 +-
 .../cluster/IndicesClusterStateService.java   |  19 +-
 .../indices/recovery/RecoveryState.java       |   9 +-
 .../admin/indices/RestShrinkIndexAction.java  |  61 ++++++
 .../snapshots/RestoreService.java             |   2 +-
 .../admin/indices/create/CreateIndexIT.java   | 102 ++++++++++
 .../shrink/TransportShrinkActionTests.java    | 136 +++++++++++++
 .../MetaDataCreateIndexServiceTests.java      | 166 ++++++++++++++++
 .../cluster/routing/ShardRoutingHelper.java   |   6 +
 .../index/shard/IndexShardTests.java          |  96 ++++++++-
 .../index/shard/StoreRecoveryTests.java       | 132 +++++++++++++
 .../elasticsearch/index/store/StoreTests.java |   2 -
 .../IndexingMemoryControllerTests.java        |   2 +-
 ...dicesLifecycleListenerSingleNodeTests.java |   2 +-
 docs/reference/indices.asciidoc               |   1 +
 docs/reference/indices/shrink-index.asciidoc  |  83 ++++++++
 .../index/store/SmbMMapDirectoryTests.java    |   3 +-
 .../store/SmbSimpleFSDirectoryTests.java      |   3 +-
 .../rest-api-spec/api/indices.shrink.json     |  35 ++++
 .../test/indices.shrink/10_basic.yaml         |  72 +++++++
 .../index/store/EsBaseDirectoryTestCase.java  |   7 +-
 40 files changed, 1867 insertions(+), 68 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkAction.java
 create mode 100644 core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java
 create mode 100644 core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequestBuilder.java
 create mode 100644 core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkResponse.java
 create mode 100644 core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java
 create mode 100644 core/src/main/java/org/elasticsearch/index/shard/LocalShardSnapshot.java
 create mode 100644 core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java
 create mode 100644 core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java
 create mode 100644 core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
 create mode 100644 core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java
 create mode 100644 docs/reference/indices/shrink-index.asciidoc
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yaml
 rename plugins/store-smb/src/test/java/org/elasticsearch/index/store/ESBaseDirectoryTestCase.java => test/framework/src/main/java/org/elasticsearch/index/store/EsBaseDirectoryTestCase.java (87%)

diff --git a/core/src/main/java/org/elasticsearch/action/ActionModule.java b/core/src/main/java/org/elasticsearch/action/ActionModule.java
index bab3dcb2ed2..7e975b922d4 100644
--- a/core/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/core/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -115,6 +115,8 @@ import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettin
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction;
 import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresAction;
 import org.elasticsearch.action.admin.indices.shards.TransportIndicesShardStoresAction;
+import org.elasticsearch.action.admin.indices.shrink.ShrinkAction;
+import org.elasticsearch.action.admin.indices.shrink.TransportShrinkAction;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
 import org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction;
 import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateAction;
@@ -286,6 +288,7 @@ public class ActionModule extends AbstractModule {
         registerAction(IndicesSegmentsAction.INSTANCE, TransportIndicesSegmentsAction.class);
         registerAction(IndicesShardStoresAction.INSTANCE, TransportIndicesShardStoresAction.class);
         registerAction(CreateIndexAction.INSTANCE, TransportCreateIndexAction.class);
+        registerAction(ShrinkAction.INSTANCE, TransportShrinkAction.class);
         registerAction(DeleteIndexAction.INSTANCE, TransportDeleteIndexAction.class);
         registerAction(GetIndexAction.INSTANCE, TransportGetIndexAction.class);
         registerAction(OpenIndexAction.INSTANCE, TransportOpenIndexAction.class);
diff --git a/core/src/main/java/org/elasticsearch/action/IndicesRequest.java b/core/src/main/java/org/elasticsearch/action/IndicesRequest.java
index 4c62a7e849b..3ef699818b6 100644
--- a/core/src/main/java/org/elasticsearch/action/IndicesRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/IndicesRequest.java
@@ -40,7 +40,7 @@ public interface IndicesRequest {
      */
     IndicesOptions indicesOptions();
 
-    static interface Replaceable extends IndicesRequest {
+    interface Replaceable extends IndicesRequest {
         /**
          * Sets the indices that the action relates to.
          */
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java
index 72fe0553f60..8d7e75b1626 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java
@@ -24,6 +24,7 @@ import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.transport.TransportMessage;
 
 import java.util.HashMap;
@@ -40,6 +41,7 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ
     private final String cause;
     private final String index;
     private final boolean updateAllTypes;
+    private Index shrinkFrom;
 
     private IndexMetaData.State state = IndexMetaData.State.OPEN;
 
@@ -54,7 +56,7 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ
     private final Set<ClusterBlock> blocks = new HashSet<>();
 
 
-    CreateIndexClusterStateUpdateRequest(TransportMessage originalMessage, String cause, String index, boolean updateAllTypes) {
+    public CreateIndexClusterStateUpdateRequest(TransportMessage originalMessage, String cause, String index, boolean updateAllTypes) {
         this.originalMessage = originalMessage;
         this.cause = cause;
         this.index = index;
@@ -91,6 +93,11 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ
         return this;
     }
 
+    public CreateIndexClusterStateUpdateRequest shrinkFrom(Index shrinkFrom) {
+        this.shrinkFrom = shrinkFrom;
+        return this;
+    }
+
     public TransportMessage originalMessage() {
         return originalMessage;
     }
@@ -127,6 +134,10 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ
         return blocks;
     }
 
+    public Index shrinkFrom() {
+        return shrinkFrom;
+    }
+
     /** True if all fields that span multiple types should be updated, false otherwise */
     public boolean updateAllTypes() {
         return updateAllTypes;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java
index 485f0a53f69..b9282002349 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexResponse.java
@@ -30,10 +30,10 @@ import java.io.IOException;
  */
 public class CreateIndexResponse extends AcknowledgedResponse {
 
-    CreateIndexResponse() {
+    protected CreateIndexResponse() {
     }
 
-    CreateIndexResponse(boolean acknowledged) {
+    protected CreateIndexResponse(boolean acknowledged) {
         super(acknowledged);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkAction.java
new file mode 100644
index 00000000000..4c09241ad75
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkAction.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.shrink;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.client.ElasticsearchClient;
+
+/**
+ */
+public class ShrinkAction extends Action<ShrinkRequest, ShrinkResponse, ShrinkRequestBuilder> {
+
+    public static final ShrinkAction INSTANCE = new ShrinkAction();
+    public static final String NAME = "indices:admin/shrink";
+
+    private ShrinkAction() {
+        super(NAME);
+    }
+
+    @Override
+    public ShrinkResponse newResponse() {
+        return new ShrinkResponse();
+    }
+
+    @Override
+    public ShrinkRequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new ShrinkRequestBuilder(client, this);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java
new file mode 100644
index 00000000000..069c27ce475
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.admin.indices.shrink;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * Request class to shrink an index into a single shard
+ */
+public class ShrinkRequest extends AcknowledgedRequest<ShrinkRequest> implements IndicesRequest {
+
+    private CreateIndexRequest shrinkIndexRequest;
+    private String sourceIndex;
+
+    ShrinkRequest() {}
+
+    public ShrinkRequest(String targetIndex, String sourceindex) {
+        this.shrinkIndexRequest = new CreateIndexRequest(targetIndex);
+        this.sourceIndex = sourceindex;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = shrinkIndexRequest == null ? null : shrinkIndexRequest.validate();
+        if (sourceIndex == null) {
+            validationException = addValidationError("source index is missing", validationException);
+        }
+        if (shrinkIndexRequest == null) {
+            validationException = addValidationError("shrink index request is missing", validationException);
+        }
+        return validationException;
+    }
+
+    public void setSourceIndex(String index) {
+        this.sourceIndex = index;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        shrinkIndexRequest = new CreateIndexRequest();
+        shrinkIndexRequest.readFrom(in);
+        sourceIndex = in.readString();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        shrinkIndexRequest.writeTo(out);
+        out.writeString(sourceIndex);
+    }
+
+    @Override
+    public String[] indices() {
+        return new String[] {sourceIndex};
+    }
+
+    @Override
+    public IndicesOptions indicesOptions() {
+        return IndicesOptions.lenientExpandOpen();
+    }
+
+    public void setShrinkIndex(CreateIndexRequest shrinkIndexRequest) {
+        this.shrinkIndexRequest = Objects.requireNonNull(shrinkIndexRequest, "shrink index request must not be null");
+    }
+
+    /**
+     * Returns the {@link CreateIndexRequest} for the shrink index
+     */
+    public CreateIndexRequest getShrinkIndexReqeust() {
+        return shrinkIndexRequest;
+    }
+
+    /**
+     * Returns the source index name
+     */
+    public String getSourceIndex() {
+        return sourceIndex;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequestBuilder.java
new file mode 100644
index 00000000000..a098215b750
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequestBuilder.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.admin.indices.shrink;
+
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.settings.Settings;
+
+public class ShrinkRequestBuilder extends AcknowledgedRequestBuilder<ShrinkRequest, ShrinkResponse,
+    ShrinkRequestBuilder> {
+    public ShrinkRequestBuilder(ElasticsearchClient client, ShrinkAction action) {
+        super(client, action, new ShrinkRequest());
+    }
+
+
+    public ShrinkRequestBuilder setTargetIndex(CreateIndexRequest request) {
+        this.request.setShrinkIndex(request);
+        return this;
+    }
+
+    public ShrinkRequestBuilder setSourceIndex(String index) {
+        this.request.setSourceIndex(index);
+        return this;
+    }
+
+    public ShrinkRequestBuilder setSettings(Settings settings) {
+        this.request.getShrinkIndexReqeust().settings(settings);
+        return this;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkResponse.java
new file mode 100644
index 00000000000..4835471ae4c
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkResponse.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.shrink;
+
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+
+public final class ShrinkResponse extends CreateIndexResponse {
+    ShrinkResponse() {
+    }
+
+    ShrinkResponse(boolean acknowledged) {
+        super(acknowledged);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java
new file mode 100644
index 00000000000..ca007e31216
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.shrink;
+
+import org.apache.lucene.index.IndexWriter;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.shard.DocsStats;
+import org.elasticsearch.indices.IndexAlreadyExistsException;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
+
+/**
+ * Main class to initiate shrinking an index into a new index with a single shard
+ */
+public class TransportShrinkAction extends TransportMasterNodeAction<ShrinkRequest, ShrinkResponse> {
+
+    private final MetaDataCreateIndexService createIndexService;
+    private final Client client;
+
+    @Inject
+    public TransportShrinkAction(Settings settings, TransportService transportService, ClusterService clusterService,
+                                 ThreadPool threadPool, MetaDataCreateIndexService createIndexService,
+                                 ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Client client) {
+        super(settings, ShrinkAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver,
+            ShrinkRequest::new);
+        this.createIndexService = createIndexService;
+        this.client = client;
+    }
+
+    @Override
+    protected String executor() {
+        // we go async right away
+        return ThreadPool.Names.SAME;
+    }
+
+    @Override
+    protected ShrinkResponse newResponse() {
+        return new ShrinkResponse();
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(ShrinkRequest request, ClusterState state) {
+        return state.blocks().indexBlockedException(ClusterBlockLevel.METADATA_WRITE, request.getShrinkIndexReqeust().index());
+    }
+
+    @Override
+    protected void masterOperation(final ShrinkRequest shrinkRequest, final ClusterState state,
+                                   final ActionListener<ShrinkResponse> listener) {
+        final String sourceIndex = indexNameExpressionResolver.resolveDateMathExpression(shrinkRequest.getSourceIndex());
+        client.admin().indices().prepareStats(sourceIndex).clear().setDocs(true).execute(new ActionListener<IndicesStatsResponse>() {
+            @Override
+            public void onResponse(IndicesStatsResponse indicesStatsResponse) {
+                CreateIndexClusterStateUpdateRequest updateRequest = prepareCreateIndexRequest(shrinkRequest, state,
+                    indicesStatsResponse.getTotal().getDocs(), indexNameExpressionResolver);
+                createIndexService.createIndex(updateRequest, new ActionListener<ClusterStateUpdateResponse>() {
+                    @Override
+                    public void onResponse(ClusterStateUpdateResponse response) {
+                        listener.onResponse(new ShrinkResponse(response.isAcknowledged()));
+                    }
+
+                    @Override
+                    public void onFailure(Throwable t) {
+                        if (t instanceof IndexAlreadyExistsException) {
+                            logger.trace("[{}] failed to create shrink index", t, updateRequest.index());
+                        } else {
+                            logger.debug("[{}] failed to create shrink index", t, updateRequest.index());
+                        }
+                        listener.onFailure(t);
+                    }
+                });
+            }
+
+            @Override
+            public void onFailure(Throwable e) {
+                listener.onFailure(e);
+            }
+        });
+
+    }
+
+    // static for unittesting this method
+    static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final ShrinkRequest shrinkReqeust, final ClusterState state
+        , final DocsStats docsStats, IndexNameExpressionResolver indexNameExpressionResolver) {
+        final String sourceIndex = indexNameExpressionResolver.resolveDateMathExpression(shrinkReqeust.getSourceIndex());
+        final CreateIndexRequest targetIndex = shrinkReqeust.getShrinkIndexReqeust();
+        final String targetIndexName = indexNameExpressionResolver.resolveDateMathExpression(targetIndex.index());
+        final IndexMetaData metaData = state.metaData().index(sourceIndex);
+        final Settings targetIndexSettings = Settings.builder().put(targetIndex.settings())
+            .normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX).build();
+        long count = docsStats.getCount();
+        if (count >= IndexWriter.MAX_DOCS) {
+            throw new IllegalStateException("Can't merge index with more than [" + IndexWriter.MAX_DOCS
+                + "] docs -  too many documents");
+        }
+        targetIndex.cause("shrink_index");
+        targetIndex.settings(Settings.builder()
+            .put(targetIndexSettings)
+            // we can only shrink to 1 index so far!
+            .put("index.number_of_shards", 1)
+        );
+
+        return new CreateIndexClusterStateUpdateRequest(targetIndex,
+            "shrink_index", targetIndexName, true)
+            // mappings are updated on the node when merging in the shards, this prevents race-conditions since all mapping must be
+            // applied once we took the snapshot and if somebody fucks things up and switches the index read/write and adds docs we miss
+            // the mappings for everything is corrupted and hard to debug
+            .ackTimeout(targetIndex.timeout())
+            .masterNodeTimeout(targetIndex.masterNodeTimeout())
+            .settings(targetIndex.settings())
+            .aliases(targetIndex.aliases())
+            .customs(targetIndex.customs())
+            .shrinkFrom(metaData.getIndex());
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java b/core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java
index 1cd3a9ad57e..ddc25c88535 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/ESPolicy.java
@@ -35,12 +35,12 @@ import java.util.Map;
 
 /** custom policy for union of static and dynamic permissions */
 final class ESPolicy extends Policy {
-    
+
     /** template policy file, the one used in tests */
     static final String POLICY_RESOURCE = "security.policy";
     /** limited policy for scripts */
     static final String UNTRUSTED_RESOURCE = "untrusted.policy";
-    
+
     final Policy template;
     final Policy untrusted;
     final Policy system;
@@ -60,7 +60,7 @@ final class ESPolicy extends Policy {
     }
 
     @Override @SuppressForbidden(reason = "fast equals check is desired")
-    public boolean implies(ProtectionDomain domain, Permission permission) {        
+    public boolean implies(ProtectionDomain domain, Permission permission) {
         CodeSource codeSource = domain.getCodeSource();
         // codesource can be null when reducing privileges via doPrivileged()
         if (codeSource == null) {
diff --git a/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java b/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java
index a475ce15d4e..10275f22d66 100644
--- a/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java
+++ b/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java
@@ -92,6 +92,9 @@ import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRespons
 import org.elasticsearch.action.admin.indices.shards.IndicesShardStoreRequestBuilder;
 import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresRequest;
 import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse;
+import org.elasticsearch.action.admin.indices.shrink.ShrinkRequest;
+import org.elasticsearch.action.admin.indices.shrink.ShrinkRequestBuilder;
+import org.elasticsearch.action.admin.indices.shrink.ShrinkResponse;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
@@ -782,4 +785,19 @@ public interface IndicesAdminClient extends ElasticsearchClient {
      */
     GetSettingsRequestBuilder prepareGetSettings(String... indices);
 
+    /**
+     * Shrinks an index using an explicit request allowing to specify the settings, mappings and aliases of the target index of the index.
+     */
+    ShrinkRequestBuilder prepareShrinkIndex(String sourceIndex, String targetIndex);
+
+    /**
+     * Shrinks an index using an explicit request allowing to specify the settings, mappings and aliases of the target index of the index.
+     */
+    ActionFuture<ShrinkResponse> shrinkIndex(ShrinkRequest request);
+
+    /**
+     * Shrinks an index using an explicit request allowing to specify the settings, mappings and aliases of the target index of the index.
+     */
+    void shrinkIndex(ShrinkRequest request, ActionListener<ShrinkResponse> listener);
+
 }
diff --git a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
index e36bc4b8d77..6481bec1b83 100644
--- a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
+++ b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
@@ -212,6 +212,10 @@ import org.elasticsearch.action.admin.indices.shards.IndicesShardStoreRequestBui
 import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresAction;
 import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresRequest;
 import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse;
+import org.elasticsearch.action.admin.indices.shrink.ShrinkAction;
+import org.elasticsearch.action.admin.indices.shrink.ShrinkRequest;
+import org.elasticsearch.action.admin.indices.shrink.ShrinkRequestBuilder;
+import org.elasticsearch.action.admin.indices.shrink.ShrinkResponse;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder;
@@ -1684,6 +1688,22 @@ public abstract class AbstractClient extends AbstractComponent implements Client
             return new GetSettingsRequestBuilder(this, GetSettingsAction.INSTANCE, indices);
         }
 
+        @Override
+        public ShrinkRequestBuilder prepareShrinkIndex(String sourceIndex, String targetIndex) {
+            return new ShrinkRequestBuilder(this, ShrinkAction.INSTANCE).setSourceIndex(sourceIndex)
+                .setTargetIndex(new CreateIndexRequest(targetIndex));
+        }
+
+        @Override
+        public ActionFuture<ShrinkResponse> shrinkIndex(ShrinkRequest request) {
+            return execute(ShrinkAction.INSTANCE, request);
+        }
+
+        @Override
+        public void shrinkIndex(ShrinkRequest request, ActionListener<ShrinkResponse> listener) {
+            execute(ShrinkAction.INSTANCE, request, listener);
+        }
+
         @Override
         public ActionFuture<GetSettingsResponse> getSettings(GetSettingsRequest request) {
             return execute(GetSettingsAction.INSTANCE, request);
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
index 52721411f46..3a1856c7481 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
@@ -383,6 +383,14 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         return mappings.get(mappingType);
     }
 
+    public static final Setting<String> INDEX_SHRINK_SOURCE_UUID = Setting.simpleString("index.shrink.source.uuid");
+    public static final Setting<String> INDEX_SHRINK_SOURCE_NAME = Setting.simpleString("index.shrink.source.name");
+
+
+    public Index getMergeSourceIndex() {
+        return INDEX_SHRINK_SOURCE_UUID.exists(settings) ? new Index(INDEX_SHRINK_SOURCE_NAME.get(settings),  INDEX_SHRINK_SOURCE_UUID.get(settings)) : null;
+    }
+
     /**
      * Sometimes, the default mapping exists and an actual mapping is not created yet (introduced),
      * in this case, we want to return the default mapping in case it has some default mapping definitions.
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
index c8c352fc46d..5532d4788ad 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
@@ -21,6 +21,7 @@ package org.elasticsearch.cluster.metadata;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
@@ -31,11 +32,15 @@ import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.cluster.block.ClusterBlock;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexMetaData.Custom;
 import org.elasticsearch.cluster.metadata.IndexMetaData.State;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -55,17 +60,18 @@ import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.indices.IndexCreationException;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.InvalidIndexNameException;
-import org.elasticsearch.script.ScriptService;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 
@@ -80,6 +86,8 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_CREATION_DATE;
@@ -264,7 +272,6 @@ public class MetaDataCreateIndexService extends AbstractComponent {
                                     templatesAliases.put(aliasMetaData.alias(), aliasMetaData);
                                 }
                             }
-
                             Settings.Builder indexSettingsBuilder = Settings.builder();
                             // apply templates, here, in reverse order, since first ones are better matching
                             for (int i = templates.size() - 1; i >= 0; i--) {
@@ -293,6 +300,11 @@ public class MetaDataCreateIndexService extends AbstractComponent {
                             }
 
                             indexSettingsBuilder.put(SETTING_INDEX_UUID, UUIDs.randomBase64UUID());
+                            final Index shrinkFromIndex = request.shrinkFrom();
+                            if (shrinkFromIndex != null) {
+                                prepareShrinkIndexSettings(currentState, mappings.keySet(), indexSettingsBuilder, shrinkFromIndex,
+                                    request.index());
+                            }
 
                             Settings actualIndexSettings = indexSettingsBuilder.build();
 
@@ -481,4 +493,82 @@ public class MetaDataCreateIndexService extends AbstractComponent {
             return Regex.simpleMatch(template.template(), request.index());
         }
     }
+
+    /**
+     * Validates the settings and mappings for shrinking an index.
+     * @return the list of nodes at least one instance of the source index shards are allocated
+     */
+    static List<String> validateShrinkIndex(ClusterState state, String sourceIndex,
+                                        Set<String> targetIndexMappingsTypes, String targetIndexName,
+                                        Settings targetIndexSettings) {
+        if (state.metaData().hasIndex(targetIndexName)) {
+            throw new IndexAlreadyExistsException(state.metaData().index(targetIndexName).getIndex());
+        }
+        final IndexMetaData sourceMetaData = state.metaData().index(sourceIndex);
+        if (sourceMetaData == null) {
+            throw new IndexNotFoundException(sourceIndex);
+        }
+        // ensure index is read-only
+        if (state.blocks().indexBlocked(ClusterBlockLevel.WRITE, sourceIndex) == false) {
+            throw new IllegalStateException("index " + sourceIndex + " must be read-only to shrink index. use \"index.blocks.write=true\"");
+        }
+
+        if (sourceMetaData.getNumberOfShards() == 1) {
+            throw new IllegalArgumentException("can't shrink an index with only one shard");
+        }
+
+        if ((targetIndexMappingsTypes.size() > 1 ||
+            (targetIndexMappingsTypes.isEmpty() || targetIndexMappingsTypes.contains(MapperService.DEFAULT_MAPPING)) == false)) {
+            throw new IllegalArgumentException("mappings are not allowed when shrinking indices" +
+                ", all mappings are copied from the source index");
+        }
+        if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)
+            && IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings) > 1) {
+            throw new IllegalArgumentException("can not shrink index into more than one shard");
+        }
+
+        // now check that index is all on one node
+        final IndexRoutingTable table = state.routingTable().index(sourceIndex);
+        Map<String, AtomicInteger> nodesToNumRouting = new HashMap<>();
+        int numShards = sourceMetaData.getNumberOfShards();
+        for (ShardRouting routing : table.shardsWithState(ShardRoutingState.STARTED)) {
+            nodesToNumRouting.computeIfAbsent(routing.currentNodeId(), (s) -> new AtomicInteger(0)).incrementAndGet();
+        }
+        List<String> nodesToAllocateOn = new ArrayList<>();
+        for (Map.Entry<String, AtomicInteger> entries : nodesToNumRouting.entrySet()) {
+            int numAllocations = entries.getValue().get();
+            assert numAllocations <= numShards : "wait what? " + numAllocations + " is > than num shards " + numShards;
+            if (numAllocations == numShards) {
+                nodesToAllocateOn.add(entries.getKey());
+            }
+        }
+        if (nodesToAllocateOn.isEmpty()) {
+            throw new IllegalStateException("index " + sourceIndex +
+                " must have all shards allocated on the same node to shrink index");
+        }
+        return nodesToAllocateOn;
+    }
+
+    static void prepareShrinkIndexSettings(ClusterState currentState, Set<String> mappingKeys, Settings.Builder indexSettingsBuilder, Index shrinkFromIndex, String shrinkIntoName) {
+        final IndexMetaData sourceMetaData = currentState.metaData().index(shrinkFromIndex.getName());
+        final List<String> nodesToAllocateOn = validateShrinkIndex(currentState, shrinkFromIndex.getName(),
+            mappingKeys, shrinkIntoName, indexSettingsBuilder.build());
+        final Predicate<String> analysisSimilarityPredicate = (s) -> s.startsWith("index.similarity.")
+            || s.startsWith("index.analysis.");
+        indexSettingsBuilder
+            // we can only shrink to 1 index so far!
+            .put("index.number_of_shards", 1)
+            // we use "i.r.a.include" rather than "i.r.a.require" since it's allows one of the nodes holding an
+            // instanceof all shards.
+            .put("index.routing.allocation.include._id",
+                Strings.arrayToCommaDelimitedString(nodesToAllocateOn.toArray()))
+            // we only try once and then give up with a shrink index
+            .put("index.allocation.max_retries", 1)
+            // now copy all similarity / analysis settings - this overrides all settings from the user unless they
+            // wanna add extra settings
+            .put(sourceMetaData.getSettings().filter(analysisSimilarityPredicate))
+            .put(IndexMetaData.INDEX_SHRINK_SOURCE_NAME.getKey(), shrinkFromIndex.getName())
+            .put(IndexMetaData.INDEX_SHRINK_SOURCE_UUID.getKey(), shrinkFromIndex.getUUID());
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
index 8e099eb14f6..a57601abffc 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
@@ -19,12 +19,14 @@
 
 package org.elasticsearch.cluster.routing;
 
+import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.shard.ShardId;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
index 201e5297511..fd9191fd713 100644
--- a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
+++ b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
@@ -72,6 +72,7 @@ import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestGetStoredSc
 import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestPutSearchTemplateAction;
 import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestPutStoredScriptAction;
 import org.elasticsearch.rest.action.admin.cluster.tasks.RestPendingClusterTasksAction;
+import org.elasticsearch.rest.action.admin.indices.RestShrinkIndexAction;
 import org.elasticsearch.rest.action.admin.indices.alias.RestIndicesAliasesAction;
 import org.elasticsearch.rest.action.admin.indices.alias.delete.RestIndexDeleteAliasesAction;
 import org.elasticsearch.rest.action.admin.indices.alias.get.RestGetAliasesAction;
@@ -209,6 +210,7 @@ public class NetworkModule extends AbstractModule {
         RestIndexPutAliasAction.class,
         RestIndicesAliasesAction.class,
         RestCreateIndexAction.class,
+        RestShrinkIndexAction.class,
         RestDeleteIndexAction.class,
         RestCloseIndexAction.class,
         RestOpenIndexAction.class,
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index d4a147d475b..adcf36f694f 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -35,8 +35,11 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequest;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RestoreSource;
+import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.common.Booleans;
@@ -55,7 +58,10 @@ import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.Callback;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.SuspendableRefContainer;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexModule;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.cache.IndexCache;
@@ -102,6 +108,7 @@ import org.elasticsearch.index.translog.TranslogStats;
 import org.elasticsearch.index.warmer.ShardIndexWarmerService;
 import org.elasticsearch.index.warmer.WarmerStats;
 import org.elasticsearch.indices.IndexingMemoryController;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.recovery.RecoveryFailedException;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.indices.recovery.RecoveryTargetService;
@@ -125,6 +132,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiConsumer;
 
 public class IndexShard extends AbstractIndexShardComponent {
 
@@ -920,8 +928,7 @@ public class IndexShard extends AbstractIndexShardComponent {
         } else {
             openMode = EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG;
         }
-        final EngineConfig config = newEngineConfig(openMode, translogConfig, cachingPolicy,
-            new IndexShardRecoveryPerformer(shardId, mapperService, logger));
+        final EngineConfig config = newEngineConfig(openMode);
         // we disable deletes since we allow for operations to be executed against the shard while recovering
         // but we need to make sure we don't loose deletes until we are done recovering
         config.setEnableGcDeletes(false);
@@ -1109,20 +1116,37 @@ public class IndexShard extends AbstractIndexShardComponent {
         return path;
     }
 
-    public boolean recoverFromStore(DiscoveryNode localNode) {
+    public boolean recoverFromLocalShards(BiConsumer<String, MappingMetaData> mappingUpdateConsumer, List<IndexShard> localShards) throws IOException {
+        final List<LocalShardSnapshot> snapshots = new ArrayList<>();
+        try {
+            for (IndexShard shard : localShards) {
+                snapshots.add(new LocalShardSnapshot(shard));
+            }
+
+            // we are the first primary, recover from the gateway
+            // if its post api allocation, the index should exists
+            assert shardRouting.primary() : "recover from local shards only makes sense if the shard is a primary shard";
+            StoreRecovery storeRecovery = new StoreRecovery(shardId, logger);
+            return storeRecovery.recoverFromLocalShards(mappingUpdateConsumer, this, snapshots);
+        } finally {
+            IOUtils.close(snapshots);
+        }
+    }
+
+    public boolean recoverFromStore() {
         // we are the first primary, recover from the gateway
         // if its post api allocation, the index should exists
         assert shardRouting.primary() : "recover from store only makes sense if the shard is a primary shard";
         boolean shouldExist = shardRouting.allocatedPostIndexCreate(indexSettings.getIndexMetaData());
 
         StoreRecovery storeRecovery = new StoreRecovery(shardId, logger);
-        return storeRecovery.recoverFromStore(this, shouldExist, localNode);
+        return storeRecovery.recoverFromStore(this, shouldExist);
     }
 
-    public boolean restoreFromRepository(IndexShardRepository repository, DiscoveryNode localNode) {
+    public boolean restoreFromRepository(IndexShardRepository repository) {
         assert shardRouting.primary() : "recover from store only makes sense if the shard is a primary shard";
         StoreRecovery storeRecovery = new StoreRecovery(shardId, logger);
-        return storeRecovery.recoverFromRepository(this, repository, localNode);
+        return storeRecovery.recoverFromRepository(this, repository);
     }
 
     /**
@@ -1337,7 +1361,8 @@ public class IndexShard extends AbstractIndexShardComponent {
     }
 
     public void startRecovery(DiscoveryNode localNode, DiscoveryNode sourceNode, RecoveryTargetService recoveryTargetService,
-                              RecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService) {
+                              RecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService,
+                              BiConsumer<String, MappingMetaData> mappingUpdateConsumer, IndicesService indicesService) {
         final RestoreSource restoreSource = shardRouting.restoreSource();
 
         if (shardRouting.isPeerRecovery()) {
@@ -1357,19 +1382,59 @@ public class IndexShard extends AbstractIndexShardComponent {
             }
         } else if (restoreSource == null) {
             // recover from filesystem store
-            final RecoveryState recoveryState = new RecoveryState(shardId(), shardRouting.primary(),
-                RecoveryState.Type.STORE, localNode, localNode);
-            markAsRecovering("from store", recoveryState); // mark the shard as recovering on the cluster state thread
-            threadPool.generic().execute(() -> {
-                try {
-                    if (recoverFromStore(localNode)) {
-                        recoveryListener.onRecoveryDone(recoveryState);
-                    }
-                } catch (Throwable t) {
-                    recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(shardId, sourceNode, localNode, t), true);
-                }
 
-            });
+            IndexMetaData indexMetaData = indexSettings().getIndexMetaData();
+            Index mergeSourceIndex = indexMetaData.getMergeSourceIndex();
+            final boolean recoverFromLocalShards = mergeSourceIndex != null && shardRouting.allocatedPostIndexCreate(indexMetaData) == false && shardRouting.primary();
+            final RecoveryState recoveryState = new RecoveryState(shardId(), shardRouting.primary(),
+                recoverFromLocalShards ? RecoveryState.Type.LOCAL_SHARDS : RecoveryState.Type.STORE, localNode, localNode);
+            if (recoverFromLocalShards) {
+                final List<IndexShard> startedShards = new ArrayList<>();
+                final IndexService sourceIndexService = indicesService.indexService(mergeSourceIndex);
+                final int numShards = sourceIndexService != null ? sourceIndexService.getIndexSettings().getNumberOfShards() : -1;
+                if (sourceIndexService != null) {
+                    for (IndexShard shard : sourceIndexService) {
+                        if (shard.state() == IndexShardState.STARTED) {
+                            startedShards.add(shard);
+                        }
+                    }
+                }
+                if (numShards == startedShards.size()) {
+                    markAsRecovering("from local shards", recoveryState); // mark the shard as recovering on the cluster state thread
+                    threadPool.generic().execute(() -> {
+                        try {
+                            if (recoverFromLocalShards(mappingUpdateConsumer, startedShards)) {
+                                recoveryListener.onRecoveryDone(recoveryState);
+                            }
+                        } catch (Throwable t) {
+                            recoveryListener.onRecoveryFailure(recoveryState,
+                                new RecoveryFailedException(shardId, localNode, localNode, t), true);
+                        }
+                    });
+                } else {
+                    final Throwable t;
+                    if (numShards == -1) {
+                        t = new IndexNotFoundException(mergeSourceIndex);
+                    } else {
+                        t = new IllegalStateException("not all shards from index " + mergeSourceIndex
+                            + " are started yet, expected " + numShards + " found " + startedShards.size() + " can't recover shard "
+                            + shardId());
+                    }
+                    recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(shardId, localNode, localNode, t), true);
+                }
+            } else {
+                markAsRecovering("from store", recoveryState); // mark the shard as recovering on the cluster state thread
+                threadPool.generic().execute(() -> {
+                    try {
+                        if (recoverFromStore()) {
+                            recoveryListener.onRecoveryDone(recoveryState);
+                        }
+                    } catch (Throwable t) {
+                        recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(shardId, sourceNode, localNode, t), true);
+                    }
+
+                });
+            }
         } else {
             // recover from a restore
             final RecoveryState recoveryState = new RecoveryState(shardId(), shardRouting.primary(),
@@ -1378,7 +1443,7 @@ public class IndexShard extends AbstractIndexShardComponent {
             threadPool.generic().execute(() -> {
                 try {
                     final IndexShardRepository indexShardRepository = repositoriesService.indexShardRepository(restoreSource.snapshotId().getRepository());
-                    if (restoreFromRepository(indexShardRepository, localNode)) {
+                    if (restoreFromRepository(indexShardRepository)) {
                         recoveryListener.onRecoveryDone(recoveryState);
                     }
                 } catch (Throwable first) {
@@ -1461,7 +1526,8 @@ public class IndexShard extends AbstractIndexShardComponent {
         return mapperService.documentMapperWithAutoCreate(type);
     }
 
-    private final EngineConfig newEngineConfig(EngineConfig.OpenMode openMode, TranslogConfig translogConfig, QueryCachingPolicy cachingPolicy, TranslogRecoveryPerformer translogRecoveryPerformer) {
+    private final EngineConfig newEngineConfig(EngineConfig.OpenMode openMode) {
+        final IndexShardRecoveryPerformer translogRecoveryPerformer = new IndexShardRecoveryPerformer(shardId, mapperService, logger);
         return new EngineConfig(openMode, shardId,
             threadPool, indexSettings, warmer, store, deletionPolicy, indexSettings.getMergePolicy(),
             mapperService.indexAnalyzer(), similarityService.similarity(mapperService), codecService, shardEventListener, translogRecoveryPerformer, indexCache.query(), cachingPolicy, translogConfig,
@@ -1627,5 +1693,4 @@ public class IndexShard extends AbstractIndexShardComponent {
             IndexShard.this.delete(engine, engineDelete);
         }
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/LocalShardSnapshot.java b/core/src/main/java/org/elasticsearch/index/shard/LocalShardSnapshot.java
new file mode 100644
index 00000000000..25c59caef99
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/shard/LocalShardSnapshot.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.shard;
+
+import org.apache.lucene.index.IndexCommit;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterDirectory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.Lock;
+import org.apache.lucene.store.NoLockFactory;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.store.Store;
+import org.elasticsearch.indices.recovery.RecoveryState;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ */
+final class LocalShardSnapshot implements Closeable {
+    private final IndexShard shard;
+    private final Store store;
+    private final IndexCommit indexCommit;
+    private final AtomicBoolean closed = new AtomicBoolean(false);
+
+    public LocalShardSnapshot(IndexShard shard) {
+        this.shard = shard;
+        store = shard.store();
+        store.incRef();
+        boolean success = false;
+        try {
+            indexCommit = shard.snapshotIndex(true);
+            success = true;
+        } finally {
+            if (success == false) {
+                store.decRef();
+            }
+        }
+    }
+
+    Index getIndex() {
+        return shard.indexSettings().getIndex();
+    }
+
+    Directory getSnapshotDirectory() {
+        /* this directory will not be used for anything else but reading / copying files to another directory
+         * we prevent all write operations on this directory with UOE - nobody should close it either. */
+        return new FilterDirectory(store.directory()) {
+            @Override
+            public String[] listAll() throws IOException {
+                Collection<String> fileNames = indexCommit.getFileNames();
+                final String[] fileNameArray = fileNames.toArray(new String[fileNames.size()]);
+                return fileNameArray;
+            }
+
+            @Override
+            public void deleteFile(String name) throws IOException {
+                throw new UnsupportedOperationException("this directory is read-only");
+            }
+
+            @Override
+            public void sync(Collection<String> names) throws IOException {
+                throw new UnsupportedOperationException("this directory is read-only");
+            }
+
+            @Override
+            public void renameFile(String source, String dest) throws IOException {
+                throw new UnsupportedOperationException("this directory is read-only");
+            }
+
+            @Override
+            public IndexOutput createOutput(String name, IOContext context) throws IOException {
+                throw new UnsupportedOperationException("this directory is read-only");
+            }
+
+            @Override
+            public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
+                throw new UnsupportedOperationException("this directory is read-only");
+            }
+
+            @Override
+            public Lock obtainLock(String name) throws IOException {
+                    /* we do explicitly a no-lock instance since we hold an index commit from a SnapshotDeletionPolicy so we
+                     * can we certain that nobody messes with the files on disk. We also hold a ref on the store which means
+                     * no external source will delete files either.*/
+                return NoLockFactory.INSTANCE.obtainLock(in, name);
+            }
+
+            @Override
+            public void close() throws IOException {
+                throw new UnsupportedOperationException("nobody should close this directory wrapper");
+            }
+        };
+    }
+
+    @Override
+    public void close() throws IOException {
+        if (closed.compareAndSet(false, true)) {
+            try {
+                shard.releaseSnapshot(indexCommit);
+            } finally {
+                store.decRef();
+            }
+        }
+    }
+
+    ImmutableOpenMap<String, MappingMetaData> getMappings() {
+        return shard.indexSettings.getIndexMetaData().getMappings();
+    }
+
+    @Override
+    public String toString() {
+        return "local_shard_snapshot:[" + shard.shardId() + " indexCommit: " + indexCommit + "]";
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
index 574e68d8cc9..c7d6f35eff2 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
@@ -19,20 +19,27 @@
 
 package org.elasticsearch.index.shard;
 
+import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterDirectory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.routing.RestoreSource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.engine.EngineException;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.snapshots.IndexShardRepository;
 import org.elasticsearch.index.snapshots.IndexShardRestoreFailedException;
 import org.elasticsearch.index.store.Store;
@@ -40,6 +47,11 @@ import org.elasticsearch.indices.recovery.RecoveryState;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 
@@ -63,12 +75,11 @@ final class StoreRecovery {
      * files / transaction logs. This
      * @param indexShard the index shard instance to recovery the shard into
      * @param indexShouldExists <code>true</code> iff the index should exist on disk ie. has the shard been allocated previously on the shards store.
-     * @param localNode the reference to the local node
      * @return  <code>true</code> if the shard has been recovered successfully, <code>false</code> if the recovery
      * has been ignored due to a concurrent modification of if the clusters state has changed due to async updates.
      * @see Store
      */
-    boolean recoverFromStore(final IndexShard indexShard, final boolean indexShouldExists, DiscoveryNode localNode) {
+    boolean recoverFromStore(final IndexShard indexShard, final boolean indexShouldExists) {
         if (canRecover(indexShard)) {
             if (indexShard.routingEntry().restoreSource() != null) {
                 throw new IllegalStateException("can't recover - restore source is not null");
@@ -81,6 +92,141 @@ final class StoreRecovery {
         return false;
     }
 
+    boolean recoverFromLocalShards(BiConsumer<String, MappingMetaData> mappingUpdateConsumer, final IndexShard indexShard, final List<LocalShardSnapshot> shards) throws IOException {
+        if (canRecover(indexShard)) {
+            assert indexShard.recoveryState().getType() == RecoveryState.Type.LOCAL_SHARDS : "invalid recovery type: " + indexShard.recoveryState().getType();
+            if (indexShard.routingEntry().restoreSource() != null) {
+                throw new IllegalStateException("can't recover - restore source is not null");
+            }
+            if (shards.isEmpty()) {
+                throw new IllegalArgumentException("shards must not be empty");
+            }
+            Set<Index> indices = shards.stream().map((s) -> s.getIndex()).collect(Collectors.toSet());
+            if (indices.size() > 1) {
+                throw new IllegalArgumentException("can't add shards from more than one index");
+            }
+            for (ObjectObjectCursor<String, MappingMetaData> mapping : shards.get(0).getMappings()) {
+                mappingUpdateConsumer.accept(mapping.key, mapping.value);
+            }
+            for (ObjectObjectCursor<String, MappingMetaData> mapping : shards.get(0).getMappings()) {
+                indexShard.mapperService().merge(mapping.key,mapping.value.source(), MapperService.MergeReason.MAPPING_RECOVERY, true);
+            }
+            return executeRecovery(indexShard, () -> {
+                logger.debug("starting recovery from local shards {}", shards);
+                try {
+                    final Directory directory = indexShard.store().directory(); // don't close this directory!!
+                    addIndices(indexShard.recoveryState().getIndex(), directory, shards.stream().map(s -> s.getSnapshotDirectory())
+                        .collect(Collectors.toList()).toArray(new Directory[shards.size()]));
+                    internalRecoverFromStore(indexShard, true);
+                    // just trigger a merge to do housekeeping on the
+                    // copied segments - we will also see them in stats etc.
+                    indexShard.getEngine().forceMerge(false, -1, false, false, false);
+                } catch (IOException ex) {
+                    throw new IndexShardRecoveryException(indexShard.shardId(), "failed to recover from local shards", ex);
+                }
+
+            });
+        }
+        return false;
+    }
+
+    final void addIndices(RecoveryState.Index indexRecoveryStats, Directory target, Directory... sources) throws IOException {
+        /*
+         * TODO: once we upgraded to Lucene 6.1 use HardlinkCopyDirectoryWrapper to enable hardlinks if possible and enable it
+         * in the security.policy:
+         *
+         * grant codeBase "${codebase.lucene-misc-6.1.0.jar}" {
+         *  // needed to allow shard shrinking to use hard-links if possible via lucenes HardlinkCopyDirectoryWrapper
+         *  permission java.nio.file.LinkPermission "hard";
+         * };
+         * target = new org.apache.lucene.store.HardlinkCopyDirectoryWrapper(target);
+         */
+        try (IndexWriter writer = new IndexWriter(new StatsDirectoryWrapper(target, indexRecoveryStats),
+            new IndexWriterConfig(null)
+                .setCommitOnClose(false)
+                // we don't want merges to happen here - we call maybe merge on the engine
+                // later once we stared it up otherwise we would need to wait for it here
+                // we also don't specify a codec here and merges should use the engines for this index
+                .setMergePolicy(NoMergePolicy.INSTANCE)
+                .setOpenMode(IndexWriterConfig.OpenMode.CREATE))) {
+            writer.addIndexes(sources);
+            writer.commit();
+        }
+    }
+
+    /**
+     * Directory wrapper that records copy process for recovery statistics
+     */
+    static final class StatsDirectoryWrapper extends FilterDirectory {
+        private final RecoveryState.Index index;
+
+        StatsDirectoryWrapper(Directory in, RecoveryState.Index indexRecoveryStats) {
+            super(in);
+            this.index = indexRecoveryStats;
+        }
+
+        @Override
+        public void copyFrom(Directory from, String src, String dest, IOContext context) throws IOException {
+            final long l = from.fileLength(src);
+            final AtomicBoolean copies = new AtomicBoolean(false);
+            // here we wrap the index input form the source directory to report progress of file copy for the recovery stats.
+            // we increment the num bytes recovered in the readBytes method below, if users pull statistics they can see immediately
+            // how much has been recovered.
+            in.copyFrom(new FilterDirectory(from) {
+                @Override
+                public IndexInput openInput(String name, IOContext context) throws IOException {
+                    index.addFileDetail(dest, l, false);
+                    copies.set(true);
+                    final IndexInput input = in.openInput(name, context);
+                    return new IndexInput("StatsDirectoryWrapper(" + input.toString() + ")") {
+                        @Override
+                        public void close() throws IOException {
+                            input.close();
+                        }
+
+                        @Override
+                        public long getFilePointer() {
+                            throw new UnsupportedOperationException("only straight copies are supported");
+                        }
+
+                        @Override
+                        public void seek(long pos) throws IOException {
+                            throw new UnsupportedOperationException("seeks are not supported");
+                        }
+
+                        @Override
+                        public long length() {
+                            return input.length();
+                        }
+
+                        @Override
+                        public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
+                            throw new UnsupportedOperationException("slices are not supported");
+                        }
+
+                        @Override
+                        public byte readByte() throws IOException {
+                           throw new UnsupportedOperationException("use a buffer if you wanna perform well");
+                        }
+
+                        @Override
+                        public void readBytes(byte[] b, int offset, int len) throws IOException {
+                            // we rely on the fact that copyFrom uses a buffer
+                            input.readBytes(b, offset, len);
+                            index.addRecoveredBytesToFile(dest, len);
+                        }
+                    };
+                }
+            }, src, dest, context);
+            if (copies.get() == false) {
+                index.addFileDetail(dest, l, true); // hardlinked - we treat it as reused since the file was already somewhat there
+            } else {
+                assert index.getFileDetails(dest) != null : "File [" + dest + "] has no file details";
+                assert index.getFileDetails(dest).recovered() == l : index.getFileDetails(dest).toString();
+            }
+        }
+    }
+
     /**
      * Recovers an index from a given {@link IndexShardRepository}. This method restores a
      * previously created index snapshot into an existing initializing shard.
@@ -89,7 +235,7 @@ final class StoreRecovery {
      * @return <code>true</code> if the shard has been recovered successfully, <code>false</code> if the recovery
      * has been ignored due to a concurrent modification of if the clusters state has changed due to async updates.
      */
-    boolean recoverFromRepository(final IndexShard indexShard, IndexShardRepository repository, DiscoveryNode localNode) {
+    boolean recoverFromRepository(final IndexShard indexShard, IndexShardRepository repository) {
         if (canRecover(indexShard)) {
             final ShardRouting shardRouting = indexShard.routingEntry();
             if (shardRouting.restoreSource() == null) {
@@ -200,29 +346,28 @@ final class StoreRecovery {
                         // it exists on the directory, but shouldn't exist on the FS, its a leftover (possibly dangling)
                         // its a "new index create" API, we have to do something, so better to clean it than use same data
                         logger.trace("cleaning existing shard, shouldn't exists");
-                        IndexWriter writer = new IndexWriter(store.directory(), new IndexWriterConfig(Lucene.STANDARD_ANALYZER).setOpenMode(IndexWriterConfig.OpenMode.CREATE));
-                        writer.close();
+                        Lucene.cleanLuceneIndex(store.directory());
                     }
                 }
             } catch (Throwable e) {
                 throw new IndexShardRecoveryException(shardId, "failed to fetch index version after copying it over", e);
             }
             recoveryState.getIndex().updateVersion(version);
-
             // since we recover from local, just fill the files and size
             try {
                 final RecoveryState.Index index = recoveryState.getIndex();
-                if (si != null) {
-                    final Directory directory = store.directory();
-                    for (String name : Lucene.files(si)) {
-                        long length = directory.fileLength(name);
-                        index.addFileDetail(name, length, true);
-                    }
+                if (si != null && recoveryState.getType() == RecoveryState.Type.STORE) {
+                    addRecoveredFileDetails(si, store, index);
                 }
             } catch (IOException e) {
                 logger.debug("failed to list file details", e);
             }
-            indexShard.performTranslogRecovery(indexShouldExists);
+            if (recoveryState.getType() == RecoveryState.Type.LOCAL_SHARDS) {
+                assert indexShouldExists;
+                indexShard.skipTranslogRecovery();
+            } else {
+                indexShard.performTranslogRecovery(indexShouldExists);
+            }
             indexShard.finalizeRecovery();
             indexShard.postRecovery("post recovery from shard_store");
         } catch (EngineException | IOException e) {
@@ -232,6 +377,14 @@ final class StoreRecovery {
         }
     }
 
+    private void addRecoveredFileDetails(SegmentInfos si, Store store, RecoveryState.Index index) throws IOException {
+        final Directory directory = store.directory();
+        for (String name : Lucene.files(si)) {
+            long length = directory.fileLength(name);
+            index.addFileDetail(name, length, true);
+        }
+    }
+
     /**
      * Restores shard from {@link RestoreSource} associated with this shard in routing table
      */
@@ -260,4 +413,5 @@ final class StoreRecovery {
             throw new IndexShardRestoreFailedException(shardId, "restore failed", t);
         }
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/index/store/Store.java b/core/src/main/java/org/elasticsearch/index/store/Store.java
index bb162880384..a720f5cb258 100644
--- a/core/src/main/java/org/elasticsearch/index/store/Store.java
+++ b/core/src/main/java/org/elasticsearch/index/store/Store.java
@@ -682,7 +682,7 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
         return refCounter.refCount();
     }
 
-    private static final class StoreDirectory extends FilterDirectory {
+    static final class StoreDirectory extends FilterDirectory {
 
         private final ESLogger deletesLogger;
 
@@ -715,7 +715,6 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
         public String toString() {
             return "store(" + in.toString() + ")";
         }
-
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
index 6f9b1684b9e..23518d4b3c3 100644
--- a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
+++ b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.indices.cluster;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
@@ -68,6 +69,8 @@ import org.elasticsearch.search.SearchService;
 import org.elasticsearch.snapshots.RestoreService;
 import org.elasticsearch.threadpool.ThreadPool;
 
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -473,7 +476,6 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
         }
 
         DiscoveryNodes nodes = event.state().nodes();
-
         for (final ShardRouting shardRouting : routingNode) {
             final IndexService indexService = indicesService.indexService(shardRouting.index());
             if (indexService == null) {
@@ -611,7 +613,17 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
         }
 
         indexShard.startRecovery(nodes.getLocalNode(), sourceNode, recoveryTargetService,
-            new RecoveryListener(shardRouting, indexService), repositoriesService);
+            new RecoveryListener(shardRouting, indexService), repositoriesService, (type, mapping) -> {
+                try {
+                    nodeServicesProvider.getClient().admin().indices().preparePutMapping()
+                        .setConcreteIndex(indexService.index()) // concrete index - no name clash, it uses uuid
+                        .setType(type)
+                        .setSource(mapping.source().string())
+                        .get();
+                } catch (IOException ex) {
+                    throw new ElasticsearchException("failed to stringify mapping source", ex);
+                }
+            }, indicesService);
     }
 
     /**
@@ -667,7 +679,8 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
                 case STORE: return "after recovery from store";
                 case PRIMARY_RELOCATION: return "after recovery (primary relocation) from node [" + state.getSourceNode() + "]";
                 case REPLICA: return "after recovery (replica) from node [" + state.getSourceNode() + "]";
-                default: throw new IllegalArgumentException(state.getType().name());
+                case LOCAL_SHARDS: return "after recovery from local shards";
+                default: throw new IllegalArgumentException("Unknown recovery type: " + state.getType().name());
             }
         }
 
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java
index 326f5362b7e..6e2b300c00c 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java
@@ -96,11 +96,12 @@ public class RecoveryState implements ToXContent, Streamable {
         }
     }
 
-    public static enum Type {
+    public enum Type {
         STORE((byte) 0),
         SNAPSHOT((byte) 1),
         REPLICA((byte) 2),
-        PRIMARY_RELOCATION((byte) 3);
+        PRIMARY_RELOCATION((byte) 3),
+        LOCAL_SHARDS((byte) 4);
 
         private static final Type[] TYPES = new Type[Type.values().length];
 
@@ -978,5 +979,9 @@ public class RecoveryState implements ToXContent, Streamable {
                 return "{ \"error\" : \"" + e.getMessage() + "\"}";
             }
         }
+
+        public File getFileDetails(String dest) {
+            return fileDetails.get(dest);
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java
new file mode 100644
index 00000000000..b5cc861b65e
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.action.admin.indices;
+
+import org.elasticsearch.action.admin.indices.shrink.ShrinkRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.support.AcknowledgedRestListener;
+
+/**
+ *
+ */
+public class RestShrinkIndexAction extends BaseRestHandler {
+
+    @Inject
+    public RestShrinkIndexAction(Settings settings, RestController controller, Client client) {
+        super(settings, client);
+        controller.registerHandler(RestRequest.Method.PUT, "/{index}/_shrink/{target}", this);
+        controller.registerHandler(RestRequest.Method.POST, "/{index}/_shrink/{target}", this);
+    }
+
+    @SuppressWarnings({"unchecked"})
+    @Override
+    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+        if (request.param("target") == null) {
+            throw new IllegalArgumentException("no target index");
+        }
+        if (request.param("index") == null) {
+            throw new IllegalArgumentException("no source index");
+        }
+        ShrinkRequest shrinkIndexRequest = new ShrinkRequest(request.param("target"), request.param("index"));
+        if (request.hasContent()) {
+            shrinkIndexRequest.getShrinkIndexReqeust().source(request.content());
+        }
+        shrinkIndexRequest.timeout(request.paramAsTime("timeout", shrinkIndexRequest.timeout()));
+        shrinkIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", shrinkIndexRequest.masterNodeTimeout()));
+        client.admin().indices().shrinkIndex(shrinkIndexRequest, new AcknowledgedRestListener<>(channel));
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
index 5338f927005..f6e6c4aaed5 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
@@ -112,7 +112,7 @@ import static org.elasticsearch.common.util.set.Sets.newHashSet;
  * method.
  * <p>
  * Individual shards are getting restored as part of normal recovery process in
- * {@link IndexShard#restoreFromRepository(IndexShardRepository, DiscoveryNode)} )}
+ * {@link IndexShard#restoreFromRepository(IndexShardRepository)} )}
  * method, which detects that shard should be restored from snapshot rather than recovered from gateway by looking
  * at the {@link org.elasticsearch.cluster.routing.ShardRouting#restoreSource()} property.
  * <p>
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
index 3b1040446e0..8ee38c86dc8 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
@@ -28,23 +28,31 @@ import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.query.RangeQueryBuilder;
+import org.elasticsearch.index.query.TermsQueryBuilder;
+import org.elasticsearch.node.service.NodeService;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
+import org.junit.Ignore;
 
 import java.util.HashMap;
+import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBlocked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -277,4 +285,98 @@ public class CreateIndexIT extends ESIntegTestCase {
         internalCluster().fullRestart();
         ensureGreen("test");
     }
+
+    public void testCreateShrinkIndex() {
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        prepareCreate("source").setSettings(Settings.builder().put(indexSettings()).put("number_of_shards", randomIntBetween(2, 7))).get();
+        for (int i = 0; i < 20; i++) {
+            client().prepareIndex("source", randomFrom("t1", "t2", "t3")).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get();
+        }
+        ImmutableOpenMap<String, DiscoveryNode> dataNodes = client().admin().cluster().prepareState().get().getState().nodes()
+            .getDataNodes();
+        assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2);
+        DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class);
+        String mergeNode = discoveryNodes[0].getName();
+        // relocate all shards to one node such that we can merge it.
+        client().admin().indices().prepareUpdateSettings("source")
+            .setSettings(Settings.builder()
+                .put("index.routing.allocation.require._name", mergeNode)
+                .put("index.blocks.write", true)).get();
+        ensureGreen();
+        // now merge source into a single shard index
+        assertAcked(client().admin().indices().prepareShrinkIndex("source", "target")
+            .setSettings(Settings.builder().put("index.number_of_replicas", 0).build()).get());
+        ensureGreen();
+        assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+        // let it be allocated anywhere and bump replicas
+        client().admin().indices().prepareUpdateSettings("target")
+            .setSettings(Settings.builder()
+                .putNull("index.routing.allocation.include._id")
+                .put("index.number_of_replicas", 1)).get();
+        ensureGreen();
+        assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+
+        for (int i = 20; i < 40; i++) {
+            client().prepareIndex("target", randomFrom("t1", "t2", "t3")).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get();
+        }
+        flushAndRefresh();
+        assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 40);
+        assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+
+    }
+    /**
+     * Tests that we can manually recover from a failed allocation due to shards being moved away etc.
+     */
+    public void testCreateShrinkIndexFails() throws Exception {
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        prepareCreate("source").setSettings(Settings.builder().put(indexSettings())
+            .put("number_of_shards", randomIntBetween(2, 7))
+            .put("number_of_replicas", 0)).get();
+        for (int i = 0; i < 20; i++) {
+            client().prepareIndex("source", randomFrom("t1", "t2", "t3")).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get();
+        }
+        ImmutableOpenMap<String, DiscoveryNode> dataNodes = client().admin().cluster().prepareState().get().getState().nodes()
+            .getDataNodes();
+        assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2);
+        DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class);
+        String spareNode = discoveryNodes[0].getName();
+        String mergeNode = discoveryNodes[1].getName();
+        // relocate all shards to one node such that we can merge it.
+        client().admin().indices().prepareUpdateSettings("source")
+            .setSettings(Settings.builder().put("index.routing.allocation.require._name", mergeNode)
+                .put("index.blocks.write", true)).get();
+        ensureGreen();
+        // now merge source into a single shard index
+        client().admin().indices().prepareShrinkIndex("source", "target")
+            .setSettings(Settings.builder()
+            .put("index.routing.allocation.exclude._name", mergeNode) // we manually exclude the merge node to forcefully fuck it up
+            .put("index.number_of_replicas", 0)
+            .put("index.allocation.max_retries", 1).build()).get();
+
+        // now we move all shards away from the merge node
+        client().admin().indices().prepareUpdateSettings("source")
+            .setSettings(Settings.builder().put("index.routing.allocation.require._name", spareNode)
+                .put("index.blocks.write", true)).get();
+        ensureGreen("source");
+
+        client().admin().indices().prepareUpdateSettings("target") // erase the forcefully fuckup!
+            .setSettings(Settings.builder().putNull("index.routing.allocation.exclude._name")).get();
+        // wait until it fails
+        assertBusy(() -> {
+            ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
+            RoutingTable routingTables = clusterStateResponse.getState().routingTable();
+            assertTrue(routingTables.index("target").shard(0).getShards().get(0).unassigned());
+            assertEquals(UnassignedInfo.Reason.ALLOCATION_FAILED,
+                routingTables.index("target").shard(0).getShards().get(0).unassignedInfo().getReason());
+            assertEquals(1,
+                routingTables.index("target").shard(0).getShards().get(0).unassignedInfo().getNumFailedAllocations());
+        });
+        client().admin().indices().prepareUpdateSettings("source") // now relocate them all to the right node
+            .setSettings(Settings.builder()
+                .put("index.routing.allocation.require._name", mergeNode)).get();
+        ensureGreen("source");
+        client().admin().cluster().prepareReroute().setRetryFailed(true).get(); // kick off a retry and wait until it's done!
+        ensureGreen();
+        assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java
new file mode 100644
index 00000000000..50bf8715f19
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.shrink;
+
+import org.apache.lucene.index.IndexWriter;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.block.ClusterBlocks;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.shard.DocsStats;
+import org.elasticsearch.indices.IndexAlreadyExistsException;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.gateway.NoopGatewayAllocator;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+
+import static java.util.Collections.emptyMap;
+
+public class TransportShrinkActionTests extends ESTestCase {
+
+    private ClusterState createClusterState(String name, int numShards, int numReplicas, Settings settings) {
+        MetaData.Builder metaBuilder = MetaData.builder();
+        IndexMetaData indexMetaData = IndexMetaData.builder(name).settings(settings(Version.CURRENT)
+            .put(settings))
+            .numberOfShards(numShards).numberOfReplicas(numReplicas).build();
+        metaBuilder.put(indexMetaData, false);
+        MetaData metaData = metaBuilder.build();
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        routingTableBuilder.addAsNew(metaData.index(name));
+
+        RoutingTable routingTable = routingTableBuilder.build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+            .metaData(metaData).routingTable(routingTable).blocks(ClusterBlocks.builder().addBlocks(indexMetaData)).build();
+        return clusterState;
+    }
+
+    public void testErrorCondition() {
+        ClusterState state = createClusterState("source", randomIntBetween(2, 100), randomIntBetween(0, 10),
+            Settings.builder().put("index.blocks.write", true).build());
+        DocsStats stats = new DocsStats(randomIntBetween(0, IndexWriter.MAX_DOCS-1), randomIntBetween(1, 1000));
+
+        assertEquals("Can't merge index with more than [2147483519] docs -  too many documents",
+            expectThrows(IllegalStateException.class, () ->
+            TransportShrinkAction.prepareCreateIndexRequest(new ShrinkRequest("target", "source"), state,
+                new DocsStats(Integer.MAX_VALUE, randomIntBetween(1, 1000)), new IndexNameExpressionResolver(Settings.EMPTY))
+        ).getMessage());
+
+
+        // create one that won't fail
+        ClusterState clusterState = ClusterState.builder(createClusterState("source", randomIntBetween(2, 10), 0,
+            Settings.builder().put("index.blocks.write", true).build())).nodes(DiscoveryNodes.builder().put(newNode("node1")))
+            .build();
+        AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
+            Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))),
+            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
+
+        RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        // now we start the shard
+        routingTable = service.applyStartedShards(clusterState,
+            routingTable.index("source").shardsWithState(ShardRoutingState.INITIALIZING)).routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+
+        TransportShrinkAction.prepareCreateIndexRequest(new ShrinkRequest("target", "source"), clusterState, stats,
+            new IndexNameExpressionResolver(Settings.EMPTY));
+    }
+
+    public void testShrinkIndexSettings() {
+        String indexName = randomAsciiOfLength(10);
+        // create one that won't fail
+        ClusterState clusterState = ClusterState.builder(createClusterState(indexName, randomIntBetween(2, 10), 0,
+            Settings.builder()
+                .put("index.blocks.write", true)
+                .build())).nodes(DiscoveryNodes.builder().put(newNode("node1")))
+            .build();
+        AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
+            Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))),
+            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
+
+        RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        // now we start the shard
+        routingTable = service.applyStartedShards(clusterState,
+            routingTable.index(indexName).shardsWithState(ShardRoutingState.INITIALIZING)).routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+
+        DocsStats stats = new DocsStats(randomIntBetween(0, IndexWriter.MAX_DOCS-1), randomIntBetween(1, 1000));
+        ShrinkRequest target = new ShrinkRequest("target", indexName);
+        CreateIndexClusterStateUpdateRequest request = TransportShrinkAction.prepareCreateIndexRequest(
+            target, clusterState, stats, new IndexNameExpressionResolver(Settings.EMPTY));
+        assertNotNull(request.shrinkFrom());
+        assertEquals(indexName, request.shrinkFrom().getName());
+        assertEquals("1", request.settings().get("index.number_of_shards"));
+        assertEquals("shrink_index", request.cause());
+    }
+
+    private DiscoveryNode newNode(String nodeId) {
+        return new DiscoveryNode(nodeId, DummyTransportAddress.INSTANCE, emptyMap(),
+            Collections.unmodifiableSet(new HashSet<>(Arrays.asList(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA))), Version.CURRENT);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
new file mode 100644
index 00000000000..49d1da0ddec
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.block.ClusterBlocks;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.indices.IndexAlreadyExistsException;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.gateway.NoopGatewayAllocator;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+
+import static java.util.Collections.emptyMap;
+
+public class MetaDataCreateIndexServiceTests extends ESTestCase {
+
+    private ClusterState createClusterState(String name, int numShards, int numReplicas, Settings settings) {
+        MetaData.Builder metaBuilder = MetaData.builder();
+        IndexMetaData indexMetaData = IndexMetaData.builder(name).settings(settings(Version.CURRENT)
+            .put(settings))
+            .numberOfShards(numShards).numberOfReplicas(numReplicas).build();
+        metaBuilder.put(indexMetaData, false);
+        MetaData metaData = metaBuilder.build();
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        routingTableBuilder.addAsNew(metaData.index(name));
+
+        RoutingTable routingTable = routingTableBuilder.build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+            .metaData(metaData).routingTable(routingTable).blocks(ClusterBlocks.builder().addBlocks(indexMetaData)).build();
+        return clusterState;
+    }
+
+    public void testValidateShrinkIndex() {
+        ClusterState state = createClusterState("source", randomIntBetween(2, 100), randomIntBetween(0, 10),
+            Settings.builder().put("index.blocks.write", true).build());
+
+        assertEquals("index [source] already exists",
+            expectThrows(IndexAlreadyExistsException.class, () ->
+                MetaDataCreateIndexService.validateShrinkIndex(state, "target", Collections.emptySet(), "source", Settings.EMPTY)
+            ).getMessage());
+
+        assertEquals("no such index",
+            expectThrows(IndexNotFoundException.class, () ->
+                MetaDataCreateIndexService.validateShrinkIndex(state, "no such index", Collections.emptySet(), "target", Settings.EMPTY)
+            ).getMessage());
+
+        assertEquals("can't shrink an index with only one shard",
+            expectThrows(IllegalArgumentException.class, () ->
+                    MetaDataCreateIndexService.validateShrinkIndex(createClusterState("source", 1, 0,
+                        Settings.builder().put("index.blocks.write", true).build()), "source", Collections.emptySet(),
+                        "target", Settings.EMPTY)
+            ).getMessage());
+
+        assertEquals("index source must be read-only to shrink index. use \"index.blocks.write=true\"",
+            expectThrows(IllegalStateException.class, () ->
+                    MetaDataCreateIndexService.validateShrinkIndex(
+                        createClusterState("source", randomIntBetween(2, 100), randomIntBetween(0, 10), Settings.EMPTY)
+                        , "source", Collections.emptySet(), "target", Settings.EMPTY)
+            ).getMessage());
+
+        assertEquals("index source must have all shards allocated on the same node to shrink index",
+            expectThrows(IllegalStateException.class, () ->
+                MetaDataCreateIndexService.validateShrinkIndex(state, "source", Collections.emptySet(), "target", Settings.EMPTY)
+
+            ).getMessage());
+
+        assertEquals("can not shrink index into more than one shard",
+            expectThrows(IllegalArgumentException.class, () ->
+                MetaDataCreateIndexService.validateShrinkIndex(state, "source", Collections.emptySet(), "target",
+                    Settings.builder().put("index.number_of_shards", 2).build())
+            ).getMessage());
+
+        assertEquals("mappings are not allowed when shrinking indices, all mappings are copied from the source index",
+            expectThrows(IllegalArgumentException.class, () -> {
+                MetaDataCreateIndexService.validateShrinkIndex(state, "source", Collections.singleton("foo"),
+                    "target", Settings.EMPTY);
+                }
+            ).getMessage());
+
+        // create one that won't fail
+        ClusterState clusterState = ClusterState.builder(createClusterState("source", randomIntBetween(2, 10), 0,
+            Settings.builder().put("index.blocks.write", true).build())).nodes(DiscoveryNodes.builder().put(newNode("node1")))
+            .build();
+        AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
+            Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))),
+            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
+
+        RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        // now we start the shard
+        routingTable = service.applyStartedShards(clusterState,
+            routingTable.index("source").shardsWithState(ShardRoutingState.INITIALIZING)).routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+
+        MetaDataCreateIndexService.validateShrinkIndex(clusterState, "source", Collections.emptySet(), "target", Settings.EMPTY);
+    }
+
+    public void testShrinkIndexSettings() {
+        String indexName = randomAsciiOfLength(10);
+        // create one that won't fail
+        ClusterState clusterState = ClusterState.builder(createClusterState(indexName, randomIntBetween(2, 10), 0,
+            Settings.builder()
+                .put("index.blocks.write", true)
+                .put("index.similarity.default.type", "BM25")
+                .put("index.analysis.analyzer.my_analyzer.tokenizer", "keyword")
+                .build())).nodes(DiscoveryNodes.builder().put(newNode("node1")))
+            .build();
+        AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
+            Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))),
+            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
+
+        RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        // now we start the shard
+        routingTable = service.applyStartedShards(clusterState,
+            routingTable.index(indexName).shardsWithState(ShardRoutingState.INITIALIZING)).routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+
+        Settings.Builder builder = Settings.builder();
+        MetaDataCreateIndexService.prepareShrinkIndexSettings(
+            clusterState, Collections.emptySet(), builder, clusterState.metaData().index(indexName).getIndex(), "target");
+        assertEquals("1", builder.build().get("index.number_of_shards"));
+        assertEquals("similarity settings must be copied", "BM25", builder.build().get("index.similarity.default.type"));
+        assertEquals("analysis settings must be copied",
+            "keyword", builder.build().get("index.analysis.analyzer.my_analyzer.tokenizer"));
+        assertEquals("node1", builder.build().get("index.routing.allocation.include._id"));
+        assertEquals("1", builder.build().get("index.allocation.max_retries"));
+    }
+
+    private DiscoveryNode newNode(String nodeId) {
+        return new DiscoveryNode(nodeId, DummyTransportAddress.INSTANCE, emptyMap(),
+            Collections.unmodifiableSet(new HashSet<>(Arrays.asList(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA))), Version.CURRENT);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java
index 81d5c45c39b..707bfd9057e 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingHelper.java
@@ -52,6 +52,12 @@ public class ShardRoutingHelper {
         return routing.reinitializeShard().updateUnassignedInfo(new UnassignedInfo(reason, "test_reinit"));
     }
 
+    public static ShardRouting initWithSameId(ShardRouting copy) {
+        return new ShardRouting(copy.shardId(), copy.currentNodeId(), copy.relocatingNodeId(), copy.restoreSource(),
+            copy.primary(), ShardRoutingState.INITIALIZING, new UnassignedInfo(UnassignedInfo.Reason.REINITIALIZED, null),
+            copy.allocationId(), copy.getExpectedShardSize());
+    }
+
     public static ShardRouting moveToUnassigned(ShardRouting routing, UnassignedInfo info) {
         return routing.moveToUnassigned(info);
     }
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index f96b546855b..ccc1f0a6b36 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -31,9 +31,11 @@ import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.IndexStats;
@@ -46,6 +48,7 @@ import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.InternalClusterInfoService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.AllocationId;
@@ -58,6 +61,7 @@ import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.lease.Releasable;
@@ -118,6 +122,7 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiConsumer;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
@@ -975,7 +980,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         newShard.updateRoutingEntry(routing, false);
         DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
         newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode));
-        assertTrue(newShard.recoverFromStore(localNode));
+        assertTrue(newShard.recoverFromStore());
         assertEquals(translogOps, newShard.recoveryState().getTranslog().recoveredOperations());
         assertEquals(translogOps, newShard.recoveryState().getTranslog().totalOperations());
         assertEquals(translogOps, newShard.recoveryState().getTranslog().totalOperationsOnStart());
@@ -1003,7 +1008,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
         newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode,
             localNode));
-        assertTrue(newShard.recoverFromStore(localNode));
+        assertTrue(newShard.recoverFromStore());
         assertEquals(0, newShard.recoveryState().getTranslog().recoveredOperations());
         assertEquals(0, newShard.recoveryState().getTranslog().totalOperations());
         assertEquals(0, newShard.recoveryState().getTranslog().totalOperationsOnStart());
@@ -1037,7 +1042,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         newShard.updateRoutingEntry(routing, false);
         newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode));
         try {
-            newShard.recoverFromStore(localNode);
+            newShard.recoverFromStore();
             fail("index not there!");
         } catch (IndexShardRecoveryException ex) {
             assertTrue(ex.getMessage().contains("failed to fetch index version after copying it over"));
@@ -1056,7 +1061,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         newShard = test.createShard(routing);
         newShard.updateRoutingEntry(routing, false);
         newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode));
-        assertTrue("recover even if there is nothing to recover", newShard.recoverFromStore(localNode));
+        assertTrue("recover even if there is nothing to recover", newShard.recoverFromStore());
 
         newShard.updateRoutingEntry(getInitializingShardRouting(routing).moveToStarted(), true);
         SearchResponse response = client().prepareSearch().get();
@@ -1142,7 +1147,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
             @Override
             public void verify(String verificationToken) {
             }
-        }, localNode));
+        }));
 
         test_target_shard.updateRoutingEntry(routing.moveToStarted(), true);
         assertHitCount(client().prepareSearch("test_target").get(), 1);
@@ -1398,7 +1403,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
     public  static final IndexShard recoverShard(IndexShard newShard) throws IOException {
         DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
         newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), newShard.routingEntry().primary(), RecoveryState.Type.STORE, localNode, localNode));
-        assertTrue(newShard.recoverFromStore(localNode));
+        assertTrue(newShard.recoverFromStore());
         newShard.updateRoutingEntry(newShard.routingEntry().moveToStarted(), true);
         return newShard;
     }
@@ -1507,4 +1512,83 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         // Shard should now be active since we did recover:
         assertTrue(newShard.isActive());
     }
+
+    public void testRecoverFromLocalShard() throws IOException {
+        createIndex("index");
+        createIndex("index_1");
+        createIndex("index_2");
+        client().admin().indices().preparePutMapping("index").setType("test").setSource(jsonBuilder().startObject()
+            .startObject("test")
+            .startObject("properties")
+            .startObject("foo")
+            .field("type", "text")
+            .endObject()
+            .endObject().endObject().endObject()).get();
+        client().prepareIndex("index", "test", "0").setSource("{\"foo\" : \"bar\"}").setRefresh(true).get();
+        client().prepareIndex("index", "test", "1").setSource("{\"foo\" : \"bar\"}").setRefresh(true).get();
+
+
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        IndexService test = indicesService.indexService(resolveIndex("index_1"));
+        IndexShard shard = test.getShardOrNull(0);
+        ShardRouting routing = ShardRoutingHelper.initWithSameId(shard.routingEntry());
+        test.removeShard(0, "b/c simon says so");
+        DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        {
+            final IndexShard newShard = test.createShard(routing);
+            newShard.updateRoutingEntry(routing, false);
+            newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.LOCAL_SHARDS, localNode, localNode));
+
+            BiConsumer<String, MappingMetaData> mappingConsumer = (type, mapping) -> {
+                try {
+                    client().admin().indices().preparePutMapping().setConcreteIndex(newShard.indexSettings().getIndex())
+                        .setType(type)
+                        .setSource(mapping.source().string())
+                        .get();
+                } catch (IOException ex) {
+                    throw new ElasticsearchException("failed to stringify mapping source", ex);
+                }
+            };
+            expectThrows(IllegalArgumentException.class, () -> {
+                IndexService index = indicesService.indexService(resolveIndex("index"));
+                IndexService index_2 = indicesService.indexService(resolveIndex("index_2"));
+                newShard.recoverFromLocalShards(mappingConsumer, Arrays.asList(index.getShard(0), index_2.getShard(0)));
+            });
+
+            IndexService indexService = indicesService.indexService(resolveIndex("index"));
+            assertTrue(newShard.recoverFromLocalShards(mappingConsumer, Arrays.asList(indexService.getShard(0))));
+            RecoveryState recoveryState = newShard.recoveryState();
+            assertEquals(RecoveryState.Stage.DONE, recoveryState.getStage());
+            assertTrue(recoveryState.getIndex().fileDetails().size() > 0);
+            for (RecoveryState.File file : recoveryState.getIndex().fileDetails()) {
+                if (file.reused()) {
+                    assertEquals(file.recovered(), 0);
+                } else {
+                    assertEquals(file.recovered(), file.length());
+                }
+            }
+            routing = ShardRoutingHelper.moveToStarted(routing);
+            newShard.updateRoutingEntry(routing, true);
+            assertHitCount(client().prepareSearch("index_1").get(), 2);
+        }
+        // now check that it's persistent ie. that the added shards are committed
+        {
+            routing = shard.routingEntry();
+            test.removeShard(0, "b/c simon says so");
+            routing = ShardRoutingHelper.reinit(routing);
+            final IndexShard newShard = test.createShard(routing);
+            newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.LOCAL_SHARDS, localNode, localNode));
+            assertTrue(newShard.recoverFromStore());
+            routing = ShardRoutingHelper.moveToStarted(routing);
+            newShard.updateRoutingEntry(routing, true);
+            assertHitCount(client().prepareSearch("index_1").get(), 2);
+        }
+
+        GetMappingsResponse mappingsResponse = client().admin().indices().prepareGetMappings("index_1").get();
+        ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = mappingsResponse.getMappings();
+        assertNotNull(mappings.get("index_1"));
+        assertNotNull(mappings.get("index_1").get("test"));
+        assertEquals(mappings.get("index_1").get("test").get().source().string(), "{\"test\":{\"properties\":{\"foo\":{\"type\":\"text\"}}}}");
+
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java b/core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java
new file mode 100644
index 00000000000..a46e0f7562d
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.shard;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.SegmentCommitInfo;
+import org.apache.lucene.index.SegmentInfos;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.Version;
+import org.elasticsearch.indices.recovery.RecoveryState;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.security.AccessControlException;
+import java.util.Arrays;
+import java.util.function.Predicate;
+
+public class StoreRecoveryTests extends ESTestCase {
+
+    public void testAddIndices() throws IOException {
+        Directory[] dirs = new Directory[randomIntBetween(1, 10)];
+        final int numDocs = randomIntBetween(50, 100);
+        int id = 0;
+        for (int i = 0; i < dirs.length; i++) {
+            dirs[i] = newFSDirectory(createTempDir());
+            IndexWriter writer = new IndexWriter(dirs[i], newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE)
+                .setOpenMode(IndexWriterConfig.OpenMode.CREATE));
+            for (int j = 0; j < numDocs; j++) {
+                writer.addDocument(Arrays.asList(new StringField("id", Integer.toString(id++), Field.Store.YES)));
+            }
+
+            writer.commit();
+            writer.close();
+        }
+        StoreRecovery storeRecovery = new StoreRecovery(new ShardId("foo", "bar", 1), logger);
+        RecoveryState.Index indexStats = new RecoveryState.Index();
+        Directory target = newFSDirectory(createTempDir());
+        storeRecovery.addIndices(indexStats, target, dirs);
+        int numFiles = 0;
+        Predicate<String> filesFilter = (f) -> f.startsWith("segments") == false && f.equals("write.lock") == false
+            && f.startsWith("extra") == false;
+        for (Directory d : dirs) {
+            numFiles += Arrays.asList(d.listAll()).stream().filter(filesFilter).count();
+        }
+        final long targetNumFiles = Arrays.asList(target.listAll()).stream().filter(filesFilter).count();
+        assertEquals(numFiles, targetNumFiles);
+        assertEquals(indexStats.totalFileCount(), targetNumFiles);
+        if (hardLinksSupported(createTempDir())) {
+            assertEquals("upgrade to HardlinkCopyDirectoryWrapper in Lucene 6.1", Version.LATEST, Version.LUCENE_6_0_0);
+            // assertEquals(indexStats.reusedFileCount(), targetNumFiles); -- uncomment this once upgraded to Lucene 6.1
+            assertEquals(indexStats.reusedFileCount(), 0);
+        } else {
+            assertEquals(indexStats.reusedFileCount(), 0);
+        }
+        DirectoryReader reader = DirectoryReader.open(target);
+        SegmentInfos segmentCommitInfos = SegmentInfos.readLatestCommit(target);
+        for (SegmentCommitInfo info : segmentCommitInfos) { // check that we didn't merge
+            assertEquals("all sources must be flush", info.info.getDiagnostics().get("source"), "flush");
+        }
+        assertEquals(reader.numDeletedDocs(), 0);
+        assertEquals(reader.numDocs(), id);
+        reader.close();
+        target.close();
+        IOUtils.close(dirs);
+    }
+
+    public void testStatsDirWrapper() throws IOException {
+        Directory dir = newDirectory();
+        Directory target = newDirectory();
+        RecoveryState.Index indexStats = new RecoveryState.Index();
+        StoreRecovery.StatsDirectoryWrapper wrapper = new StoreRecovery.StatsDirectoryWrapper(target, indexStats);
+        try (IndexOutput output = dir.createOutput("foo.bar", IOContext.DEFAULT)) {
+            CodecUtil.writeHeader(output, "foo", 0);
+            int numBytes = randomIntBetween(100, 20000);
+            for (int i = 0; i < numBytes; i++) {
+                output.writeByte((byte)i);
+            }
+            CodecUtil.writeFooter(output);
+        }
+        wrapper.copyFrom(dir, "foo.bar", "bar.foo", IOContext.DEFAULT);
+        assertNotNull(indexStats.getFileDetails("bar.foo"));
+        assertNull(indexStats.getFileDetails("foo.bar"));
+        assertEquals(dir.fileLength("foo.bar"), indexStats.getFileDetails("bar.foo").length());
+        assertEquals(dir.fileLength("foo.bar"), indexStats.getFileDetails("bar.foo").recovered());
+        assertFalse(indexStats.getFileDetails("bar.foo").reused());
+        IOUtils.close(dir, target);
+    }
+
+    public boolean hardLinksSupported(Path path) throws IOException {
+        try {
+            Files.createFile(path.resolve("foo.bar"));
+            Files.createLink(path.resolve("test"), path.resolve("foo.bar"));
+            BasicFileAttributes destAttr = Files.readAttributes(path.resolve("test"), BasicFileAttributes.class);
+            BasicFileAttributes sourceAttr = Files.readAttributes(path.resolve("foo.bar"), BasicFileAttributes.class);
+            // we won't get here - no permission ;)
+            return destAttr.fileKey() != null
+                && destAttr.fileKey().equals(sourceAttr.fileKey());
+        } catch (AccessControlException ex) {
+            return true; // if we run into that situation we know it's supported.
+        } catch (UnsupportedOperationException ex) {
+            return false;
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/store/StoreTests.java b/core/src/test/java/org/elasticsearch/index/store/StoreTests.java
index 8f64390dbd3..be45b1ad330 100644
--- a/core/src/test/java/org/elasticsearch/index/store/StoreTests.java
+++ b/core/src/test/java/org/elasticsearch/index/store/StoreTests.java
@@ -95,9 +95,7 @@ import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
 
 public class StoreTests extends ESTestCase {
 
diff --git a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
index 4b11bb1bc96..2722fc9d9d3 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
@@ -453,7 +453,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase {
             newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode));
 
             assertEquals(1, imc.availableShards().size());
-            assertTrue(newShard.recoverFromStore(localNode));
+            assertTrue(newShard.recoverFromStore());
             assertTrue("we should have flushed in IMC at least once but did: " + flushes.get(), flushes.get() >= 1);
             newShard.updateRoutingEntry(routing.moveToStarted(), true);
         } finally {
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java
index bbb1e8bf2bf..8d59da7da01 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java
@@ -106,7 +106,7 @@ public class IndicesLifecycleListenerSingleNodeTests extends ESSingleNodeTestCas
             final DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE,
                     emptyMap(), emptySet(), Version.CURRENT);
             shard.markAsRecovering("store", new RecoveryState(shard.shardId(), newRouting.primary(), RecoveryState.Type.SNAPSHOT, newRouting.restoreSource(), localNode));
-            shard.recoverFromStore(localNode);
+            shard.recoverFromStore();
             newRouting = ShardRoutingHelper.moveToStarted(newRouting);
             shard.updateRoutingEntry(newRouting, true);
         } finally {
diff --git a/docs/reference/indices.asciidoc b/docs/reference/indices.asciidoc
index 7543a3f0bbb..86b8bfbf342 100644
--- a/docs/reference/indices.asciidoc
+++ b/docs/reference/indices.asciidoc
@@ -15,6 +15,7 @@ index settings, aliases, mappings, and index templates.
 * <<indices-get-index>>
 * <<indices-exists>>
 * <<indices-open-close>>
+* <<indices-shrink-index>>
 
 [float]
 [[mapping-management]]
diff --git a/docs/reference/indices/shrink-index.asciidoc b/docs/reference/indices/shrink-index.asciidoc
new file mode 100644
index 00000000000..28532db5240
--- /dev/null
+++ b/docs/reference/indices/shrink-index.asciidoc
@@ -0,0 +1,83 @@
+[[indices-shrink-index]]
+== Shrink Index
+
+The shrink index API allows to shrink an existing index into a new index with a single shard.
+In order to shrink an index, all its shards must be allocated on a single node in the cluster.
+This is required since the shrink command will copy all shards index files into the target index
+data folder when the primary of the target index is initially allocated.
+
+When an index is shrunk no write operations should happen to the source index. Elasticsearch will
+enforce the `read-only` property when the shrink command is executed. All operations necessary to shrink the
+source index are executed during initial primary recovery. Once the target index primary shard is started the
+shrink operation has successfully finished. To monitor status and progress use <<cat-recovery>>
+
+
+To shrink and index all shards of that index must be allocated on a single node.
+
+[source,js]
+--------------------------------------------------
+$ curl -XPUT 'http://localhost:9200/logs/_settings' -d '{
+    "settings" : {
+        "index.routing.allocation.require._name" : "shrink_node_name", <1>
+        "index.blocks.write" : true <2>
+    }
+}'
+--------------------------------------------------
+<1> Forces the relocation of all of the indices shards to the node `shrink_node_name`
+<2> Prevents write operations to this index while still allowing metadata changes like deleting the index.
+
+The above second curl example shows how an index called `logs` can be
+forced to allocate at least one copy of each shard on a specific node in the cluster.
+
+The `_shrink` API is similar to <<indices-create-index>> and accepts `settings` and `aliases` for the target index.
+
+[source,js]
+--------------------------------------------------
+$ curl -XPUT 'http://localhost:9200/logs/_shrink/logs_single_shard' -d '{
+    "settings" : {
+        "index.codec" : "best_compression", <1>
+        "index.number_of_replicas" : 0 <2>
+    }
+}'
+--------------------------------------------------
+<1> Enables `best_compression` codec on the target index
+<2> Sets the number of replicas on the target index to `0` to ensure the cluster is green once the shard initialized
+
+The API call above returns immediately once the target index is created but doesn't wait
+for the shrink operation to start. Once the target indices primary shard moves to state `initializing`
+the shrink operation has started.
+
+Once the index is shrunk replicas can be set to `1` and allocation filtering can be removed.
+
+[source,js]
+--------------------------------------------------
+$ curl -XPUT 'http://localhost:9200/logs_single_shard/_settings' -d '{
+    "settings" : {
+        "index.routing.allocation.include._id" : null, <1>
+        "index.number_of_replicas" : 1 <2>
+    }
+}'
+--------------------------------------------------
+
+<1> Resets the allocation filtering for the new shrunk index to allow replica allocation
+<2> Bumps the number of replicas to 1
+
+
+
+[float]
+[[shrink-index-limitations]]
+=== Limitations
+
+Indices can only be shrunk into a single shard if they fully the following requirements:
+
+ * an instance of all of the indices shards must be allocated on a single node
+ * the index must not contain more than `2.14 billion` documents (`2147483519`) in total (sum of all shards)
+  This is the maximum shard size elasticsearch can support.
+ * the index must have more than one shard
+ * the index must be `read-only`, ie. have a cluster block set `index.blocks.write=true`
+ * the target index must not exist
+ * all `index.analysis.*` and `index.similarity.*` settings passed to the `_shrink` call will be overwritten with the
+  source indices settings.
+ * if the target index can't be allocated on the shrink node, due to throttling or other allocation deciders,
+  its primary shard will stay `unassigned` until it can be allocated on that node
+
diff --git a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbMMapDirectoryTests.java b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbMMapDirectoryTests.java
index b368a3e99e7..3dad35adaa2 100644
--- a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbMMapDirectoryTests.java
+++ b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbMMapDirectoryTests.java
@@ -23,9 +23,8 @@ import java.io.IOException;
 import java.nio.file.Path;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MMapDirectory;
-import org.elasticsearch.index.store.SmbDirectoryWrapper;
 
-public class SmbMMapDirectoryTests extends ESBaseDirectoryTestCase {
+public class SmbMMapDirectoryTests extends EsBaseDirectoryTestCase {
 
     @Override
     protected Directory getDirectory(Path file) throws IOException {
diff --git a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbSimpleFSDirectoryTests.java b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbSimpleFSDirectoryTests.java
index d18057a106b..659f6eff7ac 100644
--- a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbSimpleFSDirectoryTests.java
+++ b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbSimpleFSDirectoryTests.java
@@ -23,9 +23,8 @@ import java.io.IOException;
 import java.nio.file.Path;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.SimpleFSDirectory;
-import org.elasticsearch.index.store.SmbDirectoryWrapper;
 
-public class SmbSimpleFSDirectoryTests extends ESBaseDirectoryTestCase {
+public class SmbSimpleFSDirectoryTests extends EsBaseDirectoryTestCase {
 
     @Override
     protected Directory getDirectory(Path file) throws IOException {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json
new file mode 100644
index 00000000000..633e9e16093
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json
@@ -0,0 +1,35 @@
+{
+  "indices.shrink": {
+    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-shrink-index.html",
+    "methods": ["PUT", "POST"],
+    "url": {
+      "path": "/{index}/_shrink/{target}",
+      "paths": ["/{index}/_shrink/{target}"],
+      "parts": {
+        "index": {
+          "type" : "string",
+          "required" : true,
+          "description" : "The name of the source index to shrink"
+        },
+        "target": {
+          "type" : "string",
+          "required" : true,
+          "description" : "The name of the target index to shrink into"
+        }
+      },
+      "params": {
+        "timeout": {
+          "type" : "time",
+          "description" : "Explicit operation timeout"
+        },
+        "master_timeout": {
+          "type" : "time",
+          "description" : "Specify timeout for connection to master"
+        }
+      }
+    },
+    "body": {
+      "description" : "The configuration for the target index (`settings` and `aliases`)"
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yaml
new file mode 100644
index 00000000000..4ef428b9600
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yaml
@@ -0,0 +1,72 @@
+---
+"Shrink index via API":
+  # creates an index with one document.
+  # relocates all it's shards to one node
+  # shrinks it into a new index with a single shard
+  - do:
+      indices.create:
+        index: source
+        body:
+          settings:
+            number_of_replicas: "0"
+  - do:
+      index:
+        index: source
+        type:  test
+        id:    "1"
+        body:  { "foo": "hello world" }
+
+  - do:
+      get:
+        index: source
+        type:  test
+        id:    "1"
+
+  - match: { _index:   source }
+  - match: { _type:    test   }
+  - match: { _id:      "1"     }
+  - match: { _source:  { foo: "hello world" } }
+
+  - do:
+      cluster.state: {}
+
+  # Get master node id
+  - set: { master_node: master }
+
+  # relocate everything to the master node and make it read-only
+  - do:
+      indices.put_settings:
+        index: source
+        body:
+          index.routing.allocation.include._id: $master
+          index.blocks.write: true
+          index.number_of_replicas: 0
+
+  - do:
+      cluster.health:
+        wait_for_status: green
+        index: source
+        wait_for_relocating_shards: 0
+
+  # now we do the actual shrink
+  - do:
+      indices.shrink:
+        index: "source"
+        target: "target"
+        body:
+          index.number_of_replicas: 0
+
+  - do:
+      cluster.health:
+        wait_for_status: green
+
+  - do:
+      get:
+        index: target
+        type:  test
+        id:    "1"
+
+  - match: { _index:   target }
+  - match: { _type:    test   }
+  - match: { _id:      "1"     }
+  - match: { _source:  { foo: "hello world" } }
diff --git a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/ESBaseDirectoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/store/EsBaseDirectoryTestCase.java
similarity index 87%
rename from plugins/store-smb/src/test/java/org/elasticsearch/index/store/ESBaseDirectoryTestCase.java
rename to test/framework/src/main/java/org/elasticsearch/index/store/EsBaseDirectoryTestCase.java
index 855ddce9ebd..638d24e7f9c 100644
--- a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/ESBaseDirectoryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/store/EsBaseDirectoryTestCase.java
@@ -25,6 +25,8 @@ import org.apache.lucene.store.BaseDirectoryTestCase;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.bootstrap.BootstrapForTesting;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.test.junit.listeners.ReproduceInfoPrinter;
 
 /**
@@ -36,8 +38,11 @@ import org.elasticsearch.test.junit.listeners.ReproduceInfoPrinter;
 @TimeoutSuite(millis = TimeUnits.HOUR)
 @LuceneTestCase.SuppressReproduceLine
 @LuceneTestCase.SuppressSysoutChecks(bugUrl = "we log a lot on purpose")
-public abstract class ESBaseDirectoryTestCase extends BaseDirectoryTestCase {
+public abstract class EsBaseDirectoryTestCase extends BaseDirectoryTestCase {
     static {
         BootstrapForTesting.ensureInitialized();
     }
+
+    protected final ESLogger logger = Loggers.getLogger(getClass());
+
 }

From 867d35c08485de5b0ebbec21c06da17f36982420 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Tue, 31 May 2016 11:31:52 +0200
Subject: [PATCH 0524/1311] Never trip circuit breaker in liveness request
 (#18627)

We don't want transport clients to disconnect due to circuit breakers
preventing the liveness request from executing.

Relates to #17951
---
 .../admin/cluster/node/liveness/TransportLivenessAction.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/liveness/TransportLivenessAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/liveness/TransportLivenessAction.java
index 8b29b9379ac..a437fc5ee49 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/liveness/TransportLivenessAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/liveness/TransportLivenessAction.java
@@ -38,7 +38,8 @@ public final class TransportLivenessAction implements TransportRequestHandler<Li
                                    ClusterService clusterService, TransportService transportService) {
         this.clusterService = clusterService;
         this.clusterName = clusterName;
-        transportService.registerRequestHandler(NAME, LivenessRequest::new, ThreadPool.Names.SAME, this);
+        transportService.registerRequestHandler(NAME, LivenessRequest::new, ThreadPool.Names.SAME,
+            false, false /*can not trip circuit breaker*/, this);
     }
 
     @Override

From 589b6c63c6b1eb4d6f0b4f58ea1bf29b336c4ce3 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Tue, 31 May 2016 11:50:50 +0200
Subject: [PATCH 0525/1311] Include shrink-index.asciidoc

---
 docs/reference/indices.asciidoc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/reference/indices.asciidoc b/docs/reference/indices.asciidoc
index 86b8bfbf342..e49340fe15f 100644
--- a/docs/reference/indices.asciidoc
+++ b/docs/reference/indices.asciidoc
@@ -73,6 +73,8 @@ include::indices/indices-exists.asciidoc[]
 
 include::indices/open-close.asciidoc[]
 
+include::indices/shrink-index.asciidoc[]
+
 include::indices/put-mapping.asciidoc[]
 
 include::indices/get-mapping.asciidoc[]

From 85bf48b4c1395950c4d1e1a138b597f93e56d6cf Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Thu, 26 May 2016 13:49:22 +0200
Subject: [PATCH 0526/1311] Added release notes for 5.0.0-alpha3

---
 docs/reference/index.asciidoc                 |   2 +-
 docs/reference/release-notes.asciidoc         |   2 +
 .../release-notes/5.0.0-alpha3.asciidoc       | 329 ++++++++++++++++++
 3 files changed, 332 insertions(+), 1 deletion(-)
 create mode 100644 docs/reference/release-notes/5.0.0-alpha3.asciidoc

diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc
index 8812a2bed95..132d763f714 100644
--- a/docs/reference/index.asciidoc
+++ b/docs/reference/index.asciidoc
@@ -1,7 +1,7 @@
 [[elasticsearch-reference]]
 = Elasticsearch Reference
 
-:version:       5.0.0-alpha2
+:version:       5.0.0-alpha3
 :major-version: 5.x
 :branch:        master
 :jdk:           1.8.0_73
diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc
index 76fe55d3d34..90492dee33e 100644
--- a/docs/reference/release-notes.asciidoc
+++ b/docs/reference/release-notes.asciidoc
@@ -5,11 +5,13 @@
 --
 This section summarizes the changes in each release.
 
+* <<release-notes-5.0.0-alpha3>>
 * <<release-notes-5.0.0-alpha2>>
 * <<release-notes-5.0.0-alpha1>>
 * <<release-notes-5.0.0-alpha1-2x>>
 
 --
+include::release-notes/5.0.0-alpha3.asciidoc[]
 include::release-notes/5.0.0-alpha2.asciidoc[]
 include::release-notes/5.0.0-alpha1.asciidoc[]
 include::release-notes/5.0.0-alpha1-2x.asciidoc[]
diff --git a/docs/reference/release-notes/5.0.0-alpha3.asciidoc b/docs/reference/release-notes/5.0.0-alpha3.asciidoc
new file mode 100644
index 00000000000..09c6f384c4f
--- /dev/null
+++ b/docs/reference/release-notes/5.0.0-alpha3.asciidoc
@@ -0,0 +1,329 @@
+[[release-notes-5.0.0-alpha3]]
+== 5.0.0-alpha3 Release Notes
+
+Also see <<breaking-changes-5.0>>.
+
+[[breaking-5.0.0-alpha3]]
+[float]
+=== Breaking changes
+
+Allocation::
+* Move parsing of allocation commands into REST and remove support for plugins to register allocation commands {pull}17802[#17802] (issue: {issue}17894[#17894])
+
+CAT API::
+* Row-centric output for _cat/fielddata {pull}18068[#18068] (issue: {issue}10249[#10249])
+
+Discovery::
+* Introduce node handshake {pull}15971[#15971] (issue: {issue}9061[#9061])
+
+Inner Hits::
+* Drop top level inner hits in favour of inner hits defined in the query dsl {pull}17816[#17816] (issue: {issue}11118[#11118])
+
+Internal::
+* Actually bound the generic thread pool {pull}17017[#17017]
+
+Packaging::
+* Require /bin/bash in packaging {pull}18259[#18259] (issue: {issue}18251[#18251])
+* Remove plugin script parsing of system properties {pull}18207[#18207] (issue: {issue}18140[#18140])
+
+Percolator::
+* Move the percolator from core to its own module {pull}18511[#18511]
+* Remove percolator cache {pull}18434[#18434]
+
+Plugin Delete By Query::
+* Remove Delete-By-Query plugin {pull}18516[#18516] (issue: {issue}18469[#18469])
+
+Query DSL::
+* Lessen leniency of the query dsl. {pull}18276[#18276]
+
+Scripting::
+* Remove LeafSearchScript.runAsFloat(): Nothing calls it. {pull}18364[#18364]
+* Allow only a single extension for a scripting engine {pull}18332[#18332] (issue: {issue}10598[#10598])
+* Remove 'sandbox' option for script settings, allow only registering a single language. {pull}18226[#18226] (issues: {issue}10598[#10598], {issue}17114[#17114])
+
+Search::
+* Refactor of query profile classes to make way for other profile implementations {pull}18370[#18370] (issue: {issue}10538[#10538])
+
+Settings::
+* Register `indices.query.bool.max_clause_count` setting {pull}18341[#18341] (issue: {issue}18336[#18336])
+* Remove settings and system properties entanglement {pull}18198[#18198] (issue: {issue}18197[#18197])
+
+
+
+[[feature-5.0.0-alpha3]]
+[float]
+=== New features
+
+Ingest::
+* Add a Sort ingest processor {pull}17999[#17999]
+* Add date_index_name processor {pull}17973[#17973] (issue: {issue}17814[#17814])
+
+Reindex API::
+* Port Delete By Query to Reindex infrastructure {pull}18329[#18329] (issue: {issue}16883[#16883])
+
+Snapshot/Restore::
+* Add Google Cloud Storage repository plugin {pull}13578[#13578] (issue: {issue}12880[#12880])
+
+Stats::
+* Extend field stats to report searchable/aggregatable fields {pull}17980[#17980] (issue: {issue}17750[#17750])
+
+
+
+[[enhancement-5.0.0-alpha3]]
+[float]
+=== Enhancements
+
+Aggregations::
+* Rename AggregatorBuilder to AggregationBuilder {pull}18377[#18377] (issue: {issue}18367[#18367])
+* Add the ability to use the breadth_first mode with nested aggregations (such as `top_hits`) which require access to score information. {pull}18127[#18127] (issue: {issue}9825[#9825])
+* Make significant terms work on fields that are indexed with points. {pull}18031[#18031]
+
+Allocation::
+* Limit retries of failed allocations per index {pull}18467[#18467] (issue: {issue}18417[#18417])
+* Immutable ShardRouting {pull}17821[#17821]
+* Add the shard's store status to the explain API {pull}17689[#17689] (issue: {issue}17372[#17372])
+
+Analysis::
+* Core: better error message when analyzer created without tokenizer or… {pull}18455[#18455] (issue: {issue}15492[#15492])
+* Move AsciiFolding earlier in FingerprintAnalyzer filter chain {pull}18281[#18281] (issue: {issue}18266[#18266])
+
+CAT API::
+* Add node name to Cat Recovery {pull}18187[#18187] (issue: {issue}8041[#8041])
+* Add support for documented byte/size units and for micros as a time unit in _cat API {pull}17779[#17779]
+
+Core::
+* Log OS and JVM on startup {pull}18557[#18557]
+* Add GC overhead logging {pull}18419[#18419]
+* Refactor JvmGcMonitorService for testing {pull}18378[#18378]
+* Default to server VM and add client VM check {pull}18155[#18155]
+* Add system bootstrap checks escape hatch {pull}18088[#18088]
+* Avoid sliced locked contention in internal engine {pull}18060[#18060] (issue: {issue}18053[#18053])
+
+Dates::
+* Support full range of Java Long for epoch DateTime {pull}18509[#18509] (issue: {issue}17936[#17936])
+
+Discovery::
+* Log warning if minimum_master_nodes set to less than quorum {pull}15625[#15625]
+
+Exceptions::
+* Make the index-too-old exception more explicit {pull}18438[#18438]
+* Add index name in IndexAlreadyExistsException default message {pull}18274[#18274]
+
+Expressions::
+* Support geo_point fields in lucene expressions {pull}18096[#18096]
+* Add support for .empty to expressions, and some docs improvements {pull}18077[#18077]
+
+Ingest::
+* Expose underlying processor to blame for thrown exception within CompoundProcessor {pull}18342[#18342] (issue: {issue}17823[#17823])
+* Avoid string concatentation in IngestDocument.FieldPath {pull}18108[#18108]
+* add ability to specify multiple grok patterns {pull}18074[#18074] (issue: {issue}17903[#17903])
+* add ability to disable ability to override values of existing fields in set processor {pull}17902[#17902] (issue: {issue}17659[#17659])
+
+Inner Hits::
+* Change scriptFields member in InnerHitBuilder to set {pull}18092[#18092] (issue: {issue}5831[#5831])
+
+Internal::
+* Do not automatically close XContent objects/arrays {pull}18549[#18549] (issue: {issue}18433[#18433])
+* Remove use of a Fields class in snapshot responses {pull}18497[#18497]
+* Removes multiple toXContent entry points for SnapshotInfo {pull}18494[#18494]
+* Removes unused methods in the o/e/common/Strings class {pull}18346[#18346]
+* Determine content length eagerly in HttpServer {pull}18203[#18203]
+* Consolidate query generation in QueryShardContext {pull}18129[#18129]
+* Make reset in QueryShardContext private {pull}18113[#18113]
+* Remove Strings#splitStringToArray {pull}18110[#18110]
+* Add toString() to GetResponse {pull}18102[#18102]
+* ConstructingObjectParser adapts ObjectParser for ctor args {pull}17596[#17596] (issue: {issue}17352[#17352])
+
+Java API::
+* Improve adding clauses to `span_near` and `span_or` query {pull}18485[#18485] (issue: {issue}18478[#18478])
+* QueryBuilder does not need generics. {pull}18133[#18133]
+
+Mapping::
+* Adds a methods to find (and dynamically create) the mappers for the parents of a field with dots in the field name {pull}18106[#18106] (issue: {issue}15951[#15951])
+
+Network::
+* Netty request/response tracer should wait for send {pull}18500[#18500]
+* Exclude specific transport actions from request size limit check {pull}17951[#17951]
+
+Packaging::
+* Don't mkdir directly in deb init script {pull}18503[#18503] (issue: {issue}18307[#18307])
+* Increase default heap size to 2g {pull}18311[#18311] (issues: {issue}16334[#16334], {issue}17686[#17686], {issue}18309[#18309])
+* Switch init.d scripts to use bash {pull}18308[#18308] (issue: {issue}18259[#18259])
+* Switch scripts to use bash {pull}18251[#18251] (issue: {issue}14002[#14002])
+* Further simplifications of plugin script {pull}18239[#18239] (issue: {issue}18207[#18207])
+* Pass ES_JAVA_OPTS to JVM for plugins script {pull}18140[#18140] (issue: {issue}16790[#16790])
+
+Parent/Child::
+* Allow adding additional child types that point to an existing parent type {pull}18446[#18446] (issue: {issue}17956[#17956])
+
+Plugin Lang Painless::
+* improve painless whitelist coverage of java api {pull}18533[#18533]
+* Definition cleanup {pull}18463[#18463]
+* Made def variable casting consistent with invokedynamic rules {pull}18425[#18425]
+* Use Java 9 Indy String Concats, if available {pull}18400[#18400] (issue: {issue}18398[#18398])
+* Add method overloading based on arity {pull}18385[#18385]
+* Refactor WriterUtils to extend ASM GeneratorAdapter {pull}18382[#18382]
+* Whitelist expansion {pull}18372[#18372]
+* Remove boxing when loading and storing values in "def" fields/arrays, remove boxing onsimple method calls of "def" methods {pull}18359[#18359]
+* Some cleanups {pull}18352[#18352]
+* Use isAssignableFrom instead of relying on ClassCastException {pull}18350[#18350]
+* Build descriptor of array and field load/store in code {pull}18338[#18338]
+* Rename the dynamic call site factory to DefBootstrap {pull}18335[#18335]
+* Cleanup of DynamicCallSite {pull}18323[#18323]
+* Improve exception stacktraces {pull}18319[#18319]
+* Make Line Number Available in Painless {pull}18298[#18298]
+* Remove input, support params instead {pull}18287[#18287]
+* Decouple ANTLR AST from Painless {pull}18286[#18286]
+* _value support in painess? {pull}18284[#18284]
+* Long priority over Float {pull}18282[#18282]
+* _score as double, not float {pull}18277[#18277]
+* Add 'ctx' keyword to painless. {pull}18264[#18264]
+* Painless doc access {pull}18262[#18262]
+* Retrieve _score directly from Scorer {pull}18258[#18258]
+* Implement needsScore() correctly. {pull}18247[#18247]
+* Add synthetic length property as alias to Lists, so they can be used like arrays {pull}18241[#18241]
+* Use better typing for dynamic method calls {pull}18234[#18234]
+* Array load/store and length with invokedynamic {pull}18232[#18232] (issue: {issue}18201[#18201])
+* Switch painless dynamic calls to invokedynamic, remove perf hack/cheat {pull}18201[#18201]
+* Add fielddata accessors (.value/.values/.distance()/etc) {pull}18169[#18169]
+* painless: optimize/simplify dynamic field and method access {pull}18151[#18151]
+* Painless: Single-Quoted Strings {pull}18150[#18150]
+
+Plugins::
+* Add plugin information for Verbose mode {pull}18051[#18051] (issue: {issue}16375[#16375])
+
+Query DSL::
+* Enforce MatchQueryBuilder#maxExpansions() to be strictly positive {pull}18464[#18464]
+* Don't allow `fuzziness` for `multi_match` types `cross_fields`, `phrase` and `phrase_prefix` {pull}18322[#18322] (issues: {issue}6866[#6866], {issue}7764[#7764])
+
+REST::
+* CORS handling triggered whether User-Agent is a browser or not {pull}18283[#18283]
+* Add semicolon query string parameter delimiter {pull}18186[#18186] (issue: {issue}18175[#18175])
+* Enable HTTP compression by default with compression level 3 {pull}18066[#18066] (issue: {issue}7309[#7309])
+
+Reindex API::
+* Make Reindex cancellation tests more uniform {pull}18498[#18498]
+* Makes DeleteByQueryRequest implements IndicesRequest {pull}18466[#18466]
+* Switch default batch size for reindex to 1000 {pull}18340[#18340]
+* Teach reindex to retry on search failures {pull}18331[#18331] (issue: {issue}18059[#18059])
+* Remove ReindexResponse in favor of BulkIndexByScrollResponse {pull}18205[#18205]
+* Stricter validation of Reindex's requests_per_second {pull}18028[#18028]
+
+Search::
+* Introduces GeoValidationMethod to GeoDistanceSortBuilder {pull}18036[#18036]
+* Switches from empty boolean query to matchNoDocs {pull}18007[#18007] (issue: {issue}17981[#17981])
+* Allow binary sort values. {pull}17959[#17959] (issue: {issue}6077[#6077])
+
+Search Refactoring::
+* Removes the now obsolete SearchParseElement implementations {pull}18233[#18233]
+
+Snapshot/Restore::
+* Change BlobPath.buildAsString() method {pull}18461[#18461]
+* Remove the Snapshot class in favor of using SnapshotInfo {pull}18167[#18167] (issue: {issue}18156[#18156])
+
+Stats::
+* Do not return fieldstats information for fields that exist in the mapping but not in the index. {pull}18212[#18212] (issue: {issue}17980[#17980])
+* Add whether the shard state fetch is pending to the allocation explain API {pull}18119[#18119] (issue: {issue}17372[#17372])
+* Add Failure Details to every NodesResponse {pull}17964[#17964] (issue: {issue}3740[#3740])
+* Add I/O statistics on Linux {pull}15915[#15915] (issue: {issue}15296[#15296])
+
+Translog::
+* FSync translog outside of the writers global lock {pull}18360[#18360]
+
+
+
+[[bug-5.0.0-alpha3]]
+[float]
+=== Bug fixes
+
+Aggregations::
+* Fix TimeUnitRounding for hour, minute and second units {pull}18415[#18415] (issue: {issue}18326[#18326])
+* Aggregations fix: support include/exclude strings for IP and dates {pull}18408[#18408] (issue: {issue}17705[#17705])
+* Fix xcontent rendering of ip terms aggs. {pull}18003[#18003]
+* Improving parsing of sigma param for Extended Stats Bucket Aggregation {pull}17562[#17562] (issue: {issue}17499[#17499])
+
+Analysis::
+* Add `Character.MODIFIER_SYMBOL` to the list of symbol categories. {pull}18402[#18402] (issue: {issue}18388[#18388])
+
+Bulk::
+* Add not-null precondition check in BulkRequest {pull}18347[#18347] (issue: {issue}12038[#12038])
+
+CAT API::
+* Expand wildcards to closed indices in /_cat/indices {pull}18545[#18545] (issues: {issue}16419[#16419], {issue}17395[#17395])
+
+Circuit Breakers::
+* Free bytes reserved on request breaker {pull}18204[#18204] (issue: {issue}18144[#18144])
+
+Cluster::
+* Dangling indices are not imported if a tombstone for the index exists {pull}18250[#18250] (issue: {issue}18249[#18249])
+* Fix issue with tombstones matching active indices in cluster state {pull}18058[#18058] (issue: {issue}18054[#18054])
+
+Core::
+* Fix concurrency bug in IMC that could cause it to check too infrequently {pull}18357[#18357]
+* Iterables.flatten should not pre-cache the first iterator {pull}18355[#18355] (issue: {issue}18353[#18353])
+* Avoid race while retiring executors {pull}18333[#18333]
+* Don't try to compute completion stats on a reader after we already closed it {pull}18094[#18094]
+
+Highlighting::
+* Skip all geo point queries in plain highlighter {pull}18495[#18495] (issue: {issue}17537[#17537])
+* Exclude all but string fields from highlighting if wildcards are used… {pull}18183[#18183] (issue: {issue}17537[#17537])
+
+Ingest::
+* Pipeline Stats: Fix concurrent modification exception {pull}18177[#18177] (issue: {issue}18126[#18126])
+* Validate properties values according to database type {pull}17940[#17940] (issue: {issue}17683[#17683])
+
+Internal::
+* Add XPointValues {pull}18011[#18011] (issue: {issue}18010[#18010])
+
+Mapping::
+* Make doc_values accessible for _type {pull}18220[#18220]
+* Fix and test handling of `null_value`. {pull}18090[#18090] (issue: {issue}18085[#18085])
+* Fail automatic string upgrade if the value of `index` is not recognized. {pull}18082[#18082] (issue: {issue}18062[#18062])
+
+Packaging::
+* Filter client/server VM options from jvm.options {pull}18473[#18473]
+* Preserve config files from RPM install {pull}18188[#18188] (issue: {issue}18158[#18158])
+* Fix typo in message for variable setup ES_MAX_MEM {pull}18168[#18168]
+* Don't run `mkdir` when $DATA_DIR contains a comma-separated list {pull}17419[#17419] (issue: {issue}16992[#16992])
+
+Percolator::
+* Add support for MatchNoDocsQuery in percolator's query terms extract service {pull}18492[#18492]
+
+Plugin Discovery EC2::
+* Add TAG_SETTING to list of allowed tags for the ec2 discovery plugin. {pull}18257[#18257]
+
+Plugin Lang Painless::
+* Remove Grammar Ambiguities {pull}18531[#18531]
+* Remove if/else ANTLR ambiguity. {pull}18428[#18428]
+* Fix insanely slow compilation {pull}18410[#18410] (issue: {issue}18398[#18398])
+* Fix Bug in Painless Assignment {pull}18379[#18379]
+* Fix bracket shortcuts {pull}18263[#18263]
+
+Plugin Repository Azure::
+* Fix azure files removal {pull}18451[#18451] (issues: {issue}16472[#16472], {issue}18436[#18436])
+
+REST::
+* Do not decode path when sending error {pull}18477[#18477] (issue: {issue}18476[#18476])
+* CORS should permit same origin requests {pull}18278[#18278] (issue: {issue}18256[#18256])
+
+Search::
+* Fix _only_nodes preferences {pull}18483[#18483] (issues: {issue}12546[#12546], {issue}12700[#12700])
+* Speed up named queries. {pull}18470[#18470]
+* Fix parsing single `rescore` element in SearchSourceBuilder {pull}18440[#18440] (issue: {issue}18439[#18439])
+* Fail queries on not indexed fields. {pull}18014[#18014]
+
+Settings::
+* Use object equality to compare versions in IndexSettings {pull}18103[#18103]
+* fix exists method for list settings when using numbered setting format {pull}17949[#17949]
+
+Snapshot/Restore::
+* Fix race condition in snapshot initialization {pull}18426[#18426] (issue: {issue}18121[#18121])
+* Handle indices=["_all"] when restoring a snapshot {pull}18025[#18025]
+
+Stats::
+* Add missing builder.endObject() in FsInfo {pull}18443[#18443] (issues: {issue}15915[#15915], {issue}18433[#18433])
+
+Translog::
+* Snapshotting and sync could cause a dead lock TranslogWriter {pull}18481[#18481] (issues: {issue}1[#1], {issue}18360[#18360], {issue}2[#2])
+
+

From 5c525e660693b2201c59c873fcc7ba2ca965dac5 Mon Sep 17 00:00:00 2001
From: Mike McCandless <mikemccand@apache.org>
Date: Tue, 31 May 2016 06:29:29 -0400
Subject: [PATCH 0527/1311] Remove index_writer_max_memory stat from segment
 stats

---
 .../elasticsearch/index/engine/Engine.java    |  1 -
 .../index/engine/InternalEngine.java          |  1 -
 .../index/engine/SegmentsStats.java           | 22 -------------------
 .../rest/action/cat/RestIndicesAction.java    |  6 -----
 .../rest/action/cat/RestNodesAction.java      |  2 --
 .../rest/action/cat/RestShardsAction.java     |  2 --
 .../indices/stats/IndexStatsIT.java           |  1 -
 docs/reference/cat/nodes.asciidoc             |  2 --
 docs/reference/cluster/stats.asciidoc         |  2 --
 .../test/cat.shards/10_basic.yaml             |  1 -
 10 files changed, 40 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/engine/Engine.java b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
index 76f9a5f8a81..ab142f9dd51 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -503,7 +503,6 @@ public abstract class Engine implements Closeable {
         // by default we don't have a writer here... subclasses can override this
         stats.addVersionMapMemoryInBytes(0);
         stats.addIndexWriterMemoryInBytes(0);
-        stats.addIndexWriterMaxMemoryInBytes(0);
     }
 
     /** How much heap is used that would be freed by a refresh.  Note that this may throw {@link AlreadyClosedException}. */
diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index cf96fb06358..c120b07fce8 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -851,7 +851,6 @@ public class InternalEngine extends Engine {
     protected final void writerSegmentStats(SegmentsStats stats) {
         stats.addVersionMapMemoryInBytes(versionMap.ramBytesUsed());
         stats.addIndexWriterMemoryInBytes(indexWriter.ramBytesUsed());
-        stats.addIndexWriterMaxMemoryInBytes((long) (indexWriter.getConfig().getRAMBufferSizeMB() * 1024 * 1024));
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java b/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java
index dcd97dfef0e..8f0388aef05 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java
@@ -43,7 +43,6 @@ public class SegmentsStats implements Streamable, ToXContent {
     private long pointsMemoryInBytes;
     private long docValuesMemoryInBytes;
     private long indexWriterMemoryInBytes;
-    private long indexWriterMaxMemoryInBytes;
     private long versionMapMemoryInBytes;
     private long bitsetMemoryInBytes;
     private ImmutableOpenMap<String, Long> fileSizes = ImmutableOpenMap.of();
@@ -111,10 +110,6 @@ public class SegmentsStats implements Streamable, ToXContent {
         this.indexWriterMemoryInBytes += indexWriterMemoryInBytes;
     }
 
-    public void addIndexWriterMaxMemoryInBytes(long indexWriterMaxMemoryInBytes) {
-        this.indexWriterMaxMemoryInBytes += indexWriterMaxMemoryInBytes;
-    }
-
     public void addVersionMapMemoryInBytes(long versionMapMemoryInBytes) {
         this.versionMapMemoryInBytes += versionMapMemoryInBytes;
     }
@@ -151,7 +146,6 @@ public class SegmentsStats implements Streamable, ToXContent {
         addPointsMemoryInBytes(mergeStats.pointsMemoryInBytes);
         addDocValuesMemoryInBytes(mergeStats.docValuesMemoryInBytes);
         addIndexWriterMemoryInBytes(mergeStats.indexWriterMemoryInBytes);
-        addIndexWriterMaxMemoryInBytes(mergeStats.indexWriterMaxMemoryInBytes);
         addVersionMapMemoryInBytes(mergeStats.versionMapMemoryInBytes);
         addBitsetMemoryInBytes(mergeStats.bitsetMemoryInBytes);
         addFileSizes(mergeStats.fileSizes);
@@ -252,17 +246,6 @@ public class SegmentsStats implements Streamable, ToXContent {
         return new ByteSizeValue(indexWriterMemoryInBytes);
     }
 
-    /**
-     * Maximum memory index writer may use before it must write buffered documents to a new segment.
-     */
-    public long getIndexWriterMaxMemoryInBytes() {
-        return this.indexWriterMaxMemoryInBytes;
-    }
-
-    public ByteSizeValue getIndexWriterMaxMemory() {
-        return new ByteSizeValue(indexWriterMaxMemoryInBytes);
-    }
-
     /**
      * Estimation of the memory usage by version map
      */
@@ -307,7 +290,6 @@ public class SegmentsStats implements Streamable, ToXContent {
         builder.byteSizeField(Fields.POINTS_MEMORY_IN_BYTES, Fields.POINTS_MEMORY, pointsMemoryInBytes);
         builder.byteSizeField(Fields.DOC_VALUES_MEMORY_IN_BYTES, Fields.DOC_VALUES_MEMORY, docValuesMemoryInBytes);
         builder.byteSizeField(Fields.INDEX_WRITER_MEMORY_IN_BYTES, Fields.INDEX_WRITER_MEMORY, indexWriterMemoryInBytes);
-        builder.byteSizeField(Fields.INDEX_WRITER_MAX_MEMORY_IN_BYTES, Fields.INDEX_WRITER_MAX_MEMORY, indexWriterMaxMemoryInBytes);
         builder.byteSizeField(Fields.VERSION_MAP_MEMORY_IN_BYTES, Fields.VERSION_MAP_MEMORY, versionMapMemoryInBytes);
         builder.byteSizeField(Fields.FIXED_BIT_SET_MEMORY_IN_BYTES, Fields.FIXED_BIT_SET, bitsetMemoryInBytes);
         builder.startObject(Fields.FILE_SIZES);
@@ -342,8 +324,6 @@ public class SegmentsStats implements Streamable, ToXContent {
         static final String DOC_VALUES_MEMORY_IN_BYTES = "doc_values_memory_in_bytes";
         static final String INDEX_WRITER_MEMORY = "index_writer_memory";
         static final String INDEX_WRITER_MEMORY_IN_BYTES = "index_writer_memory_in_bytes";
-        static final String INDEX_WRITER_MAX_MEMORY = "index_writer_max_memory";
-        static final String INDEX_WRITER_MAX_MEMORY_IN_BYTES = "index_writer_max_memory_in_bytes";
         static final String VERSION_MAP_MEMORY = "version_map_memory";
         static final String VERSION_MAP_MEMORY_IN_BYTES = "version_map_memory_in_bytes";
         static final String FIXED_BIT_SET = "fixed_bit_set";
@@ -366,7 +346,6 @@ public class SegmentsStats implements Streamable, ToXContent {
         docValuesMemoryInBytes = in.readLong();
         indexWriterMemoryInBytes = in.readLong();
         versionMapMemoryInBytes = in.readLong();
-        indexWriterMaxMemoryInBytes = in.readLong();
         bitsetMemoryInBytes = in.readLong();
 
         int size = in.readVInt();
@@ -391,7 +370,6 @@ public class SegmentsStats implements Streamable, ToXContent {
         out.writeLong(docValuesMemoryInBytes);
         out.writeLong(indexWriterMemoryInBytes);
         out.writeLong(versionMapMemoryInBytes);
-        out.writeLong(indexWriterMaxMemoryInBytes);
         out.writeLong(bitsetMemoryInBytes);
 
         out.writeVInt(fileSizes.size());
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
index ac8cefcc5aa..eaf9ad4a081 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
@@ -281,9 +281,6 @@ public class RestIndicesAction extends AbstractCatAction {
         table.addCell("segments.index_writer_memory", "sibling:pri;alias:siwm,segmentsIndexWriterMemory;default:false;text-align:right;desc:memory used by index writer");
         table.addCell("pri.segments.index_writer_memory", "default:false;text-align:right;desc:memory used by index writer");
 
-        table.addCell("segments.index_writer_max_memory", "sibling:pri;alias:siwmx,segmentsIndexWriterMaxMemory;default:false;text-align:right;desc:maximum memory index writer may use before it must write buffered documents to a new segment");
-        table.addCell("pri.segments.index_writer_max_memory", "default:false;text-align:right;desc:maximum memory index writer may use before it must write buffered documents to a new segment");
-
         table.addCell("segments.version_map_memory", "sibling:pri;alias:svmm,segmentsVersionMapMemory;default:false;text-align:right;desc:memory used by version map");
         table.addCell("pri.segments.version_map_memory", "default:false;text-align:right;desc:memory used by version map");
 
@@ -480,9 +477,6 @@ public class RestIndicesAction extends AbstractCatAction {
             table.addCell(indexStats == null ? null : indexStats.getTotal().getSegments().getIndexWriterMemory());
             table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSegments().getIndexWriterMemory());
 
-            table.addCell(indexStats == null ? null : indexStats.getTotal().getSegments().getIndexWriterMaxMemory());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSegments().getIndexWriterMaxMemory());
-
             table.addCell(indexStats == null ? null : indexStats.getTotal().getSegments().getVersionMapMemory());
             table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSegments().getVersionMapMemory());
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
index f23f0e0a029..1069a879d94 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
@@ -204,7 +204,6 @@ public class RestNodesAction extends AbstractCatAction {
         table.addCell("segments.count", "alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments");
         table.addCell("segments.memory", "alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments");
         table.addCell("segments.index_writer_memory", "alias:siwm,segmentsIndexWriterMemory;default:false;text-align:right;desc:memory used by index writer");
-        table.addCell("segments.index_writer_max_memory", "alias:siwmx,segmentsIndexWriterMaxMemory;default:false;text-align:right;desc:maximum memory index writer may use before it must write buffered documents to a new segment");
         table.addCell("segments.version_map_memory", "alias:svmm,segmentsVersionMapMemory;default:false;text-align:right;desc:memory used by version map");
         table.addCell("segments.fixed_bitset_memory", "alias:sfbm,fixedBitsetMemory;default:false;text-align:right;desc:memory used by fixed bit sets for nested object field types and type filters for types referred in _parent fields");
 
@@ -359,7 +358,6 @@ public class RestNodesAction extends AbstractCatAction {
             table.addCell(segmentsStats == null ? null : segmentsStats.getCount());
             table.addCell(segmentsStats == null ? null : segmentsStats.getMemory());
             table.addCell(segmentsStats == null ? null : segmentsStats.getIndexWriterMemory());
-            table.addCell(segmentsStats == null ? null : segmentsStats.getIndexWriterMaxMemory());
             table.addCell(segmentsStats == null ? null : segmentsStats.getVersionMapMemory());
             table.addCell(segmentsStats == null ? null : segmentsStats.getBitsetMemory());
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
index 6538b405fce..0f6bec367d5 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
@@ -156,7 +156,6 @@ public class RestShardsAction extends AbstractCatAction {
         table.addCell("segments.count", "alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments");
         table.addCell("segments.memory", "alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments");
         table.addCell("segments.index_writer_memory", "alias:siwm,segmentsIndexWriterMemory;default:false;text-align:right;desc:memory used by index writer");
-        table.addCell("segments.index_writer_max_memory", "alias:siwmx,segmentsIndexWriterMaxMemory;default:false;text-align:right;desc:maximum memory index writer may use before it must write buffered documents to a new segment");
         table.addCell("segments.version_map_memory", "alias:svmm,segmentsVersionMapMemory;default:false;text-align:right;desc:memory used by version map");
         table.addCell("segments.fixed_bitset_memory", "alias:sfbm,fixedBitsetMemory;default:false;text-align:right;desc:memory used by fixed bit sets for nested object field types and type filters for types referred in _parent fields");
 
@@ -293,7 +292,6 @@ public class RestShardsAction extends AbstractCatAction {
             table.addCell(commonStats == null ? null : commonStats.getSegments().getCount());
             table.addCell(commonStats == null ? null : commonStats.getSegments().getMemory());
             table.addCell(commonStats == null ? null : commonStats.getSegments().getIndexWriterMemory());
-            table.addCell(commonStats == null ? null : commonStats.getSegments().getIndexWriterMaxMemory());
             table.addCell(commonStats == null ? null : commonStats.getSegments().getVersionMapMemory());
             table.addCell(commonStats == null ? null : commonStats.getSegments().getBitsetMemory());
 
diff --git a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
index 66aed327f70..3565cf0147d 100644
--- a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
@@ -544,7 +544,6 @@ public class IndexStatsIT extends ESIntegTestCase {
 
         IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get();
         assertThat(stats.getTotal().getSegments().getIndexWriterMemoryInBytes(), greaterThan(0L));
-        assertThat(stats.getTotal().getSegments().getIndexWriterMaxMemoryInBytes(), greaterThan(0L));
         assertThat(stats.getTotal().getSegments().getVersionMapMemoryInBytes(), greaterThan(0L));
 
         client().admin().indices().prepareFlush().get();
diff --git a/docs/reference/cat/nodes.asciidoc b/docs/reference/cat/nodes.asciidoc
index bd0bc9d5ba2..288a28f3642 100644
--- a/docs/reference/cat/nodes.asciidoc
+++ b/docs/reference/cat/nodes.asciidoc
@@ -201,8 +201,6 @@ operations |9
 segments |1.4kb
 |`segments.index_writer_memory` |`siwm`, `segmentsIndexWriterMemory` |No
 |Memory used by index writer |18mb
-|`segments.index_writer_max_memory` |`siwmx`, `segmentsIndexWriterMaxMemory` |No
-|Maximum memory index writer may use before it must write buffered documents to a new segment |32mb
 |`segments.version_map_memory` |`svmm`, `segmentsVersionMapMemory` |No
 |Memory used by version map |1.0kb
 |=======================================================================
diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc
index 7365aa1af70..4758ea2b0c4 100644
--- a/docs/reference/cluster/stats.asciidoc
+++ b/docs/reference/cluster/stats.asciidoc
@@ -87,8 +87,6 @@ Will return, for example:
          "doc_values_memory_in_bytes": 744,
          "index_writer_memory": "0b",
          "index_writer_memory_in_bytes": 0,
-         "index_writer_max_memory": "2.5gb",
-         "index_writer_max_memory_in_bytes": 2684354560,
          "version_map_memory": "0b",
          "version_map_memory_in_bytes": 0,
          "fixed_bit_set": "0b",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml
index 97ffae1f802..28da3be4511 100755
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yaml
@@ -63,7 +63,6 @@
                     segments.count                   .+   \n
                     segments.memory                  .+   \n
                     segments.index_writer_memory     .+   \n
-                    segments.index_writer_max_memory .+   \n
                     segments.version_map_memory      .+   \n
                     segments.fixed_bitset_memory     .+   \n
                     warmer.current                   .+   \n

From adf47121646974e7e15403b4c406fb230bd9b3c5 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Thu, 26 May 2016 15:26:17 +0200
Subject: [PATCH 0528/1311] Make ip fields backward-compatible at query time.
 #18593

The fact that ip fields used a different doc values representation in 2.x causes
issues when querying 2.x and 5.0 indices in the same request. This changes 2.x
doc values on ip fields/2.x to be hidden behind binary doc values that use the
same encoding as 5.0. This way the coordinating node will be able to merge shard
responses that have different major versions.

One known issue is that this makes sorting/aggregating slower on ip fields for
indices that have been generated with elasticsearch 2.x.
---
 .../index/mapper/ip/LegacyIpFieldMapper.java  |  16 +-
 .../mapper/ip/LegacyIpIndexFieldData.java     | 145 ++++++++++++++++++
 .../elasticsearch/search/DocValueFormat.java  |  10 +-
 .../bucket/range/BinaryRangeAggregator.java   | 104 ++++++++++++-
 .../bwcompat/IpFieldBwCompatIT.java           |  94 ++++++++++++
 .../range/BinaryRangeAggregatorTests.java     |  99 ++++++++++++
 6 files changed, 457 insertions(+), 11 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpIndexFieldData.java
 create mode 100644 core/src/test/java/org/elasticsearch/bwcompat/IpFieldBwCompatIT.java

diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpFieldMapper.java
index 19141a675a3..f1812c93735 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpFieldMapper.java
@@ -35,20 +35,21 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.network.Cidrs;
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.fielddata.IndexFieldData;
-import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
-import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.core.LegacyLongFieldMapper;
 import org.elasticsearch.index.mapper.core.LegacyLongFieldMapper.CustomLongNumericField;
 import org.elasticsearch.index.mapper.core.LegacyNumberFieldMapper;
 import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.search.DocValueFormat;
 import org.joda.time.DateTimeZone;
 
@@ -249,7 +250,14 @@ public class LegacyIpFieldMapper extends LegacyNumberFieldMapper {
         @Override
         public IndexFieldData.Builder fielddataBuilder() {
             failIfNoDocValues();
-            return new DocValuesIndexFieldData.Builder().numericType(NumericType.LONG);
+            return new IndexFieldData.Builder() {
+                @Override
+                public IndexFieldData<?> build(IndexSettings indexSettings,
+                        MappedFieldType fieldType, IndexFieldDataCache cache,
+                        CircuitBreakerService breakerService, MapperService mapperService) {
+                    return new LegacyIpIndexFieldData(indexSettings.getIndex(), name());
+                }
+            };
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpIndexFieldData.java
new file mode 100644
index 00000000000..b94855d25bf
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpIndexFieldData.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.mapper.ip;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+
+import org.apache.lucene.document.InetAddressPoint;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.AtomicFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.ScriptDocValues;
+import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
+import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
+import org.elasticsearch.search.MultiValueMode;
+
+final class LegacyIpIndexFieldData implements IndexFieldData<AtomicFieldData> {
+
+    protected final Index index;
+    protected final String fieldName;
+    protected final ESLogger logger;
+
+    public LegacyIpIndexFieldData(Index index, String fieldName) {
+        this.index = index;
+        this.fieldName = fieldName;
+        this.logger = Loggers.getLogger(getClass());
+    }
+
+    public final String getFieldName() {
+        return fieldName;
+    }
+
+    public final void clear() {
+        // nothing to do
+    }
+
+    public final void clear(IndexReader reader) {
+        // nothing to do
+    }
+
+    public final Index index() {
+        return index;
+    }
+
+    @Override
+    public AtomicFieldData load(LeafReaderContext context) {
+        return new AtomicFieldData() {
+            
+            @Override
+            public void close() {
+                // no-op
+            }
+            
+            @Override
+            public long ramBytesUsed() {
+                return 0;
+            }
+            
+            @Override
+            public ScriptDocValues<?> getScriptValues() {
+                throw new UnsupportedOperationException("Cannot run scripts on ip fields");
+            }
+            
+            @Override
+            public SortedBinaryDocValues getBytesValues() {
+                SortedNumericDocValues values;
+                try {
+                    values = DocValues.getSortedNumeric(context.reader(), fieldName);
+                } catch (IOException e) {
+                    throw new IllegalStateException("Cannot load doc values", e);
+                }
+                return new SortedBinaryDocValues() {
+
+                    final ByteBuffer scratch = ByteBuffer.allocate(4);
+
+                    @Override
+                    public BytesRef valueAt(int index) {
+                        // we do not need to reorder ip addresses since both the numeric
+                        // encoding of LegacyIpFieldMapper and the binary encoding of
+                        // IpFieldMapper match the sort order of ip addresses
+                        long ip = values.valueAt(index);
+                        scratch.putInt(0, (int) ip);
+                        InetAddress inet;
+                        try {
+                            inet = InetAddress.getByAddress(scratch.array());
+                        } catch (UnknownHostException e) {
+                            throw new IllegalStateException("Cannot happen", e);
+                        }
+                        byte[] encoded = InetAddressPoint.encode(inet);
+                        return new BytesRef(encoded);
+                    }
+                    
+                    @Override
+                    public void setDocument(int docId) {
+                        values.setDocument(docId);
+                    }
+                    
+                    @Override
+                    public int count() {
+                        return values.count();
+                    }
+                };
+            }
+        };
+    }
+
+    @Override
+    public AtomicFieldData loadDirect(LeafReaderContext context)
+            throws Exception {
+        return load(context);
+    }
+
+    @Override
+    public IndexFieldData.XFieldComparatorSource comparatorSource(
+            Object missingValue, MultiValueMode sortMode, Nested nested) {
+        return new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/DocValueFormat.java b/core/src/main/java/org/elasticsearch/search/DocValueFormat.java
index c7c993b82b6..9fc36c38e76 100644
--- a/core/src/main/java/org/elasticsearch/search/DocValueFormat.java
+++ b/core/src/main/java/org/elasticsearch/search/DocValueFormat.java
@@ -30,7 +30,6 @@ import org.elasticsearch.common.joda.FormatDateTimeFormatter;
 import org.elasticsearch.common.joda.Joda;
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.network.NetworkAddress;
-import org.elasticsearch.index.mapper.ip.LegacyIpFieldMapper;
 import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
@@ -284,12 +283,12 @@ public interface DocValueFormat extends NamedWriteable {
 
         @Override
         public String format(long value) {
-            return LegacyIpFieldMapper.longToIp(value);
+            throw new UnsupportedOperationException();
         }
 
         @Override
         public String format(double value) {
-            return format((long) value);
+            throw new UnsupportedOperationException();
         }
 
         @Override
@@ -301,13 +300,12 @@ public interface DocValueFormat extends NamedWriteable {
 
         @Override
         public long parseLong(String value, boolean roundUp, Callable<Long> now) {
-            // TODO: throw exception in 6.0
-            return LegacyIpFieldMapper.ipToLong(value);
+            throw new UnsupportedOperationException();
         }
 
         @Override
         public double parseDouble(String value, boolean roundUp, Callable<Long> now) {
-            return parseLong(value, roundUp, now);
+            throw new UnsupportedOperationException();
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java
index 87a3e917ddd..9909d6eca06 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java
@@ -27,6 +27,7 @@ import java.util.Map;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
@@ -104,8 +105,16 @@ public final class BinaryRangeAggregator extends BucketsAggregator {
                     collectBucket(sub, doc, bucket);
                 }
             };
+        } else {
+            SortedBinaryDocValues values = valuesSource.bytesValues(ctx);
+            return new SortedBinaryRangeLeafCollector(values, ranges, sub) {
+                @Override
+                protected void doCollect(LeafBucketCollector sub, int doc, long bucket)
+                        throws IOException {
+                    collectBucket(sub, doc, bucket);
+                }
+            };
         }
-        throw new IllegalArgumentException("binary range aggregation expects a values source that supports ordinals");
     }
 
     static abstract class SortedSetRangeLeafCollector extends LeafBucketCollectorBase {
@@ -214,6 +223,99 @@ public final class BinaryRangeAggregator extends BucketsAggregator {
         protected abstract void doCollect(LeafBucketCollector sub, int doc, long bucket) throws IOException;
     }
 
+    static abstract class SortedBinaryRangeLeafCollector extends LeafBucketCollectorBase {
+
+        final Range[] ranges;
+        final BytesRef[] maxTos;
+        final SortedBinaryDocValues values;
+        final LeafBucketCollector sub;
+
+        SortedBinaryRangeLeafCollector(SortedBinaryDocValues values,
+                Range[] ranges, LeafBucketCollector sub) {
+            super(sub, values);
+            for (int i = 1; i < ranges.length; ++i) {
+                if (RANGE_COMPARATOR.compare(ranges[i-1], ranges[i]) > 0) {
+                    throw new IllegalArgumentException("Ranges must be sorted");
+                }
+            }
+            this.values = values;
+            this.sub = sub;
+            this.ranges = ranges;
+            maxTos = new BytesRef[ranges.length];
+            if (ranges.length > 0) {
+                maxTos[0] = ranges[0].to;
+            }
+            for (int i = 1; i < ranges.length; ++i) {
+                if (compare(ranges[i].to, maxTos[i-1], -1) >= 0) {
+                    maxTos[i] = ranges[i].to;
+                } else {
+                    maxTos[i] = maxTos[i-1];
+                }
+            }
+        }
+
+        @Override
+        public void collect(int doc, long bucket) throws IOException {
+            values.setDocument(doc);
+            final int valuesCount = values.count();
+            for (int i = 0, lo = 0; i < valuesCount; ++i) {
+                final BytesRef value = values.valueAt(i);
+                lo = collect(doc, value, bucket, lo);
+            }
+        }
+
+        private int collect(int doc, BytesRef value, long bucket, int lowBound) throws IOException {
+            int lo = lowBound, hi = ranges.length - 1; // all candidates are between these indexes
+            int mid = (lo + hi) >>> 1;
+            while (lo <= hi) {
+                if (compare(value, ranges[mid].from, 1) < 0) {
+                    hi = mid - 1;
+                } else if (compare(value, maxTos[mid], -1) >= 0) {
+                    lo = mid + 1;
+                } else {
+                    break;
+                }
+                mid = (lo + hi) >>> 1;
+            }
+            if (lo > hi) return lo; // no potential candidate
+
+            // binary search the lower bound
+            int startLo = lo, startHi = mid;
+            while (startLo <= startHi) {
+                final int startMid = (startLo + startHi) >>> 1;
+                if (compare(value, maxTos[startMid], -1) >= 0) {
+                    startLo = startMid + 1;
+                } else {
+                    startHi = startMid - 1;
+                }
+            }
+
+            // binary search the upper bound
+            int endLo = mid, endHi = hi;
+            while (endLo <= endHi) {
+                final int endMid = (endLo + endHi) >>> 1;
+                if (compare(value, ranges[endMid].from, 1) < 0) {
+                    endHi = endMid - 1;
+                } else {
+                    endLo = endMid + 1;
+                }
+            }
+
+            assert startLo == lowBound || compare(value, maxTos[startLo - 1], -1) >= 0;
+            assert endHi == ranges.length - 1 || compare(value, ranges[endHi + 1].from, 1) < 0;
+
+            for (int i = startLo; i <= endHi; ++i) {
+                if (compare(value, ranges[i].to, -1) < 0) {
+                    doCollect(sub, doc, bucket * ranges.length + i);
+                }
+            }
+
+            return endHi + 1;
+        }
+
+        protected abstract void doCollect(LeafBucketCollector sub, int doc, long bucket) throws IOException;
+    }
+
     @Override
     public InternalAggregation buildAggregation(long bucket) throws IOException {
         InternalBinaryRange.Bucket[] buckets = new InternalBinaryRange.Bucket[ranges.length];
diff --git a/core/src/test/java/org/elasticsearch/bwcompat/IpFieldBwCompatIT.java b/core/src/test/java/org/elasticsearch/bwcompat/IpFieldBwCompatIT.java
new file mode 100644
index 00000000000..b2370df172c
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/bwcompat/IpFieldBwCompatIT.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.bwcompat;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+
+import java.util.Collection;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.bucket.range.Range;
+import org.elasticsearch.search.aggregations.bucket.terms.Terms;
+import org.elasticsearch.search.sort.SortBuilders;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
+
+@ESIntegTestCase.SuiteScopeTestCase
+public class IpFieldBwCompatIT extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return pluginList(InternalSettingsPlugin.class); // uses index.merge.enabled
+    }
+
+    @Override
+    public void setupSuiteScopeCluster() throws Exception {
+        assertAcked(prepareCreate("old_index")
+                .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_3.id)
+                .addMapping("type", "ip_field", "type=ip"));
+        assertAcked(prepareCreate("new_index")
+                .addMapping("type", "ip_field", "type=ip"));
+
+        indexRandom(true,
+                client().prepareIndex("old_index", "type", "1").setSource("ip_field", "127.0.0.1"),
+                client().prepareIndex("new_index", "type", "1").setSource("ip_field", "127.0.0.1"),
+                client().prepareIndex("new_index", "type", "2").setSource("ip_field", "::1"));
+    }
+
+    public void testSort() {
+        SearchResponse response = client().prepareSearch("old_index", "new_index")
+                .addSort(SortBuilders.fieldSort("ip_field")).get();
+        assertNoFailures(response);
+        assertEquals(3, response.getHits().totalHits());
+        assertEquals("::1", response.getHits().getAt(0).getSortValues()[0]);
+        assertEquals("127.0.0.1", response.getHits().getAt(1).getSortValues()[0]);
+        assertEquals("127.0.0.1", response.getHits().getAt(2).getSortValues()[0]);
+    }
+
+    public void testRangeAgg() {
+        SearchResponse response = client().prepareSearch("old_index", "new_index")
+                .addAggregation(AggregationBuilders.ipRange("ip_range").field("ip_field")
+                        .addMaskRange("127.0.0.1/16")
+                        .addMaskRange("::1/64")).get();
+        assertNoFailures(response);
+        assertEquals(3, response.getHits().totalHits());
+        Range range = response.getAggregations().get("ip_range");
+        assertEquals(2, range.getBuckets().size());
+        assertEquals("::1/64", range.getBuckets().get(0).getKeyAsString());
+        assertEquals(3, range.getBuckets().get(0).getDocCount());
+        assertEquals("127.0.0.1/16", range.getBuckets().get(1).getKeyAsString());
+        assertEquals(2, range.getBuckets().get(1).getDocCount());
+    }
+
+    public void testTermsAgg() {
+        SearchResponse response = client().prepareSearch("old_index", "new_index")
+                .addAggregation(AggregationBuilders.terms("ip_terms").field("ip_field")).get();
+        assertNoFailures(response);
+        assertEquals(3, response.getHits().totalHits());
+        Terms terms = response.getAggregations().get("ip_terms");
+        assertEquals(2, terms.getBuckets().size());
+        assertEquals(2, terms.getBucketByKey("127.0.0.1").getDocCount());
+        assertEquals(1, terms.getBucketByKey("::1").getDocCount());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java
index b3ad7771bec..6ba6da3b555 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregatorTests.java
@@ -26,7 +26,9 @@ import java.util.Set;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.TestUtil;
+import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
+import org.elasticsearch.search.aggregations.bucket.range.BinaryRangeAggregator.SortedBinaryRangeLeafCollector;
 import org.elasticsearch.search.aggregations.bucket.range.BinaryRangeAggregator.SortedSetRangeLeafCollector;
 import org.elasticsearch.test.ESTestCase;
 
@@ -139,4 +141,101 @@ public class BinaryRangeAggregatorTests extends ESTestCase {
             doTestSortedSetRangeLeafCollector(5);
         }
     }
+
+    private static class FakeSortedBinaryDocValues extends SortedBinaryDocValues {
+
+        private final BytesRef[] terms;
+        long[] ords;
+
+        FakeSortedBinaryDocValues(BytesRef[] terms) {
+            this.terms = terms;
+        }
+
+        @Override
+        public void setDocument(int docID) {
+            // no-op
+        }
+
+        @Override
+        public int count() {
+            return ords.length;
+        }
+
+        @Override
+        public BytesRef valueAt(int index) {
+            return terms[(int) ords[index]];
+        }
+
+    }
+
+    private void doTestSortedBinaryRangeLeafCollector(int maxNumValuesPerDoc) throws Exception {
+        final Set<BytesRef> termSet = new HashSet<>();
+        final int numTerms = TestUtil.nextInt(random(), maxNumValuesPerDoc, 100);
+        while (termSet.size() < numTerms) {
+            termSet.add(new BytesRef(TestUtil.randomSimpleString(random(), randomInt(2))));
+        }
+        final BytesRef[] terms = termSet.toArray(new BytesRef[0]);
+        Arrays.sort(terms);
+
+        final int numRanges = randomIntBetween(1, 10);
+        BinaryRangeAggregator.Range[] ranges = new BinaryRangeAggregator.Range[numRanges];
+        for (int i = 0; i < numRanges; ++i) {
+            ranges[i] = new BinaryRangeAggregator.Range(Integer.toString(i),
+                    randomBoolean() ? null : new BytesRef(TestUtil.randomSimpleString(random(), randomInt(2))),
+                    randomBoolean() ? null : new BytesRef(TestUtil.randomSimpleString(random(), randomInt(2))));
+        }
+        Arrays.sort(ranges, BinaryRangeAggregator.RANGE_COMPARATOR);
+
+        FakeSortedBinaryDocValues values = new FakeSortedBinaryDocValues(terms);
+        final int[] counts = new int[ranges.length];
+        SortedBinaryRangeLeafCollector collector = new SortedBinaryRangeLeafCollector(values, ranges, null) {
+            @Override
+            protected void doCollect(LeafBucketCollector sub, int doc, long bucket) throws IOException {
+                counts[(int) bucket]++;
+            }
+        };
+
+        final int[] expectedCounts = new int[ranges.length];
+        final int maxDoc = randomIntBetween(5, 10);
+        for (int doc = 0; doc < maxDoc; ++doc) {
+            LongHashSet ordinalSet = new LongHashSet();
+            final int numValues = randomInt(maxNumValuesPerDoc);
+            while (ordinalSet.size() < numValues) {
+                ordinalSet.add(random().nextInt(terms.length));
+            }
+            final long[] ords = ordinalSet.toArray();
+            Arrays.sort(ords);
+            values.ords = ords;
+
+            // simulate aggregation
+            collector.collect(doc);
+
+            // now do it the naive way
+            for (int i = 0; i < ranges.length; ++i) {
+                for (long ord : ords) {
+                    BytesRef term = terms[(int) ord];
+                    if ((ranges[i].from == null || ranges[i].from.compareTo(term) <= 0)
+                            && (ranges[i].to == null || ranges[i].to.compareTo(term) > 0)) {
+                        expectedCounts[i]++;
+                        break;
+                    }
+                }
+            }
+        }
+        assertArrayEquals(expectedCounts, counts);
+    }
+
+    public void testSortedBinaryRangeLeafCollectorSingleValued() throws Exception {
+        final int iters = randomInt(10);
+        for (int i = 0; i < iters; ++i) {
+            doTestSortedBinaryRangeLeafCollector(1);
+        }
+    }
+
+    public void testSortedBinaryRangeLeafCollectorMultiValued() throws Exception {
+        final int iters = randomInt(10);
+        for (int i = 0; i < iters; ++i) {
+            doTestSortedBinaryRangeLeafCollector(5);
+        }
+    }
 }

From 2d1eb89aefb21e46ee0be60ba7cff702bf757c64 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 31 May 2016 09:32:33 -0400
Subject: [PATCH 0529/1311] improve date api for expressions/painless fields

---
 .../index/fielddata/ScriptDocValues.java      |   3 +-
 .../elasticsearch/script/ClassPermission.java |   6 +-
 .../modules/scripting/expression.asciidoc     |  40 +++--
 .../script/expression/DateField.java          |  14 ++
 .../script/expression/DateObject.java         | 161 ++++++++++++++++++
 .../expression/DateObjectValueSource.java     |  91 ++++++++++
 .../ExpressionScriptEngineService.java        |  33 +++-
 .../script/expression/GeoField.java           |  16 +-
 .../script/expression/NumericField.java       |  12 ++
 .../expression/MoreExpressionTests.java       |  28 +++
 .../elasticsearch/painless/Definition.java    |   3 +-
 .../org/elasticsearch/painless/Variables.java |   2 +-
 .../org/elasticsearch/painless/java.util.txt  |   2 +
 .../org/elasticsearch/painless/joda.time.txt  |  60 +++++++
 .../painless/org.elasticsearch.txt            |  23 ++-
 .../elasticsearch/painless/OverloadTests.java |  10 +-
 16 files changed, 467 insertions(+), 37 deletions(-)
 create mode 100644 modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObject.java
 create mode 100644 modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java
 create mode 100644 modules/lang-painless/src/main/resources/org/elasticsearch/painless/joda.time.txt

diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
index 9e0d0fcc40d..afaaa9a1df8 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.joda.time.DateTimeZone;
 import org.joda.time.MutableDateTime;
+import org.joda.time.ReadableDateTime;
 
 import java.util.AbstractList;
 import java.util.Collections;
@@ -131,7 +132,7 @@ public interface ScriptDocValues<T> extends List<T> {
             return Collections.unmodifiableList(this);
         }
 
-        public MutableDateTime getDate() {
+        public ReadableDateTime getDate() {
             date.setMillis(getValue());
             return date;
         }
diff --git a/core/src/main/java/org/elasticsearch/script/ClassPermission.java b/core/src/main/java/org/elasticsearch/script/ClassPermission.java
index 0effe85578a..f636a0190c4 100644
--- a/core/src/main/java/org/elasticsearch/script/ClassPermission.java
+++ b/core/src/main/java/org/elasticsearch/script/ClassPermission.java
@@ -69,6 +69,8 @@ import java.util.Set;
  *   <li>{@link org.joda.time.DateTimeUtils}</li>
  *   <li>{@link org.joda.time.DateTimeZone}</li>
  *   <li>{@link org.joda.time.Instant}</li>
+ *   <li>{@link org.joda.time.ReadableDateTime}</li>
+ *   <li>{@link org.joda.time.ReadableInstant}</li>
  * </ul>
  */
 public final class ClassPermission extends BasicPermission {
@@ -102,7 +104,9 @@ public final class ClassPermission extends BasicPermission {
             org.joda.time.DateTime.class.getName(),
             org.joda.time.DateTimeUtils.class.getName(),
             org.joda.time.DateTimeZone.class.getName(),
-            org.joda.time.Instant.class.getName()
+            org.joda.time.Instant.class.getName(),
+            org.joda.time.ReadableDateTime.class.getName(),
+            org.joda.time.ReadableInstant.class.getName()
      )));
 
     /**
diff --git a/docs/reference/modules/scripting/expression.asciidoc b/docs/reference/modules/scripting/expression.asciidoc
index 61e28e05298..fb7739261ef 100644
--- a/docs/reference/modules/scripting/expression.asciidoc
+++ b/docs/reference/modules/scripting/expression.asciidoc
@@ -42,6 +42,8 @@ scripts, simply set the `lang` parameter to `expression`.
 |`doc['field_name'].empty` |A boolean indicating if the field has no
 values within the doc.
 
+|`doc['field_name'].length` |The number of values in this document.
+
 |`doc['field_name'].min()` |The minimum value of the field in this document.
 
 |`doc['field_name'].max()` |The maximum value of the field in this document.
@@ -51,8 +53,6 @@ values within the doc.
 |`doc['field_name'].avg()` |The average of the values in this document.
 
 |`doc['field_name'].sum()` |The sum of the values in this document.
-
-|`doc['field_name'].count()` |The number of values in this document.
 |=======================================================================
 
 When a document is missing the field completely, by default the value will be treated as `0`.
@@ -69,27 +69,47 @@ For example: `doc['on_sale'].value ? doc['price'].value * 0.5 : doc['price'].val
 [float]
 === Date field API
 Date fields are treated as the number of milliseconds since January 1, 1970 and
-support the Numeric Fields API above, with these additional methods:
+support the Numeric Fields API above, plus access to some date-specific fields:
 
 [cols="<,<",options="header",]
 |=======================================================================
 |Expression |Description
-|`doc['field_name'].getYear()` |Year component, e.g. `1970`.
+|`doc['field_name'].date.centuryOfEra`|Century (1-2920000)
 
-|`doc['field_name'].getMonth()` |Month component (0-11), e.g. `0` for January.
+|`doc['field_name'].date.dayOfMonth`|Day (1-31), e.g. `1` for the first of the month.
 
-|`doc['field_name'].getDayOfMonth()` |Day component, e.g. `1` for the first of the month.
+|`doc['field_name'].date.dayOfWeek`|Day of the week (1-7), e.g. `1` for Monday.
 
-|`doc['field_name'].getHourOfDay()` |Hour component (0-23)
+|`doc['field_name'].date.dayOfYear`|Day of the year, e.g. `1` for January 1.
 
-|`doc['field_name'].getMinutes()` |Minutes component (0-59)
+|`doc['field_name'].date.era`|Era: `0` for BC, `1` for AD.
 
-|`doc['field_name'].getSeconds()` |Seconds component (0-59)
+|`doc['field_name'].date.hourOfDay`|Hour (0-23).
+
+|`doc['field_name'].date.millisOfDay`|Milliseconds within the day (0-86399999).
+
+|`doc['field_name'].date.millisOfSecond`|Milliseconds within the second (0-999).
+
+|`doc['field_name'].date.minuteOfDay`|Minute within the day (0-1439).
+
+|`doc['field_name'].date.minuteOfHour`|Minute within the hour (0-59).
+
+|`doc['field_name'].date.monthOfYear`|Month within the year (1-12), e.g. `1` for January.
+
+|`doc['field_name'].date.secondOfDay`|Second within the day (0-86399).
+
+|`doc['field_name'].date.secondOfMinute`|Second within the minute (0-59).
+
+|`doc['field_name'].date.year`|Year (-292000000 - 292000000).
+
+|`doc['field_name'].date.yearOfCentury`|Year within the century (1-100).
+
+|`doc['field_name'].date.yearOfEra`|Year within the era (1-292000000).
 |=======================================================================
 
 The following example shows the difference in years between the `date` fields date0 and date1:
 
-`doc['date1'].getYear() - doc['date0'].getYear()`
+`doc['date1'].date.year - doc['date0'].date.year`
 
 [float]
 === `geo_point` field API
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateField.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateField.java
index e4648887772..5c769774f61 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateField.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateField.java
@@ -35,14 +35,20 @@ final class DateField {
     // supported variables
     static final String VALUE_VARIABLE          = "value";
     static final String EMPTY_VARIABLE          = "empty";
+    static final String LENGTH_VARIABLE         = "length";
     
     // supported methods
+    static final String GETVALUE_METHOD         = "getValue";
+    static final String ISEMPTY_METHOD          = "isEmpty";
+    static final String SIZE_METHOD             = "size";
     static final String MINIMUM_METHOD          = "min";
     static final String MAXIMUM_METHOD          = "max";
     static final String AVERAGE_METHOD          = "avg";
     static final String MEDIAN_METHOD           = "median";
     static final String SUM_METHOD              = "sum";
     static final String COUNT_METHOD            = "count";
+    
+    // date-specific
     static final String GET_YEAR_METHOD         = "getYear";
     static final String GET_MONTH_METHOD        = "getMonth";
     static final String GET_DAY_OF_MONTH_METHOD = "getDayOfMonth";
@@ -56,6 +62,8 @@ final class DateField {
                 return new FieldDataValueSource(fieldData, MultiValueMode.MIN);
             case EMPTY_VARIABLE:
                 return new EmptyMemberValueSource(fieldData);
+            case LENGTH_VARIABLE:
+                return new CountMethodValueSource(fieldData);
             default:
                 throw new IllegalArgumentException("Member variable [" + variable + "] does not exist for date field [" + fieldName + "].");
         }
@@ -63,6 +71,12 @@ final class DateField {
     
     static ValueSource getMethod(IndexFieldData<?> fieldData, String fieldName, String method) {
         switch (method) {
+            case GETVALUE_METHOD:
+                return new FieldDataValueSource(fieldData, MultiValueMode.MIN);
+            case ISEMPTY_METHOD:
+                return new EmptyMemberValueSource(fieldData);
+            case SIZE_METHOD:
+                return new CountMethodValueSource(fieldData);
             case MINIMUM_METHOD:
                 return new FieldDataValueSource(fieldData, MultiValueMode.MIN);
             case MAXIMUM_METHOD:
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObject.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObject.java
new file mode 100644
index 00000000000..66cc56370a3
--- /dev/null
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObject.java
@@ -0,0 +1,161 @@
+package org.elasticsearch.script.expression;
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.lucene.queries.function.ValueSource;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.search.MultiValueMode;
+import org.joda.time.ReadableDateTime;
+
+/**
+ * Expressions API for date objects (.date)
+ */
+final class DateObject {
+    // no instance
+    private DateObject() {}
+    
+    // supported variables
+    static final String CENTURY_OF_ERA_VARIABLE       = "centuryOfEra";
+    static final String DAY_OF_MONTH_VARIABLE         = "dayOfMonth";
+    static final String DAY_OF_WEEK_VARIABLE          = "dayOfWeek";
+    static final String DAY_OF_YEAR_VARIABLE          = "dayOfYear";
+    static final String ERA_VARIABLE                  = "era";
+    static final String HOUR_OF_DAY_VARIABLE          = "hourOfDay";
+    static final String MILLIS_OF_DAY_VARIABLE        = "millisOfDay";
+    static final String MILLIS_OF_SECOND_VARIABLE     = "millisOfSecond";
+    static final String MINUTE_OF_DAY_VARIABLE        = "minuteOfDay";
+    static final String MINUTE_OF_HOUR_VARIABLE       = "minuteOfHour";
+    static final String MONTH_OF_YEAR_VARIABLE        = "monthOfYear";
+    static final String SECOND_OF_DAY_VARIABLE        = "secondOfDay";
+    static final String SECOND_OF_MINUTE_VARIABLE     = "secondOfMinute";
+    static final String WEEK_OF_WEEK_YEAR_VARIABLE    = "weekOfWeekyear";
+    static final String WEEK_YEAR_VARIABLE            = "weekyear";
+    static final String YEAR_VARIABLE                 = "year";
+    static final String YEAR_OF_CENTURY_VARIABLE      = "yearOfCentury";
+    static final String YEAR_OF_ERA_VARIABLE          = "yearOfEra";
+    
+    // supported methods
+    static final String GETCENTURY_OF_ERA_METHOD      = "getCenturyOfEra";
+    static final String GETDAY_OF_MONTH_METHOD        = "getDayOfMonth";
+    static final String GETDAY_OF_WEEK_METHOD         = "getDayOfWeek";
+    static final String GETDAY_OF_YEAR_METHOD         = "getDayOfYear";
+    static final String GETERA_METHOD                 = "getEra";
+    static final String GETHOUR_OF_DAY_METHOD         = "getHourOfDay";
+    static final String GETMILLIS_OF_DAY_METHOD       = "getMillisOfDay";
+    static final String GETMILLIS_OF_SECOND_METHOD    = "getMillisOfSecond";
+    static final String GETMINUTE_OF_DAY_METHOD       = "getMinuteOfDay";
+    static final String GETMINUTE_OF_HOUR_METHOD      = "getMinuteOfHour";
+    static final String GETMONTH_OF_YEAR_METHOD       = "getMonthOfYear";
+    static final String GETSECOND_OF_DAY_METHOD       = "getSecondOfDay";
+    static final String GETSECOND_OF_MINUTE_METHOD    = "getSecondOfMinute";
+    static final String GETWEEK_OF_WEEK_YEAR_METHOD   = "getWeekOfWeekyear";
+    static final String GETWEEK_YEAR_METHOD           = "getWeekyear";
+    static final String GETYEAR_METHOD                = "getYear";
+    static final String GETYEAR_OF_CENTURY_METHOD     = "getYearOfCentury";
+    static final String GETYEAR_OF_ERA_METHOD         = "getYearOfEra";
+    
+    static ValueSource getVariable(IndexFieldData<?> fieldData, String fieldName, String variable) {
+        switch (variable) {
+            case CENTURY_OF_ERA_VARIABLE:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, variable, ReadableDateTime::getCenturyOfEra);
+            case DAY_OF_MONTH_VARIABLE:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, variable, ReadableDateTime::getDayOfMonth);
+            case DAY_OF_WEEK_VARIABLE:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, variable, ReadableDateTime::getDayOfWeek);
+            case DAY_OF_YEAR_VARIABLE:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, variable, ReadableDateTime::getDayOfYear);
+            case ERA_VARIABLE:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, variable, ReadableDateTime::getEra);
+            case HOUR_OF_DAY_VARIABLE:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, variable, ReadableDateTime::getHourOfDay);
+            case MILLIS_OF_DAY_VARIABLE:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, variable, ReadableDateTime::getMillisOfDay);
+            case MILLIS_OF_SECOND_VARIABLE:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, variable, ReadableDateTime::getMillisOfSecond);
+            case MINUTE_OF_DAY_VARIABLE:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, variable, ReadableDateTime::getMinuteOfDay);
+            case MINUTE_OF_HOUR_VARIABLE:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, variable, ReadableDateTime::getMinuteOfHour);
+            case MONTH_OF_YEAR_VARIABLE:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, variable, ReadableDateTime::getMonthOfYear);
+            case SECOND_OF_DAY_VARIABLE:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, variable, ReadableDateTime::getSecondOfDay);
+            case SECOND_OF_MINUTE_VARIABLE:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, variable, ReadableDateTime::getSecondOfMinute);
+            case WEEK_OF_WEEK_YEAR_VARIABLE:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, variable, ReadableDateTime::getWeekOfWeekyear);
+            case WEEK_YEAR_VARIABLE:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, variable, ReadableDateTime::getWeekyear);
+            case YEAR_VARIABLE:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, variable, ReadableDateTime::getYear);
+            case YEAR_OF_CENTURY_VARIABLE:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, variable, ReadableDateTime::getYearOfCentury);
+            case YEAR_OF_ERA_VARIABLE:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, variable, ReadableDateTime::getYearOfEra);
+            default:
+                throw new IllegalArgumentException("Member variable [" + variable + 
+                                                   "] does not exist for date object on field [" + fieldName + "].");
+        }
+    }
+    
+    static ValueSource getMethod(IndexFieldData<?> fieldData, String fieldName, String method) {
+        switch (method) {
+            case GETCENTURY_OF_ERA_METHOD:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, method, ReadableDateTime::getCenturyOfEra);
+            case GETDAY_OF_MONTH_METHOD:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, method, ReadableDateTime::getDayOfMonth);
+            case GETDAY_OF_WEEK_METHOD:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, method, ReadableDateTime::getDayOfWeek);
+            case GETDAY_OF_YEAR_METHOD:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, method, ReadableDateTime::getDayOfYear);
+            case GETERA_METHOD:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, method, ReadableDateTime::getEra);
+            case GETHOUR_OF_DAY_METHOD:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, method, ReadableDateTime::getHourOfDay);
+            case GETMILLIS_OF_DAY_METHOD:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, method, ReadableDateTime::getMillisOfDay);
+            case GETMILLIS_OF_SECOND_METHOD:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, method, ReadableDateTime::getMillisOfSecond);
+            case GETMINUTE_OF_DAY_METHOD:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, method, ReadableDateTime::getMinuteOfDay);
+            case GETMINUTE_OF_HOUR_METHOD:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, method, ReadableDateTime::getMinuteOfHour);
+            case GETMONTH_OF_YEAR_METHOD:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, method, ReadableDateTime::getMonthOfYear);
+            case GETSECOND_OF_DAY_METHOD:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, method, ReadableDateTime::getSecondOfDay);
+            case GETSECOND_OF_MINUTE_METHOD:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, method, ReadableDateTime::getSecondOfMinute);
+            case GETWEEK_OF_WEEK_YEAR_METHOD:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, method, ReadableDateTime::getWeekOfWeekyear);
+            case GETWEEK_YEAR_METHOD:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, method, ReadableDateTime::getWeekyear);
+            case GETYEAR_METHOD:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, method, ReadableDateTime::getYear);
+            case GETYEAR_OF_CENTURY_METHOD:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, method, ReadableDateTime::getYearOfCentury);
+            case GETYEAR_OF_ERA_METHOD:
+                return new DateObjectValueSource(fieldData, MultiValueMode.MIN, method, ReadableDateTime::getYearOfEra);
+            default:
+                throw new IllegalArgumentException("Member method [" + method + 
+                                                   "] does not exist for date object on field [" + fieldName + "].");
+        }
+    }
+}
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java
new file mode 100644
index 00000000000..a9f2018d398
--- /dev/null
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/DateObjectValueSource.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script.expression;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.ToIntFunction;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.docvalues.DoubleDocValues;
+import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.NumericDoubleValues;
+import org.elasticsearch.search.MultiValueMode;
+import org.joda.time.DateTimeZone;
+import org.joda.time.MutableDateTime;
+import org.joda.time.ReadableDateTime;
+
+/** Extracts a portion of a date field with joda time */
+class DateObjectValueSource extends FieldDataValueSource {
+
+    final String methodName;
+    final ToIntFunction<ReadableDateTime> function;
+
+    DateObjectValueSource(IndexFieldData<?> indexFieldData, MultiValueMode multiValueMode, 
+                          String methodName, ToIntFunction<ReadableDateTime> function) {
+        super(indexFieldData, multiValueMode);
+
+        Objects.requireNonNull(methodName);
+
+        this.methodName = methodName;
+        this.function = function;
+    }
+
+    @Override
+    @SuppressWarnings("rawtypes") // ValueSource uses a rawtype
+    public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException {
+        AtomicNumericFieldData leafData = (AtomicNumericFieldData) fieldData.load(leaf);
+        MutableDateTime joda = new MutableDateTime(0, DateTimeZone.UTC);
+        NumericDoubleValues docValues = multiValueMode.select(leafData.getDoubleValues(), 0d);
+        return new DoubleDocValues(this) {
+          @Override
+          public double doubleVal(int docId) {
+            long millis = (long)docValues.get(docId);
+            joda.setMillis(millis);
+            return function.applyAsInt(joda);
+          }
+        };
+    }
+
+    @Override
+    public String description() {
+        return methodName + ": field(" + fieldData.getFieldName() + ")";
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        if (!super.equals(o)) return false;
+
+        DateObjectValueSource that = (DateObjectValueSource) o;
+        return methodName.equals(that.methodName);
+    }
+
+    @Override
+    public int hashCode() {
+        int result = super.hashCode();
+        result = 31 * result + methodName.hashCode();
+        return result;
+    }
+}
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
index 5c4ff3ca94d..dce8d8a7800 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
@@ -51,6 +51,7 @@ import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.text.ParseException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
@@ -148,6 +149,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                     String fieldname = null;
                     String methodname = null;
                     String variablename = "value"; // .value is the default for doc['field'], its optional.
+                    boolean dateAccessor = false; // true if the variable is of type doc['field'].date.xxx
                     VariableContext[] parts = VariableContext.parse(variable);
                     if (parts[0].text.equals("doc") == false) {
                         throw new ParseException("Unknown variable [" + parts[0].text + "]", 0);
@@ -167,7 +169,19 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                         }
                     }
                     if (parts.length > 3) {
-                        throw new IllegalArgumentException("Variable [" + variable + "] does not follow an allowed format of either doc['field'] or doc['field'].method()");
+                        // access to the .date "object" within the field
+                        if (parts.length == 4 && ("date".equals(parts[2].text) || "getDate".equals(parts[2].text))) {
+                            if (parts[3].type == VariableContext.Type.METHOD) {
+                                methodname = parts[3].text;
+                                dateAccessor = true;
+                            } else if (parts[3].type == VariableContext.Type.MEMBER) {
+                                variablename = parts[3].text;
+                                dateAccessor = true;
+                            }
+                        }
+                        if (!dateAccessor) {
+                            throw new IllegalArgumentException("Variable [" + variable + "] does not follow an allowed format of either doc['field'] or doc['field'].method()");
+                        }
                     }
 
                     MappedFieldType fieldType = mapper.fullName(fieldname);
@@ -191,11 +205,20 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                         }
                     } else if (fieldType instanceof LegacyDateFieldMapper.DateFieldType || 
                             fieldType instanceof DateFieldMapper.DateFieldType) {
-                        // date
-                        if (methodname == null) {
-                            valueSource = DateField.getVariable(fieldData, fieldname, variablename);
+                        if (dateAccessor) {
+                            // date object
+                            if (methodname == null) {
+                                valueSource = DateObject.getVariable(fieldData, fieldname, variablename);
+                            } else {
+                                valueSource = DateObject.getMethod(fieldData, fieldname, methodname);
+                            }
                         } else {
-                            valueSource = DateField.getMethod(fieldData, fieldname, methodname);
+                            // date field itself
+                            if (methodname == null) {
+                                valueSource = DateField.getVariable(fieldData, fieldname, variablename);
+                            } else {
+                                valueSource = DateField.getMethod(fieldData, fieldname, methodname);
+                            }
                         }
                     } else if (fieldData instanceof IndexNumericFieldData) {
                         // number
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoField.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoField.java
index e830813e3a3..0ce74a187df 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoField.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoField.java
@@ -34,6 +34,11 @@ final class GeoField {
     static final String LAT_VARIABLE          = "lat";
     static final String LON_VARIABLE          = "lon";
     
+    // supported methods
+    static final String ISEMPTY_METHOD        = "isEmpty";
+    static final String GETLAT_METHOD         = "getLat";
+    static final String GETLON_METHOD         = "getLon";
+    
     static ValueSource getVariable(IndexFieldData<?> fieldData, String fieldName, String variable) {
         switch (variable) {
             case EMPTY_VARIABLE:
@@ -48,6 +53,15 @@ final class GeoField {
     }
     
     static ValueSource getMethod(IndexFieldData<?> fieldData, String fieldName, String method) {
-        throw new IllegalArgumentException("Member method [" + method + "] does not exist for geo field [" + fieldName + "].");
+        switch (method) {
+            case ISEMPTY_METHOD:
+                return new GeoEmptyValueSource(fieldData);
+            case GETLAT_METHOD:
+                return new GeoLatitudeValueSource(fieldData);
+            case GETLON_METHOD:
+                return new GeoLongitudeValueSource(fieldData);
+            default:
+                throw new IllegalArgumentException("Member method [" + method + "] does not exist for geo field [" + fieldName + "].");
+        }
     }
 }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/NumericField.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/NumericField.java
index 4147d0e97d9..7cd918cf914 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/NumericField.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/NumericField.java
@@ -33,8 +33,12 @@ final class NumericField {
     // supported variables
     static final String VALUE_VARIABLE          = "value";
     static final String EMPTY_VARIABLE          = "empty";
+    static final String LENGTH_VARIABLE         = "length";
     
     // supported methods
+    static final String GETVALUE_METHOD         = "getValue";
+    static final String ISEMPTY_METHOD          = "isEmpty";
+    static final String SIZE_METHOD             = "size";
     static final String MINIMUM_METHOD          = "min";
     static final String MAXIMUM_METHOD          = "max";
     static final String AVERAGE_METHOD          = "avg";
@@ -48,6 +52,8 @@ final class NumericField {
                 return new FieldDataValueSource(fieldData, MultiValueMode.MIN);
             case EMPTY_VARIABLE:
                 return new EmptyMemberValueSource(fieldData);
+            case LENGTH_VARIABLE:
+                return new CountMethodValueSource(fieldData);
             default:
                 throw new IllegalArgumentException("Member variable [" + variable + "] does not exist for " + 
                                                    "numeric field [" + fieldName + "].");
@@ -56,6 +62,12 @@ final class NumericField {
     
     static ValueSource getMethod(IndexFieldData<?> fieldData, String fieldName, String method) {
         switch (method) {
+            case GETVALUE_METHOD:
+                return new FieldDataValueSource(fieldData, MultiValueMode.MIN);
+            case ISEMPTY_METHOD:
+                return new EmptyMemberValueSource(fieldData);
+            case SIZE_METHOD:
+                return new CountMethodValueSource(fieldData);
             case MINIMUM_METHOD:
                 return new FieldDataValueSource(fieldData, MultiValueMode.MIN);
             case MAXIMUM_METHOD:
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
index 3f2466e52e9..da2f117c78f 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
@@ -169,6 +169,34 @@ public class MoreExpressionTests extends ESIntegTestCase {
         assertEquals(1985.0, hits.getAt(0).field("foo").getValue(), 0.0D);
         assertEquals(1983.0, hits.getAt(1).field("foo").getValue(), 0.0D);
     }
+    
+    public void testDateObjectMethods() throws Exception {
+        ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "date0", "type=date", "date1", "type=date"));
+        ensureGreen("test");
+        indexRandom(true,
+                client().prepareIndex("test", "doc", "1").setSource("date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"),
+                client().prepareIndex("test", "doc", "2").setSource("date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z"));
+        SearchResponse rsp = buildRequest("doc['date0'].date.secondOfMinute - doc['date0'].date.minuteOfHour").get();
+        assertEquals(2, rsp.getHits().getTotalHits());
+        SearchHits hits = rsp.getHits();
+        assertEquals(5.0, hits.getAt(0).field("foo").getValue(), 0.0D);
+        assertEquals(-11.0, hits.getAt(1).field("foo").getValue(), 0.0D);
+        rsp = buildRequest("doc['date0'].date.getHourOfDay() + doc['date1'].date.dayOfMonth").get();
+        assertEquals(2, rsp.getHits().getTotalHits());
+        hits = rsp.getHits();
+        assertEquals(5.0, hits.getAt(0).field("foo").getValue(), 0.0D);
+        assertEquals(24.0, hits.getAt(1).field("foo").getValue(), 0.0D);
+        rsp = buildRequest("doc['date1'].date.monthOfYear + 1").get();
+        assertEquals(2, rsp.getHits().getTotalHits());
+        hits = rsp.getHits();
+        assertEquals(10.0, hits.getAt(0).field("foo").getValue(), 0.0D);
+        assertEquals(11.0, hits.getAt(1).field("foo").getValue(), 0.0D);
+        rsp = buildRequest("doc['date1'].date.year").get();
+        assertEquals(2, rsp.getHits().getTotalHits());
+        hits = rsp.getHits();
+        assertEquals(1985.0, hits.getAt(0).field("foo").getValue(), 0.0D);
+        assertEquals(1983.0, hits.getAt(1).field("foo").getValue(), 0.0D);
+    }
 
     public void testMultiValueMethods() throws Exception {
         ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "double0", "type=double", "double1", "type=double", "double2", "type=double"));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 0923cffc59d..669323da803 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -50,7 +50,8 @@ public final class Definition {
                       "java.time.zone.txt",
                       "java.util.txt",
                       "java.util.function.txt",
-                      "java.util.stream.txt"));
+                      "java.util.stream.txt",
+                      "joda.time.txt"));
 
     private static final Definition INSTANCE = new Definition();
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
index 4443b1dc0f8..f50ab29096f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
@@ -99,7 +99,7 @@ public final class Variables {
         // Method variables.
 
         // This reference.  Internal use only.
-        addVariable("[" + Reserved.THIS + "]", Definition.getType("Executable"), Reserved.THIS, true, true);
+        addVariable("[" + Reserved.THIS + "]", Definition.getType("Object"), Reserved.THIS, true, true);
 
         // Input map of variables passed to the script.
         addVariable("[" + Reserved.PARAMS + "]", Definition.getType("Map"), Reserved.PARAMS, true, true);
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt
index ffa80da79c1..5e240d0d45a 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt
@@ -643,9 +643,11 @@ class GregorianCalendar -> java.util.GregorianCalendar extends Calendar,Comparab
   GregorianCalendar <init>(int,int,int,int,int,int)
   GregorianCalendar <init>(TimeZone)
   GregorianCalendar <init>(TimeZone,Locale)
+  GregorianCalendar from(ZonedDateTime)
   Date getGregorianChange()
   boolean isLeapYear(int)
   void setGregorianChange(Date)
+  ZonedDateTime toZonedDateTime()
 }
 
 class HashMap -> java.util.HashMap extends AbstractMap,Map,Object {
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/joda.time.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/joda.time.txt
new file mode 100644
index 00000000000..02d95921534
--- /dev/null
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/joda.time.txt
@@ -0,0 +1,60 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#
+# Painless definition file. This defines the hierarchy of classes,
+# what methods and fields they have, etc.
+#
+
+# NOTE: this just minimal whitelisting of joda time, just to provide
+# convenient access via the scripting API. classes are fully qualified to avoid
+# any confusion with java.time
+
+class org.joda.time.ReadableInstant -> org.joda.time.ReadableInstant extends Comparable {
+  boolean equals(Object)
+  long getMillis()
+  int hashCode()
+  boolean isAfter(org.joda.time.ReadableInstant)
+  boolean isBefore(org.joda.time.ReadableInstant)
+  boolean isEqual(org.joda.time.ReadableInstant)
+  String toString()
+}
+
+class org.joda.time.ReadableDateTime -> org.joda.time.ReadableDateTime extends org.joda.time.ReadableInstant,Comparable {
+  int getCenturyOfEra()
+  int getDayOfMonth()
+  int getDayOfWeek()
+  int getDayOfYear()
+  int getEra()
+  int getHourOfDay()
+  int getMillisOfDay()
+  int getMillisOfSecond()
+  int getMinuteOfDay()
+  int getMinuteOfHour()
+  int getMonthOfYear()
+  int getSecondOfDay()
+  int getSecondOfMinute()
+  int getWeekOfWeekyear()
+  int getWeekyear()
+  int getYear()
+  int getYearOfCentury()
+  int getYearOfEra()
+  String toString(String)
+  String toString(String,Locale)
+}
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt
index a4af7c318ec..8627de26d26 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt
@@ -59,28 +59,29 @@ class def -> java.lang.Object {
 
 #### ES Scripting API
 
-class GeoPoint -> org.elasticsearch.common.geo.GeoPoint extends Object {
+class org.elasticsearch.common.geo.GeoPoint -> org.elasticsearch.common.geo.GeoPoint extends Object {
   double getLat()
   double getLon()
 }
 
-class Strings -> org.elasticsearch.index.fielddata.ScriptDocValues$Strings extends List,Collection,Iterable,Object {
+class org.elasticsearch.index.fielddata.ScriptDocValues.Strings -> org.elasticsearch.index.fielddata.ScriptDocValues$Strings extends List,Collection,Iterable,Object {
   String getValue()
   List getValues()
 }
 
-class Longs -> org.elasticsearch.index.fielddata.ScriptDocValues$Longs extends List,Collection,Iterable,Object {
+class org.elasticsearch.index.fielddata.ScriptDocValues.Longs -> org.elasticsearch.index.fielddata.ScriptDocValues$Longs extends List,Collection,Iterable,Object {
   long getValue()
   List getValues()
+  org.joda.time.ReadableDateTime getDate()
 }
 
-class Doubles -> org.elasticsearch.index.fielddata.ScriptDocValues$Doubles extends List,Collection,Iterable,Object {
+class org.elasticsearch.index.fielddata.ScriptDocValues.Doubles -> org.elasticsearch.index.fielddata.ScriptDocValues$Doubles extends List,Collection,Iterable,Object {
   double getValue()
   List getValues()
 }
 
-class GeoPoints -> org.elasticsearch.index.fielddata.ScriptDocValues$GeoPoints extends List,Collection,Iterable,Object {
-  GeoPoint getValue()
+class org.elasticsearch.index.fielddata.ScriptDocValues.GeoPoints -> org.elasticsearch.index.fielddata.ScriptDocValues$GeoPoints extends List,Collection,Iterable,Object {
+  org.elasticsearch.common.geo.GeoPoint getValue()
   List getValues()
   double getLat()
   double getLon()
@@ -111,9 +112,9 @@ class GeoPoints -> org.elasticsearch.index.fielddata.ScriptDocValues$GeoPoints e
 
 # for testing.
 # currently FeatureTest exposes overloaded constructor, field load store, and overloaded static methods
-class FeatureTest -> org.elasticsearch.painless.FeatureTest extends Object {
-  FeatureTest <init>()
-  FeatureTest <init>(int,int)
+class org.elasticsearch.painless.FeatureTest -> org.elasticsearch.painless.FeatureTest extends Object {
+  org.elasticsearch.painless.FeatureTest <init>()
+  org.elasticsearch.painless.FeatureTest <init>(int,int)
   int getX()
   int getY()
   void setX(int)
@@ -121,7 +122,3 @@ class FeatureTest -> org.elasticsearch.painless.FeatureTest extends Object {
   boolean overloadedStatic()
   boolean overloadedStatic(boolean)
 }
-
-# currently needed internally
-class Executable -> org.elasticsearch.painless.Executable {
-}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java
index 6662aa95fbd..00c86836a67 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java
@@ -41,12 +41,14 @@ public class OverloadTests extends ScriptTestCase {
     }
     
     public void testConstructor() {
-        assertEquals(true, exec("FeatureTest f = new FeatureTest(); return f.x == 0 && f.y == 0;"));
-        assertEquals(true, exec("FeatureTest f = new FeatureTest(1, 2); return f.x == 1 && f.y == 2;"));
+        assertEquals(true, exec("org.elasticsearch.painless.FeatureTest f = new org.elasticsearch.painless.FeatureTest();" +
+                                "return f.x == 0 && f.y == 0;"));
+        assertEquals(true, exec("org.elasticsearch.painless.FeatureTest f = new org.elasticsearch.painless.FeatureTest(1, 2);" +
+                                "return f.x == 1 && f.y == 2;"));
     }
     
     public void testStatic() {
-        assertEquals(true, exec("return FeatureTest.overloadedStatic();"));
-        assertEquals(false, exec("return FeatureTest.overloadedStatic(false);"));
+        assertEquals(true, exec("return org.elasticsearch.painless.FeatureTest.overloadedStatic();"));
+        assertEquals(false, exec("return org.elasticsearch.painless.FeatureTest.overloadedStatic(false);"));
     }
 }

From de7688ae5d0ee226c498fa91cdf2b99670c5cd7f Mon Sep 17 00:00:00 2001
From: Christoph Wurm <cwurm@users.noreply.github.com>
Date: Tue, 31 May 2016 15:43:16 +0200
Subject: [PATCH 0530/1311] Update configuration.asciidoc

Correct configuration of `path.conf`
---
 docs/reference/setup/configuration.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/setup/configuration.asciidoc b/docs/reference/setup/configuration.asciidoc
index 68f73fc96b8..52bf5ffcbff 100644
--- a/docs/reference/setup/configuration.asciidoc
+++ b/docs/reference/setup/configuration.asciidoc
@@ -26,7 +26,7 @@ setting, as follows:
 
 [source,sh]
 -------------------------------
-./bin/elasticsearch -Epath.conf=/path/to/my/config/
+./bin/elasticsearch -Ees.path.conf=/path/to/my/config/
 -------------------------------
 
 [float]

From 740bdc8d99c6235392de4d19bdc4a7d722516962 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Tue, 31 May 2016 14:22:26 +0200
Subject: [PATCH 0531/1311] Treat zero token in `common` terms query as
 MatchNoDocsQuery

Currently we return `null` when the query in a common terms query has
zero tokens after analysis. It would be better if query builders
`toQuery()` would never return null and return a meaningful lucene
query instead. Since an ExtendedCommonTermsQuery with no terms gets
rewritten to a MatchNoDocsQuery later, it is enough to leave out the
check for zero tokens.
---
 .../index/query/CommonTermsQueryBuilder.java    |  5 +----
 .../query/CommonTermsQueryBuilderTests.java     | 17 +++++++----------
 2 files changed, 8 insertions(+), 14 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
index 6279187dfd2..f7b35d8dfa2 100644
--- a/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
@@ -401,7 +401,7 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQue
         return parseQueryString(commonsQuery, text, field, analyzerObj, lowFreqMinimumShouldMatch, highFreqMinimumShouldMatch);
     }
 
-    static Query parseQueryString(ExtendedCommonTermsQuery query, Object queryString, String field, Analyzer analyzer,
+    private static Query parseQueryString(ExtendedCommonTermsQuery query, Object queryString, String field, Analyzer analyzer,
                                          String lowFreqMinimumShouldMatch, String highFreqMinimumShouldMatch) throws IOException {
         // Logic similar to QueryParser#getFieldQuery
         int count = 0;
@@ -417,9 +417,6 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQue
             }
         }
 
-        if (count == 0) {
-            return null;
-        }
         query.setLowFreqMinimumNumberShouldMatch(lowFreqMinimumShouldMatch);
         query.setHighFreqMinimumNumberShouldMatch(highFreqMinimumShouldMatch);
         return query;
diff --git a/core/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java
index 0b293cd0157..dd3eabe27fd 100644
--- a/core/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/CommonTermsQueryBuilderTests.java
@@ -37,12 +37,16 @@ public class CommonTermsQueryBuilderTests extends AbstractQueryTestCase<CommonTe
     protected CommonTermsQueryBuilder doCreateTestQueryBuilder() {
         CommonTermsQueryBuilder query;
 
+        int numberOfTerms = randomIntBetween(0, 10);
+        StringBuilder text = new StringBuilder("");
+        for (int i = 0; i < numberOfTerms; i++) {
+            text.append(randomAsciiOfLengthBetween(1, 10)).append(" ");
+        }
         // mapped or unmapped field
-        String text = randomAsciiOfLengthBetween(1, 10);
         if (randomBoolean()) {
-            query = new CommonTermsQueryBuilder(STRING_FIELD_NAME, text);
+            query = new CommonTermsQueryBuilder(STRING_FIELD_NAME, text.toString());
         } else {
-            query = new CommonTermsQueryBuilder(randomAsciiOfLengthBetween(1, 10), text);
+            query = new CommonTermsQueryBuilder(randomAsciiOfLengthBetween(1, 10), text.toString());
         }
 
         if (randomBoolean()) {
@@ -134,13 +138,6 @@ public class CommonTermsQueryBuilderTests extends AbstractQueryTestCase<CommonTe
         assertEquals(query, "nelly the elephant not as a cartoon", queryBuilder.value());
     }
 
-    public void testNoTermsFromQueryString() throws IOException {
-        CommonTermsQueryBuilder builder = new CommonTermsQueryBuilder(STRING_FIELD_NAME, "");
-        QueryShardContext context = createShardContext();
-        context.setAllowUnmappedFields(true);
-        assertNull(builder.toQuery(context));
-    }
-
     public void testCommonTermsQuery1() throws IOException {
         String query = copyToStringFromClasspath("/org/elasticsearch/index/query/commonTerms-query1.json");
         Query parsedQuery = parseQuery(query).toQuery(createShardContext());

From 82645563bf075074268b6aaa0b6ac29a337229d2 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Tue, 31 May 2016 16:33:38 +0200
Subject: [PATCH 0532/1311] Estimate shard size for shrinked indices (#18659)

When we shrink an index we can estimate the shards size for the primary
from the source index. This is important for allocation decisions since we
should try out best to ensure we have enough space on the node we shrink the
index.
---
 .../allocator/BalancedShardsAllocator.java    |   5 +-
 .../decider/DiskThresholdDecider.java         |  39 +++++--
 .../admin/indices/create/CreateIndexIT.java   |  15 ++-
 .../DiskThresholdDeciderUnitTests.java        | 103 +++++++++++++++---
 4 files changed, 135 insertions(+), 27 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
index b1d78fa44d0..1f69b771fdb 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
@@ -32,6 +32,7 @@ import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision.Type;
+import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
@@ -721,11 +722,13 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
                     }
                     assert decision != null && minNode != null || decision == null && minNode == null;
                     if (minNode != null) {
-                        long shardSize = allocation.clusterInfo().getShardSize(shard, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
+                        final long shardSize = DiskThresholdDecider.getExpectedShardSize(shard, allocation,
+                            ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
                         if (decision.type() == Type.YES) {
                             if (logger.isTraceEnabled()) {
                                 logger.trace("Assigned shard [{}] to [{}]", shard, minNode.getNodeId());
                             }
+
                             shard = routingNodes.initialize(shard, minNode.getNodeId(), null, shardSize);
                             minNode.addShard(shard);
                             changed = true;
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
index 4d56711c097..f64791d834a 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
@@ -29,6 +29,7 @@ import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.DiskUsage;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
@@ -44,6 +45,7 @@ import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.RatioValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.index.Index;
 
 import java.util.Set;
 
@@ -303,26 +305,23 @@ public class DiskThresholdDecider extends AllocationDecider {
      * If subtractShardsMovingAway is set then the size of shards moving away is subtracted from the total size
      * of all shards
      */
-    public static long sizeOfRelocatingShards(RoutingNode node, ClusterInfo clusterInfo,
+    public static long sizeOfRelocatingShards(RoutingNode node, RoutingAllocation allocation,
                                               boolean subtractShardsMovingAway, String dataPath) {
+        ClusterInfo clusterInfo = allocation.clusterInfo();
         long totalSize = 0;
         for (ShardRouting routing : node.shardsWithState(ShardRoutingState.RELOCATING, ShardRoutingState.INITIALIZING)) {
             String actualPath = clusterInfo.getDataPath(routing);
             if (dataPath.equals(actualPath)) {
                 if (routing.initializing() && routing.relocatingNodeId() != null) {
-                    totalSize += getShardSize(routing, clusterInfo);
+                    totalSize += getExpectedShardSize(routing, allocation, 0);
                 } else if (subtractShardsMovingAway && routing.relocating()) {
-                    totalSize -= getShardSize(routing, clusterInfo);
+                    totalSize -= getExpectedShardSize(routing, allocation, 0);
                 }
             }
         }
         return totalSize;
     }
 
-    static long getShardSize(ShardRouting routing, ClusterInfo clusterInfo) {
-        Long shardSize = clusterInfo.getShardSize(routing);
-        return shardSize == null ? 0 : shardSize;
-    }
 
     @Override
     public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
@@ -426,7 +425,7 @@ public class DiskThresholdDecider extends AllocationDecider {
         }
 
         // Secondly, check that allocating the shard to this node doesn't put it above the high watermark
-        final long shardSize = getShardSize(shardRouting, allocation.clusterInfo());
+        final long shardSize = getExpectedShardSize(shardRouting, allocation, 0);
         double freeSpaceAfterShard = freeDiskPercentageAfterShardAssigned(usage, shardSize);
         long freeBytesAfterShard = freeBytes - shardSize;
         if (freeBytesAfterShard < freeBytesThresholdHigh.bytes()) {
@@ -518,7 +517,7 @@ public class DiskThresholdDecider extends AllocationDecider {
         }
 
         if (includeRelocations) {
-            long relocatingShardsSize = sizeOfRelocatingShards(node, clusterInfo, true, usage.getPath());
+            long relocatingShardsSize = sizeOfRelocatingShards(node, allocation, true, usage.getPath());
             DiskUsage usageIncludingRelocations = new DiskUsage(node.nodeId(), node.node().getName(), usage.getPath(),
                     usage.getTotalBytes(), usage.getFreeBytes() - relocatingShardsSize);
             if (logger.isTraceEnabled()) {
@@ -643,4 +642,26 @@ public class DiskThresholdDecider extends AllocationDecider {
         }
         return null;
     }
+
+    /**
+     * Returns the expected shard size for the given shard or the default value provided if not enough information are available
+     * to estimate the shards size.
+     */
+    public static final long getExpectedShardSize(ShardRouting shard, RoutingAllocation allocation, long defaultValue) {
+        final IndexMetaData metaData = allocation.metaData().getIndexSafe(shard.index());
+        final ClusterInfo info = allocation.clusterInfo();
+        if (metaData.getMergeSourceIndex() != null && shard.allocatedPostIndexCreate(metaData) == false) {
+            // in the shrink index case we sum up the source index shards since we basically make a copy of the shard in
+            // the worst case
+            Index mergeSourceIndex = metaData.getMergeSourceIndex();
+            long targetShardSize = 0;
+            for (IndexShardRoutingTable shardRoutingTable : allocation.routingTable().index(mergeSourceIndex.getName())) {
+                targetShardSize += info.getShardSize(shardRoutingTable.primaryShard(), 0);
+            }
+            return targetShardSize == 0 ? defaultValue : targetShardSize;
+        } else {
+            return info.getShardSize(shard, defaultValue);
+        }
+
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
index 8ee38c86dc8..af7d7b94ada 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
@@ -21,11 +21,14 @@ package org.elasticsearch.action.admin.indices.create;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.UnavailableShardsException;
+import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.InternalClusterInfoService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -346,6 +349,7 @@ public class CreateIndexIT extends ESIntegTestCase {
             .setSettings(Settings.builder().put("index.routing.allocation.require._name", mergeNode)
                 .put("index.blocks.write", true)).get();
         ensureGreen();
+
         // now merge source into a single shard index
         client().admin().indices().prepareShrinkIndex("source", "target")
             .setSettings(Settings.builder()
@@ -375,7 +379,16 @@ public class CreateIndexIT extends ESIntegTestCase {
             .setSettings(Settings.builder()
                 .put("index.routing.allocation.require._name", mergeNode)).get();
         ensureGreen("source");
-        client().admin().cluster().prepareReroute().setRetryFailed(true).get(); // kick off a retry and wait until it's done!
+
+        final InternalClusterInfoService infoService = (InternalClusterInfoService) internalCluster().getInstance(ClusterInfoService.class,
+            internalCluster().getMasterName());
+        infoService.refresh();
+        // kick off a retry and wait until it's done!
+        ClusterRerouteResponse clusterRerouteResponse = client().admin().cluster().prepareReroute().setRetryFailed(true).get();
+        long expectedShardSize = clusterRerouteResponse.getState().routingTable().index("target")
+            .shard(0).getShards().get(0).getExpectedShardSize();
+        // we support the expected shard size in the allocator to sum up over the source index shards
+        assertTrue("expected shard size must be set but wasn't: " + expectedShardSize, expectedShardSize > 0);
         ensureGreen();
         assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
     }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
index 008884cbb8d..52ba45ab7ab 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
@@ -34,7 +34,9 @@ import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingHelper;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -44,7 +46,7 @@ import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ESAllocationTestCase;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -57,7 +59,7 @@ import static org.hamcrest.CoreMatchers.equalTo;
 /**
  * Unit tests for the DiskThresholdDecider
  */
-public class DiskThresholdDeciderUnitTests extends ESTestCase {
+public class DiskThresholdDeciderUnitTests extends ESAllocationTestCase {
     public void testDynamicSettings() {
         ClusterSettings nss = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
 
@@ -233,7 +235,18 @@ public class DiskThresholdDeciderUnitTests extends ESTestCase {
         shardSizes.put("[test][2][r]", 1000L);
         shardSizes.put("[other][0][p]", 10000L);
         ClusterInfo info = new DevNullClusterInfo(ImmutableOpenMap.of(), ImmutableOpenMap.of(), shardSizes.build());
-        final Index index = new Index("test", "_na_");
+        MetaData.Builder metaBuilder = MetaData.builder();
+        metaBuilder.put(IndexMetaData.builder("test").settings(settings(Version.CURRENT).put("index.uuid", "1234")).numberOfShards(3).numberOfReplicas(1));
+        metaBuilder.put(IndexMetaData.builder("other").settings(settings(Version.CURRENT).put("index.uuid", "5678")).numberOfShards(1).numberOfReplicas(1));
+        MetaData metaData = metaBuilder.build();
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        routingTableBuilder.addAsNew(metaData.index("test"));
+        routingTableBuilder.addAsNew(metaData.index("other"));
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+            .metaData(metaData).routingTable(routingTableBuilder.build()).build();
+        RoutingAllocation allocation = new RoutingAllocation(null, null, clusterState, info, 0, false);
+
+        final Index index = new Index("test", "1234");
         ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         test_0 = ShardRoutingHelper.initialize(test_0, "node1");
         test_0 = ShardRoutingHelper.moveToStarted(test_0);
@@ -248,24 +261,24 @@ public class DiskThresholdDeciderUnitTests extends ESTestCase {
         test_2 = ShardRoutingHelper.initialize(test_2, "node1");
         test_2 = ShardRoutingHelper.moveToStarted(test_2);
 
-        assertEquals(1000L, DiskThresholdDecider.getShardSize(test_2, info));
-        assertEquals(100L, DiskThresholdDecider.getShardSize(test_1, info));
-        assertEquals(10L, DiskThresholdDecider.getShardSize(test_0, info));
+        assertEquals(1000L, DiskThresholdDecider.getExpectedShardSize(test_2, allocation, 0));
+        assertEquals(100L, DiskThresholdDecider.getExpectedShardSize(test_1, allocation, 0));
+        assertEquals(10L, DiskThresholdDecider.getExpectedShardSize(test_0, allocation, 0));
 
         RoutingNode node = new RoutingNode("node1", new DiscoveryNode("node1", new LocalTransportAddress("test"),
                 emptyMap(), emptySet(), Version.CURRENT), test_0, test_1.buildTargetRelocatingShard(), test_2);
-        assertEquals(100L, DiskThresholdDecider.sizeOfRelocatingShards(node, info, false, "/dev/null"));
-        assertEquals(90L, DiskThresholdDecider.sizeOfRelocatingShards(node, info, true, "/dev/null"));
-        assertEquals(0L, DiskThresholdDecider.sizeOfRelocatingShards(node, info, true, "/dev/some/other/dev"));
-        assertEquals(0L, DiskThresholdDecider.sizeOfRelocatingShards(node, info, true, "/dev/some/other/dev"));
+        assertEquals(100L, DiskThresholdDecider.sizeOfRelocatingShards(node, allocation, false, "/dev/null"));
+        assertEquals(90L, DiskThresholdDecider.sizeOfRelocatingShards(node, allocation, true, "/dev/null"));
+        assertEquals(0L, DiskThresholdDecider.sizeOfRelocatingShards(node, allocation, true, "/dev/some/other/dev"));
+        assertEquals(0L, DiskThresholdDecider.sizeOfRelocatingShards(node, allocation, true, "/dev/some/other/dev"));
 
         ShardRouting test_3 = ShardRouting.newUnassigned(new ShardId(index, 3), null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         test_3 = ShardRoutingHelper.initialize(test_3, "node1");
         test_3 = ShardRoutingHelper.moveToStarted(test_3);
-        assertEquals(0L, DiskThresholdDecider.getShardSize(test_3, info));
+        assertEquals(0L, DiskThresholdDecider.getExpectedShardSize(test_3, allocation, 0));
 
 
-        ShardRouting other_0 = ShardRouting.newUnassigned(new ShardId("other", "_NA_", 0), null, randomBoolean(), new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        ShardRouting other_0 = ShardRouting.newUnassigned(new ShardId("other", "5678", 0), null, randomBoolean(), new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         other_0 = ShardRoutingHelper.initialize(other_0, "node2");
         other_0 = ShardRoutingHelper.moveToStarted(other_0);
         other_0 = ShardRoutingHelper.relocate(other_0, "node1");
@@ -273,12 +286,70 @@ public class DiskThresholdDeciderUnitTests extends ESTestCase {
         node = new RoutingNode("node1", new DiscoveryNode("node1", new LocalTransportAddress("test"),
                 emptyMap(), emptySet(), Version.CURRENT), test_0, test_1.buildTargetRelocatingShard(), test_2, other_0.buildTargetRelocatingShard());
         if (other_0.primary()) {
-            assertEquals(10100L, DiskThresholdDecider.sizeOfRelocatingShards(node, info, false, "/dev/null"));
-            assertEquals(10090L, DiskThresholdDecider.sizeOfRelocatingShards(node, info, true, "/dev/null"));
+            assertEquals(10100L, DiskThresholdDecider.sizeOfRelocatingShards(node, allocation, false, "/dev/null"));
+            assertEquals(10090L, DiskThresholdDecider.sizeOfRelocatingShards(node, allocation, true, "/dev/null"));
         } else {
-            assertEquals(100L, DiskThresholdDecider.sizeOfRelocatingShards(node, info, false, "/dev/null"));
-            assertEquals(90L, DiskThresholdDecider.sizeOfRelocatingShards(node, info, true, "/dev/null"));
+            assertEquals(100L, DiskThresholdDecider.sizeOfRelocatingShards(node, allocation, false, "/dev/null"));
+            assertEquals(90L, DiskThresholdDecider.sizeOfRelocatingShards(node, allocation, true, "/dev/null"));
         }
+    }
+
+    public void testSizeShrinkIndex() {
+        ImmutableOpenMap.Builder<String, Long> shardSizes = ImmutableOpenMap.builder();
+        shardSizes.put("[test][0][p]", 10L);
+        shardSizes.put("[test][1][p]", 100L);
+        shardSizes.put("[test][2][p]", 1000L);
+        ClusterInfo info = new DevNullClusterInfo(ImmutableOpenMap.of(), ImmutableOpenMap.of(), shardSizes.build());
+        MetaData.Builder metaBuilder = MetaData.builder();
+        metaBuilder.put(IndexMetaData.builder("test").settings(settings(Version.CURRENT).put("index.uuid", "1234"))
+            .numberOfShards(3).numberOfReplicas(0));
+        metaBuilder.put(IndexMetaData.builder("target").settings(settings(Version.CURRENT).put("index.uuid", "5678")
+            .put("index.shrink.source.name", "test").put("index.shrink.source.uuid", "1234")).numberOfShards(1).numberOfReplicas(0));
+        MetaData metaData = metaBuilder.build();
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        routingTableBuilder.addAsNew(metaData.index("test"));
+        routingTableBuilder.addAsNew(metaData.index("target"));
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+            .metaData(metaData).routingTable(routingTableBuilder.build()).build();
+
+        AllocationService allocationService = createAllocationService();
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")))
+            .build();
+        RoutingAllocation.Result result = allocationService.reroute(clusterState, "foo");
+        clusterState = ClusterState.builder(clusterState).routingTable(result.routingTable()).build();
+
+        result = allocationService.applyStartedShards(clusterState,
+            clusterState.getRoutingTable().index("test").shardsWithState(ShardRoutingState.UNASSIGNED));
+        clusterState = ClusterState.builder(clusterState).routingTable(result.routingTable()).build();
+
+        RoutingAllocation allocation = new RoutingAllocation(null, clusterState.getRoutingNodes(), clusterState, info, 0, false);
+
+        final Index index = new Index("test", "1234");
+        ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), null, true,
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        test_0 = ShardRoutingHelper.initialize(test_0, "node1");
+        test_0 = ShardRoutingHelper.moveToStarted(test_0);
+
+        ShardRouting test_1 = ShardRouting.newUnassigned(new ShardId(index, 1), null, true,
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        test_1 = ShardRoutingHelper.initialize(test_1, "node2");
+        test_1 = ShardRoutingHelper.moveToStarted(test_1);
+
+        ShardRouting test_2 = ShardRouting.newUnassigned(new ShardId(index, 2), null, true,
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        test_2 = ShardRoutingHelper.initialize(test_2, "node1");
+
+        assertEquals(1000L, DiskThresholdDecider.getExpectedShardSize(test_2, allocation, 0));
+        assertEquals(100L, DiskThresholdDecider.getExpectedShardSize(test_1, allocation, 0));
+        assertEquals(10L, DiskThresholdDecider.getExpectedShardSize(test_0, allocation, 0));
+
+
+        ShardRouting target = ShardRouting.newUnassigned(new ShardId(new Index("target", "5678"), 0),
+            null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+
+
+        assertEquals(1110L, DiskThresholdDecider.getExpectedShardSize(target, allocation, 0));
+
 
     }
 

From 58a103aca2f0bf1ad3e71c0ff75cccb311df5347 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Tue, 31 May 2016 16:37:58 +0200
Subject: [PATCH 0533/1311] Remove custom iterators from RoutingNodes (#18615)

Contains a number of cleanups related to recent changes in RoutingNodes:

- PR #17821 (Immutable ShardRouting) changed RoutingNode to use a map indexed by ShardId to manage ShardRouting elements. This means that we can directly select the right ShardRouting without iterating over all elements. This lets us get rid of RoutingNodeIterator and all kind of iterations all over the place.
- Second cleanup is an extension of #18390 (Expose cluster state before reroute in RoutingAllocation instead of RoutingNodes). We should not reexpose RoutingTable in RoutingNodes and only use it in the constructor. This makes it clear that the RoutingTable is only used to construct the RoutingNodes and can diverge from it afterwards (only RoutingNodes is mutable).
- Remove AllocationService.applyNewNodes() (that is already done as part of construction of RoutingNodes)
---
 ...ansportClusterAllocationExplainAction.java |   5 +-
 .../action/shard/ShardStateAction.java        |  14 +-
 .../cluster/routing/IndexRoutingTable.java    |   4 +-
 .../cluster/routing/RoutingNode.java          |   9 +-
 .../cluster/routing/RoutingNodes.java         | 232 +++++-------------
 .../cluster/routing/RoutingTable.java         |  10 +-
 .../routing/allocation/AllocationService.java | 174 +++++--------
 .../routing/allocation/RoutingAllocation.java |   5 +-
 .../command/CancelAllocationCommand.java      | 107 +++-----
 .../decider/AwarenessAllocationDecider.java   |   2 +-
 .../decider/NodeVersionAllocationDecider.java |   2 +-
 ...alanceOnlyWhenActiveAllocationDecider.java |   4 +-
 ...caAfterPrimaryActiveAllocationDecider.java |   2 +-
 .../decider/SameShardAllocationDecider.java   |   2 +-
 .../gateway/ReplicaShardAllocator.java        |  12 +-
 .../cluster/ClusterInfoServiceIT.java         |   3 +-
 .../cluster/ClusterStateDiffIT.java           |   2 +-
 .../org/elasticsearch/cluster/ack/AckIT.java  |   8 +-
 .../DelayedAllocationServiceTests.java        |  10 +-
 .../allocation/BalanceConfigurationTests.java |  31 +--
 .../ExpectedShardSizeAllocationTests.java     |   4 +-
 .../NodeVersionAllocationDeciderTests.java    |   4 +-
 .../index/store/CorruptedFileIT.java          |   8 +-
 .../index/store/CorruptedTranslogIT.java      |   2 +-
 24 files changed, 211 insertions(+), 445 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
index c5e1f6cc2a1..121dfe6f682 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
@@ -38,7 +38,6 @@ import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
-import org.elasticsearch.cluster.routing.RoutingNodes.RoutingNodesIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
@@ -226,10 +225,8 @@ public class TransportClusterAllocationExplainAction
         // get the existing unassigned info if available
         UnassignedInfo ui = shard.unassignedInfo();
 
-        RoutingNodesIterator iter = routingNodes.nodes();
         Map<DiscoveryNode, Decision> nodeToDecision = new HashMap<>();
-        while (iter.hasNext()) {
-            RoutingNode node = iter.next();
+        for (RoutingNode node : routingNodes) {
             DiscoveryNode discoNode = node.node();
             if (discoNode.isDataNode()) {
                 Decision d = tryShardOnNode(shard, node, allocation, includeYesDecisions);
diff --git a/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
index eb43322cdd0..edeaf95443f 100644
--- a/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
+++ b/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
@@ -30,7 +30,7 @@ import org.elasticsearch.cluster.MasterNodeChangePredicate;
 import org.elasticsearch.cluster.NotMasterException;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
-import org.elasticsearch.cluster.routing.RoutingNodes;
+import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
@@ -301,13 +301,11 @@ public class ShardStateAction extends AbstractComponent {
                 }
             }
 
-            RoutingNodes.RoutingNodeIterator routingNodeIterator =
-                currentState.getRoutingNodes().routingNodeIter(task.getShardRouting().currentNodeId());
-            if (routingNodeIterator != null) {
-                for (ShardRouting maybe : routingNodeIterator) {
-                    if (task.getShardRouting().isSameAllocation(maybe)) {
-                        return ValidationResult.VALID;
-                    }
+            RoutingNode routingNode = currentState.getRoutingNodes().node(task.getShardRouting().currentNodeId());
+            if (routingNode != null) {
+                ShardRouting maybe = routingNode.getByShardId(task.getShardRouting().shardId());
+                if (maybe != null && maybe.isSameAllocation(task.getShardRouting())) {
+                    return ValidationResult.VALID;
                 }
             }
             return ValidationResult.SHARD_MISSING;
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
index 1d38dcbdb7e..8bb0f51e22e 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
@@ -486,10 +486,10 @@ public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> imple
          * Adds a new shard routing (makes a copy of it), with reference data used from the index shard routing table
          * if it needs to be created.
          */
-        public Builder addShard(IndexShardRoutingTable refData, ShardRouting shard) {
+        public Builder addShard(ShardRouting shard) {
             IndexShardRoutingTable indexShard = shards.get(shard.id());
             if (indexShard == null) {
-                indexShard = new IndexShardRoutingTable.Builder(refData.shardId()).addShard(shard).build();
+                indexShard = new IndexShardRoutingTable.Builder(shard.shardId()).addShard(shard).build();
             } else {
                 indexShard = new IndexShardRoutingTable.Builder(indexShard).addShard(shard).build();
             }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
index a57601abffc..b4a04e4d104 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
@@ -71,10 +71,6 @@ public class RoutingNode implements Iterable<ShardRouting> {
         return Collections.unmodifiableCollection(shards.values()).iterator();
     }
 
-    Iterator<ShardRouting> mutableIterator() {
-        return shards.values().iterator();
-    }
-
     /**
      * Returns the nodes {@link DiscoveryNode}.
      *
@@ -121,6 +117,11 @@ public class RoutingNode implements Iterable<ShardRouting> {
         assert previousValue == oldShard : "expected shard " + previousValue + " but was " + oldShard;
     }
 
+    void remove(ShardRouting shard) {
+        ShardRouting previousValue = shards.remove(shard.shardId());
+        assert previousValue == shard : "expected shard " + previousValue + " but was " + shard;
+    }
+
     /**
      * Determine the number of shards with a specific state
      * @param states set of states which should be counted
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
index 159dfd99d1f..fbbedcdefde 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
@@ -23,6 +23,7 @@ import com.carrotsearch.hppc.ObjectIntHashMap;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Randomness;
@@ -51,8 +52,6 @@ import java.util.function.Predicate;
  */
 public class RoutingNodes implements Iterable<RoutingNode> {
 
-    private final RoutingTable routingTable;
-
     private final Map<String, RoutingNode> nodesToShards = new HashMap<>();
 
     private final UnassignedShards unassignedShards = new UnassignedShards(this);
@@ -76,7 +75,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
 
     public RoutingNodes(ClusterState clusterState, boolean readOnly) {
         this.readOnly = readOnly;
-        this.routingTable = clusterState.routingTable();
+        final RoutingTable routingTable = clusterState.routingTable();
 
         Map<String, LinkedHashMap<ShardId, ShardRouting>> nodesToShards = new HashMap<>();
         // fill in the nodeToShards with the "live" nodes
@@ -109,7 +108,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
                             // add the counterpart shard with relocatingNodeId reflecting the source from which
                             // it's relocating from.
                             ShardRouting targetShardRouting = shard.buildTargetRelocatingShard();
-                            addInitialRecovery(targetShardRouting);
+                            addInitialRecovery(targetShardRouting, routingTable);
                             previousValue = entries.put(targetShardRouting.shardId(), targetShardRouting);
                             if (previousValue != null) {
                                 throw new IllegalArgumentException("Cannot have two different shards with same shard id on same node");
@@ -120,7 +119,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
                                 inactivePrimaryCount++;
                             }
                             inactiveShardCount++;
-                            addInitialRecovery(shard);
+                            addInitialRecovery(shard, routingTable);
                         }
                     } else {
                         unassignedShards.add(shard);
@@ -135,18 +134,18 @@ public class RoutingNodes implements Iterable<RoutingNode> {
     }
 
     private void addRecovery(ShardRouting routing) {
-        addRecovery(routing, true, false);
+        addRecovery(routing, true, null);
     }
 
     private void removeRecovery(ShardRouting routing) {
-        addRecovery(routing, false, false);
+        addRecovery(routing, false, null);
     }
 
-    public void addInitialRecovery(ShardRouting routing) {
-        addRecovery(routing,true, true);
+    private void addInitialRecovery(ShardRouting routing, RoutingTable routingTable) {
+        addRecovery(routing, true, routingTable);
     }
 
-    private void addRecovery(final ShardRouting routing, final boolean increment, final boolean initializing) {
+    private void addRecovery(final ShardRouting routing, final boolean increment, final RoutingTable routingTable) {
         final int howMany = increment ? 1 : -1;
         assert routing.initializing() : "routing must be initializing: " + routing;
         Recoveries.getOrAdd(recoveriesPerNode, routing.currentNodeId()).addIncoming(howMany);
@@ -155,7 +154,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
             sourceNodeId = routing.relocatingNodeId();
             if (routing.primary() && increment == false) { // primary is done relocating
                 int numRecoveringReplicas = 0;
-                for (ShardRouting assigned : assignedShards(routing)) {
+                for (ShardRouting assigned : assignedShards(routing.shardId())) {
                     if (assigned.primary() == false && assigned.initializing() && assigned.relocatingNodeId() == null) {
                         numRecoveringReplicas++;
                     }
@@ -166,7 +165,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
             }
         } else if (routing.primary() == false) { // primary without relocationID is initial recovery
             ShardRouting primary = findPrimary(routing);
-            if (primary == null && initializing) {
+            if (primary == null && routingTable != null) {
                 primary = routingTable.index(routing.index().getName()).shard(routing.shardId().id()).primary;
             } else if (primary == null) {
                 throw new IllegalStateException("replica is initializing but primary is unassigned");
@@ -212,22 +211,14 @@ public class RoutingNodes implements Iterable<RoutingNode> {
         return Collections.unmodifiableCollection(nodesToShards.values()).iterator();
     }
 
-    public RoutingTable routingTable() {
-        return routingTable;
-    }
-
-    public RoutingTable getRoutingTable() {
-        return routingTable();
+    public Iterator<RoutingNode> mutableIterator() {
+        return nodesToShards.values().iterator();
     }
 
     public UnassignedShards unassigned() {
         return this.unassignedShards;
     }
 
-    public RoutingNodesIterator nodes() {
-        return new RoutingNodesIterator(nodesToShards.values().iterator());
-    }
-
     public RoutingNode node(String nodeId) {
         return nodesToShards.get(nodeId);
     }
@@ -277,11 +268,20 @@ public class RoutingNodes implements Iterable<RoutingNode> {
     }
 
     /**
-     * Returns the active primary shard for the given ShardRouting or <code>null</code> if
+     * Returns all shards that are not in the state UNASSIGNED with the same shard
+     * ID as the given shard.
+     */
+    public List<ShardRouting> assignedShards(ShardId shardId) {
+        final List<ShardRouting> replicaSet = assignedShards.get(shardId);
+        return replicaSet == null ? EMPTY : Collections.unmodifiableList(replicaSet);
+    }
+
+    /**
+     * Returns the active primary shard for the given shard id or <code>null</code> if
      * no primary is found or the primary is not active.
      */
-    public ShardRouting activePrimary(ShardRouting shard) {
-        for (ShardRouting shardRouting : assignedShards(shard.shardId())) {
+    public ShardRouting activePrimary(ShardId shardId) {
+        for (ShardRouting shardRouting : assignedShards(shardId)) {
             if (shardRouting.primary() && shardRouting.active()) {
                 return shardRouting;
             }
@@ -290,11 +290,11 @@ public class RoutingNodes implements Iterable<RoutingNode> {
     }
 
     /**
-     * Returns one active replica shard for the given ShardRouting shard ID or <code>null</code> if
+     * Returns one active replica shard for the given shard id or <code>null</code> if
      * no active replica is found.
      */
-    public ShardRouting activeReplica(ShardRouting shard) {
-        for (ShardRouting shardRouting : assignedShards(shard.shardId())) {
+    public ShardRouting activeReplica(ShardId shardId) {
+        for (ShardRouting shardRouting : assignedShards(shardId)) {
             if (!shardRouting.primary() && shardRouting.active()) {
                 return shardRouting;
             }
@@ -302,20 +302,12 @@ public class RoutingNodes implements Iterable<RoutingNode> {
         return null;
     }
 
-    /**
-     * Returns all shards that are not in the state UNASSIGNED with the same shard
-     * ID as the given shard.
-     */
-    public Iterable<ShardRouting> assignedShards(ShardRouting shard) {
-        return assignedShards(shard.shardId());
-    }
-
     /**
      * Returns <code>true</code> iff all replicas are active for the given shard routing. Otherwise <code>false</code>
      */
-    public boolean allReplicasActive(ShardRouting shardRouting) {
-        final List<ShardRouting> shards = assignedShards(shardRouting.shardId());
-        if (shards.isEmpty() || shards.size() < this.routingTable.index(shardRouting.index().getName()).shard(shardRouting.id()).size()) {
+    public boolean allReplicasActive(ShardId shardId, MetaData metaData) {
+        final List<ShardRouting> shards = assignedShards(shardId);
+        if (shards.isEmpty() || shards.size() < metaData.getIndexSafe(shardId.getIndex()).getNumberOfReplicas() + 1) {
             return false; // if we are empty nothing is active if we have less than total at least one is unassigned
         }
         for (ShardRouting shard : shards) {
@@ -429,7 +421,7 @@ public class RoutingNodes implements Iterable<RoutingNode> {
      */
     public ShardRouting started(ShardRouting shard) {
         ensureMutable();
-        assert !shard.active() : "expected an initializing shard " + shard;
+        assert shard.initializing() : "expected an initializing shard " + shard;
         if (shard.relocatingNodeId() == null) {
             // if this is not a target shard for relocation, we need to update statistics
             inactiveShardCount--;
@@ -475,18 +467,15 @@ public class RoutingNodes implements Iterable<RoutingNode> {
 
     private static final List<ShardRouting> EMPTY = Collections.emptyList();
 
-    private List<ShardRouting> assignedShards(ShardId shardId) {
-        final List<ShardRouting> replicaSet = assignedShards.get(shardId);
-        return replicaSet == null ? EMPTY : Collections.unmodifiableList(replicaSet);
-    }
-
     /**
      * Cancels the give shard from the Routing nodes internal statistics and cancels
      * the relocation if the shard is relocating.
      */
-    private void remove(ShardRouting shard) {
+    public void remove(ShardRouting shard) {
         ensureMutable();
-        if (!shard.active() && shard.relocatingNodeId() == null) {
+        assert shard.unassigned() == false : "only assigned shards can be removed here (" + shard + ")";
+        node(shard.currentNodeId()).remove(shard);
+        if (shard.initializing() && shard.relocatingNodeId() == null) {
             inactiveShardCount--;
             assert inactiveShardCount >= 0;
             if (shard.primary()) {
@@ -501,6 +490,20 @@ public class RoutingNodes implements Iterable<RoutingNode> {
         }
     }
 
+    /**
+     * Removes relocation source of an initializing non-primary shard. This allows the replica shard to continue recovery from
+     * the primary even though its non-primary relocation source has failed.
+     */
+    public ShardRouting removeRelocationSource(ShardRouting shard) {
+        assert shard.isRelocationTarget() : "only relocation target shards can have their relocation source removed (" + shard + ")";
+        ensureMutable();
+        ShardRouting relocationMarkerRemoved = shard.removeRelocationSource();
+        updateAssigned(shard, relocationMarkerRemoved);
+        inactiveShardCount++; // relocation targets are not counted as inactive shards whereas initializing shards are
+        Recoveries.getOrAdd(recoveriesPerNode, shard.relocatingNodeId()).addOutgoing(-1);
+        return relocationMarkerRemoved;
+    }
+
     private void assignedShardsAdd(ShardRouting shard) {
         assert shard.unassigned() == false : "unassigned shard " + shard + " cannot be added to list of assigned shards";
         List<ShardRouting> shards = assignedShards.computeIfAbsent(shard.shardId(), k -> new ArrayList<>());
@@ -531,28 +534,6 @@ public class RoutingNodes implements Iterable<RoutingNode> {
         assert false : "No shard found to remove";
     }
 
-    public boolean isKnown(DiscoveryNode node) {
-        return nodesToShards.containsKey(node.getId());
-    }
-
-    public void addNode(DiscoveryNode node) {
-        ensureMutable();
-        RoutingNode routingNode = new RoutingNode(node.getId(), node);
-        nodesToShards.put(routingNode.nodeId(), routingNode);
-    }
-
-    public RoutingNodeIterator routingNodeIter(String nodeId) {
-        final RoutingNode routingNode = nodesToShards.get(nodeId);
-        if (routingNode == null) {
-            return null;
-        }
-        return new RoutingNodeIterator(routingNode);
-    }
-
-    public RoutingNode[] toArray() {
-        return nodesToShards.values().toArray(new RoutingNode[nodesToShards.size()]);
-    }
-
     public ShardRouting reinitShadowPrimary(ShardRouting candidate) {
         ensureMutable();
         if (candidate.relocating()) {
@@ -579,6 +560,15 @@ public class RoutingNodes implements Iterable<RoutingNode> {
         shardsWithMatchingShardId.set(previousShardIndex, newShard);
     }
 
+    public ShardRouting moveToUnassigned(ShardRouting shard, UnassignedInfo unassignedInfo) {
+        ensureMutable();
+        assert shard.unassigned() == false : "only assigned shards can be moved to unassigned (" + shard + ")";
+        remove(shard);
+        ShardRouting unassigned = shard.moveToUnassigned(unassignedInfo);
+        unassignedShards.add(unassigned);
+        return unassigned;
+    }
+
     /**
      * Returns the number of routing nodes
      */
@@ -896,110 +886,6 @@ public class RoutingNodes implements Iterable<RoutingNode> {
         return true;
     }
 
-
-    public class RoutingNodesIterator implements Iterator<RoutingNode>, Iterable<ShardRouting> {
-        private RoutingNode current;
-        private final Iterator<RoutingNode> delegate;
-
-        public RoutingNodesIterator(Iterator<RoutingNode> iterator) {
-            delegate = iterator;
-        }
-
-        @Override
-        public boolean hasNext() {
-            return delegate.hasNext();
-        }
-
-        @Override
-        public RoutingNode next() {
-            return current = delegate.next();
-        }
-
-        public RoutingNodeIterator nodeShards() {
-            return new RoutingNodeIterator(current);
-        }
-
-        @Override
-        public void remove() {
-           delegate.remove();
-        }
-
-        @Override
-        public Iterator<ShardRouting> iterator() {
-            return nodeShards();
-        }
-    }
-
-    public final class RoutingNodeIterator implements Iterator<ShardRouting>, Iterable<ShardRouting> {
-        private final RoutingNode iterable;
-        private ShardRouting shard;
-        private final Iterator<ShardRouting> delegate;
-        private boolean removed = false;
-
-        public RoutingNodeIterator(RoutingNode iterable) {
-            this.delegate = iterable.mutableIterator();
-            this.iterable = iterable;
-        }
-
-        @Override
-        public boolean hasNext() {
-            return delegate.hasNext();
-        }
-
-        @Override
-        public ShardRouting next() {
-            removed = false;
-            return shard = delegate.next();
-        }
-
-        @Override
-        public void remove() {
-            ensureMutable();
-            delegate.remove();
-            RoutingNodes.this.remove(shard);
-            removed = true;
-        }
-
-
-        /** returns true if {@link #remove()} or {@link #moveToUnassigned(UnassignedInfo)} were called on the current shard */
-        public boolean isRemoved() {
-            return removed;
-        }
-
-        @Override
-        public Iterator<ShardRouting> iterator() {
-            return iterable.iterator();
-        }
-
-        public ShardRouting moveToUnassigned(UnassignedInfo unassignedInfo) {
-            ensureMutable();
-            if (isRemoved() == false) {
-                remove();
-            }
-            ShardRouting unassigned = shard.moveToUnassigned(unassignedInfo);
-            unassignedShards.add(unassigned);
-            return unassigned;
-        }
-
-        /**
-         * Removes relocation source of an initializing non-primary shard. This allows the replica shard to continue recovery from
-         * the primary even though its non-primary relocation source has failed.
-         */
-        public ShardRouting removeRelocationSource() {
-            assert shard.isRelocationTarget();
-            ensureMutable();
-            ShardRouting relocationMarkerRemoved = shard.removeRelocationSource();
-            updateAssigned(shard, relocationMarkerRemoved);
-            inactiveShardCount++; // relocation targets are not counted as inactive shards whereas initializing shards are
-            Recoveries.getOrAdd(recoveriesPerNode, shard.relocatingNodeId()).addOutgoing(-1);
-            return relocationMarkerRemoved;
-        }
-
-        public ShardRouting current() {
-            return shard;
-        }
-    }
-
     private void ensureMutable() {
         if (readOnly) {
             throw new IllegalStateException("can't modify RoutingNodes - readonly");
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java
index 86973b8c83f..f43517ec559 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java
@@ -404,9 +404,9 @@ public class RoutingTable implements Iterable<IndexRoutingTable>, Diffable<Routi
             }
         }
 
-        public Builder updateNodes(RoutingNodes routingNodes) {
+        public Builder updateNodes(long version, RoutingNodes routingNodes) {
             // this is being called without pre initializing the routing table, so we must copy over the version as well
-            this.version = routingNodes.routingTable().version();
+            this.version = version;
 
             Map<String, IndexRoutingTable.Builder> indexRoutingTableBuilders = new HashMap<>();
             for (RoutingNode routingNode : routingNodes) {
@@ -422,8 +422,7 @@ public class RoutingTable implements Iterable<IndexRoutingTable>, Diffable<Routi
                         indexRoutingTableBuilders.put(index.getName(), indexBuilder);
                     }
 
-                    IndexShardRoutingTable refData = routingNodes.routingTable().index(shardRoutingEntry.index().getName()).shard(shardRoutingEntry.id());
-                    indexBuilder.addShard(refData, shardRoutingEntry);
+                    indexBuilder.addShard(shardRoutingEntry);
                 }
             }
 
@@ -436,8 +435,7 @@ public class RoutingTable implements Iterable<IndexRoutingTable>, Diffable<Routi
                     indexBuilder = new IndexRoutingTable.Builder(index);
                     indexRoutingTableBuilders.put(index.getName(), indexBuilder);
                 }
-                IndexShardRoutingTable refData = routingNodes.routingTable().index(shardRoutingEntry.index().getName()).shard(shardRoutingEntry.id());
-                indexBuilder.addShard(refData, shardRoutingEntry);
+                indexBuilder.addShard(shardRoutingEntry);
             }
 
             for (IndexRoutingTable.Builder indexBuilder : indexRoutingTableBuilders.values()) {
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
index 83f7ac9f2af..55954f0f1e4 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
@@ -19,14 +19,12 @@
 
 package org.elasticsearch.cluster.routing.allocation;
 
-import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.health.ClusterStateHealth;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.AllocationId;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
@@ -40,6 +38,7 @@ import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.gateway.GatewayAllocator;
 import org.elasticsearch.index.shard.ShardId;
@@ -47,6 +46,7 @@ import org.elasticsearch.index.shard.ShardId;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
@@ -93,7 +93,7 @@ public class AllocationService extends AbstractComponent {
         // shuffle the unassigned nodes, just so we won't have things like poison failed shards
         routingNodes.unassigned().shuffle();
         StartedRerouteAllocation allocation = new StartedRerouteAllocation(allocationDeciders, routingNodes, clusterState, startedShards, clusterInfoService.getClusterInfo(), currentNanoTime());
-        boolean changed = applyStartedShards(routingNodes, startedShards);
+        boolean changed = applyStartedShards(allocation, startedShards);
         if (!changed) {
             return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData());
         }
@@ -114,7 +114,7 @@ public class AllocationService extends AbstractComponent {
         MetaData oldMetaData = allocation.metaData();
         RoutingTable oldRoutingTable = allocation.routingTable();
         RoutingNodes newRoutingNodes = allocation.routingNodes();
-        final RoutingTable newRoutingTable = new RoutingTable.Builder().updateNodes(newRoutingNodes).build();
+        final RoutingTable newRoutingTable = new RoutingTable.Builder().updateNodes(oldRoutingTable.version(), newRoutingNodes).build();
         MetaData newMetaData = updateMetaDataWithRoutingTable(oldMetaData, oldRoutingTable, newRoutingTable);
         assert newRoutingTable.validate(newMetaData); // validates the routing table is coherent with the cluster state metadata
         logClusterHealthStateChange(
@@ -339,9 +339,6 @@ public class AllocationService extends AbstractComponent {
         // first, clear from the shards any node id they used to belong to that is now dead
         changed |= deassociateDeadNodes(allocation);
 
-        // create a sorted list of from nodes with least number of shards to the maximum ones
-        applyNewNodes(allocation);
-
         // elect primaries *before* allocating unassigned, so backups of primaries that failed
         // will be moved to primary state and not wait for primaries to be allocated and recovered (*from gateway*)
         changed |= electPrimariesAndUnassignedDanglingReplicas(allocation);
@@ -372,7 +369,7 @@ public class AllocationService extends AbstractComponent {
             if (shardEntry.primary()) {
                 // remove dangling replicas that are initializing for primary shards
                 changed |= failReplicasForUnassignedPrimary(allocation, shardEntry);
-                ShardRouting candidate = allocation.routingNodes().activeReplica(shardEntry);
+                ShardRouting candidate = allocation.routingNodes().activeReplica(shardEntry.shardId());
                 if (candidate != null) {
                     shardEntry = unassignedIterator.demotePrimaryToReplicaShard();
                     ShardRouting primarySwappedCandidate = routingNodes.promoteAssignedReplicaShardToPrimary(candidate);
@@ -401,23 +398,9 @@ public class AllocationService extends AbstractComponent {
         return changed;
     }
 
-    /**
-     * Applies the new nodes to the routing nodes and returns them (just the
-     * new nodes);
-     */
-    private void applyNewNodes(RoutingAllocation allocation) {
-        final RoutingNodes routingNodes = allocation.routingNodes();
-        for (ObjectCursor<DiscoveryNode> cursor : allocation.nodes().getDataNodes().values()) {
-            DiscoveryNode node = cursor.value;
-            if (!routingNodes.isKnown(node)) {
-                routingNodes.addNode(node);
-            }
-        }
-    }
-
     private boolean deassociateDeadNodes(RoutingAllocation allocation) {
         boolean changed = false;
-        for (RoutingNodes.RoutingNodesIterator it = allocation.routingNodes().nodes(); it.hasNext(); ) {
+        for (Iterator<RoutingNode> it = allocation.routingNodes().mutableIterator(); it.hasNext(); ) {
             RoutingNode node = it.next();
             if (allocation.nodes().getDataNodes().containsKey(node.nodeId())) {
                 // its a live node, continue
@@ -441,7 +424,7 @@ public class AllocationService extends AbstractComponent {
 
     private boolean failReplicasForUnassignedPrimary(RoutingAllocation allocation, ShardRouting primary) {
         List<ShardRouting> replicas = new ArrayList<>();
-        for (ShardRouting routing : allocation.routingNodes().assignedShards(primary)) {
+        for (ShardRouting routing : allocation.routingNodes().assignedShards(primary.shardId())) {
             if (!routing.primary() && routing.initializing()) {
                 replicas.add(routing);
             }
@@ -455,19 +438,19 @@ public class AllocationService extends AbstractComponent {
         return changed;
     }
 
-    private boolean applyStartedShards(RoutingNodes routingNodes, Iterable<? extends ShardRouting> startedShardEntries) {
+    private boolean applyStartedShards(RoutingAllocation routingAllocation, Iterable<? extends ShardRouting> startedShardEntries) {
         boolean dirty = false;
         // apply shards might be called several times with the same shard, ignore it
+        RoutingNodes routingNodes = routingAllocation.routingNodes();
         for (ShardRouting startedShard : startedShardEntries) {
             assert startedShard.initializing();
 
             // validate index still exists. strictly speaking this is not needed but it gives clearer logs
-            if (routingNodes.routingTable().index(startedShard.index()) == null) {
+            if (routingAllocation.metaData().index(startedShard.index()) == null) {
                 logger.debug("{} ignoring shard started, unknown index (routing: {})", startedShard.shardId(), startedShard);
                 continue;
             }
 
-
             RoutingNode currentRoutingNode = routingNodes.node(startedShard.currentNodeId());
             if (currentRoutingNode == null) {
                 logger.debug("{} failed to find shard in order to start it [failed to find node], ignoring (routing: {})", startedShard.shardId(), startedShard);
@@ -480,32 +463,21 @@ public class AllocationService extends AbstractComponent {
             } else if (matchingShard.isSameAllocation(startedShard) == false) {
                 logger.debug("{} failed to find shard with matching allocation id in order to start it [failed to find matching shard], ignoring (routing: {}, matched shard routing: {})", startedShard.shardId(), startedShard, matchingShard);
             } else {
-                if (matchingShard.active()) {
+                startedShard = matchingShard;
+                if (startedShard.active()) {
                     logger.trace("{} shard is already started, ignoring (routing: {})", startedShard.shardId(), startedShard);
                 } else {
+                    assert startedShard.initializing();
                     dirty = true;
-                    // override started shard with the latest copy.
-                    startedShard = matchingShard;
-                    routingNodes.started(matchingShard);
+                    routingNodes.started(startedShard);
                     logger.trace("{} marked shard as started (routing: {})", startedShard.shardId(), startedShard);
-                }
-            }
 
-            // startedShard is the current state of the shard (post relocation for example)
-            // this means that after relocation, the state will be started and the currentNodeId will be
-            // the node we relocated to
-            if (startedShard.relocatingNodeId() == null) {
-                continue;
-            }
-
-            RoutingNodes.RoutingNodeIterator sourceRoutingNode = routingNodes.routingNodeIter(startedShard.relocatingNodeId());
-            if (sourceRoutingNode != null) {
-                while (sourceRoutingNode.hasNext()) {
-                    ShardRouting shard = sourceRoutingNode.next();
-                    if (shard.isRelocationSourceOf(startedShard)) {
-                        dirty = true;
-                        sourceRoutingNode.remove();
-                        break;
+                    if (startedShard.relocatingNodeId() != null) {
+                        // relocation target has been started, remove relocation source
+                        RoutingNode relocationSourceNode = routingNodes.node(startedShard.relocatingNodeId());
+                        ShardRouting relocationSourceShard = relocationSourceNode.getByShardId(startedShard.shardId());
+                        assert relocationSourceShard.isRelocationSourceOf(startedShard);
+                        routingNodes.remove(relocationSourceShard);
                     }
                 }
             }
@@ -525,91 +497,73 @@ public class AllocationService extends AbstractComponent {
         }
         RoutingNodes routingNodes = allocation.routingNodes();
 
-        RoutingNodes.RoutingNodeIterator matchedNode = routingNodes.routingNodeIter(failedShard.currentNodeId());
+        RoutingNode matchedNode = routingNodes.node(failedShard.currentNodeId());
         if (matchedNode == null) {
             logger.debug("{} ignoring shard failure, unknown node in {} ({})", failedShard.shardId(), failedShard, unassignedInfo.shortSummary());
             return false;
         }
 
-        boolean matchedShard = false;
-        while (matchedNode.hasNext()) {
-            ShardRouting routing = matchedNode.next();
-            if (routing.isSameAllocation(failedShard)) {
-                matchedShard = true;
-                logger.debug("{} failed shard {} found in routingNodes, failing it ({})", failedShard.shardId(), failedShard, unassignedInfo.shortSummary());
-                break;
-            }
-        }
-
-        if (matchedShard == false) {
+        ShardRouting matchedShard = matchedNode.getByShardId(failedShard.shardId());
+        if (matchedShard != null && matchedShard.isSameAllocation(failedShard)) {
+            logger.debug("{} failed shard {} found in routingNodes, failing it ({})", failedShard.shardId(), failedShard, unassignedInfo.shortSummary());
+            // replace incoming instance to make sure we work on the latest one
+            failedShard = matchedShard;
+        } else {
             logger.debug("{} ignoring shard failure, unknown allocation id in {} ({})", failedShard.shardId(), failedShard, unassignedInfo.shortSummary());
             return false;
         }
+
         if (failedShard.primary()) {
             // fail replicas first otherwise we move RoutingNodes into an inconsistent state
             failReplicasForUnassignedPrimary(allocation, failedShard);
         }
-        // replace incoming instance to make sure we work on the latest one
-        failedShard = matchedNode.current();
-
-        // remove the current copy of the shard
-        matchedNode.remove();
 
         if (addToIgnoreList) {
             // make sure we ignore this shard on the relevant node
             allocation.addIgnoreShardForNode(failedShard.shardId(), failedShard.currentNodeId());
         }
 
-        if (failedShard.relocatingNodeId() != null && failedShard.initializing()) {
-            // The shard is a target of a relocating shard. In that case we only
-            // need to remove the target shard and cancel the source relocation.
-            // No shard is left unassigned
-            logger.trace("{} is a relocation target, resolving source to cancel relocation ({})", failedShard, unassignedInfo.shortSummary());
-            RoutingNode relocatingFromNode = routingNodes.node(failedShard.relocatingNodeId());
-            if (relocatingFromNode != null) {
-                for (ShardRouting shardRouting : relocatingFromNode) {
-                    if (shardRouting.isRelocationSourceOf(failedShard)) {
-                        logger.trace("{}, resolved source to [{}]. canceling relocation ... ({})", failedShard.shardId(), shardRouting, unassignedInfo.shortSummary());
-                        routingNodes.cancelRelocation(shardRouting);
-                        break;
-                    }
-                }
-            }
+        cancelShard(logger, failedShard, unassignedInfo, routingNodes);
+        assert matchedNode.getByShardId(failedShard.shardId()) == null : "failedShard " + failedShard + " was matched but wasn't removed";
+        return true;
+    }
+
+    public static void cancelShard(ESLogger logger, ShardRouting cancelledShard, UnassignedInfo unassignedInfo, RoutingNodes routingNodes) {
+        if (cancelledShard.relocatingNodeId() == null) {
+            routingNodes.moveToUnassigned(cancelledShard, unassignedInfo);
         } else {
-            // The fail shard is the main copy of the current shard routing.
-
-            boolean addAsUnassigned = true;
-            if (failedShard.relocatingNodeId() != null) {
+            if (cancelledShard.initializing()) {
+                // The shard is a target of a relocating shard. In that case we only
+                // need to remove the target shard and cancel the source relocation.
+                // No shard is left unassigned
+                logger.trace("{} is a relocation target, resolving source to cancel relocation ({})", cancelledShard, unassignedInfo.shortSummary());
+                RoutingNode sourceNode = routingNodes.node(cancelledShard.relocatingNodeId());
+                ShardRouting sourceShard = sourceNode.getByShardId(cancelledShard.shardId());
+                assert sourceShard.isRelocationSourceOf(cancelledShard);
+                logger.trace("{}, resolved source to [{}]. canceling relocation ... ({})", cancelledShard.shardId(), sourceShard, unassignedInfo.shortSummary());
+                routingNodes.cancelRelocation(sourceShard);
+                routingNodes.remove(cancelledShard);
+            } else {
+                assert cancelledShard.relocating();
+                // The cancelled shard is the main copy of the current shard routing.
                 // now, find the shard that is initializing on the target node
-                assert failedShard.relocating();
-
-                RoutingNodes.RoutingNodeIterator initializingNode = routingNodes.routingNodeIter(failedShard.relocatingNodeId());
-                if (initializingNode != null) {
-                    while (initializingNode.hasNext()) {
-                        ShardRouting shardRouting = initializingNode.next();
-                        if (shardRouting.isRelocationTargetOf(failedShard)) {
-                            if (failedShard.primary()) {
-                                logger.trace("{} is removed due to the failure of the source shard", shardRouting);
-                                // cancel and remove target shard
-                                initializingNode.remove();
-                            } else {
-                                logger.trace("{}, relocation source failed, mark as initializing without relocation source", shardRouting);
-                                // promote to initializing shard without relocation source and ensure that removed relocation source
-                                // is not added back as unassigned shard
-                                initializingNode.removeRelocationSource();
-                                addAsUnassigned = false;
-                            }
-                            break;
-                        }
-                    }
+                RoutingNode targetNode = routingNodes.node(cancelledShard.relocatingNodeId());
+                ShardRouting targetShard = targetNode.getByShardId(cancelledShard.shardId());
+                assert targetShard.isRelocationTargetOf(cancelledShard);
+                if (cancelledShard.primary()) {
+                    logger.trace("{} is removed due to the failure/cancellation of the source shard", targetShard);
+                    // cancel and remove target shard
+                    routingNodes.remove(targetShard);
+                    routingNodes.moveToUnassigned(cancelledShard, unassignedInfo);
+                } else {
+                    logger.trace("{}, relocation source failed / cancelled, mark as initializing without relocation source", targetShard);
+                    // promote to initializing shard without relocation source and ensure that removed relocation source
+                    // is not added back as unassigned shard
+                    routingNodes.removeRelocationSource(targetShard);
+                    routingNodes.remove(cancelledShard);
                 }
             }
-            if (addAsUnassigned) {
-                matchedNode.moveToUnassigned(unassignedInfo);
-            }
         }
-        assert matchedNode.isRemoved() : "failedShard " + failedShard + " was matched but wasn't removed";
-        return true;
     }
 
     private RoutingNodes getMutableRoutingNodes(ClusterState clusterState) {
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
index 584b141913f..f58ff54fc14 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java
@@ -120,6 +120,8 @@ public class RoutingAllocation {
 
     private final MetaData metaData;
 
+    private final RoutingTable routingTable;
+
     private final DiscoveryNodes nodes;
 
     private final ImmutableOpenMap<String, ClusterState.Custom> customs;
@@ -152,6 +154,7 @@ public class RoutingAllocation {
         this.deciders = deciders;
         this.routingNodes = routingNodes;
         this.metaData = clusterState.metaData();
+        this.routingTable = clusterState.routingTable();
         this.nodes = clusterState.nodes();
         this.customs = clusterState.customs();
         this.clusterInfo = clusterInfo;
@@ -177,7 +180,7 @@ public class RoutingAllocation {
      * @return current routing table
      */
     public RoutingTable routingTable() {
-        return routingNodes.routingTable();
+        return routingTable;
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java
index 0aaacaf89ff..230e7929cc8 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java
@@ -20,25 +20,29 @@
 package org.elasticsearch.cluster.routing.allocation.command;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RerouteExplanation;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.index.shard.ShardId;
 
 import java.io.IOException;
+import java.util.Locale;
 import java.util.Objects;
 
-import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
-
 /**
  * A command that cancels relocation, or recovery of a given shard on a node.
  */
@@ -120,87 +124,38 @@ public class CancelAllocationCommand implements AllocationCommand {
     @Override
     public RerouteExplanation execute(RoutingAllocation allocation, boolean explain) {
         DiscoveryNode discoNode = allocation.nodes().resolveNode(node);
-        boolean found = false;
-        for (RoutingNodes.RoutingNodeIterator it = allocation.routingNodes().routingNodeIter(discoNode.getId()); it.hasNext(); ) {
-            ShardRouting shardRouting = it.next();
-            if (!shardRouting.shardId().getIndex().getName().equals(index)) {
-                continue;
-            }
-            if (shardRouting.shardId().id() != shardId) {
-                continue;
-            }
-            found = true;
-            if (shardRouting.relocatingNodeId() != null) {
-                if (shardRouting.initializing()) {
-                    // the shard is initializing and recovering from another node, simply cancel the recovery
-                    it.remove();
-                    // and cancel the relocating state from the shard its being relocated from
-                    RoutingNode relocatingFromNode = allocation.routingNodes().node(shardRouting.relocatingNodeId());
-                    if (relocatingFromNode != null) {
-                        for (ShardRouting fromShardRouting : relocatingFromNode) {
-                            if (fromShardRouting.isSameShard(shardRouting) && fromShardRouting.state() == RELOCATING) {
-                                allocation.routingNodes().cancelRelocation(fromShardRouting);
-                                break;
-                            }
-                        }
-                    }
-                } else if (shardRouting.relocating()) {
-
-                    if (!allowPrimary && shardRouting.primary()) {
-                        // can't cancel a primary shard being initialized
-                        if (explain) {
-                            return new RerouteExplanation(this, allocation.decision(Decision.NO, "cancel_allocation_command",
-                                    "can't cancel " + shardId + " on node " + discoNode + ", shard is primary and initializing its state"));
-                        }
-                        throw new IllegalArgumentException("[cancel_allocation] can't cancel " + shardId + " on node " +
-                                discoNode + ", shard is primary and initializing its state");
-                    }
-                    it.remove();
-                    boolean addAsUnassigned = true;
-                    // now, find the shard that is initializing on the target node
-                    RoutingNodes.RoutingNodeIterator initializingNode = allocation.routingNodes().routingNodeIter(shardRouting.relocatingNodeId());
-                    if (initializingNode != null) {
-                        while (initializingNode.hasNext()) {
-                            ShardRouting initializingShardRouting = initializingNode.next();
-                            if (initializingShardRouting.isRelocationTargetOf(shardRouting)) {
-                                if (shardRouting.primary()) {
-                                    // cancel and remove target shard
-                                    initializingNode.remove();
-                                } else {
-                                    // promote to initializing shard without relocation source and ensure that removed relocation source
-                                    // is not added back as unassigned shard
-                                    initializingNode.removeRelocationSource();
-                                    addAsUnassigned = false;
-                                }
-                                break;
-                            }
-                        }
-                    }
-                    if (addAsUnassigned) {
-                        it.moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.REROUTE_CANCELLED, null));
-                    }
-                }
-            } else {
-                // the shard is not relocating, its either started, or initializing, just cancel it and move on...
-                if (!allowPrimary && shardRouting.primary()) {
-                    // can't cancel a primary shard being initialized
-                    if (explain) {
-                        return new RerouteExplanation(this, allocation.decision(Decision.NO, "cancel_allocation_command",
-                                "can't cancel " + shardId + " on node " + discoNode + ", shard is primary and started"));
-                    }
-                    throw new IllegalArgumentException("[cancel_allocation] can't cancel " + shardId + " on node " +
-                            discoNode + ", shard is primary and started");
-                }
-                it.moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.REROUTE_CANCELLED, null));
+        ShardRouting shardRouting = null;
+        RoutingNodes routingNodes = allocation.routingNodes();
+        RoutingNode routingNode = routingNodes.node(discoNode.getId());
+        if (routingNode != null) {
+            IndexMetaData indexMetaData = allocation.metaData().index(index());
+            if (indexMetaData == null) {
+                throw new IndexNotFoundException(index());
             }
+            ShardId shardId = new ShardId(indexMetaData.getIndex(), shardId());
+            shardRouting = routingNode.getByShardId(shardId);
         }
-        if (!found) {
+        if (shardRouting == null) {
             if (explain) {
                 return new RerouteExplanation(this, allocation.decision(Decision.NO, "cancel_allocation_command",
-                        "can't cancel " + shardId + ", failed to find it on node " + discoNode));
+                    "can't cancel " + shardId + ", failed to find it on node " + discoNode));
             }
             throw new IllegalArgumentException("[cancel_allocation] can't cancel " + shardId + ", failed to find it on node " + discoNode);
         }
+        if (shardRouting.primary() && allowPrimary == false) {
+            if ((shardRouting.initializing() && shardRouting.relocatingNodeId() != null) == false) {
+                // only allow cancelling initializing shard of primary relocation without allowPrimary flag
+                if (explain) {
+                    return new RerouteExplanation(this, allocation.decision(Decision.NO, "cancel_allocation_command",
+                        "can't cancel " + shardId + " on node " + discoNode + ", shard is primary and " +
+                            shardRouting.state().name().toLowerCase(Locale.ROOT)));
+                }
+                throw new IllegalArgumentException("[cancel_allocation] can't cancel " + shardId + " on node " +
+                    discoNode + ", shard is primary and " + shardRouting.state().name().toLowerCase(Locale.ROOT));
+            }
+        }
+        AllocationService.cancelShard(Loggers.getLogger(CancelAllocationCommand.class), shardRouting,
+            new UnassignedInfo(UnassignedInfo.Reason.REROUTE_CANCELLED, null), routingNodes);
         return new RerouteExplanation(this, allocation.decision(Decision.YES, "cancel_allocation_command",
                 "shard " + shardId + " on node " + discoNode + " can be cancelled"));
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java
index 8e6dbd989e2..ed7ab3bd68e 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java
@@ -167,7 +167,7 @@ public class AwarenessAllocationDecider extends AllocationDecider {
 
             // build the count of shards per attribute value
             ObjectIntHashMap<String> shardPerAttribute = new ObjectIntHashMap<>();
-            for (ShardRouting assignedShard : allocation.routingNodes().assignedShards(shardRouting)) {
+            for (ShardRouting assignedShard : allocation.routingNodes().assignedShards(shardRouting.shardId())) {
                 if (assignedShard.started() || assignedShard.initializing()) {
                     // Note: this also counts relocation targets as that will be the new location of the shard.
                     // Relocation sources should not be counted as the shard is moving away
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeVersionAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeVersionAllocationDecider.java
index 17ac486a8ce..9d9f78c3c15 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeVersionAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeVersionAllocationDecider.java
@@ -59,7 +59,7 @@ public class NodeVersionAllocationDecider extends AllocationDecider {
                 return isVersionCompatible(allocation.routingNodes(), shardRouting.currentNodeId(), node, allocation);
             }
         } else {
-            final ShardRouting primary = allocation.routingNodes().activePrimary(shardRouting);
+            final ShardRouting primary = allocation.routingNodes().activePrimary(shardRouting.shardId());
             // check that active primary has a newer version so that peer recovery works
             if (primary != null) {
                 return isVersionCompatible(allocation.routingNodes(), primary.currentNodeId(), node, allocation);
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RebalanceOnlyWhenActiveAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RebalanceOnlyWhenActiveAllocationDecider.java
index 869c6313069..b6c675597c0 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RebalanceOnlyWhenActiveAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RebalanceOnlyWhenActiveAllocationDecider.java
@@ -38,9 +38,7 @@ public class RebalanceOnlyWhenActiveAllocationDecider extends AllocationDecider
 
     @Override
     public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation allocation) {
-        // its ok to check for active here, since in relocation, a shard is split into two in routing
-        // nodes, once relocating, and one initializing
-        if (!allocation.routingNodes().allReplicasActive(shardRouting)) {
+        if (!allocation.routingNodes().allReplicasActive(shardRouting.shardId(), allocation.metaData())) {
             return allocation.decision(Decision.NO, NAME, "rebalancing can not occur if not all replicas are active in the cluster");
         }
         return allocation.decision(Decision.YES, NAME, "all replicas are active in the cluster, rebalancing can occur");
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ReplicaAfterPrimaryActiveAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ReplicaAfterPrimaryActiveAllocationDecider.java
index 59ab67c309e..c23f0930924 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ReplicaAfterPrimaryActiveAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ReplicaAfterPrimaryActiveAllocationDecider.java
@@ -47,7 +47,7 @@ public class ReplicaAfterPrimaryActiveAllocationDecider extends AllocationDecide
         if (shardRouting.primary()) {
             return allocation.decision(Decision.YES, NAME, "shard is primary and can be allocated");
         }
-        ShardRouting primary = allocation.routingNodes().activePrimary(shardRouting);
+        ShardRouting primary = allocation.routingNodes().activePrimary(shardRouting.shardId());
         if (primary == null) {
             return allocation.decision(Decision.NO, NAME, "primary shard for this replica is not yet active");
         }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java
index f0b4fdf35c6..fca8a34936e 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java
@@ -58,7 +58,7 @@ public class SameShardAllocationDecider extends AllocationDecider {
 
     @Override
     public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
-        Iterable<ShardRouting> assignedShards = allocation.routingNodes().assignedShards(shardRouting);
+        Iterable<ShardRouting> assignedShards = allocation.routingNodes().assignedShards(shardRouting.shardId());
         for (ShardRouting assignedShard : assignedShards) {
             if (node.nodeId().equals(assignedShard.currentNodeId())) {
                 return allocation.decision(Decision.NO, NAME,
diff --git a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
index cf8a54dfc64..13d08590216 100644
--- a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
+++ b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
@@ -59,10 +59,8 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
     public boolean processExistingRecoveries(RoutingAllocation allocation) {
         boolean changed = false;
         MetaData metaData = allocation.metaData();
-        for (RoutingNodes.RoutingNodesIterator nodes = allocation.routingNodes().nodes(); nodes.hasNext(); ) {
-            nodes.next();
-            for (RoutingNodes.RoutingNodeIterator it = nodes.nodeShards(); it.hasNext(); ) {
-                ShardRouting shard = it.next();
+        for (RoutingNode routingNode : allocation.routingNodes()) {
+            for (ShardRouting shard : routingNode) {
                 if (shard.primary() == true) {
                     continue;
                 }
@@ -85,7 +83,7 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
                     continue; // still fetching
                 }
 
-                ShardRouting primaryShard = allocation.routingNodes().activePrimary(shard);
+                ShardRouting primaryShard = allocation.routingNodes().activePrimary(shard.shardId());
                 assert primaryShard != null : "the replica shard can be allocated on at least one node, so there must be an active primary";
                 TransportNodesListShardStoreMetaData.StoreFilesMetaData primaryStore = findStore(primaryShard, allocation, shardStores);
                 if (primaryStore == null || primaryStore.allocated() == false) {
@@ -106,7 +104,7 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
                         // so we found a better one, cancel this one
                         logger.debug("cancelling allocation of replica on [{}], sync id match found on node [{}]",
                                 currentNode, nodeWithHighestMatch);
-                        it.moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.REALLOCATED_REPLICA,
+                        allocation.routingNodes().moveToUnassigned(shard, new UnassignedInfo(UnassignedInfo.Reason.REALLOCATED_REPLICA,
                                 "existing allocation of replica to [" + currentNode + "] cancelled, sync id match found on node [" + nodeWithHighestMatch + "]",
                                 null, 0, allocation.getCurrentNanoTime(), System.currentTimeMillis(), false));
                         changed = true;
@@ -149,7 +147,7 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
                 continue; // still fetching
             }
 
-            ShardRouting primaryShard = routingNodes.activePrimary(shard);
+            ShardRouting primaryShard = routingNodes.activePrimary(shard.shardId());
             assert primaryShard != null : "the replica shard can be allocated on at least one node, so there must be an active primary";
             TransportNodesListShardStoreMetaData.StoreFilesMetaData primaryStore = findStore(primaryShard, allocation, shardStores);
             if (primaryStore == null || primaryStore.allocated() == false) {
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
index 4338a43fa2d..8b50d125a40 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
@@ -170,8 +170,7 @@ public class ClusterInfoServiceIT extends ESIntegTestCase {
         }
         ClusterService clusterService = internalTestCluster.getInstance(ClusterService.class, internalTestCluster.getMasterName());
         ClusterState state = clusterService.state();
-        RoutingNodes routingNodes = state.getRoutingNodes();
-        for (ShardRouting shard : routingNodes.getRoutingTable().allShards()) {
+        for (ShardRouting shard : state.routingTable().allShards()) {
             String dataPath = info.getDataPath(shard);
             assertNotNull(dataPath);
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
index 65f451ac85c..1d013737a30 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
@@ -260,7 +260,7 @@ public class ClusterStateDiffIT extends ESIntegTestCase {
         for (ObjectCursor<IndexShardRoutingTable> indexShardRoutingTable :  original.shards().values()) {
             for (ShardRouting shardRouting : indexShardRoutingTable.value.shards()) {
                 final ShardRouting updatedShardRouting = randomChange(shardRouting, nodes);
-                builder.addShard(indexShardRoutingTable.value, updatedShardRouting);
+                builder.addShard(updatedShardRouting);
             }
         }
         return builder.build();
diff --git a/core/src/test/java/org/elasticsearch/cluster/ack/AckIT.java b/core/src/test/java/org/elasticsearch/cluster/ack/AckIT.java
index 0a0732252b7..2ea4f755156 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ack/AckIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ack/AckIT.java
@@ -99,7 +99,7 @@ public class AckIT extends ESIntegTestCase {
 
         for (Client client : clients()) {
             ClusterState clusterState = getLocalClusterState(client);
-            for (ShardRouting shardRouting : clusterState.getRoutingNodes().routingNodeIter(moveAllocationCommand.fromNode())) {
+            for (ShardRouting shardRouting : clusterState.getRoutingNodes().node(moveAllocationCommand.fromNode())) {
                 //if the shard that we wanted to move is still on the same node, it must be relocating
                 if (shardRouting.shardId().equals(commandShard)) {
                     assertThat(shardRouting.relocating(), equalTo(true));
@@ -108,7 +108,7 @@ public class AckIT extends ESIntegTestCase {
             }
 
             boolean found = false;
-            for (ShardRouting shardRouting : clusterState.getRoutingNodes().routingNodeIter(moveAllocationCommand.toNode())) {
+            for (ShardRouting shardRouting : clusterState.getRoutingNodes().node(moveAllocationCommand.toNode())) {
                 if (shardRouting.shardId().equals(commandShard)) {
                     assertThat(shardRouting.state(), anyOf(equalTo(ShardRoutingState.INITIALIZING), equalTo(ShardRoutingState.STARTED)));
                     found = true;
@@ -150,7 +150,7 @@ public class AckIT extends ESIntegTestCase {
         //all nodes hold the same cluster state version. We only know there was no need to change anything, thus no need for ack on this update.
         ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
         boolean found = false;
-        for (ShardRouting shardRouting : clusterStateResponse.getState().getRoutingNodes().routingNodeIter(moveAllocationCommand.fromNode())) {
+        for (ShardRouting shardRouting : clusterStateResponse.getState().getRoutingNodes().node(moveAllocationCommand.fromNode())) {
             //the shard that we wanted to move is still on the same node, as we had dryRun flag
             if (shardRouting.shardId().equals(commandShard)) {
                 assertThat(shardRouting.started(), equalTo(true));
@@ -160,7 +160,7 @@ public class AckIT extends ESIntegTestCase {
         }
         assertThat(found, equalTo(true));
 
-        for (ShardRouting shardRouting : clusterStateResponse.getState().getRoutingNodes().routingNodeIter(moveAllocationCommand.toNode())) {
+        for (ShardRouting shardRouting : clusterStateResponse.getState().getRoutingNodes().node(moveAllocationCommand.toNode())) {
             if (shardRouting.shardId().equals(commandShard)) {
                 fail("shard [" + shardRouting + "] shouldn't be on node [" + moveAllocationCommand.toString() + "]");
             }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
index be8e85c0fa8..630e5b034af 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
@@ -156,7 +156,7 @@ public class DelayedAllocationServiceTests extends ESAllocationTestCase {
             .build();
         assertFalse("no shards should be unassigned", clusterState.getRoutingNodes().unassigned().size() > 0);
         String nodeId = null;
-        final List<ShardRouting> allShards = clusterState.getRoutingNodes().routingTable().allShards("test");
+        final List<ShardRouting> allShards = clusterState.getRoutingTable().allShards("test");
         // we need to find the node with the replica otherwise we will not reroute
         for (ShardRouting shardRouting : allShards) {
             if (shardRouting.primary() == false) {
@@ -254,7 +254,7 @@ public class DelayedAllocationServiceTests extends ESAllocationTestCase {
 
         // find replica of short_delay
         ShardRouting shortDelayReplica = null;
-        for (ShardRouting shardRouting : clusterState.getRoutingNodes().routingTable().allShards("short_delay")) {
+        for (ShardRouting shardRouting : clusterState.getRoutingTable().allShards("short_delay")) {
             if (shardRouting.primary() == false) {
                 shortDelayReplica = shardRouting;
                 break;
@@ -264,7 +264,7 @@ public class DelayedAllocationServiceTests extends ESAllocationTestCase {
 
         // find replica of long_delay
         ShardRouting longDelayReplica = null;
-        for (ShardRouting shardRouting : clusterState.getRoutingNodes().routingTable().allShards("long_delay")) {
+        for (ShardRouting shardRouting : clusterState.getRoutingTable().allShards("long_delay")) {
             if (shardRouting.primary() == false) {
                 longDelayReplica = shardRouting;
                 break;
@@ -413,7 +413,7 @@ public class DelayedAllocationServiceTests extends ESAllocationTestCase {
             .build();
         assertFalse("no shards should be unassigned", clusterState.getRoutingNodes().unassigned().size() > 0);
         String nodeIdOfFooReplica = null;
-        for (ShardRouting shardRouting : clusterState.getRoutingNodes().routingTable().allShards("foo")) {
+        for (ShardRouting shardRouting : clusterState.getRoutingTable().allShards("foo")) {
             if (shardRouting.primary() == false) {
                 nodeIdOfFooReplica = shardRouting.currentNodeId();
                 break;
@@ -456,7 +456,7 @@ public class DelayedAllocationServiceTests extends ESAllocationTestCase {
         } else {
             // node leaves with replica shard of index bar that has shorter delay
             String nodeIdOfBarReplica = null;
-            for (ShardRouting shardRouting : stateWithDelayedShard.getRoutingNodes().routingTable().allShards("bar")) {
+            for (ShardRouting shardRouting : stateWithDelayedShard.getRoutingTable().allShards("bar")) {
                 if (shardRouting.primary() == false) {
                     nodeIdOfBarReplica = shardRouting.currentNodeId();
                     break;
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
index 37c5c4d4f45..e606f007dee 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
@@ -33,7 +33,6 @@ import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
-import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
@@ -74,13 +73,13 @@ public class BalanceConfigurationTests extends ESAllocationTestCase {
         AllocationService strategy = createAllocationService(settings.build());
 
         ClusterState clusterState = initCluster(strategy);
-        assertIndexBalance(logger, clusterState.getRoutingNodes(), numberOfNodes, numberOfIndices, numberOfReplicas, numberOfShards, balanceTreshold);
+        assertIndexBalance(clusterState.getRoutingTable(), clusterState.getRoutingNodes(), numberOfNodes, numberOfIndices, numberOfReplicas, numberOfShards, balanceTreshold);
 
         clusterState = addNode(clusterState, strategy);
-        assertIndexBalance(logger, clusterState.getRoutingNodes(), numberOfNodes + 1, numberOfIndices, numberOfReplicas, numberOfShards, balanceTreshold);
+        assertIndexBalance(clusterState.getRoutingTable(), clusterState.getRoutingNodes(), numberOfNodes + 1, numberOfIndices, numberOfReplicas, numberOfShards, balanceTreshold);
 
         clusterState = removeNodes(clusterState, strategy);
-        assertIndexBalance(logger, clusterState.getRoutingNodes(), (numberOfNodes + 1) - (numberOfNodes + 1) / 2, numberOfIndices, numberOfReplicas, numberOfShards, balanceTreshold);
+        assertIndexBalance(clusterState.getRoutingTable(), clusterState.getRoutingNodes(), (numberOfNodes + 1) - (numberOfNodes + 1) / 2, numberOfIndices, numberOfReplicas, numberOfShards, balanceTreshold);
 
     }
 
@@ -242,14 +241,14 @@ public class BalanceConfigurationTests extends ESAllocationTestCase {
         }
     }
 
-    private void assertIndexBalance(ESLogger logger, RoutingNodes nodes, int numberOfNodes, int numberOfIndices, int numberOfReplicas, int numberOfShards, float treshold) {
+    private void assertIndexBalance(RoutingTable routingTable, RoutingNodes nodes, int numberOfNodes, int numberOfIndices, int numberOfReplicas, int numberOfShards, float treshold) {
 
         final int numShards = numberOfShards * (numberOfReplicas + 1);
         final float avgNumShards = (float) (numShards) / (float) (numberOfNodes);
         final int minAvgNumberOfShards = Math.round(Math.round(Math.floor(avgNumShards - treshold)));
         final int maxAvgNumberOfShards = Math.round(Math.round(Math.ceil(avgNumShards + treshold)));
 
-        for (ObjectCursor<String> index : nodes.getRoutingTable().indicesRouting().keys()) {
+        for (ObjectCursor<String> index : routingTable.indicesRouting().keys()) {
             for (RoutingNode node : nodes) {
 //              logger.info(node.nodeId() +":"+index+ ": " + node.shardsWithState(index, INITIALIZING, STARTED).size() + " shards ("+minAvgNumberOfShards+" to "+maxAvgNumberOfShards+")");
                 assertThat(node.shardsWithState(index.value, STARTED).size(), Matchers.greaterThanOrEqualTo(minAvgNumberOfShards));
@@ -258,26 +257,6 @@ public class BalanceConfigurationTests extends ESAllocationTestCase {
         }
     }
 
-    private void assertPrimaryBalance(ESLogger logger, RoutingNodes nodes, int numberOfNodes, int numberOfIndices, int numberOfReplicas, int numberOfShards, float treshold) {
-
-        final int numShards = numberOfShards;
-        final float avgNumShards = (float) (numShards) / (float) (numberOfNodes);
-        final int minAvgNumberOfShards = Math.round(Math.round(Math.floor(avgNumShards - treshold)));
-        final int maxAvgNumberOfShards = Math.round(Math.round(Math.ceil(avgNumShards + treshold)));
-
-        for (ObjectCursor<String> index : nodes.getRoutingTable().indicesRouting().keys()) {
-            for (RoutingNode node : nodes) {
-                int primaries = 0;
-                for (ShardRouting shard : node.shardsWithState(index.value, STARTED)) {
-                    primaries += shard.primary() ? 1 : 0;
-                }
-//                logger.info(node.nodeId() + ": " + primaries + " primaries ("+minAvgNumberOfShards+" to "+maxAvgNumberOfShards+")");
-                assertThat(primaries, Matchers.greaterThanOrEqualTo(minAvgNumberOfShards));
-                assertThat(primaries, Matchers.lessThanOrEqualTo(maxAvgNumberOfShards));
-            }
-        }
-    }
-
     public void testPersistedSettings() {
         Settings.Builder settings = Settings.builder();
         settings.put(BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING.getKey(), 0.2);
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java
index 29644f07944..19591eef425 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java
@@ -85,7 +85,7 @@ public class ExpectedShardSizeAllocationTests extends ESAllocationTestCase {
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
 
         assertEquals(1, clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.INITIALIZING));
-        assertEquals(byteSize, clusterState.getRoutingNodes().getRoutingTable().shardsWithState(ShardRoutingState.INITIALIZING).get(0).getExpectedShardSize());
+        assertEquals(byteSize, clusterState.getRoutingTable().shardsWithState(ShardRoutingState.INITIALIZING).get(0).getExpectedShardSize());
         logger.info("Start the primary shard");
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
         routingTable = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING)).routingTable();
@@ -100,7 +100,7 @@ public class ExpectedShardSizeAllocationTests extends ESAllocationTestCase {
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
 
         assertEquals(1, clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.INITIALIZING));
-        assertEquals(byteSize, clusterState.getRoutingNodes().getRoutingTable().shardsWithState(ShardRoutingState.INITIALIZING).get(0).getExpectedShardSize());
+        assertEquals(byteSize, clusterState.getRoutingTable().shardsWithState(ShardRoutingState.INITIALIZING).get(0).getExpectedShardSize());
     }
 
     public void testExpectedSizeOnMove() {
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
index d0fc64b4b6b..8eb3e2f8a8b 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
@@ -418,7 +418,7 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase {
                     toId, routingNodes.node(toId).node().getVersion());
                 assertTrue(routingNodes.node(toId).node().getVersion().onOrAfter(routingNodes.node(fromId).node().getVersion()));
             } else {
-                ShardRouting primary = routingNodes.activePrimary(r);
+                ShardRouting primary = routingNodes.activePrimary(r.shardId());
                 assertThat(primary, notNullValue());
                 String fromId = primary.currentNodeId();
                 String toId = r.relocatingNodeId();
@@ -431,7 +431,7 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase {
         mutableShardRoutings = routingNodes.shardsWithState(ShardRoutingState.INITIALIZING);
         for (ShardRouting r : mutableShardRoutings) {
             if (!r.primary()) {
-                ShardRouting primary = routingNodes.activePrimary(r);
+                ShardRouting primary = routingNodes.activePrimary(r.shardId());
                 assertThat(primary, notNullValue());
                 String fromId = primary.currentNodeId();
                 String toId = r.currentNodeId();
diff --git a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
index 0000a6d65dd..6508336d9f8 100644
--- a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
+++ b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
@@ -285,7 +285,7 @@ public class CorruptedFileIT extends ESIntegTestCase {
         }
         assertThat(response.getStatus(), is(ClusterHealthStatus.RED));
         ClusterState state = client().admin().cluster().prepareState().get().getState();
-        GroupShardsIterator shardIterators = state.getRoutingNodes().getRoutingTable().activePrimaryShardsGrouped(new String[]{"test"}, false);
+        GroupShardsIterator shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[]{"test"}, false);
         for (ShardIterator iterator : shardIterators) {
             ShardRouting routing;
             while ((routing = iterator.nextOrNull()) != null) {
@@ -446,7 +446,7 @@ public class CorruptedFileIT extends ESIntegTestCase {
         // we are green so primaries got not corrupted.
         // ensure that no shard is actually allocated on the unlucky node
         ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
-        for (IndexShardRoutingTable table : clusterStateResponse.getState().getRoutingNodes().getRoutingTable().index("test")) {
+        for (IndexShardRoutingTable table : clusterStateResponse.getState().getRoutingTable().index("test")) {
             for (ShardRouting routing : table) {
                 if (unluckyNode.getNode().getId().equals(routing.currentNodeId())) {
                     assertThat(routing.state(), not(equalTo(ShardRoutingState.STARTED)));
@@ -585,7 +585,7 @@ public class CorruptedFileIT extends ESIntegTestCase {
 
     private int numShards(String... index) {
         ClusterState state = client().admin().cluster().prepareState().get().getState();
-        GroupShardsIterator shardIterators = state.getRoutingNodes().getRoutingTable().activePrimaryShardsGrouped(index, false);
+        GroupShardsIterator shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(index, false);
         return shardIterators.size();
     }
 
@@ -613,7 +613,7 @@ public class CorruptedFileIT extends ESIntegTestCase {
     private ShardRouting corruptRandomPrimaryFile(final boolean includePerCommitFiles) throws IOException {
         ClusterState state = client().admin().cluster().prepareState().get().getState();
         Index test = state.metaData().index("test").getIndex();
-        GroupShardsIterator shardIterators = state.getRoutingNodes().getRoutingTable().activePrimaryShardsGrouped(new String[]{"test"}, false);
+        GroupShardsIterator shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[]{"test"}, false);
         List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
         ShardIterator shardIterator = RandomPicks.randomFrom(random(), iterators);
         ShardRouting shardRouting = shardIterator.nextOrNull();
diff --git a/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java b/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
index 018bce16566..62ef2e029b9 100644
--- a/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
+++ b/core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
@@ -110,7 +110,7 @@ public class CorruptedTranslogIT extends ESIntegTestCase {
 
     private void corruptRandomTranslogFiles() throws IOException {
         ClusterState state = client().admin().cluster().prepareState().get().getState();
-        GroupShardsIterator shardIterators = state.getRoutingNodes().getRoutingTable().activePrimaryShardsGrouped(new String[]{"test"}, false);
+        GroupShardsIterator shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[]{"test"}, false);
         final Index test = state.metaData().index("test").getIndex();
         List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
         ShardIterator shardIterator = RandomPicks.randomFrom(random(), iterators);

From 7df5fcd4b6d69fcffca0ed8ac1ec04622a63901b Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Tue, 31 May 2016 18:32:31 +0200
Subject: [PATCH 0534/1311] Changed version back to 5.0.0

---
 buildSrc/version.properties                       | 2 +-
 core/src/main/java/org/elasticsearch/Version.java | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index 6669abb90b3..b6e64a3c263 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,4 +1,4 @@
-elasticsearch     = 5.0.0-alpha3
+elasticsearch     = 5.0.0
 lucene            = 6.0.0
 
 # optional dependencies
diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java
index c220c297388..76df1052015 100644
--- a/core/src/main/java/org/elasticsearch/Version.java
+++ b/core/src/main/java/org/elasticsearch/Version.java
@@ -76,7 +76,9 @@ public class Version {
     public static final Version V_5_0_0_alpha2 = new Version(V_5_0_0_alpha2_ID, org.apache.lucene.util.Version.LUCENE_6_0_0);
     public static final int V_5_0_0_alpha3_ID = 5000003;
     public static final Version V_5_0_0_alpha3 = new Version(V_5_0_0_alpha3_ID, org.apache.lucene.util.Version.LUCENE_6_0_0);
-    public static final Version CURRENT = V_5_0_0_alpha3;
+    public static final int V_5_0_0_ID = 5000099;
+    public static final Version V_5_0_0 = new Version(V_5_0_0_ID, org.apache.lucene.util.Version.LUCENE_6_0_0);
+    public static final Version CURRENT = V_5_0_0;
 
     static {
         assert CURRENT.luceneVersion.equals(org.apache.lucene.util.Version.LATEST) : "Version must be upgraded to ["
@@ -89,6 +91,8 @@ public class Version {
 
     public static Version fromId(int id) {
         switch (id) {
+            case V_5_0_0_ID:
+                return V_5_0_0;
             case V_5_0_0_alpha3_ID:
                 return V_5_0_0_alpha3;
             case V_5_0_0_alpha2_ID:

From ad093a78c563bc0626c82e10f99ad201a51913cc Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Tue, 31 May 2016 19:42:17 +0200
Subject: [PATCH 0535/1311] Fix ConcurrentModificationException in
 ReplicaShardAllocator

---
 .../gateway/ReplicaShardAllocator.java         | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
index 13d08590216..602a05c4321 100644
--- a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
+++ b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
@@ -34,6 +34,7 @@ import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -41,6 +42,8 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.store.StoreFileMetaData;
 import org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -59,7 +62,9 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
     public boolean processExistingRecoveries(RoutingAllocation allocation) {
         boolean changed = false;
         MetaData metaData = allocation.metaData();
-        for (RoutingNode routingNode : allocation.routingNodes()) {
+        RoutingNodes routingNodes = allocation.routingNodes();
+        List<Tuple<ShardRouting, UnassignedInfo>> recoveriesToCancel = new ArrayList<>();
+        for (RoutingNode routingNode : routingNodes) {
             for (ShardRouting shard : routingNode) {
                 if (shard.primary() == true) {
                     continue;
@@ -104,14 +109,19 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
                         // so we found a better one, cancel this one
                         logger.debug("cancelling allocation of replica on [{}], sync id match found on node [{}]",
                                 currentNode, nodeWithHighestMatch);
-                        allocation.routingNodes().moveToUnassigned(shard, new UnassignedInfo(UnassignedInfo.Reason.REALLOCATED_REPLICA,
-                                "existing allocation of replica to [" + currentNode + "] cancelled, sync id match found on node [" + nodeWithHighestMatch + "]",
-                                null, 0, allocation.getCurrentNanoTime(), System.currentTimeMillis(), false));
+                        UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.REALLOCATED_REPLICA,
+                            "existing allocation of replica to [" + currentNode + "] cancelled, sync id match found on node ["+ nodeWithHighestMatch + "]",
+                            null, 0, allocation.getCurrentNanoTime(), System.currentTimeMillis(), false);
+                        // don't cancel shard in the loop as it will cause a ConcurrentModificationException
+                        recoveriesToCancel.add(new Tuple<>(shard, unassignedInfo));
                         changed = true;
                     }
                 }
             }
         }
+        for (Tuple<ShardRouting, UnassignedInfo> cancellation : recoveriesToCancel) {
+            routingNodes.moveToUnassigned(cancellation.v1(), cancellation.v2());
+        }
         return changed;
     }
 

From e21d8b31f153d72222ab50f8b99dd820fffd3491 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 31 May 2016 14:51:58 -0400
Subject: [PATCH 0536/1311] Remove thread pool from page cache recycler

The page cache recycler has a dependency on thread pool that was there
for historical reasons but is no longer needed. This commit removes this
now unneeded dependency.

Relates #18664
---
 .../cache/recycler/PageCacheRecycler.java     | 36 ++++++-------------
 .../http/netty/NettyHttpChannelTests.java     |  2 +-
 .../netty/NettyHttpServerPipeliningTests.java |  2 +-
 .../netty/NettyHttpServerTransportTests.java  |  2 +-
 .../elasticsearch/index/IndexModuleTests.java |  2 +-
 .../NettySizeHeaderFrameDecoderTests.java     |  2 +-
 .../netty/NettyTransportMultiPortTests.java   |  2 +-
 .../cache/recycler/MockPageCacheRecycler.java |  5 ++-
 8 files changed, 19 insertions(+), 34 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cache/recycler/PageCacheRecycler.java b/core/src/main/java/org/elasticsearch/cache/recycler/PageCacheRecycler.java
index 2fad8678649..69ff997a85a 100644
--- a/core/src/main/java/org/elasticsearch/cache/recycler/PageCacheRecycler.java
+++ b/core/src/main/java/org/elasticsearch/cache/recycler/PageCacheRecycler.java
@@ -30,7 +30,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Arrays;
 import java.util.Locale;
@@ -70,25 +69,12 @@ public class PageCacheRecycler extends AbstractComponent implements Releasable {
         objectPage.close();
     }
 
-    private static int maximumSearchThreadPoolSize(ThreadPool threadPool, Settings settings) {
-        ThreadPool.Info searchThreadPool = threadPool.info(ThreadPool.Names.SEARCH);
-        assert searchThreadPool != null;
-        final int maxSize = searchThreadPool.getMax();
-        if (maxSize <= 0) {
-            // happens with cached thread pools, let's assume there are at most 2x ${number of processors} threads
-            return 2 * EsExecutors.boundedNumberOfProcessors(settings);
-        } else {
-            return maxSize;
-        }
-    }
-
     @Inject
-    public PageCacheRecycler(Settings settings, ThreadPool threadPool) {
+    public PageCacheRecycler(Settings settings) {
         super(settings);
         final Type type = TYPE_SETTING .get(settings);
         final long limit = LIMIT_HEAP_SETTING .get(settings).bytes();
         final int availableProcessors = EsExecutors.boundedNumberOfProcessors(settings);
-        final int searchThreadPoolSize = maximumSearchThreadPoolSize(threadPool, settings);
 
         // We have a global amount of memory that we need to divide across data types.
         // Since some types are more useful than other ones we give them different weights.
@@ -112,7 +98,7 @@ public class PageCacheRecycler extends AbstractComponent implements Releasable {
         final int maxPageCount = (int) Math.min(Integer.MAX_VALUE, limit / BigArrays.PAGE_SIZE_IN_BYTES);
 
         final int maxBytePageCount = (int) (bytesWeight * maxPageCount / totalWeight);
-        bytePage = build(type, maxBytePageCount, searchThreadPoolSize, availableProcessors, new AbstractRecyclerC<byte[]>() {
+        bytePage = build(type, maxBytePageCount, availableProcessors, new AbstractRecyclerC<byte[]>() {
             @Override
             public byte[] newInstance(int sizing) {
                 return new byte[BigArrays.BYTE_PAGE_SIZE];
@@ -124,7 +110,7 @@ public class PageCacheRecycler extends AbstractComponent implements Releasable {
         });
 
         final int maxIntPageCount = (int) (intsWeight * maxPageCount / totalWeight);
-        intPage = build(type, maxIntPageCount, searchThreadPoolSize, availableProcessors, new AbstractRecyclerC<int[]>() {
+        intPage = build(type, maxIntPageCount, availableProcessors, new AbstractRecyclerC<int[]>() {
             @Override
             public int[] newInstance(int sizing) {
                 return new int[BigArrays.INT_PAGE_SIZE];
@@ -136,7 +122,7 @@ public class PageCacheRecycler extends AbstractComponent implements Releasable {
         });
 
         final int maxLongPageCount = (int) (longsWeight * maxPageCount / totalWeight);
-        longPage = build(type, maxLongPageCount, searchThreadPoolSize, availableProcessors, new AbstractRecyclerC<long[]>() {
+        longPage = build(type, maxLongPageCount, availableProcessors, new AbstractRecyclerC<long[]>() {
             @Override
             public long[] newInstance(int sizing) {
                 return new long[BigArrays.LONG_PAGE_SIZE];
@@ -148,7 +134,7 @@ public class PageCacheRecycler extends AbstractComponent implements Releasable {
         });
 
         final int maxObjectPageCount = (int) (objectsWeight * maxPageCount / totalWeight);
-        objectPage = build(type, maxObjectPageCount, searchThreadPoolSize, availableProcessors, new AbstractRecyclerC<Object[]>() {
+        objectPage = build(type, maxObjectPageCount, availableProcessors, new AbstractRecyclerC<Object[]>() {
             @Override
             public Object[] newInstance(int sizing) {
                 return new Object[BigArrays.OBJECT_PAGE_SIZE];
@@ -191,12 +177,12 @@ public class PageCacheRecycler extends AbstractComponent implements Releasable {
         return objectPage.obtain();
     }
 
-    private static <T> Recycler<T> build(Type type, int limit, int estimatedThreadPoolSize, int availableProcessors, Recycler.C<T> c) {
+    private static <T> Recycler<T> build(Type type, int limit, int availableProcessors, Recycler.C<T> c) {
         final Recycler<T> recycler;
         if (limit == 0) {
             recycler = none(c);
         } else {
-            recycler = type.build(c, limit, estimatedThreadPoolSize, availableProcessors);
+            recycler = type.build(c, limit, availableProcessors);
         }
         return recycler;
     }
@@ -204,19 +190,19 @@ public class PageCacheRecycler extends AbstractComponent implements Releasable {
     public enum Type {
         QUEUE {
             @Override
-            <T> Recycler<T> build(Recycler.C<T> c, int limit, int estimatedThreadPoolSize, int availableProcessors) {
+            <T> Recycler<T> build(Recycler.C<T> c, int limit, int availableProcessors) {
                 return concurrentDeque(c, limit);
             }
         },
         CONCURRENT {
             @Override
-            <T> Recycler<T> build(Recycler.C<T> c, int limit, int estimatedThreadPoolSize, int availableProcessors) {
+            <T> Recycler<T> build(Recycler.C<T> c, int limit, int availableProcessors) {
                 return concurrent(dequeFactory(c, limit / availableProcessors), availableProcessors);
             }
         },
         NONE {
             @Override
-            <T> Recycler<T> build(Recycler.C<T> c, int limit, int estimatedThreadPoolSize, int availableProcessors) {
+            <T> Recycler<T> build(Recycler.C<T> c, int limit, int availableProcessors) {
                 return none(c);
             }
         };
@@ -229,6 +215,6 @@ public class PageCacheRecycler extends AbstractComponent implements Releasable {
             }
         }
 
-        abstract <T> Recycler<T> build(Recycler.C<T> c, int limit, int estimatedThreadPoolSize, int availableProcessors);
+        abstract <T> Recycler<T> build(Recycler.C<T> c, int limit, int availableProcessors);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java
index 0602e963ed8..8098d557a05 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java
@@ -72,7 +72,7 @@ public class NettyHttpChannelTests extends ESTestCase {
     public void setup() throws Exception {
         networkService = new NetworkService(Settings.EMPTY);
         threadPool = new ThreadPool("test");
-        MockPageCacheRecycler mockPageCacheRecycler = new MockPageCacheRecycler(Settings.EMPTY, threadPool);
+        MockPageCacheRecycler mockPageCacheRecycler = new MockPageCacheRecycler(Settings.EMPTY);
         bigArrays = new MockBigArrays(mockPageCacheRecycler, new NoneCircuitBreakerService());
     }
 
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java
index ece00173627..f233512e4ce 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java
@@ -76,7 +76,7 @@ public class NettyHttpServerPipeliningTests extends ESTestCase {
     public void setup() throws Exception {
         networkService = new NetworkService(Settings.EMPTY);
         threadPool = new ThreadPool("test");
-        mockPageCacheRecycler = new MockPageCacheRecycler(Settings.EMPTY, threadPool);
+        mockPageCacheRecycler = new MockPageCacheRecycler(Settings.EMPTY);
         bigArrays = new MockBigArrays(mockPageCacheRecycler, new NoneCircuitBreakerService());
     }
 
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerTransportTests.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerTransportTests.java
index 13ac26b010d..3089225e453 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerTransportTests.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerTransportTests.java
@@ -57,7 +57,7 @@ public class NettyHttpServerTransportTests extends ESTestCase {
     public void setup() throws Exception {
         networkService = new NetworkService(Settings.EMPTY);
         threadPool = new ThreadPool("test");
-        mockPageCacheRecycler = new MockPageCacheRecycler(Settings.EMPTY, threadPool);
+        mockPageCacheRecycler = new MockPageCacheRecycler(Settings.EMPTY);
         bigArrays = new MockBigArrays(mockPageCacheRecycler, new NoneCircuitBreakerService());
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
index 49e5d0c4595..6e901a9a671 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
@@ -113,7 +113,7 @@ public class IndexModuleTests extends ESTestCase {
         // TODO this can be used in other place too - lets first refactor the IndicesQueriesRegistry
         ThreadPool threadPool = new ThreadPool("test");
         CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService();
-        PageCacheRecycler recycler = new PageCacheRecycler(settings, threadPool);
+        PageCacheRecycler recycler = new PageCacheRecycler(settings);
         BigArrays bigArrays = new BigArrays(recycler, circuitBreakerService);
         Set<ScriptEngineService> scriptEngines = Collections.emptySet();
         scriptEngines.addAll(Arrays.asList(scriptEngineServices));
diff --git a/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java b/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java
index d2d88b87593..98d440bb446 100644
--- a/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java
@@ -67,7 +67,7 @@ public class NettySizeHeaderFrameDecoderTests extends ESTestCase {
         threadPool = new ThreadPool(settings);
         threadPool.setClusterSettings(new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
         NetworkService networkService = new NetworkService(settings);
-        BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(settings, threadPool), new NoneCircuitBreakerService());
+        BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(settings), new NoneCircuitBreakerService());
         nettyTransport = new NettyTransport(settings, threadPool, networkService, bigArrays, Version.CURRENT, new NamedWriteableRegistry(),
             new NoneCircuitBreakerService());
         nettyTransport.start();
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java
index 8ffd89428ba..78aa90cf8fc 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java
@@ -134,7 +134,7 @@ public class NettyTransportMultiPortTests extends ESTestCase {
     }
 
     private NettyTransport startNettyTransport(Settings settings, ThreadPool threadPool) {
-        BigArrays bigArrays = new MockBigArrays(new PageCacheRecycler(settings, threadPool), new NoneCircuitBreakerService());
+        BigArrays bigArrays = new MockBigArrays(new PageCacheRecycler(settings), new NoneCircuitBreakerService());
 
         NettyTransport nettyTransport = new NettyTransport(settings, threadPool, new NetworkService(settings), bigArrays, Version.CURRENT,
             new NamedWriteableRegistry(), new NoneCircuitBreakerService());
diff --git a/test/framework/src/main/java/org/elasticsearch/cache/recycler/MockPageCacheRecycler.java b/test/framework/src/main/java/org/elasticsearch/cache/recycler/MockPageCacheRecycler.java
index 80b2c3279bf..ae7fc9142ea 100644
--- a/test/framework/src/main/java/org/elasticsearch/cache/recycler/MockPageCacheRecycler.java
+++ b/test/framework/src/main/java/org/elasticsearch/cache/recycler/MockPageCacheRecycler.java
@@ -24,7 +24,6 @@ import org.elasticsearch.common.recycler.Recycler.V;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.threadpool.ThreadPool;
 
 import java.lang.reflect.Array;
 import java.util.Arrays;
@@ -60,8 +59,8 @@ public class MockPageCacheRecycler extends PageCacheRecycler {
     private final Random random;
 
     @Inject
-    public MockPageCacheRecycler(Settings settings, ThreadPool threadPool) {
-        super(settings, threadPool);
+    public MockPageCacheRecycler(Settings settings) {
+        super(settings);
         // we always initialize with 0 here since we really only wanna have some random bytes / ints / longs
         // and given the fact that it's called concurrently it won't reproduces anyway the same order other than in a unittest
         // for the latter 0 is just fine

From 9bfa6139014dde630a50f60807985c5f5b153c6e Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Tue, 31 May 2016 14:20:21 -0700
Subject: [PATCH 0537/1311] Build: Rework eclipse settings copy so it does not
 get automatically cleaned

Gradle has "rules" for certain task names, and clean is one of these.
When you run clean, it searches for any tasks named cleanX, and tries to
reverse engineer the X task. For eclipse, this means it finds
cleanEclipse, and internally runs it (but this does not show up as a
dependency of clean in tasks list!!). Since we added .settings as an
additional file to delete with cleanEclipse, this gets deleted when
running just "clean". It doesn't delete the other files because those
have their own clean methods, and dependencies are not followed in this
insanity. This change simply makes a separate task for cleaning eclipse
settings.
---
 build.gradle | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index b52d299ad94..5c6ec366930 100644
--- a/build.gradle
+++ b/build.gradle
@@ -303,9 +303,10 @@ allprojects {
     into '.settings'
   }
   // otherwise .settings is not nuked entirely
-  tasks.cleanEclipse {
+  task wipeEclipseSettings(type: Delete) {
     delete '.settings'
   }
+  tasks.cleanEclipse.dependsOn(wipeEclipseSettings)
   // otherwise the eclipse merging is *super confusing*
   tasks.eclipse.dependsOn(cleanEclipse, copyEclipseSettings)
 }

From 0efac76f01da120084ab6d9a5768f97f06917919 Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Tue, 31 May 2016 19:22:55 -0400
Subject: [PATCH 0538/1311] Clarify the semantics of the BlobContainer
 interface

This commit clarifies the behavior that must be adhered to by any
implementors of the BlobContainer interface.  This is done through
expanded Javadocs.

Closes #18157
Closes #15580
---
 .../common/blobstore/BlobContainer.java       | 102 +++++++++++++++---
 .../common/blobstore/BlobMetaData.java        |   8 +-
 .../common/blobstore/BlobPath.java            |   3 +-
 .../common/blobstore/BlobStore.java           |   8 +-
 .../common/blobstore/fs/FsBlobContainer.java  |   5 +
 .../support/AbstractBlobContainer.java        |   6 +-
 .../snapshots/BlobStoreFormatIT.java          |   1 +
 .../repositories/hdfs/HdfsBlobContainer.java  |   2 +-
 .../ESBlobStoreContainerTestCase.java         |  14 +++
 9 files changed, 129 insertions(+), 20 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java b/core/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java
index 5d00e36ddbd..10f5154d863 100644
--- a/core/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java
+++ b/core/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java
@@ -27,60 +27,138 @@ import java.util.Collection;
 import java.util.Map;
 
 /**
- *
+ * An interface for managing a repository of blob entries, where each blob entry is just a named group of bytes.
  */
 public interface BlobContainer {
 
+    /**
+     * Gets the {@link BlobPath} that defines the implementation specific paths to where the blobs are contained.
+     *
+     * @return  the BlobPath where the blobs are contained
+     */
     BlobPath path();
 
+    /**
+     * Tests whether a blob with the given blob name exists in the container.
+     *
+     * @param   blobName
+     *          The name of the blob whose existence is to be determined.
+     * @return  {@code true} if a blob exists in the {@link BlobContainer} with the given name, and {@code false} otherwise.
+     * @throws  IOException if any error occurred while attempting to ascertain if the blob exists
+     */
     boolean blobExists(String blobName);
 
     /**
-     * Creates a new InputStream for the given blob name
+     * Creates a new {@link InputStream} for the given blob name.
+     *
+     * @param   blobName
+     *          The name of the blob to get an {@link InputStream} for.
+     * @return  The {@code InputStream} to read the blob.
+     * @throws  IOException if the blob does not exist or can not be read.
      */
     InputStream readBlob(String blobName) throws IOException;
 
     /**
-     * Reads blob content from the input stream and writes it to the blob store
+     * Reads blob content from the input stream and writes it to the container in a new blob with the given name.
+     * This method assumes the container does not already contain a blob of the same blobName.  If a blob by the
+     * same name already exists, the operation will fail and an {@link IOException} will be thrown.
+     *
+     * @param   blobName
+     *          The name of the blob to write the contents of the input stream to.
+     * @param   inputStream
+     *          The input stream from which to retrieve the bytes to write to the blob.
+     * @param   blobSize
+     *          The size of the blob to be written, in bytes.  It is implementation dependent whether
+     *          this value is used in writing the blob to the repository.
+     * @throws  IOException if the input stream could not be read, a blob by the same name already exists,
+     *          or the target blob could not be written to.
      */
     void writeBlob(String blobName, InputStream inputStream, long blobSize) throws IOException;
 
     /**
-     * Writes bytes to the blob
+     * Writes the input bytes to a new blob in the container with the given name.  This method assumes the
+     * container does not already contain a blob of the same blobName.  If a blob by the same name already
+     * exists, the operation will fail and an {@link IOException} will be thrown.
+     *
+     * TODO: Remove this in favor of a single {@link #writeBlob(String, InputStream, long)} method.
+     *       See https://github.com/elastic/elasticsearch/issues/18528
+     *
+     * @param   blobName
+     *          The name of the blob to write the contents of the input stream to.
+     * @param   bytes
+     *          The bytes to write to the blob.
+     * @throws  IOException if a blob by the same name already exists, or the target blob could not be written to.
      */
     void writeBlob(String blobName, BytesReference bytes) throws IOException;
 
     /**
-     * Deletes a blob with giving name.
+     * Deletes a blob with giving name, if the blob exists.  If the blob does not exist, this method throws an IOException.
      *
-     * If a blob exists but cannot be deleted an exception has to be thrown.
+     * @param   blobName
+     *          The name of the blob to delete.
+     * @throws  IOException if the blob does not exist, or if the blob exists but could not be deleted.
      */
     void deleteBlob(String blobName) throws IOException;
 
     /**
-     * Deletes blobs with giving names.
+     * Deletes blobs with the given names.  If any subset of the names do not exist in the container, this method has no
+     * effect for those names, and will delete the blobs for those names that do exist.  If any of the blobs failed
+     * to delete, those blobs that were processed before it and successfully deleted will remain deleted.  An exception
+     * is thrown at the first blob entry that fails to delete (TODO: is this the right behavior?  Should we collect
+     * all the failed deletes into a single IOException instead?)
      *
-     * If a blob exists but cannot be deleted an exception has to be thrown.
+     * TODO: remove, see https://github.com/elastic/elasticsearch/issues/18529
+     *
+     * @param   blobNames
+     *          The collection of blob names to delete from the container.
+     * @throws  IOException if any of the blobs in the collection exists but could not be deleted.
      */
     void deleteBlobs(Collection<String> blobNames) throws IOException;
 
     /**
-     * Deletes all blobs in the container that match the specified prefix.
+     * Deletes all blobs in the container that match the specified prefix.  If any of the blobs failed to delete,
+     * those blobs that were processed before it and successfully deleted will remain deleted.  An exception is
+     * thrown at the first blob entry that fails to delete (TODO: is this the right behavior?  Should we collect
+     * all the failed deletes into a single IOException instead?)
+     *
+     * TODO: remove, see: https://github.com/elastic/elasticsearch/issues/18529
+     *
+     * @param   blobNamePrefix
+     *          The prefix to match against blob names in the container.  Any blob whose name has the prefix will be deleted.
+     * @throws  IOException if any of the matching blobs failed to delete.
      */
     void deleteBlobsByPrefix(String blobNamePrefix) throws IOException;
 
     /**
-     * Lists all blobs in the container
+     * Lists all blobs in the container.
+     *
+     * @return  A map of all the blobs in the container.  The keys in the map are the names of the blobs and
+     *          the values are {@link BlobMetaData}, containing basic information about each blob.
+     * @throws  IOException if there were any failures in reading from the blob container.
      */
     Map<String, BlobMetaData> listBlobs() throws IOException;
 
     /**
-     * Lists all blobs in the container that match specified prefix
+     * Lists all blobs in the container that match the specified prefix.
+     *
+     * @param   blobNamePrefix
+     *          The prefix to match against blob names in the container.
+     * @return  A map of the matching blobs in the container.  The keys in the map are the names of the blobs
+     *          and the values are {@link BlobMetaData}, containing basic information about each blob.
+     * @throws  IOException if there were any failures in reading from the blob container.
      */
     Map<String, BlobMetaData> listBlobsByPrefix(String blobNamePrefix) throws IOException;
 
     /**
-     * Atomically renames source blob into target blob
+     * Atomically renames the source blob into the target blob.  If the source blob does not exist or the
+     * target blob already exists, an exception is thrown.
+     *
+     * @param   sourceBlobName
+     *          The blob to rename.
+     * @param   targetBlobName
+     *          The name of the blob after the renaming.
+     * @throws  IOException if the source blob does not exist, the target blob already exists,
+     *          or there were any failures in reading from the blob container.
      */
     void move(String sourceBlobName, String targetBlobName) throws IOException;
 }
diff --git a/core/src/main/java/org/elasticsearch/common/blobstore/BlobMetaData.java b/core/src/main/java/org/elasticsearch/common/blobstore/BlobMetaData.java
index 3f69e268034..da6c277aa2a 100644
--- a/core/src/main/java/org/elasticsearch/common/blobstore/BlobMetaData.java
+++ b/core/src/main/java/org/elasticsearch/common/blobstore/BlobMetaData.java
@@ -20,11 +20,17 @@
 package org.elasticsearch.common.blobstore;
 
 /**
- *
+ * An interface for providing basic metadata about a blob.
  */
 public interface BlobMetaData {
 
+    /**
+     * Gets the name of the blob.
+     */
     String name();
 
+    /**
+     * Gets the size of the blob in bytes.
+     */
     long length();
 }
diff --git a/core/src/main/java/org/elasticsearch/common/blobstore/BlobPath.java b/core/src/main/java/org/elasticsearch/common/blobstore/BlobPath.java
index 858486d282c..9092e13eb1b 100644
--- a/core/src/main/java/org/elasticsearch/common/blobstore/BlobPath.java
+++ b/core/src/main/java/org/elasticsearch/common/blobstore/BlobPath.java
@@ -19,14 +19,13 @@
 
 package org.elasticsearch.common.blobstore;
 
-
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
 /**
- *
+ * The list of paths where a blob can reside.  The contents of the paths are dependent upon the implementation of {@link BlobContainer}.
  */
 public class BlobPath implements Iterable<String> {
 
diff --git a/core/src/main/java/org/elasticsearch/common/blobstore/BlobStore.java b/core/src/main/java/org/elasticsearch/common/blobstore/BlobStore.java
index 9275b379158..e4cdb148a15 100644
--- a/core/src/main/java/org/elasticsearch/common/blobstore/BlobStore.java
+++ b/core/src/main/java/org/elasticsearch/common/blobstore/BlobStore.java
@@ -22,12 +22,18 @@ import java.io.Closeable;
 import java.io.IOException;
 
 /**
- *
+ * An interface for storing blobs.
  */
 public interface BlobStore extends Closeable {
 
+    /**
+     * Get a blob container instance for storing blobs at the given {@link BlobPath}.
+     */
     BlobContainer blobContainer(BlobPath path);
 
+    /**
+     * Delete the blob store at the given {@link BlobPath}.
+     */
     void delete(BlobPath path) throws IOException;
 
 }
diff --git a/core/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java b/core/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java
index c62166a23a3..822f8d1721a 100644
--- a/core/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java
+++ b/core/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java
@@ -41,7 +41,12 @@ import java.util.Map;
 import static java.util.Collections.unmodifiableMap;
 
 /**
+ * A file system based implementation of {@link org.elasticsearch.common.blobstore.BlobContainer}.
+ * All blobs in the container are stored on a file system, the location of which is specified by the {@link BlobPath}.
  *
+ * Note that the methods in this implementation of {@link org.elasticsearch.common.blobstore.BlobContainer} may
+ * additionally throw a {@link java.lang.SecurityException} if the configured {@link java.lang.SecurityManager}
+ * does not permit read and/or write access to the underlying files.
  */
 public class FsBlobContainer extends AbstractBlobContainer {
 
diff --git a/core/src/main/java/org/elasticsearch/common/blobstore/support/AbstractBlobContainer.java b/core/src/main/java/org/elasticsearch/common/blobstore/support/AbstractBlobContainer.java
index 8f83bbf8098..9929de2674d 100644
--- a/core/src/main/java/org/elasticsearch/common/blobstore/support/AbstractBlobContainer.java
+++ b/core/src/main/java/org/elasticsearch/common/blobstore/support/AbstractBlobContainer.java
@@ -30,7 +30,7 @@ import java.util.Collection;
 import java.util.Map;
 
 /**
- *
+ * A base abstract blob container that implements higher level container methods.
  */
 public abstract class AbstractBlobContainer implements BlobContainer {
 
@@ -55,11 +55,11 @@ public abstract class AbstractBlobContainer implements BlobContainer {
 
     @Override
     public void deleteBlobs(Collection<String> blobNames) throws IOException {
-        for(String blob: blobNames) {
+        for (String blob: blobNames) {
             deleteBlob(blob);
         }
     }
-    
+
     @Override
     public void writeBlob(String blobName, BytesReference bytes) throws IOException {
         try (InputStream stream = bytes.streamInput()) {
diff --git a/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java b/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java
index 666ef9dfe39..b2b9e780205 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java
@@ -283,6 +283,7 @@ public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase {
             int location = randomIntBetween(0, buffer.length - 1);
             buffer[location] = (byte) (buffer[location] ^ 42);
         } while (originalChecksum == checksum(buffer));
+        blobContainer.deleteBlob(blobName); // delete original before writing new blob
         blobContainer.writeBlob(blobName, new BytesArray(buffer));
     }
 
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
index 135e2f77810..6ba726e2b24 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
@@ -154,4 +154,4 @@ final class HdfsBlobContainer extends AbstractBlobContainer {
     public Map<String, BlobMetaData> listBlobs() throws IOException {
         return listBlobsByPrefix(null);
     }
-}
\ No newline at end of file
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java
index 8462cf007f0..6ff0b71cdcc 100644
--- a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java
@@ -112,5 +112,19 @@ public abstract class ESBlobStoreContainerTestCase extends ESTestCase {
         }
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/15579")
+    public void testOverwriteFails() throws IOException {
+        try (final BlobStore store = newBlobStore()) {
+            final String blobName = "foobar";
+            final BlobContainer container = store.blobContainer(new BlobPath());
+            byte[] data = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16)));
+            final BytesArray bytesArray = new BytesArray(data);
+            container.writeBlob(blobName, bytesArray);
+            expectThrows(IOException.class, () -> container.writeBlob(blobName, bytesArray));
+            container.deleteBlob(blobName);
+            container.writeBlob(blobName, bytesArray); // deleted it, so should be able to write it again
+        }
+    }
+
     protected abstract BlobStore newBlobStore() throws IOException;
 }

From dac322a32d21ab169c289858ba33d6f3ad097713 Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Tue, 31 May 2016 19:56:55 -0400
Subject: [PATCH 0539/1311] Fix javadoc that stated a throws clause that didn't
 exist.

---
 .../java/org/elasticsearch/common/blobstore/BlobContainer.java   | 1 -
 1 file changed, 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java b/core/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java
index 10f5154d863..46b7b6fc68f 100644
--- a/core/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java
+++ b/core/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java
@@ -44,7 +44,6 @@ public interface BlobContainer {
      * @param   blobName
      *          The name of the blob whose existence is to be determined.
      * @return  {@code true} if a blob exists in the {@link BlobContainer} with the given name, and {@code false} otherwise.
-     * @throws  IOException if any error occurred while attempting to ascertain if the blob exists
      */
     boolean blobExists(String blobName);
 

From f12b10c48a729a3ff30ace34b9b8574632b33ccf Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Tue, 31 May 2016 23:03:50 -0400
Subject: [PATCH 0540/1311] Make cluster health classes immutable and have them
 implement Writeable instead of Streamable

Closes #18673
---
 .../cluster/health/ClusterHealthResponse.java |  72 ++++----
 .../cluster/health/ClusterIndexHealth.java    | 170 ++++++++----------
 .../cluster/health/ClusterShardHealth.java    |  91 +++++-----
 .../cluster/health/ClusterStateHealth.java    | 112 ++++++------
 .../health/ClusterStateHealthTests.java       |   2 +-
 5 files changed, 214 insertions(+), 233 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java
index e336576368c..0d3bfb970af 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java
@@ -182,7 +182,7 @@ public class ClusterHealthResponse extends ActionResponse implements StatusToXCo
         super.readFrom(in);
         clusterName = in.readString();
         clusterHealthStatus = ClusterHealthStatus.fromValue(in.readByte());
-        clusterStateHealth = ClusterStateHealth.readClusterHealth(in);
+        clusterStateHealth = new ClusterStateHealth(in);
         numberOfPendingTasks = in.readInt();
         timedOut = in.readBoolean();
         numberOfInFlightFetch = in.readInt();
@@ -222,50 +222,48 @@ public class ClusterHealthResponse extends ActionResponse implements StatusToXCo
         return isTimedOut() ? RestStatus.REQUEST_TIMEOUT : RestStatus.OK;
     }
 
-    static final class Fields {
-        static final String CLUSTER_NAME = "cluster_name";
-        static final String STATUS = "status";
-        static final String TIMED_OUT = "timed_out";
-        static final String NUMBER_OF_NODES = "number_of_nodes";
-        static final String NUMBER_OF_DATA_NODES = "number_of_data_nodes";
-        static final String NUMBER_OF_PENDING_TASKS = "number_of_pending_tasks";
-        static final String NUMBER_OF_IN_FLIGHT_FETCH = "number_of_in_flight_fetch";
-        static final String DELAYED_UNASSIGNED_SHARDS = "delayed_unassigned_shards";
-        static final String TASK_MAX_WAIT_TIME_IN_QUEUE = "task_max_waiting_in_queue";
-        static final String TASK_MAX_WAIT_TIME_IN_QUEUE_IN_MILLIS = "task_max_waiting_in_queue_millis";
-        static final String ACTIVE_SHARDS_PERCENT_AS_NUMBER = "active_shards_percent_as_number";
-        static final String ACTIVE_SHARDS_PERCENT = "active_shards_percent";
-        static final String ACTIVE_PRIMARY_SHARDS = "active_primary_shards";
-        static final String ACTIVE_SHARDS = "active_shards";
-        static final String RELOCATING_SHARDS = "relocating_shards";
-        static final String INITIALIZING_SHARDS = "initializing_shards";
-        static final String UNASSIGNED_SHARDS = "unassigned_shards";
-        static final String INDICES = "indices";
-    }
+    private static final String CLUSTER_NAME = "cluster_name";
+    private static final String STATUS = "status";
+    private static final String TIMED_OUT = "timed_out";
+    private static final String NUMBER_OF_NODES = "number_of_nodes";
+    private static final String NUMBER_OF_DATA_NODES = "number_of_data_nodes";
+    private static final String NUMBER_OF_PENDING_TASKS = "number_of_pending_tasks";
+    private static final String NUMBER_OF_IN_FLIGHT_FETCH = "number_of_in_flight_fetch";
+    private static final String DELAYED_UNASSIGNED_SHARDS = "delayed_unassigned_shards";
+    private static final String TASK_MAX_WAIT_TIME_IN_QUEUE = "task_max_waiting_in_queue";
+    private static final String TASK_MAX_WAIT_TIME_IN_QUEUE_IN_MILLIS = "task_max_waiting_in_queue_millis";
+    private static final String ACTIVE_SHARDS_PERCENT_AS_NUMBER = "active_shards_percent_as_number";
+    private static final String ACTIVE_SHARDS_PERCENT = "active_shards_percent";
+    private static final String ACTIVE_PRIMARY_SHARDS = "active_primary_shards";
+    private static final String ACTIVE_SHARDS = "active_shards";
+    private static final String RELOCATING_SHARDS = "relocating_shards";
+    private static final String INITIALIZING_SHARDS = "initializing_shards";
+    private static final String UNASSIGNED_SHARDS = "unassigned_shards";
+    private static final String INDICES = "indices";
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(Fields.CLUSTER_NAME, getClusterName());
-        builder.field(Fields.STATUS, getStatus().name().toLowerCase(Locale.ROOT));
-        builder.field(Fields.TIMED_OUT, isTimedOut());
-        builder.field(Fields.NUMBER_OF_NODES, getNumberOfNodes());
-        builder.field(Fields.NUMBER_OF_DATA_NODES, getNumberOfDataNodes());
-        builder.field(Fields.ACTIVE_PRIMARY_SHARDS, getActivePrimaryShards());
-        builder.field(Fields.ACTIVE_SHARDS, getActiveShards());
-        builder.field(Fields.RELOCATING_SHARDS, getRelocatingShards());
-        builder.field(Fields.INITIALIZING_SHARDS, getInitializingShards());
-        builder.field(Fields.UNASSIGNED_SHARDS, getUnassignedShards());
-        builder.field(Fields.DELAYED_UNASSIGNED_SHARDS, getDelayedUnassignedShards());
-        builder.field(Fields.NUMBER_OF_PENDING_TASKS, getNumberOfPendingTasks());
-        builder.field(Fields.NUMBER_OF_IN_FLIGHT_FETCH, getNumberOfInFlightFetch());
-        builder.timeValueField(Fields.TASK_MAX_WAIT_TIME_IN_QUEUE_IN_MILLIS, Fields.TASK_MAX_WAIT_TIME_IN_QUEUE, getTaskMaxWaitingTime());
-        builder.percentageField(Fields.ACTIVE_SHARDS_PERCENT_AS_NUMBER, Fields.ACTIVE_SHARDS_PERCENT, getActiveShardsPercent());
+        builder.field(CLUSTER_NAME, getClusterName());
+        builder.field(STATUS, getStatus().name().toLowerCase(Locale.ROOT));
+        builder.field(TIMED_OUT, isTimedOut());
+        builder.field(NUMBER_OF_NODES, getNumberOfNodes());
+        builder.field(NUMBER_OF_DATA_NODES, getNumberOfDataNodes());
+        builder.field(ACTIVE_PRIMARY_SHARDS, getActivePrimaryShards());
+        builder.field(ACTIVE_SHARDS, getActiveShards());
+        builder.field(RELOCATING_SHARDS, getRelocatingShards());
+        builder.field(INITIALIZING_SHARDS, getInitializingShards());
+        builder.field(UNASSIGNED_SHARDS, getUnassignedShards());
+        builder.field(DELAYED_UNASSIGNED_SHARDS, getDelayedUnassignedShards());
+        builder.field(NUMBER_OF_PENDING_TASKS, getNumberOfPendingTasks());
+        builder.field(NUMBER_OF_IN_FLIGHT_FETCH, getNumberOfInFlightFetch());
+        builder.timeValueField(TASK_MAX_WAIT_TIME_IN_QUEUE_IN_MILLIS, TASK_MAX_WAIT_TIME_IN_QUEUE, getTaskMaxWaitingTime());
+        builder.percentageField(ACTIVE_SHARDS_PERCENT_AS_NUMBER, ACTIVE_SHARDS_PERCENT, getActiveShardsPercent());
 
         String level = params.param("level", "cluster");
         boolean outputIndices = "indices".equals(level) || "shards".equals(level);
 
         if (outputIndices) {
-            builder.startObject(Fields.INDICES);
+            builder.startObject(INDICES);
             for (ClusterIndexHealth indexHealth : clusterStateHealth.getIndices().values()) {
                 builder.startObject(indexHealth.getIndex());
                 indexHealth.toXContent(builder, params);
diff --git a/core/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java b/core/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java
index 9116e9689e8..9bc483ace68 100644
--- a/core/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java
+++ b/core/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java
@@ -24,7 +24,7 @@ import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
@@ -34,34 +34,20 @@ import java.util.Iterator;
 import java.util.Locale;
 import java.util.Map;
 
-import static org.elasticsearch.cluster.health.ClusterShardHealth.readClusterShardHealth;
-
-public final class ClusterIndexHealth implements Iterable<ClusterShardHealth>, Streamable, ToXContent {
-
-    private String index;
-
-    private int numberOfShards;
-
-    private int numberOfReplicas;
-
-    private int activeShards = 0;
-
-    private int relocatingShards = 0;
-
-    private int initializingShards = 0;
-
-    private int unassignedShards = 0;
-
-    private int activePrimaryShards = 0;
-
-    private ClusterHealthStatus status = ClusterHealthStatus.RED;
+public final class ClusterIndexHealth implements Iterable<ClusterShardHealth>, Writeable, ToXContent {
 
+    private final String index;
+    private final int numberOfShards;
+    private final int numberOfReplicas;
+    private final int activeShards;
+    private final int relocatingShards;
+    private final int initializingShards;
+    private final int unassignedShards;
+    private final int activePrimaryShards;
+    private final ClusterHealthStatus status;
     private final Map<Integer, ClusterShardHealth> shards = new HashMap<>();
 
-    private ClusterIndexHealth() {
-    }
-
-    public ClusterIndexHealth(IndexMetaData indexMetaData, IndexRoutingTable indexRoutingTable) {
+    public ClusterIndexHealth(final IndexMetaData indexMetaData, final IndexRoutingTable indexRoutingTable) {
         this.index = indexMetaData.getIndex().getName();
         this.numberOfShards = indexMetaData.getNumberOfShards();
         this.numberOfReplicas = indexMetaData.getNumberOfReplicas();
@@ -72,26 +58,55 @@ public final class ClusterIndexHealth implements Iterable<ClusterShardHealth>, S
         }
 
         // update the index status
-        status = ClusterHealthStatus.GREEN;
-
+        ClusterHealthStatus computeStatus = ClusterHealthStatus.GREEN;
+        int computeActivePrimaryShards = 0;
+        int computeActiveShards = 0;
+        int computeRelocatingShards = 0;
+        int computeInitializingShards = 0;
+        int computeUnassignedShards = 0;
         for (ClusterShardHealth shardHealth : shards.values()) {
             if (shardHealth.isPrimaryActive()) {
-                activePrimaryShards++;
+                computeActivePrimaryShards++;
             }
-            activeShards += shardHealth.getActiveShards();
-            relocatingShards += shardHealth.getRelocatingShards();
-            initializingShards += shardHealth.getInitializingShards();
-            unassignedShards += shardHealth.getUnassignedShards();
+            computeActiveShards += shardHealth.getActiveShards();
+            computeRelocatingShards += shardHealth.getRelocatingShards();
+            computeInitializingShards += shardHealth.getInitializingShards();
+            computeUnassignedShards += shardHealth.getUnassignedShards();
 
             if (shardHealth.getStatus() == ClusterHealthStatus.RED) {
-                status = ClusterHealthStatus.RED;
-            } else if (shardHealth.getStatus() == ClusterHealthStatus.YELLOW && status != ClusterHealthStatus.RED) {
+                computeStatus = ClusterHealthStatus.RED;
+            } else if (shardHealth.getStatus() == ClusterHealthStatus.YELLOW && computeStatus != ClusterHealthStatus.RED) {
                 // do not override an existing red
-                status = ClusterHealthStatus.YELLOW;
+                computeStatus = ClusterHealthStatus.YELLOW;
             }
         }
         if (shards.isEmpty()) { // might be since none has been created yet (two phase index creation)
-            status = ClusterHealthStatus.RED;
+            computeStatus = ClusterHealthStatus.RED;
+        }
+
+        this.status = computeStatus;
+        this.activePrimaryShards = computeActivePrimaryShards;
+        this.activeShards = computeActiveShards;
+        this.relocatingShards = computeRelocatingShards;
+        this.initializingShards = computeInitializingShards;
+        this.unassignedShards = computeUnassignedShards;
+    }
+
+    public ClusterIndexHealth(final StreamInput in) throws IOException {
+        index = in.readString();
+        numberOfShards = in.readVInt();
+        numberOfReplicas = in.readVInt();
+        activePrimaryShards = in.readVInt();
+        activeShards = in.readVInt();
+        relocatingShards = in.readVInt();
+        initializingShards = in.readVInt();
+        unassignedShards = in.readVInt();
+        status = ClusterHealthStatus.fromValue(in.readByte());
+
+        int size = in.readVInt();
+        for (int i = 0; i < size; i++) {
+            ClusterShardHealth shardHealth = new ClusterShardHealth(in);
+            shards.put(shardHealth.getId(), shardHealth);
         }
     }
 
@@ -140,33 +155,8 @@ public final class ClusterIndexHealth implements Iterable<ClusterShardHealth>, S
         return shards.values().iterator();
     }
 
-    public static ClusterIndexHealth readClusterIndexHealth(StreamInput in) throws IOException {
-        ClusterIndexHealth indexHealth = new ClusterIndexHealth();
-        indexHealth.readFrom(in);
-        return indexHealth;
-    }
-
     @Override
-    public void readFrom(StreamInput in) throws IOException {
-        index = in.readString();
-        numberOfShards = in.readVInt();
-        numberOfReplicas = in.readVInt();
-        activePrimaryShards = in.readVInt();
-        activeShards = in.readVInt();
-        relocatingShards = in.readVInt();
-        initializingShards = in.readVInt();
-        unassignedShards = in.readVInt();
-        status = ClusterHealthStatus.fromValue(in.readByte());
-
-        int size = in.readVInt();
-        for (int i = 0; i < size; i++) {
-            ClusterShardHealth shardHealth = readClusterShardHealth(in);
-            shards.put(shardHealth.getId(), shardHealth);
-        }
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
+    public void writeTo(final StreamOutput out) throws IOException {
         out.writeString(index);
         out.writeVInt(numberOfShards);
         out.writeVInt(numberOfReplicas);
@@ -183,42 +173,40 @@ public final class ClusterIndexHealth implements Iterable<ClusterShardHealth>, S
         }
     }
 
-    static final class Fields {
-        static final String STATUS = "status";
-        static final String NUMBER_OF_SHARDS = "number_of_shards";
-        static final String NUMBER_OF_REPLICAS = "number_of_replicas";
-        static final String ACTIVE_PRIMARY_SHARDS = "active_primary_shards";
-        static final String ACTIVE_SHARDS = "active_shards";
-        static final String RELOCATING_SHARDS = "relocating_shards";
-        static final String INITIALIZING_SHARDS = "initializing_shards";
-        static final String UNASSIGNED_SHARDS = "unassigned_shards";
-        static final String SHARDS = "shards";
-        static final String PRIMARY_ACTIVE = "primary_active";
-    }
+    private static final String STATUS = "status";
+    private static final String NUMBER_OF_SHARDS = "number_of_shards";
+    private static final String NUMBER_OF_REPLICAS = "number_of_replicas";
+    private static final String ACTIVE_PRIMARY_SHARDS = "active_primary_shards";
+    private static final String ACTIVE_SHARDS = "active_shards";
+    private static final String RELOCATING_SHARDS = "relocating_shards";
+    private static final String INITIALIZING_SHARDS = "initializing_shards";
+    private static final String UNASSIGNED_SHARDS = "unassigned_shards";
+    private static final String SHARDS = "shards";
+    private static final String PRIMARY_ACTIVE = "primary_active";
 
     @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(Fields.STATUS, getStatus().name().toLowerCase(Locale.ROOT));
-        builder.field(Fields.NUMBER_OF_SHARDS, getNumberOfShards());
-        builder.field(Fields.NUMBER_OF_REPLICAS, getNumberOfReplicas());
-        builder.field(Fields.ACTIVE_PRIMARY_SHARDS, getActivePrimaryShards());
-        builder.field(Fields.ACTIVE_SHARDS, getActiveShards());
-        builder.field(Fields.RELOCATING_SHARDS, getRelocatingShards());
-        builder.field(Fields.INITIALIZING_SHARDS, getInitializingShards());
-        builder.field(Fields.UNASSIGNED_SHARDS, getUnassignedShards());
+    public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
+        builder.field(STATUS, getStatus().name().toLowerCase(Locale.ROOT));
+        builder.field(NUMBER_OF_SHARDS, getNumberOfShards());
+        builder.field(NUMBER_OF_REPLICAS, getNumberOfReplicas());
+        builder.field(ACTIVE_PRIMARY_SHARDS, getActivePrimaryShards());
+        builder.field(ACTIVE_SHARDS, getActiveShards());
+        builder.field(RELOCATING_SHARDS, getRelocatingShards());
+        builder.field(INITIALIZING_SHARDS, getInitializingShards());
+        builder.field(UNASSIGNED_SHARDS, getUnassignedShards());
 
         if ("shards".equals(params.param("level", "indices"))) {
-            builder.startObject(Fields.SHARDS);
+            builder.startObject(SHARDS);
 
             for (ClusterShardHealth shardHealth : shards.values()) {
                 builder.startObject(Integer.toString(shardHealth.getId()));
 
-                builder.field(Fields.STATUS, shardHealth.getStatus().name().toLowerCase(Locale.ROOT));
-                builder.field(Fields.PRIMARY_ACTIVE, shardHealth.isPrimaryActive());
-                builder.field(Fields.ACTIVE_SHARDS, shardHealth.getActiveShards());
-                builder.field(Fields.RELOCATING_SHARDS, shardHealth.getRelocatingShards());
-                builder.field(Fields.INITIALIZING_SHARDS, shardHealth.getInitializingShards());
-                builder.field(Fields.UNASSIGNED_SHARDS, shardHealth.getUnassignedShards());
+                builder.field(STATUS, shardHealth.getStatus().name().toLowerCase(Locale.ROOT));
+                builder.field(PRIMARY_ACTIVE, shardHealth.isPrimaryActive());
+                builder.field(ACTIVE_SHARDS, shardHealth.getActiveShards());
+                builder.field(RELOCATING_SHARDS, shardHealth.getRelocatingShards());
+                builder.field(INITIALIZING_SHARDS, shardHealth.getInitializingShards());
+                builder.field(UNASSIGNED_SHARDS, shardHealth.getUnassignedShards());
 
                 builder.endObject();
             }
diff --git a/core/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java b/core/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java
index 725f89121b7..2c3479b94d8 100644
--- a/core/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java
+++ b/core/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java
@@ -23,57 +23,69 @@ import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.io.stream.Writeable;
 
 import java.io.IOException;
 
-public final class ClusterShardHealth implements Streamable {
+public final class ClusterShardHealth implements Writeable {
 
-    private int shardId;
+    private final int shardId;
+    private final ClusterHealthStatus status;
+    private final int activeShards;
+    private final int relocatingShards;
+    private final int initializingShards;
+    private final int unassignedShards;
+    private final boolean primaryActive;
 
-    ClusterHealthStatus status = ClusterHealthStatus.RED;
-
-    private int activeShards = 0;
-
-    private int relocatingShards = 0;
-
-    private int initializingShards = 0;
-
-    private int unassignedShards = 0;
-
-    private boolean primaryActive = false;
-
-    private ClusterShardHealth() {
-
-    }
-
-    public ClusterShardHealth(int shardId, final IndexShardRoutingTable shardRoutingTable) {
+    public ClusterShardHealth(final int shardId, final IndexShardRoutingTable shardRoutingTable) {
         this.shardId = shardId;
+        int computeActiveShards = 0;
+        int computeRelocatingShards = 0;
+        int computeInitializingShards = 0;
+        int computeUnassignedShards = 0;
+        boolean computePrimaryActive = false;
         for (ShardRouting shardRouting : shardRoutingTable) {
             if (shardRouting.active()) {
-                activeShards++;
+                computeActiveShards++;
                 if (shardRouting.relocating()) {
                     // the shard is relocating, the one it is relocating to will be in initializing state, so we don't count it
-                    relocatingShards++;
+                    computeRelocatingShards++;
                 }
                 if (shardRouting.primary()) {
-                    primaryActive = true;
+                    computePrimaryActive = true;
                 }
             } else if (shardRouting.initializing()) {
-                initializingShards++;
+                computeInitializingShards++;
             } else if (shardRouting.unassigned()) {
-                unassignedShards++;
+                computeUnassignedShards++;
             }
         }
-        if (primaryActive) {
-            if (activeShards == shardRoutingTable.size()) {
-                status = ClusterHealthStatus.GREEN;
+        ClusterHealthStatus computeStatus;
+        if (computePrimaryActive) {
+            if (computeActiveShards == shardRoutingTable.size()) {
+                computeStatus = ClusterHealthStatus.GREEN;
             } else {
-                status = ClusterHealthStatus.YELLOW;
+                computeStatus = ClusterHealthStatus.YELLOW;
             }
         } else {
-            status = ClusterHealthStatus.RED;
+            computeStatus = ClusterHealthStatus.RED;
         }
+        this.status = computeStatus;
+        this.activeShards = computeActiveShards;
+        this.relocatingShards = computeRelocatingShards;
+        this.initializingShards = computeInitializingShards;
+        this.unassignedShards = computeUnassignedShards;
+        this.primaryActive = computePrimaryActive;
+    }
+
+    public ClusterShardHealth(final StreamInput in) throws IOException {
+        shardId = in.readVInt();
+        status = ClusterHealthStatus.fromValue(in.readByte());
+        activeShards = in.readVInt();
+        relocatingShards = in.readVInt();
+        initializingShards = in.readVInt();
+        unassignedShards = in.readVInt();
+        primaryActive = in.readBoolean();
     }
 
     public int getId() {
@@ -104,25 +116,8 @@ public final class ClusterShardHealth implements Streamable {
         return unassignedShards;
     }
 
-    static ClusterShardHealth readClusterShardHealth(StreamInput in) throws IOException {
-        ClusterShardHealth ret = new ClusterShardHealth();
-        ret.readFrom(in);
-        return ret;
-    }
-
     @Override
-    public void readFrom(StreamInput in) throws IOException {
-        shardId = in.readVInt();
-        status = ClusterHealthStatus.fromValue(in.readByte());
-        activeShards = in.readVInt();
-        relocatingShards = in.readVInt();
-        initializingShards = in.readVInt();
-        unassignedShards = in.readVInt();
-        primaryActive = in.readBoolean();
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
+    public void writeTo(final StreamOutput out) throws IOException {
         out.writeVInt(shardId);
         out.writeByte(status.value());
         out.writeVInt(activeShards);
diff --git a/core/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java b/core/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java
index 5855ef9445c..5d267fa142d 100644
--- a/core/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java
+++ b/core/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java
@@ -27,7 +27,7 @@ import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
@@ -37,29 +37,18 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.cluster.health.ClusterIndexHealth.readClusterIndexHealth;
+public final class ClusterStateHealth implements Iterable<ClusterIndexHealth>, Writeable {
 
-public final class ClusterStateHealth implements Iterable<ClusterIndexHealth>, Streamable {
-    private int numberOfNodes = 0;
-    private int numberOfDataNodes = 0;
-    private int activeShards = 0;
-    private int relocatingShards = 0;
-    private int activePrimaryShards = 0;
-    private int initializingShards = 0;
-    private int unassignedShards = 0;
-    private double activeShardsPercent = 100;
-    private ClusterHealthStatus status = ClusterHealthStatus.RED;
-    private Map<String, ClusterIndexHealth> indices = new HashMap<>();
-
-    public static ClusterStateHealth readClusterHealth(StreamInput in) throws IOException {
-        ClusterStateHealth clusterStateHealth = new ClusterStateHealth();
-        clusterStateHealth.readFrom(in);
-        return clusterStateHealth;
-    }
-
-    ClusterStateHealth() {
-        // only intended for serialization
-    }
+    private final int numberOfNodes;
+    private final int numberOfDataNodes;
+    private final int activeShards;
+    private final int relocatingShards;
+    private final int activePrimaryShards;
+    private final int initializingShards;
+    private final int unassignedShards;
+    private final double activeShardsPercent;
+    private final ClusterHealthStatus status;
+    private final Map<String, ClusterIndexHealth> indices = new HashMap<>();
 
     /**
      * Creates a new <code>ClusterStateHealth</code> instance based on cluster meta data and its routing table as a convenience.
@@ -67,7 +56,7 @@ public final class ClusterStateHealth implements Iterable<ClusterIndexHealth>, S
      * @param clusterMetaData Current cluster meta data. Must not be null.
      * @param routingTables   Current routing table. Must not be null.
      */
-    public ClusterStateHealth(MetaData clusterMetaData, RoutingTable routingTables) {
+    public ClusterStateHealth(final MetaData clusterMetaData, final RoutingTable routingTables) {
         this(ClusterState.builder(ClusterName.DEFAULT).metaData(clusterMetaData).routingTable(routingTables).build());
     }
 
@@ -76,7 +65,7 @@ public final class ClusterStateHealth implements Iterable<ClusterIndexHealth>, S
      *
      * @param clusterState The current cluster state. Must not be null.
      */
-    public ClusterStateHealth(ClusterState clusterState) {
+    public ClusterStateHealth(final ClusterState clusterState) {
         this(clusterState, clusterState.metaData().getConcreteAllIndices());
     }
 
@@ -86,7 +75,7 @@ public final class ClusterStateHealth implements Iterable<ClusterIndexHealth>, S
      * @param clusterState    The current cluster state. Must not be null.
      * @param concreteIndices An array of index names to consider. Must not be null but may be empty.
      */
-    public ClusterStateHealth(ClusterState clusterState, String[] concreteIndices) {
+    public ClusterStateHealth(final ClusterState clusterState, final String[] concreteIndices) {
         numberOfNodes = clusterState.nodes().getSize();
         numberOfDataNodes = clusterState.nodes().getDataNodes().size();
 
@@ -102,27 +91,39 @@ public final class ClusterStateHealth implements Iterable<ClusterIndexHealth>, S
             indices.put(indexHealth.getIndex(), indexHealth);
         }
 
-        status = ClusterHealthStatus.GREEN;
+        ClusterHealthStatus computeStatus = ClusterHealthStatus.GREEN;
+        int computeActivePrimaryShards = 0;
+        int computeActiveShards = 0;
+        int computeRelocatingShards = 0;
+        int computeInitializingShards = 0;
+        int computeUnassignedShards = 0;
 
         for (ClusterIndexHealth indexHealth : indices.values()) {
-            activePrimaryShards += indexHealth.getActivePrimaryShards();
-            activeShards += indexHealth.getActiveShards();
-            relocatingShards += indexHealth.getRelocatingShards();
-            initializingShards += indexHealth.getInitializingShards();
-            unassignedShards += indexHealth.getUnassignedShards();
+            computeActivePrimaryShards += indexHealth.getActivePrimaryShards();
+            computeActiveShards += indexHealth.getActiveShards();
+            computeRelocatingShards += indexHealth.getRelocatingShards();
+            computeInitializingShards += indexHealth.getInitializingShards();
+            computeUnassignedShards += indexHealth.getUnassignedShards();
             if (indexHealth.getStatus() == ClusterHealthStatus.RED) {
-                status = ClusterHealthStatus.RED;
-            } else if (indexHealth.getStatus() == ClusterHealthStatus.YELLOW && status != ClusterHealthStatus.RED) {
-                status = ClusterHealthStatus.YELLOW;
+                computeStatus = ClusterHealthStatus.RED;
+            } else if (indexHealth.getStatus() == ClusterHealthStatus.YELLOW && computeStatus != ClusterHealthStatus.RED) {
+                computeStatus = ClusterHealthStatus.YELLOW;
             }
         }
 
         if (clusterState.blocks().hasGlobalBlock(RestStatus.SERVICE_UNAVAILABLE)) {
-            status = ClusterHealthStatus.RED;
+            computeStatus = ClusterHealthStatus.RED;
         }
 
+        this.status = computeStatus;
+        this.activePrimaryShards = computeActivePrimaryShards;
+        this.activeShards = computeActiveShards;
+        this.relocatingShards = computeRelocatingShards;
+        this.initializingShards = computeInitializingShards;
+        this.unassignedShards = computeUnassignedShards;
+
         // shortcut on green
-        if (status.equals(ClusterHealthStatus.GREEN)) {
+        if (computeStatus.equals(ClusterHealthStatus.GREEN)) {
             this.activeShardsPercent = 100;
         } else {
             List<ShardRouting> shardRoutings = clusterState.getRoutingTable().allShards();
@@ -136,6 +137,23 @@ public final class ClusterStateHealth implements Iterable<ClusterIndexHealth>, S
         }
     }
 
+    public ClusterStateHealth(final StreamInput in) throws IOException {
+        activePrimaryShards = in.readVInt();
+        activeShards = in.readVInt();
+        relocatingShards = in.readVInt();
+        initializingShards = in.readVInt();
+        unassignedShards = in.readVInt();
+        numberOfNodes = in.readVInt();
+        numberOfDataNodes = in.readVInt();
+        status = ClusterHealthStatus.fromValue(in.readByte());
+        int size = in.readVInt();
+        for (int i = 0; i < size; i++) {
+            ClusterIndexHealth indexHealth = new ClusterIndexHealth(in);
+            indices.put(indexHealth.getIndex(), indexHealth);
+        }
+        activeShardsPercent = in.readDouble();
+    }
+
     public int getActiveShards() {
         return activeShards;
     }
@@ -182,25 +200,7 @@ public final class ClusterStateHealth implements Iterable<ClusterIndexHealth>, S
     }
 
     @Override
-    public void readFrom(StreamInput in) throws IOException {
-        activePrimaryShards = in.readVInt();
-        activeShards = in.readVInt();
-        relocatingShards = in.readVInt();
-        initializingShards = in.readVInt();
-        unassignedShards = in.readVInt();
-        numberOfNodes = in.readVInt();
-        numberOfDataNodes = in.readVInt();
-        status = ClusterHealthStatus.fromValue(in.readByte());
-        int size = in.readVInt();
-        for (int i = 0; i < size; i++) {
-            ClusterIndexHealth indexHealth = readClusterIndexHealth(in);
-            indices.put(indexHealth.getIndex(), indexHealth);
-        }
-        activeShardsPercent = in.readDouble();
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
+    public void writeTo(final StreamOutput out) throws IOException {
         out.writeVInt(activePrimaryShards);
         out.writeVInt(activeShards);
         out.writeVInt(relocatingShards);
diff --git a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
index 93c33a97cdc..dde35e33993 100644
--- a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
@@ -170,7 +170,7 @@ public class ClusterStateHealthTests extends ESTestCase {
             BytesStreamOutput out = new BytesStreamOutput();
             clusterStateHealth.writeTo(out);
             StreamInput in = StreamInput.wrap(out.bytes());
-            clusterStateHealth = ClusterStateHealth.readClusterHealth(in);
+            clusterStateHealth = new ClusterStateHealth(in);
         }
         return clusterStateHealth;
     }

From 93c5a9dacd04e1d9cc8cf51bf3fbe29f5f2c4e31 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Wed, 1 Jun 2016 09:11:08 +0200
Subject: [PATCH 0541/1311] [TEST] Set BWC version to 5.0.0-SNAP since this is
 it's min compat version

---
 qa/backwards-5.0/build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qa/backwards-5.0/build.gradle b/qa/backwards-5.0/build.gradle
index 70889ad5009..93d361c989c 100644
--- a/qa/backwards-5.0/build.gradle
+++ b/qa/backwards-5.0/build.gradle
@@ -18,6 +18,6 @@ integTest {
   cluster {
     numNodes = 2
     numBwcNodes = 1
-    bwcVersion = "5.0.0-alpha3-SNAPSHOT" // this is the same as the current version until we released the first RC
+    bwcVersion = "5.0.0-SNAPSHOT" // this is the same as the current version until we released the first RC
   }
 }

From fd762911314f6ed38fb5ff9a0aff003c1f93f13c Mon Sep 17 00:00:00 2001
From: Jun Ohtani <johtani@gmail.com>
Date: Thu, 4 Dec 2014 17:42:40 +0900
Subject: [PATCH 0542/1311] Index Template: parse and validate mappings in
 template when template puts

Share applying template with MetaDataCreateIndexService and MetaDataIndexTemplateService
Add some unit test
Collapse addMappingsToMapperService and move it to MapperService
Extract validateTemplate method
use expectThrows in testcase
Add TODO comment

Closes #2415
---
 .../metadata/MetaDataCreateIndexService.java  |  39 ++----
 .../MetaDataIndexTemplateService.java         |  97 ++++++++++----
 .../index/mapper/MapperService.java           |  31 +++++
 .../MetaDataIndexTemplateServiceTests.java    | 110 +++++++++++++++-
 .../MetaDataCreateIndexServiceTests.java      |  38 ++++++
 .../index/mapper/MapperServiceTests.java      |  23 +++-
 .../template/SimpleIndexTemplateIT.java       | 123 ++++++++++--------
 .../SharedClusterSnapshotRestoreIT.java       |  11 +-
 8 files changed, 359 insertions(+), 113 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
index 5532d4788ad..f5539f28e9a 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
@@ -55,9 +55,7 @@ import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
@@ -220,7 +218,7 @@ public class MetaDataCreateIndexService extends AbstractComponent {
                             List<String> templateNames = new ArrayList<>();
 
                             for (Map.Entry<String, String> entry : request.mappings().entrySet()) {
-                                mappings.put(entry.getKey(), parseMapping(entry.getValue()));
+                                mappings.put(entry.getKey(), MapperService.parseMapping(entry.getValue()));
                             }
 
                             for (Map.Entry<String, Custom> entry : request.customs().entrySet()) {
@@ -232,9 +230,9 @@ public class MetaDataCreateIndexService extends AbstractComponent {
                                 templateNames.add(template.getName());
                                 for (ObjectObjectCursor<String, CompressedXContent> cursor : template.mappings()) {
                                     if (mappings.containsKey(cursor.key)) {
-                                        XContentHelper.mergeDefaults(mappings.get(cursor.key), parseMapping(cursor.value.string()));
+                                        XContentHelper.mergeDefaults(mappings.get(cursor.key), MapperService.parseMapping(cursor.value.string()));
                                     } else {
-                                        mappings.put(cursor.key, parseMapping(cursor.value.string()));
+                                        mappings.put(cursor.key, MapperService.parseMapping(cursor.value.string()));
                                     }
                                 }
                                 // handle custom
@@ -315,26 +313,11 @@ public class MetaDataCreateIndexService extends AbstractComponent {
                             createdIndex = indexService.index();
                             // now add the mappings
                             MapperService mapperService = indexService.mapperService();
-                            // first, add the default mapping
-                            if (mappings.containsKey(MapperService.DEFAULT_MAPPING)) {
-                                try {
-                                    mapperService.merge(MapperService.DEFAULT_MAPPING, new CompressedXContent(XContentFactory.jsonBuilder().map(mappings.get(MapperService.DEFAULT_MAPPING)).string()), MapperService.MergeReason.MAPPING_UPDATE, request.updateAllTypes());
-                                } catch (Exception e) {
-                                    removalReason = "failed on parsing default mapping on index creation";
-                                    throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, MapperService.DEFAULT_MAPPING, e.getMessage());
-                                }
-                            }
-                            for (Map.Entry<String, Map<String, Object>> entry : mappings.entrySet()) {
-                                if (entry.getKey().equals(MapperService.DEFAULT_MAPPING)) {
-                                    continue;
-                                }
-                                try {
-                                    // apply the default here, its the first time we parse it
-                                    mapperService.merge(entry.getKey(), new CompressedXContent(XContentFactory.jsonBuilder().map(entry.getValue()).string()), MapperService.MergeReason.MAPPING_UPDATE, request.updateAllTypes());
-                                } catch (Exception e) {
-                                    removalReason = "failed on parsing mappings on index creation";
-                                    throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
-                                }
+                            try {
+                                mapperService.merge(mappings, request.updateAllTypes());
+                            } catch (MapperParsingException mpe) {
+                                removalReason = "failed on parsing default mapping/mappings on index creation";
+                                throw mpe;
                             }
 
                             final QueryShardContext queryShardContext = indexService.newQueryShardContext();
@@ -426,12 +409,6 @@ public class MetaDataCreateIndexService extends AbstractComponent {
                 });
     }
 
-    private Map<String, Object> parseMapping(String mappingSource) throws Exception {
-        try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource)) {
-            return parser.map();
-        }
-    }
-
     private List<IndexTemplateMetaData> findTemplates(CreateIndexClusterStateUpdateRequest request, ClusterState state, IndexTemplateFilter indexTemplateFilter) throws IOException {
         List<IndexTemplateMetaData> templates = new ArrayList<>();
         for (ObjectCursor<IndexTemplateMetaData> cursor : state.metaData().templates().values()) {
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
index 0b3652b3672..167dea5cf42 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.cluster.metadata;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.cluster.ClusterState;
@@ -26,17 +27,25 @@ import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.ValidationException;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.NodeServicesProvider;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.indices.IndexTemplateAlreadyExistsException;
 import org.elasticsearch.indices.IndexTemplateMissingException;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.InvalidIndexTemplateException;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -51,14 +60,18 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
 
     private final ClusterService clusterService;
     private final AliasValidator aliasValidator;
+    private final IndicesService indicesService;
     private final MetaDataCreateIndexService metaDataCreateIndexService;
+    private final NodeServicesProvider nodeServicesProvider;
 
     @Inject
-    public MetaDataIndexTemplateService(Settings settings, ClusterService clusterService, MetaDataCreateIndexService metaDataCreateIndexService, AliasValidator aliasValidator) {
+    public MetaDataIndexTemplateService(Settings settings, ClusterService clusterService, MetaDataCreateIndexService metaDataCreateIndexService, AliasValidator aliasValidator, IndicesService indicesService, NodeServicesProvider nodeServicesProvider) {
         super(settings);
         this.clusterService = clusterService;
         this.aliasValidator = aliasValidator;
+        this.indicesService = indicesService;
         this.metaDataCreateIndexService = metaDataCreateIndexService;
+        this.nodeServicesProvider = nodeServicesProvider;
     }
 
     public void removeTemplates(final RemoveRequest request, final RemoveListener listener) {
@@ -126,28 +139,7 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
             return;
         }
 
-        IndexTemplateMetaData.Builder templateBuilder;
-        try {
-            templateBuilder = IndexTemplateMetaData.builder(request.name);
-            templateBuilder.order(request.order);
-            templateBuilder.template(request.template);
-            templateBuilder.settings(request.settings);
-            for (Map.Entry<String, String> entry : request.mappings.entrySet()) {
-                templateBuilder.putMapping(entry.getKey(), entry.getValue());
-            }
-            for (Alias alias : request.aliases) {
-                AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter())
-                        .indexRouting(alias.indexRouting()).searchRouting(alias.searchRouting()).build();
-                templateBuilder.putAlias(aliasMetaData);
-            }
-            for (Map.Entry<String, IndexMetaData.Custom> entry : request.customs.entrySet()) {
-                templateBuilder.putCustom(entry.getKey(), entry.getValue());
-            }
-        } catch (Throwable e) {
-            listener.onFailure(e);
-            return;
-        }
-        final IndexTemplateMetaData template = templateBuilder.build();
+        final IndexTemplateMetaData.Builder templateBuilder = IndexTemplateMetaData.builder(request.name);
 
         clusterService.submitStateUpdateTask("create-index-template [" + request.name + "], cause [" + request.cause + "]",
                 new ClusterStateUpdateTask(Priority.URGENT) {
@@ -163,10 +155,23 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
             }
 
             @Override
-            public ClusterState execute(ClusterState currentState) {
+            public ClusterState execute(ClusterState currentState) throws Exception {
                 if (request.create && currentState.metaData().templates().containsKey(request.name)) {
                     throw new IndexTemplateAlreadyExistsException(request.name);
                 }
+
+                validateAndAddTemplate(request, templateBuilder, indicesService, nodeServicesProvider, metaDataCreateIndexService);
+
+                for (Alias alias : request.aliases) {
+                    AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter())
+                        .indexRouting(alias.indexRouting()).searchRouting(alias.searchRouting()).build();
+                    templateBuilder.putAlias(aliasMetaData);
+                }
+                for (Map.Entry<String, IndexMetaData.Custom> entry : request.customs.entrySet()) {
+                    templateBuilder.putCustom(entry.getKey(), entry.getValue());
+                }
+                IndexTemplateMetaData template = templateBuilder.build();
+
                 MetaData.Builder builder = MetaData.builder(currentState.metaData()).put(template);
 
                 return ClusterState.builder(currentState).metaData(builder).build();
@@ -174,11 +179,53 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
 
             @Override
             public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                listener.onResponse(new PutResponse(true, template));
+                listener.onResponse(new PutResponse(true, templateBuilder.build()));
             }
         });
     }
 
+    private static void validateAndAddTemplate(final PutRequest request, IndexTemplateMetaData.Builder templateBuilder, IndicesService indicesService,
+                                               NodeServicesProvider nodeServicesProvider, MetaDataCreateIndexService metaDataCreateIndexService) throws Exception {
+        Index createdIndex = null;
+        final String temporaryIndexName = UUIDs.randomBase64UUID();
+        try {
+
+            //create index service for parsing and validating "mappings"
+            Settings dummySettings = Settings.builder()
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(request.settings)
+                .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+                .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
+                .build();
+
+            final IndexMetaData tmpIndexMetadata = IndexMetaData.builder(temporaryIndexName).settings(dummySettings).build();
+            IndexService dummyIndexService = indicesService.createIndex(nodeServicesProvider, tmpIndexMetadata, Collections.emptyList());
+            createdIndex = dummyIndexService.index();
+
+            templateBuilder.order(request.order);
+            templateBuilder.template(request.template);
+            templateBuilder.settings(request.settings);
+
+            Map<String, Map<String, Object>> mappingsForValidation = new HashMap<>();
+            for (Map.Entry<String, String> entry : request.mappings.entrySet()) {
+                try {
+                    templateBuilder.putMapping(entry.getKey(), entry.getValue());
+                } catch (Exception e) {
+                    throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
+                }
+                mappingsForValidation.put(entry.getKey(), MapperService.parseMapping(entry.getValue()));
+            }
+
+            dummyIndexService.mapperService().merge(mappingsForValidation, false);
+
+        } finally {
+            if (createdIndex != null) {
+                indicesService.removeIndex(createdIndex, " created for parsing template mapping");
+            }
+        }
+    }
+
     private void validate(PutRequest request) {
         List<String> validationErrors = new ArrayList<>();
         if (request.name.contains(" ")) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
index adb87929b84..e49846ba6b2 100755
--- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
@@ -29,6 +29,8 @@ import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalysisService;
@@ -175,6 +177,35 @@ public class MapperService extends AbstractIndexComponent {
         return this.documentParser;
     }
 
+    public static Map<String, Object> parseMapping(String mappingSource) throws Exception {
+        try (XContentParser parser = XContentFactory.xContent(mappingSource).createParser(mappingSource)) {
+            return parser.map();
+        }
+    }
+
+    //TODO: make this atomic
+    public void merge(Map<String, Map<String, Object>> mappings, boolean updateAllTypes) throws MapperParsingException {
+        // first, add the default mapping
+        if (mappings.containsKey(DEFAULT_MAPPING)) {
+            try {
+                this.merge(DEFAULT_MAPPING, new CompressedXContent(XContentFactory.jsonBuilder().map(mappings.get(DEFAULT_MAPPING)).string()), MergeReason.MAPPING_UPDATE, updateAllTypes);
+            } catch (Exception e) {
+                throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, DEFAULT_MAPPING, e.getMessage());
+            }
+        }
+        for (Map.Entry<String, Map<String, Object>> entry : mappings.entrySet()) {
+            if (entry.getKey().equals(DEFAULT_MAPPING)) {
+                continue;
+            }
+            try {
+                // apply the default here, its the first time we parse it
+                this.merge(entry.getKey(), new CompressedXContent(XContentFactory.jsonBuilder().map(entry.getValue()).string()), MergeReason.MAPPING_UPDATE, updateAllTypes);
+            } catch (Exception e) {
+                throw new MapperParsingException("Failed to parse mapping [{}]: {}", e, entry.getKey(), e.getMessage());
+            }
+        }
+    }
+
     public DocumentMapper merge(String type, CompressedXContent mappingSource, MergeReason reason, boolean updateAllTypes) {
         if (DEFAULT_MAPPING.equals(type)) {
             // verify we can parse it
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
index 0dcbd82d56a..4c3f21f988d 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
@@ -26,9 +26,14 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
 import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService;
 import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService.PutRequest;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.NodeServicesProvider;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.InvalidIndexTemplateException;
-import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.ESSingleNodeTestCase;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -36,12 +41,14 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
 
-public class MetaDataIndexTemplateServiceTests extends ESTestCase {
+public class MetaDataIndexTemplateServiceTests extends ESSingleNodeTestCase {
     public void testIndexTemplateInvalidNumberOfShards() {
         PutRequest request = new PutRequest("test", "test_shards");
         request.template("test_shards*");
@@ -83,6 +90,68 @@ public class MetaDataIndexTemplateServiceTests extends ESTestCase {
         assertThat(errors.get(0).getMessage(), equalTo("Alias [foobar] cannot be the same as the template pattern [foobar]"));
     }
 
+    public void testIndexTemplateWithValidateEmptyMapping() throws Exception {
+        PutRequest request = new PutRequest("api", "validate_template");
+        request.template("validate_template");
+        request.putMapping("type1", "{}");
+
+        List<Throwable> errors = putTemplateDetail(request);
+        assertThat(errors.size(), equalTo(1));
+        assertThat(errors.get(0), instanceOf(MapperParsingException.class));
+        assertThat(errors.get(0).getMessage(), containsString("malformed mapping no root object found"));
+    }
+
+    public void testIndexTemplateWithValidateMapping() throws Exception {
+        PutRequest request = new PutRequest("api", "validate_template");
+        request.template("te*");
+        request.putMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties")
+            .startObject("field2").field("type", "string").field("analyzer", "custom_1").endObject()
+            .endObject().endObject().endObject().string());
+
+        List<Throwable> errors = putTemplateDetail(request);
+        assertThat(errors.size(), equalTo(1));
+        assertThat(errors.get(0), instanceOf(MapperParsingException.class));
+        assertThat(errors.get(0).getMessage(), containsString("analyzer [custom_1] not found for field [field2]"));
+    }
+
+    public void testBrokenMapping() throws Exception {
+        PutRequest request = new PutRequest("api", "broken_mapping");
+        request.template("te*");
+        request.putMapping("type1", "abcde");
+
+        List<Throwable> errors = putTemplateDetail(request);
+        assertThat(errors.size(), equalTo(1));
+        assertThat(errors.get(0), instanceOf(MapperParsingException.class));
+        assertThat(errors.get(0).getMessage(), containsString("Failed to parse mapping "));
+    }
+
+    public void testBlankMapping() throws Exception {
+        PutRequest request = new PutRequest("api", "blank_mapping");
+        request.template("te*");
+        request.putMapping("type1", "{}");
+
+        List<Throwable> errors = putTemplateDetail(request);
+        assertThat(errors.size(), equalTo(1));
+        assertThat(errors.get(0), instanceOf(MapperParsingException.class));
+        assertThat(errors.get(0).getMessage(), containsString("malformed mapping no root object found"));
+    }
+
+    public void testAliasInvalidFilterInvalidJson() throws Exception {
+        //invalid json: put index template fails
+        PutRequest request = new PutRequest("api", "blank_mapping");
+        request.template("te*");
+        request.putMapping("type1", "{}");
+        Set<Alias> aliases = new HashSet<>();
+        aliases.add(new Alias("invalid_alias").filter("abcde"));
+        request.aliases(aliases);
+
+        List<Throwable> errors = putTemplateDetail(request);
+        assertThat(errors.size(), equalTo(1));
+        assertThat(errors.get(0), instanceOf(IllegalArgumentException.class));
+        assertThat(errors.get(0).getMessage(), equalTo("failed to parse filter for alias [invalid_alias]"));
+    }
+
+
     private static List<Throwable> putTemplate(PutRequest request) {
         MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(
                 Settings.EMPTY,
@@ -94,7 +163,7 @@ public class MetaDataIndexTemplateServiceTests extends ESTestCase {
                 new HashSet<>(),
                 null,
                 null, null);
-        MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(Settings.EMPTY, null, createIndexService, new AliasValidator(Settings.EMPTY));
+        MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(Settings.EMPTY, null, createIndexService, new AliasValidator(Settings.EMPTY), null, null);
 
         final List<Throwable> throwables = new ArrayList<>();
         service.putTemplate(request, new MetaDataIndexTemplateService.PutListener() {
@@ -108,7 +177,42 @@ public class MetaDataIndexTemplateServiceTests extends ESTestCase {
                 throwables.add(t);
             }
         });
+        return throwables;
+    }
 
+    private List<Throwable> putTemplateDetail(PutRequest request) throws Exception {
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        ClusterService clusterService = getInstanceFromNode(ClusterService.class);
+        NodeServicesProvider nodeServicesProvider = getInstanceFromNode(NodeServicesProvider.class);
+        MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(
+            Settings.EMPTY,
+            clusterService,
+            indicesService,
+            null,
+            Version.CURRENT,
+            null,
+            new HashSet<>(),
+            null,
+            nodeServicesProvider,
+            null);
+        MetaDataIndexTemplateService service = new MetaDataIndexTemplateService(
+            Settings.EMPTY, clusterService, createIndexService, new AliasValidator(Settings.EMPTY), indicesService, nodeServicesProvider);
+
+        final List<Throwable> throwables = new ArrayList<>();
+        final CountDownLatch latch = new CountDownLatch(1);
+        service.putTemplate(request, new MetaDataIndexTemplateService.PutListener() {
+            @Override
+            public void onResponse(MetaDataIndexTemplateService.PutResponse response) {
+                latch.countDown();
+            }
+
+            @Override
+            public void onFailure(Throwable t) {
+                throwables.add(t);
+                latch.countDown();
+            }
+        });
+        latch.await();
         return throwables;
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
index 49d1da0ddec..12292eb6598 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.cluster.metadata;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
 import org.elasticsearch.cluster.block.ClusterBlocks;
@@ -31,10 +32,12 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.indices.IndexAlreadyExistsException;
+import org.elasticsearch.indices.InvalidIndexNameException;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.gateway.NoopGatewayAllocator;
 
@@ -43,6 +46,7 @@ import java.util.Collections;
 import java.util.HashSet;
 
 import static java.util.Collections.emptyMap;
+import static org.hamcrest.Matchers.endsWith;
 
 public class MetaDataCreateIndexServiceTests extends ESTestCase {
 
@@ -163,4 +167,38 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
         return new DiscoveryNode(nodeId, DummyTransportAddress.INSTANCE, emptyMap(),
             Collections.unmodifiableSet(new HashSet<>(Arrays.asList(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA))), Version.CURRENT);
     }
+
+    public void testValidateIndexName() throws Exception {
+
+        validateIndexName("index?name", "must not contain the following characters " + Strings.INVALID_FILENAME_CHARS);
+
+        validateIndexName("index#name", "must not contain '#'");
+
+        validateIndexName("_indexname", "must not start with '_'");
+
+        validateIndexName("INDEXNAME", "must be lowercase");
+
+        validateIndexName("..", "must not be '.' or '..'");
+
+    }
+
+    private void validateIndexName(String indexName, String errorMessage) {
+        InvalidIndexNameException e = expectThrows(InvalidIndexNameException.class,
+            () -> getCreateIndexService().validateIndexName(indexName, ClusterState.builder(ClusterName.DEFAULT).build()));
+        assertThat(e.getMessage(), endsWith(errorMessage));
+    }
+
+    private MetaDataCreateIndexService getCreateIndexService() {
+        return new MetaDataCreateIndexService(
+            Settings.EMPTY,
+            null,
+            null,
+            null,
+            Version.CURRENT,
+            null,
+            new HashSet<>(),
+            null,
+            null,
+            null);
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
index e4c24f51c2b..c93c181f860 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
@@ -23,7 +23,9 @@ import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.function.Function;
 
@@ -38,8 +40,9 @@ import org.elasticsearch.index.mapper.core.NumberFieldMapper.NumberFieldType;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
 import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.instanceOf;
-
+import static org.hamcrest.Matchers.startsWith;
 
 public class MapperServiceTests extends ESSingleNodeTestCase {
 
@@ -166,4 +169,22 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
         assertThat(mapperService.unmappedFieldType("string"), instanceOf(KeywordFieldType.class));
     }
 
+
+    public void testMergeWithMap() throws Throwable {
+        IndexService indexService1 = createIndex("index1");
+        MapperService mapperService = indexService1.mapperService();
+        Map<String, Map<String, Object>> mappings = new HashMap<>();
+
+        mappings.put(MapperService.DEFAULT_MAPPING, MapperService.parseMapping("{}"));
+        MapperException e = expectThrows(MapperParsingException.class,
+            () -> mapperService.merge(mappings, false));
+        assertThat(e.getMessage(), startsWith("Failed to parse mapping [" + MapperService.DEFAULT_MAPPING + "]: "));
+
+        mappings.clear();
+        mappings.put("type1", MapperService.parseMapping("{}"));
+
+        e = expectThrows( MapperParsingException.class,
+            () -> mapperService.merge(mappings, false));
+        assertThat(e.getMessage(), startsWith("Failed to parse mapping [type1]: "));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
index 24cb856c326..416dbd19761 100644
--- a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.indices.template;
 
-import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
@@ -32,9 +31,11 @@ import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.indices.IndexTemplateAlreadyExistsException;
 import org.elasticsearch.indices.InvalidAliasNameException;
+import org.elasticsearch.indices.InvalidIndexNameException;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -296,22 +297,15 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
         GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates().get();
         assertThat(response.getIndexTemplates(), empty());
 
-        client().admin().indices().preparePutTemplate("template_1")
+        MapperParsingException e = expectThrows( MapperParsingException.class,
+            () -> client().admin().indices().preparePutTemplate("template_1")
                 .setTemplate("te*")
                 .addMapping("type1", "abcde")
-                .get();
+                .get());
+        assertThat(e.getMessage(), containsString("Failed to parse mapping "));
 
         response = client().admin().indices().prepareGetTemplates().get();
-        assertThat(response.getIndexTemplates(), hasSize(1));
-        assertThat(response.getIndexTemplates().get(0).getMappings().size(), equalTo(1));
-        assertThat(response.getIndexTemplates().get(0).getMappings().get("type1").string(), equalTo("abcde"));
-
-        try {
-            createIndex("test");
-            fail("create index should have failed due to broken index templates mapping");
-        } catch(ElasticsearchParseException e) {
-            //everything fine
-        }
+        assertThat(response.getIndexTemplates(), hasSize(0));
     }
 
     public void testInvalidSettings() throws Exception {
@@ -322,15 +316,12 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
         GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates().get();
         assertThat(response.getIndexTemplates(), empty());
 
-        try {
-            client().admin().indices().preparePutTemplate("template_1")
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> client().admin().indices().preparePutTemplate("template_1")
                 .setTemplate("te*")
                 .setSettings(Settings.builder().put("does_not_exist", "test"))
-                .get();
-            fail();
-        } catch (IllegalArgumentException ex) {
-            assertEquals("unknown setting [index.does_not_exist]", ex.getMessage());
-        }
+                .get());
+        assertEquals("unknown setting [index.does_not_exist]", e.getMessage());
 
         response = client().admin().indices().prepareGetTemplates().get();
         assertEquals(0, response.getIndexTemplates().size());
@@ -490,14 +481,11 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
         assertThat(response.getIndexTemplates().get(0).getAliases().size(), equalTo(1));
         assertThat(response.getIndexTemplates().get(0).getAliases().get("invalid_alias").filter().string(), equalTo("{\"invalid\":{}}"));
 
-        try {
-            createIndex("test");
-            fail("index creation should have failed due to invalid alias filter in matching index template");
-        } catch(IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("failed to parse filter for alias [invalid_alias]"));
-            assertThat(e.getCause(), instanceOf(ParsingException.class));
-            assertThat(e.getCause().getMessage(), equalTo("no [query] registered for [invalid]"));
-        }
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> createIndex("test"));
+        assertThat(e.getMessage(), equalTo("failed to parse filter for alias [invalid_alias]"));
+        assertThat(e.getCause(), instanceOf(ParsingException.class));
+        assertThat(e.getCause().getMessage(), equalTo("no [query] registered for [invalid]"));
     }
 
     public void testAliasInvalidFilterInvalidJson() throws Exception {
@@ -506,11 +494,9 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
                 .setTemplate("te*")
                 .addAlias(new Alias("invalid_alias").filter("abcde"));
 
-        try {
-            putIndexTemplateRequestBuilder.get();
-        } catch(IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("failed to parse filter for alias [invalid_alias]"));
-        }
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> putIndexTemplateRequestBuilder.get());
+        assertThat(e.getMessage(), equalTo("failed to parse filter for alias [invalid_alias]"));
 
         GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates("template_1").get();
         assertThat(response.getIndexTemplates().size(), equalTo(0));
@@ -523,12 +509,9 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
                 .setTemplate("te*")
                 .addAlias(new Alias("index")).get();
 
-        try {
-            createIndex("test");
-            fail("index creation should have failed due to alias with existing index name in matching index template");
-        } catch(InvalidAliasNameException e) {
-            assertThat(e.getMessage(), equalTo("Invalid alias name [index], an index exists with the same name as the alias"));
-        }
+        InvalidAliasNameException e = expectThrows(InvalidAliasNameException.class,
+            () -> createIndex("test"));
+        assertThat(e.getMessage(), equalTo("Invalid alias name [index], an index exists with the same name as the alias"));
     }
 
     public void testAliasEmptyName() throws Exception {
@@ -536,12 +519,9 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
                 .setTemplate("te*")
                 .addAlias(new Alias("  ").indexRouting("1,2,3"));
 
-        try {
-            putIndexTemplateRequestBuilder.get();
-            fail("put template should have failed due to alias with empty name");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("alias name is required"));
-        }
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> putIndexTemplateRequestBuilder.get());
+        assertThat(e.getMessage(), equalTo("alias name is required"));
     }
 
     public void testAliasWithMultipleIndexRoutings() throws Exception {
@@ -549,12 +529,9 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
                 .setTemplate("te*")
                 .addAlias(new Alias("alias").indexRouting("1,2,3"));
 
-        try {
-            putIndexTemplateRequestBuilder.get();
-            fail("put template should have failed due to alias with multiple index routings");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("alias [alias] has several index routing values associated with it"));
-        }
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> putIndexTemplateRequestBuilder.get());
+        assertThat(e.getMessage(), equalTo("alias [alias] has several index routing values associated with it"));
     }
 
     public void testMultipleAliasesPrecedence() throws Exception {
@@ -664,4 +641,48 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
         assertThat(response.getItems()[0].getVersion(), equalTo(1L));
     }
 
+    public void testCombineTemplates() throws Exception{
+        // clean all templates setup by the framework.
+        client().admin().indices().prepareDeleteTemplate("*").get();
+
+        // check get all templates on an empty index.
+        GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates().get();
+        assertThat(response.getIndexTemplates(), empty());
+
+        //Now, a complete mapping with two separated templates is error
+        // base template
+        client().admin().indices().preparePutTemplate("template_1")
+            .setTemplate("*")
+            .setSettings(
+                "    {\n" +
+                    "        \"index\" : {\n" +
+                    "            \"analysis\" : {\n" +
+                    "                \"analyzer\" : {\n" +
+                    "                    \"custom_1\" : {\n" +
+                    "                        \"tokenizer\" : \"whitespace\"\n" +
+                    "                    }\n" +
+                    "                }\n" +
+                    "            }\n" +
+                    "         }\n" +
+                    "    }\n")
+            .get();
+
+        // put template using custom_1 analyzer
+        MapperParsingException e = expectThrows(MapperParsingException.class,
+            () -> client().admin().indices().preparePutTemplate("template_2")
+                    .setTemplate("test*")
+                    .setCreate(true)
+                    .setOrder(1)
+                    .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties")
+                        .startObject("field2").field("type", "string").field("analyzer", "custom_1").endObject()
+                        .endObject().endObject().endObject())
+                .get());
+        assertThat(e.getMessage(), containsString("analyzer [custom_1] not found for field [field2]"));
+
+        response = client().admin().indices().prepareGetTemplates().get();
+        assertThat(response.getIndexTemplates(), hasSize(1));
+
+    }
+
+
 }
diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
index e9d10d31a04..3996aad25e3 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -59,6 +59,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.store.IndexStore;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.indices.InvalidIndexNameException;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.RepositoryException;
@@ -416,7 +417,10 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
                 .setType("fs").setSettings(Settings.builder().put("location", randomRepoPath())));
 
         logger.info("-->  creating test template");
-        assertThat(client.admin().indices().preparePutTemplate("test-template").setTemplate("te*").addMapping("test-mapping", "{}").get().isAcknowledged(), equalTo(true));
+        assertThat(client.admin().indices().preparePutTemplate("test-template").setTemplate("te*").addMapping("test-mapping", XContentFactory.jsonBuilder().startObject().startObject("test-mapping").startObject("properties")
+            .startObject("field1").field("type", "string").field("store", "yes").endObject()
+            .startObject("field2").field("type", "string").field("store", "yes").field("index", "not_analyzed").endObject()
+            .endObject().endObject().endObject()).get().isAcknowledged(), equalTo(true));
 
         logger.info("--> snapshot");
         CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setIndices().setWaitForCompletion(true).get();
@@ -449,7 +453,10 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
                 .setType("fs").setSettings(Settings.builder().put("location", location)));
 
         logger.info("-->  creating test template");
-        assertThat(client.admin().indices().preparePutTemplate("test-template").setTemplate("te*").addMapping("test-mapping", "{}").get().isAcknowledged(), equalTo(true));
+        assertThat(client.admin().indices().preparePutTemplate("test-template").setTemplate("te*").addMapping("test-mapping", XContentFactory.jsonBuilder().startObject().startObject("test-mapping").startObject("properties")
+            .startObject("field1").field("type", "string").field("store", "yes").endObject()
+            .startObject("field2").field("type", "string").field("store", "yes").field("index", "not_analyzed").endObject()
+            .endObject().endObject().endObject()).get().isAcknowledged(), equalTo(true));
 
         logger.info("--> snapshot without global state");
         CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-no-global-state").setIndices().setIncludeGlobalState(false).setWaitForCompletion(true).get();

From 88800e8e470973d36fe0c0fc467aac17e6698700 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Wed, 1 Jun 2016 09:43:11 +0200
Subject: [PATCH 0543/1311] Move PageCacheRecycler into BigArrays (#18666)

PageCacheRecycler is really just an implementation detail of
BigArrays. There is no need to leak this class anywhere outside of it.
---
 .../query/TransportValidateQueryAction.java       |  9 ++-------
 .../action/explain/TransportExplainAction.java    |  9 +++------
 .../client/transport/TransportClient.java         |  4 ++--
 .../elasticsearch/common/recycler/Recycler.java   |  8 ++++----
 .../common/settings/ClusterSettings.java          |  2 +-
 .../common/util/AbstractBigArray.java             |  1 -
 .../org/elasticsearch/common/util/BigArrays.java  | 15 ++++++++++-----
 .../util}/PageCacheRecycler.java                  | 11 ++++-------
 .../main/java/org/elasticsearch/node/Node.java    |  4 ++--
 .../java/org/elasticsearch/node/NodeModule.java   |  7 -------
 .../org/elasticsearch/search/SearchService.java   |  9 ++-------
 .../aggregations/support/AggregationContext.java  |  5 -----
 .../search/internal/DefaultSearchContext.java     | 10 +---------
 .../search/internal/FilteredSearchContext.java    |  6 ------
 .../search/internal/SearchContext.java            |  4 ----
 .../common/bytes/PagedBytesReferenceTests.java    |  2 +-
 .../elasticsearch/common/util/BigArraysTests.java |  8 +++-----
 .../common/util/BytesRefHashTests.java            |  5 ++---
 .../elasticsearch/common/util/LongHashTests.java  |  5 ++---
 .../common/util/LongObjectHashMapTests.java       |  5 ++---
 .../http/netty/NettyHttpChannelTests.java         |  4 +---
 .../netty/NettyHttpServerPipeliningTests.java     |  5 +----
 .../http/netty/NettyHttpServerTransportTests.java |  6 +-----
 .../org/elasticsearch/index/IndexModuleTests.java |  4 +---
 .../elasticsearch/index/SearchSlowLogTests.java   |  4 +---
 .../NettySizeHeaderFrameDecoderTests.java         |  3 +--
 .../netty/NettyTransportMultiPortTests.java       |  3 +--
 .../elasticsearch/common/util/MockBigArrays.java  | 12 ++++++------
 .../util}/MockPageCacheRecycler.java              |  6 ++----
 .../org/elasticsearch/node/NodeMocksPlugin.java   |  2 --
 .../elasticsearch/search/MockSearchService.java   |  5 ++---
 .../elasticsearch/test/ESSingleNodeTestCase.java  |  5 ++---
 .../java/org/elasticsearch/test/ESTestCase.java   |  4 +---
 .../elasticsearch/test/InternalTestCluster.java   |  2 +-
 .../org/elasticsearch/test/TestSearchContext.java | 11 +----------
 35 files changed, 63 insertions(+), 142 deletions(-)
 rename core/src/main/java/org/elasticsearch/{cache/recycler => common/util}/PageCacheRecycler.java (97%)
 rename test/framework/src/main/java/org/elasticsearch/{cache/recycler => common/util}/MockPageCacheRecycler.java (97%)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java
index 7c8d52b6a60..d1405e92e1c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java
@@ -28,7 +28,6 @@ import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
 import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException;
 import org.elasticsearch.action.support.broadcast.TransportBroadcastAction;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
@@ -43,7 +42,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.indices.IndicesService;
@@ -73,8 +71,6 @@ public class TransportValidateQueryAction extends TransportBroadcastAction<Valid
 
     private final ScriptService scriptService;
 
-    private final PageCacheRecycler pageCacheRecycler;
-
     private final BigArrays bigArrays;
 
     private final FetchPhase fetchPhase;
@@ -82,13 +78,12 @@ public class TransportValidateQueryAction extends TransportBroadcastAction<Valid
     @Inject
     public TransportValidateQueryAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
             TransportService transportService, IndicesService indicesService, ScriptService scriptService,
-            PageCacheRecycler pageCacheRecycler, BigArrays bigArrays, ActionFilters actionFilters,
+            BigArrays bigArrays, ActionFilters actionFilters,
             IndexNameExpressionResolver indexNameExpressionResolver, FetchPhase fetchPhase) {
         super(settings, ValidateQueryAction.NAME, threadPool, clusterService, transportService, actionFilters,
                 indexNameExpressionResolver, ValidateQueryRequest::new, ShardValidateQueryRequest::new, ThreadPool.Names.SEARCH);
         this.indicesService = indicesService;
         this.scriptService = scriptService;
-        this.pageCacheRecycler = pageCacheRecycler;
         this.bigArrays = bigArrays;
         this.fetchPhase = fetchPhase;
     }
@@ -176,7 +171,7 @@ public class TransportValidateQueryAction extends TransportBroadcastAction<Valid
 
         DefaultSearchContext searchContext = new DefaultSearchContext(0,
                 new ShardSearchLocalRequest(request.types(), request.nowInMillis(), request.filteringAliases()), null, searcher,
-                indexService, indexShard, scriptService, pageCacheRecycler, bigArrays, threadPool.estimatedTimeInMillisCounter(),
+                indexService, indexShard, scriptService, bigArrays, threadPool.estimatedTimeInMillisCounter(),
                 parseFieldMatcher, SearchService.NO_TIMEOUT, fetchPhase);
         SearchContext.setCurrent(searchContext);
         try {
diff --git a/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java b/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
index 284e31406e2..88e7c66c1b1 100644
--- a/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
+++ b/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
@@ -26,7 +26,6 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.RoutingMissingException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardIterator;
@@ -65,7 +64,6 @@ public class TransportExplainAction extends TransportSingleShardAction<ExplainRe
 
     private final ScriptService scriptService;
 
-    private final PageCacheRecycler pageCacheRecycler;
 
     private final BigArrays bigArrays;
 
@@ -74,13 +72,12 @@ public class TransportExplainAction extends TransportSingleShardAction<ExplainRe
     @Inject
     public TransportExplainAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
             TransportService transportService, IndicesService indicesService, ScriptService scriptService,
-            PageCacheRecycler pageCacheRecycler, BigArrays bigArrays, ActionFilters actionFilters,
-            IndexNameExpressionResolver indexNameExpressionResolver, FetchPhase fetchPhase) {
+            BigArrays bigArrays, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+            FetchPhase fetchPhase) {
         super(settings, ExplainAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
                 ExplainRequest::new, ThreadPool.Names.GET);
         this.indicesService = indicesService;
         this.scriptService = scriptService;
-        this.pageCacheRecycler = pageCacheRecycler;
         this.bigArrays = bigArrays;
         this.fetchPhase = fetchPhase;
     }
@@ -117,7 +114,7 @@ public class TransportExplainAction extends TransportSingleShardAction<ExplainRe
 
         SearchContext context = new DefaultSearchContext(0,
                 new ShardSearchLocalRequest(new String[] { request.type() }, request.nowInMillis, request.filteringAlias()), null,
-                result.searcher(), indexService, indexShard, scriptService, pageCacheRecycler, bigArrays,
+                result.searcher(), indexService, indexShard, scriptService, bigArrays,
                 threadPool.estimatedTimeInMillisCounter(), parseFieldMatcher, SearchService.NO_TIMEOUT, fetchPhase);
         SearchContext.setCurrent(context);
 
diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
index 269984a7858..76c4eafad78 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
@@ -26,7 +26,6 @@ import org.elasticsearch.action.ActionModule;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.client.support.AbstractClient;
 import org.elasticsearch.client.transport.support.TransportProxyClient;
 import org.elasticsearch.cluster.ClusterNameModule;
@@ -41,6 +40,7 @@ import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.indices.breaker.CircuitBreakerModule;
 import org.elasticsearch.monitor.MonitorService;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
@@ -272,7 +272,7 @@ public class TransportClient extends AbstractClient {
             // ignore
         }
 
-        injector.getInstance(PageCacheRecycler.class).close();
+        injector.getInstance(BigArrays.class).close();
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/common/recycler/Recycler.java b/core/src/main/java/org/elasticsearch/common/recycler/Recycler.java
index 520888d6066..0cf7420af85 100644
--- a/core/src/main/java/org/elasticsearch/common/recycler/Recycler.java
+++ b/core/src/main/java/org/elasticsearch/common/recycler/Recycler.java
@@ -25,13 +25,13 @@ import org.elasticsearch.common.lease.Releasable;
  * A recycled object, note, implementations should support calling obtain and then recycle
  * on different threads.
  */
-public interface Recycler<T> {
+public interface Recycler<T> extends Releasable {
 
-    public static interface Factory<T> {
+    interface Factory<T> {
         Recycler<T> build();
     }
 
-    public static interface C<T> {
+    interface C<T> {
 
         /** Create a new empty instance of the given size. */
         T newInstance(int sizing);
@@ -43,7 +43,7 @@ public interface Recycler<T> {
         void destroy(T value);
     }
 
-    public static interface V<T> extends Releasable {
+    interface V<T> extends Releasable {
 
         /** Reference to the value. */
         T v();
diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index b29e7604a98..3510b952660 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -24,7 +24,7 @@ import org.elasticsearch.action.support.AutoCreateIndex;
 import org.elasticsearch.action.support.DestructiveOperations;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
 import org.elasticsearch.bootstrap.BootstrapSettings;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
+import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.transport.TransportClientNodesService;
 import org.elasticsearch.cluster.ClusterModule;
diff --git a/core/src/main/java/org/elasticsearch/common/util/AbstractBigArray.java b/core/src/main/java/org/elasticsearch/common/util/AbstractBigArray.java
index 29beee99d5a..f26dad1fdb5 100644
--- a/core/src/main/java/org/elasticsearch/common/util/AbstractBigArray.java
+++ b/core/src/main/java/org/elasticsearch/common/util/AbstractBigArray.java
@@ -21,7 +21,6 @@ package org.elasticsearch.common.util;
 
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.RamUsageEstimator;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.recycler.Recycler;
 
diff --git a/core/src/main/java/org/elasticsearch/common/util/BigArrays.java b/core/src/main/java/org/elasticsearch/common/util/BigArrays.java
index dfef49ca9d4..017881a9218 100644
--- a/core/src/main/java/org/elasticsearch/common/util/BigArrays.java
+++ b/core/src/main/java/org/elasticsearch/common/util/BigArrays.java
@@ -22,7 +22,6 @@ package org.elasticsearch.common.util;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.RamUsageEstimator;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.breaker.CircuitBreakingException;
@@ -30,14 +29,15 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.recycler.Recycler;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 
 import java.util.Arrays;
 
 /** Utility class to work with arrays. */
-public class BigArrays {
+public class BigArrays implements Releasable {
 
-    public static final BigArrays NON_RECYCLING_INSTANCE = new BigArrays(null, null);
+    public static final BigArrays NON_RECYCLING_INSTANCE = new BigArrays(null, null, false);
 
     /** Page size in bytes: 16KB */
     public static final int PAGE_SIZE_IN_BYTES = 1 << 14;
@@ -84,6 +84,11 @@ public class BigArrays {
         return index == (int) index;
     }
 
+    @Override
+    public void close() {
+        recycler.close();
+    }
+
     private static abstract class AbstractArrayWrapper extends AbstractArray implements BigArray {
 
         protected static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(ByteArrayWrapper.class);
@@ -369,9 +374,9 @@ public class BigArrays {
     private final BigArrays circuitBreakingInstance;
 
     @Inject
-    public BigArrays(PageCacheRecycler recycler, @Nullable final CircuitBreakerService breakerService) {
+    public BigArrays(Settings settings, @Nullable final CircuitBreakerService breakerService) {
         // Checking the breaker is disabled if not specified
-        this(recycler, breakerService, false);
+        this(new PageCacheRecycler(settings), breakerService, false);
     }
 
     public BigArrays(PageCacheRecycler recycler, @Nullable final CircuitBreakerService breakerService, boolean checkBreaker) {
diff --git a/core/src/main/java/org/elasticsearch/cache/recycler/PageCacheRecycler.java b/core/src/main/java/org/elasticsearch/common/util/PageCacheRecycler.java
similarity index 97%
rename from core/src/main/java/org/elasticsearch/cache/recycler/PageCacheRecycler.java
rename to core/src/main/java/org/elasticsearch/common/util/PageCacheRecycler.java
index 69ff997a85a..eed357dee74 100644
--- a/core/src/main/java/org/elasticsearch/cache/recycler/PageCacheRecycler.java
+++ b/core/src/main/java/org/elasticsearch/common/util/PageCacheRecycler.java
@@ -17,11 +17,12 @@
  * under the License.
  */
 
-package org.elasticsearch.cache.recycler;
+package org.elasticsearch.common.util;
 
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.recycler.AbstractRecyclerC;
 import org.elasticsearch.common.recycler.Recycler;
 import org.elasticsearch.common.settings.Setting;
@@ -63,14 +64,10 @@ public class PageCacheRecycler extends AbstractComponent implements Releasable {
 
     @Override
     public void close() {
-        bytePage.close();
-        intPage.close();
-        longPage.close();
-        objectPage.close();
+        Releasables.close(true, bytePage, intPage, longPage, objectPage);
     }
 
-    @Inject
-    public PageCacheRecycler(Settings settings) {
+    protected PageCacheRecycler(Settings settings) {
         super(settings);
         final Type type = TYPE_SETTING .get(settings);
         final long limit = LIMIT_HEAP_SETTING .get(settings).bytes();
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index 2b7651de7b6..cf33770fd16 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -26,7 +26,6 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionModule;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.node.NodeClientModule;
 import org.elasticsearch.cluster.ClusterModule;
@@ -62,6 +61,7 @@ import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.discovery.DiscoveryModule;
@@ -526,7 +526,7 @@ public class Node implements Closeable {
 
 
         toClose.add(injector.getInstance(NodeEnvironment.class));
-        toClose.add(injector.getInstance(PageCacheRecycler.class));
+        toClose.add(injector.getInstance(BigArrays.class));
 
         if (logger.isTraceEnabled()) {
             logger.trace("Close times for each service:\n{}", stopWatch.prettyPrint());
diff --git a/core/src/main/java/org/elasticsearch/node/NodeModule.java b/core/src/main/java/org/elasticsearch/node/NodeModule.java
index 8c9b3e362fc..08b60970229 100644
--- a/core/src/main/java/org/elasticsearch/node/NodeModule.java
+++ b/core/src/main/java/org/elasticsearch/node/NodeModule.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.node;
 
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.ingest.ProcessorsRegistry;
@@ -56,7 +55,6 @@ public class NodeModule extends AbstractModule {
     private final ProcessorsRegistry.Builder processorsRegistryBuilder;
 
     // pkg private so tests can mock
-    Class<? extends PageCacheRecycler> pageCacheRecyclerImpl = PageCacheRecycler.class;
     Class<? extends BigArrays> bigArraysImpl = BigArrays.class;
 
     public NodeModule(Node node, MonitorService monitorService) {
@@ -84,11 +82,6 @@ public class NodeModule extends AbstractModule {
 
     @Override
     protected void configure() {
-        if (pageCacheRecyclerImpl == PageCacheRecycler.class) {
-            bind(PageCacheRecycler.class).asEagerSingleton();
-        } else {
-            bind(PageCacheRecycler.class).to(pageCacheRecyclerImpl).asEagerSingleton();
-        }
         if (bigArraysImpl == BigArrays.class) {
             bind(BigArrays.class).asEagerSingleton();
         } else {
diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java
index 4cf440f0a95..2f0ea96c6fc 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchService.java
@@ -33,10 +33,8 @@ import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.lucene.search.FieldDoc;
-import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.TopDocs;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
@@ -134,8 +132,6 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
 
     private final ScriptService scriptService;
 
-    private final PageCacheRecycler pageCacheRecycler;
-
     private final BigArrays bigArrays;
 
     private final DfsPhase dfsPhase;
@@ -162,7 +158,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
 
     @Inject
     public SearchService(Settings settings, ClusterSettings clusterSettings, ClusterService clusterService, IndicesService indicesService,
-                         ThreadPool threadPool, ScriptService scriptService, PageCacheRecycler pageCacheRecycler, BigArrays bigArrays, DfsPhase dfsPhase,
+                         ThreadPool threadPool, ScriptService scriptService, BigArrays bigArrays, DfsPhase dfsPhase,
                          QueryPhase queryPhase, FetchPhase fetchPhase, AggregatorParsers aggParsers, Suggesters suggesters) {
         super(settings);
         this.aggParsers = aggParsers;
@@ -172,7 +168,6 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
         this.clusterService = clusterService;
         this.indicesService = indicesService;
         this.scriptService = scriptService;
-        this.pageCacheRecycler = pageCacheRecycler;
         this.bigArrays = bigArrays;
         this.dfsPhase = dfsPhase;
         this.queryPhase = queryPhase;
@@ -548,7 +543,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
 
         DefaultSearchContext context = new DefaultSearchContext(idGenerator.incrementAndGet(), request, shardTarget, engineSearcher,
                 indexService,
-                indexShard, scriptService, pageCacheRecycler, bigArrays, threadPool.estimatedTimeInMillisCounter(), parseFieldMatcher,
+                indexShard, scriptService, bigArrays, threadPool.estimatedTimeInMillisCounter(), parseFieldMatcher,
                 defaultSearchTimeout, fetchPhase);
         SearchContext.setCurrent(context);
         request.rewrite(context.getQueryShardContext());
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java
index 1ee72d2b572..d66a1664c8f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.search.aggregations.support;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
@@ -51,10 +50,6 @@ public class AggregationContext {
         return searchContext;
     }
 
-    public PageCacheRecycler pageCacheRecycler() {
-        return searchContext.pageCacheRecycler();
-    }
-
     public BigArrays bigArrays() {
         return searchContext.bigArrays();
     }
diff --git a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
index e69e4094f44..a001ab22ac4 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
@@ -29,7 +29,6 @@ import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Counter;
 import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.lease.Releasables;
@@ -91,7 +90,6 @@ public class DefaultSearchContext extends SearchContext {
     private SearchType searchType;
     private final Engine.Searcher engineSearcher;
     private final ScriptService scriptService;
-    private final PageCacheRecycler pageCacheRecycler;
     private final BigArrays bigArrays;
     private final IndexShard indexShard;
     private final IndexService indexService;
@@ -153,7 +151,7 @@ public class DefaultSearchContext extends SearchContext {
     private FetchPhase fetchPhase;
 
     public DefaultSearchContext(long id, ShardSearchRequest request, SearchShardTarget shardTarget, Engine.Searcher engineSearcher,
-            IndexService indexService, IndexShard indexShard, ScriptService scriptService, PageCacheRecycler pageCacheRecycler,
+            IndexService indexService, IndexShard indexShard, ScriptService scriptService,
             BigArrays bigArrays, Counter timeEstimateCounter, ParseFieldMatcher parseFieldMatcher, TimeValue timeout,
             FetchPhase fetchPhase) {
         super(parseFieldMatcher);
@@ -164,7 +162,6 @@ public class DefaultSearchContext extends SearchContext {
         this.shardTarget = shardTarget;
         this.engineSearcher = engineSearcher;
         this.scriptService = scriptService;
-        this.pageCacheRecycler = pageCacheRecycler;
         // SearchContexts use a BigArrays that can circuit break
         this.bigArrays = bigArrays.withCircuitBreaking();
         this.dfsResult = new DfsSearchResult(id, shardTarget);
@@ -474,11 +471,6 @@ public class DefaultSearchContext extends SearchContext {
         return scriptService;
     }
 
-    @Override
-    public PageCacheRecycler pageCacheRecycler() {
-        return pageCacheRecycler;
-    }
-
     @Override
     public BigArrays bigArrays() {
         return bigArrays;
diff --git a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
index ac283f1f7ef..822ac6f5422 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
@@ -24,7 +24,6 @@ import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.Counter;
 import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.analysis.AnalysisService;
@@ -249,11 +248,6 @@ public abstract class FilteredSearchContext extends SearchContext {
         return in.scriptService();
     }
 
-    @Override
-    public PageCacheRecycler pageCacheRecycler() {
-        return in.pageCacheRecycler();
-    }
-
     @Override
     public BigArrays bigArrays() {
         return in.bigArrays();
diff --git a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
index 55b66c42a60..5135f42ab56 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
@@ -22,10 +22,8 @@ package org.elasticsearch.search.internal;
 import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Sort;
 import org.apache.lucene.util.Counter;
 import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.lease.Releasable;
@@ -222,8 +220,6 @@ public abstract class SearchContext implements Releasable {
 
     public abstract ScriptService scriptService();
 
-    public abstract PageCacheRecycler pageCacheRecycler();
-
     public abstract BigArrays bigArrays();
 
     public abstract BitsetFilterCache bitsetFilterCache();
diff --git a/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java b/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java
index 90b253ec441..9cb633e410e 100644
--- a/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java
+++ b/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java
@@ -51,7 +51,7 @@ public class PagedBytesReferenceTests extends ESTestCase {
     @Before
     public void setUp() throws Exception {
         super.setUp();
-        bigarrays = new BigArrays(null, new NoneCircuitBreakerService());
+        bigarrays = new BigArrays(null, new NoneCircuitBreakerService(), false);
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/common/util/BigArraysTests.java b/core/src/test/java/org/elasticsearch/common/util/BigArraysTests.java
index 3192c8242b5..6fadb6e5506 100644
--- a/core/src/test/java/org/elasticsearch/common/util/BigArraysTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/BigArraysTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.common.util;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.breaker.CircuitBreakingException;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -38,8 +37,7 @@ import java.util.Arrays;
 public class BigArraysTests extends ESSingleNodeTestCase {
 
     private BigArrays randombigArrays() {
-        final PageCacheRecycler recycler = randomBoolean() ? null : getInstanceFromNode(PageCacheRecycler.class);
-        return new MockBigArrays(recycler, new NoneCircuitBreakerService());
+        return new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
     }
 
     private BigArrays bigArrays;
@@ -339,7 +337,7 @@ public class BigArraysTests extends ESSingleNodeTestCase {
                             .put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), size - 1, ByteSizeUnit.BYTES)
                             .build(),
                     new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
-            BigArrays bigArrays = new BigArrays(null, hcbs).withCircuitBreaking();
+            BigArrays bigArrays = new BigArrays(null, hcbs, false).withCircuitBreaking();
             Method create = BigArrays.class.getMethod("new" + type + "Array", long.class);
             try {
                 create.invoke(bigArrays, size);
@@ -359,7 +357,7 @@ public class BigArraysTests extends ESSingleNodeTestCase {
                             .put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), maxSize, ByteSizeUnit.BYTES)
                             .build(),
                     new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
-            BigArrays bigArrays = new BigArrays(null, hcbs).withCircuitBreaking();
+            BigArrays bigArrays = new BigArrays(null, hcbs, false).withCircuitBreaking();
             Method create = BigArrays.class.getMethod("new" + type + "Array", long.class);
             final int size = scaledRandomIntBetween(1, 20);
             BigArray array = (BigArray) create.invoke(bigArrays, size);
diff --git a/core/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java b/core/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java
index 8b0051228b4..9f90fcf151d 100644
--- a/core/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/BytesRefHashTests.java
@@ -25,7 +25,7 @@ import com.carrotsearch.hppc.cursors.ObjectLongCursor;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
 import org.apache.lucene.util.TestUtil;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
@@ -41,8 +41,7 @@ public class BytesRefHashTests extends ESSingleNodeTestCase {
     BytesRefHash hash;
 
     private BigArrays randombigArrays() {
-        final PageCacheRecycler recycler = randomBoolean() ? null : getInstanceFromNode(PageCacheRecycler.class);
-        return new MockBigArrays(recycler, new NoneCircuitBreakerService());
+        return new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
     }
 
     private void newHash() {
diff --git a/core/src/test/java/org/elasticsearch/common/util/LongHashTests.java b/core/src/test/java/org/elasticsearch/common/util/LongHashTests.java
index 9439044a7be..66c9e689c96 100644
--- a/core/src/test/java/org/elasticsearch/common/util/LongHashTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/LongHashTests.java
@@ -22,7 +22,7 @@ package org.elasticsearch.common.util;
 import com.carrotsearch.hppc.LongLongHashMap;
 import com.carrotsearch.hppc.LongLongMap;
 import com.carrotsearch.hppc.cursors.LongLongCursor;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
@@ -36,8 +36,7 @@ public class LongHashTests extends ESSingleNodeTestCase {
     LongHash hash;
 
     private BigArrays randombigArrays() {
-        final PageCacheRecycler recycler = randomBoolean() ? null : getInstanceFromNode(PageCacheRecycler.class);
-        return new MockBigArrays(recycler, new NoneCircuitBreakerService());
+        return new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
     }
 
     private void newHash() {
diff --git a/core/src/test/java/org/elasticsearch/common/util/LongObjectHashMapTests.java b/core/src/test/java/org/elasticsearch/common/util/LongObjectHashMapTests.java
index 1775f86199a..f783317808d 100644
--- a/core/src/test/java/org/elasticsearch/common/util/LongObjectHashMapTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/LongObjectHashMapTests.java
@@ -20,15 +20,14 @@
 package org.elasticsearch.common.util;
 
 import com.carrotsearch.hppc.LongObjectHashMap;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
 public class LongObjectHashMapTests extends ESSingleNodeTestCase {
 
     private BigArrays randombigArrays() {
-        final PageCacheRecycler recycler = randomBoolean() ? null : getInstanceFromNode(PageCacheRecycler.class);
-        return new MockBigArrays(recycler, new NoneCircuitBreakerService());
+        return new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
     }
 
     public void testDuel() {
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java
index 8098d557a05..e7e3c41820b 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.http.netty;
 
-import org.elasticsearch.cache.recycler.MockPageCacheRecycler;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.network.NetworkService;
@@ -72,8 +71,7 @@ public class NettyHttpChannelTests extends ESTestCase {
     public void setup() throws Exception {
         networkService = new NetworkService(Settings.EMPTY);
         threadPool = new ThreadPool("test");
-        MockPageCacheRecycler mockPageCacheRecycler = new MockPageCacheRecycler(Settings.EMPTY);
-        bigArrays = new MockBigArrays(mockPageCacheRecycler, new NoneCircuitBreakerService());
+        bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
     }
 
     @After
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java
index f233512e4ce..db48a56a122 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java
@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.http.netty;
 
-import org.elasticsearch.cache.recycler.MockPageCacheRecycler;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
@@ -68,7 +67,6 @@ import static org.jboss.netty.handler.codec.http.HttpVersion.HTTP_1_1;
 public class NettyHttpServerPipeliningTests extends ESTestCase {
     private NetworkService networkService;
     private ThreadPool threadPool;
-    private MockPageCacheRecycler mockPageCacheRecycler;
     private MockBigArrays bigArrays;
     private CustomNettyHttpServerTransport httpServerTransport;
 
@@ -76,8 +74,7 @@ public class NettyHttpServerPipeliningTests extends ESTestCase {
     public void setup() throws Exception {
         networkService = new NetworkService(Settings.EMPTY);
         threadPool = new ThreadPool("test");
-        mockPageCacheRecycler = new MockPageCacheRecycler(Settings.EMPTY);
-        bigArrays = new MockBigArrays(mockPageCacheRecycler, new NoneCircuitBreakerService());
+        bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
     }
 
     @After
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerTransportTests.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerTransportTests.java
index 3089225e453..da548c93985 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerTransportTests.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerTransportTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.http.netty;
 
-import org.elasticsearch.cache.recycler.MockPageCacheRecycler;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
@@ -50,15 +49,13 @@ import static org.hamcrest.Matchers.equalTo;
 public class NettyHttpServerTransportTests extends ESTestCase {
     private NetworkService networkService;
     private ThreadPool threadPool;
-    private MockPageCacheRecycler mockPageCacheRecycler;
     private MockBigArrays bigArrays;
 
     @Before
     public void setup() throws Exception {
         networkService = new NetworkService(Settings.EMPTY);
         threadPool = new ThreadPool("test");
-        mockPageCacheRecycler = new MockPageCacheRecycler(Settings.EMPTY);
-        bigArrays = new MockBigArrays(mockPageCacheRecycler, new NoneCircuitBreakerService());
+        bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
     }
 
     @After
@@ -68,7 +65,6 @@ public class NettyHttpServerTransportTests extends ESTestCase {
         }
         threadPool = null;
         networkService = null;
-        mockPageCacheRecycler = null;
         bigArrays = null;
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
index 6e901a9a671..38b7341cd24 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
@@ -32,7 +32,6 @@ import org.apache.lucene.search.similarities.BM25Similarity;
 import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.util.SetOnce.AlreadySetException;
 import org.elasticsearch.Version;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -113,8 +112,7 @@ public class IndexModuleTests extends ESTestCase {
         // TODO this can be used in other place too - lets first refactor the IndicesQueriesRegistry
         ThreadPool threadPool = new ThreadPool("test");
         CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService();
-        PageCacheRecycler recycler = new PageCacheRecycler(settings);
-        BigArrays bigArrays = new BigArrays(recycler, circuitBreakerService);
+        BigArrays bigArrays = new BigArrays(settings, circuitBreakerService);
         Set<ScriptEngineService> scriptEngines = Collections.emptySet();
         scriptEngines.addAll(Arrays.asList(scriptEngineServices));
         ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.emptyList());
diff --git a/core/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java b/core/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java
index 49283d7b73a..3c82d3063af 100644
--- a/core/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java
+++ b/core/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java
@@ -21,7 +21,6 @@ package org.elasticsearch.index;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
@@ -49,9 +48,8 @@ public class SearchSlowLogTests extends ESSingleNodeTestCase {
     protected SearchContext createSearchContext(IndexService indexService) {
         BigArrays bigArrays = indexService.getBigArrays();
         ThreadPool threadPool = indexService.getThreadPool();
-        PageCacheRecycler pageCacheRecycler = node().injector().getInstance(PageCacheRecycler.class);
         ScriptService scriptService = node().injector().getInstance(ScriptService.class);
-        return new TestSearchContext(threadPool, pageCacheRecycler, bigArrays, scriptService, indexService) {
+        return new TestSearchContext(threadPool, bigArrays, scriptService, indexService) {
             @Override
             public ShardSearchRequest request() {
                 return new ShardSearchRequest() {
diff --git a/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java b/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java
index 98d440bb446..ca5faf68b51 100644
--- a/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.transport;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.cache.recycler.MockPageCacheRecycler;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
@@ -67,7 +66,7 @@ public class NettySizeHeaderFrameDecoderTests extends ESTestCase {
         threadPool = new ThreadPool(settings);
         threadPool.setClusterSettings(new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
         NetworkService networkService = new NetworkService(settings);
-        BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(settings), new NoneCircuitBreakerService());
+        BigArrays bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
         nettyTransport = new NettyTransport(settings, threadPool, networkService, bigArrays, Version.CURRENT, new NamedWriteableRegistry(),
             new NoneCircuitBreakerService());
         nettyTransport.start();
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java
index 78aa90cf8fc..352e7bb5a59 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.transport.netty;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
@@ -134,7 +133,7 @@ public class NettyTransportMultiPortTests extends ESTestCase {
     }
 
     private NettyTransport startNettyTransport(Settings settings, ThreadPool threadPool) {
-        BigArrays bigArrays = new MockBigArrays(new PageCacheRecycler(settings), new NoneCircuitBreakerService());
+        BigArrays bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
 
         NettyTransport nettyTransport = new NettyTransport(settings, threadPool, new NetworkService(settings), bigArrays, Version.CURRENT,
             new NamedWriteableRegistry(), new NoneCircuitBreakerService());
diff --git a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
index 50f1886431a..26e021bf031 100644
--- a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
+++ b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
@@ -24,8 +24,8 @@ import com.carrotsearch.randomizedtesting.SeedUtils;
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.Accountables;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
@@ -75,11 +75,11 @@ public class MockBigArrays extends BigArrays {
     private final CircuitBreakerService breakerService;
 
     @Inject
-    public MockBigArrays(PageCacheRecycler recycler, CircuitBreakerService breakerService) {
-        this(recycler, breakerService, false);
+    public MockBigArrays(Settings settings, CircuitBreakerService breakerService) {
+        this(new MockPageCacheRecycler(settings), breakerService, false);
     }
 
-    public MockBigArrays(PageCacheRecycler recycler, CircuitBreakerService breakerService, boolean checkBreaker) {
+    private MockBigArrays(PageCacheRecycler recycler, CircuitBreakerService breakerService, boolean checkBreaker) {
         super(recycler, breakerService, checkBreaker);
         this.recycler = recycler;
         this.breakerService = breakerService;
@@ -371,7 +371,7 @@ public class MockBigArrays extends BigArrays {
         public void fill(long fromIndex, long toIndex, int value) {
             in.fill(fromIndex, toIndex, value);
         }
-        
+
         @Override
         public Collection<Accountable> getChildResources() {
             return Collections.singleton(Accountables.namedAccountable("delegate", in));
@@ -416,7 +416,7 @@ public class MockBigArrays extends BigArrays {
         public void fill(long fromIndex, long toIndex, long value) {
             in.fill(fromIndex, toIndex, value);
         }
-        
+
         @Override
         public Collection<Accountable> getChildResources() {
             return Collections.singleton(Accountables.namedAccountable("delegate", in));
diff --git a/test/framework/src/main/java/org/elasticsearch/cache/recycler/MockPageCacheRecycler.java b/test/framework/src/main/java/org/elasticsearch/common/util/MockPageCacheRecycler.java
similarity index 97%
rename from test/framework/src/main/java/org/elasticsearch/cache/recycler/MockPageCacheRecycler.java
rename to test/framework/src/main/java/org/elasticsearch/common/util/MockPageCacheRecycler.java
index ae7fc9142ea..766d68f45dd 100644
--- a/test/framework/src/main/java/org/elasticsearch/cache/recycler/MockPageCacheRecycler.java
+++ b/test/framework/src/main/java/org/elasticsearch/common/util/MockPageCacheRecycler.java
@@ -17,9 +17,8 @@
  * under the License.
  */
 
-package org.elasticsearch.cache.recycler;
+package org.elasticsearch.common.util;
 
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.recycler.Recycler.V;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
@@ -58,8 +57,7 @@ public class MockPageCacheRecycler extends PageCacheRecycler {
 
     private final Random random;
 
-    @Inject
-    public MockPageCacheRecycler(Settings settings) {
+    MockPageCacheRecycler(Settings settings) {
         super(settings);
         // we always initialize with 0 here since we really only wanna have some random bytes / ints / longs
         // and given the fact that it's called concurrently it won't reproduces anyway the same order other than in a unittest
diff --git a/test/framework/src/main/java/org/elasticsearch/node/NodeMocksPlugin.java b/test/framework/src/main/java/org/elasticsearch/node/NodeMocksPlugin.java
index f958b2b752e..598f5a7ee7c 100644
--- a/test/framework/src/main/java/org/elasticsearch/node/NodeMocksPlugin.java
+++ b/test/framework/src/main/java/org/elasticsearch/node/NodeMocksPlugin.java
@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.node;
 
-import org.elasticsearch.cache.recycler.MockPageCacheRecycler;
 import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.plugins.Plugin;
 
@@ -35,7 +34,6 @@ public class NodeMocksPlugin extends Plugin {
     }
 
     public void onModule(NodeModule module) {
-        module.pageCacheRecyclerImpl = MockPageCacheRecycler.class;
         module.bigArraysImpl = MockBigArrays.class;
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java b/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
index 2519606af3b..629a5df3fee 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.search;
 
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -84,10 +83,10 @@ public class MockSearchService extends SearchService {
 
     @Inject
     public MockSearchService(Settings settings, ClusterSettings clusterSettings, ClusterService clusterService,
-            IndicesService indicesService, ThreadPool threadPool, ScriptService scriptService, PageCacheRecycler pageCacheRecycler,
+            IndicesService indicesService, ThreadPool threadPool, ScriptService scriptService,
             BigArrays bigArrays, DfsPhase dfsPhase, QueryPhase queryPhase, FetchPhase fetchPhase,
             AggregatorParsers aggParsers, Suggesters suggesters) {
-        super(settings, clusterSettings, clusterService, indicesService, threadPool, scriptService, pageCacheRecycler, bigArrays, dfsPhase,
+        super(settings, clusterSettings, clusterService, indicesService, threadPool, scriptService, bigArrays, dfsPhase,
                 queryPhase, fetchPhase, aggParsers, suggesters);
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
index 7875f8fd20b..c86c111f645 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
@@ -23,7 +23,6 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.ClusterName;
@@ -35,6 +34,7 @@ import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.env.Environment;
@@ -282,9 +282,8 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
     protected SearchContext createSearchContext(IndexService indexService) {
         BigArrays bigArrays = indexService.getBigArrays();
         ThreadPool threadPool = indexService.getThreadPool();
-        PageCacheRecycler pageCacheRecycler = node().injector().getInstance(PageCacheRecycler.class);
         ScriptService scriptService = node().injector().getInstance(ScriptService.class);
-        return new TestSearchContext(threadPool, pageCacheRecycler, bigArrays, scriptService, indexService);
+        return new TestSearchContext(threadPool, bigArrays, scriptService, indexService);
     }
 
     /**
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 6ed9a746f46..22216efa15b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -37,7 +37,7 @@ import org.apache.lucene.util.TestUtil;
 import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.Version;
 import org.elasticsearch.bootstrap.BootstrapForTesting;
-import org.elasticsearch.cache.recycler.MockPageCacheRecycler;
+import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -82,9 +82,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
-import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.function.BooleanSupplier;
 import java.util.function.Consumer;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index c67ffc07a2b..1217662480a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -31,7 +31,6 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.cluster.ClusterName;
@@ -60,6 +59,7 @@ import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.discovery.DiscoverySettings;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
index d8ea1f60bbd..8122abe483d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
@@ -27,7 +27,6 @@ import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.Counter;
 import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.cache.recycler.PageCacheRecycler;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.IndexService;
@@ -67,7 +66,6 @@ import org.elasticsearch.threadpool.ThreadPool;
 
 public class TestSearchContext extends SearchContext {
 
-    final PageCacheRecycler pageCacheRecycler;
     final BigArrays bigArrays;
     final IndexService indexService;
     final IndexFieldDataService indexFieldDataService;
@@ -92,9 +90,8 @@ public class TestSearchContext extends SearchContext {
     private final long originNanoTime = System.nanoTime();
     private final Map<String, FetchSubPhaseContext> subPhaseContexts = new HashMap<>();
 
-    public TestSearchContext(ThreadPool threadPool,PageCacheRecycler pageCacheRecycler, BigArrays bigArrays, ScriptService scriptService, IndexService indexService) {
+    public TestSearchContext(ThreadPool threadPool, BigArrays bigArrays, ScriptService scriptService, IndexService indexService) {
         super(ParseFieldMatcher.STRICT);
-        this.pageCacheRecycler = pageCacheRecycler;
         this.bigArrays = bigArrays.withCircuitBreaking();
         this.indexService = indexService;
         this.indexFieldDataService = indexService.fieldData();
@@ -107,7 +104,6 @@ public class TestSearchContext extends SearchContext {
 
     public TestSearchContext(QueryShardContext queryShardContext) {
         super(ParseFieldMatcher.STRICT);
-        this.pageCacheRecycler = null;
         this.bigArrays = null;
         this.indexService = null;
         this.indexFieldDataService = null;
@@ -299,11 +295,6 @@ public class TestSearchContext extends SearchContext {
         return scriptService;
     }
 
-    @Override
-    public PageCacheRecycler pageCacheRecycler() {
-        return pageCacheRecycler;
-    }
-
     @Override
     public BigArrays bigArrays() {
         return bigArrays;

From a78c7d9911617c97e7f5d82b6fb03397ad1914a4 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Tue, 31 May 2016 15:30:42 +0200
Subject: [PATCH 0544/1311] AggregatorBuilder and PipelineAggregatorBuilder do
 not need generics. #18368

Similar reasoning as #18133 but for the aggs API. One important change is that
I moved the base PipelineAggregatorBuilder class to the o.e.s.aggregations
package instead of o.e.s.aggregations.pipeline so that the create method does
not need to be public.
---
 .../action/search/SearchRequestBuilder.java   |   4 +-
 .../elasticsearch/search/SearchModule.java    |   6 +-
 .../AbstractAggregationBuilder.java           | 182 ++++++++++++++++++
 .../aggregations/AggregationBuilder.java      | 150 ++-------------
 .../search/aggregations/Aggregator.java       |   2 +-
 .../aggregations/AggregatorFactories.java     |  77 ++++----
 .../aggregations/AggregatorParsers.java       |   5 +-
 .../PipelineAggregatorBuilder.java            |  84 ++++++++
 .../filter/FilterAggregationBuilder.java      |   4 +-
 .../filters/FiltersAggregationBuilder.java    |   4 +-
 .../global/GlobalAggregationBuilder.java      |   4 +-
 .../nested/NestedAggregationBuilder.java      |   4 +-
 .../ReverseNestedAggregationBuilder.java      |   4 +-
 .../sampler/SamplerAggregationBuilder.java    |   4 +-
 .../ScriptedMetricAggregationBuilder.java     |   4 +-
 .../tophits/TopHitsAggregationBuilder.java    |   4 +-
 ...=> AbstractPipelineAggregatorBuilder.java} |  60 ++----
 .../aggregations/pipeline/BucketHelpers.java  |   4 +-
 .../pipeline/PipelineAggregator.java          |   3 +-
 ...ucketMetricsPipelineAggregatorBuilder.java |   7 +-
 .../AvgBucketPipelineAggregatorBuilder.java   |   4 +-
 .../MaxBucketPipelineAggregatorBuilder.java   |   4 +-
 .../MinBucketPipelineAggregatorBuilder.java   |   4 +-
 ...ntilesBucketPipelineAggregatorBuilder.java |   5 +-
 .../StatsBucketPipelineAggregatorBuilder.java |   4 +-
 ...dStatsBucketPipelineAggregatorBuilder.java |   4 +-
 .../SumBucketPipelineAggregatorBuilder.java   |   4 +-
 ...BucketScriptPipelineAggregatorBuilder.java |   4 +-
 ...cketSelectorPipelineAggregatorBuilder.java |   4 +-
 ...umulativeSumPipelineAggregatorBuilder.java |   7 +-
 .../DerivativePipelineAggregatorBuilder.java  |   7 +-
 .../MovAvgPipelineAggregatorBuilder.java      |   7 +-
 .../SerialDiffPipelineAggregatorBuilder.java  |   4 +-
 .../ValuesSourceAggregationBuilder.java       |   4 +-
 .../search/builder/SearchSourceBuilder.java   |   4 +-
 .../aggregations/BaseAggregationTestCase.java |   6 +-
 .../BasePipelineAggregationTestCase.java      |  12 +-
 .../AbstractBucketMetricsTestCase.java        |   2 +-
 .../pipeline/moving/avg/MovAvgTests.java      |   8 +-
 .../percolator/PercolateRequestBuilder.java   |   4 +-
 .../percolator/PercolateSourceBuilder.java    |  14 +-
 41 files changed, 430 insertions(+), 302 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java
 create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregatorBuilder.java
 rename core/src/main/java/org/elasticsearch/search/aggregations/pipeline/{PipelineAggregatorBuilder.java => AbstractPipelineAggregatorBuilder.java} (73%)

diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
index 5732d43b4c7..c5666fa16a5 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
@@ -29,7 +29,7 @@ import org.elasticsearch.script.Script;
 import org.elasticsearch.script.Template;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.highlight.HighlightBuilder;
 import org.elasticsearch.search.rescore.RescoreBuilder;
@@ -373,7 +373,7 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
     /**
      * Adds an aggregation to the search operation.
      */
-    public SearchRequestBuilder addAggregation(AggregationBuilder<?> aggregation) {
+    public SearchRequestBuilder addAggregation(AggregationBuilder aggregation) {
         sourceBuilder().aggregation(aggregation);
         return this;
     }
diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java
index f601edd50a4..a8893c4c27c 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -98,6 +98,7 @@ import org.elasticsearch.search.aggregations.AggregationPhase;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
+import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.bucket.children.ChildrenAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.children.InternalChildren;
 import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
@@ -201,7 +202,6 @@ import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggreg
 import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountParser;
 import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.InternalBucketMetricValue;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.avg.AvgBucketPipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.avg.AvgBucketPipelineAggregatorBuilder;
@@ -422,7 +422,7 @@ public class SearchModule extends AbstractModule {
      * @param aggregationName names by which the aggregation may be parsed. The first name is special because it is the name that the reader
      *        is registered under.
      */
-    public <AB extends AggregationBuilder<AB>> void registerAggregation(Writeable.Reader<AB> reader, Aggregator.Parser aggregationParser,
+    public void registerAggregation(Writeable.Reader<? extends AggregationBuilder> reader, Aggregator.Parser aggregationParser,
                                                                         ParseField aggregationName) {
         aggregationParserRegistry.register(aggregationParser, aggregationName);
         namedWriteableRegistry.register(AggregationBuilder.class, aggregationName.getPreferredName(), reader);
@@ -436,7 +436,7 @@ public class SearchModule extends AbstractModule {
      * @param aggregationName names by which the aggregation may be parsed. The first name is special because it is the name that the reader
      *        is registered under.
      */
-    public <AB extends PipelineAggregatorBuilder<AB>> void registerPipelineAggregation(Writeable.Reader<AB> reader,
+    public void registerPipelineAggregation(Writeable.Reader<? extends PipelineAggregatorBuilder> reader,
             PipelineAggregator.Parser aggregationParser, ParseField aggregationName) {
         pipelineAggregationParserRegistry.register(aggregationParser, aggregationName);
         namedWriteableRegistry.register(PipelineAggregatorBuilder.class, aggregationName.getPreferredName(), reader);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java
new file mode 100644
index 00000000000..ef54cdf0ef5
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.aggregations.InternalAggregation.Type;
+import org.elasticsearch.search.aggregations.support.AggregationContext;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Base implementation of a {@link AggregationBuilder}.
+ */
+public abstract class AbstractAggregationBuilder<AB extends AbstractAggregationBuilder<AB>>
+    extends AggregationBuilder {
+
+    protected Map<String, Object> metaData;
+
+    /**
+     * Constructs a new aggregation builder.
+     *
+     * @param name  The aggregation name
+     * @param type  The aggregation type
+     */
+    public AbstractAggregationBuilder(String name, Type type) {
+        super(name, type);
+    }
+
+    /**
+     * Read from a stream.
+     */
+    protected AbstractAggregationBuilder(StreamInput in, Type type) throws IOException {
+        super(in.readString(), type);
+        factoriesBuilder = new AggregatorFactories.Builder(in);
+        metaData = in.readMap();
+    }
+
+    @Override
+    public final void writeTo(StreamOutput out) throws IOException {
+        out.writeString(name);
+        factoriesBuilder.writeTo(out);
+        out.writeMap(metaData);
+        doWriteTo(out);
+    }
+
+    protected abstract void doWriteTo(StreamOutput out) throws IOException;
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public AB subAggregation(AggregationBuilder aggregation) {
+        if (aggregation == null) {
+            throw new IllegalArgumentException("[aggregation] must not be null: [" + name + "]");
+        }
+        factoriesBuilder.addAggregator(aggregation);
+        return (AB) this;
+    }
+
+    /**
+     * Add a sub aggregation to this aggregation.
+     */
+    @SuppressWarnings("unchecked")
+    @Override
+    public AB subAggregation(PipelineAggregatorBuilder aggregation) {
+        if (aggregation == null) {
+            throw new IllegalArgumentException("[aggregation] must not be null: [" + name + "]");
+        }
+        factoriesBuilder.addPipelineAggregator(aggregation);
+        return (AB) this;
+    }
+
+    /**
+     * Registers sub-factories with this factory. The sub-factory will be
+     * responsible for the creation of sub-aggregators under the aggregator
+     * created by this factory.
+     *
+     * @param subFactories
+     *            The sub-factories
+     * @return this factory (fluent interface)
+     */
+    @SuppressWarnings("unchecked")
+    @Override
+    public AB subAggregations(AggregatorFactories.Builder subFactories) {
+        if (subFactories == null) {
+            throw new IllegalArgumentException("[subFactories] must not be null: [" + name + "]");
+        }
+        this.factoriesBuilder = subFactories;
+        return (AB) this;
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public AB setMetaData(Map<String, Object> metaData) {
+        if (metaData == null) {
+            throw new IllegalArgumentException("[metaData] must not be null: [" + name + "]");
+        }
+        this.metaData = metaData;
+        return (AB) this;
+    }
+
+    public String getType() {
+        return type.name();
+    }
+
+    @Override
+    public final AggregatorFactory<?> build(AggregationContext context, AggregatorFactory<?> parent) throws IOException {
+        AggregatorFactory<?> factory = doBuild(context, parent, factoriesBuilder);
+        return factory;
+    }
+
+    protected abstract AggregatorFactory<?> doBuild(AggregationContext context, AggregatorFactory<?> parent,
+            AggregatorFactories.Builder subfactoriesBuilder) throws IOException;
+
+    @Override
+    public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(name);
+
+        if (this.metaData != null) {
+            builder.field("meta", this.metaData);
+        }
+        builder.field(type.name());
+        internalXContent(builder, params);
+
+        if (factoriesBuilder != null && (factoriesBuilder.count()) > 0) {
+            builder.field("aggregations");
+            factoriesBuilder.toXContent(builder, params);
+
+        }
+
+        return builder.endObject();
+    }
+
+    protected abstract XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException;
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(factoriesBuilder, metaData, name, type, doHashCode());
+    }
+
+    protected abstract int doHashCode();
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null)
+            return false;
+        if (getClass() != obj.getClass())
+            return false;
+        @SuppressWarnings("unchecked")
+        AbstractAggregationBuilder<AB> other = (AbstractAggregationBuilder<AB>) obj;
+        if (!Objects.equals(name, other.name))
+            return false;
+        if (!Objects.equals(type, other.type))
+            return false;
+        if (!Objects.equals(metaData, other.metaData))
+            return false;
+        if (!Objects.equals(factoriesBuilder, other.factoriesBuilder))
+            return false;
+        return doEquals(obj);
+    }
+
+    protected abstract boolean doEquals(Object obj);
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
index e0336247c75..1a16f653159 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
@@ -21,29 +21,23 @@ package org.elasticsearch.search.aggregations;
 
 import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.io.stream.NamedWriteable;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 
 import java.io.IOException;
 import java.util.Map;
-import java.util.Objects;
 
 /**
  * A factory that knows how to create an {@link Aggregator} of a specific type.
  */
-public abstract class AggregationBuilder<AB extends AggregationBuilder<AB>>
+public abstract class AggregationBuilder
     extends ToXContentToBytes
     implements NamedWriteable, ToXContent {
 
-    protected String name;
-    protected Type type;
+    protected final String name;
+    protected final Type type;
     protected AggregatorFactories.Builder factoriesBuilder = AggregatorFactories.builder();
-    protected Map<String, Object> metaData;
 
     /**
      * Constructs a new aggregation builder.
@@ -51,7 +45,7 @@ public abstract class AggregationBuilder<AB extends AggregationBuilder<AB>>
      * @param name  The aggregation name
      * @param type  The aggregation type
      */
-    public AggregationBuilder(String name, Type type) {
+    protected AggregationBuilder(String name, Type type) {
         if (name == null) {
             throw new IllegalArgumentException("[name] must not be null: [" + name + "]");
         }
@@ -62,136 +56,32 @@ public abstract class AggregationBuilder<AB extends AggregationBuilder<AB>>
         this.type = type;
     }
 
-    /**
-     * Read from a stream.
-     */
-    protected AggregationBuilder(StreamInput in, Type type) throws IOException {
-        name = in.readString();
-        this.type = type;
-        factoriesBuilder = new AggregatorFactories.Builder(in);
-        metaData = in.readMap();
+    /** Return this aggregation's name. */
+    public String getName() {
+        return name;
     }
 
-    @Override
-    public final void writeTo(StreamOutput out) throws IOException {
-        out.writeString(name);
-        factoriesBuilder.writeTo(out);
-        out.writeMap(metaData);
-        doWriteTo(out);
-    }
+    /** Internal: build an {@link AggregatorFactory} based on the configuration of this builder. */
+    protected abstract AggregatorFactory<?> build(AggregationContext context, AggregatorFactory<?> parent) throws IOException;
 
-    protected abstract void doWriteTo(StreamOutput out) throws IOException;
+    /** Associate metadata with this {@link AggregationBuilder}. */
+    public abstract AggregationBuilder setMetaData(Map<String, Object> metaData);
+
+    /** Add a sub aggregation to this builder. */
+    public abstract AggregationBuilder subAggregation(AggregationBuilder aggregation);
+
+    /** Add a sub aggregation to this builder. */
+    public abstract AggregationBuilder subAggregation(PipelineAggregatorBuilder aggregation);
 
     /**
-     * Add a sub aggregation to this aggregation.
-     */
-    @SuppressWarnings("unchecked")
-    public AB subAggregation(AggregationBuilder<?> aggregation) {
-        if (aggregation == null) {
-            throw new IllegalArgumentException("[aggregation] must not be null: [" + name + "]");
-        }
-        factoriesBuilder.addAggregator(aggregation);
-        return (AB) this;
-    }
-
-    /**
-     * Add a sub aggregation to this aggregation.
-     */
-    @SuppressWarnings("unchecked")
-    public AB subAggregation(PipelineAggregatorBuilder<?> aggregation) {
-        if (aggregation == null) {
-            throw new IllegalArgumentException("[aggregation] must not be null: [" + name + "]");
-        }
-        factoriesBuilder.addPipelineAggregator(aggregation);
-        return (AB) this;
-    }
-
-    /**
-     * Registers sub-factories with this factory. The sub-factory will be
+     * Internal: Registers sub-factories with this factory. The sub-factory will be
      * responsible for the creation of sub-aggregators under the aggregator
-     * created by this factory.
+     * created by this factory. This is only for use by {@link AggregatorParsers}.
      *
      * @param subFactories
      *            The sub-factories
      * @return this factory (fluent interface)
      */
-    @SuppressWarnings("unchecked")
-    public AB subAggregations(AggregatorFactories.Builder subFactories) {
-        if (subFactories == null) {
-            throw new IllegalArgumentException("[subFactories] must not be null: [" + name + "]");
-        }
-        this.factoriesBuilder = subFactories;
-        return (AB) this;
-    }
-
-    @SuppressWarnings("unchecked")
-    public AB setMetaData(Map<String, Object> metaData) {
-        if (metaData == null) {
-            throw new IllegalArgumentException("[metaData] must not be null: [" + name + "]");
-        }
-        this.metaData = metaData;
-        return (AB) this;
-    }
-
-    public String getType() {
-        return type.name();
-    }
-
-    public final AggregatorFactory<?> build(AggregationContext context, AggregatorFactory<?> parent) throws IOException {
-        AggregatorFactory<?> factory = doBuild(context, parent, factoriesBuilder);
-        return factory;
-    }
-
-    protected abstract AggregatorFactory<?> doBuild(AggregationContext context, AggregatorFactory<?> parent,
-            AggregatorFactories.Builder subfactoriesBuilder) throws IOException;
-
-    @Override
-    public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(name);
-
-        if (this.metaData != null) {
-            builder.field("meta", this.metaData);
-        }
-        builder.field(type.name());
-        internalXContent(builder, params);
-
-        if (factoriesBuilder != null && (factoriesBuilder.count()) > 0) {
-            builder.field("aggregations");
-            factoriesBuilder.toXContent(builder, params);
-
-        }
-
-        return builder.endObject();
-    }
-
-    protected abstract XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException;
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(factoriesBuilder, metaData, name, type, doHashCode());
-    }
-
-    protected abstract int doHashCode();
-
-    @Override
-    public boolean equals(Object obj) {
-        if (obj == null)
-            return false;
-        if (getClass() != obj.getClass())
-            return false;
-        @SuppressWarnings("unchecked")
-        AggregationBuilder<AB> other = (AggregationBuilder<AB>) obj;
-        if (!Objects.equals(name, other.name))
-            return false;
-        if (!Objects.equals(type, other.type))
-            return false;
-        if (!Objects.equals(metaData, other.metaData))
-            return false;
-        if (!Objects.equals(factoriesBuilder, other.factoriesBuilder))
-            return false;
-        return doEquals(obj);
-    }
-
-    protected abstract boolean doEquals(Object obj);
+    protected abstract AggregationBuilder subAggregations(AggregatorFactories.Builder subFactories);
 
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java
index faceada6415..bd81a397f04 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/Aggregator.java
@@ -55,7 +55,7 @@ public abstract class Aggregator extends BucketCollector implements Releasable {
          * @return                  The resolved aggregator factory or {@code null} in case the aggregation should be skipped
          * @throws java.io.IOException      When parsing fails
          */
-        AggregationBuilder<?> parse(String aggregationName, QueryParseContext context) throws IOException;
+        AggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException;
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
index 4e07ffcc4d8..38fb0e6a431 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
@@ -24,7 +24,6 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.AggregationPath;
 import org.elasticsearch.search.aggregations.support.AggregationPath.PathElement;
@@ -45,18 +44,18 @@ import java.util.Set;
 public class AggregatorFactories {
 
     public static final AggregatorFactories EMPTY = new AggregatorFactories(null, new AggregatorFactory<?>[0],
-            new ArrayList<PipelineAggregatorBuilder<?>>());
+            new ArrayList<PipelineAggregatorBuilder>());
 
     private AggregatorFactory<?> parent;
     private AggregatorFactory<?>[] factories;
-    private List<PipelineAggregatorBuilder<?>> pipelineAggregatorFactories;
+    private List<PipelineAggregatorBuilder> pipelineAggregatorFactories;
 
     public static Builder builder() {
         return new Builder();
     }
 
     private AggregatorFactories(AggregatorFactory<?> parent, AggregatorFactory<?>[] factories,
-            List<PipelineAggregatorBuilder<?>> pipelineAggregators) {
+            List<PipelineAggregatorBuilder> pipelineAggregators) {
         this.parent = parent;
         this.factories = factories;
         this.pipelineAggregatorFactories = pipelineAggregators;
@@ -64,7 +63,7 @@ public class AggregatorFactories {
 
     public List<PipelineAggregator> createPipelineAggregators() throws IOException {
         List<PipelineAggregator> pipelineAggregators = new ArrayList<>();
-        for (PipelineAggregatorBuilder<?> factory : this.pipelineAggregatorFactories) {
+        for (PipelineAggregatorBuilder factory : this.pipelineAggregatorFactories) {
             pipelineAggregators.add(factory.create());
         }
         return pipelineAggregators;
@@ -117,15 +116,15 @@ public class AggregatorFactories {
         for (AggregatorFactory<?> factory : factories) {
             factory.validate();
         }
-        for (PipelineAggregatorBuilder<?> factory : pipelineAggregatorFactories) {
+        for (PipelineAggregatorBuilder factory : pipelineAggregatorFactories) {
             factory.validate(parent, factories, pipelineAggregatorFactories);
         }
     }
 
     public static class Builder extends ToXContentToBytes implements Writeable {
         private final Set<String> names = new HashSet<>();
-        private final List<AggregationBuilder<?>> aggregationBuilders = new ArrayList<>();
-        private final List<PipelineAggregatorBuilder<?>> pipelineAggregatorBuilders = new ArrayList<>();
+        private final List<AggregationBuilder> aggregationBuilders = new ArrayList<>();
+        private final List<PipelineAggregatorBuilder> pipelineAggregatorBuilders = new ArrayList<>();
         private boolean skipResolveOrder;
 
         /**
@@ -151,11 +150,11 @@ public class AggregatorFactories {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeVInt(this.aggregationBuilders.size());
-            for (AggregationBuilder<?> factory : aggregationBuilders) {
+            for (AggregationBuilder factory : aggregationBuilders) {
                 out.writeNamedWriteable(factory);
             }
             out.writeVInt(this.pipelineAggregatorBuilders.size());
-            for (PipelineAggregatorBuilder<?> factory : pipelineAggregatorBuilders) {
+            for (PipelineAggregatorBuilder factory : pipelineAggregatorBuilders) {
                 out.writeNamedWriteable(factory);
             }
         }
@@ -164,7 +163,7 @@ public class AggregatorFactories {
             throw new UnsupportedOperationException("This needs to be removed");
         }
 
-        public Builder addAggregator(AggregationBuilder<?> factory) {
+        public Builder addAggregator(AggregationBuilder factory) {
             if (!names.add(factory.name)) {
                 throw new IllegalArgumentException("Two sibling aggregations cannot have the same name: [" + factory.name + "]");
             }
@@ -172,7 +171,7 @@ public class AggregatorFactories {
             return this;
         }
 
-        public Builder addPipelineAggregator(PipelineAggregatorBuilder<?> pipelineAggregatorFactory) {
+        public Builder addPipelineAggregator(PipelineAggregatorBuilder pipelineAggregatorFactory) {
             this.pipelineAggregatorBuilders.add(pipelineAggregatorFactory);
             return this;
         }
@@ -189,7 +188,7 @@ public class AggregatorFactories {
             if (aggregationBuilders.isEmpty() && pipelineAggregatorBuilders.isEmpty()) {
                 return EMPTY;
             }
-            List<PipelineAggregatorBuilder<?>> orderedpipelineAggregators = null;
+            List<PipelineAggregatorBuilder> orderedpipelineAggregators = null;
             if (skipResolveOrder) {
                 orderedpipelineAggregators = new ArrayList<>(pipelineAggregatorBuilders);
             } else {
@@ -202,31 +201,31 @@ public class AggregatorFactories {
             return new AggregatorFactories(parent, aggFactories, orderedpipelineAggregators);
         }
 
-        private List<PipelineAggregatorBuilder<?>> resolvePipelineAggregatorOrder(
-                List<PipelineAggregatorBuilder<?>> pipelineAggregatorBuilders, List<AggregationBuilder<?>> aggBuilders) {
-            Map<String, PipelineAggregatorBuilder<?>> pipelineAggregatorBuildersMap = new HashMap<>();
-            for (PipelineAggregatorBuilder<?> builder : pipelineAggregatorBuilders) {
+        private List<PipelineAggregatorBuilder> resolvePipelineAggregatorOrder(
+                List<PipelineAggregatorBuilder> pipelineAggregatorBuilders, List<AggregationBuilder> aggBuilders) {
+            Map<String, PipelineAggregatorBuilder> pipelineAggregatorBuildersMap = new HashMap<>();
+            for (PipelineAggregatorBuilder builder : pipelineAggregatorBuilders) {
                 pipelineAggregatorBuildersMap.put(builder.getName(), builder);
             }
-            Map<String, AggregationBuilder<?>> aggBuildersMap = new HashMap<>();
-            for (AggregationBuilder<?> aggBuilder : aggBuilders) {
+            Map<String, AggregationBuilder> aggBuildersMap = new HashMap<>();
+            for (AggregationBuilder aggBuilder : aggBuilders) {
                 aggBuildersMap.put(aggBuilder.name, aggBuilder);
             }
-            List<PipelineAggregatorBuilder<?>> orderedPipelineAggregatorrs = new LinkedList<>();
-            List<PipelineAggregatorBuilder<?>> unmarkedBuilders = new ArrayList<PipelineAggregatorBuilder<?>>(pipelineAggregatorBuilders);
-            Set<PipelineAggregatorBuilder<?>> temporarilyMarked = new HashSet<PipelineAggregatorBuilder<?>>();
+            List<PipelineAggregatorBuilder> orderedPipelineAggregatorrs = new LinkedList<>();
+            List<PipelineAggregatorBuilder> unmarkedBuilders = new ArrayList<PipelineAggregatorBuilder>(pipelineAggregatorBuilders);
+            Set<PipelineAggregatorBuilder> temporarilyMarked = new HashSet<PipelineAggregatorBuilder>();
             while (!unmarkedBuilders.isEmpty()) {
-                PipelineAggregatorBuilder<?> builder = unmarkedBuilders.get(0);
+                PipelineAggregatorBuilder builder = unmarkedBuilders.get(0);
                 resolvePipelineAggregatorOrder(aggBuildersMap, pipelineAggregatorBuildersMap, orderedPipelineAggregatorrs, unmarkedBuilders,
                         temporarilyMarked, builder);
             }
             return orderedPipelineAggregatorrs;
         }
 
-        private void resolvePipelineAggregatorOrder(Map<String, AggregationBuilder<?>> aggBuildersMap,
-                Map<String, PipelineAggregatorBuilder<?>> pipelineAggregatorBuildersMap,
-                List<PipelineAggregatorBuilder<?>> orderedPipelineAggregators, List<PipelineAggregatorBuilder<?>> unmarkedBuilders,
-                Set<PipelineAggregatorBuilder<?>> temporarilyMarked, PipelineAggregatorBuilder<?> builder) {
+        private void resolvePipelineAggregatorOrder(Map<String, AggregationBuilder> aggBuildersMap,
+                Map<String, PipelineAggregatorBuilder> pipelineAggregatorBuildersMap,
+                List<PipelineAggregatorBuilder> orderedPipelineAggregators, List<PipelineAggregatorBuilder> unmarkedBuilders,
+                Set<PipelineAggregatorBuilder> temporarilyMarked, PipelineAggregatorBuilder builder) {
             if (temporarilyMarked.contains(builder)) {
                 throw new IllegalArgumentException("Cyclical dependency found with pipeline aggregator [" + builder.getName() + "]");
             } else if (unmarkedBuilders.contains(builder)) {
@@ -238,7 +237,7 @@ public class AggregatorFactories {
                     if (bucketsPath.equals("_count") || bucketsPath.equals("_key")) {
                         continue;
                     } else if (aggBuildersMap.containsKey(firstAggName)) {
-                        AggregationBuilder<?> aggBuilder = aggBuildersMap.get(firstAggName);
+                        AggregationBuilder aggBuilder = aggBuildersMap.get(firstAggName);
                         for (int i = 1; i < bucketsPathElements.size(); i++) {
                             PathElement pathElement = bucketsPathElements.get(i);
                             String aggName = pathElement.name;
@@ -247,9 +246,9 @@ public class AggregatorFactories {
                             } else {
                                 // Check the non-pipeline sub-aggregator
                                 // factories
-                                AggregationBuilder<?>[] subBuilders = aggBuilder.factoriesBuilder.getAggregatorFactories();
+                                AggregationBuilder[] subBuilders = aggBuilder.factoriesBuilder.getAggregatorFactories();
                                 boolean foundSubBuilder = false;
-                                for (AggregationBuilder<?> subBuilder : subBuilders) {
+                                for (AggregationBuilder subBuilder : subBuilders) {
                                     if (aggName.equals(subBuilder.name)) {
                                         aggBuilder = subBuilder;
                                         foundSubBuilder = true;
@@ -258,9 +257,9 @@ public class AggregatorFactories {
                                 }
                                 // Check the pipeline sub-aggregator factories
                                 if (!foundSubBuilder && (i == bucketsPathElements.size() - 1)) {
-                                    List<PipelineAggregatorBuilder<?>> subPipelineBuilders = aggBuilder.factoriesBuilder.pipelineAggregatorBuilders;
-                                    for (PipelineAggregatorBuilder<?> subFactory : subPipelineBuilders) {
-                                        if (aggName.equals(subFactory.name())) {
+                                    List<PipelineAggregatorBuilder> subPipelineBuilders = aggBuilder.factoriesBuilder.pipelineAggregatorBuilders;
+                                    for (PipelineAggregatorBuilder subFactory : subPipelineBuilders) {
+                                        if (aggName.equals(subFactory.getName())) {
                                             foundSubBuilder = true;
                                             break;
                                         }
@@ -274,7 +273,7 @@ public class AggregatorFactories {
                         }
                         continue;
                     } else {
-                        PipelineAggregatorBuilder<?> matchingBuilder = pipelineAggregatorBuildersMap.get(firstAggName);
+                        PipelineAggregatorBuilder matchingBuilder = pipelineAggregatorBuildersMap.get(firstAggName);
                         if (matchingBuilder != null) {
                             resolvePipelineAggregatorOrder(aggBuildersMap, pipelineAggregatorBuildersMap, orderedPipelineAggregators,
                                     unmarkedBuilders, temporarilyMarked, matchingBuilder);
@@ -289,11 +288,11 @@ public class AggregatorFactories {
             }
         }
 
-        AggregationBuilder<?>[] getAggregatorFactories() {
-            return this.aggregationBuilders.toArray(new AggregationBuilder<?>[this.aggregationBuilders.size()]);
+        AggregationBuilder[] getAggregatorFactories() {
+            return this.aggregationBuilders.toArray(new AggregationBuilder[this.aggregationBuilders.size()]);
         }
 
-        List<PipelineAggregatorBuilder<?>> getPipelineAggregatorFactories() {
+        List<PipelineAggregatorBuilder> getPipelineAggregatorFactories() {
             return this.pipelineAggregatorBuilders;
         }
 
@@ -305,12 +304,12 @@ public class AggregatorFactories {
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
             if (aggregationBuilders != null) {
-                for (AggregationBuilder<?> subAgg : aggregationBuilders) {
+                for (AggregationBuilder subAgg : aggregationBuilders) {
                     subAgg.toXContent(builder, params);
                 }
             }
             if (pipelineAggregatorBuilders != null) {
-                for (PipelineAggregatorBuilder<?> subAgg : pipelineAggregatorBuilders) {
+                for (PipelineAggregatorBuilder subAgg : pipelineAggregatorBuilders) {
                     subAgg.toXContent(builder, params);
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorParsers.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorParsers.java
index 55345d6e5ec..178c1666b18 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorParsers.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorParsers.java
@@ -24,7 +24,6 @@ import org.elasticsearch.common.xcontent.ParseFieldRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
 
 import java.io.IOException;
 import java.util.Map;
@@ -104,8 +103,8 @@ public class AggregatorParsers {
                         + token + "], expected a [" + XContentParser.Token.START_OBJECT + "].");
             }
 
-            AggregationBuilder<?> aggFactory = null;
-            PipelineAggregatorBuilder<?> pipelineAggregatorFactory = null;
+            AggregationBuilder aggFactory = null;
+            PipelineAggregatorBuilder pipelineAggregatorFactory = null;
             AggregatorFactories.Builder subFactories = null;
 
             Map<String, Object> metaData = null;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregatorBuilder.java
new file mode 100644
index 00000000000..1931ec232d5
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregatorBuilder.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations;
+
+import org.elasticsearch.action.support.ToXContentToBytes;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A factory that knows how to create an {@link PipelineAggregator} of a
+ * specific type.
+ */
+public abstract class PipelineAggregatorBuilder extends ToXContentToBytes
+        implements NamedWriteable {
+
+    protected final String name;
+    protected final String[] bucketsPaths;
+
+    /**
+     * Constructs a new pipeline aggregator factory.
+     *
+     * @param name
+     *            The aggregation name
+     */
+    protected PipelineAggregatorBuilder(String name, String[] bucketsPaths) {
+        if (name == null) {
+            throw new IllegalArgumentException("[name] must not be null: [" + name + "]");
+        }
+        if (bucketsPaths == null) {
+            throw new IllegalArgumentException("[bucketsPaths] must not be null: [" + name + "]");
+        }
+        this.name = name;
+        this.bucketsPaths = bucketsPaths;
+    }
+
+    /** Return this aggregation's name. */
+    public String getName() {
+        return name;
+    }
+
+    /** Return the consumed buckets paths. */
+    public final String[] getBucketsPaths() {
+        return bucketsPaths;
+    }
+
+    /**
+     * Internal: Validates the state of this factory (makes sure the factory is properly
+     * configured)
+     */
+    protected abstract void validate(AggregatorFactory<?> parent, AggregatorFactory<?>[] factories,
+            List<PipelineAggregatorBuilder> pipelineAggregatorFactories);
+
+    /**
+     * Creates the pipeline aggregator
+     *
+     * @return The created aggregator
+     */
+    protected abstract PipelineAggregator create() throws IOException;
+
+    /** Associate metadata with this {@link PipelineAggregatorBuilder}. */
+    public abstract PipelineAggregatorBuilder setMetaData(Map<String, Object> metaData);
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
index 1a73471c32e..a971035b5f6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
@@ -28,7 +28,7 @@ import org.elasticsearch.index.query.EmptyQueryBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
@@ -36,7 +36,7 @@ import org.elasticsearch.search.aggregations.support.AggregationContext;
 import java.io.IOException;
 import java.util.Objects;
 
-public class FilterAggregationBuilder extends AggregationBuilder<FilterAggregationBuilder> {
+public class FilterAggregationBuilder extends AbstractAggregationBuilder<FilterAggregationBuilder> {
     public static final String NAME = InternalFilter.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregationBuilder.java
index 45744db3e30..5056e3a67bb 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregationBuilder.java
@@ -28,7 +28,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregator.KeyedFilter;
@@ -43,7 +43,7 @@ import java.util.Objects;
 
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 
-public class FiltersAggregationBuilder extends AggregationBuilder<FiltersAggregationBuilder> {
+public class FiltersAggregationBuilder extends AbstractAggregationBuilder<FiltersAggregationBuilder> {
     public static final String NAME = InternalFilters.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java
index 0f7e0713598..2772f6aa880 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java
@@ -24,14 +24,14 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 
 import java.io.IOException;
 
-public class GlobalAggregationBuilder extends AggregationBuilder<GlobalAggregationBuilder> {
+public class GlobalAggregationBuilder extends AbstractAggregationBuilder<GlobalAggregationBuilder> {
     public static final String NAME = InternalGlobal.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java
index 33771910f16..9cde938fb4f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregationBuilder.java
@@ -26,7 +26,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
@@ -34,7 +34,7 @@ import org.elasticsearch.search.aggregations.support.AggregationContext;
 import java.io.IOException;
 import java.util.Objects;
 
-public class NestedAggregationBuilder extends AggregationBuilder<NestedAggregationBuilder> {
+public class NestedAggregationBuilder extends AbstractAggregationBuilder<NestedAggregationBuilder> {
     public static final String NAME = InternalNested.TYPE.name();
     public static final ParseField AGGREGATION_FIELD_NAME = new ParseField(NAME);
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java
index 6257f219daf..01daa27f3d6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregationBuilder.java
@@ -26,7 +26,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
@@ -34,7 +34,7 @@ import org.elasticsearch.search.aggregations.support.AggregationContext;
 import java.io.IOException;
 import java.util.Objects;
 
-public class ReverseNestedAggregationBuilder extends AggregationBuilder<ReverseNestedAggregationBuilder> {
+public class ReverseNestedAggregationBuilder extends AbstractAggregationBuilder<ReverseNestedAggregationBuilder> {
     public static final String NAME = InternalReverseNested.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java
index 1220a2ddd42..bdbff785043 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java
@@ -26,7 +26,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
@@ -34,7 +34,7 @@ import org.elasticsearch.search.aggregations.support.AggregationContext;
 import java.io.IOException;
 import java.util.Objects;
 
-public class SamplerAggregationBuilder extends AggregationBuilder<SamplerAggregationBuilder> {
+public class SamplerAggregationBuilder extends AbstractAggregationBuilder<SamplerAggregationBuilder> {
     public static final String NAME = InternalSampler.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java
index d5bdf2f5626..5a5f9894f33 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java
@@ -29,7 +29,7 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptParameterParser;
 import org.elasticsearch.script.ScriptParameterParser.ScriptParameterValue;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
@@ -40,7 +40,7 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 
-public class ScriptedMetricAggregationBuilder extends AggregationBuilder<ScriptedMetricAggregationBuilder> {
+public class ScriptedMetricAggregationBuilder extends AbstractAggregationBuilder<ScriptedMetricAggregationBuilder> {
 
     public static final String NAME = InternalScriptedMetric.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java
index 7195482f147..00d783c12e3 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java
@@ -29,8 +29,8 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregationInitializationException;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
@@ -51,7 +51,7 @@ import java.util.List;
 import java.util.Objects;
 import java.util.Set;
 
-public class TopHitsAggregationBuilder extends AggregationBuilder<TopHitsAggregationBuilder> {
+public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHitsAggregationBuilder> {
     public static final String NAME = InternalTopHits.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregatorBuilder.java
similarity index 73%
rename from core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregatorBuilder.java
index bdcefa054fc..fc308b1bcdd 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregatorBuilder.java
@@ -18,13 +18,12 @@
  */
 package org.elasticsearch.search.aggregations.pipeline;
 
-import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -33,52 +32,32 @@ import java.util.Map;
 import java.util.Objects;
 
 /**
- * A factory that knows how to create an {@link PipelineAggregator} of a
- * specific type.
+ * Base implementation of a {@link PipelineAggregatorBuilder}.
  */
-public abstract class PipelineAggregatorBuilder<PAB extends PipelineAggregatorBuilder<PAB>> extends ToXContentToBytes
-        implements NamedWriteable {
+public abstract class AbstractPipelineAggregatorBuilder<PAB extends AbstractPipelineAggregatorBuilder<PAB>>
+        extends PipelineAggregatorBuilder {
 
     /**
      * Field shared by many parsers.
      */
     public static final ParseField BUCKETS_PATH_FIELD = new ParseField("buckets_path");
 
-    protected final String name;
     protected final String type;
-    protected final String[] bucketsPaths;
     protected Map<String, Object> metaData;
 
-    /**
-     * Constructs a new pipeline aggregator factory.
-     *
-     * @param name
-     *            The aggregation name
-     * @param type
-     *            The aggregation type
-     */
-    protected PipelineAggregatorBuilder(String name, String type, String[] bucketsPaths) {
-        if (name == null) {
-            throw new IllegalArgumentException("[name] must not be null: [" + name + "]");
-        }
+    protected AbstractPipelineAggregatorBuilder(String name, String type, String[] bucketsPaths) {
+        super(name, bucketsPaths);
         if (type == null) {
             throw new IllegalArgumentException("[type] must not be null: [" + name + "]");
         }
-        if (bucketsPaths == null) {
-            throw new IllegalArgumentException("[bucketsPaths] must not be null: [" + name + "]");
-        }
-        this.name = name;
         this.type = type;
-        this.bucketsPaths = bucketsPaths;
     }
 
     /**
      * Read from a stream.
      */
-    protected PipelineAggregatorBuilder(StreamInput in, String type) throws IOException {
-        name = in.readString();
-        this.type = type;
-        bucketsPaths = in.readStringArray();
+    protected AbstractPipelineAggregatorBuilder(StreamInput in, String type) throws IOException {
+        this(in.readString(), type, in.readStringArray());
         metaData = in.readMap();
     }
 
@@ -92,10 +71,6 @@ public abstract class PipelineAggregatorBuilder<PAB extends PipelineAggregatorBu
 
     protected abstract void doWriteTo(StreamOutput out) throws IOException;
 
-    public String name() {
-        return name;
-    }
-
     public String type() {
         return type;
     }
@@ -104,8 +79,9 @@ public abstract class PipelineAggregatorBuilder<PAB extends PipelineAggregatorBu
      * Validates the state of this factory (makes sure the factory is properly
      * configured)
      */
+    @Override
     public final void validate(AggregatorFactory<?> parent, AggregatorFactory<?>[] factories,
-            List<PipelineAggregatorBuilder<?>> pipelineAggregatorFactories) {
+            List<PipelineAggregatorBuilder> pipelineAggregatorFactories) {
         doValidate(parent, factories, pipelineAggregatorFactories);
     }
 
@@ -116,29 +92,23 @@ public abstract class PipelineAggregatorBuilder<PAB extends PipelineAggregatorBu
      *
      * @return The created aggregator
      */
+    @Override
     public final PipelineAggregator create() throws IOException {
         PipelineAggregator aggregator = createInternal(this.metaData);
         return aggregator;
     }
 
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] factories,
-            List<PipelineAggregatorBuilder<?>> pipelineAggregatorFactories) {
+            List<PipelineAggregatorBuilder> pipelineAggregatorFactories) {
     }
 
     @SuppressWarnings("unchecked")
+    @Override
     public PAB setMetaData(Map<String, Object> metaData) {
         this.metaData = metaData;
         return (PAB) this;
     }
 
-    public String getName() {
-        return name;
-    }
-
-    public String[] getBucketsPaths() {
-        return bucketsPaths;
-    }
-
     @Override
     public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(getName());
@@ -164,7 +134,7 @@ public abstract class PipelineAggregatorBuilder<PAB extends PipelineAggregatorBu
     }
 
     /**
-     * @return <code>true</code> if the {@link PipelineAggregatorBuilder}
+     * @return <code>true</code> if the {@link AbstractPipelineAggregatorBuilder}
      *         overrides the XContent rendering of the bucketPath option.
      */
     protected boolean overrideBucketsPath() {
@@ -187,7 +157,7 @@ public abstract class PipelineAggregatorBuilder<PAB extends PipelineAggregatorBu
         if (getClass() != obj.getClass())
             return false;
         @SuppressWarnings("unchecked")
-        PipelineAggregatorBuilder<PAB> other = (PipelineAggregatorBuilder<PAB>) obj;
+        AbstractPipelineAggregatorBuilder<PAB> other = (AbstractPipelineAggregatorBuilder<PAB>) obj;
         if (!Objects.equals(name, other.name))
             return false;
         if (!Objects.equals(type, other.type))
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpers.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpers.java
index 21337b73043..ea0973dd1a2 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpers.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpers.java
@@ -156,7 +156,7 @@ public class BucketHelpers {
         try {
             Object propertyValue = bucket.getProperty(agg.getName(), aggPathAsList);
             if (propertyValue == null) {
-                throw new AggregationExecutionException(PipelineAggregatorBuilder.BUCKETS_PATH_FIELD.getPreferredName()
+                throw new AggregationExecutionException(AbstractPipelineAggregatorBuilder.BUCKETS_PATH_FIELD.getPreferredName()
                         + " must reference either a number value or a single value numeric metric aggregation");
             } else {
                 double value;
@@ -165,7 +165,7 @@ public class BucketHelpers {
                 } else if (propertyValue instanceof InternalNumericMetricsAggregation.SingleValue) {
                     value = ((InternalNumericMetricsAggregation.SingleValue) propertyValue).value();
                 } else {
-                    throw new AggregationExecutionException(PipelineAggregatorBuilder.BUCKETS_PATH_FIELD.getPreferredName()
+                    throw new AggregationExecutionException(AbstractPipelineAggregatorBuilder.BUCKETS_PATH_FIELD.getPreferredName()
                             + " must reference either a number value or a single value numeric metric aggregation, got: "
                             + propertyValue.getClass().getCanonicalName());
                 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregator.java
index e80a7dbfcf9..4b0dea560a8 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregator.java
@@ -28,6 +28,7 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
 import org.elasticsearch.search.aggregations.InternalAggregation.Type;
+import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
 
 import java.io.IOException;
 import java.util.Map;
@@ -55,7 +56,7 @@ public abstract class PipelineAggregator implements Streamable {
          * @throws java.io.IOException
          *             When parsing fails
          */
-        PipelineAggregatorBuilder<?> parse(String pipelineAggregatorName, QueryParseContext context)
+        PipelineAggregatorBuilder parse(String pipelineAggregatorName, QueryParseContext context)
                 throws IOException;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsPipelineAggregatorBuilder.java
index 981ddcc13ad..4945fd3a2e5 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsPipelineAggregatorBuilder.java
@@ -24,9 +24,10 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
+import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
 
 import java.io.IOException;
 import java.util.List;
@@ -34,7 +35,7 @@ import java.util.Map;
 import java.util.Objects;
 
 public abstract class BucketMetricsPipelineAggregatorBuilder<AF extends BucketMetricsPipelineAggregatorBuilder<AF>>
-        extends PipelineAggregatorBuilder<AF> {
+        extends AbstractPipelineAggregatorBuilder<AF> {
 
     private String format = null;
     private GapPolicy gapPolicy = GapPolicy.SKIP;
@@ -106,7 +107,7 @@ public abstract class BucketMetricsPipelineAggregatorBuilder<AF extends BucketMe
 
     @Override
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] aggFactories,
-            List<PipelineAggregatorBuilder<?>> pipelineAggregatorFactories) {
+            List<PipelineAggregatorBuilder> pipelineAggregatorFactories) {
         if (bucketsPaths.length != 1) {
             throw new IllegalStateException(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                     + " must contain a single entry for aggregation [" + name + "]");
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/avg/AvgBucketPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/avg/AvgBucketPipelineAggregatorBuilder.java
index 248186b24bc..509b45868b1 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/avg/AvgBucketPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/avg/AvgBucketPipelineAggregatorBuilder.java
@@ -24,8 +24,8 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsParser;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsPipelineAggregatorBuilder;
 
@@ -60,7 +60,7 @@ public class AvgBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
 
     @Override
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] aggFactories,
-            List<PipelineAggregatorBuilder<?>> pipelineAggregatorFactories) {
+            List<PipelineAggregatorBuilder> pipelineAggregatorFactories) {
         if (bucketsPaths.length != 1) {
             throw new IllegalStateException(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                     + " must contain a single entry for aggregation [" + name + "]");
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/max/MaxBucketPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/max/MaxBucketPipelineAggregatorBuilder.java
index 1ea6dcb59ff..2800a63661d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/max/MaxBucketPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/max/MaxBucketPipelineAggregatorBuilder.java
@@ -24,8 +24,8 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsParser;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsPipelineAggregatorBuilder;
 
@@ -60,7 +60,7 @@ public class MaxBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
 
     @Override
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] aggFactories,
-            List<PipelineAggregatorBuilder<?>> pipelineAggregatorFactories) {
+            List<PipelineAggregatorBuilder> pipelineAggregatorFactories) {
         if (bucketsPaths.length != 1) {
             throw new IllegalStateException(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                     + " must contain a single entry for aggregation [" + name + "]");
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/min/MinBucketPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/min/MinBucketPipelineAggregatorBuilder.java
index 1681cc1b7b2..ee7ae9838cc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/min/MinBucketPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/min/MinBucketPipelineAggregatorBuilder.java
@@ -24,8 +24,8 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsParser;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsPipelineAggregatorBuilder;
 
@@ -60,7 +60,7 @@ public class MinBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
 
     @Override
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] aggFactories,
-            List<PipelineAggregatorBuilder<?>> pipelineAggregatorFactories) {
+            List<PipelineAggregatorBuilder> pipelineAggregatorFactories) {
         if (bucketsPaths.length != 1) {
             throw new IllegalStateException(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                     + " must contain a single entry for aggregation [" + name + "]");
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregatorBuilder.java
index 8f3ee4d9eea..ddea1a65545 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregatorBuilder.java
@@ -27,13 +27,12 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsParser;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsPipelineAggregatorBuilder;
 
 import java.io.IOException;
-import java.text.ParseException;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
@@ -97,7 +96,7 @@ public class PercentilesBucketPipelineAggregatorBuilder
 
     @Override
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] aggFactories,
-            List<PipelineAggregatorBuilder<?>> pipelineAggregatorFactories) {
+            List<PipelineAggregatorBuilder> pipelineAggregatorFactories) {
         if (bucketsPaths.length != 1) {
             throw new IllegalStateException(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                     + " must contain a single entry for aggregation [" + name + "]");
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregatorBuilder.java
index 24ca14bff28..a3a37bfe9d6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregatorBuilder.java
@@ -24,9 +24,9 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsParser;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsPipelineAggregatorBuilder;
 
@@ -62,7 +62,7 @@ public class StatsBucketPipelineAggregatorBuilder extends BucketMetricsPipelineA
 
     @Override
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] aggFactories,
-            List<PipelineAggregatorBuilder<?>> pipelineAggregatorFactories) {
+            List<PipelineAggregatorBuilder> pipelineAggregatorFactories) {
         if (bucketsPaths.length != 1) {
             throw new IllegalStateException(Parser.BUCKETS_PATH.getPreferredName()
                     + " must contain a single entry for aggregation [" + name + "]");
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketPipelineAggregatorBuilder.java
index b27ce374978..8b17cf7db18 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketPipelineAggregatorBuilder.java
@@ -24,9 +24,9 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsPipelineAggregatorBuilder;
 
 import java.io.IOException;
@@ -85,7 +85,7 @@ public class ExtendedStatsBucketPipelineAggregatorBuilder
 
     @Override
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] aggFactories,
-            List<PipelineAggregatorBuilder<?>> pipelineAggregatorFactories) {
+            List<PipelineAggregatorBuilder> pipelineAggregatorFactories) {
         if (bucketsPaths.length != 1) {
             throw new IllegalStateException(Parser.BUCKETS_PATH.getPreferredName()
                     + " must contain a single entry for aggregation [" + name + "]");
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/sum/SumBucketPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/sum/SumBucketPipelineAggregatorBuilder.java
index 3fb2322dedc..72ba158a2b2 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/sum/SumBucketPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/sum/SumBucketPipelineAggregatorBuilder.java
@@ -24,8 +24,8 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsParser;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsPipelineAggregatorBuilder;
 
@@ -60,7 +60,7 @@ public class SumBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
 
     @Override
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] aggFactories,
-            List<PipelineAggregatorBuilder<?>> pipelineAggregatorFactories) {
+            List<PipelineAggregatorBuilder> pipelineAggregatorFactories) {
         if (bucketsPaths.length != 1) {
             throw new IllegalStateException(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                     + " must contain a single entry for aggregation [" + name + "]");
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregatorBuilder.java
index 6e266b92f6b..946c48080db 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregatorBuilder.java
@@ -29,9 +29,9 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.Script.ScriptField;
 import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -46,7 +46,7 @@ import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.FORMAT;
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.GAP_POLICY;
 
-public class BucketScriptPipelineAggregatorBuilder extends PipelineAggregatorBuilder<BucketScriptPipelineAggregatorBuilder> {
+public class BucketScriptPipelineAggregatorBuilder extends AbstractPipelineAggregatorBuilder<BucketScriptPipelineAggregatorBuilder> {
     public static final String NAME = BucketScriptPipelineAggregator.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregatorBuilder.java
index f109e1c61e4..8e1b854b350 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregatorBuilder.java
@@ -28,9 +28,9 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.Script.ScriptField;
+import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -44,7 +44,7 @@ import java.util.TreeMap;
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.BUCKETS_PATH;
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.GAP_POLICY;
 
-public class BucketSelectorPipelineAggregatorBuilder extends PipelineAggregatorBuilder<BucketSelectorPipelineAggregatorBuilder> {
+public class BucketSelectorPipelineAggregatorBuilder extends AbstractPipelineAggregatorBuilder<BucketSelectorPipelineAggregatorBuilder> {
     public static final String NAME = BucketSelectorPipelineAggregator.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregatorBuilder.java
index 7f2208c5480..a4b2178d0aa 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregatorBuilder.java
@@ -28,9 +28,10 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.AbstractHistogramAggregatorFactory;
+import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsParser;
 
 import java.io.IOException;
@@ -42,7 +43,7 @@ import java.util.Objects;
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.BUCKETS_PATH;
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.FORMAT;
 
-public class CumulativeSumPipelineAggregatorBuilder extends PipelineAggregatorBuilder<CumulativeSumPipelineAggregatorBuilder> {
+public class CumulativeSumPipelineAggregatorBuilder extends AbstractPipelineAggregatorBuilder<CumulativeSumPipelineAggregatorBuilder> {
     public static final String NAME = CumulativeSumPipelineAggregator.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
@@ -98,7 +99,7 @@ public class CumulativeSumPipelineAggregatorBuilder extends PipelineAggregatorBu
 
     @Override
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] aggFactories,
-            List<PipelineAggregatorBuilder<?>> pipelineAggregatorFactories) {
+            List<PipelineAggregatorBuilder> pipelineAggregatorFactories) {
         if (bucketsPaths.length != 1) {
             throw new IllegalStateException(BUCKETS_PATH.getPreferredName()
                     + " must contain a single entry for aggregation [" + name + "]");
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregatorBuilder.java
index ac970720330..0aba20e37c4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregatorBuilder.java
@@ -30,12 +30,13 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.AbstractHistogramAggregatorFactory;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregatorFactory;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
+import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
 import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
@@ -44,7 +45,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
-public class DerivativePipelineAggregatorBuilder extends PipelineAggregatorBuilder<DerivativePipelineAggregatorBuilder> {
+public class DerivativePipelineAggregatorBuilder extends AbstractPipelineAggregatorBuilder<DerivativePipelineAggregatorBuilder> {
     public static final String NAME = DerivativePipelineAggregator.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
@@ -156,7 +157,7 @@ public class DerivativePipelineAggregatorBuilder extends PipelineAggregatorBuild
 
     @Override
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] aggFactories,
-            List<PipelineAggregatorBuilder<?>> pipelineAggregatoractories) {
+            List<PipelineAggregatorBuilder> pipelineAggregatoractories) {
         if (bucketsPaths.length != 1) {
             throw new IllegalStateException(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                     + " must contain a single entry for aggregation [" + name + "]");
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregatorBuilder.java
index a1df9f2e896..2cf8a1bd046 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregatorBuilder.java
@@ -29,10 +29,11 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.AbstractHistogramAggregatorFactory;
+import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.MovAvgModel;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.MovAvgModelBuilder;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.SimpleModel;
@@ -48,7 +49,7 @@ import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.FORMAT;
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.GAP_POLICY;
 
-public class MovAvgPipelineAggregatorBuilder extends PipelineAggregatorBuilder<MovAvgPipelineAggregatorBuilder> {
+public class MovAvgPipelineAggregatorBuilder extends AbstractPipelineAggregatorBuilder<MovAvgPipelineAggregatorBuilder> {
     public static final String NAME = MovAvgPipelineAggregator.TYPE.name();
     public static final ParseField AGGREGATION_FIELD_NAME = new ParseField(NAME);
 
@@ -256,7 +257,7 @@ public class MovAvgPipelineAggregatorBuilder extends PipelineAggregatorBuilder<M
 
     @Override
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] aggFactories,
-            List<PipelineAggregatorBuilder<?>> pipelineAggregatoractories) {
+            List<PipelineAggregatorBuilder> pipelineAggregatoractories) {
         if (minimize != null && minimize && !model.canBeMinimized()) {
             // If the user asks to minimize, but this model doesn't support
             // it, throw exception
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregatorBuilder.java
index 2361683a6bc..5f9096542ea 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregatorBuilder.java
@@ -27,9 +27,9 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -40,7 +40,7 @@ import java.util.Objects;
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.BUCKETS_PATH;
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.FORMAT;
 
-public class SerialDiffPipelineAggregatorBuilder extends PipelineAggregatorBuilder<SerialDiffPipelineAggregatorBuilder> {
+public class SerialDiffPipelineAggregatorBuilder extends AbstractPipelineAggregatorBuilder<SerialDiffPipelineAggregatorBuilder> {
     public static final String NAME = SerialDiffPipelineAggregator.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
index 78d2a2da10f..582ae6befe0 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
@@ -31,7 +31,7 @@ import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregationInitializationException;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
@@ -47,7 +47,7 @@ import java.util.Objects;
  *
  */
 public abstract class ValuesSourceAggregationBuilder<VS extends ValuesSource, AB extends ValuesSourceAggregationBuilder<VS, AB>>
-        extends AggregationBuilder<AB> {
+        extends AbstractAggregationBuilder<AB> {
 
     public static abstract class LeafOnly<VS extends ValuesSource, AB extends ValuesSourceAggregationBuilder<VS, AB>>
             extends ValuesSourceAggregationBuilder<VS, AB> {
diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index dc45a99fa87..a2c2f3a0fa6 100644
--- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -44,7 +44,7 @@ import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
 import org.elasticsearch.search.fetch.source.FetchSourceContext;
 import org.elasticsearch.search.highlight.HighlightBuilder;
 import org.elasticsearch.search.internal.SearchContext;
@@ -600,7 +600,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
     /**
      * Add an aggregation to perform as part of the search.
      */
-    public SearchSourceBuilder aggregation(AggregationBuilder<?> aggregation) {
+    public SearchSourceBuilder aggregation(AggregationBuilder aggregation) {
             if (aggregations == null) {
             aggregations = AggregatorFactories.builder();
             }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
index e7933fb1d0c..c81ee0bb9c2 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
@@ -79,7 +79,7 @@ import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
 import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.equalTo;
 
-public abstract class BaseAggregationTestCase<AB extends AggregationBuilder<AB>> extends ESTestCase {
+public abstract class BaseAggregationTestCase<AB extends AbstractAggregationBuilder<AB>> extends ESTestCase {
 
     protected static final String STRING_FIELD_NAME = "mapped_string";
     protected static final String INT_FIELD_NAME = "mapped_int";
@@ -237,7 +237,7 @@ public abstract class BaseAggregationTestCase<AB extends AggregationBuilder<AB>>
         assertSame(XContentParser.Token.FIELD_NAME, parser.nextToken());
         assertEquals(testAgg.type.name(), parser.currentName());
         assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        AggregationBuilder<?> newAgg = aggParsers.parser(testAgg.getType(), ParseFieldMatcher.STRICT).parse(testAgg.name, parseContext);
+        AggregationBuilder newAgg = aggParsers.parser(testAgg.getType(), ParseFieldMatcher.STRICT).parse(testAgg.name, parseContext);
         assertSame(XContentParser.Token.END_OBJECT, parser.currentToken());
         assertSame(XContentParser.Token.END_OBJECT, parser.nextToken());
         assertSame(XContentParser.Token.END_OBJECT, parser.nextToken());
@@ -257,7 +257,7 @@ public abstract class BaseAggregationTestCase<AB extends AggregationBuilder<AB>>
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             output.writeNamedWriteable(testAgg);
             try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
-                AggregationBuilder<?> deserialized = in.readNamedWriteable(AggregationBuilder.class);
+                AggregationBuilder deserialized = in.readNamedWriteable(AggregationBuilder.class);
                 assertEquals(testAgg, deserialized);
                 assertEquals(testAgg.hashCode(), deserialized.hashCode());
                 assertNotSame(testAgg, deserialized);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
index 8fdc3de325d..078c04d5ffe 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
@@ -59,7 +59,7 @@ import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptSettings;
 import org.elasticsearch.search.SearchModule;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregatorBuilder;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -80,7 +80,7 @@ import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
 import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.equalTo;
 
-public abstract class BasePipelineAggregationTestCase<AF extends PipelineAggregatorBuilder> extends ESTestCase {
+public abstract class BasePipelineAggregationTestCase<AF extends AbstractPipelineAggregatorBuilder<AF>> extends ESTestCase {
 
     protected static final String STRING_FIELD_NAME = "mapped_string";
     protected static final String INT_FIELD_NAME = "mapped_int";
@@ -234,13 +234,13 @@ public abstract class BasePipelineAggregationTestCase<AF extends PipelineAggrega
         logger.info("Content string: {}", contentString);
         assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
         assertSame(XContentParser.Token.FIELD_NAME, parser.nextToken());
-        assertEquals(testAgg.name(), parser.currentName());
+        assertEquals(testAgg.getName(), parser.currentName());
         assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
         assertSame(XContentParser.Token.FIELD_NAME, parser.nextToken());
         assertEquals(testAgg.type(), parser.currentName());
         assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        PipelineAggregatorBuilder<?> newAgg = aggParsers.pipelineParser(testAgg.getWriteableName(), ParseFieldMatcher.STRICT)
-                .parse(testAgg.name(), parseContext);
+        PipelineAggregatorBuilder newAgg = aggParsers.pipelineParser(testAgg.getWriteableName(), ParseFieldMatcher.STRICT)
+                .parse(testAgg.getName(), parseContext);
         assertSame(XContentParser.Token.END_OBJECT, parser.currentToken());
         assertSame(XContentParser.Token.END_OBJECT, parser.nextToken());
         assertSame(XContentParser.Token.END_OBJECT, parser.nextToken());
@@ -260,7 +260,7 @@ public abstract class BasePipelineAggregationTestCase<AF extends PipelineAggrega
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             output.writeNamedWriteable(testAgg);
             try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
-                PipelineAggregatorBuilder<?> deserializedQuery = in.readNamedWriteable(PipelineAggregatorBuilder.class);
+                PipelineAggregatorBuilder deserializedQuery = in.readNamedWriteable(PipelineAggregatorBuilder.class);
                 assertEquals(deserializedQuery, testAgg);
                 assertEquals(deserializedQuery.hashCode(), testAgg.hashCode());
                 assertNotSame(deserializedQuery, testAgg);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/AbstractBucketMetricsTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/AbstractBucketMetricsTestCase.java
index fdc52c37c76..20c4a614ba2 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/AbstractBucketMetricsTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/AbstractBucketMetricsTestCase.java
@@ -22,7 +22,7 @@ package org.elasticsearch.search.aggregations.pipeline.bucketmetrics;
 import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 
-public abstract class AbstractBucketMetricsTestCase<PAF extends BucketMetricsPipelineAggregatorBuilder>
+public abstract class AbstractBucketMetricsTestCase<PAF extends BucketMetricsPipelineAggregatorBuilder<PAF>>
         extends BasePipelineAggregationTestCase<PAF> {
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgTests.java
index 0bfebf08074..8aa294e987f 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgTests.java
@@ -24,8 +24,8 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
+import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.movavg.MovAvgPipelineAggregatorBuilder;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.EwmaModel;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.HoltLinearModel;
@@ -110,13 +110,13 @@ public class MovAvgTests extends BasePipelineAggregationTestCase<MovAvgPipelineA
         QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
         assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
         assertSame(XContentParser.Token.FIELD_NAME, parser.nextToken());
-        assertEquals(expected.name(), parser.currentName());
+        assertEquals(expected.getName(), parser.currentName());
         assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
         assertSame(XContentParser.Token.FIELD_NAME, parser.nextToken());
         assertEquals(expected.type(), parser.currentName());
         assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        PipelineAggregatorBuilder<?> newAgg = aggParsers.pipelineParser(expected.getWriteableName(), ParseFieldMatcher.STRICT)
-                .parse(expected.name(), parseContext);
+        PipelineAggregatorBuilder newAgg = aggParsers.pipelineParser(expected.getWriteableName(), ParseFieldMatcher.STRICT)
+                .parse(expected.getName(), parseContext);
         assertSame(XContentParser.Token.END_OBJECT, parser.currentToken());
         assertSame(XContentParser.Token.END_OBJECT, parser.nextToken());
         assertSame(XContentParser.Token.END_OBJECT, parser.nextToken());
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequestBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequestBuilder.java
index 7d42382e76e..36eca70049a 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequestBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequestBuilder.java
@@ -28,7 +28,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
 import org.elasticsearch.search.highlight.HighlightBuilder;
 import org.elasticsearch.search.sort.SortBuilder;
 
@@ -170,7 +170,7 @@ public class PercolateRequestBuilder extends ActionRequestBuilder<PercolateReque
      * Delegates to
      * {@link PercolateSourceBuilder#addAggregation(AggregationBuilder)}
      */
-    public PercolateRequestBuilder addAggregation(AggregationBuilder<?> aggregationBuilder) {
+    public PercolateRequestBuilder addAggregation(AggregationBuilder aggregationBuilder) {
         sourceBuilder().addAggregation(aggregationBuilder);
         return this;
     }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateSourceBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateSourceBuilder.java
index 88e709447f7..8de722260f0 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateSourceBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateSourceBuilder.java
@@ -30,7 +30,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.highlight.HighlightBuilder;
 import org.elasticsearch.search.sort.ScoreSortBuilder;
@@ -56,8 +56,8 @@ public class PercolateSourceBuilder extends ToXContentToBytes {
     private List<SortBuilder<?>> sorts;
     private Boolean trackScores;
     private HighlightBuilder highlightBuilder;
-    private List<AggregationBuilder<?>> aggregationBuilders;
-    private List<PipelineAggregatorBuilder<?>> pipelineAggregationBuilders;
+    private List<AggregationBuilder> aggregationBuilders;
+    private List<PipelineAggregatorBuilder> pipelineAggregationBuilders;
 
     /**
      * Sets the document to run the percolate queries against.
@@ -129,7 +129,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes {
     /**
      * Add an aggregation definition.
      */
-    public PercolateSourceBuilder addAggregation(AggregationBuilder<?> aggregationBuilder) {
+    public PercolateSourceBuilder addAggregation(AggregationBuilder aggregationBuilder) {
         if (aggregationBuilders == null) {
             aggregationBuilders = new ArrayList<>();
         }
@@ -140,7 +140,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes {
     /**
      * Add an aggregation definition.
      */
-    public PercolateSourceBuilder addAggregation(PipelineAggregatorBuilder<?> aggregationBuilder) {
+    public PercolateSourceBuilder addAggregation(PipelineAggregatorBuilder aggregationBuilder) {
         if (pipelineAggregationBuilders == null) {
             pipelineAggregationBuilders = new ArrayList<>();
         }
@@ -178,12 +178,12 @@ public class PercolateSourceBuilder extends ToXContentToBytes {
             builder.field("aggregations");
             builder.startObject();
             if (aggregationBuilders != null) {
-                for (AggregationBuilder<?> aggregation : aggregationBuilders) {
+                for (AggregationBuilder aggregation : aggregationBuilders) {
                     aggregation.toXContent(builder, params);
                 }
             }
             if (pipelineAggregationBuilders != null) {
-                for (PipelineAggregatorBuilder<?> aggregation : pipelineAggregationBuilders) {
+                for (PipelineAggregatorBuilder aggregation : pipelineAggregationBuilders) {
                     aggregation.toXContent(builder, params);
                 }
             }

From 966bfc3f2da795c771e07359aa0547dabedcacfd Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Wed, 1 Jun 2016 10:28:35 +0200
Subject: [PATCH 0545/1311] Throw IllegalStateException when handshake fails
 due to version or cluster mismatch (#18676)

We do throw ConnectTransportException which is logged in trace level hiding a potentially
important information  when an old or wrong node wants to connect. We should throw ISE and
log as warn.
---
 .../transport/TransportService.java           | 12 +--
 .../NettyTransportServiceHandshakeTests.java  | 85 ++++++++-----------
 2 files changed, 43 insertions(+), 54 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/transport/TransportService.java b/core/src/main/java/org/elasticsearch/transport/TransportService.java
index f53d63740ff..c8f7be07b94 100644
--- a/core/src/main/java/org/elasticsearch/transport/TransportService.java
+++ b/core/src/main/java/org/elasticsearch/transport/TransportService.java
@@ -316,8 +316,8 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
      * @param checkClusterName whether or not to ignore cluster name
      *                         mismatches
      * @return the connected node
-     * @throws ConnectTransportException if the connection or the
-     *                                   handshake failed
+     * @throws ConnectTransportException if the connection failed
+     * @throws IllegalStateException if the handshake failed
      */
     public DiscoveryNode connectToNodeLightAndHandshake(
             final DiscoveryNode node,
@@ -329,7 +329,7 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
         transport.connectToNodeLight(node);
         try {
             return handshake(node, handshakeTimeout, checkClusterName);
-        } catch (ConnectTransportException e) {
+        } catch (ConnectTransportException | IllegalStateException e) {
             transport.disconnectFromNode(node);
             throw e;
         }
@@ -353,13 +353,13 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
                     }
                 }).txGet();
         } catch (Exception e) {
-            throw new ConnectTransportException(node, "handshake failed", e);
+            throw new IllegalStateException("handshake failed with " + node, e);
         }
 
         if (checkClusterName && !Objects.equals(clusterName, response.clusterName)) {
-            throw new ConnectTransportException(node, "handshake failed, mismatched cluster name [" + response.clusterName + "]");
+            throw new IllegalStateException("handshake failed, mismatched cluster name [" + response.clusterName + "] - " + node);
         } else if (!isVersionCompatible(response.version)) {
-            throw new ConnectTransportException(node, "handshake failed, incompatible version [" + response.version + "]");
+            throw new IllegalStateException("handshake failed, incompatible version [" + response.version + "] - " + node);
         }
 
         return response.discoveryNode;
diff --git a/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java b/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java
index e1f9bf75874..6a7e0a0de36 100644
--- a/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java
@@ -109,21 +109,20 @@ public class NettyTransportServiceHandshakeTests extends ESTestCase {
                         settings,
                         VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.CURRENT),
                         test);
-
+        DiscoveryNode discoveryNode = new DiscoveryNode(
+            "",
+            handleB.discoveryNode.getAddress(),
+            emptyMap(),
+            emptySet(),
+            Version.CURRENT.minimumCompatibilityVersion());
         DiscoveryNode connectedNode =
-                handleA.transportService.connectToNodeLightAndHandshake(
-                        new DiscoveryNode(
-                                "",
-                                handleB.discoveryNode.getAddress(),
-                                emptyMap(),
-                                emptySet(),
-                                Version.CURRENT.minimumCompatibilityVersion()),
-                        timeout);
+                handleA.transportService.connectToNodeLightAndHandshake(discoveryNode, timeout);
         assertNotNull(connectedNode);
 
         // the name and version should be updated
         assertEquals(connectedNode.getName(), "TS_B");
         assertEquals(connectedNode.getVersion(), handleB.discoveryNode.getVersion());
+        assertTrue(handleA.transportService.nodeConnected(discoveryNode));
     }
 
     public void testMismatchedClusterName() {
@@ -131,21 +130,17 @@ public class NettyTransportServiceHandshakeTests extends ESTestCase {
 
         NetworkHandle handleA = startServices("TS_A", settings, Version.CURRENT, new ClusterName("a"));
         NetworkHandle handleB = startServices("TS_B", settings, Version.CURRENT, new ClusterName("b"));
-
-        try {
-            handleA.transportService.connectToNodeLightAndHandshake(
-                    new DiscoveryNode(
-                            "",
-                            handleB.discoveryNode.getAddress(),
-                            emptyMap(),
-                            emptySet(),
-                            Version.CURRENT.minimumCompatibilityVersion()),
-                    timeout);
-            fail("expected handshake to fail from mismatched cluster names");
-        } catch (ConnectTransportException e) {
-            assertThat(e.getMessage(), containsString("handshake failed, mismatched cluster name [Cluster [b]]"));
-        }
-    }
+        DiscoveryNode discoveryNode = new DiscoveryNode(
+            "",
+            handleB.discoveryNode.getAddress(),
+            emptyMap(),
+            emptySet(),
+            Version.CURRENT.minimumCompatibilityVersion());
+        IllegalStateException ex = expectThrows(IllegalStateException.class, () -> handleA.transportService.connectToNodeLightAndHandshake(
+                discoveryNode, timeout));
+        assertThat(ex.getMessage(), containsString("handshake failed, mismatched cluster name [Cluster [b]]"));
+        assertFalse(handleA.transportService.nodeConnected(discoveryNode));
+}
 
     public void testIncompatibleVersions() {
         Settings settings = Settings.EMPTY;
@@ -154,20 +149,16 @@ public class NettyTransportServiceHandshakeTests extends ESTestCase {
         NetworkHandle handleA = startServices("TS_A", settings, Version.CURRENT, test);
         NetworkHandle handleB =
                 startServices("TS_B", settings, VersionUtils.getPreviousVersion(Version.CURRENT.minimumCompatibilityVersion()), test);
-
-        try {
-            handleA.transportService.connectToNodeLightAndHandshake(
-                    new DiscoveryNode(
-                            "",
-                            handleB.discoveryNode.getAddress(),
-                            emptyMap(),
-                            emptySet(),
-                            Version.CURRENT.minimumCompatibilityVersion()),
-                    timeout);
-            fail("expected handshake to fail from incompatible versions");
-        } catch (ConnectTransportException e) {
-            assertThat(e.getMessage(), containsString("handshake failed, incompatible version"));
-        }
+        DiscoveryNode discoveryNode = new DiscoveryNode(
+            "",
+            handleB.discoveryNode.getAddress(),
+            emptyMap(),
+            emptySet(),
+            Version.CURRENT.minimumCompatibilityVersion());
+        IllegalStateException ex = expectThrows(IllegalStateException.class, () -> handleA.transportService.connectToNodeLightAndHandshake(
+            discoveryNode, timeout));
+        assertThat(ex.getMessage(), containsString("handshake failed, incompatible version"));
+        assertFalse(handleA.transportService.nodeConnected(discoveryNode));
     }
 
     public void testIgnoreMismatchedClusterName() {
@@ -181,19 +172,17 @@ public class NettyTransportServiceHandshakeTests extends ESTestCase {
                         VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.CURRENT),
                         new ClusterName("b")
                 );
-
-        DiscoveryNode connectedNode = handleA.transportService.connectToNodeLightAndHandshake(
-                new DiscoveryNode(
-                        "",
-                        handleB.discoveryNode.getAddress(),
-                        emptyMap(),
-                        emptySet(),
-                        Version.CURRENT.minimumCompatibilityVersion()),
-                timeout,
-                false);
+        DiscoveryNode discoveryNode = new DiscoveryNode(
+            "",
+            handleB.discoveryNode.getAddress(),
+            emptyMap(),
+            emptySet(),
+            Version.CURRENT.minimumCompatibilityVersion());
+        DiscoveryNode connectedNode = handleA.transportService.connectToNodeLightAndHandshake(discoveryNode, timeout, false);
         assertNotNull(connectedNode);
         assertEquals(connectedNode.getName(), "TS_B");
         assertEquals(connectedNode.getVersion(), handleB.discoveryNode.getVersion());
+        assertTrue(handleA.transportService.nodeConnected(discoveryNode));
     }
 
     private static class NetworkHandle {

From 766789b0f090915ec687c52cc832d9e45339a042 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Tue, 31 May 2016 11:58:20 +0200
Subject: [PATCH 0546/1311] ingest: added `ignore_failure` option to all
 processors

If this option is enabled on a processor it silently catches any processor related failure and continues executing the rest of the pipeline.

 Closes #18493
---
 .../resources/checkstyle_suppressions.xml     |  5 ---
 .../ingest/core/CompoundProcessor.java        | 14 ++++++-
 .../ingest/core/ConfigurationUtils.java       |  6 ++-
 .../elasticsearch/ingest/core/Pipeline.java   |  2 +-
 .../processor/TrackingResultProcessor.java    |  2 +-
 .../ingest/SimulateExecutionServiceTests.java |  2 +-
 .../ingest/PipelineExecutionServiceTests.java | 14 ++++---
 .../ingest/PipelineStoreTests.java            | 12 ++++--
 .../ingest/core/CompoundProcessorTests.java   | 41 +++++++++++++------
 .../ingest/core/PipelineFactoryTests.java     | 29 ++++++++++++-
 .../ingest/core/ValueSourceTests.java         |  6 ++-
 .../processor/ForEachProcessorTests.java      |  2 +-
 .../TrackingResultProcessorTests.java         |  4 +-
 docs/reference/ingest/ingest-node.asciidoc    | 24 +++++++++++
 .../test/ingest/50_on_failure.yaml            | 40 ++++++++++++++++++
 15 files changed, 163 insertions(+), 40 deletions(-)

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 6432ccb3ddd..c77ef0f9d5b 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -1037,11 +1037,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]store[/\\]IndicesStoreIntegrationIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]store[/\\]IndicesStoreTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]template[/\\]SimpleIndexTemplateIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]PipelineExecutionServiceTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]PipelineStoreTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]CompoundProcessorTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]PipelineFactoryTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]ValueSourceTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]AbstractStringProcessorTestCase.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]AppendProcessorTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]DateFormatTests.java" checks="LineLength" />
diff --git a/core/src/main/java/org/elasticsearch/ingest/core/CompoundProcessor.java b/core/src/main/java/org/elasticsearch/ingest/core/CompoundProcessor.java
index fb10f8a1927..652a2c184a2 100644
--- a/core/src/main/java/org/elasticsearch/ingest/core/CompoundProcessor.java
+++ b/core/src/main/java/org/elasticsearch/ingest/core/CompoundProcessor.java
@@ -38,19 +38,25 @@ public class CompoundProcessor implements Processor {
     public static final String ON_FAILURE_PROCESSOR_TYPE_FIELD = "on_failure_processor_type";
     public static final String ON_FAILURE_PROCESSOR_TAG_FIELD = "on_failure_processor_tag";
 
+    private final boolean ignoreFailure;
     private final List<Processor> processors;
     private final List<Processor> onFailureProcessors;
 
     public CompoundProcessor(Processor... processor) {
-        this(Arrays.asList(processor), Collections.emptyList());
+        this(false, Arrays.asList(processor), Collections.emptyList());
     }
 
-    public CompoundProcessor(List<Processor> processors, List<Processor> onFailureProcessors) {
+    public CompoundProcessor(boolean ignoreFailure, List<Processor> processors, List<Processor> onFailureProcessors) {
         super();
+        this.ignoreFailure = ignoreFailure;
         this.processors = processors;
         this.onFailureProcessors = onFailureProcessors;
     }
 
+    public boolean isIgnoreFailure() {
+        return ignoreFailure;
+    }
+
     public List<Processor> getOnFailureProcessors() {
         return onFailureProcessors;
     }
@@ -93,6 +99,10 @@ public class CompoundProcessor implements Processor {
             try {
                 processor.execute(ingestDocument);
             } catch (Exception e) {
+                if (ignoreFailure) {
+                    continue;
+                }
+
                 ElasticsearchException compoundProcessorException = newCompoundProcessorException(e, processor.getType(), processor.getTag());
                 if (onFailureProcessors.isEmpty()) {
                     throw compoundProcessorException;
diff --git a/core/src/main/java/org/elasticsearch/ingest/core/ConfigurationUtils.java b/core/src/main/java/org/elasticsearch/ingest/core/ConfigurationUtils.java
index 685f6f41de3..20370430bdc 100644
--- a/core/src/main/java/org/elasticsearch/ingest/core/ConfigurationUtils.java
+++ b/core/src/main/java/org/elasticsearch/ingest/core/ConfigurationUtils.java
@@ -250,6 +250,7 @@ public final class ConfigurationUtils {
     private static Processor readProcessor(ProcessorsRegistry processorRegistry, String type, Map<String, Object> config) throws Exception {
         Processor.Factory factory = processorRegistry.getProcessorFactory(type);
         if (factory != null) {
+            boolean ignoreFailure = ConfigurationUtils.readBooleanProperty(null, null, config, "ignore_failure", false);
             List<Map<String, Map<String, Object>>> onFailureProcessorConfigs =
                 ConfigurationUtils.readOptionalList(null, null, config, Pipeline.ON_FAILURE_KEY);
 
@@ -260,10 +261,11 @@ public final class ConfigurationUtils {
                 throw new ElasticsearchParseException("processor [{}] doesn't support one or more provided configuration parameters {}",
                     type, Arrays.toString(config.keySet().toArray()));
             }
-            if (onFailureProcessors.isEmpty()) {
+            if (onFailureProcessors.size() > 0 || ignoreFailure) {
+                return new CompoundProcessor(ignoreFailure, Collections.singletonList(processor), onFailureProcessors);
+            } else {
                 return processor;
             }
-            return new CompoundProcessor(Collections.singletonList(processor), onFailureProcessors);
         }
         throw new ElasticsearchParseException("No processor type exists with name [" + type + "]");
     }
diff --git a/core/src/main/java/org/elasticsearch/ingest/core/Pipeline.java b/core/src/main/java/org/elasticsearch/ingest/core/Pipeline.java
index aaae929e0a9..d0b1ca7f6cf 100644
--- a/core/src/main/java/org/elasticsearch/ingest/core/Pipeline.java
+++ b/core/src/main/java/org/elasticsearch/ingest/core/Pipeline.java
@@ -109,7 +109,7 @@ public final class Pipeline {
             if (config.isEmpty() == false) {
                 throw new ElasticsearchParseException("pipeline [" + id + "] doesn't support one or more provided configuration parameters " + Arrays.toString(config.keySet().toArray()));
             }
-            CompoundProcessor compoundProcessor = new CompoundProcessor(Collections.unmodifiableList(processors), Collections.unmodifiableList(onFailureProcessors));
+            CompoundProcessor compoundProcessor = new CompoundProcessor(false, Collections.unmodifiableList(processors), Collections.unmodifiableList(onFailureProcessors));
             return new Pipeline(id, description, compoundProcessor);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/TrackingResultProcessor.java b/core/src/main/java/org/elasticsearch/ingest/processor/TrackingResultProcessor.java
index af820318d83..861b0d68d2d 100644
--- a/core/src/main/java/org/elasticsearch/ingest/processor/TrackingResultProcessor.java
+++ b/core/src/main/java/org/elasticsearch/ingest/processor/TrackingResultProcessor.java
@@ -83,7 +83,7 @@ public final class TrackingResultProcessor implements Processor {
                 onFailureProcessors.add(new TrackingResultProcessor(processor, processorResultList));
             }
         }
-        return new CompoundProcessor(processors, onFailureProcessors);
+        return new CompoundProcessor(false, processors, onFailureProcessors);
     }
 }
 
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
index 30c29f8db5d..f7f2f0212dd 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
@@ -128,7 +128,7 @@ public class SimulateExecutionServiceTests extends ESTestCase {
         TestProcessor processor2 = new TestProcessor("processor_1", "mock", ingestDocument -> {});
         TestProcessor processor3 = new TestProcessor("processor_2", "mock", ingestDocument -> {});
         Pipeline pipeline = new Pipeline("_id", "_description",
-                new CompoundProcessor(new CompoundProcessor(Collections.singletonList(processor1),
+                new CompoundProcessor(new CompoundProcessor(false, Collections.singletonList(processor1),
                                 Collections.singletonList(processor2)), processor3));
         SimulateDocumentResult actualItemResponse = executionService.executeDocument(pipeline, ingestDocument, true);
         assertThat(processor1.getInvokedCounter(), equalTo(1));
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
index 254057d2ede..1b96939eb38 100644
--- a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
@@ -98,7 +98,8 @@ public class PipelineExecutionServiceTests extends ESTestCase {
 
         IndexRequest indexRequest1 = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id");
         bulkRequest.add(indexRequest1);
-        IndexRequest indexRequest2 = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("does_not_exist");
+        IndexRequest indexRequest2 =
+                new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("does_not_exist");
         bulkRequest.add(indexRequest2);
         @SuppressWarnings("unchecked")
         BiConsumer<IndexRequest, Throwable> failureHandler = mock(BiConsumer.class);
@@ -192,7 +193,8 @@ public class PipelineExecutionServiceTests extends ESTestCase {
         when(processor.getType()).thenReturn("mock_processor_type");
         when(processor.getTag()).thenReturn("mock_processor_tag");
         Processor onFailureProcessor = mock(Processor.class);
-        CompoundProcessor compoundProcessor = new CompoundProcessor(Collections.singletonList(processor), Collections.singletonList(new CompoundProcessor(onFailureProcessor)));
+        CompoundProcessor compoundProcessor = new CompoundProcessor(false, Collections.singletonList(processor),
+                Collections.singletonList(new CompoundProcessor(onFailureProcessor)));
         when(store.get("_id")).thenReturn(new Pipeline("_id", "_description", compoundProcessor));
         IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id");
         doThrow(new RuntimeException()).when(processor).execute(eqID("_index", "_type", "_id", Collections.emptyMap()));
@@ -208,7 +210,8 @@ public class PipelineExecutionServiceTests extends ESTestCase {
     public void testExecuteFailureWithOnFailure() throws Exception {
         Processor processor = mock(Processor.class);
         Processor onFailureProcessor = mock(Processor.class);
-        CompoundProcessor compoundProcessor = new CompoundProcessor(Collections.singletonList(processor), Collections.singletonList(new CompoundProcessor(onFailureProcessor)));
+        CompoundProcessor compoundProcessor = new CompoundProcessor(false, Collections.singletonList(processor),
+                Collections.singletonList(new CompoundProcessor(onFailureProcessor)));
         when(store.get("_id")).thenReturn(new Pipeline("_id", "_description", compoundProcessor));
         IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id");
         doThrow(new RuntimeException()).when(processor).execute(eqID("_index", "_type", "_id", Collections.emptyMap()));
@@ -227,8 +230,9 @@ public class PipelineExecutionServiceTests extends ESTestCase {
         Processor processor = mock(Processor.class);
         Processor onFailureProcessor = mock(Processor.class);
         Processor onFailureOnFailureProcessor = mock(Processor.class);
-        CompoundProcessor compoundProcessor = new CompoundProcessor(Collections.singletonList(processor),
-            Collections.singletonList(new CompoundProcessor(Collections.singletonList(onFailureProcessor), Collections.singletonList(onFailureOnFailureProcessor))));
+        CompoundProcessor compoundProcessor = new CompoundProcessor(false, Collections.singletonList(processor),
+            Collections.singletonList(new CompoundProcessor(false, Collections.singletonList(onFailureProcessor),
+                    Collections.singletonList(onFailureOnFailureProcessor))));
         when(store.get("_id")).thenReturn(new Pipeline("_id", "_description", compoundProcessor));
         IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id");
         doThrow(new RuntimeException()).when(onFailureOnFailureProcessor).execute(eqID("_index", "_type", "_id", Collections.emptyMap()));
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
index 10321e30301..c60bb3eecde 100644
--- a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
@@ -61,7 +61,8 @@ public class PipelineStoreTests extends ESTestCase {
         store = new PipelineStore(Settings.EMPTY);
         ProcessorsRegistry.Builder registryBuilder = new ProcessorsRegistry.Builder();
         registryBuilder.registerProcessor("set", (templateService, registry) -> new SetProcessor.Factory(TestTemplateService.instance()));
-        registryBuilder.registerProcessor("remove", (templateService, registry) -> new RemoveProcessor.Factory(TestTemplateService.instance()));
+        registryBuilder.registerProcessor("remove", (templateService, registry) ->
+                new RemoveProcessor.Factory(TestTemplateService.instance()));
         store.buildProcessorFactoryRegistry(registryBuilder, null);
     }
 
@@ -190,7 +191,8 @@ public class PipelineStoreTests extends ESTestCase {
         assertThat(pipeline, nullValue());
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); // Start empty
 
-        PutPipelineRequest putRequest = new PutPipelineRequest(id, new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"));
+        PutPipelineRequest putRequest = new PutPipelineRequest(id,
+                new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"));
         clusterState = store.innerPut(putRequest, clusterState);
         store.innerUpdatePipelines(clusterState);
         pipeline = store.get(id);
@@ -208,7 +210,8 @@ public class PipelineStoreTests extends ESTestCase {
     }
 
     public void testValidate() throws Exception {
-        PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}},{\"remove\" : {\"field\": \"_field\"}}]}"));
+        PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray(
+                "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}},{\"remove\" : {\"field\": \"_field\"}}]}"));
 
         DiscoveryNode node1 = new DiscoveryNode("_node_id1", new LocalTransportAddress("_id"),
                 emptyMap(), emptySet(), Version.CURRENT);
@@ -230,7 +233,8 @@ public class PipelineStoreTests extends ESTestCase {
     }
 
     public void testValidateNoIngestInfo() throws Exception {
-        PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"));
+        PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray(
+                "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"));
         try {
             store.validatePipeline(Collections.emptyMap(), putRequest);
             fail("exception expected");
diff --git a/core/src/test/java/org/elasticsearch/ingest/core/CompoundProcessorTests.java b/core/src/test/java/org/elasticsearch/ingest/core/CompoundProcessorTests.java
index b4ee7eca07c..0e110288597 100644
--- a/core/src/test/java/org/elasticsearch/ingest/core/CompoundProcessorTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/core/CompoundProcessorTests.java
@@ -24,6 +24,7 @@ import org.elasticsearch.ingest.TestProcessor;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -31,6 +32,7 @@ import java.util.Map;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.sameInstance;
 
 public class CompoundProcessorTests extends ESTestCase {
     private IngestDocument ingestDocument;
@@ -51,7 +53,7 @@ public class CompoundProcessorTests extends ESTestCase {
         TestProcessor processor = new TestProcessor(ingestDocument -> {});
         CompoundProcessor compoundProcessor = new CompoundProcessor(processor);
         assertThat(compoundProcessor.getProcessors().size(), equalTo(1));
-        assertThat(compoundProcessor.getProcessors().get(0), equalTo(processor));
+        assertThat(compoundProcessor.getProcessors().get(0), sameInstance(processor));
         assertThat(compoundProcessor.getOnFailureProcessors().isEmpty(), is(true));
         compoundProcessor.execute(ingestDocument);
         assertThat(processor.getInvokedCounter(), equalTo(1));
@@ -61,7 +63,7 @@ public class CompoundProcessorTests extends ESTestCase {
         TestProcessor processor = new TestProcessor(ingestDocument -> {throw new RuntimeException("error");});
         CompoundProcessor compoundProcessor = new CompoundProcessor(processor);
         assertThat(compoundProcessor.getProcessors().size(), equalTo(1));
-        assertThat(compoundProcessor.getProcessors().get(0), equalTo(processor));
+        assertThat(compoundProcessor.getProcessors().get(0), sameInstance(processor));
         assertThat(compoundProcessor.getOnFailureProcessors().isEmpty(), is(true));
         try {
             compoundProcessor.execute(ingestDocument);
@@ -72,6 +74,16 @@ public class CompoundProcessorTests extends ESTestCase {
         assertThat(processor.getInvokedCounter(), equalTo(1));
     }
 
+    public void testIgnoreFailure() throws Exception {
+        TestProcessor processor1 = new TestProcessor(ingestDocument -> {throw new RuntimeException("error");});
+        TestProcessor processor2 = new TestProcessor(ingestDocument -> {ingestDocument.setFieldValue("field", "value");});
+        CompoundProcessor compoundProcessor = new CompoundProcessor(true, Arrays.asList(processor1, processor2), Collections.emptyList());
+        compoundProcessor.execute(ingestDocument);
+        assertThat(processor1.getInvokedCounter(), equalTo(1));
+        assertThat(processor2.getInvokedCounter(), equalTo(1));
+        assertThat(ingestDocument.getFieldValue("field", String.class), equalTo("value"));
+    }
+
     public void testSingleProcessorWithOnFailureProcessor() throws Exception {
         TestProcessor processor1 = new TestProcessor("id", "first", ingestDocument -> {throw new RuntimeException("error");});
         TestProcessor processor2 = new TestProcessor(ingestDocument -> {
@@ -82,7 +94,8 @@ public class CompoundProcessorTests extends ESTestCase {
             assertThat(ingestMetadata.get(CompoundProcessor.ON_FAILURE_PROCESSOR_TAG_FIELD), equalTo("id"));
         });
 
-        CompoundProcessor compoundProcessor = new CompoundProcessor(Collections.singletonList(processor1), Collections.singletonList(processor2));
+        CompoundProcessor compoundProcessor = new CompoundProcessor(false, Collections.singletonList(processor1),
+                Collections.singletonList(processor2));
         compoundProcessor.execute(ingestDocument);
 
         assertThat(processor1.getInvokedCounter(), equalTo(1));
@@ -106,8 +119,10 @@ public class CompoundProcessorTests extends ESTestCase {
             assertThat(ingestMetadata.get(CompoundProcessor.ON_FAILURE_PROCESSOR_TYPE_FIELD), equalTo("second"));
             assertThat(ingestMetadata.get(CompoundProcessor.ON_FAILURE_PROCESSOR_TAG_FIELD), equalTo("id2"));
         });
-        CompoundProcessor compoundOnFailProcessor = new CompoundProcessor(Collections.singletonList(processorToFail), Collections.singletonList(lastProcessor));
-        CompoundProcessor compoundProcessor = new CompoundProcessor(Collections.singletonList(processor), Collections.singletonList(compoundOnFailProcessor));
+        CompoundProcessor compoundOnFailProcessor = new CompoundProcessor(false, Collections.singletonList(processorToFail),
+                Collections.singletonList(lastProcessor));
+        CompoundProcessor compoundProcessor = new CompoundProcessor(false, Collections.singletonList(processor),
+                Collections.singletonList(compoundOnFailProcessor));
         compoundProcessor.execute(ingestDocument);
 
         assertThat(processorToFail.getInvokedCounter(), equalTo(1));
@@ -126,7 +141,7 @@ public class CompoundProcessorTests extends ESTestCase {
 
         CompoundProcessor failCompoundProcessor = new CompoundProcessor(firstProcessor);
 
-        CompoundProcessor compoundProcessor = new CompoundProcessor(Collections.singletonList(failCompoundProcessor),
+        CompoundProcessor compoundProcessor = new CompoundProcessor(false, Collections.singletonList(failCompoundProcessor),
             Collections.singletonList(secondProcessor));
         compoundProcessor.execute(ingestDocument);
 
@@ -136,7 +151,8 @@ public class CompoundProcessorTests extends ESTestCase {
 
     public void testCompoundProcessorExceptionFail() throws Exception {
         TestProcessor firstProcessor = new TestProcessor("id1", "first", ingestDocument -> {throw new RuntimeException("error");});
-        TestProcessor failProcessor = new TestProcessor("tag_fail", "fail", ingestDocument -> {throw new RuntimeException("custom error message");});
+        TestProcessor failProcessor =
+                new TestProcessor("tag_fail", "fail", ingestDocument -> {throw new RuntimeException("custom error message");});
         TestProcessor secondProcessor = new TestProcessor("id3", "second", ingestDocument -> {
             Map<String, String> ingestMetadata = ingestDocument.getIngestMetadata();
             assertThat(ingestMetadata.entrySet(), hasSize(3));
@@ -145,10 +161,10 @@ public class CompoundProcessorTests extends ESTestCase {
             assertThat(ingestMetadata.get(CompoundProcessor.ON_FAILURE_PROCESSOR_TAG_FIELD), equalTo("tag_fail"));
         });
 
-        CompoundProcessor failCompoundProcessor = new CompoundProcessor(Collections.singletonList(firstProcessor),
+        CompoundProcessor failCompoundProcessor = new CompoundProcessor(false, Collections.singletonList(firstProcessor),
             Collections.singletonList(failProcessor));
 
-        CompoundProcessor compoundProcessor = new CompoundProcessor(Collections.singletonList(failCompoundProcessor),
+        CompoundProcessor compoundProcessor = new CompoundProcessor(false, Collections.singletonList(failCompoundProcessor),
             Collections.singletonList(secondProcessor));
         compoundProcessor.execute(ingestDocument);
 
@@ -158,7 +174,8 @@ public class CompoundProcessorTests extends ESTestCase {
 
     public void testCompoundProcessorExceptionFailInOnFailure() throws Exception {
         TestProcessor firstProcessor = new TestProcessor("id1", "first", ingestDocument -> {throw new RuntimeException("error");});
-        TestProcessor failProcessor = new TestProcessor("tag_fail", "fail", ingestDocument -> {throw new RuntimeException("custom error message");});
+        TestProcessor failProcessor =
+                new TestProcessor("tag_fail", "fail", ingestDocument -> {throw new RuntimeException("custom error message");});
         TestProcessor secondProcessor = new TestProcessor("id3", "second", ingestDocument -> {
             Map<String, String> ingestMetadata = ingestDocument.getIngestMetadata();
             assertThat(ingestMetadata.entrySet(), hasSize(3));
@@ -167,10 +184,10 @@ public class CompoundProcessorTests extends ESTestCase {
             assertThat(ingestMetadata.get(CompoundProcessor.ON_FAILURE_PROCESSOR_TAG_FIELD), equalTo("tag_fail"));
         });
 
-        CompoundProcessor failCompoundProcessor = new CompoundProcessor(Collections.singletonList(firstProcessor),
+        CompoundProcessor failCompoundProcessor = new CompoundProcessor(false, Collections.singletonList(firstProcessor),
             Collections.singletonList(new CompoundProcessor(failProcessor)));
 
-        CompoundProcessor compoundProcessor = new CompoundProcessor(Collections.singletonList(failCompoundProcessor),
+        CompoundProcessor compoundProcessor = new CompoundProcessor(false, Collections.singletonList(failCompoundProcessor),
             Collections.singletonList(secondProcessor));
         compoundProcessor.execute(ingestDocument);
 
diff --git a/core/src/test/java/org/elasticsearch/ingest/core/PipelineFactoryTests.java b/core/src/test/java/org/elasticsearch/ingest/core/PipelineFactoryTests.java
index 537d8f020e6..449f1836f2b 100644
--- a/core/src/test/java/org/elasticsearch/ingest/core/PipelineFactoryTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/core/PipelineFactoryTests.java
@@ -35,6 +35,7 @@ import java.util.Map;
 import java.util.prefs.PreferencesFactory;
 
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 
 public class PipelineFactoryTests extends ESTestCase {
@@ -45,7 +46,8 @@ public class PipelineFactoryTests extends ESTestCase {
         processorConfig0.put(AbstractProcessorFactory.TAG_KEY, "first-processor");
         Map<String, Object> pipelineConfig = new HashMap<>();
         pipelineConfig.put(Pipeline.DESCRIPTION_KEY, "_description");
-        pipelineConfig.put(Pipeline.PROCESSORS_KEY, Arrays.asList(Collections.singletonMap("test", processorConfig0), Collections.singletonMap("test", processorConfig1)));
+        pipelineConfig.put(Pipeline.PROCESSORS_KEY,
+                Arrays.asList(Collections.singletonMap("test", processorConfig0), Collections.singletonMap("test", processorConfig1)));
         Pipeline.Factory factory = new Pipeline.Factory();
         ProcessorsRegistry processorRegistry = createProcessorRegistry(Collections.singletonMap("test", new TestProcessor.Factory()));
         Pipeline pipeline = factory.create("_id", pipelineConfig, processorRegistry);
@@ -87,6 +89,28 @@ public class PipelineFactoryTests extends ESTestCase {
         assertThat(pipeline.getOnFailureProcessors().get(0).getType(), equalTo("test-processor"));
     }
 
+    public void testCreateWithPipelineIgnoreFailure() throws Exception {
+        Map<String, Object> processorConfig = new HashMap<>();
+        processorConfig.put("ignore_failure", true);
+
+        ProcessorsRegistry processorRegistry = createProcessorRegistry(Collections.singletonMap("test", new TestProcessor.Factory()));
+        Pipeline.Factory factory = new Pipeline.Factory();
+        Map<String, Object> pipelineConfig = new HashMap<>();
+        pipelineConfig.put(Pipeline.DESCRIPTION_KEY, "_description");
+        pipelineConfig.put(Pipeline.PROCESSORS_KEY,
+                Collections.singletonList(Collections.singletonMap("test", processorConfig)));
+
+        Pipeline pipeline = factory.create("_id", pipelineConfig, processorRegistry);
+        assertThat(pipeline.getId(), equalTo("_id"));
+        assertThat(pipeline.getDescription(), equalTo("_description"));
+        assertThat(pipeline.getProcessors().size(), equalTo(1));
+        assertThat(pipeline.getOnFailureProcessors().size(), equalTo(0));
+
+        CompoundProcessor processor = (CompoundProcessor) pipeline.getProcessors().get(0);
+        assertThat(processor.isIgnoreFailure(), is(true));
+        assertThat(processor.getProcessors().get(0).getType(), equalTo("test-processor"));
+    }
+
     public void testCreateUnusedProcessorOptions() throws Exception {
         Map<String, Object> processorConfig = new HashMap<>();
         processorConfig.put("unused", "value");
@@ -121,7 +145,8 @@ public class PipelineFactoryTests extends ESTestCase {
     public void testFlattenProcessors() throws Exception {
         TestProcessor testProcessor = new TestProcessor(ingestDocument -> {});
         CompoundProcessor processor1 = new CompoundProcessor(testProcessor, testProcessor);
-        CompoundProcessor processor2 = new CompoundProcessor(Collections.singletonList(testProcessor), Collections.singletonList(testProcessor));
+        CompoundProcessor processor2 =
+                new CompoundProcessor(false, Collections.singletonList(testProcessor), Collections.singletonList(testProcessor));
         Pipeline pipeline = new Pipeline("_id", "_description", new CompoundProcessor(processor1, processor2));
         List<Processor> flattened = pipeline.flattenAllProcessors();
         assertThat(flattened.size(), equalTo(4));
diff --git a/core/src/test/java/org/elasticsearch/ingest/core/ValueSourceTests.java b/core/src/test/java/org/elasticsearch/ingest/core/ValueSourceTests.java
index f2aa9f32bcd..57a6eb7ee68 100644
--- a/core/src/test/java/org/elasticsearch/ingest/core/ValueSourceTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/core/ValueSourceTests.java
@@ -51,7 +51,8 @@ public class ValueSourceTests extends ESTestCase {
         myPreciousMap.put("field2", "value2");
 
         IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
-        ingestDocument.setFieldValue(TestTemplateService.instance().compile("field1"), ValueSource.wrap(myPreciousMap, TestTemplateService.instance()));
+        ingestDocument.setFieldValue(TestTemplateService.instance().compile("field1"),
+                ValueSource.wrap(myPreciousMap, TestTemplateService.instance()));
         ingestDocument.removeField("field1.field2");
 
         assertThat(myPreciousMap.size(), equalTo(1));
@@ -63,7 +64,8 @@ public class ValueSourceTests extends ESTestCase {
         myPreciousList.add("value");
 
         IngestDocument ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
-        ingestDocument.setFieldValue(TestTemplateService.instance().compile("field1"), ValueSource.wrap(myPreciousList, TestTemplateService.instance()));
+        ingestDocument.setFieldValue(TestTemplateService.instance().compile("field1"),
+                ValueSource.wrap(myPreciousList, TestTemplateService.instance()));
         ingestDocument.removeField("field1.0");
 
         assertThat(myPreciousList.size(), equalTo(1));
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/ForEachProcessorTests.java b/core/src/test/java/org/elasticsearch/ingest/processor/ForEachProcessorTests.java
index 9f76a32e0c2..d1c2c6f1777 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/ForEachProcessorTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/processor/ForEachProcessorTests.java
@@ -91,7 +91,7 @@ public class ForEachProcessorTests extends ESTestCase {
         Processor onFailureProcessor = new TestProcessor(ingestDocument1 -> {});
         processor = new ForEachProcessor(
             "_tag", "values",
-            Collections.singletonList(new CompoundProcessor(Arrays.asList(testProcessor), Arrays.asList(onFailureProcessor)))
+            Collections.singletonList(new CompoundProcessor(false, Arrays.asList(testProcessor), Arrays.asList(onFailureProcessor)))
         );
         processor.execute(ingestDocument);
         assertThat(testProcessor.getInvokedCounter(), equalTo(3));
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/TrackingResultProcessorTests.java b/core/src/test/java/org/elasticsearch/ingest/processor/TrackingResultProcessorTests.java
index 2b53a9d08bc..e19a4df05c9 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/TrackingResultProcessorTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/processor/TrackingResultProcessorTests.java
@@ -91,8 +91,8 @@ public class TrackingResultProcessorTests extends ESTestCase {
         RuntimeException exception = new RuntimeException("fail");
         TestProcessor failProcessor = new TestProcessor("fail", "test", ingestDocument -> {  throw exception; });
         TestProcessor onFailureProcessor = new TestProcessor("success", "test", ingestDocument -> {});
-        CompoundProcessor actualProcessor = new CompoundProcessor(
-            Arrays.asList(new CompoundProcessor(
+        CompoundProcessor actualProcessor = new CompoundProcessor(false,
+            Arrays.asList(new CompoundProcessor(false,
                 Arrays.asList(failProcessor, onFailureProcessor),
                 Arrays.asList(onFailureProcessor, failProcessor))),
                 Arrays.asList(onFailureProcessor));
diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc
index 54a10f766fd..9cf51374ab4 100644
--- a/docs/reference/ingest/ingest-node.asciidoc
+++ b/docs/reference/ingest/ingest-node.asciidoc
@@ -566,6 +566,30 @@ the index to which failed documents get sent.
 }
 --------------------------------------------------
 
+Alternatively instead of defining behaviour in case of processor failure, it is also possible
+to ignore a failure and continue with the next processor by specifying the `ignore_failure` setting.
+
+In case in the example below the field `foo` doesn't exist the failure will be caught and the pipeline
+continues to execute, which in this case means that the pipeline does nothing.
+
+[source,js]
+--------------------------------------------------
+{
+  "description" : "my first pipeline with handled exceptions",
+  "processors" : [
+    {
+      "rename" : {
+        "field" : "foo",
+        "target_field" : "bar",
+        "ignore_failure" : true
+      }
+    }
+  ]
+}
+--------------------------------------------------
+
+The `ignore_failure` can be set on any processor and defaults to `false`.
+
 [float]
 [[accessing-error-metadata]]
 === Accessing Error Metadata From Processors Handling Exceptions
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/50_on_failure.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/50_on_failure.yaml
index b0c6c53fe67..bc0a765e2c0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/50_on_failure.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/50_on_failure.yaml
@@ -106,3 +106,43 @@
   - match: { _source.field1: "value1" }
   - match: { _source.foofield: "exists" }
   - match: { _source.foofield2: "ran" }
+
+---
+"Test pipeline with ignore_failure in a processor":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "fail" : {
+                  "message" : "_message",
+                  "ignore_failure": true
+                }
+              },
+              {
+                "set" : {
+                  "field" : "field",
+                  "value" : "value"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: {}
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.field: "value" }

From d182e171a49312d3158fc44417868fde11b8d775 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Tue, 31 May 2016 11:17:22 +0200
Subject: [PATCH 0547/1311] Upgrade to Lucene 6.0.1.

---
 buildSrc/version.properties                                     | 2 +-
 core/src/main/java/org/elasticsearch/Version.java               | 2 +-
 .../main/resources/org/elasticsearch/bootstrap/security.policy  | 2 +-
 .../resources/org/elasticsearch/bootstrap/test-framework.policy | 2 +-
 distribution/licenses/lucene-analyzers-common-6.0.0.jar.sha1    | 1 -
 distribution/licenses/lucene-analyzers-common-6.0.1.jar.sha1    | 1 +
 distribution/licenses/lucene-backward-codecs-6.0.0.jar.sha1     | 1 -
 distribution/licenses/lucene-backward-codecs-6.0.1.jar.sha1     | 1 +
 distribution/licenses/lucene-core-6.0.0.jar.sha1                | 1 -
 distribution/licenses/lucene-core-6.0.1.jar.sha1                | 1 +
 distribution/licenses/lucene-grouping-6.0.0.jar.sha1            | 1 -
 distribution/licenses/lucene-grouping-6.0.1.jar.sha1            | 1 +
 distribution/licenses/lucene-highlighter-6.0.0.jar.sha1         | 1 -
 distribution/licenses/lucene-highlighter-6.0.1.jar.sha1         | 1 +
 distribution/licenses/lucene-join-6.0.0.jar.sha1                | 1 -
 distribution/licenses/lucene-join-6.0.1.jar.sha1                | 1 +
 distribution/licenses/lucene-memory-6.0.0.jar.sha1              | 1 -
 distribution/licenses/lucene-memory-6.0.1.jar.sha1              | 1 +
 distribution/licenses/lucene-misc-6.0.0.jar.sha1                | 1 -
 distribution/licenses/lucene-misc-6.0.1.jar.sha1                | 1 +
 distribution/licenses/lucene-queries-6.0.0.jar.sha1             | 1 -
 distribution/licenses/lucene-queries-6.0.1.jar.sha1             | 1 +
 distribution/licenses/lucene-queryparser-6.0.0.jar.sha1         | 1 -
 distribution/licenses/lucene-queryparser-6.0.1.jar.sha1         | 1 +
 distribution/licenses/lucene-sandbox-6.0.0.jar.sha1             | 1 -
 distribution/licenses/lucene-sandbox-6.0.1.jar.sha1             | 1 +
 distribution/licenses/lucene-spatial-6.0.0.jar.sha1             | 1 -
 distribution/licenses/lucene-spatial-6.0.1.jar.sha1             | 1 +
 distribution/licenses/lucene-spatial-extras-6.0.0.jar.sha1      | 1 -
 distribution/licenses/lucene-spatial-extras-6.0.1.jar.sha1      | 1 +
 distribution/licenses/lucene-spatial3d-6.0.0.jar.sha1           | 1 -
 distribution/licenses/lucene-spatial3d-6.0.1.jar.sha1           | 1 +
 distribution/licenses/lucene-suggest-6.0.0.jar.sha1             | 1 -
 distribution/licenses/lucene-suggest-6.0.1.jar.sha1             | 1 +
 .../lang-expression/licenses/lucene-expressions-6.0.0.jar.sha1  | 1 -
 .../lang-expression/licenses/lucene-expressions-6.0.1.jar.sha1  | 1 +
 .../analysis-icu/licenses/lucene-analyzers-icu-6.0.0.jar.sha1   | 1 -
 .../analysis-icu/licenses/lucene-analyzers-icu-6.0.1.jar.sha1   | 1 +
 .../licenses/lucene-analyzers-kuromoji-6.0.0.jar.sha1           | 1 -
 .../licenses/lucene-analyzers-kuromoji-6.0.1.jar.sha1           | 1 +
 .../licenses/lucene-analyzers-phonetic-6.0.0.jar.sha1           | 1 -
 .../licenses/lucene-analyzers-phonetic-6.0.1.jar.sha1           | 1 +
 .../licenses/lucene-analyzers-smartcn-6.0.0.jar.sha1            | 1 -
 .../licenses/lucene-analyzers-smartcn-6.0.1.jar.sha1            | 1 +
 .../licenses/lucene-analyzers-stempel-6.0.0.jar.sha1            | 1 -
 .../licenses/lucene-analyzers-stempel-6.0.1.jar.sha1            | 1 +
 46 files changed, 25 insertions(+), 25 deletions(-)
 delete mode 100644 distribution/licenses/lucene-analyzers-common-6.0.0.jar.sha1
 create mode 100644 distribution/licenses/lucene-analyzers-common-6.0.1.jar.sha1
 delete mode 100644 distribution/licenses/lucene-backward-codecs-6.0.0.jar.sha1
 create mode 100644 distribution/licenses/lucene-backward-codecs-6.0.1.jar.sha1
 delete mode 100644 distribution/licenses/lucene-core-6.0.0.jar.sha1
 create mode 100644 distribution/licenses/lucene-core-6.0.1.jar.sha1
 delete mode 100644 distribution/licenses/lucene-grouping-6.0.0.jar.sha1
 create mode 100644 distribution/licenses/lucene-grouping-6.0.1.jar.sha1
 delete mode 100644 distribution/licenses/lucene-highlighter-6.0.0.jar.sha1
 create mode 100644 distribution/licenses/lucene-highlighter-6.0.1.jar.sha1
 delete mode 100644 distribution/licenses/lucene-join-6.0.0.jar.sha1
 create mode 100644 distribution/licenses/lucene-join-6.0.1.jar.sha1
 delete mode 100644 distribution/licenses/lucene-memory-6.0.0.jar.sha1
 create mode 100644 distribution/licenses/lucene-memory-6.0.1.jar.sha1
 delete mode 100644 distribution/licenses/lucene-misc-6.0.0.jar.sha1
 create mode 100644 distribution/licenses/lucene-misc-6.0.1.jar.sha1
 delete mode 100644 distribution/licenses/lucene-queries-6.0.0.jar.sha1
 create mode 100644 distribution/licenses/lucene-queries-6.0.1.jar.sha1
 delete mode 100644 distribution/licenses/lucene-queryparser-6.0.0.jar.sha1
 create mode 100644 distribution/licenses/lucene-queryparser-6.0.1.jar.sha1
 delete mode 100644 distribution/licenses/lucene-sandbox-6.0.0.jar.sha1
 create mode 100644 distribution/licenses/lucene-sandbox-6.0.1.jar.sha1
 delete mode 100644 distribution/licenses/lucene-spatial-6.0.0.jar.sha1
 create mode 100644 distribution/licenses/lucene-spatial-6.0.1.jar.sha1
 delete mode 100644 distribution/licenses/lucene-spatial-extras-6.0.0.jar.sha1
 create mode 100644 distribution/licenses/lucene-spatial-extras-6.0.1.jar.sha1
 delete mode 100644 distribution/licenses/lucene-spatial3d-6.0.0.jar.sha1
 create mode 100644 distribution/licenses/lucene-spatial3d-6.0.1.jar.sha1
 delete mode 100644 distribution/licenses/lucene-suggest-6.0.0.jar.sha1
 create mode 100644 distribution/licenses/lucene-suggest-6.0.1.jar.sha1
 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-6.0.0.jar.sha1
 create mode 100644 modules/lang-expression/licenses/lucene-expressions-6.0.1.jar.sha1
 delete mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-6.0.0.jar.sha1
 create mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-6.0.1.jar.sha1
 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.0.0.jar.sha1
 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.0.1.jar.sha1
 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.0.0.jar.sha1
 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.0.1.jar.sha1
 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.0.0.jar.sha1
 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.0.1.jar.sha1
 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.0.0.jar.sha1
 create mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.0.1.jar.sha1

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index b6e64a3c263..d9e3908df22 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 elasticsearch     = 5.0.0
-lucene            = 6.0.0
+lucene            = 6.0.1
 
 # optional dependencies
 spatial4j         = 0.6
diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java
index 76df1052015..5bdbf76265d 100644
--- a/core/src/main/java/org/elasticsearch/Version.java
+++ b/core/src/main/java/org/elasticsearch/Version.java
@@ -77,7 +77,7 @@ public class Version {
     public static final int V_5_0_0_alpha3_ID = 5000003;
     public static final Version V_5_0_0_alpha3 = new Version(V_5_0_0_alpha3_ID, org.apache.lucene.util.Version.LUCENE_6_0_0);
     public static final int V_5_0_0_ID = 5000099;
-    public static final Version V_5_0_0 = new Version(V_5_0_0_ID, org.apache.lucene.util.Version.LUCENE_6_0_0);
+    public static final Version V_5_0_0 = new Version(V_5_0_0_ID, org.apache.lucene.util.Version.LUCENE_6_0_1);
     public static final Version CURRENT = V_5_0_0;
 
     static {
diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
index 32f7b2bf0dd..ff0ea773667 100644
--- a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
+++ b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
@@ -31,7 +31,7 @@ grant codeBase "${codebase.securesm-1.0.jar}" {
 //// Very special jar permissions:
 //// These are dangerous permissions that we don't want to grant to everything.
 
-grant codeBase "${codebase.lucene-core-6.0.0.jar}" {
+grant codeBase "${codebase.lucene-core-6.0.1.jar}" {
   // needed to allow MMapDirectory's "unmap hack" (die unmap hack, die)
   // java 8 package
   permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
index 3c703fa72c7..d4ab6e01ab9 100644
--- a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
+++ b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
@@ -31,7 +31,7 @@ grant codeBase "${codebase.securemock-1.2.jar}" {
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
 };
 
-grant codeBase "${codebase.lucene-test-framework-6.0.0.jar}" {
+grant codeBase "${codebase.lucene-test-framework-6.0.1.jar}" {
   // needed by RamUsageTester
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
 };
diff --git a/distribution/licenses/lucene-analyzers-common-6.0.0.jar.sha1 b/distribution/licenses/lucene-analyzers-common-6.0.0.jar.sha1
deleted file mode 100644
index 28a2c9c3cd5..00000000000
--- a/distribution/licenses/lucene-analyzers-common-6.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-abb404e5a429f92f6b43ce11bef81e95e9bac6ab
\ No newline at end of file
diff --git a/distribution/licenses/lucene-analyzers-common-6.0.1.jar.sha1 b/distribution/licenses/lucene-analyzers-common-6.0.1.jar.sha1
new file mode 100644
index 00000000000..b581809a004
--- /dev/null
+++ b/distribution/licenses/lucene-analyzers-common-6.0.1.jar.sha1
@@ -0,0 +1 @@
+53953c1a9b097f83209c84a422cf8f9d271f47c1
\ No newline at end of file
diff --git a/distribution/licenses/lucene-backward-codecs-6.0.0.jar.sha1 b/distribution/licenses/lucene-backward-codecs-6.0.0.jar.sha1
deleted file mode 100644
index d594796d25d..00000000000
--- a/distribution/licenses/lucene-backward-codecs-6.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-daaa8f7a00915bf2c5dd3b74489f49736b50d2bd
\ No newline at end of file
diff --git a/distribution/licenses/lucene-backward-codecs-6.0.1.jar.sha1 b/distribution/licenses/lucene-backward-codecs-6.0.1.jar.sha1
new file mode 100644
index 00000000000..5433f09c993
--- /dev/null
+++ b/distribution/licenses/lucene-backward-codecs-6.0.1.jar.sha1
@@ -0,0 +1 @@
+3647088603be84b8f4916ef86954e3336b98d254
\ No newline at end of file
diff --git a/distribution/licenses/lucene-core-6.0.0.jar.sha1 b/distribution/licenses/lucene-core-6.0.0.jar.sha1
deleted file mode 100644
index d6c2fd6a2d9..00000000000
--- a/distribution/licenses/lucene-core-6.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3aa09d24a630d2249fdd7056ce569a72ff54ebcf
\ No newline at end of file
diff --git a/distribution/licenses/lucene-core-6.0.1.jar.sha1 b/distribution/licenses/lucene-core-6.0.1.jar.sha1
new file mode 100644
index 00000000000..6bcd7fc87f4
--- /dev/null
+++ b/distribution/licenses/lucene-core-6.0.1.jar.sha1
@@ -0,0 +1 @@
+40ccd40bec54266a10aa1f81c565914ede8c0ca0
\ No newline at end of file
diff --git a/distribution/licenses/lucene-grouping-6.0.0.jar.sha1 b/distribution/licenses/lucene-grouping-6.0.0.jar.sha1
deleted file mode 100644
index 684d63951db..00000000000
--- a/distribution/licenses/lucene-grouping-6.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a7664be36b0c765fc1394a1d65aef6a224ba4fa4
\ No newline at end of file
diff --git a/distribution/licenses/lucene-grouping-6.0.1.jar.sha1 b/distribution/licenses/lucene-grouping-6.0.1.jar.sha1
new file mode 100644
index 00000000000..b132acc9112
--- /dev/null
+++ b/distribution/licenses/lucene-grouping-6.0.1.jar.sha1
@@ -0,0 +1 @@
+010daaae60227fbe719ca95e9b6fcdb5c38d4eba
\ No newline at end of file
diff --git a/distribution/licenses/lucene-highlighter-6.0.0.jar.sha1 b/distribution/licenses/lucene-highlighter-6.0.0.jar.sha1
deleted file mode 100644
index 0b050c97a2d..00000000000
--- a/distribution/licenses/lucene-highlighter-6.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b81357ee48293500dd0876d33b59cbdc5e03c5f5
\ No newline at end of file
diff --git a/distribution/licenses/lucene-highlighter-6.0.1.jar.sha1 b/distribution/licenses/lucene-highlighter-6.0.1.jar.sha1
new file mode 100644
index 00000000000..95d4b3edab9
--- /dev/null
+++ b/distribution/licenses/lucene-highlighter-6.0.1.jar.sha1
@@ -0,0 +1 @@
+65d74c3642e6a86ba905045473b17cc84826527e
\ No newline at end of file
diff --git a/distribution/licenses/lucene-join-6.0.0.jar.sha1 b/distribution/licenses/lucene-join-6.0.0.jar.sha1
deleted file mode 100644
index 06eed8b2413..00000000000
--- a/distribution/licenses/lucene-join-6.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f8cef5f84d7c37f04d1bd0e8aac9ffcb685624b2
\ No newline at end of file
diff --git a/distribution/licenses/lucene-join-6.0.1.jar.sha1 b/distribution/licenses/lucene-join-6.0.1.jar.sha1
new file mode 100644
index 00000000000..07392cf260f
--- /dev/null
+++ b/distribution/licenses/lucene-join-6.0.1.jar.sha1
@@ -0,0 +1 @@
+2105e2826ce93d1f764e5a0a3afa9ee461d556c1
\ No newline at end of file
diff --git a/distribution/licenses/lucene-memory-6.0.0.jar.sha1 b/distribution/licenses/lucene-memory-6.0.0.jar.sha1
deleted file mode 100644
index 21df706dbd1..00000000000
--- a/distribution/licenses/lucene-memory-6.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ba02cc33668273258f66c61ffd18461e2a96c9d7
\ No newline at end of file
diff --git a/distribution/licenses/lucene-memory-6.0.1.jar.sha1 b/distribution/licenses/lucene-memory-6.0.1.jar.sha1
new file mode 100644
index 00000000000..b9820103d3f
--- /dev/null
+++ b/distribution/licenses/lucene-memory-6.0.1.jar.sha1
@@ -0,0 +1 @@
+e2cde0688e487a27d08df0c2d81d492b1f4cdc2a
\ No newline at end of file
diff --git a/distribution/licenses/lucene-misc-6.0.0.jar.sha1 b/distribution/licenses/lucene-misc-6.0.0.jar.sha1
deleted file mode 100644
index 104be50fc85..00000000000
--- a/distribution/licenses/lucene-misc-6.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-92bcb9af0b849addd6c4bf5a740ded646563e0f5
\ No newline at end of file
diff --git a/distribution/licenses/lucene-misc-6.0.1.jar.sha1 b/distribution/licenses/lucene-misc-6.0.1.jar.sha1
new file mode 100644
index 00000000000..2670ab628df
--- /dev/null
+++ b/distribution/licenses/lucene-misc-6.0.1.jar.sha1
@@ -0,0 +1 @@
+e6e59996fc324319d695e41cf25e30e5f1e4c182
\ No newline at end of file
diff --git a/distribution/licenses/lucene-queries-6.0.0.jar.sha1 b/distribution/licenses/lucene-queries-6.0.0.jar.sha1
deleted file mode 100644
index cb48f222957..00000000000
--- a/distribution/licenses/lucene-queries-6.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6d88cb7b8666ea809f8c894996b682ceb9581d07
\ No newline at end of file
diff --git a/distribution/licenses/lucene-queries-6.0.1.jar.sha1 b/distribution/licenses/lucene-queries-6.0.1.jar.sha1
new file mode 100644
index 00000000000..acaa53f1f8e
--- /dev/null
+++ b/distribution/licenses/lucene-queries-6.0.1.jar.sha1
@@ -0,0 +1 @@
+09b0e5862a676ff9e55a1bc6ca37ad578a25cb38
\ No newline at end of file
diff --git a/distribution/licenses/lucene-queryparser-6.0.0.jar.sha1 b/distribution/licenses/lucene-queryparser-6.0.0.jar.sha1
deleted file mode 100644
index 160c73d335d..00000000000
--- a/distribution/licenses/lucene-queryparser-6.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-89ac493f4bc038632abc1b4df9737bc6d433b052
\ No newline at end of file
diff --git a/distribution/licenses/lucene-queryparser-6.0.1.jar.sha1 b/distribution/licenses/lucene-queryparser-6.0.1.jar.sha1
new file mode 100644
index 00000000000..48c91d68f44
--- /dev/null
+++ b/distribution/licenses/lucene-queryparser-6.0.1.jar.sha1
@@ -0,0 +1 @@
+24f7ba0707aa01be2dd7749adff1659262be8f33
\ No newline at end of file
diff --git a/distribution/licenses/lucene-sandbox-6.0.0.jar.sha1 b/distribution/licenses/lucene-sandbox-6.0.0.jar.sha1
deleted file mode 100644
index de87353a1b4..00000000000
--- a/distribution/licenses/lucene-sandbox-6.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5ab7f57a1ab03aad36e15b75dbc391c73866d347
\ No newline at end of file
diff --git a/distribution/licenses/lucene-sandbox-6.0.1.jar.sha1 b/distribution/licenses/lucene-sandbox-6.0.1.jar.sha1
new file mode 100644
index 00000000000..ef843328aa0
--- /dev/null
+++ b/distribution/licenses/lucene-sandbox-6.0.1.jar.sha1
@@ -0,0 +1 @@
+0faf4c0d7e0adb6fccd830a2d5797d4176b579fe
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial-6.0.0.jar.sha1 b/distribution/licenses/lucene-spatial-6.0.0.jar.sha1
deleted file mode 100644
index 00e3d62dcb2..00000000000
--- a/distribution/licenses/lucene-spatial-6.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4e44de59bd0afb244bf7acd788e00c438e072ba1
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial-6.0.1.jar.sha1 b/distribution/licenses/lucene-spatial-6.0.1.jar.sha1
new file mode 100644
index 00000000000..25e7232ac14
--- /dev/null
+++ b/distribution/licenses/lucene-spatial-6.0.1.jar.sha1
@@ -0,0 +1 @@
+4d94d006251c904de3f1503c64746400877d6fa3
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial-extras-6.0.0.jar.sha1 b/distribution/licenses/lucene-spatial-extras-6.0.0.jar.sha1
deleted file mode 100644
index fbb30ddbfc1..00000000000
--- a/distribution/licenses/lucene-spatial-extras-6.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-886185979fc9e548fb125db80707f96b288fb8de
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial-extras-6.0.1.jar.sha1 b/distribution/licenses/lucene-spatial-extras-6.0.1.jar.sha1
new file mode 100644
index 00000000000..d421e1b053c
--- /dev/null
+++ b/distribution/licenses/lucene-spatial-extras-6.0.1.jar.sha1
@@ -0,0 +1 @@
+3de19dbdb889fe87791dae291ac3b340586854c4
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial3d-6.0.0.jar.sha1 b/distribution/licenses/lucene-spatial3d-6.0.0.jar.sha1
deleted file mode 100644
index e330e87543b..00000000000
--- a/distribution/licenses/lucene-spatial3d-6.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-898ce3070d5f833430e02dfef99bfbd67c38845d
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial3d-6.0.1.jar.sha1 b/distribution/licenses/lucene-spatial3d-6.0.1.jar.sha1
new file mode 100644
index 00000000000..348f501bb52
--- /dev/null
+++ b/distribution/licenses/lucene-spatial3d-6.0.1.jar.sha1
@@ -0,0 +1 @@
+5b1b7a754e83e2d58a819afa279b20b08b48c9c1
\ No newline at end of file
diff --git a/distribution/licenses/lucene-suggest-6.0.0.jar.sha1 b/distribution/licenses/lucene-suggest-6.0.0.jar.sha1
deleted file mode 100644
index d464f1ba02e..00000000000
--- a/distribution/licenses/lucene-suggest-6.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-46a915473878ab5379f41293fbec6da49ba706e1
\ No newline at end of file
diff --git a/distribution/licenses/lucene-suggest-6.0.1.jar.sha1 b/distribution/licenses/lucene-suggest-6.0.1.jar.sha1
new file mode 100644
index 00000000000..2cb6272d826
--- /dev/null
+++ b/distribution/licenses/lucene-suggest-6.0.1.jar.sha1
@@ -0,0 +1 @@
+55886bdaf16ecc6948e94b527837eaa1f16fe988
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-6.0.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-6.0.0.jar.sha1
deleted file mode 100644
index d145d6b8973..00000000000
--- a/modules/lang-expression/licenses/lucene-expressions-6.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-06cf4d232592ce7dcb8d8ae10dcde13904f6bfb2
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-6.0.1.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-6.0.1.jar.sha1
new file mode 100644
index 00000000000..7b3f5a1cef9
--- /dev/null
+++ b/modules/lang-expression/licenses/lucene-expressions-6.0.1.jar.sha1
@@ -0,0 +1 @@
+2b76056dbd40fb51dc5e8ef71e1919ad23e635a1
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.0.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.0.0.jar.sha1
deleted file mode 100644
index 46731426ef5..00000000000
--- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7835cfa15f52676eec0dcd2ca8cf4f53f8aefd5a
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.0.1.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.0.1.jar.sha1
new file mode 100644
index 00000000000..95dab25e74a
--- /dev/null
+++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.0.1.jar.sha1
@@ -0,0 +1 @@
+da08d9919f54efd2e09968d49fe05f6ce3f0c7ce
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.0.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.0.0.jar.sha1
deleted file mode 100644
index 4f2deed5514..00000000000
--- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d56f94ae9e67297db77c59d105d9ac0ba0501593
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.0.1.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.0.1.jar.sha1
new file mode 100644
index 00000000000..70f83bf52cc
--- /dev/null
+++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.0.1.jar.sha1
@@ -0,0 +1 @@
+77905f563c47994a764a6ab3d5ec198c174567a7
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.0.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.0.0.jar.sha1
deleted file mode 100644
index b6bf0ddd8f0..00000000000
--- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-848fbed30403e67d47d27b50eb28c31f70e969a2
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.0.1.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.0.1.jar.sha1
new file mode 100644
index 00000000000..8e2f7ab8b98
--- /dev/null
+++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.0.1.jar.sha1
@@ -0,0 +1 @@
+3ee5d909c269e5da7a92715f41ead88943b38123
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.0.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.0.0.jar.sha1
deleted file mode 100644
index b7c9262f37b..00000000000
--- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c0acd2a995e1c9adb5165554b3506828cf544986
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.0.1.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.0.1.jar.sha1
new file mode 100644
index 00000000000..981855d5a97
--- /dev/null
+++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.0.1.jar.sha1
@@ -0,0 +1 @@
+3b7bdbf9efa84f8d8875bd7f1d8734276930b9c3
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.0.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.0.0.jar.sha1
deleted file mode 100644
index 0bb34afc492..00000000000
--- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.0.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-71aa64f1a206da34a1e72607b8d6457ae2e127e8
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.0.1.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.0.1.jar.sha1
new file mode 100644
index 00000000000..4ff0afee687
--- /dev/null
+++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.0.1.jar.sha1
@@ -0,0 +1 @@
+e80e912621276e1009b72c06d5def188976c5426
\ No newline at end of file

From 3b8c9d87b4153a69d86e4829b566d94675a26a9f Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Wed, 1 Jun 2016 10:54:51 +0200
Subject: [PATCH 0548/1311] Fix StoreRecoveryTests after 6.0.1 upgrade.

---
 .../java/org/elasticsearch/index/shard/StoreRecoveryTests.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java b/core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java
index a46e0f7562d..ffb64f991cc 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java
@@ -74,7 +74,7 @@ public class StoreRecoveryTests extends ESTestCase {
         assertEquals(numFiles, targetNumFiles);
         assertEquals(indexStats.totalFileCount(), targetNumFiles);
         if (hardLinksSupported(createTempDir())) {
-            assertEquals("upgrade to HardlinkCopyDirectoryWrapper in Lucene 6.1", Version.LATEST, Version.LUCENE_6_0_0);
+            assertEquals("upgrade to HardlinkCopyDirectoryWrapper in Lucene 6.1", Version.LATEST, Version.LUCENE_6_0_1);
             // assertEquals(indexStats.reusedFileCount(), targetNumFiles); -- uncomment this once upgraded to Lucene 6.1
             assertEquals(indexStats.reusedFileCount(), 0);
         } else {

From f32f35bec4e103a6f5eb347895bbf73bec0fdf76 Mon Sep 17 00:00:00 2001
From: Alexander Kazakov <alexshadow007@gmail.com>
Date: Wed, 1 Jun 2016 14:27:11 +0300
Subject: [PATCH 0549/1311] Register "cloud.node.auto_attributes" setting
 (#18678)

---
 .../elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
index 7582e138c61..21e31b9f076 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
@@ -135,6 +135,9 @@ public class Ec2DiscoveryPlugin extends Plugin {
         settingsModule.registerSetting(AwsEc2Service.DISCOVERY_EC2.AVAILABILITY_ZONES_SETTING);
         settingsModule.registerSetting(AwsEc2Service.DISCOVERY_EC2.NODE_CACHE_TIME_SETTING);
         settingsModule.registerSetting(AwsEc2Service.DISCOVERY_EC2.TAG_SETTING);
+
+        // Register cloud node settings: cloud.node
+        settingsModule.registerSetting(AwsEc2Service.AUTO_ATTRIBUTE_SETTING);
     }
 
     /**

From bdd1d0703dff32309b78d5f288fd71ef196f0347 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Wed, 1 Jun 2016 15:22:55 +0200
Subject: [PATCH 0550/1311] Acknowledge index deletion requests based on
 standard cluster state acknowledgment (#18602)

Index deletion requests currently use a custom acknowledgement mechanism that wait for the data nodes to actually delete the data before acknowledging the request to the client. This was initially put into place as a new index with same name could only be created if the old index was wiped as we used the index name as data folder on the data nodes. With PR #16442, we now use the index uuid as folder name which avoids collision between indices that are named the same (deleted and recreated). This allows us to get rid of the custom acknowledgment mechanism altogether and rely on the standard cluster state-based acknowledgment instead.

Closes #18558
---
 .../DeleteIndexClusterStateUpdateRequest.java |  31 +++
 .../indices/delete/DeleteIndexRequest.java    |  33 +--
 .../delete/TransportDeleteIndexAction.java    |  13 +-
 .../support/master/AcknowledgedRequest.java   |   1 -
 .../elasticsearch/cluster/ClusterModule.java  |   2 -
 .../action/index/NodeIndexDeletedAction.java  | 199 ------------------
 .../cluster/metadata/IndexGraveyard.java      |   6 +-
 .../metadata/MetaDataDeleteIndexService.java  | 151 ++-----------
 .../cluster/IndicesClusterStateService.java   |  25 +--
 .../indices/IndicesServiceTests.java          |   1 -
 10 files changed, 63 insertions(+), 399 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexClusterStateUpdateRequest.java
 delete mode 100644 core/src/main/java/org/elasticsearch/cluster/action/index/NodeIndexDeletedAction.java

diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexClusterStateUpdateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexClusterStateUpdateRequest.java
new file mode 100644
index 00000000000..90139487ee8
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexClusterStateUpdateRequest.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.admin.indices.delete;
+
+import org.elasticsearch.cluster.ack.IndicesClusterStateUpdateRequest;
+
+/**
+ * Cluster state update request that allows to close one or more indices
+ */
+public class DeleteIndexClusterStateUpdateRequest extends IndicesClusterStateUpdateRequest<DeleteIndexClusterStateUpdateRequest> {
+
+    DeleteIndexClusterStateUpdateRequest() {
+
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java
index 7c957eaebfc..f1d7d38f6ac 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/delete/DeleteIndexRequest.java
@@ -23,26 +23,22 @@ import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
-import org.elasticsearch.action.support.master.MasterNodeRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.CollectionUtils;
 
 import java.io.IOException;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
-import static org.elasticsearch.common.unit.TimeValue.readTimeValue;
 
 /**
  * A request to delete an index. Best created with {@link org.elasticsearch.client.Requests#deleteIndexRequest(String)}.
  */
-public class DeleteIndexRequest extends MasterNodeRequest<DeleteIndexRequest> implements IndicesRequest.Replaceable {
+public class DeleteIndexRequest extends AcknowledgedRequest<DeleteIndexRequest> implements IndicesRequest.Replaceable {
 
     private String[] indices;
     // Delete index should work by default on both open and closed indices.
     private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true);
-    private TimeValue timeout = AcknowledgedRequest.DEFAULT_ACK_TIMEOUT;
 
     public DeleteIndexRequest() {
     }
@@ -98,37 +94,11 @@ public class DeleteIndexRequest extends MasterNodeRequest<DeleteIndexRequest> im
         return indices;
     }
 
-    /**
-     * Timeout to wait for the index deletion to be acknowledged by current cluster nodes. Defaults
-     * to <tt>10s</tt>.
-     */
-    public TimeValue timeout() {
-        return timeout;
-    }
-
-    /**
-     * Timeout to wait for the index deletion to be acknowledged by current cluster nodes. Defaults
-     * to <tt>10s</tt>.
-     */
-    public DeleteIndexRequest timeout(TimeValue timeout) {
-        this.timeout = timeout;
-        return this;
-    }
-
-    /**
-     * Timeout to wait for the index deletion to be acknowledged by current cluster nodes. Defaults
-     * to <tt>10s</tt>.
-     */
-    public DeleteIndexRequest timeout(String timeout) {
-        return timeout(TimeValue.parseTimeValue(timeout, null, getClass().getSimpleName() + ".timeout"));
-    }
-
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         indices = in.readStringArray();
         indicesOptions = IndicesOptions.readIndicesOptions(in);
-        timeout = readTimeValue(in);
     }
 
     @Override
@@ -136,6 +106,5 @@ public class DeleteIndexRequest extends MasterNodeRequest<DeleteIndexRequest> im
         super.writeTo(out);
         out.writeStringArray(indices);
         indicesOptions.writeIndicesOptions(out);
-        timeout.writeTo(out);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java
index 489001d9b89..0ae8444ea7c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java
@@ -24,6 +24,7 @@ import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.DestructiveOperations;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -85,15 +86,21 @@ public class TransportDeleteIndexAction extends TransportMasterNodeAction<Delete
             listener.onResponse(new DeleteIndexResponse(true));
             return;
         }
-        deleteIndexService.deleteIndices(new MetaDataDeleteIndexService.Request(concreteIndices).timeout(request.timeout()).masterTimeout(request.masterNodeTimeout()), new MetaDataDeleteIndexService.Listener() {
+
+        DeleteIndexClusterStateUpdateRequest deleteRequest = new DeleteIndexClusterStateUpdateRequest()
+            .ackTimeout(request.timeout()).masterNodeTimeout(request.masterNodeTimeout())
+            .indices(concreteIndices.toArray(new Index[concreteIndices.size()]));
+
+        deleteIndexService.deleteIndices(deleteRequest, new ActionListener<ClusterStateUpdateResponse>() {
 
             @Override
-            public void onResponse(MetaDataDeleteIndexService.Response response) {
-                listener.onResponse(new DeleteIndexResponse(response.acknowledged()));
+            public void onResponse(ClusterStateUpdateResponse response) {
+                listener.onResponse(new DeleteIndexResponse(response.isAcknowledged()));
             }
 
             @Override
             public void onFailure(Throwable t) {
+                logger.debug("failed to delete indices [{}]", t, concreteIndices);
                 listener.onFailure(t);
             }
         });
diff --git a/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java b/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java
index 5d45b7b11e6..9e45bccc547 100644
--- a/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java
@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.action.support.master;
 
-import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.cluster.ack.AckedRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index 15ed58346e7..b9084d52a91 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.cluster;
 
 import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
-import org.elasticsearch.cluster.action.index.NodeIndexDeletedAction;
 import org.elasticsearch.cluster.action.index.NodeMappingRefreshAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -155,7 +154,6 @@ public class ClusterModule extends AbstractModule {
         bind(RoutingService.class).asEagerSingleton();
         bind(DelayedAllocationService.class).asEagerSingleton();
         bind(ShardStateAction.class).asEagerSingleton();
-        bind(NodeIndexDeletedAction.class).asEagerSingleton();
         bind(NodeMappingRefreshAction.class).asEagerSingleton();
         bind(MappingUpdatedAction.class).asEagerSingleton();
         bind(TaskResultsService.class).asEagerSingleton();
diff --git a/core/src/main/java/org/elasticsearch/cluster/action/index/NodeIndexDeletedAction.java b/core/src/main/java/org/elasticsearch/cluster/action/index/NodeIndexDeletedAction.java
deleted file mode 100644
index 377addd48f3..00000000000
--- a/core/src/main/java/org/elasticsearch/cluster/action/index/NodeIndexDeletedAction.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cluster.action.index;
-
-import org.apache.lucene.store.LockObtainFailedException;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.EmptyTransportResponseHandler;
-import org.elasticsearch.transport.TransportChannel;
-import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.transport.TransportRequestHandler;
-import org.elasticsearch.transport.TransportResponse;
-import org.elasticsearch.transport.TransportService;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.TimeUnit;
-
-/**
- *
- */
-public class NodeIndexDeletedAction extends AbstractComponent {
-
-    public static final String INDEX_DELETED_ACTION_NAME = "internal:cluster/node/index/deleted";
-    public static final String INDEX_STORE_DELETED_ACTION_NAME = "internal:cluster/node/index_store/deleted";
-
-    private final ThreadPool threadPool;
-    private final TransportService transportService;
-    private final List<Listener> listeners = new CopyOnWriteArrayList<>();
-    private final IndicesService indicesService;
-
-    @Inject
-    public NodeIndexDeletedAction(Settings settings, ThreadPool threadPool, TransportService transportService, IndicesService indicesService) {
-        super(settings);
-        this.threadPool = threadPool;
-        this.transportService = transportService;
-        transportService.registerRequestHandler(INDEX_DELETED_ACTION_NAME, NodeIndexDeletedMessage::new, ThreadPool.Names.SAME, new NodeIndexDeletedTransportHandler());
-        transportService.registerRequestHandler(INDEX_STORE_DELETED_ACTION_NAME, NodeIndexStoreDeletedMessage::new, ThreadPool.Names.SAME, new NodeIndexStoreDeletedTransportHandler());
-        this.indicesService = indicesService;
-    }
-
-    public void add(Listener listener) {
-        listeners.add(listener);
-    }
-
-    public void remove(Listener listener) {
-        listeners.remove(listener);
-    }
-
-    public void nodeIndexDeleted(final ClusterState clusterState, final Index index, final IndexSettings indexSettings, final String nodeId) {
-        final DiscoveryNodes nodes = clusterState.nodes();
-        transportService.sendRequest(clusterState.nodes().getMasterNode(),
-                INDEX_DELETED_ACTION_NAME, new NodeIndexDeletedMessage(index, nodeId), EmptyTransportResponseHandler.INSTANCE_SAME);
-        if (nodes.getLocalNode().isDataNode() == false) {
-            logger.trace("[{}] not acking store deletion (not a data node)", index);
-            return;
-        }
-        threadPool.generic().execute(new AbstractRunnable() {
-            @Override
-            public void onFailure(Throwable t) {
-                logger.warn("[{}] failed to ack index store deleted for index", t, index);
-            }
-
-            @Override
-            protected void doRun() throws Exception {
-                lockIndexAndAck(index, nodes, nodeId, clusterState, indexSettings);
-            }
-        });
-    }
-
-    private void lockIndexAndAck(Index index, DiscoveryNodes nodes, String nodeId, ClusterState clusterState, IndexSettings indexSettings) throws IOException {
-        try {
-            // we are waiting until we can lock the index / all shards on the node and then we ack the delete of the store to the
-            // master. If we can't acquire the locks here immediately there might be a shard of this index still holding on to the lock
-            // due to a "currently canceled recovery" or so. The shard will delete itself BEFORE the lock is released so it's guaranteed to be
-            // deleted by the time we get the lock
-            indicesService.processPendingDeletes(indexSettings.getIndex(), indexSettings, new TimeValue(30, TimeUnit.MINUTES));
-            transportService.sendRequest(clusterState.nodes().getMasterNode(),
-                    INDEX_STORE_DELETED_ACTION_NAME, new NodeIndexStoreDeletedMessage(index, nodeId), EmptyTransportResponseHandler.INSTANCE_SAME);
-        } catch (LockObtainFailedException exc) {
-            logger.warn("[{}] failed to lock all shards for index - timed out after 30 seconds", index);
-        } catch (InterruptedException e) {
-            logger.warn("[{}] failed to lock all shards for index - interrupted", index);
-        }
-    }
-
-    public interface Listener {
-        void onNodeIndexDeleted(Index index, String nodeId);
-
-        void onNodeIndexStoreDeleted(Index index, String nodeId);
-    }
-
-    private class NodeIndexDeletedTransportHandler implements TransportRequestHandler<NodeIndexDeletedMessage> {
-
-        @Override
-        public void messageReceived(NodeIndexDeletedMessage message, TransportChannel channel) throws Exception {
-            for (Listener listener : listeners) {
-                listener.onNodeIndexDeleted(message.index, message.nodeId);
-            }
-            channel.sendResponse(TransportResponse.Empty.INSTANCE);
-        }
-    }
-
-    private class NodeIndexStoreDeletedTransportHandler implements TransportRequestHandler<NodeIndexStoreDeletedMessage> {
-
-        @Override
-        public void messageReceived(NodeIndexStoreDeletedMessage message, TransportChannel channel) throws Exception {
-            for (Listener listener : listeners) {
-                listener.onNodeIndexStoreDeleted(message.index, message.nodeId);
-            }
-            channel.sendResponse(TransportResponse.Empty.INSTANCE);
-        }
-    }
-
-    public static class NodeIndexDeletedMessage extends TransportRequest {
-
-        Index index;
-        String nodeId;
-
-        public NodeIndexDeletedMessage() {
-        }
-
-        NodeIndexDeletedMessage(Index index, String nodeId) {
-            this.index = index;
-            this.nodeId = nodeId;
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            index.writeTo(out);
-            out.writeString(nodeId);
-        }
-
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            index = new Index(in);
-            nodeId = in.readString();
-        }
-    }
-
-    public static class NodeIndexStoreDeletedMessage extends TransportRequest {
-
-        Index index;
-        String nodeId;
-
-        public NodeIndexStoreDeletedMessage() {
-        }
-
-        NodeIndexStoreDeletedMessage(Index index, String nodeId) {
-            this.index = index;
-            this.nodeId = nodeId;
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            super.writeTo(out);
-            index.writeTo(out);
-            out.writeString(nodeId);
-        }
-
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            super.readFrom(in);
-            index = new Index(in);
-            nodeId = in.readString();
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
index 7831862784e..6130ed2317a 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
@@ -219,8 +219,10 @@ final public class IndexGraveyard implements MetaData.Custom {
         /**
          * Add a set of deleted indexes to the list of tombstones in the cluster state.
          */
-        public Builder addTombstones(final Set<Index> indices) {
-            indices.stream().forEach(this::addTombstone);
+        public Builder addTombstones(final Index[] indices) {
+            for (Index index : indices) {
+                addTombstone(index);
+            }
             return this;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java
index 9f373fcda97..7e97d792430 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataDeleteIndexService.java
@@ -19,10 +19,11 @@
 
 package org.elasticsearch.cluster.metadata;
 
-import org.elasticsearch.action.support.master.MasterNodeRequest;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexClusterStateUpdateRequest;
+import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
-import org.elasticsearch.cluster.action.index.NodeIndexDeletedAction;
+import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
@@ -32,16 +33,11 @@ import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.snapshots.SnapshotsService;
-import org.elasticsearch.threadpool.ThreadPool;
 
+import java.util.Arrays;
 import java.util.Set;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
 /**
@@ -49,46 +45,37 @@ import java.util.stream.Collectors;
  */
 public class MetaDataDeleteIndexService extends AbstractComponent {
 
-    private final ThreadPool threadPool;
-
     private final ClusterService clusterService;
 
     private final AllocationService allocationService;
 
-    private final NodeIndexDeletedAction nodeIndexDeletedAction;
-
     @Inject
-    public MetaDataDeleteIndexService(Settings settings, ThreadPool threadPool, ClusterService clusterService, AllocationService allocationService,
-                                      NodeIndexDeletedAction nodeIndexDeletedAction) {
+    public MetaDataDeleteIndexService(Settings settings, ClusterService clusterService, AllocationService allocationService) {
         super(settings);
-        this.threadPool = threadPool;
         this.clusterService = clusterService;
         this.allocationService = allocationService;
-        this.nodeIndexDeletedAction = nodeIndexDeletedAction;
     }
 
-    public void deleteIndices(final Request request, final Listener userListener) {
-        final DeleteIndexListener listener = new DeleteIndexListener(userListener);
+    public void deleteIndices(final DeleteIndexClusterStateUpdateRequest request, final ActionListener<ClusterStateUpdateResponse> listener) {
+        if (request.indices() == null || request.indices().length == 0) {
+            throw new IllegalArgumentException("Index name is required");
+        }
 
-        clusterService.submitStateUpdateTask("delete-index " + request.indices, new ClusterStateUpdateTask(Priority.URGENT) {
+        clusterService.submitStateUpdateTask("delete-index " + request.indices(),
+            new AckedClusterStateUpdateTask<ClusterStateUpdateResponse>(Priority.URGENT, request, listener) {
 
             @Override
-            public TimeValue timeout() {
-                return request.masterTimeout;
-            }
-
-            @Override
-            public void onFailure(String source, Throwable t) {
-                listener.onFailure(t);
+            protected ClusterStateUpdateResponse newResponse(boolean acknowledged) {
+                return new ClusterStateUpdateResponse(acknowledged);
             }
 
             @Override
             public ClusterState execute(final ClusterState currentState) {
                 final MetaData meta = currentState.metaData();
-                final Set<IndexMetaData> metaDatas = request.indices.stream().map(i -> meta.getIndexSafe(i)).collect(Collectors.toSet());
+                final Index[] indices = request.indices();
+                final Set<IndexMetaData> metaDatas = Arrays.asList(indices).stream().map(i -> meta.getIndexSafe(i)).collect(Collectors.toSet());
                 // Check if index deletion conflicts with any running snapshots
                 SnapshotsService.checkIndexDeletion(currentState, metaDatas);
-                final Set<Index> indices = request.indices;
                 RoutingTable.Builder routingTableBuilder = RoutingTable.builder(currentState.routingTable());
                 MetaData.Builder metaDataBuilder = MetaData.builder(meta);
                 ClusterBlocks.Builder clusterBlocksBuilder = ClusterBlocks.builder().blocks(currentState.blocks());
@@ -108,40 +95,6 @@ public class MetaDataDeleteIndexService extends AbstractComponent {
                 logger.trace("{} tombstones purged from the cluster state. Previous tombstone size: {}. Current tombstone size: {}.",
                     graveyardBuilder.getNumPurged(), previousGraveyardSize, currentGraveyard.getTombstones().size());
 
-                // wait for events from all nodes that it has been removed from their respective metadata...
-                int count = currentState.nodes().getSize();
-                // add the notifications that the store was deleted from *data* nodes
-                count += currentState.nodes().getDataNodes().size();
-                final AtomicInteger counter = new AtomicInteger(count * indices.size());
-
-                // this listener will be notified once we get back a notification based on the cluster state change below.
-                final NodeIndexDeletedAction.Listener nodeIndexDeleteListener = new NodeIndexDeletedAction.Listener() {
-                    @Override
-                    public void onNodeIndexDeleted(Index deleted, String nodeId) {
-                        if (indices.contains(deleted)) {
-                            if (counter.decrementAndGet() == 0) {
-                                listener.onResponse(new Response(true));
-                                nodeIndexDeletedAction.remove(this);
-                            }
-                        }
-                    }
-
-                    @Override
-                    public void onNodeIndexStoreDeleted(Index deleted, String nodeId) {
-                        if (indices.contains(deleted)) {
-                            if (counter.decrementAndGet() == 0) {
-                                listener.onResponse(new Response(true));
-                                nodeIndexDeletedAction.remove(this);
-                            }
-                        }
-                    }
-                };
-                nodeIndexDeletedAction.add(nodeIndexDeleteListener);
-                listener.future = threadPool.schedule(request.timeout, ThreadPool.Names.SAME, () -> {
-                    listener.onResponse(new Response(false));
-                    nodeIndexDeletedAction.remove(nodeIndexDeleteListener);
-                });
-
                 MetaData newMetaData = metaDataBuilder.build();
                 ClusterBlocks blocks = clusterBlocksBuilder.build();
                 RoutingAllocation.Result routingResult = allocationService.reroute(
@@ -149,78 +102,6 @@ public class MetaDataDeleteIndexService extends AbstractComponent {
                         "deleted indices [" + indices + "]");
                 return ClusterState.builder(currentState).routingResult(routingResult).metaData(newMetaData).blocks(blocks).build();
             }
-
-            @Override
-            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-            }
         });
     }
-
-    class DeleteIndexListener implements Listener {
-
-        private final AtomicBoolean notified = new AtomicBoolean();
-        private final Listener listener;
-        volatile ScheduledFuture<?> future;
-
-        private DeleteIndexListener(Listener listener) {
-            this.listener = listener;
-        }
-
-        @Override
-        public void onResponse(final Response response) {
-            if (notified.compareAndSet(false, true)) {
-                FutureUtils.cancel(future);
-                listener.onResponse(response);
-            }
-        }
-
-        @Override
-        public void onFailure(Throwable t) {
-            if (notified.compareAndSet(false, true)) {
-                FutureUtils.cancel(future);
-                listener.onFailure(t);
-            }
-        }
-    }
-
-    public interface Listener {
-
-        void onResponse(Response response);
-
-        void onFailure(Throwable t);
-    }
-
-    public static class Request {
-
-        final Set<Index> indices;
-
-        TimeValue timeout = TimeValue.timeValueSeconds(10);
-        TimeValue masterTimeout = MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT;
-
-        public Request(Set<Index> indices) {
-            this.indices = indices;
-        }
-
-        public Request timeout(TimeValue timeout) {
-            this.timeout = timeout;
-            return this;
-        }
-
-        public Request masterTimeout(TimeValue masterTimeout) {
-            this.masterTimeout = masterTimeout;
-            return this;
-        }
-    }
-
-    public static class Response {
-        private final boolean acknowledged;
-
-        public Response(boolean acknowledged) {
-            this.acknowledged = acknowledged;
-        }
-
-        public boolean acknowledged() {
-            return acknowledged;
-        }
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
index 23518d4b3c3..dcf055275e5 100644
--- a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
+++ b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
@@ -24,14 +24,12 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
-import org.elasticsearch.cluster.action.index.NodeIndexDeletedAction;
 import org.elasticsearch.cluster.action.index.NodeMappingRefreshAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingNode;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -48,7 +46,6 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexShardAlreadyExistsException;
 import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.mapper.DocumentMapper;
@@ -89,7 +86,6 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
     private final ThreadPool threadPool;
     private final RecoveryTargetService recoveryTargetService;
     private final ShardStateAction shardStateAction;
-    private final NodeIndexDeletedAction nodeIndexDeletedAction;
     private final NodeMappingRefreshAction nodeMappingRefreshAction;
     private final NodeServicesProvider nodeServicesProvider;
 
@@ -112,7 +108,6 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
     public IndicesClusterStateService(Settings settings, IndicesService indicesService, ClusterService clusterService,
                                       ThreadPool threadPool, RecoveryTargetService recoveryTargetService,
                                       ShardStateAction shardStateAction,
-                                      NodeIndexDeletedAction nodeIndexDeletedAction,
                                       NodeMappingRefreshAction nodeMappingRefreshAction,
                                       RepositoriesService repositoriesService, RestoreService restoreService,
                                       SearchService searchService, SyncedFlushService syncedFlushService,
@@ -124,7 +119,6 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
         this.threadPool = threadPool;
         this.recoveryTargetService = recoveryTargetService;
         this.shardStateAction = shardStateAction;
-        this.nodeIndexDeletedAction = nodeIndexDeletedAction;
         this.nodeMappingRefreshAction = nodeMappingRefreshAction;
         this.restoreService = restoreService;
         this.repositoriesService = repositoriesService;
@@ -219,14 +213,11 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
                 logger.debug("[{}] cleaning index, no longer part of the metadata", index);
             }
             final IndexService idxService = indicesService.indexService(index);
-            final IndexSettings indexSettings;
             if (idxService != null) {
-                indexSettings = idxService.getIndexSettings();
                 deleteIndex(index, "index no longer part of the metadata");
             } else if (previousState.metaData().hasIndex(index.getName())) {
                 // The deleted index was part of the previous cluster state, but not loaded on the local node
                 final IndexMetaData metaData = previousState.metaData().index(index);
-                indexSettings = new IndexSettings(metaData, settings);
                 indicesService.deleteUnassignedIndex("deleted index was not assigned to local node", metaData, event.state());
             } else {
                 // The previous cluster state's metadata also does not contain the index,
@@ -236,21 +227,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
                 // First, though, verify the precondition for applying this case by
                 // asserting that the previous cluster state is not initialized/recovered.
                 assert previousState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK);
-                final IndexMetaData metaData = indicesService.verifyIndexIsDeleted(index, event.state());
-                if (metaData != null) {
-                    indexSettings = new IndexSettings(metaData, settings);
-                } else {
-                    indexSettings = null;
-                }
-            }
-            // indexSettings can only be null if there was no IndexService and no metadata existed
-            // on disk for this index, so it won't need to go through the node deleted action anyway
-            if (indexSettings != null) {
-                try {
-                    nodeIndexDeletedAction.nodeIndexDeleted(event.state(), index, indexSettings, localNodeId);
-                } catch (Exception e) {
-                    logger.debug("failed to send to master index {} deleted event", e, index);
-                }
+                indicesService.verifyIndexIsDeleted(index, event.state());
             }
         }
 
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
index 2abf07164bf..76f7a30e078 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
@@ -105,7 +105,6 @@ public class IndicesServiceTests extends ESSingleNodeTestCase {
             indicesService.canDeleteShardContent(notAllocated, test.getIndexSettings()));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/18558")
     public void testDeleteIndexStore() throws Exception {
         IndicesService indicesService = getIndicesService();
         IndexService test = createIndex("test");

From c20bf5d7475268a61a82e1b83cd1d1c314c30883 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Wed, 1 Jun 2016 17:02:09 +0200
Subject: [PATCH 0551/1311] [TEST] Fix tests that rely on assumption that data
 dirs are removed after index deletion (#18681)

Relates to #18602
---
 .../index/IndexWithShadowReplicasIT.java              |  6 +++---
 .../elasticsearch/index/shard/IndexShardTests.java    |  6 +++---
 .../main/java/org/elasticsearch/test/ESTestCase.java  | 11 +++++++----
 3 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
index 4eb674397be..a280d4e687c 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
@@ -586,7 +586,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
         logger.info("--> deleting index " + IDX);
         assertAcked(client().admin().indices().prepareDelete(IDX));
 
-        assertPathHasBeenCleared(dataPath);
+        assertBusyPathHasBeenCleared(dataPath);
         //norelease
         //TODO: uncomment the test below when https://github.com/elastic/elasticsearch/issues/17695 is resolved.
         //assertIndicesDirsDeleted(nodes);
@@ -647,7 +647,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
 
         assertAcked(client().admin().indices().prepareDelete(IDX));
 
-        assertPathHasBeenCleared(dataPath);
+        assertBusyPathHasBeenCleared(dataPath);
         //norelease
         //TODO: uncomment the test below when https://github.com/elastic/elasticsearch/issues/17695 is resolved.
         //assertIndicesDirsDeleted(nodes);
@@ -839,7 +839,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
         logger.info("--> deleting closed index");
         client().admin().indices().prepareDelete(IDX).get();
 
-        assertPathHasBeenCleared(dataPath);
+        assertBusyPathHasBeenCleared(dataPath);
         assertIndicesDirsDeleted(nodes);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index ccc1f0a6b36..52aa07b3eb3 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -556,7 +556,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         SearchResponse response = client().prepareSearch("test").get();
         assertHitCount(response, 1L);
         client().admin().indices().prepareDelete("test").get();
-        assertPathHasBeenCleared(idxPath);
+        assertBusyPathHasBeenCleared(idxPath);
     }
 
     public void testExpectedShardSizeIsPresent() throws InterruptedException {
@@ -639,8 +639,8 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         assertThat("found the hit", resp.getHits().getTotalHits(), equalTo(1L));
 
         assertAcked(client().admin().indices().prepareDelete(INDEX));
-        assertPathHasBeenCleared(startDir.toAbsolutePath().toString());
-        assertPathHasBeenCleared(endDir.toAbsolutePath().toString());
+        assertBusyPathHasBeenCleared(startDir.toAbsolutePath());
+        assertBusyPathHasBeenCleared(endDir.toAbsolutePath());
     }
 
     public void testShardStats() throws IOException {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 22216efa15b..972c1a50fbc 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -70,6 +70,7 @@ import org.junit.Rule;
 import org.junit.rules.RuleChain;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -672,16 +673,16 @@ public abstract class ESTestCase extends LuceneTestCase {
     }
 
     /**
-     * Asserts that there are no files in the specified path
+     * Asserts busily that there are no files in the specified path
      */
-    public void assertPathHasBeenCleared(String path) throws Exception {
-        assertPathHasBeenCleared(PathUtils.get(path));
+    public void assertBusyPathHasBeenCleared(Path path) throws Exception {
+        assertBusy(() -> assertPathHasBeenCleared(path));
     }
 
     /**
      * Asserts that there are no files in the specified path
      */
-    public void assertPathHasBeenCleared(Path path) throws Exception {
+    public void assertPathHasBeenCleared(Path path) {
         logger.info("--> checking that [{}] has been cleared", path);
         int count = 0;
         StringBuilder sb = new StringBuilder();
@@ -702,6 +703,8 @@ public abstract class ESTestCase extends LuceneTestCase {
                         sb.append("\n");
                     }
                 }
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
             }
         }
         sb.append("]");

From 21f0d7da1d62f92550c69498220ebdee87eec623 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Wed, 1 Jun 2016 18:47:25 +0200
Subject: [PATCH 0552/1311] [TEST] Increase timeout to wait on folder deletion
 in IndexWithShadowReplicasIT

---
 .../org/elasticsearch/index/IndexWithShadowReplicasIT.java | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
index a280d4e687c..815b356fcd2 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
@@ -70,6 +70,7 @@ import java.util.List;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -586,7 +587,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
         logger.info("--> deleting index " + IDX);
         assertAcked(client().admin().indices().prepareDelete(IDX));
 
-        assertBusyPathHasBeenCleared(dataPath);
+        assertBusy(() -> assertPathHasBeenCleared(dataPath), 1, TimeUnit.MINUTES);
         //norelease
         //TODO: uncomment the test below when https://github.com/elastic/elasticsearch/issues/17695 is resolved.
         //assertIndicesDirsDeleted(nodes);
@@ -647,7 +648,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
 
         assertAcked(client().admin().indices().prepareDelete(IDX));
 
-        assertBusyPathHasBeenCleared(dataPath);
+        assertBusy(() -> assertPathHasBeenCleared(dataPath), 1, TimeUnit.MINUTES);
         //norelease
         //TODO: uncomment the test below when https://github.com/elastic/elasticsearch/issues/17695 is resolved.
         //assertIndicesDirsDeleted(nodes);
@@ -839,7 +840,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
         logger.info("--> deleting closed index");
         client().admin().indices().prepareDelete(IDX).get();
 
-        assertBusyPathHasBeenCleared(dataPath);
+        assertBusy(() -> assertPathHasBeenCleared(dataPath), 1, TimeUnit.MINUTES);
         assertIndicesDirsDeleted(nodes);
     }
 

From 1b66d4a97f46c5ead617fcaf9d2e145019ee5746 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 1 Jun 2016 12:53:59 -0400
Subject: [PATCH 0553/1311] Add more logging to reindex rethrottle

The tests are failing in CI and we can't track down the cause. This should
help!
---
 .../index/reindex/BulkByScrollTask.java       | 20 +++++++++++++++++++
 .../index/reindex/RethrottleTests.java        |  2 +-
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
index f062287a5de..40ee492b4a9 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
@@ -22,6 +22,8 @@ package org.elasticsearch.index.reindex;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
@@ -47,6 +49,8 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
  * Task storing information about a currently running BulkByScroll request.
  */
 public class BulkByScrollTask extends CancellableTask {
+    private static final ESLogger logger = ESLoggerFactory.getLogger(BulkByScrollTask.class.getName());
+
     /**
      * The total number of documents this request will process. 0 means we don't yet know or, possibly, there are actually 0 documents
      * to process. Its ok that these have the same meaning because any request with 0 actual documents should be quite short lived.
@@ -436,10 +440,16 @@ public class BulkByScrollTask extends CancellableTask {
 
     void rethrottle(float newRequestsPerSecond) {
         synchronized (delayedPrepareBulkRequestReference) {
+            if (logger.isDebugEnabled()) {
+                logger.debug("[{}]: Rethrottling to [{}] requests per second", getId(), newRequestsPerSecond);
+            }
             setRequestsPerSecond(newRequestsPerSecond);
 
             DelayedPrepareBulkRequest delayedPrepareBulkRequest = this.delayedPrepareBulkRequestReference.get();
             if (delayedPrepareBulkRequest == null) {
+                if (logger.isDebugEnabled()) {
+                    logger.debug("[{}]: Skipping rescheduling because there is no scheduled task", getId());
+                }
                 // No request has been queued yet so nothing to reschedule.
                 return;
             }
@@ -478,6 +488,10 @@ public class BulkByScrollTask extends CancellableTask {
                  * The user is attempting to slow the request down. We'll let the change in throttle take effect the next time we delay
                  * prepareBulkRequest. We can't just reschedule the request further out in the future the bulk context might time out.
                  */
+                if (logger.isDebugEnabled()) {
+                    logger.debug("[{}]: Skipping rescheduling because the new throttle [{}] is slower than the old one [{}].", getId(),
+                            newRequestsPerSecond, requestsPerSecond);
+                }
                 return this;
             }
 
@@ -485,6 +499,9 @@ public class BulkByScrollTask extends CancellableTask {
             // Actually reschedule the task
             if (false == FutureUtils.cancel(future)) {
                 // Couldn't cancel, probably because the task has finished or been scheduled. Either way we have nothing to do here.
+                if (logger.isDebugEnabled()) {
+                    logger.debug("[{}]: Skipping rescheduling we couldn't cancel the task.", getId());
+                }
                 return this;
             }
 
@@ -493,6 +510,9 @@ public class BulkByScrollTask extends CancellableTask {
              * test it you'll find that requests sneak through. So each request is given a runOnce boolean to prevent that.
              */
             TimeValue newDelay = newDelay(remainingDelay, newRequestsPerSecond);
+            if (logger.isDebugEnabled()) {
+                logger.debug("[{}]: Rescheduling for [{}] in the future.", getId(), newDelay);
+            }
             return new DelayedPrepareBulkRequest(threadPool, requestsPerSecond, newDelay, command);
         }
 
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
index dfa17ae4778..98575a457cf 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
@@ -53,7 +53,7 @@ public class RethrottleTests extends ReindexTestCase {
                 client().prepareIndex("test", "test", "3").setSource("foo", "bar"));
 
         // Start a request that will never finish unless we rethrottle it
-        request.setRequestsPerSecond(.000001f);  // Throttle forever
+        request.setRequestsPerSecond(.000001f);  // Throttle "forever"
         request.source().setSize(1);             // Make sure we use multiple batches
         ListenableActionFuture<? extends BulkIndexByScrollResponse> responseListener = request.execute();
 

From a98856663b560bd81ae3b1d72319c9329ed64f44 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Wed, 1 Jun 2016 20:16:12 +0200
Subject: [PATCH 0554/1311] Update reindex.asciidoc (#18687)

Potentially fixing some copy/paste errors
# Conflicts:
#	docs/reference/docs/reindex.asciidoc
---
 docs/reference/docs/reindex.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index bde6ad8b611..0bfb44b2aaf 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -237,7 +237,7 @@ POST _reindex
     "version_type": "external"
   },
   "script": {
-    "script": "if (ctx._source.foo == 'bar') {ctx._version++; ctx._source.remove('foo')}"
+    "inline": "if (ctx._source.foo == 'bar') {ctx._version++; ctx._source.remove('foo')}"
   }
 }
 --------------------------------------------------

From 8e2a7d0fe1cf54478cd70b733c80f82ee651232c Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 1 Jun 2016 16:25:51 -0400
Subject: [PATCH 0555/1311] Rename boostrap.mlockall to bootstrap.memory_lock

The setting bootstrap.mlockall is useful on both POSIX-like systems
(POSIX mlockall) and Windows (Win32 VirtualLock). But mlockall is really
a POSIX only thing so the name should not be tied POSIX. This commit
renames the setting to "bootstrap.memory_lock".

Relates #18669
---
 .../main/java/org/elasticsearch/bootstrap/Bootstrap.java  | 3 +--
 .../java/org/elasticsearch/bootstrap/BootstrapCheck.java  | 2 +-
 .../org/elasticsearch/bootstrap/BootstrapSettings.java    | 4 ++--
 .../elasticsearch/common/settings/ClusterSettings.java    | 3 +--
 .../elasticsearch/bootstrap/BootstrapSettingsTests.java   | 2 +-
 distribution/src/main/packaging/env/elasticsearch         | 2 +-
 .../src/main/packaging/systemd/elasticsearch.service      | 2 +-
 distribution/src/main/resources/config/elasticsearch.yml  | 2 +-
 docs/reference/migration/migrate_5_0/settings.asciidoc    | 5 +++++
 docs/reference/setup/bootstrap-checks.asciidoc            | 6 +++---
 docs/reference/setup/important-settings.asciidoc          | 8 ++++----
 docs/reference/setup/install/sysconfig-file.asciidoc      | 2 +-
 docs/reference/setup/sysconfig/swap.asciidoc              | 4 ++--
 13 files changed, 24 insertions(+), 21 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
index 305a4fd30ae..2688e8b012e 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
@@ -45,7 +45,6 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
 import java.nio.file.Path;
-import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 
@@ -151,7 +150,7 @@ final class Bootstrap {
     private void setup(boolean addShutdownHook, Settings settings, Environment environment) throws Exception {
         initializeNatives(
                 environment.tmpFile(),
-                BootstrapSettings.MLOCKALL_SETTING.get(settings),
+                BootstrapSettings.MEMORY_LOCK_SETTING.get(settings),
                 BootstrapSettings.SECCOMP_SETTING.get(settings),
                 BootstrapSettings.CTRLHANDLER_SETTING.get(settings));
 
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
index 0ee5099dc69..51ddb752b44 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
@@ -152,7 +152,7 @@ final class BootstrapCheck {
         final FileDescriptorCheck fileDescriptorCheck
             = Constants.MAC_OS_X ? new OsXFileDescriptorCheck() : new FileDescriptorCheck();
         checks.add(fileDescriptorCheck);
-        checks.add(new MlockallCheck(BootstrapSettings.MLOCKALL_SETTING.get(settings)));
+        checks.add(new MlockallCheck(BootstrapSettings.MEMORY_LOCK_SETTING.get(settings)));
         if (Constants.LINUX) {
             checks.add(new MaxNumberOfThreadsCheck());
         }
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java
index b2059380181..ad37916881b 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapSettings.java
@@ -31,8 +31,8 @@ public final class BootstrapSettings {
     public static final Setting<Boolean> SECURITY_FILTER_BAD_DEFAULTS_SETTING =
             Setting.boolSetting("security.manager.filter_bad_defaults", true, Property.NodeScope);
 
-    public static final Setting<Boolean> MLOCKALL_SETTING =
-        Setting.boolSetting("bootstrap.mlockall", false, Property.NodeScope);
+    public static final Setting<Boolean> MEMORY_LOCK_SETTING =
+        Setting.boolSetting("bootstrap.memory_lock", false, Property.NodeScope);
     public static final Setting<Boolean> SECCOMP_SETTING =
         Setting.boolSetting("bootstrap.seccomp", true, Property.NodeScope);
     public static final Setting<Boolean> CTRLHANDLER_SETTING =
diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index 3510b952660..c9abafbbe54 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -81,7 +81,6 @@ import org.elasticsearch.monitor.jvm.JvmService;
 import org.elasticsearch.monitor.os.OsService;
 import org.elasticsearch.monitor.process.ProcessService;
 import org.elasticsearch.node.Node;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.repositories.uri.URLRepository;
@@ -407,7 +406,7 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     PageCacheRecycler.TYPE_SETTING,
                     PluginsService.MANDATORY_SETTING,
                     BootstrapSettings.SECURITY_FILTER_BAD_DEFAULTS_SETTING,
-                    BootstrapSettings.MLOCKALL_SETTING,
+                    BootstrapSettings.MEMORY_LOCK_SETTING,
                     BootstrapSettings.SECCOMP_SETTING,
                     BootstrapSettings.CTRLHANDLER_SETTING,
                     BootstrapSettings.IGNORE_SYSTEM_BOOTSTRAP_CHECKS,
diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapSettingsTests.java b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapSettingsTests.java
index c032d3ddee8..128c82e9533 100644
--- a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapSettingsTests.java
@@ -26,7 +26,7 @@ public class BootstrapSettingsTests extends ESTestCase {
 
     public void testDefaultSettings() {
         assertTrue(BootstrapSettings.SECURITY_FILTER_BAD_DEFAULTS_SETTING.get(Settings.EMPTY));
-        assertFalse(BootstrapSettings.MLOCKALL_SETTING.get(Settings.EMPTY));
+        assertFalse(BootstrapSettings.MEMORY_LOCK_SETTING.get(Settings.EMPTY));
         assertTrue(BootstrapSettings.SECCOMP_SETTING.get(Settings.EMPTY));
         assertTrue(BootstrapSettings.CTRLHANDLER_SETTING.get(Settings.EMPTY));
     }
diff --git a/distribution/src/main/packaging/env/elasticsearch b/distribution/src/main/packaging/env/elasticsearch
index 9b7f2fd2443..34eff906661 100644
--- a/distribution/src/main/packaging/env/elasticsearch
+++ b/distribution/src/main/packaging/env/elasticsearch
@@ -53,7 +53,7 @@ ES_STARTUP_SLEEP_TIME=5
 #MAX_OPEN_FILES=65536
 
 # The maximum number of bytes of memory that may be locked into RAM
-# Set to "unlimited" if you use the 'bootstrap.mlockall: true' option
+# Set to "unlimited" if you use the 'bootstrap.memory_lock: true' option
 # in elasticsearch.yml.
 # When using Systemd, the LimitMEMLOCK property must be set
 # in /usr/lib/systemd/system/elasticsearch.service
diff --git a/distribution/src/main/packaging/systemd/elasticsearch.service b/distribution/src/main/packaging/systemd/elasticsearch.service
index 0c99464c4f6..8971bc2bc19 100644
--- a/distribution/src/main/packaging/systemd/elasticsearch.service
+++ b/distribution/src/main/packaging/systemd/elasticsearch.service
@@ -32,7 +32,7 @@ StandardError=inherit
 LimitNOFILE=65536
 
 # Specifies the maximum number of bytes of memory that may be locked into RAM
-# Set to "infinity" if you use the 'bootstrap.mlockall: true' option
+# Set to "infinity" if you use the 'bootstrap.memory_lock: true' option
 # in elasticsearch.yml and 'MAX_LOCKED_MEMORY=unlimited' in ${path.env}
 #LimitMEMLOCK=infinity
 
diff --git a/distribution/src/main/resources/config/elasticsearch.yml b/distribution/src/main/resources/config/elasticsearch.yml
index fe103052218..27e038ab3b2 100644
--- a/distribution/src/main/resources/config/elasticsearch.yml
+++ b/distribution/src/main/resources/config/elasticsearch.yml
@@ -40,7 +40,7 @@
 #
 # Lock the memory on startup:
 #
-# bootstrap.mlockall: true
+# bootstrap.memory_lock: true
 #
 # Make sure that the heap size is set to about half the memory available
 # on the system and that the owner of the process is allowed to use this
diff --git a/docs/reference/migration/migrate_5_0/settings.asciidoc b/docs/reference/migration/migrate_5_0/settings.asciidoc
index 0fa7d42e874..5edd4e449e8 100644
--- a/docs/reference/migration/migrate_5_0/settings.asciidoc
+++ b/docs/reference/migration/migrate_5_0/settings.asciidoc
@@ -259,3 +259,8 @@ set `script.line: true` or `script.stored: true`.
 The setting `index.query.bool.max_clause_count` has been removed. In order to
 set the maximum number of boolean clauses `indices.query.bool.max_clause_count`
 should be used instead.
+
+==== Memory lock settings
+
+The setting `bootstrap.mlockall` has been renamed to
+`bootstrap.memory_lock`.
diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc
index fea711a52f0..2a5c55bf8b3 100644
--- a/docs/reference/setup/bootstrap-checks.asciidoc
+++ b/docs/reference/setup/bootstrap-checks.asciidoc
@@ -34,7 +34,7 @@ If a JVM is started with unequal initial and max heap size, it can be
 prone to pauses as the JVM heap is resized during system usage. To avoid
 these resize pauses, it's best to start the JVM with the initial heap
 size equal to the maximum heap size. Additionally, if
-<<bootstrap.mlockall,`bootstrap.mlockall`>> is enabled, the JVM will
+<<bootstrap.memory_lock,`bootstrap.memory_lock`>> is enabled, the JVM will
 lock the initial size of the heap on startup. If the initial heap size
 is not equal to the maximum heap size, after a resize it will not be the
 case that all of the JVM heap is locked in memory. To pass the heap size
@@ -62,11 +62,11 @@ Elasticsearch would much rather use to service requests. There are
 several ways to configure a system to disallow swapping. One way is by
 requesting the JVM to lock the heap in memory through `mlockall` (Unix)
 or virtual lock (Windows). This is done via the Elasticsearch setting
-<<bootstrap.mlockall,`bootstrap.mlockall`>>. However, there are cases
+<<bootstrap.memory_lock,`bootstrap.memory_lock`>>. However, there are cases
 where this setting can be passed to Elasticsearch but Elasticsearch is
 not able to lock the heap (e.g., if the `elasticsearch` user does not
 have `memlock unlimited`). The memory lock check verifies that *if* the
-`bootstrap.mlockall` setting is enabled, that the JVM was successfully
+`bootstrap.memory_lock` setting is enabled, that the JVM was successfully
 able to lock the heap. To pass the memory lock check, you might have to
 configure <<mlockall,`mlockall`>>.
 
diff --git a/docs/reference/setup/important-settings.asciidoc b/docs/reference/setup/important-settings.asciidoc
index 867bd208d56..e3be0b780c4 100644
--- a/docs/reference/setup/important-settings.asciidoc
+++ b/docs/reference/setup/important-settings.asciidoc
@@ -8,7 +8,7 @@ configured before going into production.
 * <<path-settings,`path.data` and `path.logs`>>
 * <<cluster.name,`cluster.name`>>
 * <<node.name,`node.name`>>
-* <<bootstrap.mlockall,`bootstrap.mlockall`>>
+* <<bootstrap.memory_lock,`bootstrap.memory_lock`>>
 * <<network.host,`network.host`>>
 * <<unicast.hosts,`discovery.zen.ping.unicast.hosts`>>
 * <<minimum_master_nodes,`discovery.zen.minimum_master_nodes`>>
@@ -90,12 +90,12 @@ node.name: ${HOSTNAME}
 --------------------------------------------------
 
 [float]
-[[bootstrap.mlockall]]
-=== `bootstrap.mlockall`
+[[bootstrap.memory_lock]]
+=== `bootstrap.memory_lock`
 
 It is vitally important to the health of your node that none of the JVM is
 ever swapped out to disk.  One way of achieving that is set the
-`bootstrap.mlockall` setting to `true`.
+`bootstrap.memory_lock` setting to `true`.
 
 For this setting to have effect, other system settings need to be configured
 first. See <<mlockall>> for more details about how to set up memory locking
diff --git a/docs/reference/setup/install/sysconfig-file.asciidoc b/docs/reference/setup/install/sysconfig-file.asciidoc
index 55aa0532b93..da3f0910964 100644
--- a/docs/reference/setup/install/sysconfig-file.asciidoc
+++ b/docs/reference/setup/install/sysconfig-file.asciidoc
@@ -18,7 +18,7 @@
 `MAX_LOCKED_MEMORY`::
 
     Maximum locked memory size. Set to `unlimited if you use the
-    `bootstrap.mlockall` option in elasticsearch.yml.
+    `bootstrap.memory_lock` option in elasticsearch.yml.
 
 `MAX_MAP_COUNT`::
 
diff --git a/docs/reference/setup/sysconfig/swap.asciidoc b/docs/reference/setup/sysconfig/swap.asciidoc
index f9de8979b5f..b84db6f415c 100644
--- a/docs/reference/setup/sysconfig/swap.asciidoc
+++ b/docs/reference/setup/sysconfig/swap.asciidoc
@@ -13,7 +13,7 @@ disconnect from the cluster.
 There are three approaches to disabling swapping:
 
 [[mlockall]]
-==== Enable `bootstrap.mlockall`
+==== Enable `bootstrap.memory_lock`
 
 The first option is to use
 http://opengroup.org/onlinepubs/007908799/xsh/mlockall.html[mlockall] on Linux/Unix systems, or https://msdn.microsoft.com/en-us/library/windows/desktop/aa366895%28v=vs.85%29.aspx[VirtualLock] on Windows, to
@@ -23,7 +23,7 @@ to the `config/elasticsearch.yml` file:
 
 [source,yaml]
 --------------
-bootstrap.mlockall: true
+bootstrap.memory_lock: true
 --------------
 
 WARNING: `mlockall` might cause the JVM or shell session to exit if it tries

From fb893a993f06d470d23ac52e592fbfd7ccaceaed Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 1 Jun 2016 16:31:16 -0400
Subject: [PATCH 0556/1311] Add note regarding Windows service heap size

This commit adds a note regarding the difference in configuration for
the Windows service heap size from any other installation of
Elasticsearch.

Relates #18606
---
 docs/reference/setup/install/windows.asciidoc     | 6 ++++++
 docs/reference/setup/sysconfig/heap_size.asciidoc | 7 +++++++
 2 files changed, 13 insertions(+)

diff --git a/docs/reference/setup/install/windows.asciidoc b/docs/reference/setup/install/windows.asciidoc
index edf91e08890..c18fc0669d3 100644
--- a/docs/reference/setup/install/windows.asciidoc
+++ b/docs/reference/setup/install/windows.asciidoc
@@ -173,6 +173,12 @@ The Elasticsearch service can be configured prior to installation by setting the
 NOTE: At its core, `service.bat` relies on http://commons.apache.org/proper/commons-daemon/[Apache Commons Daemon] project
 to install the service. Environment variables set prior to the service installation are copied and will be used during the service lifecycle. This means any changes made to them after the installation will not be picked up unless the service is reinstalled.
 
+NOTE: On Windows, the <<heap-size,heap size>> can be configured as for
+any other Elasticsearch installation when running Elasticsearch from the
+command line, or when installing Elasticsearch as a service for the
+first time. To adjust the heap size for an already installed service,
+use the service manager: `bin\service.bat manager`.
+
 Using the Manager GUI::
 
 It is also possible to configure the service after it's been installed using the manager GUI (`elasticsearch-service-mgr.exe`), which offers insight into the installed service, including its status, startup type, JVM, start and stop settings amongst other things.  Simply invoking `service.bat manager` from the command-line will open up the manager window:
diff --git a/docs/reference/setup/sysconfig/heap_size.asciidoc b/docs/reference/setup/sysconfig/heap_size.asciidoc
index 0e3a3b2a44c..00c4553b97f 100644
--- a/docs/reference/setup/sysconfig/heap_size.asciidoc
+++ b/docs/reference/setup/sysconfig/heap_size.asciidoc
@@ -65,3 +65,10 @@ ES_JAVA_OPTS="-Xms4000m -Xmx4000m" ./bin/elasticsearch <2>
 ------------------
 <1> Set the minimum and maximum heap size to 2 GB.
 <2> Set the minimum and maximum heap size to 4000 MB.
+
+NOTE: Configuring the heap for the <<windows-service,Windows service>>
+is different than the above. The values initiallly populated for the
+Windows service can be configured as above but are different after the
+service has been installed. Consult the
+<<windows-service,Windows service documentation>> for additional
+details.

From f449666c592db7dc3ecaf1a98249e7b724310739 Mon Sep 17 00:00:00 2001
From: Nicholas Knize <nknize@gmail.com>
Date: Thu, 12 May 2016 12:39:51 -0500
Subject: [PATCH 0557/1311] New Matrix Stats Aggregation module

This commit adds a new aggs-matrix-stats module. The module presents a new class of aggregations called Matrix Aggregations. Matrix aggregations work on multiple fields and produce a matrix as output. The first matrix aggregation provided by the module is matrix_stats aggregation. This aggregation computes the following statistics over a set of fields:

* Covariance
* Correlation

For completeness (and interpretation purposes) the following per-field statistics are also provided:

* sample count
* population mean
* population variance
* population skewness
* population kurtosis
---
 modules/aggs-matrix-stats/build.gradle        |  35 ++
 .../MatrixStatsAggregationBuilders.java       |  33 ++
 .../matrix/MatrixAggregationPlugin.java       |  52 +++
 .../matrix/stats/InternalMatrixStats.java     | 268 +++++++++++++
 .../matrix/stats/MatrixStats.java             |  50 +++
 .../matrix/stats/MatrixStatsAggregator.java   | 150 ++++++++
 .../stats/MatrixStatsAggregatorBuilder.java   |  87 +++++
 .../stats/MatrixStatsAggregatorFactory.java   |  59 +++
 .../matrix/stats/MatrixStatsParser.java       |  50 +++
 .../matrix/stats/MatrixStatsResults.java      | 242 ++++++++++++
 .../matrix/stats/RunningStats.java            | 358 ++++++++++++++++++
 .../matrix/MatrixAggregationRestIT.java       |  41 ++
 .../matrix/stats/MatrixStatsTestCase.java     | 172 +++++++++
 .../matrix/stats/RunningStatsTests.java       |  72 ++++
 .../rest-api-spec/test/stats/10_basic.yaml    |  13 +
 .../test/stats/20_empty_bucket.yaml           |  48 +++
 .../test/stats/30_single_value_field.yaml     | 184 +++++++++
 .../test/stats/40_multi_value_field.yaml      | 184 +++++++++
 settings.gradle                               |   1 +
 19 files changed, 2099 insertions(+)
 create mode 100644 modules/aggs-matrix-stats/build.gradle
 create mode 100644 modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/MatrixStatsAggregationBuilders.java
 create mode 100644 modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/MatrixAggregationPlugin.java
 create mode 100644 modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java
 create mode 100644 modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStats.java
 create mode 100644 modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java
 create mode 100644 modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorBuilder.java
 create mode 100644 modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java
 create mode 100644 modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsParser.java
 create mode 100644 modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java
 create mode 100644 modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStats.java
 create mode 100644 modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixAggregationRestIT.java
 create mode 100644 modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsTestCase.java
 create mode 100644 modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStatsTests.java
 create mode 100644 modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/10_basic.yaml
 create mode 100644 modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/20_empty_bucket.yaml
 create mode 100644 modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yaml
 create mode 100644 modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yaml

diff --git a/modules/aggs-matrix-stats/build.gradle b/modules/aggs-matrix-stats/build.gradle
new file mode 100644
index 00000000000..78938837806
--- /dev/null
+++ b/modules/aggs-matrix-stats/build.gradle
@@ -0,0 +1,35 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+esplugin {
+    description 'Adds aggregations whose input are a list of numeric fields and output includes a matrix.'
+    classname 'org.elasticsearch.search.aggregations.matrix.MatrixAggregationPlugin'
+}
+
+dependencies {
+    testCompile project(path: ':plugins:lang-javascript', configuration: 'runtime')
+}
+
+integTest {
+    cluster {
+        plugin 'lang-javascript', project(':plugins:lang-javascript')
+        setting 'script.inline', 'true'
+        setting 'script.stored', 'true'
+    }
+}
\ No newline at end of file
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/MatrixStatsAggregationBuilders.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/MatrixStatsAggregationBuilders.java
new file mode 100644
index 00000000000..321f4fbceae
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/MatrixStatsAggregationBuilders.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations;
+
+import org.elasticsearch.search.aggregations.matrix.stats.MatrixStats;
+import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsAggregatorBuilder;
+
+/**
+ */
+public class MatrixStatsAggregationBuilders {
+    /**
+     * Create a new {@link MatrixStats} aggregation with the given name.
+     */
+    public static MatrixStatsAggregatorBuilder matrixStats(String name) {
+        return new MatrixStatsAggregatorBuilder(name);
+    }
+}
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/MatrixAggregationPlugin.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/MatrixAggregationPlugin.java
new file mode 100644
index 00000000000..c1397fa410e
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/MatrixAggregationPlugin.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.matrix;
+
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.aggregations.matrix.stats.InternalMatrixStats;
+import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsParser;
+
+import java.io.IOException;
+
+public class MatrixAggregationPlugin extends Plugin {
+    static {
+        InternalMatrixStats.registerStreams();
+    }
+
+    public MatrixAggregationPlugin() throws IOException {
+    }
+
+    @Override
+    public String name() {
+        return "aggs-matrix-stats";
+    }
+
+    @Override
+    public String description() {
+        return "Adds aggregations whose input are a list of numeric fields and output includes a matrix.";
+    }
+
+    public void onModule(SearchModule searchModule) {
+        searchModule.registerAggregation(MatrixStatsAggregatorBuilder::new, new MatrixStatsParser(),
+            MatrixStatsAggregatorBuilder.AGGREGATION_NAME_FIELD);
+    }
+}
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java
new file mode 100644
index 00000000000..da7683b4b3d
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java
@@ -0,0 +1,268 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.matrix.stats;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.aggregations.AggregationStreams;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.metrics.InternalMetricsAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.io.IOException;
+import java.util.*;
+
+/**
+ * Computes distribution statistics over multiple fields
+ */
+public class InternalMatrixStats extends InternalMetricsAggregation implements MatrixStats {
+
+    public final static Type TYPE = new Type("matrix_stats");
+    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+        @Override
+        public InternalMatrixStats readResult(StreamInput in) throws IOException {
+            InternalMatrixStats result = new InternalMatrixStats();
+            result.readFrom(in);
+            return result;
+        }
+    };
+
+    public static void registerStreams() {
+        AggregationStreams.registerStream(STREAM, TYPE.stream());
+    }
+
+    /** per shard stats needed to compute stats */
+    protected RunningStats stats;
+    /** final result */
+    protected MatrixStatsResults results;
+
+    protected InternalMatrixStats() {
+    }
+
+    /** per shard ctor */
+    protected InternalMatrixStats(String name, long count, RunningStats multiFieldStatsResults, MatrixStatsResults results,
+                                  List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+        super(name, pipelineAggregators, metaData);
+        assert count >= 0;
+        this.stats = multiFieldStatsResults;
+        this.results = results;
+    }
+
+    @Override
+    public Type type() {
+        return TYPE;
+    }
+
+    @Override
+    public long getDocCount() {
+        return stats.docCount;
+    }
+
+    @Override
+    public long getFieldCount(String field) {
+        if (results == null) {
+            return 0;
+        }
+        return results.getFieldCount(field);
+    }
+
+    @Override
+    public Double getMean(String field) {
+        if (results == null) {
+            return null;
+        }
+        return results.getMean(field);
+    }
+
+    @Override
+    public Double getVariance(String field) {
+        if (results == null) {
+            return null;
+        }
+        return results.getVariance(field);
+    }
+
+    @Override
+    public Double getSkewness(String field) {
+        if (results == null) {
+            return null;
+        }
+        return results.getSkewness(field);
+    }
+
+    @Override
+    public Double getKurtosis(String field) {
+        if (results == null) {
+            return null;
+        }
+        return results.getKurtosis(field);
+    }
+
+    @Override
+    public Double getCovariance(String fieldX, String fieldY) {
+        if (results == null) {
+            return null;
+        }
+        return results.getCovariance(fieldX, fieldY);
+    }
+
+    @Override
+    public Map<String, HashMap<String, Double>> getCovariance() {
+        return results.getCovariances();
+    }
+
+    @Override
+    public Double getCorrelation(String fieldX, String fieldY) {
+        if (results == null) {
+            return null;
+        }
+        return results.getCorrelation(fieldX, fieldY);
+    }
+
+    @Override
+    public Map<String, HashMap<String, Double>> getCorrelation() {
+        return results.getCorrelations();
+    }
+
+    static class Fields {
+        public static final String COUNT = "count";
+        public static final String MEAN = "mean";
+        public static final String VARIANCE = "variance";
+        public static final String SKEWNESS = "skewness";
+        public static final String KURTOSIS = "kurtosis";
+        public static final String COVARIANCE = "covariance";
+        public static final String CORRELATION = "correlation";
+    }
+
+    @Override
+    public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        if (results != null) {
+            Set<String> fieldNames = results.getFieldCounts().keySet();
+            builder.field("field", fieldNames);
+            builder.field(Fields.COUNT, results.getFieldCounts().values());
+            builder.field(Fields.MEAN, results.getMeans().values());
+            builder.field(Fields.VARIANCE, results.getVariances().values());
+            builder.field(Fields.SKEWNESS, results.getSkewness().values());
+            builder.field(Fields.KURTOSIS, results.getKurtosis().values());
+            ArrayList<ArrayList<Double>> cov = new ArrayList<>(fieldNames.size());
+            ArrayList<ArrayList<Double>> cor = new ArrayList<>(fieldNames.size());
+            for (String y : fieldNames) {
+                ArrayList<Double> covRow = new ArrayList<>(fieldNames.size());
+                ArrayList<Double> corRow = new ArrayList<>(fieldNames.size());
+                for (String x : fieldNames) {
+                    covRow.add(results.getCovariance(x, y));
+                    corRow.add(results.getCorrelation(x, y));
+                }
+                cov.add(covRow);
+                cor.add(corRow);
+            }
+            builder.field(Fields.COVARIANCE, cov);
+            builder.field(Fields.CORRELATION, cor);
+        }
+
+        return builder;
+    }
+
+    @Override
+    public Object getProperty(List<String> path) {
+        if (path.isEmpty()) {
+            return this;
+        } else if (path.size() == 1) {
+            String coordinate = path.get(0);
+            if (results == null) {
+                results = MatrixStatsResults.EMPTY();
+            }
+            switch (coordinate) {
+                case "counts":
+                    return results.getFieldCounts();
+                case "means":
+                    return results.getMeans();
+                case "variances":
+                    return results.getVariances();
+                case "skewness":
+                    return results.getSkewness();
+                case "kurtosis":
+                    return results.getKurtosis();
+                case "covariance":
+                    return results.getCovariances();
+                case "correlation":
+                    return results.getCorrelations();
+                default:
+                    throw new IllegalArgumentException("Found unknown path element [" + coordinate + "] in [" + getName() + "]");
+            }
+        } else {
+            throw new IllegalArgumentException("path not supported for [" + getName() + "]: " + path);
+        }
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        // write running stats
+        if (stats == null || stats.docCount == 0) {
+            out.writeVLong(0);
+        } else {
+            out.writeVLong(stats.docCount);
+            stats.writeTo(out);
+        }
+
+        // write results
+        if (results == null || results.getDocCount() == 0) {
+            out.writeVLong(0);
+        } else {
+            out.writeVLong(results.getDocCount());
+            results.writeTo(out);
+        }
+    }
+
+    @Override
+    protected void doReadFrom(StreamInput in) throws IOException {
+        // read stats count
+        final long statsCount = in.readVLong();
+        if (statsCount > 0) {
+            stats = new RunningStats(in);
+            stats.docCount = statsCount;
+        }
+
+        // read count
+        final long count = in.readVLong();
+        if (count > 0) {
+            results = new MatrixStatsResults(in);
+        }
+    }
+
+    @Override
+    public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
+        // merge stats across all shards
+        aggregations.removeIf(p -> ((InternalMatrixStats)p).stats == null);
+
+        // return empty result iff all stats are null
+        if (aggregations.isEmpty()) {
+            return new InternalMatrixStats(name, 0, null, MatrixStatsResults.EMPTY(), pipelineAggregators(), getMetaData());
+        }
+
+        RunningStats runningStats = ((InternalMatrixStats) aggregations.get(0)).stats;
+        for (int i=1; i < aggregations.size(); ++i) {
+            runningStats.merge(((InternalMatrixStats) aggregations.get(i)).stats);
+        }
+        MatrixStatsResults results = new MatrixStatsResults(stats);
+
+        return new InternalMatrixStats(name, results.getDocCount(), runningStats, results, pipelineAggregators(), getMetaData());
+    }
+}
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStats.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStats.java
new file mode 100644
index 00000000000..1de20faae93
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStats.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.matrix.stats;
+
+import org.elasticsearch.search.aggregations.Aggregation;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Interface for MatrixStats Metric Aggregation
+ */
+public interface MatrixStats extends Aggregation {
+    /** return the total document count */
+    long getDocCount();
+    /** return total field count (differs from docCount if there are missing values) */
+    long getFieldCount(String field);
+    /** return the field mean */
+    Double getMean(String field);
+    /** return the field variance */
+    Double getVariance(String field);
+    /** return the skewness of the distribution */
+    Double getSkewness(String field);
+    /** return the kurtosis of the distribution */
+    Double getKurtosis(String field);
+    /** return the upper triangle of the covariance matrix */
+    Map<String, HashMap<String, Double>> getCovariance();
+    /** return the covariance between field x and field y */
+    Double getCovariance(String fieldX, String fieldY);
+    /** return the upper triangle of the pearson product-moment correlation matrix */
+    Map<String, HashMap<String, Double>> getCorrelation();
+    /** return the correlation coefficient of field x and field y */
+    Double getCorrelation(String fieldX, String fieldY);
+}
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java
new file mode 100644
index 00000000000..275163bf103
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.matrix.stats;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.ObjectArray;
+import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.LeafBucketCollector;
+import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
+import org.elasticsearch.search.aggregations.metrics.MetricsAggregator;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Metric Aggregation for computing the pearson product correlation coefficient between multiple fields
+ **/
+public class MatrixStatsAggregator extends MetricsAggregator {
+    /** Multiple ValuesSource with field names */
+    final Map<String, ValuesSource.Numeric> valuesSources;
+
+    /** array of descriptive stats, per shard, needed to compute the correlation */
+    ObjectArray<RunningStats> stats;
+
+    public MatrixStatsAggregator(String name, Map<String, ValuesSource.Numeric> valuesSources, AggregationContext context,
+                                 Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+                                 Map<String,Object> metaData) throws IOException {
+        super(name, context, parent, pipelineAggregators, metaData);
+        this.valuesSources = valuesSources;
+        if (valuesSources != null && !valuesSources.isEmpty()) {
+            stats = context.bigArrays().newObjectArray(1);
+        }
+    }
+
+    @Override
+    public boolean needsScores() {
+        boolean needsScores = false;
+        if (valuesSources != null) {
+            for (Map.Entry<String, ValuesSource.Numeric> valueSource : valuesSources.entrySet()) {
+                needsScores |= valueSource.getValue().needsScores();
+            }
+        }
+        return needsScores;
+    }
+
+    @Override
+    public LeafBucketCollector getLeafCollector(LeafReaderContext ctx,
+                                                final LeafBucketCollector sub) throws IOException {
+        if (valuesSources == null || valuesSources.isEmpty()) {
+            return LeafBucketCollector.NO_OP_COLLECTOR;
+        }
+        final BigArrays bigArrays = context.bigArrays();
+        final HashMap<String, SortedNumericDoubleValues> values = new HashMap<>(valuesSources.size());
+        for (Map.Entry<String, ValuesSource.Numeric> valuesSource : valuesSources.entrySet()) {
+            values.put(valuesSource.getKey(), valuesSource.getValue().doubleValues(ctx));
+        }
+
+        return new LeafBucketCollectorBase(sub, values) {
+            @Override
+            public void collect(int doc, long bucket) throws IOException {
+                // get fields
+                Map<String, Double> fields = getFields(doc);
+                if (fields != null) {
+                    stats = bigArrays.grow(stats, bucket + 1);
+                    RunningStats stat = stats.get(bucket);
+                    // add document fields to correlation stats
+                    if (stat == null) {
+                        stat = new RunningStats(fields);
+                    } else {
+                        stat.add(fields);
+                    }
+                    stats.set(bucket, stat);
+                }
+            }
+
+            /**
+             * return a map of field names and data
+             */
+            private Map<String, Double> getFields(int doc) {
+                // get fieldNames to use as hash keys
+                ArrayList<String> fieldNames = new ArrayList<>(values.keySet());
+                HashMap<String, Double> fields = new HashMap<>(fieldNames.size());
+
+                // loop over fields
+                for (String fieldName : fieldNames) {
+                    final SortedNumericDoubleValues doubleValues = values.get(fieldName);
+                    doubleValues.setDocument(doc);
+                    final int valuesCount = doubleValues.count();
+                    // if document contains an empty field we omit the doc from the correlation
+                    if (valuesCount <= 0) {
+                        return null;
+                    }
+                    // get the field value (multi-value is the average of all the values)
+                    double fieldValue = 0;
+                    for (int i = 0; i < valuesCount; ++i) {
+                        if (Double.isNaN(doubleValues.valueAt(i)) == false) {
+                            fieldValue += doubleValues.valueAt(i);
+                        }
+                    }
+                    fields.put(fieldName, fieldValue / valuesCount);
+                }
+                return fields;
+            }
+        };
+    }
+
+    @Override
+    public InternalAggregation buildAggregation(long bucket) {
+        if (valuesSources == null || bucket >= stats.size()) {
+            return buildEmptyAggregation();
+        }
+        return new InternalMatrixStats(name, stats.size(), stats.get(bucket), null, pipelineAggregators(), metaData());
+    }
+
+    @Override
+    public InternalAggregation buildEmptyAggregation() {
+        return new InternalMatrixStats(name, 0, null, null, pipelineAggregators(), metaData());
+    }
+
+    @Override
+    public void doClose() {
+        Releasables.close(stats);
+    }
+}
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorBuilder.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorBuilder.java
new file mode 100644
index 00000000000..ea20a3c32ad
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorBuilder.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.matrix.stats;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.aggregations.support.MultiValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.aggregations.support.ValueType;
+import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ */
+public class MatrixStatsAggregatorBuilder
+    extends MultiValuesSourceAggregatorBuilder.LeafOnly<ValuesSource.Numeric, MatrixStatsAggregatorBuilder> {
+    public static final String NAME = InternalMatrixStats.TYPE.name();
+    public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
+
+    public MatrixStatsAggregatorBuilder(String name) {
+        super(name, InternalMatrixStats.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public MatrixStatsAggregatorBuilder(StreamInput in) throws IOException {
+        super(in, InternalMatrixStats.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+    }
+
+    @Override
+    protected void innerWriteTo(StreamOutput out) {
+        // Do nothing, no extra state to write to stream
+    }
+
+    @Override
+    protected MatrixStatsAggregatorFactory innerBuild(AggregationContext context, Map<String, ValuesSourceConfig<Numeric>> configs,
+                                                      AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
+        return new MatrixStatsAggregatorFactory(name, type, configs, context, parent, subFactoriesBuilder, metaData);
+    }
+
+    @Override
+    public XContentBuilder doXContentBody(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        return builder;
+    }
+
+    @Override
+    protected int innerHashCode() {
+        return 0;
+    }
+
+    @Override
+    protected boolean innerEquals(Object obj) {
+        return true;
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+}
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java
new file mode 100644
index 00000000000..30389c05836
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.matrix.stats;
+
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.aggregations.support.MultiValuesSourceAggregatorFactory;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+/**
+ */
+public class MatrixStatsAggregatorFactory
+    extends MultiValuesSourceAggregatorFactory<ValuesSource.Numeric, MatrixStatsAggregatorFactory> {
+
+    public MatrixStatsAggregatorFactory(String name, InternalAggregation.Type type,
+                                        Map<String, ValuesSourceConfig<ValuesSource.Numeric>> configs, AggregationContext context,
+                                        AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+                                        Map<String, Object> metaData) throws IOException {
+        super(name, type, configs, context, parent, subFactoriesBuilder, metaData);
+    }
+
+    @Override
+    protected Aggregator createUnmapped(Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
+        throws IOException {
+        return new MatrixStatsAggregator(name, null, context, parent, pipelineAggregators, metaData);
+    }
+
+    @Override
+    protected Aggregator doCreateInternal(Map<String, ValuesSource.Numeric> valuesSources, Aggregator parent,
+                                          boolean collectsFromSingleBucket, List<PipelineAggregator> pipelineAggregators,
+                                          Map<String, Object> metaData) throws IOException {
+        return new MatrixStatsAggregator(name, valuesSources, context, parent, pipelineAggregators, metaData);
+    }
+}
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsParser.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsParser.java
new file mode 100644
index 00000000000..950bd078086
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsParser.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.matrix.stats;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.support.MultiValuesSourceParser.NumericValuesSourceParser;
+import org.elasticsearch.search.aggregations.support.ValueType;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ */
+public class MatrixStatsParser extends NumericValuesSourceParser {
+
+    public MatrixStatsParser() {
+        super(true, true, false);
+    }
+
+    @Override
+    protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, XContentParser parser,
+                            ParseFieldMatcher parseFieldMatcher, Map<ParseField, Object> otherOptions) throws IOException {
+        return false;
+    }
+
+    @Override
+    protected MatrixStatsAggregatorBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
+                                                         ValueType targetValueType, Map<ParseField, Object> otherOptions) {
+        return new MatrixStatsAggregatorBuilder(aggregationName);
+    }
+}
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java
new file mode 100644
index 00000000000..65cedbc2676
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.matrix.stats;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Streamable;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Descriptive stats gathered per shard. Coordinating node computes final pearson product coefficient
+ * based on these descriptive stats
+ *
+ * @internal
+ */
+class MatrixStatsResults implements Streamable {
+    /** object holding results - computes results in place */
+    final RunningStats results;
+    /** pearson product correlation coefficients */
+    protected HashMap<String, HashMap<String, Double>> correlation;
+
+    /** Base ctor */
+    private MatrixStatsResults() {
+        results = RunningStats.EMPTY();
+        this.correlation = new HashMap<>();
+    }
+
+    /** creates and computes result from provided stats */
+    public MatrixStatsResults(RunningStats stats) {
+        try {
+            this.results = stats.clone();
+            this.correlation = new HashMap<>();
+        } catch (CloneNotSupportedException e) {
+            throw new ElasticsearchException("Error trying to create multifield_stats results", e);
+        }
+        this.compute();
+    }
+
+    /** creates a results object from the given stream */
+    protected MatrixStatsResults(StreamInput in) {
+        try {
+            results = new RunningStats(in);
+            this.readFrom(in);
+        } catch (IOException e) {
+            throw new ElasticsearchException("Error trying to create multifield_stats results from stream input", e);
+        }
+    }
+
+    /** create an empty results object **/
+    protected static MatrixStatsResults EMPTY() {
+        return new MatrixStatsResults();
+    }
+
+    /** return document count */
+    public final long getDocCount() {
+        return results.docCount;
+    }
+
+    /** return the field counts */
+    public Map<String, Long> getFieldCounts() {
+        return Collections.unmodifiableMap(results.counts);
+    }
+
+    /** return the fied count for the requested field */
+    public long getFieldCount(String field) {
+        if (results.counts.containsKey(field) == false) {
+            return 0;
+        }
+        return results.counts.get(field);
+    }
+
+    /** return the means */
+    public Map<String, Double> getMeans() {
+        return Collections.unmodifiableMap(results.means);
+    }
+
+    /** return the mean for the requested field */
+    public Double getMean(String field) {
+        return results.means.get(field);
+    }
+
+    /** return the variances */
+    public Map<String, Double> getVariances() {
+        return Collections.unmodifiableMap(results.variances);
+    }
+
+    /** return the variance for the requested field */
+    public Double getVariance(String field) {
+        return results.variances.get(field);
+    }
+
+    /** return the skewness */
+    public Map<String, Double> getSkewness() {
+        return Collections.unmodifiableMap(results.skewness);
+    }
+
+    /** return the skewness for the requested field */
+    public Double getSkewness(String field) {
+        return results.skewness.get(field);
+    }
+
+    /** return the kurtosis */
+    public Map<String, Double> getKurtosis() {
+        return Collections.unmodifiableMap(results.kurtosis);
+    }
+
+    /** return the kurtosis for the requested field */
+    public Double getKurtosis(String field) {
+        return results.kurtosis.get(field);
+    }
+
+    /** return the covariances */
+    public Map<String, HashMap<String, Double>> getCovariances() {
+        return Collections.unmodifiableMap(results.covariances);
+    }
+
+    /** return the covariance between two fields */
+    public Double getCovariance(String fieldX, String fieldY) {
+        if (fieldX.equals(fieldY)) {
+            return results.variances.get(fieldX);
+        }
+        return getValFromUpperTriangularMatrix(results.covariances, fieldX, fieldY);
+    }
+
+    public Map<String, HashMap<String, Double>> getCorrelations() {
+        return Collections.unmodifiableMap(correlation);
+    }
+
+    /** return the correlation coefficient between two fields */
+    public Double getCorrelation(String fieldX, String fieldY) {
+        if (fieldX.equals(fieldY)) {
+            return 1.0;
+        }
+        return getValFromUpperTriangularMatrix(correlation, fieldX, fieldY);
+    }
+
+    /** return the value for two fields in an upper triangular matrix, regardless of row col location. */
+    private Double getValFromUpperTriangularMatrix(HashMap<String, HashMap<String, Double>> map, String fieldX, String fieldY) {
+        // for the co-value to exist, one of the two (or both) fields has to be a row key
+        if (map.containsKey(fieldX) == false && map.containsKey(fieldY) == false) {
+            return null;
+        } else if (map.containsKey(fieldX)) {
+            // fieldX exists as a row key
+            if (map.get(fieldX).containsKey(fieldY)) {
+                // fieldY exists as a col key to fieldX
+                return map.get(fieldX).get(fieldY);
+            } else {
+                // otherwise fieldX is the col key to fieldY
+                return map.get(fieldY).get(fieldX);
+            }
+        } else if (map.containsKey(fieldY)) {
+            // fieldX did not exist as a row key, it must be a col key
+            return map.get(fieldY).get(fieldX);
+        }
+        throw new IllegalArgumentException("Coefficient not computed between fields: " + fieldX + " and " + fieldY);
+    }
+
+    /** Computes final covariance, variance, and correlation */
+    private void compute() {
+        final double nM1 = results.docCount - 1D;
+        // compute final skewness and kurtosis
+        for (String fieldName : results.means.keySet()) {
+            final double var = results.variances.get(fieldName);
+            // update skewness
+            results.skewness.put(fieldName, Math.sqrt(results.docCount) * results.skewness.get(fieldName) / Math.pow(var, 1.5D));
+            // update kurtosis
+            results.kurtosis.put(fieldName, (double)results.docCount * results.kurtosis.get(fieldName) / (var * var));
+            // update variances
+            results.variances.put(fieldName, results.variances.get(fieldName) / nM1);
+        }
+
+        // compute final covariances and correlation
+        double cor;
+        for (Map.Entry<String, HashMap<String, Double>> row : results.covariances.entrySet()) {
+            final String rowName = row.getKey();
+            final HashMap<String, Double> covRow = row.getValue();
+            final HashMap<String, Double> corRow = new HashMap<>();
+            for (Map.Entry<String, Double> col : covRow.entrySet()) {
+                final String colName = col.getKey();
+                // update covariance
+                covRow.put(colName, covRow.get(colName) / nM1);
+                // update correlation
+                // if there is no variance in the data then correlation is NaN
+                if (results.variances.get(rowName) == 0d || results.variances.get(colName) == 0d) {
+                    cor = Double.NaN;
+                } else {
+                    final double corDen = Math.sqrt(results.variances.get(rowName)) * Math.sqrt(results.variances.get(colName));
+                    cor = covRow.get(colName) / corDen;
+                }
+                corRow.put(colName, cor);
+            }
+            results.covariances.put(rowName, covRow);
+            correlation.put(rowName, corRow);
+        }
+    }
+
+    /** Unmarshalls MatrixStatsResults */
+    @Override
+    @SuppressWarnings("unchecked")
+    public void readFrom(StreamInput in) throws IOException {
+        if (in.readBoolean()) {
+            correlation = (HashMap<String, HashMap<String, Double>>) (in.readGenericValue());
+        } else {
+            correlation = null;
+        }
+    }
+
+    /** Marshalls MatrixStatsResults */
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        // marshall results
+        results.writeTo(out);
+        // marshall correlation
+        if (correlation != null) {
+            out.writeBoolean(true);
+            out.writeGenericValue(correlation);
+        } else {
+            out.writeBoolean(false);
+        }
+    }
+}
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStats.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStats.java
new file mode 100644
index 00000000000..57a8a8948fe
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStats.java
@@ -0,0 +1,358 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.matrix.stats;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Streamable;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Descriptive stats gathered per shard. Coordinating node computes final correlation and covariance stats
+ * based on these descriptive stats. This single pass, parallel approach is based on:
+ *
+ * http://prod.sandia.gov/techlib/access-control.cgi/2008/086212.pdf
+ *
+ * @internal
+ */
+public class RunningStats implements Streamable, Cloneable {
+    /** count of observations (same number of observations per field) */
+    protected long docCount = 0;
+    /** per field sum of observations */
+    protected HashMap<String, Double> fieldSum;
+    /** counts */
+    protected HashMap<String, Long> counts;
+    /** mean values (first moment) */
+    protected HashMap<String, Double> means;
+    /** variance values (second moment) */
+    protected HashMap<String, Double> variances;
+    /** skewness values (third moment) */
+    protected HashMap<String, Double> skewness;
+    /** kurtosis values (fourth moment) */
+    protected HashMap<String, Double> kurtosis;
+    /** covariance values */
+    protected HashMap<String, HashMap<String, Double>> covariances;
+
+    private RunningStats() {
+        init();
+    }
+
+    /** Ctor to create an instance of running statistics */
+    public RunningStats(StreamInput in) throws IOException {
+        this();
+        this.readFrom(in);
+    }
+
+    public RunningStats(Map<String, Double> doc) {
+        if (doc != null && doc.isEmpty() == false) {
+            init();
+            this.add(doc);
+        }
+    }
+
+    private void init() {
+        counts = new HashMap<>();
+        fieldSum = new HashMap<>();
+        means = new HashMap<>();
+        skewness = new HashMap<>();
+        kurtosis = new HashMap<>();
+        covariances = new HashMap<>();
+        variances = new HashMap<>();
+    }
+
+    /** create an empty instance */
+    protected static RunningStats EMPTY() {
+        return new RunningStats();
+    }
+
+    /** updates running statistics with a documents field values **/
+    public void add(Map<String, Double> doc) {
+        if (doc == null || doc.isEmpty()) {
+            return;
+        }
+
+        // update total, mean, and variance
+        ++docCount;
+        String fieldName;
+        double fieldValue;
+        double m1, m2, m3, m4;  // moments
+        double d, dn, dn2, t1;
+        final HashMap<String, Double> deltas = new HashMap<>();
+        for (Map.Entry<String, Double> field : doc.entrySet()) {
+            fieldName = field.getKey();
+            fieldValue = field.getValue();
+
+            // update counts
+            counts.put(fieldName, 1 + (counts.containsKey(fieldName) ? counts.get(fieldName) : 0));
+            // update running sum
+            fieldSum.put(fieldName, fieldValue + (fieldSum.containsKey(fieldName) ? fieldSum.get(fieldName) : 0));
+            // update running deltas
+            deltas.put(fieldName, fieldValue * docCount - fieldSum.get(fieldName));
+
+            // update running mean, variance, skewness, kurtosis
+            if (means.containsKey(fieldName) == true) {
+                // update running means
+                m1 = means.get(fieldName);
+                d = fieldValue - m1;
+                means.put(fieldName, m1 + d / docCount);
+                // update running variances
+                dn = d / docCount;
+                t1 = d * dn * (docCount - 1);
+                m2 = variances.get(fieldName);
+                variances.put(fieldName, m2 + t1);
+                m3 = skewness.get(fieldName);
+                skewness.put(fieldName, m3 + (t1 * dn * (docCount - 2D) - 3D * dn * m2));
+                dn2 = dn * dn;
+                m4 = t1 * dn2 * (docCount * docCount - 3D * docCount + 3D) + 6D * dn2 * m2 - 4D * dn * m3;
+                kurtosis.put(fieldName, kurtosis.get(fieldName) + m4);
+            } else {
+                means.put(fieldName, fieldValue);
+                variances.put(fieldName, 0.0);
+                skewness.put(fieldName, 0.0);
+                kurtosis.put(fieldName, 0.0);
+            }
+        }
+
+        this.updateCovariance(doc, deltas);
+    }
+
+    /** Update covariance matrix */
+    private void updateCovariance(final Map<String, Double> doc, final Map<String, Double> deltas) {
+        // deep copy of hash keys (field names)
+        ArrayList<String> cFieldNames = new ArrayList<>(doc.keySet());
+        String fieldName;
+        double dR, newVal;
+        for (Map.Entry<String, Double> field : doc.entrySet()) {
+            fieldName = field.getKey();
+            cFieldNames.remove(fieldName);
+            // update running covariances
+            dR = deltas.get(fieldName);
+            HashMap<String, Double> cFieldVals = (covariances.get(fieldName) != null) ? covariances.get(fieldName) : new HashMap<>();
+            for (String cFieldName : cFieldNames) {
+                if (cFieldVals.containsKey(cFieldName) == true) {
+                    newVal = cFieldVals.get(cFieldName) + 1.0 / (docCount * (docCount - 1.0)) * dR * deltas.get(cFieldName);
+                    cFieldVals.put(cFieldName, newVal);
+                } else {
+                    cFieldVals.put(cFieldName, 0.0);
+                }
+            }
+            if (cFieldVals.size() > 0) {
+                covariances.put(fieldName, cFieldVals);
+            }
+        }
+    }
+
+    /**
+     * Merges the descriptive statistics of a second data set (e.g., per shard)
+     *
+     * running computations taken from: http://prod.sandia.gov/techlib/access-control.cgi/2008/086212.pdf
+     **/
+    public void merge(final RunningStats other) {
+        if (other == null) {
+            return;
+        }
+        final double nA = docCount;
+        final double nB = other.docCount;
+        // merge count
+        docCount += other.docCount;
+
+        final HashMap<String, Double> deltas = new HashMap<>();
+        double meanA, varA, skewA, kurtA, meanB, varB, skewB, kurtB;
+        double d, d2, d3, d4, n2, nA2, nB2;
+        double newSkew, nk;
+        // across fields
+        for (Map.Entry<String, Double> fs : other.means.entrySet()) {
+            final String fieldName = fs.getKey();
+            meanA = means.get(fieldName);
+            varA = variances.get(fieldName);
+            skewA = skewness.get(fieldName);
+            kurtA = kurtosis.get(fieldName);
+            meanB = other.means.get(fieldName);
+            varB = other.variances.get(fieldName);
+            skewB = other.skewness.get(fieldName);
+            kurtB = other.kurtosis.get(fieldName);
+
+            // merge counts of two sets
+            counts.put(fieldName, counts.get(fieldName) + other.counts.get(fieldName));
+
+            // merge means of two sets
+            means.put(fieldName, (nA * means.get(fieldName) + nB * other.means.get(fieldName)) / (nA + nB));
+
+            // merge deltas
+            deltas.put(fieldName, other.fieldSum.get(fieldName) / nB - fieldSum.get(fieldName) / nA);
+
+            // merge totals
+            fieldSum.put(fieldName, fieldSum.get(fieldName) + other.fieldSum.get(fieldName));
+
+            // merge variances, skewness, and kurtosis of two sets
+            d = meanB - meanA;          // delta mean
+            d2 = d * d;                 // delta mean squared
+            d3 = d * d2;                // delta mean cubed
+            d4 = d2 * d2;               // delta mean 4th power
+            n2 = docCount * docCount;   // num samples squared
+            nA2 = nA * nA;              // doc A num samples squared
+            nB2 = nB * nB;              // doc B num samples squared
+            // variance
+            variances.put(fieldName, varA + varB + d2 * nA * other.docCount / docCount);
+            // skeewness
+            newSkew = skewA + skewB + d3 * nA * nB * (nA - nB) / n2;
+            skewness.put(fieldName, newSkew + 3D * d * (nA * varB - nB * varA) / docCount);
+            // kurtosis
+            nk = kurtA + kurtB + d4 * nA * nB * (nA2 - nA * nB + nB2) / (n2 * docCount);
+            kurtosis.put(fieldName, nk + 6D * d2 * (nA2 * varB + nB2 * varA) / n2 + 4D * d * (nA * skewB - nB * skewA) / docCount);
+        }
+
+        this.mergeCovariance(other, deltas);
+    }
+
+    /** Merges two covariance matrices */
+    private void mergeCovariance(final RunningStats other, final HashMap<String, Double> deltas) {
+        final double countA = docCount - other.docCount;
+        double f, dR, newVal;
+        for (Map.Entry<String, Double> fs : other.means.entrySet()) {
+            final String fieldName = fs.getKey();
+            // merge covariances of two sets
+            f = countA * other.docCount / this.docCount;
+            dR = deltas.get(fieldName);
+            // merge covariances
+            if (covariances.containsKey(fieldName)) {
+                HashMap<String, Double> cFieldVals = covariances.get(fieldName);
+                for (String cFieldName : cFieldVals.keySet()) {
+                    newVal = cFieldVals.get(cFieldName);
+                    if (other.covariances.containsKey(fieldName) && other.covariances.get(fieldName).containsKey(cFieldName)) {
+                        newVal += other.covariances.get(fieldName).get(cFieldName) + f * dR * deltas.get(cFieldName);
+                    } else {
+                        newVal += other.covariances.get(cFieldName).get(fieldName) + f * dR * deltas.get(cFieldName);
+                    }
+                    cFieldVals.put(cFieldName, newVal);
+                }
+                covariances.put(fieldName, cFieldVals);
+            }
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        // marshall fieldSum
+        if (fieldSum != null) {
+            out.writeBoolean(true);
+            out.writeGenericValue(fieldSum);
+        } else {
+            out.writeBoolean(false);
+        }
+        // counts
+        if (counts != null) {
+            out.writeBoolean(true);
+            out.writeGenericValue(counts);
+        } else {
+            out.writeBoolean(false);
+        }
+        // mean
+        if (means != null) {
+            out.writeBoolean(true);
+            out.writeGenericValue(means);
+        } else {
+            out.writeBoolean(false);
+        }
+        // variances
+        if (variances != null) {
+            out.writeBoolean(true);
+            out.writeGenericValue(variances);
+        } else {
+            out.writeBoolean(false);
+        }
+        // skewness
+        if (skewness != null) {
+            out.writeBoolean(true);
+            out.writeGenericValue(skewness);
+        } else {
+            out.writeBoolean(false);
+        }
+        // kurtosis
+        if (kurtosis != null) {
+            out.writeBoolean(true);
+            out.writeGenericValue(kurtosis);
+        } else {
+            out.writeBoolean(false);
+        }
+        // covariances
+        if (covariances != null) {
+            out.writeBoolean(true);
+            out.writeGenericValue(covariances);
+        } else {
+            out.writeBoolean(false);
+        }
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public void readFrom(StreamInput in) throws IOException {
+        // read fieldSum
+        if (in.readBoolean()) {
+            fieldSum = (HashMap<String, Double>)(in.readGenericValue());
+        } else {
+            fieldSum = null;
+        }
+        // counts
+        if (in.readBoolean()) {
+            counts = (HashMap<String, Long>)(in.readGenericValue());
+        } else {
+            counts = null;
+        }
+        // means
+        if (in.readBoolean()) {
+            means = (HashMap<String, Double>)(in.readGenericValue());
+        } else {
+            means = null;
+        }
+        // variances
+        if (in.readBoolean()) {
+            variances = (HashMap<String, Double>)(in.readGenericValue());
+        } else {
+            variances = null;
+        }
+        // skewness
+        if (in.readBoolean()) {
+            skewness = (HashMap<String, Double>)(in.readGenericValue());
+        } else {
+            skewness = null;
+        }
+        // kurtosis
+        if (in.readBoolean()) {
+            kurtosis = (HashMap<String, Double>)(in.readGenericValue());
+        } else {
+            kurtosis = null;
+        }
+        // read covariances
+        if (in.readBoolean()) {
+            covariances = (HashMap<String, HashMap<String, Double>>) (in.readGenericValue());
+        } else {
+            covariances = null;
+        }
+    }
+
+    @Override
+    public RunningStats clone() throws CloneNotSupportedException {
+        return (RunningStats)super.clone();
+    }
+}
diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixAggregationRestIT.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixAggregationRestIT.java
new file mode 100644
index 00000000000..69ddd46edc2
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixAggregationRestIT.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.matrix;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.RestTestCandidate;
+import org.elasticsearch.test.rest.parser.RestTestParseException;
+
+import java.io.IOException;
+
+/**
+ *
+ */
+public class MatrixAggregationRestIT extends ESRestTestCase {
+    public MatrixAggregationRestIT(@Name("yaml")RestTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws IOException, RestTestParseException {
+        return ESRestTestCase.createParameters(0, 1);
+    }
+}
diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsTestCase.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsTestCase.java
new file mode 100644
index 00000000000..84b032b82b6
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsTestCase.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.matrix.stats;
+
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ *
+ */
+public class MatrixStatsTestCase extends ESTestCase {
+    protected final int numObs = atLeast(10000);
+    protected final ArrayList<Double> fieldA = new ArrayList<>(numObs);
+    protected final ArrayList<Double> fieldB = new ArrayList<>(numObs);
+    protected final MultiPassStats actualStats = new MultiPassStats();
+    protected final static String fieldAKey = "fieldA";
+    protected final static String fieldBKey = "fieldB";
+
+    @Before
+    public void setup() {
+        createStats();
+    }
+
+    public void createStats() {
+        for (int n = 0; n < numObs; ++n) {
+            fieldA.add(randomDouble());
+            fieldB.add(randomDouble());
+        }
+        actualStats.computeStats(fieldA, fieldB);
+    }
+
+    static class MultiPassStats {
+        long count;
+        HashMap<String, Double> means = new HashMap<>();
+        HashMap<String, Double> variances = new HashMap<>();
+        HashMap<String, Double> skewness = new HashMap<>();
+        HashMap<String, Double> kurtosis = new HashMap<>();
+        HashMap<String, HashMap<String, Double>> covariances = new HashMap<>();
+        HashMap<String, HashMap<String, Double>> correlations = new HashMap<>();
+
+        @SuppressWarnings("unchecked")
+        void computeStats(final ArrayList<Double> fieldA, final ArrayList<Double> fieldB) {
+            // set count
+            count = fieldA.size();
+            double meanA = 0d;
+            double meanB = 0d;
+
+            // compute mean
+            for (int n = 0; n < count; ++n) {
+                // fieldA
+                meanA += fieldA.get(n);
+                meanB += fieldB.get(n);
+            }
+            means.put(fieldAKey, meanA/count);
+            means.put(fieldBKey, meanB/count);
+
+            // compute variance, skewness, and kurtosis
+            double dA;
+            double dB;
+            double skewA = 0d;
+            double skewB = 0d;
+            double kurtA = 0d;
+            double kurtB = 0d;
+            double varA = 0d;
+            double varB = 0d;
+            double cVar = 0d;
+            for (int n = 0; n < count; ++n) {
+                dA = fieldA.get(n) - means.get(fieldAKey);
+                varA += dA * dA;
+                skewA += dA * dA * dA;
+                kurtA += dA * dA * dA * dA;
+                dB = fieldB.get(n) - means.get(fieldBKey);
+                varB += dB * dB;
+                skewB += dB * dB * dB;
+                kurtB += dB * dB * dB * dB;
+                cVar += dA * dB;
+            }
+            variances.put(fieldAKey, varA / (count - 1));
+            final double stdA = Math.sqrt(variances.get(fieldAKey));
+            variances.put(fieldBKey, varB / (count - 1));
+            final double stdB = Math.sqrt(variances.get(fieldBKey));
+            skewness.put(fieldAKey, skewA / ((count - 1) * variances.get(fieldAKey) * stdA));
+            skewness.put(fieldBKey, skewB / ((count - 1) * variances.get(fieldBKey) * stdB));
+            kurtosis.put(fieldAKey, kurtA / ((count - 1) * variances.get(fieldAKey) * variances.get(fieldAKey)));
+            kurtosis.put(fieldBKey, kurtB / ((count - 1) * variances.get(fieldBKey) * variances.get(fieldBKey)));
+
+            // compute covariance
+            final HashMap<String, Double> fieldACovar = new HashMap<>(2);
+            fieldACovar.put(fieldAKey, 1d);
+            cVar /= count - 1;
+            fieldACovar.put(fieldBKey, cVar);
+            covariances.put(fieldAKey, fieldACovar);
+            final HashMap<String, Double> fieldBCovar = new HashMap<>(2);
+            fieldBCovar.put(fieldAKey, cVar);
+            fieldBCovar.put(fieldBKey, 1d);
+            covariances.put(fieldBKey, fieldBCovar);
+
+            // compute correlation
+            final HashMap<String, Double> fieldACorr = new HashMap<>();
+            fieldACorr.put(fieldAKey, 1d);
+            double corr = covariances.get(fieldAKey).get(fieldBKey);
+            corr /= stdA * stdB;
+            fieldACorr.put(fieldBKey, corr);
+            correlations.put(fieldAKey, fieldACorr);
+            final HashMap<String, Double> fieldBCorr = new HashMap<>();
+            fieldBCorr.put(fieldAKey, corr);
+            fieldBCorr.put(fieldBKey, 1d);
+            correlations.put(fieldBKey, fieldBCorr);
+        }
+
+        public void assertNearlyEqual(MatrixStatsResults stats) {
+            assertThat(count, equalTo(stats.getDocCount()));
+            assertThat(count, equalTo(stats.getFieldCount(fieldAKey)));
+            assertThat(count, equalTo(stats.getFieldCount(fieldBKey)));
+            // means
+            assertTrue(nearlyEqual(means.get(fieldAKey), stats.getMean(fieldAKey), 1e-7));
+            assertTrue(nearlyEqual(means.get(fieldBKey), stats.getMean(fieldBKey), 1e-7));
+            // variances
+            assertTrue(nearlyEqual(variances.get(fieldAKey), stats.getVariance(fieldAKey), 1e-7));
+            assertTrue(nearlyEqual(variances.get(fieldBKey), stats.getVariance(fieldBKey), 1e-7));
+            // skewness (multi-pass is more susceptible to round-off error so we need to slightly relax the tolerance)
+            assertTrue(nearlyEqual(skewness.get(fieldAKey), stats.getSkewness(fieldAKey), 1e-4));
+            assertTrue(nearlyEqual(skewness.get(fieldBKey), stats.getSkewness(fieldBKey), 1e-4));
+            // kurtosis (multi-pass is more susceptible to round-off error so we need to slightly relax the tolerance)
+            assertTrue(nearlyEqual(kurtosis.get(fieldAKey), stats.getKurtosis(fieldAKey), 1e-4));
+            assertTrue(nearlyEqual(kurtosis.get(fieldBKey), stats.getKurtosis(fieldBKey), 1e-4));
+            // covariances
+            assertTrue(nearlyEqual(covariances.get(fieldAKey).get(fieldBKey), stats.getCovariance(fieldAKey, fieldBKey), 1e-7));
+            assertTrue(nearlyEqual(covariances.get(fieldBKey).get(fieldAKey), stats.getCovariance(fieldBKey, fieldAKey), 1e-7));
+            // correlation
+            assertTrue(nearlyEqual(correlations.get(fieldAKey).get(fieldBKey), stats.getCorrelation(fieldAKey, fieldBKey), 1e-7));
+            assertTrue(nearlyEqual(correlations.get(fieldBKey).get(fieldAKey), stats.getCorrelation(fieldBKey, fieldAKey), 1e-7));
+        }
+    }
+
+    private static boolean nearlyEqual(double a, double b, double epsilon) {
+        final double absA = Math.abs(a);
+        final double absB = Math.abs(b);
+        final double diff = Math.abs(a - b);
+
+        if (a == b) { // shortcut, handles infinities
+            return true;
+        } else if (a == 0 || b == 0 || diff < Double.MIN_NORMAL) {
+            // a or b is zero or both are extremely close to it
+            // relative error is less meaningful here
+            return diff < (epsilon * Double.MIN_NORMAL);
+        } else { // use relative error
+            return diff / Math.min((absA + absB), Double.MAX_VALUE) < epsilon;
+        }
+    }
+}
diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStatsTests.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStatsTests.java
new file mode 100644
index 00000000000..2992fd57a7e
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStatsTests.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.matrix.stats;
+
+import java.util.HashMap;
+import java.util.List;
+
+/**
+ *
+ */
+public class RunningStatsTests extends MatrixStatsTestCase {
+
+    /** test running stats */
+    public void testRunningStats() throws Exception {
+        final MatrixStatsResults results = new MatrixStatsResults(createRunningStats(fieldA, fieldB));
+        actualStats.assertNearlyEqual(results);
+    }
+
+    /** Test merging stats across observation shards */
+    public void testMergedStats() throws Exception {
+        // slice observations into shards
+        int numShards = randomIntBetween(2, 10);
+        double obsPerShard = Math.floor(numObs / numShards);
+        int start = 0;
+        RunningStats stats = null;
+        List<Double> fieldAShard, fieldBShard;
+        for (int s = 0; s < numShards-1; start = ++s * (int)obsPerShard) {
+            fieldAShard = fieldA.subList(start, start + (int)obsPerShard);
+            fieldBShard = fieldB.subList(start, start + (int)obsPerShard);
+            if (stats == null) {
+                stats = createRunningStats(fieldAShard, fieldBShard);
+            } else {
+                stats.merge(createRunningStats(fieldAShard, fieldBShard));
+            }
+        }
+        stats.merge(createRunningStats(fieldA.subList(start, fieldA.size()), fieldB.subList(start, fieldB.size())));
+
+        final MatrixStatsResults results = new MatrixStatsResults(stats);
+        actualStats.assertNearlyEqual(results);
+    }
+
+    private RunningStats createRunningStats(List<Double> fieldAObs, List<Double> fieldBObs) {
+        RunningStats stats = RunningStats.EMPTY();
+        // create a document with two numeric fields
+        final HashMap<String, Double> doc = new HashMap<>(2);
+
+        // running stats computation
+        for (int n = 0; n < fieldAObs.size(); ++n) {
+            doc.put(fieldAKey, fieldAObs.get(n));
+            doc.put(fieldBKey, fieldBObs.get(n));
+            stats.add(doc);
+        }
+        return stats;
+    }
+
+}
diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/10_basic.yaml b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/10_basic.yaml
new file mode 100644
index 00000000000..5ca9a323387
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/10_basic.yaml
@@ -0,0 +1,13 @@
+# Integration tests for Matrix Aggs Plugin
+#
+"Matrix stats aggs loaded":
+    - do:
+        cluster.state: {}
+
+    # Get master node id
+    - set: { master_node: master }
+
+    - do:
+        nodes.info: {}
+
+    - match:  { nodes.$master.modules.0.name: aggs-matrix-stats }
diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/20_empty_bucket.yaml b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/20_empty_bucket.yaml
new file mode 100644
index 00000000000..d91c53e9e42
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/20_empty_bucket.yaml
@@ -0,0 +1,48 @@
+---
+"Empty Bucket Aggregation":
+  - do:
+    indices.create:
+      index: empty_bucket_idx
+      body:
+        settings:
+          number_of_shards: "3"
+        mappings:
+          test:
+            "properties":
+              "value":
+                "type": "integer"
+              "val1":
+                "type": "double"
+
+  - do:
+    index:
+      index:  empty_bucket_idx
+      type:   test
+      id:     1
+      body:   { "value": 0, "val1": 3.1 }
+
+  - do:
+    index:
+      index:  empty_bucket_idx
+      type:   test
+      id:     2
+      body:   { "value": 2, "val1": -3.1 }
+
+  - do:
+    indices.refresh:
+      index: [empty_bucket_idx]
+
+  - do:
+    search:
+      index: empty_bucket_idx
+      type: test
+
+  - match: {hits.total: 2}
+
+  - do:
+      search:
+        index: empty_bucket_idx
+        type:  test
+        body: {"aggs": {"histo": {"histogram": {"field": "val1", "interval": 1, "min_doc_count": 0}, "aggs": { "mfs" : { "matrix_stats": {"field": ["value", "val1"]} } } } } }
+
+  - match: {hits.total: 2}
diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yaml b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yaml
new file mode 100644
index 00000000000..0fe3590af48
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yaml
@@ -0,0 +1,184 @@
+---
+setup:
+
+  - do:
+    indices.create:
+      index: test
+      body:
+        settings:
+          number_of_shards: 3
+        mappings:
+          test:
+            "properties":
+              "val1":
+                "type": "double"
+              "val2":
+                "type": "double"
+              "val3":
+                "type": "double"
+
+  - do:
+    indices.create:
+      index: unmapped
+      body:
+        settings:
+          number_of_shards: 3
+
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     1
+      body:   { "val1": 1.9, "val2": 3.1, "val3": 2.3 }
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     2
+      body:   { "val1": -5.2, "val2": -3.4, "val3": 2.3}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     3
+      body:   { "val1": -5.2, "val3": 2.3}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     4
+      body:   { "val1": 18.3, "val2": 104.4, "val3": 2.3}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     5
+      body:   { "val1": -53.2, "val2": -322.4, "val3": 2.3}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     6
+      body:   { "val1": -578.9, "val2": 69.9, "val3": 2.3}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     7
+      body:   { "val1": 16.2, "val2": 17.2, "val3": 2.3}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     8
+      body:   { "val1": -4222.63, "val2": 316.44, "val3": 2.3}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     9
+      body:   { "val1": -59999.55, "val2": -3163.4, "val3": 2.3}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     10
+      body:   { "val1": 782.7, "val2": 789.7, "val3": 2.3}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     11
+      body:   { "val1": -1.2, "val2": 6.3, "val3": 2.3}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     12
+      body:   { "val1": 0, "val2": 1.11, "val3": 2.3}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     13
+      body:   { "val1": 0.1, "val2": 0.92, "val3": 2.3}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     14
+      body:   { "val1": 0.12, "val2": -82.4, "val3": 2.3}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     15
+      body:   { "val1": 98.2, "val2": 32.4, "val3": 2.3}
+
+  - do:
+    indices.refresh:
+      index: [test, unmapped]
+
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+
+---
+"Unmapped":
+
+  - do:
+      search:
+        index: unmapped
+        type:  test
+        body: {"aggs": { "mfs" : { "matrix_stats": {"field": ["val1", "val2", "val3"]} } } }
+
+  - match: {hits.total: 0}
+
+---
+"Single value field":
+
+  - do:
+      search:
+        index: test
+        type:  test
+        body: {"aggs": { "mfs" : { "matrix_stats": {"field": ["val1", "val3"]} } } }
+
+  - match: {hits.total: 15}
+  - match: {aggregations.mfs.count.0: 15}
+
+---
+"Partially unmapped":
+
+  - do:
+      search:
+        index: [test, unmapped]
+        type:  test
+        body: {"aggs": { "mfs" : { "matrix_stats": {"field": ["val1", "val2", "val3"]} } } }
+
+  - match: {hits.total: 15}
+  - match: {aggregations.mfs.count.0: 14}
+
+---
+"With script":
+
+  - do:
+      search:
+        index: test
+        type:  test
+        body: {"aggs": { "mfs" : { "matrix_stats": {"field": ["val1", "val2"], "script" : { "my_script" : {"inline" : "1 + doc['val1'].value", "lang" : "js"} } } } } }
+
+  - match: {hits.total: 15}
+  - match: {aggregations.mfs.count.0: 14}
+
+---
+"With script params":
+
+  - do:
+      search:
+        index: test
+        type:  test
+        body: {"aggs": { "mfs" : { "matrix_stats": {"field": ["val1", "val2", "val3"], "script" : { "my_script" : {"inline" : "my_var + doc['val1'].value", "params" : { "my_var" : 1 }, "lang" : "js" } } } } } }
+
+  - match: {hits.total: 15}
+  - match: {aggregations.mfs.count.0: 14}
+  - match: {aggregations.mfs.correlation.1.2: 0.9569513137793205}
diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yaml b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yaml
new file mode 100644
index 00000000000..87b5dea19fe
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yaml
@@ -0,0 +1,184 @@
+---
+setup:
+
+  - do:
+    indices.create:
+      index: test
+      body:
+        settings:
+          number_of_shards: 3
+        mappings:
+          test:
+            "properties":
+              "val1":
+                "type": "double"
+              "val2":
+                "type": "double"
+              "val3":
+                "type": "double"
+
+  - do:
+    indices.create:
+      index: unmapped
+      body:
+        settings:
+          number_of_shards: 3
+
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     1
+      body:   { "val1": 1.9, "val2": 3.1, "val3": 2.3, "vals" : [1.9, 16.143] }
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     2
+      body:   { "val1": -5.2, "val2": -3.4, "val3": 2.3, "vals" : [155, 16.23]}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     3
+      body:   { "val1": -5.2, "val3": 2.3, "vals" : [-455, -32.32]}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     4
+      body:   { "val1": 18.3, "val2": 104.4, "val3": 2.3, "vals" : [0.14, 92.1]}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     5
+      body:   { "val1": -53.2, "val2": -322.4, "val3": 2.3, "vals" : [16, 16]}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     6
+      body:   { "val1": -578.9, "val2": 69.9, "val3": 2.3}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     7
+      body:   { "val1": 16.2, "val2": 17.2, "val3": 2.3, "vals" : [1234.3, -3433]}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     8
+      body:   { "val1": -4222.63, "val2": 316.44, "val3": 2.3, "vals" : [177.2, -93.333]}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     9
+      body:   { "val1": -59999.55, "val2": -3163.4, "val3": 2.3, "vals" : [-29.9, 163.0]}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     10
+      body:   { "val1": 782.7, "val2": 789.7, "val3": 2.3, "vals" : [-0.2, 1343.3]}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     11
+      body:   { "val1": -1.2, "val2": 6.3, "val3": 2.3, "vals" : [15.3, 16.9]}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     12
+      body:   { "val1": 0, "val2": 1.11, "val3": 2.3, "vals" : [-644.4, -644.4]}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     13
+      body:   { "val1": 0.1, "val2": 0.92, "val3": 2.3, "vals" : [73.2, 0.12]}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     14
+      body:   { "val1": 0.12, "val2": -82.4, "val3": 2.3, "vals" : [-0.001, 1295.3]}
+  - do:
+    index:
+      index:  test
+      type:   test
+      id:     15
+      body:   { "val1": 98.2, "val2": 32.4, "val3": 2.3, "vals" : [15.5, 16.5]}
+
+  - do:
+    indices.refresh:
+      index: [test, unmapped]
+
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+
+---
+"Unmapped":
+
+  - do:
+      search:
+        index: unmapped
+        type:  test
+        body: {"aggs": { "mfs" : { "matrix_stats": {"field": ["val1", "val2", "vals"]} } } }
+
+  - match: {hits.total: 0}
+
+---
+"Multi value field":
+
+  - do:
+      search:
+        index: test
+        type:  test
+        body: {"aggs": { "mfs" : { "matrix_stats": {"field": ["val1", "val3"]} } } }
+
+  - match: {hits.total: 15}
+  - match: {aggregations.mfs.count.0: 15}
+
+---
+"Partially unmapped":
+
+  - do:
+      search:
+        index: [test, unmapped]
+        type:  test
+        body: {"aggs": { "mfs" : { "matrix_stats": {"field": ["val1", "val2", "vals"]} } } }
+
+  - match: {hits.total: 15}
+  - match: {aggregations.mfs.count.0: 13}
+
+---
+"With script":
+
+  - do:
+      search:
+        index: test
+        type:  test
+        body: {"aggs": { "mfs" : { "matrix_stats": {"field": ["vals", "val3"], "script" : { "my_script" : {"inline" : "1 + doc['val1'].value", "lang" : "js"} } } } } }
+
+  - match: {hits.total: 15}
+  - match: {aggregations.mfs.count.0: 14}
+
+---
+"With script params":
+
+  - do:
+      search:
+        index: test
+        type:  test
+        body: {"aggs": { "mfs" : { "matrix_stats": {"field": ["val1", "val3", "vals"], "script" : { "my_script" : {"inline" : "my_var + doc['val1'].value", "params" : { "my_var" : 1 }, "lang" : "js" } } } } } }
+
+  - match: {hits.total: 15}
+  - match: {aggregations.mfs.count.0: 14}
+  - match: {aggregations.mfs.correlation.1.2: -0.055971032866899535}
diff --git a/settings.gradle b/settings.gradle
index 86673589e10..215b436e20a 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -14,6 +14,7 @@ List projects = [
   'test:fixtures:example-fixture',
   'test:fixtures:hdfs-fixture',
   'test:logger-usage',
+  'modules:aggs-matrix-stats',
   'modules:ingest-grok',
   'modules:lang-expression',
   'modules:lang-groovy',

From 90b8f5d0d8ab316a88ae17a60f04aa12187cd671 Mon Sep 17 00:00:00 2001
From: Nicholas Knize <nknize@gmail.com>
Date: Fri, 13 May 2016 13:37:05 -0500
Subject: [PATCH 0558/1311] Adding MultiValuesSource support classes and
 documentation to matrix stats agg module

---
 .../org/elasticsearch/common/ParseField.java  |   8 +
 .../aggregations/AggregationBuilder.java      |   5 +
 .../aggregations/InternalAggregation.java     |   4 +-
 .../support/ValuesSourceConfig.java           |  15 +
 docs/reference/modules.asciidoc               |   6 +
 .../modules/aggregations-matrix.asciidoc      |   9 +
 .../aggregations/matrix/stats.asciidoc        | 112 ++++++
 modules/aggs-matrix-stats/build.gradle        |  12 -
 .../MatrixStatsAggregationBuilders.java       |   6 +-
 .../matrix/MatrixAggregationPlugin.java       |  10 +-
 .../matrix/stats/InternalMatrixStats.java     | 148 ++++---
 .../matrix/stats/MatrixStats.java             |  19 +-
 ...ava => MatrixStatsAggregationBuilder.java} |  27 +-
 .../matrix/stats/MatrixStatsAggregator.java   |  72 ++--
 .../stats/MatrixStatsAggregatorFactory.java   |  14 +-
 .../matrix/stats/MatrixStatsParser.java       |  22 +-
 .../matrix/stats/MatrixStatsResults.java      | 111 +++---
 .../matrix/stats/RunningStats.java            | 208 ++++------
 .../support/MultiValuesSource.java            |  90 +++++
 .../MultiValuesSourceAggregationBuilder.java  | 366 ++++++++++++++++++
 .../MultiValuesSourceAggregatorFactory.java   |  68 ++++
 .../support/MultiValuesSourceParser.java      | 239 ++++++++++++
 ...Case.java => BaseMatrixStatsTestCase.java} |   2 +-
 .../matrix/stats/RunningStatsTests.java       |  16 +-
 .../test/stats/20_empty_bucket.yaml           |   2 +-
 .../test/stats/30_single_value_field.yaml     |  37 +-
 .../test/stats/40_multi_value_field.yaml      |  53 ++-
 27 files changed, 1283 insertions(+), 398 deletions(-)
 create mode 100644 docs/reference/modules/aggregations-matrix.asciidoc
 create mode 100644 docs/reference/modules/aggregations/matrix/stats.asciidoc
 rename modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/{MatrixStatsAggregatorBuilder.java => MatrixStatsAggregationBuilder.java} (75%)
 create mode 100644 modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSource.java
 create mode 100644 modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
 create mode 100644 modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java
 create mode 100644 modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParser.java
 rename modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/{MatrixStatsTestCase.java => BaseMatrixStatsTestCase.java} (99%)

diff --git a/core/src/main/java/org/elasticsearch/common/ParseField.java b/core/src/main/java/org/elasticsearch/common/ParseField.java
index 6516f7952a7..c04bcb14dcb 100644
--- a/core/src/main/java/org/elasticsearch/common/ParseField.java
+++ b/core/src/main/java/org/elasticsearch/common/ParseField.java
@@ -107,4 +107,12 @@ public class ParseField {
     public String[] getDeprecatedNames() {
         return deprecatedNames;
     }
+
+    public static class CommonFields {
+        public static final ParseField FIELD = new ParseField("field");
+        public static final ParseField FIELDS = new ParseField("fields");
+        public static final ParseField FORMAT = new ParseField("format");
+        public static final ParseField MISSING = new ParseField("missing");
+        public static final ParseField TIME_ZONE = new ParseField("time_zone");
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
index 1a16f653159..d718227aae8 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
@@ -20,6 +20,7 @@ package org.elasticsearch.search.aggregations;
 
 
 import org.elasticsearch.action.support.ToXContentToBytes;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.search.aggregations.InternalAggregation.Type;
@@ -84,4 +85,8 @@ public abstract class AggregationBuilder
      */
     protected abstract AggregationBuilder subAggregations(AggregatorFactories.Builder subFactories);
 
+    /** Common xcontent fields shared among aggregator builders */
+    public static final class CommonFields extends ParseField.CommonFields {
+        public static final ParseField VALUE_TYPE = new ParseField("value_type");
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
index 5ae78417f6f..b4fab7093f8 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
@@ -20,6 +20,7 @@ package org.elasticsearch.search.aggregations;
 
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -250,7 +251,8 @@ public abstract class InternalAggregation implements Aggregation, ToXContent, St
     /**
      * Common xcontent fields that are shared among addAggregation
      */
-    public static final class CommonFields {
+    public static final class CommonFields extends ParseField.CommonFields {
+        // todo convert these to ParseField
         public static final String META = "meta";
         public static final String BUCKETS = "buckets";
         public static final String VALUE = "value";
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
index 2c5e6f2f335..acc2b063ac2 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
@@ -76,6 +76,21 @@ public class ValuesSourceConfig<VS extends ValuesSource> {
         return this;
     }
 
+    public ValuesSourceConfig<VS> format(final DocValueFormat format) {
+        this.format = format;
+        return this;
+    }
+
+    public ValuesSourceConfig<VS> missing(final Object missing) {
+        this.missing = missing;
+        return this;
+    }
+
+    public ValuesSourceConfig<VS> timezone(final DateTimeZone timeZone) {
+        this.timeZone= timeZone;
+        return this;
+    }
+
     public DocValueFormat format() {
         return format;
     }
diff --git a/docs/reference/modules.asciidoc b/docs/reference/modules.asciidoc
index 5a39cdfd790..01b0edd0676 100644
--- a/docs/reference/modules.asciidoc
+++ b/docs/reference/modules.asciidoc
@@ -18,6 +18,10 @@ These settings can be dynamically updated on a live cluster with the
 
 The modules in this section are:
 
+<<modules-aggregations-matrix,Matrix Aggregations>>::
+
+    A family of aggregations that operate on multiple document fields and produce a matrix as output.
+
 <<modules-cluster,Cluster-level routing and shard allocation>>::
 
     Settings to control where, when, and how shards are allocated to nodes.
@@ -80,6 +84,8 @@ The modules in this section are:
 --
 
 
+include::modules/aggregations-matrix.asciidoc[]
+
 include::modules/cluster.asciidoc[]
 
 include::modules/discovery.asciidoc[]
diff --git a/docs/reference/modules/aggregations-matrix.asciidoc b/docs/reference/modules/aggregations-matrix.asciidoc
new file mode 100644
index 00000000000..e8f741e7df6
--- /dev/null
+++ b/docs/reference/modules/aggregations-matrix.asciidoc
@@ -0,0 +1,9 @@
+[[modules-aggregations-matrix]]
+== Matrix Aggregations
+
+experimental[]
+
+The aggregations in this family operate on multiple fields and produce a matrix result based on the values extracted from
+the requested document fields. Unlike metric and bucket aggregations, this aggregation family does not yet support scripting.
+
+include::aggregations/matrix/stats.asciidoc[]
diff --git a/docs/reference/modules/aggregations/matrix/stats.asciidoc b/docs/reference/modules/aggregations/matrix/stats.asciidoc
new file mode 100644
index 00000000000..2649ecd5fe7
--- /dev/null
+++ b/docs/reference/modules/aggregations/matrix/stats.asciidoc
@@ -0,0 +1,112 @@
+[[modules-matrix-aggregations-stats]]
+=== Matrix Stats
+
+The `matrix_stats` aggregation is a numeric aggregation that computes the following statistics over a set of document fields:
+
+[horizontal]
+`count`:: Number of per field samples included in the calculation.
+`mean`:: The average value for each field.
+`variance`:: Per field Measurement for how spread out the samples are from the mean.
+`skewness`:: Per field measurement quantifying the asymmetric distribution around the mean.
+`kurtosis`:: Per field measurement quantifying the shape of the distribution.
+`covariance`:: A matrix that quantitatively describes how changes in one field are associated with another.
+`correlation`:: The covariance matrix scaled to a range of -1 to 1, inclusive. Describes the relationship between field
+            distributions.
+
+The following example demonstrates the use of matrix stats to describe the relationship between income and poverty.
+
+[source,js]
+--------------------------------------------------
+{
+    "aggs": {
+        "matrixstats": {
+            "matrix_stats": {
+                "fields": ["poverty", "income"]
+            }
+        }
+    }
+}
+--------------------------------------------------
+
+The aggregation type is `matrix_stats` and the `fields` setting defines the set of fields (as an array) for computing
+the statistics. The above request returns the following response:
+
+[source,js]
+--------------------------------------------------
+{
+    ...
+    "aggregations": {
+        "matrixstats": {
+            "fields": [{
+                "name": "income",
+                "count": 50,
+                "mean": 51985.1,
+                "variance": 7.383377037755103E7,
+                "skewness": 0.5595114003506483,
+                "kurtosis": 2.5692365287787124,
+                "covariance": {
+                    "income": 7.383377037755103E7,
+                    "poverty": -21093.65836734694
+                },
+                "correlation": {
+                    "income": 1.0,
+                    "poverty": -0.8352655256272504
+                }
+            }, {
+                "name": "poverty",
+                "count": 50,
+                "mean": 12.732000000000001,
+                "variance": 8.637730612244896,
+                "skewness": 0.4516049811903419,
+                "kurtosis": 2.8615929677997767,
+                "covariance": {
+                    "income": -21093.65836734694,
+                    "poverty": 8.637730612244896
+                },
+                "correlation": {
+                    "income": -0.8352655256272504,
+                    "poverty": 1.0
+                }
+            }]
+        }
+    }
+}
+--------------------------------------------------
+
+==== Multi Value Fields
+
+The `matrix_stats` aggregation treats each document field as an independent sample. The `mode` parameter controls what
+array value the aggregation will use for array or multi-valued fields. This parameter can take one of the following:
+
+[horizontal]
+`avg`:: (default) Use the average of all values.
+`min`:: Pick the lowest value.
+`max`:: Pick the highest value.
+`sum`:: Use the sum of all values.
+`median`:: Use the median of all values.
+
+==== Missing Values
+
+The `missing` parameter defines how documents that are missing a value should be treated.
+By default they will be ignored but it is also possible to treat them as if they had a value.
+This is done by adding a set of fieldname : value mappings to specify default values per field.
+
+[source,js]
+--------------------------------------------------
+{
+    "aggs": {
+        "matrixstats": {
+            "matrix_stats": {
+                "fields": ["poverty", "income"],
+                "missing": {"income" : 50000} <1>
+            }
+        }
+    }
+}
+--------------------------------------------------
+
+<1> Documents without a value in the `income` field will have the default value `50000`.
+
+==== Script
+
+This aggregation family does not yet support scripting.
diff --git a/modules/aggs-matrix-stats/build.gradle b/modules/aggs-matrix-stats/build.gradle
index 78938837806..b3060fa1786 100644
--- a/modules/aggs-matrix-stats/build.gradle
+++ b/modules/aggs-matrix-stats/build.gradle
@@ -21,15 +21,3 @@ esplugin {
     description 'Adds aggregations whose input are a list of numeric fields and output includes a matrix.'
     classname 'org.elasticsearch.search.aggregations.matrix.MatrixAggregationPlugin'
 }
-
-dependencies {
-    testCompile project(path: ':plugins:lang-javascript', configuration: 'runtime')
-}
-
-integTest {
-    cluster {
-        plugin 'lang-javascript', project(':plugins:lang-javascript')
-        setting 'script.inline', 'true'
-        setting 'script.stored', 'true'
-    }
-}
\ No newline at end of file
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/MatrixStatsAggregationBuilders.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/MatrixStatsAggregationBuilders.java
index 321f4fbceae..10758979ed9 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/MatrixStatsAggregationBuilders.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/MatrixStatsAggregationBuilders.java
@@ -19,7 +19,7 @@
 package org.elasticsearch.search.aggregations;
 
 import org.elasticsearch.search.aggregations.matrix.stats.MatrixStats;
-import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsAggregationBuilder;
 
 /**
  */
@@ -27,7 +27,7 @@ public class MatrixStatsAggregationBuilders {
     /**
      * Create a new {@link MatrixStats} aggregation with the given name.
      */
-    public static MatrixStatsAggregatorBuilder matrixStats(String name) {
-        return new MatrixStatsAggregatorBuilder(name);
+    public static MatrixStatsAggregationBuilder matrixStats(String name) {
+        return new MatrixStatsAggregationBuilder(name);
     }
 }
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/MatrixAggregationPlugin.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/MatrixAggregationPlugin.java
index c1397fa410e..156ff05ad65 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/MatrixAggregationPlugin.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/MatrixAggregationPlugin.java
@@ -22,15 +22,12 @@ package org.elasticsearch.search.aggregations.matrix;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.aggregations.matrix.stats.InternalMatrixStats;
-import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsAggregatorBuilder;
+import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsAggregationBuilder;
 import org.elasticsearch.search.aggregations.matrix.stats.MatrixStatsParser;
 
 import java.io.IOException;
 
 public class MatrixAggregationPlugin extends Plugin {
-    static {
-        InternalMatrixStats.registerStreams();
-    }
 
     public MatrixAggregationPlugin() throws IOException {
     }
@@ -46,7 +43,8 @@ public class MatrixAggregationPlugin extends Plugin {
     }
 
     public void onModule(SearchModule searchModule) {
-        searchModule.registerAggregation(MatrixStatsAggregatorBuilder::new, new MatrixStatsParser(),
-            MatrixStatsAggregatorBuilder.AGGREGATION_NAME_FIELD);
+        InternalMatrixStats.registerStreams();
+        searchModule.registerAggregation(MatrixStatsAggregationBuilder::new, new MatrixStatsParser(),
+            MatrixStatsAggregationBuilder.AGGREGATION_NAME_FIELD);
     }
 }
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java
index da7683b4b3d..edef75389c8 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java
@@ -27,7 +27,9 @@ import org.elasticsearch.search.aggregations.metrics.InternalMetricsAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
 
 /**
  * Computes distribution statistics over multiple fields
@@ -70,11 +72,13 @@ public class InternalMatrixStats extends InternalMetricsAggregation implements M
         return TYPE;
     }
 
+    /** get the number of documents */
     @Override
     public long getDocCount() {
         return stats.docCount;
     }
 
+    /** get the number of samples for the given field. == docCount - numMissing */
     @Override
     public long getFieldCount(String field) {
         if (results == null) {
@@ -83,65 +87,63 @@ public class InternalMatrixStats extends InternalMetricsAggregation implements M
         return results.getFieldCount(field);
     }
 
+    /** get the mean for the given field */
     @Override
-    public Double getMean(String field) {
+    public double getMean(String field) {
         if (results == null) {
-            return null;
+            return Double.NaN;
         }
         return results.getMean(field);
     }
 
+    /** get the variance for the given field */
     @Override
-    public Double getVariance(String field) {
+    public double getVariance(String field) {
         if (results == null) {
-            return null;
+            return Double.NaN;
         }
         return results.getVariance(field);
     }
 
+    /** get the distribution skewness for the given field */
     @Override
-    public Double getSkewness(String field) {
+    public double getSkewness(String field) {
         if (results == null) {
-            return null;
+            return Double.NaN;
         }
         return results.getSkewness(field);
     }
 
+    /** get the distribution shape for the given field */
     @Override
-    public Double getKurtosis(String field) {
+    public double getKurtosis(String field) {
         if (results == null) {
-            return null;
+            return Double.NaN;
         }
         return results.getKurtosis(field);
     }
 
+    /** get the covariance between the two fields */
     @Override
-    public Double getCovariance(String fieldX, String fieldY) {
+    public double getCovariance(String fieldX, String fieldY) {
         if (results == null) {
-            return null;
+            return Double.NaN;
         }
         return results.getCovariance(fieldX, fieldY);
     }
 
+    /** get the correlation between the two fields */
     @Override
-    public Map<String, HashMap<String, Double>> getCovariance() {
-        return results.getCovariances();
-    }
-
-    @Override
-    public Double getCorrelation(String fieldX, String fieldY) {
+    public double getCorrelation(String fieldX, String fieldY) {
         if (results == null) {
-            return null;
+            return Double.NaN;
         }
         return results.getCorrelation(fieldX, fieldY);
     }
 
-    @Override
-    public Map<String, HashMap<String, Double>> getCorrelation() {
-        return results.getCorrelations();
-    }
-
     static class Fields {
+        public static final String FIELDS = "fields";
+        public static final String NAME = "name";
         public static final String COUNT = "count";
         public static final String MEAN = "mean";
         public static final String VARIANCE = "variance";
@@ -153,30 +155,38 @@ public class InternalMatrixStats extends InternalMetricsAggregation implements M
 
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
-        if (results != null) {
-            Set<String> fieldNames = results.getFieldCounts().keySet();
-            builder.field("field", fieldNames);
-            builder.field(Fields.COUNT, results.getFieldCounts().values());
-            builder.field(Fields.MEAN, results.getMeans().values());
-            builder.field(Fields.VARIANCE, results.getVariances().values());
-            builder.field(Fields.SKEWNESS, results.getSkewness().values());
-            builder.field(Fields.KURTOSIS, results.getKurtosis().values());
-            ArrayList<ArrayList<Double>> cov = new ArrayList<>(fieldNames.size());
-            ArrayList<ArrayList<Double>> cor = new ArrayList<>(fieldNames.size());
-            for (String y : fieldNames) {
-                ArrayList<Double> covRow = new ArrayList<>(fieldNames.size());
-                ArrayList<Double> corRow = new ArrayList<>(fieldNames.size());
-                for (String x : fieldNames) {
-                    covRow.add(results.getCovariance(x, y));
-                    corRow.add(results.getCorrelation(x, y));
+        if (results != null && results.getFieldCounts().keySet().isEmpty() == false) {
+            builder.startArray(Fields.FIELDS);
+            for (String fieldName : results.getFieldCounts().keySet()) {
+                builder.startObject();
+                // name
+                builder.field(Fields.NAME, fieldName);
+                // count
+                builder.field(Fields.COUNT, results.getFieldCount(fieldName));
+                // mean
+                builder.field(Fields.MEAN, results.getMean(fieldName));
+                // variance
+                builder.field(Fields.VARIANCE, results.getVariance(fieldName));
+                // skewness
+                builder.field(Fields.SKEWNESS, results.getSkewness(fieldName));
+                // kurtosis
+                builder.field(Fields.KURTOSIS, results.getKurtosis(fieldName));
+                // covariance
+                builder.startObject(Fields.COVARIANCE);
+                for (String fieldB : results.getFieldCounts().keySet()) {
+                    builder.field(fieldB, results.getCovariance(fieldName, fieldB));
                 }
-                cov.add(covRow);
-                cor.add(corRow);
+                builder.endObject();
+                // correlation
+                builder.startObject(Fields.CORRELATION);
+                for (String fieldB : results.getFieldCounts().keySet()) {
+                    builder.field(fieldB, results.getCorrelation(fieldName, fieldB));
+                }
+                builder.endObject();
+                builder.endObject();
             }
-            builder.field(Fields.COVARIANCE, cov);
-            builder.field(Fields.CORRELATION, cor);
+            builder.endArray();
         }
-
         return builder;
     }
 
@@ -185,11 +195,11 @@ public class InternalMatrixStats extends InternalMetricsAggregation implements M
         if (path.isEmpty()) {
             return this;
         } else if (path.size() == 1) {
-            String coordinate = path.get(0);
+            String element = path.get(0);
             if (results == null) {
-                results = MatrixStatsResults.EMPTY();
+                results = new MatrixStatsResults();
             }
-            switch (coordinate) {
+            switch (element) {
                 case "counts":
                     return results.getFieldCounts();
                 case "means":
@@ -205,7 +215,7 @@ public class InternalMatrixStats extends InternalMetricsAggregation implements M
                 case "correlation":
                     return results.getCorrelations();
                 default:
-                    throw new IllegalArgumentException("Found unknown path element [" + coordinate + "] in [" + getName() + "]");
+                    throw new IllegalArgumentException("Found unknown path element [" + element + "] in [" + getName() + "]");
             }
         } else {
             throw new IllegalArgumentException("path not supported for [" + getName() + "]: " + path);
@@ -215,53 +225,35 @@ public class InternalMatrixStats extends InternalMetricsAggregation implements M
     @Override
     protected void doWriteTo(StreamOutput out) throws IOException {
         // write running stats
-        if (stats == null || stats.docCount == 0) {
-            out.writeVLong(0);
-        } else {
-            out.writeVLong(stats.docCount);
-            stats.writeTo(out);
-        }
-
+        out.writeOptionalWriteable(stats);
         // write results
-        if (results == null || results.getDocCount() == 0) {
-            out.writeVLong(0);
-        } else {
-            out.writeVLong(results.getDocCount());
-            results.writeTo(out);
-        }
+        out.writeOptionalWriteable(results);
     }
 
     @Override
     protected void doReadFrom(StreamInput in) throws IOException {
         // read stats count
-        final long statsCount = in.readVLong();
-        if (statsCount > 0) {
-            stats = new RunningStats(in);
-            stats.docCount = statsCount;
-        }
-
+        stats = in.readOptionalWriteable(RunningStats::new);
         // read count
-        final long count = in.readVLong();
-        if (count > 0) {
-            results = new MatrixStatsResults(in);
-        }
+        results = in.readOptionalWriteable(MatrixStatsResults::new);
     }
 
     @Override
     public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
         // merge stats across all shards
-        aggregations.removeIf(p -> ((InternalMatrixStats)p).stats == null);
+        List<InternalAggregation> aggs = new ArrayList<>(aggregations);
+        aggs.removeIf(p -> ((InternalMatrixStats)p).stats == null);
 
         // return empty result iff all stats are null
-        if (aggregations.isEmpty()) {
-            return new InternalMatrixStats(name, 0, null, MatrixStatsResults.EMPTY(), pipelineAggregators(), getMetaData());
+        if (aggs.isEmpty()) {
+            return new InternalMatrixStats(name, 0, null, new MatrixStatsResults(), pipelineAggregators(), getMetaData());
         }
 
-        RunningStats runningStats = ((InternalMatrixStats) aggregations.get(0)).stats;
-        for (int i=1; i < aggregations.size(); ++i) {
-            runningStats.merge(((InternalMatrixStats) aggregations.get(i)).stats);
+        RunningStats runningStats = new RunningStats();
+        for (int i=0; i < aggs.size(); ++i) {
+            runningStats.merge(((InternalMatrixStats) aggs.get(i)).stats);
         }
-        MatrixStatsResults results = new MatrixStatsResults(stats);
+        MatrixStatsResults results = new MatrixStatsResults(runningStats);
 
         return new InternalMatrixStats(name, results.getDocCount(), runningStats, results, pipelineAggregators(), getMetaData());
     }
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStats.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStats.java
index 1de20faae93..f7604ff18ec 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStats.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStats.java
@@ -20,9 +20,6 @@ package org.elasticsearch.search.aggregations.matrix.stats;
 
 import org.elasticsearch.search.aggregations.Aggregation;
 
-import java.util.HashMap;
-import java.util.Map;
-
 /**
  * Interface for MatrixStats Metric Aggregation
  */
@@ -32,19 +29,15 @@ public interface MatrixStats extends Aggregation {
     /** return total field count (differs from docCount if there are missing values) */
     long getFieldCount(String field);
     /** return the field mean */
-    Double getMean(String field);
+    double getMean(String field);
     /** return the field variance */
-    Double getVariance(String field);
+    double getVariance(String field);
     /** return the skewness of the distribution */
-    Double getSkewness(String field);
+    double getSkewness(String field);
     /** return the kurtosis of the distribution */
-    Double getKurtosis(String field);
-    /** return the upper triangle of the covariance matrix */
-    Map<String, HashMap<String, Double>> getCovariance();
+    double getKurtosis(String field);
     /** return the covariance between field x and field y */
-    Double getCovariance(String fieldX, String fieldY);
-    /** return the upper triangle of the pearson product-moment correlation matrix */
-    Map<String, HashMap<String, Double>> getCorrelation();
+    double getCovariance(String fieldX, String fieldY);
     /** return the correlation coefficient of field x and field y */
-    Double getCorrelation(String fieldX, String fieldY);
+    double getCorrelation(String fieldX, String fieldY);
 }
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorBuilder.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java
similarity index 75%
rename from modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorBuilder.java
rename to modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java
index ea20a3c32ad..5c6c3767177 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorBuilder.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregationBuilder.java
@@ -23,10 +23,11 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
-import org.elasticsearch.search.aggregations.support.MultiValuesSourceAggregatorBuilder;
+import org.elasticsearch.search.aggregations.support.MultiValuesSourceAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
@@ -38,19 +39,21 @@ import java.util.Map;
 
 /**
  */
-public class MatrixStatsAggregatorBuilder
-    extends MultiValuesSourceAggregatorBuilder.LeafOnly<ValuesSource.Numeric, MatrixStatsAggregatorBuilder> {
+public class MatrixStatsAggregationBuilder
+    extends MultiValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, MatrixStatsAggregationBuilder> {
     public static final String NAME = InternalMatrixStats.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
-    public MatrixStatsAggregatorBuilder(String name) {
+    private MultiValueMode multiValueMode = MultiValueMode.AVG;
+
+    public MatrixStatsAggregationBuilder(String name) {
         super(name, InternalMatrixStats.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
-    public MatrixStatsAggregatorBuilder(StreamInput in) throws IOException {
+    public MatrixStatsAggregationBuilder(StreamInput in) throws IOException {
         super(in, InternalMatrixStats.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
@@ -59,14 +62,24 @@ public class MatrixStatsAggregatorBuilder
         // Do nothing, no extra state to write to stream
     }
 
+    public MatrixStatsAggregationBuilder multiValueMode(MultiValueMode multiValueMode) {
+        this.multiValueMode = multiValueMode;
+        return this;
+    }
+
+    public MultiValueMode multiValueMode() {
+        return this.multiValueMode;
+    }
+
     @Override
     protected MatrixStatsAggregatorFactory innerBuild(AggregationContext context, Map<String, ValuesSourceConfig<Numeric>> configs,
-                                                      AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
-        return new MatrixStatsAggregatorFactory(name, type, configs, context, parent, subFactoriesBuilder, metaData);
+            AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
+        return new MatrixStatsAggregatorFactory(name, type, configs, multiValueMode, context, parent, subFactoriesBuilder, metaData);
     }
 
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        builder.field(MULTIVALUE_MODE_FIELD.getPreferredName(), multiValueMode);
         return builder;
     }
 
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java
index 275163bf103..75f0c37d89f 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregator.java
@@ -22,7 +22,8 @@ import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.ObjectArray;
-import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
+import org.elasticsearch.index.fielddata.NumericDoubleValues;
+import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
@@ -30,11 +31,10 @@ import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.metrics.MetricsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
+import org.elasticsearch.search.aggregations.support.MultiValuesSource.NumericMultiValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -43,89 +43,75 @@ import java.util.Map;
  **/
 public class MatrixStatsAggregator extends MetricsAggregator {
     /** Multiple ValuesSource with field names */
-    final Map<String, ValuesSource.Numeric> valuesSources;
+    final NumericMultiValuesSource valuesSources;
 
     /** array of descriptive stats, per shard, needed to compute the correlation */
     ObjectArray<RunningStats> stats;
 
     public MatrixStatsAggregator(String name, Map<String, ValuesSource.Numeric> valuesSources, AggregationContext context,
-                                 Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+                                 Aggregator parent, MultiValueMode multiValueMode, List<PipelineAggregator> pipelineAggregators,
                                  Map<String,Object> metaData) throws IOException {
         super(name, context, parent, pipelineAggregators, metaData);
-        this.valuesSources = valuesSources;
         if (valuesSources != null && !valuesSources.isEmpty()) {
+            this.valuesSources = new NumericMultiValuesSource(valuesSources, multiValueMode);
             stats = context.bigArrays().newObjectArray(1);
+        } else {
+            this.valuesSources = null;
         }
     }
 
     @Override
     public boolean needsScores() {
-        boolean needsScores = false;
-        if (valuesSources != null) {
-            for (Map.Entry<String, ValuesSource.Numeric> valueSource : valuesSources.entrySet()) {
-                needsScores |= valueSource.getValue().needsScores();
-            }
-        }
-        return needsScores;
+        return (valuesSources == null) ? false : valuesSources.needsScores();
     }
 
     @Override
     public LeafBucketCollector getLeafCollector(LeafReaderContext ctx,
                                                 final LeafBucketCollector sub) throws IOException {
-        if (valuesSources == null || valuesSources.isEmpty()) {
+        if (valuesSources == null) {
             return LeafBucketCollector.NO_OP_COLLECTOR;
         }
         final BigArrays bigArrays = context.bigArrays();
-        final HashMap<String, SortedNumericDoubleValues> values = new HashMap<>(valuesSources.size());
-        for (Map.Entry<String, ValuesSource.Numeric> valuesSource : valuesSources.entrySet()) {
-            values.put(valuesSource.getKey(), valuesSource.getValue().doubleValues(ctx));
+        final NumericDoubleValues[] values = new NumericDoubleValues[valuesSources.fieldNames().length];
+        for (int i = 0; i < values.length; ++i) {
+            values[i] = valuesSources.getField(i, ctx);
         }
 
         return new LeafBucketCollectorBase(sub, values) {
+            final String[] fieldNames = valuesSources.fieldNames();
+            final double[] fieldVals = new double[fieldNames.length];
+
             @Override
             public void collect(int doc, long bucket) throws IOException {
                 // get fields
-                Map<String, Double> fields = getFields(doc);
-                if (fields != null) {
+                if (includeDocument(doc) == true) {
                     stats = bigArrays.grow(stats, bucket + 1);
                     RunningStats stat = stats.get(bucket);
                     // add document fields to correlation stats
                     if (stat == null) {
-                        stat = new RunningStats(fields);
+                        stat = new RunningStats(fieldNames, fieldVals);
+                        stats.set(bucket, stat);
                     } else {
-                        stat.add(fields);
+                        stat.add(fieldNames, fieldVals);
                     }
-                    stats.set(bucket, stat);
                 }
             }
 
             /**
              * return a map of field names and data
              */
-            private Map<String, Double> getFields(int doc) {
-                // get fieldNames to use as hash keys
-                ArrayList<String> fieldNames = new ArrayList<>(values.keySet());
-                HashMap<String, Double> fields = new HashMap<>(fieldNames.size());
-
+            private boolean includeDocument(int doc) {
                 // loop over fields
-                for (String fieldName : fieldNames) {
-                    final SortedNumericDoubleValues doubleValues = values.get(fieldName);
-                    doubleValues.setDocument(doc);
-                    final int valuesCount = doubleValues.count();
-                    // if document contains an empty field we omit the doc from the correlation
-                    if (valuesCount <= 0) {
-                        return null;
+                for (int i = 0; i < fieldVals.length; ++i) {
+                    final NumericDoubleValues doubleValues = values[i];
+                    final double value = doubleValues.get(doc);
+                    // skip if value is missing
+                    if (value == Double.NEGATIVE_INFINITY) {
+                        return false;
                     }
-                    // get the field value (multi-value is the average of all the values)
-                    double fieldValue = 0;
-                    for (int i = 0; i < valuesCount; ++i) {
-                        if (Double.isNaN(doubleValues.valueAt(i)) == false) {
-                            fieldValue += doubleValues.valueAt(i);
-                        }
-                    }
-                    fields.put(fieldName, fieldValue / valuesCount);
+                    fieldVals[i] = value;
                 }
-                return fields;
+                return true;
             }
         };
     }
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java
index 30389c05836..26e7910dcac 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsAggregatorFactory.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.search.aggregations.matrix.stats;
 
+import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
@@ -37,23 +38,26 @@ import java.util.Map;
 public class MatrixStatsAggregatorFactory
     extends MultiValuesSourceAggregatorFactory<ValuesSource.Numeric, MatrixStatsAggregatorFactory> {
 
+    private final MultiValueMode multiValueMode;
+
     public MatrixStatsAggregatorFactory(String name, InternalAggregation.Type type,
-                                        Map<String, ValuesSourceConfig<ValuesSource.Numeric>> configs, AggregationContext context,
-                                        AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
-                                        Map<String, Object> metaData) throws IOException {
+            Map<String, ValuesSourceConfig<ValuesSource.Numeric>> configs, MultiValueMode multiValueMode,
+            AggregationContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+            Map<String, Object> metaData) throws IOException {
         super(name, type, configs, context, parent, subFactoriesBuilder, metaData);
+        this.multiValueMode = multiValueMode;
     }
 
     @Override
     protected Aggregator createUnmapped(Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
         throws IOException {
-        return new MatrixStatsAggregator(name, null, context, parent, pipelineAggregators, metaData);
+        return new MatrixStatsAggregator(name, null, context, parent, multiValueMode, pipelineAggregators, metaData);
     }
 
     @Override
     protected Aggregator doCreateInternal(Map<String, ValuesSource.Numeric> valuesSources, Aggregator parent,
                                           boolean collectsFromSingleBucket, List<PipelineAggregator> pipelineAggregators,
                                           Map<String, Object> metaData) throws IOException {
-        return new MatrixStatsAggregator(name, valuesSources, context, parent, pipelineAggregators, metaData);
+        return new MatrixStatsAggregator(name, valuesSources, context, parent, multiValueMode, pipelineAggregators, metaData);
     }
 }
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsParser.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsParser.java
index 950bd078086..ea383b642c2 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsParser.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsParser.java
@@ -21,6 +21,7 @@ package org.elasticsearch.search.aggregations.matrix.stats;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.search.aggregations.support.MultiValuesSourceParser.NumericValuesSourceParser;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
@@ -28,23 +29,36 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import java.io.IOException;
 import java.util.Map;
 
+import static org.elasticsearch.search.aggregations.support.MultiValuesSourceAggregationBuilder.MULTIVALUE_MODE_FIELD;
+
 /**
  */
 public class MatrixStatsParser extends NumericValuesSourceParser {
 
     public MatrixStatsParser() {
-        super(true, true, false);
+        super(true);
     }
 
     @Override
     protected boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, XContentParser parser,
                             ParseFieldMatcher parseFieldMatcher, Map<ParseField, Object> otherOptions) throws IOException {
+        if (parseFieldMatcher.match(currentFieldName, MULTIVALUE_MODE_FIELD)) {
+            if (token == XContentParser.Token.VALUE_STRING) {
+                otherOptions.put(MULTIVALUE_MODE_FIELD, parser.text());
+                return true;
+            }
+        }
         return false;
     }
 
     @Override
-    protected MatrixStatsAggregatorBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
-                                                         ValueType targetValueType, Map<ParseField, Object> otherOptions) {
-        return new MatrixStatsAggregatorBuilder(aggregationName);
+    protected MatrixStatsAggregationBuilder createFactory(String aggregationName, ValuesSourceType valuesSourceType,
+                                                          ValueType targetValueType, Map<ParseField, Object> otherOptions) {
+        MatrixStatsAggregationBuilder builder = new MatrixStatsAggregationBuilder(aggregationName);
+        String mode = (String)otherOptions.get(MULTIVALUE_MODE_FIELD);
+        if (mode != null) {
+            builder.multiValueMode(MultiValueMode.fromString(mode));
+        }
+        return builder;
     }
 }
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java
index 65cedbc2676..2652b523bde 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java
@@ -21,7 +21,7 @@ package org.elasticsearch.search.aggregations.matrix.stats;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.io.stream.Writeable;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -34,42 +34,43 @@ import java.util.Map;
  *
  * @internal
  */
-class MatrixStatsResults implements Streamable {
+class MatrixStatsResults implements Writeable {
     /** object holding results - computes results in place */
-    final RunningStats results;
+    final protected RunningStats results;
     /** pearson product correlation coefficients */
-    protected HashMap<String, HashMap<String, Double>> correlation;
+    final protected Map<String, HashMap<String, Double>> correlation;
 
     /** Base ctor */
-    private MatrixStatsResults() {
-        results = RunningStats.EMPTY();
+    public MatrixStatsResults() {
+        results = new RunningStats();
         this.correlation = new HashMap<>();
     }
 
     /** creates and computes result from provided stats */
     public MatrixStatsResults(RunningStats stats) {
-        try {
-            this.results = stats.clone();
-            this.correlation = new HashMap<>();
-        } catch (CloneNotSupportedException e) {
-            throw new ElasticsearchException("Error trying to create multifield_stats results", e);
-        }
+        this.results = stats.clone();
+        this.correlation = new HashMap<>();
         this.compute();
     }
 
     /** creates a results object from the given stream */
+    @SuppressWarnings("unchecked")
     protected MatrixStatsResults(StreamInput in) {
         try {
             results = new RunningStats(in);
-            this.readFrom(in);
+            correlation = (Map<String, HashMap<String, Double>>) in.readGenericValue();
         } catch (IOException e) {
             throw new ElasticsearchException("Error trying to create multifield_stats results from stream input", e);
         }
     }
 
-    /** create an empty results object **/
-    protected static MatrixStatsResults EMPTY() {
-        return new MatrixStatsResults();
+    /** Marshalls MatrixStatsResults */
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        // marshall results
+        results.writeTo(out);
+        // marshall correlation
+        out.writeGenericValue(correlation);
     }
 
     /** return document count */
@@ -77,8 +78,8 @@ class MatrixStatsResults implements Streamable {
         return results.docCount;
     }
 
-    /** return the field counts */
-    public Map<String, Long> getFieldCounts() {
+    /** return the field counts - not public, used for getProperty() */
+    protected Map<String, Long> getFieldCounts() {
         return Collections.unmodifiableMap(results.counts);
     }
 
@@ -90,60 +91,66 @@ class MatrixStatsResults implements Streamable {
         return results.counts.get(field);
     }
 
-    /** return the means */
-    public Map<String, Double> getMeans() {
+    /** return the means - not public, used for getProperty() */
+    protected Map<String, Double> getMeans() {
         return Collections.unmodifiableMap(results.means);
     }
 
     /** return the mean for the requested field */
-    public Double getMean(String field) {
+    public double getMean(String field) {
+        checkField(field, results.means);
         return results.means.get(field);
     }
 
-    /** return the variances */
-    public Map<String, Double> getVariances() {
+    /** return the variances - not public, used for getProperty() */
+    protected Map<String, Double> getVariances() {
         return Collections.unmodifiableMap(results.variances);
     }
 
     /** return the variance for the requested field */
-    public Double getVariance(String field) {
+    public double getVariance(String field) {
+        checkField(field, results.variances);
         return results.variances.get(field);
     }
 
-    /** return the skewness */
-    public Map<String, Double> getSkewness() {
+    /** return the skewness - not public, used for getProperty() */
+    protected Map<String, Double> getSkewness() {
         return Collections.unmodifiableMap(results.skewness);
     }
 
     /** return the skewness for the requested field */
-    public Double getSkewness(String field) {
+    public double getSkewness(String field) {
+        checkField(field, results.skewness);
         return results.skewness.get(field);
     }
 
     /** return the kurtosis */
-    public Map<String, Double> getKurtosis() {
+    protected Map<String, Double> getKurtosis() {
         return Collections.unmodifiableMap(results.kurtosis);
     }
 
     /** return the kurtosis for the requested field */
-    public Double getKurtosis(String field) {
+    public double getKurtosis(String field) {
+        checkField(field, results.kurtosis);
         return results.kurtosis.get(field);
     }
 
-    /** return the covariances */
-    public Map<String, HashMap<String, Double>> getCovariances() {
+    /** return the covariances as a map - not public, used for getProperty() */
+    protected Map<String, HashMap<String, Double>> getCovariances() {
         return Collections.unmodifiableMap(results.covariances);
     }
 
     /** return the covariance between two fields */
-    public Double getCovariance(String fieldX, String fieldY) {
+    public double getCovariance(String fieldX, String fieldY) {
         if (fieldX.equals(fieldY)) {
+            checkField(fieldX, results.variances);
             return results.variances.get(fieldX);
         }
         return getValFromUpperTriangularMatrix(results.covariances, fieldX, fieldY);
     }
 
-    public Map<String, HashMap<String, Double>> getCorrelations() {
+    /** return the correlations as a map - not public, used for getProperty() */
+    protected Map<String, HashMap<String, Double>> getCorrelations() {
         return Collections.unmodifiableMap(correlation);
     }
 
@@ -156,10 +163,10 @@ class MatrixStatsResults implements Streamable {
     }
 
     /** return the value for two fields in an upper triangular matrix, regardless of row col location. */
-    private Double getValFromUpperTriangularMatrix(HashMap<String, HashMap<String, Double>> map, String fieldX, String fieldY) {
+    private double getValFromUpperTriangularMatrix(Map<String, HashMap<String, Double>> map, String fieldX, String fieldY) {
         // for the co-value to exist, one of the two (or both) fields has to be a row key
         if (map.containsKey(fieldX) == false && map.containsKey(fieldY) == false) {
-            return null;
+            throw new IllegalArgumentException("neither field " + fieldX + " nor " + fieldY + " exist");
         } else if (map.containsKey(fieldX)) {
             // fieldX exists as a row key
             if (map.get(fieldX).containsKey(fieldY)) {
@@ -176,6 +183,15 @@ class MatrixStatsResults implements Streamable {
         throw new IllegalArgumentException("Coefficient not computed between fields: " + fieldX + " and " + fieldY);
     }
 
+    private void checkField(String field, Map<String, ?> map) {
+        if (field == null) {
+            throw new IllegalArgumentException("field name cannot be null");
+        }
+        if (map.containsKey(field) == false) {
+            throw new IllegalArgumentException("field " + field + " does not exist");
+        }
+    }
+
     /** Computes final covariance, variance, and correlation */
     private void compute() {
         final double nM1 = results.docCount - 1D;
@@ -214,29 +230,4 @@ class MatrixStatsResults implements Streamable {
             correlation.put(rowName, corRow);
         }
     }
-
-    /** Unmarshalls MatrixStatsResults */
-    @Override
-    @SuppressWarnings("unchecked")
-    public void readFrom(StreamInput in) throws IOException {
-        if (in.readBoolean()) {
-            correlation = (HashMap<String, HashMap<String, Double>>) (in.readGenericValue());
-        } else {
-            correlation = null;
-        }
-    }
-
-    /** Marshalls MatrixStatsResults */
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        // marshall results
-        results.writeTo(out);
-        // marshall correlation
-        if (correlation != null) {
-            out.writeBoolean(true);
-            out.writeGenericValue(correlation);
-        } else {
-            out.writeBoolean(false);
-        }
-    }
 }
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStats.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStats.java
index 57a8a8948fe..1aa36e38bd6 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStats.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStats.java
@@ -18,12 +18,14 @@
  */
 package org.elasticsearch.search.aggregations.matrix.stats;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.io.stream.Writeable;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -35,7 +37,7 @@ import java.util.Map;
  *
  * @internal
  */
-public class RunningStats implements Streamable, Cloneable {
+public class RunningStats implements Writeable, Cloneable {
     /** count of observations (same number of observations per field) */
     protected long docCount = 0;
     /** per field sum of observations */
@@ -53,20 +55,14 @@ public class RunningStats implements Streamable, Cloneable {
     /** covariance values */
     protected HashMap<String, HashMap<String, Double>> covariances;
 
-    private RunningStats() {
+    public RunningStats() {
         init();
     }
 
-    /** Ctor to create an instance of running statistics */
-    public RunningStats(StreamInput in) throws IOException {
-        this();
-        this.readFrom(in);
-    }
-
-    public RunningStats(Map<String, Double> doc) {
-        if (doc != null && doc.isEmpty() == false) {
+    public RunningStats(final String[] fieldNames, final double[] fieldVals) {
+        if (fieldVals != null && fieldVals.length > 0) {
             init();
-            this.add(doc);
+            this.add(fieldNames, fieldVals);
         }
     }
 
@@ -80,15 +76,52 @@ public class RunningStats implements Streamable, Cloneable {
         variances = new HashMap<>();
     }
 
-    /** create an empty instance */
-    protected static RunningStats EMPTY() {
-        return new RunningStats();
+    /** Ctor to create an instance of running statistics */
+    @SuppressWarnings("unchecked")
+    public RunningStats(StreamInput in) throws IOException {
+        this();
+        // read fieldSum
+        fieldSum = (HashMap<String, Double>)in.readGenericValue();
+        // counts
+        counts = (HashMap<String, Long>)in.readGenericValue();
+        // means
+        means = (HashMap<String, Double>)in.readGenericValue();
+        // variances
+        variances = (HashMap<String, Double>)in.readGenericValue();
+        // skewness
+        skewness = (HashMap<String, Double>)in.readGenericValue();
+        // kurtosis
+        kurtosis = (HashMap<String, Double>)in.readGenericValue();
+        // read covariances
+        covariances = (HashMap<String, HashMap<String, Double>>)in.readGenericValue();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        // marshall fieldSum
+        out.writeGenericValue(fieldSum);
+        // counts
+        out.writeGenericValue(counts);
+        // mean
+        out.writeGenericValue(means);
+        // variances
+        out.writeGenericValue(variances);
+        // skewness
+        out.writeGenericValue(skewness);
+        // kurtosis
+        out.writeGenericValue(kurtosis);
+        // covariances
+        out.writeGenericValue(covariances);
     }
 
     /** updates running statistics with a documents field values **/
-    public void add(Map<String, Double> doc) {
-        if (doc == null || doc.isEmpty()) {
-            return;
+    public void add(final String[] fieldNames, final double[] fieldVals) {
+        if (fieldNames == null) {
+            throw new IllegalArgumentException("Cannot add statistics without field names.");
+        } else if (fieldVals == null) {
+            throw new IllegalArgumentException("Cannot add statistics without field values.");
+        } else if (fieldNames.length != fieldVals.length) {
+            throw new IllegalArgumentException("Number of field values do not match number of field names.");
         }
 
         // update total, mean, and variance
@@ -98,9 +131,9 @@ public class RunningStats implements Streamable, Cloneable {
         double m1, m2, m3, m4;  // moments
         double d, dn, dn2, t1;
         final HashMap<String, Double> deltas = new HashMap<>();
-        for (Map.Entry<String, Double> field : doc.entrySet()) {
-            fieldName = field.getKey();
-            fieldValue = field.getValue();
+        for (int i = 0; i < fieldNames.length; ++i) {
+            fieldName = fieldNames[i];
+            fieldValue = fieldVals[i];
 
             // update counts
             counts.put(fieldName, 1 + (counts.containsKey(fieldName) ? counts.get(fieldName) : 0));
@@ -133,17 +166,17 @@ public class RunningStats implements Streamable, Cloneable {
             }
         }
 
-        this.updateCovariance(doc, deltas);
+        this.updateCovariance(fieldNames, deltas);
     }
 
     /** Update covariance matrix */
-    private void updateCovariance(final Map<String, Double> doc, final Map<String, Double> deltas) {
+    private void updateCovariance(final String[] fieldNames, final Map<String, Double> deltas) {
         // deep copy of hash keys (field names)
-        ArrayList<String> cFieldNames = new ArrayList<>(doc.keySet());
+        ArrayList<String> cFieldNames = new ArrayList<>(Arrays.asList(fieldNames));
         String fieldName;
         double dR, newVal;
-        for (Map.Entry<String, Double> field : doc.entrySet()) {
-            fieldName = field.getKey();
+        for (int i = 0; i < fieldNames.length; ++i) {
+            fieldName = fieldNames[i];
             cFieldNames.remove(fieldName);
             // update running covariances
             dR = deltas.get(fieldName);
@@ -170,6 +203,21 @@ public class RunningStats implements Streamable, Cloneable {
     public void merge(final RunningStats other) {
         if (other == null) {
             return;
+        } else if (this.docCount == 0) {
+            for (Map.Entry<String, Double> fs : other.means.entrySet()) {
+                final String fieldName = fs.getKey();
+                this.means.put(fieldName, fs.getValue().doubleValue());
+                this.counts.put(fieldName, other.counts.get(fieldName).longValue());
+                this.fieldSum.put(fieldName, other.fieldSum.get(fieldName).doubleValue());
+                this.variances.put(fieldName, other.variances.get(fieldName).doubleValue());
+                this.skewness.put(fieldName , other.skewness.get(fieldName).doubleValue());
+                this.kurtosis.put(fieldName, other.kurtosis.get(fieldName).doubleValue());
+                if (other.covariances.containsKey(fieldName) == true) {
+                    this.covariances.put(fieldName, other.covariances.get(fieldName));
+                }
+                this.docCount = other.docCount;
+            }
+            return;
         }
         final double nA = docCount;
         final double nB = other.docCount;
@@ -226,7 +274,7 @@ public class RunningStats implements Streamable, Cloneable {
     }
 
     /** Merges two covariance matrices */
-    private void mergeCovariance(final RunningStats other, final HashMap<String, Double> deltas) {
+    private void mergeCovariance(final RunningStats other, final Map<String, Double> deltas) {
         final double countA = docCount - other.docCount;
         double f, dR, newVal;
         for (Map.Entry<String, Double> fs : other.means.entrySet()) {
@@ -252,107 +300,11 @@ public class RunningStats implements Streamable, Cloneable {
     }
 
     @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        // marshall fieldSum
-        if (fieldSum != null) {
-            out.writeBoolean(true);
-            out.writeGenericValue(fieldSum);
-        } else {
-            out.writeBoolean(false);
+    public RunningStats clone() {
+        try {
+            return (RunningStats) super.clone();
+        } catch (CloneNotSupportedException e) {
+            throw new ElasticsearchException("Error trying to create a copy of RunningStats");
         }
-        // counts
-        if (counts != null) {
-            out.writeBoolean(true);
-            out.writeGenericValue(counts);
-        } else {
-            out.writeBoolean(false);
-        }
-        // mean
-        if (means != null) {
-            out.writeBoolean(true);
-            out.writeGenericValue(means);
-        } else {
-            out.writeBoolean(false);
-        }
-        // variances
-        if (variances != null) {
-            out.writeBoolean(true);
-            out.writeGenericValue(variances);
-        } else {
-            out.writeBoolean(false);
-        }
-        // skewness
-        if (skewness != null) {
-            out.writeBoolean(true);
-            out.writeGenericValue(skewness);
-        } else {
-            out.writeBoolean(false);
-        }
-        // kurtosis
-        if (kurtosis != null) {
-            out.writeBoolean(true);
-            out.writeGenericValue(kurtosis);
-        } else {
-            out.writeBoolean(false);
-        }
-        // covariances
-        if (covariances != null) {
-            out.writeBoolean(true);
-            out.writeGenericValue(covariances);
-        } else {
-            out.writeBoolean(false);
-        }
-    }
-
-    @Override
-    @SuppressWarnings("unchecked")
-    public void readFrom(StreamInput in) throws IOException {
-        // read fieldSum
-        if (in.readBoolean()) {
-            fieldSum = (HashMap<String, Double>)(in.readGenericValue());
-        } else {
-            fieldSum = null;
-        }
-        // counts
-        if (in.readBoolean()) {
-            counts = (HashMap<String, Long>)(in.readGenericValue());
-        } else {
-            counts = null;
-        }
-        // means
-        if (in.readBoolean()) {
-            means = (HashMap<String, Double>)(in.readGenericValue());
-        } else {
-            means = null;
-        }
-        // variances
-        if (in.readBoolean()) {
-            variances = (HashMap<String, Double>)(in.readGenericValue());
-        } else {
-            variances = null;
-        }
-        // skewness
-        if (in.readBoolean()) {
-            skewness = (HashMap<String, Double>)(in.readGenericValue());
-        } else {
-            skewness = null;
-        }
-        // kurtosis
-        if (in.readBoolean()) {
-            kurtosis = (HashMap<String, Double>)(in.readGenericValue());
-        } else {
-            kurtosis = null;
-        }
-        // read covariances
-        if (in.readBoolean()) {
-            covariances = (HashMap<String, HashMap<String, Double>>) (in.readGenericValue());
-        } else {
-            covariances = null;
-        }
-    }
-
-    @Override
-    public RunningStats clone() throws CloneNotSupportedException {
-        return (RunningStats)super.clone();
     }
 }
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSource.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSource.java
new file mode 100644
index 00000000000..0274c1748dd
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSource.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.support;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.index.fielddata.NumericDoubleValues;
+import org.elasticsearch.search.MultiValueMode;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Class to encapsulate a set of ValuesSource objects labeled by field name
+ */
+public abstract class MultiValuesSource <VS extends ValuesSource> {
+    protected MultiValueMode multiValueMode;
+    protected String[] names;
+    protected VS[] values;
+
+    public static class NumericMultiValuesSource extends MultiValuesSource<ValuesSource.Numeric> {
+        public NumericMultiValuesSource(Map<String, ValuesSource.Numeric> valuesSources, MultiValueMode multiValueMode) {
+            super(valuesSources, multiValueMode);
+            if (valuesSources != null) {
+                this.values = valuesSources.values().toArray(new ValuesSource.Numeric[0]);
+            } else {
+                this.values = new ValuesSource.Numeric[0];
+            }
+        }
+
+        public NumericDoubleValues getField(final int ordinal, LeafReaderContext ctx) throws IOException {
+            if (ordinal > names.length) {
+                throw new IndexOutOfBoundsException("ValuesSource array index " + ordinal + " out of bounds");
+            }
+            return multiValueMode.select(values[ordinal].doubleValues(ctx), Double.NEGATIVE_INFINITY);
+        }
+    }
+
+    public static class BytesMultiValuesSource extends MultiValuesSource<ValuesSource.Bytes> {
+        public BytesMultiValuesSource(Map<String, ValuesSource.Bytes> valuesSources, MultiValueMode multiValueMode) {
+            super(valuesSources, multiValueMode);
+            this.values = valuesSources.values().toArray(new ValuesSource.Bytes[0]);
+        }
+
+        public Object getField(final int ordinal, LeafReaderContext ctx) throws IOException {
+            return values[ordinal].bytesValues(ctx);
+        }
+    }
+
+    public static class GeoPointValuesSource extends MultiValuesSource<ValuesSource.GeoPoint> {
+        public GeoPointValuesSource(Map<String, ValuesSource.GeoPoint> valuesSources, MultiValueMode multiValueMode) {
+            super(valuesSources, multiValueMode);
+            this.values = valuesSources.values().toArray(new ValuesSource.GeoPoint[0]);
+        }
+    }
+
+    private MultiValuesSource(Map<String, ?> valuesSources, MultiValueMode multiValueMode) {
+        if (valuesSources != null) {
+            this.names = valuesSources.keySet().toArray(new String[0]);
+        }
+        this.multiValueMode = multiValueMode;
+    }
+
+    public boolean needsScores() {
+        boolean needsScores = false;
+        for (ValuesSource value : values) {
+            needsScores |= value.needsScores();
+        }
+        return needsScores;
+    }
+
+    public String[] fieldNames() {
+        return this.names;
+    }
+}
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
new file mode 100644
index 00000000000..e4ea8df0497
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
@@ -0,0 +1,366 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.aggregations.support;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.AggregationInitializationException;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.InternalAggregation.Type;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ *
+ */
+public abstract class MultiValuesSourceAggregationBuilder<VS extends ValuesSource, AB extends MultiValuesSourceAggregationBuilder<VS, AB>>
+        extends AggregationBuilder<AB> {
+
+    public static final ParseField MULTIVALUE_MODE_FIELD = new ParseField("mode");
+
+    public static abstract class LeafOnly<VS extends ValuesSource, AB extends MultiValuesSourceAggregationBuilder<VS, AB>>
+            extends MultiValuesSourceAggregationBuilder<VS, AB> {
+
+        protected LeafOnly(String name, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType) {
+            super(name, type, valuesSourceType, targetValueType);
+        }
+
+        /**
+         * Read from a stream that does not serialize its targetValueType. This should be used by most subclasses.
+         */
+        protected LeafOnly(StreamInput in, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType) throws IOException {
+            super(in, type, valuesSourceType, targetValueType);
+        }
+
+        /**
+         * Read an aggregation from a stream that serializes its targetValueType. This should only be used by subclasses that override
+         * {@link #serializeTargetValueType()} to return true.
+         */
+        protected LeafOnly(StreamInput in, Type type, ValuesSourceType valuesSourceType) throws IOException {
+            super(in, type, valuesSourceType);
+        }
+
+        @Override
+        public AB subAggregations(Builder subFactories) {
+            throw new AggregationInitializationException("Aggregator [" + name + "] of type [" +
+                type + "] cannot accept sub-aggregations");
+        }
+    }
+
+    private final ValuesSourceType valuesSourceType;
+    private final ValueType targetValueType;
+    private List<String> fields = Collections.emptyList();
+    private ValueType valueType = null;
+    private String format = null;
+    private Object missing = null;
+    private Map<String, Object> missingMap = Collections.emptyMap();
+
+    protected MultiValuesSourceAggregationBuilder(String name, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType) {
+        super(name, type);
+        if (valuesSourceType == null) {
+            throw new IllegalArgumentException("[valuesSourceType] must not be null: [" + name + "]");
+        }
+        this.valuesSourceType = valuesSourceType;
+        this.targetValueType = targetValueType;
+    }
+
+    protected MultiValuesSourceAggregationBuilder(StreamInput in, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType)
+        throws IOException {
+        super(in, type);
+        assert false == serializeTargetValueType() : "Wrong read constructor called for subclass that provides its targetValueType";
+        this.valuesSourceType = valuesSourceType;
+        this.targetValueType = targetValueType;
+        read(in);
+    }
+
+    protected MultiValuesSourceAggregationBuilder(StreamInput in, Type type, ValuesSourceType valuesSourceType) throws IOException {
+        super(in, type);
+        assert serializeTargetValueType() : "Wrong read constructor called for subclass that serializes its targetValueType";
+        this.valuesSourceType = valuesSourceType;
+        this.targetValueType = in.readOptionalWriteable(ValueType::readFromStream);
+        read(in);
+    }
+
+    /**
+     * Read from a stream.
+     */
+    @SuppressWarnings("unchecked")
+    private void read(StreamInput in) throws IOException {
+        fields = (ArrayList<String>)in.readGenericValue();
+        valueType = in.readOptionalWriteable(ValueType::readFromStream);
+        format = in.readOptionalString();
+        missingMap = in.readMap();
+    }
+
+    @Override
+    protected final void doWriteTo(StreamOutput out) throws IOException {
+        if (serializeTargetValueType()) {
+            out.writeOptionalWriteable(targetValueType);
+        }
+        out.writeGenericValue(fields);
+        out.writeOptionalWriteable(valueType);
+        out.writeOptionalString(format);
+        out.writeMap(missingMap);
+        innerWriteTo(out);
+    }
+
+    /**
+     * Write subclass' state to the stream
+     */
+    protected abstract void innerWriteTo(StreamOutput out) throws IOException;
+
+    /**
+     * Sets the field to use for this aggregation.
+     */
+    @SuppressWarnings("unchecked")
+    public AB fields(List<String> fields) {
+        if (fields == null) {
+            throw new IllegalArgumentException("[field] must not be null: [" + name + "]");
+        }
+        this.fields = fields;
+        return (AB) this;
+    }
+
+    /**
+     * Gets the field to use for this aggregation.
+     */
+    public List<String> fields() {
+        return fields;
+    }
+
+    /**
+     * Sets the {@link ValueType} for the value produced by this aggregation
+     */
+    @SuppressWarnings("unchecked")
+    public AB valueType(ValueType valueType) {
+        if (valueType == null) {
+            throw new IllegalArgumentException("[valueType] must not be null: [" + name + "]");
+        }
+        this.valueType = valueType;
+        return (AB) this;
+    }
+
+    /**
+     * Gets the {@link ValueType} for the value produced by this aggregation
+     */
+    public ValueType valueType() {
+        return valueType;
+    }
+
+    /**
+     * Sets the format to use for the output of the aggregation.
+     */
+    @SuppressWarnings("unchecked")
+    public AB format(String format) {
+        if (format == null) {
+            throw new IllegalArgumentException("[format] must not be null: [" + name + "]");
+        }
+        this.format = format;
+        return (AB) this;
+    }
+
+    /**
+     * Gets the format to use for the output of the aggregation.
+     */
+    public String format() {
+        return format;
+    }
+
+    /**
+     * Sets the value to use when the aggregation finds a missing value in a
+     * document
+     */
+    @SuppressWarnings("unchecked")
+    public AB missingMap(Map<String, Object> missingMap) {
+        if (missingMap == null) {
+            throw new IllegalArgumentException("[missing] must not be null: [" + name + "]");
+        }
+        this.missingMap = missingMap;
+        return (AB) this;
+    }
+
+    /**
+     * Gets the value to use when the aggregation finds a missing value in a
+     * document
+     */
+    public Map<String, Object> missingMap() {
+        return missingMap;
+    }
+
+    @Override
+    protected final MultiValuesSourceAggregatorFactory<VS, ?> doBuild(AggregationContext context, AggregatorFactory<?> parent,
+            AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
+        Map<String, ValuesSourceConfig<VS>> configs = resolveConfig(context);
+        MultiValuesSourceAggregatorFactory<VS, ?> factory = innerBuild(context, configs, parent, subFactoriesBuilder);
+        return factory;
+    }
+
+    protected Map<String, ValuesSourceConfig<VS>> resolveConfig(AggregationContext context) {
+        HashMap<String, ValuesSourceConfig<VS>> configs = new HashMap<>();
+        for (String field : fields) {
+            ValuesSourceConfig<VS> config = config(context, field, null);
+            configs.put(field, config);
+        }
+        return configs;
+    }
+
+    protected abstract MultiValuesSourceAggregatorFactory<VS, ?> innerBuild(AggregationContext context,
+            Map<String, ValuesSourceConfig<VS>> configs, AggregatorFactory<?> parent,
+            AggregatorFactories.Builder subFactoriesBuilder) throws IOException;
+
+    public ValuesSourceConfig<VS> config(AggregationContext context, String field, Script script) {
+
+        ValueType valueType = this.valueType != null ? this.valueType : targetValueType;
+
+        if (field == null) {
+            if (script == null) {
+                @SuppressWarnings("unchecked")
+                ValuesSourceConfig<VS> config = new ValuesSourceConfig<>(ValuesSourceType.ANY);
+                return config.format(resolveFormat(null, valueType));
+            }
+            ValuesSourceType valuesSourceType = valueType != null ? valueType.getValuesSourceType() : this.valuesSourceType;
+            if (valuesSourceType == null || valuesSourceType == ValuesSourceType.ANY) {
+                // the specific value source type is undefined, but for scripts,
+                // we need to have a specific value source
+                // type to know how to handle the script values, so we fallback
+                // on Bytes
+                valuesSourceType = ValuesSourceType.BYTES;
+            }
+            ValuesSourceConfig<VS> config = new ValuesSourceConfig<>(valuesSourceType);
+            config.missing(missingMap.get(field));
+            return config.format(resolveFormat(format, valueType));
+        }
+
+        MappedFieldType fieldType = context.searchContext().smartNameFieldType(field);
+        if (fieldType == null) {
+            ValuesSourceType valuesSourceType = valueType != null ? valueType.getValuesSourceType() : this.valuesSourceType;
+            ValuesSourceConfig<VS> config = new ValuesSourceConfig<>(valuesSourceType);
+            config.missing(missingMap.get(field));
+            config.format(resolveFormat(format, valueType));
+            return config.unmapped(true);
+        }
+
+        IndexFieldData<?> indexFieldData = context.searchContext().fieldData().getForField(fieldType);
+
+        ValuesSourceConfig<VS> config;
+        if (valuesSourceType == ValuesSourceType.ANY) {
+            if (indexFieldData instanceof IndexNumericFieldData) {
+                config = new ValuesSourceConfig<>(ValuesSourceType.NUMERIC);
+            } else if (indexFieldData instanceof IndexGeoPointFieldData) {
+                config = new ValuesSourceConfig<>(ValuesSourceType.GEOPOINT);
+            } else {
+                config = new ValuesSourceConfig<>(ValuesSourceType.BYTES);
+            }
+        } else {
+            config = new ValuesSourceConfig<>(valuesSourceType);
+        }
+
+        config.fieldContext(new FieldContext(field, indexFieldData, fieldType));
+        config.missing(missingMap.get(field));
+        return config.format(fieldType.docValueFormat(format, null));
+    }
+
+    private static DocValueFormat resolveFormat(@Nullable String format, @Nullable ValueType valueType) {
+        if (valueType == null) {
+            return DocValueFormat.RAW; // we can't figure it out
+        }
+        DocValueFormat valueFormat = valueType.defaultFormat();
+        if (valueFormat instanceof DocValueFormat.Decimal && format != null) {
+            valueFormat = new DocValueFormat.Decimal(format);
+        }
+        return valueFormat;
+    }
+
+    /**
+     * Should this builder serialize its targetValueType? Defaults to false. All subclasses that override this to true
+     * should use the three argument read constructor rather than the four argument version.
+     */
+    protected boolean serializeTargetValueType() {
+        return false;
+    }
+
+    @Override
+    public final XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        // todo add ParseField support to XContentBuilder
+        if (fields != null) {
+            builder.field(CommonFields.FIELDS.getPreferredName(), fields);
+        }
+        if (missing != null) {
+            builder.field(CommonFields.MISSING.getPreferredName(), missing);
+        }
+        if (format != null) {
+            builder.field(CommonFields.FORMAT.getPreferredName(), format);
+        }
+        if (valueType != null) {
+            builder.field(CommonFields.VALUE_TYPE.getPreferredName(), valueType.getPreferredName());
+        }
+        doXContentBody(builder, params);
+        builder.endObject();
+        return builder;
+    }
+
+    protected abstract XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException;
+
+    @Override
+    protected final int doHashCode() {
+        return Objects.hash(fields, format, missing, targetValueType, valueType, valuesSourceType,
+                innerHashCode());
+    }
+
+    protected abstract int innerHashCode();
+
+    @Override
+    protected final boolean doEquals(Object obj) {
+        MultiValuesSourceAggregationBuilder<?, ?> other = (MultiValuesSourceAggregationBuilder<?, ?>) obj;
+        if (!Objects.equals(fields, other.fields))
+            return false;
+        if (!Objects.equals(format, other.format))
+            return false;
+        if (!Objects.equals(missing, other.missing))
+            return false;
+        if (!Objects.equals(targetValueType, other.targetValueType))
+            return false;
+        if (!Objects.equals(valueType, other.valueType))
+            return false;
+        if (!Objects.equals(valuesSourceType, other.valuesSourceType))
+            return false;
+        return innerEquals(obj);
+    }
+
+    protected abstract boolean innerEquals(Object obj);
+}
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java
new file mode 100644
index 00000000000..956f953da10
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregatorFactory.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.support;
+
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.InternalAggregation.Type;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public abstract class MultiValuesSourceAggregatorFactory<VS extends ValuesSource, AF extends MultiValuesSourceAggregatorFactory<VS, AF>>
+        extends AggregatorFactory<AF> {
+
+    protected Map<String, ValuesSourceConfig<VS>> configs;
+
+    public MultiValuesSourceAggregatorFactory(String name, Type type, Map<String, ValuesSourceConfig<VS>> configs,
+        AggregationContext context, AggregatorFactory<?> parent, AggregatorFactories.Builder subFactoriesBuilder,
+        Map<String, Object> metaData) throws IOException {
+        super(name, type, context, parent, subFactoriesBuilder, metaData);
+        this.configs = configs;
+    }
+
+    @Override
+    public Aggregator createInternal(Aggregator parent, boolean collectsFromSingleBucket, List<PipelineAggregator> pipelineAggregators,
+                                     Map<String, Object> metaData) throws IOException {
+        HashMap<String, VS> valuesSources = new HashMap<>();
+
+        for (Map.Entry<String, ValuesSourceConfig<VS>> config : configs.entrySet()) {
+            VS vs = context.valuesSource(config.getValue(), context.searchContext());
+            if (vs != null) {
+                valuesSources.put(config.getKey(), vs);
+            }
+        }
+        if (valuesSources.isEmpty()) {
+            return createUnmapped(parent, pipelineAggregators, metaData);
+        }
+        return doCreateInternal(valuesSources, parent, collectsFromSingleBucket, pipelineAggregators, metaData);
+    }
+
+    protected abstract Aggregator createUnmapped(Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+                                                 Map<String, Object> metaData) throws IOException;
+
+    protected abstract Aggregator doCreateInternal(Map<String, VS> valuesSources, Aggregator parent, boolean collectsFromSingleBucket,
+        List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) throws IOException;
+
+}
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParser.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParser.java
new file mode 100644
index 00000000000..dd2b69696f3
--- /dev/null
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParser.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.support;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.script.Script.ScriptField;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregationBuilder.CommonFields;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ *
+ */
+public abstract class MultiValuesSourceParser<VS extends ValuesSource> implements Aggregator.Parser {
+
+    public abstract static class AnyValuesSourceParser extends MultiValuesSourceParser<ValuesSource> {
+
+        protected AnyValuesSourceParser(boolean formattable) {
+            super(formattable, ValuesSourceType.ANY, null);
+        }
+    }
+
+    public abstract static class NumericValuesSourceParser extends MultiValuesSourceParser<ValuesSource.Numeric> {
+
+        protected NumericValuesSourceParser(boolean formattable) {
+            super(formattable, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        }
+    }
+
+    public abstract static class BytesValuesSourceParser extends MultiValuesSourceParser<ValuesSource.Bytes> {
+
+        protected BytesValuesSourceParser(boolean formattable) {
+            super(formattable, ValuesSourceType.BYTES, ValueType.STRING);
+        }
+    }
+
+    public abstract static class GeoPointValuesSourceParser extends MultiValuesSourceParser<ValuesSource.GeoPoint> {
+
+        protected GeoPointValuesSourceParser(boolean formattable) {
+            super(formattable, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
+        }
+    }
+
+    private boolean formattable = false;
+    private ValuesSourceType valuesSourceType = null;
+    private ValueType targetValueType = null;
+
+    private MultiValuesSourceParser(boolean formattable, ValuesSourceType valuesSourceType, ValueType targetValueType) {
+        this.valuesSourceType = valuesSourceType;
+        this.targetValueType = targetValueType;
+        this.formattable = formattable;
+    }
+
+    @Override
+    public final MultiValuesSourceAggregationBuilder<VS, ?> parse(String aggregationName, QueryParseContext context)
+            throws IOException {
+
+        XContentParser parser = context.parser();
+        List<String> fields = null;
+        ValueType valueType = null;
+        String format = null;
+        Map<String, Object> missingMap = null;
+        Map<ParseField, Object> otherOptions = new HashMap<>();
+        final ParseFieldMatcher parseFieldMatcher = context.getParseFieldMatcher();
+
+        XContentParser.Token token;
+        String currentFieldName = null;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token == XContentParser.Token.VALUE_STRING) {
+                if (parseFieldMatcher.match(currentFieldName, CommonFields.FIELDS)) {
+                    fields = Collections.singletonList(parser.text());
+                } else if (formattable && parseFieldMatcher.match(currentFieldName, CommonFields.FORMAT)) {
+                    format = parser.text();
+                } else if (parseFieldMatcher.match(currentFieldName, CommonFields.VALUE_TYPE)) {
+                    throw new ParsingException(parser.getTokenLocation(),
+                        "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "]. " +
+                            "Multi-field aggregations do not support scripts.");
+                } else if (!token(aggregationName, currentFieldName, token, parser, context.getParseFieldMatcher(), otherOptions)) {
+                    throw new ParsingException(parser.getTokenLocation(),
+                            "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
+                }
+            } else if (token == XContentParser.Token.START_OBJECT) {
+                if (parseFieldMatcher.match(currentFieldName, CommonFields.MISSING)) {
+                    missingMap = new HashMap<>();
+                    while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+                        parseMissingAndAdd(aggregationName, currentFieldName, parser, missingMap);
+                    }
+                } else if (context.getParseFieldMatcher().match(currentFieldName, ScriptField.SCRIPT)) {
+                    throw new ParsingException(parser.getTokenLocation(),
+                        "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "]. " +
+                            "Multi-field aggregations do not support scripts.");
+
+                } else if (!token(aggregationName, currentFieldName, token, parser, context.getParseFieldMatcher(), otherOptions)) {
+                    throw new ParsingException(parser.getTokenLocation(),
+                            "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
+                }
+            } else if (token == XContentParser.Token.START_ARRAY) {
+                if (context.getParseFieldMatcher().match(currentFieldName, ScriptField.SCRIPT)) {
+                    throw new ParsingException(parser.getTokenLocation(),
+                        "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "]. " +
+                            "Multi-field aggregations do not support scripts.");
+                } else if (parseFieldMatcher.match(currentFieldName, CommonFields.FIELDS)) {
+                    fields = new ArrayList<>();
+                    while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
+                        if (token == XContentParser.Token.VALUE_STRING) {
+                            fields.add(parser.text());
+                        } else {
+                            throw new ParsingException(parser.getTokenLocation(),
+                                    "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
+                        }
+                    }
+                } else if (!token(aggregationName, currentFieldName, token, parser, context.getParseFieldMatcher(), otherOptions)) {
+                    throw new ParsingException(parser.getTokenLocation(),
+                            "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
+                }
+            } else if (!token(aggregationName, currentFieldName, token, parser, context.getParseFieldMatcher(), otherOptions)) {
+                throw new ParsingException(parser.getTokenLocation(),
+                        "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "].");
+            }
+        }
+
+        MultiValuesSourceAggregationBuilder<VS, ?> factory = createFactory(aggregationName, this.valuesSourceType, this.targetValueType,
+                otherOptions);
+        if (fields != null) {
+            factory.fields(fields);
+        }
+        if (valueType != null) {
+            factory.valueType(valueType);
+        }
+        if (format != null) {
+            factory.format(format);
+        }
+        if (missingMap != null) {
+            factory.missingMap(missingMap);
+        }
+        return factory;
+    }
+
+    private final void parseMissingAndAdd(final String aggregationName, final String currentFieldName,
+                                          XContentParser parser, final Map<String, Object> missing) throws IOException {
+        XContentParser.Token token = parser.currentToken();
+        if (token == null) {
+            token = parser.nextToken();
+        }
+
+        if (token == XContentParser.Token.FIELD_NAME) {
+            final String fieldName = parser.currentName();
+            if (missing.containsKey(fieldName)) {
+                throw new ParsingException(parser.getTokenLocation(),
+                    "Missing field [" + fieldName + "] already defined as [" + missing.get(fieldName)
+                        + "] in [" + aggregationName + "].");
+            }
+            parser.nextToken();
+            missing.put(fieldName, parser.objectText());
+        } else {
+            throw new ParsingException(parser.getTokenLocation(),
+                "Unexpected token " + token + " [" + currentFieldName + "] in [" + aggregationName + "]");
+        }
+    }
+
+    /**
+     * Creates a {@link ValuesSourceAggregationBuilder} from the information
+     * gathered by the subclass. Options parsed in
+     * {@link MultiValuesSourceParser} itself will be added to the factory
+     * after it has been returned by this method.
+     *
+     * @param aggregationName
+     *            the name of the aggregation
+     * @param valuesSourceType
+     *            the type of the {@link ValuesSource}
+     * @param targetValueType
+     *            the target type of the final value output by the aggregation
+     * @param otherOptions
+     *            a {@link Map} containing the extra options parsed by the
+     *            {@link #token(String, String, org.elasticsearch.common.xcontent.XContentParser.Token,
+     *             XContentParser, ParseFieldMatcher, Map)}
+     *            method
+     * @return the created factory
+     */
+    protected abstract MultiValuesSourceAggregationBuilder<VS, ?> createFactory(String aggregationName, ValuesSourceType valuesSourceType,
+        ValueType targetValueType, Map<ParseField, Object> otherOptions);
+
+    /**
+     * Allows subclasses of {@link MultiValuesSourceParser} to parse extra
+     * parameters and store them in a {@link Map} which will later be passed to
+     * {@link #createFactory(String, ValuesSourceType, ValueType, Map)}.
+     *
+     * @param aggregationName
+     *            the name of the aggregation
+     * @param currentFieldName
+     *            the name of the current field being parsed
+     * @param token
+     *            the current token for the parser
+     * @param parser
+     *            the parser
+     * @param parseFieldMatcher
+     *            the {@link ParseFieldMatcher} to use to match field names
+     * @param otherOptions
+     *            a {@link Map} of options to be populated by successive calls
+     *            to this method which will then be passed to the
+     *            {@link #createFactory(String, ValuesSourceType, ValueType, Map)}
+     *            method
+     * @return <code>true</code> if the current token was correctly parsed,
+     *         <code>false</code> otherwise
+     * @throws IOException
+     *             if an error occurs whilst parsing
+     */
+    protected abstract boolean token(String aggregationName, String currentFieldName, XContentParser.Token token, XContentParser parser,
+        ParseFieldMatcher parseFieldMatcher, Map<ParseField, Object> otherOptions) throws IOException;
+}
diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsTestCase.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/BaseMatrixStatsTestCase.java
similarity index 99%
rename from modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsTestCase.java
rename to modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/BaseMatrixStatsTestCase.java
index 84b032b82b6..b1296bb1146 100644
--- a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsTestCase.java
+++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/BaseMatrixStatsTestCase.java
@@ -29,7 +29,7 @@ import static org.hamcrest.Matchers.equalTo;
 /**
  *
  */
-public class MatrixStatsTestCase extends ESTestCase {
+public abstract class BaseMatrixStatsTestCase extends ESTestCase {
     protected final int numObs = atLeast(10000);
     protected final ArrayList<Double> fieldA = new ArrayList<>(numObs);
     protected final ArrayList<Double> fieldB = new ArrayList<>(numObs);
diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStatsTests.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStatsTests.java
index 2992fd57a7e..f90f00d2a79 100644
--- a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStatsTests.java
+++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStatsTests.java
@@ -18,13 +18,12 @@
  */
 package org.elasticsearch.search.aggregations.matrix.stats;
 
-import java.util.HashMap;
 import java.util.List;
 
 /**
  *
  */
-public class RunningStatsTests extends MatrixStatsTestCase {
+public class RunningStatsTests extends BaseMatrixStatsTestCase {
 
     /** test running stats */
     public void testRunningStats() throws Exception {
@@ -56,15 +55,18 @@ public class RunningStatsTests extends MatrixStatsTestCase {
     }
 
     private RunningStats createRunningStats(List<Double> fieldAObs, List<Double> fieldBObs) {
-        RunningStats stats = RunningStats.EMPTY();
+        RunningStats stats = new RunningStats();
         // create a document with two numeric fields
-        final HashMap<String, Double> doc = new HashMap<>(2);
+        final String[] fieldNames = new String[2];
+        fieldNames[0] = fieldAKey;
+        fieldNames[1] = fieldBKey;
+        final double[] fieldVals = new double[2];
 
         // running stats computation
         for (int n = 0; n < fieldAObs.size(); ++n) {
-            doc.put(fieldAKey, fieldAObs.get(n));
-            doc.put(fieldBKey, fieldBObs.get(n));
-            stats.add(doc);
+            fieldVals[0] = fieldAObs.get(n);
+            fieldVals[1] = fieldBObs.get(n);
+            stats.add(fieldNames, fieldVals);
         }
         return stats;
     }
diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/20_empty_bucket.yaml b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/20_empty_bucket.yaml
index d91c53e9e42..b6fade2cf18 100644
--- a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/20_empty_bucket.yaml
+++ b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/20_empty_bucket.yaml
@@ -43,6 +43,6 @@
       search:
         index: empty_bucket_idx
         type:  test
-        body: {"aggs": {"histo": {"histogram": {"field": "val1", "interval": 1, "min_doc_count": 0}, "aggs": { "mfs" : { "matrix_stats": {"field": ["value", "val1"]} } } } } }
+        body: {"aggs": {"histo": {"histogram": {"field": "val1", "interval": 1, "min_doc_count": 0}, "aggs": { "mfs" : { "matrix_stats": {"fields": ["value", "val1"]} } } } } }
 
   - match: {hits.total: 2}
diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yaml b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yaml
index 0fe3590af48..6c2098f181e 100644
--- a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yaml
+++ b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/30_single_value_field.yaml
@@ -130,7 +130,7 @@ setup:
       search:
         index: unmapped
         type:  test
-        body: {"aggs": { "mfs" : { "matrix_stats": {"field": ["val1", "val2", "val3"]} } } }
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2", "val3"]} } } }
 
   - match: {hits.total: 0}
 
@@ -141,10 +141,10 @@ setup:
       search:
         index: test
         type:  test
-        body: {"aggs": { "mfs" : { "matrix_stats": {"field": ["val1", "val3"]} } } }
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val3"]} } } }
 
   - match: {hits.total: 15}
-  - match: {aggregations.mfs.count.0: 15}
+  - match: {aggregations.mfs.fields.0.count: 15}
 
 ---
 "Partially unmapped":
@@ -153,32 +153,41 @@ setup:
       search:
         index: [test, unmapped]
         type:  test
-        body: {"aggs": { "mfs" : { "matrix_stats": {"field": ["val1", "val2", "val3"]} } } }
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2", "val3"]} } } }
 
   - match: {hits.total: 15}
-  - match: {aggregations.mfs.count.0: 14}
+  - match: {aggregations.mfs.fields.0.count: 14}
+  - match: {aggregations.mfs.fields.2.correlation.val2: 0.9569513137793205}
+
+---
+"Partially unmapped with missing default":
+
+  - do:
+      search:
+        index: [test, unmapped]
+        type:  test
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2", "val3"], "missing" : {"val2" : 10} } } } }
+
+  - match: {hits.total: 15}
+  - match: {aggregations.mfs.fields.0.count: 15}
+  - match: {aggregations.mfs.fields.2.correlation.val2: 0.9567970467908384}
 
 ---
 "With script":
 
   - do:
+      catch: /parsing_exception/
       search:
         index: test
         type:  test
-        body: {"aggs": { "mfs" : { "matrix_stats": {"field": ["val1", "val2"], "script" : { "my_script" : {"inline" : "1 + doc['val1'].value", "lang" : "js"} } } } } }
-
-  - match: {hits.total: 15}
-  - match: {aggregations.mfs.count.0: 14}
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2"], "script" : { "my_script" : {"inline" : "1 + doc['val1'].value", "lang" : "js"} } } } } }
 
 ---
 "With script params":
 
   - do:
+      catch: /parsing_exception/
       search:
         index: test
         type:  test
-        body: {"aggs": { "mfs" : { "matrix_stats": {"field": ["val1", "val2", "val3"], "script" : { "my_script" : {"inline" : "my_var + doc['val1'].value", "params" : { "my_var" : 1 }, "lang" : "js" } } } } } }
-
-  - match: {hits.total: 15}
-  - match: {aggregations.mfs.count.0: 14}
-  - match: {aggregations.mfs.correlation.1.2: 0.9569513137793205}
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2", "val3"], "script" : { "my_script" : {"inline" : "my_var + doc['val1'].value", "params" : { "my_var" : 1 }, "lang" : "js" } } } } } }
diff --git a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yaml b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yaml
index 87b5dea19fe..10b722555d8 100644
--- a/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yaml
+++ b/modules/aggs-matrix-stats/src/test/resources/rest-api-spec/test/stats/40_multi_value_field.yaml
@@ -130,21 +130,35 @@ setup:
       search:
         index: unmapped
         type:  test
-        body: {"aggs": { "mfs" : { "matrix_stats": {"field": ["val1", "val2", "vals"]} } } }
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2", "vals"]} } } }
 
   - match: {hits.total: 0}
 
 ---
-"Multi value field":
+"Multi value field Max":
 
   - do:
       search:
         index: test
         type:  test
-        body: {"aggs": { "mfs" : { "matrix_stats": {"field": ["val1", "val3"]} } } }
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "vals"], "mode" : "max"} } } }
 
   - match: {hits.total: 15}
-  - match: {aggregations.mfs.count.0: 15}
+  - match: {aggregations.mfs.fields.0.count: 14}
+  - match: {aggregations.mfs.fields.0.correlation.val1: 0.06838646533369998}
+
+---
+"Multi value field Min":
+
+  - do:
+      search:
+        index: test
+        type:  test
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "vals"], "mode" : "min"} } } }
+
+  - match: {hits.total: 15}
+  - match: {aggregations.mfs.fields.0.count: 14}
+  - match: {aggregations.mfs.fields.0.correlation.val1: -0.09777682707831963}
 
 ---
 "Partially unmapped":
@@ -153,32 +167,41 @@ setup:
       search:
         index: [test, unmapped]
         type:  test
-        body: {"aggs": { "mfs" : { "matrix_stats": {"field": ["val1", "val2", "vals"]} } } }
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2", "vals"]} } } }
 
   - match: {hits.total: 15}
-  - match: {aggregations.mfs.count.0: 13}
+  - match: {aggregations.mfs.fields.0.count: 13}
+  - match: {aggregations.mfs.fields.0.correlation.val1: -0.044997535185684244}
+
+---
+"Partially unmapped with missing defaults":
+
+  - do:
+      search:
+        index: [test, unmapped]
+        type:  test
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val2", "vals"], "missing" : {"val2" : 10, "vals" : 5 } } } } }
+
+  - match: {hits.total: 15}
+  - match: {aggregations.mfs.fields.0.count: 15}
+  - match: {aggregations.mfs.fields.0.correlation.val2: 0.04028024709708195}
 
 ---
 "With script":
 
   - do:
+      catch: /parsing_exception/
       search:
         index: test
         type:  test
-        body: {"aggs": { "mfs" : { "matrix_stats": {"field": ["vals", "val3"], "script" : { "my_script" : {"inline" : "1 + doc['val1'].value", "lang" : "js"} } } } } }
-
-  - match: {hits.total: 15}
-  - match: {aggregations.mfs.count.0: 14}
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["vals", "val3"], "script" : { "my_script" : {"inline" : "1 + doc['val1'].value", "lang" : "js"} } } } } }
 
 ---
 "With script params":
 
   - do:
+      catch: /parsing_exception/
       search:
         index: test
         type:  test
-        body: {"aggs": { "mfs" : { "matrix_stats": {"field": ["val1", "val3", "vals"], "script" : { "my_script" : {"inline" : "my_var + doc['val1'].value", "params" : { "my_var" : 1 }, "lang" : "js" } } } } } }
-
-  - match: {hits.total: 15}
-  - match: {aggregations.mfs.count.0: 14}
-  - match: {aggregations.mfs.correlation.1.2: -0.055971032866899535}
+        body: {"aggs": { "mfs" : { "matrix_stats": {"fields": ["val1", "val3", "vals"], "script" : { "my_script" : {"inline" : "my_var + doc['val1'].value", "params" : { "my_var" : 1 }, "lang" : "js" } } } } } }

From 54575e55caf9a7781d95e0d2e5f41d163eac3ce2 Mon Sep 17 00:00:00 2001
From: Nicholas Knize <nknize@gmail.com>
Date: Tue, 31 May 2016 12:28:10 -0500
Subject: [PATCH 0559/1311] set ValuesSourceConfig fields as private

---
 .../support/AggregationContext.java           | 73 ++++++++++---------
 .../ValuesSourceAggregationBuilder.java       | 32 ++++----
 .../ValuesSourceAggregatorFactory.java        |  4 +-
 .../support/ValuesSourceConfig.java           | 35 ++++++---
 .../MultiValuesSourceAggregationBuilder.java  |  4 +-
 5 files changed, 83 insertions(+), 65 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java
index d66a1664c8f..c4d76304d09 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java
@@ -61,29 +61,30 @@ public class AggregationContext {
         assert config.valid() : "value source config is invalid - must have either a field context or a script or marked as unmapped";
 
         final VS vs;
-        if (config.unmapped) {
-            if (config.missing == null) {
+        if (config.unmapped()) {
+            if (config.missing() == null) {
                 // otherwise we will have values because of the missing value
                 vs = null;
-            } else if (config.valueSourceType == ValuesSourceType.NUMERIC) {
+            } else if (config.valueSourceType() == ValuesSourceType.NUMERIC) {
                 vs = (VS) ValuesSource.Numeric.EMPTY;
-            } else if (config.valueSourceType == ValuesSourceType.GEOPOINT) {
+            } else if (config.valueSourceType() == ValuesSourceType.GEOPOINT) {
                 vs = (VS) ValuesSource.GeoPoint.EMPTY;
-            } else if (config.valueSourceType == ValuesSourceType.ANY || config.valueSourceType == ValuesSourceType.BYTES) {
+            } else if (config.valueSourceType() == ValuesSourceType.ANY || config.valueSourceType() == ValuesSourceType.BYTES) {
                 vs = (VS) ValuesSource.Bytes.EMPTY;
             } else {
-                throw new SearchParseException(searchContext, "Can't deal with unmapped ValuesSource type " + config.valueSourceType, null);
+                throw new SearchParseException(searchContext, "Can't deal with unmapped ValuesSource type "
+                    + config.valueSourceType(), null);
             }
         } else {
             vs = originalValuesSource(config);
         }
 
-        if (config.missing == null) {
+        if (config.missing() == null) {
             return vs;
         }
 
         if (vs instanceof ValuesSource.Bytes) {
-            final BytesRef missing = new BytesRef(config.missing.toString());
+            final BytesRef missing = new BytesRef(config.missing().toString());
             if (vs instanceof ValuesSource.Bytes.WithOrdinals) {
                 return (VS) MissingValues.replaceMissing((ValuesSource.Bytes.WithOrdinals) vs, missing);
             } else {
@@ -91,20 +92,20 @@ public class AggregationContext {
             }
         } else if (vs instanceof ValuesSource.Numeric) {
             Number missing = null;
-            if (config.missing instanceof Number) {
-                missing = (Number) config.missing;
+            if (config.missing() instanceof Number) {
+                missing = (Number) config.missing();
             } else {
-                if (config.fieldContext != null && config.fieldContext.fieldType() != null) {
-                    missing = config.fieldContext.fieldType().docValueFormat(null, DateTimeZone.UTC)
-                            .parseDouble(config.missing.toString(), false, context.nowCallable());
+                if (config.fieldContext() != null && config.fieldContext().fieldType() != null) {
+                    missing = config.fieldContext().fieldType().docValueFormat(null, DateTimeZone.UTC)
+                            .parseDouble(config.missing().toString(), false, context.nowCallable());
                 } else {
-                    missing = Double.parseDouble(config.missing.toString());
+                    missing = Double.parseDouble(config.missing().toString());
                 }
             }
             return (VS) MissingValues.replaceMissing((ValuesSource.Numeric) vs, missing);
         } else if (vs instanceof ValuesSource.GeoPoint) {
             // TODO: also support the structured formats of geo points
-            final GeoPoint missing = GeoUtils.parseGeoPoint(config.missing.toString(), new GeoPoint());
+            final GeoPoint missing = GeoUtils.parseGeoPoint(config.missing().toString(), new GeoPoint());
             return (VS) MissingValues.replaceMissing((ValuesSource.GeoPoint) vs, missing);
         } else {
             // Should not happen
@@ -116,21 +117,21 @@ public class AggregationContext {
      * Return the original values source, before we apply `missing`.
      */
     private <VS extends ValuesSource> VS originalValuesSource(ValuesSourceConfig<VS> config) throws IOException {
-        if (config.fieldContext == null) {
-            if (config.valueSourceType == ValuesSourceType.NUMERIC) {
+        if (config.fieldContext() == null) {
+            if (config.valueSourceType() == ValuesSourceType.NUMERIC) {
                 return (VS) numericScript(config);
             }
-            if (config.valueSourceType == ValuesSourceType.BYTES) {
+            if (config.valueSourceType() == ValuesSourceType.BYTES) {
                 return (VS) bytesScript(config);
             }
-            throw new AggregationExecutionException("value source of type [" + config.valueSourceType.name()
+            throw new AggregationExecutionException("value source of type [" + config.valueSourceType().name()
                     + "] is not supported by scripts");
         }
 
-        if (config.valueSourceType == ValuesSourceType.NUMERIC) {
+        if (config.valueSourceType() == ValuesSourceType.NUMERIC) {
             return (VS) numericField(config);
         }
-        if (config.valueSourceType == ValuesSourceType.GEOPOINT) {
+        if (config.valueSourceType() == ValuesSourceType.GEOPOINT) {
             return (VS) geoPointField(config);
         }
         // falling back to bytes values
@@ -138,25 +139,25 @@ public class AggregationContext {
     }
 
     private ValuesSource.Numeric numericScript(ValuesSourceConfig<?> config) throws IOException {
-        return new ValuesSource.Numeric.Script(config.script, config.scriptValueType);
+        return new ValuesSource.Numeric.Script(config.script(), config.scriptValueType());
     }
 
     private ValuesSource.Numeric numericField(ValuesSourceConfig<?> config) throws IOException {
 
-        if (!(config.fieldContext.indexFieldData() instanceof IndexNumericFieldData)) {
-            throw new IllegalArgumentException("Expected numeric type on field [" + config.fieldContext.field() +
-                    "], but got [" + config.fieldContext.fieldType().typeName() + "]");
+        if (!(config.fieldContext().indexFieldData() instanceof IndexNumericFieldData)) {
+            throw new IllegalArgumentException("Expected numeric type on field [" + config.fieldContext().field() +
+                    "], but got [" + config.fieldContext().fieldType().typeName() + "]");
         }
 
-        ValuesSource.Numeric dataSource = new ValuesSource.Numeric.FieldData((IndexNumericFieldData) config.fieldContext.indexFieldData());
-        if (config.script != null) {
-            dataSource = new ValuesSource.Numeric.WithScript(dataSource, config.script);
+        ValuesSource.Numeric dataSource = new ValuesSource.Numeric.FieldData((IndexNumericFieldData)config.fieldContext().indexFieldData());
+        if (config.script() != null) {
+            dataSource = new ValuesSource.Numeric.WithScript(dataSource, config.script());
         }
         return dataSource;
     }
 
     private ValuesSource bytesField(ValuesSourceConfig<?> config) throws IOException {
-        final IndexFieldData<?> indexFieldData = config.fieldContext.indexFieldData();
+        final IndexFieldData<?> indexFieldData = config.fieldContext().indexFieldData();
         ValuesSource dataSource;
         if (indexFieldData instanceof ParentChildIndexFieldData) {
             dataSource = new ValuesSource.Bytes.WithOrdinals.ParentChild((ParentChildIndexFieldData) indexFieldData);
@@ -165,24 +166,24 @@ public class AggregationContext {
         } else {
             dataSource = new ValuesSource.Bytes.FieldData(indexFieldData);
         }
-        if (config.script != null) {
-            dataSource = new ValuesSource.WithScript(dataSource, config.script);
+        if (config.script() != null) {
+            dataSource = new ValuesSource.WithScript(dataSource, config.script());
         }
         return dataSource;
     }
 
     private ValuesSource.Bytes bytesScript(ValuesSourceConfig<?> config) throws IOException {
-        return new ValuesSource.Bytes.Script(config.script);
+        return new ValuesSource.Bytes.Script(config.script());
     }
 
     private ValuesSource.GeoPoint geoPointField(ValuesSourceConfig<?> config) throws IOException {
 
-        if (!(config.fieldContext.indexFieldData() instanceof IndexGeoPointFieldData)) {
-            throw new IllegalArgumentException("Expected geo_point type on field [" + config.fieldContext.field() +
-                    "], but got [" + config.fieldContext.fieldType().typeName() + "]");
+        if (!(config.fieldContext().indexFieldData() instanceof IndexGeoPointFieldData)) {
+            throw new IllegalArgumentException("Expected geo_point type on field [" + config.fieldContext().field() +
+                    "], but got [" + config.fieldContext().fieldType().typeName() + "]");
         }
 
-        return new ValuesSource.GeoPoint.Fielddata((IndexGeoPointFieldData) config.fieldContext.indexFieldData());
+        return new ValuesSource.GeoPoint.Fielddata((IndexGeoPointFieldData) config.fieldContext().indexFieldData());
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
index 582ae6befe0..cd03ceb9eca 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
@@ -317,7 +317,7 @@ public abstract class ValuesSourceAggregationBuilder<VS extends ValuesSource, AB
             if (script == null) {
                 @SuppressWarnings("unchecked")
                 ValuesSourceConfig<VS> config = new ValuesSourceConfig(ValuesSourceType.ANY);
-                config.format = resolveFormat(null, valueType);
+                config.format(resolveFormat(null, valueType));
                 return config;
             }
             ValuesSourceType valuesSourceType = valueType != null ? valueType.getValuesSourceType() : this.valuesSourceType;
@@ -329,11 +329,11 @@ public abstract class ValuesSourceAggregationBuilder<VS extends ValuesSource, AB
                 valuesSourceType = ValuesSourceType.BYTES;
             }
             ValuesSourceConfig<VS> config = new ValuesSourceConfig<VS>(valuesSourceType);
-            config.missing = missing;
-            config.timeZone = timeZone;
-            config.format = resolveFormat(format, valueType);
-            config.script = createScript(script, context.searchContext());
-            config.scriptValueType = valueType;
+            config.missing(missing);
+            config.timezone(timeZone);
+            config.format(resolveFormat(format, valueType));
+            config.script(createScript(script, context.searchContext()));
+            config.scriptValueType(valueType);
             return config;
         }
 
@@ -341,13 +341,13 @@ public abstract class ValuesSourceAggregationBuilder<VS extends ValuesSource, AB
         if (fieldType == null) {
             ValuesSourceType valuesSourceType = valueType != null ? valueType.getValuesSourceType() : this.valuesSourceType;
             ValuesSourceConfig<VS> config = new ValuesSourceConfig<>(valuesSourceType);
-            config.missing = missing;
-            config.timeZone = timeZone;
-            config.format = resolveFormat(format, valueType);
-            config.unmapped = true;
+            config.missing(missing);
+            config.timezone(timeZone);
+            config.format(resolveFormat(format, valueType));
+            config.unmapped(true);
             if (valueType != null) {
                 // todo do we really need this for unmapped?
-                config.scriptValueType = valueType;
+                config.scriptValueType(valueType);
             }
             return config;
         }
@@ -367,11 +367,11 @@ public abstract class ValuesSourceAggregationBuilder<VS extends ValuesSource, AB
             config = new ValuesSourceConfig(valuesSourceType);
         }
 
-        config.fieldContext = new FieldContext(field, indexFieldData, fieldType);
-        config.missing = missing;
-        config.timeZone = timeZone;
-        config.script = createScript(script, context.searchContext());
-        config.format = fieldType.docValueFormat(format, timeZone);
+        config.fieldContext(new FieldContext(field, indexFieldData, fieldType));
+        config.missing(missing);
+        config.timezone(timeZone);
+        config.script(createScript(script, context.searchContext()));
+        config.format(fieldType.docValueFormat(format, timeZone));
         return config;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java
index c273430dda0..36ab0d505b9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregatorFactory.java
@@ -42,7 +42,7 @@ public abstract class ValuesSourceAggregatorFactory<VS extends ValuesSource, AF
     }
 
     public DateTimeZone timeZone() {
-        return config.timeZone;
+        return config.timezone();
         }
 
     @Override
@@ -62,4 +62,4 @@ public abstract class ValuesSourceAggregatorFactory<VS extends ValuesSource, AF
             boolean collectsFromSingleBucket, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
             throws IOException;
 
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
index acc2b063ac2..5e53812e726 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
@@ -27,15 +27,15 @@ import org.joda.time.DateTimeZone;
  */
 public class ValuesSourceConfig<VS extends ValuesSource> {
 
-    final ValuesSourceType valueSourceType;
-    FieldContext fieldContext;
-    SearchScript script;
-    ValueType scriptValueType;
-    boolean unmapped = false;
-    String formatPattern;
-    DocValueFormat format = DocValueFormat.RAW;
-    Object missing;
-    DateTimeZone timeZone;
+    private final ValuesSourceType valueSourceType;
+    private FieldContext fieldContext;
+    private SearchScript script;
+    private ValueType scriptValueType;
+    private boolean unmapped = false;
+    private String formatPattern;
+    private DocValueFormat format = DocValueFormat.RAW;
+    private Object missing;
+    private DateTimeZone timeZone;
 
     public ValuesSourceConfig(ValuesSourceType valueSourceType) {
         this.valueSourceType = valueSourceType;
@@ -71,6 +71,15 @@ public class ValuesSourceConfig<VS extends ValuesSource> {
         return this;
     }
 
+    public ValuesSourceConfig<VS> scriptValueType(ValueType scriptValueType) {
+        this.scriptValueType = scriptValueType;
+        return this;
+    }
+
+    public ValueType scriptValueType() {
+        return this.scriptValueType;
+    }
+
     public ValuesSourceConfig<VS> unmapped(boolean unmapped) {
         this.unmapped = unmapped;
         return this;
@@ -86,11 +95,19 @@ public class ValuesSourceConfig<VS extends ValuesSource> {
         return this;
     }
 
+    public Object missing() {
+        return this.missing;
+    }
+
     public ValuesSourceConfig<VS> timezone(final DateTimeZone timeZone) {
         this.timeZone= timeZone;
         return this;
     }
 
+    public DateTimeZone timezone() {
+        return this.timeZone;
+    }
+
     public DocValueFormat format() {
         return format;
     }
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
index e4ea8df0497..51e5ce1cf27 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
@@ -30,7 +30,7 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregationInitializationException;
-import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
@@ -48,7 +48,7 @@ import java.util.Objects;
  *
  */
 public abstract class MultiValuesSourceAggregationBuilder<VS extends ValuesSource, AB extends MultiValuesSourceAggregationBuilder<VS, AB>>
-        extends AggregationBuilder<AB> {
+        extends AbstractAggregationBuilder<AB> {
 
     public static final ParseField MULTIVALUE_MODE_FIELD = new ParseField("mode");
 

From 2d57bbdacd016c1ff7af25752c7b3a0d55118e56 Mon Sep 17 00:00:00 2001
From: Nicholas Knize <nknize@gmail.com>
Date: Wed, 1 Jun 2016 17:30:20 -0500
Subject: [PATCH 0560/1311] remove unrecognized javadoc tag from matrix
 aggregation module

---
 .../search/aggregations/matrix/stats/MatrixStatsResults.java    | 2 --
 .../search/aggregations/matrix/stats/RunningStats.java          | 2 --
 2 files changed, 4 deletions(-)

diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java
index 2652b523bde..96b7b74ab4e 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java
@@ -31,8 +31,6 @@ import java.util.Map;
 /**
  * Descriptive stats gathered per shard. Coordinating node computes final pearson product coefficient
  * based on these descriptive stats
- *
- * @internal
  */
 class MatrixStatsResults implements Writeable {
     /** object holding results - computes results in place */
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStats.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStats.java
index 1aa36e38bd6..0019bfef5d6 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStats.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/RunningStats.java
@@ -34,8 +34,6 @@ import java.util.Map;
  * based on these descriptive stats. This single pass, parallel approach is based on:
  *
  * http://prod.sandia.gov/techlib/access-control.cgi/2008/086212.pdf
- *
- * @internal
  */
 public class RunningStats implements Writeable, Cloneable {
     /** count of observations (same number of observations per field) */

From 56d86ef875533a85738c514340ecb87004fbd7da Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Thu, 2 Jun 2016 10:18:16 +0200
Subject: [PATCH 0561/1311] Add upgrade-not-supported warning to alpha release
 notes

---
 docs/reference/release-notes/5.0.0-alpha1.asciidoc | 1 +
 docs/reference/release-notes/5.0.0-alpha2.asciidoc | 2 ++
 docs/reference/release-notes/5.0.0-alpha3.asciidoc | 2 ++
 3 files changed, 5 insertions(+)

diff --git a/docs/reference/release-notes/5.0.0-alpha1.asciidoc b/docs/reference/release-notes/5.0.0-alpha1.asciidoc
index c1fe00d1379..8d1db3e09a5 100644
--- a/docs/reference/release-notes/5.0.0-alpha1.asciidoc
+++ b/docs/reference/release-notes/5.0.0-alpha1.asciidoc
@@ -7,6 +7,7 @@ in the 2.x series are listed in <<release-notes-5.0.0-alpha1-2x>>.
 
 Also see <<breaking-changes-5.0>>.
 
+IMPORTANT: This is an alpha release and is intended for _testing purposes only_. Indices created in this version will *not be compatible with Elasticsearch 5.0.0 GA*. Upgrading 5.0.0-alpha1 to any other version is not supported.
 
 [[breaking-5.0.0-alpha1]]
 [float]
diff --git a/docs/reference/release-notes/5.0.0-alpha2.asciidoc b/docs/reference/release-notes/5.0.0-alpha2.asciidoc
index ad6c62653f0..812daee137c 100644
--- a/docs/reference/release-notes/5.0.0-alpha2.asciidoc
+++ b/docs/reference/release-notes/5.0.0-alpha2.asciidoc
@@ -3,6 +3,8 @@
 
 Also see <<breaking-changes-5.0>>.
 
+IMPORTANT: This is an alpha release and is intended for _testing purposes only_. Indices created in this version will *not be compatible with Elasticsearch 5.0.0 GA*. Upgrading 5.0.0-alpha2 to any other version is not supported.
+
 [[breaking-5.0.0-alpha2]]
 [float]
 === Breaking changes
diff --git a/docs/reference/release-notes/5.0.0-alpha3.asciidoc b/docs/reference/release-notes/5.0.0-alpha3.asciidoc
index 09c6f384c4f..69802c8a4cc 100644
--- a/docs/reference/release-notes/5.0.0-alpha3.asciidoc
+++ b/docs/reference/release-notes/5.0.0-alpha3.asciidoc
@@ -3,6 +3,8 @@
 
 Also see <<breaking-changes-5.0>>.
 
+IMPORTANT: This is an alpha release and is intended for _testing purposes only_. Indices created in this version will *not be compatible with Elasticsearch 5.0.0 GA*. Upgrading 5.0.0-alpha3 to any other version is not supported.
+
 [[breaking-5.0.0-alpha3]]
 [float]
 === Breaking changes

From 3ccd59592a46aa26b0675025248a69b4ade3d516 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 2 Jun 2016 05:03:28 -0400
Subject: [PATCH 0562/1311] Remove allow running as root

This commit removes the escape hatch for running Elasticsearch as root.

Relates #18694
---
 .../main/java/org/elasticsearch/bootstrap/Bootstrap.java    | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
index 2688e8b012e..54cea62e12e 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
@@ -86,11 +86,7 @@ final class Bootstrap {
 
         // check if the user is running as root, and bail
         if (Natives.definitelyRunningAsRoot()) {
-            if (Boolean.parseBoolean(System.getProperty("es.insecure.allow.root"))) {
-                logger.warn("running as ROOT user. this is a bad idea!");
-            } else {
-                throw new RuntimeException("don't run elasticsearch as root.");
-            }
+            throw new RuntimeException("can not run elasticsearch as root");
         }
 
         // enable secure computing mode

From b2724c0d08267486efc261a30bdb686cb9977f5d Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Thu, 2 Jun 2016 11:23:44 +0200
Subject: [PATCH 0563/1311] Mute failing assertions in
 IndexWithShadowReplicasIT until fix

---
 .../org/elasticsearch/index/IndexWithShadowReplicasIT.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
index 815b356fcd2..1b7c71c644f 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
@@ -587,7 +587,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
         logger.info("--> deleting index " + IDX);
         assertAcked(client().admin().indices().prepareDelete(IDX));
 
-        assertBusy(() -> assertPathHasBeenCleared(dataPath), 1, TimeUnit.MINUTES);
+        // assertBusy(() -> assertPathHasBeenCleared(dataPath), 1, TimeUnit.MINUTES);
         //norelease
         //TODO: uncomment the test below when https://github.com/elastic/elasticsearch/issues/17695 is resolved.
         //assertIndicesDirsDeleted(nodes);
@@ -648,7 +648,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
 
         assertAcked(client().admin().indices().prepareDelete(IDX));
 
-        assertBusy(() -> assertPathHasBeenCleared(dataPath), 1, TimeUnit.MINUTES);
+        // assertBusy(() -> assertPathHasBeenCleared(dataPath), 1, TimeUnit.MINUTES);
         //norelease
         //TODO: uncomment the test below when https://github.com/elastic/elasticsearch/issues/17695 is resolved.
         //assertIndicesDirsDeleted(nodes);

From 359f45988fa5550ff44a0f805fa050eb9a221405 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Wed, 20 Apr 2016 13:19:12 +0200
Subject: [PATCH 0564/1311] Handle empty query bodies at parse time and remove
 EmptyQueryBuilder

Currently we support empty query clauses like the filter in

"constant_score" : {  "filter" : { } }

How these clauses are handled depends on the surrounding query.
They later are either ignored, converted to match all or no documents or
passed up further in the query hierarchy. During parsing these claues are
currently represented as EmptyQueryBuilders. When not handled anywhere else,
these special cases need to be checked for on the shard when building the
lucene query.

This is trappy, so this PR changes the parsing of compound queries. Instead
of returning QueryBuilder, the core query parsing method
QueryShardContext#parseInnerQueryBuilder() now return an Optional which can
be empty in the case of empty query clauses. This has the advantage of forcing
callers to deal with this sooner or later. When encountering empty Optionals,
compound query builders now have the choice to ignore them, pass them on or
rewrite to a different query, depending on context.
---
 .../cluster/metadata/AliasValidator.java      |  8 +-
 .../org/elasticsearch/index/IndexService.java |  8 +-
 .../index/query/BoolQueryBuilder.java         | 30 +++----
 .../index/query/BoostingQueryBuilder.java     | 14 ++--
 .../index/query/CommonTermsQueryBuilder.java  |  7 +-
 .../query/ConstantScoreQueryBuilder.java      | 14 +++-
 .../index/query/DisMaxQueryBuilder.java       | 11 ++-
 .../index/query/ExistsQueryBuilder.java       |  5 +-
 .../query/FieldMaskingSpanQueryBuilder.java   | 11 +--
 .../index/query/FuzzyQueryBuilder.java        |  7 +-
 .../query/GeoBoundingBoxQueryBuilder.java     |  5 +-
 .../index/query/GeoDistanceQueryBuilder.java  |  5 +-
 .../query/GeoDistanceRangeQueryBuilder.java   |  5 +-
 .../index/query/GeoPolygonQueryBuilder.java   |  5 +-
 .../index/query/GeoShapeQueryBuilder.java     |  5 +-
 .../index/query/GeohashCellQuery.java         |  7 +-
 .../index/query/HasChildQueryBuilder.java     | 15 +++-
 .../index/query/HasParentQueryBuilder.java    | 13 ++-
 .../index/query/IdsQueryBuilder.java          |  5 +-
 .../index/query/IndicesQueryBuilder.java      | 12 +--
 .../index/query/InnerHitBuilder.java          |  5 +-
 .../index/query/MatchAllQueryBuilder.java     |  5 +-
 .../index/query/MatchNoneQueryBuilder.java    |  5 +-
 .../query/MatchPhrasePrefixQueryBuilder.java  |  5 +-
 .../index/query/MatchPhraseQueryBuilder.java  |  6 +-
 .../index/query/MatchQueryBuilder.java        |  5 +-
 .../index/query/MoreLikeThisQueryBuilder.java |  4 +-
 .../index/query/MultiMatchQueryBuilder.java   |  7 +-
 .../index/query/NestedQueryBuilder.java       | 14 +++-
 .../index/query/ParentIdQueryBuilder.java     |  5 +-
 .../index/query/PrefixQueryBuilder.java       |  7 +-
 .../index/query/QueryParseContext.java        | 22 +++--
 .../index/query/QueryParser.java              | 10 ++-
 .../index/query/QueryStringQueryBuilder.java  |  5 +-
 .../index/query/RangeQueryBuilder.java        |  5 +-
 .../index/query/RegexpQueryBuilder.java       |  7 +-
 .../index/query/ScriptQueryBuilder.java       |  9 +-
 .../index/query/SimpleQueryStringBuilder.java |  5 +-
 .../query/SpanContainingQueryBuilder.java     | 17 ++--
 .../index/query/SpanFirstQueryBuilder.java    | 11 +--
 .../query/SpanMultiTermQueryBuilder.java      | 11 +--
 .../index/query/SpanNearQueryBuilder.java     | 11 +--
 .../index/query/SpanNotQueryBuilder.java      | 17 ++--
 .../index/query/SpanOrQueryBuilder.java       | 11 +--
 .../index/query/SpanTermQueryBuilder.java     |  5 +-
 .../index/query/SpanWithinQueryBuilder.java   | 17 ++--
 .../index/query/TemplateQueryBuilder.java     |  9 +-
 .../index/query/TermQueryBuilder.java         |  5 +-
 .../index/query/TermsQueryBuilder.java        |  7 +-
 .../index/query/TypeQueryBuilder.java         |  7 +-
 .../index/query/WildcardQueryBuilder.java     |  7 +-
 .../index/query/WrapperQueryBuilder.java      |  8 +-
 .../FunctionScoreQueryBuilder.java            | 16 ++--
 .../elasticsearch/search/SearchModule.java    |  4 -
 .../filter/FilterAggregationBuilder.java      | 20 +----
 .../filters/FiltersAggregationBuilder.java    |  9 +-
 .../bucket/filters/FiltersAggregator.java     |  7 +-
 .../significant/SignificantTermsParser.java   |  7 +-
 .../search/builder/SearchSourceBuilder.java   |  6 +-
 .../highlight/AbstractHighlighterBuilder.java |  2 +-
 .../search/rescore/QueryRescorerBuilder.java  |  3 +-
 .../search/sort/FieldSortBuilder.java         |  7 +-
 .../search/sort/GeoDistanceSortBuilder.java   | 11 +--
 .../search/sort/ScriptSortBuilder.java        |  7 +-
 .../suggest/phrase/PhraseSuggester.java       |  6 +-
 .../index/query/BoolQueryBuilderTests.java    | 25 ++++++
 .../query/BoostingQueryBuilderTests.java      | 39 +++++++++
 .../query/ConstantScoreQueryBuilderTests.java | 25 ++++++
 .../index/query/DisMaxQueryBuilderTests.java  | 18 ++--
 .../query/HasChildQueryBuilderTests.java      | 43 +++++++---
 .../query/HasParentQueryBuilderTests.java     | 36 ++++++--
 .../index/query/NestedQueryBuilderTests.java  | 10 +--
 .../index/query/RandomQueryBuilder.java       |  4 +-
 .../index/query/TermsQueryBuilderTests.java   |  8 +-
 .../FunctionScoreQueryBuilderTests.java       | 27 +++++-
 .../query/plugin/CustomQueryParserIT.java     | 22 ++---
 .../query/plugin/DummyQueryBuilder.java}      | 83 +++++++++----------
 .../query/plugin/DummyQueryParserPlugin.java  | 61 +-------------
 .../search/aggregations/bucket/FilterIT.java  | 21 ++++-
 .../search/aggregations/bucket/FiltersIT.java | 26 ++++--
 .../builder/SearchSourceBuilderTests.java     | 24 ++++--
 .../messy/tests/TemplateQueryParserTests.java | 12 +--
 .../percolator/PercolateQueryBuilder.java     |  7 +-
 .../percolator/PercolatorFieldMapper.java     |  3 +-
 .../percolator/TransportPercolateAction.java  |  4 +-
 .../PercolatorFieldMapperTests.java           |  2 +-
 .../test/AbstractQueryTestCase.java           | 26 +++---
 87 files changed, 653 insertions(+), 444 deletions(-)
 rename core/src/{main/java/org/elasticsearch/index/query/EmptyQueryBuilder.java => test/java/org/elasticsearch/index/query/plugin/DummyQueryBuilder.java} (55%)

diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java
index 735916504da..647e30cd85c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java
@@ -32,6 +32,7 @@ import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.indices.InvalidAliasNameException;
 
 import java.io.IOException;
+import java.util.Optional;
 
 /**
  * Validator for an alias, to be used before adding an alias to the index metadata
@@ -141,7 +142,10 @@ public class AliasValidator extends AbstractComponent {
 
     private static void validateAliasFilter(XContentParser parser, QueryShardContext queryShardContext) throws IOException {
         QueryParseContext queryParseContext = queryShardContext.newParseContext(parser);
-        QueryBuilder queryBuilder = QueryBuilder.rewriteQuery(queryParseContext.parseInnerQueryBuilder(), queryShardContext);
-        queryBuilder.toFilter(queryShardContext);
+        Optional<QueryBuilder> parseInnerQueryBuilder = queryParseContext.parseInnerQueryBuilder();
+        if (parseInnerQueryBuilder.isPresent()) {
+            QueryBuilder queryBuilder = QueryBuilder.rewriteQuery(parseInnerQueryBuilder.get(), queryShardContext);
+            queryBuilder.toFilter(queryShardContext);
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java
index f5e5ce91d80..e2d2ea4b8fe 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexService.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexService.java
@@ -51,6 +51,7 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.ParsedQuery;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexSearcherWrapper;
@@ -80,6 +81,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
@@ -601,7 +603,11 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
         try {
             byte[] filterSource = alias.filter().uncompressed();
             try (XContentParser parser = XContentFactory.xContent(filterSource).createParser(filterSource)) {
-                ParsedQuery parsedFilter = shardContext.toFilter(shardContext.newParseContext(parser).parseInnerQueryBuilder());
+                Optional<QueryBuilder> innerQueryBuilder = shardContext.newParseContext(parser).parseInnerQueryBuilder();
+                ParsedQuery parsedFilter = null;
+                if (innerQueryBuilder.isPresent()) {
+                    parsedFilter = shardContext.toFilter(innerQueryBuilder.get());
+                }
                 return parsedFilter == null ? null : parsedFilter.query();
             }
         } catch (IOException ex) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
index e5aa774addc..863bdb25c71 100644
--- a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
@@ -37,6 +37,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.function.Consumer;
 
 import static org.elasticsearch.common.lucene.search.Queries.fixNegativeQueryIfNeeded;
@@ -300,7 +301,7 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
         builder.endArray();
     }
 
-    public static BoolQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException, ParsingException {
+    public static Optional<BoolQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException, ParsingException {
         XContentParser parser = parseContext.parser();
 
         boolean disableCoord = BoolQueryBuilder.DISABLE_COORD_DEFAULT;
@@ -316,7 +317,6 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
 
         String currentFieldName = null;
         XContentParser.Token token;
-        QueryBuilder query;
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
@@ -325,21 +325,17 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
             } else if (token == XContentParser.Token.START_OBJECT) {
                 switch (currentFieldName) {
                 case MUST:
-                    query = parseContext.parseInnerQueryBuilder();
-                    mustClauses.add(query);
+                    parseContext.parseInnerQueryBuilder().ifPresent(mustClauses::add);
                     break;
                 case SHOULD:
-                    query = parseContext.parseInnerQueryBuilder();
-                    shouldClauses.add(query);
+                    parseContext.parseInnerQueryBuilder().ifPresent(shouldClauses::add);
                     break;
                 case FILTER:
-                    query = parseContext.parseInnerQueryBuilder();
-                    filterClauses.add(query);
+                    parseContext.parseInnerQueryBuilder().ifPresent(filterClauses::add);
                     break;
                 case MUST_NOT:
                 case MUSTNOT:
-                    query = parseContext.parseInnerQueryBuilder();
-                    mustNotClauses.add(query);
+                    parseContext.parseInnerQueryBuilder().ifPresent(mustNotClauses::add);
                     break;
                 default:
                     throw new ParsingException(parser.getTokenLocation(), "[bool] query does not support [" + currentFieldName + "]");
@@ -348,21 +344,17 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
                 while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                     switch (currentFieldName) {
                     case MUST:
-                        query = parseContext.parseInnerQueryBuilder();
-                        mustClauses.add(query);
+                        parseContext.parseInnerQueryBuilder().ifPresent(mustClauses::add);
                         break;
                     case SHOULD:
-                        query = parseContext.parseInnerQueryBuilder();
-                        shouldClauses.add(query);
+                        parseContext.parseInnerQueryBuilder().ifPresent(shouldClauses::add);
                         break;
                     case FILTER:
-                        query = parseContext.parseInnerQueryBuilder();
-                        filterClauses.add(query);
+                        parseContext.parseInnerQueryBuilder().ifPresent(filterClauses::add);
                         break;
                     case MUST_NOT:
                     case MUSTNOT:
-                        query = parseContext.parseInnerQueryBuilder();
-                        mustNotClauses.add(query);
+                        parseContext.parseInnerQueryBuilder().ifPresent(mustNotClauses::add);
                         break;
                     default:
                         throw new ParsingException(parser.getTokenLocation(), "bool query does not support [" + currentFieldName + "]");
@@ -404,7 +396,7 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
         boolQuery.adjustPureNegative(adjustPureNegative);
         boolQuery.minimumNumberShouldMatch(minimumShouldMatch);
         boolQuery.queryName(queryName);
-        return boolQuery;
+        return Optional.of(boolQuery);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java
index 496cb7ec8a7..3afa4339b5c 100644
--- a/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java
@@ -31,6 +31,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import java.io.IOException;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * The BoostingQuery class can be used to effectively demote results that match a given query.
@@ -138,12 +139,12 @@ public class BoostingQueryBuilder extends AbstractQueryBuilder<BoostingQueryBuil
         builder.endObject();
     }
 
-    public static BoostingQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<BoostingQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
-        QueryBuilder positiveQuery = null;
+        Optional<QueryBuilder> positiveQuery = null;
         boolean positiveQueryFound = false;
-        QueryBuilder negativeQuery = null;
+        Optional<QueryBuilder> negativeQuery = null;
         boolean negativeQueryFound = false;
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
         float negativeBoost = -1;
@@ -187,12 +188,15 @@ public class BoostingQueryBuilder extends AbstractQueryBuilder<BoostingQueryBuil
             throw new ParsingException(parser.getTokenLocation(),
                     "[boosting] query requires 'negative_boost' to be set to be a positive value'");
         }
+        if (positiveQuery.isPresent() == false || negativeQuery.isPresent() == false) {
+            return Optional.empty();
+        }
 
-        BoostingQueryBuilder boostingQuery = new BoostingQueryBuilder(positiveQuery, negativeQuery);
+        BoostingQueryBuilder boostingQuery = new BoostingQueryBuilder(positiveQuery.get(), negativeQuery.get());
         boostingQuery.negativeBoost(negativeBoost);
         boostingQuery.boost(boost);
         boostingQuery.queryName(queryName);
-        return boostingQuery;
+        return Optional.of(boostingQuery);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
index f7b35d8dfa2..692c157c1b1 100644
--- a/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
@@ -40,6 +40,7 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * CommonTermsQuery query is a query that executes high-frequency terms in a
@@ -263,7 +264,7 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQue
         builder.endObject();
     }
 
-    public static CommonTermsQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<CommonTermsQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
         XContentParser.Token token = parser.nextToken();
         if (token != XContentParser.Token.FIELD_NAME) {
@@ -352,7 +353,7 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQue
         if (text == null) {
             throw new ParsingException(parser.getTokenLocation(), "No text specified for text query");
         }
-        return new CommonTermsQueryBuilder(fieldName, text)
+        return Optional.of(new CommonTermsQueryBuilder(fieldName, text)
                 .lowFreqMinimumShouldMatch(lowFreqMinimumShouldMatch)
                 .highFreqMinimumShouldMatch(highFreqMinimumShouldMatch)
                 .analyzer(analyzer)
@@ -361,7 +362,7 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQue
                 .disableCoord(disableCoord)
                 .cutoffFrequency(cutoffFrequency)
                 .boost(boost)
-                .queryName(queryName);
+                .queryName(queryName));
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java
index 306a4568576..44b4458b572 100644
--- a/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java
@@ -31,6 +31,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import java.io.IOException;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * A query that wraps a filter and simply returns a constant score equal to the
@@ -87,10 +88,10 @@ public class ConstantScoreQueryBuilder extends AbstractQueryBuilder<ConstantScor
         builder.endObject();
     }
 
-    public static ConstantScoreQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<ConstantScoreQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
-        QueryBuilder query = null;
+        Optional<QueryBuilder> query = Optional.empty();
         boolean queryFound = false;
         String queryName = null;
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
@@ -131,10 +132,15 @@ public class ConstantScoreQueryBuilder extends AbstractQueryBuilder<ConstantScor
             throw new ParsingException(parser.getTokenLocation(), "[constant_score] requires a 'filter' element");
         }
 
-        ConstantScoreQueryBuilder constantScoreBuilder = new ConstantScoreQueryBuilder(query);
+        if (query.isPresent() == false) {
+            // if inner query is empty, bubble this up to caller so they can decide how to deal with it
+            return Optional.empty();
+        }
+
+        ConstantScoreQueryBuilder constantScoreBuilder = new ConstantScoreQueryBuilder(query.get());
         constantScoreBuilder.boost(boost);
         constantScoreBuilder.queryName(queryName);
-        return constantScoreBuilder;
+        return Optional.of(constantScoreBuilder);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java
index f38b2c09b16..e480417e8fb 100644
--- a/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java
@@ -33,6 +33,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * A query that generates the union of documents produced by its sub-queries, and that scores each document
@@ -122,7 +123,7 @@ public class DisMaxQueryBuilder extends AbstractQueryBuilder<DisMaxQueryBuilder>
         builder.endObject();
     }
 
-    public static DisMaxQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<DisMaxQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
@@ -140,8 +141,7 @@ public class DisMaxQueryBuilder extends AbstractQueryBuilder<DisMaxQueryBuilder>
             } else if (token == XContentParser.Token.START_OBJECT) {
                 if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERIES_FIELD)) {
                     queriesFound = true;
-                    QueryBuilder query = parseContext.parseInnerQueryBuilder();
-                    queries.add(query);
+                    parseContext.parseInnerQueryBuilder().ifPresent(queries::add);
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[dis_max] query does not support [" + currentFieldName + "]");
                 }
@@ -149,8 +149,7 @@ public class DisMaxQueryBuilder extends AbstractQueryBuilder<DisMaxQueryBuilder>
                 if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERIES_FIELD)) {
                     queriesFound = true;
                     while (token != XContentParser.Token.END_ARRAY) {
-                        QueryBuilder query = parseContext.parseInnerQueryBuilder();
-                        queries.add(query);
+                        parseContext.parseInnerQueryBuilder().ifPresent(queries::add);
                         token = parser.nextToken();
                     }
                 } else {
@@ -180,7 +179,7 @@ public class DisMaxQueryBuilder extends AbstractQueryBuilder<DisMaxQueryBuilder>
         for (QueryBuilder query : queries) {
             disMaxQuery.add(query);
         }
-        return disMaxQuery;
+        return Optional.of(disMaxQuery);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java
index afcff164bec..c933bae68c0 100644
--- a/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java
@@ -37,6 +37,7 @@ import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * Constructs a query that only match on documents that the field has a value in them.
@@ -85,7 +86,7 @@ public class ExistsQueryBuilder extends AbstractQueryBuilder<ExistsQueryBuilder>
         builder.endObject();
     }
 
-    public static ExistsQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<ExistsQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         String fieldPattern = null;
@@ -121,7 +122,7 @@ public class ExistsQueryBuilder extends AbstractQueryBuilder<ExistsQueryBuilder>
         ExistsQueryBuilder builder = new ExistsQueryBuilder(fieldPattern);
         builder.queryName(queryName);
         builder.boost(boost);
-        return builder;
+        return Optional.of(builder);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java
index 0382f353cb3..801b0ae5a81 100644
--- a/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java
@@ -33,6 +33,7 @@ import org.elasticsearch.index.mapper.MappedFieldType;
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Optional;
 
 public class FieldMaskingSpanQueryBuilder extends AbstractQueryBuilder<FieldMaskingSpanQueryBuilder>
         implements SpanQueryBuilder {
@@ -103,7 +104,7 @@ public class FieldMaskingSpanQueryBuilder extends AbstractQueryBuilder<FieldMask
         builder.endObject();
     }
 
-    public static FieldMaskingSpanQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<FieldMaskingSpanQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
@@ -119,11 +120,11 @@ public class FieldMaskingSpanQueryBuilder extends AbstractQueryBuilder<FieldMask
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
                 if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
-                    QueryBuilder query = parseContext.parseInnerQueryBuilder();
-                    if (!(query instanceof SpanQueryBuilder)) {
+                    Optional<QueryBuilder> query = parseContext.parseInnerQueryBuilder();
+                    if (query.isPresent() == false || query.get() instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "[field_masking_span] query must be of type span query");
                     }
-                    inner = (SpanQueryBuilder) query;
+                    inner = (SpanQueryBuilder) query.get();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[field_masking_span] query does not support ["
                             + currentFieldName + "]");
@@ -151,7 +152,7 @@ public class FieldMaskingSpanQueryBuilder extends AbstractQueryBuilder<FieldMask
         FieldMaskingSpanQueryBuilder queryBuilder = new FieldMaskingSpanQueryBuilder(inner, field);
         queryBuilder.boost(boost);
         queryBuilder.queryName(queryName);
-        return queryBuilder;
+        return Optional.of(queryBuilder);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryBuilder.java
index 70b4f5eb7f5..6e60c19d2fe 100644
--- a/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryBuilder.java
@@ -37,6 +37,7 @@ import org.elasticsearch.index.query.support.QueryParsers;
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * A Query that does fuzzy matching for a specific value.
@@ -257,7 +258,7 @@ public class FuzzyQueryBuilder extends AbstractQueryBuilder<FuzzyQueryBuilder> i
         builder.endObject();
     }
 
-    public static FuzzyQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<FuzzyQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         XContentParser.Token token = parser.nextToken();
@@ -317,14 +318,14 @@ public class FuzzyQueryBuilder extends AbstractQueryBuilder<FuzzyQueryBuilder> i
         if (value == null) {
             throw new ParsingException(parser.getTokenLocation(), "no value specified for fuzzy query");
         }
-        return new FuzzyQueryBuilder(fieldName, value)
+        return Optional.of(new FuzzyQueryBuilder(fieldName, value)
                 .fuzziness(fuzziness)
                 .prefixLength(prefixLength)
                 .maxExpansions(maxExpansions)
                 .transpositions(transpositions)
                 .rewrite(rewrite)
                 .boost(boost)
-                .queryName(queryName);
+                .queryName(queryName));
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
index 7f5d5e2ec8d..c1925fe3f39 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java
@@ -43,6 +43,7 @@ import org.elasticsearch.index.search.geo.IndexedGeoBoundingBoxQuery;
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * Creates a Lucene query that will filter for all documents that lie within the specified
@@ -385,7 +386,7 @@ public class GeoBoundingBoxQueryBuilder extends AbstractQueryBuilder<GeoBounding
         builder.endObject();
     }
 
-    public static GeoBoundingBoxQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<GeoBoundingBoxQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         String fieldName = null;
@@ -496,7 +497,7 @@ public class GeoBoundingBoxQueryBuilder extends AbstractQueryBuilder<GeoBounding
         } else {
             builder.setValidationMethod(GeoValidationMethod.infer(coerce, ignoreMalformed));
         }
-        return builder;
+        return Optional.of(builder);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java
index 0f2b4694e27..710cb529c07 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java
@@ -45,6 +45,7 @@ import org.elasticsearch.index.search.geo.GeoDistanceRangeQuery;
 import java.io.IOException;
 import java.util.Locale;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * Filter results of a query to include only those within a specific distance to some
@@ -330,7 +331,7 @@ public class GeoDistanceQueryBuilder extends AbstractQueryBuilder<GeoDistanceQue
         builder.endObject();
     }
 
-    public static GeoDistanceQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<GeoDistanceQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         XContentParser.Token token;
@@ -445,7 +446,7 @@ public class GeoDistanceQueryBuilder extends AbstractQueryBuilder<GeoDistanceQue
         qb.boost(boost);
         qb.queryName(queryName);
         qb.ignoreUnmapped(ignoreUnmapped);
-        return qb;
+        return Optional.of(qb);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java
index ea22461243c..06f30a3477e 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java
@@ -46,6 +46,7 @@ import org.elasticsearch.index.search.geo.GeoDistanceRangeQuery;
 import java.io.IOException;
 import java.util.Locale;
 import java.util.Objects;
+import java.util.Optional;
 
 import static org.apache.lucene.spatial.util.GeoEncodingUtils.TOLERANCE;
 
@@ -391,7 +392,7 @@ public class GeoDistanceRangeQueryBuilder extends AbstractQueryBuilder<GeoDistan
         builder.endObject();
     }
 
-    public static GeoDistanceRangeQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<GeoDistanceRangeQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         XContentParser.Token token;
@@ -603,7 +604,7 @@ public class GeoDistanceRangeQueryBuilder extends AbstractQueryBuilder<GeoDistan
             queryBuilder.setValidationMethod(GeoValidationMethod.infer(coerce, ignoreMalformed));
         }
         queryBuilder.ignoreUnmapped(ignoreUnmapped);
-        return queryBuilder;
+        return Optional.of(queryBuilder);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java
index 35ab98e25de..97c9990a9f7 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoPolygonQueryBuilder.java
@@ -43,6 +43,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 
 public class GeoPolygonQueryBuilder extends AbstractQueryBuilder<GeoPolygonQueryBuilder> {
 
@@ -241,7 +242,7 @@ public class GeoPolygonQueryBuilder extends AbstractQueryBuilder<GeoPolygonQuery
         builder.endObject();
     }
 
-    public static GeoPolygonQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<GeoPolygonQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         String fieldName = null;
@@ -322,7 +323,7 @@ public class GeoPolygonQueryBuilder extends AbstractQueryBuilder<GeoPolygonQuery
             builder.boost(boost);
         }
         builder.ignoreUnmapped(ignoreUnmapped);
-        return builder;
+        return Optional.of(builder);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
index 8b3c8a62486..0b04629e2cc 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
@@ -47,6 +47,7 @@ import org.elasticsearch.index.mapper.geo.GeoShapeFieldMapper;
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * {@link QueryBuilder} that builds a GeoShape Query
@@ -453,7 +454,7 @@ public class GeoShapeQueryBuilder extends AbstractQueryBuilder<GeoShapeQueryBuil
         builder.endObject();
     }
 
-    public static GeoShapeQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<GeoShapeQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         String fieldName = null;
@@ -559,7 +560,7 @@ public class GeoShapeQueryBuilder extends AbstractQueryBuilder<GeoShapeQueryBuil
         }
         builder.boost(boost);
         builder.ignoreUnmapped(ignoreUnmapped);
-        return builder;
+        return Optional.of(builder);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeohashCellQuery.java b/core/src/main/java/org/elasticsearch/index/query/GeohashCellQuery.java
index 4f34d6db31b..f9f21bac7b5 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeohashCellQuery.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeohashCellQuery.java
@@ -21,12 +21,12 @@ package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
-import org.elasticsearch.common.geo.GeoHashUtils;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.geo.GeoHashUtils;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -42,6 +42,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * A geohash cell filter that filters {@link GeoPoint}s by their geohashes. Basically the a
@@ -280,7 +281,7 @@ public class GeohashCellQuery {
             builder.endObject();
         }
 
-        public static Builder fromXContent(QueryParseContext parseContext) throws IOException {
+        public static Optional<Builder> fromXContent(QueryParseContext parseContext) throws IOException {
             XContentParser parser = parseContext.parser();
 
             String fieldName = null;
@@ -362,7 +363,7 @@ public class GeohashCellQuery {
                 builder.boost(boost);
             }
             builder.ignoreUnmapped(ignoreUnmapped);
-            return builder;
+            return Optional.of(builder);
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
index 990b5a35fd9..410fda41aef 100644
--- a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
@@ -43,6 +43,7 @@ import java.io.IOException;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * A query builder for <tt>has_child</tt> query.
@@ -226,7 +227,7 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
         builder.endObject();
     }
 
-    public static HasChildQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<HasChildQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
         String childType = null;
@@ -238,7 +239,7 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
         InnerHitBuilder innerHitBuilder = null;
         String currentFieldName = null;
         XContentParser.Token token;
-        QueryBuilder iqb = null;
+        Optional<QueryBuilder> iqb = Optional.empty();
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
@@ -272,7 +273,13 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
                 }
             }
         }
-        HasChildQueryBuilder hasChildQueryBuilder = new HasChildQueryBuilder(childType, iqb, scoreMode);
+
+        if (iqb.isPresent() == false) {
+            // if inner query is empty, bubble this up to caller so they can decide how to deal with it
+            return Optional.empty();
+        }
+
+        HasChildQueryBuilder hasChildQueryBuilder = new HasChildQueryBuilder(childType, iqb.get(), scoreMode);
         if (innerHitBuilder != null) {
             hasChildQueryBuilder.innerHit(innerHitBuilder);
         }
@@ -280,7 +287,7 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
         hasChildQueryBuilder.queryName(queryName);
         hasChildQueryBuilder.boost(boost);
         hasChildQueryBuilder.ignoreUnmapped(ignoreUnmapped);
-        return hasChildQueryBuilder;
+        return Optional.of(hasChildQueryBuilder);
     }
 
     public static ScoreMode parseScoreMode(String scoreModeString) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java
index 96356c276e7..7e788a16e87 100644
--- a/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java
@@ -38,6 +38,7 @@ import java.io.IOException;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 
 /**
@@ -227,7 +228,7 @@ public class HasParentQueryBuilder extends AbstractQueryBuilder<HasParentQueryBu
         builder.endObject();
     }
 
-    public static HasParentQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<HasParentQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
         String parentType = null;
@@ -238,7 +239,7 @@ public class HasParentQueryBuilder extends AbstractQueryBuilder<HasParentQueryBu
 
         String currentFieldName = null;
         XContentParser.Token token;
-        QueryBuilder iqb = null;
+        Optional<QueryBuilder> iqb = Optional.empty();
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
             if (token == XContentParser.Token.FIELD_NAME) {
                 currentFieldName = parser.currentName();
@@ -276,14 +277,18 @@ public class HasParentQueryBuilder extends AbstractQueryBuilder<HasParentQueryBu
                 }
             }
         }
-        HasParentQueryBuilder queryBuilder =  new HasParentQueryBuilder(parentType, iqb, score)
+        if (iqb.isPresent() == false) {
+            // if inner query is empty, bubble this up to caller so they can decide how to deal with it
+            return Optional.empty();
+        }
+        HasParentQueryBuilder queryBuilder =  new HasParentQueryBuilder(parentType, iqb.get(), score)
                 .ignoreUnmapped(ignoreUnmapped)
                 .queryName(queryName)
                 .boost(boost);
         if (innerHits != null) {
             queryBuilder.innerHit(innerHits);
         }
-        return queryBuilder;
+        return Optional.of(queryBuilder);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java
index 7845e2732ea..d2c847c55c0 100644
--- a/core/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java
@@ -40,6 +40,7 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 
 /**
@@ -127,7 +128,7 @@ public class IdsQueryBuilder extends AbstractQueryBuilder<IdsQueryBuilder> {
         builder.endObject();
     }
 
-    public static IdsQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<IdsQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
         List<String> ids = new ArrayList<>();
         List<String> types = new ArrayList<>();
@@ -191,7 +192,7 @@ public class IdsQueryBuilder extends AbstractQueryBuilder<IdsQueryBuilder> {
         IdsQueryBuilder query = new IdsQueryBuilder(types.toArray(new String[types.size()]));
         query.addIds(ids.toArray(new String[ids.size()]));
         query.boost(boost).queryName(queryName);
-        return query;
+        return Optional.of(query);
     }
 
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/IndicesQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/IndicesQueryBuilder.java
index 7cfdf1baa1e..9758ad3f542 100644
--- a/core/src/main/java/org/elasticsearch/index/query/IndicesQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/IndicesQueryBuilder.java
@@ -34,6 +34,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * A query that will execute the wrapped query only for the specified indices,
@@ -141,7 +142,7 @@ public class IndicesQueryBuilder extends AbstractQueryBuilder<IndicesQueryBuilde
         builder.endObject();
     }
 
-    public static IndicesQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException, ParsingException {
+    public static Optional<IndicesQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException, ParsingException {
         XContentParser parser = parseContext.parser();
 
         QueryBuilder innerQuery = null;
@@ -158,9 +159,10 @@ public class IndicesQueryBuilder extends AbstractQueryBuilder<IndicesQueryBuilde
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
                 if (parseContext.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
-                    innerQuery = parseContext.parseInnerQueryBuilder();
+                    // the 2.0 behaviour when encountering "query" : {} is to return no docs for matching indices
+                    innerQuery = parseContext.parseInnerQueryBuilder().orElse(new MatchNoneQueryBuilder());
                 } else if (parseContext.getParseFieldMatcher().match(currentFieldName, NO_MATCH_QUERY)) {
-                    noMatchQuery = parseContext.parseInnerQueryBuilder();
+                    noMatchQuery = parseContext.parseInnerQueryBuilder().orElse(defaultNoMatchQuery());
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[indices] query does not support [" + currentFieldName + "]");
                 }
@@ -203,10 +205,10 @@ public class IndicesQueryBuilder extends AbstractQueryBuilder<IndicesQueryBuilde
         if (indices.isEmpty()) {
             throw new ParsingException(parser.getTokenLocation(), "[indices] requires 'indices' or 'index' element");
         }
-        return new IndicesQueryBuilder(innerQuery, indices.toArray(new String[indices.size()]))
+        return Optional.of(new IndicesQueryBuilder(innerQuery, indices.toArray(new String[indices.size()]))
                 .noMatchQuery(noMatchQuery)
                 .boost(boost)
-                .queryName(queryName);
+                .queryName(queryName));
     }
 
     static QueryBuilder parseNoMatchQuery(String type) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
index b62b5a18a1d..a017159b451 100644
--- a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
@@ -60,6 +60,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
 
     public static final ParseField NAME_FIELD = new ParseField("name");
     public static final ParseField INNER_HITS_FIELD = new ParseField("inner_hits");
+    public static final QueryBuilder DEFAULT_INNER_HIT_QUERY = new MatchAllQueryBuilder();
 
     private final static ObjectParser<InnerHitBuilder, QueryParseContext> PARSER = new ObjectParser<>("inner_hits", InnerHitBuilder::new);
 
@@ -131,7 +132,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
     private boolean trackScores;
 
     private List<String> fieldNames;
-    private QueryBuilder query = new MatchAllQueryBuilder();
+    private QueryBuilder query = DEFAULT_INNER_HIT_QUERY;
     private List<SortBuilder<?>> sorts;
     private List<String> fieldDataFields;
     private Set<ScriptField> scriptFields;
@@ -394,7 +395,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         return this;
     }
 
-    public InnerHitBuilder addSort(SortBuilder sort) {
+    public InnerHitBuilder addSort(SortBuilder<?> sort) {
         if (sorts == null) {
             sorts = new ArrayList<>();
         }
diff --git a/core/src/main/java/org/elasticsearch/index/query/MatchAllQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MatchAllQueryBuilder.java
index 2d415c2b2a8..21742eb8771 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MatchAllQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MatchAllQueryBuilder.java
@@ -29,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.Optional;
 
 /**
  * A query that matches on all documents.
@@ -60,7 +61,7 @@ public class MatchAllQueryBuilder extends AbstractQueryBuilder<MatchAllQueryBuil
         builder.endObject();
     }
 
-    public static MatchAllQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<MatchAllQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         String currentFieldName = null;
@@ -87,7 +88,7 @@ public class MatchAllQueryBuilder extends AbstractQueryBuilder<MatchAllQueryBuil
         MatchAllQueryBuilder queryBuilder = new MatchAllQueryBuilder();
         queryBuilder.boost(boost);
         queryBuilder.queryName(queryName);
-        return queryBuilder;
+        return Optional.of(queryBuilder);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java
index 068b0e86d8e..f96c6160ee3 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MatchNoneQueryBuilder.java
@@ -29,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.Optional;
 
 /**
  * A query that matches no document.
@@ -60,7 +61,7 @@ public class MatchNoneQueryBuilder extends AbstractQueryBuilder<MatchNoneQueryBu
         builder.endObject();
     }
 
-    public static MatchNoneQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<MatchNoneQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         String currentFieldName = null;
@@ -88,7 +89,7 @@ public class MatchNoneQueryBuilder extends AbstractQueryBuilder<MatchNoneQueryBu
         MatchNoneQueryBuilder matchNoneQueryBuilder = new MatchNoneQueryBuilder();
         matchNoneQueryBuilder.boost(boost);
         matchNoneQueryBuilder.queryName(queryName);
-        return matchNoneQueryBuilder;
+        return Optional.of(matchNoneQueryBuilder);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilder.java
index cb20d448f89..002b5e23430 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MatchPhrasePrefixQueryBuilder.java
@@ -31,6 +31,7 @@ import org.elasticsearch.index.search.MatchQuery;
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * Match query is a query that analyzes the text and constructs a phrase prefix
@@ -191,7 +192,7 @@ public class MatchPhrasePrefixQueryBuilder extends AbstractQueryBuilder<MatchPhr
         return Objects.hash(fieldName, value, analyzer, slop, maxExpansions);
     }
 
-    public static MatchPhrasePrefixQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<MatchPhrasePrefixQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         XContentParser.Token token = parser.nextToken();
@@ -257,6 +258,6 @@ public class MatchPhrasePrefixQueryBuilder extends AbstractQueryBuilder<MatchPhr
         matchQuery.maxExpansions(maxExpansion);
         matchQuery.queryName(queryName);
         matchQuery.boost(boost);
-        return matchQuery;
+        return Optional.of(matchQuery);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/MatchPhraseQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MatchPhraseQueryBuilder.java
index 7fef776bbc0..2621c2dc509 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MatchPhraseQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MatchPhraseQueryBuilder.java
@@ -27,8 +27,10 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.search.MatchQuery;
+
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * Match query is a query that analyzes the text and constructs a phrase query
@@ -161,7 +163,7 @@ public class MatchPhraseQueryBuilder extends AbstractQueryBuilder<MatchPhraseQue
         return Objects.hash(fieldName, value, analyzer, slop);
     }
 
-    public static MatchPhraseQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<MatchPhraseQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         XContentParser.Token token = parser.nextToken();
@@ -223,6 +225,6 @@ public class MatchPhraseQueryBuilder extends AbstractQueryBuilder<MatchPhraseQue
         matchQuery.slop(slop);
         matchQuery.queryName(queryName);
         matchQuery.boost(boost);
-        return matchQuery;
+        return Optional.of(matchQuery);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java
index 6c867590653..04fa9120a76 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java
@@ -38,6 +38,7 @@ import org.elasticsearch.index.search.MatchQuery.ZeroTermsQuery;
 import java.io.IOException;
 import java.util.Locale;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * Match query is a query that analyzes the text and constructs a query as the
@@ -507,7 +508,7 @@ public class MatchQueryBuilder extends AbstractQueryBuilder<MatchQueryBuilder> {
         return NAME;
     }
 
-    public static MatchQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<MatchQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         XContentParser.Token token = parser.nextToken();
@@ -633,7 +634,7 @@ public class MatchQueryBuilder extends AbstractQueryBuilder<MatchQueryBuilder> {
         matchQuery.zeroTermsQuery(zeroTermsQuery);
         matchQuery.queryName(queryName);
         matchQuery.boost(boost);
-        return matchQuery;
+        return Optional.of(matchQuery);
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
index 66f623cbbb3..5fa36ec9777 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
@@ -805,7 +805,7 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ
         builder.endObject();
     }
 
-    public static MoreLikeThisQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<MoreLikeThisQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         // document inputs
@@ -955,7 +955,7 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ
         if (stopWords != null) {
             moreLikeThisQueryBuilder.stopWords(stopWords);
         }
-        return moreLikeThisQueryBuilder;
+        return Optional.of(moreLikeThisQueryBuilder);
     }
 
     private static void parseLikeField(QueryParseContext parseContext, List<String> texts, List<Item> items) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java
index 032feac9193..0d93b1331bb 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java
@@ -43,6 +43,7 @@ import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.TreeMap;
 
 /**
@@ -556,7 +557,7 @@ public class MultiMatchQueryBuilder extends AbstractQueryBuilder<MultiMatchQuery
         builder.endObject();
     }
 
-    public static MultiMatchQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<MultiMatchQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         Object value = null;
@@ -660,7 +661,7 @@ public class MultiMatchQueryBuilder extends AbstractQueryBuilder<MultiMatchQuery
                     "Fuziness not allowed for type [" + type.parseField.getPreferredName() + "]");
         }
 
-        return new MultiMatchQueryBuilder(value)
+        return Optional.of(new MultiMatchQueryBuilder(value)
                 .fields(fieldsBoosts)
                 .type(type)
                 .analyzer(analyzer)
@@ -677,7 +678,7 @@ public class MultiMatchQueryBuilder extends AbstractQueryBuilder<MultiMatchQuery
                 .tieBreaker(tieBreaker)
                 .zeroTermsQuery(zeroTermsQuery)
                 .boost(boost)
-                .queryName(queryName);
+                .queryName(queryName));
     }
 
     private static void parseFieldAndBoost(XContentParser parser, Map<String, Float> fieldsBoosts) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
index 5d74b540116..1d0e6d11f5f 100644
--- a/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
@@ -36,6 +36,7 @@ import org.elasticsearch.index.mapper.object.ObjectMapper;
 import java.io.IOException;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 
 public class NestedQueryBuilder extends AbstractQueryBuilder<NestedQueryBuilder> {
 
@@ -156,12 +157,12 @@ public class NestedQueryBuilder extends AbstractQueryBuilder<NestedQueryBuilder>
         builder.endObject();
     }
 
-    public static NestedQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<NestedQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
         ScoreMode scoreMode = ScoreMode.Avg;
         String queryName = null;
-        QueryBuilder query = null;
+        Optional<QueryBuilder> query = Optional.empty();
         String path = null;
         String currentFieldName = null;
         InnerHitBuilder innerHitBuilder = null;
@@ -194,14 +195,19 @@ public class NestedQueryBuilder extends AbstractQueryBuilder<NestedQueryBuilder>
                 }
             }
         }
-        NestedQueryBuilder queryBuilder =  new NestedQueryBuilder(path, query, scoreMode)
+
+        if (query.isPresent() == false) {
+            // if inner query is empty, bubble this up to caller so they can decide how to deal with it
+            return Optional.empty();
+        }
+        NestedQueryBuilder queryBuilder =  new NestedQueryBuilder(path, query.get(), scoreMode)
                 .ignoreUnmapped(ignoreUnmapped)
                 .queryName(queryName)
                 .boost(boost);
         if (innerHitBuilder != null) {
             queryBuilder.innerHit(innerHitBuilder);
         }
-        return queryBuilder;
+        return Optional.of(queryBuilder);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/ParentIdQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ParentIdQueryBuilder.java
index 419149ce992..ede449cbd39 100644
--- a/core/src/main/java/org/elasticsearch/index/query/ParentIdQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/ParentIdQueryBuilder.java
@@ -38,6 +38,7 @@ import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Optional;
 
 public final class ParentIdQueryBuilder extends AbstractQueryBuilder<ParentIdQueryBuilder> {
 
@@ -117,7 +118,7 @@ public final class ParentIdQueryBuilder extends AbstractQueryBuilder<ParentIdQue
         builder.endObject();
     }
 
-    public static ParentIdQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<ParentIdQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
         String type = null;
@@ -151,7 +152,7 @@ public final class ParentIdQueryBuilder extends AbstractQueryBuilder<ParentIdQue
         queryBuilder.queryName(queryName);
         queryBuilder.boost(boost);
         queryBuilder.ignoreUnmapped(ignoreUnmapped);
-        return queryBuilder;
+        return Optional.of(queryBuilder);
     }
 
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java
index 44677d1840d..f92e1d521df 100644
--- a/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java
@@ -36,6 +36,7 @@ import org.elasticsearch.index.query.support.QueryParsers;
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * A Query that matches documents containing terms with a specified prefix.
@@ -118,7 +119,7 @@ public class PrefixQueryBuilder extends AbstractQueryBuilder<PrefixQueryBuilder>
         builder.endObject();
     }
 
-    public static PrefixQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<PrefixQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         String fieldName = parser.currentName();
@@ -163,10 +164,10 @@ public class PrefixQueryBuilder extends AbstractQueryBuilder<PrefixQueryBuilder>
         if (value == null) {
             throw new ParsingException(parser.getTokenLocation(), "No value specified for prefix query");
         }
-        return new PrefixQueryBuilder(fieldName, value)
+        return Optional.of(new PrefixQueryBuilder(fieldName, value)
                 .rewrite(rewrite)
                 .boost(boost)
-                .queryName(queryName);
+                .queryName(queryName));
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
index 62662914fd8..93bd07c7bf9 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
@@ -23,14 +23,19 @@ import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Optional;
 
 public class QueryParseContext implements ParseFieldMatcherSupplier {
 
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(QueryParseContext.class));
+
     private static final ParseField CACHE = new ParseField("_cache").withAllDeprecated("Elasticsearch makes its own caching decisions");
     private static final ParseField CACHE_KEY = new ParseField("_cache_key").withAllDeprecated("Filters are always used as cache keys");
 
@@ -62,7 +67,7 @@ public class QueryParseContext implements ParseFieldMatcherSupplier {
                 if (token == XContentParser.Token.FIELD_NAME) {
                     String fieldName = parser.currentName();
                     if ("query".equals(fieldName)) {
-                        queryBuilder = parseInnerQueryBuilder();
+                        queryBuilder = parseInnerQueryBuilder().orElse(null);
                     } else {
                         throw new ParsingException(parser.getTokenLocation(), "request does not support [" + parser.currentName() + "]");
                     }
@@ -82,7 +87,7 @@ public class QueryParseContext implements ParseFieldMatcherSupplier {
     /**
      * Parses a query excluding the query element that wraps it
      */
-    public QueryBuilder parseInnerQueryBuilder() throws IOException {
+    public Optional<QueryBuilder> parseInnerQueryBuilder() throws IOException {
         // move to START object
         XContentParser.Token token;
         if (parser.currentToken() != XContentParser.Token.START_OBJECT) {
@@ -93,8 +98,13 @@ public class QueryParseContext implements ParseFieldMatcherSupplier {
         }
         token = parser.nextToken();
         if (token == XContentParser.Token.END_OBJECT) {
-            // empty query
-            return new EmptyQueryBuilder();
+            // we encountered '{}' for a query clause
+            String msg = "query malformed, empty clause found at [" + parser.getTokenLocation() +"]";
+            DEPRECATION_LOGGER.deprecated(msg);
+            if (parseFieldMatcher.isStrict()) {
+                throw new IllegalArgumentException(msg);
+            }
+            return Optional.empty();
         }
         if (token != XContentParser.Token.FIELD_NAME) {
             throw new ParsingException(parser.getTokenLocation(), "[_na] query malformed, no field after start_object");
@@ -105,7 +115,9 @@ public class QueryParseContext implements ParseFieldMatcherSupplier {
         if (token != XContentParser.Token.START_OBJECT && token != XContentParser.Token.START_ARRAY) {
             throw new ParsingException(parser.getTokenLocation(), "[_na] query malformed, no field after start_object");
         }
-        QueryBuilder result = indicesQueriesRegistry.lookup(queryName, parseFieldMatcher, parser.getTokenLocation()).fromXContent(this);
+        @SuppressWarnings("unchecked")
+        Optional<QueryBuilder> result = (Optional<QueryBuilder>) indicesQueriesRegistry.lookup(queryName, parseFieldMatcher,
+                parser.getTokenLocation()).fromXContent(this);
         if (parser.currentToken() == XContentParser.Token.END_OBJECT || parser.currentToken() == XContentParser.Token.END_ARRAY) {
             // if we are at END_OBJECT, move to the next one...
             parser.nextToken();
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryParser.java b/core/src/main/java/org/elasticsearch/index/query/QueryParser.java
index 069dc86cf8a..222d3bd4963 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryParser.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryParser.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.query;
 
 import java.io.IOException;
+import java.util.Optional;
 
 /**
  * Defines a query parser that is able to parse {@link QueryBuilder}s from {@link org.elasticsearch.common.xcontent.XContent}.
@@ -36,5 +37,12 @@ public interface QueryParser<QB extends QueryBuilder> {
      *            call
      * @return the new QueryBuilder
      */
-    QB fromXContent(QueryParseContext parseContext) throws IOException;
+    Optional<QB> fromXContent(QueryParseContext parseContext) throws IOException;
+
+    /**
+     * @return an empty {@link QueryBuilder} instance for this parser that can be used for deserialization
+     */
+    default QB getBuilderPrototype() { // TODO remove this when nothing implements it
+        throw new UnsupportedOperationException();
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java
index 6806ae944c2..c390507a785 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java
@@ -46,6 +46,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.TreeMap;
 
 /**
@@ -631,7 +632,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
         builder.endObject();
     }
 
-    public static QueryStringQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<QueryStringQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
         String currentFieldName = null;
         XContentParser.Token token;
@@ -792,7 +793,7 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder<QueryStringQue
         queryStringQuery.locale(locale);
         queryStringQuery.boost(boost);
         queryStringQuery.queryName(queryName);
-        return queryStringQuery;
+        return Optional.of(queryStringQuery);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
index 241c38475ab..1c51b6a2dd5 100644
--- a/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java
@@ -42,6 +42,7 @@ import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * A Query that matches documents within an range of terms.
@@ -297,7 +298,7 @@ public class RangeQueryBuilder extends AbstractQueryBuilder<RangeQueryBuilder> i
         builder.endObject();
     }
 
-    public static RangeQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<RangeQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         String fieldName = null;
@@ -381,7 +382,7 @@ public class RangeQueryBuilder extends AbstractQueryBuilder<RangeQueryBuilder> i
         if (format != null) {
             rangeQuery.format(format);
         }
-        return rangeQuery;
+        return Optional.of(rangeQuery);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java
index 703b2463b11..27462cfb26e 100644
--- a/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java
@@ -37,6 +37,7 @@ import org.elasticsearch.index.query.support.QueryParsers;
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * A Query that does fuzzy matching for a specific value.
@@ -179,7 +180,7 @@ public class RegexpQueryBuilder extends AbstractQueryBuilder<RegexpQueryBuilder>
         builder.endObject();
     }
 
-    public static RegexpQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<RegexpQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         String fieldName = parser.currentName();
@@ -237,12 +238,12 @@ public class RegexpQueryBuilder extends AbstractQueryBuilder<RegexpQueryBuilder>
         if (value == null) {
             throw new ParsingException(parser.getTokenLocation(), "No value specified for regexp query");
         }
-        return new RegexpQueryBuilder(fieldName, value)
+        return Optional.of(new RegexpQueryBuilder(fieldName, value)
                 .flags(flagsValue)
                 .maxDeterminizedStates(maxDeterminizedStates)
                 .rewrite(rewrite)
                 .boost(boost)
-                .queryName(queryName);
+                .queryName(queryName));
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
index 12aba8ae879..6d563f22a06 100644
--- a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
@@ -35,9 +35,9 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.Script.ScriptField;
-import org.elasticsearch.script.ScriptParameterParser.ScriptParameterValue;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptParameterParser;
+import org.elasticsearch.script.ScriptParameterParser.ScriptParameterValue;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.lookup.SearchLookup;
@@ -47,6 +47,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 
 public class ScriptQueryBuilder extends AbstractQueryBuilder<ScriptQueryBuilder> {
 
@@ -94,7 +95,7 @@ public class ScriptQueryBuilder extends AbstractQueryBuilder<ScriptQueryBuilder>
         builder.endObject();
     }
 
-    public static ScriptQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<ScriptQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
         ScriptParameterParser scriptParameterParser = new ScriptParameterParser();
 
@@ -149,9 +150,9 @@ public class ScriptQueryBuilder extends AbstractQueryBuilder<ScriptQueryBuilder>
             throw new ParsingException(parser.getTokenLocation(), "script must be provided with a [script] filter");
         }
 
-        return new ScriptQueryBuilder(script)
+        return Optional.of(new ScriptQueryBuilder(script)
                 .boost(boost)
-                .queryName(queryName);
+                .queryName(queryName));
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java
index 5e2c88af923..0bedf67820b 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java
@@ -39,6 +39,7 @@ import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.TreeMap;
 
 /**
@@ -417,7 +418,7 @@ public class SimpleQueryStringBuilder extends AbstractQueryBuilder<SimpleQuerySt
         builder.endObject();
     }
 
-    public static SimpleQueryStringBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<SimpleQueryStringBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         String currentFieldName = null;
@@ -514,7 +515,7 @@ public class SimpleQueryStringBuilder extends AbstractQueryBuilder<SimpleQuerySt
         qb.boost(boost).fields(fieldsAndWeights).analyzer(analyzerName).queryName(queryName).minimumShouldMatch(minimumShouldMatch);
         qb.flags(flags).defaultOperator(defaultOperator).locale(locale).lowercaseExpandedTerms(lowercaseExpandedTerms);
         qb.lenient(lenient).analyzeWildcard(analyzeWildcard).boost(boost);
-        return qb;
+        return Optional.of(qb);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java
index ae4297e431d..5d22df1557c 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanContainingQueryBuilder.java
@@ -31,6 +31,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * Builder for {@link org.apache.lucene.search.spans.SpanContainingQuery}.
@@ -102,7 +103,7 @@ public class SpanContainingQueryBuilder extends AbstractQueryBuilder<SpanContain
         builder.endObject();
     }
 
-    public static SpanContainingQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<SpanContainingQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
         String queryName = null;
@@ -116,17 +117,17 @@ public class SpanContainingQueryBuilder extends AbstractQueryBuilder<SpanContain
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
                 if (parseContext.getParseFieldMatcher().match(currentFieldName, BIG_FIELD)) {
-                    QueryBuilder query = parseContext.parseInnerQueryBuilder();
-                    if (!(query instanceof SpanQueryBuilder)) {
+                    Optional<QueryBuilder> query = parseContext.parseInnerQueryBuilder();
+                    if (query.isPresent() == false || query.get() instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "span_containing [big] must be of type span query");
                     }
-                    big = (SpanQueryBuilder) query;
+                    big = (SpanQueryBuilder) query.get();
                 } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LITTLE_FIELD)) {
-                    QueryBuilder query = parseContext.parseInnerQueryBuilder();
-                    if (!(query instanceof SpanQueryBuilder)) {
+                    Optional<QueryBuilder> query = parseContext.parseInnerQueryBuilder();
+                    if (query.isPresent() == false || query.get() instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "span_containing [little] must be of type span query");
                     }
-                    little = (SpanQueryBuilder) query;
+                    little = (SpanQueryBuilder) query.get();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "[span_containing] query does not support [" + currentFieldName + "]");
@@ -143,7 +144,7 @@ public class SpanContainingQueryBuilder extends AbstractQueryBuilder<SpanContain
 
         SpanContainingQueryBuilder query = new SpanContainingQueryBuilder(big, little);
         query.boost(boost).queryName(queryName);
-        return query;
+        return Optional.of(query);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java
index 009f8a39652..b0b096b19ec 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanFirstQueryBuilder.java
@@ -31,6 +31,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Optional;
 
 public class SpanFirstQueryBuilder extends AbstractQueryBuilder<SpanFirstQueryBuilder> implements SpanQueryBuilder {
 
@@ -101,7 +102,7 @@ public class SpanFirstQueryBuilder extends AbstractQueryBuilder<SpanFirstQueryBu
         builder.endObject();
     }
 
-    public static SpanFirstQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<SpanFirstQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
@@ -117,11 +118,11 @@ public class SpanFirstQueryBuilder extends AbstractQueryBuilder<SpanFirstQueryBu
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
                 if (parseContext.getParseFieldMatcher().match(currentFieldName, MATCH_FIELD)) {
-                    QueryBuilder query = parseContext.parseInnerQueryBuilder();
-                    if (!(query instanceof SpanQueryBuilder)) {
+                    Optional<QueryBuilder> query = parseContext.parseInnerQueryBuilder();
+                    if (query.isPresent() == false || query.get() instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "spanFirst [match] must be of type span query");
                     }
-                    match = (SpanQueryBuilder) query;
+                    match = (SpanQueryBuilder) query.get();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[span_first] query does not support [" + currentFieldName + "]");
                 }
@@ -145,7 +146,7 @@ public class SpanFirstQueryBuilder extends AbstractQueryBuilder<SpanFirstQueryBu
         }
         SpanFirstQueryBuilder queryBuilder = new SpanFirstQueryBuilder(match, end);
         queryBuilder.boost(boost).queryName(queryName);
-        return queryBuilder;
+        return Optional.of(queryBuilder);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java
index 86418903f70..0849d1c6a6e 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java
@@ -33,6 +33,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * Query that allows wraping a {@link MultiTermQueryBuilder} (one of wildcard, fuzzy, prefix, term, range or regexp query)
@@ -82,7 +83,7 @@ public class SpanMultiTermQueryBuilder extends AbstractQueryBuilder<SpanMultiTer
         builder.endObject();
     }
 
-    public static SpanMultiTermQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<SpanMultiTermQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
         String currentFieldName = null;
         MultiTermQueryBuilder subQuery = null;
@@ -94,12 +95,12 @@ public class SpanMultiTermQueryBuilder extends AbstractQueryBuilder<SpanMultiTer
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
                 if (parseContext.getParseFieldMatcher().match(currentFieldName, MATCH_FIELD)) {
-                    QueryBuilder innerQuery = parseContext.parseInnerQueryBuilder();
-                    if (innerQuery instanceof MultiTermQueryBuilder == false) {
+                    Optional<QueryBuilder> query = parseContext.parseInnerQueryBuilder();
+                    if (query.isPresent() == false || query.get() instanceof MultiTermQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(),
                                 "[span_multi] [" + MATCH_FIELD.getPreferredName() + "] must be of type multi term query");
                     }
-                    subQuery = (MultiTermQueryBuilder) innerQuery;
+                    subQuery = (MultiTermQueryBuilder) query.get();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[span_multi] query does not support [" + currentFieldName + "]");
                 }
@@ -119,7 +120,7 @@ public class SpanMultiTermQueryBuilder extends AbstractQueryBuilder<SpanMultiTer
                     "[span_multi] must have [" + MATCH_FIELD.getPreferredName() + "] multi term query clause");
         }
 
-        return new SpanMultiTermQueryBuilder(subQuery).queryName(queryName).boost(boost);
+        return Optional.of(new SpanMultiTermQueryBuilder(subQuery).queryName(queryName).boost(boost));
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
index cebc72c077b..c11164ae7a4 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanNearQueryBuilder.java
@@ -34,6 +34,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * Matches spans which are near one another. One can specify slop, the maximum number
@@ -146,7 +147,7 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
         builder.endObject();
     }
 
-    public static SpanNearQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<SpanNearQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
@@ -164,11 +165,11 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
             } else if (token == XContentParser.Token.START_ARRAY) {
                 if (parseContext.getParseFieldMatcher().match(currentFieldName, CLAUSES_FIELD)) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                        QueryBuilder query = parseContext.parseInnerQueryBuilder();
-                        if (!(query instanceof SpanQueryBuilder)) {
+                        Optional<QueryBuilder> query = parseContext.parseInnerQueryBuilder();
+                        if (query.isPresent() == false || query.get() instanceof SpanQueryBuilder == false) {
                             throw new ParsingException(parser.getTokenLocation(), "spanNear [clauses] must be of type span query");
                         }
-                        clauses.add((SpanQueryBuilder) query);
+                        clauses.add((SpanQueryBuilder) query.get());
                     }
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[span_near] query does not support [" + currentFieldName + "]");
@@ -207,7 +208,7 @@ public class SpanNearQueryBuilder extends AbstractQueryBuilder<SpanNearQueryBuil
         queryBuilder.inOrder(inOrder);
         queryBuilder.boost(boost);
         queryBuilder.queryName(queryName);
-        return queryBuilder;
+        return Optional.of(queryBuilder);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java
index 02ce431de1e..6bb635b2005 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java
@@ -31,6 +31,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Optional;
 
 public class SpanNotQueryBuilder extends AbstractQueryBuilder<SpanNotQueryBuilder> implements SpanQueryBuilder {
 
@@ -162,7 +163,7 @@ public class SpanNotQueryBuilder extends AbstractQueryBuilder<SpanNotQueryBuilde
         builder.endObject();
     }
 
-    public static SpanNotQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<SpanNotQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
@@ -183,17 +184,17 @@ public class SpanNotQueryBuilder extends AbstractQueryBuilder<SpanNotQueryBuilde
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
                 if (parseContext.getParseFieldMatcher().match(currentFieldName, INCLUDE_FIELD)) {
-                    QueryBuilder query = parseContext.parseInnerQueryBuilder();
-                    if (!(query instanceof SpanQueryBuilder)) {
+                    Optional<QueryBuilder> query = parseContext.parseInnerQueryBuilder();
+                    if (query.isPresent() == false || query.get() instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "spanNot [include] must be of type span query");
                     }
-                    include = (SpanQueryBuilder) query;
+                    include = (SpanQueryBuilder) query.get();
                 } else if (parseContext.getParseFieldMatcher().match(currentFieldName, EXCLUDE_FIELD)) {
-                    QueryBuilder query = parseContext.parseInnerQueryBuilder();
-                    if (!(query instanceof SpanQueryBuilder)) {
+                    Optional<QueryBuilder> query = parseContext.parseInnerQueryBuilder();
+                    if (query.isPresent() == false || query.get() instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "spanNot [exclude] must be of type span query");
                     }
-                    exclude = (SpanQueryBuilder) query;
+                    exclude = (SpanQueryBuilder) query.get();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[span_not] query does not support [" + currentFieldName + "]");
                 }
@@ -235,7 +236,7 @@ public class SpanNotQueryBuilder extends AbstractQueryBuilder<SpanNotQueryBuilde
         }
         spanNotQuery.boost(boost);
         spanNotQuery.queryName(queryName);
-        return spanNotQuery;
+        return Optional.of(spanNotQuery);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java
index 98698307d0c..5169e0d933b 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java
@@ -34,6 +34,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * Span query that matches the union of its clauses. Maps to {@link SpanOrQuery}.
@@ -99,7 +100,7 @@ public class SpanOrQueryBuilder extends AbstractQueryBuilder<SpanOrQueryBuilder>
         builder.endObject();
     }
 
-    public static SpanOrQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<SpanOrQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
@@ -115,11 +116,11 @@ public class SpanOrQueryBuilder extends AbstractQueryBuilder<SpanOrQueryBuilder>
             } else if (token == XContentParser.Token.START_ARRAY) {
                 if (parseContext.getParseFieldMatcher().match(currentFieldName, CLAUSES_FIELD)) {
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                        QueryBuilder query = parseContext.parseInnerQueryBuilder();
-                        if (!(query instanceof SpanQueryBuilder)) {
+                        Optional<QueryBuilder> query = parseContext.parseInnerQueryBuilder();
+                        if (query.isPresent() == false || query.get() instanceof SpanQueryBuilder == false) {
                             throw new ParsingException(parser.getTokenLocation(), "spanOr [clauses] must be of type span query");
                         }
-                        clauses.add((SpanQueryBuilder) query);
+                        clauses.add((SpanQueryBuilder) query.get());
                     }
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "[span_or] query does not support [" + currentFieldName + "]");
@@ -145,7 +146,7 @@ public class SpanOrQueryBuilder extends AbstractQueryBuilder<SpanOrQueryBuilder>
         }
         queryBuilder.boost(boost);
         queryBuilder.queryName(queryName);
-        return queryBuilder;
+        return Optional.of(queryBuilder);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java
index 3bb374ff276..56581c39918 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java
@@ -31,6 +31,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.MappedFieldType;
 
 import java.io.IOException;
+import java.util.Optional;
 
 /**
  * A Span Query that matches documents containing a term.
@@ -93,7 +94,7 @@ public class SpanTermQueryBuilder extends BaseTermQueryBuilder<SpanTermQueryBuil
         return new SpanTermQuery(term);
     }
 
-    public static SpanTermQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException, ParsingException {
+    public static Optional<SpanTermQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException, ParsingException {
         XContentParser parser = parseContext.parser();
 
         XContentParser.Token token = parser.currentToken();
@@ -142,7 +143,7 @@ public class SpanTermQueryBuilder extends BaseTermQueryBuilder<SpanTermQueryBuil
 
         SpanTermQueryBuilder result = new SpanTermQueryBuilder(fieldName, value);
         result.boost(boost).queryName(queryName);
-        return result;
+        return Optional.of(result);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java
index eaedc80bab4..2975063f78d 100644
--- a/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java
@@ -31,6 +31,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * Builder for {@link org.apache.lucene.search.spans.SpanWithinQuery}.
@@ -107,7 +108,7 @@ public class SpanWithinQueryBuilder extends AbstractQueryBuilder<SpanWithinQuery
         builder.endObject();
     }
 
-    public static SpanWithinQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<SpanWithinQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
@@ -122,17 +123,17 @@ public class SpanWithinQueryBuilder extends AbstractQueryBuilder<SpanWithinQuery
                 currentFieldName = parser.currentName();
             } else if (token == XContentParser.Token.START_OBJECT) {
                 if (parseContext.getParseFieldMatcher().match(currentFieldName, BIG_FIELD)) {
-                    QueryBuilder query = parseContext.parseInnerQueryBuilder();
-                    if (query instanceof SpanQueryBuilder == false) {
+                    Optional<QueryBuilder> query = parseContext.parseInnerQueryBuilder();
+                    if (query.isPresent() == false || query.get() instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "span_within [big] must be of type span query");
                     }
-                    big = (SpanQueryBuilder) query;
+                    big = (SpanQueryBuilder) query.get();
                 } else if (parseContext.getParseFieldMatcher().match(currentFieldName, LITTLE_FIELD)) {
-                    QueryBuilder query = parseContext.parseInnerQueryBuilder();
-                    if (query instanceof SpanQueryBuilder == false) {
+                    Optional<QueryBuilder> query = parseContext.parseInnerQueryBuilder();
+                    if (query.isPresent() == false || query.get() instanceof SpanQueryBuilder == false) {
                         throw new ParsingException(parser.getTokenLocation(), "span_within [little] must be of type span query");
                     }
-                    little = (SpanQueryBuilder) query;
+                    little = (SpanQueryBuilder) query.get();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
                             "[span_within] query does not support [" + currentFieldName + "]");
@@ -155,7 +156,7 @@ public class SpanWithinQueryBuilder extends AbstractQueryBuilder<SpanWithinQuery
 
         SpanWithinQueryBuilder query = new SpanWithinQueryBuilder(big, little);
         query.boost(boost).queryName(queryName);
-        return query;
+        return Optional.of(query);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java
index 1a4d05374b3..3b41abbbe82 100644
--- a/core/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.query;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -37,6 +38,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * Facilitates creating template query requests.
@@ -120,10 +122,10 @@ public class TemplateQueryBuilder extends AbstractQueryBuilder<TemplateQueryBuil
      * In the simplest case, parse template string and variables from the request,
      * compile the template and execute the template against the given variables.
      */
-    public static TemplateQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<TemplateQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
         Template template = parse(parser, parseContext.getParseFieldMatcher());
-        return new TemplateQueryBuilder(template);
+        return Optional.of(new TemplateQueryBuilder(template));
     }
 
     public static Template parse(XContentParser parser, ParseFieldMatcher parseFieldMatcher, String... parameters) throws IOException {
@@ -179,7 +181,8 @@ public class TemplateQueryBuilder extends AbstractQueryBuilder<TemplateQueryBuil
         BytesReference querySource = (BytesReference) executable.run();
         try (XContentParser qSourceParser = XContentFactory.xContent(querySource).createParser(querySource)) {
             final QueryParseContext queryParseContext = queryRewriteContext.newParseContext(qSourceParser);
-            final QueryBuilder queryBuilder = queryParseContext.parseInnerQueryBuilder();
+            final QueryBuilder queryBuilder = queryParseContext.parseInnerQueryBuilder().orElseThrow(
+                    () -> new ParsingException(qSourceParser.getTokenLocation(), "inner query in [" + NAME + "] cannot be empty"));;
             if (boost() != DEFAULT_BOOST || queryName() != null) {
                 final BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
                 boolQueryBuilder.must(queryBuilder);
diff --git a/core/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java
index ffdf6a3337d..bfb55c9956c 100644
--- a/core/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/TermQueryBuilder.java
@@ -30,6 +30,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.MappedFieldType;
 
 import java.io.IOException;
+import java.util.Optional;
 
 /**
  * A Query that matches documents containing a term.
@@ -84,7 +85,7 @@ public class TermQueryBuilder extends BaseTermQueryBuilder<TermQueryBuilder> {
         super(in);
     }
 
-    public static TermQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<TermQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         String queryName = null;
@@ -140,7 +141,7 @@ public class TermQueryBuilder extends BaseTermQueryBuilder<TermQueryBuilder> {
         if (queryName != null) {
             termQuery.queryName(queryName);
         }
-        return termQuery;
+        return Optional.of(termQuery);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
index 5b0650eb7e4..7d27a911f88 100644
--- a/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
@@ -47,6 +47,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
@@ -238,7 +239,7 @@ public class TermsQueryBuilder extends AbstractQueryBuilder<TermsQueryBuilder> {
         builder.endObject();
     }
 
-    public static TermsQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<TermsQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         String fieldName = null;
@@ -289,9 +290,9 @@ public class TermsQueryBuilder extends AbstractQueryBuilder<TermsQueryBuilder> {
             throw new ParsingException(parser.getTokenLocation(), "[" + TermsQueryBuilder.NAME + "] query requires a field name, " +
                     "followed by array of terms or a document lookup specification");
         }
-        return new TermsQueryBuilder(fieldName, values, termsLookup)
+        return Optional.of(new TermsQueryBuilder(fieldName, values, termsLookup)
                 .boost(boost)
-                .queryName(queryName);
+                .queryName(queryName));
     }
 
     private static List<Object> parseValues(XContentParser parser) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/index/query/TypeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/TypeQueryBuilder.java
index 129b0275527..00cd108c003 100644
--- a/core/src/main/java/org/elasticsearch/index/query/TypeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/TypeQueryBuilder.java
@@ -33,6 +33,7 @@ import org.elasticsearch.index.mapper.DocumentMapper;
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Optional;
 
 public class TypeQueryBuilder extends AbstractQueryBuilder<TypeQueryBuilder> {
     public static final String NAME = "type";
@@ -81,7 +82,7 @@ public class TypeQueryBuilder extends AbstractQueryBuilder<TypeQueryBuilder> {
         builder.endObject();
     }
 
-    public static TypeQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<TypeQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
         BytesRef type = null;
 
@@ -114,9 +115,9 @@ public class TypeQueryBuilder extends AbstractQueryBuilder<TypeQueryBuilder> {
             throw new ParsingException(parser.getTokenLocation(),
                     "[" + TypeQueryBuilder.NAME + "] filter needs to be provided with a value for the type");
         }
-        return new TypeQueryBuilder(type)
+        return Optional.of(new TypeQueryBuilder(type)
                 .boost(boost)
-                .queryName(queryName);
+                .queryName(queryName));
     }
 
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java
index ef49472b8cc..007dff92340 100644
--- a/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java
@@ -36,6 +36,7 @@ import org.elasticsearch.index.query.support.QueryParsers;
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * Implements the wildcard search query. Supported wildcards are <tt>*</tt>, which
@@ -135,7 +136,7 @@ public class WildcardQueryBuilder extends AbstractQueryBuilder<WildcardQueryBuil
         builder.endObject();
     }
 
-    public static WildcardQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<WildcardQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         XContentParser.Token token = parser.nextToken();
@@ -180,10 +181,10 @@ public class WildcardQueryBuilder extends AbstractQueryBuilder<WildcardQueryBuil
         if (value == null) {
             throw new ParsingException(parser.getTokenLocation(), "No value specified for wildcard query");
         }
-        return new WildcardQueryBuilder(fieldName, value)
+        return Optional.of(new WildcardQueryBuilder(fieldName, value)
                 .rewrite(rewrite)
                 .boost(boost)
-                .queryName(queryName);
+                .queryName(queryName));
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java
index 4037666393d..4e1eb83272a 100644
--- a/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java
@@ -33,6 +33,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
+import java.util.Optional;
 
 /**
  * A Query builder which allows building a query given JSON string or binary data provided as input. This is useful when you want
@@ -116,7 +117,7 @@ public class WrapperQueryBuilder extends AbstractQueryBuilder<WrapperQueryBuilde
         builder.endObject();
     }
 
-    public static WrapperQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<WrapperQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         XContentParser.Token token = parser.nextToken();
@@ -136,7 +137,7 @@ public class WrapperQueryBuilder extends AbstractQueryBuilder<WrapperQueryBuilde
         if (source == null) {
             throw new ParsingException(parser.getTokenLocation(), "wrapper query has no [query] specified");
         }
-        return new WrapperQueryBuilder(source);
+        return Optional.of(new WrapperQueryBuilder(source));
     }
 
     @Override
@@ -164,7 +165,8 @@ public class WrapperQueryBuilder extends AbstractQueryBuilder<WrapperQueryBuilde
         try (XContentParser qSourceParser = XContentFactory.xContent(source).createParser(source)) {
             QueryParseContext parseContext = context.newParseContext(qSourceParser);
 
-            final QueryBuilder queryBuilder = parseContext.parseInnerQueryBuilder();
+            final QueryBuilder queryBuilder = parseContext.parseInnerQueryBuilder().orElseThrow(
+                    () -> new ParsingException(qSourceParser.getTokenLocation(), "inner query cannot be empty"));
             if (boost() != DEFAULT_BOOST || queryName() != null) {
                 final BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
                 boolQueryBuilder.must(queryBuilder);
diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java
index 98f878d1c2e..3e52322d22f 100644
--- a/core/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilder.java
@@ -37,12 +37,13 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
+import org.elasticsearch.index.query.InnerHitBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.index.query.InnerHitBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -51,6 +52,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * A query that uses a filters with a script associated with them to compute the
@@ -431,15 +433,13 @@ public class FunctionScoreQueryBuilder extends AbstractQueryBuilder<FunctionScor
         return this;
     }
 
-
-
     @Override
     protected void extractInnerHitBuilders(Map<String, InnerHitBuilder> innerHits) {
         InnerHitBuilder.extractInnerHits(query(), innerHits);
     }
 
-    public static FunctionScoreQueryBuilder fromXContent(ParseFieldRegistry<ScoreFunctionParser<?>> scoreFunctionsRegistry,
-                                                         QueryParseContext parseContext) throws IOException {
+    public static Optional<FunctionScoreQueryBuilder> fromXContent(ParseFieldRegistry<ScoreFunctionParser<?>> scoreFunctionsRegistry,
+            QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
         QueryBuilder query = null;
@@ -468,7 +468,7 @@ public class FunctionScoreQueryBuilder extends AbstractQueryBuilder<FunctionScor
                         throw new ParsingException(parser.getTokenLocation(), "failed to parse [{}] query. [query] is already defined.",
                                 NAME);
                     }
-                    query = parseContext.parseInnerQueryBuilder();
+                    query = parseContext.parseInnerQueryBuilder().orElse(QueryBuilders.matchAllQuery());
                 } else {
                     if (singleFunctionFound) {
                         throw new ParsingException(parser.getTokenLocation(),
@@ -556,7 +556,7 @@ public class FunctionScoreQueryBuilder extends AbstractQueryBuilder<FunctionScor
         }
         functionScoreQueryBuilder.boost(boost);
         functionScoreQueryBuilder.queryName(queryName);
-        return functionScoreQueryBuilder;
+        return Optional.of(functionScoreQueryBuilder);
     }
 
     private static void handleMisplacedFunctionsDeclaration(XContentLocation contentLocation, String errorString) {
@@ -584,7 +584,7 @@ public class FunctionScoreQueryBuilder extends AbstractQueryBuilder<FunctionScor
                         currentFieldName = parser.currentName();
                     } else if (token == XContentParser.Token.START_OBJECT) {
                         if (parseContext.getParseFieldMatcher().match(currentFieldName, FILTER_FIELD)) {
-                            filter = parseContext.parseInnerQueryBuilder();
+                            filter = parseContext.parseInnerQueryBuilder().orElse(QueryBuilders.matchAllQuery());
                         } else {
                             if (scoreFunction != null) {
                                 throw new ParsingException(parser.getTokenLocation(),
diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java
index a8893c4c27c..8afda85c58f 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -37,7 +37,6 @@ import org.elasticsearch.index.query.BoostingQueryBuilder;
 import org.elasticsearch.index.query.CommonTermsQueryBuilder;
 import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
 import org.elasticsearch.index.query.DisMaxQueryBuilder;
-import org.elasticsearch.index.query.EmptyQueryBuilder;
 import org.elasticsearch.index.query.ExistsQueryBuilder;
 import org.elasticsearch.index.query.FieldMaskingSpanQueryBuilder;
 import org.elasticsearch.index.query.FuzzyQueryBuilder;
@@ -698,9 +697,6 @@ public class SearchModule extends AbstractModule {
         if (ShapesAvailability.JTS_AVAILABLE && ShapesAvailability.SPATIAL4J_AVAILABLE) {
             registerQuery(GeoShapeQueryBuilder::new, GeoShapeQueryBuilder::fromXContent, GeoShapeQueryBuilder.QUERY_NAME_FIELD);
         }
-        // EmptyQueryBuilder is not registered as query parser but used internally.
-        // We need to register it with the NamedWriteableRegistry in order to serialize it
-        namedWriteableRegistry.register(QueryBuilder.class, EmptyQueryBuilder.NAME, EmptyQueryBuilder::new);
     }
 
     static {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
index a971035b5f6..a973d69da4f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
@@ -24,8 +24,6 @@ import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.query.EmptyQueryBuilder;
-import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
@@ -55,11 +53,7 @@ public class FilterAggregationBuilder extends AbstractAggregationBuilder<FilterA
         if (filter == null) {
             throw new IllegalArgumentException("[filter] must not be null: [" + name + "]");
         }
-        if (filter instanceof EmptyQueryBuilder) {
-            this.filter = new MatchAllQueryBuilder();
-        } else {
-            this.filter = filter;
-        }
+        this.filter = filter;
     }
 
     /**
@@ -89,18 +83,12 @@ public class FilterAggregationBuilder extends AbstractAggregationBuilder<FilterA
         return builder;
     }
 
-    public static FilterAggregationBuilder parse(String aggregationName, QueryParseContext context)
-            throws IOException {
-        QueryBuilder filter = context.parseInnerQueryBuilder();
-
-        if (filter == null) {
-            throw new ParsingException(null, "filter cannot be null in filter aggregation [{}]", aggregationName);
-        }
-
+    public static FilterAggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
+        QueryBuilder filter = context.parseInnerQueryBuilder().orElseThrow(() -> new ParsingException(context.parser().getTokenLocation(),
+                "filter cannot be null in filter aggregation [{}]", aggregationName));
         return new FilterAggregationBuilder(aggregationName, filter);
     }
 
-
     @Override
     protected int doHashCode() {
         return Objects.hash(filter);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregationBuilder.java
index 5056e3a67bb..9b2b9cee7ff 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregationBuilder.java
@@ -26,7 +26,6 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
@@ -235,8 +234,8 @@ public class FiltersAggregationBuilder extends AbstractAggregationBuilder<Filter
                         if (token == XContentParser.Token.FIELD_NAME) {
                             key = parser.currentName();
                         } else {
-                            QueryBuilder filter = context.parseInnerQueryBuilder();
-                            keyedFilters.add(new FiltersAggregator.KeyedFilter(key, filter == null ? matchAllQuery() : filter));
+                            QueryBuilder filter = context.parseInnerQueryBuilder().orElse(matchAllQuery());
+                            keyedFilters.add(new FiltersAggregator.KeyedFilter(key, filter));
                         }
                     }
                 } else {
@@ -247,8 +246,8 @@ public class FiltersAggregationBuilder extends AbstractAggregationBuilder<Filter
                 if (context.getParseFieldMatcher().match(currentFieldName, FILTERS_FIELD)) {
                     nonKeyedFilters = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
-                        QueryBuilder filter = context.parseInnerQueryBuilder();
-                        nonKeyedFilters.add(filter == null ? QueryBuilders.matchAllQuery() : filter);
+                        QueryBuilder filter = context.parseInnerQueryBuilder().orElse(matchAllQuery());
+                        nonKeyedFilters.add(filter);
                     }
                 } else {
                     throw new ParsingException(parser.getTokenLocation(),
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregator.java
index a5ce89c4666..08bbdaf3e3b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregator.java
@@ -29,7 +29,6 @@ import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.query.EmptyQueryBuilder;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.aggregations.Aggregator;
@@ -69,11 +68,7 @@ public class FiltersAggregator extends BucketsAggregator {
                 throw new IllegalArgumentException("[filter] must not be null");
             }
             this.key = key;
-            if (filter instanceof EmptyQueryBuilder) {
-                this.filter = new MatchAllQueryBuilder();
-            } else {
-                this.filter = filter;
-            }
+            this.filter = filter;
         }
 
         /**
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParser.java
index 33db8f97335..ba87f0917a0 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsParser.java
@@ -38,6 +38,7 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 
 import java.io.IOException;
 import java.util.Map;
+import java.util.Optional;
 
 /**
  *
@@ -91,8 +92,10 @@ public class SignificantTermsParser extends AbstractTermsParser {
                 return true;
             } else if (parseFieldMatcher.match(currentFieldName, SignificantTermsAggregationBuilder.BACKGROUND_FILTER)) {
                 QueryParseContext queryParseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
-                QueryBuilder filter = queryParseContext.parseInnerQueryBuilder();
-                otherOptions.put(SignificantTermsAggregationBuilder.BACKGROUND_FILTER, filter);
+                Optional<QueryBuilder> filter = queryParseContext.parseInnerQueryBuilder();
+                if (filter.isPresent()) {
+                    otherOptions.put(SignificantTermsAggregationBuilder.BACKGROUND_FILTER, filter.get());
+                }
                 return true;
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index a2c2f3a0fa6..4c47304330a 100644
--- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -611,7 +611,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
     /**
      * Add an aggregation to perform as part of the search.
      */
-    public SearchSourceBuilder aggregation(PipelineAggregatorBuilder aggregation) {
+    public SearchSourceBuilder aggregation(PipelineAggregatorBuilder<?> aggregation) {
             if (aggregations == null) {
             aggregations = AggregatorFactories.builder();
             }
@@ -1003,9 +1003,9 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                 }
             } else if (token == XContentParser.Token.START_OBJECT) {
                 if (context.getParseFieldMatcher().match(currentFieldName, QUERY_FIELD)) {
-                    queryBuilder = context.parseInnerQueryBuilder();
+                    queryBuilder = context.parseInnerQueryBuilder().orElse(null);
                 } else if (context.getParseFieldMatcher().match(currentFieldName, POST_FILTER_FIELD)) {
-                    postQueryBuilder = context.parseInnerQueryBuilder();
+                    postQueryBuilder = context.parseInnerQueryBuilder().orElse(null);
                 } else if (context.getParseFieldMatcher().match(currentFieldName, _SOURCE_FIELD)) {
                     fetchSourceContext = FetchSourceContext.parse(context);
                 } else if (context.getParseFieldMatcher().match(currentFieldName, SCRIPT_FIELDS_FIELD)) {
diff --git a/core/src/main/java/org/elasticsearch/search/highlight/AbstractHighlighterBuilder.java b/core/src/main/java/org/elasticsearch/search/highlight/AbstractHighlighterBuilder.java
index 557567fe354..816be79a9e5 100644
--- a/core/src/main/java/org/elasticsearch/search/highlight/AbstractHighlighterBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/highlight/AbstractHighlighterBuilder.java
@@ -539,7 +539,7 @@ public abstract class AbstractHighlighterBuilder<HB extends AbstractHighlighterB
         }, OPTIONS_FIELD);
         parser.declareObject(HB::highlightQuery, (XContentParser p, QueryParseContext c) -> {
             try {
-                return c.parseInnerQueryBuilder();
+                return c.parseInnerQueryBuilder().orElse(null);
             } catch (IOException e) {
                 throw new RuntimeException("Error parsing query", e);
             }
diff --git a/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java
index e91468c805f..6f59b446dc2 100644
--- a/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java
@@ -26,6 +26,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.search.rescore.QueryRescorer.QueryRescoreContext;
@@ -56,7 +57,7 @@ public class QueryRescorerBuilder extends RescoreBuilder<QueryRescorerBuilder> {
     static {
         QUERY_RESCORE_PARSER.declareObject(InnerBuilder::setQueryBuilder, (p, c) -> {
             try {
-                return c.parseInnerQueryBuilder();
+                return c.parseInnerQueryBuilder().orElse(QueryBuilders.matchAllQuery());
             } catch (IOException e) {
                 throw new ParsingException(p.getTokenLocation(), "Could not parse inner query", e);
             }
diff --git a/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
index 892673f890a..d519f740870 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
@@ -39,6 +39,7 @@ import org.elasticsearch.search.MultiValueMode;
 
 import java.io.IOException;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * A sort builder to sort based on a document field.
@@ -328,7 +329,7 @@ public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {
     public static FieldSortBuilder fromXContent(QueryParseContext context, String fieldName) throws IOException {
         XContentParser parser = context.parser();
 
-        QueryBuilder nestedFilter = null;
+        Optional<QueryBuilder> nestedFilter = Optional.empty();
         String nestedPath = null;
         Object missing = null;
         SortOrder order = null;
@@ -371,9 +372,7 @@ public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {
         }
 
         FieldSortBuilder builder = new FieldSortBuilder(fieldName);
-        if (nestedFilter != null) {
-            builder.setNestedFilter(nestedFilter);
-        }
+        nestedFilter.ifPresent(builder::setNestedFilter);
         if (nestedPath != null) {
             builder.setNestedPath(nestedPath);
         }
diff --git a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
index dce9a7ec3fe..f33dd0e2b15 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
@@ -60,6 +60,7 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * A geo distance based sorting on a geo point like field.
@@ -257,7 +258,7 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
     }
 
     /**
-     * Sets validation method for this sort builder. 
+     * Sets validation method for this sort builder.
      */
     public GeoDistanceSortBuilder validation(GeoValidationMethod method) {
         this.validation = method;
@@ -265,7 +266,7 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
     }
 
     /**
-     * Returns the validation method to use for this sort builder. 
+     * Returns the validation method to use for this sort builder.
      */
     public GeoValidationMethod validation() {
         return validation;
@@ -407,7 +408,7 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
         GeoDistance geoDistance = GeoDistance.DEFAULT;
         SortOrder order = SortOrder.ASC;
         SortMode sortMode = null;
-        QueryBuilder nestedFilter = null;
+        Optional<QueryBuilder> nestedFilter = Optional.empty();
         String nestedPath = null;
 
         boolean coerce = GeoValidationMethod.DEFAULT_LENIENT_PARSING;
@@ -492,7 +493,7 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
         if (sortMode != null) {
             result.sortMode(sortMode);
         }
-        result.setNestedFilter(nestedFilter);
+        nestedFilter.ifPresent(result::setNestedFilter);
         result.setNestedPath(nestedPath);
         if (validation == null) {
             // looks like either validation was left unset or we are parsing old validation json
@@ -517,7 +518,7 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
             for (GeoPoint point : localPoints) {
                 if (GeoUtils.isValidLatitude(point.lat()) == false) {
                     throw new ElasticsearchParseException(
-                            "illegal latitude value [{}] for [GeoDistanceSort] for field [{}].", 
+                            "illegal latitude value [{}] for [GeoDistanceSort] for field [{}].",
                             point.lat(),
                             fieldName);
                 }
diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
index eeb418c0a97..f7baee509de 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
@@ -61,6 +61,7 @@ import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * Script sort builder allows to sort based on a custom script expression.
@@ -237,7 +238,7 @@ public class ScriptSortBuilder extends SortBuilder<ScriptSortBuilder> {
         ScriptSortType type = null;
         SortMode sortMode = null;
         SortOrder order = null;
-        QueryBuilder nestedFilter = null;
+        Optional<QueryBuilder> nestedFilter = Optional.empty();
         String nestedPath = null;
         Map<String, Object> params = new HashMap<>();
 
@@ -292,9 +293,7 @@ public class ScriptSortBuilder extends SortBuilder<ScriptSortBuilder> {
         if (sortMode != null) {
             result.sortMode(sortMode);
         }
-        if (nestedFilter != null) {
-            result.setNestedFilter(nestedFilter);
-        }
+        nestedFilter.ifPresent(result::setNestedFilter);
         if (nestedPath != null) {
             result.setNestedPath(nestedPath);
         }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
index 9aed90f55e5..25f589794f3 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggester.java
@@ -34,7 +34,9 @@ import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.MatchNoneQueryBuilder;
 import org.elasticsearch.index.query.ParsedQuery;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.script.CompiledScript;
@@ -53,6 +55,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
     private final BytesRef SEPARATOR = new BytesRef(" ");
@@ -123,7 +126,8 @@ public final class PhraseSuggester extends Suggester<PhraseSuggestionContext> {
                     final ExecutableScript executable = scriptService.executable(collateScript, vars);
                     final BytesReference querySource = (BytesReference) executable.run();
                     try (XContentParser parser = XContentFactory.xContent(querySource).createParser(querySource)) {
-                        final ParsedQuery parsedQuery = shardContext.toQuery(shardContext.newParseContext(parser).parseInnerQueryBuilder());
+                        Optional<QueryBuilder> innerQueryBuilder = shardContext.newParseContext(parser).parseInnerQueryBuilder();
+                        final ParsedQuery parsedQuery = shardContext.toQuery(innerQueryBuilder.orElse(new MatchNoneQueryBuilder()));
                         collateMatch = Lucene.exists(searcher, parsedQuery.query());
                     }
                 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
index 15180b9d989..89da227df87 100644
--- a/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
@@ -24,6 +24,7 @@ import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
+import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -43,6 +44,7 @@ import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.startsWith;
 
 public class BoolQueryBuilderTests extends AbstractQueryTestCase<BoolQueryBuilder> {
     @Override
@@ -345,6 +347,29 @@ public class BoolQueryBuilderTests extends AbstractQueryTestCase<BoolQueryBuilde
         assertEquals(query, "kimchy", ((TermQueryBuilder)queryBuilder.must().get(0)).value());
     }
 
+    /**
+     * we ignore empty query bodies if we are not in strict mode
+     */
+    public void testFromJsonEmptyQueryBody() throws IOException {
+        String query =
+                "{" +
+                "\"bool\" : {" +
+                "  \"must\" : [ { } ]," +
+                "  \"filter\" : { }," +
+                "  \"must_not\" : [ { \"constant_score\" : {\"filter\" : { } } } ]" +
+                "}" +
+              "}";
+
+        BoolQueryBuilder queryBuilder = (BoolQueryBuilder) parseQuery(query, ParseFieldMatcher.EMPTY);
+        assertEquals(query, 0, queryBuilder.must().size());
+        assertEquals(query, 0, queryBuilder.filter().size());
+        assertEquals(query, 0, queryBuilder.mustNot().size());
+        assertEquals(query, 0, queryBuilder.should().size());
+
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> parseQuery(query, ParseFieldMatcher.STRICT));
+        assertThat(ex.getMessage(), startsWith("query malformed, empty clause found at"));
+    }
+
     public void testRewrite() throws IOException {
         BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
         boolean mustRewrite = false;
diff --git a/core/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java
index 34c15d63577..343c6270746 100644
--- a/core/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/BoostingQueryBuilderTests.java
@@ -21,12 +21,17 @@ package org.elasticsearch.index.query;
 
 import org.apache.lucene.queries.BoostingQuery;
 import org.apache.lucene.search.Query;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
+import java.util.Optional;
 
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.CoreMatchers.startsWith;;
 
 public class BoostingQueryBuilderTests extends AbstractQueryTestCase<BoostingQueryBuilder> {
 
@@ -105,6 +110,40 @@ public class BoostingQueryBuilderTests extends AbstractQueryTestCase<BoostingQue
         assertEquals(query, 5, queryBuilder.positiveQuery().boost(), 0.00001);
     }
 
+    /**
+     * we bubble up empty inner clauses as an empty optional
+     */
+    public void testFromJsonEmptyQueryBody() throws IOException {
+        String query =
+                "{ \"boosting\" : {" +
+                "    \"positive\" : { }, " +
+                "    \"negative\" : { \"match_all\" : {} }, " +
+                "    \"negative_boost\" : 23.0" +
+                "  }" +
+                "}";
+        XContentParser parser = XContentFactory.xContent(query).createParser(query);
+        QueryParseContext context = createParseContext(parser, ParseFieldMatcher.EMPTY);
+        Optional<QueryBuilder> innerQueryBuilder = context.parseInnerQueryBuilder();
+        assertTrue(innerQueryBuilder.isPresent() == false);
+
+        query =
+                "{ \"boosting\" : {" +
+                "    \"positive\" : { \"match_all\" : {} }, " +
+                "    \"negative\" : { }, " +
+                "    \"negative_boost\" : 23.0" +
+                "  }" +
+                "}";
+        parser = XContentFactory.xContent(query).createParser(query);
+        context = createParseContext(parser, ParseFieldMatcher.EMPTY);
+        innerQueryBuilder = context.parseInnerQueryBuilder();
+        assertTrue(innerQueryBuilder.isPresent() == false);
+
+        parser = XContentFactory.xContent(query).createParser(query);
+        QueryParseContext otherContext = createParseContext(parser, ParseFieldMatcher.STRICT);
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> otherContext.parseInnerQueryBuilder());
+        assertThat(ex.getMessage(), startsWith("query malformed, empty clause found at"));
+    }
+
     public void testRewrite() throws IOException {
         QueryBuilder positive = randomBoolean() ? new MatchAllQueryBuilder() : new WrapperQueryBuilder(new TermQueryBuilder("pos", "bar").toString());
         QueryBuilder negative = randomBoolean() ? new MatchAllQueryBuilder() : new WrapperQueryBuilder(new TermQueryBuilder("neg", "bar").toString());
diff --git a/core/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java
index c24dd25b987..86381c135a5 100644
--- a/core/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/ConstantScoreQueryBuilderTests.java
@@ -21,13 +21,18 @@ package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.Query;
+import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
+import java.util.Optional;
 
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.nullValue;
+import static org.hamcrest.CoreMatchers.startsWith;
 import static org.hamcrest.Matchers.containsString;
 
 public class ConstantScoreQueryBuilderTests extends AbstractQueryTestCase<ConstantScoreQueryBuilder> {
@@ -126,4 +131,24 @@ public class ConstantScoreQueryBuilderTests extends AbstractQueryTestCase<Consta
         assertEquals(json, 42.0, parsed.innerQuery().boost(), 0.0001);
     }
 
+    /**
+     * we bubble up empty query bodies as an empty optional
+     */
+    public void testFromJsonEmptyQueryBody() throws IOException {
+        String query =
+                "{ \"constant_score\" : {" +
+                "    \"filter\" : { }" +
+                "  }" +
+                "}";
+        XContentParser parser = XContentFactory.xContent(query).createParser(query);
+        QueryParseContext context = createParseContext(parser, ParseFieldMatcher.EMPTY);
+        Optional<QueryBuilder> innerQueryBuilder = context.parseInnerQueryBuilder();
+        assertTrue(innerQueryBuilder.isPresent() == false);
+
+        parser = XContentFactory.xContent(query).createParser(query);
+        QueryParseContext otherContext = createParseContext(parser, ParseFieldMatcher.STRICT);
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> otherContext.parseInnerQueryBuilder());
+        assertThat(ex.getMessage(), startsWith("query malformed, empty clause found at"));
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
index 6d2348660c6..61c786d1b01 100644
--- a/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
@@ -24,6 +24,7 @@ import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
+import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -96,15 +97,16 @@ public class DisMaxQueryBuilderTests extends AbstractQueryTestCase<DisMaxQueryBu
     }
 
     /**
-     * Test inner query parsing to null. Current DSL allows inner filter element to parse to <tt>null</tt>.
-     * Those should be ignored upstream. To test this, we use inner {@link ConstantScoreQueryBuilder}
-     * with empty inner filter.
+     * Test with empty inner query body, this should be ignored upstream.
+     * To test this, we use inner {@link ConstantScoreQueryBuilder} with empty inner filter.
      */
-    public void testInnerQueryReturnsNull() throws IOException {
-        String queryString = "{ \"" + ConstantScoreQueryBuilder.NAME + "\" : { \"filter\" : { } } }";
-        QueryBuilder innerQueryBuilder = parseQuery(queryString);
-        DisMaxQueryBuilder disMaxBuilder = new DisMaxQueryBuilder().add(innerQueryBuilder);
-        assertNull(disMaxBuilder.toQuery(createShardContext()));
+    public void testInnerQueryEmptyIgnored() throws IOException {
+        String queryString = "{ \"" + DisMaxQueryBuilder.NAME + "\" :"
+                + "             { \"queries\" : [ {\"" + ConstantScoreQueryBuilder.NAME + "\" : { \"filter\" : { } } } ] "
+                + "             }"
+                + "           }";
+        DisMaxQueryBuilder builder = (DisMaxQueryBuilder) parseQuery(queryString, ParseFieldMatcher.EMPTY);
+        assertTrue("the inner query has an empty body, so it should be ignored", builder.innerQueries().isEmpty());
     }
 
     public void testIllegalArguments() {
diff --git a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java
index 1c6a99ca099..d0a924be0ac 100644
--- a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java
@@ -35,9 +35,12 @@ import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
+import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
@@ -56,11 +59,13 @@ import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.CoreMatchers.startsWith;
 import static org.hamcrest.Matchers.is;
 
 public class HasChildQueryBuilderTests extends AbstractQueryTestCase<HasChildQueryBuilder> {
@@ -116,16 +121,11 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase<HasChildQue
 
     @Override
     protected void doAssertLuceneQuery(HasChildQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
-        QueryBuilder innerQueryBuilder = queryBuilder.query();
-        if (innerQueryBuilder instanceof EmptyQueryBuilder) {
-            assertNull(query);
-        } else {
-            assertThat(query, instanceOf(HasChildQueryBuilder.LateParsingQuery.class));
-            HasChildQueryBuilder.LateParsingQuery lpq = (HasChildQueryBuilder.LateParsingQuery) query;
-            assertEquals(queryBuilder.minChildren(), lpq.getMinChildren());
-            assertEquals(queryBuilder.maxChildren(), lpq.getMaxChildren());
-            assertEquals(queryBuilder.scoreMode(), lpq.getScoreMode()); // WTF is this why do we have two?
-        }
+        assertThat(query, instanceOf(HasChildQueryBuilder.LateParsingQuery.class));
+        HasChildQueryBuilder.LateParsingQuery lpq = (HasChildQueryBuilder.LateParsingQuery) query;
+        assertEquals(queryBuilder.minChildren(), lpq.getMinChildren());
+        assertEquals(queryBuilder.maxChildren(), lpq.getMaxChildren());
+        assertEquals(queryBuilder.scoreMode(), lpq.getScoreMode()); // WTF is this why do we have two?
         if (queryBuilder.innerHit() != null) {
             SearchContext searchContext = SearchContext.current();
             assertNotNull(searchContext);
@@ -224,8 +224,29 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase<HasChildQue
                 .setSize(100)
                 .addSort(new FieldSortBuilder("mapped_string").order(SortOrder.ASC));
         assertEquals(query, queryBuilder.innerHit(), expected);
-
     }
+
+    /**
+     * we resolve empty inner clauses by representing this whole query as empty optional upstream
+     */
+    public void testFromJsonEmptyQueryBody() throws IOException {
+        String query =  "{\n" +
+                "  \"has_child\" : {\n" +
+                "    \"query\" : { },\n" +
+                "    \"type\" : \"child\"" +
+                "   }" +
+                "}";
+        XContentParser parser = XContentFactory.xContent(query).createParser(query);
+        QueryParseContext context = createParseContext(parser, ParseFieldMatcher.EMPTY);
+        Optional<QueryBuilder> innerQueryBuilder = context.parseInnerQueryBuilder();
+        assertTrue(innerQueryBuilder.isPresent() == false);
+
+        parser = XContentFactory.xContent(query).createParser(query);
+        QueryParseContext otherContext = createParseContext(parser, ParseFieldMatcher.STRICT);
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> otherContext.parseInnerQueryBuilder());
+        assertThat(ex.getMessage(), startsWith("query malformed, empty clause found at"));
+    }
+
     public void testToQueryInnerQueryType() throws IOException {
         String[] searchTypes = new String[]{PARENT_TYPE};
         QueryShardContext shardContext = createShardContext();
diff --git a/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java
index 7b700cbfc7f..c4ba395db51 100644
--- a/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java
@@ -32,6 +32,7 @@ import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.script.Script.ScriptParseException;
 import org.elasticsearch.search.fetch.innerhits.InnerHitsContext;
@@ -45,11 +46,13 @@ import org.junit.BeforeClass;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.CoreMatchers.startsWith;
 
 public class HasParentQueryBuilderTests extends AbstractQueryTestCase<HasParentQueryBuilder> {
     protected static final String PARENT_TYPE = "parent";
@@ -99,14 +102,10 @@ public class HasParentQueryBuilderTests extends AbstractQueryTestCase<HasParentQ
 
     @Override
     protected void doAssertLuceneQuery(HasParentQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
-        QueryBuilder innerQueryBuilder = queryBuilder.query();
-        if (innerQueryBuilder instanceof EmptyQueryBuilder) {
-            assertNull(query);
-        } else {
-            assertThat(query, instanceOf(HasChildQueryBuilder.LateParsingQuery.class));
-            HasChildQueryBuilder.LateParsingQuery lpq = (HasChildQueryBuilder.LateParsingQuery) query;
-            assertEquals(queryBuilder.score() ? ScoreMode.Max : ScoreMode.None, lpq.getScoreMode());
-        }
+        assertThat(query, instanceOf(HasChildQueryBuilder.LateParsingQuery.class));
+        HasChildQueryBuilder.LateParsingQuery lpq = (HasChildQueryBuilder.LateParsingQuery) query;
+        assertEquals(queryBuilder.score() ? ScoreMode.Max : ScoreMode.None, lpq.getScoreMode());
+
         if (queryBuilder.innerHit() != null) {
             SearchContext searchContext = SearchContext.current();
             assertNotNull(searchContext);
@@ -231,6 +230,27 @@ public class HasParentQueryBuilderTests extends AbstractQueryTestCase<HasParentQ
         assertEquals(json, "something", ((TermQueryBuilder) parsed.query()).value());
     }
 
+    /**
+     * we resolve empty inner clauses by representing this whole query as empty optional upstream
+     */
+    public void testFromJsonEmptyQueryBody() throws IOException {
+        String query =  "{\n" +
+                "  \"has_parent\" : {\n" +
+                "    \"query\" : { },\n" +
+                "    \"parent_type\" : \"blog\"" +
+                "   }" +
+                "}";
+        XContentParser parser = XContentFactory.xContent(query).createParser(query);
+        QueryParseContext context = createParseContext(parser, ParseFieldMatcher.EMPTY);
+        Optional<QueryBuilder> innerQueryBuilder = context.parseInnerQueryBuilder();
+        assertTrue(innerQueryBuilder.isPresent() == false);
+
+        parser = XContentFactory.xContent(query).createParser(query);
+        QueryParseContext otherContext = createParseContext(parser, ParseFieldMatcher.STRICT);
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> otherContext.parseInnerQueryBuilder());
+        assertThat(ex.getMessage(), startsWith("query malformed, empty clause found at"));
+    }
+
     public void testIgnoreUnmapped() throws IOException {
         final HasParentQueryBuilder queryBuilder = new HasParentQueryBuilder("unmapped", new MatchAllQueryBuilder(), false);
         queryBuilder.ignoreUnmapped(true);
diff --git a/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java
index 04bbadb02c2..d93bcce6dca 100644
--- a/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java
@@ -83,13 +83,9 @@ public class NestedQueryBuilderTests extends AbstractQueryTestCase<NestedQueryBu
     @Override
     protected void doAssertLuceneQuery(NestedQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
         QueryBuilder innerQueryBuilder = queryBuilder.query();
-        if (innerQueryBuilder instanceof EmptyQueryBuilder) {
-            assertNull(query);
-        } else {
-            assertThat(query, instanceOf(ToParentBlockJoinQuery.class));
-            ToParentBlockJoinQuery parentBlockJoinQuery = (ToParentBlockJoinQuery) query;
-            //TODO how to assert this?
-        }
+        assertThat(query, instanceOf(ToParentBlockJoinQuery.class));
+        ToParentBlockJoinQuery parentBlockJoinQuery = (ToParentBlockJoinQuery) query;
+        // TODO how to assert this?
         if (queryBuilder.innerHit() != null) {
             SearchContext searchContext = SearchContext.current();
             assertNotNull(searchContext);
diff --git a/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java b/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java
index cf4921b7a43..c244273d13a 100644
--- a/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java
+++ b/core/src/test/java/org/elasticsearch/index/query/RandomQueryBuilder.java
@@ -38,7 +38,7 @@ public class RandomQueryBuilder {
      * @return a random {@link QueryBuilder}
      */
     public static QueryBuilder createQuery(Random r) {
-        switch (RandomInts.randomIntBetween(r, 0, 4)) {
+        switch (RandomInts.randomIntBetween(r, 0, 3)) {
             case 0:
                 return new MatchAllQueryBuilderTests().createTestQueryBuilder();
             case 1:
@@ -47,8 +47,6 @@ public class RandomQueryBuilder {
                 return new IdsQueryBuilderTests().createTestQueryBuilder();
             case 3:
                 return createMultiTermQuery(r);
-            case 4:
-                return new EmptyQueryBuilder();
             default:
                 throw new UnsupportedOperationException();
         }
diff --git a/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
index a8bdeb69362..985669394d9 100644
--- a/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/TermsQueryBuilderTests.java
@@ -239,25 +239,25 @@ public class TermsQueryBuilderTests extends AbstractQueryTestCase<TermsQueryBuil
     public void testNumeric() throws IOException {
         {
             TermsQueryBuilder builder = new TermsQueryBuilder("foo", new int[]{1, 3, 4});
-            TermsQueryBuilder copy = assertSerialization(builder);
+            TermsQueryBuilder copy = (TermsQueryBuilder) assertSerialization(builder);
             List<Object> values = copy.values();
             assertEquals(Arrays.asList(1, 3, 4), values);
         }
         {
             TermsQueryBuilder builder = new TermsQueryBuilder("foo", new double[]{1, 3, 4});
-            TermsQueryBuilder copy = assertSerialization(builder);
+            TermsQueryBuilder copy = (TermsQueryBuilder) assertSerialization(builder);
             List<Object> values = copy.values();
             assertEquals(Arrays.asList(1d, 3d, 4d), values);
         }
         {
             TermsQueryBuilder builder = new TermsQueryBuilder("foo", new float[]{1, 3, 4});
-            TermsQueryBuilder copy = assertSerialization(builder);
+            TermsQueryBuilder copy = (TermsQueryBuilder) assertSerialization(builder);
             List<Object> values = copy.values();
             assertEquals(Arrays.asList(1f, 3f, 4f), values);
         }
         {
             TermsQueryBuilder builder = new TermsQueryBuilder("foo", new long[]{1, 3, 4});
-            TermsQueryBuilder copy = assertSerialization(builder);
+            TermsQueryBuilder copy = (TermsQueryBuilder) assertSerialization(builder);
             List<Object> values = copy.values();
             assertEquals(Arrays.asList(1L, 3L, 4L), values);
         }
diff --git a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
index 52632edcf47..2597675095a 100644
--- a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
@@ -26,6 +26,7 @@ import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -599,8 +600,30 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase<Functi
                 "  }\n" +
                 "}";
 
-        FunctionScoreQueryBuilder parsed = (FunctionScoreQueryBuilder) parseQuery(json);
-        checkGeneratedJson(json, parsed);
+        FunctionScoreQueryBuilder parsed = (FunctionScoreQueryBuilder) parseQuery(json, ParseFieldMatcher.EMPTY);
+        // this should be equivalent to the same with a match_all query
+        String expected =
+                "{\n" +
+                    "  \"function_score\" : {\n" +
+                    "    \"query\" : { \"match_all\" : {} },\n" +
+                    "    \"functions\" : [ {\n" +
+                    "      \"filter\" : { },\n" +
+                    "      \"weight\" : 23.0,\n" +
+                    "      \"random_score\" : { }\n" +
+                    "    }, {\n" +
+                    "      \"filter\" : { },\n" +
+                    "      \"weight\" : 5.0\n" +
+                    "    } ],\n" +
+                    "    \"score_mode\" : \"multiply\",\n" +
+                    "    \"boost_mode\" : \"multiply\",\n" +
+                    "    \"max_boost\" : 100.0,\n" +
+                    "    \"min_score\" : 1.0,\n" +
+                    "    \"boost\" : 42.0\n" +
+                    "  }\n" +
+                    "}";
+
+        FunctionScoreQueryBuilder expectedParsed = (FunctionScoreQueryBuilder) parseQuery(json, ParseFieldMatcher.EMPTY);
+        assertEquals(expectedParsed, parsed);
 
         assertEquals(json, 2, parsed.filterFunctionBuilders().length);
         assertEquals(json, 42, parsed.boost(), 0.0001);
diff --git a/core/src/test/java/org/elasticsearch/index/query/plugin/CustomQueryParserIT.java b/core/src/test/java/org/elasticsearch/index/query/plugin/CustomQueryParserIT.java
index ec405bd8407..ba87b293701 100644
--- a/core/src/test/java/org/elasticsearch/index/query/plugin/CustomQueryParserIT.java
+++ b/core/src/test/java/org/elasticsearch/index/query/plugin/CustomQueryParserIT.java
@@ -59,11 +59,11 @@ public class CustomQueryParserIT extends ESIntegTestCase {
     }
 
     public void testCustomDummyQuery() {
-        assertHitCount(client().prepareSearch("index").setQuery(new DummyQueryParserPlugin.DummyQueryBuilder()).get(), 1L);
+        assertHitCount(client().prepareSearch("index").setQuery(new DummyQueryBuilder()).get(), 1L);
     }
 
     public void testCustomDummyQueryWithinBooleanQuery() {
-        assertHitCount(client().prepareSearch("index").setQuery(new BoolQueryBuilder().must(new DummyQueryParserPlugin.DummyQueryBuilder())).get(), 1L);
+        assertHitCount(client().prepareSearch("index").setQuery(new BoolQueryBuilder().must(new DummyQueryBuilder())).get(), 1L);
     }
 
     private static QueryShardContext queryShardContext() {
@@ -73,7 +73,7 @@ public class CustomQueryParserIT extends ESIntegTestCase {
 
     //see #11120
     public void testConstantScoreParsesFilter() throws Exception {
-        Query q = constantScoreQuery(new DummyQueryParserPlugin.DummyQueryBuilder()).toQuery(queryShardContext());
+        Query q = constantScoreQuery(new DummyQueryBuilder()).toQuery(queryShardContext());
         Query inner = ((ConstantScoreQuery) q).getQuery();
         assertThat(inner, instanceOf(DummyQueryParserPlugin.DummyQuery.class));
         assertEquals(true, ((DummyQueryParserPlugin.DummyQuery) inner).isFilter);
@@ -83,10 +83,10 @@ public class CustomQueryParserIT extends ESIntegTestCase {
     public void testBooleanParsesFilter() throws Exception {
         // single clause, serialized as inner object
         Query q = boolQuery()
-                .should(new DummyQueryParserPlugin.DummyQueryBuilder())
-                .must(new DummyQueryParserPlugin.DummyQueryBuilder())
-                .filter(new DummyQueryParserPlugin.DummyQueryBuilder())
-                .mustNot(new DummyQueryParserPlugin.DummyQueryBuilder()).toQuery(queryShardContext());
+                .should(new DummyQueryBuilder())
+                .must(new DummyQueryBuilder())
+                .filter(new DummyQueryBuilder())
+                .mustNot(new DummyQueryBuilder()).toQuery(queryShardContext());
         assertThat(q, instanceOf(BooleanQuery.class));
         BooleanQuery bq = (BooleanQuery) q;
         assertEquals(4, bq.clauses().size());
@@ -108,10 +108,10 @@ public class CustomQueryParserIT extends ESIntegTestCase {
 
         // multiple clauses, serialized as inner arrays
         q = boolQuery()
-                .should(new DummyQueryParserPlugin.DummyQueryBuilder()).should(new DummyQueryParserPlugin.DummyQueryBuilder())
-                .must(new DummyQueryParserPlugin.DummyQueryBuilder()).must(new DummyQueryParserPlugin.DummyQueryBuilder())
-                .filter(new DummyQueryParserPlugin.DummyQueryBuilder()).filter(new DummyQueryParserPlugin.DummyQueryBuilder())
-                .mustNot(new DummyQueryParserPlugin.DummyQueryBuilder()).mustNot(new DummyQueryParserPlugin.DummyQueryBuilder()).toQuery(queryShardContext());
+                .should(new DummyQueryBuilder()).should(new DummyQueryBuilder())
+                .must(new DummyQueryBuilder()).must(new DummyQueryBuilder())
+                .filter(new DummyQueryBuilder()).filter(new DummyQueryBuilder())
+                .mustNot(new DummyQueryBuilder()).mustNot(new DummyQueryBuilder()).toQuery(queryShardContext());
         assertThat(q, instanceOf(BooleanQuery.class));
         bq = (BooleanQuery) q;
         assertEquals(8, bq.clauses().size());
diff --git a/core/src/main/java/org/elasticsearch/index/query/EmptyQueryBuilder.java b/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryBuilder.java
similarity index 55%
rename from core/src/main/java/org/elasticsearch/index/query/EmptyQueryBuilder.java
rename to core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryBuilder.java
index 3d09bb41b0a..334244559c9 100644
--- a/core/src/main/java/org/elasticsearch/index/query/EmptyQueryBuilder.java
+++ b/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryBuilder.java
@@ -17,67 +17,66 @@
  * under the License.
  */
 
-package org.elasticsearch.index.query;
+package org.elasticsearch.index.query.plugin;
 
 import org.apache.lucene.search.Query;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.AbstractQueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.query.plugin.DummyQueryParserPlugin.DummyQuery;
 
 import java.io.IOException;
+import java.util.Optional;
 
-/**
- * A {@link QueryBuilder} that is a stand in replacement for an empty query clause in the DSL.
- * The current DSL allows parsing inner queries / filters like "{ }", in order to have a
- * valid non-null representation of these clauses that actually do nothing we can use this class.
- */
-public final class EmptyQueryBuilder extends AbstractQueryBuilder<EmptyQueryBuilder> {
+public class DummyQueryBuilder extends AbstractQueryBuilder<DummyQueryBuilder> {
+    private static final String NAME = "dummy";
+    static final ParseField QUERY_NAME_FIELD = new ParseField(NAME);
 
-    public static final String NAME = "empty_query";
-
-    /**
-     * Construct an empty query. This query can *technically* be named and given a boost.
-     */
-    public EmptyQueryBuilder() {
+    public DummyQueryBuilder() {
     }
 
-    /**
-     * Read from a stream.
-     */
-    public EmptyQueryBuilder(StreamInput in) throws IOException {
+    public DummyQueryBuilder(StreamInput in) throws IOException {
         super(in);
     }
 
     @Override
     protected void doWriteTo(StreamOutput out) throws IOException {
+        // only the superclass has state
+    }
+
+    @Override
+    protected void doXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(NAME).endObject();
+    }
+
+    public static Optional<DummyQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
+        XContentParser.Token token = parseContext.parser().nextToken();
+        assert token == XContentParser.Token.END_OBJECT;
+        return Optional.of(new DummyQueryBuilder());
+    }
+
+    @Override
+    protected Query doToQuery(QueryShardContext context) throws IOException {
+        return new DummyQuery(context.isFilter());
+    }
+
+    @Override
+    protected int doHashCode() {
+        return 0;
+    }
+
+    @Override
+    protected boolean doEquals(DummyQueryBuilder other) {
+        return true;
     }
 
     @Override
     public String getWriteableName() {
         return NAME;
     }
-
-    @Override
-    protected Query doToQuery(QueryShardContext context) throws IOException {
-        return null;
-    }
-
-    @Override
-    public String getName() {
-        return getWriteableName();
-    }
-
-    @Override
-    protected void doXContent(XContentBuilder builder, Params params) throws IOException {
-    }
-
-    @Override
-    protected int doHashCode() {
-        return 31;
-    }
-
-    @Override
-    protected boolean doEquals(EmptyQueryBuilder other) {
-        return true;
-    }
-}
+}
\ No newline at end of file
diff --git a/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java b/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java
index 54d1026a7f0..fbb744f799c 100644
--- a/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java
+++ b/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java
@@ -23,14 +23,6 @@ import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Weight;
-import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.index.query.AbstractQueryBuilder;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchModule;
 
@@ -52,62 +44,11 @@ public class DummyQueryParserPlugin extends Plugin {
         module.registerQuery(DummyQueryBuilder::new, DummyQueryBuilder::fromXContent, DummyQueryBuilder.QUERY_NAME_FIELD);
     }
 
-    public static class DummyQueryBuilder extends AbstractQueryBuilder<DummyQueryBuilder> {
-        private static final String NAME = "dummy";
-        private static final ParseField QUERY_NAME_FIELD = new ParseField(NAME);
-
-        public DummyQueryBuilder() {
-        }
-
-        /**
-         * Read from a stream.
-         */
-        public DummyQueryBuilder(StreamInput in) throws IOException {
-            super(in);
-        }
-
-        @Override
-        protected void doWriteTo(StreamOutput out) throws IOException {
-            // only the superclass has state
-        }
-
-        @Override
-        protected void doXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.startObject(NAME).endObject();
-        }
-
-        public static DummyQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
-            XContentParser.Token token = parseContext.parser().nextToken();
-            assert token == XContentParser.Token.END_OBJECT;
-            return new DummyQueryBuilder();
-        }
-
-        @Override
-        protected Query doToQuery(QueryShardContext context) throws IOException {
-            return new DummyQuery(context.isFilter());
-        }
-
-        @Override
-        protected int doHashCode() {
-            return 0;
-        }
-
-        @Override
-        protected boolean doEquals(DummyQueryBuilder other) {
-            return true;
-        }
-
-        @Override
-        public String getWriteableName() {
-            return NAME;
-        }
-    }
-
     public static class DummyQuery extends Query {
         public final boolean isFilter;
         private final Query matchAllDocsQuery = new MatchAllDocsQuery();
 
-        private DummyQuery(boolean isFilter) {
+        public DummyQuery(boolean isFilter) {
             this.isFilter = isFilter;
         }
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java
index 9859936b410..ddb62d57ffe 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java
@@ -21,11 +21,17 @@ package org.elasticsearch.search.aggregations.bucket;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.EmptyQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
+import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.metrics.avg.Avg;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -42,8 +48,8 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.filter;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.core.IsNull.notNullValue;
 
 /**
@@ -119,9 +125,16 @@ public class FilterIT extends ESIntegTestCase {
         assertThat(filter.getDocCount(), equalTo((long) numDocs));
     }
 
+    /**
+     * test that "{ "filter" : {} }" is regarded as match_all when not parsing strict
+     */
     public void testEmptyFilter() throws Exception {
-        QueryBuilder emptyFilter = new EmptyQueryBuilder();
-        SearchResponse response = client().prepareSearch("idx").addAggregation(filter("tag1", emptyFilter)).execute().actionGet();
+        String emtpyFilterBody = "{ }";
+        XContentParser parser = XContentFactory.xContent(emtpyFilterBody).createParser(emtpyFilterBody);
+        QueryParseContext parseContext = new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.EMPTY);
+        AggregationBuilder<?> filterAgg = FilterAggregationBuilder.parse("tag1", parseContext);
+
+        SearchResponse response = client().prepareSearch("idx").addAggregation(filterAgg).execute().actionGet();
 
         assertSearchResponse(response);
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java
index fd4db82f2db..0be0f76de90 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java
@@ -22,11 +22,17 @@ package org.elasticsearch.search.aggregations.bucket;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.EmptyQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.filters.Filters;
+import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregator.KeyedFilter;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.metrics.avg.Avg;
@@ -207,8 +213,13 @@ public class FiltersIT extends ESIntegTestCase {
     }
 
     public void testEmptyFilter() throws Exception {
-        QueryBuilder emptyFilter = new EmptyQueryBuilder();
-        SearchResponse response = client().prepareSearch("idx").addAggregation(filters("tag1", emptyFilter)).execute().actionGet();
+        String emtpyFilterBody = "{ \"filters\" : [ {} ] }";
+        XContentParser parser = XContentFactory.xContent(emtpyFilterBody).createParser(emtpyFilterBody);
+        parser.nextToken();
+        QueryParseContext parseContext = new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.EMPTY);
+        AggregationBuilder<?> filtersAgg = FiltersAggregationBuilder.parse("tag1", parseContext);
+
+        SearchResponse response = client().prepareSearch("idx").addAggregation(filtersAgg).execute().actionGet();
 
         assertSearchResponse(response);
 
@@ -219,8 +230,13 @@ public class FiltersIT extends ESIntegTestCase {
     }
 
     public void testEmptyKeyedFilter() throws Exception {
-        QueryBuilder emptyFilter = new EmptyQueryBuilder();
-        SearchResponse response = client().prepareSearch("idx").addAggregation(filters("tag1", new KeyedFilter("foo", emptyFilter)))
+        String emtpyFilterBody = "{ \"filters\" : {\"foo\" : {} } }";
+        XContentParser parser = XContentFactory.xContent(emtpyFilterBody).createParser(emtpyFilterBody);
+        parser.nextToken();
+        QueryParseContext parseContext = new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.EMPTY);
+        AggregationBuilder<?> filtersAgg = FiltersAggregationBuilder.parse("tag1", parseContext);
+
+        SearchResponse response = client().prepareSearch("idx").addAggregation(filtersAgg)
                 .execute().actionGet();
 
         assertSearchResponse(response);
diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index f8236f66e48..fc3c90b5165 100644
--- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -48,8 +48,6 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.EnvironmentModule;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.test.AbstractQueryTestCase;
-import org.elasticsearch.index.query.EmptyQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.indices.IndicesModule;
@@ -80,6 +78,7 @@ import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.search.suggest.SuggestBuilderTests;
 import org.elasticsearch.search.suggest.Suggesters;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -440,11 +439,17 @@ public class SearchSourceBuilderTests extends ESTestCase {
         assertParseSearchSource(testSearchSourceBuilder, builder.bytes());
     }
 
-    private void assertParseSearchSource(SearchSourceBuilder testBuilder, BytesReference searchSourceAsBytes) throws IOException {
+    private static void assertParseSearchSource(SearchSourceBuilder testBuilder, BytesReference searchSourceAsBytes) throws IOException {
+        assertParseSearchSource(testBuilder, searchSourceAsBytes, ParseFieldMatcher.STRICT);
+    }
+
+    private static void assertParseSearchSource(SearchSourceBuilder testBuilder, BytesReference searchSourceAsBytes, ParseFieldMatcher pfm)
+            throws IOException {
         XContentParser parser = XContentFactory.xContent(searchSourceAsBytes).createParser(searchSourceAsBytes);
-        QueryParseContext parseContext = createParseContext(parser);
+        QueryParseContext parseContext = new QueryParseContext(indicesQueriesRegistry, parser, pfm);
         if (randomBoolean()) {
-            parser.nextToken(); // sometimes we move it on the START_OBJECT to test the embedded case
+            parser.nextToken(); // sometimes we move it on the START_OBJECT to
+                                // test the embedded case
         }
         SearchSourceBuilder newBuilder = SearchSourceBuilder.fromXContent(parseContext, aggParsers, suggesters);
         assertNull(parser.nextToken());
@@ -614,8 +619,13 @@ public class SearchSourceBuilderTests extends ESTestCase {
 
     public void testEmptyPostFilter() throws IOException {
         SearchSourceBuilder builder = new SearchSourceBuilder();
-        builder.postFilter(new EmptyQueryBuilder());
         String query = "{ \"post_filter\": {} }";
-        assertParseSearchSource(builder, new BytesArray(query));
+        assertParseSearchSource(builder, new BytesArray(query), ParseFieldMatcher.EMPTY);
+    }
+
+    public void testEmptyQuery() throws IOException {
+        SearchSourceBuilder builder = new SearchSourceBuilder();
+        String query = "{ \"query\": {} }";
+        assertParseSearchSource(builder, new BytesArray(query), ParseFieldMatcher.EMPTY);
     }
 }
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
index 7097718c6fc..9aa5e1892ef 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
@@ -169,7 +169,7 @@ public class TemplateQueryParserTests extends ESTestCase {
         QueryShardContext context = contextFactory.get();
         templateSourceParser.nextToken();
 
-        Query query = QueryBuilder.rewriteQuery(TemplateQueryBuilder.fromXContent(context.newParseContext(templateSourceParser)),
+        Query query = QueryBuilder.rewriteQuery(TemplateQueryBuilder.fromXContent(context.newParseContext(templateSourceParser)).get(),
                 context).toQuery(context);
         assertTrue("Parsing template query failed.", query instanceof MatchAllDocsQuery);
     }
@@ -180,7 +180,9 @@ public class TemplateQueryParserTests extends ESTestCase {
         XContentParser templateSourceParser = XContentFactory.xContent(templateString).createParser(templateString);
         QueryShardContext context = contextFactory.get();
 
-        Query query = QueryBuilder.rewriteQuery(TemplateQueryBuilder.fromXContent(context.newParseContext(templateSourceParser)), context).toQuery(context);
+        Query query = QueryBuilder
+                .rewriteQuery(TemplateQueryBuilder.fromXContent(context.newParseContext(templateSourceParser)).get(), context)
+                .toQuery(context);
         assertTrue("Parsing template query failed.", query instanceof MatchAllDocsQuery);
     }
 
@@ -197,7 +199,7 @@ public class TemplateQueryParserTests extends ESTestCase {
         QueryShardContext context = contextFactory.get();
 
         try {
-            TemplateQueryBuilder.fromXContent(context.newParseContext(templateSourceParser)).rewrite(context);
+            TemplateQueryBuilder.fromXContent(context.newParseContext(templateSourceParser)).get().rewrite(context);
             fail("Expected ParsingException");
         } catch (ParsingException e) {
             assertThat(e.getMessage(), containsString("query malformed, no field after start_object"));
@@ -212,7 +214,7 @@ public class TemplateQueryParserTests extends ESTestCase {
         templateSourceParser.nextToken();
 
 
-        Query query = QueryBuilder.rewriteQuery(TemplateQueryBuilder.fromXContent(context.newParseContext(templateSourceParser)),
+        Query query = QueryBuilder.rewriteQuery(TemplateQueryBuilder.fromXContent(context.newParseContext(templateSourceParser)).get(),
                 context).toQuery(context);
         assertTrue("Parsing template query failed.", query instanceof MatchAllDocsQuery);
     }
@@ -224,7 +226,7 @@ public class TemplateQueryParserTests extends ESTestCase {
         QueryShardContext context = contextFactory.get();
         templateSourceParser.nextToken();
         try {
-            TemplateQueryBuilder.fromXContent(context.newParseContext(templateSourceParser)).toQuery(context);
+            TemplateQueryBuilder.fromXContent(context.newParseContext(templateSourceParser)).get().toQuery(context);
             fail();
         } catch (UnsupportedOperationException ex) {
             assertEquals("this query must be rewritten first", ex.getMessage());
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
index 627cb09da19..aadcabda006 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
@@ -76,6 +76,7 @@ import org.elasticsearch.index.query.QueryShardException;
 import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 
 import static org.elasticsearch.index.mapper.SourceToParse.source;
 import static org.elasticsearch.percolator.PercolatorFieldMapper.parseQuery;
@@ -235,7 +236,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
         builder.endObject();
     }
 
-    public static PercolateQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException {
+    public static Optional<PercolateQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
 
@@ -316,7 +317,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
         }
         queryBuilder.queryName(queryName);
         queryBuilder.boost(boost);
-        return queryBuilder;
+        return Optional.of(queryBuilder);
     }
 
     @Override
@@ -446,7 +447,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
         return document;
     }
 
-    private IndexSearcher createMultiDocumentSearcher(Analyzer analyzer, ParsedDocument doc) {
+    private static IndexSearcher createMultiDocumentSearcher(Analyzer analyzer, ParsedDocument doc) {
         IndexReader[] memoryIndices = new IndexReader[doc.docs().size()];
         List<ParseContext.Document> docs = doc.docs();
         int rootDocIndex = docs.size() - 1;
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
index bf5c4c0ae8b..a19a207eb76 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
@@ -250,7 +250,8 @@ public class PercolatorFieldMapper extends FieldMapper {
 
     private static QueryBuilder parseQueryBuilder(QueryParseContext context, XContentLocation location) {
         try {
-            return context.parseInnerQueryBuilder();
+            return context.parseInnerQueryBuilder()
+                    .orElseThrow(() -> new ParsingException(location, "Failed to parse inner query, was empty"));
         } catch (IOException e) {
             throw new ParsingException(location, "Failed to parse", e);
         }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java
index f3ee9230eaa..9e86ae44cdd 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java
@@ -42,7 +42,6 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
@@ -202,9 +201,8 @@ public class TransportPercolateAction extends HandledTransportAction<PercolateRe
         if (querySource != null) {
             try (XContentParser parser = XContentHelper.createParser(querySource)) {
                 QueryParseContext queryParseContext = new QueryParseContext(queryRegistry, parser, parseFieldMatcher);
-                QueryBuilder queryBuilder = queryParseContext.parseInnerQueryBuilder();
                 BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
-                boolQueryBuilder.must(queryBuilder);
+                queryParseContext.parseInnerQueryBuilder().ifPresent(boolQueryBuilder::must);
                 boolQueryBuilder.filter(percolateQueryBuilder);
                 searchSource.field("query", boolQueryBuilder);
             }
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
index bcb9ed8d40d..2175dabfefb 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
@@ -272,6 +272,6 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
         XContentParser sourceParser = PercolatorFieldMapper.QUERY_BUILDER_CONTENT_TYPE.xContent()
                 .createParser(actual.bytes, actual.offset, actual.length);
         QueryParseContext qsc = indexService.newQueryShardContext().newParseContext(sourceParser);
-        assertThat(qsc.parseInnerQueryBuilder(), equalTo(expected));
+        assertThat(qsc.parseInnerQueryBuilder().get(), equalTo(expected));
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
index c27617a7287..2e0bfe28911 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
@@ -336,11 +336,11 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
     /**
      * Parses the query provided as string argument and compares it with the expected result provided as argument as a {@link QueryBuilder}
      */
-    protected final void assertParsedQuery(String queryAsString, QueryBuilder expectedQuery) throws IOException {
+    protected final static void assertParsedQuery(String queryAsString, QueryBuilder expectedQuery) throws IOException {
         assertParsedQuery(queryAsString, expectedQuery, ParseFieldMatcher.STRICT);
     }
 
-    protected final void assertParsedQuery(String queryAsString, QueryBuilder expectedQuery, ParseFieldMatcher matcher) throws IOException {
+    protected final static void assertParsedQuery(String queryAsString, QueryBuilder expectedQuery, ParseFieldMatcher matcher) throws IOException {
         QueryBuilder newQuery = parseQuery(queryAsString, matcher);
         assertNotSame(newQuery, expectedQuery);
         assertEquals(expectedQuery, newQuery);
@@ -350,38 +350,39 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
     /**
      * Parses the query provided as bytes argument and compares it with the expected result provided as argument as a {@link QueryBuilder}
      */
-    protected final void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery) throws IOException {
+    protected final static void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery) throws IOException {
         assertParsedQuery(queryAsBytes, expectedQuery, ParseFieldMatcher.STRICT);
     }
 
-    protected final void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery, ParseFieldMatcher matcher) throws IOException {
+    protected final static void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery, ParseFieldMatcher matcher) throws IOException {
         QueryBuilder newQuery = parseQuery(queryAsBytes, matcher);
         assertNotSame(newQuery, expectedQuery);
         assertEquals(expectedQuery, newQuery);
         assertEquals(expectedQuery.hashCode(), newQuery.hashCode());
     }
 
-    protected final QueryBuilder parseQuery(String queryAsString) throws IOException {
+    protected final static QueryBuilder parseQuery(String queryAsString) throws IOException {
         return parseQuery(queryAsString, ParseFieldMatcher.STRICT);
     }
 
-    protected final QueryBuilder parseQuery(String queryAsString, ParseFieldMatcher matcher) throws IOException {
+    protected final static QueryBuilder parseQuery(String queryAsString, ParseFieldMatcher matcher) throws IOException {
         XContentParser parser = XContentFactory.xContent(queryAsString).createParser(queryAsString);
         return parseQuery(parser, matcher);
     }
 
-    protected final QueryBuilder parseQuery(BytesReference queryAsBytes) throws IOException {
+    protected final static QueryBuilder parseQuery(BytesReference queryAsBytes) throws IOException {
         return parseQuery(queryAsBytes, ParseFieldMatcher.STRICT);
     }
 
-    protected final QueryBuilder parseQuery(BytesReference queryAsBytes, ParseFieldMatcher matcher) throws IOException {
+    protected final static QueryBuilder parseQuery(BytesReference queryAsBytes, ParseFieldMatcher matcher) throws IOException {
         XContentParser parser = XContentFactory.xContent(queryAsBytes).createParser(queryAsBytes);
         return parseQuery(parser, matcher);
     }
 
-    private QueryBuilder parseQuery(XContentParser parser, ParseFieldMatcher matcher) throws IOException {
+    private static QueryBuilder parseQuery(XContentParser parser, ParseFieldMatcher matcher) throws IOException {
         QueryParseContext context = createParseContext(parser, matcher);
-        QueryBuilder parseInnerQueryBuilder = context.parseInnerQueryBuilder();
+        QueryBuilder parseInnerQueryBuilder = context.parseInnerQueryBuilder()
+                .orElseThrow(() -> new IllegalArgumentException("inner query body cannot be empty"));
         assertNull(parser.nextToken());
         return parseInnerQueryBuilder;
     }
@@ -518,8 +519,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
     /**
      * Serialize the given query builder and asserts that both are equal
      */
-    @SuppressWarnings("unchecked")
-    protected <QB extends QueryBuilder> QB assertSerialization(QB testQuery) throws IOException {
+    protected QueryBuilder assertSerialization(QueryBuilder testQuery) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             output.writeNamedWriteable(testQuery);
             try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), serviceHolder.namedWriteableRegistry)) {
@@ -527,7 +527,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
                 assertEquals(testQuery, deserializedQuery);
                 assertEquals(testQuery.hashCode(), deserializedQuery.hashCode());
                 assertNotSame(testQuery, deserializedQuery);
-                return (QB) deserializedQuery;
+                return deserializedQuery;
             }
         }
     }

From cb145aec684ccb029943bcbf5d0fb6a51557e26c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Wed, 25 May 2016 11:18:40 +0200
Subject: [PATCH 0565/1311] Removing handling of null lucene query where we
 catch this at parse time

---
 .../org/elasticsearch/index/IndexService.java     |  6 ++----
 .../index/query/BoolQueryBuilder.java             |  4 +---
 .../index/query/BoostingQueryBuilder.java         |  6 ------
 .../index/query/ConstantScoreQueryBuilder.java    |  4 ----
 .../index/query/DisMaxQueryBuilder.java           |  8 ++------
 .../index/query/HasChildQueryBuilder.java         |  3 ---
 .../index/query/HasParentQueryBuilder.java        |  3 ---
 .../index/query/NestedQueryBuilder.java           |  3 ---
 .../bucket/filter/FilterAggregationBuilder.java   |  5 ++---
 .../index/query/DisMaxQueryBuilderTests.java      | 15 ++++-----------
 10 files changed, 11 insertions(+), 46 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java
index e2d2ea4b8fe..5780dc256a5 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexService.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexService.java
@@ -50,7 +50,6 @@ import org.elasticsearch.index.engine.EngineFactory;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.IndexEventListener;
@@ -604,11 +603,10 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
             byte[] filterSource = alias.filter().uncompressed();
             try (XContentParser parser = XContentFactory.xContent(filterSource).createParser(filterSource)) {
                 Optional<QueryBuilder> innerQueryBuilder = shardContext.newParseContext(parser).parseInnerQueryBuilder();
-                ParsedQuery parsedFilter = null;
                 if (innerQueryBuilder.isPresent()) {
-                    parsedFilter = shardContext.toFilter(innerQueryBuilder.get());
+                    return shardContext.toFilter(innerQueryBuilder.get()).query();
                 }
-                return parsedFilter == null ? null : parsedFilter.query();
+                return null;
             }
         } catch (IOException ex) {
             throw new AliasFilterParsingException(shardContext.index(), alias.getAlias(), "Invalid alias filter", ex);
diff --git a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
index 863bdb25c71..052ddd85663 100644
--- a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
@@ -441,9 +441,7 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
                     luceneQuery = query.toFilter(context);
                     break;
             }
-            if (luceneQuery != null) {
-                booleanQueryBuilder.add(new BooleanClause(luceneQuery, occurs));
-            }
+            booleanQueryBuilder.add(new BooleanClause(luceneQuery, occurs));
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java
index 3afa4339b5c..3912bcff26e 100644
--- a/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java
@@ -208,12 +208,6 @@ public class BoostingQueryBuilder extends AbstractQueryBuilder<BoostingQueryBuil
     protected Query doToQuery(QueryShardContext context) throws IOException {
         Query positive = positiveQuery.toQuery(context);
         Query negative = negativeQuery.toQuery(context);
-        // make upstream queries ignore this query by returning `null`
-        // if either inner query builder returns null
-        if (positive == null || negative == null) {
-            return null;
-        }
-
         return new BoostingQuery(positive, negative, negativeBoost);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java
index 44b4458b572..574835c6b7a 100644
--- a/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java
@@ -146,10 +146,6 @@ public class ConstantScoreQueryBuilder extends AbstractQueryBuilder<ConstantScor
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
         Query innerFilter = filterBuilder.toFilter(context);
-        if (innerFilter == null ) {
-            // return null so that parent queries (e.g. bool) also ignore this
-            return null;
-        }
         return new ConstantScoreQuery(innerFilter);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java
index e480417e8fb..3ceca1a07a0 100644
--- a/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java
@@ -168,8 +168,8 @@ public class DisMaxQueryBuilder extends AbstractQueryBuilder<DisMaxQueryBuilder>
             }
         }
 
-        if (!queriesFound) {
-            throw new ParsingException(parser.getTokenLocation(), "[dis_max] requires 'queries' field");
+        if (!queriesFound || queries.isEmpty()) {
+            throw new ParsingException(parser.getTokenLocation(), "[dis_max] requires 'queries' field with at least one clause");
         }
 
         DisMaxQueryBuilder disMaxQuery = new DisMaxQueryBuilder();
@@ -186,10 +186,6 @@ public class DisMaxQueryBuilder extends AbstractQueryBuilder<DisMaxQueryBuilder>
     protected Query doToQuery(QueryShardContext context) throws IOException {
         // return null if there are no queries at all
         Collection<Query> luceneQueries = toQueries(queries, context);
-        if (luceneQueries.isEmpty()) {
-            return null;
-        }
-
         return new DisjunctionMaxQuery(luceneQueries, tieBreaker);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
index 410fda41aef..9fb44845f22 100644
--- a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
@@ -330,9 +330,6 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
             context.setTypes(previousTypes);
         }
 
-        if (innerQuery == null) {
-            return null;
-        }
         DocumentMapper childDocMapper = context.getMapperService().documentMapper(type);
         if (childDocMapper == null) {
             if (ignoreUnmapped) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java
index 7e788a16e87..3bda0782355 100644
--- a/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java
@@ -162,9 +162,6 @@ public class HasParentQueryBuilder extends AbstractQueryBuilder<HasParentQueryBu
             context.setTypes(previousTypes);
         }
 
-        if (innerQuery == null) {
-            return null;
-        }
         DocumentMapper parentDocMapper = context.getMapperService().documentMapper(type);
         if (parentDocMapper == null) {
             if (ignoreUnmapped) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
index 1d0e6d11f5f..f16913a397b 100644
--- a/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java
@@ -255,9 +255,6 @@ public class NestedQueryBuilder extends AbstractQueryBuilder<NestedQueryBuilder>
         try {
             context.nestedScope().nextLevel(nestedObjectMapper);
             innerQuery = this.query.toQuery(context);
-            if (innerQuery == null) {
-                return null;
-            }
         } finally {
             context.nestedScope().previousLevel();
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
index a973d69da4f..4026f846e6d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
@@ -20,10 +20,10 @@
 package org.elasticsearch.search.aggregations.bucket.filter;
 
 import org.elasticsearch.common.ParseField;
-import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
@@ -84,8 +84,7 @@ public class FilterAggregationBuilder extends AbstractAggregationBuilder<FilterA
     }
 
     public static FilterAggregationBuilder parse(String aggregationName, QueryParseContext context) throws IOException {
-        QueryBuilder filter = context.parseInnerQueryBuilder().orElseThrow(() -> new ParsingException(context.parser().getTokenLocation(),
-                "filter cannot be null in filter aggregation [{}]", aggregationName));
+        QueryBuilder filter = context.parseInnerQueryBuilder().orElse(new MatchAllQueryBuilder());
         return new FilterAggregationBuilder(aggregationName, filter);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
index 61c786d1b01..f18db46346c 100644
--- a/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
@@ -25,6 +25,7 @@ import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -88,25 +89,17 @@ public class DisMaxQueryBuilderTests extends AbstractQueryTestCase<DisMaxQueryBu
         return alternateVersions;
     }
 
-    /**
-     * test `null`return value for missing inner queries
-     */
-    public void testNoInnerQueries() throws IOException {
-        DisMaxQueryBuilder disMaxBuilder = new DisMaxQueryBuilder();
-        assertNull(disMaxBuilder.toQuery(createShardContext()));
-    }
-
     /**
      * Test with empty inner query body, this should be ignored upstream.
      * To test this, we use inner {@link ConstantScoreQueryBuilder} with empty inner filter.
      */
-    public void testInnerQueryEmptyIgnored() throws IOException {
+    public void testInnerQueryEmptyException() throws IOException {
         String queryString = "{ \"" + DisMaxQueryBuilder.NAME + "\" :"
                 + "             { \"queries\" : [ {\"" + ConstantScoreQueryBuilder.NAME + "\" : { \"filter\" : { } } } ] "
                 + "             }"
                 + "           }";
-        DisMaxQueryBuilder builder = (DisMaxQueryBuilder) parseQuery(queryString, ParseFieldMatcher.EMPTY);
-        assertTrue("the inner query has an empty body, so it should be ignored", builder.innerQueries().isEmpty());
+        ParsingException ex = expectThrows(ParsingException.class, () -> parseQuery(queryString, ParseFieldMatcher.EMPTY));
+        assertEquals("[dis_max] requires 'queries' field with at least one clause", ex.getMessage());
     }
 
     public void testIllegalArguments() {

From e2b6dbc020838dc36efd7921118f817754774faf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Tue, 31 May 2016 16:51:06 +0200
Subject: [PATCH 0566/1311] Add tests to check that toQuery() doesn't return
 null

---
 .../test/AbstractQueryTestCase.java           | 72 ++++++++++---------
 1 file changed, 37 insertions(+), 35 deletions(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
index 2e0bfe28911..cb73ebc0752 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
@@ -19,27 +19,33 @@
 
 package org.elasticsearch.test;
 
-import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.core.io.JsonStringEncoder;
-import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.common.inject.Module;
-import org.elasticsearch.common.settings.IndexScopedSettings;
-import org.elasticsearch.index.query.AbstractQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.index.query.QueryRewriteContext;
-import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.PluginsModule;
-import org.elasticsearch.plugins.PluginsService;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.either;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.not;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
 
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.spans.SpanBoostQuery;
 import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
@@ -62,6 +68,7 @@ import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.Injector;
+import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.inject.ModulesBuilder;
 import org.elasticsearch.common.inject.multibindings.Multibinder;
 import org.elasticsearch.common.inject.util.Providers;
@@ -69,6 +76,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.unit.Fuzziness;
@@ -87,6 +95,11 @@ import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.query.AbstractQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.support.QueryParsers;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.similarity.SimilarityService;
@@ -96,6 +109,10 @@ import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.mapper.MapperRegistry;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.node.internal.InternalSettingsPreparer;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.PluginsModule;
+import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.Script.ScriptParseException;
 import org.elasticsearch.script.ScriptContext;
@@ -115,26 +132,9 @@ import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 
-import java.io.Closeable;
-import java.io.IOException;
-import java.lang.reflect.InvocationHandler;
-import java.lang.reflect.Method;
-import java.lang.reflect.Proxy;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ExecutionException;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.either;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.not;
+import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.io.JsonStringEncoder;
 
 public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>> extends ESTestCase {
 
@@ -399,6 +399,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             QB controlQuery = copyQuery(firstQuery);
             setSearchContext(randomTypes, context); // only set search context for toQuery to be more realistic
             Query firstLuceneQuery = rewriteQuery(firstQuery, context).toQuery(context);
+            assertNotNull("toQuery should not return null", firstLuceneQuery);
             assertLuceneQuery(firstQuery, firstLuceneQuery, context);
             SearchContext.removeCurrent(); // remove after assertLuceneQuery since the assertLuceneQuery impl might access the context as well
             assertTrue(
@@ -1005,6 +1006,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             this.namedWriteableRegistry = injector.getInstance(NamedWriteableRegistry.class);
         }
 
+        @Override
         public void close() throws IOException {
             injector.getInstance(ClusterService.class).close();
             try {

From 22dfc41521c55aae3cf0e9dce1bb92f602be754c Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Thu, 2 Jun 2016 15:38:51 +0200
Subject: [PATCH 0567/1311] Only filter intial recovery (post API) when
 shrinking an index (#18661)

Today we use `index.routing.allocation.include._id` to filter the allocation
for the shrink target index. That has the sideeffect that the user has to
delete that setting / change it once the primary has been recovered (shrink is done)
This PR adds a dedicated filter that can only be set internally that only filters
allocation for unassigned shards.
---
 .../cluster/metadata/IndexMetaData.java       |  22 ++-
 .../metadata/MetaDataCreateIndexService.java  |   8 +-
 .../decider/FilterAllocationDecider.java      |  13 ++
 .../admin/indices/create/CreateIndexIT.java   |   3 +-
 .../MetaDataCreateIndexServiceTests.java      |   2 +-
 .../FilterAllocationDeciderTests.java         | 125 ++++++++++++++++++
 docs/reference/indices/shrink-index.asciidoc  |  19 ---
 7 files changed, 164 insertions(+), 28 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterAllocationDeciderTests.java

diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
index 3a1856c7481..a87c887a4a1 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
@@ -30,8 +30,10 @@ import org.elasticsearch.cluster.Diffable;
 import org.elasticsearch.cluster.DiffableUtils;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeFilters;
 import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseFieldMatcher;
@@ -215,6 +217,8 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         Setting.groupSetting("index.routing.allocation.include.", Property.Dynamic, Property.IndexScope);
     public static final Setting<Settings> INDEX_ROUTING_EXCLUDE_GROUP_SETTING =
         Setting.groupSetting("index.routing.allocation.exclude.", Property.Dynamic, Property.IndexScope);
+    public static final Setting<Settings> INDEX_ROUTING_INITIAL_RECOVERY_GROUP_SETTING =
+        Setting.groupSetting("index.routing.allocation.initial_recovery."); // this is only setable internally not a registered setting!!
 
     public static final IndexMetaData PROTO = IndexMetaData.builder("")
             .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))
@@ -254,6 +258,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
     private final DiscoveryNodeFilters requireFilters;
     private final DiscoveryNodeFilters includeFilters;
     private final DiscoveryNodeFilters excludeFilters;
+    private final DiscoveryNodeFilters initialRecoveryFilters;
 
     private final Version indexCreatedVersion;
     private final Version indexUpgradedVersion;
@@ -262,7 +267,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
     private IndexMetaData(Index index, long version, long[] primaryTerms, State state, int numberOfShards, int numberOfReplicas, Settings settings,
                           ImmutableOpenMap<String, MappingMetaData> mappings, ImmutableOpenMap<String, AliasMetaData> aliases,
                           ImmutableOpenMap<String, Custom> customs, ImmutableOpenIntMap<Set<String>> activeAllocationIds,
-                          DiscoveryNodeFilters requireFilters, DiscoveryNodeFilters includeFilters, DiscoveryNodeFilters excludeFilters,
+                          DiscoveryNodeFilters requireFilters, DiscoveryNodeFilters initialRecoveryFilters, DiscoveryNodeFilters includeFilters, DiscoveryNodeFilters excludeFilters,
                           Version indexCreatedVersion, Version indexUpgradedVersion, org.apache.lucene.util.Version minimumCompatibleLuceneVersion) {
 
         this.index = index;
@@ -281,6 +286,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         this.requireFilters = requireFilters;
         this.includeFilters = includeFilters;
         this.excludeFilters = excludeFilters;
+        this.initialRecoveryFilters = initialRecoveryFilters;
         this.indexCreatedVersion = indexCreatedVersion;
         this.indexUpgradedVersion = indexUpgradedVersion;
         this.minimumCompatibleLuceneVersion = minimumCompatibleLuceneVersion;
@@ -430,6 +436,11 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         return requireFilters;
     }
 
+    @Nullable
+    public DiscoveryNodeFilters getInitialRecoveryFilters() {
+        return initialRecoveryFilters;
+    }
+
     @Nullable
     public DiscoveryNodeFilters includeFilters() {
         return includeFilters;
@@ -892,6 +903,13 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
             } else {
                 excludeFilters = DiscoveryNodeFilters.buildFromKeyValue(OR, excludeMap);
             }
+            Map<String, String> initialRecoveryMap = INDEX_ROUTING_INITIAL_RECOVERY_GROUP_SETTING.get(settings).getAsMap();
+            final DiscoveryNodeFilters initialRecoveryFilters;
+            if (initialRecoveryMap.isEmpty()) {
+                initialRecoveryFilters = null;
+            } else {
+                initialRecoveryFilters = DiscoveryNodeFilters.buildFromKeyValue(OR, initialRecoveryMap);
+            }
             Version indexCreatedVersion = Version.indexCreated(settings);
             Version indexUpgradedVersion = settings.getAsVersion(IndexMetaData.SETTING_VERSION_UPGRADED, indexCreatedVersion);
             String stringLuceneVersion = settings.get(SETTING_VERSION_MINIMUM_COMPATIBLE);
@@ -915,7 +933,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
 
             final String uuid = settings.get(SETTING_INDEX_UUID, INDEX_UUID_NA_VALUE);
             return new IndexMetaData(new Index(index, uuid), version, primaryTerms, state, numberOfShards, numberOfReplicas, tmpSettings, mappings.build(),
-                tmpAliases.build(), customs.build(), filledActiveAllocationIds.build(), requireFilters, includeFilters, excludeFilters,
+                tmpAliases.build(), customs.build(), filledActiveAllocationIds.build(), requireFilters, initialRecoveryFilters, includeFilters, excludeFilters,
                 indexCreatedVersion, indexUpgradedVersion, minimumCompatibleLuceneVersion);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
index f5539f28e9a..47528428928 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
@@ -533,11 +533,11 @@ public class MetaDataCreateIndexService extends AbstractComponent {
         final Predicate<String> analysisSimilarityPredicate = (s) -> s.startsWith("index.similarity.")
             || s.startsWith("index.analysis.");
         indexSettingsBuilder
-            // we can only shrink to 1 index so far!
+            // we can only shrink to 1 shard so far!
             .put("index.number_of_shards", 1)
-            // we use "i.r.a.include" rather than "i.r.a.require" since it's allows one of the nodes holding an
-            // instanceof all shards.
-            .put("index.routing.allocation.include._id",
+            // we use "i.r.a.initial_recovery" rather than "i.r.a.require|include" since we want the replica to allocate right away
+            // once we are allocated.
+            .put("index.routing.allocation.initial_recovery._id",
                 Strings.arrayToCommaDelimitedString(nodesToAllocateOn.toArray()))
             // we only try once and then give up with a shrink index
             .put("index.allocation.max_retries", 1)
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java
index c862865cb16..63bd588114a 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java
@@ -88,6 +88,19 @@ public class FilterAllocationDecider extends AllocationDecider {
 
     @Override
     public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
+        if (shardRouting.unassigned()) {
+            // only for unassigned - we filter allocation right after the index creation ie. for shard shrinking etc. to ensure
+            // that once it has been allocated post API the replicas can be allocated elsewhere without user interaction
+            // this is a setting that can only be set within the system!
+            IndexMetaData indexMd = allocation.metaData().getIndexSafe(shardRouting.index());
+            DiscoveryNodeFilters initialRecoveryFilters = indexMd.getInitialRecoveryFilters();
+            if (shardRouting.allocatedPostIndexCreate(indexMd) == false &&
+                initialRecoveryFilters != null &&
+                initialRecoveryFilters.match(node.node()) == false) {
+                return allocation.decision(Decision.NO, NAME, "node does not match index initial recovery filters [%s]",
+                    indexMd.includeFilters());
+            }
+        }
         return shouldFilter(shardRouting, node, allocation);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
index af7d7b94ada..31c5749697b 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
@@ -311,10 +311,9 @@ public class CreateIndexIT extends ESIntegTestCase {
             .setSettings(Settings.builder().put("index.number_of_replicas", 0).build()).get());
         ensureGreen();
         assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
-        // let it be allocated anywhere and bump replicas
+        // bump replicas
         client().admin().indices().prepareUpdateSettings("target")
             .setSettings(Settings.builder()
-                .putNull("index.routing.allocation.include._id")
                 .put("index.number_of_replicas", 1)).get();
         ensureGreen();
         assertHitCount(client().prepareSearch("target").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
index 12292eb6598..aa226e311dd 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
@@ -159,7 +159,7 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
         assertEquals("similarity settings must be copied", "BM25", builder.build().get("index.similarity.default.type"));
         assertEquals("analysis settings must be copied",
             "keyword", builder.build().get("index.analysis.analyzer.my_analyzer.tokenizer"));
-        assertEquals("node1", builder.build().get("index.routing.allocation.include._id"));
+        assertEquals("node1", builder.build().get("index.routing.allocation.initial_recovery._id"));
         assertEquals("1", builder.build().get("index.allocation.max_retries"));
     }
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterAllocationDeciderTests.java
new file mode 100644
index 00000000000..44437237dc9
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterAllocationDeciderTests.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.routing.allocation;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.decider.Decision;
+import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESAllocationTestCase;
+import org.elasticsearch.test.gateway.NoopGatewayAllocator;
+
+import java.util.Collections;
+
+import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
+import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
+import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED;
+
+public class FilterAllocationDeciderTests extends ESAllocationTestCase {
+
+    public void testFilterInitialAllocation() {
+        FilterAllocationDecider filterAllocationDecider = new FilterAllocationDecider(Settings.EMPTY,
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
+        AllocationDeciders allocationDeciders = new AllocationDeciders(Settings.EMPTY,
+            Collections.singleton(filterAllocationDecider));
+        AllocationService service = new AllocationService(Settings.builder().build(), allocationDeciders,
+            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE);
+        ClusterState state = createInitialClusterState(service, Settings.builder().put("index.routing.allocation.initial_recovery._id",
+            "node2").build());
+        RoutingTable routingTable = state.routingTable();
+
+        // we can initally only allocate on node2
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).currentNodeId(), "node2");
+        routingTable = service.applyFailedShard(state, routingTable.index("idx").shard(0).shards().get(0)).routingTable();
+        state = ClusterState.builder(state).routingTable(routingTable).build();
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), UNASSIGNED);
+        assertNull(routingTable.index("idx").shard(0).shards().get(0).currentNodeId());
+
+        RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, state.getRoutingNodes(), state,
+            null, 0, false);
+        assertEquals(filterAllocationDecider.canAllocate(routingTable.index("idx").shard(0).shards().get(0),
+            state.getRoutingNodes().node("node2")
+            ,allocation), Decision.YES);
+        assertEquals(filterAllocationDecider.canAllocate(routingTable.index("idx").shard(0).shards().get(0),
+            state.getRoutingNodes().node("node1")
+            ,allocation), Decision.NO);
+
+        // after failing the shard we are unassigned since the node is blacklisted and we can't initialize on the other node
+        state = stateFromResult(state, service.reroute(state, "try allocate again"));
+        routingTable = state.routingTable();
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).currentNodeId(), "node2");
+
+        state = stateFromResult(state, service.applyStartedShards(state, routingTable.index("idx").shard(0).shards()));
+        routingTable = state.routingTable();
+
+        // ok now we are started and can be allocated anywhere!! lets see...
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), STARTED);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).currentNodeId(), "node2");
+        assertTrue(routingTable.index("idx").shard(0).shards().get(0).allocatedPostIndexCreate(state.getMetaData().index("idx")));
+
+        // we fail it again to check if we are initializing immediately on the other node
+        state = stateFromResult(state, service.applyFailedShard(state, routingTable.index("idx").shard(0).shards().get(0)));
+        routingTable = state.routingTable();
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING);
+        assertEquals(routingTable.index("idx").shard(0).shards().get(0).currentNodeId(), "node1");
+        assertTrue(routingTable.index("idx").shard(0).shards().get(0).allocatedPostIndexCreate(state.getMetaData().index("idx")));
+
+        allocation = new RoutingAllocation(allocationDeciders, state.getRoutingNodes(), state,
+            null, 0, false);
+        assertEquals(filterAllocationDecider.canAllocate(routingTable.index("idx").shard(0).shards().get(0),
+            state.getRoutingNodes().node("node2")
+            ,allocation), Decision.YES);
+        assertEquals(filterAllocationDecider.canAllocate(routingTable.index("idx").shard(0).shards().get(0),
+            state.getRoutingNodes().node("node1")
+            ,allocation), Decision.YES);
+    }
+
+    private ClusterState stateFromResult(ClusterState previousState, RoutingAllocation.Result result) {
+        return ClusterState.builder(previousState).routingTable(result.routingTable()).metaData(result.metaData()).build();
+    }
+
+    private ClusterState createInitialClusterState(AllocationService service, Settings settings) {
+        MetaData.Builder metaBuilder = MetaData.builder();
+        metaBuilder.put(IndexMetaData.builder("idx").settings(settings(Version.CURRENT).put(settings))
+            .numberOfShards(1).numberOfReplicas(0));
+        MetaData metaData = metaBuilder.build();
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        routingTableBuilder.addAsNew(metaData.index("idx"));
+
+        RoutingTable routingTable = routingTableBuilder.build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+            .metaData(metaData).routingTable(routingTable).build();
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")))
+            .build();
+        routingTable = service.reroute(clusterState, "reroute", false).routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        return clusterState;
+    }
+}
diff --git a/docs/reference/indices/shrink-index.asciidoc b/docs/reference/indices/shrink-index.asciidoc
index 28532db5240..db6f717a82b 100644
--- a/docs/reference/indices/shrink-index.asciidoc
+++ b/docs/reference/indices/shrink-index.asciidoc
@@ -36,34 +36,15 @@ The `_shrink` API is similar to <<indices-create-index>> and accepts `settings`
 $ curl -XPUT 'http://localhost:9200/logs/_shrink/logs_single_shard' -d '{
     "settings" : {
         "index.codec" : "best_compression", <1>
-        "index.number_of_replicas" : 0 <2>
     }
 }'
 --------------------------------------------------
 <1> Enables `best_compression` codec on the target index
-<2> Sets the number of replicas on the target index to `0` to ensure the cluster is green once the shard initialized
 
 The API call above returns immediately once the target index is created but doesn't wait
 for the shrink operation to start. Once the target indices primary shard moves to state `initializing`
 the shrink operation has started.
 
-Once the index is shrunk replicas can be set to `1` and allocation filtering can be removed.
-
-[source,js]
---------------------------------------------------
-$ curl -XPUT 'http://localhost:9200/logs_single_shard/_settings' -d '{
-    "settings" : {
-        "index.routing.allocation.include._id" : null, <1>
-        "index.number_of_replicas" : 1 <2>
-    }
-}'
---------------------------------------------------
-
-<1> Resets the allocation filtering for the new shrunk index to allow replica allocation
-<2> Bumps the number of replicas to 1
-
-
-
 [float]
 [[shrink-index-limitations]]
 === Limitations

From 9067407cdd10bb6035feaac56632aa5698feb064 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Wed, 1 Jun 2016 19:48:22 +0200
Subject: [PATCH 0568/1311] Adressing review comments

---
 .../query/ConstantScoreQueryBuilder.java      |  2 +-
 .../index/query/DisMaxQueryBuilder.java       |  7 ++-
 .../search/builder/SearchSourceBuilder.java   |  2 +-
 .../search/rescore/QueryRescorerBuilder.java  |  3 ++
 .../index/query/DisMaxQueryBuilderTests.java  | 30 ++++++-----
 .../search/aggregations/bucket/FilterIT.java  |  2 +-
 .../search/aggregations/bucket/FiltersIT.java |  4 +-
 .../rescore/QueryRescoreBuilderTests.java     |  6 +++
 .../test/AbstractQueryTestCase.java           | 50 +++++++++----------
 9 files changed, 59 insertions(+), 47 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java
index 574835c6b7a..8848a5ce558 100644
--- a/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java
@@ -91,7 +91,7 @@ public class ConstantScoreQueryBuilder extends AbstractQueryBuilder<ConstantScor
     public static Optional<ConstantScoreQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException {
         XContentParser parser = parseContext.parser();
 
-        Optional<QueryBuilder> query = Optional.empty();
+        Optional<QueryBuilder> query = null;
         boolean queryFound = false;
         String queryName = null;
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
diff --git a/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java
index 3ceca1a07a0..b2d8f8b8d15 100644
--- a/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryBuilder.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 
@@ -168,7 +169,7 @@ public class DisMaxQueryBuilder extends AbstractQueryBuilder<DisMaxQueryBuilder>
             }
         }
 
-        if (!queriesFound || queries.isEmpty()) {
+        if (!queriesFound) {
             throw new ParsingException(parser.getTokenLocation(), "[dis_max] requires 'queries' field with at least one clause");
         }
 
@@ -186,6 +187,10 @@ public class DisMaxQueryBuilder extends AbstractQueryBuilder<DisMaxQueryBuilder>
     protected Query doToQuery(QueryShardContext context) throws IOException {
         // return null if there are no queries at all
         Collection<Query> luceneQueries = toQueries(queries, context);
+        if (luceneQueries.isEmpty()) {
+            return Queries.newMatchNoDocsQuery("no clauses for dismax query.");
+        }
+
         return new DisjunctionMaxQuery(luceneQueries, tieBreaker);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index 4c47304330a..8c4be225127 100644
--- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -611,7 +611,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
     /**
      * Add an aggregation to perform as part of the search.
      */
-    public SearchSourceBuilder aggregation(PipelineAggregatorBuilder<?> aggregation) {
+    public SearchSourceBuilder aggregation(PipelineAggregatorBuilder aggregation) {
             if (aggregations == null) {
             aggregations = AggregatorFactories.builder();
             }
diff --git a/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java
index 6f59b446dc2..5603446914c 100644
--- a/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorerBuilder.java
@@ -72,6 +72,9 @@ public class QueryRescorerBuilder extends RescoreBuilder<QueryRescorerBuilder> {
      * @param builder the query builder to build the rescore query from
      */
     public QueryRescorerBuilder(QueryBuilder builder) {
+        if (builder == null) {
+            throw new IllegalArgumentException("rescore_query cannot be null");
+        }
         this.queryBuilder = builder;
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
index f18db46346c..ebd6446f80a 100644
--- a/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/DisMaxQueryBuilderTests.java
@@ -25,7 +25,7 @@ import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.test.AbstractQueryTestCase;
 
 import java.io.IOException;
@@ -35,7 +35,6 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-import static org.hamcrest.CoreMatchers.nullValue;
 import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -60,17 +59,13 @@ public class DisMaxQueryBuilderTests extends AbstractQueryTestCase<DisMaxQueryBu
     @Override
     protected void doAssertLuceneQuery(DisMaxQueryBuilder queryBuilder, Query query, QueryShardContext context) throws IOException {
         Collection<Query> queries = AbstractQueryBuilder.toQueries(queryBuilder.innerQueries(), context);
-        if (queries.isEmpty()) {
-            assertThat(query, nullValue());
-        } else {
-            assertThat(query, instanceOf(DisjunctionMaxQuery.class));
-            DisjunctionMaxQuery disjunctionMaxQuery = (DisjunctionMaxQuery) query;
-            assertThat(disjunctionMaxQuery.getTieBreakerMultiplier(), equalTo(queryBuilder.tieBreaker()));
-            assertThat(disjunctionMaxQuery.getDisjuncts().size(), equalTo(queries.size()));
-            Iterator<Query> queryIterator = queries.iterator();
-            for (int i = 0; i < disjunctionMaxQuery.getDisjuncts().size(); i++) {
-                assertThat(disjunctionMaxQuery.getDisjuncts().get(i), equalTo(queryIterator.next()));
-            }
+        assertThat(query, instanceOf(DisjunctionMaxQuery.class));
+        DisjunctionMaxQuery disjunctionMaxQuery = (DisjunctionMaxQuery) query;
+        assertThat(disjunctionMaxQuery.getTieBreakerMultiplier(), equalTo(queryBuilder.tieBreaker()));
+        assertThat(disjunctionMaxQuery.getDisjuncts().size(), equalTo(queries.size()));
+        Iterator<Query> queryIterator = queries.iterator();
+        for (int i = 0; i < disjunctionMaxQuery.getDisjuncts().size(); i++) {
+            assertThat(disjunctionMaxQuery.getDisjuncts().get(i), equalTo(queryIterator.next()));
         }
     }
 
@@ -90,7 +85,7 @@ public class DisMaxQueryBuilderTests extends AbstractQueryTestCase<DisMaxQueryBu
     }
 
     /**
-     * Test with empty inner query body, this should be ignored upstream.
+     * Test with empty inner query body, this should be converted to a {@link MatchNoDocsQuery}.
      * To test this, we use inner {@link ConstantScoreQueryBuilder} with empty inner filter.
      */
     public void testInnerQueryEmptyException() throws IOException {
@@ -98,8 +93,11 @@ public class DisMaxQueryBuilderTests extends AbstractQueryTestCase<DisMaxQueryBu
                 + "             { \"queries\" : [ {\"" + ConstantScoreQueryBuilder.NAME + "\" : { \"filter\" : { } } } ] "
                 + "             }"
                 + "           }";
-        ParsingException ex = expectThrows(ParsingException.class, () -> parseQuery(queryString, ParseFieldMatcher.EMPTY));
-        assertEquals("[dis_max] requires 'queries' field with at least one clause", ex.getMessage());
+        QueryBuilder queryBuilder = parseQuery(queryString, ParseFieldMatcher.EMPTY);
+        QueryShardContext context = createShardContext();
+        Query luceneQuery = queryBuilder.toQuery(context);
+        assertThat(luceneQuery, instanceOf(MatchNoDocsQuery.class));
+        assertThat(((MatchNoDocsQuery) luceneQuery).toString(), equalTo("MatchNoDocsQuery[\"no clauses for dismax query.\"]"));
     }
 
     public void testIllegalArguments() {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java
index ddb62d57ffe..a48facc4d66 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java
@@ -132,7 +132,7 @@ public class FilterIT extends ESIntegTestCase {
         String emtpyFilterBody = "{ }";
         XContentParser parser = XContentFactory.xContent(emtpyFilterBody).createParser(emtpyFilterBody);
         QueryParseContext parseContext = new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.EMPTY);
-        AggregationBuilder<?> filterAgg = FilterAggregationBuilder.parse("tag1", parseContext);
+        AggregationBuilder filterAgg = FilterAggregationBuilder.parse("tag1", parseContext);
 
         SearchResponse response = client().prepareSearch("idx").addAggregation(filterAgg).execute().actionGet();
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java
index 0be0f76de90..a95df3ff5e6 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java
@@ -217,7 +217,7 @@ public class FiltersIT extends ESIntegTestCase {
         XContentParser parser = XContentFactory.xContent(emtpyFilterBody).createParser(emtpyFilterBody);
         parser.nextToken();
         QueryParseContext parseContext = new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.EMPTY);
-        AggregationBuilder<?> filtersAgg = FiltersAggregationBuilder.parse("tag1", parseContext);
+        AggregationBuilder filtersAgg = FiltersAggregationBuilder.parse("tag1", parseContext);
 
         SearchResponse response = client().prepareSearch("idx").addAggregation(filtersAgg).execute().actionGet();
 
@@ -234,7 +234,7 @@ public class FiltersIT extends ESIntegTestCase {
         XContentParser parser = XContentFactory.xContent(emtpyFilterBody).createParser(emtpyFilterBody);
         parser.nextToken();
         QueryParseContext parseContext = new QueryParseContext(new IndicesQueriesRegistry(), parser, ParseFieldMatcher.EMPTY);
-        AggregationBuilder<?> filtersAgg = FiltersAggregationBuilder.parse("tag1", parseContext);
+        AggregationBuilder filtersAgg = FiltersAggregationBuilder.parse("tag1", parseContext);
 
         SearchResponse response = client().prepareSearch("idx").addAggregation(filtersAgg)
                 .execute().actionGet();
diff --git a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
index 5cc59046433..f965d3ac5fd 100644
--- a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
@@ -54,6 +54,7 @@ import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
+
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 
@@ -180,6 +181,11 @@ public class QueryRescoreBuilderTests extends ESTestCase {
         }
     }
 
+    public void testRescoreQueryNull() throws IOException {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new QueryRescorerBuilder((QueryBuilder) null));
+        assertEquals("rescore_query cannot be null", e.getMessage());
+    }
+
     /**
      * test parsing exceptions for incorrect rescorer syntax
      */
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
index cb73ebc0752..b7a0d5a16f6 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
@@ -19,26 +19,9 @@
 
 package org.elasticsearch.test;
 
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.either;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.not;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.lang.reflect.InvocationHandler;
-import java.lang.reflect.Method;
-import java.lang.reflect.Proxy;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ExecutionException;
+import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.io.JsonStringEncoder;
 
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.Query;
@@ -132,9 +115,26 @@ import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
 
-import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.core.io.JsonStringEncoder;
+import java.io.Closeable;
+import java.io.IOException;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.either;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.not;
 
 public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>> extends ESTestCase {
 
@@ -418,13 +418,13 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             }
             setSearchContext(randomTypes, context);
             Query secondLuceneQuery = rewriteQuery(secondQuery, context).toQuery(context);
+            assertNotNull("toQuery should not return null", secondLuceneQuery);
             assertLuceneQuery(secondQuery, secondLuceneQuery, context);
             SearchContext.removeCurrent();
 
             assertEquals("two equivalent query builders lead to different lucene queries", rewrite(secondLuceneQuery), rewrite(firstLuceneQuery));
 
-            // if the initial lucene query is null, changing its boost won't have any effect, we shouldn't test that
-            if (firstLuceneQuery != null && supportsBoostAndQueryName()) {
+            if (supportsBoostAndQueryName()) {
                 secondQuery.boost(firstQuery.boost() + 1f + randomFloat());
                 setSearchContext(randomTypes, context);
                 Query thirdLuceneQuery = rewriteQuery(secondQuery, context).toQuery(context);

From f67e5807ca8bd8f06cd6bf1f913001f2da8c65a5 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 2 Jun 2016 13:12:21 -0400
Subject: [PATCH 0569/1311] Do not start scheduled pings until transport start

Today, scheduled pings in NettyTransport can start before the transport
is started. Instead, these pings should not be scheduled until after the
transport is started. This commit modifies NettyTransport so that this
is the case.

Relates #18702
---
 .../org/elasticsearch/transport/netty/NettyTransport.java   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
index c5d8f29e39a..a5542f9b403 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
@@ -287,9 +287,6 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
 
         this.scheduledPing = new ScheduledPing();
         this.pingSchedule = PING_SCHEDULE.get(settings);
-        if (pingSchedule.millis() > 0) {
-            threadPool.schedule(pingSchedule, ThreadPool.Names.GENERIC, scheduledPing);
-        }
         this.namedWriteableRegistry = namedWriteableRegistry;
         this.circuitBreakerService = circuitBreakerService;
     }
@@ -366,6 +363,9 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
                     bindServerBootstrap(name, mergedSettings);
                 }
             }
+            if (pingSchedule.millis() > 0) {
+                threadPool.schedule(pingSchedule, ThreadPool.Names.GENERIC, scheduledPing);
+            }
             success = true;
         } finally {
             if (success == false) {

From f82ab787a554460c742826d2cb1e35465d8649e9 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 1 Jun 2016 13:02:54 -0400
Subject: [PATCH 0570/1311] Add test rethrottle test case for delete-by-query

and remove some type parameters that we don't need that were getting
in the way.
---
 .../index/reindex/AbstractBulkByScrollRequestBuilder.java | 8 +++-----
 .../reindex/AbstractBulkIndexByScrollRequestBuilder.java  | 2 +-
 .../index/reindex/DeleteByQueryRequestBuilder.java        | 2 +-
 .../elasticsearch/index/reindex/RethrottleRequest.java    | 1 +
 .../java/org/elasticsearch/index/reindex/CancelTests.java | 5 +----
 .../org/elasticsearch/index/reindex/RethrottleTests.java  | 6 +++++-
 .../java/org/elasticsearch/index/reindex/RetryTests.java  | 4 ++--
 7 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
index bd1a63cb4e9..3ce01201499 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
@@ -21,7 +21,6 @@ package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequestBuilder;
-import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.WriteConsistencyLevel;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -30,13 +29,12 @@ import org.elasticsearch.index.query.QueryBuilder;
 
 public abstract class AbstractBulkByScrollRequestBuilder<
                 Request extends AbstractBulkByScrollRequest<Request>,
-                Response extends ActionResponse,
-                Self extends AbstractBulkByScrollRequestBuilder<Request, Response, Self>>
-        extends ActionRequestBuilder<Request, Response, Self> {
+                Self extends AbstractBulkByScrollRequestBuilder<Request, Self>>
+        extends ActionRequestBuilder<Request, BulkIndexByScrollResponse, Self> {
     private final SearchRequestBuilder source;
 
     protected AbstractBulkByScrollRequestBuilder(ElasticsearchClient client,
-            Action<Request, Response, Self> action, SearchRequestBuilder source, Request request) {
+            Action<Request, BulkIndexByScrollResponse, Self> action, SearchRequestBuilder source, Request request) {
         super(client, action, request);
         this.source = source;
     }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java
index 3fbeea078b7..e1b32baa4af 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequestBuilder.java
@@ -27,7 +27,7 @@ import org.elasticsearch.script.Script;
 public abstract class AbstractBulkIndexByScrollRequestBuilder<
                 Request extends AbstractBulkIndexByScrollRequest<Request>,
                 Self extends AbstractBulkIndexByScrollRequestBuilder<Request, Self>>
-        extends AbstractBulkByScrollRequestBuilder<Request, BulkIndexByScrollResponse, Self> {
+        extends AbstractBulkByScrollRequestBuilder<Request, Self> {
 
     protected AbstractBulkIndexByScrollRequestBuilder(ElasticsearchClient client,
             Action<Request, BulkIndexByScrollResponse, Self> action, SearchRequestBuilder search, Request request) {
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java
index f4d8a91f4cb..eb528793236 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/DeleteByQueryRequestBuilder.java
@@ -25,7 +25,7 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 
 public class DeleteByQueryRequestBuilder extends
-        AbstractBulkByScrollRequestBuilder<DeleteByQueryRequest, BulkIndexByScrollResponse, DeleteByQueryRequestBuilder> {
+        AbstractBulkByScrollRequestBuilder<DeleteByQueryRequest, DeleteByQueryRequestBuilder> {
 
     public DeleteByQueryRequestBuilder(ElasticsearchClient client,
                                        Action<DeleteByQueryRequest, BulkIndexByScrollResponse, DeleteByQueryRequestBuilder> action) {
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RethrottleRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RethrottleRequest.java
index 93f36d7034b..968d998974b 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RethrottleRequest.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RethrottleRequest.java
@@ -70,6 +70,7 @@ public class RethrottleRequest extends BaseTasksRequest<RethrottleRequest> {
             switch (action) {
             case ReindexAction.NAME:
             case UpdateByQueryAction.NAME:
+            case DeleteByQueryAction.NAME:
                 continue;
             default:
                 validationException = addValidationError(
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
index e2cc5b43f0a..1f9cd6b1394 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
@@ -80,10 +80,7 @@ public class CancelTests extends ReindexTestCase {
     /**
      * Executes the cancellation test
      */
-    private void testCancel(String action,
-                            AbstractBulkByScrollRequestBuilder<?, ? extends BulkIndexByScrollResponse, ?> builder,
-                            CancelAssertion assertion) throws Exception {
-
+    private void testCancel(String action, AbstractBulkByScrollRequestBuilder<?, ?> builder, CancelAssertion assertion) throws Exception {
         createIndex(INDEX);
 
         // Total number of documents created for this test (~10 per primary shard)
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
index 98575a457cf..acbc31bff99 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
@@ -43,7 +43,11 @@ public class RethrottleTests extends ReindexTestCase {
         testCase(updateByQuery().source("test"), UpdateByQueryAction.NAME);
     }
 
-    private void testCase(AbstractBulkIndexByScrollRequestBuilder<?, ?> request, String actionName)
+    public void testDeleteByQuery() throws Exception {
+        testCase(deleteByQuery().source("test"), DeleteByQueryAction.NAME);
+    }
+
+    private void testCase(AbstractBulkByScrollRequestBuilder<?, ?> request, String actionName)
             throws Exception {
         // Use a single shard so the reindex has to happen in multiple batches
         client().admin().indices().prepareCreate("test").setSettings("index.number_of_shards", 1).get();
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
index 478f7707dfd..699ba483e30 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
@@ -107,8 +107,8 @@ public class RetryTests extends ESSingleNodeTestCase {
                 matcher().deleted(DOC_COUNT));
     }
 
-    private void testCase(String action, AbstractBulkByScrollRequestBuilder<?, BulkIndexByScrollResponse, ?> request,
-            BulkIndexByScrollResponseMatcher matcher) throws Exception {
+    private void testCase(String action, AbstractBulkByScrollRequestBuilder<?, ?> request, BulkIndexByScrollResponseMatcher matcher)
+            throws Exception {
         logger.info("Blocking search");
         CyclicBarrier initialSearchBlock = blockExecutor(ThreadPool.Names.SEARCH);
 

From b720216395b39b350e027882efb051a22ecf5b16 Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Thu, 2 Jun 2016 17:01:14 -0400
Subject: [PATCH 0571/1311] Adds UUIDs to snapshots

This commit adds a UUID for each snapshot, in addition to the already
existing repository and snapshot name. The addition of UUIDs will enable
more robust handling of the deletion of previous snapshots and lingering
files from partially failed delete operations, on top of being able to
uniquely track each snapshot.

Closes #18228
Relates #18156
---
 .../create/TransportCreateSnapshotAction.java |  18 +-
 .../delete/TransportDeleteSnapshotAction.java |   4 +-
 .../snapshots/get/GetSnapshotsResponse.java   |   2 +-
 .../get/TransportGetSnapshotsAction.java      |  56 ++-
 .../TransportRestoreSnapshotAction.java       |  15 +-
 .../snapshots/status/SnapshotStatus.java      |  45 +-
 .../status/TransportNodesSnapshotsStatus.java |  38 +-
 .../TransportSnapshotsStatusAction.java       |  99 +++--
 .../cluster/RestoreInProgress.java            | 103 ++---
 .../cluster/SnapshotsInProgress.java          |  99 +++--
 .../cluster/metadata/SnapshotId.java          | 127 ------
 .../cluster/routing/IndexRoutingTable.java    |  10 +-
 .../cluster/routing/RestoreSource.java        |  53 ++-
 .../elasticsearch/index/shard/IndexShard.java |   3 +-
 .../index/shard/StoreRecovery.java            |   4 +-
 .../index/snapshots/IndexShardRepository.java |   2 +-
 .../BlobStoreIndexShardRepository.java        |  14 +-
 .../cluster/IndicesClusterStateService.java   |   4 +-
 .../repositories/Repository.java              |  12 +-
 .../blobstore/BlobStoreRepository.java        | 185 ++++++---
 .../repositories/uri/URLRepository.java       |   3 +-
 .../rest/action/cat/RestRecoveryAction.java   |   4 +-
 .../rest/action/cat/RestSnapshotAction.java   |   4 +-
 .../ConcurrentSnapshotExecutionException.java |   8 +-
 .../InvalidSnapshotNameException.java         |   5 +-
 .../snapshots/RestoreService.java             | 220 +++++-----
 .../org/elasticsearch/snapshots/Snapshot.java | 102 +++++
 .../snapshots/SnapshotCreationException.java  |  17 +-
 .../snapshots/SnapshotException.java          |  63 ++-
 .../elasticsearch/snapshots/SnapshotId.java   | 118 ++++++
 .../elasticsearch/snapshots/SnapshotInfo.java |  79 ++--
 .../snapshots/SnapshotMissingException.java   |  13 +-
 .../snapshots/SnapshotRestoreException.java   |  13 +-
 .../snapshots/SnapshotShardsService.java      |  90 ++--
 .../snapshots/SnapshotsService.java           | 387 ++++++++++--------
 .../ExceptionSerializationTests.java          |  16 +-
 .../cluster/snapshots/SnapshotBlocksIT.java   |   2 +-
 .../bwcompat/RepositoryUpgradabilityIT.java   | 195 +++++++++
 .../bwcompat/RestoreBackwardsCompatIT.java    |   1 -
 .../cluster/ClusterStateDiffIT.java           |   7 +-
 .../cluster/routing/ShardRoutingTests.java    |  13 +-
 .../cluster/routing/UnassignedInfoTests.java  |   8 +-
 .../NodeVersionAllocationDeciderTests.java    |   7 +-
 .../gateway/PrimaryShardAllocatorTests.java   |   8 +-
 .../index/shard/IndexShardTests.java          |   7 +-
 .../blobstore/BlobStoreRepositoryTests.java   | 230 +++++++++++
 .../action/cat/RestRecoveryActionTests.java   |  15 +-
 .../AbstractSnapshotIntegTestCase.java        |  33 +-
 .../DedicatedClusterSnapshotRestoreIT.java    |   8 +-
 .../SharedClusterSnapshotRestoreIT.java       | 271 +++++++++---
 .../SnapshotBackwardsCompatibilityIT.java     |   2 +-
 .../snapshots/SnapshotTests.java              |  57 +++
 .../snapshots/mockstore/MockRepository.java   |   6 +-
 .../repositories/azure/AzureRepository.java   |   4 +-
 .../storage/AzureStorageServiceMock.java      |  11 +-
 .../common/io/FileTestUtils.java              |  44 ++
 .../test/ESSingleNodeTestCase.java            |   6 +-
 57 files changed, 2019 insertions(+), 951 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/cluster/metadata/SnapshotId.java
 create mode 100644 core/src/main/java/org/elasticsearch/snapshots/Snapshot.java
 create mode 100644 core/src/main/java/org/elasticsearch/snapshots/SnapshotId.java
 create mode 100644 core/src/test/java/org/elasticsearch/bwcompat/RepositoryUpgradabilityIT.java
 create mode 100644 core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
 create mode 100644 core/src/test/java/org/elasticsearch/snapshots/SnapshotTests.java

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java
index 2654ac0c269..53b1bf86f66 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java
@@ -26,10 +26,10 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -72,7 +72,7 @@ public class TransportCreateSnapshotAction extends TransportMasterNodeAction<Cre
     @Override
     protected void masterOperation(final CreateSnapshotRequest request, ClusterState state, final ActionListener<CreateSnapshotResponse> listener) {
         SnapshotsService.SnapshotRequest snapshotRequest =
-                new SnapshotsService.SnapshotRequest("create_snapshot [" + request.snapshot() + "]", request.snapshot(), request.repository())
+                new SnapshotsService.SnapshotRequest(request.repository(), request.snapshot(), "create_snapshot [" + request.snapshot() + "]")
                         .indices(request.indices())
                         .indicesOptions(request.indicesOptions())
                         .partial(request.partial())
@@ -84,19 +84,19 @@ public class TransportCreateSnapshotAction extends TransportMasterNodeAction<Cre
             public void onResponse() {
                 if (request.waitForCompletion()) {
                     snapshotsService.addListener(new SnapshotsService.SnapshotCompletionListener() {
-                        SnapshotId snapshotId = new SnapshotId(request.repository(), request.snapshot());
-
                         @Override
-                        public void onSnapshotCompletion(SnapshotId snapshotId, SnapshotInfo snapshot) {
-                            if (this.snapshotId.equals(snapshotId)) {
-                                listener.onResponse(new CreateSnapshotResponse(snapshot));
+                        public void onSnapshotCompletion(Snapshot snapshot, SnapshotInfo snapshotInfo) {
+                            if (snapshot.getRepository().equals(request.repository()) &&
+                                    snapshot.getSnapshotId().getName().equals(request.snapshot())) {
+                                listener.onResponse(new CreateSnapshotResponse(snapshotInfo));
                                 snapshotsService.removeListener(this);
                             }
                         }
 
                         @Override
-                        public void onSnapshotFailure(SnapshotId snapshotId, Throwable t) {
-                            if (this.snapshotId.equals(snapshotId)) {
+                        public void onSnapshotFailure(Snapshot snapshot, Throwable t) {
+                            if (snapshot.getRepository().equals(request.repository()) &&
+                                    snapshot.getSnapshotId().getName().equals(request.snapshot())) {
                                 listener.onFailure(t);
                                 snapshotsService.removeListener(this);
                             }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
index 423e38cd258..d2857dc9e1a 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
@@ -26,7 +26,6 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -66,8 +65,7 @@ public class TransportDeleteSnapshotAction extends TransportMasterNodeAction<Del
 
     @Override
     protected void masterOperation(final DeleteSnapshotRequest request, ClusterState state, final ActionListener<DeleteSnapshotResponse> listener) {
-        SnapshotId snapshotIds = new SnapshotId(request.repository(), request.snapshot());
-        snapshotsService.deleteSnapshot(snapshotIds, new SnapshotsService.DeleteSnapshotListener() {
+        snapshotsService.deleteSnapshot(request.repository(), request.snapshot(), new SnapshotsService.DeleteSnapshotListener() {
             @Override
             public void onResponse() {
                 listener.onResponse(new DeleteSnapshotResponse(true));
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
index ec996e6d366..924f5a90d42 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java
@@ -42,7 +42,7 @@ public class GetSnapshotsResponse extends ActionResponse implements ToXContent {
     }
 
     GetSnapshotsResponse(List<SnapshotInfo> snapshots) {
-        this.snapshots = snapshots;
+        this.snapshots = Collections.unmodifiableList(snapshots);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
index 833b1a62289..f734201e7bd 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
@@ -26,20 +26,22 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
+import org.elasticsearch.snapshots.SnapshotMissingException;
 import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.util.ArrayList;
-import java.util.Collections;
+import java.util.HashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 /**
@@ -52,7 +54,8 @@ public class TransportGetSnapshotsAction extends TransportMasterNodeAction<GetSn
     public TransportGetSnapshotsAction(Settings settings, TransportService transportService, ClusterService clusterService,
                                        ThreadPool threadPool, SnapshotsService snapshotsService, ActionFilters actionFilters,
                                        IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, GetSnapshotsAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, GetSnapshotsRequest::new);
+        super(settings, GetSnapshotsAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver,
+              GetSnapshotsRequest::new);
         this.snapshotsService = snapshotsService;
     }
 
@@ -72,36 +75,51 @@ public class TransportGetSnapshotsAction extends TransportMasterNodeAction<GetSn
     }
 
     @Override
-    protected void masterOperation(final GetSnapshotsRequest request, ClusterState state, final ActionListener<GetSnapshotsResponse> listener) {
+    protected void masterOperation(final GetSnapshotsRequest request, ClusterState state,
+                                   final ActionListener<GetSnapshotsResponse> listener) {
         try {
+            final String repository = request.repository();
             List<SnapshotInfo> snapshotInfoBuilder = new ArrayList<>();
             if (isAllSnapshots(request.snapshots())) {
-                snapshotInfoBuilder.addAll(snapshotsService.snapshots(request.repository(), request.ignoreUnavailable()));
+                snapshotInfoBuilder.addAll(snapshotsService.currentSnapshots(repository));
+                snapshotInfoBuilder.addAll(snapshotsService.snapshots(repository,
+                                                                      snapshotsService.snapshotIds(repository),
+                                                                      request.ignoreUnavailable()));
             } else if (isCurrentSnapshots(request.snapshots())) {
-                snapshotInfoBuilder.addAll(snapshotsService.currentSnapshots(request.repository()));
+                snapshotInfoBuilder.addAll(snapshotsService.currentSnapshots(repository));
             } else {
-                Set<String> snapshotsToGet = new LinkedHashSet<>(); // to keep insertion order
-                List<SnapshotInfo> snapshots = null;
+                final Map<String, SnapshotId> allSnapshotIds = new HashMap<>();
+                for (SnapshotInfo snapshotInfo : snapshotsService.currentSnapshots(repository)) {
+                    SnapshotId snapshotId = snapshotInfo.snapshotId();
+                    allSnapshotIds.put(snapshotId.getName(), snapshotId);
+                }
+                for (SnapshotId snapshotId : snapshotsService.snapshotIds(repository)) {
+                    allSnapshotIds.put(snapshotId.getName(), snapshotId);
+                }
+                final Set<SnapshotId> toResolve = new LinkedHashSet<>(); // maintain order
                 for (String snapshotOrPattern : request.snapshots()) {
                     if (Regex.isSimpleMatchPattern(snapshotOrPattern) == false) {
-                        snapshotsToGet.add(snapshotOrPattern);
-                    } else {
-                        if (snapshots == null) { // lazily load snapshots
-                            snapshots = snapshotsService.snapshots(request.repository(), request.ignoreUnavailable());
+                        if (allSnapshotIds.containsKey(snapshotOrPattern)) {
+                            toResolve.add(allSnapshotIds.get(snapshotOrPattern));
+                        } else if (request.ignoreUnavailable() == false) {
+                            throw new SnapshotMissingException(repository, snapshotOrPattern);
                         }
-                        for (SnapshotInfo snapshot : snapshots) {
-                            if (Regex.simpleMatch(snapshotOrPattern, snapshot.name())) {
-                                snapshotsToGet.add(snapshot.name());
+                    } else {
+                        for (Map.Entry<String, SnapshotId> entry : allSnapshotIds.entrySet()) {
+                            if (Regex.simpleMatch(snapshotOrPattern, entry.getKey())) {
+                                toResolve.add(entry.getValue());
                             }
                         }
                     }
                 }
-                for (String snapshot : snapshotsToGet) {
-                    SnapshotId snapshotId = new SnapshotId(request.repository(), snapshot);
-                    snapshotInfoBuilder.add(snapshotsService.snapshot(snapshotId));
+
+                if (toResolve.isEmpty() && request.ignoreUnavailable() == false) {
+                    throw new SnapshotMissingException(repository, request.snapshots()[0]);
                 }
+
+                snapshotInfoBuilder.addAll(snapshotsService.snapshots(repository, new ArrayList<>(toResolve), request.ignoreUnavailable()));
             }
-            listener.onResponse(new GetSnapshotsResponse(Collections.unmodifiableList(snapshotInfoBuilder)));
+            listener.onResponse(new GetSnapshotsResponse(snapshotInfoBuilder));
         } catch (Throwable t) {
             listener.onFailure(t);
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java
index cbbc195370c..a38fbce46c2 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java
@@ -26,12 +26,12 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.snapshots.RestoreInfo;
 import org.elasticsearch.snapshots.RestoreService;
+import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -72,23 +72,22 @@ public class TransportRestoreSnapshotAction extends TransportMasterNodeAction<Re
     }
 
     @Override
-    protected void masterOperation(final RestoreSnapshotRequest request, ClusterState state, final ActionListener<RestoreSnapshotResponse> listener) {
-        RestoreService.RestoreRequest restoreRequest = new RestoreService.RestoreRequest(
-                "restore_snapshot[" + request.snapshot() + "]", request.repository(), request.snapshot(),
+    protected void masterOperation(final RestoreSnapshotRequest request, final ClusterState state, final ActionListener<RestoreSnapshotResponse> listener) {
+        RestoreService.RestoreRequest restoreRequest = new RestoreService.RestoreRequest(request.repository(), request.snapshot(),
                 request.indices(), request.indicesOptions(), request.renamePattern(), request.renameReplacement(),
                 request.settings(), request.masterNodeTimeout(), request.includeGlobalState(), request.partial(), request.includeAliases(),
-                request.indexSettings(), request.ignoreIndexSettings());
+                request.indexSettings(), request.ignoreIndexSettings(), "restore_snapshot[" + request.snapshot() + "]");
 
         restoreService.restoreSnapshot(restoreRequest, new ActionListener<RestoreInfo>() {
             @Override
             public void onResponse(RestoreInfo restoreInfo) {
                 if (restoreInfo == null && request.waitForCompletion()) {
                     restoreService.addListener(new ActionListener<RestoreService.RestoreCompletionResponse>() {
-                        SnapshotId snapshotId = new SnapshotId(request.repository(), request.snapshot());
-
                         @Override
                         public void onResponse(RestoreService.RestoreCompletionResponse restoreCompletionResponse) {
-                            if (this.snapshotId.equals(restoreCompletionResponse.getSnapshotId())) {
+                            final Snapshot snapshot = restoreCompletionResponse.getSnapshot();
+                            if (snapshot.getRepository().equals(request.repository()) &&
+                                    snapshot.getSnapshotId().getName().equals(request.snapshot())) {
                                 listener.onResponse(new RestoreSnapshotResponse(restoreCompletionResponse.getRestoreInfo()));
                                 restoreService.removeListener(this);
                             }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java
index 54f87f3c5fc..1a5ef9ab933 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotStatus.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.action.admin.cluster.snapshots.status;
 
 import org.elasticsearch.cluster.SnapshotsInProgress.State;
-import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
@@ -35,6 +35,7 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
 import static java.util.Collections.unmodifiableMap;
@@ -44,7 +45,7 @@ import static java.util.Collections.unmodifiableMap;
  */
 public class SnapshotStatus implements ToXContent, Streamable {
 
-    private SnapshotId snapshotId;
+    private Snapshot snapshot;
 
     private State state;
 
@@ -56,11 +57,10 @@ public class SnapshotStatus implements ToXContent, Streamable {
 
     private SnapshotStats stats;
 
-
-    SnapshotStatus(SnapshotId snapshotId, State state, List<SnapshotIndexShardStatus> shards) {
-        this.snapshotId = snapshotId;
-        this.state = state;
-        this.shards = shards;
+    SnapshotStatus(final Snapshot snapshot, final State state, final List<SnapshotIndexShardStatus> shards) {
+        this.snapshot = Objects.requireNonNull(snapshot);
+        this.state = Objects.requireNonNull(state);
+        this.shards = Objects.requireNonNull(shards);
         shardsStats = new SnapshotShardsStats(shards);
         updateShardStats();
     }
@@ -69,10 +69,10 @@ public class SnapshotStatus implements ToXContent, Streamable {
     }
 
     /**
-     * Returns snapshot id
+     * Returns snapshot
      */
-    public SnapshotId getSnapshotId() {
-        return snapshotId;
+    public Snapshot getSnapshot() {
+        return snapshot;
     }
 
     /**
@@ -124,7 +124,7 @@ public class SnapshotStatus implements ToXContent, Streamable {
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
-        snapshotId = SnapshotId.readSnapshotId(in);
+        snapshot = new Snapshot(in);
         state = State.fromValue(in.readByte());
         int size = in.readVInt();
         List<SnapshotIndexShardStatus> builder = new ArrayList<>();
@@ -137,7 +137,7 @@ public class SnapshotStatus implements ToXContent, Streamable {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        snapshotId.writeTo(out);
+        snapshot.writeTo(out);
         out.writeByte(state.value());
         out.writeVInt(shards.size());
         for (SnapshotIndexShardStatus shard : shards) {
@@ -170,7 +170,6 @@ public class SnapshotStatus implements ToXContent, Streamable {
         }
     }
 
-
     /**
      * Returns number of files in the snapshot
      */
@@ -178,22 +177,22 @@ public class SnapshotStatus implements ToXContent, Streamable {
         return stats;
     }
 
-    static final class Fields {
-        static final String SNAPSHOT = "snapshot";
-        static final String REPOSITORY = "repository";
-        static final String STATE = "state";
-        static final String INDICES = "indices";
-    }
+    private static final String SNAPSHOT = "snapshot";
+    private static final String REPOSITORY = "repository";
+    private static final String UUID = "uuid";
+    private static final String STATE = "state";
+    private static final String INDICES = "indices";
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field(Fields.SNAPSHOT, snapshotId.getSnapshot());
-        builder.field(Fields.REPOSITORY, snapshotId.getRepository());
-        builder.field(Fields.STATE, state.name());
+        builder.field(SNAPSHOT, snapshot.getSnapshotId().getName());
+        builder.field(REPOSITORY, snapshot.getRepository());
+        builder.field(UUID, snapshot.getSnapshotId().getUUID());
+        builder.field(STATE, state.name());
         shardsStats.toXContent(builder, params);
         stats.toXContent(builder, params);
-        builder.startObject(Fields.INDICES);
+        builder.startObject(INDICES);
         for (SnapshotIndexStatus indexStatus : getIndices().values()) {
             indexStatus.toXContent(builder, params);
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java
index bc139389460..8e7361c1928 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java
@@ -29,7 +29,7 @@ import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
@@ -94,11 +94,11 @@ public class TransportNodesSnapshotsStatus extends TransportNodesAction<Transpor
 
     @Override
     protected NodeSnapshotStatus nodeOperation(NodeRequest request) {
-        Map<SnapshotId, Map<ShardId, SnapshotIndexShardStatus>> snapshotMapBuilder = new HashMap<>();
+        Map<Snapshot, Map<ShardId, SnapshotIndexShardStatus>> snapshotMapBuilder = new HashMap<>();
         try {
             String nodeId = clusterService.localNode().getId();
-            for (SnapshotId snapshotId : request.snapshotIds) {
-                Map<ShardId, IndexShardSnapshotStatus> shardsStatus = snapshotShardsService.currentSnapshotShards(snapshotId);
+            for (Snapshot snapshot : request.snapshots) {
+                Map<ShardId, IndexShardSnapshotStatus> shardsStatus = snapshotShardsService.currentSnapshotShards(snapshot);
                 if (shardsStatus == null) {
                     continue;
                 }
@@ -114,7 +114,7 @@ public class TransportNodesSnapshotsStatus extends TransportNodesAction<Transpor
                     }
                     shardMapBuilder.put(shardEntry.getKey(), shardStatus);
                 }
-                snapshotMapBuilder.put(snapshotId, unmodifiableMap(shardMapBuilder));
+                snapshotMapBuilder.put(snapshot, unmodifiableMap(shardMapBuilder));
             }
             return new NodeSnapshotStatus(clusterService.localNode(), unmodifiableMap(snapshotMapBuilder));
         } catch (Exception e) {
@@ -129,7 +129,7 @@ public class TransportNodesSnapshotsStatus extends TransportNodesAction<Transpor
 
     public static class Request extends BaseNodesRequest<Request> {
 
-        private SnapshotId[] snapshotIds;
+        private Snapshot[] snapshots;
 
         public Request() {
         }
@@ -138,8 +138,8 @@ public class TransportNodesSnapshotsStatus extends TransportNodesAction<Transpor
             super(nodesIds);
         }
 
-        public Request snapshotIds(SnapshotId[] snapshotIds) {
-            this.snapshotIds = snapshotIds;
+        public Request snapshots(Snapshot[] snapshots) {
+            this.snapshots = snapshots;
             return this;
         }
 
@@ -179,42 +179,42 @@ public class TransportNodesSnapshotsStatus extends TransportNodesAction<Transpor
 
     public static class NodeRequest extends BaseNodeRequest {
 
-        private List<SnapshotId> snapshotIds;
+        private List<Snapshot> snapshots;
 
         public NodeRequest() {
         }
 
         NodeRequest(String nodeId, TransportNodesSnapshotsStatus.Request request) {
             super(nodeId);
-            snapshotIds = Arrays.asList(request.snapshotIds);
+            snapshots = Arrays.asList(request.snapshots);
         }
 
         @Override
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
-            snapshotIds = in.readList(SnapshotId::readSnapshotId);
+            snapshots = in.readList(Snapshot::new);
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            out.writeStreamableList(snapshotIds);
+            out.writeList(snapshots);
         }
     }
 
     public static class NodeSnapshotStatus extends BaseNodeResponse {
 
-        private Map<SnapshotId, Map<ShardId, SnapshotIndexShardStatus>> status;
+        private Map<Snapshot, Map<ShardId, SnapshotIndexShardStatus>> status;
 
         NodeSnapshotStatus() {
         }
 
-        public NodeSnapshotStatus(DiscoveryNode node, Map<SnapshotId, Map<ShardId, SnapshotIndexShardStatus>> status) {
+        public NodeSnapshotStatus(DiscoveryNode node, Map<Snapshot, Map<ShardId, SnapshotIndexShardStatus>> status) {
             super(node);
             this.status = status;
         }
 
-        public Map<SnapshotId, Map<ShardId, SnapshotIndexShardStatus>> status() {
+        public Map<Snapshot, Map<ShardId, SnapshotIndexShardStatus>> status() {
             return status;
         }
 
@@ -222,9 +222,9 @@ public class TransportNodesSnapshotsStatus extends TransportNodesAction<Transpor
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
             int numberOfSnapshots = in.readVInt();
-            Map<SnapshotId, Map<ShardId, SnapshotIndexShardStatus>> snapshotMapBuilder = new HashMap<>(numberOfSnapshots);
+            Map<Snapshot, Map<ShardId, SnapshotIndexShardStatus>> snapshotMapBuilder = new HashMap<>(numberOfSnapshots);
             for (int i = 0; i < numberOfSnapshots; i++) {
-                SnapshotId snapshotId = SnapshotId.readSnapshotId(in);
+                Snapshot snapshot = new Snapshot(in);
                 int numberOfShards = in.readVInt();
                 Map<ShardId, SnapshotIndexShardStatus> shardMapBuilder = new HashMap<>(numberOfShards);
                 for (int j = 0; j < numberOfShards; j++) {
@@ -232,7 +232,7 @@ public class TransportNodesSnapshotsStatus extends TransportNodesAction<Transpor
                     SnapshotIndexShardStatus status = SnapshotIndexShardStatus.readShardSnapshotStatus(in);
                     shardMapBuilder.put(shardId, status);
                 }
-                snapshotMapBuilder.put(snapshotId, unmodifiableMap(shardMapBuilder));
+                snapshotMapBuilder.put(snapshot, unmodifiableMap(shardMapBuilder));
             }
             status = unmodifiableMap(snapshotMapBuilder);
         }
@@ -242,7 +242,7 @@ public class TransportNodesSnapshotsStatus extends TransportNodesAction<Transpor
             super.writeTo(out);
             if (status != null) {
                 out.writeVInt(status.size());
-                for (Map.Entry<SnapshotId, Map<ShardId, SnapshotIndexShardStatus>> entry : status.entrySet()) {
+                for (Map.Entry<Snapshot, Map<ShardId, SnapshotIndexShardStatus>> entry : status.entrySet()) {
                     entry.getKey().writeTo(out);
                     out.writeVInt(entry.getValue().size());
                     for (Map.Entry<ShardId, SnapshotIndexShardStatus> shardEntry : entry.getValue().entrySet()) {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
index ee60a919da6..afbb2179c0a 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
@@ -29,26 +29,32 @@ import org.elasticsearch.cluster.SnapshotsInProgress;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
+import org.elasticsearch.snapshots.Snapshot;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.snapshots.SnapshotInfo;
+import org.elasticsearch.snapshots.SnapshotMissingException;
 import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 /**
  */
@@ -87,8 +93,8 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction<Sn
     protected void masterOperation(final SnapshotsStatusRequest request,
                                    final ClusterState state,
                                    final ActionListener<SnapshotsStatusResponse> listener) throws Exception {
-        List<SnapshotsInProgress.Entry> currentSnapshots = snapshotsService.currentSnapshots(request.repository(), request.snapshots());
-
+        List<SnapshotsInProgress.Entry> currentSnapshots =
+            snapshotsService.currentSnapshots(request.repository(), Arrays.asList(request.snapshots()));
         if (currentSnapshots.isEmpty()) {
             listener.onResponse(buildResponse(request, currentSnapshots, null));
             return;
@@ -105,19 +111,19 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction<Sn
 
         if (!nodesIds.isEmpty()) {
             // There are still some snapshots running - check their progress
-            SnapshotId[] snapshotIds = new SnapshotId[currentSnapshots.size()];
+            Snapshot[] snapshots = new Snapshot[currentSnapshots.size()];
             for (int i = 0; i < currentSnapshots.size(); i++) {
-                snapshotIds[i] = currentSnapshots.get(i).snapshotId();
+                snapshots[i] = currentSnapshots.get(i).snapshot();
             }
 
             TransportNodesSnapshotsStatus.Request nodesRequest = new TransportNodesSnapshotsStatus.Request(nodesIds.toArray(new String[nodesIds.size()]))
-                    .snapshotIds(snapshotIds).timeout(request.masterNodeTimeout());
+                    .snapshots(snapshots).timeout(request.masterNodeTimeout());
             transportNodesSnapshotsStatus.execute(nodesRequest, new ActionListener<TransportNodesSnapshotsStatus.NodesSnapshotStatus>() {
                         @Override
                         public void onResponse(TransportNodesSnapshotsStatus.NodesSnapshotStatus nodeSnapshotStatuses) {
                             try {
                                 List<SnapshotsInProgress.Entry> currentSnapshots =
-                                        snapshotsService.currentSnapshots(request.repository(), request.snapshots());
+                                        snapshotsService.currentSnapshots(request.repository(), Arrays.asList(request.snapshots()));
                                 listener.onResponse(buildResponse(request, currentSnapshots, nodeSnapshotStatuses));
                             } catch (Throwable e) {
                                 listener.onFailure(e);
@@ -136,12 +142,12 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction<Sn
 
     }
 
-    private SnapshotsStatusResponse buildResponse(SnapshotsStatusRequest request, List<SnapshotsInProgress.Entry> currentSnapshots,
+    private SnapshotsStatusResponse buildResponse(SnapshotsStatusRequest request, List<SnapshotsInProgress.Entry> currentSnapshotEntries,
                                                   TransportNodesSnapshotsStatus.NodesSnapshotStatus nodeSnapshotStatuses) throws IOException {
         // First process snapshot that are currently processed
         List<SnapshotStatus> builder = new ArrayList<>();
-        Set<SnapshotId> currentSnapshotIds = new HashSet<>();
-        if (!currentSnapshots.isEmpty()) {
+        Set<String> currentSnapshotNames = new HashSet<>();
+        if (!currentSnapshotEntries.isEmpty()) {
             Map<String, TransportNodesSnapshotsStatus.NodeSnapshotStatus> nodeSnapshotStatusMap;
             if (nodeSnapshotStatuses != null) {
                 nodeSnapshotStatusMap = nodeSnapshotStatuses.getNodesMap();
@@ -149,8 +155,8 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction<Sn
                 nodeSnapshotStatusMap = new HashMap<>();
             }
 
-            for (SnapshotsInProgress.Entry entry : currentSnapshots) {
-                currentSnapshotIds.add(entry.snapshotId());
+            for (SnapshotsInProgress.Entry entry : currentSnapshotEntries) {
+                currentSnapshotNames.add(entry.snapshot().getSnapshotId().getName());
                 List<SnapshotIndexShardStatus> shardStatusBuilder = new ArrayList<>();
                 for (ObjectObjectCursor<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shardEntry : entry.shards()) {
                     SnapshotsInProgress.ShardSnapshotStatus status = shardEntry.value;
@@ -158,7 +164,7 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction<Sn
                         // We should have information about this shard from the shard:
                         TransportNodesSnapshotsStatus.NodeSnapshotStatus nodeStatus = nodeSnapshotStatusMap.get(status.nodeId());
                         if (nodeStatus != null) {
-                            Map<ShardId, SnapshotIndexShardStatus> shardStatues = nodeStatus.status().get(entry.snapshotId());
+                            Map<ShardId, SnapshotIndexShardStatus> shardStatues = nodeStatus.status().get(entry.snapshot());
                             if (shardStatues != null) {
                                 SnapshotIndexShardStatus shardStatus = shardStatues.get(shardEntry.key);
                                 if (shardStatus != null) {
@@ -190,41 +196,50 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction<Sn
                     SnapshotIndexShardStatus shardStatus = new SnapshotIndexShardStatus(shardEntry.key, stage);
                     shardStatusBuilder.add(shardStatus);
                 }
-                builder.add(new SnapshotStatus(entry.snapshotId(), entry.state(), Collections.unmodifiableList(shardStatusBuilder)));
+                builder.add(new SnapshotStatus(entry.snapshot(), entry.state(), Collections.unmodifiableList(shardStatusBuilder)));
             }
         }
         // Now add snapshots on disk that are not currently running
-        if (Strings.hasText(request.repository())) {
-            if (request.snapshots() != null && request.snapshots().length > 0) {
-                for (String snapshotName : request.snapshots()) {
-                    SnapshotId snapshotId = new SnapshotId(request.repository(), snapshotName);
-                    if (currentSnapshotIds.contains(snapshotId)) {
-                        // This is a snapshot the is currently running - skipping
+        final String repositoryName = request.repository();
+        if (Strings.hasText(repositoryName) && request.snapshots() != null && request.snapshots().length > 0) {
+            final Set<String> requestedSnapshotNames = Sets.newHashSet(request.snapshots());
+            final Map<String, SnapshotId> matchedSnapshotIds = snapshotsService.snapshotIds(repositoryName).stream()
+                .filter(s -> requestedSnapshotNames.contains(s.getName()))
+                .collect(Collectors.toMap(SnapshotId::getName, Function.identity()));
+            for (final String snapshotName : request.snapshots()) {
+                SnapshotId snapshotId = matchedSnapshotIds.get(snapshotName);
+                if (snapshotId == null) {
+                    if (currentSnapshotNames.contains(snapshotName)) {
+                        // we've already found this snapshot in the current snapshot entries, so skip over
                         continue;
+                    } else {
+                        // neither in the current snapshot entries nor found in the repository
+                        throw new SnapshotMissingException(repositoryName, snapshotName);
                     }
-                    SnapshotInfo snapshot = snapshotsService.snapshot(snapshotId);
-                    List<SnapshotIndexShardStatus> shardStatusBuilder = new ArrayList<>();
-                    if (snapshot.state().completed()) {
-                        Map<ShardId, IndexShardSnapshotStatus> shardStatues = snapshotsService.snapshotShards(snapshotId);
-                        for (Map.Entry<ShardId, IndexShardSnapshotStatus> shardStatus : shardStatues.entrySet()) {
-                            shardStatusBuilder.add(new SnapshotIndexShardStatus(shardStatus.getKey(), shardStatus.getValue()));
-                        }
-                        final SnapshotsInProgress.State state;
-                        switch (snapshot.state()) {
-                            case FAILED:
-                                state = SnapshotsInProgress.State.FAILED;
-                                break;
-                            case SUCCESS:
-                            case PARTIAL:
-                                // Translating both PARTIAL and SUCCESS to SUCCESS for now
-                                // TODO: add the differentiation on the metadata level in the next major release
-                                state = SnapshotsInProgress.State.SUCCESS;
-                                break;
-                            default:
-                                throw new IllegalArgumentException("Unknown snapshot state " + snapshot.state());
-                        }
-                        builder.add(new SnapshotStatus(snapshotId, state, Collections.unmodifiableList(shardStatusBuilder)));
+                }
+                SnapshotInfo snapshotInfo = snapshotsService.snapshot(repositoryName, snapshotId);
+                List<SnapshotIndexShardStatus> shardStatusBuilder = new ArrayList<>();
+                if (snapshotInfo.state().completed()) {
+                    Map<ShardId, IndexShardSnapshotStatus> shardStatues =
+                        snapshotsService.snapshotShards(request.repository(), snapshotInfo);
+                    for (Map.Entry<ShardId, IndexShardSnapshotStatus> shardStatus : shardStatues.entrySet()) {
+                        shardStatusBuilder.add(new SnapshotIndexShardStatus(shardStatus.getKey(), shardStatus.getValue()));
                     }
+                    final SnapshotsInProgress.State state;
+                    switch (snapshotInfo.state()) {
+                        case FAILED:
+                            state = SnapshotsInProgress.State.FAILED;
+                            break;
+                        case SUCCESS:
+                        case PARTIAL:
+                            // Translating both PARTIAL and SUCCESS to SUCCESS for now
+                            // TODO: add the differentiation on the metadata level in the next major release
+                            state = SnapshotsInProgress.State.SUCCESS;
+                            break;
+                        default:
+                            throw new IllegalArgumentException("Unknown snapshot state " + snapshotInfo.state());
+                    }
+                    builder.add(new SnapshotStatus(new Snapshot(repositoryName, snapshotInfo.snapshotId()), state, Collections.unmodifiableList(shardStatusBuilder)));
                 }
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java b/core/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java
index a083476ea2f..55a09f87f75 100644
--- a/core/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java
+++ b/core/src/main/java/org/elasticsearch/cluster/RestoreInProgress.java
@@ -21,7 +21,7 @@ package org.elasticsearch.cluster;
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.cluster.ClusterState.Custom;
-import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -34,6 +34,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * Meta data about restore processes that are currently executing
@@ -73,22 +74,6 @@ public class RestoreInProgress extends AbstractDiffable<Custom> implements Custo
         return this.entries;
     }
 
-    /**
-     * Returns currently running restore process with corresponding snapshot id or null if this snapshot is not being
-     * restored
-     *
-     * @param snapshotId snapshot id
-     * @return restore metadata or null
-     */
-    public Entry snapshot(SnapshotId snapshotId) {
-        for (Entry entry : entries) {
-            if (snapshotId.equals(entry.snapshotId())) {
-                return entry;
-            }
-        }
-        return null;
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -111,22 +96,22 @@ public class RestoreInProgress extends AbstractDiffable<Custom> implements Custo
      */
     public static class Entry {
         private final State state;
-        private final SnapshotId snapshotId;
+        private final Snapshot snapshot;
         private final ImmutableOpenMap<ShardId, ShardRestoreStatus> shards;
         private final List<String> indices;
 
         /**
          * Creates new restore metadata
          *
-         * @param snapshotId snapshot id
+         * @param snapshot   snapshot
          * @param state      current state of the restore process
          * @param indices    list of indices being restored
          * @param shards     map of shards being restored to their current restore status
          */
-        public Entry(SnapshotId snapshotId, State state, List<String> indices, ImmutableOpenMap<ShardId, ShardRestoreStatus> shards) {
-            this.snapshotId = snapshotId;
-            this.state = state;
-            this.indices = indices;
+        public Entry(Snapshot snapshot, State state, List<String> indices, ImmutableOpenMap<ShardId, ShardRestoreStatus> shards) {
+            this.snapshot = Objects.requireNonNull(snapshot);
+            this.state = Objects.requireNonNull(state);
+            this.indices = Objects.requireNonNull(indices);
             if (shards == null) {
                 this.shards = ImmutableOpenMap.of();
             } else {
@@ -135,12 +120,12 @@ public class RestoreInProgress extends AbstractDiffable<Custom> implements Custo
         }
 
         /**
-         * Returns snapshot id
+         * Returns snapshot
          *
-         * @return snapshot id
+         * @return snapshot
          */
-        public SnapshotId snapshotId() {
-            return this.snapshotId;
+        public Snapshot snapshot() {
+            return this.snapshot;
         }
 
         /**
@@ -172,26 +157,22 @@ public class RestoreInProgress extends AbstractDiffable<Custom> implements Custo
 
         @Override
         public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-
-            Entry entry = (Entry) o;
-
-            if (!indices.equals(entry.indices)) return false;
-            if (!snapshotId.equals(entry.snapshotId)) return false;
-            if (!shards.equals(entry.shards)) return false;
-            if (state != entry.state) return false;
-
-            return true;
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+            @SuppressWarnings("unchecked") Entry entry = (Entry) o;
+            return snapshot.equals(entry.snapshot) &&
+                       state == entry.state &&
+                       indices.equals(entry.indices) &&
+                       shards.equals(entry.shards);
         }
 
         @Override
         public int hashCode() {
-            int result = state.hashCode();
-            result = 31 * result + snapshotId.hashCode();
-            result = 31 * result + shards.hashCode();
-            result = 31 * result + indices.hashCode();
-            return result;
+            return Objects.hash(snapshot, state, indices, shards);
         }
     }
 
@@ -301,31 +282,29 @@ public class RestoreInProgress extends AbstractDiffable<Custom> implements Custo
 
         @Override
         public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
 
-            ShardRestoreStatus status = (ShardRestoreStatus) o;
-
-            if (nodeId != null ? !nodeId.equals(status.nodeId) : status.nodeId != null) return false;
-            if (reason != null ? !reason.equals(status.reason) : status.reason != null) return false;
-            if (state != status.state) return false;
-
-            return true;
+            @SuppressWarnings("unchecked") ShardRestoreStatus status = (ShardRestoreStatus) o;
+            return state == status.state &&
+                       Objects.equals(nodeId, status.nodeId) &&
+                       Objects.equals(reason, status.reason);
         }
 
         @Override
         public int hashCode() {
-            int result = state != null ? state.hashCode() : 0;
-            result = 31 * result + (nodeId != null ? nodeId.hashCode() : 0);
-            result = 31 * result + (reason != null ? reason.hashCode() : 0);
-            return result;
+            return Objects.hash(state, nodeId, reason);
         }
     }
 
     /**
      * Shard restore process state
      */
-    public static enum State {
+    public enum State {
         /**
          * Initializing state
          */
@@ -409,7 +388,7 @@ public class RestoreInProgress extends AbstractDiffable<Custom> implements Custo
     public RestoreInProgress readFrom(StreamInput in) throws IOException {
         Entry[] entries = new Entry[in.readVInt()];
         for (int i = 0; i < entries.length; i++) {
-            SnapshotId snapshotId = SnapshotId.readSnapshotId(in);
+            Snapshot snapshot = new Snapshot(in);
             State state = State.fromValue(in.readByte());
             int indices = in.readVInt();
             List<String> indexBuilder = new ArrayList<>();
@@ -423,7 +402,7 @@ public class RestoreInProgress extends AbstractDiffable<Custom> implements Custo
                 ShardRestoreStatus shardState = ShardRestoreStatus.readShardRestoreStatus(in);
                 builder.put(shardId, shardState);
             }
-            entries[i] = new Entry(snapshotId, state, Collections.unmodifiableList(indexBuilder), builder.build());
+            entries[i] = new Entry(snapshot, state, Collections.unmodifiableList(indexBuilder), builder.build());
         }
         return new RestoreInProgress(entries);
     }
@@ -435,7 +414,7 @@ public class RestoreInProgress extends AbstractDiffable<Custom> implements Custo
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVInt(entries.size());
         for (Entry entry : entries) {
-            entry.snapshotId().writeTo(out);
+            entry.snapshot().writeTo(out);
             out.writeByte(entry.state().value());
             out.writeVInt(entry.indices().size());
             for (String index : entry.indices()) {
@@ -471,8 +450,8 @@ public class RestoreInProgress extends AbstractDiffable<Custom> implements Custo
      */
     public void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException {
         builder.startObject();
-        builder.field("snapshot", entry.snapshotId().getSnapshot());
-        builder.field("repository", entry.snapshotId().getRepository());
+        builder.field("snapshot", entry.snapshot().getSnapshotId().getName());
+        builder.field("repository", entry.snapshot().getRepository());
         builder.field("state", entry.state());
         builder.startArray("indices");
         {
diff --git a/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java b/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
index 5432c1f0f19..f0a0fdec665 100644
--- a/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
+++ b/core/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
@@ -23,13 +23,13 @@ import com.carrotsearch.hppc.ObjectContainer;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.cluster.ClusterState.Custom;
-import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.snapshots.Snapshot;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -66,7 +66,7 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
 
     public static class Entry {
         private final State state;
-        private final SnapshotId snapshotId;
+        private final Snapshot snapshot;
         private final boolean includeGlobalState;
         private final boolean partial;
         private final ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards;
@@ -74,9 +74,10 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
         private final ImmutableOpenMap<String, List<ShardId>> waitingIndices;
         private final long startTime;
 
-        public Entry(SnapshotId snapshotId, boolean includeGlobalState, boolean partial, State state, List<String> indices, long startTime, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
+        public Entry(Snapshot snapshot, boolean includeGlobalState, boolean partial, State state, List<String> indices, long startTime,
+                     ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
             this.state = state;
-            this.snapshotId = snapshotId;
+            this.snapshot = snapshot;
             this.includeGlobalState = includeGlobalState;
             this.partial = partial;
             this.indices = indices;
@@ -91,15 +92,15 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
         }
 
         public Entry(Entry entry, State state, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
-            this(entry.snapshotId, entry.includeGlobalState, entry.partial, state, entry.indices, entry.startTime, shards);
+            this(entry.snapshot, entry.includeGlobalState, entry.partial, state, entry.indices, entry.startTime, shards);
         }
 
         public Entry(Entry entry, ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
             this(entry, entry.state, shards);
         }
 
-        public SnapshotId snapshotId() {
-            return this.snapshotId;
+        public Snapshot snapshot() {
+            return this.snapshot;
         }
 
         public ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards() {
@@ -142,7 +143,7 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
             if (startTime != entry.startTime) return false;
             if (!indices.equals(entry.indices)) return false;
             if (!shards.equals(entry.shards)) return false;
-            if (!snapshotId.equals(entry.snapshotId)) return false;
+            if (!snapshot.equals(entry.snapshot)) return false;
             if (state != entry.state) return false;
             if (!waitingIndices.equals(entry.waitingIndices)) return false;
 
@@ -152,7 +153,7 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
         @Override
         public int hashCode() {
             int result = state.hashCode();
-            result = 31 * result + snapshotId.hashCode();
+            result = 31 * result + snapshot.hashCode();
             result = 31 * result + (includeGlobalState ? 1 : 0);
             result = 31 * result + (partial ? 1 : 0);
             result = 31 * result + shards.hashCode();
@@ -162,6 +163,11 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
             return result;
         }
 
+        @Override
+        public String toString() {
+            return snapshot.toString();
+        }
+
         private ImmutableOpenMap<String, List<ShardId>> findWaitingIndices(ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards) {
             Map<String, List<ShardId>> waitingIndicesMap = new HashMap<>();
             for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> entry : shards) {
@@ -277,7 +283,7 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
         }
     }
 
-    public static enum State {
+    public enum State {
         INIT((byte) 0, false, false),
         STARTED((byte) 1, false, false),
         SUCCESS((byte) 2, true, false),
@@ -347,9 +353,10 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
         return this.entries;
     }
 
-    public Entry snapshot(SnapshotId snapshotId) {
+    public Entry snapshot(final Snapshot snapshot) {
         for (Entry entry : entries) {
-            if (snapshotId.equals(entry.snapshotId())) {
+            final Snapshot curr = entry.snapshot();
+            if (curr.equals(snapshot)) {
                 return entry;
             }
         }
@@ -365,7 +372,7 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
     public SnapshotsInProgress readFrom(StreamInput in) throws IOException {
         Entry[] entries = new Entry[in.readVInt()];
         for (int i = 0; i < entries.length; i++) {
-            SnapshotId snapshotId = SnapshotId.readSnapshotId(in);
+            Snapshot snapshot = new Snapshot(in);
             boolean includeGlobalState = in.readBoolean();
             boolean partial = in.readBoolean();
             State state = State.fromValue(in.readByte());
@@ -383,7 +390,13 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
                 State shardState = State.fromValue(in.readByte());
                 builder.put(shardId, new ShardSnapshotStatus(nodeId, shardState));
             }
-            entries[i] = new Entry(snapshotId, includeGlobalState, partial, state, Collections.unmodifiableList(indexBuilder), startTime, builder.build());
+            entries[i] = new Entry(snapshot,
+                                   includeGlobalState,
+                                   partial,
+                                   state,
+                                   Collections.unmodifiableList(indexBuilder),
+                                   startTime,
+                                   builder.build());
         }
         return new SnapshotsInProgress(entries);
     }
@@ -392,7 +405,7 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVInt(entries.size());
         for (Entry entry : entries) {
-            entry.snapshotId().writeTo(out);
+            entry.snapshot().writeTo(out);
             out.writeBoolean(entry.includeGlobalState());
             out.writeBoolean(entry.partial());
             out.writeByte(entry.state().value());
@@ -410,25 +423,24 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
         }
     }
 
-    static final class Fields {
-        static final String REPOSITORY = "repository";
-        static final String SNAPSHOTS = "snapshots";
-        static final String SNAPSHOT = "snapshot";
-        static final String INCLUDE_GLOBAL_STATE = "include_global_state";
-        static final String PARTIAL = "partial";
-        static final String STATE = "state";
-        static final String INDICES = "indices";
-        static final String START_TIME_MILLIS = "start_time_millis";
-        static final String START_TIME = "start_time";
-        static final String SHARDS = "shards";
-        static final String INDEX = "index";
-        static final String SHARD = "shard";
-        static final String NODE = "node";
-    }
+    private static final String REPOSITORY = "repository";
+    private static final String SNAPSHOTS = "snapshots";
+    private static final String SNAPSHOT = "snapshot";
+    private static final String UUID = "uuid";
+    private static final String INCLUDE_GLOBAL_STATE = "include_global_state";
+    private static final String PARTIAL = "partial";
+    private static final String STATE = "state";
+    private static final String INDICES = "indices";
+    private static final String START_TIME_MILLIS = "start_time_millis";
+    private static final String START_TIME = "start_time";
+    private static final String SHARDS = "shards";
+    private static final String INDEX = "index";
+    private static final String SHARD = "shard";
+    private static final String NODE = "node";
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
-        builder.startArray(Fields.SNAPSHOTS);
+        builder.startArray(SNAPSHOTS);
         for (Entry entry : entries) {
             toXContent(entry, builder, params);
         }
@@ -438,30 +450,31 @@ public class SnapshotsInProgress extends AbstractDiffable<Custom> implements Cus
 
     public void toXContent(Entry entry, XContentBuilder builder, ToXContent.Params params) throws IOException {
         builder.startObject();
-        builder.field(Fields.REPOSITORY, entry.snapshotId().getRepository());
-        builder.field(Fields.SNAPSHOT, entry.snapshotId().getSnapshot());
-        builder.field(Fields.INCLUDE_GLOBAL_STATE, entry.includeGlobalState());
-        builder.field(Fields.PARTIAL, entry.partial());
-        builder.field(Fields.STATE, entry.state());
-        builder.startArray(Fields.INDICES);
+        builder.field(REPOSITORY, entry.snapshot().getRepository());
+        builder.field(SNAPSHOT, entry.snapshot().getSnapshotId().getName());
+        builder.field(UUID, entry.snapshot().getSnapshotId().getUUID());
+        builder.field(INCLUDE_GLOBAL_STATE, entry.includeGlobalState());
+        builder.field(PARTIAL, entry.partial());
+        builder.field(STATE, entry.state());
+        builder.startArray(INDICES);
         {
             for (String index : entry.indices()) {
                 builder.value(index);
             }
         }
         builder.endArray();
-        builder.timeValueField(Fields.START_TIME_MILLIS, Fields.START_TIME, entry.startTime());
-        builder.startArray(Fields.SHARDS);
+        builder.timeValueField(START_TIME_MILLIS, START_TIME, entry.startTime());
+        builder.startArray(SHARDS);
         {
             for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shardEntry : entry.shards) {
                 ShardId shardId = shardEntry.key;
                 ShardSnapshotStatus status = shardEntry.value;
                 builder.startObject();
                 {
-                    builder.field(Fields.INDEX, shardId.getIndex());
-                    builder.field(Fields.SHARD, shardId.getId());
-                    builder.field(Fields.STATE, status.state());
-                    builder.field(Fields.NODE, status.nodeId());
+                    builder.field(INDEX, shardId.getIndex());
+                    builder.field(SHARD, shardId.getId());
+                    builder.field(STATE, status.state());
+                    builder.field(NODE, status.nodeId());
                 }
                 builder.endObject();
             }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/SnapshotId.java b/core/src/main/java/org/elasticsearch/cluster/metadata/SnapshotId.java
deleted file mode 100644
index 88c60f1f07e..00000000000
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/SnapshotId.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cluster.metadata;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
-
-import java.io.IOException;
-
-/**
- * Snapshot ID - repository name + snapshot name
- */
-public class SnapshotId implements Streamable {
-
-    private String repository;
-
-    private String snapshot;
-
-    // Caching hash code
-    private int hashCode;
-
-    private SnapshotId() {
-    }
-
-    /**
-     * Constructs new snapshot id
-     *
-     * @param repository repository name
-     * @param snapshot   snapshot name
-     */
-    public SnapshotId(String repository, String snapshot) {
-        this.repository = repository;
-        this.snapshot = snapshot;
-        this.hashCode = computeHashCode();
-    }
-
-    /**
-     * Returns repository name
-     *
-     * @return repository name
-     */
-    public String getRepository() {
-        return repository;
-    }
-
-    /**
-     * Returns snapshot name
-     *
-     * @return snapshot name
-     */
-    public String getSnapshot() {
-        return snapshot;
-    }
-
-    @Override
-    public String toString() {
-        return repository + ":" + snapshot;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null) return false;
-        SnapshotId snapshotId = (SnapshotId) o;
-        return snapshot.equals(snapshotId.snapshot) && repository.equals(snapshotId.repository);
-    }
-
-    @Override
-    public int hashCode() {
-        return hashCode;
-    }
-
-    private int computeHashCode() {
-        int result = repository != null ? repository.hashCode() : 0;
-        result = 31 * result + snapshot.hashCode();
-        return result;
-    }
-
-    /**
-     * Reads snapshot id from stream input
-     *
-     * @param in stream input
-     * @return snapshot id
-     */
-    public static SnapshotId readSnapshotId(StreamInput in) throws IOException {
-        SnapshotId snapshot = new SnapshotId();
-        snapshot.readFrom(in);
-        return snapshot;
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        repository = in.readString();
-        snapshot = in.readString();
-        hashCode = computeHashCode();
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeString(repository);
-        out.writeString(snapshot);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
index 8bb0f51e22e..0fe49369177 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java
@@ -376,14 +376,20 @@ public class IndexRoutingTable extends AbstractDiffable<IndexRoutingTable> imple
          * Initializes a new empty index, to be restored from a snapshot
          */
         public Builder initializeAsNewRestore(IndexMetaData indexMetaData, RestoreSource restoreSource, IntSet ignoreShards) {
-            return initializeAsRestore(indexMetaData, restoreSource, ignoreShards, true, new UnassignedInfo(UnassignedInfo.Reason.NEW_INDEX_RESTORED, "restore_source[" + restoreSource.snapshotId().getRepository() + "/" + restoreSource.snapshotId().getSnapshot() + "]"));
+            final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NEW_INDEX_RESTORED,
+                                                                     "restore_source[" + restoreSource.snapshot().getRepository() + "/" +
+                                                                         restoreSource.snapshot().getSnapshotId().getName() + "]");
+            return initializeAsRestore(indexMetaData, restoreSource, ignoreShards, true, unassignedInfo);
         }
 
         /**
          * Initializes an existing index, to be restored from a snapshot
          */
         public Builder initializeAsRestore(IndexMetaData indexMetaData, RestoreSource restoreSource) {
-            return initializeAsRestore(indexMetaData, restoreSource, null, false, new UnassignedInfo(UnassignedInfo.Reason.EXISTING_INDEX_RESTORED, "restore_source[" + restoreSource.snapshotId().getRepository() + "/" + restoreSource.snapshotId().getSnapshot() + "]"));
+            final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.EXISTING_INDEX_RESTORED,
+                                                                     "restore_source[" + restoreSource.snapshot().getRepository() + "/" +
+                                                                         restoreSource.snapshot().getSnapshotId().getName() + "]");
+            return initializeAsRestore(indexMetaData, restoreSource, null, false, unassignedInfo);
         }
 
         /**
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RestoreSource.java b/core/src/main/java/org/elasticsearch/cluster/routing/RestoreSource.java
index c091f71798b..f80e55ed8b3 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RestoreSource.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RestoreSource.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.cluster.routing;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
@@ -28,13 +28,14 @@ import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.Objects;
 
 /**
  * Represents snapshot and index from which a recovering index should be restored
  */
 public class RestoreSource implements Streamable, ToXContent {
 
-    private SnapshotId snapshotId;
+    private Snapshot snapshot;
 
     private String index;
 
@@ -43,14 +44,14 @@ public class RestoreSource implements Streamable, ToXContent {
     RestoreSource() {
     }
 
-    public RestoreSource(SnapshotId snapshotId, Version version, String index) {
-        this.snapshotId = snapshotId;
-        this.version = version;
-        this.index = index;
+    public RestoreSource(Snapshot snapshot, Version version, String index) {
+        this.snapshot = Objects.requireNonNull(snapshot);
+        this.version = Objects.requireNonNull(version);
+        this.index = Objects.requireNonNull(index);
     }
 
-    public SnapshotId snapshotId() {
-        return snapshotId;
+    public Snapshot snapshot() {
+        return snapshot;
     }
 
     public String index() {
@@ -61,26 +62,20 @@ public class RestoreSource implements Streamable, ToXContent {
         return version;
     }
 
-    public static RestoreSource readRestoreSource(StreamInput in) throws IOException {
-        RestoreSource restoreSource = new RestoreSource();
-        restoreSource.readFrom(in);
-        return restoreSource;
-    }
-
     public static RestoreSource readOptionalRestoreSource(StreamInput in) throws IOException {
         return in.readOptionalStreamable(RestoreSource::new);
     }
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
-        snapshotId = SnapshotId.readSnapshotId(in);
+        snapshot = new Snapshot(in);
         version = Version.readVersion(in);
         index = in.readString();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        snapshotId.writeTo(out);
+        snapshot.writeTo(out);
         Version.writeVersion(version, out);
         out.writeString(index);
     }
@@ -88,8 +83,8 @@ public class RestoreSource implements Streamable, ToXContent {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         return builder.startObject()
-                .field("repository", snapshotId.getRepository())
-                .field("snapshot", snapshotId.getSnapshot())
+                .field("repository", snapshot.getRepository())
+                .field("snapshot", snapshot.getSnapshotId().getName())
                 .field("version", version.toString())
                 .field("index", index)
                 .endObject();
@@ -97,26 +92,24 @@ public class RestoreSource implements Streamable, ToXContent {
 
     @Override
     public String toString() {
-        return snapshotId.toString();
+        return snapshot.toString();
     }
 
     @Override
     public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
 
-        RestoreSource that = (RestoreSource) o;
-
-        if (!index.equals(that.index)) return false;
-        if (!snapshotId.equals(that.snapshotId)) return false;
-
-        return true;
+        @SuppressWarnings("unchecked") RestoreSource that = (RestoreSource) o;
+        return snapshot.equals(that.snapshot) && index.equals(that.index);
     }
 
     @Override
     public int hashCode() {
-        int result = snapshotId.hashCode();
-        result = 31 * result + index.hashCode();
-        return result;
+        return Objects.hash(snapshot, index);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index adcf36f694f..44445f0b6dd 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -39,7 +39,6 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RestoreSource;
-import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.common.Booleans;
@@ -1442,7 +1441,7 @@ public class IndexShard extends AbstractIndexShardComponent {
             markAsRecovering("from snapshot", recoveryState); // mark the shard as recovering on the cluster state thread
             threadPool.generic().execute(() -> {
                 try {
-                    final IndexShardRepository indexShardRepository = repositoriesService.indexShardRepository(restoreSource.snapshotId().getRepository());
+                    final IndexShardRepository indexShardRepository = repositoriesService.indexShardRepository(restoreSource.snapshot().getRepository());
                     if (restoreFromRepository(indexShardRepository)) {
                         recoveryListener.onRecoveryDone(recoveryState);
                     }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
index c7d6f35eff2..62173f936c5 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
@@ -395,7 +395,7 @@ final class StoreRecovery {
             throw new IndexShardRestoreFailedException(shardId, "empty restore source");
         }
         if (logger.isTraceEnabled()) {
-            logger.trace("[{}] restoring shard [{}]", restoreSource.snapshotId(), shardId);
+            logger.trace("[{}] restoring shard [{}]", restoreSource.snapshot(), shardId);
         }
         try {
             translogState.totalOperations(0);
@@ -405,7 +405,7 @@ final class StoreRecovery {
             if (!shardId.getIndexName().equals(restoreSource.index())) {
                 snapshotShardId = new ShardId(restoreSource.index(), IndexMetaData.INDEX_UUID_NA_VALUE, shardId.id());
             }
-            indexShardRepository.restore(restoreSource.snapshotId(), restoreSource.version(), shardId, snapshotShardId, indexShard.recoveryState());
+            indexShardRepository.restore(restoreSource.snapshot().getSnapshotId(), restoreSource.version(), shardId, snapshotShardId, indexShard.recoveryState());
             indexShard.skipTranslogRecovery();
             indexShard.finalizeRecovery();
             indexShard.postRecovery("restore done");
diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardRepository.java b/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardRepository.java
index ca481e1430d..5988d82def2 100644
--- a/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardRepository.java
+++ b/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardRepository.java
@@ -21,7 +21,7 @@ package org.elasticsearch.index.snapshots;
 
 import org.apache.lucene.index.IndexCommit;
 import org.elasticsearch.Version;
-import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.recovery.RecoveryState;
 
diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java
index c15d2cfcdbe..e0032fe503b 100644
--- a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java
+++ b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java
@@ -32,7 +32,7 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
-import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParseFieldMatcher;
@@ -204,7 +204,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements
         try {
             snapshotContext.restore();
         } catch (Throwable e) {
-            throw new IndexShardRestoreFailedException(shardId, "failed to restore snapshot [" + snapshotId.getSnapshot() + "]", e);
+            throw new IndexShardRestoreFailedException(shardId, "failed to restore snapshot [" + snapshotId + "]", e);
         }
     }
 
@@ -318,7 +318,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements
             int fileListGeneration = tuple.v2();
 
             try {
-                indexShardSnapshotFormat(version).delete(blobContainer, snapshotId.getSnapshot());
+                indexShardSnapshotFormat(version).delete(blobContainer, snapshotId.getName());
             } catch (IOException e) {
                 logger.debug("[{}] [{}] failed to delete shard snapshot file", shardId, snapshotId);
             }
@@ -326,7 +326,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements
             // Build a list of snapshots that should be preserved
             List<SnapshotFiles> newSnapshotsList = new ArrayList<>();
             for (SnapshotFiles point : snapshots) {
-                if (!point.snapshot().equals(snapshotId.getSnapshot())) {
+                if (!point.snapshot().equals(snapshotId.getName())) {
                     newSnapshotsList.add(point);
                 }
             }
@@ -339,7 +339,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements
          */
         public BlobStoreIndexShardSnapshot loadSnapshot() {
             try {
-                return indexShardSnapshotFormat(version).read(blobContainer, snapshotId.getSnapshot());
+                return indexShardSnapshotFormat(version).read(blobContainer, snapshotId.getName());
             } catch (IOException ex) {
                 throw new IndexShardRestoreFailedException(shardId, "failed to read shard snapshot file", ex);
             }
@@ -605,14 +605,14 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements
                 // now create and write the commit point
                 snapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.FINALIZE);
 
-                BlobStoreIndexShardSnapshot snapshot = new BlobStoreIndexShardSnapshot(snapshotId.getSnapshot(),
+                BlobStoreIndexShardSnapshot snapshot = new BlobStoreIndexShardSnapshot(snapshotId.getName(),
                         snapshotIndexCommit.getGeneration(), indexCommitPointFiles, snapshotStatus.startTime(),
                         // snapshotStatus.startTime() is assigned on the same machine, so it's safe to use with VLong
                         System.currentTimeMillis() - snapshotStatus.startTime(), indexNumberOfFiles, indexTotalFilesSize);
                 //TODO: The time stored in snapshot doesn't include cleanup time.
                 logger.trace("[{}] [{}] writing shard snapshot file", shardId, snapshotId);
                 try {
-                    indexShardSnapshotFormat.write(snapshot, blobContainer, snapshotId.getSnapshot());
+                    indexShardSnapshotFormat.write(snapshot, blobContainer, snapshotId.getName());
                 } catch (IOException e) {
                     throw new IndexShardSnapshotFailedException(shardId, "Failed to write commit point", e);
                 }
diff --git a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
index dcf055275e5..719fb812c74 100644
--- a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
+++ b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
@@ -645,7 +645,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
         @Override
         public void onRecoveryDone(RecoveryState state) {
             if (state.getType() == RecoveryState.Type.SNAPSHOT) {
-                restoreService.indexShardRestoreCompleted(state.getRestoreSource().snapshotId(), shardRouting.shardId());
+                restoreService.indexShardRestoreCompleted(state.getRestoreSource().snapshot(), shardRouting.shardId());
             }
             shardStateAction.shardStarted(shardRouting, message(state), SHARD_STATE_ACTION_LISTENER);
         }
@@ -666,7 +666,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
             if (state.getType() == RecoveryState.Type.SNAPSHOT) {
                 try {
                     if (Lucene.isCorruptionException(e.getCause())) {
-                        restoreService.failRestore(state.getRestoreSource().snapshotId(), shardRouting.shardId());
+                        restoreService.failRestore(state.getRestoreSource().snapshot(), shardRouting.shardId());
                     }
                 } catch (Throwable inner) {
                     e.addSuppressed(inner);
diff --git a/core/src/main/java/org/elasticsearch/repositories/Repository.java b/core/src/main/java/org/elasticsearch/repositories/Repository.java
index f3c44f29226..0cfe74a0e34 100644
--- a/core/src/main/java/org/elasticsearch/repositories/Repository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/Repository.java
@@ -20,7 +20,7 @@ package org.elasticsearch.repositories;
 
 import org.apache.lucene.index.IndexCommit;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
@@ -29,6 +29,7 @@ import org.elasticsearch.snapshots.SnapshotShardFailure;
 
 import java.io.IOException;
 import java.util.List;
+import java.util.function.Predicate;
 
 /**
  * Snapshot repository interface.
@@ -39,7 +40,7 @@ import java.util.List;
  * <p>
  * Typical snapshot usage pattern:
  * <ul>
- * <li>Master calls {@link #initializeSnapshot(org.elasticsearch.cluster.metadata.SnapshotId, List, org.elasticsearch.cluster.metadata.MetaData)}
+ * <li>Master calls {@link #initializeSnapshot(SnapshotId, List, org.elasticsearch.cluster.metadata.MetaData)}
  * with list of indices that will be included into the snapshot</li>
  * <li>Data nodes call {@link org.elasticsearch.index.snapshots.IndexShardRepository#snapshot(SnapshotId, ShardId, IndexCommit, IndexShardSnapshotStatus)} for each shard</li>
  * <li>When all shard calls return master calls {@link #finalizeSnapshot}
@@ -51,7 +52,7 @@ public interface Repository extends LifecycleComponent<Repository> {
     /**
      * Reads snapshot description from repository.
      *
-     * @param snapshotId snapshot ID
+     * @param snapshotId  snapshot id
      * @return information about snapshot
      */
     SnapshotInfo readSnapshot(SnapshotId snapshotId);
@@ -65,10 +66,11 @@ public interface Repository extends LifecycleComponent<Repository> {
      * @param indices    list of indices
      * @return information about snapshot
      */
-    MetaData readSnapshotMetaData(SnapshotId snapshotId, SnapshotInfo snapshot, List<String> indices) throws IOException;
+    MetaData readSnapshotMetaData(SnapshotInfo snapshot, List<String> indices) throws IOException;
 
     /**
-     * Returns the list of snapshots currently stored in the repository
+     * Returns the list of snapshots currently stored in the repository that match the given predicate on the snapshot name.
+     * To get all snapshots, the predicate filter should return true regardless of the input.
      *
      * @return snapshot list
      */
diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index 121df3e5832..506e64cb9a5 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -24,7 +24,7 @@ import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
@@ -34,6 +34,7 @@ import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.compress.NotXContentException;
 import org.elasticsearch.common.io.Streams;
@@ -56,7 +57,6 @@ import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.RepositorySettings;
 import org.elasticsearch.repositories.RepositoryVerificationException;
-import org.elasticsearch.snapshots.InvalidSnapshotNameException;
 import org.elasticsearch.snapshots.SnapshotCreationException;
 import org.elasticsearch.snapshots.SnapshotException;
 import org.elasticsearch.snapshots.SnapshotInfo;
@@ -71,6 +71,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 /**
  * BlobStore - based implementation of Snapshot Repository
@@ -130,7 +131,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
 
     private static final String SNAPSHOT_CODEC = "snapshot";
 
-    private static final String SNAPSHOTS_FILE = "index";
+    static final String SNAPSHOTS_FILE = "index"; // package private for unit testing
 
     private static final String TESTS_FILE = "tests-";
 
@@ -268,20 +269,25 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
             throw new RepositoryException(this.repositoryName, "cannot create snapshot in a readonly repository");
         }
         try {
-            if (snapshotFormat.exists(snapshotsBlobContainer, snapshotId.getSnapshot()) ||
-                    snapshotLegacyFormat.exists(snapshotsBlobContainer, snapshotId.getSnapshot())) {
-                throw new InvalidSnapshotNameException(snapshotId, "snapshot with such name already exists");
+            final String snapshotName = snapshotId.getName();
+            // check if the snapshot name already exists in the repository
+            if (snapshots().stream().anyMatch(s -> s.getName().equals(snapshotName))) {
+                throw new SnapshotCreationException(repositoryName, snapshotId, "snapshot with the same name already exists");
+            }
+            if (snapshotFormat.exists(snapshotsBlobContainer, blobId(snapshotId)) ||
+                    snapshotLegacyFormat.exists(snapshotsBlobContainer, snapshotName)) {
+                throw new SnapshotCreationException(repositoryName, snapshotId, "snapshot with such name already exists");
             }
             // Write Global MetaData
-            globalMetaDataFormat.write(metaData, snapshotsBlobContainer, snapshotId.getSnapshot());
+            globalMetaDataFormat.write(metaData, snapshotsBlobContainer, snapshotName);
             for (String index : indices) {
                 final IndexMetaData indexMetaData = metaData.index(index);
                 final BlobPath indexPath = basePath().add("indices").add(index);
                 final BlobContainer indexMetaDataBlobContainer = blobStore().blobContainer(indexPath);
-                indexMetaDataFormat.write(indexMetaData, indexMetaDataBlobContainer, snapshotId.getSnapshot());
+                indexMetaDataFormat.write(indexMetaData, indexMetaDataBlobContainer, snapshotName);
             }
         } catch (IOException ex) {
-            throw new SnapshotCreationException(snapshotId, ex);
+            throw new SnapshotCreationException(repositoryName, snapshotId, ex);
         }
     }
 
@@ -314,35 +320,27 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
             logger.warn("cannot read metadata for snapshot [{}]", ex, snapshotId);
         }
         try {
+            final String snapshotName = snapshotId.getName();
             // Delete snapshot file first so we wouldn't end up with partially deleted snapshot that looks OK
             if (snapshot != null) {
-                snapshotFormat(snapshot.version()).delete(snapshotsBlobContainer, snapshotId.getSnapshot());
-                globalMetaDataFormat(snapshot.version()).delete(snapshotsBlobContainer, snapshotId.getSnapshot());
+                snapshotFormat(snapshot.version()).delete(snapshotsBlobContainer, blobId(snapshotId));
+                globalMetaDataFormat(snapshot.version()).delete(snapshotsBlobContainer, snapshotName);
             } else {
                 // We don't know which version was the snapshot created with - try deleting both current and legacy formats
-                snapshotFormat.delete(snapshotsBlobContainer, snapshotId.getSnapshot());
-                snapshotLegacyFormat.delete(snapshotsBlobContainer, snapshotId.getSnapshot());
-                globalMetaDataLegacyFormat.delete(snapshotsBlobContainer, snapshotId.getSnapshot());
-                globalMetaDataFormat.delete(snapshotsBlobContainer, snapshotId.getSnapshot());
+                snapshotFormat.delete(snapshotsBlobContainer, blobId(snapshotId));
+                snapshotLegacyFormat.delete(snapshotsBlobContainer, snapshotName);
+                globalMetaDataLegacyFormat.delete(snapshotsBlobContainer, snapshotName);
+                globalMetaDataFormat.delete(snapshotsBlobContainer, snapshotName);
             }
             // Delete snapshot from the snapshot list
-            List<SnapshotId> snapshotIds = snapshots();
-            if (snapshotIds.contains(snapshotId)) {
-                List<SnapshotId> builder = new ArrayList<>();
-                for (SnapshotId id : snapshotIds) {
-                    if (!snapshotId.equals(id)) {
-                        builder.add(id);
-                    }
-                }
-                snapshotIds = Collections.unmodifiableList(builder);
-            }
+            List<SnapshotId> snapshotIds = snapshots().stream().filter(id -> snapshotId.equals(id) == false).collect(Collectors.toList());
             writeSnapshotList(snapshotIds);
             // Now delete all indices
             for (String index : indices) {
                 BlobPath indexPath = basePath().add("indices").add(index);
                 BlobContainer indexMetaDataBlobContainer = blobStore().blobContainer(indexPath);
                 try {
-                    indexMetaDataFormat(snapshot.version()).delete(indexMetaDataBlobContainer, snapshotId.getSnapshot());
+                    indexMetaDataFormat(snapshot.version()).delete(indexMetaDataBlobContainer, snapshotId.getName());
                 } catch (IOException ex) {
                     logger.warn("[{}] failed to delete metadata for index [{}]", ex, snapshotId, index);
                 }
@@ -368,10 +366,21 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
      * {@inheritDoc}
      */
     @Override
-    public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, List<String> indices, long startTime, String failure, int totalShards, List<SnapshotShardFailure> shardFailures) {
+    public SnapshotInfo finalizeSnapshot(final SnapshotId snapshotId,
+                                         final List<String> indices,
+                                         final long startTime,
+                                         final String failure,
+                                         final int totalShards,
+                                         final List<SnapshotShardFailure> shardFailures) {
         try {
-            SnapshotInfo blobStoreSnapshot = new SnapshotInfo(snapshotId.getSnapshot(), indices, startTime, failure, System.currentTimeMillis(), totalShards, shardFailures);
-            snapshotFormat.write(blobStoreSnapshot, snapshotsBlobContainer, snapshotId.getSnapshot());
+            SnapshotInfo blobStoreSnapshot = new SnapshotInfo(snapshotId,
+                                                              indices,
+                                                              startTime,
+                                                              failure,
+                                                              System.currentTimeMillis(),
+                                                              totalShards,
+                                                              shardFailures);
+            snapshotFormat.write(blobStoreSnapshot, snapshotsBlobContainer, blobId(snapshotId));
             List<SnapshotId> snapshotIds = snapshots();
             if (!snapshotIds.contains(snapshotId)) {
                 snapshotIds = new ArrayList<>(snapshotIds);
@@ -405,15 +414,22 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
             for (BlobMetaData md : blobs.values()) {
                 String blobName = md.name();
                 final String name;
+                String uuid;
                 if (blobName.startsWith(SNAPSHOT_PREFIX) && blobName.length() > legacyPrefixLength) {
-                    name = blobName.substring(prefixLength, blobName.length() - suffixLength);
+                    final String str = blobName.substring(prefixLength, blobName.length() - suffixLength);
+                    // TODO: this will go away once we make the snapshot file writes atomic and
+                    // use it as the source of truth for the snapshots list instead of listing blobs
+                    Tuple<String, String> pair = parseNameUUIDFromBlobName(str);
+                    name = pair.v1();
+                    uuid = pair.v2();
                 } else if (blobName.startsWith(LEGACY_SNAPSHOT_PREFIX) && blobName.length() > suffixLength + prefixLength) {
                     name = blobName.substring(legacyPrefixLength);
+                    uuid = SnapshotId.UNASSIGNED_UUID;
                 } else {
                     // not sure what it was - ignore
                     continue;
                 }
-                snapshots.add(new SnapshotId(repositoryName, name));
+                snapshots.add(new SnapshotId(name, uuid));
             }
             return Collections.unmodifiableList(snapshots);
         } catch (IOException ex) {
@@ -425,28 +441,25 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
      * {@inheritDoc}
      */
     @Override
-    public MetaData readSnapshotMetaData(SnapshotId snapshotId, SnapshotInfo snapshot, List<String> indices) throws IOException {
-        return readSnapshotMetaData(snapshotId, snapshot.version(), indices, false);
+    public MetaData readSnapshotMetaData(SnapshotInfo snapshot, List<String> indices) throws IOException {
+        return readSnapshotMetaData(snapshot.snapshotId(), snapshot.version(), indices, false);
     }
 
-    /**
-     * {@inheritDoc}
-     */
     @Override
-    public SnapshotInfo readSnapshot(SnapshotId snapshotId) {
+    public SnapshotInfo readSnapshot(final SnapshotId snapshotId) {
         try {
-            return snapshotFormat.read(snapshotsBlobContainer, snapshotId.getSnapshot());
+            return snapshotFormat.read(snapshotsBlobContainer, blobId(snapshotId));
         } catch (FileNotFoundException | NoSuchFileException ex) {
             // File is missing - let's try legacy format instead
             try {
-                return snapshotLegacyFormat.read(snapshotsBlobContainer, snapshotId.getSnapshot());
+                return snapshotLegacyFormat.read(snapshotsBlobContainer, snapshotId.getName());
             } catch (FileNotFoundException | NoSuchFileException ex1) {
-                throw new SnapshotMissingException(snapshotId, ex);
+                throw new SnapshotMissingException(repositoryName, snapshotId, ex);
             } catch (IOException | NotXContentException ex1) {
-                throw new SnapshotException(snapshotId, "failed to get snapshots", ex1);
+                throw new SnapshotException(repositoryName, snapshotId, "failed to get snapshots", ex1);
             }
         } catch (IOException | NotXContentException ex) {
-            throw new SnapshotException(snapshotId, "failed to get snapshots", ex);
+            throw new SnapshotException(repositoryName, snapshotId, "failed to get snapshots", ex);
         }
     }
 
@@ -456,27 +469,27 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
             // When we delete corrupted snapshots we might not know which version we are dealing with
             // We can try detecting the version based on the metadata file format
             assert ignoreIndexErrors;
-            if (globalMetaDataFormat.exists(snapshotsBlobContainer, snapshotId.getSnapshot())) {
+            if (globalMetaDataFormat.exists(snapshotsBlobContainer, snapshotId.getName())) {
                 snapshotVersion = Version.CURRENT;
-            } else if (globalMetaDataLegacyFormat.exists(snapshotsBlobContainer, snapshotId.getSnapshot())) {
-                throw new SnapshotException(snapshotId, "snapshot is too old");
+            } else if (globalMetaDataLegacyFormat.exists(snapshotsBlobContainer, snapshotId.getName())) {
+                throw new SnapshotException(repositoryName, snapshotId, "snapshot is too old");
             } else {
-                throw new SnapshotMissingException(snapshotId);
+                throw new SnapshotMissingException(repositoryName, snapshotId);
             }
         }
         try {
-            metaData = globalMetaDataFormat(snapshotVersion).read(snapshotsBlobContainer, snapshotId.getSnapshot());
+            metaData = globalMetaDataFormat(snapshotVersion).read(snapshotsBlobContainer, snapshotId.getName());
         } catch (FileNotFoundException | NoSuchFileException ex) {
-            throw new SnapshotMissingException(snapshotId, ex);
+            throw new SnapshotMissingException(repositoryName, snapshotId, ex);
         } catch (IOException ex) {
-            throw new SnapshotException(snapshotId, "failed to get snapshots", ex);
+            throw new SnapshotException(repositoryName, snapshotId, "failed to get snapshots", ex);
         }
         MetaData.Builder metaDataBuilder = MetaData.builder(metaData);
         for (String index : indices) {
             BlobPath indexPath = basePath().add("indices").add(index);
             BlobContainer indexMetaDataBlobContainer = blobStore().blobContainer(indexPath);
             try {
-                metaDataBuilder.put(indexMetaDataFormat(snapshotVersion).read(indexMetaDataBlobContainer, snapshotId.getSnapshot()), false);
+                metaDataBuilder.put(indexMetaDataFormat(snapshotVersion).read(indexMetaDataBlobContainer, snapshotId.getName()), false);
             } catch (ElasticsearchParseException | IOException ex) {
                 if (ignoreIndexErrors) {
                     logger.warn("[{}] [{}] failed to read metadata for index", ex, snapshotId, index);
@@ -547,6 +560,10 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
         }
     }
 
+    private static final String SNAPSHOTS = "snapshots";
+    private static final String NAME = "name";
+    private static final String UUID = "uuid";
+
     /**
      * Writes snapshot index file
      * <p>
@@ -561,9 +578,12 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
             try(StreamOutput stream = new OutputStreamStreamOutput(bStream)) {
                 XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream);
                 builder.startObject();
-                builder.startArray("snapshots");
+                builder.startArray(SNAPSHOTS);
                 for (SnapshotId snapshot : snapshots) {
-                    builder.value(snapshot.getSnapshot());
+                    builder.startObject();
+                    builder.field(NAME, snapshot.getName());
+                    builder.field(UUID, snapshot.getUUID());
+                    builder.endObject();
                 }
                 builder.endArray();
                 builder.endObject();
@@ -594,10 +614,29 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
                 if (parser.nextToken() == XContentParser.Token.START_OBJECT) {
                     if (parser.nextToken() == XContentParser.Token.FIELD_NAME) {
                         String currentFieldName = parser.currentName();
-                        if ("snapshots".equals(currentFieldName)) {
+                        if (SNAPSHOTS.equals(currentFieldName)) {
                             if (parser.nextToken() == XContentParser.Token.START_ARRAY) {
                                 while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
-                                    snapshots.add(new SnapshotId(repositoryName, parser.text()));
+                                    // the new format from 5.0 which contains the snapshot name and uuid
+                                    String name = null;
+                                    String uuid = null;
+                                    if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
+                                        while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+                                            currentFieldName = parser.currentName();
+                                            parser.nextToken();
+                                            if (NAME.equals(currentFieldName)) {
+                                                name = parser.text();
+                                            } else if (UUID.equals(currentFieldName)) {
+                                                uuid = parser.text();
+                                            }
+                                        }
+                                        snapshots.add(new SnapshotId(name, uuid));
+                                    }
+                                    // the old format pre 5.0 that only contains the snapshot name, use the name as the uuid too
+                                    else {
+                                        name = parser.text();
+                                        snapshots.add(new SnapshotId(name, SnapshotId.UNASSIGNED_UUID));
+                                    }
                                 }
                             }
                         }
@@ -669,4 +708,42 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
     public boolean readOnly() {
         return readOnly;
     }
+
+    // package private, only use for testing
+    BlobContainer blobContainer() {
+        return snapshotsBlobContainer;
+    }
+
+    // TODO: this will go away once readSnapshotsList uses the index file instead of listing blobs
+    // to know which snapshots are part of a repository.  See #18156
+    // Package private for testing.
+    static Tuple<String, String> parseNameUUIDFromBlobName(final String str) {
+        final String name;
+        final String uuid;
+        final int sizeOfUUID = 22; // uuid is 22 chars in length
+        // unreliable, but highly unlikely to have a snapshot name with a dash followed by 22 characters,
+        // and this will go away before a release (see #18156).
+        //norelease
+        if (str.length() > sizeOfUUID + 1 && str.charAt(str.length() - sizeOfUUID - 1) == '-') {
+            // new naming convention, snapshot blob id has name and uuid
+            final int idx = str.length() - sizeOfUUID - 1;
+            name = str.substring(0, idx);
+            uuid = str.substring(idx + 1);
+        } else {
+            // old naming convention, before snapshots had UUIDs
+            name = str;
+            uuid = SnapshotId.UNASSIGNED_UUID;
+        }
+        return Tuple.tuple(name, uuid);
+    }
+
+    // Package private for testing
+    static String blobId(final SnapshotId snapshotId) {
+        final String uuid = snapshotId.getUUID();
+        if (uuid.equals(SnapshotId.UNASSIGNED_UUID)) {
+            // the old snapshot blob naming
+            return snapshotId.getName();
+        }
+        return snapshotId.getName() + "-" + uuid;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java b/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
index 77d4f1cc816..baea7fe28f9 100644
--- a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.repositories.uri;
 
-import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.blobstore.url.URLBlobStore;
@@ -42,6 +42,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.function.Function;
+import java.util.function.Predicate;
 
 /**
  * Read-only URL-based implementation of the BlobStoreRepository
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java
index 4764462d958..07c2611f2ce 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java
@@ -154,8 +154,8 @@ public class RestRecoveryAction extends AbstractCatAction {
                 t.addCell(state.getSourceNode() == null ? "n/a" : state.getSourceNode().getName());
                 t.addCell(state.getTargetNode().getHostName());
                 t.addCell(state.getTargetNode().getName());
-                t.addCell(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshotId().getRepository());
-                t.addCell(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshotId().getSnapshot());
+                t.addCell(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshot().getRepository());
+                t.addCell(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshot().getSnapshotId().getName());
                 t.addCell(state.getIndex().totalRecoverFiles());
                 t.addCell(state.getIndex().recoveredFileCount());
                 t.addCell(String.format(Locale.ROOT, "%1.1f%%", state.getIndex().recoveredFilesPercent()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java
index 0d98dd20278..94d178e4db9 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java
@@ -79,7 +79,7 @@ public class RestSnapshotAction extends AbstractCatAction {
     protected Table getTableWithHeader(RestRequest request) {
         return new Table()
                 .startHeaders()
-                .addCell("id", "alias:id,snapshotId;desc:unique snapshot id")
+                .addCell("id", "alias:id,snapshot;desc:unique snapshot")
                 .addCell("status", "alias:s,status;text-align:right;desc:snapshot name")
                 .addCell("start_epoch", "alias:ste,startEpoch;desc:start time in seconds since 1970-01-01 00:00:00")
                 .addCell("start_time", "alias:sti,startTime;desc:start time in HH:MM:SS")
@@ -101,7 +101,7 @@ public class RestSnapshotAction extends AbstractCatAction {
         for (SnapshotInfo snapshotStatus : getSnapshotsResponse.getSnapshots()) {
             table.startRow();
 
-            table.addCell(snapshotStatus.name());
+            table.addCell(snapshotStatus.snapshotId().getName());
             table.addCell(snapshotStatus.state());
             table.addCell(TimeUnit.SECONDS.convert(snapshotStatus.startTime(), TimeUnit.MILLISECONDS));
             table.addCell(dateFormat.print(snapshotStatus.startTime()));
diff --git a/core/src/main/java/org/elasticsearch/snapshots/ConcurrentSnapshotExecutionException.java b/core/src/main/java/org/elasticsearch/snapshots/ConcurrentSnapshotExecutionException.java
index 4a844fb4907..73762e2d3a7 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/ConcurrentSnapshotExecutionException.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/ConcurrentSnapshotExecutionException.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.snapshots;
 
-import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.rest.RestStatus;
 
@@ -29,7 +28,12 @@ import java.io.IOException;
  * Thrown when a user tries to start multiple snapshots at the same time
  */
 public class ConcurrentSnapshotExecutionException extends SnapshotException {
-    public ConcurrentSnapshotExecutionException(SnapshotId snapshot, String msg) {
+
+    public ConcurrentSnapshotExecutionException(final String repositoryName, final String snapshotName, final String msg) {
+        super(repositoryName, snapshotName, msg);
+    }
+
+    public ConcurrentSnapshotExecutionException(final Snapshot snapshot, final String msg) {
         super(snapshot, msg);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/snapshots/InvalidSnapshotNameException.java b/core/src/main/java/org/elasticsearch/snapshots/InvalidSnapshotNameException.java
index 48949c11d4c..2acc82add36 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/InvalidSnapshotNameException.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/InvalidSnapshotNameException.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.snapshots;
 
-import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.rest.RestStatus;
 
@@ -30,8 +29,8 @@ import java.io.IOException;
  */
 public class InvalidSnapshotNameException extends SnapshotException {
 
-    public InvalidSnapshotNameException(SnapshotId snapshot, String desc) {
-        super(snapshot, "Invalid snapshot name [" + snapshot.getSnapshot() + "], " + desc);
+    public InvalidSnapshotNameException(final String repositoryName, final String snapshotName, String desc) {
+        super(repositoryName, snapshotName, "Invalid snapshot name [" + snapshotName + "], " + desc);
     }
 
     public InvalidSnapshotNameException(StreamInput in) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
index f6e6c4aaed5..44ca6fb972e 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
@@ -39,7 +39,6 @@ import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
 import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
 import org.elasticsearch.cluster.metadata.RepositoriesMetaData;
-import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
@@ -85,6 +84,8 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.CopyOnWriteArrayList;
@@ -116,7 +117,7 @@ import static org.elasticsearch.common.util.set.Sets.newHashSet;
  * method, which detects that shard should be restored from snapshot rather than recovered from gateway by looking
  * at the {@link org.elasticsearch.cluster.routing.ShardRouting#restoreSource()} property.
  * <p>
- * At the end of the successful restore process {@code IndexShardSnapshotAndRestoreService} calls {@link #indexShardRestoreCompleted(SnapshotId, ShardId)},
+ * At the end of the successful restore process {@code IndexShardSnapshotAndRestoreService} calls {@link #indexShardRestoreCompleted(Snapshot, ShardId)},
  * which updates {@link RestoreInProgress} in cluster state or removes it when all shards are completed. In case of
  * restore failure a normal recovery fail-over process kicks in.
  */
@@ -153,8 +154,6 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
 
     private final MetaDataCreateIndexService createIndexService;
 
-    private final ClusterSettings dynamicSettings;
-
     private final MetaDataIndexUpgradeService metaDataIndexUpgradeService;
 
     private final CopyOnWriteArrayList<ActionListener<RestoreCompletionResponse>> listeners = new CopyOnWriteArrayList<>();
@@ -164,7 +163,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
 
     @Inject
     public RestoreService(Settings settings, ClusterService clusterService, RepositoriesService repositoriesService, TransportService transportService,
-                          AllocationService allocationService, MetaDataCreateIndexService createIndexService, ClusterSettings dynamicSettings,
+                          AllocationService allocationService, MetaDataCreateIndexService createIndexService,
                           MetaDataIndexUpgradeService metaDataIndexUpgradeService, ClusterSettings clusterSettings) {
         super(settings);
         this.clusterService = clusterService;
@@ -172,7 +171,6 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
         this.transportService = transportService;
         this.allocationService = allocationService;
         this.createIndexService = createIndexService;
-        this.dynamicSettings = dynamicSettings;
         this.metaDataIndexUpgradeService = metaDataIndexUpgradeService;
         transportService.registerRequestHandler(UPDATE_RESTORE_ACTION_NAME, UpdateIndexShardRestoreStatusRequest::new, ThreadPool.Names.SAME, new UpdateRestoreStateRequestHandler());
         clusterService.add(this);
@@ -188,14 +186,20 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
     public void restoreSnapshot(final RestoreRequest request, final ActionListener<RestoreInfo> listener) {
         try {
             // Read snapshot info and metadata from the repository
-            Repository repository = repositoriesService.repository(request.repository());
-            final SnapshotId snapshotId = new SnapshotId(request.repository(), request.name());
-            final SnapshotInfo snapshot = repository.readSnapshot(snapshotId);
-            List<String> filteredIndices = SnapshotUtils.filterIndices(snapshot.indices(), request.indices(), request.indicesOptions());
-            MetaData metaDataIn = repository.readSnapshotMetaData(snapshotId, snapshot, filteredIndices);
+            Repository repository = repositoriesService.repository(request.repositoryName);
+            final Optional<SnapshotId> matchingSnapshotId = repository.snapshots().stream()
+                .filter(s -> request.snapshotName.equals(s.getName())).findFirst();
+            if (matchingSnapshotId.isPresent() == false) {
+                throw new SnapshotRestoreException(request.repositoryName, request.snapshotName, "snapshot does not exist");
+            }
+            final SnapshotId snapshotId = matchingSnapshotId.get();
+            final SnapshotInfo snapshotInfo = repository.readSnapshot(snapshotId);
+            final Snapshot snapshot = new Snapshot(request.repositoryName, snapshotId);
+            List<String> filteredIndices = SnapshotUtils.filterIndices(snapshotInfo.indices(), request.indices(), request.indicesOptions());
+            MetaData metaDataIn = repository.readSnapshotMetaData(snapshotInfo, filteredIndices);
 
             final MetaData metaData;
-            if (snapshot.version().before(Version.V_2_0_0_beta1)) {
+            if (snapshotInfo.version().before(Version.V_2_0_0_beta1)) {
                 // ES 2.0 now requires units for all time and byte-sized settings, so we add the default unit if it's missing in this snapshot:
                 metaData = MetaData.addDefaultUnitsIfNeeded(logger, metaDataIn);
             } else {
@@ -204,7 +208,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
             }
 
             // Make sure that we can restore from this snapshot
-            validateSnapshotRestorable(snapshotId, snapshot);
+            validateSnapshotRestorable(request.repositoryName, snapshotInfo);
 
             // Find list of indices that we need to restore
             final Map<String, String> renamedIndices = renamedIndices(request, filteredIndices);
@@ -220,7 +224,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
                     // same time
                     RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE);
                     if (restoreInProgress != null && !restoreInProgress.entries().isEmpty()) {
-                        throw new ConcurrentSnapshotExecutionException(snapshotId, "Restore process is already running in this cluster");
+                        throw new ConcurrentSnapshotExecutionException(snapshot, "Restore process is already running in this cluster");
                     }
 
                     // Updating cluster state
@@ -236,14 +240,14 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
                         for (Map.Entry<String, String> indexEntry : renamedIndices.entrySet()) {
                             String index = indexEntry.getValue();
                             boolean partial = checkPartial(index);
-                            RestoreSource restoreSource = new RestoreSource(snapshotId, snapshot.version(), index);
+                            RestoreSource restoreSource = new RestoreSource(snapshot, snapshotInfo.version(), index);
                             String renamedIndexName = indexEntry.getKey();
                             IndexMetaData snapshotIndexMetaData = metaData.index(index);
                             snapshotIndexMetaData = updateIndexSettings(snapshotIndexMetaData, request.indexSettings, request.ignoreIndexSettings);
                             try {
                                 snapshotIndexMetaData = metaDataIndexUpgradeService.upgradeIndexMetaData(snapshotIndexMetaData);
                             } catch (Exception ex) {
-                                throw new SnapshotRestoreException(snapshotId, "cannot restore index [" + index + "] because it cannot be upgraded", ex);
+                                throw new SnapshotRestoreException(snapshot, "cannot restore index [" + index + "] because it cannot be upgraded", ex);
                             }
                             // Check that the index is closed or doesn't exist
                             IndexMetaData currentIndexMetaData = currentState.metaData().index(renamedIndexName);
@@ -309,7 +313,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
                         }
 
                         shards = shardsBuilder.build();
-                        RestoreInProgress.Entry restoreEntry = new RestoreInProgress.Entry(snapshotId, RestoreInProgress.State.INIT, Collections.unmodifiableList(new ArrayList<>(renamedIndices.keySet())), shards);
+                        RestoreInProgress.Entry restoreEntry = new RestoreInProgress.Entry(snapshot, RestoreInProgress.State.INIT, Collections.unmodifiableList(new ArrayList<>(renamedIndices.keySet())), shards);
                         builder.putCustom(RestoreInProgress.TYPE, new RestoreInProgress(restoreEntry));
                     } else {
                         shards = ImmutableOpenMap.of();
@@ -322,28 +326,30 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
 
                     if (completed(shards)) {
                         // We don't have any indices to restore - we are done
-                        restoreInfo = new RestoreInfo(request.name(), Collections.unmodifiableList(new ArrayList<>(renamedIndices.keySet())),
-                                shards.size(), shards.size() - failedShards(shards));
+                        restoreInfo = new RestoreInfo(snapshotId.getName(),
+                                                      Collections.unmodifiableList(new ArrayList<>(renamedIndices.keySet())),
+                                                      shards.size(),
+                                                      shards.size() - failedShards(shards));
                     }
 
                     RoutingTable rt = rtBuilder.build();
                     ClusterState updatedState = builder.metaData(mdBuilder).blocks(blocks).routingTable(rt).build();
                     RoutingAllocation.Result routingResult = allocationService.reroute(
                             ClusterState.builder(updatedState).routingTable(rt).build(),
-                            "restored snapshot [" + snapshotId + "]");
+                            "restored snapshot [" + snapshot + "]");
                     return ClusterState.builder(updatedState).routingResult(routingResult).build();
                 }
 
                 private void checkAliasNameConflicts(Map<String, String> renamedIndices, Set<String> aliases) {
                     for (Map.Entry<String, String> renamedIndex : renamedIndices.entrySet()) {
                         if (aliases.contains(renamedIndex.getKey())) {
-                            throw new SnapshotRestoreException(snapshotId, "cannot rename index [" + renamedIndex.getValue() + "] into [" + renamedIndex.getKey() + "] because of conflict with an alias with the same name");
+                            throw new SnapshotRestoreException(snapshot, "cannot rename index [" + renamedIndex.getValue() + "] into [" + renamedIndex.getKey() + "] because of conflict with an alias with the same name");
                         }
                     }
                 }
 
                 private void populateIgnoredShards(String index, IntSet ignoreShards) {
-                    for (SnapshotShardFailure failure : snapshot.shardFailures()) {
+                    for (SnapshotShardFailure failure : snapshotInfo.shardFailures()) {
                         if (index.equals(failure.index())) {
                             ignoreShards.add(failure.shardId());
                         }
@@ -352,11 +358,11 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
 
                 private boolean checkPartial(String index) {
                     // Make sure that index was fully snapshotted
-                    if (failed(snapshot, index)) {
+                    if (failed(snapshotInfo, index)) {
                         if (request.partial()) {
                             return true;
                         } else {
-                            throw new SnapshotRestoreException(snapshotId, "index [" + index + "] wasn't fully snapshotted - cannot restore");
+                            throw new SnapshotRestoreException(snapshot, "index [" + index + "] wasn't fully snapshotted - cannot restore");
                         }
                     } else {
                         return false;
@@ -367,15 +373,15 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
                     // Index exist - checking that it's closed
                     if (currentIndexMetaData.getState() != IndexMetaData.State.CLOSE) {
                         // TODO: Enable restore for open indices
-                        throw new SnapshotRestoreException(snapshotId, "cannot restore index [" + renamedIndex + "] because it's open");
+                        throw new SnapshotRestoreException(snapshot, "cannot restore index [" + renamedIndex + "] because it's open");
                     }
                     // Index exist - checking if it's partial restore
                     if (partial) {
-                        throw new SnapshotRestoreException(snapshotId, "cannot restore partial index [" + renamedIndex + "] because such index already exists");
+                        throw new SnapshotRestoreException(snapshot, "cannot restore partial index [" + renamedIndex + "] because such index already exists");
                     }
                     // Make sure that the number of shards is the same. That's the only thing that we cannot change
                     if (currentIndexMetaData.getNumberOfShards() != snapshotIndexMetaData.getNumberOfShards()) {
-                        throw new SnapshotRestoreException(snapshotId, "cannot restore index [" + renamedIndex + "] with [" + currentIndexMetaData.getNumberOfShards() +
+                        throw new SnapshotRestoreException(snapshot, "cannot restore index [" + renamedIndex + "] with [" + currentIndexMetaData.getNumberOfShards() +
                                 "] shard from snapshot with [" + snapshotIndexMetaData.getNumberOfShards() + "] shards");
                     }
                 }
@@ -395,7 +401,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
                     for (String ignoredSetting : ignoreSettings) {
                         if (!Regex.isSimpleMatchPattern(ignoredSetting)) {
                             if (UNREMOVABLE_SETTINGS.contains(ignoredSetting)) {
-                                throw new SnapshotRestoreException(snapshotId, "cannot remove setting [" + ignoredSetting + "] on restore");
+                                throw new SnapshotRestoreException(snapshot, "cannot remove setting [" + ignoredSetting + "] on restore");
                             } else {
                                 settingsMap.remove(ignoredSetting);
                             }
@@ -417,7 +423,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
                     }
                     for(Map.Entry<String, String> entry : normalizedChangeSettings.getAsMap().entrySet()) {
                         if (UNMODIFIABLE_SETTINGS.contains(entry.getKey())) {
-                            throw new SnapshotRestoreException(snapshotId, "cannot modify setting [" + entry.getKey() + "] on restore");
+                            throw new SnapshotRestoreException(snapshot, "cannot modify setting [" + entry.getKey() + "] on restore");
                         } else {
                             settingsMap.put(entry.getKey(), entry.getValue());
                         }
@@ -471,7 +477,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
 
 
         } catch (Throwable e) {
-            logger.warn("[{}][{}] failed to restore snapshot", e, request.repository(), request.name());
+            logger.warn("[{}] failed to restore snapshot", e, request.repositoryName + ":" + request.snapshotName);
             listener.onFailure(e);
         }
     }
@@ -480,28 +486,28 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
      * This method is used by {@link IndexShard} to notify
      * {@code RestoreService} about shard restore completion.
      *
-     * @param snapshotId snapshot id
+     * @param snapshot   snapshot
      * @param shardId    shard id
      */
-    public void indexShardRestoreCompleted(SnapshotId snapshotId, ShardId shardId) {
-        logger.trace("[{}] successfully restored shard  [{}]", snapshotId, shardId);
-        UpdateIndexShardRestoreStatusRequest request = new UpdateIndexShardRestoreStatusRequest(snapshotId, shardId,
+    public void indexShardRestoreCompleted(Snapshot snapshot, ShardId shardId) {
+        logger.trace("[{}] successfully restored shard  [{}]", snapshot, shardId);
+        UpdateIndexShardRestoreStatusRequest request = new UpdateIndexShardRestoreStatusRequest(snapshot, shardId,
                 new ShardRestoreStatus(clusterService.state().nodes().getLocalNodeId(), RestoreInProgress.State.SUCCESS));
-            transportService.sendRequest(clusterService.state().nodes().getMasterNode(),
-                    UPDATE_RESTORE_ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME);
+        transportService.sendRequest(clusterService.state().nodes().getMasterNode(),
+                UPDATE_RESTORE_ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME);
     }
 
     public final static class RestoreCompletionResponse {
-        private final SnapshotId snapshotId;
+        private final Snapshot snapshot;
         private final RestoreInfo restoreInfo;
 
-        private RestoreCompletionResponse(SnapshotId snapshotId, RestoreInfo restoreInfo) {
-            this.snapshotId = snapshotId;
+        private RestoreCompletionResponse(final Snapshot snapshot, final RestoreInfo restoreInfo) {
+            this.snapshot = snapshot;
             this.restoreInfo = restoreInfo;
         }
 
-        public SnapshotId getSnapshotId() {
-            return snapshotId;
+        public Snapshot getSnapshot() {
+            return snapshot;
         }
 
         public RestoreInfo getRestoreInfo() {
@@ -520,7 +526,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
 
         clusterService.submitStateUpdateTask("update snapshot state", new ClusterStateUpdateTask() {
             private final List<UpdateIndexShardRestoreStatusRequest> drainedRequests = new ArrayList<>();
-            private Map<SnapshotId, Tuple<RestoreInfo, ImmutableOpenMap<ShardId, ShardRestoreStatus>>> batchedRestoreInfo = null;
+            private Map<Snapshot, Tuple<RestoreInfo, ImmutableOpenMap<ShardId, ShardRestoreStatus>>> batchedRestoreInfo = null;
 
             @Override
             public ClusterState execute(ClusterState currentState) {
@@ -549,8 +555,8 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
                             final UpdateIndexShardRestoreStatusRequest updateSnapshotState = drainedRequests.get(i);
                             updateSnapshotState.processed = true;
 
-                            if (entry.snapshotId().equals(updateSnapshotState.snapshotId())) {
-                                logger.trace("[{}] Updating shard [{}] with status [{}]", updateSnapshotState.snapshotId(), updateSnapshotState.shardId(), updateSnapshotState.status().state());
+                            if (entry.snapshot().equals(updateSnapshotState.snapshot())) {
+                                logger.trace("[{}] Updating shard [{}] with status [{}]", updateSnapshotState.snapshot(), updateSnapshotState.shardId(), updateSnapshotState.status().state());
                                 if (shardsBuilder == null) {
                                     shardsBuilder = ImmutableOpenMap.builder(entry.shards());
                                 }
@@ -562,16 +568,19 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
                         if (shardsBuilder != null) {
                             ImmutableOpenMap<ShardId, ShardRestoreStatus> shards = shardsBuilder.build();
                             if (!completed(shards)) {
-                                entries.add(new RestoreInProgress.Entry(entry.snapshotId(), RestoreInProgress.State.STARTED, entry.indices(), shards));
+                                entries.add(new RestoreInProgress.Entry(entry.snapshot(), RestoreInProgress.State.STARTED, entry.indices(), shards));
                             } else {
-                                logger.info("restore [{}] is done", entry.snapshotId());
+                                logger.info("restore [{}] is done", entry.snapshot());
                                 if (batchedRestoreInfo == null) {
                                     batchedRestoreInfo = new HashMap<>();
                                 }
-                                assert !batchedRestoreInfo.containsKey(entry.snapshotId());
-                                batchedRestoreInfo.put(entry.snapshotId(),
+                                assert !batchedRestoreInfo.containsKey(entry.snapshot());
+                                batchedRestoreInfo.put(entry.snapshot(),
                                     new Tuple<>(
-                                        new RestoreInfo(entry.snapshotId().getSnapshot(), entry.indices(), shards.size(), shards.size() - failedShards(shards)),
+                                        new RestoreInfo(entry.snapshot().getSnapshotId().getName(),
+                                                        entry.indices(),
+                                                        shards.size(),
+                                                        shards.size() - failedShards(shards)),
                                         shards));
                             }
                         } else {
@@ -592,15 +601,15 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
             @Override
             public void onFailure(String source, @Nullable Throwable t) {
                 for (UpdateIndexShardRestoreStatusRequest request : drainedRequests) {
-                    logger.warn("[{}][{}] failed to update snapshot status to [{}]", t, request.snapshotId(), request.shardId(), request.status());
+                    logger.warn("[{}][{}] failed to update snapshot status to [{}]", t, request.snapshot(), request.shardId(), request.status());
                 }
             }
 
             @Override
             public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                 if (batchedRestoreInfo != null) {
-                    for (final Entry<SnapshotId, Tuple<RestoreInfo, ImmutableOpenMap<ShardId, ShardRestoreStatus>>> entry : batchedRestoreInfo.entrySet()) {
-                        final SnapshotId snapshotId = entry.getKey();
+                    for (final Entry<Snapshot, Tuple<RestoreInfo, ImmutableOpenMap<ShardId, ShardRestoreStatus>>> entry : batchedRestoreInfo.entrySet()) {
+                        final Snapshot snapshot = entry.getKey();
                         final RestoreInfo restoreInfo = entry.getValue().v1();
                         final ImmutableOpenMap<ShardId, ShardRestoreStatus> shards = entry.getValue().v2();
                         RoutingTable routingTable = newState.getRoutingTable();
@@ -610,13 +619,13 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
                                 ShardId shardId = shard.key;
                                 ShardRouting shardRouting = findPrimaryShard(routingTable, shardId);
                                 if (shardRouting != null && !shardRouting.active()) {
-                                    logger.trace("[{}][{}] waiting for the shard to start", snapshotId, shardId);
+                                    logger.trace("[{}][{}] waiting for the shard to start", snapshot, shardId);
                                     waitForStarted.add(shardId);
                                 }
                             }
                         }
                         if (waitForStarted.isEmpty()) {
-                            notifyListeners(snapshotId, restoreInfo);
+                            notifyListeners(snapshot, restoreInfo);
                         } else {
                             clusterService.addLast(new ClusterStateListener() {
                                 @Override
@@ -629,12 +638,12 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
                                             // Shard disappeared (index deleted) or became active
                                             if (shardRouting == null || shardRouting.active()) {
                                                 iterator.remove();
-                                                logger.trace("[{}][{}] shard disappeared or started - removing", snapshotId, shardId);
+                                                logger.trace("[{}][{}] shard disappeared or started - removing", snapshot, shardId);
                                             }
                                         }
                                     }
                                     if (waitForStarted.isEmpty()) {
-                                        notifyListeners(snapshotId, restoreInfo);
+                                        notifyListeners(snapshot, restoreInfo);
                                         clusterService.remove(this);
                                     }
                                 }
@@ -655,10 +664,10 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
                 return null;
             }
 
-            private void notifyListeners(SnapshotId snapshotId, RestoreInfo restoreInfo) {
+            private void notifyListeners(Snapshot snapshot, RestoreInfo restoreInfo) {
                 for (ActionListener<RestoreCompletionResponse> listener : listeners) {
                     try {
-                        listener.onResponse(new RestoreCompletionResponse(snapshotId, restoreInfo));
+                        listener.onResponse(new RestoreCompletionResponse(snapshot, restoreInfo));
                     } catch (Throwable e) {
                         logger.warn("failed to update snapshot status for [{}]", e, listener);
                     }
@@ -695,7 +704,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
             }
             String previousIndex = renamedIndices.put(renamedIndex, index);
             if (previousIndex != null) {
-                throw new SnapshotRestoreException(new SnapshotId(request.repository(), request.name()),
+                throw new SnapshotRestoreException(request.repositoryName, request.snapshotName,
                         "indices [" + index + "] and [" + previousIndex + "] are renamed into the same index [" + renamedIndex + "]");
             }
         }
@@ -705,16 +714,18 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
     /**
      * Checks that snapshots can be restored and have compatible version
      *
-     * @param snapshotId snapshot id
-     * @param snapshot   snapshot metadata
+     * @param repository      repository name
+     * @param snapshotInfo    snapshot metadata
      */
-    private void validateSnapshotRestorable(SnapshotId snapshotId, SnapshotInfo snapshot) {
-        if (!snapshot.state().restorable()) {
-            throw new SnapshotRestoreException(snapshotId, "unsupported snapshot state [" + snapshot.state() + "]");
+    private void validateSnapshotRestorable(final String repository, final SnapshotInfo snapshotInfo) {
+        if (!snapshotInfo.state().restorable()) {
+            throw new SnapshotRestoreException(new Snapshot(repository, snapshotInfo.snapshotId()),
+                                               "unsupported snapshot state [" + snapshotInfo.state() + "]");
         }
-        if (Version.CURRENT.before(snapshot.version())) {
-            throw new SnapshotRestoreException(snapshotId, "the snapshot was created with Elasticsearch version [" +
-                    snapshot.version() + "] which is higher than the version of this node [" + Version.CURRENT + "]");
+        if (Version.CURRENT.before(snapshotInfo.version())) {
+            throw new SnapshotRestoreException(new Snapshot(repository, snapshotInfo.snapshotId()),
+                                               "the snapshot was created with Elasticsearch version [" + snapshotInfo.version() +
+                                                   "] which is higher than the version of this node [" + Version.CURRENT + "]");
         }
     }
 
@@ -746,8 +757,8 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
                 }
                 if (shardsToFail != null) {
                     for (ShardId shardId : shardsToFail) {
-                        logger.trace("[{}] failing running shard restore [{}]", entry.snapshotId(), shardId);
-                        updateRestoreStateOnMaster(new UpdateIndexShardRestoreStatusRequest(entry.snapshotId(), shardId, new ShardRestoreStatus(null, RestoreInProgress.State.FAILURE, "index was deleted")));
+                        logger.trace("[{}] failing running shard restore [{}]", entry.snapshot(), shardId);
+                        updateRestoreStateOnMaster(new UpdateIndexShardRestoreStatusRequest(entry.snapshot(), shardId, new ShardRestoreStatus(null, RestoreInProgress.State.FAILURE, "index was deleted")));
                     }
                 }
             }
@@ -757,12 +768,12 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
     /**
      * Fails the given snapshot restore operation for the given shard
      */
-    public void failRestore(SnapshotId snapshotId, ShardId shardId) {
-        logger.debug("[{}] failed to restore shard  [{}]", snapshotId, shardId);
-        UpdateIndexShardRestoreStatusRequest request = new UpdateIndexShardRestoreStatusRequest(snapshotId, shardId,
+    public void failRestore(Snapshot snapshot, ShardId shardId) {
+        logger.debug("[{}] failed to restore shard  [{}]", snapshot, shardId);
+        UpdateIndexShardRestoreStatusRequest request = new UpdateIndexShardRestoreStatusRequest(snapshot, shardId,
                 new ShardRestoreStatus(clusterService.state().nodes().getLocalNodeId(), RestoreInProgress.State.FAILURE));
-            transportService.sendRequest(clusterService.state().nodes().getMasterNode(),
-                    UPDATE_RESTORE_ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME);
+        transportService.sendRequest(clusterService.state().nodes().getMasterNode(),
+                                     UPDATE_RESTORE_ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME);
     }
 
     private boolean failed(SnapshotInfo snapshot, String index) {
@@ -846,7 +857,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
         RestoreInProgress snapshots = clusterState.custom(RestoreInProgress.TYPE);
         if (snapshots != null) {
             for (RestoreInProgress.Entry snapshot : snapshots.entries()) {
-                if (repository.equals(snapshot.snapshotId().getRepository())) {
+                if (repository.equals(snapshot.snapshot().getRepository())) {
                     return true;
                 }
             }
@@ -861,9 +872,9 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
 
         final private String cause;
 
-        final private String name;
+        final private String repositoryName;
 
-        final private String repository;
+        final private String snapshotName;
 
         final private String[] indices;
 
@@ -890,9 +901,8 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
         /**
          * Constructs new restore request
          *
-         * @param cause              cause for restoring the snapshot
-         * @param repository         repository name
-         * @param name               snapshot name
+         * @param repositoryName     repositoryName
+         * @param snapshotName       snapshotName
          * @param indices            list of indices to restore
          * @param indicesOptions     indices options
          * @param renamePattern      pattern to rename indices
@@ -903,14 +913,14 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
          * @param partial            allow partial restore
          * @param indexSettings      index settings that should be changed on restore
          * @param ignoreIndexSettings index settings that shouldn't be restored
+         * @param cause              cause for restoring the snapshot
          */
-        public RestoreRequest(String cause, String repository, String name, String[] indices, IndicesOptions indicesOptions,
+        public RestoreRequest(String repositoryName, String snapshotName, String[] indices, IndicesOptions indicesOptions,
                               String renamePattern, String renameReplacement, Settings settings,
                               TimeValue masterNodeTimeout, boolean includeGlobalState, boolean partial, boolean includeAliases,
-                              Settings indexSettings, String[] ignoreIndexSettings ) {
-            this.cause = cause;
-            this.name = name;
-            this.repository = repository;
+                              Settings indexSettings, String[] ignoreIndexSettings, String cause) {
+            this.repositoryName = Objects.requireNonNull(repositoryName);
+            this.snapshotName = Objects.requireNonNull(snapshotName);
             this.indices = indices;
             this.renamePattern = renamePattern;
             this.renameReplacement = renameReplacement;
@@ -922,7 +932,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
             this.includeAliases = includeAliases;
             this.indexSettings = indexSettings;
             this.ignoreIndexSettings = ignoreIndexSettings;
-
+            this.cause = cause;
         }
 
         /**
@@ -934,22 +944,22 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
             return cause;
         }
 
-        /**
-         * Returns snapshot name
-         *
-         * @return snapshot name
-         */
-        public String name() {
-            return name;
-        }
-
         /**
          * Returns repository name
          *
          * @return repository name
          */
-        public String repository() {
-            return repository;
+        public String repositoryName() {
+            return repositoryName;
+        }
+
+        /**
+         * Returns snapshot name
+         *
+         * @return snapshot name
+         */
+        public String snapshotName() {
+            return snapshotName;
         }
 
         /**
@@ -1058,7 +1068,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
      * Internal class that is used to send notifications about finished shard restore operations to master node
      */
     public static class UpdateIndexShardRestoreStatusRequest extends TransportRequest {
-        private SnapshotId snapshotId;
+        private Snapshot snapshot;
         private ShardId shardId;
         private ShardRestoreStatus status;
 
@@ -1068,8 +1078,8 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
 
         }
 
-        private UpdateIndexShardRestoreStatusRequest(SnapshotId snapshotId, ShardId shardId, ShardRestoreStatus status) {
-            this.snapshotId = snapshotId;
+        private UpdateIndexShardRestoreStatusRequest(Snapshot snapshot, ShardId shardId, ShardRestoreStatus status) {
+            this.snapshot = snapshot;
             this.shardId = shardId;
             this.status = status;
         }
@@ -1077,7 +1087,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
         @Override
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
-            snapshotId = SnapshotId.readSnapshotId(in);
+            snapshot = new Snapshot(in);
             shardId = ShardId.readShardId(in);
             status = ShardRestoreStatus.readShardRestoreStatus(in);
         }
@@ -1085,13 +1095,13 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            snapshotId.writeTo(out);
+            snapshot.writeTo(out);
             shardId.writeTo(out);
             status.writeTo(out);
         }
 
-        public SnapshotId snapshotId() {
-            return snapshotId;
+        public Snapshot snapshot() {
+            return snapshot;
         }
 
         public ShardId shardId() {
@@ -1104,7 +1114,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
 
         @Override
         public String toString() {
-            return "" + snapshotId + ", shardId [" + shardId + "], status [" + status.state() + "]";
+            return "" + snapshot + ", shardId [" + shardId + "], status [" + status.state() + "]";
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/snapshots/Snapshot.java b/core/src/main/java/org/elasticsearch/snapshots/Snapshot.java
new file mode 100644
index 00000000000..314cd4053dd
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/snapshots/Snapshot.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.snapshots;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Basic information about a snapshot - a SnapshotId and the repository that the snapshot belongs to.
+ */
+public final class Snapshot implements Writeable {
+
+    private final String repository;
+    private final SnapshotId snapshotId;
+    private final int hashCode;
+
+    /**
+     * Constructs a snapshot.
+     */
+    public Snapshot(final String repository, final SnapshotId snapshotId) {
+        this.repository = Objects.requireNonNull(repository);
+        this.snapshotId = Objects.requireNonNull(snapshotId);
+        this.hashCode = computeHashCode();
+    }
+
+    /**
+     * Constructs a snapshot from the stream input.
+     */
+    public Snapshot(final StreamInput in) throws IOException {
+        repository = in.readString();
+        snapshotId = new SnapshotId(in);
+        hashCode = computeHashCode();
+    }
+
+    /**
+     * Gets the repository name for the snapshot.
+     */
+    public String getRepository() {
+        return repository;
+    }
+
+    /**
+     * Gets the snapshot id for the snapshot.
+     */
+    public SnapshotId getSnapshotId() {
+        return snapshotId;
+    }
+
+    @Override
+    public String toString() {
+        return repository + ":" + snapshotId.toString();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        @SuppressWarnings("unchecked") Snapshot that = (Snapshot) o;
+        return repository.equals(that.repository) && snapshotId.equals(that.snapshotId);
+    }
+
+    @Override
+    public int hashCode() {
+        return hashCode;
+    }
+
+    private int computeHashCode() {
+        return Objects.hash(repository, snapshotId);
+    }
+
+    @Override
+    public void writeTo(final StreamOutput out) throws IOException {
+        out.writeString(repository);
+        snapshotId.writeTo(out);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotCreationException.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotCreationException.java
index 58faecda4a4..32d3992f243 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotCreationException.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotCreationException.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.snapshots;
 
-import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.common.io.stream.StreamInput;
 
 import java.io.IOException;
@@ -29,11 +28,19 @@ import java.io.IOException;
  */
 public class SnapshotCreationException extends SnapshotException {
 
+    public SnapshotCreationException(final String repositoryName, final String snapshotName, final Throwable cause) {
+        super(repositoryName, snapshotName, "failed to create snapshot", cause);
+    }
+
+    public SnapshotCreationException(final String repositoryName, final SnapshotId snapshotId, final Throwable cause) {
+        super(repositoryName, snapshotId, "failed to create snapshot", cause);
+    }
+
+    public SnapshotCreationException(final String repositoryName, final SnapshotId snapshotId, final String msg) {
+        super(repositoryName, snapshotId, msg);
+    }
+
     public SnapshotCreationException(StreamInput in) throws IOException {
         super(in);
     }
-
-    public SnapshotCreationException(SnapshotId snapshot, Throwable cause) {
-        super(snapshot, "failed to create snapshot", cause);
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotException.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotException.java
index b109c46f46b..0acd73d62ea 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotException.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotException.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.snapshots;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 
@@ -30,33 +30,68 @@ import java.io.IOException;
  * Generic snapshot exception
  */
 public class SnapshotException extends ElasticsearchException {
-    private final SnapshotId snapshot;
 
-    public SnapshotException(SnapshotId snapshot, String msg) {
+    @Nullable
+    private final String repositoryName;
+    @Nullable
+    private final String snapshotName;
+
+    public SnapshotException(final Snapshot snapshot, final String msg) {
         this(snapshot, msg, null);
     }
 
-    public SnapshotException(SnapshotId snapshot, String msg, Throwable cause) {
+    public SnapshotException(final Snapshot snapshot, final String msg, final Throwable cause) {
         super("[" + (snapshot == null ? "_na" : snapshot) + "] " + msg, cause);
-        this.snapshot = snapshot;
+        if (snapshot != null) {
+            this.repositoryName = snapshot.getRepository();
+            this.snapshotName = snapshot.getSnapshotId().getName();
+        } else {
+            this.repositoryName = null;
+            this.snapshotName = null;
+        }
     }
 
-    public SnapshotException(StreamInput in) throws IOException {
+    public SnapshotException(final String repositoryName, final SnapshotId snapshotId, final String msg) {
+        this(repositoryName, snapshotId, msg, null);
+    }
+
+    public SnapshotException(final String repositoryName, final SnapshotId snapshotId, final String msg, final Throwable cause) {
+        super("[" + repositoryName + ":" + snapshotId + "] " + msg, cause);
+        this.repositoryName = repositoryName;
+        this.snapshotName = snapshotId.getName();
+    }
+
+    public SnapshotException(final String repositoryName, final String snapshotName, final String msg) {
+        this(repositoryName, snapshotName, msg, null);
+    }
+
+    public SnapshotException(final String repositoryName, final String snapshotName, final String msg, final Throwable cause) {
+        super("[" + repositoryName + ":" + snapshotName + "]" + msg, cause);
+        this.repositoryName = repositoryName;
+        this.snapshotName = snapshotName;
+    }
+
+    public SnapshotException(final StreamInput in) throws IOException {
         super(in);
-        if (in.readBoolean()) {
-            snapshot = SnapshotId.readSnapshotId(in);
-        } else {
-            snapshot = null;
-        }
+        repositoryName = in.readOptionalString();
+        snapshotName = in.readOptionalString();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeOptionalStreamable(snapshot);
+        out.writeOptionalString(repositoryName);
+        out.writeOptionalString(snapshotName);
     }
 
-    public SnapshotId snapshot() {
-        return snapshot;
+    @Nullable
+    public String getRepositoryName() {
+        return repositoryName;
     }
+
+    @Nullable
+    public String getSnapshotName() {
+        return snapshotName;
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotId.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotId.java
new file mode 100644
index 00000000000..16f371b28f7
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotId.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.snapshots;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * SnapshotId - snapshot name + snapshot UUID
+ */
+public final class SnapshotId implements Writeable {
+
+    /**
+     * This value is for older snapshots that don't have a UUID.
+     */
+    public static final String UNASSIGNED_UUID = "_na_";
+
+    private final String name;
+    private final String uuid;
+
+    // Caching hash code
+    private final int hashCode;
+
+    /**
+     * Constructs a new snapshot
+     *
+     * @param name   snapshot name
+     * @param uuid   snapshot uuid
+     */
+    public SnapshotId(final String name, final String uuid) {
+        this.name = Objects.requireNonNull(name);
+        this.uuid = Objects.requireNonNull(uuid);
+        this.hashCode = computeHashCode();
+    }
+
+    /**
+     * Constructs a new snapshot from a input stream
+     *
+     * @param in  input stream
+     */
+    public SnapshotId(final StreamInput in) throws IOException {
+        name = in.readString();
+        uuid = in.readString();
+        hashCode = computeHashCode();
+    }
+
+    /**
+     * Returns snapshot name
+     *
+     * @return snapshot name
+     */
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * Returns the snapshot UUID
+     *
+     * @return snapshot uuid
+     */
+    public String getUUID() {
+        return uuid;
+    }
+
+    @Override
+    public String toString() {
+        return name + "/" + uuid;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        @SuppressWarnings("unchecked") final SnapshotId that = (SnapshotId) o;
+        return name.equals(that.name) && uuid.equals(that.uuid);
+    }
+
+    @Override
+    public int hashCode() {
+        return hashCode;
+    }
+
+    private int computeHashCode() {
+        return Objects.hash(name, uuid);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(name);
+        out.writeString(uuid);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
index 2b8ea8ace31..2159fda2237 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java
@@ -37,17 +37,19 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * Information about a snapshot
  */
 public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent, FromXContentBuilder<SnapshotInfo>, Writeable {
 
-    public static final SnapshotInfo PROTO = new SnapshotInfo("", Collections.emptyList(), 0);
+    public static final SnapshotInfo PROTO = new SnapshotInfo(new SnapshotId("", ""), Collections.emptyList(), 0);
     public static final String CONTEXT_MODE_PARAM = "context_mode";
     public static final String CONTEXT_MODE_SNAPSHOT = "SNAPSHOT";
     private static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("strictDateOptionalTime");
     private static final String SNAPSHOT = "snapshot";
+    private static final String UUID = "uuid";
     private static final String INDICES = "indices";
     private static final String STATE = "state";
     private static final String REASON = "reason";
@@ -68,7 +70,7 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
     private static final String TOTAL_SHARDS = "total_shards";
     private static final String SUCCESSFUL_SHARDS = "successful_shards";
 
-    private final String name;
+    private final SnapshotId snapshotId;
 
     private final SnapshotState state;
 
@@ -88,39 +90,35 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
 
     private final List<SnapshotShardFailure> shardFailures;
 
-    public SnapshotInfo(String name, List<String> indices, long startTime) {
-        this(name, indices, SnapshotState.IN_PROGRESS, null, Version.CURRENT, startTime, 0L, 0, 0, Collections.emptyList());
+    public SnapshotInfo(SnapshotId snapshotId, List<String> indices, long startTime) {
+        this(snapshotId, indices, SnapshotState.IN_PROGRESS, null, Version.CURRENT, startTime, 0L, 0, 0, Collections.emptyList());
     }
 
-    public SnapshotInfo(String name, List<String> indices, long startTime, String reason, long endTime,
+    public SnapshotInfo(SnapshotId snapshotId, List<String> indices, long startTime, String reason, long endTime,
                         int totalShards, List<SnapshotShardFailure> shardFailures) {
-        this(name, indices, snapshotState(reason, shardFailures), reason, Version.CURRENT,
+        this(snapshotId, indices, snapshotState(reason, shardFailures), reason, Version.CURRENT,
              startTime, endTime, totalShards, totalShards - shardFailures.size(), shardFailures);
     }
 
-    private SnapshotInfo(String name, List<String> indices, SnapshotState state, String reason, Version version, long startTime,
-                         long endTime, int totalShards, int successfulShards, List<SnapshotShardFailure> shardFailures) {
-        assert name != null;
-        assert indices != null;
-        assert state != null;
-        assert shardFailures != null;
-        this.name = name;
-        this.indices = indices;
-        this.state = state;
+    private SnapshotInfo(SnapshotId snapshotId, List<String> indices, SnapshotState state, String reason, Version version,
+                         long startTime, long endTime, int totalShards, int successfulShards, List<SnapshotShardFailure> shardFailures) {
+        this.snapshotId = Objects.requireNonNull(snapshotId);
+        this.indices = Objects.requireNonNull(indices);
+        this.state = Objects.requireNonNull(state);
         this.reason = reason;
         this.version = version;
         this.startTime = startTime;
         this.endTime = endTime;
         this.totalShards = totalShards;
         this.successfulShards = successfulShards;
-        this.shardFailures = shardFailures;
+        this.shardFailures = Objects.requireNonNull(shardFailures);
     }
 
     /**
      * Constructs snapshot information from stream input
      */
     public SnapshotInfo(final StreamInput in) throws IOException {
-        name = in.readString();
+        snapshotId = new SnapshotId(in);
         int size = in.readVInt();
         List<String> indicesListBuilder = new ArrayList<>();
         for (int i = 0; i < size; i++) {
@@ -147,12 +145,12 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
     }
 
     /**
-     * Returns snapshot name
+     * Returns snapshot id
      *
-     * @return snapshot name
+     * @return snapshot id
      */
-    public String name() {
-        return name;
+    public SnapshotId snapshotId() {
+        return snapshotId;
     }
 
     /**
@@ -270,16 +268,21 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
         }
 
         final SnapshotInfo that = (SnapshotInfo) o;
-        return startTime == that.startTime && name.equals(that.name);
+        return startTime == that.startTime && snapshotId.equals(that.snapshotId);
     }
 
     @Override
     public int hashCode() {
-        int result = name.hashCode();
+        int result = snapshotId.hashCode();
         result = 31 * result + Long.hashCode(startTime);
         return result;
     }
 
+    @Override
+    public String toString() {
+        return "SnapshotInfo[snapshotId=" + snapshotId + ", state=" + state + ", indices=" + indices + "]";
+    }
+
     /**
      * Returns snapshot REST status
      */
@@ -303,7 +306,8 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
 
         // write snapshot info for the API and any other situations
         builder.startObject();
-        builder.field(SNAPSHOT, name);
+        builder.field(SNAPSHOT, snapshotId.getName());
+        builder.field(UUID, snapshotId.getUUID());
         builder.field(VERSION_ID, version.id);
         builder.field(VERSION, version.toString());
         builder.startArray(INDICES);
@@ -342,7 +346,8 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
 
     private XContentBuilder toXContentSnapshot(final XContentBuilder builder, final ToXContent.Params params) throws IOException {
         builder.startObject(SNAPSHOT);
-        builder.field(NAME, name);
+        builder.field(NAME, snapshotId.getName());
+        builder.field(UUID, snapshotId.getUUID());
         builder.field(VERSION_ID, version.id);
         builder.startArray(INDICES);
         for (String index : indices) {
@@ -380,13 +385,14 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
      */
     public static SnapshotInfo fromXContent(final XContentParser parser) throws IOException {
         String name = null;
+        String uuid = null;
         Version version = Version.CURRENT;
         SnapshotState state = SnapshotState.IN_PROGRESS;
         String reason = null;
         List<String> indices = Collections.emptyList();
         long startTime = 0;
         long endTime = 0;
-        int totalShard = 0;
+        int totalShards = 0;
         int successfulShards = 0;
         List<SnapshotShardFailure> shardFailures = Collections.emptyList();
         if (parser.currentToken() == null) { // fresh parser? move to the first token
@@ -406,6 +412,8 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
                         if (token.isValue()) {
                             if (NAME.equals(currentFieldName)) {
                                 name = parser.text();
+                            } else if (UUID.equals(currentFieldName)) {
+                                uuid = parser.text();
                             } else if (STATE.equals(currentFieldName)) {
                                 state = SnapshotState.valueOf(parser.text());
                             } else if (REASON.equals(currentFieldName)) {
@@ -415,7 +423,7 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
                             } else if (END_TIME.equals(currentFieldName)) {
                                 endTime = parser.longValue();
                             } else if (TOTAL_SHARDS.equals(currentFieldName)) {
-                                totalShard = parser.intValue();
+                                totalShards = parser.intValue();
                             } else if (SUCCESSFUL_SHARDS.equals(currentFieldName)) {
                                 successfulShards = parser.intValue();
                             } else if (VERSION_ID.equals(currentFieldName)) {
@@ -448,12 +456,25 @@ public final class SnapshotInfo implements Comparable<SnapshotInfo>, ToXContent,
         } else {
             throw new ElasticsearchParseException("unexpected token  [" + token + "]");
         }
-        return new SnapshotInfo(name, indices, state, reason, version, startTime, endTime, totalShard, successfulShards, shardFailures);
+        if (uuid == null) {
+            // the old format where there wasn't a UUID
+            uuid = SnapshotId.UNASSIGNED_UUID;
+        }
+        return new SnapshotInfo(new SnapshotId(name, uuid),
+                                indices,
+                                state,
+                                reason,
+                                version,
+                                startTime,
+                                endTime,
+                                totalShards,
+                                successfulShards,
+                                shardFailures);
     }
 
     @Override
     public void writeTo(final StreamOutput out) throws IOException {
-        out.writeString(name);
+        snapshotId.writeTo(out);
         out.writeVInt(indices.size());
         for (String index : indices) {
             out.writeString(index);
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotMissingException.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotMissingException.java
index 27fe3de51e0..5f0979e38d8 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotMissingException.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotMissingException.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.snapshots;
 
-import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.rest.RestStatus;
 
@@ -30,12 +29,16 @@ import java.io.IOException;
  */
 public class SnapshotMissingException extends SnapshotException {
 
-    public SnapshotMissingException(SnapshotId snapshot, Throwable cause) {
-        super(snapshot, "is missing", cause);
+    public SnapshotMissingException(final String repositoryName, final SnapshotId snapshotId, final Throwable cause) {
+        super(repositoryName, snapshotId, "is missing", cause);
     }
 
-    public SnapshotMissingException(SnapshotId snapshot) {
-        super(snapshot, "is missing");
+    public SnapshotMissingException(final String repositoryName, final SnapshotId snapshotId) {
+        super(repositoryName, snapshotId, "is missing");
+    }
+
+    public SnapshotMissingException(final String repositoryName, final String snapshotName) {
+        super(repositoryName, snapshotName, "is missing");
     }
 
     public SnapshotMissingException(StreamInput in) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotRestoreException.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotRestoreException.java
index 940f8162e6b..9003a08a54a 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotRestoreException.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotRestoreException.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.snapshots;
 
-import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.common.io.stream.StreamInput;
 
 import java.io.IOException;
@@ -28,11 +27,19 @@ import java.io.IOException;
  * Snapshot restore exception
  */
 public class SnapshotRestoreException extends SnapshotException {
-    public SnapshotRestoreException(SnapshotId snapshot, String message) {
+    public SnapshotRestoreException(final String repositoryName, final String snapshotName, final String message) {
+        super(repositoryName, snapshotName, message);
+    }
+
+    public SnapshotRestoreException(final String repositoryName, final String snapshotName, final String message, final Throwable cause) {
+        super(repositoryName, snapshotName, message, cause);
+    }
+
+    public SnapshotRestoreException(final Snapshot snapshot, final String message) {
         super(snapshot, message);
     }
 
-    public SnapshotRestoreException(SnapshotId snapshot, String message, Throwable cause) {
+    public SnapshotRestoreException(final Snapshot snapshot, final String message, final Throwable cause) {
         super(snapshot, message, cause);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
index c2a33680efa..b9d8c729223 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
@@ -27,7 +27,6 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.SnapshotsInProgress;
-import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -94,7 +93,7 @@ public class SnapshotShardsService extends AbstractLifecycleComponent<SnapshotSh
 
     private final Condition shutdownCondition = shutdownLock.newCondition();
 
-    private volatile Map<SnapshotId, SnapshotShards> shardSnapshots = emptyMap();
+    private volatile Map<Snapshot, SnapshotShards> shardSnapshots = emptyMap();
 
     private final BlockingQueue<UpdateIndexShardSnapshotStatusRequest> updatedSnapshotStateQueue = ConcurrentCollections.newBlockingQueue();
 
@@ -176,11 +175,11 @@ public class SnapshotShardsService extends AbstractLifecycleComponent<SnapshotSh
      * This method is executed on data node
      * </p>
      *
-     * @param snapshotId snapshot id
+     * @param snapshot  snapshot
      * @return map of shard id to snapshot status
      */
-    public Map<ShardId, IndexShardSnapshotStatus> currentSnapshotShards(SnapshotId snapshotId) {
-        SnapshotShards snapshotShards = shardSnapshots.get(snapshotId);
+    public Map<ShardId, IndexShardSnapshotStatus> currentSnapshotShards(Snapshot snapshot) {
+        SnapshotShards snapshotShards = shardSnapshots.get(snapshot);
         if (snapshotShards == null) {
             return null;
         } else {
@@ -195,24 +194,25 @@ public class SnapshotShardsService extends AbstractLifecycleComponent<SnapshotSh
      */
     private void processIndexShardSnapshots(ClusterChangedEvent event) {
         SnapshotsInProgress snapshotsInProgress = event.state().custom(SnapshotsInProgress.TYPE);
-        Map<SnapshotId, SnapshotShards> survivors = new HashMap<>();
+        Map<Snapshot, SnapshotShards> survivors = new HashMap<>();
         // First, remove snapshots that are no longer there
-        for (Map.Entry<SnapshotId, SnapshotShards> entry : shardSnapshots.entrySet()) {
-            if (snapshotsInProgress != null && snapshotsInProgress.snapshot(entry.getKey()) != null) {
+        for (Map.Entry<Snapshot, SnapshotShards> entry : shardSnapshots.entrySet()) {
+            final Snapshot snapshot = entry.getKey();
+            if (snapshotsInProgress != null && snapshotsInProgress.snapshot(snapshot) != null) {
                 survivors.put(entry.getKey(), entry.getValue());
             }
         }
 
         // For now we will be mostly dealing with a single snapshot at a time but might have multiple simultaneously running
         // snapshots in the future
-        Map<SnapshotId, Map<ShardId, IndexShardSnapshotStatus>> newSnapshots = new HashMap<>();
+        Map<Snapshot, Map<ShardId, IndexShardSnapshotStatus>> newSnapshots = new HashMap<>();
         // Now go through all snapshots and update existing or create missing
         final String localNodeId = clusterService.localNode().getId();
         if (snapshotsInProgress != null) {
             for (SnapshotsInProgress.Entry entry : snapshotsInProgress.entries()) {
                 if (entry.state() == SnapshotsInProgress.State.STARTED) {
                     Map<ShardId, IndexShardSnapshotStatus> startedShards = new HashMap<>();
-                    SnapshotShards snapshotShards = shardSnapshots.get(entry.snapshotId());
+                    SnapshotShards snapshotShards = shardSnapshots.get(entry.snapshot());
                     for (ObjectObjectCursor<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shard : entry.shards()) {
                         // Add all new shards to start processing on
                         if (localNodeId.equals(shard.value.nodeId())) {
@@ -223,7 +223,7 @@ public class SnapshotShardsService extends AbstractLifecycleComponent<SnapshotSh
                         }
                     }
                     if (!startedShards.isEmpty()) {
-                        newSnapshots.put(entry.snapshotId(), startedShards);
+                        newSnapshots.put(entry.snapshot(), startedShards);
                         if (snapshotShards != null) {
                             // We already saw this snapshot but we need to add more started shards
                             Map<ShardId, IndexShardSnapshotStatus> shards = new HashMap<>();
@@ -231,15 +231,15 @@ public class SnapshotShardsService extends AbstractLifecycleComponent<SnapshotSh
                             shards.putAll(snapshotShards.shards);
                             // Put all newly started shards
                             shards.putAll(startedShards);
-                            survivors.put(entry.snapshotId(), new SnapshotShards(unmodifiableMap(shards)));
+                            survivors.put(entry.snapshot(), new SnapshotShards(unmodifiableMap(shards)));
                         } else {
                             // Brand new snapshot that we haven't seen before
-                            survivors.put(entry.snapshotId(), new SnapshotShards(unmodifiableMap(startedShards)));
+                            survivors.put(entry.snapshot(), new SnapshotShards(unmodifiableMap(startedShards)));
                         }
                     }
                 } else if (entry.state() == SnapshotsInProgress.State.ABORTED) {
                     // Abort all running shards for this snapshot
-                    SnapshotShards snapshotShards = shardSnapshots.get(entry.snapshotId());
+                    SnapshotShards snapshotShards = shardSnapshots.get(entry.snapshot());
                     if (snapshotShards != null) {
                         for (ObjectObjectCursor<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shard : entry.shards()) {
                             IndexShardSnapshotStatus snapshotStatus = snapshotShards.shards.get(shard.key);
@@ -250,16 +250,16 @@ public class SnapshotShardsService extends AbstractLifecycleComponent<SnapshotSh
                                         snapshotStatus.abort();
                                         break;
                                     case FINALIZE:
-                                        logger.debug("[{}] trying to cancel snapshot on shard [{}] that is finalizing, letting it finish", entry.snapshotId(), shard.key);
+                                        logger.debug("[{}] trying to cancel snapshot on shard [{}] that is finalizing, letting it finish", entry.snapshot(), shard.key);
                                         break;
                                     case DONE:
-                                        logger.debug("[{}] trying to cancel snapshot on the shard [{}] that is already done, updating status on the master", entry.snapshotId(), shard.key);
-                                        updateIndexShardSnapshotStatus(entry.snapshotId(), shard.key,
+                                        logger.debug("[{}] trying to cancel snapshot on the shard [{}] that is already done, updating status on the master", entry.snapshot(), shard.key);
+                                        updateIndexShardSnapshotStatus(entry.snapshot(), shard.key,
                                                 new SnapshotsInProgress.ShardSnapshotStatus(event.state().nodes().getLocalNodeId(), SnapshotsInProgress.State.SUCCESS));
                                         break;
                                     case FAILURE:
-                                        logger.debug("[{}] trying to cancel snapshot on the shard [{}] that has already failed, updating status on the master", entry.snapshotId(), shard.key);
-                                        updateIndexShardSnapshotStatus(entry.snapshotId(), shard.key,
+                                        logger.debug("[{}] trying to cancel snapshot on the shard [{}] that has already failed, updating status on the master", entry.snapshot(), shard.key);
+                                        updateIndexShardSnapshotStatus(entry.snapshot(), shard.key,
                                                 new SnapshotsInProgress.ShardSnapshotStatus(event.state().nodes().getLocalNodeId(), SnapshotsInProgress.State.FAILED, snapshotStatus.failure()));
                                         break;
                                     default:
@@ -288,7 +288,7 @@ public class SnapshotShardsService extends AbstractLifecycleComponent<SnapshotSh
         // We have new shards to starts
         if (newSnapshots.isEmpty() == false) {
             Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
-            for (final Map.Entry<SnapshotId, Map<ShardId, IndexShardSnapshotStatus>> entry : newSnapshots.entrySet()) {
+            for (final Map.Entry<Snapshot, Map<ShardId, IndexShardSnapshotStatus>> entry : newSnapshots.entrySet()) {
                 for (final Map.Entry<ShardId, IndexShardSnapshotStatus> shardEntry : entry.getValue().entrySet()) {
                     final ShardId shardId = shardEntry.getKey();
                     try {
@@ -318,11 +318,11 @@ public class SnapshotShardsService extends AbstractLifecycleComponent<SnapshotSh
     /**
      * Creates shard snapshot
      *
-     * @param snapshotId     snapshot id
+     * @param snapshot       snapshot
      * @param snapshotStatus snapshot status
      */
-    private void snapshot(final IndexShard indexShard, final SnapshotId snapshotId, final IndexShardSnapshotStatus snapshotStatus) {
-        IndexShardRepository indexShardRepository = snapshotsService.getRepositoriesService().indexShardRepository(snapshotId.getRepository());
+    private void snapshot(final IndexShard indexShard, final Snapshot snapshot, final IndexShardSnapshotStatus snapshotStatus) {
+        IndexShardRepository indexShardRepository = snapshotsService.getRepositoriesService().indexShardRepository(snapshot.getRepository());
         ShardId shardId = indexShard.shardId();
         if (!indexShard.routingEntry().primary()) {
             throw new IndexShardSnapshotFailedException(shardId, "snapshot should be performed only on primary");
@@ -340,11 +340,11 @@ public class SnapshotShardsService extends AbstractLifecycleComponent<SnapshotSh
             // we flush first to make sure we get the latest writes snapshotted
             IndexCommit snapshotIndexCommit = indexShard.snapshotIndex(true);
             try {
-                indexShardRepository.snapshot(snapshotId, shardId, snapshotIndexCommit, snapshotStatus);
+                indexShardRepository.snapshot(snapshot.getSnapshotId(), shardId, snapshotIndexCommit, snapshotStatus);
                 if (logger.isDebugEnabled()) {
                     StringBuilder sb = new StringBuilder();
                     sb.append("    index    : version [").append(snapshotStatus.indexVersion()).append("], number_of_files [").append(snapshotStatus.numberOfFiles()).append("] with total_size [").append(new ByteSizeValue(snapshotStatus.totalSize())).append("]\n");
-                    logger.debug("snapshot ({}) completed to {}, took [{}]\n{}", snapshotId.getSnapshot(), indexShardRepository,
+                    logger.debug("snapshot ({}) completed to {}, took [{}]\n{}", snapshot, indexShardRepository,
                         TimeValue.timeValueMillis(snapshotStatus.time()), sb);
                 }
             } finally {
@@ -369,7 +369,7 @@ public class SnapshotShardsService extends AbstractLifecycleComponent<SnapshotSh
         }
         for (SnapshotsInProgress.Entry snapshot : snapshotsInProgress.entries()) {
             if (snapshot.state() == SnapshotsInProgress.State.STARTED || snapshot.state() == SnapshotsInProgress.State.ABORTED) {
-                Map<ShardId, IndexShardSnapshotStatus> localShards = currentSnapshotShards(snapshot.snapshotId());
+                Map<ShardId, IndexShardSnapshotStatus> localShards = currentSnapshotShards(snapshot.snapshot());
                 if (localShards != null) {
                     ImmutableOpenMap<ShardId, SnapshotsInProgress.ShardSnapshotStatus> masterShards = snapshot.shards();
                     for(Map.Entry<ShardId, IndexShardSnapshotStatus> localShard : localShards.entrySet()) {
@@ -380,13 +380,13 @@ public class SnapshotShardsService extends AbstractLifecycleComponent<SnapshotSh
                             // Master knows about the shard and thinks it has not completed
                             if (localShardStatus.stage() == IndexShardSnapshotStatus.Stage.DONE) {
                                 // but we think the shard is done - we need to make new master know that the shard is done
-                                logger.debug("[{}] new master thinks the shard [{}] is not completed but the shard is done locally, updating status on the master", snapshot.snapshotId(), shardId);
-                                updateIndexShardSnapshotStatus(snapshot.snapshotId(), shardId,
+                                logger.debug("[{}] new master thinks the shard [{}] is not completed but the shard is done locally, updating status on the master", snapshot.snapshot(), shardId);
+                                updateIndexShardSnapshotStatus(snapshot.snapshot(), shardId,
                                         new SnapshotsInProgress.ShardSnapshotStatus(event.state().nodes().getLocalNodeId(), SnapshotsInProgress.State.SUCCESS));
                             } else if (localShard.getValue().stage() == IndexShardSnapshotStatus.Stage.FAILURE) {
                                 // but we think the shard failed - we need to make new master know that the shard failed
-                                logger.debug("[{}] new master thinks the shard [{}] is not completed but the shard failed locally, updating status on master", snapshot.snapshotId(), shardId);
-                                updateIndexShardSnapshotStatus(snapshot.snapshotId(), shardId,
+                                logger.debug("[{}] new master thinks the shard [{}] is not completed but the shard failed locally, updating status on master", snapshot.snapshot(), shardId);
+                                updateIndexShardSnapshotStatus(snapshot.snapshot(), shardId,
                                         new SnapshotsInProgress.ShardSnapshotStatus(event.state().nodes().getLocalNodeId(), SnapshotsInProgress.State.FAILED, localShardStatus.failure()));
 
                             }
@@ -413,7 +413,7 @@ public class SnapshotShardsService extends AbstractLifecycleComponent<SnapshotSh
      * Internal request that is used to send changes in snapshot status to master
      */
     public static class UpdateIndexShardSnapshotStatusRequest extends TransportRequest {
-        private SnapshotId snapshotId;
+        private Snapshot snapshot;
         private ShardId shardId;
         private SnapshotsInProgress.ShardSnapshotStatus status;
 
@@ -423,8 +423,8 @@ public class SnapshotShardsService extends AbstractLifecycleComponent<SnapshotSh
 
         }
 
-        public UpdateIndexShardSnapshotStatusRequest(SnapshotId snapshotId, ShardId shardId, SnapshotsInProgress.ShardSnapshotStatus status) {
-            this.snapshotId = snapshotId;
+        public UpdateIndexShardSnapshotStatusRequest(Snapshot snapshot, ShardId shardId, SnapshotsInProgress.ShardSnapshotStatus status) {
+            this.snapshot = snapshot;
             this.shardId = shardId;
             this.status = status;
         }
@@ -432,7 +432,7 @@ public class SnapshotShardsService extends AbstractLifecycleComponent<SnapshotSh
         @Override
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
-            snapshotId = SnapshotId.readSnapshotId(in);
+            snapshot = new Snapshot(in);
             shardId = ShardId.readShardId(in);
             status = SnapshotsInProgress.ShardSnapshotStatus.readShardSnapshotStatus(in);
         }
@@ -440,13 +440,13 @@ public class SnapshotShardsService extends AbstractLifecycleComponent<SnapshotSh
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            snapshotId.writeTo(out);
+            snapshot.writeTo(out);
             shardId.writeTo(out);
             status.writeTo(out);
         }
 
-        public SnapshotId snapshotId() {
-            return snapshotId;
+        public Snapshot snapshot() {
+            return snapshot;
         }
 
         public ShardId shardId() {
@@ -459,7 +459,7 @@ public class SnapshotShardsService extends AbstractLifecycleComponent<SnapshotSh
 
         @Override
         public String toString() {
-            return "" + snapshotId + ", shardId [" + shardId + "], status [" + status.state() + "]";
+            return "" + snapshot + ", shardId [" + shardId + "], status [" + status.state() + "]";
         }
 
         public void markAsProcessed() {
@@ -474,8 +474,8 @@ public class SnapshotShardsService extends AbstractLifecycleComponent<SnapshotSh
     /**
      * Updates the shard status
      */
-    public void updateIndexShardSnapshotStatus(SnapshotId snapshotId, ShardId shardId, SnapshotsInProgress.ShardSnapshotStatus status) {
-        UpdateIndexShardSnapshotStatusRequest request = new UpdateIndexShardSnapshotStatusRequest(snapshotId, shardId, status);
+    public void updateIndexShardSnapshotStatus(Snapshot snapshot, ShardId shardId, SnapshotsInProgress.ShardSnapshotStatus status) {
+        UpdateIndexShardSnapshotStatusRequest request = new UpdateIndexShardSnapshotStatusRequest(snapshot, shardId, status);
         try {
             if (clusterService.state().nodes().isLocalNodeElectedMaster()) {
                 innerUpdateSnapshotState(request);
@@ -484,7 +484,7 @@ public class SnapshotShardsService extends AbstractLifecycleComponent<SnapshotSh
                         UPDATE_SNAPSHOT_ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME);
             }
         } catch (Throwable t) {
-            logger.warn("[{}] [{}] failed to update snapshot state", t, request.snapshotId(), request.status());
+            logger.warn("[{}] [{}] failed to update snapshot state", t, request.snapshot(), request.status());
         }
     }
 
@@ -528,8 +528,8 @@ public class SnapshotShardsService extends AbstractLifecycleComponent<SnapshotSh
                             final UpdateIndexShardSnapshotStatusRequest updateSnapshotState = drainedRequests.get(i);
                             updateSnapshotState.markAsProcessed();
 
-                            if (entry.snapshotId().equals(updateSnapshotState.snapshotId())) {
-                                logger.trace("[{}] Updating shard [{}] with status [{}]", updateSnapshotState.snapshotId(), updateSnapshotState.shardId(), updateSnapshotState.status().state());
+                            if (entry.snapshot().equals(updateSnapshotState.snapshot())) {
+                                logger.trace("[{}] Updating shard [{}] with status [{}]", updateSnapshotState.snapshot(), updateSnapshotState.shardId(), updateSnapshotState.status().state());
                                 if (updated == false) {
                                     shards.putAll(entry.shards());
                                     updated = true;
@@ -549,7 +549,7 @@ public class SnapshotShardsService extends AbstractLifecycleComponent<SnapshotSh
                                 entries.add(updatedEntry);
                                 // Finalize snapshot in the repository
                                 snapshotsService.endSnapshot(updatedEntry);
-                                logger.info("snapshot [{}] is done", updatedEntry.snapshotId());
+                                logger.info("snapshot [{}] is done", updatedEntry.snapshot());
                             }
                         } else {
                             entries.add(entry);
@@ -568,7 +568,7 @@ public class SnapshotShardsService extends AbstractLifecycleComponent<SnapshotSh
             @Override
             public void onFailure(String source, Throwable t) {
                 for (UpdateIndexShardSnapshotStatusRequest request : drainedRequests) {
-                    logger.warn("[{}][{}] failed to update snapshot status to [{}]", t, request.snapshotId(), request.shardId(), request.status());
+                    logger.warn("[{}][{}] failed to update snapshot status to [{}]", t, request.snapshot(), request.shardId(), request.status());
                 }
             }
         });
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
index 5b52e915195..d6e610a184f 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
@@ -37,7 +37,6 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.RepositoriesMetaData;
-import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
@@ -47,6 +46,7 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
@@ -72,8 +72,11 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.stream.Collectors;
 
 import static java.util.Collections.unmodifiableMap;
 import static org.elasticsearch.cluster.SnapshotsInProgress.completed;
@@ -92,7 +95,7 @@ import static org.elasticsearch.cluster.SnapshotsInProgress.completed;
  * <li>Once shard snapshot is created data node updates state of the shard in the cluster state using the {@link SnapshotShardsService#updateIndexShardSnapshotStatus} method</li>
  * <li>When last shard is completed master node in {@link SnapshotShardsService#innerUpdateSnapshotState} method marks the snapshot as completed</li>
  * <li>After cluster state is updated, the {@link #endSnapshot(SnapshotsInProgress.Entry)} finalizes snapshot in the repository,
- * notifies all {@link #snapshotCompletionListeners} that snapshot is completed, and finally calls {@link #removeSnapshotFromClusterState(SnapshotId, SnapshotInfo, Throwable)} to remove snapshot from cluster state</li>
+ * notifies all {@link #snapshotCompletionListeners} that snapshot is completed, and finally calls {@link #removeSnapshotFromClusterState(Snapshot, SnapshotInfo, Throwable)} to remove snapshot from cluster state</li>
  * </ul>
  */
 public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsService> implements ClusterStateListener {
@@ -121,49 +124,67 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
         }
     }
 
+    /**
+     * Retrieves list of snapshot ids that are present in a repository
+     *
+     * @param repositoryName repository name
+     * @return list of snapshot ids
+     */
+    public List<SnapshotId> snapshotIds(final String repositoryName) {
+        Repository repository = repositoriesService.repository(repositoryName);
+        assert repository != null; // should only be called once we've validated the repository exists
+        return repository.snapshots();
+    }
+
     /**
      * Retrieves snapshot from repository
      *
-     * @param snapshotId snapshot id
+     * @param repositoryName  repository name
+     * @param snapshotId      snapshot id
      * @return snapshot
      * @throws SnapshotMissingException if snapshot is not found
      */
-    public SnapshotInfo snapshot(SnapshotId snapshotId) {
-        validate(snapshotId);
-        List<SnapshotsInProgress.Entry> entries = currentSnapshots(snapshotId.getRepository(), new String[]{snapshotId.getSnapshot()});
+    public SnapshotInfo snapshot(final String repositoryName, final SnapshotId snapshotId) {
+        List<SnapshotsInProgress.Entry> entries = currentSnapshots(repositoryName, Arrays.asList(snapshotId.getName()));
         if (!entries.isEmpty()) {
             return inProgressSnapshot(entries.iterator().next());
         }
-        return repositoriesService.repository(snapshotId.getRepository()).readSnapshot(snapshotId);
+        return repositoriesService.repository(repositoryName).readSnapshot(snapshotId);
     }
 
     /**
      * Returns a list of snapshots from repository sorted by snapshot creation date
      *
      * @param repositoryName repository name
+     * @param snapshotIds       snapshots for which to fetch snapshot information
+     * @param ignoreUnavailable if true, snapshots that could not be read will only be logged with a warning,
+     *                          if false, they will throw an error
      * @return list of snapshots
      */
-    public List<SnapshotInfo> snapshots(String repositoryName, boolean ignoreUnavailable) {
-        Set<SnapshotInfo> snapshotSet = new HashSet<>();
-        List<SnapshotsInProgress.Entry> entries = currentSnapshots(repositoryName, null);
+    public List<SnapshotInfo> snapshots(final String repositoryName, List<SnapshotId> snapshotIds, final boolean ignoreUnavailable) {
+        final Set<SnapshotInfo> snapshotSet = new HashSet<>();
+        final Set<SnapshotId> snapshotIdsToIterate = new HashSet<>(snapshotIds);
+        // first, look at the snapshots in progress
+        final List<SnapshotsInProgress.Entry> entries =
+            currentSnapshots(repositoryName, snapshotIdsToIterate.stream().map(SnapshotId::getName).collect(Collectors.toList()));
         for (SnapshotsInProgress.Entry entry : entries) {
             snapshotSet.add(inProgressSnapshot(entry));
+            snapshotIdsToIterate.remove(entry.snapshot().getSnapshotId());
         }
-        Repository repository = repositoriesService.repository(repositoryName);
-        List<SnapshotId> snapshotIds = repository.snapshots();
-        for (SnapshotId snapshotId : snapshotIds) {
+        // then, look in the repository
+        final Repository repository = repositoriesService.repository(repositoryName);
+        for (SnapshotId snapshotId : snapshotIdsToIterate) {
             try {
                 snapshotSet.add(repository.readSnapshot(snapshotId));
             } catch (Exception ex) {
                 if (ignoreUnavailable) {
                     logger.warn("failed to get snapshot [{}]", ex, snapshotId);
                 } else {
-                    throw new SnapshotException(snapshotId, "Snapshot could not be read", ex);
+                    throw new SnapshotException(repositoryName, snapshotId, "Snapshot could not be read", ex);
                 }
             }
         }
-
-        ArrayList<SnapshotInfo> snapshotList = new ArrayList<>(snapshotSet);
+        final ArrayList<SnapshotInfo> snapshotList = new ArrayList<>(snapshotSet);
         CollectionUtil.timSort(snapshotList);
         return Collections.unmodifiableList(snapshotList);
     }
@@ -174,9 +195,9 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
      * @param repositoryName repository name
      * @return list of snapshots
      */
-    public List<SnapshotInfo> currentSnapshots(String repositoryName) {
+    public List<SnapshotInfo> currentSnapshots(final String repositoryName) {
         List<SnapshotInfo> snapshotList = new ArrayList<>();
-        List<SnapshotsInProgress.Entry> entries = currentSnapshots(repositoryName, null);
+        List<SnapshotsInProgress.Entry> entries = currentSnapshots(repositoryName, Collections.emptyList());
         for (SnapshotsInProgress.Entry entry : entries) {
             snapshotList.add(inProgressSnapshot(entry));
         }
@@ -194,8 +215,11 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
      * @param listener snapshot creation listener
      */
     public void createSnapshot(final SnapshotRequest request, final CreateSnapshotListener listener) {
-        final SnapshotId snapshotId = new SnapshotId(request.repository(), request.name());
-        validate(snapshotId);
+        final String repositoryName = request.repositoryName;
+        final String snapshotName = request.snapshotName;
+        validate(repositoryName, snapshotName);
+        final SnapshotId snapshotId = new SnapshotId(snapshotName, UUIDs.randomBase64UUID()); // new UUID for the snapshot
+
         clusterService.submitStateUpdateTask(request.cause(), new ClusterStateUpdateTask() {
 
             private SnapshotsInProgress.Entry newSnapshot = null;
@@ -208,19 +232,25 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
                 if (snapshots == null || snapshots.entries().isEmpty()) {
                     // Store newSnapshot here to be processed in clusterStateProcessed
                     List<String> indices = Arrays.asList(indexNameExpressionResolver.concreteIndexNames(currentState, request.indicesOptions(), request.indices()));
-                    logger.trace("[{}][{}] creating snapshot for indices [{}]", request.repository(), request.name(), indices);
-                    newSnapshot = new SnapshotsInProgress.Entry(snapshotId, request.includeGlobalState(), request.partial(), State.INIT, indices, System.currentTimeMillis(), null);
+                    logger.trace("[{}][{}] creating snapshot for indices [{}]", repositoryName, snapshotName, indices);
+                    newSnapshot = new SnapshotsInProgress.Entry(new Snapshot(repositoryName, snapshotId),
+                                                                request.includeGlobalState(),
+                                                                request.partial(),
+                                                                State.INIT,
+                                                                indices,
+                                                                System.currentTimeMillis(),
+                                                                null);
                     snapshots = new SnapshotsInProgress(newSnapshot);
                 } else {
                     // TODO: What should we do if a snapshot is already running?
-                    throw new ConcurrentSnapshotExecutionException(snapshotId, "a snapshot is already running");
+                    throw new ConcurrentSnapshotExecutionException(repositoryName, snapshotName, "a snapshot is already running");
                 }
                 return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, snapshots).build();
             }
 
             @Override
             public void onFailure(String source, Throwable t) {
-                logger.warn("[{}][{}] failed to create snapshot", t, request.repository(), request.name());
+                logger.warn("[{}][{}] failed to create snapshot", t, repositoryName, snapshotName);
                 newSnapshot = null;
                 listener.onFailure(t);
             }
@@ -228,12 +258,9 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
             @Override
             public void clusterStateProcessed(String source, ClusterState oldState, final ClusterState newState) {
                 if (newSnapshot != null) {
-                    threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new Runnable() {
-                        @Override
-                        public void run() {
-                            beginSnapshot(newState, newSnapshot, request.partial(), listener);
-                        }
-                    });
+                    threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() ->
+                        beginSnapshot(newState, newSnapshot, request.partial(), listener)
+                    );
                 }
             }
 
@@ -253,34 +280,36 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
      */
     private void validate(SnapshotRequest request, ClusterState state) {
         RepositoriesMetaData repositoriesMetaData = state.getMetaData().custom(RepositoriesMetaData.TYPE);
-        if (repositoriesMetaData == null || repositoriesMetaData.repository(request.repository()) == null) {
-            throw new RepositoryMissingException(request.repository());
+        final String repository = request.repositoryName;
+        if (repositoriesMetaData == null || repositoriesMetaData.repository(repository) == null) {
+            throw new RepositoryMissingException(repository);
         }
-        validate(new SnapshotId(request.repository(), request.name()));
+        validate(repository, request.snapshotName);
     }
 
-    private static void validate(SnapshotId snapshotId) {
-        String name = snapshotId.getSnapshot();
-        if (!Strings.hasLength(name)) {
-            throw new InvalidSnapshotNameException(snapshotId, "cannot be empty");
+    private static void validate(final String repositoryName, final String snapshotName) {
+        if (Strings.hasLength(snapshotName) == false) {
+            throw new InvalidSnapshotNameException(repositoryName, snapshotName, "cannot be empty");
         }
-        if (name.contains(" ")) {
-            throw new InvalidSnapshotNameException(snapshotId, "must not contain whitespace");
+        if (snapshotName.contains(" ")) {
+            throw new InvalidSnapshotNameException(repositoryName, snapshotName, "must not contain whitespace");
         }
-        if (name.contains(",")) {
-            throw new InvalidSnapshotNameException(snapshotId, "must not contain ','");
+        if (snapshotName.contains(",")) {
+            throw new InvalidSnapshotNameException(repositoryName, snapshotName, "must not contain ','");
         }
-        if (name.contains("#")) {
-            throw new InvalidSnapshotNameException(snapshotId, "must not contain '#'");
+        if (snapshotName.contains("#")) {
+            throw new InvalidSnapshotNameException(repositoryName, snapshotName, "must not contain '#'");
         }
-        if (name.charAt(0) == '_') {
-            throw new InvalidSnapshotNameException(snapshotId, "must not start with '_'");
+        if (snapshotName.charAt(0) == '_') {
+            throw new InvalidSnapshotNameException(repositoryName, snapshotName, "must not start with '_'");
         }
-        if (!name.toLowerCase(Locale.ROOT).equals(name)) {
-            throw new InvalidSnapshotNameException(snapshotId, "must be lowercase");
+        if (snapshotName.toLowerCase(Locale.ROOT).equals(snapshotName) == false) {
+            throw new InvalidSnapshotNameException(repositoryName, snapshotName, "must be lowercase");
         }
-        if (!Strings.validFileName(name)) {
-            throw new InvalidSnapshotNameException(snapshotId, "must not contain the following characters " + Strings.INVALID_FILENAME_CHARS);
+        if (Strings.validFileName(snapshotName) == false) {
+            throw new InvalidSnapshotNameException(repositoryName,
+                                                   snapshotName,
+                                                   "must not contain the following characters " + Strings.INVALID_FILENAME_CHARS);
         }
     }
 
@@ -294,10 +323,13 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
      * @param partial                    allow partial snapshots
      * @param userCreateSnapshotListener listener
      */
-    private void beginSnapshot(ClusterState clusterState, final SnapshotsInProgress.Entry snapshot, final boolean partial, final CreateSnapshotListener userCreateSnapshotListener) {
+    private void beginSnapshot(final ClusterState clusterState,
+                               final SnapshotsInProgress.Entry snapshot,
+                               final boolean partial,
+                               final CreateSnapshotListener userCreateSnapshotListener) {
         boolean snapshotCreated = false;
         try {
-            Repository repository = repositoriesService.repository(snapshot.snapshotId().getRepository());
+            Repository repository = repositoriesService.repository(snapshot.snapshot().getRepository());
 
             MetaData metaData = clusterState.metaData();
             if (!snapshot.includeGlobalState()) {
@@ -309,7 +341,7 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
                 metaData = builder.build();
             }
 
-            repository.initializeSnapshot(snapshot.snapshotId(), snapshot.indices(), metaData);
+            repository.initializeSnapshot(snapshot.snapshot().getSnapshotId(), snapshot.indices(), metaData);
             snapshotCreated = true;
             if (snapshot.indices().isEmpty()) {
                 // No indices in this snapshot - we are done
@@ -317,7 +349,7 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
                 endSnapshot(snapshot);
                 return;
             }
-            clusterService.submitStateUpdateTask("update_snapshot [" + snapshot.snapshotId().getSnapshot() + "]", new ClusterStateUpdateTask() {
+            clusterService.submitStateUpdateTask("update_snapshot [" + snapshot.snapshot() + "]", new ClusterStateUpdateTask() {
                 boolean accepted = false;
                 SnapshotsInProgress.Entry updatedSnapshot;
                 String failure = null;
@@ -327,7 +359,7 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
                     SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
                     List<SnapshotsInProgress.Entry> entries = new ArrayList<>();
                     for (SnapshotsInProgress.Entry entry : snapshots.entries()) {
-                        if (entry.snapshotId().equals(snapshot.snapshotId())) {
+                        if (entry.snapshot().equals(snapshot.snapshot())) {
                             // Replace the snapshot that was just created
                             ImmutableOpenMap<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shards = shards(currentState, entry.indices());
                             if (!partial) {
@@ -362,13 +394,15 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
                             entries.add(entry);
                         }
                     }
-                    return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, new SnapshotsInProgress(Collections.unmodifiableList(entries))).build();
+                    return ClusterState.builder(currentState)
+                                       .putCustom(SnapshotsInProgress.TYPE, new SnapshotsInProgress(Collections.unmodifiableList(entries)))
+                                       .build();
                 }
 
                 @Override
                 public void onFailure(String source, Throwable t) {
-                    logger.warn("[{}] failed to create snapshot", t, snapshot.snapshotId());
-                    removeSnapshotFromClusterState(snapshot.snapshotId(), null, t, new CleanupAfterErrorListener(snapshot, true, userCreateSnapshotListener, t));
+                    logger.warn("[{}] failed to create snapshot", t, snapshot.snapshot().getSnapshotId());
+                    removeSnapshotFromClusterState(snapshot.snapshot(), null, t, new CleanupAfterErrorListener(snapshot, true, userCreateSnapshotListener, t));
                 }
 
                 @Override
@@ -389,8 +423,8 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
                 }
             });
         } catch (Throwable t) {
-            logger.warn("failed to create snapshot [{}]", t, snapshot.snapshotId());
-            removeSnapshotFromClusterState(snapshot.snapshotId(), null, t, new CleanupAfterErrorListener(snapshot, snapshotCreated, userCreateSnapshotListener, t));
+            logger.warn("failed to create snapshot [{}]", t, snapshot.snapshot().getSnapshotId());
+            removeSnapshotFromClusterState(snapshot.snapshot(), null, t, new CleanupAfterErrorListener(snapshot, snapshotCreated, userCreateSnapshotListener, t));
         }
     }
 
@@ -421,10 +455,15 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
         private void cleanupAfterError() {
             if(snapshotCreated) {
                 try {
-                    repositoriesService.repository(snapshot.snapshotId().getRepository()).finalizeSnapshot(
-                        snapshot.snapshotId(), snapshot.indices(), snapshot.startTime(), ExceptionsHelper.detailedMessage(t), 0, Collections.<SnapshotShardFailure>emptyList());
+                    repositoriesService.repository(snapshot.snapshot().getRepository())
+                                       .finalizeSnapshot(snapshot.snapshot().getSnapshotId(),
+                                                         snapshot.indices(),
+                                                         snapshot.startTime(),
+                                                         ExceptionsHelper.detailedMessage(t),
+                                                         0,
+                                                         Collections.emptyList());
                 } catch (Throwable t2) {
-                    logger.warn("[{}] failed to close snapshot in repository", snapshot.snapshotId());
+                    logger.warn("[{}] failed to close snapshot in repository", snapshot.snapshot());
                 }
             }
             userCreateSnapshotListener.onFailure(t);
@@ -433,7 +472,7 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
     }
 
     private SnapshotInfo inProgressSnapshot(SnapshotsInProgress.Entry entry) {
-        return new SnapshotInfo(entry.snapshotId().getSnapshot(), entry.indices(), entry.startTime());
+        return new SnapshotInfo(entry.snapshot().getSnapshotId(), entry.indices(), entry.startTime());
     }
 
     /**
@@ -443,10 +482,10 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
      * </p>
      *
      * @param repository repository id
-     * @param snapshots  optional list of snapshots that will be used as a filter
+     * @param snapshots  list of snapshots that will be used as a filter, empty list means no snapshots are filtered
      * @return list of metadata for currently running snapshots
      */
-    public List<SnapshotsInProgress.Entry> currentSnapshots(String repository, String[] snapshots) {
+    public List<SnapshotsInProgress.Entry> currentSnapshots(final String repository, final List<String> snapshots) {
         SnapshotsInProgress snapshotsInProgress = clusterService.state().custom(SnapshotsInProgress.TYPE);
         if (snapshotsInProgress == null || snapshotsInProgress.entries().isEmpty()) {
             return Collections.emptyList();
@@ -458,12 +497,12 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
             // Most likely scenario - one snapshot is currently running
             // Check this snapshot against the query
             SnapshotsInProgress.Entry entry = snapshotsInProgress.entries().get(0);
-            if (!entry.snapshotId().getRepository().equals(repository)) {
+            if (entry.snapshot().getRepository().equals(repository) == false) {
                 return Collections.emptyList();
             }
-            if (snapshots != null && snapshots.length > 0) {
+            if (snapshots.isEmpty() == false) {
                 for (String snapshot : snapshots) {
-                    if (entry.snapshotId().getSnapshot().equals(snapshot)) {
+                    if (entry.snapshot().getSnapshotId().getName().equals(snapshot)) {
                         return snapshotsInProgress.entries();
                     }
                 }
@@ -474,12 +513,12 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
         }
         List<SnapshotsInProgress.Entry> builder = new ArrayList<>();
         for (SnapshotsInProgress.Entry entry : snapshotsInProgress.entries()) {
-            if (!entry.snapshotId().getRepository().equals(repository)) {
+            if (entry.snapshot().getRepository().equals(repository) == false) {
                 continue;
             }
-            if (snapshots != null && snapshots.length > 0) {
+            if (snapshots.isEmpty() == false) {
                 for (String snapshot : snapshots) {
-                    if (entry.snapshotId().getSnapshot().equals(snapshot)) {
+                    if (entry.snapshot().getSnapshotId().getName().equals(snapshot)) {
                         builder.add(entry);
                         break;
                     }
@@ -494,34 +533,35 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
     /**
      * Returns status of shards  currently finished snapshots
      * <p>
-     * This method is executed on master node and it's complimentary to the {@link SnapshotShardsService#currentSnapshotShards(SnapshotId)} because it
+     * This method is executed on master node and it's complimentary to the {@link SnapshotShardsService#currentSnapshotShards(Snapshot)} because it
      * returns similar information but for already finished snapshots.
      * </p>
      *
-     * @param snapshotId snapshot id
+     * @param repositoryName  repository name
+     * @param snapshotInfo    snapshot info
      * @return map of shard id to snapshot status
      */
-    public Map<ShardId, IndexShardSnapshotStatus> snapshotShards(SnapshotId snapshotId) throws IOException {
-        validate(snapshotId);
+    public Map<ShardId, IndexShardSnapshotStatus> snapshotShards(final String repositoryName,
+                                                                 final SnapshotInfo snapshotInfo) throws IOException {
         Map<ShardId, IndexShardSnapshotStatus> shardStatus = new HashMap<>();
-        Repository repository = repositoriesService.repository(snapshotId.getRepository());
-        IndexShardRepository indexShardRepository = repositoriesService.indexShardRepository(snapshotId.getRepository());
-        SnapshotInfo snapshot = repository.readSnapshot(snapshotId);
-        MetaData metaData = repository.readSnapshotMetaData(snapshotId, snapshot, snapshot.indices());
-        for (String index : snapshot.indices()) {
+        Repository repository = repositoriesService.repository(repositoryName);
+        IndexShardRepository indexShardRepository = repositoriesService.indexShardRepository(repositoryName);
+        MetaData metaData = repository.readSnapshotMetaData(snapshotInfo, snapshotInfo.indices());
+        for (String index : snapshotInfo.indices()) {
             IndexMetaData indexMetaData = metaData.indices().get(index);
             if (indexMetaData != null) {
                 int numberOfShards = indexMetaData.getNumberOfShards();
                 for (int i = 0; i < numberOfShards; i++) {
                     ShardId shardId = new ShardId(indexMetaData.getIndex(), i);
-                    SnapshotShardFailure shardFailure = findShardFailure(snapshot.shardFailures(), shardId);
+                    SnapshotShardFailure shardFailure = findShardFailure(snapshotInfo.shardFailures(), shardId);
                     if (shardFailure != null) {
                         IndexShardSnapshotStatus shardSnapshotStatus = new IndexShardSnapshotStatus();
                         shardSnapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.FAILURE);
                         shardSnapshotStatus.failure(shardFailure.reason());
                         shardStatus.put(shardId, shardSnapshotStatus);
                     } else {
-                        IndexShardSnapshotStatus shardSnapshotStatus = indexShardRepository.snapshotStatus(snapshotId, snapshot.version(), shardId);
+                        IndexShardSnapshotStatus shardSnapshotStatus =
+                            indexShardRepository.snapshotStatus(snapshotInfo.snapshotId(), snapshotInfo.version(), shardId);
                         shardStatus.put(shardId, shardSnapshotStatus);
                     }
                 }
@@ -606,15 +646,15 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
                             entries.add(updatedSnapshot);
                         } else if (snapshot.state() == State.INIT && newMaster) {
                             // Clean up the snapshot that failed to start from the old master
-                            deleteSnapshot(snapshot.snapshotId(), new DeleteSnapshotListener() {
+                            deleteSnapshot(snapshot.snapshot(), new DeleteSnapshotListener() {
                                 @Override
                                 public void onResponse() {
-                                    logger.debug("cleaned up abandoned snapshot {} in INIT state", snapshot.snapshotId());
+                                    logger.debug("cleaned up abandoned snapshot {} in INIT state", snapshot.snapshot());
                                 }
 
                                 @Override
                                 public void onFailure(Throwable t) {
-                                    logger.warn("failed to clean up abandoned snapshot {} in INIT state", snapshot.snapshotId());
+                                    logger.warn("failed to clean up abandoned snapshot {} in INIT state", snapshot.snapshot());
                                 }
                             });
                         } else if (snapshot.state() == State.SUCCESS && newMaster) {
@@ -810,10 +850,10 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
         threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new Runnable() {
             @Override
             public void run() {
-                SnapshotId snapshotId = entry.snapshotId();
+                final Snapshot snapshot = entry.snapshot();
                 try {
-                    final Repository repository = repositoriesService.repository(snapshotId.getRepository());
-                    logger.trace("[{}] finalizing snapshot in repository, state: [{}], failure[{}]", snapshotId, entry.state(), failure);
+                    final Repository repository = repositoriesService.repository(snapshot.getRepository());
+                    logger.trace("[{}] finalizing snapshot in repository, state: [{}], failure[{}]", snapshot, entry.state(), failure);
                     ArrayList<ShardSearchFailure> failures = new ArrayList<>();
                     ArrayList<SnapshotShardFailure> shardFailures = new ArrayList<>();
                     for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shardStatus : entry.shards()) {
@@ -824,11 +864,11 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
                             shardFailures.add(new SnapshotShardFailure(status.nodeId(), shardId, status.reason()));
                         }
                     }
-                    SnapshotInfo snapshot = repository.finalizeSnapshot(snapshotId, entry.indices(), entry.startTime(), failure, entry.shards().size(), Collections.unmodifiableList(shardFailures));
-                    removeSnapshotFromClusterState(snapshotId, snapshot, null);
+                    SnapshotInfo snapshotInfo = repository.finalizeSnapshot(snapshot.getSnapshotId(), entry.indices(), entry.startTime(), failure, entry.shards().size(), Collections.unmodifiableList(shardFailures));
+                    removeSnapshotFromClusterState(snapshot, snapshotInfo, null);
                 } catch (Throwable t) {
-                    logger.warn("[{}] failed to finalize snapshot", t, snapshotId);
-                    removeSnapshotFromClusterState(snapshotId, null, t);
+                    logger.warn("[{}] failed to finalize snapshot", t, snapshot);
+                    removeSnapshotFromClusterState(snapshot, null, t);
                 }
             }
         });
@@ -837,23 +877,23 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
     /**
      * Removes record of running snapshot from cluster state
      *
-     * @param snapshotId snapshot id
-     * @param snapshot   snapshot info if snapshot was successful
-     * @param t          exception if snapshot failed
+     * @param snapshot       snapshot
+     * @param snapshotInfo   snapshot info if snapshot was successful
+     * @param t              exception if snapshot failed
      */
-    private void removeSnapshotFromClusterState(final SnapshotId snapshotId, final SnapshotInfo snapshot, final Throwable t) {
-        removeSnapshotFromClusterState(snapshotId, snapshot, t, null);
+    private void removeSnapshotFromClusterState(final Snapshot snapshot, final SnapshotInfo snapshotInfo, final Throwable t) {
+        removeSnapshotFromClusterState(snapshot, snapshotInfo, t, null);
     }
 
     /**
      * Removes record of running snapshot from cluster state and notifies the listener when this action is complete
      *
-     * @param snapshotId snapshot id
+     * @param snapshot   snapshot
      * @param snapshot   snapshot info if snapshot was successful
      * @param t          exception if snapshot failed
      * @param listener   listener to notify when snapshot information is removed from the cluster state
      */
-    private void removeSnapshotFromClusterState(final SnapshotId snapshotId, final SnapshotInfo snapshot, final Throwable t,
+    private void removeSnapshotFromClusterState(final Snapshot snapshot, final SnapshotInfo snapshotInfo, final Throwable t,
                                                 @Nullable ActionListener<SnapshotInfo> listener) {
         clusterService.submitStateUpdateTask("remove snapshot metadata", new ClusterStateUpdateTask() {
             @Override
@@ -863,7 +903,7 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
                     boolean changed = false;
                     ArrayList<SnapshotsInProgress.Entry> entries = new ArrayList<>();
                     for (SnapshotsInProgress.Entry entry : snapshots.entries()) {
-                        if (entry.snapshotId().equals(snapshotId)) {
+                        if (entry.snapshot().equals(snapshot)) {
                             changed = true;
                         } else {
                             entries.add(entry);
@@ -879,7 +919,7 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
 
             @Override
             public void onFailure(String source, Throwable t) {
-                logger.warn("[{}] failed to remove snapshot metadata", t, snapshotId);
+                logger.warn("[{}] failed to remove snapshot metadata", t, snapshot);
                 if (listener != null) {
                     listener.onFailure(t);
                 }
@@ -889,32 +929,54 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
             public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                 for (SnapshotCompletionListener listener : snapshotCompletionListeners) {
                     try {
-                        if (snapshot != null) {
-                            listener.onSnapshotCompletion(snapshotId, snapshot);
+                        if (snapshotInfo != null) {
+                            listener.onSnapshotCompletion(snapshot, snapshotInfo);
                         } else {
-                            listener.onSnapshotFailure(snapshotId, t);
+                            listener.onSnapshotFailure(snapshot, t);
                         }
                     } catch (Throwable t) {
                         logger.warn("failed to notify listener [{}]", t, listener);
                     }
                 }
                 if (listener != null) {
-                    listener.onResponse(snapshot);
+                    listener.onResponse(snapshotInfo);
                 }
             }
         });
     }
 
+    /**
+     * Deletes a snapshot from the repository, looking up the {@link Snapshot} reference before deleting.
+     * If the snapshot is still running cancels the snapshot first and then deletes it from the repository.
+     *
+     * @param repositoryName  repositoryName
+     * @param snapshotName    snapshotName
+     * @param listener        listener
+     */
+    public void deleteSnapshot(final String repositoryName, final String snapshotName, final DeleteSnapshotListener listener) {
+        // First, look for the snapshot in the repository
+        final Repository repository = repositoriesService.repository(repositoryName);
+        Optional<SnapshotId> matchedEntry = repository.snapshots().stream().filter(s -> s.getName().equals(snapshotName)).findFirst();
+        // if nothing found by the same name, then look in the cluster state for current in progress snapshots
+        if (matchedEntry.isPresent() == false) {
+            matchedEntry = currentSnapshots(repositoryName, Collections.emptyList()).stream()
+                               .map(e -> e.snapshot().getSnapshotId()).filter(s -> s.getName().equals(snapshotName)).findFirst();
+        }
+        if (matchedEntry.isPresent() == false) {
+            throw new SnapshotMissingException(repositoryName, snapshotName);
+        }
+        deleteSnapshot(new Snapshot(repositoryName, matchedEntry.get()), listener);
+    }
+
     /**
      * Deletes snapshot from repository.
      * <p>
      * If the snapshot is still running cancels the snapshot first and then deletes it from the repository.
      *
-     * @param snapshotId snapshot id
-     * @param listener   listener
+     * @param snapshot  snapshot
+     * @param listener  listener
      */
-    public void deleteSnapshot(final SnapshotId snapshotId, final DeleteSnapshotListener listener) {
-        validate(snapshotId);
+    public void deleteSnapshot(final Snapshot snapshot, final DeleteSnapshotListener listener) {
         clusterService.submitStateUpdateTask("delete snapshot", new ClusterStateUpdateTask() {
 
             boolean waitForSnapshot = false;
@@ -926,22 +988,22 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
                     // No snapshots running - we can continue
                     return currentState;
                 }
-                SnapshotsInProgress.Entry snapshot = snapshots.snapshot(snapshotId);
-                if (snapshot == null) {
+                SnapshotsInProgress.Entry snapshotEntry = snapshots.snapshot(snapshot);
+                if (snapshotEntry == null) {
                     // This snapshot is not running - continue
                     if (!snapshots.entries().isEmpty()) {
                         // However other snapshots are running - cannot continue
-                        throw new ConcurrentSnapshotExecutionException(snapshotId, "another snapshot is currently running cannot delete");
+                        throw new ConcurrentSnapshotExecutionException(snapshot, "another snapshot is currently running cannot delete");
                     }
                     return currentState;
                 } else {
                     // This snapshot is currently running - stopping shards first
                     waitForSnapshot = true;
                     ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards;
-                    if (snapshot.state() == State.STARTED && snapshot.shards() != null) {
+                    if (snapshotEntry.state() == State.STARTED && snapshotEntry.shards() != null) {
                         // snapshot is currently running - stop started shards
                         ImmutableOpenMap.Builder<ShardId, ShardSnapshotStatus> shardsBuilder = ImmutableOpenMap.builder();
-                        for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shardEntry : snapshot.shards()) {
+                        for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shardEntry : snapshotEntry.shards()) {
                             ShardSnapshotStatus status = shardEntry.value;
                             if (!status.state().completed()) {
                                 shardsBuilder.put(shardEntry.key, new ShardSnapshotStatus(status.nodeId(), State.ABORTED));
@@ -950,14 +1012,14 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
                             }
                         }
                         shards = shardsBuilder.build();
-                    } else if (snapshot.state() == State.INIT) {
+                    } else if (snapshotEntry.state() == State.INIT) {
                         // snapshot hasn't started yet - end it
-                        shards = snapshot.shards();
-                        endSnapshot(snapshot);
+                        shards = snapshotEntry.shards();
+                        endSnapshot(snapshotEntry);
                     } else {
                         boolean hasUncompletedShards = false;
                         // Cleanup in case a node gone missing and snapshot wasn't updated for some reason
-                        for (ObjectCursor<ShardSnapshotStatus> shardStatus : snapshot.shards().values()) {
+                        for (ObjectCursor<ShardSnapshotStatus> shardStatus : snapshotEntry.shards().values()) {
                             // Check if we still have shard running on existing nodes
                             if (shardStatus.value.state().completed() == false && shardStatus.value.nodeId() != null
                                     && currentState.nodes().get(shardStatus.value.nodeId()) != null) {
@@ -972,11 +1034,11 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
                         } else {
                             // no shards to wait for - finish the snapshot
                             logger.debug("trying to delete completed snapshot with no finalizing shards - can delete immediately");
-                            shards = snapshot.shards();
-                            endSnapshot(snapshot);
+                            shards = snapshotEntry.shards();
+                            endSnapshot(snapshotEntry);
                         }
                     }
-                    SnapshotsInProgress.Entry newSnapshot = new SnapshotsInProgress.Entry(snapshot, State.ABORTED, shards);
+                    SnapshotsInProgress.Entry newSnapshot = new SnapshotsInProgress.Entry(snapshotEntry, State.ABORTED, shards);
                     snapshots = new SnapshotsInProgress(newSnapshot);
                     return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, snapshots).build();
                 }
@@ -993,26 +1055,26 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
                     logger.trace("adding snapshot completion listener to wait for deleted snapshot to finish");
                     addListener(new SnapshotCompletionListener() {
                         @Override
-                        public void onSnapshotCompletion(SnapshotId completedSnapshotId, SnapshotInfo snapshot) {
-                            if (completedSnapshotId.equals(snapshotId)) {
+                        public void onSnapshotCompletion(Snapshot completedSnapshot, SnapshotInfo snapshotInfo) {
+                            if (completedSnapshot.equals(snapshot)) {
                                 logger.trace("deleted snapshot completed - deleting files");
                                 removeListener(this);
-                                deleteSnapshotFromRepository(snapshotId, listener);
+                                deleteSnapshotFromRepository(snapshot, listener);
                             }
                         }
 
                         @Override
-                        public void onSnapshotFailure(SnapshotId failedSnapshotId, Throwable t) {
-                            if (failedSnapshotId.equals(snapshotId)) {
+                        public void onSnapshotFailure(Snapshot failedSnapshot, Throwable t) {
+                            if (failedSnapshot.equals(snapshot)) {
                                 logger.trace("deleted snapshot failed - deleting files", t);
                                 removeListener(this);
-                                deleteSnapshotFromRepository(snapshotId, listener);
+                                deleteSnapshotFromRepository(snapshot, listener);
                             }
                         }
                     });
                 } else {
                     logger.trace("deleted snapshot is not running - deleting files");
-                    deleteSnapshotFromRepository(snapshotId, listener);
+                    deleteSnapshotFromRepository(snapshot, listener);
                 }
             }
         });
@@ -1029,7 +1091,7 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
         SnapshotsInProgress snapshots = clusterState.custom(SnapshotsInProgress.TYPE);
         if (snapshots != null) {
             for (SnapshotsInProgress.Entry snapshot : snapshots.entries()) {
-                if (repository.equals(snapshot.snapshotId().getRepository())) {
+                if (repository.equals(snapshot.snapshot().getRepository())) {
                     return true;
                 }
             }
@@ -1040,20 +1102,17 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
     /**
      * Deletes snapshot from repository
      *
-     * @param snapshotId snapshot id
+     * @param snapshot   snapshot
      * @param listener   listener
      */
-    private void deleteSnapshotFromRepository(final SnapshotId snapshotId, final DeleteSnapshotListener listener) {
-        threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new Runnable() {
-            @Override
-            public void run() {
-                try {
-                    Repository repository = repositoriesService.repository(snapshotId.getRepository());
-                    repository.deleteSnapshot(snapshotId);
-                    listener.onResponse();
-                } catch (Throwable t) {
-                    listener.onFailure(t);
-                }
+    private void deleteSnapshotFromRepository(final Snapshot snapshot, final DeleteSnapshotListener listener) {
+        threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> {
+            try {
+                Repository repository = repositoriesService.repository(snapshot.getRepository());
+                repository.deleteSnapshot(snapshot.getSnapshotId());
+                listener.onResponse();
+            } catch (Throwable t) {
+                listener.onFailure(t);
             }
         });
     }
@@ -1203,7 +1262,7 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
     /**
      * Listener for create snapshot operation
      */
-    public static interface CreateSnapshotListener {
+    public interface CreateSnapshotListener {
 
         /**
          * Called when snapshot has successfully started
@@ -1219,7 +1278,7 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
     /**
      * Listener for delete snapshot operation
      */
-    public static interface DeleteSnapshotListener {
+    public interface DeleteSnapshotListener {
 
         /**
          * Called if delete operation was successful
@@ -1232,11 +1291,11 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
         void onFailure(Throwable t);
     }
 
-    public static interface SnapshotCompletionListener {
+    public interface SnapshotCompletionListener {
 
-        void onSnapshotCompletion(SnapshotId snapshotId, SnapshotInfo snapshot);
+        void onSnapshotCompletion(Snapshot snapshot, SnapshotInfo snapshotInfo);
 
-        void onSnapshotFailure(SnapshotId snapshotId, Throwable t);
+        void onSnapshotFailure(Snapshot snapshot, Throwable t);
     }
 
     /**
@@ -1244,11 +1303,11 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
      */
     public static class SnapshotRequest {
 
-        private String cause;
+        private final String cause;
 
-        private String name;
+        private final String repositoryName;
 
-        private String repository;
+        private final String snapshotName;
 
         private String[] indices;
 
@@ -1265,14 +1324,14 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
         /**
          * Constructs new snapshot creation request
          *
-         * @param cause      cause for snapshot operation
-         * @param name       name of the snapshot
-         * @param repository name of the repository
+         * @param repositoryName  repository name
+         * @param snapshotName    snapshot name
+         * @param cause           cause for snapshot operation
          */
-        public SnapshotRequest(String cause, String name, String repository) {
-            this.cause = cause;
-            this.name = name;
-            this.repository = repository;
+        public SnapshotRequest(final String repositoryName, final String snapshotName, final String cause) {
+            this.repositoryName = Objects.requireNonNull(repositoryName);
+            this.snapshotName = Objects.requireNonNull(snapshotName);
+            this.cause = Objects.requireNonNull(cause);
         }
 
         /**
@@ -1351,21 +1410,17 @@ public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsServic
         }
 
         /**
-         * Returns snapshot name
-         *
-         * @return snapshot name
+         * Returns the repository name
          */
-        public String name() {
-            return name;
+        public String repositoryName() {
+            return repositoryName;
         }
 
         /**
-         * Returns snapshot repository
-         *
-         * @return snapshot repository
+         * Returns the snapshot name
          */
-        public String repository() {
-            return repository;
+        public String snapshotName() {
+            return snapshotName;
         }
 
         /**
diff --git a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
index 96f9cbb3522..682eed35f2e 100644
--- a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
+++ b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
@@ -27,13 +27,13 @@ import org.elasticsearch.action.support.replication.ReplicationOperation;
 import org.elasticsearch.client.AbstractClientHeadersTestCase;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.block.ClusterBlockException;
-import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.IllegalShardRoutingStateException;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.breaker.CircuitBreakingException;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -70,7 +70,9 @@ import org.elasticsearch.search.SearchException;
 import org.elasticsearch.search.SearchParseException;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotException;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.TestSearchContext;
 import org.elasticsearch.test.VersionUtils;
@@ -303,14 +305,16 @@ public class ExceptionSerializationTests extends ESTestCase {
     }
 
     public void testSnapshotException() throws IOException {
-        SnapshotException ex = serialize(
-                new SnapshotException(new SnapshotId("repo", "snap"), "no such snapshot", new NullPointerException()));
-        assertEquals(ex.snapshot(), new SnapshotId("repo", "snap"));
-        assertEquals(ex.getMessage(), "[repo:snap] no such snapshot");
+        final Snapshot snapshot = new Snapshot("repo", new SnapshotId("snap", UUIDs.randomBase64UUID()));
+        SnapshotException ex = serialize(new SnapshotException(snapshot, "no such snapshot", new NullPointerException()));
+        assertEquals(ex.getRepositoryName(), snapshot.getRepository());
+        assertEquals(ex.getSnapshotName(), snapshot.getSnapshotId().getName());
+        assertEquals(ex.getMessage(), "[" + snapshot + "] no such snapshot");
         assertTrue(ex.getCause() instanceof NullPointerException);
 
         ex = serialize(new SnapshotException(null, "no such snapshot", new NullPointerException()));
-        assertEquals(ex.snapshot(), null);
+        assertNull(ex.getRepositoryName());
+        assertNull(ex.getSnapshotName());
         assertEquals(ex.getMessage(), "[_na] no such snapshot");
         assertTrue(ex.getCause() instanceof NullPointerException);
     }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java
index 10d8659cf2c..c66fa4b244f 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java
@@ -157,7 +157,7 @@ public class SnapshotBlocksIT extends ESIntegTestCase {
             setClusterReadOnly(true);
             GetSnapshotsResponse response = client().admin().cluster().prepareGetSnapshots(REPOSITORY_NAME).execute().actionGet();
             assertThat(response.getSnapshots(), hasSize(1));
-            assertThat(response.getSnapshots().get(0).name(), equalTo(SNAPSHOT_NAME));
+            assertThat(response.getSnapshots().get(0).snapshotId().getName(), equalTo(SNAPSHOT_NAME));
         } finally {
             setClusterReadOnly(false);
         }
diff --git a/core/src/test/java/org/elasticsearch/bwcompat/RepositoryUpgradabilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/RepositoryUpgradabilityIT.java
new file mode 100644
index 00000000000..e1873850703
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/bwcompat/RepositoryUpgradabilityIT.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.bwcompat;
+
+import org.elasticsearch.common.io.FileTestUtils;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
+import org.elasticsearch.snapshots.SnapshotId;
+import org.elasticsearch.snapshots.SnapshotInfo;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+
+/**
+ * Tests that a repository can handle both snapshots of previous version formats and new version formats,
+ * as blob names and repository blob formats have changed between the snapshot versions.
+ */
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
+public class RepositoryUpgradabilityIT extends AbstractSnapshotIntegTestCase {
+
+    /**
+     * This tests that a repository can inter-operate with snapshots that both have and don't have a UUID,
+     * namely when a repository was created in an older version with snapshots created in the old format
+     * (only snapshot name, no UUID) and then the repository is loaded into newer versions where subsequent
+     * snapshots have a name and a UUID.
+     */
+    public void testRepositoryWorksWithCrossVersions() throws Exception {
+        final List<String> repoVersions = listRepoVersions();
+        // run the test for each supported version
+        for (final String version : repoVersions) {
+            final String repoName = "test-repo-" + version;
+            logger.info("-->  creating repository [{}] for version [{}]", repoName, version);
+            createRepository(version, repoName);
+
+            logger.info("--> get the snapshots");
+            final String originalIndex = "index-" + version;
+            final Set<String> indices = Sets.newHashSet(originalIndex);
+            final Set<SnapshotInfo> snapshotInfos = Sets.newHashSet(getSnapshots(repoName));
+            assertThat(snapshotInfos.size(), equalTo(1));
+            SnapshotInfo originalSnapshot = snapshotInfos.iterator().next();
+            assertThat(originalSnapshot.snapshotId(), equalTo(new SnapshotId("test_1", SnapshotId.UNASSIGNED_UUID)));
+            assertThat(Sets.newHashSet(originalSnapshot.indices()), equalTo(indices));
+
+            logger.info("--> restore the original snapshot");
+            final Set<String> restoredIndices = Sets.newHashSet(
+                restoreSnapshot(repoName, originalSnapshot.snapshotId().getName())
+            );
+            assertThat(restoredIndices, equalTo(indices));
+            // make sure it has documents
+            for (final String searchIdx : restoredIndices) {
+                assertThat(client().prepareSearch(searchIdx).setSize(0).get().getHits().totalHits(), greaterThan(0L));
+            }
+            deleteIndices(restoredIndices); // delete so we can restore again later
+
+            final String snapshotName2 = "test_2";
+            logger.info("--> take a new snapshot of the old index");
+            final int addedDocSize = 10;
+            for (int i = 0; i < addedDocSize; i++) {
+                index(originalIndex, "doc", Integer.toString(i), "foo", "new-bar-" + i);
+            }
+            refresh();
+            snapshotInfos.add(createSnapshot(repoName, snapshotName2));
+
+            logger.info("--> get the snapshots with the newly created snapshot [{}]", snapshotName2);
+            Set<SnapshotInfo> snapshotInfosFromRepo = Sets.newHashSet(getSnapshots(repoName));
+            assertThat(snapshotInfosFromRepo, equalTo(snapshotInfos));
+            snapshotInfosFromRepo.forEach(snapshotInfo -> {
+                assertThat(Sets.newHashSet(snapshotInfo.indices()), equalTo(indices));
+            });
+
+            final String snapshotName3 = "test_3";
+            final String indexName2 = "index2";
+            logger.info("--> take a new snapshot with a new index");
+            createIndex(indexName2);
+            indices.add(indexName2);
+            for (int i = 0; i < addedDocSize; i++) {
+                index(indexName2, "doc", Integer.toString(i), "foo", "new-bar-" + i);
+            }
+            refresh();
+            snapshotInfos.add(createSnapshot(repoName, snapshotName3));
+
+            logger.info("--> get the snapshots with the newly created snapshot [{}]", snapshotName3);
+            snapshotInfosFromRepo = Sets.newHashSet(getSnapshots(repoName));
+            assertThat(snapshotInfosFromRepo, equalTo(snapshotInfos));
+            snapshotInfosFromRepo.forEach(snapshotInfo -> {
+                if (snapshotInfo.snapshotId().getName().equals(snapshotName3)) {
+                    // only the last snapshot has all the indices
+                    assertThat(Sets.newHashSet(snapshotInfo.indices()), equalTo(indices));
+                } else {
+                    assertThat(Sets.newHashSet(snapshotInfo.indices()), equalTo(Sets.newHashSet(originalIndex)));
+                }
+            });
+            deleteIndices(indices); // clean up indices
+
+            logger.info("--> restore the old snapshot again");
+            Set<String> oldRestoredIndices = Sets.newHashSet(restoreSnapshot(repoName, originalSnapshot.snapshotId().getName()));
+            assertThat(oldRestoredIndices, equalTo(Sets.newHashSet(originalIndex)));
+            for (final String searchIdx : oldRestoredIndices) {
+                assertThat(client().prepareSearch(searchIdx).setSize(0).get().getHits().totalHits(),
+                           greaterThanOrEqualTo((long)addedDocSize));
+            }
+            deleteIndices(oldRestoredIndices);
+
+            logger.info("--> restore the new snapshot");
+            Set<String> newSnapshotIndices = Sets.newHashSet(restoreSnapshot(repoName, snapshotName3));
+            assertThat(newSnapshotIndices, equalTo(Sets.newHashSet(originalIndex, indexName2)));
+            for (final String searchIdx : newSnapshotIndices) {
+                assertThat(client().prepareSearch(searchIdx).setSize(0).get().getHits().totalHits(),
+                           greaterThanOrEqualTo((long)addedDocSize));
+            }
+            deleteIndices(newSnapshotIndices); // clean up indices before starting again
+        }
+    }
+
+    private List<String> listRepoVersions() throws Exception {
+        final String prefix = "repo";
+        final List<String> repoVersions = new ArrayList<>();
+        final Path repoFiles = getBwcIndicesPath();
+        try (final DirectoryStream<Path> dirStream = Files.newDirectoryStream(repoFiles, prefix + "-*.zip")) {
+            for (final Path entry : dirStream) {
+                final String fileName = entry.getFileName().toString();
+                String version = fileName.substring(prefix.length() + 1);
+                version = version.substring(0, version.length() - ".zip".length());
+                repoVersions.add(version);
+            }
+        }
+        return Collections.unmodifiableList(repoVersions);
+    }
+
+    private void createRepository(final String version, final String repoName) throws Exception {
+        final String prefix = "repo";
+        final Path repoFile = getBwcIndicesPath().resolve(prefix + "-" + version + ".zip");
+        final Path repoPath = randomRepoPath();
+        FileTestUtils.unzip(repoFile, repoPath, "repo/");
+        assertAcked(client().admin().cluster().preparePutRepository(repoName)
+                                              .setType("fs")
+                                              .setSettings(Settings.builder().put("location", repoPath)));
+    }
+
+    private List<SnapshotInfo> getSnapshots(final String repoName) throws Exception {
+        return client().admin().cluster().prepareGetSnapshots(repoName)
+                                         .addSnapshots("_all")
+                                         .get()
+                                         .getSnapshots();
+    }
+
+    private SnapshotInfo createSnapshot(final String repoName, final String snapshotName) throws Exception {
+        return client().admin().cluster().prepareCreateSnapshot(repoName, snapshotName)
+                                         .setWaitForCompletion(true)
+                                         .get()
+                                         .getSnapshotInfo();
+    }
+
+    private List<String> restoreSnapshot(final String repoName, final String snapshotName) throws Exception {
+        return client().admin().cluster().prepareRestoreSnapshot(repoName, snapshotName)
+                                         .setWaitForCompletion(true)
+                                         .get()
+                                         .getRestoreInfo()
+                                         .indices();
+    }
+
+    private void deleteIndices(final Set<String> indices) throws Exception {
+        client().admin().indices().prepareDelete(indices.toArray(new String[indices.size()])).get();
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java
index ad16267cde9..419104bfe34 100644
--- a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java
+++ b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java
@@ -40,7 +40,6 @@ import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.VersionUtils;
 
 import java.io.IOException;
-import java.lang.reflect.Modifier;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.file.DirectoryStream;
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
index 1d013737a30..b82b5e0ba60 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
@@ -30,7 +30,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.RepositoriesMetaData;
-import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
@@ -51,6 +51,7 @@ import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.Collections;
@@ -653,7 +654,7 @@ public class ClusterStateDiffIT extends ESIntegTestCase {
                 switch (randomIntBetween(0, 1)) {
                     case 0:
                         return new SnapshotsInProgress(new SnapshotsInProgress.Entry(
-                                new SnapshotId(randomName("repo"), randomName("snap")),
+                                new Snapshot(randomName("repo"), new SnapshotId(randomName("snap"), UUIDs.randomBase64UUID())),
                                 randomBoolean(),
                                 randomBoolean(),
                                 SnapshotsInProgress.State.fromValue((byte) randomIntBetween(0, 6)),
@@ -662,7 +663,7 @@ public class ClusterStateDiffIT extends ESIntegTestCase {
                                 ImmutableOpenMap.of()));
                     case 1:
                         return new RestoreInProgress(new RestoreInProgress.Entry(
-                                new SnapshotId(randomName("repo"), randomName("snap")),
+                                new Snapshot(randomName("repo"), new SnapshotId(randomName("snap"), UUIDs.randomBase64UUID())),
                                 RestoreInProgress.State.fromValue((byte) randomIntBetween(0, 3)),
                                 emptyList(),
                                 ImmutableOpenMap.of()));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java
index 31f48a68654..7267252b19f 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java
@@ -20,18 +20,15 @@
 package org.elasticsearch.cluster.routing;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 
-import static org.hamcrest.Matchers.equalTo;
-
 public class ShardRoutingTests extends ESTestCase {
 
     public void testIsSameAllocation() {
@@ -155,8 +152,8 @@ public class ShardRoutingTests extends ESTestCase {
                 case 4:
                     // change restore source
                     otherRouting = TestShardRouting.newShardRouting(otherRouting.getIndexName(), otherRouting.id(), otherRouting.currentNodeId(), otherRouting.relocatingNodeId(),
-                            otherRouting.restoreSource() == null ? new RestoreSource(new SnapshotId("test", "s1"), Version.CURRENT, "test") :
-                                    new RestoreSource(otherRouting.restoreSource().snapshotId(), Version.CURRENT, otherRouting.index() + "_1"),
+                            otherRouting.restoreSource() == null ? new RestoreSource(new Snapshot("test", new SnapshotId("s1", UUIDs.randomBase64UUID())), Version.CURRENT, "test") :
+                                    new RestoreSource(otherRouting.restoreSource().snapshot(), Version.CURRENT, otherRouting.index() + "_1"),
                             otherRouting.primary(), otherRouting.state(), otherRouting.unassignedInfo());
                     break;
                 case 5:
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
index a0ef14ee98d..82703c6f29e 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
@@ -26,7 +26,8 @@ import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
@@ -35,6 +36,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.test.ESAllocationTestCase;
 
 import java.util.Collections;
@@ -130,7 +132,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
                 .build();
         ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
                 .metaData(metaData)
-                .routingTable(RoutingTable.builder().addAsNewRestore(metaData.index("test"), new RestoreSource(new SnapshotId("rep1", "snp1"), Version.CURRENT, "test"), new IntHashSet()).build()).build();
+                .routingTable(RoutingTable.builder().addAsNewRestore(metaData.index("test"), new RestoreSource(new Snapshot("rep1", new SnapshotId("snp1", UUIDs.randomBase64UUID())), Version.CURRENT, "test"), new IntHashSet()).build()).build();
         for (ShardRouting shard : clusterState.getRoutingNodes().shardsWithState(UNASSIGNED)) {
             assertThat(shard.unassignedInfo().getReason(), equalTo(UnassignedInfo.Reason.NEW_INDEX_RESTORED));
         }
@@ -142,7 +144,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
                 .build();
         ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
                 .metaData(metaData)
-                .routingTable(RoutingTable.builder().addAsRestore(metaData.index("test"), new RestoreSource(new SnapshotId("rep1", "snp1"), Version.CURRENT, "test")).build()).build();
+                .routingTable(RoutingTable.builder().addAsRestore(metaData.index("test"), new RestoreSource(new Snapshot("rep1", new SnapshotId("snp1", UUIDs.randomBase64UUID())), Version.CURRENT, "test")).build()).build();
         for (ShardRouting shard : clusterState.getRoutingNodes().shardsWithState(UNASSIGNED)) {
             assertThat(shard.unassignedInfo().getReason(), equalTo(UnassignedInfo.Reason.EXISTING_INDEX_RESTORED));
         }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
index 8eb3e2f8a8b..54df74db1f5 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
@@ -25,7 +25,8 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
@@ -48,6 +49,7 @@ import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.test.ESAllocationTestCase;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.gateway.NoopGatewayAllocator;
@@ -360,7 +362,8 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase {
 
         ClusterState state = ClusterState.builder(ClusterName.DEFAULT)
             .metaData(metaData)
-            .routingTable(RoutingTable.builder().addAsRestore(metaData.index("test"), new RestoreSource(new SnapshotId("rep1", "snp1"),
+            .routingTable(RoutingTable.builder().addAsRestore(metaData.index("test"),
+                new RestoreSource(new Snapshot("rep1", new SnapshotId("snp1", UUIDs.randomBase64UUID())),
                 Version.CURRENT, "test")).build())
             .nodes(DiscoveryNodes.builder().put(newNode).put(oldNode1).put(oldNode2)).build();
         AllocationDeciders allocationDeciders = new AllocationDeciders(Settings.EMPTY, new AllocationDecider[]{
diff --git a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
index ee1cf7280e7..3e088f4f82e 100644
--- a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
@@ -24,7 +24,7 @@ import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RestoreSource;
@@ -40,6 +40,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardStateMetaData;
+import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.test.ESAllocationTestCase;
 import org.junit.Before;
 
@@ -339,8 +340,9 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
                 .putActiveAllocationIds(0, hasActiveAllocation ? Sets.newHashSet("allocId") : Collections.emptySet()))
             .build();
 
+        final Snapshot snapshot = new Snapshot("test", new SnapshotId("test", UUIDs.randomBase64UUID()));
         RoutingTable routingTable = RoutingTable.builder()
-            .addAsRestore(metaData.index(shardId.getIndex()), new RestoreSource(new SnapshotId("test", "test"), version, shardId.getIndexName()))
+            .addAsRestore(metaData.index(shardId.getIndex()), new RestoreSource(snapshot, version, shardId.getIndexName()))
             .build();
         ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
             .metaData(metaData)
@@ -419,7 +421,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
             .build();
 
         RoutingTable routingTable = RoutingTable.builder()
-            .addAsRestore(metaData.index(shardId.getIndex()), new RestoreSource(new SnapshotId("test", "test"), Version.CURRENT, shardId.getIndexName()))
+            .addAsRestore(metaData.index(shardId.getIndex()), new RestoreSource(new Snapshot("test", new SnapshotId("test", UUIDs.randomBase64UUID())), Version.CURRENT, shardId.getIndexName()))
             .build();
         ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
             .metaData(metaData)
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 52aa07b3eb3..7774537c734 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -49,7 +49,8 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.InternalClusterInfoService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.AllocationId;
 import org.elasticsearch.cluster.routing.RestoreSource;
@@ -97,6 +98,7 @@ import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.test.DummyShardLock;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.FieldMaskingReader;
@@ -1110,7 +1112,8 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         client().admin().indices().prepareFlush("test").get(); // only flush test
         final ShardRouting origRouting = test_target.getShardOrNull(0).routingEntry();
         ShardRouting routing = ShardRoutingHelper.reinit(origRouting);
-        routing = ShardRoutingHelper.newWithRestoreSource(routing, new RestoreSource(new SnapshotId("foo", "bar"), Version.CURRENT, "test"));
+        final Snapshot snapshot = new Snapshot("foo", new SnapshotId("bar", UUIDs.randomBase64UUID()));
+        routing = ShardRoutingHelper.newWithRestoreSource(routing, new RestoreSource(snapshot, Version.CURRENT, "test"));
         test_target.removeShard(0, "just do it man!");
         final IndexShard test_target_shard = test_target.createShard(routing);
         Store sourceStore = test_shard.store();
diff --git a/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java b/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
new file mode 100644
index 00000000000..3d46c0bbacf
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.blobstore;
+
+import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
+import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.snapshots.SnapshotId;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.blobId;
+import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.parseNameUUIDFromBlobName;
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * Tests for the {@link BlobStoreRepository} and its subclasses.
+ */
+public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
+
+    public void testRetrieveSnapshots() throws Exception {
+        final Client client = client();
+        final Path location = ESIntegTestCase.randomRepoPath(node().settings());
+        final String repositoryName = "test-repo";
+
+        logger.info("-->  creating repository");
+        PutRepositoryResponse putRepositoryResponse =
+            client.admin().cluster().preparePutRepository(repositoryName)
+                                    .setType("fs")
+                                    .setSettings(Settings.builder().put(node().settings()).put("location", location))
+                                    .get();
+        assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
+
+        logger.info("--> creating an index and indexing documents");
+        final String indexName = "test-idx";
+        createIndex(indexName);
+        ensureGreen();
+        int numDocs = randomIntBetween(10, 20);
+        for (int i = 0; i < numDocs; i++) {
+            String id = Integer.toString(i);
+            client().prepareIndex(indexName, "type1", id).setSource("text", "sometext").get();
+        }
+        client().admin().indices().prepareFlush(indexName).setWaitIfOngoing(true).get();
+
+        logger.info("--> create first snapshot");
+        CreateSnapshotResponse createSnapshotResponse = client.admin()
+                                                              .cluster()
+                                                              .prepareCreateSnapshot(repositoryName, "test-snap-1")
+                                                              .setWaitForCompletion(true)
+                                                              .setIndices(indexName)
+                                                              .get();
+        final SnapshotId snapshotId1 = createSnapshotResponse.getSnapshotInfo().snapshotId();
+
+        logger.info("--> create second snapshot");
+        createSnapshotResponse = client.admin()
+                                       .cluster()
+                                       .prepareCreateSnapshot(repositoryName, "test-snap-2")
+                                       .setWaitForCompletion(true)
+                                       .setIndices(indexName)
+                                       .get();
+        final SnapshotId snapshotId2 = createSnapshotResponse.getSnapshotInfo().snapshotId();
+
+        logger.info("--> make sure the node's repository can resolve the snapshots");
+        final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class);
+        @SuppressWarnings("unchecked") final BlobStoreRepository repository =
+            (BlobStoreRepository) repositoriesService.repository(repositoryName);
+        final List<SnapshotId> originalSnapshots = Arrays.asList(snapshotId1, snapshotId2);
+
+        List<SnapshotId> snapshotIds = repository.snapshots().stream()
+                                                             .sorted((s1, s2) -> s1.getName().compareTo(s2.getName()))
+                                                             .collect(Collectors.toList());
+        assertThat(snapshotIds, equalTo(originalSnapshots));
+    }
+
+    public void testSnapshotIndexFile() throws Exception {
+        final Client client = client();
+        final Path location = ESIntegTestCase.randomRepoPath(node().settings());
+        final String repositoryName = "test-repo";
+
+        PutRepositoryResponse putRepositoryResponse =
+            client.admin().cluster().preparePutRepository(repositoryName)
+                                    .setType("fs")
+                                    .setSettings(Settings.builder().put(node().settings()).put("location", location))
+                                    .get();
+        assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
+
+        final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class);
+        @SuppressWarnings("unchecked") final BlobStoreRepository repository =
+            (BlobStoreRepository) repositoriesService.repository(repositoryName);
+
+        // write to and read from a snapshot file with no entries
+        repository.writeSnapshotList(Collections.emptyList());
+        List<SnapshotId> readSnapshotIds = repository.readSnapshotList();
+        assertThat(readSnapshotIds.size(), equalTo(0));
+
+        // write to and read from a snapshot file with a random number of entries
+        final int numSnapshots = randomIntBetween(1, 1000);
+        final List<SnapshotId> snapshotIds = new ArrayList<>(numSnapshots);
+        for (int i = 0; i < numSnapshots; i++) {
+            snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
+        }
+        repository.writeSnapshotList(snapshotIds);
+        readSnapshotIds = repository.readSnapshotList();
+        assertThat(readSnapshotIds, equalTo(snapshotIds));
+    }
+
+    public void testOldIndexFileFormat() throws Exception {
+        final Client client = client();
+        final Path location = ESIntegTestCase.randomRepoPath(node().settings());
+        final String repositoryName = "test-repo";
+
+        PutRepositoryResponse putRepositoryResponse =
+            client.admin().cluster().preparePutRepository(repositoryName)
+                .setType("fs")
+                .setSettings(Settings.builder().put(node().settings()).put("location", location))
+                .get();
+        assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
+
+        final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class);
+        @SuppressWarnings("unchecked") final BlobStoreRepository repository =
+            (BlobStoreRepository) repositoriesService.repository(repositoryName);
+
+        // write old index file format
+        final int numOldSnapshots = randomIntBetween(1, 50);
+        final List<SnapshotId> snapshotIds = new ArrayList<>();
+        for (int i = 0; i < numOldSnapshots; i++) {
+            snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), SnapshotId.UNASSIGNED_UUID));
+        }
+        writeOldFormat(repository, snapshotIds.stream().map(SnapshotId::getName).collect(Collectors.toList()));
+        List<SnapshotId> readSnapshotIds = repository.readSnapshotList();
+        assertThat(Sets.newHashSet(readSnapshotIds), equalTo(Sets.newHashSet(snapshotIds)));
+
+        // write to and read from a snapshot file with a random number of new entries added
+        final int numSnapshots = randomIntBetween(1, 1000);
+        for (int i = 0; i < numSnapshots; i++) {
+            snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
+        }
+        repository.writeSnapshotList(snapshotIds);
+        readSnapshotIds = repository.readSnapshotList();
+        assertThat(Sets.newHashSet(readSnapshotIds), equalTo(Sets.newHashSet(snapshotIds)));
+    }
+
+    public void testParseUUIDFromBlobName() {
+        String blobStr = "abc123";
+        Tuple<String, String> pair = parseNameUUIDFromBlobName(blobStr);
+        assertThat(pair.v1(), equalTo(blobStr)); // snapshot name
+        assertThat(pair.v2(), equalTo(SnapshotId.UNASSIGNED_UUID)); // snapshot uuid
+        blobStr = "abcefghijklmnopqrstuvwxyz";
+        pair = parseNameUUIDFromBlobName(blobStr);
+        assertThat(pair.v1(), equalTo(blobStr));
+        assertThat(pair.v2(), equalTo(SnapshotId.UNASSIGNED_UUID));
+        blobStr = "abc123-xyz"; // not enough characters after '-' to have a uuid
+        pair = parseNameUUIDFromBlobName(blobStr);
+        assertThat(pair.v1(), equalTo(blobStr));
+        assertThat(pair.v2(), equalTo(SnapshotId.UNASSIGNED_UUID));
+        blobStr = "abc123-a1b2c3d4e5f6g7h8i9j0k1";
+        pair = parseNameUUIDFromBlobName(blobStr);
+        assertThat(pair.v1(), equalTo("abc123"));
+        assertThat(pair.v2(), equalTo("a1b2c3d4e5f6g7h8i9j0k1"));
+    }
+
+    public void testBlobId() {
+        SnapshotId snapshotId = new SnapshotId("abc123", SnapshotId.UNASSIGNED_UUID);
+        assertThat(blobId(snapshotId), equalTo("abc123")); // just the snapshot name
+        snapshotId = new SnapshotId("abc-123", SnapshotId.UNASSIGNED_UUID);
+        assertThat(blobId(snapshotId), equalTo("abc-123")); // just the snapshot name
+        String uuid = UUIDs.randomBase64UUID();
+        snapshotId = new SnapshotId("abc123", uuid);
+        assertThat(blobId(snapshotId), equalTo("abc123-" + uuid)); // snapshot name + '-' + uuid
+        uuid = UUIDs.randomBase64UUID();
+        snapshotId = new SnapshotId("abc-123", uuid);
+        assertThat(blobId(snapshotId), equalTo("abc-123-" + uuid)); // snapshot name + '-' + uuid
+    }
+
+    private void writeOldFormat(final BlobStoreRepository repository, final List<String> snapshotNames) throws Exception {
+        final BytesReference bRef;
+        try (BytesStreamOutput bStream = new BytesStreamOutput()) {
+            try (StreamOutput stream = new OutputStreamStreamOutput(bStream)) {
+                XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream);
+                builder.startObject();
+                builder.startArray("snapshots");
+                for (final String snapshotName : snapshotNames) {
+                    builder.value(snapshotName);
+                }
+                builder.endArray();
+                builder.endObject();
+                builder.close();
+            }
+            bRef = bStream.bytes();
+        }
+        repository.blobContainer().writeBlob(BlobStoreRepository.SNAPSHOTS_FILE, bRef); // write to index file
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java
index 978d226da40..b603ded8697 100644
--- a/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java
@@ -21,7 +21,8 @@ package org.elasticsearch.rest.action.cat;
 
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
-import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.RestoreSource;
 import org.elasticsearch.common.Randomness;
@@ -32,6 +33,7 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.rest.RestController;
+import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
@@ -76,10 +78,9 @@ public class RestRecoveryActionTests extends ESTestCase {
 
             final RestoreSource restoreSource = randomBoolean() ? mock(RestoreSource.class) : null;
             if (restoreSource != null) {
-                final SnapshotId snapshotId = mock(SnapshotId.class);
-                when(snapshotId.getRepository()).thenReturn(randomAsciiOfLength(8));
-                when(snapshotId.getSnapshot()).thenReturn(randomAsciiOfLength(8));
-                when(restoreSource.snapshotId()).thenReturn(snapshotId);
+                final Snapshot snapshot = new Snapshot(randomAsciiOfLength(8),
+                                                       new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
+                when(restoreSource.snapshot()).thenReturn(snapshot);
             }
 
             RecoveryState.Index index = mock(RecoveryState.Index.class);
@@ -166,10 +167,10 @@ public class RestRecoveryActionTests extends ESTestCase {
             assertThat(cells.get(8).value, equalTo(state.getTargetNode().getName()));
             assertThat(
                     cells.get(9).value,
-                    equalTo(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshotId().getRepository()));
+                    equalTo(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshot().getRepository()));
             assertThat(
                     cells.get(10).value,
-                    equalTo(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshotId().getSnapshot()));
+                    equalTo(state.getRestoreSource() == null ? "n/a" : state.getRestoreSource().snapshot().getSnapshotId().getName()));
             assertThat(cells.get(11).value, equalTo(state.getIndex().totalRecoverFiles()));
             assertThat(cells.get(12).value, equalTo(state.getIndex().recoveredFileCount()));
             assertThat(cells.get(13).value, equalTo(percent(state.getIndex().recoveredFilesPercent())));
diff --git a/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
index 102173626f3..50fb3f9074b 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
@@ -25,7 +25,6 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.SnapshotsInProgress;
-import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.PendingClusterTask;
@@ -108,18 +107,29 @@ public abstract class AbstractSnapshotIntegTestCase extends ESIntegTestCase {
         fail("Timeout!!!");
     }
 
-    public SnapshotInfo waitForCompletion(String repository, String snapshot, TimeValue timeout) throws InterruptedException {
+    public SnapshotInfo waitForCompletion(String repository, String snapshotName, TimeValue timeout) throws InterruptedException {
         long start = System.currentTimeMillis();
-        SnapshotId snapshotId = new SnapshotId(repository, snapshot);
         while (System.currentTimeMillis() - start < timeout.millis()) {
-            List<SnapshotInfo> snapshotInfos = client().admin().cluster().prepareGetSnapshots(repository).setSnapshots(snapshot).get().getSnapshots();
+            List<SnapshotInfo> snapshotInfos = client().admin().cluster().prepareGetSnapshots(repository).setSnapshots(snapshotName).get().getSnapshots();
             assertThat(snapshotInfos.size(), equalTo(1));
             if (snapshotInfos.get(0).state().completed()) {
                 // Make sure that snapshot clean up operations are finished
                 ClusterStateResponse stateResponse = client().admin().cluster().prepareState().get();
                 SnapshotsInProgress snapshotsInProgress = stateResponse.getState().custom(SnapshotsInProgress.TYPE);
-                if (snapshotsInProgress == null || snapshotsInProgress.snapshot(snapshotId) == null) {
+                if (snapshotsInProgress == null) {
                     return snapshotInfos.get(0);
+                } else {
+                    boolean found = false;
+                    for (SnapshotsInProgress.Entry entry : snapshotsInProgress.entries()) {
+                        final Snapshot curr = entry.snapshot();
+                        if (curr.getRepository().equals(repository) && curr.getSnapshotId().getName().equals(snapshotName)) {
+                            found = true;
+                            break;
+                        }
+                    }
+                    if (found == false) {
+                        return snapshotInfos.get(0);
+                    }
                 }
             }
             Thread.sleep(100);
@@ -128,12 +138,13 @@ public abstract class AbstractSnapshotIntegTestCase extends ESIntegTestCase {
         return null;
     }
 
-    public static String blockNodeWithIndex(String index) {
-        for(String node : internalCluster().nodesInclude("test-idx")) {
-            ((MockRepository)internalCluster().getInstance(RepositoriesService.class, node).repository("test-repo")).blockOnDataFiles(true);
+    public static String blockNodeWithIndex(final String repositoryName, final String indexName) {
+        for(String node : internalCluster().nodesInclude(indexName)) {
+            ((MockRepository)internalCluster().getInstance(RepositoriesService.class, node).repository(repositoryName))
+                .blockOnDataFiles(true);
             return node;
         }
-        fail("No nodes for the index " + index + " found");
+        fail("No nodes for the index " + indexName + " found");
         return null;
     }
 
@@ -163,8 +174,8 @@ public abstract class AbstractSnapshotIntegTestCase extends ESIntegTestCase {
         }
     }
 
-    public static void unblockNode(String node) {
-        ((MockRepository)internalCluster().getInstance(RepositoriesService.class, node).repository("test-repo")).unblock();
+    public static void unblockNode(final String repository, final String node) {
+        ((MockRepository)internalCluster().getInstance(RepositoriesService.class, node).repository(repository)).unblock();
     }
 
     protected void assertBusyPendingTasks(final String taskPrefix, final int expectedCount) throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
index 34333a583af..3ba6c875b68 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
@@ -313,7 +313,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
         assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
 
         // Pick one node and block it
-        String blockedNode = blockNodeWithIndex("test-idx");
+        String blockedNode = blockNodeWithIndex("test-repo", "test-idx");
 
         logger.info("--> snapshot");
         client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(false).setIndices("test-idx").get();
@@ -322,7 +322,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
         waitForBlock(blockedNode, "test-repo", TimeValue.timeValueSeconds(60));
 
         logger.info("--> execution was blocked on node [{}], shutting it down", blockedNode);
-        unblockNode(blockedNode);
+        unblockNode("test-repo", blockedNode);
 
         logger.info("--> stopping node [{}]", blockedNode);
         stopNode(blockedNode);
@@ -361,7 +361,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
         assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
 
         // Pick one node and block it
-        String blockedNode = blockNodeWithIndex("test-idx");
+        String blockedNode = blockNodeWithIndex("test-repo", "test-idx");
         // Remove it from the list of available nodes
         nodes.remove(blockedNode);
 
@@ -377,7 +377,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
         ListenableActionFuture<DeleteSnapshotResponse> deleteSnapshotResponseFuture = internalCluster().client(nodes.get(0)).admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap").execute();
         // Make sure that abort makes some progress
         Thread.sleep(100);
-        unblockNode(blockedNode);
+        unblockNode("test-repo", blockedNode);
         logger.info("--> stopping node [{}]", blockedNode);
         stopNode(blockedNode);
         try {
diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
index 3996aad25e3..19b46710fea 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -50,7 +50,6 @@ import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
-import org.elasticsearch.cluster.metadata.SnapshotId;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -73,9 +72,11 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Locale;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
@@ -917,7 +918,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
 
         logger.info("--> truncate snapshot file to make it unreadable");
-        Path snapshotPath = repo.resolve("snap-test-snap-1.dat");
+        Path snapshotPath = repo.resolve("snap-test-snap-1-" + createSnapshotResponse.getSnapshotInfo().snapshotId().getUUID() + ".dat");
         try(SeekableByteChannel outChan = Files.newByteChannel(snapshotPath, StandardOpenOption.WRITE)) {
             outChan.truncate(randomInt(10));
         }
@@ -1120,7 +1121,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
 
         // Pick one node and block it
-        String blockedNode = blockNodeWithIndex("test-idx");
+        String blockedNode = blockNodeWithIndex("test-repo", "test-idx");
 
         logger.info("--> snapshot");
         client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(false).setIndices("test-idx").get();
@@ -1133,7 +1134,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         client().admin().indices().prepareUpdateSettings("test-idx").setSettings(excludeSettings).get();
 
         logger.info("--> unblocking blocked node");
-        unblockNode(blockedNode);
+        unblockNode("test-repo", blockedNode);
         logger.info("--> waiting for completion");
         SnapshotInfo snapshotInfo = waitForCompletion("test-repo", "test-snap", TimeValue.timeValueSeconds(600));
         logger.info("Number of failed shards [{}]", snapshotInfo.shardFailures().size());
@@ -1183,7 +1184,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
 
         // Pick one node and block it
-        String blockedNode = blockNodeWithIndex("test-idx");
+        String blockedNode = blockNodeWithIndex("test-repo", "test-idx");
 
         logger.info("--> snapshot");
         client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(false).setIndices("test-idx").get();
@@ -1215,7 +1216,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
                 .setType("fs").setSettings(Settings.builder().put("location", repositoryLocation.resolve("test"))));
 
         logger.info("--> unblocking blocked node");
-        unblockNode(blockedNode);
+        unblockNode("test-repo", blockedNode);
         logger.info("--> waiting for completion");
         SnapshotInfo snapshotInfo = waitForCompletion("test-repo", "test-snap", TimeValue.timeValueSeconds(600));
         logger.info("Number of failed shards [{}]", snapshotInfo.shardFailures().size());
@@ -1436,7 +1437,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
 
         // Pick one node and block it
-        String blockedNode = blockNodeWithIndex("test-idx");
+        String blockedNode = blockNodeWithIndex("test-repo", "test-idx");
 
         logger.info("--> snapshot");
         client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(false).setIndices("test-idx").get();
@@ -1477,7 +1478,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         assertThat(snapshotInfo.state(), equalTo(SnapshotState.IN_PROGRESS));
 
         logger.info("--> unblocking blocked node");
-        unblockNode(blockedNode);
+        unblockNode("test-repo", blockedNode);
 
         snapshotInfo = waitForCompletion("test-repo", "test-snap", TimeValue.timeValueSeconds(600));
         logger.info("Number of failed shards [{}]", snapshotInfo.shardFailures().size());
@@ -1904,7 +1905,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         } finally {
             if (initBlocking) {
                 logger.info("--> unblock running master node");
-                unblockNode(internalCluster().getMasterName());
+                unblockNode("test-repo", internalCluster().getMasterName());
             } else {
                 logger.info("--> unblock all data nodes");
                 unblockAllDataNodes("test-repo");
@@ -1993,14 +1994,17 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         Client client = client();
 
         logger.info("-->  creating repository");
-        assertAcked(client.admin().cluster().preparePutRepository("test-repo")
+        final String repositoryName = "test-repo";
+        assertAcked(client.admin().cluster().preparePutRepository(repositoryName)
                 .setType("mock").setSettings(Settings.builder()
                                 .put("location", randomRepoPath())
                                 .put("compress", randomBoolean())
                                 .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
                 ));
 
-        createIndex("test-idx");
+        logger.info("--> create the index");
+        final String idxName = "test-idx";
+        createIndex(idxName);
         ensureGreen();
 
         ClusterService clusterService = internalCluster().getInstance(ClusterService.class, internalCluster().getMasterName());
@@ -2008,7 +2012,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         final CountDownLatch countDownLatch = new CountDownLatch(1);
 
         logger.info("--> snapshot");
-        CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get();
+        final String snapshotName = "test-snap";
+        CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshotName).setWaitForCompletion(true).setIndices(idxName).get();
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
 
@@ -2020,11 +2025,18 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
             public ClusterState execute(ClusterState currentState) {
                 // Simulate orphan snapshot
                 ImmutableOpenMap.Builder<ShardId, ShardSnapshotStatus> shards = ImmutableOpenMap.builder();
-                shards.put(new ShardId("test-idx", "_na_", 0), new ShardSnapshotStatus("unknown-node", State.ABORTED));
-                shards.put(new ShardId("test-idx", "_na_", 1), new ShardSnapshotStatus("unknown-node", State.ABORTED));
-                shards.put(new ShardId("test-idx", "_na_", 2), new ShardSnapshotStatus("unknown-node", State.ABORTED));
+                shards.put(new ShardId(idxName, "_na_", 0), new ShardSnapshotStatus("unknown-node", State.ABORTED));
+                shards.put(new ShardId(idxName, "_na_", 1), new ShardSnapshotStatus("unknown-node", State.ABORTED));
+                shards.put(new ShardId(idxName, "_na_", 2), new ShardSnapshotStatus("unknown-node", State.ABORTED));
                 List<Entry> entries = new ArrayList<>();
-                entries.add(new Entry(new SnapshotId("test-repo", "test-snap"), true, false, State.ABORTED, Collections.singletonList("test-idx"), System.currentTimeMillis(), shards.build()));
+                entries.add(new Entry(new Snapshot(repositoryName,
+                                                   createSnapshotResponse.getSnapshotInfo().snapshotId()),
+                                                   true,
+                                                   false,
+                                                   State.ABORTED,
+                                                   Collections.singletonList(idxName),
+                                                   System.currentTimeMillis(),
+                                                   shards.build()));
                 return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, new SnapshotsInProgress(Collections.unmodifiableList(entries))).build();
             }
 
@@ -2042,8 +2054,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         countDownLatch.await();
         logger.info("--> try deleting the orphan snapshot");
 
-        assertAcked(client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap").get("10s"));
-
+        assertAcked(client.admin().cluster().prepareDeleteSnapshot(repositoryName, snapshotName).get("10s"));
     }
 
     private boolean waitForIndex(final String index, TimeValue timeout) throws InterruptedException {
@@ -2143,33 +2154,14 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
                         .put("compress", randomBoolean())
                         .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
 
-        try {
-            client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("_foo").get();
-            fail("shouldn't be here");
-        } catch (InvalidSnapshotNameException ex) {
-            assertThat(ex.getMessage(), containsString("Invalid snapshot name"));
-        }
-
-        try {
-            client.admin().cluster().prepareCreateSnapshot("test-repo", "_foo").get();
-            fail("shouldn't be here");
-        } catch (InvalidSnapshotNameException ex) {
-            assertThat(ex.getMessage(), containsString("Invalid snapshot name"));
-        }
-
-        try {
-            client.admin().cluster().prepareDeleteSnapshot("test-repo", "_foo").get();
-            fail("shouldn't be here");
-        } catch (InvalidSnapshotNameException ex) {
-            assertThat(ex.getMessage(), containsString("Invalid snapshot name"));
-        }
-
-        try {
-            client.admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("_foo").get();
-            fail("shouldn't be here");
-        } catch (InvalidSnapshotNameException ex) {
-            assertThat(ex.getMessage(), containsString("Invalid snapshot name"));
-        }
+        expectThrows(InvalidSnapshotNameException.class,
+                     () -> client.admin().cluster().prepareCreateSnapshot("test-repo", "_foo").get());
+        expectThrows(SnapshotMissingException.class,
+                     () -> client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("_foo").get());
+        expectThrows(SnapshotMissingException.class,
+                     () -> client.admin().cluster().prepareDeleteSnapshot("test-repo", "_foo").get());
+        expectThrows(SnapshotMissingException.class,
+                     () -> client.admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("_foo").get());
     }
 
     public void testListCorruptedSnapshot() throws Exception {
@@ -2199,7 +2191,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
 
         logger.info("--> truncate snapshot file to make it unreadable");
-        Path snapshotPath = repo.resolve("snap-test-snap-2.dat");
+        Path snapshotPath = repo.resolve("snap-test-snap-2-" + createSnapshotResponse.getSnapshotInfo().snapshotId().getUUID() + ".dat");
         try(SeekableByteChannel outChan = Files.newByteChannel(snapshotPath, StandardOpenOption.WRITE)) {
             outChan.truncate(randomInt(10));
         }
@@ -2211,13 +2203,194 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         assertThat(snapshotInfos.size(), equalTo(1));
         assertThat(snapshotInfos.get(0).state(), equalTo(SnapshotState.SUCCESS));
-        assertThat(snapshotInfos.get(0).name(), equalTo("test-snap-1"));
+        assertThat(snapshotInfos.get(0).snapshotId().getName(), equalTo("test-snap-1"));
 
         try {
             client.admin().cluster().prepareGetSnapshots("test-repo").setIgnoreUnavailable(false).get().getSnapshots();
         } catch (SnapshotException ex) {
-            assertThat(ex.snapshot().getRepository(), equalTo("test-repo"));
-            assertThat(ex.snapshot().getSnapshot(), equalTo("test-snap-2"));
+            assertThat(ex.getRepositoryName(), equalTo("test-repo"));
+            assertThat(ex.getSnapshotName(), equalTo("test-snap-2"));
         }
     }
+
+    public void testCannotCreateSnapshotsWithSameName() throws Exception {
+        final String repositoryName = "test-repo";
+        final String snapshotName = "test-snap";
+        final String indexName = "test-idx";
+        final Client client = client();
+        final Path repo = randomRepoPath();
+
+        logger.info("-->  creating repository at {}", repo.toAbsolutePath());
+        assertAcked(client.admin().cluster().preparePutRepository(repositoryName)
+                                            .setType("fs").setSettings(Settings.builder()
+                                                            .put("location", repo)
+                                                            .put("compress", false)
+                                                            .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
+        logger.info("--> creating an index and indexing documents");
+        createIndex(indexName);
+        ensureGreen();
+        for (int i = 0; i < 10; i++) {
+            index(indexName, "doc", Integer.toString(i), "foo", "bar" + i);
+        }
+        refresh();
+
+        logger.info("--> take first snapshot");
+        CreateSnapshotResponse createSnapshotResponse = client.admin()
+                                                              .cluster()
+                                                              .prepareCreateSnapshot(repositoryName, snapshotName)
+                                                              .setWaitForCompletion(true)
+                                                              .setIndices(indexName)
+                                                              .get();
+        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
+        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(),
+                   equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
+
+        logger.info("--> index more documents");
+        for (int i = 10; i < 20; i++) {
+            index(indexName, "doc", Integer.toString(i), "foo", "bar" + i);
+        }
+        refresh();
+
+        logger.info("--> second snapshot of the same name should fail");
+        try {
+            createSnapshotResponse = client.admin()
+                                           .cluster()
+                                           .prepareCreateSnapshot(repositoryName, snapshotName)
+                                           .setWaitForCompletion(true)
+                                           .setIndices(indexName)
+                                           .get();
+            fail("should not be allowed to create a snapshot with the same name as an already existing snapshot: " +
+                 createSnapshotResponse.getSnapshotInfo().snapshotId());
+        } catch (SnapshotCreationException e) {
+            assertThat(e.getMessage(), containsString("snapshot with the same name already exists"));
+        }
+
+        logger.info("--> delete the first snapshot");
+        client.admin().cluster().prepareDeleteSnapshot(repositoryName, snapshotName).get();
+
+        logger.info("--> try creating a snapshot with the same name, now it should work because the first one was deleted");
+        createSnapshotResponse = client.admin()
+                                       .cluster()
+                                       .prepareCreateSnapshot(repositoryName, snapshotName)
+                                       .setWaitForCompletion(true)
+                                       .setIndices(indexName)
+                                       .get();
+        assertThat(createSnapshotResponse.getSnapshotInfo().snapshotId().getName(), equalTo(snapshotName));
+    }
+
+    public void testGetSnapshotsRequest() throws Exception {
+        final String repositoryName = "test-repo";
+        final String indexName = "test-idx";
+        final Client client = client();
+        final Path repo = randomRepoPath();
+
+        logger.info("-->  creating repository at {}", repo.toAbsolutePath());
+        assertAcked(client.admin().cluster().preparePutRepository(repositoryName)
+                          .setType("mock").setSettings(Settings.builder()
+                                                               .put("location", repo)
+                                                               .put("compress", false)
+                                                               .put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
+                                                               .put("wait_after_unblock", 200)));
+
+        logger.info("--> get snapshots on an empty repository");
+        expectThrows(SnapshotMissingException.class, () -> client.admin()
+                                                                 .cluster()
+                                                                 .prepareGetSnapshots(repositoryName)
+                                                                 .addSnapshots("non-existent-snapshot")
+                                                                 .get());
+        // with ignore unavailable set to true, should not throw an exception
+        GetSnapshotsResponse getSnapshotsResponse = client.admin()
+                                                          .cluster()
+                                                          .prepareGetSnapshots(repositoryName)
+                                                          .setIgnoreUnavailable(true)
+                                                          .addSnapshots("non-existent-snapshot")
+                                                          .get();
+        assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(0));
+
+        logger.info("--> creating an index and indexing documents");
+        // Create index on 2 nodes and make sure each node has a primary by setting no replicas
+        assertAcked(prepareCreate(indexName, 1, Settings.builder().put("number_of_replicas", 0)));
+        ensureGreen();
+        for (int i = 0; i < 10; i++) {
+            index(indexName, "doc", Integer.toString(i), "foo", "bar" + i);
+        }
+        refresh();
+
+        final int numSnapshots = randomIntBetween(1, 3) + 1;
+        logger.info("--> take {} snapshot(s)", numSnapshots);
+        final String[] snapshotNames = new String[numSnapshots];
+        for (int i = 0; i < numSnapshots - 1; i++) {
+            final String snapshotName = randomAsciiOfLength(8).toLowerCase(Locale.ROOT);
+            CreateSnapshotResponse createSnapshotResponse = client.admin()
+                                                                  .cluster()
+                                                                  .prepareCreateSnapshot(repositoryName, snapshotName)
+                                                                  .setWaitForCompletion(true)
+                                                                  .setIndices(indexName)
+                                                                  .get();
+            assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
+            snapshotNames[i] = snapshotName;
+        }
+        logger.info("--> take another snapshot to be in-progress");
+        // add documents so there are data files to block on
+        for (int i = 10; i < 20; i++) {
+            index(indexName, "doc", Integer.toString(i), "foo", "bar" + i);
+        }
+        refresh();
+
+        final String inProgressSnapshot = randomAsciiOfLength(8).toLowerCase(Locale.ROOT);
+        snapshotNames[numSnapshots - 1] = inProgressSnapshot;
+        // block a node so the create snapshot operation can remain in progress
+        final String blockedNode = blockNodeWithIndex(repositoryName, indexName);
+        client.admin().cluster().prepareCreateSnapshot(repositoryName, inProgressSnapshot)
+                                .setWaitForCompletion(false)
+                                .setIndices(indexName)
+                                .get();
+        waitForBlock(blockedNode, repositoryName, TimeValue.timeValueSeconds(60)); // wait for block to kick in
+
+        logger.info("--> get all snapshots with a current in-progress");
+        // with ignore unavailable set to true, should not throw an exception
+        getSnapshotsResponse = client.admin().cluster()
+                                             .prepareGetSnapshots(repositoryName)
+                                             .addSnapshots("_all")
+                                             .get();
+        List<String> sortedNames = Arrays.asList(snapshotNames);
+        Collections.sort(sortedNames);
+        assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(numSnapshots));
+        assertThat(getSnapshotsResponse.getSnapshots().stream()
+                                                      .map(s -> s.snapshotId().getName())
+                                                      .sorted()
+                                                      .collect(Collectors.toList()), equalTo(sortedNames));
+
+        getSnapshotsResponse = client.admin().cluster()
+                                   .prepareGetSnapshots(repositoryName)
+                                   .addSnapshots(snapshotNames)
+                                   .get();
+        sortedNames = Arrays.asList(snapshotNames);
+        Collections.sort(sortedNames);
+        assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(numSnapshots));
+        assertThat(getSnapshotsResponse.getSnapshots().stream()
+                                                      .map(s -> s.snapshotId().getName())
+                                                      .sorted()
+                                                      .collect(Collectors.toList()), equalTo(sortedNames));
+
+        logger.info("--> make sure duplicates are not returned in the response");
+        String regexName = snapshotNames[randomIntBetween(0, numSnapshots - 1)];
+        final int splitPos = regexName.length() / 2;
+        final String firstRegex = regexName.substring(0, splitPos) + "*";
+        final String secondRegex = "*" + regexName.substring(splitPos);
+        getSnapshotsResponse = client.admin().cluster()
+                                             .prepareGetSnapshots(repositoryName)
+                                             .addSnapshots(snapshotNames)
+                                             .addSnapshots(firstRegex, secondRegex)
+                                             .get();
+        assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(numSnapshots));
+        assertThat(getSnapshotsResponse.getSnapshots().stream()
+                                                      .map(s -> s.snapshotId().getName())
+                                                      .sorted()
+                                                      .collect(Collectors.toList()), equalTo(sortedNames));
+
+        unblockNode(repositoryName, blockedNode); // unblock node
+        waitForCompletion(repositoryName, inProgressSnapshot, TimeValue.timeValueSeconds(60));
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/snapshots/SnapshotBackwardsCompatibilityIT.java b/core/src/test/java/org/elasticsearch/snapshots/SnapshotBackwardsCompatibilityIT.java
index e6379468382..31c0a193f07 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/SnapshotBackwardsCompatibilityIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/SnapshotBackwardsCompatibilityIT.java
@@ -86,7 +86,7 @@ public class SnapshotBackwardsCompatibilityIT extends ESBackcompatTestCase {
             counts[i] = client().prepareSearch(indices[i]).setSize(0).get().getHits().totalHits();
         }
 
-        logger.info("--> snapshot subset of indices before upgrage");
+        logger.info("--> snapshot subset of indices before upgrade");
         CreateSnapshotResponse createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1").setWaitForCompletion(true).setIndices("index_before_*").get();
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
         assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
diff --git a/core/src/test/java/org/elasticsearch/snapshots/SnapshotTests.java b/core/src/test/java/org/elasticsearch/snapshots/SnapshotTests.java
new file mode 100644
index 00000000000..cb297785e4b
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/snapshots/SnapshotTests.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.snapshots;
+
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+
+/**
+ * Tests for the {@link Snapshot} class.
+ */
+public class SnapshotTests extends ESTestCase {
+
+    public void testSnapshotEquals() {
+        final SnapshotId snapshotId = new SnapshotId("snap", UUIDs.randomBase64UUID());
+        final Snapshot original = new Snapshot("repo", snapshotId);
+        final Snapshot expected = new Snapshot(original.getRepository(), original.getSnapshotId());
+        assertThat(expected, equalTo(original));
+        assertThat(expected.getRepository(), equalTo(original.getRepository()));
+        assertThat(expected.getSnapshotId(), equalTo(original.getSnapshotId()));
+        assertThat(expected.getSnapshotId().getName(), equalTo(original.getSnapshotId().getName()));
+        assertThat(expected.getSnapshotId().getUUID(), equalTo(original.getSnapshotId().getUUID()));
+    }
+
+    public void testSerialization() throws IOException {
+        final SnapshotId snapshotId = new SnapshotId(randomAsciiOfLength(randomIntBetween(2, 8)), UUIDs.randomBase64UUID());
+        final Snapshot original = new Snapshot(randomAsciiOfLength(randomIntBetween(2, 8)), snapshotId);
+        final BytesStreamOutput out = new BytesStreamOutput();
+        original.writeTo(out);
+        final ByteBufferStreamInput in = new ByteBufferStreamInput(ByteBuffer.wrap(out.bytes().toBytes()));
+        assertThat(new Snapshot(in), equalTo(original));
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java b/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
index 7df44738076..8b02f90f51c 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
@@ -21,7 +21,7 @@ package org.elasticsearch.snapshots.mockstore;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobMetaData;
@@ -174,6 +174,10 @@ public class MockRepository extends FsRepository {
         blockOnControlFiles = blocked;
     }
 
+    public boolean blockOnDataFiles() {
+        return blockOnDataFiles;
+    }
+
     public synchronized void unblockExecution() {
         blocked = false;
         // Clean blocking flags, so we wouldn't try to block again
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
index 8af614df605..4d3459cdcd4 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
@@ -24,7 +24,7 @@ import com.microsoft.azure.storage.StorageException;
 import org.elasticsearch.cloud.azure.blobstore.AzureBlobStore;
 import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
@@ -166,7 +166,7 @@ public class AzureRepository extends BlobStoreRepository {
             super.initializeSnapshot(snapshotId, indices, metaData);
         } catch (StorageException | URISyntaxException e) {
             logger.warn("can not initialize container [{}]: [{}]", blobStore.container(), e.getMessage());
-            throw new SnapshotCreationException(snapshotId, e);
+            throw new SnapshotCreationException(repositoryName, snapshotId, e);
         }
     }
 
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java
index 5a1c76df413..8160c560325 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java
@@ -94,8 +94,15 @@ public class AzureStorageServiceMock extends AbstractLifecycleComponent<AzureSto
     public Map<String, BlobMetaData> listBlobsByPrefix(String account, LocationMode mode, String container, String keyPath, String prefix) {
         MapBuilder<String, BlobMetaData> blobsBuilder = MapBuilder.newMapBuilder();
         for (String blobName : blobs.keySet()) {
-            if (startsWithIgnoreCase(blobName, prefix)) {
-                blobsBuilder.put(blobName, new PlainBlobMetaData(blobName, blobs.get(blobName).size()));
+            final String checkBlob;
+            if (keyPath != null) {
+                // strip off key path from the beginning of the blob name
+                checkBlob = blobName.replace(keyPath, "");
+            } else {
+                checkBlob = blobName;
+            }
+            if (startsWithIgnoreCase(checkBlob, prefix)) {
+                blobsBuilder.put(blobName, new PlainBlobMetaData(checkBlob, blobs.get(blobName).size()));
             }
         }
         return blobsBuilder.immutableMap();
diff --git a/test/framework/src/main/java/org/elasticsearch/common/io/FileTestUtils.java b/test/framework/src/main/java/org/elasticsearch/common/io/FileTestUtils.java
index 50d677c600c..98155967514 100644
--- a/test/framework/src/main/java/org/elasticsearch/common/io/FileTestUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/common/io/FileTestUtils.java
@@ -19,11 +19,15 @@
 
 package org.elasticsearch.common.io;
 
+import org.elasticsearch.common.Nullable;
 import org.junit.Assert;
 
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists;
 import static org.hamcrest.CoreMatchers.equalTo;
@@ -50,4 +54,44 @@ public class FileTestUtils {
             Assert.assertThat(fileContent.trim(), equalTo(expected.trim()));
         }
     }
+
+    /**
+     * Unzip a zip file to a destination directory.  If the zip file does not exist, an IOException is thrown.
+     * If the destination directory does not exist, it will be created.
+     *
+     * @param zip      zip file to unzip
+     * @param destDir  directory to unzip the file to
+     * @param prefixToRemove  the (optional) prefix in the zip file path to remove when writing to the destination directory
+     * @throws IOException if zip file does not exist, or there was an error reading from the zip file or
+     *                     writing to the destination directory
+     */
+    public static void unzip(final Path zip, final Path destDir, @Nullable final String prefixToRemove) throws IOException {
+        if (Files.notExists(zip)) {
+            throw new IOException("[" + zip + "] zip file must exist");
+        }
+        Files.createDirectories(destDir);
+
+        try (final ZipInputStream zipInput = new ZipInputStream(Files.newInputStream(zip))) {
+            ZipEntry entry;
+            while ((entry = zipInput.getNextEntry()) != null) {
+                final String entryPath;
+                if (prefixToRemove != null) {
+                    if (entry.getName().startsWith(prefixToRemove)) {
+                        entryPath = entry.getName().substring(prefixToRemove.length());
+                    } else {
+                        throw new IOException("prefix not found: " + prefixToRemove);
+                    }
+                } else {
+                    entryPath = entry.getName();
+                }
+                final Path path = Paths.get(destDir.toString(), entryPath);
+                if (entry.isDirectory()) {
+                    Files.createDirectories(path);
+                } else {
+                    Files.copy(zipInput, path);
+                }
+                zipInput.closeEntry();
+            }
+        }
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
index c86c111f645..f4d5b4dd482 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
@@ -54,6 +54,7 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
+import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -171,14 +172,15 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
     }
 
     private Node newNode() {
+        final Path tempDir = createTempDir();
         Settings settings = Settings.builder()
             .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", randomLong()))
-            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+            .put(Environment.PATH_HOME_SETTING.getKey(), tempDir)
+            .put(Environment.PATH_REPO_SETTING.getKey(), tempDir.resolve("repo"))
             // TODO: use a consistent data path for custom paths
             // This needs to tie into the ESIntegTestCase#indexSettings() method
             .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), createTempDir().getParent())
             .put("node.name", nodeName())
-
             .put("script.inline", "true")
             .put("script.stored", "true")
             .put(EsExecutors.PROCESSORS_SETTING.getKey(), 1) // limit the number of threads created

From b5393ce55e0cfc95dcb85f5bf1f621b59c2264ea Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Thu, 2 Jun 2016 18:23:47 -0400
Subject: [PATCH 0572/1311] Improve painless compile-time exceptions

---
 .../painless/AnalyzerCaster.java              |  15 +-
 .../org/elasticsearch/painless/Compiler.java  |   4 +-
 .../elasticsearch/painless/Definition.java    |   5 +
 .../org/elasticsearch/painless/Location.java  | 111 ++++++++++++
 .../elasticsearch/painless/MethodWriter.java  |  19 +-
 .../painless/PainlessScriptEngineService.java |  69 +++++++-
 .../elasticsearch/painless/ScriptImpl.java    |   3 +-
 .../org/elasticsearch/painless/Variables.java |  52 +++---
 .../org/elasticsearch/painless/Writer.java    |  45 +----
 .../painless/antlr/ErrorHandlingLexer.java    |  15 +-
 .../painless/antlr/ParserErrorStrategy.java   |  34 ++--
 .../elasticsearch/painless/antlr/Walker.java  | 163 +++++++++---------
 .../painless/node/AExpression.java            |  23 +--
 .../elasticsearch/painless/node/ALink.java    |   5 +-
 .../elasticsearch/painless/node/ANode.java    |  27 +--
 .../painless/node/AStatement.java             |   5 +-
 .../elasticsearch/painless/node/EBinary.java  |  97 ++++++-----
 .../elasticsearch/painless/node/EBool.java    |  11 +-
 .../elasticsearch/painless/node/EBoolean.java |   7 +-
 .../elasticsearch/painless/node/ECast.java    |   9 +-
 .../elasticsearch/painless/node/EChain.java   |  38 ++--
 .../elasticsearch/painless/node/EComp.java    |  55 +++---
 .../painless/node/EConditional.java           |   9 +-
 .../painless/node/EConstant.java              |  11 +-
 .../elasticsearch/painless/node/EDecimal.java |  11 +-
 .../painless/node/EExplicit.java              |   9 +-
 .../painless/node/EFunctionRef.java           |  10 +-
 .../elasticsearch/painless/node/ENull.java    |   7 +-
 .../elasticsearch/painless/node/ENumeric.java |  19 +-
 .../elasticsearch/painless/node/EUnary.java   |  25 +--
 .../painless/node/LArrayLength.java           |  15 +-
 .../elasticsearch/painless/node/LBrace.java   |  19 +-
 .../elasticsearch/painless/node/LCall.java    |  19 +-
 .../elasticsearch/painless/node/LCast.java    |  15 +-
 .../painless/node/LDefArray.java              |   9 +-
 .../elasticsearch/painless/node/LDefCall.java |   9 +-
 .../painless/node/LDefField.java              |   9 +-
 .../elasticsearch/painless/node/LField.java   |  27 +--
 .../painless/node/LListShortcut.java          |  17 +-
 .../painless/node/LMapShortcut.java           |  17 +-
 .../painless/node/LNewArray.java              |  17 +-
 .../elasticsearch/painless/node/LNewObj.java  |  21 +--
 .../painless/node/LShortcut.java              |  17 +-
 .../elasticsearch/painless/node/LStatic.java  |  15 +-
 .../elasticsearch/painless/node/LString.java  |  13 +-
 .../painless/node/LVariable.java              |   9 +-
 .../elasticsearch/painless/node/SBlock.java   |   9 +-
 .../elasticsearch/painless/node/SBreak.java   |   7 +-
 .../elasticsearch/painless/node/SCatch.java   |  11 +-
 .../painless/node/SContinue.java              |   9 +-
 .../painless/node/SDeclBlock.java             |   5 +-
 .../painless/node/SDeclaration.java           |  11 +-
 .../org/elasticsearch/painless/node/SDo.java  |  15 +-
 .../painless/node/SExpression.java            |   9 +-
 .../org/elasticsearch/painless/node/SFor.java |  23 +--
 .../org/elasticsearch/painless/node/SIf.java  |  11 +-
 .../elasticsearch/painless/node/SIfElse.java  |  13 +-
 .../elasticsearch/painless/node/SReturn.java  |   7 +-
 .../elasticsearch/painless/node/SSource.java  |  10 +-
 .../elasticsearch/painless/node/SThrow.java   |   7 +-
 .../org/elasticsearch/painless/node/STry.java |   9 +-
 .../elasticsearch/painless/node/SWhile.java   |  17 +-
 .../painless/node/package-info.java           |  16 +-
 .../painless/ConditionalTests.java            |  20 +--
 .../elasticsearch/painless/DivisionTests.java |   4 +-
 .../painless/FunctionRefTests.java            |   2 +-
 .../elasticsearch/painless/OverloadTests.java |   2 +-
 .../painless/RemainderTests.java              |  14 +-
 .../painless/ReservedWordTests.java           |  16 +-
 .../painless/WhenThingsGoWrongTests.java      |   8 +-
 .../painless/antlr/ParserTests.java           |   4 +-
 .../rest-api-spec/test/plan_a/16_update2.yaml |   2 +-
 72 files changed, 791 insertions(+), 631 deletions(-)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/Location.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
index a16a9818ab3..6e542d6aca9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
@@ -22,6 +22,7 @@ package org.elasticsearch.painless;
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.node.ANode;
 
 /**
  * Used during the analysis phase to collect legal type casts and promotions
@@ -29,7 +30,7 @@ import org.elasticsearch.painless.Definition.Type;
  */
 public final class AnalyzerCaster {
 
-    public static Cast getLegalCast(String location, Type actual, Type expected, boolean explicit, boolean internal) {
+    public static Cast getLegalCast(Location location, Type actual, Type expected, boolean explicit, boolean internal) {
         if (actual.equals(expected)) {
             return null;
         }
@@ -653,11 +654,11 @@ public final class AnalyzerCaster {
             explicit && actual.clazz.isAssignableFrom(expected.clazz)) {
             return new Cast(actual, expected, explicit);
         } else {
-            throw new ClassCastException("Error" + location + ": Cannot cast from [" + actual.name + "] to [" + expected.name + "].");
+            throw location.createError(new ClassCastException("Cannot cast from [" + actual.name + "] to [" + expected.name + "]."));
         }
     }
 
-    public static Object constCast(final String location, final Object constant, final Cast cast) {
+    public static Object constCast(Location location, final Object constant, final Cast cast) {
         final Sort fsort = cast.from.sort;
         final Sort tsort = cast.to.sort;
 
@@ -685,12 +686,12 @@ public final class AnalyzerCaster {
                 case FLOAT:  return number.floatValue();
                 case DOUBLE: return number.doubleValue();
                 default:
-                    throw new IllegalStateException("Error" + location + ": Cannot cast from " +
-                        "[" + cast.from.clazz.getCanonicalName() + "] to [" + cast.to.clazz.getCanonicalName() + "].");
+                    throw location.createError(new IllegalStateException("Cannot cast from " +
+                        "[" + cast.from.clazz.getCanonicalName() + "] to [" + cast.to.clazz.getCanonicalName() + "]."));
             }
         } else {
-            throw new IllegalStateException("Error" + location + ": Cannot cast from " +
-                "[" + cast.from.clazz.getCanonicalName() + "] to [" + cast.to.clazz.getCanonicalName() + "].");
+            throw location.createError(new IllegalStateException("Cannot cast from " +
+                "[" + cast.from.clazz.getCanonicalName() + "] to [" + cast.to.clazz.getCanonicalName() + "]."));
         }
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
index 36d41056013..a780088a222 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
@@ -101,7 +101,7 @@ final class Compiler {
         }
 
         Reserved reserved = new Reserved();
-        SSource root = Walker.buildPainlessTree(source, reserved, settings);
+        SSource root = Walker.buildPainlessTree(name, source, reserved, settings);
         Variables variables = Analyzer.analyze(reserved, root);
         BitSet expressions = new BitSet(source.length());
 
@@ -132,7 +132,7 @@ final class Compiler {
         }
 
         Reserved reserved = new Reserved();
-        SSource root = Walker.buildPainlessTree(source, reserved, settings);
+        SSource root = Walker.buildPainlessTree(name, source, reserved, settings);
         Variables variables = Analyzer.analyze(reserved, root);
 
         return Writer.write(settings, name, source, variables, root, new BitSet(source.length()));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 669323da803..50ee4649074 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -164,6 +164,11 @@ public final class Definition {
 
             return result;
         }
+
+        @Override
+        public String toString() {
+            return name;
+        }
     }
 
     public static final class Constructor {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Location.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Location.java
new file mode 100644
index 00000000000..eb1df00edd2
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Location.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import java.util.Objects;
+
+/**
+ * Represents a location in script code (name of script + character offset)
+ */
+public final class Location {
+    private final String sourceName;
+    private final int offset;
+    
+    /**
+     * Create a new Location 
+     * @param sourceName script's name
+     * @param offset character offset of script element
+     */
+    public Location(String sourceName, int offset) {
+        this.sourceName = Objects.requireNonNull(sourceName);
+        this.offset = offset;
+    }
+    
+    /**
+     * Return the script's name
+     */
+    public String getSourceName() {
+        return sourceName;
+    }
+
+    /**
+     * Return the character offset
+     */
+    public int getOffset() {
+        return offset;
+    }
+
+    /**
+     * Augments an exception with this location's information.
+     */
+    public RuntimeException createError(RuntimeException exception) {
+        StackTraceElement element = new StackTraceElement(WriterConstants.CLASS_NAME, "compile", sourceName, offset + 1);
+        StackTraceElement[] oldStack = exception.getStackTrace();
+        StackTraceElement[] newStack = new StackTraceElement[oldStack.length + 1];
+        System.arraycopy(oldStack, 0, newStack, 1, oldStack.length);
+        newStack[0] = element;
+        exception.setStackTrace(newStack);
+        assert exception.getStackTrace().length == newStack.length : "non-writeable stacktrace for exception: " + exception.getClass();
+        return exception;
+    }
+
+    // This maximum length is theoretically 65535 bytes, but as it's CESU-8 encoded we dont know how large it is in bytes, so be safe
+    private static final int MAX_NAME_LENGTH = 256;
+    
+    /** Computes the file name (mostly important for stacktraces) */
+    public static String computeSourceName(String scriptName, String source) {
+        StringBuilder fileName = new StringBuilder();
+        if (scriptName.equals(PainlessScriptEngineService.INLINE_NAME)) {
+            // its an anonymous script, include at least a portion of the source to help identify which one it is
+            // but don't create stacktraces with filenames that contain newlines or huge names.
+
+            // truncate to the first newline
+            int limit = source.indexOf('\n');
+            if (limit >= 0) {
+                int limit2 = source.indexOf('\r');
+                if (limit2 >= 0) {
+                    limit = Math.min(limit, limit2);
+                }
+            } else {
+                limit = source.length();
+            }
+
+            // truncate to our limit
+            limit = Math.min(limit, MAX_NAME_LENGTH);
+            fileName.append(source, 0, limit);
+
+            // if we truncated, make it obvious
+            if (limit != source.length()) {
+                fileName.append(" ...");
+            }
+            fileName.append(" @ <inline script>");
+        } else {
+            // its a named script, just use the name
+            // but don't trust this has a reasonable length!
+            if (scriptName.length() > MAX_NAME_LENGTH) {
+                fileName.append(scriptName, 0, MAX_NAME_LENGTH);
+                fileName.append(" ...");
+            } else {
+                fileName.append(scriptName);
+            }
+        }
+        return fileName.toString();
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index 7c11c5b15bf..2663fe9c717 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -114,7 +114,8 @@ public final class MethodWriter extends GeneratorAdapter {
      * <p>
      * This is invoked for each statement boundary (leaf {@code S*} nodes).
      */
-    public void writeStatementOffset(int offset) {
+    public void writeStatementOffset(Location location) {
+        int offset = location.getOffset();
         // ensure we don't have duplicate stuff going in here. can catch bugs
         // (e.g. nodes get assigned wrong offsets by antlr walker)
         assert statements.get(offset) == false;
@@ -126,16 +127,16 @@ public final class MethodWriter extends GeneratorAdapter {
      * <p>
      * This is invoked before instructions that can hit exceptions.
      */
-    public void writeDebugInfo(int offset) {
+    public void writeDebugInfo(Location location) {
         // TODO: maybe track these in bitsets too? this is trickier...
         Label label = new Label();
         visitLabel(label);
-        visitLineNumber(offset + 1, label);
+        visitLineNumber(location.getOffset() + 1, label);
     }
 
-    public void writeLoopCounter(int slot, int count, int offset) {
+    public void writeLoopCounter(int slot, int count, Location location) {
         if (slot > -1) {
-            writeDebugInfo(offset);
+            writeDebugInfo(location);
             final Label end = new Label();
 
             iinc(slot, -count);
@@ -281,14 +282,14 @@ public final class MethodWriter extends GeneratorAdapter {
         }
     }
 
-    public void writeBinaryInstruction(final String location, final Type type, final Operation operation) {
+    public void writeBinaryInstruction(Location location, Type type, Operation operation) {
         final Sort sort = type.sort;
 
         if ((sort == Sort.FLOAT || sort == Sort.DOUBLE) &&
                 (operation == Operation.LSH || operation == Operation.USH ||
                 operation == Operation.RSH || operation == Operation.BWAND ||
                 operation == Operation.XOR || operation == Operation.BWOR)) {
-            throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
+            throw location.createError(new IllegalStateException("Illegal tree structure."));
         }
 
         if (sort == Sort.DEF) {
@@ -305,7 +306,7 @@ public final class MethodWriter extends GeneratorAdapter {
                 case XOR:   invokeStatic(DEF_UTIL_TYPE, DEF_XOR_CALL); break;
                 case BWOR:  invokeStatic(DEF_UTIL_TYPE, DEF_OR_CALL);  break;
                 default:
-                    throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
+                    throw location.createError(new IllegalStateException("Illegal tree structure."));
             }
         } else {
             switch (operation) {
@@ -321,7 +322,7 @@ public final class MethodWriter extends GeneratorAdapter {
                 case XOR:   math(GeneratorAdapter.XOR,  type.type); break;
                 case BWOR:  math(GeneratorAdapter.OR,   type.type); break;
                 default:
-                    throw new IllegalStateException("Error " + location + ": Illegal tree structure.");
+                    throw location.createError(new IllegalStateException("Illegal tree structure."));
             }
         }
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
index f9ee949e3d4..4971c08b086 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
@@ -29,6 +29,7 @@ import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.ScriptEngineService;
+import org.elasticsearch.script.ScriptException;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.lookup.SearchLookup;
 
@@ -38,7 +39,10 @@ import java.security.AccessController;
 import java.security.Permissions;
 import java.security.PrivilegedAction;
 import java.security.ProtectionDomain;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -147,13 +151,17 @@ public final class PainlessScriptEngineService extends AbstractComponent impleme
             }
         });
 
-        // Drop all permissions to actually compile the code itself.
-        return AccessController.doPrivileged(new PrivilegedAction<Executable>() {
-            @Override
-            public Executable run() {
-                return Compiler.compile(loader, scriptName == null ? INLINE_NAME : scriptName, scriptSource, compilerSettings);
-            }
-        }, COMPILATION_CONTEXT);
+        try {
+            // Drop all permissions to actually compile the code itself.
+            return AccessController.doPrivileged(new PrivilegedAction<Executable>() {
+                @Override
+                public Executable run() {
+                    return Compiler.compile(loader, scriptName == null ? INLINE_NAME : scriptName, scriptSource, compilerSettings);
+                }
+            }, COMPILATION_CONTEXT);
+        } catch (Exception e) {
+            throw convertToScriptException(scriptName == null ? scriptSource : scriptName, scriptSource, e);
+        }
     }
 
     /**
@@ -213,4 +221,51 @@ public final class PainlessScriptEngineService extends AbstractComponent impleme
     public void close() {
         // Nothing to do.
     }
+    
+    private ScriptException convertToScriptException(String scriptName, String scriptSource, Throwable t) {
+        // create a script stack: this is just the script portion
+        List<String> scriptStack = new ArrayList<>();
+        for (StackTraceElement element : t.getStackTrace()) {
+            if (WriterConstants.CLASS_NAME.equals(element.getClassName())) {
+                // found the script portion
+                int offset = element.getLineNumber();
+                if (offset == -1) {
+                    scriptStack.add("<<< unknown portion of script >>>");
+                } else {
+                    offset--; // offset is 1 based, line numbers must be!
+                    int startOffset = getPreviousStatement(scriptSource, offset);
+                    int endOffset = getNextStatement(scriptSource, offset);
+                    StringBuilder snippet = new StringBuilder();
+                    if (startOffset > 0) {
+                        snippet.append("... ");
+                    }
+                    snippet.append(scriptSource.substring(startOffset, endOffset));
+                    if (endOffset < scriptSource.length()) {
+                        snippet.append(" ...");
+                    }
+                    scriptStack.add(snippet.toString());
+                    StringBuilder pointer = new StringBuilder();
+                    if (startOffset > 0) {
+                        pointer.append("    ");
+                    }
+                    for (int i = startOffset; i < offset; i++) {
+                        pointer.append(' ');
+                    }
+                    pointer.append("^---- HERE");
+                    scriptStack.add(pointer.toString());
+                }
+                break;
+            }
+        }
+        throw new ScriptException("compile error", t, scriptStack, scriptSource, PainlessScriptEngineService.NAME);
+    }
+    
+    // very simple heuristic: +/- 25 chars. can be improved later.
+    private int getPreviousStatement(String scriptSource, int offset) {
+        return Math.max(0, offset - 25);
+    }
+    
+    private int getNextStatement(String scriptSource, int offset) {
+        return Math.min(scriptSource.length(), offset + 25);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
index 83ae7b664c1..380b6c2bf21 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
@@ -173,7 +173,8 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
     /** returns true for methods that are part of the runtime */
     private static boolean shouldFilter(StackTraceElement element) {
         return element.getClassName().startsWith("org.elasticsearch.painless.") ||
-               element.getClassName().startsWith("java.lang.invoke.");
+               element.getClassName().startsWith("java.lang.invoke.") ||
+               element.getClassName().startsWith("sun.invoke.");
     }
 
     /**
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
index f50ab29096f..16130476c38 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
@@ -69,7 +69,7 @@ public final class Variables {
     }
 
     public static final class Variable {
-        public final String location;
+        public final Location location;
         public final String name;
         public final Type type;
         public final int slot;
@@ -77,7 +77,7 @@ public final class Variables {
 
         public boolean read = false;
 
-        private Variable(String location, String name, Type type, int slot, boolean readonly) {
+        private Variable(Location location, String name, Type type, int slot, boolean readonly) {
             this.location = location;
             this.name = name;
             this.type = type;
@@ -88,6 +88,7 @@ public final class Variables {
 
     final Reserved reserved;
 
+    // TODO: this datastructure runs in linear time for nearly all operations. use linkedhashset instead?
     private final Deque<Integer> scopes = new ArrayDeque<>();
     private final Deque<Variable> variables = new ArrayDeque<>();
 
@@ -99,35 +100,35 @@ public final class Variables {
         // Method variables.
 
         // This reference.  Internal use only.
-        addVariable("[" + Reserved.THIS + "]", Definition.getType("Object"), Reserved.THIS, true, true);
+        addVariable(null, Definition.getType("Object"), Reserved.THIS, true, true);
 
         // Input map of variables passed to the script.
-        addVariable("[" + Reserved.PARAMS + "]", Definition.getType("Map"), Reserved.PARAMS, true, true);
+        addVariable(null, Definition.getType("Map"), Reserved.PARAMS, true, true);
 
         // Scorer parameter passed to the script.  Internal use only.
-        addVariable("[" + Reserved.SCORER + "]", Definition.DEF_TYPE, Reserved.SCORER, true, true);
+        addVariable(null, Definition.DEF_TYPE, Reserved.SCORER, true, true);
 
         // Doc parameter passed to the script. TODO: Currently working as a Map, we can do better?
-        addVariable("[" + Reserved.DOC + "]", Definition.getType("Map"), Reserved.DOC, true, true);
+        addVariable(null, Definition.getType("Map"), Reserved.DOC, true, true);
 
         // Aggregation _value parameter passed to the script.
-        addVariable("[" + Reserved.VALUE + "]", Definition.DEF_TYPE, Reserved.VALUE, true, true);
+        addVariable(null, Definition.DEF_TYPE, Reserved.VALUE, true, true);
 
         // Shortcut variables.
 
         // Document's score as a read-only double.
         if (reserved.score) {
-            addVariable("[" + Reserved.SCORE + "]", Definition.DOUBLE_TYPE, Reserved.SCORE, true, true);
+            addVariable(null, Definition.DOUBLE_TYPE, Reserved.SCORE, true, true);
         }
 
         // The ctx map set by executable scripts as a read-only map.
         if (reserved.ctx) {
-            addVariable("[" + Reserved.CTX + "]", Definition.getType("Map"), Reserved.CTX, true, true);
+            addVariable(null, Definition.getType("Map"), Reserved.CTX, true, true);
         }
 
         // Loop counter to catch infinite loops.  Internal use only.
         if (reserved.loop) {
-            addVariable("[" + Reserved.LOOP + "]", Definition.INT_TYPE, Reserved.LOOP, true, true);
+            addVariable(null, Definition.INT_TYPE, Reserved.LOOP, true, true);
         }
     }
 
@@ -137,19 +138,20 @@ public final class Variables {
 
     public void decrementScope() {
         int remove = scopes.pop();
-
+        
         while (remove > 0) {
-             Variable variable = variables.pop();
-
+            Variable variable = variables.pop();
+            
+            // TODO: is this working? the code reads backwards...
             if (variable.read) {
-                throw new IllegalArgumentException("Error [" + variable.location + "]: Variable [" + variable.name + "] never used.");
+                throw variable.location.createError(new IllegalArgumentException("Variable [" + variable.name + "] never used."));
             }
-
+            
             --remove;
         }
     }
 
-    public Variable getVariable(String location, String name) {
+    public Variable getVariable(Location location, String name) {
          Iterator<Variable> itr = variables.iterator();
 
         while (itr.hasNext()) {
@@ -160,20 +162,20 @@ public final class Variables {
             }
         }
 
-        if (location != null) {
-            throw new IllegalArgumentException("Error " + location + ": Variable [" + name + "] not defined.");
-        }
-
-        return null;
+        throw location.createError(new IllegalArgumentException("Variable [" + name + "] not defined."));
     }
 
-    public Variable addVariable(String location, Type type, String name, boolean readonly, boolean reserved) {
+    private boolean variableExists(String name) {
+        return variables.contains(name);
+    }
+
+    public Variable addVariable(Location location, Type type, String name, boolean readonly, boolean reserved) {
         if (!reserved && this.reserved.isReserved(name)) {
-            throw new IllegalArgumentException("Error " + location + ": Variable name [" + name + "] is reserved.");
+            throw location.createError(new IllegalArgumentException("Variable name [" + name + "] is reserved."));
         }
 
-        if (getVariable(null, name) != null) {
-            throw new IllegalArgumentException("Error " + location + ": Variable name [" + name + "] already defined.");
+        if (variableExists(name)) {
+            throw new IllegalArgumentException("Variable name [" + name + "] already defined.");
         }
 
         try {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
index 362d81f2479..f09bf5b87eb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
@@ -72,9 +72,6 @@ final class Writer {
         writeEnd();
     }
 
-    // This maximum length is theoretically 65535 bytes, but as it's CESU-8 encoded we dont know how large it is in bytes, so be safe
-    private static final int MAX_NAME_LENGTH = 256;
-
     private void writeBegin() {
         final int version = Opcodes.V1_8;
         final int access = Opcodes.ACC_PUBLIC | Opcodes.ACC_SUPER | Opcodes.ACC_FINAL;
@@ -86,47 +83,7 @@ final class Writer {
             new String[] { WriterConstants.NEEDS_SCORE_TYPE.getInternalName() } : null;
 
         writer.visit(version, access, name, null, base, interfaces);
-        writer.visitSource(computeSourceName(), null);
-    }
-
-    /** Computes the file name (mostly important for stacktraces) */
-    private String computeSourceName() {
-        StringBuilder fileName = new StringBuilder();
-        if (scriptName.equals(PainlessScriptEngineService.INLINE_NAME)) {
-            // its an anonymous script, include at least a portion of the source to help identify which one it is
-            // but don't create stacktraces with filenames that contain newlines or huge names.
-
-            // truncate to the first newline
-            int limit = source.indexOf('\n');
-            if (limit >= 0) {
-                int limit2 = source.indexOf('\r');
-                if (limit2 >= 0) {
-                    limit = Math.min(limit, limit2);
-                }
-            } else {
-                limit = source.length();
-            }
-
-            // truncate to our limit
-            limit = Math.min(limit, MAX_NAME_LENGTH);
-            fileName.append(source, 0, limit);
-
-            // if we truncated, make it obvious
-            if (limit != source.length()) {
-                fileName.append(" ...");
-            }
-            fileName.append(" @ <inline script>");
-        } else {
-            // its a named script, just use the name
-            // but don't trust this has a reasonable length!
-            if (scriptName.length() > MAX_NAME_LENGTH) {
-                fileName.append(scriptName, 0, MAX_NAME_LENGTH);
-                fileName.append(" ...");
-            } else {
-                fileName.append(scriptName);
-            }
-        }
-        return fileName.toString();
+        writer.visitSource(Location.computeSourceName(scriptName,source), null);
     }
 
     private void writeConstructor() {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ErrorHandlingLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ErrorHandlingLexer.java
index 25069a7e9d3..99453b648a8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ErrorHandlingLexer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ErrorHandlingLexer.java
@@ -22,16 +22,17 @@ package org.elasticsearch.painless.antlr;
 import org.antlr.v4.runtime.CharStream;
 import org.antlr.v4.runtime.LexerNoViableAltException;
 import org.antlr.v4.runtime.misc.Interval;
-
-import java.text.ParseException;
+import org.elasticsearch.painless.Location;
 
 /**
  * A lexer that will override the default error behavior to fail on the first error.
  */
 final class ErrorHandlingLexer extends PainlessLexer {
+    final String sourceName;
 
-    ErrorHandlingLexer(final CharStream charStream) {
+    ErrorHandlingLexer(CharStream charStream, String sourceName) {
         super(charStream);
+        this.sourceName = sourceName;
     }
 
     @Override
@@ -40,11 +41,7 @@ final class ErrorHandlingLexer extends PainlessLexer {
         final int startIndex = lnvae.getStartIndex();
         final String text = charStream.getText(Interval.of(startIndex, charStream.index()));
 
-        final ParseException parseException = new ParseException("Error [" + _tokenStartLine + ":" +
-                _tokenStartCharPositionInLine + "]: unexpected character [" +
-                getErrorDisplay(text) + "].",  _tokenStartCharIndex);
-        parseException.initCause(lnvae);
-
-        throw new RuntimeException(parseException);
+        Location location = new Location(sourceName, _tokenStartCharIndex);
+        throw location.createError(new IllegalArgumentException("unexpected character [" + getErrorDisplay(text) + "].", lnvae));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ParserErrorStrategy.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ParserErrorStrategy.java
index 40f617a660c..b7d495fbe9b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ParserErrorStrategy.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ParserErrorStrategy.java
@@ -25,13 +25,17 @@ import org.antlr.v4.runtime.NoViableAltException;
 import org.antlr.v4.runtime.Parser;
 import org.antlr.v4.runtime.RecognitionException;
 import org.antlr.v4.runtime.Token;
-
-import java.text.ParseException;
+import org.elasticsearch.painless.Location;
 
 /**
  * An error strategy that will override the default error behavior to fail on the first parser error.
  */
 final class ParserErrorStrategy extends DefaultErrorStrategy {
+    final String sourceName;
+    
+    ParserErrorStrategy(String sourceName) {
+        this.sourceName = sourceName;
+    }
 
     @Override
     public void recover(final Parser recognizer, final RecognitionException re) {
@@ -39,38 +43,32 @@ final class ParserErrorStrategy extends DefaultErrorStrategy {
         String message;
 
         if (token == null) {
-            message = "Error: no parse token found.";
+            message = "no parse token found.";
         } else if (re instanceof InputMismatchException) {
-            message = "Error[" + token.getLine() + ":" + token.getCharPositionInLine() + "]:" +
-                    " unexpected token [" + getTokenErrorDisplay(token) + "]" +
+            message = "unexpected token [" + getTokenErrorDisplay(token) + "]" +
                     " was expecting one of [" + re.getExpectedTokens().toString(recognizer.getVocabulary()) + "].";
         } else if (re instanceof NoViableAltException) {
             if (token.getType() == PainlessParser.EOF) {
-                message = "Error: unexpected end of script.";
+                message = "unexpected end of script.";
             } else {
-                message = "Error[" + token.getLine() + ":" + token.getCharPositionInLine() + "]:" +
-                        "invalid sequence of tokens near [" + getTokenErrorDisplay(token) + "].";
+                message = "invalid sequence of tokens near [" + getTokenErrorDisplay(token) + "].";
             }
         } else {
-            message = "Error[" + token.getLine() + ":" + token.getCharPositionInLine() + "]:" +
-                    " unexpected token near [" + getTokenErrorDisplay(token) + "].";
+            message =  "unexpected token near [" + getTokenErrorDisplay(token) + "].";
         }
 
-        final ParseException parseException = new ParseException(message, token == null ? -1 : token.getStartIndex());
-        parseException.initCause(re);
-
-        throw new RuntimeException(parseException);
+        Location location = new Location(sourceName, token == null ? -1 : token.getStartIndex());
+        throw location.createError(new IllegalArgumentException(message, re));
     }
 
     @Override
     public Token recoverInline(final Parser recognizer) throws RecognitionException {
         final Token token = recognizer.getCurrentToken();
-        final String message = "Error[" + token.getLine() + ":" + token.getCharPositionInLine() + "]:" +
-            " unexpected token [" + getTokenErrorDisplay(token) + "]" +
+        final String message = "unexpected token [" + getTokenErrorDisplay(token) + "]" +
             " was expecting one of [" + recognizer.getExpectedTokens().toString(recognizer.getVocabulary()) + "].";
-        final ParseException parseException = new ParseException(message, token.getStartIndex());
 
-        throw new RuntimeException(parseException);
+        Location location = new Location(sourceName, token.getStartIndex());
+        throw location.createError(new IllegalArgumentException(message));
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index 5a1c080f960..0f155ceb4c1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -28,6 +28,7 @@ import org.antlr.v4.runtime.RecognitionException;
 import org.antlr.v4.runtime.Recognizer;
 import org.antlr.v4.runtime.atn.PredictionMode;
 import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Variables.Reserved;
 import org.elasticsearch.painless.antlr.PainlessParser.AfterthoughtContext;
@@ -135,25 +136,27 @@ import java.util.List;
  */
 public final class Walker extends PainlessParserBaseVisitor<Object> {
 
-    public static SSource buildPainlessTree(String source, Reserved reserved, CompilerSettings settings) {
-        return new Walker(source, reserved, settings).source;
+    public static SSource buildPainlessTree(String name, String sourceText, Reserved reserved, CompilerSettings settings) {
+        return new Walker(name, sourceText, reserved, settings).source;
     }
 
     private final Reserved reserved;
     private final SSource source;
     private final CompilerSettings settings;
+    private final String sourceName;
 
-    private Walker(String source, Reserved reserved, CompilerSettings settings) {
+    private Walker(String name, String sourceText, Reserved reserved, CompilerSettings settings) {
         this.reserved = reserved;
         this.settings = settings;
-        this.source = (SSource)visit(buildAntlrTree(source));
+        this.sourceName = Location.computeSourceName(name, sourceText);
+        this.source = (SSource)visit(buildAntlrTree(sourceText));
     }
 
     private SourceContext buildAntlrTree(String source) {
         ANTLRInputStream stream = new ANTLRInputStream(source);
-        PainlessLexer lexer = new ErrorHandlingLexer(stream);
+        PainlessLexer lexer = new ErrorHandlingLexer(stream, sourceName);
         PainlessParser parser = new PainlessParser(new CommonTokenStream(lexer));
-        ParserErrorStrategy strategy = new ParserErrorStrategy();
+        ParserErrorStrategy strategy = new ParserErrorStrategy(sourceName);
 
         lexer.removeErrorListeners();
         parser.removeErrorListeners();
@@ -185,16 +188,8 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         parser.getInterpreter().setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION);
     }
 
-    private int line(ParserRuleContext ctx) {
-        return ctx.getStart().getLine();
-    }
-
-    private int offset(ParserRuleContext ctx) {
-        return ctx.getStart().getStartIndex();
-    }
-
-    private String location(ParserRuleContext ctx) {
-        return "[ " + ctx.getStart().getLine() + " : " + ctx.getStart().getCharPositionInLine() + " ]";
+    private Location location(ParserRuleContext ctx) {
+        return new Location(sourceName, ctx.getStart().getStartIndex());
     }
 
     @Override
@@ -205,7 +200,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
             statements.add((AStatement)visit(statement));
         }
 
-        return new SSource(line(ctx), offset(ctx), location(ctx), statements);
+        return new SSource(location(ctx), statements);
     }
 
     @Override
@@ -216,9 +211,9 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         if (ctx.trailer().size() > 1) {
             SBlock elseblock = (SBlock)visit(ctx.trailer(1));
 
-            return new SIfElse(line(ctx), offset(ctx), location(ctx), expression, ifblock, elseblock);
+            return new SIfElse(location(ctx), expression, ifblock, elseblock);
         } else {
-            return new SIf(line(ctx), offset(ctx), location(ctx), expression, ifblock);
+            return new SIf(location(ctx), expression, ifblock);
         }
     }
 
@@ -233,11 +228,11 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         if (ctx.trailer() != null) {
             SBlock block = (SBlock)visit(ctx.trailer());
 
-            return new SWhile(line(ctx), offset(ctx), location(ctx), settings.getMaxLoopCounter(), expression, block);
+            return new SWhile(location(ctx), settings.getMaxLoopCounter(), expression, block);
         } else if (ctx.empty() != null) {
-            return new SWhile(line(ctx), offset(ctx), location(ctx), settings.getMaxLoopCounter(), expression, null);
+            return new SWhile(location(ctx), settings.getMaxLoopCounter(), expression, null);
         } else {
-            throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
+            throw location(ctx).createError(new IllegalStateException(" Illegal tree structure."));
         }
     }
 
@@ -250,7 +245,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         AExpression expression = (AExpression)visitExpression(ctx.expression());
         SBlock block = (SBlock)visit(ctx.block());
 
-        return new SDo(line(ctx), offset(ctx), location(ctx), settings.getMaxLoopCounter(), block, expression);
+        return new SDo(location(ctx), settings.getMaxLoopCounter(), block, expression);
     }
 
     @Override
@@ -266,13 +261,13 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         if (ctx.trailer() != null) {
             SBlock block = (SBlock)visit(ctx.trailer());
 
-            return new SFor(line(ctx), offset(ctx), location(ctx),
+            return new SFor(location(ctx),
                 settings.getMaxLoopCounter(), initializer, expression, afterthought, block);
         } else if (ctx.empty() != null) {
-            return new SFor(line(ctx), offset(ctx), location(ctx),
+            return new SFor(location(ctx),
                 settings.getMaxLoopCounter(), initializer, expression, afterthought, null);
         } else {
-            throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
+            throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
         }
     }
 
@@ -283,19 +278,19 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
 
     @Override
     public Object visitContinue(ContinueContext ctx) {
-        return new SContinue(line(ctx), offset(ctx), location(ctx));
+        return new SContinue(location(ctx));
     }
 
     @Override
     public Object visitBreak(BreakContext ctx) {
-        return new SBreak(line(ctx), offset(ctx), location(ctx));
+        return new SBreak(location(ctx));
     }
 
     @Override
     public Object visitReturn(ReturnContext ctx) {
         AExpression expression = (AExpression)visitExpression(ctx.expression());
 
-        return new SReturn(line(ctx), offset(ctx), location(ctx), expression);
+        return new SReturn(location(ctx), expression);
     }
 
     @Override
@@ -307,21 +302,21 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
             catches.add((SCatch)visit(trap));
         }
 
-        return new STry(line(ctx), offset(ctx), location(ctx), block, catches);
+        return new STry(location(ctx), block, catches);
     }
 
     @Override
     public Object visitThrow(ThrowContext ctx) {
         AExpression expression = (AExpression)visitExpression(ctx.expression());
 
-        return new SThrow(line(ctx), offset(ctx), location(ctx), expression);
+        return new SThrow(location(ctx), expression);
     }
 
     @Override
     public Object visitExpr(ExprContext ctx) {
         AExpression expression = (AExpression)visitExpression(ctx.expression());
 
-        return new SExpression(line(ctx), offset(ctx), location(ctx), expression);
+        return new SExpression(location(ctx), expression);
     }
 
     @Override
@@ -332,9 +327,9 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
             List<AStatement> statements = new ArrayList<>();
             statements.add((AStatement)visit(ctx.statement()));
 
-            return new SBlock(line(ctx), offset(ctx), location(ctx), statements);
+            return new SBlock(location(ctx), statements);
         } else {
-            throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
+            throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
         }
     }
 
@@ -349,13 +344,13 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
                 statements.add((AStatement)visit(statement));
             }
 
-            return new SBlock(line(ctx), offset(ctx), location(ctx), statements);
+            return new SBlock(location(ctx), statements);
         }
     }
 
     @Override
     public Object visitEmpty(EmptyContext ctx) {
-        throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
+        throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
     }
 
     @Override
@@ -365,7 +360,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         } else if (ctx.expression() != null) {
             return visitExpression(ctx.expression());
         } else {
-            throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
+            throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
         }
     }
 
@@ -383,25 +378,25 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
             String name = declvar.ID().getText();
             AExpression expression = declvar.expression() == null ? null : (AExpression)visitExpression(declvar.expression());
 
-            declarations.add(new SDeclaration(line(declvar), offset(declvar), location(declvar), type, name, expression));
+            declarations.add(new SDeclaration(location(declvar), type, name, expression));
         }
 
-        return new SDeclBlock(line(ctx), offset(ctx), location(ctx), declarations);
+        return new SDeclBlock(location(ctx), declarations);
     }
 
     @Override
     public Object visitDecltype(DecltypeContext ctx) {
-        throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
+        throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
     }
 
     @Override
     public Object visitFuncref(FuncrefContext ctx) {
-        return new EFunctionRef(line(ctx), offset(ctx), location(ctx), ctx.TYPE().getText(), ctx.ID().getText());
+        return new EFunctionRef(location(ctx), ctx.TYPE().getText(), ctx.ID().getText());
     }
 
     @Override
     public Object visitDeclvar(DeclvarContext ctx) {
-        throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
+        throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
     }
 
     @Override
@@ -410,12 +405,12 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         String name = ctx.ID().getText();
         SBlock block = (SBlock)visit(ctx.block());
 
-        return new SCatch(line(ctx), offset(ctx), location(ctx), type, name, block);
+        return new SCatch(location(ctx), type, name, block);
     }
 
     @Override
     public Object visitDelimiter(DelimiterContext ctx) {
-        throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
+        throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
     }
 
     private Object visitExpression(ExpressionContext ctx) {
@@ -425,7 +420,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
             @SuppressWarnings("unchecked")
             List<ALink> links = (List<ALink>)expression;
 
-            return new EChain(line(ctx), offset(ctx), location(ctx), links, false, false, null, null);
+            return new EChain(location(ctx), links, false, false, null, null);
         } else {
             return expression;
         }
@@ -465,10 +460,10 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         } else if (ctx.BWOR() != null) {
             operation = Operation.BWOR;
         } else {
-            throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+            throw location(ctx).createError(new IllegalStateException("Unexpected state."));
         }
 
-        return new EBinary(line(ctx), offset(ctx), location(ctx), operation, left, right);
+        return new EBinary(location(ctx), operation, left, right);
     }
 
     @Override
@@ -494,10 +489,10 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         } else if (ctx.NER() != null) {
             operation = Operation.NER;
         } else {
-            throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+            throw location(ctx).createError(new IllegalStateException("Unexpected state."));
         }
 
-        return new EComp(line(ctx), offset(ctx), location(ctx), operation, left, right);
+        return new EComp(location(ctx), operation, left, right);
     }
 
     @Override
@@ -511,10 +506,10 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         } else if (ctx.BOOLOR() != null) {
             operation = Operation.OR;
         } else {
-            throw new IllegalStateException("Error " + location(ctx) + ": Unexpected state.");
+            throw location(ctx).createError(new IllegalStateException("Unexpected state."));
         }
 
-        return new EBool(line(ctx), offset(ctx), location(ctx), operation, left, right);
+        return new EBool(location(ctx), operation, left, right);
     }
 
     @Override
@@ -523,7 +518,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         AExpression left = (AExpression)visitExpression(ctx.expression(1));
         AExpression right = (AExpression)visitExpression(ctx.expression(2));
 
-        return new EConditional(line(ctx), offset(ctx), location(ctx), condition, left, right);
+        return new EConditional(location(ctx), condition, left, right);
     }
 
     @Override
@@ -557,12 +552,12 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         } else if (ctx.AOR() != null) {
             operation = Operation.BWOR;
         } else {
-            throw new IllegalStateException("Error " + location(ctx) + ": Illegal tree structure.");
+            throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
         }
 
         AExpression expression = (AExpression)visitExpression(ctx.expression());
 
-        return new EChain(line(ctx), offset(ctx), location(ctx), links, false, false, operation, expression);
+        return new EChain(location(ctx), links, false, false, operation, expression);
     }
 
     private Object visitUnary(UnaryContext ctx) {
@@ -572,7 +567,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
             @SuppressWarnings("unchecked")
             List<ALink> links = (List<ALink>)expression;
 
-            return new EChain(line(ctx), offset(ctx), location(ctx), links, false, false, null, null);
+            return new EChain(location(ctx), links, false, false, null, null);
         } else {
             return expression;
         }
@@ -589,10 +584,10 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         } else if (ctx.DECR() != null) {
             operation = Operation.DECR;
         } else {
-            throw new IllegalStateException("Error " + location(ctx) + ": Illegal tree structure.");
+            throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
         }
 
-        return new EChain(line(ctx), offset(ctx), location(ctx), links, true, false, operation, null);
+        return new EChain(location(ctx), links, true, false, operation, null);
     }
 
     @Override
@@ -606,10 +601,10 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         } else if (ctx.DECR() != null) {
             operation = Operation.DECR;
         } else {
-            throw new IllegalStateException("Error " + location(ctx) + ": Illegal tree structure.");
+            throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
         }
 
-        return new EChain(line(ctx), offset(ctx), location(ctx), links, false, true, operation, null);
+        return new EChain(location(ctx), links, false, true, operation, null);
     }
 
     @Override
@@ -622,31 +617,31 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         final boolean negate = ctx.parent instanceof OperatorContext && ((OperatorContext)ctx.parent).SUB() != null;
 
         if (ctx.DECIMAL() != null) {
-            return new EDecimal(line(ctx), offset(ctx), location(ctx), (negate ? "-" : "") + ctx.DECIMAL().getText());
+            return new EDecimal(location(ctx), (negate ? "-" : "") + ctx.DECIMAL().getText());
         } else if (ctx.HEX() != null) {
-            return new ENumeric(line(ctx), offset(ctx), location(ctx), (negate ? "-" : "") + ctx.HEX().getText().substring(2), 16);
+            return new ENumeric(location(ctx), (negate ? "-" : "") + ctx.HEX().getText().substring(2), 16);
         } else if (ctx.INTEGER() != null) {
-            return new ENumeric(line(ctx), offset(ctx), location(ctx), (negate ? "-" : "") + ctx.INTEGER().getText(), 10);
+            return new ENumeric(location(ctx), (negate ? "-" : "") + ctx.INTEGER().getText(), 10);
         } else if (ctx.OCTAL() != null) {
-            return new ENumeric(line(ctx), offset(ctx), location(ctx), (negate ? "-" : "") + ctx.OCTAL().getText().substring(1), 8);
+            return new ENumeric(location(ctx), (negate ? "-" : "") + ctx.OCTAL().getText().substring(1), 8);
         } else {
-            throw new IllegalStateException("Error " + location(ctx) + ": Illegal tree structure.");
+            throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
         }
     }
 
     @Override
     public Object visitTrue(TrueContext ctx) {
-        return new EBoolean(line(ctx), offset(ctx), location(ctx), true);
+        return new EBoolean(location(ctx), true);
     }
 
     @Override
     public Object visitFalse(FalseContext ctx) {
-        return new EBoolean(line(ctx), offset(ctx), location(ctx), false);
+        return new EBoolean(location(ctx), false);
     }
 
     @Override
     public Object visitNull(NullContext ctx) {
-        return new ENull(line(ctx), offset(ctx), location(ctx));
+        return new ENull(location(ctx));
     }
 
     @Override
@@ -666,10 +661,10 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
             } else if (ctx.SUB() != null) {
                 operation = Operation.SUB;
             } else {
-                throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
+                throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
             }
 
-            return new EUnary(line(ctx), offset(ctx), location(ctx), operation, expression);
+            return new EUnary(location(ctx), operation, expression);
         }
     }
 
@@ -681,11 +676,11 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         if (child instanceof List) {
             @SuppressWarnings("unchecked")
             List<ALink> links = (List<ALink>)child;
-            links.add(new LCast(line(ctx), offset(ctx), location(ctx), type));
+            links.add(new LCast(location(ctx), type));
 
             return links;
         } else {
-            return new EExplicit(line(ctx), offset(ctx), location(ctx), type, (AExpression)child);
+            return new EExplicit(location(ctx), type, (AExpression)child);
         }
     }
 
@@ -703,7 +698,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
 
             return links;
         } else if (!ctx.secondary().isEmpty()) {
-            throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
+            throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
         } else {
             return child;
         }
@@ -714,7 +709,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         String type = ctx.decltype().getText();
         List<ALink> links = new ArrayList<>();
 
-        links.add(new LStatic(line(ctx), offset(ctx), location(ctx), type));
+        links.add(new LStatic(location(ctx), type));
         links.add((ALink)visit(ctx.dot()));
 
         for (SecondaryContext secondary : ctx.secondary()) {
@@ -734,7 +729,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         }
 
         List<ALink> links = new ArrayList<>();
-        links.add(new LNewArray(line(ctx), offset(ctx), location(ctx), type, expressions));
+        links.add(new LNewArray(location(ctx), type, expressions));
 
         if (ctx.dot() != null) {
             links.add((ALink)visit(ctx.dot()));
@@ -743,7 +738,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
                 links.add((ALink)visit(secondary));
             }
         } else if (!ctx.secondary().isEmpty()) {
-            throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
+            throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
         }
 
         return links;
@@ -763,7 +758,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
     public Object visitString(StringContext ctx) {
         String string = ctx.STRING().getText().substring(1, ctx.STRING().getText().length() - 1);
         List<ALink> links = new ArrayList<>();
-        links.add(new LString(line(ctx), offset(ctx), location(ctx), string));
+        links.add(new LString(location(ctx), string));
 
         return links;
     }
@@ -772,7 +767,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
     public Object visitVariable(VariableContext ctx) {
         String name = ctx.ID().getText();
         List<ALink> links = new ArrayList<>();
-        links.add(new LVariable(line(ctx), offset(ctx), location(ctx), name));
+        links.add(new LVariable(location(ctx), name));
 
         reserved.markReserved(name);
 
@@ -786,7 +781,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         List<AExpression> arguments = (List<AExpression>)visit(ctx.arguments());
 
         List<ALink> links = new ArrayList<>();
-        links.add(new LNewObj(line(ctx), offset(ctx), location(ctx), type, arguments));
+        links.add(new LNewObj(location(ctx), type, arguments));
 
         return links;
     }
@@ -798,7 +793,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         } else if (ctx.brace() != null) {
             return visit(ctx.brace());
         } else {
-            throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
+            throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
         }
     }
 
@@ -808,7 +803,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         @SuppressWarnings("unchecked")
         List<AExpression> arguments = (List<AExpression>)visit(ctx.arguments());
 
-        return new LCall(line(ctx), offset(ctx), location(ctx), name, arguments);
+        return new LCall(location(ctx), name, arguments);
     }
 
     @Override
@@ -820,17 +815,17 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         } else if (ctx.DOTINTEGER() != null) {
             value = ctx.DOTINTEGER().getText();
         } else {
-            throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
+            throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
         }
 
-        return new LField(line(ctx), offset(ctx), location(ctx), value);
+        return new LField(location(ctx), value);
     }
 
     @Override
     public Object visitBraceaccess(BraceaccessContext ctx) {
         AExpression expression = (AExpression)visitExpression(ctx.expression());
 
-        return new LBrace(line(ctx), offset(ctx), location(ctx), expression);
+        return new LBrace(location(ctx), expression);
     }
 
     @Override
@@ -851,7 +846,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         } else if (ctx.funcref() != null) {
             return visit(ctx.funcref());
         } else {
-            throw new IllegalStateException("Error " + location(ctx) + " Illegal tree structure.");
+            throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
         }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
index c46181817e2..0f3cae45a36 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
@@ -21,6 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
@@ -98,8 +99,8 @@ public abstract class AExpression extends ANode {
      */
     protected Label fals = null;
 
-    public AExpression(int line, int offset, String location) {
-        super(line, offset, location);
+    public AExpression(Location location) {
+        super(location);
     }
 
     /**
@@ -124,18 +125,18 @@ public abstract class AExpression extends ANode {
             if (constant == null || this instanceof EConstant) {
                 return this;
             } else {
-                final EConstant econstant = new EConstant(line, offset, location, constant);
+                final EConstant econstant = new EConstant(location, constant);
                 econstant.analyze(variables);
 
                 if (!expected.equals(econstant.actual)) {
-                    throw new IllegalStateException(error("Illegal tree structure."));
+                    throw createError(new IllegalStateException("Illegal tree structure."));
                 }
 
                 return econstant;
             }
         } else {
             if (constant == null) {
-                final ECast ecast = new ECast(line, offset, location, this, cast);
+                final ECast ecast = new ECast(location, this, cast);
                 ecast.statement = statement;
                 ecast.actual = expected;
                 ecast.isNull = isNull;
@@ -145,28 +146,28 @@ public abstract class AExpression extends ANode {
                 if (expected.sort.constant) {
                     constant = AnalyzerCaster.constCast(location, constant, cast);
 
-                    final EConstant econstant = new EConstant(line, offset, location, constant);
+                    final EConstant econstant = new EConstant(location, constant);
                     econstant.analyze(variables);
 
                     if (!expected.equals(econstant.actual)) {
-                        throw new IllegalStateException(error("Illegal tree structure."));
+                        throw createError(new IllegalStateException("Illegal tree structure."));
                     }
 
                     return econstant;
                 } else if (this instanceof EConstant) {
-                    final ECast ecast = new ECast(line, offset, location, this, cast);
+                    final ECast ecast = new ECast(location, this, cast);
                     ecast.actual = expected;
 
                     return ecast;
                 } else {
-                    final EConstant econstant = new EConstant(line, offset, location, constant);
+                    final EConstant econstant = new EConstant(location, constant);
                     econstant.analyze(variables);
 
                     if (!actual.equals(econstant.actual)) {
-                        throw new IllegalStateException(error("Illegal tree structure."));
+                        throw createError(new IllegalStateException("Illegal tree structure."));
                     }
 
-                    final ECast ecast = new ECast(line, offset, location, econstant, cast);
+                    final ECast ecast = new ECast(location, econstant, cast);
                     ecast.actual = expected;
 
                     return ecast;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
index d68f662f913..19875ebeec5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -73,8 +74,8 @@ public abstract class ALink extends ANode {
      */
     String string = null;
 
-    ALink(int line, int offset, String location, int size) {
-        super(line, offset, location);
+    ALink(Location location, int size) {
+        super(location);
 
         this.size = size;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
index a27f6748086..08092a9e116 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
@@ -19,33 +19,22 @@
 
 package org.elasticsearch.painless.node;
 
+import org.elasticsearch.painless.Location;
+
 /**
  * The superclass for all other nodes.
  */
 public abstract class ANode {
-
     /**
-     * The line number in the original source used for debugging and errors.
+     * The identifier of the script and character offset used for debugging and errors.
      */
-    final int line;
+    final Location location;
 
-    /**
-     * The character offset in the original source used for debugging and errors.
-     */
-    final int offset;
-
-    /**
-     * The location in the original source to be printed in error messages.
-     */
-    final String location;
-
-    ANode(int line, int offset, String location) {
-        this.line = line;
-        this.offset = offset;
+    ANode(Location location) {
         this.location = location;
     }
-
-    public String error(final String message) {
-        return "Error " + location  + ": " + message;
+    
+    public RuntimeException createError(RuntimeException exception) {
+        return location.createError(exception);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
index 80d24b4cab3..b9e6679f630 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
@@ -21,6 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -107,8 +108,8 @@ public abstract class AStatement extends ANode {
      */
     Label brake = null;
 
-    AStatement(int line, int offset, String location) {
-        super(line, offset, location);
+    AStatement(Location location) {
+        super(location);
     }
 
     /**
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 23179a9ed0d..1c7da092d55 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -23,6 +23,7 @@ import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Variables;
@@ -38,8 +39,8 @@ public final class EBinary extends AExpression {
 
     boolean cat = false;
 
-    public EBinary(int line, int offset, String location, Operation operation, AExpression left, AExpression right) {
-        super(line, offset, location);
+    public EBinary(Location location, Operation operation, AExpression left, AExpression right) {
+        super(location);
 
         this.operation = operation;
         this.left = left;
@@ -71,7 +72,7 @@ public final class EBinary extends AExpression {
         } else if (operation == Operation.BWOR) {
             analyzeBWOr(variables);
         } else {
-            throw new IllegalStateException(error("Illegal tree structure."));
+            throw createError(new IllegalStateException("Illegal tree structure."));
         }
     }
 
@@ -82,7 +83,7 @@ public final class EBinary extends AExpression {
         Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
         if (promote == null) {
-            throw new ClassCastException(error("Cannot apply multiply [*] to types " +
+            throw createError(new ClassCastException("Cannot apply multiply [*] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
@@ -104,7 +105,7 @@ public final class EBinary extends AExpression {
             } else if (sort == Sort.DOUBLE) {
                 constant = (double)left.constant * (double)right.constant;
             } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
 
@@ -118,7 +119,7 @@ public final class EBinary extends AExpression {
         Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
         if (promote == null) {
-            throw new ClassCastException(error("Cannot apply divide [/] to types " +
+            throw createError(new ClassCastException("Cannot apply divide [/] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
@@ -131,16 +132,20 @@ public final class EBinary extends AExpression {
         if (left.constant != null && right.constant != null) {
             Sort sort = promote.sort;
 
-            if (sort == Sort.INT) {
-                constant = (int)left.constant / (int)right.constant;
-            } else if (sort == Sort.LONG) {
-                constant = (long)left.constant / (long)right.constant;
-            } else if (sort == Sort.FLOAT) {
-                constant = (float)left.constant / (float)right.constant;
-            } else if (sort == Sort.DOUBLE) {
-                constant = (double)left.constant / (double)right.constant;
-            } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+            try {
+                if (sort == Sort.INT) {
+                    constant = (int)left.constant / (int)right.constant;
+                } else if (sort == Sort.LONG) {
+                    constant = (long)left.constant / (long)right.constant;
+                } else if (sort == Sort.FLOAT) {
+                    constant = (float)left.constant / (float)right.constant;
+                } else if (sort == Sort.DOUBLE) {
+                    constant = (double)left.constant / (double)right.constant;
+                } else {
+                    throw createError(new IllegalStateException("Illegal tree structure."));
+                }
+            } catch (ArithmeticException e) {
+                throw createError(e);
             }
         }
 
@@ -154,7 +159,7 @@ public final class EBinary extends AExpression {
         Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
         if (promote == null) {
-            throw new ClassCastException(error("Cannot apply remainder [%] to types " +
+            throw createError(new ClassCastException("Cannot apply remainder [%] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
@@ -167,16 +172,20 @@ public final class EBinary extends AExpression {
         if (left.constant != null && right.constant != null) {
             Sort sort = promote.sort;
 
-            if (sort == Sort.INT) {
-                constant = (int)left.constant % (int)right.constant;
-            } else if (sort == Sort.LONG) {
-                constant = (long)left.constant % (long)right.constant;
-            } else if (sort == Sort.FLOAT) {
-                constant = (float)left.constant % (float)right.constant;
-            } else if (sort == Sort.DOUBLE) {
-                constant = (double)left.constant % (double)right.constant;
-            } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+            try {
+                if (sort == Sort.INT) {
+                    constant = (int)left.constant % (int)right.constant;
+                } else if (sort == Sort.LONG) {
+                    constant = (long)left.constant % (long)right.constant;
+                } else if (sort == Sort.FLOAT) {
+                    constant = (float)left.constant % (float)right.constant;
+                } else if (sort == Sort.DOUBLE) {
+                    constant = (double)left.constant % (double)right.constant;
+                } else {
+                    throw createError(new IllegalStateException("Illegal tree structure."));
+                }
+            } catch (ArithmeticException e) {
+                throw createError(e);
             }
         }
 
@@ -190,7 +199,7 @@ public final class EBinary extends AExpression {
         Type promote = AnalyzerCaster.promoteAdd(left.actual, right.actual);
 
         if (promote == null) {
-            throw new ClassCastException(error("Cannot apply add [+] to types " +
+            throw createError(new ClassCastException("Cannot apply add [+] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
@@ -228,7 +237,7 @@ public final class EBinary extends AExpression {
             } else if (sort == Sort.STRING) {
                 constant = "" + left.constant + right.constant;
             } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
 
@@ -242,7 +251,7 @@ public final class EBinary extends AExpression {
         Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
         if (promote == null) {
-            throw new ClassCastException(error("Cannot apply subtract [-] to types " +
+            throw createError(new ClassCastException("Cannot apply subtract [-] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
@@ -264,7 +273,7 @@ public final class EBinary extends AExpression {
             } else if (sort == Sort.DOUBLE) {
                 constant = (double)left.constant - (double)right.constant;
             } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
 
@@ -278,7 +287,7 @@ public final class EBinary extends AExpression {
         Type promote = AnalyzerCaster.promoteNumeric(left.actual, false);
 
         if (promote == null) {
-            throw new ClassCastException(error("Cannot apply left shift [<<] to types " +
+            throw createError(new ClassCastException("Cannot apply left shift [<<] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
@@ -297,7 +306,7 @@ public final class EBinary extends AExpression {
             } else if (sort == Sort.LONG) {
                 constant = (long)left.constant << (int)right.constant;
             } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
 
@@ -311,7 +320,7 @@ public final class EBinary extends AExpression {
         Type promote = AnalyzerCaster.promoteNumeric(left.actual, false);
 
         if (promote == null) {
-            throw new ClassCastException(error("Cannot apply right shift [>>] to types " +
+            throw createError(new ClassCastException("Cannot apply right shift [>>] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
@@ -330,7 +339,7 @@ public final class EBinary extends AExpression {
             } else if (sort == Sort.LONG) {
                 constant = (long)left.constant >> (int)right.constant;
             } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
 
@@ -344,7 +353,7 @@ public final class EBinary extends AExpression {
         Type promote = AnalyzerCaster.promoteNumeric(left.actual, false);
 
         if (promote == null) {
-            throw new ClassCastException(error("Cannot apply unsigned shift [>>>] to types " +
+            throw createError(new ClassCastException("Cannot apply unsigned shift [>>>] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
@@ -363,7 +372,7 @@ public final class EBinary extends AExpression {
             } else if (sort == Sort.LONG) {
                 constant = (long)left.constant >>> (int)right.constant;
             } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
 
@@ -377,7 +386,7 @@ public final class EBinary extends AExpression {
         Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false);
 
         if (promote == null) {
-            throw new ClassCastException(error("Cannot apply and [&] to types " +
+            throw createError(new ClassCastException("Cannot apply and [&] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
@@ -395,7 +404,7 @@ public final class EBinary extends AExpression {
             } else if (sort == Sort.LONG) {
                 constant = (long)left.constant & (long)right.constant;
             } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
 
@@ -409,7 +418,7 @@ public final class EBinary extends AExpression {
         Type promote = AnalyzerCaster.promoteXor(left.actual, right.actual);
 
         if (promote == null) {
-            throw new ClassCastException(error("Cannot apply xor [^] to types " +
+            throw createError(new ClassCastException("Cannot apply xor [^] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
@@ -429,7 +438,7 @@ public final class EBinary extends AExpression {
             } else if (sort == Sort.LONG) {
                 constant = (long)left.constant ^ (long)right.constant;
             } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
 
@@ -443,7 +452,7 @@ public final class EBinary extends AExpression {
         Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false);
 
         if (promote == null) {
-            throw new ClassCastException(error("Cannot apply or [|] to types " +
+            throw createError(new ClassCastException("Cannot apply or [|] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
@@ -461,7 +470,7 @@ public final class EBinary extends AExpression {
             } else if (sort == Sort.LONG) {
                 constant = (long)left.constant | (long)right.constant;
             } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
 
@@ -470,7 +479,7 @@ public final class EBinary extends AExpression {
 
     @Override
     void write(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         if (actual.sort == Sort.STRING && operation == Operation.ADD) {
             if (!cat) {
                 writer.writeNewStrings();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
index eda3e364795..dd221f040c4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
@@ -34,8 +35,8 @@ public final class EBool extends AExpression {
     AExpression left;
     AExpression right;
 
-    public EBool(int line, int offset, String location, Operation operation, AExpression left, AExpression right) {
-        super(line, offset, location);
+    public EBool(Location location, Operation operation, AExpression left, AExpression right) {
+        super(location);
 
         this.operation = operation;
         this.left = left;
@@ -58,7 +59,7 @@ public final class EBool extends AExpression {
             } else if (operation == Operation.OR) {
                 constant = (boolean)left.constant || (boolean)right.constant;
             } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
 
@@ -95,7 +96,7 @@ public final class EBool extends AExpression {
                     writer.mark(localtru);
                 }
             } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
         } else {
             if (operation == Operation.AND) {
@@ -131,7 +132,7 @@ public final class EBool extends AExpression {
                 writer.push(false);
                 writer.mark(end);
             } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
index a1585e13dbc..5065989600f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -28,8 +29,8 @@ import org.elasticsearch.painless.MethodWriter;
  */
 public final class EBoolean extends AExpression {
 
-    public EBoolean(int line, int offset, String location, boolean constant) {
-        super(line, offset, location);
+    public EBoolean(Location location, boolean constant) {
+        super(location);
 
         this.constant = constant;
     }
@@ -41,6 +42,6 @@ public final class EBoolean extends AExpression {
 
     @Override
     void write(MethodWriter adapter) {
-        throw new IllegalArgumentException(error("Illegal tree structure."));
+        throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
index 96e07511485..27974240125 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition.Cast;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -34,8 +35,8 @@ final class ECast extends AExpression {
 
     Cast cast = null;
 
-    ECast(int line, int offset, String location, AExpression child, Cast cast) {
-        super(line, offset, location);
+    ECast(Location location, AExpression child, Cast cast) {
+        super(location);
 
         this.type = null;
         this.child = child;
@@ -45,13 +46,13 @@ final class ECast extends AExpression {
 
     @Override
     void analyze(Variables variables) {
-        throw new IllegalStateException(error("Illegal tree structure."));
+        throw createError(new IllegalStateException("Illegal tree structure."));
     }
 
     @Override
     void write(MethodWriter writer) {
         child.write(writer);
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         writer.writeCast(cast);
         writer.writeBranch(tru, fals);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index affd1fe78c0..dfb1106616c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -23,6 +23,7 @@ import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Variables;
@@ -46,9 +47,9 @@ public final class EChain extends AExpression {
     Cast there = null;
     Cast back = null;
 
-    public EChain(int line, int offset, String location, List<ALink> links,
+    public EChain(Location location, List<ALink> links,
                   boolean pre, boolean post, Operation operation, AExpression expression) {
-        super(line, offset, location);
+        super(location);
 
         this.links = links;
         this.pre = pre;
@@ -114,40 +115,40 @@ public final class EChain extends AExpression {
         ALink last = links.get(links.size() - 1);
 
         if (pre && post) {
-            throw new IllegalStateException(error("Illegal tree structure."));
+            throw createError(new IllegalStateException("Illegal tree structure."));
         } else if (pre || post) {
             if (expression != null) {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
 
             Sort sort = last.after.sort;
 
             if (operation == Operation.INCR) {
                 if (sort == Sort.DOUBLE) {
-                    expression = new EConstant(line, offset, location, 1D);
+                    expression = new EConstant(location, 1D);
                 } else if (sort == Sort.FLOAT) {
-                    expression = new EConstant(line, offset, location, 1F);
+                    expression = new EConstant(location, 1F);
                 } else if (sort == Sort.LONG) {
-                    expression = new EConstant(line, offset, location, 1L);
+                    expression = new EConstant(location, 1L);
                 } else {
-                    expression = new EConstant(line, offset, location, 1);
+                    expression = new EConstant(location, 1);
                 }
 
                 operation = Operation.ADD;
             } else if (operation == Operation.DECR) {
                 if (sort == Sort.DOUBLE) {
-                    expression = new EConstant(line, offset, location, 1D);
+                    expression = new EConstant(location, 1D);
                 } else if (sort == Sort.FLOAT) {
-                    expression = new EConstant(line, offset, location, 1F);
+                    expression = new EConstant(location, 1F);
                 } else if (sort == Sort.LONG) {
-                    expression = new EConstant(line, offset, location, 1L);
+                    expression = new EConstant(location, 1L);
                 } else {
-                    expression = new EConstant(line, offset, location, 1);
+                    expression = new EConstant(location, 1);
                 }
 
                 operation = Operation.SUB;
             } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
     }
@@ -180,12 +181,12 @@ public final class EChain extends AExpression {
         } else if (operation == Operation.BWOR) {
             promote = AnalyzerCaster.promoteXor(last.after, expression.actual);
         } else {
-            throw new IllegalStateException(error("Illegal tree structure."));
+            throw createError(new IllegalStateException("Illegal tree structure."));
         }
 
         if (promote == null) {
-            throw new ClassCastException("Cannot apply compound assignment " +
-                "[" + operation.symbol + "=] to types [" + last.after + "] and [" + expression.actual + "].");
+            throw createError(new ClassCastException("Cannot apply compound assignment " +
+                "[" + operation.symbol + "=] to types [" + last.after + "] and [" + expression.actual + "]."));
         }
 
         cat = operation == Operation.ADD && promote.sort == Sort.STRING;
@@ -248,9 +249,8 @@ public final class EChain extends AExpression {
 
     @Override
     void write(MethodWriter writer) {
-        if (cat) {
-            writer.writeDebugInfo(offset);
-        }
+        // can cause class cast exception among other things at runtime
+        writer.writeDebugInfo(location);
 
         if (cat) {
             writer.writeNewStrings();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index 5bc6e4ee400..c122dba877a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -22,6 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Variables;
@@ -46,8 +47,8 @@ public final class EComp extends AExpression {
     AExpression left;
     AExpression right;
 
-    public EComp(int line, int offset, String location, Operation operation, AExpression left, AExpression right) {
-        super(line, offset, location);
+    public EComp(Location location, Operation operation, AExpression left, AExpression right) {
+        super(location);
 
         this.operation = operation;
         this.left = left;
@@ -73,7 +74,7 @@ public final class EComp extends AExpression {
         } else if (operation == Operation.LT) {
             analyzeLT(variables);
         } else {
-            throw new IllegalStateException(error("Illegal tree structure."));
+            throw createError(new IllegalStateException("Illegal tree structure."));
         }
     }
 
@@ -84,7 +85,7 @@ public final class EComp extends AExpression {
         Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual);
 
         if (promote == null) {
-            throw new ClassCastException(error("Cannot apply equals [==] to types " +
+            throw createError(new ClassCastException("Cannot apply equals [==] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
@@ -95,7 +96,7 @@ public final class EComp extends AExpression {
         right = right.cast(variables);
 
         if (left.isNull && right.isNull) {
-            throw new IllegalArgumentException(error("Extraneous comparison of null constants."));
+            throw createError(new IllegalArgumentException("Extraneous comparison of null constants."));
         }
 
         if ((left.constant != null || left.isNull) && (right.constant != null || right.isNull)) {
@@ -116,7 +117,7 @@ public final class EComp extends AExpression {
             } else if (!right.isNull) {
                 constant = right.constant.equals(null);
             } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
 
@@ -130,7 +131,7 @@ public final class EComp extends AExpression {
         Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual);
 
         if (promote == null) {
-            throw new ClassCastException(error("Cannot apply reference equals [===] to types " +
+            throw createError(new ClassCastException("Cannot apply reference equals [===] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
@@ -141,7 +142,7 @@ public final class EComp extends AExpression {
         right = right.cast(variables);
 
         if (left.isNull && right.isNull) {
-            throw new IllegalArgumentException(error("Extraneous comparison of null constants."));
+            throw createError(new IllegalArgumentException("Extraneous comparison of null constants."));
         }
 
         if ((left.constant != null || left.isNull) && (right.constant != null || right.isNull)) {
@@ -172,7 +173,7 @@ public final class EComp extends AExpression {
         Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual);
 
         if (promote == null) {
-            throw new ClassCastException(error("Cannot apply not equals [!=] to types " +
+            throw createError(new ClassCastException("Cannot apply not equals [!=] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
@@ -183,7 +184,7 @@ public final class EComp extends AExpression {
         right = right.cast(variables);
 
         if (left.isNull && right.isNull) {
-            throw new IllegalArgumentException(error("Extraneous comparison of null constants."));
+            throw createError(new IllegalArgumentException("Extraneous comparison of null constants."));
         }
 
         if ((left.constant != null || left.isNull) && (right.constant != null || right.isNull)) {
@@ -204,7 +205,7 @@ public final class EComp extends AExpression {
             } else if (!right.isNull) {
                 constant = !right.constant.equals(null);
             } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
 
@@ -218,7 +219,7 @@ public final class EComp extends AExpression {
         Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual);
 
         if (promote == null) {
-            throw new ClassCastException(error("Cannot apply reference not equals [!==] to types " +
+            throw createError(new ClassCastException("Cannot apply reference not equals [!==] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
@@ -229,7 +230,7 @@ public final class EComp extends AExpression {
         right = right.cast(variables);
 
         if (left.isNull && right.isNull) {
-            throw new IllegalArgumentException(error("Extraneous comparison of null constants."));
+            throw createError(new IllegalArgumentException("Extraneous comparison of null constants."));
         }
 
         if ((left.constant != null || left.isNull) && (right.constant != null || right.isNull)) {
@@ -260,7 +261,7 @@ public final class EComp extends AExpression {
         Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
         if (promote == null) {
-            throw new ClassCastException(error("Cannot apply greater than or equals [>=] to types " +
+            throw createError(new ClassCastException("Cannot apply greater than or equals [>=] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
@@ -282,7 +283,7 @@ public final class EComp extends AExpression {
             } else if (sort == Sort.DOUBLE) {
                 constant = (double)left.constant >= (double)right.constant;
             } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
 
@@ -296,7 +297,7 @@ public final class EComp extends AExpression {
         Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
         if (promote == null) {
-            throw new ClassCastException(error("Cannot apply greater than [>] to types " +
+            throw createError(new ClassCastException("Cannot apply greater than [>] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
@@ -318,7 +319,7 @@ public final class EComp extends AExpression {
             } else if (sort == Sort.DOUBLE) {
                 constant = (double)left.constant > (double)right.constant;
             } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
 
@@ -332,7 +333,7 @@ public final class EComp extends AExpression {
         Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
         if (promote == null) {
-            throw new ClassCastException(error("Cannot apply less than or equals [<=] to types " +
+            throw createError(new ClassCastException("Cannot apply less than or equals [<=] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
@@ -354,7 +355,7 @@ public final class EComp extends AExpression {
             } else if (sort == Sort.DOUBLE) {
                 constant = (double)left.constant <= (double)right.constant;
             } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
 
@@ -368,7 +369,7 @@ public final class EComp extends AExpression {
         Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
         if (promote == null) {
-            throw new ClassCastException(error("Cannot apply less than [>=] to types " +
+            throw createError(new ClassCastException("Cannot apply less than [>=] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
@@ -390,7 +391,7 @@ public final class EComp extends AExpression {
             } else if (sort == Sort.DOUBLE) {
                 constant = (double)left.constant < (double)right.constant;
             } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
 
@@ -399,7 +400,7 @@ public final class EComp extends AExpression {
 
     @Override
     void write(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         boolean branch = tru != null || fals != null;
         org.objectweb.asm.Type rtype = right.actual.type;
         Sort rsort = right.actual.sort;
@@ -429,12 +430,12 @@ public final class EComp extends AExpression {
             case BYTE:
             case SHORT:
             case CHAR:
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             case BOOL:
                 if      (eq) writer.ifZCmp(MethodWriter.EQ, jump);
                 else if (ne) writer.ifZCmp(MethodWriter.NE, jump);
                 else {
-                    throw new IllegalStateException(error("Illegal tree structure."));
+                    throw createError(new IllegalStateException("Illegal tree structure."));
                 }
 
                 break;
@@ -449,7 +450,7 @@ public final class EComp extends AExpression {
                 else if (gt)  writer.ifCmp(rtype, MethodWriter.GT, jump);
                 else if (gte) writer.ifCmp(rtype, MethodWriter.GE, jump);
                 else {
-                    throw new IllegalStateException(error("Illegal tree structure."));
+                    throw createError(new IllegalStateException("Illegal tree structure."));
                 }
 
                 break;
@@ -485,7 +486,7 @@ public final class EComp extends AExpression {
                     writer.invokeStatic(DEF_UTIL_TYPE, DEF_GTE_CALL);
                     writejump = false;
                 } else {
-                    throw new IllegalStateException(error("Illegal tree structure."));
+                    throw createError(new IllegalStateException("Illegal tree structure."));
                 }
 
                 if (branch && !writejump) {
@@ -518,7 +519,7 @@ public final class EComp extends AExpression {
                         writer.ifCmp(rtype, MethodWriter.NE, jump);
                     }
                 } else {
-                    throw new IllegalStateException(error("Illegal tree structure."));
+                    throw createError(new IllegalStateException("Illegal tree structure."));
                 }
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
index d7f7ab7e4b6..03e1f4c55c5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
@@ -21,6 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
@@ -35,8 +36,8 @@ public final class EConditional extends AExpression {
     AExpression left;
     AExpression right;
 
-    public EConditional(int line, int offset, String location, AExpression condition, AExpression left, AExpression right) {
-        super(line, offset, location);
+    public EConditional(Location location, AExpression condition, AExpression left, AExpression right) {
+        super(location);
 
         this.condition = condition;
         this.left = left;
@@ -50,7 +51,7 @@ public final class EConditional extends AExpression {
         condition = condition.cast(variables);
 
         if (condition.constant != null) {
-            throw new IllegalArgumentException(error("Extraneous conditional statement."));
+            throw createError(new IllegalArgumentException("Extraneous conditional statement."));
         }
 
         left.expected = expected;
@@ -78,7 +79,7 @@ public final class EConditional extends AExpression {
 
     @Override
     void write(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         Label localfals = new Label();
         Label end = new Label();
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
index b45e3c33bc2..8cb032c5c3f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
@@ -21,17 +21,18 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
 
 /**
- * Respresents a constant.  Note this replaces any other expression
+ * Represents a constant.  Note this replaces any other expression
  * node with a constant value set during a cast.  (Internal only.)
  */
 final class EConstant extends AExpression {
 
-    EConstant(int line, int offset, String location, Object constant) {
-        super(line, offset, location);
+    EConstant(Location location, Object constant) {
+        super(location);
 
         this.constant = constant;
     }
@@ -57,7 +58,7 @@ final class EConstant extends AExpression {
         } else if (constant instanceof Boolean) {
             actual = Definition.BOOLEAN_TYPE;
         } else {
-            throw new IllegalStateException(error("Illegal tree structure."));
+            throw createError(new IllegalStateException("Illegal tree structure."));
         }
     }
 
@@ -85,7 +86,7 @@ final class EConstant extends AExpression {
 
                 break;
             default:
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
         }
 
         if (sort != Sort.BOOL) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
index 034a4ec97f9..e59002c2a3b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -30,8 +31,8 @@ public final class EDecimal extends AExpression {
 
     final String value;
 
-    public EDecimal(int line, int offset, String location, String value) {
-        super(line, offset, location);
+    public EDecimal(Location location, String value) {
+        super(location);
 
         this.value = value;
     }
@@ -43,20 +44,20 @@ public final class EDecimal extends AExpression {
                 constant = Float.parseFloat(value.substring(0, value.length() - 1));
                 actual = Definition.FLOAT_TYPE;
             } catch (NumberFormatException exception) {
-                throw new IllegalArgumentException(error("Invalid float constant [" + value + "]."));
+                throw createError(new IllegalArgumentException("Invalid float constant [" + value + "]."));
             }
         } else {
             try {
                 constant = Double.parseDouble(value);
                 actual = Definition.DOUBLE_TYPE;
             } catch (NumberFormatException exception) {
-                throw new IllegalArgumentException(error("Invalid double constant [" + value + "]."));
+                throw createError(new IllegalArgumentException("Invalid double constant [" + value + "]."));
             }
         }
     }
 
     @Override
     void write(MethodWriter writer) {
-        throw new IllegalArgumentException(error("Illegal tree structure."));
+        throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
index 643dc2d4141..0a55727b5e9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -31,8 +32,8 @@ public final class EExplicit extends AExpression {
     final String type;
     AExpression child;
 
-    public EExplicit(int line, int offset, String location, String type, AExpression child) {
-        super(line, offset, location);
+    public EExplicit(Location location, String type, AExpression child) {
+        super(location);
 
         this.type = type;
         this.child = child;
@@ -43,7 +44,7 @@ public final class EExplicit extends AExpression {
         try {
             actual = Definition.getType(this.type);
         } catch (IllegalArgumentException exception) {
-            throw new IllegalArgumentException(error("Not a type [" + this.type + "]."));
+            throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
 
         child.expected = actual;
@@ -54,7 +55,7 @@ public final class EExplicit extends AExpression {
 
     @Override
     void write(MethodWriter writer) {
-        throw new IllegalArgumentException(error("Illegal tree structure."));
+        throw createError(new IllegalStateException("Illegal tree structure."));
     }
 
     AExpression cast(Variables variables) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index 77eb6a00e44..ec2bd6f7e47 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.painless.node;
 
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.Variables;
 
@@ -29,8 +30,8 @@ public class EFunctionRef extends AExpression {
     public String type;
     public String call;
 
-    public EFunctionRef(int line, int offset, String location, String type, String call) {
-        super(line, offset, location);
+    public EFunctionRef(Location location, String type, String call) {
+        super(location);
 
         this.type = type;
         this.call = call;
@@ -38,11 +39,12 @@ public class EFunctionRef extends AExpression {
 
     @Override
     void analyze(Variables variables) {
-        throw new UnsupportedOperationException(error("Function references [" + type + "::" + call + "] are not currently supported."));
+        throw createError(new UnsupportedOperationException("Function references [" + type + "::" + call + 
+                                                            "] are not currently supported."));
     }
 
     @Override
     void write(MethodWriter writer) {
-        throw new IllegalStateException(error("Illegal tree structure."));
+        throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
index 2bb4f4ba585..3f934da4968 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Opcodes;
 import org.elasticsearch.painless.MethodWriter;
@@ -29,8 +30,8 @@ import org.elasticsearch.painless.MethodWriter;
  */
 public final class ENull extends AExpression {
 
-    public ENull(int line, int offset, String location) {
-        super(line, offset, location);
+    public ENull(Location location) {
+        super(location);
     }
 
     @Override
@@ -39,7 +40,7 @@ public final class ENull extends AExpression {
 
         if (expected != null) {
             if (expected.sort.primitive) {
-                throw new IllegalArgumentException(error("Cannot cast null to a primitive type [" + expected.name + "]."));
+                throw createError(new IllegalArgumentException("Cannot cast null to a primitive type [" + expected.name + "]."));
             }
 
             actual = expected;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
index 0fed456dd64..53f3559c294 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
@@ -32,8 +33,8 @@ public final class ENumeric extends AExpression {
     final String value;
     int radix;
 
-    public ENumeric(int line, int offset, String location, String value, int radix) {
-        super(line, offset, location);
+    public ENumeric(Location location, String value, int radix) {
+        super(location);
 
         this.value = value;
         this.radix = radix;
@@ -43,32 +44,32 @@ public final class ENumeric extends AExpression {
     void analyze(Variables variables) {
         if (value.endsWith("d") || value.endsWith("D")) {
             if (radix != 10) {
-                throw new IllegalStateException(error("Invalid tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
 
             try {
                 constant = Double.parseDouble(value.substring(0, value.length() - 1));
                 actual = Definition.DOUBLE_TYPE;
             } catch (NumberFormatException exception) {
-                throw new IllegalArgumentException(error("Invalid double constant [" + value + "]."));
+                throw createError(new IllegalArgumentException("Invalid double constant [" + value + "]."));
             }
         } else if (value.endsWith("f") || value.endsWith("F")) {
             if (radix != 10) {
-                throw new IllegalStateException(error("Invalid tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
 
             try {
                 constant = Float.parseFloat(value.substring(0, value.length() - 1));
                 actual = Definition.FLOAT_TYPE;
             } catch (NumberFormatException exception) {
-                throw new IllegalArgumentException(error("Invalid float constant [" + value + "]."));
+                throw createError(new IllegalArgumentException("Invalid float constant [" + value + "]."));
             }
         } else if (value.endsWith("l") || value.endsWith("L")) {
             try {
                 constant = Long.parseLong(value.substring(0, value.length() - 1), radix);
                 actual = Definition.LONG_TYPE;
             } catch (NumberFormatException exception) {
-                throw new IllegalArgumentException(error("Invalid long constant [" + value + "]."));
+                throw createError(new IllegalArgumentException("Invalid long constant [" + value + "]."));
             }
         } else {
             try {
@@ -89,13 +90,13 @@ public final class ENumeric extends AExpression {
                     actual = Definition.INT_TYPE;
                 }
             } catch (NumberFormatException exception) {
-                throw new IllegalArgumentException(error("Invalid int constant [" + value + "]."));
+                throw createError(new IllegalArgumentException("Invalid int constant [" + value + "]."));
             }
         }
     }
 
     @Override
     void write(MethodWriter writer) {
-        throw new IllegalArgumentException(error("Illegal tree structure."));
+        throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index 2122174442d..aa15999718c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.AnalyzerCaster;
@@ -40,8 +41,8 @@ public final class EUnary extends AExpression {
     final Operation operation;
     AExpression child;
 
-    public EUnary(int line, int offset, String location, Operation operation, AExpression child) {
-        super(line, offset, location);
+    public EUnary(Location location, Operation operation, AExpression child) {
+        super(location);
 
         this.operation = operation;
         this.child = child;
@@ -58,7 +59,7 @@ public final class EUnary extends AExpression {
         } else if (operation == Operation.SUB) {
             analyzerSub(variables);
         } else {
-            throw new IllegalStateException(error("Illegal tree structure."));
+            throw createError(new IllegalStateException("Illegal tree structure."));
         }
     }
 
@@ -80,7 +81,7 @@ public final class EUnary extends AExpression {
         Type promote = AnalyzerCaster.promoteNumeric(child.actual, false);
 
         if (promote == null) {
-            throw new ClassCastException(error("Cannot apply not [~] to type [" + child.actual.name + "]."));
+            throw createError(new ClassCastException("Cannot apply not [~] to type [" + child.actual.name + "]."));
         }
 
         child.expected = promote;
@@ -94,7 +95,7 @@ public final class EUnary extends AExpression {
             } else if (sort == Sort.LONG) {
                 constant = ~(long)child.constant;
             } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
 
@@ -107,7 +108,7 @@ public final class EUnary extends AExpression {
         Type promote = AnalyzerCaster.promoteNumeric(child.actual, true);
 
         if (promote == null) {
-            throw new ClassCastException(error("Cannot apply positive [+] to type [" + child.actual.name + "]."));
+            throw createError(new ClassCastException("Cannot apply positive [+] to type [" + child.actual.name + "]."));
         }
 
         child.expected = promote;
@@ -125,7 +126,7 @@ public final class EUnary extends AExpression {
             } else if (sort == Sort.DOUBLE) {
                 constant = +(double)child.constant;
             } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
 
@@ -138,7 +139,7 @@ public final class EUnary extends AExpression {
         Type promote = AnalyzerCaster.promoteNumeric(child.actual, true);
 
         if (promote == null) {
-            throw new ClassCastException(error("Cannot apply negative [-] to type [" + child.actual.name + "]."));
+            throw createError(new ClassCastException("Cannot apply negative [-] to type [" + child.actual.name + "]."));
         }
 
         child.expected = promote;
@@ -156,7 +157,7 @@ public final class EUnary extends AExpression {
             } else if (sort == Sort.DOUBLE) {
                 constant = -(double)child.constant;
             } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
 
@@ -165,7 +166,7 @@ public final class EUnary extends AExpression {
 
     @Override
     void write(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         if (operation == Operation.NOT) {
             if (tru == null && fals == null) {
                 Label localfals = new Label();
@@ -199,7 +200,7 @@ public final class EUnary extends AExpression {
                     } else if (sort == Sort.LONG) {
                         writer.push(-1L);
                     } else {
-                        throw new IllegalStateException(error("Illegal tree structure."));
+                        throw createError(new IllegalStateException("Illegal tree structure."));
                     }
 
                     writer.math(MethodWriter.XOR, type);
@@ -211,7 +212,7 @@ public final class EUnary extends AExpression {
                     writer.math(MethodWriter.NEG, type);
                 }
             } else if (operation != Operation.ADD) {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
 
             writer.writeBranch(tru, fals);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
index b8e2fc7739b..c80cc8b8ada 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -30,8 +31,8 @@ public final class LArrayLength extends ALink {
 
     final String value;
 
-    LArrayLength(int line, int offset, String location, String value) {
-        super(line, offset, location, -1);
+    LArrayLength(Location location, String value) {
+        super(location, -1);
 
         this.value = value;
     }
@@ -40,14 +41,14 @@ public final class LArrayLength extends ALink {
     ALink analyze(Variables variables) {
         if ("length".equals(value)) {
             if (!load) {
-                throw new IllegalArgumentException(error("Must read array field [length]."));
+                throw createError(new IllegalArgumentException("Must read array field [length]."));
             } else if (store) {
-                throw new IllegalArgumentException(error("Cannot write to read-only array field [length]."));
+                throw createError(new IllegalArgumentException("Cannot write to read-only array field [length]."));
             }
 
             after = Definition.INT_TYPE;
         } else {
-            throw new IllegalArgumentException(error("Illegal field access [" + value + "]."));
+            throw createError(new IllegalArgumentException("Illegal field access [" + value + "]."));
         }
 
         return this;
@@ -60,12 +61,12 @@ public final class LArrayLength extends ALink {
 
     @Override
     void load(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         writer.arrayLength();
     }
 
     @Override
     void store(MethodWriter writer) {
-        throw new IllegalStateException(error("Illegal tree structure."));
+        throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
index 02e8ceacd9f..0c883d36ecf 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
@@ -34,8 +35,8 @@ public final class LBrace extends ALink {
 
     AExpression index;
 
-    public LBrace(int line, int offset, String location, AExpression index) {
-        super(line, offset, location, 2);
+    public LBrace(Location location, AExpression index) {
+        super(location, 2);
 
         this.index = index;
     }
@@ -43,7 +44,7 @@ public final class LBrace extends ALink {
     @Override
     ALink analyze(Variables variables) {
         if (before == null) {
-            throw new IllegalArgumentException(error("Illegal array access made without target."));
+            throw createError(new IllegalArgumentException("Illegal array access made without target."));
         }
 
         final Sort sort = before.sort;
@@ -57,14 +58,14 @@ public final class LBrace extends ALink {
 
             return this;
         } else if (sort == Sort.DEF) {
-            return new LDefArray(line, offset, location, index).copy(this).analyze(variables);
+            return new LDefArray(location, index).copy(this).analyze(variables);
         } else if (Map.class.isAssignableFrom(before.clazz)) {
-            return new LMapShortcut(line, offset, location, index).copy(this).analyze(variables);
+            return new LMapShortcut(location, index).copy(this).analyze(variables);
         } else if (List.class.isAssignableFrom(before.clazz)) {
-            return new LListShortcut(line, offset, location, index).copy(this).analyze(variables);
+            return new LListShortcut(location, index).copy(this).analyze(variables);
         }
 
-        throw new IllegalArgumentException(error("Illegal array access on type [" + before.name + "]."));
+        throw createError(new IllegalArgumentException("Illegal array access on type [" + before.name + "]."));
     }
 
     @Override
@@ -74,13 +75,13 @@ public final class LBrace extends ALink {
 
     @Override
     void load(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         writer.arrayLoad(after.type);
     }
 
     @Override
     void store(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         writer.arrayStore(after.type);
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
index 30775e3a6e7..f7e91790108 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Struct;
@@ -38,8 +39,8 @@ public final class LCall extends ALink {
 
     Method method = null;
 
-    public LCall(int line, int offset, String location, String name, List<AExpression> arguments) {
-        super(line, offset, location, -1);
+    public LCall(Location location, String name, List<AExpression> arguments) {
+        super(location, -1);
 
         this.name = name;
         this.arguments = arguments;
@@ -48,11 +49,11 @@ public final class LCall extends ALink {
     @Override
     ALink analyze(Variables variables) {
         if (before == null) {
-            throw new IllegalArgumentException(error("Illegal call [" + name + "] made without target."));
+            throw createError(new IllegalArgumentException("Illegal call [" + name + "] made without target."));
         } else if (before.sort == Sort.ARRAY) {
-            throw new IllegalArgumentException(error("Illegal call [" + name + "] on array type."));
+            throw createError(new IllegalArgumentException("Illegal call [" + name + "] on array type."));
         } else if (store) {
-            throw new IllegalArgumentException(error("Cannot assign a value to a call [" + name + "]."));
+            throw createError(new IllegalArgumentException("Cannot assign a value to a call [" + name + "]."));
         }
 
         Definition.MethodKey methodKey = new Definition.MethodKey(name, arguments.size());
@@ -74,13 +75,13 @@ public final class LCall extends ALink {
 
             return this;
         } else if (before.sort == Sort.DEF) {
-            ALink link = new LDefCall(line, offset, location, name, arguments);
+            ALink link = new LDefCall(location, name, arguments);
             link.copy(this);
 
             return link.analyze(variables);
         }
 
-        throw new IllegalArgumentException(error("Unknown call [" + name + "] with [" + arguments.size() +
+        throw createError(new IllegalArgumentException("Unknown call [" + name + "] with [" + arguments.size() +
                                                  "] arguments on type [" + struct.name + "]."));
     }
 
@@ -91,7 +92,7 @@ public final class LCall extends ALink {
 
     @Override
     void load(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         for (AExpression argument : arguments) {
             argument.write(writer);
         }
@@ -111,6 +112,6 @@ public final class LCall extends ALink {
 
     @Override
     void store(MethodWriter writer) {
-        throw new IllegalStateException(error("Illegal tree structure."));
+        throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
index bce3f213a71..de4a22a5ad8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.Variables;
@@ -34,8 +35,8 @@ public final class LCast extends ALink {
 
     Cast cast = null;
 
-    public LCast(int line, int offset, String location, String type) {
-        super(line, offset, location, -1);
+    public LCast(Location location, String type) {
+        super(location, -1);
 
         this.type = type;
     }
@@ -43,15 +44,15 @@ public final class LCast extends ALink {
     @Override
     ALink analyze(Variables variables) {
         if (before == null) {
-            throw new IllegalStateException(error("Illegal cast without a target."));
+            throw createError(new IllegalStateException("Illegal cast without a target."));
         } else if (store) {
-            throw new IllegalArgumentException(error("Cannot assign a value to a cast."));
+            throw createError(new IllegalArgumentException("Cannot assign a value to a cast."));
         }
 
         try {
             after = Definition.getType(type);
         } catch (IllegalArgumentException exception) {
-            throw new IllegalArgumentException(error("Not a type [" + type + "]."));
+            throw createError(new IllegalArgumentException("Not a type [" + type + "]."));
         }
 
         cast = AnalyzerCaster.getLegalCast(location, before, after, true, false);
@@ -61,7 +62,7 @@ public final class LCast extends ALink {
 
     @Override
     void write(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         writer.writeCast(cast);
     }
 
@@ -72,6 +73,6 @@ public final class LCast extends ALink {
 
     @Override
     void store(MethodWriter writer) {
-        throw new IllegalStateException(error("Illegal tree structure."));
+        throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index b00eafe3f9f..5fb2ff870ef 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Type;
@@ -34,8 +35,8 @@ final class LDefArray extends ALink implements IDefLink {
 
     AExpression index;
 
-    LDefArray(int line, int offset, String location, AExpression index) {
-        super(line, offset, location, 2);
+    LDefArray(Location location, AExpression index) {
+        super(location, 2);
 
         this.index = index;
     }
@@ -58,14 +59,14 @@ final class LDefArray extends ALink implements IDefLink {
 
     @Override
     void load(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type, index.actual.type);
         writer.invokeDynamic("arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ARRAY_LOAD);
     }
 
     @Override
     void store(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, index.actual.type, after.type);
         writer.invokeDynamic("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ARRAY_STORE);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index 0292ac3c589..3b07d367eba 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
@@ -36,8 +37,8 @@ final class LDefCall extends ALink implements IDefLink {
     final String name;
     final List<AExpression> arguments;
 
-    LDefCall(int line, int offset, String location, String name, List<AExpression> arguments) {
-        super(line, offset, location, -1);
+    LDefCall(Location location, String name, List<AExpression> arguments) {
+        super(location, -1);
 
         this.name = name;
         this.arguments = arguments;
@@ -67,7 +68,7 @@ final class LDefCall extends ALink implements IDefLink {
 
     @Override
     void load(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         StringBuilder signature = new StringBuilder();
 
         signature.append('(');
@@ -88,6 +89,6 @@ final class LDefCall extends ALink implements IDefLink {
 
     @Override
     void store(MethodWriter writer) {
-        throw new IllegalStateException(error("Illegal tree structure."));
+        throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index e762de9e862..2b32d75d89c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Type;
@@ -34,8 +35,8 @@ final class LDefField extends ALink implements IDefLink {
 
     final String value;
 
-    LDefField(int line, int offset, String location, String value) {
-        super(line, offset, location, 1);
+    LDefField(Location location, String value) {
+        super(location, 1);
 
         this.value = value;
     }
@@ -55,14 +56,14 @@ final class LDefField extends ALink implements IDefLink {
 
     @Override
     void load(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type);
         writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.LOAD);
     }
 
     @Override
     void store(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, after.type);
         writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.STORE);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
index 88173ad3f65..b9b7f525a97 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Field;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Struct;
@@ -38,8 +39,8 @@ public final class LField extends ALink {
 
     Field field;
 
-    public LField(int line, int offset, String location, String value) {
-        super(line, offset, location, 1);
+    public LField(Location location, String value) {
+        super(location, 1);
 
         this.value = value;
     }
@@ -47,15 +48,15 @@ public final class LField extends ALink {
     @Override
     ALink analyze(Variables variables) {
         if (before == null) {
-            throw new IllegalArgumentException(error("Illegal field [" + value + "] access made without target."));
+            throw createError(new IllegalArgumentException("Illegal field [" + value + "] access made without target."));
         }
 
         Sort sort = before.sort;
 
         if (sort == Sort.ARRAY) {
-            return new LArrayLength(line, offset, location, value).copy(this).analyze(variables);
+            return new LArrayLength(location, value).copy(this).analyze(variables);
         } else if (sort == Sort.DEF) {
-            return new LDefField(line, offset, location, value).copy(this).analyze(variables);
+            return new LDefField(location, value).copy(this).analyze(variables);
         }
 
         Struct struct = before.struct;
@@ -63,7 +64,7 @@ public final class LField extends ALink {
 
         if (field != null) {
             if (store && java.lang.reflect.Modifier.isFinal(field.modifiers)) {
-                throw new IllegalArgumentException(error(
+                throw createError(new IllegalArgumentException(
                     "Cannot write to read-only field [" + value + "] for type [" + struct.name + "]."));
             }
 
@@ -80,22 +81,22 @@ public final class LField extends ALink {
                     Character.toUpperCase(value.charAt(0)) + value.substring(1), 1));
 
             if (shortcut) {
-                return new LShortcut(line, offset, location, value).copy(this).analyze(variables);
+                return new LShortcut(location, value).copy(this).analyze(variables);
             } else {
-                EConstant index = new EConstant(line, offset, location, value);
+                EConstant index = new EConstant(location, value);
                 index.analyze(variables);
 
                 if (Map.class.isAssignableFrom(before.clazz)) {
-                    return new LMapShortcut(line, offset, location, index).copy(this).analyze(variables);
+                    return new LMapShortcut(location, index).copy(this).analyze(variables);
                 }
 
                 if (List.class.isAssignableFrom(before.clazz)) {
-                    return new LListShortcut(line, offset, location, index).copy(this).analyze(variables);
+                    return new LListShortcut(location, index).copy(this).analyze(variables);
                 }
             }
         }
 
-        throw new IllegalArgumentException(error("Unknown field [" + value + "] for type [" + struct.name + "]."));
+        throw createError(new IllegalArgumentException("Unknown field [" + value + "] for type [" + struct.name + "]."));
     }
 
     @Override
@@ -105,7 +106,7 @@ public final class LField extends ALink {
 
     @Override
     void load(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         if (java.lang.reflect.Modifier.isStatic(field.modifiers)) {
             writer.getStatic(field.owner.type, field.javaName, field.type.type);
         } else {
@@ -115,7 +116,7 @@ public final class LField extends ALink {
 
     @Override
     void store(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         if (java.lang.reflect.Modifier.isStatic(field.modifiers)) {
             writer.putStatic(field.owner.type, field.javaName, field.type.type);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
index b628fee5f37..f1a176181c8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Variables;
@@ -34,8 +35,8 @@ final class LListShortcut extends ALink {
     Method getter;
     Method setter;
 
-    LListShortcut(int line, int offset, String location, AExpression index) {
-        super(line, offset, location, 2);
+    LListShortcut(Location location, AExpression index) {
+        super(location, 2);
 
         this.index = index;
     }
@@ -47,16 +48,16 @@ final class LListShortcut extends ALink {
 
         if (getter != null && (getter.rtn.sort == Sort.VOID || getter.arguments.size() != 1 ||
             getter.arguments.get(0).sort != Sort.INT)) {
-            throw new IllegalArgumentException(error("Illegal list get shortcut for type [" + before.name + "]."));
+            throw createError(new IllegalArgumentException("Illegal list get shortcut for type [" + before.name + "]."));
         }
 
         if (setter != null && (setter.arguments.size() != 2 || setter.arguments.get(0).sort != Sort.INT)) {
-            throw new IllegalArgumentException(error("Illegal list set shortcut for type [" + before.name + "]."));
+            throw createError(new IllegalArgumentException("Illegal list set shortcut for type [" + before.name + "]."));
         }
 
         if (getter != null && setter != null && (!getter.arguments.get(0).equals(setter.arguments.get(0))
             || !getter.rtn.equals(setter.arguments.get(1)))) {
-            throw new IllegalArgumentException(error("Shortcut argument types must match."));
+            throw createError(new IllegalArgumentException("Shortcut argument types must match."));
         }
 
         if ((load || store) && (!load || getter != null) && (!store || setter != null)) {
@@ -66,7 +67,7 @@ final class LListShortcut extends ALink {
 
             after = setter != null ? setter.arguments.get(1) : getter.rtn;
         } else {
-            throw new IllegalArgumentException(error("Illegal list shortcut for type [" + before.name + "]."));
+            throw createError(new IllegalArgumentException("Illegal list shortcut for type [" + before.name + "]."));
         }
 
         return this;
@@ -79,7 +80,7 @@ final class LListShortcut extends ALink {
 
     @Override
     void load(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
             writer.invokeInterface(getter.owner.type, getter.method);
         } else {
@@ -93,7 +94,7 @@ final class LListShortcut extends ALink {
 
     @Override
     void store(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
             writer.invokeInterface(setter.owner.type, setter.method);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
index a500a6673fc..b7f2b96ebd0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Variables;
@@ -34,8 +35,8 @@ final class LMapShortcut extends ALink {
     Method getter;
     Method setter;
 
-    LMapShortcut(int line, int offset, String location, AExpression index) {
-        super(line, offset, location, 2);
+    LMapShortcut(Location location, AExpression index) {
+        super(location, 2);
 
         this.index = index;
     }
@@ -46,16 +47,16 @@ final class LMapShortcut extends ALink {
         setter = before.struct.methods.get(new Definition.MethodKey("put", 2));
 
         if (getter != null && (getter.rtn.sort == Sort.VOID || getter.arguments.size() != 1)) {
-            throw new IllegalArgumentException(error("Illegal map get shortcut for type [" + before.name + "]."));
+            throw createError(new IllegalArgumentException("Illegal map get shortcut for type [" + before.name + "]."));
         }
 
         if (setter != null && setter.arguments.size() != 2) {
-            throw new IllegalArgumentException(error("Illegal map set shortcut for type [" + before.name + "]."));
+            throw createError(new IllegalArgumentException("Illegal map set shortcut for type [" + before.name + "]."));
         }
 
         if (getter != null && setter != null &&
             (!getter.arguments.get(0).equals(setter.arguments.get(0)) || !getter.rtn.equals(setter.arguments.get(1)))) {
-            throw new IllegalArgumentException(error("Shortcut argument types must match."));
+            throw createError(new IllegalArgumentException("Shortcut argument types must match."));
         }
 
         if ((load || store) && (!load || getter != null) && (!store || setter != null)) {
@@ -65,7 +66,7 @@ final class LMapShortcut extends ALink {
 
             after = setter != null ? setter.arguments.get(1) : getter.rtn;
         } else {
-            throw new IllegalArgumentException(error("Illegal map shortcut for type [" + before.name + "]."));
+            throw createError(new IllegalArgumentException("Illegal map shortcut for type [" + before.name + "]."));
         }
 
         return this;
@@ -78,7 +79,7 @@ final class LMapShortcut extends ALink {
 
     @Override
     void load(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
             writer.invokeInterface(getter.owner.type, getter.method);
         } else {
@@ -92,7 +93,7 @@ final class LMapShortcut extends ALink {
 
     @Override
     void store(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
             writer.invokeInterface(setter.owner.type, setter.method);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
index c555f04ab95..cc8c0668eb4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
@@ -34,8 +35,8 @@ public final class LNewArray extends ALink {
     final String type;
     final List<AExpression> arguments;
 
-    public LNewArray(int line, int offset, String location, String type, List<AExpression> arguments) {
-        super(line, offset, location, -1);
+    public LNewArray(Location location, String type, List<AExpression> arguments) {
+        super(location, -1);
 
         this.type = type;
         this.arguments = arguments;
@@ -44,11 +45,11 @@ public final class LNewArray extends ALink {
     @Override
     ALink analyze(Variables variables) {
         if (before != null) {
-            throw new IllegalArgumentException(error("Cannot create a new array with a target already defined."));
+            throw createError(new IllegalArgumentException("Cannot create a new array with a target already defined."));
         } else if (store) {
-            throw new IllegalArgumentException(error("Cannot assign a value to a new array."));
+            throw createError(new IllegalArgumentException("Cannot assign a value to a new array."));
         } else if (!load) {
-            throw new IllegalArgumentException(error("A newly created array must be read."));
+            throw createError(new IllegalArgumentException("A newly created array must be read."));
         }
 
         final Type type;
@@ -56,7 +57,7 @@ public final class LNewArray extends ALink {
         try {
             type = Definition.getType(this.type);
         } catch (IllegalArgumentException exception) {
-            throw new IllegalArgumentException(error("Not a type [" + this.type + "]."));
+            throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
 
         for (int argument = 0; argument < arguments.size(); ++argument) {
@@ -79,7 +80,7 @@ public final class LNewArray extends ALink {
 
     @Override
     void load(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         for (AExpression argument : arguments) {
             argument.write(writer);
         }
@@ -93,6 +94,6 @@ public final class LNewArray extends ALink {
 
     @Override
     void store(MethodWriter writer) {
-        throw new IllegalStateException(error("Illegal tree structure."));
+        throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
index 44738450962..a27aa25754d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Constructor;
 import org.elasticsearch.painless.Definition.Struct;
 import org.elasticsearch.painless.Definition.Type;
@@ -29,7 +30,7 @@ import org.elasticsearch.painless.MethodWriter;
 import java.util.List;
 
 /**
- * Respresents and object instantiation.
+ * Represents and object instantiation.
  */
 public final class LNewObj extends ALink {
 
@@ -38,8 +39,8 @@ public final class LNewObj extends ALink {
 
     Constructor constructor;
 
-    public LNewObj(int line, int offset, String location, String type, List<AExpression> arguments) {
-        super(line, offset, location, -1);
+    public LNewObj(Location location, String type, List<AExpression> arguments) {
+        super(location, -1);
 
         this.type = type;
         this.arguments = arguments;
@@ -48,9 +49,9 @@ public final class LNewObj extends ALink {
     @Override
     ALink analyze(Variables variables) {
         if (before != null) {
-            throw new IllegalArgumentException(error("Illegal new call with a target already defined."));
+            throw createError(new IllegalArgumentException("Illegal new call with a target already defined."));
         } else if (store) {
-            throw new IllegalArgumentException(error("Cannot assign a value to a new call."));
+            throw createError(new IllegalArgumentException("Cannot assign a value to a new call."));
         }
 
         final Type type;
@@ -58,7 +59,7 @@ public final class LNewObj extends ALink {
         try {
             type = Definition.getType(this.type);
         } catch (IllegalArgumentException exception) {
-            throw new IllegalArgumentException(error("Not a type [" + this.type + "]."));
+            throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
 
         Struct struct = type.struct;
@@ -69,7 +70,7 @@ public final class LNewObj extends ALink {
             constructor.arguments.toArray(types);
 
             if (constructor.arguments.size() != arguments.size()) {
-                throw new IllegalArgumentException(error("When calling constructor on type [" + struct.name + "]" +
+                throw createError(new IllegalArgumentException("When calling constructor on type [" + struct.name + "]" +
                     " expected [" + constructor.arguments.size() + "] arguments, but found [" + arguments.size() + "]."));
             }
 
@@ -85,7 +86,7 @@ public final class LNewObj extends ALink {
             statement = true;
             after = type;
         } else {
-            throw new IllegalArgumentException(error("Unknown new call on type [" + struct.name + "]."));
+            throw createError(new IllegalArgumentException("Unknown new call on type [" + struct.name + "]."));
         }
 
         return this;
@@ -98,7 +99,7 @@ public final class LNewObj extends ALink {
 
     @Override
     void load(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         writer.newInstance(after.type);
 
         if (load) {
@@ -114,6 +115,6 @@ public final class LNewObj extends ALink {
 
     @Override
     void store(MethodWriter writer) {
-        throw new IllegalStateException(error("Illegal tree structure."));
+        throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
index 7eb44156e55..5e4928c04d5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Struct;
@@ -36,8 +37,8 @@ final class LShortcut extends ALink {
     Method getter = null;
     Method setter = null;
 
-    LShortcut(int line, int offset, String location, String value) {
-        super(line, offset, location, 1);
+    LShortcut(Location location, String value) {
+        super(location, 1);
 
         this.value = value;
     }
@@ -55,23 +56,23 @@ final class LShortcut extends ALink {
         setter = struct.methods.get(new Definition.MethodKey("set" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 1));
 
         if (getter != null && (getter.rtn.sort == Sort.VOID || !getter.arguments.isEmpty())) {
-            throw new IllegalArgumentException(error(
+            throw createError(new IllegalArgumentException(
                 "Illegal get shortcut on field [" + value + "] for type [" + struct.name + "]."));
         }
 
         if (setter != null && (setter.rtn.sort != Sort.VOID || setter.arguments.size() != 1)) {
-            throw new IllegalArgumentException(error(
+            throw createError(new IllegalArgumentException(
                 "Illegal set shortcut on field [" + value + "] for type [" + struct.name + "]."));
         }
 
         if (getter != null && setter != null && setter.arguments.get(0) != getter.rtn) {
-            throw new IllegalArgumentException(error("Shortcut argument types must match."));
+            throw createError(new IllegalArgumentException("Shortcut argument types must match."));
         }
 
         if ((getter != null || setter != null) && (!load || getter != null) && (!store || setter != null)) {
             after = setter != null ? setter.arguments.get(0) : getter.rtn;
         } else {
-            throw new IllegalArgumentException(error("Illegal shortcut on field [" + value + "] for type [" + struct.name + "]."));
+            throw createError(new IllegalArgumentException("Illegal shortcut on field [" + value + "] for type [" + struct.name + "]."));
         }
 
         return this;
@@ -84,7 +85,7 @@ final class LShortcut extends ALink {
 
     @Override
     void load(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
             writer.invokeInterface(getter.owner.type, getter.method);
         } else {
@@ -98,7 +99,7 @@ final class LShortcut extends ALink {
 
     @Override
     void store(MethodWriter writer) {
-        writer.writeDebugInfo(offset);
+        writer.writeDebugInfo(location);
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
             writer.invokeInterface(setter.owner.type, setter.method);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java
index 8919b75dc27..98774513188 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.Variables;
 
@@ -30,8 +31,8 @@ public final class LStatic extends ALink {
 
     final String type;
 
-    public LStatic(int line, int offset, String location, String type) {
-        super(line, offset, location, 0);
+    public LStatic(Location location, String type) {
+        super(location, 0);
 
         this.type = type;
     }
@@ -39,14 +40,14 @@ public final class LStatic extends ALink {
     @Override
     ALink analyze(Variables variables) {
         if (before != null) {
-            throw new IllegalArgumentException(error("Illegal static type [" + type + "] after target already defined."));
+            throw createError(new IllegalArgumentException("Illegal static type [" + type + "] after target already defined."));
         }
 
         try {
             after = Definition.getType(type);
             statik = true;
         } catch (IllegalArgumentException exception) {
-            throw new IllegalArgumentException(error("Not a type [" + type + "]."));
+            throw createError(new IllegalArgumentException("Not a type [" + type + "]."));
         }
 
         return this;
@@ -54,16 +55,16 @@ public final class LStatic extends ALink {
 
     @Override
     void write(MethodWriter writer) {
-        throw new IllegalStateException(error("Illegal tree structure."));
+        throw createError(new IllegalStateException("Illegal tree structure."));
     }
 
     @Override
     void load(MethodWriter writer) {
-        throw new IllegalStateException(error("Illegal tree structure."));
+        throw createError(new IllegalStateException("Illegal tree structure."));
     }
 
     @Override
     void store(MethodWriter writer) {
-        throw new IllegalStateException(error("Illegal tree structure."));
+        throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
index 446750a22bb..6afe62d02ba 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -28,8 +29,8 @@ import org.elasticsearch.painless.MethodWriter;
  */
 public final class LString extends ALink {
 
-    public LString(int line, int offset, String location, String string) {
-        super(line, offset, location, -1);
+    public LString(Location location, String string) {
+        super(location, -1);
 
         this.string = string;
     }
@@ -37,11 +38,11 @@ public final class LString extends ALink {
     @Override
     ALink analyze(Variables variables) {
         if (before != null) {
-            throw new IllegalArgumentException(error("Illegal String constant [" + string + "]."));
+            throw createError(new IllegalArgumentException("Illegal String constant [" + string + "]."));
         } else if (store) {
-            throw new IllegalArgumentException(error("Cannot write to read-only String constant [" + string + "]."));
+            throw createError(new IllegalArgumentException("Cannot write to read-only String constant [" + string + "]."));
         } else if (!load) {
-            throw new IllegalArgumentException(error("Must read String constant [" + string + "]."));
+            throw createError(new IllegalArgumentException("Must read String constant [" + string + "]."));
         }
 
         after = Definition.STRING_TYPE;
@@ -61,6 +62,6 @@ public final class LString extends ALink {
 
     @Override
     void store(MethodWriter writer) {
-        throw new IllegalStateException(error("Illegal tree structure."));
+        throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
index 3a555d93c87..8fe6f17b0b5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.painless.node;
 
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.Variables.Variable;
@@ -33,8 +34,8 @@ public final class LVariable extends ALink {
 
     int slot;
 
-    public LVariable(int line, int offset, String location, String name) {
-        super(line, offset, location, 0);
+    public LVariable(Location location, String name) {
+        super(location, 0);
 
         this.name = name;
     }
@@ -42,13 +43,13 @@ public final class LVariable extends ALink {
     @Override
     ALink analyze(Variables variables) {
         if (before != null) {
-            throw new IllegalArgumentException(error("Illegal variable [" + name + "] access with target already defined."));
+            throw createError(new IllegalArgumentException("Illegal variable [" + name + "] access with target already defined."));
         }
 
         Variable variable = variables.getVariable(location, name);
 
         if (store && variable.readonly) {
-            throw new IllegalArgumentException(error("Variable [" + variable.name + "] is read-only."));
+            throw createError(new IllegalArgumentException("Variable [" + variable.name + "] is read-only."));
         }
 
         slot = variable.slot;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
index 0f05243f994..e1dd81ee54e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
 import java.util.Collections;
@@ -32,8 +33,8 @@ public final class SBlock extends AStatement {
 
     final List<AStatement> statements;
 
-    public SBlock(int line, int offset, String location, List<AStatement> statements) {
-        super(line, offset, location);
+    public SBlock(Location location, List<AStatement> statements) {
+        super(location);
 
         this.statements = Collections.unmodifiableList(statements);
     }
@@ -41,14 +42,14 @@ public final class SBlock extends AStatement {
     @Override
     void analyze(Variables variables) {
         if (statements == null || statements.isEmpty()) {
-            throw new IllegalArgumentException(error("A block must contain at least one statement."));
+            throw createError(new IllegalArgumentException("A block must contain at least one statement."));
         }
 
         final AStatement last = statements.get(statements.size() - 1);
 
         for (AStatement statement : statements) {
             if (allEscape) {
-                throw new IllegalArgumentException(error("Unreachable statement."));
+                throw createError(new IllegalArgumentException("Unreachable statement."));
             }
 
             statement.inLoop = inLoop;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
index 4cd1decb26b..ca72dd0b55b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -27,14 +28,14 @@ import org.elasticsearch.painless.MethodWriter;
  */
 public final class SBreak extends AStatement {
 
-    public SBreak(int line, int offset, String location) {
-        super(line, offset, location);
+    public SBreak(Location location) {
+        super(location);
     }
 
     @Override
     void analyze(Variables variables) {
         if (!inLoop) {
-            throw new IllegalArgumentException(error("Break statement outside of a loop."));
+            throw createError(new IllegalArgumentException("Break statement outside of a loop."));
         }
 
         loopEscape = true;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
index 3fb8c8b4680..d78a4539580 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.Variables.Variable;
@@ -42,8 +43,8 @@ public final class SCatch extends AStatement {
     Label end;
     Label exception;
 
-    public SCatch(int line, int offset, String location, String type, String name, SBlock block) {
-        super(line, offset, location);
+    public SCatch(Location location, String type, String name, SBlock block) {
+        super(location);
 
         this.type = type;
         this.name = name;
@@ -57,11 +58,11 @@ public final class SCatch extends AStatement {
         try {
             type = Definition.getType(this.type);
         } catch (IllegalArgumentException exception) {
-            throw new IllegalArgumentException(error("Not a type [" + this.type + "]."));
+            throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
 
         if (!Exception.class.isAssignableFrom(type.clazz)) {
-            throw new ClassCastException(error("Not an exception type [" + this.type + "]."));
+            throw createError(new ClassCastException("Not an exception type [" + this.type + "]."));
         }
 
         variable = variables.addVariable(location, type, name, true, false);
@@ -84,7 +85,7 @@ public final class SCatch extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writer.writeStatementOffset(offset);
+        writer.writeStatementOffset(location);
         Label jump = new Label();
 
         writer.mark(jump);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
index 18ce5c81231..ef80766bdd1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -27,18 +28,18 @@ import org.elasticsearch.painless.MethodWriter;
  */
 public final class SContinue extends AStatement {
 
-    public SContinue(int line, int offset, String location) {
-        super(line, offset, location);
+    public SContinue(Location location) {
+        super(location);
     }
 
     @Override
     void analyze(Variables variables) {
         if (!inLoop) {
-            throw new IllegalArgumentException(error("Continue statement outside of a loop."));
+            throw createError(new IllegalArgumentException("Continue statement outside of a loop."));
         }
 
         if (lastLoop) {
-            throw new IllegalArgumentException(error("Extraneous continue statement."));
+            throw createError(new IllegalArgumentException("Extraneous continue statement."));
         }
 
         allEscape = true;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
index ddfe54c22eb..e52819c7959 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
 import java.util.Collections;
@@ -32,8 +33,8 @@ public final class SDeclBlock extends AStatement {
 
     final List<SDeclaration> declarations;
 
-    public SDeclBlock(int line, int offset, String location, List<SDeclaration> declarations) {
-        super(line, offset, location);
+    public SDeclBlock(Location location, List<SDeclaration> declarations) {
+        super(location);
 
         this.declarations = Collections.unmodifiableList(declarations);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
index 526d90dca26..38c9bad315b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.Variables.Variable;
@@ -37,8 +38,8 @@ public final class SDeclaration extends AStatement {
 
     Variable variable;
 
-    public SDeclaration(int line, int offset, String location, String type, String name, AExpression expression) {
-        super(line, offset, location);
+    public SDeclaration(Location location, String type, String name, AExpression expression) {
+        super(location);
 
         this.type = type;
         this.name = name;
@@ -52,7 +53,7 @@ public final class SDeclaration extends AStatement {
         try {
             type = Definition.getType(this.type);
         } catch (IllegalArgumentException exception) {
-            throw new IllegalArgumentException(error("Not a type [" + this.type + "]."));
+            throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
 
         if (expression != null) {
@@ -66,10 +67,10 @@ public final class SDeclaration extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writer.writeStatementOffset(offset);
+        writer.writeStatementOffset(location);
         if (expression == null) {
             switch (variable.type.sort) {
-                case VOID:   throw new IllegalStateException(error("Illegal tree structure."));
+                case VOID:   throw createError(new IllegalStateException("Illegal tree structure."));
                 case BOOL:
                 case BYTE:
                 case SHORT:
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
index 3b854ea4ebf..a8c76c98dcc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.MethodWriter;
@@ -33,8 +34,8 @@ public final class SDo extends AStatement {
     final SBlock block;
     AExpression condition;
 
-    public SDo(int line, int offset, String location, int maxLoopCounter, SBlock block, AExpression condition) {
-        super(line, offset, location);
+    public SDo(Location location, int maxLoopCounter, SBlock block, AExpression condition) {
+        super(location);
 
         this.condition = condition;
         this.block = block;
@@ -46,7 +47,7 @@ public final class SDo extends AStatement {
         variables.incrementScope();
 
         if (block == null) {
-            throw new IllegalArgumentException(error("Extraneous do while loop."));
+            throw createError(new IllegalArgumentException("Extraneous do while loop."));
         }
 
         block.beginLoop = true;
@@ -55,7 +56,7 @@ public final class SDo extends AStatement {
         block.analyze(variables);
 
         if (block.loopEscape && !block.anyContinue) {
-            throw new IllegalArgumentException(error("Extraneous do while loop."));
+            throw createError(new IllegalArgumentException("Extraneous do while loop."));
         }
 
         condition.expected = Definition.BOOLEAN_TYPE;
@@ -66,7 +67,7 @@ public final class SDo extends AStatement {
             final boolean continuous = (boolean)condition.constant;
 
             if (!continuous) {
-                throw new IllegalArgumentException(error("Extraneous do while loop."));
+                throw createError(new IllegalArgumentException("Extraneous do while loop."));
             }
 
             if (!block.anyBreak) {
@@ -86,7 +87,7 @@ public final class SDo extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writer.writeStatementOffset(offset);
+        writer.writeStatementOffset(location);
         Label start = new Label();
         Label begin = new Label();
         Label end = new Label();
@@ -102,7 +103,7 @@ public final class SDo extends AStatement {
         condition.fals = end;
         condition.write(writer);
 
-        writer.writeLoopCounter(loopCounterSlot, Math.max(1, block.statementCount), offset);
+        writer.writeLoopCounter(loopCounterSlot, Math.max(1, block.statementCount), location);
 
         writer.goTo(start);
         writer.mark(end);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index 37f1cbeb24c..1bea07d5599 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
@@ -31,8 +32,8 @@ public final class SExpression extends AStatement {
 
     AExpression expression;
 
-    public SExpression(int line, int offset, String location, AExpression expression) {
-        super(line, offset, location);
+    public SExpression(Location location, AExpression expression) {
+        super(location);
 
         this.expression = expression;
     }
@@ -43,7 +44,7 @@ public final class SExpression extends AStatement {
         expression.analyze(variables);
 
         if (!lastSource && !expression.statement) {
-            throw new IllegalArgumentException(error("Not a statement."));
+            throw createError(new IllegalArgumentException("Not a statement."));
         }
 
         final boolean rtn = lastSource && expression.actual.sort != Sort.VOID;
@@ -60,7 +61,7 @@ public final class SExpression extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writer.writeStatementOffset(offset);
+        writer.writeStatementOffset(location);
         expression.write(writer);
 
         if (methodEscape) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index ac8e721a1f6..58fdb33fb3c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.MethodWriter;
@@ -35,9 +36,9 @@ public final class SFor extends AStatement {
     AExpression afterthought;
     final SBlock block;
 
-    public SFor(int line, int offset, String location, int maxLoopCounter,
+    public SFor(Location location, int maxLoopCounter,
                 ANode initializer, AExpression condition, AExpression afterthought, SBlock block) {
-        super(line, offset, location);
+        super(location);
 
         this.initializer = initializer;
         this.condition = condition;
@@ -62,10 +63,10 @@ public final class SFor extends AStatement {
                 initializer.analyze(variables);
 
                 if (!initializer.statement) {
-                    throw new IllegalArgumentException(initializer.error("Not a statement."));
+                    throw createError(new IllegalArgumentException("Not a statement."));
                 }
             } else {
-                throw new IllegalStateException(error("Illegal tree structure."));
+                throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
 
@@ -78,11 +79,11 @@ public final class SFor extends AStatement {
                 continuous = (boolean)condition.constant;
 
                 if (!continuous) {
-                    throw new IllegalArgumentException(error("Extraneous for loop."));
+                    throw createError(new IllegalArgumentException("Extraneous for loop."));
                 }
 
                 if (block == null) {
-                    throw new IllegalArgumentException(error("For loop has no escape."));
+                    throw createError(new IllegalArgumentException("For loop has no escape."));
                 }
             }
         } else {
@@ -94,7 +95,7 @@ public final class SFor extends AStatement {
             afterthought.analyze(variables);
 
             if (!afterthought.statement) {
-                throw new IllegalArgumentException(afterthought.error("Not a statement."));
+                throw createError(new IllegalArgumentException("Not a statement."));
             }
         }
 
@@ -105,7 +106,7 @@ public final class SFor extends AStatement {
             block.analyze(variables);
 
             if (block.loopEscape && !block.anyContinue) {
-                throw new IllegalArgumentException(error("Extraneous for loop."));
+                throw createError(new IllegalArgumentException("Extraneous for loop."));
             }
 
             if (continuous && !block.anyBreak) {
@@ -127,7 +128,7 @@ public final class SFor extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writer.writeStatementOffset(offset);
+        writer.writeStatementOffset(location);
         Label start = new Label();
         Label begin = afterthought == null ? start : new Label();
         Label end = new Label();
@@ -159,10 +160,10 @@ public final class SFor extends AStatement {
                 ++statementCount;
             }
 
-            writer.writeLoopCounter(loopCounterSlot, statementCount, offset);
+            writer.writeLoopCounter(loopCounterSlot, statementCount, location);
             block.write(writer);
         } else {
-            writer.writeLoopCounter(loopCounterSlot, 1, offset);
+            writer.writeLoopCounter(loopCounterSlot, 1, location);
         }
 
         if (afterthought != null) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
index 1f498215478..5c871db5eb9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.MethodWriter;
@@ -32,8 +33,8 @@ public final class SIf extends AStatement {
     AExpression condition;
     final SBlock ifblock;
 
-    public SIf(int line, int offset, String location, AExpression condition, SBlock ifblock) {
-        super(line, offset, location);
+    public SIf(Location location, AExpression condition, SBlock ifblock) {
+        super(location);
 
         this.condition = condition;
         this.ifblock = ifblock;
@@ -46,11 +47,11 @@ public final class SIf extends AStatement {
         condition = condition.cast(variables);
 
         if (condition.constant != null) {
-            throw new IllegalArgumentException(error("Extraneous if statement."));
+            throw createError(new IllegalArgumentException("Extraneous if statement."));
         }
 
         if (ifblock == null) {
-            throw new IllegalArgumentException(error("Extraneous if statement."));
+            throw createError(new IllegalArgumentException("Extraneous if statement."));
         }
 
         ifblock.lastSource = lastSource;
@@ -68,7 +69,7 @@ public final class SIf extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writer.writeStatementOffset(offset);
+        writer.writeStatementOffset(location);
         Label fals = new Label();
 
         condition.fals = fals;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
index d8fd5c0756e..7c552ac40e2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.MethodWriter;
@@ -33,8 +34,8 @@ public final class SIfElse extends AStatement {
     final SBlock ifblock;
     final SBlock elseblock;
 
-    public SIfElse(int line, int offset, String location, AExpression condition, SBlock ifblock, SBlock elseblock) {
-        super(line, offset, location);
+    public SIfElse(Location location, AExpression condition, SBlock ifblock, SBlock elseblock) {
+        super(location);
 
         this.condition = condition;
         this.ifblock = ifblock;
@@ -48,11 +49,11 @@ public final class SIfElse extends AStatement {
         condition = condition.cast(variables);
 
         if (condition.constant != null) {
-            throw new IllegalArgumentException(error("Extraneous if statement."));
+            throw createError(new IllegalArgumentException("Extraneous if statement."));
         }
 
         if (ifblock == null) {
-            throw new IllegalArgumentException(error("Extraneous if statement."));
+            throw createError(new IllegalArgumentException("Extraneous if statement."));
         }
 
         ifblock.lastSource = lastSource;
@@ -68,7 +69,7 @@ public final class SIfElse extends AStatement {
         statementCount = ifblock.statementCount;
 
         if (elseblock == null) {
-            throw new IllegalArgumentException(error("Extraneous else statement."));
+            throw createError(new IllegalArgumentException("Extraneous else statement."));
         }
 
         elseblock.lastSource = lastSource;
@@ -89,7 +90,7 @@ public final class SIfElse extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writer.writeStatementOffset(offset);
+        writer.writeStatementOffset(location);
         Label end = new Label();
         Label fals = elseblock != null ? new Label() : end;
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
index d6cc5598263..bd9707eb6a4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -30,8 +31,8 @@ public final class SReturn extends AStatement {
 
     AExpression expression;
 
-    public SReturn(int line, int offset, String location, AExpression expression) {
-        super(line, offset, location);
+    public SReturn(Location location, AExpression expression) {
+        super(location);
 
         this.expression = expression;
     }
@@ -52,7 +53,7 @@ public final class SReturn extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writer.writeStatementOffset(offset);
+        writer.writeStatementOffset(location);
         expression.write(writer);
         writer.returnValue();
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index 899d54225dc..510c54da7a7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -21,6 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Opcodes;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
 import java.util.Collections;
@@ -33,8 +34,8 @@ public final class SSource extends AStatement {
 
     final List<AStatement> statements;
 
-    public SSource(int line, int offset, String location, List<AStatement> statements) {
-        super(line, offset, location);
+    public SSource(Location location, List<AStatement> statements) {
+        super(location);
 
         this.statements = Collections.unmodifiableList(statements);
     }
@@ -42,7 +43,7 @@ public final class SSource extends AStatement {
     @Override
     public void analyze(Variables variables) {
         if (statements == null || statements.isEmpty()) {
-            throw new IllegalArgumentException(error("Cannot generate an empty script."));
+            throw createError(new IllegalArgumentException("Cannot generate an empty script."));
         }
 
         variables.incrementScope();
@@ -50,8 +51,9 @@ public final class SSource extends AStatement {
         final AStatement last = statements.get(statements.size() - 1);
 
         for (AStatement statement : statements) {
+            // TODO: why are we checking only statements 0..n-1 (this effectively checks only the previous statement)
             if (allEscape) {
-                throw new IllegalArgumentException(error("Unreachable statement."));
+                throw createError(new IllegalArgumentException("Unreachable statement."));
             }
 
             statement.lastSource = statement == last;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
index 7c65aafbc56..af9d7a65990 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -30,8 +31,8 @@ public final class SThrow extends AStatement {
 
     AExpression expression;
 
-    public SThrow(int line, int offset, String location, AExpression expression) {
-        super(line, offset, location);
+    public SThrow(Location location, AExpression expression) {
+        super(location);
 
         this.expression = expression;
     }
@@ -50,7 +51,7 @@ public final class SThrow extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writer.writeStatementOffset(offset);
+        writer.writeStatementOffset(location);
         expression.write(writer);
         writer.throwException();
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
index 0fdb70bdc82..b522724280a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
@@ -21,6 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
 import java.util.Collections;
@@ -34,8 +35,8 @@ public final class STry extends AStatement {
     final SBlock block;
     final List<SCatch> catches;
 
-    public STry(int line, int offset, String location, SBlock block, List<SCatch> traps) {
-        super(line, offset, location);
+    public STry(Location location, SBlock block, List<SCatch> traps) {
+        super(location);
 
         this.block = block;
         this.catches = Collections.unmodifiableList(traps);
@@ -44,7 +45,7 @@ public final class STry extends AStatement {
     @Override
     void analyze(Variables variables) {
         if (block == null) {
-            throw new IllegalArgumentException(error("Extraneous try statement."));
+            throw createError(new IllegalArgumentException("Extraneous try statement."));
         }
 
         block.lastSource = lastSource;
@@ -86,7 +87,7 @@ public final class STry extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writer.writeStatementOffset(offset);
+        writer.writeStatementOffset(location);
         Label begin = new Label();
         Label end = new Label();
         Label exception = new Label();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
index 43ac824dac7..4b071be36b5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Variables;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.MethodWriter;
@@ -33,8 +34,8 @@ public final class SWhile extends AStatement {
     AExpression condition;
     final SBlock block;
 
-    public SWhile(int line, int offset, String location, int maxLoopCounter, AExpression condition, SBlock block) {
-        super(line, offset, location);
+    public SWhile(Location location, int maxLoopCounter, AExpression condition, SBlock block) {
+        super(location);
 
         this.maxLoopCounter = maxLoopCounter;
         this.condition = condition;
@@ -55,11 +56,11 @@ public final class SWhile extends AStatement {
             continuous = (boolean)condition.constant;
 
             if (!continuous) {
-                throw new IllegalArgumentException(error("Extraneous while loop."));
+                throw createError(new IllegalArgumentException("Extraneous while loop."));
             }
 
             if (block == null) {
-                throw new IllegalArgumentException(error("While loop has no escape."));
+                throw createError(new IllegalArgumentException("While loop has no escape."));
             }
         }
 
@@ -70,7 +71,7 @@ public final class SWhile extends AStatement {
             block.analyze(variables);
 
             if (block.loopEscape && !block.anyContinue) {
-                throw new IllegalArgumentException(error("Extraneous while loop."));
+                throw createError(new IllegalArgumentException("Extraneous while loop."));
             }
 
             if (continuous && !block.anyBreak) {
@@ -92,7 +93,7 @@ public final class SWhile extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        writer.writeStatementOffset(offset);
+        writer.writeStatementOffset(location);
         Label begin = new Label();
         Label end = new Label();
 
@@ -102,13 +103,13 @@ public final class SWhile extends AStatement {
         condition.write(writer);
 
         if (block != null) {
-            writer.writeLoopCounter(loopCounterSlot, Math.max(1, block.statementCount), offset);
+            writer.writeLoopCounter(loopCounterSlot, Math.max(1, block.statementCount), location);
 
             block.continu = begin;
             block.brake = end;
             block.write(writer);
         } else {
-            writer.writeLoopCounter(loopCounterSlot, 1, offset);
+            writer.writeLoopCounter(loopCounterSlot, 1, location);
         }
 
         if (block == null || !block.allEscape) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
index 51825f41d70..13ef13bdd77 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
@@ -24,8 +24,8 @@
  * A* (abstract) - These are the abstract nodes that are the superclasses for the other types.
  * I* (interface) -- Thse are marker interfaces to denote a property of the node.
  * S* (statement) - These are nodes that represent a statement in Painless.  These are the highest level nodes.
- * E* (expression) - These are nodess that represent an expression in Painless.  These are the middle level nodes.
- * L* (link) - These are nodes that respresent a piece of a variable/method chain.  The are the lowest level nodes.
+ * E* (expression) - These are nodes that represent an expression in Painless.  These are the middle level nodes.
+ * L* (link) - These are nodes that represent a piece of a variable/method chain.  The are the lowest level nodes.
  * <p>
  * The following is a brief description of each node:
  * {@link org.elasticsearch.painless.node.AExpression} - The superclass for all E* (expression) nodes.
@@ -38,18 +38,18 @@
  * {@link org.elasticsearch.painless.node.ECast} - Represents an implicit cast in most cases.  (Internal only.)
  * {@link org.elasticsearch.painless.node.EChain} - Represents the entirety of a variable/method chain for read/write operations.
  * {@link org.elasticsearch.painless.node.EComp} - Represents a comparison expression.
- * {@link org.elasticsearch.painless.node.EConditional} - Respresents a conditional expression.
- * {@link org.elasticsearch.painless.node.EConstant} - Respresents a constant.  (Internal only.)
- * {@link org.elasticsearch.painless.node.EDecimal} - Respresents a decimal constant.
+ * {@link org.elasticsearch.painless.node.EConditional} - Represents a conditional expression.
+ * {@link org.elasticsearch.painless.node.EConstant} - Represents a constant.  (Internal only.)
+ * {@link org.elasticsearch.painless.node.EDecimal} - Represents a decimal constant.
  * {@link org.elasticsearch.painless.node.EExplicit} - Represents an explicit cast.
  * {@link org.elasticsearch.painless.node.EFunctionRef} - Represents a function reference.
  * {@link org.elasticsearch.painless.node.ENull} - Represents a null constant.
- * {@link org.elasticsearch.painless.node.ENumeric} - Respresents a non-decimal numeric constant.
+ * {@link org.elasticsearch.painless.node.ENumeric} - Represents a non-decimal numeric constant.
  * {@link org.elasticsearch.painless.node.EUnary} - Represents a unary math expression.
  * {@link org.elasticsearch.painless.node.IDefLink} - A marker interface for all LDef* (link) nodes.
  * {@link org.elasticsearch.painless.node.LArrayLength} - Represents an array length field load.
  * {@link org.elasticsearch.painless.node.LBrace} - Represents an array load/store or defers to possible shortcuts.
- * {@link org.elasticsearch.painless.node.LCall} - Represents a method call or deferes to a def call.
+ * {@link org.elasticsearch.painless.node.LCall} - Represents a method call or defers to a def call.
  * {@link org.elasticsearch.painless.node.LCast} - Represents a cast made in a variable/method chain.
  * {@link org.elasticsearch.painless.node.LDefArray} - Represents an array load/store or shortcut on a def type.  (Internal only.)
  * {@link org.elasticsearch.painless.node.LDefCall} - Represents a method call made on a def type. (Internal only.)
@@ -58,7 +58,7 @@
  * {@link org.elasticsearch.painless.node.LListShortcut} - Represents a list load/store shortcut.  (Internal only.)
  * {@link org.elasticsearch.painless.node.LMapShortcut} - Represents a map load/store shortcut. (Internal only.)
  * {@link org.elasticsearch.painless.node.LNewArray} - Represents an array instantiation.
- * {@link org.elasticsearch.painless.node.LNewObj} - Respresents and object instantiation.
+ * {@link org.elasticsearch.painless.node.LNewObj} - Represents and object instantiation.
  * {@link org.elasticsearch.painless.node.LShortcut} - Represents a field load/store shortcut.  (Internal only.)
  * {@link org.elasticsearch.painless.node.LStatic} - Represents a static type target.
  * {@link org.elasticsearch.painless.node.LString} - Represents a string constant.
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ConditionalTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ConditionalTests.java
index a3a09fee425..233f50ab93e 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ConditionalTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ConditionalTests.java
@@ -70,24 +70,20 @@ public class ConditionalTests extends ScriptTestCase {
     }
 
     public void testIncompatibleAssignment() {
-        try {
+        expectScriptThrows(ClassCastException.class, () -> {
             exec("boolean x = false; byte z = x ? 2 : 4.0F; return z;");
-            fail("expected class cast exception");
-        } catch (ClassCastException expected) {}
+        });
 
-        try {
+        expectScriptThrows(ClassCastException.class, () -> {
             exec("boolean x = false; Map z = x ? 4 : (byte)7; return z;");
-            fail("expected class cast exception");
-        } catch (ClassCastException expected) {}
+        });
 
-        try {
+        expectScriptThrows(ClassCastException.class, () -> {
             exec("boolean x = false; Map z = x ? new HashMap() : new ArrayList(); return z;");
-            fail("expected class cast exception");
-        } catch (ClassCastException expected) {}
+        });
 
-        try {
+        expectScriptThrows(ClassCastException.class, () -> {
             exec("boolean x = false; int y = 2; byte z = x ? y : 7; return z;");
-            fail("expected class cast exception");
-        } catch (ClassCastException expected) {}
+        });
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java
index 61046ab59f4..834f998723e 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java
@@ -124,11 +124,11 @@ public class DivisionTests extends ScriptTestCase {
     }
 
     public void testDivideByZeroConst() throws Exception {
-        expectThrows(ArithmeticException.class, () -> {
+        expectScriptThrows(ArithmeticException.class, () -> {
             exec("return 1/0;");
         });
 
-        expectThrows(ArithmeticException.class, () -> {
+        expectScriptThrows(ArithmeticException.class, () -> {
             exec("return 1L/0L;");
         });
     }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
index f612a6c84ed..20c48fe87d6 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
@@ -21,7 +21,7 @@ package org.elasticsearch.painless;
 
 public class FunctionRefTests extends ScriptTestCase {
     public void testUnsupported() {
-        expectThrows(UnsupportedOperationException.class, () -> {
+        expectScriptThrows(UnsupportedOperationException.class, () -> {
            exec("DoubleStream.Builder builder = DoubleStream.builder();" +
                "builder.add(2.0); builder.add(1.0); builder.add(3.0);" +
                "builder.build().reduce(Double::unsupported);");
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java
index 00c86836a67..ba095c05a9d 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java
@@ -25,7 +25,7 @@ public class OverloadTests extends ScriptTestCase {
     public void testMethod() {
         assertEquals(2, exec("return 'abc123abc'.indexOf('c');"));
         assertEquals(8, exec("return 'abc123abc'.indexOf('c', 3);"));
-        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("return 'abc123abc'.indexOf('c', 3, 'bogus');");
         });
         assertTrue(expected.getMessage().contains("[indexOf] with [3] arguments"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java
index b60de9233df..448d4262879 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java
@@ -124,18 +124,12 @@ public class RemainderTests extends ScriptTestCase {
     }
 
     public void testDivideByZeroConst() throws Exception {
-        try {
+        expectScriptThrows(ArithmeticException.class, () -> {
             exec("return 1%0;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {
-            // divide by zero
-        }
+        });
 
-        try {
+        expectScriptThrows(ArithmeticException.class, () -> {
             exec("return 1L%0L;");
-            fail("should have hit exception");
-        } catch (ArithmeticException expected) {
-            // divide by zero
-        }
+        });
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java
index e5284cd0eb9..9136261c164 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java
@@ -27,7 +27,7 @@ public class ReservedWordTests extends ScriptTestCase {
 
     /** check that we can't declare a variable of _score, its really reserved! */
     public void testScoreVar() {
-        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("int _score = 5; return _score;");
         });
         assertTrue(expected.getMessage().contains("Variable name [_score] is reserved"));
@@ -35,7 +35,7 @@ public class ReservedWordTests extends ScriptTestCase {
 
     /** check that we can't write to _score, its read-only! */
     public void testScoreStore() {
-        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("_score = 5; return _score;");
         });
         assertTrue(expected.getMessage().contains("Variable [_score] is read-only"));
@@ -43,7 +43,7 @@ public class ReservedWordTests extends ScriptTestCase {
 
     /** check that we can't declare a variable of doc, its really reserved! */
     public void testDocVar() {
-        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("int doc = 5; return doc;");
         });
         assertTrue(expected.getMessage().contains("Variable name [doc] is reserved"));
@@ -51,7 +51,7 @@ public class ReservedWordTests extends ScriptTestCase {
 
     /** check that we can't write to doc, its read-only! */
     public void testDocStore() {
-        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("doc = 5; return doc;");
         });
         assertTrue(expected.getMessage().contains("Variable [doc] is read-only"));
@@ -59,7 +59,7 @@ public class ReservedWordTests extends ScriptTestCase {
 
     /** check that we can't declare a variable of ctx, its really reserved! */
     public void testCtxVar() {
-        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("int ctx = 5; return ctx;");
         });
         assertTrue(expected.getMessage().contains("Variable name [ctx] is reserved"));
@@ -67,7 +67,7 @@ public class ReservedWordTests extends ScriptTestCase {
 
     /** check that we can't write to ctx, its read-only! */
     public void testCtxStore() {
-        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("ctx = 5; return ctx;");
         });
         assertTrue(expected.getMessage().contains("Variable [ctx] is read-only"));
@@ -80,7 +80,7 @@ public class ReservedWordTests extends ScriptTestCase {
 
     /** check that we can't declare a variable of _value, its really reserved! */
     public void testAggregationValueVar() {
-        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("int _value = 5; return _value;");
         });
         assertTrue(expected.getMessage().contains("Variable name [_value] is reserved"));
@@ -88,7 +88,7 @@ public class ReservedWordTests extends ScriptTestCase {
 
     /** check that we can't write to _value, its read-only! */
     public void testAggregationValueStore() {
-        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("_value = 5; return _value;");
         });
         assertTrue(expected.getMessage().contains("Variable [_value] is read-only"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
index 961c3561c48..ee88c0000bd 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
@@ -71,11 +71,11 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
     }
 
     public void testInvalidShift() {
-        expectThrows(ClassCastException.class, () -> {
+        expectScriptThrows(ClassCastException.class, () -> {
             exec("float x = 15F; x <<= 2; return x;");
         });
 
-        expectThrows(ClassCastException.class, () -> {
+        expectScriptThrows(ClassCastException.class, () -> {
             exec("double x = 15F; x <<= 2; return x;");
         });
     }
@@ -134,7 +134,7 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
         assertTrue(expected.getMessage().contains(
                    "The maximum number of statements that can be executed in a loop has been reached."));
 
-        RuntimeException parseException = expectThrows(RuntimeException.class, () -> {
+        RuntimeException parseException = expectScriptThrows(RuntimeException.class, () -> {
             exec("try { int x; } catch (PainlessError error) {}");
             fail("should have hit ParseException");
         });
@@ -156,7 +156,7 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
         final char[] tooManyChars = new char[Compiler.MAXIMUM_SOURCE_LENGTH + 1];
         Arrays.fill(tooManyChars, '0');
 
-        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec(new String(tooManyChars));
         });
         assertTrue(expected.getMessage().contains("Scripts may be no longer than"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/antlr/ParserTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/antlr/ParserTests.java
index aedecfc9c91..1a75c970fc4 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/antlr/ParserTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/antlr/ParserTests.java
@@ -40,9 +40,9 @@ public class ParserTests extends ScriptTestCase {
 
     private SourceContext buildAntlrTree(String source) {
         ANTLRInputStream stream = new ANTLRInputStream(source);
-        PainlessLexer lexer = new ErrorHandlingLexer(stream);
+        PainlessLexer lexer = new ErrorHandlingLexer(stream, "testing");
         PainlessParser parser = new PainlessParser(new CommonTokenStream(lexer));
-        ParserErrorStrategy strategy = new ParserErrorStrategy();
+        ParserErrorStrategy strategy = new ParserErrorStrategy("testing");
 
         lexer.removeErrorListeners();
         parser.removeErrorListeners();
diff --git a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/16_update2.yaml b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/16_update2.yaml
index 8a29e8e786b..524f203b2fc 100644
--- a/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/16_update2.yaml
+++ b/modules/lang-painless/src/test/resources/rest-api-spec/test/plan_a/16_update2.yaml
@@ -47,7 +47,7 @@
         body: { "script":  "_score * foo bar + doc['myParent.weight'].value" }
 
   - do:
-      catch: /Unable.to.parse.*/
+      catch: /compile error/
       put_script:
         id: "1"
         lang: "painless"

From 9a0d0d7cffa876639a425f785893fe83388836c1 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Thu, 2 Jun 2016 15:30:11 -0700
Subject: [PATCH 0573/1311] Fixed a grammar mistake in a comment.

---
 .../src/main/java/org/elasticsearch/painless/node/EChain.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index af111b99995..5448c53b35c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -301,7 +301,7 @@ public final class EChain extends AExpression {
 
                     if (!(expression instanceof EBinary) ||
                         ((EBinary)expression).operation != Operation.ADD || expression.actual.sort != Sort.STRING) {
-                        writer.writeAppendStrings(expression.actual); // append the expression's value unless its also a concatenation
+                        writer.writeAppendStrings(expression.actual); // append the expression's value unless it's also a concatenation
                     }
 
                     writer.writeToStrings(); // put the value of the StringBuilder on the stack

From e6aaaf11ed4a1b8f595f920de1840938eddb7a97 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Fri, 3 Jun 2016 09:50:51 +0200
Subject: [PATCH 0574/1311] Reworked docs for index-shrink API (#18705)

---
 docs/reference/indices/shrink-index.asciidoc | 163 +++++++++++++------
 1 file changed, 112 insertions(+), 51 deletions(-)

diff --git a/docs/reference/indices/shrink-index.asciidoc b/docs/reference/indices/shrink-index.asciidoc
index db6f717a82b..d531477e77c 100644
--- a/docs/reference/indices/shrink-index.asciidoc
+++ b/docs/reference/indices/shrink-index.asciidoc
@@ -1,64 +1,125 @@
 [[indices-shrink-index]]
 == Shrink Index
 
-The shrink index API allows to shrink an existing index into a new index with a single shard.
-In order to shrink an index, all its shards must be allocated on a single node in the cluster.
-This is required since the shrink command will copy all shards index files into the target index
-data folder when the primary of the target index is initially allocated.
+The shrink index API allows you to shrink an existing index into a new index
+with a single primary shard. Before shrinking, a (primary or replica) copy of
+every shard in the index must be present on the same node.
 
-When an index is shrunk no write operations should happen to the source index. Elasticsearch will
-enforce the `read-only` property when the shrink command is executed. All operations necessary to shrink the
-source index are executed during initial primary recovery. Once the target index primary shard is started the
-shrink operation has successfully finished. To monitor status and progress use <<cat-recovery>>
+Shrinking works as follows:
 
+* First, it creates a new target index with the same definition as the source
+  index, but with a single primary shard.
 
-To shrink and index all shards of that index must be allocated on a single node.
+* Then it hard-links segments from the source index into the target index. (If
+  the file system doesn't support hard-linking, then all segments are copied
+  into the new index, which is a much more time consuming process.)
 
-[source,js]
---------------------------------------------------
-$ curl -XPUT 'http://localhost:9200/logs/_settings' -d '{
-    "settings" : {
-        "index.routing.allocation.require._name" : "shrink_node_name", <1>
-        "index.blocks.write" : true <2>
-    }
-}'
---------------------------------------------------
-<1> Forces the relocation of all of the indices shards to the node `shrink_node_name`
-<2> Prevents write operations to this index while still allowing metadata changes like deleting the index.
-
-The above second curl example shows how an index called `logs` can be
-forced to allocate at least one copy of each shard on a specific node in the cluster.
-
-The `_shrink` API is similar to <<indices-create-index>> and accepts `settings` and `aliases` for the target index.
-
-[source,js]
---------------------------------------------------
-$ curl -XPUT 'http://localhost:9200/logs/_shrink/logs_single_shard' -d '{
-    "settings" : {
-        "index.codec" : "best_compression", <1>
-    }
-}'
---------------------------------------------------
-<1> Enables `best_compression` codec on the target index
-
-The API call above returns immediately once the target index is created but doesn't wait
-for the shrink operation to start. Once the target indices primary shard moves to state `initializing`
-the shrink operation has started.
+* Finally, it recovers the target index as though it were a closed index which
+  had just been re-opened.
 
 [float]
-[[shrink-index-limitations]]
-=== Limitations
+=== Preparing an index for shrinking
 
-Indices can only be shrunk into a single shard if they fully the following requirements:
+In order to shrink an index, the index must be marked as read-only, and a
+(primary or replica) copy of every shard in the index must be relocated to the
+same node and have <<cluster-health,health>> `green`.
+
+These two conditions can be achieved with the following request:
+
+[source,js]
+--------------------------------------------------
+PUT /my_source_index/_settings
+{
+  "settings": {
+    "index.routing.allocation.require._name": "shrink_node_name", <1>
+    "index.blocks.write": true <2>
+  }
+}
+--------------------------------------------------
+<1> Forces the relocation of a copy of each shard to the node with name
+    `shrink_node_name`.  See <<shard-allocation-filtering>> for more options.
+
+<2> Prevents write operations to this index while still allowing metadata
+    changes like deleting the index.
+
+It can take a while to relocate the source index.  Progress can be tracked
+with the <<cat-recovery,`_cat recovery` API>>, or the <<cluster-health,
+`cluster health` API>> can be used to wait until all shards have relocated
+with the `wait_for_relocating_shards` parameter.
+
+[float]
+=== Shrinking an index
+
+To shrink `my_source_index` into a new index called `my_target_index`, issue
+the following request:
+
+[source,js]
+--------------------------------------------------
+POST my_source_index/_shrink/my_target_index
+--------------------------------------------------
+
+The above request returns immediately once the target index has been added to
+the cluster state -- it doesn't wait for the shrink operation to start.
+
+[IMPORTANT]
+=====================================
+
+Indices can only be shrunk into a single shard if they satisfy the following requirements:
 
- * an instance of all of the indices shards must be allocated on a single node
- * the index must not contain more than `2.14 billion` documents (`2147483519`) in total (sum of all shards)
-  This is the maximum shard size elasticsearch can support.
- * the index must have more than one shard
- * the index must be `read-only`, ie. have a cluster block set `index.blocks.write=true`
  * the target index must not exist
- * all `index.analysis.*` and `index.similarity.*` settings passed to the `_shrink` call will be overwritten with the
-  source indices settings.
- * if the target index can't be allocated on the shrink node, due to throttling or other allocation deciders,
-  its primary shard will stay `unassigned` until it can be allocated on that node
+
+* The index must have more than one primary shard.
+
+* The index must not contain more than `2,147,483,519` documents in total
+  across all shards as this is the maximum number of docs that can fit into a
+  single shard.
+
+* The node handling the shrink process must have sufficient free disk space to
+  accommodate a second copy of the existing index.
+
+=====================================
+
+The `_shrink` API is similar to the <<indices-create-index, `create index` API>>
+and accepts `settings` and `aliases` parameters for the target index:
+
+[source,js]
+--------------------------------------------------
+POST my_source_index/_shrink/my_target_index
+{
+  "settings": {
+    "index.number_of_replicas": 1,
+    "index.codec": "best_compression" <1>
+  },
+  "aliases": {
+    "my_search_indices": {}
+  }
+}
+--------------------------------------------------
+
+<1> Best compression will only take affect when new writes are made to the
+    index, such as when <<indices-forcemerge,force-merging>> the shard to a single
+    segment.
+
+NOTE: Mappings may not be specified in the `_shrink` request, and all
+`index.analysis.*` and `index.similarity.*` settings will be overwritten with
+the settings from the source index.
+
+[float]
+=== Monitoring the shrink process
+
+The shrink process can be monitored with the <<cat-recovery,`_cat recovery`
+API>>, or the <<cluster-health, `cluster health` API>> can be used to wait
+until all primary shards have been allocated by setting the  `wait_for_status`
+parameter to `yellow`.
+
+The `_shrink` API returns as soon as the target index has been added to the
+cluster state, before any shards have been allocated. At this point, all
+shards are in the state `unassigned`. If, for any reason, the target index
+can't be allocated on the shrink node, its primary shard will remain
+`unassigned` until it can be allocated on that node.
+
+Once the primary shard is allocated, it moves to state `initializing`, and the
+shrink process begins. When the shrink operation completes, the shard will
+become `active`. At that  point, Elasticsearch will try to allocate any
+replicas and may decide to relocate the primary shard to another node.
 

From 78574d248cf10dad1c5a566fda79e1e0c94afe43 Mon Sep 17 00:00:00 2001
From: Britta Weber <britta.weber@elasticsearch.com>
Date: Fri, 3 Jun 2016 11:37:22 +0200
Subject: [PATCH 0575/1311] [TEST] mute
 RandomAllocationDeciderTests.testRandomDecisions

we have a pr already: https://github.com/elastic/elasticsearch/pull/18701
---
 .../cluster/routing/allocation/RandomAllocationDeciderTests.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
index 307df91c302..1ec1ed8c37d 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
@@ -54,6 +54,7 @@ public class RandomAllocationDeciderTests extends ESAllocationTestCase {
      * amount of iterations the test allows allocation unless the same shard is
      * already allocated on a node and balances the cluster to gain optimal
      * balance.*/
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/18701")
     public void testRandomDecisions() {
         RandomAllocationDecider randomAllocationDecider = new RandomAllocationDecider(random());
         AllocationService strategy = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,

From 6c28235b03fdad898986f613e198c24ca3067e4d Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Fri, 3 Jun 2016 13:42:09 +0200
Subject: [PATCH 0576/1311] Fix merge stats rendering in RestIndicesAction
 (#18720)

give the table description:
```
        table.addCell("merges.total", "sibling:pri;alias:mt,mergesTotal;default:false;text-align:right;desc:number of completed merge ops");
        table.addCell("pri.merges.total", "default:false;text-align:right;desc:number of completed merge ops");

        table.addCell("merges.total_docs", "sibling:pri;alias:mtd,mergesTotalDocs;default:false;text-align:right;desc:docs merged");
        table.addCell("pri.merges.total_docs", "default:false;text-align:right;desc:docs merged");

        table.addCell("merges.total_size", "sibling:pri;alias:mts,mergesTotalSize;default:false;text-align:right;desc:size merged");
        table.addCell("pri.merges.total_size", "default:false;text-align:right;desc:size merged");
```
this is how it should be.
---
 .../org/elasticsearch/rest/action/cat/RestIndicesAction.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
index eaf9ad4a081..38cbba6fd5a 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
@@ -412,13 +412,13 @@ public class RestIndicesAction extends AbstractCatAction {
             table.addCell(indexStats == null ? null : indexStats.getPrimaries().getIndexing().getTotal().getIndexFailedCount());
 
             table.addCell(indexStats == null ? null : indexStats.getTotal().getMerge().getCurrent());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getCurrentSize());
+            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getCurrent());
 
             table.addCell(indexStats == null ? null : indexStats.getTotal().getMerge().getCurrentNumDocs());
             table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getCurrentNumDocs());
 
             table.addCell(indexStats == null ? null : indexStats.getTotal().getMerge().getCurrentSize());
-            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getCurrent());
+            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getCurrentSize());
 
             table.addCell(indexStats == null ? null : indexStats.getTotal().getMerge().getTotal());
             table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getTotal());

From 24a7b7224bd834eb57d1026ceb0c455b8c48a206 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Fri, 3 Jun 2016 14:11:34 +0200
Subject: [PATCH 0577/1311] Fix recovery throttling to properly handle
 relocating non-primary shards (#18701)

Relocation of non-primary shards is realized by recovering from the primary shard. Recovery throttling wrongly equates non-primary relocation as recovering a shard from the non-primary relocation source, however.

Closes #18640
---
 .../cluster/routing/RoutingNodes.java         |  89 +++++------
 .../decider/ThrottlingAllocationDecider.java  | 110 ++++++++-----
 .../routing/RandomShardRoutingMutator.java    |   2 +-
 .../RandomAllocationDeciderTests.java         |   1 -
 .../allocation/ThrottlingAllocationTests.java | 150 ++++++++++--------
 5 files changed, 199 insertions(+), 153 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
index fbbedcdefde..a4e61eac739 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java
@@ -108,18 +108,18 @@ public class RoutingNodes implements Iterable<RoutingNode> {
                             // add the counterpart shard with relocatingNodeId reflecting the source from which
                             // it's relocating from.
                             ShardRouting targetShardRouting = shard.buildTargetRelocatingShard();
-                            addInitialRecovery(targetShardRouting, routingTable);
+                            addInitialRecovery(targetShardRouting, indexShard.primary);
                             previousValue = entries.put(targetShardRouting.shardId(), targetShardRouting);
                             if (previousValue != null) {
                                 throw new IllegalArgumentException("Cannot have two different shards with same shard id on same node");
                             }
                             assignedShardsAdd(targetShardRouting);
-                        } else if (shard.active() == false) { // shards that are initializing without being relocated
+                        } else if (shard.initializing()) {
                             if (shard.primary()) {
                                 inactivePrimaryCount++;
                             }
                             inactiveShardCount++;
-                            addInitialRecovery(shard, routingTable);
+                            addInitialRecovery(shard, indexShard.primary);
                         }
                     } else {
                         unassignedShards.add(shard);
@@ -134,48 +134,44 @@ public class RoutingNodes implements Iterable<RoutingNode> {
     }
 
     private void addRecovery(ShardRouting routing) {
-        addRecovery(routing, true, null);
+        updateRecoveryCounts(routing, true, findAssignedPrimaryIfPeerRecovery(routing));
     }
 
     private void removeRecovery(ShardRouting routing) {
-        addRecovery(routing, false, null);
+        updateRecoveryCounts(routing, false, findAssignedPrimaryIfPeerRecovery(routing));
     }
 
-    private void addInitialRecovery(ShardRouting routing, RoutingTable routingTable) {
-        addRecovery(routing, true, routingTable);
+    private void addInitialRecovery(ShardRouting routing, ShardRouting initialPrimaryShard) {
+        updateRecoveryCounts(routing, true, initialPrimaryShard);
     }
 
-    private void addRecovery(final ShardRouting routing, final boolean increment, final RoutingTable routingTable) {
+    private void updateRecoveryCounts(final ShardRouting routing, final boolean increment, @Nullable final ShardRouting primary) {
         final int howMany = increment ? 1 : -1;
         assert routing.initializing() : "routing must be initializing: " + routing;
+        // TODO: check primary == null || primary.active() after all tests properly add ReplicaAfterPrimaryActiveAllocationDecider
+        assert primary == null || primary.assignedToNode() :
+            "shard is initializing but its primary is not assigned to a node";
+
         Recoveries.getOrAdd(recoveriesPerNode, routing.currentNodeId()).addIncoming(howMany);
-        final String sourceNodeId;
-        if (routing.relocatingNodeId() != null) { // this is a relocation-target
-            sourceNodeId = routing.relocatingNodeId();
-            if (routing.primary() && increment == false) { // primary is done relocating
+
+        if (routing.isPeerRecovery()) {
+            // add/remove corresponding outgoing recovery on node with primary shard
+            if (primary == null) {
+                throw new IllegalStateException("shard is peer recovering but primary is unassigned");
+            }
+            Recoveries.getOrAdd(recoveriesPerNode, primary.currentNodeId()).addOutgoing(howMany);
+
+            if (increment == false && routing.primary() && routing.relocatingNodeId() != null) {
+                // primary is done relocating, move non-primary recoveries from old primary to new primary
                 int numRecoveringReplicas = 0;
                 for (ShardRouting assigned : assignedShards(routing.shardId())) {
-                    if (assigned.primary() == false && assigned.initializing() && assigned.relocatingNodeId() == null) {
+                    if (assigned.primary() == false && assigned.isPeerRecovery()) {
                         numRecoveringReplicas++;
                     }
                 }
-                // we transfer the recoveries to the relocated primary
-                recoveriesPerNode.get(sourceNodeId).addOutgoing(-numRecoveringReplicas);
+                recoveriesPerNode.get(routing.relocatingNodeId()).addOutgoing(-numRecoveringReplicas);
                 recoveriesPerNode.get(routing.currentNodeId()).addOutgoing(numRecoveringReplicas);
             }
-        } else if (routing.primary() == false) { // primary without relocationID is initial recovery
-            ShardRouting primary = findPrimary(routing);
-            if (primary == null && routingTable != null) {
-                primary = routingTable.index(routing.index().getName()).shard(routing.shardId().id()).primary;
-            } else if (primary == null) {
-                throw new IllegalStateException("replica is initializing but primary is unassigned");
-            }
-            sourceNodeId = primary.currentNodeId();
-        } else {
-            sourceNodeId = null;
-        }
-        if (sourceNodeId != null) {
-            Recoveries.getOrAdd(recoveriesPerNode, sourceNodeId).addOutgoing(howMany);
         }
     }
 
@@ -187,18 +183,21 @@ public class RoutingNodes implements Iterable<RoutingNode> {
         return recoveriesPerNode.getOrDefault(nodeId, Recoveries.EMPTY).getOutgoing();
     }
 
-    private ShardRouting findPrimary(ShardRouting routing) {
-        List<ShardRouting> shardRoutings = assignedShards.get(routing.shardId());
+    @Nullable
+    private ShardRouting findAssignedPrimaryIfPeerRecovery(ShardRouting routing) {
         ShardRouting primary = null;
-        if (shardRoutings != null) {
-            for (ShardRouting shardRouting : shardRoutings) {
-                if (shardRouting.primary()) {
-                    if (shardRouting.active()) {
-                        return shardRouting;
-                    } else if (primary == null) {
-                        primary = shardRouting;
-                    } else if (primary.relocatingNodeId() != null) {
-                        primary = shardRouting;
+        if (routing.isPeerRecovery()) {
+            List<ShardRouting> shardRoutings = assignedShards.get(routing.shardId());
+            if (shardRoutings != null) {
+                for (ShardRouting shardRouting : shardRoutings) {
+                    if (shardRouting.primary()) {
+                        if (shardRouting.active()) {
+                            return shardRouting;
+                        } else if (primary == null) {
+                            primary = shardRouting;
+                        } else if (primary.relocatingNodeId() != null) {
+                            primary = shardRouting;
+                        }
                     }
                 }
             }
@@ -500,7 +499,6 @@ public class RoutingNodes implements Iterable<RoutingNode> {
         ShardRouting relocationMarkerRemoved = shard.removeRelocationSource();
         updateAssigned(shard, relocationMarkerRemoved);
         inactiveShardCount++; // relocation targets are not counted as inactive shards whereas initializing shards are
-        Recoveries.getOrAdd(recoveriesPerNode, shard.relocatingNodeId()).addOutgoing(-1);
         return relocationMarkerRemoved;
     }
 
@@ -856,20 +854,17 @@ public class RoutingNodes implements Iterable<RoutingNode> {
                 for (ShardRouting routing : routingNode) {
                     if (routing.initializing()) {
                         incoming++;
-                    } else if (routing.relocating()) {
-                        outgoing++;
                     }
-                    if (routing.primary() && (routing.initializing() && routing.relocatingNodeId() != null) == false) { // we don't count the initialization end of the primary relocation
-                        List<ShardRouting> shardRoutings = routingNodes.assignedShards.get(routing.shardId());
-                        for (ShardRouting assigned : shardRoutings) {
-                            if (assigned.primary() == false && assigned.initializing() && assigned.relocatingNodeId() == null) {
+                    if (routing.primary() && routing.isPeerRecovery() == false) {
+                        for (ShardRouting assigned : routingNodes.assignedShards.get(routing.shardId())) {
+                            if (assigned.isPeerRecovery()) {
                                 outgoing++;
                             }
                         }
                     }
                 }
             }
-            assert incoming == value.incoming : incoming + " != " + value.incoming;
+            assert incoming == value.incoming : incoming + " != " + value.incoming + " node: " + routingNode;
             assert outgoing == value.outgoing : outgoing + " != " + value.outgoing + " node: " + routingNode;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
index 45afd07e297..286b378debc 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java
@@ -28,6 +28,9 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 
+import static org.elasticsearch.cluster.routing.allocation.decider.Decision.THROTTLE;
+import static org.elasticsearch.cluster.routing.allocation.decider.Decision.YES;
+
 /**
  * {@link ThrottlingAllocationDecider} controls the recovery process per node in
  * the cluster. It exposes two settings via the cluster update API that allow
@@ -109,50 +112,83 @@ public class ThrottlingAllocationDecider extends AllocationDecider {
 
     @Override
     public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
-        if (shardRouting.primary()) {
-            assert shardRouting.unassigned() || shardRouting.active();
-            if (shardRouting.unassigned()) {
-                // primary is unassigned, means we are going to do recovery from gateway
-                // count *just the primary* currently doing recovery on the node and check against concurrent_recoveries
-                int primariesInRecovery = 0;
-                for (ShardRouting shard : node) {
-                    // when a primary shard is INITIALIZING, it can be because of *initial recovery* or *relocation from another node*
-                    // we only count initial recoveries here, so we need to make sure that relocating node is null
-                    if (shard.initializing() && shard.primary() && shard.relocatingNodeId() == null) {
-                        primariesInRecovery++;
-                    }
+        if (shardRouting.primary() && shardRouting.unassigned()) {
+            assert initializingShard(shardRouting, node.nodeId()).isPeerRecovery() == false;
+            // primary is unassigned, means we are going to do recovery from store, snapshot or local shards
+            // count *just the primaries* currently doing recovery on the node and check against primariesInitialRecoveries
+
+            int primariesInRecovery = 0;
+            for (ShardRouting shard : node) {
+                // when a primary shard is INITIALIZING, it can be because of *initial recovery* or *relocation from another node*
+                // we only count initial recoveries here, so we need to make sure that relocating node is null
+                if (shard.initializing() && shard.primary() && shard.relocatingNodeId() == null) {
+                    primariesInRecovery++;
                 }
-                if (primariesInRecovery >= primariesInitialRecoveries) {
-                    return allocation.decision(Decision.THROTTLE, NAME, "too many primaries are currently recovering [%d], limit: [%d]",
-                            primariesInRecovery, primariesInitialRecoveries);
+            }
+            if (primariesInRecovery >= primariesInitialRecoveries) {
+                // TODO: Should index creation not be throttled for primary shards?
+                return allocation.decision(THROTTLE, NAME, "too many primaries are currently recovering [%d], limit: [%d]",
+                    primariesInRecovery, primariesInitialRecoveries);
+            } else {
+                return allocation.decision(YES, NAME, "below primary recovery limit of [%d]", primariesInitialRecoveries);
+            }
+        } else {
+            // Peer recovery
+            assert initializingShard(shardRouting, node.nodeId()).isPeerRecovery();
+
+            // Allocating a shard to this node will increase the incoming recoveries
+            int currentInRecoveries = allocation.routingNodes().getIncomingRecoveries(node.nodeId());
+            if (currentInRecoveries >= concurrentIncomingRecoveries) {
+                return allocation.decision(THROTTLE, NAME, "too many incoming shards are currently recovering [%d], limit: [%d]",
+                    currentInRecoveries, concurrentIncomingRecoveries);
+            } else {
+                // search for corresponding recovery source (= primary shard) and check number of outgoing recoveries on that node
+                ShardRouting primaryShard = allocation.routingNodes().activePrimary(shardRouting.shardId());
+                if (primaryShard == null) {
+                    return allocation.decision(Decision.NO, NAME, "primary shard for this replica is not yet active");
+                }
+                int primaryNodeOutRecoveries = allocation.routingNodes().getOutgoingRecoveries(primaryShard.currentNodeId());
+                if (primaryNodeOutRecoveries >= concurrentOutgoingRecoveries) {
+                    return allocation.decision(THROTTLE, NAME, "too many outgoing shards are currently recovering [%d], limit: [%d]",
+                        primaryNodeOutRecoveries, concurrentOutgoingRecoveries);
                 } else {
-                    return allocation.decision(Decision.YES, NAME, "below primary recovery limit of [%d]", primariesInitialRecoveries);
+                    return allocation.decision(YES, NAME, "below shard recovery limit of outgoing: [%d < %d] incoming: [%d < %d]",
+                        primaryNodeOutRecoveries,
+                        concurrentOutgoingRecoveries,
+                        currentInRecoveries,
+                        concurrentIncomingRecoveries);
                 }
             }
         }
-        // TODO should we allow shards not allocated post API to always allocate?
-        // either primary or replica doing recovery (from peer shard)
-
-        // count the number of recoveries on the node, its for both target (INITIALIZING) and source (RELOCATING)
-        return canAllocate(node, allocation);
     }
 
-    @Override
-    public Decision canAllocate(RoutingNode node, RoutingAllocation allocation) {
-        int currentOutRecoveries = allocation.routingNodes().getOutgoingRecoveries(node.nodeId());
-        int currentInRecoveries = allocation.routingNodes().getIncomingRecoveries(node.nodeId());
-        if (currentOutRecoveries >= concurrentOutgoingRecoveries) {
-            return allocation.decision(Decision.THROTTLE, NAME, "too many outgoing shards are currently recovering [%d], limit: [%d]",
-                currentOutRecoveries, concurrentOutgoingRecoveries);
-        } else if (currentInRecoveries >= concurrentIncomingRecoveries) {
-            return allocation.decision(Decision.THROTTLE, NAME, "too many incoming shards are currently recovering [%d], limit: [%d]",
-                currentInRecoveries, concurrentIncomingRecoveries);
-        }  else {
-            return allocation.decision(Decision.YES, NAME, "below shard recovery limit of outgoing: [%d < %d] incoming: [%d < %d]",
-                    currentOutRecoveries,
-                    concurrentOutgoingRecoveries,
-                    currentInRecoveries,
-                    concurrentIncomingRecoveries);
+    /**
+     * The shard routing passed to {@link #canAllocate(ShardRouting, RoutingNode, RoutingAllocation)} is not the initializing shard to this
+     * node but:
+     * - the unassigned shard routing in case if we want to assign an unassigned shard to this node.
+     * - the initializing shard routing if we want to assign the initializing shard to this node instead
+     * - the started shard routing in case if we want to check if we can relocate to this node.
+     * - the relocating shard routing if we want to relocate to this node now instead.
+     *
+     * This method returns the corresponding initializing shard that would be allocated to this node.
+     */
+    private ShardRouting initializingShard(ShardRouting shardRouting, String currentNodeId) {
+        final ShardRouting initializingShard;
+        if (shardRouting.unassigned()) {
+            initializingShard = shardRouting.initialize(currentNodeId, null, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
+        } else if (shardRouting.initializing()) {
+            initializingShard = shardRouting.moveToUnassigned(shardRouting.unassignedInfo())
+                .initialize(currentNodeId, null, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
+        } else if (shardRouting.relocating()) {
+            initializingShard = shardRouting.cancelRelocation()
+                .relocate(currentNodeId, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE)
+                .buildTargetRelocatingShard();
+        } else {
+            assert shardRouting.started();
+            initializingShard = shardRouting.relocate(currentNodeId, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE)
+                .buildTargetRelocatingShard();
         }
+        assert initializingShard.initializing();
+        return initializingShard;
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RandomShardRoutingMutator.java b/core/src/test/java/org/elasticsearch/cluster/routing/RandomShardRoutingMutator.java
index 5d24b3d1e17..a470ca9f9ae 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/RandomShardRoutingMutator.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/RandomShardRoutingMutator.java
@@ -34,7 +34,7 @@ public final class RandomShardRoutingMutator {
     public static ShardRouting randomChange(ShardRouting shardRouting, String[] nodes) {
         switch (randomInt(2)) {
             case 0:
-                if (shardRouting.unassigned() == false) {
+                if (shardRouting.unassigned() == false && shardRouting.primary() == false) {
                     shardRouting = shardRouting.moveToUnassigned(new UnassignedInfo(randomReason(), randomAsciiOfLength(10)));
                 } else if (shardRouting.unassignedInfo() != null) {
                     shardRouting = shardRouting.updateUnassignedInfo(new UnassignedInfo(randomReason(), randomAsciiOfLength(10)));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
index 1ec1ed8c37d..307df91c302 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
@@ -54,7 +54,6 @@ public class RandomAllocationDeciderTests extends ESAllocationTestCase {
      * amount of iterations the test allows allocation unless the same shard is
      * already allocated on a node and balances the cluster to gain optimal
      * balance.*/
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/18701")
     public void testRandomDecisions() {
         RandomAllocationDecider randomAllocationDecider = new RandomAllocationDecider(random());
         AllocationService strategy = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java
index 61a72bc352a..ada5f4c19f8 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java
@@ -19,11 +19,13 @@
 
 package org.elasticsearch.cluster.routing.allocation;
 
+import com.carrotsearch.hppc.IntHashSet;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RestoreSource;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
@@ -31,6 +33,8 @@ import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.snapshots.Snapshot;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.test.ESAllocationTestCase;
 
 import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING;
@@ -57,9 +61,7 @@ public class ThrottlingAllocationTests extends ESAllocationTestCase {
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(10).numberOfReplicas(1))
                 .build();
 
-        RoutingTable routingTable = RoutingTable.builder()
-                .addAsNew(metaData.index("test"))
-                .build();
+        RoutingTable routingTable = createRecoveryRoutingTable(metaData.index("test"));
 
         ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
 
@@ -118,9 +120,7 @@ public class ThrottlingAllocationTests extends ESAllocationTestCase {
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(5).numberOfReplicas(1))
                 .build();
 
-        RoutingTable routingTable = RoutingTable.builder()
-                .addAsNew(metaData.index("test"))
-                .build();
+        RoutingTable routingTable = createRecoveryRoutingTable(metaData.index("test"));
 
         ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
 
@@ -188,9 +188,7 @@ public class ThrottlingAllocationTests extends ESAllocationTestCase {
             .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(9).numberOfReplicas(0))
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder()
-            .addAsNew(metaData.index("test"))
-            .build();
+        RoutingTable routingTable = createRecoveryRoutingTable(metaData.index("test"));
 
         ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
 
@@ -242,89 +240,107 @@ public class ThrottlingAllocationTests extends ESAllocationTestCase {
         assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node1"), 1);
     }
 
-    public void testOutgoingThrottlesAllocaiton() {
-        Settings settings = Settings.builder()
-            .put("cluster.routing.allocation.node_concurrent_recoveries", 1)
-            .put("cluster.routing.allocation.node_initial_primaries_recoveries", 1)
-            .put("cluster.routing.allocation.cluster_concurrent_rebalance", 1)
-            .build();
-        AllocationService strategy = createAllocationService(settings);
+    public void testOutgoingThrottlesAllocation() {
+        AllocationService strategy = createAllocationService(Settings.builder()
+            .put("cluster.routing.allocation.node_concurrent_outgoing_recoveries", 1)
+            .build());
+
+        logger.info("Building initial routing table");
 
         MetaData metaData = MetaData.builder()
-            .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(3).numberOfReplicas(0))
+            .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(2))
             .build();
 
-        RoutingTable routingTable = RoutingTable.builder()
-            .addAsNew(metaData.index("test"))
-            .build();
+        RoutingTable routingTable = createRecoveryRoutingTable(metaData.index("test"));
 
         ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
 
-        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")).put(newNode("node3"))).build();
+        logger.info("start one node, do reroute, only 1 should initialize");
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build();
         routingTable = strategy.reroute(clusterState, "reroute").routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
-        assertThat(routingTable.shardsWithState(STARTED).size(), equalTo(0));
-        assertThat(routingTable.shardsWithState(INITIALIZING).size(), equalTo(3));
-        assertThat(routingTable.shardsWithState(UNASSIGNED).size(), equalTo(0));
-        assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node1"), 1);
-        assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node2"), 1);
-        assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node3"), 1);
-        assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node1"), 0);
-        assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node2"), 0);
-        assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node3"), 0);
 
+        assertThat(routingTable.shardsWithState(STARTED).size(), equalTo(0));
+        assertThat(routingTable.shardsWithState(INITIALIZING).size(), equalTo(1));
+        assertThat(routingTable.shardsWithState(UNASSIGNED).size(), equalTo(2));
+
+        logger.info("start initializing");
         routingTable = strategy.applyStartedShards(clusterState, routingTable.shardsWithState(INITIALIZING)).routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
 
-        assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node1"), 0);
-        assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node2"), 0);
-        assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node3"), 0);
-        assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node1"), 0);
-        assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node2"), 0);
-        assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node3"), 0);
+        assertThat(routingTable.shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(routingTable.shardsWithState(INITIALIZING).size(), equalTo(0));
+        assertThat(routingTable.shardsWithState(UNASSIGNED).size(), equalTo(2));
 
-        RoutingAllocation.Result reroute = strategy.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", clusterState.getRoutingNodes().node("node1").iterator().next().shardId().id(), "node1", "node2")), false, false);
-        assertEquals(reroute.explanations().explanations().size(), 1);
-        assertEquals(reroute.explanations().explanations().get(0).decisions().type(), Decision.Type.YES);
-        routingTable = reroute.routingTable();
+        logger.info("start one more node, first non-primary should start being allocated");
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node2"))).build();
+        routingTable = strategy.reroute(clusterState, "reroute").routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
-        assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node1"), 0);
-        assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node2"), 1);
-        assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node3"), 0);
-        assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node1"), 1);
-        assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node2"), 0);
-        assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node3"), 0);
 
-        // outgoing throttles
-        reroute = strategy.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", clusterState.getRoutingNodes().node("node3").iterator().next().shardId().id(), "node3", "node1")), true, false);
-        assertEquals(reroute.explanations().explanations().size(), 1);
-        assertEquals(reroute.explanations().explanations().get(0).decisions().type(), Decision.Type.THROTTLE);
-        assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node1"), 0);
-        assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node2"), 1);
-        assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node3"), 0);
+        assertThat(routingTable.shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(routingTable.shardsWithState(INITIALIZING).size(), equalTo(1));
+        assertThat(routingTable.shardsWithState(UNASSIGNED).size(), equalTo(1));
         assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node1"), 1);
-        assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node2"), 0);
-        assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node3"), 0);
+
+        logger.info("start initializing non-primary");
+        routingTable = strategy.applyStartedShards(clusterState, routingTable.shardsWithState(INITIALIZING)).routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        assertThat(routingTable.shardsWithState(STARTED).size(), equalTo(2));
+        assertThat(routingTable.shardsWithState(INITIALIZING).size(), equalTo(0));
+        assertThat(routingTable.shardsWithState(UNASSIGNED).size(), equalTo(1));
+        assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node1"), 0);
+
+        logger.info("start one more node, initializing second non-primary");
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node3"))).build();
+        routingTable = strategy.reroute(clusterState, "reroute").routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+
         assertThat(routingTable.shardsWithState(STARTED).size(), equalTo(2));
         assertThat(routingTable.shardsWithState(INITIALIZING).size(), equalTo(1));
-        assertThat(routingTable.shardsWithState(RELOCATING).size(), equalTo(1));
         assertThat(routingTable.shardsWithState(UNASSIGNED).size(), equalTo(0));
+        assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node1"), 1);
 
-        // incoming throttles
-        reroute = strategy.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", clusterState.getRoutingNodes().node("node3").iterator().next().shardId().id(), "node3", "node2")), true, false);
+        logger.info("start one more node");
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).put(newNode("node4"))).build();
+        routingTable = strategy.reroute(clusterState, "reroute").routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+
+        assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node1"), 1);
+
+        logger.info("move started non-primary to new node");
+        RoutingAllocation.Result reroute = strategy.reroute(clusterState, new AllocationCommands(
+            new MoveAllocationCommand("test", 0, "node2", "node4")), true, false);
         assertEquals(reroute.explanations().explanations().size(), 1);
         assertEquals(reroute.explanations().explanations().get(0).decisions().type(), Decision.Type.THROTTLE);
+        // even though it is throttled, move command still forces allocation
 
-        assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node1"), 0);
-        assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node2"), 1);
-        assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node3"), 0);
-        assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node1"), 1);
-        assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node2"), 0);
-        assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node3"), 0);
-        assertThat(routingTable.shardsWithState(STARTED).size(), equalTo(2));
-        assertThat(routingTable.shardsWithState(INITIALIZING).size(), equalTo(1));
+        clusterState = ClusterState.builder(clusterState).routingResult(reroute).build();
+        routingTable = clusterState.routingTable();
+        assertThat(routingTable.shardsWithState(STARTED).size(), equalTo(1));
         assertThat(routingTable.shardsWithState(RELOCATING).size(), equalTo(1));
+        assertThat(routingTable.shardsWithState(INITIALIZING).size(), equalTo(2));
         assertThat(routingTable.shardsWithState(UNASSIGNED).size(), equalTo(0));
-
+        assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node1"), 2);
+        assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node2"), 0);
     }
+
+    private RoutingTable createRecoveryRoutingTable(IndexMetaData indexMetaData) {
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        switch (randomInt(5)) {
+            case 0: routingTableBuilder.addAsRecovery(indexMetaData); break;
+            case 1: routingTableBuilder.addAsFromCloseToOpen(indexMetaData); break;
+            case 2: routingTableBuilder.addAsFromDangling(indexMetaData); break;
+            case 3: routingTableBuilder.addAsNewRestore(indexMetaData,
+                new RestoreSource(new Snapshot("repo", new SnapshotId("snap", "randomId")), Version.CURRENT,
+                indexMetaData.getIndex().getName()), new IntHashSet()); break;
+            case 4: routingTableBuilder.addAsRestore(indexMetaData,
+                new RestoreSource(new Snapshot("repo", new SnapshotId("snap", "randomId")), Version.CURRENT,
+                indexMetaData.getIndex().getName())); break;
+            case 5: routingTableBuilder.addAsNew(indexMetaData); break;
+            default: throw new IndexOutOfBoundsException();
+        }
+
+        return routingTableBuilder.build();
+    }
+
 }

From 3efbe95ca4f53cc9fb018d5b65a1b69c77d09616 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Mon, 22 Feb 2016 11:37:57 -0800
Subject: [PATCH 0578/1311] RestClient prototype

---
 client/build.gradle                           |  82 +++++++
 .../org/elasticsearch/client/IndexClient.java |  65 ++++++
 .../org/elasticsearch/client/RestClient.java  | 217 ++++++++++++++++++
 .../elasticsearch/client/RestClientTests.java | 152 ++++++++++++
 settings.gradle                               |   1 +
 5 files changed, 517 insertions(+)
 create mode 100644 client/build.gradle
 create mode 100644 client/src/main/java/org/elasticsearch/client/IndexClient.java
 create mode 100644 client/src/main/java/org/elasticsearch/client/RestClient.java
 create mode 100644 client/src/test/java/org/elasticsearch/client/RestClientTests.java

diff --git a/client/build.gradle b/client/build.gradle
new file mode 100644
index 00000000000..1ae32bf6e98
--- /dev/null
+++ b/client/build.gradle
@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.elasticsearch.gradle.precommit.PrecommitTasks;
+
+group = 'org.elasticsearch.client'
+apply plugin: 'elasticsearch.build'
+
+dependencies {
+  // we use the lucene test-framework here but we are not pulling in ES core or the test framework
+  testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
+  testCompile "junit:junit:${versions.junit}"
+  testCompile 'org.hamcrest:hamcrest-all:1.3'
+  testCompile "org.apache.lucene:lucene-test-framework:${versions.lucene}"
+  testCompile "org.apache.lucene:lucene-core:${versions.lucene}"
+  testCompile "org.apache.lucene:lucene-codecs:${versions.lucene}"
+
+
+  // TODO once we got rid of the client in the test framework we should use a version variable here
+  // we use httpclient here since the JDK support has several issue
+  // - httpclient supports basic and digest auth and other schemes
+  // - URLConnection has issues with SSL and not all system patches are available
+  // - URLConnection can't stream data but httpclient can
+  // - URLConnection doesn't expose responsecodes unless it's a 200
+  // - httpclient supports pipelining which we might wanna expose down the road?
+  compile "org.apache.httpcomponents:httpclient:4.5.1"
+  compile "org.apache.httpcomponents:httpcore:4.4.4"
+  compile "org.apache.httpcomponents:httpcore-nio:4.4.4" // currently unused
+  compile "commons-logging:commons-logging:1.2"
+  compile 'org.apache.httpcomponents:httpasyncclient:4.1.1' // currently unused
+}
+
+
+compileJava.options.compilerArgs << '-Xlint:-cast,-rawtypes,-try,-unchecked'
+compileTestJava.options.compilerArgs << '-Xlint:-rawtypes'
+
+// the main files are actually test files, so use the appopriate forbidden api sigs
+forbiddenApisMain {
+  bundledSignatures = ['jdk-unsafe', 'jdk-deprecated']
+  signaturesURLs = [PrecommitTasks.getResource('/forbidden/all-signatures.txt'),
+                    PrecommitTasks.getResource('/forbidden/test-signatures.txt')]
+}
+
+forbiddenApisTest.enabled=false
+forbiddenApisMain.enabled=false
+
+// dependency license are currently checked in distribution
+dependencyLicenses.enabled = false
+jarHell.enabled = false
+thirdPartyAudit.enabled = false
+thirdPartyAudit.excludes = [
+  // classes are missing
+  'javax.servlet.ServletContextEvent', 
+  'javax.servlet.ServletContextListener', 
+  'org.apache.avalon.framework.logger.Logger', 
+  'org.apache.log.Hierarchy', 
+  'org.apache.log.Logger', 
+   // we intentionally exclude the ant tasks because people were depending on them from their tests!!!!!!!
+  'org.apache.tools.ant.BuildException', 
+  'org.apache.tools.ant.DirectoryScanner', 
+  'org.apache.tools.ant.Task', 
+  'org.apache.tools.ant.types.FileSet', 
+  'org.easymock.EasyMock', 
+  'org.easymock.IArgumentMatcher', 
+  'org.jmock.core.Constraint',
+]
diff --git a/client/src/main/java/org/elasticsearch/client/IndexClient.java b/client/src/main/java/org/elasticsearch/client/IndexClient.java
new file mode 100644
index 00000000000..ce3905b2a12
--- /dev/null
+++ b/client/src/main/java/org/elasticsearch/client/IndexClient.java
@@ -0,0 +1,65 @@
+package org.elasticsearch.client;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Created by simon on 2/16/16.
+ */
+public class IndexClient {
+
+    private final RestClient client;
+
+    public IndexClient(RestClient client) {
+        this.client = client;
+    }
+
+    public void delete(String index, String type, String id) throws IOException {
+        delete(index, type, id, null);
+    }
+    public void delete(String index, String type, String id, DeleteOptions params) throws IOException {
+        Objects.requireNonNull(index, "index must not be null");
+        Objects.requireNonNull(type, "type must not be null");
+        Objects.requireNonNull(id, "id must not be null");
+        String deleteEndpoint = String.format("/%s/%s/%s", index, type, id);
+        client.httpDelete(deleteEndpoint, params == null ? Collections.emptyMap() : params.options);
+    }
+
+    public class DeleteOptions {
+        private final Map<String, Object> options = new HashMap<>();
+        /** Specific write consistency setting for the operation one of "one", "quorum", "all"*/
+        public void consistency(String consistency) {
+            options.put("consistency", consistency);
+        };
+        /** ID of parent document */
+        public void parent(String parent){
+            options.put("parent", parent);
+        };
+        /** Refresh the index after performing the operation */
+        public void refresh(Boolean refresh) {
+            options.put("refresh", refresh);
+        };
+        /** Specific routing value */
+        public void routing(String routing) {
+            options.put("routing", routing);
+        };
+        /** Explicit version number for concurrency control */
+        public void version(Number version) {
+            options.put("version", version);
+        };
+        /** Specific version type one of "internal", "external", "external_gte", "force" */
+        public void versionType(String versionType) {
+            options.put("version_type", versionType);
+        };
+        /** Explicit operation timeout */
+        public void timeout(String timeout) {
+            options.put("timeout", timeout);
+        };
+    }
+
+
+
+}
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
new file mode 100644
index 00000000000..8c2c9f0e8ee
--- /dev/null
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpResponse;
+import org.apache.http.NameValuePair;
+import org.apache.http.StatusLine;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.utils.URLEncodedUtils;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.message.BasicNameValuePair;
+
+import java.io.BufferedReader;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArraySet;
+import java.util.stream.Collectors;
+
+/**
+ */
+public class RestClient implements Closeable{
+
+    private final CloseableHttpClient client;
+    private volatile Set<HttpHost> hosts;
+    private final String scheme;
+    private final Set<HttpHost> blackList = new CopyOnWriteArraySet<>();
+
+    public RestClient(HttpHost... hosts) {
+        this("http", HttpClientBuilder.create().setDefaultRequestConfig(RequestConfig.custom().setConnectTimeout(100).build()).build(), hosts);
+    }
+
+    public RestClient(String scheme, CloseableHttpClient client, HttpHost[] hosts) {
+        if (hosts.length == 0) {
+            throw new IllegalArgumentException("hosts must note be empty");
+        }
+        this.scheme = scheme;
+        this.client = client;
+        this.hosts = new HashSet<>(Arrays.asList(hosts));
+    }
+
+
+    public HttpResponse httpGet(String endpoint, Map<String, Object> params) throws IOException {
+        return httpGet(getHostIterator(true), endpoint, params);
+    }
+
+    HttpResponse httpGet(Iterable<HttpHost> hosts, String endpoint, Map<String, Object> params) throws IOException {
+        HttpUriRequest request = new HttpGet(buildUri(endpoint, pairs(params)));
+        return execute(request, hosts);
+    }
+
+    HttpResponse httpDelete(String endpoint, Map<String, Object> params) throws IOException {
+        HttpUriRequest request = new HttpDelete(buildUri(endpoint, pairs(params)));
+        return execute(request, getHostIterator(true));
+    }
+
+    HttpResponse httpPut(String endpoint, HttpEntity body, Map<String, Object>  params) throws IOException {
+        HttpPut request = new HttpPut(buildUri(endpoint, pairs(params)));
+        request.setEntity(body);
+        return execute(request, getHostIterator(true));
+    }
+
+    HttpResponse httpPost(String endpoint, HttpEntity body, Map<String, Object> params) throws IOException {
+        HttpPost request = new HttpPost(buildUri(endpoint, pairs(params)));
+        request.setEntity(body);
+        return execute(request, getHostIterator(true));
+    }
+
+    private List<NameValuePair> pairs(Map<String, Object> options) {
+        return options.entrySet().stream().map(e -> new BasicNameValuePair(e.getKey(), e.getValue().toString()))
+            .collect(Collectors.toList());
+    }
+
+    public HttpResponse execute(HttpUriRequest request, Iterable<HttpHost> retryHosts) throws IOException {
+        IOException exception = null;
+        for (HttpHost singleHost : retryHosts) {
+            try {
+                return client.execute(singleHost, request);
+            } catch (IOException ex) {
+                if (this.hosts.contains(singleHost)) {
+                    blackList.add(singleHost);
+                }
+                if (exception != null) {
+                    exception.addSuppressed(ex);
+                } else {
+                    exception = ex;
+                }
+            }
+        }
+        throw exception;
+    }
+
+    public URI buildUri(String path, List<NameValuePair> query) {
+        try {
+            return new URI(null, null, null, -1, path, URLEncodedUtils.format(query, StandardCharsets.UTF_8), null);
+        } catch (URISyntaxException e) {
+            throw new RuntimeException(e.getMessage(), e);
+        }
+    }
+
+    public Set<HttpHost> fetchNodes(HttpHost host, boolean useClientNodes, boolean local, boolean checkAvailable) throws IOException {
+        HttpResponse httpResponse = httpGet(Collections.singleton(host), "/_cat/nodes", Collections.singletonMap("h", "http,role"));
+        StatusLine statusLine = httpResponse.getStatusLine();
+        if (statusLine.getStatusCode() != 200) {
+            throw new RuntimeException("failed to fetch nodes: " + statusLine.getReasonPhrase());
+        }
+        HttpEntity entity = httpResponse.getEntity();
+        Set<HttpHost> hosts = new HashSet<>();
+        try (BufferedReader content = new BufferedReader(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8))) {
+            String line;
+            while((line = content.readLine()) != null) {
+                final String[] split = line.split("\\s+");
+                assert split.length == 2;
+                String boundAddress = split[0];
+                String role = split[1];
+                if ("-".equals(split[0].trim()) == false) {
+                    if ("d".equals(role.trim()) == false && useClientNodes == false) {
+                        continue;
+                    }
+                    URI boundAddressAsURI = URI.create("http://" + boundAddress);
+                    HttpHost newHost = new HttpHost(boundAddressAsURI.getHost(), boundAddressAsURI.getPort(), scheme);
+                    if (checkAvailable == false || isAvailable(newHost)) {
+                        hosts.add(newHost);
+                    }
+                }
+            }
+        }
+        return hosts;
+    }
+
+    public String getClusterName(HttpHost host) throws IOException {
+        HttpResponse httpResponse = httpGet(Collections.singleton(host), "/_cat/health", Collections.singletonMap("h", "cluster"));
+        StatusLine statusLine = httpResponse.getStatusLine();
+        if (statusLine.getStatusCode() != 200) {
+            throw new RuntimeException("failed to fetch nodes: " + statusLine.getReasonPhrase());
+        }
+        HttpEntity entity = httpResponse.getEntity();
+        try (BufferedReader content = new BufferedReader(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8))) {
+            String clusterName = content.readLine().trim();
+            if (clusterName.length() == 0) {
+                throw new IllegalStateException("clustername must not be empty");
+            }
+            return clusterName;
+        }
+    }
+
+    public boolean isAvailable(HttpHost host) {
+        try {
+            HttpResponse httpResponse = httpGet(Collections.singleton(host), "/", Collections.emptyMap());
+            StatusLine statusLine = httpResponse.getStatusLine();
+            return statusLine.getStatusCode() == 200;
+        } catch (IOException ex) {
+            return false;
+        }
+    }
+
+    public synchronized void setNodes(Set<HttpHost> hosts) {
+        this.hosts = Collections.unmodifiableSet(new HashSet<>(hosts));
+        blackList.retainAll(hosts);
+    }
+
+    public Set<HttpHost> getHosts() {
+        return hosts;
+    }
+
+    protected Iterable<HttpHost> getHostIterator(boolean clearBlacklist) {
+        if (hosts.size() == blackList.size() && clearBlacklist) {
+            blackList.clear(); // lets try again
+        }
+        return () -> hosts.stream().filter((h) -> blackList.contains(h) == false).iterator();
+    }
+
+    int getNumHosts() {
+        return hosts.size();
+    }
+
+    int getNumBlacklistedHosts() {
+        return blackList.size();
+    }
+    @Override
+    public void close() throws IOException {
+        client.close();
+    }
+}
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientTests.java b/client/src/test/java/org/elasticsearch/client/RestClientTests.java
new file mode 100644
index 00000000000..366ae692339
--- /dev/null
+++ b/client/src/test/java/org/elasticsearch/client/RestClientTests.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client;
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpServer;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpResponse;
+import org.apache.lucene.util.LuceneTestCase;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+public class RestClientTests extends LuceneTestCase {
+    //TODO this should be refactored into a base test!!
+    HttpServer server;
+    protected String clusterName = "elasticsearch";
+    protected List<String> additionalNodes = Collections.emptyList();
+
+
+    public void setUp() throws Exception {
+        super.setUp();
+        server = HttpServer.create(new InetSocketAddress(0), 0);
+        server.setExecutor(null); // creates a default executor
+        server.start();
+        server.createContext("/", (t) -> {
+            handle("/", t);
+        });
+        server.createContext("/_cat/nodes", (t) -> {
+            handle("/_cat/nodes", t);
+        });
+        server.createContext("/_cat/health", (t) -> {
+            handle("/_cat/health", t);
+        });
+    }
+
+    protected void handle(String path, HttpExchange t) throws IOException {
+        final String response;
+        switch (path) {
+            case "/":
+                response = "{}";
+                break;
+            case "/_cat/nodes":
+                StringBuilder builder = new StringBuilder( "127.0.0.1:" + server.getAddress().getPort()  + " " + "d\n");
+                for (String host : additionalNodes) {
+                    builder.append(host).append("\n");
+                }
+                response = builder.toString();
+                break;
+            case "/_cat/health":
+                response = clusterName;
+                break;
+            default:
+                throw new IllegalArgumentException("no such handler " + path);
+        }
+        t.sendResponseHeaders(200, response.length());
+        OutputStream os = t.getResponseBody();
+        os.write(response.getBytes());
+        os.close();
+    }
+
+    public void tearDown() throws Exception {
+        super.tearDown();
+        server.stop(0);
+    }
+
+    public void testGetClustername() throws IOException {
+        HttpHost httpHost = new HttpHost("127.0.0.1", server.getAddress().getPort(), "http");
+        try(RestClient client = new RestClient(httpHost)) {
+            assertEquals(clusterName, client.getClusterName(httpHost));
+        }
+    }
+
+    public void testFetchNodes() throws IOException {
+        additionalNodes = Arrays.asList("127.0.0.2:9200 c", "127.0.0.3:9200 d");
+        HttpHost httpHost = new HttpHost("127.0.0.1", server.getAddress().getPort(), "http");
+        try(RestClient client = new RestClient(httpHost)) {
+            assertEquals(3, client.fetchNodes(httpHost, true, true, false).size());
+            assertTrue(client.fetchNodes(httpHost, true, true, false).toString(), client.fetchNodes(httpHost, true, true, false).contains(new HttpHost("127.0.0.2", 9200, "http")));
+            assertTrue(client.fetchNodes(httpHost, true, true, false).contains(new HttpHost("127.0.0.3", 9200, "http")));
+            assertTrue(client.fetchNodes(httpHost, true, true, false).contains(httpHost));
+            assertEquals(1, client.fetchNodes(httpHost, true, true, true).size());
+        }
+    }
+
+    public void testSimpleRetry() throws IOException{
+        additionalNodes = Arrays.asList("127.0.0.2:9200 c", "127.0.0.3:9200 d");
+        HttpHost httpHost = new HttpHost("127.0.0.1", server.getAddress().getPort(), "http");
+        try(RestClient client = new RestClient(httpHost)) {
+            client.setNodes(client.fetchNodes(httpHost, true, true, false));
+            HttpResponse httpResponse = client.httpGet("/_cat/health", Collections.emptyMap());
+            assertEquals(httpResponse.getStatusLine().getStatusCode(), 200);
+            server.stop(0);
+            try {
+                client.httpGet("/_cat/health", Collections.emptyMap());
+                fail();
+            } catch (IOException ex) {
+                assertTrue(ex.getMessage(), ex.getMessage().endsWith("failed: connect timed out") || ex.getMessage().endsWith("failed: Connection refused"));
+            }
+        }
+    }
+
+    public void testBlacklist() throws IOException{
+        additionalNodes = Arrays.asList("127.0.0.2:9200 c", "127.0.0.3:9200 d");
+        HttpHost httpHost = new HttpHost("127.0.0.1", server.getAddress().getPort(), "http");
+        try(RestClient client = new RestClient(httpHost)) {
+            client.setNodes(client.fetchNodes(httpHost, true, true, false));
+            assertEquals(3, client.getNumHosts());
+            assertEquals(0, client.getNumBlacklistedHosts());
+            server.stop(0);
+            try {
+                client.httpGet("/_cat/health", Collections.emptyMap());
+                fail();
+            } catch (IOException ex) {
+                assertTrue(ex.getMessage(), ex.getMessage().endsWith("failed: connect timed out") || ex.getMessage().endsWith("failed: Connection refused"));
+            }
+            assertEquals(3, client.getNumHosts());
+            assertEquals(3, client.getNumBlacklistedHosts());
+            int num = 0;
+            for (HttpHost host : client.getHostIterator(false)) {
+                num++; // nothing here
+            }
+            assertEquals(0, num);
+            for (HttpHost host : client.getHostIterator(true)) {
+                num++; // all there - we have to retry now
+            }
+            assertEquals(3, num);
+        }
+    }
+
+
+}
diff --git a/settings.gradle b/settings.gradle
index 215b436e20a..831b2c5d2fb 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -5,6 +5,7 @@ List projects = [
   'rest-api-spec',
   'core',
   'docs',
+  'client',
   'distribution:integ-test-zip',
   'distribution:zip',
   'distribution:tar',

From c9c33a7719d7f2bb00c507b50e08d49d1d08474c Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 8 Apr 2016 15:24:21 +0200
Subject: [PATCH 0579/1311] Fix checkstyle issues, setup thirdPartyAudit checks
 and enable forbiddenApisMain

---
 client/build.gradle                           | 69 ++++++++-----------
 .../org/elasticsearch/client/IndexClient.java | 27 ++++++--
 .../org/elasticsearch/client/RestClient.java  |  3 +-
 .../elasticsearch/client/RestClientTests.java | 14 ++--
 4 files changed, 59 insertions(+), 54 deletions(-)

diff --git a/client/build.gradle b/client/build.gradle
index 1ae32bf6e98..bd916cd2cee 100644
--- a/client/build.gradle
+++ b/client/build.gradle
@@ -23,60 +23,51 @@ group = 'org.elasticsearch.client'
 apply plugin: 'elasticsearch.build'
 
 dependencies {
-  // we use the lucene test-framework here but we are not pulling in ES core or the test framework
+  // TODO once we got rid of the client in the test framework we should use a version variable here
+  compile "org.apache.httpcomponents:httpclient:4.5.2"
+  compile "org.apache.httpcomponents:httpcore:4.4.4"
+  //compile "org.apache.httpcomponents:httpcore-nio:4.4.4"
+  //compile "org.apache.httpcomponents:httpasyncclient:4.1.1"
+  compile "commons-codec:commons-codec:1.9"
+  compile "commons-logging:commons-logging:1.2"
+
+  // we use lucene-test-framework here without pulling in ES core or its own test-framework
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   testCompile 'org.hamcrest:hamcrest-all:1.3'
   testCompile "org.apache.lucene:lucene-test-framework:${versions.lucene}"
   testCompile "org.apache.lucene:lucene-core:${versions.lucene}"
   testCompile "org.apache.lucene:lucene-codecs:${versions.lucene}"
-
-
-  // TODO once we got rid of the client in the test framework we should use a version variable here
-  // we use httpclient here since the JDK support has several issue
-  // - httpclient supports basic and digest auth and other schemes
-  // - URLConnection has issues with SSL and not all system patches are available
-  // - URLConnection can't stream data but httpclient can
-  // - URLConnection doesn't expose responsecodes unless it's a 200
-  // - httpclient supports pipelining which we might wanna expose down the road?
-  compile "org.apache.httpcomponents:httpclient:4.5.1"
-  compile "org.apache.httpcomponents:httpcore:4.4.4"
-  compile "org.apache.httpcomponents:httpcore-nio:4.4.4" // currently unused
-  compile "commons-logging:commons-logging:1.2"
-  compile 'org.apache.httpcomponents:httpasyncclient:4.1.1' // currently unused
 }
 
-
 compileJava.options.compilerArgs << '-Xlint:-cast,-rawtypes,-try,-unchecked'
 compileTestJava.options.compilerArgs << '-Xlint:-rawtypes'
 
-// the main files are actually test files, so use the appopriate forbidden api sigs
 forbiddenApisMain {
-  bundledSignatures = ['jdk-unsafe', 'jdk-deprecated']
-  signaturesURLs = [PrecommitTasks.getResource('/forbidden/all-signatures.txt'),
-                    PrecommitTasks.getResource('/forbidden/test-signatures.txt')]
+  //client does not depend on core, so only jdk signatures should be checked
+  signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
 }
 
+forbiddenApisMain.enabled=true
+//TODO remove use of sun http server and enable forbidden-apis for tests
 forbiddenApisTest.enabled=false
-forbiddenApisMain.enabled=false
+//TODO add licenses for deps and check out distribution task
+//dependency license are currently checked in distribution
+dependencyLicenses.enabled=false
+//TODO enable jarhell checks
+jarHell.enabled=false
+//NamingConventionCheck is part of test-framework, which we don't want to pull in as it depends on es core
+namingConventions.enabled=false
 
-// dependency license are currently checked in distribution
-dependencyLicenses.enabled = false
-jarHell.enabled = false
-thirdPartyAudit.enabled = false
 thirdPartyAudit.excludes = [
-  // classes are missing
-  'javax.servlet.ServletContextEvent', 
-  'javax.servlet.ServletContextListener', 
-  'org.apache.avalon.framework.logger.Logger', 
-  'org.apache.log.Hierarchy', 
-  'org.apache.log.Logger', 
-   // we intentionally exclude the ant tasks because people were depending on them from their tests!!!!!!!
-  'org.apache.tools.ant.BuildException', 
-  'org.apache.tools.ant.DirectoryScanner', 
-  'org.apache.tools.ant.Task', 
-  'org.apache.tools.ant.types.FileSet', 
-  'org.easymock.EasyMock', 
-  'org.easymock.IArgumentMatcher', 
-  'org.jmock.core.Constraint',
+  //commons-logging optional dependencies
+  'org.apache.avalon.framework.logger.Logger',
+  'org.apache.log.Hierarchy',
+  'org.apache.log.Logger',
+  'org.apache.log4j.Level',
+  'org.apache.log4j.Logger',
+  'org.apache.log4j.Priority',
+  //commons-logging provided dependencies
+  'javax.servlet.ServletContextEvent',
+  'javax.servlet.ServletContextListener'
 ]
diff --git a/client/src/main/java/org/elasticsearch/client/IndexClient.java b/client/src/main/java/org/elasticsearch/client/IndexClient.java
index ce3905b2a12..6b116b35993 100644
--- a/client/src/main/java/org/elasticsearch/client/IndexClient.java
+++ b/client/src/main/java/org/elasticsearch/client/IndexClient.java
@@ -1,14 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
 package org.elasticsearch.client;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 
-/**
- * Created by simon on 2/16/16.
- */
 public class IndexClient {
 
     private final RestClient client;
@@ -24,7 +40,7 @@ public class IndexClient {
         Objects.requireNonNull(index, "index must not be null");
         Objects.requireNonNull(type, "type must not be null");
         Objects.requireNonNull(id, "id must not be null");
-        String deleteEndpoint = String.format("/%s/%s/%s", index, type, id);
+        String deleteEndpoint = String.format(Locale.ROOT, "/%s/%s/%s", index, type, id);
         client.httpDelete(deleteEndpoint, params == null ? Collections.emptyMap() : params.options);
     }
 
@@ -59,7 +75,4 @@ public class IndexClient {
             options.put("timeout", timeout);
         };
     }
-
-
-
 }
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 8c2c9f0e8ee..25d96aa903d 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -60,7 +60,8 @@ public class RestClient implements Closeable{
     private final Set<HttpHost> blackList = new CopyOnWriteArraySet<>();
 
     public RestClient(HttpHost... hosts) {
-        this("http", HttpClientBuilder.create().setDefaultRequestConfig(RequestConfig.custom().setConnectTimeout(100).build()).build(), hosts);
+        this("http", HttpClientBuilder.create().setDefaultRequestConfig(
+                RequestConfig.custom().setConnectTimeout(100).build()).build(), hosts);
     }
 
     public RestClient(String scheme, CloseableHttpClient client, HttpHost[] hosts) {
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientTests.java b/client/src/test/java/org/elasticsearch/client/RestClientTests.java
index 366ae692339..038cb8bab3c 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientTests.java
@@ -26,11 +26,10 @@ import org.apache.lucene.util.LuceneTestCase;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.net.InetSocketAddress;
-import java.net.URISyntaxException;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
-import java.util.concurrent.ExecutionException;
+
 public class RestClientTests extends LuceneTestCase {
     //TODO this should be refactored into a base test!!
     HttpServer server;
@@ -96,7 +95,8 @@ public class RestClientTests extends LuceneTestCase {
         HttpHost httpHost = new HttpHost("127.0.0.1", server.getAddress().getPort(), "http");
         try(RestClient client = new RestClient(httpHost)) {
             assertEquals(3, client.fetchNodes(httpHost, true, true, false).size());
-            assertTrue(client.fetchNodes(httpHost, true, true, false).toString(), client.fetchNodes(httpHost, true, true, false).contains(new HttpHost("127.0.0.2", 9200, "http")));
+            assertTrue(client.fetchNodes(httpHost, true, true, false).toString(), client.fetchNodes(httpHost, true, true, false)
+                    .contains(new HttpHost("127.0.0.2", 9200, "http")));
             assertTrue(client.fetchNodes(httpHost, true, true, false).contains(new HttpHost("127.0.0.3", 9200, "http")));
             assertTrue(client.fetchNodes(httpHost, true, true, false).contains(httpHost));
             assertEquals(1, client.fetchNodes(httpHost, true, true, true).size());
@@ -115,7 +115,8 @@ public class RestClientTests extends LuceneTestCase {
                 client.httpGet("/_cat/health", Collections.emptyMap());
                 fail();
             } catch (IOException ex) {
-                assertTrue(ex.getMessage(), ex.getMessage().endsWith("failed: connect timed out") || ex.getMessage().endsWith("failed: Connection refused"));
+                assertTrue(ex.getMessage(), ex.getMessage().endsWith("failed: connect timed out") ||
+                        ex.getMessage().endsWith("failed: Connection refused"));
             }
         }
     }
@@ -132,7 +133,8 @@ public class RestClientTests extends LuceneTestCase {
                 client.httpGet("/_cat/health", Collections.emptyMap());
                 fail();
             } catch (IOException ex) {
-                assertTrue(ex.getMessage(), ex.getMessage().endsWith("failed: connect timed out") || ex.getMessage().endsWith("failed: Connection refused"));
+                assertTrue(ex.getMessage(), ex.getMessage().endsWith("failed: connect timed out") ||
+                        ex.getMessage().endsWith("failed: Connection refused"));
             }
             assertEquals(3, client.getNumHosts());
             assertEquals(3, client.getNumBlacklistedHosts());
@@ -147,6 +149,4 @@ public class RestClientTests extends LuceneTestCase {
             assertEquals(3, num);
         }
     }
-
-
 }

From 59707239452bdf24fe6a32deed5f21c5205f79e9 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Mon, 11 Apr 2016 14:32:19 +0200
Subject: [PATCH 0580/1311] clarify why jarhell and dependency license check
 are disabled, add okhttp mockwebserver dependency

Also enable forbiddenApisTest (will fail till we get rid of the sun http web server in RestClientTests that will be replaced with mockwebserver)
---
 client/build.gradle | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/client/build.gradle b/client/build.gradle
index bd916cd2cee..d94676267d0 100644
--- a/client/build.gradle
+++ b/client/build.gradle
@@ -31,13 +31,18 @@ dependencies {
   compile "commons-codec:commons-codec:1.9"
   compile "commons-logging:commons-logging:1.2"
 
-  // we use lucene-test-framework here without pulling in ES core or its own test-framework
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   testCompile 'org.hamcrest:hamcrest-all:1.3'
   testCompile "org.apache.lucene:lucene-test-framework:${versions.lucene}"
   testCompile "org.apache.lucene:lucene-core:${versions.lucene}"
   testCompile "org.apache.lucene:lucene-codecs:${versions.lucene}"
+  //mock web server
+  testCompile "com.squareup.okhttp3:mockwebserver:3.2.0"
+  testCompile "com.squareup.okhttp3:okhttp:3.2.0"
+  testCompile "com.squareup.okhttp3:okhttp-ws:3.2.0"
+  testCompile "com.squareup.okio:okio:1.6.0"
+  testCompile "org.bouncycastle:bcprov-jdk15on:1.54"
 }
 
 compileJava.options.compilerArgs << '-Xlint:-cast,-rawtypes,-try,-unchecked'
@@ -48,13 +53,15 @@ forbiddenApisMain {
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
 }
 
-forbiddenApisMain.enabled=true
-//TODO remove use of sun http server and enable forbidden-apis for tests
-forbiddenApisTest.enabled=false
-//TODO add licenses for deps and check out distribution task
+forbiddenApisTest {
+  //client does not depend on core, so only jdk signatures should be checked
+  signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
+}
+
+//TODO add licenses for dependencies and take care of distribution
 //dependency license are currently checked in distribution
 dependencyLicenses.enabled=false
-//TODO enable jarhell checks
+//JarHell is part of es core, which we don't want to pull in
 jarHell.enabled=false
 //NamingConventionCheck is part of test-framework, which we don't want to pull in as it depends on es core
 namingConventions.enabled=false
@@ -64,6 +71,7 @@ thirdPartyAudit.excludes = [
   'org.apache.avalon.framework.logger.Logger',
   'org.apache.log.Hierarchy',
   'org.apache.log.Logger',
+  'org.apache.log4j.Category',
   'org.apache.log4j.Level',
   'org.apache.log4j.Logger',
   'org.apache.log4j.Priority',

From 6ae8be55bd3e120c4ed3186295aac10feb78ac64 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Thu, 28 Apr 2016 23:16:06 +0200
Subject: [PATCH 0581/1311] Introduce notion of Transport, Connection and
 ConnectionPool

There are two implementations of connection pool, a static one that allows to enable/disable pings, and a sniffing one that sniffs nodes from the nodes info api.

Transport retrieves a stream of connections from the connection for each request and calls onSuccess or onFailure depending on the result of the request.

Transport also supports a max retry timeout to control the timeout for the request retries overall.
---
 client/build.gradle                           |   3 +-
 .../client/AbstractStaticConnectionPool.java  | 105 +++++++
 .../org/elasticsearch/client/Connection.java  |  45 +++
 .../elasticsearch/client/ConnectionPool.java  |  69 +++++
 .../client/ElasticsearchResponse.java         |  99 +++++++
 .../ElasticsearchResponseException.java       |  60 ++++
 .../client/HttpDeleteWithEntity.java          |  41 +++
 .../client/HttpGetWithEntity.java             |  41 +++
 .../org/elasticsearch/client/IndexClient.java |  78 ------
 .../java/org/elasticsearch/client/Node.java   | 125 +++++++++
 .../elasticsearch/client/RequestLogger.java   |  51 ++++
 .../org/elasticsearch/client/RestClient.java  | 189 +------------
 .../client/RetryTimeoutException.java         |  29 ++
 .../org/elasticsearch/client/Sniffer.java     | 162 +++++++++++
 .../client/SniffingConnectionPool.java        | 196 +++++++++++++
 .../client/StatefulConnection.java            | 125 +++++++++
 .../client/StaticConnectionPool.java          |  93 +++++++
 .../org/elasticsearch/client/Transport.java   | 207 ++++++++++++++
 .../java/org/elasticsearch/client/Verb.java   |  27 ++
 .../org/elasticsearch/client/NodeTests.java   | 136 +++++++++
 .../elasticsearch/client/RestClientTests.java | 152 ----------
 .../elasticsearch/client/SnifferTests.java    | 259 ++++++++++++++++++
 .../client/SniffingConnectionPoolTests.java   | 147 ++++++++++
 .../client/StaticConnectionPoolTests.java     |  87 ++++++
 .../elasticsearch/client/TransportTests.java  |  95 +++++++
 25 files changed, 2209 insertions(+), 412 deletions(-)
 create mode 100644 client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java
 create mode 100644 client/src/main/java/org/elasticsearch/client/Connection.java
 create mode 100644 client/src/main/java/org/elasticsearch/client/ConnectionPool.java
 create mode 100644 client/src/main/java/org/elasticsearch/client/ElasticsearchResponse.java
 create mode 100644 client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
 create mode 100644 client/src/main/java/org/elasticsearch/client/HttpDeleteWithEntity.java
 create mode 100644 client/src/main/java/org/elasticsearch/client/HttpGetWithEntity.java
 delete mode 100644 client/src/main/java/org/elasticsearch/client/IndexClient.java
 create mode 100644 client/src/main/java/org/elasticsearch/client/Node.java
 create mode 100644 client/src/main/java/org/elasticsearch/client/RequestLogger.java
 create mode 100644 client/src/main/java/org/elasticsearch/client/RetryTimeoutException.java
 create mode 100644 client/src/main/java/org/elasticsearch/client/Sniffer.java
 create mode 100644 client/src/main/java/org/elasticsearch/client/SniffingConnectionPool.java
 create mode 100644 client/src/main/java/org/elasticsearch/client/StatefulConnection.java
 create mode 100644 client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
 create mode 100644 client/src/main/java/org/elasticsearch/client/Transport.java
 create mode 100644 client/src/main/java/org/elasticsearch/client/Verb.java
 create mode 100644 client/src/test/java/org/elasticsearch/client/NodeTests.java
 delete mode 100644 client/src/test/java/org/elasticsearch/client/RestClientTests.java
 create mode 100644 client/src/test/java/org/elasticsearch/client/SnifferTests.java
 create mode 100644 client/src/test/java/org/elasticsearch/client/SniffingConnectionPoolTests.java
 create mode 100644 client/src/test/java/org/elasticsearch/client/StaticConnectionPoolTests.java
 create mode 100644 client/src/test/java/org/elasticsearch/client/TransportTests.java

diff --git a/client/build.gradle b/client/build.gradle
index d94676267d0..7a9377d0acb 100644
--- a/client/build.gradle
+++ b/client/build.gradle
@@ -30,10 +30,11 @@ dependencies {
   //compile "org.apache.httpcomponents:httpasyncclient:4.1.1"
   compile "commons-codec:commons-codec:1.9"
   compile "commons-logging:commons-logging:1.2"
+  compile "com.fasterxml.jackson.core:jackson-core:2.7.3"
 
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
-  testCompile 'org.hamcrest:hamcrest-all:1.3'
+  testCompile "org.hamcrest:hamcrest-all:1.3"
   testCompile "org.apache.lucene:lucene-test-framework:${versions.lucene}"
   testCompile "org.apache.lucene:lucene-core:${versions.lucene}"
   testCompile "org.apache.lucene:lucene-codecs:${versions.lucene}"
diff --git a/client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java b/client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java
new file mode 100644
index 00000000000..f30a2b9c5c1
--- /dev/null
+++ b/client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
+import java.util.stream.Stream;
+
+/**
+ * Base static connection pool implementation that deals with mutable connections. Marks connections as dead/alive when needed.
+ * Provides a stream of alive connections or dead ones that should be retried for each {@link #nextConnection()} call, which
+ * allows to filter connections through a customizable {@link Predicate}, called connection selector.
+ * In case the returned stream is empty a last resort dead connection should be retrieved by calling {@link #lastResortConnection()}
+ * and resurrected so that a single request attempt can be performed.
+ * The {@link #onSuccess(StatefulConnection)} method marks the connection provided as an argument alive.
+ * The {@link #onFailure(StatefulConnection)} method marks the connection provided as an argument dead.
+ * This base implementation doesn't define the list implementation that stores connections, so that concurrency can be
+ * handled in the subclasses depending on the usecase (e.g. defining the list volatile when needed).
+ */
+public abstract class AbstractStaticConnectionPool implements ConnectionPool<StatefulConnection> {
+
+    private static final Log logger = LogFactory.getLog(AbstractStaticConnectionPool.class);
+
+    private final Predicate<Connection> connectionSelector;
+
+    private final AtomicInteger lastConnectionIndex = new AtomicInteger(0);
+
+    protected AbstractStaticConnectionPool(Predicate<Connection> connectionSelector) {
+        Objects.requireNonNull(connectionSelector, "connection selector predicate cannot be null");
+        this.connectionSelector = connectionSelector;
+    }
+
+    protected abstract List<StatefulConnection> getConnections();
+
+    @Override
+    public final Stream<StatefulConnection> nextConnection() {
+        return nextUnfilteredConnection().filter(connectionSelector);
+    }
+
+    protected final Stream<StatefulConnection> nextUnfilteredConnection() {
+        List<StatefulConnection> connections = getConnections();
+        if (connections.isEmpty()) {
+            throw new IllegalStateException("no connections available in the connection pool");
+        }
+
+        List<StatefulConnection> sortedConnections = new ArrayList<>(connections);
+        //TODO is it possible to make this O(1)? (rotate is O(n))
+        Collections.rotate(sortedConnections, sortedConnections.size() - lastConnectionIndex.getAndIncrement());
+        return sortedConnections.stream().filter(connection -> connection.isAlive() || connection.shouldBeRetried());
+    }
+
+    protected List<StatefulConnection> createConnections(Node... nodes) {
+        List<StatefulConnection> connections = new ArrayList<>();
+        for (Node node : nodes) {
+            Objects.requireNonNull(node, "node cannot be null");
+            connections.add(new StatefulConnection(node));
+        }
+        return Collections.unmodifiableList(connections);
+    }
+
+    @Override
+    public StatefulConnection lastResortConnection() {
+        StatefulConnection statefulConnection = getConnections().stream()
+                .sorted((o1, o2) -> Long.compare(o1.getDeadUntil(), o2.getDeadUntil())).findFirst().get();
+        statefulConnection.markResurrected();
+        return statefulConnection;
+    }
+
+    @Override
+    public void onSuccess(StatefulConnection connection) {
+        connection.markAlive();
+        logger.trace("marked connection alive for " + connection.getNode());
+    }
+
+    @Override
+    public void onFailure(StatefulConnection connection) throws IOException {
+        connection.markDead();
+        logger.debug("marked connection dead for " + connection.getNode());
+    }
+}
diff --git a/client/src/main/java/org/elasticsearch/client/Connection.java b/client/src/main/java/org/elasticsearch/client/Connection.java
new file mode 100644
index 00000000000..225c812ebfa
--- /dev/null
+++ b/client/src/main/java/org/elasticsearch/client/Connection.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+/**
+ * Simplest representation of a connection to an elasticsearch node.
+ * It doesn't have any mutable state. It holds the node that the connection points to.
+ * Allows the transport to deal with very simple connection objects that are immutable.
+ * Any change to the state of connections should be made through the connection pool
+ * which is aware of the connection object that it supports.
+ */
+public class Connection {
+    private final Node node;
+
+    /**
+     * Creates a new connection pointing to the provided {@link Node} argument
+     */
+    public Connection(Node node) {
+            this.node = node;
+        }
+
+    /**
+     * Returns the {@link Node} that the connection points to
+     */
+    public Node getNode() {
+            return node;
+        }
+}
diff --git a/client/src/main/java/org/elasticsearch/client/ConnectionPool.java b/client/src/main/java/org/elasticsearch/client/ConnectionPool.java
new file mode 100644
index 00000000000..3f2eb89da97
--- /dev/null
+++ b/client/src/main/java/org/elasticsearch/client/ConnectionPool.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.stream.Stream;
+
+/**
+ * Pool of connections to the different nodes that belong to an elasticsearch cluster.
+ * It keeps track of the different nodes to communicate with and allows to retrieve a stream of connections to be used
+ * for each request. Exposes the needed hooks to be able to eventually mark connections dead or alive and execute
+ * arbitrary operations before each single request attempt.
+ *
+ * @param <C> the type of {@link Connection} that the pool supports
+ */
+public interface ConnectionPool<C extends Connection> extends Closeable {
+
+    /**
+     * Returns a stream of connections that should be used for a request call.
+     * Ideally, the first connection is retrieved from the stream and used successfully for the request.
+     * Otherwise, after each failure the next connection should be retrieved from the stream so that the request can be retried.
+     * The maximum total of attempts is equal to the number of connections that are available in the stream.
+     * It may happen that the stream is empty, in which case it means that there aren't healthy connections to use.
+     * Then {@link #lastResortConnection()} should be called to retrieve a non healthy connection and try it.
+     */
+    Stream<C> nextConnection();
+
+    /**
+     * Returns a connection that is not necessarily healthy, but can be used for a request attempt. To be called as last resort
+     * only in case {@link #nextConnection()} returns an empty stream
+     */
+    C lastResortConnection();
+
+    /**
+     * Called before each single request attempt. Allows to execute operations (e.g. ping) before each request.
+     * Receives as an argument the connection that is going to be used for the request.
+     */
+    void beforeAttempt(C connection) throws IOException;
+
+    /**
+     * Called after each successful request call.
+     * Receives as an argument the connection that was used for the successful request.
+     */
+    void onSuccess(C connection);
+
+    /**
+     * Called after each failed attempt.
+     * Receives as an argument the connection that was used for the failed attempt.
+     */
+    void onFailure(C connection) throws IOException;
+}
diff --git a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponse.java b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponse.java
new file mode 100644
index 00000000000..13c92c16453
--- /dev/null
+++ b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponse.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.RequestLine;
+import org.apache.http.StatusLine;
+import org.apache.http.client.methods.CloseableHttpResponse;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Holds an elasticsearch response. It wraps the {@link CloseableHttpResponse} response and associates it with
+ * its corresponding {@link RequestLine} and {@link Node}
+ */
+public class ElasticsearchResponse implements Closeable {
+
+    private final RequestLine requestLine;
+    private final Node node;
+    private final CloseableHttpResponse response;
+
+    ElasticsearchResponse(RequestLine requestLine, Node node, CloseableHttpResponse response) {
+        Objects.requireNonNull(requestLine, "requestLine cannot be null");
+        Objects.requireNonNull(node, "node cannot be null");
+        Objects.requireNonNull(response, "response cannot be null");
+        this.requestLine = requestLine;
+        this.node = node;
+        this.response = response;
+    }
+
+    /**
+     * Returns the request line that generated this response
+     */
+    public RequestLine getRequestLine() {
+        return requestLine;
+    }
+
+    /**
+     * Returns the node that returned this response
+     */
+    public Node getNode() {
+        return node;
+    }
+
+    /**
+     * Returns the status line of the current response
+     */
+    public StatusLine getStatusLine() {
+        return response.getStatusLine();
+    }
+
+    /**
+     * Returns all the response headers
+     */
+    public Header[] getHeaders() {
+        return response.getAllHeaders();
+    }
+
+    /**
+     * Returns the response bodyi available, null otherwise
+     */
+    public HttpEntity getEntity() {
+        return response.getEntity();
+    }
+
+    @Override
+    public String toString() {
+        return "ElasticsearchResponse{" +
+                "requestLine=" + requestLine +
+                ", node=" + node +
+                ", response=" + response.getStatusLine() +
+                '}';
+    }
+
+    @Override
+    public void close() throws IOException {
+        this.response.close();
+    }
+}
diff --git a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
new file mode 100644
index 00000000000..6c2ef2d2e0d
--- /dev/null
+++ b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.RequestLine;
+import org.apache.http.StatusLine;
+
+import java.io.IOException;
+
+public class ElasticsearchResponseException extends IOException {
+
+    private final Node node;
+    private final RequestLine requestLine;
+    private final StatusLine statusLine;
+
+    ElasticsearchResponseException(RequestLine requestLine, Node node, StatusLine statusLine) {
+        super(buildMessage(requestLine, node, statusLine));
+        this.node = node;
+        this.requestLine = requestLine;
+        this.statusLine = statusLine;
+    }
+
+    private static String buildMessage(RequestLine requestLine, Node node, StatusLine statusLine) {
+        return requestLine.getMethod() + " " + node.getHttpHost() + requestLine.getUri() + ": " + statusLine.toString();
+    }
+
+    public boolean isRecoverable() {
+        //clients don't retry on 500 because elasticsearch still misuses it instead of 400 in some places
+        return statusLine.getStatusCode() >= 502 && statusLine.getStatusCode() <= 504;
+    }
+
+    public Node getNode() {
+        return node;
+    }
+
+    public RequestLine getRequestLine() {
+        return requestLine;
+    }
+
+    public StatusLine getStatusLine() {
+        return statusLine;
+    }
+}
diff --git a/client/src/main/java/org/elasticsearch/client/HttpDeleteWithEntity.java b/client/src/main/java/org/elasticsearch/client/HttpDeleteWithEntity.java
new file mode 100644
index 00000000000..4f378f19c8f
--- /dev/null
+++ b/client/src/main/java/org/elasticsearch/client/HttpDeleteWithEntity.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client;
+
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
+
+import java.net.URI;
+
+/**
+ * Allows to send DELETE requests providing a body (not supported out of the box)
+ */
+final class HttpDeleteWithEntity extends HttpEntityEnclosingRequestBase {
+
+    final static String METHOD_NAME = HttpDelete.METHOD_NAME;
+
+    public HttpDeleteWithEntity(final URI uri) {
+        setURI(uri);
+    }
+
+    @Override
+    public String getMethod() {
+        return METHOD_NAME;
+    }
+}
diff --git a/client/src/main/java/org/elasticsearch/client/HttpGetWithEntity.java b/client/src/main/java/org/elasticsearch/client/HttpGetWithEntity.java
new file mode 100644
index 00000000000..18039ba9b34
--- /dev/null
+++ b/client/src/main/java/org/elasticsearch/client/HttpGetWithEntity.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.client;
+
+import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
+import org.apache.http.client.methods.HttpGet;
+
+import java.net.URI;
+
+/**
+ * Allows to send GET requests providing a body (not supported out of the box)
+ */
+final class HttpGetWithEntity extends HttpEntityEnclosingRequestBase {
+
+    final static String METHOD_NAME = HttpGet.METHOD_NAME;
+
+    public HttpGetWithEntity(final URI uri) {
+        setURI(uri);
+    }
+
+    @Override
+    public String getMethod() {
+        return METHOD_NAME;
+    }
+}
\ No newline at end of file
diff --git a/client/src/main/java/org/elasticsearch/client/IndexClient.java b/client/src/main/java/org/elasticsearch/client/IndexClient.java
deleted file mode 100644
index 6b116b35993..00000000000
--- a/client/src/main/java/org/elasticsearch/client/IndexClient.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.client;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-
-public class IndexClient {
-
-    private final RestClient client;
-
-    public IndexClient(RestClient client) {
-        this.client = client;
-    }
-
-    public void delete(String index, String type, String id) throws IOException {
-        delete(index, type, id, null);
-    }
-    public void delete(String index, String type, String id, DeleteOptions params) throws IOException {
-        Objects.requireNonNull(index, "index must not be null");
-        Objects.requireNonNull(type, "type must not be null");
-        Objects.requireNonNull(id, "id must not be null");
-        String deleteEndpoint = String.format(Locale.ROOT, "/%s/%s/%s", index, type, id);
-        client.httpDelete(deleteEndpoint, params == null ? Collections.emptyMap() : params.options);
-    }
-
-    public class DeleteOptions {
-        private final Map<String, Object> options = new HashMap<>();
-        /** Specific write consistency setting for the operation one of "one", "quorum", "all"*/
-        public void consistency(String consistency) {
-            options.put("consistency", consistency);
-        };
-        /** ID of parent document */
-        public void parent(String parent){
-            options.put("parent", parent);
-        };
-        /** Refresh the index after performing the operation */
-        public void refresh(Boolean refresh) {
-            options.put("refresh", refresh);
-        };
-        /** Specific routing value */
-        public void routing(String routing) {
-            options.put("routing", routing);
-        };
-        /** Explicit version number for concurrency control */
-        public void version(Number version) {
-            options.put("version", version);
-        };
-        /** Specific version type one of "internal", "external", "external_gte", "force" */
-        public void versionType(String versionType) {
-            options.put("version_type", versionType);
-        };
-        /** Explicit operation timeout */
-        public void timeout(String timeout) {
-            options.put("timeout", timeout);
-        };
-    }
-}
diff --git a/client/src/main/java/org/elasticsearch/client/Node.java b/client/src/main/java/org/elasticsearch/client/Node.java
new file mode 100644
index 00000000000..ab6d3631527
--- /dev/null
+++ b/client/src/main/java/org/elasticsearch/client/Node.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.HttpHost;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Represents an Elasticsearch node.
+ * Holds its http address as an {@link HttpHost} instance, as well as its optional set of roles and attributes.
+ * Roles and attributes can be populated in one of the two following ways:
+ * 1) using a connection pool that supports sniffing, so that all the info is retrieved from elasticsearch itself
+ * 2) manually passing the info through the {@link #Node(HttpHost, Set, Map)} constructor
+ * Roles and attributes may be taken into account as part of connection selection by the connection pool, which
+ * can be customized by passing in a predicate at connection pool creation.
+ */
+public class Node {
+    private final HttpHost httpHost;
+    private final Set<Role> roles;
+    private final Map<String, String> attributes;
+
+    /**
+     * Creates a node given its http address as an {@link HttpHost} instance.
+     * Roles are not provided hence all possible roles will be assumed, as that is the default in Elasticsearch.
+     * No attributes will be associated with the node.
+     *
+     * @param httpHost the http address of the node
+     */
+    public Node(HttpHost httpHost) {
+        this(httpHost, new HashSet<>(Arrays.asList(Role.values())), Collections.emptyMap());
+    }
+
+    /**
+     * Creates a node given its http address as an {@link HttpHost} instance, its set or roles and attributes.
+     *
+     * @param httpHost the http address of the node
+     * @param roles the set of roles that the node fulfills within the cluster
+     * @param attributes the attributes associated with the node
+     */
+    public Node(HttpHost httpHost, Set<Role> roles, Map<String, String> attributes) {
+        Objects.requireNonNull(httpHost, "host cannot be null");
+        Objects.requireNonNull(roles, "roles cannot be null");
+        Objects.requireNonNull(attributes, "attributes cannot be null");
+        this.httpHost = httpHost;
+        this.roles = Collections.unmodifiableSet(roles);
+        this.attributes = Collections.unmodifiableMap(attributes);
+    }
+
+    /**
+     * Returns the http address of the node
+     */
+    public HttpHost getHttpHost() {
+        return httpHost;
+    }
+
+    /**
+     * Returns the set of roles associated with the node
+     */
+    public Set<Role> getRoles() {
+        return roles;
+    }
+
+    /**
+     * Returns the set of attributes associated with the node
+     */
+    public Map<String, String> getAttributes() {
+        return attributes;
+    }
+
+    @Override
+    public String toString() {
+        return "Node{" +
+                "httpHost=" + httpHost +
+                ", roles=" + roles +
+                ", attributes=" + attributes +
+                '}';
+    }
+
+    /**
+     * Holds all the potential roles that a node can fulfill within a cluster
+     */
+    public enum Role {
+        /**
+         * Data node
+         */
+        DATA,
+        /**
+         * Master eligible node
+         */
+        MASTER,
+        /**
+         * Ingest node
+         */
+        INGEST;
+
+        @Override
+        public String toString() {
+            return name().toLowerCase(Locale.ROOT);
+        }
+    }
+}
diff --git a/client/src/main/java/org/elasticsearch/client/RequestLogger.java b/client/src/main/java/org/elasticsearch/client/RequestLogger.java
new file mode 100644
index 00000000000..8423dcb8ade
--- /dev/null
+++ b/client/src/main/java/org/elasticsearch/client/RequestLogger.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.commons.logging.Log;
+import org.apache.http.RequestLine;
+import org.apache.http.StatusLine;
+
+import java.io.IOException;
+
+/**
+ * Helper class that exposes static method to unify the way requests are logged
+ */
+public final class RequestLogger {
+
+    private RequestLogger() {
+    }
+
+    /**
+     * Logs a request that yielded a response
+     */
+    public static void log(Log logger, String message, RequestLine requestLine, Node node, StatusLine statusLine) {
+        logger.debug(message + " [" + requestLine.getMethod() + " " + node.getHttpHost() +
+                requestLine.getUri() + "] [" + statusLine + "]");
+    }
+
+    /**
+     * Logs a request that failed
+     */
+    public static void log(Log logger, String message, RequestLine requestLine, Node node, IOException e) {
+        logger.debug(message + " [" + requestLine.getMethod() + " " + node.getHttpHost() +
+                requestLine.getUri() + "]", e);
+    }
+}
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 25d96aa903d..a9150119667 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -19,200 +19,27 @@
 package org.elasticsearch.client;
 
 import org.apache.http.HttpEntity;
-import org.apache.http.HttpHost;
-import org.apache.http.HttpResponse;
-import org.apache.http.NameValuePair;
-import org.apache.http.StatusLine;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.HttpDelete;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.client.utils.URLEncodedUtils;
 import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.message.BasicNameValuePair;
 
-import java.io.BufferedReader;
 import java.io.Closeable;
 import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.CopyOnWriteArraySet;
-import java.util.stream.Collectors;
 
-/**
- */
-public class RestClient implements Closeable{
+public final class RestClient implements Closeable {
 
-    private final CloseableHttpClient client;
-    private volatile Set<HttpHost> hosts;
-    private final String scheme;
-    private final Set<HttpHost> blackList = new CopyOnWriteArraySet<>();
+    private final Transport transport;
 
-    public RestClient(HttpHost... hosts) {
-        this("http", HttpClientBuilder.create().setDefaultRequestConfig(
-                RequestConfig.custom().setConnectTimeout(100).build()).build(), hosts);
+    public RestClient(CloseableHttpClient client, ConnectionPool<? extends Connection> connectionPool, long maxRetryTimeout) {
+        this.transport = new Transport<>(client, connectionPool, maxRetryTimeout);
     }
 
-    public RestClient(String scheme, CloseableHttpClient client, HttpHost[] hosts) {
-        if (hosts.length == 0) {
-            throw new IllegalArgumentException("hosts must note be empty");
-        }
-        this.scheme = scheme;
-        this.client = client;
-        this.hosts = new HashSet<>(Arrays.asList(hosts));
+    public ElasticsearchResponse performRequest(Verb verb, String endpoint, Map<String, Object> params, HttpEntity entity)
+            throws IOException {
+        return transport.performRequest(verb, endpoint, params, entity);
     }
 
-
-    public HttpResponse httpGet(String endpoint, Map<String, Object> params) throws IOException {
-        return httpGet(getHostIterator(true), endpoint, params);
-    }
-
-    HttpResponse httpGet(Iterable<HttpHost> hosts, String endpoint, Map<String, Object> params) throws IOException {
-        HttpUriRequest request = new HttpGet(buildUri(endpoint, pairs(params)));
-        return execute(request, hosts);
-    }
-
-    HttpResponse httpDelete(String endpoint, Map<String, Object> params) throws IOException {
-        HttpUriRequest request = new HttpDelete(buildUri(endpoint, pairs(params)));
-        return execute(request, getHostIterator(true));
-    }
-
-    HttpResponse httpPut(String endpoint, HttpEntity body, Map<String, Object>  params) throws IOException {
-        HttpPut request = new HttpPut(buildUri(endpoint, pairs(params)));
-        request.setEntity(body);
-        return execute(request, getHostIterator(true));
-    }
-
-    HttpResponse httpPost(String endpoint, HttpEntity body, Map<String, Object> params) throws IOException {
-        HttpPost request = new HttpPost(buildUri(endpoint, pairs(params)));
-        request.setEntity(body);
-        return execute(request, getHostIterator(true));
-    }
-
-    private List<NameValuePair> pairs(Map<String, Object> options) {
-        return options.entrySet().stream().map(e -> new BasicNameValuePair(e.getKey(), e.getValue().toString()))
-            .collect(Collectors.toList());
-    }
-
-    public HttpResponse execute(HttpUriRequest request, Iterable<HttpHost> retryHosts) throws IOException {
-        IOException exception = null;
-        for (HttpHost singleHost : retryHosts) {
-            try {
-                return client.execute(singleHost, request);
-            } catch (IOException ex) {
-                if (this.hosts.contains(singleHost)) {
-                    blackList.add(singleHost);
-                }
-                if (exception != null) {
-                    exception.addSuppressed(ex);
-                } else {
-                    exception = ex;
-                }
-            }
-        }
-        throw exception;
-    }
-
-    public URI buildUri(String path, List<NameValuePair> query) {
-        try {
-            return new URI(null, null, null, -1, path, URLEncodedUtils.format(query, StandardCharsets.UTF_8), null);
-        } catch (URISyntaxException e) {
-            throw new RuntimeException(e.getMessage(), e);
-        }
-    }
-
-    public Set<HttpHost> fetchNodes(HttpHost host, boolean useClientNodes, boolean local, boolean checkAvailable) throws IOException {
-        HttpResponse httpResponse = httpGet(Collections.singleton(host), "/_cat/nodes", Collections.singletonMap("h", "http,role"));
-        StatusLine statusLine = httpResponse.getStatusLine();
-        if (statusLine.getStatusCode() != 200) {
-            throw new RuntimeException("failed to fetch nodes: " + statusLine.getReasonPhrase());
-        }
-        HttpEntity entity = httpResponse.getEntity();
-        Set<HttpHost> hosts = new HashSet<>();
-        try (BufferedReader content = new BufferedReader(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8))) {
-            String line;
-            while((line = content.readLine()) != null) {
-                final String[] split = line.split("\\s+");
-                assert split.length == 2;
-                String boundAddress = split[0];
-                String role = split[1];
-                if ("-".equals(split[0].trim()) == false) {
-                    if ("d".equals(role.trim()) == false && useClientNodes == false) {
-                        continue;
-                    }
-                    URI boundAddressAsURI = URI.create("http://" + boundAddress);
-                    HttpHost newHost = new HttpHost(boundAddressAsURI.getHost(), boundAddressAsURI.getPort(), scheme);
-                    if (checkAvailable == false || isAvailable(newHost)) {
-                        hosts.add(newHost);
-                    }
-                }
-            }
-        }
-        return hosts;
-    }
-
-    public String getClusterName(HttpHost host) throws IOException {
-        HttpResponse httpResponse = httpGet(Collections.singleton(host), "/_cat/health", Collections.singletonMap("h", "cluster"));
-        StatusLine statusLine = httpResponse.getStatusLine();
-        if (statusLine.getStatusCode() != 200) {
-            throw new RuntimeException("failed to fetch nodes: " + statusLine.getReasonPhrase());
-        }
-        HttpEntity entity = httpResponse.getEntity();
-        try (BufferedReader content = new BufferedReader(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8))) {
-            String clusterName = content.readLine().trim();
-            if (clusterName.length() == 0) {
-                throw new IllegalStateException("clustername must not be empty");
-            }
-            return clusterName;
-        }
-    }
-
-    public boolean isAvailable(HttpHost host) {
-        try {
-            HttpResponse httpResponse = httpGet(Collections.singleton(host), "/", Collections.emptyMap());
-            StatusLine statusLine = httpResponse.getStatusLine();
-            return statusLine.getStatusCode() == 200;
-        } catch (IOException ex) {
-            return false;
-        }
-    }
-
-    public synchronized void setNodes(Set<HttpHost> hosts) {
-        this.hosts = Collections.unmodifiableSet(new HashSet<>(hosts));
-        blackList.retainAll(hosts);
-    }
-
-    public Set<HttpHost> getHosts() {
-        return hosts;
-    }
-
-    protected Iterable<HttpHost> getHostIterator(boolean clearBlacklist) {
-        if (hosts.size() == blackList.size() && clearBlacklist) {
-            blackList.clear(); // lets try again
-        }
-        return () -> hosts.stream().filter((h) -> blackList.contains(h) == false).iterator();
-    }
-
-    int getNumHosts() {
-        return hosts.size();
-    }
-
-    int getNumBlacklistedHosts() {
-        return blackList.size();
-    }
     @Override
     public void close() throws IOException {
-        client.close();
+        transport.close();
     }
 }
diff --git a/client/src/main/java/org/elasticsearch/client/RetryTimeoutException.java b/client/src/main/java/org/elasticsearch/client/RetryTimeoutException.java
new file mode 100644
index 00000000000..632597bfa37
--- /dev/null
+++ b/client/src/main/java/org/elasticsearch/client/RetryTimeoutException.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import java.io.IOException;
+
+public class RetryTimeoutException extends IOException {
+
+    RetryTimeoutException(String message) {
+        super(message);
+    }
+}
diff --git a/client/src/main/java/org/elasticsearch/client/Sniffer.java b/client/src/main/java/org/elasticsearch/client/Sniffer.java
new file mode 100644
index 00000000000..07d485de879
--- /dev/null
+++ b/client/src/main/java/org/elasticsearch/client/Sniffer.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
+import org.apache.http.StatusLine;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.util.EntityUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+final class Sniffer {
+
+    private static final Log logger = LogFactory.getLog(Sniffer.class);
+
+    private final CloseableHttpClient client;
+    private final RequestConfig sniffRequestConfig;
+    private final int sniffRequestTimeout;
+    private final String scheme;
+    private final JsonFactory jsonFactory;
+
+    Sniffer(CloseableHttpClient client, RequestConfig sniffRequestConfig, int sniffRequestTimeout, String scheme) {
+        Objects.requireNonNull(client, "client cannot be null");
+        Objects.requireNonNull(sniffRequestConfig, "sniffRequestConfig cannot be null");
+        if (sniffRequestTimeout <=0) {
+            throw new IllegalArgumentException("sniffRequestTimeout must be greater than 0");
+        }
+        Objects.requireNonNull(scheme, "scheme cannot be null");
+        this.client = client;
+        this.sniffRequestConfig = sniffRequestConfig;
+        this.sniffRequestTimeout = sniffRequestTimeout;
+        this.scheme = scheme;
+        this.jsonFactory = new JsonFactory();
+    }
+
+    List<Node> sniffNodes(Node node) throws IOException {
+        HttpGet httpGet = new HttpGet("/_nodes/http?timeout=" + sniffRequestTimeout + "ms");
+        httpGet.setConfig(sniffRequestConfig);
+
+        try (CloseableHttpResponse response = client.execute(node.getHttpHost(), httpGet)) {
+            StatusLine statusLine = response.getStatusLine();
+            if (statusLine.getStatusCode() >= 300) {
+                RequestLogger.log(logger, "sniff failed", httpGet.getRequestLine(), node, statusLine);
+                EntityUtils.consume(response.getEntity());
+                throw new ElasticsearchResponseException(httpGet.getRequestLine(), node, statusLine);
+            } else {
+                List<Node> nodes = readNodes(response.getEntity());
+                RequestLogger.log(logger, "sniff succeeded", httpGet.getRequestLine(), node, statusLine);
+                return nodes;
+            }
+        } catch(IOException e) {
+            RequestLogger.log(logger, "sniff failed", httpGet.getRequestLine(), node, e);
+            throw e;
+        }
+    }
+
+    private List<Node> readNodes(HttpEntity entity) throws IOException {
+        try (InputStream inputStream = entity.getContent()) {
+            JsonParser parser = jsonFactory.createParser(inputStream);
+            if (parser.nextToken() != JsonToken.START_OBJECT) {
+                throw new IOException("expected data to start with an object");
+            }
+            List<Node> nodes = new ArrayList<>();
+            while (parser.nextToken() != JsonToken.END_OBJECT) {
+                if (parser.getCurrentToken() == JsonToken.START_OBJECT) {
+                    if ("nodes".equals(parser.getCurrentName())) {
+                        while (parser.nextToken() != JsonToken.END_OBJECT) {
+                            JsonToken token = parser.nextToken();
+                            assert token == JsonToken.START_OBJECT;
+                            String nodeId = parser.getCurrentName();
+                            Node sniffedNode = readNode(nodeId, parser, this.scheme);
+                            if (sniffedNode != null) {
+                                logger.trace("adding node [" + nodeId + "]");
+                                nodes.add(sniffedNode);
+                            }
+                        }
+                    } else {
+                        parser.skipChildren();
+                    }
+                }
+            }
+            return nodes;
+        }
+    }
+
+    private static Node readNode(String nodeId, JsonParser parser, String scheme) throws IOException {
+        HttpHost httpHost = null;
+        Set<Node.Role> roles = new HashSet<>();
+        Map<String, String> attributes = new HashMap<>();
+        String fieldName = null;
+        while (parser.nextToken() != JsonToken.END_OBJECT) {
+            if (parser.getCurrentToken() == JsonToken.FIELD_NAME) {
+                fieldName = parser.getCurrentName();
+            } else if (parser.getCurrentToken() == JsonToken.START_ARRAY && "roles".equals(fieldName)) {
+                while (parser.nextToken() != JsonToken.END_ARRAY) {
+                    roles.add(Node.Role.valueOf(parser.getValueAsString().toUpperCase(Locale.ROOT)));
+                }
+            } else if (parser.getCurrentToken() == JsonToken.START_OBJECT) {
+                if ("attributes".equals(fieldName)) {
+                    while (parser.nextToken() != JsonToken.END_OBJECT) {
+                        attributes.put(parser.getCurrentName(), parser.getValueAsString());
+                    }
+                } else if ("http".equals(fieldName)) {
+                    while (parser.nextToken() != JsonToken.END_OBJECT) {
+                        if (parser.getCurrentToken() == JsonToken.VALUE_STRING &&
+                                "publish_address".equals(parser.getCurrentName())) {
+                            URI boundAddressAsURI = URI.create(scheme + "://" + parser.getValueAsString());
+                            httpHost = new HttpHost(boundAddressAsURI.getHost(), boundAddressAsURI.getPort(),
+                                    boundAddressAsURI.getScheme());
+                        } else if (parser.getCurrentToken() == JsonToken.START_OBJECT) {
+                            parser.skipChildren();
+                        }
+                    }
+                } else {
+                    parser.skipChildren();
+                }
+            }
+        }
+        //http section is not present if http is not enabled on the node, ignore such nodes
+        if (httpHost == null) {
+            logger.debug("skipping node [" + nodeId + "] with http disabled");
+            return null;
+        }
+        return new Node(httpHost, roles, attributes);
+    }
+}
diff --git a/client/src/main/java/org/elasticsearch/client/SniffingConnectionPool.java b/client/src/main/java/org/elasticsearch/client/SniffingConnectionPool.java
new file mode 100644
index 00000000000..0488033c1ca
--- /dev/null
+++ b/client/src/main/java/org/elasticsearch/client/SniffingConnectionPool.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.impl.client.CloseableHttpClient;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Objects;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Predicate;
+import java.util.stream.Stream;
+
+public class SniffingConnectionPool extends AbstractStaticConnectionPool {
+
+    private static final Log logger = LogFactory.getLog(SniffingConnectionPool.class);
+
+    private final boolean sniffOnFailure;
+    private final Sniffer sniffer;
+    private volatile List<StatefulConnection> connections;
+    private final SnifferTask snifferTask;
+
+    public SniffingConnectionPool(int sniffInterval, boolean sniffOnFailure, int sniffAfterFailureDelay,
+                                  CloseableHttpClient client, RequestConfig sniffRequestConfig, int sniffRequestTimeout, Scheme scheme,
+                                  Predicate<Connection> connectionSelector, Node... nodes) {
+        super(connectionSelector);
+        if (sniffInterval <= 0) {
+            throw new IllegalArgumentException("sniffInterval must be greater than 0");
+        }
+        if (sniffAfterFailureDelay <= 0) {
+            throw new IllegalArgumentException("sniffAfterFailureDelay must be greater than 0");
+        }
+        Objects.requireNonNull(scheme, "scheme cannot be null");
+        if (nodes == null || nodes.length == 0) {
+            throw new IllegalArgumentException("no nodes provided");
+        }
+        this.sniffOnFailure = sniffOnFailure;
+        this.sniffer = new Sniffer(client, sniffRequestConfig, sniffRequestTimeout, scheme.toString());
+        this.connections = createConnections(nodes);
+        this.snifferTask = new SnifferTask(sniffInterval, sniffAfterFailureDelay);
+    }
+
+    @Override
+    protected List<StatefulConnection> getConnections() {
+        return this.connections;
+    }
+
+    @Override
+    public void beforeAttempt(StatefulConnection connection) throws IOException {
+
+    }
+
+    @Override
+    public void onFailure(StatefulConnection connection) throws IOException {
+        super.onFailure(connection);
+        if (sniffOnFailure) {
+            //re-sniff immediately but take out the node that failed
+            snifferTask.sniffOnFailure(connection.getNode());
+        }
+    }
+
+    @Override
+    public void close() throws IOException {
+        snifferTask.shutdown();
+    }
+
+    public enum Scheme {
+        HTTP, HTTPS;
+
+        @Override
+        public String toString() {
+            return name().toLowerCase(Locale.ROOT);
+        }
+    }
+
+    private class SnifferTask implements Runnable {
+        private final int sniffInterval;
+        private final int sniffAfterFailureDelay;
+        private final ScheduledExecutorService scheduledExecutorService;
+        private final AtomicBoolean running = new AtomicBoolean(false);
+        private volatile boolean failure = false;
+        private volatile ScheduledFuture<?> scheduledFuture;
+
+        private SnifferTask(int sniffInterval, int sniffAfterFailureDelay) {
+            this.sniffInterval = sniffInterval;
+            this.sniffAfterFailureDelay = sniffAfterFailureDelay;
+            this.scheduledExecutorService = Executors.newScheduledThreadPool(1);
+            this.scheduledFuture = this.scheduledExecutorService.schedule(this, 0, TimeUnit.MILLISECONDS);
+        }
+
+        @Override
+        public void run() {
+            sniff(node -> true);
+        }
+
+        void sniffOnFailure(Node failedNode) {
+            //sync sniff straightaway on failure
+            failure = true;
+            sniff(node -> node.getHttpHost().equals(failedNode.getHttpHost()) == false);
+        }
+
+        void sniff(Predicate<Node> nodeFilter) {
+            if (running.compareAndSet(false, true)) {
+                try {
+                    Iterator<StatefulConnection> connectionIterator = nextUnfilteredConnection().iterator();
+                    if (connectionIterator.hasNext()) {
+                        sniff(connectionIterator, nodeFilter);
+                    } else {
+                        StatefulConnection connection = lastResortConnection();
+                        logger.info("no healthy nodes available, trying " + connection.getNode());
+                        sniff(Stream.of(connection).iterator(), nodeFilter);
+                    }
+                } catch (Throwable t) {
+                    logger.error("error while sniffing nodes", t);
+                } finally {
+                    try {
+                        //regardless of whether and when the next sniff is scheduled, cancel it and schedule a new one with updated delay
+                        this.scheduledFuture.cancel(false);
+                        if (this.failure) {
+                            this.scheduledFuture = this.scheduledExecutorService.schedule(this,
+                                    sniffAfterFailureDelay, TimeUnit.MILLISECONDS);
+                            this.failure = false;
+                        } else {
+                            this.scheduledFuture = this.scheduledExecutorService.schedule(this, sniffInterval, TimeUnit.MILLISECONDS);
+                        }
+                    } catch (Throwable t) {
+                        logger.error("error while scheduling next sniffer task", t);
+                    } finally {
+                        running.set(false);
+                    }
+                }
+            }
+        }
+
+        void sniff(Iterator<StatefulConnection> connectionIterator, Predicate<Node> nodeFilter) throws IOException {
+            IOException lastSeenException = null;
+            while (connectionIterator.hasNext()) {
+                StatefulConnection connection = connectionIterator.next();
+                try {
+                    List<Node> sniffedNodes = sniffer.sniffNodes(connection.getNode());
+                    Node[] filteredNodes = sniffedNodes.stream().filter(nodeFilter).toArray(Node[]::new);
+                    logger.debug("adding " + filteredNodes.length + " nodes out of " + sniffedNodes.size() + " sniffed nodes");
+                    connections = createConnections(filteredNodes);
+                    onSuccess(connection);
+                    return;
+                } catch (IOException e) {
+                    //here we have control over the request, if it fails something is really wrong, always call onFailure
+                    onFailure(connection);
+                    if (lastSeenException != null) {
+                        e.addSuppressed(lastSeenException);
+                    }
+                    lastSeenException = e;
+                }
+            }
+            logger.warn("failed to sniff nodes", lastSeenException);
+        }
+
+        void shutdown() {
+            scheduledExecutorService.shutdown();
+            try {
+                if (scheduledExecutorService.awaitTermination(1000, TimeUnit.MILLISECONDS)) {
+                    return;
+                }
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+            scheduledExecutorService.shutdownNow();
+        }
+    }
+}
diff --git a/client/src/main/java/org/elasticsearch/client/StatefulConnection.java b/client/src/main/java/org/elasticsearch/client/StatefulConnection.java
new file mode 100644
index 00000000000..1ac72382e19
--- /dev/null
+++ b/client/src/main/java/org/elasticsearch/client/StatefulConnection.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * {@link Connection} subclass that has a mutable state, based on previous usage.
+ * When first created, a connection is in unknown state, till it is used for the first time
+ * and marked either dead or alive based on the outcome of the first usage.
+ * Should be marked alive when properly working.
+ * Should be marked dead when it caused a failure, in which case the connection may be retried some time later,
+ * as soon as {@link #shouldBeRetried()} returns true, which depends on how many consecutive failed attempts
+ * were counted and when the last one was registered.
+ * Should be marked resurrected if in dead state, as last resort in case there are no live connections available
+ * and none of the dead ones are ready to be retried yet. When marked resurrected, the number of failed attempts
+ * and its timeout is not reset so that if it gets marked dead again it returns to the exact state before resurrection.
+ */
+public final class StatefulConnection extends Connection {
+    //TODO make these values configurable through the connection pool?
+    private static final long DEFAULT_CONNECTION_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(1);
+    private static final long MAX_CONNECTION_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(30);
+
+    private volatile State state = State.UNKNOWN;
+    private volatile int failedAttempts = -1;
+    private volatile long deadUntil = -1;
+
+    /**
+     * Creates a new mutable connection pointing to the provided {@link Node} argument
+     */
+    public StatefulConnection(Node node) {
+        super(node);
+    }
+
+    /**
+     * Marks connection as dead. Should be called in case the corresponding node is not responding or caused failures.
+     * Once marked dead, the number of failed attempts will be incremented on each call to this method. A dead connection
+     * should be retried once {@link #shouldBeRetried()} returns true, which depends on the number of previous failed attempts
+     * and when the last failure was registered.
+     */
+    void markDead() {
+        synchronized (this) {
+            int failedAttempts = Math.max(this.failedAttempts, 0);
+            long timeoutMillis = (long)Math.min(DEFAULT_CONNECTION_TIMEOUT_MILLIS * 2 * Math.pow(2, failedAttempts * 0.5 - 1),
+                    MAX_CONNECTION_TIMEOUT_MILLIS);
+            this.deadUntil = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis);
+            this.failedAttempts = ++failedAttempts;
+            this.state = State.DEAD;
+        }
+    }
+
+    /**
+     * Marks this connection alive. Should be called when the corresponding node is working properly.
+     * Will reset the number of failed attempts that were counted in case the connection was previously dead,
+     * as well as its dead timeout.
+     */
+    void markAlive() {
+        if (this.state != State.ALIVE) {
+            synchronized (this) {
+                this.deadUntil = -1;
+                this.failedAttempts = 0;
+                this.state = State.ALIVE;
+            }
+        }
+    }
+
+    /**
+     * Resets the connection to its initial state, so it will be retried. To be called when all the connections in the pool
+     * are dead, so that one connection can be retried. Note that calling this method only changes the state of the connection,
+     * it doesn't reset its failed attempts and dead until timestamp. That way if the connection goes back to dead straightaway
+     * all of its previous failed attempts are taken into account.
+     */
+    void markResurrected() {
+        if (this.state == State.DEAD) {
+            synchronized (this) {
+                this.state = State.UNKNOWN;
+            }
+        }
+    }
+
+    /**
+     * Returns the timestamp till the connection is supposed to stay dead till it can be retried
+     */
+    public long getDeadUntil() {
+        return deadUntil;
+    }
+
+    /**
+     * Returns true if the connection is alive, false otherwise.
+     */
+    public boolean isAlive() {
+        return state == State.ALIVE;
+    }
+
+    /**
+     * Returns true in case the connection is not alive but should be used/retried, false otherwise.
+     * Returns true in case the connection is in unknown state (never used before) or resurrected. When the connection is dead,
+     * returns true when it is time to retry it, depending on how many failed attempts were registered and when the last failure
+     * happened (minimum 1 minute, maximum 30 minutes).
+     */
+    public boolean shouldBeRetried() {
+        return state == State.UNKNOWN || (state == State.DEAD && System.nanoTime() - deadUntil >= 0);
+    }
+
+    private enum State {
+        UNKNOWN, DEAD, ALIVE
+    }
+}
diff --git a/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java b/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
new file mode 100644
index 00000000000..5fb32341c99
--- /dev/null
+++ b/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.http.StatusLine;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpHead;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.util.EntityUtils;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Predicate;
+
+public class StaticConnectionPool extends AbstractStaticConnectionPool {
+
+    private static final Log logger = LogFactory.getLog(StaticConnectionPool.class);
+
+    private final CloseableHttpClient client;
+    private final boolean pingEnabled;
+    private final RequestConfig pingRequestConfig;
+    private final List<StatefulConnection> connections;
+
+    public StaticConnectionPool(CloseableHttpClient client, boolean pingEnabled, RequestConfig pingRequestConfig,
+                                Predicate<Connection> connectionSelector, Node... nodes) {
+        super(connectionSelector);
+        Objects.requireNonNull(client, "client cannot be null");
+        Objects.requireNonNull(pingRequestConfig, "pingRequestConfig cannot be null");
+        if (nodes == null || nodes.length == 0) {
+            throw new IllegalArgumentException("no nodes provided");
+        }
+        this.client = client;
+        this.pingEnabled = pingEnabled;
+        this.pingRequestConfig = pingRequestConfig;
+        this.connections = createConnections(nodes);
+    }
+
+    @Override
+    protected List<StatefulConnection> getConnections() {
+        return connections;
+    }
+
+    @Override
+    public void beforeAttempt(StatefulConnection connection) throws IOException {
+        if (pingEnabled && connection.shouldBeRetried()) {
+            HttpHead httpHead = new HttpHead("/");
+            httpHead.setConfig(pingRequestConfig);
+            StatusLine statusLine;
+            try(CloseableHttpResponse httpResponse = client.execute(connection.getNode().getHttpHost(), httpHead)) {
+                statusLine = httpResponse.getStatusLine();
+                EntityUtils.consume(httpResponse.getEntity());
+            } catch(IOException e) {
+                RequestLogger.log(logger, "ping failed", httpHead.getRequestLine(), connection.getNode(), e);
+                onFailure(connection);
+                throw e;
+            }
+            if (statusLine.getStatusCode() >= 300) {
+                RequestLogger.log(logger, "ping failed", httpHead.getRequestLine(), connection.getNode(), statusLine);
+                onFailure(connection);
+                throw new ElasticsearchResponseException(httpHead.getRequestLine(), connection.getNode(), statusLine);
+            } else {
+                RequestLogger.log(logger, "ping succeeded", httpHead.getRequestLine(), connection.getNode(), statusLine);
+                onSuccess(connection);
+            }
+        }
+    }
+
+    @Override
+    public void close() throws IOException {
+        //no-op nothing to close
+    }
+}
diff --git a/client/src/main/java/org/elasticsearch/client/Transport.java b/client/src/main/java/org/elasticsearch/client/Transport.java
new file mode 100644
index 00000000000..40f68ab61cc
--- /dev/null
+++ b/client/src/main/java/org/elasticsearch/client/Transport.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.http.HttpEntity;
+import org.apache.http.StatusLine;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
+import org.apache.http.client.methods.HttpHead;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.util.EntityUtils;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
+
+final class Transport<C extends Connection> implements Closeable {
+
+    private static final Log logger = LogFactory.getLog(Transport.class);
+
+    private final CloseableHttpClient client;
+    private final ConnectionPool<C> connectionPool;
+    private final long maxRetryTimeout;
+
+    Transport(CloseableHttpClient client, ConnectionPool<C> connectionPool, long maxRetryTimeout) {
+        Objects.requireNonNull(client, "client cannot be null");
+        Objects.requireNonNull(connectionPool, "connectionPool cannot be null");
+        if (maxRetryTimeout <= 0) {
+            throw new IllegalArgumentException("maxRetryTimeout must be greater than 0");
+        }
+        this.client = client;
+        this.connectionPool = connectionPool;
+        this.maxRetryTimeout = maxRetryTimeout;
+    }
+
+    ElasticsearchResponse performRequest(Verb verb, String endpoint, Map<String, Object> params, HttpEntity entity) throws IOException {
+        URI uri = buildUri(endpoint, params);
+        HttpRequestBase request = createHttpRequest(verb, uri, entity);
+        Iterator<C> connectionIterator = connectionPool.nextConnection().iterator();
+        if (connectionIterator.hasNext() == false) {
+            C connection = connectionPool.lastResortConnection();
+            logger.info("no healthy nodes available, trying " + connection.getNode());
+            return performRequest(request, Stream.of(connection).iterator());
+        }
+        return performRequest(request, connectionIterator);
+    }
+
+    private ElasticsearchResponse performRequest(HttpRequestBase request, Iterator<C> connectionIterator) throws IOException {
+        //we apply a soft margin so that e.g. if a request took 59 seconds and timeout is set to 60 we don't do another attempt
+        long retryTimeout = Math.round(this.maxRetryTimeout / (float)100 * 98);
+        IOException lastSeenException = null;
+        long startTime = System.nanoTime();
+
+        while (connectionIterator.hasNext()) {
+            C connection = connectionIterator.next();
+
+            if (lastSeenException != null) {
+                long timeElapsed = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
+                long timeout = retryTimeout - timeElapsed;
+                if (timeout <= 0) {
+                    RetryTimeoutException retryTimeoutException = new RetryTimeoutException(
+                            "request retries exceeded max retry timeout [" + retryTimeout + "]");
+                    retryTimeoutException.addSuppressed(lastSeenException);
+                    throw retryTimeoutException;
+                }
+            }
+
+            try {
+                connectionPool.beforeAttempt(connection);
+            } catch(IOException e) {
+                lastSeenException = addSuppressedException(lastSeenException, e);
+                continue;
+            }
+
+            try {
+                ElasticsearchResponse response = performRequest(request, connection);
+                connectionPool.onSuccess(connection);
+                return response;
+            } catch(ElasticsearchResponseException e) {
+                if (e.isRecoverable()) {
+                    connectionPool.onFailure(connection);
+                    lastSeenException = addSuppressedException(lastSeenException, e);
+                } else {
+                    //don't retry and call onSuccess as the error should be a request problem
+                    connectionPool.onSuccess(connection);
+                    throw e;
+                }
+            } catch(IOException e) {
+                connectionPool.onFailure(connection);
+                lastSeenException = addSuppressedException(lastSeenException, e);
+            }
+        }
+        assert lastSeenException != null;
+        throw lastSeenException;
+    }
+
+    private ElasticsearchResponse performRequest(HttpRequestBase request, C connection) throws IOException {
+        CloseableHttpResponse response;
+        try {
+            response = client.execute(connection.getNode().getHttpHost(), request);
+        } catch(IOException e) {
+            RequestLogger.log(logger, "request failed", request.getRequestLine(), connection.getNode(), e);
+            throw e;
+        } finally {
+            request.reset();
+        }
+        StatusLine statusLine = response.getStatusLine();
+        //TODO make ignore status code configurable. rest-spec and tests support that parameter.
+        if (statusLine.getStatusCode() < 300 ||
+                request.getMethod().equals(HttpHead.METHOD_NAME) && statusLine.getStatusCode() == 404) {
+            RequestLogger.log(logger, "request succeeded", request.getRequestLine(), connection.getNode(), response.getStatusLine());
+            return new ElasticsearchResponse(request.getRequestLine(), connection.getNode(), response);
+        } else {
+            EntityUtils.consume(response.getEntity());
+            RequestLogger.log(logger, "request failed", request.getRequestLine(), connection.getNode(), response.getStatusLine());
+            throw new ElasticsearchResponseException(request.getRequestLine(), connection.getNode(), statusLine);
+        }
+    }
+
+    private static IOException addSuppressedException(IOException suppressedException, IOException currentException) {
+        if (suppressedException != null) {
+            currentException.addSuppressed(suppressedException);
+        }
+        return currentException;
+    }
+
+    private static HttpRequestBase createHttpRequest(Verb verb, URI uri, HttpEntity entity) {
+        switch(verb) {
+            case DELETE:
+                HttpDeleteWithEntity httpDeleteWithEntity = new HttpDeleteWithEntity(uri);
+                addRequestBody(httpDeleteWithEntity, entity);
+                return httpDeleteWithEntity;
+            case GET:
+                HttpGetWithEntity httpGetWithEntity = new HttpGetWithEntity(uri);
+                addRequestBody(httpGetWithEntity, entity);
+                return httpGetWithEntity;
+            case HEAD:
+                if (entity != null) {
+                    throw new UnsupportedOperationException("HEAD with body is not supported");
+                }
+                return new HttpHead(uri);
+            case POST:
+                HttpPost httpPost = new HttpPost(uri);
+                addRequestBody(httpPost, entity);
+                return httpPost;
+            case PUT:
+                HttpPut httpPut = new HttpPut(uri);
+                addRequestBody(httpPut, entity);
+                return httpPut;
+            default:
+                throw new UnsupportedOperationException("http method not supported: " + verb);
+        }
+    }
+
+    private static void addRequestBody(HttpEntityEnclosingRequestBase httpRequest, HttpEntity entity) {
+        if (entity != null) {
+            httpRequest.setEntity(entity);
+        }
+    }
+
+    private static URI buildUri(String path, Map<String, Object> params) {
+        try {
+            URIBuilder uriBuilder = new URIBuilder(path);
+            for (Map.Entry<String, Object> param : params.entrySet()) {
+                uriBuilder.addParameter(param.getKey(), param.getValue().toString());
+            }
+            return uriBuilder.build();
+        } catch(URISyntaxException e) {
+            throw new IllegalArgumentException(e.getMessage(), e);
+        }
+    }
+
+    @Override
+    public void close() throws IOException {
+        connectionPool.close();
+        client.close();
+    }
+}
diff --git a/client/src/main/java/org/elasticsearch/client/Verb.java b/client/src/main/java/org/elasticsearch/client/Verb.java
new file mode 100644
index 00000000000..74a88353977
--- /dev/null
+++ b/client/src/main/java/org/elasticsearch/client/Verb.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+/**
+ * Holds the http verbs/methods supported by elasticsearch, which can be used when sending a request
+ */
+public enum Verb {
+    DELETE, GET, HEAD, POST, PUT
+}
diff --git a/client/src/test/java/org/elasticsearch/client/NodeTests.java b/client/src/test/java/org/elasticsearch/client/NodeTests.java
new file mode 100644
index 00000000000..73bad0f6d7b
--- /dev/null
+++ b/client/src/test/java/org/elasticsearch/client/NodeTests.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+import org.apache.http.HttpHost;
+import org.apache.lucene.util.LuceneTestCase;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.notNullValue;
+import static org.hamcrest.CoreMatchers.sameInstance;
+
+public class NodeTests extends LuceneTestCase {
+
+    public void testSingleArgumentConstructor() {
+        HttpHost httpHost = new HttpHost(randomHost(), randomPort(), randomScheme());
+        Node node = new Node(httpHost);
+        assertThat(node.getHttpHost(), sameInstance(httpHost));
+        assertThat(node.getAttributes(), notNullValue());
+        assertThat(node.getAttributes().size(), equalTo(0));
+        assertThat(node.getRoles(), notNullValue());
+        assertThat(node.getRoles(), equalTo(new HashSet<>(Arrays.asList(Node.Role.values()))));
+
+        try {
+            new Node(null);
+            fail("node construction should have failed");
+        } catch(NullPointerException e) {
+            assertThat(e.getMessage(), equalTo("host cannot be null"));
+        }
+
+    }
+
+    public void testThreeArgumentsConstructor() {
+        HttpHost httpHost = new HttpHost(randomHost(), randomPort(), randomScheme());
+        Set<Node.Role> roles = randomRoles();
+        Map<String, String> attributes = randomAttributes();
+        Node node = new Node(httpHost, roles, attributes);
+        assertThat(node.getHttpHost(), sameInstance(httpHost));
+        assertThat(node.getAttributes(), equalTo(attributes));
+        assertThat(node.getRoles(), equalTo(roles));
+
+        try {
+            new Node(null, roles, attributes);
+            fail("node construction should have failed");
+        } catch(NullPointerException e) {
+            assertThat(e.getMessage(), equalTo("host cannot be null"));
+        }
+
+        try {
+            new Node(httpHost, null, attributes);
+            fail("node construction should have failed");
+        } catch(NullPointerException e) {
+            assertThat(e.getMessage(), equalTo("roles cannot be null"));
+        }
+
+        try {
+            new Node(httpHost, roles, null);
+            fail("node construction should have failed");
+        } catch(NullPointerException e) {
+            assertThat(e.getMessage(), equalTo("attributes cannot be null"));
+        }
+    }
+
+    public void testToString() {
+        HttpHost httpHost = new HttpHost(randomHost(), randomPort(), randomScheme());
+        Set<Node.Role> roles = randomRoles();
+        Map<String, String> attributes = randomAttributes();
+        Node node = new Node(httpHost, roles, attributes);
+        String expectedString = "Node{" +
+                "httpHost=" + httpHost.toString() +
+                ", roles=" + roles.toString() +
+                ", attributes=" + attributes.toString() +
+                '}';
+        assertThat(node.toString(), equalTo(expectedString));
+    }
+
+    private static String randomHost() {
+        return RandomStrings.randomAsciiOfLengthBetween(random(), 5, 10);
+    }
+
+    private static int randomPort() {
+        return random().nextInt();
+    }
+
+    private static String randomScheme() {
+        if (rarely()) {
+            return null;
+        }
+        return random().nextBoolean() ? "http" : "https";
+    }
+
+    private static Map<String, String> randomAttributes() {
+        int numAttributes = RandomInts.randomIntBetween(random(), 0, 5);
+        Map<String, String> attributes = new HashMap<>(numAttributes);
+        for (int i = 0; i < numAttributes; i++) {
+            String key = RandomStrings.randomAsciiOfLengthBetween(random(), 3, 10);
+            String value = RandomStrings.randomAsciiOfLengthBetween(random(), 3, 10);
+            attributes.put(key, value);
+        }
+        return attributes;
+    }
+
+    private static Set<Node.Role> randomRoles() {
+        int numRoles = RandomInts.randomIntBetween(random(), 0, 3);
+        Set<Node.Role> roles = new HashSet<>(numRoles);
+        for (int j = 0; j < numRoles; j++) {
+            roles.add(RandomPicks.randomFrom(random(), Node.Role.values()));
+        }
+        return roles;
+    }
+}
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientTests.java b/client/src/test/java/org/elasticsearch/client/RestClientTests.java
deleted file mode 100644
index 038cb8bab3c..00000000000
--- a/client/src/test/java/org/elasticsearch/client/RestClientTests.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.client;
-import com.sun.net.httpserver.HttpExchange;
-import com.sun.net.httpserver.HttpServer;
-import org.apache.http.HttpHost;
-import org.apache.http.HttpResponse;
-import org.apache.lucene.util.LuceneTestCase;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.net.InetSocketAddress;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-public class RestClientTests extends LuceneTestCase {
-    //TODO this should be refactored into a base test!!
-    HttpServer server;
-    protected String clusterName = "elasticsearch";
-    protected List<String> additionalNodes = Collections.emptyList();
-
-
-    public void setUp() throws Exception {
-        super.setUp();
-        server = HttpServer.create(new InetSocketAddress(0), 0);
-        server.setExecutor(null); // creates a default executor
-        server.start();
-        server.createContext("/", (t) -> {
-            handle("/", t);
-        });
-        server.createContext("/_cat/nodes", (t) -> {
-            handle("/_cat/nodes", t);
-        });
-        server.createContext("/_cat/health", (t) -> {
-            handle("/_cat/health", t);
-        });
-    }
-
-    protected void handle(String path, HttpExchange t) throws IOException {
-        final String response;
-        switch (path) {
-            case "/":
-                response = "{}";
-                break;
-            case "/_cat/nodes":
-                StringBuilder builder = new StringBuilder( "127.0.0.1:" + server.getAddress().getPort()  + " " + "d\n");
-                for (String host : additionalNodes) {
-                    builder.append(host).append("\n");
-                }
-                response = builder.toString();
-                break;
-            case "/_cat/health":
-                response = clusterName;
-                break;
-            default:
-                throw new IllegalArgumentException("no such handler " + path);
-        }
-        t.sendResponseHeaders(200, response.length());
-        OutputStream os = t.getResponseBody();
-        os.write(response.getBytes());
-        os.close();
-    }
-
-    public void tearDown() throws Exception {
-        super.tearDown();
-        server.stop(0);
-    }
-
-    public void testGetClustername() throws IOException {
-        HttpHost httpHost = new HttpHost("127.0.0.1", server.getAddress().getPort(), "http");
-        try(RestClient client = new RestClient(httpHost)) {
-            assertEquals(clusterName, client.getClusterName(httpHost));
-        }
-    }
-
-    public void testFetchNodes() throws IOException {
-        additionalNodes = Arrays.asList("127.0.0.2:9200 c", "127.0.0.3:9200 d");
-        HttpHost httpHost = new HttpHost("127.0.0.1", server.getAddress().getPort(), "http");
-        try(RestClient client = new RestClient(httpHost)) {
-            assertEquals(3, client.fetchNodes(httpHost, true, true, false).size());
-            assertTrue(client.fetchNodes(httpHost, true, true, false).toString(), client.fetchNodes(httpHost, true, true, false)
-                    .contains(new HttpHost("127.0.0.2", 9200, "http")));
-            assertTrue(client.fetchNodes(httpHost, true, true, false).contains(new HttpHost("127.0.0.3", 9200, "http")));
-            assertTrue(client.fetchNodes(httpHost, true, true, false).contains(httpHost));
-            assertEquals(1, client.fetchNodes(httpHost, true, true, true).size());
-        }
-    }
-
-    public void testSimpleRetry() throws IOException{
-        additionalNodes = Arrays.asList("127.0.0.2:9200 c", "127.0.0.3:9200 d");
-        HttpHost httpHost = new HttpHost("127.0.0.1", server.getAddress().getPort(), "http");
-        try(RestClient client = new RestClient(httpHost)) {
-            client.setNodes(client.fetchNodes(httpHost, true, true, false));
-            HttpResponse httpResponse = client.httpGet("/_cat/health", Collections.emptyMap());
-            assertEquals(httpResponse.getStatusLine().getStatusCode(), 200);
-            server.stop(0);
-            try {
-                client.httpGet("/_cat/health", Collections.emptyMap());
-                fail();
-            } catch (IOException ex) {
-                assertTrue(ex.getMessage(), ex.getMessage().endsWith("failed: connect timed out") ||
-                        ex.getMessage().endsWith("failed: Connection refused"));
-            }
-        }
-    }
-
-    public void testBlacklist() throws IOException{
-        additionalNodes = Arrays.asList("127.0.0.2:9200 c", "127.0.0.3:9200 d");
-        HttpHost httpHost = new HttpHost("127.0.0.1", server.getAddress().getPort(), "http");
-        try(RestClient client = new RestClient(httpHost)) {
-            client.setNodes(client.fetchNodes(httpHost, true, true, false));
-            assertEquals(3, client.getNumHosts());
-            assertEquals(0, client.getNumBlacklistedHosts());
-            server.stop(0);
-            try {
-                client.httpGet("/_cat/health", Collections.emptyMap());
-                fail();
-            } catch (IOException ex) {
-                assertTrue(ex.getMessage(), ex.getMessage().endsWith("failed: connect timed out") ||
-                        ex.getMessage().endsWith("failed: Connection refused"));
-            }
-            assertEquals(3, client.getNumHosts());
-            assertEquals(3, client.getNumBlacklistedHosts());
-            int num = 0;
-            for (HttpHost host : client.getHostIterator(false)) {
-                num++; // nothing here
-            }
-            assertEquals(0, num);
-            for (HttpHost host : client.getHostIterator(true)) {
-                num++; // all there - we have to retry now
-            }
-            assertEquals(3, num);
-        }
-    }
-}
diff --git a/client/src/test/java/org/elasticsearch/client/SnifferTests.java b/client/src/test/java/org/elasticsearch/client/SnifferTests.java
new file mode 100644
index 00000000000..9169afd7695
--- /dev/null
+++ b/client/src/test/java/org/elasticsearch/client/SnifferTests.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import okhttp3.mockwebserver.Dispatcher;
+import okhttp3.mockwebserver.MockResponse;
+import okhttp3.mockwebserver.MockWebServer;
+import okhttp3.mockwebserver.RecordedRequest;
+import org.apache.http.HttpHost;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.io.UnsupportedEncodingException;
+import java.net.URISyntaxException;
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.LogManager;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class SnifferTests extends LuceneTestCase {
+
+    static {
+        //prevent MockWebServer from logging to stdout and stderr
+        LogManager.getLogManager().reset();
+    }
+
+    private int sniffRequestTimeout;
+    private SniffingConnectionPool.Scheme scheme;
+    private SniffResponse sniffResponse;
+    private MockWebServer server;
+
+    @Before
+    public void startMockWebServer() throws IOException {
+        this.sniffRequestTimeout = RandomInts.randomIntBetween(random(), 1000, 10000);
+        this.scheme = RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values());
+        if (rarely()) {
+            this.sniffResponse = SniffResponse.buildFailure();
+        } else {
+            this.sniffResponse = buildSniffResponse(scheme);
+        }
+        this.server = buildMockWebServer(sniffResponse, sniffRequestTimeout);
+        this.server.start();
+    }
+
+    @After
+    public void stopMockWebServer() throws IOException {
+        server.shutdown();
+    }
+
+    public void testSniffNodes() throws IOException, URISyntaxException {
+        CloseableHttpClient client = HttpClientBuilder.create().build();
+        Sniffer sniffer = new Sniffer(client, RequestConfig.DEFAULT, sniffRequestTimeout, scheme.toString());
+        HttpHost httpHost = new HttpHost(server.getHostName(), server.getPort());
+        try {
+            List<Node> sniffedNodes = sniffer.sniffNodes(new Node(httpHost));
+            if (sniffResponse.isFailure) {
+                fail("sniffNodes should have failed");
+            }
+            assertThat(sniffedNodes.size(), equalTo(sniffResponse.nodes.size()));
+            Iterator<Node> responseNodesIterator = sniffResponse.nodes.iterator();
+            for (Node sniffedNode : sniffedNodes) {
+                Node responseNode = responseNodesIterator.next();
+                assertThat(sniffedNode.getHttpHost(), equalTo(responseNode.getHttpHost()));
+                assertThat(sniffedNode.getRoles(), equalTo(responseNode.getRoles()));
+                assertThat(sniffedNode.getAttributes(), equalTo(responseNode.getAttributes()));
+            }
+        } catch(ElasticsearchResponseException e) {
+            if (sniffResponse.isFailure) {
+                assertThat(e.getMessage(), containsString("GET http://localhost:" + server.getPort() +
+                        "/_nodes/http?timeout=" + sniffRequestTimeout));
+                assertThat(e.getMessage(), containsString(Integer.toString(sniffResponse.nodesInfoResponseCode)));
+                assertThat(e.getNode().getHttpHost(), equalTo(httpHost));
+                assertThat(e.getStatusLine().getStatusCode(), equalTo(sniffResponse.nodesInfoResponseCode));
+                assertThat(e.getRequestLine().toString(), equalTo("GET /_nodes/http?timeout=" + sniffRequestTimeout + "ms HTTP/1.1"));
+            } else {
+                fail("sniffNodes should have succeeded: " + e.getStatusLine());
+            }
+        }
+    }
+
+    private static MockWebServer buildMockWebServer(SniffResponse sniffResponse, int sniffTimeout) throws UnsupportedEncodingException {
+        MockWebServer server = new MockWebServer();
+        final Dispatcher dispatcher = new Dispatcher() {
+            @Override
+            public MockResponse dispatch(RecordedRequest request) throws InterruptedException {
+                String decodedUrl;
+                try {
+                    decodedUrl = URLDecoder.decode(request.getPath(), StandardCharsets.UTF_8.name());
+                } catch (UnsupportedEncodingException e) {
+                    throw new RuntimeException(e);
+                }
+                String sniffUrl = "/_nodes/http?timeout=" + sniffTimeout + "ms";
+                if (sniffUrl.equals(decodedUrl)) {
+                    return new MockResponse().setBody(sniffResponse.nodesInfoBody).setResponseCode(sniffResponse.nodesInfoResponseCode);
+                } else {
+                    return new MockResponse().setResponseCode(404);
+                }
+            }
+        };
+        server.setDispatcher(dispatcher);
+        return server;
+    }
+
+    private static SniffResponse buildSniffResponse(SniffingConnectionPool.Scheme scheme) throws IOException {
+        int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
+        List<Node> nodes = new ArrayList<>(numNodes);
+        JsonFactory jsonFactory = new JsonFactory();
+        StringWriter writer = new StringWriter();
+        JsonGenerator generator = jsonFactory.createGenerator(writer);
+        generator.writeStartObject();
+        if (random().nextBoolean()) {
+            generator.writeStringField("cluster_name", "elasticsearch");
+        }
+        if (random().nextBoolean()) {
+            generator.writeObjectFieldStart("bogus_object");
+            generator.writeEndObject();
+        }
+        generator.writeObjectFieldStart("nodes");
+        for (int i = 0; i < numNodes; i++) {
+            String nodeId = RandomStrings.randomAsciiOfLengthBetween(random(), 5, 10);
+            generator.writeObjectFieldStart(nodeId);
+            if (random().nextBoolean()) {
+                generator.writeObjectFieldStart("bogus_object");
+                generator.writeEndObject();
+            }
+            if (random().nextBoolean()) {
+                generator.writeArrayFieldStart("bogus_array");
+                generator.writeStartObject();
+                generator.writeEndObject();
+                generator.writeEndArray();
+            }
+            boolean isHttpEnabled = rarely() == false;
+            int numRoles = RandomInts.randomIntBetween(random(), 0, 3);
+            Set<Node.Role> nodeRoles = new HashSet<>(numRoles);
+            for (int j = 0; j < numRoles; j++) {
+                Node.Role role;
+                do {
+                    role = RandomPicks.randomFrom(random(), Node.Role.values());
+                } while(nodeRoles.add(role) == false);
+            }
+
+            int numAttributes = RandomInts.randomIntBetween(random(), 0, 3);
+            Map<String, String> attributes = new HashMap<>(numAttributes);
+            for (int j = 0; j < numAttributes; j++) {
+                attributes.put("attr" + j, "value" + j);
+            }
+            if (isHttpEnabled) {
+                String host = "host" + i;
+                int port = RandomInts.randomIntBetween(random(), 9200, 9299);
+                HttpHost httpHost = new HttpHost(host, port, scheme.toString());
+                nodes.add(new Node(httpHost, nodeRoles, attributes));
+                generator.writeObjectFieldStart("http");
+                if (random().nextBoolean()) {
+                    generator.writeArrayFieldStart("bound_address");
+                    generator.writeString("[fe80::1]:" + port);
+                    generator.writeString("[::1]:" + port);
+                    generator.writeString("127.0.0.1:" + port);
+                    generator.writeEndArray();
+                }
+                if (random().nextBoolean()) {
+                    generator.writeObjectFieldStart("bogus_object");
+                    generator.writeEndObject();
+                }
+                generator.writeStringField("publish_address", httpHost.toHostString());
+                if (random().nextBoolean()) {
+                    generator.writeNumberField("max_content_length_in_bytes", 104857600);
+                }
+                generator.writeEndObject();
+            }
+            generator.writeArrayFieldStart("roles");
+            for (Node.Role nodeRole : nodeRoles) {
+                generator.writeString(nodeRole.toString());
+            }
+            generator.writeEndArray();
+            if (numAttributes > 0) {
+                generator.writeObjectFieldStart("attributes");
+            }
+            for (Map.Entry<String, String> entry : attributes.entrySet()) {
+                generator.writeStringField(entry.getKey(), entry.getValue());
+            }
+            if (numAttributes > 0) {
+                generator.writeEndObject();
+            }
+            generator.writeEndObject();
+        }
+        generator.writeEndObject();
+        generator.writeEndObject();
+        generator.close();
+        return SniffResponse.buildResponse(writer.toString(), nodes);
+    }
+
+    private static class SniffResponse {
+        private final String nodesInfoBody;
+        private final int nodesInfoResponseCode;
+        private final List<Node> nodes;
+        private final boolean isFailure;
+
+        SniffResponse(String nodesInfoBody, List<Node> nodes, boolean isFailure) {
+            this.nodesInfoBody = nodesInfoBody;
+            this.nodes = nodes;
+            this.isFailure = isFailure;
+            if (isFailure) {
+                this.nodesInfoResponseCode = randomErrorResponseCode();
+            } else {
+                this.nodesInfoResponseCode = 200;
+            }
+        }
+
+        static SniffResponse buildFailure() {
+            return new SniffResponse("", Collections.emptyList(), true);
+        }
+
+        static SniffResponse buildResponse(String nodesInfoBody, List<Node> nodes) {
+            return new SniffResponse(nodesInfoBody, nodes, false);
+        }
+    }
+
+    private static int randomErrorResponseCode() {
+        return RandomInts.randomIntBetween(random(), 400, 599);
+    }
+}
diff --git a/client/src/test/java/org/elasticsearch/client/SniffingConnectionPoolTests.java b/client/src/test/java/org/elasticsearch/client/SniffingConnectionPoolTests.java
new file mode 100644
index 00000000000..13eaadd619a
--- /dev/null
+++ b/client/src/test/java/org/elasticsearch/client/SniffingConnectionPoolTests.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import org.apache.http.HttpHost;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.lucene.util.LuceneTestCase;
+
+import java.util.logging.LogManager;
+
+public class SniffingConnectionPoolTests extends LuceneTestCase {
+
+    static {
+        LogManager.getLogManager().reset();
+    }
+
+    public void testConstructor() throws Exception {
+        CloseableHttpClient httpClient = HttpClientBuilder.create().build();
+        int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
+        Node[] nodes = new Node[numNodes];
+        for (int i = 0; i < numNodes; i++) {
+            nodes[i] = new Node(new HttpHost("localhost", 9200));
+        }
+
+        try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
+                RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0), random().nextBoolean(),
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), nodes)) {
+
+            fail("pool creation should have failed " + connectionPool);
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "sniffInterval must be greater than 0");
+        }
+
+        try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
+                RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0), httpClient, RequestConfig.DEFAULT,
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), nodes)) {
+            fail("pool creation should have failed " + connectionPool);
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "sniffAfterFailureDelay must be greater than 0");
+        }
+
+        try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), null, RequestConfig.DEFAULT,
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), nodes)) {
+            fail("pool creation should have failed " + connectionPool);
+        } catch(NullPointerException e) {
+            assertEquals(e.getMessage(), "client cannot be null");
+        }
+
+        try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, null,
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), nodes)) {
+            fail("pool creation should have failed " + connectionPool);
+        } catch(NullPointerException e) {
+            assertEquals(e.getMessage(), "sniffRequestConfig cannot be null");
+        }
+
+        try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
+                RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0),
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), nodes)) {
+            fail("pool creation should have failed " + connectionPool);
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "sniffRequestTimeout must be greater than 0");
+        }
+
+        try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), null, nodes)) {
+            fail("pool creation should have failed " + connectionPool);
+        } catch(NullPointerException e) {
+            assertEquals(e.getMessage(), "connection selector predicate cannot be null");
+        }
+
+        try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()),
+                connection -> random().nextBoolean(), (Node[])null)) {
+            fail("pool creation should have failed " + connectionPool);
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "no nodes provided");
+        }
+
+        try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(),
+                (Node)null)) {
+            fail("pool creation should have failed " + connectionPool);
+        } catch(NullPointerException e) {
+            assertEquals(e.getMessage(), "node cannot be null");
+        }
+
+        try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean())) {
+            fail("pool creation should have failed " + connectionPool);
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "no nodes provided");
+        }
+
+        try (SniffingConnectionPool sniffingConnectionPool = new SniffingConnectionPool(
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), nodes)) {
+            assertNotNull(sniffingConnectionPool);
+        }
+    }
+}
diff --git a/client/src/test/java/org/elasticsearch/client/StaticConnectionPoolTests.java b/client/src/test/java/org/elasticsearch/client/StaticConnectionPoolTests.java
new file mode 100644
index 00000000000..9b5b29df1b1
--- /dev/null
+++ b/client/src/test/java/org/elasticsearch/client/StaticConnectionPoolTests.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import org.apache.http.HttpHost;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.lucene.util.LuceneTestCase;
+
+import java.util.logging.LogManager;
+
+public class StaticConnectionPoolTests extends LuceneTestCase {
+
+    static {
+        LogManager.getLogManager().reset();
+    }
+
+    public void testConstructor() {
+        CloseableHttpClient httpClient = HttpClientBuilder.create().build();
+        int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
+        Node[] nodes = new Node[numNodes];
+        for (int i = 0; i < numNodes; i++) {
+            nodes[i] = new Node(new HttpHost("localhost", 9200));
+        }
+
+        try {
+            new StaticConnectionPool(null, random().nextBoolean(), RequestConfig.DEFAULT, connection -> random().nextBoolean(), nodes);
+        } catch(NullPointerException e) {
+            assertEquals(e.getMessage(), "client cannot be null");
+        }
+
+        try {
+            new StaticConnectionPool(httpClient, random().nextBoolean(), null, connection -> random().nextBoolean(), nodes);
+        } catch(NullPointerException e) {
+            assertEquals(e.getMessage(), "pingRequestConfig cannot be null");
+        }
+
+        try {
+            new StaticConnectionPool(httpClient, random().nextBoolean(), RequestConfig.DEFAULT, null, nodes);
+        } catch(NullPointerException e) {
+            assertEquals(e.getMessage(), "connection selector predicate cannot be null");
+        }
+
+        try {
+            new StaticConnectionPool(httpClient, random().nextBoolean(), RequestConfig.DEFAULT,
+                    connection -> random().nextBoolean(), (Node)null);
+        } catch(NullPointerException e) {
+            assertEquals(e.getMessage(), "node cannot be null");
+        }
+
+        try {
+            new StaticConnectionPool(httpClient, random().nextBoolean(), RequestConfig.DEFAULT,
+                    connection -> random().nextBoolean(), (Node[])null);
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "no nodes provided");
+        }
+
+        try {
+            new StaticConnectionPool(httpClient, random().nextBoolean(), RequestConfig.DEFAULT, connection -> random().nextBoolean());
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "no nodes provided");
+        }
+
+        StaticConnectionPool staticConnectionPool = new StaticConnectionPool(httpClient, random().nextBoolean(), RequestConfig.DEFAULT,
+                connection -> random().nextBoolean(), nodes);
+        assertNotNull(staticConnectionPool);
+    }
+}
diff --git a/client/src/test/java/org/elasticsearch/client/TransportTests.java b/client/src/test/java/org/elasticsearch/client/TransportTests.java
new file mode 100644
index 00000000000..591eb516c59
--- /dev/null
+++ b/client/src/test/java/org/elasticsearch/client/TransportTests.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.lucene.util.LuceneTestCase;
+
+import java.io.IOException;
+import java.util.logging.LogManager;
+import java.util.stream.Stream;
+
+public class TransportTests extends LuceneTestCase {
+
+    static {
+        LogManager.getLogManager().reset();
+    }
+
+    public void testConstructor() {
+        CloseableHttpClient httpClient = HttpClientBuilder.create().build();
+        ConnectionPool<Connection> connectionPool = new ConnectionPool<Connection>() {
+            @Override
+            public Stream<Connection> nextConnection() {
+                return null;
+            }
+
+            @Override
+            public Connection lastResortConnection() {
+                return null;
+            }
+
+            @Override
+            public void beforeAttempt(Connection connection) throws IOException {
+
+            }
+
+            @Override
+            public void onSuccess(Connection connection) {
+
+            }
+
+            @Override
+            public void onFailure(Connection connection) throws IOException {
+
+            }
+
+            @Override
+            public void close() throws IOException {
+
+            }
+        };
+
+        try {
+            new Transport<>(null, connectionPool, RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+            fail("transport creation should have failed");
+        } catch(NullPointerException e) {
+            assertEquals(e.getMessage(), "client cannot be null");
+        }
+
+        try {
+            new Transport<>(httpClient, null, RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+            fail("transport creation should have failed");
+        } catch(NullPointerException e) {
+            assertEquals(e.getMessage(), "connectionPool cannot be null");
+        }
+
+        try {
+            new Transport<>(httpClient, connectionPool, RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+            fail("transport creation should have failed");
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "maxRetryTimeout must be greater than 0");
+        }
+
+        Transport<Connection> transport = new Transport<>(httpClient, connectionPool, RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+        assertNotNull(transport);
+    }
+}

From 25892351e7d7f6739dacc04db0c59308b7879c88 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Wed, 4 May 2016 15:43:00 +0200
Subject: [PATCH 0582/1311] remove Verb enum

---
 .../org/elasticsearch/client/Transport.java   | 21 ++++++++++---------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/Transport.java b/client/src/main/java/org/elasticsearch/client/Transport.java
index 40f68ab61cc..8898cf6714f 100644
--- a/client/src/main/java/org/elasticsearch/client/Transport.java
+++ b/client/src/main/java/org/elasticsearch/client/Transport.java
@@ -38,6 +38,7 @@ import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.Iterator;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
@@ -62,9 +63,9 @@ final class Transport<C extends Connection> implements Closeable {
         this.maxRetryTimeout = maxRetryTimeout;
     }
 
-    ElasticsearchResponse performRequest(Verb verb, String endpoint, Map<String, Object> params, HttpEntity entity) throws IOException {
+    ElasticsearchResponse performRequest(String method, String endpoint, Map<String, Object> params, HttpEntity entity) throws IOException {
         URI uri = buildUri(endpoint, params);
-        HttpRequestBase request = createHttpRequest(verb, uri, entity);
+        HttpRequestBase request = createHttpRequest(method, uri, entity);
         Iterator<C> connectionIterator = connectionPool.nextConnection().iterator();
         if (connectionIterator.hasNext() == false) {
             C connection = connectionPool.lastResortConnection();
@@ -153,31 +154,31 @@ final class Transport<C extends Connection> implements Closeable {
         return currentException;
     }
 
-    private static HttpRequestBase createHttpRequest(Verb verb, URI uri, HttpEntity entity) {
-        switch(verb) {
-            case DELETE:
+    private static HttpRequestBase createHttpRequest(String method, URI uri, HttpEntity entity) {
+        switch(method.toUpperCase(Locale.ROOT)) {
+            case HttpDeleteWithEntity.METHOD_NAME:
                 HttpDeleteWithEntity httpDeleteWithEntity = new HttpDeleteWithEntity(uri);
                 addRequestBody(httpDeleteWithEntity, entity);
                 return httpDeleteWithEntity;
-            case GET:
+            case HttpGetWithEntity.METHOD_NAME:
                 HttpGetWithEntity httpGetWithEntity = new HttpGetWithEntity(uri);
                 addRequestBody(httpGetWithEntity, entity);
                 return httpGetWithEntity;
-            case HEAD:
+            case HttpHead.METHOD_NAME:
                 if (entity != null) {
                     throw new UnsupportedOperationException("HEAD with body is not supported");
                 }
                 return new HttpHead(uri);
-            case POST:
+            case HttpPost.METHOD_NAME:
                 HttpPost httpPost = new HttpPost(uri);
                 addRequestBody(httpPost, entity);
                 return httpPost;
-            case PUT:
+            case HttpPut.METHOD_NAME:
                 HttpPut httpPut = new HttpPut(uri);
                 addRequestBody(httpPut, entity);
                 return httpPut;
             default:
-                throw new UnsupportedOperationException("http method not supported: " + verb);
+                throw new UnsupportedOperationException("http method not supported: " + method);
         }
     }
 

From 94cf8437d0b2b729fba10590e15c052ba571b391 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Wed, 4 May 2016 15:47:38 +0200
Subject: [PATCH 0583/1311] get rid of retry timeout exception

---
 .../org/elasticsearch/client/RestClient.java  |  4 +--
 .../client/RetryTimeoutException.java         | 29 -------------------
 .../org/elasticsearch/client/Transport.java   |  2 +-
 .../java/org/elasticsearch/client/Verb.java   | 27 -----------------
 4 files changed, 3 insertions(+), 59 deletions(-)
 delete mode 100644 client/src/main/java/org/elasticsearch/client/RetryTimeoutException.java
 delete mode 100644 client/src/main/java/org/elasticsearch/client/Verb.java

diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index a9150119667..35af2723267 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -33,9 +33,9 @@ public final class RestClient implements Closeable {
         this.transport = new Transport<>(client, connectionPool, maxRetryTimeout);
     }
 
-    public ElasticsearchResponse performRequest(Verb verb, String endpoint, Map<String, Object> params, HttpEntity entity)
+    public ElasticsearchResponse performRequest(String method, String endpoint, Map<String, Object> params, HttpEntity entity)
             throws IOException {
-        return transport.performRequest(verb, endpoint, params, entity);
+        return transport.performRequest(method, endpoint, params, entity);
     }
 
     @Override
diff --git a/client/src/main/java/org/elasticsearch/client/RetryTimeoutException.java b/client/src/main/java/org/elasticsearch/client/RetryTimeoutException.java
deleted file mode 100644
index 632597bfa37..00000000000
--- a/client/src/main/java/org/elasticsearch/client/RetryTimeoutException.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.client;
-
-import java.io.IOException;
-
-public class RetryTimeoutException extends IOException {
-
-    RetryTimeoutException(String message) {
-        super(message);
-    }
-}
diff --git a/client/src/main/java/org/elasticsearch/client/Transport.java b/client/src/main/java/org/elasticsearch/client/Transport.java
index 8898cf6714f..88bf1bf6c7a 100644
--- a/client/src/main/java/org/elasticsearch/client/Transport.java
+++ b/client/src/main/java/org/elasticsearch/client/Transport.java
@@ -88,7 +88,7 @@ final class Transport<C extends Connection> implements Closeable {
                 long timeElapsed = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
                 long timeout = retryTimeout - timeElapsed;
                 if (timeout <= 0) {
-                    RetryTimeoutException retryTimeoutException = new RetryTimeoutException(
+                    IOException retryTimeoutException = new IOException(
                             "request retries exceeded max retry timeout [" + retryTimeout + "]");
                     retryTimeoutException.addSuppressed(lastSeenException);
                     throw retryTimeoutException;
diff --git a/client/src/main/java/org/elasticsearch/client/Verb.java b/client/src/main/java/org/elasticsearch/client/Verb.java
deleted file mode 100644
index 74a88353977..00000000000
--- a/client/src/main/java/org/elasticsearch/client/Verb.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.client;
-
-/**
- * Holds the http verbs/methods supported by elasticsearch, which can be used when sending a request
- */
-public enum Verb {
-    DELETE, GET, HEAD, POST, PUT
-}

From f6a5a0a4adab554bf8c19f3e7b16c2c1f0f4b6a5 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Wed, 4 May 2016 16:27:52 +0200
Subject: [PATCH 0584/1311] get rid of Node abstraction

---
 .../client/AbstractStaticConnectionPool.java  |  13 +-
 .../org/elasticsearch/client/Connection.java  |  18 +--
 .../client/ElasticsearchResponse.java         |  20 +--
 .../ElasticsearchResponseException.java       |  32 +++--
 .../java/org/elasticsearch/client/Node.java   | 125 ----------------
 .../elasticsearch/client/RequestLogger.java   |  13 +-
 .../org/elasticsearch/client/Sniffer.java     |  54 +++----
 .../client/SniffingConnectionPool.java        |  29 ++--
 .../client/StatefulConnection.java            |   8 +-
 .../client/StaticConnectionPool.java          |  19 +--
 .../org/elasticsearch/client/Transport.java   |  14 +-
 .../org/elasticsearch/client/NodeTests.java   | 136 ------------------
 .../elasticsearch/client/SnifferTests.java    |  63 ++++----
 .../client/SniffingConnectionPoolTests.java   |  28 ++--
 .../client/StaticConnectionPoolTests.java     |  22 +--
 15 files changed, 171 insertions(+), 423 deletions(-)
 delete mode 100644 client/src/main/java/org/elasticsearch/client/Node.java
 delete mode 100644 client/src/test/java/org/elasticsearch/client/NodeTests.java

diff --git a/client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java b/client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java
index f30a2b9c5c1..c043dd17059 100644
--- a/client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java
@@ -21,6 +21,7 @@ package org.elasticsearch.client;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.http.HttpHost;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -74,11 +75,11 @@ public abstract class AbstractStaticConnectionPool implements ConnectionPool<Sta
         return sortedConnections.stream().filter(connection -> connection.isAlive() || connection.shouldBeRetried());
     }
 
-    protected List<StatefulConnection> createConnections(Node... nodes) {
+    protected List<StatefulConnection> createConnections(HttpHost... hosts) {
         List<StatefulConnection> connections = new ArrayList<>();
-        for (Node node : nodes) {
-            Objects.requireNonNull(node, "node cannot be null");
-            connections.add(new StatefulConnection(node));
+        for (HttpHost host : hosts) {
+            Objects.requireNonNull(host, "host cannot be null");
+            connections.add(new StatefulConnection(host));
         }
         return Collections.unmodifiableList(connections);
     }
@@ -94,12 +95,12 @@ public abstract class AbstractStaticConnectionPool implements ConnectionPool<Sta
     @Override
     public void onSuccess(StatefulConnection connection) {
         connection.markAlive();
-        logger.trace("marked connection alive for " + connection.getNode());
+        logger.trace("marked connection alive for " + connection.getHost());
     }
 
     @Override
     public void onFailure(StatefulConnection connection) throws IOException {
         connection.markDead();
-        logger.debug("marked connection dead for " + connection.getNode());
+        logger.debug("marked connection dead for " + connection.getHost());
     }
 }
diff --git a/client/src/main/java/org/elasticsearch/client/Connection.java b/client/src/main/java/org/elasticsearch/client/Connection.java
index 225c812ebfa..51a430b2540 100644
--- a/client/src/main/java/org/elasticsearch/client/Connection.java
+++ b/client/src/main/java/org/elasticsearch/client/Connection.java
@@ -19,27 +19,29 @@
 
 package org.elasticsearch.client;
 
+import org.apache.http.HttpHost;
+
 /**
  * Simplest representation of a connection to an elasticsearch node.
- * It doesn't have any mutable state. It holds the node that the connection points to.
+ * It doesn't have any mutable state. It holds the host that the connection points to.
  * Allows the transport to deal with very simple connection objects that are immutable.
  * Any change to the state of connections should be made through the connection pool
  * which is aware of the connection object that it supports.
  */
 public class Connection {
-    private final Node node;
+    private final HttpHost host;
 
     /**
-     * Creates a new connection pointing to the provided {@link Node} argument
+     * Creates a new connection pointing to the provided {@link HttpHost} argument
      */
-    public Connection(Node node) {
-            this.node = node;
+    public Connection(HttpHost host) {
+            this.host = host;
         }
 
     /**
-     * Returns the {@link Node} that the connection points to
+     * Returns the {@link HttpHost} that the connection points to
      */
-    public Node getNode() {
-            return node;
+    public HttpHost getHost() {
+            return host;
         }
 }
diff --git a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponse.java b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponse.java
index 13c92c16453..124a233dea7 100644
--- a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponse.java
+++ b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponse.java
@@ -21,6 +21,7 @@ package org.elasticsearch.client;
 
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
 import org.apache.http.RequestLine;
 import org.apache.http.StatusLine;
 import org.apache.http.client.methods.CloseableHttpResponse;
@@ -31,20 +32,20 @@ import java.util.Objects;
 
 /**
  * Holds an elasticsearch response. It wraps the {@link CloseableHttpResponse} response and associates it with
- * its corresponding {@link RequestLine} and {@link Node}
+ * its corresponding {@link RequestLine} and {@link HttpHost}
  */
 public class ElasticsearchResponse implements Closeable {
 
     private final RequestLine requestLine;
-    private final Node node;
+    private final HttpHost host;
     private final CloseableHttpResponse response;
 
-    ElasticsearchResponse(RequestLine requestLine, Node node, CloseableHttpResponse response) {
+    ElasticsearchResponse(RequestLine requestLine, HttpHost host, CloseableHttpResponse response) {
         Objects.requireNonNull(requestLine, "requestLine cannot be null");
-        Objects.requireNonNull(node, "node cannot be null");
+        Objects.requireNonNull(host, "node cannot be null");
         Objects.requireNonNull(response, "response cannot be null");
         this.requestLine = requestLine;
-        this.node = node;
+        this.host = host;
         this.response = response;
     }
 
@@ -58,8 +59,8 @@ public class ElasticsearchResponse implements Closeable {
     /**
      * Returns the node that returned this response
      */
-    public Node getNode() {
-        return node;
+    public HttpHost getHost() {
+        return host;
     }
 
     /**
@@ -77,7 +78,8 @@ public class ElasticsearchResponse implements Closeable {
     }
 
     /**
-     * Returns the response bodyi available, null otherwise
+     * Returns the response body available, null otherwise
+     * @see HttpEntity
      */
     public HttpEntity getEntity() {
         return response.getEntity();
@@ -87,7 +89,7 @@ public class ElasticsearchResponse implements Closeable {
     public String toString() {
         return "ElasticsearchResponse{" +
                 "requestLine=" + requestLine +
-                ", node=" + node +
+                ", host=" + host +
                 ", response=" + response.getStatusLine() +
                 '}';
     }
diff --git a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
index 6c2ef2d2e0d..9f4b123bc8c 100644
--- a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
+++ b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
@@ -19,41 +19,57 @@
 
 package org.elasticsearch.client;
 
+import org.apache.http.HttpHost;
 import org.apache.http.RequestLine;
 import org.apache.http.StatusLine;
 
 import java.io.IOException;
 
+/**
+ * Exception thrown when an elasticsearch node responds to a request with a status code that indicates an error
+ */
 public class ElasticsearchResponseException extends IOException {
 
-    private final Node node;
+    private final HttpHost host;
     private final RequestLine requestLine;
     private final StatusLine statusLine;
 
-    ElasticsearchResponseException(RequestLine requestLine, Node node, StatusLine statusLine) {
-        super(buildMessage(requestLine, node, statusLine));
-        this.node = node;
+    ElasticsearchResponseException(RequestLine requestLine, HttpHost host, StatusLine statusLine) {
+        super(buildMessage(requestLine, host, statusLine));
+        this.host = host;
         this.requestLine = requestLine;
         this.statusLine = statusLine;
     }
 
-    private static String buildMessage(RequestLine requestLine, Node node, StatusLine statusLine) {
-        return requestLine.getMethod() + " " + node.getHttpHost() + requestLine.getUri() + ": " + statusLine.toString();
+    private static String buildMessage(RequestLine requestLine, HttpHost host, StatusLine statusLine) {
+        return requestLine.getMethod() + " " + host + requestLine.getUri() + ": " + statusLine.toString();
     }
 
+    /**
+     * Returns whether the error is recoverable or not, hence whether the same request should be retried on other nodes or not
+     */
     public boolean isRecoverable() {
         //clients don't retry on 500 because elasticsearch still misuses it instead of 400 in some places
         return statusLine.getStatusCode() >= 502 && statusLine.getStatusCode() <= 504;
     }
 
-    public Node getNode() {
-        return node;
+    /**
+     * Returns the {@link HttpHost} that returned the error
+     */
+    public HttpHost getHost() {
+        return host;
     }
 
+    /**
+     * Returns the {@link RequestLine} that triggered the error
+     */
     public RequestLine getRequestLine() {
         return requestLine;
     }
 
+    /**
+     * Returns the {@link StatusLine} that was returned by elasticsearch
+     */
     public StatusLine getStatusLine() {
         return statusLine;
     }
diff --git a/client/src/main/java/org/elasticsearch/client/Node.java b/client/src/main/java/org/elasticsearch/client/Node.java
deleted file mode 100644
index ab6d3631527..00000000000
--- a/client/src/main/java/org/elasticsearch/client/Node.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.client;
-
-import org.apache.http.HttpHost;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-
-/**
- * Represents an Elasticsearch node.
- * Holds its http address as an {@link HttpHost} instance, as well as its optional set of roles and attributes.
- * Roles and attributes can be populated in one of the two following ways:
- * 1) using a connection pool that supports sniffing, so that all the info is retrieved from elasticsearch itself
- * 2) manually passing the info through the {@link #Node(HttpHost, Set, Map)} constructor
- * Roles and attributes may be taken into account as part of connection selection by the connection pool, which
- * can be customized by passing in a predicate at connection pool creation.
- */
-public class Node {
-    private final HttpHost httpHost;
-    private final Set<Role> roles;
-    private final Map<String, String> attributes;
-
-    /**
-     * Creates a node given its http address as an {@link HttpHost} instance.
-     * Roles are not provided hence all possible roles will be assumed, as that is the default in Elasticsearch.
-     * No attributes will be associated with the node.
-     *
-     * @param httpHost the http address of the node
-     */
-    public Node(HttpHost httpHost) {
-        this(httpHost, new HashSet<>(Arrays.asList(Role.values())), Collections.emptyMap());
-    }
-
-    /**
-     * Creates a node given its http address as an {@link HttpHost} instance, its set or roles and attributes.
-     *
-     * @param httpHost the http address of the node
-     * @param roles the set of roles that the node fulfills within the cluster
-     * @param attributes the attributes associated with the node
-     */
-    public Node(HttpHost httpHost, Set<Role> roles, Map<String, String> attributes) {
-        Objects.requireNonNull(httpHost, "host cannot be null");
-        Objects.requireNonNull(roles, "roles cannot be null");
-        Objects.requireNonNull(attributes, "attributes cannot be null");
-        this.httpHost = httpHost;
-        this.roles = Collections.unmodifiableSet(roles);
-        this.attributes = Collections.unmodifiableMap(attributes);
-    }
-
-    /**
-     * Returns the http address of the node
-     */
-    public HttpHost getHttpHost() {
-        return httpHost;
-    }
-
-    /**
-     * Returns the set of roles associated with the node
-     */
-    public Set<Role> getRoles() {
-        return roles;
-    }
-
-    /**
-     * Returns the set of attributes associated with the node
-     */
-    public Map<String, String> getAttributes() {
-        return attributes;
-    }
-
-    @Override
-    public String toString() {
-        return "Node{" +
-                "httpHost=" + httpHost +
-                ", roles=" + roles +
-                ", attributes=" + attributes +
-                '}';
-    }
-
-    /**
-     * Holds all the potential roles that a node can fulfill within a cluster
-     */
-    public enum Role {
-        /**
-         * Data node
-         */
-        DATA,
-        /**
-         * Master eligible node
-         */
-        MASTER,
-        /**
-         * Ingest node
-         */
-        INGEST;
-
-        @Override
-        public String toString() {
-            return name().toLowerCase(Locale.ROOT);
-        }
-    }
-}
diff --git a/client/src/main/java/org/elasticsearch/client/RequestLogger.java b/client/src/main/java/org/elasticsearch/client/RequestLogger.java
index 8423dcb8ade..99ce924c164 100644
--- a/client/src/main/java/org/elasticsearch/client/RequestLogger.java
+++ b/client/src/main/java/org/elasticsearch/client/RequestLogger.java
@@ -20,13 +20,14 @@
 package org.elasticsearch.client;
 
 import org.apache.commons.logging.Log;
+import org.apache.http.HttpHost;
 import org.apache.http.RequestLine;
 import org.apache.http.StatusLine;
 
 import java.io.IOException;
 
 /**
- * Helper class that exposes static method to unify the way requests are logged
+ * Helper class that exposes static methods to unify the way requests are logged
  */
 public final class RequestLogger {
 
@@ -36,16 +37,14 @@ public final class RequestLogger {
     /**
      * Logs a request that yielded a response
      */
-    public static void log(Log logger, String message, RequestLine requestLine, Node node, StatusLine statusLine) {
-        logger.debug(message + " [" + requestLine.getMethod() + " " + node.getHttpHost() +
-                requestLine.getUri() + "] [" + statusLine + "]");
+    public static void log(Log logger, String message, RequestLine requestLine, HttpHost host, StatusLine statusLine) {
+        logger.debug(message + " [" + requestLine.getMethod() + " " + host + requestLine.getUri() + "] [" + statusLine + "]");
     }
 
     /**
      * Logs a request that failed
      */
-    public static void log(Log logger, String message, RequestLine requestLine, Node node, IOException e) {
-        logger.debug(message + " [" + requestLine.getMethod() + " " + node.getHttpHost() +
-                requestLine.getUri() + "]", e);
+    public static void log(Log logger, String message, RequestLine requestLine, HttpHost host, IOException e) {
+        logger.debug(message + " [" + requestLine.getMethod() + " " + host + requestLine.getUri() + "]", e);
     }
 }
diff --git a/client/src/main/java/org/elasticsearch/client/Sniffer.java b/client/src/main/java/org/elasticsearch/client/Sniffer.java
index 07d485de879..1bec4eb87e1 100644
--- a/client/src/main/java/org/elasticsearch/client/Sniffer.java
+++ b/client/src/main/java/org/elasticsearch/client/Sniffer.java
@@ -37,14 +37,13 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
 import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Locale;
-import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
 
+/**
+ * Calls nodes info api and returns a list of http hosts extracted from it
+ */
+//TODO this could potentially a call to _cat/nodes (although it doesn't support timeout param), but how would we handle bw comp with 2.x?
 final class Sniffer {
 
     private static final Log logger = LogFactory.getLog(Sniffer.class);
@@ -69,34 +68,34 @@ final class Sniffer {
         this.jsonFactory = new JsonFactory();
     }
 
-    List<Node> sniffNodes(Node node) throws IOException {
+    List<HttpHost> sniffNodes(HttpHost host) throws IOException {
         HttpGet httpGet = new HttpGet("/_nodes/http?timeout=" + sniffRequestTimeout + "ms");
         httpGet.setConfig(sniffRequestConfig);
 
-        try (CloseableHttpResponse response = client.execute(node.getHttpHost(), httpGet)) {
+        try (CloseableHttpResponse response = client.execute(host, httpGet)) {
             StatusLine statusLine = response.getStatusLine();
             if (statusLine.getStatusCode() >= 300) {
-                RequestLogger.log(logger, "sniff failed", httpGet.getRequestLine(), node, statusLine);
+                RequestLogger.log(logger, "sniff failed", httpGet.getRequestLine(), host, statusLine);
                 EntityUtils.consume(response.getEntity());
-                throw new ElasticsearchResponseException(httpGet.getRequestLine(), node, statusLine);
+                throw new ElasticsearchResponseException(httpGet.getRequestLine(), host, statusLine);
             } else {
-                List<Node> nodes = readNodes(response.getEntity());
-                RequestLogger.log(logger, "sniff succeeded", httpGet.getRequestLine(), node, statusLine);
+                List<HttpHost> nodes = readHosts(response.getEntity());
+                RequestLogger.log(logger, "sniff succeeded", httpGet.getRequestLine(), host, statusLine);
                 return nodes;
             }
         } catch(IOException e) {
-            RequestLogger.log(logger, "sniff failed", httpGet.getRequestLine(), node, e);
+            RequestLogger.log(logger, "sniff failed", httpGet.getRequestLine(), host, e);
             throw e;
         }
     }
 
-    private List<Node> readNodes(HttpEntity entity) throws IOException {
+    private List<HttpHost> readHosts(HttpEntity entity) throws IOException {
         try (InputStream inputStream = entity.getContent()) {
             JsonParser parser = jsonFactory.createParser(inputStream);
             if (parser.nextToken() != JsonToken.START_OBJECT) {
                 throw new IOException("expected data to start with an object");
             }
-            List<Node> nodes = new ArrayList<>();
+            List<HttpHost> hosts = new ArrayList<>();
             while (parser.nextToken() != JsonToken.END_OBJECT) {
                 if (parser.getCurrentToken() == JsonToken.START_OBJECT) {
                     if ("nodes".equals(parser.getCurrentName())) {
@@ -104,10 +103,10 @@ final class Sniffer {
                             JsonToken token = parser.nextToken();
                             assert token == JsonToken.START_OBJECT;
                             String nodeId = parser.getCurrentName();
-                            Node sniffedNode = readNode(nodeId, parser, this.scheme);
-                            if (sniffedNode != null) {
+                            HttpHost sniffedHost = readNode(nodeId, parser, this.scheme);
+                            if (sniffedHost != null) {
                                 logger.trace("adding node [" + nodeId + "]");
-                                nodes.add(sniffedNode);
+                                hosts.add(sniffedHost);
                             }
                         }
                     } else {
@@ -115,31 +114,20 @@ final class Sniffer {
                     }
                 }
             }
-            return nodes;
+            return hosts;
         }
     }
 
-    private static Node readNode(String nodeId, JsonParser parser, String scheme) throws IOException {
+    private static HttpHost readNode(String nodeId, JsonParser parser, String scheme) throws IOException {
         HttpHost httpHost = null;
-        Set<Node.Role> roles = new HashSet<>();
-        Map<String, String> attributes = new HashMap<>();
         String fieldName = null;
         while (parser.nextToken() != JsonToken.END_OBJECT) {
             if (parser.getCurrentToken() == JsonToken.FIELD_NAME) {
                 fieldName = parser.getCurrentName();
-            } else if (parser.getCurrentToken() == JsonToken.START_ARRAY && "roles".equals(fieldName)) {
-                while (parser.nextToken() != JsonToken.END_ARRAY) {
-                    roles.add(Node.Role.valueOf(parser.getValueAsString().toUpperCase(Locale.ROOT)));
-                }
             } else if (parser.getCurrentToken() == JsonToken.START_OBJECT) {
-                if ("attributes".equals(fieldName)) {
+                if ("http".equals(fieldName)) {
                     while (parser.nextToken() != JsonToken.END_OBJECT) {
-                        attributes.put(parser.getCurrentName(), parser.getValueAsString());
-                    }
-                } else if ("http".equals(fieldName)) {
-                    while (parser.nextToken() != JsonToken.END_OBJECT) {
-                        if (parser.getCurrentToken() == JsonToken.VALUE_STRING &&
-                                "publish_address".equals(parser.getCurrentName())) {
+                        if (parser.getCurrentToken() == JsonToken.VALUE_STRING && "publish_address".equals(parser.getCurrentName())) {
                             URI boundAddressAsURI = URI.create(scheme + "://" + parser.getValueAsString());
                             httpHost = new HttpHost(boundAddressAsURI.getHost(), boundAddressAsURI.getPort(),
                                     boundAddressAsURI.getScheme());
@@ -157,6 +145,6 @@ final class Sniffer {
             logger.debug("skipping node [" + nodeId + "] with http disabled");
             return null;
         }
-        return new Node(httpHost, roles, attributes);
+        return httpHost;
     }
 }
diff --git a/client/src/main/java/org/elasticsearch/client/SniffingConnectionPool.java b/client/src/main/java/org/elasticsearch/client/SniffingConnectionPool.java
index 0488033c1ca..25d17b45ee5 100644
--- a/client/src/main/java/org/elasticsearch/client/SniffingConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/SniffingConnectionPool.java
@@ -21,6 +21,7 @@ package org.elasticsearch.client;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.http.HttpHost;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.impl.client.CloseableHttpClient;
 
@@ -48,7 +49,7 @@ public class SniffingConnectionPool extends AbstractStaticConnectionPool {
 
     public SniffingConnectionPool(int sniffInterval, boolean sniffOnFailure, int sniffAfterFailureDelay,
                                   CloseableHttpClient client, RequestConfig sniffRequestConfig, int sniffRequestTimeout, Scheme scheme,
-                                  Predicate<Connection> connectionSelector, Node... nodes) {
+                                  Predicate<Connection> connectionSelector, HttpHost... hosts) {
         super(connectionSelector);
         if (sniffInterval <= 0) {
             throw new IllegalArgumentException("sniffInterval must be greater than 0");
@@ -57,12 +58,12 @@ public class SniffingConnectionPool extends AbstractStaticConnectionPool {
             throw new IllegalArgumentException("sniffAfterFailureDelay must be greater than 0");
         }
         Objects.requireNonNull(scheme, "scheme cannot be null");
-        if (nodes == null || nodes.length == 0) {
-            throw new IllegalArgumentException("no nodes provided");
+        if (hosts == null || hosts.length == 0) {
+            throw new IllegalArgumentException("no hosts provided");
         }
         this.sniffOnFailure = sniffOnFailure;
         this.sniffer = new Sniffer(client, sniffRequestConfig, sniffRequestTimeout, scheme.toString());
-        this.connections = createConnections(nodes);
+        this.connections = createConnections(hosts);
         this.snifferTask = new SnifferTask(sniffInterval, sniffAfterFailureDelay);
     }
 
@@ -81,7 +82,7 @@ public class SniffingConnectionPool extends AbstractStaticConnectionPool {
         super.onFailure(connection);
         if (sniffOnFailure) {
             //re-sniff immediately but take out the node that failed
-            snifferTask.sniffOnFailure(connection.getNode());
+            snifferTask.sniffOnFailure(connection.getHost());
         }
     }
 
@@ -119,22 +120,22 @@ public class SniffingConnectionPool extends AbstractStaticConnectionPool {
             sniff(node -> true);
         }
 
-        void sniffOnFailure(Node failedNode) {
+        void sniffOnFailure(HttpHost failedHost) {
             //sync sniff straightaway on failure
             failure = true;
-            sniff(node -> node.getHttpHost().equals(failedNode.getHttpHost()) == false);
+            sniff(host -> host.equals(failedHost) == false);
         }
 
-        void sniff(Predicate<Node> nodeFilter) {
+        void sniff(Predicate<HttpHost> hostFilter) {
             if (running.compareAndSet(false, true)) {
                 try {
                     Iterator<StatefulConnection> connectionIterator = nextUnfilteredConnection().iterator();
                     if (connectionIterator.hasNext()) {
-                        sniff(connectionIterator, nodeFilter);
+                        sniff(connectionIterator, hostFilter);
                     } else {
                         StatefulConnection connection = lastResortConnection();
-                        logger.info("no healthy nodes available, trying " + connection.getNode());
-                        sniff(Stream.of(connection).iterator(), nodeFilter);
+                        logger.info("no healthy nodes available, trying " + connection.getHost());
+                        sniff(Stream.of(connection).iterator(), hostFilter);
                     }
                 } catch (Throwable t) {
                     logger.error("error while sniffing nodes", t);
@@ -158,13 +159,13 @@ public class SniffingConnectionPool extends AbstractStaticConnectionPool {
             }
         }
 
-        void sniff(Iterator<StatefulConnection> connectionIterator, Predicate<Node> nodeFilter) throws IOException {
+        void sniff(Iterator<StatefulConnection> connectionIterator, Predicate<HttpHost> hostFilter) throws IOException {
             IOException lastSeenException = null;
             while (connectionIterator.hasNext()) {
                 StatefulConnection connection = connectionIterator.next();
                 try {
-                    List<Node> sniffedNodes = sniffer.sniffNodes(connection.getNode());
-                    Node[] filteredNodes = sniffedNodes.stream().filter(nodeFilter).toArray(Node[]::new);
+                    List<HttpHost> sniffedNodes = sniffer.sniffNodes(connection.getHost());
+                    HttpHost[] filteredNodes = sniffedNodes.stream().filter(hostFilter).toArray(HttpHost[]::new);
                     logger.debug("adding " + filteredNodes.length + " nodes out of " + sniffedNodes.size() + " sniffed nodes");
                     connections = createConnections(filteredNodes);
                     onSuccess(connection);
diff --git a/client/src/main/java/org/elasticsearch/client/StatefulConnection.java b/client/src/main/java/org/elasticsearch/client/StatefulConnection.java
index 1ac72382e19..50ac78cdde9 100644
--- a/client/src/main/java/org/elasticsearch/client/StatefulConnection.java
+++ b/client/src/main/java/org/elasticsearch/client/StatefulConnection.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.client;
 
+import org.apache.http.HttpHost;
+
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -43,10 +45,10 @@ public final class StatefulConnection extends Connection {
     private volatile long deadUntil = -1;
 
     /**
-     * Creates a new mutable connection pointing to the provided {@link Node} argument
+     * Creates a new mutable connection pointing to the provided {@link HttpHost} argument
      */
-    public StatefulConnection(Node node) {
-        super(node);
+    public StatefulConnection(HttpHost host) {
+        super(host);
     }
 
     /**
diff --git a/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java b/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
index 5fb32341c99..539b7ad0f1f 100644
--- a/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
@@ -21,6 +21,7 @@ package org.elasticsearch.client;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.http.HttpHost;
 import org.apache.http.StatusLine;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
@@ -43,17 +44,17 @@ public class StaticConnectionPool extends AbstractStaticConnectionPool {
     private final List<StatefulConnection> connections;
 
     public StaticConnectionPool(CloseableHttpClient client, boolean pingEnabled, RequestConfig pingRequestConfig,
-                                Predicate<Connection> connectionSelector, Node... nodes) {
+                                Predicate<Connection> connectionSelector, HttpHost... hosts) {
         super(connectionSelector);
         Objects.requireNonNull(client, "client cannot be null");
         Objects.requireNonNull(pingRequestConfig, "pingRequestConfig cannot be null");
-        if (nodes == null || nodes.length == 0) {
-            throw new IllegalArgumentException("no nodes provided");
+        if (hosts == null || hosts.length == 0) {
+            throw new IllegalArgumentException("no hosts provided");
         }
         this.client = client;
         this.pingEnabled = pingEnabled;
         this.pingRequestConfig = pingRequestConfig;
-        this.connections = createConnections(nodes);
+        this.connections = createConnections(hosts);
     }
 
     @Override
@@ -67,20 +68,20 @@ public class StaticConnectionPool extends AbstractStaticConnectionPool {
             HttpHead httpHead = new HttpHead("/");
             httpHead.setConfig(pingRequestConfig);
             StatusLine statusLine;
-            try(CloseableHttpResponse httpResponse = client.execute(connection.getNode().getHttpHost(), httpHead)) {
+            try(CloseableHttpResponse httpResponse = client.execute(connection.getHost(), httpHead)) {
                 statusLine = httpResponse.getStatusLine();
                 EntityUtils.consume(httpResponse.getEntity());
             } catch(IOException e) {
-                RequestLogger.log(logger, "ping failed", httpHead.getRequestLine(), connection.getNode(), e);
+                RequestLogger.log(logger, "ping failed", httpHead.getRequestLine(), connection.getHost(), e);
                 onFailure(connection);
                 throw e;
             }
             if (statusLine.getStatusCode() >= 300) {
-                RequestLogger.log(logger, "ping failed", httpHead.getRequestLine(), connection.getNode(), statusLine);
+                RequestLogger.log(logger, "ping failed", httpHead.getRequestLine(), connection.getHost(), statusLine);
                 onFailure(connection);
-                throw new ElasticsearchResponseException(httpHead.getRequestLine(), connection.getNode(), statusLine);
+                throw new ElasticsearchResponseException(httpHead.getRequestLine(), connection.getHost(), statusLine);
             } else {
-                RequestLogger.log(logger, "ping succeeded", httpHead.getRequestLine(), connection.getNode(), statusLine);
+                RequestLogger.log(logger, "ping succeeded", httpHead.getRequestLine(), connection.getHost(), statusLine);
                 onSuccess(connection);
             }
         }
diff --git a/client/src/main/java/org/elasticsearch/client/Transport.java b/client/src/main/java/org/elasticsearch/client/Transport.java
index 88bf1bf6c7a..ff0835a991c 100644
--- a/client/src/main/java/org/elasticsearch/client/Transport.java
+++ b/client/src/main/java/org/elasticsearch/client/Transport.java
@@ -69,7 +69,7 @@ final class Transport<C extends Connection> implements Closeable {
         Iterator<C> connectionIterator = connectionPool.nextConnection().iterator();
         if (connectionIterator.hasNext() == false) {
             C connection = connectionPool.lastResortConnection();
-            logger.info("no healthy nodes available, trying " + connection.getNode());
+            logger.info("no healthy nodes available, trying " + connection.getHost());
             return performRequest(request, Stream.of(connection).iterator());
         }
         return performRequest(request, connectionIterator);
@@ -127,9 +127,9 @@ final class Transport<C extends Connection> implements Closeable {
     private ElasticsearchResponse performRequest(HttpRequestBase request, C connection) throws IOException {
         CloseableHttpResponse response;
         try {
-            response = client.execute(connection.getNode().getHttpHost(), request);
+            response = client.execute(connection.getHost(), request);
         } catch(IOException e) {
-            RequestLogger.log(logger, "request failed", request.getRequestLine(), connection.getNode(), e);
+            RequestLogger.log(logger, "request failed", request.getRequestLine(), connection.getHost(), e);
             throw e;
         } finally {
             request.reset();
@@ -138,12 +138,12 @@ final class Transport<C extends Connection> implements Closeable {
         //TODO make ignore status code configurable. rest-spec and tests support that parameter.
         if (statusLine.getStatusCode() < 300 ||
                 request.getMethod().equals(HttpHead.METHOD_NAME) && statusLine.getStatusCode() == 404) {
-            RequestLogger.log(logger, "request succeeded", request.getRequestLine(), connection.getNode(), response.getStatusLine());
-            return new ElasticsearchResponse(request.getRequestLine(), connection.getNode(), response);
+            RequestLogger.log(logger, "request succeeded", request.getRequestLine(), connection.getHost(), response.getStatusLine());
+            return new ElasticsearchResponse(request.getRequestLine(), connection.getHost(), response);
         } else {
             EntityUtils.consume(response.getEntity());
-            RequestLogger.log(logger, "request failed", request.getRequestLine(), connection.getNode(), response.getStatusLine());
-            throw new ElasticsearchResponseException(request.getRequestLine(), connection.getNode(), statusLine);
+            RequestLogger.log(logger, "request failed", request.getRequestLine(), connection.getHost(), response.getStatusLine());
+            throw new ElasticsearchResponseException(request.getRequestLine(), connection.getHost(), statusLine);
         }
     }
 
diff --git a/client/src/test/java/org/elasticsearch/client/NodeTests.java b/client/src/test/java/org/elasticsearch/client/NodeTests.java
deleted file mode 100644
index 73bad0f6d7b..00000000000
--- a/client/src/test/java/org/elasticsearch/client/NodeTests.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.client;
-
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
-import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-import com.carrotsearch.randomizedtesting.generators.RandomStrings;
-import org.apache.http.HttpHost;
-import org.apache.lucene.util.LuceneTestCase;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.notNullValue;
-import static org.hamcrest.CoreMatchers.sameInstance;
-
-public class NodeTests extends LuceneTestCase {
-
-    public void testSingleArgumentConstructor() {
-        HttpHost httpHost = new HttpHost(randomHost(), randomPort(), randomScheme());
-        Node node = new Node(httpHost);
-        assertThat(node.getHttpHost(), sameInstance(httpHost));
-        assertThat(node.getAttributes(), notNullValue());
-        assertThat(node.getAttributes().size(), equalTo(0));
-        assertThat(node.getRoles(), notNullValue());
-        assertThat(node.getRoles(), equalTo(new HashSet<>(Arrays.asList(Node.Role.values()))));
-
-        try {
-            new Node(null);
-            fail("node construction should have failed");
-        } catch(NullPointerException e) {
-            assertThat(e.getMessage(), equalTo("host cannot be null"));
-        }
-
-    }
-
-    public void testThreeArgumentsConstructor() {
-        HttpHost httpHost = new HttpHost(randomHost(), randomPort(), randomScheme());
-        Set<Node.Role> roles = randomRoles();
-        Map<String, String> attributes = randomAttributes();
-        Node node = new Node(httpHost, roles, attributes);
-        assertThat(node.getHttpHost(), sameInstance(httpHost));
-        assertThat(node.getAttributes(), equalTo(attributes));
-        assertThat(node.getRoles(), equalTo(roles));
-
-        try {
-            new Node(null, roles, attributes);
-            fail("node construction should have failed");
-        } catch(NullPointerException e) {
-            assertThat(e.getMessage(), equalTo("host cannot be null"));
-        }
-
-        try {
-            new Node(httpHost, null, attributes);
-            fail("node construction should have failed");
-        } catch(NullPointerException e) {
-            assertThat(e.getMessage(), equalTo("roles cannot be null"));
-        }
-
-        try {
-            new Node(httpHost, roles, null);
-            fail("node construction should have failed");
-        } catch(NullPointerException e) {
-            assertThat(e.getMessage(), equalTo("attributes cannot be null"));
-        }
-    }
-
-    public void testToString() {
-        HttpHost httpHost = new HttpHost(randomHost(), randomPort(), randomScheme());
-        Set<Node.Role> roles = randomRoles();
-        Map<String, String> attributes = randomAttributes();
-        Node node = new Node(httpHost, roles, attributes);
-        String expectedString = "Node{" +
-                "httpHost=" + httpHost.toString() +
-                ", roles=" + roles.toString() +
-                ", attributes=" + attributes.toString() +
-                '}';
-        assertThat(node.toString(), equalTo(expectedString));
-    }
-
-    private static String randomHost() {
-        return RandomStrings.randomAsciiOfLengthBetween(random(), 5, 10);
-    }
-
-    private static int randomPort() {
-        return random().nextInt();
-    }
-
-    private static String randomScheme() {
-        if (rarely()) {
-            return null;
-        }
-        return random().nextBoolean() ? "http" : "https";
-    }
-
-    private static Map<String, String> randomAttributes() {
-        int numAttributes = RandomInts.randomIntBetween(random(), 0, 5);
-        Map<String, String> attributes = new HashMap<>(numAttributes);
-        for (int i = 0; i < numAttributes; i++) {
-            String key = RandomStrings.randomAsciiOfLengthBetween(random(), 3, 10);
-            String value = RandomStrings.randomAsciiOfLengthBetween(random(), 3, 10);
-            attributes.put(key, value);
-        }
-        return attributes;
-    }
-
-    private static Set<Node.Role> randomRoles() {
-        int numRoles = RandomInts.randomIntBetween(random(), 0, 3);
-        Set<Node.Role> roles = new HashSet<>(numRoles);
-        for (int j = 0; j < numRoles; j++) {
-            roles.add(RandomPicks.randomFrom(random(), Node.Role.values()));
-        }
-        return roles;
-    }
-}
diff --git a/client/src/test/java/org/elasticsearch/client/SnifferTests.java b/client/src/test/java/org/elasticsearch/client/SnifferTests.java
index 9169afd7695..c2caa5ab4fa 100644
--- a/client/src/test/java/org/elasticsearch/client/SnifferTests.java
+++ b/client/src/test/java/org/elasticsearch/client/SnifferTests.java
@@ -90,24 +90,21 @@ public class SnifferTests extends LuceneTestCase {
         Sniffer sniffer = new Sniffer(client, RequestConfig.DEFAULT, sniffRequestTimeout, scheme.toString());
         HttpHost httpHost = new HttpHost(server.getHostName(), server.getPort());
         try {
-            List<Node> sniffedNodes = sniffer.sniffNodes(new Node(httpHost));
+            List<HttpHost> sniffedHosts = sniffer.sniffNodes(httpHost);
             if (sniffResponse.isFailure) {
                 fail("sniffNodes should have failed");
             }
-            assertThat(sniffedNodes.size(), equalTo(sniffResponse.nodes.size()));
-            Iterator<Node> responseNodesIterator = sniffResponse.nodes.iterator();
-            for (Node sniffedNode : sniffedNodes) {
-                Node responseNode = responseNodesIterator.next();
-                assertThat(sniffedNode.getHttpHost(), equalTo(responseNode.getHttpHost()));
-                assertThat(sniffedNode.getRoles(), equalTo(responseNode.getRoles()));
-                assertThat(sniffedNode.getAttributes(), equalTo(responseNode.getAttributes()));
+            assertThat(sniffedHosts.size(), equalTo(sniffResponse.hosts.size()));
+            Iterator<HttpHost> responseHostsIterator = sniffResponse.hosts.iterator();
+            for (HttpHost sniffedHost : sniffedHosts) {
+                assertEquals(sniffedHost, responseHostsIterator.next());
             }
         } catch(ElasticsearchResponseException e) {
             if (sniffResponse.isFailure) {
                 assertThat(e.getMessage(), containsString("GET http://localhost:" + server.getPort() +
                         "/_nodes/http?timeout=" + sniffRequestTimeout));
                 assertThat(e.getMessage(), containsString(Integer.toString(sniffResponse.nodesInfoResponseCode)));
-                assertThat(e.getNode().getHttpHost(), equalTo(httpHost));
+                assertThat(e.getHost(), equalTo(httpHost));
                 assertThat(e.getStatusLine().getStatusCode(), equalTo(sniffResponse.nodesInfoResponseCode));
                 assertThat(e.getRequestLine().toString(), equalTo("GET /_nodes/http?timeout=" + sniffRequestTimeout + "ms HTTP/1.1"));
             } else {
@@ -141,7 +138,7 @@ public class SnifferTests extends LuceneTestCase {
 
     private static SniffResponse buildSniffResponse(SniffingConnectionPool.Scheme scheme) throws IOException {
         int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
-        List<Node> nodes = new ArrayList<>(numNodes);
+        List<HttpHost> hosts = new ArrayList<>(numNodes);
         JsonFactory jsonFactory = new JsonFactory();
         StringWriter writer = new StringWriter();
         JsonGenerator generator = jsonFactory.createGenerator(writer);
@@ -168,25 +165,11 @@ public class SnifferTests extends LuceneTestCase {
                 generator.writeEndArray();
             }
             boolean isHttpEnabled = rarely() == false;
-            int numRoles = RandomInts.randomIntBetween(random(), 0, 3);
-            Set<Node.Role> nodeRoles = new HashSet<>(numRoles);
-            for (int j = 0; j < numRoles; j++) {
-                Node.Role role;
-                do {
-                    role = RandomPicks.randomFrom(random(), Node.Role.values());
-                } while(nodeRoles.add(role) == false);
-            }
-
-            int numAttributes = RandomInts.randomIntBetween(random(), 0, 3);
-            Map<String, String> attributes = new HashMap<>(numAttributes);
-            for (int j = 0; j < numAttributes; j++) {
-                attributes.put("attr" + j, "value" + j);
-            }
             if (isHttpEnabled) {
                 String host = "host" + i;
                 int port = RandomInts.randomIntBetween(random(), 9200, 9299);
                 HttpHost httpHost = new HttpHost(host, port, scheme.toString());
-                nodes.add(new Node(httpHost, nodeRoles, attributes));
+                hosts.add(httpHost);
                 generator.writeObjectFieldStart("http");
                 if (random().nextBoolean()) {
                     generator.writeArrayFieldStart("bound_address");
@@ -205,11 +188,25 @@ public class SnifferTests extends LuceneTestCase {
                 }
                 generator.writeEndObject();
             }
+            String[] roles = {"master", "data", "ingest"};
+            int numRoles = RandomInts.randomIntBetween(random(), 0, 3);
+            Set<String> nodeRoles = new HashSet<>(numRoles);
+            for (int j = 0; j < numRoles; j++) {
+                String role;
+                do {
+                    role = RandomPicks.randomFrom(random(), roles);
+                } while(nodeRoles.add(role) == false);
+            }
             generator.writeArrayFieldStart("roles");
-            for (Node.Role nodeRole : nodeRoles) {
-                generator.writeString(nodeRole.toString());
+            for (String nodeRole : nodeRoles) {
+                generator.writeString(nodeRole);
             }
             generator.writeEndArray();
+            int numAttributes = RandomInts.randomIntBetween(random(), 0, 3);
+            Map<String, String> attributes = new HashMap<>(numAttributes);
+            for (int j = 0; j < numAttributes; j++) {
+                attributes.put("attr" + j, "value" + j);
+            }
             if (numAttributes > 0) {
                 generator.writeObjectFieldStart("attributes");
             }
@@ -224,18 +221,18 @@ public class SnifferTests extends LuceneTestCase {
         generator.writeEndObject();
         generator.writeEndObject();
         generator.close();
-        return SniffResponse.buildResponse(writer.toString(), nodes);
+        return SniffResponse.buildResponse(writer.toString(), hosts);
     }
 
     private static class SniffResponse {
         private final String nodesInfoBody;
         private final int nodesInfoResponseCode;
-        private final List<Node> nodes;
+        private final List<HttpHost> hosts;
         private final boolean isFailure;
 
-        SniffResponse(String nodesInfoBody, List<Node> nodes, boolean isFailure) {
+        SniffResponse(String nodesInfoBody, List<HttpHost> hosts, boolean isFailure) {
             this.nodesInfoBody = nodesInfoBody;
-            this.nodes = nodes;
+            this.hosts = hosts;
             this.isFailure = isFailure;
             if (isFailure) {
                 this.nodesInfoResponseCode = randomErrorResponseCode();
@@ -248,8 +245,8 @@ public class SnifferTests extends LuceneTestCase {
             return new SniffResponse("", Collections.emptyList(), true);
         }
 
-        static SniffResponse buildResponse(String nodesInfoBody, List<Node> nodes) {
-            return new SniffResponse(nodesInfoBody, nodes, false);
+        static SniffResponse buildResponse(String nodesInfoBody, List<HttpHost> hosts) {
+            return new SniffResponse(nodesInfoBody, hosts, false);
         }
     }
 
diff --git a/client/src/test/java/org/elasticsearch/client/SniffingConnectionPoolTests.java b/client/src/test/java/org/elasticsearch/client/SniffingConnectionPoolTests.java
index 13eaadd619a..97b7d0867c6 100644
--- a/client/src/test/java/org/elasticsearch/client/SniffingConnectionPoolTests.java
+++ b/client/src/test/java/org/elasticsearch/client/SniffingConnectionPoolTests.java
@@ -38,16 +38,16 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
     public void testConstructor() throws Exception {
         CloseableHttpClient httpClient = HttpClientBuilder.create().build();
         int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
-        Node[] nodes = new Node[numNodes];
+        HttpHost[] hosts = new HttpHost[numNodes];
         for (int i = 0; i < numNodes; i++) {
-            nodes[i] = new Node(new HttpHost("localhost", 9200));
+            hosts[i] = new HttpHost("localhost", 9200);
         }
 
         try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
                 RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), nodes)) {
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), hosts)) {
 
             fail("pool creation should have failed " + connectionPool);
         } catch(IllegalArgumentException e) {
@@ -58,7 +58,7 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0), httpClient, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), nodes)) {
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), hosts)) {
             fail("pool creation should have failed " + connectionPool);
         } catch(IllegalArgumentException e) {
             assertEquals(e.getMessage(), "sniffAfterFailureDelay must be greater than 0");
@@ -68,7 +68,7 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), null, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), nodes)) {
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), hosts)) {
             fail("pool creation should have failed " + connectionPool);
         } catch(NullPointerException e) {
             assertEquals(e.getMessage(), "client cannot be null");
@@ -78,7 +78,7 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, null,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), nodes)) {
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), hosts)) {
             fail("pool creation should have failed " + connectionPool);
         } catch(NullPointerException e) {
             assertEquals(e.getMessage(), "sniffRequestConfig cannot be null");
@@ -88,7 +88,7 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), nodes)) {
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), hosts)) {
             fail("pool creation should have failed " + connectionPool);
         } catch(IllegalArgumentException e) {
             assertEquals(e.getMessage(), "sniffRequestTimeout must be greater than 0");
@@ -98,7 +98,7 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), null, nodes)) {
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), null, hosts)) {
             fail("pool creation should have failed " + connectionPool);
         } catch(NullPointerException e) {
             assertEquals(e.getMessage(), "connection selector predicate cannot be null");
@@ -109,10 +109,10 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
                 RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()),
-                connection -> random().nextBoolean(), (Node[])null)) {
+                connection -> random().nextBoolean(), (HttpHost[])null)) {
             fail("pool creation should have failed " + connectionPool);
         } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "no nodes provided");
+            assertEquals(e.getMessage(), "no hosts provided");
         }
 
         try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
@@ -120,10 +120,10 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
                 RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(),
-                (Node)null)) {
+                (HttpHost) null)) {
             fail("pool creation should have failed " + connectionPool);
         } catch(NullPointerException e) {
-            assertEquals(e.getMessage(), "node cannot be null");
+            assertEquals(e.getMessage(), "host cannot be null");
         }
 
         try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
@@ -133,14 +133,14 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean())) {
             fail("pool creation should have failed " + connectionPool);
         } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "no nodes provided");
+            assertEquals(e.getMessage(), "no hosts provided");
         }
 
         try (SniffingConnectionPool sniffingConnectionPool = new SniffingConnectionPool(
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), nodes)) {
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), hosts)) {
             assertNotNull(sniffingConnectionPool);
         }
     }
diff --git a/client/src/test/java/org/elasticsearch/client/StaticConnectionPoolTests.java b/client/src/test/java/org/elasticsearch/client/StaticConnectionPoolTests.java
index 9b5b29df1b1..82e8c60b02f 100644
--- a/client/src/test/java/org/elasticsearch/client/StaticConnectionPoolTests.java
+++ b/client/src/test/java/org/elasticsearch/client/StaticConnectionPoolTests.java
@@ -37,51 +37,51 @@ public class StaticConnectionPoolTests extends LuceneTestCase {
     public void testConstructor() {
         CloseableHttpClient httpClient = HttpClientBuilder.create().build();
         int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
-        Node[] nodes = new Node[numNodes];
+        HttpHost[] hosts = new HttpHost[numNodes];
         for (int i = 0; i < numNodes; i++) {
-            nodes[i] = new Node(new HttpHost("localhost", 9200));
+            hosts[i] = new HttpHost("localhost", 9200);
         }
 
         try {
-            new StaticConnectionPool(null, random().nextBoolean(), RequestConfig.DEFAULT, connection -> random().nextBoolean(), nodes);
+            new StaticConnectionPool(null, random().nextBoolean(), RequestConfig.DEFAULT, connection -> random().nextBoolean(), hosts);
         } catch(NullPointerException e) {
             assertEquals(e.getMessage(), "client cannot be null");
         }
 
         try {
-            new StaticConnectionPool(httpClient, random().nextBoolean(), null, connection -> random().nextBoolean(), nodes);
+            new StaticConnectionPool(httpClient, random().nextBoolean(), null, connection -> random().nextBoolean(), hosts);
         } catch(NullPointerException e) {
             assertEquals(e.getMessage(), "pingRequestConfig cannot be null");
         }
 
         try {
-            new StaticConnectionPool(httpClient, random().nextBoolean(), RequestConfig.DEFAULT, null, nodes);
+            new StaticConnectionPool(httpClient, random().nextBoolean(), RequestConfig.DEFAULT, null, hosts);
         } catch(NullPointerException e) {
             assertEquals(e.getMessage(), "connection selector predicate cannot be null");
         }
 
         try {
             new StaticConnectionPool(httpClient, random().nextBoolean(), RequestConfig.DEFAULT,
-                    connection -> random().nextBoolean(), (Node)null);
+                    connection -> random().nextBoolean(), (HttpHost) null);
         } catch(NullPointerException e) {
-            assertEquals(e.getMessage(), "node cannot be null");
+            assertEquals(e.getMessage(), "host cannot be null");
         }
 
         try {
             new StaticConnectionPool(httpClient, random().nextBoolean(), RequestConfig.DEFAULT,
-                    connection -> random().nextBoolean(), (Node[])null);
+                    connection -> random().nextBoolean(), (HttpHost[])null);
         } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "no nodes provided");
+            assertEquals(e.getMessage(), "no hosts provided");
         }
 
         try {
             new StaticConnectionPool(httpClient, random().nextBoolean(), RequestConfig.DEFAULT, connection -> random().nextBoolean());
         } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "no nodes provided");
+            assertEquals(e.getMessage(), "no hosts provided");
         }
 
         StaticConnectionPool staticConnectionPool = new StaticConnectionPool(httpClient, random().nextBoolean(), RequestConfig.DEFAULT,
-                connection -> random().nextBoolean(), nodes);
+                connection -> random().nextBoolean(), hosts);
         assertNotNull(staticConnectionPool);
     }
 }

From ce663e9703e1086bd1565a2592f3af2f22149e31 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Wed, 4 May 2016 16:36:13 +0200
Subject: [PATCH 0585/1311] get rid of connection selector predicate

---
 .../client/AbstractStaticConnectionPool.java  | 11 -------
 .../client/SniffingConnectionPool.java        |  5 ++--
 .../client/StaticConnectionPool.java          |  5 +---
 .../client/SniffingConnectionPoolTests.java   | 30 ++++++-------------
 .../client/StaticConnectionPoolTests.java     | 20 ++++---------
 5 files changed, 18 insertions(+), 53 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java b/client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java
index c043dd17059..50923fa5406 100644
--- a/client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java
@@ -47,23 +47,12 @@ public abstract class AbstractStaticConnectionPool implements ConnectionPool<Sta
 
     private static final Log logger = LogFactory.getLog(AbstractStaticConnectionPool.class);
 
-    private final Predicate<Connection> connectionSelector;
-
     private final AtomicInteger lastConnectionIndex = new AtomicInteger(0);
 
-    protected AbstractStaticConnectionPool(Predicate<Connection> connectionSelector) {
-        Objects.requireNonNull(connectionSelector, "connection selector predicate cannot be null");
-        this.connectionSelector = connectionSelector;
-    }
-
     protected abstract List<StatefulConnection> getConnections();
 
     @Override
     public final Stream<StatefulConnection> nextConnection() {
-        return nextUnfilteredConnection().filter(connectionSelector);
-    }
-
-    protected final Stream<StatefulConnection> nextUnfilteredConnection() {
         List<StatefulConnection> connections = getConnections();
         if (connections.isEmpty()) {
             throw new IllegalStateException("no connections available in the connection pool");
diff --git a/client/src/main/java/org/elasticsearch/client/SniffingConnectionPool.java b/client/src/main/java/org/elasticsearch/client/SniffingConnectionPool.java
index 25d17b45ee5..eaaf98d0506 100644
--- a/client/src/main/java/org/elasticsearch/client/SniffingConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/SniffingConnectionPool.java
@@ -49,8 +49,7 @@ public class SniffingConnectionPool extends AbstractStaticConnectionPool {
 
     public SniffingConnectionPool(int sniffInterval, boolean sniffOnFailure, int sniffAfterFailureDelay,
                                   CloseableHttpClient client, RequestConfig sniffRequestConfig, int sniffRequestTimeout, Scheme scheme,
-                                  Predicate<Connection> connectionSelector, HttpHost... hosts) {
-        super(connectionSelector);
+                                  HttpHost... hosts) {
         if (sniffInterval <= 0) {
             throw new IllegalArgumentException("sniffInterval must be greater than 0");
         }
@@ -129,7 +128,7 @@ public class SniffingConnectionPool extends AbstractStaticConnectionPool {
         void sniff(Predicate<HttpHost> hostFilter) {
             if (running.compareAndSet(false, true)) {
                 try {
-                    Iterator<StatefulConnection> connectionIterator = nextUnfilteredConnection().iterator();
+                    Iterator<StatefulConnection> connectionIterator = nextConnection().iterator();
                     if (connectionIterator.hasNext()) {
                         sniff(connectionIterator, hostFilter);
                     } else {
diff --git a/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java b/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
index 539b7ad0f1f..602298b1b3b 100644
--- a/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
@@ -32,7 +32,6 @@ import org.apache.http.util.EntityUtils;
 import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
-import java.util.function.Predicate;
 
 public class StaticConnectionPool extends AbstractStaticConnectionPool {
 
@@ -43,9 +42,7 @@ public class StaticConnectionPool extends AbstractStaticConnectionPool {
     private final RequestConfig pingRequestConfig;
     private final List<StatefulConnection> connections;
 
-    public StaticConnectionPool(CloseableHttpClient client, boolean pingEnabled, RequestConfig pingRequestConfig,
-                                Predicate<Connection> connectionSelector, HttpHost... hosts) {
-        super(connectionSelector);
+    public StaticConnectionPool(CloseableHttpClient client, boolean pingEnabled, RequestConfig pingRequestConfig, HttpHost... hosts) {
         Objects.requireNonNull(client, "client cannot be null");
         Objects.requireNonNull(pingRequestConfig, "pingRequestConfig cannot be null");
         if (hosts == null || hosts.length == 0) {
diff --git a/client/src/test/java/org/elasticsearch/client/SniffingConnectionPoolTests.java b/client/src/test/java/org/elasticsearch/client/SniffingConnectionPoolTests.java
index 97b7d0867c6..1ae9279f78e 100644
--- a/client/src/test/java/org/elasticsearch/client/SniffingConnectionPoolTests.java
+++ b/client/src/test/java/org/elasticsearch/client/SniffingConnectionPoolTests.java
@@ -47,7 +47,7 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), hosts)) {
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), hosts)) {
 
             fail("pool creation should have failed " + connectionPool);
         } catch(IllegalArgumentException e) {
@@ -58,7 +58,7 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0), httpClient, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), hosts)) {
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), hosts)) {
             fail("pool creation should have failed " + connectionPool);
         } catch(IllegalArgumentException e) {
             assertEquals(e.getMessage(), "sniffAfterFailureDelay must be greater than 0");
@@ -68,7 +68,7 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), null, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), hosts)) {
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), hosts)) {
             fail("pool creation should have failed " + connectionPool);
         } catch(NullPointerException e) {
             assertEquals(e.getMessage(), "client cannot be null");
@@ -78,7 +78,7 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, null,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), hosts)) {
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), hosts)) {
             fail("pool creation should have failed " + connectionPool);
         } catch(NullPointerException e) {
             assertEquals(e.getMessage(), "sniffRequestConfig cannot be null");
@@ -88,7 +88,7 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), hosts)) {
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), hosts)) {
             fail("pool creation should have failed " + connectionPool);
         } catch(IllegalArgumentException e) {
             assertEquals(e.getMessage(), "sniffRequestTimeout must be greater than 0");
@@ -98,18 +98,7 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), null, hosts)) {
-            fail("pool creation should have failed " + connectionPool);
-        } catch(NullPointerException e) {
-            assertEquals(e.getMessage(), "connection selector predicate cannot be null");
-        }
-
-        try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()),
-                connection -> random().nextBoolean(), (HttpHost[])null)) {
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), (HttpHost[])null)) {
             fail("pool creation should have failed " + connectionPool);
         } catch(IllegalArgumentException e) {
             assertEquals(e.getMessage(), "no hosts provided");
@@ -119,8 +108,7 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(),
-                (HttpHost) null)) {
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), (HttpHost) null)) {
             fail("pool creation should have failed " + connectionPool);
         } catch(NullPointerException e) {
             assertEquals(e.getMessage(), "host cannot be null");
@@ -130,7 +118,7 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean())) {
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()))) {
             fail("pool creation should have failed " + connectionPool);
         } catch(IllegalArgumentException e) {
             assertEquals(e.getMessage(), "no hosts provided");
@@ -140,7 +128,7 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), connection -> random().nextBoolean(), hosts)) {
+                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), hosts)) {
             assertNotNull(sniffingConnectionPool);
         }
     }
diff --git a/client/src/test/java/org/elasticsearch/client/StaticConnectionPoolTests.java b/client/src/test/java/org/elasticsearch/client/StaticConnectionPoolTests.java
index 82e8c60b02f..60398b71c0d 100644
--- a/client/src/test/java/org/elasticsearch/client/StaticConnectionPoolTests.java
+++ b/client/src/test/java/org/elasticsearch/client/StaticConnectionPoolTests.java
@@ -43,45 +43,37 @@ public class StaticConnectionPoolTests extends LuceneTestCase {
         }
 
         try {
-            new StaticConnectionPool(null, random().nextBoolean(), RequestConfig.DEFAULT, connection -> random().nextBoolean(), hosts);
+            new StaticConnectionPool(null, random().nextBoolean(), RequestConfig.DEFAULT, hosts);
         } catch(NullPointerException e) {
             assertEquals(e.getMessage(), "client cannot be null");
         }
 
         try {
-            new StaticConnectionPool(httpClient, random().nextBoolean(), null, connection -> random().nextBoolean(), hosts);
+            new StaticConnectionPool(httpClient, random().nextBoolean(), null, hosts);
         } catch(NullPointerException e) {
             assertEquals(e.getMessage(), "pingRequestConfig cannot be null");
         }
 
         try {
-            new StaticConnectionPool(httpClient, random().nextBoolean(), RequestConfig.DEFAULT, null, hosts);
-        } catch(NullPointerException e) {
-            assertEquals(e.getMessage(), "connection selector predicate cannot be null");
-        }
-
-        try {
-            new StaticConnectionPool(httpClient, random().nextBoolean(), RequestConfig.DEFAULT,
-                    connection -> random().nextBoolean(), (HttpHost) null);
+            new StaticConnectionPool(httpClient, random().nextBoolean(), RequestConfig.DEFAULT, (HttpHost) null);
         } catch(NullPointerException e) {
             assertEquals(e.getMessage(), "host cannot be null");
         }
 
         try {
-            new StaticConnectionPool(httpClient, random().nextBoolean(), RequestConfig.DEFAULT,
-                    connection -> random().nextBoolean(), (HttpHost[])null);
+            new StaticConnectionPool(httpClient, random().nextBoolean(), RequestConfig.DEFAULT, (HttpHost[])null);
         } catch(IllegalArgumentException e) {
             assertEquals(e.getMessage(), "no hosts provided");
         }
 
         try {
-            new StaticConnectionPool(httpClient, random().nextBoolean(), RequestConfig.DEFAULT, connection -> random().nextBoolean());
+            new StaticConnectionPool(httpClient, random().nextBoolean(), RequestConfig.DEFAULT);
         } catch(IllegalArgumentException e) {
             assertEquals(e.getMessage(), "no hosts provided");
         }
 
         StaticConnectionPool staticConnectionPool = new StaticConnectionPool(httpClient, random().nextBoolean(), RequestConfig.DEFAULT,
-                connection -> random().nextBoolean(), hosts);
+                hosts);
         assertNotNull(staticConnectionPool);
     }
 }

From a472544ab4d3c608e1299617234be3da1d8d4e69 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Wed, 4 May 2016 17:06:30 +0200
Subject: [PATCH 0586/1311] move sniff related stuff to sniff package

---
 .../java/org/elasticsearch/client/{ => sniff}/Sniffer.java   | 4 +++-
 .../client/{ => sniff}/SniffingConnectionPool.java           | 4 +++-
 .../org/elasticsearch/client/{ => sniff}/SnifferTests.java   | 5 ++++-
 .../client/{ => sniff}/SniffingConnectionPoolTests.java      | 2 +-
 4 files changed, 11 insertions(+), 4 deletions(-)
 rename client/src/main/java/org/elasticsearch/client/{ => sniff}/Sniffer.java (97%)
 rename client/src/main/java/org/elasticsearch/client/{ => sniff}/SniffingConnectionPool.java (98%)
 rename client/src/test/java/org/elasticsearch/client/{ => sniff}/SnifferTests.java (98%)
 rename client/src/test/java/org/elasticsearch/client/{ => sniff}/SniffingConnectionPoolTests.java (99%)

diff --git a/client/src/main/java/org/elasticsearch/client/Sniffer.java b/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
similarity index 97%
rename from client/src/main/java/org/elasticsearch/client/Sniffer.java
rename to client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
index 1bec4eb87e1..a812804cbce 100644
--- a/client/src/main/java/org/elasticsearch/client/Sniffer.java
+++ b/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.client;
+package org.elasticsearch.client.sniff;
 
 import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonParser;
@@ -32,6 +32,8 @@ import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.ElasticsearchResponseException;
+import org.elasticsearch.client.RequestLogger;
 
 import java.io.IOException;
 import java.io.InputStream;
diff --git a/client/src/main/java/org/elasticsearch/client/SniffingConnectionPool.java b/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
similarity index 98%
rename from client/src/main/java/org/elasticsearch/client/SniffingConnectionPool.java
rename to client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
index eaaf98d0506..d0171e6b952 100644
--- a/client/src/main/java/org/elasticsearch/client/SniffingConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
@@ -17,13 +17,15 @@
  * under the License.
  */
 
-package org.elasticsearch.client;
+package org.elasticsearch.client.sniff;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.http.HttpHost;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.impl.client.CloseableHttpClient;
+import org.elasticsearch.client.AbstractStaticConnectionPool;
+import org.elasticsearch.client.StatefulConnection;
 
 import java.io.IOException;
 import java.util.Iterator;
diff --git a/client/src/test/java/org/elasticsearch/client/SnifferTests.java b/client/src/test/java/org/elasticsearch/client/sniff/SnifferTests.java
similarity index 98%
rename from client/src/test/java/org/elasticsearch/client/SnifferTests.java
rename to client/src/test/java/org/elasticsearch/client/sniff/SnifferTests.java
index c2caa5ab4fa..28a9697ac37 100644
--- a/client/src/test/java/org/elasticsearch/client/SnifferTests.java
+++ b/client/src/test/java/org/elasticsearch/client/sniff/SnifferTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.client;
+package org.elasticsearch.client.sniff;
 
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
@@ -33,6 +33,9 @@ import org.apache.http.client.config.RequestConfig;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.client.ElasticsearchResponseException;
+import org.elasticsearch.client.sniff.Sniffer;
+import org.elasticsearch.client.sniff.SniffingConnectionPool;
 import org.junit.After;
 import org.junit.Before;
 
diff --git a/client/src/test/java/org/elasticsearch/client/SniffingConnectionPoolTests.java b/client/src/test/java/org/elasticsearch/client/sniff/SniffingConnectionPoolTests.java
similarity index 99%
rename from client/src/test/java/org/elasticsearch/client/SniffingConnectionPoolTests.java
rename to client/src/test/java/org/elasticsearch/client/sniff/SniffingConnectionPoolTests.java
index 1ae9279f78e..06e6b43236b 100644
--- a/client/src/test/java/org/elasticsearch/client/SniffingConnectionPoolTests.java
+++ b/client/src/test/java/org/elasticsearch/client/sniff/SniffingConnectionPoolTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.client;
+package org.elasticsearch.client.sniff;
 
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;

From 062a21678c144262d98e1f1cf5b0b74071c31463 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 6 May 2016 09:52:33 +0200
Subject: [PATCH 0587/1311] merge Connection and StatefulConnection into one
 class, remove generic type from Transport

---
 .../client/AbstractStaticConnectionPool.java  |  32 ++---
 .../org/elasticsearch/client/Connection.java  |  88 +++++++++++-
 .../elasticsearch/client/ConnectionPool.java  |  18 ++-
 .../ElasticsearchResponseException.java       |   2 +-
 .../org/elasticsearch/client/RestClient.java  |   4 +-
 .../client/StatefulConnection.java            | 127 ------------------
 .../client/StaticConnectionPool.java          |   8 +-
 .../org/elasticsearch/client/Transport.java   |  16 +--
 .../client/sniff/SniffingConnectionPool.java  |  19 +--
 .../elasticsearch/client/TransportTests.java  |  10 +-
 10 files changed, 139 insertions(+), 185 deletions(-)
 delete mode 100644 client/src/main/java/org/elasticsearch/client/StatefulConnection.java

diff --git a/client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java b/client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java
index 50923fa5406..192c1b0f35a 100644
--- a/client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java
@@ -38,57 +38,57 @@ import java.util.stream.Stream;
  * allows to filter connections through a customizable {@link Predicate}, called connection selector.
  * In case the returned stream is empty a last resort dead connection should be retrieved by calling {@link #lastResortConnection()}
  * and resurrected so that a single request attempt can be performed.
- * The {@link #onSuccess(StatefulConnection)} method marks the connection provided as an argument alive.
- * The {@link #onFailure(StatefulConnection)} method marks the connection provided as an argument dead.
+ * The {@link #onSuccess(Connection)} method marks the connection provided as an argument alive.
+ * The {@link #onFailure(Connection)} method marks the connection provided as an argument dead.
  * This base implementation doesn't define the list implementation that stores connections, so that concurrency can be
  * handled in the subclasses depending on the usecase (e.g. defining the list volatile when needed).
  */
-public abstract class AbstractStaticConnectionPool implements ConnectionPool<StatefulConnection> {
+public abstract class AbstractStaticConnectionPool implements ConnectionPool {
 
     private static final Log logger = LogFactory.getLog(AbstractStaticConnectionPool.class);
 
     private final AtomicInteger lastConnectionIndex = new AtomicInteger(0);
 
-    protected abstract List<StatefulConnection> getConnections();
+    protected abstract List<Connection> getConnections();
 
     @Override
-    public final Stream<StatefulConnection> nextConnection() {
-        List<StatefulConnection> connections = getConnections();
+    public final Stream<Connection> nextConnection() {
+        List<Connection> connections = getConnections();
         if (connections.isEmpty()) {
             throw new IllegalStateException("no connections available in the connection pool");
         }
 
-        List<StatefulConnection> sortedConnections = new ArrayList<>(connections);
+        List<Connection> sortedConnections = new ArrayList<>(connections);
         //TODO is it possible to make this O(1)? (rotate is O(n))
         Collections.rotate(sortedConnections, sortedConnections.size() - lastConnectionIndex.getAndIncrement());
         return sortedConnections.stream().filter(connection -> connection.isAlive() || connection.shouldBeRetried());
     }
 
-    protected List<StatefulConnection> createConnections(HttpHost... hosts) {
-        List<StatefulConnection> connections = new ArrayList<>();
+    protected List<Connection> createConnections(HttpHost... hosts) {
+        List<Connection> connections = new ArrayList<>();
         for (HttpHost host : hosts) {
             Objects.requireNonNull(host, "host cannot be null");
-            connections.add(new StatefulConnection(host));
+            connections.add(new Connection(host));
         }
         return Collections.unmodifiableList(connections);
     }
 
     @Override
-    public StatefulConnection lastResortConnection() {
-        StatefulConnection statefulConnection = getConnections().stream()
+    public Connection lastResortConnection() {
+        Connection Connection = getConnections().stream()
                 .sorted((o1, o2) -> Long.compare(o1.getDeadUntil(), o2.getDeadUntil())).findFirst().get();
-        statefulConnection.markResurrected();
-        return statefulConnection;
+        Connection.markResurrected();
+        return Connection;
     }
 
     @Override
-    public void onSuccess(StatefulConnection connection) {
+    public void onSuccess(Connection connection) {
         connection.markAlive();
         logger.trace("marked connection alive for " + connection.getHost());
     }
 
     @Override
-    public void onFailure(StatefulConnection connection) throws IOException {
+    public void onFailure(Connection connection) throws IOException {
         connection.markDead();
         logger.debug("marked connection dead for " + connection.getHost());
     }
diff --git a/client/src/main/java/org/elasticsearch/client/Connection.java b/client/src/main/java/org/elasticsearch/client/Connection.java
index 51a430b2540..3d48a4eeae1 100644
--- a/client/src/main/java/org/elasticsearch/client/Connection.java
+++ b/client/src/main/java/org/elasticsearch/client/Connection.java
@@ -21,15 +21,21 @@ package org.elasticsearch.client;
 
 import org.apache.http.HttpHost;
 
+import java.util.concurrent.TimeUnit;
+
 /**
- * Simplest representation of a connection to an elasticsearch node.
- * It doesn't have any mutable state. It holds the host that the connection points to.
+ * Represents a connection to a host. It holds the host that the connection points to.
  * Allows the transport to deal with very simple connection objects that are immutable.
- * Any change to the state of connections should be made through the connection pool
- * which is aware of the connection object that it supports.
+ * Any change to the state of a connection should be made through the connection pool.
  */
 public class Connection {
+    //TODO make these values configurable through the connection pool?
+    private static final long DEFAULT_CONNECTION_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(1);
+    private static final long MAX_CONNECTION_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(30);
     private final HttpHost host;
+    private volatile State state = State.UNKNOWN;
+    private volatile int failedAttempts = -1;
+    private volatile long deadUntil = -1;
 
     /**
      * Creates a new connection pointing to the provided {@link HttpHost} argument
@@ -44,4 +50,78 @@ public class Connection {
     public HttpHost getHost() {
             return host;
         }
+
+    /**
+     * Marks connection as dead. Should be called in case the corresponding node is not responding or caused failures.
+     * Once marked dead, the number of failed attempts will be incremented on each call to this method. A dead connection
+     * should be retried once {@link #shouldBeRetried()} returns true, which depends on the number of previous failed attempts
+     * and when the last failure was registered.
+     */
+    void markDead() {
+        synchronized (this) {
+            int failedAttempts = Math.max(this.failedAttempts, 0);
+            long timeoutMillis = (long)Math.min(DEFAULT_CONNECTION_TIMEOUT_MILLIS * 2 * Math.pow(2, failedAttempts * 0.5 - 1),
+                    MAX_CONNECTION_TIMEOUT_MILLIS);
+            this.deadUntil = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis);
+            this.failedAttempts = ++failedAttempts;
+            this.state = State.DEAD;
+        }
+    }
+
+    /**
+     * Marks this connection alive. Should be called when the corresponding node is working properly.
+     * Will reset the number of failed attempts that were counted in case the connection was previously dead,
+     * as well as its dead timeout.
+     */
+    void markAlive() {
+        if (this.state != State.ALIVE) {
+            synchronized (this) {
+                this.deadUntil = -1;
+                this.failedAttempts = 0;
+                this.state = State.ALIVE;
+            }
+        }
+    }
+
+    /**
+     * Resets the connection to its initial state, so it will be retried. To be called when all the connections in the pool
+     * are dead, so that one connection can be retried. Note that calling this method only changes the state of the connection,
+     * it doesn't reset its failed attempts and dead until timestamp. That way if the connection goes back to dead straightaway
+     * all of its previous failed attempts are taken into account.
+     */
+    void markResurrected() {
+        if (this.state == State.DEAD) {
+            synchronized (this) {
+                this.state = State.UNKNOWN;
+            }
+        }
+    }
+
+    /**
+     * Returns the timestamp till the connection is supposed to stay dead till it can be retried
+     */
+    public long getDeadUntil() {
+        return deadUntil;
+    }
+
+    /**
+     * Returns true if the connection is alive, false otherwise.
+     */
+    public boolean isAlive() {
+        return state == State.ALIVE;
+    }
+
+    /**
+     * Returns true in case the connection is not alive but should be used/retried, false otherwise.
+     * Returns true in case the connection is in unknown state (never used before) or resurrected. When the connection is dead,
+     * returns true when it is time to retry it, depending on how many failed attempts were registered and when the last failure
+     * happened (minimum 1 minute, maximum 30 minutes).
+     */
+    public boolean shouldBeRetried() {
+        return state == State.UNKNOWN || (state == State.DEAD && System.nanoTime() - deadUntil >= 0);
+    }
+
+    private enum State {
+        UNKNOWN, DEAD, ALIVE
+    }
 }
diff --git a/client/src/main/java/org/elasticsearch/client/ConnectionPool.java b/client/src/main/java/org/elasticsearch/client/ConnectionPool.java
index 3f2eb89da97..7e47a819256 100644
--- a/client/src/main/java/org/elasticsearch/client/ConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/ConnectionPool.java
@@ -24,14 +24,12 @@ import java.io.IOException;
 import java.util.stream.Stream;
 
 /**
- * Pool of connections to the different nodes that belong to an elasticsearch cluster.
- * It keeps track of the different nodes to communicate with and allows to retrieve a stream of connections to be used
+ * Pool of connections to the different hosts that belong to an elasticsearch cluster.
+ * It keeps track of the different hosts to communicate with and allows to retrieve a stream of connections to be used
  * for each request. Exposes the needed hooks to be able to eventually mark connections dead or alive and execute
  * arbitrary operations before each single request attempt.
- *
- * @param <C> the type of {@link Connection} that the pool supports
  */
-public interface ConnectionPool<C extends Connection> extends Closeable {
+public interface ConnectionPool extends Closeable {
 
     /**
      * Returns a stream of connections that should be used for a request call.
@@ -41,29 +39,29 @@ public interface ConnectionPool<C extends Connection> extends Closeable {
      * It may happen that the stream is empty, in which case it means that there aren't healthy connections to use.
      * Then {@link #lastResortConnection()} should be called to retrieve a non healthy connection and try it.
      */
-    Stream<C> nextConnection();
+    Stream<Connection> nextConnection();
 
     /**
      * Returns a connection that is not necessarily healthy, but can be used for a request attempt. To be called as last resort
      * only in case {@link #nextConnection()} returns an empty stream
      */
-    C lastResortConnection();
+    Connection lastResortConnection();
 
     /**
      * Called before each single request attempt. Allows to execute operations (e.g. ping) before each request.
      * Receives as an argument the connection that is going to be used for the request.
      */
-    void beforeAttempt(C connection) throws IOException;
+    void beforeAttempt(Connection connection) throws IOException;
 
     /**
      * Called after each successful request call.
      * Receives as an argument the connection that was used for the successful request.
      */
-    void onSuccess(C connection);
+    void onSuccess(Connection connection);
 
     /**
      * Called after each failed attempt.
      * Receives as an argument the connection that was used for the failed attempt.
      */
-    void onFailure(C connection) throws IOException;
+    void onFailure(Connection connection) throws IOException;
 }
diff --git a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
index 9f4b123bc8c..2d9ceead5ed 100644
--- a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
+++ b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
@@ -34,7 +34,7 @@ public class ElasticsearchResponseException extends IOException {
     private final RequestLine requestLine;
     private final StatusLine statusLine;
 
-    ElasticsearchResponseException(RequestLine requestLine, HttpHost host, StatusLine statusLine) {
+    public ElasticsearchResponseException(RequestLine requestLine, HttpHost host, StatusLine statusLine) {
         super(buildMessage(requestLine, host, statusLine));
         this.host = host;
         this.requestLine = requestLine;
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 35af2723267..e4186edc126 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -29,8 +29,8 @@ public final class RestClient implements Closeable {
 
     private final Transport transport;
 
-    public RestClient(CloseableHttpClient client, ConnectionPool<? extends Connection> connectionPool, long maxRetryTimeout) {
-        this.transport = new Transport<>(client, connectionPool, maxRetryTimeout);
+    public RestClient(CloseableHttpClient client, ConnectionPool connectionPool, long maxRetryTimeout) {
+        this.transport = new Transport(client, connectionPool, maxRetryTimeout);
     }
 
     public ElasticsearchResponse performRequest(String method, String endpoint, Map<String, Object> params, HttpEntity entity)
diff --git a/client/src/main/java/org/elasticsearch/client/StatefulConnection.java b/client/src/main/java/org/elasticsearch/client/StatefulConnection.java
deleted file mode 100644
index 50ac78cdde9..00000000000
--- a/client/src/main/java/org/elasticsearch/client/StatefulConnection.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.client;
-
-import org.apache.http.HttpHost;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * {@link Connection} subclass that has a mutable state, based on previous usage.
- * When first created, a connection is in unknown state, till it is used for the first time
- * and marked either dead or alive based on the outcome of the first usage.
- * Should be marked alive when properly working.
- * Should be marked dead when it caused a failure, in which case the connection may be retried some time later,
- * as soon as {@link #shouldBeRetried()} returns true, which depends on how many consecutive failed attempts
- * were counted and when the last one was registered.
- * Should be marked resurrected if in dead state, as last resort in case there are no live connections available
- * and none of the dead ones are ready to be retried yet. When marked resurrected, the number of failed attempts
- * and its timeout is not reset so that if it gets marked dead again it returns to the exact state before resurrection.
- */
-public final class StatefulConnection extends Connection {
-    //TODO make these values configurable through the connection pool?
-    private static final long DEFAULT_CONNECTION_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(1);
-    private static final long MAX_CONNECTION_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(30);
-
-    private volatile State state = State.UNKNOWN;
-    private volatile int failedAttempts = -1;
-    private volatile long deadUntil = -1;
-
-    /**
-     * Creates a new mutable connection pointing to the provided {@link HttpHost} argument
-     */
-    public StatefulConnection(HttpHost host) {
-        super(host);
-    }
-
-    /**
-     * Marks connection as dead. Should be called in case the corresponding node is not responding or caused failures.
-     * Once marked dead, the number of failed attempts will be incremented on each call to this method. A dead connection
-     * should be retried once {@link #shouldBeRetried()} returns true, which depends on the number of previous failed attempts
-     * and when the last failure was registered.
-     */
-    void markDead() {
-        synchronized (this) {
-            int failedAttempts = Math.max(this.failedAttempts, 0);
-            long timeoutMillis = (long)Math.min(DEFAULT_CONNECTION_TIMEOUT_MILLIS * 2 * Math.pow(2, failedAttempts * 0.5 - 1),
-                    MAX_CONNECTION_TIMEOUT_MILLIS);
-            this.deadUntil = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis);
-            this.failedAttempts = ++failedAttempts;
-            this.state = State.DEAD;
-        }
-    }
-
-    /**
-     * Marks this connection alive. Should be called when the corresponding node is working properly.
-     * Will reset the number of failed attempts that were counted in case the connection was previously dead,
-     * as well as its dead timeout.
-     */
-    void markAlive() {
-        if (this.state != State.ALIVE) {
-            synchronized (this) {
-                this.deadUntil = -1;
-                this.failedAttempts = 0;
-                this.state = State.ALIVE;
-            }
-        }
-    }
-
-    /**
-     * Resets the connection to its initial state, so it will be retried. To be called when all the connections in the pool
-     * are dead, so that one connection can be retried. Note that calling this method only changes the state of the connection,
-     * it doesn't reset its failed attempts and dead until timestamp. That way if the connection goes back to dead straightaway
-     * all of its previous failed attempts are taken into account.
-     */
-    void markResurrected() {
-        if (this.state == State.DEAD) {
-            synchronized (this) {
-                this.state = State.UNKNOWN;
-            }
-        }
-    }
-
-    /**
-     * Returns the timestamp till the connection is supposed to stay dead till it can be retried
-     */
-    public long getDeadUntil() {
-        return deadUntil;
-    }
-
-    /**
-     * Returns true if the connection is alive, false otherwise.
-     */
-    public boolean isAlive() {
-        return state == State.ALIVE;
-    }
-
-    /**
-     * Returns true in case the connection is not alive but should be used/retried, false otherwise.
-     * Returns true in case the connection is in unknown state (never used before) or resurrected. When the connection is dead,
-     * returns true when it is time to retry it, depending on how many failed attempts were registered and when the last failure
-     * happened (minimum 1 minute, maximum 30 minutes).
-     */
-    public boolean shouldBeRetried() {
-        return state == State.UNKNOWN || (state == State.DEAD && System.nanoTime() - deadUntil >= 0);
-    }
-
-    private enum State {
-        UNKNOWN, DEAD, ALIVE
-    }
-}
diff --git a/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java b/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
index 602298b1b3b..3e25818fb39 100644
--- a/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
@@ -40,7 +40,7 @@ public class StaticConnectionPool extends AbstractStaticConnectionPool {
     private final CloseableHttpClient client;
     private final boolean pingEnabled;
     private final RequestConfig pingRequestConfig;
-    private final List<StatefulConnection> connections;
+    private final List<Connection> connections;
 
     public StaticConnectionPool(CloseableHttpClient client, boolean pingEnabled, RequestConfig pingRequestConfig, HttpHost... hosts) {
         Objects.requireNonNull(client, "client cannot be null");
@@ -55,12 +55,14 @@ public class StaticConnectionPool extends AbstractStaticConnectionPool {
     }
 
     @Override
-    protected List<StatefulConnection> getConnections() {
+    protected List<Connection> getConnections() {
         return connections;
     }
 
+    //TODO do we still need pinging? seems like a workaround for some clients that don't support connect timeout but we have that
+
     @Override
-    public void beforeAttempt(StatefulConnection connection) throws IOException {
+    public void beforeAttempt(Connection connection) throws IOException {
         if (pingEnabled && connection.shouldBeRetried()) {
             HttpHead httpHead = new HttpHead("/");
             httpHead.setConfig(pingRequestConfig);
diff --git a/client/src/main/java/org/elasticsearch/client/Transport.java b/client/src/main/java/org/elasticsearch/client/Transport.java
index ff0835a991c..b0f843ec19b 100644
--- a/client/src/main/java/org/elasticsearch/client/Transport.java
+++ b/client/src/main/java/org/elasticsearch/client/Transport.java
@@ -44,15 +44,15 @@ import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Stream;
 
-final class Transport<C extends Connection> implements Closeable {
+final class Transport implements Closeable {
 
     private static final Log logger = LogFactory.getLog(Transport.class);
 
     private final CloseableHttpClient client;
-    private final ConnectionPool<C> connectionPool;
+    private final ConnectionPool connectionPool;
     private final long maxRetryTimeout;
 
-    Transport(CloseableHttpClient client, ConnectionPool<C> connectionPool, long maxRetryTimeout) {
+    Transport(CloseableHttpClient client, ConnectionPool connectionPool, long maxRetryTimeout) {
         Objects.requireNonNull(client, "client cannot be null");
         Objects.requireNonNull(connectionPool, "connectionPool cannot be null");
         if (maxRetryTimeout <= 0) {
@@ -66,23 +66,23 @@ final class Transport<C extends Connection> implements Closeable {
     ElasticsearchResponse performRequest(String method, String endpoint, Map<String, Object> params, HttpEntity entity) throws IOException {
         URI uri = buildUri(endpoint, params);
         HttpRequestBase request = createHttpRequest(method, uri, entity);
-        Iterator<C> connectionIterator = connectionPool.nextConnection().iterator();
+        Iterator<Connection> connectionIterator = connectionPool.nextConnection().iterator();
         if (connectionIterator.hasNext() == false) {
-            C connection = connectionPool.lastResortConnection();
+            Connection connection = connectionPool.lastResortConnection();
             logger.info("no healthy nodes available, trying " + connection.getHost());
             return performRequest(request, Stream.of(connection).iterator());
         }
         return performRequest(request, connectionIterator);
     }
 
-    private ElasticsearchResponse performRequest(HttpRequestBase request, Iterator<C> connectionIterator) throws IOException {
+    private ElasticsearchResponse performRequest(HttpRequestBase request, Iterator<Connection> connectionIterator) throws IOException {
         //we apply a soft margin so that e.g. if a request took 59 seconds and timeout is set to 60 we don't do another attempt
         long retryTimeout = Math.round(this.maxRetryTimeout / (float)100 * 98);
         IOException lastSeenException = null;
         long startTime = System.nanoTime();
 
         while (connectionIterator.hasNext()) {
-            C connection = connectionIterator.next();
+            Connection connection = connectionIterator.next();
 
             if (lastSeenException != null) {
                 long timeElapsed = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
@@ -124,7 +124,7 @@ final class Transport<C extends Connection> implements Closeable {
         throw lastSeenException;
     }
 
-    private ElasticsearchResponse performRequest(HttpRequestBase request, C connection) throws IOException {
+    private ElasticsearchResponse performRequest(HttpRequestBase request, Connection connection) throws IOException {
         CloseableHttpResponse response;
         try {
             response = client.execute(connection.getHost(), request);
diff --git a/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java b/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
index d0171e6b952..9bce0b43f0b 100644
--- a/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
@@ -25,7 +25,7 @@ import org.apache.http.HttpHost;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.elasticsearch.client.AbstractStaticConnectionPool;
-import org.elasticsearch.client.StatefulConnection;
+import org.elasticsearch.client.Connection;
 
 import java.io.IOException;
 import java.util.Iterator;
@@ -46,9 +46,10 @@ public class SniffingConnectionPool extends AbstractStaticConnectionPool {
 
     private final boolean sniffOnFailure;
     private final Sniffer sniffer;
-    private volatile List<StatefulConnection> connections;
+    private volatile List<Connection> connections;
     private final SnifferTask snifferTask;
 
+    //TODO do we still need the sniff request timeout? or should we just use a low connect timeout?
     public SniffingConnectionPool(int sniffInterval, boolean sniffOnFailure, int sniffAfterFailureDelay,
                                   CloseableHttpClient client, RequestConfig sniffRequestConfig, int sniffRequestTimeout, Scheme scheme,
                                   HttpHost... hosts) {
@@ -69,17 +70,17 @@ public class SniffingConnectionPool extends AbstractStaticConnectionPool {
     }
 
     @Override
-    protected List<StatefulConnection> getConnections() {
+    protected List<Connection> getConnections() {
         return this.connections;
     }
 
     @Override
-    public void beforeAttempt(StatefulConnection connection) throws IOException {
+    public void beforeAttempt(Connection connection) throws IOException {
 
     }
 
     @Override
-    public void onFailure(StatefulConnection connection) throws IOException {
+    public void onFailure(Connection connection) throws IOException {
         super.onFailure(connection);
         if (sniffOnFailure) {
             //re-sniff immediately but take out the node that failed
@@ -130,11 +131,11 @@ public class SniffingConnectionPool extends AbstractStaticConnectionPool {
         void sniff(Predicate<HttpHost> hostFilter) {
             if (running.compareAndSet(false, true)) {
                 try {
-                    Iterator<StatefulConnection> connectionIterator = nextConnection().iterator();
+                    Iterator<Connection> connectionIterator = nextConnection().iterator();
                     if (connectionIterator.hasNext()) {
                         sniff(connectionIterator, hostFilter);
                     } else {
-                        StatefulConnection connection = lastResortConnection();
+                        Connection connection = lastResortConnection();
                         logger.info("no healthy nodes available, trying " + connection.getHost());
                         sniff(Stream.of(connection).iterator(), hostFilter);
                     }
@@ -160,10 +161,10 @@ public class SniffingConnectionPool extends AbstractStaticConnectionPool {
             }
         }
 
-        void sniff(Iterator<StatefulConnection> connectionIterator, Predicate<HttpHost> hostFilter) throws IOException {
+        void sniff(Iterator<Connection> connectionIterator, Predicate<HttpHost> hostFilter) throws IOException {
             IOException lastSeenException = null;
             while (connectionIterator.hasNext()) {
-                StatefulConnection connection = connectionIterator.next();
+                Connection connection = connectionIterator.next();
                 try {
                     List<HttpHost> sniffedNodes = sniffer.sniffNodes(connection.getHost());
                     HttpHost[] filteredNodes = sniffedNodes.stream().filter(hostFilter).toArray(HttpHost[]::new);
diff --git a/client/src/test/java/org/elasticsearch/client/TransportTests.java b/client/src/test/java/org/elasticsearch/client/TransportTests.java
index 591eb516c59..1efafd438b0 100644
--- a/client/src/test/java/org/elasticsearch/client/TransportTests.java
+++ b/client/src/test/java/org/elasticsearch/client/TransportTests.java
@@ -36,7 +36,7 @@ public class TransportTests extends LuceneTestCase {
 
     public void testConstructor() {
         CloseableHttpClient httpClient = HttpClientBuilder.create().build();
-        ConnectionPool<Connection> connectionPool = new ConnectionPool<Connection>() {
+        ConnectionPool connectionPool = new ConnectionPool() {
             @Override
             public Stream<Connection> nextConnection() {
                 return null;
@@ -69,27 +69,27 @@ public class TransportTests extends LuceneTestCase {
         };
 
         try {
-            new Transport<>(null, connectionPool, RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+            new Transport(null, connectionPool, RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
             fail("transport creation should have failed");
         } catch(NullPointerException e) {
             assertEquals(e.getMessage(), "client cannot be null");
         }
 
         try {
-            new Transport<>(httpClient, null, RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+            new Transport(httpClient, null, RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
             fail("transport creation should have failed");
         } catch(NullPointerException e) {
             assertEquals(e.getMessage(), "connectionPool cannot be null");
         }
 
         try {
-            new Transport<>(httpClient, connectionPool, RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+            new Transport(httpClient, connectionPool, RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
             fail("transport creation should have failed");
         } catch(IllegalArgumentException e) {
             assertEquals(e.getMessage(), "maxRetryTimeout must be greater than 0");
         }
 
-        Transport<Connection> transport = new Transport<>(httpClient, connectionPool, RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+        Transport transport = new Transport(httpClient, connectionPool, RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
         assertNotNull(transport);
     }
 }

From bd29dc157259246f181cf911a2fcf5d684675385 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 6 May 2016 10:12:43 +0200
Subject: [PATCH 0588/1311] Remove Transport class, move all of it within
 RestClient class

---
 .../org/elasticsearch/client/RestClient.java  | 171 +++++++++++++-
 .../org/elasticsearch/client/Transport.java   | 208 ------------------
 ...ansportTests.java => RestClientTests.java} |  15 +-
 3 files changed, 175 insertions(+), 219 deletions(-)
 delete mode 100644 client/src/main/java/org/elasticsearch/client/Transport.java
 rename client/src/test/java/org/elasticsearch/client/{TransportTests.java => RestClientTests.java} (80%)

diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index e4186edc126..3e59cc297e4 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -18,28 +18,191 @@
  */
 package org.elasticsearch.client;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.http.HttpEntity;
+import org.apache.http.StatusLine;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
+import org.apache.http.client.methods.HttpHead;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.util.EntityUtils;
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Iterator;
+import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
 
 public final class RestClient implements Closeable {
 
-    private final Transport transport;
+    private static final Log logger = LogFactory.getLog(RestClient.class);
+
+    private final CloseableHttpClient client;
+    private final ConnectionPool connectionPool;
+    private final long maxRetryTimeout;
 
     public RestClient(CloseableHttpClient client, ConnectionPool connectionPool, long maxRetryTimeout) {
-        this.transport = new Transport(client, connectionPool, maxRetryTimeout);
+        Objects.requireNonNull(client, "client cannot be null");
+        Objects.requireNonNull(connectionPool, "connectionPool cannot be null");
+        if (maxRetryTimeout <= 0) {
+            throw new IllegalArgumentException("maxRetryTimeout must be greater than 0");
+        }
+        this.client = client;
+        this.connectionPool = connectionPool;
+        this.maxRetryTimeout = maxRetryTimeout;
     }
 
     public ElasticsearchResponse performRequest(String method, String endpoint, Map<String, Object> params, HttpEntity entity)
             throws IOException {
-        return transport.performRequest(method, endpoint, params, entity);
+        URI uri = buildUri(endpoint, params);
+        HttpRequestBase request = createHttpRequest(method, uri, entity);
+        Iterator<Connection> connectionIterator = connectionPool.nextConnection().iterator();
+        if (connectionIterator.hasNext() == false) {
+            Connection connection = connectionPool.lastResortConnection();
+            logger.info("no healthy nodes available, trying " + connection.getHost());
+            return performRequest(request, Stream.of(connection).iterator());
+        }
+        return performRequest(request, connectionIterator);
+    }
+
+    private ElasticsearchResponse performRequest(HttpRequestBase request, Iterator<Connection> connectionIterator) throws IOException {
+        //we apply a soft margin so that e.g. if a request took 59 seconds and timeout is set to 60 we don't do another attempt
+        long retryTimeout = Math.round(this.maxRetryTimeout / (float)100 * 98);
+        IOException lastSeenException = null;
+        long startTime = System.nanoTime();
+
+        while (connectionIterator.hasNext()) {
+            Connection connection = connectionIterator.next();
+
+            if (lastSeenException != null) {
+                long timeElapsed = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
+                long timeout = retryTimeout - timeElapsed;
+                if (timeout <= 0) {
+                    IOException retryTimeoutException = new IOException(
+                            "request retries exceeded max retry timeout [" + retryTimeout + "]");
+                    retryTimeoutException.addSuppressed(lastSeenException);
+                    throw retryTimeoutException;
+                }
+            }
+
+            try {
+                connectionPool.beforeAttempt(connection);
+            } catch(IOException e) {
+                lastSeenException = addSuppressedException(lastSeenException, e);
+                continue;
+            }
+
+            try {
+                ElasticsearchResponse response = performRequest(request, connection);
+                connectionPool.onSuccess(connection);
+                return response;
+            } catch(ElasticsearchResponseException e) {
+                if (e.isRecoverable()) {
+                    connectionPool.onFailure(connection);
+                    lastSeenException = addSuppressedException(lastSeenException, e);
+                } else {
+                    //don't retry and call onSuccess as the error should be a request problem
+                    connectionPool.onSuccess(connection);
+                    throw e;
+                }
+            } catch(IOException e) {
+                connectionPool.onFailure(connection);
+                lastSeenException = addSuppressedException(lastSeenException, e);
+            }
+        }
+        assert lastSeenException != null;
+        throw lastSeenException;
+    }
+
+    private ElasticsearchResponse performRequest(HttpRequestBase request, Connection connection) throws IOException {
+        CloseableHttpResponse response;
+        try {
+            response = client.execute(connection.getHost(), request);
+        } catch(IOException e) {
+            RequestLogger.log(logger, "request failed", request.getRequestLine(), connection.getHost(), e);
+            throw e;
+        } finally {
+            request.reset();
+        }
+        StatusLine statusLine = response.getStatusLine();
+        //TODO make ignore status code configurable. rest-spec and tests support that parameter.
+        if (statusLine.getStatusCode() < 300 ||
+                request.getMethod().equals(HttpHead.METHOD_NAME) && statusLine.getStatusCode() == 404) {
+            RequestLogger.log(logger, "request succeeded", request.getRequestLine(), connection.getHost(), response.getStatusLine());
+            return new ElasticsearchResponse(request.getRequestLine(), connection.getHost(), response);
+        } else {
+            EntityUtils.consume(response.getEntity());
+            RequestLogger.log(logger, "request failed", request.getRequestLine(), connection.getHost(), response.getStatusLine());
+            throw new ElasticsearchResponseException(request.getRequestLine(), connection.getHost(), statusLine);
+        }
+    }
+
+    private static IOException addSuppressedException(IOException suppressedException, IOException currentException) {
+        if (suppressedException != null) {
+            currentException.addSuppressed(suppressedException);
+        }
+        return currentException;
+    }
+
+    private static HttpRequestBase createHttpRequest(String method, URI uri, HttpEntity entity) {
+        switch(method.toUpperCase(Locale.ROOT)) {
+            case HttpDeleteWithEntity.METHOD_NAME:
+                HttpDeleteWithEntity httpDeleteWithEntity = new HttpDeleteWithEntity(uri);
+                addRequestBody(httpDeleteWithEntity, entity);
+                return httpDeleteWithEntity;
+            case HttpGetWithEntity.METHOD_NAME:
+                HttpGetWithEntity httpGetWithEntity = new HttpGetWithEntity(uri);
+                addRequestBody(httpGetWithEntity, entity);
+                return httpGetWithEntity;
+            case HttpHead.METHOD_NAME:
+                if (entity != null) {
+                    throw new UnsupportedOperationException("HEAD with body is not supported");
+                }
+                return new HttpHead(uri);
+            case HttpPost.METHOD_NAME:
+                HttpPost httpPost = new HttpPost(uri);
+                addRequestBody(httpPost, entity);
+                return httpPost;
+            case HttpPut.METHOD_NAME:
+                HttpPut httpPut = new HttpPut(uri);
+                addRequestBody(httpPut, entity);
+                return httpPut;
+            default:
+                throw new UnsupportedOperationException("http method not supported: " + method);
+        }
+    }
+
+    private static void addRequestBody(HttpEntityEnclosingRequestBase httpRequest, HttpEntity entity) {
+        if (entity != null) {
+            httpRequest.setEntity(entity);
+        }
+    }
+
+    private static URI buildUri(String path, Map<String, Object> params) {
+        try {
+            URIBuilder uriBuilder = new URIBuilder(path);
+            for (Map.Entry<String, Object> param : params.entrySet()) {
+                uriBuilder.addParameter(param.getKey(), param.getValue().toString());
+            }
+            return uriBuilder.build();
+        } catch(URISyntaxException e) {
+            throw new IllegalArgumentException(e.getMessage(), e);
+        }
     }
 
     @Override
     public void close() throws IOException {
-        transport.close();
+        connectionPool.close();
+        client.close();
     }
 }
diff --git a/client/src/main/java/org/elasticsearch/client/Transport.java b/client/src/main/java/org/elasticsearch/client/Transport.java
deleted file mode 100644
index b0f843ec19b..00000000000
--- a/client/src/main/java/org/elasticsearch/client/Transport.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.client;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.http.HttpEntity;
-import org.apache.http.StatusLine;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
-import org.apache.http.client.methods.HttpHead;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.client.methods.HttpRequestBase;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.util.EntityUtils;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.Iterator;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Stream;
-
-final class Transport implements Closeable {
-
-    private static final Log logger = LogFactory.getLog(Transport.class);
-
-    private final CloseableHttpClient client;
-    private final ConnectionPool connectionPool;
-    private final long maxRetryTimeout;
-
-    Transport(CloseableHttpClient client, ConnectionPool connectionPool, long maxRetryTimeout) {
-        Objects.requireNonNull(client, "client cannot be null");
-        Objects.requireNonNull(connectionPool, "connectionPool cannot be null");
-        if (maxRetryTimeout <= 0) {
-            throw new IllegalArgumentException("maxRetryTimeout must be greater than 0");
-        }
-        this.client = client;
-        this.connectionPool = connectionPool;
-        this.maxRetryTimeout = maxRetryTimeout;
-    }
-
-    ElasticsearchResponse performRequest(String method, String endpoint, Map<String, Object> params, HttpEntity entity) throws IOException {
-        URI uri = buildUri(endpoint, params);
-        HttpRequestBase request = createHttpRequest(method, uri, entity);
-        Iterator<Connection> connectionIterator = connectionPool.nextConnection().iterator();
-        if (connectionIterator.hasNext() == false) {
-            Connection connection = connectionPool.lastResortConnection();
-            logger.info("no healthy nodes available, trying " + connection.getHost());
-            return performRequest(request, Stream.of(connection).iterator());
-        }
-        return performRequest(request, connectionIterator);
-    }
-
-    private ElasticsearchResponse performRequest(HttpRequestBase request, Iterator<Connection> connectionIterator) throws IOException {
-        //we apply a soft margin so that e.g. if a request took 59 seconds and timeout is set to 60 we don't do another attempt
-        long retryTimeout = Math.round(this.maxRetryTimeout / (float)100 * 98);
-        IOException lastSeenException = null;
-        long startTime = System.nanoTime();
-
-        while (connectionIterator.hasNext()) {
-            Connection connection = connectionIterator.next();
-
-            if (lastSeenException != null) {
-                long timeElapsed = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
-                long timeout = retryTimeout - timeElapsed;
-                if (timeout <= 0) {
-                    IOException retryTimeoutException = new IOException(
-                            "request retries exceeded max retry timeout [" + retryTimeout + "]");
-                    retryTimeoutException.addSuppressed(lastSeenException);
-                    throw retryTimeoutException;
-                }
-            }
-
-            try {
-                connectionPool.beforeAttempt(connection);
-            } catch(IOException e) {
-                lastSeenException = addSuppressedException(lastSeenException, e);
-                continue;
-            }
-
-            try {
-                ElasticsearchResponse response = performRequest(request, connection);
-                connectionPool.onSuccess(connection);
-                return response;
-            } catch(ElasticsearchResponseException e) {
-                if (e.isRecoverable()) {
-                    connectionPool.onFailure(connection);
-                    lastSeenException = addSuppressedException(lastSeenException, e);
-                } else {
-                    //don't retry and call onSuccess as the error should be a request problem
-                    connectionPool.onSuccess(connection);
-                    throw e;
-                }
-            } catch(IOException e) {
-                connectionPool.onFailure(connection);
-                lastSeenException = addSuppressedException(lastSeenException, e);
-            }
-        }
-        assert lastSeenException != null;
-        throw lastSeenException;
-    }
-
-    private ElasticsearchResponse performRequest(HttpRequestBase request, Connection connection) throws IOException {
-        CloseableHttpResponse response;
-        try {
-            response = client.execute(connection.getHost(), request);
-        } catch(IOException e) {
-            RequestLogger.log(logger, "request failed", request.getRequestLine(), connection.getHost(), e);
-            throw e;
-        } finally {
-            request.reset();
-        }
-        StatusLine statusLine = response.getStatusLine();
-        //TODO make ignore status code configurable. rest-spec and tests support that parameter.
-        if (statusLine.getStatusCode() < 300 ||
-                request.getMethod().equals(HttpHead.METHOD_NAME) && statusLine.getStatusCode() == 404) {
-            RequestLogger.log(logger, "request succeeded", request.getRequestLine(), connection.getHost(), response.getStatusLine());
-            return new ElasticsearchResponse(request.getRequestLine(), connection.getHost(), response);
-        } else {
-            EntityUtils.consume(response.getEntity());
-            RequestLogger.log(logger, "request failed", request.getRequestLine(), connection.getHost(), response.getStatusLine());
-            throw new ElasticsearchResponseException(request.getRequestLine(), connection.getHost(), statusLine);
-        }
-    }
-
-    private static IOException addSuppressedException(IOException suppressedException, IOException currentException) {
-        if (suppressedException != null) {
-            currentException.addSuppressed(suppressedException);
-        }
-        return currentException;
-    }
-
-    private static HttpRequestBase createHttpRequest(String method, URI uri, HttpEntity entity) {
-        switch(method.toUpperCase(Locale.ROOT)) {
-            case HttpDeleteWithEntity.METHOD_NAME:
-                HttpDeleteWithEntity httpDeleteWithEntity = new HttpDeleteWithEntity(uri);
-                addRequestBody(httpDeleteWithEntity, entity);
-                return httpDeleteWithEntity;
-            case HttpGetWithEntity.METHOD_NAME:
-                HttpGetWithEntity httpGetWithEntity = new HttpGetWithEntity(uri);
-                addRequestBody(httpGetWithEntity, entity);
-                return httpGetWithEntity;
-            case HttpHead.METHOD_NAME:
-                if (entity != null) {
-                    throw new UnsupportedOperationException("HEAD with body is not supported");
-                }
-                return new HttpHead(uri);
-            case HttpPost.METHOD_NAME:
-                HttpPost httpPost = new HttpPost(uri);
-                addRequestBody(httpPost, entity);
-                return httpPost;
-            case HttpPut.METHOD_NAME:
-                HttpPut httpPut = new HttpPut(uri);
-                addRequestBody(httpPut, entity);
-                return httpPut;
-            default:
-                throw new UnsupportedOperationException("http method not supported: " + method);
-        }
-    }
-
-    private static void addRequestBody(HttpEntityEnclosingRequestBase httpRequest, HttpEntity entity) {
-        if (entity != null) {
-            httpRequest.setEntity(entity);
-        }
-    }
-
-    private static URI buildUri(String path, Map<String, Object> params) {
-        try {
-            URIBuilder uriBuilder = new URIBuilder(path);
-            for (Map.Entry<String, Object> param : params.entrySet()) {
-                uriBuilder.addParameter(param.getKey(), param.getValue().toString());
-            }
-            return uriBuilder.build();
-        } catch(URISyntaxException e) {
-            throw new IllegalArgumentException(e.getMessage(), e);
-        }
-    }
-
-    @Override
-    public void close() throws IOException {
-        connectionPool.close();
-        client.close();
-    }
-}
diff --git a/client/src/test/java/org/elasticsearch/client/TransportTests.java b/client/src/test/java/org/elasticsearch/client/RestClientTests.java
similarity index 80%
rename from client/src/test/java/org/elasticsearch/client/TransportTests.java
rename to client/src/test/java/org/elasticsearch/client/RestClientTests.java
index 1efafd438b0..52e18831591 100644
--- a/client/src/test/java/org/elasticsearch/client/TransportTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientTests.java
@@ -28,13 +28,13 @@ import java.io.IOException;
 import java.util.logging.LogManager;
 import java.util.stream.Stream;
 
-public class TransportTests extends LuceneTestCase {
+public class RestClientTests extends LuceneTestCase {
 
     static {
         LogManager.getLogManager().reset();
     }
 
-    public void testConstructor() {
+    public void testConstructor() throws IOException {
         CloseableHttpClient httpClient = HttpClientBuilder.create().build();
         ConnectionPool connectionPool = new ConnectionPool() {
             @Override
@@ -69,27 +69,28 @@ public class TransportTests extends LuceneTestCase {
         };
 
         try {
-            new Transport(null, connectionPool, RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+            new RestClient(null, connectionPool, RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
             fail("transport creation should have failed");
         } catch(NullPointerException e) {
             assertEquals(e.getMessage(), "client cannot be null");
         }
 
         try {
-            new Transport(httpClient, null, RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+            new RestClient(httpClient, null, RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
             fail("transport creation should have failed");
         } catch(NullPointerException e) {
             assertEquals(e.getMessage(), "connectionPool cannot be null");
         }
 
         try {
-            new Transport(httpClient, connectionPool, RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+            new RestClient(httpClient, connectionPool, RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
             fail("transport creation should have failed");
         } catch(IllegalArgumentException e) {
             assertEquals(e.getMessage(), "maxRetryTimeout must be greater than 0");
         }
 
-        Transport transport = new Transport(httpClient, connectionPool, RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
-        assertNotNull(transport);
+        try(RestClient client = new RestClient(httpClient, connectionPool, RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE))) {
+            assertNotNull(client);
+        }
     }
 }

From 9ffdea9515ea8387e5fafc50ab57d753c7e7ebc2 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 6 May 2016 10:30:44 +0200
Subject: [PATCH 0589/1311] remove Scheme enum

---
 client/build.gradle                           |  1 +
 .../client/sniff/SniffingConnectionPool.java  | 17 +++-----
 .../client/sniff/SnifferTests.java            |  9 ++---
 .../sniff/SniffingConnectionPoolTests.java    | 39 ++++++++++++++-----
 4 files changed, 40 insertions(+), 26 deletions(-)

diff --git a/client/build.gradle b/client/build.gradle
index 7a9377d0acb..e19e860c356 100644
--- a/client/build.gradle
+++ b/client/build.gradle
@@ -30,6 +30,7 @@ dependencies {
   //compile "org.apache.httpcomponents:httpasyncclient:4.1.1"
   compile "commons-codec:commons-codec:1.9"
   compile "commons-logging:commons-logging:1.2"
+  //jackson is only needed in the sniff package
   compile "com.fasterxml.jackson.core:jackson-core:2.7.3"
 
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
diff --git a/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java b/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
index 9bce0b43f0b..399bfd8e39b 100644
--- a/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
@@ -30,7 +30,6 @@ import org.elasticsearch.client.Connection;
 import java.io.IOException;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Locale;
 import java.util.Objects;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
@@ -51,7 +50,7 @@ public class SniffingConnectionPool extends AbstractStaticConnectionPool {
 
     //TODO do we still need the sniff request timeout? or should we just use a low connect timeout?
     public SniffingConnectionPool(int sniffInterval, boolean sniffOnFailure, int sniffAfterFailureDelay,
-                                  CloseableHttpClient client, RequestConfig sniffRequestConfig, int sniffRequestTimeout, Scheme scheme,
+                                  CloseableHttpClient client, RequestConfig sniffRequestConfig, int sniffRequestTimeout, String scheme,
                                   HttpHost... hosts) {
         if (sniffInterval <= 0) {
             throw new IllegalArgumentException("sniffInterval must be greater than 0");
@@ -60,11 +59,14 @@ public class SniffingConnectionPool extends AbstractStaticConnectionPool {
             throw new IllegalArgumentException("sniffAfterFailureDelay must be greater than 0");
         }
         Objects.requireNonNull(scheme, "scheme cannot be null");
+        if (scheme.equals("http") == false && scheme.equals("https") == false) {
+            throw new IllegalArgumentException("scheme must be either http or https");
+        }
         if (hosts == null || hosts.length == 0) {
             throw new IllegalArgumentException("no hosts provided");
         }
         this.sniffOnFailure = sniffOnFailure;
-        this.sniffer = new Sniffer(client, sniffRequestConfig, sniffRequestTimeout, scheme.toString());
+        this.sniffer = new Sniffer(client, sniffRequestConfig, sniffRequestTimeout, scheme);
         this.connections = createConnections(hosts);
         this.snifferTask = new SnifferTask(sniffInterval, sniffAfterFailureDelay);
     }
@@ -93,15 +95,6 @@ public class SniffingConnectionPool extends AbstractStaticConnectionPool {
         snifferTask.shutdown();
     }
 
-    public enum Scheme {
-        HTTP, HTTPS;
-
-        @Override
-        public String toString() {
-            return name().toLowerCase(Locale.ROOT);
-        }
-    }
-
     private class SnifferTask implements Runnable {
         private final int sniffInterval;
         private final int sniffAfterFailureDelay;
diff --git a/client/src/test/java/org/elasticsearch/client/sniff/SnifferTests.java b/client/src/test/java/org/elasticsearch/client/sniff/SnifferTests.java
index 28a9697ac37..4ef17579856 100644
--- a/client/src/test/java/org/elasticsearch/client/sniff/SnifferTests.java
+++ b/client/src/test/java/org/elasticsearch/client/sniff/SnifferTests.java
@@ -34,8 +34,6 @@ import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.client.ElasticsearchResponseException;
-import org.elasticsearch.client.sniff.Sniffer;
-import org.elasticsearch.client.sniff.SniffingConnectionPool;
 import org.junit.After;
 import org.junit.Before;
 
@@ -46,6 +44,7 @@ import java.net.URISyntaxException;
 import java.net.URLDecoder;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -66,14 +65,14 @@ public class SnifferTests extends LuceneTestCase {
     }
 
     private int sniffRequestTimeout;
-    private SniffingConnectionPool.Scheme scheme;
+    private String scheme;
     private SniffResponse sniffResponse;
     private MockWebServer server;
 
     @Before
     public void startMockWebServer() throws IOException {
         this.sniffRequestTimeout = RandomInts.randomIntBetween(random(), 1000, 10000);
-        this.scheme = RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values());
+        this.scheme = RandomPicks.randomFrom(random(), Arrays.asList("http", "https"));
         if (rarely()) {
             this.sniffResponse = SniffResponse.buildFailure();
         } else {
@@ -139,7 +138,7 @@ public class SnifferTests extends LuceneTestCase {
         return server;
     }
 
-    private static SniffResponse buildSniffResponse(SniffingConnectionPool.Scheme scheme) throws IOException {
+    private static SniffResponse buildSniffResponse(String scheme) throws IOException {
         int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
         List<HttpHost> hosts = new ArrayList<>(numNodes);
         JsonFactory jsonFactory = new JsonFactory();
diff --git a/client/src/test/java/org/elasticsearch/client/sniff/SniffingConnectionPoolTests.java b/client/src/test/java/org/elasticsearch/client/sniff/SniffingConnectionPoolTests.java
index 06e6b43236b..4e54f8074db 100644
--- a/client/src/test/java/org/elasticsearch/client/sniff/SniffingConnectionPoolTests.java
+++ b/client/src/test/java/org/elasticsearch/client/sniff/SniffingConnectionPoolTests.java
@@ -37,6 +37,7 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
 
     public void testConstructor() throws Exception {
         CloseableHttpClient httpClient = HttpClientBuilder.create().build();
+        String[] schemes = new String[]{"http", "https"};
         int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
         HttpHost[] hosts = new HttpHost[numNodes];
         for (int i = 0; i < numNodes; i++) {
@@ -47,7 +48,7 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), hosts)) {
+                RandomPicks.randomFrom(random(), schemes), hosts)) {
 
             fail("pool creation should have failed " + connectionPool);
         } catch(IllegalArgumentException e) {
@@ -58,7 +59,7 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0), httpClient, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), hosts)) {
+                RandomPicks.randomFrom(random(), schemes), hosts)) {
             fail("pool creation should have failed " + connectionPool);
         } catch(IllegalArgumentException e) {
             assertEquals(e.getMessage(), "sniffAfterFailureDelay must be greater than 0");
@@ -68,7 +69,7 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), null, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), hosts)) {
+                RandomPicks.randomFrom(random(), schemes), hosts)) {
             fail("pool creation should have failed " + connectionPool);
         } catch(NullPointerException e) {
             assertEquals(e.getMessage(), "client cannot be null");
@@ -78,7 +79,7 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, null,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), hosts)) {
+                RandomPicks.randomFrom(random(), schemes), hosts)) {
             fail("pool creation should have failed " + connectionPool);
         } catch(NullPointerException e) {
             assertEquals(e.getMessage(), "sniffRequestConfig cannot be null");
@@ -88,17 +89,37 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), hosts)) {
+                RandomPicks.randomFrom(random(), schemes), hosts)) {
             fail("pool creation should have failed " + connectionPool);
         } catch(IllegalArgumentException e) {
             assertEquals(e.getMessage(), "sniffRequestTimeout must be greater than 0");
         }
 
+        try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
+                RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0),
+                null, hosts)) {
+            fail("pool creation should have failed " + connectionPool);
+        } catch(NullPointerException e) {
+            assertEquals(e.getMessage(), "scheme cannot be null");
+        }
+
+        try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
+                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
+                RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0),
+                "whatever", hosts)) {
+            fail("pool creation should have failed " + connectionPool);
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "scheme must be either http or https");
+        }
+
         try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), (HttpHost[])null)) {
+                RandomPicks.randomFrom(random(), schemes), (HttpHost[])null)) {
             fail("pool creation should have failed " + connectionPool);
         } catch(IllegalArgumentException e) {
             assertEquals(e.getMessage(), "no hosts provided");
@@ -108,7 +129,7 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), (HttpHost) null)) {
+                RandomPicks.randomFrom(random(), schemes), (HttpHost) null)) {
             fail("pool creation should have failed " + connectionPool);
         } catch(NullPointerException e) {
             assertEquals(e.getMessage(), "host cannot be null");
@@ -118,7 +139,7 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()))) {
+                RandomPicks.randomFrom(random(), schemes))) {
             fail("pool creation should have failed " + connectionPool);
         } catch(IllegalArgumentException e) {
             assertEquals(e.getMessage(), "no hosts provided");
@@ -128,7 +149,7 @@ public class SniffingConnectionPoolTests extends LuceneTestCase {
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
                 RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), SniffingConnectionPool.Scheme.values()), hosts)) {
+                RandomPicks.randomFrom(random(), schemes), hosts)) {
             assertNotNull(sniffingConnectionPool);
         }
     }

From d7c41764f2072a4d9ec3cbbdc3b0d5a2b57ed30b Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 6 May 2016 11:04:26 +0200
Subject: [PATCH 0590/1311] add some javadocs to connection pool classes

---
 .../client/AbstractStaticConnectionPool.java  | 20 +++++++++++++------
 .../client/StaticConnectionPool.java          |  3 +++
 .../elasticsearch/client/sniff/Sniffer.java   |  4 ++--
 .../client/sniff/SniffingConnectionPool.java  |  4 ++++
 4 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java b/client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java
index 192c1b0f35a..3fe4ead4bbe 100644
--- a/client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java
@@ -29,19 +29,17 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Predicate;
 import java.util.stream.Stream;
 
 /**
- * Base static connection pool implementation that deals with mutable connections. Marks connections as dead/alive when needed.
- * Provides a stream of alive connections or dead ones that should be retried for each {@link #nextConnection()} call, which
- * allows to filter connections through a customizable {@link Predicate}, called connection selector.
+ * Base static connection pool implementation that marks connections as dead/alive when needed.
+ * Provides a stream of alive connections or dead ones that should be retried for each {@link #nextConnection()} call.
  * In case the returned stream is empty a last resort dead connection should be retrieved by calling {@link #lastResortConnection()}
- * and resurrected so that a single request attempt can be performed.
+ * and resurrected so that a last resort request attempt can be performed.
  * The {@link #onSuccess(Connection)} method marks the connection provided as an argument alive.
  * The {@link #onFailure(Connection)} method marks the connection provided as an argument dead.
  * This base implementation doesn't define the list implementation that stores connections, so that concurrency can be
- * handled in the subclasses depending on the usecase (e.g. defining the list volatile when needed).
+ * handled in subclasses depending on the usecase (e.g. defining the list volatile or final when needed).
  */
 public abstract class AbstractStaticConnectionPool implements ConnectionPool {
 
@@ -49,6 +47,12 @@ public abstract class AbstractStaticConnectionPool implements ConnectionPool {
 
     private final AtomicInteger lastConnectionIndex = new AtomicInteger(0);
 
+    /**
+     * Allows to retrieve the concrete list of connections. Not defined directly as a member
+     * of this class as subclasses may need to handle concurrency if the list can change, for
+     * instance defining the field as volatile. On the other hand static implementations
+     * can just make the list final instead.
+     */
     protected abstract List<Connection> getConnections();
 
     @Override
@@ -64,6 +68,10 @@ public abstract class AbstractStaticConnectionPool implements ConnectionPool {
         return sortedConnections.stream().filter(connection -> connection.isAlive() || connection.shouldBeRetried());
     }
 
+    /**
+     * Helper method to be used by subclasses when needing to create a new list
+     * of connections given their corresponding hosts
+     */
     protected List<Connection> createConnections(HttpHost... hosts) {
         List<Connection> connections = new ArrayList<>();
         for (HttpHost host : hosts) {
diff --git a/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java b/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
index 3e25818fb39..5d54e583abe 100644
--- a/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
@@ -33,6 +33,9 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 
+/**
+ * Static implementation of {@link ConnectionPool}. Its underlying list of connections is immutable.
+ */
 public class StaticConnectionPool extends AbstractStaticConnectionPool {
 
     private static final Log logger = LogFactory.getLog(StaticConnectionPool.class);
diff --git a/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
index a812804cbce..ebed97e4a4e 100644
--- a/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
+++ b/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
@@ -43,9 +43,9 @@ import java.util.List;
 import java.util.Objects;
 
 /**
- * Calls nodes info api and returns a list of http hosts extracted from it
+ * Calls nodes info api and returns a list of http hosts extracted from it.
  */
-//TODO this could potentially a call to _cat/nodes (although it doesn't support timeout param), but how would we handle bw comp with 2.x?
+//TODO This could potentially be using _cat/nodes which wouldn't require jackson as a dependency, but we'd have bw comp problems with 2.x
 final class Sniffer {
 
     private static final Log logger = LogFactory.getLog(Sniffer.class);
diff --git a/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java b/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
index 399bfd8e39b..3fb7885fd14 100644
--- a/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
@@ -39,6 +39,10 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Predicate;
 import java.util.stream.Stream;
 
+/**
+ * Connection pool implementation that sniffs nodes from elasticsearch at regular intervals.
+ * Can optionally sniff nodes on each failure as well.
+ */
 public class SniffingConnectionPool extends AbstractStaticConnectionPool {
 
     private static final Log logger = LogFactory.getLog(SniffingConnectionPool.class);

From e85ed3eb5251a2da080d2d349f8dc4ed826e30ef Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 6 May 2016 11:16:28 +0200
Subject: [PATCH 0591/1311] remove pinging from static connection pool, can be
 replaced by a low connect timeout on each request

---
 .../elasticsearch/client/ConnectionPool.java  |  9 +---
 .../org/elasticsearch/client/RestClient.java  |  7 ---
 .../client/StaticConnectionPool.java          | 48 +------------------
 .../client/sniff/SniffingConnectionPool.java  |  5 --
 .../elasticsearch/client/RestClientTests.java |  5 --
 .../client/StaticConnectionPoolTests.java     | 25 ++--------
 6 files changed, 6 insertions(+), 93 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/ConnectionPool.java b/client/src/main/java/org/elasticsearch/client/ConnectionPool.java
index 7e47a819256..b76376abf10 100644
--- a/client/src/main/java/org/elasticsearch/client/ConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/ConnectionPool.java
@@ -26,8 +26,7 @@ import java.util.stream.Stream;
 /**
  * Pool of connections to the different hosts that belong to an elasticsearch cluster.
  * It keeps track of the different hosts to communicate with and allows to retrieve a stream of connections to be used
- * for each request. Exposes the needed hooks to be able to eventually mark connections dead or alive and execute
- * arbitrary operations before each single request attempt.
+ * for each request. Exposes the needed hooks to be able to eventually mark connections dead or alive.
  */
 public interface ConnectionPool extends Closeable {
 
@@ -47,12 +46,6 @@ public interface ConnectionPool extends Closeable {
      */
     Connection lastResortConnection();
 
-    /**
-     * Called before each single request attempt. Allows to execute operations (e.g. ping) before each request.
-     * Receives as an argument the connection that is going to be used for the request.
-     */
-    void beforeAttempt(Connection connection) throws IOException;
-
     /**
      * Called after each successful request call.
      * Receives as an argument the connection that was used for the successful request.
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 3e59cc297e4..f2ff7c61522 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -95,13 +95,6 @@ public final class RestClient implements Closeable {
                 }
             }
 
-            try {
-                connectionPool.beforeAttempt(connection);
-            } catch(IOException e) {
-                lastSeenException = addSuppressedException(lastSeenException, e);
-                continue;
-            }
-
             try {
                 ElasticsearchResponse response = performRequest(request, connection);
                 connectionPool.onSuccess(connection);
diff --git a/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java b/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
index 5d54e583abe..289f14ce6ef 100644
--- a/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
@@ -19,41 +19,22 @@
 
 package org.elasticsearch.client;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.http.HttpHost;
-import org.apache.http.StatusLine;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpHead;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.util.EntityUtils;
 
 import java.io.IOException;
 import java.util.List;
-import java.util.Objects;
 
 /**
  * Static implementation of {@link ConnectionPool}. Its underlying list of connections is immutable.
  */
 public class StaticConnectionPool extends AbstractStaticConnectionPool {
 
-    private static final Log logger = LogFactory.getLog(StaticConnectionPool.class);
-
-    private final CloseableHttpClient client;
-    private final boolean pingEnabled;
-    private final RequestConfig pingRequestConfig;
     private final List<Connection> connections;
 
-    public StaticConnectionPool(CloseableHttpClient client, boolean pingEnabled, RequestConfig pingRequestConfig, HttpHost... hosts) {
-        Objects.requireNonNull(client, "client cannot be null");
-        Objects.requireNonNull(pingRequestConfig, "pingRequestConfig cannot be null");
+    public StaticConnectionPool(HttpHost... hosts) {
         if (hosts == null || hosts.length == 0) {
             throw new IllegalArgumentException("no hosts provided");
         }
-        this.client = client;
-        this.pingEnabled = pingEnabled;
-        this.pingRequestConfig = pingRequestConfig;
         this.connections = createConnections(hosts);
     }
 
@@ -62,33 +43,6 @@ public class StaticConnectionPool extends AbstractStaticConnectionPool {
         return connections;
     }
 
-    //TODO do we still need pinging? seems like a workaround for some clients that don't support connect timeout but we have that
-
-    @Override
-    public void beforeAttempt(Connection connection) throws IOException {
-        if (pingEnabled && connection.shouldBeRetried()) {
-            HttpHead httpHead = new HttpHead("/");
-            httpHead.setConfig(pingRequestConfig);
-            StatusLine statusLine;
-            try(CloseableHttpResponse httpResponse = client.execute(connection.getHost(), httpHead)) {
-                statusLine = httpResponse.getStatusLine();
-                EntityUtils.consume(httpResponse.getEntity());
-            } catch(IOException e) {
-                RequestLogger.log(logger, "ping failed", httpHead.getRequestLine(), connection.getHost(), e);
-                onFailure(connection);
-                throw e;
-            }
-            if (statusLine.getStatusCode() >= 300) {
-                RequestLogger.log(logger, "ping failed", httpHead.getRequestLine(), connection.getHost(), statusLine);
-                onFailure(connection);
-                throw new ElasticsearchResponseException(httpHead.getRequestLine(), connection.getHost(), statusLine);
-            } else {
-                RequestLogger.log(logger, "ping succeeded", httpHead.getRequestLine(), connection.getHost(), statusLine);
-                onSuccess(connection);
-            }
-        }
-    }
-
     @Override
     public void close() throws IOException {
         //no-op nothing to close
diff --git a/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java b/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
index 3fb7885fd14..934352deee0 100644
--- a/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
@@ -80,11 +80,6 @@ public class SniffingConnectionPool extends AbstractStaticConnectionPool {
         return this.connections;
     }
 
-    @Override
-    public void beforeAttempt(Connection connection) throws IOException {
-
-    }
-
     @Override
     public void onFailure(Connection connection) throws IOException {
         super.onFailure(connection);
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientTests.java b/client/src/test/java/org/elasticsearch/client/RestClientTests.java
index 52e18831591..6c3850fafcf 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientTests.java
@@ -47,11 +47,6 @@ public class RestClientTests extends LuceneTestCase {
                 return null;
             }
 
-            @Override
-            public void beforeAttempt(Connection connection) throws IOException {
-
-            }
-
             @Override
             public void onSuccess(Connection connection) {
 
diff --git a/client/src/test/java/org/elasticsearch/client/StaticConnectionPoolTests.java b/client/src/test/java/org/elasticsearch/client/StaticConnectionPoolTests.java
index 60398b71c0d..c777666e98b 100644
--- a/client/src/test/java/org/elasticsearch/client/StaticConnectionPoolTests.java
+++ b/client/src/test/java/org/elasticsearch/client/StaticConnectionPoolTests.java
@@ -21,9 +21,6 @@ package org.elasticsearch.client;
 
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
 import org.apache.http.HttpHost;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.lucene.util.LuceneTestCase;
 
 import java.util.logging.LogManager;
@@ -35,7 +32,6 @@ public class StaticConnectionPoolTests extends LuceneTestCase {
     }
 
     public void testConstructor() {
-        CloseableHttpClient httpClient = HttpClientBuilder.create().build();
         int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
         HttpHost[] hosts = new HttpHost[numNodes];
         for (int i = 0; i < numNodes; i++) {
@@ -43,37 +39,24 @@ public class StaticConnectionPoolTests extends LuceneTestCase {
         }
 
         try {
-            new StaticConnectionPool(null, random().nextBoolean(), RequestConfig.DEFAULT, hosts);
-        } catch(NullPointerException e) {
-            assertEquals(e.getMessage(), "client cannot be null");
-        }
-
-        try {
-            new StaticConnectionPool(httpClient, random().nextBoolean(), null, hosts);
-        } catch(NullPointerException e) {
-            assertEquals(e.getMessage(), "pingRequestConfig cannot be null");
-        }
-
-        try {
-            new StaticConnectionPool(httpClient, random().nextBoolean(), RequestConfig.DEFAULT, (HttpHost) null);
+            new StaticConnectionPool((HttpHost) null);
         } catch(NullPointerException e) {
             assertEquals(e.getMessage(), "host cannot be null");
         }
 
         try {
-            new StaticConnectionPool(httpClient, random().nextBoolean(), RequestConfig.DEFAULT, (HttpHost[])null);
+            new StaticConnectionPool((HttpHost[])null);
         } catch(IllegalArgumentException e) {
             assertEquals(e.getMessage(), "no hosts provided");
         }
 
         try {
-            new StaticConnectionPool(httpClient, random().nextBoolean(), RequestConfig.DEFAULT);
+            new StaticConnectionPool();
         } catch(IllegalArgumentException e) {
             assertEquals(e.getMessage(), "no hosts provided");
         }
 
-        StaticConnectionPool staticConnectionPool = new StaticConnectionPool(httpClient, random().nextBoolean(), RequestConfig.DEFAULT,
-                hosts);
+        StaticConnectionPool staticConnectionPool = new StaticConnectionPool(hosts);
         assertNotNull(staticConnectionPool);
     }
 }

From e77ab87926d5c0025882557a5d0948029d4d193e Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 6 May 2016 15:04:53 +0200
Subject: [PATCH 0592/1311] return the response as part of
 ElasticsearchResponseException

---
 .../ElasticsearchResponseException.java       | 23 ++++++++++---------
 .../org/elasticsearch/client/RestClient.java  |  4 +---
 .../elasticsearch/client/sniff/Sniffer.java   |  4 +---
 .../client/sniff/SnifferTests.java            |  4 ++--
 4 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
index 2d9ceead5ed..291b2290e46 100644
--- a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
+++ b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
@@ -21,7 +21,7 @@ package org.elasticsearch.client;
 
 import org.apache.http.HttpHost;
 import org.apache.http.RequestLine;
-import org.apache.http.StatusLine;
+import org.apache.http.client.methods.CloseableHttpResponse;
 
 import java.io.IOException;
 
@@ -32,25 +32,26 @@ public class ElasticsearchResponseException extends IOException {
 
     private final HttpHost host;
     private final RequestLine requestLine;
-    private final StatusLine statusLine;
+    private final CloseableHttpResponse response;
 
-    public ElasticsearchResponseException(RequestLine requestLine, HttpHost host, StatusLine statusLine) {
-        super(buildMessage(requestLine, host, statusLine));
+    public ElasticsearchResponseException(RequestLine requestLine, HttpHost host, CloseableHttpResponse response) {
+        super(buildMessage(requestLine, host, response));
         this.host = host;
         this.requestLine = requestLine;
-        this.statusLine = statusLine;
+        this.response = response;
     }
 
-    private static String buildMessage(RequestLine requestLine, HttpHost host, StatusLine statusLine) {
-        return requestLine.getMethod() + " " + host + requestLine.getUri() + ": " + statusLine.toString();
+    private static String buildMessage(RequestLine requestLine, HttpHost host, CloseableHttpResponse response) {
+        return requestLine.getMethod() + " " + host + requestLine.getUri() + ": " + response.getStatusLine().toString();
     }
 
     /**
      * Returns whether the error is recoverable or not, hence whether the same request should be retried on other nodes or not
      */
     public boolean isRecoverable() {
+        int statusCode = response.getStatusLine().getStatusCode();
         //clients don't retry on 500 because elasticsearch still misuses it instead of 400 in some places
-        return statusLine.getStatusCode() >= 502 && statusLine.getStatusCode() <= 504;
+        return statusCode >= 502 && statusCode <= 504;
     }
 
     /**
@@ -68,9 +69,9 @@ public class ElasticsearchResponseException extends IOException {
     }
 
     /**
-     * Returns the {@link StatusLine} that was returned by elasticsearch
+     * Returns the {@link CloseableHttpResponse} that was returned by elasticsearch
      */
-    public StatusLine getStatusLine() {
-        return statusLine;
+    public CloseableHttpResponse getResponse() {
+        return response;
     }
 }
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index f2ff7c61522..560c285b417 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -30,7 +30,6 @@ import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.util.EntityUtils;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -134,9 +133,8 @@ public final class RestClient implements Closeable {
             RequestLogger.log(logger, "request succeeded", request.getRequestLine(), connection.getHost(), response.getStatusLine());
             return new ElasticsearchResponse(request.getRequestLine(), connection.getHost(), response);
         } else {
-            EntityUtils.consume(response.getEntity());
             RequestLogger.log(logger, "request failed", request.getRequestLine(), connection.getHost(), response.getStatusLine());
-            throw new ElasticsearchResponseException(request.getRequestLine(), connection.getHost(), statusLine);
+            throw new ElasticsearchResponseException(request.getRequestLine(), connection.getHost(), response);
         }
     }
 
diff --git a/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
index ebed97e4a4e..77eb07e00e0 100644
--- a/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
+++ b/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
@@ -31,7 +31,6 @@ import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.ElasticsearchResponseException;
 import org.elasticsearch.client.RequestLogger;
 
@@ -78,8 +77,7 @@ final class Sniffer {
             StatusLine statusLine = response.getStatusLine();
             if (statusLine.getStatusCode() >= 300) {
                 RequestLogger.log(logger, "sniff failed", httpGet.getRequestLine(), host, statusLine);
-                EntityUtils.consume(response.getEntity());
-                throw new ElasticsearchResponseException(httpGet.getRequestLine(), host, statusLine);
+                throw new ElasticsearchResponseException(httpGet.getRequestLine(), host, response);
             } else {
                 List<HttpHost> nodes = readHosts(response.getEntity());
                 RequestLogger.log(logger, "sniff succeeded", httpGet.getRequestLine(), host, statusLine);
diff --git a/client/src/test/java/org/elasticsearch/client/sniff/SnifferTests.java b/client/src/test/java/org/elasticsearch/client/sniff/SnifferTests.java
index 4ef17579856..ea5e4ba2243 100644
--- a/client/src/test/java/org/elasticsearch/client/sniff/SnifferTests.java
+++ b/client/src/test/java/org/elasticsearch/client/sniff/SnifferTests.java
@@ -107,10 +107,10 @@ public class SnifferTests extends LuceneTestCase {
                         "/_nodes/http?timeout=" + sniffRequestTimeout));
                 assertThat(e.getMessage(), containsString(Integer.toString(sniffResponse.nodesInfoResponseCode)));
                 assertThat(e.getHost(), equalTo(httpHost));
-                assertThat(e.getStatusLine().getStatusCode(), equalTo(sniffResponse.nodesInfoResponseCode));
+                assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(sniffResponse.nodesInfoResponseCode));
                 assertThat(e.getRequestLine().toString(), equalTo("GET /_nodes/http?timeout=" + sniffRequestTimeout + "ms HTTP/1.1"));
             } else {
-                fail("sniffNodes should have succeeded: " + e.getStatusLine());
+                fail("sniffNodes should have succeeded: " + e.getResponse().getStatusLine());
             }
         }
     }

From 530ad227a2e7bd63e80f72ee52b2502def1c724f Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 6 May 2016 15:51:23 +0200
Subject: [PATCH 0593/1311] prevent unclosed response entities in
 ElasticsearchResponseException, eagerly read response string in case of error
 status code

---
 .../ElasticsearchResponseException.java       | 36 +++++-----
 .../org/elasticsearch/client/RestClient.java  | 66 +++++++++----------
 .../elasticsearch/client/sniff/Sniffer.java   |  7 +-
 .../client/sniff/SnifferTests.java            |  8 +--
 4 files changed, 56 insertions(+), 61 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
index 291b2290e46..98945b9a726 100644
--- a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
+++ b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
@@ -21,7 +21,7 @@ package org.elasticsearch.client;
 
 import org.apache.http.HttpHost;
 import org.apache.http.RequestLine;
-import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.StatusLine;
 
 import java.io.IOException;
 
@@ -32,26 +32,19 @@ public class ElasticsearchResponseException extends IOException {
 
     private final HttpHost host;
     private final RequestLine requestLine;
-    private final CloseableHttpResponse response;
+    private final StatusLine statusLine;
+    private final String responseBody;
 
-    public ElasticsearchResponseException(RequestLine requestLine, HttpHost host, CloseableHttpResponse response) {
-        super(buildMessage(requestLine, host, response));
+    public ElasticsearchResponseException(RequestLine requestLine, HttpHost host, StatusLine statusLine, String responseBody) {
+        super(buildMessage(requestLine, host, statusLine));
         this.host = host;
         this.requestLine = requestLine;
-        this.response = response;
+        this.responseBody = responseBody;
+        this.statusLine = statusLine;
     }
 
-    private static String buildMessage(RequestLine requestLine, HttpHost host, CloseableHttpResponse response) {
-        return requestLine.getMethod() + " " + host + requestLine.getUri() + ": " + response.getStatusLine().toString();
-    }
-
-    /**
-     * Returns whether the error is recoverable or not, hence whether the same request should be retried on other nodes or not
-     */
-    public boolean isRecoverable() {
-        int statusCode = response.getStatusLine().getStatusCode();
-        //clients don't retry on 500 because elasticsearch still misuses it instead of 400 in some places
-        return statusCode >= 502 && statusCode <= 504;
+    private static String buildMessage(RequestLine requestLine, HttpHost host, StatusLine statusLine) {
+        return requestLine.getMethod() + " " + host + requestLine.getUri() + ": " + statusLine.toString();
     }
 
     /**
@@ -68,10 +61,11 @@ public class ElasticsearchResponseException extends IOException {
         return requestLine;
     }
 
-    /**
-     * Returns the {@link CloseableHttpResponse} that was returned by elasticsearch
-     */
-    public CloseableHttpResponse getResponse() {
-        return response;
+    public StatusLine getStatusLine() {
+        return statusLine;
+    }
+
+    public String getResponseBody() {
+        return responseBody;
     }
 }
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 560c285b417..9409a4b7ba3 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -21,7 +21,6 @@ package org.elasticsearch.client;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.http.HttpEntity;
-import org.apache.http.StatusLine;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
 import org.apache.http.client.methods.HttpHead;
@@ -30,6 +29,7 @@ import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.util.EntityUtils;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -94,50 +94,46 @@ public final class RestClient implements Closeable {
                 }
             }
 
+            CloseableHttpResponse response;
             try {
-                ElasticsearchResponse response = performRequest(request, connection);
-                connectionPool.onSuccess(connection);
-                return response;
-            } catch(ElasticsearchResponseException e) {
-                if (e.isRecoverable()) {
-                    connectionPool.onFailure(connection);
-                    lastSeenException = addSuppressedException(lastSeenException, e);
-                } else {
-                    //don't retry and call onSuccess as the error should be a request problem
-                    connectionPool.onSuccess(connection);
-                    throw e;
-                }
+                response = client.execute(connection.getHost(), request);
             } catch(IOException e) {
+                RequestLogger.log(logger, "request failed", request.getRequestLine(), connection.getHost(), e);
                 connectionPool.onFailure(connection);
                 lastSeenException = addSuppressedException(lastSeenException, e);
+                continue;
+            } finally {
+                request.reset();
+            }
+            int statusCode = response.getStatusLine().getStatusCode();
+            //TODO make ignore status code configurable. rest-spec and tests support that parameter (ignore_missing)
+            if (statusCode < 300 || request.getMethod().equals(HttpHead.METHOD_NAME) && statusCode == 404) {
+                RequestLogger.log(logger, "request succeeded", request.getRequestLine(), connection.getHost(), response.getStatusLine());
+                connectionPool.onSuccess(connection);
+                return new ElasticsearchResponse(request.getRequestLine(), connection.getHost(), response);
+            } else {
+                RequestLogger.log(logger, "request failed", request.getRequestLine(), connection.getHost(), response.getStatusLine());
+                String responseBody = null;
+                if (response.getEntity() != null) {
+                    responseBody = EntityUtils.toString(response.getEntity());
+                }
+                ElasticsearchResponseException elasticsearchResponseException = new ElasticsearchResponseException(
+                        request.getRequestLine(), connection.getHost(), response.getStatusLine(), responseBody);
+                lastSeenException = addSuppressedException(lastSeenException, elasticsearchResponseException);
+                //clients don't retry on 500 because elasticsearch still misuses it instead of 400 in some places
+                if (statusCode == 502 || statusCode == 503 || statusCode == 504) {
+                    connectionPool.onFailure(connection);
+                } else {
+                    //don't retry and call onSuccess as the error should be a request problem, the node is alive
+                    connectionPool.onSuccess(connection);
+                    break;
+                }
             }
         }
         assert lastSeenException != null;
         throw lastSeenException;
     }
 
-    private ElasticsearchResponse performRequest(HttpRequestBase request, Connection connection) throws IOException {
-        CloseableHttpResponse response;
-        try {
-            response = client.execute(connection.getHost(), request);
-        } catch(IOException e) {
-            RequestLogger.log(logger, "request failed", request.getRequestLine(), connection.getHost(), e);
-            throw e;
-        } finally {
-            request.reset();
-        }
-        StatusLine statusLine = response.getStatusLine();
-        //TODO make ignore status code configurable. rest-spec and tests support that parameter.
-        if (statusLine.getStatusCode() < 300 ||
-                request.getMethod().equals(HttpHead.METHOD_NAME) && statusLine.getStatusCode() == 404) {
-            RequestLogger.log(logger, "request succeeded", request.getRequestLine(), connection.getHost(), response.getStatusLine());
-            return new ElasticsearchResponse(request.getRequestLine(), connection.getHost(), response);
-        } else {
-            RequestLogger.log(logger, "request failed", request.getRequestLine(), connection.getHost(), response.getStatusLine());
-            throw new ElasticsearchResponseException(request.getRequestLine(), connection.getHost(), response);
-        }
-    }
-
     private static IOException addSuppressedException(IOException suppressedException, IOException currentException) {
         if (suppressedException != null) {
             currentException.addSuppressed(suppressedException);
diff --git a/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
index 77eb07e00e0..4567e27566a 100644
--- a/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
+++ b/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
@@ -31,6 +31,7 @@ import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.ElasticsearchResponseException;
 import org.elasticsearch.client.RequestLogger;
 
@@ -77,7 +78,11 @@ final class Sniffer {
             StatusLine statusLine = response.getStatusLine();
             if (statusLine.getStatusCode() >= 300) {
                 RequestLogger.log(logger, "sniff failed", httpGet.getRequestLine(), host, statusLine);
-                throw new ElasticsearchResponseException(httpGet.getRequestLine(), host, response);
+                String responseBody = null;
+                if (response.getEntity() != null) {
+                    responseBody = EntityUtils.toString(response.getEntity());
+                }
+                throw new ElasticsearchResponseException(httpGet.getRequestLine(), host, response.getStatusLine(), responseBody);
             } else {
                 List<HttpHost> nodes = readHosts(response.getEntity());
                 RequestLogger.log(logger, "sniff succeeded", httpGet.getRequestLine(), host, statusLine);
diff --git a/client/src/test/java/org/elasticsearch/client/sniff/SnifferTests.java b/client/src/test/java/org/elasticsearch/client/sniff/SnifferTests.java
index ea5e4ba2243..6cb7905151f 100644
--- a/client/src/test/java/org/elasticsearch/client/sniff/SnifferTests.java
+++ b/client/src/test/java/org/elasticsearch/client/sniff/SnifferTests.java
@@ -89,7 +89,7 @@ public class SnifferTests extends LuceneTestCase {
 
     public void testSniffNodes() throws IOException, URISyntaxException {
         CloseableHttpClient client = HttpClientBuilder.create().build();
-        Sniffer sniffer = new Sniffer(client, RequestConfig.DEFAULT, sniffRequestTimeout, scheme.toString());
+        Sniffer sniffer = new Sniffer(client, RequestConfig.DEFAULT, sniffRequestTimeout, scheme);
         HttpHost httpHost = new HttpHost(server.getHostName(), server.getPort());
         try {
             List<HttpHost> sniffedHosts = sniffer.sniffNodes(httpHost);
@@ -107,10 +107,10 @@ public class SnifferTests extends LuceneTestCase {
                         "/_nodes/http?timeout=" + sniffRequestTimeout));
                 assertThat(e.getMessage(), containsString(Integer.toString(sniffResponse.nodesInfoResponseCode)));
                 assertThat(e.getHost(), equalTo(httpHost));
-                assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(sniffResponse.nodesInfoResponseCode));
+                assertThat(e.getStatusLine().getStatusCode(), equalTo(sniffResponse.nodesInfoResponseCode));
                 assertThat(e.getRequestLine().toString(), equalTo("GET /_nodes/http?timeout=" + sniffRequestTimeout + "ms HTTP/1.1"));
             } else {
-                fail("sniffNodes should have succeeded: " + e.getResponse().getStatusLine());
+                fail("sniffNodes should have succeeded: " + e.getStatusLine());
             }
         }
     }
@@ -170,7 +170,7 @@ public class SnifferTests extends LuceneTestCase {
             if (isHttpEnabled) {
                 String host = "host" + i;
                 int port = RandomInts.randomIntBetween(random(), 9200, 9299);
-                HttpHost httpHost = new HttpHost(host, port, scheme.toString());
+                HttpHost httpHost = new HttpHost(host, port, scheme);
                 hosts.add(httpHost);
                 generator.writeObjectFieldStart("http");
                 if (random().nextBoolean()) {

From e7fe397c39712731e54fd4b35182e45010344b87 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 6 May 2016 15:53:30 +0200
Subject: [PATCH 0594/1311] add missing parentheses

---
 client/src/main/java/org/elasticsearch/client/RestClient.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 9409a4b7ba3..21ad8f7200b 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -107,7 +107,7 @@ public final class RestClient implements Closeable {
             }
             int statusCode = response.getStatusLine().getStatusCode();
             //TODO make ignore status code configurable. rest-spec and tests support that parameter (ignore_missing)
-            if (statusCode < 300 || request.getMethod().equals(HttpHead.METHOD_NAME) && statusCode == 404) {
+            if (statusCode < 300 || (request.getMethod().equals(HttpHead.METHOD_NAME) && statusCode == 404) ) {
                 RequestLogger.log(logger, "request succeeded", request.getRequestLine(), connection.getHost(), response.getStatusLine());
                 connectionPool.onSuccess(connection);
                 return new ElasticsearchResponse(request.getRequestLine(), connection.getHost(), response);

From 17a21f0272b02365160532580f2ab74e926257f6 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Mon, 9 May 2016 11:14:25 +0200
Subject: [PATCH 0595/1311] add curl format trace logging for requests and
 responses

---
 .../elasticsearch/client/RequestLogger.java   |  94 +++++++++++++--
 .../org/elasticsearch/client/RestClient.java  |   6 +-
 .../elasticsearch/client/sniff/Sniffer.java   |   6 +-
 .../client/RequestLoggerTests.java            | 107 ++++++++++++++++++
 4 files changed, 200 insertions(+), 13 deletions(-)
 create mode 100644 client/src/test/java/org/elasticsearch/client/RequestLoggerTests.java

diff --git a/client/src/main/java/org/elasticsearch/client/RequestLogger.java b/client/src/main/java/org/elasticsearch/client/RequestLogger.java
index 99ce924c164..ce2e82ea1e5 100644
--- a/client/src/main/java/org/elasticsearch/client/RequestLogger.java
+++ b/client/src/main/java/org/elasticsearch/client/RequestLogger.java
@@ -20,31 +20,111 @@
 package org.elasticsearch.client;
 
 import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpEntityEnclosingRequest;
 import org.apache.http.HttpHost;
-import org.apache.http.RequestLine;
-import org.apache.http.StatusLine;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.entity.BufferedHttpEntity;
+import org.apache.http.util.EntityUtils;
 
+import java.io.BufferedReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
 
 /**
- * Helper class that exposes static methods to unify the way requests are logged
+ * Helper class that exposes static methods to unify the way requests are logged.
+ * Includes trace logging to log complete requests and responses in curl format.
  */
 public final class RequestLogger {
 
+    private static final Log tracer = LogFactory.getLog("tracer");
+
     private RequestLogger() {
     }
 
     /**
      * Logs a request that yielded a response
      */
-    public static void log(Log logger, String message, RequestLine requestLine, HttpHost host, StatusLine statusLine) {
-        logger.debug(message + " [" + requestLine.getMethod() + " " + host + requestLine.getUri() + "] [" + statusLine + "]");
+    public static void log(Log logger, String message, HttpUriRequest request, HttpHost host, HttpResponse httpResponse) {
+        logger.debug(message + " [" + request.getMethod() + " " + host + request.getRequestLine().getUri() +
+                "] [" + httpResponse.getStatusLine() + "]");
+
+        if (tracer.isTraceEnabled()) {
+            String requestLine;
+            try {
+                requestLine = buildTraceRequest(request, host);
+            } catch(IOException e) {
+                requestLine = "";
+                tracer.trace("error while reading request for trace purposes", e);
+            }
+            String responseLine;
+            try {
+                responseLine = buildTraceResponse(httpResponse);
+            } catch(IOException e) {
+                responseLine = "";
+                tracer.trace("error while reading response for trace purposes", e);
+            }
+            tracer.trace(requestLine + '\n' + responseLine);
+        }
     }
 
     /**
      * Logs a request that failed
      */
-    public static void log(Log logger, String message, RequestLine requestLine, HttpHost host, IOException e) {
-        logger.debug(message + " [" + requestLine.getMethod() + " " + host + requestLine.getUri() + "]", e);
+    public static void log(Log logger, String message, HttpUriRequest request, HttpHost host, IOException e) {
+        logger.debug(message + " [" + request.getMethod() + " " + host + request.getRequestLine().getUri() + "]", e);
+        if (logger.isTraceEnabled()) {
+            String traceRequest;
+            try {
+                traceRequest = buildTraceRequest(request, host);
+            } catch (IOException e1) {
+                tracer.trace("error while reading request for trace purposes", e);
+                traceRequest = "";
+            }
+            tracer.trace(traceRequest);
+        }
+    }
+
+    /**
+     * Creates curl output for given request
+     */
+    static String buildTraceRequest(HttpUriRequest request, HttpHost host) throws IOException {
+        String requestLine = "curl -iX " + request.getMethod() + " '" + host + request.getRequestLine().getUri() + "'";
+        if (request instanceof  HttpEntityEnclosingRequest) {
+            HttpEntityEnclosingRequest enclosingRequest = (HttpEntityEnclosingRequest) request;
+            if (enclosingRequest.getEntity() != null) {
+                requestLine += " -d '";
+                HttpEntity entity = new BufferedHttpEntity(enclosingRequest.getEntity());
+                enclosingRequest.setEntity(entity);
+                requestLine += EntityUtils.toString(entity) + "'";
+            }
+        }
+        return requestLine;
+    }
+
+    /**
+     * Creates curl output for given response
+     */
+    static String buildTraceResponse(HttpResponse httpResponse) throws IOException {
+        String responseLine = "# " + httpResponse.getStatusLine().toString();
+        for (Header header : httpResponse.getAllHeaders()) {
+            responseLine += "\n# " + header.getName() + ": " + header.getValue();
+        }
+        responseLine += "\n#";
+        HttpEntity entity = httpResponse.getEntity();
+        if (entity != null) {
+            entity = new BufferedHttpEntity(entity);
+            httpResponse.setEntity(entity);
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(entity.getContent()))) {
+                String line;
+                while( (line = reader.readLine()) != null) {
+                    responseLine += "\n# " + line;
+                }
+            }
+        }
+        return responseLine;
     }
 }
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 21ad8f7200b..945934d5ffe 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -98,7 +98,7 @@ public final class RestClient implements Closeable {
             try {
                 response = client.execute(connection.getHost(), request);
             } catch(IOException e) {
-                RequestLogger.log(logger, "request failed", request.getRequestLine(), connection.getHost(), e);
+                RequestLogger.log(logger, "request failed", request, connection.getHost(), e);
                 connectionPool.onFailure(connection);
                 lastSeenException = addSuppressedException(lastSeenException, e);
                 continue;
@@ -108,11 +108,11 @@ public final class RestClient implements Closeable {
             int statusCode = response.getStatusLine().getStatusCode();
             //TODO make ignore status code configurable. rest-spec and tests support that parameter (ignore_missing)
             if (statusCode < 300 || (request.getMethod().equals(HttpHead.METHOD_NAME) && statusCode == 404) ) {
-                RequestLogger.log(logger, "request succeeded", request.getRequestLine(), connection.getHost(), response.getStatusLine());
+                RequestLogger.log(logger, "request succeeded", request, connection.getHost(), response);
                 connectionPool.onSuccess(connection);
                 return new ElasticsearchResponse(request.getRequestLine(), connection.getHost(), response);
             } else {
-                RequestLogger.log(logger, "request failed", request.getRequestLine(), connection.getHost(), response.getStatusLine());
+                RequestLogger.log(logger, "request failed", request, connection.getHost(), response);
                 String responseBody = null;
                 if (response.getEntity() != null) {
                     responseBody = EntityUtils.toString(response.getEntity());
diff --git a/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
index 4567e27566a..7aa4e5f5f35 100644
--- a/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
+++ b/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
@@ -77,7 +77,7 @@ final class Sniffer {
         try (CloseableHttpResponse response = client.execute(host, httpGet)) {
             StatusLine statusLine = response.getStatusLine();
             if (statusLine.getStatusCode() >= 300) {
-                RequestLogger.log(logger, "sniff failed", httpGet.getRequestLine(), host, statusLine);
+                RequestLogger.log(logger, "sniff failed", httpGet, host, response);
                 String responseBody = null;
                 if (response.getEntity() != null) {
                     responseBody = EntityUtils.toString(response.getEntity());
@@ -85,11 +85,11 @@ final class Sniffer {
                 throw new ElasticsearchResponseException(httpGet.getRequestLine(), host, response.getStatusLine(), responseBody);
             } else {
                 List<HttpHost> nodes = readHosts(response.getEntity());
-                RequestLogger.log(logger, "sniff succeeded", httpGet.getRequestLine(), host, statusLine);
+                RequestLogger.log(logger, "sniff succeeded", httpGet, host, response);
                 return nodes;
             }
         } catch(IOException e) {
-            RequestLogger.log(logger, "sniff failed", httpGet.getRequestLine(), host, e);
+            RequestLogger.log(logger, "sniff failed", httpGet, host, e);
             throw e;
         }
     }
diff --git a/client/src/test/java/org/elasticsearch/client/RequestLoggerTests.java b/client/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
new file mode 100644
index 00000000000..84bc3c032d5
--- /dev/null
+++ b/client/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import org.apache.commons.codec.Charsets;
+import org.apache.http.HttpEntityEnclosingRequest;
+import org.apache.http.HttpHost;
+import org.apache.http.ProtocolVersion;
+import org.apache.http.client.methods.HttpHead;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.message.BasicHttpResponse;
+import org.apache.http.message.BasicStatusLine;
+import org.apache.lucene.util.LuceneTestCase;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class RequestLoggerTests extends LuceneTestCase {
+
+    public void testTraceRequest() throws IOException, URISyntaxException {
+        HttpHost host = new HttpHost("localhost", 9200, random().nextBoolean() ? "http" : "https");
+        URI uri = new URI("/index/type/_api");
+
+        HttpRequestBase request;
+        int requestType = RandomInts.randomIntBetween(random(), 0, 4);
+        switch(requestType) {
+            case 0:
+                request = new HttpGetWithEntity(uri);
+                break;
+            case 1:
+                request = new HttpPost(uri);
+                break;
+            case 2:
+                request = new HttpPut(uri);
+                break;
+            case 3:
+                request = new HttpDeleteWithEntity(uri);
+                break;
+            case 4:
+                request = new HttpHead(uri);
+                break;
+            default:
+                throw new UnsupportedOperationException();
+        }
+
+        String expected = "curl -iX " + request.getMethod() + " '" + host + uri + "'";
+
+        if (request instanceof HttpEntityEnclosingRequest && random().nextBoolean()) {
+            HttpEntityEnclosingRequest enclosingRequest = (HttpEntityEnclosingRequest) request;
+            String requestBody = "{ \"field\": \"value\" }";
+            enclosingRequest.setEntity(new StringEntity(requestBody, Charsets.UTF_8));
+            expected += " -d '" + requestBody + "'";
+        }
+
+        String traceRequest = RequestLogger.buildTraceRequest(request, host);
+        assertThat(traceRequest, equalTo(expected));
+    }
+
+    public void testTraceResponse() throws IOException {
+        ProtocolVersion protocolVersion = new ProtocolVersion("HTTP", 1, 1);
+        int statusCode = RandomInts.randomIntBetween(random(), 200, 599);
+        String reasonPhrase = "REASON";
+        BasicStatusLine statusLine = new BasicStatusLine(protocolVersion, statusCode, reasonPhrase);
+        String expected = "# " + statusLine.toString();
+        BasicHttpResponse httpResponse = new BasicHttpResponse(statusLine);
+        int numHeaders = RandomInts.randomIntBetween(random(), 0, 3);
+        for (int i = 0; i < numHeaders; i++) {
+            httpResponse.setHeader("header" + i, "value");
+            expected += "\n# header" + i + ": value";
+        }
+        expected += "\n#";
+        if (random().nextBoolean()) {
+            String responseBody = "{\n  \"field\": \"value\"\n}";
+            httpResponse.setEntity(new StringEntity(responseBody, Charsets.UTF_8));
+            expected += "\n# {";
+            expected += "\n#   \"field\": \"value\"";
+            expected += "\n# }";
+        }
+
+        String traceResponse = RequestLogger.buildTraceResponse(httpResponse);
+        assertThat(traceResponse, equalTo(expected));
+    }
+}

From e040d2fc779fcf3ed12cdcb29f92e5a3e5acc0f3 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Mon, 9 May 2016 12:28:47 +0200
Subject: [PATCH 0596/1311] remove ConnectionPool interface

---
 .../client/AbstractStaticConnectionPool.java  | 103 ------------------
 .../elasticsearch/client/ConnectionPool.java  |  76 +++++++++++--
 .../client/StaticConnectionPool.java          |   2 +-
 .../client/sniff/SniffingConnectionPool.java  |   4 +-
 .../elasticsearch/client/RestClientTests.java |  12 +-
 5 files changed, 76 insertions(+), 121 deletions(-)
 delete mode 100644 client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java

diff --git a/client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java b/client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java
deleted file mode 100644
index 3fe4ead4bbe..00000000000
--- a/client/src/main/java/org/elasticsearch/client/AbstractStaticConnectionPool.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.client;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.http.HttpHost;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Objects;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Stream;
-
-/**
- * Base static connection pool implementation that marks connections as dead/alive when needed.
- * Provides a stream of alive connections or dead ones that should be retried for each {@link #nextConnection()} call.
- * In case the returned stream is empty a last resort dead connection should be retrieved by calling {@link #lastResortConnection()}
- * and resurrected so that a last resort request attempt can be performed.
- * The {@link #onSuccess(Connection)} method marks the connection provided as an argument alive.
- * The {@link #onFailure(Connection)} method marks the connection provided as an argument dead.
- * This base implementation doesn't define the list implementation that stores connections, so that concurrency can be
- * handled in subclasses depending on the usecase (e.g. defining the list volatile or final when needed).
- */
-public abstract class AbstractStaticConnectionPool implements ConnectionPool {
-
-    private static final Log logger = LogFactory.getLog(AbstractStaticConnectionPool.class);
-
-    private final AtomicInteger lastConnectionIndex = new AtomicInteger(0);
-
-    /**
-     * Allows to retrieve the concrete list of connections. Not defined directly as a member
-     * of this class as subclasses may need to handle concurrency if the list can change, for
-     * instance defining the field as volatile. On the other hand static implementations
-     * can just make the list final instead.
-     */
-    protected abstract List<Connection> getConnections();
-
-    @Override
-    public final Stream<Connection> nextConnection() {
-        List<Connection> connections = getConnections();
-        if (connections.isEmpty()) {
-            throw new IllegalStateException("no connections available in the connection pool");
-        }
-
-        List<Connection> sortedConnections = new ArrayList<>(connections);
-        //TODO is it possible to make this O(1)? (rotate is O(n))
-        Collections.rotate(sortedConnections, sortedConnections.size() - lastConnectionIndex.getAndIncrement());
-        return sortedConnections.stream().filter(connection -> connection.isAlive() || connection.shouldBeRetried());
-    }
-
-    /**
-     * Helper method to be used by subclasses when needing to create a new list
-     * of connections given their corresponding hosts
-     */
-    protected List<Connection> createConnections(HttpHost... hosts) {
-        List<Connection> connections = new ArrayList<>();
-        for (HttpHost host : hosts) {
-            Objects.requireNonNull(host, "host cannot be null");
-            connections.add(new Connection(host));
-        }
-        return Collections.unmodifiableList(connections);
-    }
-
-    @Override
-    public Connection lastResortConnection() {
-        Connection Connection = getConnections().stream()
-                .sorted((o1, o2) -> Long.compare(o1.getDeadUntil(), o2.getDeadUntil())).findFirst().get();
-        Connection.markResurrected();
-        return Connection;
-    }
-
-    @Override
-    public void onSuccess(Connection connection) {
-        connection.markAlive();
-        logger.trace("marked connection alive for " + connection.getHost());
-    }
-
-    @Override
-    public void onFailure(Connection connection) throws IOException {
-        connection.markDead();
-        logger.debug("marked connection dead for " + connection.getHost());
-    }
-}
diff --git a/client/src/main/java/org/elasticsearch/client/ConnectionPool.java b/client/src/main/java/org/elasticsearch/client/ConnectionPool.java
index b76376abf10..b0be3a27075 100644
--- a/client/src/main/java/org/elasticsearch/client/ConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/ConnectionPool.java
@@ -19,16 +19,44 @@
 
 package org.elasticsearch.client;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.http.HttpHost;
+
 import java.io.Closeable;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Stream;
 
 /**
  * Pool of connections to the different hosts that belong to an elasticsearch cluster.
  * It keeps track of the different hosts to communicate with and allows to retrieve a stream of connections to be used
- * for each request. Exposes the needed hooks to be able to eventually mark connections dead or alive.
+ * for each request. Marks connections as dead/alive when needed.
+ * Provides a stream of alive connections or dead ones that should be retried for each {@link #nextConnection()} call.
+ * In case the returned stream is empty a last resort dead connection should be retrieved by calling {@link #lastResortConnection()}
+ * and resurrected so that a last resort request attempt can be performed.
+ * The {@link #onSuccess(Connection)} method marks the connection provided as an argument alive.
+ * The {@link #onFailure(Connection)} method marks the connection provided as an argument dead.
+ * This base implementation doesn't define the list implementation that stores connections, so that concurrency can be
+ * handled in subclasses depending on the usecase (e.g. defining the list volatile or final when needed).
  */
-public interface ConnectionPool extends Closeable {
+public abstract class ConnectionPool implements Closeable {
+
+    private static final Log logger = LogFactory.getLog(ConnectionPool.class);
+
+    private final AtomicInteger lastConnectionIndex = new AtomicInteger(0);
+
+    /**
+     * Allows to retrieve the concrete list of connections. Not defined directly as a member
+     * of this class as subclasses may need to handle concurrency if the list can change, for
+     * instance defining the field as volatile. On the other hand static implementations
+     * can just make the list final instead.
+     */
+    protected abstract List<Connection> getConnections();
 
     /**
      * Returns a stream of connections that should be used for a request call.
@@ -38,23 +66,57 @@ public interface ConnectionPool extends Closeable {
      * It may happen that the stream is empty, in which case it means that there aren't healthy connections to use.
      * Then {@link #lastResortConnection()} should be called to retrieve a non healthy connection and try it.
      */
-    Stream<Connection> nextConnection();
+    public final Stream<Connection> nextConnection() {
+        List<Connection> connections = getConnections();
+        if (connections.isEmpty()) {
+            throw new IllegalStateException("no connections available in the connection pool");
+        }
+
+        List<Connection> sortedConnections = new ArrayList<>(connections);
+        //TODO is it possible to make this O(1)? (rotate is O(n))
+        Collections.rotate(sortedConnections, sortedConnections.size() - lastConnectionIndex.getAndIncrement());
+        return sortedConnections.stream().filter(connection -> connection.isAlive() || connection.shouldBeRetried());
+    }
+
+    /**
+     * Helper method to be used by subclasses when needing to create a new list
+     * of connections given their corresponding hosts
+     */
+    protected final List<Connection> createConnections(HttpHost... hosts) {
+        List<Connection> connections = new ArrayList<>();
+        for (HttpHost host : hosts) {
+            Objects.requireNonNull(host, "host cannot be null");
+            connections.add(new Connection(host));
+        }
+        return Collections.unmodifiableList(connections);
+    }
 
     /**
      * Returns a connection that is not necessarily healthy, but can be used for a request attempt. To be called as last resort
-     * only in case {@link #nextConnection()} returns an empty stream
+     * only in case {@link #nextConnection()} returns an empty stream.
      */
-    Connection lastResortConnection();
+    public final Connection lastResortConnection() {
+        Connection Connection = getConnections().stream()
+                .sorted((o1, o2) -> Long.compare(o1.getDeadUntil(), o2.getDeadUntil())).findFirst().get();
+        Connection.markResurrected();
+        return Connection;
+    }
 
     /**
      * Called after each successful request call.
      * Receives as an argument the connection that was used for the successful request.
      */
-    void onSuccess(Connection connection);
+    public void onSuccess(Connection connection) {
+        connection.markAlive();
+        logger.trace("marked connection alive for " + connection.getHost());
+    }
 
     /**
      * Called after each failed attempt.
      * Receives as an argument the connection that was used for the failed attempt.
      */
-    void onFailure(Connection connection) throws IOException;
+    public void onFailure(Connection connection) throws IOException {
+        connection.markDead();
+        logger.debug("marked connection dead for " + connection.getHost());
+    }
 }
diff --git a/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java b/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
index 289f14ce6ef..4ec0bcb3c39 100644
--- a/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
@@ -27,7 +27,7 @@ import java.util.List;
 /**
  * Static implementation of {@link ConnectionPool}. Its underlying list of connections is immutable.
  */
-public class StaticConnectionPool extends AbstractStaticConnectionPool {
+public class StaticConnectionPool extends ConnectionPool {
 
     private final List<Connection> connections;
 
diff --git a/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java b/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
index 934352deee0..a2973638ad4 100644
--- a/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
@@ -24,7 +24,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.http.HttpHost;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.impl.client.CloseableHttpClient;
-import org.elasticsearch.client.AbstractStaticConnectionPool;
+import org.elasticsearch.client.ConnectionPool;
 import org.elasticsearch.client.Connection;
 
 import java.io.IOException;
@@ -43,7 +43,7 @@ import java.util.stream.Stream;
  * Connection pool implementation that sniffs nodes from elasticsearch at regular intervals.
  * Can optionally sniff nodes on each failure as well.
  */
-public class SniffingConnectionPool extends AbstractStaticConnectionPool {
+public class SniffingConnectionPool extends ConnectionPool {
 
     private static final Log logger = LogFactory.getLog(SniffingConnectionPool.class);
 
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientTests.java b/client/src/test/java/org/elasticsearch/client/RestClientTests.java
index 6c3850fafcf..0d9930d9c9f 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientTests.java
@@ -25,8 +25,9 @@ import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.lucene.util.LuceneTestCase;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
 import java.util.logging.LogManager;
-import java.util.stream.Stream;
 
 public class RestClientTests extends LuceneTestCase {
 
@@ -38,13 +39,8 @@ public class RestClientTests extends LuceneTestCase {
         CloseableHttpClient httpClient = HttpClientBuilder.create().build();
         ConnectionPool connectionPool = new ConnectionPool() {
             @Override
-            public Stream<Connection> nextConnection() {
-                return null;
-            }
-
-            @Override
-            public Connection lastResortConnection() {
-                return null;
+            protected List<Connection> getConnections() {
+                return Collections.emptyList();
             }
 
             @Override

From 9569ebc262a20f629a9953f3da2724f7bc0c3841 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Mon, 9 May 2016 16:37:39 +0200
Subject: [PATCH 0597/1311] add builders for simple creation of RestClient and
 SniffingConnectionPool instances

We have quite some constructor parameters and some defaults should be applied, builders help simplifying creation of objects for users.
---
 .../org/elasticsearch/client/RestClient.java  | 109 +++++++++++-
 .../elasticsearch/client/sniff/Sniffer.java   |   7 -
 .../client/sniff/SniffingConnectionPool.java  | 152 +++++++++++++++--
 .../client/RestClientBuilderTests.java        | 102 ++++++++++++
 .../elasticsearch/client/RestClientTests.java |  87 ----------
 .../SniffingConnectionPoolBuilderTests.java   | 138 ++++++++++++++++
 .../sniff/SniffingConnectionPoolTests.java    | 156 ------------------
 7 files changed, 476 insertions(+), 275 deletions(-)
 create mode 100644 client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
 delete mode 100644 client/src/test/java/org/elasticsearch/client/RestClientTests.java
 create mode 100644 client/src/test/java/org/elasticsearch/client/sniff/SniffingConnectionPoolBuilderTests.java
 delete mode 100644 client/src/test/java/org/elasticsearch/client/sniff/SniffingConnectionPoolTests.java

diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 945934d5ffe..7fdc8e57e3a 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -21,6 +21,8 @@ package org.elasticsearch.client;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
+import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
 import org.apache.http.client.methods.HttpHead;
@@ -29,6 +31,8 @@ import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.apache.http.util.EntityUtils;
 
 import java.io.Closeable;
@@ -38,7 +42,6 @@ import java.net.URISyntaxException;
 import java.util.Iterator;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Stream;
 
@@ -50,12 +53,7 @@ public final class RestClient implements Closeable {
     private final ConnectionPool connectionPool;
     private final long maxRetryTimeout;
 
-    public RestClient(CloseableHttpClient client, ConnectionPool connectionPool, long maxRetryTimeout) {
-        Objects.requireNonNull(client, "client cannot be null");
-        Objects.requireNonNull(connectionPool, "connectionPool cannot be null");
-        if (maxRetryTimeout <= 0) {
-            throw new IllegalArgumentException("maxRetryTimeout must be greater than 0");
-        }
+    private RestClient(CloseableHttpClient client, ConnectionPool connectionPool, long maxRetryTimeout) {
         this.client = client;
         this.connectionPool = connectionPool;
         this.maxRetryTimeout = maxRetryTimeout;
@@ -192,4 +190,101 @@ public final class RestClient implements Closeable {
         connectionPool.close();
         client.close();
     }
+
+    /**
+     * Returns a new {@link Builder} to help with {@link RestClient} creation.
+     */
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    /**
+     * Rest client builder. Helps creating a new {@link RestClient}.
+     */
+    public static final class Builder {
+        private static final int DEFAULT_MAX_RETRY_TIMEOUT = 10000;
+
+        private ConnectionPool connectionPool;
+        private CloseableHttpClient httpClient;
+        private int maxRetryTimeout = DEFAULT_MAX_RETRY_TIMEOUT;
+        private HttpHost[] hosts;
+
+        private Builder() {
+
+        }
+
+        /**
+         * Sets the connection pool. {@link StaticConnectionPool} will be used if not specified.
+         * @see ConnectionPool
+         */
+        public Builder setConnectionPool(ConnectionPool connectionPool) {
+            this.connectionPool = connectionPool;
+            return this;
+        }
+
+        /**
+         * Sets the http client. A new default one will be created if not specified, by calling {@link #createDefaultHttpClient()}.
+         * @see CloseableHttpClient
+         */
+        public Builder setHttpClient(CloseableHttpClient httpClient) {
+            this.httpClient = httpClient;
+            return this;
+        }
+
+        /**
+         * Sets the maximum timeout to honour in case of multiple retries of the same request.
+         * {@link #DEFAULT_MAX_RETRY_TIMEOUT} if not specified.
+         * @throws IllegalArgumentException if maxRetryTimeout is not greater than 0
+         */
+        public Builder setMaxRetryTimeout(int maxRetryTimeout) {
+            if (maxRetryTimeout <= 0) {
+                throw new IllegalArgumentException("maxRetryTimeout must be greater than 0");
+            }
+            this.maxRetryTimeout = maxRetryTimeout;
+            return this;
+        }
+
+        /**
+         * Sets the hosts that the client will send requests to. Mandatory if no connection pool is specified,
+         * as the provided hosts will be used to create the default static connection pool.
+         */
+        public Builder setHosts(HttpHost... hosts) {
+            if (hosts == null || hosts.length == 0) {
+                throw new IllegalArgumentException("no hosts provided");
+            }
+            this.hosts = hosts;
+            return this;
+        }
+
+        /**
+         * Creates a new {@link RestClient} based on the provided configuration.
+         */
+        public RestClient build() {
+            if (httpClient == null) {
+                httpClient = createDefaultHttpClient();
+            }
+            if (connectionPool == null) {
+                connectionPool = new StaticConnectionPool(hosts);
+            }
+            return new RestClient(httpClient, connectionPool, maxRetryTimeout);
+        }
+
+        /**
+         * Creates an http client with default settings
+         *
+         * @see CloseableHttpClient
+         */
+        public static CloseableHttpClient createDefaultHttpClient() {
+            PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
+            //default settings may be too constraining
+            connectionManager.setDefaultMaxPerRoute(10);
+            connectionManager.setMaxTotal(30);
+
+            //default timeouts are all infinite
+            RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(500).setSocketTimeout(10000)
+                    .setConnectionRequestTimeout(500).build();
+
+            return HttpClientBuilder.create().setConnectionManager(connectionManager).setDefaultRequestConfig(requestConfig).build();
+        }
+    }
 }
diff --git a/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
index 7aa4e5f5f35..09a72fedb62 100644
--- a/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
+++ b/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
@@ -40,7 +40,6 @@ import java.io.InputStream;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Objects;
 
 /**
  * Calls nodes info api and returns a list of http hosts extracted from it.
@@ -57,12 +56,6 @@ final class Sniffer {
     private final JsonFactory jsonFactory;
 
     Sniffer(CloseableHttpClient client, RequestConfig sniffRequestConfig, int sniffRequestTimeout, String scheme) {
-        Objects.requireNonNull(client, "client cannot be null");
-        Objects.requireNonNull(sniffRequestConfig, "sniffRequestConfig cannot be null");
-        if (sniffRequestTimeout <=0) {
-            throw new IllegalArgumentException("sniffRequestTimeout must be greater than 0");
-        }
-        Objects.requireNonNull(scheme, "scheme cannot be null");
         this.client = client;
         this.sniffRequestConfig = sniffRequestConfig;
         this.sniffRequestTimeout = sniffRequestTimeout;
diff --git a/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java b/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
index a2973638ad4..525645e9598 100644
--- a/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
@@ -24,8 +24,9 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.http.HttpHost;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.impl.client.CloseableHttpClient;
-import org.elasticsearch.client.ConnectionPool;
 import org.elasticsearch.client.Connection;
+import org.elasticsearch.client.ConnectionPool;
+import org.elasticsearch.client.RestClient;
 
 import java.io.IOException;
 import java.util.Iterator;
@@ -52,23 +53,8 @@ public class SniffingConnectionPool extends ConnectionPool {
     private volatile List<Connection> connections;
     private final SnifferTask snifferTask;
 
-    //TODO do we still need the sniff request timeout? or should we just use a low connect timeout?
-    public SniffingConnectionPool(int sniffInterval, boolean sniffOnFailure, int sniffAfterFailureDelay,
-                                  CloseableHttpClient client, RequestConfig sniffRequestConfig, int sniffRequestTimeout, String scheme,
-                                  HttpHost... hosts) {
-        if (sniffInterval <= 0) {
-            throw new IllegalArgumentException("sniffInterval must be greater than 0");
-        }
-        if (sniffAfterFailureDelay <= 0) {
-            throw new IllegalArgumentException("sniffAfterFailureDelay must be greater than 0");
-        }
-        Objects.requireNonNull(scheme, "scheme cannot be null");
-        if (scheme.equals("http") == false && scheme.equals("https") == false) {
-            throw new IllegalArgumentException("scheme must be either http or https");
-        }
-        if (hosts == null || hosts.length == 0) {
-            throw new IllegalArgumentException("no hosts provided");
-        }
+    private SniffingConnectionPool(int sniffInterval, boolean sniffOnFailure, int sniffAfterFailureDelay, CloseableHttpClient client,
+                                   RequestConfig sniffRequestConfig, int sniffRequestTimeout, String scheme, HttpHost... hosts) {
         this.sniffOnFailure = sniffOnFailure;
         this.sniffer = new Sniffer(client, sniffRequestConfig, sniffRequestTimeout, scheme);
         this.connections = createConnections(hosts);
@@ -188,4 +174,134 @@ public class SniffingConnectionPool extends ConnectionPool {
             scheduledExecutorService.shutdownNow();
         }
     }
+
+    /**
+     * Returns a new {@link Builder} to help with {@link SniffingConnectionPool} creation.
+     */
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    /**
+     * Sniffing connection pool builder. Helps creating a new {@link SniffingConnectionPool}.
+     */
+    public static final class Builder {
+        private int sniffInterval = 5 * 1000 * 60;
+        private boolean sniffOnFailure = true;
+        private int sniffAfterFailureDelay = 60000;
+        private CloseableHttpClient httpClient;
+        private RequestConfig sniffRequestConfig;
+        private int sniffRequestTimeout = 1000;
+        private String scheme = "http";
+        private HttpHost[] hosts;
+
+        private Builder() {
+
+        }
+
+        /**
+         * Sets the interval between consecutive ordinary sniff executions. Will be honoured when sniffOnFailure is disabled or
+         * when there are no failures between consecutive sniff executions.
+         * @throws IllegalArgumentException if sniffInterval is not greater than 0
+         */
+        public Builder setSniffInterval(int sniffInterval) {
+            if (sniffInterval <= 0) {
+                throw new IllegalArgumentException("sniffInterval must be greater than 0");
+            }
+            this.sniffInterval = sniffInterval;
+            return this;
+        }
+
+        /**
+         * Enables/disables sniffing on failure. If enabled, at each failure nodes will be reloaded, and a new sniff execution will
+         * be scheduled after a shorter time than usual (sniffAfterFailureDelay).
+         */
+        public Builder setSniffOnFailure(boolean sniffOnFailure) {
+            this.sniffOnFailure = sniffOnFailure;
+            return this;
+        }
+
+        /**
+         * Sets the delay of a sniff execution scheduled after a failure.
+         */
+        public Builder setSniffAfterFailureDelay(int sniffAfterFailureDelay) {
+            if (sniffAfterFailureDelay <= 0) {
+                throw new IllegalArgumentException("sniffAfterFailureDelay must be greater than 0");
+            }
+            this.sniffAfterFailureDelay = sniffAfterFailureDelay;
+            return this;
+        }
+
+        /**
+         * Sets the http client. Mandatory argument. Best practice is to use the same client used
+         * within {@link org.elasticsearch.client.RestClient} which can be created manually or
+         * through {@link RestClient.Builder#createDefaultHttpClient()}.
+         * @see CloseableHttpClient
+         */
+        public Builder setHttpClient(CloseableHttpClient httpClient) {
+            this.httpClient = httpClient;
+            return this;
+        }
+
+        /**
+         * Sets the configuration to be used for each sniff request. Useful as sniff can have
+         * different timeouts compared to ordinary requests.
+         * @see RequestConfig
+         */
+        public Builder setSniffRequestConfig(RequestConfig sniffRequestConfig) {
+            this.sniffRequestConfig = sniffRequestConfig;
+            return this;
+        }
+
+        /**
+         * Sets the sniff request timeout to be passed in as a query string parameter to elasticsearch.
+         * Allows to halt the request without any failure, as only the nodes that have responded
+         * within this timeout will be returned.
+         */
+        public Builder setSniffRequestTimeout(int sniffRequestTimeout) {
+            if (sniffRequestTimeout <=0) {
+                throw new IllegalArgumentException("sniffRequestTimeout must be greater than 0");
+            }
+            this.sniffRequestTimeout = sniffRequestTimeout;
+            return this;
+        }
+
+        /**
+         * Sets the scheme to be used for sniffed nodes. This information is not returned by elasticsearch,
+         * default is http but should be customized if https is needed/enabled.
+         */
+        public Builder setScheme(String scheme) {
+            Objects.requireNonNull(scheme, "scheme cannot be null");
+            if (scheme.equals("http") == false && scheme.equals("https") == false) {
+                throw new IllegalArgumentException("scheme must be either http or https");
+            }
+            this.scheme = scheme;
+            return this;
+        }
+
+        /**
+         * Sets the hosts that the client will send requests to.
+         */
+        public Builder setHosts(HttpHost... hosts) {
+            this.hosts = hosts;
+            return this;
+        }
+
+        /**
+         * Creates the {@link SniffingConnectionPool} based on the provided configuration.
+         */
+        public SniffingConnectionPool build() {
+            Objects.requireNonNull(httpClient, "httpClient cannot be null");
+            if (hosts == null || hosts.length == 0) {
+                throw new IllegalArgumentException("no hosts provided");
+            }
+
+            if (sniffRequestConfig == null) {
+                sniffRequestConfig = RequestConfig.custom().setConnectTimeout(500).setSocketTimeout(1000)
+                        .setConnectionRequestTimeout(500).build();
+            }
+            return new SniffingConnectionPool(sniffInterval, sniffOnFailure, sniffAfterFailureDelay, httpClient, sniffRequestConfig,
+                    sniffRequestTimeout, scheme, hosts);
+        }
+    }
 }
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java b/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
new file mode 100644
index 00000000000..11008d52b19
--- /dev/null
+++ b/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import org.apache.http.HttpHost;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.lucene.util.LuceneTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.logging.LogManager;
+
+public class RestClientBuilderTests extends LuceneTestCase {
+
+    static {
+        LogManager.getLogManager().reset();
+    }
+
+    public void testBuild() throws IOException {
+        try {
+            RestClient.builder().setMaxRetryTimeout(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+            fail("should have failed");
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "maxRetryTimeout must be greater than 0");
+        }
+
+        try {
+            RestClient.builder().setHosts((HttpHost[])null);
+            fail("should have failed");
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "no hosts provided");
+        }
+
+        try {
+            RestClient.builder().setHosts();
+            fail("should have failed");
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "no hosts provided");
+        }
+
+        RestClient.Builder builder = RestClient.builder();
+        if (random().nextBoolean()) {
+            ConnectionPool connectionPool = new ConnectionPool() {
+                @Override
+                protected List<Connection> getConnections() {
+                    return Collections.emptyList();
+                }
+
+                @Override
+                public void onSuccess(Connection connection) {
+
+                }
+
+                @Override
+                public void onFailure(Connection connection) throws IOException {
+
+                }
+
+                @Override
+                public void close() throws IOException {
+
+                }
+            };
+            builder.setConnectionPool(connectionPool);
+        } else {
+            int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
+            HttpHost[] hosts = new HttpHost[numNodes];
+            for (int i = 0; i < numNodes; i++) {
+                hosts[i] = new HttpHost("localhost", 9200 + i);
+            }
+            builder.setHosts(hosts);
+        }
+        if (random().nextBoolean()) {
+            builder.setHttpClient(HttpClientBuilder.create().build());
+        }
+        if (random().nextBoolean()) {
+            builder.setMaxRetryTimeout(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+        }
+        try (RestClient restClient = builder.build()) {
+            assertNotNull(restClient);
+        }
+    }
+}
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientTests.java b/client/src/test/java/org/elasticsearch/client/RestClientTests.java
deleted file mode 100644
index 0d9930d9c9f..00000000000
--- a/client/src/test/java/org/elasticsearch/client/RestClientTests.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.client;
-
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.lucene.util.LuceneTestCase;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.logging.LogManager;
-
-public class RestClientTests extends LuceneTestCase {
-
-    static {
-        LogManager.getLogManager().reset();
-    }
-
-    public void testConstructor() throws IOException {
-        CloseableHttpClient httpClient = HttpClientBuilder.create().build();
-        ConnectionPool connectionPool = new ConnectionPool() {
-            @Override
-            protected List<Connection> getConnections() {
-                return Collections.emptyList();
-            }
-
-            @Override
-            public void onSuccess(Connection connection) {
-
-            }
-
-            @Override
-            public void onFailure(Connection connection) throws IOException {
-
-            }
-
-            @Override
-            public void close() throws IOException {
-
-            }
-        };
-
-        try {
-            new RestClient(null, connectionPool, RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
-            fail("transport creation should have failed");
-        } catch(NullPointerException e) {
-            assertEquals(e.getMessage(), "client cannot be null");
-        }
-
-        try {
-            new RestClient(httpClient, null, RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
-            fail("transport creation should have failed");
-        } catch(NullPointerException e) {
-            assertEquals(e.getMessage(), "connectionPool cannot be null");
-        }
-
-        try {
-            new RestClient(httpClient, connectionPool, RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
-            fail("transport creation should have failed");
-        } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "maxRetryTimeout must be greater than 0");
-        }
-
-        try(RestClient client = new RestClient(httpClient, connectionPool, RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE))) {
-            assertNotNull(client);
-        }
-    }
-}
diff --git a/client/src/test/java/org/elasticsearch/client/sniff/SniffingConnectionPoolBuilderTests.java b/client/src/test/java/org/elasticsearch/client/sniff/SniffingConnectionPoolBuilderTests.java
new file mode 100644
index 00000000000..b5b4eaa3ce1
--- /dev/null
+++ b/client/src/test/java/org/elasticsearch/client/sniff/SniffingConnectionPoolBuilderTests.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.sniff;
+
+import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import org.apache.http.HttpHost;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.lucene.util.LuceneTestCase;
+
+import java.util.Arrays;
+import java.util.logging.LogManager;
+
+public class SniffingConnectionPoolBuilderTests extends LuceneTestCase {
+
+    static {
+        LogManager.getLogManager().reset();
+    }
+
+    public void testBuild() throws Exception {
+
+        try {
+            SniffingConnectionPool.builder().setScheme(null);
+            fail("should have failed");
+        } catch(NullPointerException e) {
+            assertEquals(e.getMessage(), "scheme cannot be null");
+        }
+
+        try {
+            SniffingConnectionPool.builder().setScheme("whatever");
+            fail("should have failed");
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "scheme must be either http or https");
+        }
+
+        try {
+            SniffingConnectionPool.builder().setSniffInterval(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+            fail("should have failed");
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "sniffInterval must be greater than 0");
+        }
+
+        try {
+            SniffingConnectionPool.builder().setSniffRequestTimeout(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+            fail("should have failed");
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "sniffRequestTimeout must be greater than 0");
+        }
+
+        try {
+            SniffingConnectionPool.builder().setSniffAfterFailureDelay(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+            fail("should have failed");
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "sniffAfterFailureDelay must be greater than 0");
+        }
+
+        try {
+            SniffingConnectionPool.builder().build();
+            fail("should have failed");
+        } catch(NullPointerException e) {
+            assertEquals(e.getMessage(), "httpClient cannot be null");
+        }
+
+        try {
+            SniffingConnectionPool.builder().setHttpClient(HttpClientBuilder.create().build()).build();
+            fail("should have failed");
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "no hosts provided");
+        }
+
+        try {
+            SniffingConnectionPool.builder().setHttpClient(HttpClientBuilder.create().build()).setHosts((HttpHost[])null).build();
+            fail("should have failed");
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "no hosts provided");
+        }
+
+        try {
+            SniffingConnectionPool.builder().setHttpClient(HttpClientBuilder.create().build()).setHosts().build();
+            fail("should have failed");
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "no hosts provided");
+        }
+
+        int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
+        HttpHost[] hosts = new HttpHost[numNodes];
+        for (int i = 0; i < numNodes; i++) {
+            hosts[i] = new HttpHost("localhost", 9200 + i);
+        }
+
+        try (SniffingConnectionPool connectionPool = SniffingConnectionPool.builder()
+                .setHttpClient(HttpClientBuilder.create().build()).setHosts(hosts).build()) {
+            assertNotNull(connectionPool);
+        }
+
+        SniffingConnectionPool.Builder builder = SniffingConnectionPool.builder()
+                .setHttpClient(HttpClientBuilder.create().build()).setHosts(hosts);
+        if (random().nextBoolean()) {
+            builder.setScheme(RandomPicks.randomFrom(random(), Arrays.asList("http", "https")));
+        }
+        if (random().nextBoolean()) {
+            builder.setSniffInterval(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+        }
+        if (random().nextBoolean()) {
+            builder.setSniffAfterFailureDelay(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+        }
+        if (random().nextBoolean()) {
+            builder.setSniffRequestTimeout(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+        }
+        if (random().nextBoolean()) {
+            builder.setSniffOnFailure(random().nextBoolean());
+        }
+        if (random().nextBoolean()) {
+            builder.setSniffRequestConfig(RequestConfig.DEFAULT);
+        }
+        try (SniffingConnectionPool connectionPool = builder.build()) {
+            assertNotNull(connectionPool);
+        }
+    }
+}
diff --git a/client/src/test/java/org/elasticsearch/client/sniff/SniffingConnectionPoolTests.java b/client/src/test/java/org/elasticsearch/client/sniff/SniffingConnectionPoolTests.java
deleted file mode 100644
index 4e54f8074db..00000000000
--- a/client/src/test/java/org/elasticsearch/client/sniff/SniffingConnectionPoolTests.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.client.sniff;
-
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
-import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-import org.apache.http.HttpHost;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.lucene.util.LuceneTestCase;
-
-import java.util.logging.LogManager;
-
-public class SniffingConnectionPoolTests extends LuceneTestCase {
-
-    static {
-        LogManager.getLogManager().reset();
-    }
-
-    public void testConstructor() throws Exception {
-        CloseableHttpClient httpClient = HttpClientBuilder.create().build();
-        String[] schemes = new String[]{"http", "https"};
-        int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
-        HttpHost[] hosts = new HttpHost[numNodes];
-        for (int i = 0; i < numNodes; i++) {
-            hosts[i] = new HttpHost("localhost", 9200);
-        }
-
-        try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
-                RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0), random().nextBoolean(),
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), schemes), hosts)) {
-
-            fail("pool creation should have failed " + connectionPool);
-        } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "sniffInterval must be greater than 0");
-        }
-
-        try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
-                RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0), httpClient, RequestConfig.DEFAULT,
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), schemes), hosts)) {
-            fail("pool creation should have failed " + connectionPool);
-        } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "sniffAfterFailureDelay must be greater than 0");
-        }
-
-        try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), null, RequestConfig.DEFAULT,
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), schemes), hosts)) {
-            fail("pool creation should have failed " + connectionPool);
-        } catch(NullPointerException e) {
-            assertEquals(e.getMessage(), "client cannot be null");
-        }
-
-        try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, null,
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), schemes), hosts)) {
-            fail("pool creation should have failed " + connectionPool);
-        } catch(NullPointerException e) {
-            assertEquals(e.getMessage(), "sniffRequestConfig cannot be null");
-        }
-
-        try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
-                RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0),
-                RandomPicks.randomFrom(random(), schemes), hosts)) {
-            fail("pool creation should have failed " + connectionPool);
-        } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "sniffRequestTimeout must be greater than 0");
-        }
-
-        try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
-                RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0),
-                null, hosts)) {
-            fail("pool creation should have failed " + connectionPool);
-        } catch(NullPointerException e) {
-            assertEquals(e.getMessage(), "scheme cannot be null");
-        }
-
-        try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
-                RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0),
-                "whatever", hosts)) {
-            fail("pool creation should have failed " + connectionPool);
-        } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "scheme must be either http or https");
-        }
-
-        try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), schemes), (HttpHost[])null)) {
-            fail("pool creation should have failed " + connectionPool);
-        } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "no hosts provided");
-        }
-
-        try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), schemes), (HttpHost) null)) {
-            fail("pool creation should have failed " + connectionPool);
-        } catch(NullPointerException e) {
-            assertEquals(e.getMessage(), "host cannot be null");
-        }
-
-        try (SniffingConnectionPool connectionPool = new SniffingConnectionPool(
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), schemes))) {
-            fail("pool creation should have failed " + connectionPool);
-        } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "no hosts provided");
-        }
-
-        try (SniffingConnectionPool sniffingConnectionPool = new SniffingConnectionPool(
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), random().nextBoolean(),
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE), httpClient, RequestConfig.DEFAULT,
-                RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE),
-                RandomPicks.randomFrom(random(), schemes), hosts)) {
-            assertNotNull(sniffingConnectionPool);
-        }
-    }
-}

From 599dad560c3297c5bc24b9bf33284ba7f0a9bd5e Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Mon, 9 May 2016 19:45:14 +0200
Subject: [PATCH 0598/1311] remove streams and java 8 only api, build with
 source and target 1.7

---
 client/build.gradle                           | 11 +++++--
 .../elasticsearch/client/ConnectionPool.java  | 30 ++++++++++++++-----
 .../org/elasticsearch/client/RestClient.java  |  6 ++--
 .../client/sniff/SniffingConnectionPool.java  | 24 +++++++--------
 .../client/sniff/SnifferTests.java            |  4 +--
 5 files changed, 48 insertions(+), 27 deletions(-)

diff --git a/client/build.gradle b/client/build.gradle
index e19e860c356..35af3d0979b 100644
--- a/client/build.gradle
+++ b/client/build.gradle
@@ -17,11 +17,15 @@
  * under the License.
  */
 
-import org.elasticsearch.gradle.precommit.PrecommitTasks;
+import org.elasticsearch.gradle.precommit.PrecommitTasks
+import org.gradle.api.JavaVersion
 
 group = 'org.elasticsearch.client'
 apply plugin: 'elasticsearch.build'
 
+targetCompatibility = JavaVersion.VERSION_1_7
+sourceCompatibility = JavaVersion.VERSION_1_7
+
 dependencies {
   // TODO once we got rid of the client in the test framework we should use a version variable here
   compile "org.apache.httpcomponents:httpclient:4.5.2"
@@ -47,8 +51,9 @@ dependencies {
   testCompile "org.bouncycastle:bcprov-jdk15on:1.54"
 }
 
-compileJava.options.compilerArgs << '-Xlint:-cast,-rawtypes,-try,-unchecked'
-compileTestJava.options.compilerArgs << '-Xlint:-rawtypes'
+//TODO compiling from 1.8 with target 1.7 and source 1.7 is best effort, not enough to ensure we are java 7 compatible
+compileJava.options.compilerArgs << '-target' << '1.7' << '-source' << '1.7' << '-Xlint:all,-path,-serial,-options'
+compileTestJava.options.compilerArgs << '-target' << '1.7' << '-source' << '1.7'
 
 forbiddenApisMain {
   //client does not depend on core, so only jdk signatures should be checked
diff --git a/client/src/main/java/org/elasticsearch/client/ConnectionPool.java b/client/src/main/java/org/elasticsearch/client/ConnectionPool.java
index b0be3a27075..89b714f2eee 100644
--- a/client/src/main/java/org/elasticsearch/client/ConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/ConnectionPool.java
@@ -27,10 +27,11 @@ import java.io.Closeable;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Stream;
 
 /**
  * Pool of connections to the different hosts that belong to an elasticsearch cluster.
@@ -66,7 +67,7 @@ public abstract class ConnectionPool implements Closeable {
      * It may happen that the stream is empty, in which case it means that there aren't healthy connections to use.
      * Then {@link #lastResortConnection()} should be called to retrieve a non healthy connection and try it.
      */
-    public final Stream<Connection> nextConnection() {
+    public final Iterator<Connection> nextConnection() {
         List<Connection> connections = getConnections();
         if (connections.isEmpty()) {
             throw new IllegalStateException("no connections available in the connection pool");
@@ -75,7 +76,14 @@ public abstract class ConnectionPool implements Closeable {
         List<Connection> sortedConnections = new ArrayList<>(connections);
         //TODO is it possible to make this O(1)? (rotate is O(n))
         Collections.rotate(sortedConnections, sortedConnections.size() - lastConnectionIndex.getAndIncrement());
-        return sortedConnections.stream().filter(connection -> connection.isAlive() || connection.shouldBeRetried());
+        Iterator<Connection> connectionIterator = sortedConnections.iterator();
+        while (connectionIterator.hasNext()) {
+            Connection connection = connectionIterator.next();
+            if (connection.isAlive() == false && connection.shouldBeRetried() == false) {
+                connectionIterator.remove();
+            }
+        }
+        return connectionIterator;
     }
 
     /**
@@ -96,10 +104,18 @@ public abstract class ConnectionPool implements Closeable {
      * only in case {@link #nextConnection()} returns an empty stream.
      */
     public final Connection lastResortConnection() {
-        Connection Connection = getConnections().stream()
-                .sorted((o1, o2) -> Long.compare(o1.getDeadUntil(), o2.getDeadUntil())).findFirst().get();
-        Connection.markResurrected();
-        return Connection;
+        List<Connection> connections = getConnections();
+        if (connections.isEmpty()) {
+            throw new IllegalStateException("no connections available in the connection pool");
+        }
+        List<Connection> sortedConnections = new ArrayList<>(connections);
+        Collections.sort(sortedConnections, new Comparator<Connection>() {
+            @Override
+            public int compare(Connection o1, Connection o2) {
+                return Long.compare(o1.getDeadUntil(), o2.getDeadUntil());
+            }
+        });
+        return sortedConnections.get(0);
     }
 
     /**
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 7fdc8e57e3a..f13be9e4e49 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -39,11 +39,11 @@ import java.io.Closeable;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Stream;
 
 public final class RestClient implements Closeable {
 
@@ -63,11 +63,11 @@ public final class RestClient implements Closeable {
             throws IOException {
         URI uri = buildUri(endpoint, params);
         HttpRequestBase request = createHttpRequest(method, uri, entity);
-        Iterator<Connection> connectionIterator = connectionPool.nextConnection().iterator();
+        Iterator<Connection> connectionIterator = connectionPool.nextConnection();
         if (connectionIterator.hasNext() == false) {
             Connection connection = connectionPool.lastResortConnection();
             logger.info("no healthy nodes available, trying " + connection.getHost());
-            return performRequest(request, Stream.of(connection).iterator());
+            return performRequest(request, Collections.singleton(connection).iterator());
         }
         return performRequest(request, connectionIterator);
     }
diff --git a/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java b/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
index 525645e9598..6bc7ad84e36 100644
--- a/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
@@ -29,6 +29,7 @@ import org.elasticsearch.client.ConnectionPool;
 import org.elasticsearch.client.RestClient;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
@@ -37,8 +38,6 @@ import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.Predicate;
-import java.util.stream.Stream;
 
 /**
  * Connection pool implementation that sniffs nodes from elasticsearch at regular intervals.
@@ -97,25 +96,25 @@ public class SniffingConnectionPool extends ConnectionPool {
 
         @Override
         public void run() {
-            sniff(node -> true);
+            sniff(null);
         }
 
         void sniffOnFailure(HttpHost failedHost) {
             //sync sniff straightaway on failure
             failure = true;
-            sniff(host -> host.equals(failedHost) == false);
+            sniff(failedHost);
         }
 
-        void sniff(Predicate<HttpHost> hostFilter) {
+        void sniff(HttpHost excludeHost) {
             if (running.compareAndSet(false, true)) {
                 try {
-                    Iterator<Connection> connectionIterator = nextConnection().iterator();
+                    Iterator<Connection> connectionIterator = nextConnection();
                     if (connectionIterator.hasNext()) {
-                        sniff(connectionIterator, hostFilter);
+                        sniff(connectionIterator, excludeHost);
                     } else {
                         Connection connection = lastResortConnection();
                         logger.info("no healthy nodes available, trying " + connection.getHost());
-                        sniff(Stream.of(connection).iterator(), hostFilter);
+                        sniff(Collections.singleton(connection).iterator(), excludeHost);
                     }
                 } catch (Throwable t) {
                     logger.error("error while sniffing nodes", t);
@@ -139,15 +138,16 @@ public class SniffingConnectionPool extends ConnectionPool {
             }
         }
 
-        void sniff(Iterator<Connection> connectionIterator, Predicate<HttpHost> hostFilter) throws IOException {
+        void sniff(Iterator<Connection> connectionIterator, HttpHost excludeHost) throws IOException {
             IOException lastSeenException = null;
             while (connectionIterator.hasNext()) {
                 Connection connection = connectionIterator.next();
                 try {
                     List<HttpHost> sniffedNodes = sniffer.sniffNodes(connection.getHost());
-                    HttpHost[] filteredNodes = sniffedNodes.stream().filter(hostFilter).toArray(HttpHost[]::new);
-                    logger.debug("adding " + filteredNodes.length + " nodes out of " + sniffedNodes.size() + " sniffed nodes");
-                    connections = createConnections(filteredNodes);
+                    if (excludeHost != null) {
+                        sniffedNodes.remove(excludeHost);
+                    }
+                    connections = createConnections(sniffedNodes.toArray(new HttpHost[sniffedNodes.size()]));
                     onSuccess(connection);
                     return;
                 } catch (IOException e) {
diff --git a/client/src/test/java/org/elasticsearch/client/sniff/SnifferTests.java b/client/src/test/java/org/elasticsearch/client/sniff/SnifferTests.java
index 6cb7905151f..2fa9f7b3225 100644
--- a/client/src/test/java/org/elasticsearch/client/sniff/SnifferTests.java
+++ b/client/src/test/java/org/elasticsearch/client/sniff/SnifferTests.java
@@ -115,7 +115,7 @@ public class SnifferTests extends LuceneTestCase {
         }
     }
 
-    private static MockWebServer buildMockWebServer(SniffResponse sniffResponse, int sniffTimeout) throws UnsupportedEncodingException {
+    private static MockWebServer buildMockWebServer(final SniffResponse sniffResponse, final int sniffTimeout) throws UnsupportedEncodingException {
         MockWebServer server = new MockWebServer();
         final Dispatcher dispatcher = new Dispatcher() {
             @Override
@@ -244,7 +244,7 @@ public class SnifferTests extends LuceneTestCase {
         }
 
         static SniffResponse buildFailure() {
-            return new SniffResponse("", Collections.emptyList(), true);
+            return new SniffResponse("", Collections.<HttpHost>emptyList(), true);
         }
 
         static SniffResponse buildResponse(String nodesInfoBody, List<HttpHost> hosts) {

From b38ef345e2e8eac35ae2727008f15ed26210ba31 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Tue, 10 May 2016 10:50:53 +0200
Subject: [PATCH 0599/1311] remove streams leftover

Given that we don't use streams anymore, we can check straightaway if the connection iterator is empty before returning it and resurrect a connection when needed directly in the connection pool, no lastResortConnection method required.
---
 .../elasticsearch/client/ConnectionPool.java  | 59 ++++++++-----------
 .../org/elasticsearch/client/RestClient.java  |  9 +--
 .../client/sniff/SniffingConnectionPool.java  | 10 +---
 3 files changed, 28 insertions(+), 50 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/ConnectionPool.java b/client/src/main/java/org/elasticsearch/client/ConnectionPool.java
index 89b714f2eee..55e9554684e 100644
--- a/client/src/main/java/org/elasticsearch/client/ConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/ConnectionPool.java
@@ -35,11 +35,9 @@ import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  * Pool of connections to the different hosts that belong to an elasticsearch cluster.
- * It keeps track of the different hosts to communicate with and allows to retrieve a stream of connections to be used
+ * It keeps track of the different hosts to communicate with and allows to retrieve an iterator of connections to be used
  * for each request. Marks connections as dead/alive when needed.
- * Provides a stream of alive connections or dead ones that should be retried for each {@link #nextConnection()} call.
- * In case the returned stream is empty a last resort dead connection should be retrieved by calling {@link #lastResortConnection()}
- * and resurrected so that a last resort request attempt can be performed.
+ * Provides an iterator of connections to be used at each {@link #nextConnection()} call.
  * The {@link #onSuccess(Connection)} method marks the connection provided as an argument alive.
  * The {@link #onFailure(Connection)} method marks the connection provided as an argument dead.
  * This base implementation doesn't define the list implementation that stores connections, so that concurrency can be
@@ -60,12 +58,13 @@ public abstract class ConnectionPool implements Closeable {
     protected abstract List<Connection> getConnections();
 
     /**
-     * Returns a stream of connections that should be used for a request call.
-     * Ideally, the first connection is retrieved from the stream and used successfully for the request.
-     * Otherwise, after each failure the next connection should be retrieved from the stream so that the request can be retried.
-     * The maximum total of attempts is equal to the number of connections that are available in the stream.
-     * It may happen that the stream is empty, in which case it means that there aren't healthy connections to use.
-     * Then {@link #lastResortConnection()} should be called to retrieve a non healthy connection and try it.
+     * Returns an iterator of connections that should be used for a request call.
+     * Ideally, the first connection is retrieved from the iterator and used successfully for the request.
+     * Otherwise, after each failure the next connection should be retrieved from the iterator so that the request can be retried.
+     * The maximum total of attempts is equal to the number of connections that are available in the iterator.
+     * The iterator returned will never be empty, rather an {@link IllegalStateException} will be thrown in that case.
+     * In case there are no alive connections available, or dead ones that should be retried, one dead connection
+     * gets resurrected and returned.
      */
     public final Iterator<Connection> nextConnection() {
         List<Connection> connections = getConnections();
@@ -73,17 +72,30 @@ public abstract class ConnectionPool implements Closeable {
             throw new IllegalStateException("no connections available in the connection pool");
         }
 
-        List<Connection> sortedConnections = new ArrayList<>(connections);
+        List<Connection> rotatedConnections = new ArrayList<>(connections);
         //TODO is it possible to make this O(1)? (rotate is O(n))
-        Collections.rotate(sortedConnections, sortedConnections.size() - lastConnectionIndex.getAndIncrement());
-        Iterator<Connection> connectionIterator = sortedConnections.iterator();
+        Collections.rotate(rotatedConnections, rotatedConnections.size() - lastConnectionIndex.getAndIncrement());
+        Iterator<Connection> connectionIterator = rotatedConnections.iterator();
         while (connectionIterator.hasNext()) {
             Connection connection = connectionIterator.next();
             if (connection.isAlive() == false && connection.shouldBeRetried() == false) {
                 connectionIterator.remove();
             }
         }
-        return connectionIterator;
+        if (rotatedConnections.isEmpty()) {
+            List<Connection> sortedConnections = new ArrayList<>(connections);
+            Collections.sort(sortedConnections, new Comparator<Connection>() {
+                @Override
+                public int compare(Connection o1, Connection o2) {
+                    return Long.compare(o1.getDeadUntil(), o2.getDeadUntil());
+                }
+            });
+            Connection connection = sortedConnections.get(0);
+            connection.markResurrected();
+            logger.trace("marked connection resurrected for " + connection.getHost());
+            return Collections.singleton(connection).iterator();
+        }
+        return rotatedConnections.iterator();
     }
 
     /**
@@ -99,25 +111,6 @@ public abstract class ConnectionPool implements Closeable {
         return Collections.unmodifiableList(connections);
     }
 
-    /**
-     * Returns a connection that is not necessarily healthy, but can be used for a request attempt. To be called as last resort
-     * only in case {@link #nextConnection()} returns an empty stream.
-     */
-    public final Connection lastResortConnection() {
-        List<Connection> connections = getConnections();
-        if (connections.isEmpty()) {
-            throw new IllegalStateException("no connections available in the connection pool");
-        }
-        List<Connection> sortedConnections = new ArrayList<>(connections);
-        Collections.sort(sortedConnections, new Comparator<Connection>() {
-            @Override
-            public int compare(Connection o1, Connection o2) {
-                return Long.compare(o1.getDeadUntil(), o2.getDeadUntil());
-            }
-        });
-        return sortedConnections.get(0);
-    }
-
     /**
      * Called after each successful request call.
      * Receives as an argument the connection that was used for the successful request.
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index f13be9e4e49..c4d6ede623e 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -39,7 +39,6 @@ import java.io.Closeable;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.util.Collections;
 import java.util.Iterator;
 import java.util.Locale;
 import java.util.Map;
@@ -63,13 +62,7 @@ public final class RestClient implements Closeable {
             throws IOException {
         URI uri = buildUri(endpoint, params);
         HttpRequestBase request = createHttpRequest(method, uri, entity);
-        Iterator<Connection> connectionIterator = connectionPool.nextConnection();
-        if (connectionIterator.hasNext() == false) {
-            Connection connection = connectionPool.lastResortConnection();
-            logger.info("no healthy nodes available, trying " + connection.getHost());
-            return performRequest(request, Collections.singleton(connection).iterator());
-        }
-        return performRequest(request, connectionIterator);
+        return performRequest(request, connectionPool.nextConnection());
     }
 
     private ElasticsearchResponse performRequest(HttpRequestBase request, Iterator<Connection> connectionIterator) throws IOException {
diff --git a/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java b/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
index 6bc7ad84e36..6fd98666e8c 100644
--- a/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
+++ b/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
@@ -29,7 +29,6 @@ import org.elasticsearch.client.ConnectionPool;
 import org.elasticsearch.client.RestClient;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
@@ -108,14 +107,7 @@ public class SniffingConnectionPool extends ConnectionPool {
         void sniff(HttpHost excludeHost) {
             if (running.compareAndSet(false, true)) {
                 try {
-                    Iterator<Connection> connectionIterator = nextConnection();
-                    if (connectionIterator.hasNext()) {
-                        sniff(connectionIterator, excludeHost);
-                    } else {
-                        Connection connection = lastResortConnection();
-                        logger.info("no healthy nodes available, trying " + connection.getHost());
-                        sniff(Collections.singleton(connection).iterator(), excludeHost);
-                    }
+                    sniff(nextConnection(), excludeHost);
                 } catch (Throwable t) {
                     logger.error("error while sniffing nodes", t);
                 } finally {

From cdffc3d15b10eb5b40f6a56d5d01f92982dc6fc4 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 13 May 2016 11:55:18 +0200
Subject: [PATCH 0600/1311] remove notion of connection pool, turn around
 dependency between RestClient and Sniffer

RestClient exposes a setNodes method, which Sniffer can call to update its nodes.
---
 .../org/elasticsearch/client/Connection.java  |   1 -
 .../elasticsearch/client/ConnectionPool.java  | 131 --------
 .../org/elasticsearch/client/RestClient.java  | 151 ++++++---
 .../client/StaticConnectionPool.java          |  50 ---
 .../client/sniff/HostsSniffer.java            | 126 ++++++++
 .../elasticsearch/client/sniff/Sniffer.java   | 288 +++++++++++------
 .../client/sniff/SniffingConnectionPool.java  | 299 ------------------
 .../client/RestClientBuilderTests.java        |  49 +--
 .../client/StaticConnectionPoolTests.java     |  62 ----
 ...ifferTests.java => HostsSnifferTests.java} |  54 ++--
 .../client/sniff/SnifferBuilderTests.java     | 115 +++++++
 .../SniffingConnectionPoolBuilderTests.java   | 138 --------
 12 files changed, 591 insertions(+), 873 deletions(-)
 delete mode 100644 client/src/main/java/org/elasticsearch/client/ConnectionPool.java
 delete mode 100644 client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
 create mode 100644 client/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
 delete mode 100644 client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
 delete mode 100644 client/src/test/java/org/elasticsearch/client/StaticConnectionPoolTests.java
 rename client/src/test/java/org/elasticsearch/client/sniff/{SnifferTests.java => HostsSnifferTests.java} (84%)
 create mode 100644 client/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
 delete mode 100644 client/src/test/java/org/elasticsearch/client/sniff/SniffingConnectionPoolBuilderTests.java

diff --git a/client/src/main/java/org/elasticsearch/client/Connection.java b/client/src/main/java/org/elasticsearch/client/Connection.java
index 3d48a4eeae1..f17ff69e14c 100644
--- a/client/src/main/java/org/elasticsearch/client/Connection.java
+++ b/client/src/main/java/org/elasticsearch/client/Connection.java
@@ -29,7 +29,6 @@ import java.util.concurrent.TimeUnit;
  * Any change to the state of a connection should be made through the connection pool.
  */
 public class Connection {
-    //TODO make these values configurable through the connection pool?
     private static final long DEFAULT_CONNECTION_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(1);
     private static final long MAX_CONNECTION_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(30);
     private final HttpHost host;
diff --git a/client/src/main/java/org/elasticsearch/client/ConnectionPool.java b/client/src/main/java/org/elasticsearch/client/ConnectionPool.java
deleted file mode 100644
index 55e9554684e..00000000000
--- a/client/src/main/java/org/elasticsearch/client/ConnectionPool.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.client;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.http.HttpHost;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Objects;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * Pool of connections to the different hosts that belong to an elasticsearch cluster.
- * It keeps track of the different hosts to communicate with and allows to retrieve an iterator of connections to be used
- * for each request. Marks connections as dead/alive when needed.
- * Provides an iterator of connections to be used at each {@link #nextConnection()} call.
- * The {@link #onSuccess(Connection)} method marks the connection provided as an argument alive.
- * The {@link #onFailure(Connection)} method marks the connection provided as an argument dead.
- * This base implementation doesn't define the list implementation that stores connections, so that concurrency can be
- * handled in subclasses depending on the usecase (e.g. defining the list volatile or final when needed).
- */
-public abstract class ConnectionPool implements Closeable {
-
-    private static final Log logger = LogFactory.getLog(ConnectionPool.class);
-
-    private final AtomicInteger lastConnectionIndex = new AtomicInteger(0);
-
-    /**
-     * Allows to retrieve the concrete list of connections. Not defined directly as a member
-     * of this class as subclasses may need to handle concurrency if the list can change, for
-     * instance defining the field as volatile. On the other hand static implementations
-     * can just make the list final instead.
-     */
-    protected abstract List<Connection> getConnections();
-
-    /**
-     * Returns an iterator of connections that should be used for a request call.
-     * Ideally, the first connection is retrieved from the iterator and used successfully for the request.
-     * Otherwise, after each failure the next connection should be retrieved from the iterator so that the request can be retried.
-     * The maximum total of attempts is equal to the number of connections that are available in the iterator.
-     * The iterator returned will never be empty, rather an {@link IllegalStateException} will be thrown in that case.
-     * In case there are no alive connections available, or dead ones that should be retried, one dead connection
-     * gets resurrected and returned.
-     */
-    public final Iterator<Connection> nextConnection() {
-        List<Connection> connections = getConnections();
-        if (connections.isEmpty()) {
-            throw new IllegalStateException("no connections available in the connection pool");
-        }
-
-        List<Connection> rotatedConnections = new ArrayList<>(connections);
-        //TODO is it possible to make this O(1)? (rotate is O(n))
-        Collections.rotate(rotatedConnections, rotatedConnections.size() - lastConnectionIndex.getAndIncrement());
-        Iterator<Connection> connectionIterator = rotatedConnections.iterator();
-        while (connectionIterator.hasNext()) {
-            Connection connection = connectionIterator.next();
-            if (connection.isAlive() == false && connection.shouldBeRetried() == false) {
-                connectionIterator.remove();
-            }
-        }
-        if (rotatedConnections.isEmpty()) {
-            List<Connection> sortedConnections = new ArrayList<>(connections);
-            Collections.sort(sortedConnections, new Comparator<Connection>() {
-                @Override
-                public int compare(Connection o1, Connection o2) {
-                    return Long.compare(o1.getDeadUntil(), o2.getDeadUntil());
-                }
-            });
-            Connection connection = sortedConnections.get(0);
-            connection.markResurrected();
-            logger.trace("marked connection resurrected for " + connection.getHost());
-            return Collections.singleton(connection).iterator();
-        }
-        return rotatedConnections.iterator();
-    }
-
-    /**
-     * Helper method to be used by subclasses when needing to create a new list
-     * of connections given their corresponding hosts
-     */
-    protected final List<Connection> createConnections(HttpHost... hosts) {
-        List<Connection> connections = new ArrayList<>();
-        for (HttpHost host : hosts) {
-            Objects.requireNonNull(host, "host cannot be null");
-            connections.add(new Connection(host));
-        }
-        return Collections.unmodifiableList(connections);
-    }
-
-    /**
-     * Called after each successful request call.
-     * Receives as an argument the connection that was used for the successful request.
-     */
-    public void onSuccess(Connection connection) {
-        connection.markAlive();
-        logger.trace("marked connection alive for " + connection.getHost());
-    }
-
-    /**
-     * Called after each failed attempt.
-     * Receives as an argument the connection that was used for the failed attempt.
-     */
-    public void onFailure(Connection connection) throws IOException {
-        connection.markDead();
-        logger.debug("marked connection dead for " + connection.getHost());
-    }
-}
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index c4d6ede623e..f578d7ce304 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -39,38 +39,51 @@ import java.io.Closeable;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 
 public final class RestClient implements Closeable {
 
     private static final Log logger = LogFactory.getLog(RestClient.class);
 
     private final CloseableHttpClient client;
-    private final ConnectionPool connectionPool;
     private final long maxRetryTimeout;
+    private final AtomicInteger lastConnectionIndex = new AtomicInteger(0);
+    private volatile List<Connection> connections;
+    private volatile FailureListener failureListener = new FailureListener();
 
-    private RestClient(CloseableHttpClient client, ConnectionPool connectionPool, long maxRetryTimeout) {
+    private RestClient(CloseableHttpClient client, long maxRetryTimeout, HttpHost... hosts) {
         this.client = client;
-        this.connectionPool = connectionPool;
         this.maxRetryTimeout = maxRetryTimeout;
+        setNodes(hosts);
+    }
+
+    public synchronized void setNodes(HttpHost... hosts) {
+        List<Connection> connections = new ArrayList<>(hosts.length);
+        for (HttpHost host : hosts) {
+            Objects.requireNonNull(host, "host cannot be null");
+            connections.add(new Connection(host));
+        }
+        this.connections = Collections.unmodifiableList(connections);
     }
 
     public ElasticsearchResponse performRequest(String method, String endpoint, Map<String, Object> params, HttpEntity entity)
             throws IOException {
         URI uri = buildUri(endpoint, params);
         HttpRequestBase request = createHttpRequest(method, uri, entity);
-        return performRequest(request, connectionPool.nextConnection());
-    }
-
-    private ElasticsearchResponse performRequest(HttpRequestBase request, Iterator<Connection> connectionIterator) throws IOException {
         //we apply a soft margin so that e.g. if a request took 59 seconds and timeout is set to 60 we don't do another attempt
         long retryTimeout = Math.round(this.maxRetryTimeout / (float)100 * 98);
         IOException lastSeenException = null;
         long startTime = System.nanoTime();
-
+        Iterator<Connection> connectionIterator = nextConnection();
         while (connectionIterator.hasNext()) {
             Connection connection = connectionIterator.next();
 
@@ -83,6 +96,7 @@ public final class RestClient implements Closeable {
                     retryTimeoutException.addSuppressed(lastSeenException);
                     throw retryTimeoutException;
                 }
+                request.reset();
             }
 
             CloseableHttpResponse response;
@@ -90,17 +104,15 @@ public final class RestClient implements Closeable {
                 response = client.execute(connection.getHost(), request);
             } catch(IOException e) {
                 RequestLogger.log(logger, "request failed", request, connection.getHost(), e);
-                connectionPool.onFailure(connection);
+                onFailure(connection);
                 lastSeenException = addSuppressedException(lastSeenException, e);
                 continue;
-            } finally {
-                request.reset();
             }
             int statusCode = response.getStatusLine().getStatusCode();
             //TODO make ignore status code configurable. rest-spec and tests support that parameter (ignore_missing)
             if (statusCode < 300 || (request.getMethod().equals(HttpHead.METHOD_NAME) && statusCode == 404) ) {
                 RequestLogger.log(logger, "request succeeded", request, connection.getHost(), response);
-                connectionPool.onSuccess(connection);
+                onSuccess(connection);
                 return new ElasticsearchResponse(request.getRequestLine(), connection.getHost(), response);
             } else {
                 RequestLogger.log(logger, "request failed", request, connection.getHost(), response);
@@ -113,10 +125,10 @@ public final class RestClient implements Closeable {
                 lastSeenException = addSuppressedException(lastSeenException, elasticsearchResponseException);
                 //clients don't retry on 500 because elasticsearch still misuses it instead of 400 in some places
                 if (statusCode == 502 || statusCode == 503 || statusCode == 504) {
-                    connectionPool.onFailure(connection);
+                    onFailure(connection);
                 } else {
                     //don't retry and call onSuccess as the error should be a request problem, the node is alive
-                    connectionPool.onSuccess(connection);
+                    onSuccess(connection);
                     break;
                 }
             }
@@ -125,6 +137,74 @@ public final class RestClient implements Closeable {
         throw lastSeenException;
     }
 
+    /**
+     * Returns an iterator of connections that should be used for a request call.
+     * Ideally, the first connection is retrieved from the iterator and used successfully for the request.
+     * Otherwise, after each failure the next connection should be retrieved from the iterator so that the request can be retried.
+     * The maximum total of attempts is equal to the number of connections that are available in the iterator.
+     * The iterator returned will never be empty, rather an {@link IllegalStateException} will be thrown in that case.
+     * In case there are no alive connections available, or dead ones that should be retried, one dead connection
+     * gets resurrected and returned.
+     */
+    private Iterator<Connection> nextConnection() {
+        if (this.connections.isEmpty()) {
+            throw new IllegalStateException("no connections available in the connection pool");
+        }
+
+        List<Connection> rotatedConnections = new ArrayList<>(connections);
+        //TODO is it possible to make this O(1)? (rotate is O(n))
+        Collections.rotate(rotatedConnections, rotatedConnections.size() - lastConnectionIndex.getAndIncrement());
+        Iterator<Connection> connectionIterator = rotatedConnections.iterator();
+        while (connectionIterator.hasNext()) {
+            Connection connection = connectionIterator.next();
+            if (connection.isAlive() == false && connection.shouldBeRetried() == false) {
+                connectionIterator.remove();
+            }
+        }
+        if (rotatedConnections.isEmpty()) {
+            List<Connection> sortedConnections = new ArrayList<>(connections);
+            Collections.sort(sortedConnections, new Comparator<Connection>() {
+                @Override
+                public int compare(Connection o1, Connection o2) {
+                    return Long.compare(o1.getDeadUntil(), o2.getDeadUntil());
+                }
+            });
+            Connection connection = sortedConnections.get(0);
+            connection.markResurrected();
+            logger.trace("marked connection resurrected for " + connection.getHost());
+            return Collections.singleton(connection).iterator();
+        }
+        return rotatedConnections.iterator();
+    }
+
+    /**
+     * Called after each successful request call.
+     * Receives as an argument the connection that was used for the successful request.
+     */
+    public void onSuccess(Connection connection) {
+        connection.markAlive();
+        logger.trace("marked connection alive for " + connection.getHost());
+    }
+
+    /**
+     * Called after each failed attempt.
+     * Receives as an argument the connection that was used for the failed attempt.
+     */
+    private void onFailure(Connection connection) throws IOException {
+        connection.markDead();
+        logger.debug("marked connection dead for " + connection.getHost());
+        failureListener.onFailure(connection);
+    }
+
+    public synchronized void setFailureListener(FailureListener failureListener) {
+        this.failureListener = failureListener;
+    }
+
+    @Override
+    public void close() throws IOException {
+        client.close();
+    }
+
     private static IOException addSuppressedException(IOException suppressedException, IOException currentException) {
         if (suppressedException != null) {
             currentException.addSuppressed(suppressedException);
@@ -178,12 +258,6 @@ public final class RestClient implements Closeable {
         }
     }
 
-    @Override
-    public void close() throws IOException {
-        connectionPool.close();
-        client.close();
-    }
-
     /**
      * Returns a new {@link Builder} to help with {@link RestClient} creation.
      */
@@ -195,9 +269,11 @@ public final class RestClient implements Closeable {
      * Rest client builder. Helps creating a new {@link RestClient}.
      */
     public static final class Builder {
-        private static final int DEFAULT_MAX_RETRY_TIMEOUT = 10000;
+        public static final int DEFAULT_CONNECT_TIMEOUT = 500;
+        public static final int DEFAULT_SOCKET_TIMEOUT = 5000;
+        public static final int DEFAULT_MAX_RETRY_TIMEOUT = DEFAULT_SOCKET_TIMEOUT;
+        public static final int DEFAULT_CONNECTION_REQUEST_TIMEOUT = 500;
 
-        private ConnectionPool connectionPool;
         private CloseableHttpClient httpClient;
         private int maxRetryTimeout = DEFAULT_MAX_RETRY_TIMEOUT;
         private HttpHost[] hosts;
@@ -206,17 +282,9 @@ public final class RestClient implements Closeable {
 
         }
 
-        /**
-         * Sets the connection pool. {@link StaticConnectionPool} will be used if not specified.
-         * @see ConnectionPool
-         */
-        public Builder setConnectionPool(ConnectionPool connectionPool) {
-            this.connectionPool = connectionPool;
-            return this;
-        }
-
         /**
          * Sets the http client. A new default one will be created if not specified, by calling {@link #createDefaultHttpClient()}.
+         *
          * @see CloseableHttpClient
          */
         public Builder setHttpClient(CloseableHttpClient httpClient) {
@@ -227,6 +295,7 @@ public final class RestClient implements Closeable {
         /**
          * Sets the maximum timeout to honour in case of multiple retries of the same request.
          * {@link #DEFAULT_MAX_RETRY_TIMEOUT} if not specified.
+         *
          * @throws IllegalArgumentException if maxRetryTimeout is not greater than 0
          */
         public Builder setMaxRetryTimeout(int maxRetryTimeout) {
@@ -256,10 +325,7 @@ public final class RestClient implements Closeable {
             if (httpClient == null) {
                 httpClient = createDefaultHttpClient();
             }
-            if (connectionPool == null) {
-                connectionPool = new StaticConnectionPool(hosts);
-            }
-            return new RestClient(httpClient, connectionPool, maxRetryTimeout);
+            return new RestClient(httpClient, maxRetryTimeout, hosts);
         }
 
         /**
@@ -274,10 +340,21 @@ public final class RestClient implements Closeable {
             connectionManager.setMaxTotal(30);
 
             //default timeouts are all infinite
-            RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(500).setSocketTimeout(10000)
-                    .setConnectionRequestTimeout(500).build();
+            RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(DEFAULT_CONNECT_TIMEOUT)
+                    .setSocketTimeout(DEFAULT_SOCKET_TIMEOUT)
+                    .setConnectionRequestTimeout(DEFAULT_CONNECTION_REQUEST_TIMEOUT).build();
 
             return HttpClientBuilder.create().setConnectionManager(connectionManager).setDefaultRequestConfig(requestConfig).build();
         }
     }
+
+    /**
+     * Listener that allows to be notified whenever a failure happens. Useful when sniffing is enabled, so that we can sniff on failure.
+     * The default implementation is a no-op.
+     */
+    public static class FailureListener {
+        public void onFailure(Connection connection) throws IOException {
+
+        }
+    }
 }
diff --git a/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java b/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
deleted file mode 100644
index 4ec0bcb3c39..00000000000
--- a/client/src/main/java/org/elasticsearch/client/StaticConnectionPool.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.client;
-
-import org.apache.http.HttpHost;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Static implementation of {@link ConnectionPool}. Its underlying list of connections is immutable.
- */
-public class StaticConnectionPool extends ConnectionPool {
-
-    private final List<Connection> connections;
-
-    public StaticConnectionPool(HttpHost... hosts) {
-        if (hosts == null || hosts.length == 0) {
-            throw new IllegalArgumentException("no hosts provided");
-        }
-        this.connections = createConnections(hosts);
-    }
-
-    @Override
-    protected List<Connection> getConnections() {
-        return connections;
-    }
-
-    @Override
-    public void close() throws IOException {
-        //no-op nothing to close
-    }
-}
diff --git a/client/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java b/client/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
new file mode 100644
index 00000000000..29117fcd856
--- /dev/null
+++ b/client/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.sniff;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
+import org.elasticsearch.client.ElasticsearchResponse;
+import org.elasticsearch.client.RestClient;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Class responsible for sniffing the http hosts from elasticsearch through the nodes info api and returning them back
+ */
+public class HostsSniffer {
+
+    private static final Log logger = LogFactory.getLog(HostsSniffer.class);
+
+    private final RestClient restClient;
+    private final Map<String, Object> sniffRequestParams;
+    private final String scheme;
+    private final JsonFactory jsonFactory;
+
+    public HostsSniffer(RestClient restClient, int sniffRequestTimeout, String scheme) {
+        this.restClient = restClient;
+        this.sniffRequestParams = Collections.<String, Object>singletonMap("timeout", sniffRequestTimeout + "ms");
+        this.scheme = scheme;
+        this.jsonFactory = new JsonFactory();
+    }
+
+    /**
+     * Calls the elasticsearch nodes info api, parses the response and returns all the found http hosts
+     */
+    public List<HttpHost> sniffHosts() throws IOException {
+        try (ElasticsearchResponse response = restClient.performRequest("get", "/_nodes/http", sniffRequestParams, null)) {
+            return readHosts(response.getEntity());
+        }
+    }
+
+    private List<HttpHost> readHosts(HttpEntity entity) throws IOException {
+        try (InputStream inputStream = entity.getContent()) {
+            JsonParser parser = jsonFactory.createParser(inputStream);
+            if (parser.nextToken() != JsonToken.START_OBJECT) {
+                throw new IOException("expected data to start with an object");
+            }
+            List<HttpHost> hosts = new ArrayList<>();
+            while (parser.nextToken() != JsonToken.END_OBJECT) {
+                if (parser.getCurrentToken() == JsonToken.START_OBJECT) {
+                    if ("nodes".equals(parser.getCurrentName())) {
+                        while (parser.nextToken() != JsonToken.END_OBJECT) {
+                            JsonToken token = parser.nextToken();
+                            assert token == JsonToken.START_OBJECT;
+                            String nodeId = parser.getCurrentName();
+                            HttpHost sniffedHost = readHost(nodeId, parser, this.scheme);
+                            if (sniffedHost != null) {
+                                logger.trace("adding node [" + nodeId + "]");
+                                hosts.add(sniffedHost);
+                            }
+                        }
+                    } else {
+                        parser.skipChildren();
+                    }
+                }
+            }
+            return hosts;
+        }
+    }
+
+    private static HttpHost readHost(String nodeId, JsonParser parser, String scheme) throws IOException {
+        HttpHost httpHost = null;
+        String fieldName = null;
+        while (parser.nextToken() != JsonToken.END_OBJECT) {
+            if (parser.getCurrentToken() == JsonToken.FIELD_NAME) {
+                fieldName = parser.getCurrentName();
+            } else if (parser.getCurrentToken() == JsonToken.START_OBJECT) {
+                if ("http".equals(fieldName)) {
+                    while (parser.nextToken() != JsonToken.END_OBJECT) {
+                        if (parser.getCurrentToken() == JsonToken.VALUE_STRING && "publish_address".equals(parser.getCurrentName())) {
+                            URI boundAddressAsURI = URI.create(scheme + "://" + parser.getValueAsString());
+                            httpHost = new HttpHost(boundAddressAsURI.getHost(), boundAddressAsURI.getPort(),
+                                    boundAddressAsURI.getScheme());
+                        } else if (parser.getCurrentToken() == JsonToken.START_OBJECT) {
+                            parser.skipChildren();
+                        }
+                    }
+                } else {
+                    parser.skipChildren();
+                }
+            }
+        }
+        //http section is not present if http is not enabled on the node, ignore such nodes
+        if (httpHost == null) {
+            logger.debug("skipping node [" + nodeId + "] with http disabled");
+            return null;
+        }
+        return httpHost;
+    }
+}
diff --git a/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
index 09a72fedb62..43ad143738a 100644
--- a/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
+++ b/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
@@ -19,130 +19,228 @@
 
 package org.elasticsearch.client.sniff;
 
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonToken;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.http.HttpEntity;
 import org.apache.http.HttpHost;
-import org.apache.http.StatusLine;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.util.EntityUtils;
-import org.elasticsearch.client.ElasticsearchResponseException;
-import org.elasticsearch.client.RequestLogger;
+import org.elasticsearch.client.Connection;
+import org.elasticsearch.client.RestClient;
 
+import java.io.Closeable;
 import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
  * Calls nodes info api and returns a list of http hosts extracted from it.
  */
 //TODO This could potentially be using _cat/nodes which wouldn't require jackson as a dependency, but we'd have bw comp problems with 2.x
-final class Sniffer {
+public final class Sniffer extends RestClient.FailureListener implements Closeable {
 
     private static final Log logger = LogFactory.getLog(Sniffer.class);
 
-    private final CloseableHttpClient client;
-    private final RequestConfig sniffRequestConfig;
-    private final int sniffRequestTimeout;
-    private final String scheme;
-    private final JsonFactory jsonFactory;
+    private final boolean sniffOnFailure;
+    private final Task task;
 
-    Sniffer(CloseableHttpClient client, RequestConfig sniffRequestConfig, int sniffRequestTimeout, String scheme) {
-        this.client = client;
-        this.sniffRequestConfig = sniffRequestConfig;
-        this.sniffRequestTimeout = sniffRequestTimeout;
-        this.scheme = scheme;
-        this.jsonFactory = new JsonFactory();
+    public Sniffer(RestClient restClient, int sniffRequestTimeout, String scheme, int sniffInterval,
+                   boolean sniffOnFailure, int sniffAfterFailureDelay) {
+        HostsSniffer hostsSniffer = new HostsSniffer(restClient, sniffRequestTimeout, scheme);
+        this.task = new Task(hostsSniffer, restClient, sniffInterval, sniffAfterFailureDelay);
+        this.sniffOnFailure = sniffOnFailure;
+        restClient.setFailureListener(this);
     }
 
-    List<HttpHost> sniffNodes(HttpHost host) throws IOException {
-        HttpGet httpGet = new HttpGet("/_nodes/http?timeout=" + sniffRequestTimeout + "ms");
-        httpGet.setConfig(sniffRequestConfig);
-
-        try (CloseableHttpResponse response = client.execute(host, httpGet)) {
-            StatusLine statusLine = response.getStatusLine();
-            if (statusLine.getStatusCode() >= 300) {
-                RequestLogger.log(logger, "sniff failed", httpGet, host, response);
-                String responseBody = null;
-                if (response.getEntity() != null) {
-                    responseBody = EntityUtils.toString(response.getEntity());
-                }
-                throw new ElasticsearchResponseException(httpGet.getRequestLine(), host, response.getStatusLine(), responseBody);
-            } else {
-                List<HttpHost> nodes = readHosts(response.getEntity());
-                RequestLogger.log(logger, "sniff succeeded", httpGet, host, response);
-                return nodes;
-            }
-        } catch(IOException e) {
-            RequestLogger.log(logger, "sniff failed", httpGet, host, e);
-            throw e;
+    @Override
+    public void onFailure(Connection connection) throws IOException {
+        if (sniffOnFailure) {
+            //re-sniff immediately but take out the node that failed
+            task.sniffOnFailure(connection.getHost());
         }
     }
 
-    private List<HttpHost> readHosts(HttpEntity entity) throws IOException {
-        try (InputStream inputStream = entity.getContent()) {
-            JsonParser parser = jsonFactory.createParser(inputStream);
-            if (parser.nextToken() != JsonToken.START_OBJECT) {
-                throw new IOException("expected data to start with an object");
-            }
-            List<HttpHost> hosts = new ArrayList<>();
-            while (parser.nextToken() != JsonToken.END_OBJECT) {
-                if (parser.getCurrentToken() == JsonToken.START_OBJECT) {
-                    if ("nodes".equals(parser.getCurrentName())) {
-                        while (parser.nextToken() != JsonToken.END_OBJECT) {
-                            JsonToken token = parser.nextToken();
-                            assert token == JsonToken.START_OBJECT;
-                            String nodeId = parser.getCurrentName();
-                            HttpHost sniffedHost = readNode(nodeId, parser, this.scheme);
-                            if (sniffedHost != null) {
-                                logger.trace("adding node [" + nodeId + "]");
-                                hosts.add(sniffedHost);
-                            }
-                        }
-                    } else {
-                        parser.skipChildren();
+    @Override
+    public void close() throws IOException {
+        task.shutdown();
+    }
+
+    private static class Task implements Runnable {
+        private final HostsSniffer hostsSniffer;
+        private final RestClient restClient;
+
+        private final int sniffInterval;
+        private final int sniffAfterFailureDelay;
+        private final ScheduledExecutorService scheduledExecutorService;
+        private final AtomicBoolean running = new AtomicBoolean(false);
+        private volatile int nextSniffDelay;
+        private volatile ScheduledFuture<?> scheduledFuture;
+
+        private Task(HostsSniffer hostsSniffer, RestClient restClient, int sniffInterval, int sniffAfterFailureDelay) {
+            this.hostsSniffer = hostsSniffer;
+            this.restClient = restClient;
+            this.sniffInterval = sniffInterval;
+            this.sniffAfterFailureDelay = sniffAfterFailureDelay;
+            this.scheduledExecutorService = Executors.newScheduledThreadPool(1);
+            this.scheduledFuture = this.scheduledExecutorService.schedule(this, 0, TimeUnit.MILLISECONDS);
+            this.nextSniffDelay = sniffInterval;
+        }
+
+        @Override
+        public void run() {
+            sniff(null);
+        }
+
+        void sniffOnFailure(HttpHost failedHost) {
+            this.nextSniffDelay = sniffAfterFailureDelay;
+            sniff(failedHost);
+        }
+
+        void sniff(HttpHost excludeHost) {
+            if (running.compareAndSet(false, true)) {
+                try {
+                    List<HttpHost> sniffedNodes = hostsSniffer.sniffHosts();
+                    if (excludeHost != null) {
+                        sniffedNodes.remove(excludeHost);
+                    }
+                    logger.debug("sniffed nodes: " + sniffedNodes);
+                    this.restClient.setNodes(sniffedNodes.toArray(new HttpHost[sniffedNodes.size()]));
+                } catch (Throwable t) {
+                    logger.error("error while sniffing nodes", t);
+                } finally {
+                    try {
+                        //regardless of whether and when the next sniff is scheduled, cancel it and schedule a new one with updated delay
+                        this.scheduledFuture.cancel(false);
+                        logger.debug("scheduling next sniff in " + nextSniffDelay + " ms");
+                        this.scheduledFuture = this.scheduledExecutorService.schedule(this, nextSniffDelay, TimeUnit.MILLISECONDS);
+                    } catch (Throwable t) {
+                        logger.error("error while scheduling next sniffer task", t);
+                    } finally {
+                        this.nextSniffDelay = sniffInterval;
+                        running.set(false);
                     }
                 }
             }
-            return hosts;
+        }
+
+        void shutdown() {
+            scheduledExecutorService.shutdown();
+            try {
+                if (scheduledExecutorService.awaitTermination(1000, TimeUnit.MILLISECONDS)) {
+                    return;
+                }
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+            scheduledExecutorService.shutdownNow();
         }
     }
 
-    private static HttpHost readNode(String nodeId, JsonParser parser, String scheme) throws IOException {
-        HttpHost httpHost = null;
-        String fieldName = null;
-        while (parser.nextToken() != JsonToken.END_OBJECT) {
-            if (parser.getCurrentToken() == JsonToken.FIELD_NAME) {
-                fieldName = parser.getCurrentName();
-            } else if (parser.getCurrentToken() == JsonToken.START_OBJECT) {
-                if ("http".equals(fieldName)) {
-                    while (parser.nextToken() != JsonToken.END_OBJECT) {
-                        if (parser.getCurrentToken() == JsonToken.VALUE_STRING && "publish_address".equals(parser.getCurrentName())) {
-                            URI boundAddressAsURI = URI.create(scheme + "://" + parser.getValueAsString());
-                            httpHost = new HttpHost(boundAddressAsURI.getHost(), boundAddressAsURI.getPort(),
-                                    boundAddressAsURI.getScheme());
-                        } else if (parser.getCurrentToken() == JsonToken.START_OBJECT) {
-                            parser.skipChildren();
-                        }
-                    }
-                } else {
-                    parser.skipChildren();
-                }
+    /**
+     * Returns a new {@link Builder} to help with {@link Sniffer} creation.
+     */
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    /**
+     * Sniffer builder. Helps creating a new {@link Sniffer}.
+     */
+    public static final class Builder {
+        public static final int DEFAULT_SNIFF_INTERVAL = 60000 * 5; //5 minutes
+        public static final int DEFAULT_SNIFF_AFTER_FAILURE_DELAY = 60000; //1 minute
+        public static final int DEFAULT_SNIFF_REQUEST_TIMEOUT = 1000; //1 second
+
+        private int sniffRequestTimeout = DEFAULT_SNIFF_REQUEST_TIMEOUT;
+        private int sniffInterval = DEFAULT_SNIFF_INTERVAL;
+        private boolean sniffOnFailure = true;
+        private int sniffAfterFailureDelay = DEFAULT_SNIFF_AFTER_FAILURE_DELAY;
+        private String scheme = "http";
+        private RestClient restClient;
+
+        private Builder() {
+
+        }
+
+        /**
+         * Sets the interval between consecutive ordinary sniff executions. Will be honoured when sniffOnFailure is disabled or
+         * when there are no failures between consecutive sniff executions.
+         * @throws IllegalArgumentException if sniffInterval is not greater than 0
+         */
+        public Builder setSniffInterval(int sniffInterval) {
+            if (sniffInterval <= 0) {
+                throw new IllegalArgumentException("sniffInterval must be greater than 0");
             }
+            this.sniffInterval = sniffInterval;
+            return this;
         }
-        //http section is not present if http is not enabled on the node, ignore such nodes
-        if (httpHost == null) {
-            logger.debug("skipping node [" + nodeId + "] with http disabled");
-            return null;
+
+        /**
+         * Enables/disables sniffing on failure. If enabled, at each failure nodes will be reloaded, and a new sniff execution will
+         * be scheduled after a shorter time than usual (sniffAfterFailureDelay).
+         */
+        public Builder setSniffOnFailure(boolean sniffOnFailure) {
+            this.sniffOnFailure = sniffOnFailure;
+            return this;
+        }
+
+        /**
+         * Sets the delay of a sniff execution scheduled after a failure.
+         */
+        public Builder setSniffAfterFailureDelay(int sniffAfterFailureDelay) {
+            if (sniffAfterFailureDelay <= 0) {
+                throw new IllegalArgumentException("sniffAfterFailureDelay must be greater than 0");
+            }
+            this.sniffAfterFailureDelay = sniffAfterFailureDelay;
+            return this;
+        }
+
+        /**
+         * Sets the http client. Mandatory argument. Best practice is to use the same client used
+         * within {@link org.elasticsearch.client.RestClient} which can be created manually or
+         * through {@link RestClient.Builder#createDefaultHttpClient()}.
+         * @see CloseableHttpClient
+         */
+        public Builder setRestClient(RestClient restClient) {
+            this.restClient = restClient;
+            return this;
+        }
+
+        /**
+         * Sets the sniff request timeout to be passed in as a query string parameter to elasticsearch.
+         * Allows to halt the request without any failure, as only the nodes that have responded
+         * within this timeout will be returned.
+         */
+        public Builder setSniffRequestTimeout(int sniffRequestTimeout) {
+            if (sniffRequestTimeout <=0) {
+                throw new IllegalArgumentException("sniffRequestTimeout must be greater than 0");
+            }
+            this.sniffRequestTimeout = sniffRequestTimeout;
+            return this;
+        }
+
+        /**
+         * Sets the scheme to be used for sniffed nodes. This information is not returned by elasticsearch,
+         * default is http but should be customized if https is needed/enabled.
+         */
+        public Builder setScheme(String scheme) {
+            Objects.requireNonNull(scheme, "scheme cannot be null");
+            if (scheme.equals("http") == false && scheme.equals("https") == false) {
+                throw new IllegalArgumentException("scheme must be either http or https");
+            }
+            this.scheme = scheme;
+            return this;
+        }
+
+        /**
+         * Creates the {@link Sniffer} based on the provided configuration.
+         */
+        public Sniffer build() {
+            Objects.requireNonNull(restClient, "restClient cannot be null");
+            return new Sniffer(restClient, sniffRequestTimeout, scheme, sniffInterval, sniffOnFailure, sniffAfterFailureDelay);
         }
-        return httpHost;
     }
 }
diff --git a/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java b/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
deleted file mode 100644
index 6fd98666e8c..00000000000
--- a/client/src/main/java/org/elasticsearch/client/sniff/SniffingConnectionPool.java
+++ /dev/null
@@ -1,299 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.client.sniff;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.http.HttpHost;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.elasticsearch.client.Connection;
-import org.elasticsearch.client.ConnectionPool;
-import org.elasticsearch.client.RestClient;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Objects;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * Connection pool implementation that sniffs nodes from elasticsearch at regular intervals.
- * Can optionally sniff nodes on each failure as well.
- */
-public class SniffingConnectionPool extends ConnectionPool {
-
-    private static final Log logger = LogFactory.getLog(SniffingConnectionPool.class);
-
-    private final boolean sniffOnFailure;
-    private final Sniffer sniffer;
-    private volatile List<Connection> connections;
-    private final SnifferTask snifferTask;
-
-    private SniffingConnectionPool(int sniffInterval, boolean sniffOnFailure, int sniffAfterFailureDelay, CloseableHttpClient client,
-                                   RequestConfig sniffRequestConfig, int sniffRequestTimeout, String scheme, HttpHost... hosts) {
-        this.sniffOnFailure = sniffOnFailure;
-        this.sniffer = new Sniffer(client, sniffRequestConfig, sniffRequestTimeout, scheme);
-        this.connections = createConnections(hosts);
-        this.snifferTask = new SnifferTask(sniffInterval, sniffAfterFailureDelay);
-    }
-
-    @Override
-    protected List<Connection> getConnections() {
-        return this.connections;
-    }
-
-    @Override
-    public void onFailure(Connection connection) throws IOException {
-        super.onFailure(connection);
-        if (sniffOnFailure) {
-            //re-sniff immediately but take out the node that failed
-            snifferTask.sniffOnFailure(connection.getHost());
-        }
-    }
-
-    @Override
-    public void close() throws IOException {
-        snifferTask.shutdown();
-    }
-
-    private class SnifferTask implements Runnable {
-        private final int sniffInterval;
-        private final int sniffAfterFailureDelay;
-        private final ScheduledExecutorService scheduledExecutorService;
-        private final AtomicBoolean running = new AtomicBoolean(false);
-        private volatile boolean failure = false;
-        private volatile ScheduledFuture<?> scheduledFuture;
-
-        private SnifferTask(int sniffInterval, int sniffAfterFailureDelay) {
-            this.sniffInterval = sniffInterval;
-            this.sniffAfterFailureDelay = sniffAfterFailureDelay;
-            this.scheduledExecutorService = Executors.newScheduledThreadPool(1);
-            this.scheduledFuture = this.scheduledExecutorService.schedule(this, 0, TimeUnit.MILLISECONDS);
-        }
-
-        @Override
-        public void run() {
-            sniff(null);
-        }
-
-        void sniffOnFailure(HttpHost failedHost) {
-            //sync sniff straightaway on failure
-            failure = true;
-            sniff(failedHost);
-        }
-
-        void sniff(HttpHost excludeHost) {
-            if (running.compareAndSet(false, true)) {
-                try {
-                    sniff(nextConnection(), excludeHost);
-                } catch (Throwable t) {
-                    logger.error("error while sniffing nodes", t);
-                } finally {
-                    try {
-                        //regardless of whether and when the next sniff is scheduled, cancel it and schedule a new one with updated delay
-                        this.scheduledFuture.cancel(false);
-                        if (this.failure) {
-                            this.scheduledFuture = this.scheduledExecutorService.schedule(this,
-                                    sniffAfterFailureDelay, TimeUnit.MILLISECONDS);
-                            this.failure = false;
-                        } else {
-                            this.scheduledFuture = this.scheduledExecutorService.schedule(this, sniffInterval, TimeUnit.MILLISECONDS);
-                        }
-                    } catch (Throwable t) {
-                        logger.error("error while scheduling next sniffer task", t);
-                    } finally {
-                        running.set(false);
-                    }
-                }
-            }
-        }
-
-        void sniff(Iterator<Connection> connectionIterator, HttpHost excludeHost) throws IOException {
-            IOException lastSeenException = null;
-            while (connectionIterator.hasNext()) {
-                Connection connection = connectionIterator.next();
-                try {
-                    List<HttpHost> sniffedNodes = sniffer.sniffNodes(connection.getHost());
-                    if (excludeHost != null) {
-                        sniffedNodes.remove(excludeHost);
-                    }
-                    connections = createConnections(sniffedNodes.toArray(new HttpHost[sniffedNodes.size()]));
-                    onSuccess(connection);
-                    return;
-                } catch (IOException e) {
-                    //here we have control over the request, if it fails something is really wrong, always call onFailure
-                    onFailure(connection);
-                    if (lastSeenException != null) {
-                        e.addSuppressed(lastSeenException);
-                    }
-                    lastSeenException = e;
-                }
-            }
-            logger.warn("failed to sniff nodes", lastSeenException);
-        }
-
-        void shutdown() {
-            scheduledExecutorService.shutdown();
-            try {
-                if (scheduledExecutorService.awaitTermination(1000, TimeUnit.MILLISECONDS)) {
-                    return;
-                }
-            } catch (InterruptedException e) {
-                Thread.currentThread().interrupt();
-            }
-            scheduledExecutorService.shutdownNow();
-        }
-    }
-
-    /**
-     * Returns a new {@link Builder} to help with {@link SniffingConnectionPool} creation.
-     */
-    public static Builder builder() {
-        return new Builder();
-    }
-
-    /**
-     * Sniffing connection pool builder. Helps creating a new {@link SniffingConnectionPool}.
-     */
-    public static final class Builder {
-        private int sniffInterval = 5 * 1000 * 60;
-        private boolean sniffOnFailure = true;
-        private int sniffAfterFailureDelay = 60000;
-        private CloseableHttpClient httpClient;
-        private RequestConfig sniffRequestConfig;
-        private int sniffRequestTimeout = 1000;
-        private String scheme = "http";
-        private HttpHost[] hosts;
-
-        private Builder() {
-
-        }
-
-        /**
-         * Sets the interval between consecutive ordinary sniff executions. Will be honoured when sniffOnFailure is disabled or
-         * when there are no failures between consecutive sniff executions.
-         * @throws IllegalArgumentException if sniffInterval is not greater than 0
-         */
-        public Builder setSniffInterval(int sniffInterval) {
-            if (sniffInterval <= 0) {
-                throw new IllegalArgumentException("sniffInterval must be greater than 0");
-            }
-            this.sniffInterval = sniffInterval;
-            return this;
-        }
-
-        /**
-         * Enables/disables sniffing on failure. If enabled, at each failure nodes will be reloaded, and a new sniff execution will
-         * be scheduled after a shorter time than usual (sniffAfterFailureDelay).
-         */
-        public Builder setSniffOnFailure(boolean sniffOnFailure) {
-            this.sniffOnFailure = sniffOnFailure;
-            return this;
-        }
-
-        /**
-         * Sets the delay of a sniff execution scheduled after a failure.
-         */
-        public Builder setSniffAfterFailureDelay(int sniffAfterFailureDelay) {
-            if (sniffAfterFailureDelay <= 0) {
-                throw new IllegalArgumentException("sniffAfterFailureDelay must be greater than 0");
-            }
-            this.sniffAfterFailureDelay = sniffAfterFailureDelay;
-            return this;
-        }
-
-        /**
-         * Sets the http client. Mandatory argument. Best practice is to use the same client used
-         * within {@link org.elasticsearch.client.RestClient} which can be created manually or
-         * through {@link RestClient.Builder#createDefaultHttpClient()}.
-         * @see CloseableHttpClient
-         */
-        public Builder setHttpClient(CloseableHttpClient httpClient) {
-            this.httpClient = httpClient;
-            return this;
-        }
-
-        /**
-         * Sets the configuration to be used for each sniff request. Useful as sniff can have
-         * different timeouts compared to ordinary requests.
-         * @see RequestConfig
-         */
-        public Builder setSniffRequestConfig(RequestConfig sniffRequestConfig) {
-            this.sniffRequestConfig = sniffRequestConfig;
-            return this;
-        }
-
-        /**
-         * Sets the sniff request timeout to be passed in as a query string parameter to elasticsearch.
-         * Allows to halt the request without any failure, as only the nodes that have responded
-         * within this timeout will be returned.
-         */
-        public Builder setSniffRequestTimeout(int sniffRequestTimeout) {
-            if (sniffRequestTimeout <=0) {
-                throw new IllegalArgumentException("sniffRequestTimeout must be greater than 0");
-            }
-            this.sniffRequestTimeout = sniffRequestTimeout;
-            return this;
-        }
-
-        /**
-         * Sets the scheme to be used for sniffed nodes. This information is not returned by elasticsearch,
-         * default is http but should be customized if https is needed/enabled.
-         */
-        public Builder setScheme(String scheme) {
-            Objects.requireNonNull(scheme, "scheme cannot be null");
-            if (scheme.equals("http") == false && scheme.equals("https") == false) {
-                throw new IllegalArgumentException("scheme must be either http or https");
-            }
-            this.scheme = scheme;
-            return this;
-        }
-
-        /**
-         * Sets the hosts that the client will send requests to.
-         */
-        public Builder setHosts(HttpHost... hosts) {
-            this.hosts = hosts;
-            return this;
-        }
-
-        /**
-         * Creates the {@link SniffingConnectionPool} based on the provided configuration.
-         */
-        public SniffingConnectionPool build() {
-            Objects.requireNonNull(httpClient, "httpClient cannot be null");
-            if (hosts == null || hosts.length == 0) {
-                throw new IllegalArgumentException("no hosts provided");
-            }
-
-            if (sniffRequestConfig == null) {
-                sniffRequestConfig = RequestConfig.custom().setConnectTimeout(500).setSocketTimeout(1000)
-                        .setConnectionRequestTimeout(500).build();
-            }
-            return new SniffingConnectionPool(sniffInterval, sniffOnFailure, sniffAfterFailureDelay, httpClient, sniffRequestConfig,
-                    sniffRequestTimeout, scheme, hosts);
-        }
-    }
-}
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java b/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
index 11008d52b19..e0828f2bd1f 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
@@ -25,8 +25,6 @@ import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.lucene.util.LuceneTestCase;
 
 import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
 import java.util.logging.LogManager;
 
 public class RestClientBuilderTests extends LuceneTestCase {
@@ -57,38 +55,23 @@ public class RestClientBuilderTests extends LuceneTestCase {
             assertEquals(e.getMessage(), "no hosts provided");
         }
 
-        RestClient.Builder builder = RestClient.builder();
-        if (random().nextBoolean()) {
-            ConnectionPool connectionPool = new ConnectionPool() {
-                @Override
-                protected List<Connection> getConnections() {
-                    return Collections.emptyList();
-                }
-
-                @Override
-                public void onSuccess(Connection connection) {
-
-                }
-
-                @Override
-                public void onFailure(Connection connection) throws IOException {
-
-                }
-
-                @Override
-                public void close() throws IOException {
-
-                }
-            };
-            builder.setConnectionPool(connectionPool);
-        } else {
-            int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
-            HttpHost[] hosts = new HttpHost[numNodes];
-            for (int i = 0; i < numNodes; i++) {
-                hosts[i] = new HttpHost("localhost", 9200 + i);
-            }
-            builder.setHosts(hosts);
+        try {
+            RestClient.builder();
+            fail("should have failed");
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "no hosts provided");
         }
+
+        RestClient.Builder builder = RestClient.builder();
+        int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
+        HttpHost[] hosts = new HttpHost[numNodes];
+        for (int i = 0; i < numNodes; i++) {
+            hosts[i] = new HttpHost("localhost", 9200 + i);
+        }
+        builder.setHosts(hosts);
+
+        //TODO test one host is null among others
+
         if (random().nextBoolean()) {
             builder.setHttpClient(HttpClientBuilder.create().build());
         }
diff --git a/client/src/test/java/org/elasticsearch/client/StaticConnectionPoolTests.java b/client/src/test/java/org/elasticsearch/client/StaticConnectionPoolTests.java
deleted file mode 100644
index c777666e98b..00000000000
--- a/client/src/test/java/org/elasticsearch/client/StaticConnectionPoolTests.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.client;
-
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
-import org.apache.http.HttpHost;
-import org.apache.lucene.util.LuceneTestCase;
-
-import java.util.logging.LogManager;
-
-public class StaticConnectionPoolTests extends LuceneTestCase {
-
-    static {
-        LogManager.getLogManager().reset();
-    }
-
-    public void testConstructor() {
-        int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
-        HttpHost[] hosts = new HttpHost[numNodes];
-        for (int i = 0; i < numNodes; i++) {
-            hosts[i] = new HttpHost("localhost", 9200);
-        }
-
-        try {
-            new StaticConnectionPool((HttpHost) null);
-        } catch(NullPointerException e) {
-            assertEquals(e.getMessage(), "host cannot be null");
-        }
-
-        try {
-            new StaticConnectionPool((HttpHost[])null);
-        } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "no hosts provided");
-        }
-
-        try {
-            new StaticConnectionPool();
-        } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "no hosts provided");
-        }
-
-        StaticConnectionPool staticConnectionPool = new StaticConnectionPool(hosts);
-        assertNotNull(staticConnectionPool);
-    }
-}
diff --git a/client/src/test/java/org/elasticsearch/client/sniff/SnifferTests.java b/client/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
similarity index 84%
rename from client/src/test/java/org/elasticsearch/client/sniff/SnifferTests.java
rename to client/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
index 2fa9f7b3225..c9e1b48a791 100644
--- a/client/src/test/java/org/elasticsearch/client/sniff/SnifferTests.java
+++ b/client/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
@@ -29,11 +29,9 @@ import okhttp3.mockwebserver.MockResponse;
 import okhttp3.mockwebserver.MockWebServer;
 import okhttp3.mockwebserver.RecordedRequest;
 import org.apache.http.HttpHost;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.client.ElasticsearchResponseException;
+import org.elasticsearch.client.RestClient;
 import org.junit.After;
 import org.junit.Before;
 
@@ -57,7 +55,7 @@ import java.util.logging.LogManager;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
 
-public class SnifferTests extends LuceneTestCase {
+public class HostsSnifferTests extends LuceneTestCase {
 
     static {
         //prevent MockWebServer from logging to stdout and stderr
@@ -88,34 +86,36 @@ public class SnifferTests extends LuceneTestCase {
     }
 
     public void testSniffNodes() throws IOException, URISyntaxException {
-        CloseableHttpClient client = HttpClientBuilder.create().build();
-        Sniffer sniffer = new Sniffer(client, RequestConfig.DEFAULT, sniffRequestTimeout, scheme);
         HttpHost httpHost = new HttpHost(server.getHostName(), server.getPort());
-        try {
-            List<HttpHost> sniffedHosts = sniffer.sniffNodes(httpHost);
-            if (sniffResponse.isFailure) {
-                fail("sniffNodes should have failed");
-            }
-            assertThat(sniffedHosts.size(), equalTo(sniffResponse.hosts.size()));
-            Iterator<HttpHost> responseHostsIterator = sniffResponse.hosts.iterator();
-            for (HttpHost sniffedHost : sniffedHosts) {
-                assertEquals(sniffedHost, responseHostsIterator.next());
-            }
-        } catch(ElasticsearchResponseException e) {
-            if (sniffResponse.isFailure) {
-                assertThat(e.getMessage(), containsString("GET http://localhost:" + server.getPort() +
-                        "/_nodes/http?timeout=" + sniffRequestTimeout));
-                assertThat(e.getMessage(), containsString(Integer.toString(sniffResponse.nodesInfoResponseCode)));
-                assertThat(e.getHost(), equalTo(httpHost));
-                assertThat(e.getStatusLine().getStatusCode(), equalTo(sniffResponse.nodesInfoResponseCode));
-                assertThat(e.getRequestLine().toString(), equalTo("GET /_nodes/http?timeout=" + sniffRequestTimeout + "ms HTTP/1.1"));
-            } else {
-                fail("sniffNodes should have succeeded: " + e.getStatusLine());
+        try (RestClient restClient = RestClient.builder().setHosts(httpHost).build()) {
+            HostsSniffer sniffer = new HostsSniffer(restClient, sniffRequestTimeout, scheme);
+            try {
+                List<HttpHost> sniffedHosts = sniffer.sniffHosts();
+                if (sniffResponse.isFailure) {
+                    fail("sniffNodes should have failed");
+                }
+                assertThat(sniffedHosts.size(), equalTo(sniffResponse.hosts.size()));
+                Iterator<HttpHost> responseHostsIterator = sniffResponse.hosts.iterator();
+                for (HttpHost sniffedHost : sniffedHosts) {
+                    assertEquals(sniffedHost, responseHostsIterator.next());
+                }
+            } catch(ElasticsearchResponseException e) {
+                if (sniffResponse.isFailure) {
+                    assertThat(e.getMessage(), containsString("GET http://localhost:" + server.getPort() +
+                            "/_nodes/http?timeout=" + sniffRequestTimeout));
+                    assertThat(e.getMessage(), containsString(Integer.toString(sniffResponse.nodesInfoResponseCode)));
+                    assertThat(e.getHost(), equalTo(httpHost));
+                    assertThat(e.getStatusLine().getStatusCode(), equalTo(sniffResponse.nodesInfoResponseCode));
+                    assertThat(e.getRequestLine().toString(), equalTo("GET /_nodes/http?timeout=" + sniffRequestTimeout + "ms HTTP/1.1"));
+                } else {
+                    fail("sniffNodes should have succeeded: " + e.getStatusLine());
+                }
             }
         }
     }
 
-    private static MockWebServer buildMockWebServer(final SniffResponse sniffResponse, final int sniffTimeout) throws UnsupportedEncodingException {
+    private static MockWebServer buildMockWebServer(final SniffResponse sniffResponse, final int sniffTimeout)
+            throws UnsupportedEncodingException {
         MockWebServer server = new MockWebServer();
         final Dispatcher dispatcher = new Dispatcher() {
             @Override
diff --git a/client/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java b/client/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
new file mode 100644
index 00000000000..6373fae00d5
--- /dev/null
+++ b/client/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.sniff;
+
+import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import org.apache.http.HttpHost;
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.client.RestClient;
+
+import java.util.Arrays;
+import java.util.logging.LogManager;
+
+public class SnifferBuilderTests extends LuceneTestCase {
+
+    static {
+        LogManager.getLogManager().reset();
+    }
+
+    public void testBuild() throws Exception {
+
+        try {
+            Sniffer.builder().setScheme(null);
+            fail("should have failed");
+        } catch(NullPointerException e) {
+            assertEquals(e.getMessage(), "scheme cannot be null");
+        }
+
+        try {
+            Sniffer.builder().setScheme("whatever");
+            fail("should have failed");
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "scheme must be either http or https");
+        }
+
+        try {
+            Sniffer.builder().setSniffInterval(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+            fail("should have failed");
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "sniffInterval must be greater than 0");
+        }
+
+        try {
+            Sniffer.builder().setSniffRequestTimeout(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+            fail("should have failed");
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "sniffRequestTimeout must be greater than 0");
+        }
+
+        try {
+            Sniffer.builder().setSniffAfterFailureDelay(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+            fail("should have failed");
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "sniffAfterFailureDelay must be greater than 0");
+        }
+
+        try {
+            Sniffer.builder().build();
+            fail("should have failed");
+        } catch(NullPointerException e) {
+            assertEquals(e.getMessage(), "restClient cannot be null");
+        }
+
+        int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
+        HttpHost[] hosts = new HttpHost[numNodes];
+        for (int i = 0; i < numNodes; i++) {
+            hosts[i] = new HttpHost("localhost", 9200 + i);
+        }
+
+        try (RestClient client = RestClient.builder().setHosts(hosts).build()) {
+            try (Sniffer sniffer = Sniffer.builder().setRestClient(client).build()) {
+                assertNotNull(sniffer);
+            }
+        }
+
+        try (RestClient client = RestClient.builder().setHosts(hosts).build()) {
+            Sniffer.Builder builder = Sniffer.builder().setRestClient(client);
+            if (random().nextBoolean()) {
+                builder.setScheme(RandomPicks.randomFrom(random(), Arrays.asList("http", "https")));
+            }
+            if (random().nextBoolean()) {
+                builder.setSniffInterval(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+            }
+            if (random().nextBoolean()) {
+                builder.setSniffAfterFailureDelay(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+            }
+            if (random().nextBoolean()) {
+                builder.setSniffRequestTimeout(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+            }
+            if (random().nextBoolean()) {
+                builder.setSniffOnFailure(random().nextBoolean());
+            }
+            try (Sniffer connectionPool = builder.build()) {
+                assertNotNull(connectionPool);
+            }
+        }
+    }
+}
diff --git a/client/src/test/java/org/elasticsearch/client/sniff/SniffingConnectionPoolBuilderTests.java b/client/src/test/java/org/elasticsearch/client/sniff/SniffingConnectionPoolBuilderTests.java
deleted file mode 100644
index b5b4eaa3ce1..00000000000
--- a/client/src/test/java/org/elasticsearch/client/sniff/SniffingConnectionPoolBuilderTests.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.client.sniff;
-
-import com.carrotsearch.randomizedtesting.generators.RandomInts;
-import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-import org.apache.http.HttpHost;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.lucene.util.LuceneTestCase;
-
-import java.util.Arrays;
-import java.util.logging.LogManager;
-
-public class SniffingConnectionPoolBuilderTests extends LuceneTestCase {
-
-    static {
-        LogManager.getLogManager().reset();
-    }
-
-    public void testBuild() throws Exception {
-
-        try {
-            SniffingConnectionPool.builder().setScheme(null);
-            fail("should have failed");
-        } catch(NullPointerException e) {
-            assertEquals(e.getMessage(), "scheme cannot be null");
-        }
-
-        try {
-            SniffingConnectionPool.builder().setScheme("whatever");
-            fail("should have failed");
-        } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "scheme must be either http or https");
-        }
-
-        try {
-            SniffingConnectionPool.builder().setSniffInterval(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
-            fail("should have failed");
-        } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "sniffInterval must be greater than 0");
-        }
-
-        try {
-            SniffingConnectionPool.builder().setSniffRequestTimeout(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
-            fail("should have failed");
-        } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "sniffRequestTimeout must be greater than 0");
-        }
-
-        try {
-            SniffingConnectionPool.builder().setSniffAfterFailureDelay(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
-            fail("should have failed");
-        } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "sniffAfterFailureDelay must be greater than 0");
-        }
-
-        try {
-            SniffingConnectionPool.builder().build();
-            fail("should have failed");
-        } catch(NullPointerException e) {
-            assertEquals(e.getMessage(), "httpClient cannot be null");
-        }
-
-        try {
-            SniffingConnectionPool.builder().setHttpClient(HttpClientBuilder.create().build()).build();
-            fail("should have failed");
-        } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "no hosts provided");
-        }
-
-        try {
-            SniffingConnectionPool.builder().setHttpClient(HttpClientBuilder.create().build()).setHosts((HttpHost[])null).build();
-            fail("should have failed");
-        } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "no hosts provided");
-        }
-
-        try {
-            SniffingConnectionPool.builder().setHttpClient(HttpClientBuilder.create().build()).setHosts().build();
-            fail("should have failed");
-        } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "no hosts provided");
-        }
-
-        int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
-        HttpHost[] hosts = new HttpHost[numNodes];
-        for (int i = 0; i < numNodes; i++) {
-            hosts[i] = new HttpHost("localhost", 9200 + i);
-        }
-
-        try (SniffingConnectionPool connectionPool = SniffingConnectionPool.builder()
-                .setHttpClient(HttpClientBuilder.create().build()).setHosts(hosts).build()) {
-            assertNotNull(connectionPool);
-        }
-
-        SniffingConnectionPool.Builder builder = SniffingConnectionPool.builder()
-                .setHttpClient(HttpClientBuilder.create().build()).setHosts(hosts);
-        if (random().nextBoolean()) {
-            builder.setScheme(RandomPicks.randomFrom(random(), Arrays.asList("http", "https")));
-        }
-        if (random().nextBoolean()) {
-            builder.setSniffInterval(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
-        }
-        if (random().nextBoolean()) {
-            builder.setSniffAfterFailureDelay(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
-        }
-        if (random().nextBoolean()) {
-            builder.setSniffRequestTimeout(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
-        }
-        if (random().nextBoolean()) {
-            builder.setSniffOnFailure(random().nextBoolean());
-        }
-        if (random().nextBoolean()) {
-            builder.setSniffRequestConfig(RequestConfig.DEFAULT);
-        }
-        try (SniffingConnectionPool connectionPool = builder.build()) {
-            assertNotNull(connectionPool);
-        }
-    }
-}

From 85a7721185739ad8cedac3065424c42e29ea0418 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 13 May 2016 12:19:45 +0200
Subject: [PATCH 0601/1311] simplify Connection class

The connection class can be greatly simplified now that we don't ping anymore. Pings required a special initial state (UNKNOWN) for connections, to indicate that they require pinging although they are not dead. At this point we don't need the State enum anymore, as connections can only be dead or alive based on the number of failed attempts. markResurrected is also not needed, as it was again a way to make pings required. RestClient can simply pick a dead connection now and use it, no need to change its state when picking the connection.
---
 .../org/elasticsearch/client/Connection.java  | 54 ++++---------------
 .../org/elasticsearch/client/RestClient.java  | 10 ++--
 2 files changed, 15 insertions(+), 49 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/Connection.java b/client/src/main/java/org/elasticsearch/client/Connection.java
index f17ff69e14c..1f604483601 100644
--- a/client/src/main/java/org/elasticsearch/client/Connection.java
+++ b/client/src/main/java/org/elasticsearch/client/Connection.java
@@ -24,16 +24,13 @@ import org.apache.http.HttpHost;
 import java.util.concurrent.TimeUnit;
 
 /**
- * Represents a connection to a host. It holds the host that the connection points to.
- * Allows the transport to deal with very simple connection objects that are immutable.
- * Any change to the state of a connection should be made through the connection pool.
+ * Represents a connection to a host. It holds the host that the connection points to and the state of the connection to it.
  */
 public class Connection {
     private static final long DEFAULT_CONNECTION_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(1);
     private static final long MAX_CONNECTION_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(30);
     private final HttpHost host;
-    private volatile State state = State.UNKNOWN;
-    private volatile int failedAttempts = -1;
+    private volatile int failedAttempts = 0;
     private volatile long deadUntil = -1;
 
     /**
@@ -53,7 +50,7 @@ public class Connection {
     /**
      * Marks connection as dead. Should be called in case the corresponding node is not responding or caused failures.
      * Once marked dead, the number of failed attempts will be incremented on each call to this method. A dead connection
-     * should be retried once {@link #shouldBeRetried()} returns true, which depends on the number of previous failed attempts
+     * should be retried once {@link #isBlacklisted()} returns true, which depends on the number of previous failed attempts
      * and when the last failure was registered.
      */
     void markDead() {
@@ -63,64 +60,35 @@ public class Connection {
                     MAX_CONNECTION_TIMEOUT_MILLIS);
             this.deadUntil = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis);
             this.failedAttempts = ++failedAttempts;
-            this.state = State.DEAD;
         }
     }
 
     /**
      * Marks this connection alive. Should be called when the corresponding node is working properly.
-     * Will reset the number of failed attempts that were counted in case the connection was previously dead,
-     * as well as its dead timeout.
+     * Will reset the number of failed attempts that were counted in case the connection was previously dead, as well as its timeout.
      */
     void markAlive() {
-        if (this.state != State.ALIVE) {
+        if (this.failedAttempts > 0) {
             synchronized (this) {
                 this.deadUntil = -1;
                 this.failedAttempts = 0;
-                this.state = State.ALIVE;
             }
         }
     }
 
     /**
-     * Resets the connection to its initial state, so it will be retried. To be called when all the connections in the pool
-     * are dead, so that one connection can be retried. Note that calling this method only changes the state of the connection,
-     * it doesn't reset its failed attempts and dead until timestamp. That way if the connection goes back to dead straightaway
-     * all of its previous failed attempts are taken into account.
-     */
-    void markResurrected() {
-        if (this.state == State.DEAD) {
-            synchronized (this) {
-                this.state = State.UNKNOWN;
-            }
-        }
-    }
-
-    /**
-     * Returns the timestamp till the connection is supposed to stay dead till it can be retried
+     * Returns the timestamp till the connection is supposed to stay dead. After that moment the connection should be retried
      */
     public long getDeadUntil() {
         return deadUntil;
     }
 
     /**
-     * Returns true if the connection is alive, false otherwise.
+     * Returns true when the connection should be skipped due to previous failures, false in case the connection is alive
+     * or dead but ready to be retried. When the connection is dead, returns false when it is time to retry it, depending
+     * on how many failed attempts were registered and when the last failure happened (minimum 1 minute, maximum 30 minutes).
      */
-    public boolean isAlive() {
-        return state == State.ALIVE;
-    }
-
-    /**
-     * Returns true in case the connection is not alive but should be used/retried, false otherwise.
-     * Returns true in case the connection is in unknown state (never used before) or resurrected. When the connection is dead,
-     * returns true when it is time to retry it, depending on how many failed attempts were registered and when the last failure
-     * happened (minimum 1 minute, maximum 30 minutes).
-     */
-    public boolean shouldBeRetried() {
-        return state == State.UNKNOWN || (state == State.DEAD && System.nanoTime() - deadUntil >= 0);
-    }
-
-    private enum State {
-        UNKNOWN, DEAD, ALIVE
+    public boolean isBlacklisted() {
+        return failedAttempts > 0 && System.nanoTime() - deadUntil < 0;
     }
 }
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index f578d7ce304..030f67c9952 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -148,7 +148,7 @@ public final class RestClient implements Closeable {
      */
     private Iterator<Connection> nextConnection() {
         if (this.connections.isEmpty()) {
-            throw new IllegalStateException("no connections available in the connection pool");
+            throw new IllegalStateException("no connections available");
         }
 
         List<Connection> rotatedConnections = new ArrayList<>(connections);
@@ -157,7 +157,7 @@ public final class RestClient implements Closeable {
         Iterator<Connection> connectionIterator = rotatedConnections.iterator();
         while (connectionIterator.hasNext()) {
             Connection connection = connectionIterator.next();
-            if (connection.isAlive() == false && connection.shouldBeRetried() == false) {
+            if (connection.isBlacklisted()) {
                 connectionIterator.remove();
             }
         }
@@ -170,8 +170,7 @@ public final class RestClient implements Closeable {
                 }
             });
             Connection connection = sortedConnections.get(0);
-            connection.markResurrected();
-            logger.trace("marked connection resurrected for " + connection.getHost());
+            logger.trace("trying to resurrect connection for " + connection.getHost());
             return Collections.singleton(connection).iterator();
         }
         return rotatedConnections.iterator();
@@ -307,8 +306,7 @@ public final class RestClient implements Closeable {
         }
 
         /**
-         * Sets the hosts that the client will send requests to. Mandatory if no connection pool is specified,
-         * as the provided hosts will be used to create the default static connection pool.
+         * Sets the hosts that the client will send requests to.
          */
         public Builder setHosts(HttpHost... hosts) {
             if (hosts == null || hosts.length == 0) {

From c0a72c1686689417374eeea46a19c3b26cf0edda Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 13 May 2016 15:20:21 +0200
Subject: [PATCH 0602/1311] add support for headers: default ones and per
 request

---
 .../org/elasticsearch/client/RestClient.java  | 42 ++++++++++++++++---
 .../elasticsearch/client/sniff/Sniffer.java   |  3 +-
 .../client/RestClientBuilderTests.java        | 34 +++++++++++++--
 3 files changed, 69 insertions(+), 10 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 030f67c9952..e27949a9751 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -20,6 +20,7 @@ package org.elasticsearch.client;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHost;
 import org.apache.http.client.config.RequestConfig;
@@ -40,6 +41,7 @@ import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.Iterator;
@@ -75,10 +77,15 @@ public final class RestClient implements Closeable {
         this.connections = Collections.unmodifiableList(connections);
     }
 
-    public ElasticsearchResponse performRequest(String method, String endpoint, Map<String, Object> params, HttpEntity entity)
-            throws IOException {
+    public ElasticsearchResponse performRequest(String method, String endpoint, Map<String, Object> params,
+                                                HttpEntity entity, Header... headers) throws IOException {
         URI uri = buildUri(endpoint, params);
         HttpRequestBase request = createHttpRequest(method, uri, entity);
+        if (headers.length > 0) {
+            for (Header header : headers) {
+                request.addHeader(header);
+            }
+        }
         //we apply a soft margin so that e.g. if a request took 59 seconds and timeout is set to 60 we don't do another attempt
         long retryTimeout = Math.round(this.maxRetryTimeout / (float)100 * 98);
         IOException lastSeenException = null;
@@ -276,13 +283,15 @@ public final class RestClient implements Closeable {
         private CloseableHttpClient httpClient;
         private int maxRetryTimeout = DEFAULT_MAX_RETRY_TIMEOUT;
         private HttpHost[] hosts;
+        private Collection<? extends Header> defaultHeaders;
 
         private Builder() {
 
         }
 
         /**
-         * Sets the http client. A new default one will be created if not specified, by calling {@link #createDefaultHttpClient()}.
+         * Sets the http client. A new default one will be created if not
+         * specified, by calling {@link #createDefaultHttpClient(Collection)}.
          *
          * @see CloseableHttpClient
          */
@@ -316,12 +325,29 @@ public final class RestClient implements Closeable {
             return this;
         }
 
+        /**
+         * Sets the default request headers, to be used when creating the default http client instance.
+         * In case the http client is set through {@link #setHttpClient(CloseableHttpClient)}, the default headers need to be
+         * set to it externally during http client construction.
+         */
+        public Builder setDefaultHeaders(Collection<? extends Header> defaultHeaders) {
+            this.defaultHeaders = defaultHeaders;
+            return this;
+        }
+
         /**
          * Creates a new {@link RestClient} based on the provided configuration.
          */
         public RestClient build() {
             if (httpClient == null) {
-                httpClient = createDefaultHttpClient();
+                httpClient = createDefaultHttpClient(defaultHeaders);
+            } else {
+                if (defaultHeaders != null) {
+                    throw new IllegalArgumentException("defaultHeaders need to be set to the HttpClient directly when manually provided");
+                }
+            }
+            if (hosts == null || hosts.length == 0) {
+                throw new IllegalArgumentException("no hosts provided");
             }
             return new RestClient(httpClient, maxRetryTimeout, hosts);
         }
@@ -331,7 +357,7 @@ public final class RestClient implements Closeable {
          *
          * @see CloseableHttpClient
          */
-        public static CloseableHttpClient createDefaultHttpClient() {
+        public static CloseableHttpClient createDefaultHttpClient(Collection<? extends Header> defaultHeaders) {
             PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
             //default settings may be too constraining
             connectionManager.setDefaultMaxPerRoute(10);
@@ -342,7 +368,11 @@ public final class RestClient implements Closeable {
                     .setSocketTimeout(DEFAULT_SOCKET_TIMEOUT)
                     .setConnectionRequestTimeout(DEFAULT_CONNECTION_REQUEST_TIMEOUT).build();
 
-            return HttpClientBuilder.create().setConnectionManager(connectionManager).setDefaultRequestConfig(requestConfig).build();
+            HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
+            if (defaultHeaders != null) {
+                httpClientBuilder.setDefaultHeaders(defaultHeaders);
+            }
+            return httpClientBuilder.setConnectionManager(connectionManager).setDefaultRequestConfig(requestConfig).build();
         }
     }
 
diff --git a/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
index 43ad143738a..ea9fc9a26ab 100644
--- a/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
+++ b/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
@@ -28,6 +28,7 @@ import org.elasticsearch.client.RestClient;
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.Executors;
@@ -201,7 +202,7 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
         /**
          * Sets the http client. Mandatory argument. Best practice is to use the same client used
          * within {@link org.elasticsearch.client.RestClient} which can be created manually or
-         * through {@link RestClient.Builder#createDefaultHttpClient()}.
+         * through {@link RestClient.Builder#createDefaultHttpClient(Collection)}.
          * @see CloseableHttpClient
          */
         public Builder setRestClient(RestClient restClient) {
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java b/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
index e0828f2bd1f..44909336dd6 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
@@ -21,10 +21,15 @@ package org.elasticsearch.client;
 
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
 import org.apache.http.HttpHost;
+import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.message.BasicHeader;
 import org.apache.lucene.util.LuceneTestCase;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.logging.LogManager;
 
 public class RestClientBuilderTests extends LuceneTestCase {
@@ -56,12 +61,27 @@ public class RestClientBuilderTests extends LuceneTestCase {
         }
 
         try {
-            RestClient.builder();
+            RestClient.builder().build();
             fail("should have failed");
         } catch(IllegalArgumentException e) {
             assertEquals(e.getMessage(), "no hosts provided");
         }
 
+        try {
+            RestClient.builder().setHosts(new HttpHost[]{new HttpHost("localhost", 9200), null}).build();
+            fail("should have failed");
+        } catch(NullPointerException e) {
+            assertEquals(e.getMessage(), "host cannot be null");
+        }
+
+        try (CloseableHttpClient httpClient = HttpClientBuilder.create().build()) {
+            RestClient.builder().setHttpClient(httpClient)
+                    .setDefaultHeaders(Collections.singleton(new BasicHeader("header", "value"))).build();
+            fail("should have failed");
+        } catch(IllegalArgumentException e) {
+            assertEquals(e.getMessage(), "defaultHeaders need to be set to the HttpClient directly when manually provided");
+        }
+
         RestClient.Builder builder = RestClient.builder();
         int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
         HttpHost[] hosts = new HttpHost[numNodes];
@@ -70,11 +90,19 @@ public class RestClientBuilderTests extends LuceneTestCase {
         }
         builder.setHosts(hosts);
 
-        //TODO test one host is null among others
-
         if (random().nextBoolean()) {
             builder.setHttpClient(HttpClientBuilder.create().build());
+        } else {
+            if (random().nextBoolean()) {
+                int numHeaders = RandomInts.randomIntBetween(random(), 1, 5);
+                Collection<BasicHeader> headers = new ArrayList<>(numHeaders);
+                for (int i = 0; i < numHeaders; i++) {
+                    headers.add(new BasicHeader("header" + i, "value"));
+                }
+                builder.setDefaultHeaders(headers);
+            }
         }
+
         if (random().nextBoolean()) {
             builder.setMaxRetryTimeout(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
         }

From 2d7a781195c8b6436efd5bb3f355e77586dca559 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Tue, 17 May 2016 15:23:47 +0200
Subject: [PATCH 0603/1311] fix usage of deprecated apis

---
 .../java/org/elasticsearch/client/RequestLogger.java   | 10 +++++++++-
 .../org/elasticsearch/client/RequestLoggerTests.java   |  6 +++---
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/RequestLogger.java b/client/src/main/java/org/elasticsearch/client/RequestLogger.java
index ce2e82ea1e5..dd9fb269717 100644
--- a/client/src/main/java/org/elasticsearch/client/RequestLogger.java
+++ b/client/src/main/java/org/elasticsearch/client/RequestLogger.java
@@ -28,11 +28,14 @@ import org.apache.http.HttpHost;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.http.entity.BufferedHttpEntity;
+import org.apache.http.entity.ContentType;
 import org.apache.http.util.EntityUtils;
 
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 
 /**
  * Helper class that exposes static methods to unify the way requests are logged.
@@ -118,7 +121,12 @@ public final class RequestLogger {
         if (entity != null) {
             entity = new BufferedHttpEntity(entity);
             httpResponse.setEntity(entity);
-            try (BufferedReader reader = new BufferedReader(new InputStreamReader(entity.getContent()))) {
+            ContentType contentType = ContentType.get(entity);
+            Charset charset = StandardCharsets.UTF_8;
+            if (contentType != null) {
+                charset = contentType.getCharset();
+            }
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(entity.getContent(), charset))) {
                 String line;
                 while( (line = reader.readLine()) != null) {
                     responseLine += "\n# " + line;
diff --git a/client/src/test/java/org/elasticsearch/client/RequestLoggerTests.java b/client/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
index 84bc3c032d5..89fa30c7e08 100644
--- a/client/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.client;
 
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
-import org.apache.commons.codec.Charsets;
 import org.apache.http.HttpEntityEnclosingRequest;
 import org.apache.http.HttpHost;
 import org.apache.http.ProtocolVersion;
@@ -36,6 +35,7 @@ import org.apache.lucene.util.LuceneTestCase;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 
@@ -72,7 +72,7 @@ public class RequestLoggerTests extends LuceneTestCase {
         if (request instanceof HttpEntityEnclosingRequest && random().nextBoolean()) {
             HttpEntityEnclosingRequest enclosingRequest = (HttpEntityEnclosingRequest) request;
             String requestBody = "{ \"field\": \"value\" }";
-            enclosingRequest.setEntity(new StringEntity(requestBody, Charsets.UTF_8));
+            enclosingRequest.setEntity(new StringEntity(requestBody, StandardCharsets.UTF_8));
             expected += " -d '" + requestBody + "'";
         }
 
@@ -95,7 +95,7 @@ public class RequestLoggerTests extends LuceneTestCase {
         expected += "\n#";
         if (random().nextBoolean()) {
             String responseBody = "{\n  \"field\": \"value\"\n}";
-            httpResponse.setEntity(new StringEntity(responseBody, Charsets.UTF_8));
+            httpResponse.setEntity(new StringEntity(responseBody, StandardCharsets.UTF_8));
             expected += "\n# {";
             expected += "\n#   \"field\": \"value\"";
             expected += "\n# }";

From 2735897b36226f36723dc1a4bf27e53d76b70fdc Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Tue, 17 May 2016 15:24:26 +0200
Subject: [PATCH 0604/1311] use versions  from versions.properties

---
 client/build.gradle | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/client/build.gradle b/client/build.gradle
index 35af3d0979b..e24aa9878e3 100644
--- a/client/build.gradle
+++ b/client/build.gradle
@@ -20,22 +20,18 @@
 import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.gradle.api.JavaVersion
 
-group = 'org.elasticsearch.client'
 apply plugin: 'elasticsearch.build'
 
 targetCompatibility = JavaVersion.VERSION_1_7
 sourceCompatibility = JavaVersion.VERSION_1_7
 
 dependencies {
-  // TODO once we got rid of the client in the test framework we should use a version variable here
-  compile "org.apache.httpcomponents:httpclient:4.5.2"
-  compile "org.apache.httpcomponents:httpcore:4.4.4"
-  //compile "org.apache.httpcomponents:httpcore-nio:4.4.4"
-  //compile "org.apache.httpcomponents:httpasyncclient:4.1.1"
-  compile "commons-codec:commons-codec:1.9"
-  compile "commons-logging:commons-logging:1.2"
+  compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
+  compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
+  compile "commons-codec:commons-codec:${versions.commonscodec}"
+  compile "commons-logging:commons-logging:${versions.commonslogging}"
   //jackson is only needed in the sniff package
-  compile "com.fasterxml.jackson.core:jackson-core:2.7.3"
+  compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
 
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"

From 3890842fd4cacd9085dee3c8544c57fea5f06b24 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Tue, 17 May 2016 15:25:10 +0200
Subject: [PATCH 0605/1311] add project substitution for
 org.elasticsearch:client so that we can add it as a dep in other projects

---
 build.gradle | 1 +
 1 file changed, 1 insertion(+)

diff --git a/build.gradle b/build.gradle
index 5c6ec366930..af18424944b 100644
--- a/build.gradle
+++ b/build.gradle
@@ -162,6 +162,7 @@ subprojects {
   ext.projectSubstitutions = [
     "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec',
     "org.elasticsearch:elasticsearch:${version}": ':core',
+    "org.elasticsearch:client:${version}": ':client',
     "org.elasticsearch.test:framework:${version}": ':test:framework',
     "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:integ-test-zip',
     "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:zip',

From 1d06916b07b1f6a59e57b4d3d1ded9ce426bfefe Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Wed, 18 May 2016 11:39:59 +0200
Subject: [PATCH 0606/1311] adapt params argument, they can only be Strings in
 performRequest method

---
 .../main/java/org/elasticsearch/client/RestClient.java    | 8 ++++----
 .../java/org/elasticsearch/client/sniff/HostsSniffer.java | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index e27949a9751..23540049fb5 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -77,7 +77,7 @@ public final class RestClient implements Closeable {
         this.connections = Collections.unmodifiableList(connections);
     }
 
-    public ElasticsearchResponse performRequest(String method, String endpoint, Map<String, Object> params,
+    public ElasticsearchResponse performRequest(String method, String endpoint, Map<String, String> params,
                                                 HttpEntity entity, Header... headers) throws IOException {
         URI uri = buildUri(endpoint, params);
         HttpRequestBase request = createHttpRequest(method, uri, entity);
@@ -252,11 +252,11 @@ public final class RestClient implements Closeable {
         }
     }
 
-    private static URI buildUri(String path, Map<String, Object> params) {
+    private static URI buildUri(String path, Map<String, String> params) {
         try {
             URIBuilder uriBuilder = new URIBuilder(path);
-            for (Map.Entry<String, Object> param : params.entrySet()) {
-                uriBuilder.addParameter(param.getKey(), param.getValue().toString());
+            for (Map.Entry<String, String> param : params.entrySet()) {
+                uriBuilder.addParameter(param.getKey(), param.getValue());
             }
             return uriBuilder.build();
         } catch(URISyntaxException e) {
diff --git a/client/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java b/client/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
index 29117fcd856..86dca173a67 100644
--- a/client/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
+++ b/client/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
@@ -45,13 +45,13 @@ public class HostsSniffer {
     private static final Log logger = LogFactory.getLog(HostsSniffer.class);
 
     private final RestClient restClient;
-    private final Map<String, Object> sniffRequestParams;
+    private final Map<String, String> sniffRequestParams;
     private final String scheme;
     private final JsonFactory jsonFactory;
 
     public HostsSniffer(RestClient restClient, int sniffRequestTimeout, String scheme) {
         this.restClient = restClient;
-        this.sniffRequestParams = Collections.<String, Object>singletonMap("timeout", sniffRequestTimeout + "ms");
+        this.sniffRequestParams = Collections.<String, String>singletonMap("timeout", sniffRequestTimeout + "ms");
         this.scheme = scheme;
         this.jsonFactory = new JsonFactory();
     }

From 16ab491016368c04cb70bb9e5858f6a35280f380 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Wed, 18 May 2016 11:40:20 +0200
Subject: [PATCH 0607/1311] add getFirstHeader method to ElasticsearchResponse

---
 .../elasticsearch/client/ElasticsearchResponse.java | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponse.java b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponse.java
index 124a233dea7..304d10e2b94 100644
--- a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponse.java
+++ b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponse.java
@@ -77,6 +77,19 @@ public class ElasticsearchResponse implements Closeable {
         return response.getAllHeaders();
     }
 
+    /**
+     * Returns the value of the first header with a specified name of this message.
+     * If there is more than one matching header in the message the first element is returned.
+     * If there is no matching header in the message <code>null</code> is returned.
+     */
+    public String getFirstHeader(String name) {
+        Header header = response.getFirstHeader(name);
+        if (header == null) {
+            return null;
+        }
+        return header.getValue();
+    }
+
     /**
      * Returns the response body available, null otherwise
      * @see HttpEntity

From 9a38d81bec9549a7505c03001c2d9e8251e3edab Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Wed, 18 May 2016 13:51:38 +0200
Subject: [PATCH 0608/1311] Expose whole ElasticsearchResponse within
 ElasticsearchResponseException

The only small problem is that the response gets closed straightaway and its body read immediately into a string. Should be ok to load it all into memory eagerly though in case of errors. Otherwise it becomes cumbersome to have an exception implement Closeable...
---
 .../ElasticsearchResponseException.java       | 41 +++++++++----------
 .../org/elasticsearch/client/RestClient.java  | 12 ++----
 .../client/sniff/HostsSnifferTests.java       | 11 +++--
 3 files changed, 31 insertions(+), 33 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
index 98945b9a726..ad63009db34 100644
--- a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
+++ b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
@@ -22,6 +22,7 @@ package org.elasticsearch.client;
 import org.apache.http.HttpHost;
 import org.apache.http.RequestLine;
 import org.apache.http.StatusLine;
+import org.apache.http.util.EntityUtils;
 
 import java.io.IOException;
 
@@ -30,17 +31,21 @@ import java.io.IOException;
  */
 public class ElasticsearchResponseException extends IOException {
 
-    private final HttpHost host;
-    private final RequestLine requestLine;
-    private final StatusLine statusLine;
+    private ElasticsearchResponse elasticsearchResponse;
     private final String responseBody;
 
-    public ElasticsearchResponseException(RequestLine requestLine, HttpHost host, StatusLine statusLine, String responseBody) {
-        super(buildMessage(requestLine, host, statusLine));
-        this.host = host;
-        this.requestLine = requestLine;
-        this.responseBody = responseBody;
-        this.statusLine = statusLine;
+    public ElasticsearchResponseException(ElasticsearchResponse elasticsearchResponse) throws IOException {
+        super(buildMessage(elasticsearchResponse.getRequestLine(), elasticsearchResponse.getHost(), elasticsearchResponse.getStatusLine()));
+        this.elasticsearchResponse = elasticsearchResponse;
+        try {
+            if (elasticsearchResponse.getEntity() == null) {
+                this.responseBody = null;
+            } else {
+                this.responseBody = EntityUtils.toString(elasticsearchResponse.getEntity());
+            }
+        } finally {
+            elasticsearchResponse.close();
+        }
     }
 
     private static String buildMessage(RequestLine requestLine, HttpHost host, StatusLine statusLine) {
@@ -48,23 +53,17 @@ public class ElasticsearchResponseException extends IOException {
     }
 
     /**
-     * Returns the {@link HttpHost} that returned the error
+     * Returns the {@link ElasticsearchResponse} that caused this exception to be thrown
      */
-    public HttpHost getHost() {
-        return host;
+    public ElasticsearchResponse getElasticsearchResponse() {
+        return elasticsearchResponse;
     }
 
     /**
-     * Returns the {@link RequestLine} that triggered the error
+     * Returns the response body as a string or null if there wasn't any.
+     * The body is eagerly consumed when an ElasticsearchResponseException gets created, and its corresponding ElasticsearchResponse
+     * gets closed straightaway so this method is the only way to get back the response body that was returned.
      */
-    public RequestLine getRequestLine() {
-        return requestLine;
-    }
-
-    public StatusLine getStatusLine() {
-        return statusLine;
-    }
-
     public String getResponseBody() {
         return responseBody;
     }
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 23540049fb5..4e414033d4a 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -34,7 +34,6 @@ import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
-import org.apache.http.util.EntityUtils;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -115,20 +114,17 @@ public final class RestClient implements Closeable {
                 lastSeenException = addSuppressedException(lastSeenException, e);
                 continue;
             }
+            ElasticsearchResponse elasticsearchResponse = new ElasticsearchResponse(request.getRequestLine(),
+                    connection.getHost(), response);
             int statusCode = response.getStatusLine().getStatusCode();
             //TODO make ignore status code configurable. rest-spec and tests support that parameter (ignore_missing)
             if (statusCode < 300 || (request.getMethod().equals(HttpHead.METHOD_NAME) && statusCode == 404) ) {
                 RequestLogger.log(logger, "request succeeded", request, connection.getHost(), response);
                 onSuccess(connection);
-                return new ElasticsearchResponse(request.getRequestLine(), connection.getHost(), response);
+                return elasticsearchResponse;
             } else {
                 RequestLogger.log(logger, "request failed", request, connection.getHost(), response);
-                String responseBody = null;
-                if (response.getEntity() != null) {
-                    responseBody = EntityUtils.toString(response.getEntity());
-                }
-                ElasticsearchResponseException elasticsearchResponseException = new ElasticsearchResponseException(
-                        request.getRequestLine(), connection.getHost(), response.getStatusLine(), responseBody);
+                ElasticsearchResponseException elasticsearchResponseException = new ElasticsearchResponseException(elasticsearchResponse);
                 lastSeenException = addSuppressedException(lastSeenException, elasticsearchResponseException);
                 //clients don't retry on 500 because elasticsearch still misuses it instead of 400 in some places
                 if (statusCode == 502 || statusCode == 503 || statusCode == 504) {
diff --git a/client/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java b/client/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
index c9e1b48a791..e2b7a6d7c10 100644
--- a/client/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
+++ b/client/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
@@ -30,6 +30,7 @@ import okhttp3.mockwebserver.MockWebServer;
 import okhttp3.mockwebserver.RecordedRequest;
 import org.apache.http.HttpHost;
 import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.client.ElasticsearchResponse;
 import org.elasticsearch.client.ElasticsearchResponseException;
 import org.elasticsearch.client.RestClient;
 import org.junit.After;
@@ -100,15 +101,17 @@ public class HostsSnifferTests extends LuceneTestCase {
                     assertEquals(sniffedHost, responseHostsIterator.next());
                 }
             } catch(ElasticsearchResponseException e) {
+                ElasticsearchResponse response = e.getElasticsearchResponse();
                 if (sniffResponse.isFailure) {
                     assertThat(e.getMessage(), containsString("GET http://localhost:" + server.getPort() +
                             "/_nodes/http?timeout=" + sniffRequestTimeout));
                     assertThat(e.getMessage(), containsString(Integer.toString(sniffResponse.nodesInfoResponseCode)));
-                    assertThat(e.getHost(), equalTo(httpHost));
-                    assertThat(e.getStatusLine().getStatusCode(), equalTo(sniffResponse.nodesInfoResponseCode));
-                    assertThat(e.getRequestLine().toString(), equalTo("GET /_nodes/http?timeout=" + sniffRequestTimeout + "ms HTTP/1.1"));
+                    assertThat(response.getHost(), equalTo(httpHost));
+                    assertThat(response.getStatusLine().getStatusCode(), equalTo(sniffResponse.nodesInfoResponseCode));
+                    assertThat(response.getRequestLine().toString(),
+                            equalTo("GET /_nodes/http?timeout=" + sniffRequestTimeout + "ms HTTP/1.1"));
                 } else {
-                    fail("sniffNodes should have succeeded: " + e.getStatusLine());
+                    fail("sniffNodes should have succeeded: " + response.getStatusLine());
                 }
             }
         }

From 6d3f6c7fafd5b5df98c142e04a85d98a434bee37 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Wed, 18 May 2016 13:52:33 +0200
Subject: [PATCH 0609/1311] support missing OPTIONS method, it is supported in
 elasticsearch core

---
 .../src/main/java/org/elasticsearch/client/RestClient.java  | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 4e414033d4a..6c37fd58ae1 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -27,6 +27,7 @@ import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
 import org.apache.http.client.methods.HttpHead;
+import org.apache.http.client.methods.HttpOptions;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpRequestBase;
@@ -229,6 +230,11 @@ public final class RestClient implements Closeable {
                     throw new UnsupportedOperationException("HEAD with body is not supported");
                 }
                 return new HttpHead(uri);
+            case HttpOptions.METHOD_NAME:
+                if (entity != null) {
+                    throw new UnsupportedOperationException("OPTIONS with body is not supported");
+                }
+                return new HttpOptions(uri);
             case HttpPost.METHOD_NAME:
                 HttpPost httpPost = new HttpPost(uri);
                 addRequestBody(httpPost, entity);

From 325b72393077e7985975515a8c9a8b29c468da16 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Wed, 18 May 2016 15:49:28 +0200
Subject: [PATCH 0610/1311] [TEST] add rest client test dependency and replace
 usage of HttpRequestBuilder with RestClient in integration tests

---
 .../resources/checkstyle_suppressions.xml     |   1 -
 .../org/elasticsearch/client/RestClient.java  |   3 +
 .../http/netty/NettyHttpCompressionIT.java    |  82 ++++++--------
 .../DetailedErrorsDisabledIT.java             |  29 +++--
 .../DetailedErrorsEnabledIT.java              |  49 +++++----
 .../plugins/ResponseHeaderPluginIT.java       |  26 +++--
 .../org/elasticsearch/rest/CorsNotSetIT.java  |  32 +++---
 .../org/elasticsearch/rest/CorsRegexIT.java   | 103 ++++++++++--------
 .../rest/action/main/RestMainActionIT.java    |  23 ++--
 .../ContextAndHeaderTransportIT.java          |  40 +++----
 test/framework/build.gradle                   |   5 +-
 .../elasticsearch/test/ESIntegTestCase.java   |  34 ++++--
 .../hamcrest/ElasticsearchAssertions.java     |   8 +-
 .../test/hamcrest/ElasticsearchMatchers.java  |  16 +--
 14 files changed, 242 insertions(+), 209 deletions(-)

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index c77ef0f9d5b..d6a38e76e3e 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -1049,7 +1049,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]JvmGcMonitorServiceSettingsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]os[/\\]OsProbeTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]nodesinfo[/\\]NodeInfoStreamingTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]options[/\\]detailederrors[/\\]DetailedErrorsEnabledIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]PluginInfoTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugins[/\\]PluginsServiceTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]recovery[/\\]FullRollingRestartIT.java" checks="LineLength" />
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 6c37fd58ae1..31914f46282 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -20,6 +20,7 @@ package org.elasticsearch.client;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.http.Consts;
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHost;
@@ -32,6 +33,7 @@ import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.entity.ContentType;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
@@ -55,6 +57,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 public final class RestClient implements Closeable {
 
     private static final Log logger = LogFactory.getLog(RestClient.class);
+    public static ContentType JSON_CONTENT_TYPE = ContentType.create("application/json", Consts.UTF_8);
 
     private final CloseableHttpClient client;
     private final long maxRetryTimeout;
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java
index bb7a78ab43f..9e995f0a010 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java
@@ -22,27 +22,31 @@ import org.apache.http.Header;
 import org.apache.http.HttpException;
 import org.apache.http.HttpHeaders;
 import org.apache.http.HttpResponseInterceptor;
+import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicHeader;
 import org.apache.http.protocol.HttpContext;
+import org.elasticsearch.client.ElasticsearchResponse;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.http.HttpTransportSettings;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 
 import java.io.IOException;
+import java.util.Collections;
 
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 1, numClientNodes = 1)
 public class NettyHttpCompressionIT extends ESIntegTestCase {
     private static final String GZIP_ENCODING = "gzip";
 
-    private static final String SAMPLE_DOCUMENT = "{\n" +
+    private static final StringEntity SAMPLE_DOCUMENT = new StringEntity("{\n" +
         "   \"name\": {\n" +
         "      \"first name\": \"Steve\",\n" +
         "      \"last name\": \"Jobs\"\n" +
         "   }\n" +
-        "}";
+        "}", RestClient.JSON_CONTENT_TYPE);
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
@@ -55,69 +59,49 @@ public class NettyHttpCompressionIT extends ESIntegTestCase {
 
     public void testCompressesResponseIfRequested() throws Exception {
         ensureGreen();
-
         // we need to intercept early, otherwise internal logic in HttpClient will just remove the header and we cannot verify it
         ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor();
-        CloseableHttpClient internalClient = HttpClients.custom().addInterceptorFirst(headerExtractor).build();
-
-        HttpResponse response = httpClient(internalClient).path("/").addHeader(HttpHeaders.ACCEPT_ENCODING, GZIP_ENCODING).execute();
-        assertEquals(200, response.getStatusCode());
-        assertTrue(headerExtractor.hasContentEncodingHeader());
-        assertEquals(GZIP_ENCODING, headerExtractor.getContentEncodingHeader().getValue());
+        try (RestClient client = restClient(HttpClients.custom().addInterceptorFirst(headerExtractor).build())) {
+            ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null,
+                    new BasicHeader(HttpHeaders.ACCEPT_ENCODING, GZIP_ENCODING));
+            assertEquals(200, response.getStatusLine().getStatusCode());
+            assertTrue(headerExtractor.hasContentEncodingHeader());
+            assertEquals(GZIP_ENCODING, headerExtractor.getContentEncodingHeader().getValue());
+        }
     }
 
     public void testUncompressedResponseByDefault() throws Exception {
         ensureGreen();
-
         ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor();
-        CloseableHttpClient internalClient = HttpClients
-            .custom()
-            .disableContentCompression()
-            .addInterceptorFirst(headerExtractor)
-            .build();
-
-        HttpResponse response = httpClient(internalClient).path("/").execute();
-        assertEquals(200, response.getStatusCode());
-        assertFalse(headerExtractor.hasContentEncodingHeader());
+        CloseableHttpClient httpClient = HttpClients.custom().disableContentCompression().addInterceptorFirst(headerExtractor).build();
+        try (RestClient client = restClient(httpClient)) {
+            ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null);
+            assertEquals(200, response.getStatusLine().getStatusCode());
+            assertFalse(headerExtractor.hasContentEncodingHeader());
+        }
     }
 
     public void testCanInterpretUncompressedRequest() throws Exception {
         ensureGreen();
-
         ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor();
-        CloseableHttpClient internalClient = HttpClients
-            .custom()
-            // this disable content compression in both directions (request and response)
-            .disableContentCompression()
-            .addInterceptorFirst(headerExtractor)
-            .build();
-
-        HttpResponse response = httpClient(internalClient)
-            .path("/company/employees/1")
-            .method("POST")
-            .body(SAMPLE_DOCUMENT)
-            .execute();
-
-        assertEquals(201, response.getStatusCode());
-        assertFalse(headerExtractor.hasContentEncodingHeader());
+        // this disable content compression in both directions (request and response)
+        CloseableHttpClient httpClient = HttpClients.custom().disableContentCompression().addInterceptorFirst(headerExtractor).build();
+        try (RestClient client = restClient(httpClient)) {
+            ElasticsearchResponse response = client.performRequest("POST", "/company/employees/1", Collections.emptyMap(), SAMPLE_DOCUMENT);
+            assertEquals(201, response.getStatusLine().getStatusCode());
+            assertFalse(headerExtractor.hasContentEncodingHeader());
+        }
     }
 
     public void testCanInterpretCompressedRequest() throws Exception {
         ensureGreen();
-
         ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor();
         // we don't call #disableContentCompression() hence the client will send the content compressed
-        CloseableHttpClient internalClient = HttpClients.custom().addInterceptorFirst(headerExtractor).build();
-
-        HttpResponse response = httpClient(internalClient)
-            .path("/company/employees/2")
-            .method("POST")
-            .body(SAMPLE_DOCUMENT)
-            .execute();
-
-        assertEquals(201, response.getStatusCode());
-        assertTrue(headerExtractor.hasContentEncodingHeader());
-        assertEquals(GZIP_ENCODING, headerExtractor.getContentEncodingHeader().getValue());
+        try (RestClient client = restClient(HttpClients.custom().addInterceptorFirst(headerExtractor).build())) {
+            ElasticsearchResponse response = client.performRequest("POST", "/company/employees/2", Collections.emptyMap(), SAMPLE_DOCUMENT);
+            assertEquals(201, response.getStatusLine().getStatusCode());
+            assertEquals(GZIP_ENCODING, headerExtractor.getContentEncodingHeader().getValue());
+        }
     }
 
     private static class ContentEncodingHeaderExtractor implements HttpResponseInterceptor {
@@ -141,6 +125,4 @@ public class NettyHttpCompressionIT extends ESIntegTestCase {
             return contentEncodingHeader;
         }
     }
-
-
 }
diff --git a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java
index 683ae71a112..ba8840bbc2c 100644
--- a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java
+++ b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java
@@ -19,17 +19,17 @@
 
 package org.elasticsearch.options.detailederrors;
 
-import org.apache.http.impl.client.HttpClients;
+import org.elasticsearch.client.ElasticsearchResponse;
+import org.elasticsearch.client.ElasticsearchResponseException;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.HttpTransportSettings;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
-import org.elasticsearch.test.rest.client.http.HttpDeleteWithEntity;
-import org.elasticsearch.test.rest.client.http.HttpRequestBuilder;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
+
+import java.util.Collections;
 
 import static org.hamcrest.Matchers.is;
 
@@ -49,15 +49,14 @@ public class DetailedErrorsDisabledIT extends ESIntegTestCase {
     }
 
     public void testThatErrorTraceParamReturns400() throws Exception {
-        // Make the HTTP request
-        HttpResponse response = new HttpRequestBuilder(HttpClients.createDefault())
-                .httpTransport(internalCluster().getDataNodeInstance(HttpServerTransport.class))
-                .addParam("error_trace", "true")
-                .method(HttpDeleteWithEntity.METHOD_NAME)
-                .execute();
-
-        assertThat(response.getHeaders().get("Content-Type"), is("application/json"));
-        assertThat(response.getBody(), is("{\"error\":\"error traces in responses are disabled.\"}"));
-        assertThat(response.getStatusCode(), is(400));
+        try (RestClient restClient = restClient()) {
+            restClient.performRequest("DELETE", "/", Collections.singletonMap("error_trace", "true"), null);
+            fail("request should have failed");
+        } catch(ElasticsearchResponseException e) {
+            ElasticsearchResponse response = e.getElasticsearchResponse();
+            assertThat(response.getFirstHeader("Content-Type"), is("application/json"));
+            assertThat(e.getResponseBody(), is("{\"error\":\"error traces in responses are disabled.\"}"));
+            assertThat(response.getStatusLine().getStatusCode(), is(400));
+        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java
index d98db83dddc..a040c31299d 100644
--- a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java
+++ b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java
@@ -19,16 +19,16 @@
 
 package org.elasticsearch.options.detailederrors;
 
-import org.apache.http.impl.client.HttpClients;
+import org.elasticsearch.client.ElasticsearchResponse;
+import org.elasticsearch.client.ElasticsearchResponseException;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
-import org.elasticsearch.test.rest.client.http.HttpDeleteWithEntity;
-import org.elasticsearch.test.rest.client.http.HttpRequestBuilder;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
+
+import java.util.Collections;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.not;
@@ -47,25 +47,26 @@ public class DetailedErrorsEnabledIT extends ESIntegTestCase {
     }
 
     public void testThatErrorTraceWorksByDefault() throws Exception {
-        // Make the HTTP request
-        HttpResponse response = new HttpRequestBuilder(HttpClients.createDefault())
-                .httpTransport(internalCluster().getDataNodeInstance(HttpServerTransport.class))
-                .path("/")
-                .addParam("error_trace", "true")
-                .method(HttpDeleteWithEntity.METHOD_NAME)
-                .execute();
+        try (RestClient restClient = restClient()) {
+            try {
+                restClient.performRequest("DELETE", "/", Collections.singletonMap("error_trace", "true"), null);
+                fail("request should have failed");
+            } catch(ElasticsearchResponseException e) {
+                ElasticsearchResponse response = e.getElasticsearchResponse();
+                assertThat(response.getFirstHeader("Content-Type"), containsString("application/json"));
+                assertThat(e.getResponseBody(), containsString("\"stack_trace\":\"[Validation Failed: 1: index / indices is missing;]; " +
+                        "nested: ActionRequestValidationException[Validation Failed: 1:"));
+            }
 
-        assertThat(response.getHeaders().get("Content-Type"), containsString("application/json"));
-        assertThat(response.getBody(), containsString("\"stack_trace\":\"[Validation Failed: 1: index / indices is missing;]; nested: ActionRequestValidationException[Validation Failed: 1:"));
-
-        // Make the HTTP request
-        response = new HttpRequestBuilder(HttpClients.createDefault())
-                .httpTransport(internalCluster().getDataNodeInstance(HttpServerTransport.class))
-                .path("/")
-                .method(HttpDeleteWithEntity.METHOD_NAME)
-                .execute();
-
-        assertThat(response.getHeaders().get("Content-Type"), containsString("application/json"));
-        assertThat(response.getBody(), not(containsString("\"stack_trace\":\"[Validation Failed: 1: index / indices is missing;]; nested: ActionRequestValidationException[Validation Failed: 1:")));
+            try {
+                restClient.performRequest("DELETE", "/", Collections.emptyMap(), null);
+                fail("request should have failed");
+            } catch(ElasticsearchResponseException e) {
+                ElasticsearchResponse response = e.getElasticsearchResponse();
+                assertThat(response.getFirstHeader("Content-Type"), containsString("application/json"));
+                assertThat(e.getResponseBody(), not(containsString("\"stack_trace\":\"[Validation Failed: 1: index / indices is missing;]; "
+                        + "nested: ActionRequestValidationException[Validation Failed: 1:")));
+            }
+        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java b/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
index ac6d8fddd89..4ae9e3912b2 100644
--- a/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
+++ b/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
@@ -18,14 +18,18 @@
  */
 package org.elasticsearch.plugins;
 
+import org.apache.http.message.BasicHeader;
+import org.elasticsearch.client.ElasticsearchResponse;
+import org.elasticsearch.client.ElasticsearchResponseException;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.responseheader.TestResponseHeaderPlugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 
 import java.util.Collection;
+import java.util.Collections;
 
 import static org.elasticsearch.rest.RestStatus.OK;
 import static org.elasticsearch.rest.RestStatus.UNAUTHORIZED;
@@ -52,12 +56,20 @@ public class ResponseHeaderPluginIT extends ESIntegTestCase {
 
     public void testThatSettingHeadersWorks() throws Exception {
         ensureGreen();
-        HttpResponse response = httpClient().method("GET").path("/_protected").execute();
-        assertThat(response, hasStatus(UNAUTHORIZED));
-        assertThat(response.getHeaders().get("Secret"), equalTo("required"));
+        try (RestClient client = restClient()) {
+            try {
+                client.performRequest("GET", "/_protected", Collections.emptyMap(), null);
+                fail("request should have failed");
+            } catch(ElasticsearchResponseException e) {
+                ElasticsearchResponse response = e.getElasticsearchResponse();
+                assertThat(response, hasStatus(UNAUTHORIZED));
+                assertThat(response.getFirstHeader("Secret"), equalTo("required"));
+            }
 
-        HttpResponse authResponse = httpClient().method("GET").path("/_protected").addHeader("Secret", "password").execute();
-        assertThat(authResponse, hasStatus(OK));
-        assertThat(authResponse.getHeaders().get("Secret"), equalTo("granted"));
+            ElasticsearchResponse authResponse = client.performRequest("GET", "/_protected", Collections.emptyMap(), null,
+                    new BasicHeader("Secret", "password"));
+            assertThat(authResponse, hasStatus(OK));
+            assertThat(authResponse.getFirstHeader("Secret"), equalTo("granted"));
+        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java b/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java
index dfdd88d1987..e407bccf872 100644
--- a/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java
+++ b/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java
@@ -19,15 +19,18 @@
 
 package org.elasticsearch.rest;
 
+import org.apache.http.message.BasicHeader;
+import org.elasticsearch.client.ElasticsearchResponse;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 
-import static org.hamcrest.Matchers.hasKey;
+import java.util.Collections;
+
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
 
 /**
  *
@@ -44,18 +47,21 @@ public class CorsNotSetIT extends ESIntegTestCase {
 
     public void testCorsSettingDefaultBehaviourDoesNotReturnAnything() throws Exception {
         String corsValue = "http://localhost:9200";
-        HttpResponse response = httpClient().method("GET").path("/").addHeader("User-Agent", "Mozilla Bar").addHeader("Origin", corsValue).execute();
-
-        assertThat(response.getStatusCode(), is(200));
-        assertThat(response.getHeaders(), not(hasKey("Access-Control-Allow-Origin")));
-        assertThat(response.getHeaders(), not(hasKey("Access-Control-Allow-Credentials")));
+        try (RestClient restClient = restClient()) {
+            ElasticsearchResponse response = restClient.performRequest("GET", "/", Collections.emptyMap(), null,
+                    new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue));
+            assertThat(response.getStatusLine().getStatusCode(), is(200));
+            assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
+            assertThat(response.getFirstHeader("Access-Control-Allow-Credentials"), nullValue());
+        }
     }
 
     public void testThatOmittingCorsHeaderDoesNotReturnAnything() throws Exception {
-        HttpResponse response = httpClient().method("GET").path("/").execute();
-
-        assertThat(response.getStatusCode(), is(200));
-        assertThat(response.getHeaders(), not(hasKey("Access-Control-Allow-Origin")));
-        assertThat(response.getHeaders(), not(hasKey("Access-Control-Allow-Credentials")));
+        try (RestClient restClient = restClient()) {
+            ElasticsearchResponse response = restClient.performRequest("GET", "/", Collections.emptyMap(), null);
+            assertThat(response.getStatusLine().getStatusCode(), is(200));
+            assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
+            assertThat(response.getFirstHeader("Access-Control-Allow-Credentials"), nullValue());
+        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java b/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java
index 18351dcb294..729fed8e96d 100644
--- a/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java
+++ b/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java
@@ -18,6 +18,10 @@
  */
 package org.elasticsearch.rest;
 
+import org.apache.http.message.BasicHeader;
+import org.elasticsearch.client.ElasticsearchResponse;
+import org.elasticsearch.client.ElasticsearchResponseException;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkModule;
@@ -25,16 +29,16 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 import org.jboss.netty.handler.codec.http.HttpHeaders;
 
+import java.util.Collections;
+
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED;
-import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
 
 /**
  * Test CORS where the allow origin value is a regular expression.
@@ -58,64 +62,77 @@ public class CorsRegexIT extends ESIntegTestCase {
 
     public void testThatRegularExpressionWorksOnMatch() throws Exception {
         String corsValue = "http://localhost:9200";
-        HttpResponse response = httpClient().method("GET").path("/").addHeader("User-Agent", "Mozilla Bar").addHeader("Origin", corsValue).execute();
-        assertResponseWithOriginheader(response, corsValue);
+        try (RestClient client = restClient()) {
+            ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null,
+                    new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue));
+            assertResponseWithOriginheader(response, corsValue);
 
-        corsValue = "https://localhost:9200";
-        response = httpClient().method("GET").path("/").addHeader("User-Agent", "Mozilla Bar").addHeader("Origin", corsValue).execute();
-        assertResponseWithOriginheader(response, corsValue);
-        assertThat(response.getHeaders(), hasKey("Access-Control-Allow-Credentials"));
-        assertThat(response.getHeaders().get("Access-Control-Allow-Credentials"), is("true"));
+            corsValue = "https://localhost:9200";
+            response = client.performRequest("GET", "/", Collections.emptyMap(), null,
+                    new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue));
+            assertResponseWithOriginheader(response, corsValue);
+            assertThat(response.getFirstHeader("Access-Control-Allow-Credentials"), is("true"));
+        }
     }
 
     public void testThatRegularExpressionReturnsForbiddenOnNonMatch() throws Exception {
-        HttpResponse response = httpClient().method("GET").path("/").addHeader("User-Agent", "Mozilla Bar").addHeader("Origin", "http://evil-host:9200").execute();
-        // a rejected origin gets a FORBIDDEN - 403
-        assertThat(response.getStatusCode(), is(403));
-        assertThat(response.getHeaders(), not(hasKey("Access-Control-Allow-Origin")));
+        try (RestClient client = restClient()) {
+            client.performRequest("GET", "/", Collections.emptyMap(), null, new BasicHeader("User-Agent", "Mozilla Bar"),
+                    new BasicHeader("Origin", "http://evil-host:9200"));
+            fail("request should have failed");
+        } catch(ElasticsearchResponseException e) {
+            ElasticsearchResponse response = e.getElasticsearchResponse();
+            // a rejected origin gets a FORBIDDEN - 403
+            assertThat(response.getStatusLine().getStatusCode(), is(403));
+            assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
+        }
     }
 
     public void testThatSendingNoOriginHeaderReturnsNoAccessControlHeader() throws Exception {
-        HttpResponse response = httpClient().method("GET").path("/").addHeader("User-Agent", "Mozilla Bar").execute();
-        assertThat(response.getStatusCode(), is(200));
-        assertThat(response.getHeaders(), not(hasKey("Access-Control-Allow-Origin")));
+        try (RestClient client = restClient()) {
+            ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null,
+                    new BasicHeader("User-Agent", "Mozilla Bar"));
+            assertThat(response.getStatusLine().getStatusCode(), is(200));
+            assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
+        }
     }
 
     public void testThatRegularExpressionIsNotAppliedWithoutCorrectBrowserOnMatch() throws Exception {
-        HttpResponse response = httpClient().method("GET").path("/").execute();
-        assertThat(response.getStatusCode(), is(200));
-        assertThat(response.getHeaders(), not(hasKey("Access-Control-Allow-Origin")));
+        try (RestClient client = restClient()) {
+            ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null);
+            assertThat(response.getStatusLine().getStatusCode(), is(200));
+            assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
+        }
     }
 
     public void testThatPreFlightRequestWorksOnMatch() throws Exception {
         String corsValue = "http://localhost:9200";
-        HttpResponse response = httpClient().method("OPTIONS")
-                                    .path("/")
-                                    .addHeader("User-Agent", "Mozilla Bar")
-                                    .addHeader("Origin", corsValue)
-                                    .addHeader(HttpHeaders.Names.ACCESS_CONTROL_REQUEST_METHOD, "GET")
-                                    .execute();
-        assertResponseWithOriginheader(response, corsValue);
-        assertThat(response.getHeaders(), hasKey("Access-Control-Allow-Methods"));
+        try (RestClient client = restClient()) {
+            ElasticsearchResponse response = client.performRequest("OPTIONS", "/", Collections.emptyMap(), null,
+                    new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue),
+                    new BasicHeader(HttpHeaders.Names.ACCESS_CONTROL_REQUEST_METHOD, "GET"));
+            assertResponseWithOriginheader(response, corsValue);
+            assertNotNull(response.getFirstHeader("Access-Control-Allow-Methods"));
+        }
     }
 
     public void testThatPreFlightRequestReturnsNullOnNonMatch() throws Exception {
-        HttpResponse response = httpClient().method("OPTIONS")
-                                    .path("/")
-                                    .addHeader("User-Agent", "Mozilla Bar")
-                                    .addHeader("Origin", "http://evil-host:9200")
-                                    .addHeader(HttpHeaders.Names.ACCESS_CONTROL_REQUEST_METHOD, "GET")
-                                    .execute();
-        // a rejected origin gets a FORBIDDEN - 403
-        assertThat(response.getStatusCode(), is(403));
-        assertThat(response.getHeaders(), not(hasKey("Access-Control-Allow-Origin")));
-        assertThat(response.getHeaders(), not(hasKey("Access-Control-Allow-Methods")));
+        try (RestClient client = restClient()) {
+            client.performRequest("OPTIONS", "/", Collections.emptyMap(), null, new BasicHeader("User-Agent", "Mozilla Bar"),
+                    new BasicHeader("Origin", "http://evil-host:9200"),
+                    new BasicHeader(HttpHeaders.Names.ACCESS_CONTROL_REQUEST_METHOD, "GET"));
+            fail("request should have failed");
+        } catch(ElasticsearchResponseException e) {
+            ElasticsearchResponse response = e.getElasticsearchResponse();
+            // a rejected origin gets a FORBIDDEN - 403
+            assertThat(response.getStatusLine().getStatusCode(), is(403));
+            assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
+            assertThat(response.getFirstHeader("Access-Control-Allow-Methods"), nullValue());
+        }
     }
 
-    protected static void assertResponseWithOriginheader(HttpResponse response, String expectedCorsHeader) {
-        assertThat(response.getStatusCode(), is(200));
-        assertThat(response.getHeaders(), hasKey("Access-Control-Allow-Origin"));
-        assertThat(response.getHeaders().get("Access-Control-Allow-Origin"), is(expectedCorsHeader));
+    protected static void assertResponseWithOriginheader(ElasticsearchResponse response, String expectedCorsHeader) {
+        assertThat(response.getStatusLine().getStatusCode(), is(200));
+        assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), is(expectedCorsHeader));
     }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java b/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java
index 0ad40f84cf7..9c2b0284ef7 100644
--- a/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java
+++ b/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java
@@ -18,16 +18,18 @@
  */
 package org.elasticsearch.rest.action.main;
 
+import org.apache.http.util.EntityUtils;
+import org.elasticsearch.client.ElasticsearchResponse;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 
 import java.io.IOException;
+import java.util.Collections;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.nullValue;
 
 public class RestMainActionIT extends ESIntegTestCase {
     @Override
@@ -39,14 +41,19 @@ public class RestMainActionIT extends ESIntegTestCase {
     }
 
     public void testHeadRequest() throws IOException {
-        final HttpResponse response = httpClient().method("HEAD").path("/").execute();
-        assertThat(response.getStatusCode(), equalTo(200));
-        assertThat(response.getBody(), nullValue());
+        try (RestClient client = restClient()) {
+            ElasticsearchResponse response = client.performRequest("HEAD", "/", Collections.emptyMap(), null);
+            assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
+            assertNull(response.getEntity());
+        }
     }
 
     public void testGetRequest() throws IOException {
-        final HttpResponse response = httpClient().path("/").execute();
-        assertThat(response.getStatusCode(), equalTo(200));
-        assertThat(response.getBody(), containsString("cluster_name"));
+        try (RestClient client = restClient()) {
+            ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null);
+            assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
+            assertNotNull(response.getEntity());
+            assertThat(EntityUtils.toString(response.getEntity()), containsString("cluster_name"));
+        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java b/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
index eb13de34858..aae15dc778a 100644
--- a/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
+++ b/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
@@ -19,8 +19,7 @@
 
 package org.elasticsearch.transport;
 
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicHeader;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionModule;
 import org.elasticsearch.action.ActionRequest;
@@ -33,12 +32,13 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.ActionFilter;
 import org.elasticsearch.action.termvectors.MultiTermVectorsRequest;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.ElasticsearchResponse;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.GeoShapeQueryBuilder;
 import org.elasticsearch.index.query.MoreLikeThisQueryBuilder;
@@ -50,8 +50,6 @@ import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.elasticsearch.test.rest.client.http.HttpRequestBuilder;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
 import org.junit.Before;
@@ -217,26 +215,22 @@ public class ContextAndHeaderTransportIT extends ESIntegTestCase {
     }
 
     public void testThatRelevantHttpHeadersBecomeRequestHeaders() throws Exception {
-        String releventHeaderName = "relevant_" + randomHeaderKey;
-        for (RestController restController : internalCluster().getDataNodeInstances(RestController.class)) {
-            restController.registerRelevantHeaders(releventHeaderName);
+        String relevantHeaderName = "relevant_" + randomHeaderKey;
+        for (RestController restController : internalCluster().getInstances(RestController.class)) {
+            restController.registerRelevantHeaders(relevantHeaderName);
         }
 
-        CloseableHttpClient httpClient = HttpClients.createDefault();
-        HttpResponse response = new HttpRequestBuilder(httpClient)
-            .httpTransport(internalCluster().getDataNodeInstance(HttpServerTransport.class))
-            .addHeader(randomHeaderKey, randomHeaderValue)
-            .addHeader(releventHeaderName, randomHeaderValue)
-            .path("/" + queryIndex + "/_search")
-            .execute();
-
-        assertThat(response, hasStatus(OK));
-        List<RequestAndHeaders> searchRequests = getRequests(SearchRequest.class);
-        assertThat(searchRequests, hasSize(greaterThan(0)));
-        for (RequestAndHeaders requestAndHeaders : searchRequests) {
-            assertThat(requestAndHeaders.headers.containsKey(releventHeaderName), is(true));
-            // was not specified, thus is not included
-            assertThat(requestAndHeaders.headers.containsKey(randomHeaderKey), is(false));
+        try (RestClient client = restClient()) {
+            ElasticsearchResponse response = client.performRequest("GET", "/" + queryIndex + "/_search", Collections.emptyMap(), null,
+                    new BasicHeader(randomHeaderKey, randomHeaderValue), new BasicHeader(relevantHeaderName, randomHeaderValue));
+            assertThat(response, hasStatus(OK));
+            List<RequestAndHeaders> searchRequests = getRequests(SearchRequest.class);
+            assertThat(searchRequests, hasSize(greaterThan(0)));
+            for (RequestAndHeaders requestAndHeaders : searchRequests) {
+                assertThat(requestAndHeaders.headers.containsKey(relevantHeaderName), is(true));
+                // was not specified, thus is not included
+                assertThat(requestAndHeaders.headers.containsKey(randomHeaderKey), is(false));
+            }
         }
     }
 
diff --git a/test/framework/build.gradle b/test/framework/build.gradle
index af65c9ff7c9..078511611fe 100644
--- a/test/framework/build.gradle
+++ b/test/framework/build.gradle
@@ -23,14 +23,15 @@ dependencies {
   compile "org.elasticsearch:elasticsearch:${version}"
   compile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   compile "junit:junit:${versions.junit}"
-  compile 'org.hamcrest:hamcrest-all:1.3'
+  compile "org.hamcrest:hamcrest-all:1.3"
   compile "org.apache.lucene:lucene-test-framework:${versions.lucene}"
   compile "org.apache.lucene:lucene-codecs:${versions.lucene}"
+  compile "org.elasticsearch:client:${version}"
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
-  compile 'org.elasticsearch:securemock:1.2'
+  compile "org.elasticsearch:securemock:1.2"
 }
 
 compileJava.options.compilerArgs << '-Xlint:-cast,-rawtypes,-try,-unchecked'
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index 2a3eecf4cc8..67f2a654928 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -22,8 +22,8 @@ import com.carrotsearch.randomizedtesting.RandomizedContext;
 import com.carrotsearch.randomizedtesting.annotations.TestGroup;
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import org.apache.http.HttpHost;
 import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestUtil;
@@ -57,6 +57,7 @@ import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.AdminClient;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.Requests;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -118,7 +119,6 @@ import org.elasticsearch.search.MockSearchService;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.test.client.RandomizingClient;
 import org.elasticsearch.test.disruption.ServiceDisruptionScheme;
-import org.elasticsearch.test.rest.client.http.HttpRequestBuilder;
 import org.elasticsearch.test.store.MockFSIndexStore;
 import org.elasticsearch.test.transport.AssertingLocalTransport;
 import org.elasticsearch.test.transport.MockTransportService;
@@ -2033,20 +2033,34 @@ public abstract class ESIntegTestCase extends ESTestCase {
         return builder.build();
     }
 
-    protected HttpRequestBuilder httpClient() {
-        return httpClient(HttpClients.createDefault());
+    protected static RestClient restClient() {
+        return restClient(null);
     }
 
-    protected HttpRequestBuilder httpClient(CloseableHttpClient httpClient) {
+    protected static RestClient restClient(CloseableHttpClient httpClient) {
+        return restClient(httpClient, "http");
+    }
+
+    protected static RestClient restClient(CloseableHttpClient httpClient, String protocol) {
         final NodesInfoResponse nodeInfos = client().admin().cluster().prepareNodesInfo().get();
         final List<NodeInfo> nodes = nodeInfos.getNodes();
         assertFalse(nodeInfos.hasFailures());
-        TransportAddress publishAddress = randomFrom(nodes).getHttp().address().publishAddress();
-        assertEquals(1, publishAddress.uniqueAddressTypeId());
-        InetSocketAddress address = ((InetSocketTransportAddress) publishAddress).address();
-        return new HttpRequestBuilder(httpClient).host(NetworkAddress.format(address.getAddress())).port(address.getPort());
-    }
+        List<HttpHost> hosts = new ArrayList<>();
+        for (NodeInfo node : nodes) {
+            if (node.getHttp() != null) {
+                TransportAddress publishAddress = node.getHttp().address().publishAddress();
+                assertEquals(1, publishAddress.uniqueAddressTypeId());
+                InetSocketAddress address = ((InetSocketTransportAddress) publishAddress).address();
+                hosts.add(new HttpHost(NetworkAddress.format(address.getAddress()), address.getPort(), protocol));
+            }
+        }
 
+        RestClient.Builder builder = RestClient.builder().setHosts(hosts.toArray(new HttpHost[hosts.size()]));
+        if (httpClient != null) {
+            builder.setHttpClient(httpClient);
+        }
+        return builder.build();
+    }
 
     /**
      * This method is executed iff the test is annotated with {@link SuiteScopeTestCase}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
index d10c113b590..0fe65ccf752 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
@@ -42,6 +42,7 @@ import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
+import org.elasticsearch.client.ElasticsearchResponse;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
@@ -60,7 +61,6 @@ import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.suggest.Suggest;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 import org.hamcrest.CoreMatchers;
 import org.hamcrest.Matcher;
 import org.hamcrest.Matchers;
@@ -82,7 +82,6 @@ import static org.elasticsearch.test.VersionUtils.randomVersion;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.hasItem;
@@ -90,7 +89,6 @@ import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.nullValue;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -488,8 +486,8 @@ public class ElasticsearchAssertions {
         return new ElasticsearchMatchers.SearchHitHasScoreMatcher(score);
     }
 
-    public static Matcher<HttpResponse> hasStatus(RestStatus restStatus) {
-        return new ElasticsearchMatchers.HttpResponseHasStatusMatcher(restStatus);
+    public static Matcher<ElasticsearchResponse> hasStatus(RestStatus restStatus) {
+        return new ElasticsearchMatchers.ElasticsearchResponseHasStatusMatcher(restStatus);
     }
 
     public static <T extends Query> T assertBooleanSubQuery(Query query, Class<T> subqueryType, int i) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchMatchers.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchMatchers.java
index d86791fa6bd..c40e434a5a7 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchMatchers.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchMatchers.java
@@ -18,9 +18,9 @@
  */
 package org.elasticsearch.test.hamcrest;
 
+import org.elasticsearch.client.ElasticsearchResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 import org.hamcrest.Description;
 import org.hamcrest.TypeSafeMatcher;
 
@@ -118,27 +118,27 @@ public class ElasticsearchMatchers {
         }
     }
 
-    public static class HttpResponseHasStatusMatcher extends TypeSafeMatcher<HttpResponse> {
+    public static class ElasticsearchResponseHasStatusMatcher extends TypeSafeMatcher<ElasticsearchResponse> {
 
         private RestStatus restStatus;
 
-        public HttpResponseHasStatusMatcher(RestStatus restStatus) {
+        public ElasticsearchResponseHasStatusMatcher(RestStatus restStatus) {
             this.restStatus = restStatus;
         }
 
         @Override
-        protected boolean matchesSafely(HttpResponse response) {
-            return response.getStatusCode() == restStatus.getStatus();
+        protected boolean matchesSafely(ElasticsearchResponse response) {
+            return response.getStatusLine().getStatusCode() == restStatus.getStatus();
         }
 
         @Override
-        public void describeMismatchSafely(final HttpResponse response, final Description mismatchDescription) {
-            mismatchDescription.appendText(" was ").appendValue(response.getStatusCode());
+        public void describeMismatchSafely(final ElasticsearchResponse response, final Description mismatchDescription) {
+            mismatchDescription.appendText(" was ").appendValue(response.getStatusLine().getStatusCode());
         }
 
         @Override
         public void describeTo(final Description description) {
-            description.appendText("HTTP response status code should be ").appendValue(restStatus.getStatus());
+            description.appendText("Elasticsearch response status code should be ").appendValue(restStatus.getStatus());
         }
     }
 }

From eae914ae8e768ba57a10ecda41cf9b64c372c68b Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Thu, 19 May 2016 15:18:17 +0200
Subject: [PATCH 0611/1311] Replace rest test client with low level RestClient

We still have a wrapper called RestTestClient that is very specific to Rest tests, as well as RestTestResponse etc. but all the low level bits around http connections etc. are now handled by RestClient.
---
 .../resources/checkstyle_suppressions.xml     |   4 -
 .../org/elasticsearch/client/RestClient.java  |   1 -
 .../ExceptionSerializationTests.java          |   1 -
 .../test/rest/ESRestTestCase.java             |  75 ++---
 .../test/rest/RestTestExecutionContext.java   |  37 ++-
 .../org/elasticsearch/test/rest/Stash.java    |  16 +-
 .../test/rest/client/RestException.java       |  41 ---
 .../{RestClient.java => RestTestClient.java}  | 268 +++++++++---------
 ...estResponse.java => RestTestResponse.java} |  56 ++--
 .../client/http/HttpDeleteWithEntity.java     |  40 ---
 .../rest/client/http/HttpGetWithEntity.java   |  40 ---
 .../rest/client/http/HttpRequestBuilder.java  | 250 ----------------
 .../test/rest/client/http/HttpResponse.java   | 108 -------
 .../test/rest/section/DoSection.java          |  33 ++-
 14 files changed, 258 insertions(+), 712 deletions(-)
 delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/client/RestException.java
 rename test/framework/src/main/java/org/elasticsearch/test/rest/client/{RestClient.java => RestTestClient.java} (52%)
 rename test/framework/src/main/java/org/elasticsearch/test/rest/client/{RestResponse.java => RestTestResponse.java} (62%)
 delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpDeleteWithEntity.java
 delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpGetWithEntity.java
 delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpRequestBuilder.java
 delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpResponse.java

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index d6a38e76e3e..985ffefa411 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -1311,9 +1311,6 @@
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]engine[/\\]MockEngineSupport.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]hamcrest[/\\]ElasticsearchAssertions.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]junit[/\\]listeners[/\\]LoggingListener.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]ESRestTestCase.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]RestTestExecutionContext.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]client[/\\]http[/\\]HttpRequestBuilder.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]json[/\\]JsonPath.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]parser[/\\]GreaterThanEqualToParser.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]parser[/\\]GreaterThanParser.java" checks="LineLength" />
@@ -1321,7 +1318,6 @@
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]parser[/\\]LessThanParser.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]parser[/\\]RestTestSuiteParseContext.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]parser[/\\]RestTestSuiteParser.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]section[/\\]DoSection.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]section[/\\]GreaterThanAssertion.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]section[/\\]GreaterThanEqualToAssertion.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]section[/\\]LengthAssertion.java" checks="LineLength" />
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 31914f46282..b366d98f77a 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -121,7 +121,6 @@ public final class RestClient implements Closeable {
             ElasticsearchResponse elasticsearchResponse = new ElasticsearchResponse(request.getRequestLine(),
                     connection.getHost(), response);
             int statusCode = response.getStatusLine().getStatusCode();
-            //TODO make ignore status code configurable. rest-spec and tests support that parameter (ignore_missing)
             if (statusCode < 300 || (request.getMethod().equals(HttpHead.METHOD_NAME) && statusCode == 404) ) {
                 RequestLogger.log(logger, "request succeeded", request, connection.getHost(), response);
                 onSuccess(connection);
diff --git a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
index 682eed35f2e..d10fc890848 100644
--- a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
+++ b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
@@ -121,7 +121,6 @@ public class ExceptionSerializationTests extends ESTestCase {
                 .resolve("org").resolve("elasticsearch");
         final Set<? extends Class<?>> ignore = Sets.newHashSet(
                 org.elasticsearch.test.rest.parser.RestTestParseException.class,
-                org.elasticsearch.test.rest.client.RestException.class,
                 CancellableThreadsTests.CustomException.class,
                 org.elasticsearch.rest.BytesRestResponseTests.WithHeadersException.class,
                 AbstractClientHeadersTestCase.InternalException.class);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index ab2c1d2afbe..72040863704 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -19,6 +19,31 @@
 
 package org.elasticsearch.test.rest;
 
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction;
+import org.elasticsearch.client.ElasticsearchResponseException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.client.RestTestResponse;
+import org.elasticsearch.test.rest.parser.RestTestParseException;
+import org.elasticsearch.test.rest.parser.RestTestSuiteParser;
+import org.elasticsearch.test.rest.section.DoSection;
+import org.elasticsearch.test.rest.section.ExecutableSection;
+import org.elasticsearch.test.rest.section.RestTestSuite;
+import org.elasticsearch.test.rest.section.SkipSection;
+import org.elasticsearch.test.rest.section.TestSection;
+import org.elasticsearch.test.rest.spec.RestApi;
+import org.elasticsearch.test.rest.spec.RestSpec;
+import org.elasticsearch.test.rest.support.FileUtils;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
@@ -38,32 +63,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.rest.client.RestException;
-import org.elasticsearch.test.rest.client.RestResponse;
-import org.elasticsearch.test.rest.parser.RestTestParseException;
-import org.elasticsearch.test.rest.parser.RestTestSuiteParser;
-import org.elasticsearch.test.rest.section.DoSection;
-import org.elasticsearch.test.rest.section.ExecutableSection;
-import org.elasticsearch.test.rest.section.RestTestSuite;
-import org.elasticsearch.test.rest.section.SkipSection;
-import org.elasticsearch.test.rest.section.TestSection;
-import org.elasticsearch.test.rest.spec.RestApi;
-import org.elasticsearch.test.rest.spec.RestSpec;
-import org.elasticsearch.test.rest.support.FileUtils;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-
-import com.carrotsearch.randomizedtesting.RandomizedTest;
-
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.sort;
@@ -98,8 +97,8 @@ public abstract class ESRestTestCase extends ESTestCase {
     private static final String DEFAULT_SPEC_PATH = "/rest-api-spec/api";
 
     /**
-     * This separator pattern matches ',' except it is preceded by a '\'. This allows us to support ',' within paths when it is escaped with
-     * a slash.
+     * This separator pattern matches ',' except it is preceded by a '\'.
+     * This allows us to support ',' within paths when it is escaped with a slash.
      *
      * For example, the path string "/a/b/c\,d/e/f,/foo/bar,/baz" is separated to "/a/b/c\,d/e/f", "/foo/bar" and "/baz".
      *
@@ -233,7 +232,7 @@ public abstract class ESRestTestCase extends ESTestCase {
     }
 
     @BeforeClass
-    public static void initExecutionContext() throws IOException, RestException {
+    public static void initExecutionContext() throws IOException {
         String[] specPaths = resolvePathsProperty(REST_TESTS_SPEC, DEFAULT_SPEC_PATH);
         RestSpec restSpec = null;
         FileSystem fileSystem = getFileSystem();
@@ -277,9 +276,9 @@ public abstract class ESRestTestCase extends ESTestCase {
         deleteIndicesArgs.put("index", "*");
         try {
             adminExecutionContext.callApi("indices.delete", deleteIndicesArgs, Collections.emptyList(), Collections.emptyMap());
-        } catch (RestException e) {
+        } catch (ElasticsearchResponseException e) {
             // 404 here just means we had no indexes
-            if (e.statusCode() != 404) {
+            if (e.getElasticsearchResponse().getStatusLine().getStatusCode() != 404) {
                 throw e;
             }
         }
@@ -300,8 +299,8 @@ public abstract class ESRestTestCase extends ESTestCase {
      * other tests.
      */
     @After
-    public void logIfThereAreRunningTasks() throws InterruptedException, IOException, RestException {
-        RestResponse tasks = adminExecutionContext.callApi("tasks.list", emptyMap(), emptyList(), emptyMap());
+    public void logIfThereAreRunningTasks() throws InterruptedException, IOException {
+        RestTestResponse tasks = adminExecutionContext.callApi("tasks.list", emptyMap(), emptyList(), emptyMap());
         Set<String> runningTasks = runningTasks(tasks);
         // Ignore the task list API - it doens't count against us
         runningTasks.remove(ListTasksAction.NAME);
@@ -347,7 +346,7 @@ public abstract class ESRestTestCase extends ESTestCase {
     }
 
     @Before
-    public void reset() throws IOException, RestException {
+    public void reset() throws IOException {
         // admin context must be available for @After always, regardless of whether the test was blacklisted
         adminExecutionContext.initClient(clusterUrls, restAdminSettings());
         adminExecutionContext.clear();
@@ -355,7 +354,8 @@ public abstract class ESRestTestCase extends ESTestCase {
         //skip test if it matches one of the blacklist globs
         for (BlacklistedPathPatternMatcher blacklistedPathMatcher : blacklistPathMatchers) {
             String testPath = testCandidate.getSuitePath() + "/" + testCandidate.getTestSection().getName();
-            assumeFalse("[" + testCandidate.getTestPath() + "] skipped, reason: blacklisted", blacklistedPathMatcher.isSuffixMatch(testPath));
+            assumeFalse("[" + testCandidate.getTestPath() + "] skipped, reason: blacklisted", blacklistedPathMatcher
+                    .isSuffixMatch(testPath));
         }
         //The client needs non static info to get initialized, therefore it can't be initialized in the before class
         restTestExecutionContext.initClient(clusterUrls, restClientSettings());
@@ -374,7 +374,8 @@ public abstract class ESRestTestCase extends ESTestCase {
         if (skipSection.isVersionCheck()) {
             messageBuilder.append("[").append(description).append("] skipped, reason: [").append(skipSection.getReason()).append("] ");
         } else {
-            messageBuilder.append("[").append(description).append("] skipped, reason: features ").append(skipSection.getFeatures()).append(" not supported");
+            messageBuilder.append("[").append(description).append("] skipped, reason: features ")
+                    .append(skipSection.getFeatures()).append(" not supported");
         }
         return messageBuilder.toString();
     }
@@ -401,7 +402,7 @@ public abstract class ESRestTestCase extends ESTestCase {
     }
 
     @SuppressWarnings("unchecked")
-    public Set<String> runningTasks(RestResponse response) throws IOException {
+    public Set<String> runningTasks(RestTestResponse response) throws IOException {
         Set<String> runningTasks = new HashSet<>();
         Map<String, Object> nodes = (Map<String, Object>) response.evaluate("nodes");
         for (Map.Entry<String, Object> node : nodes.entrySet()) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java
index 4a7f83953de..1fa3dba5d83 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java
@@ -19,18 +19,17 @@
 package org.elasticsearch.test.rest;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.client.ElasticsearchResponseException;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.test.rest.client.RestClient;
-import org.elasticsearch.test.rest.client.RestException;
-import org.elasticsearch.test.rest.client.RestResponse;
+import org.elasticsearch.test.rest.client.RestTestClient;
+import org.elasticsearch.test.rest.client.RestTestResponse;
 import org.elasticsearch.test.rest.spec.RestSpec;
 
 import java.io.Closeable;
 import java.io.IOException;
-import java.net.InetSocketAddress;
 import java.net.URL;
 import java.util.HashMap;
 import java.util.List;
@@ -50,9 +49,9 @@ public class RestTestExecutionContext implements Closeable {
 
     private final RestSpec restSpec;
 
-    private RestClient restClient;
+    private RestTestClient restTestClient;
 
-    private RestResponse response;
+    private RestTestResponse response;
 
     public RestTestExecutionContext(RestSpec restSpec) {
         this.restSpec = restSpec;
@@ -61,10 +60,9 @@ public class RestTestExecutionContext implements Closeable {
     /**
      * Calls an elasticsearch api with the parameters and request body provided as arguments.
      * Saves the obtained response in the execution context.
-     * @throws RestException if the returned status code is non ok
      */
-    public RestResponse callApi(String apiName, Map<String, String> params, List<Map<String, Object>> bodies,
-                                Map<String, String> headers) throws IOException, RestException  {
+    public RestTestResponse callApi(String apiName, Map<String, String> params, List<Map<String, Object>> bodies,
+                                    Map<String, String> headers) throws IOException  {
         //makes a copy of the parameters before modifying them for this specific request
         HashMap<String, String> requestParams = new HashMap<>(params);
         for (Map.Entry<String, String> entry : requestParams.entrySet()) {
@@ -80,8 +78,8 @@ public class RestTestExecutionContext implements Closeable {
             //we always stash the last response body
             stash.stashResponse(response);
             return response;
-        } catch(RestException e) {
-            response = e.restResponse();
+        } catch(ElasticsearchResponseException e) {
+            response = new RestTestResponse(e);
             throw e;
         }
     }
@@ -106,8 +104,9 @@ public class RestTestExecutionContext implements Closeable {
         return XContentFactory.jsonBuilder().map(body).string();
     }
 
-    private RestResponse callApiInternal(String apiName, Map<String, String> params, String body, Map<String, String> headers) throws IOException, RestException  {
-        return restClient.callApi(apiName, params, body, headers);
+    private RestTestResponse callApiInternal(String apiName, Map<String, String> params, String body, Map<String, String> headers)
+            throws IOException  {
+        return restTestClient.callApi(apiName, params, body, headers);
     }
 
     /**
@@ -120,9 +119,9 @@ public class RestTestExecutionContext implements Closeable {
     /**
      * Creates the embedded REST client when needed. Needs to be called before each test.
      */
-    public void initClient(URL[] urls, Settings settings) throws IOException, RestException {
-        if (restClient == null) {
-            restClient = new RestClient(restSpec, settings, urls);
+    public void initClient(URL[] urls, Settings settings) throws IOException {
+        if (restTestClient == null) {
+            restTestClient = new RestTestClient(restSpec, settings, urls);
         }
     }
 
@@ -143,7 +142,7 @@ public class RestTestExecutionContext implements Closeable {
      * Returns the current es version as a string
      */
     public Version esVersion() {
-        return restClient.getEsVersion();
+        return restTestClient.getEsVersion();
     }
 
     /**
@@ -151,8 +150,8 @@ public class RestTestExecutionContext implements Closeable {
      */
     @Override
     public void close() {
-        if (restClient != null) {
-            restClient.close();
+        if (restTestClient != null) {
+            restTestClient.close();
         }
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java b/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
index 52b531e5cd1..885df395c2b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
@@ -19,17 +19,17 @@
 
 package org.elasticsearch.test.rest;
 
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.test.rest.client.RestResponse;
+import org.elasticsearch.test.rest.client.RestTestResponse;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 /**
  * Allows to cache the last obtained test response and or part of it within variables
@@ -42,7 +42,7 @@ public class Stash implements ToXContent {
     public static final Stash EMPTY = new Stash();
 
     private final Map<String, Object> stash = new HashMap<>();
-    private RestResponse response;
+    private RestTestResponse response;
 
     /**
      * Allows to saved a specific field in the stash as key-value pair
@@ -55,7 +55,7 @@ public class Stash implements ToXContent {
         }
     }
 
-    public void stashResponse(RestResponse response) throws IOException {
+    public void stashResponse(RestTestResponse response) throws IOException {
         // TODO we can almost certainly save time by lazily evaluating the body
         stashValue("body", response.getBody());
         this.response = response;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestException.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestException.java
deleted file mode 100644
index 2236134837b..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestException.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.client;
-
-/**
- * Thrown when a status code that holds an error is received (unless needs to be ignored)
- * Holds the original {@link RestResponse}
- */
-public class RestException extends Exception {
-
-    private final RestResponse restResponse;
-
-    public RestException(String message, RestResponse restResponse) {
-        super(message);
-        this.restResponse = restResponse;
-    }
-
-    public RestResponse restResponse() {
-        return restResponse;
-    }
-
-    public int statusCode() {
-        return restResponse.getStatusCode();
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java
similarity index 52%
rename from test/framework/src/main/java/org/elasticsearch/test/rest/client/RestClient.java
rename to test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java
index cb35653b103..3018af869b3 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestClient.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java
@@ -19,17 +19,25 @@
 package org.elasticsearch.test.rest.client;
 
 import com.carrotsearch.randomizedtesting.RandomizedTest;
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
 import org.apache.http.config.Registry;
 import org.apache.http.config.RegistryBuilder;
 import org.apache.http.conn.socket.ConnectionSocketFactory;
 import org.apache.http.conn.socket.PlainConnectionSocketFactory;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.SSLContexts;
+import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
-import org.apache.http.ssl.SSLContexts;
+import org.apache.http.message.BasicHeader;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
+import org.elasticsearch.client.ElasticsearchResponse;
+import org.elasticsearch.client.ElasticsearchResponseException;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.logging.ESLogger;
@@ -37,8 +45,6 @@ import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.test.rest.client.http.HttpRequestBuilder;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 import org.elasticsearch.test.rest.spec.RestApi;
 import org.elasticsearch.test.rest.spec.RestSpec;
 
@@ -46,6 +52,8 @@ import javax.net.ssl.SSLContext;
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -55,61 +63,53 @@ import java.security.KeyStoreException;
 import java.security.NoSuchAlgorithmException;
 import java.security.cert.CertificateException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-import static java.util.Objects.requireNonNull;
 
 /**
  * REST client used to test the elasticsearch REST layer
  * Holds the {@link RestSpec} used to translate api calls into REST calls
  */
-public class RestClient implements Closeable {
+public class RestTestClient implements Closeable {
 
     public static final String PROTOCOL = "protocol";
     public static final String TRUSTSTORE_PATH = "truststore.path";
     public static final String TRUSTSTORE_PASSWORD = "truststore.password";
 
-    private static final ESLogger logger = Loggers.getLogger(RestClient.class);
+
+    private static final ESLogger logger = Loggers.getLogger(RestTestClient.class);
     //query_string params that don't need to be declared in the spec, thay are supported by default
     private static final Set<String> ALWAYS_ACCEPTED_QUERY_STRING_PARAMS = Sets.newHashSet("pretty", "source", "filter_path");
 
-    private final String protocol;
     private final RestSpec restSpec;
-    private final CloseableHttpClient httpClient;
-    private final URL[] urls;
+    private final RestClient restClient;
     private final Version esVersion;
-    private final ThreadContext threadContext;
 
-    public RestClient(RestSpec restSpec, Settings settings, URL[] urls) throws IOException, RestException {
+    public RestTestClient(RestSpec restSpec, Settings settings, URL[] urls) throws IOException {
         assert urls.length > 0;
         this.restSpec = restSpec;
-        this.protocol = settings.get(PROTOCOL, "http");
-        this.httpClient = createHttpClient(settings);
-        this.threadContext = new ThreadContext(settings);
-        this.urls = urls;
-        this.esVersion = readAndCheckVersion();
+        this.restClient = createRestClient(urls, settings);
+        this.esVersion = readAndCheckVersion(urls);
         logger.info("REST client initialized {}, elasticsearch version: [{}]", urls, esVersion);
     }
 
-    private Version readAndCheckVersion() throws IOException, RestException {
-        //we make a manual call here without using callApi method, mainly because we are initializing
-        //and the randomized context doesn't exist for the current thread (would be used to choose the method otherwise)
+    private Version readAndCheckVersion(URL[] urls) throws IOException {
         RestApi restApi = restApi("info");
         assert restApi.getPaths().size() == 1;
         assert restApi.getMethods().size() == 1;
 
         String version = null;
-        for (URL url : urls) {
-            RestResponse restResponse = new RestResponse(httpRequestBuilder(url)
-                    .path(restApi.getPaths().get(0))
-                    .method(restApi.getMethods().get(0)).execute());
-            checkStatusCode(restResponse);
-
-            Object latestVersion = restResponse.evaluate("version.number");
+        for (URL ignored : urls) {
+            //we don't really use the urls here, we rely on the client doing round-robin to touch all the nodes in the cluster
+            String method = restApi.getMethods().get(0);
+            String endpoint = restApi.getPaths().get(0);
+            ElasticsearchResponse elasticsearchResponse = restClient.performRequest(method, endpoint, Collections.emptyMap(), null);
+            RestTestResponse restTestResponse = new RestTestResponse(elasticsearchResponse);
+            Object latestVersion = restTestResponse.evaluate("version.number");
             if (latestVersion == null) {
                 throw new RuntimeException("elasticsearch version not found in the response");
             }
@@ -130,77 +130,41 @@ public class RestClient implements Closeable {
 
     /**
      * Calls an api with the provided parameters and body
-     * @throws RestException if the obtained status code is non ok, unless the specific error code needs to be ignored
-     * according to the ignore parameter received as input (which won't get sent to elasticsearch)
      */
-    public RestResponse callApi(String apiName, Map<String, String> params, String body, Map<String, String> headers)
-            throws IOException, RestException {
+    public RestTestResponse callApi(String apiName, Map<String, String> params, String body, Map<String, String> headers)
+            throws IOException {
 
-        List<Integer> ignores = new ArrayList<>();
-        Map<String, String> requestParams = null;
-        if (params != null) {
-            //makes a copy of the parameters before modifying them for this specific request
-            requestParams = new HashMap<>(params);
-            //ignore is a special parameter supported by the clients, shouldn't be sent to es
-            String ignoreString = requestParams.remove("ignore");
-            if (Strings.hasLength(ignoreString)) {
-                try {
-                    ignores.add(Integer.valueOf(ignoreString));
-                } catch(NumberFormatException e) {
-                    throw new IllegalArgumentException("ignore value should be a number, found [" + ignoreString + "] instead");
-                }
-            }
-        }
-
-        HttpRequestBuilder httpRequestBuilder = callApiBuilder(apiName, requestParams, body);
-        for (Map.Entry<String, String> header : headers.entrySet()) {
-            logger.error("Adding header {}\n with value {}", header.getKey(), header.getValue());
-            httpRequestBuilder.addHeader(header.getKey(), header.getValue());
-        }
-        logger.debug("calling api [{}]", apiName);
-        HttpResponse httpResponse = httpRequestBuilder.execute();
-
-        // http HEAD doesn't support response body
-        // For the few api (exists class of api) that use it we need to accept 404 too
-        if (!httpResponse.supportsBody()) {
-            ignores.add(404);
-        }
-
-        RestResponse restResponse = new RestResponse(httpResponse);
-        checkStatusCode(restResponse, ignores);
-        return restResponse;
-    }
-
-    private void checkStatusCode(RestResponse restResponse, List<Integer> ignores) throws RestException {
-        //ignore is a catch within the client, to prevent the client from throwing error if it gets non ok codes back
-        if (ignores.contains(restResponse.getStatusCode())) {
-            if (logger.isDebugEnabled()) {
-                logger.debug("ignored non ok status codes {} as requested", ignores);
-            }
-            return;
-        }
-        checkStatusCode(restResponse);
-    }
-
-    private void checkStatusCode(RestResponse restResponse) throws RestException {
-        if (restResponse.isError()) {
-            throw new RestException("non ok status code [" + restResponse.getStatusCode() + "] returned", restResponse);
-        }
-    }
-
-    private HttpRequestBuilder callApiBuilder(String apiName, Map<String, String> params, String body) {
         if ("raw".equals(apiName)) {
             // Raw requests are bit simpler....
-            HttpRequestBuilder httpRequestBuilder = httpRequestBuilder();
-            httpRequestBuilder.method(requireNonNull(params.remove("method"), "Method must be set to use raw request"));
-            httpRequestBuilder.path("/"+ requireNonNull(params.remove("path"), "Path must be set to use raw request"));
-            httpRequestBuilder.body(body);
-
-            // And everything else is a url parameter!
-            for (Map.Entry<String, String> entry : params.entrySet()) {
-                httpRequestBuilder.addParam(entry.getKey(), entry.getValue());
+            HashMap<String, String> queryStringParams = new HashMap<>(params);
+            String method = Objects.requireNonNull(queryStringParams.remove("method"), "Method must be set to use raw request");
+            String path = "/"+ Objects.requireNonNull(queryStringParams.remove("path"), "Path must be set to use raw request");
+            HttpEntity entity = null;
+            if (body != null && body.length() > 0) {
+                entity = new StringEntity(body, RestClient.JSON_CONTENT_TYPE);
+            }
+            // And everything else is a url parameter!
+            ElasticsearchResponse response = restClient.performRequest(method, path, queryStringParams, entity);
+            return new RestTestResponse(response);
+        }
+
+        List<Integer> ignores = new ArrayList<>();
+        Map<String, String> requestParams;
+        if (params == null) {
+            requestParams = Collections.emptyMap();
+        } else {
+            requestParams = new HashMap<>(params);
+            if (params.isEmpty() == false) {
+                //ignore is a special parameter supported by the clients, shouldn't be sent to es
+                String ignoreString = requestParams.remove("ignore");
+                if (ignoreString != null) {
+                    try {
+                        ignores.add(Integer.valueOf(ignoreString));
+                    } catch (NumberFormatException e) {
+                        throw new IllegalArgumentException("ignore value should be a number, found [" + ignoreString + "] instead");
+                    }
+                }
             }
-            return httpRequestBuilder;
         }
 
         //create doesn't exist in the spec but is supported in the clients (index with op_type=create)
@@ -208,51 +172,91 @@ public class RestClient implements Closeable {
         String api = indexCreateApi ? "index" : apiName;
         RestApi restApi = restApi(api);
 
-        HttpRequestBuilder httpRequestBuilder = httpRequestBuilder();
-
         //divide params between ones that go within query string and ones that go within path
         Map<String, String> pathParts = new HashMap<>();
-        if (params != null) {
-            for (Map.Entry<String, String> entry : params.entrySet()) {
-                if (restApi.getPathParts().contains(entry.getKey())) {
-                    pathParts.put(entry.getKey(), entry.getValue());
+        Map<String, String> queryStringParams = new HashMap<>();
+        for (Map.Entry<String, String> entry : requestParams.entrySet()) {
+            if (restApi.getPathParts().contains(entry.getKey())) {
+                pathParts.put(entry.getKey(), entry.getValue());
+            } else {
+                if (restApi.getParams().contains(entry.getKey()) || ALWAYS_ACCEPTED_QUERY_STRING_PARAMS.contains(entry.getKey())) {
+                    queryStringParams.put(entry.getKey(), entry.getValue());
                 } else {
-                    if (restApi.getParams().contains(entry.getKey()) || ALWAYS_ACCEPTED_QUERY_STRING_PARAMS.contains(entry.getKey())) {
-                        httpRequestBuilder.addParam(entry.getKey(), entry.getValue());
-                    } else {
-                        throw new IllegalArgumentException("param [" + entry.getKey() +
-                                "] not supported in [" + restApi.getName() + "] api");
-                    }
+                    throw new IllegalArgumentException("param [" + entry.getKey() + "] not supported in ["
+                            + restApi.getName() + "] " + "api");
                 }
             }
         }
 
         if (indexCreateApi) {
-            httpRequestBuilder.addParam("op_type", "create");
+            queryStringParams.put("op_type", "create");
         }
 
         List<String> supportedMethods = restApi.getSupportedMethods(pathParts.keySet());
+        String requestMethod;
+        StringEntity requestBody = null;
         if (Strings.hasLength(body)) {
             if (!restApi.isBodySupported()) {
                 throw new IllegalArgumentException("body is not supported by [" + restApi.getName() + "] api");
             }
-            //test the GET with source param instead of GET/POST with body
+            //randomly test the GET with source param instead of GET/POST with body
             if (supportedMethods.contains("GET") && RandomizedTest.rarely()) {
                 logger.debug("sending the request body as source param with GET method");
-                httpRequestBuilder.addParam("source", body).method("GET");
+                queryStringParams.put("source", body);
+                requestMethod = "GET";
             } else {
-                httpRequestBuilder.body(body).method(RandomizedTest.randomFrom(supportedMethods));
+                requestMethod = RandomizedTest.randomFrom(supportedMethods);
+                requestBody = new StringEntity(body, RestClient.JSON_CONTENT_TYPE);
             }
         } else {
             if (restApi.isBodyRequired()) {
                 throw new IllegalArgumentException("body is required by [" + restApi.getName() + "] api");
             }
-            httpRequestBuilder.method(RandomizedTest.randomFrom(supportedMethods));
+            requestMethod = RandomizedTest.randomFrom(supportedMethods);
         }
 
         //the rest path to use is randomized out of the matching ones (if more than one)
         RestPath restPath = RandomizedTest.randomFrom(restApi.getFinalPaths(pathParts));
-        return httpRequestBuilder.pathParts(restPath.getPathParts());
+        //Encode rules for path and query string parameters are different. We use URI to encode the path.
+        //We need to encode each path part separately, as each one might contain slashes that need to be escaped, which needs to
+        //be done manually.
+        String requestPath;
+        if (restPath.getPathParts().length == 0) {
+            requestPath = "/";
+        } else {
+            StringBuilder finalPath = new StringBuilder();
+            for (String pathPart : restPath.getPathParts()) {
+                try {
+                    finalPath.append('/');
+                    // We append "/" to the path part to handle parts that start with - or other invalid characters
+                    URI uri = new URI(null, null, null, -1, "/" + pathPart, null, null);
+                    //manually escape any slash that each part may contain
+                    finalPath.append(uri.getRawPath().substring(1).replaceAll("/", "%2F"));
+                } catch (URISyntaxException e) {
+                    throw new RuntimeException("unable to build uri", e);
+                }
+            }
+            requestPath = finalPath.toString();
+        }
+
+        Header[] requestHeaders = new Header[headers.size()];
+        int index = 0;
+        for (Map.Entry<String, String> header : headers.entrySet()) {
+            logger.info("Adding header {}\n with value {}", header.getKey(), header.getValue());
+            requestHeaders[index++] = new BasicHeader(header.getKey(), header.getValue());
+        }
+
+        logger.debug("calling api [{}]", apiName);
+        try {
+            ElasticsearchResponse response = restClient.performRequest(requestMethod, requestPath,
+                    queryStringParams, requestBody, requestHeaders);
+            return new RestTestResponse(response);
+        } catch(ElasticsearchResponseException e) {
+            if (ignores.contains(e.getElasticsearchResponse().getStatusLine().getStatusCode())) {
+                return new RestTestResponse(e);
+            }
+            throw e;
+        }
     }
 
     private RestApi restApi(String apiName) {
@@ -263,21 +267,7 @@ public class RestClient implements Closeable {
         return restApi;
     }
 
-    protected HttpRequestBuilder httpRequestBuilder(URL url) {
-        return new HttpRequestBuilder(httpClient)
-                .addHeaders(threadContext.getHeaders())
-                .protocol(protocol)
-                .host(url.getHost())
-                .port(url.getPort());
-    }
-
-    protected HttpRequestBuilder httpRequestBuilder() {
-        //the address used is randomized between the available ones
-        URL url = RandomizedTest.randomFrom(urls);
-        return httpRequestBuilder(url);
-    }
-
-    protected CloseableHttpClient createHttpClient(Settings settings) throws IOException {
+    protected RestClient createRestClient(URL[] urls, Settings settings) throws IOException {
         SSLConnectionSocketFactory sslsf;
         String keystorePath = settings.get(TRUSTSTORE_PATH);
         if (keystorePath != null) {
@@ -307,8 +297,24 @@ public class RestClient implements Closeable {
                 .register("http", PlainConnectionSocketFactory.getSocketFactory())
                 .register("https", sslsf)
                 .build();
-        return HttpClients.createMinimal(
-                new PoolingHttpClientConnectionManager(socketFactoryRegistry, null, null, null, 15, TimeUnit.SECONDS));
+
+        List<Header> headers = new ArrayList<>();
+        try (ThreadContext threadContext = new ThreadContext(settings)) {
+            for (Map.Entry<String, String> entry : threadContext.getHeaders().entrySet()) {
+                headers.add(new BasicHeader(entry.getKey(), entry.getValue()));
+            }
+        }
+
+        CloseableHttpClient httpClient = HttpClientBuilder.create().setDefaultHeaders(headers)
+                .setConnectionManager(new PoolingHttpClientConnectionManager(socketFactoryRegistry)).build();
+
+        String protocol = settings.get(PROTOCOL, "http");
+        HttpHost[] hosts = new HttpHost[urls.length];
+        for (int i = 0; i < hosts.length; i++) {
+            URL url = urls[i];
+            hosts[i] = new HttpHost(url.getHost(), url.getPort(), protocol);
+        }
+        return RestClient.builder().setHttpClient(httpClient).setHosts(hosts).build();
     }
 
     /**
@@ -316,6 +322,6 @@ public class RestClient implements Closeable {
      */
     @Override
     public void close() {
-        IOUtils.closeWhileHandlingException(httpClient);
+        IOUtils.closeWhileHandlingException(restClient);
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestResponse.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java
similarity index 62%
rename from test/framework/src/main/java/org/elasticsearch/test/rest/client/RestResponse.java
rename to test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java
index e0b2f3ab729..abcae6b26a2 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestResponse.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java
@@ -18,31 +18,54 @@
  */
 package org.elasticsearch.test.rest.client;
 
+import org.apache.http.client.methods.HttpHead;
+import org.apache.http.util.EntityUtils;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.client.ElasticsearchResponse;
+import org.elasticsearch.client.ElasticsearchResponseException;
 import org.elasticsearch.test.rest.Stash;
-import org.elasticsearch.test.rest.client.http.HttpResponse;
 import org.elasticsearch.test.rest.json.JsonPath;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 
 /**
- * Response obtained from a REST call
- * Supports parsing the response body as json when needed and returning specific values extracted from it
+ * Response obtained from a REST call, eagerly reads the response body into a string for later optional parsing.
+ * Supports parsing the response body as json when needed and returning specific values extracted from it.
  */
-public class RestResponse {
+public class RestTestResponse {
 
-    private final HttpResponse response;
+    private final ElasticsearchResponse response;
+    private final String body;
     private JsonPath parsedResponse;
 
-    public RestResponse(HttpResponse response) {
+    public RestTestResponse(ElasticsearchResponse response) {
         this.response = response;
+        if (response.getEntity() != null) {
+            try {
+                this.body = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8);
+            } catch (IOException e) {
+                EntityUtils.consumeQuietly(response.getEntity());
+                throw new RuntimeException(e);
+            } finally {
+                IOUtils.closeWhileHandlingException(response);
+            }
+        } else {
+            this.body = null;
+        }
+    }
+
+    public RestTestResponse(ElasticsearchResponseException responseException) {
+        this.response = responseException.getElasticsearchResponse();
+        this.body = responseException.getResponseBody();
     }
 
     public int getStatusCode() {
-        return response.getStatusCode();
+        return response.getStatusLine().getStatusCode();
     }
 
     public String getReasonPhrase() {
-        return response.getReasonPhrase();
+        return response.getStatusLine().getReasonPhrase();
     }
 
     /**
@@ -57,18 +80,18 @@ public class RestResponse {
             }
             return parsedResponse.evaluate("");
         }
-        return response.getBody();
+        return body;
     }
 
     /**
      * Returns the body as a string
      */
     public String getBodyAsString() {
-        return response.getBody();
+        return body;
     }
 
     public boolean isError() {
-        return response.isError();
+        return response.getStatusLine().getStatusCode() >= 400;
     }
 
     /**
@@ -82,7 +105,6 @@ public class RestResponse {
      * Parses the response body as json and extracts a specific value from it (identified by the provided path)
      */
     public Object evaluate(String path, Stash stash) throws IOException {
-
         if (response == null) {
             return null;
         }
@@ -93,8 +115,8 @@ public class RestResponse {
             //special case: api that don't support body (e.g. exists) return true if 200, false if 404, even if no body
             //is_true: '' means the response had no body but the client returned true (caused by 200)
             //is_false: '' means the response had no body but the client returned false (caused by 404)
-            if ("".equals(path) && !response.supportsBody()) {
-                return !response.isError();
+            if ("".equals(path) && HttpHead.METHOD_NAME.equals(response.getRequestLine().getMethod())) {
+                return isError() == false;
             }
             return null;
         }
@@ -103,7 +125,7 @@ public class RestResponse {
     }
 
     private boolean isJson() {
-        String contentType = response.getHeaders().get("Content-Type");
+        String contentType = response.getFirstHeader("Content-Type");
         return contentType != null && contentType.contains("application/json");
     }
 
@@ -111,9 +133,9 @@ public class RestResponse {
         if (parsedResponse != null) {
             return parsedResponse;
         }
-        if (response == null || !response.hasBody()) {
+        if (response == null || body == null) {
             return null;
         }
-        return parsedResponse = new JsonPath(response.getBody());
+        return parsedResponse = new JsonPath(body);
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpDeleteWithEntity.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpDeleteWithEntity.java
deleted file mode 100644
index 480fc7b2f01..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpDeleteWithEntity.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.client.http;
-
-import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
-
-import java.net.URI;
-
-/**
- * Allows to send DELETE requests providing a body (not supported out of the box)
- */
-public class HttpDeleteWithEntity extends HttpEntityEnclosingRequestBase {
-
-    public final static String METHOD_NAME = "DELETE";
-
-    public HttpDeleteWithEntity(final URI uri) {
-        setURI(uri);
-    }
-
-    @Override
-    public String getMethod() {
-        return METHOD_NAME;
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpGetWithEntity.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpGetWithEntity.java
deleted file mode 100644
index aa0129f4660..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpGetWithEntity.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.client.http;
-
-import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
-
-import java.net.URI;
-
-/**
- * Allows to send GET requests providing a body (not supported out of the box)
- */
-public class HttpGetWithEntity extends HttpEntityEnclosingRequestBase {
-
-    public final static String METHOD_NAME = "GET";
-
-    public HttpGetWithEntity(final URI uri) {
-        setURI(uri);
-    }
-
-    @Override
-    public String getMethod() {
-        return METHOD_NAME;
-    }
-}
\ No newline at end of file
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpRequestBuilder.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpRequestBuilder.java
deleted file mode 100644
index 9f47111d49a..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpRequestBuilder.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.client.http;
-
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
-import org.apache.http.client.methods.HttpHead;
-import org.apache.http.client.methods.HttpOptions;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.common.network.NetworkAddress;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
-import org.elasticsearch.http.HttpServerTransport;
-
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URLEncoder;
-import java.nio.charset.Charset;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-/**
- * Executable builder for an http request
- * Holds an {@link org.apache.http.client.HttpClient} that is used to send the built http request
- */
-public class HttpRequestBuilder {
-
-    private static final ESLogger logger = Loggers.getLogger(HttpRequestBuilder.class);
-
-    static final Charset DEFAULT_CHARSET = Charset.forName("utf-8");
-
-    private final CloseableHttpClient httpClient;
-
-    private String protocol = "http";
-
-    private String host;
-
-    private int port;
-
-    private String path = "";
-
-    private final Map<String, String> params = new HashMap<>();
-
-    private final Map<String, String> headers = new HashMap<>();
-
-    private String method = HttpGetWithEntity.METHOD_NAME;
-
-    private String body;
-
-    public HttpRequestBuilder(CloseableHttpClient httpClient) {
-        this.httpClient = httpClient;
-    }
-
-    public HttpRequestBuilder host(String host) {
-        this.host = host;
-        return this;
-    }
-
-    public HttpRequestBuilder httpTransport(HttpServerTransport httpServerTransport) {
-        InetSocketTransportAddress transportAddress = (InetSocketTransportAddress) httpServerTransport.boundAddress().publishAddress();
-        return host(NetworkAddress.format(transportAddress.address().getAddress())).port(transportAddress.address().getPort());
-    }
-
-    public HttpRequestBuilder port(int port) {
-        this.port = port;
-        return this;
-    }
-
-    /**
-     * Sets the path to send the request to. Url encoding needs to be applied by the caller.
-     * Use {@link #pathParts(String...)} instead if the path needs to be encoded, part by part.
-     */
-    public HttpRequestBuilder path(String path) {
-        this.path = path;
-        return this;
-    }
-
-    /**
-     * Sets the path by providing the different parts (without slashes), which will be properly encoded.
-     */
-    public HttpRequestBuilder pathParts(String... path) {
-        //encode rules for path and query string parameters are different. We use URI to encode the path, and URLEncoder for each query string parameter (see addParam).
-        //We need to encode each path part separately though, as each one might contain slashes that need to be escaped, which needs to be done manually.
-        if (path.length == 0) {
-            this.path = "/";
-            return this;
-        }
-        StringBuilder finalPath = new StringBuilder();
-        for (String pathPart : path) {
-            try {
-                finalPath.append('/');
-                // We append "/" to the path part to handle parts that start with - or other invalid characters
-                URI uri = new URI(null, null, null, -1, "/" + pathPart, null, null);
-                //manually escape any slash that each part may contain
-                finalPath.append(uri.getRawPath().substring(1).replaceAll("/", "%2F"));
-            } catch(URISyntaxException e) {
-                throw new RuntimeException("unable to build uri", e);
-            }
-        }
-        this.path = finalPath.toString();
-        return this;
-    }
-
-    public HttpRequestBuilder addParam(String name, String value) {
-        try {
-            this.params.put(name, URLEncoder.encode(value, "utf-8"));
-            return this;
-        } catch (UnsupportedEncodingException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    public HttpRequestBuilder addHeaders(Map<String, String> headers) {
-        this.headers.putAll(headers);
-        return this;
-    }
-
-    public HttpRequestBuilder addHeader(String name, String value) {
-        this.headers.put(name, value);
-        return this;
-    }
-
-    public HttpRequestBuilder protocol(String protocol) {
-        this.protocol = protocol;
-        return this;
-    }
-
-    public HttpRequestBuilder method(String method) {
-        this.method = method;
-        return this;
-    }
-
-    public HttpRequestBuilder body(String body) {
-        if (Strings.hasLength(body)) {
-            this.body = body;
-        }
-        return this;
-    }
-
-    public HttpResponse execute() throws IOException {
-        HttpUriRequest httpUriRequest = buildRequest();
-        if (logger.isTraceEnabled()) {
-            StringBuilder stringBuilder = new StringBuilder(httpUriRequest.getMethod()).append(" ").append(httpUriRequest.getURI());
-            if (Strings.hasLength(body)) {
-                stringBuilder.append("\n").append(body);
-            }
-            logger.trace("sending request \n{}", stringBuilder.toString());
-        }
-        for (Map.Entry<String, String> entry : this.headers.entrySet()) {
-            logger.trace("adding header [{} => {}]", entry.getKey(), entry.getValue());
-            httpUriRequest.addHeader(entry.getKey(), entry.getValue());
-        }
-        try (CloseableHttpResponse closeableHttpResponse = httpClient.execute(httpUriRequest)) {
-            HttpResponse httpResponse = new HttpResponse(httpUriRequest, closeableHttpResponse);
-            logger.trace("got response \n{}\n{}", closeableHttpResponse, httpResponse.hasBody() ? httpResponse.getBody() : "");
-            return httpResponse;
-        }
-    }
-
-    private HttpUriRequest buildRequest() {
-
-        if (HttpGetWithEntity.METHOD_NAME.equalsIgnoreCase(method)) {
-            return addOptionalBody(new HttpGetWithEntity(buildUri()));
-        }
-
-        if (HttpHead.METHOD_NAME.equalsIgnoreCase(method)) {
-            checkBodyNotSupported();
-            return new HttpHead(buildUri());
-        }
-
-        if (HttpOptions.METHOD_NAME.equalsIgnoreCase(method)) {
-            checkBodyNotSupported();
-            return new HttpOptions(buildUri());
-        }
-
-        if (HttpDeleteWithEntity.METHOD_NAME.equalsIgnoreCase(method)) {
-            return addOptionalBody(new HttpDeleteWithEntity(buildUri()));
-        }
-
-        if (HttpPut.METHOD_NAME.equalsIgnoreCase(method)) {
-            return addOptionalBody(new HttpPut(buildUri()));
-        }
-
-        if (HttpPost.METHOD_NAME.equalsIgnoreCase(method)) {
-            return addOptionalBody(new HttpPost(buildUri()));
-        }
-
-        throw new UnsupportedOperationException("method [" + method + "] not supported");
-    }
-
-    private URI buildUri() {
-        StringBuilder uriBuilder = new StringBuilder(protocol).append("://").append(host).append(":").append(port).append(path);
-        if (params.size() > 0) {
-            uriBuilder.append("?").append(params.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("&")));
-        }
-        //using this constructor no url encoding happens, as we did everything upfront in addParam and pathPart methods
-        return URI.create(uriBuilder.toString());
-    }
-
-    private HttpEntityEnclosingRequestBase addOptionalBody(HttpEntityEnclosingRequestBase requestBase) {
-        if (Strings.hasText(body)) {
-            requestBase.setEntity(new StringEntity(body, DEFAULT_CHARSET));
-        }
-        return requestBase;
-    }
-
-    private void checkBodyNotSupported() {
-        if (Strings.hasText(body)) {
-            throw new IllegalArgumentException("request body not supported with head request");
-        }
-    }
-
-    @Override
-    public String toString() {
-        StringBuilder stringBuilder = new StringBuilder(method).append(" '")
-                .append(host).append(":").append(port).append(path).append("'");
-        if (!params.isEmpty()) {
-            stringBuilder.append(", params=").append(params);
-        }
-        if (Strings.hasLength(body)) {
-            stringBuilder.append(", body=\n").append(body);
-        }
-        return stringBuilder.toString();
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpResponse.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpResponse.java
deleted file mode 100644
index 37fc163ac61..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/http/HttpResponse.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.client.http;
-
-import org.apache.http.Header;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpHead;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.util.EntityUtils;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.Loggers;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Response obtained from an http request
- * Always consumes the whole response body loading it entirely into a string
- */
-public class HttpResponse {
-
-    private static final ESLogger logger = Loggers.getLogger(HttpResponse.class);
-
-    private final HttpUriRequest httpRequest;
-    private final int statusCode;
-    private final String reasonPhrase;
-    private final String body;
-    private final Map<String, String> headers = new HashMap<>();
-
-    HttpResponse(HttpUriRequest httpRequest, CloseableHttpResponse httpResponse) {
-        this.httpRequest = httpRequest;
-        this.statusCode = httpResponse.getStatusLine().getStatusCode();
-        this.reasonPhrase = httpResponse.getStatusLine().getReasonPhrase();
-        for (Header header : httpResponse.getAllHeaders()) {
-            this.headers.put(header.getName(), header.getValue());
-        }
-        if (httpResponse.getEntity() != null) {
-            try {
-                this.body = EntityUtils.toString(httpResponse.getEntity(), HttpRequestBuilder.DEFAULT_CHARSET);
-            } catch (IOException e) {
-                EntityUtils.consumeQuietly(httpResponse.getEntity());
-                throw new RuntimeException(e);
-            } finally {
-                try {
-                    httpResponse.close();
-                } catch (IOException e) {
-                    logger.error("Failed closing response", e);
-                }
-            }
-        } else {
-            this.body = null;
-        }
-    }
-
-    public boolean isError() {
-        return statusCode >= 400;
-    }
-
-    public int getStatusCode() {
-        return statusCode;
-    }
-
-    public String getReasonPhrase() {
-        return reasonPhrase;
-    }
-
-    public String getBody() {
-        return body;
-    }
-
-    public boolean hasBody() {
-        return body != null;
-    }
-
-    public boolean supportsBody() {
-        return !HttpHead.METHOD_NAME.equals(httpRequest.getMethod());
-    }
-
-    public Map<String, String> getHeaders() {
-        return headers;
-    }
-
-    @Override
-    public String toString() {
-        StringBuilder stringBuilder = new StringBuilder(statusCode).append(" ").append(reasonPhrase);
-        if (hasBody()) {
-            stringBuilder.append("\n").append(body);
-        }
-        return stringBuilder.toString();
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/DoSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/DoSection.java
index 7b392a4043e..76593d4af9d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/DoSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/DoSection.java
@@ -18,13 +18,13 @@
  */
 package org.elasticsearch.test.rest.section;
 
+import org.elasticsearch.client.ElasticsearchResponseException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.test.rest.RestTestExecutionContext;
-import org.elasticsearch.test.rest.client.RestException;
-import org.elasticsearch.test.rest.client.RestResponse;
+import org.elasticsearch.test.rest.client.RestTestResponse;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -89,7 +89,7 @@ public class DoSection implements ExecutableSection {
         }
 
         try {
-            RestResponse restResponse = executionContext.callApi(apiCallSection.getApi(), apiCallSection.getParams(),
+            RestTestResponse restTestResponse = executionContext.callApi(apiCallSection.getApi(), apiCallSection.getParams(),
                     apiCallSection.getBodies(), apiCallSection.getHeaders());
             if (Strings.hasLength(catchParam)) {
                 String catchStatusCode;
@@ -100,16 +100,18 @@ public class DoSection implements ExecutableSection {
                 } else {
                     throw new UnsupportedOperationException("catch value [" + catchParam + "] not supported");
                 }
-                fail(formatStatusCodeMessage(restResponse, catchStatusCode));
+                fail(formatStatusCodeMessage(restTestResponse, catchStatusCode));
             }
-        } catch(RestException e) {
+        } catch(ElasticsearchResponseException e) {
+            RestTestResponse restTestResponse = new RestTestResponse(e);
             if (!Strings.hasLength(catchParam)) {
-                fail(formatStatusCodeMessage(e.restResponse(), "2xx"));
+                fail(formatStatusCodeMessage(restTestResponse, "2xx"));
             } else if (catches.containsKey(catchParam)) {
-                assertStatusCode(e.restResponse());
+                assertStatusCode(restTestResponse);
             } else if (catchParam.length() > 2 && catchParam.startsWith("/") && catchParam.endsWith("/")) {
                 //the text of the error message matches regular expression
-                assertThat(formatStatusCodeMessage(e.restResponse(), "4xx|5xx"), e.statusCode(), greaterThanOrEqualTo(400));
+                assertThat(formatStatusCodeMessage(restTestResponse, "4xx|5xx"),
+                        e.getElasticsearchResponse().getStatusLine().getStatusCode(), greaterThanOrEqualTo(400));
                 Object error = executionContext.response("error");
                 assertThat("error was expected in the response", error, notNullValue());
                 //remove delimiters from regex
@@ -122,19 +124,19 @@ public class DoSection implements ExecutableSection {
         }
     }
 
-    private void assertStatusCode(RestResponse restResponse) {
+    private void assertStatusCode(RestTestResponse restTestResponse) {
         Tuple<String, org.hamcrest.Matcher<Integer>> stringMatcherTuple = catches.get(catchParam);
-        assertThat(formatStatusCodeMessage(restResponse, stringMatcherTuple.v1()),
-                restResponse.getStatusCode(), stringMatcherTuple.v2());
+        assertThat(formatStatusCodeMessage(restTestResponse, stringMatcherTuple.v1()),
+                restTestResponse.getStatusCode(), stringMatcherTuple.v2());
     }
 
-    private String formatStatusCodeMessage(RestResponse restResponse, String expected) {
+    private String formatStatusCodeMessage(RestTestResponse restTestResponse, String expected) {
         String api = apiCallSection.getApi();
         if ("raw".equals(api)) {
             api += "[method=" + apiCallSection.getParams().get("method") + " path=" + apiCallSection.getParams().get("path") + "]";
         }
-        return "expected [" + expected + "] status code but api [" + api + "] returned ["
-                + restResponse.getStatusCode() + " " + restResponse.getReasonPhrase() + "] [" + restResponse.getBodyAsString() + "]";
+        return "expected [" + expected + "] status code but api [" + api + "] returned [" + restTestResponse.getStatusCode() +
+                " " + restTestResponse.getReasonPhrase() + "] [" + restTestResponse.getBodyAsString() + "]";
     }
 
     private static Map<String, Tuple<String, org.hamcrest.Matcher<Integer>>> catches = new HashMap<>();
@@ -145,6 +147,7 @@ public class DoSection implements ExecutableSection {
         catches.put("forbidden", tuple("403", equalTo(403)));
         catches.put("request_timeout", tuple("408", equalTo(408)));
         catches.put("unavailable", tuple("503", equalTo(503)));
-        catches.put("request", tuple("4xx|5xx", allOf(greaterThanOrEqualTo(400), not(equalTo(404)), not(equalTo(408)), not(equalTo(409)), not(equalTo(403)))));
+        catches.put("request", tuple("4xx|5xx",
+                allOf(greaterThanOrEqualTo(400), not(equalTo(404)), not(equalTo(408)), not(equalTo(409)), not(equalTo(403)))));
     }
 }

From c70e08c393943e136970a9ef5afde75334341ece Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 20 May 2016 11:03:43 +0200
Subject: [PATCH 0612/1311] include response body in
 ElasticsearchResponseException error message

---
 .../ElasticsearchResponseException.java       | 36 +++++++++----------
 .../org/elasticsearch/client/RestClient.java  | 14 +++++++-
 2 files changed, 29 insertions(+), 21 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
index ad63009db34..12aef4f0d65 100644
--- a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
+++ b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
@@ -19,41 +19,37 @@
 
 package org.elasticsearch.client;
 
-import org.apache.http.HttpHost;
-import org.apache.http.RequestLine;
-import org.apache.http.StatusLine;
-import org.apache.http.util.EntityUtils;
-
 import java.io.IOException;
 
 /**
- * Exception thrown when an elasticsearch node responds to a request with a status code that indicates an error
+ * Exception thrown when an elasticsearch node responds to a request with a status code that indicates an error.
+ * Note that the response body gets passed in as a string and read eagerly, which means that the ElasticsearchResponse object
+ * is expected to be closed and available only to read metadata like status line, request line, response headers.
  */
 public class ElasticsearchResponseException extends IOException {
 
     private ElasticsearchResponse elasticsearchResponse;
     private final String responseBody;
 
-    public ElasticsearchResponseException(ElasticsearchResponse elasticsearchResponse) throws IOException {
-        super(buildMessage(elasticsearchResponse.getRequestLine(), elasticsearchResponse.getHost(), elasticsearchResponse.getStatusLine()));
+    public ElasticsearchResponseException(ElasticsearchResponse elasticsearchResponse, String responseBody) throws IOException {
+        super(buildMessage(elasticsearchResponse,responseBody));
         this.elasticsearchResponse = elasticsearchResponse;
-        try {
-            if (elasticsearchResponse.getEntity() == null) {
-                this.responseBody = null;
-            } else {
-                this.responseBody = EntityUtils.toString(elasticsearchResponse.getEntity());
-            }
-        } finally {
-            elasticsearchResponse.close();
-        }
+        this.responseBody = responseBody;
     }
 
-    private static String buildMessage(RequestLine requestLine, HttpHost host, StatusLine statusLine) {
-        return requestLine.getMethod() + " " + host + requestLine.getUri() + ": " + statusLine.toString();
+    private static String buildMessage(ElasticsearchResponse response, String responseBody) {
+        String message = response.getRequestLine().getMethod() + " " + response.getHost() + response.getRequestLine().getUri()
+                + ": " + response.getStatusLine().toString();
+        if (responseBody != null) {
+            message += "\n" + responseBody;
+        }
+        return message;
     }
 
     /**
-     * Returns the {@link ElasticsearchResponse} that caused this exception to be thrown
+     * Returns the {@link ElasticsearchResponse} that caused this exception to be thrown.
+     * Expected to be used only to read metadata like status line, request line, response headers. The response body should
+     * be retrieved using {@link #getResponseBody()}
      */
     public ElasticsearchResponse getElasticsearchResponse() {
         return elasticsearchResponse;
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index b366d98f77a..e115b8bc3b5 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -37,6 +37,7 @@ import org.apache.http.entity.ContentType;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.util.EntityUtils;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -127,7 +128,18 @@ public final class RestClient implements Closeable {
                 return elasticsearchResponse;
             } else {
                 RequestLogger.log(logger, "request failed", request, connection.getHost(), response);
-                ElasticsearchResponseException elasticsearchResponseException = new ElasticsearchResponseException(elasticsearchResponse);
+                String responseBody;
+                try {
+                    if (elasticsearchResponse.getEntity() == null) {
+                        responseBody = null;
+                    } else {
+                        responseBody = EntityUtils.toString(elasticsearchResponse.getEntity());
+                    }
+                } finally {
+                    elasticsearchResponse.close();
+                }
+                ElasticsearchResponseException elasticsearchResponseException = new ElasticsearchResponseException(
+                        elasticsearchResponse, responseBody);
                 lastSeenException = addSuppressedException(lastSeenException, elasticsearchResponseException);
                 //clients don't retry on 500 because elasticsearch still misuses it instead of 400 in some places
                 if (statusCode == 502 || statusCode == 503 || statusCode == 504) {

From e81aad972a6921a98cce3f5e2354aaf17e8b828a Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 20 May 2016 13:30:30 +0200
Subject: [PATCH 0613/1311] remove usage of deprecated api

---
 .../java/org/elasticsearch/test/rest/client/RestTestClient.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java
index 3018af869b3..5baedf8d060 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java
@@ -27,12 +27,12 @@ import org.apache.http.config.RegistryBuilder;
 import org.apache.http.conn.socket.ConnectionSocketFactory;
 import org.apache.http.conn.socket.PlainConnectionSocketFactory;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
-import org.apache.http.conn.ssl.SSLContexts;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.apache.http.message.BasicHeader;
+import org.apache.http.ssl.SSLContexts;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.client.ElasticsearchResponse;

From 6490355cb6585cac6f460d7c68417d1e757ef2c6 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Wed, 25 May 2016 18:37:10 +0200
Subject: [PATCH 0614/1311] make host state immutable

Instead of having a Connection mutable object that holds the state of the connection to each host, we now have immutable objects only. We keep two sets, one with all the hosts, one with the blacklisted ones. Once we blacklist a host we associate it with a DeadHostState which keeps track of the number of failed attempts and when the host should be retried. A new state object is created each and every time the state of the host needs to be updated.
---
 .../org/elasticsearch/client/Connection.java  |  94 ------------
 .../elasticsearch/client/DeadHostState.java   |  58 ++++++++
 .../org/elasticsearch/client/RestClient.java  | 137 +++++++++++-------
 .../elasticsearch/client/sniff/Sniffer.java   |   7 +-
 .../client/sniff/SnifferBuilderTests.java     |   4 +-
 5 files changed, 144 insertions(+), 156 deletions(-)
 delete mode 100644 client/src/main/java/org/elasticsearch/client/Connection.java
 create mode 100644 client/src/main/java/org/elasticsearch/client/DeadHostState.java

diff --git a/client/src/main/java/org/elasticsearch/client/Connection.java b/client/src/main/java/org/elasticsearch/client/Connection.java
deleted file mode 100644
index 1f604483601..00000000000
--- a/client/src/main/java/org/elasticsearch/client/Connection.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.client;
-
-import org.apache.http.HttpHost;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * Represents a connection to a host. It holds the host that the connection points to and the state of the connection to it.
- */
-public class Connection {
-    private static final long DEFAULT_CONNECTION_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(1);
-    private static final long MAX_CONNECTION_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(30);
-    private final HttpHost host;
-    private volatile int failedAttempts = 0;
-    private volatile long deadUntil = -1;
-
-    /**
-     * Creates a new connection pointing to the provided {@link HttpHost} argument
-     */
-    public Connection(HttpHost host) {
-            this.host = host;
-        }
-
-    /**
-     * Returns the {@link HttpHost} that the connection points to
-     */
-    public HttpHost getHost() {
-            return host;
-        }
-
-    /**
-     * Marks connection as dead. Should be called in case the corresponding node is not responding or caused failures.
-     * Once marked dead, the number of failed attempts will be incremented on each call to this method. A dead connection
-     * should be retried once {@link #isBlacklisted()} returns true, which depends on the number of previous failed attempts
-     * and when the last failure was registered.
-     */
-    void markDead() {
-        synchronized (this) {
-            int failedAttempts = Math.max(this.failedAttempts, 0);
-            long timeoutMillis = (long)Math.min(DEFAULT_CONNECTION_TIMEOUT_MILLIS * 2 * Math.pow(2, failedAttempts * 0.5 - 1),
-                    MAX_CONNECTION_TIMEOUT_MILLIS);
-            this.deadUntil = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(timeoutMillis);
-            this.failedAttempts = ++failedAttempts;
-        }
-    }
-
-    /**
-     * Marks this connection alive. Should be called when the corresponding node is working properly.
-     * Will reset the number of failed attempts that were counted in case the connection was previously dead, as well as its timeout.
-     */
-    void markAlive() {
-        if (this.failedAttempts > 0) {
-            synchronized (this) {
-                this.deadUntil = -1;
-                this.failedAttempts = 0;
-            }
-        }
-    }
-
-    /**
-     * Returns the timestamp till the connection is supposed to stay dead. After that moment the connection should be retried
-     */
-    public long getDeadUntil() {
-        return deadUntil;
-    }
-
-    /**
-     * Returns true when the connection should be skipped due to previous failures, false in case the connection is alive
-     * or dead but ready to be retried. When the connection is dead, returns false when it is time to retry it, depending
-     * on how many failed attempts were registered and when the last failure happened (minimum 1 minute, maximum 30 minutes).
-     */
-    public boolean isBlacklisted() {
-        return failedAttempts > 0 && System.nanoTime() - deadUntil < 0;
-    }
-}
diff --git a/client/src/main/java/org/elasticsearch/client/DeadHostState.java b/client/src/main/java/org/elasticsearch/client/DeadHostState.java
new file mode 100644
index 00000000000..30a24bc2d5d
--- /dev/null
+++ b/client/src/main/java/org/elasticsearch/client/DeadHostState.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Holds the state of a dead connection to a host. Keeps track of how many failed attempts were performed and
+ * when the host should be retried (based on number of previous failed attempts).
+ * Class is immutable, a new copy of it should be created each time the state has to be changed.
+ */
+class DeadHostState {
+
+    private static final long MIN_CONNECTION_TIMEOUT_NANOS = TimeUnit.MINUTES.toNanos(1);
+    private static final long MAX_CONNECTION_TIMEOUT_NANOS = TimeUnit.MINUTES.toNanos(30);
+
+    static final DeadHostState INITIAL_DEAD_STATE = new DeadHostState();
+
+    private final int failedAttempts;
+    private final long deadUntil;
+
+    private DeadHostState() {
+        this.failedAttempts = 1;
+        this.deadUntil = System.nanoTime() + MIN_CONNECTION_TIMEOUT_NANOS;
+    }
+
+    DeadHostState(DeadHostState previousDeadHostState) {
+        long timeoutNanos = (long)Math.min(MIN_CONNECTION_TIMEOUT_NANOS * 2 * Math.pow(2, previousDeadHostState.failedAttempts * 0.5 - 1),
+                MAX_CONNECTION_TIMEOUT_NANOS);
+        this.deadUntil = System.nanoTime() + timeoutNanos;
+        this.failedAttempts = previousDeadHostState.failedAttempts + 1;
+    }
+
+    /**
+     * Returns the timestamp (nanos) till the host is supposed to stay dead without being retried.
+     * After that the host should be retried.
+     */
+    long getDeadUntil() {
+        return deadUntil;
+    }
+}
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index e115b8bc3b5..4fc8d3c2741 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -47,11 +47,15 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -62,23 +66,25 @@ public final class RestClient implements Closeable {
 
     private final CloseableHttpClient client;
     private final long maxRetryTimeout;
-    private final AtomicInteger lastConnectionIndex = new AtomicInteger(0);
-    private volatile List<Connection> connections;
+    private final AtomicInteger lastHostIndex = new AtomicInteger(0);
+    private volatile Set<HttpHost> hosts;
+    private final ConcurrentMap<HttpHost, DeadHostState> blacklist = new ConcurrentHashMap<>();
     private volatile FailureListener failureListener = new FailureListener();
 
     private RestClient(CloseableHttpClient client, long maxRetryTimeout, HttpHost... hosts) {
         this.client = client;
         this.maxRetryTimeout = maxRetryTimeout;
-        setNodes(hosts);
+        setHosts(hosts);
     }
 
-    public synchronized void setNodes(HttpHost... hosts) {
-        List<Connection> connections = new ArrayList<>(hosts.length);
+    public synchronized void setHosts(HttpHost... hosts) {
+        Set<HttpHost> httpHosts = new HashSet<>();
         for (HttpHost host : hosts) {
             Objects.requireNonNull(host, "host cannot be null");
-            connections.add(new Connection(host));
+            httpHosts.add(host);
         }
-        this.connections = Collections.unmodifiableList(connections);
+        this.hosts = Collections.unmodifiableSet(httpHosts);
+        this.blacklist.clear();
     }
 
     public ElasticsearchResponse performRequest(String method, String endpoint, Map<String, String> params,
@@ -94,9 +100,9 @@ public final class RestClient implements Closeable {
         long retryTimeout = Math.round(this.maxRetryTimeout / (float)100 * 98);
         IOException lastSeenException = null;
         long startTime = System.nanoTime();
-        Iterator<Connection> connectionIterator = nextConnection();
-        while (connectionIterator.hasNext()) {
-            Connection connection = connectionIterator.next();
+        Iterator<HttpHost> hostIterator = nextHost();
+        while (hostIterator.hasNext()) {
+            HttpHost host = hostIterator.next();
 
             if (lastSeenException != null) {
                 long timeElapsed = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
@@ -112,22 +118,22 @@ public final class RestClient implements Closeable {
 
             CloseableHttpResponse response;
             try {
-                response = client.execute(connection.getHost(), request);
+                response = client.execute(host, request);
             } catch(IOException e) {
-                RequestLogger.log(logger, "request failed", request, connection.getHost(), e);
-                onFailure(connection);
+                RequestLogger.log(logger, "request failed", request, host, e);
+                onFailure(host);
                 lastSeenException = addSuppressedException(lastSeenException, e);
                 continue;
             }
             ElasticsearchResponse elasticsearchResponse = new ElasticsearchResponse(request.getRequestLine(),
-                    connection.getHost(), response);
+                    host, response);
             int statusCode = response.getStatusLine().getStatusCode();
             if (statusCode < 300 || (request.getMethod().equals(HttpHead.METHOD_NAME) && statusCode == 404) ) {
-                RequestLogger.log(logger, "request succeeded", request, connection.getHost(), response);
-                onSuccess(connection);
+                RequestLogger.log(logger, "request succeeded", request, host, response);
+                onSuccess(host);
                 return elasticsearchResponse;
             } else {
-                RequestLogger.log(logger, "request failed", request, connection.getHost(), response);
+                RequestLogger.log(logger, "request failed", request, host, response);
                 String responseBody;
                 try {
                     if (elasticsearchResponse.getEntity() == null) {
@@ -143,10 +149,10 @@ public final class RestClient implements Closeable {
                 lastSeenException = addSuppressedException(lastSeenException, elasticsearchResponseException);
                 //clients don't retry on 500 because elasticsearch still misuses it instead of 400 in some places
                 if (statusCode == 502 || statusCode == 503 || statusCode == 504) {
-                    onFailure(connection);
+                    onFailure(host);
                 } else {
                     //don't retry and call onSuccess as the error should be a request problem, the node is alive
-                    onSuccess(connection);
+                    onSuccess(host);
                     break;
                 }
             }
@@ -156,63 +162,78 @@ public final class RestClient implements Closeable {
     }
 
     /**
-     * Returns an iterator of connections that should be used for a request call.
-     * Ideally, the first connection is retrieved from the iterator and used successfully for the request.
-     * Otherwise, after each failure the next connection should be retrieved from the iterator so that the request can be retried.
-     * The maximum total of attempts is equal to the number of connections that are available in the iterator.
-     * The iterator returned will never be empty, rather an {@link IllegalStateException} will be thrown in that case.
-     * In case there are no alive connections available, or dead ones that should be retried, one dead connection
-     * gets resurrected and returned.
+     * Returns an iterator of hosts to be used for a request call.
+     * Ideally, the first host is retrieved from the iterator and used successfully for the request.
+     * Otherwise, after each failure the next host should be retrieved from the iterator so that the request can be retried till
+     * the iterator is exhausted. The maximum total of attempts is equal to the number of hosts that are available in the iterator.
+     * The iterator returned will never be empty, rather an {@link IllegalStateException} in case there are no hosts.
+     * In case there are no healthy hosts available, or dead ones to be be retried, one dead host gets returned.
      */
-    private Iterator<Connection> nextConnection() {
-        if (this.connections.isEmpty()) {
-            throw new IllegalStateException("no connections available");
+    private Iterator<HttpHost> nextHost() {
+        if (this.hosts.isEmpty()) {
+            throw new IllegalStateException("no hosts available");
         }
 
-        List<Connection> rotatedConnections = new ArrayList<>(connections);
-        //TODO is it possible to make this O(1)? (rotate is O(n))
-        Collections.rotate(rotatedConnections, rotatedConnections.size() - lastConnectionIndex.getAndIncrement());
-        Iterator<Connection> connectionIterator = rotatedConnections.iterator();
-        while (connectionIterator.hasNext()) {
-            Connection connection = connectionIterator.next();
-            if (connection.isBlacklisted()) {
-                connectionIterator.remove();
+        Set<HttpHost> filteredHosts = new HashSet<>(hosts);
+        for (Map.Entry<HttpHost, DeadHostState> entry : blacklist.entrySet()) {
+            if (System.nanoTime() - entry.getValue().getDeadUntil() < 0) {
+                filteredHosts.remove(entry.getKey());
             }
         }
-        if (rotatedConnections.isEmpty()) {
-            List<Connection> sortedConnections = new ArrayList<>(connections);
-            Collections.sort(sortedConnections, new Comparator<Connection>() {
+
+        if (filteredHosts.isEmpty()) {
+            //last resort: if there are no good hosts to use, return a single dead one, the one that's closest to being retried
+            List<Map.Entry<HttpHost, DeadHostState>> sortedHosts = new ArrayList<>(blacklist.entrySet());
+            Collections.sort(sortedHosts, new Comparator<Map.Entry<HttpHost, DeadHostState>>() {
                 @Override
-                public int compare(Connection o1, Connection o2) {
-                    return Long.compare(o1.getDeadUntil(), o2.getDeadUntil());
+                public int compare(Map.Entry<HttpHost, DeadHostState> o1, Map.Entry<HttpHost, DeadHostState> o2) {
+                    return Long.compare(o1.getValue().getDeadUntil(), o2.getValue().getDeadUntil());
                 }
             });
-            Connection connection = sortedConnections.get(0);
-            logger.trace("trying to resurrect connection for " + connection.getHost());
-            return Collections.singleton(connection).iterator();
+            HttpHost deadHost = sortedHosts.get(0).getKey();
+            logger.trace("resurrecting host [" + deadHost + "]");
+            return Collections.singleton(deadHost).iterator();
         }
-        return rotatedConnections.iterator();
+
+        List<HttpHost> rotatedHosts = new ArrayList<>(filteredHosts);
+        //TODO is it possible to make this O(1)? (rotate is O(n))
+        Collections.rotate(rotatedHosts, rotatedHosts.size() - lastHostIndex.getAndIncrement());
+        return rotatedHosts.iterator();
     }
 
     /**
      * Called after each successful request call.
-     * Receives as an argument the connection that was used for the successful request.
+     * Receives as an argument the host that was used for the successful request.
      */
-    public void onSuccess(Connection connection) {
-        connection.markAlive();
-        logger.trace("marked connection alive for " + connection.getHost());
+    private void onSuccess(HttpHost host) {
+        DeadHostState removedHost = this.blacklist.remove(host);
+        if (logger.isDebugEnabled() && removedHost != null) {
+            logger.debug("removed host [" + host + "] from blacklist");
+        }
     }
 
     /**
      * Called after each failed attempt.
-     * Receives as an argument the connection that was used for the failed attempt.
+     * Receives as an argument the host that was used for the failed attempt.
      */
-    private void onFailure(Connection connection) throws IOException {
-        connection.markDead();
-        logger.debug("marked connection dead for " + connection.getHost());
-        failureListener.onFailure(connection);
+    private void onFailure(HttpHost host) throws IOException {
+        while(true) {
+            DeadHostState previousDeadHostState = blacklist.putIfAbsent(host, DeadHostState.INITIAL_DEAD_STATE);
+            if (previousDeadHostState == null) {
+                logger.debug("added host [" + host + "] to blacklist");
+                break;
+            }
+            if (blacklist.replace(host, previousDeadHostState, new DeadHostState(previousDeadHostState))) {
+                logger.debug("updated host [" + host + "] already in blacklist");
+                break;
+            }
+        }
+        failureListener.onFailure(host);
     }
 
+    /**
+     * Sets a {@link FailureListener} to be notified each and every time a host fails
+     */
     public synchronized void setFailureListener(FailureListener failureListener) {
         this.failureListener = failureListener;
     }
@@ -397,7 +418,11 @@ public final class RestClient implements Closeable {
      * The default implementation is a no-op.
      */
     public static class FailureListener {
-        public void onFailure(Connection connection) throws IOException {
+
+        /**
+         * Notifies that the host provided as argument has just failed
+         */
+        public void onFailure(HttpHost host) throws IOException {
 
         }
     }
diff --git a/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
index ea9fc9a26ab..ae5180413a0 100644
--- a/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
+++ b/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
@@ -23,7 +23,6 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.http.HttpHost;
 import org.apache.http.impl.client.CloseableHttpClient;
-import org.elasticsearch.client.Connection;
 import org.elasticsearch.client.RestClient;
 
 import java.io.Closeable;
@@ -57,10 +56,10 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
     }
 
     @Override
-    public void onFailure(Connection connection) throws IOException {
+    public void onFailure(HttpHost host) throws IOException {
         if (sniffOnFailure) {
             //re-sniff immediately but take out the node that failed
-            task.sniffOnFailure(connection.getHost());
+            task.sniffOnFailure(host);
         }
     }
 
@@ -108,7 +107,7 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
                         sniffedNodes.remove(excludeHost);
                     }
                     logger.debug("sniffed nodes: " + sniffedNodes);
-                    this.restClient.setNodes(sniffedNodes.toArray(new HttpHost[sniffedNodes.size()]));
+                    this.restClient.setHosts(sniffedNodes.toArray(new HttpHost[sniffedNodes.size()]));
                 } catch (Throwable t) {
                     logger.error("error while sniffing nodes", t);
                 } finally {
diff --git a/client/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java b/client/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
index 6373fae00d5..868c5788120 100644
--- a/client/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
+++ b/client/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
@@ -107,8 +107,8 @@ public class SnifferBuilderTests extends LuceneTestCase {
             if (random().nextBoolean()) {
                 builder.setSniffOnFailure(random().nextBoolean());
             }
-            try (Sniffer connectionPool = builder.build()) {
-                assertNotNull(connectionPool);
+            try (Sniffer sniffer = builder.build()) {
+                assertNotNull(sniffer);
             }
         }
     }

From 29b1f8d44af081118eb53f7444a09b398dd22d00 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 27 May 2016 13:42:50 +0200
Subject: [PATCH 0615/1311] make some classes and methods package private

ElasticsearchResponseException, as well as ElasticsearchResponse, should only be created from o.e.client package.
RequestLogger should only be used from this package too.
---
 .../client/ElasticsearchResponseException.java              | 2 +-
 .../java/org/elasticsearch/client/HttpDeleteWithEntity.java | 2 +-
 .../java/org/elasticsearch/client/HttpGetWithEntity.java    | 2 +-
 .../main/java/org/elasticsearch/client/RequestLogger.java   | 6 +++---
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
index 12aef4f0d65..2e9d42797de 100644
--- a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
+++ b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
@@ -31,7 +31,7 @@ public class ElasticsearchResponseException extends IOException {
     private ElasticsearchResponse elasticsearchResponse;
     private final String responseBody;
 
-    public ElasticsearchResponseException(ElasticsearchResponse elasticsearchResponse, String responseBody) throws IOException {
+    ElasticsearchResponseException(ElasticsearchResponse elasticsearchResponse, String responseBody) throws IOException {
         super(buildMessage(elasticsearchResponse,responseBody));
         this.elasticsearchResponse = elasticsearchResponse;
         this.responseBody = responseBody;
diff --git a/client/src/main/java/org/elasticsearch/client/HttpDeleteWithEntity.java b/client/src/main/java/org/elasticsearch/client/HttpDeleteWithEntity.java
index 4f378f19c8f..85927b6f871 100644
--- a/client/src/main/java/org/elasticsearch/client/HttpDeleteWithEntity.java
+++ b/client/src/main/java/org/elasticsearch/client/HttpDeleteWithEntity.java
@@ -30,7 +30,7 @@ final class HttpDeleteWithEntity extends HttpEntityEnclosingRequestBase {
 
     final static String METHOD_NAME = HttpDelete.METHOD_NAME;
 
-    public HttpDeleteWithEntity(final URI uri) {
+    HttpDeleteWithEntity(final URI uri) {
         setURI(uri);
     }
 
diff --git a/client/src/main/java/org/elasticsearch/client/HttpGetWithEntity.java b/client/src/main/java/org/elasticsearch/client/HttpGetWithEntity.java
index 18039ba9b34..c25418387e4 100644
--- a/client/src/main/java/org/elasticsearch/client/HttpGetWithEntity.java
+++ b/client/src/main/java/org/elasticsearch/client/HttpGetWithEntity.java
@@ -30,7 +30,7 @@ final class HttpGetWithEntity extends HttpEntityEnclosingRequestBase {
 
     final static String METHOD_NAME = HttpGet.METHOD_NAME;
 
-    public HttpGetWithEntity(final URI uri) {
+    HttpGetWithEntity(final URI uri) {
         setURI(uri);
     }
 
diff --git a/client/src/main/java/org/elasticsearch/client/RequestLogger.java b/client/src/main/java/org/elasticsearch/client/RequestLogger.java
index dd9fb269717..c15f6abf04b 100644
--- a/client/src/main/java/org/elasticsearch/client/RequestLogger.java
+++ b/client/src/main/java/org/elasticsearch/client/RequestLogger.java
@@ -41,7 +41,7 @@ import java.nio.charset.StandardCharsets;
  * Helper class that exposes static methods to unify the way requests are logged.
  * Includes trace logging to log complete requests and responses in curl format.
  */
-public final class RequestLogger {
+final class RequestLogger {
 
     private static final Log tracer = LogFactory.getLog("tracer");
 
@@ -51,7 +51,7 @@ public final class RequestLogger {
     /**
      * Logs a request that yielded a response
      */
-    public static void log(Log logger, String message, HttpUriRequest request, HttpHost host, HttpResponse httpResponse) {
+    static void log(Log logger, String message, HttpUriRequest request, HttpHost host, HttpResponse httpResponse) {
         logger.debug(message + " [" + request.getMethod() + " " + host + request.getRequestLine().getUri() +
                 "] [" + httpResponse.getStatusLine() + "]");
 
@@ -77,7 +77,7 @@ public final class RequestLogger {
     /**
      * Logs a request that failed
      */
-    public static void log(Log logger, String message, HttpUriRequest request, HttpHost host, IOException e) {
+    static void log(Log logger, String message, HttpUriRequest request, HttpHost host, IOException e) {
         logger.debug(message + " [" + request.getMethod() + " " + host + request.getRequestLine().getUri() + "]", e);
         if (logger.isTraceEnabled()) {
             String traceRequest;

From 7f4807b29e46c7b294b1b8c15b16b444f0961e39 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 27 May 2016 14:08:29 +0200
Subject: [PATCH 0616/1311] add some javadocs

---
 .../org/elasticsearch/client/RestClient.java  | 35 +++++++++++++++++++
 .../client/sniff/HostsSniffer.java            |  1 +
 .../elasticsearch/client/sniff/Sniffer.java   |  9 +++--
 3 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 4fc8d3c2741..ab919ca3382 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -24,6 +24,7 @@ import org.apache.http.Consts;
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHost;
+import org.apache.http.client.ClientProtocolException;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
@@ -59,6 +60,19 @@ import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
+/**
+ * Client that connects to an elasticsearch cluster through http.
+ * Must be created using {@link Builder}, which allows to set all the different options or just rely on defaults.
+ * The hosts that are part of the cluster need to be provided at creation time, but can also be replaced later
+ * by calling {@link #setHosts(HttpHost...)}.
+ * The method {@link #performRequest(String, String, Map, HttpEntity, Header...)} allows to send a request to the cluster. When
+ * sending a request, a host gets selected out of the provided ones in a round-robin fashion. Failing hosts are marked dead and
+ * retried after a certain amount of time (minimum 1 minute, maximum 30 minutes), depending on how many times they previously
+ * failed (the more failures, the later they will be retried). In case of failures all of the alive nodes (or dead nodes that
+ * deserve a retry) are retried till one responds or none of them does, in which case an {@link IOException} will be thrown.
+ *
+ * Requests can be traced by enabling trace logging for "tracer". The trace logger outputs requests and responses in curl format.
+ */
 public final class RestClient implements Closeable {
 
     private static final Log logger = LogFactory.getLog(RestClient.class);
@@ -77,6 +91,10 @@ public final class RestClient implements Closeable {
         setHosts(hosts);
     }
 
+    /**
+     * Replaces the hosts that the client communicates with.
+     * @see HttpHost
+     */
     public synchronized void setHosts(HttpHost... hosts) {
         Set<HttpHost> httpHosts = new HashSet<>();
         for (HttpHost host : hosts) {
@@ -87,6 +105,23 @@ public final class RestClient implements Closeable {
         this.blacklist.clear();
     }
 
+    /**
+     * Sends a request to the elasticsearch cluster that the current client points to.
+     * Selects a host out of the provided ones in a round-robin fashion. Failing hosts are marked dead and retried after a certain
+     * amount of time (minimum 1 minute, maximum 30 minutes), depending on how many times they previously failed (the more failures,
+     * the later they will be retried). In case of failures all of the alive nodes (or dead nodes that deserve a retry) are retried
+     * till one responds or none of them does, in which case an {@link IOException} will be thrown.
+     *
+     * @param method the http method
+     * @param endpoint the path of the request (without host and port)
+     * @param params the query_string parameters
+     * @param entity the body of the request, null if not applicable
+     * @param headers the optional request headers
+     * @return the response returned by elasticsearch
+     * @throws IOException in case of a problem or the connection was aborted
+     * @throws ClientProtocolException in case of an http protocol error
+     * @throws ElasticsearchResponseException in case elasticsearch responded with a status code that indicated an error
+     */
     public ElasticsearchResponse performRequest(String method, String endpoint, Map<String, String> params,
                                                 HttpEntity entity, Header... headers) throws IOException {
         URI uri = buildUri(endpoint, params);
diff --git a/client/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java b/client/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
index 86dca173a67..cf49cfbe68a 100644
--- a/client/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
+++ b/client/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
@@ -40,6 +40,7 @@ import java.util.Map;
 /**
  * Class responsible for sniffing the http hosts from elasticsearch through the nodes info api and returning them back
  */
+//TODO This could potentially be using _cat/nodes which wouldn't require jackson as a dependency, but we'd have bw comp problems with 2.x
 public class HostsSniffer {
 
     private static final Log logger = LogFactory.getLog(HostsSniffer.class);
diff --git a/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
index ae5180413a0..ed6f8284f1e 100644
--- a/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
+++ b/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
@@ -37,9 +37,12 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
- * Calls nodes info api and returns a list of http hosts extracted from it.
+ * Class responsible for sniffing nodes from an elasticsearch cluster and setting them to a provided instance of {@link RestClient}.
+ * Must be created via {@link Builder}, which allows to set all of the different options or rely on defaults.
+ * A background task fetches the nodes from elasticsearch and updates them periodically.
+ * Supports sniffing on failure, meaning that the client will notify the sniffer at each host failure, so that nodes can be updated
+ * straightaway.
  */
-//TODO This could potentially be using _cat/nodes which wouldn't require jackson as a dependency, but we'd have bw comp problems with 2.x
 public final class Sniffer extends RestClient.FailureListener implements Closeable {
 
     private static final Log logger = LogFactory.getLog(Sniffer.class);
@@ -47,7 +50,7 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
     private final boolean sniffOnFailure;
     private final Task task;
 
-    public Sniffer(RestClient restClient, int sniffRequestTimeout, String scheme, int sniffInterval,
+    private Sniffer(RestClient restClient, int sniffRequestTimeout, String scheme, int sniffInterval,
                    boolean sniffOnFailure, int sniffAfterFailureDelay) {
         HostsSniffer hostsSniffer = new HostsSniffer(restClient, sniffRequestTimeout, scheme);
         this.task = new Task(hostsSniffer, restClient, sniffInterval, sniffAfterFailureDelay);

From 044a97c7406392f37e5ad28021c5a74795b98022 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 27 May 2016 16:01:33 +0200
Subject: [PATCH 0617/1311] move client sniffer to its own project

Create a new subproject called client-sniffer that contains the o.e.client.sniff package. Since it is going to go to a separate jar, due to its additional functionalities and dependency on jackson, it makes sense to have it as a separate project that depends on client. This way we make sure that client doesn't depend on it etc.
---
 build.gradle                                  |  1 +
 client-sniffer/build.gradle                   | 84 +++++++++++++++++++
 .../client/sniff/HostsSniffer.java            |  0
 .../elasticsearch/client/sniff/Sniffer.java   |  0
 .../client/sniff/HostsSnifferTests.java       |  0
 .../client/sniff/SnifferBuilderTests.java     |  0
 client/build.gradle                           |  2 -
 settings.gradle                               |  1 +
 8 files changed, 86 insertions(+), 2 deletions(-)
 create mode 100644 client-sniffer/build.gradle
 rename {client => client-sniffer}/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java (100%)
 rename {client => client-sniffer}/src/main/java/org/elasticsearch/client/sniff/Sniffer.java (100%)
 rename {client => client-sniffer}/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java (100%)
 rename {client => client-sniffer}/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java (100%)

diff --git a/build.gradle b/build.gradle
index af18424944b..bd36211b128 100644
--- a/build.gradle
+++ b/build.gradle
@@ -163,6 +163,7 @@ subprojects {
     "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec',
     "org.elasticsearch:elasticsearch:${version}": ':core',
     "org.elasticsearch:client:${version}": ':client',
+    "org.elasticsearch:client-sniffer:${version}": ':client-sniffer',
     "org.elasticsearch.test:framework:${version}": ':test:framework',
     "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:integ-test-zip',
     "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:zip',
diff --git a/client-sniffer/build.gradle b/client-sniffer/build.gradle
new file mode 100644
index 00000000000..d446ffc0113
--- /dev/null
+++ b/client-sniffer/build.gradle
@@ -0,0 +1,84 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.elasticsearch.gradle.precommit.PrecommitTasks
+import org.gradle.api.JavaVersion
+
+apply plugin: 'elasticsearch.build'
+
+targetCompatibility = JavaVersion.VERSION_1_7
+sourceCompatibility = JavaVersion.VERSION_1_7
+
+dependencies {
+  compile "org.elasticsearch:client:${version}"
+  compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
+  compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
+  compile "commons-codec:commons-codec:${versions.commonscodec}"
+  compile "commons-logging:commons-logging:${versions.commonslogging}"
+  compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
+
+  testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
+  testCompile "junit:junit:${versions.junit}"
+  testCompile "org.hamcrest:hamcrest-all:1.3"
+  testCompile "org.apache.lucene:lucene-test-framework:${versions.lucene}"
+  testCompile "org.apache.lucene:lucene-core:${versions.lucene}"
+  testCompile "org.apache.lucene:lucene-codecs:${versions.lucene}"
+  //mock web server
+  testCompile "com.squareup.okhttp3:mockwebserver:3.2.0"
+  testCompile "com.squareup.okhttp3:okhttp:3.2.0"
+  testCompile "com.squareup.okhttp3:okhttp-ws:3.2.0"
+  testCompile "com.squareup.okio:okio:1.6.0"
+  testCompile "org.bouncycastle:bcprov-jdk15on:1.54"
+}
+
+//TODO compiling from 1.8 with target 1.7 and source 1.7 is best effort, not enough to ensure we are java 7 compatible
+compileJava.options.compilerArgs << '-target' << '1.7' << '-source' << '1.7' << '-Xlint:all,-path,-serial,-options'
+compileTestJava.options.compilerArgs << '-target' << '1.7' << '-source' << '1.7'
+
+forbiddenApisMain {
+  //client does not depend on core, so only jdk signatures should be checked
+  signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
+}
+
+forbiddenApisTest {
+  //client does not depend on core, so only jdk signatures should be checked
+  signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
+}
+
+//TODO add licenses for dependencies and take care of distribution
+//dependency license are currently checked in distribution
+dependencyLicenses.enabled=false
+//JarHell is part of es core, which we don't want to pull in
+jarHell.enabled=false
+//NamingConventionCheck is part of test-framework, which we don't want to pull in as it depends on es core
+namingConventions.enabled=false
+
+thirdPartyAudit.excludes = [
+  //commons-logging optional dependencies
+  'org.apache.avalon.framework.logger.Logger',
+  'org.apache.log.Hierarchy',
+  'org.apache.log.Logger',
+  'org.apache.log4j.Category',
+  'org.apache.log4j.Level',
+  'org.apache.log4j.Logger',
+  'org.apache.log4j.Priority',
+  //commons-logging provided dependencies
+  'javax.servlet.ServletContextEvent',
+  'javax.servlet.ServletContextListener'
+]
diff --git a/client/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
similarity index 100%
rename from client/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
rename to client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
diff --git a/client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
similarity index 100%
rename from client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
rename to client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
diff --git a/client/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
similarity index 100%
rename from client/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
rename to client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
diff --git a/client/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
similarity index 100%
rename from client/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
rename to client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
diff --git a/client/build.gradle b/client/build.gradle
index e24aa9878e3..8cb0018e991 100644
--- a/client/build.gradle
+++ b/client/build.gradle
@@ -30,8 +30,6 @@ dependencies {
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
-  //jackson is only needed in the sniff package
-  compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
 
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
diff --git a/settings.gradle b/settings.gradle
index 831b2c5d2fb..dbaf9fa8aa9 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -6,6 +6,7 @@ List projects = [
   'core',
   'docs',
   'client',
+  'client-sniffer',
   'distribution:integ-test-zip',
   'distribution:zip',
   'distribution:tar',

From 3745305ffb785a331843e4bb94878efcb99c051f Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Mon, 30 May 2016 10:07:34 +0200
Subject: [PATCH 0618/1311] [TEST] be more specific around http method used for
 sniffing

---
 .../client/sniff/HostsSnifferTests.java       | 24 ++++++++++---------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
index e2b7a6d7c10..9d9c054fd06 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
@@ -29,6 +29,7 @@ import okhttp3.mockwebserver.MockResponse;
 import okhttp3.mockwebserver.MockWebServer;
 import okhttp3.mockwebserver.RecordedRequest;
 import org.apache.http.HttpHost;
+import org.apache.http.client.methods.HttpGet;
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.client.ElasticsearchResponse;
 import org.elasticsearch.client.ElasticsearchResponseException;
@@ -123,18 +124,19 @@ public class HostsSnifferTests extends LuceneTestCase {
         final Dispatcher dispatcher = new Dispatcher() {
             @Override
             public MockResponse dispatch(RecordedRequest request) throws InterruptedException {
-                String decodedUrl;
-                try {
-                    decodedUrl = URLDecoder.decode(request.getPath(), StandardCharsets.UTF_8.name());
-                } catch (UnsupportedEncodingException e) {
-                    throw new RuntimeException(e);
-                }
-                String sniffUrl = "/_nodes/http?timeout=" + sniffTimeout + "ms";
-                if (sniffUrl.equals(decodedUrl)) {
-                    return new MockResponse().setBody(sniffResponse.nodesInfoBody).setResponseCode(sniffResponse.nodesInfoResponseCode);
-                } else {
-                    return new MockResponse().setResponseCode(404);
+                if (request.getMethod().equals(HttpGet.METHOD_NAME)) {
+                    String decodedUrl;
+                    try {
+                        decodedUrl = URLDecoder.decode(request.getPath(), StandardCharsets.UTF_8.name());
+                    } catch (UnsupportedEncodingException e) {
+                        throw new RuntimeException(e);
+                    }
+                    String sniffUrl = "/_nodes/http?timeout=" + sniffTimeout + "ms";
+                    if (sniffUrl.equals(decodedUrl)) {
+                        return new MockResponse().setBody(sniffResponse.nodesInfoBody).setResponseCode(sniffResponse.nodesInfoResponseCode);
+                    }
                 }
+                return new MockResponse().setResponseCode(404);
             }
         };
         server.setDispatcher(dispatcher);

From 51e487fa555b03d507c985bb4eb3d7501f51da98 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Mon, 30 May 2016 17:08:20 +0200
Subject: [PATCH 0619/1311] [TEST] remove okhttp test dependency

Use sun HttpServer instead and disable forbidden-apis for test classes. It turns out to be more flexible than okhttp as it allows get & delete with body.
---
 client-sniffer/build.gradle                   | 16 ++---
 .../client/sniff/HostsSnifferTests.java       | 70 ++++++++-----------
 .../client/sniff/SnifferBuilderTests.java     |  5 --
 client/build.gradle                           | 16 ++---
 .../client/RestClientBuilderTests.java        |  5 --
 5 files changed, 44 insertions(+), 68 deletions(-)

diff --git a/client-sniffer/build.gradle b/client-sniffer/build.gradle
index d446ffc0113..40551556731 100644
--- a/client-sniffer/build.gradle
+++ b/client-sniffer/build.gradle
@@ -39,12 +39,6 @@ dependencies {
   testCompile "org.apache.lucene:lucene-test-framework:${versions.lucene}"
   testCompile "org.apache.lucene:lucene-core:${versions.lucene}"
   testCompile "org.apache.lucene:lucene-codecs:${versions.lucene}"
-  //mock web server
-  testCompile "com.squareup.okhttp3:mockwebserver:3.2.0"
-  testCompile "com.squareup.okhttp3:okhttp:3.2.0"
-  testCompile "com.squareup.okhttp3:okhttp-ws:3.2.0"
-  testCompile "com.squareup.okio:okio:1.6.0"
-  testCompile "org.bouncycastle:bcprov-jdk15on:1.54"
 }
 
 //TODO compiling from 1.8 with target 1.7 and source 1.7 is best effort, not enough to ensure we are java 7 compatible
@@ -56,10 +50,14 @@ forbiddenApisMain {
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
 }
 
-forbiddenApisTest {
+//TODO would be nice to just exclude the classes where we use com.sun.net.httpserver.* classes
+//excludes don't seem to work though and we don't want to have our own @SuppressForbidden
+forbiddenApisTest.enabled=false
+
+//forbiddenApisTest {
   //client does not depend on core, so only jdk signatures should be checked
-  signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
-}
+  //signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
+//}
 
 //TODO add licenses for dependencies and take care of distribution
 //dependency license are currently checked in distribution
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
index 9d9c054fd06..88f0480f92a 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
@@ -24,10 +24,10 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonGenerator;
-import okhttp3.mockwebserver.Dispatcher;
-import okhttp3.mockwebserver.MockResponse;
-import okhttp3.mockwebserver.MockWebServer;
-import okhttp3.mockwebserver.RecordedRequest;
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpServer;
+import org.apache.http.Consts;
 import org.apache.http.HttpHost;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.lucene.util.LuceneTestCase;
@@ -38,11 +38,10 @@ import org.junit.After;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.io.OutputStream;
 import java.io.StringWriter;
-import java.io.UnsupportedEncodingException;
+import java.net.InetSocketAddress;
 import java.net.URISyntaxException;
-import java.net.URLDecoder;
-import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -52,25 +51,19 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.logging.LogManager;
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
 
 public class HostsSnifferTests extends LuceneTestCase {
 
-    static {
-        //prevent MockWebServer from logging to stdout and stderr
-        LogManager.getLogManager().reset();
-    }
-
     private int sniffRequestTimeout;
     private String scheme;
     private SniffResponse sniffResponse;
-    private MockWebServer server;
+    private HttpServer httpServer;
 
     @Before
-    public void startMockWebServer() throws IOException {
+    public void startHttpServer() throws IOException {
         this.sniffRequestTimeout = RandomInts.randomIntBetween(random(), 1000, 10000);
         this.scheme = RandomPicks.randomFrom(random(), Arrays.asList("http", "https"));
         if (rarely()) {
@@ -78,17 +71,17 @@ public class HostsSnifferTests extends LuceneTestCase {
         } else {
             this.sniffResponse = buildSniffResponse(scheme);
         }
-        this.server = buildMockWebServer(sniffResponse, sniffRequestTimeout);
-        this.server.start();
+        this.httpServer = createHttpServer(sniffResponse, sniffRequestTimeout);
+        this.httpServer.start();
     }
 
     @After
-    public void stopMockWebServer() throws IOException {
-        server.shutdown();
+    public void stopHttpServer() throws IOException {
+        httpServer.stop(0);
     }
 
     public void testSniffNodes() throws IOException, URISyntaxException {
-        HttpHost httpHost = new HttpHost(server.getHostName(), server.getPort());
+        HttpHost httpHost = new HttpHost(httpServer.getAddress().getHostName(), httpServer.getAddress().getPort());
         try (RestClient restClient = RestClient.builder().setHosts(httpHost).build()) {
             HostsSniffer sniffer = new HostsSniffer(restClient, sniffRequestTimeout, scheme);
             try {
@@ -104,7 +97,7 @@ public class HostsSnifferTests extends LuceneTestCase {
             } catch(ElasticsearchResponseException e) {
                 ElasticsearchResponse response = e.getElasticsearchResponse();
                 if (sniffResponse.isFailure) {
-                    assertThat(e.getMessage(), containsString("GET http://localhost:" + server.getPort() +
+                    assertThat(e.getMessage(), containsString("GET http://localhost:" + httpServer.getAddress().getPort() +
                             "/_nodes/http?timeout=" + sniffRequestTimeout));
                     assertThat(e.getMessage(), containsString(Integer.toString(sniffResponse.nodesInfoResponseCode)));
                     assertThat(response.getHost(), equalTo(httpHost));
@@ -118,29 +111,26 @@ public class HostsSnifferTests extends LuceneTestCase {
         }
     }
 
-    private static MockWebServer buildMockWebServer(final SniffResponse sniffResponse, final int sniffTimeout)
-            throws UnsupportedEncodingException {
-        MockWebServer server = new MockWebServer();
-        final Dispatcher dispatcher = new Dispatcher() {
+    private static HttpServer createHttpServer(final SniffResponse sniffResponse, final int sniffTimeout) throws IOException {
+        HttpServer httpServer = HttpServer.create(new InetSocketAddress(0), 0);
+        httpServer.createContext("/_nodes/http", new HttpHandler() {
             @Override
-            public MockResponse dispatch(RecordedRequest request) throws InterruptedException {
-                if (request.getMethod().equals(HttpGet.METHOD_NAME)) {
-                    String decodedUrl;
-                    try {
-                        decodedUrl = URLDecoder.decode(request.getPath(), StandardCharsets.UTF_8.name());
-                    } catch (UnsupportedEncodingException e) {
-                        throw new RuntimeException(e);
-                    }
-                    String sniffUrl = "/_nodes/http?timeout=" + sniffTimeout + "ms";
-                    if (sniffUrl.equals(decodedUrl)) {
-                        return new MockResponse().setBody(sniffResponse.nodesInfoBody).setResponseCode(sniffResponse.nodesInfoResponseCode);
+            public void handle(HttpExchange httpExchange) throws IOException {
+                if (httpExchange.getRequestMethod().equals(HttpGet.METHOD_NAME)) {
+                    if (httpExchange.getRequestURI().getRawQuery().equals("timeout=" + sniffTimeout + "ms")) {
+                        String nodesInfoBody = sniffResponse.nodesInfoBody;
+                        httpExchange.sendResponseHeaders(sniffResponse.nodesInfoResponseCode, nodesInfoBody.length());
+                        try (OutputStream out = httpExchange.getResponseBody()) {
+                            out.write(nodesInfoBody.getBytes(Consts.UTF_8));
+                            return;
+                        }
                     }
                 }
-                return new MockResponse().setResponseCode(404);
+                httpExchange.sendResponseHeaders(404, 0);
+                httpExchange.close();
             }
-        };
-        server.setDispatcher(dispatcher);
-        return server;
+        });
+        return httpServer;
     }
 
     private static SniffResponse buildSniffResponse(String scheme) throws IOException {
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
index 868c5788120..ff3cad34bfa 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
@@ -26,14 +26,9 @@ import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.client.RestClient;
 
 import java.util.Arrays;
-import java.util.logging.LogManager;
 
 public class SnifferBuilderTests extends LuceneTestCase {
 
-    static {
-        LogManager.getLogManager().reset();
-    }
-
     public void testBuild() throws Exception {
 
         try {
diff --git a/client/build.gradle b/client/build.gradle
index 8cb0018e991..43979e84597 100644
--- a/client/build.gradle
+++ b/client/build.gradle
@@ -37,12 +37,6 @@ dependencies {
   testCompile "org.apache.lucene:lucene-test-framework:${versions.lucene}"
   testCompile "org.apache.lucene:lucene-core:${versions.lucene}"
   testCompile "org.apache.lucene:lucene-codecs:${versions.lucene}"
-  //mock web server
-  testCompile "com.squareup.okhttp3:mockwebserver:3.2.0"
-  testCompile "com.squareup.okhttp3:okhttp:3.2.0"
-  testCompile "com.squareup.okhttp3:okhttp-ws:3.2.0"
-  testCompile "com.squareup.okio:okio:1.6.0"
-  testCompile "org.bouncycastle:bcprov-jdk15on:1.54"
 }
 
 //TODO compiling from 1.8 with target 1.7 and source 1.7 is best effort, not enough to ensure we are java 7 compatible
@@ -54,10 +48,14 @@ forbiddenApisMain {
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
 }
 
-forbiddenApisTest {
+//TODO would be nice to just exclude the classes where we use com.sun.net.httpserver.* classes
+//excludes don't seem to work though and we don't want to have our own @SuppressForbidden
+forbiddenApisTest.enabled=false
+
+//forbiddenApisTest {
   //client does not depend on core, so only jdk signatures should be checked
-  signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
-}
+  //signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
+//}
 
 //TODO add licenses for dependencies and take care of distribution
 //dependency license are currently checked in distribution
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java b/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
index 44909336dd6..1d73db6ec37 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
@@ -30,14 +30,9 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.logging.LogManager;
 
 public class RestClientBuilderTests extends LuceneTestCase {
 
-    static {
-        LogManager.getLogManager().reset();
-    }
-
     public void testBuild() throws IOException {
         try {
             RestClient.builder().setMaxRetryTimeout(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));

From 83c6e736dedea2a28eac708acfbe5f88814495ce Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Tue, 31 May 2016 16:33:24 +0200
Subject: [PATCH 0620/1311] add support for PATCH and TRACE methods

Although elasticsearch doesn't support these methods (RestController doesn't even allow to register handler for them), the RestClient should allow to send requests using them.
---
 .../org/elasticsearch/client/RestClient.java  | 40 +++++++++----------
 1 file changed, 19 insertions(+), 21 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index ab919ca3382..63ee118cc4b 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -30,9 +30,11 @@ import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
 import org.apache.http.client.methods.HttpHead;
 import org.apache.http.client.methods.HttpOptions;
+import org.apache.http.client.methods.HttpPatch;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.methods.HttpTrace;
 import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.entity.ContentType;
 import org.apache.http.impl.client.CloseableHttpClient;
@@ -160,8 +162,7 @@ public final class RestClient implements Closeable {
                 lastSeenException = addSuppressedException(lastSeenException, e);
                 continue;
             }
-            ElasticsearchResponse elasticsearchResponse = new ElasticsearchResponse(request.getRequestLine(),
-                    host, response);
+            ElasticsearchResponse elasticsearchResponse = new ElasticsearchResponse(request.getRequestLine(), host, response);
             int statusCode = response.getStatusLine().getStatusCode();
             if (statusCode < 300 || (request.getMethod().equals(HttpHead.METHOD_NAME) && statusCode == 404) ) {
                 RequestLogger.log(logger, "request succeeded", request, host, response);
@@ -288,40 +289,37 @@ public final class RestClient implements Closeable {
     private static HttpRequestBase createHttpRequest(String method, URI uri, HttpEntity entity) {
         switch(method.toUpperCase(Locale.ROOT)) {
             case HttpDeleteWithEntity.METHOD_NAME:
-                HttpDeleteWithEntity httpDeleteWithEntity = new HttpDeleteWithEntity(uri);
-                addRequestBody(httpDeleteWithEntity, entity);
-                return httpDeleteWithEntity;
+                return addRequestBody(new HttpDeleteWithEntity(uri), entity);
             case HttpGetWithEntity.METHOD_NAME:
-                HttpGetWithEntity httpGetWithEntity = new HttpGetWithEntity(uri);
-                addRequestBody(httpGetWithEntity, entity);
-                return httpGetWithEntity;
+                return addRequestBody(new HttpGetWithEntity(uri), entity);
             case HttpHead.METHOD_NAME:
-                if (entity != null) {
-                    throw new UnsupportedOperationException("HEAD with body is not supported");
-                }
-                return new HttpHead(uri);
+                return addRequestBody(new HttpHead(uri), entity);
             case HttpOptions.METHOD_NAME:
-                if (entity != null) {
-                    throw new UnsupportedOperationException("OPTIONS with body is not supported");
-                }
-                return new HttpOptions(uri);
+                return addRequestBody(new HttpOptions(uri), entity);
+            case HttpPatch.METHOD_NAME:
+                return addRequestBody(new HttpPatch(uri), entity);
             case HttpPost.METHOD_NAME:
                 HttpPost httpPost = new HttpPost(uri);
                 addRequestBody(httpPost, entity);
                 return httpPost;
             case HttpPut.METHOD_NAME:
-                HttpPut httpPut = new HttpPut(uri);
-                addRequestBody(httpPut, entity);
-                return httpPut;
+                return addRequestBody(new HttpPut(uri), entity);
+            case HttpTrace.METHOD_NAME:
+                return addRequestBody(new HttpTrace(uri), entity);
             default:
                 throw new UnsupportedOperationException("http method not supported: " + method);
         }
     }
 
-    private static void addRequestBody(HttpEntityEnclosingRequestBase httpRequest, HttpEntity entity) {
+    private static HttpRequestBase addRequestBody(HttpRequestBase httpRequest, HttpEntity entity) {
         if (entity != null) {
-            httpRequest.setEntity(entity);
+            if (httpRequest instanceof HttpEntityEnclosingRequestBase) {
+                ((HttpEntityEnclosingRequestBase)httpRequest).setEntity(entity);
+            } else {
+                throw new UnsupportedOperationException(httpRequest.getMethod() + " with body is not supported");
+            }
         }
+        return httpRequest;
     }
 
     private static URI buildUri(String path, Map<String, String> params) {

From c9db111387b1808c9dfe945326fc497f54e82990 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Wed, 1 Jun 2016 10:51:51 +0200
Subject: [PATCH 0621/1311] add javadocs on closing responses

---
 .../java/org/elasticsearch/client/ElasticsearchResponse.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponse.java b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponse.java
index 304d10e2b94..2d25851b969 100644
--- a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponse.java
+++ b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponse.java
@@ -32,7 +32,8 @@ import java.util.Objects;
 
 /**
  * Holds an elasticsearch response. It wraps the {@link CloseableHttpResponse} response and associates it with
- * its corresponding {@link RequestLine} and {@link HttpHost}
+ * its corresponding {@link RequestLine} and {@link HttpHost}.
+ * It must be closed to free any resource held by it, as well as the corresponding connection in the connection pool.
  */
 public class ElasticsearchResponse implements Closeable {
 

From 6d66fbd9c15caab1f3fe089fd7ba2fc4c5c8189d Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Wed, 1 Jun 2016 14:38:46 +0200
Subject: [PATCH 0622/1311] add toString to DeadHostState class

---
 .../main/java/org/elasticsearch/client/DeadHostState.java | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/client/src/main/java/org/elasticsearch/client/DeadHostState.java b/client/src/main/java/org/elasticsearch/client/DeadHostState.java
index 30a24bc2d5d..52ef6e2bc6b 100644
--- a/client/src/main/java/org/elasticsearch/client/DeadHostState.java
+++ b/client/src/main/java/org/elasticsearch/client/DeadHostState.java
@@ -55,4 +55,12 @@ class DeadHostState {
     long getDeadUntil() {
         return deadUntil;
     }
+
+    @Override
+    public String toString() {
+        return "DeadHostState{" +
+                "failedAttempts=" + failedAttempts +
+                ", deadUntil=" + deadUntil +
+                '}';
+    }
 }

From 35dbdeeae5173bb3b0b53b93282185bf083ec2be Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Wed, 1 Jun 2016 23:38:26 +0200
Subject: [PATCH 0623/1311] check hosts is not null nor empty earlier, remove
 check from nextHost

if we check at set time, we don't need to check each single time in nextHost
---
 .../src/main/java/org/elasticsearch/client/RestClient.java | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 63ee118cc4b..7d9cebcafca 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -98,6 +98,9 @@ public final class RestClient implements Closeable {
      * @see HttpHost
      */
     public synchronized void setHosts(HttpHost... hosts) {
+        if (hosts == null || hosts.length == 0) {
+            throw new IllegalArgumentException("hosts must not be null nor empty");
+        }
         Set<HttpHost> httpHosts = new HashSet<>();
         for (HttpHost host : hosts) {
             Objects.requireNonNull(host, "host cannot be null");
@@ -206,10 +209,6 @@ public final class RestClient implements Closeable {
      * In case there are no healthy hosts available, or dead ones to be be retried, one dead host gets returned.
      */
     private Iterator<HttpHost> nextHost() {
-        if (this.hosts.isEmpty()) {
-            throw new IllegalStateException("no hosts available");
-        }
-
         Set<HttpHost> filteredHosts = new HashSet<>(hosts);
         for (Map.Entry<HttpHost, DeadHostState> entry : blacklist.entrySet()) {
             if (System.nanoTime() - entry.getValue().getDeadUntil() < 0) {

From 47e52044e452073c3205475515bee87168360812 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Thu, 2 Jun 2016 11:30:09 +0200
Subject: [PATCH 0624/1311] [TEST] add setHosts test and rename
 RestClientBuilderTests to RestClientTests

---
 ...BuilderTests.java => RestClientTests.java} | 31 ++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)
 rename client/src/test/java/org/elasticsearch/client/{RestClientBuilderTests.java => RestClientTests.java} (75%)

diff --git a/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java b/client/src/test/java/org/elasticsearch/client/RestClientTests.java
similarity index 75%
rename from client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
rename to client/src/test/java/org/elasticsearch/client/RestClientTests.java
index 1d73db6ec37..f6a297eb145 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientTests.java
@@ -31,7 +31,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 
-public class RestClientBuilderTests extends LuceneTestCase {
+public class RestClientTests extends LuceneTestCase {
 
     public void testBuild() throws IOException {
         try {
@@ -105,4 +105,33 @@ public class RestClientBuilderTests extends LuceneTestCase {
             assertNotNull(restClient);
         }
     }
+
+    public void testSetNodes() throws IOException {
+        try (RestClient restClient = RestClient.builder().setHosts(new HttpHost("localhost", 9200)).build()) {
+            try {
+                restClient.setHosts((HttpHost[]) null);
+                fail("setHosts should have failed");
+            } catch (IllegalArgumentException e) {
+                assertEquals("hosts must not be null nor empty", e.getMessage());
+            }
+            try {
+                restClient.setHosts();
+                fail("setHosts should have failed");
+            } catch (IllegalArgumentException e) {
+                assertEquals("hosts must not be null nor empty", e.getMessage());
+            }
+            try {
+                restClient.setHosts((HttpHost) null);
+                fail("setHosts should have failed");
+            } catch (NullPointerException e) {
+                assertEquals("host cannot be null", e.getMessage());
+            }
+            try {
+                restClient.setHosts(new HttpHost("localhost", 9200), null, new HttpHost("localhost", 9201));
+                fail("setHosts should have failed");
+            } catch (NullPointerException e) {
+                assertEquals("host cannot be null", e.getMessage());
+            }
+        }
+    }
 }

From 24ea585c9ee8ebd150fc87c3de403667ff1f76bf Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 3 Jun 2016 15:44:51 +0200
Subject: [PATCH 0625/1311] don't use setDefaultHeaders from HttpClient

Store default headers ourselves instead, otherwise default ones cannot be replaced. Don't allow for multiple headers with same key, last one wins and replaces previous ones with same key.

Also fail with null params or headers.
---
 .../elasticsearch/client/sniff/Sniffer.java   |  3 +-
 .../org/elasticsearch/client/RestClient.java  | 56 +++++++++++--------
 2 files changed, 33 insertions(+), 26 deletions(-)

diff --git a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
index ed6f8284f1e..855b15e1e4d 100644
--- a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
+++ b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
@@ -27,7 +27,6 @@ import org.elasticsearch.client.RestClient;
 
 import java.io.Closeable;
 import java.io.IOException;
-import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.Executors;
@@ -204,7 +203,7 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
         /**
          * Sets the http client. Mandatory argument. Best practice is to use the same client used
          * within {@link org.elasticsearch.client.RestClient} which can be created manually or
-         * through {@link RestClient.Builder#createDefaultHttpClient(Collection)}.
+         * through {@link RestClient.Builder#createDefaultHttpClient()}.
          * @see CloseableHttpClient
          */
         public Builder setRestClient(RestClient restClient) {
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 7d9cebcafca..856a50c1834 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -24,6 +24,7 @@ import org.apache.http.Consts;
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHost;
+import org.apache.http.HttpRequest;
 import org.apache.http.client.ClientProtocolException;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
@@ -47,7 +48,6 @@ import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashSet;
@@ -81,15 +81,19 @@ public final class RestClient implements Closeable {
     public static ContentType JSON_CONTENT_TYPE = ContentType.create("application/json", Consts.UTF_8);
 
     private final CloseableHttpClient client;
+    //we don't rely on default headers supported by HttpClient as those cannot be replaced, plus it would get hairy
+    //when we create the HttpClient instance on our own as there would be two different ways to set the default headers.
+    private final Header[] defaultHeaders;
     private final long maxRetryTimeout;
     private final AtomicInteger lastHostIndex = new AtomicInteger(0);
     private volatile Set<HttpHost> hosts;
     private final ConcurrentMap<HttpHost, DeadHostState> blacklist = new ConcurrentHashMap<>();
     private volatile FailureListener failureListener = new FailureListener();
 
-    private RestClient(CloseableHttpClient client, long maxRetryTimeout, HttpHost... hosts) {
+    private RestClient(CloseableHttpClient client, long maxRetryTimeout, Header[] defaultHeaders, HttpHost[] hosts) {
         this.client = client;
         this.maxRetryTimeout = maxRetryTimeout;
+        this.defaultHeaders = defaultHeaders;
         setHosts(hosts);
     }
 
@@ -131,11 +135,7 @@ public final class RestClient implements Closeable {
                                                 HttpEntity entity, Header... headers) throws IOException {
         URI uri = buildUri(endpoint, params);
         HttpRequestBase request = createHttpRequest(method, uri, entity);
-        if (headers.length > 0) {
-            for (Header header : headers) {
-                request.addHeader(header);
-            }
-        }
+        setHeaders(request, headers);
         //we apply a soft margin so that e.g. if a request took 59 seconds and timeout is set to 60 we don't do another attempt
         long retryTimeout = Math.round(this.maxRetryTimeout / (float)100 * 98);
         IOException lastSeenException = null;
@@ -200,6 +200,17 @@ public final class RestClient implements Closeable {
         throw lastSeenException;
     }
 
+    private void setHeaders(HttpRequest httpRequest, Header[] requestHeaders) {
+        Objects.requireNonNull(requestHeaders, "request headers must not be null");
+        for (Header defaultHeader : defaultHeaders) {
+            httpRequest.setHeader(defaultHeader);
+        }
+        for (Header requestHeader : requestHeaders) {
+            Objects.requireNonNull(requestHeader, "request header must not be null");
+            httpRequest.setHeader(requestHeader);
+        }
+    }
+
     /**
      * Returns an iterator of hosts to be used for a request call.
      * Ideally, the first host is retrieved from the iterator and used successfully for the request.
@@ -322,6 +333,7 @@ public final class RestClient implements Closeable {
     }
 
     private static URI buildUri(String path, Map<String, String> params) {
+        Objects.requireNonNull(params, "params must not be null");
         try {
             URIBuilder uriBuilder = new URIBuilder(path);
             for (Map.Entry<String, String> param : params.entrySet()) {
@@ -349,10 +361,12 @@ public final class RestClient implements Closeable {
         public static final int DEFAULT_MAX_RETRY_TIMEOUT = DEFAULT_SOCKET_TIMEOUT;
         public static final int DEFAULT_CONNECTION_REQUEST_TIMEOUT = 500;
 
+        private static final Header[] EMPTY_HEADERS = new Header[0];
+
         private CloseableHttpClient httpClient;
         private int maxRetryTimeout = DEFAULT_MAX_RETRY_TIMEOUT;
         private HttpHost[] hosts;
-        private Collection<? extends Header> defaultHeaders;
+        private Header[] defaultHeaders = EMPTY_HEADERS;
 
         private Builder() {
 
@@ -360,7 +374,7 @@ public final class RestClient implements Closeable {
 
         /**
          * Sets the http client. A new default one will be created if not
-         * specified, by calling {@link #createDefaultHttpClient(Collection)}.
+         * specified, by calling {@link #createDefaultHttpClient()}.
          *
          * @see CloseableHttpClient
          */
@@ -399,7 +413,11 @@ public final class RestClient implements Closeable {
          * In case the http client is set through {@link #setHttpClient(CloseableHttpClient)}, the default headers need to be
          * set to it externally during http client construction.
          */
-        public Builder setDefaultHeaders(Collection<? extends Header> defaultHeaders) {
+        public Builder setDefaultHeaders(Header[] defaultHeaders) {
+            Objects.requireNonNull(defaultHeaders, "default headers must not be null");
+            for (Header defaultHeader : defaultHeaders) {
+                Objects.requireNonNull(defaultHeader, "default header must not be null");
+            }
             this.defaultHeaders = defaultHeaders;
             return this;
         }
@@ -409,16 +427,12 @@ public final class RestClient implements Closeable {
          */
         public RestClient build() {
             if (httpClient == null) {
-                httpClient = createDefaultHttpClient(defaultHeaders);
-            } else {
-                if (defaultHeaders != null) {
-                    throw new IllegalArgumentException("defaultHeaders need to be set to the HttpClient directly when manually provided");
-                }
+                httpClient = createDefaultHttpClient();
             }
             if (hosts == null || hosts.length == 0) {
                 throw new IllegalArgumentException("no hosts provided");
             }
-            return new RestClient(httpClient, maxRetryTimeout, hosts);
+            return new RestClient(httpClient, maxRetryTimeout, defaultHeaders, hosts);
         }
 
         /**
@@ -426,7 +440,7 @@ public final class RestClient implements Closeable {
          *
          * @see CloseableHttpClient
          */
-        public static CloseableHttpClient createDefaultHttpClient(Collection<? extends Header> defaultHeaders) {
+        public static CloseableHttpClient createDefaultHttpClient() {
             PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
             //default settings may be too constraining
             connectionManager.setDefaultMaxPerRoute(10);
@@ -436,12 +450,7 @@ public final class RestClient implements Closeable {
             RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(DEFAULT_CONNECT_TIMEOUT)
                     .setSocketTimeout(DEFAULT_SOCKET_TIMEOUT)
                     .setConnectionRequestTimeout(DEFAULT_CONNECTION_REQUEST_TIMEOUT).build();
-
-            HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
-            if (defaultHeaders != null) {
-                httpClientBuilder.setDefaultHeaders(defaultHeaders);
-            }
-            return httpClientBuilder.setConnectionManager(connectionManager).setDefaultRequestConfig(requestConfig).build();
+            return HttpClientBuilder.create().setConnectionManager(connectionManager).setDefaultRequestConfig(requestConfig).build();
         }
     }
 
@@ -450,7 +459,6 @@ public final class RestClient implements Closeable {
      * The default implementation is a no-op.
      */
     public static class FailureListener {
-
         /**
          * Notifies that the host provided as argument has just failed
          */

From 9ed2d610ecd5c5be2108b9de3dd785493481d2f8 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 3 Jun 2016 15:45:52 +0200
Subject: [PATCH 0626/1311] [TEST] rename restClientTests back to
 RestClientBuilderTests

---
 ...Tests.java => RestClientBuilderTests.java} | 71 ++++++-------------
 1 file changed, 21 insertions(+), 50 deletions(-)
 rename client/src/test/java/org/elasticsearch/client/{RestClientTests.java => RestClientBuilderTests.java} (58%)

diff --git a/client/src/test/java/org/elasticsearch/client/RestClientTests.java b/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
similarity index 58%
rename from client/src/test/java/org/elasticsearch/client/RestClientTests.java
rename to client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
index f6a297eb145..3ea0fe14ed2 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
@@ -20,18 +20,15 @@
 package org.elasticsearch.client;
 
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import org.apache.http.Header;
 import org.apache.http.HttpHost;
-import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.message.BasicHeader;
 import org.apache.lucene.util.LuceneTestCase;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
 
-public class RestClientTests extends LuceneTestCase {
+public class RestClientBuilderTests extends LuceneTestCase {
 
     public void testBuild() throws IOException {
         try {
@@ -69,12 +66,18 @@ public class RestClientTests extends LuceneTestCase {
             assertEquals(e.getMessage(), "host cannot be null");
         }
 
-        try (CloseableHttpClient httpClient = HttpClientBuilder.create().build()) {
-            RestClient.builder().setHttpClient(httpClient)
-                    .setDefaultHeaders(Collections.singleton(new BasicHeader("header", "value"))).build();
+        try {
+            RestClient.builder().setDefaultHeaders(null);
             fail("should have failed");
-        } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "defaultHeaders need to be set to the HttpClient directly when manually provided");
+        } catch(NullPointerException e) {
+            assertEquals(e.getMessage(), "default headers must not be null");
+        }
+
+        try {
+            RestClient.builder().setDefaultHeaders(new Header[]{null});
+            fail("should have failed");
+        } catch(NullPointerException e) {
+            assertEquals(e.getMessage(), "default header must not be null");
         }
 
         RestClient.Builder builder = RestClient.builder();
@@ -84,20 +87,17 @@ public class RestClientTests extends LuceneTestCase {
             hosts[i] = new HttpHost("localhost", 9200 + i);
         }
         builder.setHosts(hosts);
-
         if (random().nextBoolean()) {
             builder.setHttpClient(HttpClientBuilder.create().build());
-        } else {
-            if (random().nextBoolean()) {
-                int numHeaders = RandomInts.randomIntBetween(random(), 1, 5);
-                Collection<BasicHeader> headers = new ArrayList<>(numHeaders);
-                for (int i = 0; i < numHeaders; i++) {
-                    headers.add(new BasicHeader("header" + i, "value"));
-                }
-                builder.setDefaultHeaders(headers);
-            }
         }
-
+        if (random().nextBoolean()) {
+            int numHeaders = RandomInts.randomIntBetween(random(), 1, 5);
+            Header[] headers = new Header[numHeaders];
+            for (int i = 0; i < numHeaders; i++) {
+                headers[i] = new BasicHeader("header" + i, "value");
+            }
+            builder.setDefaultHeaders(headers);
+        }
         if (random().nextBoolean()) {
             builder.setMaxRetryTimeout(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
         }
@@ -105,33 +105,4 @@ public class RestClientTests extends LuceneTestCase {
             assertNotNull(restClient);
         }
     }
-
-    public void testSetNodes() throws IOException {
-        try (RestClient restClient = RestClient.builder().setHosts(new HttpHost("localhost", 9200)).build()) {
-            try {
-                restClient.setHosts((HttpHost[]) null);
-                fail("setHosts should have failed");
-            } catch (IllegalArgumentException e) {
-                assertEquals("hosts must not be null nor empty", e.getMessage());
-            }
-            try {
-                restClient.setHosts();
-                fail("setHosts should have failed");
-            } catch (IllegalArgumentException e) {
-                assertEquals("hosts must not be null nor empty", e.getMessage());
-            }
-            try {
-                restClient.setHosts((HttpHost) null);
-                fail("setHosts should have failed");
-            } catch (NullPointerException e) {
-                assertEquals("host cannot be null", e.getMessage());
-            }
-            try {
-                restClient.setHosts(new HttpHost("localhost", 9200), null, new HttpHost("localhost", 9201));
-                fail("setHosts should have failed");
-            } catch (NullPointerException e) {
-                assertEquals("host cannot be null", e.getMessage());
-            }
-        }
-    }
 }

From 4572b69011bc8f057717a2c8d8d2fc861cebef82 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 3 Jun 2016 15:55:44 +0200
Subject: [PATCH 0627/1311] [TEST] add RestClient unit tests

 Unit tests rely on mockito to mock the internal HttpClient instance. No http request is performed, we only simulate interaction between RestClient and its internal HttpClient.
---
 client-sniffer/build.gradle                   |   1 +
 client/build.gradle                           |   1 +
 .../client/CloseableBasicHttpResponse.java    |  42 ++
 .../client/RestClientMultipleHostsTests.java  | 279 ++++++++++++
 .../client/RestClientSingleHostTests.java     | 421 ++++++++++++++++++
 .../client/RestClientTestUtil.java            |  84 ++++
 .../client/TrackingFailureListener.java       |  52 +++
 7 files changed, 880 insertions(+)
 create mode 100644 client/src/test/java/org/elasticsearch/client/CloseableBasicHttpResponse.java
 create mode 100644 client/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
 create mode 100644 client/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
 create mode 100644 client/src/test/java/org/elasticsearch/client/RestClientTestUtil.java
 create mode 100644 client/src/test/java/org/elasticsearch/client/TrackingFailureListener.java

diff --git a/client-sniffer/build.gradle b/client-sniffer/build.gradle
index 40551556731..6542453fa4f 100644
--- a/client-sniffer/build.gradle
+++ b/client-sniffer/build.gradle
@@ -39,6 +39,7 @@ dependencies {
   testCompile "org.apache.lucene:lucene-test-framework:${versions.lucene}"
   testCompile "org.apache.lucene:lucene-core:${versions.lucene}"
   testCompile "org.apache.lucene:lucene-codecs:${versions.lucene}"
+  testCompile "org.elasticsearch:securemock:1.2"
 }
 
 //TODO compiling from 1.8 with target 1.7 and source 1.7 is best effort, not enough to ensure we are java 7 compatible
diff --git a/client/build.gradle b/client/build.gradle
index 43979e84597..6d2b92f0b0c 100644
--- a/client/build.gradle
+++ b/client/build.gradle
@@ -37,6 +37,7 @@ dependencies {
   testCompile "org.apache.lucene:lucene-test-framework:${versions.lucene}"
   testCompile "org.apache.lucene:lucene-core:${versions.lucene}"
   testCompile "org.apache.lucene:lucene-codecs:${versions.lucene}"
+  testCompile "org.elasticsearch:securemock:1.2"
 }
 
 //TODO compiling from 1.8 with target 1.7 and source 1.7 is best effort, not enough to ensure we are java 7 compatible
diff --git a/client/src/test/java/org/elasticsearch/client/CloseableBasicHttpResponse.java b/client/src/test/java/org/elasticsearch/client/CloseableBasicHttpResponse.java
new file mode 100644
index 00000000000..904cbe7cfeb
--- /dev/null
+++ b/client/src/test/java/org/elasticsearch/client/CloseableBasicHttpResponse.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.StatusLine;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.message.BasicHttpResponse;
+
+import java.io.IOException;
+
+/**
+ * Simple {@link CloseableHttpResponse} impl needed to easily create http responses that are closeable given that
+ * org.apache.http.impl.execchain.HttpResponseProxy is not public.
+ */
+class CloseableBasicHttpResponse extends BasicHttpResponse implements CloseableHttpResponse {
+
+    public CloseableBasicHttpResponse(StatusLine statusline) {
+        super(statusline);
+    }
+
+    @Override
+    public void close() throws IOException {
+        //nothing to close
+    }
+}
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java b/client/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
new file mode 100644
index 00000000000..d78a03509d0
--- /dev/null
+++ b/client/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpRequest;
+import org.apache.http.ProtocolVersion;
+import org.apache.http.StatusLine;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.conn.ConnectTimeoutException;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.message.BasicStatusLine;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Before;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import java.io.IOException;
+import java.net.SocketTimeoutException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.elasticsearch.client.RestClientTestUtil.randomErrorNoRetryStatusCode;
+import static org.elasticsearch.client.RestClientTestUtil.randomErrorRetryStatusCode;
+import static org.elasticsearch.client.RestClientTestUtil.randomHttpMethod;
+import static org.elasticsearch.client.RestClientTestUtil.randomOkStatusCode;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests for {@link RestClient} behaviour against multiple hosts: fail-over, blacklisting etc.
+ * Relies on a mock http client to intercept requests and return desired responses based on request path.
+ */
+public class RestClientMultipleHostsTests extends LuceneTestCase {
+
+    private RestClient restClient;
+    private HttpHost[] httpHosts;
+    private TrackingFailureListener failureListener;
+
+    @Before
+    public void createRestClient() throws IOException {
+        CloseableHttpClient httpClient = mock(CloseableHttpClient.class);
+        when(httpClient.execute(any(HttpHost.class), any(HttpRequest.class))).thenAnswer(new Answer<CloseableHttpResponse>() {
+            @Override
+            public CloseableHttpResponse answer(InvocationOnMock invocationOnMock) throws Throwable {
+                HttpHost httpHost = (HttpHost) invocationOnMock.getArguments()[0];
+                HttpUriRequest request = (HttpUriRequest) invocationOnMock.getArguments()[1];
+                //return the desired status code or exception depending on the path
+                if (request.getURI().getPath().equals("/soe")) {
+                    throw new SocketTimeoutException(httpHost.toString());
+                } else if (request.getURI().getPath().equals("/coe")) {
+                    throw new ConnectTimeoutException(httpHost.toString());
+                } else if (request.getURI().getPath().equals("/ioe")) {
+                    throw new IOException(httpHost.toString());
+                }
+                int statusCode = Integer.parseInt(request.getURI().getPath().substring(1));
+                StatusLine statusLine = new BasicStatusLine(new ProtocolVersion("http", 1, 1), statusCode, "");
+                return new CloseableBasicHttpResponse(statusLine);
+            }
+        });
+
+        int numHosts = RandomInts.randomIntBetween(random(), 2, 5);
+        httpHosts = new HttpHost[numHosts];
+        for (int i = 0; i < numHosts; i++) {
+            httpHosts[i] = new HttpHost("localhost", 9200 + i);
+        }
+        restClient = RestClient.builder().setHosts(httpHosts).setHttpClient(httpClient).build();
+        failureListener = new TrackingFailureListener();
+        restClient.setFailureListener(failureListener);
+    }
+
+    /**
+     * Test that
+     */
+    public void testRoundRobinOkStatusCodes() throws Exception {
+        int numIters = RandomInts.randomIntBetween(random(), 1, 5);
+        for (int i = 0; i < numIters; i++) {
+            Set<HttpHost> hostsSet = new HashSet<>();
+            Collections.addAll(hostsSet, httpHosts);
+            for (int j = 0; j < httpHosts.length; j++) {
+                int statusCode = randomOkStatusCode(random());
+                try (ElasticsearchResponse response = restClient.performRequest(randomHttpMethod(random()), "/" + statusCode,
+                        Collections.<String, String>emptyMap(), null)) {
+                    assertThat(response.getStatusLine().getStatusCode(), equalTo(statusCode));
+                    assertTrue("host not found: " + response.getHost(), hostsSet.remove(response.getHost()));
+                }
+            }
+            assertEquals("every host should have been used but some weren't: " + hostsSet, 0, hostsSet.size());
+        }
+        failureListener.assertNotCalled();
+    }
+
+    public void testRoundRobinNoRetryErrors() throws Exception {
+        int numIters = RandomInts.randomIntBetween(random(), 1, 5);
+        for (int i = 0; i < numIters; i++) {
+            Set<HttpHost> hostsSet = new HashSet<>();
+            Collections.addAll(hostsSet, httpHosts);
+            for (int j = 0; j < httpHosts.length; j++) {
+                String method = randomHttpMethod(random());
+                int statusCode = randomErrorNoRetryStatusCode(random());
+                try (ElasticsearchResponse response = restClient.performRequest(method, "/" + statusCode,
+                        Collections.<String, String>emptyMap(), null)) {
+                    if (method.equals("HEAD") && statusCode == 404) {
+                        //no exception gets thrown although we got a 404
+                        assertThat(response.getStatusLine().getStatusCode(), equalTo(404));
+                        assertThat(response.getStatusLine().getStatusCode(), equalTo(statusCode));
+                        assertTrue("host not found: " + response.getHost(), hostsSet.remove(response.getHost()));
+                    } else {
+                        fail("request should have failed");
+                    }
+                } catch(ElasticsearchResponseException e) {
+                    if (method.equals("HEAD") && statusCode == 404) {
+                        throw e;
+                    }
+                    ElasticsearchResponse response = e.getElasticsearchResponse();
+                    assertThat(response.getStatusLine().getStatusCode(), equalTo(statusCode));
+                    assertTrue("host not found: " + response.getHost(), hostsSet.remove(response.getHost()));
+                    assertEquals(0, e.getSuppressed().length);
+                }
+            }
+            assertEquals("every host should have been used but some weren't: " + hostsSet, 0, hostsSet.size());
+        }
+        failureListener.assertNotCalled();
+    }
+
+    public void testRoundRobinRetryErrors() throws Exception {
+        String retryEndpoint = randomErrorRetryEndpoint();
+        try  {
+            restClient.performRequest(randomHttpMethod(random()), retryEndpoint, Collections.<String, String>emptyMap(), null);
+            fail("request should have failed");
+        } catch(ElasticsearchResponseException e) {
+            Set<HttpHost> hostsSet = new HashSet<>();
+            Collections.addAll(hostsSet, httpHosts);
+            //first request causes all the hosts to be blacklisted, the returned exception holds one suppressed exception each
+            failureListener.assertCalled(httpHosts);
+            do {
+                ElasticsearchResponse response = e.getElasticsearchResponse();
+                assertThat(response.getStatusLine().getStatusCode(), equalTo(Integer.parseInt(retryEndpoint.substring(1))));
+                assertTrue("host [" + response.getHost() + "] not found, most likely used multiple times",
+                        hostsSet.remove(response.getHost()));
+                if (e.getSuppressed().length > 0) {
+                    assertEquals(1, e.getSuppressed().length);
+                    Throwable suppressed = e.getSuppressed()[0];
+                    assertThat(suppressed, instanceOf(ElasticsearchResponseException.class));
+                    e = (ElasticsearchResponseException)suppressed;
+                } else {
+                    e = null;
+                }
+            } while(e != null);
+            assertEquals("every host should have been used but some weren't: " + hostsSet, 0, hostsSet.size());
+        } catch(IOException e) {
+            Set<HttpHost> hostsSet = new HashSet<>();
+            Collections.addAll(hostsSet, httpHosts);
+            //first request causes all the hosts to be blacklisted, the returned exception holds one suppressed exception each
+            failureListener.assertCalled(httpHosts);
+            do {
+                HttpHost httpHost = HttpHost.create(e.getMessage());
+                assertTrue("host [" + httpHost + "] not found, most likely used multiple times", hostsSet.remove(httpHost));
+                if (e.getSuppressed().length > 0) {
+                    assertEquals(1, e.getSuppressed().length);
+                    Throwable suppressed = e.getSuppressed()[0];
+                    assertThat(suppressed, instanceOf(IOException.class));
+                    e = (IOException) suppressed;
+                } else {
+                    e = null;
+                }
+            } while(e != null);
+            assertEquals("every host should have been used but some weren't: " + hostsSet, 0, hostsSet.size());
+        }
+
+        int numIters = RandomInts.randomIntBetween(random(), 2, 5);
+        for (int i = 1; i <= numIters; i++) {
+            //check that one different host is resurrected at each new attempt
+            Set<HttpHost> hostsSet = new HashSet<>();
+            Collections.addAll(hostsSet, httpHosts);
+            for (int j = 0; j < httpHosts.length; j++) {
+                retryEndpoint = randomErrorRetryEndpoint();
+                try  {
+                    restClient.performRequest(randomHttpMethod(random()), retryEndpoint, Collections.<String, String>emptyMap(), null);
+                    fail("request should have failed");
+                } catch(ElasticsearchResponseException e) {
+                    ElasticsearchResponse response = e.getElasticsearchResponse();
+                    assertThat(response.getStatusLine().getStatusCode(), equalTo(Integer.parseInt(retryEndpoint.substring(1))));
+                    assertTrue("host [" + response.getHost() + "] not found, most likely used multiple times",
+                            hostsSet.remove(response.getHost()));
+                    //after the first request, all hosts are blacklisted, a single one gets resurrected each time
+                    failureListener.assertCalled(response.getHost());
+                    assertEquals(0, e.getSuppressed().length);
+                } catch(IOException e) {
+                    HttpHost httpHost = HttpHost.create(e.getMessage());
+                    assertTrue("host [" + httpHost + "] not found, most likely used multiple times", hostsSet.remove(httpHost));
+                    //after the first request, all hosts are blacklisted, a single one gets resurrected each time
+                    failureListener.assertCalled(httpHost);
+                    assertEquals(0, e.getSuppressed().length);
+                }
+            }
+            assertEquals("every host should have been used but some weren't: " + hostsSet, 0, hostsSet.size());
+            if (random().nextBoolean()) {
+                //mark one host back alive through a successful request and check that all requests after that are sent to it
+                HttpHost selectedHost = null;
+                int iters = RandomInts.randomIntBetween(random(), 2, 10);
+                for (int y = 0; y < iters; y++) {
+                    int statusCode = randomErrorNoRetryStatusCode(random());
+                    ElasticsearchResponse response;
+                    try (ElasticsearchResponse esResponse = restClient.performRequest(randomHttpMethod(random()), "/" + statusCode,
+                            Collections.<String, String>emptyMap(), null)) {
+                        response = esResponse;
+                    }
+                    catch(ElasticsearchResponseException e) {
+                        response = e.getElasticsearchResponse();
+                    }
+                    assertThat(response.getStatusLine().getStatusCode(), equalTo(statusCode));
+                    if (selectedHost == null) {
+                        selectedHost = response.getHost();
+                    } else {
+                        assertThat(response.getHost(), equalTo(selectedHost));
+                    }
+                }
+                failureListener.assertNotCalled();
+                //let the selected host catch up on number of failures, it gets selected a consecutive number of times as it's the one
+                //selected to be retried earlier (due to lower number of failures) till all the hosts have the same number of failures
+                for (int y = 0; y < i + 1; y++) {
+                    retryEndpoint = randomErrorRetryEndpoint();
+                    try {
+                        restClient.performRequest(randomHttpMethod(random()), retryEndpoint,
+                                Collections.<String, String>emptyMap(), null);
+                        fail("request should have failed");
+                    } catch(ElasticsearchResponseException e) {
+                        ElasticsearchResponse response = e.getElasticsearchResponse();
+                        assertThat(response.getStatusLine().getStatusCode(), equalTo(Integer.parseInt(retryEndpoint.substring(1))));
+                        assertThat(response.getHost(), equalTo(selectedHost));
+                        failureListener.assertCalled(selectedHost);
+                    } catch(IOException e) {
+                        HttpHost httpHost = HttpHost.create(e.getMessage());
+                        assertThat(httpHost, equalTo(selectedHost));
+                        failureListener.assertCalled(selectedHost);
+                    }
+                }
+            }
+        }
+    }
+
+    private static String randomErrorRetryEndpoint() {
+        switch(RandomInts.randomIntBetween(random(), 0, 3)) {
+            case 0:
+                return "/" + randomErrorRetryStatusCode(random());
+            case 1:
+                return "/coe";
+            case 2:
+                return "/soe";
+            case 3:
+                return "/ioe";
+        }
+        throw new UnsupportedOperationException();
+    }
+}
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java b/client/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
new file mode 100644
index 00000000000..c3c08bee628
--- /dev/null
+++ b/client/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
@@ -0,0 +1,421 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpEntityEnclosingRequest;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpRequest;
+import org.apache.http.ProtocolVersion;
+import org.apache.http.StatusLine;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpHead;
+import org.apache.http.client.methods.HttpOptions;
+import org.apache.http.client.methods.HttpPatch;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.methods.HttpTrace;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.conn.ConnectTimeoutException;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.message.BasicStatusLine;
+import org.apache.http.util.EntityUtils;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Before;
+import org.mockito.ArgumentCaptor;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import java.io.IOException;
+import java.net.SocketTimeoutException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.client.RestClientTestUtil.getAllErrorStatusCodes;
+import static org.elasticsearch.client.RestClientTestUtil.getHttpMethods;
+import static org.elasticsearch.client.RestClientTestUtil.getOkStatusCodes;
+import static org.elasticsearch.client.RestClientTestUtil.randomHttpMethod;
+import static org.elasticsearch.client.RestClientTestUtil.randomStatusCode;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests for basic functionality of {@link RestClient} against one single host: tests http requests being sent, headers,
+ * body, different status codes and corresponding responses/exceptions.
+ * Relies on a mock http client to intercept requests and return desired responses based on request path.
+ */
+public class RestClientSingleHostTests extends LuceneTestCase {
+
+    private RestClient restClient;
+    private Header[] defaultHeaders;
+    private HttpHost httpHost;
+    private CloseableHttpClient httpClient;
+    private TrackingFailureListener failureListener;
+
+    @Before
+    public void createRestClient() throws IOException {
+        httpClient = mock(CloseableHttpClient.class);
+        when(httpClient.execute(any(HttpHost.class), any(HttpRequest.class))).thenAnswer(new Answer<CloseableHttpResponse>() {
+            @Override
+            public CloseableHttpResponse answer(InvocationOnMock invocationOnMock) throws Throwable {
+                HttpUriRequest request = (HttpUriRequest) invocationOnMock.getArguments()[1];
+                //return the desired status code or exception depending on the path
+                if (request.getURI().getPath().equals("/soe")) {
+                    throw new SocketTimeoutException();
+                } else if (request.getURI().getPath().equals("/coe")) {
+                    throw new ConnectTimeoutException();
+                }
+                int statusCode = Integer.parseInt(request.getURI().getPath().substring(1));
+                StatusLine statusLine = new BasicStatusLine(new ProtocolVersion("http", 1, 1), statusCode, "");
+
+                CloseableHttpResponse httpResponse = new CloseableBasicHttpResponse(statusLine);
+                //return the same body that was sent
+                if (request instanceof HttpEntityEnclosingRequest) {
+                    HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity();
+                    if (entity != null) {
+                        assertTrue("the entity is not repeatable, cannot set it to the response directly", entity.isRepeatable());
+                        httpResponse.setEntity(entity);
+                    }
+                }
+                //return the same headers that were sent
+                httpResponse.setHeaders(request.getAllHeaders());
+                return httpResponse;
+            }
+        });
+        int numHeaders = RandomInts.randomIntBetween(random(), 0, 3);
+        defaultHeaders = new Header[numHeaders];
+        for (int i = 0; i < numHeaders; i++) {
+            String headerName = "Header-default" + (random().nextBoolean() ? i : "");
+            String headerValue = RandomStrings.randomAsciiOfLengthBetween(random(), 3, 10);
+            defaultHeaders[i] = new BasicHeader(headerName, headerValue);
+        }
+        httpHost = new HttpHost("localhost", 9200);
+        restClient = RestClient.builder().setHosts(httpHost).setHttpClient(httpClient).setDefaultHeaders(defaultHeaders).build();
+        failureListener = new TrackingFailureListener();
+        restClient.setFailureListener(failureListener);
+    }
+
+    /**
+     * Verifies the content of the {@link HttpRequest} that's internally created and passed through to the http client
+     */
+    public void testInternalHttpRequest() throws Exception {
+        ArgumentCaptor<HttpUriRequest> requestArgumentCaptor = ArgumentCaptor.forClass(HttpUriRequest.class);
+        int times = 0;
+        for (String httpMethod : getHttpMethods()) {
+            HttpUriRequest expectedRequest = performRandomRequest(httpMethod);
+            verify(httpClient, times(++times)).execute(any(HttpHost.class), requestArgumentCaptor.capture());
+            HttpUriRequest actualRequest = requestArgumentCaptor.getValue();
+            assertEquals(expectedRequest.getURI(), actualRequest.getURI());
+            assertEquals(expectedRequest.getClass(), actualRequest.getClass());
+            assertArrayEquals(expectedRequest.getAllHeaders(), actualRequest.getAllHeaders());
+            if (expectedRequest instanceof HttpEntityEnclosingRequest) {
+                HttpEntity expectedEntity = ((HttpEntityEnclosingRequest) expectedRequest).getEntity();
+                if (expectedEntity != null) {
+                    HttpEntity actualEntity = ((HttpEntityEnclosingRequest) actualRequest).getEntity();
+                    assertEquals(EntityUtils.toString(expectedEntity), EntityUtils.toString(actualEntity));
+                }
+            }
+        }
+    }
+
+    public void testSetNodes() throws IOException {
+        try {
+            restClient.setHosts((HttpHost[]) null);
+            fail("setHosts should have failed");
+        } catch (IllegalArgumentException e) {
+            assertEquals("hosts must not be null nor empty", e.getMessage());
+        }
+        try {
+            restClient.setHosts();
+            fail("setHosts should have failed");
+        } catch (IllegalArgumentException e) {
+            assertEquals("hosts must not be null nor empty", e.getMessage());
+        }
+        try {
+            restClient.setHosts((HttpHost) null);
+            fail("setHosts should have failed");
+        } catch (NullPointerException e) {
+            assertEquals("host cannot be null", e.getMessage());
+        }
+        try {
+            restClient.setHosts(new HttpHost("localhost", 9200), null, new HttpHost("localhost", 9201));
+            fail("setHosts should have failed");
+        } catch (NullPointerException e) {
+            assertEquals("host cannot be null", e.getMessage());
+        }
+    }
+
+    /**
+     * End to end test for ok status codes
+     */
+    public void testOkStatusCodes() throws Exception {
+        for (String method : getHttpMethods()) {
+            for (int okStatusCode : getOkStatusCodes()) {
+                ElasticsearchResponse response = restClient.performRequest(method, "/" + okStatusCode,
+                        Collections.<String, String>emptyMap(), null);
+                assertThat(response.getStatusLine().getStatusCode(), equalTo(okStatusCode));
+            }
+        }
+        failureListener.assertNotCalled();
+    }
+
+    /**
+     * End to end test for error status codes: they should cause an exception to be thrown, apart from 404 with HEAD requests
+     */
+    public void testErrorStatusCodes() throws Exception {
+        for (String method : getHttpMethods()) {
+            //error status codes should cause an exception to be thrown
+            for (int errorStatusCode : getAllErrorStatusCodes()) {
+                try (ElasticsearchResponse response = restClient.performRequest(method, "/" + errorStatusCode,
+                        Collections.<String, String>emptyMap(), null)) {
+                    if (method.equals("HEAD") && errorStatusCode == 404) {
+                        //no exception gets thrown although we got a 404
+                        assertThat(response.getStatusLine().getStatusCode(), equalTo(errorStatusCode));
+                    } else {
+                        fail("request should have failed");
+                    }
+                } catch(ElasticsearchResponseException e) {
+                    if (method.equals("HEAD") && errorStatusCode == 404) {
+                        throw e;
+                    }
+                    assertThat(e.getElasticsearchResponse().getStatusLine().getStatusCode(), equalTo(errorStatusCode));
+                }
+                if (errorStatusCode <= 500) {
+                    failureListener.assertNotCalled();
+                } else {
+                    failureListener.assertCalled(httpHost);
+                }
+            }
+        }
+    }
+
+    public void testIOExceptions() throws IOException {
+        for (String method : getHttpMethods()) {
+            //IOExceptions should be let bubble up
+            try {
+                restClient.performRequest(method, "/coe", Collections.<String, String>emptyMap(), null);
+                fail("request should have failed");
+            } catch(IOException e) {
+                assertThat(e, instanceOf(ConnectTimeoutException.class));
+            }
+            failureListener.assertCalled(httpHost);
+            try {
+                restClient.performRequest(method, "/soe", Collections.<String, String>emptyMap(), null);
+                fail("request should have failed");
+            } catch(IOException e) {
+                assertThat(e, instanceOf(SocketTimeoutException.class));
+            }
+            failureListener.assertCalled(httpHost);
+        }
+    }
+
+    /**
+     * End to end test for request and response body. Exercises the mock http client ability to send back
+     * whatever body it has received.
+     */
+    public void testBody() throws Exception {
+        String body = "{ \"field\": \"value\" }";
+        StringEntity entity = new StringEntity(body);
+        for (String method : Arrays.asList("DELETE", "GET", "PATCH", "POST", "PUT")) {
+            for (int okStatusCode : getOkStatusCodes()) {
+                try (ElasticsearchResponse response = restClient.performRequest(method, "/" + okStatusCode,
+                        Collections.<String, String>emptyMap(), entity)) {
+                    assertThat(response.getStatusLine().getStatusCode(), equalTo(okStatusCode));
+                    assertThat(EntityUtils.toString(response.getEntity()), equalTo(body));
+                }
+            }
+            for (int errorStatusCode : getAllErrorStatusCodes()) {
+                try {
+                    restClient.performRequest(method, "/" + errorStatusCode, Collections.<String, String>emptyMap(), entity);
+                    fail("request should have failed");
+                } catch(ElasticsearchResponseException e) {
+                    ElasticsearchResponse response = e.getElasticsearchResponse();
+                    assertThat(response.getStatusLine().getStatusCode(), equalTo(errorStatusCode));
+                    assertThat(EntityUtils.toString(response.getEntity()), equalTo(body));
+                }
+            }
+        }
+        for (String method : Arrays.asList("HEAD", "OPTIONS", "TRACE")) {
+            try {
+                restClient.performRequest(method, "/" + randomStatusCode(random()),
+                        Collections.<String, String>emptyMap(), entity);
+                fail("request should have failed");
+            } catch(UnsupportedOperationException e) {
+                assertThat(e.getMessage(), equalTo(method + " with body is not supported"));
+            }
+        }
+    }
+
+    public void testNullHeaders() throws Exception {
+        String method = randomHttpMethod(random());
+        int statusCode = randomStatusCode(random());
+        try {
+            restClient.performRequest(method, "/" + statusCode, Collections.<String, String>emptyMap(), null, (Header[])null);
+            fail("request should have failed");
+        } catch(NullPointerException e) {
+            assertEquals("request headers must not be null", e.getMessage());
+        }
+        try {
+            restClient.performRequest(method, "/" + statusCode, Collections.<String, String>emptyMap(), null, (Header)null);
+            fail("request should have failed");
+        } catch(NullPointerException e) {
+            assertEquals("request header must not be null", e.getMessage());
+        }
+    }
+
+    public void testNullParams() throws Exception {
+        String method = randomHttpMethod(random());
+        int statusCode = randomStatusCode(random());
+        try {
+            restClient.performRequest(method, "/" + statusCode, null, null);
+            fail("request should have failed");
+        } catch(NullPointerException e) {
+            assertEquals("params must not be null", e.getMessage());
+        }
+    }
+
+    /**
+     * End to end test for request and response headers. Exercises the mock http client ability to send back
+     * whatever headers it has received.
+     */
+    public void testHeaders() throws Exception {
+        for (String method : getHttpMethods()) {
+            Map<String, String> expectedHeaders = new HashMap<>();
+            for (Header defaultHeader : defaultHeaders) {
+                expectedHeaders.put(defaultHeader.getName(), defaultHeader.getValue());
+            }
+            int numHeaders = RandomInts.randomIntBetween(random(), 1, 5);
+            Header[] headers = new Header[numHeaders];
+            for (int i = 0; i < numHeaders; i++) {
+                String headerName = "Header" + (random().nextBoolean() ? i : "");
+                String headerValue = RandomStrings.randomAsciiOfLengthBetween(random(), 3, 10);
+                headers[i] = new BasicHeader(headerName, headerValue);
+                expectedHeaders.put(headerName, headerValue);
+            }
+
+            int statusCode = randomStatusCode(random());
+            ElasticsearchResponse esResponse;
+            try (ElasticsearchResponse response = restClient.performRequest(method, "/" + statusCode,
+                    Collections.<String, String>emptyMap(), null, headers)) {
+                esResponse = response;
+            } catch(ElasticsearchResponseException e) {
+                esResponse = e.getElasticsearchResponse();
+            }
+            assertThat(esResponse.getStatusLine().getStatusCode(), equalTo(statusCode));
+            for (Header responseHeader : esResponse.getHeaders()) {
+                String headerValue = expectedHeaders.remove(responseHeader.getName());
+                assertNotNull("found response header [" + responseHeader.getName() + "] that wasn't originally sent", headerValue);
+            }
+            assertEquals("some headers that were sent weren't returned " + expectedHeaders, 0, expectedHeaders.size());
+        }
+    }
+
+    private HttpUriRequest performRandomRequest(String method) throws IOException, URISyntaxException {
+        String uriAsString = "/" + randomStatusCode(random());
+        URIBuilder uriBuilder = new URIBuilder(uriAsString);
+        Map<String, String> params = Collections.emptyMap();
+        if (random().nextBoolean()) {
+            int numParams = RandomInts.randomIntBetween(random(), 1, 3);
+            params = new HashMap<>(numParams);
+            for (int i = 0; i < numParams; i++) {
+                String paramKey = "param-" + i;
+                String paramValue = RandomStrings.randomAsciiOfLengthBetween(random(), 3, 10);
+                params.put(paramKey, paramValue);
+                uriBuilder.addParameter(paramKey, paramValue);
+            }
+        }
+        URI uri = uriBuilder.build();
+
+        HttpUriRequest request;
+        switch(method) {
+            case "DELETE":
+                request = new HttpDeleteWithEntity(uri);
+                break;
+            case "GET":
+                request = new HttpGetWithEntity(uri);
+                break;
+            case "HEAD":
+                request = new HttpHead(uri);
+                break;
+            case "OPTIONS":
+                request = new HttpOptions(uri);
+                break;
+            case "PATCH":
+                request = new HttpPatch(uri);
+                break;
+            case "POST":
+                request = new HttpPost(uri);
+                break;
+            case "PUT":
+                request = new HttpPut(uri);
+                break;
+            case "TRACE":
+                request = new HttpTrace(uri);
+                break;
+            default:
+                throw new UnsupportedOperationException("method not supported: " + method);
+        }
+
+        HttpEntity entity = null;
+        if (request instanceof HttpEntityEnclosingRequest && random().nextBoolean()) {
+            entity = new StringEntity(RandomStrings.randomAsciiOfLengthBetween(random(), 10, 100));
+            ((HttpEntityEnclosingRequest) request).setEntity(entity);
+        }
+
+        Header[] headers = new Header[0];
+        for (Header defaultHeader : defaultHeaders) {
+            //default headers are expected but not sent for each request
+            request.setHeader(defaultHeader);
+        }
+        if (random().nextBoolean()) {
+            int numHeaders = RandomInts.randomIntBetween(random(), 1, 5);
+            headers = new Header[numHeaders];
+            for (int i = 0; i < numHeaders; i++) {
+                String headerName = "Header" + (random().nextBoolean() ? i : "");
+                String headerValue = RandomStrings.randomAsciiOfLengthBetween(random(), 3, 10);
+                BasicHeader basicHeader = new BasicHeader(headerName, headerValue);
+                headers[i] = basicHeader;
+                request.setHeader(basicHeader);
+            }
+        }
+
+        try {
+            restClient.performRequest(method, uriAsString, params, entity, headers);
+        } catch(ElasticsearchResponseException e) {
+            //all good
+        }
+        return request;
+    }
+}
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientTestUtil.java b/client/src/test/java/org/elasticsearch/client/RestClientTestUtil.java
new file mode 100644
index 00000000000..4d4aa00f492
--- /dev/null
+++ b/client/src/test/java/org/elasticsearch/client/RestClientTestUtil.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+
+final class RestClientTestUtil {
+
+    private static final String[] HTTP_METHODS = new String[]{"DELETE", "HEAD", "GET", "OPTIONS", "PATCH", "POST", "PUT", "TRACE"};
+    private static final List<Integer> ALL_STATUS_CODES;
+    private static final List<Integer> OK_STATUS_CODES = Arrays.asList(200, 201);
+    private static final List<Integer> ALL_ERROR_STATUS_CODES;
+    private static List<Integer> ERROR_NO_RETRY_STATUS_CODES = Arrays.asList(400, 401, 403, 404, 405, 500);
+    private static List<Integer> ERROR_RETRY_STATUS_CODES = Arrays.asList(502, 503, 504);
+
+    static {
+        ALL_ERROR_STATUS_CODES = new ArrayList<>(ERROR_RETRY_STATUS_CODES);
+        ALL_ERROR_STATUS_CODES.addAll(ERROR_NO_RETRY_STATUS_CODES);
+        ALL_STATUS_CODES = new ArrayList<>(ALL_ERROR_STATUS_CODES);
+        ALL_STATUS_CODES.addAll(OK_STATUS_CODES);
+    }
+
+    private RestClientTestUtil() {
+
+    }
+
+    static String[] getHttpMethods() {
+        return HTTP_METHODS;
+    }
+
+    static String randomHttpMethod(Random random) {
+        return RandomPicks.randomFrom(random, HTTP_METHODS);
+    }
+
+    static int randomStatusCode(Random random) {
+        return RandomPicks.randomFrom(random, ALL_ERROR_STATUS_CODES);
+    }
+
+    static int randomOkStatusCode(Random random) {
+        return RandomPicks.randomFrom(random, OK_STATUS_CODES);
+    }
+
+    static int randomErrorNoRetryStatusCode(Random random) {
+        return RandomPicks.randomFrom(random, ERROR_NO_RETRY_STATUS_CODES);
+    }
+
+    static int randomErrorRetryStatusCode(Random random) {
+        return RandomPicks.randomFrom(random, ERROR_RETRY_STATUS_CODES);
+    }
+
+    static List<Integer> getOkStatusCodes() {
+        return OK_STATUS_CODES;
+    }
+
+    static List<Integer> getAllErrorStatusCodes() {
+        return ALL_ERROR_STATUS_CODES;
+    }
+
+    static List<Integer> getAllStatusCodes() {
+        return ALL_STATUS_CODES;
+    }
+}
diff --git a/client/src/test/java/org/elasticsearch/client/TrackingFailureListener.java b/client/src/test/java/org/elasticsearch/client/TrackingFailureListener.java
new file mode 100644
index 00000000000..35842823923
--- /dev/null
+++ b/client/src/test/java/org/elasticsearch/client/TrackingFailureListener.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.HttpHost;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+
+/**
+ * {@link org.elasticsearch.client.RestClient.FailureListener} impl that allows to track when it gets called
+ */
+class TrackingFailureListener extends RestClient.FailureListener {
+    private Set<HttpHost> hosts = new HashSet<>();
+
+    @Override
+    public void onFailure(HttpHost host) throws IOException {
+        hosts.add(host);
+    }
+
+    void assertCalled(HttpHost... hosts) {
+        assertEquals(hosts.length, this.hosts.size());
+        assertThat(this.hosts, containsInAnyOrder(hosts));
+        this.hosts.clear();
+    }
+
+    void assertNotCalled() {
+        assertEquals(0, hosts.size());
+    }
+}
\ No newline at end of file

From 13a27a34f8bed4fe7243d0c8c2ead5169009a2a0 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 3 Jun 2016 15:56:43 +0200
Subject: [PATCH 0628/1311] [TEST] add RestClient integration test

Relies on real HttpServer to test the actual interaction between RestClient and HttpClient, and how requests get sent in real life.
---
 .../client/RestClientIntegTests.java          | 204 ++++++++++++++++++
 1 file changed, 204 insertions(+)
 create mode 100644 client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java

diff --git a/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java b/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
new file mode 100644
index 00000000000..420ff6ed7f7
--- /dev/null
+++ b/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+import com.sun.net.httpserver.Headers;
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpServer;
+import org.apache.http.Consts;
+import org.apache.http.Header;
+import org.apache.http.HttpHost;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.util.EntityUtils;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.elasticsearch.client.RestClientTestUtil.getAllStatusCodes;
+import static org.elasticsearch.client.RestClientTestUtil.getHttpMethods;
+import static org.elasticsearch.client.RestClientTestUtil.randomStatusCode;
+import static org.hamcrest.CoreMatchers.equalTo;
+
+/**
+ * Integration test to check interaction between {@link RestClient} and {@link org.apache.http.client.HttpClient}.
+ * Works against a real http server, one single host.
+ */
+public class RestClientIntegTests extends LuceneTestCase {
+
+    private static HttpServer httpServer;
+    private static RestClient restClient;
+    private static Header[] defaultHeaders;
+
+    @BeforeClass
+    public static void startHttpServer() throws Exception {
+        httpServer = HttpServer.create(new InetSocketAddress(0), 0);
+        httpServer.start();
+        //returns a different status code depending on the path
+        for (int statusCode : getAllStatusCodes()) {
+            createStatusCodeContext(httpServer, statusCode);
+        }
+        int numHeaders = RandomInts.randomIntBetween(random(), 0, 3);
+        defaultHeaders = new Header[numHeaders];
+        for (int i = 0; i < numHeaders; i++) {
+            String headerName = "Header-default" + (random().nextBoolean() ? i : "");
+            String headerValue = RandomStrings.randomAsciiOfLengthBetween(random(), 3, 10);
+            defaultHeaders[i] = new BasicHeader(headerName, headerValue);
+        }
+        restClient = RestClient.builder().setDefaultHeaders(defaultHeaders)
+                .setHosts(new HttpHost(httpServer.getAddress().getHostName(), httpServer.getAddress().getPort())).build();
+    }
+
+    private static void createStatusCodeContext(HttpServer httpServer, final int statusCode) {
+        httpServer.createContext("/" + statusCode, new HttpHandler() {
+            @Override
+            public void handle(HttpExchange httpExchange) throws IOException {
+                StringBuilder body = new StringBuilder();
+                try (InputStreamReader reader = new InputStreamReader(httpExchange.getRequestBody(), Consts.UTF_8)) {
+                    char[] buffer = new char[256];
+                    int read;
+                    while ((read = reader.read(buffer)) != -1) {
+                        body.append(buffer, 0, read);
+                    }
+                }
+                Headers requestHeaders = httpExchange.getRequestHeaders();
+                Headers responseHeaders = httpExchange.getResponseHeaders();
+                for (Map.Entry<String, List<String>> header : requestHeaders.entrySet()) {
+                    responseHeaders.put(header.getKey(), header.getValue());
+                }
+                httpExchange.getRequestBody().close();
+                httpExchange.sendResponseHeaders(statusCode, body.length() == 0 ? -1 : body.length());
+                if (body.length() > 0) {
+                    try (OutputStream out = httpExchange.getResponseBody()) {
+                        out.write(body.toString().getBytes(Consts.UTF_8));
+                    }
+                }
+                httpExchange.close();
+            }
+        });
+    }
+
+    @AfterClass
+    public static void stopHttpServers() throws IOException {
+        restClient.close();
+        restClient = null;
+        httpServer.stop(0);
+        httpServer = null;
+    }
+
+    /**
+     * End to end test for headers. We test it explicitly against a real http client as there are different ways
+     * to set/add headers to the {@link org.apache.http.client.HttpClient}.
+     * Exercises the test http server ability to send back whatever headers it received.
+     */
+    public void testHeaders() throws Exception {
+        for (String method : getHttpMethods()) {
+            Set<String> standardHeaders = new HashSet<>(
+                    Arrays.asList("Accept-encoding", "Connection", "Host", "User-agent", "Date"));
+            if (method.equals("HEAD") == false) {
+                standardHeaders.add("Content-length");
+            }
+            int numHeaders = RandomInts.randomIntBetween(random(), 1, 5);
+            Map<String, String> expectedHeaders = new HashMap<>();
+            for (Header defaultHeader : defaultHeaders) {
+                expectedHeaders.put(defaultHeader.getName(), defaultHeader.getValue());
+            }
+            Header[] headers = new Header[numHeaders];
+            for (int i = 0; i < numHeaders; i++) {
+                String headerName = "Header" + (random().nextBoolean() ? i : "");
+                String headerValue = RandomStrings.randomAsciiOfLengthBetween(random(), 3, 10);
+                headers[i] = new BasicHeader(headerName, headerValue);
+                expectedHeaders.put(headerName, headerValue);
+            }
+
+            int statusCode = randomStatusCode(random());
+            ElasticsearchResponse esResponse;
+            try (ElasticsearchResponse response = restClient.performRequest(method, "/" + statusCode,
+                    Collections.<String, String>emptyMap(), null, headers)) {
+                esResponse = response;
+            } catch(ElasticsearchResponseException e) {
+                esResponse = e.getElasticsearchResponse();
+            }
+            assertThat(esResponse.getStatusLine().getStatusCode(), equalTo(statusCode));
+            for (Header responseHeader : esResponse.getHeaders()) {
+                if (responseHeader.getName().startsWith("Header")) {
+                    String headerValue = expectedHeaders.remove(responseHeader.getName());
+                    assertNotNull("found response header [" + responseHeader.getName() + "] that wasn't originally sent", headerValue);
+                } else {
+                    assertTrue("unknown header was returned " + responseHeader.getName(),
+                            standardHeaders.remove(responseHeader.getName()));
+                }
+            }
+            assertEquals("some headers that were sent weren't returned: " + expectedHeaders, 0, expectedHeaders.size());
+            assertEquals("some expected standard headers weren't returned: " + standardHeaders, 0, standardHeaders.size());
+        }
+    }
+
+    /**
+     * End to end test for delete with body. We test it explicitly as it is not supported
+     * out of the box by {@link org.apache.http.client.HttpClient}.
+     * Exercises the test http server ability to send back whatever body it received.
+     */
+    public void testDeleteWithBody() throws Exception {
+        testBody("DELETE");
+    }
+
+    /**
+     * End to end test for get with body. We test it explicitly as it is not supported
+     * out of the box by {@link org.apache.http.client.HttpClient}.
+     * Exercises the test http server ability to send back whatever body it received.
+     */
+    public void testGetWithBody() throws Exception {
+        testBody("GET");
+    }
+
+    private void testBody(String method) throws Exception {
+        String requestBody = "{ \"field\": \"value\" }";
+        StringEntity entity = new StringEntity(requestBody);
+        ElasticsearchResponse esResponse;
+        String responseBody;
+        int statusCode = randomStatusCode(random());
+        try (ElasticsearchResponse response = restClient.performRequest(method, "/" + statusCode,
+                Collections.<String, String>emptyMap(), entity)) {
+            responseBody = EntityUtils.toString(response.getEntity());
+            esResponse = response;
+        } catch(ElasticsearchResponseException e) {
+            responseBody = e.getResponseBody();
+            esResponse = e.getElasticsearchResponse();
+        }
+        assertEquals(statusCode, esResponse.getStatusLine().getStatusCode());
+        assertEquals(requestBody, responseBody);
+    }
+}

From d55f719f8abe56a25f9d78999ef356a260599a78 Mon Sep 17 00:00:00 2001
From: Britta Weber <brwe@users.noreply.github.com>
Date: Fri, 3 Jun 2016 16:37:28 +0200
Subject: [PATCH 0629/1311] [TEST] wait for yellow after setup doc tests
 (#18726)

* [TEST] wait for yellow after setup doc tests

We have many places in the doc where we expect and index to be
yellow before we execute a query. Therefore we have to
always wait for yellow after setup.
---
 .../doc/RestTestsFromSnippetsTask.groovy      |     7 +
 docs/out                                      | 11165 ++++++++++++++++
 .../reference/query-dsl/nested-query.asciidoc |     1 -
 .../query-dsl/parent-id-query.asciidoc        |     1 -
 4 files changed, 11172 insertions(+), 2 deletions(-)
 create mode 100644 docs/out

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
index 9f840df36e1..c9f5668a326 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
@@ -170,6 +170,13 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
             current.println('---')
             current.println("setup:")
             body(setup)
+            // always wait for yellow before anything is executed
+            current.println(
+                    "  - do:\n" +
+                    "      raw:\n" +
+                    "        method: GET\n" +
+                    "        path: \"_cluster/health\"\n" +
+                    "        wait_for_status: \"yellow\"")
         }
 
         private void body(Snippet snippet) {
diff --git a/docs/out b/docs/out
new file mode 100644
index 00000000000..5a0af906e79
--- /dev/null
+++ b/docs/out
@@ -0,0 +1,11165 @@
+:buildSrc:compileJava UP-TO-DATE
+:buildSrc:compileGroovy
+:buildSrc:writeVersionProperties UP-TO-DATE
+:buildSrc:processResources UP-TO-DATE
+:buildSrc:classes
+:buildSrc:jar
+:buildSrc:assemble
+:buildSrc:compileTestJava UP-TO-DATE
+:buildSrc:compileTestGroovy UP-TO-DATE
+:buildSrc:processTestResources UP-TO-DATE
+:buildSrc:testClasses UP-TO-DATE
+:buildSrc:test UP-TO-DATE
+:buildSrc:check UP-TO-DATE
+:buildSrc:build
+=======================================
+Elasticsearch Build Hamster says Hello!
+=======================================
+  Gradle Version        : 2.13
+  OS Info               : Linux 3.13.0-39-generic (amd64)
+  JDK Version           : Oracle Corporation 1.8.0_65 [Java HotSpot(TM) 64-Bit Server VM 25.65-b01]
+  JAVA_HOME             : /usr/java/jdk1.8.0_65
+:docs:clean
+:docs:processTestResources UP-TO-DATE
+:rest-api-spec:compileJava UP-TO-DATE
+:rest-api-spec:processResources UP-TO-DATE
+:rest-api-spec:classes UP-TO-DATE
+:rest-api-spec:jar UP-TO-DATE
+:docs:copyRestSpec
+:docs:buildRestTests
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "title": {
+          "type":     "text",
+          "analyzer": "standard"
+        }
+      }
+    }
+  }
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "my_number": {
+          "type": "long",
+          "fields": {
+            "keyword": {
+              "type":  "keyword"
+            }
+          }
+        }
+      }
+    }
+  }
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "analyzer": "whitespace",
+  "text":     "The quick brown fox."
+}
+emit snippet
+body part: 
+{
+  "tokenizer": "standard",
+  "filter":  [ "lowercase", "asciifolding" ],
+  "text":      "Is this déja vu?"
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "std_folded": {
+          "type": "custom",
+          "tokenizer": "standard",
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ]
+        }
+      }
+    }
+  },
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "my_text": {
+          "type": "text",
+          "analyzer": "std_folded"
+        }
+      }
+    }
+  }
+}
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "analyzer": "std_folded",
+  "text":     "Is this déjà vu?"
+}
+emit snippet
+body part: 
+{
+  "field": "my_text",
+  "text":  "Is this déjà vu?"
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "tokenizer": "classic",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "my_tokenizer"
+        }
+      },
+      "tokenizer": {
+        "my_tokenizer": {
+          "type": "classic",
+          "max_token_length": 5
+        }
+      }
+    }
+  }
+}
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "analyzer": "my_analyzer",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "tokenizer": "whitespace",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "tokenizer": "ngram",
+  "text": "Quick Fox"
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "my_tokenizer"
+        }
+      },
+      "tokenizer": {
+        "my_tokenizer": {
+          "type": "ngram",
+          "min_gram": 3,
+          "max_gram": 3,
+          "token_chars": [
+            "letter",
+            "digit"
+          ]
+        }
+      }
+    }
+  }
+}
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "analyzer": "my_analyzer",
+  "text": "2 Quick Foxes."
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "tokenizer": "letter",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "tokenizer": "pattern",
+  "text": "The foo_bar_size's default is 5."
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "my_tokenizer"
+        }
+      },
+      "tokenizer": {
+        "my_tokenizer": {
+          "type": "pattern",
+          "pattern": ","
+        }
+      }
+    }
+  }
+}
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "analyzer": "my_analyzer",
+  "text": "comma,separated,values"
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "my_tokenizer"
+        }
+      },
+      "tokenizer": {
+        "my_tokenizer": {
+          "type": "pattern",
+          "pattern": "\"((?:\\\\\"|[^\"]|\\\\\")+)\"",
+          "group": 1
+        }
+      }
+    }
+  }
+}
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "analyzer": "my_analyzer",
+  "text": "\"value\", \"value with embedded \\\" quote\""
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "tokenizer": "edge_ngram",
+  "text": "Quick Fox"
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "my_tokenizer"
+        }
+      },
+      "tokenizer": {
+        "my_tokenizer": {
+          "type": "edge_ngram",
+          "min_gram": 2,
+          "max_gram": 10,
+          "token_chars": [
+            "letter",
+            "digit"
+          ]
+        }
+      }
+    }
+  }
+}
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "analyzer": "my_analyzer",
+  "text": "2 Quick Foxes."
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "autocomplete": {
+          "tokenizer": "autocomplete",
+          "filter": [
+            "lowercase"
+          ]
+        },
+        "autocomplete_search": {
+          "tokenizer": "lowercase"
+        }
+      },
+      "tokenizer": {
+        "autocomplete": {
+          "type": "edge_ngram",
+          "min_gram": 2,
+          "max_gram": 10,
+          "token_chars": [
+            "letter"
+          ]
+        }
+      }
+    }
+  },
+  "mappings": {
+    "doc": {
+      "properties": {
+        "title": {
+          "type": "text",
+          "analyzer": "autocomplete",
+          "search_analyzer": "autocomplete_search"
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "title": "Quick Foxes"
+}
+emit snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "match": {
+      "title": {
+        "query": "Quick Fo",
+        "operator": "and"
+      }
+    }
+  }
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "tokenizer": "uax_url_email",
+  "text": "Email me at john.smith@global-international.com"
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "my_tokenizer"
+        }
+      },
+      "tokenizer": {
+        "my_tokenizer": {
+          "type": "uax_url_email",
+          "max_token_length": 5
+        }
+      }
+    }
+  }
+}
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "analyzer": "my_analyzer",
+  "text": "john.smith@global-international.com"
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "tokenizer": "path_hierarchy",
+  "text": "/one/two/three"
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "my_tokenizer"
+        }
+      },
+      "tokenizer": {
+        "my_tokenizer": {
+          "type": "path_hierarchy",
+          "delimiter": "-",
+          "replacement": "/",
+          "skip": 2
+        }
+      }
+    }
+  }
+}
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "analyzer": "my_analyzer",
+  "text": "one-two-three-four-five"
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "tokenizer": "standard",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "my_tokenizer"
+        }
+      },
+      "tokenizer": {
+        "my_tokenizer": {
+          "type": "standard",
+          "max_token_length": 5
+        }
+      }
+    }
+  }
+}
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "analyzer": "my_analyzer",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "tokenizer": "keyword",
+  "text": "New York"
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "tokenizer": "thai",
+  "text": "�ารที่ได้ต้อง�สดงว่างานดี"
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "tokenizer": "lowercase",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "analyzer": "standard",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_english_analyzer": {
+          "type": "standard",
+          "max_token_length": 5,
+          "stopwords": "_english_"
+        }
+      }
+    }
+  }
+}
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "analyzer": "my_english_analyzer",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "analyzer": "simple",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "analyzer": "whitespace",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "analyzer": "keyword",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "std_english": {
+          "type":      "standard",
+          "stopwords": "_english_"
+        }
+      }
+    }
+  },
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "my_text": {
+          "type":     "text",
+          "analyzer": "standard",
+          "fields": {
+            "english": {
+              "type":     "text",
+              "analyzer": "std_english"
+            }
+          }
+        }
+      }
+    }
+  }
+}
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "field": "my_text",
+  "text": "The old brown cow"
+}
+emit snippet
+body part: 
+{
+  "field": "my_text.english",
+  "text": "The old brown cow"
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "analyzer": "stop",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_stop_analyzer": {
+          "type": "stop",
+          "stopwords": ["the", "over"]
+        }
+      }
+    }
+  }
+}
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "analyzer": "my_stop_analyzer",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_custom_analyzer": {
+          "type":      "custom",
+          "tokenizer": "standard",
+          "char_filter": [
+            "html_strip"
+          ],
+          "filter": [
+            "lowercase",
+            "asciifolding"
+          ]
+        }
+      }
+    }
+  }
+}
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "analyzer": "my_custom_analyzer",
+  "text": "Is this <b>déjà vu</b>?"
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_custom_analyzer": {
+          "type": "custom",
+          "char_filter": [
+            "emoticons"
+          ],
+          "tokenizer": "punctuation",
+          "filter": [
+            "lowercase",
+            "english_stop"
+          ]
+        }
+      },
+      "tokenizer": {
+        "punctuation": {
+          "type": "pattern",
+          "pattern": "[ .,!?]"
+        }
+      },
+      "char_filter": {
+        "emoticons": {
+          "type": "mapping",
+          "mappings": [
+            ":) => _happy_",
+            ":( => _sad_"
+          ]
+        }
+      },
+      "filter": {
+        "english_stop": {
+          "type": "stop",
+          "stopwords": "_english_"
+        }
+      }
+    }
+  }
+}
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "analyzer": "my_custom_analyzer",
+  "text":     "I'm a :) person, and you?"
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "analyzer": "pattern",
+  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_email_analyzer": {
+          "type":      "pattern",
+          "pattern":   "\\W|_",
+          "lowercase": true
+        }
+      }
+    }
+  }
+}
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "analyzer": "my_email_analyzer",
+  "text": "John_Smith@foo-bar.com"
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "camel": {
+          "type": "pattern",
+          "pattern": "([^\\p{L}\\d]+)|(?<=\\D)(?=\\d)|(?<=\\d)(?=\\D)|(?<=[\\p{L}&&[^\\p{Lu}]])(?=\\p{Lu})|(?<=\\p{Lu})(?=\\p{Lu}[\\p{L}&&[^\\p{Lu}]])"
+        }
+      }
+    }
+  }
+}
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "analyzer": "camel",
+  "text": "MooseX::FTPClass2_beta"
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "analyzer": "fingerprint",
+  "text": "Yes yes, Gödel said this sentence is consistent and."
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_fingerprint_analyzer": {
+          "type": "fingerprint",
+          "stopwords": "_english_"
+        }
+      }
+    }
+  }
+}
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "analyzer": "my_fingerprint_analyzer",
+  "text": "Yes yes, Gödel said this sentence is consistent and."
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "tokenizer":      "keyword",
+  "char_filter":  [ "html_strip" ],
+  "text": "<p>I&apos;m so <b>happy</b>!</p>"
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "keyword",
+          "char_filter": ["my_char_filter"]
+        }
+      },
+      "char_filter": {
+        "my_char_filter": {
+          "type": "html_strip",
+          "escaped_tags": ["b"]
+        }
+      }
+    }
+  }
+}
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "analyzer": "my_analyzer",
+  "text": "<p>I&apos;m so <b>happy</b>!</p>"
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "keyword",
+          "char_filter": [
+            "my_char_filter"
+          ]
+        }
+      },
+      "char_filter": {
+        "my_char_filter": {
+          "type": "mapping",
+          "mappings": [
+            "Ù  => 0",
+            "Ù¡ => 1",
+            "Ù¢ => 2",
+            "Ù£ => 3",
+            "Ù¤ => 4",
+            "Ù¥ => 5",
+            "Ù¦ => 6",
+            "Ù§ => 7",
+            "Ù¨ => 8",
+            "Ù© => 9"
+          ]
+        }
+      }
+    }
+  }
+}
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "analyzer": "my_analyzer",
+  "text": "My license plate is ٢٥٠١٥"
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "standard",
+          "char_filter": [
+            "my_char_filter"
+          ]
+        }
+      },
+      "char_filter": {
+        "my_char_filter": {
+          "type": "mapping",
+          "mappings": [
+            ":) => _happy_",
+            ":( => _sad_"
+          ]
+        }
+      }
+    }
+  }
+}
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "analyzer": "my_analyzer",
+  "text": "I'm delighted about it :("
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "standard",
+          "char_filter": [
+            "my_char_filter"
+          ]
+        }
+      },
+      "char_filter": {
+        "my_char_filter": {
+          "type": "pattern_replace",
+          "pattern": "(\\d+)-(?=\\d)",
+          "replacement": "$1_"
+        }
+      }
+    }
+  }
+}
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "analyzer": "my_analyzer",
+  "text": "My credit card is 123-456-789"
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "my_analyzer": {
+          "tokenizer": "standard",
+          "char_filter": [
+            "my_char_filter"
+          ],
+          "filter": [
+            "lowercase"
+          ]
+        }
+      },
+      "char_filter": {
+        "my_char_filter": {
+          "type": "pattern_replace",
+          "pattern": "(?<=\\p{Lower})(?=\\p{Upper})",
+          "replacement": " "
+        }
+      }
+    }
+  },
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "text": {
+          "type": "text",
+          "analyzer": "my_analyzer"
+        }
+      }
+    }
+  }
+}
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "analyzer": "my_analyzer",
+  "text": "The fooBarBaz method"
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+query part: refresh
+body part: 
+{
+  "text": "The fooBarBaz method"
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "match": {
+      "text": "bar"
+    }
+  },
+  "highlight": {
+    "fields": {
+      "text": {}
+    }
+  }
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query" : {
+        "match_all": {}
+    },
+    "fielddata_fields" : ["test1", "test2"]
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "from" : 0, "size" : 10,
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "_source": false,
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "_source": "obj.*",
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "_source": [ "obj1.*", "obj2.*" ],
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "_source": {
+        "include": [ "obj1.*", "obj2.*" ],
+        "exclude": [ "*.description" ]
+    },
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "explain": true,
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query" : {
+        "match_all": {}
+    },
+    "script_fields" : {
+        "test1" : {
+            "script" : "doc['my_field_name'].value * 2"
+        },
+        "test2" : {
+            "script" : {
+                "inline": "doc['my_field_name'].value * factor",
+                "params" : {
+                    "factor"  : 2.0
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+    {
+        "query" : {
+            "match_all": {}
+        },
+        "script_fields" : {
+            "test1" : {
+                "script" : "_source.obj1.obj2"
+            }
+        }
+    }
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query" : {
+        "match": { "user": "kimchy" }
+    },
+    "highlight" : {
+        "fields" : {
+            "content" : {}
+        }
+    }
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query" : {
+        "match": { "user": "kimchy" }
+    },
+    "highlight" : {
+        "fields" : {
+            "content" : {"type" : "plain"}
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query" : {
+        "match": { "user": "kimchy" }
+    },
+    "highlight" : {
+        "fields" : {
+            "content" : {"force_source" : true}
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query" : {
+        "match": { "user": "kimchy" }
+    },
+    "highlight" : {
+        "pre_tags" : ["<tag1>"],
+        "post_tags" : ["</tag1>"],
+        "fields" : {
+            "_all" : {}
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query" : {
+        "match": { "user": "kimchy" }
+    },
+    "highlight" : {
+        "pre_tags" : ["<tag1>", "<tag2>"],
+        "post_tags" : ["</tag1>", "</tag2>"],
+        "fields" : {
+            "_all" : {}
+        }
+    }
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query" : {
+        "match": { "user": "kimchy" }
+    },
+    "highlight" : {
+        "tags_schema" : "styled",
+        "fields" : {
+            "content" : {}
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query" : {
+        "match": { "user": "kimchy" }
+    },
+    "highlight" : {
+        "fields" : {
+            "content" : {"fragment_size" : 150, "number_of_fragments" : 3}
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query" : {
+        "match": { "user": "kimchy" }
+    },
+    "highlight" : {
+        "order" : "score",
+        "fields" : {
+            "content" : {"fragment_size" : 150, "number_of_fragments" : 3}
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query" : {
+        "match": { "user": "kimchy" }
+    },
+    "highlight" : {
+        "fields" : {
+            "_all" : {},
+            "bio.title" : {"number_of_fragments" : 0}
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query" : {
+        "match": { "user": "kimchy" }
+    },
+    "highlight" : {
+        "fields" : {
+            "content" : {
+                "fragment_size" : 150,
+                "number_of_fragments" : 3,
+                "no_match_size": 150
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "fields": [ "_id" ],
+    "query" : {
+        "match": {
+            "content": {
+                "query": "foo bar"
+            }
+        }
+    },
+    "rescore": {
+        "window_size": 50,
+        "query": {
+            "rescore_query" : {
+                "match_phrase": {
+                    "content": {
+                        "query": "foo bar",
+                        "phrase_slop": 1
+                    }
+                }
+            },
+            "rescore_query_weight" : 10
+        }
+    },
+    "highlight" : {
+        "order" : "score",
+        "fields" : {
+            "content" : {
+                "fragment_size" : 150,
+                "number_of_fragments" : 3,
+                "highlight_query": {
+                    "bool": {
+                        "must": {
+                            "match": {
+                                "content": {
+                                    "query": "foo bar"
+                                }
+                            }
+                        },
+                        "should": {
+                            "match_phrase": {
+                                "content": {
+                                    "query": "foo bar",
+                                    "phrase_slop": 1,
+                                    "boost": 10.0
+                                }
+                            }
+                        },
+                        "minimum_should_match": 0
+                    }
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query" : {
+        "match": { "user": "kimchy" }
+    },
+    "highlight" : {
+        "number_of_fragments" : 3,
+        "fragment_size" : 150,
+        "fields" : {
+            "_all" : { "pre_tags" : ["<em>"], "post_tags" : ["</em>"] },
+            "bio.title" : { "number_of_fragments" : 0 },
+            "bio.author" : { "number_of_fragments" : 0 },
+            "bio.content" : { "number_of_fragments" : 5, "order" : "score" }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query" : {
+        "match": { "user": "kimchy" }
+    },
+    "highlight" : {
+        "require_field_match": false,
+        "fields": {
+                "_all" : { "pre_tags" : ["<em>"], "post_tags" : ["</em>"] }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "query_string": {
+            "query": "content.plain:running scissors",
+            "fields": ["content"]
+        }
+    },
+    "highlight": {
+        "order": "score",
+        "fields": {
+            "content": {
+                "matched_fields": ["content", "content.plain"],
+                "type" : "fvh"
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "query_string": {
+            "query": "running scissors",
+            "fields": ["content", "content.plain^10"]
+        }
+    },
+    "highlight": {
+        "order": "score",
+        "fields": {
+            "content": {
+                "matched_fields": ["content", "content.plain"],
+                "type" : "fvh"
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "query_string": {
+            "query": "running scissors",
+            "fields": ["content", "content.plain^10"]
+        }
+    },
+    "highlight": {
+        "order": "score",
+        "fields": {
+            "content": {
+                "matched_fields": ["content.plain"],
+                "type" : "fvh"
+            }
+        }
+    }
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "min_score": 0.5,
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+query part: preference=xyzabc123
+body part: 
+{
+    "query": {
+        "match": {
+            "title": "elasticsearch"
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool" : {
+            "should" : [
+                {"match" : { "name.first" : {"query" : "shay", "_name" : "first"} }},
+                {"match" : { "name.last" : {"query" : "banon", "_name" : "last"} }}
+            ],
+            "filter" : {
+                "terms" : {
+                    "name.last" : ["banon", "kimchy"],
+                    "_name" : "test"
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "indices_boost" : {
+        "index1" : 1.4,
+        "index2" : 1.3
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "mappings": {
+        "my_type": {
+            "properties": {
+                "post_date": { "type": "date" },
+                "user": {
+                    "type": "keyword"
+                },
+                "name": {
+                    "type": "keyword"
+                },
+                "age": { "type": "integer" }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "sort" : [
+        { "post_date" : {"order" : "asc"}},
+        "user",
+        { "name" : "desc" },
+        { "age" : "desc" },
+        "_score"
+    ],
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+query part: refresh
+body part: 
+{
+   "product": "chocolate",
+    "price": [20, 4]
+}
+emit snippet
+body part: 
+{
+   "query" : {
+      "term" : { "product" : "chocolate" }
+   },
+   "sort" : [
+      {"price" : {"order" : "asc", "mode" : "avg"}}
+   ]
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+   "query" : {
+      "term" : { "product" : "chocolate" }
+   },
+   "sort" : [
+       {
+          "offer.price" : {
+             "mode" :  "avg",
+             "order" : "asc",
+             "nested_path" : "offer",
+             "nested_filter" : {
+                "term" : { "offer.color" : "blue" }
+             }
+          }
+       }
+    ]
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "sort" : [
+        { "price" : {"missing" : "_last"} }
+    ],
+    "query" : {
+        "term" : { "product" : "chocolate" }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "sort" : [
+        { "price" : {"unmapped_type" : "long"} }
+    ],
+    "query" : {
+        "term" : { "product" : "chocolate" }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "sort" : [
+        {
+            "_geo_distance" : {
+                "pin.location" : [-70, 40],
+                "order" : "asc",
+                "unit" : "km",
+		"mode" : "min",
+		"distance_type" : "sloppy_arc"
+            }
+        }
+    ],
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "sort" : [
+        {
+            "_geo_distance" : {
+                "pin.location" : {
+                    "lat" : 40,
+                    "lon" : -70
+                },
+                "order" : "asc",
+                "unit" : "km"
+            }
+        }
+    ],
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "sort" : [
+        {
+            "_geo_distance" : {
+                "pin.location" : "40,-70",
+                "order" : "asc",
+                "unit" : "km"
+            }
+        }
+    ],
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "sort" : [
+        {
+            "_geo_distance" : {
+                "pin.location" : "drm3btev3e86",
+                "order" : "asc",
+                "unit" : "km"
+            }
+        }
+    ],
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "sort" : [
+        {
+            "_geo_distance" : {
+                "pin.location" : [-70, 40],
+                "order" : "asc",
+                "unit" : "km"
+            }
+        }
+    ],
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    },
+    "sort" : {
+        "_script" : {
+            "type" : "number",
+            "script" : {
+                "inline": "doc['field_name'].value * factor",
+                "params" : {
+                    "factor" : 1.1
+                }
+            },
+            "order" : "asc"
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "track_scores": true,
+    "sort" : [
+        { "post_date" : {"order" : "desc"} },
+        { "name" : "desc" },
+        { "age" : "desc" }
+    ],
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "version": true,
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "fields" : ["user", "postDate"],
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "fields" : [],
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
+handle snippet
+null
+emit snippet
+body part: 
+{
+    "mappings": {
+        "item": {
+            "properties": {
+                "brand": { "type": "keyword"},
+                "color": { "type": "keyword"},
+                "model": { "type": "keyword"}
+            }
+        }
+    }
+}
+emit snippet
+query part: refresh
+body part: 
+{
+    "brand": "gucci",
+    "color": "red",
+    "model": "slim"
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "bool": {
+      "filter": [
+        { "term": { "color": "red"   }},
+        { "term": { "brand": "gucci" }}
+      ]
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "bool": {
+      "filter": [
+        { "term": { "color": "red"   }},
+        { "term": { "brand": "gucci" }}
+      ]
+    }
+  },
+  "aggs": {
+    "models": {
+      "terms": { "field": "model" }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "bool": {
+      "filter": {
+        "term": { "brand": "gucci" }
+      }
+    }
+  },
+  "aggs": {
+    "colors": {
+      "terms": { "field": "color" }
+    },
+    "color_red": {
+      "filter": {
+        "term": { "color": "red" }
+      },
+      "aggs": {
+        "models": {
+          "terms": { "field": "model" }
+        }
+      }
+    }
+  },
+  "post_filter": {
+    "term": { "color": "red" }
+  }
+}
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+emit snippet
+body part: 
+{
+    "size": 10,
+    "query": {
+        "match" : {
+            "title" : "elasticsearch"
+        }
+    },
+    "sort": [
+        {"date": "asc"},
+        {"_uid": "desc"}
+    ]
+}
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+emit snippet
+body part: 
+{
+    "size": 10,
+    "query": {
+        "match" : {
+            "title" : "elasticsearch"
+        }
+    },
+    "search_after": [1463538857, "tweet#654323"],
+    "sort": [
+        {"date": "asc"},
+        {"_uid": "desc"}
+    ]
+}
+handle snippet
+null
+emit snippet
+query part: refresh
+body part: 
+{"index":{"_id":1}}
+{"user" : "kimchy", "post_date" : "2009-11-15T14:12:12", "message" : "trying out Elasticsearch"}
+{"index":{"_id":2}}
+{"user" : "kimchi", "post_date" : "2009-11-15T14:12:13", "message" : "My username is similar to @kimchy!"}
+handle snippet
+test snippet
+emit snippet
+query part: q=user:foo
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query" : {
+    "bool" : {
+      "must" : {
+        "query_string" : {
+          "query" : "*:*"
+        }
+      },
+      "filter" : {
+        "term" : { "user" : "kimchy" }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+query part: q=post_date:foo
+handle snippet
+handle snippet
+test snippet
+emit snippet
+query part: q=post_date:foo&explain=true
+handle snippet
+handle snippet
+test snippet
+emit snippet
+query part: rewrite=true
+body part: 
+{
+  "query": {
+    "match": {
+      "user": {
+        "query": "kimchy",
+        "fuzziness": "auto"
+      }
+    }
+  }
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+query part: rewrite=true
+body part: 
+{
+  "query": {
+    "more_like_this": {
+      "like": {
+        "_id": "2"
+      },
+      "boost_terms": 1
+    }
+  }
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+emit snippet
+query part: q=user:kimchy
+handle snippet
+handle snippet
+test snippet
+emit snippet
+query part: refresh
+body part: 
+{
+    "user": "kimchy"
+}
+emit snippet
+query part: q=user:kimchy
+emit snippet
+body part: 
+{
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "multi_match" : {
+      "query":    "this is a test",
+      "fields": [ "subject", "message" ]
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "multi_match" : {
+      "query":    "Will Smith",
+      "fields": [ "title", "*_name" ]
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "multi_match" : {
+      "query" : "this is a test",
+      "fields" : [ "subject^3", "message" ]
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "multi_match" : {
+      "query":      "brown fox",
+      "type":       "best_fields",
+      "fields":     [ "subject", "message" ],
+      "tie_breaker": 0.3
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "dis_max": {
+      "queries": [
+        { "match": { "subject": "brown fox" }},
+        { "match": { "message": "brown fox" }}
+      ],
+      "tie_breaker": 0.3
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "multi_match" : {
+      "query":      "Will Smith",
+      "type":       "best_fields",
+      "fields":     [ "first_name", "last_name" ],
+      "operator":   "and"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "multi_match" : {
+      "query":      "quick brown fox",
+      "type":       "most_fields",
+      "fields":     [ "title", "title.original", "title.shingles" ]
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "bool": {
+      "should": [
+        { "match": { "title":          "quick brown fox" }},
+        { "match": { "title.original": "quick brown fox" }},
+        { "match": { "title.shingles": "quick brown fox" }}
+      ]
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "multi_match" : {
+      "query":      "quick brown f",
+      "type":       "phrase_prefix",
+      "fields":     [ "subject", "message" ]
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "dis_max": {
+      "queries": [
+        { "match_phrase_prefix": { "subject": "quick brown f" }},
+        { "match_phrase_prefix": { "message": "quick brown f" }}
+      ]
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "multi_match" : {
+      "query":      "Will Smith",
+      "type":       "cross_fields",
+      "fields":     [ "first_name", "last_name" ],
+      "operator":   "and"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "multi_match" : {
+      "query":      "Jon",
+      "type":       "cross_fields",
+      "fields":     [
+        "first", "first.edge",
+        "last",  "last.edge"
+      ]
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "bool": {
+      "should": [
+        {
+          "multi_match" : {
+            "query":      "Will Smith",
+            "type":       "cross_fields",
+            "fields":     [ "first", "last" ],
+            "minimum_should_match": "50%"
+          }
+        },
+        {
+          "multi_match" : {
+            "query":      "Will Smith",
+            "type":       "cross_fields",
+            "fields":     [ "*.edge" ]
+          }
+        }
+      ]
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+   "multi_match" : {
+      "query":      "Jon",
+      "type":       "cross_fields",
+      "analyzer":   "standard",
+      "fields":     [ "first", "last", "*.edge" ]
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "query_string" : {
+            "default_field" : "content",
+            "query" : "this AND that OR thus"
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "query_string" : {
+            "fields" : ["content", "name"],
+            "query" : "this AND that"
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "query_string": {
+            "query": "(content:this OR name:this) AND (content:that OR name:that)"
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "query_string" : {
+            "fields" : ["content", "name^5"],
+            "query" : "this AND that OR thus",
+            "use_dis_max" : true
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "query_string" : {
+            "fields" : ["city.*"],
+            "query" : "this AND that OR thus",
+            "use_dis_max" : true
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "query_string" : {
+            "fields" : ["content", "name.*^5"],
+            "query" : "this AND that OR thus",
+            "use_dis_max" : true
+        }
+    }
+}
+handle snippet
+null
+emit snippet
+body part: 
+{
+    "mappings": {
+        "location": {
+            "properties": {
+                "pin": {
+                    "properties": {
+                        "location": {
+                            "type": "geo_point"
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+emit snippet
+body part: 
+{
+    "pin" : {
+        "location" : {
+            "lat" : 40.12,
+            "lon" : -71.34
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_distance" : {
+                    "distance" : "200km",
+                    "pin.location" : {
+                        "lat" : 40,
+                        "lon" : -70
+                    }
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_distance" : {
+                    "distance" : "12km",
+                    "pin.location" : {
+                        "lat" : 40,
+                        "lon" : -70
+                    }
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_distance" : {
+                    "distance" : "12km",
+                    "pin.location" : [-70, 40]
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_distance" : {
+                    "distance" : "12km",
+                    "pin.location" : "40,-70"
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_distance" : {
+                    "distance" : "12km",
+                    "pin.location" : "drm3btev3e86"
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "match_all": {}
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "match_all": { "boost" : 1.2 }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "match_none": {}
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "more_like_this" : {
+            "fields" : ["title", "description"],
+            "like" : "Once upon a time",
+            "min_term_freq" : 1,
+            "max_query_terms" : 12
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "more_like_this" : {
+            "fields" : ["title", "description"],
+            "like" : [
+            {
+                "_index" : "imdb",
+                "_type" : "movies",
+                "_id" : "1"
+            },
+            {
+                "_index" : "imdb",
+                "_type" : "movies",
+                "_id" : "2"
+            },
+            "and potentially some more text here as well"
+            ],
+            "min_term_freq" : 1,
+            "max_query_terms" : 12
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "more_like_this" : {
+            "fields" : ["name.first", "name.last"],
+            "like" : [
+            {
+                "_index" : "marvel",
+                "_type" : "quotes",
+                "doc" : {
+                    "name": {
+                        "first": "Ben",
+                        "last": "Grimm"
+                    },
+                    "tweet": "You got no idea what I'd... what I'd give to be invisible."
+                  }
+            },
+            {
+                "_index" : "marvel",
+                "_type" : "quotes",
+                "_id" : "2"
+            }
+            ],
+            "min_term_freq" : 1,
+            "max_query_terms" : 12
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "mappings": {
+        "movies": {
+            "properties": {
+                "title": {
+                    "type": "text",
+                    "term_vector": "yes"
+                },
+                "description": {
+                    "type": "text"
+                },
+                "tags": {
+                    "type": "text",
+                    "fields" : {
+                        "raw": {
+                            "type" : "text",
+                            "analyzer": "keyword",
+                            "term_vector" : "yes"
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "indices" : {
+            "indices" : ["index1", "index2"],
+            "query" : { "term" : { "tag" : "wow" } },
+            "no_match_query" : { "term" : { "tag" : "kow" } }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "range" : {
+            "age" : {
+                "gte" : 10,
+                "lte" : 20,
+                "boost" : 2.0
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "range" : {
+            "date" : {
+                "gte" : "now-1d/d",
+                "lt" :  "now/d"
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "range" : {
+            "born" : {
+                "gte": "01/01/2012",
+                "lte": "2013",
+                "format": "dd/MM/yyyy||yyyy"
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "range" : {
+            "timestamp" : {
+                "gte": "2015-01-01 00:00:00",
+                "lte": "now",
+                "time_zone": "+01:00"
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "common": {
+            "body": {
+                "query": "this is bonsai cool",
+                    "cutoff_frequency": 0.001
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "common": {
+            "body": {
+                "query": "nelly the elephant as a cartoon",
+                    "cutoff_frequency": 0.001,
+                    "low_freq_operator": "and"
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool": {
+            "must": [
+            { "term": { "body": "nelly"}},
+            { "term": { "body": "elephant"}},
+            { "term": { "body": "cartoon"}}
+            ],
+            "should": [
+            { "term": { "body": "the"}},
+            { "term": { "body": "as"}},
+            { "term": { "body": "a"}}
+            ]
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "common": {
+            "body": {
+                "query": "nelly the elephant as a cartoon",
+                "cutoff_frequency": 0.001,
+                "minimum_should_match": 2
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool": {
+            "must": {
+                "bool": {
+                    "should": [
+                    { "term": { "body": "nelly"}},
+                    { "term": { "body": "elephant"}},
+                    { "term": { "body": "cartoon"}}
+                    ],
+                    "minimum_should_match": 2
+                }
+            },
+            "should": [
+                { "term": { "body": "the"}},
+                { "term": { "body": "as"}},
+                { "term": { "body": "a"}}
+                ]
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "common": {
+            "body": {
+                "query": "nelly the elephant not as a cartoon",
+                    "cutoff_frequency": 0.001,
+                    "minimum_should_match": {
+                        "low_freq" : 2,
+                        "high_freq" : 3
+                    }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool": {
+            "must": {
+                "bool": {
+                    "should": [
+                    { "term": { "body": "nelly"}},
+                    { "term": { "body": "elephant"}},
+                    { "term": { "body": "cartoon"}}
+                    ],
+                    "minimum_should_match": 2
+                }
+            },
+            "should": {
+                "bool": {
+                    "should": [
+                    { "term": { "body": "the"}},
+                    { "term": { "body": "not"}},
+                    { "term": { "body": "as"}},
+                    { "term": { "body": "a"}}
+                    ],
+                    "minimum_should_match": 3
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "common": {
+            "body": {
+                "query": "how not to be",
+                    "cutoff_frequency": 0.001,
+                    "minimum_should_match": {
+                        "low_freq" : 2,
+                        "high_freq" : 3
+                    }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool": {
+            "should": [
+            { "term": { "body": "how"}},
+            { "term": { "body": "not"}},
+            { "term": { "body": "to"}},
+            { "term": { "body": "be"}}
+            ],
+            "minimum_should_match": "3<50%"
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "span_within" : {
+            "little" : {
+                "span_term" : { "field1" : "foo" }
+            },
+            "big" : {
+                "span_near" : {
+                    "clauses" : [
+                        { "span_term" : { "field1" : "bar" } },
+                        { "span_term" : { "field1" : "baz" } }
+                    ],
+                    "slop" : 5,
+                    "in_order" : true
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "span_near" : {
+            "clauses" : [
+                { "span_term" : { "field" : "value1" } },
+                { "span_term" : { "field" : "value2" } },
+                { "span_term" : { "field" : "value3" } }
+            ],
+            "slop" : 12,
+            "in_order" : false
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_distance_range" : {
+                    "from" : "200km",
+                    "to" : "400km",
+                    "pin.location" : {
+                        "lat" : 40,
+                        "lon" : -70
+                    }
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "simple_query_string" : {
+        "query": "\"fried eggs\" +(eggplant | potato) -frittata",
+        "analyzer": "snowball",
+        "fields": ["body^5","_all"],
+        "default_operator": "and"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "simple_query_string" : {
+            "fields" : ["content", "name.*^5"],
+            "query" : "foo bar baz"
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "simple_query_string" : {
+            "query" : "foo | bar + baz*",
+            "flags" : "OR|AND|PREFIX"
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "span_multi":{
+            "match":{
+                "prefix" : { "user" :  { "value" : "ki" } }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "span_multi":{
+            "match":{
+                "prefix" : { "user" :  { "value" : "ki", "boost" : 1.08 } }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "constant_score" : {
+            "filter" : {
+                "term" : { "user" : "kimchy"}
+            },
+            "boost" : 1.2
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "wildcard" : { "user" : "ki*y" }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "wildcard" : { "user" : { "value" : "ki*y", "boost" : 2.0 } }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "wildcard" : { "user" : { "wildcard" : "ki*y", "boost" : 2.0 } }
+    }
+}
+handle snippet
+null
+emit snippet
+body part: 
+{
+    "mappings": {
+        "type1" : {
+            "properties" : {
+                "obj1" : {
+                    "type" : "nested"
+                }
+            }
+        }
+    }
+}
+emit snippet
+query part: wait_for_status=yellow
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "nested" : {
+            "path" : "obj1",
+            "score_mode" : "avg",
+            "query" : {
+                "bool" : {
+                    "must" : [
+                    { "match" : {"obj1.name" : "blue"} },
+                    { "range" : {"obj1.count" : {"gt" : 5}} }
+                    ]
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "term" : { "user" : "Kimchy" }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "bool": {
+      "should": [
+        {
+          "term": {
+            "status": {
+              "value": "urgent",
+              "boost": 2.0
+            }
+          }
+        },
+        {
+          "term": {
+            "status": "normal"
+          }
+        }
+      ]
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "full_text": {
+          "type":  "text"
+        },
+        "exact_value": {
+          "type":  "keyword"
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "full_text":   "Quick Foxes!",
+  "exact_value": "Quick Foxes!"
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "term": {
+      "exact_value": "Quick Foxes!"
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "term": {
+      "full_text": "Quick Foxes!"
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "term": {
+      "full_text": "foxes"
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "match": {
+      "full_text": "Quick Foxes!"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "match" : {
+            "message" : "this is a test"
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "match" : {
+            "message" : {
+                "query" : "this is a test",
+                "operator" : "and"
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "match" : {
+            "message" : {
+                "query" : "to be or not to be",
+                "operator" : "and",
+                "zero_terms_query": "all"
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "match" : {
+            "message" : {
+                "query" : "to be or not to be",
+                "cutoff_frequency" : 0.001
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "bool": {
+      "must": [
+        { "match": { "title":   "Search"        }},
+        { "match": { "content": "Elasticsearch" }}
+      ],
+      "filter": [
+        { "term":  { "status": "published" }},
+        { "range": { "publish_date": { "gte": "2015-01-01" }}}
+      ]
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "ids" : {
+            "type" : "my_type",
+            "values" : ["1", "4", "100"]
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "dis_max" : {
+            "tie_breaker" : 0.7,
+                "boost" : 1.2,
+                "queries" : [
+                {
+                    "term" : { "age" : 34 }
+                },
+                {
+                    "term" : { "age" : 35 }
+                }
+                ]
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "match_phrase" : {
+            "message" : "this is a test"
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "match_phrase" : {
+            "message" : {
+                "query" : "this is a test",
+                "analyzer" : "my_analyzer"
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "script" : {
+                    "script" : "doc['num1'].value > 1"
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "script" : {
+                    "script" : {
+                        "inline" : "doc['num1'].value > param1",
+                        "params" : {
+                            "param1" : 5
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{ "query": {
+    "prefix" : { "user" : "ki" }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{ "query": {
+    "prefix" : { "user" :  { "value" : "ki", "boost" : 2.0 } }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{ "query": {
+    "prefix" : { "user" :  { "prefix" : "ki", "boost" : 2.0 } }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "bool" : {
+      "must" : {
+        "term" : { "user" : "kimchy" }
+      },
+      "filter": {
+        "term" : { "tag" : "tech" }
+      },
+      "must_not" : {
+        "range" : {
+          "age" : { "from" : 10, "to" : 20 }
+        }
+      },
+      "should" : [
+        { "term" : { "tag" : "wow" } },
+        { "term" : { "tag" : "elasticsearch" } }
+      ],
+      "minimum_should_match" : 1,
+      "boost" : 1.0
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "bool": {
+      "filter": {
+        "term": {
+          "status": "active"
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "bool": {
+      "must": {
+        "match_all": {}
+      },
+      "filter": {
+        "term": {
+          "status": "active"
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "constant_score": {
+      "filter": {
+        "term": {
+          "status": "active"
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "span_containing" : {
+            "little" : {
+                "span_term" : { "field1" : "foo" }
+            },
+            "big" : {
+                "span_near" : {
+                    "clauses" : [
+                        { "span_term" : { "field1" : "bar" } },
+                        { "span_term" : { "field1" : "baz" } }
+                    ],
+                    "slop" : 5,
+                    "in_order" : true
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "boosting" : {
+            "positive" : {
+                "term" : {
+                    "field1" : "value1"
+                }
+            },
+            "negative" : {
+                 "term" : {
+                     "field2" : "value2"
+                }
+            },
+            "negative_boost" : 0.2
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "has_child" : {
+            "type" : "blog_tag",
+                "query" : {
+                    "term" : {
+                        "tag" : "something"
+                    }
+                }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "has_child" : {
+            "type" : "blog_tag",
+                "score_mode" : "min",
+                "query" : {
+                    "term" : {
+                        "tag" : "something"
+                    }
+                }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "has_child" : {
+            "type" : "blog_tag",
+            "score_mode" : "min",
+            "min_children": 2,
+            "max_children": 10,
+            "query" : {
+                "term" : {
+                    "tag" : "something"
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "constant_score" : {
+            "filter" : {
+                "terms" : { "user" : ["kimchy", "elasticsearch"]}
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "followers" : ["1", "3"]
+}
+emit snippet
+body part: 
+{
+    "user" : "1"
+}
+emit snippet
+body part: 
+{
+    "query" : {
+        "terms" : {
+            "user" : {
+                "index" : "users",
+                "type" : "user",
+                "id" : "2",
+                "path" : "followers"
+            }
+        }
+    }
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "exists" : { "field" : "user" }
+    }
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool": {
+            "must_not": {
+                "exists": {
+                    "field": "user"
+                }
+            }
+        }
+    }
+}
+handle snippet
+null
+emit snippet
+body part: 
+{
+    "mappings" : {
+        "location": {
+            "properties": {
+                "pin": {
+                    "type": "geo_point",
+                    "geohash": true,
+                    "geohash_prefix": true,
+                    "geohash_precision": 10
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geohash_cell": {
+                    "pin": {
+                        "lat": 13.4080,
+                        "lon": 52.5186
+                    },
+                    "precision": 3,
+                    "neighbors": true
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "regexp":{
+            "name.first": "s.*y"
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "regexp":{
+            "name.first":{
+                "value":"s.*y",
+                "boost":1.2
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "regexp":{
+            "name.first": {
+                "value": "s.*y",
+                "flags" : "INTERSECTION|COMPLEMENT|EMPTY"
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "regexp":{
+            "name.first": {
+                "value": "s.*y",
+                "flags" : "INTERSECTION|COMPLEMENT|EMPTY",
+                "max_determinized_states": 20000
+            }
+        }
+    }
+}
+handle snippet
+null
+emit snippet
+body part: 
+{
+    "mappings": {
+        "blog_post": {
+            "properties": {
+                "name": {
+                    "type": "keyword"
+                }
+            }
+        },
+        "blog_tag": {
+            "_parent": {
+                "type": "blog_post"
+            },
+            "_routing": {
+                "required": true
+            }
+        }
+    }
+}
+emit snippet
+query part: wait_for_status=yellow
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "parent_id" : {
+            "type" : "blog_tag",
+                "id" : "1"
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "has_parent": {
+      "type": "blog_post",
+        "query": {
+          "term": {
+            "_id": "1"
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "has_parent" : {
+            "parent_type" : "blog",
+                "query" : {
+                    "term" : {
+                        "tag" : "something"
+                    }
+                }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "has_parent" : {
+            "parent_type" : "blog",
+                "score" : true,
+                "query" : {
+                    "term" : {
+                        "tag" : "something"
+                    }
+                }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "template": {
+            "inline": { "match": { "text": "{{query_string}}" }},
+            "params" : {
+                "query_string" : "all about search"
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "match": {
+            "text": "all about search"
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "template": {
+            "inline": "{ \"match\": { \"text\": \"{{query_string}}\" }}",
+            "params" : {
+                "query_string" : "all about search"
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "template": {
+            "file": "my_template",
+            "params" : {
+                "query_string" : "all about search"
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "template": { "match": { "text": "{{query_string}}" }}
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "template": {
+            "id": "my_template",
+            "params" : {
+                "query_string" : "all about search"
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "span_first" : {
+            "match" : {
+                "span_term" : { "user" : "kimchy" }
+            },
+            "end" : 3
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_polygon" : {
+                    "person.location" : {
+                        "points" : [
+                        {"lat" : 40, "lon" : -70},
+                        {"lat" : 30, "lon" : -80},
+                        {"lat" : 20, "lon" : -90}
+                        ]
+                    }
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_polygon" : {
+                    "person.location" : {
+                        "points" : [
+                            [-70, 40],
+                        [-80, 30],
+                            [-90, 20]
+                        ]
+                    }
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+               "geo_polygon" : {
+                    "person.location" : {
+                        "points" : [
+                            "40, -70",
+                            "30, -80",
+                            "20, -90"
+                        ]
+                    }
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+               "geo_polygon" : {
+                    "person.location" : {
+                        "points" : [
+                            "drn5x1g8cu2y",
+                            "30, -80",
+                            "20, -90"
+                        ]
+                    }
+                }
+            }
+        }
+    }
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query":{
+        "bool": {
+            "must": {
+                "match_all": {}
+            },
+            "filter": {
+                "geo_shape": {
+                    "location": {
+                        "shape": {
+                            "type": "envelope",
+                            "coordinates" : [[13.0, 53.0], [14.0, 52.0]]
+                        },
+                        "relation": "within"
+                    }
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool": {
+            "must": {
+                "match_all": {}
+            },
+                "filter": {
+                    "geo_shape": {
+                        "location": {
+                            "indexed_shape": {
+                                "id": "DEU",
+                                "type": "countries",
+                                "index": "shapes",
+                                "path": "location"
+                            }
+                        }
+                    }
+                }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "function_score": {
+            "query": {},
+            "boost": "5",
+            "random_score": {},
+            "boost_mode":"multiply"
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "function_score": {
+          "query": {},
+          "boost": "5",
+          "functions": [
+              {
+                  "filter": {},
+                  "random_score": {},
+                  "weight": 23
+              },
+              {
+                  "filter": {},
+                  "weight": 42
+              }
+          ],
+          "max_boost": 42,
+          "score_mode": "max",
+          "boost_mode": "multiply",
+          "min_score" : 42
+        }
+    }
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "function_score": {
+          "functions": [
+            {
+              "gauss": {
+                "price": {
+                  "origin": "0",
+                  "scale": "20"
+                }
+              }
+            },
+            {
+              "gauss": {
+                "location": {
+                  "origin": "11, 12",
+                  "scale": "2km"
+                }
+              }
+            }
+          ],
+          "query": {
+            "match": {
+              "properties": "balcony"
+            }
+          },
+          "score_mode": "multiply"
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "span_or" : {
+            "clauses" : [
+                { "span_term" : { "field" : "value1" } },
+                { "span_term" : { "field" : "value2" } },
+                { "span_term" : { "field" : "value3" } }
+            ]
+        }
+    }
+}
+handle snippet
+null
+emit snippet
+body part: 
+{
+    "mappings": {
+        "location": {
+            "properties": {
+                "pin": {
+                    "properties": {
+                        "location": {
+                            "type": "geo_point"
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+emit snippet
+body part: 
+{
+    "pin" : {
+        "location" : {
+            "lat" : 40.12,
+            "lon" : -71.34
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_bounding_box" : {
+                    "pin.location" : {
+                        "top_left" : {
+                            "lat" : 40.73,
+                            "lon" : -74.1
+                        },
+                        "bottom_right" : {
+                            "lat" : 40.01,
+                            "lon" : -71.12
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_bounding_box" : {
+                    "pin.location" : {
+                        "top_left" : {
+                            "lat" : 40.73,
+                            "lon" : -74.1
+                        },
+                        "bottom_right" : {
+                            "lat" : 40.01,
+                            "lon" : -71.12
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_bounding_box" : {
+                    "pin.location" : {
+                        "top_left" : [-74.1, 40.73],
+                        "bottom_right" : [-71.12, 40.01]
+                    }
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_bounding_box" : {
+                    "pin.location" : {
+                        "top_left" : "40.73, -74.1",
+                        "bottom_right" : "40.01, -71.12"
+                    }
+                }
+            }
+    }
+}
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_bounding_box" : {
+                    "pin.location" : {
+                        "top_left" : "dr5r9ydj2y73",
+                        "bottom_right" : "drj7teegpus6"
+                    }
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_bounding_box" : {
+                    "pin.location" : {
+                        "top" : 40.73,
+                        "left" : -74.1,
+                        "bottom" : 40.01,
+                        "right" : -71.12
+                    }
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "bool" : {
+            "must" : {
+                "match_all" : {}
+            },
+            "filter" : {
+                "geo_bounding_box" : {
+                    "pin.location" : {
+                        "top_left" : {
+                            "lat" : 40.73,
+                            "lon" : -74.1
+                        },
+                        "bottom_right" : {
+                            "lat" : 40.10,
+                            "lon" : -71.12
+                        }
+                    },
+                    "type" : "indexed"
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "match_phrase_prefix" : {
+            "message" : "quick brown f"
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "match_phrase_prefix" : {
+            "message" : {
+                "query" : "quick brown f",
+                "max_expansions" : 10
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "mappings": {
+        "doctype": {
+            "properties": {
+                "message": {
+                    "type": "string"
+                }
+            }
+        },
+        "queries": {
+            "properties": {
+                "query": {
+                    "type": "percolator"
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query" : {
+        "match" : {
+            "message" : "bonsai tree"
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query" : {
+        "percolate" : {
+            "field" : "query",
+            "document_type" : "doctype",
+            "document" : {
+                "message" : "A new bonsai tree in the office"
+            }
+        }
+    }
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "message" : "A new bonsai tree in the office"
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query" : {
+        "percolate" : {
+            "field": "query",
+            "document_type" : "doctype",
+            "index" : "my-index",
+            "type" : "message",
+            "id" : "1",
+            "version" : 1
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query" : {
+        "match" : {
+            "message" : "brown fox"
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query" : {
+        "match" : {
+            "message" : "lazy dog"
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query" : {
+        "percolate" : {
+            "field": "query",
+            "document_type" : "doctype",
+            "document" : {
+                "message" : "The quick brown fox jumps over the lazy dog"
+            }
+        }
+    },
+    "highlight": {
+      "fields": {
+        "message": {}
+      }
+    }
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "term" : {
+      "query.unknown_query" : ""
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "span_not" : {
+            "include" : {
+                "span_term" : { "field1" : "hoya" }
+            },
+            "exclude" : {
+                "span_near" : {
+                    "clauses" : [
+                        { "span_term" : { "field1" : "la" } },
+                        { "span_term" : { "field1" : "hoya" } }
+                    ],
+                    "slop" : 0,
+                    "in_order" : true
+                }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "type" : {
+            "value" : "my_type"
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "span_term" : { "user" : "kimchy" }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+       "span_term" : { "user" : { "value" : "kimchy", "boost" : 2.0 } }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "span_term" : { "user" : { "term" : "kimchy", "boost" : 2.0 } }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+       "fuzzy" : { "user" : "ki" }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "fuzzy" : {
+            "user" : {
+                "value" :         "ki",
+                    "boost" :         1.0,
+                    "fuzziness" :     2,
+                    "prefix_length" : 0,
+                    "max_expansions": 100
+            }
+        }
+    }
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "description" : "describe pipeline",
+  "processors" : [
+    {
+      "set" : {
+        "field": "foo",
+        "value": "bar"
+      }
+    }
+    // other processors
+  ]
+}
+handle snippet
+test snippet
+emit snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "pipeline" :
+  {
+    "description": "_description",
+    "processors": [
+      {
+        "set" : {
+          "field" : "field2",
+          "value" : "_value"
+        }
+      }
+    ]
+  },
+  "docs": [
+    {
+      "_index": "index",
+      "_type": "type",
+      "_id": "id",
+      "_source": {
+        "foo": "bar"
+      }
+    },
+    {
+      "_index": "index",
+      "_type": "type",
+      "_id": "id",
+      "_source": {
+        "foo": "rab"
+      }
+    }
+  ]
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+query part: verbose
+body part: 
+{
+  "pipeline" :
+  {
+    "description": "_description",
+    "processors": [
+      {
+        "set" : {
+          "field" : "field2",
+          "value" : "_value2"
+        }
+      },
+      {
+        "set" : {
+          "field" : "field3",
+          "value" : "_value3"
+        }
+      }
+    ]
+  },
+  "docs": [
+    {
+      "_index": "index",
+      "_type": "type",
+      "_id": "id",
+      "_source": {
+        "foo": "bar"
+      }
+    },
+    {
+      "_index": "index",
+      "_type": "type",
+      "_id": "id",
+      "_source": {
+        "foo": "rab"
+      }
+    }
+  ]
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+emit snippet
+query part: nodes=nodeId1,nodeId2
+emit snippet
+query part: nodes=nodeId1,nodeId2&actions=cluster:*
+handle snippet
+handle snippet
+test snippet
+emit snippet
+emit snippet
+query part: parent_task_id=parentTaskId:1
+handle snippet
+test snippet
+emit snippet
+query part: wait_for_completion=true&timeout=10s
+handle snippet
+test snippet
+emit snippet
+handle snippet
+test snippet
+emit snippet
+handle snippet
+test snippet
+emit snippet
+query part: node_id=nodeId1,nodeId2&actions=*reindex
+handle snippet
+test snippet
+emit snippet
+query part: group_by=parents
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "transient" : {
+    "cluster.routing.allocation.exclude._ip" : "10.0.0.1"
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "transient": {
+    "cluster.routing.allocation.include._ip": "192.168.2.*"
+  }
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "transient": {
+    "cluster.routing.allocation.disk.watermark.low": "80%",
+    "cluster.routing.allocation.disk.watermark.high": "50gb",
+    "cluster.info.update.interval": "1m"
+  }
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+null
+emit snippet
+query part: refresh
+body part: 
+{"index":{"_id":1}}
+{"first":"johnny","last":"gaudreau","goals":[9,27,1],"assists":[17,46,0],"gp":[26,82,1]}
+{"index":{"_id":2}}
+{"first":"sean","last":"monohan","goals":[7,54,26],"assists":[11,26,13],"gp":[26,82,82]}
+{"index":{"_id":3}}
+{"first":"jiri","last":"hudler","goals":[5,34,36],"assists":[11,62,42],"gp":[24,80,79]}
+{"index":{"_id":4}}
+{"first":"micheal","last":"frolik","goals":[4,6,15],"assists":[8,23,15],"gp":[26,82,82]}
+{"index":{"_id":5}}
+{"first":"sam","last":"bennett","goals":[5,0,0],"assists":[8,1,0],"gp":[26,1,0]}
+{"index":{"_id":6}}
+{"first":"dennis","last":"wideman","goals":[0,26,15],"assists":[11,30,24],"gp":[26,81,82]}
+{"index":{"_id":7}}
+{"first":"david","last":"jones","goals":[7,19,5],"assists":[3,17,4],"gp":[26,45,34]}
+{"index":{"_id":8}}
+{"first":"tj","last":"brodie","goals":[2,14,7],"assists":[8,42,30],"gp":[26,82,82]}
+{"index":{"_id":39}}
+{"first":"mark","last":"giordano","goals":[6,30,15],"assists":[3,30,24],"gp":[26,60,63]}
+{"index":{"_id":10}}
+{"first":"mikael","last":"backlund","goals":[3,15,13],"assists":[6,24,18],"gp":[26,82,82]}
+{"index":{"_id":11}}
+{"first":"joe","last":"colborne","goals":[3,18,13],"assists":[6,20,24],"gp":[26,67,82]}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "function_score": {
+      "script_score": {
+        "script": {
+          "lang": "painless",
+          "inline": "int total = 0; for (int i = 0; i < doc['goals'].length; ++i) { total += doc['goals'][i]; } return total;"
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "match_all": {}
+  },
+  "script_fields": {
+    "total_goals": {
+      "script": {
+        "lang": "painless",
+        "inline": "int total = 0; for (int i = 0; i < doc['goals'].length; ++i) { total += doc['goals'][i]; } return total;"
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "match_all": {}
+  },
+  "sort": {
+    "_script": {
+      "type": "string",
+      "order": "asc",
+      "script": {
+        "lang": "painless",
+        "inline": "doc['first'].value + ' ' + doc['last'].value"
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "fields": [
+    "_id",
+    "_source"
+  ],
+  "query": {
+    "term": {
+      "_id": 1
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "script": {
+    "lang": "painless",
+    "inline": "ctx._source.last = params.last",
+    "params": {
+      "last": "hockey"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "script": {
+    "lang": "painless",
+    "inline": "ctx._source.last = params.last; ctx._source.nick = params.nick",
+    "params": {
+      "last": "gaudreau",
+      "nick": "hockey"
+    }
+  }
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "text": "quick brown fox",
+  "popularity": 1
+}
+emit snippet
+body part: 
+{
+  "text": "quick fox",
+  "popularity": 5
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "function_score": {
+      "query": {
+        "match": {
+          "text": "quick brown fox"
+        }
+      },
+      "script_score": {
+        "script": {
+          "lang": "expression",
+          "inline": "_score * doc['popularity']"
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "cost_price": 100
+}
+emit snippet
+body part: 
+{
+  "script_fields": {
+    "sales_price": {
+      "script": {
+        "lang":   "expression",
+        "inline": "doc['cost_price'] * markup",
+        "params": {
+          "markup": 0.2
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "title": {
+          "type": "text"
+        },
+        "first_name": {
+          "type": "text",
+          "store": true
+        },
+        "last_name": {
+          "type": "text",
+          "store": true
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "title": "Mr",
+  "first_name": "Barry",
+  "last_name": "White"
+}
+emit snippet
+body part: 
+{
+  "script_fields": {
+    "source": {
+      "script": {
+        "lang": "groovy",
+        "inline": "_source.title + ' ' + _source.first_name + ' ' + _source.last_name"
+      }
+    },
+    "stored_fields": {
+      "script": {
+        "lang": "groovy",
+        "inline": "_fields['first_name'].value + ' ' + _fields['last_name'].value"
+      }
+    }
+  }
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "my_field": 5
+}
+emit snippet
+body part: 
+{
+  "script_fields": {
+    "my_doubled_field": {
+      "script": {
+        "lang":   "expression",
+        "inline": "doc['my_field'] * multiplier",
+        "params": {
+          "multiplier": 2
+        }
+      }
+    }
+  }
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "script": "log(_score * 2) + my_modifier"
+}
+handle snippet
+test snippet
+emit snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "script": {
+      "script": {
+        "lang": "groovy",
+        "id":   "calculate-score",
+        "params": {
+          "my_modifier": 2
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+catch part: /cannot set discovery.zen.minimum_master_nodes to more than the current master nodes/
+body part: 
+{
+  "transient": {
+    "discovery.zen.minimum_master_nodes": 2
+  }
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+          {"index":{"_id": "5"}}
+          {"user": "test", "message": "some message with the number 5", "date": "2009-11-15T14:12:12", "likes": 5}
+          {"index":{"_id": "6"}}
+          {"user": "test", "message": "some message with the number 6", "date": "2009-11-15T14:12:12", "likes": 6}
+          {"index":{"_id": "7"}}
+          {"user": "test", "message": "some message with the number 7", "date": "2009-11-15T14:12:12", "likes": 7}
+          {"index":{"_id": "8"}}
+          {"user": "test", "message": "some message with the number 8", "date": "2009-11-15T14:12:12", "likes": 8}
+          {"index":{"_id": "9"}}
+          {"user": "test", "message": "some message with the number 9", "date": "2009-11-15T14:12:12", "likes": 9}
+          {"index":{"_id": "10"}}
+          {"user": "test", "message": "some message with the number 10", "date": "2009-11-15T14:12:12", "likes": 10}
+          {"index":{"_id": "11"}}
+          {"user": "test", "message": "some message with the number 11", "date": "2009-11-15T14:12:12", "likes": 11}
+          {"index":{"_id": "12"}}
+          {"user": "test", "message": "some message with the number 12", "date": "2009-11-15T14:12:12", "likes": 12}
+          {"index":{"_id": "13"}}
+          {"user": "test", "message": "some message with the number 13", "date": "2009-11-15T14:12:12", "likes": 13}
+          {"index":{"_id": "14"}}
+          {"user": "test", "message": "some message with the number 14", "date": "2009-11-15T14:12:12", "likes": 14}
+          {"index":{"_id": "15"}}
+          {"user": "test", "message": "some message with the number 15", "date": "2009-11-15T14:12:12", "likes": 15}
+          {"index":{"_id": "16"}}
+          {"user": "test", "message": "some message with the number 16", "date": "2009-11-15T14:12:12", "likes": 16}
+          {"index":{"_id": "17"}}
+          {"user": "test", "message": "some message with the number 17", "date": "2009-11-15T14:12:12", "likes": 17}
+          {"index":{"_id": "18"}}
+          {"user": "test", "message": "some message with the number 18", "date": "2009-11-15T14:12:12", "likes": 18}
+          {"index":{"_id": "19"}}
+          {"user": "test", "message": "some message with the number 19", "date": "2009-11-15T14:12:12", "likes": 19}
+          {"index":{"_id": "20"}}
+          {"user": "test", "message": "some message with the number 20", "date": "2009-11-15T14:12:12", "likes": 20}
+          {"index":{"_id": "21"}}
+          {"user": "test", "message": "some message with the number 21", "date": "2009-11-15T14:12:12", "likes": 21}
+          {"index":{"_id": "22"}}
+          {"user": "test", "message": "some message with the number 22", "date": "2009-11-15T14:12:12", "likes": 22}
+          {"index":{"_id": "23"}}
+          {"user": "test", "message": "some message with the number 23", "date": "2009-11-15T14:12:12", "likes": 23}
+          {"index":{"_id": "24"}}
+          {"user": "test", "message": "some message with the number 24", "date": "2009-11-15T14:12:12", "likes": 24}
+          {"index":{"_id": "25"}}
+          {"user": "test", "message": "some message with the number 25", "date": "2009-11-15T14:12:12", "likes": 25}
+          {"index":{"_id": "26"}}
+          {"user": "test", "message": "some message with the number 26", "date": "2009-11-15T14:12:12", "likes": 26}
+          {"index":{"_id": "27"}}
+          {"user": "test", "message": "some message with the number 27", "date": "2009-11-15T14:12:12", "likes": 27}
+          {"index":{"_id": "28"}}
+          {"user": "test", "message": "some message with the number 28", "date": "2009-11-15T14:12:12", "likes": 28}
+          {"index":{"_id": "29"}}
+          {"user": "test", "message": "some message with the number 29", "date": "2009-11-15T14:12:12", "likes": 29}
+          {"index":{"_id": "30"}}
+          {"user": "test", "message": "some message with the number 30", "date": "2009-11-15T14:12:12", "likes": 30}
+          {"index":{"_id": "31"}}
+          {"user": "test", "message": "some message with the number 31", "date": "2009-11-15T14:12:12", "likes": 31}
+          {"index":{"_id": "32"}}
+          {"user": "test", "message": "some message with the number 32", "date": "2009-11-15T14:12:12", "likes": 32}
+          {"index":{"_id": "33"}}
+          {"user": "test", "message": "some message with the number 33", "date": "2009-11-15T14:12:12", "likes": 33}
+          {"index":{"_id": "34"}}
+          {"user": "test", "message": "some message with the number 34", "date": "2009-11-15T14:12:12", "likes": 34}
+          {"index":{"_id": "35"}}
+          {"user": "test", "message": "some message with the number 35", "date": "2009-11-15T14:12:12", "likes": 35}
+          {"index":{"_id": "36"}}
+          {"user": "test", "message": "some message with the number 36", "date": "2009-11-15T14:12:12", "likes": 36}
+          {"index":{"_id": "37"}}
+          {"user": "test", "message": "some message with the number 37", "date": "2009-11-15T14:12:12", "likes": 37}
+          {"index":{"_id": "38"}}
+          {"user": "test", "message": "some message with the number 38", "date": "2009-11-15T14:12:12", "likes": 38}
+          {"index":{"_id": "39"}}
+          {"user": "test", "message": "some message with the number 39", "date": "2009-11-15T14:12:12", "likes": 39}
+          {"index":{"_id": "40"}}
+          {"user": "test", "message": "some message with the number 40", "date": "2009-11-15T14:12:12", "likes": 40}
+          {"index":{"_id": "41"}}
+          {"user": "test", "message": "some message with the number 41", "date": "2009-11-15T14:12:12", "likes": 41}
+          {"index":{"_id": "42"}}
+          {"user": "test", "message": "some message with the number 42", "date": "2009-11-15T14:12:12", "likes": 42}
+          {"index":{"_id": "43"}}
+          {"user": "test", "message": "some message with the number 43", "date": "2009-11-15T14:12:12", "likes": 43}
+          {"index":{"_id": "44"}}
+          {"user": "test", "message": "some message with the number 44", "date": "2009-11-15T14:12:12", "likes": 44}
+          {"index":{"_id": "45"}}
+          {"user": "test", "message": "some message with the number 45", "date": "2009-11-15T14:12:12", "likes": 45}
+          {"index":{"_id": "46"}}
+          {"user": "test", "message": "some message with the number 46", "date": "2009-11-15T14:12:12", "likes": 46}
+          {"index":{"_id": "47"}}
+          {"user": "test", "message": "some message with the number 47", "date": "2009-11-15T14:12:12", "likes": 47}
+          {"index":{"_id": "48"}}
+          {"user": "test", "message": "some message with the number 48", "date": "2009-11-15T14:12:12", "likes": 48}
+          {"index":{"_id": "49"}}
+          {"user": "test", "message": "some message with the number 49", "date": "2009-11-15T14:12:12", "likes": 49}
+          {"index":{"_id": "50"}}
+          {"user": "test", "message": "some message with the number 50", "date": "2009-11-15T14:12:12", "likes": 50}
+          {"index":{"_id": "51"}}
+          {"user": "test", "message": "some message with the number 51", "date": "2009-11-15T14:12:12", "likes": 51}
+          {"index":{"_id": "52"}}
+          {"user": "test", "message": "some message with the number 52", "date": "2009-11-15T14:12:12", "likes": 52}
+          {"index":{"_id": "53"}}
+          {"user": "test", "message": "some message with the number 53", "date": "2009-11-15T14:12:12", "likes": 53}
+          {"index":{"_id": "54"}}
+          {"user": "test", "message": "some message with the number 54", "date": "2009-11-15T14:12:12", "likes": 54}
+          {"index":{"_id": "55"}}
+          {"user": "test", "message": "some message with the number 55", "date": "2009-11-15T14:12:12", "likes": 55}
+          {"index":{"_id": "56"}}
+          {"user": "test", "message": "some message with the number 56", "date": "2009-11-15T14:12:12", "likes": 56}
+          {"index":{"_id": "57"}}
+          {"user": "test", "message": "some message with the number 57", "date": "2009-11-15T14:12:12", "likes": 57}
+          {"index":{"_id": "58"}}
+          {"user": "test", "message": "some message with the number 58", "date": "2009-11-15T14:12:12", "likes": 58}
+          {"index":{"_id": "59"}}
+          {"user": "test", "message": "some message with the number 59", "date": "2009-11-15T14:12:12", "likes": 59}
+          {"index":{"_id": "60"}}
+          {"user": "test", "message": "some message with the number 60", "date": "2009-11-15T14:12:12", "likes": 60}
+          {"index":{"_id": "61"}}
+          {"user": "test", "message": "some message with the number 61", "date": "2009-11-15T14:12:12", "likes": 61}
+          {"index":{"_id": "62"}}
+          {"user": "test", "message": "some message with the number 62", "date": "2009-11-15T14:12:12", "likes": 62}
+          {"index":{"_id": "63"}}
+          {"user": "test", "message": "some message with the number 63", "date": "2009-11-15T14:12:12", "likes": 63}
+          {"index":{"_id": "64"}}
+          {"user": "test", "message": "some message with the number 64", "date": "2009-11-15T14:12:12", "likes": 64}
+          {"index":{"_id": "65"}}
+          {"user": "test", "message": "some message with the number 65", "date": "2009-11-15T14:12:12", "likes": 65}
+          {"index":{"_id": "66"}}
+          {"user": "test", "message": "some message with the number 66", "date": "2009-11-15T14:12:12", "likes": 66}
+          {"index":{"_id": "67"}}
+          {"user": "test", "message": "some message with the number 67", "date": "2009-11-15T14:12:12", "likes": 67}
+          {"index":{"_id": "68"}}
+          {"user": "test", "message": "some message with the number 68", "date": "2009-11-15T14:12:12", "likes": 68}
+          {"index":{"_id": "69"}}
+          {"user": "test", "message": "some message with the number 69", "date": "2009-11-15T14:12:12", "likes": 69}
+          {"index":{"_id": "70"}}
+          {"user": "test", "message": "some message with the number 70", "date": "2009-11-15T14:12:12", "likes": 70}
+          {"index":{"_id": "71"}}
+          {"user": "test", "message": "some message with the number 71", "date": "2009-11-15T14:12:12", "likes": 71}
+          {"index":{"_id": "72"}}
+          {"user": "test", "message": "some message with the number 72", "date": "2009-11-15T14:12:12", "likes": 72}
+          {"index":{"_id": "73"}}
+          {"user": "test", "message": "some message with the number 73", "date": "2009-11-15T14:12:12", "likes": 73}
+          {"index":{"_id": "74"}}
+          {"user": "test", "message": "some message with the number 74", "date": "2009-11-15T14:12:12", "likes": 74}
+          {"index":{"_id": "75"}}
+          {"user": "test", "message": "some message with the number 75", "date": "2009-11-15T14:12:12", "likes": 75}
+          {"index":{"_id": "76"}}
+          {"user": "test", "message": "some message with the number 76", "date": "2009-11-15T14:12:12", "likes": 76}
+          {"index":{"_id": "77"}}
+          {"user": "test", "message": "some message with the number 77", "date": "2009-11-15T14:12:12", "likes": 77}
+          {"index":{"_id": "78"}}
+          {"user": "test", "message": "some message with the number 78", "date": "2009-11-15T14:12:12", "likes": 78}
+          {"index":{"_id": "79"}}
+          {"user": "test", "message": "some message with the number 79", "date": "2009-11-15T14:12:12", "likes": 79}
+          {"index":{"_id": "80"}}
+          {"user": "test", "message": "some message with the number 80", "date": "2009-11-15T14:12:12", "likes": 80}
+          {"index":{"_id": "81"}}
+          {"user": "test", "message": "some message with the number 81", "date": "2009-11-15T14:12:12", "likes": 81}
+          {"index":{"_id": "82"}}
+          {"user": "test", "message": "some message with the number 82", "date": "2009-11-15T14:12:12", "likes": 82}
+          {"index":{"_id": "83"}}
+          {"user": "test", "message": "some message with the number 83", "date": "2009-11-15T14:12:12", "likes": 83}
+          {"index":{"_id": "84"}}
+          {"user": "test", "message": "some message with the number 84", "date": "2009-11-15T14:12:12", "likes": 84}
+          {"index":{"_id": "85"}}
+          {"user": "test", "message": "some message with the number 85", "date": "2009-11-15T14:12:12", "likes": 85}
+          {"index":{"_id": "86"}}
+          {"user": "test", "message": "some message with the number 86", "date": "2009-11-15T14:12:12", "likes": 86}
+          {"index":{"_id": "87"}}
+          {"user": "test", "message": "some message with the number 87", "date": "2009-11-15T14:12:12", "likes": 87}
+          {"index":{"_id": "88"}}
+          {"user": "test", "message": "some message with the number 88", "date": "2009-11-15T14:12:12", "likes": 88}
+          {"index":{"_id": "89"}}
+          {"user": "test", "message": "some message with the number 89", "date": "2009-11-15T14:12:12", "likes": 89}
+          {"index":{"_id": "90"}}
+          {"user": "test", "message": "some message with the number 90", "date": "2009-11-15T14:12:12", "likes": 90}
+          {"index":{"_id": "91"}}
+          {"user": "test", "message": "some message with the number 91", "date": "2009-11-15T14:12:12", "likes": 91}
+          {"index":{"_id": "92"}}
+          {"user": "test", "message": "some message with the number 92", "date": "2009-11-15T14:12:12", "likes": 92}
+          {"index":{"_id": "93"}}
+          {"user": "test", "message": "some message with the number 93", "date": "2009-11-15T14:12:12", "likes": 93}
+          {"index":{"_id": "94"}}
+          {"user": "test", "message": "some message with the number 94", "date": "2009-11-15T14:12:12", "likes": 94}
+          {"index":{"_id": "95"}}
+          {"user": "test", "message": "some message with the number 95", "date": "2009-11-15T14:12:12", "likes": 95}
+          {"index":{"_id": "96"}}
+          {"user": "test", "message": "some message with the number 96", "date": "2009-11-15T14:12:12", "likes": 96}
+          {"index":{"_id": "97"}}
+          {"user": "test", "message": "some message with the number 97", "date": "2009-11-15T14:12:12", "likes": 97}
+          {"index":{"_id": "98"}}
+          {"user": "test", "message": "some message with the number 98", "date": "2009-11-15T14:12:12", "likes": 98}
+          {"index":{"_id": "99"}}
+          {"user": "test", "message": "some message with the number 99", "date": "2009-11-15T14:12:12", "likes": 99}
+          {"index":{"_id": "100"}}
+          {"user": "test", "message": "some message with the number 100", "date": "2009-11-15T14:12:12", "likes": 100}
+          {"index":{"_id": "101"}}
+          {"user": "test", "message": "some message with the number 101", "date": "2009-11-15T14:12:12", "likes": 101}
+          {"index":{"_id": "102"}}
+          {"user": "test", "message": "some message with the number 102", "date": "2009-11-15T14:12:12", "likes": 102}
+          {"index":{"_id": "103"}}
+          {"user": "test", "message": "some message with the number 103", "date": "2009-11-15T14:12:12", "likes": 103}
+          {"index":{"_id": "104"}}
+          {"user": "test", "message": "some message with the number 104", "date": "2009-11-15T14:12:12", "likes": 104}
+          {"index":{"_id": "105"}}
+          {"user": "test", "message": "some message with the number 105", "date": "2009-11-15T14:12:12", "likes": 105}
+          {"index":{"_id": "106"}}
+          {"user": "test", "message": "some message with the number 106", "date": "2009-11-15T14:12:12", "likes": 106}
+          {"index":{"_id": "107"}}
+          {"user": "test", "message": "some message with the number 107", "date": "2009-11-15T14:12:12", "likes": 107}
+          {"index":{"_id": "108"}}
+          {"user": "test", "message": "some message with the number 108", "date": "2009-11-15T14:12:12", "likes": 108}
+          {"index":{"_id": "109"}}
+          {"user": "test", "message": "some message with the number 109", "date": "2009-11-15T14:12:12", "likes": 109}
+          {"index":{"_id": "110"}}
+          {"user": "test", "message": "some message with the number 110", "date": "2009-11-15T14:12:12", "likes": 110}
+          {"index":{"_id": "111"}}
+          {"user": "test", "message": "some message with the number 111", "date": "2009-11-15T14:12:12", "likes": 111}
+          {"index":{"_id": "112"}}
+          {"user": "test", "message": "some message with the number 112", "date": "2009-11-15T14:12:12", "likes": 112}
+          {"index":{"_id": "113"}}
+          {"user": "test", "message": "some message with the number 113", "date": "2009-11-15T14:12:12", "likes": 113}
+          {"index":{"_id": "114"}}
+          {"user": "test", "message": "some message with the number 114", "date": "2009-11-15T14:12:12", "likes": 114}
+          {"index":{"_id": "115"}}
+          {"user": "test", "message": "some message with the number 115", "date": "2009-11-15T14:12:12", "likes": 115}
+          {"index":{"_id": "116"}}
+          {"user": "test", "message": "some message with the number 116", "date": "2009-11-15T14:12:12", "likes": 116}
+          {"index":{"_id": "117"}}
+          {"user": "test", "message": "some message with the number 117", "date": "2009-11-15T14:12:12", "likes": 117}
+          {"index":{"_id": "118"}}
+          {"user": "test", "message": "some message with the number 118", "date": "2009-11-15T14:12:12", "likes": 118}
+          {"index":{"_id": "119"}}
+          {"user": "test", "message": "some message with the number 119", "date": "2009-11-15T14:12:12", "likes": 119}
+emit snippet
+body part: 
+{
+  "query": {
+    "match": {
+      "message": "some message"
+    }
+  }
+}
+handle snippet
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+emit snippet
+query part: conflicts=proceed
+body part: 
+{
+  "query": {
+    "match_all": {}
+  }
+}
+handle snippet
+test snippet
+emit snippet
+emit snippet
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "query": {
+    "match_all": {}
+  }
+}
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+emit snippet
+query part: routing=1
+body part: 
+{
+  "query": {
+    "range" : {
+        "age" : {
+           "gte" : 10
+        }
+    }
+  }
+}
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+emit snippet
+query part: scroll_size=5000
+body part: 
+{
+  "query": {
+    "term": {
+      "user": "kimchy"
+    }
+  }
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+query part: detailed=true&action=*/delete/byquery
+handle snippet
+handle snippet
+test snippet
+emit snippet
+handle snippet
+test snippet
+emit snippet
+query part: requests_per_second=unlimited
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "user" : "kimchy",
+    "post_date" : "2009-11-15T14:12:12",
+    "message" : "trying out Elasticsearch"
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+catch part: conflict
+query part: version=2
+body part: 
+{
+    "message" : "elasticsearch now has versioning support, double cool!"
+}
+handle snippet
+test snippet
+emit snippet
+query part: op_type=create
+body part: 
+{
+    "user" : "kimchy",
+    "post_date" : "2009-11-15T14:12:12",
+    "message" : "trying out Elasticsearch"
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "user" : "kimchy",
+    "post_date" : "2009-11-15T14:12:12",
+    "message" : "trying out Elasticsearch"
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "user" : "kimchy",
+    "post_date" : "2009-11-15T14:12:12",
+    "message" : "trying out Elasticsearch"
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+query part: routing=kimchy
+body part: 
+{
+    "user" : "kimchy",
+    "post_date" : "2009-11-15T14:12:12",
+    "message" : "trying out Elasticsearch"
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "tag_parent": {},
+    "blog_tag": {
+      "_parent": {
+        "type": "tag_parent"
+      }
+    }
+  }
+}
+emit snippet
+query part: parent=1111
+body part: 
+{
+    "tag" : "something"
+}
+handle snippet
+test snippet
+emit snippet
+query part: timestamp=2009-11-15T14:12:12
+body part: 
+{
+    "user" : "kimchy",
+    "message" : "trying out Elasticsearch"
+}
+handle snippet
+test snippet
+emit snippet
+query part: ttl=86400000ms
+body part: 
+{
+    "user": "kimchy",
+    "message": "Trying out elasticsearch, so far so good?"
+}
+handle snippet
+test snippet
+emit snippet
+query part: ttl=1d
+body part: 
+{
+    "user": "kimchy",
+    "message": "Trying out elasticsearch, so far so good?"
+}
+handle snippet
+test snippet
+emit snippet
+query part: timeout=5m
+body part: 
+{
+    "user" : "kimchy",
+    "post_date" : "2009-11-15T14:12:12",
+    "message" : "trying out Elasticsearch"
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+          {"index":{"_id": "5"}}
+          {"user": "test", "message": "some message with the number 5", "date": "2009-11-15T14:12:12", "likes": 5}
+          {"index":{"_id": "6"}}
+          {"user": "test", "message": "some message with the number 6", "date": "2009-11-15T14:12:12", "likes": 6}
+          {"index":{"_id": "7"}}
+          {"user": "test", "message": "some message with the number 7", "date": "2009-11-15T14:12:12", "likes": 7}
+          {"index":{"_id": "8"}}
+          {"user": "test", "message": "some message with the number 8", "date": "2009-11-15T14:12:12", "likes": 8}
+          {"index":{"_id": "9"}}
+          {"user": "test", "message": "some message with the number 9", "date": "2009-11-15T14:12:12", "likes": 9}
+          {"index":{"_id": "10"}}
+          {"user": "test", "message": "some message with the number 10", "date": "2009-11-15T14:12:12", "likes": 10}
+          {"index":{"_id": "11"}}
+          {"user": "test", "message": "some message with the number 11", "date": "2009-11-15T14:12:12", "likes": 11}
+          {"index":{"_id": "12"}}
+          {"user": "test", "message": "some message with the number 12", "date": "2009-11-15T14:12:12", "likes": 12}
+          {"index":{"_id": "13"}}
+          {"user": "test", "message": "some message with the number 13", "date": "2009-11-15T14:12:12", "likes": 13}
+          {"index":{"_id": "14"}}
+          {"user": "test", "message": "some message with the number 14", "date": "2009-11-15T14:12:12", "likes": 14}
+          {"index":{"_id": "15"}}
+          {"user": "test", "message": "some message with the number 15", "date": "2009-11-15T14:12:12", "likes": 15}
+          {"index":{"_id": "16"}}
+          {"user": "test", "message": "some message with the number 16", "date": "2009-11-15T14:12:12", "likes": 16}
+          {"index":{"_id": "17"}}
+          {"user": "test", "message": "some message with the number 17", "date": "2009-11-15T14:12:12", "likes": 17}
+          {"index":{"_id": "18"}}
+          {"user": "test", "message": "some message with the number 18", "date": "2009-11-15T14:12:12", "likes": 18}
+          {"index":{"_id": "19"}}
+          {"user": "test", "message": "some message with the number 19", "date": "2009-11-15T14:12:12", "likes": 19}
+          {"index":{"_id": "20"}}
+          {"user": "test", "message": "some message with the number 20", "date": "2009-11-15T14:12:12", "likes": 20}
+          {"index":{"_id": "21"}}
+          {"user": "test", "message": "some message with the number 21", "date": "2009-11-15T14:12:12", "likes": 21}
+          {"index":{"_id": "22"}}
+          {"user": "test", "message": "some message with the number 22", "date": "2009-11-15T14:12:12", "likes": 22}
+          {"index":{"_id": "23"}}
+          {"user": "test", "message": "some message with the number 23", "date": "2009-11-15T14:12:12", "likes": 23}
+          {"index":{"_id": "24"}}
+          {"user": "test", "message": "some message with the number 24", "date": "2009-11-15T14:12:12", "likes": 24}
+          {"index":{"_id": "25"}}
+          {"user": "test", "message": "some message with the number 25", "date": "2009-11-15T14:12:12", "likes": 25}
+          {"index":{"_id": "26"}}
+          {"user": "test", "message": "some message with the number 26", "date": "2009-11-15T14:12:12", "likes": 26}
+          {"index":{"_id": "27"}}
+          {"user": "test", "message": "some message with the number 27", "date": "2009-11-15T14:12:12", "likes": 27}
+          {"index":{"_id": "28"}}
+          {"user": "test", "message": "some message with the number 28", "date": "2009-11-15T14:12:12", "likes": 28}
+          {"index":{"_id": "29"}}
+          {"user": "test", "message": "some message with the number 29", "date": "2009-11-15T14:12:12", "likes": 29}
+          {"index":{"_id": "30"}}
+          {"user": "test", "message": "some message with the number 30", "date": "2009-11-15T14:12:12", "likes": 30}
+          {"index":{"_id": "31"}}
+          {"user": "test", "message": "some message with the number 31", "date": "2009-11-15T14:12:12", "likes": 31}
+          {"index":{"_id": "32"}}
+          {"user": "test", "message": "some message with the number 32", "date": "2009-11-15T14:12:12", "likes": 32}
+          {"index":{"_id": "33"}}
+          {"user": "test", "message": "some message with the number 33", "date": "2009-11-15T14:12:12", "likes": 33}
+          {"index":{"_id": "34"}}
+          {"user": "test", "message": "some message with the number 34", "date": "2009-11-15T14:12:12", "likes": 34}
+          {"index":{"_id": "35"}}
+          {"user": "test", "message": "some message with the number 35", "date": "2009-11-15T14:12:12", "likes": 35}
+          {"index":{"_id": "36"}}
+          {"user": "test", "message": "some message with the number 36", "date": "2009-11-15T14:12:12", "likes": 36}
+          {"index":{"_id": "37"}}
+          {"user": "test", "message": "some message with the number 37", "date": "2009-11-15T14:12:12", "likes": 37}
+          {"index":{"_id": "38"}}
+          {"user": "test", "message": "some message with the number 38", "date": "2009-11-15T14:12:12", "likes": 38}
+          {"index":{"_id": "39"}}
+          {"user": "test", "message": "some message with the number 39", "date": "2009-11-15T14:12:12", "likes": 39}
+          {"index":{"_id": "40"}}
+          {"user": "test", "message": "some message with the number 40", "date": "2009-11-15T14:12:12", "likes": 40}
+          {"index":{"_id": "41"}}
+          {"user": "test", "message": "some message with the number 41", "date": "2009-11-15T14:12:12", "likes": 41}
+          {"index":{"_id": "42"}}
+          {"user": "test", "message": "some message with the number 42", "date": "2009-11-15T14:12:12", "likes": 42}
+          {"index":{"_id": "43"}}
+          {"user": "test", "message": "some message with the number 43", "date": "2009-11-15T14:12:12", "likes": 43}
+          {"index":{"_id": "44"}}
+          {"user": "test", "message": "some message with the number 44", "date": "2009-11-15T14:12:12", "likes": 44}
+          {"index":{"_id": "45"}}
+          {"user": "test", "message": "some message with the number 45", "date": "2009-11-15T14:12:12", "likes": 45}
+          {"index":{"_id": "46"}}
+          {"user": "test", "message": "some message with the number 46", "date": "2009-11-15T14:12:12", "likes": 46}
+          {"index":{"_id": "47"}}
+          {"user": "test", "message": "some message with the number 47", "date": "2009-11-15T14:12:12", "likes": 47}
+          {"index":{"_id": "48"}}
+          {"user": "test", "message": "some message with the number 48", "date": "2009-11-15T14:12:12", "likes": 48}
+          {"index":{"_id": "49"}}
+          {"user": "test", "message": "some message with the number 49", "date": "2009-11-15T14:12:12", "likes": 49}
+          {"index":{"_id": "50"}}
+          {"user": "test", "message": "some message with the number 50", "date": "2009-11-15T14:12:12", "likes": 50}
+          {"index":{"_id": "51"}}
+          {"user": "test", "message": "some message with the number 51", "date": "2009-11-15T14:12:12", "likes": 51}
+          {"index":{"_id": "52"}}
+          {"user": "test", "message": "some message with the number 52", "date": "2009-11-15T14:12:12", "likes": 52}
+          {"index":{"_id": "53"}}
+          {"user": "test", "message": "some message with the number 53", "date": "2009-11-15T14:12:12", "likes": 53}
+          {"index":{"_id": "54"}}
+          {"user": "test", "message": "some message with the number 54", "date": "2009-11-15T14:12:12", "likes": 54}
+          {"index":{"_id": "55"}}
+          {"user": "test", "message": "some message with the number 55", "date": "2009-11-15T14:12:12", "likes": 55}
+          {"index":{"_id": "56"}}
+          {"user": "test", "message": "some message with the number 56", "date": "2009-11-15T14:12:12", "likes": 56}
+          {"index":{"_id": "57"}}
+          {"user": "test", "message": "some message with the number 57", "date": "2009-11-15T14:12:12", "likes": 57}
+          {"index":{"_id": "58"}}
+          {"user": "test", "message": "some message with the number 58", "date": "2009-11-15T14:12:12", "likes": 58}
+          {"index":{"_id": "59"}}
+          {"user": "test", "message": "some message with the number 59", "date": "2009-11-15T14:12:12", "likes": 59}
+          {"index":{"_id": "60"}}
+          {"user": "test", "message": "some message with the number 60", "date": "2009-11-15T14:12:12", "likes": 60}
+          {"index":{"_id": "61"}}
+          {"user": "test", "message": "some message with the number 61", "date": "2009-11-15T14:12:12", "likes": 61}
+          {"index":{"_id": "62"}}
+          {"user": "test", "message": "some message with the number 62", "date": "2009-11-15T14:12:12", "likes": 62}
+          {"index":{"_id": "63"}}
+          {"user": "test", "message": "some message with the number 63", "date": "2009-11-15T14:12:12", "likes": 63}
+          {"index":{"_id": "64"}}
+          {"user": "test", "message": "some message with the number 64", "date": "2009-11-15T14:12:12", "likes": 64}
+          {"index":{"_id": "65"}}
+          {"user": "test", "message": "some message with the number 65", "date": "2009-11-15T14:12:12", "likes": 65}
+          {"index":{"_id": "66"}}
+          {"user": "test", "message": "some message with the number 66", "date": "2009-11-15T14:12:12", "likes": 66}
+          {"index":{"_id": "67"}}
+          {"user": "test", "message": "some message with the number 67", "date": "2009-11-15T14:12:12", "likes": 67}
+          {"index":{"_id": "68"}}
+          {"user": "test", "message": "some message with the number 68", "date": "2009-11-15T14:12:12", "likes": 68}
+          {"index":{"_id": "69"}}
+          {"user": "test", "message": "some message with the number 69", "date": "2009-11-15T14:12:12", "likes": 69}
+          {"index":{"_id": "70"}}
+          {"user": "test", "message": "some message with the number 70", "date": "2009-11-15T14:12:12", "likes": 70}
+          {"index":{"_id": "71"}}
+          {"user": "test", "message": "some message with the number 71", "date": "2009-11-15T14:12:12", "likes": 71}
+          {"index":{"_id": "72"}}
+          {"user": "test", "message": "some message with the number 72", "date": "2009-11-15T14:12:12", "likes": 72}
+          {"index":{"_id": "73"}}
+          {"user": "test", "message": "some message with the number 73", "date": "2009-11-15T14:12:12", "likes": 73}
+          {"index":{"_id": "74"}}
+          {"user": "test", "message": "some message with the number 74", "date": "2009-11-15T14:12:12", "likes": 74}
+          {"index":{"_id": "75"}}
+          {"user": "test", "message": "some message with the number 75", "date": "2009-11-15T14:12:12", "likes": 75}
+          {"index":{"_id": "76"}}
+          {"user": "test", "message": "some message with the number 76", "date": "2009-11-15T14:12:12", "likes": 76}
+          {"index":{"_id": "77"}}
+          {"user": "test", "message": "some message with the number 77", "date": "2009-11-15T14:12:12", "likes": 77}
+          {"index":{"_id": "78"}}
+          {"user": "test", "message": "some message with the number 78", "date": "2009-11-15T14:12:12", "likes": 78}
+          {"index":{"_id": "79"}}
+          {"user": "test", "message": "some message with the number 79", "date": "2009-11-15T14:12:12", "likes": 79}
+          {"index":{"_id": "80"}}
+          {"user": "test", "message": "some message with the number 80", "date": "2009-11-15T14:12:12", "likes": 80}
+          {"index":{"_id": "81"}}
+          {"user": "test", "message": "some message with the number 81", "date": "2009-11-15T14:12:12", "likes": 81}
+          {"index":{"_id": "82"}}
+          {"user": "test", "message": "some message with the number 82", "date": "2009-11-15T14:12:12", "likes": 82}
+          {"index":{"_id": "83"}}
+          {"user": "test", "message": "some message with the number 83", "date": "2009-11-15T14:12:12", "likes": 83}
+          {"index":{"_id": "84"}}
+          {"user": "test", "message": "some message with the number 84", "date": "2009-11-15T14:12:12", "likes": 84}
+          {"index":{"_id": "85"}}
+          {"user": "test", "message": "some message with the number 85", "date": "2009-11-15T14:12:12", "likes": 85}
+          {"index":{"_id": "86"}}
+          {"user": "test", "message": "some message with the number 86", "date": "2009-11-15T14:12:12", "likes": 86}
+          {"index":{"_id": "87"}}
+          {"user": "test", "message": "some message with the number 87", "date": "2009-11-15T14:12:12", "likes": 87}
+          {"index":{"_id": "88"}}
+          {"user": "test", "message": "some message with the number 88", "date": "2009-11-15T14:12:12", "likes": 88}
+          {"index":{"_id": "89"}}
+          {"user": "test", "message": "some message with the number 89", "date": "2009-11-15T14:12:12", "likes": 89}
+          {"index":{"_id": "90"}}
+          {"user": "test", "message": "some message with the number 90", "date": "2009-11-15T14:12:12", "likes": 90}
+          {"index":{"_id": "91"}}
+          {"user": "test", "message": "some message with the number 91", "date": "2009-11-15T14:12:12", "likes": 91}
+          {"index":{"_id": "92"}}
+          {"user": "test", "message": "some message with the number 92", "date": "2009-11-15T14:12:12", "likes": 92}
+          {"index":{"_id": "93"}}
+          {"user": "test", "message": "some message with the number 93", "date": "2009-11-15T14:12:12", "likes": 93}
+          {"index":{"_id": "94"}}
+          {"user": "test", "message": "some message with the number 94", "date": "2009-11-15T14:12:12", "likes": 94}
+          {"index":{"_id": "95"}}
+          {"user": "test", "message": "some message with the number 95", "date": "2009-11-15T14:12:12", "likes": 95}
+          {"index":{"_id": "96"}}
+          {"user": "test", "message": "some message with the number 96", "date": "2009-11-15T14:12:12", "likes": 96}
+          {"index":{"_id": "97"}}
+          {"user": "test", "message": "some message with the number 97", "date": "2009-11-15T14:12:12", "likes": 97}
+          {"index":{"_id": "98"}}
+          {"user": "test", "message": "some message with the number 98", "date": "2009-11-15T14:12:12", "likes": 98}
+          {"index":{"_id": "99"}}
+          {"user": "test", "message": "some message with the number 99", "date": "2009-11-15T14:12:12", "likes": 99}
+          {"index":{"_id": "100"}}
+          {"user": "test", "message": "some message with the number 100", "date": "2009-11-15T14:12:12", "likes": 100}
+          {"index":{"_id": "101"}}
+          {"user": "test", "message": "some message with the number 101", "date": "2009-11-15T14:12:12", "likes": 101}
+          {"index":{"_id": "102"}}
+          {"user": "test", "message": "some message with the number 102", "date": "2009-11-15T14:12:12", "likes": 102}
+          {"index":{"_id": "103"}}
+          {"user": "test", "message": "some message with the number 103", "date": "2009-11-15T14:12:12", "likes": 103}
+          {"index":{"_id": "104"}}
+          {"user": "test", "message": "some message with the number 104", "date": "2009-11-15T14:12:12", "likes": 104}
+          {"index":{"_id": "105"}}
+          {"user": "test", "message": "some message with the number 105", "date": "2009-11-15T14:12:12", "likes": 105}
+          {"index":{"_id": "106"}}
+          {"user": "test", "message": "some message with the number 106", "date": "2009-11-15T14:12:12", "likes": 106}
+          {"index":{"_id": "107"}}
+          {"user": "test", "message": "some message with the number 107", "date": "2009-11-15T14:12:12", "likes": 107}
+          {"index":{"_id": "108"}}
+          {"user": "test", "message": "some message with the number 108", "date": "2009-11-15T14:12:12", "likes": 108}
+          {"index":{"_id": "109"}}
+          {"user": "test", "message": "some message with the number 109", "date": "2009-11-15T14:12:12", "likes": 109}
+          {"index":{"_id": "110"}}
+          {"user": "test", "message": "some message with the number 110", "date": "2009-11-15T14:12:12", "likes": 110}
+          {"index":{"_id": "111"}}
+          {"user": "test", "message": "some message with the number 111", "date": "2009-11-15T14:12:12", "likes": 111}
+          {"index":{"_id": "112"}}
+          {"user": "test", "message": "some message with the number 112", "date": "2009-11-15T14:12:12", "likes": 112}
+          {"index":{"_id": "113"}}
+          {"user": "test", "message": "some message with the number 113", "date": "2009-11-15T14:12:12", "likes": 113}
+          {"index":{"_id": "114"}}
+          {"user": "test", "message": "some message with the number 114", "date": "2009-11-15T14:12:12", "likes": 114}
+          {"index":{"_id": "115"}}
+          {"user": "test", "message": "some message with the number 115", "date": "2009-11-15T14:12:12", "likes": 115}
+          {"index":{"_id": "116"}}
+          {"user": "test", "message": "some message with the number 116", "date": "2009-11-15T14:12:12", "likes": 116}
+          {"index":{"_id": "117"}}
+          {"user": "test", "message": "some message with the number 117", "date": "2009-11-15T14:12:12", "likes": 117}
+          {"index":{"_id": "118"}}
+          {"user": "test", "message": "some message with the number 118", "date": "2009-11-15T14:12:12", "likes": 118}
+          {"index":{"_id": "119"}}
+          {"user": "test", "message": "some message with the number 119", "date": "2009-11-15T14:12:12", "likes": 119}
+emit snippet
+query part: conflicts=proceed
+handle snippet
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+emit snippet
+query part: conflicts=proceed
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+emit snippet
+query part: conflicts=proceed
+body part: 
+{
+  "query": {
+    "term": {
+      "user": "kimchy"
+    }
+  }
+}
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+emit snippet
+body part: 
+{
+  "script": {
+    "inline": "ctx._source.likes++"
+  },
+  "query": {
+    "term": {
+      "user": "kimchy"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+emit snippet
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+emit snippet
+query part: routing=1
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+emit snippet
+query part: scroll_size=100
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+emit snippet
+body part: 
+{
+  "description" : "sets foo",
+  "processors" : [ {
+      "set" : {
+        "field": "foo",
+        "value": "bar"
+      }
+  } ]
+}
+emit snippet
+query part: pipeline=set-foo
+handle snippet
+handle snippet
+test snippet
+emit snippet
+query part: detailed=true&action=*byquery
+handle snippet
+handle snippet
+test snippet
+emit snippet
+handle snippet
+test snippet
+emit snippet
+query part: requests_per_second=unlimited
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "test": {
+      "dynamic": false,
+      "properties": {
+        "text": {"type": "text"}
+      }
+    }
+  }
+}
+emit snippet
+query part: refresh
+body part: 
+{
+  "text": "words words",
+  "flag": "bar"
+}
+emit snippet
+query part: refresh
+body part: 
+{
+  "text": "words words",
+  "flag": "foo"
+}
+emit snippet
+body part: 
+{
+  "properties": {
+    "text": {"type": "text"},
+    "flag": {"type": "text", "analyzer": "keyword"}
+  }
+}
+handle snippet
+test snippet
+emit snippet
+query part: filter_path=hits.total
+body part: 
+{
+  "query": {
+    "match": {
+      "flag": "foo"
+    }
+  }
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+query part: refresh&conflicts=proceed
+emit snippet
+query part: filter_path=hits.total
+body part: 
+{
+  "query": {
+    "match": {
+      "flag": "foo"
+    }
+  }
+}
+handle snippet
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+          {"index":{"_id": "5"}}
+          {"user": "test", "message": "some message with the number 5", "date": "2009-11-15T14:12:12", "likes": 5}
+          {"index":{"_id": "6"}}
+          {"user": "test", "message": "some message with the number 6", "date": "2009-11-15T14:12:12", "likes": 6}
+          {"index":{"_id": "7"}}
+          {"user": "test", "message": "some message with the number 7", "date": "2009-11-15T14:12:12", "likes": 7}
+          {"index":{"_id": "8"}}
+          {"user": "test", "message": "some message with the number 8", "date": "2009-11-15T14:12:12", "likes": 8}
+          {"index":{"_id": "9"}}
+          {"user": "test", "message": "some message with the number 9", "date": "2009-11-15T14:12:12", "likes": 9}
+          {"index":{"_id": "10"}}
+          {"user": "test", "message": "some message with the number 10", "date": "2009-11-15T14:12:12", "likes": 10}
+          {"index":{"_id": "11"}}
+          {"user": "test", "message": "some message with the number 11", "date": "2009-11-15T14:12:12", "likes": 11}
+          {"index":{"_id": "12"}}
+          {"user": "test", "message": "some message with the number 12", "date": "2009-11-15T14:12:12", "likes": 12}
+          {"index":{"_id": "13"}}
+          {"user": "test", "message": "some message with the number 13", "date": "2009-11-15T14:12:12", "likes": 13}
+          {"index":{"_id": "14"}}
+          {"user": "test", "message": "some message with the number 14", "date": "2009-11-15T14:12:12", "likes": 14}
+          {"index":{"_id": "15"}}
+          {"user": "test", "message": "some message with the number 15", "date": "2009-11-15T14:12:12", "likes": 15}
+          {"index":{"_id": "16"}}
+          {"user": "test", "message": "some message with the number 16", "date": "2009-11-15T14:12:12", "likes": 16}
+          {"index":{"_id": "17"}}
+          {"user": "test", "message": "some message with the number 17", "date": "2009-11-15T14:12:12", "likes": 17}
+          {"index":{"_id": "18"}}
+          {"user": "test", "message": "some message with the number 18", "date": "2009-11-15T14:12:12", "likes": 18}
+          {"index":{"_id": "19"}}
+          {"user": "test", "message": "some message with the number 19", "date": "2009-11-15T14:12:12", "likes": 19}
+          {"index":{"_id": "20"}}
+          {"user": "test", "message": "some message with the number 20", "date": "2009-11-15T14:12:12", "likes": 20}
+          {"index":{"_id": "21"}}
+          {"user": "test", "message": "some message with the number 21", "date": "2009-11-15T14:12:12", "likes": 21}
+          {"index":{"_id": "22"}}
+          {"user": "test", "message": "some message with the number 22", "date": "2009-11-15T14:12:12", "likes": 22}
+          {"index":{"_id": "23"}}
+          {"user": "test", "message": "some message with the number 23", "date": "2009-11-15T14:12:12", "likes": 23}
+          {"index":{"_id": "24"}}
+          {"user": "test", "message": "some message with the number 24", "date": "2009-11-15T14:12:12", "likes": 24}
+          {"index":{"_id": "25"}}
+          {"user": "test", "message": "some message with the number 25", "date": "2009-11-15T14:12:12", "likes": 25}
+          {"index":{"_id": "26"}}
+          {"user": "test", "message": "some message with the number 26", "date": "2009-11-15T14:12:12", "likes": 26}
+          {"index":{"_id": "27"}}
+          {"user": "test", "message": "some message with the number 27", "date": "2009-11-15T14:12:12", "likes": 27}
+          {"index":{"_id": "28"}}
+          {"user": "test", "message": "some message with the number 28", "date": "2009-11-15T14:12:12", "likes": 28}
+          {"index":{"_id": "29"}}
+          {"user": "test", "message": "some message with the number 29", "date": "2009-11-15T14:12:12", "likes": 29}
+          {"index":{"_id": "30"}}
+          {"user": "test", "message": "some message with the number 30", "date": "2009-11-15T14:12:12", "likes": 30}
+          {"index":{"_id": "31"}}
+          {"user": "test", "message": "some message with the number 31", "date": "2009-11-15T14:12:12", "likes": 31}
+          {"index":{"_id": "32"}}
+          {"user": "test", "message": "some message with the number 32", "date": "2009-11-15T14:12:12", "likes": 32}
+          {"index":{"_id": "33"}}
+          {"user": "test", "message": "some message with the number 33", "date": "2009-11-15T14:12:12", "likes": 33}
+          {"index":{"_id": "34"}}
+          {"user": "test", "message": "some message with the number 34", "date": "2009-11-15T14:12:12", "likes": 34}
+          {"index":{"_id": "35"}}
+          {"user": "test", "message": "some message with the number 35", "date": "2009-11-15T14:12:12", "likes": 35}
+          {"index":{"_id": "36"}}
+          {"user": "test", "message": "some message with the number 36", "date": "2009-11-15T14:12:12", "likes": 36}
+          {"index":{"_id": "37"}}
+          {"user": "test", "message": "some message with the number 37", "date": "2009-11-15T14:12:12", "likes": 37}
+          {"index":{"_id": "38"}}
+          {"user": "test", "message": "some message with the number 38", "date": "2009-11-15T14:12:12", "likes": 38}
+          {"index":{"_id": "39"}}
+          {"user": "test", "message": "some message with the number 39", "date": "2009-11-15T14:12:12", "likes": 39}
+          {"index":{"_id": "40"}}
+          {"user": "test", "message": "some message with the number 40", "date": "2009-11-15T14:12:12", "likes": 40}
+          {"index":{"_id": "41"}}
+          {"user": "test", "message": "some message with the number 41", "date": "2009-11-15T14:12:12", "likes": 41}
+          {"index":{"_id": "42"}}
+          {"user": "test", "message": "some message with the number 42", "date": "2009-11-15T14:12:12", "likes": 42}
+          {"index":{"_id": "43"}}
+          {"user": "test", "message": "some message with the number 43", "date": "2009-11-15T14:12:12", "likes": 43}
+          {"index":{"_id": "44"}}
+          {"user": "test", "message": "some message with the number 44", "date": "2009-11-15T14:12:12", "likes": 44}
+          {"index":{"_id": "45"}}
+          {"user": "test", "message": "some message with the number 45", "date": "2009-11-15T14:12:12", "likes": 45}
+          {"index":{"_id": "46"}}
+          {"user": "test", "message": "some message with the number 46", "date": "2009-11-15T14:12:12", "likes": 46}
+          {"index":{"_id": "47"}}
+          {"user": "test", "message": "some message with the number 47", "date": "2009-11-15T14:12:12", "likes": 47}
+          {"index":{"_id": "48"}}
+          {"user": "test", "message": "some message with the number 48", "date": "2009-11-15T14:12:12", "likes": 48}
+          {"index":{"_id": "49"}}
+          {"user": "test", "message": "some message with the number 49", "date": "2009-11-15T14:12:12", "likes": 49}
+          {"index":{"_id": "50"}}
+          {"user": "test", "message": "some message with the number 50", "date": "2009-11-15T14:12:12", "likes": 50}
+          {"index":{"_id": "51"}}
+          {"user": "test", "message": "some message with the number 51", "date": "2009-11-15T14:12:12", "likes": 51}
+          {"index":{"_id": "52"}}
+          {"user": "test", "message": "some message with the number 52", "date": "2009-11-15T14:12:12", "likes": 52}
+          {"index":{"_id": "53"}}
+          {"user": "test", "message": "some message with the number 53", "date": "2009-11-15T14:12:12", "likes": 53}
+          {"index":{"_id": "54"}}
+          {"user": "test", "message": "some message with the number 54", "date": "2009-11-15T14:12:12", "likes": 54}
+          {"index":{"_id": "55"}}
+          {"user": "test", "message": "some message with the number 55", "date": "2009-11-15T14:12:12", "likes": 55}
+          {"index":{"_id": "56"}}
+          {"user": "test", "message": "some message with the number 56", "date": "2009-11-15T14:12:12", "likes": 56}
+          {"index":{"_id": "57"}}
+          {"user": "test", "message": "some message with the number 57", "date": "2009-11-15T14:12:12", "likes": 57}
+          {"index":{"_id": "58"}}
+          {"user": "test", "message": "some message with the number 58", "date": "2009-11-15T14:12:12", "likes": 58}
+          {"index":{"_id": "59"}}
+          {"user": "test", "message": "some message with the number 59", "date": "2009-11-15T14:12:12", "likes": 59}
+          {"index":{"_id": "60"}}
+          {"user": "test", "message": "some message with the number 60", "date": "2009-11-15T14:12:12", "likes": 60}
+          {"index":{"_id": "61"}}
+          {"user": "test", "message": "some message with the number 61", "date": "2009-11-15T14:12:12", "likes": 61}
+          {"index":{"_id": "62"}}
+          {"user": "test", "message": "some message with the number 62", "date": "2009-11-15T14:12:12", "likes": 62}
+          {"index":{"_id": "63"}}
+          {"user": "test", "message": "some message with the number 63", "date": "2009-11-15T14:12:12", "likes": 63}
+          {"index":{"_id": "64"}}
+          {"user": "test", "message": "some message with the number 64", "date": "2009-11-15T14:12:12", "likes": 64}
+          {"index":{"_id": "65"}}
+          {"user": "test", "message": "some message with the number 65", "date": "2009-11-15T14:12:12", "likes": 65}
+          {"index":{"_id": "66"}}
+          {"user": "test", "message": "some message with the number 66", "date": "2009-11-15T14:12:12", "likes": 66}
+          {"index":{"_id": "67"}}
+          {"user": "test", "message": "some message with the number 67", "date": "2009-11-15T14:12:12", "likes": 67}
+          {"index":{"_id": "68"}}
+          {"user": "test", "message": "some message with the number 68", "date": "2009-11-15T14:12:12", "likes": 68}
+          {"index":{"_id": "69"}}
+          {"user": "test", "message": "some message with the number 69", "date": "2009-11-15T14:12:12", "likes": 69}
+          {"index":{"_id": "70"}}
+          {"user": "test", "message": "some message with the number 70", "date": "2009-11-15T14:12:12", "likes": 70}
+          {"index":{"_id": "71"}}
+          {"user": "test", "message": "some message with the number 71", "date": "2009-11-15T14:12:12", "likes": 71}
+          {"index":{"_id": "72"}}
+          {"user": "test", "message": "some message with the number 72", "date": "2009-11-15T14:12:12", "likes": 72}
+          {"index":{"_id": "73"}}
+          {"user": "test", "message": "some message with the number 73", "date": "2009-11-15T14:12:12", "likes": 73}
+          {"index":{"_id": "74"}}
+          {"user": "test", "message": "some message with the number 74", "date": "2009-11-15T14:12:12", "likes": 74}
+          {"index":{"_id": "75"}}
+          {"user": "test", "message": "some message with the number 75", "date": "2009-11-15T14:12:12", "likes": 75}
+          {"index":{"_id": "76"}}
+          {"user": "test", "message": "some message with the number 76", "date": "2009-11-15T14:12:12", "likes": 76}
+          {"index":{"_id": "77"}}
+          {"user": "test", "message": "some message with the number 77", "date": "2009-11-15T14:12:12", "likes": 77}
+          {"index":{"_id": "78"}}
+          {"user": "test", "message": "some message with the number 78", "date": "2009-11-15T14:12:12", "likes": 78}
+          {"index":{"_id": "79"}}
+          {"user": "test", "message": "some message with the number 79", "date": "2009-11-15T14:12:12", "likes": 79}
+          {"index":{"_id": "80"}}
+          {"user": "test", "message": "some message with the number 80", "date": "2009-11-15T14:12:12", "likes": 80}
+          {"index":{"_id": "81"}}
+          {"user": "test", "message": "some message with the number 81", "date": "2009-11-15T14:12:12", "likes": 81}
+          {"index":{"_id": "82"}}
+          {"user": "test", "message": "some message with the number 82", "date": "2009-11-15T14:12:12", "likes": 82}
+          {"index":{"_id": "83"}}
+          {"user": "test", "message": "some message with the number 83", "date": "2009-11-15T14:12:12", "likes": 83}
+          {"index":{"_id": "84"}}
+          {"user": "test", "message": "some message with the number 84", "date": "2009-11-15T14:12:12", "likes": 84}
+          {"index":{"_id": "85"}}
+          {"user": "test", "message": "some message with the number 85", "date": "2009-11-15T14:12:12", "likes": 85}
+          {"index":{"_id": "86"}}
+          {"user": "test", "message": "some message with the number 86", "date": "2009-11-15T14:12:12", "likes": 86}
+          {"index":{"_id": "87"}}
+          {"user": "test", "message": "some message with the number 87", "date": "2009-11-15T14:12:12", "likes": 87}
+          {"index":{"_id": "88"}}
+          {"user": "test", "message": "some message with the number 88", "date": "2009-11-15T14:12:12", "likes": 88}
+          {"index":{"_id": "89"}}
+          {"user": "test", "message": "some message with the number 89", "date": "2009-11-15T14:12:12", "likes": 89}
+          {"index":{"_id": "90"}}
+          {"user": "test", "message": "some message with the number 90", "date": "2009-11-15T14:12:12", "likes": 90}
+          {"index":{"_id": "91"}}
+          {"user": "test", "message": "some message with the number 91", "date": "2009-11-15T14:12:12", "likes": 91}
+          {"index":{"_id": "92"}}
+          {"user": "test", "message": "some message with the number 92", "date": "2009-11-15T14:12:12", "likes": 92}
+          {"index":{"_id": "93"}}
+          {"user": "test", "message": "some message with the number 93", "date": "2009-11-15T14:12:12", "likes": 93}
+          {"index":{"_id": "94"}}
+          {"user": "test", "message": "some message with the number 94", "date": "2009-11-15T14:12:12", "likes": 94}
+          {"index":{"_id": "95"}}
+          {"user": "test", "message": "some message with the number 95", "date": "2009-11-15T14:12:12", "likes": 95}
+          {"index":{"_id": "96"}}
+          {"user": "test", "message": "some message with the number 96", "date": "2009-11-15T14:12:12", "likes": 96}
+          {"index":{"_id": "97"}}
+          {"user": "test", "message": "some message with the number 97", "date": "2009-11-15T14:12:12", "likes": 97}
+          {"index":{"_id": "98"}}
+          {"user": "test", "message": "some message with the number 98", "date": "2009-11-15T14:12:12", "likes": 98}
+          {"index":{"_id": "99"}}
+          {"user": "test", "message": "some message with the number 99", "date": "2009-11-15T14:12:12", "likes": 99}
+          {"index":{"_id": "100"}}
+          {"user": "test", "message": "some message with the number 100", "date": "2009-11-15T14:12:12", "likes": 100}
+          {"index":{"_id": "101"}}
+          {"user": "test", "message": "some message with the number 101", "date": "2009-11-15T14:12:12", "likes": 101}
+          {"index":{"_id": "102"}}
+          {"user": "test", "message": "some message with the number 102", "date": "2009-11-15T14:12:12", "likes": 102}
+          {"index":{"_id": "103"}}
+          {"user": "test", "message": "some message with the number 103", "date": "2009-11-15T14:12:12", "likes": 103}
+          {"index":{"_id": "104"}}
+          {"user": "test", "message": "some message with the number 104", "date": "2009-11-15T14:12:12", "likes": 104}
+          {"index":{"_id": "105"}}
+          {"user": "test", "message": "some message with the number 105", "date": "2009-11-15T14:12:12", "likes": 105}
+          {"index":{"_id": "106"}}
+          {"user": "test", "message": "some message with the number 106", "date": "2009-11-15T14:12:12", "likes": 106}
+          {"index":{"_id": "107"}}
+          {"user": "test", "message": "some message with the number 107", "date": "2009-11-15T14:12:12", "likes": 107}
+          {"index":{"_id": "108"}}
+          {"user": "test", "message": "some message with the number 108", "date": "2009-11-15T14:12:12", "likes": 108}
+          {"index":{"_id": "109"}}
+          {"user": "test", "message": "some message with the number 109", "date": "2009-11-15T14:12:12", "likes": 109}
+          {"index":{"_id": "110"}}
+          {"user": "test", "message": "some message with the number 110", "date": "2009-11-15T14:12:12", "likes": 110}
+          {"index":{"_id": "111"}}
+          {"user": "test", "message": "some message with the number 111", "date": "2009-11-15T14:12:12", "likes": 111}
+          {"index":{"_id": "112"}}
+          {"user": "test", "message": "some message with the number 112", "date": "2009-11-15T14:12:12", "likes": 112}
+          {"index":{"_id": "113"}}
+          {"user": "test", "message": "some message with the number 113", "date": "2009-11-15T14:12:12", "likes": 113}
+          {"index":{"_id": "114"}}
+          {"user": "test", "message": "some message with the number 114", "date": "2009-11-15T14:12:12", "likes": 114}
+          {"index":{"_id": "115"}}
+          {"user": "test", "message": "some message with the number 115", "date": "2009-11-15T14:12:12", "likes": 115}
+          {"index":{"_id": "116"}}
+          {"user": "test", "message": "some message with the number 116", "date": "2009-11-15T14:12:12", "likes": 116}
+          {"index":{"_id": "117"}}
+          {"user": "test", "message": "some message with the number 117", "date": "2009-11-15T14:12:12", "likes": 117}
+          {"index":{"_id": "118"}}
+          {"user": "test", "message": "some message with the number 118", "date": "2009-11-15T14:12:12", "likes": 118}
+          {"index":{"_id": "119"}}
+          {"user": "test", "message": "some message with the number 119", "date": "2009-11-15T14:12:12", "likes": 119}
+emit snippet
+body part: 
+{
+  "source": {
+    "index": "twitter"
+  },
+  "dest": {
+    "index": "new_twitter"
+  }
+}
+handle snippet
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+emit snippet
+body part: 
+{
+  "source": {
+    "index": "twitter"
+  },
+  "dest": {
+    "index": "new_twitter",
+    "version_type": "internal"
+  }
+}
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+emit snippet
+body part: 
+{
+  "source": {
+    "index": "twitter"
+  },
+  "dest": {
+    "index": "new_twitter",
+    "version_type": "external"
+  }
+}
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+emit snippet
+body part: 
+{
+  "source": {
+    "index": "twitter"
+  },
+  "dest": {
+    "index": "new_twitter",
+    "op_type": "create"
+  }
+}
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+emit snippet
+body part: 
+{
+  "conflicts": "proceed",
+  "source": {
+    "index": "twitter"
+  },
+  "dest": {
+    "index": "new_twitter",
+    "op_type": "create"
+  }
+}
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+emit snippet
+body part: 
+{
+  "source": {
+    "index": "twitter",
+    "type": "tweet",
+    "query": {
+      "term": {
+        "user": "kimchy"
+      }
+    }
+  },
+  "dest": {
+    "index": "new_twitter"
+  }
+}
+handle snippet
+test snippet
+emit snippet
+emit snippet
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "source": {
+    "index": ["twitter", "blog"],
+    "type": ["tweet", "post"]
+  },
+  "dest": {
+    "index": "all_together"
+  }
+}
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+emit snippet
+body part: 
+{
+  "size": 1,
+  "source": {
+    "index": "twitter"
+  },
+  "dest": {
+    "index": "new_twitter"
+  }
+}
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+emit snippet
+body part: 
+{
+  "size": 10000,
+  "source": {
+    "index": "twitter",
+    "sort": { "date": "desc" }
+  },
+  "dest": {
+    "index": "new_twitter"
+  }
+}
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+emit snippet
+body part: 
+{
+  "source": {
+    "index": "twitter"
+  },
+  "dest": {
+    "index": "new_twitter",
+    "version_type": "external"
+  },
+  "script": {
+    "inline": "if (ctx._source.foo == 'bar') {ctx._version++; ctx._source.remove('foo')}"
+  }
+}
+handle snippet
+test snippet
+emit snippet
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "source": {
+    "index": "source",
+    "query": {
+      "match": {
+        "company": "cat"
+      }
+    }
+  },
+  "dest": {
+    "index": "dest",
+    "routing": "=cat"
+  }
+}
+handle snippet
+test snippet
+emit snippet
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "source": {
+    "index": "source",
+    "size": 100
+  },
+  "dest": {
+    "index": "dest",
+    "routing": "=cat"
+  }
+}
+handle snippet
+test snippet
+emit snippet
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+body part: 
+{
+  "source": {
+    "index": "source"
+  },
+  "dest": {
+    "index": "dest",
+    "pipeline": "some_ingest_pipeline"
+  }
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+query part: detailed=true&actions=*reindex
+handle snippet
+handle snippet
+test snippet
+emit snippet
+handle snippet
+test snippet
+emit snippet
+query part: requests_per_second=unlimited
+handle snippet
+test snippet
+emit snippet
+query part: refresh
+body part: 
+{
+  "text": "words words",
+  "flag": "foo"
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "source": {
+    "index": "test"
+  },
+  "dest": {
+    "index": "test2"
+  },
+  "script": {
+    "inline": "ctx._source.tag = ctx._source.remove(\"flag\")"
+  }
+}
+handle snippet
+test snippet
+emit snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+catch part: request
+query part: pipeline=my_pipeline_id
+body part: 
+{
+  "foo": "bar"
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "transient": {
+    "cluster.routing.allocation.enable": "none"
+  }
+}
+handle snippet
+test snippet
+emit snippet
+handle snippet
+test snippet
+emit snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "transient": {
+    "cluster.routing.allocation.enable": "all"
+  }
+}
+handle snippet
+test snippet
+emit snippet
+handle snippet
+test snippet
+emit snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "persistent": {
+    "cluster.routing.allocation.enable": "none"
+  }
+}
+handle snippet
+test snippet
+emit snippet
+handle snippet
+test snippet
+emit snippet
+emit snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "persistent": {
+    "cluster.routing.allocation.enable": "all"
+  }
+}
+handle snippet
+test snippet
+emit snippet
+emit snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "user": {
+      "_all":       { "enabled": false  },
+      "properties": {
+        "title":    { "type": "text"  },
+        "name":     { "type": "text"  },
+        "age":      { "type": "integer" }
+      }
+    },
+    "blogpost": {
+      "_all":       { "enabled": false  },
+      "properties": {
+        "title":    { "type": "text"  },
+        "body":     { "type": "text"  },
+        "user_id":  {
+          "type":   "keyword"
+        },
+        "created":  {
+          "type":   "date",
+          "format": "strict_date_optional_time||epoch_millis"
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{ "count": 5 }
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "index.mapper.dynamic":false
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "session": {
+      "properties": {
+        "user_id": {
+          "type":  "keyword"
+        },
+        "last_updated": {
+          "type": "date"
+        },
+        "session_data": {
+          "enabled": false
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "user_id": "kimchy",
+  "session_data": {
+    "arbitrary_object": {
+      "some_array": [ "foo", "bar", { "baz": 2 } ]
+    }
+  },
+  "last_updated": "2015-12-06T18:20:22"
+}
+emit snippet
+body part: 
+{
+  "user_id": "jpountz",
+  "session_data": "none",
+  "last_updated": "2015-12-06T18:22:13"
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "session": {
+      "enabled": false
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "user_id": "kimchy",
+  "session_data": {
+    "arbitrary_object": {
+      "some_array": [ "foo", "bar", { "baz": 2 } ]
+    }
+  },
+  "last_updated": "2015-12-06T18:20:22"
+}
+emit snippet
+emit snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "location": {
+          "type": "geo_point",
+          "geohash_prefix": true,
+          "geohash_precision": 6
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "location": {
+    "lat": 41.12,
+    "lon": -71.34
+  }
+}
+emit snippet
+query part: fielddata_fields=location.geohash
+body part: 
+{
+  "query": {
+    "term": {
+      "location.geohash": "drm3bt"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "date": {
+          "type":   "date",
+          "format": "yyyy-MM-dd"
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "first_name": {
+          "type": "text",
+          "copy_to": "full_name"
+        },
+        "last_name": {
+          "type": "text",
+          "copy_to": "full_name"
+        },
+        "full_name": {
+          "type": "text"
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "first_name": "John",
+  "last_name": "Smith"
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "match": {
+      "full_name": {
+        "query": "John Smith",
+        "operator": "and"
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "title": {
+          "type": "text",
+          "store": true
+        },
+        "date": {
+          "type": "date",
+          "store": true
+        },
+        "content": {
+          "type": "text"
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "title":   "Some short title",
+  "date":    "2015-01-01",
+  "content": "A very long content field..."
+}
+emit snippet
+body part: 
+{
+  "fields": [ "title", "date" ]
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "status_code": {
+          "type":       "keyword",
+          "null_value": "NULL"
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "status_code": null
+}
+emit snippet
+body part: 
+{
+  "status_code": []
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "term": {
+      "status_code": "NULL"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "number_one": {
+          "type": "integer"
+        },
+        "number_two": {
+          "type": "integer",
+          "coerce": false
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "number_one": "10"
+}
+emit snippet
+catch part: request
+body part: 
+{
+  "number_two": "10"
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "index.mapping.coerce": false
+  },
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "number_one": {
+          "type": "integer",
+          "coerce": true
+        },
+        "number_two": {
+          "type": "integer"
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{ "number_one": "10" }
+emit snippet
+catch part: request
+body part: 
+{ "number_two": "10" }
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "analysis": {
+      "filter": {
+        "autocomplete_filter": {
+          "type": "edge_ngram",
+          "min_gram": 1,
+          "max_gram": 20
+        }
+      },
+      "analyzer": {
+        "autocomplete": {
+          "type": "custom",
+          "tokenizer": "standard",
+          "filter": [
+            "lowercase",
+            "autocomplete_filter"
+          ]
+        }
+      }
+    }
+  },
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "text": {
+          "type": "text",
+          "analyzer": "autocomplete",
+          "search_analyzer": "standard"
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "text": "Quick Brown Fox"
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "match": {
+      "text": {
+        "query": "Quick Br",
+        "operator": "and"
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "text": {
+          "type":        "text",
+          "term_vector": "with_positions_offsets"
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "text": "Quick brown fox"
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "match": {
+      "text": "brown fox"
+    }
+  },
+  "highlight": {
+    "fields": {
+      "text": {}
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "city": {
+          "type": "text",
+          "fields": {
+            "raw": {
+              "type":  "keyword"
+            }
+          }
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "city": "New York"
+}
+emit snippet
+body part: 
+{
+  "city": "York"
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "match": {
+      "city": "york"
+    }
+  },
+  "sort": {
+    "city.raw": "asc"
+  },
+  "aggs": {
+    "Cities": {
+      "terms": {
+        "field": "city.raw"
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "text": {
+          "type": "text",
+          "fields": {
+            "english": {
+              "type":     "text",
+              "analyzer": "english"
+            }
+          }
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{ "text": "quick brown fox" }
+emit snippet
+body part: 
+{ "text": "quick brown foxes" }
+emit snippet
+body part: 
+{
+  "query": {
+    "multi_match": {
+      "query": "quick brown foxes",
+      "fields": [
+        "text",
+        "text.english"
+      ],
+      "type": "most_fields"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "tag": {
+          "type": "text",
+          "fielddata": true,
+          "fielddata_frequency_filter": {
+            "min": 0.001,
+            "max": 0.1,
+            "min_segment_size": 500
+          }
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "number_one": {
+          "type": "integer",
+          "ignore_malformed": true
+        },
+        "number_two": {
+          "type": "integer"
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "text":       "Some text value",
+  "number_one": "foo"
+}
+emit snippet
+catch part: request
+body part: 
+{
+  "text":       "Some text value",
+  "number_two": "foo"
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "index.mapping.ignore_malformed": true
+  },
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "number_one": {
+          "type": "byte"
+        },
+        "number_two": {
+          "type": "integer",
+          "ignore_malformed": false
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "username": "johnsmith",
+  "name": {
+    "first": "John",
+    "last": "Smith"
+  }
+}
+emit snippet
+emit snippet
+body part: 
+{
+  "username": "marywhite",
+  "email": "mary@white.com",
+  "name": {
+    "first": "Mary",
+    "middle": "Alice",
+    "last": "White"
+  }
+}
+emit snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "dynamic": false,
+      "properties": {
+        "user": {
+          "properties": {
+            "name": {
+              "type": "text"
+            },
+            "social_networks": {
+              "dynamic": true,
+              "properties": {}
+            }
+          }
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "names": [ "John Abraham", "Lincoln Smith"]
+}
+emit snippet
+body part: 
+{
+    "query": {
+        "match_phrase": {
+            "names": {
+                "query": "Abraham Lincoln"
+            }
+        }
+    }
+}
+emit snippet
+body part: 
+{
+    "query": {
+        "match_phrase": {
+            "names": {
+                "query": "Abraham Lincoln",
+                "slop": 101
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "groups": {
+      "properties": {
+        "names": {
+          "type": "text",
+          "position_increment_gap": 0
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+    "names": [ "John Abraham", "Lincoln Smith"]
+}
+emit snippet
+body part: 
+{
+    "query": {
+        "match_phrase": {
+            "names": "Abraham Lincoln"
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "message": {
+          "type": "keyword",
+          "ignore_above": 20
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "message": "Syntax error"
+}
+emit snippet
+body part: 
+{
+  "message": "Syntax error with some long stacktrace"
+}
+emit snippet
+body part: 
+{
+  "aggs": {
+    "messages": {
+      "terms": {
+        "field": "message"
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "location": {
+          "type": "geo_point",
+          "lat_lon": true
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "location": {
+    "lat": 41.12,
+    "lon": -71.34
+  }
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "geo_distance": {
+      "location": {
+        "lat": 41,
+        "lon": -71
+      },
+      "distance": "50km",
+      "optimize_bbox": "indexed"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "location": {
+          "type": "geo_point",
+          "geohash_prefix": true,
+          "geohash_precision": 6
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "location": {
+    "lat": 41.12,
+    "lon": -71.34
+  }
+}
+emit snippet
+query part: fielddata_fields=location.geohash
+body part: 
+{
+  "query": {
+    "geohash_cell": {
+      "location": {
+        "lat": 41.02,
+        "lon": -71.48
+      },
+      "precision": 4,
+      "neighbors": true
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "status_code": {
+          "type":       "keyword"
+        },
+        "session_id": {
+          "type":       "keyword",
+          "doc_values": false
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "text": {
+          "type": "text",
+          "index_options": "offsets"
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "text": "Quick brown fox"
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "match": {
+      "text": "brown fox"
+    }
+  },
+  "highlight": {
+    "fields": {
+      "text": {}
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "manager": {
+          "properties": {
+            "age":  { "type": "integer" },
+            "name": { "type": "text"  }
+          }
+        },
+        "employees": {
+          "type": "nested",
+          "properties": {
+            "age":  { "type": "integer" },
+            "name": { "type": "text"  }
+          }
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "region": "US",
+  "manager": {
+    "name": "Alice White",
+    "age": 30
+  },
+  "employees": [
+    {
+      "name": "John Smith",
+      "age": 34
+    },
+    {
+      "name": "Peter Brown",
+      "age": 26
+    }
+  ]
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "match": {
+      "manager.name": "Alice White"
+    }
+  },
+  "aggs": {
+    "Employees": {
+      "nested": {
+        "path": "employees"
+      },
+      "aggs": {
+        "Employee Ages": {
+          "histogram": {
+            "field": "employees.age",
+            "interval": 5
+          }
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "title": {
+          "type": "text"
+        },
+        "content": {
+          "type": "text"
+        },
+        "date": {
+          "type": "date",
+          "include_in_all": false
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "include_in_all": false,
+      "properties": {
+        "title":          { "type": "text" },
+        "author": {
+          "include_in_all": true,
+          "properties": {
+            "first_name": { "type": "text" },
+            "last_name":  { "type": "text" }
+          }
+        },
+        "editor": {
+          "properties": {
+            "first_name": { "type": "text" },
+            "last_name":  { "type": "text", "include_in_all": true }
+          }
+        }
+      }
+    }
+  }
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+emit snippet
+body part: 
+{
+  "properties": {
+    "title": {
+      "type": "text",
+      "norms": false
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "title": {
+          "type": "text",
+          "boost": 2
+        },
+        "content": {
+          "type": "text"
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "match" : {
+            "title": {
+                "query": "quick brown fox"
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "query": {
+        "match" : {
+            "title": {
+                "query": "quick brown fox",
+                "boost": 2
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "default_field": {
+          "type": "text"
+        },
+        "bm25_field": {
+          "type": "text",
+          "similarity": "BM25"
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "location": {
+          "type": "geo_point",
+          "geohash": true
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "location": {
+    "lat": 41.12,
+    "lon": -71.34
+  }
+}
+emit snippet
+query part: fielddata_fields=location.geohash
+body part: 
+{
+  "query": {
+    "prefix": {
+      "location.geohash": "drm3b"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "text": {
+          "type": "text",
+          "fields": {
+            "english": {
+              "type":     "text",
+              "analyzer": "english"
+            }
+          }
+        }
+      }
+    }
+  }
+}
+emit snippet
+query part: wait_for_status=yellow
+emit snippet
+query part: field=text
+body part: 
+{
+  "text": "The quick Brown Foxes."
+}
+emit snippet
+query part: field=text.english
+body part: 
+{
+  "text": "The quick Brown Foxes."
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+   "settings":{
+      "analysis":{
+         "analyzer":{
+            "my_analyzer":{
+               "type":"custom",
+               "tokenizer":"standard",
+               "filter":[
+                  "lowercase"
+               ]
+            },
+            "my_stop_analyzer":{
+               "type":"custom",
+               "tokenizer":"standard",
+               "filter":[
+                  "lowercase",
+                  "english_stop"
+               ]
+            }
+         },
+         "filter":{
+            "english_stop":{
+               "type":"stop",
+               "stopwords":"_english_"
+            }
+         }
+      }
+   },
+   "mappings":{
+      "my_type":{
+         "properties":{
+            "title": {
+               "type":"text",
+               "analyzer":"my_analyzer",
+               "search_analyzer":"my_stop_analyzer",
+               "search_quote_analyzer":"my_analyzer"
+            }
+         }
+      }
+   }
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "region": "US",
+  "manager": {
+    "age":     30,
+    "name": {
+      "first": "John",
+      "last":  "Smith"
+    }
+  }
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "region": {
+          "type": "keyword"
+        },
+        "manager": {
+          "properties": {
+            "age":  { "type": "integer" },
+            "name": {
+              "properties": {
+                "first": { "type": "text" },
+                "last":  { "type": "text" }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "date": {
+          "type": "date"
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{ "date": "2015-01-01" }
+emit snippet
+body part: 
+{ "date": "2015-01-01T12:10:30Z" }
+emit snippet
+body part: 
+{ "date": 1420070400001 }
+emit snippet
+body part: 
+{
+  "sort": { "date": "asc"}
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "date": {
+          "type":   "date",
+          "format": "yyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "full_name": {
+          "type":  "text"
+        }
+      }
+    }
+  }
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "name": {
+          "type": "text",
+          "fields": {
+            "length": {
+              "type":     "token_count",
+              "analyzer": "standard"
+            }
+          }
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{ "name": "John Smith" }
+emit snippet
+body part: 
+{ "name": "Rachel Alice Williams" }
+emit snippet
+body part: 
+{
+  "query": {
+    "term": {
+      "name.length": 3
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "group" : "fans",
+  "user" : [
+    {
+      "first" : "John",
+      "last" :  "Smith"
+    },
+    {
+      "first" : "Alice",
+      "last" :  "White"
+    }
+  ]
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "bool": {
+      "must": [
+        { "match": { "user.first": "Alice" }},
+        { "match": { "user.last":  "Smith" }}
+      ]
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "user": {
+          "type": "nested"
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "group" : "fans",
+  "user" : [
+    {
+      "first" : "John",
+      "last" :  "Smith"
+    },
+    {
+      "first" : "Alice",
+      "last" :  "White"
+    }
+  ]
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "nested": {
+      "path": "user",
+      "query": {
+        "bool": {
+          "must": [
+            { "match": { "user.first": "Alice" }},
+            { "match": { "user.last":  "Smith" }}
+          ]
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "nested": {
+      "path": "user",
+      "query": {
+        "bool": {
+          "must": [
+            { "match": { "user.first": "Alice" }},
+            { "match": { "user.last":  "White" }}
+          ]
+        }
+      },
+      "inner_hits": {
+        "highlight": {
+          "fields": {
+            "user.first": {}
+          }
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "is_published": {
+          "type": "boolean"
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "is_published": true
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "term": {
+      "is_published": 1
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "is_published": true
+}
+emit snippet
+body part: 
+{
+  "is_published": false
+}
+emit snippet
+body part: 
+{
+  "aggs": {
+    "publish_state": {
+      "terms": {
+        "field": "is_published"
+      }
+    }
+  },
+  "script_fields": {
+    "is_published": {
+      "script": "doc['is_published'].value"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "number_of_bytes": {
+          "type": "integer"
+        },
+        "time_in_seconds": {
+          "type": "float"
+        }
+      }
+    }
+  }
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "tags": {
+          "type":  "keyword"
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "location": {
+          "type": "geo_point"
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "text": "Geo-point as an object",
+  "location": {
+    "lat": 41.12,
+    "lon": -71.34
+  }
+}
+emit snippet
+body part: 
+{
+  "text": "Geo-point as a string",
+  "location": "41.12,-71.34"
+}
+emit snippet
+body part: 
+{
+  "text": "Geo-point as a geohash",
+  "location": "drm3btev3e86"
+}
+emit snippet
+body part: 
+{
+  "text": "Geo-point as an array",
+  "location": [ -71.34, 41.12 ]
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "geo_bounding_box": {
+      "location": {
+        "top_left": {
+          "lat": 42,
+          "lon": -72
+        },
+        "bottom_right": {
+          "lat": 40,
+          "lon": -74
+        }
+      }
+    }
+  }
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "message": "some arrays in this document...",
+  "tags":  [ "elasticsearch", "wow" ],
+  "lists": [
+    {
+      "name": "prog_list",
+      "description": "programming list"
+    },
+    {
+      "name": "cool_list",
+      "description": "cool stuff list"
+    }
+  ]
+}
+emit snippet
+body part: 
+{
+  "message": "no arrays in this document...",
+  "tags":  "elasticsearch",
+  "lists": {
+    "name": "prog_list",
+    "description": "programming list"
+  }
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "match": {
+      "tags": "elasticsearch"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "properties": {
+        "ip_addr": {
+          "type": "ip"
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "ip_addr": "192.168.1.1"
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "term": {
+      "ip_addr": "192.168.0.0/16"
+    }
+  }
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "dynamic_templates": [
+        {
+          "integers": {
+            "match_mapping_type": "long",
+            "mapping": {
+              "type": "integer"
+            }
+          }
+        },
+        {
+          "strings": {
+            "match_mapping_type": "string",
+            "mapping": {
+              "type": "text",
+              "fields": {
+                "raw": {
+                  "type":  "keyword",
+                  "ignore_above": 256
+                }
+              }
+            }
+          }
+        }
+      ]
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "my_integer": 5,
+  "my_string": "Some string"
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "dynamic_templates": [
+        {
+          "longs_as_strings": {
+            "match_mapping_type": "string",
+            "match":   "long_*",
+            "unmatch": "*_text",
+            "mapping": {
+              "type": "long"
+            }
+          }
+        }
+      ]
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "long_num": "5",
+  "long_text": "foo"
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "dynamic_templates": [
+        {
+          "full_name": {
+            "path_match":   "name.*",
+            "path_unmatch": "*.middle",
+            "mapping": {
+              "type":       "text",
+              "copy_to":    "full_name"
+            }
+          }
+        }
+      ]
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "name": {
+    "first":  "Alice",
+    "middle": "Mary",
+    "last":   "White"
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "dynamic_templates": [
+        {
+          "named_analyzers": {
+            "match_mapping_type": "string",
+            "match": "*",
+            "mapping": {
+              "type": "text",
+              "analyzer": "{name}"
+            }
+          }
+        },
+        {
+          "no_doc_values": {
+            "match_mapping_type":"*",
+            "mapping": {
+              "type": "{dynamic_type}",
+              "doc_values": false
+            }
+          }
+        }
+      ]
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "english": "Some English text",
+  "count":   5
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "order": 0,
+  "template": "*",
+  "mappings": {
+    "_default_": {
+      "_all": {
+        "enabled": false
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "create_date": "2015/09/02"
+}
+emit snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "date_detection": false
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "create": "2015/09/02"
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "dynamic_date_formats": ["MM/dd/yyyy"]
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "create_date": "09/25/2015"
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "numeric_detection": true
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "my_float":   "1.0",
+  "my_integer": "1"
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "_default_": {
+      "_all": {
+        "enabled": false
+      }
+    },
+    "user": {},
+    "blogpost": {
+      "_all": {
+        "enabled": true
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "template":   "logs-*",
+  "settings": { "number_of_shards": 1 },
+  "mappings": {
+    "_default_": {
+      "_all": {
+        "enabled": false
+      },
+      "dynamic_templates": [
+        {
+          "strings": {
+            "match_mapping_type": "string",
+            "mapping": {
+              "type": "text",
+              "fields": {
+                "raw": {
+                  "type":  "keyword",
+                  "ignore_above": 256
+                }
+              }
+            }
+          }
+        }
+      ]
+    }
+  }
+}
+emit snippet
+body part: 
+{ "message": "error:16" }
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "_timestamp": {
+        "enabled": true
+      }
+    }
+  }
+}
+emit snippet
+query part: timestamp=2015-01-01
+body part: 
+{ "text": "Timestamp as a formatted date" }
+emit snippet
+query part: timestamp=1420070400000
+body part: 
+{ "text": "Timestamp as milliseconds since the epoch" }
+emit snippet
+body part: 
+{ "text": "Autogenerated timestamp set to now()" }
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "range": {
+      "_timestamp": {
+        "gte": "2015-01-01"
+      }
+    }
+  },
+  "aggs": {
+    "Timestamps": {
+      "terms": {
+        "field": "_timestamp",
+        "size": 10
+      }
+    }
+  },
+  "sort": [
+    {
+      "_timestamp": {
+        "order": "desc"
+      }
+    }
+  ],
+  "script_fields": {
+    "Timestamp": {
+      "script": "doc['_timestamp']"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "first_name":    "John",
+  "last_name":     "Smith",
+  "date_of_birth": "1970-10-24"
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "match": {
+      "_all": "john smith 1970"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "query_string": {
+      "query": "john smith 1970"
+    }
+  }
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "type_1": {
+      "properties": {}
+    },
+    "type_2": {
+      "_all": {
+        "enabled": false
+      },
+      "properties": {}
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "_all": {
+        "enabled": false
+      },
+      "properties": {
+        "content": {
+          "type": "text"
+        }
+      }
+    }
+  },
+  "settings": {
+    "index.query.default_field": "content"
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "mytype": {
+      "properties": {
+        "title": {
+          "type": "text",
+          "boost": 2
+        },
+        "content": {
+          "type": "text"
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "mytype": {
+      "properties": {
+        "first_name": {
+          "type":    "text",
+          "copy_to": "full_name"
+        },
+        "last_name": {
+          "type":    "text",
+          "copy_to": "full_name"
+        },
+        "full_name": {
+          "type":    "text"
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "first_name": "John",
+  "last_name": "Smith"
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "match": {
+      "full_name": "John Smith"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "mytype": {
+      "_all": {
+        "store": true
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "first_name": "John",
+  "last_name": "Smith"
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "match": {
+      "_all": "John Smith"
+    }
+  },
+  "highlight": {
+    "fields": {
+      "_all": {}
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "mytype": {
+      "_all": {}
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "first_name": "John",
+  "last_name": "Smith"
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "match": {
+      "_all": "John Smith"
+    }
+  },
+  "highlight": {
+    "fields": {
+      "*_name": {
+        "require_field_match": false
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "title": "This is a document"
+}
+emit snippet
+body part: 
+{
+  "title": "This is another document",
+  "body": "This document has a body"
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "terms": {
+      "_field_names": [ "title" ]
+    }
+  },
+  "script_fields": {
+    "Field names": {
+      "script": "doc['_field_names']"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "text": "Document in index 1"
+}
+emit snippet
+body part: 
+{
+  "text": "Document in index 2"
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "terms": {
+      "_index": ["index_1", "index_2"]
+    }
+  },
+  "aggs": {
+    "indices": {
+      "terms": {
+        "field": "_index",
+        "size": 10
+      }
+    }
+  },
+  "sort": [
+    {
+      "_index": {
+        "order": "asc"
+      }
+    }
+  ],
+  "script_fields": {
+    "index_name": {
+      "script": "doc['_index']"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "tweet": {
+      "_source": {
+        "enabled": false
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "event": {
+      "_source": {
+        "includes": [
+          "*.count",
+          "meta.*"
+        ],
+        "excludes": [
+          "meta.description",
+          "meta.other.*"
+        ]
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "requests": {
+    "count": 10,
+    "foo": "bar"
+  },
+  "meta": {
+    "name": "Some metric",
+    "description": "Some metric description",
+    "other": {
+      "foo": "one",
+      "baz": "two"
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "match": {
+      "meta.other.foo": "one"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "text": "Document with type 1"
+}
+emit snippet
+body part: 
+{
+  "text": "Document with type 2"
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "terms": {
+      "_type": [ "type_1", "type_2" ]
+    }
+  },
+  "aggs": {
+    "types": {
+      "terms": {
+        "field": "_type",
+        "size": 10
+      }
+    }
+  },
+  "sort": [
+    {
+      "_type": {
+        "order": "desc"
+      }
+    }
+  ],
+  "script_fields": {
+    "type": {
+      "script": "doc['_type']"
+    }
+  }
+}
+handle snippet
+null
+emit snippet
+query part: routing=user1
+body part: 
+{
+  "title": "This is a document"
+}
+emit snippet
+query part: routing=user1
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "terms": {
+      "_routing": [ "user1" ]
+    }
+  },
+  "script_fields": {
+    "Routing value": {
+      "script": "doc['_routing']"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+query part: routing=user1,user2
+body part: 
+{
+  "query": {
+    "match": {
+      "title": "document"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "_routing": {
+        "required": true
+      }
+    }
+  }
+}
+emit snippet
+catch part: request
+body part: 
+{
+  "text": "No routing value provided"
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "_ttl": {
+        "enabled": true
+      }
+    }
+  }
+}
+emit snippet
+query part: ttl=10m
+body part: 
+{
+  "text": "Will expire in 10 minutes"
+}
+emit snippet
+body part: 
+{
+  "text": "Will not expire"
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_type": {
+      "_ttl": {
+        "enabled": true,
+        "default": "5m"
+      }
+    }
+  }
+}
+emit snippet
+query part: ttl=10m
+body part: 
+{
+  "text": "Will expire in 10 minutes"
+}
+emit snippet
+body part: 
+{
+  "text": "Will expire in 5 minutes"
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_parent": {},
+    "my_child": {
+      "_parent": {
+        "type": "my_parent"
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "text": "This is a parent document"
+}
+emit snippet
+query part: parent=1
+body part: 
+{
+  "text": "This is a child document"
+}
+emit snippet
+query part: parent=1
+body part: 
+{
+  "text": "This is another child document"
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "has_child": {
+      "type": "my_child",
+      "query": {
+        "match": {
+          "text": "child document"
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "query": {
+    "terms": {
+      "_parent": [ "1" ]
+    }
+  },
+  "aggs": {
+    "parents": {
+      "terms": {
+        "field": "_parent",
+        "size": 10
+      }
+    }
+  },
+  "script_fields": {
+    "parent": {
+      "script": "doc['_parent']"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "my_parent": {},
+    "my_child": {
+      "_parent": {
+        "type": "my_parent",
+        "eager_global_ordinals": true
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+query part: human&fields=_parent
+emit snippet
+query part: human&fields=_parent
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "text": "Document with ID 1"
+}
+emit snippet
+body part: 
+{
+  "text": "Document with ID 2"
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "terms": {
+      "_id": [ "1", "2" ]
+    }
+  },
+  "script_fields": {
+    "UID": {
+      "script": "doc['_id']"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "text": "Document with ID 1"
+}
+emit snippet
+body part: 
+{
+  "text": "Document with ID 2"
+}
+emit snippet
+body part: 
+{
+  "query": {
+    "terms": {
+      "_uid": [ "my_type#1", "my_type#2" ]
+    }
+  },
+  "aggs": {
+    "UIDs": {
+      "terms": {
+        "field": "_uid",
+        "size": 10
+      }
+    }
+  },
+  "sort": [
+    {
+      "_uid": {
+        "order": "desc"
+      }
+    }
+  ],
+  "script_fields": {
+    "UID": {
+      "script": "doc['_uid']"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "user": {
+      "_meta": {
+        "class": "MyApp::User",
+        "version": {
+          "min": "1.0",
+          "max": "1.3"
+        }
+      }
+    }
+  }
+}
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+emit snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "index.priority": 10
+  }
+}
+emit snippet
+body part: 
+{
+  "settings": {
+    "index.priority": 5
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "index.priority": 1
+}
+handle snippet
+test snippet
+emit snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "index.unassigned.node_left.delayed_timeout": "5m"
+  }
+}
+handle snippet
+test snippet
+emit snippet
+handle snippet
+test snippet
+emit snippet
+emit snippet
+body part: 
+{
+  "settings": {
+    "index.unassigned.node_left.delayed_timeout": "0"
+  }
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+emit snippet
+body part: 
+{
+  "index.routing.allocation.include.size": "big,medium"
+}
+handle snippet
+test snippet
+emit snippet
+emit snippet
+body part: 
+{
+  "index.routing.allocation.exclude.size": "small"
+}
+handle snippet
+test snippet
+emit snippet
+emit snippet
+body part: 
+{
+  "index.routing.allocation.include.size": "big",
+  "index.routing.allocation.include.rack": "rack1"
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "index.routing.allocation.include._ip": "192.168.2.*"
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "tokenizer" : "standard",
+  "token_filter" : ["snowball"],
+  "text" : "detailed output",
+  "explain" : true,
+  "attributes" : ["keyword"]
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "template": "te*",
+  "settings": {
+    "number_of_shards": 1
+  },
+  "mappings": {
+    "type1": {
+      "_source": {
+        "enabled": false
+      },
+      "properties": {
+        "host_name": {
+          "type": "keyword"
+        },
+        "created_at": {
+          "type": "date",
+          "format": "EEE MMM dd HH:mm:ss Z YYYY"
+        }
+      }
+    }
+  }
+}
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "tweet": {
+      "properties": {
+        "message": {
+          "type": "text"
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "properties": {
+    "name": {
+      "type": "text"
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "properties": {
+    "user_name": {
+      "type": "text"
+    }
+  }
+}
+handle snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "user": {
+      "properties": {
+        "name": {
+          "properties": {
+            "first": {
+              "type": "text"
+            }
+          }
+        },
+        "user_id": {
+          "type": "keyword"
+        }
+      }
+    }
+  }
+}
+emit snippet
+body part: 
+{
+  "properties": {
+    "name": {
+      "properties": {
+        "last": {
+          "type": "text"
+        }
+      }
+    },
+    "user_id": {
+      "type": "keyword",
+      "ignore_above": 100
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "type_one": {
+      "properties": {
+        "text": {
+          "type": "text",
+          "analyzer": "standard"
+        }
+      }
+    },
+    "type_two": {
+      "properties": {
+        "text": {
+          "type": "text",
+          "analyzer": "standard"
+        }
+      }
+    }
+  }
+}
+emit snippet
+catch part: request
+body part: 
+{
+  "properties": {
+    "text": {
+      "type": "text",
+      "analyzer": "standard",
+      "search_analyzer": "whitespace"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+query part: update_all_types
+body part: 
+{
+  "properties": {
+    "text": {
+      "type": "text",
+      "analyzer": "standard",
+      "search_analyzer": "whitespace"
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+emit snippet
+body part: 
+{
+    "actions" : [
+        { "add" : { "index" : "test1", "alias" : "alias1" } }
+    ]
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "actions" : [
+        { "remove" : { "index" : "test1", "alias" : "alias1" } }
+    ]
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "actions" : [
+        { "remove" : { "index" : "test1", "alias" : "alias1" } },
+        { "add" : { "index" : "test1", "alias" : "alias2" } }
+    ]
+}
+handle snippet
+test snippet
+emit snippet
+emit snippet
+emit snippet
+body part: 
+{
+    "actions" : [
+        { "add" : { "index" : "test1", "alias" : "alias1" } },
+        { "add" : { "index" : "test2", "alias" : "alias1" } }
+    ]
+}
+handle snippet
+test snippet
+emit snippet
+emit snippet
+emit snippet
+body part: 
+{
+    "actions" : [
+        { "add" : { "indices" : ["test1", "test2"], "alias" : "alias1" } }
+    ]
+}
+handle snippet
+test snippet
+emit snippet
+emit snippet
+emit snippet
+body part: 
+{
+    "actions" : [
+        { "add" : { "index" : "test*", "alias" : "all_test_indices" } }
+    ]
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+  "mappings": {
+    "type1": {
+      "properties": {
+        "user" : {
+          "type": "keyword"
+        }
+      }
+    }
+  }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "actions" : [
+        {
+            "add" : {
+                 "index" : "test1",
+                 "alias" : "alias2",
+                 "filter" : { "term" : { "user" : "kimchy" } }
+            }
+        }
+    ]
+}
+handle snippet
+test snippet
+emit snippet
+emit snippet
+body part: 
+{
+    "actions" : [
+        {
+            "add" : {
+                 "index" : "test",
+                 "alias" : "alias1",
+                 "routing" : "1"
+            }
+        }
+    ]
+}
+handle snippet
+test snippet
+emit snippet
+emit snippet
+body part: 
+{
+    "actions" : [
+        {
+            "add" : {
+                 "index" : "test",
+                 "alias" : "alias2",
+                 "search_routing" : "1,2",
+                 "index_routing" : "2"
+            }
+        }
+    ]
+}
+handle snippet
+test snippet
+emit snippet
+query part: q=user:kimchy&routing=2,3
+handle snippet
+test snippet
+emit snippet
+emit snippet
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "mappings" : {
+        "user" : {
+            "properties" : {
+                "user_id" : {"type" : "integer"}
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "routing" : "12",
+    "filter" : {
+        "term" : {
+            "user_id" : 12
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+body part: 
+{
+    "mappings" : {
+        "type" : {
+            "properties" : {
+                "year" : {"type" : "integer"}
+            }
+        }
+    },
+    "aliases" : {
+        "current_day" : {},
+        "2016" : {
+            "filter" : {
+                "term" : {"year" : 2016 }
+            }
+        }
+    }
+}
+handle snippet
+test snippet
+emit snippet
+handle snippet
+test snippet
+emit snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+emit snippet
+emit snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+emit snippet
+handle snippet
+test snippet
+emit snippet
+emit snippet
+emit snippet
+emit snippet
+handle snippet
+test snippet
+emit snippet
+emit snippet
+query part: level=shards
+handle snippet
+handle snippet
+test snippet
+
+  - do:
+      bulk:
+        index: twitter
+        type: tweet
+        refresh: true
+        body: |
+          {"index":{"_id": "0"}}
+          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
+          {"index":{"_id": "1"}}
+          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
+          {"index":{"_id": "2"}}
+          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
+          {"index":{"_id": "3"}}
+          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
+          {"index":{"_id": "4"}}
+          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
+emit snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+test snippet
+emit snippet
+emit snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+handle snippet
+:core:compileJava UP-TO-DATE
+:core:generateModulesList UP-TO-DATE
+:core:generatePluginsList UP-TO-DATE
+:core:processResources UP-TO-DATE
+:core:classes UP-TO-DATE
+:core:jar UP-TO-DATE
+:test:framework:compileJava UP-TO-DATE
+:test:framework:processResources UP-TO-DATE
+:test:framework:classes UP-TO-DATE
+:test:framework:jar UP-TO-DATE
+:docs:compileTestJava
+:docs:testClasses
+:docs:integTest#prepareCluster.cleanShared
+:docs:integTest#clean
+:docs:integTest#checkPrevious SKIPPED
+:docs:integTest#stopPrevious SKIPPED
+:modules:aggs-matrix-stats:compileJava UP-TO-DATE
+:modules:aggs-matrix-stats:processResources UP-TO-DATE
+:modules:aggs-matrix-stats:classes UP-TO-DATE
+:modules:aggs-matrix-stats:jar UP-TO-DATE
+:modules:aggs-matrix-stats:copyPluginPropertiesTemplate
+:modules:aggs-matrix-stats:pluginProperties UP-TO-DATE
+:modules:aggs-matrix-stats:bundlePlugin UP-TO-DATE
+:modules:ingest-grok:compileJava UP-TO-DATE
+:modules:ingest-grok:processResources UP-TO-DATE
+:modules:ingest-grok:classes UP-TO-DATE
+:modules:ingest-grok:jar UP-TO-DATE
+:modules:ingest-grok:copyPluginPropertiesTemplate
+:modules:ingest-grok:pluginProperties UP-TO-DATE
+:modules:ingest-grok:bundlePlugin UP-TO-DATE
+:modules:lang-expression:compileJava UP-TO-DATE
+:modules:lang-expression:processResources UP-TO-DATE
+:modules:lang-expression:classes UP-TO-DATE
+:modules:lang-expression:jar UP-TO-DATE
+:modules:lang-expression:copyPluginPropertiesTemplate
+:modules:lang-expression:pluginProperties UP-TO-DATE
+:modules:lang-expression:bundlePlugin UP-TO-DATE
+:modules:lang-groovy:compileJava UP-TO-DATE
+:modules:lang-groovy:processResources UP-TO-DATE
+:modules:lang-groovy:classes UP-TO-DATE
+:modules:lang-groovy:jar UP-TO-DATE
+:modules:lang-groovy:copyPluginPropertiesTemplate
+:modules:lang-groovy:pluginProperties UP-TO-DATE
+:modules:lang-groovy:bundlePlugin UP-TO-DATE
+:modules:lang-mustache:compileJava UP-TO-DATE
+:modules:lang-mustache:processResources UP-TO-DATE
+:modules:lang-mustache:classes UP-TO-DATE
+:modules:lang-mustache:jar UP-TO-DATE
+:modules:lang-mustache:copyPluginPropertiesTemplate
+:modules:lang-mustache:pluginProperties UP-TO-DATE
+:modules:lang-mustache:bundlePlugin UP-TO-DATE
+:modules:lang-painless:compileJava UP-TO-DATE
+:modules:lang-painless:processResources UP-TO-DATE
+:modules:lang-painless:classes UP-TO-DATE
+:modules:lang-painless:jar UP-TO-DATE
+:modules:lang-painless:copyPluginPropertiesTemplate
+:modules:lang-painless:pluginProperties UP-TO-DATE
+:modules:lang-painless:bundlePlugin UP-TO-DATE
+:modules:percolator:compileJava UP-TO-DATE
+:modules:percolator:processResources UP-TO-DATE
+:modules:percolator:classes UP-TO-DATE
+:modules:percolator:jar UP-TO-DATE
+:modules:percolator:copyPluginPropertiesTemplate
+:modules:percolator:pluginProperties UP-TO-DATE
+:modules:percolator:bundlePlugin UP-TO-DATE
+:modules:reindex:compileJava UP-TO-DATE
+:modules:reindex:processResources UP-TO-DATE
+:modules:reindex:classes UP-TO-DATE
+:modules:reindex:jar UP-TO-DATE
+:modules:reindex:copyPluginPropertiesTemplate
+:modules:reindex:pluginProperties UP-TO-DATE
+:modules:reindex:bundlePlugin UP-TO-DATE
+:distribution:buildModules UP-TO-DATE
+:distribution:zip:buildZip UP-TO-DATE
+:docs:integTest#extract
+:docs:integTest#configure
+:docs:integTest#start
+:docs:integTest#wait
+:docs:integTest
+   [junit4] <JUnit4> says ahoj! Master seed: E40438BBAF23DBE2
+==> Test Info: seed=E40438BBAF23DBE2; jvm=1; suite=1
+Suite: org.elasticsearch.smoketest.SmokeTestDocsIT
+==> Test Summary: 1 suite (1 ignored), 0 tests
+   [junit4] JVM J0:     0.30 ..     1.64 =     1.35s
+   [junit4] Execution time total: 1.66 sec.
+   [junit4] Tests summary: 1 suite (1 ignored), 0 tests
+:docs:integTest FAILED
+:docs:integTest#stop
+
+BUILD FAILED
+
+Total time: 22.834 secs
+
+BUILD SUCCESSFUL
+Total time: 1 second
diff --git a/docs/reference/query-dsl/nested-query.asciidoc b/docs/reference/query-dsl/nested-query.asciidoc
index 6e990e07f91..176473174dd 100644
--- a/docs/reference/query-dsl/nested-query.asciidoc
+++ b/docs/reference/query-dsl/nested-query.asciidoc
@@ -23,7 +23,6 @@ PUT /my_index
     }
 }
 
-GET _cluster/health?wait_for_status=yellow
 --------------------------------------------------
 // CONSOLE
 // TESTSETUP
diff --git a/docs/reference/query-dsl/parent-id-query.asciidoc b/docs/reference/query-dsl/parent-id-query.asciidoc
index 8ea07a6d0b7..a7a28cf88e8 100644
--- a/docs/reference/query-dsl/parent-id-query.asciidoc
+++ b/docs/reference/query-dsl/parent-id-query.asciidoc
@@ -29,7 +29,6 @@ PUT /my_index
     }
 }
 
-GET /_cluster/health?wait_for_status=yellow
 ------------------------------------------
 // CONSOLE
 // TESTSETUP

From 4fa824f891f5f19597c22557627fcc4bed2de98b Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 3 Jun 2016 16:42:07 +0200
Subject: [PATCH 0630/1311] [TEST] add a lot of forgotten try with resources to
 wrap ElasticsearchResponses

---
 .../http/netty/NettyHttpCompressionIT.java    | 34 ++++++++-------
 .../plugins/ResponseHeaderPluginIT.java       |  9 ++--
 .../org/elasticsearch/rest/CorsNotSetIT.java  | 20 +++++----
 .../org/elasticsearch/rest/CorsRegexIT.java   | 42 ++++++++++---------
 .../rest/action/main/RestMainActionIT.java    | 16 +++----
 .../ContextAndHeaderTransportIT.java          | 19 +++++----
 6 files changed, 78 insertions(+), 62 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java
index 9e995f0a010..32a331d61f7 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java
@@ -62,11 +62,12 @@ public class NettyHttpCompressionIT extends ESIntegTestCase {
         // we need to intercept early, otherwise internal logic in HttpClient will just remove the header and we cannot verify it
         ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor();
         try (RestClient client = restClient(HttpClients.custom().addInterceptorFirst(headerExtractor).build())) {
-            ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null,
-                    new BasicHeader(HttpHeaders.ACCEPT_ENCODING, GZIP_ENCODING));
-            assertEquals(200, response.getStatusLine().getStatusCode());
-            assertTrue(headerExtractor.hasContentEncodingHeader());
-            assertEquals(GZIP_ENCODING, headerExtractor.getContentEncodingHeader().getValue());
+            try (ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null,
+                    new BasicHeader(HttpHeaders.ACCEPT_ENCODING, GZIP_ENCODING))) {
+                assertEquals(200, response.getStatusLine().getStatusCode());
+                assertTrue(headerExtractor.hasContentEncodingHeader());
+                assertEquals(GZIP_ENCODING, headerExtractor.getContentEncodingHeader().getValue());
+            }
         }
     }
 
@@ -75,9 +76,10 @@ public class NettyHttpCompressionIT extends ESIntegTestCase {
         ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor();
         CloseableHttpClient httpClient = HttpClients.custom().disableContentCompression().addInterceptorFirst(headerExtractor).build();
         try (RestClient client = restClient(httpClient)) {
-            ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null);
-            assertEquals(200, response.getStatusLine().getStatusCode());
-            assertFalse(headerExtractor.hasContentEncodingHeader());
+            try (ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null)) {
+                assertEquals(200, response.getStatusLine().getStatusCode());
+                assertFalse(headerExtractor.hasContentEncodingHeader());
+            }
         }
     }
 
@@ -87,9 +89,11 @@ public class NettyHttpCompressionIT extends ESIntegTestCase {
         // this disable content compression in both directions (request and response)
         CloseableHttpClient httpClient = HttpClients.custom().disableContentCompression().addInterceptorFirst(headerExtractor).build();
         try (RestClient client = restClient(httpClient)) {
-            ElasticsearchResponse response = client.performRequest("POST", "/company/employees/1", Collections.emptyMap(), SAMPLE_DOCUMENT);
-            assertEquals(201, response.getStatusLine().getStatusCode());
-            assertFalse(headerExtractor.hasContentEncodingHeader());
+            try (ElasticsearchResponse response = client.performRequest("POST", "/company/employees/1",
+                    Collections.emptyMap(), SAMPLE_DOCUMENT)) {
+                assertEquals(201, response.getStatusLine().getStatusCode());
+                assertFalse(headerExtractor.hasContentEncodingHeader());
+            }
         }
     }
 
@@ -98,9 +102,11 @@ public class NettyHttpCompressionIT extends ESIntegTestCase {
         ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor();
         // we don't call #disableContentCompression() hence the client will send the content compressed
         try (RestClient client = restClient(HttpClients.custom().addInterceptorFirst(headerExtractor).build())) {
-            ElasticsearchResponse response = client.performRequest("POST", "/company/employees/2", Collections.emptyMap(), SAMPLE_DOCUMENT);
-            assertEquals(201, response.getStatusLine().getStatusCode());
-            assertEquals(GZIP_ENCODING, headerExtractor.getContentEncodingHeader().getValue());
+            try (ElasticsearchResponse response = client.performRequest("POST", "/company/employees/2",
+                    Collections.emptyMap(), SAMPLE_DOCUMENT)) {
+                assertEquals(201, response.getStatusLine().getStatusCode());
+                assertEquals(GZIP_ENCODING, headerExtractor.getContentEncodingHeader().getValue());
+            }
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java b/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
index 4ae9e3912b2..4b905d7b1a8 100644
--- a/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
+++ b/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
@@ -66,10 +66,11 @@ public class ResponseHeaderPluginIT extends ESIntegTestCase {
                 assertThat(response.getFirstHeader("Secret"), equalTo("required"));
             }
 
-            ElasticsearchResponse authResponse = client.performRequest("GET", "/_protected", Collections.emptyMap(), null,
-                    new BasicHeader("Secret", "password"));
-            assertThat(authResponse, hasStatus(OK));
-            assertThat(authResponse.getFirstHeader("Secret"), equalTo("granted"));
+            try (ElasticsearchResponse authResponse = client.performRequest("GET", "/_protected", Collections.emptyMap(), null,
+                    new BasicHeader("Secret", "password"))) {
+                assertThat(authResponse, hasStatus(OK));
+                assertThat(authResponse.getFirstHeader("Secret"), equalTo("granted"));
+            }
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java b/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java
index e407bccf872..c6c23da102f 100644
--- a/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java
+++ b/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java
@@ -48,20 +48,22 @@ public class CorsNotSetIT extends ESIntegTestCase {
     public void testCorsSettingDefaultBehaviourDoesNotReturnAnything() throws Exception {
         String corsValue = "http://localhost:9200";
         try (RestClient restClient = restClient()) {
-            ElasticsearchResponse response = restClient.performRequest("GET", "/", Collections.emptyMap(), null,
-                    new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue));
-            assertThat(response.getStatusLine().getStatusCode(), is(200));
-            assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
-            assertThat(response.getFirstHeader("Access-Control-Allow-Credentials"), nullValue());
+            try (ElasticsearchResponse response = restClient.performRequest("GET", "/", Collections.emptyMap(), null,
+                    new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue))) {
+                assertThat(response.getStatusLine().getStatusCode(), is(200));
+                assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
+                assertThat(response.getFirstHeader("Access-Control-Allow-Credentials"), nullValue());
+            }
         }
     }
 
     public void testThatOmittingCorsHeaderDoesNotReturnAnything() throws Exception {
         try (RestClient restClient = restClient()) {
-            ElasticsearchResponse response = restClient.performRequest("GET", "/", Collections.emptyMap(), null);
-            assertThat(response.getStatusLine().getStatusCode(), is(200));
-            assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
-            assertThat(response.getFirstHeader("Access-Control-Allow-Credentials"), nullValue());
+            try (ElasticsearchResponse response = restClient.performRequest("GET", "/", Collections.emptyMap(), null)) {
+                assertThat(response.getStatusLine().getStatusCode(), is(200));
+                assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
+                assertThat(response.getFirstHeader("Access-Control-Allow-Credentials"), nullValue());
+            }
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java b/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java
index 729fed8e96d..49bfd82e613 100644
--- a/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java
+++ b/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java
@@ -63,15 +63,16 @@ public class CorsRegexIT extends ESIntegTestCase {
     public void testThatRegularExpressionWorksOnMatch() throws Exception {
         String corsValue = "http://localhost:9200";
         try (RestClient client = restClient()) {
-            ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null,
-                    new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue));
-            assertResponseWithOriginheader(response, corsValue);
-
+            try (ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null,
+                    new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue))) {
+                assertResponseWithOriginheader(response, corsValue);
+            }
             corsValue = "https://localhost:9200";
-            response = client.performRequest("GET", "/", Collections.emptyMap(), null,
-                    new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue));
-            assertResponseWithOriginheader(response, corsValue);
-            assertThat(response.getFirstHeader("Access-Control-Allow-Credentials"), is("true"));
+            try (ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null,
+                    new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue));) {
+                assertResponseWithOriginheader(response, corsValue);
+                assertThat(response.getFirstHeader("Access-Control-Allow-Credentials"), is("true"));
+            }
         }
     }
 
@@ -90,29 +91,32 @@ public class CorsRegexIT extends ESIntegTestCase {
 
     public void testThatSendingNoOriginHeaderReturnsNoAccessControlHeader() throws Exception {
         try (RestClient client = restClient()) {
-            ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null,
-                    new BasicHeader("User-Agent", "Mozilla Bar"));
-            assertThat(response.getStatusLine().getStatusCode(), is(200));
-            assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
+            try (ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null,
+                    new BasicHeader("User-Agent", "Mozilla Bar"))) {
+                assertThat(response.getStatusLine().getStatusCode(), is(200));
+                assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
+            }
         }
     }
 
     public void testThatRegularExpressionIsNotAppliedWithoutCorrectBrowserOnMatch() throws Exception {
         try (RestClient client = restClient()) {
-            ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null);
-            assertThat(response.getStatusLine().getStatusCode(), is(200));
-            assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
+            try (ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null)) {
+                assertThat(response.getStatusLine().getStatusCode(), is(200));
+                assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
+            }
         }
     }
 
     public void testThatPreFlightRequestWorksOnMatch() throws Exception {
         String corsValue = "http://localhost:9200";
         try (RestClient client = restClient()) {
-            ElasticsearchResponse response = client.performRequest("OPTIONS", "/", Collections.emptyMap(), null,
+            try (ElasticsearchResponse response = client.performRequest("OPTIONS", "/", Collections.emptyMap(), null,
                     new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue),
-                    new BasicHeader(HttpHeaders.Names.ACCESS_CONTROL_REQUEST_METHOD, "GET"));
-            assertResponseWithOriginheader(response, corsValue);
-            assertNotNull(response.getFirstHeader("Access-Control-Allow-Methods"));
+                    new BasicHeader(HttpHeaders.Names.ACCESS_CONTROL_REQUEST_METHOD, "GET"));) {
+                assertResponseWithOriginheader(response, corsValue);
+                assertNotNull(response.getFirstHeader("Access-Control-Allow-Methods"));
+            }
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java b/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java
index 9c2b0284ef7..0723a9573b4 100644
--- a/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java
+++ b/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java
@@ -42,18 +42,20 @@ public class RestMainActionIT extends ESIntegTestCase {
 
     public void testHeadRequest() throws IOException {
         try (RestClient client = restClient()) {
-            ElasticsearchResponse response = client.performRequest("HEAD", "/", Collections.emptyMap(), null);
-            assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
-            assertNull(response.getEntity());
+            try (ElasticsearchResponse response = client.performRequest("HEAD", "/", Collections.emptyMap(), null)) {
+                assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
+                assertNull(response.getEntity());
+            }
         }
     }
 
     public void testGetRequest() throws IOException {
         try (RestClient client = restClient()) {
-            ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null);
-            assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
-            assertNotNull(response.getEntity());
-            assertThat(EntityUtils.toString(response.getEntity()), containsString("cluster_name"));
+            try (ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null)) {
+                assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
+                assertNotNull(response.getEntity());
+                assertThat(EntityUtils.toString(response.getEntity()), containsString("cluster_name"));
+            }
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java b/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
index aae15dc778a..e2139d73218 100644
--- a/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
+++ b/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
@@ -221,15 +221,16 @@ public class ContextAndHeaderTransportIT extends ESIntegTestCase {
         }
 
         try (RestClient client = restClient()) {
-            ElasticsearchResponse response = client.performRequest("GET", "/" + queryIndex + "/_search", Collections.emptyMap(), null,
-                    new BasicHeader(randomHeaderKey, randomHeaderValue), new BasicHeader(relevantHeaderName, randomHeaderValue));
-            assertThat(response, hasStatus(OK));
-            List<RequestAndHeaders> searchRequests = getRequests(SearchRequest.class);
-            assertThat(searchRequests, hasSize(greaterThan(0)));
-            for (RequestAndHeaders requestAndHeaders : searchRequests) {
-                assertThat(requestAndHeaders.headers.containsKey(relevantHeaderName), is(true));
-                // was not specified, thus is not included
-                assertThat(requestAndHeaders.headers.containsKey(randomHeaderKey), is(false));
+            try (ElasticsearchResponse response = client.performRequest("GET", "/" + queryIndex + "/_search", Collections.emptyMap(), null,
+                    new BasicHeader(randomHeaderKey, randomHeaderValue), new BasicHeader(relevantHeaderName, randomHeaderValue))) {
+                assertThat(response, hasStatus(OK));
+                List<RequestAndHeaders> searchRequests = getRequests(SearchRequest.class);
+                assertThat(searchRequests, hasSize(greaterThan(0)));
+                for (RequestAndHeaders requestAndHeaders : searchRequests) {
+                    assertThat(requestAndHeaders.headers.containsKey(relevantHeaderName), is(true));
+                    // was not specified, thus is not included
+                    assertThat(requestAndHeaders.headers.containsKey(randomHeaderKey), is(false));
+                }
             }
         }
     }

From f5825b86e631905f1108f72ae9466f2bc42119b9 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 3 Jun 2016 16:56:29 +0200
Subject: [PATCH 0631/1311] [TEST] restore needed LogManager.reset to prevent
 logging to sysout

---
 .../org/elasticsearch/client/sniff/SnifferBuilderTests.java  | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
index ff3cad34bfa..868c5788120 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
@@ -26,9 +26,14 @@ import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.client.RestClient;
 
 import java.util.Arrays;
+import java.util.logging.LogManager;
 
 public class SnifferBuilderTests extends LuceneTestCase {
 
+    static {
+        LogManager.getLogManager().reset();
+    }
+
     public void testBuild() throws Exception {
 
         try {

From 23a94bb9745aeba8bc8c04c02c874deaf89ae621 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 3 Jun 2016 18:00:54 +0200
Subject: [PATCH 0632/1311] [TEST] create standard RestClient at first request
 and reuse it

A RestClient instance is now created whenever EsIntegTestCase#getRestClient is invoked for the first time. It is then kept until the cluster is cleared (depending on the cluster scope of the test).

Renamed other two restClient methods to createRestClient, as that instance needs to be closed and managed in the tests.
---
 .../http/netty/NettyHttpCompressionIT.java    |  8 +--
 .../DetailedErrorsDisabledIT.java             |  5 +-
 .../DetailedErrorsEnabledIT.java              | 37 ++++++------
 .../plugins/ResponseHeaderPluginIT.java       | 27 ++++-----
 .../org/elasticsearch/rest/CorsNotSetIT.java  | 23 +++-----
 .../org/elasticsearch/rest/CorsRegexIT.java   | 59 ++++++++-----------
 .../rest/action/main/RestMainActionIT.java    | 19 +++---
 .../ContextAndHeaderTransportIT.java          | 21 +++----
 .../elasticsearch/test/ESIntegTestCase.java   | 25 ++++++--
 9 files changed, 104 insertions(+), 120 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java
index 32a331d61f7..fb616a87810 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java
@@ -61,7 +61,7 @@ public class NettyHttpCompressionIT extends ESIntegTestCase {
         ensureGreen();
         // we need to intercept early, otherwise internal logic in HttpClient will just remove the header and we cannot verify it
         ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor();
-        try (RestClient client = restClient(HttpClients.custom().addInterceptorFirst(headerExtractor).build())) {
+        try (RestClient client = createRestClient(HttpClients.custom().addInterceptorFirst(headerExtractor).build())) {
             try (ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null,
                     new BasicHeader(HttpHeaders.ACCEPT_ENCODING, GZIP_ENCODING))) {
                 assertEquals(200, response.getStatusLine().getStatusCode());
@@ -75,7 +75,7 @@ public class NettyHttpCompressionIT extends ESIntegTestCase {
         ensureGreen();
         ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor();
         CloseableHttpClient httpClient = HttpClients.custom().disableContentCompression().addInterceptorFirst(headerExtractor).build();
-        try (RestClient client = restClient(httpClient)) {
+        try (RestClient client = createRestClient(httpClient)) {
             try (ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null)) {
                 assertEquals(200, response.getStatusLine().getStatusCode());
                 assertFalse(headerExtractor.hasContentEncodingHeader());
@@ -88,7 +88,7 @@ public class NettyHttpCompressionIT extends ESIntegTestCase {
         ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor();
         // this disable content compression in both directions (request and response)
         CloseableHttpClient httpClient = HttpClients.custom().disableContentCompression().addInterceptorFirst(headerExtractor).build();
-        try (RestClient client = restClient(httpClient)) {
+        try (RestClient client = createRestClient(httpClient)) {
             try (ElasticsearchResponse response = client.performRequest("POST", "/company/employees/1",
                     Collections.emptyMap(), SAMPLE_DOCUMENT)) {
                 assertEquals(201, response.getStatusLine().getStatusCode());
@@ -101,7 +101,7 @@ public class NettyHttpCompressionIT extends ESIntegTestCase {
         ensureGreen();
         ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor();
         // we don't call #disableContentCompression() hence the client will send the content compressed
-        try (RestClient client = restClient(HttpClients.custom().addInterceptorFirst(headerExtractor).build())) {
+        try (RestClient client = createRestClient(HttpClients.custom().addInterceptorFirst(headerExtractor).build())) {
             try (ElasticsearchResponse response = client.performRequest("POST", "/company/employees/2",
                     Collections.emptyMap(), SAMPLE_DOCUMENT)) {
                 assertEquals(201, response.getStatusLine().getStatusCode());
diff --git a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java
index ba8840bbc2c..c23b6fc1239 100644
--- a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java
+++ b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java
@@ -21,7 +21,6 @@ package org.elasticsearch.options.detailederrors;
 
 import org.elasticsearch.client.ElasticsearchResponse;
 import org.elasticsearch.client.ElasticsearchResponseException;
-import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.http.HttpTransportSettings;
@@ -49,8 +48,8 @@ public class DetailedErrorsDisabledIT extends ESIntegTestCase {
     }
 
     public void testThatErrorTraceParamReturns400() throws Exception {
-        try (RestClient restClient = restClient()) {
-            restClient.performRequest("DELETE", "/", Collections.singletonMap("error_trace", "true"), null);
+        try {
+            getRestClient().performRequest("DELETE", "/", Collections.singletonMap("error_trace", "true"), null);
             fail("request should have failed");
         } catch(ElasticsearchResponseException e) {
             ElasticsearchResponse response = e.getElasticsearchResponse();
diff --git a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java
index a040c31299d..df1cb5a6308 100644
--- a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java
+++ b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java
@@ -21,7 +21,6 @@ package org.elasticsearch.options.detailederrors;
 
 import org.elasticsearch.client.ElasticsearchResponse;
 import org.elasticsearch.client.ElasticsearchResponseException;
-import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -47,26 +46,24 @@ public class DetailedErrorsEnabledIT extends ESIntegTestCase {
     }
 
     public void testThatErrorTraceWorksByDefault() throws Exception {
-        try (RestClient restClient = restClient()) {
-            try {
-                restClient.performRequest("DELETE", "/", Collections.singletonMap("error_trace", "true"), null);
-                fail("request should have failed");
-            } catch(ElasticsearchResponseException e) {
-                ElasticsearchResponse response = e.getElasticsearchResponse();
-                assertThat(response.getFirstHeader("Content-Type"), containsString("application/json"));
-                assertThat(e.getResponseBody(), containsString("\"stack_trace\":\"[Validation Failed: 1: index / indices is missing;]; " +
-                        "nested: ActionRequestValidationException[Validation Failed: 1:"));
-            }
+        try {
+            getRestClient().performRequest("DELETE", "/", Collections.singletonMap("error_trace", "true"), null);
+            fail("request should have failed");
+        } catch(ElasticsearchResponseException e) {
+            ElasticsearchResponse response = e.getElasticsearchResponse();
+            assertThat(response.getFirstHeader("Content-Type"), containsString("application/json"));
+            assertThat(e.getResponseBody(), containsString("\"stack_trace\":\"[Validation Failed: 1: index / indices is missing;]; " +
+                    "nested: ActionRequestValidationException[Validation Failed: 1:"));
+        }
 
-            try {
-                restClient.performRequest("DELETE", "/", Collections.emptyMap(), null);
-                fail("request should have failed");
-            } catch(ElasticsearchResponseException e) {
-                ElasticsearchResponse response = e.getElasticsearchResponse();
-                assertThat(response.getFirstHeader("Content-Type"), containsString("application/json"));
-                assertThat(e.getResponseBody(), not(containsString("\"stack_trace\":\"[Validation Failed: 1: index / indices is missing;]; "
-                        + "nested: ActionRequestValidationException[Validation Failed: 1:")));
-            }
+        try {
+            getRestClient().performRequest("DELETE", "/", Collections.emptyMap(), null);
+            fail("request should have failed");
+        } catch(ElasticsearchResponseException e) {
+            ElasticsearchResponse response = e.getElasticsearchResponse();
+            assertThat(response.getFirstHeader("Content-Type"), containsString("application/json"));
+            assertThat(e.getResponseBody(), not(containsString("\"stack_trace\":\"[Validation Failed: 1: index / indices is missing;]; "
+                    + "nested: ActionRequestValidationException[Validation Failed: 1:")));
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java b/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
index 4b905d7b1a8..0fd21651ac0 100644
--- a/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
+++ b/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
@@ -21,7 +21,6 @@ package org.elasticsearch.plugins;
 import org.apache.http.message.BasicHeader;
 import org.elasticsearch.client.ElasticsearchResponse;
 import org.elasticsearch.client.ElasticsearchResponseException;
-import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.responseheader.TestResponseHeaderPlugin;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -56,21 +55,19 @@ public class ResponseHeaderPluginIT extends ESIntegTestCase {
 
     public void testThatSettingHeadersWorks() throws Exception {
         ensureGreen();
-        try (RestClient client = restClient()) {
-            try {
-                client.performRequest("GET", "/_protected", Collections.emptyMap(), null);
-                fail("request should have failed");
-            } catch(ElasticsearchResponseException e) {
-                ElasticsearchResponse response = e.getElasticsearchResponse();
-                assertThat(response, hasStatus(UNAUTHORIZED));
-                assertThat(response.getFirstHeader("Secret"), equalTo("required"));
-            }
+        try {
+            getRestClient().performRequest("GET", "/_protected", Collections.emptyMap(), null);
+            fail("request should have failed");
+        } catch(ElasticsearchResponseException e) {
+            ElasticsearchResponse response = e.getElasticsearchResponse();
+            assertThat(response, hasStatus(UNAUTHORIZED));
+            assertThat(response.getFirstHeader("Secret"), equalTo("required"));
+        }
 
-            try (ElasticsearchResponse authResponse = client.performRequest("GET", "/_protected", Collections.emptyMap(), null,
-                    new BasicHeader("Secret", "password"))) {
-                assertThat(authResponse, hasStatus(OK));
-                assertThat(authResponse.getFirstHeader("Secret"), equalTo("granted"));
-            }
+        try (ElasticsearchResponse authResponse = getRestClient().performRequest("GET", "/_protected", Collections.emptyMap(), null,
+                new BasicHeader("Secret", "password"))) {
+            assertThat(authResponse, hasStatus(OK));
+            assertThat(authResponse.getFirstHeader("Secret"), equalTo("granted"));
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java b/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java
index c6c23da102f..0f4ccd1a6cf 100644
--- a/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java
+++ b/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java
@@ -21,7 +21,6 @@ package org.elasticsearch.rest;
 
 import org.apache.http.message.BasicHeader;
 import org.elasticsearch.client.ElasticsearchResponse;
-import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -47,23 +46,19 @@ public class CorsNotSetIT extends ESIntegTestCase {
 
     public void testCorsSettingDefaultBehaviourDoesNotReturnAnything() throws Exception {
         String corsValue = "http://localhost:9200";
-        try (RestClient restClient = restClient()) {
-            try (ElasticsearchResponse response = restClient.performRequest("GET", "/", Collections.emptyMap(), null,
-                    new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue))) {
-                assertThat(response.getStatusLine().getStatusCode(), is(200));
-                assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
-                assertThat(response.getFirstHeader("Access-Control-Allow-Credentials"), nullValue());
-            }
+        try (ElasticsearchResponse response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null,
+                new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue))) {
+            assertThat(response.getStatusLine().getStatusCode(), is(200));
+            assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
+            assertThat(response.getFirstHeader("Access-Control-Allow-Credentials"), nullValue());
         }
     }
 
     public void testThatOmittingCorsHeaderDoesNotReturnAnything() throws Exception {
-        try (RestClient restClient = restClient()) {
-            try (ElasticsearchResponse response = restClient.performRequest("GET", "/", Collections.emptyMap(), null)) {
-                assertThat(response.getStatusLine().getStatusCode(), is(200));
-                assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
-                assertThat(response.getFirstHeader("Access-Control-Allow-Credentials"), nullValue());
-            }
+        try (ElasticsearchResponse response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null)) {
+            assertThat(response.getStatusLine().getStatusCode(), is(200));
+            assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
+            assertThat(response.getFirstHeader("Access-Control-Allow-Credentials"), nullValue());
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java b/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java
index 49bfd82e613..f6012702ce8 100644
--- a/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java
+++ b/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java
@@ -21,7 +21,6 @@ package org.elasticsearch.rest;
 import org.apache.http.message.BasicHeader;
 import org.elasticsearch.client.ElasticsearchResponse;
 import org.elasticsearch.client.ElasticsearchResponseException;
-import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkModule;
@@ -62,23 +61,21 @@ public class CorsRegexIT extends ESIntegTestCase {
 
     public void testThatRegularExpressionWorksOnMatch() throws Exception {
         String corsValue = "http://localhost:9200";
-        try (RestClient client = restClient()) {
-            try (ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null,
-                    new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue))) {
-                assertResponseWithOriginheader(response, corsValue);
-            }
-            corsValue = "https://localhost:9200";
-            try (ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null,
-                    new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue));) {
-                assertResponseWithOriginheader(response, corsValue);
-                assertThat(response.getFirstHeader("Access-Control-Allow-Credentials"), is("true"));
-            }
+        try (ElasticsearchResponse response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null,
+                new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue))) {
+            assertResponseWithOriginheader(response, corsValue);
+        }
+        corsValue = "https://localhost:9200";
+        try (ElasticsearchResponse response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null,
+                new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue));) {
+            assertResponseWithOriginheader(response, corsValue);
+            assertThat(response.getFirstHeader("Access-Control-Allow-Credentials"), is("true"));
         }
     }
 
     public void testThatRegularExpressionReturnsForbiddenOnNonMatch() throws Exception {
-        try (RestClient client = restClient()) {
-            client.performRequest("GET", "/", Collections.emptyMap(), null, new BasicHeader("User-Agent", "Mozilla Bar"),
+        try {
+            getRestClient().performRequest("GET", "/", Collections.emptyMap(), null, new BasicHeader("User-Agent", "Mozilla Bar"),
                     new BasicHeader("Origin", "http://evil-host:9200"));
             fail("request should have failed");
         } catch(ElasticsearchResponseException e) {
@@ -90,39 +87,33 @@ public class CorsRegexIT extends ESIntegTestCase {
     }
 
     public void testThatSendingNoOriginHeaderReturnsNoAccessControlHeader() throws Exception {
-        try (RestClient client = restClient()) {
-            try (ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null,
-                    new BasicHeader("User-Agent", "Mozilla Bar"))) {
-                assertThat(response.getStatusLine().getStatusCode(), is(200));
-                assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
-            }
+        try (ElasticsearchResponse response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null,
+                new BasicHeader("User-Agent", "Mozilla Bar"))) {
+            assertThat(response.getStatusLine().getStatusCode(), is(200));
+            assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
         }
     }
 
     public void testThatRegularExpressionIsNotAppliedWithoutCorrectBrowserOnMatch() throws Exception {
-        try (RestClient client = restClient()) {
-            try (ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null)) {
-                assertThat(response.getStatusLine().getStatusCode(), is(200));
-                assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
-            }
+        try (ElasticsearchResponse response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null)) {
+            assertThat(response.getStatusLine().getStatusCode(), is(200));
+            assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
         }
     }
 
     public void testThatPreFlightRequestWorksOnMatch() throws Exception {
         String corsValue = "http://localhost:9200";
-        try (RestClient client = restClient()) {
-            try (ElasticsearchResponse response = client.performRequest("OPTIONS", "/", Collections.emptyMap(), null,
-                    new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue),
-                    new BasicHeader(HttpHeaders.Names.ACCESS_CONTROL_REQUEST_METHOD, "GET"));) {
-                assertResponseWithOriginheader(response, corsValue);
-                assertNotNull(response.getFirstHeader("Access-Control-Allow-Methods"));
-            }
+        try (ElasticsearchResponse response = getRestClient().performRequest("OPTIONS", "/", Collections.emptyMap(), null,
+                new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue),
+                new BasicHeader(HttpHeaders.Names.ACCESS_CONTROL_REQUEST_METHOD, "GET"));) {
+            assertResponseWithOriginheader(response, corsValue);
+            assertNotNull(response.getFirstHeader("Access-Control-Allow-Methods"));
         }
     }
 
     public void testThatPreFlightRequestReturnsNullOnNonMatch() throws Exception {
-        try (RestClient client = restClient()) {
-            client.performRequest("OPTIONS", "/", Collections.emptyMap(), null, new BasicHeader("User-Agent", "Mozilla Bar"),
+        try {
+            getRestClient().performRequest("OPTIONS", "/", Collections.emptyMap(), null, new BasicHeader("User-Agent", "Mozilla Bar"),
                     new BasicHeader("Origin", "http://evil-host:9200"),
                     new BasicHeader(HttpHeaders.Names.ACCESS_CONTROL_REQUEST_METHOD, "GET"));
             fail("request should have failed");
diff --git a/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java b/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java
index 0723a9573b4..edf9b5b2c06 100644
--- a/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java
+++ b/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java
@@ -20,7 +20,6 @@ package org.elasticsearch.rest.action.main;
 
 import org.apache.http.util.EntityUtils;
 import org.elasticsearch.client.ElasticsearchResponse;
-import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -41,21 +40,17 @@ public class RestMainActionIT extends ESIntegTestCase {
     }
 
     public void testHeadRequest() throws IOException {
-        try (RestClient client = restClient()) {
-            try (ElasticsearchResponse response = client.performRequest("HEAD", "/", Collections.emptyMap(), null)) {
-                assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
-                assertNull(response.getEntity());
-            }
+        try (ElasticsearchResponse response = getRestClient().performRequest("HEAD", "/", Collections.emptyMap(), null)) {
+            assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
+            assertNull(response.getEntity());
         }
     }
 
     public void testGetRequest() throws IOException {
-        try (RestClient client = restClient()) {
-            try (ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null)) {
-                assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
-                assertNotNull(response.getEntity());
-                assertThat(EntityUtils.toString(response.getEntity()), containsString("cluster_name"));
-            }
+        try (ElasticsearchResponse response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null)) {
+            assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
+            assertNotNull(response.getEntity());
+            assertThat(EntityUtils.toString(response.getEntity()), containsString("cluster_name"));
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java b/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
index e2139d73218..ecee90f313b 100644
--- a/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
+++ b/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
@@ -33,7 +33,6 @@ import org.elasticsearch.action.support.ActionFilter;
 import org.elasticsearch.action.termvectors.MultiTermVectorsRequest;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.ElasticsearchResponse;
-import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.inject.Module;
@@ -220,17 +219,15 @@ public class ContextAndHeaderTransportIT extends ESIntegTestCase {
             restController.registerRelevantHeaders(relevantHeaderName);
         }
 
-        try (RestClient client = restClient()) {
-            try (ElasticsearchResponse response = client.performRequest("GET", "/" + queryIndex + "/_search", Collections.emptyMap(), null,
-                    new BasicHeader(randomHeaderKey, randomHeaderValue), new BasicHeader(relevantHeaderName, randomHeaderValue))) {
-                assertThat(response, hasStatus(OK));
-                List<RequestAndHeaders> searchRequests = getRequests(SearchRequest.class);
-                assertThat(searchRequests, hasSize(greaterThan(0)));
-                for (RequestAndHeaders requestAndHeaders : searchRequests) {
-                    assertThat(requestAndHeaders.headers.containsKey(relevantHeaderName), is(true));
-                    // was not specified, thus is not included
-                    assertThat(requestAndHeaders.headers.containsKey(randomHeaderKey), is(false));
-                }
+        try (ElasticsearchResponse response = getRestClient().performRequest("GET", "/" + queryIndex + "/_search", Collections.emptyMap(), null,
+                new BasicHeader(randomHeaderKey, randomHeaderValue), new BasicHeader(relevantHeaderName, randomHeaderValue))) {
+            assertThat(response, hasStatus(OK));
+            List<RequestAndHeaders> searchRequests = getRequests(SearchRequest.class);
+            assertThat(searchRequests, hasSize(greaterThan(0)));
+            for (RequestAndHeaders requestAndHeaders : searchRequests) {
+                assertThat(requestAndHeaders.headers.containsKey(relevantHeaderName), is(true));
+                // was not specified, thus is not included
+                assertThat(requestAndHeaders.headers.containsKey(randomHeaderKey), is(false));
             }
         }
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index 67f2a654928..ab25d25fac0 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -314,6 +314,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
      * By default if no {@link ClusterScope} is configured this will hold a reference to the suite cluster.
      */
     private static TestCluster currentCluster;
+    private static RestClient restClient = null;
 
     private static final double TRANSPORT_CLIENT_RATIO = transportClientRatio();
 
@@ -510,6 +511,10 @@ public abstract class ESIntegTestCase extends ESTestCase {
             IOUtils.close(clusters.values());
             clusters.clear();
         }
+        if (restClient != null) {
+            restClient.close();
+            restClient = null;
+        }
     }
 
     protected final void afterInternal(boolean afterClass) throws Exception {
@@ -2033,15 +2038,24 @@ public abstract class ESIntegTestCase extends ESTestCase {
         return builder.build();
     }
 
-    protected static RestClient restClient() {
-        return restClient(null);
+    /**
+     * Returns an instance of {@link RestClient} pointing to the current test cluster.
+     * Creates a new client if the method is invoked for the first time in the context of the current test scope.
+     * The returned client gets automatically closed when needed, it shouldn't be closed as part of tests otherwise
+     * it cannot be reused by other tests anymore.
+     */
+    protected synchronized static RestClient getRestClient() {
+        if (restClient == null) {
+            restClient = createRestClient(null);
+        }
+        return restClient;
     }
 
-    protected static RestClient restClient(CloseableHttpClient httpClient) {
-        return restClient(httpClient, "http");
+    protected static RestClient createRestClient(CloseableHttpClient httpClient) {
+        return createRestClient(httpClient, "http");
     }
 
-    protected static RestClient restClient(CloseableHttpClient httpClient, String protocol) {
+    protected static RestClient createRestClient(CloseableHttpClient httpClient, String protocol) {
         final NodesInfoResponse nodeInfos = client().admin().cluster().prepareNodesInfo().get();
         final List<NodeInfo> nodes = nodeInfos.getNodes();
         assertFalse(nodeInfos.hasFailures());
@@ -2054,7 +2068,6 @@ public abstract class ESIntegTestCase extends ESTestCase {
                 hosts.add(new HttpHost(NetworkAddress.format(address.getAddress()), address.getPort(), protocol));
             }
         }
-
         RestClient.Builder builder = RestClient.builder().setHosts(hosts.toArray(new HttpHost[hosts.size()]));
         if (httpClient != null) {
             builder.setHttpClient(httpClient);

From c48b7a7a97a629e3272583ff511d9f960d63e4a5 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 3 Jun 2016 18:08:37 +0200
Subject: [PATCH 0633/1311] [TEST] create standard RestClient at first request
 and reuse it

A RestClient instance is now created whenever EsIntegTestCase#getRestClient is invoked for the first time. It is then kept until the cluster is cleared (depending on the cluster scope of the test).

Renamed other two restClient methods to createRestClient, as that instance needs to be closed and managed in the tests.
---
 .../elasticsearch/transport/ContextAndHeaderTransportIT.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java b/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
index ecee90f313b..6dd6114ce79 100644
--- a/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
+++ b/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
@@ -219,7 +219,8 @@ public class ContextAndHeaderTransportIT extends ESIntegTestCase {
             restController.registerRelevantHeaders(relevantHeaderName);
         }
 
-        try (ElasticsearchResponse response = getRestClient().performRequest("GET", "/" + queryIndex + "/_search", Collections.emptyMap(), null,
+        try (ElasticsearchResponse response = getRestClient().performRequest(
+                "GET", "/" + queryIndex + "/_search", Collections.emptyMap(), null,
                 new BasicHeader(randomHeaderKey, randomHeaderValue), new BasicHeader(relevantHeaderName, randomHeaderValue))) {
             assertThat(response, hasStatus(OK));
             List<RequestAndHeaders> searchRequests = getRequests(SearchRequest.class);

From 29eee328fe8b7baab0f7c68c89a6c92c915dec41 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 3 Jun 2016 18:23:52 +0200
Subject: [PATCH 0634/1311] [TEST] expand RequestLoggerTests to all the
 supported http methods

---
 .../elasticsearch/client/RequestLoggerTests.java   | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/client/src/test/java/org/elasticsearch/client/RequestLoggerTests.java b/client/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
index 89fa30c7e08..64bddf88af2 100644
--- a/client/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
@@ -24,9 +24,12 @@ import org.apache.http.HttpEntityEnclosingRequest;
 import org.apache.http.HttpHost;
 import org.apache.http.ProtocolVersion;
 import org.apache.http.client.methods.HttpHead;
+import org.apache.http.client.methods.HttpOptions;
+import org.apache.http.client.methods.HttpPatch;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.methods.HttpTrace;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.message.BasicHttpResponse;
 import org.apache.http.message.BasicStatusLine;
@@ -46,7 +49,7 @@ public class RequestLoggerTests extends LuceneTestCase {
         URI uri = new URI("/index/type/_api");
 
         HttpRequestBase request;
-        int requestType = RandomInts.randomIntBetween(random(), 0, 4);
+        int requestType = RandomInts.randomIntBetween(random(), 0, 7);
         switch(requestType) {
             case 0:
                 request = new HttpGetWithEntity(uri);
@@ -63,6 +66,15 @@ public class RequestLoggerTests extends LuceneTestCase {
             case 4:
                 request = new HttpHead(uri);
                 break;
+            case 5:
+                request = new HttpTrace(uri);
+                break;
+            case 6:
+                request = new HttpOptions(uri);
+                break;
+            case 7:
+                request = new HttpPatch(uri);
+                break;
             default:
                 throw new UnsupportedOperationException();
         }

From f17f0f9247360276652a76098daa3de8de8ed1e5 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 3 Jun 2016 18:28:31 +0200
Subject: [PATCH 0635/1311] rename ElasticsearchResponse#getFirstHeader to
 getHeader

---
 .../client/ElasticsearchResponse.java            |  2 +-
 .../detailederrors/DetailedErrorsDisabledIT.java |  2 +-
 .../detailederrors/DetailedErrorsEnabledIT.java  |  4 ++--
 .../plugins/ResponseHeaderPluginIT.java          |  4 ++--
 .../org/elasticsearch/rest/CorsNotSetIT.java     |  8 ++++----
 .../java/org/elasticsearch/rest/CorsRegexIT.java | 16 ++++++++--------
 .../test/rest/client/RestTestResponse.java       |  2 +-
 7 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponse.java b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponse.java
index 2d25851b969..037f7412f66 100644
--- a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponse.java
+++ b/client/src/main/java/org/elasticsearch/client/ElasticsearchResponse.java
@@ -83,7 +83,7 @@ public class ElasticsearchResponse implements Closeable {
      * If there is more than one matching header in the message the first element is returned.
      * If there is no matching header in the message <code>null</code> is returned.
      */
-    public String getFirstHeader(String name) {
+    public String getHeader(String name) {
         Header header = response.getFirstHeader(name);
         if (header == null) {
             return null;
diff --git a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java
index c23b6fc1239..fb38f774c3d 100644
--- a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java
+++ b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java
@@ -53,7 +53,7 @@ public class DetailedErrorsDisabledIT extends ESIntegTestCase {
             fail("request should have failed");
         } catch(ElasticsearchResponseException e) {
             ElasticsearchResponse response = e.getElasticsearchResponse();
-            assertThat(response.getFirstHeader("Content-Type"), is("application/json"));
+            assertThat(response.getHeader("Content-Type"), is("application/json"));
             assertThat(e.getResponseBody(), is("{\"error\":\"error traces in responses are disabled.\"}"));
             assertThat(response.getStatusLine().getStatusCode(), is(400));
         }
diff --git a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java
index df1cb5a6308..39aeb00d05d 100644
--- a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java
+++ b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java
@@ -51,7 +51,7 @@ public class DetailedErrorsEnabledIT extends ESIntegTestCase {
             fail("request should have failed");
         } catch(ElasticsearchResponseException e) {
             ElasticsearchResponse response = e.getElasticsearchResponse();
-            assertThat(response.getFirstHeader("Content-Type"), containsString("application/json"));
+            assertThat(response.getHeader("Content-Type"), containsString("application/json"));
             assertThat(e.getResponseBody(), containsString("\"stack_trace\":\"[Validation Failed: 1: index / indices is missing;]; " +
                     "nested: ActionRequestValidationException[Validation Failed: 1:"));
         }
@@ -61,7 +61,7 @@ public class DetailedErrorsEnabledIT extends ESIntegTestCase {
             fail("request should have failed");
         } catch(ElasticsearchResponseException e) {
             ElasticsearchResponse response = e.getElasticsearchResponse();
-            assertThat(response.getFirstHeader("Content-Type"), containsString("application/json"));
+            assertThat(response.getHeader("Content-Type"), containsString("application/json"));
             assertThat(e.getResponseBody(), not(containsString("\"stack_trace\":\"[Validation Failed: 1: index / indices is missing;]; "
                     + "nested: ActionRequestValidationException[Validation Failed: 1:")));
         }
diff --git a/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java b/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
index 0fd21651ac0..2f95fe20232 100644
--- a/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
+++ b/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
@@ -61,13 +61,13 @@ public class ResponseHeaderPluginIT extends ESIntegTestCase {
         } catch(ElasticsearchResponseException e) {
             ElasticsearchResponse response = e.getElasticsearchResponse();
             assertThat(response, hasStatus(UNAUTHORIZED));
-            assertThat(response.getFirstHeader("Secret"), equalTo("required"));
+            assertThat(response.getHeader("Secret"), equalTo("required"));
         }
 
         try (ElasticsearchResponse authResponse = getRestClient().performRequest("GET", "/_protected", Collections.emptyMap(), null,
                 new BasicHeader("Secret", "password"))) {
             assertThat(authResponse, hasStatus(OK));
-            assertThat(authResponse.getFirstHeader("Secret"), equalTo("granted"));
+            assertThat(authResponse.getHeader("Secret"), equalTo("granted"));
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java b/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java
index 0f4ccd1a6cf..2a38f3bcd3a 100644
--- a/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java
+++ b/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java
@@ -49,16 +49,16 @@ public class CorsNotSetIT extends ESIntegTestCase {
         try (ElasticsearchResponse response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null,
                 new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue))) {
             assertThat(response.getStatusLine().getStatusCode(), is(200));
-            assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
-            assertThat(response.getFirstHeader("Access-Control-Allow-Credentials"), nullValue());
+            assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue());
+            assertThat(response.getHeader("Access-Control-Allow-Credentials"), nullValue());
         }
     }
 
     public void testThatOmittingCorsHeaderDoesNotReturnAnything() throws Exception {
         try (ElasticsearchResponse response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null)) {
             assertThat(response.getStatusLine().getStatusCode(), is(200));
-            assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
-            assertThat(response.getFirstHeader("Access-Control-Allow-Credentials"), nullValue());
+            assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue());
+            assertThat(response.getHeader("Access-Control-Allow-Credentials"), nullValue());
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java b/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java
index f6012702ce8..4da0b675d02 100644
--- a/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java
+++ b/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java
@@ -69,7 +69,7 @@ public class CorsRegexIT extends ESIntegTestCase {
         try (ElasticsearchResponse response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null,
                 new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue));) {
             assertResponseWithOriginheader(response, corsValue);
-            assertThat(response.getFirstHeader("Access-Control-Allow-Credentials"), is("true"));
+            assertThat(response.getHeader("Access-Control-Allow-Credentials"), is("true"));
         }
     }
 
@@ -82,7 +82,7 @@ public class CorsRegexIT extends ESIntegTestCase {
             ElasticsearchResponse response = e.getElasticsearchResponse();
             // a rejected origin gets a FORBIDDEN - 403
             assertThat(response.getStatusLine().getStatusCode(), is(403));
-            assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
+            assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue());
         }
     }
 
@@ -90,14 +90,14 @@ public class CorsRegexIT extends ESIntegTestCase {
         try (ElasticsearchResponse response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null,
                 new BasicHeader("User-Agent", "Mozilla Bar"))) {
             assertThat(response.getStatusLine().getStatusCode(), is(200));
-            assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
+            assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue());
         }
     }
 
     public void testThatRegularExpressionIsNotAppliedWithoutCorrectBrowserOnMatch() throws Exception {
         try (ElasticsearchResponse response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null)) {
             assertThat(response.getStatusLine().getStatusCode(), is(200));
-            assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
+            assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue());
         }
     }
 
@@ -107,7 +107,7 @@ public class CorsRegexIT extends ESIntegTestCase {
                 new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue),
                 new BasicHeader(HttpHeaders.Names.ACCESS_CONTROL_REQUEST_METHOD, "GET"));) {
             assertResponseWithOriginheader(response, corsValue);
-            assertNotNull(response.getFirstHeader("Access-Control-Allow-Methods"));
+            assertNotNull(response.getHeader("Access-Control-Allow-Methods"));
         }
     }
 
@@ -121,13 +121,13 @@ public class CorsRegexIT extends ESIntegTestCase {
             ElasticsearchResponse response = e.getElasticsearchResponse();
             // a rejected origin gets a FORBIDDEN - 403
             assertThat(response.getStatusLine().getStatusCode(), is(403));
-            assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), nullValue());
-            assertThat(response.getFirstHeader("Access-Control-Allow-Methods"), nullValue());
+            assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue());
+            assertThat(response.getHeader("Access-Control-Allow-Methods"), nullValue());
         }
     }
 
     protected static void assertResponseWithOriginheader(ElasticsearchResponse response, String expectedCorsHeader) {
         assertThat(response.getStatusLine().getStatusCode(), is(200));
-        assertThat(response.getFirstHeader("Access-Control-Allow-Origin"), is(expectedCorsHeader));
+        assertThat(response.getHeader("Access-Control-Allow-Origin"), is(expectedCorsHeader));
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java
index abcae6b26a2..5406fb79443 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java
@@ -125,7 +125,7 @@ public class RestTestResponse {
     }
 
     private boolean isJson() {
-        String contentType = response.getFirstHeader("Content-Type");
+        String contentType = response.getHeader("Content-Type");
         return contentType != null && contentType.contains("application/json");
     }
 

From 974c753bf6fdc00d30bab96d737447a0b9113a8f Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 3 Jun 2016 14:17:25 -0400
Subject: [PATCH 0636/1311] Fix uncaught checked exception in AzureTestUtils

This commit fixes an uncaught checked IOException now thrown in
AzureTestUtils after 3adaf096758a6015ca4f733e2e49ee5528ac3cd5.
---
 .../org/elasticsearch/cloud/azure/AzureTestUtils.java     | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AzureTestUtils.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AzureTestUtils.java
index 80840db587f..097f519db03 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AzureTestUtils.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AzureTestUtils.java
@@ -24,6 +24,8 @@ import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
 
+import java.io.IOException;
+
 public class AzureTestUtils {
     /**
      * Read settings from file when running integration tests with ThirdParty annotation.
@@ -36,7 +38,11 @@ public class AzureTestUtils {
         // if explicit, just load it and don't load from env
         try {
             if (Strings.hasText(System.getProperty("tests.config"))) {
-                settings.loadFromPath(PathUtils.get((System.getProperty("tests.config"))));
+                try {
+                    settings.loadFromPath(PathUtils.get((System.getProperty("tests.config"))));
+                } catch (IOException e) {
+                    throw new IllegalArgumentException("could not load azure tests config", e);
+                }
             } else {
                 throw new IllegalStateException("to run integration tests, you need to set -Dtests.thirdparty=true and " +
                     "-Dtests.config=/path/to/elasticsearch.yml");

From 92c6d78d4a17213ebed306f7f7646041bb0fcfe4 Mon Sep 17 00:00:00 2001
From: Stefan Scherer <scherer_stefan@icloud.com>
Date: Fri, 3 Jun 2016 21:30:18 +0200
Subject: [PATCH 0637/1311] Use java from path if JAVA_HOME is not set

This commit adds support to the Windows scripts for finding java.exe via
the path if JAVA_HOME is not set.

Relates #18685
---
 .../resources/bin/elasticsearch-plugin.bat    | 21 +++++++-------
 .../src/main/resources/bin/elasticsearch.bat  | 10 +++----
 .../main/resources/bin/elasticsearch.in.bat   | 15 ++++++----
 .../src/main/resources/bin/service.bat        | 29 +++++++++++++------
 4 files changed, 44 insertions(+), 31 deletions(-)

diff --git a/distribution/src/main/resources/bin/elasticsearch-plugin.bat b/distribution/src/main/resources/bin/elasticsearch-plugin.bat
index 5604b57dbef..ba35ad1c214 100644
--- a/distribution/src/main/resources/bin/elasticsearch-plugin.bat
+++ b/distribution/src/main/resources/bin/elasticsearch-plugin.bat
@@ -2,7 +2,15 @@
 
 SETLOCAL enabledelayedexpansion
 
-if NOT DEFINED JAVA_HOME goto err
+IF DEFINED JAVA_HOME (
+  set JAVA=%JAVA_HOME%\bin\java.exe
+) ELSE (
+  FOR %%I IN (java.exe) DO set JAVA=%%~$PATH:I
+)
+IF NOT EXIST "%JAVA%" (
+  ECHO Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME 1>&2
+  EXIT /B 1
+)
 
 set SCRIPT_DIR=%~dp0
 for %%I in ("%SCRIPT_DIR%..") do set ES_HOME=%%~dpfI
@@ -48,15 +56,6 @@ GOTO loop
 
 SET HOSTNAME=%COMPUTERNAME%
 
-"%JAVA_HOME%\bin\java" %ES_JAVA_OPTS% -Des.path.home="%ES_HOME%" !properties! -cp "%ES_HOME%/lib/*;" "org.elasticsearch.plugins.PluginCli" !args!
-goto finally
-
-
-:err
-echo JAVA_HOME environment variable must be set!
-pause
-
-
-:finally
+"%JAVA%" %ES_JAVA_OPTS% -Des.path.home="%ES_HOME%" !properties! -cp "%ES_HOME%/lib/*;" "org.elasticsearch.plugins.PluginCli" !args!
 
 ENDLOCAL
diff --git a/distribution/src/main/resources/bin/elasticsearch.bat b/distribution/src/main/resources/bin/elasticsearch.bat
index 9f1d871d0a2..37d7fc026a8 100644
--- a/distribution/src/main/resources/bin/elasticsearch.bat
+++ b/distribution/src/main/resources/bin/elasticsearch.bat
@@ -35,14 +35,14 @@ FOR /F "usebackq tokens=1* delims= " %%A IN (!params!) DO (
     SET current=%%A
     SET params='%%B'
 	SET silent=N
-	
+
 	IF "!current!" == "-s" (
 		SET silent=Y
 	)
 	IF "!current!" == "--silent" (
 		SET silent=Y
-	)	
-	
+	)
+
 	IF "!silent!" == "Y" (
 		SET nopauseonerror=Y
 	) ELSE (
@@ -52,7 +52,7 @@ FOR /F "usebackq tokens=1* delims= " %%A IN (!params!) DO (
             SET newparams=!current!
         )
 	)
-	
+
     IF "x!params!" NEQ "x" (
 		GOTO loop
 	)
@@ -79,6 +79,6 @@ IF ERRORLEVEL 1 (
 	EXIT /B %ERRORLEVEL%
 )
 
-"%JAVA_HOME%\bin\java" %ES_JAVA_OPTS% %ES_PARAMS% -cp "%ES_CLASSPATH%" "org.elasticsearch.bootstrap.Elasticsearch" !newparams!
+"%JAVA%" %ES_JAVA_OPTS% %ES_PARAMS% -cp "%ES_CLASSPATH%" "org.elasticsearch.bootstrap.Elasticsearch" !newparams!
 
 ENDLOCAL
diff --git a/distribution/src/main/resources/bin/elasticsearch.in.bat b/distribution/src/main/resources/bin/elasticsearch.in.bat
index d6f1ef308bd..2d73ed2b57b 100644
--- a/distribution/src/main/resources/bin/elasticsearch.in.bat
+++ b/distribution/src/main/resources/bin/elasticsearch.in.bat
@@ -1,12 +1,15 @@
 @echo off
 
-if DEFINED JAVA_HOME goto cont
+IF DEFINED JAVA_HOME (
+  set JAVA=%JAVA_HOME%\bin\java.exe
+) ELSE (
+  FOR %%I IN (java.exe) DO set JAVA=%%~$PATH:I
+)
+IF NOT EXIST "%JAVA%" (
+  ECHO Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME 1>&2
+  EXIT /B 1
+)
 
-:err
-ECHO JAVA_HOME environment variable must be set! 1>&2
-EXIT /B 1
-
-:cont
 set SCRIPT_DIR=%~dp0
 for %%I in ("%SCRIPT_DIR%..") do set ES_HOME=%%~dpfI
 
diff --git a/distribution/src/main/resources/bin/service.bat b/distribution/src/main/resources/bin/service.bat
index 81b6c8a5df5..344782030ee 100644
--- a/distribution/src/main/resources/bin/service.bat
+++ b/distribution/src/main/resources/bin/service.bat
@@ -28,27 +28,38 @@ if %bad_env_var% == 1 (
 )
 rem end TODO: remove for Elasticsearch 6.x
 
-if NOT DEFINED JAVA_HOME goto err
+IF DEFINED JAVA_HOME (
+  SET JAVA=%JAVA_HOME%\bin\java.exe
+) ELSE (
+  FOR %%I IN (java.exe) DO set JAVA=%%~$PATH:I
+)
+IF NOT EXIST "%JAVA%" (
+  ECHO Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME 1>&2
+  EXIT /B 1
+)
+IF DEFINED JAVA_HOME GOTO :cont
 
+IF NOT "%JAVA:~-13%" == "\bin\java.exe" (
+  FOR /f "tokens=2 delims=[]" %%I IN ('dir %JAVA%') DO @set JAVA=%%I
+)
+IF "%JAVA:~-13%" == "\bin\java.exe" (
+  SET JAVA_HOME=%JAVA:~0,-13%
+)
+
+:cont
 if not "%CONF_FILE%" == "" goto conffileset
 
 set SCRIPT_DIR=%~dp0
 for %%I in ("%SCRIPT_DIR%..") do set ES_HOME=%%~dpfI
 
-rem Detect JVM version to figure out appropriate executable to use
-if not exist "%JAVA_HOME%\bin\java.exe" (
-echo JAVA_HOME points to an invalid Java installation (no java.exe found in "%JAVA_HOME%"^). Exiting...
-goto:eof
-)
-
-"%JAVA_HOME%\bin\java" -Xmx50M -version > nul 2>&1
+"%JAVA%" -Xmx50M -version > nul 2>&1
 
 if errorlevel 1 (
 	echo Warning: Could not start JVM to detect version, defaulting to x86:
 	goto x86
 )
 
-"%JAVA_HOME%\bin\java" -Xmx50M -version 2>&1 | "%windir%\System32\find" "64-Bit" >nul:
+"%JAVA%" -Xmx50M -version 2>&1 | "%windir%\System32\find" "64-Bit" >nul:
 
 if errorlevel 1 goto x86
 set EXECUTABLE=%ES_HOME%\bin\elasticsearch-service-x64.exe

From a30437f2505a1599bffd14be7a36c67489687dad Mon Sep 17 00:00:00 2001
From: Areek Zillur <areek.zillur@elasticsearch.com>
Date: Fri, 3 Jun 2016 14:52:54 -0400
Subject: [PATCH 0638/1311] adds _rollover api

---
 .../elasticsearch/action/ActionModule.java    |   3 +
 ...dicesAliasesClusterStateUpdateRequest.java |   2 +-
 .../admin/indices/rollover/Condition.java     | 105 +++++++
 .../indices/rollover/RolloverAction.java      |  45 +++
 .../indices/rollover/RolloverRequest.java     | 164 +++++++++++
 .../rollover/RolloverRequestBuilder.java      |  58 ++++
 .../indices/rollover/RolloverResponse.java    |  62 +++++
 .../rollover/TransportRolloverAction.java     | 257 ++++++++++++++++++
 .../client/IndicesAdminClient.java            |  18 ++
 .../client/support/AbstractClient.java        |  19 ++
 .../common/network/NetworkModule.java         |   2 +
 .../indices/RestRolloverIndexAction.java      |  72 +++++
 .../admin/indices/rollover/RolloverIT.java    | 105 +++++++
 .../TransportRolloverActionTests.java         | 192 +++++++++++++
 .../rest-api-spec/api/indices.rollover.json   |  35 +++
 .../test/indices.rollover/10_basic.yaml       |  69 +++++
 16 files changed, 1207 insertions(+), 1 deletion(-)
 create mode 100644 core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
 create mode 100644 core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverAction.java
 create mode 100644 core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
 create mode 100644 core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
 create mode 100644 core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
 create mode 100644 core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
 create mode 100644 core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
 create mode 100644 core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
 create mode 100644 core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml

diff --git a/core/src/main/java/org/elasticsearch/action/ActionModule.java b/core/src/main/java/org/elasticsearch/action/ActionModule.java
index 7e975b922d4..316b2b9aaa0 100644
--- a/core/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/core/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -107,6 +107,8 @@ import org.elasticsearch.action.admin.indices.recovery.RecoveryAction;
 import org.elasticsearch.action.admin.indices.recovery.TransportRecoveryAction;
 import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
 import org.elasticsearch.action.admin.indices.refresh.TransportRefreshAction;
+import org.elasticsearch.action.admin.indices.rollover.RolloverAction;
+import org.elasticsearch.action.admin.indices.rollover.TransportRolloverAction;
 import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsAction;
 import org.elasticsearch.action.admin.indices.segments.TransportIndicesSegmentsAction;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction;
@@ -289,6 +291,7 @@ public class ActionModule extends AbstractModule {
         registerAction(IndicesShardStoresAction.INSTANCE, TransportIndicesShardStoresAction.class);
         registerAction(CreateIndexAction.INSTANCE, TransportCreateIndexAction.class);
         registerAction(ShrinkAction.INSTANCE, TransportShrinkAction.class);
+        registerAction(RolloverAction.INSTANCE, TransportRolloverAction.class);
         registerAction(DeleteIndexAction.INSTANCE, TransportDeleteIndexAction.class);
         registerAction(GetIndexAction.INSTANCE, TransportGetIndexAction.class);
         registerAction(OpenIndexAction.INSTANCE, TransportOpenIndexAction.class);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesClusterStateUpdateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesClusterStateUpdateRequest.java
index 30cbd6d9ec7..42d34d6ebd7 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesClusterStateUpdateRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/IndicesAliasesClusterStateUpdateRequest.java
@@ -28,7 +28,7 @@ public class IndicesAliasesClusterStateUpdateRequest extends ClusterStateUpdateR
 
     AliasAction[] actions;
 
-    IndicesAliasesClusterStateUpdateRequest() {
+    public IndicesAliasesClusterStateUpdateRequest() {
 
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
new file mode 100644
index 00000000000..c5ef9b4c51b
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.rollover;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
+
+import java.util.Locale;
+
+public class Condition {
+
+    public enum ConditionType {
+        MAX_SIZE((byte) 0),
+        MAX_AGE((byte) 1),
+        MAX_DOCS((byte) 2);
+
+        private final byte id;
+
+        ConditionType(byte id) {
+            this.id = id;
+        }
+
+        public byte getId() {
+            return id;
+        }
+
+        public static ConditionType fromId(byte id) {
+            if (id == 0) {
+                return MAX_SIZE;
+            } else if (id == 1) {
+                return MAX_AGE;
+            } else if (id == 2) {
+                return MAX_DOCS;
+            } else {
+                throw new IllegalArgumentException("no condition type [" + id + "]");
+            }
+        }
+
+        public static ConditionType fromString(String type) {
+            final String typeString = type.toLowerCase(Locale.ROOT);
+            switch (typeString) {
+                case "max_size":
+                    return MAX_SIZE;
+                case "max_age":
+                    return MAX_AGE;
+                case "max_docs":
+                    return MAX_DOCS;
+                default:
+                    throw new IllegalArgumentException("no condition type [" + type + "]");
+            }
+        }
+
+        public static long parseFromString(ConditionType condition, String value) {
+            switch (condition) {
+                case MAX_SIZE:
+                    return ByteSizeValue.parseBytesSizeValue(value, MAX_SIZE.name().toLowerCase(Locale.ROOT)).getBytes();
+                case MAX_AGE:
+                    return TimeValue.parseTimeValue(value, MAX_AGE.name().toLowerCase(Locale.ROOT)).getMillis();
+                case MAX_DOCS:
+                    try {
+                       return Long.valueOf(value);
+                    } catch (NumberFormatException e) {
+                        throw new ElasticsearchParseException("Failed to parse setting [{}] with value [{}] as long", e,
+                            MAX_DOCS.name().toLowerCase(Locale.ROOT), value);
+                    }
+                default:
+                    throw new ElasticsearchParseException("condition [" + condition + "] not recognized");
+            }
+        }
+    }
+
+    private final ConditionType type;
+    private final long value;
+
+    public Condition(ConditionType type, long value) {
+        this.type = type;
+        this.value = value;
+    }
+
+    public ConditionType getType() {
+        return type;
+    }
+
+    public long getValue() {
+        return value;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverAction.java
new file mode 100644
index 00000000000..dd9d50dac73
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverAction.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.rollover;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.client.ElasticsearchClient;
+
+/**
+ */
+public class RolloverAction extends Action<RolloverRequest, RolloverResponse, RolloverRequestBuilder> {
+
+    public static final RolloverAction INSTANCE = new RolloverAction();
+    public static final String NAME = "indices:admin/rollover";
+
+    private RolloverAction() {
+        super(NAME);
+    }
+
+    @Override
+    public RolloverResponse newResponse() {
+        return new RolloverResponse();
+    }
+
+    @Override
+    public RolloverRequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new RolloverRequestBuilder(client, this);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
new file mode 100644
index 00000000000..4a233bf2d81
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.admin.indices.rollover;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * Request class to swap index under an alias given some predicates
+ * TODO: documentation
+ */
+public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implements IndicesRequest {
+
+    private String sourceAlias;
+    private String optionalTargetAlias;
+    private List<Condition> conditions = new ArrayList<>();
+
+    RolloverRequest() {}
+
+    public RolloverRequest(String sourceAlias, String optionalTargetAlias) {
+        this.sourceAlias = sourceAlias;
+        this.optionalTargetAlias = optionalTargetAlias;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (sourceAlias == null) {
+            validationException = addValidationError("source alias is missing", validationException);
+        }
+        return validationException;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        sourceAlias = in.readString();
+        optionalTargetAlias = in.readOptionalString();
+        int size = in.readVInt();
+        for (int i = 0; i < size; i++) {
+            Condition.ConditionType type = Condition.ConditionType.fromId(in.readByte());
+            long value = in.readLong();
+            this.conditions.add(new Condition(type, value));
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(sourceAlias);
+        out.writeOptionalString(optionalTargetAlias);
+        out.writeVInt(conditions.size());
+        for (Condition condition : conditions) {
+            out.writeByte(condition.getType().getId());
+            out.writeLong(condition.getValue());
+        }
+    }
+
+    @Override
+    public String[] indices() {
+        return new String[] {sourceAlias};
+    }
+
+    @Override
+    public IndicesOptions indicesOptions() {
+        return IndicesOptions.strictSingleIndexNoExpandForbidClosed();
+    }
+
+    public void setSourceAlias(String sourceAlias) {
+        this.sourceAlias = sourceAlias;
+    }
+
+    public void setOptionalTargetAlias(String optionalTargetAlias) {
+        this.optionalTargetAlias = optionalTargetAlias;
+    }
+
+    public void addMaxIndexAgeCondition(String age) {
+        addCondition(Condition.ConditionType.MAX_AGE, Condition.ConditionType.parseFromString(Condition.ConditionType.MAX_AGE, age));
+    }
+
+    public void addMaxIndexDocsCondition(String docs) {
+        addCondition(Condition.ConditionType.MAX_DOCS, Condition.ConditionType.parseFromString(Condition.ConditionType.MAX_DOCS, docs));
+    }
+
+    public void addMaxIndexSizeCondition(String size) {
+        addCondition(Condition.ConditionType.MAX_SIZE, Condition.ConditionType.parseFromString(Condition.ConditionType.MAX_SIZE, size));
+    }
+
+    private void addCondition(Condition.ConditionType conditionType, long value) {
+        this.conditions.add(new Condition(conditionType, value));
+    }
+
+    public List<Condition> getConditions() {
+        return conditions;
+    }
+
+    public String getSourceAlias() {
+        return sourceAlias;
+    }
+
+    public String getOptionalTargetAlias() {
+        return optionalTargetAlias;
+    }
+
+    public void source(BytesReference source) {
+        XContentType xContentType = XContentFactory.xContentType(source);
+        if (xContentType != null) {
+            try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(source)) {
+                source(parser.map());
+            } catch (IOException e) {
+                throw new ElasticsearchParseException("failed to parse source for rollover index", e);
+            }
+        } else {
+            throw new ElasticsearchParseException("failed to parse content type for rollover index source");
+        }
+    }
+
+    private void source(Map<String, Object> map) {
+        for (Map.Entry<String, Object> entry : map.entrySet()) {
+            if (entry.getKey().equals("conditions")) {
+                final Map<String, String> conditions = (Map<String, String>) entry.getValue();
+                for (Map.Entry<String, String> conditionEntry : conditions.entrySet()) {
+                    final Condition.ConditionType conditionType = Condition.ConditionType.fromString(conditionEntry.getKey());
+                    this.addCondition(conditionType, Condition.ConditionType.parseFromString(conditionType,
+                        conditionEntry.getValue()));
+                }
+            } else {
+                throw new ElasticsearchParseException("unknown property [" + entry.getKey() + "]");
+            }
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
new file mode 100644
index 00000000000..c42bb9cbd83
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.action.admin.indices.rollover;
+
+import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+
+
+/**
+ * TODO: Documentation
+ */
+public class RolloverRequestBuilder extends MasterNodeOperationRequestBuilder<RolloverRequest, RolloverResponse,
+    RolloverRequestBuilder> {
+    public RolloverRequestBuilder(ElasticsearchClient client, RolloverAction action) {
+        super(client, action, new RolloverRequest());
+    }
+
+    public RolloverRequestBuilder setSourceAlias(String sourceAlias) {
+        this.request.setSourceAlias(sourceAlias);
+        return this;
+    }
+
+    public RolloverRequestBuilder setOptionalTargetAlias(String optionalTargetAlias) {
+        this.request.setOptionalTargetAlias(optionalTargetAlias);
+        return this;
+    }
+
+    public RolloverRequestBuilder addMaxIndexSizeCondition(String size) {
+        this.request.addMaxIndexSizeCondition(size);
+        return this;
+    }
+
+    public RolloverRequestBuilder addMaxIndexAgeCondition(String age) {
+        this.request.addMaxIndexAgeCondition(age);
+        return this;
+    }
+
+    public RolloverRequestBuilder addMaxIndexDocsCondition(String docs) {
+        this.request.addMaxIndexDocsCondition(docs);
+        return this;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
new file mode 100644
index 00000000000..79722e73755
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.rollover;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+public final class RolloverResponse extends ActionResponse {
+
+    private String oldIndex;
+    private String newIndex;
+
+    RolloverResponse() {
+    }
+
+    RolloverResponse(String oldIndex, String newIndex) {
+        this.oldIndex = oldIndex;
+        this.newIndex = newIndex;
+    }
+
+    public String getOldIndex() {
+        return oldIndex;
+    }
+
+    public String getNewIndex() {
+        return newIndex;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        oldIndex = in.readString();
+        newIndex = in.readString();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeString(oldIndex);
+        out.writeString(newIndex);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
new file mode 100644
index 00000000000..fcd9e2d96cf
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.rollover;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesClusterStateUpdateRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse;
+import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.block.ClusterBlockLevel;
+import org.elasticsearch.cluster.metadata.AliasAction;
+import org.elasticsearch.cluster.metadata.AliasOrIndex;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
+import org.elasticsearch.cluster.metadata.MetaDataIndexAliasesService;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.shard.DocsStats;
+import org.elasticsearch.index.store.StoreStats;
+import org.elasticsearch.indices.IndexAlreadyExistsException;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.List;
+
+/**
+ * Main class to swap the index pointed to by an alias, given some predicates
+ */
+public class TransportRolloverAction extends TransportMasterNodeAction<RolloverRequest, RolloverResponse> {
+
+    private final MetaDataCreateIndexService createIndexService;
+    private final MetaDataIndexAliasesService indexAliasesService;
+    private final Client client;
+
+    @Inject
+    public TransportRolloverAction(Settings settings, TransportService transportService, ClusterService clusterService,
+                                   ThreadPool threadPool, MetaDataCreateIndexService createIndexService,
+                                   ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                   MetaDataIndexAliasesService indexAliasesService, Client client) {
+        super(settings, RolloverAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver,
+            RolloverRequest::new);
+        this.createIndexService = createIndexService;
+        this.indexAliasesService = indexAliasesService;
+        this.client = client;
+    }
+
+    @Override
+    protected String executor() {
+        // we go async right away
+        return ThreadPool.Names.SAME;
+    }
+
+    @Override
+    protected RolloverResponse newResponse() {
+        return new RolloverResponse();
+    }
+
+    @Override
+    protected ClusterBlockException checkBlock(RolloverRequest request, ClusterState state) {
+        IndicesOptions indicesOptions = IndicesOptions.fromOptions(true, true,
+            request.indicesOptions().expandWildcardsOpen(), request.indicesOptions().expandWildcardsClosed());
+        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE,
+            indexNameExpressionResolver.concreteIndexNames(state, indicesOptions, request.indices()));
+    }
+
+    @Override
+    protected void masterOperation(final RolloverRequest rolloverRequest, final ClusterState state,
+                                   final ActionListener<RolloverResponse> listener) {
+        final MetaData metaData = state.metaData();
+        validate(metaData, rolloverRequest);
+        final AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(rolloverRequest.getSourceAlias());
+        final IndexMetaData indexMetaData = aliasOrIndex.getIndices().get(0);
+        final String sourceIndexName = indexMetaData.getIndex().getName();
+        client.admin().indices().prepareStats(sourceIndexName).clear().setDocs(true).setStore(true)
+            .execute(new ActionListener<IndicesStatsResponse>() {
+            @Override
+            public void onResponse(IndicesStatsResponse indicesStatsResponse) {
+                final IndexMetaData sourceIndex = metaData.index(sourceIndexName);
+                DocsStats docsStats = indicesStatsResponse.getTotal().getDocs();
+                long docCount = docsStats == null ? 0 : docsStats.getCount();
+                StoreStats storeStats = indicesStatsResponse.getTotal().getStore();
+                long sizeInBytes = storeStats == null ? 0 : storeStats.getSizeInBytes();
+                long creationDate = sourceIndex.getCreationDate();
+                if (satisfiesConditions(rolloverRequest.getConditions(), docCount, sizeInBytes, creationDate)) {
+                    final String rolloverIndexName = generateRolloverIndexName(sourceIndexName);
+                    boolean createRolloverIndex = metaData.index(rolloverIndexName) == null;
+                    if (createRolloverIndex) {
+                        CreateIndexClusterStateUpdateRequest updateRequest =
+                            prepareCreateIndexRequest(rolloverIndexName, rolloverRequest);
+                        createIndexService.createIndex(updateRequest, new ActionListener<ClusterStateUpdateResponse>() {
+                            @Override
+                            public void onResponse(ClusterStateUpdateResponse response) {
+                                indexAliasesService.indicesAliases(
+                                    prepareIndicesAliasesRequest(sourceIndexName, rolloverIndexName, rolloverRequest),
+                                    new IndicesAliasesListener(sourceIndexName, rolloverIndexName, listener));
+                            }
+
+                            @Override
+                            public void onFailure(Throwable t) {
+                                if (t instanceof IndexAlreadyExistsException) {
+                                    logger.trace("[{}] failed to create rollover index", t, updateRequest.index());
+                                } else {
+                                    logger.debug("[{}] failed to create rollover index", t, updateRequest.index());
+                                }
+                                listener.onFailure(t);
+                            }
+                        });
+                    } else {
+                        indexAliasesService.indicesAliases(
+                            prepareIndicesAliasesRequest(sourceIndexName, rolloverIndexName, rolloverRequest),
+                            new IndicesAliasesListener(sourceIndexName, rolloverIndexName, listener));
+                    }
+                } else {
+                    // conditions not met
+                    listener.onResponse(new RolloverResponse(sourceIndexName, sourceIndexName));
+                }
+            }
+
+            @Override
+            public void onFailure(Throwable e) {
+                listener.onFailure(e);
+            }
+        });
+
+    }
+
+    private static final class IndicesAliasesListener implements ActionListener<ClusterStateUpdateResponse> {
+
+        private final ActionListener<RolloverResponse> listener;
+        private final String oldIndex;
+        private final String newIndex;
+
+        public IndicesAliasesListener(String oldIndex, String newIndex, ActionListener<RolloverResponse> listener) {
+            this.oldIndex = oldIndex;
+            this.newIndex = newIndex;
+            this.listener = listener;
+        }
+
+        @Override
+        public void onResponse(ClusterStateUpdateResponse clusterStateUpdateResponse) {
+            listener.onResponse(new RolloverResponse(oldIndex, newIndex));
+        }
+
+        @Override
+        public void onFailure(Throwable e) {
+            listener.onFailure(e);
+        }
+    }
+
+    static IndicesAliasesClusterStateUpdateRequest prepareIndicesAliasesRequest(String concreteSourceIndex, String targetIndex,
+                                                                                RolloverRequest request) {
+        final IndicesAliasesClusterStateUpdateRequest updateRequest = new IndicesAliasesClusterStateUpdateRequest()
+            .ackTimeout(request.ackTimeout())
+            .masterNodeTimeout(request.masterNodeTimeout());
+        final AliasAction[] actions;
+        if (request.getOptionalTargetAlias() != null) {
+            actions = new AliasAction[3];
+            actions[0] = new AliasAction(AliasAction.Type.ADD, targetIndex, request.getSourceAlias());
+            actions[1] = new AliasAction(AliasAction.Type.ADD, targetIndex, request.getOptionalTargetAlias());
+            actions[2] = new AliasAction(AliasAction.Type.REMOVE, concreteSourceIndex, request.getSourceAlias());
+        } else {
+            actions = new AliasAction[2];
+            actions[0] = new AliasAction(AliasAction.Type.ADD, targetIndex, request.getSourceAlias());
+            actions[1] = new AliasAction(AliasAction.Type.REMOVE, concreteSourceIndex, request.getSourceAlias());
+        }
+        updateRequest.actions(actions);
+        return updateRequest;
+    }
+
+    static String generateRolloverIndexName(String sourceIndexName) {
+        int numberIndex = sourceIndexName.lastIndexOf("-");
+        int counter = 1;
+        String indexPrefix = sourceIndexName;
+        if (numberIndex != -1) {
+            try {
+                counter = Integer.parseInt(sourceIndexName.substring(numberIndex + 1));
+                counter++;
+                indexPrefix = sourceIndexName.substring(0, numberIndex);
+            } catch (NumberFormatException ignored) {
+            }
+        }
+        return String.join("-", indexPrefix, String.valueOf(counter));
+    }
+
+    static boolean satisfiesConditions(List<Condition> requestConditions, long docCount, long sizeInBytes,
+                                       long creationDate) {
+        for (Condition condition: requestConditions) {
+            switch (condition.getType()) {
+                case MAX_SIZE:
+                    if (sizeInBytes < condition.getValue()) {
+                        return false;
+                    }
+                    break;
+                case MAX_AGE:
+                    long currentAge = System.currentTimeMillis() - creationDate;
+                    if (currentAge < condition.getValue()) {
+                        return false;
+                    }
+                    break;
+                case MAX_DOCS:
+                    if (docCount < condition.getValue()) {
+                        return false;
+                    }
+                    break;
+            }
+        }
+        return true;
+    }
+
+    static void validate(MetaData metaData, RolloverRequest request) {
+        final AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(request.getSourceAlias());
+        if (aliasOrIndex == null) {
+            throw new IllegalArgumentException("source alias does not exist");
+        }
+        if (aliasOrIndex.isAlias() == false) {
+            throw new IllegalArgumentException("source alias is a concrete index");
+        }
+        if (aliasOrIndex.getIndices().size() != 1) {
+            throw new IllegalArgumentException("source alias maps to multiple indices");
+        }
+    }
+
+    static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final String targetIndexName,
+                                                                          final RolloverRequest rolloverRequest) {
+        return new CreateIndexClusterStateUpdateRequest(rolloverRequest,
+            "rollover_index", targetIndexName, true)
+            .ackTimeout(rolloverRequest.timeout())
+            .masterNodeTimeout(rolloverRequest.masterNodeTimeout());
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java b/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java
index 10275f22d66..24d190c68a1 100644
--- a/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java
+++ b/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java
@@ -80,6 +80,9 @@ import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder;
 import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
+import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
+import org.elasticsearch.action.admin.indices.rollover.RolloverRequestBuilder;
+import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
 import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse;
 import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest;
 import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequestBuilder;
@@ -800,4 +803,19 @@ public interface IndicesAdminClient extends ElasticsearchClient {
      */
     void shrinkIndex(ShrinkRequest request, ActionListener<ShrinkResponse> listener);
 
+    /**
+     * Swaps the index pointed to by an alias given all provided conditions are satisfied
+     */
+    RolloverRequestBuilder prepareRolloverIndex(String sourceAlias);
+
+    /**
+     * Swaps the index pointed to by an alias given all provided conditions are satisfied
+     */
+    ActionFuture<RolloverResponse> rolloversIndex(RolloverRequest request);
+
+    /**
+     * Swaps the index pointed to by an alias given all provided conditions are satisfied
+     */
+    void rolloverIndex(RolloverRequest request, ActionListener<RolloverResponse> listener);
+
 }
diff --git a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
index 6481bec1b83..18a3a2ae202 100644
--- a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
+++ b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
@@ -196,6 +196,10 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder;
 import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
+import org.elasticsearch.action.admin.indices.rollover.RolloverAction;
+import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
+import org.elasticsearch.action.admin.indices.rollover.RolloverRequestBuilder;
+import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
 import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse;
 import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsAction;
 import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest;
@@ -1704,6 +1708,21 @@ public abstract class AbstractClient extends AbstractComponent implements Client
             execute(ShrinkAction.INSTANCE, request, listener);
         }
 
+        @Override
+        public RolloverRequestBuilder prepareRolloverIndex(String sourceAlias) {
+            return new RolloverRequestBuilder(this, RolloverAction.INSTANCE).setSourceAlias(sourceAlias);
+        }
+
+        @Override
+        public ActionFuture<RolloverResponse> rolloversIndex(RolloverRequest request) {
+            return execute(RolloverAction.INSTANCE, request);
+        }
+
+        @Override
+        public void rolloverIndex(RolloverRequest request, ActionListener<RolloverResponse> listener) {
+            execute(RolloverAction.INSTANCE, request, listener);
+        }
+
         @Override
         public ActionFuture<GetSettingsResponse> getSettings(GetSettingsRequest request) {
             return execute(GetSettingsAction.INSTANCE, request);
diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
index fd9191fd713..6d4f671c3ef 100644
--- a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
+++ b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
@@ -72,6 +72,7 @@ import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestGetStoredSc
 import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestPutSearchTemplateAction;
 import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestPutStoredScriptAction;
 import org.elasticsearch.rest.action.admin.cluster.tasks.RestPendingClusterTasksAction;
+import org.elasticsearch.rest.action.admin.indices.RestRolloverIndexAction;
 import org.elasticsearch.rest.action.admin.indices.RestShrinkIndexAction;
 import org.elasticsearch.rest.action.admin.indices.alias.RestIndicesAliasesAction;
 import org.elasticsearch.rest.action.admin.indices.alias.delete.RestIndexDeleteAliasesAction;
@@ -211,6 +212,7 @@ public class NetworkModule extends AbstractModule {
         RestIndicesAliasesAction.class,
         RestCreateIndexAction.class,
         RestShrinkIndexAction.class,
+        RestRolloverIndexAction.class,
         RestDeleteIndexAction.class,
         RestCloseIndexAction.class,
         RestOpenIndexAction.class,
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
new file mode 100644
index 00000000000..42626b37281
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.action.admin.indices;
+
+import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
+import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestResponse;
+import org.elasticsearch.rest.action.support.RestBuilderListener;
+
+import static org.elasticsearch.rest.RestStatus.OK;
+
+/**
+ *
+ */
+public class RestRolloverIndexAction extends BaseRestHandler {
+
+    @Inject
+    public RestRolloverIndexAction(Settings settings, RestController controller, Client client) {
+        super(settings, client);
+        controller.registerHandler(RestRequest.Method.PUT, "/{alias}/_rollover/{optional_alias}", this);
+        controller.registerHandler(RestRequest.Method.POST, "/{alias}/_rollover/{optional_alias}", this);
+    }
+
+    @Override
+    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+        if (request.param("alias") == null) {
+            throw new IllegalArgumentException("no source alias");
+        }
+        RolloverRequest rolloverIndexRequest = new RolloverRequest(request.param("alias"), request.param("optional_alias"));
+        if (request.hasContent()) {
+            rolloverIndexRequest.source(request.content());
+        }
+        rolloverIndexRequest.timeout(request.paramAsTime("timeout", rolloverIndexRequest.timeout()));
+        rolloverIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", rolloverIndexRequest.masterNodeTimeout()));
+        client.admin().indices().rolloverIndex(rolloverIndexRequest, new RestBuilderListener<RolloverResponse>(channel) {
+            @Override
+            public RestResponse buildResponse(RolloverResponse rolloverResponse, XContentBuilder builder) throws Exception {
+                builder.startObject();
+                builder.field("old_index", rolloverResponse.getOldIndex());
+                builder.field("new_index", rolloverResponse.getNewIndex());
+                builder.endObject();
+                return new BytesRestResponse(OK, builder);
+            }
+        });
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
new file mode 100644
index 00000000000..f80543fd7c2
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.rollover;
+
+import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.equalTo;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
+public class RolloverIT extends ESIntegTestCase {
+
+    public void testRolloverOnEmptyIndex() throws Exception {
+        assertAcked(prepareCreate("test_index").addAlias(new Alias("test_alias")).get());
+        final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias").get();
+        assertThat(response.getOldIndex(), equalTo("test_index"));
+        assertThat(response.getNewIndex(), equalTo("test_index-1"));
+        final ClusterState state = client().admin().cluster().prepareState().get().getState();
+        final IndexMetaData oldIndex = state.metaData().index("test_index");
+        assertFalse(oldIndex.getAliases().containsKey("test_alias"));
+        final IndexMetaData newIndex = state.metaData().index("test_index-1");
+        assertTrue(newIndex.getAliases().containsKey("test_alias"));
+    }
+
+    public void testRollover() throws Exception {
+        assertAcked(prepareCreate("test_index").addAlias(new Alias("test_alias")).get());
+        index("test_index", "type1", "1", "field", "value");
+        flush("test_index");
+        final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias").get();
+        assertThat(response.getOldIndex(), equalTo("test_index"));
+        assertThat(response.getNewIndex(), equalTo("test_index-1"));
+        final ClusterState state = client().admin().cluster().prepareState().get().getState();
+        final IndexMetaData oldIndex = state.metaData().index("test_index");
+        assertFalse(oldIndex.getAliases().containsKey("test_alias"));
+        final IndexMetaData newIndex = state.metaData().index("test_index-1");
+        assertTrue(newIndex.getAliases().containsKey("test_alias"));
+    }
+
+    public void testRolloverConditionsNotMet() throws Exception {
+        assertAcked(prepareCreate("test_index").addAlias(new Alias("test_alias")).get());
+        index("test_index", "type1", "1", "field", "value");
+        flush("test_index");
+        final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias")
+            .addMaxIndexAgeCondition("7d").get();
+        assertThat(response.getOldIndex(), equalTo("test_index"));
+        assertThat(response.getNewIndex(), equalTo("test_index"));
+        final ClusterState state = client().admin().cluster().prepareState().get().getState();
+        final IndexMetaData oldIndex = state.metaData().index("test_index");
+        assertTrue(oldIndex.getAliases().containsKey("test_alias"));
+        final IndexMetaData newIndex = state.metaData().index("test_index-1");
+        assertNull(newIndex);
+    }
+
+    public void testRolloverWithOptionalTargetAlias() throws Exception {
+        assertAcked(prepareCreate("test_index").addAlias(new Alias("test_alias")).get());
+        index("test_index", "type1", "1", "field", "value");
+        flush("test_index");
+        final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias")
+            .setOptionalTargetAlias("test_alias_2").get();
+        assertThat(response.getOldIndex(), equalTo("test_index"));
+        assertThat(response.getNewIndex(), equalTo("test_index-1"));
+        final ClusterState state = client().admin().cluster().prepareState().get().getState();
+        final IndexMetaData oldIndex = state.metaData().index("test_index");
+        assertFalse(oldIndex.getAliases().containsKey("test_alias"));
+        final IndexMetaData newIndex = state.metaData().index("test_index-1");
+        assertTrue(newIndex.getAliases().containsKey("test_alias"));
+        assertTrue(newIndex.getAliases().containsKey("test_alias_2"));
+    }
+
+    public void testRolloverOnExistingIndex() throws Exception {
+        assertAcked(prepareCreate("test_index").addAlias(new Alias("test_alias")).get());
+        index("test_index", "type1", "1", "field", "value");
+        assertAcked(prepareCreate("test_index-1").get());
+        index("test_index-1", "type1", "1", "field", "value");
+        flush("test_index", "test_index-1");
+        final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias").get();
+        assertThat(response.getOldIndex(), equalTo("test_index"));
+        assertThat(response.getNewIndex(), equalTo("test_index-1"));
+        final ClusterState state = client().admin().cluster().prepareState().get().getState();
+        final IndexMetaData oldIndex = state.metaData().index("test_index");
+        assertFalse(oldIndex.getAliases().containsKey("test_alias"));
+        final IndexMetaData newIndex = state.metaData().index("test_index-1");
+        assertTrue(newIndex.getAliases().containsKey("test_alias"));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
new file mode 100644
index 00000000000..959f0d81bf0
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.rollover;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesClusterStateUpdateRequest;
+import org.elasticsearch.cluster.metadata.AliasAction;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+
+public class TransportRolloverActionTests extends ESTestCase {
+
+    public void testSatisfyConditions() throws Exception {
+        List<Condition> conditions = Collections.emptyList();
+        assertTrue(TransportRolloverAction.satisfiesConditions(conditions, randomLong(), randomLong(),
+            randomLong()));
+
+        conditions = Collections.singletonList(
+            new Condition(Condition.ConditionType.MAX_AGE, 10L));
+        assertTrue(TransportRolloverAction.satisfiesConditions(conditions, randomLong(), randomLong(),
+            System.currentTimeMillis() - randomIntBetween(10, 100)));
+        assertFalse(TransportRolloverAction.satisfiesConditions(conditions, randomLong(), randomLong(),
+            System.currentTimeMillis() - randomIntBetween(1, 9)));
+
+        conditions = Collections.singletonList(
+            new Condition(Condition.ConditionType.MAX_SIZE, 10L));
+        assertTrue(TransportRolloverAction.satisfiesConditions(conditions, randomLong(), randomIntBetween(10, 100),
+            randomLong()));
+        assertFalse(TransportRolloverAction.satisfiesConditions(conditions, randomLong(), randomIntBetween(1, 9),
+            randomLong()));
+
+        conditions = Collections.singletonList(
+            new Condition(Condition.ConditionType.MAX_DOCS, 10L));
+        assertTrue(TransportRolloverAction.satisfiesConditions(conditions, randomIntBetween(10, 100), randomLong(),
+            randomLong()));
+        assertFalse(TransportRolloverAction.satisfiesConditions(conditions, randomIntBetween(1, 9), randomLong(),
+            randomLong()));
+
+        conditions = Arrays.asList(new Condition(Condition.ConditionType.MAX_AGE, 100L),
+            new Condition(Condition.ConditionType.MAX_DOCS, 1000L));
+        assertTrue(TransportRolloverAction.satisfiesConditions(conditions, randomIntBetween(1000, 1500),
+            randomLong(), System.currentTimeMillis() - randomIntBetween(100, 500)));
+        assertFalse(TransportRolloverAction.satisfiesConditions(conditions, randomIntBetween(1, 999),
+            randomLong(), System.currentTimeMillis() - randomIntBetween(100, 500)));
+        assertFalse(TransportRolloverAction.satisfiesConditions(conditions, randomIntBetween(1000, 1500),
+            randomLong(), System.currentTimeMillis() - randomIntBetween(1, 99)));
+    }
+
+    public void testCreateUpdateAliasRequest() throws Exception {
+        String sourceAlias = randomAsciiOfLength(10);
+        String sourceIndex = randomAsciiOfLength(10);
+        String targetIndex = randomAsciiOfLength(10);
+        final RolloverRequest rolloverRequest = new RolloverRequest(sourceAlias, null);
+        final IndicesAliasesClusterStateUpdateRequest updateRequest =
+            TransportRolloverAction.prepareIndicesAliasesRequest(sourceIndex, targetIndex, rolloverRequest);
+
+        final AliasAction[] actions = updateRequest.actions();
+        assertThat(actions.length, equalTo(2));
+        boolean foundAdd = false;
+        boolean foundRemove = false;
+        for (AliasAction action : actions) {
+            if (action.actionType() == AliasAction.Type.ADD) {
+                foundAdd = true;
+                assertThat(action.index(), equalTo(targetIndex));
+                assertThat(action.alias(), equalTo(sourceAlias));
+            } else if (action.actionType() == AliasAction.Type.REMOVE) {
+                foundRemove = true;
+                assertThat(action.index(), equalTo(sourceIndex));
+                assertThat(action.alias(), equalTo(sourceAlias));
+            }
+        }
+        assertTrue(foundAdd);
+        assertTrue(foundRemove);
+    }
+
+    public void testCreateUpdateAliasRequestWithOptionalTargetAlias() throws Exception {
+        String sourceAlias = randomAsciiOfLength(10);
+        String optionalTargetAlias = randomAsciiOfLength(10);
+        String sourceIndex = randomAsciiOfLength(10);
+        String targetIndex = randomAsciiOfLength(10);
+        final RolloverRequest rolloverRequest = new RolloverRequest(sourceAlias, optionalTargetAlias);
+        final IndicesAliasesClusterStateUpdateRequest updateRequest =
+            TransportRolloverAction.prepareIndicesAliasesRequest(sourceIndex, targetIndex, rolloverRequest);
+
+        final AliasAction[] actions = updateRequest.actions();
+        assertThat(actions.length, equalTo(3));
+        boolean foundAdd = false;
+        boolean foundRemove = false;
+        for (AliasAction action : actions) {
+            if (action.actionType() == AliasAction.Type.ADD) {
+                foundAdd = true;
+                assertThat(action.index(), equalTo(targetIndex));
+                assertThat(action.alias(), anyOf(equalTo(sourceAlias),
+                    equalTo(optionalTargetAlias)));
+            } else if (action.actionType() == AliasAction.Type.REMOVE) {
+                foundRemove = true;
+                assertThat(action.index(), equalTo(sourceIndex));
+                assertThat(action.alias(), equalTo(sourceAlias));
+            }
+        }
+        assertTrue(foundAdd);
+        assertTrue(foundRemove);
+    }
+
+    public void testValidation() throws Exception {
+        String index1 = randomAsciiOfLength(10);
+        String alias = randomAsciiOfLength(10);
+        String index2 = randomAsciiOfLength(10);
+        String aliasWithMultipleIndices = randomAsciiOfLength(10);
+        final Settings settings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .build();
+        final MetaData metaData = MetaData.builder()
+            .put(IndexMetaData.builder(index1)
+                .settings(settings)
+                .putAlias(AliasMetaData.builder(alias))
+                .putAlias(AliasMetaData.builder(aliasWithMultipleIndices))
+            )
+            .put(IndexMetaData.builder(index2)
+                .settings(settings)
+                .putAlias(AliasMetaData.builder(aliasWithMultipleIndices))
+            ).build();
+
+        try {
+            TransportRolloverAction.validate(metaData, new RolloverRequest(aliasWithMultipleIndices,
+                randomBoolean() ? null : randomAsciiOfLength(10)));
+            fail("expected to throw exception");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), equalTo("source alias maps to multiple indices"));
+        }
+
+        try {
+            TransportRolloverAction.validate(metaData, new RolloverRequest(randomFrom(index1, index2),
+                randomBoolean() ? null : randomAsciiOfLength(10)));
+            fail("expected to throw exception");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), equalTo("source alias is a concrete index"));
+        }
+
+        try {
+            TransportRolloverAction.validate(metaData, new RolloverRequest(randomAsciiOfLength(5),
+                randomBoolean() ? null : randomAsciiOfLength(10)));
+            fail("expected to throw exception");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), equalTo("source alias does not exist"));
+        }
+
+        TransportRolloverAction.validate(metaData, new RolloverRequest(alias,
+            randomBoolean() ? null : randomAsciiOfLength(10)));
+    }
+
+    public void testGenerateRolloverIndexName() throws Exception {
+        String index = randomAsciiOfLength(10);
+        assertThat(TransportRolloverAction.generateRolloverIndexName(index), not(equalTo(index)));
+        assertThat(TransportRolloverAction.generateRolloverIndexName("index"), equalTo("index-1"));
+        assertThat(TransportRolloverAction.generateRolloverIndexName("index-1"), equalTo("index-2"));
+        assertThat(TransportRolloverAction.generateRolloverIndexName("index-name-1"), equalTo("index-name-2"));
+        assertThat(TransportRolloverAction.generateRolloverIndexName("index-name"), equalTo("index-name-1"));
+    }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
new file mode 100644
index 00000000000..494ff408273
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
@@ -0,0 +1,35 @@
+{
+  "indices.rollover": {
+    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-rollover-index.html",
+    "methods": ["PUT", "POST"],
+    "url": {
+      "path": "/{alias}/_rollover/{target_alias}",
+      "paths": ["/{alias}/_rollover/{target_alias}"],
+      "parts": {
+        "alias": {
+          "type" : "string",
+          "required" : true,
+          "description" : "The name of the alias to rollover"
+        },
+        "target_alias": {
+          "type" : "string",
+          "required" : false,
+          "description" : "The name of optional target alias"
+        }
+      },
+      "params": {
+        "timeout": {
+          "type" : "time",
+          "description" : "Explicit operation timeout"
+        },
+        "master_timeout": {
+          "type" : "time",
+          "description" : "Specify timeout for connection to master"
+        }
+      }
+    },
+    "body": {
+      "description" : "The conditions that needs to be met for executing rollover"
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml
new file mode 100644
index 00000000000..a9acf8d150a
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml
@@ -0,0 +1,69 @@
+---
+"Rollover index via API":
+  # create index with alias
+  - do:
+      indices.create:
+        index: logs
+        body:
+          settings:
+            number_of_replicas: "0"
+          aliases:
+            logs_index: {}
+            logs_search: {}
+
+  # index document
+  - do:
+      index:
+        index: logs
+        type:  test
+        id:    "1"
+        body:  { "foo": "hello world" }
+
+  - do:
+      get:
+        index: logs_search
+        type:  test
+        id:    "1"
+
+  - match: { _index:   logs }
+  - match: { _type:    test   }
+  - match: { _id:      "1"     }
+  - match: { _source:  { foo: "hello world" } }
+
+  # perform alias rollover
+  - do:
+    indices.rollover:
+      alias: "logs_search"
+      body:
+        condition.max_docs: 1
+
+  - match: { old_index: logs }
+  - match: { new_index: logs-1 }
+
+  # ensure new index is created
+  - do:
+      indices.exists:
+        index: logs-1
+
+  - is_true: ''
+
+  # index into new index
+  - do:
+      index:
+        index: logs-1
+        type:  test
+        id:    "2"
+        body:  { "foo": "hello world" }
+
+  # check alias points to the new index
+  - do:
+      get:
+        index: logs_search
+        type:  test
+        id:    "2"
+
+  - match: { _index:   logs-1 }
+  - match: { _type:    test   }
+  - match: { _id:      "2"     }
+  - match: { _source:  { foo: "hello world" } }
+

From be0036542c4d1c40c0044cd22f45ea1d2534b8ec Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 3 Jun 2016 16:18:46 -0400
Subject: [PATCH 0639/1311] More complete exception message in settings tests

This commit adds additional details to the exception message assertions
in the YAML settings loader tests.
---
 .../loader/YamlSettingsLoaderTests.java       | 21 +++++++++++--------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java b/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java
index 618209cf114..7c956de8f9a 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/loader/YamlSettingsLoaderTests.java
@@ -19,17 +19,16 @@
 
 package org.elasticsearch.common.settings.loader;
 
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Collections;
-
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.test.ESTestCase;
 
-import static org.hamcrest.Matchers.containsString;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+
 import static org.hamcrest.Matchers.equalTo;
 
 public class YamlSettingsLoaderTests extends ESTestCase {
@@ -75,8 +74,10 @@ public class YamlSettingsLoaderTests extends ESTestCase {
         });
         assertEquals(e.getCause().getClass(), ElasticsearchParseException.class);
         String msg = e.getCause().getMessage();
-        assertTrue(msg, msg.contains("duplicate settings key [foo] found"));
-        assertTrue(msg, msg.contains("previous value [bar], current value [baz]"));
+        assertTrue(
+            msg,
+            msg.contains("duplicate settings key [foo] found at line number [2], column number [6], " +
+                "previous value [bar], current value [baz]"));
     }
 
     public void testMissingValue() throws Exception {
@@ -85,6 +86,8 @@ public class YamlSettingsLoaderTests extends ESTestCase {
         ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> {
             Settings.builder().loadFromPath(tmp);
         });
-        assertTrue(e.getMessage(), e.getMessage().contains("null-valued setting found for key [foo] found at line"));
+        assertTrue(
+            e.getMessage(),
+            e.getMessage().contains("null-valued setting found for key [foo] found at line number [1], column number [5]"));
     }
 }

From b891c46657a9d2e031395172f00464f1ac91c90d Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 3 Jun 2016 23:25:17 +0200
Subject: [PATCH 0640/1311] [TEST] remove status matcher and hasStatus
 assertion

All it does is checking the status code of a response, which can be done with a single line in each test
---
 .../plugins/ResponseHeaderPluginIT.java       |  7 ++---
 .../ContextAndHeaderTransportIT.java          |  5 ++--
 .../hamcrest/ElasticsearchAssertions.java     |  5 ----
 .../test/hamcrest/ElasticsearchMatchers.java  | 26 -------------------
 4 files changed, 4 insertions(+), 39 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java b/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
index 2f95fe20232..6b4fdee47c6 100644
--- a/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
+++ b/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
@@ -30,9 +30,6 @@ import org.elasticsearch.test.ESIntegTestCase.Scope;
 import java.util.Collection;
 import java.util.Collections;
 
-import static org.elasticsearch.rest.RestStatus.OK;
-import static org.elasticsearch.rest.RestStatus.UNAUTHORIZED;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasStatus;
 import static org.hamcrest.Matchers.equalTo;
 
 /**
@@ -60,13 +57,13 @@ public class ResponseHeaderPluginIT extends ESIntegTestCase {
             fail("request should have failed");
         } catch(ElasticsearchResponseException e) {
             ElasticsearchResponse response = e.getElasticsearchResponse();
-            assertThat(response, hasStatus(UNAUTHORIZED));
+            assertThat(response.getStatusLine().getStatusCode(), equalTo(401));
             assertThat(response.getHeader("Secret"), equalTo("required"));
         }
 
         try (ElasticsearchResponse authResponse = getRestClient().performRequest("GET", "/_protected", Collections.emptyMap(), null,
                 new BasicHeader("Secret", "password"))) {
-            assertThat(authResponse, hasStatus(OK));
+            assertThat(authResponse.getStatusLine().getStatusCode(), equalTo(200));
             assertThat(authResponse.getHeader("Secret"), equalTo("granted"));
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java b/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
index 6dd6114ce79..9d25414f4d8 100644
--- a/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
+++ b/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
@@ -63,12 +63,11 @@ import java.util.concurrent.CopyOnWriteArrayList;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.elasticsearch.rest.RestStatus.OK;
 import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasStatus;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
@@ -222,7 +221,7 @@ public class ContextAndHeaderTransportIT extends ESIntegTestCase {
         try (ElasticsearchResponse response = getRestClient().performRequest(
                 "GET", "/" + queryIndex + "/_search", Collections.emptyMap(), null,
                 new BasicHeader(randomHeaderKey, randomHeaderValue), new BasicHeader(relevantHeaderName, randomHeaderValue))) {
-            assertThat(response, hasStatus(OK));
+            assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
             List<RequestAndHeaders> searchRequests = getRequests(SearchRequest.class);
             assertThat(searchRequests, hasSize(greaterThan(0)));
             for (RequestAndHeaders requestAndHeaders : searchRequests) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
index 0fe65ccf752..51d15e019a4 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
@@ -42,7 +42,6 @@ import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.action.support.broadcast.BroadcastResponse;
 import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.client.ElasticsearchResponse;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
@@ -486,10 +485,6 @@ public class ElasticsearchAssertions {
         return new ElasticsearchMatchers.SearchHitHasScoreMatcher(score);
     }
 
-    public static Matcher<ElasticsearchResponse> hasStatus(RestStatus restStatus) {
-        return new ElasticsearchMatchers.ElasticsearchResponseHasStatusMatcher(restStatus);
-    }
-
     public static <T extends Query> T assertBooleanSubQuery(Query query, Class<T> subqueryType, int i) {
         assertThat(query, instanceOf(BooleanQuery.class));
         BooleanQuery q = (BooleanQuery) query;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchMatchers.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchMatchers.java
index c40e434a5a7..f49cc3bd39e 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchMatchers.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchMatchers.java
@@ -18,8 +18,6 @@
  */
 package org.elasticsearch.test.hamcrest;
 
-import org.elasticsearch.client.ElasticsearchResponse;
-import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
 import org.hamcrest.Description;
 import org.hamcrest.TypeSafeMatcher;
@@ -117,28 +115,4 @@ public class ElasticsearchMatchers {
             description.appendText("searchHit score should be ").appendValue(score);
         }
     }
-
-    public static class ElasticsearchResponseHasStatusMatcher extends TypeSafeMatcher<ElasticsearchResponse> {
-
-        private RestStatus restStatus;
-
-        public ElasticsearchResponseHasStatusMatcher(RestStatus restStatus) {
-            this.restStatus = restStatus;
-        }
-
-        @Override
-        protected boolean matchesSafely(ElasticsearchResponse response) {
-            return response.getStatusLine().getStatusCode() == restStatus.getStatus();
-        }
-
-        @Override
-        public void describeMismatchSafely(final ElasticsearchResponse response, final Description mismatchDescription) {
-            mismatchDescription.appendText(" was ").appendValue(response.getStatusLine().getStatusCode());
-        }
-
-        @Override
-        public void describeTo(final Description description) {
-            description.appendText("Elasticsearch response status code should be ").appendValue(restStatus.getStatus());
-        }
-    }
 }

From c0a3a200712607ca07a738541c6b6b2e32609111 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Fri, 3 Jun 2016 23:31:06 +0200
Subject: [PATCH 0641/1311] painless: Add support for the new Java 9
 MethodHandles#arrayLength() factory (see
 https://bugs.openjdk.java.net/browse/JDK-8156915)

---
 .../java/org/elasticsearch/painless/Def.java  | 32 +++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index bd8e09f504b..f27f0ab20a9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -53,7 +53,7 @@ public final class Def {
 
     /** Helper class for isolating MethodHandles and methods to get the length of arrays
      * (to emulate a "arraystore" bytecode using MethodHandles).
-     * This should really be a method in {@link MethodHandles} class!
+     * See: https://bugs.openjdk.java.net/browse/JDK-8156915
      */
     @SuppressWarnings("unused") // getArrayLength() methods are are actually used, javac just does not know :)
     private static final class ArrayLengthHelper {
@@ -103,6 +103,8 @@ public final class Def {
     private static final MethodHandle LIST_GET;
     /** pointer to List.set(int,Object) */
     private static final MethodHandle LIST_SET;
+    /** factory for arraylength MethodHandle (intrinsic) from Java 9 */
+    private static final MethodHandle JAVA9_ARRAY_LENGTH_MH_FACTORY;
 
     static {
         final Lookup lookup = MethodHandles.publicLookup();
@@ -115,11 +117,37 @@ public final class Def {
         } catch (final ReflectiveOperationException roe) {
             throw new AssertionError(roe);
         }
+        
+        // lookup up the factory for arraylength MethodHandle (intrinsic) from Java 9:
+        // https://bugs.openjdk.java.net/browse/JDK-8156915
+        MethodHandle arrayLengthMHFactory;
+        try {
+            arrayLengthMHFactory = lookup.findStatic(MethodHandles.class, "arrayLength",
+                MethodType.methodType(MethodHandle.class, Class.class));
+        } catch (final ReflectiveOperationException roe) {
+            arrayLengthMHFactory = null;
+        }
+        JAVA9_ARRAY_LENGTH_MH_FACTORY = arrayLengthMHFactory;
     }
 
+    /** Hack to rethrow unknown Exceptions from {@link MethodHandle#invokeExact}: */
+    @SuppressWarnings("unchecked")
+    private static <T extends Throwable> void rethrow(Throwable t) throws T {
+        throw (T) t;
+    }
+    
     /** Returns an array length getter MethodHandle for the given array type */
     static MethodHandle arrayLengthGetter(Class<?> arrayType) {
-        return ArrayLengthHelper.arrayLengthGetter(arrayType);
+        if (JAVA9_ARRAY_LENGTH_MH_FACTORY != null) {
+            try {
+                return (MethodHandle) JAVA9_ARRAY_LENGTH_MH_FACTORY.invokeExact(arrayType);
+            } catch (Throwable t) {
+                rethrow(t);
+                throw new AssertionError(t);
+            }
+        } else {
+            return ArrayLengthHelper.arrayLengthGetter(arrayType);
+        }
     }
 
     /**

From b79fd4c360789c8ff917185db305f11f1ebbdd81 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 3 Jun 2016 17:56:03 -0400
Subject: [PATCH 0642/1311] Remove log file

Looks like accidentally committed.
---
 docs/out | 11165 -----------------------------------------------------
 1 file changed, 11165 deletions(-)
 delete mode 100644 docs/out

diff --git a/docs/out b/docs/out
deleted file mode 100644
index 5a0af906e79..00000000000
--- a/docs/out
+++ /dev/null
@@ -1,11165 +0,0 @@
-:buildSrc:compileJava UP-TO-DATE
-:buildSrc:compileGroovy
-:buildSrc:writeVersionProperties UP-TO-DATE
-:buildSrc:processResources UP-TO-DATE
-:buildSrc:classes
-:buildSrc:jar
-:buildSrc:assemble
-:buildSrc:compileTestJava UP-TO-DATE
-:buildSrc:compileTestGroovy UP-TO-DATE
-:buildSrc:processTestResources UP-TO-DATE
-:buildSrc:testClasses UP-TO-DATE
-:buildSrc:test UP-TO-DATE
-:buildSrc:check UP-TO-DATE
-:buildSrc:build
-=======================================
-Elasticsearch Build Hamster says Hello!
-=======================================
-  Gradle Version        : 2.13
-  OS Info               : Linux 3.13.0-39-generic (amd64)
-  JDK Version           : Oracle Corporation 1.8.0_65 [Java HotSpot(TM) 64-Bit Server VM 25.65-b01]
-  JAVA_HOME             : /usr/java/jdk1.8.0_65
-:docs:clean
-:docs:processTestResources UP-TO-DATE
-:rest-api-spec:compileJava UP-TO-DATE
-:rest-api-spec:processResources UP-TO-DATE
-:rest-api-spec:classes UP-TO-DATE
-:rest-api-spec:jar UP-TO-DATE
-:docs:copyRestSpec
-:docs:buildRestTests
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "title": {
-          "type":     "text",
-          "analyzer": "standard"
-        }
-      }
-    }
-  }
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "my_number": {
-          "type": "long",
-          "fields": {
-            "keyword": {
-              "type":  "keyword"
-            }
-          }
-        }
-      }
-    }
-  }
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "analyzer": "whitespace",
-  "text":     "The quick brown fox."
-}
-emit snippet
-body part: 
-{
-  "tokenizer": "standard",
-  "filter":  [ "lowercase", "asciifolding" ],
-  "text":      "Is this déja vu?"
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "std_folded": {
-          "type": "custom",
-          "tokenizer": "standard",
-          "filter": [
-            "lowercase",
-            "asciifolding"
-          ]
-        }
-      }
-    }
-  },
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "my_text": {
-          "type": "text",
-          "analyzer": "std_folded"
-        }
-      }
-    }
-  }
-}
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "analyzer": "std_folded",
-  "text":     "Is this déjà vu?"
-}
-emit snippet
-body part: 
-{
-  "field": "my_text",
-  "text":  "Is this déjà vu?"
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "tokenizer": "classic",
-  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "my_analyzer": {
-          "tokenizer": "my_tokenizer"
-        }
-      },
-      "tokenizer": {
-        "my_tokenizer": {
-          "type": "classic",
-          "max_token_length": 5
-        }
-      }
-    }
-  }
-}
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "analyzer": "my_analyzer",
-  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "tokenizer": "whitespace",
-  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "tokenizer": "ngram",
-  "text": "Quick Fox"
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "my_analyzer": {
-          "tokenizer": "my_tokenizer"
-        }
-      },
-      "tokenizer": {
-        "my_tokenizer": {
-          "type": "ngram",
-          "min_gram": 3,
-          "max_gram": 3,
-          "token_chars": [
-            "letter",
-            "digit"
-          ]
-        }
-      }
-    }
-  }
-}
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "analyzer": "my_analyzer",
-  "text": "2 Quick Foxes."
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "tokenizer": "letter",
-  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "tokenizer": "pattern",
-  "text": "The foo_bar_size's default is 5."
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "my_analyzer": {
-          "tokenizer": "my_tokenizer"
-        }
-      },
-      "tokenizer": {
-        "my_tokenizer": {
-          "type": "pattern",
-          "pattern": ","
-        }
-      }
-    }
-  }
-}
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "analyzer": "my_analyzer",
-  "text": "comma,separated,values"
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "my_analyzer": {
-          "tokenizer": "my_tokenizer"
-        }
-      },
-      "tokenizer": {
-        "my_tokenizer": {
-          "type": "pattern",
-          "pattern": "\"((?:\\\\\"|[^\"]|\\\\\")+)\"",
-          "group": 1
-        }
-      }
-    }
-  }
-}
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "analyzer": "my_analyzer",
-  "text": "\"value\", \"value with embedded \\\" quote\""
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "tokenizer": "edge_ngram",
-  "text": "Quick Fox"
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "my_analyzer": {
-          "tokenizer": "my_tokenizer"
-        }
-      },
-      "tokenizer": {
-        "my_tokenizer": {
-          "type": "edge_ngram",
-          "min_gram": 2,
-          "max_gram": 10,
-          "token_chars": [
-            "letter",
-            "digit"
-          ]
-        }
-      }
-    }
-  }
-}
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "analyzer": "my_analyzer",
-  "text": "2 Quick Foxes."
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "autocomplete": {
-          "tokenizer": "autocomplete",
-          "filter": [
-            "lowercase"
-          ]
-        },
-        "autocomplete_search": {
-          "tokenizer": "lowercase"
-        }
-      },
-      "tokenizer": {
-        "autocomplete": {
-          "type": "edge_ngram",
-          "min_gram": 2,
-          "max_gram": 10,
-          "token_chars": [
-            "letter"
-          ]
-        }
-      }
-    }
-  },
-  "mappings": {
-    "doc": {
-      "properties": {
-        "title": {
-          "type": "text",
-          "analyzer": "autocomplete",
-          "search_analyzer": "autocomplete_search"
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "title": "Quick Foxes"
-}
-emit snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "match": {
-      "title": {
-        "query": "Quick Fo",
-        "operator": "and"
-      }
-    }
-  }
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "tokenizer": "uax_url_email",
-  "text": "Email me at john.smith@global-international.com"
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "my_analyzer": {
-          "tokenizer": "my_tokenizer"
-        }
-      },
-      "tokenizer": {
-        "my_tokenizer": {
-          "type": "uax_url_email",
-          "max_token_length": 5
-        }
-      }
-    }
-  }
-}
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "analyzer": "my_analyzer",
-  "text": "john.smith@global-international.com"
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "tokenizer": "path_hierarchy",
-  "text": "/one/two/three"
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "my_analyzer": {
-          "tokenizer": "my_tokenizer"
-        }
-      },
-      "tokenizer": {
-        "my_tokenizer": {
-          "type": "path_hierarchy",
-          "delimiter": "-",
-          "replacement": "/",
-          "skip": 2
-        }
-      }
-    }
-  }
-}
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "analyzer": "my_analyzer",
-  "text": "one-two-three-four-five"
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "tokenizer": "standard",
-  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "my_analyzer": {
-          "tokenizer": "my_tokenizer"
-        }
-      },
-      "tokenizer": {
-        "my_tokenizer": {
-          "type": "standard",
-          "max_token_length": 5
-        }
-      }
-    }
-  }
-}
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "analyzer": "my_analyzer",
-  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "tokenizer": "keyword",
-  "text": "New York"
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "tokenizer": "thai",
-  "text": "�ารที่ได้ต้อง�สดงว่างานดี"
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "tokenizer": "lowercase",
-  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "analyzer": "standard",
-  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "my_english_analyzer": {
-          "type": "standard",
-          "max_token_length": 5,
-          "stopwords": "_english_"
-        }
-      }
-    }
-  }
-}
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "analyzer": "my_english_analyzer",
-  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "analyzer": "simple",
-  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "analyzer": "whitespace",
-  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "analyzer": "keyword",
-  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "std_english": {
-          "type":      "standard",
-          "stopwords": "_english_"
-        }
-      }
-    }
-  },
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "my_text": {
-          "type":     "text",
-          "analyzer": "standard",
-          "fields": {
-            "english": {
-              "type":     "text",
-              "analyzer": "std_english"
-            }
-          }
-        }
-      }
-    }
-  }
-}
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "field": "my_text",
-  "text": "The old brown cow"
-}
-emit snippet
-body part: 
-{
-  "field": "my_text.english",
-  "text": "The old brown cow"
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "analyzer": "stop",
-  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "my_stop_analyzer": {
-          "type": "stop",
-          "stopwords": ["the", "over"]
-        }
-      }
-    }
-  }
-}
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "analyzer": "my_stop_analyzer",
-  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "my_custom_analyzer": {
-          "type":      "custom",
-          "tokenizer": "standard",
-          "char_filter": [
-            "html_strip"
-          ],
-          "filter": [
-            "lowercase",
-            "asciifolding"
-          ]
-        }
-      }
-    }
-  }
-}
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "analyzer": "my_custom_analyzer",
-  "text": "Is this <b>déjà vu</b>?"
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "my_custom_analyzer": {
-          "type": "custom",
-          "char_filter": [
-            "emoticons"
-          ],
-          "tokenizer": "punctuation",
-          "filter": [
-            "lowercase",
-            "english_stop"
-          ]
-        }
-      },
-      "tokenizer": {
-        "punctuation": {
-          "type": "pattern",
-          "pattern": "[ .,!?]"
-        }
-      },
-      "char_filter": {
-        "emoticons": {
-          "type": "mapping",
-          "mappings": [
-            ":) => _happy_",
-            ":( => _sad_"
-          ]
-        }
-      },
-      "filter": {
-        "english_stop": {
-          "type": "stop",
-          "stopwords": "_english_"
-        }
-      }
-    }
-  }
-}
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "analyzer": "my_custom_analyzer",
-  "text":     "I'm a :) person, and you?"
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "analyzer": "pattern",
-  "text": "The 2 QUICK Brown-Foxes jumped over the lazy dog's bone."
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "my_email_analyzer": {
-          "type":      "pattern",
-          "pattern":   "\\W|_",
-          "lowercase": true
-        }
-      }
-    }
-  }
-}
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "analyzer": "my_email_analyzer",
-  "text": "John_Smith@foo-bar.com"
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "camel": {
-          "type": "pattern",
-          "pattern": "([^\\p{L}\\d]+)|(?<=\\D)(?=\\d)|(?<=\\d)(?=\\D)|(?<=[\\p{L}&&[^\\p{Lu}]])(?=\\p{Lu})|(?<=\\p{Lu})(?=\\p{Lu}[\\p{L}&&[^\\p{Lu}]])"
-        }
-      }
-    }
-  }
-}
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "analyzer": "camel",
-  "text": "MooseX::FTPClass2_beta"
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "analyzer": "fingerprint",
-  "text": "Yes yes, Gödel said this sentence is consistent and."
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "my_fingerprint_analyzer": {
-          "type": "fingerprint",
-          "stopwords": "_english_"
-        }
-      }
-    }
-  }
-}
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "analyzer": "my_fingerprint_analyzer",
-  "text": "Yes yes, Gödel said this sentence is consistent and."
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "tokenizer":      "keyword",
-  "char_filter":  [ "html_strip" ],
-  "text": "<p>I&apos;m so <b>happy</b>!</p>"
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "my_analyzer": {
-          "tokenizer": "keyword",
-          "char_filter": ["my_char_filter"]
-        }
-      },
-      "char_filter": {
-        "my_char_filter": {
-          "type": "html_strip",
-          "escaped_tags": ["b"]
-        }
-      }
-    }
-  }
-}
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "analyzer": "my_analyzer",
-  "text": "<p>I&apos;m so <b>happy</b>!</p>"
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "my_analyzer": {
-          "tokenizer": "keyword",
-          "char_filter": [
-            "my_char_filter"
-          ]
-        }
-      },
-      "char_filter": {
-        "my_char_filter": {
-          "type": "mapping",
-          "mappings": [
-            "Ù  => 0",
-            "Ù¡ => 1",
-            "Ù¢ => 2",
-            "Ù£ => 3",
-            "Ù¤ => 4",
-            "Ù¥ => 5",
-            "Ù¦ => 6",
-            "Ù§ => 7",
-            "Ù¨ => 8",
-            "Ù© => 9"
-          ]
-        }
-      }
-    }
-  }
-}
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "analyzer": "my_analyzer",
-  "text": "My license plate is ٢٥٠١٥"
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "my_analyzer": {
-          "tokenizer": "standard",
-          "char_filter": [
-            "my_char_filter"
-          ]
-        }
-      },
-      "char_filter": {
-        "my_char_filter": {
-          "type": "mapping",
-          "mappings": [
-            ":) => _happy_",
-            ":( => _sad_"
-          ]
-        }
-      }
-    }
-  }
-}
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "analyzer": "my_analyzer",
-  "text": "I'm delighted about it :("
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "my_analyzer": {
-          "tokenizer": "standard",
-          "char_filter": [
-            "my_char_filter"
-          ]
-        }
-      },
-      "char_filter": {
-        "my_char_filter": {
-          "type": "pattern_replace",
-          "pattern": "(\\d+)-(?=\\d)",
-          "replacement": "$1_"
-        }
-      }
-    }
-  }
-}
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "analyzer": "my_analyzer",
-  "text": "My credit card is 123-456-789"
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "analyzer": {
-        "my_analyzer": {
-          "tokenizer": "standard",
-          "char_filter": [
-            "my_char_filter"
-          ],
-          "filter": [
-            "lowercase"
-          ]
-        }
-      },
-      "char_filter": {
-        "my_char_filter": {
-          "type": "pattern_replace",
-          "pattern": "(?<=\\p{Lower})(?=\\p{Upper})",
-          "replacement": " "
-        }
-      }
-    }
-  },
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "text": {
-          "type": "text",
-          "analyzer": "my_analyzer"
-        }
-      }
-    }
-  }
-}
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "analyzer": "my_analyzer",
-  "text": "The fooBarBaz method"
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-query part: refresh
-body part: 
-{
-  "text": "The fooBarBaz method"
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "match": {
-      "text": "bar"
-    }
-  },
-  "highlight": {
-    "fields": {
-      "text": {}
-    }
-  }
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query" : {
-        "match_all": {}
-    },
-    "fielddata_fields" : ["test1", "test2"]
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query" : {
-        "term" : { "user" : "kimchy" }
-    }
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "from" : 0, "size" : 10,
-    "query" : {
-        "term" : { "user" : "kimchy" }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "_source": false,
-    "query" : {
-        "term" : { "user" : "kimchy" }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "_source": "obj.*",
-    "query" : {
-        "term" : { "user" : "kimchy" }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "_source": [ "obj1.*", "obj2.*" ],
-    "query" : {
-        "term" : { "user" : "kimchy" }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "_source": {
-        "include": [ "obj1.*", "obj2.*" ],
-        "exclude": [ "*.description" ]
-    },
-    "query" : {
-        "term" : { "user" : "kimchy" }
-    }
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "explain": true,
-    "query" : {
-        "term" : { "user" : "kimchy" }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query" : {
-        "match_all": {}
-    },
-    "script_fields" : {
-        "test1" : {
-            "script" : "doc['my_field_name'].value * 2"
-        },
-        "test2" : {
-            "script" : {
-                "inline": "doc['my_field_name'].value * factor",
-                "params" : {
-                    "factor"  : 2.0
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-    {
-        "query" : {
-            "match_all": {}
-        },
-        "script_fields" : {
-            "test1" : {
-                "script" : "_source.obj1.obj2"
-            }
-        }
-    }
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query" : {
-        "match": { "user": "kimchy" }
-    },
-    "highlight" : {
-        "fields" : {
-            "content" : {}
-        }
-    }
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query" : {
-        "match": { "user": "kimchy" }
-    },
-    "highlight" : {
-        "fields" : {
-            "content" : {"type" : "plain"}
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query" : {
-        "match": { "user": "kimchy" }
-    },
-    "highlight" : {
-        "fields" : {
-            "content" : {"force_source" : true}
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query" : {
-        "match": { "user": "kimchy" }
-    },
-    "highlight" : {
-        "pre_tags" : ["<tag1>"],
-        "post_tags" : ["</tag1>"],
-        "fields" : {
-            "_all" : {}
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query" : {
-        "match": { "user": "kimchy" }
-    },
-    "highlight" : {
-        "pre_tags" : ["<tag1>", "<tag2>"],
-        "post_tags" : ["</tag1>", "</tag2>"],
-        "fields" : {
-            "_all" : {}
-        }
-    }
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query" : {
-        "match": { "user": "kimchy" }
-    },
-    "highlight" : {
-        "tags_schema" : "styled",
-        "fields" : {
-            "content" : {}
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query" : {
-        "match": { "user": "kimchy" }
-    },
-    "highlight" : {
-        "fields" : {
-            "content" : {"fragment_size" : 150, "number_of_fragments" : 3}
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query" : {
-        "match": { "user": "kimchy" }
-    },
-    "highlight" : {
-        "order" : "score",
-        "fields" : {
-            "content" : {"fragment_size" : 150, "number_of_fragments" : 3}
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query" : {
-        "match": { "user": "kimchy" }
-    },
-    "highlight" : {
-        "fields" : {
-            "_all" : {},
-            "bio.title" : {"number_of_fragments" : 0}
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query" : {
-        "match": { "user": "kimchy" }
-    },
-    "highlight" : {
-        "fields" : {
-            "content" : {
-                "fragment_size" : 150,
-                "number_of_fragments" : 3,
-                "no_match_size": 150
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "fields": [ "_id" ],
-    "query" : {
-        "match": {
-            "content": {
-                "query": "foo bar"
-            }
-        }
-    },
-    "rescore": {
-        "window_size": 50,
-        "query": {
-            "rescore_query" : {
-                "match_phrase": {
-                    "content": {
-                        "query": "foo bar",
-                        "phrase_slop": 1
-                    }
-                }
-            },
-            "rescore_query_weight" : 10
-        }
-    },
-    "highlight" : {
-        "order" : "score",
-        "fields" : {
-            "content" : {
-                "fragment_size" : 150,
-                "number_of_fragments" : 3,
-                "highlight_query": {
-                    "bool": {
-                        "must": {
-                            "match": {
-                                "content": {
-                                    "query": "foo bar"
-                                }
-                            }
-                        },
-                        "should": {
-                            "match_phrase": {
-                                "content": {
-                                    "query": "foo bar",
-                                    "phrase_slop": 1,
-                                    "boost": 10.0
-                                }
-                            }
-                        },
-                        "minimum_should_match": 0
-                    }
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query" : {
-        "match": { "user": "kimchy" }
-    },
-    "highlight" : {
-        "number_of_fragments" : 3,
-        "fragment_size" : 150,
-        "fields" : {
-            "_all" : { "pre_tags" : ["<em>"], "post_tags" : ["</em>"] },
-            "bio.title" : { "number_of_fragments" : 0 },
-            "bio.author" : { "number_of_fragments" : 0 },
-            "bio.content" : { "number_of_fragments" : 5, "order" : "score" }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query" : {
-        "match": { "user": "kimchy" }
-    },
-    "highlight" : {
-        "require_field_match": false,
-        "fields": {
-                "_all" : { "pre_tags" : ["<em>"], "post_tags" : ["</em>"] }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "query_string": {
-            "query": "content.plain:running scissors",
-            "fields": ["content"]
-        }
-    },
-    "highlight": {
-        "order": "score",
-        "fields": {
-            "content": {
-                "matched_fields": ["content", "content.plain"],
-                "type" : "fvh"
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "query_string": {
-            "query": "running scissors",
-            "fields": ["content", "content.plain^10"]
-        }
-    },
-    "highlight": {
-        "order": "score",
-        "fields": {
-            "content": {
-                "matched_fields": ["content", "content.plain"],
-                "type" : "fvh"
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "query_string": {
-            "query": "running scissors",
-            "fields": ["content", "content.plain^10"]
-        }
-    },
-    "highlight": {
-        "order": "score",
-        "fields": {
-            "content": {
-                "matched_fields": ["content.plain"],
-                "type" : "fvh"
-            }
-        }
-    }
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "min_score": 0.5,
-    "query" : {
-        "term" : { "user" : "kimchy" }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-query part: preference=xyzabc123
-body part: 
-{
-    "query": {
-        "match": {
-            "title": "elasticsearch"
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool" : {
-            "should" : [
-                {"match" : { "name.first" : {"query" : "shay", "_name" : "first"} }},
-                {"match" : { "name.last" : {"query" : "banon", "_name" : "last"} }}
-            ],
-            "filter" : {
-                "terms" : {
-                    "name.last" : ["banon", "kimchy"],
-                    "_name" : "test"
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "indices_boost" : {
-        "index1" : 1.4,
-        "index2" : 1.3
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "mappings": {
-        "my_type": {
-            "properties": {
-                "post_date": { "type": "date" },
-                "user": {
-                    "type": "keyword"
-                },
-                "name": {
-                    "type": "keyword"
-                },
-                "age": { "type": "integer" }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "sort" : [
-        { "post_date" : {"order" : "asc"}},
-        "user",
-        { "name" : "desc" },
-        { "age" : "desc" },
-        "_score"
-    ],
-    "query" : {
-        "term" : { "user" : "kimchy" }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-query part: refresh
-body part: 
-{
-   "product": "chocolate",
-    "price": [20, 4]
-}
-emit snippet
-body part: 
-{
-   "query" : {
-      "term" : { "product" : "chocolate" }
-   },
-   "sort" : [
-      {"price" : {"order" : "asc", "mode" : "avg"}}
-   ]
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-   "query" : {
-      "term" : { "product" : "chocolate" }
-   },
-   "sort" : [
-       {
-          "offer.price" : {
-             "mode" :  "avg",
-             "order" : "asc",
-             "nested_path" : "offer",
-             "nested_filter" : {
-                "term" : { "offer.color" : "blue" }
-             }
-          }
-       }
-    ]
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "sort" : [
-        { "price" : {"missing" : "_last"} }
-    ],
-    "query" : {
-        "term" : { "product" : "chocolate" }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "sort" : [
-        { "price" : {"unmapped_type" : "long"} }
-    ],
-    "query" : {
-        "term" : { "product" : "chocolate" }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "sort" : [
-        {
-            "_geo_distance" : {
-                "pin.location" : [-70, 40],
-                "order" : "asc",
-                "unit" : "km",
-		"mode" : "min",
-		"distance_type" : "sloppy_arc"
-            }
-        }
-    ],
-    "query" : {
-        "term" : { "user" : "kimchy" }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "sort" : [
-        {
-            "_geo_distance" : {
-                "pin.location" : {
-                    "lat" : 40,
-                    "lon" : -70
-                },
-                "order" : "asc",
-                "unit" : "km"
-            }
-        }
-    ],
-    "query" : {
-        "term" : { "user" : "kimchy" }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "sort" : [
-        {
-            "_geo_distance" : {
-                "pin.location" : "40,-70",
-                "order" : "asc",
-                "unit" : "km"
-            }
-        }
-    ],
-    "query" : {
-        "term" : { "user" : "kimchy" }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "sort" : [
-        {
-            "_geo_distance" : {
-                "pin.location" : "drm3btev3e86",
-                "order" : "asc",
-                "unit" : "km"
-            }
-        }
-    ],
-    "query" : {
-        "term" : { "user" : "kimchy" }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "sort" : [
-        {
-            "_geo_distance" : {
-                "pin.location" : [-70, 40],
-                "order" : "asc",
-                "unit" : "km"
-            }
-        }
-    ],
-    "query" : {
-        "term" : { "user" : "kimchy" }
-    }
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query" : {
-        "term" : { "user" : "kimchy" }
-    },
-    "sort" : {
-        "_script" : {
-            "type" : "number",
-            "script" : {
-                "inline": "doc['field_name'].value * factor",
-                "params" : {
-                    "factor" : 1.1
-                }
-            },
-            "order" : "asc"
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "track_scores": true,
-    "sort" : [
-        { "post_date" : {"order" : "desc"} },
-        { "name" : "desc" },
-        { "age" : "desc" }
-    ],
-    "query" : {
-        "term" : { "user" : "kimchy" }
-    }
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "version": true,
-    "query" : {
-        "term" : { "user" : "kimchy" }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "fields" : ["user", "postDate"],
-    "query" : {
-        "term" : { "user" : "kimchy" }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "fields" : [],
-    "query" : {
-        "term" : { "user" : "kimchy" }
-    }
-}
-handle snippet
-null
-emit snippet
-body part: 
-{
-    "mappings": {
-        "item": {
-            "properties": {
-                "brand": { "type": "keyword"},
-                "color": { "type": "keyword"},
-                "model": { "type": "keyword"}
-            }
-        }
-    }
-}
-emit snippet
-query part: refresh
-body part: 
-{
-    "brand": "gucci",
-    "color": "red",
-    "model": "slim"
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "bool": {
-      "filter": [
-        { "term": { "color": "red"   }},
-        { "term": { "brand": "gucci" }}
-      ]
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "bool": {
-      "filter": [
-        { "term": { "color": "red"   }},
-        { "term": { "brand": "gucci" }}
-      ]
-    }
-  },
-  "aggs": {
-    "models": {
-      "terms": { "field": "model" }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "bool": {
-      "filter": {
-        "term": { "brand": "gucci" }
-      }
-    }
-  },
-  "aggs": {
-    "colors": {
-      "terms": { "field": "color" }
-    },
-    "color_red": {
-      "filter": {
-        "term": { "color": "red" }
-      },
-      "aggs": {
-        "models": {
-          "terms": { "field": "model" }
-        }
-      }
-    }
-  },
-  "post_filter": {
-    "term": { "color": "red" }
-  }
-}
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-emit snippet
-body part: 
-{
-    "size": 10,
-    "query": {
-        "match" : {
-            "title" : "elasticsearch"
-        }
-    },
-    "sort": [
-        {"date": "asc"},
-        {"_uid": "desc"}
-    ]
-}
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-emit snippet
-body part: 
-{
-    "size": 10,
-    "query": {
-        "match" : {
-            "title" : "elasticsearch"
-        }
-    },
-    "search_after": [1463538857, "tweet#654323"],
-    "sort": [
-        {"date": "asc"},
-        {"_uid": "desc"}
-    ]
-}
-handle snippet
-null
-emit snippet
-query part: refresh
-body part: 
-{"index":{"_id":1}}
-{"user" : "kimchy", "post_date" : "2009-11-15T14:12:12", "message" : "trying out Elasticsearch"}
-{"index":{"_id":2}}
-{"user" : "kimchi", "post_date" : "2009-11-15T14:12:13", "message" : "My username is similar to @kimchy!"}
-handle snippet
-test snippet
-emit snippet
-query part: q=user:foo
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query" : {
-    "bool" : {
-      "must" : {
-        "query_string" : {
-          "query" : "*:*"
-        }
-      },
-      "filter" : {
-        "term" : { "user" : "kimchy" }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-query part: q=post_date:foo
-handle snippet
-handle snippet
-test snippet
-emit snippet
-query part: q=post_date:foo&explain=true
-handle snippet
-handle snippet
-test snippet
-emit snippet
-query part: rewrite=true
-body part: 
-{
-  "query": {
-    "match": {
-      "user": {
-        "query": "kimchy",
-        "fuzziness": "auto"
-      }
-    }
-  }
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-query part: rewrite=true
-body part: 
-{
-  "query": {
-    "more_like_this": {
-      "like": {
-        "_id": "2"
-      },
-      "boost_terms": 1
-    }
-  }
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-emit snippet
-query part: q=user:kimchy
-handle snippet
-handle snippet
-test snippet
-emit snippet
-query part: refresh
-body part: 
-{
-    "user": "kimchy"
-}
-emit snippet
-query part: q=user:kimchy
-emit snippet
-body part: 
-{
-    "query" : {
-        "term" : { "user" : "kimchy" }
-    }
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "multi_match" : {
-      "query":    "this is a test",
-      "fields": [ "subject", "message" ]
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "multi_match" : {
-      "query":    "Will Smith",
-      "fields": [ "title", "*_name" ]
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "multi_match" : {
-      "query" : "this is a test",
-      "fields" : [ "subject^3", "message" ]
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "multi_match" : {
-      "query":      "brown fox",
-      "type":       "best_fields",
-      "fields":     [ "subject", "message" ],
-      "tie_breaker": 0.3
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "dis_max": {
-      "queries": [
-        { "match": { "subject": "brown fox" }},
-        { "match": { "message": "brown fox" }}
-      ],
-      "tie_breaker": 0.3
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "multi_match" : {
-      "query":      "Will Smith",
-      "type":       "best_fields",
-      "fields":     [ "first_name", "last_name" ],
-      "operator":   "and"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "multi_match" : {
-      "query":      "quick brown fox",
-      "type":       "most_fields",
-      "fields":     [ "title", "title.original", "title.shingles" ]
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "bool": {
-      "should": [
-        { "match": { "title":          "quick brown fox" }},
-        { "match": { "title.original": "quick brown fox" }},
-        { "match": { "title.shingles": "quick brown fox" }}
-      ]
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "multi_match" : {
-      "query":      "quick brown f",
-      "type":       "phrase_prefix",
-      "fields":     [ "subject", "message" ]
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "dis_max": {
-      "queries": [
-        { "match_phrase_prefix": { "subject": "quick brown f" }},
-        { "match_phrase_prefix": { "message": "quick brown f" }}
-      ]
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "multi_match" : {
-      "query":      "Will Smith",
-      "type":       "cross_fields",
-      "fields":     [ "first_name", "last_name" ],
-      "operator":   "and"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "multi_match" : {
-      "query":      "Jon",
-      "type":       "cross_fields",
-      "fields":     [
-        "first", "first.edge",
-        "last",  "last.edge"
-      ]
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "bool": {
-      "should": [
-        {
-          "multi_match" : {
-            "query":      "Will Smith",
-            "type":       "cross_fields",
-            "fields":     [ "first", "last" ],
-            "minimum_should_match": "50%"
-          }
-        },
-        {
-          "multi_match" : {
-            "query":      "Will Smith",
-            "type":       "cross_fields",
-            "fields":     [ "*.edge" ]
-          }
-        }
-      ]
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-   "multi_match" : {
-      "query":      "Jon",
-      "type":       "cross_fields",
-      "analyzer":   "standard",
-      "fields":     [ "first", "last", "*.edge" ]
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "query_string" : {
-            "default_field" : "content",
-            "query" : "this AND that OR thus"
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "query_string" : {
-            "fields" : ["content", "name"],
-            "query" : "this AND that"
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "query_string": {
-            "query": "(content:this OR name:this) AND (content:that OR name:that)"
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "query_string" : {
-            "fields" : ["content", "name^5"],
-            "query" : "this AND that OR thus",
-            "use_dis_max" : true
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "query_string" : {
-            "fields" : ["city.*"],
-            "query" : "this AND that OR thus",
-            "use_dis_max" : true
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "query_string" : {
-            "fields" : ["content", "name.*^5"],
-            "query" : "this AND that OR thus",
-            "use_dis_max" : true
-        }
-    }
-}
-handle snippet
-null
-emit snippet
-body part: 
-{
-    "mappings": {
-        "location": {
-            "properties": {
-                "pin": {
-                    "properties": {
-                        "location": {
-                            "type": "geo_point"
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-emit snippet
-body part: 
-{
-    "pin" : {
-        "location" : {
-            "lat" : 40.12,
-            "lon" : -71.34
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_distance" : {
-                    "distance" : "200km",
-                    "pin.location" : {
-                        "lat" : 40,
-                        "lon" : -70
-                    }
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_distance" : {
-                    "distance" : "12km",
-                    "pin.location" : {
-                        "lat" : 40,
-                        "lon" : -70
-                    }
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_distance" : {
-                    "distance" : "12km",
-                    "pin.location" : [-70, 40]
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_distance" : {
-                    "distance" : "12km",
-                    "pin.location" : "40,-70"
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_distance" : {
-                    "distance" : "12km",
-                    "pin.location" : "drm3btev3e86"
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "match_all": {}
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "match_all": { "boost" : 1.2 }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "match_none": {}
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "more_like_this" : {
-            "fields" : ["title", "description"],
-            "like" : "Once upon a time",
-            "min_term_freq" : 1,
-            "max_query_terms" : 12
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "more_like_this" : {
-            "fields" : ["title", "description"],
-            "like" : [
-            {
-                "_index" : "imdb",
-                "_type" : "movies",
-                "_id" : "1"
-            },
-            {
-                "_index" : "imdb",
-                "_type" : "movies",
-                "_id" : "2"
-            },
-            "and potentially some more text here as well"
-            ],
-            "min_term_freq" : 1,
-            "max_query_terms" : 12
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "more_like_this" : {
-            "fields" : ["name.first", "name.last"],
-            "like" : [
-            {
-                "_index" : "marvel",
-                "_type" : "quotes",
-                "doc" : {
-                    "name": {
-                        "first": "Ben",
-                        "last": "Grimm"
-                    },
-                    "tweet": "You got no idea what I'd... what I'd give to be invisible."
-                  }
-            },
-            {
-                "_index" : "marvel",
-                "_type" : "quotes",
-                "_id" : "2"
-            }
-            ],
-            "min_term_freq" : 1,
-            "max_query_terms" : 12
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "mappings": {
-        "movies": {
-            "properties": {
-                "title": {
-                    "type": "text",
-                    "term_vector": "yes"
-                },
-                "description": {
-                    "type": "text"
-                },
-                "tags": {
-                    "type": "text",
-                    "fields" : {
-                        "raw": {
-                            "type" : "text",
-                            "analyzer": "keyword",
-                            "term_vector" : "yes"
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "indices" : {
-            "indices" : ["index1", "index2"],
-            "query" : { "term" : { "tag" : "wow" } },
-            "no_match_query" : { "term" : { "tag" : "kow" } }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "range" : {
-            "age" : {
-                "gte" : 10,
-                "lte" : 20,
-                "boost" : 2.0
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "range" : {
-            "date" : {
-                "gte" : "now-1d/d",
-                "lt" :  "now/d"
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "range" : {
-            "born" : {
-                "gte": "01/01/2012",
-                "lte": "2013",
-                "format": "dd/MM/yyyy||yyyy"
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "range" : {
-            "timestamp" : {
-                "gte": "2015-01-01 00:00:00",
-                "lte": "now",
-                "time_zone": "+01:00"
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "common": {
-            "body": {
-                "query": "this is bonsai cool",
-                    "cutoff_frequency": 0.001
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "common": {
-            "body": {
-                "query": "nelly the elephant as a cartoon",
-                    "cutoff_frequency": 0.001,
-                    "low_freq_operator": "and"
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool": {
-            "must": [
-            { "term": { "body": "nelly"}},
-            { "term": { "body": "elephant"}},
-            { "term": { "body": "cartoon"}}
-            ],
-            "should": [
-            { "term": { "body": "the"}},
-            { "term": { "body": "as"}},
-            { "term": { "body": "a"}}
-            ]
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "common": {
-            "body": {
-                "query": "nelly the elephant as a cartoon",
-                "cutoff_frequency": 0.001,
-                "minimum_should_match": 2
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool": {
-            "must": {
-                "bool": {
-                    "should": [
-                    { "term": { "body": "nelly"}},
-                    { "term": { "body": "elephant"}},
-                    { "term": { "body": "cartoon"}}
-                    ],
-                    "minimum_should_match": 2
-                }
-            },
-            "should": [
-                { "term": { "body": "the"}},
-                { "term": { "body": "as"}},
-                { "term": { "body": "a"}}
-                ]
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "common": {
-            "body": {
-                "query": "nelly the elephant not as a cartoon",
-                    "cutoff_frequency": 0.001,
-                    "minimum_should_match": {
-                        "low_freq" : 2,
-                        "high_freq" : 3
-                    }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool": {
-            "must": {
-                "bool": {
-                    "should": [
-                    { "term": { "body": "nelly"}},
-                    { "term": { "body": "elephant"}},
-                    { "term": { "body": "cartoon"}}
-                    ],
-                    "minimum_should_match": 2
-                }
-            },
-            "should": {
-                "bool": {
-                    "should": [
-                    { "term": { "body": "the"}},
-                    { "term": { "body": "not"}},
-                    { "term": { "body": "as"}},
-                    { "term": { "body": "a"}}
-                    ],
-                    "minimum_should_match": 3
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "common": {
-            "body": {
-                "query": "how not to be",
-                    "cutoff_frequency": 0.001,
-                    "minimum_should_match": {
-                        "low_freq" : 2,
-                        "high_freq" : 3
-                    }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool": {
-            "should": [
-            { "term": { "body": "how"}},
-            { "term": { "body": "not"}},
-            { "term": { "body": "to"}},
-            { "term": { "body": "be"}}
-            ],
-            "minimum_should_match": "3<50%"
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "span_within" : {
-            "little" : {
-                "span_term" : { "field1" : "foo" }
-            },
-            "big" : {
-                "span_near" : {
-                    "clauses" : [
-                        { "span_term" : { "field1" : "bar" } },
-                        { "span_term" : { "field1" : "baz" } }
-                    ],
-                    "slop" : 5,
-                    "in_order" : true
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "span_near" : {
-            "clauses" : [
-                { "span_term" : { "field" : "value1" } },
-                { "span_term" : { "field" : "value2" } },
-                { "span_term" : { "field" : "value3" } }
-            ],
-            "slop" : 12,
-            "in_order" : false
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_distance_range" : {
-                    "from" : "200km",
-                    "to" : "400km",
-                    "pin.location" : {
-                        "lat" : 40,
-                        "lon" : -70
-                    }
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "simple_query_string" : {
-        "query": "\"fried eggs\" +(eggplant | potato) -frittata",
-        "analyzer": "snowball",
-        "fields": ["body^5","_all"],
-        "default_operator": "and"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "simple_query_string" : {
-            "fields" : ["content", "name.*^5"],
-            "query" : "foo bar baz"
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "simple_query_string" : {
-            "query" : "foo | bar + baz*",
-            "flags" : "OR|AND|PREFIX"
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "span_multi":{
-            "match":{
-                "prefix" : { "user" :  { "value" : "ki" } }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "span_multi":{
-            "match":{
-                "prefix" : { "user" :  { "value" : "ki", "boost" : 1.08 } }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "constant_score" : {
-            "filter" : {
-                "term" : { "user" : "kimchy"}
-            },
-            "boost" : 1.2
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "wildcard" : { "user" : "ki*y" }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "wildcard" : { "user" : { "value" : "ki*y", "boost" : 2.0 } }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "wildcard" : { "user" : { "wildcard" : "ki*y", "boost" : 2.0 } }
-    }
-}
-handle snippet
-null
-emit snippet
-body part: 
-{
-    "mappings": {
-        "type1" : {
-            "properties" : {
-                "obj1" : {
-                    "type" : "nested"
-                }
-            }
-        }
-    }
-}
-emit snippet
-query part: wait_for_status=yellow
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "nested" : {
-            "path" : "obj1",
-            "score_mode" : "avg",
-            "query" : {
-                "bool" : {
-                    "must" : [
-                    { "match" : {"obj1.name" : "blue"} },
-                    { "range" : {"obj1.count" : {"gt" : 5}} }
-                    ]
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "term" : { "user" : "Kimchy" }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "bool": {
-      "should": [
-        {
-          "term": {
-            "status": {
-              "value": "urgent",
-              "boost": 2.0
-            }
-          }
-        },
-        {
-          "term": {
-            "status": "normal"
-          }
-        }
-      ]
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "full_text": {
-          "type":  "text"
-        },
-        "exact_value": {
-          "type":  "keyword"
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "full_text":   "Quick Foxes!",
-  "exact_value": "Quick Foxes!"
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "term": {
-      "exact_value": "Quick Foxes!"
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "term": {
-      "full_text": "Quick Foxes!"
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "term": {
-      "full_text": "foxes"
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "match": {
-      "full_text": "Quick Foxes!"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "match" : {
-            "message" : "this is a test"
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "match" : {
-            "message" : {
-                "query" : "this is a test",
-                "operator" : "and"
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "match" : {
-            "message" : {
-                "query" : "to be or not to be",
-                "operator" : "and",
-                "zero_terms_query": "all"
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "match" : {
-            "message" : {
-                "query" : "to be or not to be",
-                "cutoff_frequency" : 0.001
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "bool": {
-      "must": [
-        { "match": { "title":   "Search"        }},
-        { "match": { "content": "Elasticsearch" }}
-      ],
-      "filter": [
-        { "term":  { "status": "published" }},
-        { "range": { "publish_date": { "gte": "2015-01-01" }}}
-      ]
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "ids" : {
-            "type" : "my_type",
-            "values" : ["1", "4", "100"]
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "dis_max" : {
-            "tie_breaker" : 0.7,
-                "boost" : 1.2,
-                "queries" : [
-                {
-                    "term" : { "age" : 34 }
-                },
-                {
-                    "term" : { "age" : 35 }
-                }
-                ]
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "match_phrase" : {
-            "message" : "this is a test"
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "match_phrase" : {
-            "message" : {
-                "query" : "this is a test",
-                "analyzer" : "my_analyzer"
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "script" : {
-                    "script" : "doc['num1'].value > 1"
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "script" : {
-                    "script" : {
-                        "inline" : "doc['num1'].value > param1",
-                        "params" : {
-                            "param1" : 5
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{ "query": {
-    "prefix" : { "user" : "ki" }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{ "query": {
-    "prefix" : { "user" :  { "value" : "ki", "boost" : 2.0 } }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{ "query": {
-    "prefix" : { "user" :  { "prefix" : "ki", "boost" : 2.0 } }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "bool" : {
-      "must" : {
-        "term" : { "user" : "kimchy" }
-      },
-      "filter": {
-        "term" : { "tag" : "tech" }
-      },
-      "must_not" : {
-        "range" : {
-          "age" : { "from" : 10, "to" : 20 }
-        }
-      },
-      "should" : [
-        { "term" : { "tag" : "wow" } },
-        { "term" : { "tag" : "elasticsearch" } }
-      ],
-      "minimum_should_match" : 1,
-      "boost" : 1.0
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "bool": {
-      "filter": {
-        "term": {
-          "status": "active"
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "bool": {
-      "must": {
-        "match_all": {}
-      },
-      "filter": {
-        "term": {
-          "status": "active"
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "constant_score": {
-      "filter": {
-        "term": {
-          "status": "active"
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "span_containing" : {
-            "little" : {
-                "span_term" : { "field1" : "foo" }
-            },
-            "big" : {
-                "span_near" : {
-                    "clauses" : [
-                        { "span_term" : { "field1" : "bar" } },
-                        { "span_term" : { "field1" : "baz" } }
-                    ],
-                    "slop" : 5,
-                    "in_order" : true
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "boosting" : {
-            "positive" : {
-                "term" : {
-                    "field1" : "value1"
-                }
-            },
-            "negative" : {
-                 "term" : {
-                     "field2" : "value2"
-                }
-            },
-            "negative_boost" : 0.2
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "has_child" : {
-            "type" : "blog_tag",
-                "query" : {
-                    "term" : {
-                        "tag" : "something"
-                    }
-                }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "has_child" : {
-            "type" : "blog_tag",
-                "score_mode" : "min",
-                "query" : {
-                    "term" : {
-                        "tag" : "something"
-                    }
-                }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "has_child" : {
-            "type" : "blog_tag",
-            "score_mode" : "min",
-            "min_children": 2,
-            "max_children": 10,
-            "query" : {
-                "term" : {
-                    "tag" : "something"
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "constant_score" : {
-            "filter" : {
-                "terms" : { "user" : ["kimchy", "elasticsearch"]}
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "followers" : ["1", "3"]
-}
-emit snippet
-body part: 
-{
-    "user" : "1"
-}
-emit snippet
-body part: 
-{
-    "query" : {
-        "terms" : {
-            "user" : {
-                "index" : "users",
-                "type" : "user",
-                "id" : "2",
-                "path" : "followers"
-            }
-        }
-    }
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "exists" : { "field" : "user" }
-    }
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool": {
-            "must_not": {
-                "exists": {
-                    "field": "user"
-                }
-            }
-        }
-    }
-}
-handle snippet
-null
-emit snippet
-body part: 
-{
-    "mappings" : {
-        "location": {
-            "properties": {
-                "pin": {
-                    "type": "geo_point",
-                    "geohash": true,
-                    "geohash_prefix": true,
-                    "geohash_precision": 10
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geohash_cell": {
-                    "pin": {
-                        "lat": 13.4080,
-                        "lon": 52.5186
-                    },
-                    "precision": 3,
-                    "neighbors": true
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "regexp":{
-            "name.first": "s.*y"
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "regexp":{
-            "name.first":{
-                "value":"s.*y",
-                "boost":1.2
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "regexp":{
-            "name.first": {
-                "value": "s.*y",
-                "flags" : "INTERSECTION|COMPLEMENT|EMPTY"
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "regexp":{
-            "name.first": {
-                "value": "s.*y",
-                "flags" : "INTERSECTION|COMPLEMENT|EMPTY",
-                "max_determinized_states": 20000
-            }
-        }
-    }
-}
-handle snippet
-null
-emit snippet
-body part: 
-{
-    "mappings": {
-        "blog_post": {
-            "properties": {
-                "name": {
-                    "type": "keyword"
-                }
-            }
-        },
-        "blog_tag": {
-            "_parent": {
-                "type": "blog_post"
-            },
-            "_routing": {
-                "required": true
-            }
-        }
-    }
-}
-emit snippet
-query part: wait_for_status=yellow
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "parent_id" : {
-            "type" : "blog_tag",
-                "id" : "1"
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "has_parent": {
-      "type": "blog_post",
-        "query": {
-          "term": {
-            "_id": "1"
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "has_parent" : {
-            "parent_type" : "blog",
-                "query" : {
-                    "term" : {
-                        "tag" : "something"
-                    }
-                }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "has_parent" : {
-            "parent_type" : "blog",
-                "score" : true,
-                "query" : {
-                    "term" : {
-                        "tag" : "something"
-                    }
-                }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "template": {
-            "inline": { "match": { "text": "{{query_string}}" }},
-            "params" : {
-                "query_string" : "all about search"
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "match": {
-            "text": "all about search"
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "template": {
-            "inline": "{ \"match\": { \"text\": \"{{query_string}}\" }}",
-            "params" : {
-                "query_string" : "all about search"
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "template": {
-            "file": "my_template",
-            "params" : {
-                "query_string" : "all about search"
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "template": { "match": { "text": "{{query_string}}" }}
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "template": {
-            "id": "my_template",
-            "params" : {
-                "query_string" : "all about search"
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "span_first" : {
-            "match" : {
-                "span_term" : { "user" : "kimchy" }
-            },
-            "end" : 3
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_polygon" : {
-                    "person.location" : {
-                        "points" : [
-                        {"lat" : 40, "lon" : -70},
-                        {"lat" : 30, "lon" : -80},
-                        {"lat" : 20, "lon" : -90}
-                        ]
-                    }
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_polygon" : {
-                    "person.location" : {
-                        "points" : [
-                            [-70, 40],
-                        [-80, 30],
-                            [-90, 20]
-                        ]
-                    }
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-               "geo_polygon" : {
-                    "person.location" : {
-                        "points" : [
-                            "40, -70",
-                            "30, -80",
-                            "20, -90"
-                        ]
-                    }
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-               "geo_polygon" : {
-                    "person.location" : {
-                        "points" : [
-                            "drn5x1g8cu2y",
-                            "30, -80",
-                            "20, -90"
-                        ]
-                    }
-                }
-            }
-        }
-    }
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query":{
-        "bool": {
-            "must": {
-                "match_all": {}
-            },
-            "filter": {
-                "geo_shape": {
-                    "location": {
-                        "shape": {
-                            "type": "envelope",
-                            "coordinates" : [[13.0, 53.0], [14.0, 52.0]]
-                        },
-                        "relation": "within"
-                    }
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool": {
-            "must": {
-                "match_all": {}
-            },
-                "filter": {
-                    "geo_shape": {
-                        "location": {
-                            "indexed_shape": {
-                                "id": "DEU",
-                                "type": "countries",
-                                "index": "shapes",
-                                "path": "location"
-                            }
-                        }
-                    }
-                }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "function_score": {
-            "query": {},
-            "boost": "5",
-            "random_score": {},
-            "boost_mode":"multiply"
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "function_score": {
-          "query": {},
-          "boost": "5",
-          "functions": [
-              {
-                  "filter": {},
-                  "random_score": {},
-                  "weight": 23
-              },
-              {
-                  "filter": {},
-                  "weight": 42
-              }
-          ],
-          "max_boost": 42,
-          "score_mode": "max",
-          "boost_mode": "multiply",
-          "min_score" : 42
-        }
-    }
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "function_score": {
-          "functions": [
-            {
-              "gauss": {
-                "price": {
-                  "origin": "0",
-                  "scale": "20"
-                }
-              }
-            },
-            {
-              "gauss": {
-                "location": {
-                  "origin": "11, 12",
-                  "scale": "2km"
-                }
-              }
-            }
-          ],
-          "query": {
-            "match": {
-              "properties": "balcony"
-            }
-          },
-          "score_mode": "multiply"
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "span_or" : {
-            "clauses" : [
-                { "span_term" : { "field" : "value1" } },
-                { "span_term" : { "field" : "value2" } },
-                { "span_term" : { "field" : "value3" } }
-            ]
-        }
-    }
-}
-handle snippet
-null
-emit snippet
-body part: 
-{
-    "mappings": {
-        "location": {
-            "properties": {
-                "pin": {
-                    "properties": {
-                        "location": {
-                            "type": "geo_point"
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-emit snippet
-body part: 
-{
-    "pin" : {
-        "location" : {
-            "lat" : 40.12,
-            "lon" : -71.34
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_bounding_box" : {
-                    "pin.location" : {
-                        "top_left" : {
-                            "lat" : 40.73,
-                            "lon" : -74.1
-                        },
-                        "bottom_right" : {
-                            "lat" : 40.01,
-                            "lon" : -71.12
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_bounding_box" : {
-                    "pin.location" : {
-                        "top_left" : {
-                            "lat" : 40.73,
-                            "lon" : -74.1
-                        },
-                        "bottom_right" : {
-                            "lat" : 40.01,
-                            "lon" : -71.12
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_bounding_box" : {
-                    "pin.location" : {
-                        "top_left" : [-74.1, 40.73],
-                        "bottom_right" : [-71.12, 40.01]
-                    }
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_bounding_box" : {
-                    "pin.location" : {
-                        "top_left" : "40.73, -74.1",
-                        "bottom_right" : "40.01, -71.12"
-                    }
-                }
-            }
-    }
-}
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_bounding_box" : {
-                    "pin.location" : {
-                        "top_left" : "dr5r9ydj2y73",
-                        "bottom_right" : "drj7teegpus6"
-                    }
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_bounding_box" : {
-                    "pin.location" : {
-                        "top" : 40.73,
-                        "left" : -74.1,
-                        "bottom" : 40.01,
-                        "right" : -71.12
-                    }
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "bool" : {
-            "must" : {
-                "match_all" : {}
-            },
-            "filter" : {
-                "geo_bounding_box" : {
-                    "pin.location" : {
-                        "top_left" : {
-                            "lat" : 40.73,
-                            "lon" : -74.1
-                        },
-                        "bottom_right" : {
-                            "lat" : 40.10,
-                            "lon" : -71.12
-                        }
-                    },
-                    "type" : "indexed"
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "match_phrase_prefix" : {
-            "message" : "quick brown f"
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "match_phrase_prefix" : {
-            "message" : {
-                "query" : "quick brown f",
-                "max_expansions" : 10
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "mappings": {
-        "doctype": {
-            "properties": {
-                "message": {
-                    "type": "string"
-                }
-            }
-        },
-        "queries": {
-            "properties": {
-                "query": {
-                    "type": "percolator"
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query" : {
-        "match" : {
-            "message" : "bonsai tree"
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query" : {
-        "percolate" : {
-            "field" : "query",
-            "document_type" : "doctype",
-            "document" : {
-                "message" : "A new bonsai tree in the office"
-            }
-        }
-    }
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "message" : "A new bonsai tree in the office"
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query" : {
-        "percolate" : {
-            "field": "query",
-            "document_type" : "doctype",
-            "index" : "my-index",
-            "type" : "message",
-            "id" : "1",
-            "version" : 1
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query" : {
-        "match" : {
-            "message" : "brown fox"
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query" : {
-        "match" : {
-            "message" : "lazy dog"
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query" : {
-        "percolate" : {
-            "field": "query",
-            "document_type" : "doctype",
-            "document" : {
-                "message" : "The quick brown fox jumps over the lazy dog"
-            }
-        }
-    },
-    "highlight": {
-      "fields": {
-        "message": {}
-      }
-    }
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "term" : {
-      "query.unknown_query" : ""
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "span_not" : {
-            "include" : {
-                "span_term" : { "field1" : "hoya" }
-            },
-            "exclude" : {
-                "span_near" : {
-                    "clauses" : [
-                        { "span_term" : { "field1" : "la" } },
-                        { "span_term" : { "field1" : "hoya" } }
-                    ],
-                    "slop" : 0,
-                    "in_order" : true
-                }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "type" : {
-            "value" : "my_type"
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "span_term" : { "user" : "kimchy" }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-       "span_term" : { "user" : { "value" : "kimchy", "boost" : 2.0 } }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "span_term" : { "user" : { "term" : "kimchy", "boost" : 2.0 } }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-       "fuzzy" : { "user" : "ki" }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "fuzzy" : {
-            "user" : {
-                "value" :         "ki",
-                    "boost" :         1.0,
-                    "fuzziness" :     2,
-                    "prefix_length" : 0,
-                    "max_expansions": 100
-            }
-        }
-    }
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "description" : "describe pipeline",
-  "processors" : [
-    {
-      "set" : {
-        "field": "foo",
-        "value": "bar"
-      }
-    }
-    // other processors
-  ]
-}
-handle snippet
-test snippet
-emit snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "pipeline" :
-  {
-    "description": "_description",
-    "processors": [
-      {
-        "set" : {
-          "field" : "field2",
-          "value" : "_value"
-        }
-      }
-    ]
-  },
-  "docs": [
-    {
-      "_index": "index",
-      "_type": "type",
-      "_id": "id",
-      "_source": {
-        "foo": "bar"
-      }
-    },
-    {
-      "_index": "index",
-      "_type": "type",
-      "_id": "id",
-      "_source": {
-        "foo": "rab"
-      }
-    }
-  ]
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-query part: verbose
-body part: 
-{
-  "pipeline" :
-  {
-    "description": "_description",
-    "processors": [
-      {
-        "set" : {
-          "field" : "field2",
-          "value" : "_value2"
-        }
-      },
-      {
-        "set" : {
-          "field" : "field3",
-          "value" : "_value3"
-        }
-      }
-    ]
-  },
-  "docs": [
-    {
-      "_index": "index",
-      "_type": "type",
-      "_id": "id",
-      "_source": {
-        "foo": "bar"
-      }
-    },
-    {
-      "_index": "index",
-      "_type": "type",
-      "_id": "id",
-      "_source": {
-        "foo": "rab"
-      }
-    }
-  ]
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-emit snippet
-query part: nodes=nodeId1,nodeId2
-emit snippet
-query part: nodes=nodeId1,nodeId2&actions=cluster:*
-handle snippet
-handle snippet
-test snippet
-emit snippet
-emit snippet
-query part: parent_task_id=parentTaskId:1
-handle snippet
-test snippet
-emit snippet
-query part: wait_for_completion=true&timeout=10s
-handle snippet
-test snippet
-emit snippet
-handle snippet
-test snippet
-emit snippet
-handle snippet
-test snippet
-emit snippet
-query part: node_id=nodeId1,nodeId2&actions=*reindex
-handle snippet
-test snippet
-emit snippet
-query part: group_by=parents
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "transient" : {
-    "cluster.routing.allocation.exclude._ip" : "10.0.0.1"
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "transient": {
-    "cluster.routing.allocation.include._ip": "192.168.2.*"
-  }
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "transient": {
-    "cluster.routing.allocation.disk.watermark.low": "80%",
-    "cluster.routing.allocation.disk.watermark.high": "50gb",
-    "cluster.info.update.interval": "1m"
-  }
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-null
-emit snippet
-query part: refresh
-body part: 
-{"index":{"_id":1}}
-{"first":"johnny","last":"gaudreau","goals":[9,27,1],"assists":[17,46,0],"gp":[26,82,1]}
-{"index":{"_id":2}}
-{"first":"sean","last":"monohan","goals":[7,54,26],"assists":[11,26,13],"gp":[26,82,82]}
-{"index":{"_id":3}}
-{"first":"jiri","last":"hudler","goals":[5,34,36],"assists":[11,62,42],"gp":[24,80,79]}
-{"index":{"_id":4}}
-{"first":"micheal","last":"frolik","goals":[4,6,15],"assists":[8,23,15],"gp":[26,82,82]}
-{"index":{"_id":5}}
-{"first":"sam","last":"bennett","goals":[5,0,0],"assists":[8,1,0],"gp":[26,1,0]}
-{"index":{"_id":6}}
-{"first":"dennis","last":"wideman","goals":[0,26,15],"assists":[11,30,24],"gp":[26,81,82]}
-{"index":{"_id":7}}
-{"first":"david","last":"jones","goals":[7,19,5],"assists":[3,17,4],"gp":[26,45,34]}
-{"index":{"_id":8}}
-{"first":"tj","last":"brodie","goals":[2,14,7],"assists":[8,42,30],"gp":[26,82,82]}
-{"index":{"_id":39}}
-{"first":"mark","last":"giordano","goals":[6,30,15],"assists":[3,30,24],"gp":[26,60,63]}
-{"index":{"_id":10}}
-{"first":"mikael","last":"backlund","goals":[3,15,13],"assists":[6,24,18],"gp":[26,82,82]}
-{"index":{"_id":11}}
-{"first":"joe","last":"colborne","goals":[3,18,13],"assists":[6,20,24],"gp":[26,67,82]}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "function_score": {
-      "script_score": {
-        "script": {
-          "lang": "painless",
-          "inline": "int total = 0; for (int i = 0; i < doc['goals'].length; ++i) { total += doc['goals'][i]; } return total;"
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "match_all": {}
-  },
-  "script_fields": {
-    "total_goals": {
-      "script": {
-        "lang": "painless",
-        "inline": "int total = 0; for (int i = 0; i < doc['goals'].length; ++i) { total += doc['goals'][i]; } return total;"
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "match_all": {}
-  },
-  "sort": {
-    "_script": {
-      "type": "string",
-      "order": "asc",
-      "script": {
-        "lang": "painless",
-        "inline": "doc['first'].value + ' ' + doc['last'].value"
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "fields": [
-    "_id",
-    "_source"
-  ],
-  "query": {
-    "term": {
-      "_id": 1
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "script": {
-    "lang": "painless",
-    "inline": "ctx._source.last = params.last",
-    "params": {
-      "last": "hockey"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "script": {
-    "lang": "painless",
-    "inline": "ctx._source.last = params.last; ctx._source.nick = params.nick",
-    "params": {
-      "last": "gaudreau",
-      "nick": "hockey"
-    }
-  }
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "text": "quick brown fox",
-  "popularity": 1
-}
-emit snippet
-body part: 
-{
-  "text": "quick fox",
-  "popularity": 5
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "function_score": {
-      "query": {
-        "match": {
-          "text": "quick brown fox"
-        }
-      },
-      "script_score": {
-        "script": {
-          "lang": "expression",
-          "inline": "_score * doc['popularity']"
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "cost_price": 100
-}
-emit snippet
-body part: 
-{
-  "script_fields": {
-    "sales_price": {
-      "script": {
-        "lang":   "expression",
-        "inline": "doc['cost_price'] * markup",
-        "params": {
-          "markup": 0.2
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "title": {
-          "type": "text"
-        },
-        "first_name": {
-          "type": "text",
-          "store": true
-        },
-        "last_name": {
-          "type": "text",
-          "store": true
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "title": "Mr",
-  "first_name": "Barry",
-  "last_name": "White"
-}
-emit snippet
-body part: 
-{
-  "script_fields": {
-    "source": {
-      "script": {
-        "lang": "groovy",
-        "inline": "_source.title + ' ' + _source.first_name + ' ' + _source.last_name"
-      }
-    },
-    "stored_fields": {
-      "script": {
-        "lang": "groovy",
-        "inline": "_fields['first_name'].value + ' ' + _fields['last_name'].value"
-      }
-    }
-  }
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "my_field": 5
-}
-emit snippet
-body part: 
-{
-  "script_fields": {
-    "my_doubled_field": {
-      "script": {
-        "lang":   "expression",
-        "inline": "doc['my_field'] * multiplier",
-        "params": {
-          "multiplier": 2
-        }
-      }
-    }
-  }
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "script": "log(_score * 2) + my_modifier"
-}
-handle snippet
-test snippet
-emit snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "script": {
-      "script": {
-        "lang": "groovy",
-        "id":   "calculate-score",
-        "params": {
-          "my_modifier": 2
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-catch part: /cannot set discovery.zen.minimum_master_nodes to more than the current master nodes/
-body part: 
-{
-  "transient": {
-    "discovery.zen.minimum_master_nodes": 2
-  }
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-          {"index":{"_id": "5"}}
-          {"user": "test", "message": "some message with the number 5", "date": "2009-11-15T14:12:12", "likes": 5}
-          {"index":{"_id": "6"}}
-          {"user": "test", "message": "some message with the number 6", "date": "2009-11-15T14:12:12", "likes": 6}
-          {"index":{"_id": "7"}}
-          {"user": "test", "message": "some message with the number 7", "date": "2009-11-15T14:12:12", "likes": 7}
-          {"index":{"_id": "8"}}
-          {"user": "test", "message": "some message with the number 8", "date": "2009-11-15T14:12:12", "likes": 8}
-          {"index":{"_id": "9"}}
-          {"user": "test", "message": "some message with the number 9", "date": "2009-11-15T14:12:12", "likes": 9}
-          {"index":{"_id": "10"}}
-          {"user": "test", "message": "some message with the number 10", "date": "2009-11-15T14:12:12", "likes": 10}
-          {"index":{"_id": "11"}}
-          {"user": "test", "message": "some message with the number 11", "date": "2009-11-15T14:12:12", "likes": 11}
-          {"index":{"_id": "12"}}
-          {"user": "test", "message": "some message with the number 12", "date": "2009-11-15T14:12:12", "likes": 12}
-          {"index":{"_id": "13"}}
-          {"user": "test", "message": "some message with the number 13", "date": "2009-11-15T14:12:12", "likes": 13}
-          {"index":{"_id": "14"}}
-          {"user": "test", "message": "some message with the number 14", "date": "2009-11-15T14:12:12", "likes": 14}
-          {"index":{"_id": "15"}}
-          {"user": "test", "message": "some message with the number 15", "date": "2009-11-15T14:12:12", "likes": 15}
-          {"index":{"_id": "16"}}
-          {"user": "test", "message": "some message with the number 16", "date": "2009-11-15T14:12:12", "likes": 16}
-          {"index":{"_id": "17"}}
-          {"user": "test", "message": "some message with the number 17", "date": "2009-11-15T14:12:12", "likes": 17}
-          {"index":{"_id": "18"}}
-          {"user": "test", "message": "some message with the number 18", "date": "2009-11-15T14:12:12", "likes": 18}
-          {"index":{"_id": "19"}}
-          {"user": "test", "message": "some message with the number 19", "date": "2009-11-15T14:12:12", "likes": 19}
-          {"index":{"_id": "20"}}
-          {"user": "test", "message": "some message with the number 20", "date": "2009-11-15T14:12:12", "likes": 20}
-          {"index":{"_id": "21"}}
-          {"user": "test", "message": "some message with the number 21", "date": "2009-11-15T14:12:12", "likes": 21}
-          {"index":{"_id": "22"}}
-          {"user": "test", "message": "some message with the number 22", "date": "2009-11-15T14:12:12", "likes": 22}
-          {"index":{"_id": "23"}}
-          {"user": "test", "message": "some message with the number 23", "date": "2009-11-15T14:12:12", "likes": 23}
-          {"index":{"_id": "24"}}
-          {"user": "test", "message": "some message with the number 24", "date": "2009-11-15T14:12:12", "likes": 24}
-          {"index":{"_id": "25"}}
-          {"user": "test", "message": "some message with the number 25", "date": "2009-11-15T14:12:12", "likes": 25}
-          {"index":{"_id": "26"}}
-          {"user": "test", "message": "some message with the number 26", "date": "2009-11-15T14:12:12", "likes": 26}
-          {"index":{"_id": "27"}}
-          {"user": "test", "message": "some message with the number 27", "date": "2009-11-15T14:12:12", "likes": 27}
-          {"index":{"_id": "28"}}
-          {"user": "test", "message": "some message with the number 28", "date": "2009-11-15T14:12:12", "likes": 28}
-          {"index":{"_id": "29"}}
-          {"user": "test", "message": "some message with the number 29", "date": "2009-11-15T14:12:12", "likes": 29}
-          {"index":{"_id": "30"}}
-          {"user": "test", "message": "some message with the number 30", "date": "2009-11-15T14:12:12", "likes": 30}
-          {"index":{"_id": "31"}}
-          {"user": "test", "message": "some message with the number 31", "date": "2009-11-15T14:12:12", "likes": 31}
-          {"index":{"_id": "32"}}
-          {"user": "test", "message": "some message with the number 32", "date": "2009-11-15T14:12:12", "likes": 32}
-          {"index":{"_id": "33"}}
-          {"user": "test", "message": "some message with the number 33", "date": "2009-11-15T14:12:12", "likes": 33}
-          {"index":{"_id": "34"}}
-          {"user": "test", "message": "some message with the number 34", "date": "2009-11-15T14:12:12", "likes": 34}
-          {"index":{"_id": "35"}}
-          {"user": "test", "message": "some message with the number 35", "date": "2009-11-15T14:12:12", "likes": 35}
-          {"index":{"_id": "36"}}
-          {"user": "test", "message": "some message with the number 36", "date": "2009-11-15T14:12:12", "likes": 36}
-          {"index":{"_id": "37"}}
-          {"user": "test", "message": "some message with the number 37", "date": "2009-11-15T14:12:12", "likes": 37}
-          {"index":{"_id": "38"}}
-          {"user": "test", "message": "some message with the number 38", "date": "2009-11-15T14:12:12", "likes": 38}
-          {"index":{"_id": "39"}}
-          {"user": "test", "message": "some message with the number 39", "date": "2009-11-15T14:12:12", "likes": 39}
-          {"index":{"_id": "40"}}
-          {"user": "test", "message": "some message with the number 40", "date": "2009-11-15T14:12:12", "likes": 40}
-          {"index":{"_id": "41"}}
-          {"user": "test", "message": "some message with the number 41", "date": "2009-11-15T14:12:12", "likes": 41}
-          {"index":{"_id": "42"}}
-          {"user": "test", "message": "some message with the number 42", "date": "2009-11-15T14:12:12", "likes": 42}
-          {"index":{"_id": "43"}}
-          {"user": "test", "message": "some message with the number 43", "date": "2009-11-15T14:12:12", "likes": 43}
-          {"index":{"_id": "44"}}
-          {"user": "test", "message": "some message with the number 44", "date": "2009-11-15T14:12:12", "likes": 44}
-          {"index":{"_id": "45"}}
-          {"user": "test", "message": "some message with the number 45", "date": "2009-11-15T14:12:12", "likes": 45}
-          {"index":{"_id": "46"}}
-          {"user": "test", "message": "some message with the number 46", "date": "2009-11-15T14:12:12", "likes": 46}
-          {"index":{"_id": "47"}}
-          {"user": "test", "message": "some message with the number 47", "date": "2009-11-15T14:12:12", "likes": 47}
-          {"index":{"_id": "48"}}
-          {"user": "test", "message": "some message with the number 48", "date": "2009-11-15T14:12:12", "likes": 48}
-          {"index":{"_id": "49"}}
-          {"user": "test", "message": "some message with the number 49", "date": "2009-11-15T14:12:12", "likes": 49}
-          {"index":{"_id": "50"}}
-          {"user": "test", "message": "some message with the number 50", "date": "2009-11-15T14:12:12", "likes": 50}
-          {"index":{"_id": "51"}}
-          {"user": "test", "message": "some message with the number 51", "date": "2009-11-15T14:12:12", "likes": 51}
-          {"index":{"_id": "52"}}
-          {"user": "test", "message": "some message with the number 52", "date": "2009-11-15T14:12:12", "likes": 52}
-          {"index":{"_id": "53"}}
-          {"user": "test", "message": "some message with the number 53", "date": "2009-11-15T14:12:12", "likes": 53}
-          {"index":{"_id": "54"}}
-          {"user": "test", "message": "some message with the number 54", "date": "2009-11-15T14:12:12", "likes": 54}
-          {"index":{"_id": "55"}}
-          {"user": "test", "message": "some message with the number 55", "date": "2009-11-15T14:12:12", "likes": 55}
-          {"index":{"_id": "56"}}
-          {"user": "test", "message": "some message with the number 56", "date": "2009-11-15T14:12:12", "likes": 56}
-          {"index":{"_id": "57"}}
-          {"user": "test", "message": "some message with the number 57", "date": "2009-11-15T14:12:12", "likes": 57}
-          {"index":{"_id": "58"}}
-          {"user": "test", "message": "some message with the number 58", "date": "2009-11-15T14:12:12", "likes": 58}
-          {"index":{"_id": "59"}}
-          {"user": "test", "message": "some message with the number 59", "date": "2009-11-15T14:12:12", "likes": 59}
-          {"index":{"_id": "60"}}
-          {"user": "test", "message": "some message with the number 60", "date": "2009-11-15T14:12:12", "likes": 60}
-          {"index":{"_id": "61"}}
-          {"user": "test", "message": "some message with the number 61", "date": "2009-11-15T14:12:12", "likes": 61}
-          {"index":{"_id": "62"}}
-          {"user": "test", "message": "some message with the number 62", "date": "2009-11-15T14:12:12", "likes": 62}
-          {"index":{"_id": "63"}}
-          {"user": "test", "message": "some message with the number 63", "date": "2009-11-15T14:12:12", "likes": 63}
-          {"index":{"_id": "64"}}
-          {"user": "test", "message": "some message with the number 64", "date": "2009-11-15T14:12:12", "likes": 64}
-          {"index":{"_id": "65"}}
-          {"user": "test", "message": "some message with the number 65", "date": "2009-11-15T14:12:12", "likes": 65}
-          {"index":{"_id": "66"}}
-          {"user": "test", "message": "some message with the number 66", "date": "2009-11-15T14:12:12", "likes": 66}
-          {"index":{"_id": "67"}}
-          {"user": "test", "message": "some message with the number 67", "date": "2009-11-15T14:12:12", "likes": 67}
-          {"index":{"_id": "68"}}
-          {"user": "test", "message": "some message with the number 68", "date": "2009-11-15T14:12:12", "likes": 68}
-          {"index":{"_id": "69"}}
-          {"user": "test", "message": "some message with the number 69", "date": "2009-11-15T14:12:12", "likes": 69}
-          {"index":{"_id": "70"}}
-          {"user": "test", "message": "some message with the number 70", "date": "2009-11-15T14:12:12", "likes": 70}
-          {"index":{"_id": "71"}}
-          {"user": "test", "message": "some message with the number 71", "date": "2009-11-15T14:12:12", "likes": 71}
-          {"index":{"_id": "72"}}
-          {"user": "test", "message": "some message with the number 72", "date": "2009-11-15T14:12:12", "likes": 72}
-          {"index":{"_id": "73"}}
-          {"user": "test", "message": "some message with the number 73", "date": "2009-11-15T14:12:12", "likes": 73}
-          {"index":{"_id": "74"}}
-          {"user": "test", "message": "some message with the number 74", "date": "2009-11-15T14:12:12", "likes": 74}
-          {"index":{"_id": "75"}}
-          {"user": "test", "message": "some message with the number 75", "date": "2009-11-15T14:12:12", "likes": 75}
-          {"index":{"_id": "76"}}
-          {"user": "test", "message": "some message with the number 76", "date": "2009-11-15T14:12:12", "likes": 76}
-          {"index":{"_id": "77"}}
-          {"user": "test", "message": "some message with the number 77", "date": "2009-11-15T14:12:12", "likes": 77}
-          {"index":{"_id": "78"}}
-          {"user": "test", "message": "some message with the number 78", "date": "2009-11-15T14:12:12", "likes": 78}
-          {"index":{"_id": "79"}}
-          {"user": "test", "message": "some message with the number 79", "date": "2009-11-15T14:12:12", "likes": 79}
-          {"index":{"_id": "80"}}
-          {"user": "test", "message": "some message with the number 80", "date": "2009-11-15T14:12:12", "likes": 80}
-          {"index":{"_id": "81"}}
-          {"user": "test", "message": "some message with the number 81", "date": "2009-11-15T14:12:12", "likes": 81}
-          {"index":{"_id": "82"}}
-          {"user": "test", "message": "some message with the number 82", "date": "2009-11-15T14:12:12", "likes": 82}
-          {"index":{"_id": "83"}}
-          {"user": "test", "message": "some message with the number 83", "date": "2009-11-15T14:12:12", "likes": 83}
-          {"index":{"_id": "84"}}
-          {"user": "test", "message": "some message with the number 84", "date": "2009-11-15T14:12:12", "likes": 84}
-          {"index":{"_id": "85"}}
-          {"user": "test", "message": "some message with the number 85", "date": "2009-11-15T14:12:12", "likes": 85}
-          {"index":{"_id": "86"}}
-          {"user": "test", "message": "some message with the number 86", "date": "2009-11-15T14:12:12", "likes": 86}
-          {"index":{"_id": "87"}}
-          {"user": "test", "message": "some message with the number 87", "date": "2009-11-15T14:12:12", "likes": 87}
-          {"index":{"_id": "88"}}
-          {"user": "test", "message": "some message with the number 88", "date": "2009-11-15T14:12:12", "likes": 88}
-          {"index":{"_id": "89"}}
-          {"user": "test", "message": "some message with the number 89", "date": "2009-11-15T14:12:12", "likes": 89}
-          {"index":{"_id": "90"}}
-          {"user": "test", "message": "some message with the number 90", "date": "2009-11-15T14:12:12", "likes": 90}
-          {"index":{"_id": "91"}}
-          {"user": "test", "message": "some message with the number 91", "date": "2009-11-15T14:12:12", "likes": 91}
-          {"index":{"_id": "92"}}
-          {"user": "test", "message": "some message with the number 92", "date": "2009-11-15T14:12:12", "likes": 92}
-          {"index":{"_id": "93"}}
-          {"user": "test", "message": "some message with the number 93", "date": "2009-11-15T14:12:12", "likes": 93}
-          {"index":{"_id": "94"}}
-          {"user": "test", "message": "some message with the number 94", "date": "2009-11-15T14:12:12", "likes": 94}
-          {"index":{"_id": "95"}}
-          {"user": "test", "message": "some message with the number 95", "date": "2009-11-15T14:12:12", "likes": 95}
-          {"index":{"_id": "96"}}
-          {"user": "test", "message": "some message with the number 96", "date": "2009-11-15T14:12:12", "likes": 96}
-          {"index":{"_id": "97"}}
-          {"user": "test", "message": "some message with the number 97", "date": "2009-11-15T14:12:12", "likes": 97}
-          {"index":{"_id": "98"}}
-          {"user": "test", "message": "some message with the number 98", "date": "2009-11-15T14:12:12", "likes": 98}
-          {"index":{"_id": "99"}}
-          {"user": "test", "message": "some message with the number 99", "date": "2009-11-15T14:12:12", "likes": 99}
-          {"index":{"_id": "100"}}
-          {"user": "test", "message": "some message with the number 100", "date": "2009-11-15T14:12:12", "likes": 100}
-          {"index":{"_id": "101"}}
-          {"user": "test", "message": "some message with the number 101", "date": "2009-11-15T14:12:12", "likes": 101}
-          {"index":{"_id": "102"}}
-          {"user": "test", "message": "some message with the number 102", "date": "2009-11-15T14:12:12", "likes": 102}
-          {"index":{"_id": "103"}}
-          {"user": "test", "message": "some message with the number 103", "date": "2009-11-15T14:12:12", "likes": 103}
-          {"index":{"_id": "104"}}
-          {"user": "test", "message": "some message with the number 104", "date": "2009-11-15T14:12:12", "likes": 104}
-          {"index":{"_id": "105"}}
-          {"user": "test", "message": "some message with the number 105", "date": "2009-11-15T14:12:12", "likes": 105}
-          {"index":{"_id": "106"}}
-          {"user": "test", "message": "some message with the number 106", "date": "2009-11-15T14:12:12", "likes": 106}
-          {"index":{"_id": "107"}}
-          {"user": "test", "message": "some message with the number 107", "date": "2009-11-15T14:12:12", "likes": 107}
-          {"index":{"_id": "108"}}
-          {"user": "test", "message": "some message with the number 108", "date": "2009-11-15T14:12:12", "likes": 108}
-          {"index":{"_id": "109"}}
-          {"user": "test", "message": "some message with the number 109", "date": "2009-11-15T14:12:12", "likes": 109}
-          {"index":{"_id": "110"}}
-          {"user": "test", "message": "some message with the number 110", "date": "2009-11-15T14:12:12", "likes": 110}
-          {"index":{"_id": "111"}}
-          {"user": "test", "message": "some message with the number 111", "date": "2009-11-15T14:12:12", "likes": 111}
-          {"index":{"_id": "112"}}
-          {"user": "test", "message": "some message with the number 112", "date": "2009-11-15T14:12:12", "likes": 112}
-          {"index":{"_id": "113"}}
-          {"user": "test", "message": "some message with the number 113", "date": "2009-11-15T14:12:12", "likes": 113}
-          {"index":{"_id": "114"}}
-          {"user": "test", "message": "some message with the number 114", "date": "2009-11-15T14:12:12", "likes": 114}
-          {"index":{"_id": "115"}}
-          {"user": "test", "message": "some message with the number 115", "date": "2009-11-15T14:12:12", "likes": 115}
-          {"index":{"_id": "116"}}
-          {"user": "test", "message": "some message with the number 116", "date": "2009-11-15T14:12:12", "likes": 116}
-          {"index":{"_id": "117"}}
-          {"user": "test", "message": "some message with the number 117", "date": "2009-11-15T14:12:12", "likes": 117}
-          {"index":{"_id": "118"}}
-          {"user": "test", "message": "some message with the number 118", "date": "2009-11-15T14:12:12", "likes": 118}
-          {"index":{"_id": "119"}}
-          {"user": "test", "message": "some message with the number 119", "date": "2009-11-15T14:12:12", "likes": 119}
-emit snippet
-body part: 
-{
-  "query": {
-    "match": {
-      "message": "some message"
-    }
-  }
-}
-handle snippet
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-emit snippet
-query part: conflicts=proceed
-body part: 
-{
-  "query": {
-    "match_all": {}
-  }
-}
-handle snippet
-test snippet
-emit snippet
-emit snippet
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "query": {
-    "match_all": {}
-  }
-}
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-emit snippet
-query part: routing=1
-body part: 
-{
-  "query": {
-    "range" : {
-        "age" : {
-           "gte" : 10
-        }
-    }
-  }
-}
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-emit snippet
-query part: scroll_size=5000
-body part: 
-{
-  "query": {
-    "term": {
-      "user": "kimchy"
-    }
-  }
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-query part: detailed=true&action=*/delete/byquery
-handle snippet
-handle snippet
-test snippet
-emit snippet
-handle snippet
-test snippet
-emit snippet
-query part: requests_per_second=unlimited
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "user" : "kimchy",
-    "post_date" : "2009-11-15T14:12:12",
-    "message" : "trying out Elasticsearch"
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-catch part: conflict
-query part: version=2
-body part: 
-{
-    "message" : "elasticsearch now has versioning support, double cool!"
-}
-handle snippet
-test snippet
-emit snippet
-query part: op_type=create
-body part: 
-{
-    "user" : "kimchy",
-    "post_date" : "2009-11-15T14:12:12",
-    "message" : "trying out Elasticsearch"
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "user" : "kimchy",
-    "post_date" : "2009-11-15T14:12:12",
-    "message" : "trying out Elasticsearch"
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "user" : "kimchy",
-    "post_date" : "2009-11-15T14:12:12",
-    "message" : "trying out Elasticsearch"
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-query part: routing=kimchy
-body part: 
-{
-    "user" : "kimchy",
-    "post_date" : "2009-11-15T14:12:12",
-    "message" : "trying out Elasticsearch"
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "tag_parent": {},
-    "blog_tag": {
-      "_parent": {
-        "type": "tag_parent"
-      }
-    }
-  }
-}
-emit snippet
-query part: parent=1111
-body part: 
-{
-    "tag" : "something"
-}
-handle snippet
-test snippet
-emit snippet
-query part: timestamp=2009-11-15T14:12:12
-body part: 
-{
-    "user" : "kimchy",
-    "message" : "trying out Elasticsearch"
-}
-handle snippet
-test snippet
-emit snippet
-query part: ttl=86400000ms
-body part: 
-{
-    "user": "kimchy",
-    "message": "Trying out elasticsearch, so far so good?"
-}
-handle snippet
-test snippet
-emit snippet
-query part: ttl=1d
-body part: 
-{
-    "user": "kimchy",
-    "message": "Trying out elasticsearch, so far so good?"
-}
-handle snippet
-test snippet
-emit snippet
-query part: timeout=5m
-body part: 
-{
-    "user" : "kimchy",
-    "post_date" : "2009-11-15T14:12:12",
-    "message" : "trying out Elasticsearch"
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-          {"index":{"_id": "5"}}
-          {"user": "test", "message": "some message with the number 5", "date": "2009-11-15T14:12:12", "likes": 5}
-          {"index":{"_id": "6"}}
-          {"user": "test", "message": "some message with the number 6", "date": "2009-11-15T14:12:12", "likes": 6}
-          {"index":{"_id": "7"}}
-          {"user": "test", "message": "some message with the number 7", "date": "2009-11-15T14:12:12", "likes": 7}
-          {"index":{"_id": "8"}}
-          {"user": "test", "message": "some message with the number 8", "date": "2009-11-15T14:12:12", "likes": 8}
-          {"index":{"_id": "9"}}
-          {"user": "test", "message": "some message with the number 9", "date": "2009-11-15T14:12:12", "likes": 9}
-          {"index":{"_id": "10"}}
-          {"user": "test", "message": "some message with the number 10", "date": "2009-11-15T14:12:12", "likes": 10}
-          {"index":{"_id": "11"}}
-          {"user": "test", "message": "some message with the number 11", "date": "2009-11-15T14:12:12", "likes": 11}
-          {"index":{"_id": "12"}}
-          {"user": "test", "message": "some message with the number 12", "date": "2009-11-15T14:12:12", "likes": 12}
-          {"index":{"_id": "13"}}
-          {"user": "test", "message": "some message with the number 13", "date": "2009-11-15T14:12:12", "likes": 13}
-          {"index":{"_id": "14"}}
-          {"user": "test", "message": "some message with the number 14", "date": "2009-11-15T14:12:12", "likes": 14}
-          {"index":{"_id": "15"}}
-          {"user": "test", "message": "some message with the number 15", "date": "2009-11-15T14:12:12", "likes": 15}
-          {"index":{"_id": "16"}}
-          {"user": "test", "message": "some message with the number 16", "date": "2009-11-15T14:12:12", "likes": 16}
-          {"index":{"_id": "17"}}
-          {"user": "test", "message": "some message with the number 17", "date": "2009-11-15T14:12:12", "likes": 17}
-          {"index":{"_id": "18"}}
-          {"user": "test", "message": "some message with the number 18", "date": "2009-11-15T14:12:12", "likes": 18}
-          {"index":{"_id": "19"}}
-          {"user": "test", "message": "some message with the number 19", "date": "2009-11-15T14:12:12", "likes": 19}
-          {"index":{"_id": "20"}}
-          {"user": "test", "message": "some message with the number 20", "date": "2009-11-15T14:12:12", "likes": 20}
-          {"index":{"_id": "21"}}
-          {"user": "test", "message": "some message with the number 21", "date": "2009-11-15T14:12:12", "likes": 21}
-          {"index":{"_id": "22"}}
-          {"user": "test", "message": "some message with the number 22", "date": "2009-11-15T14:12:12", "likes": 22}
-          {"index":{"_id": "23"}}
-          {"user": "test", "message": "some message with the number 23", "date": "2009-11-15T14:12:12", "likes": 23}
-          {"index":{"_id": "24"}}
-          {"user": "test", "message": "some message with the number 24", "date": "2009-11-15T14:12:12", "likes": 24}
-          {"index":{"_id": "25"}}
-          {"user": "test", "message": "some message with the number 25", "date": "2009-11-15T14:12:12", "likes": 25}
-          {"index":{"_id": "26"}}
-          {"user": "test", "message": "some message with the number 26", "date": "2009-11-15T14:12:12", "likes": 26}
-          {"index":{"_id": "27"}}
-          {"user": "test", "message": "some message with the number 27", "date": "2009-11-15T14:12:12", "likes": 27}
-          {"index":{"_id": "28"}}
-          {"user": "test", "message": "some message with the number 28", "date": "2009-11-15T14:12:12", "likes": 28}
-          {"index":{"_id": "29"}}
-          {"user": "test", "message": "some message with the number 29", "date": "2009-11-15T14:12:12", "likes": 29}
-          {"index":{"_id": "30"}}
-          {"user": "test", "message": "some message with the number 30", "date": "2009-11-15T14:12:12", "likes": 30}
-          {"index":{"_id": "31"}}
-          {"user": "test", "message": "some message with the number 31", "date": "2009-11-15T14:12:12", "likes": 31}
-          {"index":{"_id": "32"}}
-          {"user": "test", "message": "some message with the number 32", "date": "2009-11-15T14:12:12", "likes": 32}
-          {"index":{"_id": "33"}}
-          {"user": "test", "message": "some message with the number 33", "date": "2009-11-15T14:12:12", "likes": 33}
-          {"index":{"_id": "34"}}
-          {"user": "test", "message": "some message with the number 34", "date": "2009-11-15T14:12:12", "likes": 34}
-          {"index":{"_id": "35"}}
-          {"user": "test", "message": "some message with the number 35", "date": "2009-11-15T14:12:12", "likes": 35}
-          {"index":{"_id": "36"}}
-          {"user": "test", "message": "some message with the number 36", "date": "2009-11-15T14:12:12", "likes": 36}
-          {"index":{"_id": "37"}}
-          {"user": "test", "message": "some message with the number 37", "date": "2009-11-15T14:12:12", "likes": 37}
-          {"index":{"_id": "38"}}
-          {"user": "test", "message": "some message with the number 38", "date": "2009-11-15T14:12:12", "likes": 38}
-          {"index":{"_id": "39"}}
-          {"user": "test", "message": "some message with the number 39", "date": "2009-11-15T14:12:12", "likes": 39}
-          {"index":{"_id": "40"}}
-          {"user": "test", "message": "some message with the number 40", "date": "2009-11-15T14:12:12", "likes": 40}
-          {"index":{"_id": "41"}}
-          {"user": "test", "message": "some message with the number 41", "date": "2009-11-15T14:12:12", "likes": 41}
-          {"index":{"_id": "42"}}
-          {"user": "test", "message": "some message with the number 42", "date": "2009-11-15T14:12:12", "likes": 42}
-          {"index":{"_id": "43"}}
-          {"user": "test", "message": "some message with the number 43", "date": "2009-11-15T14:12:12", "likes": 43}
-          {"index":{"_id": "44"}}
-          {"user": "test", "message": "some message with the number 44", "date": "2009-11-15T14:12:12", "likes": 44}
-          {"index":{"_id": "45"}}
-          {"user": "test", "message": "some message with the number 45", "date": "2009-11-15T14:12:12", "likes": 45}
-          {"index":{"_id": "46"}}
-          {"user": "test", "message": "some message with the number 46", "date": "2009-11-15T14:12:12", "likes": 46}
-          {"index":{"_id": "47"}}
-          {"user": "test", "message": "some message with the number 47", "date": "2009-11-15T14:12:12", "likes": 47}
-          {"index":{"_id": "48"}}
-          {"user": "test", "message": "some message with the number 48", "date": "2009-11-15T14:12:12", "likes": 48}
-          {"index":{"_id": "49"}}
-          {"user": "test", "message": "some message with the number 49", "date": "2009-11-15T14:12:12", "likes": 49}
-          {"index":{"_id": "50"}}
-          {"user": "test", "message": "some message with the number 50", "date": "2009-11-15T14:12:12", "likes": 50}
-          {"index":{"_id": "51"}}
-          {"user": "test", "message": "some message with the number 51", "date": "2009-11-15T14:12:12", "likes": 51}
-          {"index":{"_id": "52"}}
-          {"user": "test", "message": "some message with the number 52", "date": "2009-11-15T14:12:12", "likes": 52}
-          {"index":{"_id": "53"}}
-          {"user": "test", "message": "some message with the number 53", "date": "2009-11-15T14:12:12", "likes": 53}
-          {"index":{"_id": "54"}}
-          {"user": "test", "message": "some message with the number 54", "date": "2009-11-15T14:12:12", "likes": 54}
-          {"index":{"_id": "55"}}
-          {"user": "test", "message": "some message with the number 55", "date": "2009-11-15T14:12:12", "likes": 55}
-          {"index":{"_id": "56"}}
-          {"user": "test", "message": "some message with the number 56", "date": "2009-11-15T14:12:12", "likes": 56}
-          {"index":{"_id": "57"}}
-          {"user": "test", "message": "some message with the number 57", "date": "2009-11-15T14:12:12", "likes": 57}
-          {"index":{"_id": "58"}}
-          {"user": "test", "message": "some message with the number 58", "date": "2009-11-15T14:12:12", "likes": 58}
-          {"index":{"_id": "59"}}
-          {"user": "test", "message": "some message with the number 59", "date": "2009-11-15T14:12:12", "likes": 59}
-          {"index":{"_id": "60"}}
-          {"user": "test", "message": "some message with the number 60", "date": "2009-11-15T14:12:12", "likes": 60}
-          {"index":{"_id": "61"}}
-          {"user": "test", "message": "some message with the number 61", "date": "2009-11-15T14:12:12", "likes": 61}
-          {"index":{"_id": "62"}}
-          {"user": "test", "message": "some message with the number 62", "date": "2009-11-15T14:12:12", "likes": 62}
-          {"index":{"_id": "63"}}
-          {"user": "test", "message": "some message with the number 63", "date": "2009-11-15T14:12:12", "likes": 63}
-          {"index":{"_id": "64"}}
-          {"user": "test", "message": "some message with the number 64", "date": "2009-11-15T14:12:12", "likes": 64}
-          {"index":{"_id": "65"}}
-          {"user": "test", "message": "some message with the number 65", "date": "2009-11-15T14:12:12", "likes": 65}
-          {"index":{"_id": "66"}}
-          {"user": "test", "message": "some message with the number 66", "date": "2009-11-15T14:12:12", "likes": 66}
-          {"index":{"_id": "67"}}
-          {"user": "test", "message": "some message with the number 67", "date": "2009-11-15T14:12:12", "likes": 67}
-          {"index":{"_id": "68"}}
-          {"user": "test", "message": "some message with the number 68", "date": "2009-11-15T14:12:12", "likes": 68}
-          {"index":{"_id": "69"}}
-          {"user": "test", "message": "some message with the number 69", "date": "2009-11-15T14:12:12", "likes": 69}
-          {"index":{"_id": "70"}}
-          {"user": "test", "message": "some message with the number 70", "date": "2009-11-15T14:12:12", "likes": 70}
-          {"index":{"_id": "71"}}
-          {"user": "test", "message": "some message with the number 71", "date": "2009-11-15T14:12:12", "likes": 71}
-          {"index":{"_id": "72"}}
-          {"user": "test", "message": "some message with the number 72", "date": "2009-11-15T14:12:12", "likes": 72}
-          {"index":{"_id": "73"}}
-          {"user": "test", "message": "some message with the number 73", "date": "2009-11-15T14:12:12", "likes": 73}
-          {"index":{"_id": "74"}}
-          {"user": "test", "message": "some message with the number 74", "date": "2009-11-15T14:12:12", "likes": 74}
-          {"index":{"_id": "75"}}
-          {"user": "test", "message": "some message with the number 75", "date": "2009-11-15T14:12:12", "likes": 75}
-          {"index":{"_id": "76"}}
-          {"user": "test", "message": "some message with the number 76", "date": "2009-11-15T14:12:12", "likes": 76}
-          {"index":{"_id": "77"}}
-          {"user": "test", "message": "some message with the number 77", "date": "2009-11-15T14:12:12", "likes": 77}
-          {"index":{"_id": "78"}}
-          {"user": "test", "message": "some message with the number 78", "date": "2009-11-15T14:12:12", "likes": 78}
-          {"index":{"_id": "79"}}
-          {"user": "test", "message": "some message with the number 79", "date": "2009-11-15T14:12:12", "likes": 79}
-          {"index":{"_id": "80"}}
-          {"user": "test", "message": "some message with the number 80", "date": "2009-11-15T14:12:12", "likes": 80}
-          {"index":{"_id": "81"}}
-          {"user": "test", "message": "some message with the number 81", "date": "2009-11-15T14:12:12", "likes": 81}
-          {"index":{"_id": "82"}}
-          {"user": "test", "message": "some message with the number 82", "date": "2009-11-15T14:12:12", "likes": 82}
-          {"index":{"_id": "83"}}
-          {"user": "test", "message": "some message with the number 83", "date": "2009-11-15T14:12:12", "likes": 83}
-          {"index":{"_id": "84"}}
-          {"user": "test", "message": "some message with the number 84", "date": "2009-11-15T14:12:12", "likes": 84}
-          {"index":{"_id": "85"}}
-          {"user": "test", "message": "some message with the number 85", "date": "2009-11-15T14:12:12", "likes": 85}
-          {"index":{"_id": "86"}}
-          {"user": "test", "message": "some message with the number 86", "date": "2009-11-15T14:12:12", "likes": 86}
-          {"index":{"_id": "87"}}
-          {"user": "test", "message": "some message with the number 87", "date": "2009-11-15T14:12:12", "likes": 87}
-          {"index":{"_id": "88"}}
-          {"user": "test", "message": "some message with the number 88", "date": "2009-11-15T14:12:12", "likes": 88}
-          {"index":{"_id": "89"}}
-          {"user": "test", "message": "some message with the number 89", "date": "2009-11-15T14:12:12", "likes": 89}
-          {"index":{"_id": "90"}}
-          {"user": "test", "message": "some message with the number 90", "date": "2009-11-15T14:12:12", "likes": 90}
-          {"index":{"_id": "91"}}
-          {"user": "test", "message": "some message with the number 91", "date": "2009-11-15T14:12:12", "likes": 91}
-          {"index":{"_id": "92"}}
-          {"user": "test", "message": "some message with the number 92", "date": "2009-11-15T14:12:12", "likes": 92}
-          {"index":{"_id": "93"}}
-          {"user": "test", "message": "some message with the number 93", "date": "2009-11-15T14:12:12", "likes": 93}
-          {"index":{"_id": "94"}}
-          {"user": "test", "message": "some message with the number 94", "date": "2009-11-15T14:12:12", "likes": 94}
-          {"index":{"_id": "95"}}
-          {"user": "test", "message": "some message with the number 95", "date": "2009-11-15T14:12:12", "likes": 95}
-          {"index":{"_id": "96"}}
-          {"user": "test", "message": "some message with the number 96", "date": "2009-11-15T14:12:12", "likes": 96}
-          {"index":{"_id": "97"}}
-          {"user": "test", "message": "some message with the number 97", "date": "2009-11-15T14:12:12", "likes": 97}
-          {"index":{"_id": "98"}}
-          {"user": "test", "message": "some message with the number 98", "date": "2009-11-15T14:12:12", "likes": 98}
-          {"index":{"_id": "99"}}
-          {"user": "test", "message": "some message with the number 99", "date": "2009-11-15T14:12:12", "likes": 99}
-          {"index":{"_id": "100"}}
-          {"user": "test", "message": "some message with the number 100", "date": "2009-11-15T14:12:12", "likes": 100}
-          {"index":{"_id": "101"}}
-          {"user": "test", "message": "some message with the number 101", "date": "2009-11-15T14:12:12", "likes": 101}
-          {"index":{"_id": "102"}}
-          {"user": "test", "message": "some message with the number 102", "date": "2009-11-15T14:12:12", "likes": 102}
-          {"index":{"_id": "103"}}
-          {"user": "test", "message": "some message with the number 103", "date": "2009-11-15T14:12:12", "likes": 103}
-          {"index":{"_id": "104"}}
-          {"user": "test", "message": "some message with the number 104", "date": "2009-11-15T14:12:12", "likes": 104}
-          {"index":{"_id": "105"}}
-          {"user": "test", "message": "some message with the number 105", "date": "2009-11-15T14:12:12", "likes": 105}
-          {"index":{"_id": "106"}}
-          {"user": "test", "message": "some message with the number 106", "date": "2009-11-15T14:12:12", "likes": 106}
-          {"index":{"_id": "107"}}
-          {"user": "test", "message": "some message with the number 107", "date": "2009-11-15T14:12:12", "likes": 107}
-          {"index":{"_id": "108"}}
-          {"user": "test", "message": "some message with the number 108", "date": "2009-11-15T14:12:12", "likes": 108}
-          {"index":{"_id": "109"}}
-          {"user": "test", "message": "some message with the number 109", "date": "2009-11-15T14:12:12", "likes": 109}
-          {"index":{"_id": "110"}}
-          {"user": "test", "message": "some message with the number 110", "date": "2009-11-15T14:12:12", "likes": 110}
-          {"index":{"_id": "111"}}
-          {"user": "test", "message": "some message with the number 111", "date": "2009-11-15T14:12:12", "likes": 111}
-          {"index":{"_id": "112"}}
-          {"user": "test", "message": "some message with the number 112", "date": "2009-11-15T14:12:12", "likes": 112}
-          {"index":{"_id": "113"}}
-          {"user": "test", "message": "some message with the number 113", "date": "2009-11-15T14:12:12", "likes": 113}
-          {"index":{"_id": "114"}}
-          {"user": "test", "message": "some message with the number 114", "date": "2009-11-15T14:12:12", "likes": 114}
-          {"index":{"_id": "115"}}
-          {"user": "test", "message": "some message with the number 115", "date": "2009-11-15T14:12:12", "likes": 115}
-          {"index":{"_id": "116"}}
-          {"user": "test", "message": "some message with the number 116", "date": "2009-11-15T14:12:12", "likes": 116}
-          {"index":{"_id": "117"}}
-          {"user": "test", "message": "some message with the number 117", "date": "2009-11-15T14:12:12", "likes": 117}
-          {"index":{"_id": "118"}}
-          {"user": "test", "message": "some message with the number 118", "date": "2009-11-15T14:12:12", "likes": 118}
-          {"index":{"_id": "119"}}
-          {"user": "test", "message": "some message with the number 119", "date": "2009-11-15T14:12:12", "likes": 119}
-emit snippet
-query part: conflicts=proceed
-handle snippet
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-emit snippet
-query part: conflicts=proceed
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-emit snippet
-query part: conflicts=proceed
-body part: 
-{
-  "query": {
-    "term": {
-      "user": "kimchy"
-    }
-  }
-}
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-emit snippet
-body part: 
-{
-  "script": {
-    "inline": "ctx._source.likes++"
-  },
-  "query": {
-    "term": {
-      "user": "kimchy"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-emit snippet
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-emit snippet
-query part: routing=1
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-emit snippet
-query part: scroll_size=100
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-emit snippet
-body part: 
-{
-  "description" : "sets foo",
-  "processors" : [ {
-      "set" : {
-        "field": "foo",
-        "value": "bar"
-      }
-  } ]
-}
-emit snippet
-query part: pipeline=set-foo
-handle snippet
-handle snippet
-test snippet
-emit snippet
-query part: detailed=true&action=*byquery
-handle snippet
-handle snippet
-test snippet
-emit snippet
-handle snippet
-test snippet
-emit snippet
-query part: requests_per_second=unlimited
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "test": {
-      "dynamic": false,
-      "properties": {
-        "text": {"type": "text"}
-      }
-    }
-  }
-}
-emit snippet
-query part: refresh
-body part: 
-{
-  "text": "words words",
-  "flag": "bar"
-}
-emit snippet
-query part: refresh
-body part: 
-{
-  "text": "words words",
-  "flag": "foo"
-}
-emit snippet
-body part: 
-{
-  "properties": {
-    "text": {"type": "text"},
-    "flag": {"type": "text", "analyzer": "keyword"}
-  }
-}
-handle snippet
-test snippet
-emit snippet
-query part: filter_path=hits.total
-body part: 
-{
-  "query": {
-    "match": {
-      "flag": "foo"
-    }
-  }
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-query part: refresh&conflicts=proceed
-emit snippet
-query part: filter_path=hits.total
-body part: 
-{
-  "query": {
-    "match": {
-      "flag": "foo"
-    }
-  }
-}
-handle snippet
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-          {"index":{"_id": "5"}}
-          {"user": "test", "message": "some message with the number 5", "date": "2009-11-15T14:12:12", "likes": 5}
-          {"index":{"_id": "6"}}
-          {"user": "test", "message": "some message with the number 6", "date": "2009-11-15T14:12:12", "likes": 6}
-          {"index":{"_id": "7"}}
-          {"user": "test", "message": "some message with the number 7", "date": "2009-11-15T14:12:12", "likes": 7}
-          {"index":{"_id": "8"}}
-          {"user": "test", "message": "some message with the number 8", "date": "2009-11-15T14:12:12", "likes": 8}
-          {"index":{"_id": "9"}}
-          {"user": "test", "message": "some message with the number 9", "date": "2009-11-15T14:12:12", "likes": 9}
-          {"index":{"_id": "10"}}
-          {"user": "test", "message": "some message with the number 10", "date": "2009-11-15T14:12:12", "likes": 10}
-          {"index":{"_id": "11"}}
-          {"user": "test", "message": "some message with the number 11", "date": "2009-11-15T14:12:12", "likes": 11}
-          {"index":{"_id": "12"}}
-          {"user": "test", "message": "some message with the number 12", "date": "2009-11-15T14:12:12", "likes": 12}
-          {"index":{"_id": "13"}}
-          {"user": "test", "message": "some message with the number 13", "date": "2009-11-15T14:12:12", "likes": 13}
-          {"index":{"_id": "14"}}
-          {"user": "test", "message": "some message with the number 14", "date": "2009-11-15T14:12:12", "likes": 14}
-          {"index":{"_id": "15"}}
-          {"user": "test", "message": "some message with the number 15", "date": "2009-11-15T14:12:12", "likes": 15}
-          {"index":{"_id": "16"}}
-          {"user": "test", "message": "some message with the number 16", "date": "2009-11-15T14:12:12", "likes": 16}
-          {"index":{"_id": "17"}}
-          {"user": "test", "message": "some message with the number 17", "date": "2009-11-15T14:12:12", "likes": 17}
-          {"index":{"_id": "18"}}
-          {"user": "test", "message": "some message with the number 18", "date": "2009-11-15T14:12:12", "likes": 18}
-          {"index":{"_id": "19"}}
-          {"user": "test", "message": "some message with the number 19", "date": "2009-11-15T14:12:12", "likes": 19}
-          {"index":{"_id": "20"}}
-          {"user": "test", "message": "some message with the number 20", "date": "2009-11-15T14:12:12", "likes": 20}
-          {"index":{"_id": "21"}}
-          {"user": "test", "message": "some message with the number 21", "date": "2009-11-15T14:12:12", "likes": 21}
-          {"index":{"_id": "22"}}
-          {"user": "test", "message": "some message with the number 22", "date": "2009-11-15T14:12:12", "likes": 22}
-          {"index":{"_id": "23"}}
-          {"user": "test", "message": "some message with the number 23", "date": "2009-11-15T14:12:12", "likes": 23}
-          {"index":{"_id": "24"}}
-          {"user": "test", "message": "some message with the number 24", "date": "2009-11-15T14:12:12", "likes": 24}
-          {"index":{"_id": "25"}}
-          {"user": "test", "message": "some message with the number 25", "date": "2009-11-15T14:12:12", "likes": 25}
-          {"index":{"_id": "26"}}
-          {"user": "test", "message": "some message with the number 26", "date": "2009-11-15T14:12:12", "likes": 26}
-          {"index":{"_id": "27"}}
-          {"user": "test", "message": "some message with the number 27", "date": "2009-11-15T14:12:12", "likes": 27}
-          {"index":{"_id": "28"}}
-          {"user": "test", "message": "some message with the number 28", "date": "2009-11-15T14:12:12", "likes": 28}
-          {"index":{"_id": "29"}}
-          {"user": "test", "message": "some message with the number 29", "date": "2009-11-15T14:12:12", "likes": 29}
-          {"index":{"_id": "30"}}
-          {"user": "test", "message": "some message with the number 30", "date": "2009-11-15T14:12:12", "likes": 30}
-          {"index":{"_id": "31"}}
-          {"user": "test", "message": "some message with the number 31", "date": "2009-11-15T14:12:12", "likes": 31}
-          {"index":{"_id": "32"}}
-          {"user": "test", "message": "some message with the number 32", "date": "2009-11-15T14:12:12", "likes": 32}
-          {"index":{"_id": "33"}}
-          {"user": "test", "message": "some message with the number 33", "date": "2009-11-15T14:12:12", "likes": 33}
-          {"index":{"_id": "34"}}
-          {"user": "test", "message": "some message with the number 34", "date": "2009-11-15T14:12:12", "likes": 34}
-          {"index":{"_id": "35"}}
-          {"user": "test", "message": "some message with the number 35", "date": "2009-11-15T14:12:12", "likes": 35}
-          {"index":{"_id": "36"}}
-          {"user": "test", "message": "some message with the number 36", "date": "2009-11-15T14:12:12", "likes": 36}
-          {"index":{"_id": "37"}}
-          {"user": "test", "message": "some message with the number 37", "date": "2009-11-15T14:12:12", "likes": 37}
-          {"index":{"_id": "38"}}
-          {"user": "test", "message": "some message with the number 38", "date": "2009-11-15T14:12:12", "likes": 38}
-          {"index":{"_id": "39"}}
-          {"user": "test", "message": "some message with the number 39", "date": "2009-11-15T14:12:12", "likes": 39}
-          {"index":{"_id": "40"}}
-          {"user": "test", "message": "some message with the number 40", "date": "2009-11-15T14:12:12", "likes": 40}
-          {"index":{"_id": "41"}}
-          {"user": "test", "message": "some message with the number 41", "date": "2009-11-15T14:12:12", "likes": 41}
-          {"index":{"_id": "42"}}
-          {"user": "test", "message": "some message with the number 42", "date": "2009-11-15T14:12:12", "likes": 42}
-          {"index":{"_id": "43"}}
-          {"user": "test", "message": "some message with the number 43", "date": "2009-11-15T14:12:12", "likes": 43}
-          {"index":{"_id": "44"}}
-          {"user": "test", "message": "some message with the number 44", "date": "2009-11-15T14:12:12", "likes": 44}
-          {"index":{"_id": "45"}}
-          {"user": "test", "message": "some message with the number 45", "date": "2009-11-15T14:12:12", "likes": 45}
-          {"index":{"_id": "46"}}
-          {"user": "test", "message": "some message with the number 46", "date": "2009-11-15T14:12:12", "likes": 46}
-          {"index":{"_id": "47"}}
-          {"user": "test", "message": "some message with the number 47", "date": "2009-11-15T14:12:12", "likes": 47}
-          {"index":{"_id": "48"}}
-          {"user": "test", "message": "some message with the number 48", "date": "2009-11-15T14:12:12", "likes": 48}
-          {"index":{"_id": "49"}}
-          {"user": "test", "message": "some message with the number 49", "date": "2009-11-15T14:12:12", "likes": 49}
-          {"index":{"_id": "50"}}
-          {"user": "test", "message": "some message with the number 50", "date": "2009-11-15T14:12:12", "likes": 50}
-          {"index":{"_id": "51"}}
-          {"user": "test", "message": "some message with the number 51", "date": "2009-11-15T14:12:12", "likes": 51}
-          {"index":{"_id": "52"}}
-          {"user": "test", "message": "some message with the number 52", "date": "2009-11-15T14:12:12", "likes": 52}
-          {"index":{"_id": "53"}}
-          {"user": "test", "message": "some message with the number 53", "date": "2009-11-15T14:12:12", "likes": 53}
-          {"index":{"_id": "54"}}
-          {"user": "test", "message": "some message with the number 54", "date": "2009-11-15T14:12:12", "likes": 54}
-          {"index":{"_id": "55"}}
-          {"user": "test", "message": "some message with the number 55", "date": "2009-11-15T14:12:12", "likes": 55}
-          {"index":{"_id": "56"}}
-          {"user": "test", "message": "some message with the number 56", "date": "2009-11-15T14:12:12", "likes": 56}
-          {"index":{"_id": "57"}}
-          {"user": "test", "message": "some message with the number 57", "date": "2009-11-15T14:12:12", "likes": 57}
-          {"index":{"_id": "58"}}
-          {"user": "test", "message": "some message with the number 58", "date": "2009-11-15T14:12:12", "likes": 58}
-          {"index":{"_id": "59"}}
-          {"user": "test", "message": "some message with the number 59", "date": "2009-11-15T14:12:12", "likes": 59}
-          {"index":{"_id": "60"}}
-          {"user": "test", "message": "some message with the number 60", "date": "2009-11-15T14:12:12", "likes": 60}
-          {"index":{"_id": "61"}}
-          {"user": "test", "message": "some message with the number 61", "date": "2009-11-15T14:12:12", "likes": 61}
-          {"index":{"_id": "62"}}
-          {"user": "test", "message": "some message with the number 62", "date": "2009-11-15T14:12:12", "likes": 62}
-          {"index":{"_id": "63"}}
-          {"user": "test", "message": "some message with the number 63", "date": "2009-11-15T14:12:12", "likes": 63}
-          {"index":{"_id": "64"}}
-          {"user": "test", "message": "some message with the number 64", "date": "2009-11-15T14:12:12", "likes": 64}
-          {"index":{"_id": "65"}}
-          {"user": "test", "message": "some message with the number 65", "date": "2009-11-15T14:12:12", "likes": 65}
-          {"index":{"_id": "66"}}
-          {"user": "test", "message": "some message with the number 66", "date": "2009-11-15T14:12:12", "likes": 66}
-          {"index":{"_id": "67"}}
-          {"user": "test", "message": "some message with the number 67", "date": "2009-11-15T14:12:12", "likes": 67}
-          {"index":{"_id": "68"}}
-          {"user": "test", "message": "some message with the number 68", "date": "2009-11-15T14:12:12", "likes": 68}
-          {"index":{"_id": "69"}}
-          {"user": "test", "message": "some message with the number 69", "date": "2009-11-15T14:12:12", "likes": 69}
-          {"index":{"_id": "70"}}
-          {"user": "test", "message": "some message with the number 70", "date": "2009-11-15T14:12:12", "likes": 70}
-          {"index":{"_id": "71"}}
-          {"user": "test", "message": "some message with the number 71", "date": "2009-11-15T14:12:12", "likes": 71}
-          {"index":{"_id": "72"}}
-          {"user": "test", "message": "some message with the number 72", "date": "2009-11-15T14:12:12", "likes": 72}
-          {"index":{"_id": "73"}}
-          {"user": "test", "message": "some message with the number 73", "date": "2009-11-15T14:12:12", "likes": 73}
-          {"index":{"_id": "74"}}
-          {"user": "test", "message": "some message with the number 74", "date": "2009-11-15T14:12:12", "likes": 74}
-          {"index":{"_id": "75"}}
-          {"user": "test", "message": "some message with the number 75", "date": "2009-11-15T14:12:12", "likes": 75}
-          {"index":{"_id": "76"}}
-          {"user": "test", "message": "some message with the number 76", "date": "2009-11-15T14:12:12", "likes": 76}
-          {"index":{"_id": "77"}}
-          {"user": "test", "message": "some message with the number 77", "date": "2009-11-15T14:12:12", "likes": 77}
-          {"index":{"_id": "78"}}
-          {"user": "test", "message": "some message with the number 78", "date": "2009-11-15T14:12:12", "likes": 78}
-          {"index":{"_id": "79"}}
-          {"user": "test", "message": "some message with the number 79", "date": "2009-11-15T14:12:12", "likes": 79}
-          {"index":{"_id": "80"}}
-          {"user": "test", "message": "some message with the number 80", "date": "2009-11-15T14:12:12", "likes": 80}
-          {"index":{"_id": "81"}}
-          {"user": "test", "message": "some message with the number 81", "date": "2009-11-15T14:12:12", "likes": 81}
-          {"index":{"_id": "82"}}
-          {"user": "test", "message": "some message with the number 82", "date": "2009-11-15T14:12:12", "likes": 82}
-          {"index":{"_id": "83"}}
-          {"user": "test", "message": "some message with the number 83", "date": "2009-11-15T14:12:12", "likes": 83}
-          {"index":{"_id": "84"}}
-          {"user": "test", "message": "some message with the number 84", "date": "2009-11-15T14:12:12", "likes": 84}
-          {"index":{"_id": "85"}}
-          {"user": "test", "message": "some message with the number 85", "date": "2009-11-15T14:12:12", "likes": 85}
-          {"index":{"_id": "86"}}
-          {"user": "test", "message": "some message with the number 86", "date": "2009-11-15T14:12:12", "likes": 86}
-          {"index":{"_id": "87"}}
-          {"user": "test", "message": "some message with the number 87", "date": "2009-11-15T14:12:12", "likes": 87}
-          {"index":{"_id": "88"}}
-          {"user": "test", "message": "some message with the number 88", "date": "2009-11-15T14:12:12", "likes": 88}
-          {"index":{"_id": "89"}}
-          {"user": "test", "message": "some message with the number 89", "date": "2009-11-15T14:12:12", "likes": 89}
-          {"index":{"_id": "90"}}
-          {"user": "test", "message": "some message with the number 90", "date": "2009-11-15T14:12:12", "likes": 90}
-          {"index":{"_id": "91"}}
-          {"user": "test", "message": "some message with the number 91", "date": "2009-11-15T14:12:12", "likes": 91}
-          {"index":{"_id": "92"}}
-          {"user": "test", "message": "some message with the number 92", "date": "2009-11-15T14:12:12", "likes": 92}
-          {"index":{"_id": "93"}}
-          {"user": "test", "message": "some message with the number 93", "date": "2009-11-15T14:12:12", "likes": 93}
-          {"index":{"_id": "94"}}
-          {"user": "test", "message": "some message with the number 94", "date": "2009-11-15T14:12:12", "likes": 94}
-          {"index":{"_id": "95"}}
-          {"user": "test", "message": "some message with the number 95", "date": "2009-11-15T14:12:12", "likes": 95}
-          {"index":{"_id": "96"}}
-          {"user": "test", "message": "some message with the number 96", "date": "2009-11-15T14:12:12", "likes": 96}
-          {"index":{"_id": "97"}}
-          {"user": "test", "message": "some message with the number 97", "date": "2009-11-15T14:12:12", "likes": 97}
-          {"index":{"_id": "98"}}
-          {"user": "test", "message": "some message with the number 98", "date": "2009-11-15T14:12:12", "likes": 98}
-          {"index":{"_id": "99"}}
-          {"user": "test", "message": "some message with the number 99", "date": "2009-11-15T14:12:12", "likes": 99}
-          {"index":{"_id": "100"}}
-          {"user": "test", "message": "some message with the number 100", "date": "2009-11-15T14:12:12", "likes": 100}
-          {"index":{"_id": "101"}}
-          {"user": "test", "message": "some message with the number 101", "date": "2009-11-15T14:12:12", "likes": 101}
-          {"index":{"_id": "102"}}
-          {"user": "test", "message": "some message with the number 102", "date": "2009-11-15T14:12:12", "likes": 102}
-          {"index":{"_id": "103"}}
-          {"user": "test", "message": "some message with the number 103", "date": "2009-11-15T14:12:12", "likes": 103}
-          {"index":{"_id": "104"}}
-          {"user": "test", "message": "some message with the number 104", "date": "2009-11-15T14:12:12", "likes": 104}
-          {"index":{"_id": "105"}}
-          {"user": "test", "message": "some message with the number 105", "date": "2009-11-15T14:12:12", "likes": 105}
-          {"index":{"_id": "106"}}
-          {"user": "test", "message": "some message with the number 106", "date": "2009-11-15T14:12:12", "likes": 106}
-          {"index":{"_id": "107"}}
-          {"user": "test", "message": "some message with the number 107", "date": "2009-11-15T14:12:12", "likes": 107}
-          {"index":{"_id": "108"}}
-          {"user": "test", "message": "some message with the number 108", "date": "2009-11-15T14:12:12", "likes": 108}
-          {"index":{"_id": "109"}}
-          {"user": "test", "message": "some message with the number 109", "date": "2009-11-15T14:12:12", "likes": 109}
-          {"index":{"_id": "110"}}
-          {"user": "test", "message": "some message with the number 110", "date": "2009-11-15T14:12:12", "likes": 110}
-          {"index":{"_id": "111"}}
-          {"user": "test", "message": "some message with the number 111", "date": "2009-11-15T14:12:12", "likes": 111}
-          {"index":{"_id": "112"}}
-          {"user": "test", "message": "some message with the number 112", "date": "2009-11-15T14:12:12", "likes": 112}
-          {"index":{"_id": "113"}}
-          {"user": "test", "message": "some message with the number 113", "date": "2009-11-15T14:12:12", "likes": 113}
-          {"index":{"_id": "114"}}
-          {"user": "test", "message": "some message with the number 114", "date": "2009-11-15T14:12:12", "likes": 114}
-          {"index":{"_id": "115"}}
-          {"user": "test", "message": "some message with the number 115", "date": "2009-11-15T14:12:12", "likes": 115}
-          {"index":{"_id": "116"}}
-          {"user": "test", "message": "some message with the number 116", "date": "2009-11-15T14:12:12", "likes": 116}
-          {"index":{"_id": "117"}}
-          {"user": "test", "message": "some message with the number 117", "date": "2009-11-15T14:12:12", "likes": 117}
-          {"index":{"_id": "118"}}
-          {"user": "test", "message": "some message with the number 118", "date": "2009-11-15T14:12:12", "likes": 118}
-          {"index":{"_id": "119"}}
-          {"user": "test", "message": "some message with the number 119", "date": "2009-11-15T14:12:12", "likes": 119}
-emit snippet
-body part: 
-{
-  "source": {
-    "index": "twitter"
-  },
-  "dest": {
-    "index": "new_twitter"
-  }
-}
-handle snippet
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-emit snippet
-body part: 
-{
-  "source": {
-    "index": "twitter"
-  },
-  "dest": {
-    "index": "new_twitter",
-    "version_type": "internal"
-  }
-}
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-emit snippet
-body part: 
-{
-  "source": {
-    "index": "twitter"
-  },
-  "dest": {
-    "index": "new_twitter",
-    "version_type": "external"
-  }
-}
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-emit snippet
-body part: 
-{
-  "source": {
-    "index": "twitter"
-  },
-  "dest": {
-    "index": "new_twitter",
-    "op_type": "create"
-  }
-}
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-emit snippet
-body part: 
-{
-  "conflicts": "proceed",
-  "source": {
-    "index": "twitter"
-  },
-  "dest": {
-    "index": "new_twitter",
-    "op_type": "create"
-  }
-}
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-emit snippet
-body part: 
-{
-  "source": {
-    "index": "twitter",
-    "type": "tweet",
-    "query": {
-      "term": {
-        "user": "kimchy"
-      }
-    }
-  },
-  "dest": {
-    "index": "new_twitter"
-  }
-}
-handle snippet
-test snippet
-emit snippet
-emit snippet
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "source": {
-    "index": ["twitter", "blog"],
-    "type": ["tweet", "post"]
-  },
-  "dest": {
-    "index": "all_together"
-  }
-}
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-emit snippet
-body part: 
-{
-  "size": 1,
-  "source": {
-    "index": "twitter"
-  },
-  "dest": {
-    "index": "new_twitter"
-  }
-}
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-emit snippet
-body part: 
-{
-  "size": 10000,
-  "source": {
-    "index": "twitter",
-    "sort": { "date": "desc" }
-  },
-  "dest": {
-    "index": "new_twitter"
-  }
-}
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-emit snippet
-body part: 
-{
-  "source": {
-    "index": "twitter"
-  },
-  "dest": {
-    "index": "new_twitter",
-    "version_type": "external"
-  },
-  "script": {
-    "inline": "if (ctx._source.foo == 'bar') {ctx._version++; ctx._source.remove('foo')}"
-  }
-}
-handle snippet
-test snippet
-emit snippet
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "source": {
-    "index": "source",
-    "query": {
-      "match": {
-        "company": "cat"
-      }
-    }
-  },
-  "dest": {
-    "index": "dest",
-    "routing": "=cat"
-  }
-}
-handle snippet
-test snippet
-emit snippet
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "source": {
-    "index": "source",
-    "size": 100
-  },
-  "dest": {
-    "index": "dest",
-    "routing": "=cat"
-  }
-}
-handle snippet
-test snippet
-emit snippet
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-body part: 
-{
-  "source": {
-    "index": "source"
-  },
-  "dest": {
-    "index": "dest",
-    "pipeline": "some_ingest_pipeline"
-  }
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-query part: detailed=true&actions=*reindex
-handle snippet
-handle snippet
-test snippet
-emit snippet
-handle snippet
-test snippet
-emit snippet
-query part: requests_per_second=unlimited
-handle snippet
-test snippet
-emit snippet
-query part: refresh
-body part: 
-{
-  "text": "words words",
-  "flag": "foo"
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "source": {
-    "index": "test"
-  },
-  "dest": {
-    "index": "test2"
-  },
-  "script": {
-    "inline": "ctx._source.tag = ctx._source.remove(\"flag\")"
-  }
-}
-handle snippet
-test snippet
-emit snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-catch part: request
-query part: pipeline=my_pipeline_id
-body part: 
-{
-  "foo": "bar"
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "transient": {
-    "cluster.routing.allocation.enable": "none"
-  }
-}
-handle snippet
-test snippet
-emit snippet
-handle snippet
-test snippet
-emit snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "transient": {
-    "cluster.routing.allocation.enable": "all"
-  }
-}
-handle snippet
-test snippet
-emit snippet
-handle snippet
-test snippet
-emit snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "persistent": {
-    "cluster.routing.allocation.enable": "none"
-  }
-}
-handle snippet
-test snippet
-emit snippet
-handle snippet
-test snippet
-emit snippet
-emit snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "persistent": {
-    "cluster.routing.allocation.enable": "all"
-  }
-}
-handle snippet
-test snippet
-emit snippet
-emit snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "user": {
-      "_all":       { "enabled": false  },
-      "properties": {
-        "title":    { "type": "text"  },
-        "name":     { "type": "text"  },
-        "age":      { "type": "integer" }
-      }
-    },
-    "blogpost": {
-      "_all":       { "enabled": false  },
-      "properties": {
-        "title":    { "type": "text"  },
-        "body":     { "type": "text"  },
-        "user_id":  {
-          "type":   "keyword"
-        },
-        "created":  {
-          "type":   "date",
-          "format": "strict_date_optional_time||epoch_millis"
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{ "count": 5 }
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "index.mapper.dynamic":false
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "session": {
-      "properties": {
-        "user_id": {
-          "type":  "keyword"
-        },
-        "last_updated": {
-          "type": "date"
-        },
-        "session_data": {
-          "enabled": false
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "user_id": "kimchy",
-  "session_data": {
-    "arbitrary_object": {
-      "some_array": [ "foo", "bar", { "baz": 2 } ]
-    }
-  },
-  "last_updated": "2015-12-06T18:20:22"
-}
-emit snippet
-body part: 
-{
-  "user_id": "jpountz",
-  "session_data": "none",
-  "last_updated": "2015-12-06T18:22:13"
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "session": {
-      "enabled": false
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "user_id": "kimchy",
-  "session_data": {
-    "arbitrary_object": {
-      "some_array": [ "foo", "bar", { "baz": 2 } ]
-    }
-  },
-  "last_updated": "2015-12-06T18:20:22"
-}
-emit snippet
-emit snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "location": {
-          "type": "geo_point",
-          "geohash_prefix": true,
-          "geohash_precision": 6
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "location": {
-    "lat": 41.12,
-    "lon": -71.34
-  }
-}
-emit snippet
-query part: fielddata_fields=location.geohash
-body part: 
-{
-  "query": {
-    "term": {
-      "location.geohash": "drm3bt"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "date": {
-          "type":   "date",
-          "format": "yyyy-MM-dd"
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "first_name": {
-          "type": "text",
-          "copy_to": "full_name"
-        },
-        "last_name": {
-          "type": "text",
-          "copy_to": "full_name"
-        },
-        "full_name": {
-          "type": "text"
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "first_name": "John",
-  "last_name": "Smith"
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "match": {
-      "full_name": {
-        "query": "John Smith",
-        "operator": "and"
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "title": {
-          "type": "text",
-          "store": true
-        },
-        "date": {
-          "type": "date",
-          "store": true
-        },
-        "content": {
-          "type": "text"
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "title":   "Some short title",
-  "date":    "2015-01-01",
-  "content": "A very long content field..."
-}
-emit snippet
-body part: 
-{
-  "fields": [ "title", "date" ]
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "status_code": {
-          "type":       "keyword",
-          "null_value": "NULL"
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "status_code": null
-}
-emit snippet
-body part: 
-{
-  "status_code": []
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "term": {
-      "status_code": "NULL"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "number_one": {
-          "type": "integer"
-        },
-        "number_two": {
-          "type": "integer",
-          "coerce": false
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "number_one": "10"
-}
-emit snippet
-catch part: request
-body part: 
-{
-  "number_two": "10"
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "index.mapping.coerce": false
-  },
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "number_one": {
-          "type": "integer",
-          "coerce": true
-        },
-        "number_two": {
-          "type": "integer"
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{ "number_one": "10" }
-emit snippet
-catch part: request
-body part: 
-{ "number_two": "10" }
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "analysis": {
-      "filter": {
-        "autocomplete_filter": {
-          "type": "edge_ngram",
-          "min_gram": 1,
-          "max_gram": 20
-        }
-      },
-      "analyzer": {
-        "autocomplete": {
-          "type": "custom",
-          "tokenizer": "standard",
-          "filter": [
-            "lowercase",
-            "autocomplete_filter"
-          ]
-        }
-      }
-    }
-  },
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "text": {
-          "type": "text",
-          "analyzer": "autocomplete",
-          "search_analyzer": "standard"
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "text": "Quick Brown Fox"
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "match": {
-      "text": {
-        "query": "Quick Br",
-        "operator": "and"
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "text": {
-          "type":        "text",
-          "term_vector": "with_positions_offsets"
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "text": "Quick brown fox"
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "match": {
-      "text": "brown fox"
-    }
-  },
-  "highlight": {
-    "fields": {
-      "text": {}
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "city": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type":  "keyword"
-            }
-          }
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "city": "New York"
-}
-emit snippet
-body part: 
-{
-  "city": "York"
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "match": {
-      "city": "york"
-    }
-  },
-  "sort": {
-    "city.raw": "asc"
-  },
-  "aggs": {
-    "Cities": {
-      "terms": {
-        "field": "city.raw"
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "text": {
-          "type": "text",
-          "fields": {
-            "english": {
-              "type":     "text",
-              "analyzer": "english"
-            }
-          }
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{ "text": "quick brown fox" }
-emit snippet
-body part: 
-{ "text": "quick brown foxes" }
-emit snippet
-body part: 
-{
-  "query": {
-    "multi_match": {
-      "query": "quick brown foxes",
-      "fields": [
-        "text",
-        "text.english"
-      ],
-      "type": "most_fields"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "tag": {
-          "type": "text",
-          "fielddata": true,
-          "fielddata_frequency_filter": {
-            "min": 0.001,
-            "max": 0.1,
-            "min_segment_size": 500
-          }
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "number_one": {
-          "type": "integer",
-          "ignore_malformed": true
-        },
-        "number_two": {
-          "type": "integer"
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "text":       "Some text value",
-  "number_one": "foo"
-}
-emit snippet
-catch part: request
-body part: 
-{
-  "text":       "Some text value",
-  "number_two": "foo"
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "index.mapping.ignore_malformed": true
-  },
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "number_one": {
-          "type": "byte"
-        },
-        "number_two": {
-          "type": "integer",
-          "ignore_malformed": false
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "username": "johnsmith",
-  "name": {
-    "first": "John",
-    "last": "Smith"
-  }
-}
-emit snippet
-emit snippet
-body part: 
-{
-  "username": "marywhite",
-  "email": "mary@white.com",
-  "name": {
-    "first": "Mary",
-    "middle": "Alice",
-    "last": "White"
-  }
-}
-emit snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "dynamic": false,
-      "properties": {
-        "user": {
-          "properties": {
-            "name": {
-              "type": "text"
-            },
-            "social_networks": {
-              "dynamic": true,
-              "properties": {}
-            }
-          }
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "names": [ "John Abraham", "Lincoln Smith"]
-}
-emit snippet
-body part: 
-{
-    "query": {
-        "match_phrase": {
-            "names": {
-                "query": "Abraham Lincoln"
-            }
-        }
-    }
-}
-emit snippet
-body part: 
-{
-    "query": {
-        "match_phrase": {
-            "names": {
-                "query": "Abraham Lincoln",
-                "slop": 101
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "groups": {
-      "properties": {
-        "names": {
-          "type": "text",
-          "position_increment_gap": 0
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-    "names": [ "John Abraham", "Lincoln Smith"]
-}
-emit snippet
-body part: 
-{
-    "query": {
-        "match_phrase": {
-            "names": "Abraham Lincoln"
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "message": {
-          "type": "keyword",
-          "ignore_above": 20
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "message": "Syntax error"
-}
-emit snippet
-body part: 
-{
-  "message": "Syntax error with some long stacktrace"
-}
-emit snippet
-body part: 
-{
-  "aggs": {
-    "messages": {
-      "terms": {
-        "field": "message"
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "location": {
-          "type": "geo_point",
-          "lat_lon": true
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "location": {
-    "lat": 41.12,
-    "lon": -71.34
-  }
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "geo_distance": {
-      "location": {
-        "lat": 41,
-        "lon": -71
-      },
-      "distance": "50km",
-      "optimize_bbox": "indexed"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "location": {
-          "type": "geo_point",
-          "geohash_prefix": true,
-          "geohash_precision": 6
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "location": {
-    "lat": 41.12,
-    "lon": -71.34
-  }
-}
-emit snippet
-query part: fielddata_fields=location.geohash
-body part: 
-{
-  "query": {
-    "geohash_cell": {
-      "location": {
-        "lat": 41.02,
-        "lon": -71.48
-      },
-      "precision": 4,
-      "neighbors": true
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "status_code": {
-          "type":       "keyword"
-        },
-        "session_id": {
-          "type":       "keyword",
-          "doc_values": false
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "text": {
-          "type": "text",
-          "index_options": "offsets"
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "text": "Quick brown fox"
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "match": {
-      "text": "brown fox"
-    }
-  },
-  "highlight": {
-    "fields": {
-      "text": {}
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "manager": {
-          "properties": {
-            "age":  { "type": "integer" },
-            "name": { "type": "text"  }
-          }
-        },
-        "employees": {
-          "type": "nested",
-          "properties": {
-            "age":  { "type": "integer" },
-            "name": { "type": "text"  }
-          }
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "region": "US",
-  "manager": {
-    "name": "Alice White",
-    "age": 30
-  },
-  "employees": [
-    {
-      "name": "John Smith",
-      "age": 34
-    },
-    {
-      "name": "Peter Brown",
-      "age": 26
-    }
-  ]
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "match": {
-      "manager.name": "Alice White"
-    }
-  },
-  "aggs": {
-    "Employees": {
-      "nested": {
-        "path": "employees"
-      },
-      "aggs": {
-        "Employee Ages": {
-          "histogram": {
-            "field": "employees.age",
-            "interval": 5
-          }
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "title": {
-          "type": "text"
-        },
-        "content": {
-          "type": "text"
-        },
-        "date": {
-          "type": "date",
-          "include_in_all": false
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "include_in_all": false,
-      "properties": {
-        "title":          { "type": "text" },
-        "author": {
-          "include_in_all": true,
-          "properties": {
-            "first_name": { "type": "text" },
-            "last_name":  { "type": "text" }
-          }
-        },
-        "editor": {
-          "properties": {
-            "first_name": { "type": "text" },
-            "last_name":  { "type": "text", "include_in_all": true }
-          }
-        }
-      }
-    }
-  }
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-emit snippet
-body part: 
-{
-  "properties": {
-    "title": {
-      "type": "text",
-      "norms": false
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "title": {
-          "type": "text",
-          "boost": 2
-        },
-        "content": {
-          "type": "text"
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "match" : {
-            "title": {
-                "query": "quick brown fox"
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "query": {
-        "match" : {
-            "title": {
-                "query": "quick brown fox",
-                "boost": 2
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "default_field": {
-          "type": "text"
-        },
-        "bm25_field": {
-          "type": "text",
-          "similarity": "BM25"
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "location": {
-          "type": "geo_point",
-          "geohash": true
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "location": {
-    "lat": 41.12,
-    "lon": -71.34
-  }
-}
-emit snippet
-query part: fielddata_fields=location.geohash
-body part: 
-{
-  "query": {
-    "prefix": {
-      "location.geohash": "drm3b"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "text": {
-          "type": "text",
-          "fields": {
-            "english": {
-              "type":     "text",
-              "analyzer": "english"
-            }
-          }
-        }
-      }
-    }
-  }
-}
-emit snippet
-query part: wait_for_status=yellow
-emit snippet
-query part: field=text
-body part: 
-{
-  "text": "The quick Brown Foxes."
-}
-emit snippet
-query part: field=text.english
-body part: 
-{
-  "text": "The quick Brown Foxes."
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-   "settings":{
-      "analysis":{
-         "analyzer":{
-            "my_analyzer":{
-               "type":"custom",
-               "tokenizer":"standard",
-               "filter":[
-                  "lowercase"
-               ]
-            },
-            "my_stop_analyzer":{
-               "type":"custom",
-               "tokenizer":"standard",
-               "filter":[
-                  "lowercase",
-                  "english_stop"
-               ]
-            }
-         },
-         "filter":{
-            "english_stop":{
-               "type":"stop",
-               "stopwords":"_english_"
-            }
-         }
-      }
-   },
-   "mappings":{
-      "my_type":{
-         "properties":{
-            "title": {
-               "type":"text",
-               "analyzer":"my_analyzer",
-               "search_analyzer":"my_stop_analyzer",
-               "search_quote_analyzer":"my_analyzer"
-            }
-         }
-      }
-   }
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "region": "US",
-  "manager": {
-    "age":     30,
-    "name": {
-      "first": "John",
-      "last":  "Smith"
-    }
-  }
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "region": {
-          "type": "keyword"
-        },
-        "manager": {
-          "properties": {
-            "age":  { "type": "integer" },
-            "name": {
-              "properties": {
-                "first": { "type": "text" },
-                "last":  { "type": "text" }
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "date": {
-          "type": "date"
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{ "date": "2015-01-01" }
-emit snippet
-body part: 
-{ "date": "2015-01-01T12:10:30Z" }
-emit snippet
-body part: 
-{ "date": 1420070400001 }
-emit snippet
-body part: 
-{
-  "sort": { "date": "asc"}
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "date": {
-          "type":   "date",
-          "format": "yyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "full_name": {
-          "type":  "text"
-        }
-      }
-    }
-  }
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "name": {
-          "type": "text",
-          "fields": {
-            "length": {
-              "type":     "token_count",
-              "analyzer": "standard"
-            }
-          }
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{ "name": "John Smith" }
-emit snippet
-body part: 
-{ "name": "Rachel Alice Williams" }
-emit snippet
-body part: 
-{
-  "query": {
-    "term": {
-      "name.length": 3
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "group" : "fans",
-  "user" : [
-    {
-      "first" : "John",
-      "last" :  "Smith"
-    },
-    {
-      "first" : "Alice",
-      "last" :  "White"
-    }
-  ]
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "bool": {
-      "must": [
-        { "match": { "user.first": "Alice" }},
-        { "match": { "user.last":  "Smith" }}
-      ]
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "user": {
-          "type": "nested"
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "group" : "fans",
-  "user" : [
-    {
-      "first" : "John",
-      "last" :  "Smith"
-    },
-    {
-      "first" : "Alice",
-      "last" :  "White"
-    }
-  ]
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "nested": {
-      "path": "user",
-      "query": {
-        "bool": {
-          "must": [
-            { "match": { "user.first": "Alice" }},
-            { "match": { "user.last":  "Smith" }}
-          ]
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "nested": {
-      "path": "user",
-      "query": {
-        "bool": {
-          "must": [
-            { "match": { "user.first": "Alice" }},
-            { "match": { "user.last":  "White" }}
-          ]
-        }
-      },
-      "inner_hits": {
-        "highlight": {
-          "fields": {
-            "user.first": {}
-          }
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "is_published": {
-          "type": "boolean"
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "is_published": true
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "term": {
-      "is_published": 1
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "is_published": true
-}
-emit snippet
-body part: 
-{
-  "is_published": false
-}
-emit snippet
-body part: 
-{
-  "aggs": {
-    "publish_state": {
-      "terms": {
-        "field": "is_published"
-      }
-    }
-  },
-  "script_fields": {
-    "is_published": {
-      "script": "doc['is_published'].value"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "number_of_bytes": {
-          "type": "integer"
-        },
-        "time_in_seconds": {
-          "type": "float"
-        }
-      }
-    }
-  }
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "tags": {
-          "type":  "keyword"
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "location": {
-          "type": "geo_point"
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "text": "Geo-point as an object",
-  "location": {
-    "lat": 41.12,
-    "lon": -71.34
-  }
-}
-emit snippet
-body part: 
-{
-  "text": "Geo-point as a string",
-  "location": "41.12,-71.34"
-}
-emit snippet
-body part: 
-{
-  "text": "Geo-point as a geohash",
-  "location": "drm3btev3e86"
-}
-emit snippet
-body part: 
-{
-  "text": "Geo-point as an array",
-  "location": [ -71.34, 41.12 ]
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "geo_bounding_box": {
-      "location": {
-        "top_left": {
-          "lat": 42,
-          "lon": -72
-        },
-        "bottom_right": {
-          "lat": 40,
-          "lon": -74
-        }
-      }
-    }
-  }
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "message": "some arrays in this document...",
-  "tags":  [ "elasticsearch", "wow" ],
-  "lists": [
-    {
-      "name": "prog_list",
-      "description": "programming list"
-    },
-    {
-      "name": "cool_list",
-      "description": "cool stuff list"
-    }
-  ]
-}
-emit snippet
-body part: 
-{
-  "message": "no arrays in this document...",
-  "tags":  "elasticsearch",
-  "lists": {
-    "name": "prog_list",
-    "description": "programming list"
-  }
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "match": {
-      "tags": "elasticsearch"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "properties": {
-        "ip_addr": {
-          "type": "ip"
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "ip_addr": "192.168.1.1"
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "term": {
-      "ip_addr": "192.168.0.0/16"
-    }
-  }
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "dynamic_templates": [
-        {
-          "integers": {
-            "match_mapping_type": "long",
-            "mapping": {
-              "type": "integer"
-            }
-          }
-        },
-        {
-          "strings": {
-            "match_mapping_type": "string",
-            "mapping": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "type":  "keyword",
-                  "ignore_above": 256
-                }
-              }
-            }
-          }
-        }
-      ]
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "my_integer": 5,
-  "my_string": "Some string"
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "dynamic_templates": [
-        {
-          "longs_as_strings": {
-            "match_mapping_type": "string",
-            "match":   "long_*",
-            "unmatch": "*_text",
-            "mapping": {
-              "type": "long"
-            }
-          }
-        }
-      ]
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "long_num": "5",
-  "long_text": "foo"
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "dynamic_templates": [
-        {
-          "full_name": {
-            "path_match":   "name.*",
-            "path_unmatch": "*.middle",
-            "mapping": {
-              "type":       "text",
-              "copy_to":    "full_name"
-            }
-          }
-        }
-      ]
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "name": {
-    "first":  "Alice",
-    "middle": "Mary",
-    "last":   "White"
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "dynamic_templates": [
-        {
-          "named_analyzers": {
-            "match_mapping_type": "string",
-            "match": "*",
-            "mapping": {
-              "type": "text",
-              "analyzer": "{name}"
-            }
-          }
-        },
-        {
-          "no_doc_values": {
-            "match_mapping_type":"*",
-            "mapping": {
-              "type": "{dynamic_type}",
-              "doc_values": false
-            }
-          }
-        }
-      ]
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "english": "Some English text",
-  "count":   5
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "order": 0,
-  "template": "*",
-  "mappings": {
-    "_default_": {
-      "_all": {
-        "enabled": false
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "create_date": "2015/09/02"
-}
-emit snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "date_detection": false
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "create": "2015/09/02"
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "dynamic_date_formats": ["MM/dd/yyyy"]
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "create_date": "09/25/2015"
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "numeric_detection": true
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "my_float":   "1.0",
-  "my_integer": "1"
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "_default_": {
-      "_all": {
-        "enabled": false
-      }
-    },
-    "user": {},
-    "blogpost": {
-      "_all": {
-        "enabled": true
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "template":   "logs-*",
-  "settings": { "number_of_shards": 1 },
-  "mappings": {
-    "_default_": {
-      "_all": {
-        "enabled": false
-      },
-      "dynamic_templates": [
-        {
-          "strings": {
-            "match_mapping_type": "string",
-            "mapping": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "type":  "keyword",
-                  "ignore_above": 256
-                }
-              }
-            }
-          }
-        }
-      ]
-    }
-  }
-}
-emit snippet
-body part: 
-{ "message": "error:16" }
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "_timestamp": {
-        "enabled": true
-      }
-    }
-  }
-}
-emit snippet
-query part: timestamp=2015-01-01
-body part: 
-{ "text": "Timestamp as a formatted date" }
-emit snippet
-query part: timestamp=1420070400000
-body part: 
-{ "text": "Timestamp as milliseconds since the epoch" }
-emit snippet
-body part: 
-{ "text": "Autogenerated timestamp set to now()" }
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "range": {
-      "_timestamp": {
-        "gte": "2015-01-01"
-      }
-    }
-  },
-  "aggs": {
-    "Timestamps": {
-      "terms": {
-        "field": "_timestamp",
-        "size": 10
-      }
-    }
-  },
-  "sort": [
-    {
-      "_timestamp": {
-        "order": "desc"
-      }
-    }
-  ],
-  "script_fields": {
-    "Timestamp": {
-      "script": "doc['_timestamp']"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "first_name":    "John",
-  "last_name":     "Smith",
-  "date_of_birth": "1970-10-24"
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "match": {
-      "_all": "john smith 1970"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "query_string": {
-      "query": "john smith 1970"
-    }
-  }
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "type_1": {
-      "properties": {}
-    },
-    "type_2": {
-      "_all": {
-        "enabled": false
-      },
-      "properties": {}
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "_all": {
-        "enabled": false
-      },
-      "properties": {
-        "content": {
-          "type": "text"
-        }
-      }
-    }
-  },
-  "settings": {
-    "index.query.default_field": "content"
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "mytype": {
-      "properties": {
-        "title": {
-          "type": "text",
-          "boost": 2
-        },
-        "content": {
-          "type": "text"
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "mytype": {
-      "properties": {
-        "first_name": {
-          "type":    "text",
-          "copy_to": "full_name"
-        },
-        "last_name": {
-          "type":    "text",
-          "copy_to": "full_name"
-        },
-        "full_name": {
-          "type":    "text"
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "first_name": "John",
-  "last_name": "Smith"
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "match": {
-      "full_name": "John Smith"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "mytype": {
-      "_all": {
-        "store": true
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "first_name": "John",
-  "last_name": "Smith"
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "match": {
-      "_all": "John Smith"
-    }
-  },
-  "highlight": {
-    "fields": {
-      "_all": {}
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "mytype": {
-      "_all": {}
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "first_name": "John",
-  "last_name": "Smith"
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "match": {
-      "_all": "John Smith"
-    }
-  },
-  "highlight": {
-    "fields": {
-      "*_name": {
-        "require_field_match": false
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "title": "This is a document"
-}
-emit snippet
-body part: 
-{
-  "title": "This is another document",
-  "body": "This document has a body"
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "terms": {
-      "_field_names": [ "title" ]
-    }
-  },
-  "script_fields": {
-    "Field names": {
-      "script": "doc['_field_names']"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "text": "Document in index 1"
-}
-emit snippet
-body part: 
-{
-  "text": "Document in index 2"
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "terms": {
-      "_index": ["index_1", "index_2"]
-    }
-  },
-  "aggs": {
-    "indices": {
-      "terms": {
-        "field": "_index",
-        "size": 10
-      }
-    }
-  },
-  "sort": [
-    {
-      "_index": {
-        "order": "asc"
-      }
-    }
-  ],
-  "script_fields": {
-    "index_name": {
-      "script": "doc['_index']"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "tweet": {
-      "_source": {
-        "enabled": false
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "event": {
-      "_source": {
-        "includes": [
-          "*.count",
-          "meta.*"
-        ],
-        "excludes": [
-          "meta.description",
-          "meta.other.*"
-        ]
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "requests": {
-    "count": 10,
-    "foo": "bar"
-  },
-  "meta": {
-    "name": "Some metric",
-    "description": "Some metric description",
-    "other": {
-      "foo": "one",
-      "baz": "two"
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "match": {
-      "meta.other.foo": "one"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "text": "Document with type 1"
-}
-emit snippet
-body part: 
-{
-  "text": "Document with type 2"
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "terms": {
-      "_type": [ "type_1", "type_2" ]
-    }
-  },
-  "aggs": {
-    "types": {
-      "terms": {
-        "field": "_type",
-        "size": 10
-      }
-    }
-  },
-  "sort": [
-    {
-      "_type": {
-        "order": "desc"
-      }
-    }
-  ],
-  "script_fields": {
-    "type": {
-      "script": "doc['_type']"
-    }
-  }
-}
-handle snippet
-null
-emit snippet
-query part: routing=user1
-body part: 
-{
-  "title": "This is a document"
-}
-emit snippet
-query part: routing=user1
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "terms": {
-      "_routing": [ "user1" ]
-    }
-  },
-  "script_fields": {
-    "Routing value": {
-      "script": "doc['_routing']"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-query part: routing=user1,user2
-body part: 
-{
-  "query": {
-    "match": {
-      "title": "document"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "_routing": {
-        "required": true
-      }
-    }
-  }
-}
-emit snippet
-catch part: request
-body part: 
-{
-  "text": "No routing value provided"
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "_ttl": {
-        "enabled": true
-      }
-    }
-  }
-}
-emit snippet
-query part: ttl=10m
-body part: 
-{
-  "text": "Will expire in 10 minutes"
-}
-emit snippet
-body part: 
-{
-  "text": "Will not expire"
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_type": {
-      "_ttl": {
-        "enabled": true,
-        "default": "5m"
-      }
-    }
-  }
-}
-emit snippet
-query part: ttl=10m
-body part: 
-{
-  "text": "Will expire in 10 minutes"
-}
-emit snippet
-body part: 
-{
-  "text": "Will expire in 5 minutes"
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_parent": {},
-    "my_child": {
-      "_parent": {
-        "type": "my_parent"
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "text": "This is a parent document"
-}
-emit snippet
-query part: parent=1
-body part: 
-{
-  "text": "This is a child document"
-}
-emit snippet
-query part: parent=1
-body part: 
-{
-  "text": "This is another child document"
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "has_child": {
-      "type": "my_child",
-      "query": {
-        "match": {
-          "text": "child document"
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "query": {
-    "terms": {
-      "_parent": [ "1" ]
-    }
-  },
-  "aggs": {
-    "parents": {
-      "terms": {
-        "field": "_parent",
-        "size": 10
-      }
-    }
-  },
-  "script_fields": {
-    "parent": {
-      "script": "doc['_parent']"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "my_parent": {},
-    "my_child": {
-      "_parent": {
-        "type": "my_parent",
-        "eager_global_ordinals": true
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-query part: human&fields=_parent
-emit snippet
-query part: human&fields=_parent
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "text": "Document with ID 1"
-}
-emit snippet
-body part: 
-{
-  "text": "Document with ID 2"
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "terms": {
-      "_id": [ "1", "2" ]
-    }
-  },
-  "script_fields": {
-    "UID": {
-      "script": "doc['_id']"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "text": "Document with ID 1"
-}
-emit snippet
-body part: 
-{
-  "text": "Document with ID 2"
-}
-emit snippet
-body part: 
-{
-  "query": {
-    "terms": {
-      "_uid": [ "my_type#1", "my_type#2" ]
-    }
-  },
-  "aggs": {
-    "UIDs": {
-      "terms": {
-        "field": "_uid",
-        "size": 10
-      }
-    }
-  },
-  "sort": [
-    {
-      "_uid": {
-        "order": "desc"
-      }
-    }
-  ],
-  "script_fields": {
-    "UID": {
-      "script": "doc['_uid']"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "user": {
-      "_meta": {
-        "class": "MyApp::User",
-        "version": {
-          "min": "1.0",
-          "max": "1.3"
-        }
-      }
-    }
-  }
-}
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-emit snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "index.priority": 10
-  }
-}
-emit snippet
-body part: 
-{
-  "settings": {
-    "index.priority": 5
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "index.priority": 1
-}
-handle snippet
-test snippet
-emit snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "index.unassigned.node_left.delayed_timeout": "5m"
-  }
-}
-handle snippet
-test snippet
-emit snippet
-handle snippet
-test snippet
-emit snippet
-emit snippet
-body part: 
-{
-  "settings": {
-    "index.unassigned.node_left.delayed_timeout": "0"
-  }
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-emit snippet
-body part: 
-{
-  "index.routing.allocation.include.size": "big,medium"
-}
-handle snippet
-test snippet
-emit snippet
-emit snippet
-body part: 
-{
-  "index.routing.allocation.exclude.size": "small"
-}
-handle snippet
-test snippet
-emit snippet
-emit snippet
-body part: 
-{
-  "index.routing.allocation.include.size": "big",
-  "index.routing.allocation.include.rack": "rack1"
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "index.routing.allocation.include._ip": "192.168.2.*"
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "tokenizer" : "standard",
-  "token_filter" : ["snowball"],
-  "text" : "detailed output",
-  "explain" : true,
-  "attributes" : ["keyword"]
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "template": "te*",
-  "settings": {
-    "number_of_shards": 1
-  },
-  "mappings": {
-    "type1": {
-      "_source": {
-        "enabled": false
-      },
-      "properties": {
-        "host_name": {
-          "type": "keyword"
-        },
-        "created_at": {
-          "type": "date",
-          "format": "EEE MMM dd HH:mm:ss Z YYYY"
-        }
-      }
-    }
-  }
-}
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "tweet": {
-      "properties": {
-        "message": {
-          "type": "text"
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "properties": {
-    "name": {
-      "type": "text"
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "properties": {
-    "user_name": {
-      "type": "text"
-    }
-  }
-}
-handle snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "user": {
-      "properties": {
-        "name": {
-          "properties": {
-            "first": {
-              "type": "text"
-            }
-          }
-        },
-        "user_id": {
-          "type": "keyword"
-        }
-      }
-    }
-  }
-}
-emit snippet
-body part: 
-{
-  "properties": {
-    "name": {
-      "properties": {
-        "last": {
-          "type": "text"
-        }
-      }
-    },
-    "user_id": {
-      "type": "keyword",
-      "ignore_above": 100
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "type_one": {
-      "properties": {
-        "text": {
-          "type": "text",
-          "analyzer": "standard"
-        }
-      }
-    },
-    "type_two": {
-      "properties": {
-        "text": {
-          "type": "text",
-          "analyzer": "standard"
-        }
-      }
-    }
-  }
-}
-emit snippet
-catch part: request
-body part: 
-{
-  "properties": {
-    "text": {
-      "type": "text",
-      "analyzer": "standard",
-      "search_analyzer": "whitespace"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-query part: update_all_types
-body part: 
-{
-  "properties": {
-    "text": {
-      "type": "text",
-      "analyzer": "standard",
-      "search_analyzer": "whitespace"
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-emit snippet
-body part: 
-{
-    "actions" : [
-        { "add" : { "index" : "test1", "alias" : "alias1" } }
-    ]
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "actions" : [
-        { "remove" : { "index" : "test1", "alias" : "alias1" } }
-    ]
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "actions" : [
-        { "remove" : { "index" : "test1", "alias" : "alias1" } },
-        { "add" : { "index" : "test1", "alias" : "alias2" } }
-    ]
-}
-handle snippet
-test snippet
-emit snippet
-emit snippet
-emit snippet
-body part: 
-{
-    "actions" : [
-        { "add" : { "index" : "test1", "alias" : "alias1" } },
-        { "add" : { "index" : "test2", "alias" : "alias1" } }
-    ]
-}
-handle snippet
-test snippet
-emit snippet
-emit snippet
-emit snippet
-body part: 
-{
-    "actions" : [
-        { "add" : { "indices" : ["test1", "test2"], "alias" : "alias1" } }
-    ]
-}
-handle snippet
-test snippet
-emit snippet
-emit snippet
-emit snippet
-body part: 
-{
-    "actions" : [
-        { "add" : { "index" : "test*", "alias" : "all_test_indices" } }
-    ]
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-  "mappings": {
-    "type1": {
-      "properties": {
-        "user" : {
-          "type": "keyword"
-        }
-      }
-    }
-  }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "actions" : [
-        {
-            "add" : {
-                 "index" : "test1",
-                 "alias" : "alias2",
-                 "filter" : { "term" : { "user" : "kimchy" } }
-            }
-        }
-    ]
-}
-handle snippet
-test snippet
-emit snippet
-emit snippet
-body part: 
-{
-    "actions" : [
-        {
-            "add" : {
-                 "index" : "test",
-                 "alias" : "alias1",
-                 "routing" : "1"
-            }
-        }
-    ]
-}
-handle snippet
-test snippet
-emit snippet
-emit snippet
-body part: 
-{
-    "actions" : [
-        {
-            "add" : {
-                 "index" : "test",
-                 "alias" : "alias2",
-                 "search_routing" : "1,2",
-                 "index_routing" : "2"
-            }
-        }
-    ]
-}
-handle snippet
-test snippet
-emit snippet
-query part: q=user:kimchy&routing=2,3
-handle snippet
-test snippet
-emit snippet
-emit snippet
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "mappings" : {
-        "user" : {
-            "properties" : {
-                "user_id" : {"type" : "integer"}
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "routing" : "12",
-    "filter" : {
-        "term" : {
-            "user_id" : 12
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-body part: 
-{
-    "mappings" : {
-        "type" : {
-            "properties" : {
-                "year" : {"type" : "integer"}
-            }
-        }
-    },
-    "aliases" : {
-        "current_day" : {},
-        "2016" : {
-            "filter" : {
-                "term" : {"year" : 2016 }
-            }
-        }
-    }
-}
-handle snippet
-test snippet
-emit snippet
-handle snippet
-test snippet
-emit snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-emit snippet
-emit snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-emit snippet
-handle snippet
-test snippet
-emit snippet
-emit snippet
-emit snippet
-emit snippet
-handle snippet
-test snippet
-emit snippet
-emit snippet
-query part: level=shards
-handle snippet
-handle snippet
-test snippet
-
-  - do:
-      bulk:
-        index: twitter
-        type: tweet
-        refresh: true
-        body: |
-          {"index":{"_id": "0"}}
-          {"user": "kimchy", "message": "trying out Elasticsearch", "date": "2009-11-15T14:12:12", "likes": 0}
-          {"index":{"_id": "1"}}
-          {"user": "test", "message": "some message with the number 1", "date": "2009-11-15T14:12:12", "likes": 1}
-          {"index":{"_id": "2"}}
-          {"user": "test", "message": "some message with the number 2", "date": "2009-11-15T14:12:12", "likes": 2}
-          {"index":{"_id": "3"}}
-          {"user": "test", "message": "some message with the number 3", "date": "2009-11-15T14:12:12", "likes": 3}
-          {"index":{"_id": "4"}}
-          {"user": "test", "message": "some message with the number 4", "date": "2009-11-15T14:12:12", "likes": 4}
-emit snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-test snippet
-emit snippet
-emit snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-handle snippet
-:core:compileJava UP-TO-DATE
-:core:generateModulesList UP-TO-DATE
-:core:generatePluginsList UP-TO-DATE
-:core:processResources UP-TO-DATE
-:core:classes UP-TO-DATE
-:core:jar UP-TO-DATE
-:test:framework:compileJava UP-TO-DATE
-:test:framework:processResources UP-TO-DATE
-:test:framework:classes UP-TO-DATE
-:test:framework:jar UP-TO-DATE
-:docs:compileTestJava
-:docs:testClasses
-:docs:integTest#prepareCluster.cleanShared
-:docs:integTest#clean
-:docs:integTest#checkPrevious SKIPPED
-:docs:integTest#stopPrevious SKIPPED
-:modules:aggs-matrix-stats:compileJava UP-TO-DATE
-:modules:aggs-matrix-stats:processResources UP-TO-DATE
-:modules:aggs-matrix-stats:classes UP-TO-DATE
-:modules:aggs-matrix-stats:jar UP-TO-DATE
-:modules:aggs-matrix-stats:copyPluginPropertiesTemplate
-:modules:aggs-matrix-stats:pluginProperties UP-TO-DATE
-:modules:aggs-matrix-stats:bundlePlugin UP-TO-DATE
-:modules:ingest-grok:compileJava UP-TO-DATE
-:modules:ingest-grok:processResources UP-TO-DATE
-:modules:ingest-grok:classes UP-TO-DATE
-:modules:ingest-grok:jar UP-TO-DATE
-:modules:ingest-grok:copyPluginPropertiesTemplate
-:modules:ingest-grok:pluginProperties UP-TO-DATE
-:modules:ingest-grok:bundlePlugin UP-TO-DATE
-:modules:lang-expression:compileJava UP-TO-DATE
-:modules:lang-expression:processResources UP-TO-DATE
-:modules:lang-expression:classes UP-TO-DATE
-:modules:lang-expression:jar UP-TO-DATE
-:modules:lang-expression:copyPluginPropertiesTemplate
-:modules:lang-expression:pluginProperties UP-TO-DATE
-:modules:lang-expression:bundlePlugin UP-TO-DATE
-:modules:lang-groovy:compileJava UP-TO-DATE
-:modules:lang-groovy:processResources UP-TO-DATE
-:modules:lang-groovy:classes UP-TO-DATE
-:modules:lang-groovy:jar UP-TO-DATE
-:modules:lang-groovy:copyPluginPropertiesTemplate
-:modules:lang-groovy:pluginProperties UP-TO-DATE
-:modules:lang-groovy:bundlePlugin UP-TO-DATE
-:modules:lang-mustache:compileJava UP-TO-DATE
-:modules:lang-mustache:processResources UP-TO-DATE
-:modules:lang-mustache:classes UP-TO-DATE
-:modules:lang-mustache:jar UP-TO-DATE
-:modules:lang-mustache:copyPluginPropertiesTemplate
-:modules:lang-mustache:pluginProperties UP-TO-DATE
-:modules:lang-mustache:bundlePlugin UP-TO-DATE
-:modules:lang-painless:compileJava UP-TO-DATE
-:modules:lang-painless:processResources UP-TO-DATE
-:modules:lang-painless:classes UP-TO-DATE
-:modules:lang-painless:jar UP-TO-DATE
-:modules:lang-painless:copyPluginPropertiesTemplate
-:modules:lang-painless:pluginProperties UP-TO-DATE
-:modules:lang-painless:bundlePlugin UP-TO-DATE
-:modules:percolator:compileJava UP-TO-DATE
-:modules:percolator:processResources UP-TO-DATE
-:modules:percolator:classes UP-TO-DATE
-:modules:percolator:jar UP-TO-DATE
-:modules:percolator:copyPluginPropertiesTemplate
-:modules:percolator:pluginProperties UP-TO-DATE
-:modules:percolator:bundlePlugin UP-TO-DATE
-:modules:reindex:compileJava UP-TO-DATE
-:modules:reindex:processResources UP-TO-DATE
-:modules:reindex:classes UP-TO-DATE
-:modules:reindex:jar UP-TO-DATE
-:modules:reindex:copyPluginPropertiesTemplate
-:modules:reindex:pluginProperties UP-TO-DATE
-:modules:reindex:bundlePlugin UP-TO-DATE
-:distribution:buildModules UP-TO-DATE
-:distribution:zip:buildZip UP-TO-DATE
-:docs:integTest#extract
-:docs:integTest#configure
-:docs:integTest#start
-:docs:integTest#wait
-:docs:integTest
-   [junit4] <JUnit4> says ahoj! Master seed: E40438BBAF23DBE2
-==> Test Info: seed=E40438BBAF23DBE2; jvm=1; suite=1
-Suite: org.elasticsearch.smoketest.SmokeTestDocsIT
-==> Test Summary: 1 suite (1 ignored), 0 tests
-   [junit4] JVM J0:     0.30 ..     1.64 =     1.35s
-   [junit4] Execution time total: 1.66 sec.
-   [junit4] Tests summary: 1 suite (1 ignored), 0 tests
-:docs:integTest FAILED
-:docs:integTest#stop
-
-BUILD FAILED
-
-Total time: 22.834 secs
-
-BUILD SUCCESSFUL
-Total time: 1 second

From b15279b5eff2515b479e7350a5583d87f31ce9bf Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 3 Jun 2016 23:59:26 +0200
Subject: [PATCH 0643/1311] Allow to pass socket facttry registry to
 createDefaultHttpClient method

---
 .../elasticsearch/client/sniff/Sniffer.java   |  3 +-
 .../org/elasticsearch/client/RestClient.java  | 17 +++++++---
 .../test/rest/client/RestTestClient.java      | 32 +++++++++----------
 3 files changed, 30 insertions(+), 22 deletions(-)

diff --git a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
index 855b15e1e4d..fa44ae7d810 100644
--- a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
+++ b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
@@ -22,6 +22,7 @@ package org.elasticsearch.client.sniff;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.http.HttpHost;
+import org.apache.http.config.Registry;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.elasticsearch.client.RestClient;
 
@@ -203,7 +204,7 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
         /**
          * Sets the http client. Mandatory argument. Best practice is to use the same client used
          * within {@link org.elasticsearch.client.RestClient} which can be created manually or
-         * through {@link RestClient.Builder#createDefaultHttpClient()}.
+         * through {@link RestClient.Builder#createDefaultHttpClient(Registry)}.
          * @see CloseableHttpClient
          */
         public Builder setRestClient(RestClient restClient) {
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 856a50c1834..3f7e5a18aec 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -37,6 +37,8 @@ import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.client.methods.HttpTrace;
 import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.config.Registry;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
 import org.apache.http.entity.ContentType;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
@@ -374,7 +376,7 @@ public final class RestClient implements Closeable {
 
         /**
          * Sets the http client. A new default one will be created if not
-         * specified, by calling {@link #createDefaultHttpClient()}.
+         * specified, by calling {@link #createDefaultHttpClient(Registry)})}.
          *
          * @see CloseableHttpClient
          */
@@ -427,7 +429,7 @@ public final class RestClient implements Closeable {
          */
         public RestClient build() {
             if (httpClient == null) {
-                httpClient = createDefaultHttpClient();
+                httpClient = createDefaultHttpClient(null);
             }
             if (hosts == null || hosts.length == 0) {
                 throw new IllegalArgumentException("no hosts provided");
@@ -436,12 +438,17 @@ public final class RestClient implements Closeable {
         }
 
         /**
-         * Creates an http client with default settings
+         * Creates a {@link CloseableHttpClient} with default settings. Used when the http client instance is not provided.
          *
          * @see CloseableHttpClient
          */
-        public static CloseableHttpClient createDefaultHttpClient() {
-            PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
+        public static CloseableHttpClient createDefaultHttpClient(Registry<ConnectionSocketFactory> socketFactoryRegistry) {
+            PoolingHttpClientConnectionManager connectionManager;
+            if (socketFactoryRegistry == null) {
+                connectionManager = new PoolingHttpClientConnectionManager();
+            } else {
+                connectionManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+            }
             //default settings may be too constraining
             connectionManager.setDefaultMaxPerRoute(10);
             connectionManager.setMaxTotal(30);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java
index 5baedf8d060..94f85422edf 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java
@@ -29,8 +29,6 @@ import org.apache.http.conn.socket.PlainConnectionSocketFactory;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.apache.http.message.BasicHeader;
 import org.apache.http.ssl.SSLContexts;
 import org.apache.lucene.util.IOUtils;
@@ -71,7 +69,8 @@ import java.util.Objects;
 import java.util.Set;
 
 /**
- * REST client used to test the elasticsearch REST layer
+ * REST client used to test the elasticsearch REST layer.
+ * Wraps a {@link RestClient} instance used to send the REST requests.
  * Holds the {@link RestSpec} used to translate api calls into REST calls
  */
 public class RestTestClient implements Closeable {
@@ -136,7 +135,7 @@ public class RestTestClient implements Closeable {
 
         if ("raw".equals(apiName)) {
             // Raw requests are bit simpler....
-            HashMap<String, String> queryStringParams = new HashMap<>(params);
+            Map<String, String> queryStringParams = new HashMap<>(params);
             String method = Objects.requireNonNull(queryStringParams.remove("method"), "Method must be set to use raw request");
             String path = "/"+ Objects.requireNonNull(queryStringParams.remove("path"), "Path must be set to use raw request");
             HttpEntity entity = null;
@@ -267,7 +266,7 @@ public class RestTestClient implements Closeable {
         return restApi;
     }
 
-    protected RestClient createRestClient(URL[] urls, Settings settings) throws IOException {
+    private static RestClient createRestClient(URL[] urls, Settings settings) throws IOException {
         SSLConnectionSocketFactory sslsf;
         String keystorePath = settings.get(TRUSTSTORE_PATH);
         if (keystorePath != null) {
@@ -297,16 +296,7 @@ public class RestTestClient implements Closeable {
                 .register("http", PlainConnectionSocketFactory.getSocketFactory())
                 .register("https", sslsf)
                 .build();
-
-        List<Header> headers = new ArrayList<>();
-        try (ThreadContext threadContext = new ThreadContext(settings)) {
-            for (Map.Entry<String, String> entry : threadContext.getHeaders().entrySet()) {
-                headers.add(new BasicHeader(entry.getKey(), entry.getValue()));
-            }
-        }
-
-        CloseableHttpClient httpClient = HttpClientBuilder.create().setDefaultHeaders(headers)
-                .setConnectionManager(new PoolingHttpClientConnectionManager(socketFactoryRegistry)).build();
+        CloseableHttpClient httpClient = RestClient.Builder.createDefaultHttpClient(socketFactoryRegistry);
 
         String protocol = settings.get(PROTOCOL, "http");
         HttpHost[] hosts = new HttpHost[urls.length];
@@ -314,7 +304,17 @@ public class RestTestClient implements Closeable {
             URL url = urls[i];
             hosts[i] = new HttpHost(url.getHost(), url.getPort(), protocol);
         }
-        return RestClient.builder().setHttpClient(httpClient).setHosts(hosts).build();
+
+        RestClient.Builder builder = RestClient.builder().setHttpClient(httpClient).setHosts(hosts);
+        try (ThreadContext threadContext = new ThreadContext(settings)) {
+            Header[] defaultHeaders = new Header[threadContext.getHeaders().size()];
+            int i = 0;
+            for (Map.Entry<String, String> entry : threadContext.getHeaders().entrySet()) {
+                defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue());
+            }
+            builder.setDefaultHeaders(defaultHeaders);
+        }
+        return builder.build();
     }
 
     /**

From f2318ed5ae1a18da9491c6057ac40179a32f6856 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Sat, 4 Jun 2016 00:45:27 +0200
Subject: [PATCH 0644/1311] Build: add missing licenses, SHAs and enable
 dependency licenses check

---
 client-sniffer/build.gradle                   |   3 -
 .../licenses/client-5.0.0-SNAPSHOT.jar.sha1   |   1 +
 client-sniffer/licenses/client-LICENSE.txt    | 202 +++++++
 client-sniffer/licenses/client-NOTICE.txt     |   5 +
 .../licenses/commons-codec-1.10.jar.sha1      |   1 +
 .../licenses/commons-codec-LICENSE.txt        | 202 +++++++
 .../licenses/commons-codec-NOTICE.txt         |  17 +
 .../licenses/commons-logging-1.1.3.jar.sha1   |   1 +
 .../licenses/commons-logging-LICENSE.txt      | 202 +++++++
 .../licenses/commons-logging-NOTICE.txt       |   6 +
 .../licenses/httpclient-4.5.2.jar.sha1        |   1 +
 .../licenses/httpclient-LICENSE.txt           | 558 ++++++++++++++++++
 client-sniffer/licenses/httpclient-NOTICE.txt |   6 +
 .../licenses/httpcore-4.4.4.jar.sha1          |   1 +
 client-sniffer/licenses/httpcore-LICENSE.txt  | 558 ++++++++++++++++++
 client-sniffer/licenses/httpcore-NOTICE.txt   |   6 +
 .../licenses/jackson-core-2.7.1.jar.sha1      |   1 +
 client-sniffer/licenses/jackson-core-LICENSE  |   8 +
 client-sniffer/licenses/jackson-core-NOTICE   |  20 +
 client/build.gradle                           |   3 -
 client/licenses/commons-codec-1.10.jar.sha1   |   1 +
 client/licenses/commons-codec-LICENSE.txt     | 202 +++++++
 client/licenses/commons-codec-NOTICE.txt      |  17 +
 .../licenses/commons-logging-1.1.3.jar.sha1   |   1 +
 client/licenses/commons-logging-LICENSE.txt   | 202 +++++++
 client/licenses/commons-logging-NOTICE.txt    |   6 +
 client/licenses/httpclient-4.5.2.jar.sha1     |   1 +
 client/licenses/httpclient-LICENSE.txt        | 558 ++++++++++++++++++
 client/licenses/httpclient-NOTICE.txt         |   6 +
 client/licenses/httpcore-4.4.4.jar.sha1       |   1 +
 client/licenses/httpcore-LICENSE.txt          | 558 ++++++++++++++++++
 client/licenses/httpcore-NOTICE.txt           |   6 +
 32 files changed, 3355 insertions(+), 6 deletions(-)
 create mode 100644 client-sniffer/licenses/client-5.0.0-SNAPSHOT.jar.sha1
 create mode 100644 client-sniffer/licenses/client-LICENSE.txt
 create mode 100644 client-sniffer/licenses/client-NOTICE.txt
 create mode 100644 client-sniffer/licenses/commons-codec-1.10.jar.sha1
 create mode 100644 client-sniffer/licenses/commons-codec-LICENSE.txt
 create mode 100644 client-sniffer/licenses/commons-codec-NOTICE.txt
 create mode 100644 client-sniffer/licenses/commons-logging-1.1.3.jar.sha1
 create mode 100644 client-sniffer/licenses/commons-logging-LICENSE.txt
 create mode 100644 client-sniffer/licenses/commons-logging-NOTICE.txt
 create mode 100644 client-sniffer/licenses/httpclient-4.5.2.jar.sha1
 create mode 100644 client-sniffer/licenses/httpclient-LICENSE.txt
 create mode 100644 client-sniffer/licenses/httpclient-NOTICE.txt
 create mode 100644 client-sniffer/licenses/httpcore-4.4.4.jar.sha1
 create mode 100644 client-sniffer/licenses/httpcore-LICENSE.txt
 create mode 100644 client-sniffer/licenses/httpcore-NOTICE.txt
 create mode 100644 client-sniffer/licenses/jackson-core-2.7.1.jar.sha1
 create mode 100644 client-sniffer/licenses/jackson-core-LICENSE
 create mode 100644 client-sniffer/licenses/jackson-core-NOTICE
 create mode 100644 client/licenses/commons-codec-1.10.jar.sha1
 create mode 100644 client/licenses/commons-codec-LICENSE.txt
 create mode 100644 client/licenses/commons-codec-NOTICE.txt
 create mode 100644 client/licenses/commons-logging-1.1.3.jar.sha1
 create mode 100644 client/licenses/commons-logging-LICENSE.txt
 create mode 100644 client/licenses/commons-logging-NOTICE.txt
 create mode 100644 client/licenses/httpclient-4.5.2.jar.sha1
 create mode 100644 client/licenses/httpclient-LICENSE.txt
 create mode 100644 client/licenses/httpclient-NOTICE.txt
 create mode 100644 client/licenses/httpcore-4.4.4.jar.sha1
 create mode 100644 client/licenses/httpcore-LICENSE.txt
 create mode 100644 client/licenses/httpcore-NOTICE.txt

diff --git a/client-sniffer/build.gradle b/client-sniffer/build.gradle
index 6542453fa4f..b0f8e1e22e0 100644
--- a/client-sniffer/build.gradle
+++ b/client-sniffer/build.gradle
@@ -60,9 +60,6 @@ forbiddenApisTest.enabled=false
   //signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
 //}
 
-//TODO add licenses for dependencies and take care of distribution
-//dependency license are currently checked in distribution
-dependencyLicenses.enabled=false
 //JarHell is part of es core, which we don't want to pull in
 jarHell.enabled=false
 //NamingConventionCheck is part of test-framework, which we don't want to pull in as it depends on es core
diff --git a/client-sniffer/licenses/client-5.0.0-SNAPSHOT.jar.sha1 b/client-sniffer/licenses/client-5.0.0-SNAPSHOT.jar.sha1
new file mode 100644
index 00000000000..de30dc58fe6
--- /dev/null
+++ b/client-sniffer/licenses/client-5.0.0-SNAPSHOT.jar.sha1
@@ -0,0 +1 @@
+4ba4746aa38f81ec7e8341da8c86784bf5384046
\ No newline at end of file
diff --git a/client-sniffer/licenses/client-LICENSE.txt b/client-sniffer/licenses/client-LICENSE.txt
new file mode 100644
index 00000000000..d6456956733
--- /dev/null
+++ b/client-sniffer/licenses/client-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/client-sniffer/licenses/client-NOTICE.txt b/client-sniffer/licenses/client-NOTICE.txt
new file mode 100644
index 00000000000..c99b9581931
--- /dev/null
+++ b/client-sniffer/licenses/client-NOTICE.txt
@@ -0,0 +1,5 @@
+Elasticsearch
+Copyright 2009-2016 Elasticsearch
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).
diff --git a/client-sniffer/licenses/commons-codec-1.10.jar.sha1 b/client-sniffer/licenses/commons-codec-1.10.jar.sha1
new file mode 100644
index 00000000000..3fe8682a1b0
--- /dev/null
+++ b/client-sniffer/licenses/commons-codec-1.10.jar.sha1
@@ -0,0 +1 @@
+4b95f4897fa13f2cd904aee711aeafc0c5295cd8
\ No newline at end of file
diff --git a/client-sniffer/licenses/commons-codec-LICENSE.txt b/client-sniffer/licenses/commons-codec-LICENSE.txt
new file mode 100644
index 00000000000..d6456956733
--- /dev/null
+++ b/client-sniffer/licenses/commons-codec-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/client-sniffer/licenses/commons-codec-NOTICE.txt b/client-sniffer/licenses/commons-codec-NOTICE.txt
new file mode 100644
index 00000000000..1da9af50f60
--- /dev/null
+++ b/client-sniffer/licenses/commons-codec-NOTICE.txt
@@ -0,0 +1,17 @@
+Apache Commons Codec
+Copyright 2002-2014 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java
+contains test data from http://aspell.net/test/orig/batch0.tab.
+Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org)
+
+===============================================================================
+
+The content of package org.apache.commons.codec.language.bm has been translated
+from the original php source code available at http://stevemorse.org/phoneticinfo.htm
+with permission from the original authors.
+Original source copyright:
+Copyright (c) 2008 Alexander Beider & Stephen P. Morse.
diff --git a/client-sniffer/licenses/commons-logging-1.1.3.jar.sha1 b/client-sniffer/licenses/commons-logging-1.1.3.jar.sha1
new file mode 100644
index 00000000000..5b8f029e582
--- /dev/null
+++ b/client-sniffer/licenses/commons-logging-1.1.3.jar.sha1
@@ -0,0 +1 @@
+f6f66e966c70a83ffbdb6f17a0919eaf7c8aca7f
\ No newline at end of file
diff --git a/client-sniffer/licenses/commons-logging-LICENSE.txt b/client-sniffer/licenses/commons-logging-LICENSE.txt
new file mode 100644
index 00000000000..d6456956733
--- /dev/null
+++ b/client-sniffer/licenses/commons-logging-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/client-sniffer/licenses/commons-logging-NOTICE.txt b/client-sniffer/licenses/commons-logging-NOTICE.txt
new file mode 100644
index 00000000000..556bd03951d
--- /dev/null
+++ b/client-sniffer/licenses/commons-logging-NOTICE.txt
@@ -0,0 +1,6 @@
+Apache Commons Logging
+Copyright 2003-2014 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
diff --git a/client-sniffer/licenses/httpclient-4.5.2.jar.sha1 b/client-sniffer/licenses/httpclient-4.5.2.jar.sha1
new file mode 100644
index 00000000000..6937112a09f
--- /dev/null
+++ b/client-sniffer/licenses/httpclient-4.5.2.jar.sha1
@@ -0,0 +1 @@
+733db77aa8d9b2d68015189df76ab06304406e50
\ No newline at end of file
diff --git a/client-sniffer/licenses/httpclient-LICENSE.txt b/client-sniffer/licenses/httpclient-LICENSE.txt
new file mode 100644
index 00000000000..32f01eda18f
--- /dev/null
+++ b/client-sniffer/licenses/httpclient-LICENSE.txt
@@ -0,0 +1,558 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+=========================================================================
+
+This project includes Public Suffix List copied from
+<https://publicsuffix.org/list/effective_tld_names.dat>
+licensed under the terms of the Mozilla Public License, v. 2.0
+
+Full license text: <http://mozilla.org/MPL/2.0/>
+
+Mozilla Public License Version 2.0
+==================================
+
+1. Definitions
+--------------
+
+1.1. "Contributor"
+    means each individual or legal entity that creates, contributes to
+    the creation of, or owns Covered Software.
+
+1.2. "Contributor Version"
+    means the combination of the Contributions of others (if any) used
+    by a Contributor and that particular Contributor's Contribution.
+
+1.3. "Contribution"
+    means Covered Software of a particular Contributor.
+
+1.4. "Covered Software"
+    means Source Code Form to which the initial Contributor has attached
+    the notice in Exhibit A, the Executable Form of such Source Code
+    Form, and Modifications of such Source Code Form, in each case
+    including portions thereof.
+
+1.5. "Incompatible With Secondary Licenses"
+    means
+
+    (a) that the initial Contributor has attached the notice described
+        in Exhibit B to the Covered Software; or
+
+    (b) that the Covered Software was made available under the terms of
+        version 1.1 or earlier of the License, but not also under the
+        terms of a Secondary License.
+
+1.6. "Executable Form"
+    means any form of the work other than Source Code Form.
+
+1.7. "Larger Work"
+    means a work that combines Covered Software with other material, in
+    a separate file or files, that is not Covered Software.
+
+1.8. "License"
+    means this document.
+
+1.9. "Licensable"
+    means having the right to grant, to the maximum extent possible,
+    whether at the time of the initial grant or subsequently, any and
+    all of the rights conveyed by this License.
+
+1.10. "Modifications"
+    means any of the following:
+
+    (a) any file in Source Code Form that results from an addition to,
+        deletion from, or modification of the contents of Covered
+        Software; or
+
+    (b) any new file in Source Code Form that contains any Covered
+        Software.
+
+1.11. "Patent Claims" of a Contributor
+    means any patent claim(s), including without limitation, method,
+    process, and apparatus claims, in any patent Licensable by such
+    Contributor that would be infringed, but for the grant of the
+    License, by the making, using, selling, offering for sale, having
+    made, import, or transfer of either its Contributions or its
+    Contributor Version.
+
+1.12. "Secondary License"
+    means either the GNU General Public License, Version 2.0, the GNU
+    Lesser General Public License, Version 2.1, the GNU Affero General
+    Public License, Version 3.0, or any later versions of those
+    licenses.
+
+1.13. "Source Code Form"
+    means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your")
+    means an individual or a legal entity exercising rights under this
+    License. For legal entities, "You" includes any entity that
+    controls, is controlled by, or is under common control with You. For
+    purposes of this definition, "control" means (a) the power, direct
+    or indirect, to cause the direction or management of such entity,
+    whether by contract or otherwise, or (b) ownership of more than
+    fifty percent (50%) of the outstanding shares or beneficial
+    ownership of such entity.
+
+2. License Grants and Conditions
+--------------------------------
+
+2.1. Grants
+
+Each Contributor hereby grants You a world-wide, royalty-free,
+non-exclusive license:
+
+(a) under intellectual property rights (other than patent or trademark)
+    Licensable by such Contributor to use, reproduce, make available,
+    modify, display, perform, distribute, and otherwise exploit its
+    Contributions, either on an unmodified basis, with Modifications, or
+    as part of a Larger Work; and
+
+(b) under Patent Claims of such Contributor to make, use, sell, offer
+    for sale, have made, import, and otherwise transfer either its
+    Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+The licenses granted in Section 2.1 with respect to any Contribution
+become effective for each Contribution on the date the Contributor first
+distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+The licenses granted in this Section 2 are the only rights granted under
+this License. No additional rights or licenses will be implied from the
+distribution or licensing of Covered Software under this License.
+Notwithstanding Section 2.1(b) above, no patent license is granted by a
+Contributor:
+
+(a) for any code that a Contributor has removed from Covered Software;
+    or
+
+(b) for infringements caused by: (i) Your and any other third party's
+    modifications of Covered Software, or (ii) the combination of its
+    Contributions with other software (except as part of its Contributor
+    Version); or
+
+(c) under Patent Claims infringed by Covered Software in the absence of
+    its Contributions.
+
+This License does not grant any rights in the trademarks, service marks,
+or logos of any Contributor (except as may be necessary to comply with
+the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+No Contributor makes additional grants as a result of Your choice to
+distribute the Covered Software under a subsequent version of this
+License (see Section 10.2) or under the terms of a Secondary License (if
+permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+Each Contributor represents that the Contributor believes its
+Contributions are its original creation(s) or it has sufficient rights
+to grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+This License is not intended to limit any rights You have under
+applicable copyright doctrines of fair use, fair dealing, or other
+equivalents.
+
+2.7. Conditions
+
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+in Section 2.1.
+
+3. Responsibilities
+-------------------
+
+3.1. Distribution of Source Form
+
+All distribution of Covered Software in Source Code Form, including any
+Modifications that You create or to which You contribute, must be under
+the terms of this License. You must inform recipients that the Source
+Code Form of the Covered Software is governed by the terms of this
+License, and how they can obtain a copy of this License. You may not
+attempt to alter or restrict the recipients' rights in the Source Code
+Form.
+
+3.2. Distribution of Executable Form
+
+If You distribute Covered Software in Executable Form then:
+
+(a) such Covered Software must also be made available in Source Code
+    Form, as described in Section 3.1, and You must inform recipients of
+    the Executable Form how they can obtain a copy of such Source Code
+    Form by reasonable means in a timely manner, at a charge no more
+    than the cost of distribution to the recipient; and
+
+(b) You may distribute such Executable Form under the terms of this
+    License, or sublicense it under different terms, provided that the
+    license for the Executable Form does not attempt to limit or alter
+    the recipients' rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+You may create and distribute a Larger Work under terms of Your choice,
+provided that You also comply with the requirements of this License for
+the Covered Software. If the Larger Work is a combination of Covered
+Software with a work governed by one or more Secondary Licenses, and the
+Covered Software is not Incompatible With Secondary Licenses, this
+License permits You to additionally distribute such Covered Software
+under the terms of such Secondary License(s), so that the recipient of
+the Larger Work may, at their option, further distribute the Covered
+Software under the terms of either this License or such Secondary
+License(s).
+
+3.4. Notices
+
+You may not remove or alter the substance of any license notices
+(including copyright notices, patent notices, disclaimers of warranty,
+or limitations of liability) contained within the Source Code Form of
+the Covered Software, except that You may alter any license notices to
+the extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+You may choose to offer, and to charge a fee for, warranty, support,
+indemnity or liability obligations to one or more recipients of Covered
+Software. However, You may do so only on Your own behalf, and not on
+behalf of any Contributor. You must make it absolutely clear that any
+such warranty, support, indemnity, or liability obligation is offered by
+You alone, and You hereby agree to indemnify every Contributor for any
+liability incurred by such Contributor as a result of warranty, support,
+indemnity or liability terms You offer. You may include additional
+disclaimers of warranty and limitations of liability specific to any
+jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+---------------------------------------------------
+
+If it is impossible for You to comply with any of the terms of this
+License with respect to some or all of the Covered Software due to
+statute, judicial order, or regulation then You must: (a) comply with
+the terms of this License to the maximum extent possible; and (b)
+describe the limitations and the code they affect. Such description must
+be placed in a text file included with all distributions of the Covered
+Software under this License. Except to the extent prohibited by statute
+or regulation, such description must be sufficiently detailed for a
+recipient of ordinary skill to be able to understand it.
+
+5. Termination
+--------------
+
+5.1. The rights granted under this License will terminate automatically
+if You fail to comply with any of its terms. However, if You become
+compliant, then the rights granted under this License from a particular
+Contributor are reinstated (a) provisionally, unless and until such
+Contributor explicitly and finally terminates Your grants, and (b) on an
+ongoing basis, if such Contributor fails to notify You of the
+non-compliance by some reasonable means prior to 60 days after You have
+come back into compliance. Moreover, Your grants from a particular
+Contributor are reinstated on an ongoing basis if such Contributor
+notifies You of the non-compliance by some reasonable means, this is the
+first time You have received notice of non-compliance with this License
+from such Contributor, and You become compliant prior to 30 days after
+Your receipt of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+infringement claim (excluding declaratory judgment actions,
+counter-claims, and cross-claims) alleging that a Contributor Version
+directly or indirectly infringes any patent, then the rights granted to
+You by any and all Contributors for the Covered Software under Section
+2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+end user license agreements (excluding distributors and resellers) which
+have been validly granted by You or Your distributors under this License
+prior to termination shall survive termination.
+
+************************************************************************
+*                                                                      *
+*  6. Disclaimer of Warranty                                           *
+*  -------------------------                                           *
+*                                                                      *
+*  Covered Software is provided under this License on an "as is"       *
+*  basis, without warranty of any kind, either expressed, implied, or  *
+*  statutory, including, without limitation, warranties that the       *
+*  Covered Software is free of defects, merchantable, fit for a        *
+*  particular purpose or non-infringing. The entire risk as to the     *
+*  quality and performance of the Covered Software is with You.        *
+*  Should any Covered Software prove defective in any respect, You     *
+*  (not any Contributor) assume the cost of any necessary servicing,   *
+*  repair, or correction. This disclaimer of warranty constitutes an   *
+*  essential part of this License. No use of any Covered Software is   *
+*  authorized under this License except under this disclaimer.         *
+*                                                                      *
+************************************************************************
+
+************************************************************************
+*                                                                      *
+*  7. Limitation of Liability                                          *
+*  --------------------------                                          *
+*                                                                      *
+*  Under no circumstances and under no legal theory, whether tort      *
+*  (including negligence), contract, or otherwise, shall any           *
+*  Contributor, or anyone who distributes Covered Software as          *
+*  permitted above, be liable to You for any direct, indirect,         *
+*  special, incidental, or consequential damages of any character      *
+*  including, without limitation, damages for lost profits, loss of    *
+*  goodwill, work stoppage, computer failure or malfunction, or any    *
+*  and all other commercial damages or losses, even if such party      *
+*  shall have been informed of the possibility of such damages. This   *
+*  limitation of liability shall not apply to liability for death or   *
+*  personal injury resulting from such party's negligence to the       *
+*  extent applicable law prohibits such limitation. Some               *
+*  jurisdictions do not allow the exclusion or limitation of           *
+*  incidental or consequential damages, so this exclusion and          *
+*  limitation may not apply to You.                                    *
+*                                                                      *
+************************************************************************
+
+8. Litigation
+-------------
+
+Any litigation relating to this License may be brought only in the
+courts of a jurisdiction where the defendant maintains its principal
+place of business and such litigation shall be governed by laws of that
+jurisdiction, without reference to its conflict-of-law provisions.
+Nothing in this Section shall prevent a party's ability to bring
+cross-claims or counter-claims.
+
+9. Miscellaneous
+----------------
+
+This License represents the complete agreement concerning the subject
+matter hereof. If any provision of this License is held to be
+unenforceable, such provision shall be reformed only to the extent
+necessary to make it enforceable. Any law or regulation which provides
+that the language of a contract shall be construed against the drafter
+shall not be used to construe this License against a Contributor.
+
+10. Versions of the License
+---------------------------
+
+10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in Section
+10.3, no one other than the license steward has the right to modify or
+publish new versions of this License. Each version will be given a
+distinguishing version number.
+
+10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version
+of the License under which You originally received the Covered Software,
+or under the terms of any subsequent version published by the license
+steward.
+
+10.3. Modified Versions
+
+If you create software not governed by this License, and you want to
+create a new license for such software, you may create and use a
+modified version of this License if you rename the license and remove
+any references to the name of the license steward (except to note that
+such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With
+Secondary Licenses under the terms of this version of the License, the
+notice described in Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+-------------------------------------------
+
+  This Source Code Form is subject to the terms of the Mozilla Public
+  License, v. 2.0. If a copy of the MPL was not distributed with this
+  file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular
+file, then You may include the notice in a location (such as a LICENSE
+file in a relevant directory) where a recipient would be likely to look
+for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+---------------------------------------------------------
+
+  This Source Code Form is "Incompatible With Secondary Licenses", as
+  defined by the Mozilla Public License, v. 2.0.
diff --git a/client-sniffer/licenses/httpclient-NOTICE.txt b/client-sniffer/licenses/httpclient-NOTICE.txt
new file mode 100644
index 00000000000..91e5c40c4c6
--- /dev/null
+++ b/client-sniffer/licenses/httpclient-NOTICE.txt
@@ -0,0 +1,6 @@
+Apache HttpComponents Client
+Copyright 1999-2016 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
diff --git a/client-sniffer/licenses/httpcore-4.4.4.jar.sha1 b/client-sniffer/licenses/httpcore-4.4.4.jar.sha1
new file mode 100644
index 00000000000..ef0c257e012
--- /dev/null
+++ b/client-sniffer/licenses/httpcore-4.4.4.jar.sha1
@@ -0,0 +1 @@
+b31526a230871fbe285fbcbe2813f9c0839ae9b0
\ No newline at end of file
diff --git a/client-sniffer/licenses/httpcore-LICENSE.txt b/client-sniffer/licenses/httpcore-LICENSE.txt
new file mode 100644
index 00000000000..32f01eda18f
--- /dev/null
+++ b/client-sniffer/licenses/httpcore-LICENSE.txt
@@ -0,0 +1,558 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+=========================================================================
+
+This project includes Public Suffix List copied from
+<https://publicsuffix.org/list/effective_tld_names.dat>
+licensed under the terms of the Mozilla Public License, v. 2.0
+
+Full license text: <http://mozilla.org/MPL/2.0/>
+
+Mozilla Public License Version 2.0
+==================================
+
+1. Definitions
+--------------
+
+1.1. "Contributor"
+    means each individual or legal entity that creates, contributes to
+    the creation of, or owns Covered Software.
+
+1.2. "Contributor Version"
+    means the combination of the Contributions of others (if any) used
+    by a Contributor and that particular Contributor's Contribution.
+
+1.3. "Contribution"
+    means Covered Software of a particular Contributor.
+
+1.4. "Covered Software"
+    means Source Code Form to which the initial Contributor has attached
+    the notice in Exhibit A, the Executable Form of such Source Code
+    Form, and Modifications of such Source Code Form, in each case
+    including portions thereof.
+
+1.5. "Incompatible With Secondary Licenses"
+    means
+
+    (a) that the initial Contributor has attached the notice described
+        in Exhibit B to the Covered Software; or
+
+    (b) that the Covered Software was made available under the terms of
+        version 1.1 or earlier of the License, but not also under the
+        terms of a Secondary License.
+
+1.6. "Executable Form"
+    means any form of the work other than Source Code Form.
+
+1.7. "Larger Work"
+    means a work that combines Covered Software with other material, in
+    a separate file or files, that is not Covered Software.
+
+1.8. "License"
+    means this document.
+
+1.9. "Licensable"
+    means having the right to grant, to the maximum extent possible,
+    whether at the time of the initial grant or subsequently, any and
+    all of the rights conveyed by this License.
+
+1.10. "Modifications"
+    means any of the following:
+
+    (a) any file in Source Code Form that results from an addition to,
+        deletion from, or modification of the contents of Covered
+        Software; or
+
+    (b) any new file in Source Code Form that contains any Covered
+        Software.
+
+1.11. "Patent Claims" of a Contributor
+    means any patent claim(s), including without limitation, method,
+    process, and apparatus claims, in any patent Licensable by such
+    Contributor that would be infringed, but for the grant of the
+    License, by the making, using, selling, offering for sale, having
+    made, import, or transfer of either its Contributions or its
+    Contributor Version.
+
+1.12. "Secondary License"
+    means either the GNU General Public License, Version 2.0, the GNU
+    Lesser General Public License, Version 2.1, the GNU Affero General
+    Public License, Version 3.0, or any later versions of those
+    licenses.
+
+1.13. "Source Code Form"
+    means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your")
+    means an individual or a legal entity exercising rights under this
+    License. For legal entities, "You" includes any entity that
+    controls, is controlled by, or is under common control with You. For
+    purposes of this definition, "control" means (a) the power, direct
+    or indirect, to cause the direction or management of such entity,
+    whether by contract or otherwise, or (b) ownership of more than
+    fifty percent (50%) of the outstanding shares or beneficial
+    ownership of such entity.
+
+2. License Grants and Conditions
+--------------------------------
+
+2.1. Grants
+
+Each Contributor hereby grants You a world-wide, royalty-free,
+non-exclusive license:
+
+(a) under intellectual property rights (other than patent or trademark)
+    Licensable by such Contributor to use, reproduce, make available,
+    modify, display, perform, distribute, and otherwise exploit its
+    Contributions, either on an unmodified basis, with Modifications, or
+    as part of a Larger Work; and
+
+(b) under Patent Claims of such Contributor to make, use, sell, offer
+    for sale, have made, import, and otherwise transfer either its
+    Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+The licenses granted in Section 2.1 with respect to any Contribution
+become effective for each Contribution on the date the Contributor first
+distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+The licenses granted in this Section 2 are the only rights granted under
+this License. No additional rights or licenses will be implied from the
+distribution or licensing of Covered Software under this License.
+Notwithstanding Section 2.1(b) above, no patent license is granted by a
+Contributor:
+
+(a) for any code that a Contributor has removed from Covered Software;
+    or
+
+(b) for infringements caused by: (i) Your and any other third party's
+    modifications of Covered Software, or (ii) the combination of its
+    Contributions with other software (except as part of its Contributor
+    Version); or
+
+(c) under Patent Claims infringed by Covered Software in the absence of
+    its Contributions.
+
+This License does not grant any rights in the trademarks, service marks,
+or logos of any Contributor (except as may be necessary to comply with
+the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+No Contributor makes additional grants as a result of Your choice to
+distribute the Covered Software under a subsequent version of this
+License (see Section 10.2) or under the terms of a Secondary License (if
+permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+Each Contributor represents that the Contributor believes its
+Contributions are its original creation(s) or it has sufficient rights
+to grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+This License is not intended to limit any rights You have under
+applicable copyright doctrines of fair use, fair dealing, or other
+equivalents.
+
+2.7. Conditions
+
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+in Section 2.1.
+
+3. Responsibilities
+-------------------
+
+3.1. Distribution of Source Form
+
+All distribution of Covered Software in Source Code Form, including any
+Modifications that You create or to which You contribute, must be under
+the terms of this License. You must inform recipients that the Source
+Code Form of the Covered Software is governed by the terms of this
+License, and how they can obtain a copy of this License. You may not
+attempt to alter or restrict the recipients' rights in the Source Code
+Form.
+
+3.2. Distribution of Executable Form
+
+If You distribute Covered Software in Executable Form then:
+
+(a) such Covered Software must also be made available in Source Code
+    Form, as described in Section 3.1, and You must inform recipients of
+    the Executable Form how they can obtain a copy of such Source Code
+    Form by reasonable means in a timely manner, at a charge no more
+    than the cost of distribution to the recipient; and
+
+(b) You may distribute such Executable Form under the terms of this
+    License, or sublicense it under different terms, provided that the
+    license for the Executable Form does not attempt to limit or alter
+    the recipients' rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+You may create and distribute a Larger Work under terms of Your choice,
+provided that You also comply with the requirements of this License for
+the Covered Software. If the Larger Work is a combination of Covered
+Software with a work governed by one or more Secondary Licenses, and the
+Covered Software is not Incompatible With Secondary Licenses, this
+License permits You to additionally distribute such Covered Software
+under the terms of such Secondary License(s), so that the recipient of
+the Larger Work may, at their option, further distribute the Covered
+Software under the terms of either this License or such Secondary
+License(s).
+
+3.4. Notices
+
+You may not remove or alter the substance of any license notices
+(including copyright notices, patent notices, disclaimers of warranty,
+or limitations of liability) contained within the Source Code Form of
+the Covered Software, except that You may alter any license notices to
+the extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+You may choose to offer, and to charge a fee for, warranty, support,
+indemnity or liability obligations to one or more recipients of Covered
+Software. However, You may do so only on Your own behalf, and not on
+behalf of any Contributor. You must make it absolutely clear that any
+such warranty, support, indemnity, or liability obligation is offered by
+You alone, and You hereby agree to indemnify every Contributor for any
+liability incurred by such Contributor as a result of warranty, support,
+indemnity or liability terms You offer. You may include additional
+disclaimers of warranty and limitations of liability specific to any
+jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+---------------------------------------------------
+
+If it is impossible for You to comply with any of the terms of this
+License with respect to some or all of the Covered Software due to
+statute, judicial order, or regulation then You must: (a) comply with
+the terms of this License to the maximum extent possible; and (b)
+describe the limitations and the code they affect. Such description must
+be placed in a text file included with all distributions of the Covered
+Software under this License. Except to the extent prohibited by statute
+or regulation, such description must be sufficiently detailed for a
+recipient of ordinary skill to be able to understand it.
+
+5. Termination
+--------------
+
+5.1. The rights granted under this License will terminate automatically
+if You fail to comply with any of its terms. However, if You become
+compliant, then the rights granted under this License from a particular
+Contributor are reinstated (a) provisionally, unless and until such
+Contributor explicitly and finally terminates Your grants, and (b) on an
+ongoing basis, if such Contributor fails to notify You of the
+non-compliance by some reasonable means prior to 60 days after You have
+come back into compliance. Moreover, Your grants from a particular
+Contributor are reinstated on an ongoing basis if such Contributor
+notifies You of the non-compliance by some reasonable means, this is the
+first time You have received notice of non-compliance with this License
+from such Contributor, and You become compliant prior to 30 days after
+Your receipt of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+infringement claim (excluding declaratory judgment actions,
+counter-claims, and cross-claims) alleging that a Contributor Version
+directly or indirectly infringes any patent, then the rights granted to
+You by any and all Contributors for the Covered Software under Section
+2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+end user license agreements (excluding distributors and resellers) which
+have been validly granted by You or Your distributors under this License
+prior to termination shall survive termination.
+
+************************************************************************
+*                                                                      *
+*  6. Disclaimer of Warranty                                           *
+*  -------------------------                                           *
+*                                                                      *
+*  Covered Software is provided under this License on an "as is"       *
+*  basis, without warranty of any kind, either expressed, implied, or  *
+*  statutory, including, without limitation, warranties that the       *
+*  Covered Software is free of defects, merchantable, fit for a        *
+*  particular purpose or non-infringing. The entire risk as to the     *
+*  quality and performance of the Covered Software is with You.        *
+*  Should any Covered Software prove defective in any respect, You     *
+*  (not any Contributor) assume the cost of any necessary servicing,   *
+*  repair, or correction. This disclaimer of warranty constitutes an   *
+*  essential part of this License. No use of any Covered Software is   *
+*  authorized under this License except under this disclaimer.         *
+*                                                                      *
+************************************************************************
+
+************************************************************************
+*                                                                      *
+*  7. Limitation of Liability                                          *
+*  --------------------------                                          *
+*                                                                      *
+*  Under no circumstances and under no legal theory, whether tort      *
+*  (including negligence), contract, or otherwise, shall any           *
+*  Contributor, or anyone who distributes Covered Software as          *
+*  permitted above, be liable to You for any direct, indirect,         *
+*  special, incidental, or consequential damages of any character      *
+*  including, without limitation, damages for lost profits, loss of    *
+*  goodwill, work stoppage, computer failure or malfunction, or any    *
+*  and all other commercial damages or losses, even if such party      *
+*  shall have been informed of the possibility of such damages. This   *
+*  limitation of liability shall not apply to liability for death or   *
+*  personal injury resulting from such party's negligence to the       *
+*  extent applicable law prohibits such limitation. Some               *
+*  jurisdictions do not allow the exclusion or limitation of           *
+*  incidental or consequential damages, so this exclusion and          *
+*  limitation may not apply to You.                                    *
+*                                                                      *
+************************************************************************
+
+8. Litigation
+-------------
+
+Any litigation relating to this License may be brought only in the
+courts of a jurisdiction where the defendant maintains its principal
+place of business and such litigation shall be governed by laws of that
+jurisdiction, without reference to its conflict-of-law provisions.
+Nothing in this Section shall prevent a party's ability to bring
+cross-claims or counter-claims.
+
+9. Miscellaneous
+----------------
+
+This License represents the complete agreement concerning the subject
+matter hereof. If any provision of this License is held to be
+unenforceable, such provision shall be reformed only to the extent
+necessary to make it enforceable. Any law or regulation which provides
+that the language of a contract shall be construed against the drafter
+shall not be used to construe this License against a Contributor.
+
+10. Versions of the License
+---------------------------
+
+10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in Section
+10.3, no one other than the license steward has the right to modify or
+publish new versions of this License. Each version will be given a
+distinguishing version number.
+
+10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version
+of the License under which You originally received the Covered Software,
+or under the terms of any subsequent version published by the license
+steward.
+
+10.3. Modified Versions
+
+If you create software not governed by this License, and you want to
+create a new license for such software, you may create and use a
+modified version of this License if you rename the license and remove
+any references to the name of the license steward (except to note that
+such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With
+Secondary Licenses under the terms of this version of the License, the
+notice described in Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+-------------------------------------------
+
+  This Source Code Form is subject to the terms of the Mozilla Public
+  License, v. 2.0. If a copy of the MPL was not distributed with this
+  file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular
+file, then You may include the notice in a location (such as a LICENSE
+file in a relevant directory) where a recipient would be likely to look
+for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+---------------------------------------------------------
+
+  This Source Code Form is "Incompatible With Secondary Licenses", as
+  defined by the Mozilla Public License, v. 2.0.
diff --git a/client-sniffer/licenses/httpcore-NOTICE.txt b/client-sniffer/licenses/httpcore-NOTICE.txt
new file mode 100644
index 00000000000..91e5c40c4c6
--- /dev/null
+++ b/client-sniffer/licenses/httpcore-NOTICE.txt
@@ -0,0 +1,6 @@
+Apache HttpComponents Client
+Copyright 1999-2016 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
diff --git a/client-sniffer/licenses/jackson-core-2.7.1.jar.sha1 b/client-sniffer/licenses/jackson-core-2.7.1.jar.sha1
new file mode 100644
index 00000000000..73831ed2d51
--- /dev/null
+++ b/client-sniffer/licenses/jackson-core-2.7.1.jar.sha1
@@ -0,0 +1 @@
+4127b62db028f981e81caa248953c0899d720f98
\ No newline at end of file
diff --git a/client-sniffer/licenses/jackson-core-LICENSE b/client-sniffer/licenses/jackson-core-LICENSE
new file mode 100644
index 00000000000..f5f45d26a49
--- /dev/null
+++ b/client-sniffer/licenses/jackson-core-LICENSE
@@ -0,0 +1,8 @@
+This copy of Jackson JSON processor streaming parser/generator is licensed under the
+Apache (Software) License, version 2.0 ("the License").
+See the License for details about distribution rights, and the
+specific rights regarding derivate works.
+
+You may obtain a copy of the License at:
+
+http://www.apache.org/licenses/LICENSE-2.0
diff --git a/client-sniffer/licenses/jackson-core-NOTICE b/client-sniffer/licenses/jackson-core-NOTICE
new file mode 100644
index 00000000000..4c976b7b4cc
--- /dev/null
+++ b/client-sniffer/licenses/jackson-core-NOTICE
@@ -0,0 +1,20 @@
+# Jackson JSON processor
+
+Jackson is a high-performance, Free/Open Source JSON processing library.
+It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has
+been in development since 2007.
+It is currently developed by a community of developers, as well as supported
+commercially by FasterXML.com.
+
+## Licensing
+
+Jackson core and extension components may licensed under different licenses.
+To find the details that apply to this artifact see the accompanying LICENSE file.
+For more information, including possible other licensing options, contact
+FasterXML.com (http://fasterxml.com).
+
+## Credits
+
+A list of contributors may be found from CREDITS file, which is included
+in some artifacts (usually source distributions); but is always available
+from the source code management (SCM) system project uses.
diff --git a/client/build.gradle b/client/build.gradle
index 6d2b92f0b0c..4c4a56d7731 100644
--- a/client/build.gradle
+++ b/client/build.gradle
@@ -58,9 +58,6 @@ forbiddenApisTest.enabled=false
   //signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
 //}
 
-//TODO add licenses for dependencies and take care of distribution
-//dependency license are currently checked in distribution
-dependencyLicenses.enabled=false
 //JarHell is part of es core, which we don't want to pull in
 jarHell.enabled=false
 //NamingConventionCheck is part of test-framework, which we don't want to pull in as it depends on es core
diff --git a/client/licenses/commons-codec-1.10.jar.sha1 b/client/licenses/commons-codec-1.10.jar.sha1
new file mode 100644
index 00000000000..3fe8682a1b0
--- /dev/null
+++ b/client/licenses/commons-codec-1.10.jar.sha1
@@ -0,0 +1 @@
+4b95f4897fa13f2cd904aee711aeafc0c5295cd8
\ No newline at end of file
diff --git a/client/licenses/commons-codec-LICENSE.txt b/client/licenses/commons-codec-LICENSE.txt
new file mode 100644
index 00000000000..d6456956733
--- /dev/null
+++ b/client/licenses/commons-codec-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/client/licenses/commons-codec-NOTICE.txt b/client/licenses/commons-codec-NOTICE.txt
new file mode 100644
index 00000000000..1da9af50f60
--- /dev/null
+++ b/client/licenses/commons-codec-NOTICE.txt
@@ -0,0 +1,17 @@
+Apache Commons Codec
+Copyright 2002-2014 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java
+contains test data from http://aspell.net/test/orig/batch0.tab.
+Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org)
+
+===============================================================================
+
+The content of package org.apache.commons.codec.language.bm has been translated
+from the original php source code available at http://stevemorse.org/phoneticinfo.htm
+with permission from the original authors.
+Original source copyright:
+Copyright (c) 2008 Alexander Beider & Stephen P. Morse.
diff --git a/client/licenses/commons-logging-1.1.3.jar.sha1 b/client/licenses/commons-logging-1.1.3.jar.sha1
new file mode 100644
index 00000000000..5b8f029e582
--- /dev/null
+++ b/client/licenses/commons-logging-1.1.3.jar.sha1
@@ -0,0 +1 @@
+f6f66e966c70a83ffbdb6f17a0919eaf7c8aca7f
\ No newline at end of file
diff --git a/client/licenses/commons-logging-LICENSE.txt b/client/licenses/commons-logging-LICENSE.txt
new file mode 100644
index 00000000000..d6456956733
--- /dev/null
+++ b/client/licenses/commons-logging-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/client/licenses/commons-logging-NOTICE.txt b/client/licenses/commons-logging-NOTICE.txt
new file mode 100644
index 00000000000..556bd03951d
--- /dev/null
+++ b/client/licenses/commons-logging-NOTICE.txt
@@ -0,0 +1,6 @@
+Apache Commons Logging
+Copyright 2003-2014 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
diff --git a/client/licenses/httpclient-4.5.2.jar.sha1 b/client/licenses/httpclient-4.5.2.jar.sha1
new file mode 100644
index 00000000000..6937112a09f
--- /dev/null
+++ b/client/licenses/httpclient-4.5.2.jar.sha1
@@ -0,0 +1 @@
+733db77aa8d9b2d68015189df76ab06304406e50
\ No newline at end of file
diff --git a/client/licenses/httpclient-LICENSE.txt b/client/licenses/httpclient-LICENSE.txt
new file mode 100644
index 00000000000..32f01eda18f
--- /dev/null
+++ b/client/licenses/httpclient-LICENSE.txt
@@ -0,0 +1,558 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+=========================================================================
+
+This project includes Public Suffix List copied from
+<https://publicsuffix.org/list/effective_tld_names.dat>
+licensed under the terms of the Mozilla Public License, v. 2.0
+
+Full license text: <http://mozilla.org/MPL/2.0/>
+
+Mozilla Public License Version 2.0
+==================================
+
+1. Definitions
+--------------
+
+1.1. "Contributor"
+    means each individual or legal entity that creates, contributes to
+    the creation of, or owns Covered Software.
+
+1.2. "Contributor Version"
+    means the combination of the Contributions of others (if any) used
+    by a Contributor and that particular Contributor's Contribution.
+
+1.3. "Contribution"
+    means Covered Software of a particular Contributor.
+
+1.4. "Covered Software"
+    means Source Code Form to which the initial Contributor has attached
+    the notice in Exhibit A, the Executable Form of such Source Code
+    Form, and Modifications of such Source Code Form, in each case
+    including portions thereof.
+
+1.5. "Incompatible With Secondary Licenses"
+    means
+
+    (a) that the initial Contributor has attached the notice described
+        in Exhibit B to the Covered Software; or
+
+    (b) that the Covered Software was made available under the terms of
+        version 1.1 or earlier of the License, but not also under the
+        terms of a Secondary License.
+
+1.6. "Executable Form"
+    means any form of the work other than Source Code Form.
+
+1.7. "Larger Work"
+    means a work that combines Covered Software with other material, in
+    a separate file or files, that is not Covered Software.
+
+1.8. "License"
+    means this document.
+
+1.9. "Licensable"
+    means having the right to grant, to the maximum extent possible,
+    whether at the time of the initial grant or subsequently, any and
+    all of the rights conveyed by this License.
+
+1.10. "Modifications"
+    means any of the following:
+
+    (a) any file in Source Code Form that results from an addition to,
+        deletion from, or modification of the contents of Covered
+        Software; or
+
+    (b) any new file in Source Code Form that contains any Covered
+        Software.
+
+1.11. "Patent Claims" of a Contributor
+    means any patent claim(s), including without limitation, method,
+    process, and apparatus claims, in any patent Licensable by such
+    Contributor that would be infringed, but for the grant of the
+    License, by the making, using, selling, offering for sale, having
+    made, import, or transfer of either its Contributions or its
+    Contributor Version.
+
+1.12. "Secondary License"
+    means either the GNU General Public License, Version 2.0, the GNU
+    Lesser General Public License, Version 2.1, the GNU Affero General
+    Public License, Version 3.0, or any later versions of those
+    licenses.
+
+1.13. "Source Code Form"
+    means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your")
+    means an individual or a legal entity exercising rights under this
+    License. For legal entities, "You" includes any entity that
+    controls, is controlled by, or is under common control with You. For
+    purposes of this definition, "control" means (a) the power, direct
+    or indirect, to cause the direction or management of such entity,
+    whether by contract or otherwise, or (b) ownership of more than
+    fifty percent (50%) of the outstanding shares or beneficial
+    ownership of such entity.
+
+2. License Grants and Conditions
+--------------------------------
+
+2.1. Grants
+
+Each Contributor hereby grants You a world-wide, royalty-free,
+non-exclusive license:
+
+(a) under intellectual property rights (other than patent or trademark)
+    Licensable by such Contributor to use, reproduce, make available,
+    modify, display, perform, distribute, and otherwise exploit its
+    Contributions, either on an unmodified basis, with Modifications, or
+    as part of a Larger Work; and
+
+(b) under Patent Claims of such Contributor to make, use, sell, offer
+    for sale, have made, import, and otherwise transfer either its
+    Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+The licenses granted in Section 2.1 with respect to any Contribution
+become effective for each Contribution on the date the Contributor first
+distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+The licenses granted in this Section 2 are the only rights granted under
+this License. No additional rights or licenses will be implied from the
+distribution or licensing of Covered Software under this License.
+Notwithstanding Section 2.1(b) above, no patent license is granted by a
+Contributor:
+
+(a) for any code that a Contributor has removed from Covered Software;
+    or
+
+(b) for infringements caused by: (i) Your and any other third party's
+    modifications of Covered Software, or (ii) the combination of its
+    Contributions with other software (except as part of its Contributor
+    Version); or
+
+(c) under Patent Claims infringed by Covered Software in the absence of
+    its Contributions.
+
+This License does not grant any rights in the trademarks, service marks,
+or logos of any Contributor (except as may be necessary to comply with
+the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+No Contributor makes additional grants as a result of Your choice to
+distribute the Covered Software under a subsequent version of this
+License (see Section 10.2) or under the terms of a Secondary License (if
+permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+Each Contributor represents that the Contributor believes its
+Contributions are its original creation(s) or it has sufficient rights
+to grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+This License is not intended to limit any rights You have under
+applicable copyright doctrines of fair use, fair dealing, or other
+equivalents.
+
+2.7. Conditions
+
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+in Section 2.1.
+
+3. Responsibilities
+-------------------
+
+3.1. Distribution of Source Form
+
+All distribution of Covered Software in Source Code Form, including any
+Modifications that You create or to which You contribute, must be under
+the terms of this License. You must inform recipients that the Source
+Code Form of the Covered Software is governed by the terms of this
+License, and how they can obtain a copy of this License. You may not
+attempt to alter or restrict the recipients' rights in the Source Code
+Form.
+
+3.2. Distribution of Executable Form
+
+If You distribute Covered Software in Executable Form then:
+
+(a) such Covered Software must also be made available in Source Code
+    Form, as described in Section 3.1, and You must inform recipients of
+    the Executable Form how they can obtain a copy of such Source Code
+    Form by reasonable means in a timely manner, at a charge no more
+    than the cost of distribution to the recipient; and
+
+(b) You may distribute such Executable Form under the terms of this
+    License, or sublicense it under different terms, provided that the
+    license for the Executable Form does not attempt to limit or alter
+    the recipients' rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+You may create and distribute a Larger Work under terms of Your choice,
+provided that You also comply with the requirements of this License for
+the Covered Software. If the Larger Work is a combination of Covered
+Software with a work governed by one or more Secondary Licenses, and the
+Covered Software is not Incompatible With Secondary Licenses, this
+License permits You to additionally distribute such Covered Software
+under the terms of such Secondary License(s), so that the recipient of
+the Larger Work may, at their option, further distribute the Covered
+Software under the terms of either this License or such Secondary
+License(s).
+
+3.4. Notices
+
+You may not remove or alter the substance of any license notices
+(including copyright notices, patent notices, disclaimers of warranty,
+or limitations of liability) contained within the Source Code Form of
+the Covered Software, except that You may alter any license notices to
+the extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+You may choose to offer, and to charge a fee for, warranty, support,
+indemnity or liability obligations to one or more recipients of Covered
+Software. However, You may do so only on Your own behalf, and not on
+behalf of any Contributor. You must make it absolutely clear that any
+such warranty, support, indemnity, or liability obligation is offered by
+You alone, and You hereby agree to indemnify every Contributor for any
+liability incurred by such Contributor as a result of warranty, support,
+indemnity or liability terms You offer. You may include additional
+disclaimers of warranty and limitations of liability specific to any
+jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+---------------------------------------------------
+
+If it is impossible for You to comply with any of the terms of this
+License with respect to some or all of the Covered Software due to
+statute, judicial order, or regulation then You must: (a) comply with
+the terms of this License to the maximum extent possible; and (b)
+describe the limitations and the code they affect. Such description must
+be placed in a text file included with all distributions of the Covered
+Software under this License. Except to the extent prohibited by statute
+or regulation, such description must be sufficiently detailed for a
+recipient of ordinary skill to be able to understand it.
+
+5. Termination
+--------------
+
+5.1. The rights granted under this License will terminate automatically
+if You fail to comply with any of its terms. However, if You become
+compliant, then the rights granted under this License from a particular
+Contributor are reinstated (a) provisionally, unless and until such
+Contributor explicitly and finally terminates Your grants, and (b) on an
+ongoing basis, if such Contributor fails to notify You of the
+non-compliance by some reasonable means prior to 60 days after You have
+come back into compliance. Moreover, Your grants from a particular
+Contributor are reinstated on an ongoing basis if such Contributor
+notifies You of the non-compliance by some reasonable means, this is the
+first time You have received notice of non-compliance with this License
+from such Contributor, and You become compliant prior to 30 days after
+Your receipt of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+infringement claim (excluding declaratory judgment actions,
+counter-claims, and cross-claims) alleging that a Contributor Version
+directly or indirectly infringes any patent, then the rights granted to
+You by any and all Contributors for the Covered Software under Section
+2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+end user license agreements (excluding distributors and resellers) which
+have been validly granted by You or Your distributors under this License
+prior to termination shall survive termination.
+
+************************************************************************
+*                                                                      *
+*  6. Disclaimer of Warranty                                           *
+*  -------------------------                                           *
+*                                                                      *
+*  Covered Software is provided under this License on an "as is"       *
+*  basis, without warranty of any kind, either expressed, implied, or  *
+*  statutory, including, without limitation, warranties that the       *
+*  Covered Software is free of defects, merchantable, fit for a        *
+*  particular purpose or non-infringing. The entire risk as to the     *
+*  quality and performance of the Covered Software is with You.        *
+*  Should any Covered Software prove defective in any respect, You     *
+*  (not any Contributor) assume the cost of any necessary servicing,   *
+*  repair, or correction. This disclaimer of warranty constitutes an   *
+*  essential part of this License. No use of any Covered Software is   *
+*  authorized under this License except under this disclaimer.         *
+*                                                                      *
+************************************************************************
+
+************************************************************************
+*                                                                      *
+*  7. Limitation of Liability                                          *
+*  --------------------------                                          *
+*                                                                      *
+*  Under no circumstances and under no legal theory, whether tort      *
+*  (including negligence), contract, or otherwise, shall any           *
+*  Contributor, or anyone who distributes Covered Software as          *
+*  permitted above, be liable to You for any direct, indirect,         *
+*  special, incidental, or consequential damages of any character      *
+*  including, without limitation, damages for lost profits, loss of    *
+*  goodwill, work stoppage, computer failure or malfunction, or any    *
+*  and all other commercial damages or losses, even if such party      *
+*  shall have been informed of the possibility of such damages. This   *
+*  limitation of liability shall not apply to liability for death or   *
+*  personal injury resulting from such party's negligence to the       *
+*  extent applicable law prohibits such limitation. Some               *
+*  jurisdictions do not allow the exclusion or limitation of           *
+*  incidental or consequential damages, so this exclusion and          *
+*  limitation may not apply to You.                                    *
+*                                                                      *
+************************************************************************
+
+8. Litigation
+-------------
+
+Any litigation relating to this License may be brought only in the
+courts of a jurisdiction where the defendant maintains its principal
+place of business and such litigation shall be governed by laws of that
+jurisdiction, without reference to its conflict-of-law provisions.
+Nothing in this Section shall prevent a party's ability to bring
+cross-claims or counter-claims.
+
+9. Miscellaneous
+----------------
+
+This License represents the complete agreement concerning the subject
+matter hereof. If any provision of this License is held to be
+unenforceable, such provision shall be reformed only to the extent
+necessary to make it enforceable. Any law or regulation which provides
+that the language of a contract shall be construed against the drafter
+shall not be used to construe this License against a Contributor.
+
+10. Versions of the License
+---------------------------
+
+10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in Section
+10.3, no one other than the license steward has the right to modify or
+publish new versions of this License. Each version will be given a
+distinguishing version number.
+
+10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version
+of the License under which You originally received the Covered Software,
+or under the terms of any subsequent version published by the license
+steward.
+
+10.3. Modified Versions
+
+If you create software not governed by this License, and you want to
+create a new license for such software, you may create and use a
+modified version of this License if you rename the license and remove
+any references to the name of the license steward (except to note that
+such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With
+Secondary Licenses under the terms of this version of the License, the
+notice described in Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+-------------------------------------------
+
+  This Source Code Form is subject to the terms of the Mozilla Public
+  License, v. 2.0. If a copy of the MPL was not distributed with this
+  file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular
+file, then You may include the notice in a location (such as a LICENSE
+file in a relevant directory) where a recipient would be likely to look
+for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+---------------------------------------------------------
+
+  This Source Code Form is "Incompatible With Secondary Licenses", as
+  defined by the Mozilla Public License, v. 2.0.
diff --git a/client/licenses/httpclient-NOTICE.txt b/client/licenses/httpclient-NOTICE.txt
new file mode 100644
index 00000000000..91e5c40c4c6
--- /dev/null
+++ b/client/licenses/httpclient-NOTICE.txt
@@ -0,0 +1,6 @@
+Apache HttpComponents Client
+Copyright 1999-2016 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
diff --git a/client/licenses/httpcore-4.4.4.jar.sha1 b/client/licenses/httpcore-4.4.4.jar.sha1
new file mode 100644
index 00000000000..ef0c257e012
--- /dev/null
+++ b/client/licenses/httpcore-4.4.4.jar.sha1
@@ -0,0 +1 @@
+b31526a230871fbe285fbcbe2813f9c0839ae9b0
\ No newline at end of file
diff --git a/client/licenses/httpcore-LICENSE.txt b/client/licenses/httpcore-LICENSE.txt
new file mode 100644
index 00000000000..32f01eda18f
--- /dev/null
+++ b/client/licenses/httpcore-LICENSE.txt
@@ -0,0 +1,558 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+=========================================================================
+
+This project includes Public Suffix List copied from
+<https://publicsuffix.org/list/effective_tld_names.dat>
+licensed under the terms of the Mozilla Public License, v. 2.0
+
+Full license text: <http://mozilla.org/MPL/2.0/>
+
+Mozilla Public License Version 2.0
+==================================
+
+1. Definitions
+--------------
+
+1.1. "Contributor"
+    means each individual or legal entity that creates, contributes to
+    the creation of, or owns Covered Software.
+
+1.2. "Contributor Version"
+    means the combination of the Contributions of others (if any) used
+    by a Contributor and that particular Contributor's Contribution.
+
+1.3. "Contribution"
+    means Covered Software of a particular Contributor.
+
+1.4. "Covered Software"
+    means Source Code Form to which the initial Contributor has attached
+    the notice in Exhibit A, the Executable Form of such Source Code
+    Form, and Modifications of such Source Code Form, in each case
+    including portions thereof.
+
+1.5. "Incompatible With Secondary Licenses"
+    means
+
+    (a) that the initial Contributor has attached the notice described
+        in Exhibit B to the Covered Software; or
+
+    (b) that the Covered Software was made available under the terms of
+        version 1.1 or earlier of the License, but not also under the
+        terms of a Secondary License.
+
+1.6. "Executable Form"
+    means any form of the work other than Source Code Form.
+
+1.7. "Larger Work"
+    means a work that combines Covered Software with other material, in
+    a separate file or files, that is not Covered Software.
+
+1.8. "License"
+    means this document.
+
+1.9. "Licensable"
+    means having the right to grant, to the maximum extent possible,
+    whether at the time of the initial grant or subsequently, any and
+    all of the rights conveyed by this License.
+
+1.10. "Modifications"
+    means any of the following:
+
+    (a) any file in Source Code Form that results from an addition to,
+        deletion from, or modification of the contents of Covered
+        Software; or
+
+    (b) any new file in Source Code Form that contains any Covered
+        Software.
+
+1.11. "Patent Claims" of a Contributor
+    means any patent claim(s), including without limitation, method,
+    process, and apparatus claims, in any patent Licensable by such
+    Contributor that would be infringed, but for the grant of the
+    License, by the making, using, selling, offering for sale, having
+    made, import, or transfer of either its Contributions or its
+    Contributor Version.
+
+1.12. "Secondary License"
+    means either the GNU General Public License, Version 2.0, the GNU
+    Lesser General Public License, Version 2.1, the GNU Affero General
+    Public License, Version 3.0, or any later versions of those
+    licenses.
+
+1.13. "Source Code Form"
+    means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your")
+    means an individual or a legal entity exercising rights under this
+    License. For legal entities, "You" includes any entity that
+    controls, is controlled by, or is under common control with You. For
+    purposes of this definition, "control" means (a) the power, direct
+    or indirect, to cause the direction or management of such entity,
+    whether by contract or otherwise, or (b) ownership of more than
+    fifty percent (50%) of the outstanding shares or beneficial
+    ownership of such entity.
+
+2. License Grants and Conditions
+--------------------------------
+
+2.1. Grants
+
+Each Contributor hereby grants You a world-wide, royalty-free,
+non-exclusive license:
+
+(a) under intellectual property rights (other than patent or trademark)
+    Licensable by such Contributor to use, reproduce, make available,
+    modify, display, perform, distribute, and otherwise exploit its
+    Contributions, either on an unmodified basis, with Modifications, or
+    as part of a Larger Work; and
+
+(b) under Patent Claims of such Contributor to make, use, sell, offer
+    for sale, have made, import, and otherwise transfer either its
+    Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+The licenses granted in Section 2.1 with respect to any Contribution
+become effective for each Contribution on the date the Contributor first
+distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+The licenses granted in this Section 2 are the only rights granted under
+this License. No additional rights or licenses will be implied from the
+distribution or licensing of Covered Software under this License.
+Notwithstanding Section 2.1(b) above, no patent license is granted by a
+Contributor:
+
+(a) for any code that a Contributor has removed from Covered Software;
+    or
+
+(b) for infringements caused by: (i) Your and any other third party's
+    modifications of Covered Software, or (ii) the combination of its
+    Contributions with other software (except as part of its Contributor
+    Version); or
+
+(c) under Patent Claims infringed by Covered Software in the absence of
+    its Contributions.
+
+This License does not grant any rights in the trademarks, service marks,
+or logos of any Contributor (except as may be necessary to comply with
+the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+No Contributor makes additional grants as a result of Your choice to
+distribute the Covered Software under a subsequent version of this
+License (see Section 10.2) or under the terms of a Secondary License (if
+permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+Each Contributor represents that the Contributor believes its
+Contributions are its original creation(s) or it has sufficient rights
+to grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+This License is not intended to limit any rights You have under
+applicable copyright doctrines of fair use, fair dealing, or other
+equivalents.
+
+2.7. Conditions
+
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+in Section 2.1.
+
+3. Responsibilities
+-------------------
+
+3.1. Distribution of Source Form
+
+All distribution of Covered Software in Source Code Form, including any
+Modifications that You create or to which You contribute, must be under
+the terms of this License. You must inform recipients that the Source
+Code Form of the Covered Software is governed by the terms of this
+License, and how they can obtain a copy of this License. You may not
+attempt to alter or restrict the recipients' rights in the Source Code
+Form.
+
+3.2. Distribution of Executable Form
+
+If You distribute Covered Software in Executable Form then:
+
+(a) such Covered Software must also be made available in Source Code
+    Form, as described in Section 3.1, and You must inform recipients of
+    the Executable Form how they can obtain a copy of such Source Code
+    Form by reasonable means in a timely manner, at a charge no more
+    than the cost of distribution to the recipient; and
+
+(b) You may distribute such Executable Form under the terms of this
+    License, or sublicense it under different terms, provided that the
+    license for the Executable Form does not attempt to limit or alter
+    the recipients' rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+You may create and distribute a Larger Work under terms of Your choice,
+provided that You also comply with the requirements of this License for
+the Covered Software. If the Larger Work is a combination of Covered
+Software with a work governed by one or more Secondary Licenses, and the
+Covered Software is not Incompatible With Secondary Licenses, this
+License permits You to additionally distribute such Covered Software
+under the terms of such Secondary License(s), so that the recipient of
+the Larger Work may, at their option, further distribute the Covered
+Software under the terms of either this License or such Secondary
+License(s).
+
+3.4. Notices
+
+You may not remove or alter the substance of any license notices
+(including copyright notices, patent notices, disclaimers of warranty,
+or limitations of liability) contained within the Source Code Form of
+the Covered Software, except that You may alter any license notices to
+the extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+You may choose to offer, and to charge a fee for, warranty, support,
+indemnity or liability obligations to one or more recipients of Covered
+Software. However, You may do so only on Your own behalf, and not on
+behalf of any Contributor. You must make it absolutely clear that any
+such warranty, support, indemnity, or liability obligation is offered by
+You alone, and You hereby agree to indemnify every Contributor for any
+liability incurred by such Contributor as a result of warranty, support,
+indemnity or liability terms You offer. You may include additional
+disclaimers of warranty and limitations of liability specific to any
+jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+---------------------------------------------------
+
+If it is impossible for You to comply with any of the terms of this
+License with respect to some or all of the Covered Software due to
+statute, judicial order, or regulation then You must: (a) comply with
+the terms of this License to the maximum extent possible; and (b)
+describe the limitations and the code they affect. Such description must
+be placed in a text file included with all distributions of the Covered
+Software under this License. Except to the extent prohibited by statute
+or regulation, such description must be sufficiently detailed for a
+recipient of ordinary skill to be able to understand it.
+
+5. Termination
+--------------
+
+5.1. The rights granted under this License will terminate automatically
+if You fail to comply with any of its terms. However, if You become
+compliant, then the rights granted under this License from a particular
+Contributor are reinstated (a) provisionally, unless and until such
+Contributor explicitly and finally terminates Your grants, and (b) on an
+ongoing basis, if such Contributor fails to notify You of the
+non-compliance by some reasonable means prior to 60 days after You have
+come back into compliance. Moreover, Your grants from a particular
+Contributor are reinstated on an ongoing basis if such Contributor
+notifies You of the non-compliance by some reasonable means, this is the
+first time You have received notice of non-compliance with this License
+from such Contributor, and You become compliant prior to 30 days after
+Your receipt of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+infringement claim (excluding declaratory judgment actions,
+counter-claims, and cross-claims) alleging that a Contributor Version
+directly or indirectly infringes any patent, then the rights granted to
+You by any and all Contributors for the Covered Software under Section
+2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+end user license agreements (excluding distributors and resellers) which
+have been validly granted by You or Your distributors under this License
+prior to termination shall survive termination.
+
+************************************************************************
+*                                                                      *
+*  6. Disclaimer of Warranty                                           *
+*  -------------------------                                           *
+*                                                                      *
+*  Covered Software is provided under this License on an "as is"       *
+*  basis, without warranty of any kind, either expressed, implied, or  *
+*  statutory, including, without limitation, warranties that the       *
+*  Covered Software is free of defects, merchantable, fit for a        *
+*  particular purpose or non-infringing. The entire risk as to the     *
+*  quality and performance of the Covered Software is with You.        *
+*  Should any Covered Software prove defective in any respect, You     *
+*  (not any Contributor) assume the cost of any necessary servicing,   *
+*  repair, or correction. This disclaimer of warranty constitutes an   *
+*  essential part of this License. No use of any Covered Software is   *
+*  authorized under this License except under this disclaimer.         *
+*                                                                      *
+************************************************************************
+
+************************************************************************
+*                                                                      *
+*  7. Limitation of Liability                                          *
+*  --------------------------                                          *
+*                                                                      *
+*  Under no circumstances and under no legal theory, whether tort      *
+*  (including negligence), contract, or otherwise, shall any           *
+*  Contributor, or anyone who distributes Covered Software as          *
+*  permitted above, be liable to You for any direct, indirect,         *
+*  special, incidental, or consequential damages of any character      *
+*  including, without limitation, damages for lost profits, loss of    *
+*  goodwill, work stoppage, computer failure or malfunction, or any    *
+*  and all other commercial damages or losses, even if such party      *
+*  shall have been informed of the possibility of such damages. This   *
+*  limitation of liability shall not apply to liability for death or   *
+*  personal injury resulting from such party's negligence to the       *
+*  extent applicable law prohibits such limitation. Some               *
+*  jurisdictions do not allow the exclusion or limitation of           *
+*  incidental or consequential damages, so this exclusion and          *
+*  limitation may not apply to You.                                    *
+*                                                                      *
+************************************************************************
+
+8. Litigation
+-------------
+
+Any litigation relating to this License may be brought only in the
+courts of a jurisdiction where the defendant maintains its principal
+place of business and such litigation shall be governed by laws of that
+jurisdiction, without reference to its conflict-of-law provisions.
+Nothing in this Section shall prevent a party's ability to bring
+cross-claims or counter-claims.
+
+9. Miscellaneous
+----------------
+
+This License represents the complete agreement concerning the subject
+matter hereof. If any provision of this License is held to be
+unenforceable, such provision shall be reformed only to the extent
+necessary to make it enforceable. Any law or regulation which provides
+that the language of a contract shall be construed against the drafter
+shall not be used to construe this License against a Contributor.
+
+10. Versions of the License
+---------------------------
+
+10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in Section
+10.3, no one other than the license steward has the right to modify or
+publish new versions of this License. Each version will be given a
+distinguishing version number.
+
+10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version
+of the License under which You originally received the Covered Software,
+or under the terms of any subsequent version published by the license
+steward.
+
+10.3. Modified Versions
+
+If you create software not governed by this License, and you want to
+create a new license for such software, you may create and use a
+modified version of this License if you rename the license and remove
+any references to the name of the license steward (except to note that
+such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With
+Secondary Licenses under the terms of this version of the License, the
+notice described in Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+-------------------------------------------
+
+  This Source Code Form is subject to the terms of the Mozilla Public
+  License, v. 2.0. If a copy of the MPL was not distributed with this
+  file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular
+file, then You may include the notice in a location (such as a LICENSE
+file in a relevant directory) where a recipient would be likely to look
+for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+---------------------------------------------------------
+
+  This Source Code Form is "Incompatible With Secondary Licenses", as
+  defined by the Mozilla Public License, v. 2.0.
diff --git a/client/licenses/httpcore-NOTICE.txt b/client/licenses/httpcore-NOTICE.txt
new file mode 100644
index 00000000000..91e5c40c4c6
--- /dev/null
+++ b/client/licenses/httpcore-NOTICE.txt
@@ -0,0 +1,6 @@
+Apache HttpComponents Client
+Copyright 1999-2016 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+

From 56e689e1b36183d9b2a5e214a07d44548fd2faba Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Sat, 4 Jun 2016 01:05:53 +0200
Subject: [PATCH 0645/1311] [TEST] remove unused method

---
 .../java/org/elasticsearch/test/rest/ESRestTestCase.java     | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index 72040863704..c27d3ed5604 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -340,11 +340,6 @@ public abstract class ESRestTestCase extends ESTestCase {
         return restClientSettings(); // default to the same client settings
     }
 
-    /** Returns the addresses the client uses to connect to the test cluster. */
-    protected URL[] getClusterUrls() {
-        return clusterUrls;
-    }
-
     @Before
     public void reset() throws IOException {
         // admin context must be available for @After always, regardless of whether the test was blacklisted

From 05e26a46d736ff5caf6e6d6394e674828283e3d7 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Sat, 4 Jun 2016 01:06:18 +0200
Subject: [PATCH 0646/1311] raise default socket timeout to 10s and default
 connect timeout to 1s

---
 client/src/main/java/org/elasticsearch/client/RestClient.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 3f7e5a18aec..161abbd40a7 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -358,8 +358,8 @@ public final class RestClient implements Closeable {
      * Rest client builder. Helps creating a new {@link RestClient}.
      */
     public static final class Builder {
-        public static final int DEFAULT_CONNECT_TIMEOUT = 500;
-        public static final int DEFAULT_SOCKET_TIMEOUT = 5000;
+        public static final int DEFAULT_CONNECT_TIMEOUT = 1000;
+        public static final int DEFAULT_SOCKET_TIMEOUT = 10000;
         public static final int DEFAULT_MAX_RETRY_TIMEOUT = DEFAULT_SOCKET_TIMEOUT;
         public static final int DEFAULT_CONNECTION_REQUEST_TIMEOUT = 500;
 

From 08f7f79b2e66706daa9ce6eb49674012c68097cb Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 3 Jun 2016 16:32:35 -0400
Subject: [PATCH 0647/1311] Wrap lines at 140 characters (:qa projects)

---
 .../src/main/resources/checkstyle_suppressions.xml   |  8 --------
 .../elasticsearch/bootstrap/ESPolicyUnitTests.java   |  3 ++-
 .../elasticsearch/bootstrap/EvilSecurityTests.java   |  3 ++-
 .../java/org/elasticsearch/tribe/TribeUnitTests.java |  5 ++---
 .../ingest/IngestDocumentMustacheIT.java             | 12 ++++++++----
 .../ingest/IngestMustacheSetProcessorIT.java         |  6 ++++--
 6 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index c77ef0f9d5b..0245cda2fe1 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -1273,14 +1273,6 @@
   <suppress files="plugins[/\\]repository-s3[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]aws[/\\]blobstore[/\\]MockDefaultS3OutputStream.java" checks="LineLength" />
   <suppress files="plugins[/\\]repository-s3[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]s3[/\\]AbstractS3SnapshotRestoreTest.java" checks="LineLength" />
   <suppress files="plugins[/\\]store-smb[/\\]src[/\\]main[/\\]java[/\\]org[/\\]apache[/\\]lucene[/\\]store[/\\]SmbDirectoryWrapper.java" checks="LineLength" />
-  <suppress files="qa[/\\]evil-tests[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]ESPolicyUnitTests.java" checks="LineLength" />
-  <suppress files="qa[/\\]evil-tests[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]EvilSecurityTests.java" checks="LineLength" />
-  <suppress files="qa[/\\]evil-tests[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]cli[/\\]CheckFileCommandTests.java" checks="LineLength" />
-  <suppress files="qa[/\\]evil-tests[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]tribe[/\\]TribeUnitTests.java" checks="LineLength" />
-  <suppress files="qa[/\\]smoke-test-client[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]smoketest[/\\]ESSmokeClientTestCase.java" checks="LineLength" />
-  <suppress files="qa[/\\]smoke-test-ingest-with-all-dependencies[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]CombineProcessorsTests.java" checks="LineLength" />
-  <suppress files="qa[/\\]smoke-test-ingest-with-all-dependencies[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]IngestDocumentMustacheIT.java" checks="LineLength" />
-  <suppress files="qa[/\\]smoke-test-ingest-with-all-dependencies[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]IngestMustacheSetProcessorIT.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]BootstrapForTesting.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]MockInternalClusterInfoService.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]TestShardRouting.java" checks="LineLength" />
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java
index 220d093301c..255dfb3ec5c 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/ESPolicyUnitTests.java
@@ -64,6 +64,7 @@ public class ESPolicyUnitTests extends ESTestCase {
         assumeTrue("test cannot run with security manager", System.getSecurityManager() == null);
         PermissionCollection noPermissions = new Permissions();
         ESPolicy policy = new ESPolicy(noPermissions, Collections.emptyMap(), true);
-        assertFalse(policy.implies(new ProtectionDomain(new CodeSource(null, (Certificate[])null), noPermissions), new FilePermission("foo", "read")));
+        assertFalse(policy.implies(new ProtectionDomain(new CodeSource(null, (Certificate[]) null), noPermissions),
+                new FilePermission("foo", "read")));
     }
 }
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java
index df54cf87fa4..6850f0026e6 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java
@@ -77,7 +77,8 @@ public class EvilSecurityTests extends ESTestCase {
         settingsBuilder.put(Environment.PATH_HOME_SETTING.getKey(), esHome.resolve("home").toString());
         settingsBuilder.put(Environment.PATH_CONF_SETTING.getKey(), esHome.resolve("conf").toString());
         settingsBuilder.put(Environment.PATH_SCRIPTS_SETTING.getKey(), esHome.resolve("scripts").toString());
-        settingsBuilder.putArray(Environment.PATH_DATA_SETTING.getKey(), esHome.resolve("data1").toString(), esHome.resolve("data2").toString());
+        settingsBuilder.putArray(Environment.PATH_DATA_SETTING.getKey(), esHome.resolve("data1").toString(),
+                esHome.resolve("data2").toString());
         settingsBuilder.put(Environment.PATH_SHARED_DATA_SETTING.getKey(), esHome.resolve("custom").toString());
         settingsBuilder.put(Environment.PATH_LOGS_SETTING.getKey(), esHome.resolve("logs").toString());
         settingsBuilder.put(Environment.PIDFILE_SETTING.getKey(), esHome.resolve("test.pid").toString());
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java
index f9cdf5b4f66..4199a5d67cd 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java
@@ -28,7 +28,6 @@ import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.node.Node;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.InternalTestCluster;
@@ -111,8 +110,8 @@ public class TribeUnitTests extends ESTestCase {
                         assertThat(state.getClusterName().value(), equalTo("tribe_node_cluster"));
                         assertThat(state.getNodes().getSize(), equalTo(5));
                         for (DiscoveryNode discoveryNode : state.getNodes()) {
-                            assertThat(discoveryNode.getName(), either(equalTo("tribe1_node")).or(equalTo("tribe2_node")).or(equalTo("tribe_node"))
-                                    .or(equalTo("tribe_node/t1")).or(equalTo("tribe_node/t2")));
+                            assertThat(discoveryNode.getName(), either(equalTo("tribe1_node")).or(equalTo("tribe2_node"))
+                                    .or(equalTo("tribe_node")).or(equalTo("tribe_node/t1")).or(equalTo("tribe_node/t2")));
                         }
                     }
                 });
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java
index c8c50603625..2314e273fc6 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java
@@ -52,10 +52,12 @@ public class IngestDocumentMustacheIT extends AbstractMustacheTestCase {
         innerObject.put("qux", Collections.singletonMap("fubar", "hello qux and fubar"));
         document.put("foo", innerObject);
         IngestDocument ingestDocument = new IngestDocument("index", "type", "id", null, null, null, null, document);
-        ingestDocument.setFieldValue(templateService.compile("field1"), ValueSource.wrap("1 {{foo.bar}} {{foo.baz}} {{foo.qux.fubar}}", templateService));
+        ingestDocument.setFieldValue(templateService.compile("field1"),
+                ValueSource.wrap("1 {{foo.bar}} {{foo.baz}} {{foo.qux.fubar}}", templateService));
         assertThat(ingestDocument.getFieldValue("field1", String.class), equalTo("1 hello bar hello baz hello qux and fubar"));
 
-        ingestDocument.setFieldValue(templateService.compile("field1"), ValueSource.wrap("2 {{_source.foo.bar}} {{_source.foo.baz}} {{_source.foo.qux.fubar}}", templateService));
+        ingestDocument.setFieldValue(templateService.compile("field1"),
+                ValueSource.wrap("2 {{_source.foo.bar}} {{_source.foo.baz}} {{_source.foo.qux.fubar}}", templateService));
         assertThat(ingestDocument.getFieldValue("field1", String.class), equalTo("2 hello bar hello baz hello qux and fubar"));
     }
 
@@ -79,7 +81,9 @@ public class IngestDocumentMustacheIT extends AbstractMustacheTestCase {
         ingestMap.put("timestamp", "bogus_timestamp");
         document.put("_ingest", ingestMap);
         IngestDocument ingestDocument = new IngestDocument("index", "type", "id", null, null, null, null, document);
-        ingestDocument.setFieldValue(templateService.compile("ingest_timestamp"), ValueSource.wrap("{{_ingest.timestamp}} and {{_source._ingest.timestamp}}", templateService));
-        assertThat(ingestDocument.getFieldValue("ingest_timestamp", String.class), equalTo(ingestDocument.getIngestMetadata().get("timestamp") + " and bogus_timestamp"));
+        ingestDocument.setFieldValue(templateService.compile("ingest_timestamp"),
+                ValueSource.wrap("{{_ingest.timestamp}} and {{_source._ingest.timestamp}}", templateService));
+        assertThat(ingestDocument.getFieldValue("ingest_timestamp", String.class),
+                equalTo(ingestDocument.getIngestMetadata().get("timestamp") + " and bogus_timestamp"));
     }
 }
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestMustacheSetProcessorIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestMustacheSetProcessorIT.java
index ed5ad65466b..f088f84d144 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestMustacheSetProcessorIT.java
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestMustacheSetProcessorIT.java
@@ -50,11 +50,13 @@ public class IngestMustacheSetProcessorIT extends AbstractMustacheTestCase {
     }
 
     public void testSetWithTemplates() throws Exception {
-        IngestDocument.MetaData randomMetaData = randomFrom(IngestDocument.MetaData.INDEX, IngestDocument.MetaData.TYPE, IngestDocument.MetaData.ID);
+        IngestDocument.MetaData randomMetaData = randomFrom(IngestDocument.MetaData.INDEX, IngestDocument.MetaData.TYPE,
+                IngestDocument.MetaData.ID);
         Processor processor = createSetProcessor("field{{_type}}", "_value {{" + randomMetaData.getFieldName() + "}}");
         IngestDocument ingestDocument = createIngestDocument(new HashMap<>());
         processor.execute(ingestDocument);
-        assertThat(ingestDocument.getFieldValue("field_type", String.class), Matchers.equalTo("_value " + ingestDocument.getFieldValue(randomMetaData.getFieldName(), String.class)));
+        assertThat(ingestDocument.getFieldValue("field_type", String.class),
+                Matchers.equalTo("_value " + ingestDocument.getFieldValue(randomMetaData.getFieldName(), String.class)));
     }
 
     private SetProcessor createSetProcessor(String fieldName, Object fieldValue) throws Exception {

From 4ca04d6f6cbc819da030ecdea3c3694192b7aa08 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Fri, 3 Jun 2016 16:15:04 +0200
Subject: [PATCH 0648/1311] Close SearchContext if query rewrite failed

 If a query failed to be rewritten and throws an exception, the SearchContext is not properly closed, skewing the ref count on the underlying Store.
---
 .../index/query/WrapperQueryBuilder.java      |   2 +-
 .../action/search/RestMultiSearchAction.java  |   2 +-
 .../elasticsearch/search/SearchService.java   |  14 +--
 .../search/SearchServiceTests.java            | 100 ++++++++++++++++++
 4 files changed, 109 insertions(+), 9 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java
index 4e1eb83272a..7e469d6bbe4 100644
--- a/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java
@@ -126,7 +126,7 @@ public class WrapperQueryBuilder extends AbstractQueryBuilder<WrapperQueryBuilde
         }
         String fieldName = parser.currentName();
         if (! parseContext.getParseFieldMatcher().match(fieldName, QUERY_FIELD)) {
-            throw new ParsingException(parser.getTokenLocation(), "[wrapper] query malformed, expected `query` but was" + fieldName);
+            throw new ParsingException(parser.getTokenLocation(), "[wrapper] query malformed, expected `query` but was " + fieldName);
         }
         parser.nextToken();
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
index 879a1e750b6..8c76525c857 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
@@ -157,7 +157,7 @@ public class RestMultiSearchAction extends BaseRestHandler {
                         Object value = entry.getValue();
                         if ("index".equals(entry.getKey()) || "indices".equals(entry.getKey())) {
                             if (!allowExplicitIndex) {
-                                throw new IllegalArgumentException("explicit index in multi percolate is not allowed");
+                                throw new IllegalArgumentException("explicit index in multi search is not allowed");
                             }
                             searchRequest.indices(nodeStringArrayValue(value));
                         } else if ("type".equals(entry.getKey()) || "types".equals(entry.getKey())) {
diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java
index 2f0ea96c6fc..b4704fe5a7f 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchService.java
@@ -546,14 +546,14 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
                 indexShard, scriptService, bigArrays, threadPool.estimatedTimeInMillisCounter(), parseFieldMatcher,
                 defaultSearchTimeout, fetchPhase);
         SearchContext.setCurrent(context);
-        request.rewrite(context.getQueryShardContext());
-        // reset that we have used nowInMillis from the context since it may
-        // have been rewritten so its no longer in the query and the request can
-        // be cached. If it is still present in the request (e.g. in a range
-        // aggregation) it will still be caught when the aggregation is
-        // evaluated.
-        context.resetNowInMillisUsed();
         try {
+            request.rewrite(context.getQueryShardContext());
+            // reset that we have used nowInMillis from the context since it may
+            // have been rewritten so its no longer in the query and the request can
+            // be cached. If it is still present in the request (e.g. in a range
+            // aggregation) it will still be caught when the aggregation is
+            // evaluated.
+            context.resetNowInMillisUsed();
             if (request.scroll() != null) {
                 context.scrollContext(new ScrollContext());
                 context.scrollContext().scroll = request.scroll();
diff --git a/core/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/core/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index a222ff52722..2ac00b541af 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -19,9 +19,25 @@
 package org.elasticsearch.search;
 
 
+import org.apache.lucene.search.Query;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.query.AbstractQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryRewriteContext;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
+import java.io.IOException;
+import java.util.Collection;
 import java.util.concurrent.ExecutionException;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -35,6 +51,11 @@ public class SearchServiceTests extends ESSingleNodeTestCase {
         return true;
     }
 
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(FailOnRewriteQueryPlugin.class);
+    }
+
     public void testClearOnClose() throws ExecutionException, InterruptedException {
         createIndex("index");
         client().prepareIndex("index", "type", "1").setSource("field", "value").setRefresh(true).get();
@@ -70,4 +91,83 @@ public class SearchServiceTests extends ESSingleNodeTestCase {
         assertAcked(client().admin().indices().prepareDelete("index"));
         assertEquals(0, service.getActiveContexts());
     }
+
+    public void testCloseSearchContextOnRewriteException() {
+        createIndex("index");
+        client().prepareIndex("index", "type", "1").setSource("field", "value").setRefresh(true).get();
+
+        SearchService service = getInstanceFromNode(SearchService.class);
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        IndexService indexService = indicesService.indexServiceSafe(resolveIndex("index"));
+        IndexShard indexShard = indexService.getShard(0);
+
+        final int activeContexts = service.getActiveContexts();
+        final int activeRefs = indexShard.store().refCount();
+        expectThrows(SearchPhaseExecutionException.class, () ->
+                client().prepareSearch("index").setQuery(new FailOnRewriteQueryBuilder()).get());
+        assertEquals(activeContexts, service.getActiveContexts());
+        assertEquals(activeRefs, indexShard.store().refCount());
+    }
+
+    public static class FailOnRewriteQueryPlugin extends Plugin {
+
+        @Override
+        public String name() {
+            return FailOnRewriteQueryPlugin.class.getSimpleName();
+        }
+
+        @Override
+        public String description() {
+            return "This plugin registers a query that always fails at rewrite phase";
+        }
+
+        public void onModule(SearchModule module) {
+            module.registerQuery(FailOnRewriteQueryBuilder::new, parseContext -> {
+                throw new UnsupportedOperationException("No query parser for this plugin");
+            }, new ParseField("fail_on_rewrite_query"));
+        }
+    }
+
+    public static class FailOnRewriteQueryBuilder extends AbstractQueryBuilder<FailOnRewriteQueryBuilder> {
+
+        public FailOnRewriteQueryBuilder(StreamInput in) throws IOException {
+            super(in);
+        }
+
+        public FailOnRewriteQueryBuilder() {
+        }
+
+        @Override
+        protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOException {
+            throw new IllegalStateException("Fail on rewrite phase");
+        }
+
+        @Override
+        protected void doWriteTo(StreamOutput out) throws IOException {
+        }
+
+        @Override
+        protected void doXContent(XContentBuilder builder, Params params) throws IOException {
+        }
+
+        @Override
+        protected Query doToQuery(QueryShardContext context) throws IOException {
+            return null;
+        }
+
+        @Override
+        protected boolean doEquals(FailOnRewriteQueryBuilder other) {
+            return false;
+        }
+
+        @Override
+        protected int doHashCode() {
+            return 0;
+        }
+
+        @Override
+        public String getWriteableName() {
+            return null;
+        }
+    }
 }

From a1172d816cc3d8404347b8770c162b4f938fc486 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Fri, 27 May 2016 16:34:53 +0200
Subject: [PATCH 0649/1311] Implement ctx.op = "delete" on _update_by_query and
 _reindex

closes #18043
---
 docs/reference/docs/reindex.asciidoc          |  20 +++
 docs/reference/docs/update-by-query.asciidoc  |  25 ++-
 .../AbstractAsyncBulkIndexByScrollAction.java | 145 ++++++++++++++-
 .../AbstractBaseReindexRestHandler.java       |   3 +-
 .../index/reindex/BulkByScrollTask.java       |  10 +-
 .../reindex/RestDeleteByQueryAction.java      |   2 +-
 .../index/reindex/RestReindexAction.java      |   2 +-
 .../reindex/RestUpdateByQueryAction.java      |   2 +-
 ...BulkIndexByScrollActionScriptTestCase.java |  30 +++-
 .../rest-api-spec/test/reindex/10_script.yaml |  92 ++++++++++
 .../test/update_by_query/10_script.yaml       | 168 ++++++++++++++++++
 11 files changed, 468 insertions(+), 31 deletions(-)

diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index 0bfb44b2aaf..3311049c699 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -30,6 +30,7 @@ That will return something like this:
   "timed_out": false,
   "created": 120,
   "updated": 0,
+  "deleted": 0,
   "batches": 1,
   "version_conflicts": 0,
   "noops": 0,
@@ -244,6 +245,24 @@ POST _reindex
 // CONSOLE
 // TEST[setup:twitter]
 
+Just as in `_update_by_query`, you can set `ctx.op` to change the
+operation that is executed on the destination index:
+
+`noop`::
+
+Set `ctx.op = "noop"` if your script decides that the document doesn't have
+to be indexed in the destination index. This no operation will be reported
+in the `noop` counter in the <<docs-reindex-response-body, response body>>.
+
+`delete`::
+
+Set `ctx.op = "delete"` if your script decides that the document must be
+ deleted from the destination index. The deletion will be reported in the
+ `deleted` counter in the <<docs-reindex-response-body, response body>>.
+
+Setting `ctx.op` to anything else is an error. Setting any
+other field in `ctx` is an error.
+
 Think of the possibilities! Just be careful! With great power.... You can
 change:
 
@@ -377,6 +396,7 @@ starting the next set. This is "bursty" instead of "smooth". The default is
 `unlimited` which is also the only non-number value that it accepts.
 
 [float]
+[[docs-reindex-response-body]]
 === Response body
 
 The JSON response looks like this:
diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc
index ac4da4251be..f3a147e36c1 100644
--- a/docs/reference/docs/update-by-query.asciidoc
+++ b/docs/reference/docs/update-by-query.asciidoc
@@ -23,6 +23,7 @@ That will return something like this:
   "took" : 147,
   "timed_out": false,
   "updated": 120,
+  "deleted": 0,
   "batches": 1,
   "version_conflicts": 0,
   "noops": 0,
@@ -115,11 +116,24 @@ POST twitter/_update_by_query
 // CONSOLE
 // TEST[setup:twitter]
 
-Just as in <<docs-update,Update API>> you can set `ctx.op = "noop"` if
-your script decides that it doesn't have to make any changes. That will cause
-`_update_by_query` to omit that document from its updates. Setting `ctx.op` to
-anything else is an error. If you want to delete by a query you can use the
-<<docs-delete-by-query,Delete By Query API>> instead. Setting any
+Just as in <<docs-update,Update API>> you can set `ctx.op` to change the
+operation that is executed:
+
+
+`noop`::
+
+Set `ctx.op = "noop"` if your script decides that it doesn't have to make any
+changes. That will cause `_update_by_query` to omit that document from its updates.
+ This no operation will be reported in the `noop` counter in the
+<<docs-update-by-query-response-body, response body>>.
+
+`delete`::
+
+Set `ctx.op = "delete"` if your script decides that the document must be
+ deleted. The deletion will be reported in the `deleted` counter in the
+<<docs-update-by-query-response-body, response body>>.
+
+Setting `ctx.op` to anything else is an error. Setting any
 other field in `ctx` is an error.
 
 Note that we stopped specifying `conflicts=proceed`. In this case we want a
@@ -212,6 +226,7 @@ starting the next set. This is "bursty" instead of "smooth". The default is
 `unlimited` which is also the only non-number value that it accepts.
 
 [float]
+[[docs-update-by-query-response-body]]
 === Response body
 
 The JSON response looks like this:
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java
index 0e3f3678851..6cb62207506 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java
@@ -48,7 +48,9 @@ import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.threadpool.ThreadPool;
 
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.function.BiFunction;
@@ -173,18 +175,30 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
 
         void setIndex(String index);
 
+        String getIndex();
+
         void setType(String type);
 
+        String getType();
+
         void setId(String id);
 
+        String getId();
+
         void setVersion(long version);
 
+        long getVersion();
+
         void setVersionType(VersionType versionType);
 
         void setParent(String parent);
 
+        String getParent();
+
         void setRouting(String routing);
 
+        String getRouting();
+
         void setTimestamp(String timestamp);
 
         void setTtl(Long ttl);
@@ -212,21 +226,41 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
             request.index(index);
         }
 
+        @Override
+        public String getIndex() {
+            return request.index();
+        }
+
         @Override
         public void setType(String type) {
             request.type(type);
         }
 
+        @Override
+        public String getType() {
+            return request.type();
+        }
+
         @Override
         public void setId(String id) {
             request.id(id);
         }
 
+        @Override
+        public String getId() {
+            return request.id();
+        }
+
         @Override
         public void setVersion(long version) {
             request.version(version);
         }
 
+        @Override
+        public long getVersion() {
+            return request.version();
+        }
+
         @Override
         public void setVersionType(VersionType versionType) {
             request.versionType(versionType);
@@ -237,11 +271,21 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
             request.parent(parent);
         }
 
+        @Override
+        public String getParent() {
+            return request.parent();
+        }
+
         @Override
         public void setRouting(String routing) {
             request.routing(routing);
         }
 
+        @Override
+        public String getRouting() {
+            return request.routing();
+        }
+
         @Override
         public void setTimestamp(String timestamp) {
             request.timestamp(timestamp);
@@ -295,21 +339,41 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
             request.index(index);
         }
 
+        @Override
+        public String getIndex() {
+            return request.index();
+        }
+
         @Override
         public void setType(String type) {
             request.type(type);
         }
 
+        @Override
+        public String getType() {
+            return request.type();
+        }
+
         @Override
         public void setId(String id) {
             request.id(id);
         }
 
+        @Override
+        public String getId() {
+            return request.id();
+        }
+
         @Override
         public void setVersion(long version) {
             request.version(version);
         }
 
+        @Override
+        public long getVersion() {
+            return request.version();
+        }
+
         @Override
         public void setVersionType(VersionType versionType) {
             request.versionType(versionType);
@@ -320,11 +384,21 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
             request.parent(parent);
         }
 
+        @Override
+        public String getParent() {
+            return request.parent();
+        }
+
         @Override
         public void setRouting(String routing) {
             request.routing(routing);
         }
 
+        @Override
+        public String getRouting() {
+            return request.routing();
+        }
+
         @Override
         public void setTimestamp(String timestamp) {
             throw new UnsupportedOperationException("unable to set [timestamp] on action request [" + request.getClass() + "]");
@@ -409,21 +483,17 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
             Long oldTTL = fieldValue(doc, TTLFieldMapper.NAME);
             context.put(TTLFieldMapper.NAME, oldTTL);
             context.put(SourceFieldMapper.NAME, request.getSource());
-            context.put("op", "update");
+
+            OpType oldOpType = OpType.INDEX;
+            context.put("op", oldOpType.toString());
+
             executable.setNextVar("ctx", context);
             executable.run();
 
             Map<String, Object> resultCtx = (Map<String, Object>) executable.unwrap(context);
             String newOp = (String) resultCtx.remove("op");
             if (newOp == null) {
-                throw new IllegalArgumentException("Script cleared op!");
-            }
-            if ("noop".equals(newOp)) {
-                task.countNoop();
-                return null;
-            }
-            if (false == "update".equals(newOp)) {
-                throw new IllegalArgumentException("Invalid op [" + newOp + ']');
+                throw new IllegalArgumentException("Script cleared operation type");
             }
 
             /*
@@ -468,12 +538,35 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
             if (false == Objects.equals(oldTTL, newValue)) {
                 scriptChangedTTL(request, newValue);
             }
+
+            OpType newOpType = OpType.fromString(newOp);
+            if (newOpType !=  oldOpType) {
+                return scriptChangedOpType(request, oldOpType, newOpType);
+            }
+
             if (false == context.isEmpty()) {
                 throw new IllegalArgumentException("Invalid fields added to context [" + String.join(",", context.keySet()) + ']');
             }
             return request;
         }
 
+        protected RequestWrapper<?> scriptChangedOpType(RequestWrapper<?> request, OpType oldOpType, OpType newOpType) {
+            switch (newOpType) {
+            case NOOP:
+                task.countNoop();
+                return null;
+            case DELETE:
+                RequestWrapper<DeleteRequest> delete = wrap(new DeleteRequest(request.getIndex(), request.getType(), request.getId()));
+                delete.setVersion(request.getVersion());
+                delete.setVersionType(VersionType.INTERNAL);
+                delete.setParent(request.getParent());
+                delete.setRouting(request.getRouting());
+                return delete;
+            default:
+                throw new IllegalArgumentException("Unsupported operation type change from [" + oldOpType + "] to [" + newOpType + "]");
+            }
+        }
+
         protected abstract void scriptChangedIndex(RequestWrapper<?> request, Object to);
 
         protected abstract void scriptChangedType(RequestWrapper<?> request, Object to);
@@ -489,5 +582,39 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
         protected abstract void scriptChangedTimestamp(RequestWrapper<?> request, Object to);
 
         protected abstract void scriptChangedTTL(RequestWrapper<?> request, Object to);
+
+    }
+
+    public enum OpType {
+
+        NOOP("noop"),
+        INDEX("index"),
+        DELETE("delete");
+
+        private final String id;
+
+        OpType(String id) {
+            this.id = id;
+        }
+
+        public static OpType fromString(String opType) {
+            String lowerOpType = opType.toLowerCase(Locale.ROOT);
+            switch (lowerOpType) {
+                case "noop":
+                    return OpType.NOOP;
+                case "index":
+                    return OpType.INDEX;
+                case "delete":
+                    return OpType.DELETE;
+                default:
+                    throw new IllegalArgumentException("Operation type [" + lowerOpType + "] not allowed, only " +
+                            Arrays.toString(values()) + " are allowed");
+            }
+        }
+
+        @Override
+        public String toString() {
+            return id.toLowerCase(Locale.ROOT);
+        }
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
index e78a6a9c350..3aea4dbce8a 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
@@ -64,7 +64,7 @@ public abstract class AbstractBaseReindexRestHandler<
     }
 
     protected void handleRequest(RestRequest request, RestChannel channel,
-                                 boolean includeCreated, boolean includeUpdated, boolean includeDeleted) throws IOException {
+                                 boolean includeCreated, boolean includeUpdated) throws IOException {
 
         // Build the internal request
         Request internal = setCommonOptions(request, buildRequest(request));
@@ -74,7 +74,6 @@ public abstract class AbstractBaseReindexRestHandler<
             Map<String, String> params = new HashMap<>();
             params.put(BulkByScrollTask.Status.INCLUDE_CREATED, Boolean.toString(includeCreated));
             params.put(BulkByScrollTask.Status.INCLUDE_UPDATED, Boolean.toString(includeUpdated));
-            params.put(BulkByScrollTask.Status.INCLUDE_DELETED, Boolean.toString(includeDeleted));
 
             action.execute(internal, new BulkIndexByScrollResponseContentListener<>(channel, params));
             return;
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
index 40ee492b4a9..5197cdae079 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
@@ -126,12 +126,6 @@ public class BulkByScrollTask extends CancellableTask {
          */
         public static final String INCLUDE_UPDATED = "include_updated";
 
-        /**
-         * XContent param name to indicate if "deleted" count must be included
-         * in the response.
-         */
-        public static final String INCLUDE_DELETED = "include_deleted";
-
         private final long total;
         private final long updated;
         private final long created;
@@ -213,9 +207,7 @@ public class BulkByScrollTask extends CancellableTask {
             if (params.paramAsBoolean(INCLUDE_CREATED, true)) {
                 builder.field("created", created);
             }
-            if (params.paramAsBoolean(INCLUDE_DELETED, true)) {
-                builder.field("deleted", deleted);
-            }
+            builder.field("deleted", deleted);
             builder.field("batches", batches);
             builder.field("version_conflicts", versionConflicts);
             builder.field("noops", noops);
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java
index 4750fe22313..bb894584c8b 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java
@@ -55,7 +55,7 @@ public class RestDeleteByQueryAction extends AbstractBulkByQueryRestHandler<Dele
         if (false == request.hasContent()) {
             throw new ElasticsearchException("_delete_by_query requires a request body");
         }
-        handleRequest(request, channel, false, false, true);
+        handleRequest(request, channel, false, false);
     }
 
     @Override
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
index 267994672d4..22fcd390430 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
@@ -115,7 +115,7 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexReq
         if (false == request.hasContent()) {
             throw new ElasticsearchException("_reindex requires a request body");
         }
-        handleRequest(request, channel, true, true, false);
+        handleRequest(request, channel, true, true);
     }
 
     @Override
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
index f7dbbf893a8..8eb0e30fe38 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
@@ -52,7 +52,7 @@ public class RestUpdateByQueryAction extends AbstractBulkByQueryRestHandler<Upda
 
     @Override
     protected void handleRequest(RestRequest request, RestChannel channel, Client client) throws Exception {
-        handleRequest(request, channel, false, true, false);
+        handleRequest(request, channel, false, true);
     }
 
     @Override
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionScriptTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionScriptTestCase.java
index 27a8a42d5e6..0318e4ddb01 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionScriptTestCase.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionScriptTestCase.java
@@ -19,9 +19,13 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.reindex.AbstractAsyncBulkIndexByScrollAction.OpType;
+import org.elasticsearch.index.reindex.AbstractAsyncBulkIndexByScrollAction.RequestWrapper;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
@@ -56,7 +60,8 @@ public abstract class AbstractAsyncBulkIndexByScrollActionScriptTestCase<
         scriptService = mock(ScriptService.class);
     }
 
-    protected IndexRequest applyScript(Consumer<Map<String, Object>> scriptBody) {
+    @SuppressWarnings("unchecked")
+    protected <T extends ActionRequest<?>> T applyScript(Consumer<Map<String, Object>> scriptBody) {
         IndexRequest index = new IndexRequest("index", "type", "1").source(singletonMap("foo", "bar"));
         Map<String, SearchHitField> fields = new HashMap<>();
         InternalSearchHit doc = new InternalSearchHit(0, "id", new Text("type"), fields);
@@ -66,8 +71,8 @@ public abstract class AbstractAsyncBulkIndexByScrollActionScriptTestCase<
         when(scriptService.executable(any(CompiledScript.class), Matchers.<Map<String, Object>>any()))
                 .thenReturn(executableScript);
         AbstractAsyncBulkIndexByScrollAction<Request> action = action(scriptService, request().setScript(EMPTY_SCRIPT));
-        action.buildScriptApplier().apply(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc);
-        return index;
+        RequestWrapper<?> result = action.buildScriptApplier().apply(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc);
+        return (result != null) ? (T) result.self() : null;
     }
 
     public void testScriptAddingJunkToCtxIsError() {
@@ -88,5 +93,24 @@ public abstract class AbstractAsyncBulkIndexByScrollActionScriptTestCase<
         assertEquals("cat", index.sourceAsMap().get("bar"));
     }
 
+    public void testSetOpTypeNoop() throws Exception {
+        assertThat(task.getStatus().getNoops(), equalTo(0L));
+        assertNull(applyScript((Map<String, Object> ctx) -> ctx.put("op", OpType.NOOP.toString())));
+        assertThat(task.getStatus().getNoops(), equalTo(1L));
+    }
+
+    public void testSetOpTypeDelete() throws Exception {
+        DeleteRequest delete = applyScript((Map<String, Object> ctx) -> ctx.put("op", OpType.DELETE.toString()));
+        assertThat(delete.index(), equalTo("index"));
+        assertThat(delete.type(), equalTo("type"));
+        assertThat(delete.id(), equalTo("1"));
+    }
+
+    public void testSetOpTypeUnknown() throws Exception {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> applyScript((Map<String, Object> ctx) -> ctx.put("op", "unknown")));
+        assertThat(e.getMessage(), equalTo("Operation type [unknown] not allowed, only [noop, index, delete] are allowed"));
+    }
+
     protected abstract AbstractAsyncBulkIndexByScrollAction<Request> action(ScriptService scriptService, Request request);
 }
diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/10_script.yaml b/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/10_script.yaml
index aa553a5c9dc..783838d7f8e 100644
--- a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/10_script.yaml
+++ b/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/10_script.yaml
@@ -284,6 +284,11 @@
               user: notfoo
   - match: { hits.total: 0 }
 
+  - do:
+      count:
+        index: new_twitter
+  - match: {count: 1}
+
 ---
 "Noop all docs":
   - do:
@@ -313,6 +318,11 @@
   - match: {updated: 0}
   - match: {noops: 2}
 
+  - do:
+      indices.exists:
+        index: new_twitter
+  - is_false: ''
+
 ---
 "Set version to null to force an update":
   - do:
@@ -443,3 +453,85 @@
             match:
               user: another
   - match: { hits.total: 1 }
+
+---
+"Reindex all docs with one doc deletion":
+  # Source index
+  - do:
+      index:
+        index:  index1
+        type:   type1
+        id:     1
+        body:   { "lang": "en", "id": 123 }
+  - do:
+      index:
+        index:  index1
+        type:   type1
+        id:     2
+        body:   { "lang": "en", "id": 456 }
+  - do:
+      index:
+        index:  index1
+        type:   type1
+        id:     3
+        body:   { "lang": "fr", "id": 789 }
+  # Destination index
+  - do:
+      index:
+        index:  index2
+        type:   type2
+        id:     fr_789
+        body:   { "lang": "fr", "id": 789 }
+  - do:
+      index:
+        index:  index2
+        type:   type2
+        id:     en_123
+        body:   { "lang": "en", "id": 123 }
+  - do:
+      indices.refresh: {}
+
+  # Reindex all documents from "index1" into "index2", changing their type
+  # to "type2" and their id to the concatened lang+id fields,
+  # trashing all non-english pre existing ones
+  - do:
+      reindex:
+        refresh: true
+        body:
+          source:
+            index: index1
+          dest:
+            index: index2
+            type: type2
+          script:
+            inline: "ctx._id = ctx._source.lang + '_' + ctx._source.id;
+                     if (ctx._source.lang != \"en\" ) {ctx.op = 'delete'}"
+  - match: {created: 1}
+  - match: {noops: 0}
+  - match: {updated: 1}
+  - match: {deleted: 1}
+
+  - do:
+      mget:
+        body:
+          docs:
+            - { _index: index2, _type: type2, _id: en_123}
+            - { _index: index2, _type: type2, _id: en_456}
+            - { _index: index2, _type: type2, _id: fr_789}
+
+  - is_true: docs.0.found
+  - match:  { docs.0._index:     index2      }
+  - match:  { docs.0._type:      type2       }
+  - match:  { docs.0._id:        en_123      }
+  - match:  { docs.0._version:   2           }
+
+  - is_true: docs.1.found
+  - match:  { docs.1._index:     index2      }
+  - match:  { docs.1._type:      type2       }
+  - match:  { docs.1._id:        en_456      }
+  - match:  { docs.1._version:   1           }
+
+  - is_false: docs.2.found
+  - match:  { docs.2._index:     index2      }
+  - match:  { docs.2._type:      type2       }
+  - match:  { docs.2._id:        fr_789      }
diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml b/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml
index 54a79ac1e32..e4fef86b1d1 100644
--- a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml
+++ b/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml
@@ -138,3 +138,171 @@
         body:
           script:
             inline: ctx._id = "stuff"
+
+---
+"Update all docs with one doc deletion":
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     1
+        body:   { "level": 9, "last_updated": "2016-01-01T12:10:30Z" }
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     2
+        body:   { "level": 10, "last_updated": "2016-01-01T12:10:30Z" }
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     3
+        body:   { "level": 11, "last_updated": "2016-01-01T12:10:30Z" }
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     4
+        body:   { "level": 12, "last_updated": "2016-01-01T12:10:30Z" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      update_by_query:
+        refresh: true
+        index:   twitter
+        body:
+          script:
+            inline: if (ctx._source.level != 11) {ctx._source.last_updated = "2016-01-02T00:00:00Z"} else {ctx.op = "delete"}
+  - match: {updated: 3}
+  - match: {deleted: 1}
+  - match: {noops: 0}
+
+  - do:
+      search:
+        index: twitter
+        body:
+          query:
+            match:
+              last_updated: "2016-01-02T00:00:00Z"
+  - match: { hits.total: 3 }
+
+  - do:
+      search:
+        index: twitter
+        body:
+          query:
+            term:
+              level: 11
+  - match: { hits.total: 0 }
+
+  - do:
+      count:
+        index: twitter
+  - match: {count: 3}
+
+---
+"Update all docs with one deletion and one noop":
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     1
+        body:   { "level": 9, "last_updated": "2016-01-01T12:10:30Z" }
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     2
+        body:   { "level": 10, "last_updated": "2016-01-01T12:10:30Z" }
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     3
+        body:   { "level": 11, "last_updated": "2016-01-01T12:10:30Z" }
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     4
+        body:   { "level": 12, "last_updated": "2016-01-01T12:10:30Z" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      update_by_query:
+        refresh: true
+        index:   twitter
+        body:
+          script:
+            inline: "switch (ctx._source.level % 3) {
+                      case 0:
+                        ctx._source.last_updated = \"2016-01-02T00:00:00Z\";
+                        break;
+                      case 1:
+                        ctx.op = \"noop\";
+                        break;
+                      case 2:
+                        ctx.op = \"delete\";
+                        break;
+                    }"
+  - match: {updated: 2}
+  - match: {deleted: 1}
+  - match: {noops: 1}
+
+  - do:
+      search:
+        index: twitter
+        body:
+          query:
+            match:
+              last_updated: "2016-01-02T00:00:00Z"
+  - match: { hits.total: 2 }
+
+  - do:
+      search:
+        index: twitter
+        body:
+          query:
+            match:
+              last_updated: "2016-01-01T12:10:30Z"
+  - match: { hits.total: 1 }
+
+  - do:
+      search:
+        index: twitter
+        body:
+          query:
+            term:
+              level: 11
+  - match: { hits.total: 0 }
+
+---
+"Set unsupported operation type":
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     1
+        body:   { "user": "kimchy" }
+  - do:
+      index:
+        index:  twitter
+        type:   tweet
+        id:     2
+        body:   { "user": "foo" }
+  - do:
+      indices.refresh: {}
+
+  - do:
+      catch: request
+      update_by_query:
+        refresh: true
+        index:   twitter
+        body:
+          script:
+            inline: if (ctx._source.user == "kimchy") {ctx.op = "update"} else {ctx.op = "junk"}
+
+  - match: { error.reason: 'Operation type [junk] not allowed, only [noop, index, delete] are allowed' }

From 371c73e14023917f3892090d2d3d7ff9ac1216b9 Mon Sep 17 00:00:00 2001
From: Nicholas Knize <nknize@gmail.com>
Date: Thu, 2 Jun 2016 16:30:50 -0500
Subject: [PATCH 0650/1311] refactor matrix agg documentation from modules to
 main agg section

---
 docs/reference/aggregations.asciidoc                       | 7 +++++++
 .../matrix.asciidoc}                                       | 4 ++--
 .../matrix/stats-aggregation.asciidoc}                     | 2 +-
 docs/reference/modules.asciidoc                            | 6 ------
 4 files changed, 10 insertions(+), 9 deletions(-)
 rename docs/reference/{modules/aggregations-matrix.asciidoc => aggregations/matrix.asciidoc} (78%)
 rename docs/reference/{modules/aggregations/matrix/stats.asciidoc => aggregations/matrix/stats-aggregation.asciidoc} (98%)

diff --git a/docs/reference/aggregations.asciidoc b/docs/reference/aggregations.asciidoc
index f3f24e8704c..ada134238bb 100644
--- a/docs/reference/aggregations.asciidoc
+++ b/docs/reference/aggregations.asciidoc
@@ -23,6 +23,11 @@ it is often easier to break them into three main families:
 <<search-aggregations-metrics, _Metric_>>::
 				Aggregations that keep track and compute metrics over a set of documents.
 
+<<search-aggregations-matrix, _Matrix_>>::
+                A family of aggregations that operate on multiple fields and produce a matrix result based on the
+                values extracted from the requested document fields. Unlike metric and bucket aggregations, this
+                aggregation family does not yet support scripting.
+
 <<search-aggregations-pipeline, _Pipeline_>>::
 				Aggregations that aggregate the output of other aggregations and their associated metrics
 
@@ -100,4 +105,6 @@ include::aggregations/bucket.asciidoc[]
 
 include::aggregations/pipeline.asciidoc[]
 
+include::aggregations/matrix.asciidoc[]
+
 include::aggregations/misc.asciidoc[]
diff --git a/docs/reference/modules/aggregations-matrix.asciidoc b/docs/reference/aggregations/matrix.asciidoc
similarity index 78%
rename from docs/reference/modules/aggregations-matrix.asciidoc
rename to docs/reference/aggregations/matrix.asciidoc
index e8f741e7df6..e0d3d489ce1 100644
--- a/docs/reference/modules/aggregations-matrix.asciidoc
+++ b/docs/reference/aggregations/matrix.asciidoc
@@ -1,4 +1,4 @@
-[[modules-aggregations-matrix]]
+[[search-aggregations-matrix]]
 == Matrix Aggregations
 
 experimental[]
@@ -6,4 +6,4 @@ experimental[]
 The aggregations in this family operate on multiple fields and produce a matrix result based on the values extracted from
 the requested document fields. Unlike metric and bucket aggregations, this aggregation family does not yet support scripting.
 
-include::aggregations/matrix/stats.asciidoc[]
+include::matrix/stats-aggregation.asciidoc[]
\ No newline at end of file
diff --git a/docs/reference/modules/aggregations/matrix/stats.asciidoc b/docs/reference/aggregations/matrix/stats-aggregation.asciidoc
similarity index 98%
rename from docs/reference/modules/aggregations/matrix/stats.asciidoc
rename to docs/reference/aggregations/matrix/stats-aggregation.asciidoc
index 2649ecd5fe7..8dafb252f08 100644
--- a/docs/reference/modules/aggregations/matrix/stats.asciidoc
+++ b/docs/reference/aggregations/matrix/stats-aggregation.asciidoc
@@ -1,4 +1,4 @@
-[[modules-matrix-aggregations-stats]]
+[[search-aggregations-matrix-stats-aggregation]]
 === Matrix Stats
 
 The `matrix_stats` aggregation is a numeric aggregation that computes the following statistics over a set of document fields:
diff --git a/docs/reference/modules.asciidoc b/docs/reference/modules.asciidoc
index 01b0edd0676..5a39cdfd790 100644
--- a/docs/reference/modules.asciidoc
+++ b/docs/reference/modules.asciidoc
@@ -18,10 +18,6 @@ These settings can be dynamically updated on a live cluster with the
 
 The modules in this section are:
 
-<<modules-aggregations-matrix,Matrix Aggregations>>::
-
-    A family of aggregations that operate on multiple document fields and produce a matrix as output.
-
 <<modules-cluster,Cluster-level routing and shard allocation>>::
 
     Settings to control where, when, and how shards are allocated to nodes.
@@ -84,8 +80,6 @@ The modules in this section are:
 --
 
 
-include::modules/aggregations-matrix.asciidoc[]
-
 include::modules/cluster.asciidoc[]
 
 include::modules/discovery.asciidoc[]

From a461dd84d22186774beaa4fbe931ea3ebf49bd04 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Mon, 6 Jun 2016 15:02:52 +0200
Subject: [PATCH 0651/1311] Build: add hamcrest and securemock to
 version.properties

---
 buildSrc/version.properties | 2 ++
 client-sniffer/build.gradle | 4 ++--
 client/build.gradle         | 4 ++--
 test/framework/build.gradle | 4 ++--
 4 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index d9e3908df22..5aaaca745fa 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -17,3 +17,5 @@ httpclient        = 4.5.2
 httpcore          = 4.4.4
 commonslogging    = 1.1.3
 commonscodec      = 1.10
+hamcrest          = 1.3
+securemock        = 1.2
\ No newline at end of file
diff --git a/client-sniffer/build.gradle b/client-sniffer/build.gradle
index b0f8e1e22e0..d561e007184 100644
--- a/client-sniffer/build.gradle
+++ b/client-sniffer/build.gradle
@@ -35,11 +35,11 @@ dependencies {
 
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
-  testCompile "org.hamcrest:hamcrest-all:1.3"
+  testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
   testCompile "org.apache.lucene:lucene-test-framework:${versions.lucene}"
   testCompile "org.apache.lucene:lucene-core:${versions.lucene}"
   testCompile "org.apache.lucene:lucene-codecs:${versions.lucene}"
-  testCompile "org.elasticsearch:securemock:1.2"
+  testCompile "org.elasticsearch:securemock:${versions.securemock}"
 }
 
 //TODO compiling from 1.8 with target 1.7 and source 1.7 is best effort, not enough to ensure we are java 7 compatible
diff --git a/client/build.gradle b/client/build.gradle
index 4c4a56d7731..2c82fd74569 100644
--- a/client/build.gradle
+++ b/client/build.gradle
@@ -33,11 +33,11 @@ dependencies {
 
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
-  testCompile "org.hamcrest:hamcrest-all:1.3"
+  testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
   testCompile "org.apache.lucene:lucene-test-framework:${versions.lucene}"
   testCompile "org.apache.lucene:lucene-core:${versions.lucene}"
   testCompile "org.apache.lucene:lucene-codecs:${versions.lucene}"
-  testCompile "org.elasticsearch:securemock:1.2"
+  testCompile "org.elasticsearch:securemock:${versions.securemock}"
 }
 
 //TODO compiling from 1.8 with target 1.7 and source 1.7 is best effort, not enough to ensure we are java 7 compatible
diff --git a/test/framework/build.gradle b/test/framework/build.gradle
index 078511611fe..6ddcb7598a1 100644
--- a/test/framework/build.gradle
+++ b/test/framework/build.gradle
@@ -23,7 +23,7 @@ dependencies {
   compile "org.elasticsearch:elasticsearch:${version}"
   compile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   compile "junit:junit:${versions.junit}"
-  compile "org.hamcrest:hamcrest-all:1.3"
+  compile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
   compile "org.apache.lucene:lucene-test-framework:${versions.lucene}"
   compile "org.apache.lucene:lucene-codecs:${versions.lucene}"
   compile "org.elasticsearch:client:${version}"
@@ -31,7 +31,7 @@ dependencies {
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
-  compile "org.elasticsearch:securemock:1.2"
+  compile "org.elasticsearch:securemock:${versions.securemock}"
 }
 
 compileJava.options.compilerArgs << '-Xlint:-cast,-rawtypes,-try,-unchecked'

From 467dfbda0ceab516091983fdb7b369c889040081 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Mon, 6 Jun 2016 15:12:13 +0200
Subject: [PATCH 0652/1311] use TimeUnit and long to keep track of time

---
 .../client/sniff/HostsSniffer.java            |  2 +-
 .../elasticsearch/client/sniff/Sniffer.java   | 24 +++++++++----------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
index cf49cfbe68a..4af9d205589 100644
--- a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
+++ b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
@@ -50,7 +50,7 @@ public class HostsSniffer {
     private final String scheme;
     private final JsonFactory jsonFactory;
 
-    public HostsSniffer(RestClient restClient, int sniffRequestTimeout, String scheme) {
+    public HostsSniffer(RestClient restClient, long sniffRequestTimeout, String scheme) {
         this.restClient = restClient;
         this.sniffRequestParams = Collections.<String, String>singletonMap("timeout", sniffRequestTimeout + "ms");
         this.scheme = scheme;
diff --git a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
index fa44ae7d810..2cd146bd80d 100644
--- a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
+++ b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
@@ -50,8 +50,8 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
     private final boolean sniffOnFailure;
     private final Task task;
 
-    private Sniffer(RestClient restClient, int sniffRequestTimeout, String scheme, int sniffInterval,
-                   boolean sniffOnFailure, int sniffAfterFailureDelay) {
+    private Sniffer(RestClient restClient, long sniffRequestTimeout, String scheme, long sniffInterval,
+                   boolean sniffOnFailure, long sniffAfterFailureDelay) {
         HostsSniffer hostsSniffer = new HostsSniffer(restClient, sniffRequestTimeout, scheme);
         this.task = new Task(hostsSniffer, restClient, sniffInterval, sniffAfterFailureDelay);
         this.sniffOnFailure = sniffOnFailure;
@@ -75,14 +75,14 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
         private final HostsSniffer hostsSniffer;
         private final RestClient restClient;
 
-        private final int sniffInterval;
-        private final int sniffAfterFailureDelay;
+        private final long sniffInterval;
+        private final long sniffAfterFailureDelay;
         private final ScheduledExecutorService scheduledExecutorService;
         private final AtomicBoolean running = new AtomicBoolean(false);
-        private volatile int nextSniffDelay;
+        private volatile long nextSniffDelay;
         private volatile ScheduledFuture<?> scheduledFuture;
 
-        private Task(HostsSniffer hostsSniffer, RestClient restClient, int sniffInterval, int sniffAfterFailureDelay) {
+        private Task(HostsSniffer hostsSniffer, RestClient restClient, long sniffInterval, long sniffAfterFailureDelay) {
             this.hostsSniffer = hostsSniffer;
             this.restClient = restClient;
             this.sniffInterval = sniffInterval;
@@ -153,14 +153,14 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
      * Sniffer builder. Helps creating a new {@link Sniffer}.
      */
     public static final class Builder {
-        public static final int DEFAULT_SNIFF_INTERVAL = 60000 * 5; //5 minutes
-        public static final int DEFAULT_SNIFF_AFTER_FAILURE_DELAY = 60000; //1 minute
-        public static final int DEFAULT_SNIFF_REQUEST_TIMEOUT = 1000; //1 second
+        public static final long DEFAULT_SNIFF_INTERVAL = TimeUnit.MINUTES.toMillis(5);
+        public static final long DEFAULT_SNIFF_AFTER_FAILURE_DELAY = TimeUnit.MINUTES.toMillis(1);
+        public static final long DEFAULT_SNIFF_REQUEST_TIMEOUT = TimeUnit.SECONDS.toMillis(1);
 
-        private int sniffRequestTimeout = DEFAULT_SNIFF_REQUEST_TIMEOUT;
-        private int sniffInterval = DEFAULT_SNIFF_INTERVAL;
+        private long sniffRequestTimeout = DEFAULT_SNIFF_REQUEST_TIMEOUT;
+        private long sniffInterval = DEFAULT_SNIFF_INTERVAL;
         private boolean sniffOnFailure = true;
-        private int sniffAfterFailureDelay = DEFAULT_SNIFF_AFTER_FAILURE_DELAY;
+        private long sniffAfterFailureDelay = DEFAULT_SNIFF_AFTER_FAILURE_DELAY;
         private String scheme = "http";
         private RestClient restClient;
 

From 1f7f6e2709fc988d2b1ae93223fc2a1861999dd6 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Mon, 6 Jun 2016 15:13:42 +0200
Subject: [PATCH 0653/1311] wrap log statement with logger.isDebugEnabled

---
 .../main/java/org/elasticsearch/client/RequestLogger.java   | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/RequestLogger.java b/client/src/main/java/org/elasticsearch/client/RequestLogger.java
index c15f6abf04b..cdee2d9944d 100644
--- a/client/src/main/java/org/elasticsearch/client/RequestLogger.java
+++ b/client/src/main/java/org/elasticsearch/client/RequestLogger.java
@@ -52,8 +52,10 @@ final class RequestLogger {
      * Logs a request that yielded a response
      */
     static void log(Log logger, String message, HttpUriRequest request, HttpHost host, HttpResponse httpResponse) {
-        logger.debug(message + " [" + request.getMethod() + " " + host + request.getRequestLine().getUri() +
-                "] [" + httpResponse.getStatusLine() + "]");
+        if (logger.isDebugEnabled()) {
+            logger.debug(message + " [" + request.getMethod() + " " + host + request.getRequestLine().getUri() +
+                    "] [" + httpResponse.getStatusLine() + "]");
+        }
 
         if (tracer.isTraceEnabled()) {
             String requestLine;

From 0a8afa2e7269ddbec3755b8f10033e93f55149c2 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Mon, 6 Jun 2016 15:14:09 +0200
Subject: [PATCH 0654/1311] Add back pending deletes (#18698)

Triggering the pending deletes logic was accidentally removed in the clean up PR #18602.
---
 .../elasticsearch/indices/IndicesService.java | 17 ++++++++
 .../cluster/IndicesClusterStateService.java   | 40 ++++++++++++++++++-
 .../index/IndexWithShadowReplicasIT.java      | 13 +++---
 .../index/shard/IndexShardTests.java          |  8 ++--
 .../indices/IndicesServiceTests.java          |  4 ++
 .../org/elasticsearch/test/ESTestCase.java    | 11 ++---
 6 files changed, 76 insertions(+), 17 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index 188e3608cb3..ba512379868 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -112,6 +112,7 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
@@ -141,6 +142,7 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
     private final CircuitBreakerService circuitBreakerService;
     private volatile Map<String, IndexService> indices = emptyMap();
     private final Map<Index, List<PendingDelete>> pendingDeletes = new HashMap<>();
+    private final AtomicInteger numUncompletedDeletes = new AtomicInteger();
     private final OldShardsStats oldShardsStats = new OldShardsStats();
     private final IndexStoreConfig indexStoreConfig;
     private final MapperRegistry mapperRegistry;
@@ -782,6 +784,7 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
                 pendingDeletes.put(index, list);
             }
             list.add(pendingDelete);
+            numUncompletedDeletes.incrementAndGet();
         }
     }
 
@@ -840,6 +843,7 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
         logger.debug("{} processing pending deletes", index);
         final long startTimeNS = System.nanoTime();
         final List<ShardLock> shardLocks = nodeEnv.lockAllForIndex(index, indexSettings, timeout.millis());
+        int numRemoved = 0;
         try {
             Map<ShardId, ShardLock> locks = new HashMap<>();
             for (ShardLock lock : shardLocks) {
@@ -850,6 +854,7 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
                  remove = pendingDeletes.remove(index);
             }
             if (remove != null && remove.isEmpty() == false) {
+                numRemoved = remove.size();
                 CollectionUtil.timSort(remove); // make sure we delete indices first
                 final long maxSleepTimeMs = 10 * 1000; // ensure we retry after 10 sec
                 long sleepTime = 10;
@@ -896,6 +901,10 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
             }
         } finally {
             IOUtils.close(shardLocks);
+            if (numRemoved > 0) {
+                int remainingUncompletedDeletes = numUncompletedDeletes.addAndGet(-numRemoved);
+                assert remainingUncompletedDeletes >= 0;
+            }
         }
     }
 
@@ -909,6 +918,14 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
         }
     }
 
+    /**
+     * Checks if all pending deletes have completed. Used by tests to ensure we don't check directory contents while deletion still ongoing.
+     * The reason is that, on Windows, browsing the directory contents can interfere with the deletion process and delay it unnecessarily.
+     */
+    public boolean hasUncompletedPendingDeletes() {
+        return numUncompletedDeletes.get() > 0;
+    }
+
     /**
      * Returns this nodes {@link IndicesQueriesRegistry}
      */
diff --git a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
index 719fb812c74..bbb1a063e8f 100644
--- a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
+++ b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.indices.cluster;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
+import org.apache.lucene.store.LockObtainFailedException;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
@@ -41,11 +42,14 @@ import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.Callback;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexShardAlreadyExistsException;
 import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.mapper.DocumentMapper;
@@ -67,7 +71,6 @@ import org.elasticsearch.snapshots.RestoreService;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -75,6 +78,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  *
@@ -213,11 +218,14 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
                 logger.debug("[{}] cleaning index, no longer part of the metadata", index);
             }
             final IndexService idxService = indicesService.indexService(index);
+            final IndexSettings indexSettings;
             if (idxService != null) {
+                indexSettings = idxService.getIndexSettings();
                 deleteIndex(index, "index no longer part of the metadata");
             } else if (previousState.metaData().hasIndex(index.getName())) {
                 // The deleted index was part of the previous cluster state, but not loaded on the local node
                 final IndexMetaData metaData = previousState.metaData().index(index);
+                indexSettings = new IndexSettings(metaData, settings);
                 indicesService.deleteUnassignedIndex("deleted index was not assigned to local node", metaData, event.state());
             } else {
                 // The previous cluster state's metadata also does not contain the index,
@@ -227,7 +235,35 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
                 // First, though, verify the precondition for applying this case by
                 // asserting that the previous cluster state is not initialized/recovered.
                 assert previousState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK);
-                indicesService.verifyIndexIsDeleted(index, event.state());
+                final IndexMetaData metaData = indicesService.verifyIndexIsDeleted(index, event.state());
+                if (metaData != null) {
+                    indexSettings = new IndexSettings(metaData, settings);
+                } else {
+                    indexSettings = null;
+                }
+            }
+            if (indexSettings != null) {
+                threadPool.generic().execute(new AbstractRunnable() {
+                    @Override
+                    public void onFailure(Throwable t) {
+                        logger.warn("[{}] failed to complete pending deletion for index", t, index);
+                    }
+
+                    @Override
+                    protected void doRun() throws Exception {
+                        try {
+                            // we are waiting until we can lock the index / all shards on the node and then we ack the delete of the store to the
+                            // master. If we can't acquire the locks here immediately there might be a shard of this index still holding on to the lock
+                            // due to a "currently canceled recovery" or so. The shard will delete itself BEFORE the lock is released so it's guaranteed to be
+                            // deleted by the time we get the lock
+                            indicesService.processPendingDeletes(index, indexSettings, new TimeValue(30, TimeUnit.MINUTES));
+                        } catch (LockObtainFailedException exc) {
+                            logger.warn("[{}] failed to lock all shards for index - timed out after 30 seconds", index);
+                        } catch (InterruptedException e) {
+                            logger.warn("[{}] failed to lock all shards for index - interrupted", index);
+                        }
+                    }
+                });
             }
         }
 
diff --git a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
index 1b7c71c644f..f0e12abeac8 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
@@ -70,7 +70,6 @@ import java.util.List;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -586,8 +585,8 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
 
         logger.info("--> deleting index " + IDX);
         assertAcked(client().admin().indices().prepareDelete(IDX));
-
-        // assertBusy(() -> assertPathHasBeenCleared(dataPath), 1, TimeUnit.MINUTES);
+        assertAllIndicesRemovedAndDeletionCompleted(internalCluster().getInstances(IndicesService.class));
+        assertPathHasBeenCleared(dataPath);
         //norelease
         //TODO: uncomment the test below when https://github.com/elastic/elasticsearch/issues/17695 is resolved.
         //assertIndicesDirsDeleted(nodes);
@@ -647,8 +646,8 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
         assertHitCount(resp, docCount);
 
         assertAcked(client().admin().indices().prepareDelete(IDX));
-
-        // assertBusy(() -> assertPathHasBeenCleared(dataPath), 1, TimeUnit.MINUTES);
+        assertAllIndicesRemovedAndDeletionCompleted(internalCluster().getInstances(IndicesService.class));
+        assertPathHasBeenCleared(dataPath);
         //norelease
         //TODO: uncomment the test below when https://github.com/elastic/elasticsearch/issues/17695 is resolved.
         //assertIndicesDirsDeleted(nodes);
@@ -839,8 +838,8 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
 
         logger.info("--> deleting closed index");
         client().admin().indices().prepareDelete(IDX).get();
-
-        assertBusy(() -> assertPathHasBeenCleared(dataPath), 1, TimeUnit.MINUTES);
+        assertAllIndicesRemovedAndDeletionCompleted(internalCluster().getInstances(IndicesService.class));
+        assertPathHasBeenCleared(dataPath);
         assertIndicesDirsDeleted(nodes);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 7774537c734..e7a7eb46377 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -558,7 +558,8 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         SearchResponse response = client().prepareSearch("test").get();
         assertHitCount(response, 1L);
         client().admin().indices().prepareDelete("test").get();
-        assertBusyPathHasBeenCleared(idxPath);
+        assertAllIndicesRemovedAndDeletionCompleted(Collections.singleton(getInstanceFromNode(IndicesService.class)));
+        assertPathHasBeenCleared(idxPath);
     }
 
     public void testExpectedShardSizeIsPresent() throws InterruptedException {
@@ -641,8 +642,9 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         assertThat("found the hit", resp.getHits().getTotalHits(), equalTo(1L));
 
         assertAcked(client().admin().indices().prepareDelete(INDEX));
-        assertBusyPathHasBeenCleared(startDir.toAbsolutePath());
-        assertBusyPathHasBeenCleared(endDir.toAbsolutePath());
+        assertAllIndicesRemovedAndDeletionCompleted(Collections.singleton(getInstanceFromNode(IndicesService.class)));
+        assertPathHasBeenCleared(startDir.toAbsolutePath());
+        assertPathHasBeenCleared(endDir.toAbsolutePath());
     }
 
     public void testShardStats() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
index 76f7a30e078..fa4183f689a 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
@@ -190,10 +190,12 @@ public class IndicesServiceTests extends ESSingleNodeTestCase {
         assertTrue(path.exists());
 
         assertEquals(indicesService.numPendingDeletes(test.index()), numPending);
+        assertTrue(indicesService.hasUncompletedPendingDeletes());
 
         // shard lock released... we can now delete
         indicesService.processPendingDeletes(test.index(), test.getIndexSettings(), new TimeValue(0, TimeUnit.MILLISECONDS));
         assertEquals(indicesService.numPendingDeletes(test.index()), 0);
+        assertFalse(indicesService.hasUncompletedPendingDeletes());
         assertFalse(path.exists());
 
         if (randomBoolean()) {
@@ -201,9 +203,11 @@ public class IndicesServiceTests extends ESSingleNodeTestCase {
             indicesService.addPendingDelete(new ShardId(test.index(), 1), test.getIndexSettings());
             indicesService.addPendingDelete(new ShardId("bogus", "_na_", 1), test.getIndexSettings());
             assertEquals(indicesService.numPendingDeletes(test.index()), 2);
+            assertTrue(indicesService.hasUncompletedPendingDeletes());
             // shard lock released... we can now delete
             indicesService.processPendingDeletes(test.index(), test.getIndexSettings(), new TimeValue(0, TimeUnit.MILLISECONDS));
             assertEquals(indicesService.numPendingDeletes(test.index()), 0);
+            assertFalse(indicesService.hasUncompletedPendingDeletes());
         }
         assertAcked(client().admin().indices().prepareOpen("test"));
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 972c1a50fbc..0859a8e86ea 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -56,6 +56,7 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.analysis.AnalysisService;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.search.MockSearchService;
 import org.elasticsearch.test.junit.listeners.LoggingListener;
@@ -672,11 +673,11 @@ public abstract class ESTestCase extends LuceneTestCase {
         return enabled;
     }
 
-    /**
-     * Asserts busily that there are no files in the specified path
-     */
-    public void assertBusyPathHasBeenCleared(Path path) throws Exception {
-        assertBusy(() -> assertPathHasBeenCleared(path));
+    public void assertAllIndicesRemovedAndDeletionCompleted(Iterable<IndicesService> indicesServices) throws Exception {
+        for (IndicesService indicesService : indicesServices) {
+            assertBusy(() -> assertFalse(indicesService.iterator().hasNext()), 1, TimeUnit.MINUTES);
+            assertBusy(() -> assertFalse(indicesService.hasUncompletedPendingDeletes()), 1, TimeUnit.MINUTES);
+        }
     }
 
     /**

From 6dbf7ab1ea1c2afee4c2ae2ad4b43cde2f018564 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 6 Jun 2016 09:27:19 -0400
Subject: [PATCH 0655/1311] initial messy impl of painless method references

---
 .../src/main/antlr/PainlessParser.g4          |   2 +-
 .../java/org/elasticsearch/painless/Def.java  | 137 ++++++++---
 .../elasticsearch/painless/DefBootstrap.java  |  16 +-
 .../elasticsearch/painless/Definition.java    |  79 +++++--
 .../elasticsearch/painless/FunctionRef.java   | 126 ++++++++++
 .../painless/PainlessScriptEngineService.java |   1 -
 .../elasticsearch/painless/ScriptImpl.java    |   2 +-
 .../painless/WriterConstants.java             |  14 +-
 .../painless/antlr/PainlessParser.java        | 221 +++++++++---------
 .../elasticsearch/painless/antlr/Walker.java  |  10 +-
 .../painless/node/EFunctionRef.java           |  41 +++-
 .../elasticsearch/painless/node/LCall.java    |   2 +-
 .../painless/node/LDefArray.java              |   4 +-
 .../elasticsearch/painless/node/LDefCall.java |  14 +-
 .../painless/node/LDefField.java              |   4 +-
 .../elasticsearch/painless/node/LNewObj.java  |   6 +-
 .../org/elasticsearch/painless/java.lang.txt  |  14 +-
 .../org/elasticsearch/painless/java.math.txt  |   2 +
 .../org/elasticsearch/painless/java.text.txt  |   1 +
 .../painless/java.time.chrono.txt             |  53 +++--
 .../org/elasticsearch/painless/java.time.txt  |  32 ++-
 .../elasticsearch/painless/java.time.zone.txt |   1 +
 .../org/elasticsearch/painless/java.util.txt  |  43 ++--
 .../painless/org.elasticsearch.txt            |   4 +
 .../elasticsearch/painless/BasicAPITests.java |  11 +-
 .../painless/DefBootstrapTests.java           |   6 +-
 .../painless/FunctionRefTests.java            |  59 ++++-
 .../elasticsearch/painless/OverloadTests.java |   2 +-
 28 files changed, 659 insertions(+), 248 deletions(-)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java

diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index 5125e7b29a6..0abab56dad2 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -73,7 +73,7 @@ decltype
     ;
 
 funcref
-    : TYPE REF ID
+    : TYPE REF ( ID | NEW )
     ;
 
 declvar
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index bd8e09f504b..60c42e2af55 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -22,6 +22,9 @@ package org.elasticsearch.painless;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.RuntimeClass;
 
+import java.lang.invoke.CallSite;
+import java.lang.invoke.LambdaConversionException;
+import java.lang.invoke.LambdaMetafactory;
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodHandles.Lookup;
@@ -123,7 +126,52 @@ public final class Def {
     }
 
     /**
-     * Looks up handle for a dynamic method call.
+     * Looks up method entry for a dynamic method call.
+     * <p>
+     * A dynamic method call for variable {@code x} of type {@code def} looks like:
+     * {@code x.method(args...)}
+     * <p>
+     * This method traverses {@code recieverClass}'s class hierarchy (including interfaces)
+     * until it finds a matching whitelisted method. If one is not found, it throws an exception.
+     * Otherwise it returns the matching method.
+     * <p>
+     * @param receiverClass Class of the object to invoke the method on.
+     * @param name Name of the method.
+     * @param arity arity of method
+     * @return matching method to invoke. never returns null.
+     * @throws IllegalArgumentException if no matching whitelisted method was found.
+     */
+    static Method lookupMethodInternal(Class<?> receiverClass, String name, int arity) {
+        Definition.MethodKey key = new Definition.MethodKey(name, arity);
+        // check whitelist for matching method
+        for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
+            RuntimeClass struct = Definition.getRuntimeClass(clazz);
+
+            if (struct != null) {
+                Method method = struct.methods.get(key);
+                if (method != null) {
+                    return method;
+                }
+            }
+
+            for (Class<?> iface : clazz.getInterfaces()) {
+                struct = Definition.getRuntimeClass(iface);
+
+                if (struct != null) {
+                    Method method = struct.methods.get(key);
+                    if (method != null) {
+                        return method;
+                    }
+                }
+            }
+        }
+        
+        throw new IllegalArgumentException("Unable to find dynamic method [" + name + "] with [" + arity + "] arguments " +
+                                           "for class [" + receiverClass.getCanonicalName() + "].");
+    }
+
+    /**
+     * Looks up handle for a dynamic method call, with lambda replacement
      * <p>
      * A dynamic method call for variable {@code x} of type {@code def} looks like:
      * {@code x.method(args...)}
@@ -134,41 +182,72 @@ public final class Def {
      * <p>
      * @param receiverClass Class of the object to invoke the method on.
      * @param name Name of the method.
-     * @param type Callsite signature. Need not match exactly, except the number of parameters.
+     * @param args args passed to callsite
+     * @param recipe bitset marking functional parameters
      * @return pointer to matching method to invoke. never returns null.
      * @throws IllegalArgumentException if no matching whitelisted method was found.
      */
-     static MethodHandle lookupMethod(Class<?> receiverClass, String name, MethodType type) {
-         // we don't consider receiver an argument/counting towards arity
-         type = type.dropParameterTypes(0, 1);
-         Definition.MethodKey key = new Definition.MethodKey(name, type.parameterCount());
-         // check whitelist for matching method
-         for (Class<?> clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) {
-             RuntimeClass struct = Definition.getRuntimeClass(clazz);
+     static MethodHandle lookupMethod(Class<?> receiverClass, String name, Object args[], long recipe) {
+         Method method = lookupMethodInternal(receiverClass, name, args.length - 1);
+         MethodHandle handle = method.handle;
+         MethodHandle filters[] = new MethodHandle[args.length];
 
-             if (struct != null) {
-                 Method method = struct.methods.get(key);
-                 if (method != null) {
-                     return method.handle;
-                 }
-             }
-
-             for (final Class<?> iface : clazz.getInterfaces()) {
-                 struct = Definition.getRuntimeClass(iface);
-
-                 if (struct != null) {
-                     Method method = struct.methods.get(key);
-                     if (method != null) {
-                         return method.handle;
-                     }
+         if (recipe != 0) {
+             for (int i = 0; i < args.length; i++) {
+                 // its a functional reference, replace the argument with an impl
+                 if ((recipe & (1L << (i - 1))) != 0) {
+                     filters[i] = lookupReference(method.arguments.get(i - 1).clazz, (String) args[i]);
                  }
              }
          }
-
-         // no matching methods in whitelist found
-         throw new IllegalArgumentException("Unable to find dynamic method [" + name + "] with signature [" + type + "] " +
-                 "for class [" + receiverClass.getCanonicalName() + "].");
-    }
+         handle = MethodHandles.filterArguments(handle, 0, filters);
+         
+         return handle;
+     }
+     
+     /** Returns a method handle to an implementation of clazz, given method reference signature */
+     private static MethodHandle lookupReference(Class<?> clazz, String signature) {
+         int separator = signature.indexOf('.');
+         FunctionRef ref = new FunctionRef(clazz, signature.substring(0, separator), signature.substring(separator+1));
+         MethodHandles.Lookup lookup = MethodHandles.lookup(); // XXX: no lookuping needed, we should pass this from DefBootstrap!
+         final CallSite callSite;
+         // XXX: clean all this up to use handles in FunctionRef, deal with ASM in EFunctionRef differently
+         MethodType invokedType = MethodType.fromMethodDescriptorString(ref.invokedType.getDescriptor(), Def.class.getClassLoader());
+         MethodType samMethodType = MethodType.fromMethodDescriptorString(ref.samMethodType.getDescriptor(), Def.class.getClassLoader());
+         MethodType interfaceType = MethodType.fromMethodDescriptorString(ref.interfaceType.getDescriptor(), Def.class.getClassLoader());
+         try {
+             if (ref.interfaceType.equals(ref.samMethodType)) {
+                 callSite = LambdaMetafactory.altMetafactory(lookup, 
+                         ref.invokedName, 
+                         invokedType,
+                         samMethodType,
+                         ref.implMethodHandle,
+                         samMethodType,
+                         0);
+             } else {
+                 callSite = LambdaMetafactory.altMetafactory(lookup, 
+                         ref.invokedName, 
+                         invokedType,
+                         samMethodType,
+                         ref.implMethodHandle,
+                         samMethodType,
+                         LambdaMetafactory.FLAG_BRIDGES,
+                         1,
+                         interfaceType);
+             }
+         } catch (LambdaConversionException e) {
+             throw new RuntimeException(e);
+         }
+         try {
+             // create an implementation of the interface (instance)
+             Object instance = callSite.dynamicInvoker().asType(MethodType.methodType(clazz)).invoke();
+             // bind this instance as a constant replacement for the parameter
+             return MethodHandles.dropArguments(MethodHandles.constant(clazz, instance), 0, Object.class);
+         } catch (Throwable e) {
+             throw new RuntimeException(e);
+         }
+     }
+     
 
     /**
      * Looks up handle for a dynamic field getter (field load)
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index 40b9cc6cbe8..4c30891dd2d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -66,12 +66,16 @@ public final class DefBootstrap {
 
         private final String name;
         private final int flavor;
+        private final long recipe;
         int depth; // pkg-protected for testing
 
-        PIC(String name, MethodType type, int flavor) {
+        PIC(String name, MethodType type, int flavor, long recipe) {
             super(type);
             this.name = name;
             this.flavor = flavor;
+            this.recipe = recipe;
+            assert recipe == 0 || flavor == METHOD_CALL;
+            assert Long.bitCount(flavor) <= type.parameterCount();
 
             final MethodHandle fallback = FALLBACK.bindTo(this)
               .asCollector(Object[].class, type.parameterCount())
@@ -91,10 +95,10 @@ public final class DefBootstrap {
         /**
          * Does a slow lookup against the whitelist.
          */
-        private static MethodHandle lookup(int flavor, Class<?> clazz, String name, MethodType type) {
+        private static MethodHandle lookup(int flavor, Class<?> clazz, String name, Object[] args, long recipe) {
             switch(flavor) {
                 case METHOD_CALL:
-                    return Def.lookupMethod(clazz, name, type);
+                    return Def.lookupMethod(clazz, name, args, recipe);
                 case LOAD:
                     return Def.lookupGetter(clazz, name);
                 case STORE:
@@ -115,7 +119,7 @@ public final class DefBootstrap {
             final MethodType type = type();
             final Object receiver = args[0];
             final Class<?> receiverClass = receiver.getClass();
-            final MethodHandle target = lookup(flavor, receiverClass, name, type).asType(type);
+            final MethodHandle target = lookup(flavor, receiverClass, name, args, recipe).asType(type);
 
             if (depth >= MAX_DEPTH) {
                 // revert to a vtable call
@@ -157,8 +161,8 @@ public final class DefBootstrap {
      * <p>
      * see https://docs.oracle.com/javase/specs/jvms/se7/html/jvms-6.html#jvms-6.5.invokedynamic
      */
-    public static CallSite bootstrap(Lookup lookup, String name, MethodType type, int flavor) {
-        return new PIC(name, type, flavor);
+    public static CallSite bootstrap(Lookup lookup, String name, MethodType type, int flavor, long recipe) {
+        return new PIC(name, type, flavor, recipe);
     }
 
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 50ee4649074..061ef0d95d7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -24,6 +24,7 @@ import java.io.InputStreamReader;
 import java.io.LineNumberReader;
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
+import java.lang.reflect.Modifier;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Collections;
@@ -31,6 +32,8 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.PrimitiveIterator;
+import java.util.Spliterator;
 
 /**
  * The entire API for Painless.  Also used as a whitelist for checking for legal
@@ -171,21 +174,6 @@ public final class Definition {
         }
     }
 
-    public static final class Constructor {
-        public final String name;
-        public final Struct owner;
-        public final List<Type> arguments;
-        public final org.objectweb.asm.commons.Method method;
-
-        private Constructor(String name, Struct owner, List<Type> arguments,
-                            org.objectweb.asm.commons.Method method) {
-            this.name = name;
-            this.owner = owner;
-            this.arguments = Collections.unmodifiableList(arguments);
-            this.method = method;
-        }
-    }
-
     public static class Method {
         public final String name;
         public final Struct owner;
@@ -287,7 +275,7 @@ public final class Definition {
         public final Class<?> clazz;
         public final org.objectweb.asm.Type type;
 
-        public final Map<MethodKey, Constructor> constructors;
+        public final Map<MethodKey, Method> constructors;
         public final Map<MethodKey, Method> staticMethods;
         public final Map<MethodKey, Method> methods;
 
@@ -439,6 +427,18 @@ public final class Definition {
         for (Map.Entry<String,List<String>> clazz : hierarchy.entrySet()) {
             copyStruct(clazz.getKey(), clazz.getValue());
         }
+        // if someone declares an interface type, its still an Object
+        for (Map.Entry<String,Struct> clazz : structsMap.entrySet()) {
+            String name = clazz.getKey();
+            Class<?> javaPeer = clazz.getValue().clazz;
+            if (javaPeer.isInterface()) {
+                copyStruct(name, Collections.singletonList("Object"));
+            } else if (name.equals("def") == false && name.equals("Object") == false && javaPeer.isPrimitive() == false) {
+                // but otherwise, unless its a primitive type, it really should
+                assert hierarchy.get(name) != null : "class '" + name + "' does not extend Object!";
+                assert hierarchy.get(name).contains("Object") : "class '" + name + "' does not extend Object!";
+            }
+        }
         // precompute runtime classes
         for (Struct struct : structsMap.values()) {
           addRuntimeClass(struct);
@@ -573,7 +573,7 @@ public final class Definition {
                 "Owner struct [" + struct + "] not defined for constructor [" + name + "].");
         }
 
-        if (!name.matches("^[_a-zA-Z][_a-zA-Z0-9]*$")) {
+        if (!name.matches("<init>")) {
             throw new IllegalArgumentException(
                 "Invalid constructor name [" + name + "] with the struct [" + owner.name + "].");
         }
@@ -611,7 +611,18 @@ public final class Definition {
         }
 
         final org.objectweb.asm.commons.Method asm = org.objectweb.asm.commons.Method.getMethod(reflect);
-        final Constructor constructor = new Constructor(name, owner, Arrays.asList(args), asm);
+        final Type returnType = getTypeInternal("void");
+        final MethodHandle handle;
+        
+        try {
+            handle = MethodHandles.publicLookup().in(owner.clazz).unreflectConstructor(reflect);
+        } catch (final IllegalAccessException exception) {
+            throw new IllegalArgumentException("Constructor " +
+                " not found for class [" + owner.clazz.getName() + "]" +
+                " with arguments " + Arrays.toString(classes) + ".");
+        }
+        
+        final Method constructor = new Method(name, owner, returnType, Arrays.asList(args), asm, reflect.getModifiers(), handle);
 
         owner.constructors.put(methodKey, constructor);
     }
@@ -653,7 +664,7 @@ public final class Definition {
                 if (!elements[0].equals(className)) {
                     throw new IllegalArgumentException("Constructors must return their own type");
                 }
-                addConstructorInternal(className, "new", args);
+                addConstructorInternal(className, "<init>", args);
             } else {
                 if (methodName.indexOf('/') >= 0) {
                     String nameAndAlias[] = methodName.split("/");
@@ -720,7 +731,7 @@ public final class Definition {
                 " method [" + name + "]" +
                 " within the struct [" + owner.name + "].");
         }
-
+        
         final org.objectweb.asm.commons.Method asm = org.objectweb.asm.commons.Method.getMethod(reflect);
 
         MethodHandle handle;
@@ -821,11 +832,37 @@ public final class Definition {
                 throw new ClassCastException("Child struct [" + child.name + "]" +
                     " is not a super type of owner struct [" + owner.name + "] in copy.");
             }
-
+            
             for (Map.Entry<MethodKey,Method> kvPair : child.methods.entrySet()) {
                 MethodKey methodKey = kvPair.getKey();
                 Method method = kvPair.getValue();
                 if (owner.methods.get(methodKey) == null) {
+                    // sanity check, look for missing covariant/generic override
+                    if (owner.clazz.isInterface() && child.clazz == Object.class) {
+                        // ok
+                    } else if (child.clazz == Spliterator.OfPrimitive.class || child.clazz == PrimitiveIterator.class) {
+                        // ok, we rely on generics erasure for these (its guaranteed in the javadocs though!!!!)
+                    } else {
+                        try {
+                            Class<?> arguments[] = new Class<?>[method.arguments.size()];
+                            for (int i = 0; i < method.arguments.size(); i++) {
+                                arguments[i] = method.arguments.get(i).clazz;
+                            }
+                            java.lang.reflect.Method m = owner.clazz.getMethod(method.method.getName(), arguments);
+                            if (m.getReturnType() != method.rtn.clazz) {
+                                throw new IllegalStateException("missing covariant override for: " + m + " in " + owner.name);
+                            }
+                            if (m.isBridge() && !Modifier.isVolatile(method.modifiers)) {
+                                // its a bridge in the destination, but not in the source, but it might still be ok, check generics:
+                                java.lang.reflect.Method source = child.clazz.getMethod(method.method.getName(), arguments);
+                                if (!Arrays.equals(source.getGenericParameterTypes(), source.getParameterTypes())) {
+                                    throw new IllegalStateException("missing generic override for: " + m + " in " + owner.name);
+                                }
+                            }
+                        } catch (ReflectiveOperationException e) {
+                            throw new AssertionError(e);
+                        }
+                    }
                     owner.methods.put(methodKey,
                         new Method(method.name, owner, method.rtn, method.arguments, method.method, method.modifiers, method.handle));
                 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
new file mode 100644
index 00000000000..5b121d3a31a
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.objectweb.asm.Handle;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.Type;
+
+import java.lang.invoke.MethodHandle;
+import java.lang.reflect.Modifier;
+import java.util.HashSet;
+import java.util.Set;
+
+/** 
+ * computes "everything you need" to call LambdaMetaFactory, given an expected interface,
+ * and reference class + method name
+ */
+public class FunctionRef {
+    // XXX: this is a mess, because of ASM versus MethodHandle types
+    // clean all this up, move reflection out of here into definition, etc etc
+    public final String invokedName;
+    public final Type invokedType;
+    public final Handle implMethod;
+    public final Type samMethodType;
+    public final Type interfaceType;
+
+    public final MethodHandle implMethodHandle;
+    
+    public FunctionRef(Class<?> expected, String type, String call) {
+        boolean isCtorReference = "new".equals(call);
+        // check its really a functional interface
+        // for e.g. Comparable
+        java.lang.reflect.Method method = getFunctionalMethod(type, call, expected);
+        // e.g. compareTo
+        invokedName = method.getName();
+        // e.g. (Object)Comparator
+        invokedType = Type.getMethodType(Type.getType(expected));
+        // e.g. (Object,Object)int
+        interfaceType = Type.getMethodType(Type.getMethodDescriptor(method));
+        // lookup requested method
+        Definition.Struct struct = Definition.getType(type).struct;
+        final Definition.Method impl;
+        // ctor ref
+        if (isCtorReference) {
+            impl = struct.constructors.get(new Definition.MethodKey("<init>", method.getParameterCount()));
+        } else {
+            // look for a static impl first
+            Definition.Method staticImpl = struct.staticMethods.get(new Definition.MethodKey(call, method.getParameterCount()));
+            if (staticImpl == null) {
+                // otherwise a virtual impl
+                impl = struct.methods.get(new Definition.MethodKey(call, method.getParameterCount()-1));
+            } else {
+                impl = staticImpl;
+            }
+        }
+        if (impl == null) {
+            throw new IllegalArgumentException("Unknown reference [" + type + "::" + call + "] matching " +
+                                               "[" + expected + "]");
+        }
+        
+        final int tag;
+        if (isCtorReference) {
+            tag = Opcodes.H_NEWINVOKESPECIAL;
+        } else if (Modifier.isStatic(impl.modifiers)) {
+            tag = Opcodes.H_INVOKESTATIC;
+        } else {
+            tag = Opcodes.H_INVOKEVIRTUAL;
+        }
+        implMethod = new Handle(tag, struct.type.getInternalName(), impl.name, impl.method.getDescriptor());
+        implMethodHandle = impl.handle;
+        if (isCtorReference) {
+            samMethodType = Type.getMethodType(interfaceType.getReturnType(), impl.method.getArgumentTypes());
+        } else if (Modifier.isStatic(impl.modifiers)) {
+            samMethodType = Type.getMethodType(impl.method.getReturnType(), impl.method.getArgumentTypes());
+        } else {
+            Type[] argTypes = impl.method.getArgumentTypes();
+            Type[] params = new Type[argTypes.length + 1];
+            System.arraycopy(argTypes, 0, params, 1, argTypes.length);
+            params[0] = struct.type;
+            samMethodType = Type.getMethodType(impl.method.getReturnType(), params);
+        }
+    }
+    
+    static final Set<Definition.MethodKey> OBJECT_METHODS = new HashSet<>();
+    static {
+        for (java.lang.reflect.Method m : Object.class.getMethods()) {
+            OBJECT_METHODS.add(new Definition.MethodKey(m.getName(), m.getParameterCount()));
+        }
+    }
+
+    // TODO: move all this crap out, to Definition to compute up front
+    java.lang.reflect.Method getFunctionalMethod(String type, String call, Class<?> clazz) {
+        if (!clazz.isInterface()) {
+            throw new IllegalArgumentException("Cannot convert function reference [" 
+                                               + type + "::" + call + "] to [" + clazz + "]");
+        }
+        for (java.lang.reflect.Method m : clazz.getMethods()) {
+            if (m.isDefault()) {
+                continue;
+            }
+            if (OBJECT_METHODS.contains(new Definition.MethodKey(m.getName(), m.getParameterCount()))) {
+                continue;
+            }
+            return m;
+        }
+        throw new IllegalArgumentException("Cannot convert function reference [" 
+                                           + type + "::" + call + "] to [" + clazz + "]");
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
index 4971c08b086..0d8d9fa4a8b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
@@ -40,7 +40,6 @@ import java.security.Permissions;
 import java.security.PrivilegedAction;
 import java.security.ProtectionDomain;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
index 380b6c2bf21..52388a5942e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
@@ -119,7 +119,7 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
     public Object run() {
         try {
             return executable.execute(variables, scorer, doc, aggregationValue);
-        } catch (PainlessError | Exception t) {
+        } catch (PainlessError | BootstrapMethodError | Exception t) {
             throw convertToScriptException(t);
         }
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index a4804cde434..1e10446e7b4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -27,6 +27,8 @@ import org.objectweb.asm.Type;
 import org.objectweb.asm.commons.Method;
 
 import java.lang.invoke.CallSite;
+import java.lang.invoke.LambdaMetafactory;
+import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodType;
 import java.util.BitSet;
@@ -62,7 +64,7 @@ public final class WriterConstants {
 
     /** dynamic callsite bootstrap signature */
     public final static MethodType DEF_BOOTSTRAP_TYPE =
-        MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, int.class);
+        MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, int.class, long.class);
     public final static Handle DEF_BOOTSTRAP_HANDLE =
         new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(DefBootstrap.class),
             "bootstrap", DEF_BOOTSTRAP_TYPE.toMethodDescriptorString());
@@ -101,7 +103,15 @@ public final class WriterConstants {
     public final static Method DEF_LTE_CALL = getAsmMethod(boolean.class, "lte", Object.class, Object.class);
     public final static Method DEF_GT_CALL  = getAsmMethod(boolean.class, "gt" , Object.class, Object.class);
     public final static Method DEF_GTE_CALL = getAsmMethod(boolean.class, "gte", Object.class, Object.class);
-
+    
+    /** invokedynamic bootstrap for lambda expression/method references */
+    public final static MethodType LAMBDA_BOOTSTRAP_TYPE =
+            MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, 
+                                  MethodType.class, Object[].class);
+    public final static Handle LAMBDA_BOOTSTRAP_HANDLE =
+            new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(LambdaMetafactory.class),
+                "altMetafactory", LAMBDA_BOOTSTRAP_TYPE.toMethodDescriptorString());
+        
     /** dynamic invokedynamic bootstrap for indy string concats (Java 9+) */
     public final static Handle INDY_STRING_CONCAT_BOOTSTRAP_HANDLE;
     static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
index 98eb16732fd..a0903de5574 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
@@ -986,6 +986,7 @@ class PainlessParser extends Parser {
     public TerminalNode TYPE() { return getToken(PainlessParser.TYPE, 0); }
     public TerminalNode REF() { return getToken(PainlessParser.REF, 0); }
     public TerminalNode ID() { return getToken(PainlessParser.ID, 0); }
+    public TerminalNode NEW() { return getToken(PainlessParser.NEW, 0); }
     public FuncrefContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
     }
@@ -1000,6 +1001,7 @@ class PainlessParser extends Parser {
   public final FuncrefContext funcref() throws RecognitionException {
     FuncrefContext _localctx = new FuncrefContext(_ctx, getState());
     enterRule(_localctx, 18, RULE_funcref);
+    int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
@@ -1008,7 +1010,12 @@ class PainlessParser extends Parser {
       setState(162);
       match(REF);
       setState(163);
-      match(ID);
+      _la = _input.LA(1);
+      if ( !(_la==NEW || _la==ID) ) {
+      _errHandler.recoverInline(this);
+      } else {
+        consume();
+      }
       }
     }
     catch (RecognitionException re) {
@@ -2623,112 +2630,112 @@ class PainlessParser extends Parser {
     "\24\3\24\3\24\3\24\3\24\5\24\u015f\n\24\3\25\3\25\3\25\3\25\3\26\3\26"+
     "\3\26\3\26\7\26\u0169\n\26\f\26\16\26\u016c\13\26\5\26\u016e\n\26\3\26"+
     "\3\26\3\27\3\27\5\27\u0174\n\27\3\27\2\3\34\30\2\4\6\b\n\f\16\20\22\24"+
-    "\26\30\32\34\36 \"$&(*,\2\r\3\3\r\r\3\2\66A\3\2\34\36\3\2\37 \3\2!#\3"+
-    "\2$\'\3\2(+\3\2\64\65\3\2BE\4\2\32\33\37 \3\2LM\u019b\2\61\3\2\2\2\4{"+
-    "\3\2\2\2\6\177\3\2\2\2\b\u0081\3\2\2\2\n\u008a\3\2\2\2\f\u008e\3\2\2\2"+
-    "\16\u0090\3\2\2\2\20\u0092\3\2\2\2\22\u009b\3\2\2\2\24\u00a3\3\2\2\2\26"+
-    "\u00a7\3\2\2\2\30\u00ac\3\2\2\2\32\u00b3\3\2\2\2\34\u00be\3\2\2\2\36\u011b"+
-    "\3\2\2\2 \u013f\3\2\2\2\"\u0151\3\2\2\2$\u0157\3\2\2\2&\u015e\3\2\2\2"+
-    "(\u0160\3\2\2\2*\u0164\3\2\2\2,\u0173\3\2\2\2.\60\5\4\3\2/.\3\2\2\2\60"+
-    "\63\3\2\2\2\61/\3\2\2\2\61\62\3\2\2\2\62\64\3\2\2\2\63\61\3\2\2\2\64\65"+
-    "\7\2\2\3\65\3\3\2\2\2\66\67\7\16\2\2\678\7\t\2\289\5\34\17\29:\7\n\2\2"+
-    ":>\5\6\4\2;<\7\17\2\2<?\5\6\4\2=?\6\3\2\2>;\3\2\2\2>=\3\2\2\2?|\3\2\2"+
-    "\2@A\7\20\2\2AB\7\t\2\2BC\5\34\17\2CF\7\n\2\2DG\5\6\4\2EG\5\n\6\2FD\3"+
-    "\2\2\2FE\3\2\2\2G|\3\2\2\2HI\7\21\2\2IJ\5\b\5\2JK\7\20\2\2KL\7\t\2\2L"+
-    "M\5\34\17\2MN\7\n\2\2NO\5\32\16\2O|\3\2\2\2PQ\7\22\2\2QS\7\t\2\2RT\5\f"+
-    "\7\2SR\3\2\2\2ST\3\2\2\2TU\3\2\2\2UW\7\r\2\2VX\5\34\17\2WV\3\2\2\2WX\3"+
-    "\2\2\2XY\3\2\2\2Y[\7\r\2\2Z\\\5\16\b\2[Z\3\2\2\2[\\\3\2\2\2\\]\3\2\2\2"+
-    "]`\7\n\2\2^a\5\6\4\2_a\5\n\6\2`^\3\2\2\2`_\3\2\2\2a|\3\2\2\2bc\5\20\t"+
-    "\2cd\5\32\16\2d|\3\2\2\2ef\7\23\2\2f|\5\32\16\2gh\7\24\2\2h|\5\32\16\2"+
-    "ij\7\25\2\2jk\5\34\17\2kl\5\32\16\2l|\3\2\2\2mn\7\27\2\2np\5\b\5\2oq\5"+
-    "\30\r\2po\3\2\2\2qr\3\2\2\2rp\3\2\2\2rs\3\2\2\2s|\3\2\2\2tu\7\31\2\2u"+
-    "v\5\34\17\2vw\5\32\16\2w|\3\2\2\2xy\5\34\17\2yz\5\32\16\2z|\3\2\2\2{\66"+
-    "\3\2\2\2{@\3\2\2\2{H\3\2\2\2{P\3\2\2\2{b\3\2\2\2{e\3\2\2\2{g\3\2\2\2{"+
-    "i\3\2\2\2{m\3\2\2\2{t\3\2\2\2{x\3\2\2\2|\5\3\2\2\2}\u0080\5\b\5\2~\u0080"+
-    "\5\4\3\2\177}\3\2\2\2\177~\3\2\2\2\u0080\7\3\2\2\2\u0081\u0085\7\5\2\2"+
-    "\u0082\u0084\5\4\3\2\u0083\u0082\3\2\2\2\u0084\u0087\3\2\2\2\u0085\u0083"+
-    "\3\2\2\2\u0085\u0086\3\2\2\2\u0086\u0088\3\2\2\2\u0087\u0085\3\2\2\2\u0088"+
-    "\u0089\7\6\2\2\u0089\t\3\2\2\2\u008a\u008b\7\r\2\2\u008b\13\3\2\2\2\u008c"+
-    "\u008f\5\20\t\2\u008d\u008f\5\34\17\2\u008e\u008c\3\2\2\2\u008e\u008d"+
-    "\3\2\2\2\u008f\r\3\2\2\2\u0090\u0091\5\34\17\2\u0091\17\3\2\2\2\u0092"+
-    "\u0093\5\22\n\2\u0093\u0098\5\26\f\2\u0094\u0095\7\f\2\2\u0095\u0097\5"+
-    "\26\f\2\u0096\u0094\3\2\2\2\u0097\u009a\3\2\2\2\u0098\u0096\3\2\2\2\u0098"+
-    "\u0099\3\2\2\2\u0099\21\3\2\2\2\u009a\u0098\3\2\2\2\u009b\u00a0\7J\2\2"+
-    "\u009c\u009d\7\7\2\2\u009d\u009f\7\b\2\2\u009e\u009c\3\2\2\2\u009f\u00a2"+
-    "\3\2\2\2\u00a0\u009e\3\2\2\2\u00a0\u00a1\3\2\2\2\u00a1\23\3\2\2\2\u00a2"+
-    "\u00a0\3\2\2\2\u00a3\u00a4\7J\2\2\u00a4\u00a5\7\63\2\2\u00a5\u00a6\7K"+
-    "\2\2\u00a6\25\3\2\2\2\u00a7\u00aa\7K\2\2\u00a8\u00a9\7\66\2\2\u00a9\u00ab"+
-    "\5\34\17\2\u00aa\u00a8\3\2\2\2\u00aa\u00ab\3\2\2\2\u00ab\27\3\2\2\2\u00ac"+
-    "\u00ad\7\30\2\2\u00ad\u00ae\7\t\2\2\u00ae\u00af\7J\2\2\u00af\u00b0\7K"+
-    "\2\2\u00b0\u00b1\7\n\2\2\u00b1\u00b2\5\b\5\2\u00b2\31\3\2\2\2\u00b3\u00b4"+
-    "\t\2\2\2\u00b4\33\3\2\2\2\u00b5\u00b6\b\17\1\2\u00b6\u00b7\5 \21\2\u00b7"+
-    "\u00b8\t\3\2\2\u00b8\u00b9\5\34\17\3\u00b9\u00ba\b\17\1\2\u00ba\u00bf"+
-    "\3\2\2\2\u00bb\u00bc\5\36\20\2\u00bc\u00bd\b\17\1\2\u00bd\u00bf\3\2\2"+
-    "\2\u00be\u00b5\3\2\2\2\u00be\u00bb\3\2\2\2\u00bf\u00fb\3\2\2\2\u00c0\u00c1"+
-    "\f\16\2\2\u00c1\u00c2\t\4\2\2\u00c2\u00c3\5\34\17\17\u00c3\u00c4\b\17"+
-    "\1\2\u00c4\u00fa\3\2\2\2\u00c5\u00c6\f\r\2\2\u00c6\u00c7\t\5\2\2\u00c7"+
-    "\u00c8\5\34\17\16\u00c8\u00c9\b\17\1\2\u00c9\u00fa\3\2\2\2\u00ca\u00cb"+
-    "\f\f\2\2\u00cb\u00cc\t\6\2\2\u00cc\u00cd\5\34\17\r\u00cd\u00ce\b\17\1"+
-    "\2\u00ce\u00fa\3\2\2\2\u00cf\u00d0\f\13\2\2\u00d0\u00d1\t\7\2\2\u00d1"+
-    "\u00d2\5\34\17\f\u00d2\u00d3\b\17\1\2\u00d3\u00fa\3\2\2\2\u00d4\u00d5"+
-    "\f\n\2\2\u00d5\u00d6\t\b\2\2\u00d6\u00d7\5\34\17\13\u00d7\u00d8\b\17\1"+
-    "\2\u00d8\u00fa\3\2\2\2\u00d9\u00da\f\t\2\2\u00da\u00db\7,\2\2\u00db\u00dc"+
-    "\5\34\17\n\u00dc\u00dd\b\17\1\2\u00dd\u00fa\3\2\2\2\u00de\u00df\f\b\2"+
-    "\2\u00df\u00e0\7-\2\2\u00e0\u00e1\5\34\17\t\u00e1\u00e2\b\17\1\2\u00e2"+
-    "\u00fa\3\2\2\2\u00e3\u00e4\f\7\2\2\u00e4\u00e5\7.\2\2\u00e5\u00e6\5\34"+
-    "\17\b\u00e6\u00e7\b\17\1\2\u00e7\u00fa\3\2\2\2\u00e8\u00e9\f\6\2\2\u00e9"+
-    "\u00ea\7/\2\2\u00ea\u00eb\5\34\17\7\u00eb\u00ec\b\17\1\2\u00ec\u00fa\3"+
-    "\2\2\2\u00ed\u00ee\f\5\2\2\u00ee\u00ef\7\60\2\2\u00ef\u00f0\5\34\17\6"+
-    "\u00f0\u00f1\b\17\1\2\u00f1\u00fa\3\2\2\2\u00f2\u00f3\f\4\2\2\u00f3\u00f4"+
-    "\7\61\2\2\u00f4\u00f5\5\34\17\2\u00f5\u00f6\7\62\2\2\u00f6\u00f7\5\34"+
-    "\17\4\u00f7\u00f8\b\17\1\2\u00f8\u00fa\3\2\2\2\u00f9\u00c0\3\2\2\2\u00f9"+
-    "\u00c5\3\2\2\2\u00f9\u00ca\3\2\2\2\u00f9\u00cf\3\2\2\2\u00f9\u00d4\3\2"+
-    "\2\2\u00f9\u00d9\3\2\2\2\u00f9\u00de\3\2\2\2\u00f9\u00e3\3\2\2\2\u00f9"+
-    "\u00e8\3\2\2\2\u00f9\u00ed\3\2\2\2\u00f9\u00f2\3\2\2\2\u00fa\u00fd\3\2"+
-    "\2\2\u00fb\u00f9\3\2\2\2\u00fb\u00fc\3\2\2\2\u00fc\35\3\2\2\2\u00fd\u00fb"+
-    "\3\2\2\2\u00fe\u00ff\6\20\16\3\u00ff\u0100\t\t\2\2\u0100\u011c\5 \21\2"+
-    "\u0101\u0102\6\20\17\3\u0102\u0103\5 \21\2\u0103\u0104\t\t\2\2\u0104\u011c"+
-    "\3\2\2\2\u0105\u0106\6\20\20\3\u0106\u011c\5 \21\2\u0107\u0108\6\20\21"+
-    "\3\u0108\u0109\t\n\2\2\u0109\u011c\b\20\1\2\u010a\u010b\6\20\22\3\u010b"+
-    "\u010c\7G\2\2\u010c\u011c\b\20\1\2\u010d\u010e\6\20\23\3\u010e\u010f\7"+
-    "H\2\2\u010f\u011c\b\20\1\2\u0110\u0111\6\20\24\3\u0111\u0112\7I\2\2\u0112"+
-    "\u011c\b\20\1\2\u0113\u0114\6\20\25\3\u0114\u0115\t\13\2\2\u0115\u011c"+
-    "\5\36\20\2\u0116\u0117\7\t\2\2\u0117\u0118\5\22\n\2\u0118\u0119\7\n\2"+
-    "\2\u0119\u011a\5\36\20\2\u011a\u011c\3\2\2\2\u011b\u00fe\3\2\2\2\u011b"+
-    "\u0101\3\2\2\2\u011b\u0105\3\2\2\2\u011b\u0107\3\2\2\2\u011b\u010a\3\2"+
-    "\2\2\u011b\u010d\3\2\2\2\u011b\u0110\3\2\2\2\u011b\u0113\3\2\2\2\u011b"+
-    "\u0116\3\2\2\2\u011c\37\3\2\2\2\u011d\u0121\5\"\22\2\u011e\u0120\5$\23"+
-    "\2\u011f\u011e\3\2\2\2\u0120\u0123\3\2\2\2\u0121\u011f\3\2\2\2\u0121\u0122"+
-    "\3\2\2\2\u0122\u0140\3\2\2\2\u0123\u0121\3\2\2\2\u0124\u0125\5\22\n\2"+
-    "\u0125\u0129\5&\24\2\u0126\u0128\5$\23\2\u0127\u0126\3\2\2\2\u0128\u012b"+
-    "\3\2\2\2\u0129\u0127\3\2\2\2\u0129\u012a\3\2\2\2\u012a\u0140\3\2\2\2\u012b"+
-    "\u0129\3\2\2\2\u012c\u012d\7\26\2\2\u012d\u0132\7J\2\2\u012e\u012f\7\7"+
-    "\2\2\u012f\u0130\5\34\17\2\u0130\u0131\7\b\2\2\u0131\u0133\3\2\2\2\u0132"+
-    "\u012e\3\2\2\2\u0133\u0134\3\2\2\2\u0134\u0132\3\2\2\2\u0134\u0135\3\2"+
-    "\2\2\u0135\u013d\3\2\2\2\u0136\u013a\5&\24\2\u0137\u0139\5$\23\2\u0138"+
-    "\u0137\3\2\2\2\u0139\u013c\3\2\2\2\u013a\u0138\3\2\2\2\u013a\u013b\3\2"+
-    "\2\2\u013b\u013e\3\2\2\2\u013c\u013a\3\2\2\2\u013d\u0136\3\2\2\2\u013d"+
-    "\u013e\3\2\2\2\u013e\u0140\3\2\2\2\u013f\u011d\3\2\2\2\u013f\u0124\3\2"+
-    "\2\2\u013f\u012c\3\2\2\2\u0140!\3\2\2\2\u0141\u0142\6\22\26\3\u0142\u0143"+
-    "\7\t\2\2\u0143\u0144\5\34\17\2\u0144\u0145\7\n\2\2\u0145\u0146\b\22\1"+
-    "\2\u0146\u0152\3\2\2\2\u0147\u0148\6\22\27\3\u0148\u0149\7\t\2\2\u0149"+
-    "\u014a\5\36\20\2\u014a\u014b\7\n\2\2\u014b\u0152\3\2\2\2\u014c\u0152\7"+
-    "F\2\2\u014d\u0152\7K\2\2\u014e\u014f\7\26\2\2\u014f\u0150\7J\2\2\u0150"+
-    "\u0152\5*\26\2\u0151\u0141\3\2\2\2\u0151\u0147\3\2\2\2\u0151\u014c\3\2"+
-    "\2\2\u0151\u014d\3\2\2\2\u0151\u014e\3\2\2\2\u0152#\3\2\2\2\u0153\u0154"+
-    "\6\23\30\3\u0154\u0158\5&\24\2\u0155\u0156\6\23\31\3\u0156\u0158\5(\25"+
-    "\2\u0157\u0153\3\2\2\2\u0157\u0155\3\2\2\2\u0158%\3\2\2\2\u0159\u015a"+
-    "\7\13\2\2\u015a\u015b\7M\2\2\u015b\u015f\5*\26\2\u015c\u015d\7\13\2\2"+
-    "\u015d\u015f\t\f\2\2\u015e\u0159\3\2\2\2\u015e\u015c\3\2\2\2\u015f\'\3"+
-    "\2\2\2\u0160\u0161\7\7\2\2\u0161\u0162\5\34\17\2\u0162\u0163\7\b\2\2\u0163"+
-    ")\3\2\2\2\u0164\u016d\7\t\2\2\u0165\u016a\5,\27\2\u0166\u0167\7\f\2\2"+
-    "\u0167\u0169\5,\27\2\u0168\u0166\3\2\2\2\u0169\u016c\3\2\2\2\u016a\u0168"+
-    "\3\2\2\2\u016a\u016b\3\2\2\2\u016b\u016e\3\2\2\2\u016c\u016a\3\2\2\2\u016d"+
-    "\u0165\3\2\2\2\u016d\u016e\3\2\2\2\u016e\u016f\3\2\2\2\u016f\u0170\7\n"+
-    "\2\2\u0170+\3\2\2\2\u0171\u0174\5\34\17\2\u0172\u0174\5\24\13\2\u0173"+
-    "\u0171\3\2\2\2\u0173\u0172\3\2\2\2\u0174-\3\2\2\2!\61>FSW[`r{\177\u0085"+
-    "\u008e\u0098\u00a0\u00aa\u00be\u00f9\u00fb\u011b\u0121\u0129\u0134\u013a"+
-    "\u013d\u013f\u0151\u0157\u015e\u016a\u016d\u0173";
+    "\26\30\32\34\36 \"$&(*,\2\16\4\2\26\26KK\3\3\r\r\3\2\66A\3\2\34\36\3\2"+
+    "\37 \3\2!#\3\2$\'\3\2(+\3\2\64\65\3\2BE\4\2\32\33\37 \3\2LM\u019b\2\61"+
+    "\3\2\2\2\4{\3\2\2\2\6\177\3\2\2\2\b\u0081\3\2\2\2\n\u008a\3\2\2\2\f\u008e"+
+    "\3\2\2\2\16\u0090\3\2\2\2\20\u0092\3\2\2\2\22\u009b\3\2\2\2\24\u00a3\3"+
+    "\2\2\2\26\u00a7\3\2\2\2\30\u00ac\3\2\2\2\32\u00b3\3\2\2\2\34\u00be\3\2"+
+    "\2\2\36\u011b\3\2\2\2 \u013f\3\2\2\2\"\u0151\3\2\2\2$\u0157\3\2\2\2&\u015e"+
+    "\3\2\2\2(\u0160\3\2\2\2*\u0164\3\2\2\2,\u0173\3\2\2\2.\60\5\4\3\2/.\3"+
+    "\2\2\2\60\63\3\2\2\2\61/\3\2\2\2\61\62\3\2\2\2\62\64\3\2\2\2\63\61\3\2"+
+    "\2\2\64\65\7\2\2\3\65\3\3\2\2\2\66\67\7\16\2\2\678\7\t\2\289\5\34\17\2"+
+    "9:\7\n\2\2:>\5\6\4\2;<\7\17\2\2<?\5\6\4\2=?\6\3\2\2>;\3\2\2\2>=\3\2\2"+
+    "\2?|\3\2\2\2@A\7\20\2\2AB\7\t\2\2BC\5\34\17\2CF\7\n\2\2DG\5\6\4\2EG\5"+
+    "\n\6\2FD\3\2\2\2FE\3\2\2\2G|\3\2\2\2HI\7\21\2\2IJ\5\b\5\2JK\7\20\2\2K"+
+    "L\7\t\2\2LM\5\34\17\2MN\7\n\2\2NO\5\32\16\2O|\3\2\2\2PQ\7\22\2\2QS\7\t"+
+    "\2\2RT\5\f\7\2SR\3\2\2\2ST\3\2\2\2TU\3\2\2\2UW\7\r\2\2VX\5\34\17\2WV\3"+
+    "\2\2\2WX\3\2\2\2XY\3\2\2\2Y[\7\r\2\2Z\\\5\16\b\2[Z\3\2\2\2[\\\3\2\2\2"+
+    "\\]\3\2\2\2]`\7\n\2\2^a\5\6\4\2_a\5\n\6\2`^\3\2\2\2`_\3\2\2\2a|\3\2\2"+
+    "\2bc\5\20\t\2cd\5\32\16\2d|\3\2\2\2ef\7\23\2\2f|\5\32\16\2gh\7\24\2\2"+
+    "h|\5\32\16\2ij\7\25\2\2jk\5\34\17\2kl\5\32\16\2l|\3\2\2\2mn\7\27\2\2n"+
+    "p\5\b\5\2oq\5\30\r\2po\3\2\2\2qr\3\2\2\2rp\3\2\2\2rs\3\2\2\2s|\3\2\2\2"+
+    "tu\7\31\2\2uv\5\34\17\2vw\5\32\16\2w|\3\2\2\2xy\5\34\17\2yz\5\32\16\2"+
+    "z|\3\2\2\2{\66\3\2\2\2{@\3\2\2\2{H\3\2\2\2{P\3\2\2\2{b\3\2\2\2{e\3\2\2"+
+    "\2{g\3\2\2\2{i\3\2\2\2{m\3\2\2\2{t\3\2\2\2{x\3\2\2\2|\5\3\2\2\2}\u0080"+
+    "\5\b\5\2~\u0080\5\4\3\2\177}\3\2\2\2\177~\3\2\2\2\u0080\7\3\2\2\2\u0081"+
+    "\u0085\7\5\2\2\u0082\u0084\5\4\3\2\u0083\u0082\3\2\2\2\u0084\u0087\3\2"+
+    "\2\2\u0085\u0083\3\2\2\2\u0085\u0086\3\2\2\2\u0086\u0088\3\2\2\2\u0087"+
+    "\u0085\3\2\2\2\u0088\u0089\7\6\2\2\u0089\t\3\2\2\2\u008a\u008b\7\r\2\2"+
+    "\u008b\13\3\2\2\2\u008c\u008f\5\20\t\2\u008d\u008f\5\34\17\2\u008e\u008c"+
+    "\3\2\2\2\u008e\u008d\3\2\2\2\u008f\r\3\2\2\2\u0090\u0091\5\34\17\2\u0091"+
+    "\17\3\2\2\2\u0092\u0093\5\22\n\2\u0093\u0098\5\26\f\2\u0094\u0095\7\f"+
+    "\2\2\u0095\u0097\5\26\f\2\u0096\u0094\3\2\2\2\u0097\u009a\3\2\2\2\u0098"+
+    "\u0096\3\2\2\2\u0098\u0099\3\2\2\2\u0099\21\3\2\2\2\u009a\u0098\3\2\2"+
+    "\2\u009b\u00a0\7J\2\2\u009c\u009d\7\7\2\2\u009d\u009f\7\b\2\2\u009e\u009c"+
+    "\3\2\2\2\u009f\u00a2\3\2\2\2\u00a0\u009e\3\2\2\2\u00a0\u00a1\3\2\2\2\u00a1"+
+    "\23\3\2\2\2\u00a2\u00a0\3\2\2\2\u00a3\u00a4\7J\2\2\u00a4\u00a5\7\63\2"+
+    "\2\u00a5\u00a6\t\2\2\2\u00a6\25\3\2\2\2\u00a7\u00aa\7K\2\2\u00a8\u00a9"+
+    "\7\66\2\2\u00a9\u00ab\5\34\17\2\u00aa\u00a8\3\2\2\2\u00aa\u00ab\3\2\2"+
+    "\2\u00ab\27\3\2\2\2\u00ac\u00ad\7\30\2\2\u00ad\u00ae\7\t\2\2\u00ae\u00af"+
+    "\7J\2\2\u00af\u00b0\7K\2\2\u00b0\u00b1\7\n\2\2\u00b1\u00b2\5\b\5\2\u00b2"+
+    "\31\3\2\2\2\u00b3\u00b4\t\3\2\2\u00b4\33\3\2\2\2\u00b5\u00b6\b\17\1\2"+
+    "\u00b6\u00b7\5 \21\2\u00b7\u00b8\t\4\2\2\u00b8\u00b9\5\34\17\3\u00b9\u00ba"+
+    "\b\17\1\2\u00ba\u00bf\3\2\2\2\u00bb\u00bc\5\36\20\2\u00bc\u00bd\b\17\1"+
+    "\2\u00bd\u00bf\3\2\2\2\u00be\u00b5\3\2\2\2\u00be\u00bb\3\2\2\2\u00bf\u00fb"+
+    "\3\2\2\2\u00c0\u00c1\f\16\2\2\u00c1\u00c2\t\5\2\2\u00c2\u00c3\5\34\17"+
+    "\17\u00c3\u00c4\b\17\1\2\u00c4\u00fa\3\2\2\2\u00c5\u00c6\f\r\2\2\u00c6"+
+    "\u00c7\t\6\2\2\u00c7\u00c8\5\34\17\16\u00c8\u00c9\b\17\1\2\u00c9\u00fa"+
+    "\3\2\2\2\u00ca\u00cb\f\f\2\2\u00cb\u00cc\t\7\2\2\u00cc\u00cd\5\34\17\r"+
+    "\u00cd\u00ce\b\17\1\2\u00ce\u00fa\3\2\2\2\u00cf\u00d0\f\13\2\2\u00d0\u00d1"+
+    "\t\b\2\2\u00d1\u00d2\5\34\17\f\u00d2\u00d3\b\17\1\2\u00d3\u00fa\3\2\2"+
+    "\2\u00d4\u00d5\f\n\2\2\u00d5\u00d6\t\t\2\2\u00d6\u00d7\5\34\17\13\u00d7"+
+    "\u00d8\b\17\1\2\u00d8\u00fa\3\2\2\2\u00d9\u00da\f\t\2\2\u00da\u00db\7"+
+    ",\2\2\u00db\u00dc\5\34\17\n\u00dc\u00dd\b\17\1\2\u00dd\u00fa\3\2\2\2\u00de"+
+    "\u00df\f\b\2\2\u00df\u00e0\7-\2\2\u00e0\u00e1\5\34\17\t\u00e1\u00e2\b"+
+    "\17\1\2\u00e2\u00fa\3\2\2\2\u00e3\u00e4\f\7\2\2\u00e4\u00e5\7.\2\2\u00e5"+
+    "\u00e6\5\34\17\b\u00e6\u00e7\b\17\1\2\u00e7\u00fa\3\2\2\2\u00e8\u00e9"+
+    "\f\6\2\2\u00e9\u00ea\7/\2\2\u00ea\u00eb\5\34\17\7\u00eb\u00ec\b\17\1\2"+
+    "\u00ec\u00fa\3\2\2\2\u00ed\u00ee\f\5\2\2\u00ee\u00ef\7\60\2\2\u00ef\u00f0"+
+    "\5\34\17\6\u00f0\u00f1\b\17\1\2\u00f1\u00fa\3\2\2\2\u00f2\u00f3\f\4\2"+
+    "\2\u00f3\u00f4\7\61\2\2\u00f4\u00f5\5\34\17\2\u00f5\u00f6\7\62\2\2\u00f6"+
+    "\u00f7\5\34\17\4\u00f7\u00f8\b\17\1\2\u00f8\u00fa\3\2\2\2\u00f9\u00c0"+
+    "\3\2\2\2\u00f9\u00c5\3\2\2\2\u00f9\u00ca\3\2\2\2\u00f9\u00cf\3\2\2\2\u00f9"+
+    "\u00d4\3\2\2\2\u00f9\u00d9\3\2\2\2\u00f9\u00de\3\2\2\2\u00f9\u00e3\3\2"+
+    "\2\2\u00f9\u00e8\3\2\2\2\u00f9\u00ed\3\2\2\2\u00f9\u00f2\3\2\2\2\u00fa"+
+    "\u00fd\3\2\2\2\u00fb\u00f9\3\2\2\2\u00fb\u00fc\3\2\2\2\u00fc\35\3\2\2"+
+    "\2\u00fd\u00fb\3\2\2\2\u00fe\u00ff\6\20\16\3\u00ff\u0100\t\n\2\2\u0100"+
+    "\u011c\5 \21\2\u0101\u0102\6\20\17\3\u0102\u0103\5 \21\2\u0103\u0104\t"+
+    "\n\2\2\u0104\u011c\3\2\2\2\u0105\u0106\6\20\20\3\u0106\u011c\5 \21\2\u0107"+
+    "\u0108\6\20\21\3\u0108\u0109\t\13\2\2\u0109\u011c\b\20\1\2\u010a\u010b"+
+    "\6\20\22\3\u010b\u010c\7G\2\2\u010c\u011c\b\20\1\2\u010d\u010e\6\20\23"+
+    "\3\u010e\u010f\7H\2\2\u010f\u011c\b\20\1\2\u0110\u0111\6\20\24\3\u0111"+
+    "\u0112\7I\2\2\u0112\u011c\b\20\1\2\u0113\u0114\6\20\25\3\u0114\u0115\t"+
+    "\f\2\2\u0115\u011c\5\36\20\2\u0116\u0117\7\t\2\2\u0117\u0118\5\22\n\2"+
+    "\u0118\u0119\7\n\2\2\u0119\u011a\5\36\20\2\u011a\u011c\3\2\2\2\u011b\u00fe"+
+    "\3\2\2\2\u011b\u0101\3\2\2\2\u011b\u0105\3\2\2\2\u011b\u0107\3\2\2\2\u011b"+
+    "\u010a\3\2\2\2\u011b\u010d\3\2\2\2\u011b\u0110\3\2\2\2\u011b\u0113\3\2"+
+    "\2\2\u011b\u0116\3\2\2\2\u011c\37\3\2\2\2\u011d\u0121\5\"\22\2\u011e\u0120"+
+    "\5$\23\2\u011f\u011e\3\2\2\2\u0120\u0123\3\2\2\2\u0121\u011f\3\2\2\2\u0121"+
+    "\u0122\3\2\2\2\u0122\u0140\3\2\2\2\u0123\u0121\3\2\2\2\u0124\u0125\5\22"+
+    "\n\2\u0125\u0129\5&\24\2\u0126\u0128\5$\23\2\u0127\u0126\3\2\2\2\u0128"+
+    "\u012b\3\2\2\2\u0129\u0127\3\2\2\2\u0129\u012a\3\2\2\2\u012a\u0140\3\2"+
+    "\2\2\u012b\u0129\3\2\2\2\u012c\u012d\7\26\2\2\u012d\u0132\7J\2\2\u012e"+
+    "\u012f\7\7\2\2\u012f\u0130\5\34\17\2\u0130\u0131\7\b\2\2\u0131\u0133\3"+
+    "\2\2\2\u0132\u012e\3\2\2\2\u0133\u0134\3\2\2\2\u0134\u0132\3\2\2\2\u0134"+
+    "\u0135\3\2\2\2\u0135\u013d\3\2\2\2\u0136\u013a\5&\24\2\u0137\u0139\5$"+
+    "\23\2\u0138\u0137\3\2\2\2\u0139\u013c\3\2\2\2\u013a\u0138\3\2\2\2\u013a"+
+    "\u013b\3\2\2\2\u013b\u013e\3\2\2\2\u013c\u013a\3\2\2\2\u013d\u0136\3\2"+
+    "\2\2\u013d\u013e\3\2\2\2\u013e\u0140\3\2\2\2\u013f\u011d\3\2\2\2\u013f"+
+    "\u0124\3\2\2\2\u013f\u012c\3\2\2\2\u0140!\3\2\2\2\u0141\u0142\6\22\26"+
+    "\3\u0142\u0143\7\t\2\2\u0143\u0144\5\34\17\2\u0144\u0145\7\n\2\2\u0145"+
+    "\u0146\b\22\1\2\u0146\u0152\3\2\2\2\u0147\u0148\6\22\27\3\u0148\u0149"+
+    "\7\t\2\2\u0149\u014a\5\36\20\2\u014a\u014b\7\n\2\2\u014b\u0152\3\2\2\2"+
+    "\u014c\u0152\7F\2\2\u014d\u0152\7K\2\2\u014e\u014f\7\26\2\2\u014f\u0150"+
+    "\7J\2\2\u0150\u0152\5*\26\2\u0151\u0141\3\2\2\2\u0151\u0147\3\2\2\2\u0151"+
+    "\u014c\3\2\2\2\u0151\u014d\3\2\2\2\u0151\u014e\3\2\2\2\u0152#\3\2\2\2"+
+    "\u0153\u0154\6\23\30\3\u0154\u0158\5&\24\2\u0155\u0156\6\23\31\3\u0156"+
+    "\u0158\5(\25\2\u0157\u0153\3\2\2\2\u0157\u0155\3\2\2\2\u0158%\3\2\2\2"+
+    "\u0159\u015a\7\13\2\2\u015a\u015b\7M\2\2\u015b\u015f\5*\26\2\u015c\u015d"+
+    "\7\13\2\2\u015d\u015f\t\r\2\2\u015e\u0159\3\2\2\2\u015e\u015c\3\2\2\2"+
+    "\u015f\'\3\2\2\2\u0160\u0161\7\7\2\2\u0161\u0162\5\34\17\2\u0162\u0163"+
+    "\7\b\2\2\u0163)\3\2\2\2\u0164\u016d\7\t\2\2\u0165\u016a\5,\27\2\u0166"+
+    "\u0167\7\f\2\2\u0167\u0169\5,\27\2\u0168\u0166\3\2\2\2\u0169\u016c\3\2"+
+    "\2\2\u016a\u0168\3\2\2\2\u016a\u016b\3\2\2\2\u016b\u016e\3\2\2\2\u016c"+
+    "\u016a\3\2\2\2\u016d\u0165\3\2\2\2\u016d\u016e\3\2\2\2\u016e\u016f\3\2"+
+    "\2\2\u016f\u0170\7\n\2\2\u0170+\3\2\2\2\u0171\u0174\5\34\17\2\u0172\u0174"+
+    "\5\24\13\2\u0173\u0171\3\2\2\2\u0173\u0172\3\2\2\2\u0174-\3\2\2\2!\61"+
+    ">FSW[`r{\177\u0085\u008e\u0098\u00a0\u00aa\u00be\u00f9\u00fb\u011b\u0121"+
+    "\u0129\u0134\u013a\u013d\u013f\u0151\u0157\u015e\u016a\u016d\u0173";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index 0f155ceb4c1..66d854fd220 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -391,7 +391,15 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
 
     @Override
     public Object visitFuncref(FuncrefContext ctx) {
-        return new EFunctionRef(location(ctx), ctx.TYPE().getText(), ctx.ID().getText());
+        final String methodText;
+        if (ctx.ID() != null) {
+            methodText = ctx.ID().getText();
+        } else if (ctx.NEW() != null ){
+            methodText = ctx.NEW().getText();
+        } else {
+            throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
+        }
+        return new EFunctionRef(location(ctx), ctx.TYPE().getText(), methodText);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index ec2bd6f7e47..dd050a549ea 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -19,16 +19,24 @@
 
 package org.elasticsearch.painless.node;
 
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.FunctionRef;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.Variables;
 
+import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE;
+
+import java.lang.invoke.LambdaMetafactory;
+
 /**
  * Represents a function reference.
  */
 public class EFunctionRef extends AExpression {
-    public String type;
-    public String call;
+    public final String type;
+    public final String call;
+    
+    private FunctionRef ref;
 
     public EFunctionRef(Location location, String type, String call) {
         super(location);
@@ -39,12 +47,35 @@ public class EFunctionRef extends AExpression {
 
     @Override
     void analyze(Variables variables) {
-        throw createError(new UnsupportedOperationException("Function references [" + type + "::" + call + 
-                                                            "] are not currently supported."));
+        if (expected == null) {
+            ref = null;
+            actual = Definition.getType("String");
+        } else {
+            try {
+                ref = new FunctionRef(expected.clazz, type, call);
+            } catch (IllegalArgumentException e) {
+                throw createError(e);
+            }
+            actual = expected;
+        }
     }
 
     @Override
     void write(MethodWriter writer) {
-        throw createError(new IllegalStateException("Illegal tree structure."));
+        if (ref == null) {
+            writer.push(type + "." + call);
+        } else {
+            writer.writeDebugInfo(location);
+            // currently if the interface differs, we ask for a bridge, but maybe we should do smarter checking?
+            // either way, stuff will fail if its wrong :)
+            if (ref.interfaceType.equals(ref.samMethodType)) {
+                writer.invokeDynamic(ref.invokedName, ref.invokedType.getDescriptor(), LAMBDA_BOOTSTRAP_HANDLE, 
+                                     ref.samMethodType, ref.implMethod, ref.samMethodType, 0);
+            } else {
+                writer.invokeDynamic(ref.invokedName, ref.invokedType.getDescriptor(), LAMBDA_BOOTSTRAP_HANDLE, 
+                                     ref.samMethodType, ref.implMethod, ref.samMethodType, 
+                                     LambdaMetafactory.FLAG_BRIDGES, 1, ref.interfaceType);
+            }
+        }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
index f7e91790108..7696e775ab6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
@@ -82,7 +82,7 @@ public final class LCall extends ALink {
         }
 
         throw createError(new IllegalArgumentException("Unknown call [" + name + "] with [" + arguments.size() +
-                                                 "] arguments on type [" + struct.name + "]."));
+                                                       "] arguments on type [" + struct.name + "]."));
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index 5fb2ff870ef..b51d7612297 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -61,13 +61,13 @@ final class LDefArray extends ALink implements IDefLink {
     void load(MethodWriter writer) {
         writer.writeDebugInfo(location);
         String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type, index.actual.type);
-        writer.invokeDynamic("arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ARRAY_LOAD);
+        writer.invokeDynamic("arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ARRAY_LOAD, 0);
     }
 
     @Override
     void store(MethodWriter writer) {
         writer.writeDebugInfo(location);
         String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, index.actual.type, after.type);
-        writer.invokeDynamic("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ARRAY_STORE);
+        writer.invokeDynamic("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ARRAY_STORE, 0);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index 3b07d367eba..2bd31999ed2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -36,6 +36,7 @@ final class LDefCall extends ALink implements IDefLink {
 
     final String name;
     final List<AExpression> arguments;
+    long recipe;
 
     LDefCall(Location location, String name, List<AExpression> arguments) {
         super(location, -1);
@@ -46,9 +47,20 @@ final class LDefCall extends ALink implements IDefLink {
 
     @Override
     ALink analyze(Variables variables) {
+        if (arguments.size() > 63) {
+            // technically, the limitation is just methods with > 63 params, containing method references.
+            // this is because we are lazy and use a long as a bitset. we can always change to a "string" if need be.
+            // but NEED NOT BE. nothing with this many parameters is in the whitelist and we do not support varargs.
+            throw new UnsupportedOperationException("methods with > 63 arguments are currently not supported");
+        }
+        
+        recipe = 0;
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
 
+            if (expression instanceof EFunctionRef) {
+                recipe |= (1L << argument); // mark argument as deferred reference
+            }
             expression.internal = true;
             expression.analyze(variables);
             expression.expected = expression.actual;
@@ -84,7 +96,7 @@ final class LDefCall extends ALink implements IDefLink {
         // return value
         signature.append(after.type.getDescriptor());
 
-        writer.invokeDynamic(name, signature.toString(), DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.METHOD_CALL);
+        writer.invokeDynamic(name, signature.toString(), DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.METHOD_CALL, recipe);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index 2b32d75d89c..c9e94ce887e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -58,13 +58,13 @@ final class LDefField extends ALink implements IDefLink {
     void load(MethodWriter writer) {
         writer.writeDebugInfo(location);
         String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type);
-        writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.LOAD);
+        writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.LOAD, 0);
     }
 
     @Override
     void store(MethodWriter writer) {
         writer.writeDebugInfo(location);
         String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, after.type);
-        writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.STORE);
+        writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.STORE, 0);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
index a27aa25754d..2f80b254350 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
@@ -21,7 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.Definition.Constructor;
+import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Struct;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Variables;
@@ -37,7 +37,7 @@ public final class LNewObj extends ALink {
     final String type;
     final List<AExpression> arguments;
 
-    Constructor constructor;
+    Method constructor;
 
     public LNewObj(Location location, String type, List<AExpression> arguments) {
         super(location, -1);
@@ -63,7 +63,7 @@ public final class LNewObj extends ALink {
         }
 
         Struct struct = type.struct;
-        constructor = struct.constructors.get(new Definition.MethodKey("new", arguments.size()));
+        constructor = struct.constructors.get(new Definition.MethodKey("<init>", arguments.size()));
 
         if (constructor != null) {
             Type[] types = new Type[constructor.arguments.size()];
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
index 755c8d44c0f..ec1a46e633f 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
@@ -63,6 +63,7 @@ class Boolean -> java.lang.Boolean extends Comparable,Object {
   Boolean FALSE
   boolean booleanValue()
   int compare(boolean,boolean)
+  int compareTo(Boolean)
   int hashCode(boolean)
   boolean logicalAnd(boolean,boolean)
   boolean logicalOr(boolean,boolean)
@@ -78,6 +79,7 @@ class Byte -> java.lang.Byte extends Number,Comparable,Object {
   byte MIN_VALUE
   int SIZE
   int compare(byte,byte)
+  int compareTo(Byte)
   Byte decode(String)
   int hashCode(byte)
   byte parseByte(String)
@@ -163,6 +165,7 @@ class Character -> java.lang.Character extends Comparable,Object {
   int codePointBefore(CharSequence,int)
   int codePointCount(CharSequence,int,int)
   int compare(char,char)
+  int compareTo(Character)
   int digit(int,int)
   char forDigit(int,int)
   byte getDirectionality(int)
@@ -454,6 +457,7 @@ class Double -> java.lang.Double extends Number,Comparable,Object {
   double POSITIVE_INFINITY
   int SIZE
   int compare(double,double)
+  int compareTo(Double)
   long doubleToLongBits(double)
   long doubleToRawLongBits(double)
   int hashCode(double)
@@ -473,6 +477,7 @@ class Double -> java.lang.Double extends Number,Comparable,Object {
 }
 
 class Enum -> java.lang.Enum extends Comparable,Object {
+  int compareTo(Enum)
   String name();
   int ordinal();
 }
@@ -489,6 +494,7 @@ class Float -> java.lang.Float extends Number,Comparable,Object {
   float POSITIVE_INFINITY
   int SIZE
   int compare(float,float)
+  int compareTo(Float)
   int floatToIntBits(float)
   int floatToRawIntBits(float)
   int hashCode(float)
@@ -516,6 +522,7 @@ class Integer -> java.lang.Integer extends Number,Comparable,Object {
   int SIZE
   int bitCount(int)
   int compare(int,int)
+  int compareTo(Integer)
   int compareUnsigned(int,int)
   Integer decode(String)
   int divideUnsigned(int,int)
@@ -555,6 +562,7 @@ class Long -> java.lang.Long extends Number,Comparable,Object {
   int SIZE
   int bitCount(long)
   int compare(long,long)
+  int compareTo(Long)
   int compareUnsigned(long,long)
   Long decode(String)
   long divideUnsigned(long,long)
@@ -587,7 +595,7 @@ class Long -> java.lang.Long extends Number,Comparable,Object {
   Long valueOf(String,int)
 }
 
-class Math -> java.lang.Math {
+class Math -> java.lang.Math extends Object {
   double E
   double PI
   double abs(double)
@@ -658,6 +666,7 @@ class Short -> java.lang.Short extends Number,Comparable,Object {
   short MIN_VALUE
   int SIZE
   int compare(short,short)
+  int compareTo(Short)
   Short decode(String)
   int hashCode(short)
   short parseShort(String)
@@ -679,7 +688,7 @@ class StackTraceElement -> java.lang.StackTraceElement extends Object {
   boolean isNativeMethod()
 }
 
-class StrictMath -> java.lang.StrictMath {
+class StrictMath -> java.lang.StrictMath extends Object {
   double E
   double PI
   double abs(double)
@@ -726,6 +735,7 @@ class String -> java.lang.String extends CharSequence,Comparable,Object {
   int codePointAt(int)
   int codePointBefore(int)
   int codePointCount(int,int)
+  int compareTo(String)
   int compareToIgnoreCase(String)
   String concat(String)
   boolean contains(CharSequence)
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.math.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.math.txt
index 42680f8a428..e3d13a0959c 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.math.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.math.txt
@@ -35,6 +35,7 @@ class BigDecimal -> java.math.BigDecimal extends Number,Comparable,Object {
   BigDecimal add(BigDecimal)
   BigDecimal add(BigDecimal,MathContext)
   byte byteValueExact()
+  int compareTo(BigDecimal)
   BigDecimal divide(BigDecimal)
   BigDecimal divide(BigDecimal,MathContext)
   BigDecimal[] divideAndRemainder(BigDecimal)
@@ -90,6 +91,7 @@ class BigInteger -> java.math.BigInteger extends Number,Comparable,Object {
   int bitLength()
   byte byteValueExact()
   BigInteger clearBit(int)
+  int compareTo(BigInteger)
   BigInteger divide(BigInteger)
   BigInteger[] divideAndRemainder(BigInteger)
   BigInteger flipBit(int)
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.text.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.text.txt
index eac725cfb94..2e93b14ab3d 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.text.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.text.txt
@@ -149,6 +149,7 @@ class CollationElementIterator -> java.text.CollationElementIterator extends Obj
 }
 
 class CollationKey -> java.text.CollationKey extends Comparable,Object {
+  int compareTo(CollationKey)
   String getSourceString()
   byte[] toByteArray()
 }
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.chrono.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.chrono.txt
index 8dd0478605e..7374efea04b 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.chrono.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.chrono.txt
@@ -24,8 +24,9 @@
 
 #### Interfaces
 
-class ChronoLocalDate -> java.time.chrono.ChronoLocalDate extends Comparable,Temporal,TemporalAccessor,TemporalAdjuster {
+class ChronoLocalDate -> java.time.chrono.ChronoLocalDate extends Temporal,TemporalAccessor,TemporalAdjuster,Comparable {
   ChronoLocalDateTime atTime(LocalTime)
+  int compareTo(ChronoLocalDate)
   boolean equals(Object)
   String format(DateTimeFormatter)
   ChronoLocalDate from(TemporalAccessor)
@@ -50,8 +51,9 @@ class ChronoLocalDate -> java.time.chrono.ChronoLocalDate extends Comparable,Tem
   ChronoLocalDate with(TemporalField,long)
 }
 
-class ChronoLocalDateTime -> java.time.chrono.ChronoLocalDateTime extends Comparable,Temporal,TemporalAccessor,TemporalAdjuster {
+class ChronoLocalDateTime -> java.time.chrono.ChronoLocalDateTime extends Temporal,TemporalAccessor,TemporalAdjuster,Comparable {
   ChronoZonedDateTime atZone(ZoneId)
+  int compareTo(ChronoLocalDateTime)
   boolean equals(Object)
   String format(DateTimeFormatter)
   ChronoLocalDateTime from(TemporalAccessor)
@@ -75,6 +77,7 @@ class ChronoLocalDateTime -> java.time.chrono.ChronoLocalDateTime extends Compar
 }
 
 class Chronology -> java.time.chrono.Chronology extends Comparable {
+  int compareTo(Chronology)
   ChronoLocalDate date(TemporalAccessor)
   ChronoLocalDate date(Era,int,int,int)
   ChronoLocalDate date(int,int,int)
@@ -121,7 +124,8 @@ class ChronoPeriod -> java.time.chrono.ChronoPeriod extends TemporalAmount {
   String toString()
 }
 
-class ChronoZonedDateTime -> java.time.chrono.ChronoZonedDateTime extends Comparable,Temporal,TemporalAccessor {
+class ChronoZonedDateTime -> java.time.chrono.ChronoZonedDateTime extends Temporal,TemporalAccessor,Comparable {
+  int compareTo(ChronoZonedDateTime)
   boolean equals(Object)
   String format(DateTimeFormatter)
   ChronoZonedDateTime from(TemporalAccessor)
@@ -158,10 +162,10 @@ class Era -> java.time.chrono.Era extends TemporalAccessor,TemporalAdjuster {
 
 #### Classes
 
-class AbstractChronology -> java.time.chrono.Chronology extends Comparable,Chronology,Object {
+class AbstractChronology -> java.time.chrono.Chronology extends Chronology,Comparable,Object {
 }
 
-class HijrahChronology -> java.time.chrono.HijrahChronology extends AbstractChronology,Comparable,Chronology,Object {
+class HijrahChronology -> java.time.chrono.HijrahChronology extends AbstractChronology,Chronology,Comparable,Object {
   HijrahChronology INSTANCE
   HijrahDate date(TemporalAccessor)
   HijrahDate date(int,int,int)
@@ -172,25 +176,26 @@ class HijrahChronology -> java.time.chrono.HijrahChronology extends AbstractChro
   HijrahDate dateYearDay(int,int)
   HijrahDate dateYearDay(Era,int,int)
   HijrahEra eraOf(int)
+  HijrahDate resolveDate(Map,ResolverStyle)
 }
 
-class HijrahDate -> java.time.chrono.HijrahDate extends Comparable,ChronoLocalDate,Temporal,TemporalAccessor,TemporalAdjuster,Object {
-  HijrahDate now()
-  HijrahDate now(ZoneId)
-  HijrahDate of(int,int,int)
+class HijrahDate -> java.time.chrono.HijrahDate extends ChronoLocalDate,Temporal,TemporalAccessor,TemporalAdjuster,Comparable,Object {
   HijrahDate from(TemporalAccessor)
   HijrahChronology getChronology()
   HijrahEra getEra()
+  HijrahDate minus(TemporalAmount)
+  HijrahDate minus(long,TemporalUnit)
+  HijrahDate now()
+  HijrahDate now(ZoneId)
+  HijrahDate of(int,int,int)
+  HijrahDate plus(TemporalAmount)
+  HijrahDate plus(long,TemporalUnit)
   HijrahDate with(TemporalField,long)
   HijrahDate with(TemporalAdjuster)
   HijrahDate withVariant(HijrahChronology)
-  HijrahDate plus(TemporalAmount)
-  HijrahDate minus(TemporalAmount)
-  HijrahDate plus(long,TemporalUnit)
-  HijrahDate minus(long,TemporalUnit)
 }
 
-class IsoChronology -> java.time.chrono.IsoChronology extends AbstractChronology,Comparable,Chronology,Object {
+class IsoChronology -> java.time.chrono.IsoChronology extends AbstractChronology,Chronology,Comparable,Object {
   IsoChronology INSTANCE
   LocalDate date(TemporalAccessor)
   LocalDate date(int,int,int)
@@ -201,9 +206,14 @@ class IsoChronology -> java.time.chrono.IsoChronology extends AbstractChronology
   LocalDate dateYearDay(int,int)
   LocalDate dateYearDay(Era,int,int)
   IsoEra eraOf(int)
+  LocalDateTime localDateTime(TemporalAccessor)
+  Period period(int,int,int)
+  LocalDate resolveDate(Map,ResolverStyle)
+  ZonedDateTime zonedDateTime(TemporalAccessor)
+  ZonedDateTime zonedDateTime(Instant,ZoneId)
 }
 
-class JapaneseChronology -> java.time.chrono.JapaneseChronology extends AbstractChronology,Comparable,Chronology,Object {
+class JapaneseChronology -> java.time.chrono.JapaneseChronology extends AbstractChronology,Chronology,Comparable,Object {
   JapaneseChronology INSTANCE
   JapaneseDate date(TemporalAccessor)
   JapaneseDate date(int,int,int)
@@ -214,9 +224,10 @@ class JapaneseChronology -> java.time.chrono.JapaneseChronology extends Abstract
   JapaneseDate dateYearDay(int,int)
   JapaneseDate dateYearDay(Era,int,int)
   JapaneseEra eraOf(int)
+  JapaneseDate resolveDate(Map,ResolverStyle)
 }
 
-class JapaneseDate -> java.time.chrono.JapaneseDate extends Comparable,ChronoLocalDate,Temporal,TemporalAccessor,TemporalAdjuster,Object {
+class JapaneseDate -> java.time.chrono.JapaneseDate extends ChronoLocalDate,Temporal,TemporalAccessor,TemporalAdjuster,Comparable,Object {
   JapaneseDate now()
   JapaneseDate now(ZoneId)
   JapaneseDate of(int,int,int)
@@ -242,7 +253,7 @@ class JapaneseEra -> java.time.chrono.JapaneseEra extends Era,TemporalAccessor,T
   JapaneseEra[] values()
 }
 
-class MinguoChronology -> java.time.chrono.MinguoChronology extends AbstractChronology,Comparable,Chronology,Object {
+class MinguoChronology -> java.time.chrono.MinguoChronology extends AbstractChronology,Chronology,Comparable,Object {
   MinguoChronology INSTANCE
   MinguoDate date(TemporalAccessor)
   MinguoDate date(int,int,int)
@@ -253,9 +264,10 @@ class MinguoChronology -> java.time.chrono.MinguoChronology extends AbstractChro
   MinguoDate dateYearDay(int,int)
   MinguoDate dateYearDay(Era,int,int)
   MinguoEra eraOf(int)
+  MinguoDate resolveDate(Map,ResolverStyle)
 }
 
-class MinguoDate -> java.time.chrono.MinguoDate extends Comparable,ChronoLocalDate,Temporal,TemporalAccessor,TemporalAdjuster,Object {
+class MinguoDate -> java.time.chrono.MinguoDate extends ChronoLocalDate,Temporal,TemporalAccessor,TemporalAdjuster,Comparable,Object {
   MinguoDate now()
   MinguoDate now(ZoneId)
   MinguoDate of(int,int,int)
@@ -270,7 +282,7 @@ class MinguoDate -> java.time.chrono.MinguoDate extends Comparable,ChronoLocalDa
   MinguoDate minus(long,TemporalUnit)
 }
 
-class ThaiBuddhistChronology -> java.time.chrono.ThaiBuddhistChronology extends AbstractChronology,Comparable,Chronology,Object {
+class ThaiBuddhistChronology -> java.time.chrono.ThaiBuddhistChronology extends AbstractChronology,Chronology,Comparable,Object {
   ThaiBuddhistChronology INSTANCE
   ThaiBuddhistDate date(TemporalAccessor)
   ThaiBuddhistDate date(int,int,int)
@@ -281,9 +293,10 @@ class ThaiBuddhistChronology -> java.time.chrono.ThaiBuddhistChronology extends
   ThaiBuddhistDate dateYearDay(int,int)
   ThaiBuddhistDate dateYearDay(Era,int,int)
   ThaiBuddhistEra eraOf(int)
+  ThaiBuddhistDate resolveDate(Map,ResolverStyle)
 }
 
-class ThaiBuddhistDate -> java.time.chrono.ThaiBuddhistDate extends Comparable,ChronoLocalDate,Temporal,TemporalAccessor,TemporalAdjuster,Object {
+class ThaiBuddhistDate -> java.time.chrono.ThaiBuddhistDate extends ChronoLocalDate,Temporal,TemporalAccessor,TemporalAdjuster,Comparable,Object {
   ThaiBuddhistDate now()
   ThaiBuddhistDate now(ZoneId)
   ThaiBuddhistDate of(int,int,int)
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.txt
index 1b7d19a6c9c..4481004fdf6 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.txt
@@ -43,6 +43,7 @@ class Duration -> java.time.Duration extends Comparable,TemporalAmount,Object {
   Duration ZERO
   Duration abs()
   Duration between(Temporal,Temporal)
+  int compareTo(Duration)
   Duration dividedBy(long)
   Duration from(TemporalAmount)
   int getNano()
@@ -91,6 +92,7 @@ class Instant -> java.time.Instant extends Comparable,Temporal,TemporalAccessor,
   Instant MIN
   OffsetDateTime atOffset(ZoneOffset)
   ZonedDateTime atZone(ZoneId)
+  int compareTo(Instant)
   Instant from(TemporalAccessor)
   long getEpochSecond()
   int getNano()
@@ -118,7 +120,7 @@ class Instant -> java.time.Instant extends Comparable,Temporal,TemporalAccessor,
   Instant with(TemporalField,long)
 }
 
-class LocalDate -> java.time.LocalDate extends Comparable,ChronoLocalDate,Temporal,TemporalAccessor,TemporalAdjuster,Object {
+class LocalDate -> java.time.LocalDate extends ChronoLocalDate,Temporal,TemporalAccessor,TemporalAdjuster,Comparable,Object {
   LocalDate MAX
   LocalDate MIN
   LocalDateTime atStartOfDay()
@@ -154,6 +156,7 @@ class LocalDate -> java.time.LocalDate extends Comparable,ChronoLocalDate,Tempor
   LocalDate plusMonths(long)
   LocalDate plusWeeks(long)
   LocalDate plusDays(long)
+  Period until(ChronoLocalDate)
   LocalDate with(TemporalAdjuster)
   LocalDate with(TemporalField,long)
   LocalDate withDayOfMonth(int)
@@ -162,7 +165,7 @@ class LocalDate -> java.time.LocalDate extends Comparable,ChronoLocalDate,Tempor
   LocalDate withYear(int)
 }
 
-class LocalDateTime -> java.time.LocalDateTime extends Comparable,ChronoLocalDateTime,Temporal,TemporalAccessor,TemporalAdjuster,Object {
+class LocalDateTime -> java.time.LocalDateTime extends ChronoLocalDateTime,Temporal,TemporalAccessor,TemporalAdjuster,Comparable,Object {
   LocalDateTime MIN
   LocalDateTime MAX
   OffsetDateTime atOffset(ZoneOffset)
@@ -208,6 +211,7 @@ class LocalDateTime -> java.time.LocalDateTime extends Comparable,ChronoLocalDat
   LocalDateTime plusSeconds(long)
   LocalDateTime plusWeeks(long)
   LocalDateTime plusYears(long)
+  LocalDate toLocalDate()
   LocalDateTime truncatedTo(TemporalUnit)
   LocalDateTime with(TemporalAdjuster)
   LocalDateTime with(TemporalField,long)
@@ -220,13 +224,14 @@ class LocalDateTime -> java.time.LocalDateTime extends Comparable,ChronoLocalDat
   LocalDateTime withYear(int)
 }
 
-class LocalTime -> java.time.LocalTime extends Comparable,Temporal,TemporalAccessor,TemporalAdjuster,Object {
+class LocalTime -> java.time.LocalTime extends Temporal,TemporalAccessor,TemporalAdjuster,Comparable,Object {
   LocalTime MAX
   LocalTime MIDNIGHT
   LocalTime MIN
   LocalTime NOON
   LocalDateTime atDate(LocalDate)
   OffsetTime atOffset(ZoneOffset)
+  int compareTo(LocalTime)
   String format(DateTimeFormatter)
   LocalTime from(TemporalAccessor)
   int getHour()
@@ -267,8 +272,9 @@ class LocalTime -> java.time.LocalTime extends Comparable,Temporal,TemporalAcces
   LocalTime withSecond(int)
 }
 
-class MonthDay -> java.time.MonthDay extends Comparable,TemporalAccessor,TemporalAdjuster,Object {
+class MonthDay -> java.time.MonthDay extends TemporalAccessor,TemporalAdjuster,Comparable,Object {
   LocalDate atYear(int)
+  int compareTo(MonthDay)
   String format(DateTimeFormatter)
   MonthDay from(TemporalAccessor)
   int getMonthValue()
@@ -287,11 +293,12 @@ class MonthDay -> java.time.MonthDay extends Comparable,TemporalAccessor,Tempora
   MonthDay withMonth(int)
 }
 
-class OffsetDateTime -> java.time.OffsetDateTime extends Comparable,Temporal,TemporalAccessor,TemporalAdjuster,Object {
+class OffsetDateTime -> java.time.OffsetDateTime extends Temporal,TemporalAccessor,TemporalAdjuster,Comparable,Object {
   OffsetDateTime MAX
   OffsetDateTime MIN
   ZonedDateTime atZoneSameInstant(ZoneId)
   ZonedDateTime atZoneSimilarLocal(ZoneId)
+  int compareTo(OffsetDateTime)
   String format(DateTimeFormatter)
   OffsetDateTime from(TemporalAccessor)
   int getDayOfMonth()
@@ -359,9 +366,10 @@ class OffsetDateTime -> java.time.OffsetDateTime extends Comparable,Temporal,Tem
   OffsetDateTime withOffsetSameInstant(ZoneOffset)
 }
 
-class OffsetTime -> java.time.OffsetTime extends Comparable,Temporal,TemporalAccessor,TemporalAdjuster,Object {
+class OffsetTime -> java.time.OffsetTime extends Temporal,TemporalAccessor,TemporalAdjuster,Comparable,Object {
   OffsetTime MAX
   OffsetTime MIN
+  int compareTo(OffsetTime)
   String format(DateTimeFormatter)
   OffsetTime from(TemporalAccessor)
   int getHour()
@@ -434,12 +442,13 @@ class Period -> java.time.Period extends ChronoPeriod,TemporalAmount,Object {
   Period withYears(int)
 }
 
-class Year -> java.time.Year extends Comparable,Temporal,TemporalAccessor,TemporalAdjuster,Object {
+class Year -> java.time.Year extends Temporal,TemporalAccessor,TemporalAdjuster,Comparable,Object {
   int MAX_VALUE
   int MIN_VALUE
   LocalDate atDay(int)
   YearMonth atMonth(int)
   LocalDate atMonthDay(MonthDay)
+  int compareTo(Year)
   String format(DateTimeFormatter)
   Year from(TemporalAccessor)
   int getValue()
@@ -463,9 +472,10 @@ class Year -> java.time.Year extends Comparable,Temporal,TemporalAccessor,Tempor
   Year with(TemporalField,long)
 }
 
-class YearMonth -> java.time.YearMonth extends Comparable,Temporal,TemporalAccessor,TemporalAdjuster,Object {
+class YearMonth -> java.time.YearMonth extends Temporal,TemporalAccessor,TemporalAdjuster,Comparable,Object {
   LocalDate atDay(int)
   LocalDate atEndOfMonth()
+  int compareTo(YearMonth)
   String format(DateTimeFormatter)
   YearMonth from(TemporalAccessor)
   Month getMonth()
@@ -496,7 +506,7 @@ class YearMonth -> java.time.YearMonth extends Comparable,Temporal,TemporalAcces
   YearMonth withMonth(int)
 }
 
-class ZonedDateTime -> java.time.ZonedDateTime extends Comparable,ChronoZonedDateTime,Temporal,TemporalAccessor,Object {
+class ZonedDateTime -> java.time.ZonedDateTime extends ChronoZonedDateTime,Temporal,TemporalAccessor,Comparable,Object {
   int getDayOfMonth()
   DayOfWeek getDayOfWeek()
   int getDayOfYear()
@@ -589,7 +599,7 @@ class ZoneOffset -> java.time.ZoneOffset extends ZoneId,Object {
 
 #### Enums
 
-class DayOfWeek -> java.time.DayOfWeek extends Enum,Comparable,TemporalAccessor,TemporalAdjuster,Object {
+class DayOfWeek -> java.time.DayOfWeek extends Enum,TemporalAccessor,TemporalAdjuster,Comparable,Object {
   DayOfWeek FRIDAY
   DayOfWeek MONDAY
   DayOfWeek SATURDAY
@@ -607,7 +617,7 @@ class DayOfWeek -> java.time.DayOfWeek extends Enum,Comparable,TemporalAccessor,
   DayOfWeek[] values()
 }
 
-class Month -> java.time.Month extends Enum,Comparable,TemporalAccessor,TemporalAdjuster,Object {
+class Month -> java.time.Month extends Enum,TemporalAccessor,TemporalAdjuster,Comparable,Object {
   Month APRIL
   Month AUGUST
   Month DECEMBER
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.zone.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.zone.txt
index 036e2a734f9..d9d1cce5c10 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.zone.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.time.zone.txt
@@ -25,6 +25,7 @@
 #### Classes
 
 class ZoneOffsetTransition -> java.time.zone.ZoneOffsetTransition extends Comparable,Object {
+  int compareTo(ZoneOffsetTransition)
   LocalDateTime getDateTimeAfter()
   LocalDateTime getDateTimeBefore()
   Duration getDuration()
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt
index 5e240d0d45a..cccbf60f1a3 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt
@@ -454,6 +454,7 @@ class Calendar -> java.util.Calendar extends Comparable,Object {
   void clear()
   void clear(int)
   def clone()
+  int compareTo(Calendar)
   int get(int)
   int getActualMaximum(int)
   int getActualMinimum(int)
@@ -584,6 +585,7 @@ class Date -> java.util.Date extends Comparable,Object {
   boolean after(Date)
   boolean before(Date)
   def clone()
+  int compareTo(Date)
   Date from(Instant)
   long getTime()
   void setTime(long)
@@ -999,6 +1001,7 @@ class TreeSet -> java.util.TreeSet extends AbstractSet,NavigableSet,SortedSet,Se
 
 class UUID -> java.util.UUID extends Comparable,Object {
   UUID <init>(long,long)
+  int compareTo(UUID)
   int clockSequence()
   UUID fromString(String)
   long getLeastSignificantBits()
@@ -1054,101 +1057,101 @@ class Locale.FilteringMode -> java.util.Locale$FilteringMode extends Enum,Compar
 
 #### Exceptions
 
-class ConcurrentModificationException -> java.util.ConcurrentModificationException extends RuntimeException,Exception {
+class ConcurrentModificationException -> java.util.ConcurrentModificationException extends RuntimeException,Exception,Object {
   ConcurrentModificationException <init>()
   ConcurrentModificationException <init>(String)
 }
 
-class DuplicateFormatFlagsException -> java.util.DuplicateFormatFlagsException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception {
+class DuplicateFormatFlagsException -> java.util.DuplicateFormatFlagsException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception,Object {
   DuplicateFormatFlagsException <init>(String)
   String getFlags()
 }
 
-class EmptyStackException -> java.util.EmptyStackException extends RuntimeException,Exception {
+class EmptyStackException -> java.util.EmptyStackException extends RuntimeException,Exception,Object {
   EmptyStackException <init>()
 }
 
-class FormatFlagsConversionMismatchException -> java.util.FormatFlagsConversionMismatchException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception {
+class FormatFlagsConversionMismatchException -> java.util.FormatFlagsConversionMismatchException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception,Object {
   FormatFlagsConversionMismatchException <init>(String,char)
   char getConversion()
   String getFlags()
 }
 
-class FormatterClosedException -> java.util.FormatterClosedException extends IllegalStateException,RuntimeException,Exception {
+class FormatterClosedException -> java.util.FormatterClosedException extends IllegalStateException,RuntimeException,Exception,Object {
   FormatterClosedException <init>()
 }
 
-class IllegalFormatCodePointException -> java.util.IllegalFormatCodePointException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception {
+class IllegalFormatCodePointException -> java.util.IllegalFormatCodePointException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception,Object {
   IllegalFormatCodePointException <init>(int)
   int getCodePoint()
 }
 
-class IllegalFormatConversionException -> java.util.IllegalFormatConversionException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception {
+class IllegalFormatConversionException -> java.util.IllegalFormatConversionException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception,Object {
   char getConversion()
 }
 
-class IllegalFormatException -> java.util.IllegalFormatException extends IllegalArgumentException,RuntimeException,Exception {
+class IllegalFormatException -> java.util.IllegalFormatException extends IllegalArgumentException,RuntimeException,Exception,Object {
 }
 
-class IllegalFormatFlagsException -> java.util.IllegalFormatFlagsException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception {
+class IllegalFormatFlagsException -> java.util.IllegalFormatFlagsException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception,Object {
   IllegalFormatFlagsException <init>(String)
   String getFlags()
 }
 
-class IllegalFormatPrecisionException -> java.util.IllegalFormatPrecisionException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception {
+class IllegalFormatPrecisionException -> java.util.IllegalFormatPrecisionException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception,Object {
   IllegalFormatPrecisionException <init>(int)
   int getPrecision()
 }
 
-class IllegalFormatWidthException -> java.util.IllegalFormatWidthException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception {
+class IllegalFormatWidthException -> java.util.IllegalFormatWidthException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception,Object {
   IllegalFormatWidthException <init>(int)
   int getWidth()
 }
 
-class IllformedLocaleException -> java.util.IllformedLocaleException extends RuntimeException,Exception {
+class IllformedLocaleException -> java.util.IllformedLocaleException extends RuntimeException,Exception,Object {
   IllformedLocaleException <init>()
   IllformedLocaleException <init>(String)
   IllformedLocaleException <init>(String,int)
   int getErrorIndex()
 }
 
-class InputMismatchException -> java.util.InputMismatchException extends NoSuchElementException,RuntimeException,Exception {
+class InputMismatchException -> java.util.InputMismatchException extends NoSuchElementException,RuntimeException,Exception,Object {
   InputMismatchException <init>()
   InputMismatchException <init>(String)
 }
 
-class MissingFormatArgumentException -> java.util.MissingFormatArgumentException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception {
+class MissingFormatArgumentException -> java.util.MissingFormatArgumentException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception,Object {
   MissingFormatArgumentException <init>(String)
   String getFormatSpecifier()
 }
 
-class MissingFormatWidthException -> java.util.MissingFormatWidthException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception {
+class MissingFormatWidthException -> java.util.MissingFormatWidthException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception,Object {
   MissingFormatWidthException <init>(String)
   String getFormatSpecifier()
 }
 
-class MissingResourceException -> java.util.MissingResourceException extends RuntimeException,Exception {
+class MissingResourceException -> java.util.MissingResourceException extends RuntimeException,Exception,Object {
   MissingResourceException <init>(String,String,String)
   String getClassName()
   String getKey()
 }
 
-class NoSuchElementException -> java.util.NoSuchElementException extends RuntimeException,Exception {
+class NoSuchElementException -> java.util.NoSuchElementException extends RuntimeException,Exception,Object {
   NoSuchElementException <init>()
   NoSuchElementException <init>(String)
 }
 
-class TooManyListenersException -> java.util.TooManyListenersException extends Exception {
+class TooManyListenersException -> java.util.TooManyListenersException extends Exception,Object {
   TooManyListenersException <init>()
   TooManyListenersException <init>(String)
 }
 
-class UnknownFormatConversionException -> java.util.UnknownFormatConversionException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception {
+class UnknownFormatConversionException -> java.util.UnknownFormatConversionException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception,Object {
   UnknownFormatConversionException <init>(String)
   String getConversion()
 }
 
-class UnknownFormatFlagsException -> java.util.UnknownFormatFlagsException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception {
+class UnknownFormatFlagsException -> java.util.UnknownFormatFlagsException extends IllegalFormatException,IllegalArgumentException,RuntimeException,Exception,Object {
   UnknownFormatFlagsException <init>(String)
   String getFlags()
 }
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt
index 8627de26d26..7fff9ac0aa4 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt
@@ -65,22 +65,26 @@ class org.elasticsearch.common.geo.GeoPoint -> org.elasticsearch.common.geo.GeoP
 }
 
 class org.elasticsearch.index.fielddata.ScriptDocValues.Strings -> org.elasticsearch.index.fielddata.ScriptDocValues$Strings extends List,Collection,Iterable,Object {
+  String get(int)
   String getValue()
   List getValues()
 }
 
 class org.elasticsearch.index.fielddata.ScriptDocValues.Longs -> org.elasticsearch.index.fielddata.ScriptDocValues$Longs extends List,Collection,Iterable,Object {
+  Long get(int)
   long getValue()
   List getValues()
   org.joda.time.ReadableDateTime getDate()
 }
 
 class org.elasticsearch.index.fielddata.ScriptDocValues.Doubles -> org.elasticsearch.index.fielddata.ScriptDocValues$Doubles extends List,Collection,Iterable,Object {
+  Double get(int)
   double getValue()
   List getValues()
 }
 
 class org.elasticsearch.index.fielddata.ScriptDocValues.GeoPoints -> org.elasticsearch.index.fielddata.ScriptDocValues$GeoPoints extends List,Collection,Iterable,Object {
+  org.elasticsearch.common.geo.GeoPoint get(int)
   org.elasticsearch.common.geo.GeoPoint getValue()
   List getValues()
   double getLat()
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
index 46609a76173..885ce060aaa 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless;
 
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -111,7 +110,13 @@ public class BasicAPITests extends ScriptTestCase {
         assertBytecodeExists("def x = 1D", "INVOKESTATIC java/lang/Double.valueOf (D)Ljava/lang/Double;");
     }
 
-    void testStream() {
-        assertEquals(11, exec("params.list.stream().sum()", Collections.singletonMap("list", Arrays.asList(1,2,3,5))));
+    public void testInterfaceDefaultMethods() {
+        assertEquals(1, exec("Map map = new HashMap(); return map.getOrDefault(5,1);"));
+        assertEquals(1, exec("def map = new HashMap(); return map.getOrDefault(5,1);"));
+    }
+
+    public void testInterfacesHaveObject() {
+        assertEquals("{}", exec("Map map = new HashMap(); return map.toString();"));
+        assertEquals("{}", exec("def map = new HashMap(); return map.toString();"));
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
index 4469279c8be..7a55844a224 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
@@ -33,7 +33,7 @@ public class DefBootstrapTests extends ESTestCase {
         CallSite site = DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                   "toString", 
                                                   MethodType.methodType(String.class, Object.class), 
-                                                  DefBootstrap.METHOD_CALL);
+                                                  DefBootstrap.METHOD_CALL, 0);
         MethodHandle handle = site.dynamicInvoker();
         assertDepthEquals(site, 0);
 
@@ -50,7 +50,7 @@ public class DefBootstrapTests extends ESTestCase {
         CallSite site = DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                   "toString", 
                                                   MethodType.methodType(String.class, Object.class), 
-                                                  DefBootstrap.METHOD_CALL);
+                                                  DefBootstrap.METHOD_CALL, 0);
         MethodHandle handle = site.dynamicInvoker();
         assertDepthEquals(site, 0);
 
@@ -72,7 +72,7 @@ public class DefBootstrapTests extends ESTestCase {
         CallSite site = DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                   "toString", 
                                                   MethodType.methodType(String.class, Object.class), 
-                                                  DefBootstrap.METHOD_CALL);
+                                                  DefBootstrap.METHOD_CALL, 0);
         MethodHandle handle = site.dynamicInvoker();
         assertDepthEquals(site, 0);
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
index 20c48fe87d6..12265f161ab 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
@@ -20,11 +20,60 @@
 package org.elasticsearch.painless;
 
 public class FunctionRefTests extends ScriptTestCase {
-    public void testUnsupported() {
-        expectScriptThrows(UnsupportedOperationException.class, () -> {
-           exec("DoubleStream.Builder builder = DoubleStream.builder();" +
-               "builder.add(2.0); builder.add(1.0); builder.add(3.0);" +
-               "builder.build().reduce(Double::unsupported);");
+
+    public void testStaticMethodReference() {
+        assertEquals(1, exec("List l = new ArrayList(); l.add(2); l.add(1); l.sort(Integer::compare); return l.get(0);"));
+    }
+    
+    public void testStaticMethodReferenceDef() {
+        assertEquals(1, exec("def l = new ArrayList(); l.add(2); l.add(1); l.sort(Integer::compare); return l.get(0);"));
+    }
+
+    public void testVirtualMethodReference() {
+        assertEquals(2, exec("List l = new ArrayList(); l.add(1); l.add(1); return l.stream().mapToInt(Integer::intValue).sum();"));
+    }
+    
+    public void testVirtualMethodReferenceDef() {
+        assertEquals(2, exec("def l = new ArrayList(); l.add(1); l.add(1); return l.stream().mapToInt(Integer::intValue).sum();"));
+    }
+
+    public void testCtorMethodReference() {
+        assertEquals(3.0D, 
+            exec("List l = new ArrayList(); l.add(1.0); l.add(2.0); " + 
+                 "DoubleStream doubleStream = l.stream().mapToDouble(Double::doubleValue);" + 
+                 "DoubleSummaryStatistics stats = doubleStream.collect(DoubleSummaryStatistics::new, " +
+                                                                      "DoubleSummaryStatistics::accept, " +
+                                                                      "DoubleSummaryStatistics::combine); " + 
+                 "return stats.getSum()"));
+    }
+    
+    public void testCtorMethodReferenceDef() {
+        assertEquals(3.0D, 
+            exec("def l = new ArrayList(); l.add(1.0); l.add(2.0); " + 
+                 "def doubleStream = l.stream().mapToDouble(Double::doubleValue);" + 
+                 "def stats = doubleStream.collect(DoubleSummaryStatistics::new, " +
+                                                  "DoubleSummaryStatistics::accept, " +
+                                                  "DoubleSummaryStatistics::combine); " + 
+                 "return stats.getSum()"));
+    }
+
+    public void testMethodMissing() {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("List l = new ArrayList(); l.add(2); l.add(1); l.sort(Integer::bogus); return l.get(0);");
+        });
+        assertTrue(expected.getMessage().contains("Unknown reference"));
+    }
+
+    public void testNotFunctionalInterface() {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("List l = new ArrayList(); l.add(2); l.add(1); l.add(Integer::bogus); return l.get(0);");
+        });
+        assertTrue(expected.getMessage().contains("Cannot convert function reference"));
+    }
+
+    public void testIncompatible() {
+        expectScriptThrows(BootstrapMethodError.class, () -> {
+            exec("List l = new ArrayList(); l.add(2); l.add(1); l.sort(String::startsWith); return l.get(0);");
         });
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java
index ba095c05a9d..fce827e686c 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OverloadTests.java
@@ -37,7 +37,7 @@ public class OverloadTests extends ScriptTestCase {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("def x = 'abc123abc'; return x.indexOf('c', 3, 'bogus');");
         });
-        assertTrue(expected.getMessage().contains("dynamic method [indexOf] with signature [(String,int,String)"));
+        assertTrue(expected.getMessage().contains("dynamic method [indexOf]"));
     }
     
     public void testConstructor() {

From 881e4058ee8e6a3e7e1b45b14b530f5ff7aae5fa Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 6 Jun 2016 09:39:39 -0400
Subject: [PATCH 0656/1311] don't do a no-op filter, that was just for testing

---
 .../src/main/java/org/elasticsearch/painless/Def.java         | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 60c42e2af55..c9572d15dc1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -190,17 +190,17 @@ public final class Def {
      static MethodHandle lookupMethod(Class<?> receiverClass, String name, Object args[], long recipe) {
          Method method = lookupMethodInternal(receiverClass, name, args.length - 1);
          MethodHandle handle = method.handle;
-         MethodHandle filters[] = new MethodHandle[args.length];
 
          if (recipe != 0) {
+             MethodHandle filters[] = new MethodHandle[args.length];
              for (int i = 0; i < args.length; i++) {
                  // its a functional reference, replace the argument with an impl
                  if ((recipe & (1L << (i - 1))) != 0) {
                      filters[i] = lookupReference(method.arguments.get(i - 1).clazz, (String) args[i]);
                  }
              }
+             handle = MethodHandles.filterArguments(handle, 0, filters);
          }
-         handle = MethodHandles.filterArguments(handle, 0, filters);
          
          return handle;
      }

From 2cf04c08772ab7ca3ea5fd65d7c5878e3852d9e8 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Mon, 6 Jun 2016 15:41:11 +0200
Subject: [PATCH 0657/1311] wrap entity only when not repeatable and improve
 RequestLoggerTests

---
 .../elasticsearch/client/RequestLogger.java   | 13 ++++--
 .../client/RequestLoggerTests.java            | 44 +++++++++++++++----
 2 files changed, 44 insertions(+), 13 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/RequestLogger.java b/client/src/main/java/org/elasticsearch/client/RequestLogger.java
index cdee2d9944d..f1b406b5cf3 100644
--- a/client/src/main/java/org/elasticsearch/client/RequestLogger.java
+++ b/client/src/main/java/org/elasticsearch/client/RequestLogger.java
@@ -102,9 +102,12 @@ final class RequestLogger {
             HttpEntityEnclosingRequest enclosingRequest = (HttpEntityEnclosingRequest) request;
             if (enclosingRequest.getEntity() != null) {
                 requestLine += " -d '";
-                HttpEntity entity = new BufferedHttpEntity(enclosingRequest.getEntity());
-                enclosingRequest.setEntity(entity);
-                requestLine += EntityUtils.toString(entity) + "'";
+                HttpEntity entity = enclosingRequest.getEntity();
+                if (entity.isRepeatable() == false) {
+                    entity = new BufferedHttpEntity(enclosingRequest.getEntity());
+                    enclosingRequest.setEntity(entity);
+                }
+                requestLine += EntityUtils.toString(entity, StandardCharsets.UTF_8) + "'";
             }
         }
         return requestLine;
@@ -121,7 +124,9 @@ final class RequestLogger {
         responseLine += "\n#";
         HttpEntity entity = httpResponse.getEntity();
         if (entity != null) {
-            entity = new BufferedHttpEntity(entity);
+            if (entity.isRepeatable() == false) {
+                entity = new BufferedHttpEntity(entity);
+            }
             httpResponse.setEntity(entity);
             ContentType contentType = ContentType.get(entity);
             Charset charset = StandardCharsets.UTF_8;
diff --git a/client/src/test/java/org/elasticsearch/client/RequestLoggerTests.java b/client/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
index 64bddf88af2..c16d9e7876e 100644
--- a/client/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.client;
 
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import org.apache.http.HttpEntity;
 import org.apache.http.HttpEntityEnclosingRequest;
 import org.apache.http.HttpHost;
 import org.apache.http.ProtocolVersion;
@@ -30,11 +31,14 @@ import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.client.methods.HttpTrace;
+import org.apache.http.entity.InputStreamEntity;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.message.BasicHttpResponse;
 import org.apache.http.message.BasicStatusLine;
+import org.apache.http.util.EntityUtils;
 import org.apache.lucene.util.LuceneTestCase;
 
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -80,16 +84,27 @@ public class RequestLoggerTests extends LuceneTestCase {
         }
 
         String expected = "curl -iX " + request.getMethod() + " '" + host + uri + "'";
-
-        if (request instanceof HttpEntityEnclosingRequest && random().nextBoolean()) {
-            HttpEntityEnclosingRequest enclosingRequest = (HttpEntityEnclosingRequest) request;
-            String requestBody = "{ \"field\": \"value\" }";
-            enclosingRequest.setEntity(new StringEntity(requestBody, StandardCharsets.UTF_8));
+        boolean hasBody = request instanceof HttpEntityEnclosingRequest && random().nextBoolean();
+        String requestBody = "{ \"field\": \"value\" }";
+        if (hasBody) {
             expected += " -d '" + requestBody + "'";
+            HttpEntityEnclosingRequest enclosingRequest = (HttpEntityEnclosingRequest) request;
+            HttpEntity entity;
+            if (random().nextBoolean()) {
+                entity = new StringEntity(requestBody, StandardCharsets.UTF_8);
+            } else {
+                entity = new InputStreamEntity(new ByteArrayInputStream(requestBody.getBytes(StandardCharsets.UTF_8)));
+            }
+            enclosingRequest.setEntity(entity);
         }
 
         String traceRequest = RequestLogger.buildTraceRequest(request, host);
         assertThat(traceRequest, equalTo(expected));
+        if (hasBody) {
+            //check that the body is still readable as most entities are not repeatable
+            String body = EntityUtils.toString(((HttpEntityEnclosingRequest) request).getEntity(), StandardCharsets.UTF_8);
+            assertThat(body, equalTo(requestBody));
+        }
     }
 
     public void testTraceResponse() throws IOException {
@@ -105,15 +120,26 @@ public class RequestLoggerTests extends LuceneTestCase {
             expected += "\n# header" + i + ": value";
         }
         expected += "\n#";
-        if (random().nextBoolean()) {
-            String responseBody = "{\n  \"field\": \"value\"\n}";
-            httpResponse.setEntity(new StringEntity(responseBody, StandardCharsets.UTF_8));
+        boolean hasBody = random().nextBoolean();
+        String responseBody = "{\n  \"field\": \"value\"\n}";
+        if (hasBody) {
             expected += "\n# {";
             expected += "\n#   \"field\": \"value\"";
             expected += "\n# }";
+            HttpEntity entity;
+            if (random().nextBoolean()) {
+                entity = new StringEntity(responseBody, StandardCharsets.UTF_8);
+            } else {
+                entity = new InputStreamEntity(new ByteArrayInputStream(responseBody.getBytes(StandardCharsets.UTF_8)));
+            }
+            httpResponse.setEntity(entity);
         }
-
         String traceResponse = RequestLogger.buildTraceResponse(httpResponse);
         assertThat(traceResponse, equalTo(expected));
+        if (hasBody) {
+            //check that the body is still readable as most entities are not repeatable
+            String body = EntityUtils.toString(httpResponse.getEntity(), StandardCharsets.UTF_8);
+            assertThat(body, equalTo(responseBody));
+        }
     }
 }

From e3e8f101032545bd86244d609dbac4975f45884e Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Mon, 6 Jun 2016 15:45:57 +0200
Subject: [PATCH 0658/1311] [TEST] fix assertion that index was fully deleted

---
 .../java/org/elasticsearch/indices/IndicesServiceTests.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
index fa4183f689a..5761997ed46 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
@@ -207,7 +207,7 @@ public class IndicesServiceTests extends ESSingleNodeTestCase {
             // shard lock released... we can now delete
             indicesService.processPendingDeletes(test.index(), test.getIndexSettings(), new TimeValue(0, TimeUnit.MILLISECONDS));
             assertEquals(indicesService.numPendingDeletes(test.index()), 0);
-            assertFalse(indicesService.hasUncompletedPendingDeletes());
+            assertTrue(indicesService.hasUncompletedPendingDeletes()); // "bogus" index has not been removed
         }
         assertAcked(client().admin().indices().prepareOpen("test"));
 

From 200d76e6f07fbaa2d0957b79fa8ddc279671e182 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 6 Jun 2016 11:27:26 -0400
Subject: [PATCH 0659/1311] Throw exception if using a closed transport client

Today when attempting to use a closed transport client, an exception
saying that no nodes are available is thrown. This is because when a
transport client is closed, its internal list of nodes is cleared. But
this exception is puzzling and can be made clearer. This commit changes
the behavior so that attempting to execute a request using a closed
transport client throws an illegal state exception.

Relates #18722
---
 .../TransportClientNodesService.java          | 13 +++++-
 .../transport/TransportClientTests.java       | 41 +++++++++++++++++++
 2 files changed, 53 insertions(+), 1 deletion(-)
 create mode 100644 core/src/test/java/org/elasticsearch/client/transport/TransportClientTests.java

diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
index c3379c9ceaf..71c5895669c 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
@@ -211,7 +211,18 @@ public class TransportClientNodesService extends AbstractComponent {
     }
 
     public <Response> void execute(NodeListenerCallback<Response> callback, ActionListener<Response> listener) {
-        List<DiscoveryNode> nodes = this.nodes;
+        // we first read nodes before checking the closed state; this
+        // is because otherwise we could be subject to a race where we
+        // read the state as not being closed, and then the client is
+        // closed and the nodes list is cleared, and then a
+        // NoNodeAvailableException is thrown
+        // it is important that the order of first setting the state of
+        // closed and then clearing the list of nodes is maintained in
+        // the close method
+        final List<DiscoveryNode> nodes = this.nodes;
+        if (closed) {
+            throw new IllegalStateException("transport client is closed");
+        }
         ensureNodesAreAvailable(nodes);
         int index = getNodeNumber();
         RetryListener<Response> retryListener = new RetryListener<>(callback, listener, nodes, index);
diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientTests.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientTests.java
new file mode 100644
index 00000000000..ec2065b67e2
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientTests.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.transport;
+
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.concurrent.ExecutionException;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.object.HasToString.hasToString;
+
+public class TransportClientTests extends ESTestCase {
+
+    public void testThatUsingAClosedClientThrowsAnException() throws ExecutionException, InterruptedException {
+        final TransportClient client = TransportClient.builder().settings(Settings.EMPTY).build();
+        client.close();
+        final IllegalStateException e =
+            expectThrows(IllegalStateException.class, () -> client.admin().cluster().health(new ClusterHealthRequest()).get());
+        assertThat(e, hasToString(containsString("transport client is closed")));
+    }
+
+}

From d8056c8213ea7d2fbd26dbe15e23e96acfbd180c Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 6 Jun 2016 10:45:44 -0400
Subject: [PATCH 0660/1311] Add support for waiting until a refresh occurs

This adds support for setting the refresh request parameter to
`wait_for` in the `index`, `delete`, `update`, and `bulk` APIs. When
`refresh=wait_for` is set those APIs will not return until their
results have been made visible to search by a refresh.

Also it adds a `forced_refresh` field to the response of `index`,
`delete`, `update`, and to each item in a bulk response. This will
be true for requests with `?refresh` or `?refresh=true` and will be
true for some requests (see below) with `refresh=wait_for` but ought
to otherwise always be false.

`refresh=wait_for` is implemented as a list of
`Tuple<Translog.Location, Consumer<Boolean>>`s in the new `RefreshListeners`
class that is managed by `IndexShard`. The dynamic, index scoped
`index.max_refresh_listeners` setting controls a maximum number of
listeners allowed in any shard. If more than that many listeners
accumulate in the engine then a refresh will be forced, the thread that
adds the listener will be blocked until the refresh completes, and then the
listener will be called with a `forcedRefresh` flag so it knows that it was
the "straw that broke the camel's back". These listeners are only used by
`refresh=wait_for` and that flag manifests itself as `forced_refresh` being
`true` in the response.

About half of this change comes from piping async-ness down to the appropriate
layer in a way that is compatible with the ongoing with with sequence ids.

Closes #1063

You can look up the winding story of all the commits here:
https://github.com/elastic/elasticsearch/pull/17986

Here are the commit messages in case they are intersting to you:
commit 59a753b89109828d2b8f0de05cb104fc663cf95e
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon Jun 6 10:18:23 2016 -0400

    Replace a method reference with implementing an interface

    Saves a single allocation and forces more commonality
    between the WriteResults.

commit 31f7861a85b457fb7378a6f27fa0a0c171538f68
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon Jun 6 10:07:55 2016 -0400

    Revert "Replace static method that takes consumer with delegate class that takes an interface"

    This reverts commit 777e23a6592c75db0081a53458cc760f4db69507.

commit 777e23a6592c75db0081a53458cc760f4db69507
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon Jun 6 09:29:35 2016 -0400

    Replace static method that takes consumer with delegate class that takes an interface

    Same number of allocations, much less code duplication.

commit 9b49a480ca9587a0a16ebe941662849f38289644
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon Jun 6 08:25:38 2016 -0400

    Patch from boaz

commit c2bc36524fda119fd0514415127e8901d94409c8
Author: Nik Everett <nik9000@gmail.com>
Date:   Thu Jun 2 14:46:27 2016 -0400

    Fix docs

    After updating to master we are actually testing them.

commit 03975ac056e44954eb0a371149d410dcf303e212
Author: Nik Everett <nik9000@gmail.com>
Date:   Thu Jun 2 14:20:11 2016 -0400

    Cleanup after merge from master

commit 9c9a1deb002c5bebb2a997c89fa12b3d7978e02e
Author: Nik Everett <nik9000@gmail.com>
Date:   Thu Jun 2 14:09:14 2016 -0400

    Breaking changes notes

commit 1c3e64ae06c07a85f7af80534fab88279adb30b4
Merge: 9e63ad6 f67e580
Author: Nik Everett <nik9000@gmail.com>
Date:   Thu Jun 2 14:00:05 2016 -0400

    Merge branch 'master' into block_until_refresh2

commit 9e63ad6de52d0b28f0b6d7203721baf1ebf6f56b
Author: Nik Everett <nik9000@gmail.com>
Date:   Thu Jun 2 13:21:27 2016 -0400

    Test for TransportWriteAction

commit 522ecb59d39b3c9e8df0d3b8df34b9e7aeaf0ce9
Author: Nik Everett <nik9000@gmail.com>
Date:   Thu Jun 2 10:30:18 2016 -0400

    Document deprecation

commit 0cd67b947f58867e704a1f0e66928a6fb5a11f11
Author: Nik Everett <nik9000@gmail.com>
Date:   Thu Jun 2 10:26:23 2016 -0400

    Deprecate setRefresh(boolean)

    Users should use `setRefresh(RefreshPolicy)` instead.

commit aeb1be3f2c501990b33fb1f8230d496035f498ef
Author: Nik Everett <nik9000@gmail.com>
Date:   Thu Jun 2 10:12:27 2016 -0400

    Remove checkstyle suppression

    It is fixed

commit 00d09a9caa638b6f90f4896b5502dd98d8fad56e
Author: Nik Everett <nik9000@gmail.com>
Date:   Thu Jun 2 10:08:28 2016 -0400

    Improve comment

commit 788164b898a6ee2878a273961230122b7386c3c9
Author: Nik Everett <nik9000@gmail.com>
Date:   Thu Jun 2 10:01:01 2016 -0400

    S/ReplicatedWriteResponse/WriteResponse/

    Now it lines up with WriteRequest.

commit b74cf3fe778352b140355afcaa08d3d4412d749d
Author: Nik Everett <nik9000@gmail.com>
Date:   Wed Jun 1 18:27:52 2016 -0400

    Preserve `?refresh` behavior

    `?refresh` means the same things as `?refresh=true`.

commit 30f972bdaeaaa0de6fe67746cdb8628aa86f5a8c
Author: Nik Everett <nik9000@gmail.com>
Date:   Wed Jun 1 17:39:05 2016 -0400

    Handle hanging documents

    If a document is added to the index during a refresh we weren't properly
    firing its refresh listener. This happened because the way we detect
    whether a refresh makes something visible or not is imperfect. It is
    ok because it always errs on the side of thinking that something isn't
    yet visible.

    So when a document arrives during a refresh the refresh listeners
    won't think it made it into a refresh when, often, it does. The way
    we work around this is by telling Elasticsearch that it ought to
    trigger a refresh if there are any pending refresh listeners even
    if there aren't pending documents to update. Lucene short circuits
    the refresh so it doesn't take that much effort, but the refresh
    listeners still get the signal that a refresh has come in and they
    still pick up the change and notify the listener.

    This means that the time that a listener can wait is actually slightly
    longer than the refresh interval.

commit d523b5702b60c7ba309fb0dcf3cd3a4798f11960
Author: Nik Everett <nik9000@gmail.com>
Date:   Wed Jun 1 14:34:01 2016 -0400

    Explain Integer.MAX_VALUE

commit 4ffb7c0e954343cc1c04b3d7be2ebad66d3a016b
Author: Nik Everett <nik9000@gmail.com>
Date:   Wed Jun 1 14:27:39 2016 -0400

    Fire all refresh listeners in a single thread

    Rather than queueing a runnable each.

commit 19606ec3bbe612095df45eba734c5b7eb2709c01
Author: Nik Everett <nik9000@gmail.com>
Date:   Wed Jun 1 14:09:52 2016 -0400

    Assert translog ordering

commit 6bb4e5c75e850f4a42518f06fbc955f7ec76d245
Author: Nik Everett <nik9000@gmail.com>
Date:   Wed Jun 1 13:17:44 2016 -0400

    Support null RefreshListeners in InternalEngine

    Just skip using it.

commit 74be1480d6e44af2b354ff9ea47c234d4870b6c2
Author: Nik Everett <nik9000@gmail.com>
Date:   Tue May 31 18:02:03 2016 -0400

    Move funny ShardInfo hack for bulk into bulk

    This should make it easier to understand because it is closer to where it
    matters....

commit 2b771f8dabd488e056cfdc9989608d18264ddfb0
Author: Nik Everett <nik9000@gmail.com>
Date:   Tue May 31 17:39:46 2016 -0400

    Pull listener out into an inner class with javadoc and stuff

commit 058481ad72019c0492b03a7a4ac32a48673697d3
Author: Nik Everett <nik9000@gmail.com>
Date:   Tue May 31 17:33:42 2016 -0400

    Fix javadoc links

commit d2123b1cabf29bce8ff561d4a4c1c1d5b42bccad
Author: Nik Everett <nik9000@gmail.com>
Date:   Tue May 31 17:28:09 2016 -0400

    Make more stuff final

commit 8453fc4f7850f6a02fb5971c17a942a3e3fd9f7b
Author: Nik Everett <nik9000@gmail.com>
Date:   Tue May 31 17:26:48 2016 -0400

    Javadoc

commit fb16d2fc7016c1e8e1621d481e8781c7ef43326c
Author: Nik Everett <nik9000@gmail.com>
Date:   Tue May 31 16:14:48 2016 -0400

    Rewrite refresh docs

commit 5797d1b1c4d233c0db918c0d08c21731ddccd05e
Author: Nik Everett <nik9000@gmail.com>
Date:   Tue May 31 15:02:34 2016 -0400

    Fix forced_refresh flag

    It wasn't being set.

commit 43ce50a1de250a9e073a2ca6cbf55c1b4c74b11b
Author: Nik Everett <nik9000@gmail.com>
Date:   Tue May 31 14:02:56 2016 -0400

    Delay translog sync and flush until after refresh

    The sync might have occurred for us during the refresh so we
    have less work to do. Maybe.

commit bb2739202e084703baf02cfa58f09517598cf14e
Author: Nik Everett <nik9000@gmail.com>
Date:   Tue May 31 13:08:08 2016 -0400

    Remove duplication in WritePrimaryResult and WriteReplicaResult

commit 2f579f89b4867a880396f2e7fcffc508449ff2de
Author: Nik Everett <nik9000@gmail.com>
Date:   Tue May 31 12:19:05 2016 -0400

    Clean up registration of RefreshListeners

commit 87ab6e60ca5ba945bf0fba84784b2bbe53506abf
Author: Nik Everett <nik9000@gmail.com>
Date:   Tue May 31 11:28:30 2016 -0400

    Shorten lock time in RefreshListeners

    Also use null to represent no listeners rather than an empty list.
    This saves allocating a new ArrayList every refresh cycle on every
    index.

commit 0d49d9c5720dadfb67da3fa760397bf6d874601c
Author: Nik Everett <nik9000@gmail.com>
Date:   Tue May 24 10:46:18 2016 -0400

    Flip relationship between RefreshListeners and Engine

    Now RefreshListeners comes to Engine from EngineConfig.

commit b2704b8a39382953f8f91a9743e894ee289f7514
Author: Nik Everett <nik9000@gmail.com>
Date:   Tue May 24 09:37:58 2016 -0400

    Remove unused imports

    Maybe I added them?

commit 04343a22647f19304d9dc716b3fac9b183227f63
Author: Nik Everett <nik9000@gmail.com>
Date:   Tue May 24 09:37:52 2016 -0400

    Javadoc

commit da1e765678890a02d61d8a29aa433274beb5e00c
Author: Nik Everett <nik9000@gmail.com>
Date:   Tue May 24 09:26:35 2016 -0400

    Reply with non-null

    Also move the fsync and flush to before the refresh listener stuff.

commit 5d8eecd0d904b497844b4c81c46477bd6178ed3a
Author: Nik Everett <nik9000@gmail.com>
Date:   Tue May 24 08:58:47 2016 -0400

    Remove funky synchronization in AsyncReplicaAction

commit 1ec71eea0f4e1228ae1497d982307be818ef4b65
Author: Nik Everett <nik9000@gmail.com>
Date:   Tue May 24 08:01:14 2016 -0400

    s/LinkedTransferQueue/ArrayList/

commit 7da36a4ceed2ccf7955138c3b005237fa41efcb4
Author: Nik Everett <nik9000@gmail.com>
Date:   Tue May 24 07:46:38 2016 -0400

    More cleanup for RefreshListeners

commit 957e9b77007c32ee75dde152c6622bab065d5993
Author: Nik Everett <nik9000@gmail.com>
Date:   Tue May 24 07:34:13 2016 -0400

    /Consumer<Runnable>/Executor/

commit 4d8bf5d4a70dcc56150c8d8d14165cd23d308b3c
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon May 23 22:20:42 2016 -0400

    explain

commit 15d948a348089bb2937eec5ac4e96f3ec67dbe32
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon May 23 22:17:59 2016 -0400

    Better....

commit dc28951d02973fc03b4d51913b5f96de14b75607
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon May 23 21:09:20 2016 -0400

    Javadocs and compromises

commit 8eebaa89c0a1ee74982fbe0d56d1485ca2ae09db
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon May 23 20:52:49 2016 -0400

    Take boaz's changes to their logic conclusion and unbreak important stuff like bulk

commit 7056b96ea412f275005b93e3570bcff895859ed5
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon May 23 15:49:32 2016 -0400

    Patch from boaz

commit 87be7eaed09a274cc6a99d1a3da81d2d7bf9dd64
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon May 23 15:49:13 2016 -0400

    Revert "Move async parts of replica operation outside of the lock"

    This reverts commit 13807ad10b6f5ecd39f98c9f20874f9f352c5bc2.

commit 13807ad10b6f5ecd39f98c9f20874f9f352c5bc2
Author: Nik Everett <nik9000@gmail.com>
Date:   Fri May 20 22:53:15 2016 -0400

    Move async parts of replica operation outside of the lock

commit b8cadcef565908b276484f7f5f988fd58b38d8b6
Author: Nik Everett <nik9000@gmail.com>
Date:   Fri May 20 16:17:20 2016 -0400

    Docs

commit 91149e0580233bf79c2273b419fe9374ca746648
Author: Nik Everett <nik9000@gmail.com>
Date:   Fri May 20 15:17:40 2016 -0400

    Finally!

commit 1ff50c2faf56665d221f00a18d9ac88745904bf5
Author: Nik Everett <nik9000@gmail.com>
Date:   Fri May 20 15:01:53 2016 -0400

    Remove Translog#lastWriteLocation

    I wasn't being careful enough with locks so it wasn't right anyway.
    Instead this builds a synthetic Tranlog.Location when you call
    getWriteLocation with much more relaxed equality guarantees. Rather
    than being equal to the last Translog.Location returned it is
    simply guaranteed to be greater than the last translog returned
    and less than the next.

commit 55596ea68b5484490c3637fbad0d95564236478b
Author: Nik Everett <nik9000@gmail.com>
Date:   Fri May 20 14:40:06 2016 -0400

    Remove listener from shardOperationOnPrimary

    Create instead asyncShardOperationOnPrimary which is called after
    all of the replica operations are started to handle any async
    operations.

commit 3322e26211bf681b37132274ee158ae330afc28b
Author: Nik Everett <nik9000@gmail.com>
Date:   Tue May 17 17:20:02 2016 -0400

    Increase default maximum number of listeners to 1000

commit 88171a8322a424e624d48960fb4c98dd43e4d671
Author: Nik Everett <nik9000@gmail.com>
Date:   Tue May 17 16:40:57 2016 -0400

    Rename test

commit 179c27c4f829f2c6ded65967652cf85adaf2ae52
Author: Nik Everett <nik9000@gmail.com>
Date:   Tue May 17 16:35:27 2016 -0400

    Move refresh listeners into their own class

    They still live at the IndexShard level but they live on their
    own in RefreshListeners which interacts with IndexShard using a
    couple of callbacks and a registration method. This lets us test
    the listeners without standing up an entire IndexShard. We still
    test the listeners against an InternalEngine, because the interplay
    between InternalEngine, Translog, and RefreshListeners is complex
    and important to get right.

commit d8926d5fc1d24b4da8ccff7e0f0907b98c583c41
Author: Nik Everett <nik9000@gmail.com>
Date:   Tue May 17 11:02:38 2016 -0400

    Move refresh listeners into IndexShard

commit df91cde398eb720143a85a8c6fa19bdc3a74e07d
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon May 16 16:01:03 2016 -0400

    unused import

commit 066da45b08148b266e4173166662fc1b3f66ed53
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon May 16 15:54:11 2016 -0400

    Remove RefreshListener interface

    Just pass a Translog.Location and a Consumer<Boolean> when registering.

commit b971d6d3301c7522b2e7eb90d5d8dd96a77fa625
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon May 16 14:41:06 2016 -0400

    Docs for setForcedRefresh

commit 6c43be821eaf61141d3ec520f988aad3a96a3941
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon May 16 14:34:39 2016 -0400

    Rename refresh setter and getter

commit e61b7391f91263a4c4d6107bfbc2a828bbcc805c
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon Apr 25 22:48:09 2016 -0400

    Trigger listeners even when there is no refresh

    Each refresh gives us an opportunity to pick up any listeners we may
    have left behind.

commit 0c9b0477085c021f503db775640d25668e02f635
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon Apr 25 20:30:06 2016 -0400

    REST

commit 8250343240de7e63118c663a230a7a314807a754
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon Apr 25 19:34:22 2016 -0400

    Switch to estimated count

    We don't need a linear time count of the number of listeners - a volatile
    variable is good enough to guess. It probably undercounts more than it
    overcounts but it isn't a huge problem.

commit bd531167fe54f1bde6f6d4ddb0a8de5a7bcc18a2
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon Apr 25 18:21:02 2016 -0400

    Don't try and set forced refresh on bulk items without a response

    NullPointerExceptions are bad. If the entire request fails then the user
    has worse problems then "did these force a refresh".

commit bcfded11515af5e0b3c3e36f3c2f73f5cd26512e
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon Apr 25 18:14:20 2016 -0400

    Replace LinkedList and synchronized with LinkedTransferQueue

commit 8a80cc70a76375a7593745884cb987535b37ca80
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon Apr 25 17:38:24 2016 -0400

    Support for update

commit 1f36966742f851b7328015151ef6fc8f95299af2
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon Apr 25 15:46:06 2016 -0400

    Cleanup translog tests

commit 8d121bf35eb265b8a0aee9710afeb1b054a113d4
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon Apr 25 15:40:53 2016 -0400

    Cleanup listener implementation

    Much more testing too!

commit 2058f4a808762c4588309f21b13b677245832f2c
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon Apr 25 11:45:55 2016 -0400

    Pass back information about whether we refreshed

commit e445cb0cb91ebdbcfdbf566696edb2bf1c84a882
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon Apr 25 11:03:31 2016 -0400

    Javadoc

commit 611cbeeaeb458f4b428bfc43a1ee6652adf4baff
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon Apr 25 11:01:40 2016 -0400

    Move ReplicationResponse

    now it is in the same package as its request

commit 9919758b644fd73895fb88cd6a4909a8387eb2e2
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon Apr 25 11:00:14 2016 -0400

    Oh boy that wasn't working

commit 247cb483c4459dea8e95e0e3bd2e4bf8d452c598
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon Apr 25 10:29:37 2016 -0400

    Basic block_until_refresh exposed to java client

    and basic "is it plugged in" style tests.

commit 46c855c9971cb2b748206d2afa6a2d88724be3ba
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon Apr 25 10:11:10 2016 -0400

    Move test to own class

commit a5ffd892d0a352ae7e9757f2640fc2a1fa656bf2
Author: Nik Everett <nik9000@gmail.com>
Date:   Mon Apr 25 07:44:25 2016 -0400

    WIP

commit 213bebb6ece11b85d17e44af9a54fc2e5e332d39
Author: Nik Everett <nik9000@gmail.com>
Date:   Fri Apr 22 21:35:52 2016 -0400

    Add refresh listeners

commit a2bc7f30e6d4857a1224ef5a89909b36c8f33731
Author: Nik Everett <nik9000@gmail.com>
Date:   Fri Apr 22 21:11:55 2016 -0400

    Return last written location from refresh

commit 85033a87551da89f36a23d4dfd5016db218e08ee
Author: Nik Everett <nik9000@gmail.com>
Date:   Fri Apr 22 20:28:21 2016 -0400

    Never reply to replica actions while you have the operation lock

    This last thing was causing periodic test failures because we were
    replying while we had the operation lock. Now, we probably could get
    away with that in most cases but the tests don't like it and it isn't
    a good idea to do network io while you have a lock anyway. So this
    prevents it.

commit 1f25cf35e796835b3827b8a4110e09e5de61784c
Author: Nik Everett <nik9000@gmail.com>
Date:   Fri Apr 22 19:56:18 2016 -0400

    Cleanup

commit 52c5f7c3f04710901f503334239a611c0e21c85a
Author: Nik Everett <nik9000@gmail.com>
Date:   Fri Apr 22 19:33:00 2016 -0400

    Add a listener to shard operations

commit 5b142dc331214c8eef90587144f4b3f959f9eced
Author: Nik Everett <nik9000@gmail.com>
Date:   Fri Apr 22 18:03:52 2016 -0400

    Cleanup

commit 3d22b2d7ceb473db339259452a7c4f117ce86069
Author: Nik Everett <nik9000@gmail.com>
Date:   Fri Apr 22 17:59:55 2016 -0400

    Push the listener into shardOperationOnPrimary

commit 34b378943b8185451acf6350f661c0ad33b5836d
Author: Nik Everett <nik9000@gmail.com>
Date:   Fri Apr 22 17:48:47 2016 -0400

    Doc

commit b42b8da968d42cc7414020c7b199606a5dcce50a
Author: Nik Everett <nik9000@gmail.com>
Date:   Fri Apr 22 17:45:40 2016 -0400

    Don't finish early if the primary finishes early

    We use a "fake" pending shard that we resolve when the replicas have
    all started.

commit 0fc045b56e1e02a48c30383ac50a281d5af7e0b6
Author: Nik Everett <nik9000@gmail.com>
Date:   Fri Apr 22 17:30:06 2016 -0400

    Make performOnPrimary asyncS

    Instead of returning Tuple<Response, ReplicaRequest> it returns
    ReplicaRequest and takes a ActionListener<Response> as an argument.
    We call the listener immediately to preserve backwards compatibility
    for now.

commit 80119b9a26ede96a865af45904c3ac69d5b19b59
Author: Nik Everett <nik9000@gmail.com>
Date:   Fri Apr 22 16:51:53 2016 -0400

    Factor out common code in shardOperationOnPrimary

commit 0642083676702618f900fa842c08802a04c1a53e
Author: Nik Everett <nik9000@gmail.com>
Date:   Fri Apr 22 16:32:29 2016 -0400

    Factor out common code from shardOperationOnReplica

commit 8bdc415fedaaa9f2d0c555590a13ec4699a7c3f7
Author: Nik Everett <nik9000@gmail.com>
Date:   Fri Apr 22 16:23:28 2016 -0400

    Create ReplicatedMutationRequest

    Superclass for index, delete, and bulkShard requests.

commit 0f8fa846a2822c4293df32fed18c9b99660b39ff
Author: Nik Everett <nik9000@gmail.com>
Date:   Fri Apr 22 16:10:30 2016 -0400

    Create TransportReplicatedMutationAction

    It is the superclass of replication actions that mutate data: index, delete,
    and shardBulk. shardFlush and shardRefresh are replication actions but they
    do not extend TransportReplicatedMutationAction because they don't change
    the data, only shuffle it around.
---
 .../resources/checkstyle_suppressions.xml     |   2 -
 .../action/DocWriteResponse.java              |  27 +-
 .../indices/flush/TransportFlushAction.java   |   2 +-
 .../flush/TransportShardFlushAction.java      |  10 +-
 .../refresh/TransportRefreshAction.java       |   2 +-
 .../refresh/TransportShardRefreshAction.java  |  20 +-
 .../action/bulk/BulkRequest.java              |  38 +--
 .../action/bulk/BulkRequestBuilder.java       |  14 +-
 .../action/bulk/BulkShardRequest.java         |  25 +-
 .../action/bulk/BulkShardResponse.java        |  20 +-
 .../action/bulk/TransportBulkAction.java      |   3 +-
 .../action/bulk/TransportShardBulkAction.java |  50 +--
 .../action/delete/DeleteRequest.java          |  21 +-
 .../action/delete/DeleteRequestBuilder.java   |  14 +-
 .../action/delete/TransportDeleteAction.java  |  37 +--
 .../action/index/IndexRequest.java            |  21 +-
 .../action/index/IndexRequestBuilder.java     |  14 +-
 .../action/index/TransportIndexAction.java    |  42 +--
 .../action/ingest/IngestActionFilter.java     |   2 +-
 .../action/support/WriteRequest.java          | 109 +++++++
 .../action/support/WriteRequestBuilder.java   |  50 +++
 .../action/support/WriteResponse.java         |  40 +++
 .../replication/ReplicatedWriteRequest.java   |  72 +++++
 .../replication/ReplicationOperation.java     | 105 ++++---
 .../replication/ReplicationRequest.java       |   6 +-
 .../replication}/ReplicationResponse.java     |  10 +-
 .../TransportBroadcastReplicationAction.java  |   1 -
 .../TransportReplicationAction.java           | 147 +++++----
 .../replication/TransportWriteAction.java     | 227 ++++++++++++++
 .../action/update/TransportUpdateAction.java  |   7 +-
 .../action/update/UpdateHelper.java           |   7 +-
 .../action/update/UpdateRequest.java          |  25 +-
 .../action/update/UpdateRequestBuilder.java   |  17 +-
 .../common/settings/IndexScopedSettings.java  |   1 +
 .../elasticsearch/index/IndexSettings.java    |  21 ++
 .../elasticsearch/index/engine/Engine.java    |   7 +
 .../index/engine/EngineConfig.java            |  16 +-
 .../index/engine/InternalEngine.java          |   4 +
 .../index/engine/ShadowEngine.java            |   4 +-
 .../elasticsearch/index/shard/IndexShard.java |  38 ++-
 .../index/shard/RefreshListeners.java         | 208 +++++++++++++
 .../index/shard/ShadowIndexShard.java         |  13 +
 .../index/translog/Translog.java              |  15 +
 .../index/translog/TranslogWriter.java        |   1 +
 .../rest/action/bulk/RestBulkAction.java      |   2 +-
 .../rest/action/delete/RestDeleteAction.java  |   2 +-
 .../rest/action/index/RestIndexAction.java    |   2 +-
 .../rest/action/update/RestUpdateAction.java  |   2 +-
 .../action/bulk/BulkRequestTests.java         |  17 +-
 .../action/bulk/BulkShardRequestTests.java    |   7 +-
 .../BroadcastReplicationTests.java            |   1 -
 .../ReplicationOperationTests.java            |  67 ++--
 .../TransportReplicationActionTests.java      |  62 ++--
 .../TransportWriteActionTests.java            | 190 ++++++++++++
 .../elasticsearch/aliases/IndexAliasesIT.java |  14 +-
 .../cluster/allocation/ClusterRerouteIT.java  |   3 +-
 .../elasticsearch/document/ShardInfoIT.java   |   2 +-
 .../index/WaitUntilRefreshIT.java             | 217 +++++++++++++
 .../index/engine/InternalEngineTests.java     |  35 ++-
 .../index/engine/ShadowEngineTests.java       |  28 +-
 ...lFieldMapperPositionIncrementGapTests.java |   3 +-
 .../index/shard/RefreshListenersTests.java    | 292 ++++++++++++++++++
 .../index/translog/TranslogTests.java         |  24 ++
 .../elasticsearch/routing/AliasRoutingIT.java |  29 +-
 .../routing/SimpleRoutingIT.java              |  28 +-
 .../search/child/ChildQuerySearchIT.java      |   8 +-
 .../suggest/CompletionSuggestSearch2xIT.java  |  25 +-
 docs/reference/docs.asciidoc                  |   4 +
 docs/reference/docs/bulk.asciidoc             |   8 +-
 docs/reference/docs/delete.asciidoc           |   8 +-
 docs/reference/docs/index_.asciidoc           |  15 +-
 docs/reference/docs/refresh.asciidoc          | 109 +++++++
 docs/reference/docs/update.asciidoc           |   5 +-
 docs/reference/index-modules.asciidoc         |   5 +
 .../migration/migrate_5_0/docs.asciidoc       |  10 +
 .../migration/migrate_5_0/java.asciidoc       |   5 +
 .../elasticsearch/messy/tests/BulkTests.java  |   9 +-
 .../percolator/PercolatorIT.java              |  39 +--
 .../resources/rest-api-spec/api/bulk.json     |   5 +-
 .../resources/rest-api-spec/api/delete.json   |   5 +-
 .../resources/rest-api-spec/api/index.json    |   5 +-
 .../resources/rest-api-spec/api/update.json   |   5 +-
 .../rest-api-spec/test/bulk/50_refresh.yaml   |  48 +++
 .../rest-api-spec/test/create/60_refresh.yaml |  42 ++-
 .../rest-api-spec/test/delete/50_refresh.yaml |  76 ++++-
 .../rest-api-spec/test/index/60_refresh.yaml  |  42 ++-
 .../rest-api-spec/test/update/60_refresh.yaml |  70 ++++-
 87 files changed, 2542 insertions(+), 548 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/action/support/WriteRequest.java
 create mode 100644 core/src/main/java/org/elasticsearch/action/support/WriteRequestBuilder.java
 create mode 100644 core/src/main/java/org/elasticsearch/action/support/WriteResponse.java
 create mode 100644 core/src/main/java/org/elasticsearch/action/support/replication/ReplicatedWriteRequest.java
 rename core/src/main/java/org/elasticsearch/action/{ => support/replication}/ReplicationResponse.java (96%)
 create mode 100644 core/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java
 create mode 100644 core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
 create mode 100644 core/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
 create mode 100644 core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java
 create mode 100644 core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
 create mode 100644 docs/reference/docs/refresh.asciidoc
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/bulk/50_refresh.yaml

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 0245cda2fe1..3c5caf2d4e2 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -19,7 +19,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]Action.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ActionModule.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ActionRequestBuilder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]ReplicationResponse.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]health[/\\]ClusterHealthRequestBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]health[/\\]TransportClusterHealthAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]node[/\\]hotthreads[/\\]NodesHotThreadsRequestBuilder.java" checks="LineLength" />
@@ -101,7 +100,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]open[/\\]TransportOpenIndexAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]recovery[/\\]TransportRecoveryAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]refresh[/\\]TransportRefreshAction.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]refresh[/\\]TransportShardRefreshAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]segments[/\\]IndexSegments.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]segments[/\\]IndicesSegmentResponse.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]segments[/\\]IndicesSegmentsRequestBuilder.java" checks="LineLength" />
diff --git a/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java b/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java
index 4df43b75401..0925c744144 100644
--- a/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java
@@ -18,10 +18,15 @@
  */
 package org.elasticsearch.action;
 
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.action.support.WriteResponse;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.action.support.replication.ReplicationResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.StatusToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 
@@ -30,12 +35,13 @@ import java.io.IOException;
 /**
  * A base class for the response of a write operation that involves a single doc
  */
-public abstract class DocWriteResponse extends ReplicationResponse implements StatusToXContent {
+public abstract class DocWriteResponse extends ReplicationResponse implements WriteResponse, StatusToXContent {
 
     private ShardId shardId;
     private String id;
     private String type;
     private long version;
+    private boolean forcedRefresh;
 
     public DocWriteResponse(ShardId shardId, String type, String id, long version) {
         this.shardId = shardId;
@@ -84,6 +90,20 @@ public abstract class DocWriteResponse extends ReplicationResponse implements St
         return this.version;
     }
 
+    /**
+     * Did this request force a refresh? Requests that set {@link WriteRequest#setRefreshPolicy(RefreshPolicy)} to
+     * {@link RefreshPolicy#IMMEDIATE} will always return true for this. Requests that set it to {@link RefreshPolicy#WAIT_UNTIL} will
+     * only return true here if they run out of refresh listener slots (see {@link IndexSettings#MAX_REFRESH_LISTENERS_PER_SHARD}).
+     */
+    public boolean forcedRefresh() {
+        return forcedRefresh;
+    }
+
+    @Override
+    public void setForcedRefresh(boolean forcedRefresh) {
+        this.forcedRefresh = forcedRefresh;
+    }
+
     /** returns the rest status for this response (based on {@link ShardInfo#status()} */
     public RestStatus status() {
         return getShardInfo().status();
@@ -97,6 +117,7 @@ public abstract class DocWriteResponse extends ReplicationResponse implements St
         type = in.readString();
         id = in.readString();
         version = in.readZLong();
+        forcedRefresh = in.readBoolean();
     }
 
     @Override
@@ -106,6 +127,7 @@ public abstract class DocWriteResponse extends ReplicationResponse implements St
         out.writeString(type);
         out.writeString(id);
         out.writeZLong(version);
+        out.writeBoolean(forcedRefresh);
     }
 
     static final class Fields {
@@ -121,7 +143,8 @@ public abstract class DocWriteResponse extends ReplicationResponse implements St
         builder.field(Fields._INDEX, shardId.getIndexName())
             .field(Fields._TYPE, type)
             .field(Fields._ID, id)
-            .field(Fields._VERSION, version);
+            .field(Fields._VERSION, version)
+            .field("forced_refresh", forcedRefresh);
         shardInfo.toXContent(builder, params);
         return builder;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java
index 8bb124d8fc4..a29918b438e 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.admin.indices.flush;
 
-import org.elasticsearch.action.ReplicationResponse;
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.replication.ReplicationResponse;
 import org.elasticsearch.action.support.replication.TransportBroadcastReplicationAction;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java
index 7e750b97677..82fb6d70ca4 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java
@@ -19,14 +19,13 @@
 
 package org.elasticsearch.action.admin.indices.flush;
 
-import org.elasticsearch.action.ReplicationResponse;
 import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.replication.ReplicationResponse;
 import org.elasticsearch.action.support.replication.TransportReplicationAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.shard.IndexShard;
@@ -55,18 +54,19 @@ public class TransportShardFlushAction extends TransportReplicationAction<ShardF
     }
 
     @Override
-    protected Tuple<ReplicationResponse, ShardFlushRequest> shardOperationOnPrimary(ShardFlushRequest shardRequest) {
+    protected PrimaryResult shardOperationOnPrimary(ShardFlushRequest shardRequest) {
         IndexShard indexShard = indicesService.indexServiceSafe(shardRequest.shardId().getIndex()).getShard(shardRequest.shardId().id());
         indexShard.flush(shardRequest.getRequest());
         logger.trace("{} flush request executed on primary", indexShard.shardId());
-        return new Tuple<>(new ReplicationResponse(), shardRequest);
+        return new PrimaryResult(shardRequest, new ReplicationResponse());
     }
 
     @Override
-    protected void shardOperationOnReplica(ShardFlushRequest request) {
+    protected ReplicaResult shardOperationOnReplica(ShardFlushRequest request) {
         IndexShard indexShard = indicesService.indexServiceSafe(request.shardId().getIndex()).getShard(request.shardId().id());
         indexShard.flush(request.getRequest());
         logger.trace("{} flush request executed on replica", indexShard.shardId());
+        return new ReplicaResult();
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java
index 34bf39daabd..ac64e276778 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java
@@ -19,10 +19,10 @@
 
 package org.elasticsearch.action.admin.indices.refresh;
 
-import org.elasticsearch.action.ReplicationResponse;
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.replication.BasicReplicationRequest;
+import org.elasticsearch.action.support.replication.ReplicationResponse;
 import org.elasticsearch.action.support.replication.TransportBroadcastReplicationAction;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java
index 0670c1f3cc6..d7d0c289953 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java
@@ -19,15 +19,14 @@
 
 package org.elasticsearch.action.admin.indices.refresh;
 
-import org.elasticsearch.action.ReplicationResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.replication.BasicReplicationRequest;
+import org.elasticsearch.action.support.replication.ReplicationResponse;
 import org.elasticsearch.action.support.replication.TransportReplicationAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.shard.IndexShard;
@@ -36,10 +35,8 @@ import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-/**
- *
- */
-public class TransportShardRefreshAction extends TransportReplicationAction<BasicReplicationRequest, BasicReplicationRequest, ReplicationResponse> {
+public class TransportShardRefreshAction
+        extends TransportReplicationAction<BasicReplicationRequest, BasicReplicationRequest, ReplicationResponse> {
 
     public static final String NAME = RefreshAction.NAME + "[s]";
 
@@ -47,8 +44,8 @@ public class TransportShardRefreshAction extends TransportReplicationAction<Basi
     public TransportShardRefreshAction(Settings settings, TransportService transportService, ClusterService clusterService,
                                        IndicesService indicesService, ThreadPool threadPool, ShardStateAction shardStateAction,
                                        ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, NAME, transportService, clusterService, indicesService, threadPool, shardStateAction,
-            actionFilters, indexNameExpressionResolver, BasicReplicationRequest::new, BasicReplicationRequest::new, ThreadPool.Names.REFRESH);
+        super(settings, NAME, transportService, clusterService, indicesService, threadPool, shardStateAction, actionFilters,
+                indexNameExpressionResolver, BasicReplicationRequest::new, BasicReplicationRequest::new, ThreadPool.Names.REFRESH);
     }
 
     @Override
@@ -57,19 +54,20 @@ public class TransportShardRefreshAction extends TransportReplicationAction<Basi
     }
 
     @Override
-    protected Tuple<ReplicationResponse, BasicReplicationRequest> shardOperationOnPrimary(BasicReplicationRequest shardRequest) {
+    protected PrimaryResult shardOperationOnPrimary(BasicReplicationRequest shardRequest) {
         IndexShard indexShard = indicesService.indexServiceSafe(shardRequest.shardId().getIndex()).getShard(shardRequest.shardId().id());
         indexShard.refresh("api");
         logger.trace("{} refresh request executed on primary", indexShard.shardId());
-        return new Tuple<>(new ReplicationResponse(), shardRequest);
+        return new PrimaryResult(shardRequest, new ReplicationResponse());
     }
 
     @Override
-    protected void shardOperationOnReplica(BasicReplicationRequest request) {
+    protected ReplicaResult shardOperationOnReplica(BasicReplicationRequest request) {
         final ShardId shardId = request.shardId();
         IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).getShard(shardId.id());
         indexShard.refresh("api");
         logger.trace("{} refresh request executed on replica", indexShard.shardId());
+        return new ReplicaResult();
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
index a5775656475..85d7147ada0 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
@@ -26,6 +26,7 @@ import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.WriteConsistencyLevel;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
@@ -54,16 +55,21 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
  * Note that we only support refresh on the bulk request not per item.
  * @see org.elasticsearch.client.Client#bulk(BulkRequest)
  */
-public class BulkRequest extends ActionRequest<BulkRequest> implements CompositeIndicesRequest {
+public class BulkRequest extends ActionRequest<BulkRequest> implements CompositeIndicesRequest, WriteRequest<BulkRequest> {
 
     private static final int REQUEST_OVERHEAD = 50;
 
+    /**
+     * Requests that are part of this request. It is only possible to add things that are both {@link ActionRequest}s and
+     * {@link WriteRequest}s to this but java doesn't support syntax to declare that everything in the array has both types so we declare
+     * the one with the least casts.
+     */
     final List<ActionRequest<?>> requests = new ArrayList<>();
     List<Object> payloads = null;
 
     protected TimeValue timeout = BulkShardRequest.DEFAULT_TIMEOUT;
     private WriteConsistencyLevel consistencyLevel = WriteConsistencyLevel.DEFAULT;
-    private boolean refresh = false;
+    private RefreshPolicy refreshPolicy = RefreshPolicy.NONE;
 
     private long sizeInBytes = 0;
 
@@ -437,18 +443,15 @@ public class BulkRequest extends ActionRequest<BulkRequest> implements Composite
         return this.consistencyLevel;
     }
 
-    /**
-     * Should a refresh be executed post this bulk operation causing the operations to
-     * be searchable. Note, heavy indexing should not set this to <tt>true</tt>. Defaults
-     * to <tt>false</tt>.
-     */
-    public BulkRequest refresh(boolean refresh) {
-        this.refresh = refresh;
+    @Override
+    public BulkRequest setRefreshPolicy(RefreshPolicy refreshPolicy) {
+        this.refreshPolicy = refreshPolicy;
         return this;
     }
 
-    public boolean refresh() {
-        return this.refresh;
+    @Override
+    public RefreshPolicy getRefreshPolicy() {
+        return refreshPolicy;
     }
 
     /**
@@ -483,7 +486,7 @@ public class BulkRequest extends ActionRequest<BulkRequest> implements Composite
      * @return Whether this bulk request contains index request with an ingest pipeline enabled.
      */
     public boolean hasIndexRequestsWithPipelines() {
-        for (ActionRequest actionRequest : requests) {
+        for (ActionRequest<?> actionRequest : requests) {
             if (actionRequest instanceof IndexRequest) {
                 IndexRequest indexRequest = (IndexRequest) actionRequest;
                 if (Strings.hasText(indexRequest.getPipeline())) {
@@ -503,10 +506,9 @@ public class BulkRequest extends ActionRequest<BulkRequest> implements Composite
         }
         for (ActionRequest<?> request : requests) {
             // We first check if refresh has been set
-            if ((request instanceof DeleteRequest && ((DeleteRequest)request).refresh()) ||
-                    (request instanceof UpdateRequest && ((UpdateRequest)request).refresh()) ||
-                    (request instanceof IndexRequest && ((IndexRequest)request).refresh())) {
-                    validationException = addValidationError("Refresh is not supported on an item request, set the refresh flag on the BulkRequest instead.", validationException);
+            if (((WriteRequest<?>) request).getRefreshPolicy() != RefreshPolicy.NONE) {
+                validationException = addValidationError(
+                        "RefreshPolicy is not supported on an item request. Set it on the BulkRequest instead.", validationException);
             }
             ActionRequestValidationException ex = request.validate();
             if (ex != null) {
@@ -541,7 +543,7 @@ public class BulkRequest extends ActionRequest<BulkRequest> implements Composite
                 requests.add(request);
             }
         }
-        refresh = in.readBoolean();
+        refreshPolicy = RefreshPolicy.readFrom(in);
         timeout = TimeValue.readTimeValue(in);
     }
 
@@ -560,7 +562,7 @@ public class BulkRequest extends ActionRequest<BulkRequest> implements Composite
             }
             request.writeTo(out);
         }
-        out.writeBoolean(refresh);
+        refreshPolicy.writeTo(out);
         timeout.writeTo(out);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java
index 3744055d26c..4f2b7aa702e 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java
@@ -25,6 +25,7 @@ import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.delete.DeleteRequestBuilder;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.support.WriteRequestBuilder;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
@@ -35,7 +36,8 @@ import org.elasticsearch.common.unit.TimeValue;
  * A bulk request holds an ordered {@link IndexRequest}s and {@link DeleteRequest}s and allows to executes
  * it in a single batch.
  */
-public class BulkRequestBuilder extends ActionRequestBuilder<BulkRequest, BulkResponse, BulkRequestBuilder> {
+public class BulkRequestBuilder extends ActionRequestBuilder<BulkRequest, BulkResponse, BulkRequestBuilder>
+        implements WriteRequestBuilder<BulkRequestBuilder> {
 
     public BulkRequestBuilder(ElasticsearchClient client, BulkAction action) {
         super(client, action, new BulkRequest());
@@ -116,16 +118,6 @@ public class BulkRequestBuilder extends ActionRequestBuilder<BulkRequest, BulkRe
         return this;
     }
 
-    /**
-     * Should a refresh be executed post this bulk operation causing the operations to
-     * be searchable. Note, heavy indexing should not set this to <tt>true</tt>. Defaults
-     * to <tt>false</tt>.
-     */
-    public BulkRequestBuilder setRefresh(boolean refresh) {
-        request.refresh(refresh);
-        return this;
-    }
-
     /**
      * A timeout to wait if the index operation can't be performed immediately. Defaults to <tt>1m</tt>.
      */
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java
index 874789e8d61..321b7e2a8e5 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.action.bulk;
 
-import org.elasticsearch.action.support.replication.ReplicationRequest;
+import org.elasticsearch.action.support.replication.ReplicatedWriteRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.shard.ShardId;
@@ -31,23 +31,17 @@ import java.util.List;
 /**
  *
  */
-public class BulkShardRequest extends ReplicationRequest<BulkShardRequest> {
+public class BulkShardRequest extends ReplicatedWriteRequest<BulkShardRequest> {
 
     private BulkItemRequest[] items;
 
-    private boolean refresh;
-
     public BulkShardRequest() {
     }
 
-    BulkShardRequest(BulkRequest bulkRequest, ShardId shardId, boolean refresh, BulkItemRequest[] items) {
+    BulkShardRequest(BulkRequest bulkRequest, ShardId shardId, RefreshPolicy refreshPolicy, BulkItemRequest[] items) {
         super(shardId);
         this.items = items;
-        this.refresh = refresh;
-    }
-
-    boolean refresh() {
-        return this.refresh;
+        setRefreshPolicy(refreshPolicy);
     }
 
     BulkItemRequest[] items() {
@@ -77,7 +71,6 @@ public class BulkShardRequest extends ReplicationRequest<BulkShardRequest> {
                 out.writeBoolean(false);
             }
         }
-        out.writeBoolean(refresh);
     }
 
     @Override
@@ -89,7 +82,6 @@ public class BulkShardRequest extends ReplicationRequest<BulkShardRequest> {
                 items[i] = BulkItemRequest.readBulkItem(in);
             }
         }
-        refresh = in.readBoolean();
     }
 
     @Override
@@ -97,8 +89,15 @@ public class BulkShardRequest extends ReplicationRequest<BulkShardRequest> {
         // This is included in error messages so we'll try to make it somewhat user friendly.
         StringBuilder b = new StringBuilder("BulkShardRequest to [");
         b.append(index).append("] containing [").append(items.length).append("] requests");
-        if (refresh) {
+        switch (getRefreshPolicy()) {
+        case IMMEDIATE:
             b.append(" and a refresh");
+            break;
+        case WAIT_UNTIL:
+            b.append(" blocking until refresh");
+            break;
+        case NONE:
+            break;
         }
         return b.toString();
     }
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java
index 76c80a9b064..22260181bb1 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java
@@ -19,7 +19,9 @@
 
 package org.elasticsearch.action.bulk;
 
-import org.elasticsearch.action.ReplicationResponse;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.support.WriteResponse;
+import org.elasticsearch.action.support.replication.ReplicationResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.shard.ShardId;
@@ -29,7 +31,7 @@ import java.io.IOException;
 /**
  *
  */
-public class BulkShardResponse extends ReplicationResponse {
+public class BulkShardResponse extends ReplicationResponse implements WriteResponse {
 
     private ShardId shardId;
     private BulkItemResponse[] responses;
@@ -50,6 +52,20 @@ public class BulkShardResponse extends ReplicationResponse {
         return responses;
     }
 
+    @Override
+    public void setForcedRefresh(boolean forcedRefresh) {
+        /*
+         * Each DocWriteResponse already has a location for whether or not it forced a refresh so we just set that information on the
+         * response.
+         */
+        for (BulkItemResponse response : responses) {
+            DocWriteResponse r = response.getResponse();
+            if (r != null) {
+                r.setForcedRefresh(forcedRefresh);
+            }
+        }
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
index 667e691f6c8..4cbebd0739a 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
@@ -344,7 +344,8 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
         for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) {
             final ShardId shardId = entry.getKey();
             final List<BulkItemRequest> requests = entry.getValue();
-            BulkShardRequest bulkShardRequest = new BulkShardRequest(bulkRequest, shardId, bulkRequest.refresh(), requests.toArray(new BulkItemRequest[requests.size()]));
+            BulkShardRequest bulkShardRequest = new BulkShardRequest(bulkRequest, shardId, bulkRequest.getRefreshPolicy(),
+                    requests.toArray(new BulkItemRequest[requests.size()]));
             bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel());
             bulkShardRequest.timeout(bulkRequest.timeout());
             if (task != null) {
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
index a2f642374b7..4ad1136e668 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
@@ -30,7 +30,8 @@ import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.index.TransportIndexAction;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.replication.ReplicationRequest;
-import org.elasticsearch.action.support.replication.TransportReplicationAction;
+import org.elasticsearch.action.support.replication.TransportWriteAction;
+import org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo;
 import org.elasticsearch.action.update.UpdateHelper;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateResponse;
@@ -53,6 +54,7 @@ import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.translog.Translog;
+import org.elasticsearch.index.translog.Translog.Location;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -67,7 +69,7 @@ import static org.elasticsearch.action.support.replication.ReplicationOperation.
 /**
  * Performs the index operation.
  */
-public class TransportShardBulkAction extends TransportReplicationAction<BulkShardRequest, BulkShardRequest, BulkShardResponse> {
+public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequest, BulkShardResponse> {
 
     private final static String OP_TYPE_UPDATE = "update";
     private final static String OP_TYPE_DELETE = "delete";
@@ -83,9 +85,8 @@ public class TransportShardBulkAction extends TransportReplicationAction<BulkSha
                                     IndicesService indicesService, ThreadPool threadPool, ShardStateAction shardStateAction,
                                     MappingUpdatedAction mappingUpdatedAction, UpdateHelper updateHelper, ActionFilters actionFilters,
                                     IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, ACTION_NAME, transportService, clusterService, indicesService, threadPool, shardStateAction,
-            actionFilters, indexNameExpressionResolver,
-                BulkShardRequest::new, BulkShardRequest::new, ThreadPool.Names.BULK);
+        super(settings, ACTION_NAME, transportService, clusterService, indicesService, threadPool, shardStateAction, actionFilters,
+                indexNameExpressionResolver, BulkShardRequest::new, ThreadPool.Names.BULK);
         this.updateHelper = updateHelper;
         this.allowIdGeneration = settings.getAsBoolean("action.allow_id_generation", true);
         this.mappingUpdatedAction = mappingUpdatedAction;
@@ -107,10 +108,9 @@ public class TransportShardBulkAction extends TransportReplicationAction<BulkSha
     }
 
     @Override
-    protected Tuple<BulkShardResponse, BulkShardRequest> shardOperationOnPrimary(BulkShardRequest request) {
+    protected WriteResult<BulkShardResponse> onPrimaryShard(BulkShardRequest request, IndexShard indexShard) throws Exception {
         ShardId shardId = request.shardId();
         final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
-        final IndexShard indexShard = indexService.getShard(shardId.getId());
         final IndexMetaData metaData = indexService.getIndexSettings().getIndexMetaData();
 
         long[] preVersions = new long[request.items().length];
@@ -121,13 +121,13 @@ public class TransportShardBulkAction extends TransportReplicationAction<BulkSha
             location = handleItem(metaData, request, indexShard, preVersions, preVersionTypes, location, requestIndex, item);
         }
 
-        processAfterWrite(request.refresh(), indexShard, location);
         BulkItemResponse[] responses = new BulkItemResponse[request.items().length];
         BulkItemRequest[] items = request.items();
         for (int i = 0; i < items.length; i++) {
             responses[i] = items[i].getPrimaryResponse();
         }
-        return new Tuple<>(new BulkShardResponse(request.shardId(), responses), request);
+        BulkShardResponse response = new BulkShardResponse(request.shardId(), responses);
+        return new WriteResult<>(response, location);
     }
 
     private Translog.Location handleItem(IndexMetaData metaData, BulkShardRequest request, IndexShard indexShard, long[] preVersions, VersionType[] preVersionTypes, Translog.Location location, int requestIndex, BulkItemRequest item) {
@@ -154,9 +154,9 @@ public class TransportShardBulkAction extends TransportReplicationAction<BulkSha
         preVersionTypes[requestIndex] = indexRequest.versionType();
         try {
             WriteResult<IndexResponse> result = shardIndexOperation(request, indexRequest, metaData, indexShard, true);
-            location = locationToSync(location, result.location);
+            location = locationToSync(location, result.getLocation());
             // add the response
-            IndexResponse indexResponse = result.response();
+            IndexResponse indexResponse = result.getResponse();
             setResponse(item, new BulkItemResponse(item.id(), indexRequest.opType().lowercase(), indexResponse));
         } catch (Throwable e) {
             // rethrow the failure if we are going to retry on primary and let parent failure to handle it
@@ -197,8 +197,8 @@ public class TransportShardBulkAction extends TransportReplicationAction<BulkSha
         try {
             // add the response
             final WriteResult<DeleteResponse> writeResult = TransportDeleteAction.executeDeleteRequestOnPrimary(deleteRequest, indexShard);
-            DeleteResponse deleteResponse = writeResult.response();
-            location = locationToSync(location, writeResult.location);
+            DeleteResponse deleteResponse = writeResult.getResponse();
+            location = locationToSync(location, writeResult.getLocation());
             setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_DELETE, deleteResponse));
         } catch (Throwable e) {
             // rethrow the failure if we are going to retry on primary and let parent failure to handle it
@@ -237,16 +237,17 @@ public class TransportShardBulkAction extends TransportReplicationAction<BulkSha
             }
             if (updateResult.success()) {
                 if (updateResult.writeResult != null) {
-                    location = locationToSync(location, updateResult.writeResult.location);
+                    location = locationToSync(location, updateResult.writeResult.getLocation());
                 }
                 switch (updateResult.result.operation()) {
                     case UPSERT:
                     case INDEX:
+                        @SuppressWarnings("unchecked")
                         WriteResult<IndexResponse> result = updateResult.writeResult;
                         IndexRequest indexRequest = updateResult.request();
                         BytesReference indexSourceAsBytes = indexRequest.source();
                         // add the response
-                        IndexResponse indexResponse = result.response();
+                        IndexResponse indexResponse = result.getResponse();
                         UpdateResponse updateResponse = new UpdateResponse(indexResponse.getShardInfo(), indexResponse.getShardId(), indexResponse.getType(), indexResponse.getId(), indexResponse.getVersion(), indexResponse.isCreated());
                         if (updateRequest.fields() != null && updateRequest.fields().length > 0) {
                             Tuple<XContentType, Map<String, Object>> sourceAndContent = XContentHelper.convertToMap(indexSourceAsBytes, true);
@@ -256,8 +257,9 @@ public class TransportShardBulkAction extends TransportReplicationAction<BulkSha
                         setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse));
                         break;
                     case DELETE:
+                        @SuppressWarnings("unchecked")
                         WriteResult<DeleteResponse> writeResult = updateResult.writeResult;
-                        DeleteResponse response = writeResult.response();
+                        DeleteResponse response = writeResult.getResponse();
                         DeleteRequest deleteRequest = updateResult.request();
                         updateResponse = new UpdateResponse(response.getShardInfo(), response.getShardId(), response.getType(), response.getId(), response.getVersion(), false);
                         updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest, request.index(), response.getVersion(), updateResult.result.updatedSourceAsMap(), updateResult.result.updateSourceContentType(), null));
@@ -326,11 +328,14 @@ public class TransportShardBulkAction extends TransportReplicationAction<BulkSha
         request.setPrimaryResponse(response);
         if (response.isFailed()) {
             request.setIgnoreOnReplica();
+        } else {
+            // Set the ShardInfo to 0 so we can safely send it to the replicas. We won't use it in the real response though.
+            response.getResponse().setShardInfo(new ShardInfo());
         }
     }
 
-    private WriteResult shardIndexOperation(BulkShardRequest request, IndexRequest indexRequest, IndexMetaData metaData,
-                                            IndexShard indexShard, boolean processed) throws Throwable {
+    private WriteResult<IndexResponse> shardIndexOperation(BulkShardRequest request, IndexRequest indexRequest, IndexMetaData metaData,
+            IndexShard indexShard, boolean processed) throws Throwable {
 
         MappingMetaData mappingMd = metaData.mappingOrDefault(indexRequest.type());
         if (!processed) {
@@ -431,12 +436,8 @@ public class TransportShardBulkAction extends TransportReplicationAction<BulkSha
         }
     }
 
-
     @Override
-    protected void shardOperationOnReplica(BulkShardRequest request) {
-        final ShardId shardId = request.shardId();
-        IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
-        IndexShard indexShard = indexService.getShard(shardId.id());
+    protected Location onReplicaShard(BulkShardRequest request, IndexShard indexShard) {
         Translog.Location location = null;
         for (int i = 0; i < request.items().length; i++) {
             BulkItemRequest item = request.items()[i];
@@ -472,8 +473,7 @@ public class TransportShardBulkAction extends TransportReplicationAction<BulkSha
                 throw new IllegalStateException("Unexpected index operation: " + item.request());
             }
         }
-
-        processAfterWrite(request.refresh(), indexShard, location);
+        return location;
     }
 
     private void applyVersion(BulkItemRequest item, long version, VersionType versionType) {
diff --git a/core/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java b/core/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java
index cbd10553522..bdf09e3e532 100644
--- a/core/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java
@@ -21,7 +21,7 @@ package org.elasticsearch.action.delete;
 
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.DocumentRequest;
-import org.elasticsearch.action.support.replication.ReplicationRequest;
+import org.elasticsearch.action.support.replication.ReplicatedWriteRequest;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -43,7 +43,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
  * @see org.elasticsearch.client.Client#delete(DeleteRequest)
  * @see org.elasticsearch.client.Requests#deleteRequest(String)
  */
-public class DeleteRequest extends ReplicationRequest<DeleteRequest> implements DocumentRequest<DeleteRequest> {
+public class DeleteRequest extends ReplicatedWriteRequest<DeleteRequest> implements DocumentRequest<DeleteRequest> {
 
     private String type;
     private String id;
@@ -51,7 +51,6 @@ public class DeleteRequest extends ReplicationRequest<DeleteRequest> implements
     private String routing;
     @Nullable
     private String parent;
-    private boolean refresh;
     private long version = Versions.MATCH_ANY;
     private VersionType versionType = VersionType.INTERNAL;
 
@@ -165,20 +164,6 @@ public class DeleteRequest extends ReplicationRequest<DeleteRequest> implements
         return this.routing;
     }
 
-    /**
-     * Should a refresh be executed post this index operation causing the operation to
-     * be searchable. Note, heavy indexing should not set this to <tt>true</tt>. Defaults
-     * to <tt>false</tt>.
-     */
-    public DeleteRequest refresh(boolean refresh) {
-        this.refresh = refresh;
-        return this;
-    }
-
-    public boolean refresh() {
-        return this.refresh;
-    }
-
     /**
      * Sets the version, which will cause the delete operation to only be performed if a matching
      * version exists and no changes happened on the doc since then.
@@ -208,7 +193,6 @@ public class DeleteRequest extends ReplicationRequest<DeleteRequest> implements
         id = in.readString();
         routing = in.readOptionalString();
         parent = in.readOptionalString();
-        refresh = in.readBoolean();
         version = in.readLong();
         versionType = VersionType.fromValue(in.readByte());
     }
@@ -220,7 +204,6 @@ public class DeleteRequest extends ReplicationRequest<DeleteRequest> implements
         out.writeString(id);
         out.writeOptionalString(routing());
         out.writeOptionalString(parent());
-        out.writeBoolean(refresh);
         out.writeLong(version);
         out.writeByte(versionType.getValue());
     }
diff --git a/core/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java
index 0ce907bac1d..b9b0f95f8de 100644
--- a/core/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.delete;
 
+import org.elasticsearch.action.support.WriteRequestBuilder;
 import org.elasticsearch.action.support.replication.ReplicationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.Nullable;
@@ -27,7 +28,8 @@ import org.elasticsearch.index.VersionType;
 /**
  * A delete document action request builder.
  */
-public class DeleteRequestBuilder extends ReplicationRequestBuilder<DeleteRequest, DeleteResponse, DeleteRequestBuilder> {
+public class DeleteRequestBuilder extends ReplicationRequestBuilder<DeleteRequest, DeleteResponse, DeleteRequestBuilder>
+        implements WriteRequestBuilder<DeleteRequestBuilder> {
 
     public DeleteRequestBuilder(ElasticsearchClient client, DeleteAction action) {
         super(client, action, new DeleteRequest());
@@ -71,16 +73,6 @@ public class DeleteRequestBuilder extends ReplicationRequestBuilder<DeleteReques
         return this;
     }
 
-    /**
-     * Should a refresh be executed post this index operation causing the operation to
-     * be searchable. Note, heavy indexing should not set this to <tt>true</tt>. Defaults
-     * to <tt>false</tt>.
-     */
-    public DeleteRequestBuilder setRefresh(boolean refresh) {
-        request.refresh(refresh);
-        return this;
-    }
-
     /**
      * Sets the version, which will cause the delete operation to only be performed if a matching
      * version exists and no changes happened on the doc since then.
diff --git a/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java b/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java
index 62d46766c47..beced23c338 100644
--- a/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java
+++ b/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java
@@ -27,19 +27,19 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.AutoCreateIndex;
-import org.elasticsearch.action.support.replication.TransportReplicationAction;
+import org.elasticsearch.action.support.replication.TransportWriteAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.translog.Translog.Location;
 import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.tasks.Task;
@@ -49,7 +49,7 @@ import org.elasticsearch.transport.TransportService;
 /**
  * Performs the delete operation.
  */
-public class TransportDeleteAction extends TransportReplicationAction<DeleteRequest, DeleteRequest, DeleteResponse> {
+public class TransportDeleteAction extends TransportWriteAction<DeleteRequest, DeleteResponse> {
 
     private final AutoCreateIndex autoCreateIndex;
     private final TransportCreateIndexAction createIndexAction;
@@ -60,9 +60,8 @@ public class TransportDeleteAction extends TransportReplicationAction<DeleteRequ
                                  TransportCreateIndexAction createIndexAction, ActionFilters actionFilters,
                                  IndexNameExpressionResolver indexNameExpressionResolver,
                                  AutoCreateIndex autoCreateIndex) {
-        super(settings, DeleteAction.NAME, transportService, clusterService, indicesService, threadPool, shardStateAction,
-            actionFilters, indexNameExpressionResolver,
-            DeleteRequest::new, DeleteRequest::new, ThreadPool.Names.INDEX);
+        super(settings, DeleteAction.NAME, transportService, clusterService, indicesService, threadPool, shardStateAction, actionFilters,
+                indexNameExpressionResolver, DeleteRequest::new, ThreadPool.Names.INDEX);
         this.createIndexAction = createIndexAction;
         this.autoCreateIndex = autoCreateIndex;
     }
@@ -119,11 +118,13 @@ public class TransportDeleteAction extends TransportReplicationAction<DeleteRequ
     }
 
     @Override
-    protected Tuple<DeleteResponse, DeleteRequest> shardOperationOnPrimary(DeleteRequest request) {
-        IndexShard indexShard = indicesService.indexServiceSafe(request.shardId().getIndex()).getShard(request.shardId().id());
-        final WriteResult<DeleteResponse> result = executeDeleteRequestOnPrimary(request, indexShard);
-        processAfterWrite(request.refresh(), indexShard, result.location);
-        return new Tuple<>(result.response, request);
+    protected WriteResult<DeleteResponse> onPrimaryShard(DeleteRequest request, IndexShard indexShard) {
+        return executeDeleteRequestOnPrimary(request, indexShard);
+    }
+
+    @Override
+    protected Location onReplicaShard(DeleteRequest request, IndexShard indexShard) {
+        return executeDeleteRequestOnReplica(request, indexShard).getTranslogLocation();
     }
 
     public static WriteResult<DeleteResponse> executeDeleteRequestOnPrimary(DeleteRequest request, IndexShard indexShard) {
@@ -134,9 +135,8 @@ public class TransportDeleteAction extends TransportReplicationAction<DeleteRequ
         request.version(delete.version());
 
         assert request.versionType().validateVersionForWrites(request.version());
-        return new WriteResult<>(
-            new DeleteResponse(indexShard.shardId(), request.type(), request.id(), delete.version(), delete.found()),
-            delete.getTranslogLocation());
+        DeleteResponse response = new DeleteResponse(indexShard.shardId(), request.type(), request.id(), delete.version(), delete.found());
+        return new WriteResult<>(response, delete.getTranslogLocation());
     }
 
     public static Engine.Delete executeDeleteRequestOnReplica(DeleteRequest request, IndexShard indexShard) {
@@ -144,13 +144,4 @@ public class TransportDeleteAction extends TransportReplicationAction<DeleteRequ
         indexShard.delete(delete);
         return delete;
     }
-
-    @Override
-    protected void shardOperationOnReplica(DeleteRequest request) {
-        final ShardId shardId = request.shardId();
-        IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).getShard(shardId.id());
-        Engine.Delete delete = executeDeleteRequestOnReplica(request, indexShard);
-        processAfterWrite(request.refresh(), indexShard, delete.getTranslogLocation());
-    }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java
index 31accebc3b2..bc1e631e559 100644
--- a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java
@@ -24,7 +24,7 @@ import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.DocumentRequest;
 import org.elasticsearch.action.RoutingMissingException;
 import org.elasticsearch.action.TimestampParsingException;
-import org.elasticsearch.action.support.replication.ReplicationRequest;
+import org.elasticsearch.action.support.replication.ReplicatedWriteRequest;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -67,7 +67,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
  * @see org.elasticsearch.client.Requests#indexRequest(String)
  * @see org.elasticsearch.client.Client#index(IndexRequest)
  */
-public class IndexRequest extends ReplicationRequest<IndexRequest> implements DocumentRequest<IndexRequest> {
+public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implements DocumentRequest<IndexRequest> {
 
     /**
      * Operation type controls if the type of the index operation.
@@ -145,7 +145,6 @@ public class IndexRequest extends ReplicationRequest<IndexRequest> implements Do
 
     private OpType opType = OpType.INDEX;
 
-    private boolean refresh = false;
     private long version = Versions.MATCH_ANY;
     private VersionType versionType = VersionType.INTERNAL;
 
@@ -542,20 +541,6 @@ public class IndexRequest extends ReplicationRequest<IndexRequest> implements Do
         return this.opType;
     }
 
-    /**
-     * Should a refresh be executed post this index operation causing the operation to
-     * be searchable. Note, heavy indexing should not set this to <tt>true</tt>. Defaults
-     * to <tt>false</tt>.
-     */
-    public IndexRequest refresh(boolean refresh) {
-        this.refresh = refresh;
-        return this;
-    }
-
-    public boolean refresh() {
-        return this.refresh;
-    }
-
     /**
      * Sets the version, which will cause the index operation to only be performed if a matching
      * version exists and no changes happened on the doc since then.
@@ -652,7 +637,6 @@ public class IndexRequest extends ReplicationRequest<IndexRequest> implements Do
         source = in.readBytesReference();
 
         opType = OpType.fromId(in.readByte());
-        refresh = in.readBoolean();
         version = in.readLong();
         versionType = VersionType.fromValue(in.readByte());
         pipeline = in.readOptionalString();
@@ -674,7 +658,6 @@ public class IndexRequest extends ReplicationRequest<IndexRequest> implements Do
         }
         out.writeBytesReference(source);
         out.writeByte(opType.id());
-        out.writeBoolean(refresh);
         out.writeLong(version);
         out.writeByte(versionType.getValue());
         out.writeOptionalString(pipeline);
diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java
index 4116755e4eb..20587bf0ea9 100644
--- a/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.index;
 
+import org.elasticsearch.action.support.WriteRequestBuilder;
 import org.elasticsearch.action.support.replication.ReplicationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.Nullable;
@@ -33,7 +34,8 @@ import java.util.Map;
 /**
  * An index document action request builder.
  */
-public class IndexRequestBuilder extends ReplicationRequestBuilder<IndexRequest, IndexResponse, IndexRequestBuilder> {
+public class IndexRequestBuilder extends ReplicationRequestBuilder<IndexRequest, IndexResponse, IndexRequestBuilder>
+        implements WriteRequestBuilder<IndexRequestBuilder> {
 
     public IndexRequestBuilder(ElasticsearchClient client, IndexAction action) {
         super(client, action, new IndexRequest());
@@ -220,16 +222,6 @@ public class IndexRequestBuilder extends ReplicationRequestBuilder<IndexRequest,
         return this;
     }
 
-    /**
-     * Should a refresh be executed post this index operation causing the operation to
-     * be searchable. Note, heavy indexing should not set this to <tt>true</tt>. Defaults
-     * to <tt>false</tt>.
-     */
-    public IndexRequestBuilder setRefresh(boolean refresh) {
-        request.refresh(refresh);
-        return this;
-    }
-
     /**
      * Sets the version, which will cause the index operation to only be performed if a matching
      * version exists and no changes happened on the doc since then.
diff --git a/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java b/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
index 10e18c82b86..00be64757ae 100644
--- a/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
@@ -27,7 +27,7 @@ import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.AutoCreateIndex;
 import org.elasticsearch.action.support.replication.ReplicationOperation;
-import org.elasticsearch.action.support.replication.TransportReplicationAction;
+import org.elasticsearch.action.support.replication.TransportWriteAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
@@ -36,16 +36,14 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.mapper.Mapping;
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.index.translog.Translog;
+import org.elasticsearch.index.translog.Translog.Location;
 import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.tasks.Task;
@@ -62,7 +60,7 @@ import org.elasticsearch.transport.TransportService;
  * <li><b>allowIdGeneration</b>: If the id is set not, should it be generated. Defaults to <tt>true</tt>.
  * </ul>
  */
-public class TransportIndexAction extends TransportReplicationAction<IndexRequest, IndexRequest, IndexResponse> {
+public class TransportIndexAction extends TransportWriteAction<IndexRequest, IndexResponse> {
 
     private final AutoCreateIndex autoCreateIndex;
     private final boolean allowIdGeneration;
@@ -78,7 +76,7 @@ public class TransportIndexAction extends TransportReplicationAction<IndexReques
                                 ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
                                 AutoCreateIndex autoCreateIndex) {
         super(settings, IndexAction.NAME, transportService, clusterService, indicesService, threadPool, shardStateAction,
-            actionFilters, indexNameExpressionResolver, IndexRequest::new, IndexRequest::new, ThreadPool.Names.INDEX);
+            actionFilters, indexNameExpressionResolver, IndexRequest::new, ThreadPool.Names.INDEX);
         this.mappingUpdatedAction = mappingUpdatedAction;
         this.createIndexAction = createIndexAction;
         this.autoCreateIndex = autoCreateIndex;
@@ -141,26 +139,13 @@ public class TransportIndexAction extends TransportReplicationAction<IndexReques
     }
 
     @Override
-    protected Tuple<IndexResponse, IndexRequest> shardOperationOnPrimary(IndexRequest request) throws Exception {
-
-        IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex());
-        IndexShard indexShard = indexService.getShard(request.shardId().id());
-
-        final WriteResult<IndexResponse> result = executeIndexRequestOnPrimary(request, indexShard, mappingUpdatedAction);
-
-        final IndexResponse response = result.response;
-        final Translog.Location location = result.location;
-        processAfterWrite(request.refresh(), indexShard, location);
-        return new Tuple<>(response, request);
+    protected WriteResult<IndexResponse> onPrimaryShard(IndexRequest request, IndexShard indexShard) throws Exception {
+        return executeIndexRequestOnPrimary(request, indexShard, mappingUpdatedAction);
     }
 
     @Override
-    protected void shardOperationOnReplica(IndexRequest request) {
-        final ShardId shardId = request.shardId();
-        IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
-        IndexShard indexShard = indexService.getShard(shardId.id());
-        final Engine.Index operation = executeIndexRequestOnReplica(request, indexShard);
-        processAfterWrite(request.refresh(), indexShard, operation.getTranslogLocation());
+    protected Location onReplicaShard(IndexRequest request, IndexShard indexShard) {
+        return executeIndexRequestOnReplica(request, indexShard).getTranslogLocation();
     }
 
     /**
@@ -188,11 +173,8 @@ public class TransportIndexAction extends TransportReplicationAction<IndexReques
         return indexShard.prepareIndexOnPrimary(sourceToParse, request.version(), request.versionType());
     }
 
-    /**
-     * Execute the given {@link IndexRequest} on a primary shard, throwing a
-     * {@link ReplicationOperation.RetryOnPrimaryException} if the operation needs to be re-tried.
-     */
-    public static WriteResult<IndexResponse> executeIndexRequestOnPrimary(IndexRequest request, IndexShard indexShard, MappingUpdatedAction mappingUpdatedAction) throws Exception {
+    public static WriteResult<IndexResponse> executeIndexRequestOnPrimary(IndexRequest request, IndexShard indexShard,
+            MappingUpdatedAction mappingUpdatedAction) throws Exception {
         Engine.Index operation = prepareIndexOperationOnPrimary(request, indexShard);
         Mapping update = operation.parsedDoc().dynamicMappingsUpdate();
         final ShardId shardId = indexShard.shardId();
@@ -214,8 +196,8 @@ public class TransportIndexAction extends TransportReplicationAction<IndexReques
 
         assert request.versionType().validateVersionForWrites(request.version());
 
-        return new WriteResult<>(new IndexResponse(shardId, request.type(), request.id(), request.version(), created), operation.getTranslogLocation());
+        IndexResponse response = new IndexResponse(shardId, request.type(), request.id(), request.version(), created); 
+        return new WriteResult<>(response, operation.getTranslogLocation());
     }
-
 }
 
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/IngestActionFilter.java b/core/src/main/java/org/elasticsearch/action/ingest/IngestActionFilter.java
index 1eb9337c814..850cac040dd 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/IngestActionFilter.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/IngestActionFilter.java
@@ -162,7 +162,7 @@ public final class IngestActionFilter extends AbstractComponent implements Actio
                 return bulkRequest;
             } else {
                 BulkRequest modifiedBulkRequest = new BulkRequest();
-                modifiedBulkRequest.refresh(bulkRequest.refresh());
+                modifiedBulkRequest.setRefreshPolicy(bulkRequest.getRefreshPolicy());
                 modifiedBulkRequest.consistencyLevel(bulkRequest.consistencyLevel());
                 modifiedBulkRequest.timeout(bulkRequest.timeout());
 
diff --git a/core/src/main/java/org/elasticsearch/action/support/WriteRequest.java b/core/src/main/java/org/elasticsearch/action/support/WriteRequest.java
new file mode 100644
index 00000000000..6379a4fb259
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/support/WriteRequest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.support;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.support.replication.ReplicatedWriteRequest;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+
+/**
+ * Interface implemented by requests that modify the documents in an index like {@link IndexRequest}, {@link UpdateRequest}, and
+ * {@link BulkRequest}. Rather than implement this directly most implementers should extend {@link ReplicatedWriteRequest}.
+ */
+public interface WriteRequest<R extends WriteRequest<R>> extends Streamable {
+    /**
+     * Should this request trigger a refresh ({@linkplain RefreshPolicy#IMMEDIATE}), wait for a refresh (
+     * {@linkplain RefreshPolicy#WAIT_UNTIL}), or proceed ignore refreshes entirely ({@linkplain RefreshPolicy#NONE}, the default).
+     */
+    R setRefreshPolicy(RefreshPolicy refreshPolicy);
+
+    /**
+     * Parse the refresh policy from a string, only modifying it if the string is non null. Convenient to use with request parsing.
+     */
+    @SuppressWarnings("unchecked")
+    default R setRefreshPolicy(String refreshPolicy) {
+        if (refreshPolicy != null) {
+            setRefreshPolicy(RefreshPolicy.parse(refreshPolicy));
+        }
+        return (R) this;
+    }
+
+    /**
+     * Should this request trigger a refresh ({@linkplain RefreshPolicy#IMMEDIATE}), wait for a refresh (
+     * {@linkplain RefreshPolicy#WAIT_UNTIL}), or proceed ignore refreshes entirely ({@linkplain RefreshPolicy#NONE}, the default).
+     */
+    RefreshPolicy getRefreshPolicy();
+
+    ActionRequestValidationException validate();
+
+    enum RefreshPolicy implements Writeable {
+        /**
+         * Don't refresh after this request. The default.
+         */
+        NONE,
+        /**
+         * Force a refresh as part of this request. This refresh policy does not scale for high indexing or search throughput but is useful
+         * to present a consistent view to for indices with very low traffic. And it is wonderful for tests!
+         */
+        IMMEDIATE,
+        /**
+         * Leave this request open until a refresh has made the contents of this request visible to search. This refresh policy is
+         * compatible with high indexing and search throughput but it causes the request to wait to reply until a refresh occurs.
+         */
+        WAIT_UNTIL;
+
+        /**
+         * Parse the string representation of a refresh policy, usually from a request parameter.
+         */
+        public static RefreshPolicy parse(String string) {
+            switch (string) {
+            case "false":
+                return NONE;
+            /*
+             * Empty string is IMMEDIATE because that makes "POST /test/test/1?refresh" perform a refresh which reads well and is what folks
+             * are used to.
+             */
+            case "":
+            case "true":
+                return IMMEDIATE;
+            case "wait_for":
+                return WAIT_UNTIL;
+            }
+            throw new IllegalArgumentException("Unknown value for refresh: [" + string + "].");
+        }
+
+        public static RefreshPolicy readFrom(StreamInput in) throws IOException {
+            return RefreshPolicy.values()[in.readByte()];
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeByte((byte) ordinal());
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/support/WriteRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/support/WriteRequestBuilder.java
new file mode 100644
index 00000000000..a87fd043452
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/support/WriteRequestBuilder.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.support;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+
+public interface WriteRequestBuilder<B extends WriteRequestBuilder<B>> {
+    WriteRequest<?> request();
+
+    /**
+     * Should this request trigger a refresh ({@linkplain RefreshPolicy#IMMEDIATE}), wait for a refresh (
+     * {@linkplain RefreshPolicy#WAIT_UNTIL}), or proceed ignore refreshes entirely ({@linkplain RefreshPolicy#NONE}, the default).
+     */
+    @SuppressWarnings("unchecked")
+    default B setRefreshPolicy(RefreshPolicy refreshPolicy) {
+        request().setRefreshPolicy(refreshPolicy);
+        return (B) this;
+    }
+
+    /**
+     * If set to true then this request will force an immediate refresh. Backwards compatibility layer for Elasticsearch's old
+     * {@code setRefresh} calls.
+     *
+     * @deprecated use {@link #setRefreshPolicy(RefreshPolicy)} with {@link RefreshPolicy#IMMEDIATE} or {@link RefreshPolicy#NONE} instead.
+     *             Will be removed in 6.0.
+     */
+    @Deprecated
+    default B setRefresh(boolean refresh) {
+        assert Version.CURRENT.major < 6 : "Remove setRefresh(boolean) in 6.0";
+        return setRefreshPolicy(refresh ? RefreshPolicy.IMMEDIATE : RefreshPolicy.NONE);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/support/WriteResponse.java b/core/src/main/java/org/elasticsearch/action/support/WriteResponse.java
new file mode 100644
index 00000000000..07f5ea695d9
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/support/WriteResponse.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.support;
+
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.index.IndexSettings;
+
+/**
+ * Interface implemented by responses for actions that modify the documents in an index like {@link IndexResponse}, {@link UpdateResponse},
+ * and {@link BulkResponse}. Rather than implement this directly most implementers should extend {@link DocWriteResponse}.
+ */
+public interface WriteResponse {
+    /**
+     * Mark the response as having forced a refresh? Requests that set {@link WriteRequest#setRefreshPolicy(RefreshPolicy)} to
+     * {@link RefreshPolicy#IMMEDIATE} should always mark this as true. Requests that set it to {@link RefreshPolicy#WAIT_UNTIL} will only
+     * set this to true if they run out of refresh listener slots (see {@link IndexSettings#MAX_REFRESH_LISTENERS_PER_SHARD}).
+     */
+    public abstract void setForcedRefresh(boolean forcedRefresh);
+}
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicatedWriteRequest.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicatedWriteRequest.java
new file mode 100644
index 00000000000..fa02dac9e1e
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicatedWriteRequest.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.support.replication;
+
+import org.elasticsearch.action.bulk.BulkShardRequest;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.index.shard.ShardId;
+
+import java.io.IOException;
+
+/**
+ * Requests that are both {@linkplain ReplicationRequest}s (run on a shard's primary first, then the replica) and {@linkplain WriteRequest}
+ * (modify documents on a shard), for example {@link BulkShardRequest}, {@link IndexRequest}, and {@link DeleteRequest}.
+ */
+public abstract class ReplicatedWriteRequest<R extends ReplicatedWriteRequest<R>> extends ReplicationRequest<R> implements WriteRequest<R> {
+    private RefreshPolicy refreshPolicy = RefreshPolicy.NONE;
+
+    /**
+     * Constructor for deserialization.
+     */
+    public ReplicatedWriteRequest() {
+    }
+
+    public ReplicatedWriteRequest(ShardId shardId) {
+        super(shardId);
+    }
+
+    @Override
+    @SuppressWarnings("unchecked")
+    public R setRefreshPolicy(RefreshPolicy refreshPolicy) {
+        this.refreshPolicy = refreshPolicy;
+        return (R) this;
+    }
+
+    @Override
+    public RefreshPolicy getRefreshPolicy() {
+        return refreshPolicy;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        refreshPolicy = RefreshPolicy.readFrom(in);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        refreshPolicy.writeTo(out);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
index 1f7313c1943..8442e705257 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
@@ -21,7 +21,6 @@ package org.elasticsearch.action.support.replication;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ReplicationResponse;
 import org.elasticsearch.action.UnavailableShardsException;
 import org.elasticsearch.action.WriteConsistencyLevel;
 import org.elasticsearch.action.support.TransportActions;
@@ -29,7 +28,6 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
@@ -47,28 +45,41 @@ import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
 
-public class ReplicationOperation<Request extends ReplicationRequest<Request>, ReplicaRequest extends ReplicationRequest<ReplicaRequest>,
-    Response extends ReplicationResponse> {
+public class ReplicationOperation<
+            Request extends ReplicationRequest<Request>,
+            ReplicaRequest extends ReplicationRequest<ReplicaRequest>,
+            PrimaryResultT extends ReplicationOperation.PrimaryResult<ReplicaRequest>
+        > {
     final private ESLogger logger;
     final private Request request;
     final private Supplier<ClusterState> clusterStateSupplier;
     final private String opType;
     final private AtomicInteger totalShards = new AtomicInteger();
+    /**
+     * The number of pending sub-operations in this operation. This is incremented when the following operations start and decremented when
+     * they complete:
+     * <ul>
+     * <li>The operation on the primary</li>
+     * <li>The operation on each replica</li>
+     * <li>Coordination of the operation as a whole. This prevents the operation from terminating early if we haven't started any replica
+     * operations and the primary finishes.</li>
+     * </ul>
+     */
     final private AtomicInteger pendingShards = new AtomicInteger();
     final private AtomicInteger successfulShards = new AtomicInteger();
     final private boolean executeOnReplicas;
     final private boolean checkWriteConsistency;
-    final private Primary<Request, ReplicaRequest, Response> primary;
+    final private Primary<Request, ReplicaRequest, PrimaryResultT> primary;
     final private Replicas<ReplicaRequest> replicasProxy;
     final private AtomicBoolean finished = new AtomicBoolean();
-    final protected ActionListener<Response> finalResponseListener;
+    final protected ActionListener<PrimaryResultT> resultListener;
 
-    private volatile Response finalResponse = null;
+    private volatile PrimaryResultT primaryResult = null;
 
     private final List<ReplicationResponse.ShardInfo.Failure> shardReplicaFailures = Collections.synchronizedList(new ArrayList<>());
 
-    ReplicationOperation(Request request, Primary<Request, ReplicaRequest, Response> primary,
-                         ActionListener<Response> listener,
+    ReplicationOperation(Request request, Primary<Request, ReplicaRequest, PrimaryResultT> primary,
+                         ActionListener<PrimaryResultT> listener,
                          boolean executeOnReplicas, boolean checkWriteConsistency,
                          Replicas<ReplicaRequest> replicas,
                          Supplier<ClusterState> clusterStateSupplier, ESLogger logger, String opType) {
@@ -76,7 +87,7 @@ public class ReplicationOperation<Request extends ReplicationRequest<Request>, R
         this.executeOnReplicas = executeOnReplicas;
         this.replicasProxy = replicas;
         this.primary = primary;
-        this.finalResponseListener = listener;
+        this.resultListener = listener;
         this.logger = logger;
         this.request = request;
         this.clusterStateSupplier = clusterStateSupplier;
@@ -85,28 +96,27 @@ public class ReplicationOperation<Request extends ReplicationRequest<Request>, R
 
     void execute() throws Exception {
         final String writeConsistencyFailure = checkWriteConsistency ? checkWriteConsistency() : null;
-        final ShardId shardId = primary.routingEntry().shardId();
+        final ShardRouting primaryRouting = primary.routingEntry();
+        final ShardId primaryId = primaryRouting.shardId();
         if (writeConsistencyFailure != null) {
-            finishAsFailed(new UnavailableShardsException(shardId,
+            finishAsFailed(new UnavailableShardsException(primaryId,
                 "{} Timeout: [{}], request: [{}]", writeConsistencyFailure, request.timeout(), request));
             return;
         }
 
         totalShards.incrementAndGet();
-        pendingShards.incrementAndGet(); // increase by 1 until we finish all primary coordination
-        Tuple<Response, ReplicaRequest> primaryResponse = primary.perform(request);
-        successfulShards.incrementAndGet(); // mark primary as successful
-        finalResponse = primaryResponse.v1();
-        ReplicaRequest replicaRequest = primaryResponse.v2();
+        pendingShards.incrementAndGet();
+        primaryResult = primary.perform(request);
+        final ReplicaRequest replicaRequest = primaryResult.replicaRequest();
         assert replicaRequest.primaryTerm() > 0 : "replicaRequest doesn't have a primary term";
         if (logger.isTraceEnabled()) {
-            logger.trace("[{}] op [{}] completed on primary for request [{}]", shardId, opType, request);
+            logger.trace("[{}] op [{}] completed on primary for request [{}]", primaryId, opType, request);
         }
         // we have to get a new state after successfully indexing into the primary in order to honour recovery semantics.
         // we have to make sure that every operation indexed into the primary after recovery start will also be replicated
         // to the recovery target. If we use an old cluster state, we may miss a relocation that has started since then.
         // If the index gets deleted after primary operation, we skip replication
-        List<ShardRouting> shards = getShards(shardId, clusterStateSupplier.get());
+        final List<ShardRouting> shards = getShards(primaryId, clusterStateSupplier.get());
         final String localNodeId = primary.routingEntry().currentNodeId();
         for (final ShardRouting shard : shards) {
             if (executeOnReplicas == false || shard.unassigned()) {
@@ -125,8 +135,8 @@ public class ReplicationOperation<Request extends ReplicationRequest<Request>, R
             }
         }
 
-        // decrement pending and finish (if there are no replicas, or those are done)
-        decPendingAndFinishIfNeeded(); // incremented in the beginning of this method
+        successfulShards.incrementAndGet();
+        decPendingAndFinishIfNeeded();
     }
 
     private void performOnReplica(final ShardRouting shard, final ReplicaRequest replicaRequest) {
@@ -241,19 +251,19 @@ public class ReplicationOperation<Request extends ReplicationRequest<Request>, R
                 failuresArray = new ReplicationResponse.ShardInfo.Failure[shardReplicaFailures.size()];
                 shardReplicaFailures.toArray(failuresArray);
             }
-            finalResponse.setShardInfo(new ReplicationResponse.ShardInfo(
+            primaryResult.setShardInfo(new ReplicationResponse.ShardInfo(
                     totalShards.get(),
                     successfulShards.get(),
                     failuresArray
                 )
             );
-            finalResponseListener.onResponse(finalResponse);
+            resultListener.onResponse(primaryResult);
         }
     }
 
     private void finishAsFailed(Throwable throwable) {
         if (finished.compareAndSet(false, true)) {
-            finalResponseListener.onFailure(throwable);
+            resultListener.onFailure(throwable);
         }
     }
 
@@ -284,22 +294,31 @@ public class ReplicationOperation<Request extends ReplicationRequest<Request>, R
     }
 
 
-    interface Primary<Request extends ReplicationRequest<Request>, ReplicaRequest extends ReplicationRequest<ReplicaRequest>,
-        Response extends ReplicationResponse> {
+    interface Primary<
+                Request extends ReplicationRequest<Request>,
+                ReplicaRequest extends ReplicationRequest<ReplicaRequest>,
+                PrimaryResultT extends PrimaryResult<ReplicaRequest>
+            > {
 
-        /** routing entry for this primary */
+        /**
+         * routing entry for this primary
+         */
         ShardRouting routingEntry();
 
-        /** fail the primary, typically due to the fact that the operation has learned the primary has been demoted by the master */
+        /**
+         * fail the primary, typically due to the fact that the operation has learned the primary has been demoted by the master
+         */
         void failShard(String message, Throwable throwable);
 
         /**
-         * Performs the given request on this primary
+         * Performs the given request on this primary. Yes, this returns as soon as it can with the request for the replicas and calls a
+         * listener when the primary request is completed. Yes, the primary request might complete before the method returns. Yes, it might
+         * also complete after. Deal with it.
          *
-         * @return A tuple containing not null values, as first value the result of the primary operation and as second value
-         * the request to be executed on the replica shards.
+         * @param request the request to perform
+         * @return the request to send to the repicas
          */
-        Tuple<Response, ReplicaRequest> perform(Request request) throws Exception;
+        PrimaryResultT perform(Request request) throws Exception;
 
     }
 
@@ -308,19 +327,20 @@ public class ReplicationOperation<Request extends ReplicationRequest<Request>, R
         /**
          * performs the the given request on the specified replica
          *
-         * @param replica {@link ShardRouting} of the shard this request should be executed on
+         * @param replica        {@link ShardRouting} of the shard this request should be executed on
          * @param replicaRequest operation to peform
-         * @param listener a callback to call once the operation has been complicated, either successfully or with an error.
+         * @param listener       a callback to call once the operation has been complicated, either successfully or with an error.
          */
         void performOn(ShardRouting replica, ReplicaRequest replicaRequest, ActionListener<TransportResponse.Empty> listener);
 
         /**
          * Fail the specified shard, removing it from the current set of active shards
-         * @param replica shard to fail
-         * @param primary the primary shard that requested the failure
-         * @param message a (short) description of the reason
-         * @param throwable the original exception which caused the ReplicationOperation to request the shard to be failed
-         * @param onSuccess a callback to call when the shard has been successfully removed from the active set.
+         *
+         * @param replica          shard to fail
+         * @param primary          the primary shard that requested the failure
+         * @param message          a (short) description of the reason
+         * @param throwable        the original exception which caused the ReplicationOperation to request the shard to be failed
+         * @param onSuccess        a callback to call when the shard has been successfully removed from the active set.
          * @param onPrimaryDemoted a callback to call when the shard can not be failed because the current primary has been demoted
          *                         by the master.
          * @param onIgnoredFailure a callback to call when failing a shard has failed, but it that failure can be safely ignored and the
@@ -345,4 +365,11 @@ public class ReplicationOperation<Request extends ReplicationRequest<Request>, R
             super(in);
         }
     }
+
+    interface PrimaryResult<R extends ReplicationRequest<R>> {
+
+        R replicaRequest();
+
+        void setShardInfo(ReplicationResponse.ShardInfo shardInfo);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java
index 3e88575b717..44c420598b5 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java
@@ -23,6 +23,8 @@ import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.WriteConsistencyLevel;
+import org.elasticsearch.action.admin.indices.refresh.TransportShardRefreshAction;
+import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -38,7 +40,8 @@ import java.util.concurrent.TimeUnit;
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
 /**
- *
+ * Requests that are run on a particular replica, first on the primary and then on the replicas like {@link IndexRequest} or
+ * {@link TransportShardRefreshAction}.
  */
 public abstract class ReplicationRequest<Request extends ReplicationRequest<Request>> extends ActionRequest<Request>
         implements IndicesRequest {
@@ -65,7 +68,6 @@ public abstract class ReplicationRequest<Request extends ReplicationRequest<Requ
 
     }
 
-
     /**
      * Creates a new request with resolved shard id
      */
diff --git a/core/src/main/java/org/elasticsearch/action/ReplicationResponse.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
similarity index 96%
rename from core/src/main/java/org/elasticsearch/action/ReplicationResponse.java
rename to core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
index df2f90b0020..181e0e0ea74 100644
--- a/core/src/main/java/org/elasticsearch/action/ReplicationResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
@@ -17,10 +17,12 @@
  * under the License.
  */
 
-package org.elasticsearch.action;
+package org.elasticsearch.action.support.replication;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -79,14 +81,16 @@ public class ReplicationResponse extends ActionResponse {
         }
 
         /**
-         * @return the total number of shards the write should go to (replicas and primaries). This includes relocating shards, so this number can be higher than the number of shards.
+         * @return the total number of shards the write should go to (replicas and primaries). This includes relocating shards, so this
+         *         number can be higher than the number of shards.
          */
         public int getTotal() {
             return total;
         }
 
         /**
-         * @return the total number of shards the write succeeded on (replicas and primaries). This includes relocating shards, so this number can be higher than the number of shards.
+         * @return the total number of shards the write succeeded on (replicas and primaries). This includes relocating shards, so this
+         *         number can be higher than the number of shards.
          */
         public int getSuccessful() {
             return successful;
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java
index 25de821e227..2cab7d78317 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java
@@ -22,7 +22,6 @@ package org.elasticsearch.action.support.replication;
 import com.carrotsearch.hppc.cursors.IntObjectCursor;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ReplicationResponse;
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.DefaultShardOperationFailedException;
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
index 8a721dfe508..87e602bf427 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
@@ -22,7 +22,6 @@ package org.elasticsearch.action.support.replication;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionListenerResponseHandler;
-import org.elasticsearch.action.ReplicationResponse;
 import org.elasticsearch.action.UnavailableShardsException;
 import org.elasticsearch.action.WriteConsistencyLevel;
 import org.elasticsearch.action.support.ActionFilters;
@@ -41,7 +40,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.settings.Settings;
@@ -53,7 +51,6 @@ import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.tasks.Task;
@@ -66,6 +63,7 @@ import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequestHandler;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportResponse;
+import org.elasticsearch.transport.TransportResponse.Empty;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
@@ -81,9 +79,11 @@ import java.util.function.Supplier;
  * primary node to validate request before primary operation followed by sampling state again for resolving
  * nodes with replica copies to perform replication.
  */
-public abstract class TransportReplicationAction<Request extends ReplicationRequest<Request>,
-    ReplicaRequest extends ReplicationRequest<ReplicaRequest>,
-    Response extends ReplicationResponse> extends TransportAction<Request, Response> {
+public abstract class TransportReplicationAction<
+            Request extends ReplicationRequest<Request>,
+            ReplicaRequest extends ReplicationRequest<ReplicaRequest>,
+            Response extends ReplicationResponse
+        > extends TransportAction<Request, Response> {
 
     final protected TransportService transportService;
     final protected ClusterService clusterService;
@@ -149,17 +149,17 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
     }
 
     /**
-     * Primary operation on node with primary copy
+     * Primary operation on node with primary copy.
      *
-     * @return A tuple containing not null values, as first value the result of the primary operation and as second value
-     * the request to be executed on the replica shards.
+     * @param shardRequest the request to the primary shard
      */
-    protected abstract Tuple<Response, ReplicaRequest> shardOperationOnPrimary(Request shardRequest) throws Exception;
+    protected abstract PrimaryResult shardOperationOnPrimary(Request shardRequest) throws Exception;
 
     /**
-     * Replica operation on nodes with replica copies
+     * Synchronous replica operation on nodes with replica copies. This is done under the lock form
+     * {@link #acquireReplicaOperationLock(ShardId, long)}.
      */
-    protected abstract void shardOperationOnReplica(ReplicaRequest shardRequest);
+    protected abstract ReplicaResult shardOperationOnReplica(ReplicaRequest shardRequest);
 
     /**
      * True if write consistency should be checked for an implementation
@@ -198,26 +198,6 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
             || TransportActions.isShardNotAvailableException(e);
     }
 
-    protected static class WriteResult<T extends ReplicationResponse> {
-
-        public final T response;
-        public final Translog.Location location;
-
-        public WriteResult(T response, Translog.Location location) {
-            this.response = response;
-            this.location = location;
-        }
-
-        @SuppressWarnings("unchecked")
-        public <T extends ReplicationResponse> T response() {
-            // this sets total, pending and failed to 0 and this is ok, because we will embed this into the replica
-            // request and not use it
-            response.setShardInfo(new ReplicationResponse.ShardInfo());
-            return (T) response;
-        }
-
-    }
-
     class OperationTransportHandler implements TransportRequestHandler<Request> {
         @Override
         public void messageReceived(final Request request, final TransportChannel channel, Task task) throws Exception {
@@ -289,7 +269,17 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
                     final IndexMetaData indexMetaData = clusterService.state().getMetaData().index(request.shardId().getIndex());
                     final boolean executeOnReplicas = (indexMetaData == null) || shouldExecuteReplication(indexMetaData.getSettings());
                     final ActionListener<Response> listener = createResponseListener(channel, replicationTask, primaryShardReference);
-                    createReplicatedOperation(request, listener, primaryShardReference, executeOnReplicas).execute();
+                    createReplicatedOperation(request, new ActionListener<PrimaryResult>() {
+                        @Override
+                        public void onResponse(PrimaryResult result) {
+                            result.respond(listener);
+                        }
+
+                        @Override
+                        public void onFailure(Throwable e) {
+                            listener.onFailure(e);
+                        }
+                    }, primaryShardReference, executeOnReplicas).execute();
                     success = true;
                 }
             } finally {
@@ -299,9 +289,9 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
             }
         }
 
-        protected ReplicationOperation<Request, ReplicaRequest, Response>
-        createReplicatedOperation(Request request, ActionListener<Response> listener,
-                                  PrimaryShardReference primaryShardReference, boolean executeOnReplicas) {
+        protected ReplicationOperation<Request, ReplicaRequest, PrimaryResult> createReplicatedOperation(
+                Request request, ActionListener<PrimaryResult> listener,
+                PrimaryShardReference primaryShardReference, boolean executeOnReplicas) {
             return new ReplicationOperation<>(request, primaryShardReference, listener,
                 executeOnReplicas, checkWriteConsistency(), replicasProxy, clusterService::state, logger, actionName
             );
@@ -339,6 +329,41 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
         }
     }
 
+    protected class PrimaryResult implements ReplicationOperation.PrimaryResult<ReplicaRequest> {
+        final ReplicaRequest replicaRequest;
+        final Response finalResponse;
+
+        public PrimaryResult(ReplicaRequest replicaRequest, Response finalResponse) {
+            this.replicaRequest = replicaRequest;
+            this.finalResponse = finalResponse;
+        }
+
+        @Override
+        public ReplicaRequest replicaRequest() {
+            return replicaRequest;
+        }
+
+        @Override
+        public void setShardInfo(ReplicationResponse.ShardInfo shardInfo) {
+            finalResponse.setShardInfo(shardInfo);
+        }
+
+        public void respond(ActionListener<Response> listener) {
+            listener.onResponse(finalResponse);
+        }
+    }
+
+    protected class ReplicaResult {
+        /**
+         * Public constructor so subclasses can call it.
+         */
+        public ReplicaResult() {}
+
+        public void respond(ActionListener<TransportResponse.Empty> listener) {
+            listener.onResponse(TransportResponse.Empty.INSTANCE);
+        }
+    }
+
     class ReplicaOperationTransportHandler implements TransportRequestHandler<ReplicaRequest> {
         @Override
         public void messageReceived(final ReplicaRequest request, final TransportChannel channel) throws Exception {
@@ -426,15 +451,35 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
         protected void doRun() throws Exception {
             setPhase(task, "replica");
             assert request.shardId() != null : "request shardId must be set";
+            ReplicaResult result;
             try (Releasable ignored = acquireReplicaOperationLock(request.shardId(), request.primaryTerm())) {
-                shardOperationOnReplica(request);
+                result = shardOperationOnReplica(request);
+            }
+            result.respond(new ResponseListener());
+        }
+
+        /**
+         * Listens for the response on the replica and sends the response back to the primary.
+         */
+        private class ResponseListener implements ActionListener<TransportResponse.Empty> {
+            @Override
+            public void onResponse(Empty response) {
                 if (logger.isTraceEnabled()) {
                     logger.trace("action [{}] completed on shard [{}] for request [{}]", transportReplicaAction, request.shardId(),
-                        request);
+                            request);
+                }
+                setPhase(task, "finished");
+                try {
+                    channel.sendResponse(response);
+                } catch (Exception e) {
+                    onFailure(e);
                 }
             }
-            setPhase(task, "finished");
-            channel.sendResponse(TransportResponse.Empty.INSTANCE);
+
+            @Override
+            public void onFailure(Throwable e) {
+                responseWithFailure(e);
+            }
         }
     }
 
@@ -722,7 +767,7 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
         return IndexMetaData.isIndexUsingShadowReplicas(settings) == false;
     }
 
-    class PrimaryShardReference implements ReplicationOperation.Primary<Request, ReplicaRequest, Response>, Releasable {
+    class PrimaryShardReference implements ReplicationOperation.Primary<Request, ReplicaRequest, PrimaryResult>, Releasable {
 
         private final IndexShard indexShard;
         private final Releasable operationLock;
@@ -751,9 +796,9 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
         }
 
         @Override
-        public Tuple<Response, ReplicaRequest> perform(Request request) throws Exception {
-            Tuple<Response, ReplicaRequest> result = shardOperationOnPrimary(request);
-            result.v2().primaryTerm(indexShard.getPrimaryTerm());
+        public PrimaryResult perform(Request request) throws Exception {
+            PrimaryResult result = shardOperationOnPrimary(request);
+            result.replicaRequest().primaryTerm(indexShard.getPrimaryTerm());
             return result;
         }
 
@@ -805,20 +850,6 @@ public abstract class TransportReplicationAction<Request extends ReplicationRequ
         }
     }
 
-    protected final void processAfterWrite(boolean refresh, IndexShard indexShard, Translog.Location location) {
-        if (refresh) {
-            try {
-                indexShard.refresh("refresh_flag_index");
-            } catch (Throwable e) {
-                // ignore
-            }
-        }
-        if (indexShard.getTranslogDurability() == Translog.Durability.REQUEST && location != null) {
-            indexShard.sync(location);
-        }
-        indexShard.maybeFlush();
-    }
-
     /**
      * Sets the current phase on the task if it isn't null. Pulled into its own
      * method because its more convenient that way.
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java
new file mode 100644
index 00000000000..e50ad7f1306
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.support.replication;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.action.support.WriteResponse;
+import org.elasticsearch.cluster.action.shard.ShardStateAction;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.translog.Translog;
+import org.elasticsearch.index.translog.Translog.Location;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportResponse;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.function.Supplier;
+
+/**
+ * Base class for transport actions that modify data in some shard like index, delete, and shardBulk.
+ */
+public abstract class TransportWriteAction<
+            Request extends ReplicatedWriteRequest<Request>,
+            Response extends ReplicationResponse & WriteResponse
+        > extends TransportReplicationAction<Request, Request, Response> {
+
+    protected TransportWriteAction(Settings settings, String actionName, TransportService transportService,
+            ClusterService clusterService, IndicesService indicesService, ThreadPool threadPool, ShardStateAction shardStateAction,
+            ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Supplier<Request> request,
+            String executor) {
+        super(settings, actionName, transportService, clusterService, indicesService, threadPool, shardStateAction, actionFilters,
+                indexNameExpressionResolver, request, request, executor);
+    }
+
+    /**
+     * Called on the primary with a reference to the {@linkplain IndexShard} to modify.
+     */
+    protected abstract WriteResult<Response> onPrimaryShard(Request request, IndexShard indexShard) throws Exception;
+
+    /**
+     * Called once per replica with a reference to the {@linkplain IndexShard} to modify.
+     *
+     * @return the translog location of the {@linkplain IndexShard} after the write was completed or null if no write occurred
+     */
+    protected abstract Translog.Location onReplicaShard(Request request, IndexShard indexShard);
+
+    @Override
+    protected final WritePrimaryResult shardOperationOnPrimary(Request request) throws Exception {
+        IndexShard indexShard = indexShard(request);
+        WriteResult<Response> result = onPrimaryShard(request, indexShard);
+        return new WritePrimaryResult(request, result.getResponse(), result.getLocation(), indexShard);
+    }
+
+    @Override
+    protected final WriteReplicaResult shardOperationOnReplica(Request request) {
+        IndexShard indexShard = indexShard(request);
+        Translog.Location location = onReplicaShard(request, indexShard);
+        return new WriteReplicaResult(indexShard, request, location);
+    }
+
+    /**
+     * Fetch the IndexShard for the request. Protected so it can be mocked in tests.
+     */
+    protected IndexShard indexShard(Request request) {
+        final ShardId shardId = request.shardId();
+        IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
+        return indexService.getShard(shardId.id());
+    }
+
+    /**
+     * Simple result from a write action. Write actions have static method to return these so they can integrate with bulk.
+     */
+    public static class WriteResult<Response extends ReplicationResponse> {
+        private final Response response;
+        private final Translog.Location location;
+
+        public WriteResult(Response response, @Nullable Location location) {
+            this.response = response;
+            this.location = location;
+        }
+
+        public Response getResponse() {
+            return response;
+        }
+
+        public Translog.Location getLocation() {
+            return location;
+        }
+    }
+
+    /**
+     * Result of taking the action on the primary.
+     */
+    class WritePrimaryResult extends PrimaryResult implements RespondingWriteResult {
+        boolean finishedAsyncActions;
+        ActionListener<Response> listener = null;
+
+        public WritePrimaryResult(Request request, Response finalResponse,
+                                  @Nullable Translog.Location location,
+                                  IndexShard indexShard) {
+            super(request, finalResponse);
+            /*
+             * We call this before replication because this might wait for a refresh and that can take a while. This way we wait for the
+             * refresh in parallel on the primary and on the replica.
+             */
+            postWriteActions(indexShard, request, location, this, logger);
+        }
+
+        @Override
+        public synchronized void respond(ActionListener<Response> listener) {
+            this.listener = listener;
+            respondIfPossible();
+        }
+
+        /**
+         * Respond if the refresh has occurred and the listener is ready. Always called while synchronized on {@code this}.
+         */
+        protected void respondIfPossible() {
+            if (finishedAsyncActions && listener != null) {
+                super.respond(listener);
+            }
+        }
+
+        @Override
+        public synchronized void respondAfterAsyncAction(boolean forcedRefresh) {
+            finalResponse.setForcedRefresh(forcedRefresh);
+            finishedAsyncActions = true;
+            respondIfPossible();
+        }
+    }
+
+    /**
+     * Result of taking the action on the replica.
+     */
+    class WriteReplicaResult extends ReplicaResult implements RespondingWriteResult {
+        boolean finishedAsyncActions;
+        private ActionListener<TransportResponse.Empty> listener;
+
+        public WriteReplicaResult(IndexShard indexShard, ReplicatedWriteRequest<?> request, Translog.Location location) {
+            postWriteActions(indexShard, request, location, this, logger);
+        }
+
+        @Override
+        public void respond(ActionListener<TransportResponse.Empty> listener) {
+            this.listener = listener;
+            respondIfPossible();
+        }
+
+        /**
+         * Respond if the refresh has occurred and the listener is ready. Always called while synchronized on {@code this}.
+         */
+        protected void respondIfPossible() {
+            if (finishedAsyncActions && listener != null) {
+                super.respond(listener);
+            }
+        }
+
+        @Override
+        public synchronized void respondAfterAsyncAction(boolean forcedRefresh) {
+            finishedAsyncActions = true;
+            respondIfPossible();
+        }
+    }
+
+    private interface RespondingWriteResult {
+        void respondAfterAsyncAction(boolean forcedRefresh);
+    }
+
+    static void postWriteActions(final IndexShard indexShard,
+                                 final WriteRequest<?> request,
+                                 @Nullable final Translog.Location location,
+                                 final RespondingWriteResult respond,
+                                 final ESLogger logger) {
+        boolean pendingOps = false;
+        boolean immediateRefresh = false;
+        switch (request.getRefreshPolicy()) {
+            case IMMEDIATE:
+                indexShard.refresh("refresh_flag_index");
+                immediateRefresh = true;
+                break;
+            case WAIT_UNTIL:
+                if (location != null) {
+                    pendingOps = true;
+                    indexShard.addRefreshListener(location, forcedRefresh -> {
+                        logger.warn("block_until_refresh request ran out of slots and forced a refresh: [{}]", request);
+                        respond.respondAfterAsyncAction(forcedRefresh);
+                    });
+                }
+                break;
+            case NONE:
+                break;
+        }
+        boolean fsyncTranslog = indexShard.getTranslogDurability() == Translog.Durability.REQUEST && location != null;
+        if (fsyncTranslog) {
+            indexShard.sync(location);
+        }
+        indexShard.maybeFlush();
+        if (pendingOps == false) {
+            respond.respondAfterAsyncAction(immediateRefresh);
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
index 0363ef8fe43..ca55a63c1d6 100644
--- a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
+++ b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
@@ -26,6 +26,7 @@ import org.elasticsearch.action.RoutingMissingException;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
+import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.delete.TransportDeleteAction;
 import org.elasticsearch.action.index.IndexRequest;
@@ -187,6 +188,7 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
                         } else {
                             update.setGetResult(null);
                         }
+                        update.setForcedRefresh(response.forcedRefresh());
                         listener.onResponse(update);
                     }
 
@@ -219,6 +221,7 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
                     public void onResponse(IndexResponse response) {
                         UpdateResponse update = new UpdateResponse(response.getShardInfo(), response.getShardId(), response.getType(), response.getId(), response.getVersion(), response.isCreated());
                         update.setGetResult(updateHelper.extractGetResult(request, request.concreteIndex(), response.getVersion(), result.updatedSourceAsMap(), result.updateSourceContentType(), indexSourceBytes));
+                        update.setForcedRefresh(response.forcedRefresh());
                         listener.onResponse(update);
                     }
 
@@ -241,11 +244,13 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
                 });
                 break;
             case DELETE:
-                deleteAction.execute(result.action(), new ActionListener<DeleteResponse>() {
+                DeleteRequest deleteRequest = result.action();
+                deleteAction.execute(deleteRequest, new ActionListener<DeleteResponse>() {
                     @Override
                     public void onResponse(DeleteResponse response) {
                         UpdateResponse update = new UpdateResponse(response.getShardInfo(), response.getShardId(), response.getType(), response.getId(), response.getVersion(), false);
                         update.setGetResult(updateHelper.extractGetResult(request, request.concreteIndex(), response.getVersion(), result.updatedSourceAsMap(), result.updateSourceContentType(), null));
+                        update.setForcedRefresh(response.forcedRefresh());
                         listener.onResponse(update);
                     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java b/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
index 9ac77050202..0c9c1c67978 100644
--- a/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
+++ b/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
@@ -131,7 +131,7 @@ public class UpdateHelper extends AbstractComponent {
                     // it has to be a "create!"
                     .create(true)
                     .ttl(ttl)
-                    .refresh(request.refresh())
+                    .setRefreshPolicy(request.getRefreshPolicy())
                     .routing(request.routing())
                     .parent(request.parent())
                     .consistencyLevel(request.consistencyLevel());
@@ -229,12 +229,13 @@ public class UpdateHelper extends AbstractComponent {
                     .version(updateVersion).versionType(request.versionType())
                     .consistencyLevel(request.consistencyLevel())
                     .timestamp(timestamp).ttl(ttl)
-                    .refresh(request.refresh());
+                    .setRefreshPolicy(request.getRefreshPolicy());
             return new Result(indexRequest, Operation.INDEX, updatedSourceAsMap, updateSourceContentType);
         } else if ("delete".equals(operation)) {
             DeleteRequest deleteRequest = Requests.deleteRequest(request.index()).type(request.type()).id(request.id()).routing(routing).parent(parent)
                     .version(updateVersion).versionType(request.versionType())
-                    .consistencyLevel(request.consistencyLevel());
+                    .consistencyLevel(request.consistencyLevel())
+                    .setRefreshPolicy(request.getRefreshPolicy());
             return new Result(deleteRequest, Operation.DELETE, updatedSourceAsMap, updateSourceContentType);
         } else if ("none".equals(operation)) {
             UpdateResponse update = new UpdateResponse(shardId, getResult.getType(), getResult.getId(), getResult.getVersion(), false);
diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java
index 31f219fd4c7..e0846c1ce5d 100644
--- a/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java
@@ -23,6 +23,7 @@ import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.DocumentRequest;
 import org.elasticsearch.action.WriteConsistencyLevel;
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.action.support.single.instance.InstanceShardOperationRequest;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseFieldMatcher;
@@ -53,7 +54,8 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
 
 /**
  */
-public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest> implements DocumentRequest<UpdateRequest> {
+public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
+        implements DocumentRequest<UpdateRequest>, WriteRequest<UpdateRequest> {
 
     private String type;
     private String id;
@@ -72,7 +74,7 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
     private VersionType versionType = VersionType.INTERNAL;
     private int retryOnConflict = 0;
 
-    private boolean refresh = false;
+    private RefreshPolicy refreshPolicy = RefreshPolicy.NONE;
 
     private WriteConsistencyLevel consistencyLevel = WriteConsistencyLevel.DEFAULT;
 
@@ -422,18 +424,15 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
         return this.versionType;
     }
 
-    /**
-     * Should a refresh be executed post this update operation causing the operation to
-     * be searchable. Note, heavy indexing should not set this to <tt>true</tt>. Defaults
-     * to <tt>false</tt>.
-     */
-    public UpdateRequest refresh(boolean refresh) {
-        this.refresh = refresh;
+    @Override
+    public UpdateRequest setRefreshPolicy(RefreshPolicy refreshPolicy) {
+        this.refreshPolicy = refreshPolicy;
         return this;
     }
 
-    public boolean refresh() {
-        return this.refresh;
+    @Override
+    public RefreshPolicy getRefreshPolicy() {
+        return refreshPolicy;
     }
 
     public WriteConsistencyLevel consistencyLevel() {
@@ -730,7 +729,7 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
             script = new Script(in);
         }
         retryOnConflict = in.readVInt();
-        refresh = in.readBoolean();
+        refreshPolicy = RefreshPolicy.readFrom(in);
         if (in.readBoolean()) {
             doc = new IndexRequest();
             doc.readFrom(in);
@@ -767,7 +766,7 @@ public class UpdateRequest extends InstanceShardOperationRequest<UpdateRequest>
             script.writeTo(out);
         }
         out.writeVInt(retryOnConflict);
-        out.writeBoolean(refresh);
+        refreshPolicy.writeTo(out);
         if (doc == null) {
             out.writeBoolean(false);
         } else {
diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java
index 30b636f4efc..403f4265fcd 100644
--- a/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java
@@ -21,6 +21,7 @@ package org.elasticsearch.action.update;
 
 import org.elasticsearch.action.WriteConsistencyLevel;
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.support.WriteRequestBuilder;
 import org.elasticsearch.action.support.single.instance.InstanceShardOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -32,9 +33,8 @@ import org.elasticsearch.script.Script;
 
 import java.util.Map;
 
-/**
- */
-public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder<UpdateRequest, UpdateResponse, UpdateRequestBuilder> {
+public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder<UpdateRequest, UpdateResponse, UpdateRequestBuilder>
+        implements WriteRequestBuilder<UpdateRequestBuilder> {
 
     public UpdateRequestBuilder(ElasticsearchClient client, UpdateAction action) {
         super(client, action, new UpdateRequest());
@@ -121,17 +121,6 @@ public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder<U
         return this;
     }
 
-
-    /**
-     * Should a refresh be executed post this update operation causing the operation to
-     * be searchable. Note, heavy indexing should not set this to <tt>true</tt>. Defaults
-     * to <tt>false</tt>.
-     */
-    public UpdateRequestBuilder setRefresh(boolean refresh) {
-        request.refresh(refresh);
-        return this;
-    }
-
     /**
      * Sets the consistency level of write. Defaults to {@link org.elasticsearch.action.WriteConsistencyLevel#DEFAULT}
      */
diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
index 027100b3469..b9d0c6b4c70 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
@@ -115,6 +115,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         IndexSettings.QUERY_STRING_LENIENT_SETTING,
         IndexSettings.ALLOW_UNMAPPED,
         IndexSettings.INDEX_CHECK_ON_STARTUP,
+        IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD,
         ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING,
         IndexSettings.INDEX_GC_DELETES_SETTING,
         IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING,
diff --git a/core/src/main/java/org/elasticsearch/index/IndexSettings.java b/core/src/main/java/org/elasticsearch/index/IndexSettings.java
index 7c8cb4ff8c8..592c1ff1125 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexSettings.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexSettings.java
@@ -115,6 +115,11 @@ public final class IndexSettings {
     public static final Setting<TimeValue> INDEX_GC_DELETES_SETTING =
         Setting.timeSetting("index.gc_deletes", DEFAULT_GC_DELETES, new TimeValue(-1, TimeUnit.MILLISECONDS), Property.Dynamic,
             Property.IndexScope);
+    /**
+     * The maximum number of refresh listeners allows on this shard.
+     */
+    public static final Setting<Integer> MAX_REFRESH_LISTENERS_PER_SHARD = Setting.intSetting("index.max_refresh_listeners", 1000, 0,
+            Property.Dynamic, Property.IndexScope);
 
     private final Index index;
     private final Version version;
@@ -145,6 +150,10 @@ public final class IndexSettings {
     private volatile int maxResultWindow;
     private volatile int maxRescoreWindow;
     private volatile boolean TTLPurgeDisabled;
+    /**
+     * The maximum number of refresh listeners allows on this shard.
+     */
+    private volatile int maxRefreshListeners;
 
     /**
      * Returns the default search field for this index.
@@ -229,6 +238,7 @@ public final class IndexSettings {
         maxResultWindow = scopedSettings.get(MAX_RESULT_WINDOW_SETTING);
         maxRescoreWindow = scopedSettings.get(MAX_RESCORE_WINDOW_SETTING);
         TTLPurgeDisabled = scopedSettings.get(INDEX_TTL_DISABLE_PURGE_SETTING);
+        maxRefreshListeners = scopedSettings.get(MAX_REFRESH_LISTENERS_PER_SHARD);
         this.mergePolicyConfig = new MergePolicyConfig(logger, this);
         assert indexNameMatcher.test(indexMetaData.getIndex().getName());
 
@@ -251,6 +261,7 @@ public final class IndexSettings {
         scopedSettings.addSettingsUpdateConsumer(INDEX_GC_DELETES_SETTING, this::setGCDeletes);
         scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING, this::setTranslogFlushThresholdSize);
         scopedSettings.addSettingsUpdateConsumer(INDEX_REFRESH_INTERVAL_SETTING, this::setRefreshInterval);
+        scopedSettings.addSettingsUpdateConsumer(MAX_REFRESH_LISTENERS_PER_SHARD, this::setMaxRefreshListeners);
     }
 
     private void setTranslogFlushThresholdSize(ByteSizeValue byteSizeValue) {
@@ -499,6 +510,16 @@ public final class IndexSettings {
         return scopedSettings.get(setting);
     }
 
+    /**
+     * The maximum number of refresh listeners allows on this shard.
+     */
+    public int getMaxRefreshListeners() {
+        return maxRefreshListeners;
+    }
+
+    private void setMaxRefreshListeners(int maxRefreshListeners) {
+        this.maxRefreshListeners = maxRefreshListeners;
+    }
 
     IndexScopedSettings getScopedSettings() { return scopedSettings;}
 }
diff --git a/core/src/main/java/org/elasticsearch/index/engine/Engine.java b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
index ab142f9dd51..87ffb9331a6 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -607,6 +607,7 @@ public abstract class Engine implements Closeable {
      * Synchronously refreshes the engine for new search operations to reflect the latest
      * changes.
      */
+    @Nullable
     public abstract void refresh(String source) throws EngineException;
 
     /**
@@ -999,6 +1000,9 @@ public abstract class Engine implements Closeable {
 
         public static final GetResult NOT_EXISTS = new GetResult(false, Versions.NOT_FOUND, null);
 
+        /**
+         * Build a realtime get result from the translog.
+         */
         public GetResult(boolean exists, long version, @Nullable Translog.Source source) {
             this.source = source;
             this.exists = exists;
@@ -1007,6 +1011,9 @@ public abstract class Engine implements Closeable {
             this.searcher = null;
         }
 
+        /**
+         * Build a non-realtime get result from the searcher.
+         */
         public GetResult(Searcher searcher, Versions.DocIdAndVersion docIdAndVersion) {
             this.exists = true;
             this.source = null;
diff --git a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
index 8a56feff70f..13408408e7e 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java
@@ -25,14 +25,15 @@ import org.apache.lucene.index.SnapshotDeletionPolicy;
 import org.apache.lucene.search.QueryCache;
 import org.apache.lucene.search.QueryCachingPolicy;
 import org.apache.lucene.search.similarities.Similarity;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.codec.CodecService;
+import org.elasticsearch.index.shard.RefreshListeners;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.TranslogRecoveryPerformer;
 import org.elasticsearch.index.store.Store;
@@ -40,8 +41,6 @@ import org.elasticsearch.index.translog.TranslogConfig;
 import org.elasticsearch.indices.IndexingMemoryController;
 import org.elasticsearch.threadpool.ThreadPool;
 
-import java.util.function.Function;
-
 /*
  * Holds all the configuration that is used to create an {@link Engine}.
  * Once {@link Engine} has been created with this object, changes to this
@@ -66,6 +65,8 @@ public final class EngineConfig {
     private final Engine.EventListener eventListener;
     private final QueryCache queryCache;
     private final QueryCachingPolicy queryCachingPolicy;
+    @Nullable
+    private final RefreshListeners refreshListeners;
 
     /**
      * Index setting to change the low level lucene codec used for writing new segments.
@@ -99,7 +100,7 @@ public final class EngineConfig {
                         MergePolicy mergePolicy,Analyzer analyzer,
                         Similarity similarity, CodecService codecService, Engine.EventListener eventListener,
                         TranslogRecoveryPerformer translogRecoveryPerformer, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy,
-                        TranslogConfig translogConfig, TimeValue flushMergesAfter) {
+                        TranslogConfig translogConfig, TimeValue flushMergesAfter, RefreshListeners refreshListeners) {
         if (openMode == null) {
             throw new IllegalArgumentException("openMode must not be null");
         }
@@ -125,6 +126,7 @@ public final class EngineConfig {
         this.translogConfig = translogConfig;
         this.flushMergesAfter = flushMergesAfter;
         this.openMode = openMode;
+        this.refreshListeners = refreshListeners;
     }
 
     /**
@@ -303,4 +305,10 @@ public final class EngineConfig {
         OPEN_INDEX_AND_TRANSLOG;
     }
 
+    /**
+     * {@linkplain RefreshListeners} instance to configure.
+     */
+    public RefreshListeners getRefreshListeners() {
+        return refreshListeners;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index c120b07fce8..15667e79421 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -154,6 +154,10 @@ public class InternalEngine extends Engine {
             this.versionMap.setManager(searcherManager);
             // don't allow commits until we are done with recovering
             allowCommits.compareAndSet(true, openMode != EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG);
+            if (engineConfig.getRefreshListeners() != null) {
+                searcherManager.addListener(engineConfig.getRefreshListeners());
+                engineConfig.getRefreshListeners().setTranslog(translog);
+            }
             success = true;
         } finally {
             if (success == false) {
diff --git a/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java b/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java
index c30b2e9bf50..0a55803a5ec 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java
@@ -30,7 +30,6 @@ import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ReleasableLock;
-import org.elasticsearch.index.shard.TranslogRecoveryPerformer;
 import org.elasticsearch.index.translog.Translog;
 
 import java.io.IOException;
@@ -68,6 +67,9 @@ public class ShadowEngine extends Engine {
 
     public ShadowEngine(EngineConfig engineConfig)  {
         super(engineConfig);
+        if (engineConfig.getRefreshListeners() != null) {
+            throw new IllegalArgumentException("ShadowEngine doesn't support RefreshListeners");
+        }
         SearcherFactory searcherFactory = new EngineSearcherFactory(engineConfig);
         final long nonexistentRetryTime = engineConfig.getIndexSettings().getSettings()
                 .getAsTime(NONEXISTENT_INDEX_RETRY_WAIT, DEFAULT_NONEXISTENT_INDEX_RETRY_WAIT)
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 44445f0b6dd..33d1aec7ed0 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -131,6 +131,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
 import java.util.function.BiConsumer;
 
 public class IndexShard extends AbstractIndexShardComponent {
@@ -203,6 +204,12 @@ public class IndexShard extends AbstractIndexShardComponent {
      * IndexingMemoryController}).
      */
     private final AtomicBoolean active = new AtomicBoolean();
+    /**
+     * Allows for the registration of listeners that are called when a change becomes visible for search. This is nullable because
+     * {@linkplain ShadowIndexShard} doesn't support this.
+     */
+    @Nullable
+    private final RefreshListeners refreshListeners;
 
     public IndexShard(ShardRouting shardRouting, IndexSettings indexSettings, ShardPath path, Store store, IndexCache indexCache,
                       MapperService mapperService, SimilarityService similarityService, IndexFieldDataService indexFieldDataService,
@@ -255,6 +262,7 @@ public class IndexShard extends AbstractIndexShardComponent {
         suspendableRefContainer = new SuspendableRefContainer();
         searcherWrapper = indexSearcherWrapper;
         primaryTerm = indexSettings.getIndexMetaData().primaryTerm(shardId.id());
+        refreshListeners = buildRefreshListeners();
         persistMetadata(shardRouting, null);
     }
 
@@ -579,6 +587,7 @@ public class IndexShard extends AbstractIndexShardComponent {
      */
     public void refresh(String source) {
         verifyNotClosed();
+        
         if (canIndex()) {
             long bytes = getEngine().getIndexBufferRAMBytesUsed();
             writingBytes.addAndGet(bytes);
@@ -1530,7 +1539,7 @@ public class IndexShard extends AbstractIndexShardComponent {
         return new EngineConfig(openMode, shardId,
             threadPool, indexSettings, warmer, store, deletionPolicy, indexSettings.getMergePolicy(),
             mapperService.indexAnalyzer(), similarityService.similarity(mapperService), codecService, shardEventListener, translogRecoveryPerformer, indexCache.query(), cachingPolicy, translogConfig,
-            IndexingMemoryController.SHARD_INACTIVE_TIME_SETTING.get(indexSettings.getSettings()));
+            IndexingMemoryController.SHARD_INACTIVE_TIME_SETTING.get(indexSettings.getSettings()), refreshListeners);
     }
 
     public Releasable acquirePrimaryOperationLock() {
@@ -1626,6 +1635,17 @@ public class IndexShard extends AbstractIndexShardComponent {
         return false;
     }
 
+    /**
+     * Build {@linkplain RefreshListeners} for this shard. Protected so {@linkplain ShadowIndexShard} can override it to return null.
+     */
+    protected RefreshListeners buildRefreshListeners() {
+        return new RefreshListeners(
+                indexSettings::getMaxRefreshListeners,
+                () -> refresh("too_many_listeners"),
+                threadPool.executor(ThreadPool.Names.LISTENER)::execute,
+                logger);
+    }
+
     /**
      * Simple struct encapsulating a shard failure
      *
@@ -1651,14 +1671,26 @@ public class IndexShard extends AbstractIndexShardComponent {
     }
 
     /**
-     * Returns <code>true</code> iff one or more changes to the engine are not visible to via the current searcher.
+     * Returns <code>true</code> iff one or more changes to the engine are not visible to via the current searcher *or* there are pending
+     * refresh listeners.
      * Otherwise <code>false</code>.
      *
      * @throws EngineClosedException  if the engine is already closed
      * @throws AlreadyClosedException if the internal indexwriter in the engine is already closed
      */
     public boolean isRefreshNeeded() {
-        return getEngine().refreshNeeded();
+        return getEngine().refreshNeeded() || (refreshListeners != null && refreshListeners.refreshNeeded());
+    }
+
+    /**
+     * Add a listener for refreshes.
+     *
+     * @param location the location to listen for
+     * @param listener for the refresh. Called with true if registering the listener ran it out of slots and forced a refresh. Called with
+     *        false otherwise.
+     */
+    public void addRefreshListener(Translog.Location location, Consumer<Boolean> listener) {
+        refreshListeners.addOrNotify(location, listener);
     }
 
     private class IndexShardRecoveryPerformer extends TranslogRecoveryPerformer {
diff --git a/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java b/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
new file mode 100644
index 00000000000..ab3e334714a
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.shard;
+
+import org.apache.lucene.search.ReferenceManager;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.index.translog.Translog;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Executor;
+import java.util.function.Consumer;
+import java.util.function.IntSupplier;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * Allows for the registration of listeners that are called when a change becomes visible for search. This functionality is exposed from
+ * {@link IndexShard} but kept here so it can be tested without standing up the entire thing. 
+ */
+public final class RefreshListeners implements ReferenceManager.RefreshListener {
+    private final IntSupplier getMaxRefreshListeners;
+    private final Runnable forceRefresh;
+    private final Executor listenerExecutor;
+    private final ESLogger logger;
+
+    /**
+     * List of refresh listeners. Defaults to null and built on demand because most refresh cycles won't need it. Entries are never removed
+     * from it, rather, it is nulled and rebuilt when needed again. The (hopefully) rare entries that didn't make the current refresh cycle
+     * are just added back to the new list. Both the reference and the contents are always modified while synchronized on {@code this}.
+     */
+    private volatile List<Tuple<Translog.Location, Consumer<Boolean>>> refreshListeners = null;
+    /**
+     * The translog location that was last made visible by a refresh.
+     */
+    private volatile Translog.Location lastRefreshedLocation;
+
+    public RefreshListeners(IntSupplier getMaxRefreshListeners, Runnable forceRefresh, Executor listenerExecutor, ESLogger logger) {
+        this.getMaxRefreshListeners = getMaxRefreshListeners;
+        this.forceRefresh = forceRefresh;
+        this.listenerExecutor = listenerExecutor;
+        this.logger = logger;
+    }
+
+    /**
+     * Add a listener for refreshes, calling it immediately if the location is already visible. If this runs out of listener slots then it
+     * forces a refresh and calls the listener immediately as well.
+     *
+     * @param location the location to listen for
+     * @param listener for the refresh. Called with true if registering the listener ran it out of slots and forced a refresh. Called with
+     *        false otherwise.
+     */
+    public void addOrNotify(Translog.Location location, Consumer<Boolean> listener) {
+        requireNonNull(listener, "listener cannot be null");
+        requireNonNull(location, "location cannot be null");
+
+        if (lastRefreshedLocation != null && lastRefreshedLocation.compareTo(location) >= 0) {
+            // Location already visible, just call the listener
+            listener.accept(false);
+            return;
+        }
+        synchronized (this) {
+            if (refreshListeners == null) {
+                refreshListeners = new ArrayList<>();
+            }
+            if (refreshListeners.size() < getMaxRefreshListeners.getAsInt()) {
+                // We have a free slot so register the listener
+                refreshListeners.add(new Tuple<>(location, listener));
+                return;
+            }
+        }
+        // No free slot so force a refresh and call the listener in this thread
+        forceRefresh.run();
+        listener.accept(true);
+    }
+
+    /**
+     * Returns true if there are pending listeners.
+     */
+    public boolean refreshNeeded() {
+        // No need to synchronize here because we're doing a single volatile read
+        return refreshListeners != null;
+    }
+
+    /**
+     * Setup the translog used to find the last refreshed location.
+     */
+    public void setTranslog(Translog translog) {
+        this.translog = translog;
+    }
+
+    // Implementation of ReferenceManager.RefreshListener that adapts Lucene's RefreshListener into Elasticsearch's refresh listeners.
+    private Translog translog;
+    /**
+     * Snapshot of the translog location before the current refresh if there is a refresh going on or null. Doesn't have to be volatile
+     * because when it is used by the refreshing thread.
+     */
+    private Translog.Location currentRefreshLocation;
+
+    @Override
+    public void beforeRefresh() throws IOException {
+        currentRefreshLocation = translog.getLastWriteLocation();
+    }
+
+    @Override
+    public void afterRefresh(boolean didRefresh) throws IOException {
+        /*
+         * We intentionally ignore didRefresh here because our timing is a little off. It'd be a useful flag if we knew everything that made
+         * it into the refresh, but the way we snapshot the translog position before the refresh, things can sneak into the refresh that we
+         * don't know about.
+         */
+        if (null == currentRefreshLocation) {
+            /*
+             * The translog had an empty last write location at the start of the refresh so we can't alert anyone to anything. This
+             * usually happens during recovery. The next refresh cycle out to pick up this refresh.
+             */
+            return;
+        }
+        // First check if we've actually moved forward. If not then just bail immediately.
+        assert lastRefreshedLocation == null || currentRefreshLocation.compareTo(lastRefreshedLocation) >= 0;
+        if (lastRefreshedLocation != null && currentRefreshLocation.compareTo(lastRefreshedLocation) == 0) {
+            return;
+        }
+        /*
+         * Set the lastRefreshedLocation so listeners that come in for locations before that will just execute inline without messing
+         * around with refreshListeners or synchronizing at all.
+         */
+        lastRefreshedLocation = currentRefreshLocation;
+        /*
+         * Grab the current refresh listeners and replace them with null while synchronized. Any listeners that come in after this won't be
+         * in the list we iterate over and very likely won't be candidates for refresh anyway because we've already moved the
+         * lastRefreshedLocation.
+         */
+        List<Tuple<Translog.Location, Consumer<Boolean>>> candidates;
+        synchronized (this) {
+            candidates = refreshListeners;
+            // No listeners to check so just bail early
+            if (candidates == null) {
+                return;
+            }
+            refreshListeners = null;
+        }
+        // Iterate the list of listeners, copying the listeners to fire to one list and those to preserve to another list.
+        List<Consumer<Boolean>> listenersToFire = null;
+        List<Tuple<Translog.Location, Consumer<Boolean>>> preservedListeners = null;
+        for (Tuple<Translog.Location, Consumer<Boolean>> tuple : candidates) {
+            Translog.Location location = tuple.v1();
+            Consumer<Boolean> listener = tuple.v2();
+            if (location.compareTo(currentRefreshLocation) <= 0) {
+                if (listenersToFire == null) {
+                    listenersToFire = new ArrayList<>();
+                }
+                listenersToFire.add(listener);
+            } else {
+                if (preservedListeners == null) {
+                    preservedListeners = new ArrayList<>();
+                }
+                preservedListeners.add(tuple);
+            }
+        }
+        /*
+         * Now add any preserved listeners back to the running list of refresh listeners while under lock. We'll try them next time. While
+         * we were iterating the list of listeners new listeners could have come in. That means that adding all of our preserved listeners
+         * might push our list of listeners above the maximum number of slots allowed. This seems unlikely because we expect few listeners
+         * to be preserved. And the next listener while we're full will trigger a refresh anyway.
+         */
+        if (preservedListeners != null) {
+            synchronized (this) {
+                if (refreshListeners == null) {
+                    refreshListeners = new ArrayList<>();
+                }
+                refreshListeners.addAll(preservedListeners);
+            }
+        }
+        // Lastly, fire the listeners that are ready on the listener thread pool
+        if (listenersToFire != null) {
+            final List<Consumer<Boolean>> finalListenersToFire = listenersToFire;
+            listenerExecutor.execute(() -> {
+                for (Consumer<Boolean> listener : finalListenersToFire) {
+                    try {
+                        listener.accept(false);
+                    } catch (Throwable t) {
+                        logger.warn("Error firing refresh listener", t);
+                    }
+                }
+            });
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java
index bf35d02fea2..e22f684637e 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java
@@ -31,12 +31,14 @@ import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.merge.MergeStats;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.index.store.Store;
+import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.index.translog.TranslogStats;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
+import java.util.function.Consumer;
 
 /**
  * ShadowIndexShard extends {@link IndexShard} to add file synchronization
@@ -86,6 +88,12 @@ public final class ShadowIndexShard extends IndexShard {
         return engineFactory.newReadOnlyEngine(config);
     }
 
+    @Override
+    protected RefreshListeners buildRefreshListeners() {
+        // ShadowEngine doesn't have a translog so it shouldn't try to support RefreshListeners.
+        return null;
+    }
+
     @Override
     public boolean shouldFlush() {
         // we don't need to flush since we don't write - all dominated by the primary
@@ -96,4 +104,9 @@ public final class ShadowIndexShard extends IndexShard {
     public TranslogStats translogStats() {
         return null; // shadow engine has no translog
     }
+
+    @Override
+    public void addRefreshListener(Translog.Location location, Consumer<Boolean> listener) {
+        throw new UnsupportedOperationException("Can't listen for a refresh on a shadow engine because it doesn't have a translog");
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/translog/Translog.java b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
index b66c82d4083..57847972e42 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/Translog.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
@@ -447,6 +447,21 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
         }
     }
 
+    /**
+     * The a {@linkplain Location} that will sort after the {@linkplain Location} returned by the last write but before any locations which
+     * can be returned by the next write.
+     */
+    public Location getLastWriteLocation() {
+        try (ReleasableLock lock = readLock.acquire()) {
+            /*
+             * We use position = current - 1 and size = Integer.MAX_VALUE here instead of position current and size = 0 for two reasons:
+             * 1. Translog.Location's compareTo doesn't actually pay attention to size even though it's equals method does.
+             * 2. It feels more right to return a *position* that is before the next write's position rather than rely on the size.
+             */
+            return new Location(current.generation, current.sizeInBytes() - 1, Integer.MAX_VALUE);
+        }
+    }
+
     boolean assertBytesAtLocation(Translog.Location location, BytesReference expectedBytes) throws IOException {
         // tests can override this
         ByteBuffer buffer = ByteBuffer.allocate(location.size);
diff --git a/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java b/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
index 0109995f80f..b2c0cc88cf9 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
@@ -26,6 +26,7 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.Channels;
+import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.index.shard.ShardId;
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/bulk/RestBulkAction.java b/core/src/main/java/org/elasticsearch/rest/action/bulk/RestBulkAction.java
index 620418eb087..d9dbb21e804 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/bulk/RestBulkAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/bulk/RestBulkAction.java
@@ -84,7 +84,7 @@ public class RestBulkAction extends BaseRestHandler {
             bulkRequest.consistencyLevel(WriteConsistencyLevel.fromString(consistencyLevel));
         }
         bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT));
-        bulkRequest.refresh(request.paramAsBoolean("refresh", bulkRequest.refresh()));
+        bulkRequest.setRefreshPolicy(request.param("refresh"));
         bulkRequest.add(request.content(), defaultIndex, defaultType, defaultRouting, defaultFields, defaultPipeline, null, allowExplicitIndex);
 
         client.bulk(bulkRequest, new RestBuilderListener<BulkResponse>(channel) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/delete/RestDeleteAction.java b/core/src/main/java/org/elasticsearch/rest/action/delete/RestDeleteAction.java
index 8e3449344c4..29316893504 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/delete/RestDeleteAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/delete/RestDeleteAction.java
@@ -51,7 +51,7 @@ public class RestDeleteAction extends BaseRestHandler {
         deleteRequest.routing(request.param("routing"));
         deleteRequest.parent(request.param("parent")); // order is important, set it after routing, so it will set the routing
         deleteRequest.timeout(request.paramAsTime("timeout", DeleteRequest.DEFAULT_TIMEOUT));
-        deleteRequest.refresh(request.paramAsBoolean("refresh", deleteRequest.refresh()));
+        deleteRequest.setRefreshPolicy(request.param("refresh"));
         deleteRequest.version(RestActions.parseVersion(request));
         deleteRequest.versionType(VersionType.fromString(request.param("version_type"), deleteRequest.versionType()));
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java
index 26dd1eca78d..f807e68088a 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java
@@ -80,7 +80,7 @@ public class RestIndexAction extends BaseRestHandler {
         indexRequest.setPipeline(request.param("pipeline"));
         indexRequest.source(request.content());
         indexRequest.timeout(request.paramAsTime("timeout", IndexRequest.DEFAULT_TIMEOUT));
-        indexRequest.refresh(request.paramAsBoolean("refresh", indexRequest.refresh()));
+        indexRequest.setRefreshPolicy(request.param("refresh"));
         indexRequest.version(RestActions.parseVersion(request));
         indexRequest.versionType(VersionType.fromString(request.param("version_type"), indexRequest.versionType()));
         String sOpType = request.param("op_type");
diff --git a/core/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java b/core/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java
index 88f90374523..bdea4e33e6d 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java
@@ -58,7 +58,7 @@ public class RestUpdateAction extends BaseRestHandler {
         updateRequest.routing(request.param("routing"));
         updateRequest.parent(request.param("parent"));
         updateRequest.timeout(request.paramAsTime("timeout", updateRequest.timeout()));
-        updateRequest.refresh(request.paramAsBoolean("refresh", updateRequest.refresh()));
+        updateRequest.setRefreshPolicy(request.param("refresh"));
         String consistencyLevel = request.param("consistency");
         if (consistencyLevel != null) {
             updateRequest.consistencyLevel(WriteConsistencyLevel.fromString(consistencyLevel));
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
index f2afc1e7f6e..337f881d41b 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
@@ -24,6 +24,7 @@ import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.Strings;
@@ -180,22 +181,22 @@ public class BulkRequestTests extends ESTestCase {
     public void testBulkRequestWithRefresh() throws Exception {
         BulkRequest bulkRequest = new BulkRequest();
         // We force here a "id is missing" validation error
-        bulkRequest.add(new DeleteRequest("index", "type", null).refresh(true));
+        bulkRequest.add(new DeleteRequest("index", "type", null).setRefreshPolicy(RefreshPolicy.IMMEDIATE));
         // We force here a "type is missing" validation error
         bulkRequest.add(new DeleteRequest("index", null, "id"));
-        bulkRequest.add(new DeleteRequest("index", "type", "id").refresh(true));
-        bulkRequest.add(new UpdateRequest("index", "type", "id").doc("{}").refresh(true));
-        bulkRequest.add(new IndexRequest("index", "type", "id").source("{}").refresh(true));
+        bulkRequest.add(new DeleteRequest("index", "type", "id").setRefreshPolicy(RefreshPolicy.IMMEDIATE));
+        bulkRequest.add(new UpdateRequest("index", "type", "id").doc("{}").setRefreshPolicy(RefreshPolicy.IMMEDIATE));
+        bulkRequest.add(new IndexRequest("index", "type", "id").source("{}").setRefreshPolicy(RefreshPolicy.IMMEDIATE));
         ActionRequestValidationException validate = bulkRequest.validate();
         assertThat(validate, notNullValue());
         assertThat(validate.validationErrors(), not(empty()));
         assertThat(validate.validationErrors(), contains(
-                "Refresh is not supported on an item request, set the refresh flag on the BulkRequest instead.",
+                "RefreshPolicy is not supported on an item request. Set it on the BulkRequest instead.",
                 "id is missing",
                 "type is missing",
-                "Refresh is not supported on an item request, set the refresh flag on the BulkRequest instead.",
-                "Refresh is not supported on an item request, set the refresh flag on the BulkRequest instead.",
-                "Refresh is not supported on an item request, set the refresh flag on the BulkRequest instead."));
+                "RefreshPolicy is not supported on an item request. Set it on the BulkRequest instead.",
+                "RefreshPolicy is not supported on an item request. Set it on the BulkRequest instead.",
+                "RefreshPolicy is not supported on an item request. Set it on the BulkRequest instead."));
     }
 
     // issue 15120
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java
index ff1a24d6900..b26d2531ff0 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.bulk;
 
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
 
@@ -28,9 +29,11 @@ public class BulkShardRequestTests extends ESTestCase {
     public void testToString() {
         String index = randomSimpleString(random(), 10);
         int count = between(1, 100);
-        BulkShardRequest r = new BulkShardRequest(null, new ShardId(index, "ignored", 0), false, new BulkItemRequest[count]);
+        BulkShardRequest r = new BulkShardRequest(null, new ShardId(index, "ignored", 0), RefreshPolicy.NONE, new BulkItemRequest[count]);
         assertEquals("BulkShardRequest to [" + index + "] containing [" + count + "] requests", r.toString());
-        r = new BulkShardRequest(null, new ShardId(index, "ignored", 0), true, new BulkItemRequest[count]);
+        r = new BulkShardRequest(null, new ShardId(index, "ignored", 0), RefreshPolicy.IMMEDIATE, new BulkItemRequest[count]);
         assertEquals("BulkShardRequest to [" + index + "] containing [" + count + "] requests and a refresh", r.toString());
+        r = new BulkShardRequest(null, new ShardId(index, "ignored", 0), RefreshPolicy.WAIT_UNTIL, new BulkItemRequest[count]);
+        assertEquals("BulkShardRequest to [" + index + "] containing [" + count + "] requests blocking until refresh", r.toString());
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
index 6175f822b6a..ebaa5b5c01e 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
@@ -21,7 +21,6 @@ package org.elasticsearch.action.support.replication;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.NoShardAvailableActionException;
-import org.elasticsearch.action.ReplicationResponse;
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.UnavailableShardsException;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java
index cc7558d1de8..55e2a9d3cf2 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java
@@ -21,16 +21,15 @@ package org.elasticsearch.action.support.replication;
 import org.apache.lucene.index.CorruptIndexException;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ReplicationResponse;
 import org.elasticsearch.action.UnavailableShardsException;
 import org.elasticsearch.action.WriteConsistencyLevel;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.logging.ESLogger;
@@ -102,7 +101,7 @@ public class ReplicationOperationTests extends ESTestCase {
         }
 
         Request request = new Request(shardId);
-        PlainActionFuture<Response> listener = new PlainActionFuture<>();
+        PlainActionFuture<TestPrimary.Result> listener = new PlainActionFuture<>();
         final ClusterState finalState = state;
         final TestReplicaProxy replicasProxy = new TestReplicaProxy(expectedFailures);
         final TestReplicationOperation op = new TestReplicationOperation(request,
@@ -114,7 +113,7 @@ public class ReplicationOperationTests extends ESTestCase {
         assertThat(request.processedOnReplicas, equalTo(expectedReplicas));
         assertThat(replicasProxy.failedReplicas, equalTo(expectedFailedShards));
         assertTrue("listener is not marked as done", listener.isDone());
-        Response.ShardInfo shardInfo = listener.actionGet().getShardInfo();
+        ShardInfo shardInfo = listener.actionGet().getShardInfo();
         assertThat(shardInfo.getFailed(), equalTo(expectedFailedShards.size()));
         assertThat(shardInfo.getFailures(), arrayWithSize(expectedFailedShards.size()));
         assertThat(shardInfo.getSuccessful(), equalTo(1 + expectedReplicas.size() - expectedFailures.size()));
@@ -135,7 +134,7 @@ public class ReplicationOperationTests extends ESTestCase {
         final ShardRouting primaryShard = indexShardRoutingTable.primaryShard();
 
         Request request = new Request(shardId);
-        PlainActionFuture<Response> listener = new PlainActionFuture<>();
+        PlainActionFuture<TestPrimary.Result> listener = new PlainActionFuture<>();
         final TestReplicationOperation op = new TestReplicationOperation(request,
             new TestPrimary(primaryShard, primaryTerm), listener, false, false,
             new TestReplicaProxy(), () -> state, logger, "test");
@@ -143,7 +142,7 @@ public class ReplicationOperationTests extends ESTestCase {
         assertThat("request was not processed on primary", request.processedOnPrimary.get(), equalTo(true));
         assertThat(request.processedOnReplicas, equalTo(Collections.emptySet()));
         assertTrue("listener is not marked as done", listener.isDone());
-        Response.ShardInfo shardInfo = listener.actionGet().getShardInfo();
+        ShardInfo shardInfo = listener.actionGet().getShardInfo();
         assertThat(shardInfo.getFailed(), equalTo(0));
         assertThat(shardInfo.getFailures(), arrayWithSize(0));
         assertThat(shardInfo.getSuccessful(), equalTo(1));
@@ -172,7 +171,7 @@ public class ReplicationOperationTests extends ESTestCase {
         expectedFailures.put(failedReplica, new CorruptIndexException("simulated", (String) null));
 
         Request request = new Request(shardId);
-        PlainActionFuture<Response> listener = new PlainActionFuture<>();
+        PlainActionFuture<TestPrimary.Result> listener = new PlainActionFuture<>();
         final ClusterState finalState = state;
         final TestReplicaProxy replicasProxy = new TestReplicaProxy(expectedFailures) {
             @Override
@@ -233,16 +232,16 @@ public class ReplicationOperationTests extends ESTestCase {
         final ShardRouting primaryShard = state.get().routingTable().shardRoutingTable(shardId).primaryShard();
         final TestPrimary primary = new TestPrimary(primaryShard, primaryTerm) {
             @Override
-            public Tuple<Response, Request> perform(Request request) throws Exception {
-                final Tuple<Response, Request> tuple = super.perform(request);
+            public Result perform(Request request) throws Exception {
+                Result result = super.perform(request);
                 state.set(changedState);
                 logger.debug("--> state after primary operation:\n{}", state.get().prettyPrint());
-                return tuple;
+                return result;
             }
         };
 
         Request request = new Request(shardId);
-        PlainActionFuture<Response> listener = new PlainActionFuture<>();
+        PlainActionFuture<TestPrimary.Result> listener = new PlainActionFuture<>();
         final TestReplicationOperation op = new TestReplicationOperation(request, primary, listener,
             new TestReplicaProxy(), state::get);
         op.execute();
@@ -296,7 +295,7 @@ public class ReplicationOperationTests extends ESTestCase {
             state.prettyPrint());
         final long primaryTerm = state.metaData().index(index).primaryTerm(shardId.id());
         final IndexShardRoutingTable shardRoutingTable = state.routingTable().index(index).shard(shardId.id());
-        PlainActionFuture<Response> listener = new PlainActionFuture<>();
+        PlainActionFuture<TestPrimary.Result> listener = new PlainActionFuture<>();
         final ShardRouting primaryShard = shardRoutingTable.primaryShard();
         final TestReplicationOperation op = new TestReplicationOperation(request,
             new TestPrimary(primaryShard, primaryTerm),
@@ -362,10 +361,7 @@ public class ReplicationOperationTests extends ESTestCase {
         }
     }
 
-    static class Response extends ReplicationResponse {
-    }
-
-    static class TestPrimary implements ReplicationOperation.Primary<Request, Request, Response> {
+    static class TestPrimary implements ReplicationOperation.Primary<Request, Request, TestPrimary.Result> {
         final ShardRouting routing;
         final long term;
 
@@ -385,12 +381,35 @@ public class ReplicationOperationTests extends ESTestCase {
         }
 
         @Override
-        public Tuple<Response, Request> perform(Request request) throws Exception {
+        public Result perform(Request request) throws Exception {
             if (request.processedOnPrimary.compareAndSet(false, true) == false) {
                 fail("processed [" + request + "] twice");
             }
             request.primaryTerm(term);
-            return new Tuple<>(new Response(), request);
+            return new Result(request);
+        }
+
+        static class Result implements ReplicationOperation.PrimaryResult<Request> {
+            private final Request request;
+            private ShardInfo shardInfo;
+
+            public Result(Request request) {
+                this.request = request;
+            }
+
+            @Override
+            public Request replicaRequest() {
+                return request;
+            }
+
+            @Override
+            public void setShardInfo(ShardInfo shardInfo) {
+                this.shardInfo = shardInfo;
+            }
+
+            public ShardInfo getShardInfo() {
+                return shardInfo;
+            }
         }
     }
 
@@ -436,15 +455,15 @@ public class ReplicationOperationTests extends ESTestCase {
         }
     }
 
-    class TestReplicationOperation extends ReplicationOperation<Request, Request, Response> {
-        public TestReplicationOperation(Request request, Primary<Request, Request, Response> primary, ActionListener<Response> listener,
-                                        Replicas<Request> replicas, Supplier<ClusterState> clusterStateSupplier) {
+    class TestReplicationOperation extends ReplicationOperation<Request, Request, TestPrimary.Result> {
+        public TestReplicationOperation(Request request, Primary<Request, Request, TestPrimary.Result> primary,
+                ActionListener<TestPrimary.Result> listener, Replicas<Request> replicas, Supplier<ClusterState> clusterStateSupplier) {
             this(request, primary, listener, true, false, replicas, clusterStateSupplier, ReplicationOperationTests.this.logger, "test");
         }
 
-        public TestReplicationOperation(Request request, Primary<Request, Request, Response> primary, ActionListener<Response> listener,
-                                        boolean executeOnReplicas, boolean checkWriteConsistency, Replicas<Request> replicas,
-                                        Supplier<ClusterState> clusterStateSupplier, ESLogger logger, String opType) {
+        public TestReplicationOperation(Request request, Primary<Request, Request, TestPrimary.Result> primary,
+                ActionListener<TestPrimary.Result> listener, boolean executeOnReplicas, boolean checkWriteConsistency,
+                Replicas<Request> replicas, Supplier<ClusterState> clusterStateSupplier, ESLogger logger, String opType) {
             super(request, primary, listener, executeOnReplicas, checkWriteConsistency, replicas, clusterStateSupplier, logger, opType);
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
index 2e81ec712eb..afa72ec7526 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
@@ -20,7 +20,6 @@ package org.elasticsearch.action.support.replication;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ReplicationResponse;
 import org.elasticsearch.action.UnavailableShardsException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
@@ -43,7 +42,6 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lease.Releasable;
@@ -155,7 +153,7 @@ public class TransportReplicationActionTests extends ESTestCase {
         ClusterBlocks.Builder block = ClusterBlocks.builder()
             .addGlobalBlock(new ClusterBlock(1, "non retryable", false, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL));
         setState(clusterService, ClusterState.builder(clusterService.state()).blocks(block));
-        TransportReplicationAction.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener);
+        Action.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener);
         reroutePhase.run();
         assertListenerThrows("primary phase should fail operation", listener, ClusterBlockException.class);
         assertPhase(task, "failed");
@@ -199,7 +197,7 @@ public class TransportReplicationActionTests extends ESTestCase {
 
         Request request = new Request(shardId).timeout("1ms");
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
-        TransportReplicationAction.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener);
+        Action.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener);
         reroutePhase.run();
         assertListenerThrows("unassigned primary didn't cause a timeout", listener, UnavailableShardsException.class);
         assertPhase(task, "failed");
@@ -245,7 +243,7 @@ public class TransportReplicationActionTests extends ESTestCase {
 
         Request request = new Request(shardId).timeout("1ms").routedBasedOnClusterVersion(clusterService.state().version() + 1);
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
-        TransportReplicationAction.ReroutePhase reroutePhase = action.new ReroutePhase(null, request, listener);
+        Action.ReroutePhase reroutePhase = action.new ReroutePhase(null, request, listener);
         reroutePhase.run();
         assertListenerThrows("cluster state too old didn't cause a timeout", listener, UnavailableShardsException.class);
 
@@ -285,7 +283,7 @@ public class TransportReplicationActionTests extends ESTestCase {
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
         ReplicationTask task = maybeTask();
 
-        TransportReplicationAction.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener);
+        Action.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener);
         reroutePhase.run();
         assertListenerThrows("must throw index not found exception", listener, IndexNotFoundException.class);
         assertPhase(task, "failed");
@@ -312,7 +310,7 @@ public class TransportReplicationActionTests extends ESTestCase {
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
         ReplicationTask task = maybeTask();
 
-        TransportReplicationAction.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener);
+        Action.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener);
         reroutePhase.run();
         CapturingTransport.CapturedRequest[] capturedRequests = transport.getCapturedRequestsAndClear();
         assertThat(capturedRequests, arrayWithSize(1));
@@ -364,7 +362,7 @@ public class TransportReplicationActionTests extends ESTestCase {
         Request request = new Request(shardId);
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
 
-        TransportReplicationAction.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener);
+        Action.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener);
         reroutePhase.run();
         assertThat(request.shardId(), equalTo(shardId));
         logger.info("--> primary is assigned to [{}], checking request forwarded", primaryNodeId);
@@ -393,9 +391,9 @@ public class TransportReplicationActionTests extends ESTestCase {
         AtomicBoolean executed = new AtomicBoolean();
         Action.PrimaryOperationTransportHandler primaryPhase = action.new PrimaryOperationTransportHandler() {
             @Override
-            protected ReplicationOperation createReplicatedOperation(Request request, ActionListener<Response> actionListener,
-                                                                     Action.PrimaryShardReference primaryShardReference,
-                                                                     boolean executeOnReplicas) {
+            protected ReplicationOperation<Request, Request, Action.PrimaryResult> createReplicatedOperation(Request request,
+                    ActionListener<Action.PrimaryResult> actionListener, Action.PrimaryShardReference primaryShardReference,
+                    boolean executeOnReplicas) {
                 return new NoopReplicationOperation(request, actionListener) {
                     public void execute() throws Exception {
                         assertPhase(task, "primary");
@@ -448,9 +446,9 @@ public class TransportReplicationActionTests extends ESTestCase {
         AtomicBoolean executed = new AtomicBoolean();
         Action.PrimaryOperationTransportHandler primaryPhase = action.new PrimaryOperationTransportHandler() {
             @Override
-            protected ReplicationOperation createReplicatedOperation(Request request, ActionListener<Response> actionListener,
-                                                                     Action.PrimaryShardReference primaryShardReference,
-                                                                     boolean executeOnReplicas) {
+            protected ReplicationOperation<Request, Request, Action.PrimaryResult> createReplicatedOperation(Request request,
+                    ActionListener<Action.PrimaryResult> actionListener, Action.PrimaryShardReference primaryShardReference,
+                    boolean executeOnReplicas) {
                 return new NoopReplicationOperation(request, actionListener) {
                     public void execute() throws Exception {
                         assertPhase(task, "primary");
@@ -478,9 +476,9 @@ public class TransportReplicationActionTests extends ESTestCase {
         };
         Action.PrimaryShardReference primary = action.new PrimaryShardReference(shard, releasable);
         final Request request = new Request();
-        Tuple<Response, Request> result = primary.perform(request);
+        Request replicaRequest = primary.perform(request).replicaRequest;
 
-        assertThat(result.v2().primaryTerm(), equalTo(primaryTerm));
+        assertThat(replicaRequest.primaryTerm(), equalTo(primaryTerm));
 
         final ElasticsearchException exception = new ElasticsearchException("testing");
         primary.failShard("test", exception);
@@ -582,9 +580,9 @@ public class TransportReplicationActionTests extends ESTestCase {
         setState(clusterService, state);
         Action.PrimaryOperationTransportHandler primaryPhase = action.new PrimaryOperationTransportHandler() {
             @Override
-            protected ReplicationOperation createReplicatedOperation(Request request, ActionListener<Response> actionListener,
-                                                                     Action.PrimaryShardReference primaryShardReference,
-                                                                     boolean executeOnReplicas) {
+            protected ReplicationOperation<Request, Request, Action.PrimaryResult> createReplicatedOperation(Request request,
+                    ActionListener<Action.PrimaryResult> actionListener, Action.PrimaryShardReference primaryShardReference,
+                    boolean executeOnReplicas) {
                 assertFalse(executeOnReplicas);
                 return new NoopReplicationOperation(request, actionListener);
             }
@@ -608,9 +606,9 @@ public class TransportReplicationActionTests extends ESTestCase {
         Action.PrimaryOperationTransportHandler primaryPhase = action.new PrimaryOperationTransportHandler() {
 
             @Override
-            protected ReplicationOperation createReplicatedOperation(Request request, ActionListener<Response> listener,
-                                                                     Action.PrimaryShardReference primaryShardReference,
-                                                                     boolean executeOnReplicas) {
+            protected ReplicationOperation<Request, Request, Action.PrimaryResult> createReplicatedOperation(Request request,
+                    ActionListener<Action.PrimaryResult> listener, Action.PrimaryShardReference primaryShardReference,
+                    boolean executeOnReplicas) {
                 assertIndexShardCounter(1);
                 if (throwExceptionOnCreation) {
                     throw new ElasticsearchException("simulated exception, during createReplicatedOperation");
@@ -623,7 +621,7 @@ public class TransportReplicationActionTests extends ESTestCase {
                         if (throwExceptionOnRun) {
                             throw new ElasticsearchException("simulated exception, during performOnPrimary");
                         } else if (respondWithError) {
-                            this.finalResponseListener.onFailure(new ElasticsearchException("simulated exception, as a response"));
+                            this.resultListener.onFailure(new ElasticsearchException("simulated exception, as a response"));
                         } else {
                             super.execute();
                         }
@@ -667,13 +665,13 @@ public class TransportReplicationActionTests extends ESTestCase {
         final ReplicationTask task = maybeTask();
         Action action = new Action(Settings.EMPTY, "testActionWithExceptions", transportService, clusterService, threadPool) {
             @Override
-            protected void shardOperationOnReplica(Request request) {
+            protected ReplicaResult shardOperationOnReplica(Request request) {
                 assertIndexShardCounter(1);
                 assertPhase(task, "replica");
                 if (throwException) {
                     throw new ElasticsearchException("simulated");
                 }
-                super.shardOperationOnReplica(request);
+                return new ReplicaResult();
             }
         };
         final Action.ReplicaOperationTransportHandler replicaOperationTransportHandler = action.new ReplicaOperationTransportHandler();
@@ -765,15 +763,16 @@ public class TransportReplicationActionTests extends ESTestCase {
         }
 
         @Override
-        protected Tuple<Response, Request> shardOperationOnPrimary(Request shardRequest) throws Exception {
+        protected PrimaryResult shardOperationOnPrimary(Request shardRequest) throws Exception {
             boolean executedBefore = shardRequest.processedOnPrimary.getAndSet(true);
             assert executedBefore == false : "request has already been executed on the primary";
-            return new Tuple<>(new Response(), shardRequest);
+            return new PrimaryResult(shardRequest, new Response());
         }
 
         @Override
-        protected void shardOperationOnReplica(Request request) {
+        protected ReplicaResult shardOperationOnReplica(Request request) {
             request.processedOnReplicas.incrementAndGet();
+            return new ReplicaResult();
         }
 
         @Override
@@ -822,15 +821,14 @@ public class TransportReplicationActionTests extends ESTestCase {
         }
     }
 
-    class NoopReplicationOperation extends ReplicationOperation<Request, Request, Response> {
-
-        public NoopReplicationOperation(Request request, ActionListener<Response> listener) {
+    class NoopReplicationOperation extends ReplicationOperation<Request, Request, Action.PrimaryResult> {
+        public NoopReplicationOperation(Request request, ActionListener<Action.PrimaryResult> listener) {
             super(request, null, listener, true, true, null, null, TransportReplicationActionTests.this.logger, "noop");
         }
 
         @Override
         public void execute() throws Exception {
-            this.finalResponseListener.onResponse(new Response());
+            this.resultListener.onResponse(action.new PrimaryResult(null, new Response()));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
new file mode 100644
index 00000000000..7b312959631
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.support.replication;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.action.support.WriteResponse;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.translog.Translog;
+import org.elasticsearch.index.translog.Translog.Location;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.junit.Before;
+import org.mockito.ArgumentCaptor;
+
+import java.util.HashSet;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+
+public class TransportWriteActionTests extends ESTestCase {
+    private IndexShard indexShard;
+    private Translog.Location location;
+
+    @Before
+    public void initCommonMocks() {
+        indexShard = mock(IndexShard.class);
+        location = mock(Translog.Location.class);
+    }
+
+    public void testPrimaryNoRefreshCall() throws Exception {
+        noRefreshCall(TestAction::shardOperationOnPrimary, TestAction.WritePrimaryResult::respond);
+    }
+
+    public void testReplicaNoRefreshCall() throws Exception {
+        noRefreshCall(TestAction::shardOperationOnReplica, TestAction.WriteReplicaResult::respond);
+    }
+
+    private <Result, Response> void noRefreshCall(ThrowingBiFunction<TestAction, TestRequest, Result> action,
+                                        BiConsumer<Result, CapturingActionListener<Response>> responder)
+            throws Exception {
+        TestRequest request = new TestRequest();
+        request.setRefreshPolicy(RefreshPolicy.NONE); // The default, but we'll set it anyway just to be explicit
+        Result result = action.apply(new TestAction(), request);
+        CapturingActionListener<Response> listener = new CapturingActionListener<>();
+        responder.accept(result, listener);
+        assertNotNull(listener.response);
+        verify(indexShard, never()).refresh(any());
+        verify(indexShard, never()).addRefreshListener(any(), any());
+    }
+
+    public void testPrimaryImmediateRefresh() throws Exception {
+        immediateRefresh(TestAction::shardOperationOnPrimary, TestAction.WritePrimaryResult::respond, r -> assertTrue(r.forcedRefresh));
+    }
+
+    public void testReplicaImmediateRefresh() throws Exception {
+        immediateRefresh(TestAction::shardOperationOnReplica, TestAction.WriteReplicaResult::respond, r -> {});
+    }
+
+    private <Result, Response> void immediateRefresh(ThrowingBiFunction<TestAction, TestRequest, Result> action,
+                                                     BiConsumer<Result, CapturingActionListener<Response>> responder,
+                                                     Consumer<Response> responseChecker) throws Exception {
+        TestRequest request = new TestRequest();
+        request.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
+        Result result = action.apply(new TestAction(), request);
+        CapturingActionListener<Response> listener = new CapturingActionListener<>();
+        responder.accept(result, listener);
+        assertNotNull(listener.response);
+        responseChecker.accept(listener.response);
+        verify(indexShard).refresh("refresh_flag_index");
+        verify(indexShard, never()).addRefreshListener(any(), any());
+    }
+
+    public void testPrimaryWaitForRefresh() throws Exception {
+        waitForRefresh(TestAction::shardOperationOnPrimary, TestAction.WritePrimaryResult::respond,
+            (r, forcedRefresh) -> assertEquals(forcedRefresh, r.forcedRefresh));
+    }
+
+    public void testReplicaWaitForRefresh() throws Exception {
+        waitForRefresh(TestAction::shardOperationOnReplica, TestAction.WriteReplicaResult::respond, (r, forcedRefresh) -> {});
+    }
+
+    private <Result, Response> void waitForRefresh(ThrowingBiFunction<TestAction, TestRequest, Result> action,
+                                                   BiConsumer<Result, CapturingActionListener<Response>> responder,
+                                         BiConsumer<Response, Boolean> resultChecker) throws Exception {
+        TestRequest request = new TestRequest();
+        request.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL);
+        Result result = action.apply(new TestAction(), request);
+        CapturingActionListener<Response> listener = new CapturingActionListener<>();
+        responder.accept(result, listener);
+        assertNull(listener.response); // Haven't reallresponded yet
+
+        @SuppressWarnings({ "unchecked", "rawtypes" })
+        ArgumentCaptor<Consumer<Boolean>> refreshListener = ArgumentCaptor.forClass((Class) Consumer.class);
+        verify(indexShard, never()).refresh(any());
+        verify(indexShard).addRefreshListener(any(), refreshListener.capture());
+
+        // Now we can fire the listener manually and we'll get a response
+        boolean forcedRefresh = randomBoolean();
+        refreshListener.getValue().accept(forcedRefresh);
+        assertNotNull(listener.response);
+        resultChecker.accept(listener.response, forcedRefresh);
+    }
+
+    private class TestAction extends TransportWriteAction<TestRequest, TestResponse> {
+        protected TestAction() {
+            super(Settings.EMPTY, "test", mock(TransportService.class), null, null, null, null, new ActionFilters(new HashSet<>()),
+                    new IndexNameExpressionResolver(Settings.EMPTY), TestRequest::new, ThreadPool.Names.SAME);
+        }
+
+        @Override
+        protected IndexShard indexShard(TestRequest request) {
+            return indexShard;
+        }
+
+        @Override
+        protected WriteResult<TestResponse> onPrimaryShard(TestRequest request, IndexShard indexShard) throws Exception {
+            return new WriteResult<>(new TestResponse(), location);
+        }
+
+        @Override
+        protected Location onReplicaShard(TestRequest request, IndexShard indexShard) {
+            return location;
+        }
+
+        @Override
+        protected TestResponse newResponseInstance() {
+            return new TestResponse();
+        }
+    }
+
+    private static class TestRequest extends ReplicatedWriteRequest<TestRequest> {
+        public TestRequest() {
+            setShardId(new ShardId("test", "test", 1));
+        }
+    }
+
+    private static class TestResponse extends ReplicationResponse implements WriteResponse {
+        boolean forcedRefresh;
+
+        @Override
+        public void setForcedRefresh(boolean forcedRefresh) {
+            this.forcedRefresh = forcedRefresh;
+        }
+    }
+
+    private static class CapturingActionListener<R> implements ActionListener<R> {
+        private R response;
+
+        @Override
+        public void onResponse(R response) {
+            this.response = response;
+        }
+
+        @Override
+        public void onFailure(Throwable e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private interface ThrowingBiFunction<A, B, R> {
+        R apply(A a, B b) throws Exception;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
index 08f8970f1e4..2a16625d037 100644
--- a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
+++ b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
@@ -29,6 +29,7 @@ import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.AliasAction;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
@@ -175,10 +176,15 @@ public class IndexAliasesIT extends ESIntegTestCase {
         assertAcked(admin().indices().prepareAliases().addAlias("test", "tests", termQuery("name", "test")));
 
         logger.info("--> indexing against [test]");
-        client().index(indexRequest("test").type("type1").id("1").source(source("1", "foo test")).refresh(true)).actionGet();
-        client().index(indexRequest("test").type("type1").id("2").source(source("2", "bar test")).refresh(true)).actionGet();
-        client().index(indexRequest("test").type("type1").id("3").source(source("3", "baz test")).refresh(true)).actionGet();
-        client().index(indexRequest("test").type("type1").id("4").source(source("4", "something else")).refresh(true)).actionGet();
+        client().index(indexRequest("test").type("type1").id("1").source(source("1", "foo test")).setRefreshPolicy(RefreshPolicy.IMMEDIATE))
+                .actionGet();
+        client().index(indexRequest("test").type("type1").id("2").source(source("2", "bar test")).setRefreshPolicy(RefreshPolicy.IMMEDIATE))
+                .actionGet();
+        client().index(indexRequest("test").type("type1").id("3").source(source("3", "baz test")).setRefreshPolicy(RefreshPolicy.IMMEDIATE))
+                .actionGet();
+        client().index(
+                indexRequest("test").type("type1").id("4").source(source("4", "something else")).setRefreshPolicy(RefreshPolicy.IMMEDIATE))
+                .actionGet();
 
         logger.info("--> checking single filtering alias search");
         SearchResponse searchResponse = client().prepareSearch("foos").setQuery(QueryBuilders.matchAllQuery()).get();
diff --git a/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java b/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java
index 0d43580c9bc..c86535e40c5 100644
--- a/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java
@@ -22,6 +22,7 @@ package org.elasticsearch.cluster.allocation;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -225,7 +226,7 @@ public class ClusterRerouteIT extends ESIntegTestCase {
         assertThat(state.getRoutingNodes().unassigned().size(), equalTo(1));
         assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_1).getId()).iterator().next().state(), equalTo(ShardRoutingState.STARTED));
 
-        client().prepareIndex("test", "type", "1").setSource("field", "value").setRefresh(true).execute().actionGet();
+        client().prepareIndex("test", "type", "1").setSource("field", "value").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
         final Index index = resolveIndex("test");
 
         logger.info("--> closing all nodes");
diff --git a/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java b/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java
index 4f28cf19d7b..765cee3b6e8 100644
--- a/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java
+++ b/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.document;
 
-import org.elasticsearch.action.ReplicationResponse;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
 import org.elasticsearch.action.bulk.BulkItemResponse;
@@ -27,6 +26,7 @@ import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.support.replication.ReplicationResponse;
 import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
diff --git a/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java b/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java
new file mode 100644
index 00000000000..b2cb2d96818
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index;
+
+import org.elasticsearch.action.ListenableActionFuture;
+import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.delete.DeleteResponse;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
+import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.NativeScriptFactory;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptModule;
+import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.junit.Before;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singleton;
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoSearchHits;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits;
+
+/**
+ * Tests that requests with RefreshPolicy.WAIT_UNTIL will be visible when they return.
+ */
+public class WaitUntilRefreshIT extends ESIntegTestCase {
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(NetworkModule.HTTP_ENABLED.getKey(), true).build();
+    }
+
+    @Override
+    public Settings indexSettings() {
+        // Use a shorter refresh interval to speed up the tests. We'll be waiting on this interval several times.
+        return Settings.builder().put(super.indexSettings()).put("index.refresh_interval", "40ms").build();
+    }
+
+    @Before
+    public void createTestIndex() {
+        createIndex("test");
+    }
+
+    public void testIndex() {
+        IndexResponse index = client().prepareIndex("test", "index", "1").setSource("foo", "bar").setRefreshPolicy(RefreshPolicy.WAIT_UNTIL)
+                .get();
+        assertEquals(RestStatus.CREATED, index.status());
+        assertFalse("request shouldn't have forced a refresh", index.forcedRefresh());
+        assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get(), "1");
+    }
+
+    public void testDelete() throws InterruptedException, ExecutionException {
+        // Index normally
+        indexRandom(true, client().prepareIndex("test", "test", "1").setSource("foo", "bar"));
+        assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get(), "1");
+
+        // Now delete with blockUntilRefresh
+        DeleteResponse delete = client().prepareDelete("test", "test", "1").setRefreshPolicy(RefreshPolicy.WAIT_UNTIL).get();
+        assertTrue("document was deleted", delete.isFound());
+        assertFalse("request shouldn't have forced a refresh", delete.forcedRefresh());
+        assertNoSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get());
+    }
+
+    public void testUpdate() throws InterruptedException, ExecutionException {
+        // Index normally
+        indexRandom(true, client().prepareIndex("test", "test", "1").setSource("foo", "bar"));
+        assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get(), "1");
+
+        // Update with RefreshPolicy.WAIT_UNTIL
+        UpdateResponse update = client().prepareUpdate("test", "test", "1").setDoc("foo", "baz").setRefreshPolicy(RefreshPolicy.WAIT_UNTIL)
+                .get();
+        assertEquals(2, update.getVersion());
+        assertFalse("request shouldn't have forced a refresh", update.forcedRefresh());
+        assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "baz")).get(), "1");
+
+        // Upsert with RefreshPolicy.WAIT_UNTIL
+        update = client().prepareUpdate("test", "test", "2").setDocAsUpsert(true).setDoc("foo", "cat")
+                .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL).get();
+        assertEquals(1, update.getVersion());
+        assertFalse("request shouldn't have forced a refresh", update.forcedRefresh());
+        assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "cat")).get(), "2");
+
+        // Update-becomes-delete with RefreshPolicy.WAIT_UNTIL
+        update = client().prepareUpdate("test", "test", "2").setScript(new Script("delete_plz", ScriptType.INLINE, "native", emptyMap()))
+                .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL).get();
+        assertEquals(2, update.getVersion());
+        assertFalse("request shouldn't have forced a refresh", update.forcedRefresh());
+        assertNoSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "cat")).get());
+    }
+
+    public void testBulk() {
+        // Index by bulk with RefreshPolicy.WAIT_UNTIL
+        BulkRequestBuilder bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL);
+        bulk.add(client().prepareIndex("test", "test", "1").setSource("foo", "bar"));
+        assertBulkSuccess(bulk.get());
+        assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get(), "1");
+
+        // Update by bulk with RefreshPolicy.WAIT_UNTIL
+        bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL);
+        bulk.add(client().prepareUpdate("test", "test", "1").setDoc("foo", "baz"));
+        assertBulkSuccess(bulk.get());
+        assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "baz")).get(), "1");
+
+        // Delete by bulk with RefreshPolicy.WAIT_UNTIL
+        bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL);
+        bulk.add(client().prepareDelete("test", "test", "1"));
+        assertBulkSuccess(bulk.get());
+        assertNoSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get());
+
+        // Update makes a noop
+        bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL);
+        bulk.add(client().prepareDelete("test", "test", "1"));
+        assertBulkSuccess(bulk.get());
+    }
+
+    /**
+     * Tests that an explicit request makes block_until_refresh return. It doesn't check that block_until_refresh doesn't return until the
+     * explicit refresh if the interval is -1 because we don't have that kind of control over refresh. It can happen all on its own.
+     */
+    public void testNoRefreshInterval() throws InterruptedException, ExecutionException {
+        client().admin().indices().prepareUpdateSettings("test").setSettings(singletonMap("index.refresh_interval", -1)).get();
+        ListenableActionFuture<IndexResponse> index = client().prepareIndex("test", "index", "1").setSource("foo", "bar")
+                .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL).execute();
+        while (false == index.isDone()) {
+            client().admin().indices().prepareRefresh("test").get();
+        }
+        assertEquals(RestStatus.CREATED, index.get().status());
+        assertFalse("request shouldn't have forced a refresh", index.get().forcedRefresh());
+        assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get(), "1");
+    }
+
+    private void assertBulkSuccess(BulkResponse response) {
+        assertNoFailures(response);
+        for (BulkItemResponse item : response) {
+            assertFalse("request shouldn't have forced a refresh", item.getResponse().forcedRefresh());
+        }
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return singleton(DeletePlzPlugin.class);
+    }
+
+    public static class DeletePlzPlugin extends Plugin {
+        @Override
+        public String name() {
+            return "delete_please";
+        }
+
+        @Override
+        public String description() {
+            return "adds a script that converts any update into a delete for testing";
+        }
+
+        public void onModule(ScriptModule scriptModule) {
+            scriptModule.registerScript("delete_plz", DeletePlzFactory.class);
+        }
+    }
+
+    public static class DeletePlzFactory implements NativeScriptFactory {
+        @Override
+        public ExecutableScript newScript(Map<String, Object> params) {
+            return new ExecutableScript() {
+                private Map<String, Object> ctx;
+
+                @Override
+                @SuppressWarnings("unchecked")  // Elasicsearch convention
+                public void setNextVar(String name, Object value) {
+                    if (name.equals("ctx")) {
+                        ctx = (Map<String, Object>) value;
+                    }
+                }
+
+                @Override
+                public Object run() {
+                    ctx.put("op", "delete");
+                    return null;
+                }
+            };
+        }
+
+        @Override
+        public boolean needsScores() {
+            return false;
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 51df3ee0386..b7da3860003 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -170,6 +170,8 @@ public class InternalEngineTests extends ESTestCase {
                 .put(IndexSettings.INDEX_GC_DELETES_SETTING, "1h") // make sure this doesn't kick in on us
                 .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), codecName)
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD,
+                        between(10, 10 * IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD.get(Settings.EMPTY)))
                 .build()); // TODO randomize more settings
         threadPool = new ThreadPool(getClass().getName());
         store = createStore();
@@ -200,7 +202,7 @@ public class InternalEngineTests extends ESTestCase {
         return new EngineConfig(openMode, config.getShardId(), config.getThreadPool(), config.getIndexSettings(), config.getWarmer(),
             config.getStore(), config.getDeletionPolicy(), config.getMergePolicy(), config.getAnalyzer(), config.getSimilarity(),
             new CodecService(null, logger), config.getEventListener(), config.getTranslogRecoveryPerformer(), config.getQueryCache(),
-            config.getQueryCachingPolicy(), config.getTranslogConfig(), config.getFlushMergesAfter());
+            config.getQueryCachingPolicy(), config.getTranslogConfig(), config.getFlushMergesAfter(), config.getRefreshListeners());
     }
 
     @Override
@@ -291,14 +293,16 @@ public class InternalEngineTests extends ESTestCase {
         } catch (IOException e) {
             throw new ElasticsearchException("can't find index?", e);
         }
-        EngineConfig config = new EngineConfig(openMode, shardId, threadPool, indexSettings
-                , null, store, createSnapshotDeletionPolicy(), mergePolicy,
-                iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), new Engine.EventListener() {
+        Engine.EventListener listener = new Engine.EventListener() {
             @Override
             public void onFailedEngine(String reason, @Nullable Throwable t) {
                 // we don't need to notify anybody in this test
             }
-        }, new TranslogHandler(shardId.getIndexName(), logger), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5));
+        };
+        EngineConfig config = new EngineConfig(openMode, shardId, threadPool, indexSettings, null, store, createSnapshotDeletionPolicy(),
+                mergePolicy, iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), listener,
+                new TranslogHandler(shardId.getIndexName(), logger), IndexSearcher.getDefaultQueryCache(),
+                IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5), null);
 
         return config;
     }
@@ -309,18 +313,20 @@ public class InternalEngineTests extends ESTestCase {
 
     public void testSegments() throws Exception {
         try (Store store = createStore();
-             Engine engine = createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE)) {
+            Engine engine = createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE)) {
             List<Segment> segments = engine.segments(false);
             assertThat(segments.isEmpty(), equalTo(true));
             assertThat(engine.segmentsStats(false).getCount(), equalTo(0L));
             assertThat(engine.segmentsStats(false).getMemoryInBytes(), equalTo(0L));
 
-            // create a doc and refresh
+            // create two docs and refresh
             ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null);
-            engine.index(new Engine.Index(newUid("1"), doc));
-
+            Engine.Index first = new Engine.Index(newUid("1"), doc);
+            engine.index(first);
             ParsedDocument doc2 = testParsedDocument("2", "2", "test", null, -1, -1, testDocumentWithTextField(), B_2, null);
-            engine.index(new Engine.Index(newUid("2"), doc2));
+            Engine.Index second = new Engine.Index(newUid("2"), doc2);
+            engine.index(second);
+            assertThat(second.getTranslogLocation(), greaterThan(first.getTranslogLocation()));
             engine.refresh("test");
 
             segments = engine.segments(false);
@@ -2064,10 +2070,11 @@ public class InternalEngineTests extends ESTestCase {
         /* create a TranslogConfig that has been created with a different UUID */
         TranslogConfig translogConfig = new TranslogConfig(shardId, translog.location(), config.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE);
 
-        EngineConfig brokenConfig = new EngineConfig(EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG, shardId, threadPool, config.getIndexSettings()
-                , null, store, createSnapshotDeletionPolicy(), newMergePolicy(),
-                config.getAnalyzer(), config.getSimilarity(), new CodecService(null, logger), config.getEventListener()
-                , config.getTranslogRecoveryPerformer(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5));
+        EngineConfig brokenConfig = new EngineConfig(EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG, shardId, threadPool,
+                config.getIndexSettings(), null, store, createSnapshotDeletionPolicy(), newMergePolicy(), config.getAnalyzer(),
+                config.getSimilarity(), new CodecService(null, logger), config.getEventListener(), config.getTranslogRecoveryPerformer(),
+                IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig,
+                TimeValue.timeValueMinutes(5), config.getRefreshListeners());
 
         try {
             InternalEngine internalEngine = new InternalEngine(brokenConfig);
diff --git a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
index f71f2d72019..bed597b7cc5 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
@@ -55,6 +55,7 @@ import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
 import org.elasticsearch.index.mapper.internal.UidFieldMapper;
+import org.elasticsearch.index.shard.RefreshListeners;
 import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardUtils;
@@ -213,7 +214,7 @@ public class ShadowEngineTests extends ESTestCase {
     }
 
     protected ShadowEngine createShadowEngine(IndexSettings indexSettings, Store store) {
-        return new ShadowEngine(config(indexSettings, store, null, null));
+        return new ShadowEngine(config(indexSettings, store, null, null, null));
     }
 
     protected InternalEngine createInternalEngine(IndexSettings indexSettings, Store store, Path translogPath) {
@@ -221,11 +222,12 @@ public class ShadowEngineTests extends ESTestCase {
     }
 
     protected InternalEngine createInternalEngine(IndexSettings indexSettings, Store store, Path translogPath, MergePolicy mergePolicy) {
-        EngineConfig config = config(indexSettings, store, translogPath, mergePolicy);
+        EngineConfig config = config(indexSettings, store, translogPath, mergePolicy, null);
         return new InternalEngine(config);
     }
 
-    public EngineConfig config(IndexSettings indexSettings, Store store, Path translogPath, MergePolicy mergePolicy) {
+    public EngineConfig config(IndexSettings indexSettings, Store store, Path translogPath, MergePolicy mergePolicy,
+            RefreshListeners refreshListeners) {
         IndexWriterConfig iwc = newIndexWriterConfig();
         final EngineConfig.OpenMode openMode;
         try {
@@ -237,14 +239,17 @@ public class ShadowEngineTests extends ESTestCase {
         } catch (IOException e) {
             throw new ElasticsearchException("can't find index?", e);
         }
-        TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, indexSettings, BigArrays.NON_RECYCLING_INSTANCE);
-        EngineConfig config = new EngineConfig(openMode, shardId, threadPool, indexSettings
-                , null, store, createSnapshotDeletionPolicy(), mergePolicy,
-                iwc.getAnalyzer(), iwc.getSimilarity() , new CodecService(null, logger), new Engine.EventListener() {
+        Engine.EventListener eventListener = new Engine.EventListener() {
             @Override
             public void onFailedEngine(String reason, @Nullable Throwable t) {
                 // we don't need to notify anybody in this test
-        }}, null, IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5));
+            }
+        };
+        TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, indexSettings, BigArrays.NON_RECYCLING_INSTANCE);
+        EngineConfig config = new EngineConfig(openMode, shardId, threadPool, indexSettings, null, store, createSnapshotDeletionPolicy(),
+                mergePolicy, iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), eventListener, null,
+                IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig,
+                TimeValue.timeValueMinutes(5), refreshListeners);
 
         return config;
     }
@@ -1011,4 +1016,11 @@ public class ShadowEngineTests extends ESTestCase {
         assertEquals(0, docStats.getDeleted());
         primaryEngine.forceMerge(randomBoolean(), 1, false, false, false);
     }
+
+    public void testRefreshListenersFails() throws IOException {
+        EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(),
+                new RefreshListeners(null, null, null, logger));
+        Exception e = expectThrows(IllegalArgumentException.class, () -> new ShadowEngine(config));
+        assertEquals("ShadowEngine doesn't support RefreshListeners", e.getMessage());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/all/AllFieldMapperPositionIncrementGapTests.java b/core/src/test/java/org/elasticsearch/index/mapper/all/AllFieldMapperPositionIncrementGapTests.java
index 702c9b85da4..7b106863341 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/all/AllFieldMapperPositionIncrementGapTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/all/AllFieldMapperPositionIncrementGapTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.mapper.all;
 
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
 import org.elasticsearch.plugins.Plugin;
@@ -87,7 +88,7 @@ public class AllFieldMapperPositionIncrementGapTests extends ESSingleNodeTestCas
     private static void testGap(Client client, String indexName,
                                 String type, int positionIncrementGap) throws IOException {
         client.prepareIndex(indexName, type, "position_gap_test")
-            .setSource("string1", "one", "string2", "two three").setRefresh(true).get();
+            .setSource("string1", "one", "string2", "two three").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
 
         // Baseline - phrase query finds matches in the same field value
         assertHitCount(client.prepareSearch(indexName)
diff --git a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
new file mode 100644
index 00000000000..5c83e6805ab
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.shard;
+
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy;
+import org.apache.lucene.index.SnapshotDeletionPolicy;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.lucene.uid.Versions;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.concurrent.FutureUtils;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.codec.CodecService;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.engine.EngineConfig;
+import org.elasticsearch.index.engine.InternalEngine;
+import org.elasticsearch.index.engine.InternalEngineTests.TranslogHandler;
+import org.elasticsearch.index.fieldvisitor.SingleFieldsVisitor;
+import org.elasticsearch.index.mapper.ParseContext.Document;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.mapper.internal.UidFieldMapper;
+import org.elasticsearch.index.store.DirectoryService;
+import org.elasticsearch.index.store.Store;
+import org.elasticsearch.index.translog.TranslogConfig;
+import org.elasticsearch.test.DummyShardLock;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+
+import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
+
+/**
+ * Tests how {@linkplain RefreshListeners} interacts with {@linkplain InternalEngine}.
+ */
+public class RefreshListenersTests extends ESTestCase {
+    private RefreshListeners listeners;
+    private Engine engine;
+    private volatile int maxListeners;
+    private ThreadPool threadPool;
+    private Store store;
+
+    @Before
+    public void setupListeners() throws Exception {
+        // Setup dependencies of the listeners
+        maxListeners = randomIntBetween(1, 1000);
+        listeners = new RefreshListeners(
+                () -> maxListeners,
+                () -> engine.refresh("too-many-listeners"),
+                // Immediately run listeners rather than adding them to the listener thread pool like IndexShard does to simplify the test.
+                Runnable::run,
+                logger
+                );
+
+        // Now setup the InternalEngine which is much more complicated because we aren't mocking anything
+        threadPool = new ThreadPool(getTestName());
+        IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("index", Settings.EMPTY);
+        ShardId shardId = new ShardId(new Index("index", "_na_"), 1);
+        Directory directory = newDirectory();
+        DirectoryService directoryService = new DirectoryService(shardId, indexSettings) {
+            @Override
+            public Directory newDirectory() throws IOException {
+                return directory;
+            }
+
+            @Override
+            public long throttleTimeInNanos() {
+                return 0;
+            }
+        };
+        store = new Store(shardId, indexSettings, directoryService, new DummyShardLock(shardId));
+        IndexWriterConfig iwc = newIndexWriterConfig();
+        TranslogConfig translogConfig = new TranslogConfig(shardId, createTempDir("translog"), indexSettings,
+                BigArrays.NON_RECYCLING_INSTANCE);
+        Engine.EventListener eventListener = new Engine.EventListener() {
+            @Override
+            public void onFailedEngine(String reason, @Nullable Throwable t) {
+                // we don't need to notify anybody in this test
+            }
+        };
+        EngineConfig config = new EngineConfig(EngineConfig.OpenMode.CREATE_INDEX_AND_TRANSLOG, shardId, threadPool, indexSettings, null,
+                store, new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()), newMergePolicy(), iwc.getAnalyzer(),
+                iwc.getSimilarity(), new CodecService(null, logger), eventListener, new TranslogHandler(shardId.getIndexName(), logger),
+                IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig,
+                TimeValue.timeValueMinutes(5), listeners);
+        engine = new InternalEngine(config);
+    }
+
+    @After
+    public void tearDownListeners() throws Exception {
+        IOUtils.close(engine, store);
+        terminate(threadPool);
+    }
+
+    public void testTooMany() throws Exception {
+        assertFalse(listeners.refreshNeeded());
+        Engine.Index index = index("1");
+
+        // Fill the listener slots
+        List<DummyRefreshListener> nonForcedListeners = new ArrayList<>(maxListeners);
+        for (int i = 0; i < maxListeners; i++) {
+            DummyRefreshListener listener = new DummyRefreshListener();
+            nonForcedListeners.add(listener);
+            listeners.addOrNotify(index.getTranslogLocation(), listener);
+            assertTrue(listeners.refreshNeeded());
+        }
+
+        // We shouldn't have called any of them
+        for (DummyRefreshListener listener : nonForcedListeners) {
+            assertNull("Called listener too early!", listener.forcedRefresh.get());
+        }
+
+        // Add one more listener which should cause a refresh.
+        DummyRefreshListener forcingListener = new DummyRefreshListener();
+        listeners.addOrNotify(index.getTranslogLocation(), forcingListener);
+        assertTrue("Forced listener wasn't forced?", forcingListener.forcedRefresh.get());
+
+        // That forces all the listeners through. It would be on the listener ThreadPool but we've made all of those execute immediately.
+        for (DummyRefreshListener listener : nonForcedListeners) {
+            assertEquals("Expected listener called with unforced refresh!", Boolean.FALSE, listener.forcedRefresh.get());
+        }
+        assertFalse(listeners.refreshNeeded());
+    }
+
+    public void testAfterRefresh() throws Exception {
+        Engine.Index index = index("1");
+        engine.refresh("I said so");
+        if (randomBoolean()) {
+            index(randomFrom("1" /* same document */, "2" /* different document */));
+            if (randomBoolean()) {
+                engine.refresh("I said so");
+            }
+        }
+
+        DummyRefreshListener listener = new DummyRefreshListener();
+        listeners.addOrNotify(index.getTranslogLocation(), listener);
+        assertFalse(listener.forcedRefresh.get());
+    }
+
+    /**
+     * Attempts to add a listener at the same time as a refresh occurs by having a background thread force a refresh as fast as it can while
+     * adding listeners. This can catch the situation where a refresh happens right as the listener is being added such that the listener
+     * misses the refresh and has to catch the next one. If the listener wasn't able to properly catch the next one then this would fail.
+     */
+    public void testConcurrentRefresh() throws Exception {
+        AtomicBoolean run = new AtomicBoolean(true);
+        Thread refresher = new Thread(() -> {
+            while (run.get()) {
+                engine.refresh("test");
+            }
+        });
+        refresher.start();
+        try {
+            for (int i = 0; i < 100; i++) {
+                Engine.Index index = index("1");
+
+                DummyRefreshListener listener = new DummyRefreshListener();
+                listeners.addOrNotify(index.getTranslogLocation(), listener);
+                assertBusy(() -> assertNotNull(listener.forcedRefresh.get()));
+                assertFalse(listener.forcedRefresh.get());
+            }
+        } finally {
+            run.set(false);
+            refresher.join();
+        }
+    }
+
+    /**
+     * Uses a bunch of threads to index, wait for refresh, and non-realtime get documents to validate that they are visible after waiting
+     * regardless of what crazy sequence of events causes the refresh listener to fire.
+     */
+    public void testLotsOfThreads() throws Exception {
+        int threadCount = between(3, 10);
+        maxListeners = between(1, threadCount * 2);
+
+        // This thread just refreshes every once in a while to cause trouble.
+        ScheduledFuture<?> refresher = threadPool.scheduleWithFixedDelay(() -> engine.refresh("because test"), timeValueMillis(100));
+
+        // These threads add and block until the refresh makes the change visible and then do a non-realtime get.
+        Thread[] indexers = new Thread[threadCount];
+        for (int thread = 0; thread < threadCount; thread++) {
+            final String threadId = String.format(Locale.ROOT, "%04d", thread);
+            indexers[thread] = new Thread(() -> {
+                for (int iteration = 1; iteration <= 50; iteration++) {
+                    try {
+                        String testFieldValue = String.format(Locale.ROOT, "%s%04d", threadId, iteration);
+                        Engine.Index index = index(threadId, testFieldValue);
+                        assertEquals(iteration, index.version());
+
+                        DummyRefreshListener listener = new DummyRefreshListener();
+                        listeners.addOrNotify(index.getTranslogLocation(), listener);
+                        assertBusy(() -> assertNotNull("listener never called", listener.forcedRefresh.get()));
+                        if (threadCount < maxListeners) {
+                            assertFalse(listener.forcedRefresh.get());
+                        }
+
+                        Engine.Get get = new Engine.Get(false, index.uid());
+                        try (Engine.GetResult getResult = engine.get(get)) {
+                            assertTrue("document not found", getResult.exists());
+                            assertEquals(iteration, getResult.version());
+                            SingleFieldsVisitor visitor = new SingleFieldsVisitor("test");
+                            getResult.docIdAndVersion().context.reader().document(getResult.docIdAndVersion().docId, visitor);
+                            assertEquals(Arrays.asList(testFieldValue), visitor.fields().get("test"));
+                        }
+                    } catch (Throwable t) {
+                        throw new RuntimeException("failure on the [" + iteration + "] iteration of thread [" + threadId + "]", t);
+                    }
+                }
+            });
+            indexers[thread].start();
+        }
+
+        for (Thread indexer: indexers) {
+            indexer.join();
+        }
+        FutureUtils.cancel(refresher);
+    }
+
+    private Engine.Index index(String id) {
+        return index(id, "test");
+    }
+
+    private Engine.Index index(String id, String testFieldValue) {
+        String type = "test";
+        String uid = type + ":" + id;
+        Document document = new Document();
+        document.add(new TextField("test", testFieldValue, Field.Store.YES));
+        Field uidField = new Field("_uid", type + ":" + id, UidFieldMapper.Defaults.FIELD_TYPE);
+        Field versionField = new NumericDocValuesField("_version", Versions.MATCH_ANY);
+        document.add(uidField);
+        document.add(versionField);
+        BytesReference source = new BytesArray(new byte[] { 1 });
+        ParsedDocument doc = new ParsedDocument(versionField, id, type, null, -1, -1, Arrays.asList(document), source, null); 
+        Engine.Index index = new Engine.Index(new Term("_uid", uid), doc);
+        engine.index(index);
+        return index;
+    }
+
+    private static class DummyRefreshListener implements Consumer<Boolean> {
+        /**
+         * When the listener is called this captures it's only argument.
+         */
+        private AtomicReference<Boolean> forcedRefresh = new AtomicReference<>();
+
+        @Override
+        public void accept(Boolean forcedRefresh) {
+            assertNotNull(forcedRefresh);
+            Boolean oldValue = this.forcedRefresh.getAndSet(forcedRefresh);
+            assertNull("Listener called twice", oldValue);
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
index b4d2423921c..889897e21f6 100644
--- a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
+++ b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.translog;
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+
 import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.mockfile.FilterFileChannel;
@@ -44,6 +45,7 @@ import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.translog.Translog.Location;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.hamcrest.Matchers;
@@ -204,18 +206,40 @@ public class TranslogTests extends ESTestCase {
     }
 
     public void testRead() throws IOException {
+        Location loc0 = translog.getLastWriteLocation();
+        assertNotNull(loc0);
+
         Translog.Location loc1 = translog.add(new Translog.Index("test", "1", new byte[]{1}));
+        assertThat(loc1, greaterThan(loc0));
+        assertThat(translog.getLastWriteLocation(), greaterThan(loc1));
         Translog.Location loc2 = translog.add(new Translog.Index("test", "2", new byte[]{2}));
+        assertThat(loc2, greaterThan(loc1));
+        assertThat(translog.getLastWriteLocation(), greaterThan(loc2));
         assertThat(translog.read(loc1).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{1})));
         assertThat(translog.read(loc2).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{2})));
+
+        Translog.Location lastLocBeforeSync = translog.getLastWriteLocation();
         translog.sync();
+        assertEquals(lastLocBeforeSync, translog.getLastWriteLocation());
         assertThat(translog.read(loc1).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{1})));
         assertThat(translog.read(loc2).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{2})));
+
         Translog.Location loc3 = translog.add(new Translog.Index("test", "2", new byte[]{3}));
+        assertThat(loc3, greaterThan(loc2));
+        assertThat(translog.getLastWriteLocation(), greaterThan(loc3));
         assertThat(translog.read(loc3).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{3})));
+
+        lastLocBeforeSync = translog.getLastWriteLocation();
         translog.sync();
+        assertEquals(lastLocBeforeSync, translog.getLastWriteLocation());
         assertThat(translog.read(loc3).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{3})));
         translog.prepareCommit();
+        /*
+         * The commit adds to the lastWriteLocation even though is isn't really a write. This is just an implementation artifact but it can
+         * safely be ignored because the lastWriteLocation continues to be greater than the Location returned from the last write operation
+         * and less than the location of the next write operation.
+         */
+        assertThat(translog.getLastWriteLocation(), greaterThan(lastLocBeforeSync));
         assertThat(translog.read(loc3).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{3})));
         translog.commit();
         assertNull(translog.read(loc1));
diff --git a/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java
index 6a4de4b9ff2..6a4d965706a 100644
--- a/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java
+++ b/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java
@@ -21,6 +21,7 @@ package org.elasticsearch.routing;
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -45,7 +46,7 @@ public class AliasRoutingIT extends ESIntegTestCase {
         assertAcked(admin().indices().prepareAliases().addAliasAction(newAddAliasAction("test", "alias0").routing("0")));
 
         logger.info("--> indexing with id [1], and routing [0] using alias");
-        client().prepareIndex("alias0", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet();
+        client().prepareIndex("alias0", "type1", "1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
         logger.info("--> verifying get with no routing, should not find anything");
         for (int i = 0; i < 5; i++) {
             assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false));
@@ -72,7 +73,7 @@ public class AliasRoutingIT extends ESIntegTestCase {
 
 
         logger.info("--> deleting with no routing, should not delete anything");
-        client().prepareDelete("test", "type1", "1").setRefresh(true).execute().actionGet();
+        client().prepareDelete("test", "type1", "1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
         for (int i = 0; i < 5; i++) {
             assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false));
             assertThat(client().prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true));
@@ -80,7 +81,7 @@ public class AliasRoutingIT extends ESIntegTestCase {
         }
 
         logger.info("--> deleting with routing alias, should delete");
-        client().prepareDelete("alias0", "type1", "1").setRefresh(true).execute().actionGet();
+        client().prepareDelete("alias0", "type1", "1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
         for (int i = 0; i < 5; i++) {
             assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false));
             assertThat(client().prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(false));
@@ -88,7 +89,7 @@ public class AliasRoutingIT extends ESIntegTestCase {
         }
 
         logger.info("--> indexing with id [1], and routing [0] using alias");
-        client().prepareIndex("alias0", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet();
+        client().prepareIndex("alias0", "type1", "1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
         logger.info("--> verifying get with no routing, should not find anything");
         for (int i = 0; i < 5; i++) {
             assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false));
@@ -110,7 +111,7 @@ public class AliasRoutingIT extends ESIntegTestCase {
                 .addAliasAction(newAddAliasAction("test", "alias01").searchRouting("0,1")));
 
         logger.info("--> indexing with id [1], and routing [0] using alias");
-        client().prepareIndex("alias0", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet();
+        client().prepareIndex("alias0", "type1", "1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
         logger.info("--> verifying get with no routing, should not find anything");
         for (int i = 0; i < 5; i++) {
             assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false));
@@ -142,7 +143,7 @@ public class AliasRoutingIT extends ESIntegTestCase {
         }
 
         logger.info("--> indexing with id [2], and routing [1] using alias");
-        client().prepareIndex("alias1", "type1", "2").setSource("field", "value1").setRefresh(true).execute().actionGet();
+        client().prepareIndex("alias1", "type1", "2").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
 
         logger.info("--> search with no routing, should fine two");
         for (int i = 0; i < 5; i++) {
@@ -207,7 +208,7 @@ public class AliasRoutingIT extends ESIntegTestCase {
                 .addAliasAction(newAddAliasAction("test-b", "alias-ab").searchRouting("1")));
         ensureGreen(); // wait for events again to make sure we got the aliases on all nodes
         logger.info("--> indexing with id [1], and routing [0] using alias to test-a");
-        client().prepareIndex("alias-a0", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet();
+        client().prepareIndex("alias-a0", "type1", "1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
         logger.info("--> verifying get with no routing, should not find anything");
         for (int i = 0; i < 5; i++) {
             assertThat(client().prepareGet("test-a", "type1", "1").execute().actionGet().isExists(), equalTo(false));
@@ -218,7 +219,7 @@ public class AliasRoutingIT extends ESIntegTestCase {
         }
 
         logger.info("--> indexing with id [0], and routing [1] using alias to test-b");
-        client().prepareIndex("alias-b1", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet();
+        client().prepareIndex("alias-b1", "type1", "1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
         logger.info("--> verifying get with no routing, should not find anything");
         for (int i = 0; i < 5; i++) {
             assertThat(client().prepareGet("test-a", "type1", "1").execute().actionGet().isExists(), equalTo(false));
@@ -261,9 +262,9 @@ public class AliasRoutingIT extends ESIntegTestCase {
                 .addAliasAction(newAddAliasAction("index", "index_1").routing("1")));
 
         logger.info("--> indexing on index_1 which is an alias for index with routing [1]");
-        client().prepareIndex("index_1", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet();
+        client().prepareIndex("index_1", "type1", "1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
         logger.info("--> indexing on index_2 which is a concrete index");
-        client().prepareIndex("index_2", "type2", "2").setSource("field", "value2").setRefresh(true).execute().actionGet();
+        client().prepareIndex("index_2", "type2", "2").setSource("field", "value2").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
 
 
         logger.info("--> search all on index_* should find two");
@@ -286,9 +287,9 @@ public class AliasRoutingIT extends ESIntegTestCase {
                 .addAliasAction(newAddAliasAction("index", "index_1").routing("1")));
 
         logger.info("--> indexing on index_1 which is an alias for index with routing [1]");
-        client().prepareIndex("index_1", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet();
+        client().prepareIndex("index_1", "type1", "1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
         logger.info("--> indexing on index_2 which is a concrete index");
-        client().prepareIndex("index_2", "type2", "2").setSource("field", "value2").setRefresh(true).execute().actionGet();
+        client().prepareIndex("index_2", "type2", "2").setSource("field", "value2").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
 
         SearchResponse searchResponse = client().prepareSearch("index_*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(1).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet();
 
@@ -307,7 +308,7 @@ public class AliasRoutingIT extends ESIntegTestCase {
                 .addAliasAction(newAddAliasAction("test", "alias").routing("3")));
 
         logger.info("--> indexing with id [0], and routing [3]");
-        client().prepareIndex("alias", "type1", "0").setSource("field", "value1").setRefresh(true).execute().actionGet();
+        client().prepareIndex("alias", "type1", "0").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
         logger.info("--> verifying get with no routing, should not find anything");
 
         logger.info("--> verifying get and search with routing, should find");
@@ -332,7 +333,7 @@ public class AliasRoutingIT extends ESIntegTestCase {
                 .addAliasAction(newAddAliasAction("test", "alias").searchRouting("3,4").indexRouting("4")));
 
         logger.info("--> indexing with id [1], and routing [4]");
-        client().prepareIndex("alias", "type1", "1").setSource("field", "value2").setRefresh(true).execute().actionGet();
+        client().prepareIndex("alias", "type1", "1").setSource("field", "value2").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
         logger.info("--> verifying get with no routing, should not find anything");
 
         logger.info("--> verifying get and search with routing, should find");
diff --git a/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java
index 03e6cbf9ef1..d8cf1e7b5ec 100644
--- a/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java
+++ b/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java
@@ -28,6 +28,7 @@ import org.elasticsearch.action.explain.ExplainResponse;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.get.MultiGetRequest;
 import org.elasticsearch.action.get.MultiGetResponse;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.action.termvectors.MultiTermVectorsResponse;
 import org.elasticsearch.action.termvectors.TermVectorsRequest;
 import org.elasticsearch.action.termvectors.TermVectorsResponse;
@@ -56,7 +57,8 @@ public class SimpleRoutingIT extends ESIntegTestCase {
         ensureGreen();
 
         logger.info("--> indexing with id [1], and routing [0]");
-        client().prepareIndex("test", "type1", "1").setRouting("0").setSource("field", "value1").setRefresh(true).execute().actionGet();
+        client().prepareIndex("test", "type1", "1").setRouting("0").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+                .get();
         logger.info("--> verifying get with no routing, should not find anything");
         for (int i = 0; i < 5; i++) {
             assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false));
@@ -67,21 +69,22 @@ public class SimpleRoutingIT extends ESIntegTestCase {
         }
 
         logger.info("--> deleting with no routing, should not delete anything");
-        client().prepareDelete("test", "type1", "1").setRefresh(true).execute().actionGet();
+        client().prepareDelete("test", "type1", "1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
         for (int i = 0; i < 5; i++) {
             assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false));
             assertThat(client().prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true));
         }
 
         logger.info("--> deleting with routing, should delete");
-        client().prepareDelete("test", "type1", "1").setRouting("0").setRefresh(true).execute().actionGet();
+        client().prepareDelete("test", "type1", "1").setRouting("0").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
         for (int i = 0; i < 5; i++) {
             assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false));
             assertThat(client().prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(false));
         }
 
         logger.info("--> indexing with id [1], and routing [0]");
-        client().prepareIndex("test", "type1", "1").setRouting("0").setSource("field", "value1").setRefresh(true).execute().actionGet();
+        client().prepareIndex("test", "type1", "1").setRouting("0").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+                .get();
         logger.info("--> verifying get with no routing, should not find anything");
         for (int i = 0; i < 5; i++) {
             assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false));
@@ -97,7 +100,8 @@ public class SimpleRoutingIT extends ESIntegTestCase {
         ensureGreen();
 
         logger.info("--> indexing with id [1], and routing [0]");
-        client().prepareIndex("test", "type1", "1").setRouting("0").setSource("field", "value1").setRefresh(true).execute().actionGet();
+        client().prepareIndex("test", "type1", "1").setRouting("0").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+                .get();
         logger.info("--> verifying get with no routing, should not find anything");
         for (int i = 0; i < 5; i++) {
             assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false));
@@ -125,7 +129,7 @@ public class SimpleRoutingIT extends ESIntegTestCase {
         }
 
         logger.info("--> indexing with id [2], and routing [1]");
-        client().prepareIndex("test", "type1", "2").setRouting("1").setSource("field", "value1").setRefresh(true).execute().actionGet();
+        client().prepareIndex("test", "type1", "2").setRouting("1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
 
         logger.info("--> search with no routing, should fine two");
         for (int i = 0; i < 5; i++) {
@@ -165,12 +169,13 @@ public class SimpleRoutingIT extends ESIntegTestCase {
         ensureGreen();
 
         logger.info("--> indexing with id [1], and routing [0]");
-        client().prepareIndex(indexOrAlias(), "type1", "1").setRouting("0").setSource("field", "value1").setRefresh(true).execute().actionGet();
+        client().prepareIndex(indexOrAlias(), "type1", "1").setRouting("0").setSource("field", "value1")
+                .setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
         logger.info("--> verifying get with no routing, should fail");
 
         logger.info("--> indexing with id [1], with no routing, should fail");
         try {
-            client().prepareIndex(indexOrAlias(), "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet();
+            client().prepareIndex(indexOrAlias(), "type1", "1").setSource("field", "value1").get();
             fail("index with missing routing when routing is required should fail");
         } catch (ElasticsearchException e) {
             assertThat(e.unwrapCause(), instanceOf(RoutingMissingException.class));
@@ -183,7 +188,7 @@ public class SimpleRoutingIT extends ESIntegTestCase {
 
         logger.info("--> deleting with no routing, should fail");
         try {
-            client().prepareDelete(indexOrAlias(), "type1", "1").setRefresh(true).execute().actionGet();
+            client().prepareDelete(indexOrAlias(), "type1", "1").get();
             fail("delete with missing routing when routing is required should fail");
         } catch (ElasticsearchException e) {
             assertThat(e.unwrapCause(), instanceOf(RoutingMissingException.class));
@@ -223,7 +228,7 @@ public class SimpleRoutingIT extends ESIntegTestCase {
             assertThat(getResponse.getSourceAsMap().get("field"), equalTo("value2"));
         }
 
-        client().prepareDelete(indexOrAlias(), "type1", "1").setRouting("0").setRefresh(true).execute().actionGet();
+        client().prepareDelete(indexOrAlias(), "type1", "1").setRouting("0").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
 
         for (int i = 0; i < 5; i++) {
             try {
@@ -320,7 +325,8 @@ public class SimpleRoutingIT extends ESIntegTestCase {
         logger.info("--> indexing with id [1], and routing [0]");
         client().prepareIndex(indexOrAlias(), "type1", "1").setRouting("0").setSource("field", "value1").get();
         logger.info("--> indexing with id [2], and routing [0]");
-        client().prepareIndex(indexOrAlias(), "type1", "2").setRouting("0").setSource("field", "value2").setRefresh(true).get();
+        client().prepareIndex(indexOrAlias(), "type1", "2").setRouting("0").setSource("field", "value2")
+                .setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
 
         logger.info("--> verifying get with id [1] with routing [0], should succeed");
         assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true));
diff --git a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
index 90f5c65c066..f8ca1e1aaf7 100644
--- a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
@@ -26,6 +26,7 @@ import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.common.lucene.search.function.CombineFunction;
 import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery;
 import org.elasticsearch.common.settings.Settings;
@@ -754,10 +755,11 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         assertNoFailures(response);
         assertThat(response.getHits().totalHits(), equalTo(0L));
 
-        client().prepareIndex("test", "child1").setSource(jsonBuilder().startObject().field("text", "value").endObject()).setRefresh(true)
-                .get();
+        client().prepareIndex("test", "child1").setSource(jsonBuilder().startObject().field("text", "value").endObject())
+                .setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
 
-        response = client().prepareSearch("test").setQuery(QueryBuilders.hasChildQuery("child", matchQuery("text", "value"), ScoreMode.None)).get();
+        response = client().prepareSearch("test")
+                .setQuery(QueryBuilders.hasChildQuery("child", matchQuery("text", "value"), ScoreMode.None)).get();
         assertNoFailures(response);
         assertThat(response.getHits().totalHits(), equalTo(0L));
 
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearch2xIT.java b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearch2xIT.java
index 812e43918d9..484edf61f7b 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearch2xIT.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearch2xIT.java
@@ -31,6 +31,7 @@ import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.Fuzziness;
@@ -473,7 +474,7 @@ public class CompletionSuggestSearch2xIT extends ESIntegTestCase {
             .setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id))
             .addMapping(TYPE, mapping));
         client().prepareIndex(INDEX, TYPE, "1")
-            .setRefresh(true)
+            .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
             .setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()).get();
         ensureGreen(INDEX);
 
@@ -496,7 +497,7 @@ public class CompletionSuggestSearch2xIT extends ESIntegTestCase {
         ).execute().actionGet();
         assertSuggestions(suggestResponse, "suggs");
 
-        client().prepareIndex(INDEX, TYPE, "1").setRefresh(true)
+        client().prepareIndex(INDEX, TYPE, "1").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
             .setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()).get();
         ensureGreen(INDEX);
 
@@ -522,7 +523,7 @@ public class CompletionSuggestSearch2xIT extends ESIntegTestCase {
             .addMapping(TYPE, mapping)
             .setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)));
         client().prepareIndex(INDEX, TYPE, "1")
-            .setRefresh(true)
+            .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
             .setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()).get();
         ensureGreen(INDEX);
 
@@ -545,7 +546,7 @@ public class CompletionSuggestSearch2xIT extends ESIntegTestCase {
         ).execute().actionGet();
         assertSuggestions(suggestResponse, "suggs");
 
-        client().prepareIndex(INDEX, TYPE, "1").setRefresh(true)
+        client().prepareIndex(INDEX, TYPE, "1").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
             .setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()).get();
         ensureGreen(INDEX);
 
@@ -731,10 +732,10 @@ public class CompletionSuggestSearch2xIT extends ESIntegTestCase {
         assertThat(putMappingResponse.isAcknowledged(), is(true));
 
         // Index two entities
-        client().prepareIndex(INDEX, TYPE, "1").setRefresh(true)
+        client().prepareIndex(INDEX, TYPE, "1").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
             .setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").field(otherField, "WHATEVER").endObject())
             .get();
-        client().prepareIndex(INDEX, TYPE, "2").setRefresh(true)
+        client().prepareIndex(INDEX, TYPE, "2").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
             .setSource(jsonBuilder().startObject().field(FIELD, "Bar Fighters").field(otherField, "WHATEVER2").endObject())
             .get();
 
@@ -1040,7 +1041,7 @@ public class CompletionSuggestSearch2xIT extends ESIntegTestCase {
                 .startArray("input").value(str).endArray()
                 .field("output", "foobar")
                 .endObject().endObject()
-            ).setRefresh(true).get();
+            ).setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
             // need to flush and refresh, because we keep changing the same document
             // we have to make sure that segments without any live documents are deleted
             flushAndRefresh();
@@ -1074,7 +1075,7 @@ public class CompletionSuggestSearch2xIT extends ESIntegTestCase {
             .startArray("input").value(longString).endArray()
             .field("output", "foobar")
             .endObject().endObject()
-        ).setRefresh(true).get();
+        ).setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
 
     }
 
@@ -1096,7 +1097,7 @@ public class CompletionSuggestSearch2xIT extends ESIntegTestCase {
                 .startArray("input").value(string).endArray()
                 .field("output", "foobar")
                 .endObject().endObject()
-            ).setRefresh(true).get();
+            ).setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
             fail("expected MapperParsingException");
         } catch (MapperParsingException expected) {}
     }
@@ -1116,7 +1117,7 @@ public class CompletionSuggestSearch2xIT extends ESIntegTestCase {
             .startObject()
             .field(FIELD, string)
             .endObject()
-        ).setRefresh(true).get();
+        ).setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
 
         try {
             client().prepareSearch(INDEX).addAggregation(AggregationBuilders.terms("suggest_agg").field(FIELD)
@@ -1148,11 +1149,11 @@ public class CompletionSuggestSearch2xIT extends ESIntegTestCase {
         ensureGreen();
 
         client().prepareIndex(INDEX, TYPE, "1").setSource(FIELD, "strings make me happy", FIELD + "_1", "nulls make me sad")
-            .setRefresh(true).get();
+        .setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
 
         try {
             client().prepareIndex(INDEX, TYPE, "2").setSource(FIELD, null, FIELD + "_1", "nulls make me sad")
-                .setRefresh(true).get();
+                .setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
             fail("Expected MapperParsingException for null value");
         } catch (MapperParsingException e) {
             // make sure that the exception has the name of the field causing the error
diff --git a/docs/reference/docs.asciidoc b/docs/reference/docs.asciidoc
index f3b30e7f0c3..04049663e84 100644
--- a/docs/reference/docs.asciidoc
+++ b/docs/reference/docs.asciidoc
@@ -15,6 +15,8 @@ This section describes the following CRUD APIs:
 .Multi-document APIs
 * <<docs-multi-get>>
 * <<docs-bulk>>
+* <<docs-update-by-query>>
+* <<docs-reindex>>
 
 NOTE: All CRUD APIs are single-index APIs. The `index` parameter accepts a single
 index name, or an `alias` which points to a single index.
@@ -42,3 +44,5 @@ include::docs/reindex.asciidoc[]
 include::docs/termvectors.asciidoc[]
 
 include::docs/multi-termvectors.asciidoc[]
+
+include::docs/refresh.asciidoc[]
diff --git a/docs/reference/docs/bulk.asciidoc b/docs/reference/docs/bulk.asciidoc
index ee8f75c6cdd..6b0a5b2e40d 100644
--- a/docs/reference/docs/bulk.asciidoc
+++ b/docs/reference/docs/bulk.asciidoc
@@ -167,12 +167,8 @@ are the same).
 [[bulk-refresh]]
 === Refresh
 
-The `refresh` parameter can be set to `true` in order to refresh the relevant
-primary and replica shards immediately after the bulk operation has occurred
-and make it searchable, instead of waiting for the normal refresh interval to
-expire. Setting it to `true` can trigger additional load, and may slow down
-indexing. Due to its costly nature, the `refresh` parameter is set on the bulk request level
-and is not supported on each individual bulk item.
+Control when the changes made by this request are visible to search. See
+<<docs-refresh>>.
 
 [float]
 [[bulk-update]]
diff --git a/docs/reference/docs/delete.asciidoc b/docs/reference/docs/delete.asciidoc
index 175c07d005e..18a370fc416 100644
--- a/docs/reference/docs/delete.asciidoc
+++ b/docs/reference/docs/delete.asciidoc
@@ -113,11 +113,9 @@ is the same).
 [[delete-refresh]]
 === Refresh
 
-The `refresh` parameter can be set to `true` in order to refresh the relevant
-primary and replica shards after the delete operation has occurred and make it
-searchable. Setting it to `true` should be done after careful thought and
-verification that this does not cause a heavy load on the system (and slows
-down indexing).
+Control when the changes made by this request are visible to search. See
+<<docs-refresh>>.
+
 
 [float]
 [[delete-timeout]]
diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc
index eb1e45d6160..aa62b65292e 100644
--- a/docs/reference/docs/index_.asciidoc
+++ b/docs/reference/docs/index_.asciidoc
@@ -30,7 +30,8 @@ The result of the above index operation is:
     "_type" : "tweet",
     "_id" : "1",
     "_version" : 1,
-    "created" : true
+    "created" : true,
+    "forced_refresh": false
 }
 --------------------------------------------------
 // TESTRESPONSE[s/"successful" : 2/"successful" : 1/]
@@ -221,7 +222,8 @@ The result of the above index operation is:
     "_type" : "tweet",
     "_id" : "6a8ca01c-7896-48e9-81cc-9f70661fcb32",
     "_version" : 1,
-    "created" : true
+    "created" : true,
+    "forced_refresh": false
 }
 --------------------------------------------------
 // TESTRESPONSE[s/6a8ca01c-7896-48e9-81cc-9f70661fcb32/$body._id/ s/"successful" : 2/"successful" : 1/]
@@ -385,13 +387,8 @@ replication group have indexed the document (sync replication).
 [[index-refresh]]
 === Refresh
 
-To refresh the shard (not the whole index) immediately after the operation
-occurs, so that the document appears in search results immediately, the
-`refresh` parameter can be set to `true`. Setting this option to `true` should
-*ONLY* be done after careful thought and verification that it does not lead to
-poor performance, both from an indexing and a search standpoint. Note, getting
-a document using the get API is completely realtime and doesn't require a
-refresh.
+Control when the changes made by this request are visible to search. See
+<<docs-refresh>>.
 
 [float]
 [[index-noop]]
diff --git a/docs/reference/docs/refresh.asciidoc b/docs/reference/docs/refresh.asciidoc
new file mode 100644
index 00000000000..3e5153341c8
--- /dev/null
+++ b/docs/reference/docs/refresh.asciidoc
@@ -0,0 +1,109 @@
+[[docs-refresh]]
+== `?refresh`
+
+The <<docs-index_,Index>>, <<docs-update,Update>>, <<docs-delete,Delete>>, and
+<<docs-bulk,Bulk>> APIs support setting `refresh` to control when changes made
+by this request are made visible to search. These are the allowed values:
+
+Empty string or `true`::
+
+Refresh the relevant primary and replica shards (not the whole index)
+immediately after the operation occurs, so that the updated document appears
+in search results immediately. This should *ONLY* be done after careful thought
+and verification that it does not lead to poor performance, both from an
+indexing and a search standpoint.
+
+`wait_for`::
+
+Wait for the changes made by the request to be made visible by a refresh before
+replying. This doesn't force an immediate refresh, rather, it waits for a
+refresh happen. Elasticsearch automatically refreshes shards that have changed
+every `index.refresh_interval` which defaults to one second. That setting is
+<<dynamic-index-settings,dynamic>>. The <<indices-refresh>> API will also
+cause the request to return, as will setting `refresh` to `true` on any of the
+APIs that support it.
+
+`false` (the default)::
+
+Take no refresh related actions. The changes made by this request will be made
+visible at some point after the request returns.
+
+=== Choosing which setting to use
+
+Unless you have a good reason to wait for the change to become visible always
+use `refresh=false`, or, because that is the default, just leave the `refresh`
+parameter out of the URL. That is the simplest and fastest choice.
+
+If you absolutely must have the changes made by a request visible synchronously
+with the request then you must get to pick between putting more load on
+Elasticsearch (`true`) and waiting longer for the response (`wait_for`). Here
+are a few points that should inform that decision:
+
+* The more changes being made to the index the more work `wait_for` saves
+compared to `true`. In the case that the index is only changed once every
+`index.refresh_interval` then it saves no work.
+* `true` creates less efficient indexes constructs (tiny segments) that must
+later be merged into more efficient index constructs (larger segments). Meaning
+that the cost of `true` is payed at index time to create the tiny segment, at
+search time to search the tiny segment, and at merge time to make the larger
+segments.
+* Never start multiple `refresh=wait_for` requests in a row. Instead batch them
+into a single bulk request with `refresh=wait_for` and Elasticsearch will start
+them all in parallel and return only when they have all finished.
+* If the refresh interval is set to `-1`, disabling the automatic refreshes,
+then requests with `refresh=wait_for` will wait indefinitely until some action
+causes a refresh. Conversely, setting `index.refresh_interval` to something
+shorter than the default like `200ms` will make `refresh=wait_for` come back
+faster, but it'll still generate inefficient segments.
+* `refresh=wait_for` only affects the request that it is on, but, by forcing a
+refresh immediately, `refresh=true` will affect other ongoing request. In
+general, if you have a running system you don't wish to disturb then
+`refresh=wait_for` is a smaller modification.
+
+=== `refresh=wait_for` Can Force a Refresh
+
+If a `refresh=wait_for` request comes in when there are already
+`index.max_refresh_listeners` (defaults to 1000) requests waiting for a refresh
+on that shard then that request will behave just as though it had `refresh` set
+to `true` instead: it will force a refresh. This keeps the promise that when a
+`refresh=wait_for` request returns that its changes are visible for search
+while preventing unchecked resource usage for blocked requests. If a request
+forced a refresh because it ran out of listener slots then its response will
+contain `"forced_refresh": true`.
+
+Bulk requests only take up one slot on each shard that they touch no matter how
+many times they modify the shard.
+
+=== Examples
+
+These will create a document and immediately refresh the index so it is visible:
+
+[source,js]
+--------------------------------------------------
+PUT /test/test/1?refresh
+{"test": "test"}
+PUT /test/test/2?refresh=true
+{"test": "test"}
+--------------------------------------------------
+// CONSOLE
+
+These will create a document without doing anything to make it visible for
+search:
+
+[source,js]
+--------------------------------------------------
+PUT /test/test/3
+{"test": "test"}
+PUT /test/test/4?refresh=true
+{"test": "test"}
+--------------------------------------------------
+// CONSOLE
+
+This will create a document and wait for it to become visible for search:
+
+[source,js]
+--------------------------------------------------
+PUT /test/test/4?refresh=wait_for
+{"test": "test"}
+--------------------------------------------------
+// CONSOLE
diff --git a/docs/reference/docs/update.asciidoc b/docs/reference/docs/update.asciidoc
index 405f9b0494b..39261c5d21f 100644
--- a/docs/reference/docs/update.asciidoc
+++ b/docs/reference/docs/update.asciidoc
@@ -235,9 +235,8 @@ The write consistency of the index/delete operation.
 
 `refresh`::
 
-Refresh the relevant primary and replica shards (not the whole index)
-immediately after the operation occurs, so that the updated document appears
-in search results  immediately.
+Control when the changes made by this request are visible to search. See
+<<docs-refresh>>.
 
 `fields`::
 
diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc
index c7119af4168..3cc95e5af9a 100644
--- a/docs/reference/index-modules.asciidoc
+++ b/docs/reference/index-modules.asciidoc
@@ -136,6 +136,11 @@ specific index module:
     experimental[] Disables the purge of <<mapping-ttl-field,expired docs>> on
     the current index.
 
+`index.max_refresh_listeners`::
+
+    Maximum number of refresh listeners available on each shard of the index.
+    These listeners are used to implement <<docs-refresh,`refresh=wait_for`>>.
+
 
 [float]
 === Settings in other index modules
diff --git a/docs/reference/migration/migrate_5_0/docs.asciidoc b/docs/reference/migration/migrate_5_0/docs.asciidoc
index 85e4e901e5c..9149eed6142 100644
--- a/docs/reference/migration/migrate_5_0/docs.asciidoc
+++ b/docs/reference/migration/migrate_5_0/docs.asciidoc
@@ -1,6 +1,16 @@
 [[breaking_50_document_api_changes]]
 === Document API changes
 
+==== `?refresh` no longer supports truthy and falsy values
+The `?refresh` request parameter used to accept any value other than `false`,
+`0`, `off`, and `no` to mean "make the changes from this request visible for
+search immediately." Now it only accepts `?refresh` and `?refresh=true` to
+mean that. You can set it to `?refresh=false` and the request will take no
+refresh-related action. The same is true if you leave `refresh` off of the
+url entirely. If you add `?refresh=wait_for` Elasticsearch will wait for the
+changes to become visible before replying to the request but won't take any
+immediate refresh related action. See <<docs-refresh>>.
+
 ==== Reindex and Update By Query
 Before 5.0.0 `_reindex` and `_update_by_query` only retried bulk failures so
 they used the following response format:
diff --git a/docs/reference/migration/migrate_5_0/java.asciidoc b/docs/reference/migration/migrate_5_0/java.asciidoc
index da97d360b43..a16f2a9ff9b 100644
--- a/docs/reference/migration/migrate_5_0/java.asciidoc
+++ b/docs/reference/migration/migrate_5_0/java.asciidoc
@@ -304,3 +304,8 @@ The `setQuery(BytesReference)` method have been removed in favor of using `setQu
 
 Removed the `getMemoryAvailable` method from `OsStats`, which could be previously accessed calling
 `clusterStatsResponse.getNodesStats().getOs().getMemoryAvailable()`.
+
+=== setRefresh(boolean) has been deprecated
+
+`setRefresh(boolean)` has been deprecated in favor of `setRefreshPolicy(RefreshPolicy)` because there
+are now three options. It will be removed in 5.0.
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java
index dd9daa75e76..13868566eac 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java
@@ -29,6 +29,7 @@ import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateRequestBuilder;
 import org.elasticsearch.action.update.UpdateResponse;
@@ -584,7 +585,7 @@ public class BulkTests extends ESIntegTestCase {
                 .add(new IndexRequest("test", "type", "4").source("{ \"title\" : \"Great Title of doc 4\" }"))
                 .add(new IndexRequest("test", "type", "5").source("{ \"title\" : \"Great Title of doc 5\" }"))
                 .add(new IndexRequest("test", "type", "6").source("{ \"title\" : \"Great Title of doc 6\" }"))
-                .setRefresh(true)
+                .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
                 .get();
         assertNoFailures(indexBulkItemResponse);
 
@@ -622,7 +623,7 @@ public class BulkTests extends ESIntegTestCase {
                    .add(new IndexRequest("bulkindex2", "index2_type").source("text", "hallo2"))
                    .add(new UpdateRequest("bulkindex2", "index2_type", "2").doc("foo", "bar"))
                    .add(new DeleteRequest("bulkindex2", "index2_type", "3"))
-                   .refresh(true);
+                   .setRefreshPolicy(RefreshPolicy.IMMEDIATE);
 
         client().bulk(bulkRequest).get();
         SearchResponse searchResponse = client().prepareSearch("bulkindex*").get();
@@ -643,10 +644,10 @@ public class BulkTests extends ESIntegTestCase {
         client().prepareIndex("bulkindex1", "index1_type", "1").setSource("text", "test").get();
         assertAcked(client().admin().indices().prepareClose("bulkindex1"));
 
-        BulkRequest bulkRequest = new BulkRequest();
+        BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(RefreshPolicy.IMMEDIATE);
         bulkRequest.add(new IndexRequest("bulkindex1", "index1_type", "1").source("text", "hallo1"))
                 .add(new UpdateRequest("bulkindex1", "index1_type", "1").doc("foo", "bar"))
-                .add(new DeleteRequest("bulkindex1", "index1_type", "1")).refresh(true);
+                .add(new DeleteRequest("bulkindex1", "index1_type", "1"));
 
         BulkResponse bulkResponse = client().bulk(bulkRequest).get();
         assertThat(bulkResponse.hasFailures(), is(true));
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
index ed9b65130a9..20c8c7f8b58 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
@@ -26,6 +26,7 @@ import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.geo.builders.ShapeBuilders;
 import org.elasticsearch.common.lucene.search.function.CombineFunction;
@@ -294,8 +295,8 @@ public class PercolatorIT extends ESIntegTestCase {
                         .field("color", "blue")
                         .field("query", termQuery("field1", "value1"))
                         .endObject())
-                .setRefresh(true)
-                .execute().actionGet();
+                .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+                .get();
 
         cluster().wipeIndices("test");
         createIndex("test");
@@ -308,8 +309,8 @@ public class PercolatorIT extends ESIntegTestCase {
                         .field("color", "blue")
                         .field("query", termQuery("field1", "value1"))
                         .endObject())
-                .setRefresh(true)
-                .execute().actionGet();
+                .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+                .get();
     }
 
     // see #2814
@@ -338,8 +339,8 @@ public class PercolatorIT extends ESIntegTestCase {
                         .field("source", "productizer")
                         .field("query", QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("filingcategory:s")))
                         .endObject())
-                .setRefresh(true)
-                .execute().actionGet();
+                .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+                .get();
         refresh();
 
         PercolateResponse percolate = preparePercolate(client())
@@ -417,8 +418,8 @@ public class PercolatorIT extends ESIntegTestCase {
                         .field("color", "blue")
                         .field("query", termQuery("field1", "value1"))
                         .endObject())
-                .setRefresh(true)
-                .execute().actionGet();
+                .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+                .get();
 
         logger.info("--> register a query 2");
         client().prepareIndex(INDEX_NAME, TYPE_NAME, "bubu")
@@ -426,8 +427,8 @@ public class PercolatorIT extends ESIntegTestCase {
                         .field("color", "green")
                         .field("query", termQuery("field1", "value2"))
                         .endObject())
-                .setRefresh(true)
-                .execute().actionGet();
+                .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+                .get();
 
         PercolateResponse percolate = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type1")
@@ -461,8 +462,8 @@ public class PercolatorIT extends ESIntegTestCase {
                         .field("color", "blue")
                         .field("query", termQuery("field1", "value1"))
                         .endObject())
-                .setRefresh(true)
-                .execute().actionGet();
+                .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+                .get();
 
         PercolateResponse percolate = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type1")
@@ -478,8 +479,8 @@ public class PercolatorIT extends ESIntegTestCase {
                         .field("color", "green")
                         .field("query", termQuery("field1", "value2"))
                         .endObject())
-                .setRefresh(true)
-                .execute().actionGet();
+                .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+                .get();
 
         percolate = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type1")
@@ -495,8 +496,8 @@ public class PercolatorIT extends ESIntegTestCase {
                         .field("color", "red")
                         .field("query", termQuery("field1", "value2"))
                         .endObject())
-                .setRefresh(true)
-                .execute().actionGet();
+                .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+                .get();
 
         PercolateSourceBuilder sourceBuilder = new PercolateSourceBuilder()
                 .setDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "value2").endObject()))
@@ -510,7 +511,7 @@ public class PercolatorIT extends ESIntegTestCase {
         assertThat(convertFromTextArray(percolate.getMatches(), INDEX_NAME), arrayContaining("susu"));
 
         logger.info("--> deleting query 1");
-        client().prepareDelete(INDEX_NAME, TYPE_NAME, "kuku").setRefresh(true).execute().actionGet();
+        client().prepareDelete(INDEX_NAME, TYPE_NAME, "kuku").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
 
         percolate = preparePercolate(client())
                 .setIndices(INDEX_NAME).setDocumentType("type1")
@@ -1461,8 +1462,8 @@ public class PercolatorIT extends ESIntegTestCase {
                                 .must(QueryBuilders.queryStringQuery("root"))
                                 .must(QueryBuilders.termQuery("message", "tree"))))
                         .endObject())
-                .setRefresh(true)
-                .execute().actionGet();
+                .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
+                .get();
         refresh();
 
         PercolateResponse percolate = preparePercolate(client())
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json b/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json
index 590054b04a4..a75daf35204 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json
@@ -22,8 +22,9 @@
           "description" : "Explicit write consistency setting for the operation"
         },
         "refresh": {
-          "type" : "boolean",
-          "description" : "Refresh the index after performing the operation"
+          "type" : "enum",
+          "options": ["true", "false", "wait_for"],
+          "description" : "If `true` then refresh the effected shards to make this operation visible to search, if `wait_for` then wait for a refresh to make this operation visible to search, if `false` (the default) then do nothing with refreshes."
         },
         "routing": {
           "type" : "string",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json
index be09c0179d4..5bb0e3fed4c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json
@@ -33,8 +33,9 @@
           "description" : "ID of parent document"
         },
         "refresh": {
-          "type" : "boolean",
-          "description" : "Refresh the index after performing the operation"
+          "type" : "enum",
+          "options": ["true", "false", "wait_for"],
+          "description" : "If `true` then refresh the effected shards to make this operation visible to search, if `wait_for` then wait for a refresh to make this operation visible to search, if `false` (the default) then do nothing with refreshes."
         },
         "routing": {
           "type" : "string",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/index.json b/rest-api-spec/src/main/resources/rest-api-spec/api/index.json
index 5c13f67c212..b7f7eeb9ef5 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/index.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/index.json
@@ -38,8 +38,9 @@
           "description" : "ID of the parent document"
         },
         "refresh": {
-          "type" : "boolean",
-          "description" : "Refresh the index after performing the operation"
+          "type" : "enum",
+          "options": ["true", "false", "wait_for"],
+          "description" : "If `true` then refresh the effected shards to make this operation visible to search, if `wait_for` then wait for a refresh to make this operation visible to search, if `false` (the default) then do nothing with refreshes."
         },
         "routing": {
           "type" : "string",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/update.json b/rest-api-spec/src/main/resources/rest-api-spec/api/update.json
index 20fc3524283..4a3f134301d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/update.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/update.json
@@ -41,8 +41,9 @@
           "description": "ID of the parent document. Is is only used for routing and when for the upsert request"
         },
         "refresh": {
-          "type": "boolean",
-          "description": "Refresh the index after performing the operation"
+          "type" : "enum",
+          "options": ["true", "false", "wait_for"],
+          "description" : "If `true` then refresh the effected shards to make this operation visible to search, if `wait_for` then wait for a refresh to make this operation visible to search, if `false` (the default) then do nothing with refreshes."
         },
         "retry_on_conflict": {
           "type": "number",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/50_refresh.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/50_refresh.yaml
new file mode 100644
index 00000000000..4906975bfab
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/50_refresh.yaml
@@ -0,0 +1,48 @@
+---
+"refresh=true immediately makes changes are visible in search":
+  - do:
+      bulk:
+        refresh: true
+        body: |
+          {"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id"}}
+          {"f1": "v1", "f2": 42}
+          {"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id2"}}
+          {"f1": "v2", "f2": 47}
+
+  - do:
+      count:
+        index: test_index
+  - match: {count: 2}
+
+---
+"refresh=empty string immediately makes changes are visible in search":
+  - do:
+      bulk:
+        refresh: ""
+        body: |
+          {"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id"}}
+          {"f1": "v1", "f2": 42}
+          {"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id2"}}
+          {"f1": "v2", "f2": 47}
+
+  - do:
+      count:
+        index: test_index
+  - match: {count: 2}
+
+
+---
+"refresh=wait_for waits until changes are visible in search":
+  - do:
+      bulk:
+        refresh: wait_for
+        body: |
+          {"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id"}}
+          {"f1": "v1", "f2": 42}
+          {"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id2"}}
+          {"f1": "v2", "f2": 47}
+
+  - do:
+      count:
+        index: test_index
+  - match: {count: 2}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/60_refresh.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/60_refresh.yaml
index 99bfbc3cff6..90dc28bcfc0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/create/60_refresh.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/create/60_refresh.yaml
@@ -33,8 +33,9 @@
           index:   test_1
           type:    test
           id:      2
-          refresh: 1
+          refresh: true
           body:    { foo: bar }
+ - is_true: forced_refresh
 
  - do:
       search:
@@ -44,3 +45,42 @@
             query:   { term: { _id: 2 }}
 
  - match: { hits.total: 1 }
+
+---
+"When refresh url parameter is an empty string that means \"refresh immediately\"":
+ - do:
+      create:
+          index:   test_1
+          type:    test
+          id:      1
+          refresh: ""
+          body:    { foo: bar }
+ - is_true: forced_refresh
+
+ - do:
+      search:
+          index:   test_1
+          type:    test
+          body:
+            query:   { term: { _id: 1 }}
+
+ - match: { hits.total: 1 }
+
+---
+"refresh=wait_for waits until changes are visible in search":
+ - do:
+      index:
+          index:   test_1
+          type:    test
+          id:      1
+          body:    { foo: bar }
+          refresh: wait_for
+ - is_false: forced_refresh
+
+ - do:
+      search:
+          index:   test_1
+          type:    test
+          body:
+            query:   { term: { _id: 1 }}
+ - match: { hits.total: 1 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/50_refresh.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/50_refresh.yaml
index 4d3f9fe039d..9ea6bc033de 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/50_refresh.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/50_refresh.yaml
@@ -19,7 +19,7 @@
           type:    test
           id:      1
           body:    { foo: bar }
-          refresh: 1
+          refresh: true
 
 # If you wonder why this document get 3 as an id instead of 2, it is because the
 # current routing algorithm would route 1 and 2 to the same shard while we need
@@ -30,7 +30,8 @@
           type:    test
           id:      3
           body:    { foo: bar }
-          refresh: 1
+          refresh: true
+ - is_true: forced_refresh
 
  - do:
       search:
@@ -61,7 +62,7 @@
           index:   test_1
           type:    test
           id:      3
-          refresh: 1
+          refresh: true
 
 # If a replica shard where doc 1 is located gets initialized at this point, doc 1
 # won't be found by the following search as the shard gets automatically refreshed
@@ -75,3 +76,72 @@
             query:   { terms: { _id: [1,3] }}
 
  - match: { hits.total: 1 }
+
+---
+"When refresh url parameter is an empty string that means \"refresh immediately\"":
+ - do:
+      index:
+          index:   test_1
+          type:    test
+          id:      1
+          body:    { foo: bar }
+          refresh: true
+ - is_true: forced_refresh
+
+ - do:
+      search:
+          index:   test_1
+          type:    test
+          body:
+            query:   { term: { _id: 1 }}
+ - match: { hits.total: 1 }
+
+ - do:
+      delete:
+          index:   test_1
+          type:    test
+          id:      1
+          refresh: ""
+
+ - do:
+      search:
+          index:   test_1
+          type:    test
+          body:
+            query:   { term: { _id: 1 }}
+ - match: { hits.total: 0 }
+
+---
+"refresh=wait_for waits until changes are visible in search":
+ - do:
+      index:
+          index:   test_1
+          type:    test
+          id:      1
+          body:    { foo: bar }
+          refresh: true
+ - is_true: forced_refresh
+
+ - do:
+      search:
+          index:   test_1
+          type:    test
+          body:
+            query:   { term: { _id: 1 }}
+ - match: { hits.total: 1 }
+
+ - do:
+     delete:
+         index:   test_1
+         type:    test
+         id:      1
+         refresh: wait_for
+ - is_false: forced_refresh
+
+ - do:
+      search:
+          index:   test_1
+          type:    test
+          body:
+            query:   { term: { _id: 1 }}
+ - match: { hits.total: 0 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/60_refresh.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/60_refresh.yaml
index af6ea59766f..4ee26411432 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/60_refresh.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/60_refresh.yaml
@@ -33,8 +33,9 @@
           index:   test_1
           type:    test
           id:      2
-          refresh: 1
+          refresh: true
           body:    { foo: bar }
+ - is_true: forced_refresh
 
  - do:
       search:
@@ -44,3 +45,42 @@
             query:   { term: { _id: 2 }}
 
  - match: { hits.total: 1 }
+
+---
+"When refresh url parameter is an empty string that means \"refresh immediately\"":
+ - do:
+      index:
+          index:   test_1
+          type:    test
+          id:      1
+          refresh: ""
+          body:    { foo: bar }
+ - is_true: forced_refresh
+
+ - do:
+      search:
+          index:   test_1
+          type:    test
+          body:
+            query:   { term: { _id: 1 }}
+
+ - match: { hits.total: 1 }
+
+---
+"refresh=wait_for waits until changes are visible in search":
+ - do:
+      index:
+          index:   test_1
+          type:    test
+          id:      1
+          body:    { foo: bar }
+          refresh: wait_for
+ - is_false: forced_refresh
+
+ - do:
+      search:
+          index:   test_1
+          type:    test
+          body:
+            query:   { term: { _id: 1 }}
+ - match: { hits.total: 1 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/60_refresh.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/60_refresh.yaml
index 6048292ceab..8c0e7e66c97 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/60_refresh.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/60_refresh.yaml
@@ -35,10 +35,11 @@
           index:   test_1
           type:    test
           id:      2
-          refresh: 1
+          refresh: true
           body:
             doc:        { foo: baz }
             upsert:     { foo: bar }
+ - is_true: forced_refresh
 
  - do:
       search:
@@ -48,3 +49,70 @@
             query:   { term: { _id: 2 }}
 
  - match: { hits.total: 1 }
+
+---
+"When refresh url parameter is an empty string that means \"refresh immediately\"":
+ - do:
+      index:
+          index:   test_1
+          type:    test
+          id:      1
+          refresh: true
+          body:    { foo: bar }
+ - is_true: forced_refresh
+
+ - do:
+     update:
+         index:   test_1
+         type:    test
+         id:      1
+         refresh: ""
+         body:
+           doc: {cat: dog}
+ - is_true: forced_refresh
+
+ - do:
+      search:
+          index:   test_1
+          type:    test
+          body:
+            query:   { term: { cat: dog }}
+
+ - match: { hits.total: 1 }
+
+---
+"refresh=wait_for waits until changes are visible in search":
+ - do:
+      index:
+          index:   test_1
+          type:    test
+          id:      1
+          body:    { foo: bar }
+          refresh: true
+ - is_true: forced_refresh
+
+ - do:
+      search:
+          index:   test_1
+          type:    test
+          body:
+            query:   { term: { _id: 1 }}
+ - match: { hits.total: 1 }
+
+ - do:
+     update:
+         index:   test_1
+         type:    test
+         id:      1
+         refresh: wait_for
+         body:
+           doc: { test: asdf }
+ - is_false: forced_refresh
+
+ - do:
+      search:
+          index:   test_1
+          type:    test
+          body:
+            query:   { match: { test: asdf } }
+ - match: { hits.total: 1 }

From d594d6c07cb33635cf72edc84efaa7e4f4ee31c6 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Mon, 6 Jun 2016 18:14:05 +0200
Subject: [PATCH 0661/1311] [TEST] ensure green before we filter allocations
 otherwise follow up ensureGreen() will fail

---
 .../action/admin/indices/create/CreateIndexIT.java        | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
index 31c5749697b..60c4d0f486e 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
@@ -300,6 +300,10 @@ public class CreateIndexIT extends ESIntegTestCase {
         assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2);
         DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class);
         String mergeNode = discoveryNodes[0].getName();
+        // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
+        // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
+        // to the require._name below.
+        ensureGreen();
         // relocate all shards to one node such that we can merge it.
         client().admin().indices().prepareUpdateSettings("source")
             .setSettings(Settings.builder()
@@ -343,6 +347,10 @@ public class CreateIndexIT extends ESIntegTestCase {
         DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class);
         String spareNode = discoveryNodes[0].getName();
         String mergeNode = discoveryNodes[1].getName();
+        // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
+        // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
+        // to the require._name below.
+        ensureGreen();
         // relocate all shards to one node such that we can merge it.
         client().admin().indices().prepareUpdateSettings("source")
             .setSettings(Settings.builder().put("index.routing.allocation.require._name", mergeNode)

From 260b0fd40f117b65902a63d838d00a63cdc4d50e Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Mon, 6 Jun 2016 10:49:40 -0700
Subject: [PATCH 0662/1311] Stubbed foreach node.

---
 .../elasticsearch/painless/antlr/Walker.java  | 12 +++++
 .../elasticsearch/painless/node/SEach.java    | 51 +++++++++++++++++++
 .../painless/node/package-info.java           |  3 +-
 3 files changed, 65 insertions(+), 1 deletion(-)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index 0f155ceb4c1..3e315bbcb24 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -53,6 +53,7 @@ import org.elasticsearch.painless.antlr.PainlessParser.DeclvarContext;
 import org.elasticsearch.painless.antlr.PainlessParser.DelimiterContext;
 import org.elasticsearch.painless.antlr.PainlessParser.DoContext;
 import org.elasticsearch.painless.antlr.PainlessParser.DynamicContext;
+import org.elasticsearch.painless.antlr.PainlessParser.EachContext;
 import org.elasticsearch.painless.antlr.PainlessParser.EmptyContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ExprContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ExpressionContext;
@@ -118,6 +119,7 @@ import org.elasticsearch.painless.node.SContinue;
 import org.elasticsearch.painless.node.SDeclBlock;
 import org.elasticsearch.painless.node.SDeclaration;
 import org.elasticsearch.painless.node.SDo;
+import org.elasticsearch.painless.node.SEach;
 import org.elasticsearch.painless.node.SExpression;
 import org.elasticsearch.painless.node.SFor;
 import org.elasticsearch.painless.node.SIf;
@@ -271,6 +273,16 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         }
     }
 
+    @Override
+    public Object visitEach(EachContext ctx) {
+        String type = ctx.decltype().getText();
+        String name = ctx.ID().getText();
+        AExpression expression = (AExpression)visit(ctx.expression());
+        SBlock block = (SBlock)visit(ctx.trailer());
+
+        return new SEach(location(ctx), type, name, expression, block);
+    }
+
     @Override
     public Object visitDecl(DeclContext ctx) {
         return visit(ctx.declaration());
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
new file mode 100644
index 00000000000..aacde9b48b1
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.Location;
+import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.Variables;
+
+public class SEach extends AStatement {
+    final String type;
+    final String name;
+    AExpression expression;
+    AStatement block;
+
+    public SEach(final Location location, final String type, final String name, final AExpression expression, final SBlock block) {
+        super(location);
+
+        this.type = type;
+        this.name = name;
+        this.expression = expression;
+        this.block = block;
+    }
+
+
+    @Override
+    AStatement analyze(Variables variables) {
+        return null;
+    }
+
+    @Override
+    void write(MethodWriter writer) {
+
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
index 13ef13bdd77..9e693510359 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
@@ -90,7 +90,8 @@
  * <p>
  * Generally, statement nodes have member data that evaluate legal control-flow during the analysis phase.
  * The typical order for statement nodes is for each node to call analyze on it's children during the analysis phase
- * and write on it's children during the writing phase.  No modifications are made to the structure of statement nodes.
+ * and write on it's children during the writing phase.  Upon analysis completion, a statement will return either
+ * itself or another statement node depending on if a shortcut or def type was found.
  * <p>
  * Generally, expression nodes have member data that evaluate static types.  The typical order for an expression node
  * during the analysis phase looks like the following:

From b2e648dfdefc413ecf6f05d5a32dd59eb9ef32ad Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Mon, 6 Jun 2016 12:33:25 -0700
Subject: [PATCH 0663/1311] Implementation of iterable.

---
 .../org/elasticsearch/painless/Variables.java |   6 +-
 .../elasticsearch/painless/antlr/Walker.java  |  12 +-
 .../elasticsearch/painless/node/LBrace.java   |   2 +-
 .../elasticsearch/painless/node/SEach.java    | 121 +++++++++++++++++-
 .../org/elasticsearch/painless/node/SFor.java |   2 +-
 .../painless/BasicStatementTests.java         |   5 +
 6 files changed, 136 insertions(+), 12 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
index 16130476c38..4905011520a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
@@ -138,15 +138,15 @@ public final class Variables {
 
     public void decrementScope() {
         int remove = scopes.pop();
-        
+
         while (remove > 0) {
             Variable variable = variables.pop();
-            
+
             // TODO: is this working? the code reads backwards...
             if (variable.read) {
                 throw variable.location.createError(new IllegalArgumentException("Variable [" + variable.name + "] never used."));
             }
-            
+
             --remove;
         }
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index 3e315bbcb24..a309fccdd27 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -263,11 +263,9 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         if (ctx.trailer() != null) {
             SBlock block = (SBlock)visit(ctx.trailer());
 
-            return new SFor(location(ctx),
-                settings.getMaxLoopCounter(), initializer, expression, afterthought, block);
+            return new SFor(location(ctx), settings.getMaxLoopCounter(), initializer, expression, afterthought, block);
         } else if (ctx.empty() != null) {
-            return new SFor(location(ctx),
-                settings.getMaxLoopCounter(), initializer, expression, afterthought, null);
+            return new SFor(location(ctx), settings.getMaxLoopCounter(), initializer, expression, afterthought, null);
         } else {
             throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
         }
@@ -275,12 +273,16 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
 
     @Override
     public Object visitEach(EachContext ctx) {
+        if (settings.getMaxLoopCounter() > 0) {
+            reserved.usesLoop();
+        }
+
         String type = ctx.decltype().getText();
         String name = ctx.ID().getText();
         AExpression expression = (AExpression)visit(ctx.expression());
         SBlock block = (SBlock)visit(ctx.trailer());
 
-        return new SEach(location(ctx), type, name, expression, block);
+        return new SEach(location(ctx), settings.getMaxLoopCounter(), type, name, expression, block);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
index 4a13e03a490..b0816540c5e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
@@ -47,7 +47,7 @@ public final class LBrace extends ALink {
             throw createError(new IllegalArgumentException("Illegal array access made without target."));
         }
 
-        final Sort sort = before.sort;
+        Sort sort = before.sort;
 
         if (sort == Sort.ARRAY) {
             index.expected = Definition.INT_TYPE;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index aacde9b48b1..8d4052326fc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -19,19 +19,40 @@
 
 package org.elasticsearch.painless.node;
 
+import org.elasticsearch.painless.AnalyzerCaster;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Cast;
+import org.elasticsearch.painless.Definition.Method;
+import org.elasticsearch.painless.Definition.MethodKey;
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Variables.Variable;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.Opcodes;
 
 public class SEach extends AStatement {
+
+    final int maxLoopCounter;
     final String type;
     final String name;
     AExpression expression;
     AStatement block;
 
-    public SEach(final Location location, final String type, final String name, final AExpression expression, final SBlock block) {
+    Variable variable = null;
+    Variable iterator = null;
+    Method method = null;
+    Method hasNext = null;
+    Method next = null;
+    Cast cast = null;
+
+    public SEach(final Location location, final int maxLoopCounter,
+                 final String type, final String name, final AExpression expression, final SBlock block) {
         super(location);
 
+        this.maxLoopCounter = maxLoopCounter;
         this.type = type;
         this.name = name;
         this.expression = expression;
@@ -41,11 +62,107 @@ public class SEach extends AStatement {
 
     @Override
     AStatement analyze(Variables variables) {
-        return null;
+        expression.analyze(variables);
+        expression.expected = expression.actual;
+        expression = expression.cast(variables);
+
+        Sort sort = expression.actual.sort;
+
+        if (sort == Sort.ARRAY) {
+            throw location.createError(new UnsupportedOperationException("Cannot execute for each against array type."));
+        } else if (sort == Sort.DEF) {
+            throw location.createError(new UnsupportedOperationException("Cannot execute for each against def type."));
+        } else if (Iterable.class.isAssignableFrom(expression.actual.clazz)) {
+            final Type type;
+
+            try {
+                type = Definition.getType(this.type);
+            } catch (IllegalArgumentException exception) {
+                throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
+            }
+
+            variables.incrementScope();
+
+            Type itr = Definition.getType("Iterator");
+
+            variable = variables.addVariable(location, type, name, false, false);
+            iterator = variables.addVariable(location, itr, "#itr" + location.getOffset(), true, false);
+
+            method = expression.actual.struct.methods.get(new MethodKey("iterator", 0));
+
+            if (method == null) {
+                throw location.createError(new IllegalArgumentException(
+                    "Unable to create iterator for the type [" + expression.actual.name + "]."));
+            }
+
+            hasNext = itr.struct.methods.get(new MethodKey("hasNext", 0));
+
+            if (hasNext == null) {
+                throw location.createError(new IllegalArgumentException("Method [hasNext] does not exist for type [Iterator]."));
+            } else if (hasNext.rtn.sort != Sort.BOOL) {
+                throw location.createError(new IllegalArgumentException("Method [hasNext] does not return type [boolean]."));
+            }
+
+            next = itr.struct.methods.get(new MethodKey("next", 0));
+
+            if (next == null) {
+                throw location.createError(new IllegalArgumentException("Method [next] does not exist for type [Iterator]."));
+            } else if (next.rtn.sort != Sort.OBJECT) {
+                throw location.createError(new IllegalArgumentException("Method [next] does not return type [Object]."));
+            }
+
+            cast = AnalyzerCaster.getLegalCast(location, Definition.getType("Object"), type, true, true);
+
+            if (block == null) {
+                throw location.createError(new IllegalArgumentException("Extraneous for each loop."));
+            }
+
+            block = block.analyze(variables);
+            block.statementCount = Math.max(1, block.statementCount);
+
+            if (block.loopEscape && !block.anyContinue) {
+                throw createError(new IllegalArgumentException("Extraneous for loop."));
+            }
+
+            statementCount = 1;
+
+            if (maxLoopCounter > 0) {
+                loopCounterSlot = variables.getVariable(location, "#loop").slot;
+            }
+
+            variables.decrementScope();
+
+            return this;
+        } else {
+            throw location.createError(new IllegalArgumentException("Illegal for each type [" + expression.actual.name + "]."));
+        }
     }
 
     @Override
     void write(MethodWriter writer) {
+        expression.write(writer);
 
+        if (java.lang.reflect.Modifier.isInterface(method.owner.clazz.getModifiers())) {
+            writer.invokeInterface(method.owner.type, method.method);
+        } else {
+            writer.invokeVirtual(method.owner.type, method.method);
+        }
+
+        writer.visitVarInsn(iterator.type.type.getOpcode(Opcodes.ISTORE), iterator.slot);
+
+        Label end = new Label();
+
+        writer.visitVarInsn(iterator.type.type.getOpcode(Opcodes.ILOAD), iterator.slot);
+        writer.invokeInterface(hasNext.owner.type, hasNext.method);
+        writer.ifZCmp(MethodWriter.EQ, end);
+
+        writer.visitVarInsn(iterator.type.type.getOpcode(Opcodes.ILOAD), iterator.slot);
+        writer.invokeInterface(next.owner.type, next.method);
+        writer.writeCast(cast);
+        writer.visitVarInsn(variable.type.type.getOpcode(Opcodes.ISTORE), variable.slot);
+
+        block.write(writer);
+
+        writer.mark(end);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index 75909dff81c..6085c1ce8d9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -40,11 +40,11 @@ public final class SFor extends AStatement {
                 ANode initializer, AExpression condition, AExpression afterthought, SBlock block) {
         super(location);
 
+        this.maxLoopCounter = maxLoopCounter;
         this.initializer = initializer;
         this.condition = condition;
         this.afterthought = afterthought;
         this.block = block;
-        this.maxLoopCounter = maxLoopCounter;
     }
 
     @Override
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
index 0d6a54b515b..1fffcf1ee29 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
@@ -125,6 +125,11 @@ public class BasicStatementTests extends ScriptTestCase {
         }
     }
 
+    public void testEachStatement() {
+        assertEquals(6, exec("List l = new ArrayList(); l.add(1); l.add(2); l.add(3); int total = 0;" +
+            " for (int x : l) total += x; return x"));
+    }
+
     public void testDeclarationStatement() {
         assertEquals((byte)2, exec("byte a = 2; return a;"));
         assertEquals((short)2, exec("short a = 2; return a;"));

From 3a2cc22afffb2af453dfec785e062f19de9d1804 Mon Sep 17 00:00:00 2001
From: Areek Zillur <areek.zillur@elasticsearch.com>
Date: Mon, 6 Jun 2016 16:10:36 -0400
Subject: [PATCH 0664/1311] simplify conditions and rollover request

---
 .../admin/indices/rollover/Condition.java     | 121 +++++++--------
 .../indices/rollover/RolloverRequest.java     |  76 +++-------
 .../rollover/RolloverRequestBuilder.java      |  15 +-
 .../rollover/TransportRolloverAction.java     | 142 ++++++++----------
 .../indices/RestRolloverIndexAction.java      |   6 +-
 5 files changed, 153 insertions(+), 207 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
index c5ef9b4c51b..bd70dda62fc 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
@@ -19,87 +19,82 @@
 
 package org.elasticsearch.action.admin.indices.rollover;
 
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcherSupplier;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ObjectParser;
 
-import java.util.Locale;
+import java.io.IOException;
+import java.util.Set;
 
-public class Condition {
+public abstract class Condition<T> implements NamedWriteable {
+    public static ObjectParser<Set<Condition>, ParseFieldMatcherSupplier> PARSER =
+        new ObjectParser<>("conditions", null);
+    static {
+        PARSER.declareString((conditions, s) ->
+            conditions.add(new MaxAge(TimeValue.parseTimeValue(s, MaxAge.NAME))), new ParseField(MaxAge.NAME));
+        PARSER.declareLong((conditions, value) ->
+            conditions.add(new MaxDocs(value)), new ParseField(MaxDocs.NAME));
+    }
 
-    public enum ConditionType {
-        MAX_SIZE((byte) 0),
-        MAX_AGE((byte) 1),
-        MAX_DOCS((byte) 2);
+    public static class MaxAge extends Condition<TimeValue> {
+        public final static String NAME = "max_age";
 
-        private final byte id;
-
-        ConditionType(byte id) {
-            this.id = id;
+        public MaxAge(TimeValue value) {
+            this.value = value;
         }
 
-        public byte getId() {
-            return id;
+        public MaxAge(StreamInput in) throws IOException {
+            this.value = TimeValue.timeValueMillis(in.readLong());
         }
 
-        public static ConditionType fromId(byte id) {
-            if (id == 0) {
-                return MAX_SIZE;
-            } else if (id == 1) {
-                return MAX_AGE;
-            } else if (id == 2) {
-                return MAX_DOCS;
-            } else {
-                throw new IllegalArgumentException("no condition type [" + id + "]");
-            }
+        @Override
+        public boolean matches(TimeValue value) {
+            return this.value.getMillis() <= value.getMillis();
         }
 
-        public static ConditionType fromString(String type) {
-            final String typeString = type.toLowerCase(Locale.ROOT);
-            switch (typeString) {
-                case "max_size":
-                    return MAX_SIZE;
-                case "max_age":
-                    return MAX_AGE;
-                case "max_docs":
-                    return MAX_DOCS;
-                default:
-                    throw new IllegalArgumentException("no condition type [" + type + "]");
-            }
+        @Override
+        public String getWriteableName() {
+            return NAME;
         }
 
-        public static long parseFromString(ConditionType condition, String value) {
-            switch (condition) {
-                case MAX_SIZE:
-                    return ByteSizeValue.parseBytesSizeValue(value, MAX_SIZE.name().toLowerCase(Locale.ROOT)).getBytes();
-                case MAX_AGE:
-                    return TimeValue.parseTimeValue(value, MAX_AGE.name().toLowerCase(Locale.ROOT)).getMillis();
-                case MAX_DOCS:
-                    try {
-                       return Long.valueOf(value);
-                    } catch (NumberFormatException e) {
-                        throw new ElasticsearchParseException("Failed to parse setting [{}] with value [{}] as long", e,
-                            MAX_DOCS.name().toLowerCase(Locale.ROOT), value);
-                    }
-                default:
-                    throw new ElasticsearchParseException("condition [" + condition + "] not recognized");
-            }
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeLong(value.getMillis());
         }
     }
 
-    private final ConditionType type;
-    private final long value;
+    public static class MaxDocs extends Condition<Long> {
+        public final static String NAME = "max_docs";
 
-    public Condition(ConditionType type, long value) {
-        this.type = type;
-        this.value = value;
+        public MaxDocs(Long value) {
+            this.value = value;
+        }
+
+        public MaxDocs(StreamInput in) throws IOException {
+            this.value = in.readLong();
+        }
+
+        @Override
+        public boolean matches(Long value) {
+            return this.value <= value;
+        }
+
+        @Override
+        public String getWriteableName() {
+            return NAME;
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeLong(value);
+        }
     }
 
-    public ConditionType getType() {
-        return type;
-    }
+    protected T value;
 
-    public long getValue() {
-        return value;
-    }
+    public abstract boolean matches(T value);
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
index 4a233bf2d81..03c288f53dd 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
@@ -23,17 +23,21 @@ import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
+import java.util.HashSet;
+import java.util.Set;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
@@ -44,14 +48,20 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
 public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implements IndicesRequest {
 
     private String sourceAlias;
-    private String optionalTargetAlias;
-    private List<Condition> conditions = new ArrayList<>();
+    private Set<Condition> conditions = new HashSet<>(2);
+
+    public static ObjectParser<Set<Condition>, ParseFieldMatcherSupplier> TLP_PARSER =
+        new ObjectParser<>("conditions", null);
+    static {
+        TLP_PARSER.declareField((parser, conditions, parseFieldMatcherSupplier) ->
+        Condition.PARSER.parse(parser, conditions,  () -> ParseFieldMatcher.EMPTY),
+            new ParseField("conditions"), ObjectParser.ValueType.OBJECT);
+    }
 
     RolloverRequest() {}
 
-    public RolloverRequest(String sourceAlias, String optionalTargetAlias) {
+    public RolloverRequest(String sourceAlias) {
         this.sourceAlias = sourceAlias;
-        this.optionalTargetAlias = optionalTargetAlias;
     }
 
     @Override
@@ -67,12 +77,9 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         sourceAlias = in.readString();
-        optionalTargetAlias = in.readOptionalString();
         int size = in.readVInt();
         for (int i = 0; i < size; i++) {
-            Condition.ConditionType type = Condition.ConditionType.fromId(in.readByte());
-            long value = in.readLong();
-            this.conditions.add(new Condition(type, value));
+            this.conditions.add(in.readNamedWriteable(Condition.class));
         }
     }
 
@@ -80,11 +87,9 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeString(sourceAlias);
-        out.writeOptionalString(optionalTargetAlias);
         out.writeVInt(conditions.size());
         for (Condition condition : conditions) {
-            out.writeByte(condition.getType().getId());
-            out.writeLong(condition.getValue());
+            out.writeNamedWriteable(condition);
         }
     }
 
@@ -102,27 +107,15 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
         this.sourceAlias = sourceAlias;
     }
 
-    public void setOptionalTargetAlias(String optionalTargetAlias) {
-        this.optionalTargetAlias = optionalTargetAlias;
+    public void addMaxIndexAgeCondition(TimeValue age) {
+        this.conditions.add(new Condition.MaxAge(age));
     }
 
-    public void addMaxIndexAgeCondition(String age) {
-        addCondition(Condition.ConditionType.MAX_AGE, Condition.ConditionType.parseFromString(Condition.ConditionType.MAX_AGE, age));
+    public void addMaxIndexDocsCondition(long docs) {
+        this.conditions.add(new Condition.MaxDocs(docs));
     }
 
-    public void addMaxIndexDocsCondition(String docs) {
-        addCondition(Condition.ConditionType.MAX_DOCS, Condition.ConditionType.parseFromString(Condition.ConditionType.MAX_DOCS, docs));
-    }
-
-    public void addMaxIndexSizeCondition(String size) {
-        addCondition(Condition.ConditionType.MAX_SIZE, Condition.ConditionType.parseFromString(Condition.ConditionType.MAX_SIZE, size));
-    }
-
-    private void addCondition(Condition.ConditionType conditionType, long value) {
-        this.conditions.add(new Condition(conditionType, value));
-    }
-
-    public List<Condition> getConditions() {
+    public Set<Condition> getConditions() {
         return conditions;
     }
 
@@ -130,15 +123,11 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
         return sourceAlias;
     }
 
-    public String getOptionalTargetAlias() {
-        return optionalTargetAlias;
-    }
-
     public void source(BytesReference source) {
         XContentType xContentType = XContentFactory.xContentType(source);
         if (xContentType != null) {
             try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(source)) {
-                source(parser.map());
+                TLP_PARSER.parse(parser, this.conditions, () -> ParseFieldMatcher.EMPTY);
             } catch (IOException e) {
                 throw new ElasticsearchParseException("failed to parse source for rollover index", e);
             }
@@ -146,19 +135,4 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
             throw new ElasticsearchParseException("failed to parse content type for rollover index source");
         }
     }
-
-    private void source(Map<String, Object> map) {
-        for (Map.Entry<String, Object> entry : map.entrySet()) {
-            if (entry.getKey().equals("conditions")) {
-                final Map<String, String> conditions = (Map<String, String>) entry.getValue();
-                for (Map.Entry<String, String> conditionEntry : conditions.entrySet()) {
-                    final Condition.ConditionType conditionType = Condition.ConditionType.fromString(conditionEntry.getKey());
-                    this.addCondition(conditionType, Condition.ConditionType.parseFromString(conditionType,
-                        conditionEntry.getValue()));
-                }
-            } else {
-                throw new ElasticsearchParseException("unknown property [" + entry.getKey() + "]");
-            }
-        }
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
index c42bb9cbd83..2a95468de69 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
@@ -20,6 +20,7 @@ package org.elasticsearch.action.admin.indices.rollover;
 
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.unit.TimeValue;
 
 
 /**
@@ -36,22 +37,12 @@ public class RolloverRequestBuilder extends MasterNodeOperationRequestBuilder<Ro
         return this;
     }
 
-    public RolloverRequestBuilder setOptionalTargetAlias(String optionalTargetAlias) {
-        this.request.setOptionalTargetAlias(optionalTargetAlias);
-        return this;
-    }
-
-    public RolloverRequestBuilder addMaxIndexSizeCondition(String size) {
-        this.request.addMaxIndexSizeCondition(size);
-        return this;
-    }
-
-    public RolloverRequestBuilder addMaxIndexAgeCondition(String age) {
+    public RolloverRequestBuilder addMaxIndexAgeCondition(TimeValue age) {
         this.request.addMaxIndexAgeCondition(age);
         return this;
     }
 
-    public RolloverRequestBuilder addMaxIndexDocsCondition(String docs) {
+    public RolloverRequestBuilder addMaxIndexDocsCondition(long docs) {
         this.request.addMaxIndexDocsCondition(docs);
         return this;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
index fcd9e2d96cf..fa8782aca03 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -41,13 +41,13 @@ import org.elasticsearch.cluster.metadata.MetaDataIndexAliasesService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.shard.DocsStats;
-import org.elasticsearch.index.store.StoreStats;
 import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-import java.util.List;
+import java.util.Set;
 
 /**
  * Main class to swap the index pointed to by an alias, given some predicates
@@ -97,56 +97,55 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
         final AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(rolloverRequest.getSourceAlias());
         final IndexMetaData indexMetaData = aliasOrIndex.getIndices().get(0);
         final String sourceIndexName = indexMetaData.getIndex().getName();
-        client.admin().indices().prepareStats(sourceIndexName).clear().setDocs(true).setStore(true)
-            .execute(new ActionListener<IndicesStatsResponse>() {
-            @Override
-            public void onResponse(IndicesStatsResponse indicesStatsResponse) {
-                final IndexMetaData sourceIndex = metaData.index(sourceIndexName);
-                DocsStats docsStats = indicesStatsResponse.getTotal().getDocs();
-                long docCount = docsStats == null ? 0 : docsStats.getCount();
-                StoreStats storeStats = indicesStatsResponse.getTotal().getStore();
-                long sizeInBytes = storeStats == null ? 0 : storeStats.getSizeInBytes();
-                long creationDate = sourceIndex.getCreationDate();
-                if (satisfiesConditions(rolloverRequest.getConditions(), docCount, sizeInBytes, creationDate)) {
-                    final String rolloverIndexName = generateRolloverIndexName(sourceIndexName);
-                    boolean createRolloverIndex = metaData.index(rolloverIndexName) == null;
-                    if (createRolloverIndex) {
-                        CreateIndexClusterStateUpdateRequest updateRequest =
-                            prepareCreateIndexRequest(rolloverIndexName, rolloverRequest);
-                        createIndexService.createIndex(updateRequest, new ActionListener<ClusterStateUpdateResponse>() {
-                            @Override
-                            public void onResponse(ClusterStateUpdateResponse response) {
-                                indexAliasesService.indicesAliases(
-                                    prepareIndicesAliasesRequest(sourceIndexName, rolloverIndexName, rolloverRequest),
-                                    new IndicesAliasesListener(sourceIndexName, rolloverIndexName, listener));
-                            }
-
-                            @Override
-                            public void onFailure(Throwable t) {
-                                if (t instanceof IndexAlreadyExistsException) {
-                                    logger.trace("[{}] failed to create rollover index", t, updateRequest.index());
-                                } else {
-                                    logger.debug("[{}] failed to create rollover index", t, updateRequest.index());
+        client.admin().indices().prepareStats(sourceIndexName).clear().setDocs(true).execute(
+            new ActionListener<IndicesStatsResponse>() {
+                @Override
+                public void onResponse(IndicesStatsResponse indicesStatsResponse) {
+                    final IndexMetaData sourceIndex = metaData.index(sourceIndexName);
+                    DocsStats docsStats = indicesStatsResponse.getTotal().getDocs();
+                    long docCount = docsStats == null ? 0 : docsStats.getCount();
+                    long indexAge = System.currentTimeMillis() - sourceIndex.getCreationDate();
+                    if (satisfiesConditions(rolloverRequest.getConditions(), docCount, indexAge)) {
+                        final String rolloverIndexName = generateRolloverIndexName(sourceIndexName);
+                        boolean createRolloverIndex = metaData.index(rolloverIndexName) == null;
+                        if (createRolloverIndex) {
+                            CreateIndexClusterStateUpdateRequest updateRequest =
+                                prepareCreateIndexRequest(rolloverIndexName, rolloverRequest);
+                            createIndexService.createIndex(updateRequest, new ActionListener<ClusterStateUpdateResponse>() {
+                                @Override
+                                public void onResponse(ClusterStateUpdateResponse response) {
+                                    indexAliasesService.indicesAliases(
+                                        prepareIndicesAliasesRequest(sourceIndexName, rolloverIndexName, rolloverRequest),
+                                        new IndicesAliasesListener(sourceIndexName, rolloverIndexName, listener));
                                 }
-                                listener.onFailure(t);
-                            }
-                        });
+
+                                @Override
+                                public void onFailure(Throwable t) {
+                                    if (t instanceof IndexAlreadyExistsException) {
+                                        logger.trace("[{}] failed to create rollover index", t, updateRequest.index());
+                                    } else {
+                                        logger.debug("[{}] failed to create rollover index", t, updateRequest.index());
+                                    }
+                                    listener.onFailure(t);
+                                }
+                            });
+                        } else {
+                            indexAliasesService.indicesAliases(
+                                prepareIndicesAliasesRequest(sourceIndexName, rolloverIndexName, rolloverRequest),
+                                new IndicesAliasesListener(sourceIndexName, rolloverIndexName, listener));
+                        }
                     } else {
-                        indexAliasesService.indicesAliases(
-                            prepareIndicesAliasesRequest(sourceIndexName, rolloverIndexName, rolloverRequest),
-                            new IndicesAliasesListener(sourceIndexName, rolloverIndexName, listener));
+                        // conditions not met
+                        listener.onResponse(new RolloverResponse(sourceIndexName, sourceIndexName));
                     }
-                } else {
-                    // conditions not met
-                    listener.onResponse(new RolloverResponse(sourceIndexName, sourceIndexName));
+                }
+
+                @Override
+                public void onFailure(Throwable e) {
+                    listener.onFailure(e);
                 }
             }
-
-            @Override
-            public void onFailure(Throwable e) {
-                listener.onFailure(e);
-            }
-        });
+        );
 
     }
 
@@ -178,17 +177,9 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
         final IndicesAliasesClusterStateUpdateRequest updateRequest = new IndicesAliasesClusterStateUpdateRequest()
             .ackTimeout(request.ackTimeout())
             .masterNodeTimeout(request.masterNodeTimeout());
-        final AliasAction[] actions;
-        if (request.getOptionalTargetAlias() != null) {
-            actions = new AliasAction[3];
-            actions[0] = new AliasAction(AliasAction.Type.ADD, targetIndex, request.getSourceAlias());
-            actions[1] = new AliasAction(AliasAction.Type.ADD, targetIndex, request.getOptionalTargetAlias());
-            actions[2] = new AliasAction(AliasAction.Type.REMOVE, concreteSourceIndex, request.getSourceAlias());
-        } else {
-            actions = new AliasAction[2];
-            actions[0] = new AliasAction(AliasAction.Type.ADD, targetIndex, request.getSourceAlias());
-            actions[1] = new AliasAction(AliasAction.Type.REMOVE, concreteSourceIndex, request.getSourceAlias());
-        }
+        AliasAction[] actions = new AliasAction[2];
+        actions[0] = new AliasAction(AliasAction.Type.ADD, targetIndex, request.getSourceAlias());
+        actions[1] = new AliasAction(AliasAction.Type.REMOVE, concreteSourceIndex, request.getSourceAlias());
         updateRequest.actions(actions);
         return updateRequest;
     }
@@ -208,26 +199,21 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
         return String.join("-", indexPrefix, String.valueOf(counter));
     }
 
-    static boolean satisfiesConditions(List<Condition> requestConditions, long docCount, long sizeInBytes,
-                                       long creationDate) {
-        for (Condition condition: requestConditions) {
-            switch (condition.getType()) {
-                case MAX_SIZE:
-                    if (sizeInBytes < condition.getValue()) {
-                        return false;
-                    }
-                    break;
-                case MAX_AGE:
-                    long currentAge = System.currentTimeMillis() - creationDate;
-                    if (currentAge < condition.getValue()) {
-                        return false;
-                    }
-                    break;
-                case MAX_DOCS:
-                    if (docCount < condition.getValue()) {
-                        return false;
-                    }
-                    break;
+    static boolean satisfiesConditions(Set<Condition> conditions, long docCount, long indexAge) {
+        for (Condition condition: conditions) {
+            if (condition instanceof Condition.MaxAge) {
+                Condition.MaxAge maxAge = (Condition.MaxAge) condition;
+                final TimeValue age = TimeValue.timeValueMillis(indexAge);
+                if (maxAge.matches(age) == false) {
+                    return false;
+                }
+            } else if (condition instanceof Condition.MaxDocs) {
+                final Condition.MaxDocs maxDocs = (Condition.MaxDocs) condition;
+                if (maxDocs.matches(docCount) == false) {
+                    return false;
+                }
+            } else {
+                throw new IllegalArgumentException("unknown condition [" + condition.getClass().getSimpleName() + "]");
             }
         }
         return true;
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
index 42626b37281..50ec45e9229 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
@@ -43,8 +43,8 @@ public class RestRolloverIndexAction extends BaseRestHandler {
     @Inject
     public RestRolloverIndexAction(Settings settings, RestController controller, Client client) {
         super(settings, client);
-        controller.registerHandler(RestRequest.Method.PUT, "/{alias}/_rollover/{optional_alias}", this);
-        controller.registerHandler(RestRequest.Method.POST, "/{alias}/_rollover/{optional_alias}", this);
+        controller.registerHandler(RestRequest.Method.PUT, "/{alias}/_rollover", this);
+        controller.registerHandler(RestRequest.Method.POST, "/{alias}/_rollover", this);
     }
 
     @Override
@@ -52,7 +52,7 @@ public class RestRolloverIndexAction extends BaseRestHandler {
         if (request.param("alias") == null) {
             throw new IllegalArgumentException("no source alias");
         }
-        RolloverRequest rolloverIndexRequest = new RolloverRequest(request.param("alias"), request.param("optional_alias"));
+        RolloverRequest rolloverIndexRequest = new RolloverRequest(request.param("alias"));
         if (request.hasContent()) {
             rolloverIndexRequest.source(request.content());
         }

From d96fe20e3a45d3d891d39967d22abeecac8a7452 Mon Sep 17 00:00:00 2001
From: Areek Zillur <areek.zillur@elasticsearch.com>
Date: Mon, 6 Jun 2016 16:11:46 -0400
Subject: [PATCH 0665/1311] add named writable registry glue

---
 .../elasticsearch/indices/IndicesModule.java  | 12 ++-
 .../java/org/elasticsearch/node/Node.java     |  2 +-
 .../admin/indices/rollover/RolloverIT.java    | 19 +---
 .../TransportRolloverActionTests.java         | 87 +++++--------------
 .../elasticsearch/index/IndexModuleTests.java |  3 +-
 .../index/engine/InternalEngineTests.java     |  3 +-
 .../internal/FieldNamesFieldMapperTests.java  |  3 +-
 .../aggregations/AggregatorParsingTests.java  |  2 +-
 .../aggregations/BaseAggregationTestCase.java |  2 +-
 .../BasePipelineAggregationTestCase.java      |  2 +-
 .../builder/SearchSourceBuilderTests.java     |  2 +-
 .../messy/tests/TemplateQueryParserTests.java |  2 +-
 .../attachments/AttachmentUnitTestCase.java   |  3 +-
 .../index/mapper/size/SizeMappingTests.java   |  3 +-
 .../rest-api-spec/api/indices.rollover.json   |  9 +-
 .../elasticsearch/index/MapperTestUtils.java  |  3 +-
 .../test/AbstractQueryTestCase.java           |  2 +-
 17 files changed, 56 insertions(+), 103 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
index 8cedea931c0..885e2d78eca 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
@@ -19,10 +19,12 @@
 
 package org.elasticsearch.indices;
 
+import org.elasticsearch.action.admin.indices.rollover.Condition;
 import org.elasticsearch.action.update.UpdateHelper;
 import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
 import org.elasticsearch.common.geo.ShapesAvailability;
 import org.elasticsearch.common.inject.AbstractModule;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
@@ -74,10 +76,18 @@ public class IndicesModule extends AbstractModule {
     // Use a LinkedHashMap for metadataMappers because iteration order matters
     private final Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers
         = new LinkedHashMap<>();
+    private final NamedWriteableRegistry namedWritableRegistry;
 
-    public IndicesModule() {
+    public IndicesModule(NamedWriteableRegistry namedWriteableRegistry) {
+        this.namedWritableRegistry = namedWriteableRegistry;
         registerBuiltInMappers();
         registerBuiltInMetadataMappers();
+        registerBuildInWritables();
+    }
+
+    private void registerBuildInWritables() {
+        namedWritableRegistry.register(Condition.class, Condition.MaxAge.NAME, Condition.MaxAge::new);
+        namedWritableRegistry.register(Condition.class, Condition.MaxDocs.NAME, Condition.MaxDocs::new);
     }
 
     private void registerBuiltInMappers() {
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index cf33770fd16..ad287549931 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -235,7 +235,7 @@ public class Node implements Closeable {
             modules.add(new ThreadPoolModule(threadPool));
             modules.add(new DiscoveryModule(this.settings));
             modules.add(new ClusterModule(this.settings));
-            modules.add(new IndicesModule());
+            modules.add(new IndicesModule(namedWriteableRegistry));
             modules.add(new SearchModule(settings, namedWriteableRegistry));
             modules.add(new ActionModule(DiscoveryNode.isIngestNode(settings), false));
             modules.add(new GatewayModule(settings));
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
index f80543fd7c2..3ab9c8cc040 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
@@ -22,6 +22,7 @@ package org.elasticsearch.action.admin.indices.rollover;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -61,7 +62,7 @@ public class RolloverIT extends ESIntegTestCase {
         index("test_index", "type1", "1", "field", "value");
         flush("test_index");
         final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias")
-            .addMaxIndexAgeCondition("7d").get();
+            .addMaxIndexAgeCondition(TimeValue.timeValueHours(4)).get();
         assertThat(response.getOldIndex(), equalTo("test_index"));
         assertThat(response.getNewIndex(), equalTo("test_index"));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
@@ -71,22 +72,6 @@ public class RolloverIT extends ESIntegTestCase {
         assertNull(newIndex);
     }
 
-    public void testRolloverWithOptionalTargetAlias() throws Exception {
-        assertAcked(prepareCreate("test_index").addAlias(new Alias("test_alias")).get());
-        index("test_index", "type1", "1", "field", "value");
-        flush("test_index");
-        final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias")
-            .setOptionalTargetAlias("test_alias_2").get();
-        assertThat(response.getOldIndex(), equalTo("test_index"));
-        assertThat(response.getNewIndex(), equalTo("test_index-1"));
-        final ClusterState state = client().admin().cluster().prepareState().get().getState();
-        final IndexMetaData oldIndex = state.metaData().index("test_index");
-        assertFalse(oldIndex.getAliases().containsKey("test_alias"));
-        final IndexMetaData newIndex = state.metaData().index("test_index-1");
-        assertTrue(newIndex.getAliases().containsKey("test_alias"));
-        assertTrue(newIndex.getAliases().containsKey("test_alias_2"));
-    }
-
     public void testRolloverOnExistingIndex() throws Exception {
         assertAcked(prepareCreate("test_index").addAlias(new Alias("test_alias")).get());
         index("test_index", "type1", "1", "field", "value");
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
index 959f0d81bf0..ff121ff3dfb 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
@@ -27,59 +27,49 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.test.ESTestCase;
 
-import java.util.Arrays;
 import java.util.Collections;
-import java.util.List;
+import java.util.Set;
 
-import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 
 public class TransportRolloverActionTests extends ESTestCase {
 
     public void testSatisfyConditions() throws Exception {
-        List<Condition> conditions = Collections.emptyList();
-        assertTrue(TransportRolloverAction.satisfiesConditions(conditions, randomLong(), randomLong(),
+        Set<Condition> conditions = Collections.emptySet();
+        assertTrue(TransportRolloverAction.satisfiesConditions(conditions, randomLong(),
             randomLong()));
 
-        conditions = Collections.singletonList(
-            new Condition(Condition.ConditionType.MAX_AGE, 10L));
-        assertTrue(TransportRolloverAction.satisfiesConditions(conditions, randomLong(), randomLong(),
-            System.currentTimeMillis() - randomIntBetween(10, 100)));
-        assertFalse(TransportRolloverAction.satisfiesConditions(conditions, randomLong(), randomLong(),
-            System.currentTimeMillis() - randomIntBetween(1, 9)));
+        conditions = Collections.singleton(new Condition.MaxAge(TimeValue.timeValueMillis(10)));
+        assertTrue(TransportRolloverAction.satisfiesConditions(conditions, randomLong(),
+            TimeValue.timeValueMillis(randomIntBetween(10, 100)).getMillis()));
+        assertFalse(TransportRolloverAction.satisfiesConditions(conditions, randomLong(),
+            TimeValue.timeValueMillis(randomIntBetween(0, 9)).getMillis()));
 
-        conditions = Collections.singletonList(
-            new Condition(Condition.ConditionType.MAX_SIZE, 10L));
-        assertTrue(TransportRolloverAction.satisfiesConditions(conditions, randomLong(), randomIntBetween(10, 100),
+        conditions = Collections.singleton(new Condition.MaxDocs(10L));
+        assertTrue(TransportRolloverAction.satisfiesConditions(conditions, randomIntBetween(10, 100),
             randomLong()));
-        assertFalse(TransportRolloverAction.satisfiesConditions(conditions, randomLong(), randomIntBetween(1, 9),
+        assertFalse(TransportRolloverAction.satisfiesConditions(conditions, randomIntBetween(1, 9),
             randomLong()));
 
-        conditions = Collections.singletonList(
-            new Condition(Condition.ConditionType.MAX_DOCS, 10L));
-        assertTrue(TransportRolloverAction.satisfiesConditions(conditions, randomIntBetween(10, 100), randomLong(),
-            randomLong()));
-        assertFalse(TransportRolloverAction.satisfiesConditions(conditions, randomIntBetween(1, 9), randomLong(),
-            randomLong()));
-
-        conditions = Arrays.asList(new Condition(Condition.ConditionType.MAX_AGE, 100L),
-            new Condition(Condition.ConditionType.MAX_DOCS, 1000L));
+        conditions = Sets.newHashSet(new Condition.MaxAge(TimeValue.timeValueMillis(100)), new Condition.MaxDocs(1000L));
         assertTrue(TransportRolloverAction.satisfiesConditions(conditions, randomIntBetween(1000, 1500),
-            randomLong(), System.currentTimeMillis() - randomIntBetween(100, 500)));
+            TimeValue.timeValueMillis(randomIntBetween(100, 1000)).getMillis()));
         assertFalse(TransportRolloverAction.satisfiesConditions(conditions, randomIntBetween(1, 999),
-            randomLong(), System.currentTimeMillis() - randomIntBetween(100, 500)));
+            TimeValue.timeValueMillis(randomIntBetween(100, 1000)).getMillis()));
         assertFalse(TransportRolloverAction.satisfiesConditions(conditions, randomIntBetween(1000, 1500),
-            randomLong(), System.currentTimeMillis() - randomIntBetween(1, 99)));
+            TimeValue.timeValueMillis(randomIntBetween(0, 99)).getMillis()));
     }
 
     public void testCreateUpdateAliasRequest() throws Exception {
         String sourceAlias = randomAsciiOfLength(10);
         String sourceIndex = randomAsciiOfLength(10);
         String targetIndex = randomAsciiOfLength(10);
-        final RolloverRequest rolloverRequest = new RolloverRequest(sourceAlias, null);
+        final RolloverRequest rolloverRequest = new RolloverRequest(sourceAlias);
         final IndicesAliasesClusterStateUpdateRequest updateRequest =
             TransportRolloverAction.prepareIndicesAliasesRequest(sourceIndex, targetIndex, rolloverRequest);
 
@@ -102,35 +92,6 @@ public class TransportRolloverActionTests extends ESTestCase {
         assertTrue(foundRemove);
     }
 
-    public void testCreateUpdateAliasRequestWithOptionalTargetAlias() throws Exception {
-        String sourceAlias = randomAsciiOfLength(10);
-        String optionalTargetAlias = randomAsciiOfLength(10);
-        String sourceIndex = randomAsciiOfLength(10);
-        String targetIndex = randomAsciiOfLength(10);
-        final RolloverRequest rolloverRequest = new RolloverRequest(sourceAlias, optionalTargetAlias);
-        final IndicesAliasesClusterStateUpdateRequest updateRequest =
-            TransportRolloverAction.prepareIndicesAliasesRequest(sourceIndex, targetIndex, rolloverRequest);
-
-        final AliasAction[] actions = updateRequest.actions();
-        assertThat(actions.length, equalTo(3));
-        boolean foundAdd = false;
-        boolean foundRemove = false;
-        for (AliasAction action : actions) {
-            if (action.actionType() == AliasAction.Type.ADD) {
-                foundAdd = true;
-                assertThat(action.index(), equalTo(targetIndex));
-                assertThat(action.alias(), anyOf(equalTo(sourceAlias),
-                    equalTo(optionalTargetAlias)));
-            } else if (action.actionType() == AliasAction.Type.REMOVE) {
-                foundRemove = true;
-                assertThat(action.index(), equalTo(sourceIndex));
-                assertThat(action.alias(), equalTo(sourceAlias));
-            }
-        }
-        assertTrue(foundAdd);
-        assertTrue(foundRemove);
-    }
-
     public void testValidation() throws Exception {
         String index1 = randomAsciiOfLength(10);
         String alias = randomAsciiOfLength(10);
@@ -154,31 +115,27 @@ public class TransportRolloverActionTests extends ESTestCase {
             ).build();
 
         try {
-            TransportRolloverAction.validate(metaData, new RolloverRequest(aliasWithMultipleIndices,
-                randomBoolean() ? null : randomAsciiOfLength(10)));
+            TransportRolloverAction.validate(metaData, new RolloverRequest(aliasWithMultipleIndices));
             fail("expected to throw exception");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), equalTo("source alias maps to multiple indices"));
         }
 
         try {
-            TransportRolloverAction.validate(metaData, new RolloverRequest(randomFrom(index1, index2),
-                randomBoolean() ? null : randomAsciiOfLength(10)));
+            TransportRolloverAction.validate(metaData, new RolloverRequest(randomFrom(index1, index2)));
             fail("expected to throw exception");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), equalTo("source alias is a concrete index"));
         }
 
         try {
-            TransportRolloverAction.validate(metaData, new RolloverRequest(randomAsciiOfLength(5),
-                randomBoolean() ? null : randomAsciiOfLength(10)));
+            TransportRolloverAction.validate(metaData, new RolloverRequest(randomAsciiOfLength(5)));
             fail("expected to throw exception");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), equalTo("source alias does not exist"));
         }
 
-        TransportRolloverAction.validate(metaData, new RolloverRequest(alias,
-            randomBoolean() ? null : randomAsciiOfLength(10)));
+        TransportRolloverAction.validate(metaData, new RolloverRequest(alias));
     }
 
     public void testGenerateRolloverIndexName() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
index 38b7341cd24..6078b0a8a17 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
@@ -35,6 +35,7 @@ import org.elasticsearch.Version;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -134,7 +135,7 @@ public class IndexModuleTests extends ESTestCase {
         environment = new Environment(settings);
         nodeServicesProvider = newNodeServiceProvider(settings, environment, null);
         nodeEnvironment = new NodeEnvironment(settings, environment);
-        mapperRegistry = new IndicesModule().getMapperRegistry();
+        mapperRegistry = new IndicesModule(new NamedWriteableRegistry()).getMapperRegistry();
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 51df3ee0386..7dd60de6c40 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -58,6 +58,7 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.FileSystemUtils;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.uid.Versions;
@@ -2015,7 +2016,7 @@ public class InternalEngineTests extends ESTestCase {
             IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings);
             AnalysisService analysisService = new AnalysisService(indexSettings, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
             SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
-            MapperRegistry mapperRegistry = new IndicesModule().getMapperRegistry();
+            MapperRegistry mapperRegistry = new IndicesModule(new NamedWriteableRegistry()).getMapperRegistry();
             MapperService mapperService = new MapperService(indexSettings, analysisService, similarityService, mapperRegistry, () -> null);
             DocumentMapper.Builder b = new DocumentMapper.Builder(rootBuilder, mapperService);
             this.docMapper = b.build(mapperService);
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java
index cd7f46844fc..2ec5aee9d15 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java
@@ -24,6 +24,7 @@ import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexService;
@@ -234,7 +235,7 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase {
 
     public void testSeesFieldsFromPlugins() throws IOException {
         IndexService indexService = createIndex("test");
-        IndicesModule indicesModule = new IndicesModule();
+        IndicesModule indicesModule = new IndicesModule(new NamedWriteableRegistry());
         indicesModule.registerMetadataMapper("_dummy", new DummyMetadataFieldMapper.TypeParser());
         final MapperRegistry mapperRegistry = indicesModule.getMapperRegistry();
         MapperService mapperService = new MapperService(indexService.getIndexSettings(), indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::newQueryShardContext);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
index 8801b3da45e..09c16e4ae46 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
@@ -151,7 +151,7 @@ public class AggregatorParsingTests extends ESTestCase {
         };
         scriptModule.prepareSettings(settingsModule);
         injector = new ModulesBuilder().add(new EnvironmentModule(new Environment(settings)), settingsModule,
-                new ThreadPoolModule(threadPool), scriptModule, new IndicesModule() {
+                new ThreadPoolModule(threadPool), scriptModule, new IndicesModule(namedWriteableRegistry) {
 
                     @Override
                     protected void configure() {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
index c81ee0bb9c2..d43c54a0e09 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
@@ -169,7 +169,7 @@ public abstract class BaseAggregationTestCase<AB extends AbstractAggregationBuil
                 settingsModule,
                 new ThreadPoolModule(threadPool),
                 scriptModule,
-                new IndicesModule() {
+                new IndicesModule(namedWriteableRegistry) {
 
                     @Override
                     protected void configure() {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
index 078c04d5ffe..4576dc14863 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
@@ -168,7 +168,7 @@ public abstract class BasePipelineAggregationTestCase<AF extends AbstractPipelin
                 settingsModule,
                 new ThreadPoolModule(threadPool),
                 scriptModule,
-                new IndicesModule() {
+                new IndicesModule(namedWriteableRegistry) {
 
                     @Override
                     protected void configure() {
diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index f8236f66e48..a1e33e91005 100644
--- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -173,7 +173,7 @@ public class SearchSourceBuilderTests extends ESTestCase {
         injector = new ModulesBuilder().add(
                 new EnvironmentModule(new Environment(settings)), settingsModule,
                 new ThreadPoolModule(threadPool),
-                scriptModule, new IndicesModule() {
+                scriptModule, new IndicesModule(namedWriteableRegistry) {
                     @Override
                     protected void configure() {
                         bindMapperExtension();
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
index 7097718c6fc..6d1e4a358bb 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
@@ -134,7 +134,7 @@ public class TemplateQueryParserTests extends ESTestCase {
         AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
         ScriptService scriptService = injector.getInstance(ScriptService.class);
         SimilarityService similarityService = new SimilarityService(idxSettings, Collections.emptyMap());
-        MapperRegistry mapperRegistry = new IndicesModule().getMapperRegistry();
+        MapperRegistry mapperRegistry = new IndicesModule(new NamedWriteableRegistry()).getMapperRegistry();
         MapperService mapperService = new MapperService(idxSettings, analysisService, similarityService, mapperRegistry, () ->
             contextFactory.get());
         IndicesFieldDataCache cache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() {});
diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/AttachmentUnitTestCase.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/AttachmentUnitTestCase.java
index 2d55587961b..e0d5d7a2ec6 100644
--- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/AttachmentUnitTestCase.java
+++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/AttachmentUnitTestCase.java
@@ -21,6 +21,7 @@ package org.elasticsearch.mapper.attachments;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.indices.IndicesModule;
@@ -32,7 +33,7 @@ public abstract class AttachmentUnitTestCase extends ESTestCase {
     protected Settings testSettings;
 
     protected static IndicesModule getIndicesModuleWithRegisteredAttachmentMapper() {
-        IndicesModule indicesModule = new IndicesModule();
+        IndicesModule indicesModule = new IndicesModule(new NamedWriteableRegistry());
         indicesModule.registerMapper(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser());
         return indicesModule;
     }
diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java
index 352d84e975a..ce44fef59a5 100644
--- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java
+++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper.size;
 
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.DocumentMapper;
@@ -47,7 +48,7 @@ public class SizeMappingTests extends ESSingleNodeTestCase {
     @Before
     public void before() {
         indexService = createIndex("test");
-        IndicesModule indices = new IndicesModule();
+        IndicesModule indices = new IndicesModule(new NamedWriteableRegistry());
         indices.registerMetadataMapper(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser());
         mapperService = new MapperService(indexService.getIndexSettings(), indexService.analysisService(), indexService.similarityService(), indices.getMapperRegistry(), indexService::newQueryShardContext);
         parser = mapperService.documentMapperParser();
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
index 494ff408273..f7b1fdc010a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
@@ -3,18 +3,13 @@
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-rollover-index.html",
     "methods": ["PUT", "POST"],
     "url": {
-      "path": "/{alias}/_rollover/{target_alias}",
-      "paths": ["/{alias}/_rollover/{target_alias}"],
+      "path": "/{alias}/_rollover",
+      "paths": ["/{alias}/_rollover", "/{alias}/_rollover/{target_alias}"],
       "parts": {
         "alias": {
           "type" : "string",
           "required" : true,
           "description" : "The name of the alias to rollover"
-        },
-        "target_alias": {
-          "type" : "string",
-          "required" : false,
-          "description" : "The name of optional target alias"
         }
       },
       "params": {
diff --git a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
index b2b172e88ba..fe434246035 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
@@ -39,7 +40,7 @@ import java.util.Collections;
 public class MapperTestUtils {
 
     public static MapperService newMapperService(Path tempDir, Settings indexSettings) throws IOException {
-        IndicesModule indicesModule = new IndicesModule();
+        IndicesModule indicesModule = new IndicesModule(new NamedWriteableRegistry());
         return newMapperService(tempDir, indexSettings, indicesModule);
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
index c27617a7287..db086cf16e1 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
@@ -934,7 +934,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
                     new EnvironmentModule(new Environment(settings)),
                     settingsModule,
                     new ThreadPoolModule(threadPool),
-                    new IndicesModule() {
+                    new IndicesModule(namedWriteableRegistry) {
                         @Override
                         public void configure() {
                             // skip services

From a4ffaa6e7a5660071dbd84d0de641321e7124b3a Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Mon, 6 Jun 2016 13:53:31 -0700
Subject: [PATCH 0666/1311] Working iterable foreach with tests.

---
 .../src/main/antlr/PainlessParser.g4              |  2 +-
 .../org/elasticsearch/painless/antlr/Walker.java  |  2 +-
 .../org/elasticsearch/painless/node/SEach.java    | 15 +++++++++++----
 .../painless/BasicStatementTests.java             |  9 +++++++--
 4 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index dad8db62117..6a5ea6bf7fe 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -74,7 +74,7 @@ decltype
     ;
 
 funcref
-    : TYPE REF ID
+    : ( TYPE | ID ) REF ID
     ;
 
 declvar
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index a309fccdd27..4ac090a4228 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -279,7 +279,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
 
         String type = ctx.decltype().getText();
         String name = ctx.ID().getText();
-        AExpression expression = (AExpression)visit(ctx.expression());
+        AExpression expression = (AExpression)visitExpression(ctx.expression());
         SBlock block = (SBlock)visit(ctx.trailer());
 
         return new SEach(location(ctx), settings.getMaxLoopCounter(), type, name, expression, block);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index 8d4052326fc..d7942583387 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -33,6 +33,9 @@ import org.elasticsearch.painless.Variables.Variable;
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
 
+import java.util.HashMap;
+import java.util.Map;
+
 public class SEach extends AStatement {
 
     final int maxLoopCounter;
@@ -85,7 +88,7 @@ public class SEach extends AStatement {
 
             Type itr = Definition.getType("Iterator");
 
-            variable = variables.addVariable(location, type, name, false, false);
+            variable = variables.addVariable(location, type, name, true, false);
             iterator = variables.addVariable(location, itr, "#itr" + location.getOffset(), true, false);
 
             method = expression.actual.struct.methods.get(new MethodKey("iterator", 0));
@@ -107,11 +110,11 @@ public class SEach extends AStatement {
 
             if (next == null) {
                 throw location.createError(new IllegalArgumentException("Method [next] does not exist for type [Iterator]."));
-            } else if (next.rtn.sort != Sort.OBJECT) {
-                throw location.createError(new IllegalArgumentException("Method [next] does not return type [Object]."));
+            } else if (next.rtn.sort != Sort.DEF) {
+                throw location.createError(new IllegalArgumentException("Method [next] does not return type [def]."));
             }
 
-            cast = AnalyzerCaster.getLegalCast(location, Definition.getType("Object"), type, true, true);
+            cast = AnalyzerCaster.getLegalCast(location, Definition.DEF_TYPE, type, true, true);
 
             if (block == null) {
                 throw location.createError(new IllegalArgumentException("Extraneous for each loop."));
@@ -150,8 +153,11 @@ public class SEach extends AStatement {
 
         writer.visitVarInsn(iterator.type.type.getOpcode(Opcodes.ISTORE), iterator.slot);
 
+        Label begin = new Label();
         Label end = new Label();
 
+        writer.mark(begin);
+
         writer.visitVarInsn(iterator.type.type.getOpcode(Opcodes.ILOAD), iterator.slot);
         writer.invokeInterface(hasNext.owner.type, hasNext.method);
         writer.ifZCmp(MethodWriter.EQ, end);
@@ -163,6 +169,7 @@ public class SEach extends AStatement {
 
         block.write(writer);
 
+        writer.goTo(begin);
         writer.mark(end);
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
index 1fffcf1ee29..7a2846433a6 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
@@ -125,9 +125,14 @@ public class BasicStatementTests extends ScriptTestCase {
         }
     }
 
-    public void testEachStatement() {
+    public void testIterableForEachStatement() {
         assertEquals(6, exec("List l = new ArrayList(); l.add(1); l.add(2); l.add(3); int total = 0;" +
-            " for (int x : l) total += x; return x"));
+            " for (int x : l) total += x; return total"));
+        assertEquals("123", exec("List l = new ArrayList(); l.add('1'); l.add('2'); l.add('3'); String cat = '';" +
+            " for (String x : l) cat += x; return cat"));
+        assertEquals("1236", exec("Map m = new HashMap(); m.put('1', 1); m.put('2', 2); m.put('3', 3);" +
+            " String cat = ''; int total = 0;" +
+            " for (Map.Entry e : m.entrySet()) { cat += e.getKey(); total += e.getValue(); } return cat + total"));
     }
 
     public void testDeclarationStatement() {

From 080946c91551cc3008db41a8fed5a780becf78b3 Mon Sep 17 00:00:00 2001
From: Jared Carey <jared.carey@elastic.co>
Date: Mon, 6 Jun 2016 15:00:13 -0600
Subject: [PATCH 0667/1311] Fix typo in template validation message

This commit addresses a typo in a template validation message in
MetaDataIndexTemplateService.java.

Relates #18754
---
 .../cluster/metadata/MetaDataIndexTemplateService.java          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
index 167dea5cf42..8f59ea3a527 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
@@ -256,7 +256,7 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
             validationErrors.add("template must not start with '_'");
         }
         if (!Strings.validFileNameExcludingAstrix(request.template)) {
-            validationErrors.add("template must not container the following characters " + Strings.INVALID_FILENAME_CHARS);
+            validationErrors.add("template must not contain the following characters " + Strings.INVALID_FILENAME_CHARS);
         }
 
         List<String> indexSettingsValidation = metaDataCreateIndexService.getIndexSettingsValidationErrors(request.settings);

From b32f6abb2611e54bfeedce3b27cdd83c2d86d3b8 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 6 Jun 2016 17:29:03 -0400
Subject: [PATCH 0668/1311] remove unnecessary caching

---
 .../src/main/java/org/elasticsearch/painless/Def.java | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index c9572d15dc1..bfa230a26c7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -238,14 +238,9 @@ public final class Def {
          } catch (LambdaConversionException e) {
              throw new RuntimeException(e);
          }
-         try {
-             // create an implementation of the interface (instance)
-             Object instance = callSite.dynamicInvoker().asType(MethodType.methodType(clazz)).invoke();
-             // bind this instance as a constant replacement for the parameter
-             return MethodHandles.dropArguments(MethodHandles.constant(clazz, instance), 0, Object.class);
-         } catch (Throwable e) {
-             throw new RuntimeException(e);
-         }
+         // we could actually invoke and cache here (in non-capturing cases), but this is not a speedup.
+         MethodHandle factory = callSite.dynamicInvoker().asType(MethodType.methodType(clazz));
+         return MethodHandles.dropArguments(factory, 0, Object.class);
      }
      
 

From 9695caa3fb4a07061fcd825b6397e464fce71d37 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Sun, 5 Jun 2016 23:46:59 -0400
Subject: [PATCH 0669/1311] Bootstrap check for OnOutOfMemoryError and seccomp

This commit adds a bootstrap check for the JVM option OnOutOfMemoryError
being in use and seccomp being enabled. These two options are
incompatible because OnOutOfMemoryError allows the user to specify an
arbitrary program to fork when the JVM encounters an
OutOfMemoryError, and seccomp enables system call filters that prevents
forking.

This commit also adds support for bootstrap checks that are always
enforced, whether or not Elasticsearch is in production mode.
---
 .../bootstrap/BootstrapCheck.java             | 49 +++++++++++-
 .../elasticsearch/monitor/jvm/JvmInfo.java    | 12 +++
 .../bootstrap/BootstrapCheckTests.java        | 76 ++++++++++++++++++-
 .../reference/setup/bootstrap-checks.asciidoc | 16 ++++
 4 files changed, 148 insertions(+), 5 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
index 51ddb752b44..f5b28b19b60 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
@@ -106,7 +106,7 @@ final class BootstrapCheck {
 
         for (final Check check : checks) {
             if (check.check()) {
-                if (!enforceLimits || (check.isSystemCheck() && ignoreSystemChecks)) {
+                if ((!enforceLimits || (check.isSystemCheck() && ignoreSystemChecks)) && !check.alwaysEnforce()) {
                     ignoredErrors.add(check.errorMessage());
                 } else {
                     errors.add(check.errorMessage());
@@ -164,6 +164,7 @@ final class BootstrapCheck {
             checks.add(new MaxMapCountCheck());
         }
         checks.add(new ClientJvmCheck());
+        checks.add(new OnOutOfMemoryErrorCheck());
         return Collections.unmodifiableList(checks);
     }
 
@@ -194,6 +195,10 @@ final class BootstrapCheck {
          */
         boolean isSystemCheck();
 
+        default boolean alwaysEnforce() {
+            return false;
+        }
+
     }
 
     static class HeapSizeCheck implements BootstrapCheck.Check {
@@ -245,7 +250,6 @@ final class BootstrapCheck {
 
     }
 
-    // visible for testing
     static class FileDescriptorCheck implements Check {
 
         private final int limit;
@@ -288,7 +292,6 @@ final class BootstrapCheck {
 
     }
 
-    // visible for testing
     static class MlockallCheck implements Check {
 
         private final boolean mlockallSet;
@@ -504,4 +507,44 @@ final class BootstrapCheck {
 
     }
 
+    static class OnOutOfMemoryErrorCheck implements BootstrapCheck.Check {
+
+        @Override
+        public boolean check() {
+            final String onOutOfMemoryError = onOutOfMemoryError();
+            return isSeccompInstalled() && onOutOfMemoryError != null && !onOutOfMemoryError.equals("");
+        }
+
+        // visible for testing
+        boolean isSeccompInstalled() {
+            return Natives.isSeccompInstalled();
+        }
+
+        // visible for testing
+        String onOutOfMemoryError() {
+            return JvmInfo.jvmInfo().onOutOfMemoryError();
+        }
+
+        @Override
+        public String errorMessage() {
+            return String.format(
+                Locale.ROOT,
+                "OnOutOfMemoryError [%s] requires forking but is prevented by system call filters ([%s=true]);" +
+                    " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError",
+                onOutOfMemoryError(),
+                BootstrapSettings.SECCOMP_SETTING.getKey());
+        }
+
+        @Override
+        public boolean isSystemCheck() {
+            return false;
+        }
+
+        @Override
+        public boolean alwaysEnforce() {
+            return true;
+        }
+
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java
index 3e39edaa47c..b2e1990a680 100644
--- a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java
+++ b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java
@@ -115,6 +115,12 @@ public class JvmInfo implements Streamable, ToXContent {
             Method vmOptionMethod = clazz.getMethod("getVMOption", String.class);
             Method valueMethod = vmOptionClazz.getMethod("getValue");
 
+            try {
+                Object onOutOfMemoryError = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "OnOutOfMemoryError");
+                info.onOutOfMemoryError = (String) valueMethod.invoke(onOutOfMemoryError);
+            } catch (Exception ignored) {
+            }
+
             try {
                 Object useCompressedOopsVmOption = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "UseCompressedOops");
                 info.useCompressedOops = (String) valueMethod.invoke(useCompressedOopsVmOption);
@@ -179,6 +185,8 @@ public class JvmInfo implements Streamable, ToXContent {
     String[] gcCollectors = Strings.EMPTY_ARRAY;
     String[] memoryPools = Strings.EMPTY_ARRAY;
 
+    private String onOutOfMemoryError;
+
     private String useCompressedOops = "unknown";
 
     private String useG1GC = "unknown";
@@ -314,6 +322,10 @@ public class JvmInfo implements Streamable, ToXContent {
         return configuredMaxHeapSize;
     }
 
+    public String onOutOfMemoryError() {
+        return onOutOfMemoryError;
+    }
+
     /**
      * The value of the JVM flag UseCompressedOops, if available otherwise
      * "unknown". The value "unknown" indicates that an attempt was
diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
index d354adc7544..41609f7fc5d 100644
--- a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
+++ b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
@@ -30,6 +30,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -383,7 +384,7 @@ public class BootstrapCheckTests extends ESTestCase {
             }
         };
 
-        RuntimeException e = expectThrows(
+        final RuntimeException e = expectThrows(
                 RuntimeException.class,
                 () -> BootstrapCheck.check(true, false, Collections.singletonList(check), "testClientJvmCheck"));
         assertThat(
@@ -395,8 +396,50 @@ public class BootstrapCheckTests extends ESTestCase {
         BootstrapCheck.check(true, false, Collections.singletonList(check), "testClientJvmCheck");
     }
 
+    public void testOnOutOfMemoryErrorCheck() {
+        final AtomicBoolean isSeccompInstalled = new AtomicBoolean();
+        final AtomicReference<String> onOutOfMemoryError = new AtomicReference<>();
+        final BootstrapCheck.Check check = new BootstrapCheck.OnOutOfMemoryErrorCheck() {
+            @Override
+            boolean isSeccompInstalled() {
+                return isSeccompInstalled.get();
+            }
+
+            @Override
+            String onOutOfMemoryError() {
+                return onOutOfMemoryError.get();
+            }
+        };
+
+        // if seccomp is disabled, nothing should happen
+        isSeccompInstalled.set(false);
+        onOutOfMemoryError.set(randomBoolean() ? "" : randomAsciiOfLength(16));
+        BootstrapCheck.check(true, false, Collections.singletonList(check), "testOnOutOfMemoryErrorCheck");
+
+        // if seccomp is enabled, and OnOutOfMemoryError is not, nothing
+        // should happen
+        isSeccompInstalled.set(true);
+        onOutOfMemoryError.set("");
+        BootstrapCheck.check(true, false, Collections.singletonList(check), "testOnOutOfMemoryErrorCheck");
+
+        // if seccomp is enabled, and OnOutOfMemoryError is, the check
+        // should be enforced, regardless of bootstrap checks being
+        // enabled or not
+        isSeccompInstalled.set(true);
+        final String command = randomAsciiOfLength(16);
+        onOutOfMemoryError.set(command);
+        final RuntimeException e = expectThrows(
+            RuntimeException.class,
+            () -> BootstrapCheck.check(randomBoolean(), randomBoolean(), Collections.singletonList(check), "testOnOutOfMemoryErrorCheck"));
+        assertThat(
+            e.getMessage(),
+            containsString(
+                "OnOutOfMemoryError [" + command + "] requires forking but is prevented by system call filters ([bootstrap.seccomp=true]);"
+                    + " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError"));
+    }
+
     public void testIgnoringSystemChecks() {
-        BootstrapCheck.Check check = new BootstrapCheck.Check() {
+        final BootstrapCheck.Check check = new BootstrapCheck.Check() {
             @Override
             public boolean check() {
                 return true;
@@ -430,4 +473,33 @@ public class BootstrapCheckTests extends ESTestCase {
         verify(logger).warn("error");
     }
 
+    public void testAlwaysEnforcedChecks() {
+        final BootstrapCheck.Check check = new BootstrapCheck.Check() {
+            @Override
+            public boolean check() {
+                return true;
+            }
+
+            @Override
+            public String errorMessage() {
+                return "error";
+            }
+
+            @Override
+            public boolean isSystemCheck() {
+                return randomBoolean();
+            }
+
+            @Override
+            public boolean alwaysEnforce() {
+                return true;
+            }
+        };
+
+        final RuntimeException alwaysEnforced = expectThrows(
+            RuntimeException.class,
+            () -> BootstrapCheck.check(randomBoolean(), randomBoolean(), Collections.singletonList(check), "testAlwaysEnforcedChecks"));
+        assertThat(alwaysEnforced, hasToString(containsString("error")));
+    }
+
 }
diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc
index 2a5c55bf8b3..81d14f24c8f 100644
--- a/docs/reference/setup/bootstrap-checks.asciidoc
+++ b/docs/reference/setup/bootstrap-checks.asciidoc
@@ -16,6 +16,10 @@ checks that fail appear as warnings in the Elasticsearch log. If
 Elasticsearch is in production mode, any bootstrap checks that fail will
 cause Elasticsearch to refuse to start.
 
+There are some bootstrap checks that are always enforced to prevent
+Elasticsearch from running with incompatible settings. These checks are
+documented individually.
+
 [float]
 === Development vs. production mode
 
@@ -152,3 +156,15 @@ JVM check, you must start Elasticsearch with the server VM. On modern
 systems and operating systems, the server VM is the
 default. Additionally, Elasticsearch is configured by default to force
 the server VM.
+
+=== OnOutOfMemoryError check
+
+The JVM option `OnOutOfMemoryError` enables executing an arbitrary
+command if the JVM throws an `OutOfMemoryError`. However, by default,
+Elasticsearch system call filters (seccomp) are enabled and these
+filters prevent forking. Thus, using `OnOutOfMemoryError` and system
+call filters are incompatible. The `OnOutOfMemoryError` check prevents
+Elasticsearch from starting if this JVM option is used and system call
+filters are enabled. This check is always enforced. To pass this check
+do not enable `OnOutOfMemoryError`; instead, upgrade to Java 8u92 and
+use the JVM flag `ExitOnOutOfMemoryError`.

From 23d7cdd043a14367f88f8f74a81d6789bf7bbf40 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 6 Jun 2016 17:58:00 -0400
Subject: [PATCH 0670/1311] ban slow methods in MethodHandle

---
 buildSrc/src/main/resources/forbidden/es-core-signatures.txt | 5 +++++
 .../main/java/org/elasticsearch/painless/DefBootstrap.java   | 3 +++
 2 files changed, 8 insertions(+)

diff --git a/buildSrc/src/main/resources/forbidden/es-core-signatures.txt b/buildSrc/src/main/resources/forbidden/es-core-signatures.txt
index 059be403a67..31084c7556c 100644
--- a/buildSrc/src/main/resources/forbidden/es-core-signatures.txt
+++ b/buildSrc/src/main/resources/forbidden/es-core-signatures.txt
@@ -92,3 +92,8 @@ org.joda.time.DateTime#<init>(int, int, int, int, int, int)
 org.joda.time.DateTime#<init>(int, int, int, int, int, int, int)
 org.joda.time.DateTime#now()
 org.joda.time.DateTimeZone#getDefault()
+
+@defaultMessage Don't use MethodHandles in slow ways, except in tests.
+java.lang.invoke.MethodHandle#invoke(java.lang.Object[])
+java.lang.invoke.MethodHandle#invokeWithArguments(java.lang.Object[])
+java.lang.invoke.MethodHandle#invokeWithArguments(java.util.List)
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index 4c30891dd2d..e28dc0242dd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -1,5 +1,7 @@
 package org.elasticsearch.painless;
 
+import org.elasticsearch.common.SuppressForbidden;
+
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -115,6 +117,7 @@ public final class DefBootstrap {
          * Called when a new type is encountered (or, when we have encountered more than {@code MAX_DEPTH}
          * types at this call site and given up on caching).
          */
+        @SuppressForbidden(reason = "slow path")
         Object fallback(Object[] args) throws Throwable {
             final MethodType type = type();
             final Object receiver = args[0];

From 9a78f6955bd2d40306830ee211226f123c9c2d85 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Mon, 6 Jun 2016 15:25:09 -0700
Subject: [PATCH 0671/1311] Added foreach for array types.

---
 .../painless/node/AStatement.java             |  19 +++
 .../painless/node/SArrayEach.java             | 130 ++++++++++++++++++
 .../elasticsearch/painless/node/SEach.java    |  10 +-
 .../painless/BasicStatementTests.java         |   9 ++
 4 files changed, 163 insertions(+), 5 deletions(-)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SArrayEach.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
index 7b5d4a2b4dc..f6b1048028c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
@@ -124,4 +124,23 @@ public abstract class AStatement extends ANode {
      * Writes ASM based on the data collected during the analysis phase.
      */
     abstract void write(MethodWriter writer);
+
+    /**
+     * Used to copy statement data from one to another during analysis in the case of replacement.
+     */
+    final AStatement copy(AStatement statement) {
+        lastSource      = statement.lastSource;
+        beginLoop       = statement.beginLoop;
+        inLoop          = statement.inLoop;
+        lastLoop        = statement.lastLoop;
+        methodEscape    = statement.methodEscape;
+        loopEscape      = statement.loopEscape;
+        allEscape       = statement.allEscape;
+        anyContinue     = statement.anyContinue;
+        anyBreak        = statement.anyBreak;
+        loopCounterSlot = statement.loopCounterSlot;
+        statementCount  = statement.statementCount;
+
+        return this;
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SArrayEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SArrayEach.java
new file mode 100644
index 00000000000..1c3f90ed93e
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SArrayEach.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.AnalyzerCaster;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Cast;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Location;
+import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Variables.Variable;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.Opcodes;
+
+class SArrayEach extends AStatement {
+    final int maxLoopCounter;
+    final String type;
+    final String name;
+    AExpression expression;
+    AStatement block;
+
+    Variable variable = null;
+    Variable array = null;
+    Variable index = null;
+    Type indexed = null;
+    Cast cast = null;
+
+    SArrayEach(final Location location, final int maxLoopCounter,
+               final String type, final String name, final AExpression expression, final SBlock block) {
+        super(location);
+
+        this.maxLoopCounter = maxLoopCounter;
+        this.type = type;
+        this.name = name;
+        this.expression = expression;
+        this.block = block;
+    }
+
+    @Override
+    AStatement analyze(Variables variables) {
+        final Type type;
+
+        try {
+            type = Definition.getType(this.type);
+        } catch (IllegalArgumentException exception) {
+            throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
+        }
+
+        variables.incrementScope();
+
+        variable = variables.addVariable(location, type, name, true, false);
+        array = variables.addVariable(location, expression.actual, "#array" + location.getOffset(), true, false);
+        index = variables.addVariable(location, Definition.INT_TYPE, "#index" + location.getOffset(), true, false);
+        indexed = Definition.getType(expression.actual.struct, expression.actual.dimensions - 1);
+        cast = AnalyzerCaster.getLegalCast(location, indexed, type, true, true);
+
+        if (block == null) {
+            throw location.createError(new IllegalArgumentException("Extraneous for each loop."));
+        }
+
+        block.beginLoop = true;
+        block.inLoop = true;
+        block = block.analyze(variables);
+        block.statementCount = Math.max(1, block.statementCount);
+
+        if (block.loopEscape && !block.anyContinue) {
+            throw createError(new IllegalArgumentException("Extraneous for loop."));
+        }
+
+        statementCount = 1;
+
+        if (maxLoopCounter > 0) {
+            loopCounterSlot = variables.getVariable(location, "#loop").slot;
+        }
+
+        variables.decrementScope();
+
+        return this;
+    }
+
+    @Override
+    void write(MethodWriter writer) {
+        writer.writeStatementOffset(location);
+
+        expression.write(writer);
+        writer.visitVarInsn(array.type.type.getOpcode(Opcodes.ISTORE), array.slot);
+        writer.push(-1);
+        writer.visitVarInsn(index.type.type.getOpcode(Opcodes.ISTORE), index.slot);
+
+        Label begin = new Label();
+        Label end = new Label();
+
+        writer.mark(begin);
+
+        writer.visitIincInsn(index.slot, 1);
+        writer.visitVarInsn(index.type.type.getOpcode(Opcodes.ILOAD), index.slot);
+        writer.visitVarInsn(array.type.type.getOpcode(Opcodes.ILOAD), array.slot);
+        writer.arrayLength();
+        writer.ifICmp(MethodWriter.GE, end);
+
+        writer.visitVarInsn(array.type.type.getOpcode(Opcodes.ILOAD), array.slot);
+        writer.visitVarInsn(index.type.type.getOpcode(Opcodes.ILOAD), index.slot);
+        writer.arrayLoad(indexed.type);
+        writer.writeCast(cast);
+        writer.visitVarInsn(variable.type.type.getOpcode(Opcodes.ISTORE), variable.slot);
+
+        block.write(writer);
+
+        writer.goTo(begin);
+        writer.mark(end);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index d7942583387..5dd1ee0936d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -33,9 +33,6 @@ import org.elasticsearch.painless.Variables.Variable;
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
 
-import java.util.HashMap;
-import java.util.Map;
-
 public class SEach extends AStatement {
 
     final int maxLoopCounter;
@@ -62,7 +59,6 @@ public class SEach extends AStatement {
         this.block = block;
     }
 
-
     @Override
     AStatement analyze(Variables variables) {
         expression.analyze(variables);
@@ -72,7 +68,7 @@ public class SEach extends AStatement {
         Sort sort = expression.actual.sort;
 
         if (sort == Sort.ARRAY) {
-            throw location.createError(new UnsupportedOperationException("Cannot execute for each against array type."));
+            return new SArrayEach(location, maxLoopCounter, type, name, expression, (SBlock)block).copy(this).analyze(variables);
         } else if (sort == Sort.DEF) {
             throw location.createError(new UnsupportedOperationException("Cannot execute for each against def type."));
         } else if (Iterable.class.isAssignableFrom(expression.actual.clazz)) {
@@ -120,6 +116,8 @@ public class SEach extends AStatement {
                 throw location.createError(new IllegalArgumentException("Extraneous for each loop."));
             }
 
+            block.beginLoop = true;
+            block.inLoop = true;
             block = block.analyze(variables);
             block.statementCount = Math.max(1, block.statementCount);
 
@@ -143,6 +141,8 @@ public class SEach extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
+        writer.writeStatementOffset(location);
+
         expression.write(writer);
 
         if (java.lang.reflect.Modifier.isInterface(method.owner.clazz.getModifiers())) {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
index 7a2846433a6..6ddb4067426 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
@@ -135,6 +135,15 @@ public class BasicStatementTests extends ScriptTestCase {
             " for (Map.Entry e : m.entrySet()) { cat += e.getKey(); total += e.getValue(); } return cat + total"));
     }
 
+    public void testArrayForEachStatement() {
+        assertEquals(6, exec("int[] a = new int[3]; a[0] = 1; a[1] = 2; a[2] = 3; int total = 0;" +
+            " for (int x : a) total += x; return total"));
+        assertEquals("123", exec("String[] a = new String[3]; a[0] = '1'; a[1] = '2'; a[2] = '3'; def total = '';" +
+            " for (String x : a) total += x; return total"));
+        assertEquals(6, exec("int[][] i = new int[3][1]; i[0][0] = 1; i[1][0] = 2; i[2][0] = 3; int total = 0;" +
+            " for (int[] j : i) total += j[0]; return total"));
+    }
+
     public void testDeclarationStatement() {
         assertEquals((byte)2, exec("byte a = 2; return a;"));
         assertEquals((short)2, exec("short a = 2; return a;"));

From f89734229677fbaf83233e117eb81d5979152273 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Mon, 6 Jun 2016 15:34:51 -0700
Subject: [PATCH 0672/1311] Updated package info documentation.

---
 .../main/java/org/elasticsearch/painless/node/SArrayEach.java  | 3 +++
 .../src/main/java/org/elasticsearch/painless/node/SEach.java   | 3 +++
 .../java/org/elasticsearch/painless/node/package-info.java     | 2 ++
 3 files changed, 8 insertions(+)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SArrayEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SArrayEach.java
index 1c3f90ed93e..9abb7a18a9a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SArrayEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SArrayEach.java
@@ -30,6 +30,9 @@ import org.elasticsearch.painless.Variables.Variable;
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
 
+/**
+ * Represents a for-each loop shortcut for arrays.  (Internal only.)
+ */
 class SArrayEach extends AStatement {
     final int maxLoopCounter;
     final String type;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index 5dd1ee0936d..0d7e81c2073 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -33,6 +33,9 @@ import org.elasticsearch.painless.Variables.Variable;
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
 
+/**
+ * Represents a for-each loop shortcut for iterables.  Defers to other S-nodes for non-iterable types.
+ */
 public class SEach extends AStatement {
 
     final int maxLoopCounter;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
index 9e693510359..04727507edc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
@@ -63,6 +63,7 @@
  * {@link org.elasticsearch.painless.node.LStatic} - Represents a static type target.
  * {@link org.elasticsearch.painless.node.LString} - Represents a string constant.
  * {@link org.elasticsearch.painless.node.LVariable} - Represents a variable load/store.
+ * {@link org.elasticsearch.painless.node.SArrayEach} - Represents a for each loop shortcut for arrays.  (Internal only.)
  * {@link org.elasticsearch.painless.node.SBlock} - Represents a set of statements as a branch of control-flow.
  * {@link org.elasticsearch.painless.node.SBreak} - Represents a break statement.
  * {@link org.elasticsearch.painless.node.SCatch} - Represents a catch block as part of a try-catch block.
@@ -70,6 +71,7 @@
  * {@link org.elasticsearch.painless.node.SDeclaration} - Represents a single variable declaration.
  * {@link org.elasticsearch.painless.node.SDeclBlock} - Represents a series of declarations.
  * {@link org.elasticsearch.painless.node.SDo} - Represents a do-while loop.
+ * {@link org.elasticsearch.painless.node.SEach} - Represents a for each loop shortcut for iterables.
  * {@link org.elasticsearch.painless.node.SExpression} - Represents the top-level node for an expression as a statement.
  * {@link org.elasticsearch.painless.node.SFor} - Represents a for loop.
  * {@link org.elasticsearch.painless.node.SIf} - Represents an if block.

From 1e329099f99d86dea0e3aaa08374cf8b9345f701 Mon Sep 17 00:00:00 2001
From: Areek Zillur <areek.zillur@elasticsearch.com>
Date: Mon, 6 Jun 2016 18:50:45 -0400
Subject: [PATCH 0673/1311] enhance rollover request

---
 .../admin/indices/rollover/Condition.java     | 14 ++++
 .../indices/rollover/RolloverRequest.java     | 12 +++
 .../rollover/RolloverRequestBuilder.java      |  5 ++
 .../indices/rollover/RolloverResponse.java    | 79 ++++++++++++++++++-
 .../rollover/TransportRolloverAction.java     | 56 ++++++++-----
 .../indices/RestRolloverIndexAction.java      | 21 +----
 .../indices/rollover/ConditionTests.java      | 41 ++++++++++
 .../admin/indices/rollover/RolloverIT.java    | 38 +++++++++
 .../TransportRolloverActionTests.java         | 73 ++++-------------
 9 files changed, 244 insertions(+), 95 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java

diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
index bd70dda62fc..fee08433e9c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
@@ -44,10 +44,12 @@ public abstract class Condition<T> implements NamedWriteable {
         public final static String NAME = "max_age";
 
         public MaxAge(TimeValue value) {
+            super(NAME);
             this.value = value;
         }
 
         public MaxAge(StreamInput in) throws IOException {
+            super(NAME);
             this.value = TimeValue.timeValueMillis(in.readLong());
         }
 
@@ -71,10 +73,12 @@ public abstract class Condition<T> implements NamedWriteable {
         public final static String NAME = "max_docs";
 
         public MaxDocs(Long value) {
+            super(NAME);
             this.value = value;
         }
 
         public MaxDocs(StreamInput in) throws IOException {
+            super(NAME);
             this.value = in.readLong();
         }
 
@@ -95,6 +99,16 @@ public abstract class Condition<T> implements NamedWriteable {
     }
 
     protected T value;
+    protected final String name;
+
+    protected Condition(String name) {
+        this.name = name;
+    }
 
     public abstract boolean matches(T value);
+
+    @Override
+    public final String toString() {
+        return "[" + name + ": " + value + "]";
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
index 03c288f53dd..0d55d256390 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
@@ -48,6 +48,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
 public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implements IndicesRequest {
 
     private String sourceAlias;
+    private boolean simulate;
     private Set<Condition> conditions = new HashSet<>(2);
 
     public static ObjectParser<Set<Condition>, ParseFieldMatcherSupplier> TLP_PARSER =
@@ -77,6 +78,7 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         sourceAlias = in.readString();
+        simulate = in.readBoolean();
         int size = in.readVInt();
         for (int i = 0; i < size; i++) {
             this.conditions.add(in.readNamedWriteable(Condition.class));
@@ -87,6 +89,7 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeString(sourceAlias);
+        out.writeBoolean(simulate);
         out.writeVInt(conditions.size());
         for (Condition condition : conditions) {
             out.writeNamedWriteable(condition);
@@ -107,6 +110,10 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
         this.sourceAlias = sourceAlias;
     }
 
+    public void simulate(boolean simulate) {
+        this.simulate = simulate;
+    }
+
     public void addMaxIndexAgeCondition(TimeValue age) {
         this.conditions.add(new Condition.MaxAge(age));
     }
@@ -115,6 +122,10 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
         this.conditions.add(new Condition.MaxDocs(docs));
     }
 
+    public boolean isSimulate() {
+        return simulate;
+    }
+
     public Set<Condition> getConditions() {
         return conditions;
     }
@@ -135,4 +146,5 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
             throw new ElasticsearchParseException("failed to parse content type for rollover index source");
         }
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
index 2a95468de69..ea495bc6eba 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
@@ -46,4 +46,9 @@ public class RolloverRequestBuilder extends MasterNodeOperationRequestBuilder<Ro
         this.request.addMaxIndexDocsCondition(docs);
         return this;
     }
+
+    public RolloverRequestBuilder simulate(boolean simulate) {
+        this.request.simulate(simulate);
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
index 79722e73755..a2abe044930 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
@@ -22,20 +22,36 @@ package org.elasticsearch.action.admin.indices.rollover;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
+import java.util.AbstractMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 
-public final class RolloverResponse extends ActionResponse {
+public final class RolloverResponse extends ActionResponse implements ToXContent {
 
     private String oldIndex;
     private String newIndex;
+    private Set<Map.Entry<String, Boolean>> conditionStatus;
+    private boolean simulate;
+    private boolean rolledOver;
+    private boolean rolloverIndexCreated;
 
     RolloverResponse() {
     }
 
-    RolloverResponse(String oldIndex, String newIndex) {
+    RolloverResponse(String oldIndex, String newIndex, Set<Map.Entry<String, Boolean>> conditionStatus,
+                     boolean simulate, boolean rolledOver, boolean rolloverIndexCreated) {
         this.oldIndex = oldIndex;
         this.newIndex = newIndex;
+        this.simulate = simulate;
+        this.rolledOver = rolledOver;
+        this.rolloverIndexCreated = rolloverIndexCreated;
+        this.conditionStatus = conditionStatus;
     }
 
     public String getOldIndex() {
@@ -46,11 +62,38 @@ public final class RolloverResponse extends ActionResponse {
         return newIndex;
     }
 
+    public Set<Map.Entry<String, Boolean>> getConditionStatus() {
+        return conditionStatus;
+    }
+
+    public boolean isSimulate() {
+        return simulate;
+    }
+
+    public boolean isRolledOver() {
+        return rolledOver;
+    }
+
+    public boolean isRolloverIndexCreated() {
+        return rolloverIndexCreated;
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         oldIndex = in.readString();
         newIndex = in.readString();
+        int conditionSize = in.readVInt();
+        Set<Map.Entry<String, Boolean>> conditions = new HashSet<>(conditionSize);
+        for (int i = 0; i < conditionSize; i++) {
+            String condition = in.readString();
+            boolean satisfied = in.readBoolean();
+            conditions.add(new AbstractMap.SimpleEntry<>(condition, satisfied));
+        }
+        conditionStatus = conditions;
+        simulate = in.readBoolean();
+        rolledOver = in.readBoolean();
+        rolloverIndexCreated = in.readBoolean();
     }
 
     @Override
@@ -58,5 +101,37 @@ public final class RolloverResponse extends ActionResponse {
         super.writeTo(out);
         out.writeString(oldIndex);
         out.writeString(newIndex);
+        out.writeVInt(conditionStatus.size());
+        for (Map.Entry<String, Boolean> entry : conditionStatus) {
+            out.writeString(entry.getKey());
+            out.writeBoolean(entry.getValue());
+        }
+        out.writeBoolean(simulate);
+        out.writeBoolean(rolledOver);
+        out.writeBoolean(rolloverIndexCreated);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field(Fields.OLD_INDEX, oldIndex);
+        builder.field(Fields.NEW_INDEX, newIndex);
+        builder.field(Fields.ROLLED_OVER, rolledOver);
+        builder.field(Fields.SIMULATED, simulate);
+        builder.field(Fields.ROLLOVER_INDEX_CREATED, rolloverIndexCreated);
+        builder.startObject(Fields.CONDITIONS);
+        for (Map.Entry<String, Boolean> entry : conditionStatus) {
+            builder.field(entry.getKey(), entry.getValue());
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    static final class Fields {
+        static final String NEW_INDEX = "new_index";
+        static final String OLD_INDEX = "old_index";
+        static final String SIMULATED = "simulated";
+        static final String ROLLED_OVER = "rolled_over";
+        static final String ROLLOVER_INDEX_CREATED = "rollover_index_created";
+        static final String CONDITIONS = "conditions";
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
index fa8782aca03..05bc674b8d3 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -47,7 +47,11 @@ import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.util.AbstractMap;
+import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * Main class to swap the index pointed to by an alias, given some predicates
@@ -105,9 +109,23 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
                     DocsStats docsStats = indicesStatsResponse.getTotal().getDocs();
                     long docCount = docsStats == null ? 0 : docsStats.getCount();
                     long indexAge = System.currentTimeMillis() - sourceIndex.getCreationDate();
-                    if (satisfiesConditions(rolloverRequest.getConditions(), docCount, indexAge)) {
-                        final String rolloverIndexName = generateRolloverIndexName(sourceIndexName);
-                        boolean createRolloverIndex = metaData.index(rolloverIndexName) == null;
+                    final Set<Map.Entry<Condition, Boolean>> evaluatedConditions =
+                        evaluateConditions(rolloverRequest.getConditions(), docCount, indexAge);
+                    final Set<Map.Entry<String, Boolean>> conditionStatus = evaluatedConditions.stream()
+                        .map(entry -> new AbstractMap.SimpleEntry<>(entry.getKey().toString(), entry.getValue()))
+                        .collect(Collectors.toSet());
+                    final String rolloverIndexName = generateRolloverIndexName(sourceIndexName);
+                    final boolean createRolloverIndex = metaData.index(rolloverIndexName) == null;
+                    if (rolloverRequest.isSimulate()) {
+                        listener.onResponse(
+                            new RolloverResponse(sourceIndexName, rolloverIndexName, conditionStatus, true, false,
+                                false));
+                        return;
+                    }
+                    if (conditionStatus.stream().allMatch(Map.Entry::getValue)) {
+                        final RolloverResponse rolloverResponse =
+                            new RolloverResponse(sourceIndexName, rolloverIndexName, conditionStatus, false, true,
+                                createRolloverIndex);
                         if (createRolloverIndex) {
                             CreateIndexClusterStateUpdateRequest updateRequest =
                                 prepareCreateIndexRequest(rolloverIndexName, rolloverRequest);
@@ -116,7 +134,7 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
                                 public void onResponse(ClusterStateUpdateResponse response) {
                                     indexAliasesService.indicesAliases(
                                         prepareIndicesAliasesRequest(sourceIndexName, rolloverIndexName, rolloverRequest),
-                                        new IndicesAliasesListener(sourceIndexName, rolloverIndexName, listener));
+                                        new IndicesAliasesListener(rolloverResponse, listener));
                                 }
 
                                 @Override
@@ -132,11 +150,14 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
                         } else {
                             indexAliasesService.indicesAliases(
                                 prepareIndicesAliasesRequest(sourceIndexName, rolloverIndexName, rolloverRequest),
-                                new IndicesAliasesListener(sourceIndexName, rolloverIndexName, listener));
+                                new IndicesAliasesListener(rolloverResponse, listener));
                         }
                     } else {
                         // conditions not met
-                        listener.onResponse(new RolloverResponse(sourceIndexName, sourceIndexName));
+                        listener.onResponse(
+                            new RolloverResponse(sourceIndexName, sourceIndexName, conditionStatus, false, false,
+                                false)
+                        );
                     }
                 }
 
@@ -152,18 +173,16 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
     private static final class IndicesAliasesListener implements ActionListener<ClusterStateUpdateResponse> {
 
         private final ActionListener<RolloverResponse> listener;
-        private final String oldIndex;
-        private final String newIndex;
+        private final RolloverResponse response;
 
-        public IndicesAliasesListener(String oldIndex, String newIndex, ActionListener<RolloverResponse> listener) {
-            this.oldIndex = oldIndex;
-            this.newIndex = newIndex;
+        public IndicesAliasesListener(RolloverResponse response, ActionListener<RolloverResponse> listener) {
+            this.response = response;
             this.listener = listener;
         }
 
         @Override
         public void onResponse(ClusterStateUpdateResponse clusterStateUpdateResponse) {
-            listener.onResponse(new RolloverResponse(oldIndex, newIndex));
+            listener.onResponse(response);
         }
 
         @Override
@@ -199,24 +218,21 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
         return String.join("-", indexPrefix, String.valueOf(counter));
     }
 
-    static boolean satisfiesConditions(Set<Condition> conditions, long docCount, long indexAge) {
+    static Set<Map.Entry<Condition, Boolean>> evaluateConditions(Set<Condition> conditions, long docCount, long indexAge) {
+        Set<Map.Entry<Condition, Boolean>> result = new HashSet<>(conditions.size());
         for (Condition condition: conditions) {
             if (condition instanceof Condition.MaxAge) {
                 Condition.MaxAge maxAge = (Condition.MaxAge) condition;
                 final TimeValue age = TimeValue.timeValueMillis(indexAge);
-                if (maxAge.matches(age) == false) {
-                    return false;
-                }
+                result.add(new AbstractMap.SimpleEntry<>(condition, maxAge.matches(age)));
             } else if (condition instanceof Condition.MaxDocs) {
                 final Condition.MaxDocs maxDocs = (Condition.MaxDocs) condition;
-                if (maxDocs.matches(docCount) == false) {
-                    return false;
-                }
+                result.add(new AbstractMap.SimpleEntry<>(condition, maxDocs.matches(docCount)));
             } else {
                 throw new IllegalArgumentException("unknown condition [" + condition.getClass().getSimpleName() + "]");
             }
         }
-        return true;
+        return result;
     }
 
     static void validate(MetaData metaData, RolloverRequest request) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
index 50ec45e9229..b56b04166fb 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
@@ -20,20 +20,14 @@
 package org.elasticsearch.rest.action.admin.indices;
 
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
-import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestResponse;
-import org.elasticsearch.rest.action.support.RestBuilderListener;
-
-import static org.elasticsearch.rest.RestStatus.OK;
+import org.elasticsearch.rest.action.support.RestToXContentListener;
 
 /**
  *
@@ -45,6 +39,7 @@ public class RestRolloverIndexAction extends BaseRestHandler {
         super(settings, client);
         controller.registerHandler(RestRequest.Method.PUT, "/{alias}/_rollover", this);
         controller.registerHandler(RestRequest.Method.POST, "/{alias}/_rollover", this);
+        controller.registerHandler(RestRequest.Method.GET, "/{alias}/_rollover", this);
     }
 
     @Override
@@ -56,17 +51,9 @@ public class RestRolloverIndexAction extends BaseRestHandler {
         if (request.hasContent()) {
             rolloverIndexRequest.source(request.content());
         }
+        rolloverIndexRequest.simulate(request.method() == RestRequest.Method.GET || request.paramAsBoolean("simulate", false));
         rolloverIndexRequest.timeout(request.paramAsTime("timeout", rolloverIndexRequest.timeout()));
         rolloverIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", rolloverIndexRequest.masterNodeTimeout()));
-        client.admin().indices().rolloverIndex(rolloverIndexRequest, new RestBuilderListener<RolloverResponse>(channel) {
-            @Override
-            public RestResponse buildResponse(RolloverResponse rolloverResponse, XContentBuilder builder) throws Exception {
-                builder.startObject();
-                builder.field("old_index", rolloverResponse.getOldIndex());
-                builder.field("new_index", rolloverResponse.getNewIndex());
-                builder.endObject();
-                return new BytesRestResponse(OK, builder);
-            }
-        });
+        client.admin().indices().rolloverIndex(rolloverIndexRequest, new RestToXContentListener<>(channel));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java
new file mode 100644
index 00000000000..3ce1b84d82c
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.rollover;
+
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class ConditionTests extends ESTestCase {
+
+    public void testMaxAge() throws Exception {
+        final Condition.MaxAge maxAge = new Condition.MaxAge(TimeValue.timeValueMillis(10));
+        assertThat(maxAge.matches(TimeValue.timeValueMillis(randomIntBetween(0, 9))), equalTo(false));
+        assertThat(maxAge.matches(TimeValue.timeValueMillis(randomIntBetween(10, 100))), equalTo(true));
+    }
+
+    public void testMaxDocs() throws Exception {
+        final Condition.MaxDocs maxDocs = new Condition.MaxDocs(10L);
+        assertThat(maxDocs.matches((long) randomIntBetween(0, 9)), equalTo(false));
+        assertThat(maxDocs.matches((long) randomIntBetween(10, 100)), equalTo(true));
+
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
index 3ab9c8cc040..9c0f3b7b96a 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
@@ -25,6 +25,8 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESIntegTestCase;
 
+import java.util.Map;
+
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -36,6 +38,10 @@ public class RolloverIT extends ESIntegTestCase {
         final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias").get();
         assertThat(response.getOldIndex(), equalTo("test_index"));
         assertThat(response.getNewIndex(), equalTo("test_index-1"));
+        assertThat(response.isSimulate(), equalTo(false));
+        assertThat(response.isRolledOver(), equalTo(true));
+        assertThat(response.isRolloverIndexCreated(), equalTo(true));
+        assertThat(response.getConditionStatus().size(), equalTo(0));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
         final IndexMetaData oldIndex = state.metaData().index("test_index");
         assertFalse(oldIndex.getAliases().containsKey("test_alias"));
@@ -50,6 +56,10 @@ public class RolloverIT extends ESIntegTestCase {
         final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias").get();
         assertThat(response.getOldIndex(), equalTo("test_index"));
         assertThat(response.getNewIndex(), equalTo("test_index-1"));
+        assertThat(response.isSimulate(), equalTo(false));
+        assertThat(response.isRolledOver(), equalTo(true));
+        assertThat(response.isRolloverIndexCreated(), equalTo(true));
+        assertThat(response.getConditionStatus().size(), equalTo(0));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
         final IndexMetaData oldIndex = state.metaData().index("test_index");
         assertFalse(oldIndex.getAliases().containsKey("test_alias"));
@@ -57,6 +67,24 @@ public class RolloverIT extends ESIntegTestCase {
         assertTrue(newIndex.getAliases().containsKey("test_alias"));
     }
 
+    public void testRolloverSimulate() throws Exception {
+        assertAcked(prepareCreate("test_index").addAlias(new Alias("test_alias")).get());
+        index("test_index", "type1", "1", "field", "value");
+        flush("test_index");
+        final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias").simulate(true).get();
+        assertThat(response.getOldIndex(), equalTo("test_index"));
+        assertThat(response.getNewIndex(), equalTo("test_index-1"));
+        assertThat(response.isSimulate(), equalTo(true));
+        assertThat(response.isRolledOver(), equalTo(false));
+        assertThat(response.isRolloverIndexCreated(), equalTo(false));
+        assertThat(response.getConditionStatus().size(), equalTo(0));
+        final ClusterState state = client().admin().cluster().prepareState().get().getState();
+        final IndexMetaData oldIndex = state.metaData().index("test_index");
+        assertTrue(oldIndex.getAliases().containsKey("test_alias"));
+        final IndexMetaData newIndex = state.metaData().index("test_index-1");
+        assertNull(newIndex);
+    }
+
     public void testRolloverConditionsNotMet() throws Exception {
         assertAcked(prepareCreate("test_index").addAlias(new Alias("test_alias")).get());
         index("test_index", "type1", "1", "field", "value");
@@ -65,6 +93,13 @@ public class RolloverIT extends ESIntegTestCase {
             .addMaxIndexAgeCondition(TimeValue.timeValueHours(4)).get();
         assertThat(response.getOldIndex(), equalTo("test_index"));
         assertThat(response.getNewIndex(), equalTo("test_index"));
+        assertThat(response.isSimulate(), equalTo(false));
+        assertThat(response.isRolledOver(), equalTo(false));
+        assertThat(response.isRolloverIndexCreated(), equalTo(false));
+        assertThat(response.getConditionStatus().size(), equalTo(1));
+        final Map.Entry<String, Boolean> conditionEntry = response.getConditionStatus().iterator().next();
+        assertThat(conditionEntry.getKey(), equalTo(new Condition.MaxAge(TimeValue.timeValueHours(4)).toString()));
+        assertThat(conditionEntry.getValue(), equalTo(false));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
         final IndexMetaData oldIndex = state.metaData().index("test_index");
         assertTrue(oldIndex.getAliases().containsKey("test_alias"));
@@ -81,6 +116,9 @@ public class RolloverIT extends ESIntegTestCase {
         final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias").get();
         assertThat(response.getOldIndex(), equalTo("test_index"));
         assertThat(response.getNewIndex(), equalTo("test_index-1"));
+        assertThat(response.isSimulate(), equalTo(false));
+        assertThat(response.isRolledOver(), equalTo(true));
+        assertThat(response.isRolloverIndexCreated(), equalTo(false));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
         final IndexMetaData oldIndex = state.metaData().index("test_index");
         assertFalse(oldIndex.getAliases().containsKey("test_alias"));
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
index ff121ff3dfb..636378a5872 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
@@ -27,44 +27,13 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.test.ESTestCase;
 
-import java.util.Collections;
-import java.util.Set;
-
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 
 public class TransportRolloverActionTests extends ESTestCase {
 
-    public void testSatisfyConditions() throws Exception {
-        Set<Condition> conditions = Collections.emptySet();
-        assertTrue(TransportRolloverAction.satisfiesConditions(conditions, randomLong(),
-            randomLong()));
-
-        conditions = Collections.singleton(new Condition.MaxAge(TimeValue.timeValueMillis(10)));
-        assertTrue(TransportRolloverAction.satisfiesConditions(conditions, randomLong(),
-            TimeValue.timeValueMillis(randomIntBetween(10, 100)).getMillis()));
-        assertFalse(TransportRolloverAction.satisfiesConditions(conditions, randomLong(),
-            TimeValue.timeValueMillis(randomIntBetween(0, 9)).getMillis()));
-
-        conditions = Collections.singleton(new Condition.MaxDocs(10L));
-        assertTrue(TransportRolloverAction.satisfiesConditions(conditions, randomIntBetween(10, 100),
-            randomLong()));
-        assertFalse(TransportRolloverAction.satisfiesConditions(conditions, randomIntBetween(1, 9),
-            randomLong()));
-
-        conditions = Sets.newHashSet(new Condition.MaxAge(TimeValue.timeValueMillis(100)), new Condition.MaxDocs(1000L));
-        assertTrue(TransportRolloverAction.satisfiesConditions(conditions, randomIntBetween(1000, 1500),
-            TimeValue.timeValueMillis(randomIntBetween(100, 1000)).getMillis()));
-        assertFalse(TransportRolloverAction.satisfiesConditions(conditions, randomIntBetween(1, 999),
-            TimeValue.timeValueMillis(randomIntBetween(100, 1000)).getMillis()));
-        assertFalse(TransportRolloverAction.satisfiesConditions(conditions, randomIntBetween(1000, 1500),
-            TimeValue.timeValueMillis(randomIntBetween(0, 99)).getMillis()));
-    }
-
     public void testCreateUpdateAliasRequest() throws Exception {
         String sourceAlias = randomAsciiOfLength(10);
         String sourceIndex = randomAsciiOfLength(10);
@@ -114,35 +83,27 @@ public class TransportRolloverActionTests extends ESTestCase {
                 .putAlias(AliasMetaData.builder(aliasWithMultipleIndices))
             ).build();
 
-        try {
-            TransportRolloverAction.validate(metaData, new RolloverRequest(aliasWithMultipleIndices));
-            fail("expected to throw exception");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("source alias maps to multiple indices"));
-        }
-
-        try {
-            TransportRolloverAction.validate(metaData, new RolloverRequest(randomFrom(index1, index2)));
-            fail("expected to throw exception");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("source alias is a concrete index"));
-        }
-
-        try {
-            TransportRolloverAction.validate(metaData, new RolloverRequest(randomAsciiOfLength(5)));
-            fail("expected to throw exception");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("source alias does not exist"));
-        }
-
+        expectThrows(IllegalArgumentException.class, () ->
+            TransportRolloverAction.validate(metaData, new RolloverRequest(aliasWithMultipleIndices)));
+        expectThrows(IllegalArgumentException.class, () ->
+            TransportRolloverAction.validate(metaData, new RolloverRequest(randomFrom(index1, index2))));
+        expectThrows(IllegalArgumentException.class, () ->
+            TransportRolloverAction.validate(metaData, new RolloverRequest(randomAsciiOfLength(5)))
+        );
         TransportRolloverAction.validate(metaData, new RolloverRequest(alias));
     }
 
     public void testGenerateRolloverIndexName() throws Exception {
-        String index = randomAsciiOfLength(10);
-        assertThat(TransportRolloverAction.generateRolloverIndexName(index), not(equalTo(index)));
-        assertThat(TransportRolloverAction.generateRolloverIndexName("index"), equalTo("index-1"));
-        assertThat(TransportRolloverAction.generateRolloverIndexName("index-1"), equalTo("index-2"));
+        String indexNotEndingInNumbers = randomAsciiOfLength(10) + "A";
+        assertThat(TransportRolloverAction.generateRolloverIndexName(indexNotEndingInNumbers),
+            not(equalTo(indexNotEndingInNumbers)));
+        assertThat(TransportRolloverAction.generateRolloverIndexName(indexNotEndingInNumbers),
+            equalTo(indexNotEndingInNumbers + "-1"));
+        int num = randomIntBetween(0, 100);
+        final String indexPrefix = randomAsciiOfLength(10);
+        String indexEndingInNumbers = indexPrefix + "-" + num;
+        assertThat(TransportRolloverAction.generateRolloverIndexName(indexEndingInNumbers),
+            equalTo(indexPrefix + "-" + (num + 1)));
         assertThat(TransportRolloverAction.generateRolloverIndexName("index-name-1"), equalTo("index-name-2"));
         assertThat(TransportRolloverAction.generateRolloverIndexName("index-name"), equalTo("index-name-1"));
     }

From d8d60d294ea89a4e94749a86b394257bb431ee84 Mon Sep 17 00:00:00 2001
From: Areek Zillur <areek.zillur@elasticsearch.com>
Date: Mon, 6 Jun 2016 18:54:20 -0400
Subject: [PATCH 0674/1311] remove put method

---
 .../rest/action/admin/indices/RestRolloverIndexAction.java       | 1 -
 1 file changed, 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
index b56b04166fb..963080c3122 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
@@ -37,7 +37,6 @@ public class RestRolloverIndexAction extends BaseRestHandler {
     @Inject
     public RestRolloverIndexAction(Settings settings, RestController controller, Client client) {
         super(settings, client);
-        controller.registerHandler(RestRequest.Method.PUT, "/{alias}/_rollover", this);
         controller.registerHandler(RestRequest.Method.POST, "/{alias}/_rollover", this);
         controller.registerHandler(RestRequest.Method.GET, "/{alias}/_rollover", this);
     }

From a088d367f7d22267cc16bbcb7146f26a27168bb1 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Mon, 6 Jun 2016 15:58:56 -0700
Subject: [PATCH 0675/1311] More comments.

---
 .../elasticsearch/painless/node/EChain.java   | 12 +++++-----
 .../elasticsearch/painless/node/SDefEach.java | 23 +++++++++++++++++++
 .../elasticsearch/painless/node/SEach.java    |  3 +++
 3 files changed, 32 insertions(+), 6 deletions(-)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDefEach.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index df0d670320d..9b2968db565 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -269,14 +269,14 @@ public final class EChain extends AExpression {
      */
     @Override
     void write(MethodWriter writer) {
+        writer.writeDebugInfo(location);
+
         // For the case where the chain represents a String concatenation
-        // we must first write debug information, and then depending on the
-        // Java version write a StringBuilder or track types going onto the
-        // stack.  This must be done before the links in the chain are read
-        // because we need the StringBuilder to be placed on the stack
-        // ahead of any potential concatenation arguments.
+        // we must, depending on the Java version, write a StringBuilder or
+        // track types going onto the stack.  This must be done before the
+        // links in the chain are read because we need the StringBuilder to
+        // be placed on the stack ahead of any potential concatenation arguments.
         if (cat) {
-            writer.writeDebugInfo(location);
             writer.writeNewStrings();
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDefEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDefEach.java
new file mode 100644
index 00000000000..000fc42c7a0
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDefEach.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+public class SDefEach {
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index 0d7e81c2073..3266fdc9ba6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -88,6 +88,9 @@ public class SEach extends AStatement {
             Type itr = Definition.getType("Iterator");
 
             variable = variables.addVariable(location, type, name, true, false);
+
+            // We must store the iterator as a variable for securing a slot on the stack, and
+            // also add the location offset to make the name unique in case of nested for each loops.
             iterator = variables.addVariable(location, itr, "#itr" + location.getOffset(), true, false);
 
             method = expression.actual.struct.methods.get(new MethodKey("iterator", 0));

From aeaf39f36d512a2dc37d37916198c1101a14a9bd Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Mon, 6 Jun 2016 16:07:12 -0700
Subject: [PATCH 0676/1311] Added a for each stub node for a target type of
 def.

---
 .../painless/node/SArrayEach.java             |   6 +-
 .../elasticsearch/painless/node/SDefEach.java | 134 +++++++++++++++++-
 .../elasticsearch/painless/node/SEach.java    |   5 +-
 3 files changed, 139 insertions(+), 6 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SArrayEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SArrayEach.java
index 9abb7a18a9a..032483f59f2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SArrayEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SArrayEach.java
@@ -46,8 +46,7 @@ class SArrayEach extends AStatement {
     Type indexed = null;
     Cast cast = null;
 
-    SArrayEach(final Location location, final int maxLoopCounter,
-               final String type, final String name, final AExpression expression, final SBlock block) {
+    public SArrayEach(Location location, int maxLoopCounter, String type, String name, AExpression expression, SBlock block) {
         super(location);
 
         this.maxLoopCounter = maxLoopCounter;
@@ -59,6 +58,9 @@ class SArrayEach extends AStatement {
 
     @Override
     AStatement analyze(Variables variables) {
+        // Note that we do not need to analyze the expression as this must already be done
+        // in the parent to determine that the for each target type is an array.
+
         final Type type;
 
         try {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDefEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDefEach.java
index 000fc42c7a0..ab823698d3e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDefEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDefEach.java
@@ -19,5 +19,137 @@
 
 package org.elasticsearch.painless.node;
 
-public class SDefEach {
+import org.elasticsearch.painless.Location;
+import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.Variables;
+
+public class SDefEach extends AStatement {
+
+    final int maxLoopCounter;
+    final String type;
+    final String name;
+    AExpression expression;
+    AStatement block;
+
+    public SDefEach(Location location, int maxLoopCounter, String type, String name, AExpression expression, SBlock block) {
+        super(location);
+
+        this.maxLoopCounter = maxLoopCounter;
+        this.type = type;
+        this.name = name;
+        this.expression = expression;
+        this.block = block;
+    }
+
+    @Override
+    AStatement analyze(Variables variables) {
+        // Note that we do not need to analyze the expression as this must already be done
+        // in the parent to determine that the for each target type is def.
+
+        throw location.createError(new UnsupportedOperationException("Cannot execute for each against def type."));
+
+        /*
+        try {
+            type = Definition.getType(this.type);
+        } catch (IllegalArgumentException exception) {
+            throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
+        }
+
+        variables.incrementScope();
+
+        Type itr = Definition.getType("Iterator");
+
+        variable = variables.addVariable(location, type, name, true, false);
+
+        // We must store the iterator as a variable for securing a slot on the stack, and
+        // also add the location offset to make the name unique in case of nested for each loops.
+        iterator = variables.addVariable(location, itr, "#itr" + location.getOffset(), true, false);
+
+        method = expression.actual.struct.methods.get(new MethodKey("iterator", 0));
+
+        if (method == null) {
+            throw location.createError(new IllegalArgumentException(
+                "Unable to create iterator for the type [" + expression.actual.name + "]."));
+        }
+
+        hasNext = itr.struct.methods.get(new MethodKey("hasNext", 0));
+
+        if (hasNext == null) {
+            throw location.createError(new IllegalArgumentException("Method [hasNext] does not exist for type [Iterator]."));
+        } else if (hasNext.rtn.sort != Sort.BOOL) {
+            throw location.createError(new IllegalArgumentException("Method [hasNext] does not return type [boolean]."));
+        }
+
+        next = itr.struct.methods.get(new MethodKey("next", 0));
+
+        if (next == null) {
+            throw location.createError(new IllegalArgumentException("Method [next] does not exist for type [Iterator]."));
+        } else if (next.rtn.sort != Sort.DEF) {
+            throw location.createError(new IllegalArgumentException("Method [next] does not return type [def]."));
+        }
+
+        cast = AnalyzerCaster.getLegalCast(location, Definition.DEF_TYPE, type, true, true);
+
+        if (block == null) {
+            throw location.createError(new IllegalArgumentException("Extraneous for each loop."));
+        }
+
+        block.beginLoop = true;
+        block.inLoop = true;
+        block = block.analyze(variables);
+        block.statementCount = Math.max(1, block.statementCount);
+
+        if (block.loopEscape && !block.anyContinue) {
+            throw createError(new IllegalArgumentException("Extraneous for loop."));
+        }
+
+        statementCount = 1;
+
+        if (maxLoopCounter > 0) {
+            loopCounterSlot = variables.getVariable(location, "#loop").slot;
+        }
+
+        variables.decrementScope();
+
+        return this;
+        */
+    }
+
+    @Override
+    void write(MethodWriter writer) {
+        throw location.createError(new IllegalStateException("Illegal tree structure."));
+
+        /*
+        writer.writeStatementOffset(location);
+
+        expression.write(writer);
+
+        if (java.lang.reflect.Modifier.isInterface(method.owner.clazz.getModifiers())) {
+            writer.invokeInterface(method.owner.type, method.method);
+        } else {
+            writer.invokeVirtual(method.owner.type, method.method);
+        }
+
+        writer.visitVarInsn(iterator.type.type.getOpcode(Opcodes.ISTORE), iterator.slot);
+
+        Label begin = new Label();
+        Label end = new Label();
+
+        writer.mark(begin);
+
+        writer.visitVarInsn(iterator.type.type.getOpcode(Opcodes.ILOAD), iterator.slot);
+        writer.invokeInterface(hasNext.owner.type, hasNext.method);
+        writer.ifZCmp(MethodWriter.EQ, end);
+
+        writer.visitVarInsn(iterator.type.type.getOpcode(Opcodes.ILOAD), iterator.slot);
+        writer.invokeInterface(next.owner.type, next.method);
+        writer.writeCast(cast);
+        writer.visitVarInsn(variable.type.type.getOpcode(Opcodes.ISTORE), variable.slot);
+
+        block.write(writer);
+
+        writer.goTo(begin);
+        writer.mark(end);
+        */
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index 3266fdc9ba6..79395a4acc7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -51,8 +51,7 @@ public class SEach extends AStatement {
     Method next = null;
     Cast cast = null;
 
-    public SEach(final Location location, final int maxLoopCounter,
-                 final String type, final String name, final AExpression expression, final SBlock block) {
+    public SEach(Location location, int maxLoopCounter, String type, String name, AExpression expression, SBlock block) {
         super(location);
 
         this.maxLoopCounter = maxLoopCounter;
@@ -73,7 +72,7 @@ public class SEach extends AStatement {
         if (sort == Sort.ARRAY) {
             return new SArrayEach(location, maxLoopCounter, type, name, expression, (SBlock)block).copy(this).analyze(variables);
         } else if (sort == Sort.DEF) {
-            throw location.createError(new UnsupportedOperationException("Cannot execute for each against def type."));
+            return new SDefEach(location, maxLoopCounter, type, name, expression, (SBlock)block).copy(this).analyze(variables);
         } else if (Iterable.class.isAssignableFrom(expression.actual.clazz)) {
             final Type type;
 

From d331f8768e31ffde0af5d29e1ab94ec216e8d55e Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 6 Jun 2016 19:41:37 -0400
Subject: [PATCH 0677/1311] implement dynamic case for iterables

---
 .../java/org/elasticsearch/painless/Def.java  |  18 ++
 .../elasticsearch/painless/DefBootstrap.java  |   4 +
 .../elasticsearch/painless/node/SDefEach.java | 155 ------------------
 .../elasticsearch/painless/node/SEach.java    |  25 ++-
 .../painless/BasicStatementTests.java         |  20 +++
 5 files changed, 59 insertions(+), 163 deletions(-)
 delete mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDefEach.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index f27f0ab20a9..269386bf944 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -27,6 +27,7 @@ import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodHandles.Lookup;
 import java.lang.invoke.MethodType;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
@@ -103,6 +104,8 @@ public final class Def {
     private static final MethodHandle LIST_GET;
     /** pointer to List.set(int,Object) */
     private static final MethodHandle LIST_SET;
+    /** pointer to Iterable.iterator() */
+    private static final MethodHandle ITERATOR;
     /** factory for arraylength MethodHandle (intrinsic) from Java 9 */
     private static final MethodHandle JAVA9_ARRAY_LENGTH_MH_FACTORY;
 
@@ -114,6 +117,7 @@ public final class Def {
             MAP_PUT  = lookup.findVirtual(Map.class , "put", MethodType.methodType(Object.class, Object.class, Object.class));
             LIST_GET = lookup.findVirtual(List.class, "get", MethodType.methodType(Object.class, int.class));
             LIST_SET = lookup.findVirtual(List.class, "set", MethodType.methodType(Object.class, int.class, Object.class));
+            ITERATOR = lookup.findVirtual(Iterable.class, "iterator", MethodType.methodType(Iterator.class));
         } catch (final ReflectiveOperationException roe) {
             throw new AssertionError(roe);
         }
@@ -374,6 +378,20 @@ public final class Def {
         throw new IllegalArgumentException("Attempting to address a non-array type " +
                                            "[" + receiverClass.getCanonicalName() + "] as an array.");
     }
+    
+    /**
+     * Returns a method handle to do iteration (for enhanced for loop)
+     * @param receiverClass Class of the array to load the value from
+     * @return a MethodHandle that accepts the receiver as first argument, returns iterator
+     */
+    static MethodHandle lookupIterator(Class<?> receiverClass) {
+        if (Iterable.class.isAssignableFrom(receiverClass)) {
+            return ITERATOR;
+        } else {
+            // TODO: arrays
+            throw new IllegalArgumentException("Cannot iterate over [" + receiverClass.getCanonicalName() + "]");
+        }
+    }
 
     // NOTE: Below methods are not cached, instead invoked directly because they are performant.
     //       We also check for Long values first when possible since the type is more
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index 40b9cc6cbe8..1d2ca64abe0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -56,6 +56,8 @@ public final class DefBootstrap {
     public static final int ARRAY_LOAD = 3;
     /** static bootstrap parameter indicating a dynamic array store, e.g. foo[bar] = baz */
     public static final int ARRAY_STORE = 4;
+    /** static bootstrap parameter indicating a dynamic iteration, e.g. for (x : y) */
+    public static final int ITERATOR = 5;
 
     /**
      * CallSite that implements the polymorphic inlining cache (PIC).
@@ -103,6 +105,8 @@ public final class DefBootstrap {
                     return Def.lookupArrayLoad(clazz);
                 case ARRAY_STORE:
                     return Def.lookupArrayStore(clazz);
+                case ITERATOR:
+                    return Def.lookupIterator(clazz);
                 default: throw new AssertionError();
             }
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDefEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDefEach.java
deleted file mode 100644
index ab823698d3e..00000000000
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDefEach.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless.node;
-
-import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.MethodWriter;
-import org.elasticsearch.painless.Variables;
-
-public class SDefEach extends AStatement {
-
-    final int maxLoopCounter;
-    final String type;
-    final String name;
-    AExpression expression;
-    AStatement block;
-
-    public SDefEach(Location location, int maxLoopCounter, String type, String name, AExpression expression, SBlock block) {
-        super(location);
-
-        this.maxLoopCounter = maxLoopCounter;
-        this.type = type;
-        this.name = name;
-        this.expression = expression;
-        this.block = block;
-    }
-
-    @Override
-    AStatement analyze(Variables variables) {
-        // Note that we do not need to analyze the expression as this must already be done
-        // in the parent to determine that the for each target type is def.
-
-        throw location.createError(new UnsupportedOperationException("Cannot execute for each against def type."));
-
-        /*
-        try {
-            type = Definition.getType(this.type);
-        } catch (IllegalArgumentException exception) {
-            throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
-        }
-
-        variables.incrementScope();
-
-        Type itr = Definition.getType("Iterator");
-
-        variable = variables.addVariable(location, type, name, true, false);
-
-        // We must store the iterator as a variable for securing a slot on the stack, and
-        // also add the location offset to make the name unique in case of nested for each loops.
-        iterator = variables.addVariable(location, itr, "#itr" + location.getOffset(), true, false);
-
-        method = expression.actual.struct.methods.get(new MethodKey("iterator", 0));
-
-        if (method == null) {
-            throw location.createError(new IllegalArgumentException(
-                "Unable to create iterator for the type [" + expression.actual.name + "]."));
-        }
-
-        hasNext = itr.struct.methods.get(new MethodKey("hasNext", 0));
-
-        if (hasNext == null) {
-            throw location.createError(new IllegalArgumentException("Method [hasNext] does not exist for type [Iterator]."));
-        } else if (hasNext.rtn.sort != Sort.BOOL) {
-            throw location.createError(new IllegalArgumentException("Method [hasNext] does not return type [boolean]."));
-        }
-
-        next = itr.struct.methods.get(new MethodKey("next", 0));
-
-        if (next == null) {
-            throw location.createError(new IllegalArgumentException("Method [next] does not exist for type [Iterator]."));
-        } else if (next.rtn.sort != Sort.DEF) {
-            throw location.createError(new IllegalArgumentException("Method [next] does not return type [def]."));
-        }
-
-        cast = AnalyzerCaster.getLegalCast(location, Definition.DEF_TYPE, type, true, true);
-
-        if (block == null) {
-            throw location.createError(new IllegalArgumentException("Extraneous for each loop."));
-        }
-
-        block.beginLoop = true;
-        block.inLoop = true;
-        block = block.analyze(variables);
-        block.statementCount = Math.max(1, block.statementCount);
-
-        if (block.loopEscape && !block.anyContinue) {
-            throw createError(new IllegalArgumentException("Extraneous for loop."));
-        }
-
-        statementCount = 1;
-
-        if (maxLoopCounter > 0) {
-            loopCounterSlot = variables.getVariable(location, "#loop").slot;
-        }
-
-        variables.decrementScope();
-
-        return this;
-        */
-    }
-
-    @Override
-    void write(MethodWriter writer) {
-        throw location.createError(new IllegalStateException("Illegal tree structure."));
-
-        /*
-        writer.writeStatementOffset(location);
-
-        expression.write(writer);
-
-        if (java.lang.reflect.Modifier.isInterface(method.owner.clazz.getModifiers())) {
-            writer.invokeInterface(method.owner.type, method.method);
-        } else {
-            writer.invokeVirtual(method.owner.type, method.method);
-        }
-
-        writer.visitVarInsn(iterator.type.type.getOpcode(Opcodes.ISTORE), iterator.slot);
-
-        Label begin = new Label();
-        Label end = new Label();
-
-        writer.mark(begin);
-
-        writer.visitVarInsn(iterator.type.type.getOpcode(Opcodes.ILOAD), iterator.slot);
-        writer.invokeInterface(hasNext.owner.type, hasNext.method);
-        writer.ifZCmp(MethodWriter.EQ, end);
-
-        writer.visitVarInsn(iterator.type.type.getOpcode(Opcodes.ILOAD), iterator.slot);
-        writer.invokeInterface(next.owner.type, next.method);
-        writer.writeCast(cast);
-        writer.visitVarInsn(variable.type.type.getOpcode(Opcodes.ISTORE), variable.slot);
-
-        block.write(writer);
-
-        writer.goTo(begin);
-        writer.mark(end);
-        */
-    }
-}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index 79395a4acc7..309c6d97967 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.AnalyzerCaster;
+import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Definition.Method;
@@ -33,6 +34,8 @@ import org.elasticsearch.painless.Variables.Variable;
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
 
+import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
+
 /**
  * Represents a for-each loop shortcut for iterables.  Defers to other S-nodes for non-iterable types.
  */
@@ -71,9 +74,7 @@ public class SEach extends AStatement {
 
         if (sort == Sort.ARRAY) {
             return new SArrayEach(location, maxLoopCounter, type, name, expression, (SBlock)block).copy(this).analyze(variables);
-        } else if (sort == Sort.DEF) {
-            return new SDefEach(location, maxLoopCounter, type, name, expression, (SBlock)block).copy(this).analyze(variables);
-        } else if (Iterable.class.isAssignableFrom(expression.actual.clazz)) {
+        } else if (sort == Sort.DEF || Iterable.class.isAssignableFrom(expression.actual.clazz)) {
             final Type type;
 
             try {
@@ -92,11 +93,15 @@ public class SEach extends AStatement {
             // also add the location offset to make the name unique in case of nested for each loops.
             iterator = variables.addVariable(location, itr, "#itr" + location.getOffset(), true, false);
 
-            method = expression.actual.struct.methods.get(new MethodKey("iterator", 0));
+            if (sort == Sort.DEF) {
+                method = null;
+            } else {
+                method = expression.actual.struct.methods.get(new MethodKey("iterator", 0));
 
-            if (method == null) {
-                throw location.createError(new IllegalArgumentException(
-                    "Unable to create iterator for the type [" + expression.actual.name + "]."));
+                if (method == null) {
+                    throw location.createError(new IllegalArgumentException(
+                            "Unable to create iterator for the type [" + expression.actual.name + "]."));
+                }
             }
 
             hasNext = itr.struct.methods.get(new MethodKey("hasNext", 0));
@@ -150,7 +155,11 @@ public class SEach extends AStatement {
 
         expression.write(writer);
 
-        if (java.lang.reflect.Modifier.isInterface(method.owner.clazz.getModifiers())) {
+        if (method == null) {
+            Type itr = Definition.getType("Iterator");
+            String desc = org.objectweb.asm.Type.getMethodDescriptor(itr.type, Definition.DEF_TYPE.type);
+            writer.invokeDynamic("iterator", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ITERATOR);
+        } else if (java.lang.reflect.Modifier.isInterface(method.owner.clazz.getModifiers())) {
             writer.invokeInterface(method.owner.type, method.method);
         } else {
             writer.invokeVirtual(method.owner.type, method.method);
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
index 6ddb4067426..64137bfbcb3 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
@@ -134,6 +134,16 @@ public class BasicStatementTests extends ScriptTestCase {
             " String cat = ''; int total = 0;" +
             " for (Map.Entry e : m.entrySet()) { cat += e.getKey(); total += e.getValue(); } return cat + total"));
     }
+    
+    public void testIterableForEachStatementDef() {
+        assertEquals(6, exec("def l = new ArrayList(); l.add(1); l.add(2); l.add(3); int total = 0;" +
+            " for (int x : l) total += x; return total"));
+        assertEquals("123", exec("def l = new ArrayList(); l.add('1'); l.add('2'); l.add('3'); String cat = '';" +
+            " for (String x : l) cat += x; return cat"));
+        assertEquals("1236", exec("def m = new HashMap(); m.put('1', 1); m.put('2', 2); m.put('3', 3);" +
+            " String cat = ''; int total = 0;" +
+            " for (Map.Entry e : m.entrySet()) { cat += e.getKey(); total += e.getValue(); } return cat + total"));
+    }
 
     public void testArrayForEachStatement() {
         assertEquals(6, exec("int[] a = new int[3]; a[0] = 1; a[1] = 2; a[2] = 3; int total = 0;" +
@@ -143,6 +153,16 @@ public class BasicStatementTests extends ScriptTestCase {
         assertEquals(6, exec("int[][] i = new int[3][1]; i[0][0] = 1; i[1][0] = 2; i[2][0] = 3; int total = 0;" +
             " for (int[] j : i) total += j[0]; return total"));
     }
+    
+    @AwaitsFix(bugUrl = "working on it")
+    public void testArrayForEachStatementDef() {
+        assertEquals(6, exec("def a = new int[3]; a[0] = 1; a[1] = 2; a[2] = 3; int total = 0;" +
+            " for (int x : a) total += x; return total"));
+        assertEquals("123", exec("def a = new String[3]; a[0] = '1'; a[1] = '2'; a[2] = '3'; def total = '';" +
+            " for (String x : a) total += x; return total"));
+        assertEquals(6, exec("def i = new int[3][1]; i[0][0] = 1; i[1][0] = 2; i[2][0] = 3; int total = 0;" +
+            " for (int[] j : i) total += j[0]; return total"));
+    }
 
     public void testDeclarationStatement() {
         assertEquals((byte)2, exec("byte a = 2; return a;"));

From 2852e82ab9ff18aadda1cb22bf4380d6568719f0 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 6 Jun 2016 20:07:32 -0400
Subject: [PATCH 0678/1311] add def iteration over arrays

---
 .../java/org/elasticsearch/painless/Def.java  | 99 +++++++++++++++++++
 1 file changed, 99 insertions(+)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 269386bf944..614490d0295 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -379,6 +379,103 @@ public final class Def {
                                            "[" + receiverClass.getCanonicalName() + "] as an array.");
     }
     
+    /** Helper class for isolating MethodHandles and methods to get iterators over arrays
+     * (to emulate "enhanced for loop" using MethodHandles). These cause boxing, and are not as efficient
+     * as they could be, but works.
+     */
+    @SuppressWarnings("unused") // iterator() methods are are actually used, javac just does not know :)
+    private static final class ArrayIteratorHelper {
+        private static final Lookup PRIV_LOOKUP = MethodHandles.lookup();
+
+        private static final Map<Class<?>,MethodHandle> ARRAY_TYPE_MH_MAPPING = Collections.unmodifiableMap(
+            Stream.of(boolean[].class, byte[].class, short[].class, int[].class, long[].class,
+                char[].class, float[].class, double[].class, Object[].class)
+                .collect(Collectors.toMap(Function.identity(), type -> {
+                    try {
+                        return PRIV_LOOKUP.findStatic(PRIV_LOOKUP.lookupClass(), "iterator", MethodType.methodType(Iterator.class, type));
+                    } catch (ReflectiveOperationException e) {
+                        throw new AssertionError(e);
+                    }
+                }))
+        );
+
+        private static final MethodHandle OBJECT_ARRAY_MH = ARRAY_TYPE_MH_MAPPING.get(Object[].class);
+
+        static Iterator<Boolean> iterator(final boolean[] array) {
+            return new Iterator<Boolean>() {
+                int index = 0;
+                @Override public boolean hasNext() { return index < array.length; }
+                @Override public Boolean next() { return array[index++]; }
+            };
+        }
+        static Iterator<Byte> iterator(final byte[] array) {
+            return new Iterator<Byte>() {
+                int index = 0;
+                @Override public boolean hasNext() { return index < array.length; }
+                @Override public Byte next() { return array[index++]; }
+            };
+        }
+        static Iterator<Short> iterator(final short[] array) {
+            return new Iterator<Short>() {
+                int index = 0;
+                @Override public boolean hasNext() { return index < array.length; }
+                @Override public Short next() { return array[index++]; }
+            };
+        }
+        static Iterator<Integer> iterator(final int[] array) {
+            return new Iterator<Integer>() {
+                int index = 0;
+                @Override public boolean hasNext() { return index < array.length; }
+                @Override public Integer next() { return array[index++]; }
+            };
+        }
+        static Iterator<Long> iterator(final long[] array) {
+            return new Iterator<Long>() {
+                int index = 0;
+                @Override public boolean hasNext() { return index < array.length; }
+                @Override public Long next() { return array[index++]; }
+            };
+        }
+        static Iterator<Character> iterator(final char[] array) {
+            return new Iterator<Character>() {
+                int index = 0;
+                @Override public boolean hasNext() { return index < array.length; }
+                @Override public Character next() { return array[index++]; }
+            };
+        }
+        static Iterator<Float> iterator(final float[] array) {
+            return new Iterator<Float>() {
+                int index = 0;
+                @Override public boolean hasNext() { return index < array.length; }
+                @Override public Float next() { return array[index++]; }
+            };
+        }
+        static Iterator<Double> iterator(final double[] array) {
+            return new Iterator<Double>() {
+                int index = 0;
+                @Override public boolean hasNext() { return index < array.length; }
+                @Override public Double next() { return array[index++]; }
+            };
+        }
+        static Iterator<Object> iterator(final Object[] array) {
+            return new Iterator<Object>() {
+                int index = 0;
+                @Override public boolean hasNext() { return index < array.length; }
+                @Override public Object next() { return array[index++]; }
+            };
+        }
+
+        static MethodHandle newIterator(Class<?> arrayType) {
+            if (!arrayType.isArray()) {
+                throw new IllegalArgumentException("type must be an array");
+            }
+            return (ARRAY_TYPE_MH_MAPPING.containsKey(arrayType)) ?
+                ARRAY_TYPE_MH_MAPPING.get(arrayType) :
+                OBJECT_ARRAY_MH.asType(OBJECT_ARRAY_MH.type().changeParameterType(0, arrayType));
+        }
+
+        private ArrayIteratorHelper() {}
+    }
     /**
      * Returns a method handle to do iteration (for enhanced for loop)
      * @param receiverClass Class of the array to load the value from
@@ -387,6 +484,8 @@ public final class Def {
     static MethodHandle lookupIterator(Class<?> receiverClass) {
         if (Iterable.class.isAssignableFrom(receiverClass)) {
             return ITERATOR;
+        } else if (receiverClass.isArray()) {
+            return ArrayIteratorHelper.newIterator(receiverClass);
         } else {
             // TODO: arrays
             throw new IllegalArgumentException("Cannot iterate over [" + receiverClass.getCanonicalName() + "]");

From 8db9a971e510169dd08ddd87d9a74c3460033cac Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 6 Jun 2016 20:30:42 -0400
Subject: [PATCH 0679/1311] enable test

---
 .../java/org/elasticsearch/painless/BasicStatementTests.java     | 1 -
 1 file changed, 1 deletion(-)

diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
index 64137bfbcb3..c392e86806e 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
@@ -154,7 +154,6 @@ public class BasicStatementTests extends ScriptTestCase {
             " for (int[] j : i) total += j[0]; return total"));
     }
     
-    @AwaitsFix(bugUrl = "working on it")
     public void testArrayForEachStatementDef() {
         assertEquals(6, exec("def a = new int[3]; a[0] = 1; a[1] = 2; a[2] = 3; int total = 0;" +
             " for (int x : a) total += x; return total"));

From 6dace47c1f493aea0b5e29b5bfc7d6e5358c7939 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Mon, 6 Jun 2016 17:59:39 -0700
Subject: [PATCH 0680/1311] Reverted S-node design change.

---
 .../src/main/antlr/PainlessParser.g4          |   2 +-
 .../painless/node/AStatement.java             |  24 +-
 .../painless/node/SArrayEach.java             | 135 ----------
 .../elasticsearch/painless/node/SBlock.java   |  21 +-
 .../elasticsearch/painless/node/SBreak.java   |   4 +-
 .../elasticsearch/painless/node/SCatch.java   |   8 +-
 .../painless/node/SContinue.java              |   4 +-
 .../painless/node/SDeclBlock.java             |  13 +-
 .../painless/node/SDeclaration.java           |   4 +-
 .../org/elasticsearch/painless/node/SDo.java  |   8 +-
 .../elasticsearch/painless/node/SEach.java    | 242 ++++++++++++------
 .../painless/node/SExpression.java            |   4 +-
 .../org/elasticsearch/painless/node/SFor.java |  10 +-
 .../org/elasticsearch/painless/node/SIf.java  |   8 +-
 .../elasticsearch/painless/node/SIfElse.java  |  12 +-
 .../elasticsearch/painless/node/SReturn.java  |   4 +-
 .../elasticsearch/painless/node/SSource.java  |  16 +-
 .../elasticsearch/painless/node/SThrow.java   |   4 +-
 .../org/elasticsearch/painless/node/STry.java |  17 +-
 .../elasticsearch/painless/node/SWhile.java   |   8 +-
 .../painless/node/package-info.java           |   4 +-
 21 files changed, 226 insertions(+), 326 deletions(-)
 delete mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SArrayEach.java

diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index 6a5ea6bf7fe..dad8db62117 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -74,7 +74,7 @@ decltype
     ;
 
 funcref
-    : ( TYPE | ID ) REF ID
+    : TYPE REF ID
     ;
 
 declvar
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
index f6b1048028c..b9e6679f630 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
@@ -114,33 +114,11 @@ public abstract class AStatement extends ANode {
 
     /**
      * Checks for errors and collects data for the writing phase.
-     * @return The new child node for the parent node calling this method.
-     * Possibly returns a different {@link AStatement} node if a type is
-     * def or if a different specialization is used. Otherwise, returns itself.
      */
-    abstract AStatement analyze(Variables variables);
+    abstract void analyze(Variables variables);
 
     /**
      * Writes ASM based on the data collected during the analysis phase.
      */
     abstract void write(MethodWriter writer);
-
-    /**
-     * Used to copy statement data from one to another during analysis in the case of replacement.
-     */
-    final AStatement copy(AStatement statement) {
-        lastSource      = statement.lastSource;
-        beginLoop       = statement.beginLoop;
-        inLoop          = statement.inLoop;
-        lastLoop        = statement.lastLoop;
-        methodEscape    = statement.methodEscape;
-        loopEscape      = statement.loopEscape;
-        allEscape       = statement.allEscape;
-        anyContinue     = statement.anyContinue;
-        anyBreak        = statement.anyBreak;
-        loopCounterSlot = statement.loopCounterSlot;
-        statementCount  = statement.statementCount;
-
-        return this;
-    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SArrayEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SArrayEach.java
deleted file mode 100644
index 032483f59f2..00000000000
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SArrayEach.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless.node;
-
-import org.elasticsearch.painless.AnalyzerCaster;
-import org.elasticsearch.painless.Definition;
-import org.elasticsearch.painless.Definition.Cast;
-import org.elasticsearch.painless.Definition.Type;
-import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.MethodWriter;
-import org.elasticsearch.painless.Variables;
-import org.elasticsearch.painless.Variables.Variable;
-import org.objectweb.asm.Label;
-import org.objectweb.asm.Opcodes;
-
-/**
- * Represents a for-each loop shortcut for arrays.  (Internal only.)
- */
-class SArrayEach extends AStatement {
-    final int maxLoopCounter;
-    final String type;
-    final String name;
-    AExpression expression;
-    AStatement block;
-
-    Variable variable = null;
-    Variable array = null;
-    Variable index = null;
-    Type indexed = null;
-    Cast cast = null;
-
-    public SArrayEach(Location location, int maxLoopCounter, String type, String name, AExpression expression, SBlock block) {
-        super(location);
-
-        this.maxLoopCounter = maxLoopCounter;
-        this.type = type;
-        this.name = name;
-        this.expression = expression;
-        this.block = block;
-    }
-
-    @Override
-    AStatement analyze(Variables variables) {
-        // Note that we do not need to analyze the expression as this must already be done
-        // in the parent to determine that the for each target type is an array.
-
-        final Type type;
-
-        try {
-            type = Definition.getType(this.type);
-        } catch (IllegalArgumentException exception) {
-            throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
-        }
-
-        variables.incrementScope();
-
-        variable = variables.addVariable(location, type, name, true, false);
-        array = variables.addVariable(location, expression.actual, "#array" + location.getOffset(), true, false);
-        index = variables.addVariable(location, Definition.INT_TYPE, "#index" + location.getOffset(), true, false);
-        indexed = Definition.getType(expression.actual.struct, expression.actual.dimensions - 1);
-        cast = AnalyzerCaster.getLegalCast(location, indexed, type, true, true);
-
-        if (block == null) {
-            throw location.createError(new IllegalArgumentException("Extraneous for each loop."));
-        }
-
-        block.beginLoop = true;
-        block.inLoop = true;
-        block = block.analyze(variables);
-        block.statementCount = Math.max(1, block.statementCount);
-
-        if (block.loopEscape && !block.anyContinue) {
-            throw createError(new IllegalArgumentException("Extraneous for loop."));
-        }
-
-        statementCount = 1;
-
-        if (maxLoopCounter > 0) {
-            loopCounterSlot = variables.getVariable(location, "#loop").slot;
-        }
-
-        variables.decrementScope();
-
-        return this;
-    }
-
-    @Override
-    void write(MethodWriter writer) {
-        writer.writeStatementOffset(location);
-
-        expression.write(writer);
-        writer.visitVarInsn(array.type.type.getOpcode(Opcodes.ISTORE), array.slot);
-        writer.push(-1);
-        writer.visitVarInsn(index.type.type.getOpcode(Opcodes.ISTORE), index.slot);
-
-        Label begin = new Label();
-        Label end = new Label();
-
-        writer.mark(begin);
-
-        writer.visitIincInsn(index.slot, 1);
-        writer.visitVarInsn(index.type.type.getOpcode(Opcodes.ILOAD), index.slot);
-        writer.visitVarInsn(array.type.type.getOpcode(Opcodes.ILOAD), array.slot);
-        writer.arrayLength();
-        writer.ifICmp(MethodWriter.GE, end);
-
-        writer.visitVarInsn(array.type.type.getOpcode(Opcodes.ILOAD), array.slot);
-        writer.visitVarInsn(index.type.type.getOpcode(Opcodes.ILOAD), index.slot);
-        writer.arrayLoad(indexed.type);
-        writer.writeCast(cast);
-        writer.visitVarInsn(variable.type.type.getOpcode(Opcodes.ISTORE), variable.slot);
-
-        block.write(writer);
-
-        writer.goTo(begin);
-        writer.mark(end);
-    }
-}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
index 175eb26566c..4dbbd80de54 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
@@ -23,6 +23,7 @@ import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
+import java.util.Collections;
 import java.util.List;
 
 /**
@@ -35,27 +36,29 @@ public final class SBlock extends AStatement {
     public SBlock(Location location, List<AStatement> statements) {
         super(location);
 
-        this.statements = statements;
+        this.statements = Collections.unmodifiableList(statements);
     }
 
     @Override
-    AStatement analyze(Variables variables) {
+    void analyze(Variables variables) {
         if (statements == null || statements.isEmpty()) {
             throw createError(new IllegalArgumentException("A block must contain at least one statement."));
         }
 
-        for (int index = 0; index < statements.size(); ++index) {
+        AStatement last = statements.get(statements.size() - 1);
+
+        for (AStatement statement : statements) {
+            // Note that we do not need to check after the last statement because
+            // there is no statement that can be unreachable after the last.
             if (allEscape) {
                 throw createError(new IllegalArgumentException("Unreachable statement."));
             }
 
-            AStatement statement = statements.get(index);
-
             statement.inLoop = inLoop;
-            statement.lastSource = lastSource && index == statements.size() - 1;
-            statement.lastLoop = (beginLoop || lastLoop) && index == statements.size() - 1;
+            statement.lastSource = lastSource && statement == last;
+            statement.lastLoop = (beginLoop || lastLoop) && statement == last;
 
-            statements.set(index, statement.analyze(variables));
+            statement.analyze(variables);
 
             methodEscape = statement.methodEscape;
             loopEscape = statement.loopEscape;
@@ -64,8 +67,6 @@ public final class SBlock extends AStatement {
             anyBreak |= statement.anyBreak;
             statementCount += statement.statementCount;
         }
-
-        return this;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
index 6e5c984814f..ca72dd0b55b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
@@ -33,7 +33,7 @@ public final class SBreak extends AStatement {
     }
 
     @Override
-    AStatement analyze(Variables variables) {
+    void analyze(Variables variables) {
         if (!inLoop) {
             throw createError(new IllegalArgumentException("Break statement outside of a loop."));
         }
@@ -42,8 +42,6 @@ public final class SBreak extends AStatement {
         allEscape = true;
         anyBreak = true;
         statementCount = 1;
-
-        return this;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
index 873192b1933..8bcaf9d22cf 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
@@ -35,7 +35,7 @@ public final class SCatch extends AStatement {
 
     final String type;
     final String name;
-    AStatement block;
+    final SBlock block;
 
     Variable variable;
 
@@ -52,7 +52,7 @@ public final class SCatch extends AStatement {
     }
 
     @Override
-    AStatement analyze(Variables variables) {
+    void analyze(Variables variables) {
         final Type type;
 
         try {
@@ -72,7 +72,7 @@ public final class SCatch extends AStatement {
             block.inLoop = inLoop;
             block.lastLoop = lastLoop;
 
-            block = block.analyze(variables);
+            block.analyze(variables);
 
             methodEscape = block.methodEscape;
             loopEscape = block.loopEscape;
@@ -81,8 +81,6 @@ public final class SCatch extends AStatement {
             anyBreak = block.anyBreak;
             statementCount = block.statementCount;
         }
-
-        return this;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
index 7c8eb6fefcc..ef80766bdd1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
@@ -33,7 +33,7 @@ public final class SContinue extends AStatement {
     }
 
     @Override
-    AStatement analyze(Variables variables) {
+    void analyze(Variables variables) {
         if (!inLoop) {
             throw createError(new IllegalArgumentException("Continue statement outside of a loop."));
         }
@@ -45,8 +45,6 @@ public final class SContinue extends AStatement {
         allEscape = true;
         anyContinue = true;
         statementCount = 1;
-
-        return this;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
index c7d1c93efb6..1ff187afe29 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
@@ -23,6 +23,7 @@ import org.elasticsearch.painless.Variables;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
+import java.util.Collections;
 import java.util.List;
 
 /**
@@ -35,20 +36,16 @@ public final class SDeclBlock extends AStatement {
     public SDeclBlock(Location location, List<SDeclaration> declarations) {
         super(location);
 
-        this.declarations = declarations;
+        this.declarations = Collections.unmodifiableList(declarations);
     }
 
     @Override
-    AStatement analyze(Variables variables) {
-        for (int index = 0; index < declarations.size(); ++index) {
-            AStatement declaration = declarations.get(index);
-
-            declarations.set(index, (SDeclaration)declaration.analyze(variables));
+    void analyze(Variables variables) {
+        for (SDeclaration declaration : declarations) {
+            declaration.analyze(variables);
         }
 
         statementCount = declarations.size();
-
-        return this;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
index 8430382d099..5f039184cb6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
@@ -47,7 +47,7 @@ public final class SDeclaration extends AStatement {
     }
 
     @Override
-    AStatement analyze(Variables variables) {
+    void analyze(Variables variables) {
         final Type type;
 
         try {
@@ -63,8 +63,6 @@ public final class SDeclaration extends AStatement {
         }
 
         variable = variables.addVariable(location, type, name, false, false);
-
-        return this;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
index 1a0aa132d5e..4989fb77a79 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -31,7 +31,7 @@ import org.elasticsearch.painless.MethodWriter;
 public final class SDo extends AStatement {
 
     final int maxLoopCounter;
-    AStatement block;
+    final SBlock block;
     AExpression condition;
 
     public SDo(Location location, int maxLoopCounter, SBlock block, AExpression condition) {
@@ -43,7 +43,7 @@ public final class SDo extends AStatement {
     }
 
     @Override
-    AStatement analyze(Variables variables) {
+    void analyze(Variables variables) {
         variables.incrementScope();
 
         if (block == null) {
@@ -53,7 +53,7 @@ public final class SDo extends AStatement {
         block.beginLoop = true;
         block.inLoop = true;
 
-        block = block.analyze(variables);
+        block.analyze(variables);
 
         if (block.loopEscape && !block.anyContinue) {
             throw createError(new IllegalArgumentException("Extraneous do while loop."));
@@ -83,8 +83,6 @@ public final class SDo extends AStatement {
         }
 
         variables.decrementScope();
-
-        return this;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index 309c6d97967..3d6d5e07fdc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -45,14 +45,22 @@ public class SEach extends AStatement {
     final String type;
     final String name;
     AExpression expression;
-    AStatement block;
+    final SBlock block;
 
+    // Members for all cases.
     Variable variable = null;
+    Cast cast = null;
+
+    // Members for the array case.
+    Variable array = null;
+    Variable index = null;
+    Type indexed = null;
+
+    // Members for the iterable case.
     Variable iterator = null;
     Method method = null;
     Method hasNext = null;
     Method next = null;
-    Cast cast = null;
 
     public SEach(Location location, int maxLoopCounter, String type, String name, AExpression expression, SBlock block) {
         super(location);
@@ -65,92 +73,172 @@ public class SEach extends AStatement {
     }
 
     @Override
-    AStatement analyze(Variables variables) {
+    void analyze(Variables variables) {
         expression.analyze(variables);
         expression.expected = expression.actual;
         expression = expression.cast(variables);
 
-        Sort sort = expression.actual.sort;
-
-        if (sort == Sort.ARRAY) {
-            return new SArrayEach(location, maxLoopCounter, type, name, expression, (SBlock)block).copy(this).analyze(variables);
-        } else if (sort == Sort.DEF || Iterable.class.isAssignableFrom(expression.actual.clazz)) {
-            final Type type;
-
-            try {
-                type = Definition.getType(this.type);
-            } catch (IllegalArgumentException exception) {
-                throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
-            }
-
-            variables.incrementScope();
-
-            Type itr = Definition.getType("Iterator");
-
-            variable = variables.addVariable(location, type, name, true, false);
-
-            // We must store the iterator as a variable for securing a slot on the stack, and
-            // also add the location offset to make the name unique in case of nested for each loops.
-            iterator = variables.addVariable(location, itr, "#itr" + location.getOffset(), true, false);
-
-            if (sort == Sort.DEF) {
-                method = null;
-            } else {
-                method = expression.actual.struct.methods.get(new MethodKey("iterator", 0));
-
-                if (method == null) {
-                    throw location.createError(new IllegalArgumentException(
-                            "Unable to create iterator for the type [" + expression.actual.name + "]."));
-                }
-            }
-
-            hasNext = itr.struct.methods.get(new MethodKey("hasNext", 0));
-
-            if (hasNext == null) {
-                throw location.createError(new IllegalArgumentException("Method [hasNext] does not exist for type [Iterator]."));
-            } else if (hasNext.rtn.sort != Sort.BOOL) {
-                throw location.createError(new IllegalArgumentException("Method [hasNext] does not return type [boolean]."));
-            }
-
-            next = itr.struct.methods.get(new MethodKey("next", 0));
-
-            if (next == null) {
-                throw location.createError(new IllegalArgumentException("Method [next] does not exist for type [Iterator]."));
-            } else if (next.rtn.sort != Sort.DEF) {
-                throw location.createError(new IllegalArgumentException("Method [next] does not return type [def]."));
-            }
-
-            cast = AnalyzerCaster.getLegalCast(location, Definition.DEF_TYPE, type, true, true);
-
-            if (block == null) {
-                throw location.createError(new IllegalArgumentException("Extraneous for each loop."));
-            }
-
-            block.beginLoop = true;
-            block.inLoop = true;
-            block = block.analyze(variables);
-            block.statementCount = Math.max(1, block.statementCount);
-
-            if (block.loopEscape && !block.anyContinue) {
-                throw createError(new IllegalArgumentException("Extraneous for loop."));
-            }
-
-            statementCount = 1;
-
-            if (maxLoopCounter > 0) {
-                loopCounterSlot = variables.getVariable(location, "#loop").slot;
-            }
-
-            variables.decrementScope();
-
-            return this;
+        if (expression.actual.sort == Sort.ARRAY) {
+            analyzeArray(variables);
+        } else if (expression.actual.sort == Sort.DEF || Iterable.class.isAssignableFrom(expression.actual.clazz)) {
+            analyzeIterable(variables);
         } else {
             throw location.createError(new IllegalArgumentException("Illegal for each type [" + expression.actual.name + "]."));
         }
     }
 
+    void analyzeArray(Variables variables) {
+        final Type type;
+
+        try {
+            type = Definition.getType(this.type);
+        } catch (IllegalArgumentException exception) {
+            throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
+        }
+
+        variables.incrementScope();
+
+        variable = variables.addVariable(location, type, name, true, false);
+        array = variables.addVariable(location, expression.actual, "#array" + location.getOffset(), true, false);
+        index = variables.addVariable(location, Definition.INT_TYPE, "#index" + location.getOffset(), true, false);
+        indexed = Definition.getType(expression.actual.struct, expression.actual.dimensions - 1);
+        cast = AnalyzerCaster.getLegalCast(location, indexed, type, true, true);
+
+        if (block == null) {
+            throw location.createError(new IllegalArgumentException("Extraneous for each loop."));
+        }
+
+        block.beginLoop = true;
+        block.inLoop = true;
+        block.analyze(variables);
+        block.statementCount = Math.max(1, block.statementCount);
+
+        if (block.loopEscape && !block.anyContinue) {
+            throw createError(new IllegalArgumentException("Extraneous for loop."));
+        }
+
+        statementCount = 1;
+
+        if (maxLoopCounter > 0) {
+            loopCounterSlot = variables.getVariable(location, "#loop").slot;
+        }
+
+        variables.decrementScope();
+    }
+
+    void analyzeIterable(Variables variables) {
+        final Type type;
+
+        try {
+            type = Definition.getType(this.type);
+        } catch (IllegalArgumentException exception) {
+            throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
+        }
+
+        variables.incrementScope();
+
+        Type itr = Definition.getType("Iterator");
+
+        variable = variables.addVariable(location, type, name, true, false);
+
+        // We must store the iterator as a variable for securing a slot on the stack, and
+        // also add the location offset to make the name unique in case of nested for each loops.
+        iterator = variables.addVariable(location, itr, "#itr" + location.getOffset(), true, false);
+
+        if (expression.actual.sort == Sort.DEF) {
+            method = null;
+        } else {
+            method = expression.actual.struct.methods.get(new MethodKey("iterator", 0));
+
+            if (method == null) {
+                throw location.createError(new IllegalArgumentException(
+                    "Unable to create iterator for the type [" + expression.actual.name + "]."));
+            }
+        }
+
+        hasNext = itr.struct.methods.get(new MethodKey("hasNext", 0));
+
+        if (hasNext == null) {
+            throw location.createError(new IllegalArgumentException("Method [hasNext] does not exist for type [Iterator]."));
+        } else if (hasNext.rtn.sort != Sort.BOOL) {
+            throw location.createError(new IllegalArgumentException("Method [hasNext] does not return type [boolean]."));
+        }
+
+        next = itr.struct.methods.get(new MethodKey("next", 0));
+
+        if (next == null) {
+            throw location.createError(new IllegalArgumentException("Method [next] does not exist for type [Iterator]."));
+        } else if (next.rtn.sort != Sort.DEF) {
+            throw location.createError(new IllegalArgumentException("Method [next] does not return type [def]."));
+        }
+
+        cast = AnalyzerCaster.getLegalCast(location, Definition.DEF_TYPE, type, true, true);
+
+        if (block == null) {
+            throw location.createError(new IllegalArgumentException("Extraneous for each loop."));
+        }
+
+        block.beginLoop = true;
+        block.inLoop = true;
+        block.analyze(variables);
+        block.statementCount = Math.max(1, block.statementCount);
+
+        if (block.loopEscape && !block.anyContinue) {
+            throw createError(new IllegalArgumentException("Extraneous for loop."));
+        }
+
+        statementCount = 1;
+
+        if (maxLoopCounter > 0) {
+            loopCounterSlot = variables.getVariable(location, "#loop").slot;
+        }
+
+        variables.decrementScope();
+    }
+
     @Override
     void write(MethodWriter writer) {
+        if (array != null) {
+            writeArray(writer);
+        } else if (iterator != null) {
+            writeIterable(writer);
+        } else {
+            throw location.createError(new IllegalStateException("Illegal tree structure."));
+        }
+    }
+
+    void writeArray(MethodWriter writer) {
+        writer.writeStatementOffset(location);
+
+        expression.write(writer);
+        writer.visitVarInsn(array.type.type.getOpcode(Opcodes.ISTORE), array.slot);
+        writer.push(-1);
+        writer.visitVarInsn(index.type.type.getOpcode(Opcodes.ISTORE), index.slot);
+
+        Label begin = new Label();
+        Label end = new Label();
+
+        writer.mark(begin);
+
+        writer.visitIincInsn(index.slot, 1);
+        writer.visitVarInsn(index.type.type.getOpcode(Opcodes.ILOAD), index.slot);
+        writer.visitVarInsn(array.type.type.getOpcode(Opcodes.ILOAD), array.slot);
+        writer.arrayLength();
+        writer.ifICmp(MethodWriter.GE, end);
+
+        writer.visitVarInsn(array.type.type.getOpcode(Opcodes.ILOAD), array.slot);
+        writer.visitVarInsn(index.type.type.getOpcode(Opcodes.ILOAD), index.slot);
+        writer.arrayLoad(indexed.type);
+        writer.writeCast(cast);
+        writer.visitVarInsn(variable.type.type.getOpcode(Opcodes.ISTORE), variable.slot);
+
+        block.write(writer);
+
+        writer.goTo(begin);
+        writer.mark(end);
+    }
+
+    void writeIterable(MethodWriter writer) {
         writer.writeStatementOffset(location);
 
         expression.write(writer);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index 87b412c5d52..1bea07d5599 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -39,7 +39,7 @@ public final class SExpression extends AStatement {
     }
 
     @Override
-    AStatement analyze(Variables variables) {
+    void analyze(Variables variables) {
         expression.read = lastSource;
         expression.analyze(variables);
 
@@ -57,8 +57,6 @@ public final class SExpression extends AStatement {
         loopEscape = rtn;
         allEscape = rtn;
         statementCount = 1;
-
-        return this;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index 6085c1ce8d9..04475a91a1a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -34,7 +34,7 @@ public final class SFor extends AStatement {
     ANode initializer;
     AExpression condition;
     AExpression afterthought;
-    AStatement block;
+    final SBlock block;
 
     public SFor(Location location, int maxLoopCounter,
                 ANode initializer, AExpression condition, AExpression afterthought, SBlock block) {
@@ -48,14 +48,14 @@ public final class SFor extends AStatement {
     }
 
     @Override
-    AStatement analyze(Variables variables) {
+    void analyze(Variables variables) {
         variables.incrementScope();
 
         boolean continuous = false;
 
         if (initializer != null) {
             if (initializer instanceof AStatement) {
-                initializer = ((AStatement)initializer).analyze(variables);
+                ((AStatement)initializer).analyze(variables);
             } else if (initializer instanceof AExpression) {
                 AExpression initializer = (AExpression)this.initializer;
 
@@ -103,7 +103,7 @@ public final class SFor extends AStatement {
             block.beginLoop = true;
             block.inLoop = true;
 
-            block = block.analyze(variables);
+            block.analyze(variables);
 
             if (block.loopEscape && !block.anyContinue) {
                 throw createError(new IllegalArgumentException("Extraneous for loop."));
@@ -124,8 +124,6 @@ public final class SFor extends AStatement {
         }
 
         variables.decrementScope();
-
-        return this;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
index aa24c8acc39..954ddac9c6a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
@@ -31,7 +31,7 @@ import org.elasticsearch.painless.MethodWriter;
 public final class SIf extends AStatement {
 
     AExpression condition;
-    AStatement ifblock;
+    final SBlock ifblock;
 
     public SIf(Location location, AExpression condition, SBlock ifblock) {
         super(location);
@@ -41,7 +41,7 @@ public final class SIf extends AStatement {
     }
 
     @Override
-    AStatement analyze(Variables variables) {
+    void analyze(Variables variables) {
         condition.expected = Definition.BOOLEAN_TYPE;
         condition.analyze(variables);
         condition = condition.cast(variables);
@@ -59,14 +59,12 @@ public final class SIf extends AStatement {
         ifblock.lastLoop = lastLoop;
 
         variables.incrementScope();
-        ifblock = ifblock.analyze(variables);
+        ifblock.analyze(variables);
         variables.decrementScope();
 
         anyContinue = ifblock.anyContinue;
         anyBreak = ifblock.anyBreak;
         statementCount = ifblock.statementCount;
-
-        return this;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
index 01e1ceb5750..1d801267054 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
@@ -31,8 +31,8 @@ import org.elasticsearch.painless.MethodWriter;
 public final class SIfElse extends AStatement {
 
     AExpression condition;
-    AStatement ifblock;
-    AStatement elseblock;
+    final SBlock ifblock;
+    final SBlock elseblock;
 
     public SIfElse(Location location, AExpression condition, SBlock ifblock, SBlock elseblock) {
         super(location);
@@ -43,7 +43,7 @@ public final class SIfElse extends AStatement {
     }
 
     @Override
-    AStatement analyze(Variables variables) {
+    void analyze(Variables variables) {
         condition.expected = Definition.BOOLEAN_TYPE;
         condition.analyze(variables);
         condition = condition.cast(variables);
@@ -61,7 +61,7 @@ public final class SIfElse extends AStatement {
         ifblock.lastLoop = lastLoop;
 
         variables.incrementScope();
-        ifblock = ifblock.analyze(variables);
+        ifblock.analyze(variables);
         variables.decrementScope();
 
         anyContinue = ifblock.anyContinue;
@@ -77,7 +77,7 @@ public final class SIfElse extends AStatement {
         elseblock.lastLoop = lastLoop;
 
         variables.incrementScope();
-        elseblock = elseblock.analyze(variables);
+        elseblock.analyze(variables);
         variables.decrementScope();
 
         methodEscape = ifblock.methodEscape && elseblock.methodEscape;
@@ -86,8 +86,6 @@ public final class SIfElse extends AStatement {
         anyContinue |= elseblock.anyContinue;
         anyBreak |= elseblock.anyBreak;
         statementCount = Math.max(ifblock.statementCount, elseblock.statementCount);
-
-        return this;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
index bab12fb52f5..bd9707eb6a4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -38,7 +38,7 @@ public final class SReturn extends AStatement {
     }
 
     @Override
-    AStatement analyze(Variables variables) {
+    void analyze(Variables variables) {
         expression.expected = Definition.OBJECT_TYPE;
         expression.internal = true;
         expression.analyze(variables);
@@ -49,8 +49,6 @@ public final class SReturn extends AStatement {
         allEscape = true;
 
         statementCount = 1;
-
-        return this;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index 57acc425c46..22fd2e3e28c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -24,6 +24,7 @@ import org.objectweb.asm.Opcodes;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
+import java.util.Collections;
 import java.util.List;
 
 /**
@@ -36,36 +37,35 @@ public final class SSource extends AStatement {
     public SSource(Location location, List<AStatement> statements) {
         super(location);
 
-        this.statements = statements;
+        this.statements = Collections.unmodifiableList(statements);
     }
 
     @Override
-    public AStatement analyze(Variables variables) {
+    public void analyze(Variables variables) {
         if (statements == null || statements.isEmpty()) {
             throw createError(new IllegalArgumentException("Cannot generate an empty script."));
         }
 
         variables.incrementScope();
 
-        for (int index = 0; index < statements.size(); ++index) {
-            AStatement statement = statements.get(index);
+        AStatement last = statements.get(statements.size() - 1);
 
+        for (AStatement statement : statements) {
             // Note that we do not need to check after the last statement because
             // there is no statement that can be unreachable after the last.
             if (allEscape) {
                 throw createError(new IllegalArgumentException("Unreachable statement."));
             }
 
-            statement.lastSource = index == statements.size() - 1;
-            statements.set(index, statement.analyze(variables));
+            statement.lastSource = statement == last;
+
+            statement.analyze(variables);
 
             methodEscape = statement.methodEscape;
             allEscape = statement.allEscape;
         }
 
         variables.decrementScope();
-
-        return this;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
index c8c02f9daa1..af9d7a65990 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
@@ -38,7 +38,7 @@ public final class SThrow extends AStatement {
     }
 
     @Override
-    AStatement analyze(Variables variables) {
+    void analyze(Variables variables) {
         expression.expected = Definition.EXCEPTION_TYPE;
         expression.analyze(variables);
         expression = expression.cast(variables);
@@ -47,8 +47,6 @@ public final class SThrow extends AStatement {
         loopEscape = true;
         allEscape = true;
         statementCount = 1;
-
-        return this;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
index 65e66a9d9a5..c24c8273dba 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
@@ -24,6 +24,7 @@ import org.objectweb.asm.Label;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
+import java.util.Collections;
 import java.util.List;
 
 /**
@@ -31,18 +32,18 @@ import java.util.List;
  */
 public final class STry extends AStatement {
 
-    AStatement block;
+    final SBlock block;
     final List<SCatch> catches;
 
     public STry(Location location, SBlock block, List<SCatch> catches) {
         super(location);
 
         this.block = block;
-        this.catches = catches;
+        this.catches = Collections.unmodifiableList(catches);
     }
 
     @Override
-    AStatement analyze(Variables variables) {
+    void analyze(Variables variables) {
         if (block == null) {
             throw createError(new IllegalArgumentException("Extraneous try statement."));
         }
@@ -52,7 +53,7 @@ public final class STry extends AStatement {
         block.lastLoop = lastLoop;
 
         variables.incrementScope();
-        block = block.analyze(variables);
+        block.analyze(variables);
         variables.decrementScope();
 
         methodEscape = block.methodEscape;
@@ -63,15 +64,13 @@ public final class STry extends AStatement {
 
         int statementCount = 0;
 
-        for (int index = 0; index < catches.size(); ++index) {
-            SCatch catc = catches.get(index);
-
+        for (SCatch catc : catches) {
             catc.lastSource = lastSource;
             catc.inLoop = inLoop;
             catc.lastLoop = lastLoop;
 
             variables.incrementScope();
-            catches.set(index, (SCatch)catc.analyze(variables));
+            catc.analyze(variables);
             variables.decrementScope();
 
             methodEscape &= catc.methodEscape;
@@ -84,8 +83,6 @@ public final class STry extends AStatement {
         }
 
         this.statementCount = block.statementCount + statementCount;
-
-        return this;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
index a886c3632c3..59c1bb75ee8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -32,7 +32,7 @@ public final class SWhile extends AStatement {
 
     final int maxLoopCounter;
     AExpression condition;
-    AStatement block;
+    final SBlock block;
 
     public SWhile(Location location, int maxLoopCounter, AExpression condition, SBlock block) {
         super(location);
@@ -43,7 +43,7 @@ public final class SWhile extends AStatement {
     }
 
     @Override
-    AStatement analyze(Variables variables) {
+    void analyze(Variables variables) {
         variables.incrementScope();
 
         condition.expected = Definition.BOOLEAN_TYPE;
@@ -68,7 +68,7 @@ public final class SWhile extends AStatement {
             block.beginLoop = true;
             block.inLoop = true;
 
-            block = block.analyze(variables);
+            block.analyze(variables);
 
             if (block.loopEscape && !block.anyContinue) {
                 throw createError(new IllegalArgumentException("Extraneous while loop."));
@@ -89,8 +89,6 @@ public final class SWhile extends AStatement {
         }
 
         variables.decrementScope();
-
-        return this;
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
index 04727507edc..71cde33e979 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
@@ -63,7 +63,6 @@
  * {@link org.elasticsearch.painless.node.LStatic} - Represents a static type target.
  * {@link org.elasticsearch.painless.node.LString} - Represents a string constant.
  * {@link org.elasticsearch.painless.node.LVariable} - Represents a variable load/store.
- * {@link org.elasticsearch.painless.node.SArrayEach} - Represents a for each loop shortcut for arrays.  (Internal only.)
  * {@link org.elasticsearch.painless.node.SBlock} - Represents a set of statements as a branch of control-flow.
  * {@link org.elasticsearch.painless.node.SBreak} - Represents a break statement.
  * {@link org.elasticsearch.painless.node.SCatch} - Represents a catch block as part of a try-catch block.
@@ -92,8 +91,7 @@
  * <p>
  * Generally, statement nodes have member data that evaluate legal control-flow during the analysis phase.
  * The typical order for statement nodes is for each node to call analyze on it's children during the analysis phase
- * and write on it's children during the writing phase.  Upon analysis completion, a statement will return either
- * itself or another statement node depending on if a shortcut or def type was found.
+ * and write on it's children during the writing phase.
  * <p>
  * Generally, expression nodes have member data that evaluate static types.  The typical order for an expression node
  * during the analysis phase looks like the following:

From 231268c89d1ddd29ce2c603d542c5523dcd86830 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 6 Jun 2016 21:04:59 -0400
Subject: [PATCH 0681/1311] remove outdated TODO

---
 .../src/main/java/org/elasticsearch/painless/Def.java            | 1 -
 1 file changed, 1 deletion(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 614490d0295..f85dff2ed21 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -487,7 +487,6 @@ public final class Def {
         } else if (receiverClass.isArray()) {
             return ArrayIteratorHelper.newIterator(receiverClass);
         } else {
-            // TODO: arrays
             throw new IllegalArgumentException("Cannot iterate over [" + receiverClass.getCanonicalName() + "]");
         }
     }

From b3804c47f7e8ffdf2a98a34ac34dd48b8e97e39a Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Mon, 6 Jun 2016 18:21:08 -0700
Subject: [PATCH 0682/1311] Cleaned up SEach node.

---
 .../painless/WriterConstants.java             |  5 +
 .../elasticsearch/painless/node/SEach.java    | 99 +++++--------------
 2 files changed, 30 insertions(+), 74 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index a4804cde434..fce40d5a6e0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -30,6 +30,7 @@ import java.lang.invoke.CallSite;
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodType;
 import java.util.BitSet;
+import java.util.Iterator;
 import java.util.Map;
 
 /**
@@ -56,6 +57,10 @@ public final class WriterConstants {
     public final static Type MAP_TYPE  = Type.getType(Map.class);
     public final static Method MAP_GET = getAsmMethod(Object.class, "get", Object.class);
 
+    public final static Type ITERATOR_TYPE = Type.getType(Iterator.class);
+    public final static Method ITERATOR_HASNEXT = getAsmMethod(boolean.class, "hasNext");
+    public final static Method ITERATOR_NEXT = getAsmMethod(Object.class, "next");
+
     public final static Type UTILITY_TYPE = Type.getType(Utility.class);
     public final static Method STRING_TO_CHAR = getAsmMethod(char.class, "StringTochar", String.class);
     public final static Method CHAR_TO_STRING = getAsmMethod(String.class, "charToString", char.class);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index 3d6d5e07fdc..2a8fb0e4063 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -35,6 +35,9 @@ import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
 
 import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
+import static org.elasticsearch.painless.WriterConstants.ITERATOR_HASNEXT;
+import static org.elasticsearch.painless.WriterConstants.ITERATOR_NEXT;
+import static org.elasticsearch.painless.WriterConstants.ITERATOR_TYPE;
 
 /**
  * Represents a for-each loop shortcut for iterables.  Defers to other S-nodes for non-iterable types.
@@ -59,8 +62,6 @@ public class SEach extends AStatement {
     // Members for the iterable case.
     Variable iterator = null;
     Method method = null;
-    Method hasNext = null;
-    Method next = null;
 
     public SEach(Location location, int maxLoopCounter, String type, String name, AExpression expression, SBlock block) {
         super(location);
@@ -78,16 +79,6 @@ public class SEach extends AStatement {
         expression.expected = expression.actual;
         expression = expression.cast(variables);
 
-        if (expression.actual.sort == Sort.ARRAY) {
-            analyzeArray(variables);
-        } else if (expression.actual.sort == Sort.DEF || Iterable.class.isAssignableFrom(expression.actual.clazz)) {
-            analyzeIterable(variables);
-        } else {
-            throw location.createError(new IllegalArgumentException("Illegal for each type [" + expression.actual.name + "]."));
-        }
-    }
-
-    void analyzeArray(Variables variables) {
         final Type type;
 
         try {
@@ -99,10 +90,14 @@ public class SEach extends AStatement {
         variables.incrementScope();
 
         variable = variables.addVariable(location, type, name, true, false);
-        array = variables.addVariable(location, expression.actual, "#array" + location.getOffset(), true, false);
-        index = variables.addVariable(location, Definition.INT_TYPE, "#index" + location.getOffset(), true, false);
-        indexed = Definition.getType(expression.actual.struct, expression.actual.dimensions - 1);
-        cast = AnalyzerCaster.getLegalCast(location, indexed, type, true, true);
+
+        if (expression.actual.sort == Sort.ARRAY) {
+            analyzeArray(variables, type);
+        } else if (expression.actual.sort == Sort.DEF || Iterable.class.isAssignableFrom(expression.actual.clazz)) {
+            analyzeIterable(variables, type);
+        } else {
+            throw location.createError(new IllegalArgumentException("Illegal for each type [" + expression.actual.name + "]."));
+        }
 
         if (block == null) {
             throw location.createError(new IllegalArgumentException("Extraneous for each loop."));
@@ -126,24 +121,19 @@ public class SEach extends AStatement {
         variables.decrementScope();
     }
 
-    void analyzeIterable(Variables variables) {
-        final Type type;
-
-        try {
-            type = Definition.getType(this.type);
-        } catch (IllegalArgumentException exception) {
-            throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
-        }
-
-        variables.incrementScope();
-
-        Type itr = Definition.getType("Iterator");
-
-        variable = variables.addVariable(location, type, name, true, false);
+    void analyzeArray(Variables variables, Type type) {
+        // We must store the array and index as variables for securing slots on the stack, and
+        // also add the location offset to make the names unique in case of nested for each loops.
+        array = variables.addVariable(location, expression.actual, "#array" + location.getOffset(), true, false);
+        index = variables.addVariable(location, Definition.INT_TYPE, "#index" + location.getOffset(), true, false);
+        indexed = Definition.getType(expression.actual.struct, expression.actual.dimensions - 1);
+        cast = AnalyzerCaster.getLegalCast(location, indexed, type, true, true);
+    }
 
+    void analyzeIterable(Variables variables, Type type) {
         // We must store the iterator as a variable for securing a slot on the stack, and
         // also add the location offset to make the name unique in case of nested for each loops.
-        iterator = variables.addVariable(location, itr, "#itr" + location.getOffset(), true, false);
+        iterator = variables.addVariable(location, Definition.getType("Iterator"), "#itr" + location.getOffset(), true, false);
 
         if (expression.actual.sort == Sort.DEF) {
             method = null;
@@ -156,48 +146,13 @@ public class SEach extends AStatement {
             }
         }
 
-        hasNext = itr.struct.methods.get(new MethodKey("hasNext", 0));
-
-        if (hasNext == null) {
-            throw location.createError(new IllegalArgumentException("Method [hasNext] does not exist for type [Iterator]."));
-        } else if (hasNext.rtn.sort != Sort.BOOL) {
-            throw location.createError(new IllegalArgumentException("Method [hasNext] does not return type [boolean]."));
-        }
-
-        next = itr.struct.methods.get(new MethodKey("next", 0));
-
-        if (next == null) {
-            throw location.createError(new IllegalArgumentException("Method [next] does not exist for type [Iterator]."));
-        } else if (next.rtn.sort != Sort.DEF) {
-            throw location.createError(new IllegalArgumentException("Method [next] does not return type [def]."));
-        }
-
         cast = AnalyzerCaster.getLegalCast(location, Definition.DEF_TYPE, type, true, true);
-
-        if (block == null) {
-            throw location.createError(new IllegalArgumentException("Extraneous for each loop."));
-        }
-
-        block.beginLoop = true;
-        block.inLoop = true;
-        block.analyze(variables);
-        block.statementCount = Math.max(1, block.statementCount);
-
-        if (block.loopEscape && !block.anyContinue) {
-            throw createError(new IllegalArgumentException("Extraneous for loop."));
-        }
-
-        statementCount = 1;
-
-        if (maxLoopCounter > 0) {
-            loopCounterSlot = variables.getVariable(location, "#loop").slot;
-        }
-
-        variables.decrementScope();
     }
 
     @Override
     void write(MethodWriter writer) {
+        writer.writeStatementOffset(location);
+
         if (array != null) {
             writeArray(writer);
         } else if (iterator != null) {
@@ -208,8 +163,6 @@ public class SEach extends AStatement {
     }
 
     void writeArray(MethodWriter writer) {
-        writer.writeStatementOffset(location);
-
         expression.write(writer);
         writer.visitVarInsn(array.type.type.getOpcode(Opcodes.ISTORE), array.slot);
         writer.push(-1);
@@ -239,8 +192,6 @@ public class SEach extends AStatement {
     }
 
     void writeIterable(MethodWriter writer) {
-        writer.writeStatementOffset(location);
-
         expression.write(writer);
 
         if (method == null) {
@@ -261,11 +212,11 @@ public class SEach extends AStatement {
         writer.mark(begin);
 
         writer.visitVarInsn(iterator.type.type.getOpcode(Opcodes.ILOAD), iterator.slot);
-        writer.invokeInterface(hasNext.owner.type, hasNext.method);
+        writer.invokeInterface(ITERATOR_TYPE, ITERATOR_HASNEXT);
         writer.ifZCmp(MethodWriter.EQ, end);
 
         writer.visitVarInsn(iterator.type.type.getOpcode(Opcodes.ILOAD), iterator.slot);
-        writer.invokeInterface(next.owner.type, next.method);
+        writer.invokeInterface(ITERATOR_TYPE, ITERATOR_NEXT);
         writer.writeCast(cast);
         writer.visitVarInsn(variable.type.type.getOpcode(Opcodes.ISTORE), variable.slot);
 

From da74323141a69bd06fc19a70bc072143e9f0e986 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 6 Jun 2016 22:09:12 -0400
Subject: [PATCH 0683/1311] Register thread pool settings

This commit refactors the handling of thread pool settings so that the
individual settings can be registered rather than registering the top
level group. With this refactoring, individual plugins must now register
their own settings for custom thread pools that they need, but a
dedicated API is provided for this in the thread pool module. This
commit also renames the prefix on the thread pool settings from
"threadpool" to "thread_pool". This enables a hard break on the settings
so that:
 - some of the settings can be given more sensible names (e.g., the max
   number of threads in a scaling thread pool is now named "max" instead
   of "size")
 - change the soft limit on the number of threads in the bulk and
   indexing thread pools to a hard limit
 - the settings names for custom plugins for thread pools can be
   prefixed (e.g., "xpack.watcher.thread_pool.size")
 - remove dynamic thread pool settings

Relates #18674
---
 .../common/settings/ClusterSettings.java      |   4 +-
 .../common/util/concurrent/EsExecutors.java   |  13 +-
 .../java/org/elasticsearch/node/Node.java     |  15 +-
 .../org/elasticsearch/plugins/Plugin.java     |  14 +
 .../elasticsearch/plugins/PluginsService.java |   9 +
 .../threadpool/ExecutorBuilder.java           |  91 ++++
 .../threadpool/FixedExecutorBuilder.java      | 135 ++++++
 .../threadpool/ScalingExecutorBuilder.java    | 129 ++++++
 .../elasticsearch/threadpool/ThreadPool.java  | 415 +++---------------
 .../threadpool/ThreadPoolModule.java          |  13 +-
 .../action/RejectionActionIT.java             |  12 +-
 .../node/tasks/TaskManagerTestCase.java       |   3 +-
 .../action/bulk/BulkProcessorRetryIT.java     |   6 +-
 .../bulk/TransportBulkActionTookTests.java    |   3 +-
 .../support/ListenableActionFutureTests.java  |   3 +-
 .../TransportBroadcastByNodeActionTests.java  |   3 +-
 .../TransportMasterNodeActionTests.java       |   3 +-
 .../nodes/TransportNodesActionTests.java      |   3 +-
 .../BroadcastReplicationTests.java            |   3 +-
 .../TransportReplicationActionTests.java      |   3 +-
 ...ortInstanceSingleOperationActionTests.java |   3 +-
 .../TransportClientNodesServiceTests.java     |   3 +-
 .../action/shard/ShardStateActionTests.java   |   3 +-
 .../health/ClusterStateHealthTests.java       |   3 +-
 .../DelayedAllocationServiceTests.java        |   3 +-
 .../cluster/service/ClusterServiceTests.java  |   3 +-
 .../concurrent/PrioritizedExecutorsTests.java |   3 +-
 .../discovery/ZenFaultDetectionTests.java     |   3 +-
 .../zen/NodeJoinControllerTests.java          |   3 +-
 .../zen/ping/unicast/UnicastZenPingIT.java    |   3 +-
 .../PublishClusterStateActionTests.java       |   3 +-
 .../gateway/AsyncShardFetchTests.java         |   3 +-
 .../http/netty/NettyHttpChannelTests.java     |   3 +-
 .../netty/NettyHttpServerPipeliningTests.java |   3 +-
 .../netty/NettyHttpServerTransportTests.java  |   3 +-
 .../elasticsearch/index/IndexModuleTests.java |   3 +-
 .../index/engine/InternalEngineTests.java     |   3 +-
 .../index/engine/ShadowEngineTests.java       |   3 +-
 .../fielddata/IndexFieldDataServiceTests.java |   3 +-
 .../mapper/DynamicMappingDisabledTests.java   |   3 +-
 .../indices/store/IndicesStoreTests.java      |   3 +-
 .../jvm/JvmGcMonitorServiceSettingsTests.java |   3 +-
 .../script/NativeScriptTests.java             |   3 +-
 .../search/SearchWithRejectionsIT.java        |   4 +-
 .../threadpool/FixedThreadPoolTests.java      |  17 +-
 .../threadpool/ScalingThreadPoolTests.java    |  88 +---
 .../threadpool/SimpleThreadPoolIT.java        |  57 ---
 .../ThreadPoolSerializationTests.java         |   2 +-
 .../UpdateThreadPoolSettingsTests.java        | 283 +++---------
 .../AbstractSimpleTransportTestCase.java      |   3 +-
 .../NettySizeHeaderFrameDecoderTests.java     |   1 -
 .../NettyTransportServiceHandshakeTests.java  |   3 +-
 .../netty/NettyScheduledPingTests.java        |   3 +-
 .../netty/NettyTransportMultiPortTests.java   |  11 +-
 .../watcher/ResourceWatcherServiceTests.java  |   5 +-
 .../migration/migrate_5_0/settings.asciidoc   |  13 +
 docs/reference/modules/threadpool.asciidoc    |  24 +-
 .../messy/tests/TemplateQueryParserTests.java |   3 +-
 ...tAsyncBulkIndexByScrollActionTestCase.java |   3 +-
 .../reindex/AsyncBulkByScrollActionTests.java |   9 +-
 .../index/reindex/BulkByScrollTaskTests.java  |   5 +-
 .../index/reindex/RetryTests.java             |   8 +-
 .../discovery/ec2/Ec2DiscoveryTests.java      |   3 +-
 .../discovery/gce/GceDiscoveryTests.java      |   3 +-
 .../org/elasticsearch/test/ESTestCase.java    |   1 -
 .../elasticsearch/test/client/NoOpClient.java |   3 +-
 .../threadpool/TestThreadPool.java            |  31 ++
 67 files changed, 705 insertions(+), 827 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/threadpool/ExecutorBuilder.java
 create mode 100644 core/src/main/java/org/elasticsearch/threadpool/FixedExecutorBuilder.java
 create mode 100644 core/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java
 create mode 100644 test/framework/src/main/java/org/elasticsearch/threadpool/TestThreadPool.java

diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index c9abafbbe54..6eb8df68242 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -190,7 +190,6 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     RecoverySettings.INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING,
                     RecoverySettings.INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT_SETTING,
                     RecoverySettings.INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT_SETTING,
-                    ThreadPool.THREADPOOL_GROUP_SETTING,
                     ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING,
                     ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING,
                     ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_OUTGOING_RECOVERIES_SETTING,
@@ -419,6 +418,7 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     ResourceWatcherService.RELOAD_INTERVAL_HIGH,
                     ResourceWatcherService.RELOAD_INTERVAL_MEDIUM,
                     ResourceWatcherService.RELOAD_INTERVAL_LOW,
-                    SearchModule.INDICES_MAX_CLAUSE_COUNT_SETTING
+                    SearchModule.INDICES_MAX_CLAUSE_COUNT_SETTING,
+                    ThreadPool.ESTIMATED_TIME_INTERVAL_SETTING
             )));
 }
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
index 2d45a6fecff..5ac94f8b386 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java
@@ -83,13 +83,16 @@ public class EsExecutors {
     }
 
     public static String threadName(Settings settings, String namePrefix) {
-        String name = settings.get("node.name");
-        if (name == null) {
-            name = "elasticsearch";
+        String nodeName = settings.get("node.name");
+        if (nodeName == null) {
+            return threadName("", namePrefix);
         } else {
-            name = "elasticsearch[" + name + "]";
+            return threadName(nodeName, namePrefix);
         }
-        return name + "[" + namePrefix + "]";
+    }
+
+    public static String threadName(final String nodeName, final String namePrefix) {
+        return "elasticsearch" + (nodeName.isEmpty() ? "" : "[") + nodeName + (nodeName.isEmpty() ? "" : "]") + "[" + namePrefix + "]";
     }
 
     public static ThreadFactory daemonThreadFactory(Settings settings, String namePrefix) {
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index cf33770fd16..04063ce5864 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -98,6 +98,7 @@ import org.elasticsearch.search.SearchService;
 import org.elasticsearch.snapshots.SnapshotShardsService;
 import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.tasks.TaskResultsService;
+import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPoolModule;
 import org.elasticsearch.transport.TransportService;
@@ -210,11 +211,12 @@ public class Node implements Closeable {
             throw new IllegalStateException("Failed to created node environment", ex);
         }
         final NetworkService networkService = new NetworkService(settings);
-        final ThreadPool threadPool = new ThreadPool(settings);
+        final List<ExecutorBuilder<?>> executorBuilders = pluginsService.getExecutorBuilders(settings);
+        final ThreadPool threadPool = new ThreadPool(settings, executorBuilders.toArray(new ExecutorBuilder[0]));
+
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
         boolean success = false;
         try {
-            final MonitorService monitorService = new MonitorService(settings, nodeEnvironment, threadPool);
             ModulesBuilder modules = new ModulesBuilder();
             modules.add(new Version.Module(version));
             modules.add(new CircuitBreakerModule(settings));
@@ -222,6 +224,7 @@ public class Node implements Closeable {
             for (Module pluginModule : pluginsService.nodeModules()) {
                 modules.add(pluginModule);
             }
+            final MonitorService monitorService = new MonitorService(settings, nodeEnvironment, threadPool);
             modules.add(new PluginsModule(pluginsService));
             SettingsModule settingsModule = new SettingsModule(this.settings);
             modules.add(settingsModule);
@@ -232,7 +235,8 @@ public class Node implements Closeable {
             modules.add(scriptModule);
             modules.add(new NodeEnvironmentModule(nodeEnvironment));
             modules.add(new ClusterNameModule(this.settings));
-            modules.add(new ThreadPoolModule(threadPool));
+            final ThreadPoolModule threadPoolModule = new ThreadPoolModule(threadPool);
+            modules.add(threadPoolModule);
             modules.add(new DiscoveryModule(this.settings));
             modules.add(new ClusterModule(this.settings));
             modules.add(new IndicesModule());
@@ -246,11 +250,14 @@ public class Node implements Closeable {
             modules.add(new AnalysisModule(environment));
 
             pluginsService.processModules(modules);
+
             scriptModule.prepareSettings(settingsModule);
+
+            threadPoolModule.prepareSettings(settingsModule);
+
             injector = modules.createInjector();
 
             client = injector.getInstance(Client.class);
-            threadPool.setClusterSettings(injector.getInstance(ClusterSettings.class));
             success = true;
         } catch (IOException ex) {
             throw new ElasticsearchException("failed to bind service", ex);
diff --git a/core/src/main/java/org/elasticsearch/plugins/Plugin.java b/core/src/main/java/org/elasticsearch/plugins/Plugin.java
index 1efc151836d..0283567bf80 100644
--- a/core/src/main/java/org/elasticsearch/plugins/Plugin.java
+++ b/core/src/main/java/org/elasticsearch/plugins/Plugin.java
@@ -23,9 +23,12 @@ import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexModule;
+import org.elasticsearch.threadpool.ExecutorBuilder;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 
 /**
  * An extension point allowing to plug in custom functionality.
@@ -80,4 +83,15 @@ public abstract class Plugin {
      */
     @Deprecated
     public final void onModule(IndexModule indexModule) {}
+
+    /**
+     * Provides the list of this plugin's custom thread pools, empty if
+     * none.
+     *
+     * @param settings the current settings
+     * @return executors builders for this plugin's custom thread pools
+     */
+    public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) {
+        return Collections.emptyList();
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
index f373da6987d..4f123625387 100644
--- a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
+++ b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
@@ -40,6 +40,7 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexModule;
+import org.elasticsearch.threadpool.ExecutorBuilder;
 
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
@@ -261,6 +262,14 @@ public class PluginsService extends AbstractComponent {
         return modules;
     }
 
+    public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) {
+        final ArrayList<ExecutorBuilder<?>> builders = new ArrayList<>();
+        for (final Tuple<PluginInfo, Plugin> plugin : plugins) {
+            builders.addAll(plugin.v2().getExecutorBuilders(settings));
+        }
+        return builders;
+    }
+
     public Collection<Class<? extends LifecycleComponent>> nodeServices() {
         List<Class<? extends LifecycleComponent>> services = new ArrayList<>();
         for (Tuple<PluginInfo, Plugin> plugin : plugins) {
diff --git a/core/src/main/java/org/elasticsearch/threadpool/ExecutorBuilder.java b/core/src/main/java/org/elasticsearch/threadpool/ExecutorBuilder.java
new file mode 100644
index 00000000000..434e6fc509c
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/threadpool/ExecutorBuilder.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.threadpool;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+
+import java.util.List;
+
+/**
+ * Base class for executor builders.
+ *
+ * @param <U> the underlying type of the executor settings
+ */
+public abstract class ExecutorBuilder<U extends ExecutorBuilder.ExecutorSettings> {
+
+    private final String name;
+
+    public ExecutorBuilder(String name) {
+        this.name = name;
+    }
+
+    protected String name() {
+        return name;
+    }
+
+    protected static String settingsKey(final String prefix, final String key) {
+        return String.join(".", prefix, key);
+    }
+
+    /**
+     * The list of settings this builder will register.
+     *
+     * @return the list of registered settings
+     */
+    abstract List<Setting<?>> getRegisteredSettings();
+
+    /**
+     * Return an executor settings object from the node-level settings.
+     *
+     * @param settings the node-level settings
+     * @return the executor settings object
+     */
+    abstract U getSettings(Settings settings);
+
+    /**
+     * Builds the executor with the specified executor settings.
+     *
+     * @param settings      the executor settings
+     * @param threadContext the current thread context
+     * @return a new executor built from the specified executor settings
+     */
+    abstract ThreadPool.ExecutorHolder build(U settings, ThreadContext threadContext);
+
+    /**
+     * Format the thread pool info object for this executor.
+     *
+     * @param info the thread pool info object to format
+     * @return a formatted thread pool info (useful for logging)
+     */
+    abstract String formatInfo(ThreadPool.Info info);
+
+    static abstract class ExecutorSettings {
+
+        protected final String nodeName;
+
+        public ExecutorSettings(String nodeName) {
+            this.nodeName = nodeName;
+        }
+
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/threadpool/FixedExecutorBuilder.java b/core/src/main/java/org/elasticsearch/threadpool/FixedExecutorBuilder.java
new file mode 100644
index 00000000000..0735774d972
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/threadpool/FixedExecutorBuilder.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.threadpool;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.SizeValue;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.node.Node;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ThreadFactory;
+
+/**
+ * A builder for fixed executors.
+ */
+public final class FixedExecutorBuilder extends ExecutorBuilder<FixedExecutorBuilder.FixedExecutorSettings> {
+
+    private final Setting<Integer> sizeSetting;
+    private final Setting<Integer> queueSizeSetting;
+
+    /**
+     * Construct a fixed executor builder; the settings will have the
+     * key prefix "thread_pool." followed by the executor name.
+     *
+     * @param settings  the node-level settings
+     * @param name      the name of the executor
+     * @param size      the fixed number of threads
+     * @param queueSize the size of the backing queue, -1 for unbounded
+     */
+    FixedExecutorBuilder(final Settings settings, final String name, final int size, final int queueSize) {
+        this(settings, name, size, queueSize, "thread_pool." + name);
+    }
+
+    /**
+     * Construct a fixed executor builder.
+     *
+     * @param settings  the node-level settings
+     * @param name      the name of the executor
+     * @param size      the fixed number of threads
+     * @param queueSize the size of the backing queue, -1 for unbounded
+     * @param prefix    the prefix for the settings keys
+     */
+    public FixedExecutorBuilder(final Settings settings, final String name, final int size, final int queueSize, final String prefix) {
+        super(name);
+        final String sizeKey = settingsKey(prefix, "size");
+        this.sizeSetting =
+            new Setting<>(
+                sizeKey,
+                s -> Integer.toString(size),
+                s -> Setting.parseInt(s, 1, applyHardSizeLimit(settings, name), sizeKey),
+                Setting.Property.NodeScope);
+        final String queueSizeKey = settingsKey(prefix, "queue_size");
+        this.queueSizeSetting =
+            Setting.intSetting(queueSizeKey, queueSize, Setting.Property.NodeScope);
+    }
+
+    private int applyHardSizeLimit(final Settings settings, final String name) {
+        if (name.equals(ThreadPool.Names.BULK) || name.equals(ThreadPool.Names.INDEX)) {
+            return 1 + EsExecutors.boundedNumberOfProcessors(settings);
+        } else {
+            return Integer.MAX_VALUE;
+        }
+    }
+
+    @Override
+    List<Setting<?>> getRegisteredSettings() {
+        return Arrays.asList(sizeSetting, queueSizeSetting);
+    }
+
+    @Override
+    FixedExecutorSettings getSettings(Settings settings) {
+        final String nodeName = Node.NODE_NAME_SETTING.get(settings);
+        final int size = sizeSetting.get(settings);
+        final int queueSize = queueSizeSetting.get(settings);
+        return new FixedExecutorSettings(nodeName, size, queueSize);
+    }
+
+    @Override
+    ThreadPool.ExecutorHolder build(final FixedExecutorSettings settings, final ThreadContext threadContext) {
+        int size = settings.size;
+        int queueSize = settings.queueSize;
+        final ThreadFactory threadFactory = EsExecutors.daemonThreadFactory(EsExecutors.threadName(settings.nodeName, name()));
+        Executor executor = EsExecutors.newFixed(name(), size, queueSize, threadFactory, threadContext);
+        final ThreadPool.Info info =
+            new ThreadPool.Info(name(), ThreadPool.ThreadPoolType.FIXED, size, size, null, queueSize < 0 ? null : new SizeValue(queueSize));
+        return new ThreadPool.ExecutorHolder(executor, info);
+    }
+
+    @Override
+    String formatInfo(ThreadPool.Info info) {
+        return String.format(
+            Locale.ROOT,
+            "name [%s], size [%d], queue size [%s]",
+            info.getName(),
+            info.getMax(),
+            info.getQueueSize() == null ? "unbounded" : info.getQueueSize());
+    }
+
+    static class FixedExecutorSettings extends ExecutorBuilder.ExecutorSettings {
+
+        private final int size;
+        private final int queueSize;
+
+        public FixedExecutorSettings(final String nodeName, final int size, final int queueSize) {
+            super(nodeName);
+            this.size = size;
+            this.queueSize = queueSize;
+        }
+
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java b/core/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java
new file mode 100644
index 00000000000..68c70c83c19
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.threadpool;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.node.Node;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A builder for scaling executors.
+ */
+public final class ScalingExecutorBuilder extends ExecutorBuilder<ScalingExecutorBuilder.ScalingExecutorSettings> {
+
+    private final Setting<Integer> coreSetting;
+    private final Setting<Integer> maxSetting;
+    private final Setting<TimeValue> keepAliveSetting;
+
+    /**
+     * Construct a scaling executor builder; the settings will have the
+     * key prefix "thread_pool." followed by the executor name.
+     *
+     * @param name      the name of the executor
+     * @param core      the minimum number of threads in the pool
+     * @param max       the maximum number of threads in the pool
+     * @param keepAlive the time that spare threads above {@code core}
+     *                  threads will be kept alive
+     */
+    public ScalingExecutorBuilder(final String name, final int core, final int max, final TimeValue keepAlive) {
+        this(name, core, max, keepAlive, "thread_pool." + name);
+    }
+
+    /**
+     * Construct a scaling executor builder; the settings will have the
+     * specified key prefix.
+     *
+     * @param name      the name of the executor
+     * @param core      the minimum number of threads in the pool
+     * @param max       the maximum number of threads in the pool
+     * @param keepAlive the time that spare threads above {@code core}
+     *                  threads will be kept alive
+     * @param prefix    the prefix for the settings keys
+     */
+    public ScalingExecutorBuilder(final String name, final int core, final int max, final TimeValue keepAlive, final String prefix) {
+        super(name);
+        this.coreSetting =
+            Setting.intSetting(settingsKey(prefix, "core"), core, Setting.Property.NodeScope);
+        this.maxSetting = Setting.intSetting(settingsKey(prefix, "max"), max, Setting.Property.NodeScope);
+        this.keepAliveSetting =
+            Setting.timeSetting(settingsKey(prefix, "keep_alive"), keepAlive, Setting.Property.NodeScope);
+    }
+
+    @Override
+    List<Setting<?>> getRegisteredSettings() {
+        return Arrays.asList(coreSetting, maxSetting, keepAliveSetting);
+    }
+
+    @Override
+    ScalingExecutorSettings getSettings(Settings settings) {
+        final String nodeName = Node.NODE_NAME_SETTING.get(settings);
+        final int coreThreads = coreSetting.get(settings);
+        final int maxThreads = maxSetting.get(settings);
+        final TimeValue keepAlive = keepAliveSetting.get(settings);
+        return new ScalingExecutorSettings(nodeName, coreThreads, maxThreads, keepAlive);
+    }
+
+    ThreadPool.ExecutorHolder build(final ScalingExecutorSettings settings, final ThreadContext threadContext) {
+        TimeValue keepAlive = settings.keepAlive;
+        int core = settings.core;
+        int max = settings.max;
+        final ThreadPool.Info info = new ThreadPool.Info(name(), ThreadPool.ThreadPoolType.SCALING, core, max, keepAlive, null);
+        final ThreadFactory threadFactory = EsExecutors.daemonThreadFactory(EsExecutors.threadName(settings.nodeName, name()));
+        final Executor executor =
+            EsExecutors.newScaling(name(), core, max, keepAlive.millis(), TimeUnit.MILLISECONDS, threadFactory, threadContext);
+        return new ThreadPool.ExecutorHolder(executor, info);
+    }
+
+    @Override
+    String formatInfo(ThreadPool.Info info) {
+        return String.format(
+            Locale.ROOT,
+            "name [%s], core [%d], max [%d], keep alive [%s]",
+            info.getName(),
+            info.getMin(),
+            info.getMax(),
+            info.getKeepAlive());
+    }
+
+    static class ScalingExecutorSettings extends ExecutorBuilder.ExecutorSettings {
+
+        private final int core;
+        private final int max;
+        private final TimeValue keepAlive;
+
+        public ScalingExecutorSettings(final String nodeName, final int core, final int max, final TimeValue keepAlive) {
+            super(nodeName);
+            this.core = core;
+            this.max = max;
+            this.keepAlive = keepAlive;
+        }
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
index 0d24bb74e17..5c31323b3d8 100644
--- a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
+++ b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
@@ -26,11 +26,8 @@ import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.unit.SizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsAbortPolicy;
@@ -45,31 +42,22 @@ import org.elasticsearch.node.Node;
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.RejectedExecutionHandler;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
 
 import static java.util.Collections.unmodifiableMap;
-import static org.elasticsearch.common.unit.SizeValue.parseSizeValue;
-import static org.elasticsearch.common.unit.TimeValue.timeValueMinutes;
 
-/**
- *
- */
 public class ThreadPool extends AbstractComponent implements Closeable {
 
     public static class Names {
@@ -146,164 +134,85 @@ public class ThreadPool extends AbstractComponent implements Closeable {
         THREAD_POOL_TYPES = Collections.unmodifiableMap(map);
     }
 
-    private static void add(Map<String, Settings> executorSettings, ExecutorSettingsBuilder builder) {
-        Settings settings = builder.build();
-        String name = settings.get("name");
-        executorSettings.put(name, settings);
-    }
-
-    private static abstract class ExecutorSettingsBuilder<T extends ExecutorSettingsBuilder<T>> {
-
-        private final Settings.Builder builder;
-
-        protected ExecutorSettingsBuilder(String name, ThreadPoolType threadPoolType) {
-            if (THREAD_POOL_TYPES.get(name) != threadPoolType) {
-                throw new IllegalArgumentException("thread pool [" + name + "] must be of type [" + threadPoolType + "]");
-            }
-            builder = Settings.builder();
-            builder.put("name", name);
-            builder.put("type", threadPoolType.getType());
-        }
-
-        public T keepAlive(String keepAlive) {
-            return add("keep_alive", keepAlive);
-        }
-
-        public T queueSize(int queueSize) {
-            return add("queue_size", queueSize);
-        }
-
-        protected T add(String setting, int value) {
-            return add(setting, Integer.toString(value));
-        }
-
-
-        protected T add(String setting, String value) {
-            builder.put(setting, value);
-            @SuppressWarnings("unchecked") final T executor = (T)this;
-            return executor;
-        }
-
-        public final Settings build() { return builder.build(); }
-
-    }
-
-    private static class FixedExecutorSettingsBuilder extends ExecutorSettingsBuilder<FixedExecutorSettingsBuilder> {
-
-        public FixedExecutorSettingsBuilder(String name) {
-            super(name, ThreadPoolType.FIXED);
-        }
-
-        public FixedExecutorSettingsBuilder size(int size) {
-            return add("size", Integer.toString(size));
-        }
-
-    }
-
-    private static class ScalingExecutorSettingsBuilder extends ExecutorSettingsBuilder<ScalingExecutorSettingsBuilder> {
-
-        public ScalingExecutorSettingsBuilder(String name) {
-            super(name, ThreadPoolType.SCALING);
-        }
-
-        public ScalingExecutorSettingsBuilder min(int min) {
-            return add("min", min);
-        }
-
-
-        public ScalingExecutorSettingsBuilder size(int size) {
-            return add("size", size);
-        }
-    }
-
-    public static final Setting<Settings> THREADPOOL_GROUP_SETTING =
-        Setting.groupSetting("threadpool.", Property.Dynamic, Property.NodeScope);
-
-    private volatile Map<String, ExecutorHolder> executors;
-
-    private final Map<String, Settings> defaultExecutorTypeSettings;
-
-    private final Queue<ExecutorHolder> retiredExecutors = new ConcurrentLinkedQueue<>();
+    private Map<String, ExecutorHolder> executors = new HashMap<>();
 
     private final ScheduledThreadPoolExecutor scheduler;
 
     private final EstimatedTimeThread estimatedTimeThread;
 
-    private final AtomicBoolean settingsListenerIsSet = new AtomicBoolean(false);
-
     static final Executor DIRECT_EXECUTOR = command -> command.run();
 
     private final ThreadContext threadContext;
 
-    public ThreadPool(String name) {
-        this(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), name).build());
+    private final Map<String, ExecutorBuilder> builders;
+
+    public Collection<ExecutorBuilder> builders() {
+        return Collections.unmodifiableCollection(builders.values());
     }
 
-    public ThreadPool(Settings settings) {
+    public static Setting<TimeValue> ESTIMATED_TIME_INTERVAL_SETTING =
+        Setting.timeSetting("thread_pool.estimated_time_interval", TimeValue.timeValueMillis(200), Setting.Property.NodeScope);
+
+    public ThreadPool(final Settings settings, final ExecutorBuilder<?>... customBuilders) {
         super(settings);
 
-        assert Node.NODE_NAME_SETTING.exists(settings) : "ThreadPool's settings should contain a name";
-        threadContext = new ThreadContext(settings);
-        Map<String, Settings> groupSettings = THREADPOOL_GROUP_SETTING.get(settings).getAsGroups();
-        validate(groupSettings);
+        assert Node.NODE_NAME_SETTING.exists(settings);
 
-        int availableProcessors = EsExecutors.boundedNumberOfProcessors(settings);
-        int halfProcMaxAt5 = halfNumberOfProcessorsMaxFive(availableProcessors);
-        int halfProcMaxAt10 = halfNumberOfProcessorsMaxTen(availableProcessors);
-        Map<String, Settings> defaultExecutorTypeSettings = new HashMap<>();
-        int genericThreadPoolMax = boundedBy(4 * availableProcessors, 128, 512);
-        add(defaultExecutorTypeSettings, new ScalingExecutorSettingsBuilder(Names.GENERIC).min(4).size(genericThreadPoolMax).keepAlive("30s"));
-        add(defaultExecutorTypeSettings, new FixedExecutorSettingsBuilder(Names.INDEX).size(availableProcessors).queueSize(200));
-        add(defaultExecutorTypeSettings, new FixedExecutorSettingsBuilder(Names.BULK).size(availableProcessors).queueSize(50));
-        add(defaultExecutorTypeSettings, new FixedExecutorSettingsBuilder(Names.GET).size(availableProcessors).queueSize(1000));
-        add(defaultExecutorTypeSettings, new FixedExecutorSettingsBuilder(Names.SEARCH).size(((availableProcessors * 3) / 2) + 1).queueSize(1000));
-        add(defaultExecutorTypeSettings, new ScalingExecutorSettingsBuilder(Names.MANAGEMENT).min(1).size(5).keepAlive("5m"));
+        final Map<String, ExecutorBuilder> builders = new HashMap<>();
+        final int availableProcessors = EsExecutors.boundedNumberOfProcessors(settings);
+        final int halfProcMaxAt5 = halfNumberOfProcessorsMaxFive(availableProcessors);
+        final int halfProcMaxAt10 = halfNumberOfProcessorsMaxTen(availableProcessors);
+        final int genericThreadPoolMax = boundedBy(4 * availableProcessors, 128, 512);
+        builders.put(Names.GENERIC, new ScalingExecutorBuilder(Names.GENERIC, 4, genericThreadPoolMax, TimeValue.timeValueSeconds(30)));
+        builders.put(Names.INDEX, new FixedExecutorBuilder(settings, Names.INDEX, availableProcessors, 200));
+        builders.put(Names.BULK, new FixedExecutorBuilder(settings, Names.BULK, availableProcessors, 50));
+        builders.put(Names.GET, new FixedExecutorBuilder(settings, Names.GET, availableProcessors, 1000));
+        builders.put(Names.SEARCH, new FixedExecutorBuilder(settings, Names.SEARCH, ((availableProcessors * 3) / 2) + 1, 1000));
+        builders.put(Names.MANAGEMENT, new ScalingExecutorBuilder(Names.MANAGEMENT, 1, 5, TimeValue.timeValueMinutes(5)));
         // no queue as this means clients will need to handle rejections on listener queue even if the operation succeeded
         // the assumption here is that the listeners should be very lightweight on the listeners side
-        add(defaultExecutorTypeSettings, new FixedExecutorSettingsBuilder(Names.LISTENER).size(halfProcMaxAt10));
-        add(defaultExecutorTypeSettings, new ScalingExecutorSettingsBuilder(Names.FLUSH).min(1).size(halfProcMaxAt5).keepAlive("5m"));
-        add(defaultExecutorTypeSettings, new ScalingExecutorSettingsBuilder(Names.REFRESH).min(1).size(halfProcMaxAt10).keepAlive("5m"));
-        add(defaultExecutorTypeSettings, new ScalingExecutorSettingsBuilder(Names.WARMER).min(1).size(halfProcMaxAt5).keepAlive("5m"));
-        add(defaultExecutorTypeSettings, new ScalingExecutorSettingsBuilder(Names.SNAPSHOT).min(1).size(halfProcMaxAt5).keepAlive("5m"));
-        add(defaultExecutorTypeSettings, new FixedExecutorSettingsBuilder(Names.FORCE_MERGE).size(1));
-        add(defaultExecutorTypeSettings, new ScalingExecutorSettingsBuilder(Names.FETCH_SHARD_STARTED).min(1).size(availableProcessors * 2).keepAlive("5m"));
-        add(defaultExecutorTypeSettings, new ScalingExecutorSettingsBuilder(Names.FETCH_SHARD_STORE).min(1).size(availableProcessors * 2).keepAlive("5m"));
-
-        this.defaultExecutorTypeSettings = unmodifiableMap(defaultExecutorTypeSettings);
-
-        Map<String, ExecutorHolder> executors = new HashMap<>();
-        for (Map.Entry<String, Settings> executor : defaultExecutorTypeSettings.entrySet()) {
-            executors.put(executor.getKey(), build(executor.getKey(), groupSettings.get(executor.getKey()), executor.getValue()));
-        }
-
-        // Building custom thread pools
-        for (Map.Entry<String, Settings> entry : groupSettings.entrySet()) {
-            if (executors.containsKey(entry.getKey())) {
-                continue;
+        builders.put(Names.LISTENER, new FixedExecutorBuilder(settings, Names.LISTENER, halfProcMaxAt10, -1));
+        builders.put(Names.FLUSH, new ScalingExecutorBuilder(Names.FLUSH, 1, halfProcMaxAt5, TimeValue.timeValueMinutes(5)));
+        builders.put(Names.REFRESH, new ScalingExecutorBuilder(Names.REFRESH, 1, halfProcMaxAt10, TimeValue.timeValueMinutes(5)));
+        builders.put(Names.WARMER, new ScalingExecutorBuilder(Names.WARMER, 1, halfProcMaxAt5, TimeValue.timeValueMinutes(5)));
+        builders.put(Names.SNAPSHOT, new ScalingExecutorBuilder(Names.SNAPSHOT, 1, halfProcMaxAt5, TimeValue.timeValueMinutes(5)));
+        builders.put(Names.FETCH_SHARD_STARTED, new ScalingExecutorBuilder(Names.FETCH_SHARD_STARTED, 1, 2 * availableProcessors, TimeValue.timeValueMinutes(5)));
+        builders.put(Names.FORCE_MERGE, new FixedExecutorBuilder(settings, Names.FORCE_MERGE, 1, -1));
+        builders.put(Names.FETCH_SHARD_STORE, new ScalingExecutorBuilder(Names.FETCH_SHARD_STORE, 1, 2 * availableProcessors, TimeValue.timeValueMinutes(5)));
+        for (final ExecutorBuilder<?> builder : customBuilders) {
+            if (builders.containsKey(builder.name())) {
+                throw new IllegalArgumentException("builder with name [" + builder.name() + "] already exists");
             }
-            executors.put(entry.getKey(), build(entry.getKey(), entry.getValue(), Settings.EMPTY));
+            builders.put(builder.name(), builder);
+        }
+        this.builders = Collections.unmodifiableMap(builders);
+
+        threadContext = new ThreadContext(settings);
+
+        final Map<String, ExecutorHolder> executors = new HashMap<>();
+        for (@SuppressWarnings("unchecked") final Map.Entry<String, ExecutorBuilder> entry : builders.entrySet()) {
+            final ExecutorBuilder.ExecutorSettings executorSettings = entry.getValue().getSettings(settings);
+            final ExecutorHolder executorHolder = entry.getValue().build(executorSettings, threadContext);
+            if (executors.containsKey(executorHolder.info.getName())) {
+                throw new IllegalStateException("duplicate executors with name [" + executorHolder.info.getName() + "] registered");
+            }
+            logger.debug("created thread pool: " + entry.getValue().formatInfo(executorHolder.info));
+            executors.put(entry.getKey(), executorHolder);
         }
 
         executors.put(Names.SAME, new ExecutorHolder(DIRECT_EXECUTOR, new Info(Names.SAME, ThreadPoolType.DIRECT)));
         this.executors = unmodifiableMap(executors);
+
         this.scheduler = new ScheduledThreadPoolExecutor(1, EsExecutors.daemonThreadFactory(settings, "scheduler"), new EsAbortPolicy());
         this.scheduler.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
         this.scheduler.setContinueExistingPeriodicTasksAfterShutdownPolicy(false);
         this.scheduler.setRemoveOnCancelPolicy(true);
 
-        TimeValue estimatedTimeInterval = settings.getAsTime("threadpool.estimated_time_interval", TimeValue.timeValueMillis(200));
+        TimeValue estimatedTimeInterval = ESTIMATED_TIME_INTERVAL_SETTING.get(settings);
         this.estimatedTimeThread = new EstimatedTimeThread(EsExecutors.threadName(settings, "[timer]"), estimatedTimeInterval.millis());
         this.estimatedTimeThread.start();
     }
 
-    public void setClusterSettings(ClusterSettings clusterSettings) {
-        if(settingsListenerIsSet.compareAndSet(false, true)) {
-            clusterSettings.addSettingsUpdateConsumer(THREADPOOL_GROUP_SETTING, this::updateSettings, (s) -> validate(s.getAsGroups()));
-        } else {
-            throw new IllegalStateException("the node settings listener was set more then once");
-        }
-    }
-
     public long estimatedTimeInMillis() {
         return estimatedTimeThread.estimatedTimeInMillis();
     }
@@ -440,12 +349,6 @@ public class ThreadPool extends AbstractComponent implements Closeable {
                 ((ThreadPoolExecutor) executor.executor()).shutdownNow();
             }
         }
-
-        ExecutorHolder holder;
-        while ((holder = retiredExecutors.poll()) != null) {
-            ThreadPoolExecutor executor = (ThreadPoolExecutor) holder.executor();
-            executor.shutdownNow();
-        }
     }
 
     public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
@@ -456,142 +359,10 @@ public class ThreadPool extends AbstractComponent implements Closeable {
             }
         }
 
-        ExecutorHolder holder;
-        while ((holder = retiredExecutors.poll()) != null) {
-            ThreadPoolExecutor executor = (ThreadPoolExecutor) holder.executor();
-            result &= executor.awaitTermination(timeout, unit);
-        }
-
         estimatedTimeThread.join(unit.toMillis(timeout));
         return result;
     }
 
-    private ExecutorHolder build(String name, @Nullable Settings settings, Settings defaultSettings) {
-        return rebuild(name, null, settings, defaultSettings);
-    }
-
-    private ExecutorHolder rebuild(String name, ExecutorHolder previousExecutorHolder, @Nullable Settings settings, Settings defaultSettings) {
-        if (Names.SAME.equals(name)) {
-            // Don't allow to change the "same" thread executor
-            return previousExecutorHolder;
-        }
-        if (settings == null) {
-            settings = Settings.Builder.EMPTY_SETTINGS;
-        }
-        Info previousInfo = previousExecutorHolder != null ? previousExecutorHolder.info : null;
-        String type = settings.get("type", previousInfo != null ? previousInfo.getThreadPoolType().getType() : defaultSettings.get("type"));
-        ThreadPoolType threadPoolType = ThreadPoolType.fromType(type);
-        ThreadFactory threadFactory = EsExecutors.daemonThreadFactory(this.settings, name);
-        if (ThreadPoolType.DIRECT == threadPoolType) {
-            if (previousExecutorHolder != null) {
-                logger.debug("updating thread pool [{}], type [{}]", name, type);
-            } else {
-                logger.debug("creating thread pool [{}], type [{}]", name, type);
-            }
-            return new ExecutorHolder(DIRECT_EXECUTOR, new Info(name, threadPoolType));
-        } else if (ThreadPoolType.FIXED == threadPoolType) {
-            int defaultSize = defaultSettings.getAsInt("size", EsExecutors.boundedNumberOfProcessors(settings));
-            SizeValue defaultQueueSize = getAsSizeOrUnbounded(defaultSettings, "queue", getAsSizeOrUnbounded(defaultSettings, "queue_size", null));
-
-            if (previousExecutorHolder != null) {
-                assert previousInfo != null;
-                if (ThreadPoolType.FIXED == previousInfo.getThreadPoolType()) {
-                    SizeValue updatedQueueSize = getAsSizeOrUnbounded(settings, "capacity", getAsSizeOrUnbounded(settings, "queue", getAsSizeOrUnbounded(settings, "queue_size", previousInfo.getQueueSize())));
-                    if (Objects.equals(previousInfo.getQueueSize(), updatedQueueSize)) {
-                        int updatedSize = applyHardSizeLimit(name, settings.getAsInt("size", previousInfo.getMax()));
-                        if (previousInfo.getMax() != updatedSize) {
-                            logger.debug("updating thread pool [{}], type [{}], size [{}], queue_size [{}]", name, type, updatedSize, updatedQueueSize);
-                            // if you think this code is crazy: that's because it is!
-                            if (updatedSize > previousInfo.getMax()) {
-                                ((EsThreadPoolExecutor) previousExecutorHolder.executor()).setMaximumPoolSize(updatedSize);
-                                ((EsThreadPoolExecutor) previousExecutorHolder.executor()).setCorePoolSize(updatedSize);
-                            } else {
-                                ((EsThreadPoolExecutor) previousExecutorHolder.executor()).setCorePoolSize(updatedSize);
-                                ((EsThreadPoolExecutor) previousExecutorHolder.executor()).setMaximumPoolSize(updatedSize);
-                            }
-                            return new ExecutorHolder(previousExecutorHolder.executor(), new Info(name, threadPoolType, updatedSize, updatedSize, null, updatedQueueSize));
-                        }
-                        return previousExecutorHolder;
-                    }
-                }
-                if (previousInfo.getMax() >= 0) {
-                    defaultSize = previousInfo.getMax();
-                }
-                defaultQueueSize = previousInfo.getQueueSize();
-            }
-
-            int size = applyHardSizeLimit(name, settings.getAsInt("size", defaultSize));
-            SizeValue queueSize = getAsSizeOrUnbounded(settings, "capacity", getAsSizeOrUnbounded(settings, "queue", getAsSizeOrUnbounded(settings, "queue_size", defaultQueueSize)));
-            logger.debug("creating thread pool [{}], type [{}], size [{}], queue_size [{}]", name, type, size, queueSize);
-            Executor executor = EsExecutors.newFixed(name, size, queueSize == null ? -1 : (int) queueSize.singles(), threadFactory, threadContext);
-            return new ExecutorHolder(executor, new Info(name, threadPoolType, size, size, null, queueSize));
-        } else if (ThreadPoolType.SCALING == threadPoolType) {
-            TimeValue defaultKeepAlive = defaultSettings.getAsTime("keep_alive", timeValueMinutes(5));
-            int defaultMin = defaultSettings.getAsInt("min", 1);
-            int defaultSize = defaultSettings.getAsInt("size", EsExecutors.boundedNumberOfProcessors(settings));
-            final Integer queueSize = settings.getAsInt("queue_size", defaultSettings.getAsInt("queue_size", null));
-            if (queueSize != null) {
-                throw new IllegalArgumentException("thread pool [" + name + "] of type scaling can not have its queue re-sized but was [" + queueSize + "]");
-            }
-            if (previousExecutorHolder != null) {
-                if (ThreadPoolType.SCALING == previousInfo.getThreadPoolType()) {
-                    TimeValue updatedKeepAlive = settings.getAsTime("keep_alive", previousInfo.getKeepAlive());
-                    int updatedMin = settings.getAsInt("min", previousInfo.getMin());
-                    int updatedSize = settings.getAsInt("max", settings.getAsInt("size", previousInfo.getMax()));
-                    if (!previousInfo.getKeepAlive().equals(updatedKeepAlive) || previousInfo.getMin() != updatedMin || previousInfo.getMax() != updatedSize) {
-                        logger.debug("updating thread pool [{}], type [{}], keep_alive [{}]", name, type, updatedKeepAlive);
-                        if (!previousInfo.getKeepAlive().equals(updatedKeepAlive)) {
-                            ((EsThreadPoolExecutor) previousExecutorHolder.executor()).setKeepAliveTime(updatedKeepAlive.millis(), TimeUnit.MILLISECONDS);
-                        }
-                        if (previousInfo.getMin() != updatedMin) {
-                            ((EsThreadPoolExecutor) previousExecutorHolder.executor()).setCorePoolSize(updatedMin);
-                        }
-                        if (previousInfo.getMax() != updatedSize) {
-                            ((EsThreadPoolExecutor) previousExecutorHolder.executor()).setMaximumPoolSize(updatedSize);
-                        }
-                        return new ExecutorHolder(previousExecutorHolder.executor(), new Info(name, threadPoolType, updatedMin, updatedSize, updatedKeepAlive, null));
-                    }
-                    return previousExecutorHolder;
-                }
-                if (previousInfo.getKeepAlive() != null) {
-                    defaultKeepAlive = previousInfo.getKeepAlive();
-                }
-                if (previousInfo.getMin() >= 0) {
-                    defaultMin = previousInfo.getMin();
-                }
-                if (previousInfo.getMax() >= 0) {
-                    defaultSize = previousInfo.getMax();
-                }
-            }
-            TimeValue keepAlive = settings.getAsTime("keep_alive", defaultKeepAlive);
-            int min = settings.getAsInt("min", defaultMin);
-            int size = settings.getAsInt("max", settings.getAsInt("size", defaultSize));
-            if (previousExecutorHolder != null) {
-                logger.debug("updating thread pool [{}], type [{}], min [{}], size [{}], keep_alive [{}]", name, type, min, size, keepAlive);
-            } else {
-                logger.debug("creating thread pool [{}], type [{}], min [{}], size [{}], keep_alive [{}]", name, type, min, size, keepAlive);
-            }
-            Executor executor = EsExecutors.newScaling(name, min, size, keepAlive.millis(), TimeUnit.MILLISECONDS, threadFactory, threadContext);
-            return new ExecutorHolder(executor, new Info(name, threadPoolType, min, size, keepAlive, null));
-        }
-        throw new IllegalArgumentException("No type found [" + type + "], for [" + name + "]");
-    }
-
-    private int applyHardSizeLimit(String name, int size) {
-        int availableProcessors = EsExecutors.boundedNumberOfProcessors(settings);
-        if ((name.equals(Names.BULK) || name.equals(Names.INDEX)) && size > availableProcessors) {
-            // We use a hard max size for the indexing pools, because if too many threads enter Lucene's IndexWriter, it means
-            // too many segments written, too frequently, too much merging, etc:
-            // TODO: I would love to be loud here (throw an exception if you ask for a too-big size), but I think this is dangerous
-            // because on upgrade this setting could be in cluster state and hard for the user to correct?
-            logger.warn("requested thread pool size [{}] for [{}] is too large; setting to maximum [{}] instead",
-                        size, name, availableProcessors);
-            size = availableProcessors;
-        }
-
-        return size;
-    }
-
     /**
      * Constrains a value between minimum and maximum values
      * (inclusive).
@@ -618,92 +389,6 @@ public class ThreadPool extends AbstractComponent implements Closeable {
         return boundedBy(2 * numberOfProcessors, 2, Integer.MAX_VALUE);
     }
 
-    private void updateSettings(Settings settings) {
-        Map<String, Settings> groupSettings = settings.getAsGroups();
-        if (groupSettings.isEmpty()) {
-            return;
-        }
-
-        for (Map.Entry<String, Settings> executor : defaultExecutorTypeSettings.entrySet()) {
-            Settings updatedSettings = groupSettings.get(executor.getKey());
-            if (updatedSettings == null) {
-                continue;
-            }
-
-            ExecutorHolder oldExecutorHolder = executors.get(executor.getKey());
-            ExecutorHolder newExecutorHolder = rebuild(executor.getKey(), oldExecutorHolder, updatedSettings, executor.getValue());
-            if (!oldExecutorHolder.equals(newExecutorHolder)) {
-                Map<String, ExecutorHolder> newExecutors = new HashMap<>(executors);
-                newExecutors.put(executor.getKey(), newExecutorHolder);
-                executors = unmodifiableMap(newExecutors);
-                if (!oldExecutorHolder.executor().equals(newExecutorHolder.executor()) && oldExecutorHolder.executor() instanceof EsThreadPoolExecutor) {
-                    retiredExecutors.add(oldExecutorHolder);
-                    ((EsThreadPoolExecutor) oldExecutorHolder.executor()).shutdown(new ExecutorShutdownListener(oldExecutorHolder));
-                }
-            }
-        }
-
-        // Building custom thread pools
-        for (Map.Entry<String, Settings> entry : groupSettings.entrySet()) {
-            if (defaultExecutorTypeSettings.containsKey(entry.getKey())) {
-                continue;
-            }
-
-            ExecutorHolder oldExecutorHolder = executors.get(entry.getKey());
-            ExecutorHolder newExecutorHolder = rebuild(entry.getKey(), oldExecutorHolder, entry.getValue(), Settings.EMPTY);
-            // Can't introduce new thread pools at runtime, because The oldExecutorHolder variable will be null in the
-            // case the settings contains a thread pool not defined in the initial settings in the constructor. The if
-            // statement will then fail and so this prevents the addition of new thread groups at runtime, which is desired.
-            if (!newExecutorHolder.equals(oldExecutorHolder)) {
-                Map<String, ExecutorHolder> newExecutors = new HashMap<>(executors);
-                newExecutors.put(entry.getKey(), newExecutorHolder);
-                executors = unmodifiableMap(newExecutors);
-                if (!oldExecutorHolder.executor().equals(newExecutorHolder.executor()) && oldExecutorHolder.executor() instanceof EsThreadPoolExecutor) {
-                    retiredExecutors.add(oldExecutorHolder);
-                    ((EsThreadPoolExecutor) oldExecutorHolder.executor()).shutdown(new ExecutorShutdownListener(oldExecutorHolder));
-                }
-            }
-        }
-    }
-
-    private void validate(Map<String, Settings> groupSettings) {
-        for (String key : groupSettings.keySet()) {
-            if (!THREAD_POOL_TYPES.containsKey(key)) {
-                continue;
-            }
-            String type = groupSettings.get(key).get("type");
-            ThreadPoolType correctThreadPoolType = THREAD_POOL_TYPES.get(key);
-            // TODO: the type equality check can be removed after #3760/#6732 are addressed
-            if (type != null && !correctThreadPoolType.getType().equals(type)) {
-                throw new IllegalArgumentException("setting " + THREADPOOL_GROUP_SETTING.getKey() + key + ".type to " + type + " is not permitted; must be " + correctThreadPoolType.getType());
-            }
-        }
-    }
-
-    /**
-     * A thread pool size can also be unbounded and is represented by -1, which is not supported by SizeValue (which only supports positive numbers)
-     */
-    private SizeValue getAsSizeOrUnbounded(Settings settings, String setting, SizeValue defaultValue) throws SettingsException {
-        if ("-1".equals(settings.get(setting))) {
-            return null;
-        }
-        return parseSizeValue(settings.get(setting), defaultValue);
-    }
-
-    class ExecutorShutdownListener implements EsThreadPoolExecutor.ShutdownListener {
-
-        private ExecutorHolder holder;
-
-        public ExecutorShutdownListener(ExecutorHolder holder) {
-            this.holder = holder;
-        }
-
-        @Override
-        public void onTerminated() {
-            retiredExecutors.remove(holder);
-        }
-    }
-
     class LoggingRunnable implements Runnable {
 
         private final Runnable runnable;
diff --git a/core/src/main/java/org/elasticsearch/threadpool/ThreadPoolModule.java b/core/src/main/java/org/elasticsearch/threadpool/ThreadPoolModule.java
index 9a507f883ac..843febfef8c 100644
--- a/core/src/main/java/org/elasticsearch/threadpool/ThreadPoolModule.java
+++ b/core/src/main/java/org/elasticsearch/threadpool/ThreadPoolModule.java
@@ -20,20 +20,25 @@
 package org.elasticsearch.threadpool;
 
 import org.elasticsearch.common.inject.AbstractModule;
+import org.elasticsearch.common.settings.SettingsModule;
 
-/**
- *
- */
 public class ThreadPoolModule extends AbstractModule {
 
     private final ThreadPool threadPool;
 
-    public ThreadPoolModule(ThreadPool threadPool) {
+    public ThreadPoolModule(final ThreadPool threadPool) {
         this.threadPool = threadPool;
     }
 
+    public void prepareSettings(SettingsModule settingsModule) {
+        for (final ExecutorBuilder<?> builder : threadPool.builders()) {
+            builder.getRegisteredSettings().forEach(settingsModule::registerSetting);
+        }
+    }
+
     @Override
     protected void configure() {
         bind(ThreadPool.class).toInstance(threadPool);
     }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/action/RejectionActionIT.java b/core/src/test/java/org/elasticsearch/action/RejectionActionIT.java
index fb0283db48f..6f100170250 100644
--- a/core/src/test/java/org/elasticsearch/action/RejectionActionIT.java
+++ b/core/src/test/java/org/elasticsearch/action/RejectionActionIT.java
@@ -45,12 +45,12 @@ public class RejectionActionIT extends ESIntegTestCase {
     protected Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
-                .put("threadpool.search.size", 1)
-                .put("threadpool.search.queue_size", 1)
-                .put("threadpool.index.size", 1)
-                .put("threadpool.index.queue_size", 1)
-                .put("threadpool.get.size", 1)
-                .put("threadpool.get.queue_size", 1)
+                .put("thread_pool.search.size", 1)
+                .put("thread_pool.search.queue_size", 1)
+                .put("thread_pool.index.size", 1)
+                .put("thread_pool.index.queue_size", 1)
+                .put("thread_pool.get.size", 1)
+                .put("thread_pool.get.queue_size", 1)
                 .build();
     }
 
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
index f04c758ef9a..a83a05b8ad8 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
@@ -42,6 +42,7 @@ import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.tasks.MockTaskManager;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.local.LocalTransport;
@@ -72,7 +73,7 @@ public abstract class TaskManagerTestCase extends ESTestCase {
 
     @BeforeClass
     public static void beforeClass() {
-        threadPool = new ThreadPool(TransportTasksActionTests.class.getSimpleName());
+        threadPool = new TestThreadPool(TransportTasksActionTests.class.getSimpleName());
     }
 
     @AfterClass
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java
index e950da05d37..87249bc8b5a 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkProcessorRetryIT.java
@@ -52,10 +52,10 @@ public class BulkProcessorRetryIT extends ESIntegTestCase {
                 .put(super.nodeSettings(nodeOrdinal))
                 // don't mess with this one! It's quite sensitive to a low queue size
                 // (see also ThreadedActionListener which is happily spawning threads even when we already got rejected)
-                //.put("threadpool.listener.queue_size", 1)
-                .put("threadpool.get.queue_size", 1)
+                //.put("thread_pool.listener.queue_size", 1)
+                .put("thread_pool.get.queue_size", 1)
                 // default is 50
-                .put("threadpool.bulk.queue_size", 30)
+                .put("thread_pool.bulk.queue_size", 30)
                 .build();
     }
 
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
index 5131ddebca7..908fb2ddd4d 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
@@ -38,6 +38,7 @@ import org.elasticsearch.common.util.concurrent.AtomicArray;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.CapturingTransport;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.junit.After;
@@ -63,7 +64,7 @@ public class TransportBulkActionTookTests extends ESTestCase {
 
     @BeforeClass
     public static void beforeClass() {
-        threadPool = new ThreadPool("TransportBulkActionTookTests");
+        threadPool = new TestThreadPool("TransportBulkActionTookTests");
     }
 
     @AfterClass
diff --git a/core/src/test/java/org/elasticsearch/action/support/ListenableActionFutureTests.java b/core/src/test/java/org/elasticsearch/action/support/ListenableActionFutureTests.java
index a6c21ca3d66..80492f0be61 100644
--- a/core/src/test/java/org/elasticsearch/action/support/ListenableActionFutureTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/ListenableActionFutureTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.action.support;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.Transports;
 
@@ -31,7 +32,7 @@ import java.util.concurrent.atomic.AtomicReference;
 public class ListenableActionFutureTests extends ESTestCase {
 
     public void testListenerIsCallableFromNetworkThreads() throws Throwable {
-        ThreadPool threadPool = new ThreadPool("testListenerIsCallableFromNetworkThreads");
+        ThreadPool threadPool = new TestThreadPool("testListenerIsCallableFromNetworkThreads");
         try {
             final PlainListenableActionFuture<Object> future = new PlainListenableActionFuture<>(threadPool);
             final CountDownLatch listenerCalled = new CountDownLatch(1);
diff --git a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
index 0fac744625f..07ddfa8e49f 100644
--- a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
@@ -55,6 +55,7 @@ import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.CapturingTransport;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportResponse;
@@ -182,7 +183,7 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase {
 
     @BeforeClass
     public static void startThreadPool() {
-        THREAD_POOL = new ThreadPool(TransportBroadcastByNodeActionTests.class.getSimpleName());
+        THREAD_POOL = new TestThreadPool(TransportBroadcastByNodeActionTests.class.getSimpleName());
     }
 
     @Before
diff --git a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
index b92ba64c2fb..d6edb972ef4 100644
--- a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
@@ -45,6 +45,7 @@ import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.CapturingTransport;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.TransportService;
@@ -76,7 +77,7 @@ public class TransportMasterNodeActionTests extends ESTestCase {
 
     @BeforeClass
     public static void beforeClass() {
-        threadPool = new ThreadPool("TransportMasterNodeActionTests");
+        threadPool = new TestThreadPool("TransportMasterNodeActionTests");
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
index c53adb08ce2..1b0a6f8d622 100644
--- a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
@@ -35,6 +35,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.CapturingTransport;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.junit.After;
@@ -162,7 +163,7 @@ public class TransportNodesActionTests extends ESTestCase {
 
     @BeforeClass
     public static void startThreadPool() {
-        THREAD_POOL = new ThreadPool(TransportBroadcastByNodeActionTests.class.getSimpleName());
+        THREAD_POOL = new TestThreadPool(TransportBroadcastByNodeActionTests.class.getSimpleName());
     }
 
     @AfterClass
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
index ebaa5b5c01e..ea4e55c02f4 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
@@ -44,6 +44,7 @@ import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.local.LocalTransport;
@@ -80,7 +81,7 @@ public class BroadcastReplicationTests extends ESTestCase {
 
     @BeforeClass
     public static void beforeClass() {
-        threadPool = new ThreadPool("BroadcastReplicationTests");
+        threadPool = new TestThreadPool("BroadcastReplicationTests");
         circuitBreakerService = new NoneCircuitBreakerService();
     }
 
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
index afa72ec7526..8ee2499d040 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
@@ -57,6 +57,7 @@ import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESAllocationTestCase;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.CapturingTransport;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportException;
@@ -109,7 +110,7 @@ public class TransportReplicationActionTests extends ESTestCase {
 
     @BeforeClass
     public static void beforeClass() {
-        threadPool = new ThreadPool("ShardReplicationTests");
+        threadPool = new TestThreadPool("ShardReplicationTests");
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
index 10132af1b1d..6d7ac6128c9 100644
--- a/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
@@ -43,6 +43,7 @@ import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.CapturingTransport;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.TransportException;
@@ -133,7 +134,7 @@ public class TransportInstanceSingleOperationActionTests extends ESTestCase {
 
     @BeforeClass
     public static void startThreadPool() {
-        THREAD_POOL = new ThreadPool(TransportInstanceSingleOperationActionTests.class.getSimpleName());
+        THREAD_POOL = new TestThreadPool(TransportInstanceSingleOperationActionTests.class.getSimpleName());
     }
 
     @Before
diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
index b566d764231..0dddb301444 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
@@ -28,6 +28,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.BaseTransportResponseHandler;
 import org.elasticsearch.transport.TransportException;
@@ -63,7 +64,7 @@ public class TransportClientNodesServiceTests extends ESTestCase {
 
         TestIteration() {
             ClusterName clusterName = new ClusterName("test");
-            threadPool = new ThreadPool("transport-client-nodes-service-tests");
+            threadPool = new TestThreadPool("transport-client-nodes-service-tests");
             transport = new FailAndRetryMockTransport<TestResponse>(random(), clusterName) {
                 @Override
                 public List<String> getLocalAddresses() {
diff --git a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
index 261da3e2bb8..bfc3929dd14 100644
--- a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
@@ -38,6 +38,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.CapturingTransport;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.NodeDisconnectedException;
 import org.elasticsearch.transport.NodeNotConnectedException;
@@ -97,7 +98,7 @@ public class ShardStateActionTests extends ESTestCase {
 
     @BeforeClass
     public static void startThreadPool() {
-        THREAD_POOL = new ThreadPool("ShardStateActionTest");
+        THREAD_POOL = new TestThreadPool("ShardStateActionTest");
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
index dde35e33993..e99c862bfec 100644
--- a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
@@ -41,6 +41,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.gateway.NoopGatewayAllocator;
 import org.elasticsearch.test.transport.CapturingTransport;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.junit.After;
@@ -72,7 +73,7 @@ public class ClusterStateHealthTests extends ESTestCase {
 
     @BeforeClass
     public static void beforeClass() {
-        threadPool = new ThreadPool("ClusterStateHealthTests");
+        threadPool = new TestThreadPool("ClusterStateHealthTests");
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
index 630e5b034af..2e1cdea56a4 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
@@ -37,6 +37,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.gateway.GatewayAllocator;
 import org.elasticsearch.test.ESAllocationTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
 import org.junit.Before;
@@ -73,7 +74,7 @@ public class DelayedAllocationServiceTests extends ESAllocationTestCase {
 
     @Before
     public void createDelayedAllocationService() {
-        threadPool = new ThreadPool(getTestName());
+        threadPool = new TestThreadPool(getTestName());
         clusterService = mock(ClusterService.class);
         allocationService = createAllocationService(Settings.EMPTY, new DelayedShardsMockGatewayAllocator());
         delayedAllocationService = new TestDelayAllocationService(Settings.EMPTY, threadPool, clusterService, allocationService);
diff --git a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
index 81fc9d3752d..bf85f67e48e 100644
--- a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
@@ -42,6 +42,7 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.MockLogAppender;
 import org.elasticsearch.test.junit.annotations.TestLogging;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -76,7 +77,7 @@ public class ClusterServiceTests extends ESTestCase {
 
     @BeforeClass
     public static void createThreadPool() {
-        threadPool = new ThreadPool(ClusterServiceTests.class.getName());
+        threadPool = new TestThreadPool(ClusterServiceTests.class.getName());
     }
 
     @AfterClass
diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/PrioritizedExecutorsTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/PrioritizedExecutorsTests.java
index 50b7d5f775c..df51e6e2e0d 100644
--- a/core/src/test/java/org/elasticsearch/common/util/concurrent/PrioritizedExecutorsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/PrioritizedExecutorsTests.java
@@ -22,6 +22,7 @@ import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.ArrayList;
@@ -244,7 +245,7 @@ public class PrioritizedExecutorsTests extends ESTestCase {
     }
 
     public void testTimeoutCleanup() throws Exception {
-        ThreadPool threadPool = new ThreadPool("test");
+        ThreadPool threadPool = new TestThreadPool("test");
         final ScheduledThreadPoolExecutor timer = (ScheduledThreadPoolExecutor) threadPool.scheduler();
         final AtomicBoolean timeoutCalled = new AtomicBoolean();
         PrioritizedEsThreadPoolExecutor executor = EsExecutors.newSinglePrioritizing(getTestName(), EsExecutors.daemonThreadFactory(getTestName()), holder);
diff --git a/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java b/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
index d3da4513d95..ea81e958342 100644
--- a/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
@@ -39,6 +39,7 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportConnectionListener;
 import org.elasticsearch.transport.TransportRequestOptions;
@@ -85,7 +86,7 @@ public class ZenFaultDetectionTests extends ESTestCase {
             .put(HierarchyCircuitBreakerService.IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), new ByteSizeValue(0))
             .build();
         ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-        threadPool = new ThreadPool(getClass().getName());
+        threadPool = new TestThreadPool(getClass().getName());
         clusterServiceA = createClusterService(threadPool);
         clusterServiceB = createClusterService(threadPool);
         circuitBreakerService = new HierarchyCircuitBreakerService(settings, clusterSettings);
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
index 651eab421ad..b7db9f9e609 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
@@ -43,6 +43,7 @@ import org.elasticsearch.discovery.zen.elect.ElectMasterService;
 import org.elasticsearch.discovery.zen.membership.MembershipAction;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.junit.annotations.TestLogging;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -81,7 +82,7 @@ public class NodeJoinControllerTests extends ESTestCase {
 
     @BeforeClass
     public static void beforeClass() {
-        threadPool = new ThreadPool("ShardReplicationTests");
+        threadPool = new TestThreadPool("ShardReplicationTests");
     }
 
     @AfterClass
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java b/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java
index 7847d7027d0..2845e6a5bf0 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java
@@ -37,6 +37,7 @@ import org.elasticsearch.discovery.zen.ping.ZenPing;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportConnectionListener;
 import org.elasticsearch.transport.TransportService;
@@ -60,7 +61,7 @@ public class UnicastZenPingIT extends ESTestCase {
         int endPort = startPort + 10;
         settings = Settings.builder().put(settings).put(TransportSettings.PORT.getKey(), startPort + "-" + endPort).build();
 
-        ThreadPool threadPool = new ThreadPool(getClass().getName());
+        ThreadPool threadPool = new TestThreadPool(getClass().getName());
         ClusterName test = new ClusterName("test");
         ClusterName mismatch = new ClusterName("mismatch");
         NetworkService networkService = new NetworkService(settings);
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java
index e6b160eabf8..7cc319c8579 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java
@@ -47,6 +47,7 @@ import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.BytesTransportRequest;
 import org.elasticsearch.transport.TransportChannel;
@@ -217,7 +218,7 @@ public class PublishClusterStateActionTests extends ESTestCase {
     @Before
     public void setUp() throws Exception {
         super.setUp();
-        threadPool = new ThreadPool(getClass().getName());
+        threadPool = new TestThreadPool(getClass().getName());
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java b/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
index b710aa50ee0..9ce2aa44ab6 100644
--- a/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
 import org.junit.Before;
@@ -60,7 +61,7 @@ public class AsyncShardFetchTests extends ESTestCase {
     @Before
     public void setUp() throws Exception {
         super.setUp();
-        this.threadPool = new ThreadPool(getTestName());
+        this.threadPool = new TestThreadPool(getTestName());
         this.test = new TestFetch(threadPool);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java
index e7e3c41820b..a56e9993434 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java
@@ -30,6 +30,7 @@ import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.jboss.netty.buffer.ChannelBuffer;
 import org.jboss.netty.buffer.ChannelBuffers;
@@ -70,7 +71,7 @@ public class NettyHttpChannelTests extends ESTestCase {
     @Before
     public void setup() throws Exception {
         networkService = new NetworkService(Settings.EMPTY);
-        threadPool = new ThreadPool("test");
+        threadPool = new TestThreadPool("test");
         bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
     }
 
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java
index db48a56a122..6fc9a4e674a 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java
@@ -29,6 +29,7 @@ import org.elasticsearch.http.netty.pipelining.OrderedDownstreamChannelEvent;
 import org.elasticsearch.http.netty.pipelining.OrderedUpstreamMessageEvent;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.jboss.netty.buffer.ChannelBuffer;
 import org.jboss.netty.buffer.ChannelBuffers;
@@ -73,7 +74,7 @@ public class NettyHttpServerPipeliningTests extends ESTestCase {
     @Before
     public void setup() throws Exception {
         networkService = new NetworkService(Settings.EMPTY);
-        threadPool = new ThreadPool("test");
+        threadPool = new TestThreadPool("test");
         bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
     }
 
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerTransportTests.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerTransportTests.java
index da548c93985..3cf9c1aa029 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerTransportTests.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerTransportTests.java
@@ -26,6 +26,7 @@ import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.http.netty.cors.CorsConfig;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.jboss.netty.handler.codec.http.HttpMethod;
 import org.junit.After;
@@ -54,7 +55,7 @@ public class NettyHttpServerTransportTests extends ESTestCase {
     @Before
     public void setup() throws Exception {
         networkService = new NetworkService(Settings.EMPTY);
-        threadPool = new ThreadPool("test");
+        threadPool = new TestThreadPool("test");
         bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
index 38b7341cd24..f552ab9cf17 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
@@ -76,6 +76,7 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.TestSearchContext;
 import org.elasticsearch.test.engine.MockEngineFactory;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
 
@@ -110,7 +111,7 @@ public class IndexModuleTests extends ESTestCase {
 
     static NodeServicesProvider newNodeServiceProvider(Settings settings, Environment environment, Client client, ScriptEngineService... scriptEngineServices) throws IOException {
         // TODO this can be used in other place too - lets first refactor the IndicesQueriesRegistry
-        ThreadPool threadPool = new ThreadPool("test");
+        ThreadPool threadPool = new TestThreadPool("test");
         CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService();
         BigArrays bigArrays = new BigArrays(settings, circuitBreakerService);
         Set<ScriptEngineService> scriptEngines = Collections.emptySet();
diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index b7da3860003..05428ee3cd5 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -98,6 +98,7 @@ import org.elasticsearch.indices.mapper.MapperRegistry;
 import org.elasticsearch.test.DummyShardLock;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.hamcrest.MatcherAssert;
 import org.junit.After;
@@ -173,7 +174,7 @@ public class InternalEngineTests extends ESTestCase {
                 .put(IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD,
                         between(10, 10 * IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD.get(Settings.EMPTY)))
                 .build()); // TODO randomize more settings
-        threadPool = new ThreadPool(getClass().getName());
+        threadPool = new TestThreadPool(getClass().getName());
         store = createStore();
         storeReplica = createStore();
         Lucene.cleanLuceneIndex(store.directory());
diff --git a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
index bed597b7cc5..ef443d1e102 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
@@ -67,6 +67,7 @@ import org.elasticsearch.index.translog.TranslogConfig;
 import org.elasticsearch.test.DummyShardLock;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.hamcrest.MatcherAssert;
 import org.junit.After;
@@ -126,7 +127,7 @@ public class ShadowEngineTests extends ESTestCase {
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                 .build()); // TODO randomize more settings
 
-        threadPool = new ThreadPool(getClass().getName());
+        threadPool = new TestThreadPool(getClass().getName());
         dirPath = createTempDir();
         store = createStore(dirPath);
         storeReplica = createStore(dirPath);
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
index 2887d9b2561..bd6decfb60e 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
@@ -55,6 +55,7 @@ import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.InternalSettingsPlugin;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Arrays;
@@ -177,7 +178,7 @@ public class IndexFieldDataServiceTests extends ESSingleNodeTestCase {
     }
 
     private void doTestRequireDocValues(MappedFieldType ft) {
-        ThreadPool threadPool = new ThreadPool("random_threadpool_name");
+        ThreadPool threadPool = new TestThreadPool("random_threadpool_name");
         try {
             IndicesFieldDataCache cache = new IndicesFieldDataCache(Settings.EMPTY, null);
             IndexFieldDataService ifds = new IndexFieldDataService(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY), cache, null, null);
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
index 26399218829..8960f9dfe6e 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
@@ -35,6 +35,7 @@ import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.local.LocalTransport;
@@ -64,7 +65,7 @@ public class DynamicMappingDisabledTests extends ESSingleNodeTestCase {
 
     @BeforeClass
     public static void createThreadPool() {
-        THREAD_POOL = new ThreadPool("DynamicMappingDisabledTests");
+        THREAD_POOL = new TestThreadPool("DynamicMappingDisabledTests");
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
index f877ae6629b..f197073033d 100644
--- a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
@@ -35,6 +35,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.junit.After;
@@ -74,7 +75,7 @@ public class IndicesStoreTests extends ESTestCase {
 
     @BeforeClass
     public static void beforeClass() {
-        threadPool = new ThreadPool("ShardReplicationTests");
+        threadPool = new TestThreadPool("ShardReplicationTests");
     }
 
     @AfterClass
diff --git a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java
index 97a2b065627..a1f4d381911 100644
--- a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java
@@ -22,6 +22,7 @@ package org.elasticsearch.monitor.jvm;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.AbstractMap;
@@ -128,7 +129,7 @@ public class JvmGcMonitorServiceSettingsTests extends ESTestCase {
         assert constructionShouldFail == (asserts == null);
         ThreadPool threadPool = null;
         try {
-            threadPool = new ThreadPool(JvmGcMonitorServiceSettingsTests.class.getCanonicalName()) {
+            threadPool = new TestThreadPool(JvmGcMonitorServiceSettingsTests.class.getCanonicalName()) {
                 @Override
                 public ScheduledFuture<?> scheduleWithFixedDelay(Runnable command, TimeValue interval) {
                     return scheduler.apply(command, interval);
diff --git a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
index 5e1dc740f9e..e0eed8387a6 100644
--- a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
@@ -55,9 +55,10 @@ public class NativeScriptTests extends ESTestCase {
         ScriptModule scriptModule = new ScriptModule();
         scriptModule.prepareSettings(settingsModule);
         scriptModule.registerScript("my", MyNativeScriptFactory.class);
+        final ThreadPool threadPool = new ThreadPool(settings);
         Injector injector = new ModulesBuilder().add(
                 new EnvironmentModule(new Environment(settings)),
-                new ThreadPoolModule(new ThreadPool(settings)),
+                new ThreadPoolModule(threadPool),
                 new SettingsModule(settings),
                 scriptModule).createInjector();
 
diff --git a/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java b/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java
index 68b4d980928..2bb39ad10ea 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java
@@ -36,8 +36,8 @@ public class SearchWithRejectionsIT extends ESIntegTestCase {
     @Override
     public Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder().put(super.nodeSettings(nodeOrdinal))
-                .put("threadpool.search.size", 1)
-                .put("threadpool.search.queue_size", 1)
+                .put("thread_pool.search.size", 1)
+                .put("thread_pool.search.queue_size", 1)
                 .build();
     }
 
diff --git a/core/src/test/java/org/elasticsearch/threadpool/FixedThreadPoolTests.java b/core/src/test/java/org/elasticsearch/threadpool/FixedThreadPoolTests.java
index 85daebb86d5..48ea8b6c8c9 100644
--- a/core/src/test/java/org/elasticsearch/threadpool/FixedThreadPoolTests.java
+++ b/core/src/test/java/org/elasticsearch/threadpool/FixedThreadPoolTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.threadpool;
 
-import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
@@ -42,8 +41,8 @@ public class FixedThreadPoolTests extends ESThreadPoolTestCase {
         final Settings nodeSettings =
             Settings.builder()
                 .put("node.name", "testRejectedExecutionCounter")
-                .put("threadpool." + threadPoolName + ".size", size)
-                .put("threadpool." + threadPoolName + ".queue_size", queueSize)
+                .put("thread_pool." + threadPoolName + ".size", size)
+                .put("thread_pool." + threadPoolName + ".queue_size", queueSize)
                 .build();
         try {
             threadPool = new ThreadPool(nodeSettings);
@@ -86,18 +85,6 @@ public class FixedThreadPoolTests extends ESThreadPoolTestCase {
 
             assertThat(counter, equalTo(rejections));
             assertThat(stats(threadPool, threadPoolName).getRejected(), equalTo(rejections));
-
-            // the rejected execution count resets to zero when the
-            // queue is resized
-            final ClusterSettings clusterSettings =
-                new ClusterSettings(nodeSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-            threadPool.setClusterSettings(clusterSettings);
-            clusterSettings.applySettings(
-                Settings.builder()
-                    .put("threadpool." + threadPoolName + ".queue_size", queueSize + 1)
-                    .build());
-            assertThat(stats(threadPool, threadPoolName).getRejected(), equalTo(0L));
-
         } finally {
             terminateThreadPoolIfNeeded(threadPool);
         }
diff --git a/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java b/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
index 2212f162eb6..d065abb884c 100644
--- a/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
+++ b/core/src/test/java/org/elasticsearch/threadpool/ScalingThreadPoolTests.java
@@ -40,35 +40,35 @@ public class ScalingThreadPoolTests extends ESThreadPoolTestCase {
         final String threadPoolName = randomThreadPool(ThreadPool.ThreadPoolType.SCALING);
         final Settings.Builder builder = Settings.builder();
 
-        final int min;
+        final int core;
         if (randomBoolean()) {
-            min = randomIntBetween(0, 8);
-            builder.put("threadpool." + threadPoolName + ".min", min);
+            core = randomIntBetween(0, 8);
+            builder.put("thread_pool." + threadPoolName + ".core", core);
         } else {
-            min = "generic".equals(threadPoolName) ? 4 : 1; // the defaults
+            core = "generic".equals(threadPoolName) ? 4 : 1; // the defaults
         }
 
-        final int sizeBasedOnNumberOfProcessors;
+        final int maxBasedOnNumberOfProcessors;
         if (randomBoolean()) {
             final int processors = randomIntBetween(1, 64);
-            sizeBasedOnNumberOfProcessors = expectedSize(threadPoolName, processors);
+            maxBasedOnNumberOfProcessors = expectedSize(threadPoolName, processors);
             builder.put("processors", processors);
         } else {
-            sizeBasedOnNumberOfProcessors = expectedSize(threadPoolName, Math.min(32, Runtime.getRuntime().availableProcessors()));
+            maxBasedOnNumberOfProcessors = expectedSize(threadPoolName, Math.min(32, Runtime.getRuntime().availableProcessors()));
         }
 
-        final int expectedSize;
-        if (sizeBasedOnNumberOfProcessors < min || randomBoolean()) {
-            expectedSize = randomIntBetween(Math.max(1, min), 16);
-            builder.put("threadpool." + threadPoolName + ".size", expectedSize);
+        final int expectedMax;
+        if (maxBasedOnNumberOfProcessors < core || randomBoolean()) {
+            expectedMax = randomIntBetween(Math.max(1, core), 16);
+            builder.put("thread_pool." + threadPoolName + ".max", expectedMax);
         }  else {
-            expectedSize = sizeBasedOnNumberOfProcessors;
+            expectedMax = maxBasedOnNumberOfProcessors;
         }
 
         final long keepAlive;
         if (randomBoolean()) {
             keepAlive = randomIntBetween(1, 300);
-            builder.put("threadpool." + threadPoolName + ".keep_alive", keepAlive + "s");
+            builder.put("thread_pool." + threadPoolName + ".keep_alive", keepAlive + "s");
         } else {
             keepAlive = "generic".equals(threadPoolName) ? 30 : 300; // the defaults
         }
@@ -88,10 +88,10 @@ public class ScalingThreadPoolTests extends ESThreadPoolTestCase {
             assertNull(info.getQueueSize());
             assertThat(esThreadPoolExecutor.getQueue().remainingCapacity(), equalTo(Integer.MAX_VALUE));
 
-            assertThat(info.getMin(), equalTo(min));
-            assertThat(esThreadPoolExecutor.getCorePoolSize(), equalTo(min));
-            assertThat(info.getMax(), equalTo(expectedSize));
-            assertThat(esThreadPoolExecutor.getMaximumPoolSize(), equalTo(expectedSize));
+            assertThat(info.getMin(), equalTo(core));
+            assertThat(esThreadPoolExecutor.getCorePoolSize(), equalTo(core));
+            assertThat(info.getMax(), equalTo(expectedMax));
+            assertThat(esThreadPoolExecutor.getMaximumPoolSize(), equalTo(expectedMax));
         });
     }
 
@@ -113,23 +113,10 @@ public class ScalingThreadPoolTests extends ESThreadPoolTestCase {
         return sizes.get(threadPoolName).size(numberOfProcessors);
     }
 
-    public void testValidDynamicKeepAlive() throws InterruptedException {
-        final String threadPoolName = randomThreadPool(ThreadPool.ThreadPoolType.SCALING);
-        runScalingThreadPoolTest(Settings.EMPTY, (clusterSettings, threadPool) -> {
-            final Executor beforeExecutor = threadPool.executor(threadPoolName);
-            final long seconds = randomIntBetween(1, 300);
-            clusterSettings.applySettings(settings("threadpool." + threadPoolName + ".keep_alive", seconds + "s"));
-            final Executor afterExecutor = threadPool.executor(threadPoolName);
-            assertSame(beforeExecutor, afterExecutor);
-            final ThreadPool.Info info = info(threadPool, threadPoolName);
-            assertThat(info.getKeepAlive().seconds(), equalTo(seconds));
-        });
-    }
-
     public void testScalingThreadPoolIsBounded() throws InterruptedException {
         final String threadPoolName = randomThreadPool(ThreadPool.ThreadPoolType.SCALING);
         final int size = randomIntBetween(32, 512);
-        final Settings settings = Settings.builder().put("threadpool." + threadPoolName + ".size", size).build();
+        final Settings settings = Settings.builder().put("thread_pool." + threadPoolName + ".max", size).build();
         runScalingThreadPoolTest(settings, (clusterSettings, threadPool) -> {
             final CountDownLatch latch = new CountDownLatch(1);
             final int numberOfTasks = 2 * size;
@@ -161,8 +148,8 @@ public class ScalingThreadPoolTests extends ESThreadPoolTestCase {
         final int min = "generic".equals(threadPoolName) ? 4 : 1;
         final Settings settings =
                 Settings.builder()
-                        .put("threadpool." + threadPoolName + ".size", 128)
-                        .put("threadpool." + threadPoolName + ".keep_alive", "1ms")
+                        .put("thread_pool." + threadPoolName + ".max", 128)
+                        .put("thread_pool." + threadPoolName + ".keep_alive", "1ms")
                         .build();
         runScalingThreadPoolTest(settings, ((clusterSettings, threadPool) -> {
             final CountDownLatch latch = new CountDownLatch(1);
@@ -197,40 +184,6 @@ public class ScalingThreadPoolTests extends ESThreadPoolTestCase {
         }));
     }
 
-    public void testDynamicThreadPoolSize() throws InterruptedException {
-        final String threadPoolName = randomThreadPool(ThreadPool.ThreadPoolType.SCALING);
-        runScalingThreadPoolTest(Settings.EMPTY, (clusterSettings, threadPool) -> {
-            final Executor beforeExecutor = threadPool.executor(threadPoolName);
-            int expectedMin = "generic".equals(threadPoolName) ? 4 : 1;
-            final int size = randomIntBetween(expectedMin, Integer.MAX_VALUE);
-            clusterSettings.applySettings(settings("threadpool." + threadPoolName + ".size", size));
-            final Executor afterExecutor = threadPool.executor(threadPoolName);
-            assertSame(beforeExecutor, afterExecutor);
-            final ThreadPool.Info info = info(threadPool, threadPoolName);
-            assertThat(info.getMin(), equalTo(expectedMin));
-            assertThat(info.getMax(), equalTo(size));
-
-            assertThat(afterExecutor, instanceOf(EsThreadPoolExecutor.class));
-            final EsThreadPoolExecutor executor = (EsThreadPoolExecutor)afterExecutor;
-            assertThat(executor.getCorePoolSize(), equalTo(expectedMin));
-            assertThat(executor.getMaximumPoolSize(), equalTo(size));
-        });
-    }
-
-    public void testResizingScalingThreadPoolQueue() throws InterruptedException {
-        final String threadPoolName = randomThreadPool(ThreadPool.ThreadPoolType.SCALING);
-        runScalingThreadPoolTest(Settings.EMPTY, (clusterSettings, threadPool) -> {
-            final int size = randomIntBetween(1, Integer.MAX_VALUE);
-            final IllegalArgumentException e = expectThrows(
-                    IllegalArgumentException.class,
-                    () -> clusterSettings.applySettings(settings("threadpool." + threadPoolName + ".queue_size", size)));
-            assertThat(e, hasToString(
-                    "java.lang.IllegalArgumentException: thread pool [" + threadPoolName +
-                        "] of type scaling can not have its queue re-sized but was [" +
-                            size + "]"));
-        });
-    }
-
     public void runScalingThreadPoolTest(
             final Settings settings,
             final BiConsumer<ClusterSettings, ThreadPool> consumer) throws InterruptedException {
@@ -240,7 +193,6 @@ public class ScalingThreadPoolTests extends ESThreadPoolTestCase {
             final Settings nodeSettings = Settings.builder().put(settings).put("node.name", test).build();
             threadPool = new ThreadPool(nodeSettings);
             final ClusterSettings clusterSettings = new ClusterSettings(nodeSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-            threadPool.setClusterSettings(clusterSettings);
             consumer.accept(clusterSettings, threadPool);
         } finally {
             terminateThreadPoolIfNeeded(threadPool);
diff --git a/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java b/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java
index baf017bc7b9..28267e9beb7 100644
--- a/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java
+++ b/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java
@@ -125,63 +125,6 @@ public class SimpleThreadPoolIT extends ESIntegTestCase {
         }
     }
 
-    public void testUpdatingThreadPoolSettings() throws Exception {
-        internalCluster().startNodesAsync(2).get();
-        ThreadPool threadPool = internalCluster().getDataNodeInstance(ThreadPool.class);
-        // Check that settings are changed
-        assertThat(((ThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getQueue().remainingCapacity(), equalTo(1000));
-        client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put("threadpool.search.queue_size", 2000).build()).execute().actionGet();
-        assertThat(((ThreadPoolExecutor) threadPool.executor(Names.SEARCH)).getQueue().remainingCapacity(), equalTo(2000));
-
-        // Make sure that threads continue executing when executor is replaced
-        final CyclicBarrier barrier = new CyclicBarrier(2);
-        Executor oldExecutor = threadPool.executor(Names.SEARCH);
-        threadPool.executor(Names.SEARCH).execute(() -> {
-                    try {
-                        barrier.await();
-                    } catch (InterruptedException ex) {
-                        Thread.currentThread().interrupt();
-                    } catch (BrokenBarrierException ex) {
-                        //
-                    }
-                });
-        client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put("threadpool.search.queue_size", 1000).build()).execute().actionGet();
-        assertThat(threadPool.executor(Names.SEARCH), not(sameInstance(oldExecutor)));
-        assertThat(((ThreadPoolExecutor) oldExecutor).isShutdown(), equalTo(true));
-        assertThat(((ThreadPoolExecutor) oldExecutor).isTerminating(), equalTo(true));
-        assertThat(((ThreadPoolExecutor) oldExecutor).isTerminated(), equalTo(false));
-        barrier.await(10, TimeUnit.SECONDS);
-
-        // Make sure that new thread executor is functional
-        threadPool.executor(Names.SEARCH).execute(() -> {
-                    try {
-                        barrier.await();
-                    } catch (InterruptedException ex) {
-                        Thread.currentThread().interrupt();
-                    } catch (BrokenBarrierException ex) {
-                        //
-                    }
-                }
-        );
-        client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put("threadpool.search.queue_size", 500)).execute().actionGet();
-        barrier.await(10, TimeUnit.SECONDS);
-
-        // Check that node info is correct
-        NodesInfoResponse nodesInfoResponse = client().admin().cluster().prepareNodesInfo().all().execute().actionGet();
-        assertEquals(2, nodesInfoResponse.getNodes().size());
-        for (NodeInfo nodeInfo : nodesInfoResponse.getNodes()) {
-            boolean found = false;
-            for (ThreadPool.Info info : nodeInfo.getThreadPool()) {
-                if (info.getName().equals(Names.SEARCH)) {
-                    assertEquals(info.getThreadPoolType(), ThreadPool.ThreadPoolType.FIXED);
-                    found = true;
-                    break;
-                }
-            }
-            assertThat(found, equalTo(true));
-        }
-    }
-
     public void testThreadPoolLeakingThreadsWithTribeNode() {
         Settings settings = Settings.builder()
                 .put("node.name", "thread_pool_leaking_threads_tribe_node")
diff --git a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java
index 19dd25a7e25..486b0635c64 100644
--- a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java
+++ b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java
@@ -96,7 +96,7 @@ public class ThreadPoolSerializationTests extends ESTestCase {
     }
 
     public void testThatNegativeSettingAllowsToStart() throws InterruptedException {
-        Settings settings = Settings.builder().put("node.name", "index").put("threadpool.index.queue_size", "-1").build();
+        Settings settings = Settings.builder().put("node.name", "index").put("thread_pool.index.queue_size", "-1").build();
         ThreadPool threadPool = new ThreadPool(settings);
         assertThat(threadPool.info("index").getQueueSize(), is(nullValue()));
         terminate(threadPool);
diff --git a/core/src/test/java/org/elasticsearch/threadpool/UpdateThreadPoolSettingsTests.java b/core/src/test/java/org/elasticsearch/threadpool/UpdateThreadPoolSettingsTests.java
index 43e8e7e7af5..87accf057ad 100644
--- a/core/src/test/java/org/elasticsearch/threadpool/UpdateThreadPoolSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/threadpool/UpdateThreadPoolSettingsTests.java
@@ -19,32 +19,24 @@
 
 package org.elasticsearch.threadpool;
 
-import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor;
 import org.elasticsearch.threadpool.ThreadPool.Names;
 
 import java.lang.reflect.Field;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Executor;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
+import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.nullValue;
-import static org.hamcrest.Matchers.sameInstance;
 
-/**
- */
 public class UpdateThreadPoolSettingsTests extends ESThreadPoolTestCase {
 
     public void testCorrectThreadPoolTypePermittedInSettings() throws InterruptedException {
@@ -53,12 +45,12 @@ public class UpdateThreadPoolSettingsTests extends ESThreadPoolTestCase {
         ThreadPool threadPool = null;
         try {
             threadPool = new ThreadPool(Settings.builder()
-                    .put("node.name", "testCorrectThreadPoolTypePermittedInSettings")
-                    .put("threadpool." + threadPoolName + ".type", correctThreadPoolType.getType())
-                    .build());
+                .put("node.name", "testCorrectThreadPoolTypePermittedInSettings")
+                .put("thread_pool." + threadPoolName + ".type", correctThreadPoolType.getType())
+                .build());
             ThreadPool.Info info = info(threadPool, threadPoolName);
             if (ThreadPool.Names.SAME.equals(threadPoolName)) {
-                assertNull(info); // we don't report on the "same" threadpool
+                assertNull(info); // we don't report on the "same" thread pool
             } else {
                 // otherwise check we have the expected type
                 assertEquals(info.getThreadPoolType(), correctThreadPoolType);
@@ -68,97 +60,31 @@ public class UpdateThreadPoolSettingsTests extends ESThreadPoolTestCase {
         }
     }
 
-    public void testThreadPoolCanNotOverrideThreadPoolType() throws InterruptedException {
-        String threadPoolName = randomThreadPoolName();
-        ThreadPool.ThreadPoolType incorrectThreadPoolType = randomIncorrectThreadPoolType(threadPoolName);
-        ThreadPool.ThreadPoolType correctThreadPoolType = ThreadPool.THREAD_POOL_TYPES.get(threadPoolName);
-        ThreadPool threadPool = null;
-        try {
-            threadPool = new ThreadPool(
-                    Settings.builder()
-                            .put("node.name", "testThreadPoolCanNotOverrideThreadPoolType")
-                            .put("threadpool." + threadPoolName + ".type", incorrectThreadPoolType.getType())
-                            .build());
-            terminate(threadPool);
-            fail("expected IllegalArgumentException");
-        } catch (IllegalArgumentException e) {
-            assertThat(
-                    e.getMessage(),
-                    is("setting threadpool." + threadPoolName + ".type to " + incorrectThreadPoolType.getType() + " is not permitted; must be " + correctThreadPoolType.getType()));
-        } finally {
-            terminateThreadPoolIfNeeded(threadPool);
-        }
-    }
-
     public void testIndexingThreadPoolsMaxSize() throws InterruptedException {
-        String threadPoolName = randomThreadPoolName();
-        for (String name : new String[] {ThreadPool.Names.BULK, ThreadPool.Names.INDEX}) {
-            ThreadPool threadPool = null;
-            try {
+        final String name = randomFrom(Names.BULK, Names.INDEX);
+        final int maxSize = 1 + EsExecutors.boundedNumberOfProcessors(Settings.EMPTY);
+        final int tooBig = randomIntBetween(1 + maxSize, Integer.MAX_VALUE);
 
-                int maxSize = EsExecutors.boundedNumberOfProcessors(Settings.EMPTY);
+        // try to create a too big thread pool
+        final IllegalArgumentException initial =
+            expectThrows(
+                IllegalArgumentException.class,
+                () -> {
+                    ThreadPool tp = null;
+                    try {
+                        tp = new ThreadPool(Settings.builder()
+                            .put("node.name", "testIndexingThreadPoolsMaxSize")
+                            .put("thread_pool." + name + ".size", tooBig)
+                            .build());
+                    } finally {
+                        terminateThreadPoolIfNeeded(tp);
+                    }
+                });
 
-                // try to create a too-big (maxSize+1) thread pool
-                threadPool = new ThreadPool(Settings.builder()
-                                               .put("node.name", "testIndexingThreadPoolsMaxSize")
-                                               .put("threadpool." + name + ".size", maxSize+1)
-                                               .build());
-
-                // confirm it clipped us at the maxSize:
-                assertEquals(maxSize, ((ThreadPoolExecutor) threadPool.executor(name)).getMaximumPoolSize());
-
-                ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-                threadPool.setClusterSettings(clusterSettings);
-
-                // update it to a tiny size:
-                clusterSettings.applySettings(
-                        Settings.builder()
-                        .put("threadpool." + name + ".size", 1)
-                        .build()
-                );
-
-                // confirm it worked:
-                assertEquals(1, ((ThreadPoolExecutor) threadPool.executor(name)).getMaximumPoolSize());
-
-                // try to update to too-big size:
-                clusterSettings.applySettings(
-                        Settings.builder()
-                        .put("threadpool." + name + ".size", maxSize+1)
-                        .build()
-                );
-
-                // confirm it clipped us at the maxSize:
-                assertEquals(maxSize, ((ThreadPoolExecutor) threadPool.executor(name)).getMaximumPoolSize());
-            } finally {
-                terminateThreadPoolIfNeeded(threadPool);
-            }
-        }
-    }
-
-    public void testUpdateSettingsCanNotChangeThreadPoolType() throws InterruptedException {
-        String threadPoolName = randomThreadPoolName();
-        ThreadPool.ThreadPoolType invalidThreadPoolType = randomIncorrectThreadPoolType(threadPoolName);
-        ThreadPool.ThreadPoolType validThreadPoolType = ThreadPool.THREAD_POOL_TYPES.get(threadPoolName);
-        ThreadPool threadPool = null;
-        try {
-            threadPool = new ThreadPool(Settings.builder().put("node.name", "testUpdateSettingsCanNotChangeThreadPoolType").build());
-            ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-            threadPool.setClusterSettings(clusterSettings);
-
-            clusterSettings.applySettings(
-                    Settings.builder()
-                            .put("threadpool." + threadPoolName + ".type", invalidThreadPoolType.getType())
-                            .build()
-            );
-            fail("expected IllegalArgumentException");
-        } catch (IllegalArgumentException e) {
-            assertEquals("illegal value can't update [threadpool.] from [{}] to [{" + threadPoolName + ".type=" + invalidThreadPoolType.getType() + "}]", e.getMessage());
-            assertThat(
-                    e.getCause().getMessage(),
-                    is("setting threadpool." + threadPoolName + ".type to " + invalidThreadPoolType.getType() + " is not permitted; must be " + validThreadPoolType.getType()));
-        } finally {
-            terminateThreadPoolIfNeeded(threadPool);
-        }
+        assertThat(
+            initial,
+            hasToString(containsString(
+                "Failed to parse value [" + tooBig + "] for setting [thread_pool." + name + ".size] must be ")));
     }
 
     private static int getExpectedThreadPoolSize(Settings settings, String name, int size) {
@@ -174,17 +100,14 @@ public class UpdateThreadPoolSettingsTests extends ESThreadPoolTestCase {
         ThreadPool threadPool = null;
 
         try {
+            int expectedSize = getExpectedThreadPoolSize(Settings.EMPTY, threadPoolName, 15);
             Settings nodeSettings = Settings.builder()
-                    .put("node.name", "testFixedExecutorType").build();
+                .put("node.name", "testFixedExecutorType")
+                .put("thread_pool." + threadPoolName + ".size", expectedSize)
+                .build();
             threadPool = new ThreadPool(nodeSettings);
-            ClusterSettings clusterSettings = new ClusterSettings(nodeSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-            threadPool.setClusterSettings(clusterSettings);
             assertThat(threadPool.executor(threadPoolName), instanceOf(EsThreadPoolExecutor.class));
-            Settings settings = clusterSettings.applySettings(Settings.builder()
-                    .put("threadpool." + threadPoolName + ".size", "15")
-                    .build());
 
-            int expectedSize = getExpectedThreadPoolSize(nodeSettings, threadPoolName, 15);
             assertEquals(info(threadPool, threadPoolName).getThreadPoolType(), ThreadPool.ThreadPoolType.FIXED);
             assertThat(threadPool.executor(threadPoolName), instanceOf(EsThreadPoolExecutor.class));
             assertThat(((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getCorePoolSize(), equalTo(expectedSize));
@@ -193,37 +116,6 @@ public class UpdateThreadPoolSettingsTests extends ESThreadPoolTestCase {
             assertThat(info(threadPool, threadPoolName).getMax(), equalTo(expectedSize));
             // keep alive does not apply to fixed thread pools
             assertThat(((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getKeepAliveTime(TimeUnit.MINUTES), equalTo(0L));
-
-            // Put old type back
-            settings = clusterSettings.applySettings(Settings.EMPTY);
-            assertEquals(info(threadPool, threadPoolName).getThreadPoolType(), ThreadPool.ThreadPoolType.FIXED);
-            // Make sure keep alive value is not used
-            assertThat(info(threadPool, threadPoolName).getKeepAlive(), nullValue());
-            // Make sure keep pool size value were reused
-            assertThat(info(threadPool, threadPoolName).getMin(), equalTo(expectedSize));
-            assertThat(info(threadPool, threadPoolName).getMax(), equalTo(expectedSize));
-            assertThat(threadPool.executor(threadPoolName), instanceOf(EsThreadPoolExecutor.class));
-            assertThat(((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getCorePoolSize(), equalTo(expectedSize));
-            assertThat(((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getMaximumPoolSize(), equalTo(expectedSize));
-
-            // Change size
-            Executor oldExecutor = threadPool.executor(threadPoolName);
-            settings = clusterSettings.applySettings(Settings.builder().put(settings).put("threadpool." + threadPoolName + ".size", "10").build());
-
-            expectedSize = getExpectedThreadPoolSize(nodeSettings, threadPoolName, 10);
-
-            // Make sure size values changed
-            assertThat(info(threadPool, threadPoolName).getMax(), equalTo(expectedSize));
-            assertThat(info(threadPool, threadPoolName).getMin(), equalTo(expectedSize));
-            assertThat(((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getMaximumPoolSize(), equalTo(expectedSize));
-            assertThat(((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getCorePoolSize(), equalTo(expectedSize));
-            // Make sure executor didn't change
-            assertEquals(info(threadPool, threadPoolName).getThreadPoolType(), ThreadPool.ThreadPoolType.FIXED);
-            assertThat(threadPool.executor(threadPoolName), sameInstance(oldExecutor));
-
-            // Change queue capacity
-            clusterSettings.applySettings(Settings.builder().put(settings).put("threadpool." + threadPoolName + ".queue", "500")
-                    .build());
         } finally {
             terminateThreadPoolIfNeeded(threadPool);
         }
@@ -234,11 +126,10 @@ public class UpdateThreadPoolSettingsTests extends ESThreadPoolTestCase {
         ThreadPool threadPool = null;
         try {
             Settings nodeSettings = Settings.builder()
-                    .put("threadpool." + threadPoolName + ".size", 10)
-                    .put("node.name", "testScalingExecutorType").build();
+                .put("thread_pool." + threadPoolName + ".max", 10)
+                .put("node.name", "testScalingExecutorType")
+                .build();
             threadPool = new ThreadPool(nodeSettings);
-            ClusterSettings clusterSettings = new ClusterSettings(nodeSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-            threadPool.setClusterSettings(clusterSettings);
             final int expectedMinimum = "generic".equals(threadPoolName) ? 4 : 1;
             assertThat(info(threadPool, threadPoolName).getMin(), equalTo(expectedMinimum));
             assertThat(info(threadPool, threadPoolName).getMax(), equalTo(10));
@@ -246,24 +137,6 @@ public class UpdateThreadPoolSettingsTests extends ESThreadPoolTestCase {
             assertThat(info(threadPool, threadPoolName).getKeepAlive().seconds(), equalTo(expectedKeepAlive));
             assertEquals(info(threadPool, threadPoolName).getThreadPoolType(), ThreadPool.ThreadPoolType.SCALING);
             assertThat(threadPool.executor(threadPoolName), instanceOf(EsThreadPoolExecutor.class));
-
-            // Change settings that doesn't require pool replacement
-            Executor oldExecutor = threadPool.executor(threadPoolName);
-            clusterSettings.applySettings(Settings.builder()
-                    .put("threadpool." + threadPoolName + ".keep_alive", "10m")
-                    .put("threadpool." + threadPoolName + ".min", "2")
-                    .put("threadpool." + threadPoolName + ".size", "15")
-                    .build());
-            assertEquals(info(threadPool, threadPoolName).getThreadPoolType(), ThreadPool.ThreadPoolType.SCALING);
-            assertThat(threadPool.executor(threadPoolName), instanceOf(EsThreadPoolExecutor.class));
-            assertThat(((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getCorePoolSize(), equalTo(2));
-            assertThat(((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getMaximumPoolSize(), equalTo(15));
-            assertThat(info(threadPool, threadPoolName).getMin(), equalTo(2));
-            assertThat(info(threadPool, threadPoolName).getMax(), equalTo(15));
-            // Make sure keep alive value changed
-            assertThat(info(threadPool, threadPoolName).getKeepAlive().minutes(), equalTo(10L));
-            assertThat(((EsThreadPoolExecutor) threadPool.executor(threadPoolName)).getKeepAliveTime(TimeUnit.MINUTES), equalTo(10L));
-            assertThat(threadPool.executor(threadPoolName), sameInstance(oldExecutor));
         } finally {
             terminateThreadPoolIfNeeded(threadPool);
         }
@@ -274,17 +147,18 @@ public class UpdateThreadPoolSettingsTests extends ESThreadPoolTestCase {
         ThreadPool threadPool = null;
         try {
             Settings nodeSettings = Settings.builder()
-                    .put("threadpool." + threadPoolName + ".queue_size", 1000)
-                    .put("node.name", "testShutdownNowInterrupts").build();
+                .put("thread_pool." + threadPoolName + ".queue_size", 1000)
+                .put("node.name", "testShutdownNowInterrupts")
+                .build();
             threadPool = new ThreadPool(nodeSettings);
-            ClusterSettings clusterSettings = new ClusterSettings(nodeSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-            threadPool.setClusterSettings(clusterSettings);
             assertEquals(info(threadPool, threadPoolName).getQueueSize().getSingles(), 1000L);
 
+            final CountDownLatch shutDownLatch = new CountDownLatch(1);
             final CountDownLatch latch = new CountDownLatch(1);
             ThreadPoolExecutor oldExecutor = (ThreadPoolExecutor) threadPool.executor(threadPoolName);
             threadPool.executor(threadPoolName).execute(() -> {
                         try {
+                            shutDownLatch.countDown();
                             new CountDownLatch(1).await();
                         } catch (InterruptedException ex) {
                             latch.countDown();
@@ -292,13 +166,11 @@ public class UpdateThreadPoolSettingsTests extends ESThreadPoolTestCase {
                         }
                     }
             );
-            clusterSettings.applySettings(Settings.builder().put("threadpool." + threadPoolName + ".queue_size", 2000).build());
-            assertThat(threadPool.executor(threadPoolName), not(sameInstance(oldExecutor)));
+            shutDownLatch.await();
+            threadPool.shutdownNow();
+            latch.await(3, TimeUnit.SECONDS); // if this throws then ThreadPool#shutdownNow did not interrupt
             assertThat(oldExecutor.isShutdown(), equalTo(true));
-            assertThat(oldExecutor.isTerminating(), equalTo(true));
-            assertThat(oldExecutor.isTerminated(), equalTo(false));
-            threadPool.shutdownNow(); // should interrupt the thread
-            latch.await(3, TimeUnit.SECONDS); // If this throws then ThreadPool#shutdownNow didn't interrupt
+            assertThat(oldExecutor.isTerminating() || oldExecutor.isTerminated(), equalTo(true));
         } finally {
             terminateThreadPoolIfNeeded(threadPool);
         }
@@ -307,18 +179,19 @@ public class UpdateThreadPoolSettingsTests extends ESThreadPoolTestCase {
     public void testCustomThreadPool() throws Exception {
         ThreadPool threadPool = null;
         try {
-            Settings nodeSettings = Settings.builder()
-                    .put("threadpool.my_pool1.type", "scaling")
-                    .put("threadpool.my_pool1.min", 1)
-                    .put("threadpool.my_pool1.size", EsExecutors.boundedNumberOfProcessors(Settings.EMPTY))
-                    .put("threadpool.my_pool1.keep_alive", "1m")
-                    .put("threadpool.my_pool2.type", "fixed")
-                    .put("threadpool.my_pool2.size", "1")
-                    .put("threadpool.my_pool2.queue_size", "1")
-                    .put("node.name", "testCustomThreadPool").build();
-            threadPool = new ThreadPool(nodeSettings);
-            ClusterSettings clusterSettings = new ClusterSettings(nodeSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
-            threadPool.setClusterSettings(clusterSettings);
+
+
+            final ScalingExecutorBuilder scaling =
+                new ScalingExecutorBuilder(
+                    "my_pool1",
+                    1,
+                    EsExecutors.boundedNumberOfProcessors(Settings.EMPTY),
+                    TimeValue.timeValueMinutes(1));
+
+            final FixedExecutorBuilder fixed = new FixedExecutorBuilder(Settings.EMPTY, "my_pool2", 1, 1);
+
+            threadPool = new ThreadPool(Settings.builder().put("node.name", "testCustomThreadPool").build(), scaling, fixed);
+
             ThreadPoolInfo groups = threadPool.info();
             boolean foundPool1 = false;
             boolean foundPool2 = false;
@@ -345,39 +218,6 @@ public class UpdateThreadPoolSettingsTests extends ESThreadPoolTestCase {
             }
             assertThat(foundPool1, is(true));
             assertThat(foundPool2, is(true));
-
-            // Updating my_pool2
-            Settings settings = Settings.builder()
-                    .put("threadpool.my_pool2.size", "10")
-                    .build();
-            clusterSettings.applySettings(settings);
-
-            groups = threadPool.info();
-            foundPool1 = false;
-            foundPool2 = false;
-            outer:
-            for (ThreadPool.Info info : groups) {
-                if ("my_pool1".equals(info.getName())) {
-                    foundPool1 = true;
-                    assertEquals(info.getThreadPoolType(), ThreadPool.ThreadPoolType.SCALING);
-                } else if ("my_pool2".equals(info.getName())) {
-                    foundPool2 = true;
-                    assertThat(info.getMax(), equalTo(10));
-                    assertThat(info.getMin(), equalTo(10));
-                    assertThat(info.getQueueSize().singles(), equalTo(1L));
-                    assertEquals(info.getThreadPoolType(), ThreadPool.ThreadPoolType.FIXED);
-                } else {
-                    for (Field field : Names.class.getFields()) {
-                        if (info.getName().equalsIgnoreCase(field.getName())) {
-                            // This is ok it is a default thread pool
-                            continue outer;
-                        }
-                    }
-                    fail("Unexpected pool name: " + info.getName());
-                }
-            }
-            assertThat(foundPool1, is(true));
-            assertThat(foundPool2, is(true));
         } finally {
             terminateThreadPoolIfNeeded(threadPool);
         }
@@ -388,11 +228,4 @@ public class UpdateThreadPoolSettingsTests extends ESThreadPoolTestCase {
         return randomFrom(threadPoolNames.toArray(new String[threadPoolNames.size()]));
     }
 
-    private ThreadPool.ThreadPoolType randomIncorrectThreadPoolType(String threadPoolName) {
-        Set<ThreadPool.ThreadPoolType> set = new HashSet<>();
-        set.addAll(Arrays.asList(ThreadPool.ThreadPoolType.values()));
-        set.remove(ThreadPool.THREAD_POOL_TYPES.get(threadPoolName));
-        return randomFrom(set.toArray(new ThreadPool.ThreadPoolType[set.size()]));
-    }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
index f285a1db52e..68c4c76d7e8 100644
--- a/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
+++ b/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
@@ -31,6 +31,7 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
 import org.junit.Before;
@@ -71,7 +72,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
     @Before
     public void setUp() throws Exception {
         super.setUp();
-        threadPool = new ThreadPool(getClass().getName());
+        threadPool = new TestThreadPool(getClass().getName());
         serviceA = build(
                 Settings.builder()
                     .put("name", "TS_A")
diff --git a/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java b/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java
index ca5faf68b51..3c382c39509 100644
--- a/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java
@@ -64,7 +64,6 @@ public class NettySizeHeaderFrameDecoderTests extends ESTestCase {
     @Before
     public void startThreadPool() {
         threadPool = new ThreadPool(settings);
-        threadPool.setClusterSettings(new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
         NetworkService networkService = new NetworkService(settings);
         BigArrays bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
         nettyTransport = new NettyTransport(settings, threadPool, networkService, bigArrays, Version.CURRENT, new NamedWriteableRegistry(),
diff --git a/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java b/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java
index 6a7e0a0de36..a111911e994 100644
--- a/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java
@@ -30,6 +30,7 @@ import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.netty.NettyTransport;
 import org.junit.After;
@@ -51,7 +52,7 @@ public class NettyTransportServiceHandshakeTests extends ESTestCase {
 
     @BeforeClass
     public static void startThreadPool() {
-        threadPool = new ThreadPool(NettyTransportServiceHandshakeTests.class.getSimpleName());
+        threadPool = new TestThreadPool(NettyTransportServiceHandshakeTests.class.getSimpleName());
     }
 
     private List<TransportService> transportServices = new ArrayList<>();
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java
index 8c00ae01b74..de156659082 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java
@@ -30,6 +30,7 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.BaseTransportResponseHandler;
 import org.elasticsearch.transport.TransportChannel;
@@ -52,7 +53,7 @@ import static org.hamcrest.Matchers.greaterThan;
  */
 public class NettyScheduledPingTests extends ESTestCase {
     public void testScheduledPing() throws Exception {
-        ThreadPool threadPool = new ThreadPool(getClass().getName());
+        ThreadPool threadPool = new TestThreadPool(getClass().getName());
 
         Settings settings = Settings.builder()
             .put(NettyTransport.PING_SCHEDULE.getKey(), "5ms")
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java
index 352e7bb5a59..0ed0cf3ec97 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java
@@ -28,6 +28,7 @@ import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.TransportSettings;
@@ -56,7 +57,7 @@ public class NettyTransportMultiPortTests extends ESTestCase {
                 .put("transport.profiles.client1.port", 0)
                 .build();
 
-        ThreadPool threadPool = new ThreadPool("tst");
+        ThreadPool threadPool = new TestThreadPool("tst");
         try (NettyTransport transport = startNettyTransport(settings, threadPool)) {
             assertEquals(1, transport.profileBoundAddresses().size());
             assertEquals(1, transport.boundAddress().boundAddresses().length);
@@ -72,7 +73,7 @@ public class NettyTransportMultiPortTests extends ESTestCase {
                 .put("transport.profiles.client1.port", 0)
                 .build();
 
-        ThreadPool threadPool = new ThreadPool("tst");
+        ThreadPool threadPool = new TestThreadPool("tst");
         try (NettyTransport transport = startNettyTransport(settings, threadPool)) {
             assertEquals(1, transport.profileBoundAddresses().size());
             assertEquals(1, transport.boundAddress().boundAddresses().length);
@@ -89,7 +90,7 @@ public class NettyTransportMultiPortTests extends ESTestCase {
                 .put("transport.profiles.client1.whatever", "foo")
                 .build();
 
-        ThreadPool threadPool = new ThreadPool("tst");
+        ThreadPool threadPool = new TestThreadPool("tst");
         try (NettyTransport transport = startNettyTransport(settings, threadPool)) {
             assertEquals(0, transport.profileBoundAddresses().size());
             assertEquals(1, transport.boundAddress().boundAddresses().length);
@@ -105,7 +106,7 @@ public class NettyTransportMultiPortTests extends ESTestCase {
                 .put("transport.profiles.default.port", 0)
                 .build();
 
-        ThreadPool threadPool = new ThreadPool("tst");
+        ThreadPool threadPool = new TestThreadPool("tst");
         try (NettyTransport transport = startNettyTransport(settings, threadPool)) {
             assertEquals(0, transport.profileBoundAddresses().size());
             assertEquals(1, transport.boundAddress().boundAddresses().length);
@@ -123,7 +124,7 @@ public class NettyTransportMultiPortTests extends ESTestCase {
                 .put("transport.profiles..port", 23) // will not actually bind to this
                 .build();
 
-        ThreadPool threadPool = new ThreadPool("tst");
+        ThreadPool threadPool = new TestThreadPool("tst");
         try (NettyTransport transport = startNettyTransport(settings, threadPool)) {
             assertEquals(0, transport.profileBoundAddresses().size());
             assertEquals(1, transport.boundAddress().boundAddresses().length);
diff --git a/core/src/test/java/org/elasticsearch/watcher/ResourceWatcherServiceTests.java b/core/src/test/java/org/elasticsearch/watcher/ResourceWatcherServiceTests.java
index 6c6c45e9cfd..82a3a55868a 100644
--- a/core/src/test/java/org/elasticsearch/watcher/ResourceWatcherServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/watcher/ResourceWatcherServiceTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.watcher;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
@@ -33,7 +34,7 @@ import static org.hamcrest.Matchers.notNullValue;
  */
 public class ResourceWatcherServiceTests extends ESTestCase {
     public void testSettings() throws Exception {
-        ThreadPool threadPool = new ThreadPool("test");
+        ThreadPool threadPool = new TestThreadPool("test");
 
         // checking the defaults
         Settings settings = Settings.builder().build();
@@ -65,7 +66,7 @@ public class ResourceWatcherServiceTests extends ESTestCase {
     }
 
     public void testHandle() throws Exception {
-        ThreadPool threadPool = new ThreadPool("test");
+        ThreadPool threadPool = new TestThreadPool("test");
         Settings settings = Settings.builder().build();
         ResourceWatcherService service = new ResourceWatcherService(settings, threadPool);
         ResourceWatcher watcher = new ResourceWatcher() {
diff --git a/docs/reference/migration/migrate_5_0/settings.asciidoc b/docs/reference/migration/migrate_5_0/settings.asciidoc
index 5edd4e449e8..5702d51fc12 100644
--- a/docs/reference/migration/migrate_5_0/settings.asciidoc
+++ b/docs/reference/migration/migrate_5_0/settings.asciidoc
@@ -79,6 +79,19 @@ on the thread pool type, `keep_alive`, `queue_size`, etc.).
 The `suggest` threadpool has been removed, now suggest requests use the
 `search` threadpool.
 
+The prefix on all thread pool settings has been changed from
+`threadpool` to `thread_pool`.
+
+The minimum size setting for a scaling thread pool has been changed
+from `min` to `core`.
+
+The maximum size setting for a scaling thread pool has been changed
+from `size` to `max`.
+
+The queue size setting for a fixed thread pool must be `queue_size`
+(all other variants that were previously supported are no longer
+supported).
+
 ==== Analysis settings
 
 The `index.analysis.analyzer.default_index` analyzer is not supported anymore.
diff --git a/docs/reference/modules/threadpool.asciidoc b/docs/reference/modules/threadpool.asciidoc
index 65069e25940..d8f8545c213 100644
--- a/docs/reference/modules/threadpool.asciidoc
+++ b/docs/reference/modules/threadpool.asciidoc
@@ -17,7 +17,7 @@ There are several thread pools, but the important ones include:
     For index/delete operations. Thread pool type is `fixed`
     with a size of `# of available processors`,
     queue_size of `200`.  The maximum size for this pool
-    is `# of available processors`.
+    is `1 + # of available processors`.
 
 `search`::
     For count/search/suggest operations. Thread pool type is `fixed`
@@ -33,7 +33,7 @@ There are several thread pools, but the important ones include:
     For bulk operations. Thread pool type is `fixed`
     with a size of `# of available processors`,
     queue_size of `50`.  The maximum size for this pool
-    is `# of available processors`.
+    is `1 + # of available processors`.
 
 `percolate`::
     For percolate operations. Thread pool type is `fixed`
@@ -42,26 +42,26 @@ There are several thread pools, but the important ones include:
 
 `snapshot`::
     For snapshot/restore operations. Thread pool type is `scaling` with a
-    keep-alive of `5m` and a size of `min(5, (# of available processors)/2)`.
+    keep-alive of `5m` and a max of `min(5, (# of available processors)/2)`.
 
 `warmer`::
     For segment warm-up operations. Thread pool type is `scaling` with a
-    keep-alive of `5m` and a size of `min(5, (# of available processors)/2)`.
+    keep-alive of `5m` and a max of `min(5, (# of available processors)/2)`.
 
 `refresh`::
     For refresh operations. Thread pool type is `scaling` with a
-    keep-alive of `5m` and a size of `min(10, (# of available processors)/2)`.
+    keep-alive of `5m` and a max of `min(10, (# of available processors)/2)`.
 
 `listener`::
     Mainly for java client executing of action when listener threaded is set to true.
-    Thread pool type is `scaling` with a default size of `min(10, (# of available processors)/2)`.
+    Thread pool type is `scaling` with a default max of `min(10, (# of available processors)/2)`.
 
 Changing a specific thread pool can be done by setting its type-specific parameters; for example, changing the `index`
 thread pool to have more threads:
 
 [source,js]
 --------------------------------------------------
-threadpool:
+thread_pool:
     index:
         size: 30
 --------------------------------------------------
@@ -91,7 +91,7 @@ full, it will abort the request.
 
 [source,js]
 --------------------------------------------------
-threadpool:
+thread_pool:
     index:
         size: 30
         queue_size: 1000
@@ -102,17 +102,17 @@ threadpool:
 
 The `scaling` thread pool holds a dynamic number of threads. This
 number is proportional to the workload and varies between the value of
-the `min` and `size` parameters.
+the `core` and `max` parameters.
 
 The `keep_alive` parameter determines how long a thread should be kept
 around in the thread pool without it doing any work.
 
 [source,js]
 --------------------------------------------------
-threadpool:
+thread_pool:
     warmer:
-        min: 1
-        size: 8
+        core: 1
+        max: 8
         keep_alive: 2m
 --------------------------------------------------
 
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
index 9aa5e1892ef..ddb330b01a7 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
@@ -108,10 +108,11 @@ public class TemplateQueryParserTests extends ESTestCase {
         // TODO: make this use a mock engine instead of mustache and it will no longer be messy!
         scriptModule.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(MustacheScriptEngineService.class, MustacheScriptEngineService.NAME, true));
         settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
+        final ThreadPool threadPool = new ThreadPool(settings);
         injector = new ModulesBuilder().add(
                 new EnvironmentModule(new Environment(settings)),
                 settingsModule,
-                new ThreadPoolModule(new ThreadPool(settings)),
+                new ThreadPoolModule(threadPool),
                 new SearchModule(settings, new NamedWriteableRegistry()) {
                     @Override
                     protected void configureSearch() {
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionTestCase.java
index b9489e9f5d9..66c636f4f1d 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionTestCase.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionTestCase.java
@@ -22,6 +22,7 @@ package org.elasticsearch.index.reindex;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
 import org.junit.Before;
@@ -35,7 +36,7 @@ public abstract class AbstractAsyncBulkIndexByScrollActionTestCase<
 
     @Before
     public void setupForTest() {
-        threadPool = new ThreadPool(getTestName());
+        threadPool = new TestThreadPool(getTestName());
         task = new BulkByScrollTask(1, "test", "test", "test", TaskId.EMPTY_TASK_ID, 0);
     }
 
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
index 9ce505bacd3..2af63deb3a9 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
@@ -72,6 +72,7 @@ import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.client.NoOpClient;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
 import org.junit.Before;
@@ -126,7 +127,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
     @Before
     public void setupForTest() {
         client = new MyMockClient(new NoOpClient(getTestName()));
-        threadPool = new ThreadPool(getTestName());
+        threadPool = new TestThreadPool(getTestName());
         firstSearchRequest = new SearchRequest();
         testRequest = new DummyAbstractBulkByScrollRequest(firstSearchRequest);
         listener = new PlainActionFuture<>();
@@ -311,7 +312,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
     public void testThreadPoolRejectionsAbortRequest() throws Exception {
         TimeValue expectedDelay = parseTimeValue(randomPositiveTimeValue(), "test");
         threadPool.shutdown();
-        threadPool = new ThreadPool(getTestName()) {
+        threadPool = new TestThreadPool(getTestName()) {
             @Override
             public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable command) {
                 assertEquals(expectedDelay, delay); // While we're here we can check that the sleep made it through
@@ -444,7 +445,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         AtomicReference<TimeValue> capturedDelay = new AtomicReference<>();
         AtomicReference<Runnable> capturedCommand = new AtomicReference<>();
         threadPool.shutdown();
-        threadPool = new ThreadPool(getTestName()) {
+        threadPool = new TestThreadPool(getTestName()) {
             @Override
             public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable command) {
                 capturedDelay.set(delay);
@@ -612,7 +613,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
          * is a delay.
          */
         threadPool.shutdown();
-        threadPool = new ThreadPool(getTestName()) {
+        threadPool = new TestThreadPool(getTestName()) {
             @Override
             public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable command) {
                 /*
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
index d64c69ba362..fd1a17a439d 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
@@ -26,6 +26,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.Before;
 
@@ -168,7 +169,7 @@ public class BulkByScrollTaskTests extends ESTestCase {
         task.rethrottle(originalRequestsPerSecond);
         TimeValue maxDelay = timeValueSeconds(between(1, 5));
         assertThat(maxDelay.nanos(), greaterThanOrEqualTo(0L));
-        ThreadPool threadPool = new ThreadPool(getTestName()) {
+        ThreadPool threadPool = new TestThreadPool(getTestName()) {
             @Override
             public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable command) {
                 assertThat(delay.nanos(), both(greaterThanOrEqualTo(0L)).and(lessThanOrEqualTo(maxDelay.nanos())));
@@ -220,7 +221,7 @@ public class BulkByScrollTaskTests extends ESTestCase {
 
     public void testDelayNeverNegative() throws IOException {
         // Thread pool that returns a ScheduledFuture that claims to have a negative delay
-        ThreadPool threadPool = new ThreadPool("test") {
+        ThreadPool threadPool = new TestThreadPool("test") {
             public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable command) {
                 return new ScheduledFuture<Void>() {
                     @Override
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
index 699ba483e30..09945c9372b 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
@@ -63,11 +63,11 @@ public class RetryTests extends ESSingleNodeTestCase {
     protected Settings nodeSettings() {
         Settings.Builder settings = Settings.builder().put(super.nodeSettings());
         // Use pools of size 1 so we can block them
-        settings.put("threadpool.bulk.size", 1);
-        settings.put("threadpool.search.size", 1);
+        settings.put("thread_pool.bulk.size", 1);
+        settings.put("thread_pool.search.size", 1);
         // Use queues of size 1 because size 0 is broken and because search requests need the queue to function
-        settings.put("threadpool.bulk.queue_size", 1);
-        settings.put("threadpool.search.queue_size", 1);
+        settings.put("thread_pool.bulk.queue_size", 1);
+        settings.put("thread_pool.search.queue_size", 1);
         return settings.build();
     }
 
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
index c4863680613..956732113c1 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
@@ -30,6 +30,7 @@ import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.junit.AfterClass;
@@ -53,7 +54,7 @@ public class Ec2DiscoveryTests extends ESTestCase {
 
     @BeforeClass
     public static void createThreadPool() {
-        threadPool = new ThreadPool(Ec2DiscoveryTests.class.getName());
+        threadPool = new TestThreadPool(Ec2DiscoveryTests.class.getName());
     }
 
     @AfterClass
diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java
index 4525b1ece1d..78d96fd0163 100644
--- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java
+++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -72,7 +73,7 @@ public class GceDiscoveryTests extends ESTestCase {
 
     @BeforeClass
     public static void createThreadPool() {
-        threadPool = new ThreadPool(GceDiscoveryTests.class.getName());
+        threadPool = new TestThreadPool(GceDiscoveryTests.class.getName());
     }
 
     @AfterClass
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 0859a8e86ea..8f3fd5010e6 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -28,7 +28,6 @@ import com.carrotsearch.randomizedtesting.generators.RandomInts;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter;
-
 import org.apache.lucene.uninverting.UninvertingReader;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/client/NoOpClient.java b/test/framework/src/main/java/org/elasticsearch/test/client/NoOpClient.java
index 5f2237640e6..6ff45608700 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/client/NoOpClient.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/client/NoOpClient.java
@@ -27,6 +27,7 @@ import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.client.support.AbstractClient;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.concurrent.TimeUnit;
@@ -34,7 +35,7 @@ import java.util.concurrent.TimeUnit;
 public class NoOpClient extends AbstractClient {
 
     public NoOpClient(String testName) {
-        super(Settings.EMPTY, new ThreadPool(testName));
+        super(Settings.EMPTY, new TestThreadPool(testName));
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/threadpool/TestThreadPool.java b/test/framework/src/main/java/org/elasticsearch/threadpool/TestThreadPool.java
new file mode 100644
index 00000000000..0d525f7f59a
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/threadpool/TestThreadPool.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.threadpool;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.node.Node;
+
+public class TestThreadPool extends ThreadPool {
+
+    public TestThreadPool(String name) {
+        super(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), name).build());
+    }
+
+}

From e94408c0d2748ce8829fb5096559221763026889 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 6 Jun 2016 18:28:34 -0400
Subject: [PATCH 0684/1311] Bootstrap check for OnError and seccomp

This commit adds a bootstrap check for the JVM option OnError being in
use and seccomp being enabled. These two options are incompatible
because OnError allows the user to specify an arbitrary program to fork
when the JVM encounters an fatal error, and seccomp enables system call
filters that prevents forking.
---
 .../bootstrap/BootstrapCheck.java             |  64 ++++++++--
 .../elasticsearch/monitor/jvm/JvmInfo.java    |  12 ++
 .../bootstrap/BootstrapCheckTests.java        | 114 +++++++++++++++---
 .../reference/setup/bootstrap-checks.asciidoc |  22 ++--
 4 files changed, 172 insertions(+), 40 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
index f5b28b19b60..fc36e4fb671 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
@@ -164,6 +164,7 @@ final class BootstrapCheck {
             checks.add(new MaxMapCountCheck());
         }
         checks.add(new ClientJvmCheck());
+        checks.add(new OnErrorCheck());
         checks.add(new OnOutOfMemoryErrorCheck());
         return Collections.unmodifiableList(checks);
     }
@@ -507,12 +508,11 @@ final class BootstrapCheck {
 
     }
 
-    static class OnOutOfMemoryErrorCheck implements BootstrapCheck.Check {
+    static abstract class MightForkCheck implements BootstrapCheck.Check {
 
         @Override
         public boolean check() {
-            final String onOutOfMemoryError = onOutOfMemoryError();
-            return isSeccompInstalled() && onOutOfMemoryError != null && !onOutOfMemoryError.equals("");
+            return isSeccompInstalled() && mightFork();
         }
 
         // visible for testing
@@ -520,6 +520,54 @@ final class BootstrapCheck {
             return Natives.isSeccompInstalled();
         }
 
+        // visible for testing
+        abstract boolean mightFork();
+
+        @Override
+        public final boolean isSystemCheck() {
+            return false;
+        }
+
+        @Override
+        public final boolean alwaysEnforce() {
+            return true;
+        }
+
+    }
+
+    static class OnErrorCheck extends MightForkCheck {
+
+        @Override
+        boolean mightFork() {
+            final String onError = onError();
+            return onError != null && !onError.equals("");
+        }
+
+        // visible for testing
+        String onError() {
+            return JvmInfo.jvmInfo().onError();
+        }
+
+        @Override
+        public String errorMessage() {
+            return String.format(
+                Locale.ROOT,
+                "OnError [%s] requires forking but is prevented by system call filters ([%s=true]);" +
+                    " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError",
+                onError(),
+                BootstrapSettings.SECCOMP_SETTING.getKey());
+        }
+
+    }
+
+    static class OnOutOfMemoryErrorCheck extends MightForkCheck {
+
+        @Override
+        boolean mightFork() {
+            final String onOutOfMemoryError = onOutOfMemoryError();
+            return onOutOfMemoryError != null && !onOutOfMemoryError.equals("");
+        }
+
         // visible for testing
         String onOutOfMemoryError() {
             return JvmInfo.jvmInfo().onOutOfMemoryError();
@@ -535,16 +583,6 @@ final class BootstrapCheck {
                 BootstrapSettings.SECCOMP_SETTING.getKey());
         }
 
-        @Override
-        public boolean isSystemCheck() {
-            return false;
-        }
-
-        @Override
-        public boolean alwaysEnforce() {
-            return true;
-        }
-
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java
index b2e1990a680..69abd0752f6 100644
--- a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java
+++ b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java
@@ -115,6 +115,12 @@ public class JvmInfo implements Streamable, ToXContent {
             Method vmOptionMethod = clazz.getMethod("getVMOption", String.class);
             Method valueMethod = vmOptionClazz.getMethod("getValue");
 
+            try {
+                Object onError = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "OnError");
+                info.onError = (String) valueMethod.invoke(onError);
+            } catch (Exception ignored) {
+            }
+
             try {
                 Object onOutOfMemoryError = vmOptionMethod.invoke(hotSpotDiagnosticMXBean, "OnOutOfMemoryError");
                 info.onOutOfMemoryError = (String) valueMethod.invoke(onOutOfMemoryError);
@@ -185,6 +191,8 @@ public class JvmInfo implements Streamable, ToXContent {
     String[] gcCollectors = Strings.EMPTY_ARRAY;
     String[] memoryPools = Strings.EMPTY_ARRAY;
 
+    private String onError;
+
     private String onOutOfMemoryError;
 
     private String useCompressedOops = "unknown";
@@ -322,6 +330,10 @@ public class JvmInfo implements Streamable, ToXContent {
         return configuredMaxHeapSize;
     }
 
+    public String onError() {
+        return onError;
+    }
+
     public String onOutOfMemoryError() {
         return onOutOfMemoryError;
     }
diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
index 41609f7fc5d..ddd4f01ef8b 100644
--- a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
+++ b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
@@ -33,6 +33,7 @@ import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
 
 import static org.hamcrest.CoreMatchers.allOf;
 import static org.hamcrest.CoreMatchers.containsString;
@@ -396,10 +397,66 @@ public class BootstrapCheckTests extends ESTestCase {
         BootstrapCheck.check(true, false, Collections.singletonList(check), "testClientJvmCheck");
     }
 
+    public void testMightForkCheck() {
+        final AtomicBoolean isSeccompInstalled = new AtomicBoolean();
+        final AtomicBoolean mightFork = new AtomicBoolean();
+        final BootstrapCheck.MightForkCheck check = new BootstrapCheck.MightForkCheck() {
+            @Override
+            boolean isSeccompInstalled() {
+                return isSeccompInstalled.get();
+            }
+
+            @Override
+            boolean mightFork() {
+                return mightFork.get();
+            }
+
+            @Override
+            public String errorMessage() {
+                return "error";
+            }
+        };
+
+        runMightForkTest(
+            check,
+            isSeccompInstalled,
+            () -> mightFork.set(false),
+            () -> mightFork.set(true),
+            e -> assertThat(e.getMessage(), containsString("error")));
+    }
+
+    public void testOnErrorCheck() {
+        final AtomicBoolean isSeccompInstalled = new AtomicBoolean();
+        final AtomicReference<String> onError = new AtomicReference<>();
+        final BootstrapCheck.MightForkCheck check = new BootstrapCheck.OnErrorCheck() {
+            @Override
+            boolean isSeccompInstalled() {
+                return isSeccompInstalled.get();
+            }
+
+            @Override
+            String onError() {
+                return onError.get();
+            }
+        };
+
+        final String command = randomAsciiOfLength(16);
+        runMightForkTest(
+            check,
+            isSeccompInstalled,
+            () -> onError.set(randomBoolean() ? "" : null),
+            () -> onError.set(command),
+            e -> assertThat(
+                e.getMessage(),
+                containsString(
+                    "OnError [" + command + "] requires forking but is prevented by system call filters ([bootstrap.seccomp=true]);"
+                        + " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError")));
+    }
+
     public void testOnOutOfMemoryErrorCheck() {
         final AtomicBoolean isSeccompInstalled = new AtomicBoolean();
         final AtomicReference<String> onOutOfMemoryError = new AtomicReference<>();
-        final BootstrapCheck.Check check = new BootstrapCheck.OnOutOfMemoryErrorCheck() {
+        final BootstrapCheck.MightForkCheck check = new BootstrapCheck.OnOutOfMemoryErrorCheck() {
             @Override
             boolean isSeccompInstalled() {
                 return isSeccompInstalled.get();
@@ -411,31 +468,52 @@ public class BootstrapCheckTests extends ESTestCase {
             }
         };
 
+        final String command = randomAsciiOfLength(16);
+        runMightForkTest(
+            check,
+            isSeccompInstalled,
+            () -> onOutOfMemoryError.set(randomBoolean() ? "" : null),
+            () -> onOutOfMemoryError.set(command),
+            e -> assertThat(
+                e.getMessage(),
+                containsString(
+                    "OnOutOfMemoryError [" + command + "]"
+                        + " requires forking but is prevented by system call filters ([bootstrap.seccomp=true]);"
+                        + " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError")));
+    }
+
+    private void runMightForkTest(
+        final BootstrapCheck.MightForkCheck check,
+        final AtomicBoolean isSeccompInstalled,
+        final Runnable disableMightFork,
+        final Runnable enableMightFork,
+        final Consumer<RuntimeException> consumer) {
+
         // if seccomp is disabled, nothing should happen
         isSeccompInstalled.set(false);
-        onOutOfMemoryError.set(randomBoolean() ? "" : randomAsciiOfLength(16));
-        BootstrapCheck.check(true, false, Collections.singletonList(check), "testOnOutOfMemoryErrorCheck");
+        if (randomBoolean()) {
+            disableMightFork.run();
+        } else {
+            enableMightFork.run();
+        }
+        BootstrapCheck.check(true, randomBoolean(), Collections.singletonList(check), "testMightFork");
 
-        // if seccomp is enabled, and OnOutOfMemoryError is not, nothing
-        // should happen
+        // if seccomp is enabled, but we will not fork, nothing should
+        // happen
         isSeccompInstalled.set(true);
-        onOutOfMemoryError.set("");
-        BootstrapCheck.check(true, false, Collections.singletonList(check), "testOnOutOfMemoryErrorCheck");
+        disableMightFork.run();
+        BootstrapCheck.check(true, randomBoolean(), Collections.singletonList(check), "testMightFork");
 
-        // if seccomp is enabled, and OnOutOfMemoryError is, the check
-        // should be enforced, regardless of bootstrap checks being
-        // enabled or not
+        // if seccomp is enabled, and we might fork, the check should
+        // be enforced, regardless of bootstrap checks being enabled or
+        // not
         isSeccompInstalled.set(true);
-        final String command = randomAsciiOfLength(16);
-        onOutOfMemoryError.set(command);
+        enableMightFork.run();
+
         final RuntimeException e = expectThrows(
             RuntimeException.class,
-            () -> BootstrapCheck.check(randomBoolean(), randomBoolean(), Collections.singletonList(check), "testOnOutOfMemoryErrorCheck"));
-        assertThat(
-            e.getMessage(),
-            containsString(
-                "OnOutOfMemoryError [" + command + "] requires forking but is prevented by system call filters ([bootstrap.seccomp=true]);"
-                    + " upgrade to at least Java 8u92 and use ExitOnOutOfMemoryError"));
+            () -> BootstrapCheck.check(randomBoolean(), randomBoolean(), Collections.singletonList(check), "testMightFork"));
+        consumer.accept(e);
     }
 
     public void testIgnoringSystemChecks() {
diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc
index 81d14f24c8f..1da61360033 100644
--- a/docs/reference/setup/bootstrap-checks.asciidoc
+++ b/docs/reference/setup/bootstrap-checks.asciidoc
@@ -157,14 +157,18 @@ systems and operating systems, the server VM is the
 default. Additionally, Elasticsearch is configured by default to force
 the server VM.
 
-=== OnOutOfMemoryError check
+=== OnError and OnOutOfMemoryError checks
 
-The JVM option `OnOutOfMemoryError` enables executing an arbitrary
-command if the JVM throws an `OutOfMemoryError`. However, by default,
+The JVM options `OnError` and `OnOutOfMemoryError` enable executing
+arbitrary commands if the JVM encounters a fatal error (`OnError`) or an
+`OutOfMemoryError` (`OnOutOfMemoryError`). However, by default,
 Elasticsearch system call filters (seccomp) are enabled and these
-filters prevent forking. Thus, using `OnOutOfMemoryError` and system
-call filters are incompatible. The `OnOutOfMemoryError` check prevents
-Elasticsearch from starting if this JVM option is used and system call
-filters are enabled. This check is always enforced. To pass this check
-do not enable `OnOutOfMemoryError`; instead, upgrade to Java 8u92 and
-use the JVM flag `ExitOnOutOfMemoryError`.
+filters prevent forking. Thus, using `OnError` or `OnOutOfMemoryError`
+and system call filters are incompatible. The `OnError` and
+`OnOutOfMemoryError` checks prevent Elasticsearch from starting if
+either of these JVM options are used and system call filters are
+enabled. This check is always enforced. To pass this check do not enable
+`OnError` nor `OnOutOfMemoryError`; instead, upgrade to Java 8u92 and
+use the JVM flag `ExitOnOutOfMemoryError`. While this does not have the
+full capabilities of `OnError` nor `OnOutOfMemoryError`, arbitrary
+forking will not be supported with seccomp enabled.

From acc9cea8f606b5093df074a4213e094844cc3ecd Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 6 Jun 2016 23:26:22 -0400
Subject: [PATCH 0685/1311] Fix compilation issue in RefreshListenersTests

This commit fixes a compilation issue in RefreshListenersTests that
arose from code being integrated into master, and then a large pull
request refactoring the handling of thread pools was later merged into
master.
---
 .../org/elasticsearch/index/shard/RefreshListenersTests.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
index 5c83e6805ab..2543668f557 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
@@ -54,6 +54,7 @@ import org.elasticsearch.index.translog.TranslogConfig;
 import org.elasticsearch.test.DummyShardLock;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
 import org.junit.Before;
@@ -93,7 +94,7 @@ public class RefreshListenersTests extends ESTestCase {
                 );
 
         // Now setup the InternalEngine which is much more complicated because we aren't mocking anything
-        threadPool = new ThreadPool(getTestName());
+        threadPool = new TestThreadPool(getTestName());
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("index", Settings.EMPTY);
         ShardId shardId = new ShardId(new Index("index", "_na_"), 1);
         Directory directory = newDirectory();

From b2c4c323e1014563dd02e9cc2f3a9175fd831e78 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Tue, 7 Jun 2016 10:06:41 +0200
Subject: [PATCH 0686/1311] Allow `_shrink` to N shards if source shards is a
 multiple of N (#18699)

Today we allow to shrink to 1 shard but that might not be possible due to
too many document or a single shard doesn't meet the requirements for the index.
The logic can be expanded to N shards if the source index shards is a multiple of N.
This guarantees that there are not hotspots created due to different number of shards
being shrunk into one.
---
 .../admin/indices/shrink/ShrinkRequest.java   |   2 +-
 .../indices/shrink/ShrinkRequestBuilder.java  |   2 +-
 .../indices/shrink/TransportShrinkAction.java |  56 +++---
 .../cluster/metadata/IndexMetaData.java       | 119 +++++++++++-
 .../metadata/MetaDataCreateIndexService.java  |  24 ++-
 .../cluster/routing/OperationRouting.java     |   6 +-
 .../decider/DiskThresholdDecider.java         |   9 +-
 .../elasticsearch/index/shard/IndexShard.java |   7 +-
 .../admin/indices/RestShrinkIndexAction.java  |   2 +-
 .../admin/indices/create/CreateIndexIT.java   |  70 +++++++
 .../shrink/TransportShrinkActionTests.java    |  37 ++--
 .../cluster/metadata/IndexMetaDataTests.java  | 126 +++++++++++++
 .../MetaDataCreateIndexServiceTests.java      |  40 ++--
 .../routing/OperationRoutingTests.java        | 173 ++++++++++++++++++
 .../DiskThresholdDeciderUnitTests.java        |  23 ++-
 docs/reference/indices/shrink-index.asciidoc  |  33 +++-
 16 files changed, 640 insertions(+), 89 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java
 create mode 100644 core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java

diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java
index 069c27ce475..a14a3f37a86 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java
@@ -94,7 +94,7 @@ public class ShrinkRequest extends AcknowledgedRequest<ShrinkRequest> implements
     /**
      * Returns the {@link CreateIndexRequest} for the shrink index
      */
-    public CreateIndexRequest getShrinkIndexReqeust() {
+    public CreateIndexRequest getShrinkIndexRequest() {
         return shrinkIndexRequest;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequestBuilder.java
index a098215b750..ab392a7f824 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequestBuilder.java
@@ -41,7 +41,7 @@ public class ShrinkRequestBuilder extends AcknowledgedRequestBuilder<ShrinkReque
     }
 
     public ShrinkRequestBuilder setSettings(Settings settings) {
-        this.request.getShrinkIndexReqeust().settings(settings);
+        this.request.getShrinkIndexRequest().settings(settings);
         return this;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java
index ca007e31216..a00dd23b764 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java
@@ -23,6 +23,7 @@ import org.apache.lucene.index.IndexWriter;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
@@ -34,27 +35,17 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
-import org.elasticsearch.cluster.routing.IndexRoutingTable;
-import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.ShardRoutingState;
-import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.shard.DocsStats;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Predicate;
+import java.util.function.IntFunction;
 
 /**
  * Main class to initiate shrinking an index into a new index with a single shard
@@ -87,7 +78,7 @@ public class TransportShrinkAction extends TransportMasterNodeAction<ShrinkReque
 
     @Override
     protected ClusterBlockException checkBlock(ShrinkRequest request, ClusterState state) {
-        return state.blocks().indexBlockedException(ClusterBlockLevel.METADATA_WRITE, request.getShrinkIndexReqeust().index());
+        return state.blocks().indexBlockedException(ClusterBlockLevel.METADATA_WRITE, request.getShrinkIndexRequest().index());
     }
 
     @Override
@@ -98,7 +89,10 @@ public class TransportShrinkAction extends TransportMasterNodeAction<ShrinkReque
             @Override
             public void onResponse(IndicesStatsResponse indicesStatsResponse) {
                 CreateIndexClusterStateUpdateRequest updateRequest = prepareCreateIndexRequest(shrinkRequest, state,
-                    indicesStatsResponse.getTotal().getDocs(), indexNameExpressionResolver);
+                    (i) -> {
+                        IndexShardStats shard = indicesStatsResponse.getIndex(sourceIndex).getIndexShards().get(i);
+                        return shard == null ? null : shard.getPrimary().getDocs();
+                    }, indexNameExpressionResolver);
                 createIndexService.createIndex(updateRequest, new ActionListener<ClusterStateUpdateResponse>() {
                     @Override
                     public void onResponse(ClusterStateUpdateResponse response) {
@@ -127,24 +121,36 @@ public class TransportShrinkAction extends TransportMasterNodeAction<ShrinkReque
 
     // static for unittesting this method
     static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final ShrinkRequest shrinkReqeust, final ClusterState state
-        , final DocsStats docsStats, IndexNameExpressionResolver indexNameExpressionResolver) {
+        , final IntFunction<DocsStats> perShardDocStats, IndexNameExpressionResolver indexNameExpressionResolver) {
         final String sourceIndex = indexNameExpressionResolver.resolveDateMathExpression(shrinkReqeust.getSourceIndex());
-        final CreateIndexRequest targetIndex = shrinkReqeust.getShrinkIndexReqeust();
+        final CreateIndexRequest targetIndex = shrinkReqeust.getShrinkIndexRequest();
         final String targetIndexName = indexNameExpressionResolver.resolveDateMathExpression(targetIndex.index());
         final IndexMetaData metaData = state.metaData().index(sourceIndex);
         final Settings targetIndexSettings = Settings.builder().put(targetIndex.settings())
             .normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX).build();
-        long count = docsStats.getCount();
-        if (count >= IndexWriter.MAX_DOCS) {
-            throw new IllegalStateException("Can't merge index with more than [" + IndexWriter.MAX_DOCS
-                + "] docs -  too many documents");
+        int numShards = 1;
+        if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) {
+            numShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings);
+        }
+        for (int i = 0; i < numShards; i++) {
+            Set<ShardId> shardIds = IndexMetaData.selectShrinkShards(i, metaData, numShards);
+            long count = 0;
+            for (ShardId id : shardIds) {
+                DocsStats docsStats = perShardDocStats.apply(id.id());
+                if (docsStats != null) {
+                    count += docsStats.getCount();
+                }
+                if (count > IndexWriter.MAX_DOCS) {
+                    throw new IllegalStateException("Can't merge index with more than [" + IndexWriter.MAX_DOCS
+                        + "] docs - too many documents in shards " + shardIds);
+                }
+            }
+
         }
         targetIndex.cause("shrink_index");
-        targetIndex.settings(Settings.builder()
-            .put(targetIndexSettings)
-            // we can only shrink to 1 index so far!
-            .put("index.number_of_shards", 1)
-        );
+        Settings.Builder settingsBuilder = Settings.builder().put(targetIndexSettings);
+        settingsBuilder.put("index.number_of_shards", numShards);
+        targetIndex.settings(settingsBuilder);
 
         return new CreateIndexClusterStateUpdateRequest(targetIndex,
             "shrink_index", targetIndexName, true)
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
index a87c887a4a1..89d8f704c77 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
@@ -30,10 +30,8 @@ import org.elasticsearch.cluster.Diffable;
 import org.elasticsearch.cluster.DiffableUtils;
 import org.elasticsearch.cluster.block.ClusterBlock;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeFilters;
 import org.elasticsearch.cluster.routing.RoutingTable;
-import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseFieldMatcher;
@@ -56,6 +54,7 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.gateway.MetaDataStateFormat;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
@@ -226,6 +225,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
 
     public static final String KEY_ACTIVE_ALLOCATIONS = "active_allocations";
     static final String KEY_VERSION = "version";
+    static final String KEY_ROUTING_NUM_SHARDS = "routing_num_shards";
     static final String KEY_SETTINGS = "settings";
     static final String KEY_STATE = "state";
     static final String KEY_MAPPINGS = "mappings";
@@ -233,6 +233,8 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
     public static final String KEY_PRIMARY_TERMS = "primary_terms";
 
     public static final String INDEX_STATE_FILE_PREFIX = "state-";
+    private final int routingNumShards;
+    private final int routingFactor;
 
     private final int numberOfShards;
     private final int numberOfReplicas;
@@ -268,7 +270,8 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
                           ImmutableOpenMap<String, MappingMetaData> mappings, ImmutableOpenMap<String, AliasMetaData> aliases,
                           ImmutableOpenMap<String, Custom> customs, ImmutableOpenIntMap<Set<String>> activeAllocationIds,
                           DiscoveryNodeFilters requireFilters, DiscoveryNodeFilters initialRecoveryFilters, DiscoveryNodeFilters includeFilters, DiscoveryNodeFilters excludeFilters,
-                          Version indexCreatedVersion, Version indexUpgradedVersion, org.apache.lucene.util.Version minimumCompatibleLuceneVersion) {
+                          Version indexCreatedVersion, Version indexUpgradedVersion, org.apache.lucene.util.Version minimumCompatibleLuceneVersion,
+                          int routingNumShards) {
 
         this.index = index;
         this.version = version;
@@ -290,6 +293,9 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         this.indexCreatedVersion = indexCreatedVersion;
         this.indexUpgradedVersion = indexUpgradedVersion;
         this.minimumCompatibleLuceneVersion = minimumCompatibleLuceneVersion;
+        this.routingNumShards = routingNumShards;
+        this.routingFactor = routingNumShards / numberOfShards;
+        assert numberOfShards * routingFactor == routingNumShards :  routingNumShards + " must be a multiple of " + numberOfShards;
     }
 
     public Index getIndex() {
@@ -484,7 +490,12 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         if (!customs.equals(that.customs)) {
             return false;
         }
-
+        if (routingNumShards != that.routingNumShards) {
+            return false;
+        }
+        if (routingFactor != that.routingFactor) {
+            return false;
+        }
         if (Arrays.equals(primaryTerms, that.primaryTerms) == false) {
             return false;
         }
@@ -503,6 +514,8 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         result = 31 * result + settings.hashCode();
         result = 31 * result + mappings.hashCode();
         result = 31 * result + customs.hashCode();
+        result = 31 * result + Long.hashCode(routingFactor);
+        result = 31 * result + Long.hashCode(routingNumShards);
         result = 31 * result + Arrays.hashCode(primaryTerms);
         result = 31 * result + activeAllocationIds.hashCode();
         return result;
@@ -533,6 +546,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
     private static class IndexMetaDataDiff implements Diff<IndexMetaData> {
 
         private final String index;
+        private final int routingNumShards;
         private final long version;
         private final long[] primaryTerms;
         private final State state;
@@ -545,6 +559,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         public IndexMetaDataDiff(IndexMetaData before, IndexMetaData after) {
             index = after.index.getName();
             version = after.version;
+            routingNumShards = after.routingNumShards;
             state = after.state;
             settings = after.settings;
             primaryTerms = after.primaryTerms;
@@ -557,6 +572,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
 
         public IndexMetaDataDiff(StreamInput in) throws IOException {
             index = in.readString();
+            routingNumShards = in.readInt();
             version = in.readLong();
             state = State.fromId(in.readByte());
             settings = Settings.readSettingsFromStream(in);
@@ -582,6 +598,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeString(index);
+            out.writeInt(routingNumShards);
             out.writeLong(version);
             out.writeByte(state.id);
             Settings.writeSettingsToStream(settings, out);
@@ -596,6 +613,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         public IndexMetaData apply(IndexMetaData part) {
             Builder builder = builder(index);
             builder.version(version);
+            builder.setRoutingNumShards(routingNumShards);
             builder.state(state);
             builder.settings(settings);
             builder.primaryTerms(primaryTerms);
@@ -611,6 +629,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
     public IndexMetaData readFrom(StreamInput in) throws IOException {
         Builder builder = new Builder(in.readString());
         builder.version(in.readLong());
+        builder.setRoutingNumShards(in.readInt());
         builder.state(State.fromId(in.readByte()));
         builder.settings(readSettingsFromStream(in));
         builder.primaryTerms(in.readVLongArray());
@@ -643,6 +662,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(index.getName()); // uuid will come as part of settings
         out.writeLong(version);
+        out.writeInt(routingNumShards);
         out.writeByte(state.id());
         writeSettingsToStream(settings, out);
         out.writeVLongArray(primaryTerms);
@@ -685,6 +705,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
         private final ImmutableOpenMap.Builder<String, AliasMetaData> aliases;
         private final ImmutableOpenMap.Builder<String, Custom> customs;
         private final ImmutableOpenIntMap.Builder<Set<String>> activeAllocationIds;
+        private Integer routingNumShards;
 
         public Builder(String index) {
             this.index = index;
@@ -703,6 +724,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
             this.mappings = ImmutableOpenMap.builder(indexMetaData.mappings);
             this.aliases = ImmutableOpenMap.builder(indexMetaData.aliases);
             this.customs = ImmutableOpenMap.builder(indexMetaData.customs);
+            this.routingNumShards = indexMetaData.routingNumShards;
             this.activeAllocationIds = ImmutableOpenIntMap.builder(indexMetaData.activeAllocationIds);
         }
 
@@ -720,6 +742,26 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
             return this;
         }
 
+        /**
+         * Sets the number of shards that should be used for routing. This should only be used if the number of shards in
+         * an index has changed ie if the index is shrunk.
+         */
+        public Builder setRoutingNumShards(int routingNumShards) {
+            this.routingNumShards = routingNumShards;
+            return this;
+        }
+
+        /**
+         * Returns number of shards that should be used for routing. By default this method will return the number of shards
+         * for this index.
+         *
+         * @see #setRoutingNumShards(int)
+         * @see #numberOfShards()
+         */
+        public int getRoutingNumShards() {
+            return routingNumShards == null ? numberOfShards() : routingNumShards;
+        }
+
         public int numberOfShards() {
             return settings.getAsInt(SETTING_NUMBER_OF_SHARDS, -1);
         }
@@ -934,13 +976,14 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
             final String uuid = settings.get(SETTING_INDEX_UUID, INDEX_UUID_NA_VALUE);
             return new IndexMetaData(new Index(index, uuid), version, primaryTerms, state, numberOfShards, numberOfReplicas, tmpSettings, mappings.build(),
                 tmpAliases.build(), customs.build(), filledActiveAllocationIds.build(), requireFilters, initialRecoveryFilters, includeFilters, excludeFilters,
-                indexCreatedVersion, indexUpgradedVersion, minimumCompatibleLuceneVersion);
+                indexCreatedVersion, indexUpgradedVersion, minimumCompatibleLuceneVersion, getRoutingNumShards());
         }
 
         public static void toXContent(IndexMetaData indexMetaData, XContentBuilder builder, ToXContent.Params params) throws IOException {
             builder.startObject(indexMetaData.getIndex().getName());
 
             builder.field(KEY_VERSION, indexMetaData.getVersion());
+            builder.field(KEY_ROUTING_NUM_SHARDS, indexMetaData.getRoutingNumShards());
             builder.field(KEY_STATE, indexMetaData.getState().toString().toLowerCase(Locale.ENGLISH));
 
             boolean binary = params.paramAsBoolean("binary", false);
@@ -1101,6 +1144,8 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
                         builder.state(State.fromString(parser.text()));
                     } else if (KEY_VERSION.equals(currentFieldName)) {
                         builder.version(parser.longValue());
+                    } else if (KEY_ROUTING_NUM_SHARDS.equals(currentFieldName)) {
+                        builder.setRoutingNumShards(parser.intValue());
                     } else {
                         throw new IllegalArgumentException("Unexpected field [" + currentFieldName + "]");
                     }
@@ -1175,4 +1220,68 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
             return Builder.fromXContent(parser);
         }
     };
+
+    /**
+     * Returns the number of shards that should be used for routing. This basically defines the hash space we use in
+     * {@link org.elasticsearch.cluster.routing.OperationRouting#generateShardId(IndexMetaData, String, String)} to route documents
+     * to shards based on their ID or their specific routing value. The default value is {@link #getNumberOfShards()}. This value only
+     * changes if and index is shrunk.
+     */
+    public int getRoutingNumShards() {
+        return routingNumShards;
+    }
+
+    /**
+     * Returns the routing factor for this index. The default is <tt>1</tt>.
+     *
+     * @see #getRoutingFactor(IndexMetaData, int) for details
+     */
+    public int getRoutingFactor() {
+        return routingFactor;
+    }
+
+    /**
+     * Returns the source shard ids to shrink into the given shard id.
+     * @param shardId the id of the target shard to shrink to
+     * @param sourceIndexMetadata the source index metadata
+     * @param numTargetShards the total number of shards in the target index
+     * @return a set of shard IDs to shrink into the given shard ID.
+     */
+    public static Set<ShardId> selectShrinkShards(int shardId, IndexMetaData sourceIndexMetadata, int numTargetShards) {
+        if (shardId >= numTargetShards) {
+            throw new IllegalArgumentException("the number of target shards (" + numTargetShards + ") must be greater than the shard id: "
+                + shardId);
+        }
+        int routingFactor = getRoutingFactor(sourceIndexMetadata, numTargetShards);
+        Set<ShardId> shards = new HashSet<>(routingFactor);
+        for (int i = shardId * routingFactor; i < routingFactor*shardId + routingFactor; i++) {
+            shards.add(new ShardId(sourceIndexMetadata.getIndex(), i));
+        }
+        return shards;
+    }
+
+    /**
+     * Returns the routing factor for and shrunk index with the given number of target shards.
+     * This factor is used in the hash function in
+     * {@link org.elasticsearch.cluster.routing.OperationRouting#generateShardId(IndexMetaData, String, String)} to guarantee consistent
+     * hashing / routing of documents even if the number of shards changed (ie. a shrunk index).
+     *
+     * @param sourceIndexMetadata the metadata of the source index
+     * @param targetNumberOfShards the total number of shards in the target index
+     * @return the routing factor for and shrunk index with the given number of target shards.
+     * @throw IllegalArgumentException if the number of source shards is greater than the number of target shards or if the source shards
+     * are not divisible by the number of target shards.
+     */
+    public static int getRoutingFactor(IndexMetaData sourceIndexMetadata, int targetNumberOfShards) {
+        int sourceNumberOfShards = sourceIndexMetadata.getNumberOfShards();
+        if (sourceNumberOfShards < targetNumberOfShards) {
+            throw new IllegalArgumentException("the number of target shards must be less that the number of source shards");
+        }
+        int factor = sourceNumberOfShards / targetNumberOfShards;
+        if (factor * targetNumberOfShards != sourceNumberOfShards || factor <= 1) {
+            throw new IllegalArgumentException("the number of source shards [" + sourceNumberOfShards + "] must be a must be a multiple of ["
+                + targetNumberOfShards + "]");
+        }
+        return factor;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
index 47528428928..2b482635879 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
@@ -21,7 +21,6 @@ package org.elasticsearch.cluster.metadata;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
-import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
@@ -65,7 +64,6 @@ import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.indices.IndexCreationException;
 import org.elasticsearch.indices.IndicesService;
@@ -299,15 +297,19 @@ public class MetaDataCreateIndexService extends AbstractComponent {
 
                             indexSettingsBuilder.put(SETTING_INDEX_UUID, UUIDs.randomBase64UUID());
                             final Index shrinkFromIndex = request.shrinkFrom();
+                            int routingNumShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(indexSettingsBuilder.build());;
                             if (shrinkFromIndex != null) {
                                 prepareShrinkIndexSettings(currentState, mappings.keySet(), indexSettingsBuilder, shrinkFromIndex,
                                     request.index());
+                                IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(shrinkFromIndex);
+                                routingNumShards = sourceMetaData.getRoutingNumShards();
                             }
 
                             Settings actualIndexSettings = indexSettingsBuilder.build();
-
+                            IndexMetaData.Builder tmpImdBuilder = IndexMetaData.builder(request.index())
+                                .setRoutingNumShards(routingNumShards);
                             // Set up everything, now locally create the index to see that things are ok, and apply
-                            final IndexMetaData tmpImd = IndexMetaData.builder(request.index()).settings(actualIndexSettings).build();
+                            final IndexMetaData tmpImd = tmpImdBuilder.settings(actualIndexSettings).build();
                             // create the index here (on the master) to validate it can be created, as well as adding the mapping
                             final IndexService indexService = indicesService.createIndex(nodeServicesProvider, tmpImd, Collections.emptyList());
                             createdIndex = indexService.index();
@@ -339,7 +341,9 @@ public class MetaDataCreateIndexService extends AbstractComponent {
                                 mappingsMetaData.put(mapper.type(), mappingMd);
                             }
 
-                            final IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder(request.index()).settings(actualIndexSettings);
+                            final IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder(request.index())
+                                .settings(actualIndexSettings)
+                                .setRoutingNumShards(routingNumShards);
                             for (MappingMetaData mappingMd : mappingsMetaData.values()) {
                                 indexMetaDataBuilder.putMapping(mappingMd);
                             }
@@ -494,14 +498,16 @@ public class MetaDataCreateIndexService extends AbstractComponent {
             throw new IllegalArgumentException("can't shrink an index with only one shard");
         }
 
+
         if ((targetIndexMappingsTypes.size() > 1 ||
             (targetIndexMappingsTypes.isEmpty() || targetIndexMappingsTypes.contains(MapperService.DEFAULT_MAPPING)) == false)) {
             throw new IllegalArgumentException("mappings are not allowed when shrinking indices" +
                 ", all mappings are copied from the source index");
         }
-        if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)
-            && IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings) > 1) {
-            throw new IllegalArgumentException("can not shrink index into more than one shard");
+        if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) {
+            // this method applies all necessary checks ie. if the target shards are less than the source shards
+            // of if the source shards are divisible by the number of target shards
+            IndexMetaData.getRoutingFactor(sourceMetaData, IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings));
         }
 
         // now check that index is all on one node
@@ -533,8 +539,6 @@ public class MetaDataCreateIndexService extends AbstractComponent {
         final Predicate<String> analysisSimilarityPredicate = (s) -> s.startsWith("index.similarity.")
             || s.startsWith("index.analysis.");
         indexSettingsBuilder
-            // we can only shrink to 1 shard so far!
-            .put("index.number_of_shards", 1)
             // we use "i.r.a.initial_recovery" rather than "i.r.a.require|include" since we want the replica to allocate right away
             // once we are allocated.
             .put("index.routing.allocation.initial_recovery._id",
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
index 3fb6f55a919..9b1c82e7c06 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
@@ -218,14 +218,16 @@ public class OperationRouting extends AbstractComponent {
         return new ShardId(indexMetaData.getIndex(), generateShardId(indexMetaData, id, routing));
     }
 
-    private int generateShardId(IndexMetaData indexMetaData, String id, @Nullable String routing) {
+    static int generateShardId(IndexMetaData indexMetaData, String id, @Nullable String routing) {
         final int hash;
         if (routing == null) {
             hash = Murmur3HashFunction.hash(id);
         } else {
             hash = Murmur3HashFunction.hash(routing);
         }
-        return Math.floorMod(hash, indexMetaData.getNumberOfShards());
+        // we don't use IMD#getNumberOfShards since the index might have been shrunk such that we need to use the size
+        // of original index to hash documents
+        return Math.floorMod(hash, indexMetaData.getRoutingNumShards()) / indexMetaData.getRoutingFactor();
     }
 
     private void ensureNodeIdExists(DiscoveryNodes nodes, String nodeId) {
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
index f64791d834a..ca341451fed 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
@@ -46,6 +46,7 @@ import org.elasticsearch.common.unit.RatioValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.shard.ShardId;
 
 import java.util.Set;
 
@@ -653,10 +654,14 @@ public class DiskThresholdDecider extends AllocationDecider {
         if (metaData.getMergeSourceIndex() != null && shard.allocatedPostIndexCreate(metaData) == false) {
             // in the shrink index case we sum up the source index shards since we basically make a copy of the shard in
             // the worst case
-            Index mergeSourceIndex = metaData.getMergeSourceIndex();
             long targetShardSize = 0;
+            final Index mergeSourceIndex = metaData.getMergeSourceIndex();
+            final IndexMetaData sourceIndexMeta = allocation.metaData().getIndexSafe(metaData.getMergeSourceIndex());
+            final Set<ShardId> shardIds = IndexMetaData.selectShrinkShards(shard.id(), sourceIndexMeta, metaData.getNumberOfShards());
             for (IndexShardRoutingTable shardRoutingTable : allocation.routingTable().index(mergeSourceIndex.getName())) {
-                targetShardSize += info.getShardSize(shardRoutingTable.primaryShard(), 0);
+                if (shardIds.contains(shardRoutingTable.shardId())) {
+                    targetShardSize += info.getShardSize(shardRoutingTable.primaryShard(), 0);
+                }
             }
             return targetShardSize == 0 ? defaultValue : targetShardSize;
         } else {
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 33d1aec7ed0..105d1e5b057 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -126,6 +126,7 @@ import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -133,6 +134,7 @@ import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
 
 public class IndexShard extends AbstractIndexShardComponent {
 
@@ -1411,7 +1413,10 @@ public class IndexShard extends AbstractIndexShardComponent {
                     markAsRecovering("from local shards", recoveryState); // mark the shard as recovering on the cluster state thread
                     threadPool.generic().execute(() -> {
                         try {
-                            if (recoverFromLocalShards(mappingUpdateConsumer, startedShards)) {
+                            final Set<ShardId> shards = IndexMetaData.selectShrinkShards(shardId().id(), sourceIndexService.getMetaData(),
+                                indexMetaData.getNumberOfShards());
+                            if (recoverFromLocalShards(mappingUpdateConsumer, startedShards.stream()
+                                .filter((s) -> shards.contains(s.shardId())).collect(Collectors.toList()))) {
                                 recoveryListener.onRecoveryDone(recoveryState);
                             }
                         } catch (Throwable t) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java
index b5cc861b65e..42117cbf328 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java
@@ -52,7 +52,7 @@ public class RestShrinkIndexAction extends BaseRestHandler {
         }
         ShrinkRequest shrinkIndexRequest = new ShrinkRequest(request.param("target"), request.param("index"));
         if (request.hasContent()) {
-            shrinkIndexRequest.getShrinkIndexReqeust().source(request.content());
+            shrinkIndexRequest.getShrinkIndexRequest().source(request.content());
         }
         shrinkIndexRequest.timeout(request.paramAsTime("timeout", shrinkIndexRequest.timeout()));
         shrinkIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", shrinkIndexRequest.masterNodeTimeout()));
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
index 60c4d0f486e..8fbb489d9c2 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
@@ -289,6 +289,76 @@ public class CreateIndexIT extends ESIntegTestCase {
         ensureGreen("test");
     }
 
+    public void testCreateShrinkIndexToN() {
+        int[][] possibleShardSplits = new int[][] {{8,4,2}, {9, 3, 1}, {4, 2, 1}, {15,5,1}};
+        int[] shardSplits = randomFrom(possibleShardSplits);
+        assertEquals(shardSplits[0], (shardSplits[0] / shardSplits[1]) * shardSplits[1]);
+        assertEquals(shardSplits[1], (shardSplits[1] / shardSplits[2]) * shardSplits[2]);
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        prepareCreate("source").setSettings(Settings.builder().put(indexSettings()).put("number_of_shards", shardSplits[0])).get();
+        for (int i = 0; i < 20; i++) {
+            client().prepareIndex("source", "t1", Integer.toString(i)).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get();
+        }
+        ImmutableOpenMap<String, DiscoveryNode> dataNodes = client().admin().cluster().prepareState().get().getState().nodes()
+            .getDataNodes();
+        assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2);
+        DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class);
+        String mergeNode = discoveryNodes[0].getName();
+        // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
+        // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
+        // to the require._name below.
+        ensureGreen();
+        // relocate all shards to one node such that we can merge it.
+        client().admin().indices().prepareUpdateSettings("source")
+            .setSettings(Settings.builder()
+                .put("index.routing.allocation.require._name", mergeNode)
+                .put("index.blocks.write", true)).get();
+        ensureGreen();
+        // now merge source into a 4 shard index
+        assertAcked(client().admin().indices().prepareShrinkIndex("source", "first_shrink")
+            .setSettings(Settings.builder()
+                .put("index.number_of_replicas", 0)
+                .put("index.number_of_shards", shardSplits[1]).build()).get());
+        ensureGreen();
+        assertHitCount(client().prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+
+        for (int i = 0; i < 20; i++) { // now update
+            client().prepareIndex("first_shrink", "t1", Integer.toString(i)).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get();
+        }
+        flushAndRefresh();
+        assertHitCount(client().prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+        assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+
+        // relocate all shards to one node such that we can merge it.
+        client().admin().indices().prepareUpdateSettings("first_shrink")
+            .setSettings(Settings.builder()
+                .put("index.routing.allocation.require._name", mergeNode)
+                .put("index.blocks.write", true)).get();
+        ensureGreen();
+        // now merge source into a 2 shard index
+        assertAcked(client().admin().indices().prepareShrinkIndex("first_shrink", "second_shrink")
+            .setSettings(Settings.builder()
+                .put("index.number_of_replicas", 0)
+                .put("index.number_of_shards", shardSplits[2]).build()).get());
+        ensureGreen();
+        assertHitCount(client().prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+        // let it be allocated anywhere and bump replicas
+        client().admin().indices().prepareUpdateSettings("second_shrink")
+            .setSettings(Settings.builder()
+                .putNull("index.routing.allocation.include._id")
+                .put("index.number_of_replicas", 1)).get();
+        ensureGreen();
+        assertHitCount(client().prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+
+        for (int i = 0; i < 20; i++) { // now update
+            client().prepareIndex("second_shrink", "t1", Integer.toString(i)).setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}").get();
+        }
+        flushAndRefresh();
+        assertHitCount(client().prepareSearch("second_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+        assertHitCount(client().prepareSearch("first_shrink").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+        assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), 20);
+    }
+
     public void testCreateShrinkIndex() {
         internalCluster().ensureAtLeastNumDataNodes(2);
         prepareCreate("source").setSettings(Settings.builder().put(indexSettings()).put("number_of_shards", randomIntBetween(2, 7))).get();
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java
index 50bf8715f19..01be8da8944 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java
@@ -28,7 +28,6 @@ import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
@@ -39,9 +38,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.DummyTransportAddress;
-import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.shard.DocsStats;
-import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.gateway.NoopGatewayAllocator;
 
@@ -70,15 +67,26 @@ public class TransportShrinkActionTests extends ESTestCase {
     }
 
     public void testErrorCondition() {
-        ClusterState state = createClusterState("source", randomIntBetween(2, 100), randomIntBetween(0, 10),
+        ClusterState state = createClusterState("source", randomIntBetween(2, 42), randomIntBetween(0, 10),
             Settings.builder().put("index.blocks.write", true).build());
-        DocsStats stats = new DocsStats(randomIntBetween(0, IndexWriter.MAX_DOCS-1), randomIntBetween(1, 1000));
-
-        assertEquals("Can't merge index with more than [2147483519] docs -  too many documents",
+        assertTrue(
             expectThrows(IllegalStateException.class, () ->
             TransportShrinkAction.prepareCreateIndexRequest(new ShrinkRequest("target", "source"), state,
-                new DocsStats(Integer.MAX_VALUE, randomIntBetween(1, 1000)), new IndexNameExpressionResolver(Settings.EMPTY))
-        ).getMessage());
+                (i) -> new DocsStats(Integer.MAX_VALUE, randomIntBetween(1, 1000)), new IndexNameExpressionResolver(Settings.EMPTY))
+        ).getMessage().startsWith("Can't merge index with more than [2147483519] docs - too many documents in shards "));
+
+
+        assertTrue(
+            expectThrows(IllegalStateException.class, () -> {
+                ShrinkRequest req = new ShrinkRequest("target", "source");
+                req.getShrinkIndexRequest().settings(Settings.builder().put("index.number_of_shards", 4));
+                ClusterState clusterState = createClusterState("source", 8, 1,
+                    Settings.builder().put("index.blocks.write", true).build());
+                    TransportShrinkAction.prepareCreateIndexRequest(req, clusterState,
+                        (i) -> i == 2 || i == 3 ? new DocsStats(Integer.MAX_VALUE/2, randomIntBetween(1, 1000)) : null,
+                        new IndexNameExpressionResolver(Settings.EMPTY));
+                }
+            ).getMessage().startsWith("Can't merge index with more than [2147483519] docs - too many documents in shards "));
 
 
         // create one that won't fail
@@ -96,8 +104,8 @@ public class TransportShrinkActionTests extends ESTestCase {
             routingTable.index("source").shardsWithState(ShardRoutingState.INITIALIZING)).routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
 
-        TransportShrinkAction.prepareCreateIndexRequest(new ShrinkRequest("target", "source"), clusterState, stats,
-            new IndexNameExpressionResolver(Settings.EMPTY));
+        TransportShrinkAction.prepareCreateIndexRequest(new ShrinkRequest("target", "source"), clusterState,
+            (i) -> new DocsStats(randomIntBetween(1, 1000), randomIntBetween(1, 1000)), new IndexNameExpressionResolver(Settings.EMPTY));
     }
 
     public void testShrinkIndexSettings() {
@@ -118,11 +126,12 @@ public class TransportShrinkActionTests extends ESTestCase {
         routingTable = service.applyStartedShards(clusterState,
             routingTable.index(indexName).shardsWithState(ShardRoutingState.INITIALIZING)).routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
-
-        DocsStats stats = new DocsStats(randomIntBetween(0, IndexWriter.MAX_DOCS-1), randomIntBetween(1, 1000));
+        int numSourceShards = clusterState.metaData().index(indexName).getNumberOfShards();
+        DocsStats stats = new DocsStats(randomIntBetween(0, (IndexWriter.MAX_DOCS) / numSourceShards), randomIntBetween(1, 1000));
         ShrinkRequest target = new ShrinkRequest("target", indexName);
         CreateIndexClusterStateUpdateRequest request = TransportShrinkAction.prepareCreateIndexRequest(
-            target, clusterState, stats, new IndexNameExpressionResolver(Settings.EMPTY));
+            target, clusterState, (i) -> stats,
+            new IndexNameExpressionResolver(Settings.EMPTY));
         assertNotNull(request.shrinkFrom());
         assertEquals(indexName, request.shrinkFrom().getName());
         assertEquals("1", request.settings().get("index.number_of_shards"));
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java
new file mode 100644
index 00000000000..0c9827587ea
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cluster.metadata;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Set;
+
+public class IndexMetaDataTests extends ESTestCase {
+
+    public void testIndexMetaDataSerialization() throws IOException {
+        Integer numShard = randomFrom(1, 2, 4, 8, 16);
+        int numberOfReplicas = randomIntBetween(0, 10);
+        IndexMetaData metaData = IndexMetaData.builder("foo")
+            .settings(Settings.builder()
+                .put("index.version.created", 1)
+                .put("index.number_of_shards", numShard)
+                .put("index.number_of_replicas", numberOfReplicas)
+                .build())
+            .creationDate(randomLong())
+            .primaryTerm(0, 2)
+            .setRoutingNumShards(32)
+            .build();
+
+        final XContentBuilder builder = JsonXContent.contentBuilder();
+        builder.startObject();
+        metaData.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+        XContentParser parser = XContentType.JSON.xContent().createParser(builder.bytes());
+        final IndexMetaData fromXContentMeta = IndexMetaData.PROTO.fromXContent(parser, null);
+        assertEquals(metaData, fromXContentMeta);
+        assertEquals(metaData.hashCode(), fromXContentMeta.hashCode());
+
+        assertEquals(metaData.getNumberOfReplicas(), fromXContentMeta.getNumberOfReplicas());
+        assertEquals(metaData.getNumberOfShards(), fromXContentMeta.getNumberOfShards());
+        assertEquals(metaData.getCreationVersion(), fromXContentMeta.getCreationVersion());
+        assertEquals(metaData.getRoutingNumShards(), fromXContentMeta.getRoutingNumShards());
+        assertEquals(metaData.getCreationDate(), fromXContentMeta.getCreationDate());
+        assertEquals(metaData.getRoutingFactor(), fromXContentMeta.getRoutingFactor());
+        assertEquals(metaData.primaryTerm(0), fromXContentMeta.primaryTerm(0));
+
+        final BytesStreamOutput out = new BytesStreamOutput();
+        metaData.writeTo(out);
+        IndexMetaData deserialized = IndexMetaData.PROTO.readFrom(StreamInput.wrap(out.bytes()));
+        assertEquals(metaData, deserialized);
+        assertEquals(metaData.hashCode(), deserialized.hashCode());
+
+        assertEquals(metaData.getNumberOfReplicas(), deserialized.getNumberOfReplicas());
+        assertEquals(metaData.getNumberOfShards(), deserialized.getNumberOfShards());
+        assertEquals(metaData.getCreationVersion(), deserialized.getCreationVersion());
+        assertEquals(metaData.getRoutingNumShards(), deserialized.getRoutingNumShards());
+        assertEquals(metaData.getCreationDate(), deserialized.getCreationDate());
+        assertEquals(metaData.getRoutingFactor(), deserialized.getRoutingFactor());
+        assertEquals(metaData.primaryTerm(0), deserialized.primaryTerm(0));
+    }
+
+    public void testGetRoutingFactor() {
+        int numberOfReplicas = randomIntBetween(0, 10);
+        IndexMetaData metaData = IndexMetaData.builder("foo")
+            .settings(Settings.builder()
+                .put("index.version.created", 1)
+                .put("index.number_of_shards", 32)
+                .put("index.number_of_replicas", numberOfReplicas)
+                .build())
+            .creationDate(randomLong())
+            .build();
+        Integer numShard = randomFrom(1, 2, 4, 8, 16);
+        int routingFactor = IndexMetaData.getRoutingFactor(metaData, numShard);
+        assertEquals(routingFactor * numShard, metaData.getNumberOfShards());
+
+        Integer brokenNumShards = randomFrom(3, 5, 9, 12, 29, 42, 64);
+        expectThrows(IllegalArgumentException.class, () -> IndexMetaData.getRoutingFactor(metaData, brokenNumShards));
+    }
+
+    public void testSelectShrinkShards() {
+        int numberOfReplicas = randomIntBetween(0, 10);
+        IndexMetaData metaData = IndexMetaData.builder("foo")
+            .settings(Settings.builder()
+                .put("index.version.created", 1)
+                .put("index.number_of_shards", 32)
+                .put("index.number_of_replicas", numberOfReplicas)
+                .build())
+            .creationDate(randomLong())
+            .build();
+        Set<ShardId> shardIds = IndexMetaData.selectShrinkShards(0, metaData, 8);
+        assertEquals(shardIds, Sets.newHashSet(new ShardId(metaData.getIndex(), 0), new ShardId(metaData.getIndex(), 1),
+            new ShardId(metaData.getIndex(), 2), new ShardId(metaData.getIndex(), 3)));
+        shardIds = IndexMetaData.selectShrinkShards(1, metaData, 8);
+        assertEquals(shardIds, Sets.newHashSet(new ShardId(metaData.getIndex(), 4), new ShardId(metaData.getIndex(), 5),
+            new ShardId(metaData.getIndex(), 6), new ShardId(metaData.getIndex(), 7)));
+        shardIds = IndexMetaData.selectShrinkShards(7, metaData, 8);
+        assertEquals(shardIds, Sets.newHashSet(new ShardId(metaData.getIndex(), 28), new ShardId(metaData.getIndex(), 29),
+            new ShardId(metaData.getIndex(), 30), new ShardId(metaData.getIndex(), 31)));
+
+        assertEquals("the number of target shards (8) must be greater than the shard id: 8",
+            expectThrows(IllegalArgumentException.class, () -> IndexMetaData.selectShrinkShards(8, metaData, 8)).getMessage());
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
index aa226e311dd..53dffc799b1 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
@@ -66,8 +66,15 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
         return clusterState;
     }
 
+    public static boolean isShrinkable(int source, int target) {
+        int x = source / target;
+        assert source > target : source  + " <= " + target;
+        return target * x == source;
+    }
+
     public void testValidateShrinkIndex() {
-        ClusterState state = createClusterState("source", randomIntBetween(2, 100), randomIntBetween(0, 10),
+        int numShards = randomIntBetween(2, 42);
+        ClusterState state = createClusterState("source", numShards, randomIntBetween(0, 10),
             Settings.builder().put("index.blocks.write", true).build());
 
         assertEquals("index [source] already exists",
@@ -81,12 +88,18 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
             ).getMessage());
 
         assertEquals("can't shrink an index with only one shard",
-            expectThrows(IllegalArgumentException.class, () ->
-                    MetaDataCreateIndexService.validateShrinkIndex(createClusterState("source", 1, 0,
-                        Settings.builder().put("index.blocks.write", true).build()), "source", Collections.emptySet(),
+            expectThrows(IllegalArgumentException.class, () -> MetaDataCreateIndexService.validateShrinkIndex(createClusterState("source",
+                1, 0, Settings.builder().put("index.blocks.write", true).build()), "source", Collections.emptySet(),
                         "target", Settings.EMPTY)
             ).getMessage());
 
+        assertEquals("the number of target shards must be less that the number of source shards",
+            expectThrows(IllegalArgumentException.class, () -> MetaDataCreateIndexService.validateShrinkIndex(createClusterState("source",
+                5, 0, Settings.builder().put("index.blocks.write", true).build()), "source", Collections.emptySet(),
+                "target", Settings.builder().put("index.number_of_shards", 10).build())
+            ).getMessage());
+
+
         assertEquals("index source must be read-only to shrink index. use \"index.blocks.write=true\"",
             expectThrows(IllegalStateException.class, () ->
                     MetaDataCreateIndexService.validateShrinkIndex(
@@ -99,11 +112,11 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
                 MetaDataCreateIndexService.validateShrinkIndex(state, "source", Collections.emptySet(), "target", Settings.EMPTY)
 
             ).getMessage());
-
-        assertEquals("can not shrink index into more than one shard",
+        assertEquals("the number of source shards [8] must be a must be a multiple of [3]",
             expectThrows(IllegalArgumentException.class, () ->
-                MetaDataCreateIndexService.validateShrinkIndex(state, "source", Collections.emptySet(), "target",
-                    Settings.builder().put("index.number_of_shards", 2).build())
+                    MetaDataCreateIndexService.validateShrinkIndex(createClusterState("source", 8, randomIntBetween(0, 10),
+                        Settings.builder().put("index.blocks.write", true).build()), "source", Collections.emptySet(), "target",
+                        Settings.builder().put("index.number_of_shards", 3).build())
             ).getMessage());
 
         assertEquals("mappings are not allowed when shrinking indices, all mappings are copied from the source index",
@@ -114,7 +127,7 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
             ).getMessage());
 
         // create one that won't fail
-        ClusterState clusterState = ClusterState.builder(createClusterState("source", randomIntBetween(2, 10), 0,
+        ClusterState clusterState = ClusterState.builder(createClusterState("source", numShards, 0,
             Settings.builder().put("index.blocks.write", true).build())).nodes(DiscoveryNodes.builder().put(newNode("node1")))
             .build();
         AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY,
@@ -127,8 +140,12 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
         routingTable = service.applyStartedShards(clusterState,
             routingTable.index("source").shardsWithState(ShardRoutingState.INITIALIZING)).routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
-
-        MetaDataCreateIndexService.validateShrinkIndex(clusterState, "source", Collections.emptySet(), "target", Settings.EMPTY);
+        int targetShards;
+        do {
+            targetShards = randomIntBetween(1, numShards/2);
+        } while (isShrinkable(numShards, targetShards) == false);
+        MetaDataCreateIndexService.validateShrinkIndex(clusterState, "source", Collections.emptySet(), "target",
+            Settings.builder().put("index.number_of_shards", targetShards).build());
     }
 
     public void testShrinkIndexSettings() {
@@ -155,7 +172,6 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
         Settings.Builder builder = Settings.builder();
         MetaDataCreateIndexService.prepareShrinkIndexSettings(
             clusterState, Collections.emptySet(), builder, clusterState.metaData().index(indexName).getIndex(), "target");
-        assertEquals("1", builder.build().get("index.number_of_shards"));
         assertEquals("similarity settings must be copied", "BM25", builder.build().get("index.similarity.default.type"));
         assertEquals("analysis settings must be copied",
             "keyword", builder.build().get("index.analysis.analyzer.my_analyzer.tokenizer"));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java
new file mode 100644
index 00000000000..e515e0f9bf4
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.cluster.routing;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+public class OperationRoutingTests extends ESTestCase{
+
+    public void testGenerateShardId() {
+        int[][] possibleValues = new int[][] {
+            {8,4,2}, {20, 10, 2}, {36, 12, 3}, {15,5,1}
+        };
+        for (int i = 0; i < 10; i++) {
+            int[] shardSplits = randomFrom(possibleValues);
+            assertEquals(shardSplits[0], (shardSplits[0] / shardSplits[1]) * shardSplits[1]);
+            assertEquals(shardSplits[1], (shardSplits[1] / shardSplits[2]) * shardSplits[2]);
+            IndexMetaData metaData = IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shardSplits[0])
+                .numberOfReplicas(1).build();
+            String term = randomAsciiOfLength(10);
+            final int shard = OperationRouting.generateShardId(metaData, term, null);
+            IndexMetaData shrunk = IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shardSplits[1])
+                .numberOfReplicas(1)
+                .setRoutingNumShards(shardSplits[0]).build();
+            int shrunkShard = OperationRouting.generateShardId(shrunk, term, null);
+            Set<ShardId> shardIds = IndexMetaData.selectShrinkShards(shrunkShard, metaData, shrunk.getNumberOfShards());
+            assertEquals(1, shardIds.stream().filter((sid) -> sid.id() == shard).count());
+
+            shrunk = IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shardSplits[2]).numberOfReplicas(1)
+                .setRoutingNumShards(shardSplits[0]).build();
+            shrunkShard = OperationRouting.generateShardId(shrunk, term, null);
+            shardIds = IndexMetaData.selectShrinkShards(shrunkShard, metaData, shrunk.getNumberOfShards());
+            assertEquals(Arrays.toString(shardSplits), 1, shardIds.stream().filter((sid) -> sid.id() == shard).count());
+        }
+    }
+
+    /**
+     * Ensures that all changes to the hash-function / shard selection are BWC
+     */
+    public void testBWC() {
+        Map<String, Integer> termToShard = new TreeMap<>();
+        termToShard.put("sEERfFzPSI", 1);
+        termToShard.put("cNRiIrjzYd", 7);
+        termToShard.put("BgfLBXUyWT", 5);
+        termToShard.put("cnepjZhQnb", 3);
+        termToShard.put("OKCmuYkeCK", 6);
+        termToShard.put("OutXGRQUja", 5);
+        termToShard.put("yCdyocKWou", 1);
+        termToShard.put("KXuNWWNgVj", 2);
+        termToShard.put("DGJOYrpESx", 4);
+        termToShard.put("upLDybdTGs", 5);
+        termToShard.put("yhZhzCPQby", 1);
+        termToShard.put("EyCVeiCouA", 1);
+        termToShard.put("tFyVdQauWR", 6);
+        termToShard.put("nyeRYDnDQr", 6);
+        termToShard.put("hswhrppvDH", 0);
+        termToShard.put("BSiWvDOsNE", 5);
+        termToShard.put("YHicpFBSaY", 1);
+        termToShard.put("EquPtdKaBZ", 4);
+        termToShard.put("rSjLZHCDfT", 5);
+        termToShard.put("qoZALVcite", 7);
+        termToShard.put("yDCCPVBiCm", 7);
+        termToShard.put("ngizYtQgGK", 5);
+        termToShard.put("FYQRIBcNqz", 0);
+        termToShard.put("EBzEDAPODe", 2);
+        termToShard.put("YePigbXgKb", 1);
+        termToShard.put("PeGJjomyik", 3);
+        termToShard.put("cyQIvDmyYD", 7);
+        termToShard.put("yIEfZrYfRk", 5);
+        termToShard.put("kblouyFUbu", 7);
+        termToShard.put("xvIGbRiGJF", 3);
+        termToShard.put("KWimwsREPf", 4);
+        termToShard.put("wsNavvIcdk", 7);
+        termToShard.put("xkWaPcCmpT", 0);
+        termToShard.put("FKKTOnJMDy", 7);
+        termToShard.put("RuLzobYixn", 2);
+        termToShard.put("mFohLeFRvF", 4);
+        termToShard.put("aAMXnamRJg", 7);
+        termToShard.put("zKBMYJDmBI", 0);
+        termToShard.put("ElSVuJQQuw", 7);
+        termToShard.put("pezPtTQAAm", 7);
+        termToShard.put("zBjjNEjAex", 2);
+        termToShard.put("PGgHcLNPYX", 7);
+        termToShard.put("hOkpeQqTDF", 3);
+        termToShard.put("chZXraUPBH", 7);
+        termToShard.put("FAIcSmmNXq", 5);
+        termToShard.put("EZmDicyayC", 0);
+        termToShard.put("GRIueBeIyL", 7);
+        termToShard.put("qCChjGZYLp", 3);
+        termToShard.put("IsSZQwwnUT", 3);
+        termToShard.put("MGlxLFyyCK", 3);
+        termToShard.put("YmscwrKSpB", 0);
+        termToShard.put("czSljcjMop", 5);
+        termToShard.put("XhfGWwNlng", 1);
+        termToShard.put("cWpKJjlzgj", 7);
+        termToShard.put("eDzIfMKbvk", 1);
+        termToShard.put("WFFWYBfnTb", 0);
+        termToShard.put("oDdHJxGxja", 7);
+        termToShard.put("PDOQQqgIKE", 1);
+        termToShard.put("bGEIEBLATe", 6);
+        termToShard.put("xpRkJPWVpu", 2);
+        termToShard.put("kTwZnPEeIi", 2);
+        termToShard.put("DifcuqSsKk", 1);
+        termToShard.put("CEmLmljpXe", 5);
+        termToShard.put("cuNKtLtyJQ", 7);
+        termToShard.put("yNjiAnxAmt", 5);
+        termToShard.put("bVDJDCeaFm", 2);
+        termToShard.put("vdnUhGLFtl", 0);
+        termToShard.put("LnqSYezXbr", 5);
+        termToShard.put("EzHgydDCSR", 3);
+        termToShard.put("ZSKjhJlcpn", 1);
+        termToShard.put("WRjUoZwtUz", 3);
+        termToShard.put("RiBbcCdIgk", 4);
+        termToShard.put("yizTqyjuDn", 4);
+        termToShard.put("QnFjcpcZUT", 4);
+        termToShard.put("agYhXYUUpl", 7);
+        termToShard.put("UOjiTugjNC", 7);
+        termToShard.put("nICGuWTdfV", 0);
+        termToShard.put("NrnSmcnUVF", 2);
+        termToShard.put("ZSzFcbpDqP", 3);
+        termToShard.put("YOhahLSzzE", 5);
+        termToShard.put("iWswCilUaT", 1);
+        termToShard.put("zXAamKsRwj", 2);
+        termToShard.put("aqGsrUPHFq", 5);
+        termToShard.put("eDItImYWTS", 1);
+        termToShard.put("JAYDZMRcpW", 4);
+        termToShard.put("lmvAaEPflK", 7);
+        termToShard.put("IKuOwPjKCx", 5);
+        termToShard.put("schsINzlYB", 1);
+        termToShard.put("OqbFNxrKrF", 2);
+        termToShard.put("QrklDfvEJU", 6);
+        termToShard.put("VLxKRKdLbx", 4);
+        termToShard.put("imoydNTZhV", 1);
+        termToShard.put("uFZyTyOMRO", 4);
+        termToShard.put("nVAZVMPNNx", 3);
+        termToShard.put("rPIdESYaAO", 5);
+        termToShard.put("nbZWPWJsIM", 0);
+        termToShard.put("wRZXPSoEgd", 3);
+        termToShard.put("nGzpgwsSBc", 4);
+        termToShard.put("AITyyoyLLs", 4);
+        IndexMetaData metaData = IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(8)
+            .numberOfReplicas(1).build();
+        for (Map.Entry<String, Integer> entry : termToShard.entrySet()) {
+            String key = entry.getKey();
+            int shard = randomBoolean() ?
+                OperationRouting.generateShardId(metaData, key, null) : OperationRouting.generateShardId(metaData, "foobar", key);
+            assertEquals(shard, entry.getValue().intValue());
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
index 52ba45ab7ab..aaed025a33e 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
@@ -298,17 +298,22 @@ public class DiskThresholdDeciderUnitTests extends ESAllocationTestCase {
         ImmutableOpenMap.Builder<String, Long> shardSizes = ImmutableOpenMap.builder();
         shardSizes.put("[test][0][p]", 10L);
         shardSizes.put("[test][1][p]", 100L);
-        shardSizes.put("[test][2][p]", 1000L);
+        shardSizes.put("[test][2][p]", 500L);
+        shardSizes.put("[test][3][p]", 500L);
+
         ClusterInfo info = new DevNullClusterInfo(ImmutableOpenMap.of(), ImmutableOpenMap.of(), shardSizes.build());
         MetaData.Builder metaBuilder = MetaData.builder();
         metaBuilder.put(IndexMetaData.builder("test").settings(settings(Version.CURRENT).put("index.uuid", "1234"))
-            .numberOfShards(3).numberOfReplicas(0));
+            .numberOfShards(4).numberOfReplicas(0));
         metaBuilder.put(IndexMetaData.builder("target").settings(settings(Version.CURRENT).put("index.uuid", "5678")
             .put("index.shrink.source.name", "test").put("index.shrink.source.uuid", "1234")).numberOfShards(1).numberOfReplicas(0));
+        metaBuilder.put(IndexMetaData.builder("target2").settings(settings(Version.CURRENT).put("index.uuid", "9101112")
+            .put("index.shrink.source.name", "test").put("index.shrink.source.uuid", "1234")).numberOfShards(2).numberOfReplicas(0));
         MetaData metaData = metaBuilder.build();
         RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
         routingTableBuilder.addAsNew(metaData.index("test"));
         routingTableBuilder.addAsNew(metaData.index("target"));
+        routingTableBuilder.addAsNew(metaData.index("target2"));
         ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
             .metaData(metaData).routingTable(routingTableBuilder.build()).build();
 
@@ -339,18 +344,26 @@ public class DiskThresholdDeciderUnitTests extends ESAllocationTestCase {
             new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         test_2 = ShardRoutingHelper.initialize(test_2, "node1");
 
-        assertEquals(1000L, DiskThresholdDecider.getExpectedShardSize(test_2, allocation, 0));
+        ShardRouting test_3 = ShardRouting.newUnassigned(new ShardId(index, 3), null, true,
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        test_3 = ShardRoutingHelper.initialize(test_3, "node1");
+        assertEquals(500l, DiskThresholdDecider.getExpectedShardSize(test_3, allocation, 0));
+        assertEquals(500L, DiskThresholdDecider.getExpectedShardSize(test_2, allocation, 0));
         assertEquals(100L, DiskThresholdDecider.getExpectedShardSize(test_1, allocation, 0));
         assertEquals(10L, DiskThresholdDecider.getExpectedShardSize(test_0, allocation, 0));
 
 
         ShardRouting target = ShardRouting.newUnassigned(new ShardId(new Index("target", "5678"), 0),
             null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
-
-
         assertEquals(1110L, DiskThresholdDecider.getExpectedShardSize(target, allocation, 0));
 
+        ShardRouting target2 = ShardRouting.newUnassigned(new ShardId(new Index("target2", "9101112"), 0),
+            null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        assertEquals(110L, DiskThresholdDecider.getExpectedShardSize(target2, allocation, 0));
 
+        target2 = ShardRouting.newUnassigned(new ShardId(new Index("target2", "9101112"), 1),
+            null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
+        assertEquals(1000L, DiskThresholdDecider.getExpectedShardSize(target2, allocation, 0));
     }
 
 }
diff --git a/docs/reference/indices/shrink-index.asciidoc b/docs/reference/indices/shrink-index.asciidoc
index d531477e77c..39c5134e23c 100644
--- a/docs/reference/indices/shrink-index.asciidoc
+++ b/docs/reference/indices/shrink-index.asciidoc
@@ -2,13 +2,18 @@
 == Shrink Index
 
 The shrink index API allows you to shrink an existing index into a new index
-with a single primary shard. Before shrinking, a (primary or replica) copy of
-every shard in the index must be present on the same node.
+with fewer primary shards. The number of primary shards in the target index
+must be a factor of the shards in the source index. For example an index with
+`8` primary shards can be shrunk into `4`, `2` or `1` primary shards or an index
+with `15` primary shards can be shrunk into `5`, `3` or `1`. If the number
+of shards in the index is a prime number it can only be shrunk into a single
+primary shard. Before shrinking, a (primary or replica) copy of every shard
+in the index must be present on the same node.
 
 Shrinking works as follows:
 
 * First, it creates a new target index with the same definition as the source
-  index, but with a single primary shard.
+  index, but with a smaller number of primary shards.
 
 * Then it hard-links segments from the source index into the target index. (If
   the file system doesn't support hard-linking, then all segments are copied
@@ -64,15 +69,19 @@ the cluster state -- it doesn't wait for the shrink operation to start.
 [IMPORTANT]
 =====================================
 
-Indices can only be shrunk into a single shard if they satisfy the following requirements:
+Indices can only be shrunk if they satisfy the following requirements:
 
- * the target index must not exist
+* the target index must not exist
 
-* The index must have more than one primary shard.
+* The index must have more primary shards than the target index.
+
+* The number of primary shards in the target index must be a factor of the
+  number of primary shards in the source index. must have more primary shards
+   than the target index.
 
 * The index must not contain more than `2,147,483,519` documents in total
-  across all shards as this is the maximum number of docs that can fit into a
-  single shard.
+  across all shards that will be shrunk into a single shard on the target index
+  as this is the maximum number of docs that can fit into a single shard.
 
 * The node handling the shrink process must have sufficient free disk space to
   accommodate a second copy of the existing index.
@@ -88,7 +97,8 @@ POST my_source_index/_shrink/my_target_index
 {
   "settings": {
     "index.number_of_replicas": 1,
-    "index.codec": "best_compression" <1>
+    "index.number_of_shards": 1, <1>
+    "index.codec": "best_compression" <2>
   },
   "aliases": {
     "my_search_indices": {}
@@ -96,10 +106,13 @@ POST my_source_index/_shrink/my_target_index
 }
 --------------------------------------------------
 
-<1> Best compression will only take affect when new writes are made to the
+<1> The number of shards in the target index. This must be a factor of the
+    number of shards in the source index.
+<2> Best compression will only take affect when new writes are made to the
     index, such as when <<indices-forcemerge,force-merging>> the shard to a single
     segment.
 
+
 NOTE: Mappings may not be specified in the `_shrink` request, and all
 `index.analysis.*` and `index.similarity.*` settings will be overwritten with
 the settings from the source index.

From 0a5e06d40270b68cb0dd6880c69cd6ebe690b8ef Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Tue, 7 Jun 2016 10:22:11 +0200
Subject: [PATCH 0687/1311] fix javadocs

---
 .../java/org/elasticsearch/cluster/metadata/IndexMetaData.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
index 89d8f704c77..14d00d3fc64 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
@@ -1269,7 +1269,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
      * @param sourceIndexMetadata the metadata of the source index
      * @param targetNumberOfShards the total number of shards in the target index
      * @return the routing factor for and shrunk index with the given number of target shards.
-     * @throw IllegalArgumentException if the number of source shards is greater than the number of target shards or if the source shards
+     * @throws IllegalArgumentException if the number of source shards is greater than the number of target shards or if the source shards
      * are not divisible by the number of target shards.
      */
     public static int getRoutingFactor(IndexMetaData sourceIndexMetadata, int targetNumberOfShards) {

From c72ebba5de040e92d5d8b4dda66abc298e31c8d8 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Tue, 7 Jun 2016 11:05:28 +0200
Subject: [PATCH 0688/1311] checkstyle have your upper L

---
 .../allocation/decider/DiskThresholdDeciderUnitTests.java       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
index aaed025a33e..8966659e63c 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
@@ -347,7 +347,7 @@ public class DiskThresholdDeciderUnitTests extends ESAllocationTestCase {
         ShardRouting test_3 = ShardRouting.newUnassigned(new ShardId(index, 3), null, true,
             new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
         test_3 = ShardRoutingHelper.initialize(test_3, "node1");
-        assertEquals(500l, DiskThresholdDecider.getExpectedShardSize(test_3, allocation, 0));
+        assertEquals(500L, DiskThresholdDecider.getExpectedShardSize(test_3, allocation, 0));
         assertEquals(500L, DiskThresholdDecider.getExpectedShardSize(test_2, allocation, 0));
         assertEquals(100L, DiskThresholdDecider.getExpectedShardSize(test_1, allocation, 0));
         assertEquals(10L, DiskThresholdDecider.getExpectedShardSize(test_0, allocation, 0));

From 6913be2ec2150e6b8d75bf075c04d75c2c3b6c72 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Tue, 7 Jun 2016 12:32:59 +0200
Subject: [PATCH 0689/1311] Pass through the lookup given by invokedynamic to
 the LambdaMetaFactory. Without it real lambdas won't work, as their
 implementations are private to script class

---
 .../src/main/java/org/elasticsearch/painless/Def.java  |  7 +++----
 .../java/org/elasticsearch/painless/DefBootstrap.java  | 10 ++++++----
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 1e885c53a30..549a3cfba8b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -219,7 +219,7 @@ public final class Def {
      * @return pointer to matching method to invoke. never returns null.
      * @throws IllegalArgumentException if no matching whitelisted method was found.
      */
-     static MethodHandle lookupMethod(Class<?> receiverClass, String name, Object args[], long recipe) {
+     static MethodHandle lookupMethod(MethodHandles.Lookup lookup, Class<?> receiverClass, String name, Object args[], long recipe) {
          Method method = lookupMethodInternal(receiverClass, name, args.length - 1);
          MethodHandle handle = method.handle;
 
@@ -228,7 +228,7 @@ public final class Def {
              for (int i = 0; i < args.length; i++) {
                  // its a functional reference, replace the argument with an impl
                  if ((recipe & (1L << (i - 1))) != 0) {
-                     filters[i] = lookupReference(method.arguments.get(i - 1).clazz, (String) args[i]);
+                     filters[i] = lookupReference(lookup, method.arguments.get(i - 1).clazz, (String) args[i]);
                  }
              }
              handle = MethodHandles.filterArguments(handle, 0, filters);
@@ -238,10 +238,9 @@ public final class Def {
      }
      
      /** Returns a method handle to an implementation of clazz, given method reference signature */
-     private static MethodHandle lookupReference(Class<?> clazz, String signature) {
+     private static MethodHandle lookupReference(MethodHandles.Lookup lookup, Class<?> clazz, String signature) {
          int separator = signature.indexOf('.');
          FunctionRef ref = new FunctionRef(clazz, signature.substring(0, separator), signature.substring(separator+1));
-         MethodHandles.Lookup lookup = MethodHandles.lookup(); // XXX: no lookuping needed, we should pass this from DefBootstrap!
          final CallSite callSite;
          // XXX: clean all this up to use handles in FunctionRef, deal with ASM in EFunctionRef differently
          MethodType invokedType = MethodType.fromMethodDescriptorString(ref.invokedType.getDescriptor(), Def.class.getClassLoader());
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index 8cf85e01d6d..e140de9000c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -68,13 +68,15 @@ public final class DefBootstrap {
         /** maximum number of types before we go megamorphic */
         static final int MAX_DEPTH = 5;
 
+        private final MethodHandles.Lookup lookup;
         private final String name;
         private final int flavor;
         private final long recipe;
         int depth; // pkg-protected for testing
 
-        PIC(String name, MethodType type, int flavor, long recipe) {
+        PIC(MethodHandles.Lookup lookup, String name, MethodType type, int flavor, long recipe) {
             super(type);
+            this.lookup = lookup;
             this.name = name;
             this.flavor = flavor;
             this.recipe = recipe;
@@ -99,10 +101,10 @@ public final class DefBootstrap {
         /**
          * Does a slow lookup against the whitelist.
          */
-        private static MethodHandle lookup(int flavor, Class<?> clazz, String name, Object[] args, long recipe) {
+        private MethodHandle lookup(int flavor, Class<?> clazz, String name, Object[] args, long recipe) {
             switch(flavor) {
                 case METHOD_CALL:
-                    return Def.lookupMethod(clazz, name, args, recipe);
+                    return Def.lookupMethod(lookup, clazz, name, args, recipe);
                 case LOAD:
                     return Def.lookupGetter(clazz, name);
                 case STORE:
@@ -169,7 +171,7 @@ public final class DefBootstrap {
      * see https://docs.oracle.com/javase/specs/jvms/se7/html/jvms-6.html#jvms-6.5.invokedynamic
      */
     public static CallSite bootstrap(Lookup lookup, String name, MethodType type, int flavor, long recipe) {
-        return new PIC(name, type, flavor, recipe);
+        return new PIC(lookup, name, type, flavor, recipe);
     }
 
 }

From cac111ff7334654b58dc09143b849ff89b2b727f Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Tue, 7 Jun 2016 13:00:36 +0200
Subject: [PATCH 0690/1311] Don't hide LambdaConversionException and behave
 like real javac compiled code when a conversion fails. This works anyways,
 because fallback is allowed to throw any Throwable

---
 .../java/org/elasticsearch/painless/Def.java  | 49 +++++++++----------
 .../elasticsearch/painless/DefBootstrap.java  |  2 +-
 2 files changed, 25 insertions(+), 26 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 549a3cfba8b..c2dd960d159 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -217,9 +217,10 @@ public final class Def {
      * @param args args passed to callsite
      * @param recipe bitset marking functional parameters
      * @return pointer to matching method to invoke. never returns null.
+     * @throws LambdaConversionException if a method reference cannot be converted to an functional interface
      * @throws IllegalArgumentException if no matching whitelisted method was found.
      */
-     static MethodHandle lookupMethod(MethodHandles.Lookup lookup, Class<?> receiverClass, String name, Object args[], long recipe) {
+     static MethodHandle lookupMethod(MethodHandles.Lookup lookup, Class<?> receiverClass, String name, Object args[], long recipe) throws LambdaConversionException {
          Method method = lookupMethodInternal(receiverClass, name, args.length - 1);
          MethodHandle handle = method.handle;
 
@@ -237,8 +238,10 @@ public final class Def {
          return handle;
      }
      
-     /** Returns a method handle to an implementation of clazz, given method reference signature */
-     private static MethodHandle lookupReference(MethodHandles.Lookup lookup, Class<?> clazz, String signature) {
+     /** Returns a method handle to an implementation of clazz, given method reference signature 
+      * @throws LambdaConversionException if a method reference cannot be converted to an functional interface
+      */
+     private static MethodHandle lookupReference(MethodHandles.Lookup lookup, Class<?> clazz, String signature) throws LambdaConversionException {
          int separator = signature.indexOf('.');
          FunctionRef ref = new FunctionRef(clazz, signature.substring(0, separator), signature.substring(separator+1));
          final CallSite callSite;
@@ -246,28 +249,24 @@ public final class Def {
          MethodType invokedType = MethodType.fromMethodDescriptorString(ref.invokedType.getDescriptor(), Def.class.getClassLoader());
          MethodType samMethodType = MethodType.fromMethodDescriptorString(ref.samMethodType.getDescriptor(), Def.class.getClassLoader());
          MethodType interfaceType = MethodType.fromMethodDescriptorString(ref.interfaceType.getDescriptor(), Def.class.getClassLoader());
-         try {
-             if (ref.interfaceType.equals(ref.samMethodType)) {
-                 callSite = LambdaMetafactory.altMetafactory(lookup, 
-                         ref.invokedName, 
-                         invokedType,
-                         samMethodType,
-                         ref.implMethodHandle,
-                         samMethodType,
-                         0);
-             } else {
-                 callSite = LambdaMetafactory.altMetafactory(lookup, 
-                         ref.invokedName, 
-                         invokedType,
-                         samMethodType,
-                         ref.implMethodHandle,
-                         samMethodType,
-                         LambdaMetafactory.FLAG_BRIDGES,
-                         1,
-                         interfaceType);
-             }
-         } catch (LambdaConversionException e) {
-             throw new RuntimeException(e);
+         if (ref.interfaceType.equals(ref.samMethodType)) {
+             callSite = LambdaMetafactory.altMetafactory(lookup, 
+                     ref.invokedName, 
+                     invokedType,
+                     samMethodType,
+                     ref.implMethodHandle,
+                     samMethodType,
+                     0);
+         } else {
+             callSite = LambdaMetafactory.altMetafactory(lookup, 
+                     ref.invokedName, 
+                     invokedType,
+                     samMethodType,
+                     ref.implMethodHandle,
+                     samMethodType,
+                     LambdaMetafactory.FLAG_BRIDGES,
+                     1,
+                     interfaceType);
          }
          // we could actually invoke and cache here (in non-capturing cases), but this is not a speedup.
          MethodHandle factory = callSite.dynamicInvoker().asType(MethodType.methodType(clazz));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index e140de9000c..7318a0b65e3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -101,7 +101,7 @@ public final class DefBootstrap {
         /**
          * Does a slow lookup against the whitelist.
          */
-        private MethodHandle lookup(int flavor, Class<?> clazz, String name, Object[] args, long recipe) {
+        private MethodHandle lookup(int flavor, Class<?> clazz, String name, Object[] args, long recipe) throws Throwable {
             switch(flavor) {
                 case METHOD_CALL:
                     return Def.lookupMethod(lookup, clazz, name, args, recipe);

From 3238868cc43346c34fe4068c0d7e60eff4e4c387 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Tue, 7 Jun 2016 13:06:23 +0200
Subject: [PATCH 0691/1311] Make checkstyle happy (add Lookup import, line
 length)

---
 .../src/main/java/org/elasticsearch/painless/Def.java        | 5 +++--
 .../main/java/org/elasticsearch/painless/DefBootstrap.java   | 4 ++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index c2dd960d159..e8c682a2034 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -220,7 +220,8 @@ public final class Def {
      * @throws LambdaConversionException if a method reference cannot be converted to an functional interface
      * @throws IllegalArgumentException if no matching whitelisted method was found.
      */
-     static MethodHandle lookupMethod(MethodHandles.Lookup lookup, Class<?> receiverClass, String name, Object args[], long recipe) throws LambdaConversionException {
+     static MethodHandle lookupMethod(Lookup lookup, Class<?> receiverClass, String name,
+             Object args[], long recipe) throws LambdaConversionException {
          Method method = lookupMethodInternal(receiverClass, name, args.length - 1);
          MethodHandle handle = method.handle;
 
@@ -241,7 +242,7 @@ public final class Def {
      /** Returns a method handle to an implementation of clazz, given method reference signature 
       * @throws LambdaConversionException if a method reference cannot be converted to an functional interface
       */
-     private static MethodHandle lookupReference(MethodHandles.Lookup lookup, Class<?> clazz, String signature) throws LambdaConversionException {
+     private static MethodHandle lookupReference(Lookup lookup, Class<?> clazz, String signature) throws LambdaConversionException {
          int separator = signature.indexOf('.');
          FunctionRef ref = new FunctionRef(clazz, signature.substring(0, separator), signature.substring(separator+1));
          final CallSite callSite;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index 7318a0b65e3..5dc773672a7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -68,13 +68,13 @@ public final class DefBootstrap {
         /** maximum number of types before we go megamorphic */
         static final int MAX_DEPTH = 5;
 
-        private final MethodHandles.Lookup lookup;
+        private final Lookup lookup;
         private final String name;
         private final int flavor;
         private final long recipe;
         int depth; // pkg-protected for testing
 
-        PIC(MethodHandles.Lookup lookup, String name, MethodType type, int flavor, long recipe) {
+        PIC(Lookup lookup, String name, MethodType type, int flavor, long recipe) {
             super(type);
             this.lookup = lookup;
             this.name = name;

From c3e3a6337e95d0b38f009ea4775bcafe3f52f997 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 7 Jun 2016 09:30:58 -0400
Subject: [PATCH 0692/1311] Use method name in bootstrap check might fork test

This commit modifies the bootstrap check invocations in the might fork
tests to use the underlying test name when setting up the logging prefix
when invoking the bootstrap checks. This is done to give clear logs in
case of failure.
---
 .../org/elasticsearch/bootstrap/BootstrapCheckTests.java  | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
index ddd4f01ef8b..5336066b1e2 100644
--- a/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
+++ b/core/src/test/java/org/elasticsearch/bootstrap/BootstrapCheckTests.java
@@ -489,6 +489,8 @@ public class BootstrapCheckTests extends ESTestCase {
         final Runnable enableMightFork,
         final Consumer<RuntimeException> consumer) {
 
+        final String methodName = Thread.currentThread().getStackTrace()[2].getMethodName();
+
         // if seccomp is disabled, nothing should happen
         isSeccompInstalled.set(false);
         if (randomBoolean()) {
@@ -496,13 +498,13 @@ public class BootstrapCheckTests extends ESTestCase {
         } else {
             enableMightFork.run();
         }
-        BootstrapCheck.check(true, randomBoolean(), Collections.singletonList(check), "testMightFork");
+        BootstrapCheck.check(true, randomBoolean(), Collections.singletonList(check), methodName);
 
         // if seccomp is enabled, but we will not fork, nothing should
         // happen
         isSeccompInstalled.set(true);
         disableMightFork.run();
-        BootstrapCheck.check(true, randomBoolean(), Collections.singletonList(check), "testMightFork");
+        BootstrapCheck.check(true, randomBoolean(), Collections.singletonList(check), methodName);
 
         // if seccomp is enabled, and we might fork, the check should
         // be enforced, regardless of bootstrap checks being enabled or
@@ -512,7 +514,7 @@ public class BootstrapCheckTests extends ESTestCase {
 
         final RuntimeException e = expectThrows(
             RuntimeException.class,
-            () -> BootstrapCheck.check(randomBoolean(), randomBoolean(), Collections.singletonList(check), "testMightFork"));
+            () -> BootstrapCheck.check(randomBoolean(), randomBoolean(), Collections.singletonList(check), methodName));
         consumer.accept(e);
     }
 

From 39550354a76f7a3ac5d69de0b5a7c8adc3710e5d Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 7 Jun 2016 09:41:03 -0400
Subject: [PATCH 0693/1311] compute functional interface stuff in Definition

---
 .../elasticsearch/painless/Definition.java    | 68 +++++++++++++++++++
 1 file changed, 68 insertions(+)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 061ef0d95d7..55e78833288 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.painless;
 
+import org.apache.lucene.util.SetOnce;
+
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.LineNumberReader;
@@ -26,13 +28,16 @@ import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
 import java.lang.reflect.Modifier;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.PrimitiveIterator;
+import java.util.Set;
 import java.util.Spliterator;
 
 /**
@@ -281,6 +286,8 @@ public final class Definition {
 
         public final Map<String, Field> staticMembers;
         public final Map<String, Field> members;
+        
+        private final SetOnce<Method> functionalMethod;
 
         private Struct(final String name, final Class<?> clazz, final org.objectweb.asm.Type type) {
             this.name = name;
@@ -293,6 +300,8 @@ public final class Definition {
 
             staticMembers = new HashMap<>();
             members = new HashMap<>();
+            
+            functionalMethod = new SetOnce<Method>();
         }
 
         private Struct(final Struct struct) {
@@ -306,6 +315,8 @@ public final class Definition {
 
             staticMembers = Collections.unmodifiableMap(struct.staticMembers);
             members = Collections.unmodifiableMap(struct.members);
+            
+            functionalMethod = struct.functionalMethod;
         }
 
         private Struct freeze() {
@@ -331,6 +342,14 @@ public final class Definition {
         public int hashCode() {
             return name.hashCode();
         }
+        
+        /** 
+         * If this class is a functional interface according to JLS, returns its method.
+         * Otherwise returns null.
+         */
+        public Method getFunctionalMethod() {
+            return functionalMethod.get();
+        }
     }
 
     public static class Cast {
@@ -439,6 +458,11 @@ public final class Definition {
                 assert hierarchy.get(name).contains("Object") : "class '" + name + "' does not extend Object!";
             }
         }
+        // mark functional interfaces (or set null, to mark class is not)
+        for (Struct clazz : structsMap.values()) {
+            clazz.functionalMethod.set(computeFunctionalInterfaceMethod(clazz));
+        }
+
         // precompute runtime classes
         for (Struct struct : structsMap.values()) {
           addRuntimeClass(struct);
@@ -927,6 +951,50 @@ public final class Definition {
 
         runtimeMap.put(struct.clazz, new RuntimeClass(methods, getters, setters));
     }
+    
+    /** computes the functional interface method for a class, or returns null */
+    private Method computeFunctionalInterfaceMethod(Struct clazz) {
+        if (!clazz.clazz.isInterface()) {
+            return null;
+        }
+        // if its marked with this annotation, we fail if the conditions don't hold (means whitelist bug)
+        // otherwise, this annotation is pretty useless.
+        boolean hasAnnotation = clazz.clazz.isAnnotationPresent(FunctionalInterface.class);
+        List<java.lang.reflect.Method> methods = new ArrayList<>();
+        for (java.lang.reflect.Method m : clazz.clazz.getMethods()) {
+            // default interface methods don't count
+            if (m.isDefault()) {
+                continue;
+            }
+            // static methods don't count
+            if (Modifier.isStatic(m.getModifiers())) {
+                continue;
+            }
+            // if its from Object, it doesn't count
+            try {
+                Object.class.getMethod(m.getName(), m.getParameterTypes());
+                continue;
+            } catch (ReflectiveOperationException e) {
+                // it counts
+            }
+            methods.add(m);
+        }
+        if (methods.size() != 1) {
+            if (hasAnnotation) {
+                throw new IllegalArgumentException("Class: " + clazz.name +
+                                                   " is marked with FunctionalInterface but doesn't fit the bill: " + methods);
+            }
+            return null;
+        }
+        // inspect the one method found from the reflection API, it should match the whitelist!
+        java.lang.reflect.Method oneMethod = methods.get(0);
+        Method painless = clazz.methods.get(new Definition.MethodKey(oneMethod.getName(), oneMethod.getParameterCount()));
+        if (painless == null || painless.method.equals(org.objectweb.asm.commons.Method.getMethod(oneMethod)) == false) {
+            throw new IllegalArgumentException("Class: " + clazz.name + " is functional but the functional " +
+                                               "method is not whitelisted!");
+        }
+        return painless;
+    }
 
     private Type getTypeInternal(String name) {
         // simple types (e.g. 0 array dimensions) are a simple hash lookup for speed

From d71894a226f9388d55b87eac0d67904a6ed030ce Mon Sep 17 00:00:00 2001
From: Christoph Wurm <cwurm@users.noreply.github.com>
Date: Tue, 7 Jun 2016 15:47:38 +0200
Subject: [PATCH 0694/1311] Update ingest-node.asciidoc

Fixed forgotten rename from `match_formats` to `formats` in documentation (changed in https://github.com/elastic/elasticsearch/commit/dd2184ab259fb544659ec28ce97467729e2974fe)
---
 docs/reference/ingest/ingest-node.asciidoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc
index 9cf51374ab4..ec641383beb 100644
--- a/docs/reference/ingest/ingest-node.asciidoc
+++ b/docs/reference/ingest/ingest-node.asciidoc
@@ -744,7 +744,7 @@ in the same order they were defined as part of the processor definition.
 | Name                   | Required  | Default             | Description
 | `field`                | yes       | -                   | The field to get the date from.
 | `target_field`         | no        | @timestamp          | The field that will hold the parsed date.
-| `match_formats`        | yes       | -                   | An array of the expected date formats. Can be a Joda pattern or one of the following formats: ISO8601, UNIX, UNIX_MS, or TAI64N.
+| `formats`              | yes       | -                   | An array of the expected date formats. Can be a Joda pattern or one of the following formats: ISO8601, UNIX, UNIX_MS, or TAI64N.
 | `timezone`             | no        | UTC                 | The timezone to use when parsing the date.
 | `locale`               | no        | ENGLISH             | The locale to use when parsing the date, relevant when parsing month names or week days.
 |======
@@ -760,7 +760,7 @@ Here is an example that adds the parsed date to the `timestamp` field based on t
       "date" : {
         "field" : "initial_date",
         "target_field" : "timestamp",
-        "match_formats" : ["dd/MM/yyyy hh:mm:ss"],
+        "formats" : ["dd/MM/yyyy hh:mm:ss"],
         "timezone" : "Europe/Amsterdam"
       }
     }

From 3108c399f02823153b449ee97ed7d9d8e7f5d9be Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 7 Jun 2016 09:49:54 -0400
Subject: [PATCH 0695/1311] use painless types in FunctionRef

---
 .../java/org/elasticsearch/painless/Def.java  |  6 +--
 .../elasticsearch/painless/FunctionRef.java   | 49 +++++--------------
 .../painless/node/EFunctionRef.java           |  2 +-
 3 files changed, 17 insertions(+), 40 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index e8c682a2034..af96e23771f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -230,7 +230,7 @@ public final class Def {
              for (int i = 0; i < args.length; i++) {
                  // its a functional reference, replace the argument with an impl
                  if ((recipe & (1L << (i - 1))) != 0) {
-                     filters[i] = lookupReference(lookup, method.arguments.get(i - 1).clazz, (String) args[i]);
+                     filters[i] = lookupReference(lookup, method.arguments.get(i - 1), (String) args[i]);
                  }
              }
              handle = MethodHandles.filterArguments(handle, 0, filters);
@@ -242,7 +242,7 @@ public final class Def {
      /** Returns a method handle to an implementation of clazz, given method reference signature 
       * @throws LambdaConversionException if a method reference cannot be converted to an functional interface
       */
-     private static MethodHandle lookupReference(Lookup lookup, Class<?> clazz, String signature) throws LambdaConversionException {
+     private static MethodHandle lookupReference(Lookup lookup, Definition.Type clazz, String signature) throws LambdaConversionException {
          int separator = signature.indexOf('.');
          FunctionRef ref = new FunctionRef(clazz, signature.substring(0, separator), signature.substring(separator+1));
          final CallSite callSite;
@@ -270,7 +270,7 @@ public final class Def {
                      interfaceType);
          }
          // we could actually invoke and cache here (in non-capturing cases), but this is not a speedup.
-         MethodHandle factory = callSite.dynamicInvoker().asType(MethodType.methodType(clazz));
+         MethodHandle factory = callSite.dynamicInvoker().asType(MethodType.methodType(clazz.clazz));
          return MethodHandles.dropArguments(factory, 0, Object.class);
      }
      
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
index 5b121d3a31a..aa847535cd4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
@@ -19,14 +19,13 @@
 
 package org.elasticsearch.painless;
 
+import org.elasticsearch.painless.Definition.Method;
 import org.objectweb.asm.Handle;
 import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.Type;
 
 import java.lang.invoke.MethodHandle;
 import java.lang.reflect.Modifier;
-import java.util.HashSet;
-import java.util.Set;
 
 /** 
  * computes "everything you need" to call LambdaMetaFactory, given an expected interface,
@@ -43,29 +42,33 @@ public class FunctionRef {
 
     public final MethodHandle implMethodHandle;
     
-    public FunctionRef(Class<?> expected, String type, String call) {
+    public FunctionRef(Definition.Type expected, String type, String call) {
         boolean isCtorReference = "new".equals(call);
         // check its really a functional interface
         // for e.g. Comparable
-        java.lang.reflect.Method method = getFunctionalMethod(type, call, expected);
+        Method method = expected.struct.getFunctionalMethod();
+        if (method == null) {
+            throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
+                                               "to [" + expected.name + "], not a functional interface");
+        }
         // e.g. compareTo
-        invokedName = method.getName();
+        invokedName = method.name;
         // e.g. (Object)Comparator
-        invokedType = Type.getMethodType(Type.getType(expected));
+        invokedType = Type.getMethodType(expected.type);
         // e.g. (Object,Object)int
-        interfaceType = Type.getMethodType(Type.getMethodDescriptor(method));
+        interfaceType = Type.getMethodType(method.method.getDescriptor());
         // lookup requested method
         Definition.Struct struct = Definition.getType(type).struct;
         final Definition.Method impl;
         // ctor ref
         if (isCtorReference) {
-            impl = struct.constructors.get(new Definition.MethodKey("<init>", method.getParameterCount()));
+            impl = struct.constructors.get(new Definition.MethodKey("<init>", method.arguments.size()));
         } else {
             // look for a static impl first
-            Definition.Method staticImpl = struct.staticMethods.get(new Definition.MethodKey(call, method.getParameterCount()));
+            Definition.Method staticImpl = struct.staticMethods.get(new Definition.MethodKey(call, method.arguments.size()));
             if (staticImpl == null) {
                 // otherwise a virtual impl
-                impl = struct.methods.get(new Definition.MethodKey(call, method.getParameterCount()-1));
+                impl = struct.methods.get(new Definition.MethodKey(call, method.arguments.size()-1));
             } else {
                 impl = staticImpl;
             }
@@ -97,30 +100,4 @@ public class FunctionRef {
             samMethodType = Type.getMethodType(impl.method.getReturnType(), params);
         }
     }
-    
-    static final Set<Definition.MethodKey> OBJECT_METHODS = new HashSet<>();
-    static {
-        for (java.lang.reflect.Method m : Object.class.getMethods()) {
-            OBJECT_METHODS.add(new Definition.MethodKey(m.getName(), m.getParameterCount()));
-        }
-    }
-
-    // TODO: move all this crap out, to Definition to compute up front
-    java.lang.reflect.Method getFunctionalMethod(String type, String call, Class<?> clazz) {
-        if (!clazz.isInterface()) {
-            throw new IllegalArgumentException("Cannot convert function reference [" 
-                                               + type + "::" + call + "] to [" + clazz + "]");
-        }
-        for (java.lang.reflect.Method m : clazz.getMethods()) {
-            if (m.isDefault()) {
-                continue;
-            }
-            if (OBJECT_METHODS.contains(new Definition.MethodKey(m.getName(), m.getParameterCount()))) {
-                continue;
-            }
-            return m;
-        }
-        throw new IllegalArgumentException("Cannot convert function reference [" 
-                                           + type + "::" + call + "] to [" + clazz + "]");
-    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index dd050a549ea..a51ac3ad737 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -52,7 +52,7 @@ public class EFunctionRef extends AExpression {
             actual = Definition.getType("String");
         } else {
             try {
-                ref = new FunctionRef(expected.clazz, type, call);
+                ref = new FunctionRef(expected, type, call);
             } catch (IllegalArgumentException e) {
                 throw createError(e);
             }

From b9030bf6feee60166d57291d5d2a91368024dbf1 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Tue, 10 May 2016 12:10:55 +0200
Subject: [PATCH 0696/1311] Add the ability to partition a scroll in multiple
 slices. API:

```
curl -XGET 'localhost:9200/twitter/tweet/_search?scroll=1m' -d '{
    "slice": {
        "field": "_uid", <1>
        "id": 0, <2>
        "max": 10 <3>
    },
    "query": {
        "match" : {
            "title" : "elasticsearch"
        }
    }
}
```

<1> (optional) The field name used to do the slicing (_uid by default)
<2> The id of the slice

By default the splitting is done on the shards first and then locally on each shard using the _uid field
with the following formula:
`slice(doc) = floorMod(hashCode(doc._uid), max)`
For instance if the number of shards is equal to 2 and the user requested 4 slices then the slices 0 and 2 are assigned
to the first shard and the slices 1 and 3 are assigned to the second shard.

Each scroll is independent and can be processed in parallel like any scroll request.

Closes #13494
---
 .../action/search/SearchRequestBuilder.java   |   6 +
 .../elasticsearch/search/SearchService.java   |   9 +
 .../search/builder/SearchSourceBuilder.java   | 102 +++---
 .../search/internal/DefaultSearchContext.java |  25 +-
 .../search/internal/ScrollContext.java        |   1 -
 .../search/internal/SubSearchContext.java     |   1 -
 .../search/slice/DocValuesSliceQuery.java     |  67 ++++
 .../search/slice/SliceBuilder.java            | 251 +++++++++++++
 .../search/slice/SliceQuery.java              |  81 +++++
 .../search/slice/TermsSliceQuery.java         |  86 +++++
 .../builder/SearchSourceBuilderTests.java     |  11 +
 .../slice/DocValuesSliceQueryTests.java       | 124 +++++++
 .../search/slice/SearchSliceIT.java           | 215 +++++++++++
 .../search/slice/SliceBuilderTests.java       | 340 ++++++++++++++++++
 .../search/slice/TermsSliceQueryTests.java    | 117 ++++++
 docs/reference/search/request/scroll.asciidoc |  89 +++++
 16 files changed, 1462 insertions(+), 63 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java
 create mode 100644 core/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java
 create mode 100644 core/src/main/java/org/elasticsearch/search/slice/SliceQuery.java
 create mode 100644 core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java
 create mode 100644 core/src/test/java/org/elasticsearch/search/slice/DocValuesSliceQueryTests.java
 create mode 100644 core/src/test/java/org/elasticsearch/search/slice/SearchSliceIT.java
 create mode 100644 core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
 create mode 100644 core/src/test/java/org/elasticsearch/search/slice/TermsSliceQueryTests.java

diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
index c5666fa16a5..173c15ec1a5 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
@@ -30,6 +30,7 @@ import org.elasticsearch.script.Template;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
+import org.elasticsearch.search.slice.SliceBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.highlight.HighlightBuilder;
 import org.elasticsearch.search.rescore.RescoreBuilder;
@@ -352,6 +353,11 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
         return this;
     }
 
+    public SearchRequestBuilder slice(SliceBuilder builder) {
+        sourceBuilder().slice(builder);
+        return this;
+    }
+
     /**
      * Applies when sorting, and controls if scores will be tracked as well. Defaults to
      * <tt>false</tt>.
diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java
index b4704fe5a7f..57841466a62 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchService.java
@@ -821,6 +821,15 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
             FieldDoc fieldDoc = SearchAfterBuilder.buildFieldDoc(context.sort(), source.searchAfter());
             context.searchAfter(fieldDoc);
         }
+
+        if (source.slice() != null) {
+            if (context.scrollContext() == null) {
+                throw new SearchContextException(context, "`slice` cannot be used outside of a scroll context");
+            }
+            context.sliceFilter(source.slice().toFilter(queryShardContext,
+                context.shardTarget().getShardId().getId(),
+                queryShardContext.getIndexSettings().getNumberOfShards()));
+        }
     }
 
     private static final int[] EMPTY_DOC_IDS = new int[0];
diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index 8c4be225127..b1a5d2d03e6 100644
--- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -42,6 +42,7 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.slice.SliceBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
@@ -98,6 +99,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
     public static final ParseField EXT_FIELD = new ParseField("ext");
     public static final ParseField PROFILE_FIELD = new ParseField("profile");
     public static final ParseField SEARCH_AFTER = new ParseField("search_after");
+    public static final ParseField SLICE = new ParseField("slice");
 
     public static SearchSourceBuilder fromXContent(QueryParseContext context, AggregatorParsers aggParsers,
             Suggesters suggesters) throws IOException {
@@ -138,6 +140,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
 
     private SearchAfterBuilder searchAfterBuilder;
 
+    private SliceBuilder sliceBuilder;
+
     private Float minScore;
 
     private long timeoutInMillis = -1;
@@ -175,9 +179,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
      * Read from a stream.
      */
     public SearchSourceBuilder(StreamInput in) throws IOException {
-        if (in.readBoolean()) {
-            aggregations = new AggregatorFactories.Builder(in);
-        }
+        aggregations = in.readOptionalWriteable(AggregatorFactories.Builder::new);
         explain = in.readOptionalBoolean();
         fetchSourceContext = in.readOptionalStreamable(FetchSourceContext::new);
         boolean hasFieldDataFields = in.readBoolean();
@@ -206,15 +208,9 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                 indexBoost.put(in.readString(), in.readFloat());
             }
         }
-        if (in.readBoolean()) {
-            minScore = in.readFloat();
-        }
-        if (in.readBoolean()) {
-            postQueryBuilder = in.readNamedWriteable(QueryBuilder.class);
-        }
-        if (in.readBoolean()) {
-            queryBuilder = in.readNamedWriteable(QueryBuilder.class);
-        }
+        minScore = in.readOptionalFloat();
+        postQueryBuilder = in.readOptionalNamedWriteable(QueryBuilder.class);
+        queryBuilder = in.readOptionalNamedWriteable(QueryBuilder.class);
         if (in.readBoolean()) {
             int size = in.readVInt();
             rescoreBuilders = new ArrayList<>();
@@ -244,29 +240,20 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                 stats.add(in.readString());
             }
         }
-        if (in.readBoolean()) {
-            suggestBuilder = new SuggestBuilder(in);
-        }
+        suggestBuilder = in.readOptionalWriteable(SuggestBuilder::new);
         terminateAfter = in.readVInt();
         timeoutInMillis = in.readLong();
         trackScores = in.readBoolean();
         version = in.readOptionalBoolean();
-        if (in.readBoolean()) {
-            ext = in.readBytesReference();
-        }
+        ext = in.readOptionalBytesReference();
         profile = in.readBoolean();
-        if (in.readBoolean()) {
-            searchAfterBuilder = new SearchAfterBuilder(in);
-        }
+        searchAfterBuilder = in.readOptionalWriteable(SearchAfterBuilder::new);
+        sliceBuilder = in.readOptionalWriteable(SliceBuilder::new);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        boolean hasAggregations = aggregations != null;
-        out.writeBoolean(hasAggregations);
-        if (hasAggregations) {
-            aggregations.writeTo(out);
-        }
+        out.writeOptionalWriteable(aggregations);
         out.writeOptionalBoolean(explain);
         out.writeOptionalStreamable(fetchSourceContext);
         boolean hasFieldDataFields = fieldDataFields != null;
@@ -296,21 +283,9 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                 out.writeFloat(indexBoost.get(key.value));
             }
         }
-        boolean hasMinScore = minScore != null;
-        out.writeBoolean(hasMinScore);
-        if (hasMinScore) {
-            out.writeFloat(minScore);
-        }
-        boolean hasPostQuery = postQueryBuilder != null;
-        out.writeBoolean(hasPostQuery);
-        if (hasPostQuery) {
-            out.writeNamedWriteable(postQueryBuilder);
-        }
-        boolean hasQuery = queryBuilder != null;
-        out.writeBoolean(hasQuery);
-        if (hasQuery) {
-            out.writeNamedWriteable(queryBuilder);
-        }
+        out.writeOptionalFloat(minScore);
+        out.writeOptionalNamedWriteable(postQueryBuilder);
+        out.writeOptionalNamedWriteable(queryBuilder);
         boolean hasRescoreBuilders = rescoreBuilders != null;
         out.writeBoolean(hasRescoreBuilders);
         if (hasRescoreBuilders) {
@@ -344,26 +319,15 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                 out.writeString(stat);
             }
         }
-        boolean hasSuggestBuilder = suggestBuilder != null;
-        out.writeBoolean(hasSuggestBuilder);
-        if (hasSuggestBuilder) {
-            suggestBuilder.writeTo(out);
-        }
+        out.writeOptionalWriteable(suggestBuilder);
         out.writeVInt(terminateAfter);
         out.writeLong(timeoutInMillis);
         out.writeBoolean(trackScores);
         out.writeOptionalBoolean(version);
-        boolean hasExt = ext != null;
-        out.writeBoolean(hasExt);
-        if (hasExt) {
-            out.writeBytesReference(ext);
-        }
+        out.writeOptionalBytesReference(ext);
         out.writeBoolean(profile);
-        boolean hasSearchAfter = searchAfterBuilder != null;
-        out.writeBoolean(hasSearchAfter);
-        if (hasSearchAfter) {
-            searchAfterBuilder.writeTo(out);
-        }
+        out.writeOptionalWriteable(searchAfterBuilder);
+        out.writeOptionalWriteable(sliceBuilder);
     }
 
     /**
@@ -597,6 +561,22 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         return this;
     }
 
+    /**
+     * Sets a filter that will restrict the search hits, the top hits and the aggregations to a slice of the results
+     * of the main query.
+     */
+    public SearchSourceBuilder slice(SliceBuilder builder) {
+        this.sliceBuilder = builder;
+        return this;
+    }
+
+    /**
+     * Gets the slice used to filter the search hits, the top hits and the aggregations.
+     */
+    public SliceBuilder slice() {
+        return sliceBuilder;
+    }
+
     /**
      * Add an aggregation to perform as part of the search.
      */
@@ -943,6 +923,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
             rewrittenBuilder.rescoreBuilders = rescoreBuilders;
             rewrittenBuilder.scriptFields = scriptFields;
             rewrittenBuilder.searchAfterBuilder = searchAfterBuilder;
+            rewrittenBuilder.sliceBuilder = sliceBuilder;
             rewrittenBuilder.size = size;
             rewrittenBuilder.sorts = sorts;
             rewrittenBuilder.stats = stats;
@@ -1039,6 +1020,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                 } else if (context.getParseFieldMatcher().match(currentFieldName, EXT_FIELD)) {
                     XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().copyCurrentStructure(parser);
                     ext = xContentBuilder.bytes();
+                } else if (context.getParseFieldMatcher().match(currentFieldName, SLICE)) {
+                    sliceBuilder = SliceBuilder.fromXContent(context);
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].",
                             parser.getTokenLocation());
@@ -1193,6 +1176,10 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
             builder.field(SEARCH_AFTER.getPreferredName(), searchAfterBuilder.getSortValues());
         }
 
+        if (sliceBuilder != null) {
+            builder.field(SLICE.getPreferredName(), sliceBuilder);
+        }
+
         if (indexBoost != null) {
             builder.startObject(INDICES_BOOST_FIELD.getPreferredName());
             assert !indexBoost.containsKey(null);
@@ -1355,7 +1342,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
     public int hashCode() {
         return Objects.hash(aggregations, explain, fetchSourceContext, fieldDataFields, fieldNames, from,
                 highlightBuilder, indexBoost, minScore, postQueryBuilder, queryBuilder, rescoreBuilders, scriptFields,
-                size, sorts, searchAfterBuilder, stats, suggestBuilder, terminateAfter, timeoutInMillis, trackScores, version, profile);
+                size, sorts, searchAfterBuilder, sliceBuilder, stats, suggestBuilder, terminateAfter, timeoutInMillis, trackScores, version, profile);
     }
 
     @Override
@@ -1383,6 +1370,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                 && Objects.equals(size, other.size)
                 && Objects.equals(sorts, other.sorts)
                 && Objects.equals(searchAfterBuilder, other.searchAfterBuilder)
+                && Objects.equals(sliceBuilder, other.sliceBuilder)
                 && Objects.equals(stats, other.stats)
                 && Objects.equals(suggestBuilder, other.suggestBuilder)
                 && Objects.equals(terminateAfter, other.terminateAfter)
diff --git a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
index a001ab22ac4..30e994b7656 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
@@ -115,6 +115,9 @@ public class DefaultSearchContext extends SearchContext {
     private Float minimumScore;
     private boolean trackScores = false; // when sorting, track scores as well...
     private FieldDoc searchAfter;
+    // filter for sliced scroll
+    private Query sliceFilter;
+
     /**
      * The original query as sent by the user without the types and aliases
      * applied. Putting things in here leaks them into highlighting so don't add
@@ -122,8 +125,7 @@ public class DefaultSearchContext extends SearchContext {
      */
     private ParsedQuery originalQuery;
     /**
-     * Just like originalQuery but with the filters from types and aliases
-     * applied.
+     * Just like originalQuery but with the filters from types, aliases and slice applied.
      */
     private ParsedQuery filteredQuery;
     /**
@@ -210,7 +212,7 @@ public class DefaultSearchContext extends SearchContext {
                 if (rescoreContext.window() > maxWindow) {
                     throw new QueryPhaseExecutionException(this, "Rescore window [" + rescoreContext.window() + "] is too large. It must "
                             + "be less than [" + maxWindow + "]. This prevents allocating massive heaps for storing the results to be "
-                            + "rescored. This limit can be set by chaning the [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey()
+                            + "rescored. This limit can be set by chaining the [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey()
                             + "] index level setting.");
 
                 }
@@ -254,7 +256,17 @@ public class DefaultSearchContext extends SearchContext {
     @Override
     @Nullable
     public Query searchFilter(String[] types) {
-        return createSearchFilter(types, aliasFilter, mapperService().hasNested());
+        Query typesFilter = createSearchFilter(types, aliasFilter, mapperService().hasNested());
+        if (sliceFilter == null) {
+            return typesFilter;
+        }
+        if (typesFilter == null) {
+            return sliceFilter;
+        }
+        return new BooleanQuery.Builder()
+            .add(typesFilter, Occur.FILTER)
+            .add(sliceFilter, Occur.FILTER)
+            .build();
     }
 
     // extracted to static helper method to make writing unit tests easier:
@@ -550,6 +562,11 @@ public class DefaultSearchContext extends SearchContext {
         return searchAfter;
     }
 
+    public SearchContext sliceFilter(Query filter) {
+        this.sliceFilter = filter;
+        return this;
+    }
+
     @Override
     public SearchContext parsedPostFilter(ParsedQuery postFilter) {
         this.postFilter = postFilter;
diff --git a/core/src/main/java/org/elasticsearch/search/internal/ScrollContext.java b/core/src/main/java/org/elasticsearch/search/internal/ScrollContext.java
index 1744b6fd745..1b7bcfb93c7 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/ScrollContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/ScrollContext.java
@@ -29,5 +29,4 @@ public class ScrollContext {
     public float maxScore;
     public ScoreDoc lastEmittedDoc;
     public Scroll scroll;
-
 }
diff --git a/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
index 6cacf86d65f..37fb608fd0c 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
@@ -20,7 +20,6 @@ package org.elasticsearch.search.internal;
 
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.Counter;
-import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.search.aggregations.SearchContextAggregations;
 import org.elasticsearch.search.fetch.FetchSearchResult;
diff --git a/core/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java b/core/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java
new file mode 100644
index 00000000000..4f334ebf0d1
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.slice;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.search.RandomAccessWeight;
+import org.apache.lucene.util.Bits;
+
+import java.io.IOException;
+
+/**
+ * A {@link SliceQuery} that uses the numeric doc values of a field to do the slicing.
+ *
+ * <b>NOTE</b>: With deterministic field values this query can be used across different readers safely.
+ * If updates are accepted on the field you must ensure that the same reader is used for all `slice` queries.
+ */
+public final class DocValuesSliceQuery extends SliceQuery {
+    public DocValuesSliceQuery(String field, int id, int max) {
+        super(field, id, max);
+    }
+
+    @Override
+    public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
+        return new RandomAccessWeight(this) {
+            @Override
+            protected Bits getMatchingDocs(final LeafReaderContext context) throws IOException {
+                final SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), getField());
+                return new Bits() {
+                    @Override
+                    public boolean get(int doc) {
+                        values.setDocument(doc);
+                        for (int i = 0; i < values.count(); i++) {
+                            return contains(Long.hashCode(values.valueAt(i)));
+                        }
+                        return contains(0);
+                    }
+
+                    @Override
+                    public int length() {
+                        return context.reader().maxDoc();
+                    }
+                };
+            }
+        };
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java b/core/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java
new file mode 100644
index 00000000000..97c79aefa71
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.slice;
+
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.elasticsearch.action.support.ToXContentToBytes;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.internal.UidFieldMapper;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ *  A slice builder allowing to split a scroll in multiple partitions.
+ *  If the provided field is the "_uid" it uses a {@link org.elasticsearch.search.slice.TermsSliceQuery}
+ *  to do the slicing. The slicing is done at the shard level first and then each shard is splitted in multiple slices.
+ *  For instance if the number of shards is equal to 2 and the user requested 4 slices
+ *  then the slices 0 and 2 are assigned to the first shard and the slices 1 and 3 are assigned to the second shard.
+ *  This way the total number of bitsets that we need to build on each shard is bounded by the number of slices
+ *  (instead of {@code numShards*numSlices}).
+ *  Otherwise the provided field must be a numeric and doc_values must be enabled. In that case a
+ *  {@link org.elasticsearch.search.slice.DocValuesSliceQuery} is used to filter the results.
+ */
+public class SliceBuilder extends ToXContentToBytes implements Writeable {
+    public static final ParseField FIELD_FIELD = new ParseField("field");
+    public static final ParseField ID_FIELD = new ParseField("id");
+    public static final ParseField MAX_FIELD = new ParseField("max");
+    private final static ObjectParser<SliceBuilder, QueryParseContext> PARSER =
+        new ObjectParser<>("slice", SliceBuilder::new);
+
+    static {
+        PARSER.declareString(SliceBuilder::setField, FIELD_FIELD);
+        PARSER.declareInt(SliceBuilder::setId, ID_FIELD);
+        PARSER.declareInt(SliceBuilder::setMax, MAX_FIELD);
+    }
+
+    /** Name of field to slice against (_uid by default) */
+    private String field = UidFieldMapper.NAME;
+    /** The id of the slice */
+    private int id = -1;
+    /** Max number of slices */
+    private int max = -1;
+
+    private SliceBuilder() {}
+
+    public SliceBuilder(int id, int max) {
+        this(UidFieldMapper.NAME, id, max);
+    }
+
+    /**
+     *
+     * @param field The name of the field
+     * @param id The id of the slice
+     * @param max The maximum number of slices
+     */
+    public SliceBuilder(String field, int id, int max) {
+        setField(field);
+        setId(id);
+        setMax(max);
+    }
+
+    public SliceBuilder(StreamInput in) throws IOException {
+        this.field = in.readString();
+        this.id = in.readVInt();
+        this.max = in.readVInt();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(field);
+        out.writeVInt(id);
+        out.writeVInt(max);
+    }
+
+    private SliceBuilder setField(String field) {
+        if (Strings.isEmpty(field)) {
+            throw new IllegalArgumentException("field name is null or empty");
+        }
+        this.field = field;
+        return this;
+    }
+
+    /**
+     * The name of the field to slice against
+     */
+    public String getField() {
+        return this.field;
+    }
+
+    private SliceBuilder setId(int id) {
+        if (id < 0) {
+            throw new IllegalArgumentException("id must be greater than or equal to 0");
+        }
+        if (max != -1 && id >= max) {
+            throw new IllegalArgumentException("max must be greater than id");
+        }
+        this.id = id;
+        return this;
+    }
+
+    /**
+     * The id of the slice.
+     */
+    public int getId() {
+        return id;
+    }
+
+    private SliceBuilder setMax(int max) {
+        if (max <= 1) {
+            throw new IllegalArgumentException("max must be greater than 1");
+        }
+        if (id != -1 && id >= max) {
+            throw new IllegalArgumentException("max must be greater than id");
+        }
+        this.max = max;
+        return this;
+    }
+
+    /**
+     * The maximum number of slices.
+     */
+    public int getMax() {
+        return max;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        innerToXContent(builder);
+        builder.endObject();
+        return builder;
+    }
+
+    void innerToXContent(XContentBuilder builder) throws IOException {
+        builder.field(FIELD_FIELD.getPreferredName(), field);
+        builder.field(ID_FIELD.getPreferredName(), id);
+        builder.field(MAX_FIELD.getPreferredName(), max);
+    }
+
+    public static SliceBuilder fromXContent(QueryParseContext context) throws IOException {
+        SliceBuilder builder = PARSER.parse(context.parser(), new SliceBuilder(), context);
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (!(other instanceof SliceBuilder)) {
+            return false;
+        }
+
+        SliceBuilder o = (SliceBuilder) other;
+        return ((field == null && o.field == null) || field.equals(o.field))
+            && id == o.id && o.max == max;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(this.field, this.id, this.max);
+    }
+
+    public Query toFilter(QueryShardContext context, int shardId, int numShards) {
+        final MappedFieldType type = context.fieldMapper(field);
+        if (type == null) {
+            throw new IllegalArgumentException("field " + field + " not found");
+        }
+
+        boolean useTermQuery = false;
+        if (UidFieldMapper.NAME.equals(field)) {
+           useTermQuery = true;
+        } else if (type.hasDocValues() == false) {
+            throw new IllegalArgumentException("cannot load numeric doc values on " + field);
+        } else {
+            IndexFieldData ifm = context.getForField(type);
+            if (ifm instanceof IndexNumericFieldData == false) {
+                throw new IllegalArgumentException("cannot load numeric doc values on " + field);
+            }
+        }
+
+        if (numShards == 1) {
+            return useTermQuery ? new TermsSliceQuery(field, id, max) :
+                new DocValuesSliceQuery(field, id, max);
+        }
+        if (max >= numShards) {
+            // the number of slices is greater than the number of shards
+            // in such case we can reduce the number of requested shards by slice
+
+            // first we check if the slice is responsible of this shard
+            int targetShard = id % numShards;
+            if (targetShard != shardId) {
+                // the shard is not part of this slice, we can skip it.
+                return new MatchNoDocsQuery("this shard is not part of the slice");
+            }
+            // compute the number of slices where this shard appears
+            int numSlicesInShard = max / numShards;
+            int rest = max % numShards;
+            if (rest > targetShard) {
+                numSlicesInShard++;
+            }
+
+            if (numSlicesInShard == 1) {
+                // this shard has only one slice so we must check all the documents
+                return new MatchAllDocsQuery();
+            }
+            // get the new slice id for this shard
+            int shardSlice = id / numShards;
+
+            return useTermQuery ?
+                new TermsSliceQuery(field, shardSlice, numSlicesInShard) :
+                new DocValuesSliceQuery(field, shardSlice, numSlicesInShard);
+        }
+        // the number of shards is greater than the number of slices
+
+        // check if the shard is assigned to the slice
+        int targetSlice = shardId % max;
+        if (id != targetSlice) {
+            // the shard is not part of this slice, we can skip it.
+            return new MatchNoDocsQuery("this shard is not part of the slice");
+        }
+        return new MatchAllDocsQuery();
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/slice/SliceQuery.java b/core/src/main/java/org/elasticsearch/search/slice/SliceQuery.java
new file mode 100644
index 00000000000..0d87b275403
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/slice/SliceQuery.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.slice;
+
+import org.apache.lucene.search.Query;
+
+import java.util.Objects;
+
+/**
+ * An abstract {@link Query} that defines an hash function to partition the documents in multiple slices.
+ */
+public abstract class SliceQuery extends Query {
+    private final String field;
+    private final int id;
+    private final int max;
+
+    /**
+     * @param field The name of the field
+     * @param id    The id of the slice
+     * @param max   The maximum number of slices
+     */
+    public SliceQuery(String field, int id, int max) {
+        this.field = field;
+        this.id = id;
+        this.max = max;
+    }
+
+    // Returns true if the value matches the predicate
+    protected final boolean contains(long value) {
+        return Math.floorMod(value, max) == id;
+    }
+
+    public String getField() {
+        return field;
+    }
+
+    public int getId() {
+        return id;
+    }
+
+    public int getMax() {
+        return max;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (super.equals(o) == false) {
+            return false;
+        }
+        SliceQuery that = (SliceQuery) o;
+        return field.equals(that.field) && id == that.id && max == that.max;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), field, id, max);
+    }
+
+    @Override
+    public String toString(String f) {
+        return getClass().getSimpleName() + "[field=" + field + ", id=" + id + ", max=" + max + "]";
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java b/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java
new file mode 100644
index 00000000000..b967a6b6e71
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.slice;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.DocIdSet;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.search.ConstantScoreWeight;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ConstantScoreScorer;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.DocIdSetBuilder;
+
+import java.io.IOException;
+
+/**
+ * A {@link SliceQuery} that uses the terms dictionary of a field to do the slicing.
+ *
+ * <b>NOTE</b>: The cost of this filter is O(N*M) where N is the number of unique terms in the dictionary
+ * and M is the average number of documents per term.
+ * For each segment this filter enumerates the terms dictionary, computes the hash code for each term and fills
+ * a bit set with the documents of all terms whose hash code matches the predicate.
+ * <b>NOTE</b>: Documents with no value for that field are ignored.
+ */
+public final class TermsSliceQuery extends SliceQuery {
+    public TermsSliceQuery(String field, int id, int max) {
+        super(field, id, max);
+    }
+
+    @Override
+    public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
+        return new ConstantScoreWeight(this) {
+            @Override
+            public Scorer scorer(LeafReaderContext context) throws IOException {
+                final DocIdSet disi = build(context.reader());
+                final DocIdSetIterator leafIt = disi.iterator();
+                return new ConstantScoreScorer(this, score(), leafIt);
+            }
+        };
+    }
+
+    /**
+     * Returns a DocIdSet per segments containing the matching docs for the specified slice.
+     */
+    private DocIdSet build(LeafReader reader) throws IOException {
+        final DocIdSetBuilder builder = new DocIdSetBuilder(reader.maxDoc());
+        final Terms terms = reader.terms(getField());
+        final TermsEnum te = terms.iterator();
+        PostingsEnum docsEnum = null;
+        for (BytesRef term = te.next(); term != null; term = te.next()) {
+            int hashCode = term.hashCode();
+            if (contains(hashCode)) {
+                docsEnum = te.postings(docsEnum, PostingsEnum.NONE);
+                int docId = docsEnum.nextDoc();
+                while (docId != DocIdSetIterator.NO_MORE_DOCS) {
+                    builder.add(docId);
+                    docId = docsEnum.nextDoc();
+                }
+            }
+        }
+        return builder.build();
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index fc3c90b5165..7ef3fa30e96 100644
--- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -71,6 +71,7 @@ import org.elasticsearch.search.highlight.HighlightBuilderTests;
 import org.elasticsearch.search.rescore.QueryRescoreBuilderTests;
 import org.elasticsearch.search.rescore.QueryRescorerBuilder;
 import org.elasticsearch.search.searchafter.SearchAfterBuilder;
+import org.elasticsearch.search.slice.SliceBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.ScoreSortBuilder;
 import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType;
@@ -426,6 +427,16 @@ public class SearchSourceBuilderTests extends ESTestCase {
             xContentBuilder.endObject();
             builder.ext(xContentBuilder);
         }
+        if (randomBoolean()) {
+            String field = randomBoolean() ? null : randomAsciiOfLengthBetween(5, 20);
+            int max = randomInt(1000);
+            int id = randomInt(max-1);
+            if (field == null) {
+                builder.slice(new SliceBuilder(id, max));
+            } else {
+                builder.slice(new SliceBuilder(field, id, max));
+            }
+        }
         return builder;
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/slice/DocValuesSliceQueryTests.java b/core/src/test/java/org/elasticsearch/search/slice/DocValuesSliceQueryTests.java
new file mode 100644
index 00000000000..dee2ae67914
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/slice/DocValuesSliceQueryTests.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.slice;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.LeafCollector;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.QueryUtils;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.NumericUtils;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class DocValuesSliceQueryTests extends ESTestCase {
+
+    public void testBasics() {
+        DocValuesSliceQuery query1 =
+            new DocValuesSliceQuery("field1", 1, 10);
+        DocValuesSliceQuery query2 =
+            new DocValuesSliceQuery("field1", 1, 10);
+        DocValuesSliceQuery query3 =
+            new DocValuesSliceQuery("field2", 1, 10);
+        DocValuesSliceQuery query4 =
+            new DocValuesSliceQuery("field1", 2, 10);
+        QueryUtils.check(query1);
+        QueryUtils.checkEqual(query1, query2);
+        QueryUtils.checkUnequal(query1, query3);
+        QueryUtils.checkUnequal(query1, query4);
+    }
+
+    public void testSearch() throws Exception {
+        final int numDocs = randomIntBetween(100, 200);
+        final Directory dir = newDirectory();
+        final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+        int max = randomIntBetween(2, 10);
+        int[] sliceCounters1 = new int[max];
+        int[] sliceCounters2 = new int[max];
+        Set<String> keys = new HashSet<>();
+        for (int i = 0; i < numDocs; ++i) {
+            Document doc = new Document();
+            String uuid = UUIDs.base64UUID();
+            int intValue = randomInt();
+            long doubleValue = NumericUtils.doubleToSortableLong(randomDouble());
+            doc.add(new StringField("uuid", uuid, Field.Store.YES));
+            doc.add(new SortedNumericDocValuesField("intField", intValue));
+            doc.add(new SortedNumericDocValuesField("doubleField",  doubleValue));
+            w.addDocument(doc);
+            sliceCounters1[Math.floorMod(Long.hashCode(intValue), max)] ++;
+            sliceCounters2[Math.floorMod(Long.hashCode(doubleValue), max)] ++;
+            keys.add(uuid);
+        }
+        final IndexReader reader = w.getReader();
+        final IndexSearcher searcher = newSearcher(reader);
+
+        for (int id = 0; id < max; id++) {
+            DocValuesSliceQuery query1 =
+                new DocValuesSliceQuery("intField", id, max);
+            assertThat(searcher.count(query1), equalTo(sliceCounters1[id]));
+
+            DocValuesSliceQuery query2 =
+                new DocValuesSliceQuery("doubleField", id, max);
+            assertThat(searcher.count(query2), equalTo(sliceCounters2[id]));
+            searcher.search(query1, new Collector() {
+                @Override
+                public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
+                    return new LeafCollector() {
+                        @Override
+                        public void setScorer(Scorer scorer) throws IOException {
+                        }
+
+                        @Override
+                        public void collect(int doc) throws IOException {
+                            Document d = context.reader().document(doc, Collections.singleton("uuid"));
+                            String uuid = d.get("uuid");
+                            assertThat(keys.contains(uuid), equalTo(true));
+                            keys.remove(uuid);
+                        }
+                    };
+                }
+
+                @Override
+                public boolean needsScores() {
+                    return false;
+                }
+            });
+        }
+        assertThat(keys.size(), equalTo(0));
+        w.close();
+        reader.close();
+        dir.close();
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/slice/SearchSliceIT.java b/core/src/test/java/org/elasticsearch/search/slice/SearchSliceIT.java
new file mode 100644
index 00000000000..ad93d14f21f
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/slice/SearchSliceIT.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.slice;
+
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.search.Scroll;
+import org.elasticsearch.search.SearchContextException;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.sort.SortBuilders;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.concurrent.ExecutionException;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.startsWith;
+
+public class SearchSliceIT extends ESIntegTestCase {
+    private static final int NUM_DOCS = 1000;
+
+    private int setupIndex(boolean withDocs) throws IOException, ExecutionException, InterruptedException {
+        String mapping = XContentFactory.jsonBuilder().
+            startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("invalid_random_kw")
+                            .field("type", "keyword")
+                            .field("doc_values", "false")
+                        .endObject()
+                        .startObject("random_int")
+                            .field("type", "integer")
+                            .field("doc_values", "true")
+                        .endObject()
+                        .startObject("invalid_random_int")
+                            .field("type", "integer")
+                            .field("doc_values", "false")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject().string();
+        int numberOfShards = randomIntBetween(1, 7);
+        assertAcked(client().admin().indices().prepareCreate("test")
+            .setSettings("number_of_shards", numberOfShards)
+            .addMapping("type", mapping));
+        ensureGreen();
+
+        if (withDocs == false) {
+            return numberOfShards;
+        }
+
+        List<IndexRequestBuilder> requests = new ArrayList<>();
+        for (int i = 0; i < NUM_DOCS; i++) {
+            XContentBuilder builder = jsonBuilder();
+            builder.startObject();
+            builder.field("invalid_random_kw", randomAsciiOfLengthBetween(5, 20));
+            builder.field("random_int", randomInt());
+            builder.field("static_int", 0);
+            builder.field("invalid_random_int", randomInt());
+            builder.endObject();
+            requests.add(client().prepareIndex("test", "test").setSource(builder));
+        }
+        indexRandom(true, requests);
+        return numberOfShards;
+    }
+
+    public void testDocIdSort() throws Exception {
+        int numShards = setupIndex(true);
+        SearchResponse sr = client().prepareSearch("test")
+            .setQuery(matchAllQuery())
+            .setSize(0)
+            .get();
+        int numDocs = (int) sr.getHits().getTotalHits();
+        assertThat(numDocs, equalTo(NUM_DOCS));
+        int max = randomIntBetween(2, numShards*3);
+        for (String field : new String[]{"_uid", "random_int", "static_int"}) {
+            int fetchSize = randomIntBetween(10, 100);
+            SearchRequestBuilder request = client().prepareSearch("test")
+                .setQuery(matchAllQuery())
+                .setScroll(new Scroll(TimeValue.timeValueSeconds(10)))
+                .setSize(fetchSize)
+                .addSort(SortBuilders.fieldSort("_doc"));
+            assertSearchSlicesWithScroll(request, field, max);
+        }
+    }
+
+    public void testNumericSort() throws Exception {
+        int numShards = setupIndex(true);
+        SearchResponse sr = client().prepareSearch("test")
+            .setQuery(matchAllQuery())
+            .setSize(0)
+            .get();
+        int numDocs = (int) sr.getHits().getTotalHits();
+        assertThat(numDocs, equalTo(NUM_DOCS));
+
+        int max = randomIntBetween(2, numShards*3);
+        for (String field : new String[]{"_uid", "random_int", "static_int"}) {
+            int fetchSize = randomIntBetween(10, 100);
+            SearchRequestBuilder request = client().prepareSearch("test")
+                .setQuery(matchAllQuery())
+                .setScroll(new Scroll(TimeValue.timeValueSeconds(10)))
+                .addSort(SortBuilders.fieldSort("random_int"))
+                .setSize(fetchSize);
+            assertSearchSlicesWithScroll(request, field, max);
+        }
+    }
+
+    public void testInvalidFields() throws Exception {
+        setupIndex(false);
+        SearchPhaseExecutionException exc = expectThrows(SearchPhaseExecutionException.class,
+            () -> client().prepareSearch("test")
+                .setQuery(matchAllQuery())
+                .setScroll(new Scroll(TimeValue.timeValueSeconds(10)))
+                .slice(new SliceBuilder("invalid_random_int", 0, 10))
+                .get());
+        Throwable rootCause = findRootCause(exc);
+        assertThat(rootCause.getClass(), equalTo(IllegalArgumentException.class));
+        assertThat(rootCause.getMessage(),
+            startsWith("cannot load numeric doc values"));
+
+        exc = expectThrows(SearchPhaseExecutionException.class, () -> client().prepareSearch("test")
+            .setQuery(matchAllQuery())
+            .setScroll(new Scroll(TimeValue.timeValueSeconds(10)))
+            .slice(new SliceBuilder("invalid_random_kw", 0, 10))
+            .get());
+        rootCause = findRootCause(exc);
+        assertThat(rootCause.getClass(), equalTo(IllegalArgumentException.class));
+        assertThat(rootCause.getMessage(),
+            startsWith("cannot load numeric doc values"));
+    }
+
+    public void testInvalidQuery() throws Exception {
+        setupIndex(false);
+        SearchPhaseExecutionException exc = expectThrows(SearchPhaseExecutionException.class,
+            () -> client().prepareSearch()
+                .setQuery(matchAllQuery())
+                .slice(new SliceBuilder("invalid_random_int", 0, 10))
+                .get());
+        Throwable rootCause = findRootCause(exc);
+        assertThat(rootCause.getClass(), equalTo(SearchContextException.class));
+        assertThat(rootCause.getMessage(),
+            equalTo("`slice` cannot be used outside of a scroll context"));
+    }
+
+    private void assertSearchSlicesWithScroll(SearchRequestBuilder request, String field, int numSlice) {
+        int totalResults = 0;
+        List<String> keys = new ArrayList<>();
+        for (int id = 0; id < numSlice; id++) {
+            SliceBuilder sliceBuilder = new SliceBuilder(field, id, numSlice);
+            SearchResponse searchResponse = request.slice(sliceBuilder).get();
+            totalResults += searchResponse.getHits().getHits().length;
+            int expectedSliceResults = (int) searchResponse.getHits().getTotalHits();
+            int numSliceResults = searchResponse.getHits().getHits().length;
+            String scrollId = searchResponse.getScrollId();
+            for (SearchHit hit : searchResponse.getHits().getHits()) {
+                keys.add(hit.getId());
+            }
+            while (searchResponse.getHits().getHits().length > 0) {
+                searchResponse = client().prepareSearchScroll("test")
+                    .setScrollId(scrollId)
+                    .setScroll(new Scroll(TimeValue.timeValueSeconds(10)))
+                    .get();
+                scrollId = searchResponse.getScrollId();
+                totalResults += searchResponse.getHits().getHits().length;
+                numSliceResults += searchResponse.getHits().getHits().length;
+                for (SearchHit hit : searchResponse.getHits().getHits()) {
+                    keys.add(hit.getId());
+                }
+            }
+            assertThat(numSliceResults, equalTo(expectedSliceResults));
+            clearScroll(scrollId);
+        }
+        assertThat(totalResults, equalTo(NUM_DOCS));
+        assertThat(keys.size(), equalTo(NUM_DOCS));
+        assertThat(new HashSet(keys).size(), equalTo(NUM_DOCS));
+    }
+
+    private Throwable findRootCause(Exception e) {
+        Throwable ret = e;
+        while (ret.getCause() != null) {
+            ret = ret.getCause();
+        }
+        return ret;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
new file mode 100644
index 00000000000..554b9436e58
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
@@ -0,0 +1,340 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.slice;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.RAMDirectory;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.fielddata.IndexNumericFieldData;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.internal.UidFieldMapper;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryParser;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.containsString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class SliceBuilderTests extends ESTestCase {
+    private static final int MAX_SLICE = 20;
+    private static NamedWriteableRegistry namedWriteableRegistry;
+    private static IndicesQueriesRegistry indicesQueriesRegistry;
+
+    /**
+     * setup for the whole base test class
+     */
+    @BeforeClass
+    public static void init() {
+        namedWriteableRegistry = new NamedWriteableRegistry();
+        indicesQueriesRegistry = new IndicesQueriesRegistry();
+        QueryParser<MatchAllQueryBuilder> parser = MatchAllQueryBuilder::fromXContent;
+        indicesQueriesRegistry.register(parser, MatchAllQueryBuilder.QUERY_NAME_FIELD);
+    }
+
+    @AfterClass
+    public static void afterClass() throws Exception {
+        namedWriteableRegistry = null;
+        indicesQueriesRegistry = null;
+    }
+
+    private final SliceBuilder randomSliceBuilder() throws IOException {
+        int max = randomIntBetween(2, MAX_SLICE);
+        if (max == 0) max++;
+        int id = randomInt(max - 1);
+        String field = randomAsciiOfLengthBetween(5, 20);
+        return new SliceBuilder(field, id, max);
+    }
+
+    private static SliceBuilder serializedCopy(SliceBuilder original) throws IOException {
+        try (BytesStreamOutput output = new BytesStreamOutput()) {
+            original.writeTo(output);
+            try (StreamInput in =
+                     new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
+                return new SliceBuilder(in);
+            }
+        }
+    }
+
+    public void testSerialization() throws Exception {
+        SliceBuilder original = randomSliceBuilder();
+        SliceBuilder deserialized = serializedCopy(original);
+        assertEquals(deserialized, original);
+        assertEquals(deserialized.hashCode(), original.hashCode());
+        assertNotSame(deserialized, original);
+    }
+
+    public void testEqualsAndHashcode() throws Exception {
+        SliceBuilder firstBuilder = randomSliceBuilder();
+        assertFalse("sliceBuilder is equal to null", firstBuilder.equals(null));
+        assertFalse("sliceBuilder is equal to incompatible type", firstBuilder.equals(""));
+        assertTrue("sliceBuilder is not equal to self", firstBuilder.equals(firstBuilder));
+        assertThat("same searchFrom's hashcode returns different values if called multiple times",
+            firstBuilder.hashCode(), equalTo(firstBuilder.hashCode()));
+
+        SliceBuilder secondBuilder = serializedCopy(firstBuilder);
+        assertTrue("sliceBuilder is not equal to self", secondBuilder.equals(secondBuilder));
+        assertTrue("sliceBuilder is not equal to its copy", firstBuilder.equals(secondBuilder));
+        assertTrue("equals is not symmetric", secondBuilder.equals(firstBuilder));
+        assertThat("sliceBuilder copy's hashcode is different from original hashcode", secondBuilder.hashCode(),
+            equalTo(firstBuilder.hashCode()));
+        SliceBuilder thirdBuilder = serializedCopy(secondBuilder);
+        assertTrue("sliceBuilder is not equal to self", thirdBuilder.equals(thirdBuilder));
+        assertTrue("sliceBuilder is not equal to its copy", secondBuilder.equals(thirdBuilder));
+        assertThat("sliceBuilder copy's hashcode is different from original hashcode", secondBuilder.hashCode(),
+            equalTo(thirdBuilder.hashCode()));
+        assertTrue("equals is not transitive", firstBuilder.equals(thirdBuilder));
+        assertThat("sliceBuilder copy's hashcode is different from original hashcode", firstBuilder.hashCode(),
+            equalTo(thirdBuilder.hashCode()));
+        assertTrue("sliceBuilder is not symmetric", thirdBuilder.equals(secondBuilder));
+        assertTrue("sliceBuilder is not symmetric", thirdBuilder.equals(firstBuilder));
+    }
+
+    public void testFromXContent() throws Exception {
+        SliceBuilder sliceBuilder = randomSliceBuilder();
+        XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
+        if (randomBoolean()) {
+            builder.prettyPrint();
+        }
+        builder.startObject();
+        sliceBuilder.innerToXContent(builder);
+        builder.endObject();
+        XContentParser parser = XContentHelper.createParser(shuffleXContent(builder).bytes());
+        QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser,
+            ParseFieldMatcher.STRICT);
+        SliceBuilder secondSliceBuilder = SliceBuilder.fromXContent(context);
+        assertNotSame(sliceBuilder, secondSliceBuilder);
+        assertEquals(sliceBuilder, secondSliceBuilder);
+        assertEquals(sliceBuilder.hashCode(), secondSliceBuilder.hashCode());
+    }
+
+    public void testInvalidArguments() throws Exception {
+        Exception e = expectThrows(IllegalArgumentException.class, () -> new SliceBuilder("field", -1, 10));
+        assertEquals(e.getMessage(), "id must be greater than or equal to 0");
+
+        e = expectThrows(IllegalArgumentException.class, () -> new SliceBuilder("field", 10, -1));
+        assertEquals(e.getMessage(), "max must be greater than 1");
+
+        e = expectThrows(IllegalArgumentException.class, () -> new SliceBuilder("field", 10, 0));
+        assertEquals(e.getMessage(), "max must be greater than 1");
+
+        e = expectThrows(IllegalArgumentException.class, () -> new SliceBuilder("field", 10, 5));
+        assertEquals(e.getMessage(), "max must be greater than id");
+
+        e = expectThrows(IllegalArgumentException.class, () -> new SliceBuilder("field", 1000, 1000));
+        assertEquals(e.getMessage(), "max must be greater than id");
+        e = expectThrows(IllegalArgumentException.class, () -> new SliceBuilder("field", 1001, 1000));
+        assertEquals(e.getMessage(), "max must be greater than id");
+    }
+
+    public void testToFilter() throws IOException {
+        Directory dir = new RAMDirectory();
+        try (IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())))) {
+            writer.commit();
+        }
+        QueryShardContext context = mock(QueryShardContext.class);
+        try (IndexReader reader = DirectoryReader.open(dir)) {
+            MappedFieldType fieldType = new MappedFieldType() {
+                @Override
+                public MappedFieldType clone() {
+                    return null;
+                }
+
+                @Override
+                public String typeName() {
+                    return null;
+                }
+
+                @Override
+                public Query termQuery(Object value, @Nullable QueryShardContext context) {
+                    return null;
+                }
+            };
+            fieldType.setName(UidFieldMapper.NAME);
+            fieldType.setHasDocValues(false);
+            when(context.fieldMapper(UidFieldMapper.NAME)).thenReturn(fieldType);
+            when(context.getIndexReader()).thenReturn(reader);
+            SliceBuilder builder = new SliceBuilder(5, 10);
+            Query query = builder.toFilter(context, 0, 1);
+            assertThat(query, instanceOf(TermsSliceQuery.class));
+
+            assertThat(builder.toFilter(context, 0, 1), equalTo(query));
+            try (IndexReader newReader = DirectoryReader.open(dir)) {
+                when(context.getIndexReader()).thenReturn(newReader);
+                assertThat(builder.toFilter(context, 0, 1), equalTo(query));
+            }
+        }
+
+        try (IndexReader reader = DirectoryReader.open(dir)) {
+            MappedFieldType fieldType = new MappedFieldType() {
+                @Override
+                public MappedFieldType clone() {
+                    return null;
+                }
+
+                @Override
+                public String typeName() {
+                    return null;
+                }
+
+                @Override
+                public Query termQuery(Object value, @Nullable QueryShardContext context) {
+                    return null;
+                }
+            };
+            fieldType.setName("field_doc_values");
+            fieldType.setHasDocValues(true);
+            fieldType.setDocValuesType(DocValuesType.SORTED_NUMERIC);
+            when(context.fieldMapper("field_doc_values")).thenReturn(fieldType);
+            when(context.getIndexReader()).thenReturn(reader);
+            IndexNumericFieldData fd = mock(IndexNumericFieldData.class);
+            when(context.getForField(fieldType)).thenReturn(fd);
+            SliceBuilder builder = new SliceBuilder("field_doc_values", 5, 10);
+            Query query = builder.toFilter(context, 0, 1);
+            assertThat(query, instanceOf(DocValuesSliceQuery.class));
+
+            assertThat(builder.toFilter(context, 0, 1), equalTo(query));
+            try (IndexReader newReader = DirectoryReader.open(dir)) {
+                when(context.getIndexReader()).thenReturn(newReader);
+                assertThat(builder.toFilter(context, 0, 1), equalTo(query));
+            }
+
+            // numSlices > numShards
+            int numSlices = randomIntBetween(10, 100);
+            int numShards = randomIntBetween(1, 9);
+            Map<Integer, AtomicInteger> numSliceMap = new HashMap<>();
+            for (int i = 0; i < numSlices; i++) {
+                for (int j = 0; j < numShards; j++) {
+                    SliceBuilder slice = new SliceBuilder("_uid", i, numSlices);
+                    Query q = slice.toFilter(context, j, numShards);
+                    if (q instanceof TermsSliceQuery || q instanceof MatchAllDocsQuery) {
+                        AtomicInteger count = numSliceMap.get(j);
+                        if (count == null) {
+                            count = new AtomicInteger(0);
+                            numSliceMap.put(j, count);
+                        }
+                        count.incrementAndGet();
+                        if (q instanceof MatchAllDocsQuery) {
+                            assertThat(count.get(), equalTo(1));
+                        }
+                    } else {
+                        assertThat(q, instanceOf(MatchNoDocsQuery.class));
+                    }
+                }
+            }
+            int total = 0;
+            for (Map.Entry<Integer, AtomicInteger> e : numSliceMap.entrySet()) {
+                total += e.getValue().get();
+            }
+            assertThat(total, equalTo(numSlices));
+
+            // numShards > numSlices
+            numShards = randomIntBetween(3, 100);
+            numSlices = randomInt(numShards-1);
+            List<Integer> targetShards = new ArrayList<>();
+            for (int i = 0; i < numSlices; i++) {
+                for (int j = 0; j < numShards; j++) {
+                    SliceBuilder slice = new SliceBuilder("_uid", i, numSlices);
+                    Query q = slice.toFilter(context, j, numShards);
+                    if (q instanceof MatchNoDocsQuery == false) {
+                        assertThat(q, instanceOf(MatchAllDocsQuery.class));
+                        targetShards.add(j);
+                    }
+                }
+            }
+            assertThat(targetShards.size(), equalTo(numShards));
+            assertThat(new HashSet<>(targetShards).size(), equalTo(numShards));
+
+            // numShards == numSlices
+            numShards = randomIntBetween(2, 10);
+            numSlices = numShards;
+            for (int i = 0; i < numSlices; i++) {
+                for (int j = 0; j < numShards; j++) {
+                    SliceBuilder slice = new SliceBuilder("_uid", i, numSlices);
+                    Query q = slice.toFilter(context, j, numShards);
+                    if (i == j) {
+                        assertThat(q, instanceOf(MatchAllDocsQuery.class));
+                    } else {
+                        assertThat(q, instanceOf(MatchNoDocsQuery.class));
+                    }
+                }
+            }
+        }
+
+        try (IndexReader reader = DirectoryReader.open(dir)) {
+            MappedFieldType fieldType = new MappedFieldType() {
+                @Override
+                public MappedFieldType clone() {
+                    return null;
+                }
+
+                @Override
+                public String typeName() {
+                    return null;
+                }
+
+                @Override
+                public Query termQuery(Object value, @Nullable QueryShardContext context) {
+                    return null;
+                }
+            };
+            fieldType.setName("field_without_doc_values");
+            when(context.fieldMapper("field_without_doc_values")).thenReturn(fieldType);
+            when(context.getIndexReader()).thenReturn(reader);
+            SliceBuilder builder = new SliceBuilder("field_without_doc_values", 5, 10);
+            IllegalArgumentException exc =
+                expectThrows(IllegalArgumentException.class, () -> builder.toFilter(context, 0, 1));
+            assertThat(exc.getMessage(), containsString("cannot load numeric doc values"));
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/slice/TermsSliceQueryTests.java b/core/src/test/java/org/elasticsearch/search/slice/TermsSliceQueryTests.java
new file mode 100644
index 00000000000..e00dabc6363
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/slice/TermsSliceQueryTests.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.slice;
+
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.LeafCollector;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.QueryUtils;
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class TermsSliceQueryTests extends ESTestCase {
+
+    public void testBasics() {
+        TermsSliceQuery query1 =
+            new TermsSliceQuery("field1", 1, 10);
+        TermsSliceQuery query2 =
+            new TermsSliceQuery("field1", 1, 10);
+        TermsSliceQuery query3 =
+            new TermsSliceQuery("field2", 1, 10);
+        TermsSliceQuery query4 =
+            new TermsSliceQuery("field1", 2, 10);
+        QueryUtils.check(query1);
+        QueryUtils.checkEqual(query1, query2);
+        QueryUtils.checkUnequal(query1, query3);
+        QueryUtils.checkUnequal(query1, query4);
+    }
+
+    public void testSearch() throws Exception {
+        final int numDocs = randomIntBetween(100, 200);
+        final Directory dir = newDirectory();
+        final RandomIndexWriter w = new RandomIndexWriter(random(), dir, new KeywordAnalyzer());
+        int max = randomIntBetween(2, 10);
+        int[] sliceCounters = new int[max];
+        Set<String> keys = new HashSet<>();
+        for (int i = 0; i < numDocs; ++i) {
+            Document doc = new Document();
+            String uuid = UUIDs.base64UUID();
+            BytesRef br = new BytesRef(uuid);
+            int id = Math.floorMod(br.hashCode(), max);
+            sliceCounters[id] ++;
+            doc.add(new StringField("uuid", uuid, Field.Store.YES));
+            w.addDocument(doc);
+            keys.add(uuid);
+        }
+        final IndexReader reader = w.getReader();
+        final IndexSearcher searcher = newSearcher(reader);
+
+        for (int id = 0; id < max; id++) {
+            TermsSliceQuery query1 =
+                new TermsSliceQuery("uuid", id, max);
+            assertThat(searcher.count(query1), equalTo(sliceCounters[id]));
+            searcher.search(query1, new Collector() {
+                @Override
+                public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
+                    return new LeafCollector() {
+                        @Override
+                        public void setScorer(Scorer scorer) throws IOException {
+                        }
+
+                        @Override
+                        public void collect(int doc) throws IOException {
+                            Document d = context.reader().document(doc, Collections.singleton("uuid"));
+                            String uuid = d.get("uuid");
+                            assertThat(keys.contains(uuid), equalTo(true));
+                            keys.remove(uuid);
+                        }
+                    };
+                }
+
+                @Override
+                public boolean needsScores() {
+                    return false;
+                }
+            });
+        }
+        assertThat(keys.size(), equalTo(0));
+        w.close();
+        reader.close();
+        dir.close();
+    }
+}
diff --git a/docs/reference/search/request/scroll.asciidoc b/docs/reference/search/request/scroll.asciidoc
index e18593d21cc..9f9558aa979 100644
--- a/docs/reference/search/request/scroll.asciidoc
+++ b/docs/reference/search/request/scroll.asciidoc
@@ -175,3 +175,92 @@ curl -XDELETE localhost:9200/_search/scroll \
      -d 'c2Nhbjs2OzM0NDg1ODpzRlBLc0FXNlNyNm5JWUc1,aGVuRmV0Y2g7NTsxOnkxaDZ'
 ---------------------------------------
 
+
+==== Sliced Scroll
+
+For scroll queries that return a lot of documents it is possible to split the scroll in multiple slices which
+can be consumed independently:
+
+[source,js]
+--------------------------------------------------
+curl -XGET 'localhost:9200/twitter/tweet/_search?scroll=1m' -d '
+{
+    "slice": {
+        "id": 0, <1>
+        "max": 2 <2>
+    },
+    "query": {
+        "match" : {
+            "title" : "elasticsearch"
+        }
+    }
+}
+'
+
+curl -XGET 'localhost:9200/twitter/tweet/_search?scroll=1m' -d '
+{
+    "slice": {
+        "id": 1,
+        "max": 2
+    },
+    "query": {
+        "match" : {
+            "title" : "elasticsearch"
+        }
+    }
+}
+'
+--------------------------------------------------
+
+<1> The id of the slice
+<2> The maximum number of slices
+
+The result from the first request returned documents that belong to the first slice (id: 0) and the result from the
+second request returned documents that belong to the second slice. Since the maximum number of slices is set to 2
+ the union of the results of the two requests is equivalent to the results of a scroll query without slicing.
+By default the splitting is done on the shards first and then locally on each shard using the _uid field
+with the following formula:
+`slice(doc) = floorMod(hashCode(doc._uid), max)`
+For instance if the number of shards is equal to 2 and the user requested 4 slices then the slices 0 and 2 are assigned
+to the first shard and the slices 1 and 3 are assigned to the second shard.
+
+Each scroll is independent and can be processed in parallel like any scroll request.
+
+NOTE: If the number of slices is bigger than the number of shards the slice filter is very slow on the first calls, it has a complexity of O(N) and a memory cost equals
+to N bits per slice where N is the total number of documents in the shard.
+After few calls the filter should be cached and subsequent calls should be faster but you should limit the number of
+ sliced query you perform in parallel to avoid the memory explosion.
+
+To avoid this cost entirely it is possible to use the `doc_values` of another field to do the slicing
+but the user must ensure that the field has the following properties:
+
+    * The field is numeric.
+
+    * `doc_values` are enabled on that field
+
+    * Every document should contain a single value. If a document has multiple values for the specified field, the first value is used.
+
+    * The value for each document should be set once when the document is created and never updated. This ensures that each
+slice gets deterministic results.
+
+    * The cardinality of the field should be high. This ensures that each slice gets approximately the same amount of documents.
+
+[source,js]
+--------------------------------------------------
+curl -XGET 'localhost:9200/twitter/tweet/_search?scroll=1m' -d '
+{
+    "slice": {
+        "field": "my_random_integer_field",
+        "id": 0,
+        "max": 10
+    },
+    "query": {
+        "match" : {
+            "title" : "elasticsearch"
+        }
+    }
+}
+'
+--------------------------------------------------
+
+For append only time-based indices, the `timestamp` field can be used safely.
\ No newline at end of file

From 32965f043a8acfd66e8d9cc781927ff30b10bb3e Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 7 Jun 2016 10:23:49 -0400
Subject: [PATCH 0697/1311] convert FunctionRef/Def usage to methodhandles.

---
 .../java/org/elasticsearch/painless/Def.java  | 20 ++++++-------
 .../elasticsearch/painless/FunctionRef.java   | 28 +++++++++++++------
 .../painless/node/EFunctionRef.java           |  6 ++--
 3 files changed, 30 insertions(+), 24 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index af96e23771f..e7f56bdbeec 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -246,28 +246,24 @@ public final class Def {
          int separator = signature.indexOf('.');
          FunctionRef ref = new FunctionRef(clazz, signature.substring(0, separator), signature.substring(separator+1));
          final CallSite callSite;
-         // XXX: clean all this up to use handles in FunctionRef, deal with ASM in EFunctionRef differently
-         MethodType invokedType = MethodType.fromMethodDescriptorString(ref.invokedType.getDescriptor(), Def.class.getClassLoader());
-         MethodType samMethodType = MethodType.fromMethodDescriptorString(ref.samMethodType.getDescriptor(), Def.class.getClassLoader());
-         MethodType interfaceType = MethodType.fromMethodDescriptorString(ref.interfaceType.getDescriptor(), Def.class.getClassLoader());
-         if (ref.interfaceType.equals(ref.samMethodType)) {
+         if (ref.interfaceType.equals(ref.samType)) {
              callSite = LambdaMetafactory.altMetafactory(lookup, 
                      ref.invokedName, 
-                     invokedType,
-                     samMethodType,
+                     ref.invokedMethodType,
+                     ref.samMethodType,
                      ref.implMethodHandle,
-                     samMethodType,
+                     ref.samMethodType,
                      0);
          } else {
              callSite = LambdaMetafactory.altMetafactory(lookup, 
                      ref.invokedName, 
-                     invokedType,
-                     samMethodType,
+                     ref.invokedMethodType,
+                     ref.samMethodType,
                      ref.implMethodHandle,
-                     samMethodType,
+                     ref.samMethodType,
                      LambdaMetafactory.FLAG_BRIDGES,
                      1,
-                     interfaceType);
+                     ref.interfaceMethodType);
          }
          // we could actually invoke and cache here (in non-capturing cases), but this is not a speedup.
          MethodHandle factory = callSite.dynamicInvoker().asType(MethodType.methodType(clazz.clazz));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
index aa847535cd4..3ac6a29527b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
@@ -25,6 +25,7 @@ import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.Type;
 
 import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodType;
 import java.lang.reflect.Modifier;
 
 /** 
@@ -32,15 +33,19 @@ import java.lang.reflect.Modifier;
  * and reference class + method name
  */
 public class FunctionRef {
-    // XXX: this is a mess, because of ASM versus MethodHandle types
-    // clean all this up, move reflection out of here into definition, etc etc
     public final String invokedName;
-    public final Type invokedType;
-    public final Handle implMethod;
-    public final Type samMethodType;
-    public final Type interfaceType;
 
+    public final Type invokedType;
+    public final MethodType invokedMethodType;
+
+    public final Handle implMethod;
     public final MethodHandle implMethodHandle;
+
+    public final Type samType;
+    public final MethodType samMethodType;
+
+    public final Type interfaceType;
+    public final MethodType interfaceMethodType;
     
     public FunctionRef(Definition.Type expected, String type, String call) {
         boolean isCtorReference = "new".equals(call);
@@ -55,8 +60,10 @@ public class FunctionRef {
         invokedName = method.name;
         // e.g. (Object)Comparator
         invokedType = Type.getMethodType(expected.type);
+        invokedMethodType = MethodType.methodType(expected.clazz);
         // e.g. (Object,Object)int
         interfaceType = Type.getMethodType(method.method.getDescriptor());
+        interfaceMethodType = method.handle.type().dropParameterTypes(0, 1);
         // lookup requested method
         Definition.Struct struct = Definition.getType(type).struct;
         final Definition.Method impl;
@@ -89,15 +96,18 @@ public class FunctionRef {
         implMethod = new Handle(tag, struct.type.getInternalName(), impl.name, impl.method.getDescriptor());
         implMethodHandle = impl.handle;
         if (isCtorReference) {
-            samMethodType = Type.getMethodType(interfaceType.getReturnType(), impl.method.getArgumentTypes());
+            samType = Type.getMethodType(interfaceType.getReturnType(), impl.method.getArgumentTypes());
+            samMethodType = MethodType.methodType(interfaceMethodType.returnType(), impl.handle.type().parameterArray());
         } else if (Modifier.isStatic(impl.modifiers)) {
-            samMethodType = Type.getMethodType(impl.method.getReturnType(), impl.method.getArgumentTypes());
+            samType = Type.getMethodType(impl.method.getReturnType(), impl.method.getArgumentTypes());
+            samMethodType = impl.handle.type();
         } else {
             Type[] argTypes = impl.method.getArgumentTypes();
             Type[] params = new Type[argTypes.length + 1];
             System.arraycopy(argTypes, 0, params, 1, argTypes.length);
             params[0] = struct.type;
-            samMethodType = Type.getMethodType(impl.method.getReturnType(), params);
+            samType = Type.getMethodType(impl.method.getReturnType(), params);
+            samMethodType = impl.handle.type();
         }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index a51ac3ad737..2f8394ebc17 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -68,12 +68,12 @@ public class EFunctionRef extends AExpression {
             writer.writeDebugInfo(location);
             // currently if the interface differs, we ask for a bridge, but maybe we should do smarter checking?
             // either way, stuff will fail if its wrong :)
-            if (ref.interfaceType.equals(ref.samMethodType)) {
+            if (ref.interfaceType.equals(ref.samType)) {
                 writer.invokeDynamic(ref.invokedName, ref.invokedType.getDescriptor(), LAMBDA_BOOTSTRAP_HANDLE, 
-                                     ref.samMethodType, ref.implMethod, ref.samMethodType, 0);
+                                     ref.samType, ref.implMethod, ref.samType, 0);
             } else {
                 writer.invokeDynamic(ref.invokedName, ref.invokedType.getDescriptor(), LAMBDA_BOOTSTRAP_HANDLE, 
-                                     ref.samMethodType, ref.implMethod, ref.samMethodType, 
+                                     ref.samType, ref.implMethod, ref.samType, 
                                      LambdaMetafactory.FLAG_BRIDGES, 1, ref.interfaceType);
             }
         }

From 692c42b23aca06ad83435e66618290c82619aeff Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Tue, 7 Jun 2016 16:29:18 +0200
Subject: [PATCH 0698/1311] Fix ut

---
 .../java/org/elasticsearch/search/simple/SimpleSearchIT.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java
index 6dc10232b21..4220bcb4538 100644
--- a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java
@@ -441,6 +441,6 @@ public class SimpleSearchIT extends ESIntegTestCase {
         assertThat(e.toString(), containsString("Rescore window [" + windowSize + "] is too large. It must "
                 + "be less than [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY)));
         assertThat(e.toString(), containsString(
-                "This limit can be set by chaning the [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey() + "] index level setting."));
+                "This limit can be set by chaining the [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey() + "] index level setting."));
     }
 }

From c0da8e4060b19ef8cdbea8c95514df322a4e07ec Mon Sep 17 00:00:00 2001
From: trangvh <trangvh@users.noreply.github.com>
Date: Tue, 7 Jun 2016 21:41:39 +0700
Subject: [PATCH 0699/1311] Fix some typos (#18746)

* Update java-doc of SearchResponse.getProfileResults()

* Fix a trivial typo in Reference document
---
 .../java/org/elasticsearch/action/search/SearchResponse.java    | 2 +-
 docs/reference/index-modules/allocation/filtering.asciidoc      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java
index ab8395c72a1..b76e640fa95 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java
@@ -167,7 +167,7 @@ public class SearchResponse extends ActionResponse implements StatusToXContent {
      * If profiling was enabled, this returns an object containing the profile results from
      * each shard.  If profiling was not enabled, this will return null
      *
-     * @return The profile results or null
+     * @return The profile results or an empty map
      */
     public @Nullable Map<String, List<ProfileShardResult>> getProfileResults() {
         return internalResponse.profile();
diff --git a/docs/reference/index-modules/allocation/filtering.asciidoc b/docs/reference/index-modules/allocation/filtering.asciidoc
index 05007b46188..0eba1282759 100644
--- a/docs/reference/index-modules/allocation/filtering.asciidoc
+++ b/docs/reference/index-modules/allocation/filtering.asciidoc
@@ -9,7 +9,7 @@ conjunction with the cluster-wide allocation filters explained in
 <<shards-allocation>>.
 
 It is possible to assign arbitrary metadata attributes to each node at
-startup.  For instance, nodes could be assigned a `rack` and a `group`
+startup.  For instance, nodes could be assigned a `rack` and a `size`
 attribute as follows:
 
 [source,sh]

From f611f1c99e5b402ef0c72afb36cc1998851fc5a8 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Fri, 27 May 2016 19:37:53 +0200
Subject: [PATCH 0700/1311] ingest: Move processors from core to ingest-common
 module.

Folded grok processor into ingest-common module.

The rest tests have been moved to ingest-common module as well, because these tests don't run in the rest-api-spec module but in the distribution:integ-test-zip module
and adding a test plugin there felt just wrong to me. I think this is ok. I left a tiny ingest rest test behind in that tests with an empty pipeline.

Removed messy tests, these tests were already covered in the rest tests

Added ingest test plugin in test infra so that each module testing integration with ingest doesn't need write its own plugin

Moved reindex ingest tests to qa module

Closes #18490
---
 .../resources/checkstyle_suppressions.xml     |  17 +--
 .../ingest/SimulateExecutionService.java      |   2 +-
 .../ingest}/TrackingResultProcessor.java      |   3 +-
 .../org/elasticsearch/node/NodeModule.java    |  33 ------
 .../ingest}/TrackingResultProcessorTests.java |   5 +-
 .../elasticsearch/ingest/IngestClientIT.java  |  27 +----
 ...gestProcessorNotInstalledOnAllNodesIT.java |   2 +-
 .../ingest/PipelineStoreTests.java            |  46 +++++++-
 .../ingest/core/PipelineFactoryTests.java     |   3 -
 .../build.gradle                              |   8 +-
 .../licenses/jcodings-1.0.12.jar.sha1         |   0
 .../licenses/jcodings-LICENSE.txt             |   0
 .../licenses/jcodings-NOTICE.txt              |   0
 .../licenses/joni-2.1.6.jar.sha1              |   0
 .../licenses/joni-LICENSE.txt                 |   0
 .../licenses/joni-NOTICE.txt                  |   0
 .../ingest}/AbstractStringProcessor.java      |   2 +-
 .../ingest}/AppendProcessor.java              |   2 +-
 .../ingest}/ConvertProcessor.java             |   5 +-
 .../org/elasticsearch/ingest}/DateFormat.java |   2 +-
 .../ingest}/DateIndexNameProcessor.java       |   2 +-
 .../elasticsearch/ingest}/DateProcessor.java  |   2 +-
 .../elasticsearch/ingest}/FailProcessor.java  |   2 +-
 .../ingest}/FailProcessorException.java       |   2 +-
 .../ingest}/ForEachProcessor.java             |   3 +-
 .../java/org/elasticsearch/ingest}/Grok.java  |  11 +-
 .../elasticsearch/ingest}/GrokMatchGroup.java |   2 +-
 .../elasticsearch/ingest}/GrokProcessor.java  |   2 +-
 .../elasticsearch/ingest}/GsubProcessor.java  |   4 +-
 .../ingest/IngestCommonPlugin.java            | 107 ++++++++++++++++++
 .../elasticsearch/ingest}/JoinProcessor.java  |   2 +-
 .../ingest}/LowercaseProcessor.java           |   2 +-
 .../ingest}/RemoveProcessor.java              |   2 +-
 .../ingest}/RenameProcessor.java              |   2 +-
 .../elasticsearch/ingest}/SetProcessor.java   |   2 +-
 .../elasticsearch/ingest}/SortProcessor.java  |   5 +-
 .../elasticsearch/ingest}/SplitProcessor.java |   2 +-
 .../elasticsearch/ingest}/TrimProcessor.java  |   2 +-
 .../ingest}/UppercaseProcessor.java           |   2 +-
 .../src/main/resources/patterns/aws           |   0
 .../src/main/resources/patterns/bacula        |   0
 .../src/main/resources/patterns/bro           |   0
 .../src/main/resources/patterns/exim          |   0
 .../src/main/resources/patterns/firewalls     |   0
 .../src/main/resources/patterns/grok-patterns |   0
 .../src/main/resources/patterns/haproxy       |   0
 .../src/main/resources/patterns/java          |   0
 .../src/main/resources/patterns/junos         |   0
 .../src/main/resources/patterns/linux-syslog  |   0
 .../resources/patterns/mcollective-patterns   |   0
 .../src/main/resources/patterns/mongodb       |   0
 .../src/main/resources/patterns/nagios        |   0
 .../src/main/resources/patterns/postgresql    |   0
 .../src/main/resources/patterns/rails         |   0
 .../src/main/resources/patterns/redis         |   0
 .../src/main/resources/patterns/ruby          |   0
 .../AbstractStringProcessorTestCase.java      |   6 +-
 .../ingest}/AppendProcessorFactoryTests.java  |   3 +-
 .../ingest}/AppendProcessorTests.java         |   7 +-
 .../ingest}/ConvertProcessorFactoryTests.java |   2 +-
 .../ingest}/ConvertProcessorTests.java        |   5 +-
 .../ingest}/DateFormatTests.java              |  11 +-
 .../ingest}/DateIndexNameFactoryTests.java    |   2 +-
 .../ingest}/DateIndexNameProcessorTests.java  |   2 +-
 .../ingest}/DateProcessorFactoryTests.java    |   4 +-
 .../ingest}/DateProcessorTests.java           |   6 +-
 .../ingest}/FailProcessorFactoryTests.java    |   4 +-
 .../ingest}/FailProcessorTests.java           |   4 +-
 .../ingest}/ForEachProcessorFactoryTests.java |   5 +-
 .../ingest}/ForEachProcessorTests.java        |   4 +-
 .../ingest}/GrokProcessorFactoryTests.java    |   2 +-
 .../ingest}/GrokProcessorTests.java           |   2 +-
 .../org/elasticsearch/ingest}/GrokTests.java  |  51 ++++++---
 .../ingest}/GsubProcessorFactoryTests.java    |   2 +-
 .../ingest}/GsubProcessorTests.java           |   6 +-
 .../ingest/IngestCommonRestIT.java}           |   9 +-
 .../ingest}/JoinProcessorFactoryTests.java    |   3 +-
 .../ingest}/JoinProcessorTests.java           |   3 +-
 .../LowercaseProcessorFactoryTests.java       |   3 +-
 .../ingest}/LowercaseProcessorTests.java      |   2 +-
 .../ingest}/RemoveProcessorFactoryTests.java  |   4 +-
 .../ingest}/RemoveProcessorTests.java         |   4 +-
 .../ingest}/RenameProcessorFactoryTests.java  |   2 +-
 .../ingest}/RenameProcessorTests.java         |   6 +-
 .../ingest}/SetProcessorFactoryTests.java     |   3 +-
 .../ingest}/SetProcessorTests.java            |  10 +-
 .../ingest}/SortProcessorTests.java           |   7 +-
 .../ingest}/SplitProcessorFactoryTests.java   |   3 +-
 .../ingest}/SplitProcessorTests.java          |   9 +-
 .../ingest}/TrimProcessorFactoryTests.java    |   3 +-
 .../ingest}/TrimProcessorTests.java           |   2 +-
 .../UppercaseProcessorFactoryTests.java       |   3 +-
 .../ingest}/UppercaseProcessorTests.java      |   2 +-
 .../ingest/100_date_index_name_processor.yaml |   0
 .../rest-api-spec/test/ingest}/10_basic.yaml  |   4 +-
 .../rest-api-spec/test/ingest/110_sort.yaml   |   0
 .../rest-api-spec/test/ingest/120_grok.yaml}  |   0
 .../rest-api-spec/test/ingest/20_crud.yaml    |   0
 .../test/ingest/30_date_processor.yaml        |   0
 .../rest-api-spec/test/ingest/40_mutate.yaml  |   0
 .../test/ingest/50_on_failure.yaml            |   0
 .../rest-api-spec/test/ingest/60_fail.yaml    |   0
 .../rest-api-spec/test/ingest/70_bulk.yaml    |   0
 .../rest-api-spec/test/ingest/80_foreach.yaml |   0
 .../test/ingest/90_simulate.yaml              |   0
 .../ingest/grok/IngestGrokPlugin.java         |  87 --------------
 .../index/reindex/CancelTests.java            |  11 +-
 .../{80_throttle.yaml => 70_throttle.yaml}    |   0
 .../{70_throttle.yaml => 60_throttle.yaml}    |   0
 .../test/ingest_attachment/10_basic.yaml      |  18 +--
 .../test/ingest_geoip/10_basic.yaml           |  18 +--
 qa/smoke-test-ingest-disabled/build.gradle    |   4 +
 .../build.gradle                              |   3 +-
 .../IngestMustacheRemoveProcessorIT.java      |  38 -------
 .../ingest/IngestMustacheSetProcessorIT.java  |  74 ------------
 .../test/ingest/30_with_ingest.yaml           |   0
 .../test/ingest/40_with_ingest.yaml           |   0
 .../rest-api-spec/test/ingest/10_basic.yaml   |  45 ++++++++
 settings.gradle                               |   2 +-
 .../ingest/IngestTestPlugin.java              |  50 ++++++++
 120 files changed, 407 insertions(+), 468 deletions(-)
 rename core/src/main/java/org/elasticsearch/{ingest/processor => action/ingest}/TrackingResultProcessor.java (97%)
 rename core/src/test/java/org/elasticsearch/{ingest/processor => action/ingest}/TrackingResultProcessorTests.java (97%)
 rename modules/{ingest-grok => ingest-common}/build.gradle (77%)
 rename modules/{ingest-grok => ingest-common}/licenses/jcodings-1.0.12.jar.sha1 (100%)
 rename modules/{ingest-grok => ingest-common}/licenses/jcodings-LICENSE.txt (100%)
 rename modules/{ingest-grok => ingest-common}/licenses/jcodings-NOTICE.txt (100%)
 rename modules/{ingest-grok => ingest-common}/licenses/joni-2.1.6.jar.sha1 (100%)
 rename modules/{ingest-grok => ingest-common}/licenses/joni-LICENSE.txt (100%)
 rename modules/{ingest-grok => ingest-common}/licenses/joni-NOTICE.txt (100%)
 rename {core/src/main/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/main/java/org/elasticsearch/ingest}/AbstractStringProcessor.java (98%)
 rename {core/src/main/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/main/java/org/elasticsearch/ingest}/AppendProcessor.java (98%)
 rename {core/src/main/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/main/java/org/elasticsearch/ingest}/ConvertProcessor.java (97%)
 rename {core/src/main/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/main/java/org/elasticsearch/ingest}/DateFormat.java (98%)
 rename {core/src/main/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/main/java/org/elasticsearch/ingest}/DateIndexNameProcessor.java (99%)
 rename {core/src/main/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/main/java/org/elasticsearch/ingest}/DateProcessor.java (99%)
 rename {core/src/main/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/main/java/org/elasticsearch/ingest}/FailProcessor.java (98%)
 rename {core/src/main/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/main/java/org/elasticsearch/ingest}/FailProcessorException.java (96%)
 rename {core/src/main/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/main/java/org/elasticsearch/ingest}/ForEachProcessor.java (97%)
 rename modules/{ingest-grok/src/main/java/org/elasticsearch/ingest/grok => ingest-common/src/main/java/org/elasticsearch/ingest}/Grok.java (93%)
 rename modules/{ingest-grok/src/main/java/org/elasticsearch/ingest/grok => ingest-common/src/main/java/org/elasticsearch/ingest}/GrokMatchGroup.java (97%)
 rename modules/{ingest-grok/src/main/java/org/elasticsearch/ingest/grok => ingest-common/src/main/java/org/elasticsearch/ingest}/GrokProcessor.java (99%)
 rename {core/src/main/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/main/java/org/elasticsearch/ingest}/GsubProcessor.java (95%)
 create mode 100644 modules/ingest-common/src/main/java/org/elasticsearch/ingest/IngestCommonPlugin.java
 rename {core/src/main/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/main/java/org/elasticsearch/ingest}/JoinProcessor.java (98%)
 rename {core/src/main/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/main/java/org/elasticsearch/ingest}/LowercaseProcessor.java (97%)
 rename {core/src/main/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/main/java/org/elasticsearch/ingest}/RemoveProcessor.java (98%)
 rename {core/src/main/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/main/java/org/elasticsearch/ingest}/RenameProcessor.java (98%)
 rename {core/src/main/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/main/java/org/elasticsearch/ingest}/SetProcessor.java (98%)
 rename {core/src/main/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/main/java/org/elasticsearch/ingest}/SortProcessor.java (96%)
 rename {core/src/main/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/main/java/org/elasticsearch/ingest}/SplitProcessor.java (98%)
 rename {core/src/main/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/main/java/org/elasticsearch/ingest}/TrimProcessor.java (97%)
 rename {core/src/main/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/main/java/org/elasticsearch/ingest}/UppercaseProcessor.java (97%)
 rename modules/{ingest-grok => ingest-common}/src/main/resources/patterns/aws (100%)
 rename modules/{ingest-grok => ingest-common}/src/main/resources/patterns/bacula (100%)
 rename modules/{ingest-grok => ingest-common}/src/main/resources/patterns/bro (100%)
 rename modules/{ingest-grok => ingest-common}/src/main/resources/patterns/exim (100%)
 rename modules/{ingest-grok => ingest-common}/src/main/resources/patterns/firewalls (100%)
 rename modules/{ingest-grok => ingest-common}/src/main/resources/patterns/grok-patterns (100%)
 rename modules/{ingest-grok => ingest-common}/src/main/resources/patterns/haproxy (100%)
 rename modules/{ingest-grok => ingest-common}/src/main/resources/patterns/java (100%)
 rename modules/{ingest-grok => ingest-common}/src/main/resources/patterns/junos (100%)
 rename modules/{ingest-grok => ingest-common}/src/main/resources/patterns/linux-syslog (100%)
 rename modules/{ingest-grok => ingest-common}/src/main/resources/patterns/mcollective-patterns (100%)
 rename modules/{ingest-grok => ingest-common}/src/main/resources/patterns/mongodb (100%)
 rename modules/{ingest-grok => ingest-common}/src/main/resources/patterns/nagios (100%)
 rename modules/{ingest-grok => ingest-common}/src/main/resources/patterns/postgresql (100%)
 rename modules/{ingest-grok => ingest-common}/src/main/resources/patterns/rails (100%)
 rename modules/{ingest-grok => ingest-common}/src/main/resources/patterns/redis (100%)
 rename modules/{ingest-grok => ingest-common}/src/main/resources/patterns/ruby (100%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/AbstractStringProcessorTestCase.java (95%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/AppendProcessorFactoryTests.java (97%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/AppendProcessorTests.java (97%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/ConvertProcessorFactoryTests.java (99%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/ConvertProcessorTests.java (99%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/DateFormatTests.java (90%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/DateIndexNameFactoryTests.java (98%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/DateIndexNameProcessorTests.java (98%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/DateProcessorFactoryTests.java (98%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/DateProcessorTests.java (97%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/FailProcessorFactoryTests.java (93%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/FailProcessorTests.java (91%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/ForEachProcessorFactoryTests.java (93%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/ForEachProcessorTests.java (98%)
 rename modules/{ingest-grok/src/test/java/org/elasticsearch/ingest/grok => ingest-common/src/test/java/org/elasticsearch/ingest}/GrokProcessorFactoryTests.java (99%)
 rename modules/{ingest-grok/src/test/java/org/elasticsearch/ingest/grok => ingest-common/src/test/java/org/elasticsearch/ingest}/GrokProcessorTests.java (99%)
 rename modules/{ingest-grok/src/test/java/org/elasticsearch/ingest/grok => ingest-common/src/test/java/org/elasticsearch/ingest}/GrokTests.java (79%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/GsubProcessorFactoryTests.java (98%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/GsubProcessorTests.java (95%)
 rename modules/{ingest-grok/src/test/java/org/elasticsearch/ingest/grok/IngestGrokRestIT.java => ingest-common/src/test/java/org/elasticsearch/ingest/IngestCommonRestIT.java} (81%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/JoinProcessorFactoryTests.java (96%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/JoinProcessorTests.java (98%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/LowercaseProcessorFactoryTests.java (95%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/LowercaseProcessorTests.java (96%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/RemoveProcessorFactoryTests.java (93%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/RemoveProcessorTests.java (93%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/RenameProcessorFactoryTests.java (98%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/RenameProcessorTests.java (97%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/SetProcessorFactoryTests.java (97%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/SetProcessorTests.java (94%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/SortProcessorTests.java (98%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/SplitProcessorFactoryTests.java (96%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/SplitProcessorTests.java (94%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/TrimProcessorFactoryTests.java (95%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/TrimProcessorTests.java (97%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/UppercaseProcessorFactoryTests.java (95%)
 rename {core/src/test/java/org/elasticsearch/ingest/processor => modules/ingest-common/src/test/java/org/elasticsearch/ingest}/UppercaseProcessorTests.java (96%)
 rename rest-api-spec/src/main/resources/rest-api-spec/test/ingest/90_date_index_name_processor.yaml => modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yaml (100%)
 rename modules/{ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok => ingest-common/src/test/resources/rest-api-spec/test/ingest}/10_basic.yaml (93%)
 rename rest-api-spec/src/main/resources/rest-api-spec/test/ingest/100_sort.yaml => modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/110_sort.yaml (100%)
 rename modules/{ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok/20_grok.yaml => ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yaml} (100%)
 rename rest-api-spec/src/main/resources/rest-api-spec/test/ingest/10_crud.yaml => modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yaml (100%)
 rename rest-api-spec/src/main/resources/rest-api-spec/test/ingest/20_date_processor.yaml => modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yaml (100%)
 rename rest-api-spec/src/main/resources/rest-api-spec/test/ingest/30_mutate.yaml => modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/40_mutate.yaml (100%)
 rename {rest-api-spec/src/main => modules/ingest-common/src/test}/resources/rest-api-spec/test/ingest/50_on_failure.yaml (100%)
 rename {rest-api-spec/src/main => modules/ingest-common/src/test}/resources/rest-api-spec/test/ingest/60_fail.yaml (100%)
 rename {rest-api-spec/src/main => modules/ingest-common/src/test}/resources/rest-api-spec/test/ingest/70_bulk.yaml (100%)
 rename {rest-api-spec/src/main => modules/ingest-common/src/test}/resources/rest-api-spec/test/ingest/80_foreach.yaml (100%)
 rename rest-api-spec/src/main/resources/rest-api-spec/test/ingest/40_simulate.yaml => modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/90_simulate.yaml (100%)
 delete mode 100644 modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/IngestGrokPlugin.java
 rename modules/reindex/src/test/resources/rest-api-spec/test/reindex/{80_throttle.yaml => 70_throttle.yaml} (100%)
 rename modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/{70_throttle.yaml => 60_throttle.yaml} (100%)
 delete mode 100644 qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestMustacheRemoveProcessorIT.java
 delete mode 100644 qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestMustacheSetProcessorIT.java
 rename modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_with_ingest.yaml => qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/30_with_ingest.yaml (100%)
 rename modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_with_ingest.yaml => qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/40_with_ingest.yaml (100%)
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/ingest/10_basic.yaml
 create mode 100644 test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 3c5caf2d4e2..9bd0755ffcc 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -501,7 +501,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]CompoundProcessor.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]IngestDocument.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]Pipeline.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]ConvertProcessor.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]GcNames.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]HotThreads.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]JvmStats.java" checks="LineLength" />
@@ -1035,14 +1034,11 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]store[/\\]IndicesStoreIntegrationIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]store[/\\]IndicesStoreTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]template[/\\]SimpleIndexTemplateIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]AbstractStringProcessorTestCase.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]AppendProcessorTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]DateFormatTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]DateProcessorTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]GsubProcessorTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]RenameProcessorTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]SetProcessorTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]processor[/\\]SplitProcessorTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]PipelineExecutionServiceTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]PipelineStoreTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]CompoundProcessorTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]PipelineFactoryTests.java" checks="LineLength" />
+  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]ValueSourceTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]mget[/\\]SimpleMgetIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]JvmGcMonitorServiceSettingsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]os[/\\]OsProbeTests.java" checks="LineLength" />
@@ -1154,9 +1150,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]update[/\\]UpdateIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]validate[/\\]SimpleValidateQueryIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]versioning[/\\]SimpleVersioningIT.java" checks="LineLength" />
-  <suppress files="modules[/\\]ingest-grok[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]grok[/\\]Grok.java" checks="LineLength" />
-  <suppress files="modules[/\\]ingest-grok[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]grok[/\\]GrokProcessorTests.java" checks="LineLength" />
-  <suppress files="modules[/\\]ingest-grok[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]grok[/\\]GrokTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-expression[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]ExpressionPlugin.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-expression[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]ExpressionScriptEngineService.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-expression[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]ExpressionSearchScript.java" checks="LineLength" />
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java
index bc40a8368f0..2636dcb3ba8 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java
@@ -29,7 +29,7 @@ import org.elasticsearch.threadpool.ThreadPool;
 import java.util.ArrayList;
 import java.util.List;
 
-import static org.elasticsearch.ingest.processor.TrackingResultProcessor.decorate;
+import static org.elasticsearch.action.ingest.TrackingResultProcessor.decorate;
 
 class SimulateExecutionService {
 
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/TrackingResultProcessor.java b/core/src/main/java/org/elasticsearch/action/ingest/TrackingResultProcessor.java
similarity index 97%
rename from core/src/main/java/org/elasticsearch/ingest/processor/TrackingResultProcessor.java
rename to core/src/main/java/org/elasticsearch/action/ingest/TrackingResultProcessor.java
index 861b0d68d2d..3693b5dcac7 100644
--- a/core/src/main/java/org/elasticsearch/ingest/processor/TrackingResultProcessor.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/TrackingResultProcessor.java
@@ -17,9 +17,8 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.action.ingest;
 
-import org.elasticsearch.action.ingest.SimulateProcessorResult;
 import org.elasticsearch.ingest.core.CompoundProcessor;
 import org.elasticsearch.ingest.core.IngestDocument;
 import org.elasticsearch.ingest.core.Processor;
diff --git a/core/src/main/java/org/elasticsearch/node/NodeModule.java b/core/src/main/java/org/elasticsearch/node/NodeModule.java
index 08b60970229..a9696593ec4 100644
--- a/core/src/main/java/org/elasticsearch/node/NodeModule.java
+++ b/core/src/main/java/org/elasticsearch/node/NodeModule.java
@@ -24,22 +24,6 @@ import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.ingest.ProcessorsRegistry;
 import org.elasticsearch.ingest.core.Processor;
 import org.elasticsearch.ingest.core.TemplateService;
-import org.elasticsearch.ingest.processor.AppendProcessor;
-import org.elasticsearch.ingest.processor.ConvertProcessor;
-import org.elasticsearch.ingest.processor.DateProcessor;
-import org.elasticsearch.ingest.processor.DateIndexNameProcessor;
-import org.elasticsearch.ingest.processor.FailProcessor;
-import org.elasticsearch.ingest.processor.ForEachProcessor;
-import org.elasticsearch.ingest.processor.GsubProcessor;
-import org.elasticsearch.ingest.processor.JoinProcessor;
-import org.elasticsearch.ingest.processor.LowercaseProcessor;
-import org.elasticsearch.ingest.processor.RemoveProcessor;
-import org.elasticsearch.ingest.processor.RenameProcessor;
-import org.elasticsearch.ingest.processor.SetProcessor;
-import org.elasticsearch.ingest.processor.SortProcessor;
-import org.elasticsearch.ingest.processor.SplitProcessor;
-import org.elasticsearch.ingest.processor.TrimProcessor;
-import org.elasticsearch.ingest.processor.UppercaseProcessor;
 import org.elasticsearch.monitor.MonitorService;
 import org.elasticsearch.node.service.NodeService;
 
@@ -61,23 +45,6 @@ public class NodeModule extends AbstractModule {
         this.node = node;
         this.monitorService = monitorService;
         this.processorsRegistryBuilder = new ProcessorsRegistry.Builder();
-
-        registerProcessor(DateProcessor.TYPE, (templateService, registry) -> new DateProcessor.Factory());
-        registerProcessor(SetProcessor.TYPE, (templateService, registry) -> new SetProcessor.Factory(templateService));
-        registerProcessor(AppendProcessor.TYPE, (templateService, registry) -> new AppendProcessor.Factory(templateService));
-        registerProcessor(RenameProcessor.TYPE, (templateService, registry) -> new RenameProcessor.Factory());
-        registerProcessor(RemoveProcessor.TYPE, (templateService, registry) -> new RemoveProcessor.Factory(templateService));
-        registerProcessor(SplitProcessor.TYPE, (templateService, registry) -> new SplitProcessor.Factory());
-        registerProcessor(JoinProcessor.TYPE, (templateService, registry) -> new JoinProcessor.Factory());
-        registerProcessor(UppercaseProcessor.TYPE, (templateService, registry) -> new UppercaseProcessor.Factory());
-        registerProcessor(LowercaseProcessor.TYPE, (templateService, registry) -> new LowercaseProcessor.Factory());
-        registerProcessor(TrimProcessor.TYPE, (templateService, registry) -> new TrimProcessor.Factory());
-        registerProcessor(ConvertProcessor.TYPE, (templateService, registry) -> new ConvertProcessor.Factory());
-        registerProcessor(GsubProcessor.TYPE, (templateService, registry) -> new GsubProcessor.Factory());
-        registerProcessor(FailProcessor.TYPE, (templateService, registry) -> new FailProcessor.Factory(templateService));
-        registerProcessor(ForEachProcessor.TYPE, (templateService, registry) -> new ForEachProcessor.Factory(registry));
-        registerProcessor(DateIndexNameProcessor.TYPE, (templateService, registry) -> new DateIndexNameProcessor.Factory());
-        registerProcessor(SortProcessor.TYPE, (templateService, registry) -> new SortProcessor.Factory());
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/TrackingResultProcessorTests.java b/core/src/test/java/org/elasticsearch/action/ingest/TrackingResultProcessorTests.java
similarity index 97%
rename from core/src/test/java/org/elasticsearch/ingest/processor/TrackingResultProcessorTests.java
rename to core/src/test/java/org/elasticsearch/action/ingest/TrackingResultProcessorTests.java
index e19a4df05c9..ac7282051d5 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/TrackingResultProcessorTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/TrackingResultProcessorTests.java
@@ -17,10 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.action.ingest;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ingest.SimulateProcessorResult;
+import org.elasticsearch.action.ingest.TrackingResultProcessor;
 import org.elasticsearch.ingest.TestProcessor;
 import org.elasticsearch.ingest.core.CompoundProcessor;
 import org.elasticsearch.ingest.core.IngestDocument;
@@ -36,7 +37,7 @@ import java.util.Map;
 import static org.elasticsearch.ingest.core.CompoundProcessor.ON_FAILURE_MESSAGE_FIELD;
 import static org.elasticsearch.ingest.core.CompoundProcessor.ON_FAILURE_PROCESSOR_TAG_FIELD;
 import static org.elasticsearch.ingest.core.CompoundProcessor.ON_FAILURE_PROCESSOR_TYPE_FIELD;
-import static org.elasticsearch.ingest.processor.TrackingResultProcessor.decorate;
+import static org.elasticsearch.action.ingest.TrackingResultProcessor.decorate;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
 
diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java b/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java
index 111133a0521..72856c143e0 100644
--- a/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java
+++ b/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java
@@ -38,7 +38,6 @@ import org.elasticsearch.action.ingest.WritePipelineResponse;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.node.NodeModule;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -69,7 +68,7 @@ public class IngestClientIT extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return pluginList(IngestPlugin.class);
+        return pluginList(IngestTestPlugin.class);
     }
 
     public void testSimulate() throws Exception {
@@ -234,28 +233,4 @@ public class IngestClientIT extends ESIntegTestCase {
     protected Collection<Class<? extends Plugin>> getMockPlugins() {
         return Collections.singletonList(TestSeedPlugin.class);
     }
-
-    public static class IngestPlugin extends Plugin {
-
-        @Override
-        public String name() {
-            return "ingest";
-        }
-
-        @Override
-        public String description() {
-            return "ingest mock";
-        }
-
-        public void onModule(NodeModule nodeModule) {
-            nodeModule.registerProcessor("test", (templateService, registry) -> config ->
-                new TestProcessor("id", "test", ingestDocument -> {
-                    ingestDocument.setFieldValue("processed", true);
-                    if (ingestDocument.getFieldValue("fail", Boolean.class)) {
-                        throw new IllegalArgumentException("test processor failed");
-                    }
-                })
-            );
-        }
-    }
 }
diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java b/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java
index a415b0992a7..c6a096f238e 100644
--- a/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java
+++ b/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java
@@ -57,7 +57,7 @@ public class IngestProcessorNotInstalledOnAllNodesIT extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return installPlugin ? pluginList(IngestClientIT.IngestPlugin.class) : Collections.emptyList();
+        return installPlugin ? pluginList(IngestTestPlugin.class) : Collections.emptyList();
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
index c60bb3eecde..26dc1957757 100644
--- a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
@@ -31,11 +31,11 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.ingest.core.IngestDocument;
 import org.elasticsearch.ingest.core.IngestInfo;
 import org.elasticsearch.ingest.core.Pipeline;
+import org.elasticsearch.ingest.core.Processor;
 import org.elasticsearch.ingest.core.ProcessorInfo;
-import org.elasticsearch.ingest.processor.RemoveProcessor;
-import org.elasticsearch.ingest.processor.SetProcessor;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
@@ -60,9 +60,45 @@ public class PipelineStoreTests extends ESTestCase {
     public void init() throws Exception {
         store = new PipelineStore(Settings.EMPTY);
         ProcessorsRegistry.Builder registryBuilder = new ProcessorsRegistry.Builder();
-        registryBuilder.registerProcessor("set", (templateService, registry) -> new SetProcessor.Factory(TestTemplateService.instance()));
-        registryBuilder.registerProcessor("remove", (templateService, registry) ->
-                new RemoveProcessor.Factory(TestTemplateService.instance()));
+        registryBuilder.registerProcessor("set", (templateService, registry) -> config -> {
+            String field = (String) config.remove("field");
+            String value = (String) config.remove("value");
+            return new Processor() {
+                @Override
+                public void execute(IngestDocument ingestDocument) throws Exception {
+                    ingestDocument.setFieldValue(field, value);
+                }
+
+                @Override
+                public String getType() {
+                    return "set";
+                }
+
+                @Override
+                public String getTag() {
+                    return null;
+                }
+            };
+        });
+        registryBuilder.registerProcessor("remove", (templateService, registry) -> config -> {
+            String field = (String) config.remove("field");
+            return new Processor() {
+                @Override
+                public void execute(IngestDocument ingestDocument) throws Exception {
+                    ingestDocument.removeField(field);
+                }
+
+                @Override
+                public String getType() {
+                    return "remove";
+                }
+
+                @Override
+                public String getTag() {
+                    return null;
+                }
+            };
+        });
         store.buildProcessorFactoryRegistry(registryBuilder, null);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/ingest/core/PipelineFactoryTests.java b/core/src/test/java/org/elasticsearch/ingest/core/PipelineFactoryTests.java
index 449f1836f2b..b6840b7a554 100644
--- a/core/src/test/java/org/elasticsearch/ingest/core/PipelineFactoryTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/core/PipelineFactoryTests.java
@@ -23,8 +23,6 @@ import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ingest.ProcessorsRegistry;
 import org.elasticsearch.ingest.TestProcessor;
 import org.elasticsearch.ingest.TestTemplateService;
-import org.elasticsearch.ingest.processor.FailProcessor;
-import org.elasticsearch.ingest.processor.SetProcessor;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Arrays;
@@ -32,7 +30,6 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.prefs.PreferencesFactory;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.is;
diff --git a/modules/ingest-grok/build.gradle b/modules/ingest-common/build.gradle
similarity index 77%
rename from modules/ingest-grok/build.gradle
rename to modules/ingest-common/build.gradle
index 26722345544..e7935cd2738 100644
--- a/modules/ingest-grok/build.gradle
+++ b/modules/ingest-common/build.gradle
@@ -18,8 +18,8 @@
  */
 
 esplugin {
-    description 'Ingest processor that uses grok patterns to split text'
-    classname 'org.elasticsearch.ingest.grok.IngestGrokPlugin'
+    description 'Module for ingest processors that do not require additional security permissions or have large dependencies and resources'
+    classname 'org.elasticsearch.ingest.IngestCommonPlugin'
 }
 
 dependencies {
@@ -28,8 +28,8 @@ dependencies {
     compile 'org.jruby.jcodings:jcodings:1.0.12'
 }
 
-compileJava.options.compilerArgs << "-Xlint:-rawtypes,-unchecked,-serial"
-compileTestJava.options.compilerArgs << "-Xlint:-rawtypes,-unchecked"
+compileJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes"
+compileTestJava.options.compilerArgs << "-Xlint:-unchecked,-rawtypes"
 
 thirdPartyAudit.excludes = [
         // joni has AsmCompilerSupport, but that isn't being used:
diff --git a/modules/ingest-grok/licenses/jcodings-1.0.12.jar.sha1 b/modules/ingest-common/licenses/jcodings-1.0.12.jar.sha1
similarity index 100%
rename from modules/ingest-grok/licenses/jcodings-1.0.12.jar.sha1
rename to modules/ingest-common/licenses/jcodings-1.0.12.jar.sha1
diff --git a/modules/ingest-grok/licenses/jcodings-LICENSE.txt b/modules/ingest-common/licenses/jcodings-LICENSE.txt
similarity index 100%
rename from modules/ingest-grok/licenses/jcodings-LICENSE.txt
rename to modules/ingest-common/licenses/jcodings-LICENSE.txt
diff --git a/modules/ingest-grok/licenses/jcodings-NOTICE.txt b/modules/ingest-common/licenses/jcodings-NOTICE.txt
similarity index 100%
rename from modules/ingest-grok/licenses/jcodings-NOTICE.txt
rename to modules/ingest-common/licenses/jcodings-NOTICE.txt
diff --git a/modules/ingest-grok/licenses/joni-2.1.6.jar.sha1 b/modules/ingest-common/licenses/joni-2.1.6.jar.sha1
similarity index 100%
rename from modules/ingest-grok/licenses/joni-2.1.6.jar.sha1
rename to modules/ingest-common/licenses/joni-2.1.6.jar.sha1
diff --git a/modules/ingest-grok/licenses/joni-LICENSE.txt b/modules/ingest-common/licenses/joni-LICENSE.txt
similarity index 100%
rename from modules/ingest-grok/licenses/joni-LICENSE.txt
rename to modules/ingest-common/licenses/joni-LICENSE.txt
diff --git a/modules/ingest-grok/licenses/joni-NOTICE.txt b/modules/ingest-common/licenses/joni-NOTICE.txt
similarity index 100%
rename from modules/ingest-grok/licenses/joni-NOTICE.txt
rename to modules/ingest-common/licenses/joni-NOTICE.txt
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/AbstractStringProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/AbstractStringProcessor.java
similarity index 98%
rename from core/src/main/java/org/elasticsearch/ingest/processor/AbstractStringProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/AbstractStringProcessor.java
index e882c79ed85..2dc11d43af1 100644
--- a/core/src/main/java/org/elasticsearch/ingest/processor/AbstractStringProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/AbstractStringProcessor.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ingest.core.AbstractProcessor;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/AppendProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/AppendProcessor.java
similarity index 98%
rename from core/src/main/java/org/elasticsearch/ingest/processor/AppendProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/AppendProcessor.java
index cca31453f37..4019f8312ef 100644
--- a/core/src/main/java/org/elasticsearch/ingest/processor/AppendProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/AppendProcessor.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ingest.core.AbstractProcessor;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/ConvertProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/ConvertProcessor.java
similarity index 97%
rename from core/src/main/java/org/elasticsearch/ingest/processor/ConvertProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/ConvertProcessor.java
index 1a6ce94e3db..d019ab01ef7 100644
--- a/core/src/main/java/org/elasticsearch/ingest/processor/ConvertProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/ConvertProcessor.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ingest.core.AbstractProcessor;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
@@ -103,7 +103,8 @@ public final class ConvertProcessor extends AbstractProcessor {
             try {
                 return Type.valueOf(type.toUpperCase(Locale.ROOT));
             } catch(IllegalArgumentException e) {
-                throw newConfigurationException(TYPE, processorTag, propertyName, "type [" + type + "] not supported, cannot convert field.");
+                throw newConfigurationException(TYPE, processorTag, propertyName, "type [" + type +
+                        "] not supported, cannot convert field.");
             }
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/DateFormat.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/DateFormat.java
similarity index 98%
rename from core/src/main/java/org/elasticsearch/ingest/processor/DateFormat.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/DateFormat.java
index 282b29176bf..2e27bd72cd2 100644
--- a/core/src/main/java/org/elasticsearch/ingest/processor/DateFormat.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/DateFormat.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/DateIndexNameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/DateIndexNameProcessor.java
similarity index 99%
rename from core/src/main/java/org/elasticsearch/ingest/processor/DateIndexNameProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/DateIndexNameProcessor.java
index 2de44addaf5..cad66a3187b 100644
--- a/core/src/main/java/org/elasticsearch/ingest/processor/DateIndexNameProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/DateIndexNameProcessor.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.ingest.core.AbstractProcessor;
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/DateProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/DateProcessor.java
similarity index 99%
rename from core/src/main/java/org/elasticsearch/ingest/processor/DateProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/DateProcessor.java
index 8a1e58dc622..273b42d2b8c 100644
--- a/core/src/main/java/org/elasticsearch/ingest/processor/DateProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/DateProcessor.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.ingest.core.AbstractProcessor;
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/FailProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/FailProcessor.java
similarity index 98%
rename from core/src/main/java/org/elasticsearch/ingest/processor/FailProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/FailProcessor.java
index 7fdbca06c2b..2e0b79501e0 100644
--- a/core/src/main/java/org/elasticsearch/ingest/processor/FailProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/FailProcessor.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ingest.core.AbstractProcessor;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/FailProcessorException.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/FailProcessorException.java
similarity index 96%
rename from core/src/main/java/org/elasticsearch/ingest/processor/FailProcessorException.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/FailProcessorException.java
index bfdfe11178e..bd010b1423c 100644
--- a/core/src/main/java/org/elasticsearch/ingest/processor/FailProcessorException.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/FailProcessorException.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 /**
  * Exception class thrown by {@link FailProcessor}.
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/ForEachProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/ForEachProcessor.java
similarity index 97%
rename from core/src/main/java/org/elasticsearch/ingest/processor/ForEachProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/ForEachProcessor.java
index b6f6a85d219..2ca895779af 100644
--- a/core/src/main/java/org/elasticsearch/ingest/processor/ForEachProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/ForEachProcessor.java
@@ -17,9 +17,8 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
-import org.elasticsearch.ingest.ProcessorsRegistry;
 import org.elasticsearch.ingest.core.AbstractProcessor;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
 import org.elasticsearch.ingest.core.ConfigurationUtils;
diff --git a/modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/Grok.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/Grok.java
similarity index 93%
rename from modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/Grok.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/Grok.java
index abed8413287..7e5d1e6b516 100644
--- a/modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/Grok.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/Grok.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.grok;
+package org.elasticsearch.ingest;
 
 import org.jcodings.specific.UTF8Encoding;
 import org.joni.Matcher;
@@ -52,7 +52,8 @@ final class Grok {
             ")+" +
             ")" +
             ")?" + "\\}";
-    private static final Regex GROK_PATTERN_REGEX = new Regex(GROK_PATTERN.getBytes(StandardCharsets.UTF_8), 0, GROK_PATTERN.getBytes(StandardCharsets.UTF_8).length, Option.NONE, UTF8Encoding.INSTANCE, Syntax.DEFAULT);
+    private static final Regex GROK_PATTERN_REGEX = new Regex(GROK_PATTERN.getBytes(StandardCharsets.UTF_8), 0,
+            GROK_PATTERN.getBytes(StandardCharsets.UTF_8).length, Option.NONE, UTF8Encoding.INSTANCE, Syntax.DEFAULT);
     private final Map<String, String> patternBank;
     private final boolean namedCaptures;
     private final Regex compiledExpression;
@@ -76,7 +77,8 @@ final class Grok {
 
     public String groupMatch(String name, Region region, String pattern) {
         try {
-            int number = GROK_PATTERN_REGEX.nameToBackrefNumber(name.getBytes(StandardCharsets.UTF_8), 0, name.getBytes(StandardCharsets.UTF_8).length, region);
+            int number = GROK_PATTERN_REGEX.nameToBackrefNumber(name.getBytes(StandardCharsets.UTF_8), 0,
+                    name.getBytes(StandardCharsets.UTF_8).length, region);
             int begin = region.beg[number];
             int end = region.end[number];
             return new String(pattern.getBytes(StandardCharsets.UTF_8), begin, end - begin, StandardCharsets.UTF_8);
@@ -143,7 +145,8 @@ final class Grok {
                 String groupName = new String(e.name, e.nameP, e.nameEnd - e.nameP, StandardCharsets.UTF_8);
                 String matchValue = null;
                 if (region.beg[number] >= 0) {
-                    matchValue = new String(textAsBytes, region.beg[number], region.end[number] - region.beg[number], StandardCharsets.UTF_8);
+                    matchValue = new String(textAsBytes, region.beg[number], region.end[number] - region.beg[number],
+                            StandardCharsets.UTF_8);
                 }
                 GrokMatchGroup match = new GrokMatchGroup(groupName, matchValue);
                 fields.put(match.getName(), match.getValue());
diff --git a/modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/GrokMatchGroup.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/GrokMatchGroup.java
similarity index 97%
rename from modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/GrokMatchGroup.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/GrokMatchGroup.java
index 2cebf620c96..6fd979f179a 100644
--- a/modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/GrokMatchGroup.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/GrokMatchGroup.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.grok;
+package org.elasticsearch.ingest;
 
 final class GrokMatchGroup {
     private static final String DEFAULT_TYPE = "string";
diff --git a/modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/GrokProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/GrokProcessor.java
similarity index 99%
rename from modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/GrokProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/GrokProcessor.java
index 6a10c8db058..b709df6e7d4 100644
--- a/modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/GrokProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/GrokProcessor.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.grok;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ingest.core.AbstractProcessor;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/GsubProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/GsubProcessor.java
similarity index 95%
rename from core/src/main/java/org/elasticsearch/ingest/processor/GsubProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/GsubProcessor.java
index d986bf522e5..8b17766f845 100644
--- a/core/src/main/java/org/elasticsearch/ingest/processor/GsubProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/GsubProcessor.java
@@ -17,13 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
-import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ingest.core.AbstractProcessor;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
 import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.ConfigurationUtils;
 
 import java.util.Map;
 import java.util.regex.Matcher;
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/IngestCommonPlugin.java
new file mode 100644
index 00000000000..8e5a821b8a5
--- /dev/null
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/IngestCommonPlugin.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest;
+
+import org.elasticsearch.node.NodeModule;
+import org.elasticsearch.plugins.Plugin;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class IngestCommonPlugin extends Plugin {
+
+    public static final String NAME = "ingest-common";
+
+    private final Map<String, String> builtinPatterns;
+
+    public IngestCommonPlugin() throws IOException {
+        this.builtinPatterns = loadBuiltinPatterns();
+    }
+
+    @Override
+    public String name() {
+        return NAME;
+    }
+
+    @Override
+    public String description() {
+        return "Module for ingest processors that do not require additional security permissions or have large dependencies and resources";
+    }
+
+    public void onModule(NodeModule nodeModule) {
+        nodeModule.registerProcessor(DateProcessor.TYPE, (templateService, registry) -> new DateProcessor.Factory());
+        nodeModule.registerProcessor(SetProcessor.TYPE, (templateService, registry) -> new SetProcessor.Factory(templateService));
+        nodeModule.registerProcessor(AppendProcessor.TYPE, (templateService, registry) -> new AppendProcessor.Factory(templateService));
+        nodeModule.registerProcessor(RenameProcessor.TYPE, (templateService, registry) -> new RenameProcessor.Factory());
+        nodeModule.registerProcessor(RemoveProcessor.TYPE, (templateService, registry) -> new RemoveProcessor.Factory(templateService));
+        nodeModule.registerProcessor(SplitProcessor.TYPE, (templateService, registry) -> new SplitProcessor.Factory());
+        nodeModule.registerProcessor(JoinProcessor.TYPE, (templateService, registry) -> new JoinProcessor.Factory());
+        nodeModule.registerProcessor(UppercaseProcessor.TYPE, (templateService, registry) -> new UppercaseProcessor.Factory());
+        nodeModule.registerProcessor(LowercaseProcessor.TYPE, (templateService, registry) -> new LowercaseProcessor.Factory());
+        nodeModule.registerProcessor(TrimProcessor.TYPE, (templateService, registry) -> new TrimProcessor.Factory());
+        nodeModule.registerProcessor(ConvertProcessor.TYPE, (templateService, registry) -> new ConvertProcessor.Factory());
+        nodeModule.registerProcessor(GsubProcessor.TYPE, (templateService, registry) -> new GsubProcessor.Factory());
+        nodeModule.registerProcessor(FailProcessor.TYPE, (templateService, registry) -> new FailProcessor.Factory(templateService));
+        nodeModule.registerProcessor(ForEachProcessor.TYPE, (templateService, registry) -> new ForEachProcessor.Factory(registry));
+        nodeModule.registerProcessor(DateIndexNameProcessor.TYPE, (templateService, registry) -> new DateIndexNameProcessor.Factory());
+        nodeModule.registerProcessor(SortProcessor.TYPE, (templateService, registry) -> new SortProcessor.Factory());
+        nodeModule.registerProcessor(GrokProcessor.TYPE, (templateService, registry) -> new GrokProcessor.Factory(builtinPatterns));
+    }
+
+    // Code for loading built-in grok patterns packaged with the jar file:
+
+    private static final String[] PATTERN_NAMES = new String[] {
+            "aws", "bacula", "bro", "exim", "firewalls", "grok-patterns", "haproxy",
+            "java", "junos", "linux-syslog", "mcollective-patterns", "mongodb", "nagios",
+            "postgresql", "rails", "redis", "ruby"
+    };
+
+    public static Map<String, String> loadBuiltinPatterns() throws IOException {
+        Map<String, String> builtinPatterns = new HashMap<>();
+        for (String pattern : PATTERN_NAMES) {
+            try(InputStream is = IngestCommonPlugin.class.getResourceAsStream("/patterns/" + pattern)) {
+                loadPatterns(builtinPatterns, is);
+            }
+        }
+        return Collections.unmodifiableMap(builtinPatterns);
+    }
+
+    private static void loadPatterns(Map<String, String> patternBank, InputStream inputStream) throws IOException {
+        String line;
+        BufferedReader br = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
+        while ((line = br.readLine()) != null) {
+            String trimmedLine = line.replaceAll("^\\s+", "");
+            if (trimmedLine.startsWith("#") || trimmedLine.length() == 0) {
+                continue;
+            }
+
+            String[] parts = trimmedLine.split("\\s+", 2);
+            if (parts.length == 2) {
+                patternBank.put(parts[0], parts[1]);
+            }
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/JoinProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/JoinProcessor.java
similarity index 98%
rename from core/src/main/java/org/elasticsearch/ingest/processor/JoinProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/JoinProcessor.java
index cd7655dd953..7db5e00b145 100644
--- a/core/src/main/java/org/elasticsearch/ingest/processor/JoinProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/JoinProcessor.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ingest.core.AbstractProcessor;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/LowercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/LowercaseProcessor.java
similarity index 97%
rename from core/src/main/java/org/elasticsearch/ingest/processor/LowercaseProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/LowercaseProcessor.java
index deb2809e711..8d5aebcf115 100644
--- a/core/src/main/java/org/elasticsearch/ingest/processor/LowercaseProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/LowercaseProcessor.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import java.util.Locale;
 
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/RemoveProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/RemoveProcessor.java
similarity index 98%
rename from core/src/main/java/org/elasticsearch/ingest/processor/RemoveProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/RemoveProcessor.java
index 3d1aff2741a..6670c462d6f 100644
--- a/core/src/main/java/org/elasticsearch/ingest/processor/RemoveProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/RemoveProcessor.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ingest.core.AbstractProcessor;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/RenameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/RenameProcessor.java
similarity index 98%
rename from core/src/main/java/org/elasticsearch/ingest/processor/RenameProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/RenameProcessor.java
index ea2f02d066a..4a6e8bf3ae5 100644
--- a/core/src/main/java/org/elasticsearch/ingest/processor/RenameProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/RenameProcessor.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ingest.core.AbstractProcessor;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/SetProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/SetProcessor.java
similarity index 98%
rename from core/src/main/java/org/elasticsearch/ingest/processor/SetProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/SetProcessor.java
index 6a67aca6bfb..6f576dcafc0 100644
--- a/core/src/main/java/org/elasticsearch/ingest/processor/SetProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/SetProcessor.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ingest.core.AbstractProcessor;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/SortProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/SortProcessor.java
similarity index 96%
rename from core/src/main/java/org/elasticsearch/ingest/processor/SortProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/SortProcessor.java
index 313a8bed7fc..ee708d97d47 100644
--- a/core/src/main/java/org/elasticsearch/ingest/processor/SortProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/SortProcessor.java
@@ -17,19 +17,16 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
-import org.elasticsearch.common.ParseField;
 import org.elasticsearch.ingest.core.AbstractProcessor;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
 import org.elasticsearch.ingest.core.IngestDocument;
 import org.elasticsearch.ingest.core.ConfigurationUtils;
 
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 
 /**
  * Processor that sorts an array of items.
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/SplitProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/SplitProcessor.java
similarity index 98%
rename from core/src/main/java/org/elasticsearch/ingest/processor/SplitProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/SplitProcessor.java
index aca4fe7026f..2d43cae3f09 100644
--- a/core/src/main/java/org/elasticsearch/ingest/processor/SplitProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/SplitProcessor.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ingest.core.AbstractProcessor;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/TrimProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/TrimProcessor.java
similarity index 97%
rename from core/src/main/java/org/elasticsearch/ingest/processor/TrimProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/TrimProcessor.java
index 919264fb866..294cd59265f 100644
--- a/core/src/main/java/org/elasticsearch/ingest/processor/TrimProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/TrimProcessor.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 /**
  * Processor that trims the content of string fields.
diff --git a/core/src/main/java/org/elasticsearch/ingest/processor/UppercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/UppercaseProcessor.java
similarity index 97%
rename from core/src/main/java/org/elasticsearch/ingest/processor/UppercaseProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/UppercaseProcessor.java
index 7b3678b89ee..cd51f0dbc71 100644
--- a/core/src/main/java/org/elasticsearch/ingest/processor/UppercaseProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/UppercaseProcessor.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import java.util.Locale;
 
diff --git a/modules/ingest-grok/src/main/resources/patterns/aws b/modules/ingest-common/src/main/resources/patterns/aws
similarity index 100%
rename from modules/ingest-grok/src/main/resources/patterns/aws
rename to modules/ingest-common/src/main/resources/patterns/aws
diff --git a/modules/ingest-grok/src/main/resources/patterns/bacula b/modules/ingest-common/src/main/resources/patterns/bacula
similarity index 100%
rename from modules/ingest-grok/src/main/resources/patterns/bacula
rename to modules/ingest-common/src/main/resources/patterns/bacula
diff --git a/modules/ingest-grok/src/main/resources/patterns/bro b/modules/ingest-common/src/main/resources/patterns/bro
similarity index 100%
rename from modules/ingest-grok/src/main/resources/patterns/bro
rename to modules/ingest-common/src/main/resources/patterns/bro
diff --git a/modules/ingest-grok/src/main/resources/patterns/exim b/modules/ingest-common/src/main/resources/patterns/exim
similarity index 100%
rename from modules/ingest-grok/src/main/resources/patterns/exim
rename to modules/ingest-common/src/main/resources/patterns/exim
diff --git a/modules/ingest-grok/src/main/resources/patterns/firewalls b/modules/ingest-common/src/main/resources/patterns/firewalls
similarity index 100%
rename from modules/ingest-grok/src/main/resources/patterns/firewalls
rename to modules/ingest-common/src/main/resources/patterns/firewalls
diff --git a/modules/ingest-grok/src/main/resources/patterns/grok-patterns b/modules/ingest-common/src/main/resources/patterns/grok-patterns
similarity index 100%
rename from modules/ingest-grok/src/main/resources/patterns/grok-patterns
rename to modules/ingest-common/src/main/resources/patterns/grok-patterns
diff --git a/modules/ingest-grok/src/main/resources/patterns/haproxy b/modules/ingest-common/src/main/resources/patterns/haproxy
similarity index 100%
rename from modules/ingest-grok/src/main/resources/patterns/haproxy
rename to modules/ingest-common/src/main/resources/patterns/haproxy
diff --git a/modules/ingest-grok/src/main/resources/patterns/java b/modules/ingest-common/src/main/resources/patterns/java
similarity index 100%
rename from modules/ingest-grok/src/main/resources/patterns/java
rename to modules/ingest-common/src/main/resources/patterns/java
diff --git a/modules/ingest-grok/src/main/resources/patterns/junos b/modules/ingest-common/src/main/resources/patterns/junos
similarity index 100%
rename from modules/ingest-grok/src/main/resources/patterns/junos
rename to modules/ingest-common/src/main/resources/patterns/junos
diff --git a/modules/ingest-grok/src/main/resources/patterns/linux-syslog b/modules/ingest-common/src/main/resources/patterns/linux-syslog
similarity index 100%
rename from modules/ingest-grok/src/main/resources/patterns/linux-syslog
rename to modules/ingest-common/src/main/resources/patterns/linux-syslog
diff --git a/modules/ingest-grok/src/main/resources/patterns/mcollective-patterns b/modules/ingest-common/src/main/resources/patterns/mcollective-patterns
similarity index 100%
rename from modules/ingest-grok/src/main/resources/patterns/mcollective-patterns
rename to modules/ingest-common/src/main/resources/patterns/mcollective-patterns
diff --git a/modules/ingest-grok/src/main/resources/patterns/mongodb b/modules/ingest-common/src/main/resources/patterns/mongodb
similarity index 100%
rename from modules/ingest-grok/src/main/resources/patterns/mongodb
rename to modules/ingest-common/src/main/resources/patterns/mongodb
diff --git a/modules/ingest-grok/src/main/resources/patterns/nagios b/modules/ingest-common/src/main/resources/patterns/nagios
similarity index 100%
rename from modules/ingest-grok/src/main/resources/patterns/nagios
rename to modules/ingest-common/src/main/resources/patterns/nagios
diff --git a/modules/ingest-grok/src/main/resources/patterns/postgresql b/modules/ingest-common/src/main/resources/patterns/postgresql
similarity index 100%
rename from modules/ingest-grok/src/main/resources/patterns/postgresql
rename to modules/ingest-common/src/main/resources/patterns/postgresql
diff --git a/modules/ingest-grok/src/main/resources/patterns/rails b/modules/ingest-common/src/main/resources/patterns/rails
similarity index 100%
rename from modules/ingest-grok/src/main/resources/patterns/rails
rename to modules/ingest-common/src/main/resources/patterns/rails
diff --git a/modules/ingest-grok/src/main/resources/patterns/redis b/modules/ingest-common/src/main/resources/patterns/redis
similarity index 100%
rename from modules/ingest-grok/src/main/resources/patterns/redis
rename to modules/ingest-common/src/main/resources/patterns/redis
diff --git a/modules/ingest-grok/src/main/resources/patterns/ruby b/modules/ingest-common/src/main/resources/patterns/ruby
similarity index 100%
rename from modules/ingest-grok/src/main/resources/patterns/ruby
rename to modules/ingest-common/src/main/resources/patterns/ruby
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/AbstractStringProcessorTestCase.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/AbstractStringProcessorTestCase.java
similarity index 95%
rename from core/src/test/java/org/elasticsearch/ingest/processor/AbstractStringProcessorTestCase.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/AbstractStringProcessorTestCase.java
index 1113a4b402f..b1f517b3d88 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/AbstractStringProcessorTestCase.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/AbstractStringProcessorTestCase.java
@@ -17,11 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ingest.core.IngestDocument;
 import org.elasticsearch.ingest.core.Processor;
-import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Collections;
@@ -81,7 +80,8 @@ public abstract class AbstractStringProcessorTestCase extends ESTestCase {
             processor.execute(ingestDocument);
             fail("processor should have failed");
         } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("field [" + fieldName + "] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
+            assertThat(e.getMessage(), equalTo("field [" + fieldName +
+                    "] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/AppendProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/AppendProcessorFactoryTests.java
similarity index 97%
rename from core/src/test/java/org/elasticsearch/ingest/processor/AppendProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/AppendProcessorFactoryTests.java
index 7350e3d9c43..884bbb7903a 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/AppendProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/AppendProcessorFactoryTests.java
@@ -17,10 +17,9 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/AppendProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/AppendProcessorTests.java
similarity index 97%
rename from core/src/test/java/org/elasticsearch/ingest/processor/AppendProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/AppendProcessorTests.java
index 4a78ba621ce..63a1bb9cce2 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/AppendProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/AppendProcessorTests.java
@@ -17,12 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.ingest.core.TemplateService;
-import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.ingest.core.ValueSource;
 import org.elasticsearch.ingest.core.Processor;
 import org.elasticsearch.test.ESTestCase;
@@ -158,7 +156,8 @@ public class AppendProcessorTests extends ESTestCase {
 
     private static Processor createAppendProcessor(String fieldName, Object fieldValue) {
         TemplateService templateService = TestTemplateService.instance();
-        return new AppendProcessor(randomAsciiOfLength(10), templateService.compile(fieldName), ValueSource.wrap(fieldValue, templateService));
+        return new AppendProcessor(randomAsciiOfLength(10), templateService.compile(fieldName), ValueSource.wrap(fieldValue,
+                templateService));
     }
 
     private enum Scalar {
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/ConvertProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ConvertProcessorFactoryTests.java
similarity index 99%
rename from core/src/test/java/org/elasticsearch/ingest/processor/ConvertProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/ConvertProcessorFactoryTests.java
index f54f04c0cf8..ce043f6aed4 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/ConvertProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ConvertProcessorFactoryTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/ConvertProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ConvertProcessorTests.java
similarity index 99%
rename from core/src/test/java/org/elasticsearch/ingest/processor/ConvertProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/ConvertProcessorTests.java
index 0efacb145bd..f9ec7883190 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/ConvertProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ConvertProcessorTests.java
@@ -17,10 +17,9 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.ingest.core.Processor;
 import org.elasticsearch.test.ESTestCase;
 
@@ -31,7 +30,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
-import static org.elasticsearch.ingest.processor.ConvertProcessor.Type;
+import static org.elasticsearch.ingest.ConvertProcessor.Type;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.sameInstance;
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/DateFormatTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateFormatTests.java
similarity index 90%
rename from core/src/test/java/org/elasticsearch/ingest/processor/DateFormatTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateFormatTests.java
index 401dd44d44a..8fcff4789c7 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/DateFormatTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateFormatTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.test.ESTestCase;
 import org.joda.time.DateTime;
@@ -34,7 +34,8 @@ import static org.hamcrest.core.IsEqual.equalTo;
 public class DateFormatTests extends ESTestCase {
 
     public void testParseJoda() {
-        Function<String, DateTime> jodaFunction = DateFormat.Joda.getFunction("MMM dd HH:mm:ss Z", DateTimeZone.forOffsetHours(-8), Locale.ENGLISH);
+        Function<String, DateTime> jodaFunction = DateFormat.Joda.getFunction("MMM dd HH:mm:ss Z",
+                DateTimeZone.forOffsetHours(-8), Locale.ENGLISH);
         assertThat(Instant.ofEpochMilli(jodaFunction.apply("Nov 24 01:29:01 -0800").getMillis())
                         .atZone(ZoneId.of("GMT-8"))
                         .format(DateTimeFormatter.ofPattern("MM dd HH:mm:ss", Locale.ENGLISH)),
@@ -50,7 +51,8 @@ public class DateFormatTests extends ESTestCase {
     }
 
     public void testParseISO8601() {
-        assertThat(DateFormat.Iso8601.getFunction(null, DateTimeZone.UTC, null).apply("2001-01-01T00:00:00-0800").getMillis(), equalTo(978336000000L));
+        assertThat(DateFormat.Iso8601.getFunction(null, DateTimeZone.UTC, null).apply("2001-01-01T00:00:00-0800").getMillis(),
+                equalTo(978336000000L));
     }
 
     public void testParseISO8601Failure() {
@@ -66,7 +68,8 @@ public class DateFormatTests extends ESTestCase {
     public void testTAI64NParse() {
         String input = "4000000050d506482dbdf024";
         String expected = "2012-12-22T03:00:46.767+02:00";
-        assertThat(DateFormat.Tai64n.getFunction(null, DateTimeZone.forOffsetHours(2), null).apply((randomBoolean() ? "@" : "") + input).toString(), equalTo(expected));
+        assertThat(DateFormat.Tai64n.getFunction(null, DateTimeZone.forOffsetHours(2), null)
+                .apply((randomBoolean() ? "@" : "") + input).toString(), equalTo(expected));
     }
 
     public void testFromString() {
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/DateIndexNameFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateIndexNameFactoryTests.java
similarity index 98%
rename from core/src/test/java/org/elasticsearch/ingest/processor/DateIndexNameFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateIndexNameFactoryTests.java
index 5bb674e950a..d6b5f7a0ff7 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/DateIndexNameFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateIndexNameFactoryTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.test.ESTestCase;
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/DateIndexNameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateIndexNameProcessorTests.java
similarity index 98%
rename from core/src/test/java/org/elasticsearch/ingest/processor/DateIndexNameProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateIndexNameProcessorTests.java
index 2dd5934387c..3baa3d3ac7c 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/DateIndexNameProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateIndexNameProcessorTests.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ingest.core.IngestDocument;
 import org.elasticsearch.test.ESTestCase;
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/DateProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateProcessorFactoryTests.java
similarity index 98%
rename from core/src/test/java/org/elasticsearch/ingest/processor/DateProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateProcessorFactoryTests.java
index 68b5d175509..9933f3fbb65 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/DateProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateProcessorFactoryTests.java
@@ -17,18 +17,16 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 import org.joda.time.DateTimeZone;
 
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/DateProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateProcessorTests.java
similarity index 97%
rename from core/src/test/java/org/elasticsearch/ingest/processor/DateProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateProcessorTests.java
index 5daab95a5d0..2d32282d332 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/DateProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateProcessorTests.java
@@ -17,10 +17,9 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
@@ -111,7 +110,8 @@ public class DateProcessorTests extends ESTestCase {
         document.put("date_as_string", "12/06");
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
         dateProcessor.execute(ingestDocument);
-        assertThat(ingestDocument.getFieldValue("date_as_date", String.class), equalTo(DateTime.now().getYear() + "-06-12T00:00:00.000+02:00"));
+        assertThat(ingestDocument.getFieldValue("date_as_date", String.class), equalTo(DateTime.now().getYear() +
+                "-06-12T00:00:00.000+02:00"));
     }
 
     public void testTAI64N() {
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/FailProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/FailProcessorFactoryTests.java
similarity index 93%
rename from core/src/test/java/org/elasticsearch/ingest/processor/FailProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/FailProcessorFactoryTests.java
index 0d88710c80d..1f1e9c72b12 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/FailProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/FailProcessorFactoryTests.java
@@ -17,12 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.Processor;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/FailProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/FailProcessorTests.java
similarity index 91%
rename from core/src/test/java/org/elasticsearch/ingest/processor/FailProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/FailProcessorTests.java
index 3fdc2073c09..457aaf45c66 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/FailProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/FailProcessorTests.java
@@ -17,12 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ingest.core.IngestDocument;
 import org.elasticsearch.ingest.core.Processor;
-import org.elasticsearch.ingest.RandomDocumentPicks;
-import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.test.ESTestCase;
 
 import static org.hamcrest.Matchers.equalTo;
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/ForEachProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ForEachProcessorFactoryTests.java
similarity index 93%
rename from core/src/test/java/org/elasticsearch/ingest/processor/ForEachProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/ForEachProcessorFactoryTests.java
index d03c2c2c546..74771b5768e 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/ForEachProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ForEachProcessorFactoryTests.java
@@ -17,11 +17,8 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
-import org.elasticsearch.ingest.ProcessorsRegistry;
-import org.elasticsearch.ingest.TestProcessor;
-import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.ingest.core.Processor;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/ForEachProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ForEachProcessorTests.java
similarity index 98%
rename from core/src/test/java/org/elasticsearch/ingest/processor/ForEachProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/ForEachProcessorTests.java
index d1c2c6f1777..19f1e15f09e 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/ForEachProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ForEachProcessorTests.java
@@ -17,10 +17,8 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
-import org.elasticsearch.ingest.TestProcessor;
-import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.ingest.core.CompoundProcessor;
 import org.elasticsearch.ingest.core.IngestDocument;
 import org.elasticsearch.ingest.core.Processor;
diff --git a/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/GrokProcessorFactoryTests.java
similarity index 99%
rename from modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/GrokProcessorFactoryTests.java
index ec3b908aac0..374a15f4f25 100644
--- a/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/GrokProcessorFactoryTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.grok;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
diff --git a/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/GrokProcessorTests.java
similarity index 99%
rename from modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/GrokProcessorTests.java
index 82354a943ae..919cd5ca548 100644
--- a/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/GrokProcessorTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.grok;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.ingest.core.IngestDocument;
diff --git a/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/GrokTests.java
similarity index 79%
rename from modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/GrokTests.java
index 21ca17ad713..fd757c4962c 100644
--- a/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/GrokTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/GrokTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.grok;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
@@ -38,7 +38,7 @@ public class GrokTests extends ESTestCase {
 
     @Before
     public void setup() throws IOException {
-        basePatterns = IngestGrokPlugin.loadBuiltinPatterns();
+        basePatterns = IngestCommonPlugin.loadBuiltinPatterns();
     }
 
     public void testMatchWithoutCaptures() {
@@ -60,7 +60,8 @@ public class GrokTests extends ESTestCase {
     }
 
     public void testSyslog5424Line() {
-        String line = "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 - [id1 foo=\\\"bar\\\"][id2 baz=\\\"something\\\"] Hello, syslog.";
+        String line = "<191>1 2009-06-30T18:30:00+02:00 paxton.local grokdebug 4123 - [id1 foo=\\\"bar\\\"][id2 baz=\\\"something\\\"] " +
+                "Hello, syslog.";
         Grok grok = new Grok(basePatterns, "%{SYSLOG5424LINE}");
         Map<String, Object> matches = grok.captures(line);
         assertEquals("191", matches.get("syslog5424_pri"));
@@ -98,8 +99,12 @@ public class GrokTests extends ESTestCase {
     }
 
     public void testUnicodeSyslog() {
-        Grok grok = new Grok(basePatterns, "<%{POSINT:syslog_pri}>%{SPACE}%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(:?)(?:\\[%{GREEDYDATA:syslog_pid}\\])?(:?) %{GREEDYDATA:syslog_message}");
-        Map<String, Object> matches = grok.captures("<22>Jan  4 07:50:46 mailmaster postfix/policy-spf[9454]: : SPF permerror (Junk encountered in record 'v=spf1 mx a:mail.domain.no ip4:192.168.0.4 �all'): Envelope-from: email@domain.no");
+        Grok grok = new Grok(basePatterns, "<%{POSINT:syslog_pri}>%{SPACE}%{SYSLOGTIMESTAMP:syslog_timestamp} " +
+                "%{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(:?)(?:\\[%{GREEDYDATA:syslog_pid}\\])?(:?) " +
+                "%{GREEDYDATA:syslog_message}");
+        Map<String, Object> matches = grok.captures("<22>Jan  4 07:50:46 mailmaster postfix/policy-spf[9454]: : " +
+                "SPF permerror (Junk encountered in record 'v=spf1 mx a:mail.domain.no ip4:192.168.0.4 �all'): Envelope-from: " +
+                "email@domain.no");
         assertThat(matches.get("syslog_pri"), equalTo("22"));
         assertThat(matches.get("syslog_program"), equalTo("postfix/policy-spf"));
         assertThat(matches.get("tags"), nullValue());
@@ -212,7 +217,9 @@ public class GrokTests extends ESTestCase {
     }
 
     public void testApacheLog() {
-        String logLine = "31.184.238.164 - - [24/Jul/2014:05:35:37 +0530] \"GET /logs/access.log HTTP/1.0\" 200 69849 \"http://8rursodiol.enjin.com\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.12785 YaBrowser/13.12.1599.12785 Safari/537.36\" \"www.dlwindianrailways.com\"";
+        String logLine = "31.184.238.164 - - [24/Jul/2014:05:35:37 +0530] \"GET /logs/access.log HTTP/1.0\" 200 69849 " +
+                "\"http://8rursodiol.enjin.com\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) " +
+                "Chrome/30.0.1599.12785 YaBrowser/13.12.1599.12785 Safari/537.36\" \"www.dlwindianrailways.com\"";
         Grok grok = new Grok(basePatterns, "%{COMBINEDAPACHELOG}");
         Map<String, Object> matches = grok.captures(logLine);
 
@@ -227,13 +234,15 @@ public class GrokTests extends ESTestCase {
         assertEquals("69849", matches.get("bytes"));
         assertEquals("\"http://8rursodiol.enjin.com\"", matches.get("referrer"));
         assertEquals(null, matches.get("port"));
-        assertEquals("\"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.12785 YaBrowser/13.12.1599.12785 Safari/537.36\"", matches.get("agent"));
+        assertEquals("\"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.12785 " +
+                "YaBrowser/13.12.1599.12785 Safari/537.36\"", matches.get("agent"));
     }
 
     public void testComplete() {
         Map<String, String> bank = new HashMap<>();
         bank.put("MONTHDAY", "(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])");
-        bank.put("MONTH", "\\b(?:Jan(?:uary|uar)?|Feb(?:ruary|ruar)?|M(?:a|ä)?r(?:ch|z)?|Apr(?:il)?|Ma(?:y|i)?|Jun(?:e|i)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|O(?:c|k)?t(?:ober)?|Nov(?:ember)?|De(?:c|z)(?:ember)?)\\b");
+        bank.put("MONTH", "\\b(?:Jan(?:uary|uar)?|Feb(?:ruary|ruar)?|M(?:a|ä)?r(?:ch|z)?|Apr(?:il)?|Ma(?:y|i)?|Jun(?:e|i)" +
+                "?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|O(?:c|k)?t(?:ober)?|Nov(?:ember)?|De(?:c|z)(?:ember)?)\\b");
         bank.put("MINUTE", "(?:[0-5][0-9])");
         bank.put("YEAR", "(?>\\d\\d){1,2}");
         bank.put("HOUR", "(?:2[0123]|[01]?[0-9])");
@@ -244,8 +253,19 @@ public class GrokTests extends ESTestCase {
         bank.put("WORD", "\\b\\w+\\b");
         bank.put("BASE10NUM", "(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\\.[0-9]+)?)|(?:\\.[0-9]+)))");
         bank.put("NUMBER", "(?:%{BASE10NUM})");
-        bank.put("IPV6", "((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:)))(%.+)?");
-        bank.put("IPV4", "(?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))(?![0-9])");
+        bank.put("IPV6", "((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]" +
+                "\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4})" +
+                "{1,2})|:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:)" +
+                "{4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\" +
+                "d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]" +
+                "\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4})" +
+                "{1,5})" +
+                "|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))" +
+                "|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)" +
+                "(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}" +
+                ":((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:)))(%.+)?");
+        bank.put("IPV4", "(?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.]" +
+                "(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))(?![0-9])");
         bank.put("IP", "(?:%{IPV6}|%{IPV4})");
         bank.put("HOSTNAME", "\\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\\.?|\\b)");
         bank.put("IPORHOST", "(?:%{IP}|%{HOSTNAME})");
@@ -253,8 +273,12 @@ public class GrokTests extends ESTestCase {
         bank.put("DATA", ".*?");
         bank.put("QS", "(?>(?<!\\\\)(?>\"(?>\\\\.|[^\\\\\"]+)+\"|\"\"|(?>'(?>\\\\.|[^\\\\']+)+')|''|(?>`(?>\\\\.|[^\\\\`]+)+`)|``))");
 
-        String text = "83.149.9.216 - - [19/Jul/2015:08:13:42 +0000] \"GET /presentations/logstash-monitorama-2013/images/kibana-dashboard3.png HTTP/1.1\" 200 171717 \"http://semicomplete.com/presentations/logstash-monitorama-2013/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"";
-        String pattern = "%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \\[%{HTTPDATE:timestamp}\\] \"%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:response:int} (?:-|%{NUMBER:bytes:int}) %{QS:referrer} %{QS:agent}";
+        String text = "83.149.9.216 - - [19/Jul/2015:08:13:42 +0000] \"GET /presentations/logstash-monitorama-2013/images/" +
+                "kibana-dashboard3.png HTTP/1.1\" 200 171717 \"http://semicomplete.com/presentations/logstash-monitorama-2013/\" " +
+                "\"Mozilla" +
+                "/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"";
+        String pattern = "%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \\[%{HTTPDATE:timestamp}\\] \"%{WORD:verb} %{DATA:request} " +
+                "HTTP/%{NUMBER:httpversion}\" %{NUMBER:response:int} (?:-|%{NUMBER:bytes:int}) %{QS:referrer} %{QS:agent}";
 
         Grok grok = new Grok(bank, pattern);
 
@@ -269,7 +293,8 @@ public class GrokTests extends ESTestCase {
         expected.put("response", 200);
         expected.put("bytes", 171717);
         expected.put("referrer", "\"http://semicomplete.com/presentations/logstash-monitorama-2013/\"");
-        expected.put("agent", "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\"");
+        expected.put("agent", "\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) " +
+                "Chrome/32.0.1700.77 Safari/537.36\"");
 
         Map<String, Object> actual = grok.captures(text);
 
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/GsubProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/GsubProcessorFactoryTests.java
similarity index 98%
rename from core/src/test/java/org/elasticsearch/ingest/processor/GsubProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/GsubProcessorFactoryTests.java
index 7f56429df9e..f840576e522 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/GsubProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/GsubProcessorFactoryTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/GsubProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/GsubProcessorTests.java
similarity index 95%
rename from core/src/test/java/org/elasticsearch/ingest/processor/GsubProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/GsubProcessorTests.java
index fe44f33b614..066609f5e3a 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/GsubProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/GsubProcessorTests.java
@@ -17,10 +17,9 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.ingest.core.Processor;
 import org.elasticsearch.test.ESTestCase;
 
@@ -50,7 +49,8 @@ public class GsubProcessorTests extends ESTestCase {
             processor.execute(ingestDocument);
             fail("processor execution should have failed");
         } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("field [" + fieldName + "] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
+            assertThat(e.getMessage(), equalTo("field [" + fieldName +
+                    "] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
         }
     }
 
diff --git a/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/IngestGrokRestIT.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/IngestCommonRestIT.java
similarity index 81%
rename from modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/IngestGrokRestIT.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/IngestCommonRestIT.java
index 3f4bdf1f8b9..544e47b8e94 100644
--- a/modules/ingest-grok/src/test/java/org/elasticsearch/ingest/grok/IngestGrokRestIT.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/IngestCommonRestIT.java
@@ -17,22 +17,19 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.grok;
+package org.elasticsearch.ingest;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import org.elasticsearch.ingest.grok.IngestGrokPlugin;
-import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.RestTestCandidate;
 import org.elasticsearch.test.rest.parser.RestTestParseException;
 
 import java.io.IOException;
-import java.util.Collection;
 
-public class IngestGrokRestIT extends ESRestTestCase {
+public class IngestCommonRestIT extends ESRestTestCase {
 
-    public IngestGrokRestIT(@Name("yaml") RestTestCandidate testCandidate) {
+    public IngestCommonRestIT(@Name("yaml") RestTestCandidate testCandidate) {
         super(testCandidate);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/JoinProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/JoinProcessorFactoryTests.java
similarity index 96%
rename from core/src/test/java/org/elasticsearch/ingest/processor/JoinProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/JoinProcessorFactoryTests.java
index c374b8a3318..45e643579a4 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/JoinProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/JoinProcessorFactoryTests.java
@@ -17,11 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.Processor;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/JoinProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/JoinProcessorTests.java
similarity index 98%
rename from core/src/test/java/org/elasticsearch/ingest/processor/JoinProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/JoinProcessorTests.java
index 2aa3ac2e035..39906fb4850 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/JoinProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/JoinProcessorTests.java
@@ -17,10 +17,9 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.ingest.core.Processor;
 import org.elasticsearch.test.ESTestCase;
 
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/LowercaseProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/LowercaseProcessorFactoryTests.java
similarity index 95%
rename from core/src/test/java/org/elasticsearch/ingest/processor/LowercaseProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/LowercaseProcessorFactoryTests.java
index 09d676b3b30..b7a315b176b 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/LowercaseProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/LowercaseProcessorFactoryTests.java
@@ -17,11 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.Processor;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/LowercaseProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/LowercaseProcessorTests.java
similarity index 96%
rename from core/src/test/java/org/elasticsearch/ingest/processor/LowercaseProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/LowercaseProcessorTests.java
index 77e22b0ca1e..81e23ef4689 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/LowercaseProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/LowercaseProcessorTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import java.util.Locale;
 
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/RemoveProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/RemoveProcessorFactoryTests.java
similarity index 93%
rename from core/src/test/java/org/elasticsearch/ingest/processor/RemoveProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/RemoveProcessorFactoryTests.java
index 1b9d88160bd..672058b61e4 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/RemoveProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/RemoveProcessorFactoryTests.java
@@ -17,12 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.Processor;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/RemoveProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/RemoveProcessorTests.java
similarity index 93%
rename from core/src/test/java/org/elasticsearch/ingest/processor/RemoveProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/RemoveProcessorTests.java
index d134b0213eb..abc68aa60b6 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/RemoveProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/RemoveProcessorTests.java
@@ -17,11 +17,9 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.RandomDocumentPicks;
-import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.ingest.core.Processor;
 import org.elasticsearch.test.ESTestCase;
 
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/RenameProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/RenameProcessorFactoryTests.java
similarity index 98%
rename from core/src/test/java/org/elasticsearch/ingest/processor/RenameProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/RenameProcessorFactoryTests.java
index 768575991d9..005abb8c0d0 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/RenameProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/RenameProcessorFactoryTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/RenameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/RenameProcessorTests.java
similarity index 97%
rename from core/src/test/java/org/elasticsearch/ingest/processor/RenameProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/RenameProcessorTests.java
index 1f9bddaac6b..85faca03e9c 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/RenameProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/RenameProcessorTests.java
@@ -17,10 +17,9 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.ingest.core.Processor;
 import org.elasticsearch.test.ESTestCase;
 
@@ -102,7 +101,8 @@ public class RenameProcessorTests extends ESTestCase {
     public void testRenameNewFieldAlreadyExists() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         String fieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
-        Processor processor = new RenameProcessor(randomAsciiOfLength(10), RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument), fieldName);
+        Processor processor = new RenameProcessor(randomAsciiOfLength(10), RandomDocumentPicks.randomExistingFieldName(
+                random(), ingestDocument), fieldName);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/SetProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/SetProcessorFactoryTests.java
similarity index 97%
rename from core/src/test/java/org/elasticsearch/ingest/processor/SetProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/SetProcessorFactoryTests.java
index 8810517de8f..acb09b0c96f 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/SetProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/SetProcessorFactoryTests.java
@@ -17,10 +17,9 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/SetProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/SetProcessorTests.java
similarity index 94%
rename from core/src/test/java/org/elasticsearch/ingest/processor/SetProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/SetProcessorTests.java
index 37c20e186b9..1d28124d55a 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/SetProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/SetProcessorTests.java
@@ -17,12 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.ingest.core.TemplateService;
-import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.ingest.core.ValueSource;
 import org.elasticsearch.ingest.core.Processor;
 import org.elasticsearch.test.ESTestCase;
@@ -64,7 +62,8 @@ public class SetProcessorTests extends ESTestCase {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
         } catch(IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("cannot set [inner] with parent object of type [java.lang.String] as part of path [field.inner]"));
+            assertThat(e.getMessage(), equalTo("cannot set [inner] with parent object of type [java.lang.String] as " +
+                    "part of path [field.inner]"));
         }
     }
 
@@ -109,6 +108,7 @@ public class SetProcessorTests extends ESTestCase {
 
     private static Processor createSetProcessor(String fieldName, Object fieldValue, boolean overrideEnabled) {
         TemplateService templateService = TestTemplateService.instance();
-        return new SetProcessor(randomAsciiOfLength(10), templateService.compile(fieldName), ValueSource.wrap(fieldValue, templateService), overrideEnabled);
+        return new SetProcessor(randomAsciiOfLength(10), templateService.compile(fieldName),
+                ValueSource.wrap(fieldValue, templateService), overrideEnabled);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/SortProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/SortProcessorTests.java
similarity index 98%
rename from core/src/test/java/org/elasticsearch/ingest/processor/SortProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/SortProcessorTests.java
index 18e35d00a79..40da784bf34 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/SortProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/SortProcessorTests.java
@@ -17,12 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.ingest.core.Processor;
-import org.elasticsearch.ingest.processor.SortProcessor.SortOrder;
+import org.elasticsearch.ingest.SortProcessor.SortOrder;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
@@ -30,8 +29,6 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
-import java.util.Random;
-import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/SplitProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/SplitProcessorFactoryTests.java
similarity index 96%
rename from core/src/test/java/org/elasticsearch/ingest/processor/SplitProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/SplitProcessorFactoryTests.java
index 70fca6f501b..7d0c3c4d970 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/SplitProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/SplitProcessorFactoryTests.java
@@ -17,11 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.Processor;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/SplitProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/SplitProcessorTests.java
similarity index 94%
rename from core/src/test/java/org/elasticsearch/ingest/processor/SplitProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/SplitProcessorTests.java
index e1c8a626a4b..95bb4fe4109 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/SplitProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/SplitProcessorTests.java
@@ -17,9 +17,8 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
-import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.ingest.core.IngestDocument;
 import org.elasticsearch.ingest.core.Processor;
 import org.elasticsearch.test.ESTestCase;
@@ -75,7 +74,8 @@ public class SplitProcessorTests extends ESTestCase {
             processor.execute(ingestDocument);
             fail("split processor should have failed");
         } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("field [" + fieldName + "] of type [java.lang.Integer] cannot be cast to [java.lang.String]"));
+            assertThat(e.getMessage(), equalTo("field [" + fieldName + "] of type [java.lang.Integer] cannot be cast " +
+                    "to [java.lang.String]"));
         }
     }
 
@@ -92,6 +92,7 @@ public class SplitProcessorTests extends ESTestCase {
         List<String> flags = (List<String>)ingestDocument.getFieldValue("flags", List.class);
         assertThat(flags, equalTo(Arrays.asList("new", "hot", "super", "fun", "interesting")));
         ingestDocument.appendFieldValue("flags", "additional_flag");
-        assertThat(ingestDocument.getFieldValue("flags", List.class), equalTo(Arrays.asList("new", "hot", "super", "fun", "interesting", "additional_flag")));
+        assertThat(ingestDocument.getFieldValue("flags", List.class), equalTo(Arrays.asList("new", "hot", "super",
+                "fun", "interesting", "additional_flag")));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/TrimProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/TrimProcessorFactoryTests.java
similarity index 95%
rename from core/src/test/java/org/elasticsearch/ingest/processor/TrimProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/TrimProcessorFactoryTests.java
index 1e74b78f973..0692d53c9e3 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/TrimProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/TrimProcessorFactoryTests.java
@@ -17,11 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.Processor;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/TrimProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/TrimProcessorTests.java
similarity index 97%
rename from core/src/test/java/org/elasticsearch/ingest/processor/TrimProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/TrimProcessorTests.java
index a0e5fdeab2b..ce51dea3acb 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/TrimProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/TrimProcessorTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 public class TrimProcessorTests extends AbstractStringProcessorTestCase {
 
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/UppercaseProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/UppercaseProcessorFactoryTests.java
similarity index 95%
rename from core/src/test/java/org/elasticsearch/ingest/processor/UppercaseProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/UppercaseProcessorFactoryTests.java
index 40e14b5f14d..0b5eafd2173 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/UppercaseProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/UppercaseProcessorFactoryTests.java
@@ -17,11 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.Processor;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
diff --git a/core/src/test/java/org/elasticsearch/ingest/processor/UppercaseProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/UppercaseProcessorTests.java
similarity index 96%
rename from core/src/test/java/org/elasticsearch/ingest/processor/UppercaseProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/UppercaseProcessorTests.java
index 4ab61f7b5e3..6881d117946 100644
--- a/core/src/test/java/org/elasticsearch/ingest/processor/UppercaseProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/UppercaseProcessorTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.processor;
+package org.elasticsearch.ingest;
 
 import java.util.Locale;
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/90_date_index_name_processor.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yaml
similarity index 100%
rename from rest-api-spec/src/main/resources/rest-api-spec/test/ingest/90_date_index_name_processor.yaml
rename to modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/100_date_index_name_processor.yaml
diff --git a/modules/ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok/10_basic.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/10_basic.yaml
similarity index 93%
rename from modules/ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok/10_basic.yaml
rename to modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/10_basic.yaml
index c41793fcdb8..1f6962c6e2f 100644
--- a/modules/ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok/10_basic.yaml
+++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/10_basic.yaml
@@ -1,4 +1,4 @@
-"Ingest grok installed":
+"Ingest common installed":
     - do:
         cluster.state: {}
 
@@ -8,7 +8,7 @@
     - do:
         nodes.info: {}
 
-    - match:  { nodes.$master.modules.0.name: ingest-grok  }
+    - match:  { nodes.$master.modules.0.name: ingest-common  }
     - match:  { nodes.$master.ingest.processors.0.type: append }
     - match:  { nodes.$master.ingest.processors.1.type: convert }
     - match:  { nodes.$master.ingest.processors.2.type: date }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/100_sort.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/110_sort.yaml
similarity index 100%
rename from rest-api-spec/src/main/resources/rest-api-spec/test/ingest/100_sort.yaml
rename to modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/110_sort.yaml
diff --git a/modules/ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok/20_grok.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yaml
similarity index 100%
rename from modules/ingest-grok/src/test/resources/rest-api-spec/test/ingest_grok/20_grok.yaml
rename to modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/120_grok.yaml
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/10_crud.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yaml
similarity index 100%
rename from rest-api-spec/src/main/resources/rest-api-spec/test/ingest/10_crud.yaml
rename to modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/20_crud.yaml
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/20_date_processor.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yaml
similarity index 100%
rename from rest-api-spec/src/main/resources/rest-api-spec/test/ingest/20_date_processor.yaml
rename to modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/30_date_processor.yaml
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/30_mutate.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/40_mutate.yaml
similarity index 100%
rename from rest-api-spec/src/main/resources/rest-api-spec/test/ingest/30_mutate.yaml
rename to modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/40_mutate.yaml
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/50_on_failure.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yaml
similarity index 100%
rename from rest-api-spec/src/main/resources/rest-api-spec/test/ingest/50_on_failure.yaml
rename to modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yaml
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/60_fail.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/60_fail.yaml
similarity index 100%
rename from rest-api-spec/src/main/resources/rest-api-spec/test/ingest/60_fail.yaml
rename to modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/60_fail.yaml
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/70_bulk.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/70_bulk.yaml
similarity index 100%
rename from rest-api-spec/src/main/resources/rest-api-spec/test/ingest/70_bulk.yaml
rename to modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/70_bulk.yaml
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/80_foreach.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/80_foreach.yaml
similarity index 100%
rename from rest-api-spec/src/main/resources/rest-api-spec/test/ingest/80_foreach.yaml
rename to modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/80_foreach.yaml
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/40_simulate.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/90_simulate.yaml
similarity index 100%
rename from rest-api-spec/src/main/resources/rest-api-spec/test/ingest/40_simulate.yaml
rename to modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/90_simulate.yaml
diff --git a/modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/IngestGrokPlugin.java b/modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/IngestGrokPlugin.java
deleted file mode 100644
index 998c536b474..00000000000
--- a/modules/ingest-grok/src/main/java/org/elasticsearch/ingest/grok/IngestGrokPlugin.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.ingest.grok;
-
-import org.elasticsearch.node.NodeModule;
-import org.elasticsearch.plugins.Plugin;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-public class IngestGrokPlugin extends Plugin {
-
-    private static final String[] PATTERN_NAMES = new String[] {
-        "aws", "bacula", "bro", "exim", "firewalls", "grok-patterns", "haproxy",
-        "java", "junos", "linux-syslog", "mcollective-patterns", "mongodb", "nagios",
-        "postgresql", "rails", "redis", "ruby"
-    };
-
-    private final Map<String, String> builtinPatterns;
-
-    public IngestGrokPlugin() throws IOException {
-        this.builtinPatterns = loadBuiltinPatterns();
-    }
-
-    @Override
-    public String name() {
-        return "ingest-grok";
-    }
-
-    @Override
-    public String description() {
-        return "Ingest processor that uses grok patterns to split text";
-    }
-
-    public void onModule(NodeModule nodeModule) {
-        nodeModule.registerProcessor(GrokProcessor.TYPE, (templateService, registry) -> new GrokProcessor.Factory(builtinPatterns));
-    }
-
-    public static Map<String, String> loadBuiltinPatterns() throws IOException {
-        Map<String, String> builtinPatterns = new HashMap<>();
-        for (String pattern : PATTERN_NAMES) {
-            try(InputStream is = IngestGrokPlugin.class.getResourceAsStream("/patterns/" + pattern)) {
-                loadPatterns(builtinPatterns, is);
-            }
-        }
-        return Collections.unmodifiableMap(builtinPatterns);
-    }
-
-    private static void loadPatterns(Map<String, String> patternBank, InputStream inputStream) throws IOException {
-        String line;
-        BufferedReader br = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
-        while ((line = br.readLine()) != null) {
-            String trimmedLine = line.replaceAll("^\\s+", "");
-            if (trimmedLine.startsWith("#") || trimmedLine.length() == 0) {
-                continue;
-            }
-
-            String[] parts = trimmedLine.split("\\s+", 2);
-            if (parts.length == 2) {
-                patternBank.put(parts[0], parts[1]);
-            }
-        }
-    }
-}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
index 1f9cd6b1394..2bb7fd627e4 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
@@ -31,6 +31,7 @@ import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.Engine.Operation.Origin;
 import org.elasticsearch.index.shard.IndexingOperationListener;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.ingest.IngestTestPlugin;
 import org.junit.BeforeClass;
 
 import java.util.ArrayList;
@@ -68,6 +69,7 @@ public class CancelTests extends ReindexTestCase {
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         Collection<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
+        plugins.add(IngestTestPlugin.class);
         plugins.add(ReindexCancellationPlugin.class);
         return plugins;
     }
@@ -157,19 +159,16 @@ public class CancelTests extends ReindexTestCase {
 
     public void testUpdateByQueryCancel() throws Exception {
         BytesReference pipeline = new BytesArray("{\n" +
-                "  \"description\" : \"sets updated to true\",\n" +
+                "  \"description\" : \"sets processed to true\",\n" +
                 "  \"processors\" : [ {\n" +
-                "      \"set\" : {\n" +
-                "        \"field\": \"updated\",\n" +
-                "        \"value\": true" +
-                "      }\n" +
+                "      \"test\" : {}\n" +
                 "  } ]\n" +
                 "}");
         assertAcked(client().admin().cluster().preparePutPipeline("set-foo", pipeline).get());
 
         testCancel(UpdateByQueryAction.NAME, updateByQuery().setPipeline("set-foo").source(INDEX), (response, total, modified) -> {
             assertThat(response, matcher().updated(modified).reasonCancelled(equalTo("by user request")));
-            assertHitCount(client().prepareSearch(INDEX).setSize(0).setQuery(termQuery("updated", true)).get(), modified);
+            assertHitCount(client().prepareSearch(INDEX).setSize(0).setQuery(termQuery("processed", true)).get(), modified);
         });
 
         assertAcked(client().admin().cluster().deletePipeline(new DeletePipelineRequest("set-foo")).get());
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_throttle.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yaml
similarity index 100%
rename from modules/reindex/src/test/resources/rest-api-spec/test/reindex/80_throttle.yaml
rename to modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yaml
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/70_throttle.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yaml
similarity index 100%
rename from modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/70_throttle.yaml
rename to modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yaml
diff --git a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/10_basic.yaml b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/10_basic.yaml
index 7546090d6ad..88accac7730 100644
--- a/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/10_basic.yaml
+++ b/plugins/ingest-attachment/src/test/resources/rest-api-spec/test/ingest_attachment/10_basic.yaml
@@ -8,21 +8,5 @@
         nodes.info: {}
 
     - match:  { nodes.$master.plugins.0.name: ingest-attachment  }
-    - match:  { nodes.$master.ingest.processors.0.type: append }
-    - match:  { nodes.$master.ingest.processors.1.type: attachment }
-    - match:  { nodes.$master.ingest.processors.2.type: convert }
-    - match:  { nodes.$master.ingest.processors.3.type: date }
-    - match:  { nodes.$master.ingest.processors.4.type: date_index_name }
-    - match:  { nodes.$master.ingest.processors.5.type: fail }
-    - match:  { nodes.$master.ingest.processors.6.type: foreach }
-    - match:  { nodes.$master.ingest.processors.7.type: gsub }
-    - match:  { nodes.$master.ingest.processors.8.type: join }
-    - match:  { nodes.$master.ingest.processors.9.type: lowercase }
-    - match:  { nodes.$master.ingest.processors.10.type: remove }
-    - match:  { nodes.$master.ingest.processors.11.type: rename }
-    - match:  { nodes.$master.ingest.processors.12.type: set }
-    - match:  { nodes.$master.ingest.processors.13.type: sort }
-    - match:  { nodes.$master.ingest.processors.14.type: split }
-    - match:  { nodes.$master.ingest.processors.15.type: trim }
-    - match:  { nodes.$master.ingest.processors.16.type: uppercase }
+    - match:  { nodes.$master.ingest.processors.0.type: attachment }
 
diff --git a/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/10_basic.yaml b/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/10_basic.yaml
index 313a9b4544f..7a06326a864 100644
--- a/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/10_basic.yaml
+++ b/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/10_basic.yaml
@@ -8,20 +8,4 @@
         nodes.info: {}
 
     - match:  { nodes.$master.plugins.0.name: ingest-geoip  }
-    - match:  { nodes.$master.ingest.processors.0.type: append }
-    - match:  { nodes.$master.ingest.processors.1.type: convert }
-    - match:  { nodes.$master.ingest.processors.2.type: date }
-    - match:  { nodes.$master.ingest.processors.3.type: date_index_name }
-    - match:  { nodes.$master.ingest.processors.4.type: fail }
-    - match:  { nodes.$master.ingest.processors.5.type: foreach }
-    - match:  { nodes.$master.ingest.processors.6.type: geoip }
-    - match:  { nodes.$master.ingest.processors.7.type: gsub }
-    - match:  { nodes.$master.ingest.processors.8.type: join }
-    - match:  { nodes.$master.ingest.processors.9.type: lowercase }
-    - match:  { nodes.$master.ingest.processors.10.type: remove }
-    - match:  { nodes.$master.ingest.processors.11.type: rename }
-    - match:  { nodes.$master.ingest.processors.12.type: set }
-    - match:  { nodes.$master.ingest.processors.13.type: sort }
-    - match:  { nodes.$master.ingest.processors.14.type: split }
-    - match:  { nodes.$master.ingest.processors.15.type: trim }
-    - match:  { nodes.$master.ingest.processors.16.type: uppercase }
+    - match:  { nodes.$master.ingest.processors.0.type: geoip }
diff --git a/qa/smoke-test-ingest-disabled/build.gradle b/qa/smoke-test-ingest-disabled/build.gradle
index 09b2d1409a1..08dfbf8ae7a 100644
--- a/qa/smoke-test-ingest-disabled/build.gradle
+++ b/qa/smoke-test-ingest-disabled/build.gradle
@@ -19,6 +19,10 @@
 
 apply plugin: 'elasticsearch.rest-test'
 
+dependencies {
+    testCompile project(path: ':modules:ingest-common', configuration: 'runtime')
+}
+
 integTest {
     cluster {
         setting 'node.ingest', 'false'
diff --git a/qa/smoke-test-ingest-with-all-dependencies/build.gradle b/qa/smoke-test-ingest-with-all-dependencies/build.gradle
index f81afc729b7..60cd9f80a84 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/build.gradle
+++ b/qa/smoke-test-ingest-with-all-dependencies/build.gradle
@@ -20,9 +20,10 @@
 apply plugin: 'elasticsearch.rest-test'
 
 dependencies {
-    testCompile project(path: ':modules:ingest-grok', configuration: 'runtime')
+    testCompile project(path: ':modules:ingest-common', configuration: 'runtime')
     testCompile project(path: ':plugins:ingest-geoip', configuration: 'runtime')
     testCompile project(path: ':modules:lang-mustache', configuration: 'runtime')
+    testCompile project(path: ':modules:reindex', configuration: 'runtime')
 }
 
 integTest {
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestMustacheRemoveProcessorIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestMustacheRemoveProcessorIT.java
deleted file mode 100644
index 621cbbc2beb..00000000000
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestMustacheRemoveProcessorIT.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.ingest;
-
-import org.elasticsearch.ingest.processor.RemoveProcessor;
-import org.hamcrest.CoreMatchers;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-public class IngestMustacheRemoveProcessorIT extends AbstractMustacheTestCase {
-
-    public void testRemoveProcessorMustacheExpression() throws Exception {
-        RemoveProcessor.Factory factory = new RemoveProcessor.Factory(templateService);
-        Map<String, Object> config = new HashMap<>();
-        config.put("field", "field{{var}}");
-        RemoveProcessor processor = factory.create(config);
-        assertThat(processor.getField().execute(Collections.singletonMap("var", "_value")), CoreMatchers.equalTo("field_value"));
-    }
-}
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestMustacheSetProcessorIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestMustacheSetProcessorIT.java
deleted file mode 100644
index f088f84d144..00000000000
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestMustacheSetProcessorIT.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.ingest;
-
-
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.ValueSource;
-import org.elasticsearch.ingest.core.Processor;
-import org.elasticsearch.ingest.processor.SetProcessor;
-import org.hamcrest.Matchers;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.instanceOf;
-
-public class IngestMustacheSetProcessorIT extends AbstractMustacheTestCase {
-
-    public void testExpression() throws Exception {
-        SetProcessor processor = createSetProcessor("_index", "text {{var}}");
-        assertThat(processor.getValue(), instanceOf(ValueSource.TemplatedValue.class));
-        assertThat(processor.getValue().copyAndResolve(Collections.singletonMap("var", "_value")), equalTo("text _value"));
-    }
-
-    public void testSetMetadataWithTemplates() throws Exception {
-        IngestDocument.MetaData randomMetaData = randomFrom(IngestDocument.MetaData.values());
-        Processor processor = createSetProcessor(randomMetaData.getFieldName(), "_value {{field}}");
-        IngestDocument ingestDocument = createIngestDocument(Collections.singletonMap("field", "value"));
-        processor.execute(ingestDocument);
-        assertThat(ingestDocument.getFieldValue(randomMetaData.getFieldName(), String.class), Matchers.equalTo("_value value"));
-    }
-
-    public void testSetWithTemplates() throws Exception {
-        IngestDocument.MetaData randomMetaData = randomFrom(IngestDocument.MetaData.INDEX, IngestDocument.MetaData.TYPE,
-                IngestDocument.MetaData.ID);
-        Processor processor = createSetProcessor("field{{_type}}", "_value {{" + randomMetaData.getFieldName() + "}}");
-        IngestDocument ingestDocument = createIngestDocument(new HashMap<>());
-        processor.execute(ingestDocument);
-        assertThat(ingestDocument.getFieldValue("field_type", String.class),
-                Matchers.equalTo("_value " + ingestDocument.getFieldValue(randomMetaData.getFieldName(), String.class)));
-    }
-
-    private SetProcessor createSetProcessor(String fieldName, Object fieldValue) throws Exception {
-        SetProcessor.Factory factory = new SetProcessor.Factory(templateService);
-        Map<String, Object> config = new HashMap<>();
-        config.put("field", fieldName);
-        config.put("value", fieldValue);
-        return factory.create(config);
-    }
-
-    private IngestDocument createIngestDocument(Map<String, Object> source) {
-        return new IngestDocument("_index", "_type", "_id", null, null, null, null, source);
-    }
-
-}
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_with_ingest.yaml b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/30_with_ingest.yaml
similarity index 100%
rename from modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_with_ingest.yaml
rename to qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/30_with_ingest.yaml
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_with_ingest.yaml b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/40_with_ingest.yaml
similarity index 100%
rename from modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_with_ingest.yaml
rename to qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/40_with_ingest.yaml
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/10_basic.yaml
new file mode 100644
index 00000000000..74396249e81
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/ingest/10_basic.yaml
@@ -0,0 +1,45 @@
+---
+"Test basic pipeline crud":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      ingest.get_pipeline:
+        id: "my_pipeline"
+  - match: { pipelines.0.id: "my_pipeline" }
+  - match: { pipelines.0.config.description: "_description" }
+
+  - do:
+      ingest.delete_pipeline:
+        id: "my_pipeline"
+  - match: { acknowledged: true }
+
+  - do:
+      catch: missing
+      ingest.get_pipeline:
+        id: "my_pipeline"
+
+---
+"Test invalid config":
+  - do:
+      catch: /parse_exception/
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "set" : {
+                }
+              }
+            ]
+          }
diff --git a/settings.gradle b/settings.gradle
index 215b436e20a..457e89c906f 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -15,7 +15,7 @@ List projects = [
   'test:fixtures:hdfs-fixture',
   'test:logger-usage',
   'modules:aggs-matrix-stats',
-  'modules:ingest-grok',
+  'modules:ingest-common',
   'modules:lang-expression',
   'modules:lang-groovy',
   'modules:lang-mustache',
diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java b/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java
new file mode 100644
index 00000000000..696799d31fb
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest;
+
+import org.elasticsearch.node.NodeModule;
+import org.elasticsearch.plugins.Plugin;
+
+/**
+ * Adds an ingest processor to be used in tests.
+ */
+public class IngestTestPlugin extends Plugin {
+
+    @Override
+    public String name() {
+        return "ingest-test";
+    }
+
+    @Override
+    public String description() {
+        return "Contains an ingest processor to be used in tests";
+    }
+
+    public void onModule(NodeModule nodeModule) {
+        nodeModule.registerProcessor("test", (templateService, registry) -> config ->
+                new TestProcessor("id", "test", doc -> {
+                    doc.setFieldValue("processed", true);
+                    if (doc.hasField("fail") && doc.getFieldValue("fail", Boolean.class)) {
+                        throw new IllegalArgumentException("test processor failed");
+                    }
+                })
+        );
+    }
+}

From 89224dbc3cee4a7e5bf6af5e666bd58a1aa30b75 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 7 Jun 2016 11:56:44 -0400
Subject: [PATCH 0701/1311] switch FunctionRef over to methodhandles

---
 .../java/org/elasticsearch/painless/Def.java  | 22 +++----
 .../elasticsearch/painless/FunctionRef.java   | 58 ++++++++++---------
 .../painless/node/EFunctionRef.java           | 30 +++++++---
 3 files changed, 65 insertions(+), 45 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index e7f56bdbeec..1edaddea2c1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -246,24 +246,24 @@ public final class Def {
          int separator = signature.indexOf('.');
          FunctionRef ref = new FunctionRef(clazz, signature.substring(0, separator), signature.substring(separator+1));
          final CallSite callSite;
-         if (ref.interfaceType.equals(ref.samType)) {
+         if (ref.needsBridges()) {
              callSite = LambdaMetafactory.altMetafactory(lookup, 
                      ref.invokedName, 
-                     ref.invokedMethodType,
+                     ref.invokedType,
                      ref.samMethodType,
-                     ref.implMethodHandle,
-                     ref.samMethodType,
-                     0);
-         } else {
-             callSite = LambdaMetafactory.altMetafactory(lookup, 
-                     ref.invokedName, 
-                     ref.invokedMethodType,
-                     ref.samMethodType,
-                     ref.implMethodHandle,
+                     ref.implMethod,
                      ref.samMethodType,
                      LambdaMetafactory.FLAG_BRIDGES,
                      1,
                      ref.interfaceMethodType);
+         } else {
+             callSite = LambdaMetafactory.altMetafactory(lookup, 
+                     ref.invokedName, 
+                     ref.invokedType,
+                     ref.samMethodType,
+                     ref.implMethod,
+                     ref.samMethodType,
+                     0);
          }
          // we could actually invoke and cache here (in non-capturing cases), but this is not a speedup.
          MethodHandle factory = callSite.dynamicInvoker().asType(MethodType.methodType(clazz.clazz));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
index 3ac6a29527b..03b6cc604c9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
@@ -22,31 +22,38 @@ package org.elasticsearch.painless;
 import org.elasticsearch.painless.Definition.Method;
 import org.objectweb.asm.Handle;
 import org.objectweb.asm.Opcodes;
-import org.objectweb.asm.Type;
 
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodType;
 import java.lang.reflect.Modifier;
 
 /** 
- * computes "everything you need" to call LambdaMetaFactory, given an expected interface,
- * and reference class + method name
+ * Reference to a function or lambda. 
+ * <p>
+ * Once you have created one of these, you have "everything you need" to call LambdaMetaFactory
+ * either statically from bytecode with invokedynamic, or at runtime from Java.  
  */
 public class FunctionRef {
+    /** Function Object's method name */
     public final String invokedName;
-
-    public final Type invokedType;
-    public final MethodType invokedMethodType;
-
-    public final Handle implMethod;
-    public final MethodHandle implMethodHandle;
-
-    public final Type samType;
+    /** CallSite signature */
+    public final MethodType invokedType;
+    /** Implementation method */
+    public final MethodHandle implMethod;
+    /** Function Object's method signature */
     public final MethodType samMethodType;
-
-    public final Type interfaceType;
+    /** When bridging is required, request this bridge interface */
     public final MethodType interfaceMethodType;
     
+    /** ASM "Handle" to the method, for the constant pool */
+    public final Handle implMethodASM;
+    
+    /**
+     * Creates a new FunctionRef.
+     * @param expected interface type to implement.
+     * @param type the left hand side of a method reference expression
+     * @param call the right hand side of a method reference expression
+     */
     public FunctionRef(Definition.Type expected, String type, String call) {
         boolean isCtorReference = "new".equals(call);
         // check its really a functional interface
@@ -59,10 +66,8 @@ public class FunctionRef {
         // e.g. compareTo
         invokedName = method.name;
         // e.g. (Object)Comparator
-        invokedType = Type.getMethodType(expected.type);
-        invokedMethodType = MethodType.methodType(expected.clazz);
+        invokedType = MethodType.methodType(expected.clazz);
         // e.g. (Object,Object)int
-        interfaceType = Type.getMethodType(method.method.getDescriptor());
         interfaceMethodType = method.handle.type().dropParameterTypes(0, 1);
         // lookup requested method
         Definition.Struct struct = Definition.getType(type).struct;
@@ -93,21 +98,22 @@ public class FunctionRef {
         } else {
             tag = Opcodes.H_INVOKEVIRTUAL;
         }
-        implMethod = new Handle(tag, struct.type.getInternalName(), impl.name, impl.method.getDescriptor());
-        implMethodHandle = impl.handle;
+        implMethodASM = new Handle(tag, struct.type.getInternalName(), impl.name, impl.method.getDescriptor());
+        implMethod = impl.handle;
         if (isCtorReference) {
-            samType = Type.getMethodType(interfaceType.getReturnType(), impl.method.getArgumentTypes());
             samMethodType = MethodType.methodType(interfaceMethodType.returnType(), impl.handle.type().parameterArray());
         } else if (Modifier.isStatic(impl.modifiers)) {
-            samType = Type.getMethodType(impl.method.getReturnType(), impl.method.getArgumentTypes());
             samMethodType = impl.handle.type();
         } else {
-            Type[] argTypes = impl.method.getArgumentTypes();
-            Type[] params = new Type[argTypes.length + 1];
-            System.arraycopy(argTypes, 0, params, 1, argTypes.length);
-            params[0] = struct.type;
-            samType = Type.getMethodType(impl.method.getReturnType(), params);
-            samMethodType = impl.handle.type();
+            // ensure the receiver type is exact and not a superclass type
+            samMethodType = impl.handle.type().changeParameterType(0, struct.clazz);
         }
     }
+    
+    /** Returns true if you should ask LambdaMetaFactory to construct a bridge for the interface signature */
+    public boolean needsBridges() {
+        // currently if the interface differs, we ask for a bridge, but maybe we should do smarter checking?
+        // either way, stuff will fail if its wrong :)
+        return interfaceMethodType.equals(samMethodType) == false;
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index 2f8394ebc17..38b7a1b8ddb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -24,6 +24,7 @@ import org.elasticsearch.painless.FunctionRef;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.Variables;
+import org.objectweb.asm.Type;
 
 import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE;
 
@@ -66,15 +67,28 @@ public class EFunctionRef extends AExpression {
             writer.push(type + "." + call);
         } else {
             writer.writeDebugInfo(location);
-            // currently if the interface differs, we ask for a bridge, but maybe we should do smarter checking?
-            // either way, stuff will fail if its wrong :)
-            if (ref.interfaceType.equals(ref.samType)) {
-                writer.invokeDynamic(ref.invokedName, ref.invokedType.getDescriptor(), LAMBDA_BOOTSTRAP_HANDLE, 
-                                     ref.samType, ref.implMethod, ref.samType, 0);
+            // convert MethodTypes to asm Type for the constant pool.
+            String invokedType = ref.invokedType.toMethodDescriptorString();
+            Type samMethodType = Type.getMethodType(ref.samMethodType.toMethodDescriptorString());
+            Type interfaceType = Type.getMethodType(ref.interfaceMethodType.toMethodDescriptorString());
+            if (ref.needsBridges()) {
+                writer.invokeDynamic(ref.invokedName, 
+                                     invokedType, 
+                                     LAMBDA_BOOTSTRAP_HANDLE, 
+                                     samMethodType, 
+                                     ref.implMethodASM, 
+                                     samMethodType, 
+                                     LambdaMetafactory.FLAG_BRIDGES, 
+                                     1, 
+                                     interfaceType);
             } else {
-                writer.invokeDynamic(ref.invokedName, ref.invokedType.getDescriptor(), LAMBDA_BOOTSTRAP_HANDLE, 
-                                     ref.samType, ref.implMethod, ref.samType, 
-                                     LambdaMetafactory.FLAG_BRIDGES, 1, ref.interfaceType);
+                writer.invokeDynamic(ref.invokedName, 
+                                     invokedType, 
+                                     LAMBDA_BOOTSTRAP_HANDLE, 
+                                     samMethodType, 
+                                     ref.implMethodASM, 
+                                     samMethodType, 
+                                     0);
             }
         }
     }

From 43b419b23018b0b1628b71dccfb715270fe0d517 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Tue, 7 Jun 2016 17:58:32 +0200
Subject: [PATCH 0702/1311] rehash the docvalues in DocValuesSliceQuery using
 BitMixer.mix instead of the naive Long.hashCode.

---
 .../org/elasticsearch/search/slice/DocValuesSliceQuery.java  | 3 ++-
 .../elasticsearch/search/slice/DocValuesSliceQueryTests.java | 5 +++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java b/core/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java
index 4f334ebf0d1..95cafacedde 100644
--- a/core/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java
+++ b/core/src/main/java/org/elasticsearch/search/slice/DocValuesSliceQuery.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.search.slice;
 
+import com.carrotsearch.hppc.BitMixer;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.SortedNumericDocValues;
@@ -51,7 +52,7 @@ public final class DocValuesSliceQuery extends SliceQuery {
                     public boolean get(int doc) {
                         values.setDocument(doc);
                         for (int i = 0; i < values.count(); i++) {
-                            return contains(Long.hashCode(values.valueAt(i)));
+                            return contains(BitMixer.mix(values.valueAt(i)));
                         }
                         return contains(0);
                     }
diff --git a/core/src/test/java/org/elasticsearch/search/slice/DocValuesSliceQueryTests.java b/core/src/test/java/org/elasticsearch/search/slice/DocValuesSliceQueryTests.java
index dee2ae67914..846c411881f 100644
--- a/core/src/test/java/org/elasticsearch/search/slice/DocValuesSliceQueryTests.java
+++ b/core/src/test/java/org/elasticsearch/search/slice/DocValuesSliceQueryTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.search.slice;
 
+import com.carrotsearch.hppc.BitMixer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.SortedNumericDocValuesField;
@@ -77,8 +78,8 @@ public class DocValuesSliceQueryTests extends ESTestCase {
             doc.add(new SortedNumericDocValuesField("intField", intValue));
             doc.add(new SortedNumericDocValuesField("doubleField",  doubleValue));
             w.addDocument(doc);
-            sliceCounters1[Math.floorMod(Long.hashCode(intValue), max)] ++;
-            sliceCounters2[Math.floorMod(Long.hashCode(doubleValue), max)] ++;
+            sliceCounters1[Math.floorMod(BitMixer.mix((long) intValue), max)] ++;
+            sliceCounters2[Math.floorMod(BitMixer.mix(doubleValue), max)] ++;
             keys.add(uuid);
         }
         final IndexReader reader = w.getReader();

From 370b198f9f17aa05d09d41b62f19893256040474 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 7 Jun 2016 12:05:38 -0400
Subject: [PATCH 0703/1311] Remove explicit parallel new GC flag

Unless explicitly disabled, the parallel new collector is enabled
automatically as soon as the CMS collector is enabled:

  void Arguments::set_cms_and_parnew_gc_flags() {
    assert(
      !UseSerialGC && !UseParallelOldGC && !UseParallelGC,
      "Error");
    assert(UseConcMarkSweepGC, "CMS is expected to be on here");

    // If we are using CMS, we prefer to UseParNewGC,
    // unless explicitly forbidden.
    if (FLAG_IS_DEFAULT(UseParNewGC)) {
      FLAG_SET_ERGO(bool, UseParNewGC, true);
    }

While it's fine to be explicit, the UseParNewGC flag is deprecatd in JDK
8 and produces warning messages in JDK 9:

  Java HotSpot(TM) 64-Bit Server VM warning: Option UseParNewGC was
  deprecated in version 9.0 and will likely be removed in a future
  release.

Thus, we can and should just remove this flag from the default JVM
options.

Relates #18767
---
 distribution/src/main/resources/config/jvm.options | 1 -
 1 file changed, 1 deletion(-)

diff --git a/distribution/src/main/resources/config/jvm.options b/distribution/src/main/resources/config/jvm.options
index b27b0a0a0f0..fecb7e00493 100644
--- a/distribution/src/main/resources/config/jvm.options
+++ b/distribution/src/main/resources/config/jvm.options
@@ -33,7 +33,6 @@
 ################################################################
 
 ## GC configuration
--XX:+UseParNewGC
 -XX:+UseConcMarkSweepGC
 -XX:CMSInitiatingOccupancyFraction=75
 -XX:+UseCMSInitiatingOccupancyOnly

From feb244c14a3aba6fb11a2c22375d3650417fa6e5 Mon Sep 17 00:00:00 2001
From: Lee Hinman <lee@writequit.org>
Date: Mon, 23 May 2016 16:19:56 -0600
Subject: [PATCH 0704/1311] Remove cluster name from data path

Previously Elasticsearch used $DATA_DIR/$CLUSTER_NAME/nodes for the path
where data is stored, this commit changes that to be $DATA_DIR/nodes.

On startup, if the old folder structure is detected it will be used.
This behavior will be removed in Elasticsearch 6.0

Resolves #17810
---
 .../org/elasticsearch/bootstrap/Security.java | 26 ++++++++-
 .../org/elasticsearch/env/Environment.java    |  4 ++
 .../elasticsearch/env/NodeEnvironment.java    | 48 +++++++++++++++--
 .../env/NodeEnvironmentTests.java             | 54 +++++++++++++++++--
 .../migration/migrate_5_0/fs.asciidoc         | 15 ++++++
 .../bootstrap/EvilSecurityTests.java          |  1 +
 .../env/NodeEnvironmentEvilTests.java         |  4 +-
 7 files changed, 141 insertions(+), 11 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Security.java b/core/src/main/java/org/elasticsearch/bootstrap/Security.java
index 4437097bb35..b195d7b8a46 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Security.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Security.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.bootstrap;
 
 import org.elasticsearch.SecureSM;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.PathUtils;
@@ -256,8 +257,10 @@ final class Security {
         for (Path path : environment.dataFiles()) {
             addPath(policy, Environment.PATH_DATA_SETTING.getKey(), path, "read,readlink,write,delete");
         }
+        // TODO: this should be removed in ES 6.0! We will no longer support data paths with the cluster as a folder
+        assert Version.CURRENT.major < 6 : "cluster name is no longer used in data path";
         for (Path path : environment.dataWithClusterFiles()) {
-            addPath(policy, Environment.PATH_DATA_SETTING.getKey(), path, "read,readlink,write,delete");
+            addPathIfExists(policy, Environment.PATH_DATA_SETTING.getKey(), path, "read,readlink,write,delete");
         }
         for (Path path : environment.repoFiles()) {
             addPath(policy, Environment.PATH_REPO_SETTING.getKey(), path, "read,readlink,write,delete");
@@ -318,6 +321,27 @@ final class Security {
         policy.add(new FilePermission(path.toString() + path.getFileSystem().getSeparator() + "-", permissions));
     }
 
+    /**
+     * Add access to a directory iff it exists already
+     * @param policy current policy to add permissions to
+     * @param configurationName the configuration name associated with the path (for error messages only)
+     * @param path the path itself
+     * @param permissions set of filepermissions to grant to the path
+     */
+    static void addPathIfExists(Permissions policy, String configurationName, Path path, String permissions) {
+        if (Files.isDirectory(path)) {
+            // add each path twice: once for itself, again for files underneath it
+            policy.add(new FilePermission(path.toString(), permissions));
+            policy.add(new FilePermission(path.toString() + path.getFileSystem().getSeparator() + "-", permissions));
+            try {
+                path.getFileSystem().provider().checkAccess(path.toRealPath(), AccessMode.READ);
+            } catch (IOException e) {
+                throw new IllegalStateException("Unable to access '" + configurationName + "' (" + path + ")", e);
+            }
+        }
+    }
+
+
     /**
      * Ensures configured directory {@code path} exists.
      * @throws IOException if {@code path} exists, but is not a directory, not accessible, or broken symbolic link.
diff --git a/core/src/main/java/org/elasticsearch/env/Environment.java b/core/src/main/java/org/elasticsearch/env/Environment.java
index e022ce6ad2f..91dcfa333b9 100644
--- a/core/src/main/java/org/elasticsearch/env/Environment.java
+++ b/core/src/main/java/org/elasticsearch/env/Environment.java
@@ -200,7 +200,11 @@ public class Environment {
 
     /**
      * The data location with the cluster name as a sub directory.
+     *
+     * @deprecated Used to upgrade old data paths to new ones that do not include the cluster name, should not be used to write files to and
+     * will be removed in ES 6.0
      */
+    @Deprecated
     public Path[] dataWithClusterFiles() {
         return dataWithClusterFiles;
     }
diff --git a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
index 98b47d92246..2b2aab2ccd2 100644
--- a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
+++ b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
@@ -32,9 +32,12 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.FileSystemUtils;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -63,6 +66,7 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -161,6 +165,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
     public static final String NODES_FOLDER = "nodes";
     public static final String INDICES_FOLDER = "indices";
     public static final String NODE_LOCK_FILENAME = "node.lock";
+    public static final String UPGRADE_LOCK_FILENAME = "upgrade.lock";
 
     @Inject
     public NodeEnvironment(Settings settings, Environment environment) throws IOException {
@@ -175,7 +180,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
             localNodeId = -1;
             return;
         }
-        final NodePath[] nodePaths = new NodePath[environment.dataWithClusterFiles().length];
+        final NodePath[] nodePaths = new NodePath[environment.dataFiles().length];
         final Lock[] locks = new Lock[nodePaths.length];
         boolean success = false;
 
@@ -185,8 +190,17 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
             IOException lastException = null;
             int maxLocalStorageNodes = MAX_LOCAL_STORAGE_NODES_SETTING.get(settings);
             for (int possibleLockId = 0; possibleLockId < maxLocalStorageNodes; possibleLockId++) {
-                for (int dirIndex = 0; dirIndex < environment.dataWithClusterFiles().length; dirIndex++) {
-                    Path dir = environment.dataWithClusterFiles()[dirIndex].resolve(NODES_FOLDER).resolve(Integer.toString(possibleLockId));
+                for (int dirIndex = 0; dirIndex < environment.dataFiles().length; dirIndex++) {
+                    Path dataDirWithClusterName = environment.dataWithClusterFiles()[dirIndex];
+                    Path dataDir = environment.dataFiles()[dirIndex];
+                    // TODO: Remove this in 6.0, we are no longer going to read from the cluster name directory
+                    if (readFromDataPathWithClusterName(dataDirWithClusterName)) {
+                        DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
+                        deprecationLogger.deprecated("ES has detected the [path.data] folder using the cluster name as a folder [{}], " +
+                                        "Elasticsearch 6.0 will not allow the cluster name as a folder within the data path", dataDir);
+                        dataDir = dataDirWithClusterName;
+                    }
+                    Path dir = dataDir.resolve(NODES_FOLDER).resolve(Integer.toString(possibleLockId));
                     Files.createDirectories(dir);
 
                     try (Directory luceneDir = FSDirectory.open(dir, NativeFSLockFactory.INSTANCE)) {
@@ -218,7 +232,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
 
             if (locks[0] == null) {
                 throw new IllegalStateException("Failed to obtain node lock, is the following location writable?: "
-                    + Arrays.toString(environment.dataWithClusterFiles()), lastException);
+                    + Arrays.toString(environment.dataFiles()), lastException);
             }
 
             this.localNodeId = localNodeId;
@@ -242,6 +256,32 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
         }
     }
 
+    /** Returns true if the directory is empty */
+    private static boolean dirEmpty(final Path path) throws IOException {
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(path)) {
+            return stream.iterator().hasNext() == false;
+        }
+    }
+
+    // Visible for testing
+    /** Returns true if data should be read from the data path that includes the cluster name (ie, it has data in it) */
+    static boolean readFromDataPathWithClusterName(Path dataPathWithClusterName) throws IOException {
+        if (Files.exists(dataPathWithClusterName) == false ||          // If it doesn't exist
+                Files.isDirectory(dataPathWithClusterName) == false || // Or isn't a directory
+                dirEmpty(dataPathWithClusterName)) {                   // Or if it's empty
+            // No need to read from cluster-name folder!
+            return false;
+        }
+        // The "nodes" directory inside of the cluster name
+        Path nodesPath = dataPathWithClusterName.resolve(NODES_FOLDER);
+        if (Files.isDirectory(nodesPath)) {
+            // The cluster has data in the "nodes" so we should read from the cluster-named folder for now
+            return true;
+        }
+        // Hey the nodes directory didn't exist, so we can safely use whatever directory we feel appropriate
+        return false;
+    }
+
     private static void releaseAndNullLocks(Lock[] locks) {
         for (int i = 0; i < locks.length; i++) {
             if (locks[i] != null) {
diff --git a/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
index 3c13351a125..84809c0f991 100644
--- a/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
+++ b/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
@@ -22,6 +22,7 @@ import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.PathUtils;
@@ -47,6 +48,8 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileNotExists;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.containsString;
@@ -382,10 +385,10 @@ public class NodeEnvironmentTests extends ESTestCase {
 
         assertThat("shard paths with a custom data_path should contain only regular paths",
                 env.availableShardPaths(sid),
-                equalTo(stringsToPaths(dataPaths, "elasticsearch/nodes/0/indices/" + index.getUUID() + "/0")));
+                equalTo(stringsToPaths(dataPaths, "nodes/0/indices/" + index.getUUID() + "/0")));
 
         assertThat("index paths uses the regular template",
-                env.indexPaths(index), equalTo(stringsToPaths(dataPaths, "elasticsearch/nodes/0/indices/" + index.getUUID())));
+                env.indexPaths(index), equalTo(stringsToPaths(dataPaths, "nodes/0/indices/" + index.getUUID())));
 
         env.close();
         NodeEnvironment env2 = newNodeEnvironment(dataPaths, "/tmp",
@@ -396,14 +399,57 @@ public class NodeEnvironmentTests extends ESTestCase {
 
         assertThat("shard paths with a custom data_path should contain only regular paths",
                 env2.availableShardPaths(sid),
-                equalTo(stringsToPaths(dataPaths, "elasticsearch/nodes/0/indices/" + index.getUUID() + "/0")));
+                equalTo(stringsToPaths(dataPaths, "nodes/0/indices/" + index.getUUID() + "/0")));
 
         assertThat("index paths uses the regular template",
-                env2.indexPaths(index), equalTo(stringsToPaths(dataPaths, "elasticsearch/nodes/0/indices/" + index.getUUID())));
+                env2.indexPaths(index), equalTo(stringsToPaths(dataPaths, "nodes/0/indices/" + index.getUUID())));
 
         env2.close();
     }
 
+    public void testWhetherClusterFolderShouldBeUsed() throws Exception {
+        Path tempNoCluster = createTempDir();
+        String tempDataPathString = tempNoCluster.toAbsolutePath().toString();
+
+        Path tempPath = tempNoCluster.resolve("foo"); // "foo" is the cluster name
+        String tempClusterPathString = tempPath.toAbsolutePath().toString();
+
+        assertFalse("non-existent directory should not be used", NodeEnvironment.readFromDataPathWithClusterName(tempPath));
+        Settings settings = Settings.builder()
+                .put("cluster.name", "foo")
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath().toString())
+                .put(Environment.PATH_DATA_SETTING.getKey(), tempDataPathString).build();
+        try (NodeEnvironment env = new NodeEnvironment(settings, new Environment(settings))) {
+            Path nodeDataPath = env.nodeDataPaths()[0];
+            assertThat(nodeDataPath.toString(), equalTo(tempDataPathString + "/nodes/0"));
+        }
+        IOUtils.rm(tempNoCluster);
+
+        Files.createDirectories(tempPath);
+        assertFalse("empty directory should not be read from", NodeEnvironment.readFromDataPathWithClusterName(tempPath));
+        settings = Settings.builder()
+                .put("cluster.name", "foo")
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath().toString())
+                .put(Environment.PATH_DATA_SETTING.getKey(), tempDataPathString).build();
+        try (NodeEnvironment env = new NodeEnvironment(settings, new Environment(settings))) {
+            Path nodeDataPath = env.nodeDataPaths()[0];
+            assertThat(nodeDataPath.toString(), equalTo(tempDataPathString + "/nodes/0"));
+        }
+        IOUtils.rm(tempNoCluster);
+
+        // Create a directory for the cluster name
+        Files.createDirectories(tempPath.resolve(NodeEnvironment.NODES_FOLDER));
+        assertTrue("there is data in the directory", NodeEnvironment.readFromDataPathWithClusterName(tempPath));
+        settings = Settings.builder()
+                .put("cluster.name", "foo")
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath().toString())
+                .put(Environment.PATH_DATA_SETTING.getKey(), tempDataPathString).build();
+        try (NodeEnvironment env = new NodeEnvironment(settings, new Environment(settings))) {
+            Path nodeDataPath = env.nodeDataPaths()[0];
+            assertThat(nodeDataPath.toString(), equalTo(tempClusterPathString + "/nodes/0"));
+        }
+    }
+
     /** Converts an array of Strings to an array of Paths, adding an additional child if specified */
     private Path[] stringsToPaths(String[] strings, String additional) {
         Path[] locations = new Path[strings.length];
diff --git a/docs/reference/migration/migrate_5_0/fs.asciidoc b/docs/reference/migration/migrate_5_0/fs.asciidoc
index 2d582702690..859f3092823 100644
--- a/docs/reference/migration/migrate_5_0/fs.asciidoc
+++ b/docs/reference/migration/migrate_5_0/fs.asciidoc
@@ -8,3 +8,18 @@ there is nothing to worry about since this is only address space consumption
 and the actual memory usage of Elasticsearch will stay similar to what it was
 in 2.x. See http://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html
 for more information.
+
+=== Path to data on disk
+
+In prior versions of Elasticsearch, the `path.data` directory included a folder
+for the cluster name, so that data was in a folder such as
+`$DATA_DIR/$CLUSTER_NAME/nodes/$nodeOrdinal`. In 5.0 the cluster name as a
+directory is deprecated. Data will now be stored in
+`$DATA_DIR/nodes/$nodeOrdinal` if there is no existing data. Upon startup,
+Elasticsearch will check to see if the cluster folder exists and has data, and
+will read from it if necessary. In Elasticsearch 6.0 this backwards-compatible
+behavior will be removed.
+
+If you are using a multi-cluster setup with both instances of Elasticsearch
+pointing to the same data path, you will need to add the cluster name to the
+data path so that different clusters do not overwrite data.
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java
index c213f18f333..9b113354ba1 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java
@@ -68,6 +68,7 @@ public class EvilSecurityTests extends ESTestCase {
     }
 
     /** test generated permissions for all configured paths */
+    @SuppressWarnings("deprecation") // needs to check settings for deprecated path
     public void testEnvironmentPaths() throws Exception {
         Path path = createTempDir();
         // make a fake ES home and ensure we only grant permissions to that.
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/env/NodeEnvironmentEvilTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/env/NodeEnvironmentEvilTests.java
index 6c5fd797be4..ccf5e1b105e 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/env/NodeEnvironmentEvilTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/env/NodeEnvironmentEvilTests.java
@@ -63,7 +63,7 @@ public class NodeEnvironmentEvilTests extends ESTestCase {
         assumeTrue("posix filesystem", isPosix);
         final String[] tempPaths = tmpPaths();
         Path path = PathUtils.get(randomFrom(tempPaths));
-        Path fooIndex = path.resolve("elasticsearch").resolve("nodes").resolve("0").resolve(NodeEnvironment.INDICES_FOLDER)
+        Path fooIndex = path.resolve("nodes").resolve("0").resolve(NodeEnvironment.INDICES_FOLDER)
             .resolve("foo");
         Files.createDirectories(fooIndex);
         try (PosixPermissionsResetter attr = new PosixPermissionsResetter(fooIndex)) {
@@ -83,7 +83,7 @@ public class NodeEnvironmentEvilTests extends ESTestCase {
         assumeTrue("posix filesystem", isPosix);
         final String[] tempPaths = tmpPaths();
         Path path = PathUtils.get(randomFrom(tempPaths));
-        Path fooIndex = path.resolve("elasticsearch").resolve("nodes").resolve("0").resolve(NodeEnvironment.INDICES_FOLDER)
+        Path fooIndex = path.resolve("nodes").resolve("0").resolve(NodeEnvironment.INDICES_FOLDER)
             .resolve("foo");
         Path fooShard = fooIndex.resolve("0");
         Path fooShardIndex = fooShard.resolve("index");

From f23e8c3722d0e8ba3a22b9552403f1a511c370ae Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 7 Jun 2016 12:42:19 -0400
Subject: [PATCH 0705/1311] more cleanups

---
 .../main/java/org/elasticsearch/painless/AnalyzerCaster.java    | 1 -
 .../src/main/java/org/elasticsearch/painless/Definition.java    | 2 --
 .../main/java/org/elasticsearch/painless/WriterConstants.java   | 1 -
 3 files changed, 4 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
index 6e542d6aca9..e724fa9e810 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
@@ -22,7 +22,6 @@ package org.elasticsearch.painless;
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
-import org.elasticsearch.painless.node.ANode;
 
 /**
  * Used during the analysis phase to collect legal type casts and promotions
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 55e78833288..a68c9629a2d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -32,12 +32,10 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.PrimitiveIterator;
-import java.util.Set;
 import java.util.Spliterator;
 
 /**
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index ed8ca9796af..48fb673f742 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -28,7 +28,6 @@ import org.objectweb.asm.commons.Method;
 
 import java.lang.invoke.CallSite;
 import java.lang.invoke.LambdaMetafactory;
-import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodType;
 import java.util.BitSet;

From f9b45107c46f6e7b9f0043cc195f7f3a7a0a8ff2 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Tue, 7 Jun 2016 11:24:33 -0700
Subject: [PATCH 0706/1311] Added method to allow creation of new methods
 on-the-fly.

---
 .../elasticsearch/painless/MethodWriter.java  | 28 +++++++++----------
 .../org/elasticsearch/painless/Writer.java    |  5 ++--
 .../painless/WriterConstants.java             |  2 +-
 3 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index 2663fe9c717..7b7caec320e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -23,6 +23,7 @@ import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
 import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.ClassWriter;
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.commons.GeneratorAdapter;
@@ -87,30 +88,29 @@ import static org.elasticsearch.painless.WriterConstants.UTILITY_TYPE;
  * shared by the nodes of the Painless tree.
  */
 public final class MethodWriter extends GeneratorAdapter {
+    private final ClassWriter parent;
     private final BitSet statements;
 
     private final Deque<List<org.objectweb.asm.Type>> stringConcatArgs = (INDY_STRING_CONCAT_BOOTSTRAP_HANDLE == null) ?
             null : new ArrayDeque<>();
 
-    MethodWriter(int access, Method method, org.objectweb.asm.Type[] exceptions, ClassVisitor cv, BitSet statements) {
-        super(Opcodes.ASM5, cv.visitMethod(access, method.getName(), method.getDescriptor(), null, getInternalNames(exceptions)),
+    MethodWriter(int access, Method method, ClassWriter cw, BitSet statements) {
+        super(Opcodes.ASM5, cw.visitMethod(access, method.getName(), method.getDescriptor(), null, null),
                 access, method.getName(), method.getDescriptor());
+
+        this.parent = cw;
         this.statements = statements;
     }
 
-    private static String[] getInternalNames(final org.objectweb.asm.Type[] types) {
-        if (types == null) {
-            return null;
-        }
-        String[] names = new String[types.length];
-        for (int i = 0; i < names.length; ++i) {
-            names[i] = types[i].getInternalName();
-        }
-        return names;
+    /**
+     * @return A new {@link MethodWriter} with the specified access and signature.
+     */
+    MethodWriter newMethodWriter(int access, Method method) {
+        return new MethodWriter(access, method, parent, statements);
     }
 
-    /** 
-     * Marks a new statement boundary. 
+    /**
+     * Marks a new statement boundary.
      * <p>
      * This is invoked for each statement boundary (leaf {@code S*} nodes).
      */
@@ -122,7 +122,7 @@ public final class MethodWriter extends GeneratorAdapter {
         statements.set(offset);
     }
 
-    /** 
+    /**
      * Encodes the offset into the line number table as {@code offset + 1}.
      * <p>
      * This is invoked before instructions that can hit exceptions.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
index f09bf5b87eb..4d4a778d558 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
@@ -41,8 +41,7 @@ import java.util.BitSet;
 final class Writer {
 
     static byte[] write(CompilerSettings settings, String name, String source, Variables variables, SSource root, BitSet expressions) {
-        Writer writer = new Writer(settings, name, source, variables, root, expressions);
-        return writer.getBytes();
+        return new Writer(settings, name, source, variables, root, expressions).getBytes();
     }
 
     private final CompilerSettings settings;
@@ -66,7 +65,7 @@ final class Writer {
         writeBegin();
         writeConstructor();
 
-        adapter = new MethodWriter(Opcodes.ACC_PUBLIC, EXECUTE, null, writer, expressions);
+        adapter = new MethodWriter(Opcodes.ACC_PUBLIC, EXECUTE, writer, expressions);
 
         writeExecute();
         writeEnd();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index fce40d5a6e0..646fd515717 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -143,7 +143,7 @@ public final class WriterConstants {
 
     public final static Method CHECKEQUALS = getAsmMethod(boolean.class, "checkEquals", Object.class, Object.class);
 
-    private static Method getAsmMethod(final Class<?> rtype, final String name, final Class<?>... ptypes) {
+    public static Method getAsmMethod(final Class<?> rtype, final String name, final Class<?>... ptypes) {
         return new Method(name, MethodType.methodType(rtype, ptypes).toMethodDescriptorString());
     }
 

From a405c2ba99974beb8e35d30de6d95ac820ec873f Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 6 Jun 2016 13:43:33 -0400
Subject: [PATCH 0707/1311] Switch QueryBuilders to new MatchPhraseQueryBuilder

It was doing deprecated things with MatchQueryBuilder.
---
 .../elasticsearch/index/query/QueryBuilders.java   | 11 +++++------
 .../search/query/MultiMatchQueryIT.java            | 14 ++++++--------
 2 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java b/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
index d048f73381e..96585ce810f 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryBuilders.java
@@ -27,7 +27,6 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder;
 import org.elasticsearch.index.query.MoreLikeThisQueryBuilder.Item;
 import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
-import org.elasticsearch.index.search.MatchQuery;
 import org.elasticsearch.indices.TermsLookup;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
@@ -57,7 +56,7 @@ public abstract class QueryBuilders {
      * @param text The query text (to be analyzed).
      */
     public static MatchQueryBuilder matchQuery(String name, Object text) {
-        return new MatchQueryBuilder(name, text).type(MatchQuery.Type.BOOLEAN);
+        return new MatchQueryBuilder(name, text);
     }
 
     /**
@@ -86,8 +85,8 @@ public abstract class QueryBuilders {
      * @param name The field name.
      * @param text The query text (to be analyzed).
      */
-    public static MatchQueryBuilder matchPhraseQuery(String name, Object text) {
-        return new MatchQueryBuilder(name, text).type(MatchQuery.Type.PHRASE);
+    public static MatchPhraseQueryBuilder matchPhraseQuery(String name, Object text) {
+        return new MatchPhraseQueryBuilder(name, text);
     }
 
     /**
@@ -96,8 +95,8 @@ public abstract class QueryBuilders {
      * @param name The field name.
      * @param text The query text (to be analyzed).
      */
-    public static MatchQueryBuilder matchPhrasePrefixQuery(String name, Object text) {
-        return new MatchQueryBuilder(name, text).type(MatchQuery.Type.PHRASE_PREFIX);
+    public static MatchPhrasePrefixQueryBuilder matchPhrasePrefixQuery(String name, Object text) {
+        return new MatchPhrasePrefixQueryBuilder(name, text);
     }
 
     /**
diff --git a/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java
index d90e465e0a0..33ef2b97216 100644
--- a/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java
@@ -386,7 +386,6 @@ public class MultiMatchQueryIT extends ESIntegTestCase {
 
             {
                 String minShouldMatch = randomBoolean() ? null : "" + between(0, 1);
-                Operator op = randomBoolean() ? Operator.AND : Operator.OR;
                 SearchResponse left = client().prepareSearch("test").setSize(numDocs)
                         .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_uid"))
                         .setQuery(randomizeType(multiMatchQuery("capta", "full_name", "first_name", "last_name", "category")
@@ -396,15 +395,14 @@ public class MultiMatchQueryIT extends ESIntegTestCase {
                         .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_uid"))
                         .setQuery(boolQuery().minimumShouldMatch(minShouldMatch)
                                 .should(matchPhrasePrefixQuery("full_name", "capta"))
-                                .should(matchPhrasePrefixQuery("first_name", "capta").operator(op))
-                                .should(matchPhrasePrefixQuery("last_name", "capta").operator(op))
-                                .should(matchPhrasePrefixQuery("category", "capta").operator(op))
+                                .should(matchPhrasePrefixQuery("first_name", "capta"))
+                                .should(matchPhrasePrefixQuery("last_name", "capta"))
+                                .should(matchPhrasePrefixQuery("category", "capta"))
                         ).get();
                 assertEquivalent("capta", left, right);
             }
             {
                 String minShouldMatch = randomBoolean() ? null : "" + between(0, 1);
-                Operator op = randomBoolean() ? Operator.AND : Operator.OR;
                 SearchResponse left;
                 if (randomBoolean()) {
                     left = client().prepareSearch("test").setSize(numDocs)
@@ -421,9 +419,9 @@ public class MultiMatchQueryIT extends ESIntegTestCase {
                         .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_uid"))
                         .setQuery(boolQuery().minimumShouldMatch(minShouldMatch)
                                 .should(matchPhraseQuery("full_name", "captain america"))
-                                .should(matchPhraseQuery("first_name", "captain america").operator(op))
-                                .should(matchPhraseQuery("last_name", "captain america").operator(op))
-                                .should(matchPhraseQuery("category", "captain america").operator(op))
+                                .should(matchPhraseQuery("first_name", "captain america"))
+                                .should(matchPhraseQuery("last_name", "captain america"))
+                                .should(matchPhraseQuery("category", "captain america"))
                         ).get();
                 assertEquivalent("captain america", left, right);
             }

From 052a62250ca880f4e5d2746822ac58139b36efda Mon Sep 17 00:00:00 2001
From: Lee Hinman <lee@writequit.org>
Date: Tue, 7 Jun 2016 15:06:20 -0600
Subject: [PATCH 0708/1311] Change the default of `include_global_state` from
 true to false.

Resolves #18569
---
 .../cluster/snapshots/restore/RestoreSnapshotRequest.java    | 2 +-
 docs/reference/migration/migrate_5_0/settings.asciidoc       | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
index bf4e4f148fc..1a41a776c73 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
@@ -57,7 +57,7 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq
     private String renamePattern;
     private String renameReplacement;
     private boolean waitForCompletion;
-    private boolean includeGlobalState = true;
+    private boolean includeGlobalState = false;
     private boolean partial = false;
     private boolean includeAliases = true;
     private Settings settings = EMPTY_SETTINGS;
diff --git a/docs/reference/migration/migrate_5_0/settings.asciidoc b/docs/reference/migration/migrate_5_0/settings.asciidoc
index 5702d51fc12..321bb6aa490 100644
--- a/docs/reference/migration/migrate_5_0/settings.asciidoc
+++ b/docs/reference/migration/migrate_5_0/settings.asciidoc
@@ -277,3 +277,8 @@ should be used instead.
 
 The setting `bootstrap.mlockall` has been renamed to
 `bootstrap.memory_lock`.
+
+==== Snapshot settings
+
+The default setting `include_global_state` for taking and restoring snapshots
+has been changed from `true` to `false`.

From 762bbdbd0c047321e1464bbbd88b76378c99d78f Mon Sep 17 00:00:00 2001
From: Lee Hinman <lee@writequit.org>
Date: Tue, 7 Jun 2016 15:07:37 -0600
Subject: [PATCH 0709/1311] Revert "Change the default of
 `include_global_state` from true to false."

This reverts commit 052a62250ca880f4e5d2746822ac58139b36efda.
---
 .../cluster/snapshots/restore/RestoreSnapshotRequest.java    | 2 +-
 docs/reference/migration/migrate_5_0/settings.asciidoc       | 5 -----
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
index 1a41a776c73..bf4e4f148fc 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
@@ -57,7 +57,7 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq
     private String renamePattern;
     private String renameReplacement;
     private boolean waitForCompletion;
-    private boolean includeGlobalState = false;
+    private boolean includeGlobalState = true;
     private boolean partial = false;
     private boolean includeAliases = true;
     private Settings settings = EMPTY_SETTINGS;
diff --git a/docs/reference/migration/migrate_5_0/settings.asciidoc b/docs/reference/migration/migrate_5_0/settings.asciidoc
index 321bb6aa490..5702d51fc12 100644
--- a/docs/reference/migration/migrate_5_0/settings.asciidoc
+++ b/docs/reference/migration/migrate_5_0/settings.asciidoc
@@ -277,8 +277,3 @@ should be used instead.
 
 The setting `bootstrap.mlockall` has been renamed to
 `bootstrap.memory_lock`.
-
-==== Snapshot settings
-
-The default setting `include_global_state` for taking and restoring snapshots
-has been changed from `true` to `false`.

From 76c59b7417bc197fda3ef1e8a2e1a0d8a3b387ad Mon Sep 17 00:00:00 2001
From: Areek Zillur <areek.zillur@elasticsearch.com>
Date: Tue, 7 Jun 2016 22:37:10 -0400
Subject: [PATCH 0710/1311] remove unused request param

---
 .../src/main/resources/rest-api-spec/api/indices.rollover.json  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
index f7b1fdc010a..22ae6498149 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
@@ -4,7 +4,7 @@
     "methods": ["PUT", "POST"],
     "url": {
       "path": "/{alias}/_rollover",
-      "paths": ["/{alias}/_rollover", "/{alias}/_rollover/{target_alias}"],
+      "paths": ["/{alias}/_rollover"],
       "parts": {
         "alias": {
           "type" : "string",

From 134a4e5e528baa8bc66c6bdf3497ccd49839ce99 Mon Sep 17 00:00:00 2001
From: Areek Zillur <areek.zillur@elasticsearch.com>
Date: Tue, 7 Jun 2016 22:38:47 -0400
Subject: [PATCH 0711/1311] incorporate feedback

---
 .../admin/indices/rollover/Condition.java     | 87 +++++--------------
 .../indices/rollover/MaxAgeCondition.java     | 56 ++++++++++++
 .../indices/rollover/MaxDocsCondition.java    | 54 ++++++++++++
 .../indices/rollover/RolloverRequest.java     |  4 +-
 .../indices/rollover/RolloverResponse.java    |  6 +-
 .../rollover/TransportRolloverAction.java     | 79 ++++++-----------
 .../elasticsearch/indices/IndicesModule.java  |  6 +-
 .../indices/rollover/ConditionTests.java      | 27 ++++--
 .../admin/indices/rollover/RolloverIT.java    |  2 +-
 .../TransportRolloverActionTests.java         | 66 ++++++++++++--
 10 files changed, 252 insertions(+), 135 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxAgeCondition.java
 create mode 100644 core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxDocsCondition.java

diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
index fee08433e9c..12c3ef45993 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
@@ -22,12 +22,9 @@ package org.elasticsearch.action.admin.indices.rollover;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.NamedWriteable;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ObjectParser;
 
-import java.io.IOException;
 import java.util.Set;
 
 public abstract class Condition<T> implements NamedWriteable {
@@ -35,67 +32,9 @@ public abstract class Condition<T> implements NamedWriteable {
         new ObjectParser<>("conditions", null);
     static {
         PARSER.declareString((conditions, s) ->
-            conditions.add(new MaxAge(TimeValue.parseTimeValue(s, MaxAge.NAME))), new ParseField(MaxAge.NAME));
+            conditions.add(new MaxAgeCondition(TimeValue.parseTimeValue(s, MaxAgeCondition.NAME))), new ParseField(MaxAgeCondition.NAME));
         PARSER.declareLong((conditions, value) ->
-            conditions.add(new MaxDocs(value)), new ParseField(MaxDocs.NAME));
-    }
-
-    public static class MaxAge extends Condition<TimeValue> {
-        public final static String NAME = "max_age";
-
-        public MaxAge(TimeValue value) {
-            super(NAME);
-            this.value = value;
-        }
-
-        public MaxAge(StreamInput in) throws IOException {
-            super(NAME);
-            this.value = TimeValue.timeValueMillis(in.readLong());
-        }
-
-        @Override
-        public boolean matches(TimeValue value) {
-            return this.value.getMillis() <= value.getMillis();
-        }
-
-        @Override
-        public String getWriteableName() {
-            return NAME;
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeLong(value.getMillis());
-        }
-    }
-
-    public static class MaxDocs extends Condition<Long> {
-        public final static String NAME = "max_docs";
-
-        public MaxDocs(Long value) {
-            super(NAME);
-            this.value = value;
-        }
-
-        public MaxDocs(StreamInput in) throws IOException {
-            super(NAME);
-            this.value = in.readLong();
-        }
-
-        @Override
-        public boolean matches(Long value) {
-            return this.value <= value;
-        }
-
-        @Override
-        public String getWriteableName() {
-            return NAME;
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeLong(value);
-        }
+            conditions.add(new MaxDocsCondition(value)), new ParseField(MaxDocsCondition.NAME));
     }
 
     protected T value;
@@ -105,10 +44,30 @@ public abstract class Condition<T> implements NamedWriteable {
         this.name = name;
     }
 
-    public abstract boolean matches(T value);
+    public abstract Result evaluate(final Stats stats);
 
     @Override
     public final String toString() {
         return "[" + name + ": " + value + "]";
     }
+
+    public static class Stats {
+        public final long numDocs;
+        public final long indexCreated;
+
+        public Stats(long numDocs, long indexCreated) {
+            this.numDocs = numDocs;
+            this.indexCreated = indexCreated;
+        }
+    }
+
+    public static class Result {
+        public final Condition condition;
+        public final boolean matched;
+
+        protected Result(Condition condition, boolean matched) {
+            this.condition = condition;
+            this.matched = matched;
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxAgeCondition.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxAgeCondition.java
new file mode 100644
index 00000000000..0a7e85bb129
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxAgeCondition.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.rollover;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.TimeValue;
+
+import java.io.IOException;
+
+public class MaxAgeCondition extends Condition<TimeValue> {
+    public final static String NAME = "max_age";
+
+    public MaxAgeCondition(TimeValue value) {
+        super(NAME);
+        this.value = value;
+    }
+
+    public MaxAgeCondition(StreamInput in) throws IOException {
+        super(NAME);
+        this.value = TimeValue.timeValueMillis(in.readLong());
+    }
+
+    @Override
+    public Result evaluate(final Stats stats) {
+        long indexAge = System.currentTimeMillis() - stats.indexCreated;
+        return new Result(this, this.value.getMillis() <= indexAge);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeLong(value.getMillis());
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxDocsCondition.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxDocsCondition.java
new file mode 100644
index 00000000000..e1c599c23d4
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxDocsCondition.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.rollover;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+public class MaxDocsCondition extends Condition<Long> {
+    public final static String NAME = "max_docs";
+
+    public MaxDocsCondition(Long value) {
+        super(NAME);
+        this.value = value;
+    }
+
+    public MaxDocsCondition(StreamInput in) throws IOException {
+        super(NAME);
+        this.value = in.readLong();
+    }
+
+    @Override
+    public Result evaluate(final Stats stats) {
+        return new Result(this, this.value <= stats.numDocs);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeLong(value);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
index 0d55d256390..dd1facdb6e2 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
@@ -115,11 +115,11 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
     }
 
     public void addMaxIndexAgeCondition(TimeValue age) {
-        this.conditions.add(new Condition.MaxAge(age));
+        this.conditions.add(new MaxAgeCondition(age));
     }
 
     public void addMaxIndexDocsCondition(long docs) {
-        this.conditions.add(new Condition.MaxDocs(docs));
+        this.conditions.add(new MaxDocsCondition(docs));
     }
 
     public boolean isSimulate() {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
index a2abe044930..fbca8af99cc 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
@@ -44,14 +44,16 @@ public final class RolloverResponse extends ActionResponse implements ToXContent
     RolloverResponse() {
     }
 
-    RolloverResponse(String oldIndex, String newIndex, Set<Map.Entry<String, Boolean>> conditionStatus,
+    RolloverResponse(String oldIndex, String newIndex, Set<Condition.Result> conditionResults,
                      boolean simulate, boolean rolledOver, boolean rolloverIndexCreated) {
         this.oldIndex = oldIndex;
         this.newIndex = newIndex;
         this.simulate = simulate;
         this.rolledOver = rolledOver;
         this.rolloverIndexCreated = rolloverIndexCreated;
-        this.conditionStatus = conditionStatus;
+        this.conditionStatus = conditionResults.stream()
+            .map(result -> new AbstractMap.SimpleEntry<>(result.condition.toString(), result.matched))
+            .collect(Collectors.toSet());
     }
 
     public String getOldIndex() {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
index 05bc674b8d3..3eed74661a8 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -41,16 +41,12 @@ import org.elasticsearch.cluster.metadata.MetaDataIndexAliasesService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.shard.DocsStats;
-import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-import java.util.AbstractMap;
-import java.util.HashSet;
-import java.util.Map;
 import java.util.Set;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 /**
@@ -104,27 +100,20 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
         client.admin().indices().prepareStats(sourceIndexName).clear().setDocs(true).execute(
             new ActionListener<IndicesStatsResponse>() {
                 @Override
-                public void onResponse(IndicesStatsResponse indicesStatsResponse) {
-                    final IndexMetaData sourceIndex = metaData.index(sourceIndexName);
-                    DocsStats docsStats = indicesStatsResponse.getTotal().getDocs();
-                    long docCount = docsStats == null ? 0 : docsStats.getCount();
-                    long indexAge = System.currentTimeMillis() - sourceIndex.getCreationDate();
-                    final Set<Map.Entry<Condition, Boolean>> evaluatedConditions =
-                        evaluateConditions(rolloverRequest.getConditions(), docCount, indexAge);
-                    final Set<Map.Entry<String, Boolean>> conditionStatus = evaluatedConditions.stream()
-                        .map(entry -> new AbstractMap.SimpleEntry<>(entry.getKey().toString(), entry.getValue()))
-                        .collect(Collectors.toSet());
+                public void onResponse(IndicesStatsResponse statsResponse) {
+                    final Set<Condition.Result> conditionResults = evaluateConditions(rolloverRequest.getConditions(),
+                        statsResponse.getTotal().getDocs(), metaData.index(sourceIndexName));
                     final String rolloverIndexName = generateRolloverIndexName(sourceIndexName);
-                    final boolean createRolloverIndex = metaData.index(rolloverIndexName) == null;
                     if (rolloverRequest.isSimulate()) {
                         listener.onResponse(
-                            new RolloverResponse(sourceIndexName, rolloverIndexName, conditionStatus, true, false,
+                            new RolloverResponse(sourceIndexName, rolloverIndexName, conditionResults, true, false,
                                 false));
                         return;
                     }
-                    if (conditionStatus.stream().allMatch(Map.Entry::getValue)) {
+                    if (conditionResults.stream().anyMatch(result -> result.matched)) {
+                        boolean createRolloverIndex = metaData.index(rolloverIndexName) == null;
                         final RolloverResponse rolloverResponse =
-                            new RolloverResponse(sourceIndexName, rolloverIndexName, conditionStatus, false, true,
+                            new RolloverResponse(sourceIndexName, rolloverIndexName, conditionResults, false, true,
                                 createRolloverIndex);
                         if (createRolloverIndex) {
                             CreateIndexClusterStateUpdateRequest updateRequest =
@@ -139,11 +128,6 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
 
                                 @Override
                                 public void onFailure(Throwable t) {
-                                    if (t instanceof IndexAlreadyExistsException) {
-                                        logger.trace("[{}] failed to create rollover index", t, updateRequest.index());
-                                    } else {
-                                        logger.debug("[{}] failed to create rollover index", t, updateRequest.index());
-                                    }
                                     listener.onFailure(t);
                                 }
                             });
@@ -155,7 +139,7 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
                     } else {
                         // conditions not met
                         listener.onResponse(
-                            new RolloverResponse(sourceIndexName, sourceIndexName, conditionStatus, false, false,
+                            new RolloverResponse(sourceIndexName, sourceIndexName, conditionResults, false, false,
                                 false)
                         );
                     }
@@ -167,9 +151,10 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
                 }
             }
         );
-
     }
 
+
+
     private static final class IndicesAliasesListener implements ActionListener<ClusterStateUpdateResponse> {
 
         private final ActionListener<RolloverResponse> listener;
@@ -203,36 +188,26 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
         return updateRequest;
     }
 
+    static final Pattern INDEX_NAME_PATTERN = Pattern.compile("^.*-(\\d)+$");
+
     static String generateRolloverIndexName(String sourceIndexName) {
-        int numberIndex = sourceIndexName.lastIndexOf("-");
-        int counter = 1;
-        String indexPrefix = sourceIndexName;
-        if (numberIndex != -1) {
-            try {
-                counter = Integer.parseInt(sourceIndexName.substring(numberIndex + 1));
-                counter++;
-                indexPrefix = sourceIndexName.substring(0, numberIndex);
-            } catch (NumberFormatException ignored) {
-            }
+        if (INDEX_NAME_PATTERN.matcher(sourceIndexName).matches()) {
+            int numberIndex = sourceIndexName.lastIndexOf("-");
+            assert numberIndex != -1 : "no separator '-' found";
+            int counter = Integer.parseInt(sourceIndexName.substring(numberIndex + 1));
+            return String.join("-", sourceIndexName.substring(0, numberIndex), String.valueOf(++counter));
+        } else {
+            throw new IllegalArgumentException("index name [" + sourceIndexName + "] does not match pattern '^.*-(\\d)+$'");
         }
-        return String.join("-", indexPrefix, String.valueOf(counter));
     }
 
-    static Set<Map.Entry<Condition, Boolean>> evaluateConditions(Set<Condition> conditions, long docCount, long indexAge) {
-        Set<Map.Entry<Condition, Boolean>> result = new HashSet<>(conditions.size());
-        for (Condition condition: conditions) {
-            if (condition instanceof Condition.MaxAge) {
-                Condition.MaxAge maxAge = (Condition.MaxAge) condition;
-                final TimeValue age = TimeValue.timeValueMillis(indexAge);
-                result.add(new AbstractMap.SimpleEntry<>(condition, maxAge.matches(age)));
-            } else if (condition instanceof Condition.MaxDocs) {
-                final Condition.MaxDocs maxDocs = (Condition.MaxDocs) condition;
-                result.add(new AbstractMap.SimpleEntry<>(condition, maxDocs.matches(docCount)));
-            } else {
-                throw new IllegalArgumentException("unknown condition [" + condition.getClass().getSimpleName() + "]");
-            }
-        }
-        return result;
+    static Set<Condition.Result> evaluateConditions(final Set<Condition> conditions,
+                                                    final DocsStats docsStats, final IndexMetaData metaData) {
+        final long numDocs = docsStats == null ? 0 : docsStats.getCount();
+        final Condition.Stats stats = new Condition.Stats(numDocs, metaData.getCreationDate());
+        return conditions.stream()
+            .map(condition -> condition.evaluate(stats))
+            .collect(Collectors.toSet());
     }
 
     static void validate(MetaData metaData, RolloverRequest request) {
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
index 885e2d78eca..1817a71073c 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
@@ -20,6 +20,8 @@
 package org.elasticsearch.indices;
 
 import org.elasticsearch.action.admin.indices.rollover.Condition;
+import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition;
+import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition;
 import org.elasticsearch.action.update.UpdateHelper;
 import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
 import org.elasticsearch.common.geo.ShapesAvailability;
@@ -86,8 +88,8 @@ public class IndicesModule extends AbstractModule {
     }
 
     private void registerBuildInWritables() {
-        namedWritableRegistry.register(Condition.class, Condition.MaxAge.NAME, Condition.MaxAge::new);
-        namedWritableRegistry.register(Condition.class, Condition.MaxDocs.NAME, Condition.MaxDocs::new);
+        namedWritableRegistry.register(Condition.class, MaxAgeCondition.NAME, MaxAgeCondition::new);
+        namedWritableRegistry.register(Condition.class, MaxDocsCondition.NAME, MaxDocsCondition::new);
     }
 
     private void registerBuiltInMappers() {
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java
index 3ce1b84d82c..95f186ba0e5 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/ConditionTests.java
@@ -27,15 +27,30 @@ import static org.hamcrest.Matchers.equalTo;
 public class ConditionTests extends ESTestCase {
 
     public void testMaxAge() throws Exception {
-        final Condition.MaxAge maxAge = new Condition.MaxAge(TimeValue.timeValueMillis(10));
-        assertThat(maxAge.matches(TimeValue.timeValueMillis(randomIntBetween(0, 9))), equalTo(false));
-        assertThat(maxAge.matches(TimeValue.timeValueMillis(randomIntBetween(10, 100))), equalTo(true));
+        final MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(1));
+
+        long indexCreatedMatch = System.currentTimeMillis() - TimeValue.timeValueMinutes(61).getMillis();
+        Condition.Result evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedMatch));
+        assertThat(evaluate.condition, equalTo(maxAgeCondition));
+        assertThat(evaluate.matched, equalTo(true));
+
+        long indexCreatedNotMatch = System.currentTimeMillis() - TimeValue.timeValueMinutes(59).getMillis();
+        evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedNotMatch));
+        assertThat(evaluate.condition, equalTo(maxAgeCondition));
+        assertThat(evaluate.matched, equalTo(false));
     }
 
     public void testMaxDocs() throws Exception {
-        final Condition.MaxDocs maxDocs = new Condition.MaxDocs(10L);
-        assertThat(maxDocs.matches((long) randomIntBetween(0, 9)), equalTo(false));
-        assertThat(maxDocs.matches((long) randomIntBetween(10, 100)), equalTo(true));
+        final MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L);
 
+        long maxDocsMatch = randomIntBetween(100, 1000);
+        Condition.Result evaluate = maxDocsCondition.evaluate(new Condition.Stats(maxDocsMatch, 0));
+        assertThat(evaluate.condition, equalTo(maxDocsCondition));
+        assertThat(evaluate.matched, equalTo(true));
+
+        long maxDocsNotMatch = randomIntBetween(0, 99);
+        evaluate = maxDocsCondition.evaluate(new Condition.Stats(0, maxDocsNotMatch));
+        assertThat(evaluate.condition, equalTo(maxDocsCondition));
+        assertThat(evaluate.matched, equalTo(false));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
index 9c0f3b7b96a..71d6f5a44f7 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
@@ -98,7 +98,7 @@ public class RolloverIT extends ESIntegTestCase {
         assertThat(response.isRolloverIndexCreated(), equalTo(false));
         assertThat(response.getConditionStatus().size(), equalTo(1));
         final Map.Entry<String, Boolean> conditionEntry = response.getConditionStatus().iterator().next();
-        assertThat(conditionEntry.getKey(), equalTo(new Condition.MaxAge(TimeValue.timeValueHours(4)).toString()));
+        assertThat(conditionEntry.getKey(), equalTo(new MaxAgeCondition(TimeValue.timeValueHours(4)).toString()));
         assertThat(conditionEntry.getValue(), equalTo(false));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
         final IndexMetaData oldIndex = state.metaData().index("test_index");
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
index 636378a5872..12e1e90242e 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
@@ -27,13 +27,69 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import static org.elasticsearch.action.admin.indices.rollover.TransportRolloverAction.evaluateConditions;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 
 public class TransportRolloverActionTests extends ESTestCase {
 
+    public void testEvaluateConditions() throws Exception {
+        MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L);
+        MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(2));
+        long matchMaxDocs = randomIntBetween(100, 1000);
+        long notMatchMaxDocs = randomIntBetween(0, 99);
+        final Settings settings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .build();
+        final IndexMetaData metaData = IndexMetaData.builder(randomAsciiOfLength(10))
+            .creationDate(System.currentTimeMillis() - TimeValue.timeValueHours(3).getMillis())
+            .settings(settings)
+            .build();
+        final HashSet<Condition> conditions = Sets.newHashSet(maxDocsCondition, maxAgeCondition);
+        Set<Condition.Result> results = evaluateConditions(conditions, new DocsStats(matchMaxDocs, 0L), metaData);
+        assertThat(results.size(), equalTo(2));
+        for (Condition.Result result : results) {
+            assertThat(result.matched, equalTo(true));
+        }
+        results = evaluateConditions(conditions, new DocsStats(notMatchMaxDocs, 0), metaData);
+        assertThat(results.size(), equalTo(2));
+        for (Condition.Result result : results) {
+            if (result.condition instanceof MaxAgeCondition) {
+                assertThat(result.matched, equalTo(true));
+            } else if (result.condition instanceof MaxDocsCondition) {
+                assertThat(result.matched, equalTo(false));
+            } else {
+                fail("unknown condition result found " + result.condition);
+            }
+        }
+        results = evaluateConditions(conditions, null, metaData);
+        assertThat(results.size(), equalTo(2));
+        for (Condition.Result result : results) {
+            if (result.condition instanceof MaxAgeCondition) {
+                assertThat(result.matched, equalTo(true));
+            } else if (result.condition instanceof MaxDocsCondition) {
+                assertThat(result.matched, equalTo(false));
+            } else {
+                fail("unknown condition result found " + result.condition);
+            }
+        }
+    }
+
+
     public void testCreateUpdateAliasRequest() throws Exception {
         String sourceAlias = randomAsciiOfLength(10);
         String sourceIndex = randomAsciiOfLength(10);
@@ -94,17 +150,15 @@ public class TransportRolloverActionTests extends ESTestCase {
     }
 
     public void testGenerateRolloverIndexName() throws Exception {
-        String indexNotEndingInNumbers = randomAsciiOfLength(10) + "A";
-        assertThat(TransportRolloverAction.generateRolloverIndexName(indexNotEndingInNumbers),
-            not(equalTo(indexNotEndingInNumbers)));
-        assertThat(TransportRolloverAction.generateRolloverIndexName(indexNotEndingInNumbers),
-            equalTo(indexNotEndingInNumbers + "-1"));
+        String invalidIndexName = randomAsciiOfLength(10) + "A";
+        expectThrows(IllegalArgumentException.class, () ->
+            TransportRolloverAction.generateRolloverIndexName(invalidIndexName));
         int num = randomIntBetween(0, 100);
         final String indexPrefix = randomAsciiOfLength(10);
         String indexEndingInNumbers = indexPrefix + "-" + num;
         assertThat(TransportRolloverAction.generateRolloverIndexName(indexEndingInNumbers),
             equalTo(indexPrefix + "-" + (num + 1)));
         assertThat(TransportRolloverAction.generateRolloverIndexName("index-name-1"), equalTo("index-name-2"));
-        assertThat(TransportRolloverAction.generateRolloverIndexName("index-name"), equalTo("index-name-1"));
+        assertThat(TransportRolloverAction.generateRolloverIndexName("index-name-2"), equalTo("index-name-3"));
     }
 }

From 791db1fb48e339da05e20d4d8da30323af0ecf1b Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Wed, 8 Jun 2016 11:43:02 +0200
Subject: [PATCH 0712/1311] remove TODO around using /_cat/nodes rather than
 /_nodes, test compatibility with 2.x

 _cat/nodes returns the http address only in 5.x. Would be nice to use as we could drop the jackson dependency, but we care more about being compatible with 2.x.
 Not compatible with previous versions as the format of returned http addresses have changed since 2.0.

Also fixed test bug that caused sporadic failures.
---
 .../client/sniff/HostsSniffer.java            |  4 +--
 .../client/sniff/HostsSnifferTests.java       | 31 ++++++++++---------
 2 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
index 4af9d205589..cc3abdfec7e 100644
--- a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
+++ b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
@@ -38,9 +38,9 @@ import java.util.List;
 import java.util.Map;
 
 /**
- * Class responsible for sniffing the http hosts from elasticsearch through the nodes info api and returning them back
+ * Class responsible for sniffing the http hosts from elasticsearch through the nodes info api and returning them back.
+ * Compatible with elasticsearch 5.x and 2.x.
  */
-//TODO This could potentially be using _cat/nodes which wouldn't require jackson as a dependency, but we'd have bw comp problems with 2.x
 public class HostsSniffer {
 
     private static final Log logger = LogFactory.getLog(HostsSniffer.class);
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
index 88f0480f92a..57705c1b832 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
@@ -97,8 +97,7 @@ public class HostsSnifferTests extends LuceneTestCase {
             } catch(ElasticsearchResponseException e) {
                 ElasticsearchResponse response = e.getElasticsearchResponse();
                 if (sniffResponse.isFailure) {
-                    assertThat(e.getMessage(), containsString("GET http://localhost:" + httpServer.getAddress().getPort() +
-                            "/_nodes/http?timeout=" + sniffRequestTimeout));
+                    assertThat(e.getMessage(), containsString("GET " + httpHost + "/_nodes/http?timeout=" + sniffRequestTimeout + "ms"));
                     assertThat(e.getMessage(), containsString(Integer.toString(sniffResponse.nodesInfoResponseCode)));
                     assertThat(response.getHost(), equalTo(httpHost));
                     assertThat(response.getStatusLine().getStatusCode(), equalTo(sniffResponse.nodesInfoResponseCode));
@@ -185,20 +184,22 @@ public class HostsSnifferTests extends LuceneTestCase {
                 }
                 generator.writeEndObject();
             }
-            String[] roles = {"master", "data", "ingest"};
-            int numRoles = RandomInts.randomIntBetween(random(), 0, 3);
-            Set<String> nodeRoles = new HashSet<>(numRoles);
-            for (int j = 0; j < numRoles; j++) {
-                String role;
-                do {
-                    role = RandomPicks.randomFrom(random(), roles);
-                } while(nodeRoles.add(role) == false);
+            if (random().nextBoolean()) {
+                String[] roles = {"master", "data", "ingest"};
+                int numRoles = RandomInts.randomIntBetween(random(), 0, 3);
+                Set<String> nodeRoles = new HashSet<>(numRoles);
+                for (int j = 0; j < numRoles; j++) {
+                    String role;
+                    do {
+                        role = RandomPicks.randomFrom(random(), roles);
+                    } while(nodeRoles.add(role) == false);
+                }
+                generator.writeArrayFieldStart("roles");
+                for (String nodeRole : nodeRoles) {
+                    generator.writeString(nodeRole);
+                }
+                generator.writeEndArray();
             }
-            generator.writeArrayFieldStart("roles");
-            for (String nodeRole : nodeRoles) {
-                generator.writeString(nodeRole);
-            }
-            generator.writeEndArray();
             int numAttributes = RandomInts.randomIntBetween(random(), 0, 3);
             Map<String, String> attributes = new HashMap<>(numAttributes);
             for (int j = 0; j < numAttributes; j++) {

From 712c77264d2a830ed00e69b50c448bb0977b0b9d Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Wed, 8 Jun 2016 11:51:46 +0200
Subject: [PATCH 0713/1311] Fix ut: make sure that the number of slices is
 bigger than 1 in the SliceBuilder tests.

---
 .../org/elasticsearch/search/slice/SliceBuilderTests.java    | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
index 554b9436e58..217f97ace0a 100644
--- a/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
@@ -90,7 +90,6 @@ public class SliceBuilderTests extends ESTestCase {
 
     private final SliceBuilder randomSliceBuilder() throws IOException {
         int max = randomIntBetween(2, MAX_SLICE);
-        if (max == 0) max++;
         int id = randomInt(max - 1);
         String field = randomAsciiOfLengthBetween(5, 20);
         return new SliceBuilder(field, id, max);
@@ -279,8 +278,8 @@ public class SliceBuilderTests extends ESTestCase {
             assertThat(total, equalTo(numSlices));
 
             // numShards > numSlices
-            numShards = randomIntBetween(3, 100);
-            numSlices = randomInt(numShards-1);
+            numShards = randomIntBetween(4, 100);
+            numSlices = randomIntBetween(2, numShards-1);
             List<Integer> targetShards = new ArrayList<>();
             for (int i = 0; i < numSlices; i++) {
                 for (int j = 0; j < numShards; j++) {

From a78fe1305dd8148db1c4f9a93f0284e21443042f Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Wed, 8 Jun 2016 12:19:52 +0200
Subject: [PATCH 0714/1311] require RestClient when creating Sniffer.Builder

---
 .../elasticsearch/client/sniff/Sniffer.java   | 30 ++----
 .../client/sniff/SnifferBuilderTests.java     | 93 +++++++++----------
 2 files changed, 55 insertions(+), 68 deletions(-)

diff --git a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
index 2cd146bd80d..7ae6b0502e5 100644
--- a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
+++ b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
@@ -22,8 +22,6 @@ package org.elasticsearch.client.sniff;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.http.HttpHost;
-import org.apache.http.config.Registry;
-import org.apache.http.impl.client.CloseableHttpClient;
 import org.elasticsearch.client.RestClient;
 
 import java.io.Closeable;
@@ -145,8 +143,8 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
     /**
      * Returns a new {@link Builder} to help with {@link Sniffer} creation.
      */
-    public static Builder builder() {
-        return new Builder();
+    public static Builder builder(RestClient restClient) {
+        return new Builder(restClient);
     }
 
     /**
@@ -157,15 +155,19 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
         public static final long DEFAULT_SNIFF_AFTER_FAILURE_DELAY = TimeUnit.MINUTES.toMillis(1);
         public static final long DEFAULT_SNIFF_REQUEST_TIMEOUT = TimeUnit.SECONDS.toMillis(1);
 
+        private final RestClient restClient;
         private long sniffRequestTimeout = DEFAULT_SNIFF_REQUEST_TIMEOUT;
         private long sniffInterval = DEFAULT_SNIFF_INTERVAL;
         private boolean sniffOnFailure = true;
         private long sniffAfterFailureDelay = DEFAULT_SNIFF_AFTER_FAILURE_DELAY;
         private String scheme = "http";
-        private RestClient restClient;
-
-        private Builder() {
 
+        /**
+         * Creates a new builder instance and sets the {@link RestClient} that will be used to communicate with elasticsearch.
+         */
+        private Builder(RestClient restClient) {
+            Objects.requireNonNull(restClient, "restClient cannot be null");
+            this.restClient = restClient;
         }
 
         /**
@@ -201,24 +203,13 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
             return this;
         }
 
-        /**
-         * Sets the http client. Mandatory argument. Best practice is to use the same client used
-         * within {@link org.elasticsearch.client.RestClient} which can be created manually or
-         * through {@link RestClient.Builder#createDefaultHttpClient(Registry)}.
-         * @see CloseableHttpClient
-         */
-        public Builder setRestClient(RestClient restClient) {
-            this.restClient = restClient;
-            return this;
-        }
-
         /**
          * Sets the sniff request timeout to be passed in as a query string parameter to elasticsearch.
          * Allows to halt the request without any failure, as only the nodes that have responded
          * within this timeout will be returned.
          */
         public Builder setSniffRequestTimeout(int sniffRequestTimeout) {
-            if (sniffRequestTimeout <=0) {
+            if (sniffRequestTimeout <= 0) {
                 throw new IllegalArgumentException("sniffRequestTimeout must be greater than 0");
             }
             this.sniffRequestTimeout = sniffRequestTimeout;
@@ -242,7 +233,6 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
          * Creates the {@link Sniffer} based on the provided configuration.
          */
         public Sniffer build() {
-            Objects.requireNonNull(restClient, "restClient cannot be null");
             return new Sniffer(restClient, sniffRequestTimeout, scheme, sniffInterval, sniffOnFailure, sniffAfterFailureDelay);
         }
     }
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
index 868c5788120..9bcd567ed2a 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
@@ -35,63 +35,60 @@ public class SnifferBuilderTests extends LuceneTestCase {
     }
 
     public void testBuild() throws Exception {
-
-        try {
-            Sniffer.builder().setScheme(null);
-            fail("should have failed");
-        } catch(NullPointerException e) {
-            assertEquals(e.getMessage(), "scheme cannot be null");
-        }
-
-        try {
-            Sniffer.builder().setScheme("whatever");
-            fail("should have failed");
-        } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "scheme must be either http or https");
-        }
-
-        try {
-            Sniffer.builder().setSniffInterval(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
-            fail("should have failed");
-        } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "sniffInterval must be greater than 0");
-        }
-
-        try {
-            Sniffer.builder().setSniffRequestTimeout(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
-            fail("should have failed");
-        } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "sniffRequestTimeout must be greater than 0");
-        }
-
-        try {
-            Sniffer.builder().setSniffAfterFailureDelay(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
-            fail("should have failed");
-        } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "sniffAfterFailureDelay must be greater than 0");
-        }
-
-        try {
-            Sniffer.builder().build();
-            fail("should have failed");
-        } catch(NullPointerException e) {
-            assertEquals(e.getMessage(), "restClient cannot be null");
-        }
-
         int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
         HttpHost[] hosts = new HttpHost[numNodes];
         for (int i = 0; i < numNodes; i++) {
             hosts[i] = new HttpHost("localhost", 9200 + i);
         }
-
         try (RestClient client = RestClient.builder().setHosts(hosts).build()) {
-            try (Sniffer sniffer = Sniffer.builder().setRestClient(client).build()) {
+
+            try {
+                Sniffer.builder(client).setScheme(null);
+                fail("should have failed");
+            } catch(NullPointerException e) {
+                assertEquals(e.getMessage(), "scheme cannot be null");
+            }
+
+            try {
+                Sniffer.builder(client).setScheme("whatever");
+                fail("should have failed");
+            } catch(IllegalArgumentException e) {
+                assertEquals(e.getMessage(), "scheme must be either http or https");
+            }
+
+            try {
+                Sniffer.builder(client).setSniffInterval(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+                fail("should have failed");
+            } catch(IllegalArgumentException e) {
+                assertEquals(e.getMessage(), "sniffInterval must be greater than 0");
+            }
+
+            try {
+                Sniffer.builder(client).setSniffRequestTimeout(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+                fail("should have failed");
+            } catch(IllegalArgumentException e) {
+                assertEquals(e.getMessage(), "sniffRequestTimeout must be greater than 0");
+            }
+
+            try {
+                Sniffer.builder(client).setSniffAfterFailureDelay(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+                fail("should have failed");
+            } catch(IllegalArgumentException e) {
+                assertEquals(e.getMessage(), "sniffAfterFailureDelay must be greater than 0");
+            }
+
+            try {
+                Sniffer.builder(null).build();
+                fail("should have failed");
+            } catch(NullPointerException e) {
+                assertEquals(e.getMessage(), "restClient cannot be null");
+            }
+
+            try (Sniffer sniffer = Sniffer.builder(client).build()) {
                 assertNotNull(sniffer);
             }
-        }
 
-        try (RestClient client = RestClient.builder().setHosts(hosts).build()) {
-            Sniffer.Builder builder = Sniffer.builder().setRestClient(client);
+            Sniffer.Builder builder = Sniffer.builder(client);
             if (random().nextBoolean()) {
                 builder.setScheme(RandomPicks.randomFrom(random(), Arrays.asList("http", "https")));
             }

From 04d620da742ef57d4bba08af54934b2f2d305de0 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Wed, 8 Jun 2016 12:29:16 +0200
Subject: [PATCH 0715/1311] require hosts when creating RestClient.Builder

Also fix order of arguments when using assertEquals
---
 .../client/sniff/HostsSnifferTests.java       |  2 +-
 .../client/sniff/SnifferBuilderTests.java     | 14 +++---
 .../org/elasticsearch/client/RestClient.java  | 30 +++++-------
 .../client/RestClientBuilderTests.java        | 46 ++++++++-----------
 .../client/RestClientIntegTests.java          |  4 +-
 .../client/RestClientMultipleHostsTests.java  |  2 +-
 .../client/RestClientSingleHostTests.java     |  2 +-
 .../elasticsearch/test/ESIntegTestCase.java   |  2 +-
 .../test/rest/client/RestTestClient.java      |  2 +-
 9 files changed, 44 insertions(+), 60 deletions(-)

diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
index 57705c1b832..826d7fb35f8 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
@@ -82,7 +82,7 @@ public class HostsSnifferTests extends LuceneTestCase {
 
     public void testSniffNodes() throws IOException, URISyntaxException {
         HttpHost httpHost = new HttpHost(httpServer.getAddress().getHostName(), httpServer.getAddress().getPort());
-        try (RestClient restClient = RestClient.builder().setHosts(httpHost).build()) {
+        try (RestClient restClient = RestClient.builder(httpHost).build()) {
             HostsSniffer sniffer = new HostsSniffer(restClient, sniffRequestTimeout, scheme);
             try {
                 List<HttpHost> sniffedHosts = sniffer.sniffHosts();
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
index 9bcd567ed2a..c0bb9def964 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
@@ -40,48 +40,48 @@ public class SnifferBuilderTests extends LuceneTestCase {
         for (int i = 0; i < numNodes; i++) {
             hosts[i] = new HttpHost("localhost", 9200 + i);
         }
-        try (RestClient client = RestClient.builder().setHosts(hosts).build()) {
+        try (RestClient client = RestClient.builder(hosts).build()) {
 
             try {
                 Sniffer.builder(client).setScheme(null);
                 fail("should have failed");
             } catch(NullPointerException e) {
-                assertEquals(e.getMessage(), "scheme cannot be null");
+                assertEquals("scheme cannot be null", e.getMessage());
             }
 
             try {
                 Sniffer.builder(client).setScheme("whatever");
                 fail("should have failed");
             } catch(IllegalArgumentException e) {
-                assertEquals(e.getMessage(), "scheme must be either http or https");
+                assertEquals("scheme must be either http or https", e.getMessage());
             }
 
             try {
                 Sniffer.builder(client).setSniffInterval(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
                 fail("should have failed");
             } catch(IllegalArgumentException e) {
-                assertEquals(e.getMessage(), "sniffInterval must be greater than 0");
+                assertEquals("sniffInterval must be greater than 0", e.getMessage());
             }
 
             try {
                 Sniffer.builder(client).setSniffRequestTimeout(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
                 fail("should have failed");
             } catch(IllegalArgumentException e) {
-                assertEquals(e.getMessage(), "sniffRequestTimeout must be greater than 0");
+                assertEquals("sniffRequestTimeout must be greater than 0", e.getMessage());
             }
 
             try {
                 Sniffer.builder(client).setSniffAfterFailureDelay(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
                 fail("should have failed");
             } catch(IllegalArgumentException e) {
-                assertEquals(e.getMessage(), "sniffAfterFailureDelay must be greater than 0");
+                assertEquals("sniffAfterFailureDelay must be greater than 0", e.getMessage());
             }
 
             try {
                 Sniffer.builder(null).build();
                 fail("should have failed");
             } catch(NullPointerException e) {
-                assertEquals(e.getMessage(), "restClient cannot be null");
+                assertEquals("restClient cannot be null", e.getMessage());
             }
 
             try (Sniffer sniffer = Sniffer.builder(client).build()) {
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 161abbd40a7..a01ad694b96 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -350,8 +350,8 @@ public final class RestClient implements Closeable {
     /**
      * Returns a new {@link Builder} to help with {@link RestClient} creation.
      */
-    public static Builder builder() {
-        return new Builder();
+    public static Builder builder(HttpHost... hosts) {
+        return new Builder(hosts);
     }
 
     /**
@@ -365,13 +365,19 @@ public final class RestClient implements Closeable {
 
         private static final Header[] EMPTY_HEADERS = new Header[0];
 
+        private final HttpHost[] hosts;
         private CloseableHttpClient httpClient;
         private int maxRetryTimeout = DEFAULT_MAX_RETRY_TIMEOUT;
-        private HttpHost[] hosts;
         private Header[] defaultHeaders = EMPTY_HEADERS;
 
-        private Builder() {
-
+        /**
+         * Creates a new builder instance and sets the hosts that the client will send requests to.
+         */
+        private Builder(HttpHost... hosts) {
+            if (hosts == null || hosts.length == 0) {
+                throw new IllegalArgumentException("no hosts provided");
+            }
+            this.hosts = hosts;
         }
 
         /**
@@ -399,17 +405,6 @@ public final class RestClient implements Closeable {
             return this;
         }
 
-        /**
-         * Sets the hosts that the client will send requests to.
-         */
-        public Builder setHosts(HttpHost... hosts) {
-            if (hosts == null || hosts.length == 0) {
-                throw new IllegalArgumentException("no hosts provided");
-            }
-            this.hosts = hosts;
-            return this;
-        }
-
         /**
          * Sets the default request headers, to be used when creating the default http client instance.
          * In case the http client is set through {@link #setHttpClient(CloseableHttpClient)}, the default headers need to be
@@ -431,9 +426,6 @@ public final class RestClient implements Closeable {
             if (httpClient == null) {
                 httpClient = createDefaultHttpClient(null);
             }
-            if (hosts == null || hosts.length == 0) {
-                throw new IllegalArgumentException("no hosts provided");
-            }
             return new RestClient(httpClient, maxRetryTimeout, defaultHeaders, hosts);
         }
 
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java b/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
index 3ea0fe14ed2..75587ed8db1 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
@@ -32,61 +32,53 @@ public class RestClientBuilderTests extends LuceneTestCase {
 
     public void testBuild() throws IOException {
         try {
-            RestClient.builder().setMaxRetryTimeout(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+            RestClient.builder((HttpHost[])null);
             fail("should have failed");
         } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "maxRetryTimeout must be greater than 0");
+            assertEquals("no hosts provided", e.getMessage());
         }
 
         try {
-            RestClient.builder().setHosts((HttpHost[])null);
+            RestClient.builder();
             fail("should have failed");
         } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "no hosts provided");
+            assertEquals("no hosts provided", e.getMessage());
         }
 
         try {
-            RestClient.builder().setHosts();
-            fail("should have failed");
-        } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "no hosts provided");
-        }
-
-        try {
-            RestClient.builder().build();
-            fail("should have failed");
-        } catch(IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "no hosts provided");
-        }
-
-        try {
-            RestClient.builder().setHosts(new HttpHost[]{new HttpHost("localhost", 9200), null}).build();
+            RestClient.builder(new HttpHost[]{new HttpHost("localhost", 9200), null}).build();
             fail("should have failed");
         } catch(NullPointerException e) {
-            assertEquals(e.getMessage(), "host cannot be null");
+            assertEquals("host cannot be null", e.getMessage());
         }
 
         try {
-            RestClient.builder().setDefaultHeaders(null);
+            RestClient.builder(new HttpHost("localhost", 9200)).setMaxRetryTimeout(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
             fail("should have failed");
-        } catch(NullPointerException e) {
-            assertEquals(e.getMessage(), "default headers must not be null");
+        } catch(IllegalArgumentException e) {
+            assertEquals("maxRetryTimeout must be greater than 0", e.getMessage());
         }
 
         try {
-            RestClient.builder().setDefaultHeaders(new Header[]{null});
+            RestClient.builder(new HttpHost("localhost", 9200)).setDefaultHeaders(null);
             fail("should have failed");
         } catch(NullPointerException e) {
-            assertEquals(e.getMessage(), "default header must not be null");
+            assertEquals("default headers must not be null", e.getMessage());
+        }
+
+        try {
+            RestClient.builder(new HttpHost("localhost", 9200)).setDefaultHeaders(new Header[]{null});
+            fail("should have failed");
+        } catch(NullPointerException e) {
+            assertEquals("default header must not be null", e.getMessage());
         }
 
-        RestClient.Builder builder = RestClient.builder();
         int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
         HttpHost[] hosts = new HttpHost[numNodes];
         for (int i = 0; i < numNodes; i++) {
             hosts[i] = new HttpHost("localhost", 9200 + i);
         }
-        builder.setHosts(hosts);
+        RestClient.Builder builder = RestClient.builder(hosts);
         if (random().nextBoolean()) {
             builder.setHttpClient(HttpClientBuilder.create().build());
         }
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java b/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
index 420ff6ed7f7..c5067dca63f 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
@@ -77,8 +77,8 @@ public class RestClientIntegTests extends LuceneTestCase {
             String headerValue = RandomStrings.randomAsciiOfLengthBetween(random(), 3, 10);
             defaultHeaders[i] = new BasicHeader(headerName, headerValue);
         }
-        restClient = RestClient.builder().setDefaultHeaders(defaultHeaders)
-                .setHosts(new HttpHost(httpServer.getAddress().getHostName(), httpServer.getAddress().getPort())).build();
+        restClient = RestClient.builder(new HttpHost(httpServer.getAddress().getHostName(), httpServer.getAddress().getPort()))
+                .setDefaultHeaders(defaultHeaders).build();
     }
 
     private static void createStatusCodeContext(HttpServer httpServer, final int statusCode) {
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java b/client/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
index d78a03509d0..7ea24cef8e3 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
@@ -87,7 +87,7 @@ public class RestClientMultipleHostsTests extends LuceneTestCase {
         for (int i = 0; i < numHosts; i++) {
             httpHosts[i] = new HttpHost("localhost", 9200 + i);
         }
-        restClient = RestClient.builder().setHosts(httpHosts).setHttpClient(httpClient).build();
+        restClient = RestClient.builder(httpHosts).setHttpClient(httpClient).build();
         failureListener = new TrackingFailureListener();
         restClient.setFailureListener(failureListener);
     }
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java b/client/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
index c3c08bee628..1112f38bbee 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
@@ -122,7 +122,7 @@ public class RestClientSingleHostTests extends LuceneTestCase {
             defaultHeaders[i] = new BasicHeader(headerName, headerValue);
         }
         httpHost = new HttpHost("localhost", 9200);
-        restClient = RestClient.builder().setHosts(httpHost).setHttpClient(httpClient).setDefaultHeaders(defaultHeaders).build();
+        restClient = RestClient.builder(httpHost).setHttpClient(httpClient).setDefaultHeaders(defaultHeaders).build();
         failureListener = new TrackingFailureListener();
         restClient.setFailureListener(failureListener);
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index ab25d25fac0..054da7664b5 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -2068,7 +2068,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
                 hosts.add(new HttpHost(NetworkAddress.format(address.getAddress()), address.getPort(), protocol));
             }
         }
-        RestClient.Builder builder = RestClient.builder().setHosts(hosts.toArray(new HttpHost[hosts.size()]));
+        RestClient.Builder builder = RestClient.builder(hosts.toArray(new HttpHost[hosts.size()]));
         if (httpClient != null) {
             builder.setHttpClient(httpClient);
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java
index 94f85422edf..d21e302168b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java
@@ -305,7 +305,7 @@ public class RestTestClient implements Closeable {
             hosts[i] = new HttpHost(url.getHost(), url.getPort(), protocol);
         }
 
-        RestClient.Builder builder = RestClient.builder().setHttpClient(httpClient).setHosts(hosts);
+        RestClient.Builder builder = RestClient.builder(hosts).setHttpClient(httpClient);
         try (ThreadContext threadContext = new ThreadContext(settings)) {
             Header[] defaultHeaders = new Header[threadContext.getHeaders().size()];
             int i = 0;

From a2372778ddb7445282f8a977b81464afcc631b12 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Tue, 31 May 2016 21:42:45 +0200
Subject: [PATCH 0716/1311] Fix problem with TimeIntervalRounding on DST end

Due to an error in our current TimeIntervalRounding, two dates can
round to the same key, even when they are 1h apart when using
short interval roundings (e.g. 20m) and a time zone with DST change.

Here is an example for the CET time zone:

On 25 October 2015, 03:00:00 clocks are turned backward 1 hour to
02:00:00 local standard time. The dates
"2015-10-25T02:15:00+02:00" (1445732100000) (before DST end) and
"2015-10-25T02:15:00+01:00" (1445735700000) (after DST end)
are thus 1h apart, but currently they round to the same value
"2015-10-25T02:00:00.000+01:00" (1445734800000).

This violates an important invariant of rounding, namely that the
rounded value must be less or equal to the value that is rounded.
It also leads to wrong histogram bucket counts because documents in
[02:00:00+02:00, 02:20:00+02:00) go to the same bucket as documents
from [02:00:00+01:00, 02:20:00+01:00).

The problem happens because in TimeIntervalRounding#roundKey() we
need to perform the rounding operation in local time, but on
converting back to UTC we don't honor the original values time zone
offset. This fix changes that and adds tests both for DST start and
DST end as well as a test that demonstrates what happens to bucket
sizes when the dst change is not evently divisibly by the interval.
---
 .../common/rounding/TimeZoneRounding.java     |  5 +-
 .../rounding/TimeZoneRoundingTests.java       | 51 +++++++++++++++++++
 2 files changed, 53 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java b/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java
index 5bb868ed253..c78d6c730b6 100644
--- a/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java
+++ b/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java
@@ -216,10 +216,9 @@ public abstract class TimeZoneRounding extends Rounding {
 
         @Override
         public long roundKey(long utcMillis) {
-            long timeLocal = utcMillis;
-            timeLocal = timeZone.convertUTCToLocal(utcMillis);
+            long timeLocal = timeZone.convertUTCToLocal(utcMillis);
             long rounded = Rounding.Interval.roundValue(Rounding.Interval.roundKey(timeLocal, interval), interval);
-            return timeZone.convertLocalToUTC(rounded, false);
+            return timeZone.convertLocalToUTC(rounded, false, utcMillis);
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java b/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
index af7b53712c1..49ba26231da 100644
--- a/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
+++ b/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
@@ -258,6 +258,57 @@ public class TimeZoneRoundingTests extends ESTestCase {
         }
     }
 
+    /**
+     * test DST end with interval rounding
+     * CET: 25 October 2015, 03:00:00 clocks were turned backward 1 hour to 25 October 2015, 02:00:00 local standard time
+     */
+    public void testTimeIntervalCET_DST_End() {
+        long interval = TimeUnit.MINUTES.toMillis(20);
+        TimeZoneRounding rounding = new TimeIntervalRounding(interval, DateTimeZone.forID("CET"));
+
+        assertThat(rounding.round(time("2015-10-25T01:55:00+02:00")), equalTo(time("2015-10-25T01:40:00+02:00")));
+        assertThat(rounding.round(time("2015-10-25T02:15:00+02:00")), equalTo(time("2015-10-25T02:00:00+02:00")));
+        assertThat(rounding.round(time("2015-10-25T02:35:00+02:00")), equalTo(time("2015-10-25T02:20:00+02:00")));
+        assertThat(rounding.round(time("2015-10-25T02:55:00+02:00")), equalTo(time("2015-10-25T02:40:00+02:00")));
+        // after DST shift
+        assertThat(rounding.round(time("2015-10-25T02:15:00+01:00")), equalTo(time("2015-10-25T02:00:00+01:00")));
+        assertThat(rounding.round(time("2015-10-25T02:35:00+01:00")), equalTo(time("2015-10-25T02:20:00+01:00")));
+        assertThat(rounding.round(time("2015-10-25T02:55:00+01:00")), equalTo(time("2015-10-25T02:40:00+01:00")));
+        assertThat(rounding.round(time("2015-10-25T03:15:00+01:00")), equalTo(time("2015-10-25T03:00:00+01:00")));
+    }
+
+    /**
+     * test DST start with interval rounding
+     * CET: 27 March 2016, 02:00:00 clocks were turned forward 1 hour to 27 March 2016, 03:00:00 local daylight time
+     */
+    public void testTimeIntervalCET_DST_Start() {
+        long interval = TimeUnit.MINUTES.toMillis(20);
+        TimeZoneRounding rounding = new TimeIntervalRounding(interval, DateTimeZone.forID("CET"));
+        // test DST start
+        assertThat(rounding.round(time("2016-03-27T01:55:00+01:00")), equalTo(time("2016-03-27T01:40:00+01:00")));
+        assertThat(rounding.round(time("2016-03-27T02:00:00+01:00")), equalTo(time("2016-03-27T03:00:00+02:00")));
+        assertThat(rounding.round(time("2016-03-27T03:15:00+02:00")), equalTo(time("2016-03-27T03:00:00+02:00")));
+        assertThat(rounding.round(time("2016-03-27T03:35:00+02:00")), equalTo(time("2016-03-27T03:20:00+02:00")));
+    }
+
+    /**
+     * test DST start with offset not fitting interval, e.g. Asia/Kathmandu
+     * adding 15min on 1986-01-01T00:00:00 the interval from
+     * 1986-01-01T00:15:00+05:45 to 1986-01-01T00:20:00+05:45 to only be 5min
+     * long
+     */
+    public void testTimeInterval_Kathmandu_DST_Start() {
+        long interval = TimeUnit.MINUTES.toMillis(20);
+        TimeZoneRounding rounding = new TimeIntervalRounding(interval, DateTimeZone.forID("Asia/Kathmandu"));
+        assertThat(rounding.round(time("1985-12-31T23:55:00+05:30")), equalTo(time("1985-12-31T23:40:00+05:30")));
+        assertThat(rounding.round(time("1986-01-01T00:16:00+05:45")), equalTo(time("1986-01-01T00:15:00+05:45")));
+        assertThat(time("1986-01-01T00:15:00+05:45") - time("1985-12-31T23:40:00+05:30"), equalTo(TimeUnit.MINUTES.toMillis(20)));
+        assertThat(rounding.round(time("1986-01-01T00:26:00+05:45")), equalTo(time("1986-01-01T00:20:00+05:45")));
+        assertThat(time("1986-01-01T00:20:00+05:45") - time("1986-01-01T00:15:00+05:45"), equalTo(TimeUnit.MINUTES.toMillis(5)));
+        assertThat(rounding.round(time("1986-01-01T00:46:00+05:45")), equalTo(time("1986-01-01T00:40:00+05:45")));
+        assertThat(time("1986-01-01T00:40:00+05:45") - time("1986-01-01T00:20:00+05:45"), equalTo(TimeUnit.MINUTES.toMillis(20)));
+    }
+
     /**
      * randomized test on {@link TimeIntervalRounding} with random interval and time zone offsets
      */

From bec26015b23d032e1af28e6e8ddd921b52f6c949 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Wed, 8 Jun 2016 15:38:11 +0200
Subject: [PATCH 0717/1311] [TEST] add a dedicated test for empty files

---
 .../org/elasticsearch/index/store/StoreTests.java   | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/core/src/test/java/org/elasticsearch/index/store/StoreTests.java b/core/src/test/java/org/elasticsearch/index/store/StoreTests.java
index be45b1ad330..4f3e68a9115 100644
--- a/core/src/test/java/org/elasticsearch/index/store/StoreTests.java
+++ b/core/src/test/java/org/elasticsearch/index/store/StoreTests.java
@@ -199,6 +199,19 @@ public class StoreTests extends ESTestCase {
         IOUtils.close(indexInput, verifyingOutput, dir);
     }
 
+    public void testVerifyingIndexOutputOnEmptyFile() throws IOException {
+        Directory dir = newDirectory();
+        IndexOutput verifyingOutput = new Store.LuceneVerifyingIndexOutput(new StoreFileMetaData("foo.bar", 0, Store.digestToString(0)),
+            dir.createOutput("foo1.bar", IOContext.DEFAULT));
+        try {
+            Store.verify(verifyingOutput);
+            fail("should be a corrupted index");
+        } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
+            // ok
+        }
+        IOUtils.close(verifyingOutput, dir);
+    }
+
     public void testChecksumCorrupted() throws IOException {
         Directory dir = newDirectory();
         IndexOutput output = dir.createOutput("foo.bar", IOContext.DEFAULT);

From dec0dcc30b94972502e6a3ee5bcd24cf35344df5 Mon Sep 17 00:00:00 2001
From: Areek Zillur <areek.zillur@elasticsearch.com>
Date: Wed, 8 Jun 2016 11:25:35 -0400
Subject: [PATCH 0718/1311] minor cleanup

---
 .../rollover/TransportRolloverAction.java     | 73 ++++++++-----------
 .../indices/RestRolloverIndexAction.java      |  2 +-
 .../admin/indices/rollover/RolloverIT.java    | 60 +++++++--------
 .../TransportRolloverActionTests.java         |  5 +-
 4 files changed, 64 insertions(+), 76 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
index 3eed74661a8..49ff6f1f23e 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -54,6 +54,7 @@ import java.util.stream.Collectors;
  */
 public class TransportRolloverAction extends TransportMasterNodeAction<RolloverRequest, RolloverResponse> {
 
+    private static final Pattern INDEX_NAME_PATTERN = Pattern.compile("^.*-(\\d)+$");
     private final MetaDataCreateIndexService createIndexService;
     private final MetaDataIndexAliasesService indexAliasesService;
     private final Client client;
@@ -110,7 +111,7 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
                                 false));
                         return;
                     }
-                    if (conditionResults.stream().anyMatch(result -> result.matched)) {
+                    if (conditionResults.size() == 0 || conditionResults.stream().anyMatch(result -> result.matched)) {
                         boolean createRolloverIndex = metaData.index(rolloverIndexName) == null;
                         final RolloverResponse rolloverResponse =
                             new RolloverResponse(sourceIndexName, rolloverIndexName, conditionResults, false, true,
@@ -118,23 +119,20 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
                         if (createRolloverIndex) {
                             CreateIndexClusterStateUpdateRequest updateRequest =
                                 prepareCreateIndexRequest(rolloverIndexName, rolloverRequest);
-                            createIndexService.createIndex(updateRequest, new ActionListener<ClusterStateUpdateResponse>() {
-                                @Override
-                                public void onResponse(ClusterStateUpdateResponse response) {
-                                    indexAliasesService.indicesAliases(
-                                        prepareIndicesAliasesRequest(sourceIndexName, rolloverIndexName, rolloverRequest),
-                                        new IndicesAliasesListener(rolloverResponse, listener));
-                                }
+                            createIndexService.createIndex(updateRequest,
+                                new ActionListener<ClusterStateUpdateResponse>() {
+                                    @Override
+                                    public void onResponse(ClusterStateUpdateResponse response) {
+                                        rollover(rolloverRequest, rolloverResponse, listener);
+                                    }
 
-                                @Override
-                                public void onFailure(Throwable t) {
-                                    listener.onFailure(t);
-                                }
-                            });
+                                    @Override
+                                    public void onFailure(Throwable t) {
+                                        listener.onFailure(t);
+                                    }
+                                });
                         } else {
-                            indexAliasesService.indicesAliases(
-                                prepareIndicesAliasesRequest(sourceIndexName, rolloverIndexName, rolloverRequest),
-                                new IndicesAliasesListener(rolloverResponse, listener));
+                            rollover(rolloverRequest, rolloverResponse, listener);
                         }
                     } else {
                         // conditions not met
@@ -153,42 +151,35 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
         );
     }
 
+    private void rollover(final RolloverRequest request, final RolloverResponse response,
+                          ActionListener<RolloverResponse> listener) {
+        indexAliasesService.indicesAliases(
+            prepareRolloverAliasesUpdateRequest(response.getOldIndex(), response.getNewIndex(), request),
+            new ActionListener<ClusterStateUpdateResponse>() {
+                @Override
+                public void onResponse(ClusterStateUpdateResponse clusterStateUpdateResponse) {
+                    listener.onResponse(response);
+                }
 
-
-    private static final class IndicesAliasesListener implements ActionListener<ClusterStateUpdateResponse> {
-
-        private final ActionListener<RolloverResponse> listener;
-        private final RolloverResponse response;
-
-        public IndicesAliasesListener(RolloverResponse response, ActionListener<RolloverResponse> listener) {
-            this.response = response;
-            this.listener = listener;
-        }
-
-        @Override
-        public void onResponse(ClusterStateUpdateResponse clusterStateUpdateResponse) {
-            listener.onResponse(response);
-        }
-
-        @Override
-        public void onFailure(Throwable e) {
-            listener.onFailure(e);
-        }
+                @Override
+                public void onFailure(Throwable e) {
+                    listener.onFailure(e);
+                }
+            });
     }
 
-    static IndicesAliasesClusterStateUpdateRequest prepareIndicesAliasesRequest(String concreteSourceIndex, String targetIndex,
-                                                                                RolloverRequest request) {
+    static IndicesAliasesClusterStateUpdateRequest prepareRolloverAliasesUpdateRequest(String oldIndex, String newIndex,
+                                                                                       RolloverRequest request) {
         final IndicesAliasesClusterStateUpdateRequest updateRequest = new IndicesAliasesClusterStateUpdateRequest()
             .ackTimeout(request.ackTimeout())
             .masterNodeTimeout(request.masterNodeTimeout());
         AliasAction[] actions = new AliasAction[2];
-        actions[0] = new AliasAction(AliasAction.Type.ADD, targetIndex, request.getSourceAlias());
-        actions[1] = new AliasAction(AliasAction.Type.REMOVE, concreteSourceIndex, request.getSourceAlias());
+        actions[0] = new AliasAction(AliasAction.Type.ADD, newIndex, request.getSourceAlias());
+        actions[1] = new AliasAction(AliasAction.Type.REMOVE, oldIndex, request.getSourceAlias());
         updateRequest.actions(actions);
         return updateRequest;
     }
 
-    static final Pattern INDEX_NAME_PATTERN = Pattern.compile("^.*-(\\d)+$");
 
     static String generateRolloverIndexName(String sourceIndexName) {
         if (INDEX_NAME_PATTERN.matcher(sourceIndexName).matches()) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
index 963080c3122..08565fca9cf 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
@@ -44,7 +44,7 @@ public class RestRolloverIndexAction extends BaseRestHandler {
     @Override
     public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
         if (request.param("alias") == null) {
-            throw new IllegalArgumentException("no source alias");
+            throw new IllegalArgumentException("no alias");
         }
         RolloverRequest rolloverIndexRequest = new RolloverRequest(request.param("alias"));
         if (request.hasContent()) {
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
index 71d6f5a44f7..75a20d05682 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
@@ -34,65 +34,65 @@ import static org.hamcrest.Matchers.equalTo;
 public class RolloverIT extends ESIntegTestCase {
 
     public void testRolloverOnEmptyIndex() throws Exception {
-        assertAcked(prepareCreate("test_index").addAlias(new Alias("test_alias")).get());
+        assertAcked(prepareCreate("test_index-1").addAlias(new Alias("test_alias")).get());
         final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias").get();
-        assertThat(response.getOldIndex(), equalTo("test_index"));
-        assertThat(response.getNewIndex(), equalTo("test_index-1"));
+        assertThat(response.getOldIndex(), equalTo("test_index-1"));
+        assertThat(response.getNewIndex(), equalTo("test_index-2"));
         assertThat(response.isSimulate(), equalTo(false));
         assertThat(response.isRolledOver(), equalTo(true));
         assertThat(response.isRolloverIndexCreated(), equalTo(true));
         assertThat(response.getConditionStatus().size(), equalTo(0));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
-        final IndexMetaData oldIndex = state.metaData().index("test_index");
+        final IndexMetaData oldIndex = state.metaData().index("test_index-1");
         assertFalse(oldIndex.getAliases().containsKey("test_alias"));
-        final IndexMetaData newIndex = state.metaData().index("test_index-1");
+        final IndexMetaData newIndex = state.metaData().index("test_index-2");
         assertTrue(newIndex.getAliases().containsKey("test_alias"));
     }
 
     public void testRollover() throws Exception {
-        assertAcked(prepareCreate("test_index").addAlias(new Alias("test_alias")).get());
-        index("test_index", "type1", "1", "field", "value");
-        flush("test_index");
+        assertAcked(prepareCreate("test_index-2").addAlias(new Alias("test_alias")).get());
+        index("test_index-2", "type1", "1", "field", "value");
+        flush("test_index-2");
         final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias").get();
-        assertThat(response.getOldIndex(), equalTo("test_index"));
-        assertThat(response.getNewIndex(), equalTo("test_index-1"));
+        assertThat(response.getOldIndex(), equalTo("test_index-2"));
+        assertThat(response.getNewIndex(), equalTo("test_index-3"));
         assertThat(response.isSimulate(), equalTo(false));
         assertThat(response.isRolledOver(), equalTo(true));
         assertThat(response.isRolloverIndexCreated(), equalTo(true));
         assertThat(response.getConditionStatus().size(), equalTo(0));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
-        final IndexMetaData oldIndex = state.metaData().index("test_index");
+        final IndexMetaData oldIndex = state.metaData().index("test_index-2");
         assertFalse(oldIndex.getAliases().containsKey("test_alias"));
-        final IndexMetaData newIndex = state.metaData().index("test_index-1");
+        final IndexMetaData newIndex = state.metaData().index("test_index-3");
         assertTrue(newIndex.getAliases().containsKey("test_alias"));
     }
 
     public void testRolloverSimulate() throws Exception {
-        assertAcked(prepareCreate("test_index").addAlias(new Alias("test_alias")).get());
-        index("test_index", "type1", "1", "field", "value");
-        flush("test_index");
+        assertAcked(prepareCreate("test_index-1").addAlias(new Alias("test_alias")).get());
+        index("test_index-1", "type1", "1", "field", "value");
+        flush("test_index-1");
         final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias").simulate(true).get();
-        assertThat(response.getOldIndex(), equalTo("test_index"));
-        assertThat(response.getNewIndex(), equalTo("test_index-1"));
+        assertThat(response.getOldIndex(), equalTo("test_index-1"));
+        assertThat(response.getNewIndex(), equalTo("test_index-2"));
         assertThat(response.isSimulate(), equalTo(true));
         assertThat(response.isRolledOver(), equalTo(false));
         assertThat(response.isRolloverIndexCreated(), equalTo(false));
         assertThat(response.getConditionStatus().size(), equalTo(0));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
-        final IndexMetaData oldIndex = state.metaData().index("test_index");
+        final IndexMetaData oldIndex = state.metaData().index("test_index-1");
         assertTrue(oldIndex.getAliases().containsKey("test_alias"));
-        final IndexMetaData newIndex = state.metaData().index("test_index-1");
+        final IndexMetaData newIndex = state.metaData().index("test_index-2");
         assertNull(newIndex);
     }
 
     public void testRolloverConditionsNotMet() throws Exception {
-        assertAcked(prepareCreate("test_index").addAlias(new Alias("test_alias")).get());
-        index("test_index", "type1", "1", "field", "value");
-        flush("test_index");
+        assertAcked(prepareCreate("test_index-0").addAlias(new Alias("test_alias")).get());
+        index("test_index-0", "type1", "1", "field", "value");
+        flush("test_index-0");
         final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias")
             .addMaxIndexAgeCondition(TimeValue.timeValueHours(4)).get();
-        assertThat(response.getOldIndex(), equalTo("test_index"));
-        assertThat(response.getNewIndex(), equalTo("test_index"));
+        assertThat(response.getOldIndex(), equalTo("test_index-0"));
+        assertThat(response.getNewIndex(), equalTo("test_index-0"));
         assertThat(response.isSimulate(), equalTo(false));
         assertThat(response.isRolledOver(), equalTo(false));
         assertThat(response.isRolloverIndexCreated(), equalTo(false));
@@ -101,26 +101,26 @@ public class RolloverIT extends ESIntegTestCase {
         assertThat(conditionEntry.getKey(), equalTo(new MaxAgeCondition(TimeValue.timeValueHours(4)).toString()));
         assertThat(conditionEntry.getValue(), equalTo(false));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
-        final IndexMetaData oldIndex = state.metaData().index("test_index");
+        final IndexMetaData oldIndex = state.metaData().index("test_index-0");
         assertTrue(oldIndex.getAliases().containsKey("test_alias"));
         final IndexMetaData newIndex = state.metaData().index("test_index-1");
         assertNull(newIndex);
     }
 
     public void testRolloverOnExistingIndex() throws Exception {
-        assertAcked(prepareCreate("test_index").addAlias(new Alias("test_alias")).get());
-        index("test_index", "type1", "1", "field", "value");
+        assertAcked(prepareCreate("test_index-0").addAlias(new Alias("test_alias")).get());
+        index("test_index-0", "type1", "1", "field", "value");
         assertAcked(prepareCreate("test_index-1").get());
         index("test_index-1", "type1", "1", "field", "value");
-        flush("test_index", "test_index-1");
+        flush("test_index-0", "test_index-1");
         final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias").get();
-        assertThat(response.getOldIndex(), equalTo("test_index"));
+        assertThat(response.getOldIndex(), equalTo("test_index-0"));
         assertThat(response.getNewIndex(), equalTo("test_index-1"));
         assertThat(response.isSimulate(), equalTo(false));
         assertThat(response.isRolledOver(), equalTo(true));
         assertThat(response.isRolloverIndexCreated(), equalTo(false));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
-        final IndexMetaData oldIndex = state.metaData().index("test_index");
+        final IndexMetaData oldIndex = state.metaData().index("test_index-0");
         assertFalse(oldIndex.getAliases().containsKey("test_alias"));
         final IndexMetaData newIndex = state.metaData().index("test_index-1");
         assertTrue(newIndex.getAliases().containsKey("test_alias"));
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
index 12e1e90242e..af51645f655 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
@@ -32,10 +32,7 @@ import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.test.ESTestCase;
 
-import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Set;
 
 import static org.elasticsearch.action.admin.indices.rollover.TransportRolloverAction.evaluateConditions;
@@ -96,7 +93,7 @@ public class TransportRolloverActionTests extends ESTestCase {
         String targetIndex = randomAsciiOfLength(10);
         final RolloverRequest rolloverRequest = new RolloverRequest(sourceAlias);
         final IndicesAliasesClusterStateUpdateRequest updateRequest =
-            TransportRolloverAction.prepareIndicesAliasesRequest(sourceIndex, targetIndex, rolloverRequest);
+            TransportRolloverAction.prepareRolloverAliasesUpdateRequest(sourceIndex, targetIndex, rolloverRequest);
 
         final AliasAction[] actions = updateRequest.actions();
         assertThat(actions.length, equalTo(2));

From 5161afe5e333be7e4454b2c71411c53373b2fa1d Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 3 Jun 2016 09:15:10 -0400
Subject: [PATCH 0719/1311] Support optional ctor args in
 ConstructingObjectParser

You declare them like
```
static {
  PARSER.declareInt(optionalConstructorArg(), new ParseField("animal"));
}
```

Other than being optional they follow all of the rules of regular
`constructorArg()`s. Parsing an object with optional constructor args
is going to be slightly less efficient than parsing an object with
all required args if some of the optional args aren't specified because
ConstructingObjectParser isn't able to build the target before the
end of the json object.
---
 .../xcontent/ConstructingObjectParser.java    | 140 ++++++++++------
 .../ConstructingObjectParserTests.java        | 153 ++++++++++++++----
 2 files changed, 206 insertions(+), 87 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java
index 3c4d8875d2a..e1400463a72 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java
@@ -26,7 +26,6 @@ import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.function.BiConsumer;
 import java.util.function.BiFunction;
@@ -53,12 +52,13 @@ import java.util.function.Function;
  * </p>
  * <pre>{@code
  *   private static final ConstructingObjectParser<Thing, SomeContext> PARSER = new ConstructingObjectParser<>("thing",
- *           a -> new Thing((String) a[0], (String) a[1]));
+ *           a -> new Thing((String) a[0], (String) a[1], (Integer) a[2]));
  *   static {
  *       PARSER.declareString(constructorArg(), new ParseField("animal"));
  *       PARSER.declareString(constructorArg(), new ParseField("vegetable"));
- *       PARSER.declareInt(Thing::setMineral, new ParseField("mineral"));
+ *       PARSER.declareInt(optionalConstructorArg(), new ParseField("mineral"));
  *       PARSER.declareInt(Thing::setFruit, new ParseField("fruit"));
+ *       PARSER.declareInt(Thing::setBug, new ParseField("bug"));
  *   }
  * }</pre>
  * <p>
@@ -70,19 +70,29 @@ import java.util.function.Function;
  * it allocates <code>3 + 2 * param_count</code> objects per parse. If this overhead is too much for you then feel free to have ObjectParser
  * parse a secondary object and have that one call the target object's constructor. That ought to be rare though.
  * </p>
+ * <p>
+ * Note: if optional constructor arguments aren't specified then the number of allocations is always the worst case.
+ * </p>
  */
 public final class ConstructingObjectParser<Value, Context extends ParseFieldMatcherSupplier> extends AbstractObjectParser<Value, Context> {
     /**
-     * Consumer that marks a field as a constructor argument instead of a real object field.
+     * Consumer that marks a field as a required constructor argument instead of a real object field.
      */
-    private static final BiConsumer<Object, Object> CONSTRUCTOR_ARG_MARKER = (a, b) -> {
+    private static final BiConsumer<Object, Object> REQUIRED_CONSTRUCTOR_ARG_MARKER = (a, b) -> {
+        throw new UnsupportedOperationException("I am just a marker I should never be called.");
+    };
+
+    /**
+     * Consumer that marks a field as an optional constructor argument instead of a real object field.
+     */
+    private static final BiConsumer<Object, Object> OPTIONAL_CONSTRUCTOR_ARG_MARKER = (a, b) -> {
         throw new UnsupportedOperationException("I am just a marker I should never be called.");
     };
 
     /**
      * List of constructor names used for generating the error message if not all arrive.
      */
-    private final List<ParseField> constructorArgNames = new ArrayList<>();
+    private final List<ConstructorArgInfo> constructorArgInfos = new ArrayList<>();
     private final ObjectParser<Target, Context> objectParser;
     private final Function<Object[], Value> builder;
     /**
@@ -120,27 +130,39 @@ public final class ConstructingObjectParser<Value, Context extends ParseFieldMat
     }
 
     /**
-     * Pass the {@linkplain BiConsumer} this returns the declare methods to declare a constructor argument. See this class's javadoc for an
-     * example. The order in which these are declared matters: it is the order that they come in the array passed to {@link #builder} and
-     * the order that missing arguments are reported to the user if any are missing. When all of these parameters are parsed from the
-     * {@linkplain XContentParser} the target object is immediately built.
+     * Pass the {@linkplain BiConsumer} this returns the declare methods to declare a required constructor argument. See this class's
+     * javadoc for an example. The order in which these are declared matters: it is the order that they come in the array passed to
+     * {@link #builder} and the order that missing arguments are reported to the user if any are missing. When all of these parameters are
+     * parsed from the {@linkplain XContentParser} the target object is immediately built.
      */
     @SuppressWarnings("unchecked") // Safe because we never call the method. This is just trickery to make the interface pretty.
     public static <Value, FieldT> BiConsumer<Value, FieldT> constructorArg() {
-        return (BiConsumer<Value, FieldT>) CONSTRUCTOR_ARG_MARKER;
+        return (BiConsumer<Value, FieldT>) REQUIRED_CONSTRUCTOR_ARG_MARKER;
+    }
+
+    /**
+     * Pass the {@linkplain BiConsumer} this returns the declare methods to declare an optional constructor argument. See this class's
+     * javadoc for an example. The order in which these are declared matters: it is the order that they come in the array passed to
+     * {@link #builder} and the order that missing arguments are reported to the user if any are missing. When all of these parameters are
+     * parsed from the {@linkplain XContentParser} the target object is immediately built.
+     */
+    @SuppressWarnings("unchecked") // Safe because we never call the method. This is just trickery to make the interface pretty.
+    public static <Value, FieldT> BiConsumer<Value, FieldT> optionalConstructorArg() {
+        return (BiConsumer<Value, FieldT>) OPTIONAL_CONSTRUCTOR_ARG_MARKER;
     }
 
     @Override
     public <T> void declareField(BiConsumer<Value, T> consumer, ContextParser<Context, T> parser, ParseField parseField, ValueType type) {
-        if (consumer == CONSTRUCTOR_ARG_MARKER) {
+        if (consumer == REQUIRED_CONSTRUCTOR_ARG_MARKER || consumer == OPTIONAL_CONSTRUCTOR_ARG_MARKER) {
             /*
              * Constructor arguments are detected by this "marker" consumer. It keeps the API looking clean even if it is a bit sleezy. We
              * then build a new consumer directly against the object parser that triggers the "constructor arg just arrived behavior" of the
              * parser. Conveniently, we can close over the position of the constructor in the argument list so we don't need to do any fancy
              * or expensive lookups whenever the constructor args come in.
              */
-            int position = constructorArgNames.size();
-            constructorArgNames.add(parseField);
+            int position = constructorArgInfos.size();
+            boolean required = consumer == REQUIRED_CONSTRUCTOR_ARG_MARKER;
+            constructorArgInfos.add(new ConstructorArgInfo(parseField, required));
             objectParser.declareField((target, v) -> target.constructorArg(position, parseField, v), parser, parseField, type);
         } else {
             numberOfFields += 1;
@@ -186,7 +208,7 @@ public final class ConstructingObjectParser<Value, Context extends ParseFieldMat
         /**
          * Array of constructor args to be passed to the {@link ConstructingObjectParser#builder}.
          */
-        private final Object[] constructorArgs = new Object[constructorArgNames.size()];
+        private final Object[] constructorArgs = new Object[constructorArgInfos.size()];
         /**
          * The parser this class is working against. We store it here so we can fetch it conveniently when queueing fields to lookup the
          * location of each field so that we can give a useful error message when replaying the queue.
@@ -224,20 +246,8 @@ public final class ConstructingObjectParser<Value, Context extends ParseFieldMat
             }
             constructorArgs[position] = value;
             constructorArgsCollected++;
-            if (constructorArgsCollected != constructorArgNames.size()) {
-                return;
-            }
-            try {
-                targetObject = builder.apply(constructorArgs);
-                while (queuedFieldsCount > 0) {
-                    queuedFieldsCount -= 1;
-                    queuedFields[queuedFieldsCount].accept(targetObject);
-                }
-            } catch (ParsingException e) {
-                throw new ParsingException(e.getLineNumber(), e.getColumnNumber(),
-                        "failed to build [" + objectParser.getName() + "] after last required field arrived", e);
-            } catch (Exception e) {
-                throw new ParsingException(null, "Failed to build [" + objectParser.getName() + "] after last required field arrived", e);
+            if (constructorArgsCollected == constructorArgInfos.size()) {
+                buildTarget();
             }
         }
 
@@ -263,36 +273,62 @@ public final class ConstructingObjectParser<Value, Context extends ParseFieldMat
             if (targetObject != null) {
                 return targetObject;
             }
-            // The object hasn't been built which ought to mean we're missing some constructor arguments.
+            /*
+             * The object hasn't been built which ought to mean we're missing some constructor arguments. But they could be optional! We'll
+             * check if they are all optional and build the error message at the same time - if we don't start the error message then they
+             * were all optional!
+             */
             StringBuilder message = null;
             for (int i = 0; i < constructorArgs.length; i++) {
-                if (constructorArgs[i] == null) {
-                    ParseField arg = constructorArgNames.get(i);
-                    if (message == null) {
-                        message = new StringBuilder("Required [").append(arg);
-                    } else {
-                        message.append(", ").append(arg);
-                    }
+                if (constructorArgs[i] != null) continue;
+                ConstructorArgInfo arg = constructorArgInfos.get(i);
+                if (false == arg.required) continue;
+                if (message == null) {
+                    message = new StringBuilder("Required [").append(arg.field);
+                } else {
+                    message.append(", ").append(arg.field);
                 }
             }
+            if (message != null) {
+                // There were non-optional constructor arguments missing.
+                throw new IllegalArgumentException(message.append(']').toString());
+            }
             /*
-             * There won't be if there weren't any constructor arguments declared. That is fine, we'll just throw that error back at the to
-             * the user. This will happen every time so we can be confident that this'll be caught in testing so we can talk to the user
-             * like they are a developer. The only time a user will see this is if someone writes a parser and never tests it which seems
-             * like a bad idea.
+             * If there weren't any constructor arguments declared at all then we won't get an error message but this isn't really a valid
+             * use of ConstructingObjectParser. You should be using ObjectParser instead. Since this is more of a programmer error and the
+             * parser ought to still work we just assert this.
              */
-            if (constructorArgNames.isEmpty()) {
-                throw new IllegalStateException("[" + objectParser.getName() + "] must configure at least on constructor argument. If it "
-                        + "doens't have any it should use ObjectParser instead of ConstructingObjectParser. This is a bug in the parser "
-                        + "declaration.");
+            assert false == constructorArgInfos.isEmpty() : "[" + objectParser.getName() + "] must configure at least on constructor "
+                        + "argument. If it doesn't have any it should use ObjectParser instead of ConstructingObjectParser. This is a bug "
+                        + "in the parser declaration.";
+            // All missing constructor arguments were optional. Just build the target and return it.
+            buildTarget();
+            return targetObject;
+        }
+
+        private void buildTarget() {
+            try {
+                targetObject = builder.apply(constructorArgs);
+                while (queuedFieldsCount > 0) {
+                    queuedFieldsCount -= 1;
+                    queuedFields[queuedFieldsCount].accept(targetObject);
+                }
+            } catch (ParsingException e) {
+                throw new ParsingException(e.getLineNumber(), e.getColumnNumber(),
+                        "failed to build [" + objectParser.getName() + "] after last required field arrived", e);
+            } catch (Exception e) {
+                throw new ParsingException(null, "Failed to build [" + objectParser.getName() + "] after last required field arrived", e);
             }
-            if (message == null) {
-                throw new IllegalStateException("The targetObject wasn't built but we aren't missing any constructor args. This is a bug "
-                        + " in ConstructingObjectParser. Here are the constructor arguments " + Arrays.toString(constructorArgs)
-                        + " and here are is the count [" + constructorArgsCollected + "]. Good luck figuring out what happened."
-                        + " I'm truly sorry you got here.");
-            }
-            throw new IllegalArgumentException(message.append(']').toString());
+        }
+    }
+
+    private static class ConstructorArgInfo {
+        final ParseField field;
+        final boolean required;
+
+        public ConstructorArgInfo(ParseField field, boolean required) {
+            this.field = field;
+            this.required = required;
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java
index e04fdab12aa..f3592936765 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java
@@ -19,17 +19,26 @@
 
 package org.elasticsearch.common.xcontent;
 
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matcher;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
 
+import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.nullValue;
 
 public class ConstructingObjectParserTests extends ESTestCase {
     private static final ParseFieldMatcherSupplier MATCHER = () -> ParseFieldMatcher.STRICT;
@@ -38,7 +47,7 @@ public class ConstructingObjectParserTests extends ESTestCase {
      * Builds the object in random order and parses it.
      */
     public void testRandomOrder() throws Exception {
-        HasRequiredArguments expected = new HasRequiredArguments(randomAsciiOfLength(5), randomInt());
+        HasCtorArguments expected = new HasCtorArguments(randomAsciiOfLength(5), randomInt());
         expected.setMineral(randomInt());
         expected.setFruit(randomInt());
         expected.setA(randomBoolean() ? null : randomAsciiOfLength(5));
@@ -49,9 +58,8 @@ public class ConstructingObjectParserTests extends ESTestCase {
         expected.toXContent(builder, ToXContent.EMPTY_PARAMS);
         builder = shuffleXContent(builder);
         BytesReference bytes = builder.bytes();
-        XContentParser parser = XContentFactory.xContent(bytes).createParser(bytes);
-        try {
-            HasRequiredArguments parsed = HasRequiredArguments.PARSER.apply(parser, MATCHER);
+        try (XContentParser parser = XContentFactory.xContent(bytes).createParser(bytes)) {
+            HasCtorArguments parsed = randomFrom(HasCtorArguments.ALL_PARSERS).apply(parser, MATCHER);
             assertEquals(expected.animal, parsed.animal);
             assertEquals(expected.vegetable, parsed.vegetable);
             assertEquals(expected.mineral, parsed.mineral);
@@ -66,44 +74,84 @@ public class ConstructingObjectParserTests extends ESTestCase {
         }
     }
 
-    public void testMissingAllConstructorParams() throws IOException {
+    public void testMissingAllConstructorArgs() throws IOException {
         XContentParser parser = XContentType.JSON.xContent().createParser(
                   "{\n"
                 + "  \"mineral\": 1\n"
                 + "}");
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> HasRequiredArguments.PARSER.apply(parser, MATCHER));
-        assertEquals("Required [animal, vegetable]", e.getMessage());
+        ConstructingObjectParser<HasCtorArguments, ParseFieldMatcherSupplier> objectParser = randomBoolean() ? HasCtorArguments.PARSER
+                : HasCtorArguments.PARSER_VEGETABLE_OPTIONAL;
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> objectParser.apply(parser, MATCHER));
+        if (objectParser == HasCtorArguments.PARSER) {
+            assertEquals("Required [animal, vegetable]", e.getMessage());
+        } else {
+            assertEquals("Required [animal]", e.getMessage());
+        }
     }
 
-    public void testMissingSecondConstructorParam() throws IOException {
+    public void testMissingAllConstructorArgsButNotRequired() throws IOException {
+        XContentParser parser = XContentType.JSON.xContent().createParser(
+                "{\n"
+              + "  \"mineral\": 1\n"
+              + "}");
+        HasCtorArguments parsed = HasCtorArguments.PARSER_ALL_OPTIONAL.apply(parser, MATCHER);
+        assertEquals(1, parsed.mineral);
+    }
+
+    public void testMissingSecondConstructorArg() throws IOException {
         XContentParser parser = XContentType.JSON.xContent().createParser(
                   "{\n"
                 + "  \"mineral\": 1,\n"
                 + "  \"animal\": \"cat\"\n"
                 + "}");
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> HasRequiredArguments.PARSER.apply(parser, MATCHER));
+                () -> HasCtorArguments.PARSER.apply(parser, MATCHER));
         assertEquals("Required [vegetable]", e.getMessage());
     }
 
-    public void testMissingFirstConstructorParam() throws IOException {
+    public void testMissingSecondConstructorArgButNotRequired() throws IOException {
+        XContentParser parser = XContentType.JSON.xContent().createParser(
+                "{\n"
+              + "  \"mineral\": 1,\n"
+              + "  \"animal\": \"cat\"\n"
+              + "}");
+        @SuppressWarnings("unchecked")
+        HasCtorArguments parsed = randomFrom(HasCtorArguments.PARSER_VEGETABLE_OPTIONAL, HasCtorArguments.PARSER_ALL_OPTIONAL).apply(parser,
+                MATCHER);
+        assertEquals(1, parsed.mineral);
+        assertEquals("cat", parsed.animal);
+    }
+
+    public void testMissingFirstConstructorArg() throws IOException {
         XContentParser parser = XContentType.JSON.xContent().createParser(
                   "{\n"
                 + "  \"mineral\": 1,\n"
                 + "  \"vegetable\": 2\n"
                 + "}");
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> HasRequiredArguments.PARSER.apply(parser, MATCHER));
+        @SuppressWarnings("unchecked")
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> randomFrom(HasCtorArguments.PARSER, HasCtorArguments.PARSER_VEGETABLE_OPTIONAL).apply(parser, MATCHER));
         assertEquals("Required [animal]", e.getMessage());
     }
 
+    public void testMissingFirstConstructorArgButNotRequired() throws IOException {
+        XContentParser parser = XContentType.JSON.xContent().createParser(
+                "{\n"
+              + "  \"mineral\": 1,\n"
+              + "  \"vegetable\": 2\n"
+              + "}");
+        HasCtorArguments parsed = HasCtorArguments.PARSER_ALL_OPTIONAL.apply(parser, MATCHER);
+        assertEquals(1, parsed.mineral);
+        assertEquals((Integer) 2, parsed.vegetable);
+    }
+
     public void testRepeatedConstructorParam() throws IOException {
         XContentParser parser = XContentType.JSON.xContent().createParser(
                   "{\n"
                 + "  \"vegetable\": 1,\n"
                 + "  \"vegetable\": 2\n"
                 + "}");
-        Throwable e = expectThrows(ParsingException.class, () -> HasRequiredArguments.PARSER.apply(parser, MATCHER));
+        Throwable e = expectThrows(ParsingException.class, () -> randomFrom(HasCtorArguments.ALL_PARSERS).apply(parser, MATCHER));
         assertEquals("[has_required_arguments] failed to parse field [vegetable]", e.getMessage());
         e = e.getCause();
         assertThat(e, instanceOf(IllegalArgumentException.class));
@@ -117,7 +165,7 @@ public class ConstructingObjectParserTests extends ESTestCase {
                 + "  \"vegetable\": 2,\n"
                 + "  \"a\": \"supercalifragilisticexpialidocious\"\n"
                 + "}");
-        ParsingException e = expectThrows(ParsingException.class, () -> HasRequiredArguments.PARSER.apply(parser, MATCHER));
+        ParsingException e = expectThrows(ParsingException.class, () -> randomFrom(HasCtorArguments.ALL_PARSERS).apply(parser, MATCHER));
         assertEquals("[has_required_arguments] failed to parse field [a]", e.getMessage());
         assertEquals(4, e.getLineNumber());
         assertEquals("[a] must be less than 10 characters in length but was [supercalifragilisticexpialidocious]",
@@ -131,7 +179,7 @@ public class ConstructingObjectParserTests extends ESTestCase {
                 + "  \"animal\": \"cat\"\n,"
                 + "  \"vegetable\": 2\n"
                 + "}");
-        ParsingException e = expectThrows(ParsingException.class, () -> HasRequiredArguments.PARSER.apply(parser, MATCHER));
+        ParsingException e = expectThrows(ParsingException.class, () -> randomFrom(HasCtorArguments.ALL_PARSERS).apply(parser, MATCHER));
         assertEquals("[has_required_arguments] failed to parse field [vegetable]", e.getMessage());
         assertEquals(4, e.getLineNumber());
         e = (ParsingException) e.getCause();
@@ -149,18 +197,28 @@ public class ConstructingObjectParserTests extends ESTestCase {
         }
         ConstructingObjectParser<NoConstructorArgs, ParseFieldMatcherSupplier> parser = new ConstructingObjectParser<>(
                 "constructor_args_required", (a) -> new NoConstructorArgs());
-        Exception e = expectThrows(IllegalStateException.class, () -> parser.apply(XContentType.JSON.xContent().createParser("{}"), null));
-        assertEquals("[constructor_args_required] must configure at least on constructor argument. If it doens't have any it "
-                + "should use ObjectParser instead of ConstructingObjectParser. This is a bug in the parser declaration.", e.getMessage());
+        try {
+            parser.apply(XContentType.JSON.xContent().createParser("{}"), null);
+            fail("Expected AssertionError");
+        } catch (AssertionError e) {
+            assertEquals("[constructor_args_required] must configure at least on constructor argument. If it doesn't have any it should "
+                    + "use ObjectParser instead of ConstructingObjectParser. This is a bug in the parser declaration.", e.getMessage());
+        }
     }
 
     /**
      * Tests the non-constructor fields are only set on time.
      */
     public void testCalledOneTime() throws IOException {
+        boolean ctorArgOptional = randomBoolean();
         class CalledOneTime {
             public CalledOneTime(String yeah) {
-                assertEquals("!", yeah);
+                Matcher<String> yeahMatcher = equalTo("!");
+                if (ctorArgOptional) {
+                    // either(yeahMatcher).or(nullValue) is broken by https://github.com/hamcrest/JavaHamcrest/issues/49
+                    yeahMatcher = anyOf(yeahMatcher, nullValue());
+                }
+                assertThat(yeah, yeahMatcher);
             }
 
             boolean fooSet = false;
@@ -172,7 +230,7 @@ public class ConstructingObjectParserTests extends ESTestCase {
         ConstructingObjectParser<CalledOneTime, ParseFieldMatcherSupplier> parser = new ConstructingObjectParser<>("one_time_test",
                 (a) -> new CalledOneTime((String) a[0]));
         parser.declareString(CalledOneTime::setFoo, new ParseField("foo"));
-        parser.declareString(constructorArg(), new ParseField("yeah"));
+        parser.declareString(ctorArgOptional ? optionalConstructorArg() : constructorArg(), new ParseField("yeah"));
 
         // ctor arg first so we can test for the bug we found one time
         XContentParser xcontent = XContentType.JSON.xContent().createParser(
@@ -191,12 +249,23 @@ public class ConstructingObjectParserTests extends ESTestCase {
               + "}");
         result = parser.apply(xcontent, MATCHER);
         assertTrue(result.fooSet);
+
+        if (ctorArgOptional) {
+            // and without the constructor arg if we've made it optional
+            xcontent = XContentType.JSON.xContent().createParser(
+                    "{\n"
+                  + "  \"foo\": \"foo\"\n"
+                  + "}");
+            result = parser.apply(xcontent, MATCHER);
+        }
+        assertTrue(result.fooSet);
     }
 
-
-    private static class HasRequiredArguments implements ToXContent {
+    private static class HasCtorArguments implements ToXContent {
+        @Nullable
         final String animal;
-        final int vegetable;
+        @Nullable
+        final Integer vegetable;
         int mineral;
         int fruit;
         String a;
@@ -204,7 +273,7 @@ public class ConstructingObjectParserTests extends ESTestCase {
         String c;
         boolean d;
 
-        public HasRequiredArguments(String animal, int vegetable) {
+        public HasCtorArguments(@Nullable String animal, @Nullable Integer vegetable) {
             this.animal = animal;
             this.vegetable = vegetable;
         }
@@ -263,17 +332,31 @@ public class ConstructingObjectParserTests extends ESTestCase {
             return builder;
         }
 
-        public static final ConstructingObjectParser<HasRequiredArguments, ParseFieldMatcherSupplier> PARSER =
-                new ConstructingObjectParser<>("has_required_arguments", a -> new HasRequiredArguments((String) a[0], (Integer) a[1]));
-        static {
-            PARSER.declareString(constructorArg(), new ParseField("animal"));
-            PARSER.declareInt(constructorArg(), new ParseField("vegetable"));
-            PARSER.declareInt(HasRequiredArguments::setMineral, new ParseField("mineral"));
-            PARSER.declareInt(HasRequiredArguments::setFruit, new ParseField("fruit"));
-            PARSER.declareString(HasRequiredArguments::setA, new ParseField("a"));
-            PARSER.declareString(HasRequiredArguments::setB, new ParseField("b"));
-            PARSER.declareString(HasRequiredArguments::setC, new ParseField("c"));
-            PARSER.declareBoolean(HasRequiredArguments::setD, new ParseField("d"));
+        /*
+         * It is normal just to declare a single PARSER but we use a couple of different parsers for testing so we have all of these. Don't
+         * this this style is normal just because it is in the test.
+         */
+        public static final ConstructingObjectParser<HasCtorArguments, ParseFieldMatcherSupplier> PARSER = buildParser(true, true);
+        public static final ConstructingObjectParser<HasCtorArguments, ParseFieldMatcherSupplier> PARSER_VEGETABLE_OPTIONAL = buildParser(
+                true, false);
+        public static final ConstructingObjectParser<HasCtorArguments, ParseFieldMatcherSupplier> PARSER_ALL_OPTIONAL = buildParser(false,
+                false);
+        public static final List<ConstructingObjectParser<HasCtorArguments, ParseFieldMatcherSupplier>> ALL_PARSERS = unmodifiableList(
+                Arrays.asList(PARSER, PARSER_VEGETABLE_OPTIONAL, PARSER_ALL_OPTIONAL));
+
+        private static ConstructingObjectParser<HasCtorArguments, ParseFieldMatcherSupplier> buildParser(boolean animalRequired,
+                boolean vegetableRequired) {
+            ConstructingObjectParser<HasCtorArguments, ParseFieldMatcherSupplier> parser = new ConstructingObjectParser<>(
+                    "has_required_arguments", a -> new HasCtorArguments((String) a[0], (Integer) a[1]));
+            parser.declareString(animalRequired ? constructorArg() : optionalConstructorArg(), new ParseField("animal"));
+            parser.declareInt(vegetableRequired ? constructorArg() : optionalConstructorArg(), new ParseField("vegetable"));
+            parser.declareInt(HasCtorArguments::setMineral, new ParseField("mineral"));
+            parser.declareInt(HasCtorArguments::setFruit, new ParseField("fruit"));
+            parser.declareString(HasCtorArguments::setA, new ParseField("a"));
+            parser.declareString(HasCtorArguments::setB, new ParseField("b"));
+            parser.declareString(HasCtorArguments::setC, new ParseField("c"));
+            parser.declareBoolean(HasCtorArguments::setD, new ParseField("d"));
+            return parser;
         }
     }
 }

From 3dd3ed4905485142a2aac7b4a8751a9c279b9648 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Wed, 8 Jun 2016 16:18:42 +0200
Subject: [PATCH 0720/1311] ingest: Upgrade geoip processor's dependencies and
 database files

The database files have been doubled in size compared to the previous files being used.
For this reason the database files are now gzip compressed, which required using
`GZIPInputStream` when loading database files.
---
 docs/plugins/ingest-geoip.asciidoc            |  6 +--
 plugins/ingest-geoip/build.gradle             |  8 ++--
 .../licenses/geoip2-2.6.0.jar.sha1            |  1 -
 .../licenses/geoip2-2.7.0.jar.sha1            |  1 +
 .../licenses/maxmind-db-1.2.0.jar.sha1        |  1 -
 .../licenses/maxmind-db-1.2.1.jar.sha1        |  1 +
 .../ingest/geoip/GeoIpProcessor.java          |  2 +-
 .../ingest/geoip/IngestGeoIpPlugin.java       |  5 ++-
 .../geoip/GeoIpProcessorFactoryTests.java     | 14 +++----
 .../ingest/geoip/GeoIpProcessorTests.java     | 38 +++++++++++--------
 .../test/ingest_geoip/20_geoip_processor.yaml |  2 +-
 .../test/ingest/20_combine_processors.yaml    |  2 +-
 12 files changed, 44 insertions(+), 37 deletions(-)
 delete mode 100644 plugins/ingest-geoip/licenses/geoip2-2.6.0.jar.sha1
 create mode 100644 plugins/ingest-geoip/licenses/geoip2-2.7.0.jar.sha1
 delete mode 100644 plugins/ingest-geoip/licenses/maxmind-db-1.2.0.jar.sha1
 create mode 100644 plugins/ingest-geoip/licenses/maxmind-db-1.2.1.jar.sha1

diff --git a/docs/plugins/ingest-geoip.asciidoc b/docs/plugins/ingest-geoip.asciidoc
index 1a555090fdc..1ff763c1d6f 100644
--- a/docs/plugins/ingest-geoip.asciidoc
+++ b/docs/plugins/ingest-geoip.asciidoc
@@ -8,8 +8,8 @@ The ingest-geoip plugin ships by default with the GeoLite2 City and GeoLite2 Cou
 under the CCA-ShareAlike 3.0 license. For more details see, http://dev.maxmind.com/geoip/geoip2/geolite2/
 
 The GeoIP processor can run with other geoip2 databases from Maxmind. The files must be copied into the geoip config directory,
-and the `database_file` option should be used to specify the filename of the custom database. The geoip config directory
-is located at `$ES_HOME/config/ingest/geoip` and holds the shipped databases too.
+and the `database_file` option should be used to specify the filename of the custom database. Custom database files must be compressed
+with gzip. The geoip config directory is located at `$ES_HOME/config/ingest/geoip` and holds the shipped databases too.
 
 [[geoip-options]]
 .Geoip options
@@ -18,7 +18,7 @@ is located at `$ES_HOME/config/ingest/geoip` and holds the shipped databases too
 | Name                   | Required  | Default                                                                            | Description
 | `field`                | yes       | -                                                                                  | The field to get the ip address from for the geographical lookup.
 | `target_field`         | no        | geoip                                                                              | The field that will hold the geographical information looked up from the Maxmind database.
-| `database_file`        | no        | GeoLite2-City.mmdb                                                                 | The database filename in the geoip config directory. The ingest-geoip plugin ships with the GeoLite2-City.mmdb and GeoLite2-Country.mmdb files.
+| `database_file`        | no        | GeoLite2-City.mmdb                                                                 | The database filename in the geoip config directory. The ingest-geoip plugin ships with the GeoLite2-City.mmdb.gz and GeoLite2-Country.mmdb.gz files.
 | `properties`           | no        | [`continent_name`, `country_iso_code`, `region_name`, `city_name`, `location`] *   | Controls what properties are added to the `target_field` based on the geoip lookup.
 |======
 
diff --git a/plugins/ingest-geoip/build.gradle b/plugins/ingest-geoip/build.gradle
index eee032d6d16..d9e90a61d40 100644
--- a/plugins/ingest-geoip/build.gradle
+++ b/plugins/ingest-geoip/build.gradle
@@ -23,19 +23,19 @@ esplugin {
 }
 
 dependencies {
-  compile ('com.maxmind.geoip2:geoip2:2.6.0')
+  compile ('com.maxmind.geoip2:geoip2:2.7.0')
   // geoip2 dependencies:
   compile('com.fasterxml.jackson.core:jackson-annotations:2.7.1')
   compile('com.fasterxml.jackson.core:jackson-databind:2.7.1')
-  compile('com.maxmind.db:maxmind-db:1.2.0')
+  compile('com.maxmind.db:maxmind-db:1.2.1')
 
-  testCompile 'org.elasticsearch:geolite2-databases:20151029'
+  testCompile 'org.elasticsearch:geolite2-databases:20160608'
 }
 
 task copyDefaultGeoIp2DatabaseFiles(type: Copy) {
   from { zipTree(configurations.testCompile.files.find { it.name.contains('geolite2-databases')}) }
   into "${project.buildDir}/ingest-geoip"
-  include "*.mmdb"
+  include "*.mmdb.gz"
 }
 
 project.bundlePlugin.dependsOn(copyDefaultGeoIp2DatabaseFiles)
diff --git a/plugins/ingest-geoip/licenses/geoip2-2.6.0.jar.sha1 b/plugins/ingest-geoip/licenses/geoip2-2.6.0.jar.sha1
deleted file mode 100644
index ba5a0242ec8..00000000000
--- a/plugins/ingest-geoip/licenses/geoip2-2.6.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2574c8b878f1cd39709559f1b96f1b5f0cdd69d3
\ No newline at end of file
diff --git a/plugins/ingest-geoip/licenses/geoip2-2.7.0.jar.sha1 b/plugins/ingest-geoip/licenses/geoip2-2.7.0.jar.sha1
new file mode 100644
index 00000000000..2015e311d60
--- /dev/null
+++ b/plugins/ingest-geoip/licenses/geoip2-2.7.0.jar.sha1
@@ -0,0 +1 @@
+2010d922191f5801939b462a5703ab79a7829626
\ No newline at end of file
diff --git a/plugins/ingest-geoip/licenses/maxmind-db-1.2.0.jar.sha1 b/plugins/ingest-geoip/licenses/maxmind-db-1.2.0.jar.sha1
deleted file mode 100644
index ae48403db39..00000000000
--- a/plugins/ingest-geoip/licenses/maxmind-db-1.2.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b842823f24555f5d26608fef8122898365b3cd63
\ No newline at end of file
diff --git a/plugins/ingest-geoip/licenses/maxmind-db-1.2.1.jar.sha1 b/plugins/ingest-geoip/licenses/maxmind-db-1.2.1.jar.sha1
new file mode 100644
index 00000000000..2358241fdf6
--- /dev/null
+++ b/plugins/ingest-geoip/licenses/maxmind-db-1.2.1.jar.sha1
@@ -0,0 +1 @@
+64b6b6a8c162fc9b0004fcdf9641cf1b408ffa33
\ No newline at end of file
diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
index c524bf15a3d..9d606419e57 100644
--- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
+++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
@@ -233,7 +233,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
         public GeoIpProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
             String ipField = readStringProperty(TYPE, processorTag, config, "field");
             String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "geoip");
-            String databaseFile = readStringProperty(TYPE, processorTag, config, "database_file", "GeoLite2-City.mmdb");
+            String databaseFile = readStringProperty(TYPE, processorTag, config, "database_file", "GeoLite2-City.mmdb.gz");
             List<String> propertyNames = readOptionalList(TYPE, processorTag, config, "properties");
 
             DatabaseReader databaseReader = databaseReaders.get(databaseFile);
diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
index 7c57349f4b6..2c7edd7ee96 100644
--- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
+++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
@@ -34,6 +34,7 @@ import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.stream.Stream;
+import java.util.zip.GZIPInputStream;
 
 public class IngestGeoIpPlugin extends Plugin {
 
@@ -60,13 +61,13 @@ public class IngestGeoIpPlugin extends Plugin {
 
         Map<String, DatabaseReader> databaseReaders = new HashMap<>();
         try (Stream<Path> databaseFiles = Files.list(geoIpConfigDirectory)) {
-            PathMatcher pathMatcher = geoIpConfigDirectory.getFileSystem().getPathMatcher("glob:**.mmdb");
+            PathMatcher pathMatcher = geoIpConfigDirectory.getFileSystem().getPathMatcher("glob:**.mmdb.gz");
             // Use iterator instead of forEach otherwise IOException needs to be caught twice...
             Iterator<Path> iterator = databaseFiles.iterator();
             while (iterator.hasNext()) {
                 Path databasePath = iterator.next();
                 if (Files.isRegularFile(databasePath) && pathMatcher.matches(databasePath)) {
-                    try (InputStream inputStream = Files.newInputStream(databasePath, StandardOpenOption.READ)) {
+                    try (InputStream inputStream = new GZIPInputStream(Files.newInputStream(databasePath, StandardOpenOption.READ))) {
                         databaseReaders.put(databasePath.getFileName().toString(), new DatabaseReader.Builder(inputStream).build());
                     }
                 }
diff --git a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
index 0840b10b004..25d8f1f6f5f 100644
--- a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
+++ b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
@@ -54,8 +54,8 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Path configDir = createTempDir();
         Path geoIpConfigDir = configDir.resolve("ingest-geoip");
         Files.createDirectories(geoIpConfigDir);
-        Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-City.mmdb")), geoIpConfigDir.resolve("GeoLite2-City.mmdb"));
-        Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-Country.mmdb")), geoIpConfigDir.resolve("GeoLite2-Country.mmdb"));
+        Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-City.mmdb.gz")), geoIpConfigDir.resolve("GeoLite2-City.mmdb.gz"));
+        Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-Country.mmdb.gz")), geoIpConfigDir.resolve("GeoLite2-Country.mmdb.gz"));
         databaseReaders = IngestGeoIpPlugin.loadDatabaseReaders(geoIpConfigDir);
     }
 
@@ -89,7 +89,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-        config.put("database_file", "GeoLite2-Country.mmdb");
+        config.put("database_file", "GeoLite2-Country.mmdb.gz");
 
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
@@ -116,7 +116,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders);
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-        config.put("database_file", "GeoLite2-Country.mmdb");
+        config.put("database_file", "GeoLite2-Country.mmdb.gz");
         GeoIpProcessor processor = factory.create(config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("geoip"));
@@ -128,7 +128,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders);
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-        config.put("database_file", "GeoLite2-Country.mmdb");
+        config.put("database_file", "GeoLite2-Country.mmdb.gz");
         EnumSet<GeoIpProcessor.Property> cityOnlyProperties = EnumSet.complementOf(GeoIpProcessor.Property.ALL_COUNTRY_PROPERTIES);
         String cityProperty = RandomPicks.randomFrom(Randomness.get(), cityOnlyProperties).toString();
         config.put("properties", Collections.singletonList(cityProperty));
@@ -145,12 +145,12 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-        config.put("database_file", "does-not-exist.mmdb");
+        config.put("database_file", "does-not-exist.mmdb.gz");
         try {
             factory.create(config);
             fail("Exception expected");
         } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), equalTo("[database_file] database file [does-not-exist.mmdb] doesn't exist"));
+            assertThat(e.getMessage(), equalTo("[database_file] database file [does-not-exist.mmdb.gz] doesn't exist"));
         }
     }
 
diff --git a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
index f646895d7cf..a64a42fa95e 100644
--- a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
+++ b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
@@ -24,10 +24,12 @@ import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.ingest.core.IngestDocument;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
 import java.io.InputStream;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.zip.GZIPInputStream;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -35,33 +37,33 @@ import static org.hamcrest.Matchers.equalTo;
 public class GeoIpProcessorTests extends ESTestCase {
 
     public void testCity() throws Exception {
-        InputStream database = GeoIpProcessor.class.getResourceAsStream("/GeoLite2-City.mmdb");
+        InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
         GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field", new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class));
 
         Map<String, Object> document = new HashMap<>();
-        document.put("source_field", "82.170.213.79");
+        document.put("source_field", "8.8.8.8");
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
         processor.execute(ingestDocument);
 
-        assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("82.170.213.79"));
+        assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo("8.8.8.8"));
         @SuppressWarnings("unchecked")
         Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
         assertThat(geoData.size(), equalTo(8));
-        assertThat(geoData.get("ip"), equalTo("82.170.213.79"));
-        assertThat(geoData.get("country_iso_code"), equalTo("NL"));
-        assertThat(geoData.get("country_name"), equalTo("Netherlands"));
-        assertThat(geoData.get("continent_name"), equalTo("Europe"));
-        assertThat(geoData.get("region_name"), equalTo("North Holland"));
-        assertThat(geoData.get("city_name"), equalTo("Amsterdam"));
-        assertThat(geoData.get("timezone"), equalTo("Europe/Amsterdam"));
+        assertThat(geoData.get("ip"), equalTo("8.8.8.8"));
+        assertThat(geoData.get("country_iso_code"), equalTo("US"));
+        assertThat(geoData.get("country_name"), equalTo("United States"));
+        assertThat(geoData.get("continent_name"), equalTo("North America"));
+        assertThat(geoData.get("region_name"), equalTo("California"));
+        assertThat(geoData.get("city_name"), equalTo("Mountain View"));
+        assertThat(geoData.get("timezone"), equalTo("America/Los_Angeles"));
         Map<String, Object> location = new HashMap<>();
-        location.put("lat", 52.374d);
-        location.put("lon", 4.8897d);
+        location.put("lat", 37.386d);
+        location.put("lon", -122.0838d);
         assertThat(geoData.get("location"), equalTo(location));
     }
 
     public void testCountry() throws Exception {
-        InputStream database = GeoIpProcessor.class.getResourceAsStream("/GeoLite2-Country.mmdb");
+        InputStream database = getDatabaseFileInputStream("/GeoLite2-Country.mmdb.gz");
         GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field", new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class));
 
         Map<String, Object> document = new HashMap<>();
@@ -80,11 +82,11 @@ public class GeoIpProcessorTests extends ESTestCase {
     }
 
     public void testAddressIsNotInTheDatabase() throws Exception {
-        InputStream database = GeoIpProcessor.class.getResourceAsStream("/GeoLite2-City.mmdb");
+        InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
         GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field", new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class));
 
         Map<String, Object> document = new HashMap<>();
-        document.put("source_field", "202.45.11.11");
+        document.put("source_field", "127.0.0.1");
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
         processor.execute(ingestDocument);
         @SuppressWarnings("unchecked")
@@ -94,7 +96,7 @@ public class GeoIpProcessorTests extends ESTestCase {
 
     /** Don't silently do DNS lookups or anything trappy on bogus data */
     public void testInvalid() throws Exception {
-        InputStream database = GeoIpProcessor.class.getResourceAsStream("/GeoLite2-City.mmdb");
+        InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
         GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field", new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class));
 
         Map<String, Object> document = new HashMap<>();
@@ -109,4 +111,8 @@ public class GeoIpProcessorTests extends ESTestCase {
         }
     }
 
+    static InputStream getDatabaseFileInputStream(String path) throws IOException {
+        return new GZIPInputStream(GeoIpProcessor.class.getResourceAsStream(path));
+    }
+
 }
diff --git a/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yaml b/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yaml
index 5e8a3e7c1ff..33e9ec1ca46 100644
--- a/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yaml
+++ b/plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yaml
@@ -98,7 +98,7 @@
               {
                 "geoip" : {
                   "field" : "field1",
-                  "database_file" : "GeoLite2-Country.mmdb"
+                  "database_file" : "GeoLite2-Country.mmdb.gz"
                 }
               }
             ]
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/20_combine_processors.yaml b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/20_combine_processors.yaml
index 45bda3146a6..7ccaa64b9f4 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/20_combine_processors.yaml
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/20_combine_processors.yaml
@@ -67,7 +67,7 @@
   - match: { _source.httpversion: "1.1" }
   - match: { _source.timestamp: "2014-09-08T02:54:42.000Z" }
   - match: { _source.geoip.continent_name: "North America" }
-  - match: { _source.geoip.city_name: "Charlotte" }
+  - match: { _source.geoip.city_name: "Fayetteville" }
   - match: { _source.geoip.country_iso_code: "US" }
   - match: { _source.geoip.region_name: "North Carolina" }
 

From c637fea84b7b52e60edd851547a2ad88897501f7 Mon Sep 17 00:00:00 2001
From: Lee Hinman <lee@writequit.org>
Date: Tue, 7 Jun 2016 15:06:20 -0600
Subject: [PATCH 0721/1311] Change the default of `include_global_state` from
 true to false for restores

This changes the default value to be false *only* for restore operations.

Resolves #18569
---
 .../restore/RestoreSnapshotRequest.java       |  2 +-
 .../migration/migrate_5_0/settings.asciidoc   |  6 ++++
 docs/reference/modules/snapshots.asciidoc     | 33 +++++++++++--------
 3 files changed, 27 insertions(+), 14 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
index bf4e4f148fc..1a41a776c73 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/RestoreSnapshotRequest.java
@@ -57,7 +57,7 @@ public class RestoreSnapshotRequest extends MasterNodeRequest<RestoreSnapshotReq
     private String renamePattern;
     private String renameReplacement;
     private boolean waitForCompletion;
-    private boolean includeGlobalState = true;
+    private boolean includeGlobalState = false;
     private boolean partial = false;
     private boolean includeAliases = true;
     private Settings settings = EMPTY_SETTINGS;
diff --git a/docs/reference/migration/migrate_5_0/settings.asciidoc b/docs/reference/migration/migrate_5_0/settings.asciidoc
index 5702d51fc12..ed405428ab6 100644
--- a/docs/reference/migration/migrate_5_0/settings.asciidoc
+++ b/docs/reference/migration/migrate_5_0/settings.asciidoc
@@ -277,3 +277,9 @@ should be used instead.
 
 The setting `bootstrap.mlockall` has been renamed to
 `bootstrap.memory_lock`.
+
+==== Snapshot settings
+
+The default setting `include_global_state` for restoring snapshots has been
+changed from `true` to `false`. It has not been changed for taking snapshots and
+still defaults to `true` in that case.
diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc
index 0f87744d317..2d8106e981c 100644
--- a/docs/reference/modules/snapshots.asciidoc
+++ b/docs/reference/modules/snapshots.asciidoc
@@ -323,13 +323,17 @@ POST /_snapshot/my_backup/snapshot_1/_restore
 -----------------------------------
 // CONSOLE
 
-By default, all indices in the snapshot as well as cluster state are restored. It's possible to select indices that
-should be restored as well as prevent global cluster state from being restored by using `indices` and
-`include_global_state` options in the restore request body. The list of indices supports
-<<search-multi-index-type,multi index syntax>>. The `rename_pattern` and `rename_replacement` options can be also used to
-rename index on restore using regular expression that supports referencing the original text as explained
+By default, all indices in the snapshot are restored, and the cluster state is
+*not* restored. It's possible to select indices that should be restored as well
+as to allow the global cluster state from being restored by using `indices` and
+`include_global_state` options in the restore request body. The list of indices
+supports <<search-multi-index-type,multi index syntax>>. The `rename_pattern`
+and `rename_replacement` options can be also used to rename index on restore
+using regular expression that supports referencing the original text as
+explained
 http://docs.oracle.com/javase/6/docs/api/java/util/regex/Matcher.html#appendReplacement(java.lang.StringBuffer,%20java.lang.String)[here].
-Set `include_aliases` to `false` to prevent aliases from being restored together with associated indices
+Set `include_aliases` to `false` to prevent aliases from being restored together
+with associated indices
 
 [source,js]
 -----------------------------------
@@ -337,19 +341,22 @@ POST /_snapshot/my_backup/snapshot_1/_restore
 {
   "indices": "index_1,index_2",
   "ignore_unavailable": "true",
-  "include_global_state": false,
+  "include_global_state": true,
   "rename_pattern": "index_(.+)",
   "rename_replacement": "restored_index_$1"
 }
 -----------------------------------
 // CONSOLE
 
-The restore operation can be performed on a functioning cluster. However, an existing index can be only restored if it's
-<<indices-open-close,closed>> and has the same number of shards as the index in the snapshot.
-The restore operation automatically opens restored indices if they were closed and creates new indices if they
-didn't exist in the cluster. If cluster state is restored, the restored templates that don't currently exist in the
-cluster are added and existing templates with the same name are replaced by the restored templates. The restored
-persistent settings are added to the existing persistent settings.
+The restore operation can be performed on a functioning cluster. However, an
+existing index can be only restored if it's <<indices-open-close,closed>> and
+has the same number of shards as the index in the snapshot. The restore
+operation automatically opens restored indices if they were closed and creates
+new indices if they didn't exist in the cluster. If cluster state is restored
+with `include_cluster_state` (defaults to `false`), the restored templates that
+don't currently exist in the cluster are added and existing templates with the
+same name are replaced by the restored templates. The restored persistent
+settings are added to the existing persistent settings.
 
 [float]
 ==== Partial restore

From 4b211579065487120f33b1b23cd5a91419993df0 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 6 Jun 2016 13:18:32 -0400
Subject: [PATCH 0722/1311] Remove setRefresh

It has been replaced with `setRefreshPolicy` which has support for
waiting until refresh with `setRefreshPolicy(WAIT_FOR)`.

Related to #1063
---
 .../resources/checkstyle_suppressions.xml     |  1 -
 .../action/support/WriteRequestBuilder.java   | 15 ++----
 .../document/DocumentActionsIT.java           |  6 ++-
 .../gateway/GatewayIndexStateIT.java          | 11 +++--
 .../mapper/geo/GeoPointFieldMapperTests.java  |  7 +--
 .../multifield/MultiFieldsIntegrationIT.java  | 13 ++---
 ...gFieldMapperPositionIncrementGapTests.java | 10 ++--
 .../index/shard/IndexShardTests.java          | 48 ++++++++++---------
 .../termvectors/TermVectorsServiceTests.java  |  3 +-
 .../indices/IndicesOptionsIntegrationIT.java  |  5 +-
 .../indices/IndicesServiceTests.java          |  3 +-
 .../template/SimpleIndexTemplateIT.java       |  5 +-
 .../org/elasticsearch/mget/SimpleMgetIT.java  | 12 +++--
 .../search/SearchServiceTests.java            |  9 ++--
 .../elasticsearch/search/SearchTimeoutIT.java |  3 +-
 .../aggregations/bucket/ReverseNestedIT.java  |  3 +-
 .../functionscore/DecayFunctionScoreIT.java   | 14 ++++--
 .../search/geo/GeoBoundingBoxIT.java          | 17 +++----
 .../search/geo/GeoShapeQueryTests.java        | 28 +++++------
 .../search/innerhits/InnerHitsIT.java         |  3 +-
 .../search/nested/SimpleNestedIT.java         |  3 +-
 .../search/query/SearchQueryIT.java           | 11 +++--
 .../search/scroll/SearchScrollIT.java         |  3 +-
 .../search/simple/SimpleSearchIT.java         |  6 ++-
 .../suggest/CompletionSuggestSearchIT.java    | 18 +++----
 .../suggest/ContextSuggestSearch2xIT.java     |  3 +-
 .../suggest/CustomSuggesterSearchIT.java      |  3 +-
 .../similarity/SimilarityIT.java              |  3 +-
 .../timestamp/SimpleTimestampIT.java          |  8 ++--
 .../org/elasticsearch/ttl/SimpleTTLIT.java    |  7 +--
 .../org/elasticsearch/update/UpdateIT.java    |  5 +-
 .../migration/migrate_5_0/java.asciidoc       | 11 +++--
 .../expression/MoreExpressionTests.java       | 21 ++++----
 .../messy/tests/SearchFieldsTests.java        |  7 +--
 .../script/groovy/GroovyScriptTests.java      |  3 +-
 .../UpdateByQueryWhileModifyingTests.java     |  3 +-
 36 files changed, 186 insertions(+), 145 deletions(-)

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 9bd0755ffcc..05ca294a9f0 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -958,7 +958,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]simple[/\\]SimpleMapperTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]source[/\\]DefaultSourceMappingTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]string[/\\]SimpleStringMappingTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]string[/\\]StringFieldMapperPositionIncrementGapTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]timestamp[/\\]TimestampMappingTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ttl[/\\]TTLMappingTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]typelevels[/\\]ParseDocumentTypeLevelsTests.java" checks="LineLength" />
diff --git a/core/src/main/java/org/elasticsearch/action/support/WriteRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/support/WriteRequestBuilder.java
index a87fd043452..eaaf90fd53b 100644
--- a/core/src/main/java/org/elasticsearch/action/support/WriteRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/support/WriteRequestBuilder.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.action.support;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 
 public interface WriteRequestBuilder<B extends WriteRequestBuilder<B>> {
@@ -36,15 +35,11 @@ public interface WriteRequestBuilder<B extends WriteRequestBuilder<B>> {
     }
 
     /**
-     * If set to true then this request will force an immediate refresh. Backwards compatibility layer for Elasticsearch's old
-     * {@code setRefresh} calls.
-     *
-     * @deprecated use {@link #setRefreshPolicy(RefreshPolicy)} with {@link RefreshPolicy#IMMEDIATE} or {@link RefreshPolicy#NONE} instead.
-     *             Will be removed in 6.0.
+     * Parse the refresh policy from a string, only modifying it if the string is non null. Convenient to use with request parsing.
      */
-    @Deprecated
-    default B setRefresh(boolean refresh) {
-        assert Version.CURRENT.major < 6 : "Remove setRefresh(boolean) in 6.0";
-        return setRefreshPolicy(refresh ? RefreshPolicy.IMMEDIATE : RefreshPolicy.NONE);
+    @SuppressWarnings("unchecked")
+    default B setRefreshPolicy(String refreshPolicy) {
+        request().setRefreshPolicy(refreshPolicy);
+        return (B) this;
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java b/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java
index 6f002e8404d..065128af918 100644
--- a/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java
+++ b/core/src/test/java/org/elasticsearch/document/DocumentActionsIT.java
@@ -28,6 +28,7 @@ import org.elasticsearch.action.delete.DeleteResponse;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -45,7 +46,7 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
 
 /**
- *
+ * Integration test for document action like index, bulk, and get. It has a very long history: it was in the second commit of Elasticsearch.
  */
 public class DocumentActionsIT extends ESIntegTestCase {
     protected void createIndex() {
@@ -62,7 +63,8 @@ public class DocumentActionsIT extends ESIntegTestCase {
         logger.info("Running Cluster Health");
         ensureGreen();
         logger.info("Indexing [type1/1]");
-        IndexResponse indexResponse = client().prepareIndex().setIndex("test").setType("type1").setId("1").setSource(source("1", "test")).setRefresh(true).execute().actionGet();
+        IndexResponse indexResponse = client().prepareIndex().setIndex("test").setType("type1").setId("1").setSource(source("1", "test"))
+                .setRefreshPolicy(RefreshPolicy.IMMEDIATE).get();
         assertThat(indexResponse.getIndex(), equalTo(getConcreteIndexName()));
         assertThat(indexResponse.getId(), equalTo("1"));
         assertThat(indexResponse.getType(), equalTo("type1"));
diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
index ebf1085cfbe..d5dee3c1bdc 100644
--- a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
+++ b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
@@ -53,6 +53,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.List;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -245,7 +246,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
         internalCluster().startNodesAsync(2).get();
 
         logger.info("--> indexing a simple document");
-        client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefresh(true).execute().actionGet();
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get();
 
         logger.info("--> waiting for green status");
         ClusterHealthResponse health = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus()
@@ -285,7 +286,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
         final String node_1 = internalCluster().startNodesAsync(2).get().get(0);
 
         logger.info("--> indexing a simple document");
-        client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefresh(true).execute().actionGet();
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get();
 
         logger.info("--> waiting for green status");
         ensureGreen();
@@ -408,7 +409,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
         logger.info("--> starting one node");
         internalCluster().startNode();
         logger.info("--> indexing a simple document");
-        client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefresh(true).execute().actionGet();
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get();
         logger.info("--> waiting for green status");
         if (usually()) {
             ensureYellow();
@@ -477,7 +478,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
                 "    }\n" +
                 "  }}").get();
         logger.info("--> indexing a simple document");
-        client().prepareIndex("test", "type1", "1").setSource("field1", "value one").setRefresh(true).execute().actionGet();
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value one").setRefreshPolicy(IMMEDIATE).get();
         logger.info("--> waiting for green status");
         if (usually()) {
             ensureYellow();
@@ -521,7 +522,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
     public void testArchiveBrokenClusterSettings() throws Exception {
         logger.info("--> starting one node");
         internalCluster().startNode();
-        client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefresh(true).execute().actionGet();
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get();
         logger.info("--> waiting for green status");
         if (usually()) {
             ensureYellow();
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java
index c7142be3c6c..a1fdb7ec60f 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java
@@ -47,6 +47,7 @@ import java.util.Map;
 import java.lang.NumberFormatException;
 
 import static org.apache.lucene.spatial.util.GeoEncodingUtils.mortonHash;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.geo.GeoHashUtils.stringEncode;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
@@ -812,7 +813,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
         mappingRequest.execute().actionGet();
         client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet();
         client().prepareIndex("test", "pin", "1").setSource(jsonBuilder().startObject().startObject("location").field("lat", 40.7143528)
-                .field("lon", -74.0059731).endObject().endObject()).setRefresh(true).execute().actionGet();
+                .field("lon", -74.0059731).endObject().endObject()).setRefreshPolicy(IMMEDIATE).get();
 
         // match all search with geohash field
         SearchResponse searchResponse = client().prepareSearch().addField("location.geohash").setQuery(matchAllQuery()).execute().actionGet();
@@ -837,7 +838,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
         mappingRequest.execute().actionGet();
         client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet();
         client().prepareIndex("test", "pin", "1").setSource(jsonBuilder().startObject().startObject("location").field("lat", 40.7143528)
-                .field("lon", -74.0059731).endObject().endObject()).setRefresh(true).execute().actionGet();
+                .field("lon", -74.0059731).endObject().endObject()).setRefreshPolicy(IMMEDIATE).get();
 
         // match all search with geohash field (includes prefixes)
         SearchResponse searchResponse = client().prepareSearch().addField("location.geohash").setQuery(matchAllQuery()).execute().actionGet();
@@ -867,7 +868,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
         for (int i=0; i<numDocs; ++i) {
             final GeoPoint pt = RandomGeoGenerator.randomPoint(random());
             client().prepareIndex("test", "pin").setSource(jsonBuilder().startObject().startObject("location").field("lat", pt.lat())
-                .field("lon", pt.lon()).endObject().endObject()).setRefresh(true).execute().actionGet();
+                .field("lon", pt.lon()).endObject().endObject()).setRefreshPolicy(IMMEDIATE).get();
         }
 
         // query by geohash subfield
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldsIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldsIntegrationIT.java
index a1f6929fade..8e10ba9f007 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldsIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/multifield/MultiFieldsIntegrationIT.java
@@ -32,6 +32,7 @@ import org.elasticsearch.test.ESIntegTestCase;
 import java.io.IOException;
 import java.util.Map;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery;
 import static org.elasticsearch.index.query.QueryBuilders.geoDistanceQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
@@ -61,7 +62,7 @@ public class MultiFieldsIntegrationIT extends ESIntegTestCase {
 
         client().prepareIndex("my-index", "my-type", "1")
                 .setSource("title", "Multi fields")
-                .setRefresh(true)
+                .setRefreshPolicy(IMMEDIATE)
                 .get();
 
         SearchResponse searchResponse = client().prepareSearch("my-index")
@@ -92,7 +93,7 @@ public class MultiFieldsIntegrationIT extends ESIntegTestCase {
 
         client().prepareIndex("my-index", "my-type", "1")
                 .setSource("title", "Multi fields")
-                .setRefresh(true)
+                .setRefreshPolicy(IMMEDIATE)
                 .get();
 
         searchResponse = client().prepareSearch("my-index")
@@ -122,7 +123,7 @@ public class MultiFieldsIntegrationIT extends ESIntegTestCase {
         assertThat(bField.get("type").toString(), equalTo("keyword"));
 
         GeoPoint point = new GeoPoint(51, 19);
-        client().prepareIndex("my-index", "my-type", "1").setSource("a", point.toString()).setRefresh(true).get();
+        client().prepareIndex("my-index", "my-type", "1").setSource("a", point.toString()).setRefreshPolicy(IMMEDIATE).get();
         SearchResponse countResponse = client().prepareSearch("my-index").setSize(0)
                 .setQuery(constantScoreQuery(geoDistanceQuery("a").point(51, 19).distance(50, DistanceUnit.KILOMETERS)))
                 .get();
@@ -162,7 +163,7 @@ public class MultiFieldsIntegrationIT extends ESIntegTestCase {
         assertThat(bField.size(), equalTo(1));
         assertThat(bField.get("type").toString(), equalTo("keyword"));
 
-        client().prepareIndex("my-index", "my-type", "1").setSource("a", "my tokens").setRefresh(true).get();
+        client().prepareIndex("my-index", "my-type", "1").setSource("a", "my tokens").setRefreshPolicy(IMMEDIATE).get();
         SearchResponse countResponse = client().prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", "my tokens")).get();
         assertThat(countResponse.getHits().totalHits(), equalTo(1L));
     }
@@ -186,7 +187,7 @@ public class MultiFieldsIntegrationIT extends ESIntegTestCase {
         assertThat(bField.size(), equalTo(1));
         assertThat(bField.get("type").toString(), equalTo("keyword"));
 
-        client().prepareIndex("my-index", "my-type", "1").setSource("a", "complete me").setRefresh(true).get();
+        client().prepareIndex("my-index", "my-type", "1").setSource("a", "complete me").setRefreshPolicy(IMMEDIATE).get();
         SearchResponse countResponse = client().prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", "complete me")).get();
         assertThat(countResponse.getHits().totalHits(), equalTo(1L));
     }
@@ -210,7 +211,7 @@ public class MultiFieldsIntegrationIT extends ESIntegTestCase {
         assertThat(bField.size(), equalTo(1));
         assertThat(bField.get("type").toString(), equalTo("keyword"));
 
-        client().prepareIndex("my-index", "my-type", "1").setSource("a", "127.0.0.1").setRefresh(true).get();
+        client().prepareIndex("my-index", "my-type", "1").setSource("a", "127.0.0.1").setRefreshPolicy(IMMEDIATE).get();
         SearchResponse countResponse = client().prepareSearch("my-index").setSize(0).setQuery(matchQuery("a.b", "127.0.0.1")).get();
         assertThat(countResponse.getHits().totalHits(), equalTo(1L));
     }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/string/StringFieldMapperPositionIncrementGapTests.java b/core/src/test/java/org/elasticsearch/index/mapper/string/StringFieldMapperPositionIncrementGapTests.java
index 91a926805bf..88f815611e2 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/string/StringFieldMapperPositionIncrementGapTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/string/StringFieldMapperPositionIncrementGapTests.java
@@ -35,6 +35,7 @@ import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.index.query.QueryBuilders.matchPhraseQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.containsString;
@@ -160,19 +161,22 @@ public class StringFieldMapperPositionIncrementGapTests extends ESSingleNodeTest
     }
 
     private static void testGap(Client client, String indexName, String type, int positionIncrementGap) throws IOException {
-        client.prepareIndex(indexName, type, "position_gap_test").setSource("string", Arrays.asList("one", "two three")).setRefresh(true).get();
+        client.prepareIndex(indexName, type, "position_gap_test").setSource("string", Arrays.asList("one", "two three"))
+                .setRefreshPolicy(IMMEDIATE).get();
 
         // Baseline - phrase query finds matches in the same field value
         assertHitCount(client.prepareSearch(indexName).setQuery(matchPhraseQuery("string", "two three")).get(), 1);
 
         if (positionIncrementGap > 0) {
             // No match across gaps when slop < position gap
-            assertHitCount(client.prepareSearch(indexName).setQuery(matchPhraseQuery("string", "one two").slop(positionIncrementGap - 1)).get(),
+            assertHitCount(
+                    client.prepareSearch(indexName).setQuery(matchPhraseQuery("string", "one two").slop(positionIncrementGap - 1)).get(),
                     0);
         }
 
         // Match across gaps when slop >= position gap
         assertHitCount(client.prepareSearch(indexName).setQuery(matchPhraseQuery("string", "one two").slop(positionIncrementGap)).get(), 1);
-        assertHitCount(client.prepareSearch(indexName).setQuery(matchPhraseQuery("string", "one two").slop(positionIncrementGap + 1)).get(), 1);
+        assertHitCount(client.prepareSearch(indexName).setQuery(matchPhraseQuery("string", "one two").slop(positionIncrementGap + 1)).get(),
+                1);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index e7a7eb46377..8889ebfb910 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -49,8 +49,6 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.InternalClusterInfoService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.AllocationId;
 import org.elasticsearch.cluster.routing.RestoreSource;
@@ -60,6 +58,7 @@ import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -99,6 +98,7 @@ import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.snapshots.Snapshot;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.test.DummyShardLock;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.FieldMaskingReader;
@@ -128,6 +128,8 @@ import java.util.function.BiConsumer;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED;
@@ -524,7 +526,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
     public void testRecoverIntoLeftover() throws IOException {
         createIndex("test");
         ensureGreen("test");
-        client().prepareIndex("test", "bar", "1").setSource("{}").setRefresh(true).get();
+        client().prepareIndex("test", "bar", "1").setSource("{}").setRefreshPolicy(IMMEDIATE).get();
         client().admin().indices().prepareFlush("test").get();
         SearchResponse response = client().prepareSearch("test").get();
         assertHitCount(response, 1L);
@@ -554,7 +556,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
             .build();
         createIndex("test", idxSettings);
         ensureGreen("test");
-        client().prepareIndex("test", "bar", "1").setSource("{}").setRefresh(true).get();
+        client().prepareIndex("test", "bar", "1").setSource("{}").setRefreshPolicy(IMMEDIATE).get();
         SearchResponse response = client().prepareSearch("test").get();
         assertHitCount(response, 1L);
         client().admin().indices().prepareDelete("test").get();
@@ -601,7 +603,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         logger.info("--> creating an index with data_path [{}]", startDir.toAbsolutePath().toString());
         createIndex(INDEX, sb);
         ensureGreen(INDEX);
-        client().prepareIndex(INDEX, "bar", "1").setSource("{}").setRefresh(true).get();
+        client().prepareIndex(INDEX, "bar", "1").setSource("{}").setRefreshPolicy(IMMEDIATE).get();
 
         SearchResponse resp = client().prepareSearch(INDEX).setQuery(matchAllQuery()).get();
         assertThat("found the hit", resp.getHits().getTotalHits(), equalTo(1L));
@@ -689,7 +691,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
     public void testIndexingOperationsListeners() throws IOException {
         createIndex("test_iol");
         ensureGreen();
-        client().prepareIndex("test_iol", "test", "0").setSource("{\"foo\" : \"bar\"}").setRefresh(true).get();
+        client().prepareIndex("test_iol", "test", "0").setSource("{\"foo\" : \"bar\"}").setRefreshPolicy(IMMEDIATE).get();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         IndexService test = indicesService.indexService(resolveIndex("test_iol"));
         IndexShard shard = test.getShardOrNull(0);
@@ -812,14 +814,14 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndexShard shard = test.getShardOrNull(0);
         assertFalse(shard.shouldFlush());
         client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(133 /* size of the operation + header&footer*/, ByteSizeUnit.BYTES)).build()).get();
-        client().prepareIndex("test", "test", "0").setSource("{}").setRefresh(randomBoolean()).get();
+        client().prepareIndex("test", "test", "0").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
         assertFalse(shard.shouldFlush());
         ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, new ParseContext.Document(), new BytesArray(new byte[]{1}), null);
         Engine.Index index = new Engine.Index(new Term("_uid", "1"), doc);
         shard.index(index);
         assertTrue(shard.shouldFlush());
         assertEquals(2, shard.getEngine().getTranslog().totalOperations());
-        client().prepareIndex("test", "test", "2").setSource("{}").setRefresh(randomBoolean()).get();
+        client().prepareIndex("test", "test", "2").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
         assertBusy(() -> { // this is async
             assertFalse(shard.shouldFlush());
         });
@@ -846,7 +848,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         final IndexShard shard = test.getShardOrNull(0);
         assertFalse(shard.shouldFlush());
         client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(133/* size of the operation + header&footer*/, ByteSizeUnit.BYTES)).build()).get();
-        client().prepareIndex("test", "test", "0").setSource("{}").setRefresh(randomBoolean()).get();
+        client().prepareIndex("test", "test", "0").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
         assertFalse(shard.shouldFlush());
         final AtomicBoolean running = new AtomicBoolean(true);
         final int numThreads = randomIntBetween(2, 4);
@@ -972,7 +974,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndexService test = indicesService.indexService(resolveIndex("test"));
         final IndexShard shard = test.getShardOrNull(0);
         int translogOps = 1;
-        client().prepareIndex("test", "test", "0").setSource("{}").setRefresh(randomBoolean()).get();
+        client().prepareIndex("test", "test", "0").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
         if (randomBoolean()) {
             client().admin().indices().prepareFlush().get();
             translogOps = 0;
@@ -1000,7 +1002,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         IndexService test = indicesService.indexService(resolveIndex("test"));
         final IndexShard shard = test.getShardOrNull(0);
-        client().prepareIndex("test", "test", "0").setSource("{}").setRefresh(randomBoolean()).get();
+        client().prepareIndex("test", "test", "0").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
         if (randomBoolean()) {
             client().admin().indices().prepareFlush().get();
         }
@@ -1030,7 +1032,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndexService test = indicesService.indexService(resolveIndex("test"));
         final IndexShard shard = test.getShardOrNull(0);
 
-        client().prepareIndex("test", "test", "0").setSource("{}").setRefresh(randomBoolean()).get();
+        client().prepareIndex("test", "test", "0").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
         if (randomBoolean()) {
             client().admin().indices().prepareFlush().get();
         }
@@ -1107,8 +1109,8 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndexService test_target = indicesService.indexService(resolveIndex("test_target"));
         final IndexShard test_shard = test.getShardOrNull(0);
 
-        client().prepareIndex("test", "test", "0").setSource("{}").setRefresh(randomBoolean()).get();
-        client().prepareIndex("test_target", "test", "1").setSource("{}").setRefresh(true).get();
+        client().prepareIndex("test", "test", "0").setSource("{}").setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
+        client().prepareIndex("test_target", "test", "1").setSource("{}").setRefreshPolicy(IMMEDIATE).get();
         assertHitCount(client().prepareSearch("test_target").get(), 1);
         assertSearchHits(client().prepareSearch("test_target").get(), "1");
         client().admin().indices().prepareFlush("test").get(); // only flush test
@@ -1165,8 +1167,8 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         IndexService indexService = indicesService.indexService(resolveIndex("test"));
         IndexShard shard = indexService.getShardOrNull(0);
-        client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}").setRefresh(true).get();
-        client().prepareIndex("test", "test", "1").setSource("{\"foobar\" : \"bar\"}").setRefresh(true).get();
+        client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}").setRefreshPolicy(IMMEDIATE).get();
+        client().prepareIndex("test", "test", "1").setSource("{\"foobar\" : \"bar\"}").setRefreshPolicy(IMMEDIATE).get();
 
         Engine.GetResult getResult = shard.get(new Engine.Get(false, new Term(UidFieldMapper.NAME, Uid.createUid("test", "1"))));
         assertTrue(getResult.exists());
@@ -1215,8 +1217,8 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndexService indexService = indicesService.indexService(resolveIndex("test"));
         IndexShard shard = indexService.getShardOrNull(0);
         client().admin().indices().preparePutMapping("test").setType("test").setSource("foo", "type=text,fielddata=true").get();
-        client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}").setRefresh(true).get();
-        client().prepareIndex("test", "test", "1").setSource("{\"foobar\" : \"bar\"}").setRefresh(true).get();
+        client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}").setRefreshPolicy(IMMEDIATE).get();
+        client().prepareIndex("test", "test", "1").setSource("{\"foobar\" : \"bar\"}").setRefreshPolicy(IMMEDIATE).get();
 
         IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {
             @Override
@@ -1266,7 +1268,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndexShard shard = indexService.getShardOrNull(0);
         client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}").get();
         client().prepareDelete("test", "test", "0").get();
-        client().prepareIndex("test", "test", "1").setSource("{\"foo\" : \"bar\"}").setRefresh(true).get();
+        client().prepareIndex("test", "test", "1").setSource("{\"foo\" : \"bar\"}").setRefreshPolicy(IMMEDIATE).get();
 
         IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {};
         shard.close("simon says", false);
@@ -1324,7 +1326,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndexShard shard = indexService.getShardOrNull(0);
         client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}").get();
         client().prepareDelete("test", "test", "0").get();
-        client().prepareIndex("test", "test", "1").setSource("{\"foo\" : \"bar\"}").setRefresh(true).get();
+        client().prepareIndex("test", "test", "1").setSource("{\"foo\" : \"bar\"}").setRefreshPolicy(IMMEDIATE).get();
 
         IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {};
         shard.close("simon says", false);
@@ -1376,7 +1378,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         IndexService indexService = indicesService.indexService(resolveIndex("test"));
         IndexShard shard = indexService.getShardOrNull(0);
-        client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}").setRefresh(true).get();
+        client().prepareIndex("test", "test", "0").setSource("{\"foo\" : \"bar\"}").setRefreshPolicy(IMMEDIATE).get();
         IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {
             @Override
             public DirectoryReader wrap(DirectoryReader reader) throws IOException {
@@ -1529,8 +1531,8 @@ public class IndexShardTests extends ESSingleNodeTestCase {
             .field("type", "text")
             .endObject()
             .endObject().endObject().endObject()).get();
-        client().prepareIndex("index", "test", "0").setSource("{\"foo\" : \"bar\"}").setRefresh(true).get();
-        client().prepareIndex("index", "test", "1").setSource("{\"foo\" : \"bar\"}").setRefresh(true).get();
+        client().prepareIndex("index", "test", "0").setSource("{\"foo\" : \"bar\"}").setRefreshPolicy(IMMEDIATE).get();
+        client().prepareIndex("index", "test", "1").setSource("{\"foo\" : \"bar\"}").setRefreshPolicy(IMMEDIATE).get();
 
 
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
diff --git a/core/src/test/java/org/elasticsearch/index/termvectors/TermVectorsServiceTests.java b/core/src/test/java/org/elasticsearch/index/termvectors/TermVectorsServiceTests.java
index a4b80577923..c79a61a22b9 100644
--- a/core/src/test/java/org/elasticsearch/index/termvectors/TermVectorsServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/index/termvectors/TermVectorsServiceTests.java
@@ -34,6 +34,7 @@ import java.util.stream.Stream;
 
 import static java.lang.Math.abs;
 import static java.util.stream.Collectors.toList;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
@@ -55,7 +56,7 @@ public class TermVectorsServiceTests extends ESSingleNodeTestCase {
         createIndex("test", Settings.EMPTY, "type1", mapping);
         ensureGreen();
 
-        client().prepareIndex("test", "type1", "0").setSource("field", "foo bar").setRefresh(true).execute().get();
+        client().prepareIndex("test", "type1", "0").setSource("field", "foo bar").setRefreshPolicy(IMMEDIATE).get();
 
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         IndexService test = indicesService.indexService(resolveIndex("test"));
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
index e0446fe329b..0256fab1b7f 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
@@ -53,6 +53,7 @@ import org.elasticsearch.test.ESIntegTestCase;
 import java.util.Collection;
 import java.util.function.Function;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
@@ -332,7 +333,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
         verify(getSettings(indices).setIndicesOptions(options), false);
 
         assertAcked(prepareCreate("foobar"));
-        client().prepareIndex("foobar", "type", "1").setSource("k", "v").setRefresh(true).execute().actionGet();
+        client().prepareIndex("foobar", "type", "1").setSource("k", "v").setRefreshPolicy(IMMEDIATE).get();
 
         // Verify defaults for wildcards, with one wildcard expression and one existing index
         indices = new String[]{"foo*"};
@@ -422,7 +423,7 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
 
     public void testAllMissingLenient() throws Exception {
         createIndex("test1");
-        client().prepareIndex("test1", "type", "1").setSource("k", "v").setRefresh(true).execute().actionGet();
+        client().prepareIndex("test1", "type", "1").setSource("k", "v").setRefreshPolicy(IMMEDIATE).get();
         SearchResponse response = client().prepareSearch("test2")
                 .setIndicesOptions(IndicesOptions.lenientExpandOpen())
                 .setQuery(matchAllQuery())
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
index 5761997ed46..75c968e1d67 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
@@ -49,6 +49,7 @@ import java.util.Arrays;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.containsString;
@@ -131,7 +132,7 @@ public class IndicesServiceTests extends ESSingleNodeTestCase {
 
 
         test = createIndex("test");
-        client().prepareIndex("test", "type", "1").setSource("field", "value").setRefresh(true).get();
+        client().prepareIndex("test", "type", "1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         client().admin().indices().prepareFlush("test").get();
         assertHitCount(client().prepareSearch("test").get(), 1);
         IndexMetaData secondMetaData = clusterService.state().metaData().index("test");
diff --git a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
index 416dbd19761..298cb9cd9e3 100644
--- a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
@@ -45,6 +45,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
@@ -110,7 +111,7 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
 
 
         // index something into test_index, will match on both templates
-        client().prepareIndex("test_index", "type1", "1").setSource("field1", "value1", "field2", "value 2").setRefresh(true).execute().actionGet();
+        client().prepareIndex("test_index", "type1", "1").setSource("field1", "value1", "field2", "value 2").setRefreshPolicy(IMMEDIATE).get();
 
         ensureGreen();
         SearchResponse searchResponse = client().prepareSearch("test_index")
@@ -123,7 +124,7 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
         // field2 is not stored.
         assertThat(searchResponse.getHits().getAt(0).field("field2"), nullValue());
 
-        client().prepareIndex("text_index", "type1", "1").setSource("field1", "value1", "field2", "value 2").setRefresh(true).execute().actionGet();
+        client().prepareIndex("text_index", "type1", "1").setSource("field1", "value1", "field2", "value 2").setRefreshPolicy(IMMEDIATE).get();
 
         ensureGreen();
         // now only match on one template (template_1)
diff --git a/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java b/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java
index 21be585862d..e7101869692 100644
--- a/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java
+++ b/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java
@@ -33,6 +33,7 @@ import org.elasticsearch.test.ESIntegTestCase;
 import java.io.IOException;
 import java.util.Map;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
@@ -45,7 +46,8 @@ public class SimpleMgetIT extends ESIntegTestCase {
         createIndex("test");
         ensureYellow();
 
-        client().prepareIndex("test", "test", "1").setSource(jsonBuilder().startObject().field("foo", "bar").endObject()).setRefresh(true).execute().actionGet();
+        client().prepareIndex("test", "test", "1").setSource(jsonBuilder().startObject().field("foo", "bar").endObject())
+                .setRefreshPolicy(IMMEDIATE).get();
 
         MultiGetResponse mgetResponse = client().prepareMultiGet()
                 .add(new MultiGetRequest.Item("test", "test", "1"))
@@ -86,9 +88,9 @@ public class SimpleMgetIT extends ESIntegTestCase {
                         .endObject()));
         ensureYellow();
 
-        client().prepareIndex("test", "test", "1").setParent("4").setRefresh(true)
+        client().prepareIndex("test", "test", "1").setParent("4").setRefreshPolicy(IMMEDIATE)
                 .setSource(jsonBuilder().startObject().field("foo", "bar").endObject())
-                .execute().actionGet();
+                .get();
 
         MultiGetResponse mgetResponse = client().prepareMultiGet()
                 .add(new MultiGetRequest.Item(indexOrAlias(), "test", "1").parent("4"))
@@ -154,9 +156,9 @@ public class SimpleMgetIT extends ESIntegTestCase {
         final String id = routingKeyForShard("test", "test", 0);
         final String routingOtherShard = routingKeyForShard("test", "test", 1);
 
-        client().prepareIndex("test", "test", id).setRefresh(true).setRouting(routingOtherShard)
+        client().prepareIndex("test", "test", id).setRefreshPolicy(IMMEDIATE).setRouting(routingOtherShard)
                 .setSource(jsonBuilder().startObject().field("foo", "bar").endObject())
-                .execute().actionGet();
+                .get();
 
         MultiGetResponse mgetResponse = client().prepareMultiGet()
                 .add(new MultiGetRequest.Item(indexOrAlias(), "test", id).routing(routingOtherShard))
diff --git a/core/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/core/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index 2ac00b541af..b465d25043f 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -40,6 +40,7 @@ import java.io.IOException;
 import java.util.Collection;
 import java.util.concurrent.ExecutionException;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
@@ -58,7 +59,7 @@ public class SearchServiceTests extends ESSingleNodeTestCase {
 
     public void testClearOnClose() throws ExecutionException, InterruptedException {
         createIndex("index");
-        client().prepareIndex("index", "type", "1").setSource("field", "value").setRefresh(true).get();
+        client().prepareIndex("index", "type", "1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         SearchResponse searchResponse = client().prepareSearch("index").setSize(1).setScroll("1m").get();
         assertThat(searchResponse.getScrollId(), is(notNullValue()));
         SearchService service = getInstanceFromNode(SearchService.class);
@@ -70,7 +71,7 @@ public class SearchServiceTests extends ESSingleNodeTestCase {
 
     public void testClearOnStop() throws ExecutionException, InterruptedException {
         createIndex("index");
-        client().prepareIndex("index", "type", "1").setSource("field", "value").setRefresh(true).get();
+        client().prepareIndex("index", "type", "1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         SearchResponse searchResponse = client().prepareSearch("index").setSize(1).setScroll("1m").get();
         assertThat(searchResponse.getScrollId(), is(notNullValue()));
         SearchService service = getInstanceFromNode(SearchService.class);
@@ -82,7 +83,7 @@ public class SearchServiceTests extends ESSingleNodeTestCase {
 
     public void testClearIndexDelete() throws ExecutionException, InterruptedException {
         createIndex("index");
-        client().prepareIndex("index", "type", "1").setSource("field", "value").setRefresh(true).get();
+        client().prepareIndex("index", "type", "1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         SearchResponse searchResponse = client().prepareSearch("index").setSize(1).setScroll("1m").get();
         assertThat(searchResponse.getScrollId(), is(notNullValue()));
         SearchService service = getInstanceFromNode(SearchService.class);
@@ -94,7 +95,7 @@ public class SearchServiceTests extends ESSingleNodeTestCase {
 
     public void testCloseSearchContextOnRewriteException() {
         createIndex("index");
-        client().prepareIndex("index", "type", "1").setSource("field", "value").setRefresh(true).get();
+        client().prepareIndex("index", "type", "1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
 
         SearchService service = getInstanceFromNode(SearchService.class);
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
diff --git a/core/src/test/java/org/elasticsearch/search/SearchTimeoutIT.java b/core/src/test/java/org/elasticsearch/search/SearchTimeoutIT.java
index ef922e84048..b2eb12159f8 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchTimeoutIT.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchTimeoutIT.java
@@ -36,6 +36,7 @@ import java.util.Collections;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.index.query.QueryBuilders.scriptQuery;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -55,7 +56,7 @@ public class SearchTimeoutIT extends ESIntegTestCase {
     }
 
     public void testSimpleTimeout() throws Exception {
-        client().prepareIndex("test", "type", "1").setSource("field", "value").setRefresh(true).execute().actionGet();
+        client().prepareIndex("test", "type", "1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
 
         SearchResponse searchResponse = client().prepareSearch("test").setTimeout(new TimeValue(10, TimeUnit.MILLISECONDS))
                 .setQuery(scriptQuery(new Script(NativeTestScriptedTimeout.TEST_NATIVE_SCRIPT_TIMEOUT, ScriptType.INLINE, "native", null)))
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java
index 93e6536e47c..21b4a74fe41 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java
@@ -35,6 +35,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -509,7 +510,7 @@ public class ReverseNestedIT extends ESIntegTestCase {
                         .addMapping("product", mapping)
         );
 
-        client().prepareIndex("idx3", "product", "1").setRefresh(true).setSource(
+        client().prepareIndex("idx3", "product", "1").setRefreshPolicy(IMMEDIATE).setSource(
                 jsonBuilder().startObject()
                         .startArray("sku")
                             .startObject()
diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java
index 83e62072683..e17f7bb3efd 100644
--- a/core/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java
+++ b/core/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java
@@ -51,6 +51,7 @@ import java.util.Collection;
 import java.util.List;
 import java.util.Locale;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.client.Requests.indexRequest;
 import static org.elasticsearch.client.Requests.searchRequest;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -318,8 +319,13 @@ public class DecayFunctionScoreIT extends ESIntegTestCase {
                 .setId("1")
                 .setIndex("test")
                 .setSource(
-                        jsonBuilder().startObject().field("test", "value").startObject("loc").field("lat", 20).field("lon", 11).endObject()
-                                .endObject()).setRefresh(true).get();
+                        jsonBuilder().startObject()
+                            .field("test", "value")
+                            .startObject("loc")
+                                .field("lat", 20)
+                                .field("lon", 11)
+                            .endObject()
+                        .endObject()).setRefreshPolicy(IMMEDIATE).get();
         FunctionScoreQueryBuilder baseQuery = functionScoreQuery(constantScoreQuery(termQuery("test", "value")),
                 ScoreFunctionBuilders.weightFactorFunction(randomIntBetween(1, 10)));
         GeoPoint point = new GeoPoint(20, 11);
@@ -354,8 +360,8 @@ public class DecayFunctionScoreIT extends ESIntegTestCase {
                         .endObject().startObject("num").field("type", "double").endObject().endObject().endObject().endObject()));
         ensureYellow();
 
-        client().prepareIndex().setType("type1").setId("1").setIndex("test")
-                .setSource(jsonBuilder().startObject().field("test", "value value").field("num", 1.0).endObject()).setRefresh(true).get();
+        client().prepareIndex().setType("type1").setId("1").setIndex("test").setRefreshPolicy(IMMEDIATE)
+                .setSource(jsonBuilder().startObject().field("test", "value value").field("num", 1.0).endObject()).get();
         FunctionScoreQueryBuilder baseQuery = functionScoreQuery(constantScoreQuery(termQuery("test", "value")),
                 ScoreFunctionBuilders.weightFactorFunction(2));
         // decay score should return 0.5 for this function and baseQuery should return 2.0f as it's score
diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java b/core/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java
index 46704410bd5..7e01f575822 100644
--- a/core/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java
+++ b/core/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java
@@ -34,6 +34,7 @@ import org.elasticsearch.test.VersionUtils;
 
 import java.util.Collection;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
 import static org.elasticsearch.index.query.QueryBuilders.geoBoundingBoxQuery;
@@ -249,16 +250,16 @@ public class GeoBoundingBoxIT extends ESIntegTestCase {
                 .field("title", "Place in Stockholm")
                 .startObject("location").field("lat", 59.328355000000002).field("lon", 18.036842).endObject()
                 .endObject())
-                .setRefresh(true)
-                .execute().actionGet();
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
 
         client().prepareIndex("test", "type1", "2").setSource(jsonBuilder().startObject()
                 .field("userid", 534)
                 .field("title", "Place in Montreal")
                 .startObject("location").field("lat", 45.509526999999999).field("lon", -73.570986000000005).endObject()
                 .endObject())
-                .setRefresh(true)
-                .execute().actionGet();
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
 
         SearchResponse searchResponse = client().prepareSearch()
                 .setQuery(
@@ -304,16 +305,16 @@ public class GeoBoundingBoxIT extends ESIntegTestCase {
                 .field("title", "Place in Stockholm")
                 .startObject("location").field("lat", 59.328355000000002).field("lon", 18.036842).endObject()
                 .endObject())
-                .setRefresh(true)
-                .execute().actionGet();
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
 
         client().prepareIndex("test", "type1", "2").setSource(jsonBuilder().startObject()
                 .field("userid", 534)
                 .field("title", "Place in Montreal")
                 .startObject("location").field("lat", 45.509526999999999).field("lon", -73.570986000000005).endObject()
                 .endObject())
-                .setRefresh(true)
-                .execute().actionGet();
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
 
         SearchResponse searchResponse = client().prepareSearch()
                 .setQuery(
diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java b/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
index 0fe90f133b5..7da18342edf 100644
--- a/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
+++ b/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
@@ -42,6 +42,7 @@ import org.elasticsearch.test.geo.RandomShapeGenerator;
 import java.io.IOException;
 import java.util.Locale;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.geoIntersectionQuery;
 import static org.elasticsearch.index.query.QueryBuilders.geoShapeQuery;
@@ -65,8 +66,7 @@ public class GeoShapeQueryTests extends ESSingleNodeTestCase {
         client().admin().indices().prepareCreate("test").addMapping("type1", mapping).execute().actionGet();
         ensureGreen();
 
-        client().prepareIndex("test", "type1", "aNullshape").setSource("{\"location\": null}").setRefresh(true)
-                .execute().actionGet();
+        client().prepareIndex("test", "type1", "aNullshape").setSource("{\"location\": null}").setRefreshPolicy(IMMEDIATE).get();
         GetResponse result = client().prepareGet("test", "type1", "aNullshape").execute().actionGet();
         assertThat(result.getField("location"), nullValue());
     }
@@ -87,7 +87,7 @@ public class GeoShapeQueryTests extends ESSingleNodeTestCase {
                 .field("type", "point")
                 .startArray("coordinates").value(-30).value(-30).endArray()
                 .endObject()
-                .endObject()).setRefresh(true).execute().actionGet();
+                .endObject()).setRefreshPolicy(IMMEDIATE).get();
 
         client().prepareIndex("test", "type1", "2").setSource(jsonBuilder().startObject()
                 .field("name", "Document 2")
@@ -95,7 +95,7 @@ public class GeoShapeQueryTests extends ESSingleNodeTestCase {
                 .field("type", "point")
                 .startArray("coordinates").value(-45).value(-50).endArray()
                 .endObject()
-                .endObject()).setRefresh(true).execute().actionGet();
+                .endObject()).setRefreshPolicy(IMMEDIATE).get();
 
         ShapeBuilder shape = ShapeBuilders.newEnvelope(new Coordinate(-45, 45), new Coordinate(45, -45));
 
@@ -139,7 +139,7 @@ public class GeoShapeQueryTests extends ESSingleNodeTestCase {
                 .startArray().value(-122.83).value(48.57).endArray() // close the polygon
                 .endArray().endArray()
                 .endObject()
-                .endObject()).setRefresh(true).execute().actionGet();
+                .endObject()).setRefreshPolicy(IMMEDIATE).get();
 
         ShapeBuilder query = ShapeBuilders.newEnvelope(new Coordinate(-122.88, 48.62), new Coordinate(-122.82, 48.54));
 
@@ -169,14 +169,14 @@ public class GeoShapeQueryTests extends ESSingleNodeTestCase {
         ShapeBuilder shape = ShapeBuilders.newEnvelope(new Coordinate(-45, 45), new Coordinate(45, -45));
 
         client().prepareIndex("shapes", "shape_type", "Big_Rectangle").setSource(jsonBuilder().startObject()
-                .field("shape", shape).endObject()).setRefresh(true).execute().actionGet();
+                .field("shape", shape).endObject()).setRefreshPolicy(IMMEDIATE).get();
         client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject()
                 .field("name", "Document 1")
                 .startObject("location")
                 .field("type", "point")
                 .startArray("coordinates").value(-30).value(-30).endArray()
                 .endObject()
-                .endObject()).setRefresh(true).execute().actionGet();
+                .endObject()).setRefreshPolicy(IMMEDIATE).get();
 
         SearchResponse searchResponse = client().prepareSearch("test").setTypes("type1")
                 .setQuery(geoIntersectionQuery("location", "Big_Rectangle", "shape_type"))
@@ -226,7 +226,7 @@ public class GeoShapeQueryTests extends ESSingleNodeTestCase {
                         String.format(
                                 Locale.ROOT, "{ %s, \"1\" : { %s, \"2\" : { %s, \"3\" : { %s } }} }", location, location, location, location
                         )
-                ).setRefresh(true).execute().actionGet();
+                ).setRefreshPolicy(IMMEDIATE).get();
         client().prepareIndex("test", "type", "1")
                 .setSource(jsonBuilder().startObject().startObject("location")
                         .field("type", "polygon")
@@ -237,7 +237,7 @@ public class GeoShapeQueryTests extends ESSingleNodeTestCase {
                         .startArray().value(-20).value(20).endArray()
                         .startArray().value(-20).value(-20).endArray()
                         .endArray().endArray()
-                        .endObject().endObject()).setRefresh(true).execute().actionGet();
+                        .endObject().endObject()).setRefreshPolicy(IMMEDIATE).get();
 
         GeoShapeQueryBuilder filter = QueryBuilders.geoShapeQuery("location", "1", "type").relation(ShapeRelation.INTERSECTS)
                 .indexedShapeIndex("shapes")
@@ -305,7 +305,7 @@ public class GeoShapeQueryTests extends ESSingleNodeTestCase {
                 .execute().actionGet();
 
         XContentBuilder docSource = gcb.toXContent(jsonBuilder().startObject().field("location"), null).endObject();
-        client().prepareIndex("test", "type", "1").setSource(docSource).setRefresh(true).execute().actionGet();
+        client().prepareIndex("test", "type", "1").setSource(docSource).setRefreshPolicy(IMMEDIATE).get();
 
         ShapeBuilder filterShape = (gcb.getShapeAt(randomIntBetween(0, gcb.numShapes() - 1)));
 
@@ -326,12 +326,12 @@ public class GeoShapeQueryTests extends ESSingleNodeTestCase {
                 .execute().actionGet();
 
         XContentBuilder docSource = gcb.toXContent(jsonBuilder().startObject().field("location"), null).endObject();
-        client().prepareIndex("test", "type", "1").setSource(docSource).setRefresh(true).execute().actionGet();
+        client().prepareIndex("test", "type", "1").setSource(docSource).setRefreshPolicy(IMMEDIATE).get();
 
         // index the mbr of the collection
         EnvelopeBuilder env = new EnvelopeBuilder(new Coordinate(mbr.getMinX(), mbr.getMaxY()), new Coordinate(mbr.getMaxX(), mbr.getMinY()));
         docSource = env.toXContent(jsonBuilder().startObject().field("location"), null).endObject();
-        client().prepareIndex("test", "type", "2").setSource(docSource).setRefresh(true).execute().actionGet();
+        client().prepareIndex("test", "type", "2").setSource(docSource).setRefreshPolicy(IMMEDIATE).get();
 
         ShapeBuilder filterShape = (gcb.getShapeAt(randomIntBetween(0, gcb.numShapes() - 1)));
         GeoShapeQueryBuilder filter = QueryBuilders.geoShapeQuery("location", filterShape)
@@ -371,7 +371,7 @@ public class GeoShapeQueryTests extends ESSingleNodeTestCase {
                 .endArray()
                 .endObject().endObject();
         client().prepareIndex("test", "type", "1")
-                .setSource(docSource).setRefresh(true).execute().actionGet();
+                .setSource(docSource).setRefreshPolicy(IMMEDIATE).get();
 
         GeoShapeQueryBuilder filter = QueryBuilders.geoShapeQuery(
                 "location",
@@ -427,7 +427,7 @@ public class GeoShapeQueryTests extends ESSingleNodeTestCase {
         try {
             client().prepareIndex("geo_points_only", "type1", "1")
                     .setSource(jsonBuilder().startObject().field("location", shape).endObject())
-                    .setRefresh(true).execute().actionGet();
+                    .setRefreshPolicy(IMMEDIATE).get();
         } catch (MapperParsingException e) {
             // RandomShapeGenerator created something other than a POINT type, verify the correct exception is thrown
             assertThat(e.getCause().getMessage(), containsString("is configured for points only"));
diff --git a/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java b/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
index 0d1ba43659c..01f98564814 100644
--- a/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
@@ -48,6 +48,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
@@ -920,7 +921,7 @@ public class InnerHitsIT extends ESIntegTestCase {
                 .endObject()
                 .endArray()
                 .endObject())
-        .setRefresh(true)
+        .setRefreshPolicy(IMMEDIATE)
         .get();
 
         response = client().prepareSearch("index2")
diff --git a/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java b/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
index 4d2f92c3c6c..ba378a3c404 100644
--- a/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
+++ b/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
@@ -38,6 +38,7 @@ import org.elasticsearch.search.sort.SortMode;
 import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.test.ESIntegTestCase;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
@@ -306,7 +307,7 @@ public class SimpleNestedIT extends ESIntegTestCase {
                 .endObject()
                 .endArray()
                 .endObject())
-                .setRefresh(true)
+                .setRefreshPolicy(IMMEDIATE)
                 .execute().actionGet();
 
         SearchResponse searchResponse = client().prepareSearch("test")
diff --git a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
index d7d4a3fd188..5e4a7d6f77f 100644
--- a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
@@ -55,6 +55,7 @@ import java.io.IOException;
 import java.util.Random;
 import java.util.concurrent.ExecutionException;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
@@ -731,7 +732,7 @@ public class SearchQueryIT extends ESIntegTestCase {
     public void testPassQueryOrFilterAsJSONString() throws Exception {
         createIndex("test");
 
-        client().prepareIndex("test", "type1", "1").setSource("field1", "value1_1", "field2", "value2_1").setRefresh(true).get();
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value1_1", "field2", "value2_1").setRefreshPolicy(IMMEDIATE).get();
 
         WrapperQueryBuilder wrapper = new WrapperQueryBuilder("{ \"term\" : { \"field1\" : \"value1_1\" } }");
         assertHitCount(client().prepareSearch().setQuery(wrapper).get(), 1L);
@@ -1548,7 +1549,7 @@ public class SearchQueryIT extends ESIntegTestCase {
     }
 
     public void testMultiFieldQueryString() {
-        client().prepareIndex("test", "s", "1").setSource("field1", "value1", "field2", "value2").setRefresh(true).get();
+        client().prepareIndex("test", "s", "1").setSource("field1", "value1", "field2", "value2").setRefreshPolicy(IMMEDIATE).get();
 
         logger.info("regular");
         assertHitCount(client().prepareSearch("test").setQuery(queryStringQuery("value1").field("field1").field("field2")).get(), 1);
@@ -1702,7 +1703,7 @@ public class SearchQueryIT extends ESIntegTestCase {
     }
 
     public void testAllFieldEmptyMapping() throws Exception {
-        client().prepareIndex("myindex", "mytype").setId("1").setSource("{}").setRefresh(true).get();
+        client().prepareIndex("myindex", "mytype").setId("1").setSource("{}").setRefreshPolicy(IMMEDIATE).get();
         SearchResponse response = client().prepareSearch("myindex").setQuery(matchQuery("_all", "foo")).get();
         assertNoFailures(response);
     }
@@ -1712,7 +1713,7 @@ public class SearchQueryIT extends ESIntegTestCase {
         assertAcked(client().admin().indices().preparePutMapping("myindex").setType("mytype").setSource(
                 jsonBuilder().startObject().startObject("mytype").startObject("_all").field("enabled", false)
                 .endObject().endObject().endObject()));
-        client().prepareIndex("myindex", "mytype").setId("1").setSource("bar", "foo").setRefresh(true).get();
+        client().prepareIndex("myindex", "mytype").setId("1").setSource("bar", "foo").setRefreshPolicy(IMMEDIATE).get();
         SearchResponse response = client().prepareSearch("myindex").setQuery(matchQuery("_all", "foo")).get();
         assertNoFailures(response);
         assertHitCount(response, 0);
@@ -2025,7 +2026,7 @@ public class SearchQueryIT extends ESIntegTestCase {
 
 
         client().prepareIndex("test", "test", "1").setSource("origin", "C.A1234.5678")
-                .setRefresh(true)
+                .setRefreshPolicy(IMMEDIATE)
                 .get();
 
         SearchResponse searchResponse = client().prepareSearch("test")
diff --git a/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java b/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
index 9ee9e0841cf..3bce6e10f83 100644
--- a/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
+++ b/core/src/test/java/org/elasticsearch/search/scroll/SearchScrollIT.java
@@ -48,6 +48,7 @@ import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
 import java.io.IOException;
 import java.util.Map;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
@@ -421,7 +422,7 @@ public class SearchScrollIT extends ESIntegTestCase {
     public void testDeepScrollingDoesNotBlowUp() throws Exception {
         client().prepareIndex("index", "type", "1")
                 .setSource("field", "value")
-                .setRefresh(true)
+                .setRefreshPolicy(IMMEDIATE)
                 .execute().get();
 
         for (SearchType searchType : SearchType.values()) {
diff --git a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java
index 4220bcb4538..9751df26329 100644
--- a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java
@@ -36,6 +36,7 @@ import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.rescore.QueryRescorerBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -101,7 +102,8 @@ public class SimpleSearchIT extends ESIntegTestCase {
                         .endObject().endObject().endObject())
                 .execute().actionGet();
 
-        client().prepareIndex("test", "type1", "1").setSource("from", "192.168.0.5", "to", "192.168.0.10").setRefresh(true).execute().actionGet();
+        client().prepareIndex("test", "type1", "1").setSource("from", "192.168.0.5", "to", "192.168.0.10").setRefreshPolicy(IMMEDIATE)
+                .get();
 
         SearchResponse search = client().prepareSearch()
                 .setQuery(boolQuery().must(rangeQuery("from").lte("192.168.0.7")).must(rangeQuery("to").gte("192.168.0.7")))
@@ -185,7 +187,7 @@ public class SimpleSearchIT extends ESIntegTestCase {
     public void testSimpleId() {
         createIndex("test");
 
-        client().prepareIndex("test", "type", "XXX1").setSource("field", "value").setRefresh(true).execute().actionGet();
+        client().prepareIndex("test", "type", "XXX1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).get();
         // id is not indexed, but lets see that we automatically convert to
         SearchResponse searchResponse = client().prepareSearch().setQuery(QueryBuilders.termQuery("_id", "XXX1")).execute().actionGet();
         assertHitCount(searchResponse, 1L);
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java
index 9ef3c898cab..0c6179e0693 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearchIT.java
@@ -64,6 +64,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.common.util.CollectionUtils.iterableAsArrayList;
@@ -622,7 +623,8 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
                 .endObject()
                 .endObject();
         assertAcked(prepareCreate(INDEX).addMapping(TYPE, mapping));
-        client().prepareIndex(INDEX, TYPE, "1").setRefresh(true).setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()).get();
+        client().prepareIndex(INDEX, TYPE, "1").setRefreshPolicy(IMMEDIATE)
+                .setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()).get();
         ensureGreen(INDEX);
 
         PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(jsonBuilder().startObject()
@@ -643,7 +645,8 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
         ).execute().actionGet();
         assertSuggestions(searchResponse, "suggs");
 
-        client().prepareIndex(INDEX, TYPE, "1").setRefresh(true).setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()).get();
+        client().prepareIndex(INDEX, TYPE, "1").setRefreshPolicy(IMMEDIATE)
+                .setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()).get();
         ensureGreen(INDEX);
 
         SearchResponse afterReindexingResponse = client().prepareSearch(INDEX).suggest(
@@ -1089,7 +1092,7 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
                 .startObject().startObject(FIELD)
                 .startArray("input").value(longString).endArray()
                 .endObject().endObject()
-        ).setRefresh(true).get();
+        ).setRefreshPolicy(IMMEDIATE).get();
 
     }
 
@@ -1111,7 +1114,7 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
                     .startArray("input").value(string).endArray()
                     .field("output", "foobar")
                     .endObject().endObject()
-            ).setRefresh(true).get();
+            ).get();
             fail("Expected MapperParsingException");
         } catch (MapperParsingException e) {
             assertThat(e.getMessage(), containsString("failed to parse"));
@@ -1133,7 +1136,7 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
                         .startObject()
                         .field(FIELD, string)
                         .endObject()
-        ).setRefresh(true).get();
+        ).setRefreshPolicy(IMMEDIATE).get();
 
         try {
             client().prepareSearch(INDEX).addAggregation(AggregationBuilders.terms("suggest_agg").field(FIELD)
@@ -1163,11 +1166,10 @@ public class CompletionSuggestSearchIT extends ESIntegTestCase {
         ensureGreen();
 
         client().prepareIndex(INDEX, TYPE, "1").setSource(FIELD, "strings make me happy", FIELD + "_1", "nulls make me sad")
-                .setRefresh(true).get();
+                .setRefreshPolicy(IMMEDIATE).get();
 
         try {
-            client().prepareIndex(INDEX, TYPE, "2").setSource(FIELD, null, FIELD + "_1", "nulls make me sad")
-                    .setRefresh(true).get();
+            client().prepareIndex(INDEX, TYPE, "2").setSource(FIELD, null, FIELD + "_1", "nulls make me sad").get();
             fail("Expected MapperParsingException for null value");
         } catch (MapperParsingException e) {
             // make sure that the exception has the name of the field causing the error
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/ContextSuggestSearch2xIT.java b/core/src/test/java/org/elasticsearch/search/suggest/ContextSuggestSearch2xIT.java
index 9152fe61847..f5726622f62 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/ContextSuggestSearch2xIT.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/ContextSuggestSearch2xIT.java
@@ -53,6 +53,7 @@ import java.util.Map;
 import java.util.Set;
 
 import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSuggestion;
@@ -504,7 +505,7 @@ public class ContextSuggestSearch2xIT extends ESIntegTestCase {
         ensureGreen();
 
         client().prepareIndex(INDEX, TYPE, "1").setSource(FIELD, "")
-            .setRefresh(true).get();
+            .setRefreshPolicy(IMMEDIATE).get();
 
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java
index 4f97a5179ed..28f8ec49a57 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java
@@ -45,6 +45,7 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
@@ -65,7 +66,7 @@ public class CustomSuggesterSearchIT extends ESIntegTestCase {
                 .startObject()
                 .field("name", "arbitrary content")
                 .endObject())
-                .setRefresh(true).execute().actionGet();
+                .setRefreshPolicy(IMMEDIATE).get();
         ensureYellow();
 
         String randomText = randomAsciiOfLength(10);
diff --git a/core/src/test/java/org/elasticsearch/similarity/SimilarityIT.java b/core/src/test/java/org/elasticsearch/similarity/SimilarityIT.java
index a123f7133b3..66c3509c76d 100644
--- a/core/src/test/java/org/elasticsearch/similarity/SimilarityIT.java
+++ b/core/src/test/java/org/elasticsearch/similarity/SimilarityIT.java
@@ -23,6 +23,7 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.hamcrest.Matchers.equalTo;
@@ -61,7 +62,7 @@ public class SimilarityIT extends ESIntegTestCase {
 
         client().prepareIndex("test", "type1", "1").setSource("field1", "the quick brown fox jumped over the lazy dog",
                                                             "field2", "the quick brown fox jumped over the lazy dog")
-                .setRefresh(true).execute().actionGet();
+                .setRefreshPolicy(IMMEDIATE).execute().actionGet();
 
         SearchResponse bm25SearchResponse = client().prepareSearch().setQuery(matchQuery("field1", "quick brown fox")).execute().actionGet();
         assertThat(bm25SearchResponse.getHits().totalHits(), equalTo(1L));
diff --git a/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java b/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java
index 21408e7b1df..2fa94bc2357 100644
--- a/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java
+++ b/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java
@@ -29,6 +29,7 @@ import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.Locale;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
@@ -48,7 +49,7 @@ public class SimpleTimestampIT extends ESIntegTestCase {
 
         logger.info("--> check with automatic timestamp");
         long now1 = System.currentTimeMillis();
-        client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefresh(true).execute().actionGet();
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get();
         long now2 = System.currentTimeMillis();
 
         // we check both realtime get and non realtime get
@@ -70,7 +71,7 @@ public class SimpleTimestampIT extends ESIntegTestCase {
         assertThat(((Number) getResponse.getField("_timestamp").getValue()).longValue(), equalTo(timestamp));
 
         logger.info("--> check with custom timestamp (numeric)");
-        client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setTimestamp("10").setRefresh(true).execute().actionGet();
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setTimestamp("10").setRefreshPolicy(IMMEDIATE).get();
 
         getResponse = client().prepareGet("test", "type1", "1").setFields("_timestamp").setRealtime(false).execute().actionGet();
         timestamp = ((Number) getResponse.getField("_timestamp").getValue()).longValue();
@@ -80,7 +81,8 @@ public class SimpleTimestampIT extends ESIntegTestCase {
         assertThat(((Number) getResponse.getField("_timestamp").getValue()).longValue(), equalTo(timestamp));
 
         logger.info("--> check with custom timestamp (string)");
-        client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setTimestamp("1970-01-01T00:00:00.020").setRefresh(true).execute().actionGet();
+        client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setTimestamp("1970-01-01T00:00:00.020")
+                .setRefreshPolicy(IMMEDIATE).get();
 
         getResponse = client().prepareGet("test", "type1", "1").setFields("_timestamp").setRealtime(false).execute().actionGet();
         timestamp = ((Number) getResponse.getField("_timestamp").getValue()).longValue();
diff --git a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
index f6a74faac39..003c08134bc 100644
--- a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
+++ b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
@@ -38,6 +38,7 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.both;
@@ -92,10 +93,10 @@ public class SimpleTTLIT extends ESIntegTestCase {
         // Index one doc without routing, one doc with routing, one doc with not TTL and no default and one doc with default TTL
         long now = System.currentTimeMillis();
         IndexResponse indexResponse = client().prepareIndex("test", "type1", "1").setSource("field1", "value1")
-                .setTimestamp(String.valueOf(now)).setTTL(providedTTLValue).setRefresh(true).get();
+                .setTimestamp(String.valueOf(now)).setTTL(providedTTLValue).setRefreshPolicy(IMMEDIATE).get();
         assertThat(indexResponse.isCreated(), is(true));
         indexResponse = client().prepareIndex("test", "type1", "with_routing").setSource("field1", "value1")
-                .setTimestamp(String.valueOf(now)).setTTL(providedTTLValue).setRouting("routing").setRefresh(true).get();
+                .setTimestamp(String.valueOf(now)).setTTL(providedTTLValue).setRouting("routing").setRefreshPolicy(IMMEDIATE).get();
         assertThat(indexResponse.isCreated(), is(true));
         indexResponse = client().prepareIndex("test", "type1", "no_ttl").setSource("field1", "value1").get();
         assertThat(indexResponse.isCreated(), is(true));
@@ -245,7 +246,7 @@ public class SimpleTTLIT extends ESIntegTestCase {
         long secondTtl = aLongTime * 2;
         long thirdTtl = aLongTime * 1;
         IndexResponse indexResponse = client().prepareIndex("test", "type1", "1").setSource("field1", "value1")
-                .setTTL(firstTtl).setRefresh(true).get();
+                .setTTL(firstTtl).setRefreshPolicy(IMMEDIATE).get();
         assertTrue(indexResponse.isCreated());
         assertThat(getTtl("type1", 1), both(lessThanOrEqualTo(firstTtl)).and(greaterThan(secondTtl)));
 
diff --git a/core/src/test/java/org/elasticsearch/update/UpdateIT.java b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
index 0445da61096..078bcd8cc3f 100644
--- a/core/src/test/java/org/elasticsearch/update/UpdateIT.java
+++ b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
@@ -63,6 +63,7 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
@@ -694,7 +695,7 @@ public class UpdateIT extends ESIntegTestCase {
         }
 
         // check TTL is kept after an update without TTL
-        client().prepareIndex("test", "type1", "2").setSource("field", 1).setTTL(86400000L).setRefresh(true).execute().actionGet();
+        client().prepareIndex("test", "type1", "2").setSource("field", 1).setTTL(86400000L).setRefreshPolicy(IMMEDIATE).get();
         GetResponse getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
         long ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue();
         assertThat(ttl, greaterThan(0L));
@@ -713,7 +714,7 @@ public class UpdateIT extends ESIntegTestCase {
         assertThat(ttl, lessThanOrEqualTo(3600000L));
 
         // check timestamp update
-        client().prepareIndex("test", "type1", "3").setSource("field", 1).setRefresh(true).execute().actionGet();
+        client().prepareIndex("test", "type1", "3").setSource("field", 1).setRefreshPolicy(IMMEDIATE).get();
         client().prepareUpdate(indexOrAlias(), "type1", "3")
                 .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", Collections.singletonMap("_ctx", Collections.singletonMap("_timestamp", "2009-11-15T14:12:12")))).execute()
                 .actionGet();
diff --git a/docs/reference/migration/migrate_5_0/java.asciidoc b/docs/reference/migration/migrate_5_0/java.asciidoc
index a16f2a9ff9b..1ad34d11a7c 100644
--- a/docs/reference/migration/migrate_5_0/java.asciidoc
+++ b/docs/reference/migration/migrate_5_0/java.asciidoc
@@ -84,14 +84,14 @@ static factory methods in QueryBuilders accordingly.
 
 Making sure that query contains at least one clause by making initial clause mandatory
 in constructor.
-Renaming method to add clauses from `clause(SpanQueryBuilder)` to `addClause(SpanQueryBuilder)`. 
+Renaming method to add clauses from `clause(SpanQueryBuilder)` to `addClause(SpanQueryBuilder)`.
 
 ===== SpanNearQueryBuilder
 
 Removed setter for mandatory slop parameter, needs to be set in constructor now. Also
 making sure that query contains at least one clause by making initial clause mandatory
 in constructor. Updated the static factory methods in QueryBuilders accordingly.
-Renaming method to add clauses from `clause(SpanQueryBuilder)` to `addClause(SpanQueryBuilder)`. 
+Renaming method to add clauses from `clause(SpanQueryBuilder)` to `addClause(SpanQueryBuilder)`.
 
 ===== SpanNotQueryBuilder
 
@@ -305,7 +305,8 @@ The `setQuery(BytesReference)` method have been removed in favor of using `setQu
 Removed the `getMemoryAvailable` method from `OsStats`, which could be previously accessed calling
 `clusterStatsResponse.getNodesStats().getOs().getMemoryAvailable()`.
 
-=== setRefresh(boolean) has been deprecated
+=== setRefresh(boolean) has been removed
 
-`setRefresh(boolean)` has been deprecated in favor of `setRefreshPolicy(RefreshPolicy)` because there
-are now three options. It will be removed in 5.0.
+`setRefresh(boolean)` has been removed in favor of `setRefreshPolicy(RefreshPolicy)` because there
+are now three options (NONE, IMMEDIATE, and WAIT_FOR). `setRefresh(IMMEDIATE)` has the same behavior
+as `setRefresh(true)` used to have. See `setRefreshPolicy`'s javadoc for more.
diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
index da2f117c78f..6796c1e048b 100644
--- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
+++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java
@@ -59,6 +59,7 @@ import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
@@ -97,7 +98,7 @@ public class MoreExpressionTests extends ESIntegTestCase {
     public void testBasic() throws Exception {
         createIndex("test");
         ensureGreen("test");
-        client().prepareIndex("test", "doc", "1").setSource("foo", 4).setRefresh(true).get();
+        client().prepareIndex("test", "doc", "1").setSource("foo", 4).setRefreshPolicy(IMMEDIATE).get();
         SearchResponse rsp = buildRequest("doc['foo'] + 1").get();
         assertEquals(1, rsp.getHits().getTotalHits());
         assertEquals(5.0, rsp.getHits().getAt(0).field("foo").getValue(), 0.0D);
@@ -106,7 +107,7 @@ public class MoreExpressionTests extends ESIntegTestCase {
     public void testFunction() throws Exception {
         createIndex("test");
         ensureGreen("test");
-        client().prepareIndex("test", "doc", "1").setSource("foo", 4).setRefresh(true).get();
+        client().prepareIndex("test", "doc", "1").setSource("foo", 4).setRefreshPolicy(IMMEDIATE).get();
         SearchResponse rsp = buildRequest("doc['foo'] + abs(1)").get();
         assertSearchResponse(rsp);
         assertEquals(1, rsp.getHits().getTotalHits());
@@ -116,7 +117,7 @@ public class MoreExpressionTests extends ESIntegTestCase {
     public void testBasicUsingDotValue() throws Exception {
         createIndex("test");
         ensureGreen("test");
-        client().prepareIndex("test", "doc", "1").setSource("foo", 4).setRefresh(true).get();
+        client().prepareIndex("test", "doc", "1").setSource("foo", 4).setRefreshPolicy(IMMEDIATE).get();
         SearchResponse rsp = buildRequest("doc['foo'].value + 1").get();
         assertEquals(1, rsp.getHits().getTotalHits());
         assertEquals(5.0, rsp.getHits().getAt(0).field("foo").getValue(), 0.0D);
@@ -314,7 +315,7 @@ public class MoreExpressionTests extends ESIntegTestCase {
     public void testMissingField() throws Exception {
         createIndex("test");
         ensureGreen("test");
-        client().prepareIndex("test", "doc", "1").setSource("x", 4).setRefresh(true).get();
+        client().prepareIndex("test", "doc", "1").setSource("x", 4).setRefreshPolicy(IMMEDIATE).get();
         try {
             buildRequest("doc['bogus']").get();
             fail("Expected missing field to cause failure");
@@ -344,7 +345,7 @@ public class MoreExpressionTests extends ESIntegTestCase {
     }
 
     public void testCompileFailure() {
-        client().prepareIndex("test", "doc", "1").setSource("x", 1).setRefresh(true).get();
+        client().prepareIndex("test", "doc", "1").setSource("x", 1).setRefreshPolicy(IMMEDIATE).get();
         try {
             buildRequest("garbage%@#%@").get();
             fail("Expected expression compilation failure");
@@ -357,7 +358,7 @@ public class MoreExpressionTests extends ESIntegTestCase {
     }
 
     public void testNonNumericParam() {
-        client().prepareIndex("test", "doc", "1").setSource("x", 1).setRefresh(true).get();
+        client().prepareIndex("test", "doc", "1").setSource("x", 1).setRefreshPolicy(IMMEDIATE).get();
         try {
             buildRequest("a", "a", "astring").get();
             fail("Expected string parameter to cause failure");
@@ -370,7 +371,7 @@ public class MoreExpressionTests extends ESIntegTestCase {
     }
 
     public void testNonNumericField() {
-        client().prepareIndex("test", "doc", "1").setSource("text", "this is not a number").setRefresh(true).get();
+        client().prepareIndex("test", "doc", "1").setSource("text", "this is not a number").setRefreshPolicy(IMMEDIATE).get();
         try {
             buildRequest("doc['text.keyword']").get();
             fail("Expected text field to cause execution failure");
@@ -383,7 +384,7 @@ public class MoreExpressionTests extends ESIntegTestCase {
     }
 
     public void testInvalidGlobalVariable() {
-        client().prepareIndex("test", "doc", "1").setSource("foo", 5).setRefresh(true).get();
+        client().prepareIndex("test", "doc", "1").setSource("foo", 5).setRefreshPolicy(IMMEDIATE).get();
         try {
             buildRequest("bogus").get();
             fail("Expected bogus variable to cause execution failure");
@@ -396,7 +397,7 @@ public class MoreExpressionTests extends ESIntegTestCase {
     }
 
     public void testDocWithoutField() {
-        client().prepareIndex("test", "doc", "1").setSource("foo", 5).setRefresh(true).get();
+        client().prepareIndex("test", "doc", "1").setSource("foo", 5).setRefreshPolicy(IMMEDIATE).get();
         try {
             buildRequest("doc").get();
             fail("Expected doc variable without field to cause execution failure");
@@ -409,7 +410,7 @@ public class MoreExpressionTests extends ESIntegTestCase {
     }
 
     public void testInvalidFieldMember() {
-        client().prepareIndex("test", "doc", "1").setSource("foo", 5).setRefresh(true).get();
+        client().prepareIndex("test", "doc", "1").setSource("foo", 5).setRefreshPolicy(IMMEDIATE).get();
         try {
             buildRequest("doc['foo'].bogus").get();
             fail("Expected bogus field member to cause execution failure");
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
index 4e3accfdc06..bc604987a70 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
@@ -57,6 +57,7 @@ import java.util.Set;
 import java.util.concurrent.ExecutionException;
 
 import static java.util.Collections.singleton;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.client.Requests.refreshRequest;
 import static org.elasticsearch.common.util.set.Sets.newHashSet;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -456,7 +457,7 @@ public class SearchFieldsTests extends ESIntegTestCase {
         client().prepareIndex("my-index", "my-type1", "1")
                 .setRouting("1")
                 .setSource(jsonBuilder().startObject().field("field1", "value").endObject())
-                .setRefresh(true)
+                .setRefreshPolicy(IMMEDIATE)
                 .get();
 
         SearchResponse searchResponse = client().prepareSearch("my-index")
@@ -473,7 +474,7 @@ public class SearchFieldsTests extends ESIntegTestCase {
     public void testSearchFieldsNonLeafField() throws Exception {
         client().prepareIndex("my-index", "my-type1", "1")
                 .setSource(jsonBuilder().startObject().startObject("field1").field("field2", "value1").endObject().endObject())
-                .setRefresh(true)
+                .setRefreshPolicy(IMMEDIATE)
                 .get();
 
         assertFailures(client().prepareSearch("my-index").setTypes("my-type1").addField("field1"),
@@ -536,7 +537,7 @@ public class SearchFieldsTests extends ESIntegTestCase {
                 .endObject().bytes();
 
         client().prepareIndex("my-index", "my-type1", "1").setSource(source).get();
-        client().prepareIndex("my-index", "my-type2", "1").setRefresh(true).setSource(source).get();
+        client().prepareIndex("my-index", "my-type2", "1").setRefreshPolicy(IMMEDIATE).setSource(source).get();
 
 
         String field = "field1.field2.field3.field4";
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java
index 4e13219bae5..4a7b4350d23 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java
@@ -37,6 +37,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery;
 import static org.elasticsearch.index.query.QueryBuilders.functionScoreQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
@@ -58,7 +59,7 @@ public class GroovyScriptTests extends ESIntegTestCase {
     }
 
     public void testGroovyBigDecimalTransformation() {
-        client().prepareIndex("test", "doc", "1").setSource("foo", 5).setRefresh(true).get();
+        client().prepareIndex("test", "doc", "1").setSource("foo", 5).setRefreshPolicy(IMMEDIATE).get();
 
         // Test that something that would usually be a BigDecimal is transformed into a Double
         assertScript("def n = 1.23; assert n instanceof Double; return n;");
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java
index 7be073e32af..faea69b870f 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java
@@ -28,6 +28,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.apache.lucene.util.TestUtil.randomSimpleString;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -64,7 +65,7 @@ public class UpdateByQueryWhileModifyingTests extends ReindexTestCase {
                 assertEquals(value.get(), get.getSource().get("test"));
                 value.set(randomSimpleString(random()));
                 IndexRequestBuilder index = client().prepareIndex("test", "test", "test").setSource("test", value.get())
-                        .setRefresh(true);
+                        .setRefreshPolicy(IMMEDIATE);
                 /*
                  * Update by query increments the version number so concurrent
                  * indexes might get version conflict exceptions so we just

From 5b94c4a25b8a6322de573d5d9923d5838db748ec Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 3 Jun 2016 15:41:40 -0400
Subject: [PATCH 0723/1311] Fix a race condition in reindex's rethrottle

If you rethrottled the request while is was performing a scroll
request then it wouldn't properly pick up the rethrottle for that
batch. This was causing test failure and might cause issues for
users. The work around is simple though: just issue the rethrottle
again with a slightly faster throttle than the first time.

Caught by:
https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+master+multijob-os-compatibility/os=centos/525/console
---
 .../AbstractAsyncBulkByScrollAction.java      |  71 ++++-------
 .../index/reindex/BulkByScrollTask.java       |  26 ++++-
 .../reindex/AsyncBulkByScrollActionTests.java | 110 ++++++++----------
 .../index/reindex/BulkByScrollTaskTests.java  |  19 ++-
 4 files changed, 111 insertions(+), 115 deletions(-)

diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
index 0921abcb677..ceb00b2f81f 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
@@ -55,7 +55,6 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
@@ -66,7 +65,6 @@ import static java.util.Collections.emptyList;
 import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.action.bulk.BackoffPolicy.exponentialBackoff;
 import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
-import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 import static org.elasticsearch.index.reindex.AbstractBulkByScrollRequest.SIZE_ALL_MATCHES;
 import static org.elasticsearch.rest.RestStatus.CONFLICT;
 import static org.elasticsearch.search.sort.SortBuilders.fieldSort;
@@ -85,7 +83,6 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
 
     private final AtomicLong startTime = new AtomicLong(-1);
     private final AtomicReference<String> scroll = new AtomicReference<>();
-    private final AtomicLong lastBatchStartTime = new AtomicLong(-1);
     private final Set<String> destinationIndices = Collections.newSetFromMap(new ConcurrentHashMap<>());
 
     private final ESLogger logger;
@@ -147,16 +144,17 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
         }
         searchWithRetry(listener -> client.search(firstSearchRequest, listener), (SearchResponse response) -> {
             logger.debug("[{}] documents match query", response.getHits().getTotalHits());
-            onScrollResponse(timeValueSeconds(0), response);
+            onScrollResponse(timeValueNanos(System.nanoTime()), 0, response);
         });
     }
 
     /**
      * Process a scroll response.
-     * @param delay how long to delay processesing the response. This delay is how throttling is applied to the action.
+     * @param lastBatchStartTime the time when the last batch started. Used to calculate the throttling delay.
+     * @param lastBatchSize the size of the last batch. Used to calculate the throttling delay.
      * @param searchResponse the scroll response to process
      */
-    void onScrollResponse(TimeValue delay, SearchResponse searchResponse) {
+    void onScrollResponse(TimeValue lastBatchStartTime, int lastBatchSize, SearchResponse searchResponse) {
         if (task.isCancelled()) {
             finishHim(null);
             return;
@@ -179,7 +177,11 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
         AbstractRunnable prepareBulkRequestRunnable = new AbstractRunnable() {
             @Override
             protected void doRun() throws Exception {
-                prepareBulkRequest(searchResponse);
+                /*
+                 * It is important that the batch start time be calculated from here, scroll response to scroll response. That way the time
+                 * waiting on the scroll doesn't count against this batch in the throttle.
+                 */
+                prepareBulkRequest(timeValueNanos(System.nanoTime()), searchResponse);
             }
 
             @Override
@@ -188,7 +190,7 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
             }
         };
         prepareBulkRequestRunnable = (AbstractRunnable) threadPool.getThreadContext().preserveContext(prepareBulkRequestRunnable);
-        task.delayPrepareBulkRequest(threadPool, delay, prepareBulkRequestRunnable);
+        task.delayPrepareBulkRequest(threadPool, lastBatchStartTime, lastBatchSize, prepareBulkRequestRunnable);
     }
 
     /**
@@ -196,12 +198,11 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
      * delay has been slept. Uses the generic thread pool because reindex is rare enough not to need its own thread pool and because the
      * thread may be blocked by the user script.
      */
-    void prepareBulkRequest(SearchResponse searchResponse) {
+    void prepareBulkRequest(TimeValue thisBatchStartTime, SearchResponse searchResponse) {
         if (task.isCancelled()) {
             finishHim(null);
             return;
         }
-        lastBatchStartTime.set(System.nanoTime());
         SearchHit[] docs = searchResponse.getHits().getHits();
         logger.debug("scroll returned [{}] documents with a scroll id of [{}]", docs.length, searchResponse.getScrollId());
         if (docs.length == 0) {
@@ -222,7 +223,7 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
             /*
              * If we noop-ed the entire batch then just skip to the next batch or the BulkRequest would fail validation.
              */
-            startNextScroll(0);
+            startNextScroll(thisBatchStartTime, 0);
             return;
         }
         request.timeout(mainRequest.getTimeout());
@@ -231,13 +232,13 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
             logger.debug("sending [{}] entry, [{}] bulk request", request.requests().size(),
                     new ByteSizeValue(request.estimatedSizeInBytes()));
         }
-        sendBulkRequest(request);
+        sendBulkRequest(thisBatchStartTime, request);
     }
 
     /**
      * Send a bulk request, handling retries.
      */
-    void sendBulkRequest(BulkRequest request) {
+    void sendBulkRequest(TimeValue thisBatchStartTime, BulkRequest request) {
         if (task.isCancelled()) {
             finishHim(null);
             return;
@@ -245,7 +246,7 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
         bulkRetry.withAsyncBackoff(client, request, new ActionListener<BulkResponse>() {
             @Override
             public void onResponse(BulkResponse response) {
-                onBulkResponse(response);
+                onBulkResponse(thisBatchStartTime, response);
             }
 
             @Override
@@ -258,7 +259,7 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
     /**
      * Processes bulk responses, accounting for failures.
      */
-    void onBulkResponse(BulkResponse response) {
+    void onBulkResponse(TimeValue thisBatchStartTime, BulkResponse response) {
         try {
             List<Failure> failures = new ArrayList<Failure>();
             Set<String> destinationIndicesThisBatch = new HashSet<>();
@@ -306,7 +307,7 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
                 return;
             }
 
-            startNextScroll(response.getItems().length);
+            startNextScroll(thisBatchStartTime, response.getItems().length);
         } catch (Throwable t) {
             finishHim(t);
         }
@@ -318,36 +319,20 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
      * @param lastBatchSize the number of requests sent in the last batch. This is used to calculate the throttling values which are applied
      *        when the scroll returns
      */
-    void startNextScroll(int lastBatchSize) {
+    void startNextScroll(TimeValue lastBatchStartTime, int lastBatchSize) {
         if (task.isCancelled()) {
             finishHim(null);
             return;
         }
-        long earliestNextBatchStartTime = lastBatchStartTime.get() + (long) perfectlyThrottledBatchTime(lastBatchSize);
-        long waitTime = max(0, earliestNextBatchStartTime - System.nanoTime());
         SearchScrollRequest request = new SearchScrollRequest();
         // Add the wait time into the scroll timeout so it won't timeout while we wait for throttling
-        request.scrollId(scroll.get()).scroll(timeValueNanos(firstSearchRequest.scroll().keepAlive().nanos() + waitTime));
+        request.scrollId(scroll.get()).scroll(timeValueNanos(
+                firstSearchRequest.scroll().keepAlive().nanos() + task.throttleWaitTime(lastBatchStartTime, lastBatchSize).nanos()));
         searchWithRetry(listener -> client.searchScroll(request, listener), (SearchResponse response) -> {
-            onScrollResponse(timeValueNanos(max(0, earliestNextBatchStartTime - System.nanoTime())), response);
+            onScrollResponse(lastBatchStartTime, lastBatchSize, response);
         });
     }
 
-    /**
-     * How many nanoseconds should a batch of lastBatchSize have taken if it were perfectly throttled? Package private for testing.
-     */
-    float perfectlyThrottledBatchTime(int lastBatchSize) {
-        if (task.getRequestsPerSecond() == Float.POSITIVE_INFINITY) {
-            return 0;
-        }
-        //       requests
-        // ------------------- == seconds
-        // request per seconds
-        float targetBatchTimeInSeconds = lastBatchSize / task.getRequestsPerSecond();
-        // nanoseconds per seconds * seconds == nanoseconds
-        return TimeUnit.SECONDS.toNanos(1) * targetBatchTimeInSeconds;
-    }
-
     private void recordFailure(Failure failure, List<Failure> failures) {
         if (failure.getStatus() == CONFLICT) {
             task.countVersionConflict();
@@ -453,20 +438,6 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
         this.scroll.set(scroll);
     }
 
-    /**
-     * Set the last batch's start time. Exists entirely for testing.
-     */
-    void setLastBatchStartTime(long newValue) {
-        lastBatchStartTime.set(newValue);
-    }
-
-    /**
-     * Get the last batch's start time. Exists entirely for testing.
-     */
-    long getLastBatchStartTime() {
-        return lastBatchStartTime.get();
-    }
-
     /**
      * Wraps a backoffPolicy in another policy that counts the number of backoffs acquired. Used to count bulk backoffs.
      */
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
index 5197cdae079..e7a74b7a562 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
@@ -49,7 +49,7 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
  * Task storing information about a currently running BulkByScroll request.
  */
 public class BulkByScrollTask extends CancellableTask {
-    private static final ESLogger logger = ESLoggerFactory.getLogger(BulkByScrollTask.class.getName());
+    private static final ESLogger logger = ESLoggerFactory.getLogger(BulkByScrollTask.class.getPackage().getName());
 
     /**
      * The total number of documents this request will process. 0 means we don't yet know or, possibly, there are actually 0 documents
@@ -403,9 +403,11 @@ public class BulkByScrollTask extends CancellableTask {
      * Schedule prepareBulkRequestRunnable to run after some delay. This is where throttling plugs into reindexing so the request can be
      * rescheduled over and over again.
      */
-    void delayPrepareBulkRequest(ThreadPool threadPool, TimeValue delay, AbstractRunnable prepareBulkRequestRunnable) {
+    void delayPrepareBulkRequest(ThreadPool threadPool, TimeValue lastBatchStartTime, int lastBatchSize,
+            AbstractRunnable prepareBulkRequestRunnable) {
         // Synchronize so we are less likely to schedule the same request twice.
         synchronized (delayedPrepareBulkRequestReference) {
+            TimeValue delay = throttleWaitTime(lastBatchStartTime, lastBatchSize);
             AbstractRunnable oneTime = new AbstractRunnable() {
                 private final AtomicBoolean hasRun = new AtomicBoolean(false);
 
@@ -426,6 +428,26 @@ public class BulkByScrollTask extends CancellableTask {
         }
     }
 
+    TimeValue throttleWaitTime(TimeValue lastBatchStartTime, int lastBatchSize) {
+        long earliestNextBatchStartTime = lastBatchStartTime.nanos() + (long) perfectlyThrottledBatchTime(lastBatchSize);
+        return timeValueNanos(max(0, earliestNextBatchStartTime - System.nanoTime()));
+    }
+
+    /**
+     * How many nanoseconds should a batch of lastBatchSize have taken if it were perfectly throttled? Package private for testing.
+     */
+    float perfectlyThrottledBatchTime(int lastBatchSize) {
+        if (requestsPerSecond == Float.POSITIVE_INFINITY) {
+            return 0;
+        }
+        //       requests
+        // ------------------- == seconds
+        // request per seconds
+        float targetBatchTimeInSeconds = lastBatchSize / requestsPerSecond;
+        // nanoseconds per seconds * seconds == nanoseconds
+        return TimeUnit.SECONDS.toNanos(1) * targetBatchTimeInSeconds;
+    }
+
     private void setRequestsPerSecond(float requestsPerSecond) {
         this.requestsPerSecond = requestsPerSecond;
     }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
index 2af63deb3a9..9b094a9e2d3 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
@@ -97,19 +97,18 @@ import static java.util.Collections.emptySet;
 import static java.util.Collections.singleton;
 import static org.apache.lucene.util.TestUtil.randomSimpleString;
 import static org.elasticsearch.action.bulk.BackoffPolicy.constantBackoff;
-import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
-import static org.elasticsearch.common.unit.TimeValue.timeValueMinutes;
+import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
-import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.emptyCollectionOf;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
 public class AsyncBulkByScrollActionTests extends ESTestCase {
     private MyMockClient client;
@@ -189,7 +188,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         client.scrollsToReject = randomIntBetween(0, testRequest.getMaxRetries() - 1);
         DummyAbstractAsyncBulkByScrollAction action = new DummyActionWithoutBackoff();
         action.setScroll(scrollId());
-        action.startNextScroll(0);
+        action.startNextScroll(timeValueNanos(System.nanoTime()), 0);
         assertBusy(() -> assertEquals(client.scrollsToReject + 1, client.scrollAttempts.get()));
         if (listener.isDone()) {
             Object result = listener.get();
@@ -203,7 +202,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         client.scrollsToReject = testRequest.getMaxRetries() + randomIntBetween(1, 100);
         DummyAbstractAsyncBulkByScrollAction action = new DummyActionWithoutBackoff();
         action.setScroll(scrollId());
-        action.startNextScroll(0);
+        action.startNextScroll(timeValueNanos(System.nanoTime()), 0);
         assertBusy(() -> assertEquals(testRequest.getMaxRetries() + 1, client.scrollAttempts.get()));
         assertBusy(() -> assertTrue(listener.isDone()));
         ExecutionException e = expectThrows(ExecutionException.class, () -> listener.get());
@@ -219,7 +218,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         long total = randomIntBetween(0, Integer.MAX_VALUE);
         InternalSearchHits hits = new InternalSearchHits(null, total, 0);
         InternalSearchResponse searchResponse = new InternalSearchResponse(hits, null, null, null, false, false);
-        new DummyAbstractAsyncBulkByScrollAction().onScrollResponse(timeValueSeconds(0),
+        new DummyAbstractAsyncBulkByScrollAction().onScrollResponse(timeValueSeconds(0), 0,
                 new SearchResponse(searchResponse, scrollId(), 5, 4, randomLong(), null));
         assertEquals(total, testTask.getStatus().getTotal());
     }
@@ -230,24 +229,17 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
     public void testScrollResponseBatchingBehavior() throws Exception {
         int maxBatches = randomIntBetween(0, 100);
         for (int batches = 1; batches < maxBatches; batches++) {
-            long now = System.nanoTime();
             InternalSearchHit hit = new InternalSearchHit(0, "id", new Text("type"), emptyMap());
             InternalSearchHits hits = new InternalSearchHits(new InternalSearchHit[] { hit }, 0, 0);
             InternalSearchResponse searchResponse = new InternalSearchResponse(hits, null, null, null, false, false);
             DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction();
-            action.onScrollResponse(timeValueSeconds(0),
+            action.onScrollResponse(timeValueNanos(System.nanoTime()), 0,
                     new SearchResponse(searchResponse, scrollId(), 5, 4, randomLong(), null));
 
             // Use assert busy because the update happens on another thread
             final int expectedBatches = batches;
             assertBusy(() -> assertEquals(expectedBatches, testTask.getStatus().getBatches()));
 
-            /*
-             * While we're here we can check that getting a scroll response sets the last scroll start time which makes sure the wait time
-             * isn't counted as time that the last batch took.
-             */
-            assertThat(action.getLastBatchStartTime(), greaterThanOrEqualTo(now));
-
             /*
              * Also while we're here check that we preserved the headers from the last request. assertBusy because no requests might have
              * come in yet.
@@ -297,7 +289,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
                 }
                 responses[i] = new BulkItemResponse(i, opType, new IndexResponse(shardId, "type", "id" + i, randomInt(), createdResponse));
             }
-            new DummyAbstractAsyncBulkByScrollAction().onBulkResponse(new BulkResponse(responses, 0));
+            new DummyAbstractAsyncBulkByScrollAction().onBulkResponse(timeValueNanos(System.nanoTime()), new BulkResponse(responses, 0));
             assertEquals(versionConflicts, testTask.getStatus().getVersionConflicts());
             assertEquals(updated, testTask.getStatus().getUpdated());
             assertEquals(created, testTask.getStatus().getCreated());
@@ -310,20 +302,22 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
      * Mimicks a ThreadPool rejecting execution of the task.
      */
     public void testThreadPoolRejectionsAbortRequest() throws Exception {
-        TimeValue expectedDelay = parseTimeValue(randomPositiveTimeValue(), "test");
+        testTask.rethrottle(1);
         threadPool.shutdown();
         threadPool = new TestThreadPool(getTestName()) {
             @Override
             public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable command) {
-                assertEquals(expectedDelay, delay); // While we're here we can check that the sleep made it through
+                // While we're here we can check that the sleep made it through
+                assertThat(delay.nanos(), greaterThan(0L));
+                assertThat(delay.seconds(), lessThanOrEqualTo(10L));
                 ((AbstractRunnable) command).onRejection(new EsRejectedExecutionException("test"));
                 return null;
             }
         };
         InternalSearchHits hits = new InternalSearchHits(null, 0, 0);
         InternalSearchResponse searchResponse = new InternalSearchResponse(hits, null, null, null, false, false);
-        new DummyAbstractAsyncBulkByScrollAction()
-                .onScrollResponse(expectedDelay, new SearchResponse(searchResponse, scrollId(), 5, 4, randomLong(), null));
+        new DummyAbstractAsyncBulkByScrollAction().onScrollResponse(timeValueNanos(System.nanoTime()), 10,
+                new SearchResponse(searchResponse, scrollId(), 5, 4, randomLong(), null));
         try {
             listener.get();
             fail("Expected a failure");
@@ -343,7 +337,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
     public void testShardFailuresAbortRequest() throws Exception {
         ShardSearchFailure shardFailure = new ShardSearchFailure(new RuntimeException("test"));
         InternalSearchResponse internalResponse = new InternalSearchResponse(null, null, null, null, false, null);
-        new DummyAbstractAsyncBulkByScrollAction().onScrollResponse(timeValueSeconds(0),
+        new DummyAbstractAsyncBulkByScrollAction().onScrollResponse(timeValueNanos(System.nanoTime()), 0,
                 new SearchResponse(internalResponse, scrollId(), 5, 4, randomLong(), new ShardSearchFailure[] { shardFailure }));
         BulkIndexByScrollResponse response = listener.get();
         assertThat(response.getIndexingFailures(), emptyCollectionOf(Failure.class));
@@ -358,7 +352,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
      */
     public void testSearchTimeoutsAbortRequest() throws Exception {
         InternalSearchResponse internalResponse = new InternalSearchResponse(null, null, null, null, true, null);
-        new DummyAbstractAsyncBulkByScrollAction().onScrollResponse(timeValueSeconds(0),
+        new DummyAbstractAsyncBulkByScrollAction().onScrollResponse(timeValueNanos(System.nanoTime()), 0,
                 new SearchResponse(internalResponse, scrollId(), 5, 4, randomLong(), new ShardSearchFailure[0]));
         BulkIndexByScrollResponse response = listener.get();
         assertThat(response.getIndexingFailures(), emptyCollectionOf(Failure.class));
@@ -368,14 +362,14 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         assertThat(client.scrollsCleared, contains(scrollId));
     }
 
-
     /**
      * Mimicks bulk indexing failures.
      */
     public void testBulkFailuresAbortRequest() throws Exception {
         Failure failure = new Failure("index", "type", "id", new RuntimeException("test"));
         DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction();
-        action.onBulkResponse(new BulkResponse(new BulkItemResponse[] {new BulkItemResponse(0, "index", failure)}, randomLong()));
+        BulkResponse bulkResponse = new BulkResponse(new BulkItemResponse[] {new BulkItemResponse(0, "index", failure)}, randomLong());
+        action.onBulkResponse(timeValueNanos(System.nanoTime()), bulkResponse);
         BulkIndexByScrollResponse response = listener.get();
         assertThat(response.getIndexingFailures(), contains(failure));
         assertThat(response.getSearchFailures(), emptyCollectionOf(ShardSearchFailure.class));
@@ -394,15 +388,12 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         };
         InternalSearchHit hit = new InternalSearchHit(0, "id", new Text("type"), emptyMap());
         InternalSearchHits hits = new InternalSearchHits(new InternalSearchHit[] {hit}, 0, 0);
-        InternalSearchResponse searchResponse = new InternalSearchResponse(hits, null, null, null, false, false);
-        action.onScrollResponse(timeValueSeconds(0), new SearchResponse(searchResponse, scrollId(), 5, 4, randomLong(), null));
-        try {
-            listener.get();
-            fail("Expected failure.");
-        } catch (ExecutionException e) {
-            assertThat(e.getCause(), instanceOf(RuntimeException.class));
-            assertThat(e.getCause().getMessage(), equalTo("surprise"));
-        }
+        InternalSearchResponse internalResponse = new InternalSearchResponse(hits, null, null, null, false, false);
+        SearchResponse searchResponse = new SearchResponse(internalResponse, scrollId(), 5, 4, randomLong(), null);
+        action.onScrollResponse(timeValueNanos(System.nanoTime()), 0, searchResponse);
+        ExecutionException e = expectThrows(ExecutionException.class, () -> listener.get());
+        assertThat(e.getCause(), instanceOf(RuntimeException.class));
+        assertThat(e.getCause().getMessage(), equalTo("surprise"));
     }
 
     /**
@@ -426,17 +417,6 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         assertEquals(testRequest.getMaxRetries(), testTask.getStatus().getBulkRetries());
     }
 
-    public void testPerfectlyThrottledBatchTime() {
-        DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction();
-        testRequest.setRequestsPerSecond(Float.POSITIVE_INFINITY);
-        assertThat((double) action.perfectlyThrottledBatchTime(randomInt()), closeTo(0f, 0f));
-
-        int total = between(0, 1000000);
-        testTask.rethrottle(1);
-        assertThat((double) action.perfectlyThrottledBatchTime(total),
-                closeTo(TimeUnit.SECONDS.toNanos(total), TimeUnit.SECONDS.toNanos(1)));
-    }
-
     public void testScrollDelay() throws Exception {
         /*
          * Replace the thread pool with one that will save the delay sent for the command. We'll use that to check that we used a proper
@@ -460,12 +440,10 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         // Set the base for the scroll to wait - this is added to the figure we calculate below
         firstSearchRequest.scroll(timeValueSeconds(10));
 
-        // We'd like to get about 1 request a second
+        // Set throttle to 1 request per second to make the math simpler
         testTask.rethrottle(1f);
-        // Make the last scroll look nearly instant
-        action.setLastBatchStartTime(System.nanoTime());
-        // The last batch had 100 documents
-        action.startNextScroll(100);
+        // Make the last batch look nearly instant but have 100 documents
+        action.startNextScroll(timeValueNanos(System.nanoTime()), 100);
 
         // So the next request is going to have to wait an extra 100 seconds or so (base was 10 seconds, so 110ish)
         assertThat(client.lastScroll.get().request.scroll().keepAlive().seconds(), either(equalTo(110L)).or(equalTo(109L)));
@@ -473,11 +451,20 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         // Now we can simulate a response and check the delay that we used for the task
         InternalSearchHit hit = new InternalSearchHit(0, "id", new Text("type"), emptyMap());
         InternalSearchHits hits = new InternalSearchHits(new InternalSearchHit[] { hit }, 0, 0);
-        InternalSearchResponse searchResponse = new InternalSearchResponse(hits, null, null, null, false, false);
-        client.lastScroll.get().listener.onResponse(new SearchResponse(searchResponse, scrollId(), 5, 4, randomLong(), null));
+        InternalSearchResponse internalResponse = new InternalSearchResponse(hits, null, null, null, false, false);
+        SearchResponse searchResponse = new SearchResponse(internalResponse, scrollId(), 5, 4, randomLong(), null);
 
-        // The delay is still 100ish seconds because there hasn't been much time between when we requested the bulk and when we got it.
-        assertThat(capturedDelay.get().seconds(), either(equalTo(100L)).or(equalTo(99L)));
+        if (randomBoolean()) {
+            client.lastScroll.get().listener.onResponse(searchResponse);
+            // The delay is still 100ish seconds because there hasn't been much time between when we requested the bulk and when we got it.
+            assertThat(capturedDelay.get().seconds(), either(equalTo(100L)).or(equalTo(99L)));
+        } else {
+            // Let's rethrottle between the starting the scroll and getting the response
+            testTask.rethrottle(10f);
+            client.lastScroll.get().listener.onResponse(searchResponse);
+            // The delay uses the new throttle
+            assertThat(capturedDelay.get().seconds(), either(equalTo(10L)).or(equalTo(9L)));
+        }
 
         // Running the command ought to increment the delay counter on the task.
         capturedCommand.get().run();
@@ -501,7 +488,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         CountDownLatch successLatch = new CountDownLatch(1);
         DummyAbstractAsyncBulkByScrollAction action = new DummyActionWithoutBackoff() {
             @Override
-            void startNextScroll(int lastBatchSize) {
+            void startNextScroll(TimeValue lastBatchStartTime, int lastBatchSize) {
                 successLatch.countDown();
             }
         };
@@ -509,7 +496,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         for (int i = 0; i < size + 1; i++) {
             request.add(new IndexRequest("index", "type", "id" + i));
         }
-        action.sendBulkRequest(request);
+        action.sendBulkRequest(timeValueNanos(System.nanoTime()), request);
         if (failWithRejection) {
             BulkIndexByScrollResponse response = listener.get();
             assertThat(response.getIndexingFailures(), hasSize(1));
@@ -576,22 +563,23 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
 
     public void testCancelBeforeScrollResponse() throws Exception {
         // We bail so early we don't need to pass in a half way valid response.
-        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.onScrollResponse(timeValueSeconds(0), null));
+        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.onScrollResponse(timeValueNanos(System.nanoTime()), 1,
+                null));
     }
 
     public void testCancelBeforeSendBulkRequest() throws Exception {
         // We bail so early we don't need to pass in a half way valid request.
-        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.sendBulkRequest(null));
+        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.sendBulkRequest(timeValueNanos(System.nanoTime()), null));
     }
 
     public void testCancelBeforeOnBulkResponse() throws Exception {
         // We bail so early we don't need to pass in a half way valid response.
         cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) ->
-                action.onBulkResponse(new BulkResponse(new BulkItemResponse[0], 0)));
+                action.onBulkResponse(timeValueNanos(System.nanoTime()), new BulkResponse(new BulkItemResponse[0], 0)));
     }
 
     public void testCancelBeforeStartNextScroll() throws Exception {
-        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.startNextScroll(0));
+        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.startNextScroll(timeValueNanos(System.nanoTime()), 0));
     }
 
     public void testCancelBeforeStartNormalTermination() throws Exception {
@@ -640,7 +628,9 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         InternalSearchHits hits = new InternalSearchHits(null, total, 0);
         InternalSearchResponse searchResponse = new InternalSearchResponse(hits, null, null, null, false, false);
         // Use a long delay here so the test will time out if the cancellation doesn't reschedule the throttled task
-        action.onScrollResponse(timeValueMinutes(10), new SearchResponse(searchResponse, scrollId(), 5, 4, randomLong(), null));
+        SearchResponse scrollResponse = new SearchResponse(searchResponse, scrollId(), 5, 4, randomLong(), null);
+        testTask.rethrottle(1);
+        action.onScrollResponse(timeValueNanos(System.nanoTime()), 1000, scrollResponse);
 
         // Now that we've got our cancel we'll just verify that it all came through all right
         assertEquals(reason, listener.get(10, TimeUnit.SECONDS).getReasonCancelled());
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
index fd1a17a439d..05699c6f7af 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
@@ -42,8 +42,10 @@ import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
+import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 import static org.hamcrest.Matchers.both;
+import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -165,10 +167,11 @@ public class BulkByScrollTaskTests extends ESTestCase {
          * We never end up waiting this long because the test rethrottles over and over again, ratcheting down the delay a random amount
          * each time.
          */
-        float originalRequestsPerSecond = (float) randomDoubleBetween(0, 10000, true);
+        float originalRequestsPerSecond = (float) randomDoubleBetween(1, 10000, true);
         task.rethrottle(originalRequestsPerSecond);
         TimeValue maxDelay = timeValueSeconds(between(1, 5));
         assertThat(maxDelay.nanos(), greaterThanOrEqualTo(0L));
+        int batchSizeForMaxDelay = (int) (maxDelay.seconds() * originalRequestsPerSecond);
         ThreadPool threadPool = new TestThreadPool(getTestName()) {
             @Override
             public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable command) {
@@ -177,7 +180,7 @@ public class BulkByScrollTaskTests extends ESTestCase {
             }
         };
         try {
-            task.delayPrepareBulkRequest(threadPool, maxDelay, new AbstractRunnable() {
+            task.delayPrepareBulkRequest(threadPool, timeValueNanos(System.nanoTime()), batchSizeForMaxDelay, new AbstractRunnable() {
                 @Override
                 protected void doRun() throws Exception {
                     boolean oldValue = done.getAndSet(true);
@@ -263,7 +266,7 @@ public class BulkByScrollTaskTests extends ESTestCase {
         };
         try {
             // Have the task use the thread pool to delay a task that does nothing
-            task.delayPrepareBulkRequest(threadPool, timeValueSeconds(0), new AbstractRunnable() {
+            task.delayPrepareBulkRequest(threadPool, timeValueSeconds(0), 1, new AbstractRunnable() {
                 @Override
                 protected void doRun() throws Exception {
                 }
@@ -284,4 +287,14 @@ public class BulkByScrollTaskTests extends ESTestCase {
         task.getStatus().toXContent(builder, ToXContent.EMPTY_PARAMS);
         assertThat(builder.string(), containsString("\"requests_per_second\":\"unlimited\""));
     }
+
+    public void testPerfectlyThrottledBatchTime() {
+        task.rethrottle(Float.POSITIVE_INFINITY);
+        assertThat((double) task.perfectlyThrottledBatchTime(randomInt()), closeTo(0f, 0f));
+
+        int total = between(0, 1000000);
+        task.rethrottle(1);
+        assertThat((double) task.perfectlyThrottledBatchTime(total),
+                closeTo(TimeUnit.SECONDS.toNanos(total), TimeUnit.SECONDS.toNanos(1)));
+    }
 }

From 2437313e4ec4d7b036978fcab64ab485e09a83fd Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 3 Jun 2016 16:05:46 -0400
Subject: [PATCH 0724/1311] Remove extra logging

The test shouldn't be failing any more.
---
 .../java/org/elasticsearch/index/reindex/RethrottleTests.java | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
index acbc31bff99..38f9dbc6d97 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RethrottleTests.java
@@ -21,7 +21,6 @@ package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.ListenableActionFuture;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 
 import static org.hamcrest.Matchers.hasSize;
 
@@ -30,9 +29,6 @@ import static org.hamcrest.Matchers.hasSize;
  * too but this is the only place that tests running against multiple nodes so it is the only integration tests that checks for
  * serialization.
  */
-// Extra logging in case of failure. We couldn't explain the last failure:
-// https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+master+g1gc/359/consoleFull
-@TestLogging("_root:DEBUG")
 public class RethrottleTests extends ReindexTestCase {
 
     public void testReindex() throws Exception {

From bd1af3450654140bff603a587ea575fbda6072fb Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 8 Jun 2016 14:00:15 -0400
Subject: [PATCH 0725/1311] [reindex] Extract runnable to inner class

Makes it more readable
---
 .../index/reindex/BulkByScrollTask.java       | 44 ++++++++++++-------
 1 file changed, 27 insertions(+), 17 deletions(-)

diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
index e7a74b7a562..7af32740be2 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
@@ -408,23 +408,8 @@ public class BulkByScrollTask extends CancellableTask {
         // Synchronize so we are less likely to schedule the same request twice.
         synchronized (delayedPrepareBulkRequestReference) {
             TimeValue delay = throttleWaitTime(lastBatchStartTime, lastBatchSize);
-            AbstractRunnable oneTime = new AbstractRunnable() {
-                private final AtomicBoolean hasRun = new AtomicBoolean(false);
-
-                @Override
-                protected void doRun() throws Exception {
-                    // Paranoia to prevent furiously rethrottling from running the command multiple times. Without this we totally can.
-                    if (hasRun.compareAndSet(false, true)) {
-                        prepareBulkRequestRunnable.run();
-                    }
-                }
-
-                @Override
-                public void onFailure(Throwable t) {
-                    prepareBulkRequestRunnable.onFailure(t);
-                }
-            };
-            delayedPrepareBulkRequestReference.set(new DelayedPrepareBulkRequest(threadPool, getRequestsPerSecond(), delay, oneTime));
+            delayedPrepareBulkRequestReference.set(new DelayedPrepareBulkRequest(threadPool, getRequestsPerSecond(),
+                    delay, new RunOnce(prepareBulkRequestRunnable)));
         }
     }
 
@@ -540,4 +525,29 @@ public class BulkByScrollTask extends CancellableTask {
             return timeValueNanos(round(remainingDelay * requestsPerSecond / newRequestsPerSecond));
         }
     }
+
+    /**
+     * Runnable that can only be run one time. This is paranoia to prevent furiously rethrottling from running the command multiple times.
+     * Without it the command would be run multiple times.
+     */
+    private static class RunOnce extends AbstractRunnable {
+        private final AtomicBoolean hasRun = new AtomicBoolean(false);
+        private final AbstractRunnable delegate;
+
+        public RunOnce(AbstractRunnable delegate) {
+            this.delegate = delegate;
+        }
+
+        @Override
+        protected void doRun() throws Exception {
+            if (hasRun.compareAndSet(false, true)) {
+                delegate.run();
+            }
+        }
+
+        @Override
+        public void onFailure(Throwable t) {
+            delegate.onFailure(t);
+        }
+    }
 }

From 06d26635de9089aa9707dba79e9ddc66924b015b Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 8 Jun 2016 14:03:48 -0400
Subject: [PATCH 0726/1311] Packaging tests: ingest-grok is now ingest-common

---
 .../packaging/scripts/module_and_plugin_test_cases.bash       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
index 32118fda389..ac54bf3b664 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
@@ -228,8 +228,8 @@ fi
     install_and_check_plugin ingest geoip geoip2-*.jar jackson-annotations-*.jar jackson-databind-*.jar maxmind-db-*.jar
 }
 
-@test "[$GROUP] check ingest-grok module" {
-    check_module ingest-grok jcodings-*.jar joni-*.jar
+@test "[$GROUP] check ingest-common module" {
+    check_module ingest-common jcodings-*.jar joni-*.jar
 }
 
 @test "[$GROUP] check lang-expression module" {

From 0c6d19c40ca6f375a7d867361c6b5b528b83551a Mon Sep 17 00:00:00 2001
From: Areek Zillur <areek.zillur@elasticsearch.com>
Date: Wed, 8 Jun 2016 14:16:06 -0400
Subject: [PATCH 0727/1311] add body support for create index request

---
 .../indices/rollover/RolloverRequest.java     | 40 +++++++--
 .../rollover/RolloverRequestBuilder.java      | 17 ++++
 .../rollover/TransportRolloverAction.java     | 19 ++--
 .../admin/indices/rollover/RolloverIT.java    | 27 ++++++
 .../rollover/RolloverRequestTests.java        | 90 +++++++++++++++++++
 .../TransportRolloverActionTests.java         | 20 ++++-
 6 files changed, 201 insertions(+), 12 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java

diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
index dd1facdb6e2..19659171a67 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
@@ -21,6 +21,7 @@ package org.elasticsearch.action.admin.indices.rollover;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.common.ParseField;
@@ -37,6 +38,8 @@ import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
 import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
@@ -50,13 +53,26 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
     private String sourceAlias;
     private boolean simulate;
     private Set<Condition> conditions = new HashSet<>(2);
+    private CreateIndexRequest createIndexRequest = new CreateIndexRequest("_na_");
 
-    public static ObjectParser<Set<Condition>, ParseFieldMatcherSupplier> TLP_PARSER =
+    public static ObjectParser<RolloverRequest, ParseFieldMatcherSupplier> PARSER =
         new ObjectParser<>("conditions", null);
     static {
-        TLP_PARSER.declareField((parser, conditions, parseFieldMatcherSupplier) ->
-        Condition.PARSER.parse(parser, conditions,  () -> ParseFieldMatcher.EMPTY),
+        PARSER.declareField((parser, request, parseFieldMatcherSupplier) ->
+            Condition.PARSER.parse(parser, request.conditions, parseFieldMatcherSupplier),
             new ParseField("conditions"), ObjectParser.ValueType.OBJECT);
+        PARSER.declareField((parser, request, parseFieldMatcherSupplier) ->
+            request.createIndexRequest.settings(parser.map()),
+            new ParseField("settings"), ObjectParser.ValueType.OBJECT);
+        PARSER.declareField((parser, request, parseFieldMatcherSupplier) -> {
+            for (Map.Entry<String, Object> mappingsEntry : parser.map().entrySet()) {
+                request.createIndexRequest.mapping(mappingsEntry.getKey(),
+                    (Map<String, Object>) mappingsEntry.getValue());
+            }
+        }, new ParseField("mappings"), ObjectParser.ValueType.OBJECT);
+        PARSER.declareField((parser, request, parseFieldMatcherSupplier) ->
+            request.createIndexRequest.aliases(parser.map()),
+            new ParseField("aliases"), ObjectParser.ValueType.OBJECT);
     }
 
     RolloverRequest() {}
@@ -67,10 +83,13 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
 
     @Override
     public ActionRequestValidationException validate() {
-        ActionRequestValidationException validationException = null;
+        ActionRequestValidationException validationException = createIndexRequest == null ? null : createIndexRequest.validate();
         if (sourceAlias == null) {
             validationException = addValidationError("source alias is missing", validationException);
         }
+        if (createIndexRequest == null) {
+            validationException = addValidationError("create index request is missing", validationException);
+        }
         return validationException;
     }
 
@@ -83,6 +102,8 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
         for (int i = 0; i < size; i++) {
             this.conditions.add(in.readNamedWriteable(Condition.class));
         }
+        createIndexRequest = new CreateIndexRequest();
+        createIndexRequest.readFrom(in);
     }
 
     @Override
@@ -94,6 +115,7 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
         for (Condition condition : conditions) {
             out.writeNamedWriteable(condition);
         }
+        createIndexRequest.writeTo(out);
     }
 
     @Override
@@ -134,11 +156,19 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
         return sourceAlias;
     }
 
+    public CreateIndexRequest getCreateIndexRequest() {
+        return createIndexRequest;
+    }
+
+    public void setCreateIndexRequest(CreateIndexRequest createIndexRequest) {
+        this.createIndexRequest = Objects.requireNonNull(createIndexRequest, "create index request must not be null");;
+    }
+
     public void source(BytesReference source) {
         XContentType xContentType = XContentFactory.xContentType(source);
         if (xContentType != null) {
             try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(source)) {
-                TLP_PARSER.parse(parser, this.conditions, () -> ParseFieldMatcher.EMPTY);
+                PARSER.parse(parser, this, () -> ParseFieldMatcher.EMPTY);
             } catch (IOException e) {
                 throw new ElasticsearchParseException("failed to parse source for rollover index", e);
             }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
index ea495bc6eba..bef680110ab 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
@@ -18,8 +18,10 @@
  */
 package org.elasticsearch.action.admin.indices.rollover;
 
+import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 
 
@@ -51,4 +53,19 @@ public class RolloverRequestBuilder extends MasterNodeOperationRequestBuilder<Ro
         this.request.simulate(simulate);
         return this;
     }
+
+    public RolloverRequestBuilder settings(Settings settings) {
+        this.request.getCreateIndexRequest().settings(settings);
+        return this;
+    }
+
+    public RolloverRequestBuilder alias(Alias alias) {
+        this.request.getCreateIndexRequest().alias(alias);
+        return this;
+    }
+
+    public RolloverRequestBuilder mapping(String type, String source) {
+        this.request.getCreateIndexRequest().mapping(type, source);
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
index 49ff6f1f23e..a3ff8fe23cd 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -22,6 +22,7 @@ package org.elasticsearch.action.admin.indices.rollover;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesClusterStateUpdateRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -117,9 +118,7 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
                             new RolloverResponse(sourceIndexName, rolloverIndexName, conditionResults, false, true,
                                 createRolloverIndex);
                         if (createRolloverIndex) {
-                            CreateIndexClusterStateUpdateRequest updateRequest =
-                                prepareCreateIndexRequest(rolloverIndexName, rolloverRequest);
-                            createIndexService.createIndex(updateRequest,
+                            createIndexService.createIndex(prepareCreateIndexRequest(rolloverIndexName, rolloverRequest),
                                 new ActionListener<ClusterStateUpdateResponse>() {
                                     @Override
                                     public void onResponse(ClusterStateUpdateResponse response) {
@@ -216,10 +215,18 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
 
     static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final String targetIndexName,
                                                                           final RolloverRequest rolloverRequest) {
-        return new CreateIndexClusterStateUpdateRequest(rolloverRequest,
+
+        final CreateIndexRequest createIndexRequest = rolloverRequest.getCreateIndexRequest();
+        createIndexRequest.cause("rollover_index");
+        createIndexRequest.index(targetIndexName);
+        return new CreateIndexClusterStateUpdateRequest(createIndexRequest,
             "rollover_index", targetIndexName, true)
-            .ackTimeout(rolloverRequest.timeout())
-            .masterNodeTimeout(rolloverRequest.masterNodeTimeout());
+            .ackTimeout(createIndexRequest.timeout())
+            .masterNodeTimeout(createIndexRequest.masterNodeTimeout())
+            .settings(createIndexRequest.settings())
+            .aliases(createIndexRequest.aliases())
+            .customs(createIndexRequest.customs())
+            .mappings(createIndexRequest.mappings());
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
index 75a20d05682..bf28bdfdbc8 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
@@ -22,6 +22,7 @@ package org.elasticsearch.action.admin.indices.rollover;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESIntegTestCase;
 
@@ -67,6 +68,32 @@ public class RolloverIT extends ESIntegTestCase {
         assertTrue(newIndex.getAliases().containsKey("test_alias"));
     }
 
+    public void testRolloverWithIndexSettings() throws Exception {
+        assertAcked(prepareCreate("test_index-2").addAlias(new Alias("test_alias")).get());
+        index("test_index-2", "type1", "1", "field", "value");
+        flush("test_index-2");
+        final Settings settings = Settings.builder()
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .build();
+        final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias")
+            .settings(settings).alias(new Alias("extra_alias")).get();
+        assertThat(response.getOldIndex(), equalTo("test_index-2"));
+        assertThat(response.getNewIndex(), equalTo("test_index-3"));
+        assertThat(response.isSimulate(), equalTo(false));
+        assertThat(response.isRolledOver(), equalTo(true));
+        assertThat(response.isRolloverIndexCreated(), equalTo(true));
+        assertThat(response.getConditionStatus().size(), equalTo(0));
+        final ClusterState state = client().admin().cluster().prepareState().get().getState();
+        final IndexMetaData oldIndex = state.metaData().index("test_index-2");
+        assertFalse(oldIndex.getAliases().containsKey("test_alias"));
+        final IndexMetaData newIndex = state.metaData().index("test_index-3");
+        assertThat(newIndex.getNumberOfShards(), equalTo(1));
+        assertThat(newIndex.getNumberOfReplicas(), equalTo(0));
+        assertTrue(newIndex.getAliases().containsKey("test_alias"));
+        assertTrue(newIndex.getAliases().containsKey("extra_alias"));
+    }
+
     public void testRolloverSimulate() throws Exception {
         assertAcked(prepareCreate("test_index-1").addAlias(new Alias("test_alias")).get());
         index("test_index-1", "type1", "1", "field", "value");
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java
new file mode 100644
index 00000000000..754e4c419bd
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.rollover;
+
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class RolloverRequestTests extends ESTestCase {
+
+    public void testConditionsParsing() throws Exception {
+        final RolloverRequest request = new RolloverRequest(randomAsciiOfLength(10));
+        final XContentBuilder builder = XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("conditions")
+                .field("max_age", "10d")
+                .field("max_docs", 100)
+            .endObject()
+            .endObject();
+        request.source(builder.bytes());
+        Set<Condition> conditions = request.getConditions();
+        assertThat(conditions.size(), equalTo(2));
+        for (Condition condition : conditions) {
+            if (condition instanceof MaxAgeCondition) {
+                MaxAgeCondition maxAgeCondition = (MaxAgeCondition) condition;
+                assertThat(maxAgeCondition.value.getMillis(), equalTo(TimeValue.timeValueHours(24 * 10).getMillis()));
+            } else if (condition instanceof MaxDocsCondition) {
+                MaxDocsCondition maxDocsCondition = (MaxDocsCondition) condition;
+                assertThat(maxDocsCondition.value, equalTo(100L));
+            } else {
+                fail("unexpected condition " + condition);
+            }
+        }
+    }
+
+    public void testParsingWithIndexSettings() throws Exception {
+        final RolloverRequest request = new RolloverRequest(randomAsciiOfLength(10));
+        final XContentBuilder builder = XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("conditions")
+                .field("max_age", "10d")
+                .field("max_docs", 100)
+            .endObject()
+            .startObject("mappings")
+                .startObject("type1")
+                    .startObject("properties")
+                        .startObject("field1")
+                            .field("type", "string")
+                            .field("index", "not_analyzed")
+                        .endObject()
+                    .endObject()
+                .endObject()
+            .endObject()
+            .startObject("settings")
+                .field("number_of_shards", 10)
+            .endObject()
+            .startObject("aliases")
+                .startObject("alias1").endObject()
+            .endObject()
+            .endObject();
+        request.source(builder.bytes());
+        Set<Condition> conditions = request.getConditions();
+        assertThat(conditions.size(), equalTo(2));
+        assertThat(request.getCreateIndexRequest().mappings().size(), equalTo(1));
+        assertThat(request.getCreateIndexRequest().aliases().size(), equalTo(1));
+        assertThat(request.getCreateIndexRequest().settings().getAsInt("number_of_shards", 0), equalTo(10));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
index af51645f655..28c0a6e3787 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.action.admin.indices.rollover;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesClusterStateUpdateRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest;
 import org.elasticsearch.cluster.metadata.AliasAction;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -86,7 +87,6 @@ public class TransportRolloverActionTests extends ESTestCase {
         }
     }
 
-
     public void testCreateUpdateAliasRequest() throws Exception {
         String sourceAlias = randomAsciiOfLength(10);
         String sourceIndex = randomAsciiOfLength(10);
@@ -158,4 +158,22 @@ public class TransportRolloverActionTests extends ESTestCase {
         assertThat(TransportRolloverAction.generateRolloverIndexName("index-name-1"), equalTo("index-name-2"));
         assertThat(TransportRolloverAction.generateRolloverIndexName("index-name-2"), equalTo("index-name-3"));
     }
+
+    public void testCreateIndexRequest() throws Exception {
+        String alias = randomAsciiOfLength(10);
+        String rolloverIndex = randomAsciiOfLength(10);
+        final RolloverRequest rolloverRequest = new RolloverRequest(alias);
+        final Settings settings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
+            .build();
+        rolloverRequest.getCreateIndexRequest().settings(settings);
+        final CreateIndexClusterStateUpdateRequest createIndexRequest =
+            TransportRolloverAction.prepareCreateIndexRequest(rolloverIndex, rolloverRequest);
+        assertThat(createIndexRequest.settings(), equalTo(settings));
+        assertThat(createIndexRequest.index(), equalTo(rolloverIndex));
+        assertThat(createIndexRequest.cause(), equalTo("rollover_index"));
+    }
 }

From ae3eb15caaaf61f3c0b627aad1bc22fcd71ec04e Mon Sep 17 00:00:00 2001
From: Areek Zillur <areek.zillur@elasticsearch.com>
Date: Wed, 8 Jun 2016 15:03:35 -0400
Subject: [PATCH 0728/1311] fix rest tests

---
 .../indices/rollover/RolloverRequest.java     | 24 +++++------
 .../rollover/RolloverRequestBuilder.java      |  4 +-
 .../rollover/TransportRolloverAction.java     |  8 ++--
 .../client/support/AbstractClient.java        |  4 +-
 .../indices/RestRolloverIndexAction.java      | 10 ++---
 .../rest-api-spec/api/indices.rollover.json   |  8 ++--
 .../test/indices.rollover/10_basic.yaml       | 41 +++++++++++--------
 7 files changed, 52 insertions(+), 47 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
index 19659171a67..a16a6db83ca 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
@@ -50,7 +50,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
  */
 public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implements IndicesRequest {
 
-    private String sourceAlias;
+    private String alias;
     private boolean simulate;
     private Set<Condition> conditions = new HashSet<>(2);
     private CreateIndexRequest createIndexRequest = new CreateIndexRequest("_na_");
@@ -77,15 +77,15 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
 
     RolloverRequest() {}
 
-    public RolloverRequest(String sourceAlias) {
-        this.sourceAlias = sourceAlias;
+    public RolloverRequest(String alias) {
+        this.alias = alias;
     }
 
     @Override
     public ActionRequestValidationException validate() {
         ActionRequestValidationException validationException = createIndexRequest == null ? null : createIndexRequest.validate();
-        if (sourceAlias == null) {
-            validationException = addValidationError("source alias is missing", validationException);
+        if (alias == null) {
+            validationException = addValidationError("index alias is missing", validationException);
         }
         if (createIndexRequest == null) {
             validationException = addValidationError("create index request is missing", validationException);
@@ -96,7 +96,7 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        sourceAlias = in.readString();
+        alias = in.readString();
         simulate = in.readBoolean();
         int size = in.readVInt();
         for (int i = 0; i < size; i++) {
@@ -109,7 +109,7 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeString(sourceAlias);
+        out.writeString(alias);
         out.writeBoolean(simulate);
         out.writeVInt(conditions.size());
         for (Condition condition : conditions) {
@@ -120,7 +120,7 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
 
     @Override
     public String[] indices() {
-        return new String[] {sourceAlias};
+        return new String[] {alias};
     }
 
     @Override
@@ -128,8 +128,8 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
         return IndicesOptions.strictSingleIndexNoExpandForbidClosed();
     }
 
-    public void setSourceAlias(String sourceAlias) {
-        this.sourceAlias = sourceAlias;
+    public void setAlias(String alias) {
+        this.alias = alias;
     }
 
     public void simulate(boolean simulate) {
@@ -152,8 +152,8 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
         return conditions;
     }
 
-    public String getSourceAlias() {
-        return sourceAlias;
+    public String getAlias() {
+        return alias;
     }
 
     public CreateIndexRequest getCreateIndexRequest() {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
index bef680110ab..dd6e22dae70 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
@@ -34,8 +34,8 @@ public class RolloverRequestBuilder extends MasterNodeOperationRequestBuilder<Ro
         super(client, action, new RolloverRequest());
     }
 
-    public RolloverRequestBuilder setSourceAlias(String sourceAlias) {
-        this.request.setSourceAlias(sourceAlias);
+    public RolloverRequestBuilder setAlias(String sourceAlias) {
+        this.request.setAlias(sourceAlias);
         return this;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
index a3ff8fe23cd..61ae2852f68 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -96,7 +96,7 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
                                    final ActionListener<RolloverResponse> listener) {
         final MetaData metaData = state.metaData();
         validate(metaData, rolloverRequest);
-        final AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(rolloverRequest.getSourceAlias());
+        final AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(rolloverRequest.getAlias());
         final IndexMetaData indexMetaData = aliasOrIndex.getIndices().get(0);
         final String sourceIndexName = indexMetaData.getIndex().getName();
         client.admin().indices().prepareStats(sourceIndexName).clear().setDocs(true).execute(
@@ -173,8 +173,8 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
             .ackTimeout(request.ackTimeout())
             .masterNodeTimeout(request.masterNodeTimeout());
         AliasAction[] actions = new AliasAction[2];
-        actions[0] = new AliasAction(AliasAction.Type.ADD, newIndex, request.getSourceAlias());
-        actions[1] = new AliasAction(AliasAction.Type.REMOVE, oldIndex, request.getSourceAlias());
+        actions[0] = new AliasAction(AliasAction.Type.ADD, newIndex, request.getAlias());
+        actions[1] = new AliasAction(AliasAction.Type.REMOVE, oldIndex, request.getAlias());
         updateRequest.actions(actions);
         return updateRequest;
     }
@@ -201,7 +201,7 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
     }
 
     static void validate(MetaData metaData, RolloverRequest request) {
-        final AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(request.getSourceAlias());
+        final AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(request.getAlias());
         if (aliasOrIndex == null) {
             throw new IllegalArgumentException("source alias does not exist");
         }
diff --git a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
index 18a3a2ae202..d42cfb6a44a 100644
--- a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
+++ b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
@@ -1709,8 +1709,8 @@ public abstract class AbstractClient extends AbstractComponent implements Client
         }
 
         @Override
-        public RolloverRequestBuilder prepareRolloverIndex(String sourceAlias) {
-            return new RolloverRequestBuilder(this, RolloverAction.INSTANCE).setSourceAlias(sourceAlias);
+        public RolloverRequestBuilder prepareRolloverIndex(String alias) {
+            return new RolloverRequestBuilder(this, RolloverAction.INSTANCE).setAlias(alias);
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
index 08565fca9cf..cccdf22d1d4 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
@@ -37,16 +37,14 @@ public class RestRolloverIndexAction extends BaseRestHandler {
     @Inject
     public RestRolloverIndexAction(Settings settings, RestController controller, Client client) {
         super(settings, client);
-        controller.registerHandler(RestRequest.Method.POST, "/{alias}/_rollover", this);
-        controller.registerHandler(RestRequest.Method.GET, "/{alias}/_rollover", this);
+        controller.registerHandler(RestRequest.Method.POST, "/{index}/_rollover", this);
+        controller.registerHandler(RestRequest.Method.GET, "/{index}/_rollover", this);
     }
 
+    @SuppressWarnings({"unchecked"})
     @Override
     public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
-        if (request.param("alias") == null) {
-            throw new IllegalArgumentException("no alias");
-        }
-        RolloverRequest rolloverIndexRequest = new RolloverRequest(request.param("alias"));
+        RolloverRequest rolloverIndexRequest = new RolloverRequest(request.param("index"));
         if (request.hasContent()) {
             rolloverIndexRequest.source(request.content());
         }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
index 22ae6498149..ec66d48116a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
@@ -1,12 +1,12 @@
 {
   "indices.rollover": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-rollover-index.html",
-    "methods": ["PUT", "POST"],
+    "methods": ["POST"],
     "url": {
-      "path": "/{alias}/_rollover",
-      "paths": ["/{alias}/_rollover"],
+      "path": "/{index}/_rollover",
+      "paths": ["/{index}/_rollover"],
       "parts": {
-        "alias": {
+        "index": {
           "type" : "string",
           "required" : true,
           "description" : "The name of the alias to rollover"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml
index a9acf8d150a..8252a7f0254 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml
@@ -3,10 +3,8 @@
   # create index with alias
   - do:
       indices.create:
-        index: logs
+        index: logs-1
         body:
-          settings:
-            number_of_replicas: "0"
           aliases:
             logs_index: {}
             logs_search: {}
@@ -14,7 +12,7 @@
   # index document
   - do:
       index:
-        index: logs
+        index: logs-1
         type:  test
         id:    "1"
         body:  { "foo": "hello world" }
@@ -25,7 +23,7 @@
         type:  test
         id:    "1"
 
-  - match: { _index:   logs }
+  - match: { _index:   logs-1 }
   - match: { _type:    test   }
   - match: { _id:      "1"     }
   - match: { _source:  { foo: "hello world" } }
@@ -33,37 +31,46 @@
   # perform alias rollover
   - do:
     indices.rollover:
-      alias: "logs_search"
+      index: "logs_search"
       body:
-        condition.max_docs: 1
+        conditions:
+          max_docs: 1
 
-  - match: { old_index: logs }
-  - match: { new_index: logs-1 }
+  - match: { old_index: logs-1 }
+  - match: { new_index: logs-2 }
+  - match: { rolled_over: true }
+  - match: { rollover_index_created: true }
+  - match: { simulated: false }
+  - match: { conditions: { "[max_docs: 1]": true } }
 
   # ensure new index is created
   - do:
       indices.exists:
-        index: logs-1
+        index: logs-2
 
   - is_true: ''
 
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+
   # index into new index
   - do:
       index:
-        index: logs-1
+        index: logs-2
         type:  test
         id:    "2"
         body:  { "foo": "hello world" }
 
+  - do:
+      indices.refresh: {}
+
   # check alias points to the new index
   - do:
-      get:
+      search:
         index: logs_search
         type:  test
-        id:    "2"
 
-  - match: { _index:   logs-1 }
-  - match: { _type:    test   }
-  - match: { _id:      "2"     }
-  - match: { _source:  { foo: "hello world" } }
+  - match: { hits.total: 1 }
+  - match: { hits.hits.0._index: "logs-2"}
 

From 9497b704bb5a6fd477b0732ca31e4a631094d297 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Wed, 8 Jun 2016 21:40:35 +0200
Subject: [PATCH 0729/1311] [TEST] Fix NodeEnvironmentTests on Windows - use
 Path.resolve instead of platform dependent path seperator

---
 .../elasticsearch/env/NodeEnvironmentTests.java  | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
index 84809c0f991..ad425d8afc9 100644
--- a/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
+++ b/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
@@ -409,19 +409,19 @@ public class NodeEnvironmentTests extends ESTestCase {
 
     public void testWhetherClusterFolderShouldBeUsed() throws Exception {
         Path tempNoCluster = createTempDir();
-        String tempDataPathString = tempNoCluster.toAbsolutePath().toString();
+        Path tempDataPath = tempNoCluster.toAbsolutePath();
 
         Path tempPath = tempNoCluster.resolve("foo"); // "foo" is the cluster name
-        String tempClusterPathString = tempPath.toAbsolutePath().toString();
+        Path tempClusterPath = tempPath.toAbsolutePath();
 
         assertFalse("non-existent directory should not be used", NodeEnvironment.readFromDataPathWithClusterName(tempPath));
         Settings settings = Settings.builder()
                 .put("cluster.name", "foo")
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath().toString())
-                .put(Environment.PATH_DATA_SETTING.getKey(), tempDataPathString).build();
+                .put(Environment.PATH_DATA_SETTING.getKey(), tempDataPath.toString()).build();
         try (NodeEnvironment env = new NodeEnvironment(settings, new Environment(settings))) {
             Path nodeDataPath = env.nodeDataPaths()[0];
-            assertThat(nodeDataPath.toString(), equalTo(tempDataPathString + "/nodes/0"));
+            assertEquals(nodeDataPath, tempDataPath.resolve("nodes").resolve("0"));
         }
         IOUtils.rm(tempNoCluster);
 
@@ -430,10 +430,10 @@ public class NodeEnvironmentTests extends ESTestCase {
         settings = Settings.builder()
                 .put("cluster.name", "foo")
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath().toString())
-                .put(Environment.PATH_DATA_SETTING.getKey(), tempDataPathString).build();
+                .put(Environment.PATH_DATA_SETTING.getKey(), tempDataPath.toString()).build();
         try (NodeEnvironment env = new NodeEnvironment(settings, new Environment(settings))) {
             Path nodeDataPath = env.nodeDataPaths()[0];
-            assertThat(nodeDataPath.toString(), equalTo(tempDataPathString + "/nodes/0"));
+            assertEquals(nodeDataPath, tempDataPath.resolve("nodes").resolve("0"));
         }
         IOUtils.rm(tempNoCluster);
 
@@ -443,10 +443,10 @@ public class NodeEnvironmentTests extends ESTestCase {
         settings = Settings.builder()
                 .put("cluster.name", "foo")
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath().toString())
-                .put(Environment.PATH_DATA_SETTING.getKey(), tempDataPathString).build();
+                .put(Environment.PATH_DATA_SETTING.getKey(), tempClusterPath.toString()).build();
         try (NodeEnvironment env = new NodeEnvironment(settings, new Environment(settings))) {
             Path nodeDataPath = env.nodeDataPaths()[0];
-            assertThat(nodeDataPath.toString(), equalTo(tempClusterPathString + "/nodes/0"));
+            assertEquals(nodeDataPath, tempClusterPath.resolve("nodes").resolve("0"));
         }
     }
 

From 9826b64e1f03cbeedc931babdf987c357032b66a Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Wed, 8 Jun 2016 15:47:09 -0400
Subject: [PATCH 0730/1311] Exempt LocalDate from bridge method checks on java
 9

The painless whitelist has a lot of self-checking, in this case, it checks
for missing covariant overrides. It fails on java 9, because LocalDate.getEra()
now returns IsoEra instead of Era: https://bugs.openjdk.java.net/browse/JDK-8072746

To our checker, it thinks we were lazy with whitelisting :)

This means painless works on java 9 again
---
 .../src/main/java/org/elasticsearch/painless/Definition.java | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index a68c9629a2d..dbfc9993c31 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.painless;
 
+import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.SetOnce;
 
 import java.io.InputStream;
@@ -28,6 +29,7 @@ import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
 import java.lang.reflect.Modifier;
 import java.nio.charset.StandardCharsets;
+import java.time.LocalDate;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -864,6 +866,9 @@ public final class Definition {
                         // ok
                     } else if (child.clazz == Spliterator.OfPrimitive.class || child.clazz == PrimitiveIterator.class) {
                         // ok, we rely on generics erasure for these (its guaranteed in the javadocs though!!!!)
+                    } else if (Constants.JRE_IS_MINIMUM_JAVA9 && owner.clazz == LocalDate.class) {
+                        // ok, java 9 added covariant override for LocalDate.getEra() to return IsoEra:
+                        // https://bugs.openjdk.java.net/browse/JDK-8072746
                     } else {
                         try {
                             Class<?> arguments[] = new Class<?>[method.arguments.size()];

From cd0473159a87c0a75f9c7a404fa192f4d21b0fb5 Mon Sep 17 00:00:00 2001
From: Christoph Wurm <cwurm@users.noreply.github.com>
Date: Wed, 8 Jun 2016 21:55:59 +0200
Subject: [PATCH 0731/1311] Installation and removal instructions for Ingest
 Plugins (#18769)

* Add install and remove instructions to ingest-geoip and ingest-attachment docs
---
 docs/plugins/ingest-attachment.asciidoc | 30 +++++++++++++++++++++++
 docs/plugins/ingest-geoip.asciidoc      | 32 ++++++++++++++++++++++++-
 2 files changed, 61 insertions(+), 1 deletion(-)

diff --git a/docs/plugins/ingest-attachment.asciidoc b/docs/plugins/ingest-attachment.asciidoc
index 290bfabd2ff..a5544844441 100644
--- a/docs/plugins/ingest-attachment.asciidoc
+++ b/docs/plugins/ingest-attachment.asciidoc
@@ -11,6 +11,36 @@ the overhead of converting back and forth between base64, you can use the CBOR
 format instead of JSON and specify the field as a bytes array instead of a string
 representation. The processor will skip the base64 decoding then.
 
+[[ingest-attachment-install]]
+[float]
+==== Installation
+
+This plugin can be installed using the plugin manager:
+
+[source,sh]
+----------------------------------------------------------------
+sudo bin/elasticsearch-plugin install ingest-attachment
+----------------------------------------------------------------
+
+The plugin must be installed on every node in the cluster, and each node must
+be restarted after installation.
+
+[[ingest-attachment-remove]]
+[float]
+==== Removal
+
+The plugin can be removed with the following command:
+
+[source,sh]
+----------------------------------------------------------------
+sudo bin/elasticsearch-plugin remove ingest-attachment
+----------------------------------------------------------------
+
+The node must be stopped before removing the plugin.
+
+[[using-ingest-attachment]]
+==== Using the Attachment Processor in a Pipeline
+
 [[ingest-attachment-options]]
 .Attachment options
 [options="header"]
diff --git a/docs/plugins/ingest-geoip.asciidoc b/docs/plugins/ingest-geoip.asciidoc
index 1ff763c1d6f..434d713f945 100644
--- a/docs/plugins/ingest-geoip.asciidoc
+++ b/docs/plugins/ingest-geoip.asciidoc
@@ -11,7 +11,37 @@ The GeoIP processor can run with other geoip2 databases from Maxmind. The files
 and the `database_file` option should be used to specify the filename of the custom database. Custom database files must be compressed
 with gzip. The geoip config directory is located at `$ES_HOME/config/ingest/geoip` and holds the shipped databases too.
 
-[[geoip-options]]
+[[ingest-geoip-install]]
+[float]
+==== Installation
+
+This plugin can be installed using the plugin manager:
+
+[source,sh]
+----------------------------------------------------------------
+sudo bin/elasticsearch-plugin install ingest-geoip
+----------------------------------------------------------------
+
+The plugin must be installed on every node in the cluster, and each node must
+be restarted after installation.
+
+[[ingest-geoip-remove]]
+[float]
+==== Removal
+
+The plugin can be removed with the following command:
+
+[source,sh]
+----------------------------------------------------------------
+sudo bin/elasticsearch-plugin remove ingest-geoip
+----------------------------------------------------------------
+
+The node must be stopped before removing the plugin.
+
+[[using-ingest-geoip]]
+==== Using the Geoip Processor in a Pipeline
+
+[[ingest-geoip-options]]
 .Geoip options
 [options="header"]
 |======

From e9017f619e9728e349e66a47c15caad2d97381bb Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 8 Jun 2016 16:08:00 -0400
Subject: [PATCH 0732/1311] Improve performance of applyDeletedShards

This commit addresses a performance issue in
IndicesClusterStateService#applyDeletedShards. Namely, the current
implementation is O(number of indices * number of shards). This is
because of an outer loop over the indices and an inner loop over the
assigned shards, all to check if a shard is in the outer index. Instead,
we can group the shards by index, and then just do a map lookup for each
index.

Testing this on a single-node with 2500 indices, each with 2 shards,
creating an index before this optimization takes 0.90s and after this
optimization takes 0.19s.

Relates #18788
---
 .../cluster/IndicesClusterStateService.java   | 20 ++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
index bbb1a063e8f..2c77f863c47 100644
--- a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
+++ b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
@@ -71,7 +71,10 @@ import org.elasticsearch.snapshots.RestoreService;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
@@ -79,7 +82,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
 
 /**
  *
@@ -285,19 +288,18 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
         if (routingNode == null) {
             return;
         }
-        Set<String> newShardAllocationIds = new HashSet<>();
+
+        final Map<Index, Set<String>> shardsByIndex = new HashMap<>();
+        for (ShardRouting shard : routingNode) {
+            shardsByIndex.computeIfAbsent(shard.index(), k -> new HashSet<>()).add(shard.allocationId().getId());
+        }
+
         for (IndexService indexService : indicesService) {
             Index index = indexService.index();
             IndexMetaData indexMetaData = event.state().metaData().index(index);
             assert indexMetaData != null : "local index doesn't have metadata, should have been cleaned up by applyDeletedIndices: " + index;
             // now, go over and delete shards that needs to get deleted
-            newShardAllocationIds.clear();
-            for (ShardRouting shard : routingNode) {
-                if (shard.index().equals(index)) {
-                    // use the allocation id and not object so we won't be influence by relocation targets
-                    newShardAllocationIds.add(shard.allocationId().getId());
-                }
-            }
+            Set<String> newShardAllocationIds = shardsByIndex.getOrDefault(index, Collections.emptySet());
             for (IndexShard existingShard : indexService) {
                 if (newShardAllocationIds.contains(existingShard.routingEntry().allocationId().getId()) == false) {
                     if (indexMetaData.getState() == IndexMetaData.State.CLOSE) {

From 94a7978ef615fb53bdc5d739846e721a93fa6c84 Mon Sep 17 00:00:00 2001
From: Areek Zillur <areek.zillur@elasticsearch.com>
Date: Wed, 8 Jun 2016 18:32:22 -0400
Subject: [PATCH 0733/1311] add documentation

---
 .../admin/indices/rollover/Condition.java     | 13 ++-
 .../indices/rollover/MaxAgeCondition.java     |  4 +
 .../indices/rollover/MaxDocsCondition.java    |  4 +
 .../indices/rollover/RolloverRequest.java     | 55 +++++++-----
 .../rollover/RolloverRequestBuilder.java      |  3 -
 .../indices/rollover/RolloverResponse.java    | 18 ++++
 .../rollover/TransportRolloverAction.java     |  2 +-
 docs/reference/indices.asciidoc               |  1 +
 .../reference/indices/rollover-index.asciidoc | 84 +++++++++++++++++++
 9 files changed, 159 insertions(+), 25 deletions(-)
 create mode 100644 docs/reference/indices/rollover-index.asciidoc

diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
index 12c3ef45993..8d9b48f2000 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/Condition.java
@@ -27,12 +27,17 @@ import org.elasticsearch.common.xcontent.ObjectParser;
 
 import java.util.Set;
 
+/**
+ * Base class for rollover request conditions
+ */
 public abstract class Condition<T> implements NamedWriteable {
+
     public static ObjectParser<Set<Condition>, ParseFieldMatcherSupplier> PARSER =
         new ObjectParser<>("conditions", null);
     static {
         PARSER.declareString((conditions, s) ->
-            conditions.add(new MaxAgeCondition(TimeValue.parseTimeValue(s, MaxAgeCondition.NAME))), new ParseField(MaxAgeCondition.NAME));
+            conditions.add(new MaxAgeCondition(TimeValue.parseTimeValue(s, MaxAgeCondition.NAME))),
+            new ParseField(MaxAgeCondition.NAME));
         PARSER.declareLong((conditions, value) ->
             conditions.add(new MaxDocsCondition(value)), new ParseField(MaxDocsCondition.NAME));
     }
@@ -51,6 +56,9 @@ public abstract class Condition<T> implements NamedWriteable {
         return "[" + name + ": " + value + "]";
     }
 
+    /**
+     * Holder for index stats used to evaluate conditions
+     */
     public static class Stats {
         public final long numDocs;
         public final long indexCreated;
@@ -61,6 +69,9 @@ public abstract class Condition<T> implements NamedWriteable {
         }
     }
 
+    /**
+     * Holder for evaluated condition result
+     */
     public static class Result {
         public final Condition condition;
         public final boolean matched;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxAgeCondition.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxAgeCondition.java
index 0a7e85bb129..ee0641bbe18 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxAgeCondition.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxAgeCondition.java
@@ -25,6 +25,10 @@ import org.elasticsearch.common.unit.TimeValue;
 
 import java.io.IOException;
 
+/**
+ * Condition for index maximum age. Evaluates to <code>true</code>
+ * when the index is at least {@link #value} old
+ */
 public class MaxAgeCondition extends Condition<TimeValue> {
     public final static String NAME = "max_age";
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxDocsCondition.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxDocsCondition.java
index e1c599c23d4..8b327929124 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxDocsCondition.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxDocsCondition.java
@@ -24,6 +24,10 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 
 import java.io.IOException;
 
+/**
+ * Condition for maximum index docs. Evaluates to <code>true</code>
+ * when the index has at least {@link #value} docs
+ */
 public class MaxDocsCondition extends Condition<Long> {
     public final static String NAME = "max_docs";
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
index a16a6db83ca..abae56cb5fb 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
@@ -45,24 +45,18 @@ import java.util.Set;
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
 /**
- * Request class to swap index under an alias given some predicates
- * TODO: documentation
+ * Request class to swap index under an alias upon satisfying conditions
  */
 public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implements IndicesRequest {
 
-    private String alias;
-    private boolean simulate;
-    private Set<Condition> conditions = new HashSet<>(2);
-    private CreateIndexRequest createIndexRequest = new CreateIndexRequest("_na_");
-
     public static ObjectParser<RolloverRequest, ParseFieldMatcherSupplier> PARSER =
         new ObjectParser<>("conditions", null);
     static {
         PARSER.declareField((parser, request, parseFieldMatcherSupplier) ->
-            Condition.PARSER.parse(parser, request.conditions, parseFieldMatcherSupplier),
+                Condition.PARSER.parse(parser, request.conditions, parseFieldMatcherSupplier),
             new ParseField("conditions"), ObjectParser.ValueType.OBJECT);
         PARSER.declareField((parser, request, parseFieldMatcherSupplier) ->
-            request.createIndexRequest.settings(parser.map()),
+                request.createIndexRequest.settings(parser.map()),
             new ParseField("settings"), ObjectParser.ValueType.OBJECT);
         PARSER.declareField((parser, request, parseFieldMatcherSupplier) -> {
             for (Map.Entry<String, Object> mappingsEntry : parser.map().entrySet()) {
@@ -71,10 +65,15 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
             }
         }, new ParseField("mappings"), ObjectParser.ValueType.OBJECT);
         PARSER.declareField((parser, request, parseFieldMatcherSupplier) ->
-            request.createIndexRequest.aliases(parser.map()),
+                request.createIndexRequest.aliases(parser.map()),
             new ParseField("aliases"), ObjectParser.ValueType.OBJECT);
     }
 
+    private String alias;
+    private boolean simulate;
+    private Set<Condition> conditions = new HashSet<>(2);
+    private CreateIndexRequest createIndexRequest = new CreateIndexRequest("_na_");
+
     RolloverRequest() {}
 
     public RolloverRequest(String alias) {
@@ -128,42 +127,58 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
         return IndicesOptions.strictSingleIndexNoExpandForbidClosed();
     }
 
+    /**
+     * Sets the alias to rollover to another index
+     */
     public void setAlias(String alias) {
         this.alias = alias;
     }
 
+    /**
+     * Sets if the rollover should not be executed when conditions are met
+     */
     public void simulate(boolean simulate) {
         this.simulate = simulate;
     }
 
+    /**
+     * Adds condition to check if the index is at least <code>age</code> old
+     */
     public void addMaxIndexAgeCondition(TimeValue age) {
         this.conditions.add(new MaxAgeCondition(age));
     }
 
-    public void addMaxIndexDocsCondition(long docs) {
-        this.conditions.add(new MaxDocsCondition(docs));
+    /**
+     * Adds condition to check if the index has at least <code>numDocs</code>
+     */
+    public void addMaxIndexDocsCondition(long numDocs) {
+        this.conditions.add(new MaxDocsCondition(numDocs));
     }
 
-    public boolean isSimulate() {
+    /**
+     * Sets rollover index creation request to override index settings when
+     * the rolled over index has to be created
+     */
+    public void setCreateIndexRequest(CreateIndexRequest createIndexRequest) {
+        this.createIndexRequest = Objects.requireNonNull(createIndexRequest, "create index request must not be null");;
+    }
+
+    boolean isSimulate() {
         return simulate;
     }
 
-    public Set<Condition> getConditions() {
+    Set<Condition> getConditions() {
         return conditions;
     }
 
-    public String getAlias() {
+    String getAlias() {
         return alias;
     }
 
-    public CreateIndexRequest getCreateIndexRequest() {
+    CreateIndexRequest getCreateIndexRequest() {
         return createIndexRequest;
     }
 
-    public void setCreateIndexRequest(CreateIndexRequest createIndexRequest) {
-        this.createIndexRequest = Objects.requireNonNull(createIndexRequest, "create index request must not be null");;
-    }
-
     public void source(BytesReference source) {
         XContentType xContentType = XContentFactory.xContentType(source);
         if (xContentType != null) {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
index dd6e22dae70..cfce9d24ad4 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
@@ -25,9 +25,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 
 
-/**
- * TODO: Documentation
- */
 public class RolloverRequestBuilder extends MasterNodeOperationRequestBuilder<RolloverRequest, RolloverResponse,
     RolloverRequestBuilder> {
     public RolloverRequestBuilder(ElasticsearchClient client, RolloverAction action) {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
index fbca8af99cc..14feec0248f 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
@@ -56,26 +56,44 @@ public final class RolloverResponse extends ActionResponse implements ToXContent
             .collect(Collectors.toSet());
     }
 
+    /**
+     * Returns the name of the index that the request alias was pointing to
+     */
     public String getOldIndex() {
         return oldIndex;
     }
 
+    /**
+     * Returns the name of the index that the request alias currently points to
+     */
     public String getNewIndex() {
         return newIndex;
     }
 
+    /**
+     * Returns the statuses of all the request conditions
+     */
     public Set<Map.Entry<String, Boolean>> getConditionStatus() {
         return conditionStatus;
     }
 
+    /**
+     * Returns if the rollover execution was skipped even when conditions were met
+     */
     public boolean isSimulate() {
         return simulate;
     }
 
+    /**
+     * Returns if the rollover was not simulated and the conditions were met
+     */
     public boolean isRolledOver() {
         return rolledOver;
     }
 
+    /**
+     * Returns if the rollover index had to be explicitly created
+     */
     public boolean isRolloverIndexCreated() {
         return rolloverIndexCreated;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
index 61ae2852f68..0cd29b84056 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -51,7 +51,7 @@ import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 /**
- * Main class to swap the index pointed to by an alias, given some predicates
+ * Main class to swap the index pointed to by an alias, given some conditions
  */
 public class TransportRolloverAction extends TransportMasterNodeAction<RolloverRequest, RolloverResponse> {
 
diff --git a/docs/reference/indices.asciidoc b/docs/reference/indices.asciidoc
index e49340fe15f..cad1a1477bd 100644
--- a/docs/reference/indices.asciidoc
+++ b/docs/reference/indices.asciidoc
@@ -16,6 +16,7 @@ index settings, aliases, mappings, and index templates.
 * <<indices-exists>>
 * <<indices-open-close>>
 * <<indices-shrink-index>>
+* <<indices-rollover-index>>
 
 [float]
 [[mapping-management]]
diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc
new file mode 100644
index 00000000000..2c553a4e0f1
--- /dev/null
+++ b/docs/reference/indices/rollover-index.asciidoc
@@ -0,0 +1,84 @@
+[[indices-rollover-index]]
+== Rollover Index
+
+The rollover index API allows to switch the index pointed to by an alias given some predicates.
+In order to rollover an index, the provided alias has to point to a single index. Upon satisfying
+any of the predicates, the alias is switched to point to a new index, creating the index if it
+does not exist. The rollover API requires the old concrete index name to have `{index_prefix}-{num}`
+format, as rollover index name is generated following `{index_prefix}-{num+1}` format.
+
+This API is syntactic sugar for changing the index pointed to by an alias given some predicate.
+
+The rollover API must be used against an alias that points to a single index:
+
+[source,js]
+--------------------------------------------------
+$ curl -XPUT 'http://localhost:9200/index-1/' -d '{
+    "aliases" : {
+        "index_alias": {}
+    }
+}'
+--------------------------------------------------
+
+To rollover `index_alias` to point to a new index:
+
+[source,js]
+--------------------------------------------------
+$ curl -XPOST 'http://localhost:9200/index_alias/_rollover' -d '{
+    "conditions" : {
+        "max_age": "7d", <1>
+        "max_docs": 1000 <2>
+    }
+}'
+--------------------------------------------------
+<1> Sets a condition that the index has to be at least 7 days old
+<2> Sets a condition that the index has to have at least a 1000 documents
+
+The API call above switches the index pointed to by `index_alias` from `index-1` to `index-2`, if any
+of the conditions are met. If `index-2` does not exist, it is created (using matching <<indices-templates>>
+if available). The API call returns immediately if none of the conditions are met.
+
+The `_rollover` API is similar to <<indices-create-index>> and accepts `settings`, `mappings` and `aliases`
+to override the index create request for a non-existent rolled over index.
+
+[source,js]
+--------------------------------------------------
+$ curl -XPOST 'http://localhost:9200/index_alias/_rollover' -d '{
+    "conditions" : {
+        "max_age": "7d",
+        "max_docs": 1000
+    },
+    "settings": { <1>
+        "index.number_of_shards": 2
+    }
+}'
+--------------------------------------------------
+<1> Set settings to override matching index template, `mappings` and `aliases` can also be provided.
+
+Using the http `GET` method for the API runs the rollover in simulated mode, where request conditions can be
+checked without performing the actual rollover. Setting `simulate=true` as a request parameter also runs
+the request in simulated mode.
+
+An example response for the index rollover API:
+
+[source,js]
+--------------------------------------------------
+{
+	"old_index": "index-1", <1>
+	"new_index": "index-2", <2>
+	"rolled_over": true, <3>
+	"simulated": false, <4>
+	"rollover_index_created": true, <5>
+	"conditions": { <6>
+	    "[max_age: 7d]": true,
+	    "[max_docs: 1000]": true
+	}
+}
+--------------------------------------------------
+<1> name of the index the alias was pointing to
+<2> name of the index the alias currently points to
+<3> whether the alias switch was successful
+<4> whether the rollover was dry run
+<5> whether the rolled over index had to be explicitly created
+<6> status of the evaluated request conditions
+

From be5e2e145b0390153b90041348c9350b83667f16 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Thu, 9 Jun 2016 14:27:30 +0200
Subject: [PATCH 0734/1311] Decouple HostsSniffer from Sniffer

Sniffer now requires a HostsSniffer instance as a constructor argument, HostsSniffer has its won Builder helper. Also synchronized accesses to scheduledExecutorService in SnifferTask.
---
 .../client/sniff/HostsSniffer.java            | 78 +++++++++++++++-
 .../elasticsearch/client/sniff/Sniffer.java   | 91 +++++++------------
 .../sniff/HostsSnifferBuilderTests.java       | 69 ++++++++++++++
 .../client/sniff/HostsSnifferTests.java       |  9 +-
 .../client/sniff/SnifferBuilderTests.java     | 63 ++++++-------
 5 files changed, 206 insertions(+), 104 deletions(-)
 create mode 100644 client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferBuilderTests.java

diff --git a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
index cc3abdfec7e..340ff89459d 100644
--- a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
+++ b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
@@ -36,6 +36,8 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
 
 /**
  * Class responsible for sniffing the http hosts from elasticsearch through the nodes info api and returning them back.
@@ -47,14 +49,13 @@ public class HostsSniffer {
 
     private final RestClient restClient;
     private final Map<String, String> sniffRequestParams;
-    private final String scheme;
-    private final JsonFactory jsonFactory;
+    private final Scheme scheme;
+    private final JsonFactory jsonFactory = new JsonFactory();
 
-    public HostsSniffer(RestClient restClient, long sniffRequestTimeout, String scheme) {
+    protected HostsSniffer(RestClient restClient, long sniffRequestTimeout, Scheme scheme) {
         this.restClient = restClient;
         this.sniffRequestParams = Collections.<String, String>singletonMap("timeout", sniffRequestTimeout + "ms");
         this.scheme = scheme;
-        this.jsonFactory = new JsonFactory();
     }
 
     /**
@@ -95,7 +96,7 @@ public class HostsSniffer {
         }
     }
 
-    private static HttpHost readHost(String nodeId, JsonParser parser, String scheme) throws IOException {
+    private static HttpHost readHost(String nodeId, JsonParser parser, Scheme scheme) throws IOException {
         HttpHost httpHost = null;
         String fieldName = null;
         while (parser.nextToken() != JsonToken.END_OBJECT) {
@@ -124,4 +125,71 @@ public class HostsSniffer {
         }
         return httpHost;
     }
+
+    /**
+     * Returns a new {@link Builder} to help with {@link HostsSniffer} creation.
+     */
+    public static Builder builder(RestClient restClient) {
+        return new Builder(restClient);
+    }
+
+    public enum Scheme {
+        HTTP("http"), HTTPS("https");
+
+        private final String name;
+
+        Scheme(String name) {
+            this.name = name;
+        }
+
+        @Override
+        public String toString() {
+            return name;
+        }
+    }
+
+    /**
+     * HostsSniffer builder. Helps creating a new {@link HostsSniffer}.
+     */
+    public static class Builder {
+        public static final long DEFAULT_SNIFF_REQUEST_TIMEOUT = TimeUnit.SECONDS.toMillis(1);
+
+        private final RestClient restClient;
+        private long sniffRequestTimeout = DEFAULT_SNIFF_REQUEST_TIMEOUT;
+        private Scheme scheme;
+
+        private Builder(RestClient restClient) {
+            Objects.requireNonNull(restClient, "restClient cannot be null");
+            this.restClient = restClient;
+        }
+
+        /**
+         * Sets the sniff request timeout to be passed in as a query string parameter to elasticsearch.
+         * Allows to halt the request without any failure, as only the nodes that have responded
+         * within this timeout will be returned.
+         */
+        public Builder setSniffRequestTimeout(int sniffRequestTimeout) {
+            if (sniffRequestTimeout <= 0) {
+                throw new IllegalArgumentException("sniffRequestTimeout must be greater than 0");
+            }
+            this.sniffRequestTimeout = sniffRequestTimeout;
+            return this;
+        }
+
+        /**
+         * Sets the scheme to associate sniffed nodes with (as it is not returned by elasticsearch)
+         */
+        public Builder setScheme(Scheme scheme) {
+            Objects.requireNonNull(scheme, "scheme cannot be null");
+            this.scheme = scheme;
+            return this;
+        }
+
+        /**
+         * Creates a new {@link HostsSniffer} instance given the provided configuration
+         */
+        public HostsSniffer build() {
+            return new HostsSniffer(restClient, sniffRequestTimeout, scheme);
+        }
+    }
 }
diff --git a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
index 7ae6b0502e5..cd5d1d93148 100644
--- a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
+++ b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
@@ -48,9 +48,8 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
     private final boolean sniffOnFailure;
     private final Task task;
 
-    private Sniffer(RestClient restClient, long sniffRequestTimeout, String scheme, long sniffInterval,
-                   boolean sniffOnFailure, long sniffAfterFailureDelay) {
-        HostsSniffer hostsSniffer = new HostsSniffer(restClient, sniffRequestTimeout, scheme);
+    private Sniffer(RestClient restClient, HostsSniffer hostsSniffer, long sniffInterval,
+                    boolean sniffOnFailure, long sniffAfterFailureDelay) {
         this.task = new Task(hostsSniffer, restClient, sniffInterval, sniffAfterFailureDelay);
         this.sniffOnFailure = sniffOnFailure;
         restClient.setFailureListener(this);
@@ -77,8 +76,7 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
         private final long sniffAfterFailureDelay;
         private final ScheduledExecutorService scheduledExecutorService;
         private final AtomicBoolean running = new AtomicBoolean(false);
-        private volatile long nextSniffDelay;
-        private volatile ScheduledFuture<?> scheduledFuture;
+        private ScheduledFuture<?> scheduledFuture;
 
         private Task(HostsSniffer hostsSniffer, RestClient restClient, long sniffInterval, long sniffAfterFailureDelay) {
             this.hostsSniffer = hostsSniffer;
@@ -86,21 +84,34 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
             this.sniffInterval = sniffInterval;
             this.sniffAfterFailureDelay = sniffAfterFailureDelay;
             this.scheduledExecutorService = Executors.newScheduledThreadPool(1);
-            this.scheduledFuture = this.scheduledExecutorService.schedule(this, 0, TimeUnit.MILLISECONDS);
-            this.nextSniffDelay = sniffInterval;
+            scheduleNextRun(0);
+        }
+
+        synchronized void scheduleNextRun(long delayMillis) {
+            if (scheduledExecutorService.isShutdown() == false) {
+                try {
+                    if (scheduledFuture != null) {
+                        //regardless of when the next sniff is scheduled, cancel it and schedule a new one with updated delay
+                        this.scheduledFuture.cancel(false);
+                    }
+                    logger.debug("scheduling next sniff in " + delayMillis + " ms");
+                    this.scheduledFuture = this.scheduledExecutorService.schedule(this, delayMillis, TimeUnit.MILLISECONDS);
+                } catch(Throwable t) {
+                    logger.error("error while scheduling next sniffer task", t);
+                }
+            }
         }
 
         @Override
         public void run() {
-            sniff(null);
+            sniff(null, sniffInterval);
         }
 
         void sniffOnFailure(HttpHost failedHost) {
-            this.nextSniffDelay = sniffAfterFailureDelay;
-            sniff(failedHost);
+            sniff(failedHost, sniffAfterFailureDelay);
         }
 
-        void sniff(HttpHost excludeHost) {
+        void sniff(HttpHost excludeHost, long nextSniffDelayMillis) {
             if (running.compareAndSet(false, true)) {
                 try {
                     List<HttpHost> sniffedNodes = hostsSniffer.sniffHosts();
@@ -112,22 +123,13 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
                 } catch (Throwable t) {
                     logger.error("error while sniffing nodes", t);
                 } finally {
-                    try {
-                        //regardless of whether and when the next sniff is scheduled, cancel it and schedule a new one with updated delay
-                        this.scheduledFuture.cancel(false);
-                        logger.debug("scheduling next sniff in " + nextSniffDelay + " ms");
-                        this.scheduledFuture = this.scheduledExecutorService.schedule(this, nextSniffDelay, TimeUnit.MILLISECONDS);
-                    } catch (Throwable t) {
-                        logger.error("error while scheduling next sniffer task", t);
-                    } finally {
-                        this.nextSniffDelay = sniffInterval;
-                        running.set(false);
-                    }
+                    scheduleNextRun(nextSniffDelayMillis);
+                    running.set(false);
                 }
             }
         }
 
-        void shutdown() {
+        synchronized void shutdown() {
             scheduledExecutorService.shutdown();
             try {
                 if (scheduledExecutorService.awaitTermination(1000, TimeUnit.MILLISECONDS)) {
@@ -143,8 +145,8 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
     /**
      * Returns a new {@link Builder} to help with {@link Sniffer} creation.
      */
-    public static Builder builder(RestClient restClient) {
-        return new Builder(restClient);
+    public static Builder builder(RestClient restClient, HostsSniffer hostsSniffer) {
+        return new Builder(restClient, hostsSniffer);
     }
 
     /**
@@ -153,21 +155,22 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
     public static final class Builder {
         public static final long DEFAULT_SNIFF_INTERVAL = TimeUnit.MINUTES.toMillis(5);
         public static final long DEFAULT_SNIFF_AFTER_FAILURE_DELAY = TimeUnit.MINUTES.toMillis(1);
-        public static final long DEFAULT_SNIFF_REQUEST_TIMEOUT = TimeUnit.SECONDS.toMillis(1);
 
         private final RestClient restClient;
-        private long sniffRequestTimeout = DEFAULT_SNIFF_REQUEST_TIMEOUT;
+        private final HostsSniffer hostsSniffer;
         private long sniffInterval = DEFAULT_SNIFF_INTERVAL;
         private boolean sniffOnFailure = true;
         private long sniffAfterFailureDelay = DEFAULT_SNIFF_AFTER_FAILURE_DELAY;
-        private String scheme = "http";
 
         /**
-         * Creates a new builder instance and sets the {@link RestClient} that will be used to communicate with elasticsearch.
+         * Creates a new builder instance by providing the {@link RestClient} that will be used to communicate with elasticsearch,
+         * and the
          */
-        private Builder(RestClient restClient) {
+        private Builder(RestClient restClient, HostsSniffer hostsSniffer) {
             Objects.requireNonNull(restClient, "restClient cannot be null");
             this.restClient = restClient;
+            Objects.requireNonNull(hostsSniffer, "hostsSniffer cannot be null");
+            this.hostsSniffer = hostsSniffer;
         }
 
         /**
@@ -203,37 +206,11 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
             return this;
         }
 
-        /**
-         * Sets the sniff request timeout to be passed in as a query string parameter to elasticsearch.
-         * Allows to halt the request without any failure, as only the nodes that have responded
-         * within this timeout will be returned.
-         */
-        public Builder setSniffRequestTimeout(int sniffRequestTimeout) {
-            if (sniffRequestTimeout <= 0) {
-                throw new IllegalArgumentException("sniffRequestTimeout must be greater than 0");
-            }
-            this.sniffRequestTimeout = sniffRequestTimeout;
-            return this;
-        }
-
-        /**
-         * Sets the scheme to be used for sniffed nodes. This information is not returned by elasticsearch,
-         * default is http but should be customized if https is needed/enabled.
-         */
-        public Builder setScheme(String scheme) {
-            Objects.requireNonNull(scheme, "scheme cannot be null");
-            if (scheme.equals("http") == false && scheme.equals("https") == false) {
-                throw new IllegalArgumentException("scheme must be either http or https");
-            }
-            this.scheme = scheme;
-            return this;
-        }
-
         /**
          * Creates the {@link Sniffer} based on the provided configuration.
          */
         public Sniffer build() {
-            return new Sniffer(restClient, sniffRequestTimeout, scheme, sniffInterval, sniffOnFailure, sniffAfterFailureDelay);
+            return new Sniffer(restClient, hostsSniffer, sniffInterval, sniffOnFailure, sniffAfterFailureDelay);
         }
     }
 }
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferBuilderTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferBuilderTests.java
new file mode 100644
index 00000000000..8f355ecc47a
--- /dev/null
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferBuilderTests.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.sniff;
+
+import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import org.apache.http.HttpHost;
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.client.RestClient;
+
+public class HostsSnifferBuilderTests extends LuceneTestCase {
+
+    public void testBuild() throws Exception {
+        try {
+            HostsSniffer.builder(null);
+            fail("should have failed");
+        } catch(NullPointerException e) {
+            assertEquals(e.getMessage(), "restClient cannot be null");
+        }
+
+        int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
+        HttpHost[] hosts = new HttpHost[numNodes];
+        for (int i = 0; i < numNodes; i++) {
+            hosts[i] = new HttpHost("localhost", 9200 + i);
+        }
+
+        try (RestClient client = RestClient.builder(hosts).build()) {
+            try {
+                HostsSniffer.builder(client).setScheme(null);
+                fail("should have failed");
+            } catch(NullPointerException e) {
+                assertEquals(e.getMessage(), "scheme cannot be null");
+            }
+
+            try {
+                HostsSniffer.builder(client).setSniffRequestTimeout(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+                fail("should have failed");
+            } catch(IllegalArgumentException e) {
+                assertEquals(e.getMessage(), "sniffRequestTimeout must be greater than 0");
+            }
+
+            HostsSniffer.Builder builder = HostsSniffer.builder(client);
+            if (random().nextBoolean()) {
+                builder.setScheme(RandomPicks.randomFrom(random(), HostsSniffer.Scheme.values()));
+            }
+            if (random().nextBoolean()) {
+                builder.setSniffRequestTimeout(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+            }
+            assertNotNull(builder.build());
+        }
+    }
+}
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
index 826d7fb35f8..86602be81fe 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
@@ -43,7 +43,6 @@ import java.io.StringWriter;
 import java.net.InetSocketAddress;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -58,14 +57,14 @@ import static org.hamcrest.CoreMatchers.equalTo;
 public class HostsSnifferTests extends LuceneTestCase {
 
     private int sniffRequestTimeout;
-    private String scheme;
+    private HostsSniffer.Scheme scheme;
     private SniffResponse sniffResponse;
     private HttpServer httpServer;
 
     @Before
     public void startHttpServer() throws IOException {
         this.sniffRequestTimeout = RandomInts.randomIntBetween(random(), 1000, 10000);
-        this.scheme = RandomPicks.randomFrom(random(), Arrays.asList("http", "https"));
+        this.scheme = RandomPicks.randomFrom(random(), HostsSniffer.Scheme.values());
         if (rarely()) {
             this.sniffResponse = SniffResponse.buildFailure();
         } else {
@@ -132,7 +131,7 @@ public class HostsSnifferTests extends LuceneTestCase {
         return httpServer;
     }
 
-    private static SniffResponse buildSniffResponse(String scheme) throws IOException {
+    private static SniffResponse buildSniffResponse(HostsSniffer.Scheme scheme) throws IOException {
         int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
         List<HttpHost> hosts = new ArrayList<>(numNodes);
         JsonFactory jsonFactory = new JsonFactory();
@@ -164,7 +163,7 @@ public class HostsSnifferTests extends LuceneTestCase {
             if (isHttpEnabled) {
                 String host = "host" + i;
                 int port = RandomInts.randomIntBetween(random(), 9200, 9299);
-                HttpHost httpHost = new HttpHost(host, port, scheme);
+                HttpHost httpHost = new HttpHost(host, port, scheme.toString());
                 hosts.add(httpHost);
                 generator.writeObjectFieldStart("http");
                 if (random().nextBoolean()) {
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
index c0bb9def964..863830370b6 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
@@ -20,87 +20,65 @@
 package org.elasticsearch.client.sniff;
 
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
-import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import org.apache.http.HttpHost;
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.client.RestClient;
 
-import java.util.Arrays;
-import java.util.logging.LogManager;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
 
 public class SnifferBuilderTests extends LuceneTestCase {
 
-    static {
-        LogManager.getLogManager().reset();
-    }
-
     public void testBuild() throws Exception {
         int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
         HttpHost[] hosts = new HttpHost[numNodes];
         for (int i = 0; i < numNodes; i++) {
             hosts[i] = new HttpHost("localhost", 9200 + i);
         }
-        try (RestClient client = RestClient.builder(hosts).build()) {
 
+        HostsSniffer hostsSniffer = new MockHostsSniffer();
+
+        try (RestClient client = RestClient.builder(hosts).build()) {
             try {
-                Sniffer.builder(client).setScheme(null);
+                Sniffer.builder(null, hostsSniffer).build();
                 fail("should have failed");
             } catch(NullPointerException e) {
-                assertEquals("scheme cannot be null", e.getMessage());
+                assertEquals("restClient cannot be null", e.getMessage());
             }
 
             try {
-                Sniffer.builder(client).setScheme("whatever");
+                Sniffer.builder(client, null).build();
                 fail("should have failed");
-            } catch(IllegalArgumentException e) {
-                assertEquals("scheme must be either http or https", e.getMessage());
+            } catch(NullPointerException e) {
+                assertEquals("hostsSniffer cannot be null", e.getMessage());
             }
 
             try {
-                Sniffer.builder(client).setSniffInterval(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+                Sniffer.builder(client, hostsSniffer).setSniffInterval(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
                 fail("should have failed");
             } catch(IllegalArgumentException e) {
                 assertEquals("sniffInterval must be greater than 0", e.getMessage());
             }
 
             try {
-                Sniffer.builder(client).setSniffRequestTimeout(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
-                fail("should have failed");
-            } catch(IllegalArgumentException e) {
-                assertEquals("sniffRequestTimeout must be greater than 0", e.getMessage());
-            }
-
-            try {
-                Sniffer.builder(client).setSniffAfterFailureDelay(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+                Sniffer.builder(client, hostsSniffer).setSniffAfterFailureDelay(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
                 fail("should have failed");
             } catch(IllegalArgumentException e) {
                 assertEquals("sniffAfterFailureDelay must be greater than 0", e.getMessage());
             }
 
-            try {
-                Sniffer.builder(null).build();
-                fail("should have failed");
-            } catch(NullPointerException e) {
-                assertEquals("restClient cannot be null", e.getMessage());
-            }
-
-            try (Sniffer sniffer = Sniffer.builder(client).build()) {
+            try (Sniffer sniffer = Sniffer.builder(client, hostsSniffer).build()) {
                 assertNotNull(sniffer);
             }
 
-            Sniffer.Builder builder = Sniffer.builder(client);
-            if (random().nextBoolean()) {
-                builder.setScheme(RandomPicks.randomFrom(random(), Arrays.asList("http", "https")));
-            }
+            Sniffer.Builder builder = Sniffer.builder(client, hostsSniffer);
             if (random().nextBoolean()) {
                 builder.setSniffInterval(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
             }
             if (random().nextBoolean()) {
                 builder.setSniffAfterFailureDelay(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
             }
-            if (random().nextBoolean()) {
-                builder.setSniffRequestTimeout(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
-            }
             if (random().nextBoolean()) {
                 builder.setSniffOnFailure(random().nextBoolean());
             }
@@ -109,4 +87,15 @@ public class SnifferBuilderTests extends LuceneTestCase {
             }
         }
     }
+
+    private static class MockHostsSniffer extends HostsSniffer {
+        MockHostsSniffer() {
+            super(null, -1, null);
+        }
+
+        @Override
+        public List<HttpHost> sniffHosts() throws IOException {
+            return Collections.singletonList(new HttpHost("localhost", 9200));
+        }
+    }
 }

From 437c4f210bc3efe640938e463b5d3c3336ff1438 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Thu, 9 Jun 2016 14:38:32 +0200
Subject: [PATCH 0735/1311] rename ElasticsearchResponse to Response and
 ElasticsearchResponseException to ResponseException

---
 .../client/sniff/HostsSniffer.java            |  4 +--
 .../client/sniff/HostsSnifferTests.java       |  8 ++---
 ...asticsearchResponse.java => Response.java} |  6 ++--
 ...eException.java => ResponseException.java} | 22 ++++++-------
 .../org/elasticsearch/client/RestClient.java  | 12 +++----
 .../client/RestClientIntegTests.java          | 16 +++++-----
 .../client/RestClientMultipleHostsTests.java  | 32 +++++++++----------
 .../client/RestClientSingleHostTests.java     | 24 +++++++-------
 .../http/netty/NettyHttpCompressionIT.java    | 10 +++---
 .../DetailedErrorsDisabledIT.java             |  8 ++---
 .../DetailedErrorsEnabledIT.java              | 12 +++----
 .../plugins/ResponseHeaderPluginIT.java       | 10 +++---
 .../org/elasticsearch/rest/CorsNotSetIT.java  |  6 ++--
 .../org/elasticsearch/rest/CorsRegexIT.java   | 24 +++++++-------
 .../rest/action/main/RestMainActionIT.java    |  6 ++--
 .../ContextAndHeaderTransportIT.java          |  4 +--
 .../test/rest/ESRestTestCase.java             |  6 ++--
 .../test/rest/RestTestExecutionContext.java   |  4 +--
 .../test/rest/client/RestTestClient.java      | 16 +++++-----
 .../test/rest/client/RestTestResponse.java    | 12 +++----
 .../test/rest/section/DoSection.java          |  6 ++--
 21 files changed, 124 insertions(+), 124 deletions(-)
 rename client/src/main/java/org/elasticsearch/client/{ElasticsearchResponse.java => Response.java} (94%)
 rename client/src/main/java/org/elasticsearch/client/{ElasticsearchResponseException.java => ResponseException.java} (70%)

diff --git a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
index 340ff89459d..3da6e2b95cd 100644
--- a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
+++ b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
@@ -26,7 +26,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHost;
-import org.elasticsearch.client.ElasticsearchResponse;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 
 import java.io.IOException;
@@ -62,7 +62,7 @@ public class HostsSniffer {
      * Calls the elasticsearch nodes info api, parses the response and returns all the found http hosts
      */
     public List<HttpHost> sniffHosts() throws IOException {
-        try (ElasticsearchResponse response = restClient.performRequest("get", "/_nodes/http", sniffRequestParams, null)) {
+        try (Response response = restClient.performRequest("get", "/_nodes/http", sniffRequestParams, null)) {
             return readHosts(response.getEntity());
         }
     }
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
index 86602be81fe..d714e89eeb3 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
@@ -31,8 +31,8 @@ import org.apache.http.Consts;
 import org.apache.http.HttpHost;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.lucene.util.LuceneTestCase;
-import org.elasticsearch.client.ElasticsearchResponse;
-import org.elasticsearch.client.ElasticsearchResponseException;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
 import org.junit.After;
 import org.junit.Before;
@@ -93,8 +93,8 @@ public class HostsSnifferTests extends LuceneTestCase {
                 for (HttpHost sniffedHost : sniffedHosts) {
                     assertEquals(sniffedHost, responseHostsIterator.next());
                 }
-            } catch(ElasticsearchResponseException e) {
-                ElasticsearchResponse response = e.getElasticsearchResponse();
+            } catch(ResponseException e) {
+                Response response = e.getResponse();
                 if (sniffResponse.isFailure) {
                     assertThat(e.getMessage(), containsString("GET " + httpHost + "/_nodes/http?timeout=" + sniffRequestTimeout + "ms"));
                     assertThat(e.getMessage(), containsString(Integer.toString(sniffResponse.nodesInfoResponseCode)));
diff --git a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponse.java b/client/src/main/java/org/elasticsearch/client/Response.java
similarity index 94%
rename from client/src/main/java/org/elasticsearch/client/ElasticsearchResponse.java
rename to client/src/main/java/org/elasticsearch/client/Response.java
index 037f7412f66..f7685b27bb9 100644
--- a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponse.java
+++ b/client/src/main/java/org/elasticsearch/client/Response.java
@@ -35,13 +35,13 @@ import java.util.Objects;
  * its corresponding {@link RequestLine} and {@link HttpHost}.
  * It must be closed to free any resource held by it, as well as the corresponding connection in the connection pool.
  */
-public class ElasticsearchResponse implements Closeable {
+public class Response implements Closeable {
 
     private final RequestLine requestLine;
     private final HttpHost host;
     private final CloseableHttpResponse response;
 
-    ElasticsearchResponse(RequestLine requestLine, HttpHost host, CloseableHttpResponse response) {
+    Response(RequestLine requestLine, HttpHost host, CloseableHttpResponse response) {
         Objects.requireNonNull(requestLine, "requestLine cannot be null");
         Objects.requireNonNull(host, "node cannot be null");
         Objects.requireNonNull(response, "response cannot be null");
@@ -101,7 +101,7 @@ public class ElasticsearchResponse implements Closeable {
 
     @Override
     public String toString() {
-        return "ElasticsearchResponse{" +
+        return "Response{" +
                 "requestLine=" + requestLine +
                 ", host=" + host +
                 ", response=" + response.getStatusLine() +
diff --git a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java b/client/src/main/java/org/elasticsearch/client/ResponseException.java
similarity index 70%
rename from client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
rename to client/src/main/java/org/elasticsearch/client/ResponseException.java
index 2e9d42797de..44f59cce7db 100644
--- a/client/src/main/java/org/elasticsearch/client/ElasticsearchResponseException.java
+++ b/client/src/main/java/org/elasticsearch/client/ResponseException.java
@@ -23,21 +23,21 @@ import java.io.IOException;
 
 /**
  * Exception thrown when an elasticsearch node responds to a request with a status code that indicates an error.
- * Note that the response body gets passed in as a string and read eagerly, which means that the ElasticsearchResponse object
+ * Note that the response body gets passed in as a string and read eagerly, which means that the Response object
  * is expected to be closed and available only to read metadata like status line, request line, response headers.
  */
-public class ElasticsearchResponseException extends IOException {
+public class ResponseException extends IOException {
 
-    private ElasticsearchResponse elasticsearchResponse;
+    private Response response;
     private final String responseBody;
 
-    ElasticsearchResponseException(ElasticsearchResponse elasticsearchResponse, String responseBody) throws IOException {
-        super(buildMessage(elasticsearchResponse,responseBody));
-        this.elasticsearchResponse = elasticsearchResponse;
+    ResponseException(Response response, String responseBody) throws IOException {
+        super(buildMessage(response,responseBody));
+        this.response = response;
         this.responseBody = responseBody;
     }
 
-    private static String buildMessage(ElasticsearchResponse response, String responseBody) {
+    private static String buildMessage(Response response, String responseBody) {
         String message = response.getRequestLine().getMethod() + " " + response.getHost() + response.getRequestLine().getUri()
                 + ": " + response.getStatusLine().toString();
         if (responseBody != null) {
@@ -47,17 +47,17 @@ public class ElasticsearchResponseException extends IOException {
     }
 
     /**
-     * Returns the {@link ElasticsearchResponse} that caused this exception to be thrown.
+     * Returns the {@link Response} that caused this exception to be thrown.
      * Expected to be used only to read metadata like status line, request line, response headers. The response body should
      * be retrieved using {@link #getResponseBody()}
      */
-    public ElasticsearchResponse getElasticsearchResponse() {
-        return elasticsearchResponse;
+    public Response getResponse() {
+        return response;
     }
 
     /**
      * Returns the response body as a string or null if there wasn't any.
-     * The body is eagerly consumed when an ElasticsearchResponseException gets created, and its corresponding ElasticsearchResponse
+     * The body is eagerly consumed when an ResponseException gets created, and its corresponding Response
      * gets closed straightaway so this method is the only way to get back the response body that was returned.
      */
     public String getResponseBody() {
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index a01ad694b96..f0e4d702f57 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -131,10 +131,10 @@ public final class RestClient implements Closeable {
      * @return the response returned by elasticsearch
      * @throws IOException in case of a problem or the connection was aborted
      * @throws ClientProtocolException in case of an http protocol error
-     * @throws ElasticsearchResponseException in case elasticsearch responded with a status code that indicated an error
+     * @throws ResponseException in case elasticsearch responded with a status code that indicated an error
      */
-    public ElasticsearchResponse performRequest(String method, String endpoint, Map<String, String> params,
-                                                HttpEntity entity, Header... headers) throws IOException {
+    public Response performRequest(String method, String endpoint, Map<String, String> params,
+                                   HttpEntity entity, Header... headers) throws IOException {
         URI uri = buildUri(endpoint, params);
         HttpRequestBase request = createHttpRequest(method, uri, entity);
         setHeaders(request, headers);
@@ -167,7 +167,7 @@ public final class RestClient implements Closeable {
                 lastSeenException = addSuppressedException(lastSeenException, e);
                 continue;
             }
-            ElasticsearchResponse elasticsearchResponse = new ElasticsearchResponse(request.getRequestLine(), host, response);
+            Response elasticsearchResponse = new Response(request.getRequestLine(), host, response);
             int statusCode = response.getStatusLine().getStatusCode();
             if (statusCode < 300 || (request.getMethod().equals(HttpHead.METHOD_NAME) && statusCode == 404) ) {
                 RequestLogger.log(logger, "request succeeded", request, host, response);
@@ -185,9 +185,9 @@ public final class RestClient implements Closeable {
                 } finally {
                     elasticsearchResponse.close();
                 }
-                ElasticsearchResponseException elasticsearchResponseException = new ElasticsearchResponseException(
+                ResponseException responseException = new ResponseException(
                         elasticsearchResponse, responseBody);
-                lastSeenException = addSuppressedException(lastSeenException, elasticsearchResponseException);
+                lastSeenException = addSuppressedException(lastSeenException, responseException);
                 //clients don't retry on 500 because elasticsearch still misuses it instead of 400 in some places
                 if (statusCode == 502 || statusCode == 503 || statusCode == 504) {
                     onFailure(host);
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java b/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
index c5067dca63f..4dadfc1a918 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
@@ -144,12 +144,12 @@ public class RestClientIntegTests extends LuceneTestCase {
             }
 
             int statusCode = randomStatusCode(random());
-            ElasticsearchResponse esResponse;
-            try (ElasticsearchResponse response = restClient.performRequest(method, "/" + statusCode,
+            Response esResponse;
+            try (Response response = restClient.performRequest(method, "/" + statusCode,
                     Collections.<String, String>emptyMap(), null, headers)) {
                 esResponse = response;
-            } catch(ElasticsearchResponseException e) {
-                esResponse = e.getElasticsearchResponse();
+            } catch(ResponseException e) {
+                esResponse = e.getResponse();
             }
             assertThat(esResponse.getStatusLine().getStatusCode(), equalTo(statusCode));
             for (Header responseHeader : esResponse.getHeaders()) {
@@ -187,16 +187,16 @@ public class RestClientIntegTests extends LuceneTestCase {
     private void testBody(String method) throws Exception {
         String requestBody = "{ \"field\": \"value\" }";
         StringEntity entity = new StringEntity(requestBody);
-        ElasticsearchResponse esResponse;
+        Response esResponse;
         String responseBody;
         int statusCode = randomStatusCode(random());
-        try (ElasticsearchResponse response = restClient.performRequest(method, "/" + statusCode,
+        try (Response response = restClient.performRequest(method, "/" + statusCode,
                 Collections.<String, String>emptyMap(), entity)) {
             responseBody = EntityUtils.toString(response.getEntity());
             esResponse = response;
-        } catch(ElasticsearchResponseException e) {
+        } catch(ResponseException e) {
             responseBody = e.getResponseBody();
-            esResponse = e.getElasticsearchResponse();
+            esResponse = e.getResponse();
         }
         assertEquals(statusCode, esResponse.getStatusLine().getStatusCode());
         assertEquals(requestBody, responseBody);
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java b/client/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
index 7ea24cef8e3..64792c2bb1b 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
@@ -102,7 +102,7 @@ public class RestClientMultipleHostsTests extends LuceneTestCase {
             Collections.addAll(hostsSet, httpHosts);
             for (int j = 0; j < httpHosts.length; j++) {
                 int statusCode = randomOkStatusCode(random());
-                try (ElasticsearchResponse response = restClient.performRequest(randomHttpMethod(random()), "/" + statusCode,
+                try (Response response = restClient.performRequest(randomHttpMethod(random()), "/" + statusCode,
                         Collections.<String, String>emptyMap(), null)) {
                     assertThat(response.getStatusLine().getStatusCode(), equalTo(statusCode));
                     assertTrue("host not found: " + response.getHost(), hostsSet.remove(response.getHost()));
@@ -121,7 +121,7 @@ public class RestClientMultipleHostsTests extends LuceneTestCase {
             for (int j = 0; j < httpHosts.length; j++) {
                 String method = randomHttpMethod(random());
                 int statusCode = randomErrorNoRetryStatusCode(random());
-                try (ElasticsearchResponse response = restClient.performRequest(method, "/" + statusCode,
+                try (Response response = restClient.performRequest(method, "/" + statusCode,
                         Collections.<String, String>emptyMap(), null)) {
                     if (method.equals("HEAD") && statusCode == 404) {
                         //no exception gets thrown although we got a 404
@@ -131,11 +131,11 @@ public class RestClientMultipleHostsTests extends LuceneTestCase {
                     } else {
                         fail("request should have failed");
                     }
-                } catch(ElasticsearchResponseException e) {
+                } catch(ResponseException e) {
                     if (method.equals("HEAD") && statusCode == 404) {
                         throw e;
                     }
-                    ElasticsearchResponse response = e.getElasticsearchResponse();
+                    Response response = e.getResponse();
                     assertThat(response.getStatusLine().getStatusCode(), equalTo(statusCode));
                     assertTrue("host not found: " + response.getHost(), hostsSet.remove(response.getHost()));
                     assertEquals(0, e.getSuppressed().length);
@@ -151,21 +151,21 @@ public class RestClientMultipleHostsTests extends LuceneTestCase {
         try  {
             restClient.performRequest(randomHttpMethod(random()), retryEndpoint, Collections.<String, String>emptyMap(), null);
             fail("request should have failed");
-        } catch(ElasticsearchResponseException e) {
+        } catch(ResponseException e) {
             Set<HttpHost> hostsSet = new HashSet<>();
             Collections.addAll(hostsSet, httpHosts);
             //first request causes all the hosts to be blacklisted, the returned exception holds one suppressed exception each
             failureListener.assertCalled(httpHosts);
             do {
-                ElasticsearchResponse response = e.getElasticsearchResponse();
+                Response response = e.getResponse();
                 assertThat(response.getStatusLine().getStatusCode(), equalTo(Integer.parseInt(retryEndpoint.substring(1))));
                 assertTrue("host [" + response.getHost() + "] not found, most likely used multiple times",
                         hostsSet.remove(response.getHost()));
                 if (e.getSuppressed().length > 0) {
                     assertEquals(1, e.getSuppressed().length);
                     Throwable suppressed = e.getSuppressed()[0];
-                    assertThat(suppressed, instanceOf(ElasticsearchResponseException.class));
-                    e = (ElasticsearchResponseException)suppressed;
+                    assertThat(suppressed, instanceOf(ResponseException.class));
+                    e = (ResponseException)suppressed;
                 } else {
                     e = null;
                 }
@@ -201,8 +201,8 @@ public class RestClientMultipleHostsTests extends LuceneTestCase {
                 try  {
                     restClient.performRequest(randomHttpMethod(random()), retryEndpoint, Collections.<String, String>emptyMap(), null);
                     fail("request should have failed");
-                } catch(ElasticsearchResponseException e) {
-                    ElasticsearchResponse response = e.getElasticsearchResponse();
+                } catch(ResponseException e) {
+                    Response response = e.getResponse();
                     assertThat(response.getStatusLine().getStatusCode(), equalTo(Integer.parseInt(retryEndpoint.substring(1))));
                     assertTrue("host [" + response.getHost() + "] not found, most likely used multiple times",
                             hostsSet.remove(response.getHost()));
@@ -224,13 +224,13 @@ public class RestClientMultipleHostsTests extends LuceneTestCase {
                 int iters = RandomInts.randomIntBetween(random(), 2, 10);
                 for (int y = 0; y < iters; y++) {
                     int statusCode = randomErrorNoRetryStatusCode(random());
-                    ElasticsearchResponse response;
-                    try (ElasticsearchResponse esResponse = restClient.performRequest(randomHttpMethod(random()), "/" + statusCode,
+                    Response response;
+                    try (Response esResponse = restClient.performRequest(randomHttpMethod(random()), "/" + statusCode,
                             Collections.<String, String>emptyMap(), null)) {
                         response = esResponse;
                     }
-                    catch(ElasticsearchResponseException e) {
-                        response = e.getElasticsearchResponse();
+                    catch(ResponseException e) {
+                        response = e.getResponse();
                     }
                     assertThat(response.getStatusLine().getStatusCode(), equalTo(statusCode));
                     if (selectedHost == null) {
@@ -248,8 +248,8 @@ public class RestClientMultipleHostsTests extends LuceneTestCase {
                         restClient.performRequest(randomHttpMethod(random()), retryEndpoint,
                                 Collections.<String, String>emptyMap(), null);
                         fail("request should have failed");
-                    } catch(ElasticsearchResponseException e) {
-                        ElasticsearchResponse response = e.getElasticsearchResponse();
+                    } catch(ResponseException e) {
+                        Response response = e.getResponse();
                         assertThat(response.getStatusLine().getStatusCode(), equalTo(Integer.parseInt(retryEndpoint.substring(1))));
                         assertThat(response.getHost(), equalTo(selectedHost));
                         failureListener.assertCalled(selectedHost);
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java b/client/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
index 1112f38bbee..e2657722701 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
@@ -183,7 +183,7 @@ public class RestClientSingleHostTests extends LuceneTestCase {
     public void testOkStatusCodes() throws Exception {
         for (String method : getHttpMethods()) {
             for (int okStatusCode : getOkStatusCodes()) {
-                ElasticsearchResponse response = restClient.performRequest(method, "/" + okStatusCode,
+                Response response = restClient.performRequest(method, "/" + okStatusCode,
                         Collections.<String, String>emptyMap(), null);
                 assertThat(response.getStatusLine().getStatusCode(), equalTo(okStatusCode));
             }
@@ -198,7 +198,7 @@ public class RestClientSingleHostTests extends LuceneTestCase {
         for (String method : getHttpMethods()) {
             //error status codes should cause an exception to be thrown
             for (int errorStatusCode : getAllErrorStatusCodes()) {
-                try (ElasticsearchResponse response = restClient.performRequest(method, "/" + errorStatusCode,
+                try (Response response = restClient.performRequest(method, "/" + errorStatusCode,
                         Collections.<String, String>emptyMap(), null)) {
                     if (method.equals("HEAD") && errorStatusCode == 404) {
                         //no exception gets thrown although we got a 404
@@ -206,11 +206,11 @@ public class RestClientSingleHostTests extends LuceneTestCase {
                     } else {
                         fail("request should have failed");
                     }
-                } catch(ElasticsearchResponseException e) {
+                } catch(ResponseException e) {
                     if (method.equals("HEAD") && errorStatusCode == 404) {
                         throw e;
                     }
-                    assertThat(e.getElasticsearchResponse().getStatusLine().getStatusCode(), equalTo(errorStatusCode));
+                    assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(errorStatusCode));
                 }
                 if (errorStatusCode <= 500) {
                     failureListener.assertNotCalled();
@@ -250,7 +250,7 @@ public class RestClientSingleHostTests extends LuceneTestCase {
         StringEntity entity = new StringEntity(body);
         for (String method : Arrays.asList("DELETE", "GET", "PATCH", "POST", "PUT")) {
             for (int okStatusCode : getOkStatusCodes()) {
-                try (ElasticsearchResponse response = restClient.performRequest(method, "/" + okStatusCode,
+                try (Response response = restClient.performRequest(method, "/" + okStatusCode,
                         Collections.<String, String>emptyMap(), entity)) {
                     assertThat(response.getStatusLine().getStatusCode(), equalTo(okStatusCode));
                     assertThat(EntityUtils.toString(response.getEntity()), equalTo(body));
@@ -260,8 +260,8 @@ public class RestClientSingleHostTests extends LuceneTestCase {
                 try {
                     restClient.performRequest(method, "/" + errorStatusCode, Collections.<String, String>emptyMap(), entity);
                     fail("request should have failed");
-                } catch(ElasticsearchResponseException e) {
-                    ElasticsearchResponse response = e.getElasticsearchResponse();
+                } catch(ResponseException e) {
+                    Response response = e.getResponse();
                     assertThat(response.getStatusLine().getStatusCode(), equalTo(errorStatusCode));
                     assertThat(EntityUtils.toString(response.getEntity()), equalTo(body));
                 }
@@ -326,12 +326,12 @@ public class RestClientSingleHostTests extends LuceneTestCase {
             }
 
             int statusCode = randomStatusCode(random());
-            ElasticsearchResponse esResponse;
-            try (ElasticsearchResponse response = restClient.performRequest(method, "/" + statusCode,
+            Response esResponse;
+            try (Response response = restClient.performRequest(method, "/" + statusCode,
                     Collections.<String, String>emptyMap(), null, headers)) {
                 esResponse = response;
-            } catch(ElasticsearchResponseException e) {
-                esResponse = e.getElasticsearchResponse();
+            } catch(ResponseException e) {
+                esResponse = e.getResponse();
             }
             assertThat(esResponse.getStatusLine().getStatusCode(), equalTo(statusCode));
             for (Header responseHeader : esResponse.getHeaders()) {
@@ -413,7 +413,7 @@ public class RestClientSingleHostTests extends LuceneTestCase {
 
         try {
             restClient.performRequest(method, uriAsString, params, entity, headers);
-        } catch(ElasticsearchResponseException e) {
+        } catch(ResponseException e) {
             //all good
         }
         return request;
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java
index fb616a87810..4e69624376d 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java
@@ -27,7 +27,7 @@ import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.message.BasicHeader;
 import org.apache.http.protocol.HttpContext;
-import org.elasticsearch.client.ElasticsearchResponse;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
@@ -62,7 +62,7 @@ public class NettyHttpCompressionIT extends ESIntegTestCase {
         // we need to intercept early, otherwise internal logic in HttpClient will just remove the header and we cannot verify it
         ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor();
         try (RestClient client = createRestClient(HttpClients.custom().addInterceptorFirst(headerExtractor).build())) {
-            try (ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null,
+            try (Response response = client.performRequest("GET", "/", Collections.emptyMap(), null,
                     new BasicHeader(HttpHeaders.ACCEPT_ENCODING, GZIP_ENCODING))) {
                 assertEquals(200, response.getStatusLine().getStatusCode());
                 assertTrue(headerExtractor.hasContentEncodingHeader());
@@ -76,7 +76,7 @@ public class NettyHttpCompressionIT extends ESIntegTestCase {
         ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor();
         CloseableHttpClient httpClient = HttpClients.custom().disableContentCompression().addInterceptorFirst(headerExtractor).build();
         try (RestClient client = createRestClient(httpClient)) {
-            try (ElasticsearchResponse response = client.performRequest("GET", "/", Collections.emptyMap(), null)) {
+            try (Response response = client.performRequest("GET", "/", Collections.emptyMap(), null)) {
                 assertEquals(200, response.getStatusLine().getStatusCode());
                 assertFalse(headerExtractor.hasContentEncodingHeader());
             }
@@ -89,7 +89,7 @@ public class NettyHttpCompressionIT extends ESIntegTestCase {
         // this disable content compression in both directions (request and response)
         CloseableHttpClient httpClient = HttpClients.custom().disableContentCompression().addInterceptorFirst(headerExtractor).build();
         try (RestClient client = createRestClient(httpClient)) {
-            try (ElasticsearchResponse response = client.performRequest("POST", "/company/employees/1",
+            try (Response response = client.performRequest("POST", "/company/employees/1",
                     Collections.emptyMap(), SAMPLE_DOCUMENT)) {
                 assertEquals(201, response.getStatusLine().getStatusCode());
                 assertFalse(headerExtractor.hasContentEncodingHeader());
@@ -102,7 +102,7 @@ public class NettyHttpCompressionIT extends ESIntegTestCase {
         ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor();
         // we don't call #disableContentCompression() hence the client will send the content compressed
         try (RestClient client = createRestClient(HttpClients.custom().addInterceptorFirst(headerExtractor).build())) {
-            try (ElasticsearchResponse response = client.performRequest("POST", "/company/employees/2",
+            try (Response response = client.performRequest("POST", "/company/employees/2",
                     Collections.emptyMap(), SAMPLE_DOCUMENT)) {
                 assertEquals(201, response.getStatusLine().getStatusCode());
                 assertEquals(GZIP_ENCODING, headerExtractor.getContentEncodingHeader().getValue());
diff --git a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java
index fb38f774c3d..21b8706abfb 100644
--- a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java
+++ b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.options.detailederrors;
 
-import org.elasticsearch.client.ElasticsearchResponse;
-import org.elasticsearch.client.ElasticsearchResponseException;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.http.HttpTransportSettings;
@@ -51,8 +51,8 @@ public class DetailedErrorsDisabledIT extends ESIntegTestCase {
         try {
             getRestClient().performRequest("DELETE", "/", Collections.singletonMap("error_trace", "true"), null);
             fail("request should have failed");
-        } catch(ElasticsearchResponseException e) {
-            ElasticsearchResponse response = e.getElasticsearchResponse();
+        } catch(ResponseException e) {
+            Response response = e.getResponse();
             assertThat(response.getHeader("Content-Type"), is("application/json"));
             assertThat(e.getResponseBody(), is("{\"error\":\"error traces in responses are disabled.\"}"));
             assertThat(response.getStatusLine().getStatusCode(), is(400));
diff --git a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java
index 39aeb00d05d..269dc72b253 100644
--- a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java
+++ b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.options.detailederrors;
 
-import org.elasticsearch.client.ElasticsearchResponse;
-import org.elasticsearch.client.ElasticsearchResponseException;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -49,8 +49,8 @@ public class DetailedErrorsEnabledIT extends ESIntegTestCase {
         try {
             getRestClient().performRequest("DELETE", "/", Collections.singletonMap("error_trace", "true"), null);
             fail("request should have failed");
-        } catch(ElasticsearchResponseException e) {
-            ElasticsearchResponse response = e.getElasticsearchResponse();
+        } catch(ResponseException e) {
+            Response response = e.getResponse();
             assertThat(response.getHeader("Content-Type"), containsString("application/json"));
             assertThat(e.getResponseBody(), containsString("\"stack_trace\":\"[Validation Failed: 1: index / indices is missing;]; " +
                     "nested: ActionRequestValidationException[Validation Failed: 1:"));
@@ -59,8 +59,8 @@ public class DetailedErrorsEnabledIT extends ESIntegTestCase {
         try {
             getRestClient().performRequest("DELETE", "/", Collections.emptyMap(), null);
             fail("request should have failed");
-        } catch(ElasticsearchResponseException e) {
-            ElasticsearchResponse response = e.getElasticsearchResponse();
+        } catch(ResponseException e) {
+            Response response = e.getResponse();
             assertThat(response.getHeader("Content-Type"), containsString("application/json"));
             assertThat(e.getResponseBody(), not(containsString("\"stack_trace\":\"[Validation Failed: 1: index / indices is missing;]; "
                     + "nested: ActionRequestValidationException[Validation Failed: 1:")));
diff --git a/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java b/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
index 6b4fdee47c6..6a3513416f8 100644
--- a/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
+++ b/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
@@ -19,8 +19,8 @@
 package org.elasticsearch.plugins;
 
 import org.apache.http.message.BasicHeader;
-import org.elasticsearch.client.ElasticsearchResponse;
-import org.elasticsearch.client.ElasticsearchResponseException;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.responseheader.TestResponseHeaderPlugin;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -55,13 +55,13 @@ public class ResponseHeaderPluginIT extends ESIntegTestCase {
         try {
             getRestClient().performRequest("GET", "/_protected", Collections.emptyMap(), null);
             fail("request should have failed");
-        } catch(ElasticsearchResponseException e) {
-            ElasticsearchResponse response = e.getElasticsearchResponse();
+        } catch(ResponseException e) {
+            Response response = e.getResponse();
             assertThat(response.getStatusLine().getStatusCode(), equalTo(401));
             assertThat(response.getHeader("Secret"), equalTo("required"));
         }
 
-        try (ElasticsearchResponse authResponse = getRestClient().performRequest("GET", "/_protected", Collections.emptyMap(), null,
+        try (Response authResponse = getRestClient().performRequest("GET", "/_protected", Collections.emptyMap(), null,
                 new BasicHeader("Secret", "password"))) {
             assertThat(authResponse.getStatusLine().getStatusCode(), equalTo(200));
             assertThat(authResponse.getHeader("Secret"), equalTo("granted"));
diff --git a/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java b/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java
index 2a38f3bcd3a..5f272d3e7e2 100644
--- a/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java
+++ b/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.rest;
 
 import org.apache.http.message.BasicHeader;
-import org.elasticsearch.client.ElasticsearchResponse;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -46,7 +46,7 @@ public class CorsNotSetIT extends ESIntegTestCase {
 
     public void testCorsSettingDefaultBehaviourDoesNotReturnAnything() throws Exception {
         String corsValue = "http://localhost:9200";
-        try (ElasticsearchResponse response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null,
+        try (Response response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null,
                 new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue))) {
             assertThat(response.getStatusLine().getStatusCode(), is(200));
             assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue());
@@ -55,7 +55,7 @@ public class CorsNotSetIT extends ESIntegTestCase {
     }
 
     public void testThatOmittingCorsHeaderDoesNotReturnAnything() throws Exception {
-        try (ElasticsearchResponse response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null)) {
+        try (Response response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null)) {
             assertThat(response.getStatusLine().getStatusCode(), is(200));
             assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue());
             assertThat(response.getHeader("Access-Control-Allow-Credentials"), nullValue());
diff --git a/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java b/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java
index 4da0b675d02..e6c85553231 100644
--- a/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java
+++ b/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java
@@ -19,8 +19,8 @@
 package org.elasticsearch.rest;
 
 import org.apache.http.message.BasicHeader;
-import org.elasticsearch.client.ElasticsearchResponse;
-import org.elasticsearch.client.ElasticsearchResponseException;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.network.NetworkModule;
@@ -61,12 +61,12 @@ public class CorsRegexIT extends ESIntegTestCase {
 
     public void testThatRegularExpressionWorksOnMatch() throws Exception {
         String corsValue = "http://localhost:9200";
-        try (ElasticsearchResponse response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null,
+        try (Response response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null,
                 new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue))) {
             assertResponseWithOriginheader(response, corsValue);
         }
         corsValue = "https://localhost:9200";
-        try (ElasticsearchResponse response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null,
+        try (Response response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null,
                 new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue));) {
             assertResponseWithOriginheader(response, corsValue);
             assertThat(response.getHeader("Access-Control-Allow-Credentials"), is("true"));
@@ -78,8 +78,8 @@ public class CorsRegexIT extends ESIntegTestCase {
             getRestClient().performRequest("GET", "/", Collections.emptyMap(), null, new BasicHeader("User-Agent", "Mozilla Bar"),
                     new BasicHeader("Origin", "http://evil-host:9200"));
             fail("request should have failed");
-        } catch(ElasticsearchResponseException e) {
-            ElasticsearchResponse response = e.getElasticsearchResponse();
+        } catch(ResponseException e) {
+            Response response = e.getResponse();
             // a rejected origin gets a FORBIDDEN - 403
             assertThat(response.getStatusLine().getStatusCode(), is(403));
             assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue());
@@ -87,7 +87,7 @@ public class CorsRegexIT extends ESIntegTestCase {
     }
 
     public void testThatSendingNoOriginHeaderReturnsNoAccessControlHeader() throws Exception {
-        try (ElasticsearchResponse response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null,
+        try (Response response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null,
                 new BasicHeader("User-Agent", "Mozilla Bar"))) {
             assertThat(response.getStatusLine().getStatusCode(), is(200));
             assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue());
@@ -95,7 +95,7 @@ public class CorsRegexIT extends ESIntegTestCase {
     }
 
     public void testThatRegularExpressionIsNotAppliedWithoutCorrectBrowserOnMatch() throws Exception {
-        try (ElasticsearchResponse response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null)) {
+        try (Response response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null)) {
             assertThat(response.getStatusLine().getStatusCode(), is(200));
             assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue());
         }
@@ -103,7 +103,7 @@ public class CorsRegexIT extends ESIntegTestCase {
 
     public void testThatPreFlightRequestWorksOnMatch() throws Exception {
         String corsValue = "http://localhost:9200";
-        try (ElasticsearchResponse response = getRestClient().performRequest("OPTIONS", "/", Collections.emptyMap(), null,
+        try (Response response = getRestClient().performRequest("OPTIONS", "/", Collections.emptyMap(), null,
                 new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue),
                 new BasicHeader(HttpHeaders.Names.ACCESS_CONTROL_REQUEST_METHOD, "GET"));) {
             assertResponseWithOriginheader(response, corsValue);
@@ -117,8 +117,8 @@ public class CorsRegexIT extends ESIntegTestCase {
                     new BasicHeader("Origin", "http://evil-host:9200"),
                     new BasicHeader(HttpHeaders.Names.ACCESS_CONTROL_REQUEST_METHOD, "GET"));
             fail("request should have failed");
-        } catch(ElasticsearchResponseException e) {
-            ElasticsearchResponse response = e.getElasticsearchResponse();
+        } catch(ResponseException e) {
+            Response response = e.getResponse();
             // a rejected origin gets a FORBIDDEN - 403
             assertThat(response.getStatusLine().getStatusCode(), is(403));
             assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue());
@@ -126,7 +126,7 @@ public class CorsRegexIT extends ESIntegTestCase {
         }
     }
 
-    protected static void assertResponseWithOriginheader(ElasticsearchResponse response, String expectedCorsHeader) {
+    protected static void assertResponseWithOriginheader(Response response, String expectedCorsHeader) {
         assertThat(response.getStatusLine().getStatusCode(), is(200));
         assertThat(response.getHeader("Access-Control-Allow-Origin"), is(expectedCorsHeader));
     }
diff --git a/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java b/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java
index edf9b5b2c06..72f8733292c 100644
--- a/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java
+++ b/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java
@@ -19,7 +19,7 @@
 package org.elasticsearch.rest.action.main;
 
 import org.apache.http.util.EntityUtils;
-import org.elasticsearch.client.ElasticsearchResponse;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -40,14 +40,14 @@ public class RestMainActionIT extends ESIntegTestCase {
     }
 
     public void testHeadRequest() throws IOException {
-        try (ElasticsearchResponse response = getRestClient().performRequest("HEAD", "/", Collections.emptyMap(), null)) {
+        try (Response response = getRestClient().performRequest("HEAD", "/", Collections.emptyMap(), null)) {
             assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
             assertNull(response.getEntity());
         }
     }
 
     public void testGetRequest() throws IOException {
-        try (ElasticsearchResponse response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null)) {
+        try (Response response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null)) {
             assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
             assertNotNull(response.getEntity());
             assertThat(EntityUtils.toString(response.getEntity()), containsString("cluster_name"));
diff --git a/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java b/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
index 9d25414f4d8..4af8b4dd701 100644
--- a/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
+++ b/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
@@ -32,7 +32,7 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.ActionFilter;
 import org.elasticsearch.action.termvectors.MultiTermVectorsRequest;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.client.ElasticsearchResponse;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.inject.Module;
@@ -218,7 +218,7 @@ public class ContextAndHeaderTransportIT extends ESIntegTestCase {
             restController.registerRelevantHeaders(relevantHeaderName);
         }
 
-        try (ElasticsearchResponse response = getRestClient().performRequest(
+        try (Response response = getRestClient().performRequest(
                 "GET", "/" + queryIndex + "/_search", Collections.emptyMap(), null,
                 new BasicHeader(randomHeaderKey, randomHeaderValue), new BasicHeader(relevantHeaderName, randomHeaderValue))) {
             assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index c27d3ed5604..e7fb35cabea 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -22,7 +22,7 @@ package org.elasticsearch.test.rest;
 import com.carrotsearch.randomizedtesting.RandomizedTest;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction;
-import org.elasticsearch.client.ElasticsearchResponseException;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.settings.Settings;
@@ -276,9 +276,9 @@ public abstract class ESRestTestCase extends ESTestCase {
         deleteIndicesArgs.put("index", "*");
         try {
             adminExecutionContext.callApi("indices.delete", deleteIndicesArgs, Collections.emptyList(), Collections.emptyMap());
-        } catch (ElasticsearchResponseException e) {
+        } catch (ResponseException e) {
             // 404 here just means we had no indexes
-            if (e.getElasticsearchResponse().getStatusLine().getStatusCode() != 404) {
+            if (e.getResponse().getStatusLine().getStatusCode() != 404) {
                 throw e;
             }
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java
index 1fa3dba5d83..34397f03d94 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java
@@ -19,7 +19,7 @@
 package org.elasticsearch.test.rest;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.client.ElasticsearchResponseException;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
@@ -78,7 +78,7 @@ public class RestTestExecutionContext implements Closeable {
             //we always stash the last response body
             stash.stashResponse(response);
             return response;
-        } catch(ElasticsearchResponseException e) {
+        } catch(ResponseException e) {
             response = new RestTestResponse(e);
             throw e;
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java
index d21e302168b..6a9dd888504 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java
@@ -33,8 +33,8 @@ import org.apache.http.message.BasicHeader;
 import org.apache.http.ssl.SSLContexts;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
-import org.elasticsearch.client.ElasticsearchResponse;
-import org.elasticsearch.client.ElasticsearchResponseException;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.PathUtils;
@@ -106,8 +106,8 @@ public class RestTestClient implements Closeable {
             //we don't really use the urls here, we rely on the client doing round-robin to touch all the nodes in the cluster
             String method = restApi.getMethods().get(0);
             String endpoint = restApi.getPaths().get(0);
-            ElasticsearchResponse elasticsearchResponse = restClient.performRequest(method, endpoint, Collections.emptyMap(), null);
-            RestTestResponse restTestResponse = new RestTestResponse(elasticsearchResponse);
+            Response response = restClient.performRequest(method, endpoint, Collections.emptyMap(), null);
+            RestTestResponse restTestResponse = new RestTestResponse(response);
             Object latestVersion = restTestResponse.evaluate("version.number");
             if (latestVersion == null) {
                 throw new RuntimeException("elasticsearch version not found in the response");
@@ -143,7 +143,7 @@ public class RestTestClient implements Closeable {
                 entity = new StringEntity(body, RestClient.JSON_CONTENT_TYPE);
             }
             // And everything else is a url parameter!
-            ElasticsearchResponse response = restClient.performRequest(method, path, queryStringParams, entity);
+            Response response = restClient.performRequest(method, path, queryStringParams, entity);
             return new RestTestResponse(response);
         }
 
@@ -247,11 +247,11 @@ public class RestTestClient implements Closeable {
 
         logger.debug("calling api [{}]", apiName);
         try {
-            ElasticsearchResponse response = restClient.performRequest(requestMethod, requestPath,
+            Response response = restClient.performRequest(requestMethod, requestPath,
                     queryStringParams, requestBody, requestHeaders);
             return new RestTestResponse(response);
-        } catch(ElasticsearchResponseException e) {
-            if (ignores.contains(e.getElasticsearchResponse().getStatusLine().getStatusCode())) {
+        } catch(ResponseException e) {
+            if (ignores.contains(e.getResponse().getStatusLine().getStatusCode())) {
                 return new RestTestResponse(e);
             }
             throw e;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java
index 5406fb79443..5b5773d6fdc 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java
@@ -21,8 +21,8 @@ package org.elasticsearch.test.rest.client;
 import org.apache.http.client.methods.HttpHead;
 import org.apache.http.util.EntityUtils;
 import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.client.ElasticsearchResponse;
-import org.elasticsearch.client.ElasticsearchResponseException;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.test.rest.Stash;
 import org.elasticsearch.test.rest.json.JsonPath;
 
@@ -35,11 +35,11 @@ import java.nio.charset.StandardCharsets;
  */
 public class RestTestResponse {
 
-    private final ElasticsearchResponse response;
+    private final Response response;
     private final String body;
     private JsonPath parsedResponse;
 
-    public RestTestResponse(ElasticsearchResponse response) {
+    public RestTestResponse(Response response) {
         this.response = response;
         if (response.getEntity() != null) {
             try {
@@ -55,8 +55,8 @@ public class RestTestResponse {
         }
     }
 
-    public RestTestResponse(ElasticsearchResponseException responseException) {
-        this.response = responseException.getElasticsearchResponse();
+    public RestTestResponse(ResponseException responseException) {
+        this.response = responseException.getResponse();
         this.body = responseException.getResponseBody();
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/DoSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/DoSection.java
index 76593d4af9d..2547d6becea 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/DoSection.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/DoSection.java
@@ -18,7 +18,7 @@
  */
 package org.elasticsearch.test.rest.section;
 
-import org.elasticsearch.client.ElasticsearchResponseException;
+import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.logging.ESLogger;
@@ -102,7 +102,7 @@ public class DoSection implements ExecutableSection {
                 }
                 fail(formatStatusCodeMessage(restTestResponse, catchStatusCode));
             }
-        } catch(ElasticsearchResponseException e) {
+        } catch(ResponseException e) {
             RestTestResponse restTestResponse = new RestTestResponse(e);
             if (!Strings.hasLength(catchParam)) {
                 fail(formatStatusCodeMessage(restTestResponse, "2xx"));
@@ -111,7 +111,7 @@ public class DoSection implements ExecutableSection {
             } else if (catchParam.length() > 2 && catchParam.startsWith("/") && catchParam.endsWith("/")) {
                 //the text of the error message matches regular expression
                 assertThat(formatStatusCodeMessage(restTestResponse, "4xx|5xx"),
-                        e.getElasticsearchResponse().getStatusLine().getStatusCode(), greaterThanOrEqualTo(400));
+                        e.getResponse().getStatusLine().getStatusCode(), greaterThanOrEqualTo(400));
                 Object error = executionContext.response("error");
                 assertThat("error was expected in the response", error, notNullValue());
                 //remove delimiters from regex

From 053a615686b3563fcfb5ee8b122d261306f46f4f Mon Sep 17 00:00:00 2001
From: Britta Weber <britta.weber@elasticsearch.com>
Date: Thu, 9 Jun 2016 15:11:48 +0200
Subject: [PATCH 0736/1311] [TEST] wait for yellow before query execution

We can remove this once https://github.com/elastic/elasticsearch/pull/18759
is in.
---
 docs/reference/search/request/sort.asciidoc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/reference/search/request/sort.asciidoc b/docs/reference/search/request/sort.asciidoc
index 2f2e8882032..fd3dbffc1d6 100644
--- a/docs/reference/search/request/sort.asciidoc
+++ b/docs/reference/search/request/sort.asciidoc
@@ -26,6 +26,8 @@ PUT /my_index
         }
     }
 }
+
+GET /_cluster/health?wait_for_status=yellow
 --------------------------------------------------
 // CONSOLE
 

From 85606e8a4b60f86e9fe0e95bb8e7c426c5013358 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Thu, 9 Jun 2016 16:27:52 +0200
Subject: [PATCH 0737/1311] renamed all time intervals arguments and members to
 include the time unit in their name

---
 .../client/sniff/HostsSniffer.java            | 21 ++++-----
 .../elasticsearch/client/sniff/Sniffer.java   | 46 +++++++++----------
 .../sniff/HostsSnifferBuilderTests.java       |  4 +-
 .../client/sniff/SnifferBuilderTests.java     |  8 ++--
 .../org/elasticsearch/client/RestClient.java  | 38 +++++++--------
 .../client/RestClientBuilderTests.java        |  4 +-
 6 files changed, 60 insertions(+), 61 deletions(-)

diff --git a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
index 3da6e2b95cd..7cdb51066cb 100644
--- a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
+++ b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
@@ -52,9 +52,9 @@ public class HostsSniffer {
     private final Scheme scheme;
     private final JsonFactory jsonFactory = new JsonFactory();
 
-    protected HostsSniffer(RestClient restClient, long sniffRequestTimeout, Scheme scheme) {
+    protected HostsSniffer(RestClient restClient, long sniffRequestTimeoutMillis, Scheme scheme) {
         this.restClient = restClient;
-        this.sniffRequestParams = Collections.<String, String>singletonMap("timeout", sniffRequestTimeout + "ms");
+        this.sniffRequestParams = Collections.<String, String>singletonMap("timeout", sniffRequestTimeoutMillis + "ms");
         this.scheme = scheme;
     }
 
@@ -155,7 +155,7 @@ public class HostsSniffer {
         public static final long DEFAULT_SNIFF_REQUEST_TIMEOUT = TimeUnit.SECONDS.toMillis(1);
 
         private final RestClient restClient;
-        private long sniffRequestTimeout = DEFAULT_SNIFF_REQUEST_TIMEOUT;
+        private long sniffRequestTimeoutMillis = DEFAULT_SNIFF_REQUEST_TIMEOUT;
         private Scheme scheme;
 
         private Builder(RestClient restClient) {
@@ -164,15 +164,14 @@ public class HostsSniffer {
         }
 
         /**
-         * Sets the sniff request timeout to be passed in as a query string parameter to elasticsearch.
-         * Allows to halt the request without any failure, as only the nodes that have responded
-         * within this timeout will be returned.
+         * Sets the sniff request timeout (in milliseconds) to be passed in as a query string parameter to elasticsearch.
+         * Allows to halt the request without any failure, as only the nodes that have responded within this timeout will be returned.
          */
-        public Builder setSniffRequestTimeout(int sniffRequestTimeout) {
-            if (sniffRequestTimeout <= 0) {
-                throw new IllegalArgumentException("sniffRequestTimeout must be greater than 0");
+        public Builder setSniffRequestTimeoutMillis(int sniffRequestTimeoutMillis) {
+            if (sniffRequestTimeoutMillis <= 0) {
+                throw new IllegalArgumentException("sniffRequestTimeoutMillis must be greater than 0");
             }
-            this.sniffRequestTimeout = sniffRequestTimeout;
+            this.sniffRequestTimeoutMillis = sniffRequestTimeoutMillis;
             return this;
         }
 
@@ -189,7 +188,7 @@ public class HostsSniffer {
          * Creates a new {@link HostsSniffer} instance given the provided configuration
          */
         public HostsSniffer build() {
-            return new HostsSniffer(restClient, sniffRequestTimeout, scheme);
+            return new HostsSniffer(restClient, sniffRequestTimeoutMillis, scheme);
         }
     }
 }
diff --git a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
index cd5d1d93148..37240741b3d 100644
--- a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
+++ b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
@@ -72,17 +72,17 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
         private final HostsSniffer hostsSniffer;
         private final RestClient restClient;
 
-        private final long sniffInterval;
-        private final long sniffAfterFailureDelay;
+        private final long sniffIntervalMillis;
+        private final long sniffAfterFailureDelayMillis;
         private final ScheduledExecutorService scheduledExecutorService;
         private final AtomicBoolean running = new AtomicBoolean(false);
         private ScheduledFuture<?> scheduledFuture;
 
-        private Task(HostsSniffer hostsSniffer, RestClient restClient, long sniffInterval, long sniffAfterFailureDelay) {
+        private Task(HostsSniffer hostsSniffer, RestClient restClient, long sniffIntervalMillis, long sniffAfterFailureDelayMillis) {
             this.hostsSniffer = hostsSniffer;
             this.restClient = restClient;
-            this.sniffInterval = sniffInterval;
-            this.sniffAfterFailureDelay = sniffAfterFailureDelay;
+            this.sniffIntervalMillis = sniffIntervalMillis;
+            this.sniffAfterFailureDelayMillis = sniffAfterFailureDelayMillis;
             this.scheduledExecutorService = Executors.newScheduledThreadPool(1);
             scheduleNextRun(0);
         }
@@ -104,11 +104,11 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
 
         @Override
         public void run() {
-            sniff(null, sniffInterval);
+            sniff(null, sniffIntervalMillis);
         }
 
         void sniffOnFailure(HttpHost failedHost) {
-            sniff(failedHost, sniffAfterFailureDelay);
+            sniff(failedHost, sniffAfterFailureDelayMillis);
         }
 
         void sniff(HttpHost excludeHost, long nextSniffDelayMillis) {
@@ -158,9 +158,9 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
 
         private final RestClient restClient;
         private final HostsSniffer hostsSniffer;
-        private long sniffInterval = DEFAULT_SNIFF_INTERVAL;
+        private long sniffIntervalMillis = DEFAULT_SNIFF_INTERVAL;
         private boolean sniffOnFailure = true;
-        private long sniffAfterFailureDelay = DEFAULT_SNIFF_AFTER_FAILURE_DELAY;
+        private long sniffAfterFailureDelayMillis = DEFAULT_SNIFF_AFTER_FAILURE_DELAY;
 
         /**
          * Creates a new builder instance by providing the {@link RestClient} that will be used to communicate with elasticsearch,
@@ -174,21 +174,21 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
         }
 
         /**
-         * Sets the interval between consecutive ordinary sniff executions. Will be honoured when sniffOnFailure is disabled or
-         * when there are no failures between consecutive sniff executions.
-         * @throws IllegalArgumentException if sniffInterval is not greater than 0
+         * Sets the interval between consecutive ordinary sniff executions in milliseconds. Will be honoured when
+         * sniffOnFailure is disabled or when there are no failures between consecutive sniff executions.
+         * @throws IllegalArgumentException if sniffIntervalMillis is not greater than 0
          */
-        public Builder setSniffInterval(int sniffInterval) {
-            if (sniffInterval <= 0) {
-                throw new IllegalArgumentException("sniffInterval must be greater than 0");
+        public Builder setSniffIntervalMillis(int sniffIntervalMillis) {
+            if (sniffIntervalMillis <= 0) {
+                throw new IllegalArgumentException("sniffIntervalMillis must be greater than 0");
             }
-            this.sniffInterval = sniffInterval;
+            this.sniffIntervalMillis = sniffIntervalMillis;
             return this;
         }
 
         /**
          * Enables/disables sniffing on failure. If enabled, at each failure nodes will be reloaded, and a new sniff execution will
-         * be scheduled after a shorter time than usual (sniffAfterFailureDelay).
+         * be scheduled after a shorter time than usual (sniffAfterFailureDelayMillis).
          */
         public Builder setSniffOnFailure(boolean sniffOnFailure) {
             this.sniffOnFailure = sniffOnFailure;
@@ -196,13 +196,13 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
         }
 
         /**
-         * Sets the delay of a sniff execution scheduled after a failure.
+         * Sets the delay of a sniff execution scheduled after a failure (in milliseconds)
          */
-        public Builder setSniffAfterFailureDelay(int sniffAfterFailureDelay) {
-            if (sniffAfterFailureDelay <= 0) {
-                throw new IllegalArgumentException("sniffAfterFailureDelay must be greater than 0");
+        public Builder setSniffAfterFailureDelayMillis(int sniffAfterFailureDelayMillis) {
+            if (sniffAfterFailureDelayMillis <= 0) {
+                throw new IllegalArgumentException("sniffAfterFailureDelayMillis must be greater than 0");
             }
-            this.sniffAfterFailureDelay = sniffAfterFailureDelay;
+            this.sniffAfterFailureDelayMillis = sniffAfterFailureDelayMillis;
             return this;
         }
 
@@ -210,7 +210,7 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
          * Creates the {@link Sniffer} based on the provided configuration.
          */
         public Sniffer build() {
-            return new Sniffer(restClient, hostsSniffer, sniffInterval, sniffOnFailure, sniffAfterFailureDelay);
+            return new Sniffer(restClient, hostsSniffer, sniffIntervalMillis, sniffOnFailure, sniffAfterFailureDelayMillis);
         }
     }
 }
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferBuilderTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferBuilderTests.java
index 8f355ecc47a..38baccf931e 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferBuilderTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferBuilderTests.java
@@ -50,7 +50,7 @@ public class HostsSnifferBuilderTests extends LuceneTestCase {
             }
 
             try {
-                HostsSniffer.builder(client).setSniffRequestTimeout(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+                HostsSniffer.builder(client).setSniffRequestTimeoutMillis(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
                 fail("should have failed");
             } catch(IllegalArgumentException e) {
                 assertEquals(e.getMessage(), "sniffRequestTimeout must be greater than 0");
@@ -61,7 +61,7 @@ public class HostsSnifferBuilderTests extends LuceneTestCase {
                 builder.setScheme(RandomPicks.randomFrom(random(), HostsSniffer.Scheme.values()));
             }
             if (random().nextBoolean()) {
-                builder.setSniffRequestTimeout(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+                builder.setSniffRequestTimeoutMillis(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
             }
             assertNotNull(builder.build());
         }
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
index 863830370b6..3fb152906b4 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
@@ -55,14 +55,14 @@ public class SnifferBuilderTests extends LuceneTestCase {
             }
 
             try {
-                Sniffer.builder(client, hostsSniffer).setSniffInterval(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+                Sniffer.builder(client, hostsSniffer).setSniffIntervalMillis(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
                 fail("should have failed");
             } catch(IllegalArgumentException e) {
                 assertEquals("sniffInterval must be greater than 0", e.getMessage());
             }
 
             try {
-                Sniffer.builder(client, hostsSniffer).setSniffAfterFailureDelay(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+                Sniffer.builder(client, hostsSniffer).setSniffAfterFailureDelayMillis(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
                 fail("should have failed");
             } catch(IllegalArgumentException e) {
                 assertEquals("sniffAfterFailureDelay must be greater than 0", e.getMessage());
@@ -74,10 +74,10 @@ public class SnifferBuilderTests extends LuceneTestCase {
 
             Sniffer.Builder builder = Sniffer.builder(client, hostsSniffer);
             if (random().nextBoolean()) {
-                builder.setSniffInterval(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+                builder.setSniffIntervalMillis(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
             }
             if (random().nextBoolean()) {
-                builder.setSniffAfterFailureDelay(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+                builder.setSniffAfterFailureDelayMillis(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
             }
             if (random().nextBoolean()) {
                 builder.setSniffOnFailure(random().nextBoolean());
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index f0e4d702f57..81354ae7a85 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -86,15 +86,15 @@ public final class RestClient implements Closeable {
     //we don't rely on default headers supported by HttpClient as those cannot be replaced, plus it would get hairy
     //when we create the HttpClient instance on our own as there would be two different ways to set the default headers.
     private final Header[] defaultHeaders;
-    private final long maxRetryTimeout;
+    private final long maxRetryTimeoutMillis;
     private final AtomicInteger lastHostIndex = new AtomicInteger(0);
     private volatile Set<HttpHost> hosts;
     private final ConcurrentMap<HttpHost, DeadHostState> blacklist = new ConcurrentHashMap<>();
     private volatile FailureListener failureListener = new FailureListener();
 
-    private RestClient(CloseableHttpClient client, long maxRetryTimeout, Header[] defaultHeaders, HttpHost[] hosts) {
+    private RestClient(CloseableHttpClient client, long maxRetryTimeoutMillis, Header[] defaultHeaders, HttpHost[] hosts) {
         this.client = client;
-        this.maxRetryTimeout = maxRetryTimeout;
+        this.maxRetryTimeoutMillis = maxRetryTimeoutMillis;
         this.defaultHeaders = defaultHeaders;
         setHosts(hosts);
     }
@@ -139,7 +139,7 @@ public final class RestClient implements Closeable {
         HttpRequestBase request = createHttpRequest(method, uri, entity);
         setHeaders(request, headers);
         //we apply a soft margin so that e.g. if a request took 59 seconds and timeout is set to 60 we don't do another attempt
-        long retryTimeout = Math.round(this.maxRetryTimeout / (float)100 * 98);
+        long retryTimeout = Math.round(this.maxRetryTimeoutMillis / (float)100 * 98);
         IOException lastSeenException = null;
         long startTime = System.nanoTime();
         Iterator<HttpHost> hostIterator = nextHost();
@@ -358,16 +358,16 @@ public final class RestClient implements Closeable {
      * Rest client builder. Helps creating a new {@link RestClient}.
      */
     public static final class Builder {
-        public static final int DEFAULT_CONNECT_TIMEOUT = 1000;
-        public static final int DEFAULT_SOCKET_TIMEOUT = 10000;
-        public static final int DEFAULT_MAX_RETRY_TIMEOUT = DEFAULT_SOCKET_TIMEOUT;
-        public static final int DEFAULT_CONNECTION_REQUEST_TIMEOUT = 500;
+        public static final int DEFAULT_CONNECT_TIMEOUT_MILLIS = 1000;
+        public static final int DEFAULT_SOCKET_TIMEOUT_MILLIS = 10000;
+        public static final int DEFAULT_MAX_RETRY_TIMEOUT_MILLIS = DEFAULT_SOCKET_TIMEOUT_MILLIS;
+        public static final int DEFAULT_CONNECTION_REQUEST_TIMEOUT_MILLIS = 500;
 
         private static final Header[] EMPTY_HEADERS = new Header[0];
 
         private final HttpHost[] hosts;
         private CloseableHttpClient httpClient;
-        private int maxRetryTimeout = DEFAULT_MAX_RETRY_TIMEOUT;
+        private int maxRetryTimeout = DEFAULT_MAX_RETRY_TIMEOUT_MILLIS;
         private Header[] defaultHeaders = EMPTY_HEADERS;
 
         /**
@@ -392,16 +392,16 @@ public final class RestClient implements Closeable {
         }
 
         /**
-         * Sets the maximum timeout to honour in case of multiple retries of the same request.
-         * {@link #DEFAULT_MAX_RETRY_TIMEOUT} if not specified.
+         * Sets the maximum timeout (in milliseconds) to honour in case of multiple retries of the same request.
+         * {@link #DEFAULT_MAX_RETRY_TIMEOUT_MILLIS} if not specified.
          *
-         * @throws IllegalArgumentException if maxRetryTimeout is not greater than 0
+         * @throws IllegalArgumentException if maxRetryTimeoutMillis is not greater than 0
          */
-        public Builder setMaxRetryTimeout(int maxRetryTimeout) {
-            if (maxRetryTimeout <= 0) {
-                throw new IllegalArgumentException("maxRetryTimeout must be greater than 0");
+        public Builder setMaxRetryTimeoutMillis(int maxRetryTimeoutMillis) {
+            if (maxRetryTimeoutMillis <= 0) {
+                throw new IllegalArgumentException("maxRetryTimeoutMillis must be greater than 0");
             }
-            this.maxRetryTimeout = maxRetryTimeout;
+            this.maxRetryTimeout = maxRetryTimeoutMillis;
             return this;
         }
 
@@ -446,9 +446,9 @@ public final class RestClient implements Closeable {
             connectionManager.setMaxTotal(30);
 
             //default timeouts are all infinite
-            RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(DEFAULT_CONNECT_TIMEOUT)
-                    .setSocketTimeout(DEFAULT_SOCKET_TIMEOUT)
-                    .setConnectionRequestTimeout(DEFAULT_CONNECTION_REQUEST_TIMEOUT).build();
+            RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(DEFAULT_CONNECT_TIMEOUT_MILLIS)
+                    .setSocketTimeout(DEFAULT_SOCKET_TIMEOUT_MILLIS)
+                    .setConnectionRequestTimeout(DEFAULT_CONNECTION_REQUEST_TIMEOUT_MILLIS).build();
             return HttpClientBuilder.create().setConnectionManager(connectionManager).setDefaultRequestConfig(requestConfig).build();
         }
     }
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java b/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
index 75587ed8db1..e4e024457a4 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
@@ -53,7 +53,7 @@ public class RestClientBuilderTests extends LuceneTestCase {
         }
 
         try {
-            RestClient.builder(new HttpHost("localhost", 9200)).setMaxRetryTimeout(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+            RestClient.builder(new HttpHost("localhost", 9200)).setMaxRetryTimeoutMillis(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
             fail("should have failed");
         } catch(IllegalArgumentException e) {
             assertEquals("maxRetryTimeout must be greater than 0", e.getMessage());
@@ -91,7 +91,7 @@ public class RestClientBuilderTests extends LuceneTestCase {
             builder.setDefaultHeaders(headers);
         }
         if (random().nextBoolean()) {
-            builder.setMaxRetryTimeout(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+            builder.setMaxRetryTimeoutMillis(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
         }
         try (RestClient restClient = builder.build()) {
             assertNotNull(restClient);

From c028bf9250227c68f4f25622f211f34ca398b69a Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Thu, 9 Jun 2016 16:31:09 +0200
Subject: [PATCH 0738/1311] rename deadUntil to deadUntilNanos

---
 .../java/org/elasticsearch/client/DeadHostState.java | 12 ++++++------
 .../java/org/elasticsearch/client/RestClient.java    |  4 ++--
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/DeadHostState.java b/client/src/main/java/org/elasticsearch/client/DeadHostState.java
index 52ef6e2bc6b..89a1720d46e 100644
--- a/client/src/main/java/org/elasticsearch/client/DeadHostState.java
+++ b/client/src/main/java/org/elasticsearch/client/DeadHostState.java
@@ -34,17 +34,17 @@ class DeadHostState {
     static final DeadHostState INITIAL_DEAD_STATE = new DeadHostState();
 
     private final int failedAttempts;
-    private final long deadUntil;
+    private final long deadUntilNanos;
 
     private DeadHostState() {
         this.failedAttempts = 1;
-        this.deadUntil = System.nanoTime() + MIN_CONNECTION_TIMEOUT_NANOS;
+        this.deadUntilNanos = System.nanoTime() + MIN_CONNECTION_TIMEOUT_NANOS;
     }
 
     DeadHostState(DeadHostState previousDeadHostState) {
         long timeoutNanos = (long)Math.min(MIN_CONNECTION_TIMEOUT_NANOS * 2 * Math.pow(2, previousDeadHostState.failedAttempts * 0.5 - 1),
                 MAX_CONNECTION_TIMEOUT_NANOS);
-        this.deadUntil = System.nanoTime() + timeoutNanos;
+        this.deadUntilNanos = System.nanoTime() + timeoutNanos;
         this.failedAttempts = previousDeadHostState.failedAttempts + 1;
     }
 
@@ -52,15 +52,15 @@ class DeadHostState {
      * Returns the timestamp (nanos) till the host is supposed to stay dead without being retried.
      * After that the host should be retried.
      */
-    long getDeadUntil() {
-        return deadUntil;
+    long getDeadUntilNanos() {
+        return deadUntilNanos;
     }
 
     @Override
     public String toString() {
         return "DeadHostState{" +
                 "failedAttempts=" + failedAttempts +
-                ", deadUntil=" + deadUntil +
+                ", deadUntilNanos=" + deadUntilNanos +
                 '}';
     }
 }
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 81354ae7a85..c14ca8e965c 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -224,7 +224,7 @@ public final class RestClient implements Closeable {
     private Iterator<HttpHost> nextHost() {
         Set<HttpHost> filteredHosts = new HashSet<>(hosts);
         for (Map.Entry<HttpHost, DeadHostState> entry : blacklist.entrySet()) {
-            if (System.nanoTime() - entry.getValue().getDeadUntil() < 0) {
+            if (System.nanoTime() - entry.getValue().getDeadUntilNanos() < 0) {
                 filteredHosts.remove(entry.getKey());
             }
         }
@@ -235,7 +235,7 @@ public final class RestClient implements Closeable {
             Collections.sort(sortedHosts, new Comparator<Map.Entry<HttpHost, DeadHostState>>() {
                 @Override
                 public int compare(Map.Entry<HttpHost, DeadHostState> o1, Map.Entry<HttpHost, DeadHostState> o2) {
-                    return Long.compare(o1.getValue().getDeadUntil(), o2.getValue().getDeadUntil());
+                    return Long.compare(o1.getValue().getDeadUntilNanos(), o2.getValue().getDeadUntilNanos());
                 }
             });
             HttpHost deadHost = sortedHosts.get(0).getKey();

From 742f9c6eaa54f9959787e68b6a23c968d875cba2 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Thu, 9 Jun 2016 16:34:01 +0200
Subject: [PATCH 0739/1311] nextHost to return Iterable<HttpHost> rather than
 Iterator

---
 .../java/org/elasticsearch/client/RestClient.java    | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index c14ca8e965c..88adf83893c 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -53,7 +53,6 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -142,10 +141,7 @@ public final class RestClient implements Closeable {
         long retryTimeout = Math.round(this.maxRetryTimeoutMillis / (float)100 * 98);
         IOException lastSeenException = null;
         long startTime = System.nanoTime();
-        Iterator<HttpHost> hostIterator = nextHost();
-        while (hostIterator.hasNext()) {
-            HttpHost host = hostIterator.next();
-
+        for (HttpHost host : nextHost()) {
             if (lastSeenException != null) {
                 long timeElapsed = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
                 long timeout = retryTimeout - timeElapsed;
@@ -221,7 +217,7 @@ public final class RestClient implements Closeable {
      * The iterator returned will never be empty, rather an {@link IllegalStateException} in case there are no hosts.
      * In case there are no healthy hosts available, or dead ones to be be retried, one dead host gets returned.
      */
-    private Iterator<HttpHost> nextHost() {
+    private Iterable<HttpHost> nextHost() {
         Set<HttpHost> filteredHosts = new HashSet<>(hosts);
         for (Map.Entry<HttpHost, DeadHostState> entry : blacklist.entrySet()) {
             if (System.nanoTime() - entry.getValue().getDeadUntilNanos() < 0) {
@@ -240,13 +236,13 @@ public final class RestClient implements Closeable {
             });
             HttpHost deadHost = sortedHosts.get(0).getKey();
             logger.trace("resurrecting host [" + deadHost + "]");
-            return Collections.singleton(deadHost).iterator();
+            return Collections.singleton(deadHost);
         }
 
         List<HttpHost> rotatedHosts = new ArrayList<>(filteredHosts);
         //TODO is it possible to make this O(1)? (rotate is O(n))
         Collections.rotate(rotatedHosts, rotatedHosts.size() - lastHostIndex.getAndIncrement());
-        return rotatedHosts.iterator();
+        return rotatedHosts;
     }
 
     /**

From 853ea9385bde2f20dee2723adc1e9805f59c06b1 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Thu, 9 Jun 2016 16:38:43 +0200
Subject: [PATCH 0740/1311] add comments on retries

---
 .../main/java/org/elasticsearch/client/RestClient.java | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 88adf83893c..3eb36b37878 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -138,19 +138,21 @@ public final class RestClient implements Closeable {
         HttpRequestBase request = createHttpRequest(method, uri, entity);
         setHeaders(request, headers);
         //we apply a soft margin so that e.g. if a request took 59 seconds and timeout is set to 60 we don't do another attempt
-        long retryTimeout = Math.round(this.maxRetryTimeoutMillis / (float)100 * 98);
+        long retryTimeoutMillis = Math.round(this.maxRetryTimeoutMillis / (float)100 * 98);
         IOException lastSeenException = null;
         long startTime = System.nanoTime();
         for (HttpHost host : nextHost()) {
             if (lastSeenException != null) {
-                long timeElapsed = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
-                long timeout = retryTimeout - timeElapsed;
+                //in case we are retrying, check whether maxRetryTimeout has been reached, in which case an exception will be thrown
+                long timeElapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
+                long timeout = retryTimeoutMillis - timeElapsedMillis;
                 if (timeout <= 0) {
                     IOException retryTimeoutException = new IOException(
-                            "request retries exceeded max retry timeout [" + retryTimeout + "]");
+                            "request retries exceeded max retry timeout [" + retryTimeoutMillis + "]");
                     retryTimeoutException.addSuppressed(lastSeenException);
                     throw retryTimeoutException;
                 }
+                //also reset the request to make it reusable for the next attempt
                 request.reset();
             }
 

From cbdffc7965ff33d283ae27ef7d9216f03ba88920 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Thu, 9 Jun 2016 16:39:59 +0200
Subject: [PATCH 0741/1311] s/elasticsearchResponse/response

---
 .../org/elasticsearch/client/RestClient.java  | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 3eb36b37878..3539c84293a 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -156,35 +156,35 @@ public final class RestClient implements Closeable {
                 request.reset();
             }
 
-            CloseableHttpResponse response;
+            CloseableHttpResponse httpResponse;
             try {
-                response = client.execute(host, request);
+                httpResponse = client.execute(host, request);
             } catch(IOException e) {
                 RequestLogger.log(logger, "request failed", request, host, e);
                 onFailure(host);
                 lastSeenException = addSuppressedException(lastSeenException, e);
                 continue;
             }
-            Response elasticsearchResponse = new Response(request.getRequestLine(), host, response);
+            Response response = new Response(request.getRequestLine(), host, httpResponse);
             int statusCode = response.getStatusLine().getStatusCode();
             if (statusCode < 300 || (request.getMethod().equals(HttpHead.METHOD_NAME) && statusCode == 404) ) {
-                RequestLogger.log(logger, "request succeeded", request, host, response);
+                RequestLogger.log(logger, "request succeeded", request, host, httpResponse);
                 onSuccess(host);
-                return elasticsearchResponse;
+                return response;
             } else {
-                RequestLogger.log(logger, "request failed", request, host, response);
+                RequestLogger.log(logger, "request failed", request, host, httpResponse);
                 String responseBody;
                 try {
-                    if (elasticsearchResponse.getEntity() == null) {
+                    if (response.getEntity() == null) {
                         responseBody = null;
                     } else {
-                        responseBody = EntityUtils.toString(elasticsearchResponse.getEntity());
+                        responseBody = EntityUtils.toString(response.getEntity());
                     }
                 } finally {
-                    elasticsearchResponse.close();
+                    response.close();
                 }
                 ResponseException responseException = new ResponseException(
-                        elasticsearchResponse, responseBody);
+                        response, responseBody);
                 lastSeenException = addSuppressedException(lastSeenException, responseException);
                 //clients don't retry on 500 because elasticsearch still misuses it instead of 400 in some places
                 if (statusCode == 502 || statusCode == 503 || statusCode == 504) {

From ce211119d00459955cb06190a72e3ad42cede7b2 Mon Sep 17 00:00:00 2001
From: Areek Zillur <areek.zillur@elasticsearch.com>
Date: Thu, 9 Jun 2016 10:19:50 -0400
Subject: [PATCH 0742/1311] Add Shrink request source parser to parse create
 index request body

Follow up to https://github.com/elastic/elasticsearch/pull/18732#discussion_r66407196
---
 .../admin/indices/shrink/ShrinkRequest.java   | 34 +++++++++++++++++++
 .../admin/indices/RestShrinkIndexAction.java  |  2 +-
 .../test/indices.shrink/10_basic.yaml         |  3 +-
 3 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java
index a14a3f37a86..791d35220e2 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java
@@ -18,15 +18,25 @@
  */
 package org.elasticsearch.action.admin.indices.shrink;
 
+import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParseFieldMatcherSupplier;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 
 import java.io.IOException;
+import java.util.Map;
 import java.util.Objects;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
@@ -36,6 +46,17 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
  */
 public class ShrinkRequest extends AcknowledgedRequest<ShrinkRequest> implements IndicesRequest {
 
+    public static ObjectParser<ShrinkRequest, ParseFieldMatcherSupplier> PARSER =
+        new ObjectParser<>("shrink_request", null);
+    static {
+        PARSER.declareField((parser, request, parseFieldMatcherSupplier) ->
+                request.getShrinkIndexRequest().settings(parser.map()),
+            new ParseField("settings"), ObjectParser.ValueType.OBJECT);
+        PARSER.declareField((parser, request, parseFieldMatcherSupplier) ->
+                request.getShrinkIndexRequest().aliases(parser.map()),
+            new ParseField("aliases"), ObjectParser.ValueType.OBJECT);
+    }
+
     private CreateIndexRequest shrinkIndexRequest;
     private String sourceIndex;
 
@@ -104,4 +125,17 @@ public class ShrinkRequest extends AcknowledgedRequest<ShrinkRequest> implements
     public String getSourceIndex() {
         return sourceIndex;
     }
+
+    public void source(BytesReference source) {
+        XContentType xContentType = XContentFactory.xContentType(source);
+        if (xContentType != null) {
+            try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(source)) {
+                PARSER.parse(parser, this, () -> ParseFieldMatcher.EMPTY);
+            } catch (IOException e) {
+                throw new ElasticsearchParseException("failed to parse source for shrink index", e);
+            }
+        } else {
+            throw new ElasticsearchParseException("failed to parse content type for shrink index source");
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java
index 42117cbf328..f145bcbf02a 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java
@@ -52,7 +52,7 @@ public class RestShrinkIndexAction extends BaseRestHandler {
         }
         ShrinkRequest shrinkIndexRequest = new ShrinkRequest(request.param("target"), request.param("index"));
         if (request.hasContent()) {
-            shrinkIndexRequest.getShrinkIndexRequest().source(request.content());
+            shrinkIndexRequest.source(request.content());
         }
         shrinkIndexRequest.timeout(request.paramAsTime("timeout", shrinkIndexRequest.timeout()));
         shrinkIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", shrinkIndexRequest.masterNodeTimeout()));
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yaml
index 4ef428b9600..b1aac4952c4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yaml
@@ -54,7 +54,8 @@
         index: "source"
         target: "target"
         body:
-          index.number_of_replicas: 0
+          settings:
+            index.number_of_replicas: 0
 
   - do:
       cluster.health:

From de8b9fd5798a09c831be829c4a504d0369b3674e Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Thu, 9 Jun 2016 16:51:08 +0200
Subject: [PATCH 0743/1311] use switch for status codes and add comments

---
 .../org/elasticsearch/client/RestClient.java  | 44 ++++++++++---------
 1 file changed, 23 insertions(+), 21 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 3539c84293a..350b750415d 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -171,31 +171,33 @@ public final class RestClient implements Closeable {
                 RequestLogger.log(logger, "request succeeded", request, host, httpResponse);
                 onSuccess(host);
                 return response;
-            } else {
-                RequestLogger.log(logger, "request failed", request, host, httpResponse);
-                String responseBody;
-                try {
-                    if (response.getEntity() == null) {
-                        responseBody = null;
-                    } else {
-                        responseBody = EntityUtils.toString(response.getEntity());
-                    }
-                } finally {
-                    response.close();
-                }
-                ResponseException responseException = new ResponseException(
-                        response, responseBody);
-                lastSeenException = addSuppressedException(lastSeenException, responseException);
-                //clients don't retry on 500 because elasticsearch still misuses it instead of 400 in some places
-                if (statusCode == 502 || statusCode == 503 || statusCode == 504) {
-                    onFailure(host);
+            }
+            RequestLogger.log(logger, "request failed", request, host, httpResponse);
+            String responseBody;
+            try {
+                if (response.getEntity() == null) {
+                    responseBody = null;
                 } else {
-                    //don't retry and call onSuccess as the error should be a request problem, the node is alive
-                    onSuccess(host);
-                    break;
+                    responseBody = EntityUtils.toString(response.getEntity());
                 }
+            } finally {
+                response.close();
+            }
+            lastSeenException = addSuppressedException(lastSeenException, new ResponseException(response, responseBody));
+            switch(statusCode) {
+                case 502:
+                case 503:
+                case 504:
+                    //mark host dead and retry against next one
+                    onFailure(host);
+                    break;
+                default:
+                    //mark host alive and don't retry, as the error should be a request problem
+                    onSuccess(host);
+                    throw lastSeenException;
             }
         }
+        //we get here only when we tried all nodes and they all failed
         assert lastSeenException != null;
         throw lastSeenException;
     }

From 9a4971d3fd296bbaab44bbece3d096a256896ffc Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Thu, 9 Jun 2016 16:57:57 +0200
Subject: [PATCH 0744/1311] fix line length

---
 .../java/org/elasticsearch/client/RestClientBuilderTests.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java b/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
index e4e024457a4..0269e6ada60 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
@@ -53,7 +53,8 @@ public class RestClientBuilderTests extends LuceneTestCase {
         }
 
         try {
-            RestClient.builder(new HttpHost("localhost", 9200)).setMaxRetryTimeoutMillis(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+            RestClient.builder(new HttpHost("localhost", 9200))
+                    .setMaxRetryTimeoutMillis(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
             fail("should have failed");
         } catch(IllegalArgumentException e) {
             assertEquals("maxRetryTimeout must be greater than 0", e.getMessage());

From d8c0fad08f93580c94966a5151e17d03fff99b2a Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Thu, 9 Jun 2016 17:06:58 +0200
Subject: [PATCH 0745/1311] fix failing tests

---
 .../elasticsearch/client/sniff/HostsSnifferBuilderTests.java  | 2 +-
 .../org/elasticsearch/client/sniff/SnifferBuilderTests.java   | 4 ++--
 .../java/org/elasticsearch/client/RestClientBuilderTests.java | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferBuilderTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferBuilderTests.java
index 38baccf931e..ff9c150d954 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferBuilderTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferBuilderTests.java
@@ -53,7 +53,7 @@ public class HostsSnifferBuilderTests extends LuceneTestCase {
                 HostsSniffer.builder(client).setSniffRequestTimeoutMillis(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
                 fail("should have failed");
             } catch(IllegalArgumentException e) {
-                assertEquals(e.getMessage(), "sniffRequestTimeout must be greater than 0");
+                assertEquals(e.getMessage(), "sniffRequestTimeoutMillis must be greater than 0");
             }
 
             HostsSniffer.Builder builder = HostsSniffer.builder(client);
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
index 3fb152906b4..23d74e9f4f5 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
@@ -58,14 +58,14 @@ public class SnifferBuilderTests extends LuceneTestCase {
                 Sniffer.builder(client, hostsSniffer).setSniffIntervalMillis(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
                 fail("should have failed");
             } catch(IllegalArgumentException e) {
-                assertEquals("sniffInterval must be greater than 0", e.getMessage());
+                assertEquals("sniffIntervalMillis must be greater than 0", e.getMessage());
             }
 
             try {
                 Sniffer.builder(client, hostsSniffer).setSniffAfterFailureDelayMillis(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
                 fail("should have failed");
             } catch(IllegalArgumentException e) {
-                assertEquals("sniffAfterFailureDelay must be greater than 0", e.getMessage());
+                assertEquals("sniffAfterFailureDelayMillis must be greater than 0", e.getMessage());
             }
 
             try (Sniffer sniffer = Sniffer.builder(client, hostsSniffer).build()) {
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java b/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
index 0269e6ada60..88b1406d925 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
@@ -57,7 +57,7 @@ public class RestClientBuilderTests extends LuceneTestCase {
                     .setMaxRetryTimeoutMillis(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
             fail("should have failed");
         } catch(IllegalArgumentException e) {
-            assertEquals("maxRetryTimeout must be greater than 0", e.getMessage());
+            assertEquals("maxRetryTimeoutMillis must be greater than 0", e.getMessage());
         }
 
         try {

From 3474a145b0a259d0194a70203e7db7785998b3bb Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Thu, 9 Jun 2016 17:09:15 +0200
Subject: [PATCH 0746/1311] fix line length

---
 .../org/elasticsearch/client/sniff/SnifferBuilderTests.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
index 23d74e9f4f5..f09ef77f5cf 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
@@ -62,7 +62,8 @@ public class SnifferBuilderTests extends LuceneTestCase {
             }
 
             try {
-                Sniffer.builder(client, hostsSniffer).setSniffAfterFailureDelayMillis(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+                Sniffer.builder(client, hostsSniffer)
+                        .setSniffAfterFailureDelayMillis(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
                 fail("should have failed");
             } catch(IllegalArgumentException e) {
                 assertEquals("sniffAfterFailureDelayMillis must be greater than 0", e.getMessage());

From bd276ef5f1272fbbe47488010a1a994addd6dca7 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 9 Jun 2016 11:24:06 -0400
Subject: [PATCH 0747/1311] [test] Check for listener calling error

Failing to call a refresh listener is logger at WARN but that'll
cause test failure. This adds explicit assertions that there are
no errors.
---
 .../index/shard/RefreshListenersTests.java    | 24 +++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
index 2543668f557..eb0fca7ce9e 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
@@ -155,10 +155,12 @@ public class RefreshListenersTests extends ESTestCase {
         DummyRefreshListener forcingListener = new DummyRefreshListener();
         listeners.addOrNotify(index.getTranslogLocation(), forcingListener);
         assertTrue("Forced listener wasn't forced?", forcingListener.forcedRefresh.get());
+        forcingListener.assertNoError();
 
         // That forces all the listeners through. It would be on the listener ThreadPool but we've made all of those execute immediately.
         for (DummyRefreshListener listener : nonForcedListeners) {
             assertEquals("Expected listener called with unforced refresh!", Boolean.FALSE, listener.forcedRefresh.get());
+            listener.assertNoError();
         }
         assertFalse(listeners.refreshNeeded());
     }
@@ -176,6 +178,7 @@ public class RefreshListenersTests extends ESTestCase {
         DummyRefreshListener listener = new DummyRefreshListener();
         listeners.addOrNotify(index.getTranslogLocation(), listener);
         assertFalse(listener.forcedRefresh.get());
+        listener.assertNoError();
     }
 
     /**
@@ -199,6 +202,7 @@ public class RefreshListenersTests extends ESTestCase {
                 listeners.addOrNotify(index.getTranslogLocation(), listener);
                 assertBusy(() -> assertNotNull(listener.forcedRefresh.get()));
                 assertFalse(listener.forcedRefresh.get());
+                listener.assertNoError();
             }
         } finally {
             run.set(false);
@@ -234,6 +238,7 @@ public class RefreshListenersTests extends ESTestCase {
                         if (threadCount < maxListeners) {
                             assertFalse(listener.forcedRefresh.get());
                         }
+                        listener.assertNoError();
 
                         Engine.Get get = new Engine.Get(false, index.uid());
                         try (Engine.GetResult getResult = engine.get(get)) {
@@ -281,13 +286,24 @@ public class RefreshListenersTests extends ESTestCase {
         /**
          * When the listener is called this captures it's only argument.
          */
-        private AtomicReference<Boolean> forcedRefresh = new AtomicReference<>();
+        AtomicReference<Boolean> forcedRefresh = new AtomicReference<>();
+        private volatile Throwable error;
 
         @Override
         public void accept(Boolean forcedRefresh) {
-            assertNotNull(forcedRefresh);
-            Boolean oldValue = this.forcedRefresh.getAndSet(forcedRefresh);
-            assertNull("Listener called twice", oldValue);
+            try {
+                assertNotNull(forcedRefresh);
+                Boolean oldValue = this.forcedRefresh.getAndSet(forcedRefresh);
+                assertNull("Listener called twice", oldValue);
+            } catch (Throwable e) {
+                error = e;
+            }
+        }
+
+        public void assertNoError() {
+            if (error != null) {
+                throw new RuntimeException(error);
+            }
         }
     }
 }

From 33bdab1a5acbb92fc0b08bad52259675bbee7a9c Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Thu, 9 Jun 2016 17:32:51 +0200
Subject: [PATCH 0748/1311] Build: temporarily disable dependency licenses
 check (till our own get filtered out)

---
 client-sniffer/build.gradle                   |   3 +
 .../licenses/client-5.0.0-SNAPSHOT.jar.sha1   |   1 -
 client-sniffer/licenses/client-LICENSE.txt    | 202 ------------------
 client-sniffer/licenses/client-NOTICE.txt     |   5 -
 4 files changed, 3 insertions(+), 208 deletions(-)
 delete mode 100644 client-sniffer/licenses/client-5.0.0-SNAPSHOT.jar.sha1
 delete mode 100644 client-sniffer/licenses/client-LICENSE.txt
 delete mode 100644 client-sniffer/licenses/client-NOTICE.txt

diff --git a/client-sniffer/build.gradle b/client-sniffer/build.gradle
index d561e007184..7c6527abb4c 100644
--- a/client-sniffer/build.gradle
+++ b/client-sniffer/build.gradle
@@ -65,6 +65,9 @@ jarHell.enabled=false
 //NamingConventionCheck is part of test-framework, which we don't want to pull in as it depends on es core
 namingConventions.enabled=false
 
+//TODO re-enable once our own jars are properly filtered out
+dependencyLicenses.enabled=false
+
 thirdPartyAudit.excludes = [
   //commons-logging optional dependencies
   'org.apache.avalon.framework.logger.Logger',
diff --git a/client-sniffer/licenses/client-5.0.0-SNAPSHOT.jar.sha1 b/client-sniffer/licenses/client-5.0.0-SNAPSHOT.jar.sha1
deleted file mode 100644
index de30dc58fe6..00000000000
--- a/client-sniffer/licenses/client-5.0.0-SNAPSHOT.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4ba4746aa38f81ec7e8341da8c86784bf5384046
\ No newline at end of file
diff --git a/client-sniffer/licenses/client-LICENSE.txt b/client-sniffer/licenses/client-LICENSE.txt
deleted file mode 100644
index d6456956733..00000000000
--- a/client-sniffer/licenses/client-LICENSE.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
diff --git a/client-sniffer/licenses/client-NOTICE.txt b/client-sniffer/licenses/client-NOTICE.txt
deleted file mode 100644
index c99b9581931..00000000000
--- a/client-sniffer/licenses/client-NOTICE.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-Elasticsearch
-Copyright 2009-2016 Elasticsearch
-
-This product includes software developed by The Apache Software
-Foundation (http://www.apache.org/).

From bd773359d5b603bcb5db8a41bbbda6da5b6c8dd1 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Thu, 9 Jun 2016 17:39:52 +0200
Subject: [PATCH 0749/1311] [TEST] add SuppresForbidden annotation for client
 project

---
 client/build.gradle                           |  7 ++--
 .../client/RestClientIntegTests.java          |  1 +
 .../client/SuppressForbidden.java             | 34 +++++++++++++++++++
 3 files changed, 39 insertions(+), 3 deletions(-)
 create mode 100644 client/src/test/java/org/elasticsearch/client/SuppressForbidden.java

diff --git a/client/build.gradle b/client/build.gradle
index 2c82fd74569..c947f94ff26 100644
--- a/client/build.gradle
+++ b/client/build.gradle
@@ -53,10 +53,11 @@ forbiddenApisMain {
 //excludes don't seem to work though and we don't want to have our own @SuppressForbidden
 forbiddenApisTest.enabled=false
 
-//forbiddenApisTest {
+forbiddenApisTest {
   //client does not depend on core, so only jdk signatures should be checked
-  //signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
-//}
+  signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
+  suppressAnnotations = ['**.SuppressForbidden']
+}
 
 //JarHell is part of es core, which we don't want to pull in
 jarHell.enabled=false
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java b/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
index 4dadfc1a918..1505ccd55d2 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
@@ -56,6 +56,7 @@ import static org.hamcrest.CoreMatchers.equalTo;
  * Integration test to check interaction between {@link RestClient} and {@link org.apache.http.client.HttpClient}.
  * Works against a real http server, one single host.
  */
+@SuppressForbidden(reason = "uses sun HttpServer")
 public class RestClientIntegTests extends LuceneTestCase {
 
     private static HttpServer httpServer;
diff --git a/client/src/test/java/org/elasticsearch/client/SuppressForbidden.java b/client/src/test/java/org/elasticsearch/client/SuppressForbidden.java
new file mode 100644
index 00000000000..309cc666bc4
--- /dev/null
+++ b/client/src/test/java/org/elasticsearch/client/SuppressForbidden.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation to suppress forbidden-apis errors inside a whole class, a method, or a field.
+ */
+@Retention(RetentionPolicy.CLASS)
+@Target({ ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE })
+public @interface SuppressForbidden {
+    String reason();
+}
\ No newline at end of file

From 9027e8a719d81ebc9fc19f4b99bbb5dcec1d9def Mon Sep 17 00:00:00 2001
From: Areek Zillur <areek.zillur@elasticsearch.com>
Date: Thu, 9 Jun 2016 11:55:10 -0400
Subject: [PATCH 0750/1311] renamed simulated mode to dry_run mode

---
 .../indices/rollover/RolloverRequest.java      | 14 +++++++-------
 .../rollover/RolloverRequestBuilder.java       |  4 ++--
 .../indices/rollover/RolloverResponse.java     | 18 +++++++++---------
 .../rollover/TransportRolloverAction.java      |  2 +-
 .../admin/indices/RestRolloverIndexAction.java |  3 +--
 .../admin/indices/rollover/RolloverIT.java     | 16 ++++++++--------
 docs/reference/indices/rollover-index.asciidoc |  7 +++----
 7 files changed, 31 insertions(+), 33 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
index abae56cb5fb..10d780a005b 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
@@ -70,7 +70,7 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
     }
 
     private String alias;
-    private boolean simulate;
+    private boolean dryRun;
     private Set<Condition> conditions = new HashSet<>(2);
     private CreateIndexRequest createIndexRequest = new CreateIndexRequest("_na_");
 
@@ -96,7 +96,7 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         alias = in.readString();
-        simulate = in.readBoolean();
+        dryRun = in.readBoolean();
         int size = in.readVInt();
         for (int i = 0; i < size; i++) {
             this.conditions.add(in.readNamedWriteable(Condition.class));
@@ -109,7 +109,7 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeString(alias);
-        out.writeBoolean(simulate);
+        out.writeBoolean(dryRun);
         out.writeVInt(conditions.size());
         for (Condition condition : conditions) {
             out.writeNamedWriteable(condition);
@@ -137,8 +137,8 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
     /**
      * Sets if the rollover should not be executed when conditions are met
      */
-    public void simulate(boolean simulate) {
-        this.simulate = simulate;
+    public void dryRun(boolean dryRun) {
+        this.dryRun = dryRun;
     }
 
     /**
@@ -163,8 +163,8 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
         this.createIndexRequest = Objects.requireNonNull(createIndexRequest, "create index request must not be null");;
     }
 
-    boolean isSimulate() {
-        return simulate;
+    boolean isDryRun() {
+        return dryRun;
     }
 
     Set<Condition> getConditions() {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
index cfce9d24ad4..8beebc2e516 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
@@ -46,8 +46,8 @@ public class RolloverRequestBuilder extends MasterNodeOperationRequestBuilder<Ro
         return this;
     }
 
-    public RolloverRequestBuilder simulate(boolean simulate) {
-        this.request.simulate(simulate);
+    public RolloverRequestBuilder dryRun(boolean dryRun) {
+        this.request.dryRun(dryRun);
         return this;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
index 14feec0248f..4bfd7905129 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
@@ -37,7 +37,7 @@ public final class RolloverResponse extends ActionResponse implements ToXContent
     private String oldIndex;
     private String newIndex;
     private Set<Map.Entry<String, Boolean>> conditionStatus;
-    private boolean simulate;
+    private boolean dryRun;
     private boolean rolledOver;
     private boolean rolloverIndexCreated;
 
@@ -45,10 +45,10 @@ public final class RolloverResponse extends ActionResponse implements ToXContent
     }
 
     RolloverResponse(String oldIndex, String newIndex, Set<Condition.Result> conditionResults,
-                     boolean simulate, boolean rolledOver, boolean rolloverIndexCreated) {
+                     boolean dryRun, boolean rolledOver, boolean rolloverIndexCreated) {
         this.oldIndex = oldIndex;
         this.newIndex = newIndex;
-        this.simulate = simulate;
+        this.dryRun = dryRun;
         this.rolledOver = rolledOver;
         this.rolloverIndexCreated = rolloverIndexCreated;
         this.conditionStatus = conditionResults.stream()
@@ -80,8 +80,8 @@ public final class RolloverResponse extends ActionResponse implements ToXContent
     /**
      * Returns if the rollover execution was skipped even when conditions were met
      */
-    public boolean isSimulate() {
-        return simulate;
+    public boolean isDryRun() {
+        return dryRun;
     }
 
     /**
@@ -111,7 +111,7 @@ public final class RolloverResponse extends ActionResponse implements ToXContent
             conditions.add(new AbstractMap.SimpleEntry<>(condition, satisfied));
         }
         conditionStatus = conditions;
-        simulate = in.readBoolean();
+        dryRun = in.readBoolean();
         rolledOver = in.readBoolean();
         rolloverIndexCreated = in.readBoolean();
     }
@@ -126,7 +126,7 @@ public final class RolloverResponse extends ActionResponse implements ToXContent
             out.writeString(entry.getKey());
             out.writeBoolean(entry.getValue());
         }
-        out.writeBoolean(simulate);
+        out.writeBoolean(dryRun);
         out.writeBoolean(rolledOver);
         out.writeBoolean(rolloverIndexCreated);
     }
@@ -136,7 +136,7 @@ public final class RolloverResponse extends ActionResponse implements ToXContent
         builder.field(Fields.OLD_INDEX, oldIndex);
         builder.field(Fields.NEW_INDEX, newIndex);
         builder.field(Fields.ROLLED_OVER, rolledOver);
-        builder.field(Fields.SIMULATED, simulate);
+        builder.field(Fields.DRY_RUN, dryRun);
         builder.field(Fields.ROLLOVER_INDEX_CREATED, rolloverIndexCreated);
         builder.startObject(Fields.CONDITIONS);
         for (Map.Entry<String, Boolean> entry : conditionStatus) {
@@ -149,7 +149,7 @@ public final class RolloverResponse extends ActionResponse implements ToXContent
     static final class Fields {
         static final String NEW_INDEX = "new_index";
         static final String OLD_INDEX = "old_index";
-        static final String SIMULATED = "simulated";
+        static final String DRY_RUN = "dry_run";
         static final String ROLLED_OVER = "rolled_over";
         static final String ROLLOVER_INDEX_CREATED = "rollover_index_created";
         static final String CONDITIONS = "conditions";
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
index 0cd29b84056..baf743466ee 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -106,7 +106,7 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
                     final Set<Condition.Result> conditionResults = evaluateConditions(rolloverRequest.getConditions(),
                         statsResponse.getTotal().getDocs(), metaData.index(sourceIndexName));
                     final String rolloverIndexName = generateRolloverIndexName(sourceIndexName);
-                    if (rolloverRequest.isSimulate()) {
+                    if (rolloverRequest.isDryRun()) {
                         listener.onResponse(
                             new RolloverResponse(sourceIndexName, rolloverIndexName, conditionResults, true, false,
                                 false));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
index cccdf22d1d4..635e22ce83c 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
@@ -38,7 +38,6 @@ public class RestRolloverIndexAction extends BaseRestHandler {
     public RestRolloverIndexAction(Settings settings, RestController controller, Client client) {
         super(settings, client);
         controller.registerHandler(RestRequest.Method.POST, "/{index}/_rollover", this);
-        controller.registerHandler(RestRequest.Method.GET, "/{index}/_rollover", this);
     }
 
     @SuppressWarnings({"unchecked"})
@@ -48,7 +47,7 @@ public class RestRolloverIndexAction extends BaseRestHandler {
         if (request.hasContent()) {
             rolloverIndexRequest.source(request.content());
         }
-        rolloverIndexRequest.simulate(request.method() == RestRequest.Method.GET || request.paramAsBoolean("simulate", false));
+        rolloverIndexRequest.dryRun(request.paramAsBoolean("dry_run", false));
         rolloverIndexRequest.timeout(request.paramAsTime("timeout", rolloverIndexRequest.timeout()));
         rolloverIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", rolloverIndexRequest.masterNodeTimeout()));
         client.admin().indices().rolloverIndex(rolloverIndexRequest, new RestToXContentListener<>(channel));
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
index bf28bdfdbc8..112fe91eaaf 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
@@ -39,7 +39,7 @@ public class RolloverIT extends ESIntegTestCase {
         final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias").get();
         assertThat(response.getOldIndex(), equalTo("test_index-1"));
         assertThat(response.getNewIndex(), equalTo("test_index-2"));
-        assertThat(response.isSimulate(), equalTo(false));
+        assertThat(response.isDryRun(), equalTo(false));
         assertThat(response.isRolledOver(), equalTo(true));
         assertThat(response.isRolloverIndexCreated(), equalTo(true));
         assertThat(response.getConditionStatus().size(), equalTo(0));
@@ -57,7 +57,7 @@ public class RolloverIT extends ESIntegTestCase {
         final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias").get();
         assertThat(response.getOldIndex(), equalTo("test_index-2"));
         assertThat(response.getNewIndex(), equalTo("test_index-3"));
-        assertThat(response.isSimulate(), equalTo(false));
+        assertThat(response.isDryRun(), equalTo(false));
         assertThat(response.isRolledOver(), equalTo(true));
         assertThat(response.isRolloverIndexCreated(), equalTo(true));
         assertThat(response.getConditionStatus().size(), equalTo(0));
@@ -80,7 +80,7 @@ public class RolloverIT extends ESIntegTestCase {
             .settings(settings).alias(new Alias("extra_alias")).get();
         assertThat(response.getOldIndex(), equalTo("test_index-2"));
         assertThat(response.getNewIndex(), equalTo("test_index-3"));
-        assertThat(response.isSimulate(), equalTo(false));
+        assertThat(response.isDryRun(), equalTo(false));
         assertThat(response.isRolledOver(), equalTo(true));
         assertThat(response.isRolloverIndexCreated(), equalTo(true));
         assertThat(response.getConditionStatus().size(), equalTo(0));
@@ -94,14 +94,14 @@ public class RolloverIT extends ESIntegTestCase {
         assertTrue(newIndex.getAliases().containsKey("extra_alias"));
     }
 
-    public void testRolloverSimulate() throws Exception {
+    public void testRolloverDryRun() throws Exception {
         assertAcked(prepareCreate("test_index-1").addAlias(new Alias("test_alias")).get());
         index("test_index-1", "type1", "1", "field", "value");
         flush("test_index-1");
-        final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias").simulate(true).get();
+        final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias").dryRun(true).get();
         assertThat(response.getOldIndex(), equalTo("test_index-1"));
         assertThat(response.getNewIndex(), equalTo("test_index-2"));
-        assertThat(response.isSimulate(), equalTo(true));
+        assertThat(response.isDryRun(), equalTo(true));
         assertThat(response.isRolledOver(), equalTo(false));
         assertThat(response.isRolloverIndexCreated(), equalTo(false));
         assertThat(response.getConditionStatus().size(), equalTo(0));
@@ -120,7 +120,7 @@ public class RolloverIT extends ESIntegTestCase {
             .addMaxIndexAgeCondition(TimeValue.timeValueHours(4)).get();
         assertThat(response.getOldIndex(), equalTo("test_index-0"));
         assertThat(response.getNewIndex(), equalTo("test_index-0"));
-        assertThat(response.isSimulate(), equalTo(false));
+        assertThat(response.isDryRun(), equalTo(false));
         assertThat(response.isRolledOver(), equalTo(false));
         assertThat(response.isRolloverIndexCreated(), equalTo(false));
         assertThat(response.getConditionStatus().size(), equalTo(1));
@@ -143,7 +143,7 @@ public class RolloverIT extends ESIntegTestCase {
         final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias").get();
         assertThat(response.getOldIndex(), equalTo("test_index-0"));
         assertThat(response.getNewIndex(), equalTo("test_index-1"));
-        assertThat(response.isSimulate(), equalTo(false));
+        assertThat(response.isDryRun(), equalTo(false));
         assertThat(response.isRolledOver(), equalTo(true));
         assertThat(response.isRolloverIndexCreated(), equalTo(false));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc
index 2c553a4e0f1..f4c228b9870 100644
--- a/docs/reference/indices/rollover-index.asciidoc
+++ b/docs/reference/indices/rollover-index.asciidoc
@@ -55,9 +55,8 @@ $ curl -XPOST 'http://localhost:9200/index_alias/_rollover' -d '{
 --------------------------------------------------
 <1> Set settings to override matching index template, `mappings` and `aliases` can also be provided.
 
-Using the http `GET` method for the API runs the rollover in simulated mode, where request conditions can be
-checked without performing the actual rollover. Setting `simulate=true` as a request parameter also runs
-the request in simulated mode.
+The rollover API supports `dry_run` mode, where request conditions can be checked without performing the
+actual rollover. The `dry_run` mode can be enabled by setting `dry_run=true` as a request parameter.
 
 An example response for the index rollover API:
 
@@ -67,7 +66,7 @@ An example response for the index rollover API:
 	"old_index": "index-1", <1>
 	"new_index": "index-2", <2>
 	"rolled_over": true, <3>
-	"simulated": false, <4>
+	"dry_run": false, <4>
 	"rollover_index_created": true, <5>
 	"conditions": { <6>
 	    "[max_age: 7d]": true,

From fb52c258fdfcb5f76e5fa856bf8509c55b43e036 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 9 Jun 2016 12:08:36 -0400
Subject: [PATCH 0751/1311] [test] Check if RefreshListeners was called
 immediately

Return a boolean from RefreshListeners, true if we called the listener
inline and false if we didn't, and check it in the test.
---
 .../elasticsearch/index/shard/RefreshListeners.java |  8 +++++---
 .../index/shard/RefreshListenersTests.java          | 13 ++++++++-----
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java b/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
index ab3e334714a..10a9e7b290c 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
@@ -68,15 +68,16 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener
      * @param location the location to listen for
      * @param listener for the refresh. Called with true if registering the listener ran it out of slots and forced a refresh. Called with
      *        false otherwise.
+     * @return did we call the listener (true) or register the listener to call later (false)?
      */
-    public void addOrNotify(Translog.Location location, Consumer<Boolean> listener) {
+    public boolean addOrNotify(Translog.Location location, Consumer<Boolean> listener) {
         requireNonNull(listener, "listener cannot be null");
         requireNonNull(location, "location cannot be null");
 
         if (lastRefreshedLocation != null && lastRefreshedLocation.compareTo(location) >= 0) {
             // Location already visible, just call the listener
             listener.accept(false);
-            return;
+            return true;
         }
         synchronized (this) {
             if (refreshListeners == null) {
@@ -85,12 +86,13 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener
             if (refreshListeners.size() < getMaxRefreshListeners.getAsInt()) {
                 // We have a free slot so register the listener
                 refreshListeners.add(new Tuple<>(location, listener));
-                return;
+                return false;
             }
         }
         // No free slot so force a refresh and call the listener in this thread
         forceRefresh.run();
         listener.accept(true);
+        return true;
     }
 
     /**
diff --git a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
index eb0fca7ce9e..4938f686f60 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
@@ -176,7 +176,7 @@ public class RefreshListenersTests extends ESTestCase {
         }
 
         DummyRefreshListener listener = new DummyRefreshListener();
-        listeners.addOrNotify(index.getTranslogLocation(), listener);
+        assertTrue(listeners.addOrNotify(index.getTranslogLocation(), listener));
         assertFalse(listener.forcedRefresh.get());
         listener.assertNoError();
     }
@@ -195,12 +195,15 @@ public class RefreshListenersTests extends ESTestCase {
         });
         refresher.start();
         try {
-            for (int i = 0; i < 100; i++) {
+            for (int i = 0; i < 1000; i++) {
                 Engine.Index index = index("1");
-
                 DummyRefreshListener listener = new DummyRefreshListener();
-                listeners.addOrNotify(index.getTranslogLocation(), listener);
-                assertBusy(() -> assertNotNull(listener.forcedRefresh.get()));
+                boolean immediate = listeners.addOrNotify(index.getTranslogLocation(), listener);
+                if (immediate) {
+                    assertNotNull(listener.forcedRefresh.get());
+                } else {
+                    assertBusy(() -> assertNotNull(listener.forcedRefresh.get()));
+                }
                 assertFalse(listener.forcedRefresh.get());
                 listener.assertNoError();
             }

From a9f24ea2dcc5ec15c8778e3df7fea172caa422f0 Mon Sep 17 00:00:00 2001
From: Areek Zillur <areek.zillur@elasticsearch.com>
Date: Thu, 9 Jun 2016 12:38:12 -0400
Subject: [PATCH 0752/1311] fail rollover request if rollover index already
 exists

---
 .../indices/rollover/RolloverResponse.java    | 15 +---
 .../rollover/TransportRolloverAction.java     | 70 ++++++++-----------
 .../admin/indices/rollover/RolloverIT.java    | 23 ++----
 .../reference/indices/rollover-index.asciidoc | 21 +++---
 .../test/indices.rollover/10_basic.yaml       |  3 +-
 5 files changed, 49 insertions(+), 83 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
index 4bfd7905129..d3a36f7d561 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
@@ -39,18 +39,16 @@ public final class RolloverResponse extends ActionResponse implements ToXContent
     private Set<Map.Entry<String, Boolean>> conditionStatus;
     private boolean dryRun;
     private boolean rolledOver;
-    private boolean rolloverIndexCreated;
 
     RolloverResponse() {
     }
 
     RolloverResponse(String oldIndex, String newIndex, Set<Condition.Result> conditionResults,
-                     boolean dryRun, boolean rolledOver, boolean rolloverIndexCreated) {
+                     boolean dryRun, boolean rolledOver) {
         this.oldIndex = oldIndex;
         this.newIndex = newIndex;
         this.dryRun = dryRun;
         this.rolledOver = rolledOver;
-        this.rolloverIndexCreated = rolloverIndexCreated;
         this.conditionStatus = conditionResults.stream()
             .map(result -> new AbstractMap.SimpleEntry<>(result.condition.toString(), result.matched))
             .collect(Collectors.toSet());
@@ -91,13 +89,6 @@ public final class RolloverResponse extends ActionResponse implements ToXContent
         return rolledOver;
     }
 
-    /**
-     * Returns if the rollover index had to be explicitly created
-     */
-    public boolean isRolloverIndexCreated() {
-        return rolloverIndexCreated;
-    }
-
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
@@ -113,7 +104,6 @@ public final class RolloverResponse extends ActionResponse implements ToXContent
         conditionStatus = conditions;
         dryRun = in.readBoolean();
         rolledOver = in.readBoolean();
-        rolloverIndexCreated = in.readBoolean();
     }
 
     @Override
@@ -128,7 +118,6 @@ public final class RolloverResponse extends ActionResponse implements ToXContent
         }
         out.writeBoolean(dryRun);
         out.writeBoolean(rolledOver);
-        out.writeBoolean(rolloverIndexCreated);
     }
 
     @Override
@@ -137,7 +126,6 @@ public final class RolloverResponse extends ActionResponse implements ToXContent
         builder.field(Fields.NEW_INDEX, newIndex);
         builder.field(Fields.ROLLED_OVER, rolledOver);
         builder.field(Fields.DRY_RUN, dryRun);
-        builder.field(Fields.ROLLOVER_INDEX_CREATED, rolloverIndexCreated);
         builder.startObject(Fields.CONDITIONS);
         for (Map.Entry<String, Boolean> entry : conditionStatus) {
             builder.field(entry.getKey(), entry.getValue());
@@ -151,7 +139,6 @@ public final class RolloverResponse extends ActionResponse implements ToXContent
         static final String OLD_INDEX = "old_index";
         static final String DRY_RUN = "dry_run";
         static final String ROLLED_OVER = "rolled_over";
-        static final String ROLLOVER_INDEX_CREATED = "rollover_index_created";
         static final String CONDITIONS = "conditions";
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
index baf743466ee..51623f11966 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -43,6 +43,7 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.shard.DocsStats;
+import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -108,36 +109,42 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
                     final String rolloverIndexName = generateRolloverIndexName(sourceIndexName);
                     if (rolloverRequest.isDryRun()) {
                         listener.onResponse(
-                            new RolloverResponse(sourceIndexName, rolloverIndexName, conditionResults, true, false,
-                                false));
+                            new RolloverResponse(sourceIndexName, rolloverIndexName, conditionResults, true, false));
                         return;
                     }
                     if (conditionResults.size() == 0 || conditionResults.stream().anyMatch(result -> result.matched)) {
-                        boolean createRolloverIndex = metaData.index(rolloverIndexName) == null;
-                        final RolloverResponse rolloverResponse =
-                            new RolloverResponse(sourceIndexName, rolloverIndexName, conditionResults, false, true,
-                                createRolloverIndex);
-                        if (createRolloverIndex) {
-                            createIndexService.createIndex(prepareCreateIndexRequest(rolloverIndexName, rolloverRequest),
-                                new ActionListener<ClusterStateUpdateResponse>() {
-                                    @Override
-                                    public void onResponse(ClusterStateUpdateResponse response) {
-                                        rollover(rolloverRequest, rolloverResponse, listener);
-                                    }
+                        createIndexService.createIndex(prepareCreateIndexRequest(rolloverIndexName, rolloverRequest),
+                            new ActionListener<ClusterStateUpdateResponse>() {
+                                @Override
+                                public void onResponse(ClusterStateUpdateResponse response) {
+                                    // switch the alias to point to the newly created index
+                                    indexAliasesService.indicesAliases(
+                                        prepareRolloverAliasesUpdateRequest(sourceIndexName, rolloverIndexName,
+                                            rolloverRequest),
+                                        new ActionListener<ClusterStateUpdateResponse>() {
+                                            @Override
+                                            public void onResponse(ClusterStateUpdateResponse clusterStateUpdateResponse) {
+                                                listener.onResponse(
+                                                    new RolloverResponse(sourceIndexName, rolloverIndexName,
+                                                        conditionResults, false, true));
+                                            }
 
-                                    @Override
-                                    public void onFailure(Throwable t) {
-                                        listener.onFailure(t);
-                                    }
-                                });
-                        } else {
-                            rollover(rolloverRequest, rolloverResponse, listener);
-                        }
+                                            @Override
+                                            public void onFailure(Throwable e) {
+                                                listener.onFailure(e);
+                                            }
+                                        });
+                                }
+
+                                @Override
+                                public void onFailure(Throwable t) {
+                                    listener.onFailure(t);
+                                }
+                            });
                     } else {
                         // conditions not met
                         listener.onResponse(
-                            new RolloverResponse(sourceIndexName, sourceIndexName, conditionResults, false, false,
-                                false)
+                            new RolloverResponse(sourceIndexName, sourceIndexName, conditionResults, false, false)
                         );
                     }
                 }
@@ -150,23 +157,6 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
         );
     }
 
-    private void rollover(final RolloverRequest request, final RolloverResponse response,
-                          ActionListener<RolloverResponse> listener) {
-        indexAliasesService.indicesAliases(
-            prepareRolloverAliasesUpdateRequest(response.getOldIndex(), response.getNewIndex(), request),
-            new ActionListener<ClusterStateUpdateResponse>() {
-                @Override
-                public void onResponse(ClusterStateUpdateResponse clusterStateUpdateResponse) {
-                    listener.onResponse(response);
-                }
-
-                @Override
-                public void onFailure(Throwable e) {
-                    listener.onFailure(e);
-                }
-            });
-    }
-
     static IndicesAliasesClusterStateUpdateRequest prepareRolloverAliasesUpdateRequest(String oldIndex, String newIndex,
                                                                                        RolloverRequest request) {
         final IndicesAliasesClusterStateUpdateRequest updateRequest = new IndicesAliasesClusterStateUpdateRequest()
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
index 112fe91eaaf..a47c91acc0f 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
@@ -24,6 +24,7 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.Map;
@@ -41,7 +42,6 @@ public class RolloverIT extends ESIntegTestCase {
         assertThat(response.getNewIndex(), equalTo("test_index-2"));
         assertThat(response.isDryRun(), equalTo(false));
         assertThat(response.isRolledOver(), equalTo(true));
-        assertThat(response.isRolloverIndexCreated(), equalTo(true));
         assertThat(response.getConditionStatus().size(), equalTo(0));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
         final IndexMetaData oldIndex = state.metaData().index("test_index-1");
@@ -59,7 +59,6 @@ public class RolloverIT extends ESIntegTestCase {
         assertThat(response.getNewIndex(), equalTo("test_index-3"));
         assertThat(response.isDryRun(), equalTo(false));
         assertThat(response.isRolledOver(), equalTo(true));
-        assertThat(response.isRolloverIndexCreated(), equalTo(true));
         assertThat(response.getConditionStatus().size(), equalTo(0));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
         final IndexMetaData oldIndex = state.metaData().index("test_index-2");
@@ -82,7 +81,6 @@ public class RolloverIT extends ESIntegTestCase {
         assertThat(response.getNewIndex(), equalTo("test_index-3"));
         assertThat(response.isDryRun(), equalTo(false));
         assertThat(response.isRolledOver(), equalTo(true));
-        assertThat(response.isRolloverIndexCreated(), equalTo(true));
         assertThat(response.getConditionStatus().size(), equalTo(0));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
         final IndexMetaData oldIndex = state.metaData().index("test_index-2");
@@ -103,7 +101,6 @@ public class RolloverIT extends ESIntegTestCase {
         assertThat(response.getNewIndex(), equalTo("test_index-2"));
         assertThat(response.isDryRun(), equalTo(true));
         assertThat(response.isRolledOver(), equalTo(false));
-        assertThat(response.isRolloverIndexCreated(), equalTo(false));
         assertThat(response.getConditionStatus().size(), equalTo(0));
         final ClusterState state = client().admin().cluster().prepareState().get().getState();
         final IndexMetaData oldIndex = state.metaData().index("test_index-1");
@@ -122,7 +119,6 @@ public class RolloverIT extends ESIntegTestCase {
         assertThat(response.getNewIndex(), equalTo("test_index-0"));
         assertThat(response.isDryRun(), equalTo(false));
         assertThat(response.isRolledOver(), equalTo(false));
-        assertThat(response.isRolloverIndexCreated(), equalTo(false));
         assertThat(response.getConditionStatus().size(), equalTo(1));
         final Map.Entry<String, Boolean> conditionEntry = response.getConditionStatus().iterator().next();
         assertThat(conditionEntry.getKey(), equalTo(new MaxAgeCondition(TimeValue.timeValueHours(4)).toString()));
@@ -140,16 +136,11 @@ public class RolloverIT extends ESIntegTestCase {
         assertAcked(prepareCreate("test_index-1").get());
         index("test_index-1", "type1", "1", "field", "value");
         flush("test_index-0", "test_index-1");
-        final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias").get();
-        assertThat(response.getOldIndex(), equalTo("test_index-0"));
-        assertThat(response.getNewIndex(), equalTo("test_index-1"));
-        assertThat(response.isDryRun(), equalTo(false));
-        assertThat(response.isRolledOver(), equalTo(true));
-        assertThat(response.isRolloverIndexCreated(), equalTo(false));
-        final ClusterState state = client().admin().cluster().prepareState().get().getState();
-        final IndexMetaData oldIndex = state.metaData().index("test_index-0");
-        assertFalse(oldIndex.getAliases().containsKey("test_alias"));
-        final IndexMetaData newIndex = state.metaData().index("test_index-1");
-        assertTrue(newIndex.getAliases().containsKey("test_alias"));
+        try {
+            client().admin().indices().prepareRolloverIndex("test_alias").get();
+            fail("expected failure due to existing rollover index");
+        } catch (IndexAlreadyExistsException e) {
+            assertThat(e.getIndex().getName(), equalTo("test_index-1"));
+        }
     }
 }
diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc
index f4c228b9870..9deb8cc3c7f 100644
--- a/docs/reference/indices/rollover-index.asciidoc
+++ b/docs/reference/indices/rollover-index.asciidoc
@@ -3,9 +3,10 @@
 
 The rollover index API allows to switch the index pointed to by an alias given some predicates.
 In order to rollover an index, the provided alias has to point to a single index. Upon satisfying
-any of the predicates, the alias is switched to point to a new index, creating the index if it
-does not exist. The rollover API requires the old concrete index name to have `{index_prefix}-{num}`
-format, as rollover index name is generated following `{index_prefix}-{num+1}` format.
+any of the predicates, the alias is switched to point to the rollover index, if the rollover index
+already exists, the rollover fails. The rollover API requires the old concrete index name to have
+`{index_prefix}-{num}` format, as rollover index name is generated following `{index_prefix}-{num+1}`
+format.
 
 This API is syntactic sugar for changing the index pointed to by an alias given some predicate.
 
@@ -35,11 +36,11 @@ $ curl -XPOST 'http://localhost:9200/index_alias/_rollover' -d '{
 <2> Sets a condition that the index has to have at least a 1000 documents
 
 The API call above switches the index pointed to by `index_alias` from `index-1` to `index-2`, if any
-of the conditions are met. If `index-2` does not exist, it is created (using matching <<indices-templates>>
-if available). The API call returns immediately if none of the conditions are met.
+of the conditions are met. `index-2` is created (using matching <<indices-templates>> if available).
+The API call returns immediately if none of the conditions are met.
 
-The `_rollover` API is similar to <<indices-create-index>> and accepts `settings`, `mappings` and `aliases`
-to override the index create request for a non-existent rolled over index.
+The `_rollover` API is similar to <<indices-create-index>> and accepts `settings`, `mappings` and
+`aliases` to override the index create request for the rollover index.
 
 [source,js]
 --------------------------------------------------
@@ -67,8 +68,7 @@ An example response for the index rollover API:
 	"new_index": "index-2", <2>
 	"rolled_over": true, <3>
 	"dry_run": false, <4>
-	"rollover_index_created": true, <5>
-	"conditions": { <6>
+	"conditions": { <5>
 	    "[max_age: 7d]": true,
 	    "[max_docs: 1000]": true
 	}
@@ -78,6 +78,5 @@ An example response for the index rollover API:
 <2> name of the index the alias currently points to
 <3> whether the alias switch was successful
 <4> whether the rollover was dry run
-<5> whether the rolled over index had to be explicitly created
-<6> status of the evaluated request conditions
+<5> status of the evaluated request conditions
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml
index 8252a7f0254..c60907fb416 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml
@@ -39,8 +39,7 @@
   - match: { old_index: logs-1 }
   - match: { new_index: logs-2 }
   - match: { rolled_over: true }
-  - match: { rollover_index_created: true }
-  - match: { simulated: false }
+  - match: { dry_run: false }
   - match: { conditions: { "[max_docs: 1]": true } }
 
   # ensure new index is created

From 6f222b5be19f557e010ef16f87368dc212c1261b Mon Sep 17 00:00:00 2001
From: gfyoung <gfyoung17@gmail.com>
Date: Thu, 9 Jun 2016 17:39:23 +0100
Subject: [PATCH 0753/1311] Support flags in pattern replace char filter

Works just like pattern analyzer's flags param.

Closes #18362.
---
 .../PatternReplaceCharFilterFactory.java       |  6 ++++--
 .../index/analysis/CharFilterTests.java        | 18 ++++++++++++++++++
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/analysis/PatternReplaceCharFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/PatternReplaceCharFilterFactory.java
index 949e1ef0fb4..db428db153d 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/PatternReplaceCharFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/PatternReplaceCharFilterFactory.java
@@ -20,6 +20,7 @@ package org.elasticsearch.index.analysis;
 
 import org.apache.lucene.analysis.pattern.PatternReplaceCharFilter;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
@@ -35,10 +36,11 @@ public class PatternReplaceCharFilterFactory extends AbstractCharFilterFactory {
     public PatternReplaceCharFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name);
 
-        if (!Strings.hasLength(settings.get("pattern"))) {
+        String sPattern = settings.get("pattern");
+        if (!Strings.hasLength(sPattern)) {
             throw new IllegalArgumentException("pattern is missing for [" + name + "] char filter of type 'pattern_replace'");
         }
-        pattern = Pattern.compile(settings.get("pattern"));
+        pattern = Regex.compile(sPattern, settings.get("flags"));
         replacement = settings.get("replacement", ""); // when not set or set to "", use "".
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java b/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java
index 3c791e72b5f..2b2c9288f17 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java
@@ -65,4 +65,22 @@ public class CharFilterTests extends ESTokenStreamTestCase {
         // Repeat one more time to make sure that char filter is reinitialized correctly
         assertTokenStreamContents(analyzer1.tokenStream("test", "<b>hello</b>!"), new String[]{"hello"});
     }
+
+    public void testPatternReplaceCharFilter() throws Exception {
+        Settings settings = Settings.builder()
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put("index.analysis.char_filter.my_mapping.type", "pattern_replace")
+            .put("index.analysis.char_filter.my_mapping.pattern", "ab*")
+            .put("index.analysis.char_filter.my_mapping.replacement", "oo")
+            .put("index.analysis.char_filter.my_mapping.flags", "CASE_INSENSITIVE")
+            .put("index.analysis.analyzer.custom_with_char_filter.tokenizer", "standard")
+            .putArray("index.analysis.analyzer.custom_with_char_filter.char_filter", "my_mapping")
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .build();
+        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
+        AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
+        NamedAnalyzer analyzer1 = analysisService.analyzer("custom_with_char_filter");
+
+        assertTokenStreamContents(analyzer1.tokenStream("test", "faBBbBB aBbbbBf"), new String[]{"foo", "oof"});
+    }
 }

From 09cc4c449aa16cbc0aa0cb14c3fac6c206f41382 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 9 Jun 2016 12:41:20 -0400
Subject: [PATCH 0754/1311] [docs] Pattern replace char filter now support
 flags

---
 .../analysis/charfilters/pattern-replace-charfilter.asciidoc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc b/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc
index 72adefa5aec..29194412a10 100644
--- a/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc
+++ b/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc
@@ -21,6 +21,11 @@ The `pattern_replace` character filter accepts the following parameters:
     `$1`..`$9` syntax, as explained
     http://docs.oracle.com/javase/8/docs/api/java/util/regex/Matcher.html#appendReplacement-java.lang.StringBuffer-java.lang.String-[here].
 
+`flags`::
+
+    Java regular expression http://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html#field.summary[flags].
+    lags should be pipe-separated, eg `"CASE_INSENSITIVE|COMMENTS"`.
+
 [float]
 === Example configuration
 

From 756af81b0019061bbf498459eed50b06a0c3f4cc Mon Sep 17 00:00:00 2001
From: Areek Zillur <areek.zillur@elasticsearch.com>
Date: Thu, 9 Jun 2016 12:53:24 -0400
Subject: [PATCH 0755/1311] change rollover rest-api placeholder name

---
 .../main/resources/rest-api-spec/api/indices.rollover.json  | 6 +++---
 .../rest-api-spec/test/indices.rollover/10_basic.yaml       | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
index ec66d48116a..7ba7d9cd29c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
@@ -3,10 +3,10 @@
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-rollover-index.html",
     "methods": ["POST"],
     "url": {
-      "path": "/{index}/_rollover",
-      "paths": ["/{index}/_rollover"],
+      "path": "/{alias}/_rollover",
+      "paths": ["/{alias}/_rollover"],
       "parts": {
-        "index": {
+        "alias": {
           "type" : "string",
           "required" : true,
           "description" : "The name of the alias to rollover"
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml
index c60907fb416..647907614fa 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml
@@ -31,7 +31,7 @@
   # perform alias rollover
   - do:
     indices.rollover:
-      index: "logs_search"
+      alias: "logs_search"
       body:
         conditions:
           max_docs: 1

From a0585269be577dbedf441b4ec3ef93e3c1c086e2 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 9 Jun 2016 13:08:53 -0400
Subject: [PATCH 0756/1311] [docs] s/lags/Flags/

Copy and paste lots an `F`.
---
 docs/reference/analysis/analyzers/pattern-analyzer.asciidoc     | 2 +-
 .../analysis/charfilters/pattern-replace-charfilter.asciidoc    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc b/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc
index 2d5741c2b9e..bd6000c3de7 100644
--- a/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/pattern-analyzer.asciidoc
@@ -148,7 +148,7 @@ The `pattern` analyzer accepts the following parameters:
 `flags`::
 
     Java regular expression http://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html#field.summary[flags].
-    lags should be pipe-separated, eg `"CASE_INSENSITIVE|COMMENTS"`.
+    Flags should be pipe-separated, eg `"CASE_INSENSITIVE|COMMENTS"`.
 
 `lowercase`::
 
diff --git a/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc b/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc
index 29194412a10..24cd203ae26 100644
--- a/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc
+++ b/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc
@@ -24,7 +24,7 @@ The `pattern_replace` character filter accepts the following parameters:
 `flags`::
 
     Java regular expression http://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html#field.summary[flags].
-    lags should be pipe-separated, eg `"CASE_INSENSITIVE|COMMENTS"`.
+    Flags should be pipe-separated, eg `"CASE_INSENSITIVE|COMMENTS"`.
 
 [float]
 === Example configuration

From 41d31541a645dd3052d36a1596867a3ed0ea946f Mon Sep 17 00:00:00 2001
From: Areek Zillur <areek.zillur@elasticsearch.com>
Date: Thu, 9 Jun 2016 13:43:19 -0400
Subject: [PATCH 0757/1311] Allow users to override the name for the rollover
 index

---
 .../indices/rollover/RolloverRequest.java      | 16 +++++++++++++++-
 .../rollover/RolloverRequestBuilder.java       |  9 +++++++--
 .../rollover/TransportRolloverAction.java      |  4 +++-
 .../metadata/MetaDataMappingService.java       |  2 +-
 .../admin/indices/RestRolloverIndexAction.java |  3 ++-
 .../admin/indices/rollover/RolloverIT.java     | 18 ++++++++++++++++++
 .../indices/rollover/RolloverRequestTests.java |  4 ++--
 .../rollover/TransportRolloverActionTests.java | 15 +++++++++------
 docs/reference/indices/rollover-index.asciidoc | 18 ++++++++++--------
 .../rest-api-spec/api/indices.rollover.json    |  9 +++++++--
 10 files changed, 74 insertions(+), 24 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
index 10d780a005b..9e98418f184 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
@@ -70,14 +70,16 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
     }
 
     private String alias;
+    private String newIndexName;
     private boolean dryRun;
     private Set<Condition> conditions = new HashSet<>(2);
     private CreateIndexRequest createIndexRequest = new CreateIndexRequest("_na_");
 
     RolloverRequest() {}
 
-    public RolloverRequest(String alias) {
+    public RolloverRequest(String alias, String newIndexName) {
         this.alias = alias;
+        this.newIndexName = newIndexName;
     }
 
     @Override
@@ -96,6 +98,7 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         alias = in.readString();
+        newIndexName = in.readOptionalString();
         dryRun = in.readBoolean();
         int size = in.readVInt();
         for (int i = 0; i < size; i++) {
@@ -109,6 +112,7 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeString(alias);
+        out.writeOptionalString(newIndexName);
         out.writeBoolean(dryRun);
         out.writeVInt(conditions.size());
         for (Condition condition : conditions) {
@@ -134,6 +138,12 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
         this.alias = alias;
     }
 
+    /**
+     * Sets the alias to rollover to another index
+     */
+    public void setNewIndexName(String newIndexName) {
+        this.newIndexName = newIndexName;
+    }
     /**
      * Sets if the rollover should not be executed when conditions are met
      */
@@ -175,6 +185,10 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
         return alias;
     }
 
+    String getNewIndexName() {
+        return newIndexName;
+    }
+
     CreateIndexRequest getCreateIndexRequest() {
         return createIndexRequest;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
index 8beebc2e516..e9b4351fc5d 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestBuilder.java
@@ -31,8 +31,13 @@ public class RolloverRequestBuilder extends MasterNodeOperationRequestBuilder<Ro
         super(client, action, new RolloverRequest());
     }
 
-    public RolloverRequestBuilder setAlias(String sourceAlias) {
-        this.request.setAlias(sourceAlias);
+    public RolloverRequestBuilder setAlias(String alias) {
+        this.request.setAlias(alias);
+        return this;
+    }
+
+    public RolloverRequestBuilder setNewIndexName(String newIndexName) {
+        this.request.setNewIndexName(newIndexName);
         return this;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
index 51623f11966..533e24a2ad5 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -106,7 +106,9 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
                 public void onResponse(IndicesStatsResponse statsResponse) {
                     final Set<Condition.Result> conditionResults = evaluateConditions(rolloverRequest.getConditions(),
                         statsResponse.getTotal().getDocs(), metaData.index(sourceIndexName));
-                    final String rolloverIndexName = generateRolloverIndexName(sourceIndexName);
+                    final String rolloverIndexName = (rolloverRequest.getNewIndexName() != null)
+                        ? rolloverRequest.getNewIndexName()
+                        : generateRolloverIndexName(sourceIndexName);
                     if (rolloverRequest.isDryRun()) {
                         listener.onResponse(
                             new RolloverResponse(sourceIndexName, rolloverIndexName, conditionResults, true, false));
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
index a41d02d28fa..a98f2148da8 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
@@ -299,7 +299,7 @@ public class MetaDataMappingService extends AbstractComponent {
             assert mappingType != null;
 
             if (!MapperService.DEFAULT_MAPPING.equals(mappingType) && mappingType.charAt(0) == '_') {
-                throw new InvalidTypeNameException("Document mapping type name can't start with '_'");
+                throw new InvalidTypeNameException("Document mapping type name can't start with '_', found: [" + mappingType + "]");
             }
             MetaData.Builder builder = MetaData.builder(metaData);
             for (Tuple<IndexService, IndexMetaData> toUpdate : updateList) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
index 635e22ce83c..7eecaaa738c 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
@@ -38,12 +38,13 @@ public class RestRolloverIndexAction extends BaseRestHandler {
     public RestRolloverIndexAction(Settings settings, RestController controller, Client client) {
         super(settings, client);
         controller.registerHandler(RestRequest.Method.POST, "/{index}/_rollover", this);
+        controller.registerHandler(RestRequest.Method.POST, "/{index}/_rollover/{new_index}", this);
     }
 
     @SuppressWarnings({"unchecked"})
     @Override
     public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
-        RolloverRequest rolloverIndexRequest = new RolloverRequest(request.param("index"));
+        RolloverRequest rolloverIndexRequest = new RolloverRequest(request.param("index"), request.param("new_index"));
         if (request.hasContent()) {
             rolloverIndexRequest.source(request.content());
         }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
index a47c91acc0f..b45383c47e4 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
@@ -130,6 +130,24 @@ public class RolloverIT extends ESIntegTestCase {
         assertNull(newIndex);
     }
 
+    public void testRolloverWithNewIndexName() throws Exception {
+        assertAcked(prepareCreate("test_index").addAlias(new Alias("test_alias")).get());
+        index("test_index", "type1", "1", "field", "value");
+        flush("test_index");
+        final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias")
+            .setNewIndexName("test_new_index").get();
+        assertThat(response.getOldIndex(), equalTo("test_index"));
+        assertThat(response.getNewIndex(), equalTo("test_new_index"));
+        assertThat(response.isDryRun(), equalTo(false));
+        assertThat(response.isRolledOver(), equalTo(true));
+        assertThat(response.getConditionStatus().size(), equalTo(0));
+        final ClusterState state = client().admin().cluster().prepareState().get().getState();
+        final IndexMetaData oldIndex = state.metaData().index("test_index");
+        assertFalse(oldIndex.getAliases().containsKey("test_alias"));
+        final IndexMetaData newIndex = state.metaData().index("test_new_index");
+        assertTrue(newIndex.getAliases().containsKey("test_alias"));
+    }
+
     public void testRolloverOnExistingIndex() throws Exception {
         assertAcked(prepareCreate("test_index-0").addAlias(new Alias("test_alias")).get());
         index("test_index-0", "type1", "1", "field", "value");
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java
index 754e4c419bd..7f4db3b8409 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequestTests.java
@@ -31,7 +31,7 @@ import static org.hamcrest.Matchers.equalTo;
 public class RolloverRequestTests extends ESTestCase {
 
     public void testConditionsParsing() throws Exception {
-        final RolloverRequest request = new RolloverRequest(randomAsciiOfLength(10));
+        final RolloverRequest request = new RolloverRequest(randomAsciiOfLength(10), randomAsciiOfLength(10));
         final XContentBuilder builder = XContentFactory.jsonBuilder()
             .startObject()
             .startObject("conditions")
@@ -56,7 +56,7 @@ public class RolloverRequestTests extends ESTestCase {
     }
 
     public void testParsingWithIndexSettings() throws Exception {
-        final RolloverRequest request = new RolloverRequest(randomAsciiOfLength(10));
+        final RolloverRequest request = new RolloverRequest(randomAsciiOfLength(10), randomAsciiOfLength(10));
         final XContentBuilder builder = XContentFactory.jsonBuilder()
             .startObject()
             .startObject("conditions")
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
index 28c0a6e3787..7d163630afb 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java
@@ -91,7 +91,7 @@ public class TransportRolloverActionTests extends ESTestCase {
         String sourceAlias = randomAsciiOfLength(10);
         String sourceIndex = randomAsciiOfLength(10);
         String targetIndex = randomAsciiOfLength(10);
-        final RolloverRequest rolloverRequest = new RolloverRequest(sourceAlias);
+        final RolloverRequest rolloverRequest = new RolloverRequest(sourceAlias, targetIndex);
         final IndicesAliasesClusterStateUpdateRequest updateRequest =
             TransportRolloverAction.prepareRolloverAliasesUpdateRequest(sourceIndex, targetIndex, rolloverRequest);
 
@@ -137,13 +137,16 @@ public class TransportRolloverActionTests extends ESTestCase {
             ).build();
 
         expectThrows(IllegalArgumentException.class, () ->
-            TransportRolloverAction.validate(metaData, new RolloverRequest(aliasWithMultipleIndices)));
+            TransportRolloverAction.validate(metaData, new RolloverRequest(aliasWithMultipleIndices,
+                randomAsciiOfLength(10))));
         expectThrows(IllegalArgumentException.class, () ->
-            TransportRolloverAction.validate(metaData, new RolloverRequest(randomFrom(index1, index2))));
+            TransportRolloverAction.validate(metaData, new RolloverRequest(randomFrom(index1, index2),
+                randomAsciiOfLength(10))));
         expectThrows(IllegalArgumentException.class, () ->
-            TransportRolloverAction.validate(metaData, new RolloverRequest(randomAsciiOfLength(5)))
+            TransportRolloverAction.validate(metaData, new RolloverRequest(randomAsciiOfLength(5),
+                randomAsciiOfLength(10)))
         );
-        TransportRolloverAction.validate(metaData, new RolloverRequest(alias));
+        TransportRolloverAction.validate(metaData, new RolloverRequest(alias, randomAsciiOfLength(10)));
     }
 
     public void testGenerateRolloverIndexName() throws Exception {
@@ -162,7 +165,7 @@ public class TransportRolloverActionTests extends ESTestCase {
     public void testCreateIndexRequest() throws Exception {
         String alias = randomAsciiOfLength(10);
         String rolloverIndex = randomAsciiOfLength(10);
-        final RolloverRequest rolloverRequest = new RolloverRequest(alias);
+        final RolloverRequest rolloverRequest = new RolloverRequest(alias, randomAsciiOfLength(10));
         final Settings settings = Settings.builder()
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
             .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc
index 9deb8cc3c7f..b5c845f7360 100644
--- a/docs/reference/indices/rollover-index.asciidoc
+++ b/docs/reference/indices/rollover-index.asciidoc
@@ -4,9 +4,7 @@
 The rollover index API allows to switch the index pointed to by an alias given some predicates.
 In order to rollover an index, the provided alias has to point to a single index. Upon satisfying
 any of the predicates, the alias is switched to point to the rollover index, if the rollover index
-already exists, the rollover fails. The rollover API requires the old concrete index name to have
-`{index_prefix}-{num}` format, as rollover index name is generated following `{index_prefix}-{num+1}`
-format.
+already exists, the rollover fails.
 
 This API is syntactic sugar for changing the index pointed to by an alias given some predicate.
 
@@ -14,18 +12,18 @@ The rollover API must be used against an alias that points to a single index:
 
 [source,js]
 --------------------------------------------------
-$ curl -XPUT 'http://localhost:9200/index-1/' -d '{
+$ curl -XPUT 'http://localhost:9200/index1' -d '{
     "aliases" : {
         "index_alias": {}
     }
 }'
 --------------------------------------------------
 
-To rollover `index_alias` to point to a new index:
+To rollover `index_alias` to point to a new index `index2`:
 
 [source,js]
 --------------------------------------------------
-$ curl -XPOST 'http://localhost:9200/index_alias/_rollover' -d '{
+$ curl -XPOST 'http://localhost:9200/index_alias/_rollover/index2' -d '{
     "conditions" : {
         "max_age": "7d", <1>
         "max_docs": 1000 <2>
@@ -35,10 +33,14 @@ $ curl -XPOST 'http://localhost:9200/index_alias/_rollover' -d '{
 <1> Sets a condition that the index has to be at least 7 days old
 <2> Sets a condition that the index has to have at least a 1000 documents
 
-The API call above switches the index pointed to by `index_alias` from `index-1` to `index-2`, if any
-of the conditions are met. `index-2` is created (using matching <<indices-templates>> if available).
+The API call above switches the index pointed to by `index_alias` from `index1` to `index2`, if any
+of the conditions are met. `index2` is created (using matching <<indices-templates>> if available).
 The API call returns immediately if none of the conditions are met.
 
+The rollover API can be used without specifying the name for the new index. In this case, the API requires
+the old concrete index name to have `{index_prefix}-{num}` format, as rollover index name is generated
+following `{index_prefix}-{num+1}` format.
+
 The `_rollover` API is similar to <<indices-create-index>> and accepts `settings`, `mappings` and
 `aliases` to override the index create request for the rollover index.
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
index 7ba7d9cd29c..d3660c4a679 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
@@ -3,13 +3,18 @@
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-rollover-index.html",
     "methods": ["POST"],
     "url": {
-      "path": "/{alias}/_rollover",
-      "paths": ["/{alias}/_rollover"],
+      "path": "/{alias}/_rollover}",
+      "paths": ["/{alias}/_rollover", "/{alias}/_rollover/{new_index}"],
       "parts": {
         "alias": {
           "type" : "string",
           "required" : true,
           "description" : "The name of the alias to rollover"
+        },
+        "new_index": {
+          "type" : "string",
+          "required" : false,
+          "description" : "The name of the rollover index"
         }
       },
       "params": {

From e02d9f09455f5a7f12f9966d1e2a3032e5cf907e Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 9 Jun 2016 13:03:08 -0400
Subject: [PATCH 0758/1311] Squash a race condition in RefreshListeners

It presented as listeners never being called if you refresh at the same
time as the listener is added. It was caught rarely by
testConcurrentRefresh. mostly this is removing code and adding a comment:

```
Note that it is not safe for us to abort early if we haven't advanced the
position here because we set and read lastRefreshedLocation outside of a
synchronized block. We do that so that waiting for a refresh that has
already passed is just a volatile read but the cost is that any check
whether or not we've advanced the position will introduce a race between
adding the listener and the position check. We could work around this by
moving this assignment into the synchronized block below and double
checking lastRefreshedLocation in addOrNotify's synchronized block but
that doesn't seem worth it given that we already skip this process early
if there aren't any listeners to iterate.
```
---
 .../elasticsearch/index/shard/RefreshListeners.java  | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java b/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
index 10a9e7b290c..b594b31abb8 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
@@ -137,14 +137,14 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener
              */
             return;
         }
-        // First check if we've actually moved forward. If not then just bail immediately.
-        assert lastRefreshedLocation == null || currentRefreshLocation.compareTo(lastRefreshedLocation) >= 0;
-        if (lastRefreshedLocation != null && currentRefreshLocation.compareTo(lastRefreshedLocation) == 0) {
-            return;
-        }
         /*
          * Set the lastRefreshedLocation so listeners that come in for locations before that will just execute inline without messing
-         * around with refreshListeners or synchronizing at all.
+         * around with refreshListeners or synchronizing at all. Note that it is not safe for us to abort early if we haven't advanced the
+         * position here because we set and read lastRefreshedLocation outside of a synchronized block. We do that so that waiting for a
+         * refresh that has already passed is just a volatile read but the cost is that any check whether or not we've advanced the
+         * position will introduce a race between adding the listener and the position check. We could work around this by moving this
+         * assignment into the synchronized block below and double checking lastRefreshedLocation in addOrNotify's synchronized block but
+         * that doesn't seem worth it given that we already skip this process early if there aren't any listeners to iterate.
          */
         lastRefreshedLocation = currentRefreshLocation;
         /*

From f38ce72004b9041b9f3b54af5b011b43b48275e3 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Thu, 9 Jun 2016 23:44:34 +0200
Subject: [PATCH 0759/1311] make forbiddenApisTest work for client and
 client-sniffer

---
 client-sniffer/build.gradle                   | 10 +---
 .../client/sniff/HostsSnifferTests.java       | 45 +++++++++-----
 client/build.gradle                           |  5 --
 .../client/RestClientIntegTests.java          | 58 +++++++++++--------
 .../client/SuppressForbidden.java             | 34 -----------
 5 files changed, 66 insertions(+), 86 deletions(-)
 delete mode 100644 client/src/test/java/org/elasticsearch/client/SuppressForbidden.java

diff --git a/client-sniffer/build.gradle b/client-sniffer/build.gradle
index 7c6527abb4c..3f1b8c37765 100644
--- a/client-sniffer/build.gradle
+++ b/client-sniffer/build.gradle
@@ -51,14 +51,10 @@ forbiddenApisMain {
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
 }
 
-//TODO would be nice to just exclude the classes where we use com.sun.net.httpserver.* classes
-//excludes don't seem to work though and we don't want to have our own @SuppressForbidden
-forbiddenApisTest.enabled=false
-
-//forbiddenApisTest {
+forbiddenApisTest {
   //client does not depend on core, so only jdk signatures should be checked
-  //signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
-//}
+  signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
+}
 
 //JarHell is part of es core, which we don't want to pull in
 jarHell.enabled=false
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
index d714e89eeb3..4ff11a24ea5 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
@@ -31,6 +31,7 @@ import org.apache.http.Consts;
 import org.apache.http.HttpHost;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.SuppressForbidden;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
@@ -54,6 +55,7 @@ import java.util.Set;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
 
+@SuppressForbidden(reason = "uses sun HttpServer")
 public class HostsSnifferTests extends LuceneTestCase {
 
     private int sniffRequestTimeout;
@@ -109,26 +111,37 @@ public class HostsSnifferTests extends LuceneTestCase {
         }
     }
 
-    private static HttpServer createHttpServer(final SniffResponse sniffResponse, final int sniffTimeout) throws IOException {
+    private static HttpServer createHttpServer(final SniffResponse sniffResponse, final int sniffTimeoutMillis) throws IOException {
         HttpServer httpServer = HttpServer.create(new InetSocketAddress(0), 0);
-        httpServer.createContext("/_nodes/http", new HttpHandler() {
-            @Override
-            public void handle(HttpExchange httpExchange) throws IOException {
-                if (httpExchange.getRequestMethod().equals(HttpGet.METHOD_NAME)) {
-                    if (httpExchange.getRequestURI().getRawQuery().equals("timeout=" + sniffTimeout + "ms")) {
-                        String nodesInfoBody = sniffResponse.nodesInfoBody;
-                        httpExchange.sendResponseHeaders(sniffResponse.nodesInfoResponseCode, nodesInfoBody.length());
-                        try (OutputStream out = httpExchange.getResponseBody()) {
-                            out.write(nodesInfoBody.getBytes(Consts.UTF_8));
-                            return;
-                        }
+        httpServer.createContext("/_nodes/http", new ResponseHandler(sniffTimeoutMillis, sniffResponse));
+        return httpServer;
+    }
+
+    @SuppressForbidden(reason = "uses sun HttpServer")
+    private static class ResponseHandler implements HttpHandler {
+        private final int sniffTimeoutMillis;
+        private final SniffResponse sniffResponse;
+
+        ResponseHandler(int sniffTimeoutMillis, SniffResponse sniffResponse) {
+            this.sniffTimeoutMillis = sniffTimeoutMillis;
+            this.sniffResponse = sniffResponse;
+        }
+
+        @Override
+        public void handle(HttpExchange httpExchange) throws IOException {
+            if (httpExchange.getRequestMethod().equals(HttpGet.METHOD_NAME)) {
+                if (httpExchange.getRequestURI().getRawQuery().equals("timeout=" + sniffTimeoutMillis + "ms")) {
+                    String nodesInfoBody = sniffResponse.nodesInfoBody;
+                    httpExchange.sendResponseHeaders(sniffResponse.nodesInfoResponseCode, nodesInfoBody.length());
+                    try (OutputStream out = httpExchange.getResponseBody()) {
+                        out.write(nodesInfoBody.getBytes(Consts.UTF_8));
+                        return;
                     }
                 }
-                httpExchange.sendResponseHeaders(404, 0);
-                httpExchange.close();
             }
-        });
-        return httpServer;
+            httpExchange.sendResponseHeaders(404, 0);
+            httpExchange.close();
+        }
     }
 
     private static SniffResponse buildSniffResponse(HostsSniffer.Scheme scheme) throws IOException {
diff --git a/client/build.gradle b/client/build.gradle
index c947f94ff26..6cdc0d818ac 100644
--- a/client/build.gradle
+++ b/client/build.gradle
@@ -49,14 +49,9 @@ forbiddenApisMain {
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
 }
 
-//TODO would be nice to just exclude the classes where we use com.sun.net.httpserver.* classes
-//excludes don't seem to work though and we don't want to have our own @SuppressForbidden
-forbiddenApisTest.enabled=false
-
 forbiddenApisTest {
   //client does not depend on core, so only jdk signatures should be checked
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
-  suppressAnnotations = ['**.SuppressForbidden']
 }
 
 //JarHell is part of es core, which we don't want to pull in
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java b/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
index 1505ccd55d2..36a7e45b728 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
@@ -32,6 +32,7 @@ import org.apache.http.entity.StringEntity;
 import org.apache.http.message.BasicHeader;
 import org.apache.http.util.EntityUtils;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.SuppressForbidden;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
@@ -83,32 +84,41 @@ public class RestClientIntegTests extends LuceneTestCase {
     }
 
     private static void createStatusCodeContext(HttpServer httpServer, final int statusCode) {
-        httpServer.createContext("/" + statusCode, new HttpHandler() {
-            @Override
-            public void handle(HttpExchange httpExchange) throws IOException {
-                StringBuilder body = new StringBuilder();
-                try (InputStreamReader reader = new InputStreamReader(httpExchange.getRequestBody(), Consts.UTF_8)) {
-                    char[] buffer = new char[256];
-                    int read;
-                    while ((read = reader.read(buffer)) != -1) {
-                        body.append(buffer, 0, read);
-                    }
+        httpServer.createContext("/" + statusCode, new ResponseHandler(statusCode));
+    }
+
+    @SuppressForbidden(reason = "uses sun HttpServer")
+    private static class ResponseHandler implements HttpHandler {
+        private final int statusCode;
+
+        ResponseHandler(int statusCode) {
+            this.statusCode = statusCode;
+        }
+
+        @Override
+        public void handle(HttpExchange httpExchange) throws IOException {
+            StringBuilder body = new StringBuilder();
+            try (InputStreamReader reader = new InputStreamReader(httpExchange.getRequestBody(), Consts.UTF_8)) {
+                char[] buffer = new char[256];
+                int read;
+                while ((read = reader.read(buffer)) != -1) {
+                    body.append(buffer, 0, read);
                 }
-                Headers requestHeaders = httpExchange.getRequestHeaders();
-                Headers responseHeaders = httpExchange.getResponseHeaders();
-                for (Map.Entry<String, List<String>> header : requestHeaders.entrySet()) {
-                    responseHeaders.put(header.getKey(), header.getValue());
-                }
-                httpExchange.getRequestBody().close();
-                httpExchange.sendResponseHeaders(statusCode, body.length() == 0 ? -1 : body.length());
-                if (body.length() > 0) {
-                    try (OutputStream out = httpExchange.getResponseBody()) {
-                        out.write(body.toString().getBytes(Consts.UTF_8));
-                    }
-                }
-                httpExchange.close();
             }
-        });
+            Headers requestHeaders = httpExchange.getRequestHeaders();
+            Headers responseHeaders = httpExchange.getResponseHeaders();
+            for (Map.Entry<String, List<String>> header : requestHeaders.entrySet()) {
+                responseHeaders.put(header.getKey(), header.getValue());
+            }
+            httpExchange.getRequestBody().close();
+            httpExchange.sendResponseHeaders(statusCode, body.length() == 0 ? -1 : body.length());
+            if (body.length() > 0) {
+                try (OutputStream out = httpExchange.getResponseBody()) {
+                    out.write(body.toString().getBytes(Consts.UTF_8));
+                }
+            }
+            httpExchange.close();
+        }
     }
 
     @AfterClass
diff --git a/client/src/test/java/org/elasticsearch/client/SuppressForbidden.java b/client/src/test/java/org/elasticsearch/client/SuppressForbidden.java
deleted file mode 100644
index 309cc666bc4..00000000000
--- a/client/src/test/java/org/elasticsearch/client/SuppressForbidden.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.client;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-/**
- * Annotation to suppress forbidden-apis errors inside a whole class, a method, or a field.
- */
-@Retention(RetentionPolicy.CLASS)
-@Target({ ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE })
-public @interface SuppressForbidden {
-    String reason();
-}
\ No newline at end of file

From 2856ab07dee401e79719f0985fd0a1dfc6d32fb0 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Thu, 9 Jun 2016 23:48:49 +0200
Subject: [PATCH 0760/1311] Build: enable dependency licenses check for
 client-sniffer, exclude org.elasticsearch jars from checks

---
 client-sniffer/build.gradle | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/client-sniffer/build.gradle b/client-sniffer/build.gradle
index 3f1b8c37765..0677007a635 100644
--- a/client-sniffer/build.gradle
+++ b/client-sniffer/build.gradle
@@ -61,8 +61,11 @@ jarHell.enabled=false
 //NamingConventionCheck is part of test-framework, which we don't want to pull in as it depends on es core
 namingConventions.enabled=false
 
-//TODO re-enable once our own jars are properly filtered out
-dependencyLicenses.enabled=false
+dependencyLicenses {
+  dependencies = project.configurations.runtime.fileCollection {
+    it.group.startsWith('org.elasticsearch') == false
+  }
+}
 
 thirdPartyAudit.excludes = [
   //commons-logging optional dependencies

From 7229c9128987bc5d4dfd767780cf3a2282305241 Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@elastic.co>
Date: Fri, 10 Jun 2016 07:46:04 +0200
Subject: [PATCH 0761/1311] Remove trace logging from
 NettyHttpRequestSizeLimitIT

With this commit we revert back to normal behavior as the
underlying issue has been fixed with #18627.
---
 .../elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java    | 1 -
 1 file changed, 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java
index ba6b4438aaa..e1135f807cf 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java
@@ -54,7 +54,6 @@ public class NettyHttpRequestSizeLimitIT extends ESIntegTestCase {
             .build();
     }
 
-    @TestLogging("_root:DEBUG,org.elasticsearch.common.breaker:TRACE,org.elasticsearch.test:TRACE,org.elasticsearch.transport:TRACE")
     public void testLimitsInFlightRequests() throws Exception {
         ensureGreen();
 

From 439b2a96e5c922b687ecbfb1f5cdb388ea7c94d5 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Wed, 8 Jun 2016 15:28:06 +0200
Subject: [PATCH 0762/1311] Add an index setting to limit the maximum number of
 slices allowed in a scroll request (default to 1024).

---
 .../common/settings/IndexScopedSettings.java  |  1 +
 .../elasticsearch/index/IndexSettings.java    | 24 ++++++
 .../elasticsearch/search/SearchService.java   |  4 +-
 .../search/internal/DefaultSearchContext.java | 23 ++++--
 .../search/simple/SimpleSearchIT.java         |  2 +-
 .../search/slice/SearchSliceIT.java           |  5 +-
 docs/reference/search/request/scroll.asciidoc |  5 +-
 .../rest-api-spec/test/scroll/12_slices.yaml  | 75 +++++++++++++++++++
 8 files changed, 126 insertions(+), 13 deletions(-)
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yaml

diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
index b9d0c6b4c70..fb60453d467 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
@@ -116,6 +116,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         IndexSettings.ALLOW_UNMAPPED,
         IndexSettings.INDEX_CHECK_ON_STARTUP,
         IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD,
+        IndexSettings.MAX_SLICES_PER_SCROLL,
         ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING,
         IndexSettings.INDEX_GC_DELETES_SETTING,
         IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING,
diff --git a/core/src/main/java/org/elasticsearch/index/IndexSettings.java b/core/src/main/java/org/elasticsearch/index/IndexSettings.java
index 592c1ff1125..6bd6df414af 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexSettings.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexSettings.java
@@ -121,6 +121,12 @@ public final class IndexSettings {
     public static final Setting<Integer> MAX_REFRESH_LISTENERS_PER_SHARD = Setting.intSetting("index.max_refresh_listeners", 1000, 0,
             Property.Dynamic, Property.IndexScope);
 
+    /**
+     * The maximum number of slices allowed in a scroll request
+     */
+    public static final Setting<Integer> MAX_SLICES_PER_SCROLL = Setting.intSetting("index.max_slices_per_scroll",
+        1024, 1, Property.Dynamic, Property.IndexScope);
+
     private final Index index;
     private final Version version;
     private final ESLogger logger;
@@ -154,6 +160,11 @@ public final class IndexSettings {
      * The maximum number of refresh listeners allows on this shard.
      */
     private volatile int maxRefreshListeners;
+    /**
+     * The maximum number of slices allowed in a scroll request.
+     */
+    private volatile int maxSlicesPerScroll;
+
 
     /**
      * Returns the default search field for this index.
@@ -239,6 +250,7 @@ public final class IndexSettings {
         maxRescoreWindow = scopedSettings.get(MAX_RESCORE_WINDOW_SETTING);
         TTLPurgeDisabled = scopedSettings.get(INDEX_TTL_DISABLE_PURGE_SETTING);
         maxRefreshListeners = scopedSettings.get(MAX_REFRESH_LISTENERS_PER_SHARD);
+        maxSlicesPerScroll = scopedSettings.get(MAX_SLICES_PER_SCROLL);
         this.mergePolicyConfig = new MergePolicyConfig(logger, this);
         assert indexNameMatcher.test(indexMetaData.getIndex().getName());
 
@@ -262,6 +274,7 @@ public final class IndexSettings {
         scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING, this::setTranslogFlushThresholdSize);
         scopedSettings.addSettingsUpdateConsumer(INDEX_REFRESH_INTERVAL_SETTING, this::setRefreshInterval);
         scopedSettings.addSettingsUpdateConsumer(MAX_REFRESH_LISTENERS_PER_SHARD, this::setMaxRefreshListeners);
+        scopedSettings.addSettingsUpdateConsumer(MAX_SLICES_PER_SCROLL, this::setMaxSlicesPerScroll);
     }
 
     private void setTranslogFlushThresholdSize(ByteSizeValue byteSizeValue) {
@@ -521,5 +534,16 @@ public final class IndexSettings {
         this.maxRefreshListeners = maxRefreshListeners;
     }
 
+    /**
+     * The maximum number of slices allowed in a scroll request.
+     */
+    public int getMaxSlicesPerScroll() {
+        return maxSlicesPerScroll;
+    }
+
+    private void setMaxSlicesPerScroll(int value) {
+        this.maxSlicesPerScroll = value;
+    }
+
     IndexScopedSettings getScopedSettings() { return scopedSettings;}
 }
diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java
index 57841466a62..b7a6d714e63 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchService.java
@@ -826,9 +826,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
             if (context.scrollContext() == null) {
                 throw new SearchContextException(context, "`slice` cannot be used outside of a scroll context");
             }
-            context.sliceFilter(source.slice().toFilter(queryShardContext,
-                context.shardTarget().getShardId().getId(),
-                queryShardContext.getIndexSettings().getNumberOfShards()));
+            context.sliceBuilder(source.slice());
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
index 30e994b7656..06df04db8a0 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
@@ -68,6 +68,7 @@ import org.elasticsearch.search.profile.Profilers;
 import org.elasticsearch.search.query.QueryPhaseExecutionException;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.rescore.RescoreSearchContext;
+import org.elasticsearch.search.slice.SliceBuilder;
 import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.suggest.SuggestionSearchContext;
 
@@ -116,7 +117,7 @@ public class DefaultSearchContext extends SearchContext {
     private boolean trackScores = false; // when sorting, track scores as well...
     private FieldDoc searchAfter;
     // filter for sliced scroll
-    private Query sliceFilter;
+    private SliceBuilder sliceBuilder;
 
     /**
      * The original query as sent by the user without the types and aliases
@@ -212,13 +213,23 @@ public class DefaultSearchContext extends SearchContext {
                 if (rescoreContext.window() > maxWindow) {
                     throw new QueryPhaseExecutionException(this, "Rescore window [" + rescoreContext.window() + "] is too large. It must "
                             + "be less than [" + maxWindow + "]. This prevents allocating massive heaps for storing the results to be "
-                            + "rescored. This limit can be set by chaining the [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey()
+                            + "rescored. This limit can be set by changing the [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey()
                             + "] index level setting.");
 
                 }
             }
         }
 
+        if (sliceBuilder != null) {
+            int sliceLimit = indexService.getIndexSettings().getMaxSlicesPerScroll();
+            int numSlices = sliceBuilder.getMax();
+            if (numSlices > sliceLimit) {
+                throw new QueryPhaseExecutionException(this, "The number of slices [" + numSlices + "] is too large. It must "
+                    + "be less than [" + sliceLimit + "]. This limit can be set by changing the [" +
+                    IndexSettings.MAX_SLICES_PER_SCROLL.getKey() + "] index level setting.");
+            }
+        }
+
         // initialize the filtering alias based on the provided filters
         aliasFilter = indexService.aliasFilter(queryShardContext, request.filteringAliases());
 
@@ -257,9 +268,11 @@ public class DefaultSearchContext extends SearchContext {
     @Nullable
     public Query searchFilter(String[] types) {
         Query typesFilter = createSearchFilter(types, aliasFilter, mapperService().hasNested());
-        if (sliceFilter == null) {
+        if (sliceBuilder == null) {
             return typesFilter;
         }
+         Query sliceFilter = sliceBuilder.toFilter(queryShardContext, shardTarget().getShardId().getId(),
+                queryShardContext.getIndexSettings().getNumberOfShards());
         if (typesFilter == null) {
             return sliceFilter;
         }
@@ -562,8 +575,8 @@ public class DefaultSearchContext extends SearchContext {
         return searchAfter;
     }
 
-    public SearchContext sliceFilter(Query filter) {
-        this.sliceFilter = filter;
+    public SearchContext sliceBuilder(SliceBuilder sliceBuilder) {
+        this.sliceBuilder = sliceBuilder;
         return this;
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java
index 9751df26329..e94fe21a170 100644
--- a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java
@@ -443,6 +443,6 @@ public class SimpleSearchIT extends ESIntegTestCase {
         assertThat(e.toString(), containsString("Rescore window [" + windowSize + "] is too large. It must "
                 + "be less than [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.get(Settings.EMPTY)));
         assertThat(e.toString(), containsString(
-                "This limit can be set by chaining the [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey() + "] index level setting."));
+                "This limit can be set by changing the [" + IndexSettings.MAX_RESCORE_WINDOW_SETTING.getKey() + "] index level setting."));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/slice/SearchSliceIT.java b/core/src/test/java/org/elasticsearch/search/slice/SearchSliceIT.java
index ad93d14f21f..4a27dcb5964 100644
--- a/core/src/test/java/org/elasticsearch/search/slice/SearchSliceIT.java
+++ b/core/src/test/java/org/elasticsearch/search/slice/SearchSliceIT.java
@@ -35,7 +35,6 @@ import org.elasticsearch.test.ESIntegTestCase;
 import java.io.IOException;
 import java.util.List;
 import java.util.ArrayList;
-import java.util.Set;
 import java.util.HashSet;
 import java.util.concurrent.ExecutionException;
 
@@ -43,7 +42,6 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.startsWith;
 
 public class SearchSliceIT extends ESIntegTestCase {
@@ -71,7 +69,8 @@ public class SearchSliceIT extends ESIntegTestCase {
             .endObject().string();
         int numberOfShards = randomIntBetween(1, 7);
         assertAcked(client().admin().indices().prepareCreate("test")
-            .setSettings("number_of_shards", numberOfShards)
+            .setSettings("number_of_shards", numberOfShards,
+                         "index.max_slices_per_scroll", 10000)
             .addMapping("type", mapping));
         ensureGreen();
 
diff --git a/docs/reference/search/request/scroll.asciidoc b/docs/reference/search/request/scroll.asciidoc
index 9f9558aa979..a082bf3ba4c 100644
--- a/docs/reference/search/request/scroll.asciidoc
+++ b/docs/reference/search/request/scroll.asciidoc
@@ -263,4 +263,7 @@ curl -XGET 'localhost:9200/twitter/tweet/_search?scroll=1m' -d '
 '
 --------------------------------------------------
 
-For append only time-based indices, the `timestamp` field can be used safely.
\ No newline at end of file
+For append only time-based indices, the `timestamp` field can be used safely.
+
+NOTE: By default the maximum number of slices allowed per scroll is limited to 1024.
+You can update the `index.max_slices_per_scroll` index setting to bypass this limit.
\ No newline at end of file
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yaml
new file mode 100644
index 00000000000..5443059135a
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/12_slices.yaml
@@ -0,0 +1,75 @@
+---
+"Sliced scroll":
+  - do:
+      indices.create:
+        index: test_sliced_scroll
+
+  - do:
+      index:
+        index:  test_sliced_scroll
+        type:   test
+        id:     42
+        body:   { foo: 1 }
+
+  - do:
+      indices.refresh: {}
+
+  - do:
+      search:
+        index: test_sliced_scroll
+        size: 1
+        scroll: 1m
+        sort: foo
+        body:
+          slice: {
+            id: 0,
+            max: 3
+          }
+          query:
+            match_all: {}
+
+  - set: {_scroll_id: scroll_id}
+
+  - do:
+        clear_scroll:
+          scroll_id: $scroll_id
+
+  - do:
+        catch:  /query_phase_execution_exception.*The number of slices.*index.max_slices_per_scroll/
+        search:
+          index: test_sliced_scroll
+          size: 1
+          scroll: 1m
+          body:
+            slice: {
+              id: 0,
+              max: 1025
+            }
+            query:
+              match_all: {}
+
+  - do:
+        indices.put_settings:
+          index: test_sliced_scroll
+          body:
+            index.max_slices_per_scroll: 1025
+
+  - do:
+        search:
+          index: test_sliced_scroll
+          size: 1
+          scroll: 1m
+          body:
+            slice: {
+              id: 0,
+              max: 1025
+            }
+            query:
+              match_all: {}
+
+  - set: {_scroll_id: scroll_id}
+
+  - do:
+        clear_scroll:
+          scroll_id: $scroll_id
+

From 0af9d2c7677eebbbe9d23ca75a65aa585759dbc8 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 10 Jun 2016 09:51:47 +0200
Subject: [PATCH 0763/1311] Build: add animalsniffer to detect usage of java8
 apis in client and client-sniffer

---
 buildSrc/build.gradle       | 1 +
 client-sniffer/build.gradle | 8 +++++++-
 client/build.gradle         | 8 +++++++-
 3 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
index 623fdab3e3e..ce29769715c 100644
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -87,6 +87,7 @@ dependencies {
   compile 'de.thetaphi:forbiddenapis:2.1'
   compile 'com.bmuschko:gradle-nexus-plugin:2.3.1'
   compile 'org.apache.rat:apache-rat:0.11'
+  compile 'ru.vyarus:gradle-animalsniffer-plugin:1.0.1'
 }
 
 
diff --git a/client-sniffer/build.gradle b/client-sniffer/build.gradle
index 0677007a635..c772cfd084b 100644
--- a/client-sniffer/build.gradle
+++ b/client-sniffer/build.gradle
@@ -21,6 +21,7 @@ import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.gradle.api.JavaVersion
 
 apply plugin: 'elasticsearch.build'
+apply plugin: 'ru.vyarus.animalsniffer'
 
 targetCompatibility = JavaVersion.VERSION_1_7
 sourceCompatibility = JavaVersion.VERSION_1_7
@@ -40,9 +41,9 @@ dependencies {
   testCompile "org.apache.lucene:lucene-core:${versions.lucene}"
   testCompile "org.apache.lucene:lucene-codecs:${versions.lucene}"
   testCompile "org.elasticsearch:securemock:${versions.securemock}"
+  signature "org.codehaus.mojo.signature:java17:1.0@signature"
 }
 
-//TODO compiling from 1.8 with target 1.7 and source 1.7 is best effort, not enough to ensure we are java 7 compatible
 compileJava.options.compilerArgs << '-target' << '1.7' << '-source' << '1.7' << '-Xlint:all,-path,-serial,-options'
 compileTestJava.options.compilerArgs << '-target' << '1.7' << '-source' << '1.7'
 
@@ -56,6 +57,11 @@ forbiddenApisTest {
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
 }
 
+animalsnifferTest {
+    //animalsniffer doesn't like sun classes usages (e.g. com.sun.net.httpserver.HttpServer)
+    exclude('org/elasticsearch/client/sniff/HostsSnifferTests.class')
+}
+
 //JarHell is part of es core, which we don't want to pull in
 jarHell.enabled=false
 //NamingConventionCheck is part of test-framework, which we don't want to pull in as it depends on es core
diff --git a/client/build.gradle b/client/build.gradle
index 6cdc0d818ac..609e10e4abd 100644
--- a/client/build.gradle
+++ b/client/build.gradle
@@ -21,6 +21,7 @@ import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.gradle.api.JavaVersion
 
 apply plugin: 'elasticsearch.build'
+apply plugin: 'ru.vyarus.animalsniffer'
 
 targetCompatibility = JavaVersion.VERSION_1_7
 sourceCompatibility = JavaVersion.VERSION_1_7
@@ -38,9 +39,9 @@ dependencies {
   testCompile "org.apache.lucene:lucene-core:${versions.lucene}"
   testCompile "org.apache.lucene:lucene-codecs:${versions.lucene}"
   testCompile "org.elasticsearch:securemock:${versions.securemock}"
+  signature "org.codehaus.mojo.signature:java17:1.0@signature"
 }
 
-//TODO compiling from 1.8 with target 1.7 and source 1.7 is best effort, not enough to ensure we are java 7 compatible
 compileJava.options.compilerArgs << '-target' << '1.7' << '-source' << '1.7' << '-Xlint:all,-path,-serial,-options'
 compileTestJava.options.compilerArgs << '-target' << '1.7' << '-source' << '1.7'
 
@@ -54,6 +55,11 @@ forbiddenApisTest {
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
 }
 
+animalsnifferTest {
+    //animalsniffer doesn't like sun classes usages (e.g. com.sun.net.httpserver.HttpServer)
+    exclude('org/elasticsearch/client/RestClientIntegTests.class')
+}
+
 //JarHell is part of es core, which we don't want to pull in
 jarHell.enabled=false
 //NamingConventionCheck is part of test-framework, which we don't want to pull in as it depends on es core

From 1d761775107b3b4c73d2b968c352b805225284eb Mon Sep 17 00:00:00 2001
From: Colin Goodheart-Smithe <colings86@users.noreply.github.com>
Date: Tue, 17 May 2016 14:16:20 +0100
Subject: [PATCH 0764/1311] Adds aggregation profiling (not including reduce
 phase)

Add Aggregation profiling initially only be for the shard phases (i.e. the reduce phase will not be profiled in this change)

This change refactors the query profiling class to extract abstract classes where it is useful for other profiler types to share code.
---
 .../action/search/SearchResponse.java         |   3 +-
 .../aggregations/AggregatorFactories.java     |  16 +-
 .../aggregations/AggregatorFactory.java       | 243 ++++++-----
 .../cardinality/CardinalityAggregator.java    |   1 -
 .../controller/SearchPhaseController.java     |   5 +-
 .../search/internal/ContextIndexSearcher.java |   2 +-
 .../internal/InternalSearchResponse.java      |   5 +-
 .../profile/AbstractInternalProfileTree.java  | 209 +++++++++
 .../search/profile/AbstractProfiler.java      |  54 +++
 .../search/profile/ProfileShardResult.java    |  75 +---
 .../search/profile/Profilers.java             |  10 +-
 .../profile/SearchProfileShardResults.java    |  72 ++--
 .../AggregationProfileBreakdown.java          |  30 ++
 .../AggregationProfileShardResult.java        |  79 ++++
 .../aggregation/AggregationProfiler.java      |  57 +++
 .../aggregation/AggregationTimingType.java    |  34 ++
 .../InternalAggregationProfileTree.java       |  46 ++
 .../aggregation/ProfilingAggregator.java      | 103 +++++
 .../ProfilingLeafBucketCollector.java         |  43 ++
 .../query/InternalQueryProfileTree.java       | 186 +-------
 .../query/QueryProfileShardResult.java        | 104 +++++
 .../search/profile/query/QueryProfiler.java   |  41 +-
 .../search/query/QueryPhase.java              |   8 +-
 .../search/query/QuerySearchResult.java       |  18 +-
 .../aggregation/AggregationProfilerIT.java    | 403 ++++++++++++++++++
 .../search/profile/query/QueryProfilerIT.java |  45 +-
 ...fileTests.java => QueryProfilerTests.java} |  10 +-
 27 files changed, 1430 insertions(+), 472 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/search/profile/AbstractInternalProfileTree.java
 create mode 100644 core/src/main/java/org/elasticsearch/search/profile/AbstractProfiler.java
 create mode 100644 core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileBreakdown.java
 create mode 100644 core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResult.java
 create mode 100644 core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfiler.java
 create mode 100644 core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationTimingType.java
 create mode 100644 core/src/main/java/org/elasticsearch/search/profile/aggregation/InternalAggregationProfileTree.java
 create mode 100644 core/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingAggregator.java
 create mode 100644 core/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingLeafBucketCollector.java
 create mode 100644 core/src/main/java/org/elasticsearch/search/profile/query/QueryProfileShardResult.java
 create mode 100644 core/src/test/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java
 rename core/src/test/java/org/elasticsearch/search/profile/query/{ProfileTests.java => QueryProfilerTests.java} (96%)

diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java
index b76e640fa95..8e144537c84 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java
@@ -36,7 +36,6 @@ import org.elasticsearch.search.profile.ProfileShardResult;
 import org.elasticsearch.search.suggest.Suggest;
 
 import java.io.IOException;
-import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.action.search.ShardSearchFailure.readShardSearchFailure;
@@ -169,7 +168,7 @@ public class SearchResponse extends ActionResponse implements StatusToXContent {
      *
      * @return The profile results or an empty map
      */
-    public @Nullable Map<String, List<ProfileShardResult>> getProfileResults() {
+    public @Nullable Map<String, ProfileShardResult> getProfileResults() {
         return internalResponse.profile();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
index 38fb0e6a431..2d8ea318fd6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
@@ -27,6 +27,8 @@ import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.AggregationPath;
 import org.elasticsearch.search.aggregations.support.AggregationPath.PathElement;
+import org.elasticsearch.search.profile.Profilers;
+import org.elasticsearch.search.profile.aggregation.ProfilingAggregator;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -81,7 +83,12 @@ public class AggregatorFactories {
             // propagate the fact that only bucket 0 will be collected with single-bucket
             // aggs
             final boolean collectsFromSingleBucket = false;
-            aggregators[i] = factories[i].create(parent, collectsFromSingleBucket);
+            Aggregator factory = factories[i].create(parent, collectsFromSingleBucket);
+            Profilers profilers = factory.context().searchContext().getProfilers();
+            if (profilers != null) {
+                factory = new ProfilingAggregator(factory, profilers.getAggregationProfiler());
+            }
+            aggregators[i] = factory;
         }
         return aggregators;
     }
@@ -92,7 +99,12 @@ public class AggregatorFactories {
         for (int i = 0; i < factories.length; i++) {
             // top-level aggs only get called with bucket 0
             final boolean collectsFromSingleBucket = true;
-            aggregators[i] = factories[i].create(null, collectsFromSingleBucket);
+            Aggregator factory = factories[i].create(null, collectsFromSingleBucket);
+            Profilers profilers = factory.context().searchContext().getProfilers();
+            if (profilers != null) {
+                factory = new ProfilingAggregator(factory, profilers.getAggregationProfiler());
+            }
+            aggregators[i] = factory;
         }
         return aggregators;
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java
index 3fced89a014..854838b7441 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactory.java
@@ -28,13 +28,139 @@ import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.internal.SearchContext.Lifetime;
-
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
 public abstract class AggregatorFactory<AF extends AggregatorFactory<AF>> {
 
+    public static final class MultiBucketAggregatorWrapper extends Aggregator {
+        private final BigArrays bigArrays;
+        private final Aggregator parent;
+        private final AggregatorFactory<?> factory;
+        private final Aggregator first;
+        ObjectArray<Aggregator> aggregators;
+        ObjectArray<LeafBucketCollector> collectors;
+
+        MultiBucketAggregatorWrapper(BigArrays bigArrays, AggregationContext context, Aggregator parent, AggregatorFactory<?> factory,
+                Aggregator first) {
+            this.bigArrays = bigArrays;
+            this.parent = parent;
+            this.factory = factory;
+            this.first = first;
+            context.searchContext().addReleasable(this, Lifetime.PHASE);
+            aggregators = bigArrays.newObjectArray(1);
+            aggregators.set(0, first);
+            collectors = bigArrays.newObjectArray(1);
+        }
+
+        public Class<?> getWrappedClass() {
+            return first.getClass();
+        }
+
+        @Override
+        public String name() {
+            return first.name();
+        }
+
+        @Override
+        public AggregationContext context() {
+            return first.context();
+        }
+
+        @Override
+        public Aggregator parent() {
+            return first.parent();
+        }
+
+        @Override
+        public boolean needsScores() {
+            return first.needsScores();
+        }
+
+        @Override
+        public Aggregator subAggregator(String name) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void preCollection() throws IOException {
+            for (long i = 0; i < aggregators.size(); ++i) {
+                final Aggregator aggregator = aggregators.get(i);
+                if (aggregator != null) {
+                    aggregator.preCollection();
+                }
+            }
+        }
+
+        @Override
+        public void postCollection() throws IOException {
+            for (long i = 0; i < aggregators.size(); ++i) {
+                final Aggregator aggregator = aggregators.get(i);
+                if (aggregator != null) {
+                    aggregator.postCollection();
+                }
+            }
+        }
+
+        @Override
+        public LeafBucketCollector getLeafCollector(final LeafReaderContext ctx) {
+            for (long i = 0; i < collectors.size(); ++i) {
+                collectors.set(i, null);
+            }
+            return new LeafBucketCollector() {
+                Scorer scorer;
+
+                @Override
+                public void setScorer(Scorer scorer) throws IOException {
+                    this.scorer = scorer;
+                }
+
+                @Override
+                public void collect(int doc, long bucket) throws IOException {
+                    collectors = bigArrays.grow(collectors, bucket + 1);
+
+                    LeafBucketCollector collector = collectors.get(bucket);
+                    if (collector == null) {
+                        aggregators = bigArrays.grow(aggregators, bucket + 1);
+                        Aggregator aggregator = aggregators.get(bucket);
+                        if (aggregator == null) {
+                            aggregator = factory.create(parent, true);
+                            aggregator.preCollection();
+                            aggregators.set(bucket, aggregator);
+                        }
+                        collector = aggregator.getLeafCollector(ctx);
+                        collector.setScorer(scorer);
+                        collectors.set(bucket, collector);
+                    }
+                    collector.collect(doc, 0);
+                }
+
+            };
+        }
+
+        @Override
+        public InternalAggregation buildAggregation(long bucket) throws IOException {
+            if (bucket < aggregators.size()) {
+                Aggregator aggregator = aggregators.get(bucket);
+                if (aggregator != null) {
+                    return aggregator.buildAggregation(0);
+                }
+            }
+            return buildEmptyAggregation();
+        }
+
+        @Override
+        public InternalAggregation buildEmptyAggregation() {
+            return first.buildEmptyAggregation();
+        }
+
+        @Override
+        public void close() {
+            Releasables.close(aggregators, collectors);
+        }
+    }
+
     protected final String name;
     protected final Type type;
     protected final AggregatorFactory<?> parent;
@@ -112,120 +238,7 @@ public abstract class AggregatorFactory<AF extends AggregatorFactory<AF>> {
             final Aggregator parent) throws IOException {
         final Aggregator first = factory.create(parent, true);
         final BigArrays bigArrays = context.bigArrays();
-        return new Aggregator() {
-
-            ObjectArray<Aggregator> aggregators;
-            ObjectArray<LeafBucketCollector> collectors;
-
-            {
-                context.searchContext().addReleasable(this, Lifetime.PHASE);
-                aggregators = bigArrays.newObjectArray(1);
-                aggregators.set(0, first);
-                collectors = bigArrays.newObjectArray(1);
-            }
-
-            @Override
-            public String name() {
-                return first.name();
-            }
-
-            @Override
-            public AggregationContext context() {
-                return first.context();
-            }
-
-            @Override
-            public Aggregator parent() {
-                return first.parent();
-            }
-
-            @Override
-            public boolean needsScores() {
-                return first.needsScores();
-            }
-
-            @Override
-            public Aggregator subAggregator(String name) {
-                throw new UnsupportedOperationException();
-            }
-
-            @Override
-            public void preCollection() throws IOException {
-                for (long i = 0; i < aggregators.size(); ++i) {
-                    final Aggregator aggregator = aggregators.get(i);
-                    if (aggregator != null) {
-                        aggregator.preCollection();
-                    }
-                }
-            }
-
-            @Override
-            public void postCollection() throws IOException {
-                for (long i = 0; i < aggregators.size(); ++i) {
-                    final Aggregator aggregator = aggregators.get(i);
-                    if (aggregator != null) {
-                        aggregator.postCollection();
-                    }
-                }
-            }
-
-            @Override
-            public LeafBucketCollector getLeafCollector(final LeafReaderContext ctx) {
-                for (long i = 0; i < collectors.size(); ++i) {
-                    collectors.set(i, null);
-                }
-                return new LeafBucketCollector() {
-                    Scorer scorer;
-
-                    @Override
-                    public void setScorer(Scorer scorer) throws IOException {
-                        this.scorer = scorer;
-                    }
-
-                    @Override
-                    public void collect(int doc, long bucket) throws IOException {
-                        aggregators = bigArrays.grow(aggregators, bucket + 1);
-                        collectors = bigArrays.grow(collectors, bucket + 1);
-
-                        LeafBucketCollector collector = collectors.get(bucket);
-                        if (collector == null) {
-                            Aggregator aggregator = aggregators.get(bucket);
-                            if (aggregator == null) {
-                                aggregator = factory.create(parent, true);
-                                aggregator.preCollection();
-                                aggregators.set(bucket, aggregator);
-                            }
-                            collector = aggregator.getLeafCollector(ctx);
-                            collector.setScorer(scorer);
-                            collectors.set(bucket, collector);
-                        }
-                        collector.collect(doc, 0);
-                    }
-
-                };
-            }
-
-            @Override
-            public InternalAggregation buildAggregation(long bucket) throws IOException {
-                if (bucket < aggregators.size()) {
-                    Aggregator aggregator = aggregators.get(bucket);
-                    if (aggregator != null) {
-                        return aggregator.buildAggregation(0);
-                    }
-                }
-                return buildEmptyAggregation();
-            }
-
-            @Override
-            public InternalAggregation buildEmptyAggregation() {
-                return first.buildEmptyAggregation();
-            }
-
-            @Override
-            public void close() {
-                Releasables.close(aggregators, collectors);
-            }
-        };
+        return new MultiBucketAggregatorWrapper(bigArrays, context, parent, factory, first);
     }
 
 }
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java
index b5ed80a0022..0793bacf722 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java
@@ -35,7 +35,6 @@ import org.elasticsearch.common.util.LongArray;
 import org.elasticsearch.common.util.ObjectArray;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
-import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
diff --git a/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java b/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java
index 36fe562a568..b2ce044e4fc 100644
--- a/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java
+++ b/core/src/main/java/org/elasticsearch/search/controller/SearchPhaseController.java
@@ -51,8 +51,9 @@ import org.elasticsearch.search.fetch.FetchSearchResultProvider;
 import org.elasticsearch.search.internal.InternalSearchHit;
 import org.elasticsearch.search.internal.InternalSearchHits;
 import org.elasticsearch.search.internal.InternalSearchResponse;
-import org.elasticsearch.search.profile.SearchProfileShardResults;
 import org.elasticsearch.search.profile.ProfileShardResult;
+import org.elasticsearch.search.profile.SearchProfileShardResults;
+import org.elasticsearch.search.profile.query.QueryProfileShardResult;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.search.query.QuerySearchResultProvider;
 import org.elasticsearch.search.suggest.Suggest;
@@ -407,7 +408,7 @@ public class SearchPhaseController extends AbstractComponent {
         //Collect profile results
         SearchProfileShardResults shardResults = null;
         if (!queryResults.isEmpty() && firstResult.profileResults() != null) {
-            Map<String, List<ProfileShardResult>> profileResults = new HashMap<>(queryResults.size());
+            Map<String, ProfileShardResult> profileResults = new HashMap<>(queryResults.size());
             for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : queryResults) {
                 String key = entry.value.queryResult().shardTarget().toString();
                 profileResults.put(key, entry.value.queryResult().profileResults());
diff --git a/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
index df1007ebc71..50e91e082cd 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
@@ -122,7 +122,7 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable {
                 weight = super.createWeight(query, needsScores);
             } finally {
                 profile.stopAndRecordTime();
-                profiler.pollLastQuery();
+                profiler.pollLastElement();
             }
             return new ProfileWeight(query, weight, profile);
         } else {
diff --git a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java
index 9c33889dc9c..26410cc9680 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java
@@ -28,13 +28,12 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.profile.SearchProfileShardResults;
 import org.elasticsearch.search.profile.ProfileShardResult;
+import org.elasticsearch.search.profile.SearchProfileShardResults;
 import org.elasticsearch.search.suggest.Suggest;
 
 import java.io.IOException;
 import java.util.Collections;
-import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.search.internal.InternalSearchHits.readSearchHits;
@@ -99,7 +98,7 @@ public class InternalSearchResponse implements Streamable, ToXContent {
      *
      * @return Profile results
      */
-    public Map<String, List<ProfileShardResult>> profile() {
+    public Map<String, ProfileShardResult> profile() {
         if (profileResults == null) {
             return Collections.emptyMap();
         }
diff --git a/core/src/main/java/org/elasticsearch/search/profile/AbstractInternalProfileTree.java b/core/src/main/java/org/elasticsearch/search/profile/AbstractInternalProfileTree.java
new file mode 100644
index 00000000000..31cb3c21237
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/profile/AbstractInternalProfileTree.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile;
+
+import org.elasticsearch.search.profile.query.QueryProfileBreakdown;
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.List;
+import java.util.Map;
+
+public abstract class AbstractInternalProfileTree<PB extends AbstractProfileBreakdown<?>, E> {
+
+    protected ArrayList<PB> timings;
+    /** Maps the Query to it's list of children.  This is basically the dependency tree */
+    protected ArrayList<ArrayList<Integer>> tree;
+    /** A list of the original queries, keyed by index position */
+    protected ArrayList<E> elements;
+    /** A list of top-level "roots".  Each root can have its own tree of profiles */
+    protected ArrayList<Integer> roots;
+    /** A temporary stack used to record where we are in the dependency tree. */
+    protected Deque<Integer> stack;
+    private int currentToken = 0;
+
+    public AbstractInternalProfileTree() {
+        timings = new ArrayList<>(10);
+        stack = new ArrayDeque<>(10);
+        tree = new ArrayList<>(10);
+        elements = new ArrayList<>(10);
+        roots = new ArrayList<>(10);
+    }
+
+    /**
+     * Returns a {@link QueryProfileBreakdown} for a scoring query.  Scoring queries (e.g. those
+     * that are past the rewrite phase and are now being wrapped by createWeight() ) follow
+     * a recursive progression.  We can track the dependency tree by a simple stack
+     *
+     * The only hiccup is that the first scoring query will be identical to the last rewritten
+     * query, so we need to take special care to fix that
+     *
+     * @param query The scoring query we wish to profile
+     * @return      A ProfileBreakdown for this query
+     */
+    public PB getProfileBreakdown(E query) {
+        int token = currentToken;
+
+        boolean stackEmpty = stack.isEmpty();
+
+        // If the stack is empty, we are a new root query
+        if (stackEmpty) {
+
+            // We couldn't find a rewritten query to attach to, so just add it as a
+            // top-level root. This is just a precaution: it really shouldn't happen.
+            // We would only get here if a top-level query that never rewrites for some reason.
+            roots.add(token);
+
+            // Increment the token since we are adding a new node, but notably, do not
+            // updateParent() because this was added as a root
+            currentToken += 1;
+            stack.add(token);
+
+            return addDependencyNode(query, token);
+        }
+
+        updateParent(token);
+
+        // Increment the token since we are adding a new node
+        currentToken += 1;
+        stack.add(token);
+
+        return addDependencyNode(query, token);
+    }
+
+    /**
+     * Helper method to add a new node to the dependency tree.
+     *
+     * Initializes a new list in the dependency tree, saves the query and
+     * generates a new {@link QueryProfileBreakdown} to track the timings of
+     * this query
+     *
+     * @param element
+     *            The element to profile
+     * @param token
+     *            The assigned token for this element
+     * @return A ProfileBreakdown to profile this element
+     */
+    private PB addDependencyNode(E element, int token) {
+
+        // Add a new slot in the dependency tree
+        tree.add(new ArrayList<>(5));
+
+        // Save our query for lookup later
+        elements.add(element);
+
+        PB queryTimings = createProfileBreakdown();
+        timings.add(token, queryTimings);
+        return queryTimings;
+    }
+
+    protected abstract PB createProfileBreakdown();
+
+    /**
+     * Removes the last (e.g. most recent) value on the stack
+     */
+    public void pollLast() {
+        stack.pollLast();
+    }
+
+    /**
+     * After the query has been run and profiled, we need to merge the flat timing map
+     * with the dependency graph to build a data structure that mirrors the original
+     * query tree
+     *
+     * @return a hierarchical representation of the profiled query tree
+     */
+    public List<ProfileResult> getTree() {
+        ArrayList<ProfileResult> results = new ArrayList<>(5);
+        for (Integer root : roots) {
+            results.add(doGetTree(root));
+        }
+        return results;
+    }
+
+    /**
+     * Recursive helper to finalize a node in the dependency tree
+     * @param token  The node we are currently finalizing
+     * @return       A hierarchical representation of the tree inclusive of children at this level
+     */
+    private ProfileResult doGetTree(int token) {
+        E element = elements.get(token);
+        PB breakdown = timings.get(token);
+        Map<String, Long> timings = breakdown.toTimingMap();
+        List<Integer> children = tree.get(token);
+        List<ProfileResult> childrenProfileResults = Collections.emptyList();
+
+        if (children != null) {
+            childrenProfileResults = new ArrayList<>(children.size());
+            for (Integer child : children) {
+                ProfileResult childNode = doGetTree(child);
+                childrenProfileResults.add(childNode);
+            }
+        }
+
+        // TODO this would be better done bottom-up instead of top-down to avoid
+        // calculating the same times over and over...but worth the effort?
+        long nodeTime = getNodeTime(timings, childrenProfileResults);
+        String type = getTypeFromElement(element);
+        String description = getDescriptionFromElement(element);
+        return new ProfileResult(type, description, timings, childrenProfileResults, nodeTime);
+    }
+
+    protected abstract String getTypeFromElement(E element);
+
+    protected abstract String getDescriptionFromElement(E element);
+
+    /**
+     * Internal helper to add a child to the current parent node
+     *
+     * @param childToken The child to add to the current parent
+     */
+    private void updateParent(int childToken) {
+        Integer parent = stack.peekLast();
+        ArrayList<Integer> parentNode = tree.get(parent);
+        parentNode.add(childToken);
+        tree.set(parent, parentNode);
+    }
+
+    /**
+     * Internal helper to calculate the time of a node, inclusive of children
+     *
+     * @param timings
+     *            A map of breakdown timing for the node
+     * @param children
+     *            All children profile results at this node
+     * @return The total time at this node, inclusive of children
+     */
+    private static long getNodeTime(Map<String, Long> timings, List<ProfileResult> children) {
+        long nodeTime = 0;
+        for (long time : timings.values()) {
+            nodeTime += time;
+        }
+
+        // Then add up our children
+        for (ProfileResult child : children) {
+            nodeTime += getNodeTime(child.getTimeBreakdown(), child.getProfiledChildren());
+        }
+        return nodeTime;
+    }
+
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/profile/AbstractProfiler.java b/core/src/main/java/org/elasticsearch/search/profile/AbstractProfiler.java
new file mode 100644
index 00000000000..a7ccb72785e
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/profile/AbstractProfiler.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile;
+
+import java.util.List;
+
+public class AbstractProfiler<PB extends AbstractProfileBreakdown<?>, E> {
+
+    protected final AbstractInternalProfileTree<PB, E> profileTree;
+
+    public AbstractProfiler(AbstractInternalProfileTree<PB, E> profileTree) {
+        this.profileTree = profileTree;
+    }
+
+    /**
+     * Get the {@link AbstractProfileBreakdown} for the given element in the
+     * tree, potentially creating it if it did not exist.
+     */
+    public PB getQueryBreakdown(E query) {
+        return profileTree.getProfileBreakdown(query);
+    }
+
+    /**
+     * Removes the last (e.g. most recent) element on the stack.
+     */
+    public void pollLastElement() {
+        profileTree.pollLast();
+    }
+
+    /**
+     * @return a hierarchical representation of the profiled tree
+     */
+    public List<ProfileResult> getTree() {
+        return profileTree.getTree();
+    }
+
+}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/search/profile/ProfileShardResult.java b/core/src/main/java/org/elasticsearch/search/profile/ProfileShardResult.java
index 9def3db7582..2a1fb0ba9b1 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/ProfileShardResult.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/ProfileShardResult.java
@@ -22,83 +22,50 @@ package org.elasticsearch.search.profile;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.search.profile.query.CollectorResult;
+import org.elasticsearch.search.profile.aggregation.AggregationProfileShardResult;
+import org.elasticsearch.search.profile.query.QueryProfileShardResult;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
-/**
- * A container class to hold the profile results for a single shard in the request.
- * Contains a list of query profiles, a collector tree and a total rewrite tree.
- */
-public final class ProfileShardResult implements Writeable, ToXContent {
+public class ProfileShardResult implements Writeable {
 
-    private final List<ProfileResult> queryProfileResults;
+    private final List<QueryProfileShardResult> queryProfileResults;
 
-    private final CollectorResult profileCollector;
+    private final AggregationProfileShardResult aggProfileShardResult;
 
-    private final long rewriteTime;
-
-    public ProfileShardResult(List<ProfileResult> queryProfileResults, long rewriteTime,
-                              CollectorResult profileCollector) {
-        assert(profileCollector != null);
-        this.queryProfileResults = queryProfileResults;
-        this.profileCollector = profileCollector;
-        this.rewriteTime = rewriteTime;
+    public ProfileShardResult(List<QueryProfileShardResult> queryProfileResults, AggregationProfileShardResult aggProfileShardResult) {
+        this.aggProfileShardResult = aggProfileShardResult;
+        this.queryProfileResults = Collections.unmodifiableList(queryProfileResults);
     }
 
-    /**
-     * Read from a stream.
-     */
     public ProfileShardResult(StreamInput in) throws IOException {
         int profileSize = in.readVInt();
-        queryProfileResults = new ArrayList<>(profileSize);
-        for (int j = 0; j < profileSize; j++) {
-            queryProfileResults.add(new ProfileResult(in));
+        List<QueryProfileShardResult> queryProfileResults = new ArrayList<>(profileSize);
+        for (int i = 0; i < profileSize; i++) {
+            QueryProfileShardResult result = new QueryProfileShardResult(in);
+            queryProfileResults.add(result);
         }
-
-        profileCollector = new CollectorResult(in);
-        rewriteTime = in.readLong();
+        this.queryProfileResults = Collections.unmodifiableList(queryProfileResults);
+        this.aggProfileShardResult = new AggregationProfileShardResult(in);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeVInt(queryProfileResults.size());
-        for (ProfileResult p : queryProfileResults) {
-            p.writeTo(out);
+        for (QueryProfileShardResult queryShardResult : queryProfileResults) {
+            queryShardResult.writeTo(out);
         }
-        profileCollector.writeTo(out);
-        out.writeLong(rewriteTime);
+        aggProfileShardResult.writeTo(out);
     }
 
-
-    public List<ProfileResult> getQueryResults() {
-        return Collections.unmodifiableList(queryProfileResults);
+    public List<QueryProfileShardResult> getQueryProfileResults() {
+        return queryProfileResults;
     }
 
-    public long getRewriteTime() {
-        return rewriteTime;
-    }
-
-    public CollectorResult getCollectorResult() {
-        return profileCollector;
-    }
-
-    @Override
-    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startArray("query");
-        for (ProfileResult p : queryProfileResults) {
-            p.toXContent(builder, params);
-        }
-        builder.endArray();
-        builder.field("rewrite_time", rewriteTime);
-        builder.startArray("collector");
-        profileCollector.toXContent(builder, params);
-        builder.endArray();
-        return builder;
+    public AggregationProfileShardResult getAggregationProfileResults() {
+        return aggProfileShardResult;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/profile/Profilers.java b/core/src/main/java/org/elasticsearch/search/profile/Profilers.java
index e9e6d88db18..d754be41f6d 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/Profilers.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/Profilers.java
@@ -20,22 +20,25 @@
 package org.elasticsearch.search.profile;
 
 import org.elasticsearch.search.internal.ContextIndexSearcher;
+import org.elasticsearch.search.profile.aggregation.AggregationProfiler;
 import org.elasticsearch.search.profile.query.QueryProfiler;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
-/** Wrapper around several {@link QueryProfiler}s that makes management easier. */
+/** Wrapper around all the profilers that makes management easier. */
 public final class Profilers {
 
     private final ContextIndexSearcher searcher;
     private final List<QueryProfiler> queryProfilers;
+    private final AggregationProfiler aggProfiler;
 
     /** Sole constructor. This {@link Profilers} instance will initially wrap one {@link QueryProfiler}. */
     public Profilers(ContextIndexSearcher searcher) {
         this.searcher = searcher;
         this.queryProfilers = new ArrayList<>();
+        this.aggProfiler = new AggregationProfiler();
         addQueryProfiler();
     }
 
@@ -57,4 +60,9 @@ public final class Profilers {
         return Collections.unmodifiableList(queryProfilers);
     }
 
+    /** Return the {@link AggregationProfiler}. */
+    public AggregationProfiler getAggregationProfiler() {
+        return aggProfiler;
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResults.java b/core/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResults.java
index bf265dd9a7e..6794aa49399 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResults.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/SearchProfileShardResults.java
@@ -24,6 +24,9 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.profile.aggregation.AggregationProfileShardResult;
+import org.elasticsearch.search.profile.aggregation.AggregationProfiler;
+import org.elasticsearch.search.profile.query.QueryProfileShardResult;
 import org.elasticsearch.search.profile.query.QueryProfiler;
 
 import java.io.IOException;
@@ -32,7 +35,6 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 
 /**
  * A container class to hold all the profile results across all shards.  Internally
@@ -40,17 +42,10 @@ import java.util.stream.Collectors;
  */
 public final class SearchProfileShardResults implements Writeable, ToXContent{
 
-    private Map<String, List<ProfileShardResult>> shardResults;
+    private Map<String, ProfileShardResult> shardResults;
 
-    public SearchProfileShardResults(Map<String, List<ProfileShardResult>> shardResults) {
-        Map<String, List<ProfileShardResult>> transformed =
-                shardResults.entrySet()
-                        .stream()
-                        .collect(Collectors.toMap(
-                                        Map.Entry::getKey,
-                                        e -> Collections.unmodifiableList(e.getValue()))
-                        );
-        this.shardResults =  Collections.unmodifiableMap(transformed);
+    public SearchProfileShardResults(Map<String, ProfileShardResult> shardResults) {
+        this.shardResults =  Collections.unmodifiableMap(shardResults);
     }
 
     public SearchProfileShardResults(StreamInput in) throws IOException {
@@ -59,33 +54,22 @@ public final class SearchProfileShardResults implements Writeable, ToXContent{
 
         for (int i = 0; i < size; i++) {
             String key = in.readString();
-            int shardResultsSize = in.readInt();
-
-            List<ProfileShardResult> shardResult = new ArrayList<>(shardResultsSize);
-
-            for (int j = 0; j < shardResultsSize; j++) {
-                ProfileShardResult result = new ProfileShardResult(in);
-                shardResult.add(result);
-            }
-            shardResults.put(key, Collections.unmodifiableList(shardResult));
+            ProfileShardResult shardResult = new ProfileShardResult(in);
+            shardResults.put(key, shardResult);
         }
         shardResults = Collections.unmodifiableMap(shardResults);
     }
 
-    public Map<String, List<ProfileShardResult>> getShardResults() {
+    public Map<String, ProfileShardResult> getShardResults() {
         return this.shardResults;
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeInt(shardResults.size());
-        for (Map.Entry<String, List<ProfileShardResult>> entry : shardResults.entrySet()) {
+        for (Map.Entry<String, ProfileShardResult> entry : shardResults.entrySet()) {
             out.writeString(entry.getKey());
-            out.writeInt(entry.getValue().size());
-
-            for (ProfileShardResult result : entry.getValue()) {
-                result.writeTo(out);
-            }
+            entry.getValue().writeTo(out);
         }
     }
 
@@ -93,14 +77,18 @@ public final class SearchProfileShardResults implements Writeable, ToXContent{
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject("profile").startArray("shards");
 
-        for (Map.Entry<String, List<ProfileShardResult>> entry : shardResults.entrySet()) {
-            builder.startObject().field("id",entry.getKey()).startArray("searches");
-            for (ProfileShardResult result : entry.getValue()) {
+        for (Map.Entry<String, ProfileShardResult> entry : shardResults.entrySet()) {
+            builder.startObject();
+            builder.field("id", entry.getKey());
+            builder.startArray("searches");
+            for (QueryProfileShardResult result : entry.getValue().getQueryProfileResults()) {
                 builder.startObject();
                 result.toXContent(builder, params);
                 builder.endObject();
             }
-            builder.endArray().endObject();
+            builder.endArray();
+            entry.getValue().getAggregationProfileResults().toXContent(builder, params);
+            builder.endObject();
         }
 
         builder.endArray().endObject();
@@ -112,16 +100,20 @@ public final class SearchProfileShardResults implements Writeable, ToXContent{
      * can be serialized to other nodes, emitted as JSON, etc.
      *
      * @param profilers
-     *            A list of Profilers to convert into
-     *            InternalProfileShardResults
-     * @return A list of corresponding InternalProfileShardResults
+     *            The {@link Profilers} to convert into results
+     * @return A {@link ProfileShardResult} representing the results for this
+     *         shard
      */
-    public static List<ProfileShardResult> buildShardResults(List<QueryProfiler> profilers) {
-        List<ProfileShardResult> results = new ArrayList<>(profilers.size());
-        for (QueryProfiler profiler : profilers) {
-            ProfileShardResult result = new ProfileShardResult(profiler.getQueryTree(), profiler.getRewriteTime(), profiler.getCollector());
-            results.add(result);
+    public static ProfileShardResult buildShardResults(Profilers profilers) {
+        List<QueryProfiler> queryProfilers = profilers.getQueryProfilers();
+        AggregationProfiler aggProfiler = profilers.getAggregationProfiler();
+        List<QueryProfileShardResult> queryResults = new ArrayList<>(queryProfilers.size());
+        for (QueryProfiler queryProfiler : queryProfilers) {
+            QueryProfileShardResult result = new QueryProfileShardResult(queryProfiler.getTree(), queryProfiler.getRewriteTime(),
+                    queryProfiler.getCollector());
+            queryResults.add(result);
         }
-        return results;
+        AggregationProfileShardResult aggResults = new AggregationProfileShardResult(aggProfiler.getTree());
+        return new ProfileShardResult(queryResults, aggResults);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileBreakdown.java b/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileBreakdown.java
new file mode 100644
index 00000000000..b4cb1efe5d3
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileBreakdown.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile.aggregation;
+
+import org.elasticsearch.search.profile.AbstractProfileBreakdown;
+
+public class AggregationProfileBreakdown extends AbstractProfileBreakdown<AggregationTimingType> {
+
+    public AggregationProfileBreakdown() {
+        super(AggregationTimingType.values());
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResult.java b/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResult.java
new file mode 100644
index 00000000000..df55c5592d6
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfileShardResult.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile.aggregation;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.profile.ProfileResult;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * A container class to hold the profile results for a single shard in the request.
+ * Contains a list of query profiles, a collector tree and a total rewrite tree.
+ */
+public final class AggregationProfileShardResult implements Writeable, ToXContent {
+
+    private final List<ProfileResult> aggProfileResults;
+
+    public AggregationProfileShardResult(List<ProfileResult> aggProfileResults) {
+        this.aggProfileResults = aggProfileResults;
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public AggregationProfileShardResult(StreamInput in) throws IOException {
+        int profileSize = in.readVInt();
+        aggProfileResults = new ArrayList<>(profileSize);
+        for (int j = 0; j < profileSize; j++) {
+            aggProfileResults.add(new ProfileResult(in));
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(aggProfileResults.size());
+        for (ProfileResult p : aggProfileResults) {
+            p.writeTo(out);
+        }
+    }
+
+
+    public List<ProfileResult> getProfileResults() {
+        return Collections.unmodifiableList(aggProfileResults);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startArray("aggregations");
+        for (ProfileResult p : aggProfileResults) {
+            p.toXContent(builder, params);
+        }
+        builder.endArray();
+        return builder;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfiler.java b/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfiler.java
new file mode 100644
index 00000000000..45d401ccbdc
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationProfiler.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile.aggregation;
+
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.profile.AbstractProfiler;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+public class AggregationProfiler extends AbstractProfiler<AggregationProfileBreakdown, Aggregator> {
+
+    private final Map<List<String>, AggregationProfileBreakdown> profileBrakdownLookup = new HashMap<>();
+
+    public AggregationProfiler() {
+        super(new InternalAggregationProfileTree());
+    }
+
+    @Override
+    public AggregationProfileBreakdown getQueryBreakdown(Aggregator agg) {
+        List<String> path = getAggregatorPath(agg);
+        AggregationProfileBreakdown aggregationProfileBreakdown = profileBrakdownLookup.get(path);
+        if (aggregationProfileBreakdown == null) {
+            aggregationProfileBreakdown = super.getQueryBreakdown(agg);
+            profileBrakdownLookup.put(path, aggregationProfileBreakdown);
+        }
+        return aggregationProfileBreakdown;
+    }
+
+    public static List<String> getAggregatorPath(Aggregator agg) {
+        LinkedList<String> path = new LinkedList<>();
+        while (agg != null) {
+            path.addFirst(agg.name());
+            agg = agg.parent();
+        }
+        return path;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationTimingType.java b/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationTimingType.java
new file mode 100644
index 00000000000..d1c5d3dd538
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/profile/aggregation/AggregationTimingType.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile.aggregation;
+
+import java.util.Locale;
+
+public enum AggregationTimingType {
+    INITIALIZE,
+    COLLECT,
+    BUILD_AGGREGATION,
+    REDUCE;
+
+    @Override
+    public String toString() {
+        return name().toLowerCase(Locale.ROOT);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/profile/aggregation/InternalAggregationProfileTree.java b/core/src/main/java/org/elasticsearch/search/profile/aggregation/InternalAggregationProfileTree.java
new file mode 100644
index 00000000000..f367595c84c
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/profile/aggregation/InternalAggregationProfileTree.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile.aggregation;
+
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactory.MultiBucketAggregatorWrapper;
+import org.elasticsearch.search.profile.AbstractInternalProfileTree;
+
+public class InternalAggregationProfileTree extends AbstractInternalProfileTree<AggregationProfileBreakdown, Aggregator> {
+
+    @Override
+    protected AggregationProfileBreakdown createProfileBreakdown() {
+        return new AggregationProfileBreakdown();
+    }
+
+    @Override
+    protected String getTypeFromElement(Aggregator element) {
+        if (element instanceof MultiBucketAggregatorWrapper) {
+            return ((MultiBucketAggregatorWrapper) element).getWrappedClass().getName();
+        }
+        return element.getClass().getName();
+    }
+
+    @Override
+    protected String getDescriptionFromElement(Aggregator element) {
+        return element.name();
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingAggregator.java b/core/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingAggregator.java
new file mode 100644
index 00000000000..2883c2903e8
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingAggregator.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile.aggregation;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.LeafBucketCollector;
+import org.elasticsearch.search.aggregations.support.AggregationContext;
+
+import java.io.IOException;
+
+public class ProfilingAggregator extends Aggregator {
+
+    private final Aggregator delegate;
+    private final AggregationProfiler profiler;
+    private AggregationProfileBreakdown profileBreakdown;
+
+    public ProfilingAggregator(Aggregator delegate, AggregationProfiler profiler) throws IOException {
+        this.profiler = profiler;
+        this.delegate = delegate;
+    }
+
+    @Override
+    public void close() {
+        delegate.close();
+    }
+
+    @Override
+    public boolean needsScores() {
+        return delegate.needsScores();
+    }
+
+    @Override
+    public String name() {
+        return delegate.name();
+    }
+
+    @Override
+    public AggregationContext context() {
+        return delegate.context();
+    }
+
+    @Override
+    public Aggregator parent() {
+        return delegate.parent();
+    }
+
+    @Override
+    public Aggregator subAggregator(String name) {
+        return delegate.subAggregator(name);
+    }
+
+    @Override
+    public InternalAggregation buildAggregation(long bucket) throws IOException {
+        profileBreakdown.startTime(AggregationTimingType.BUILD_AGGREGATION);
+        InternalAggregation result = delegate.buildAggregation(bucket);
+        profileBreakdown.stopAndRecordTime();
+        return result;
+    }
+
+    @Override
+    public InternalAggregation buildEmptyAggregation() {
+        return delegate.buildEmptyAggregation();
+    }
+
+    @Override
+    public LeafBucketCollector getLeafCollector(LeafReaderContext ctx) throws IOException {
+        return new ProfilingLeafBucketCollector(delegate.getLeafCollector(ctx), profileBreakdown);
+    }
+
+    @Override
+    public void preCollection() throws IOException {
+        this.profileBreakdown = profiler.getQueryBreakdown(delegate);
+        profileBreakdown.startTime(AggregationTimingType.INITIALIZE);
+        delegate.preCollection();
+        profileBreakdown.stopAndRecordTime();
+        profiler.pollLastElement();
+    }
+
+    @Override
+    public void postCollection() throws IOException {
+        delegate.postCollection();
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingLeafBucketCollector.java b/core/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingLeafBucketCollector.java
new file mode 100644
index 00000000000..75c90ded709
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/profile/aggregation/ProfilingLeafBucketCollector.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile.aggregation;
+
+import org.elasticsearch.search.aggregations.LeafBucketCollector;
+
+import java.io.IOException;
+
+public class ProfilingLeafBucketCollector extends LeafBucketCollector {
+
+    private LeafBucketCollector delegate;
+    private AggregationProfileBreakdown profileBreakdown;
+
+    public ProfilingLeafBucketCollector(LeafBucketCollector delegate, AggregationProfileBreakdown profileBreakdown) {
+        this.delegate = delegate;
+        this.profileBreakdown = profileBreakdown;
+    }
+
+    @Override
+    public void collect(int doc, long bucket) throws IOException {
+        profileBreakdown.startTime(AggregationTimingType.COLLECT);
+        delegate.collect(doc, bucket);
+        profileBreakdown.stopAndRecordTime();
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/search/profile/query/InternalQueryProfileTree.java b/core/src/main/java/org/elasticsearch/search/profile/query/InternalQueryProfileTree.java
index 5b92ef8b2a9..013b7d3a506 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/query/InternalQueryProfileTree.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/query/InternalQueryProfileTree.java
@@ -20,89 +20,33 @@
 package org.elasticsearch.search.profile.query;
 
 import org.apache.lucene.search.Query;
+import org.elasticsearch.search.profile.AbstractInternalProfileTree;
 import org.elasticsearch.search.profile.ProfileResult;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Deque;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.LinkedBlockingDeque;
-
 /**
  * This class tracks the dependency tree for queries (scoring and rewriting) and
  * generates {@link QueryProfileBreakdown} for each node in the tree.  It also finalizes the tree
  * and returns a list of {@link ProfileResult} that can be serialized back to the client
  */
-final class InternalQueryProfileTree {
-
-    private ArrayList<QueryProfileBreakdown> timings;
-
-    /** Maps the Query to it's list of children.  This is basically the dependency tree */
-    private ArrayList<ArrayList<Integer>> tree;
-
-    /** A list of the original queries, keyed by index position */
-    private ArrayList<Query> queries;
-
-    /** A list of top-level "roots".  Each root can have its own tree of profiles */
-    private ArrayList<Integer> roots;
+final class InternalQueryProfileTree extends AbstractInternalProfileTree<QueryProfileBreakdown, Query> {
 
     /** Rewrite time */
     private long rewriteTime;
     private long rewriteScratch;
 
-    /** A temporary stack used to record where we are in the dependency tree.  Only used by scoring queries */
-    private Deque<Integer> stack;
-
-    private int currentToken = 0;
-
-    public InternalQueryProfileTree() {
-        timings = new ArrayList<>(10);
-        stack = new LinkedBlockingDeque<>(10);
-        tree = new ArrayList<>(10);
-        queries = new ArrayList<>(10);
-        roots = new ArrayList<>(10);
+    @Override
+    protected QueryProfileBreakdown createProfileBreakdown() {
+        return new QueryProfileBreakdown();
     }
 
-    /**
-     * Returns a {@link QueryProfileBreakdown} for a scoring query.  Scoring queries (e.g. those
-     * that are past the rewrite phase and are now being wrapped by createWeight() ) follow
-     * a recursive progression.  We can track the dependency tree by a simple stack
-     *
-     * The only hiccup is that the first scoring query will be identical to the last rewritten
-     * query, so we need to take special care to fix that
-     *
-     * @param query The scoring query we wish to profile
-     * @return      A ProfileBreakdown for this query
-     */
-    public QueryProfileBreakdown getQueryBreakdown(Query query) {
-        int token = currentToken;
+    @Override
+    protected String getTypeFromElement(Query query) {
+        return query.getClass().getSimpleName();
+    }
 
-        boolean stackEmpty = stack.isEmpty();
-
-        // If the stack is empty, we are a new root query
-        if (stackEmpty) {
-
-            // We couldn't find a rewritten query to attach to, so just add it as a
-            // top-level root. This is just a precaution: it really shouldn't happen.
-            // We would only get here if a top-level query that never rewrites for some reason.
-            roots.add(token);
-
-            // Increment the token since we are adding a new node, but notably, do not
-            // updateParent() because this was added as a root
-            currentToken += 1;
-            stack.add(token);
-
-            return addDependencyNode(query, token);
-        }
-
-        updateParent(token);
-
-        // Increment the token since we are adding a new node
-        currentToken += 1;
-        stack.add(token);
-
-        return addDependencyNode(query, token);
+    @Override
+    protected String getDescriptionFromElement(Query query) {
+        return query.toString();
     }
 
     /**
@@ -128,113 +72,7 @@ final class InternalQueryProfileTree {
         return time;
     }
 
-    /**
-     * Helper method to add a new node to the dependency tree.
-     *
-     * Initializes a new list in the dependency tree, saves the query and
-     * generates a new {@link QueryProfileBreakdown} to track the timings
-     * of this query
-     *
-     * @param query             The query to profile
-     * @param token             The assigned token for this query
-     * @return                  A ProfileBreakdown to profile this query
-     */
-    private QueryProfileBreakdown addDependencyNode(Query query, int token) {
-
-        // Add a new slot in the dependency tree
-        tree.add(new ArrayList<>(5));
-
-        // Save our query for lookup later
-        queries.add(query);
-
-        QueryProfileBreakdown queryTimings = new QueryProfileBreakdown();
-        timings.add(token, queryTimings);
-        return queryTimings;
-    }
-
-    /**
-     * Removes the last (e.g. most recent) value on the stack
-     */
-    public void pollLast() {
-        stack.pollLast();
-    }
-
-    /**
-     * After the query has been run and profiled, we need to merge the flat timing map
-     * with the dependency graph to build a data structure that mirrors the original
-     * query tree
-     *
-     * @return a hierarchical representation of the profiled query tree
-     */
-    public List<ProfileResult> getQueryTree() {
-        ArrayList<ProfileResult> results = new ArrayList<>(5);
-        for (Integer root : roots) {
-            results.add(doGetQueryTree(root));
-        }
-        return results;
-    }
-
-    /**
-     * Recursive helper to finalize a node in the dependency tree
-     * @param token  The node we are currently finalizing
-     * @return       A hierarchical representation of the tree inclusive of children at this level
-     */
-    private ProfileResult doGetQueryTree(int token) {
-        Query query = queries.get(token);
-        QueryProfileBreakdown breakdown = timings.get(token);
-        Map<String, Long> timings = breakdown.toTimingMap();
-        List<Integer> children = tree.get(token);
-        List<ProfileResult> childrenProfileResults = Collections.emptyList();
-
-        if (children != null) {
-            childrenProfileResults = new ArrayList<>(children.size());
-            for (Integer child : children) {
-                ProfileResult childNode = doGetQueryTree(child);
-                childrenProfileResults.add(childNode);
-            }
-        }
-
-        // TODO this would be better done bottom-up instead of top-down to avoid
-        // calculating the same times over and over...but worth the effort?
-        long nodeTime = getNodeTime(timings, childrenProfileResults);
-        String queryDescription = query.getClass().getSimpleName();
-        String luceneName = query.toString();
-        return new ProfileResult(queryDescription, luceneName, timings, childrenProfileResults, nodeTime);
-    }
-
     public long getRewriteTime() {
         return rewriteTime;
     }
-
-    /**
-     * Internal helper to add a child to the current parent node
-     *
-     * @param childToken The child to add to the current parent
-     */
-    private void updateParent(int childToken) {
-        Integer parent = stack.peekLast();
-        ArrayList<Integer> parentNode = tree.get(parent);
-        parentNode.add(childToken);
-        tree.set(parent, parentNode);
-    }
-
-    /**
-     * Internal helper to calculate the time of a node, inclusive of children
-     *
-     * @param timings   A map of breakdown timing for the node
-     * @param children  All children profile results at this node
-     * @return          The total time at this node, inclusive of children
-     */
-    private static long getNodeTime(Map<String, Long> timings, List<ProfileResult> children) {
-        long nodeTime = 0;
-        for (long time : timings.values()) {
-            nodeTime += time;
-        }
-
-        // Then add up our children
-        for (ProfileResult child : children) {
-            nodeTime += getNodeTime(child.getTimeBreakdown(), child.getProfiledChildren());
-        }
-        return nodeTime;
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfileShardResult.java b/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfileShardResult.java
new file mode 100644
index 00000000000..d5e00aca336
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfileShardResult.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile.query;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.profile.ProfileResult;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * A container class to hold the profile results for a single shard in the request.
+ * Contains a list of query profiles, a collector tree and a total rewrite tree.
+ */
+public final class QueryProfileShardResult implements Writeable, ToXContent {
+
+    private final List<ProfileResult> queryProfileResults;
+
+    private final CollectorResult profileCollector;
+
+    private final long rewriteTime;
+
+    public QueryProfileShardResult(List<ProfileResult> queryProfileResults, long rewriteTime,
+                              CollectorResult profileCollector) {
+        assert(profileCollector != null);
+        this.queryProfileResults = queryProfileResults;
+        this.profileCollector = profileCollector;
+        this.rewriteTime = rewriteTime;
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public QueryProfileShardResult(StreamInput in) throws IOException {
+        int profileSize = in.readVInt();
+        queryProfileResults = new ArrayList<>(profileSize);
+        for (int j = 0; j < profileSize; j++) {
+            queryProfileResults.add(new ProfileResult(in));
+        }
+
+        profileCollector = new CollectorResult(in);
+        rewriteTime = in.readLong();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVInt(queryProfileResults.size());
+        for (ProfileResult p : queryProfileResults) {
+            p.writeTo(out);
+        }
+        profileCollector.writeTo(out);
+        out.writeLong(rewriteTime);
+    }
+
+
+    public List<ProfileResult> getQueryResults() {
+        return Collections.unmodifiableList(queryProfileResults);
+    }
+
+    public long getRewriteTime() {
+        return rewriteTime;
+    }
+
+    public CollectorResult getCollectorResult() {
+        return profileCollector;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startArray("query");
+        for (ProfileResult p : queryProfileResults) {
+            p.toXContent(builder, params);
+        }
+        builder.endArray();
+        builder.field("rewrite_time", rewriteTime);
+        builder.startArray("collector");
+        profileCollector.toXContent(builder, params);
+        builder.endArray();
+        return builder;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfiler.java b/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfiler.java
index 57341ee132f..0051356e35a 100644
--- a/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfiler.java
+++ b/core/src/main/java/org/elasticsearch/search/profile/query/QueryProfiler.java
@@ -20,9 +20,8 @@
 package org.elasticsearch.search.profile.query;
 
 import org.apache.lucene.search.Query;
-import org.elasticsearch.search.profile.ProfileResult;
+import org.elasticsearch.search.profile.AbstractProfiler;
 
-import java.util.List;
 import java.util.Objects;
 
 /**
@@ -36,16 +35,16 @@ import java.util.Objects;
  * request may execute two searches (query + global agg).  A Profiler just
  * represents one of those
  */
-public final class QueryProfiler {
-
-    private final InternalQueryProfileTree queryTree = new InternalQueryProfileTree();
+public final class QueryProfiler extends AbstractProfiler<QueryProfileBreakdown, Query> {
 
     /**
      * The root Collector used in the search
      */
     private InternalProfileCollector collector;
 
-    public QueryProfiler() {}
+    public QueryProfiler() {
+        super(new InternalQueryProfileTree());
+    }
 
     /** Set the collector that is associated with this profiler. */
     public void setCollector(InternalProfileCollector collector) {
@@ -55,21 +54,12 @@ public final class QueryProfiler {
         this.collector = Objects.requireNonNull(collector);
     }
 
-    /**
-     * Get the {@link QueryProfileBreakdown} for the given query, potentially creating it if it did not exist.
-     * This should only be used for queries that will be undergoing scoring. Do not use it to profile the
-     * rewriting phase
-     */
-    public QueryProfileBreakdown getQueryBreakdown(Query query) {
-        return queryTree.getQueryBreakdown(query);
-    }
-
     /**
      * Begin timing the rewrite phase of a request.  All rewrites are accumulated together into a
      * single metric
      */
     public void startRewriteTime() {
-        queryTree.startRewriteTime();
+        ((InternalQueryProfileTree) profileTree).startRewriteTime();
     }
 
     /**
@@ -79,29 +69,14 @@ public final class QueryProfiler {
      * @return cumulative rewrite time
      */
     public long stopAndAddRewriteTime() {
-        return queryTree.stopAndAddRewriteTime();
-    }
-
-    /**
-     * Removes the last (e.g. most recent) query on the stack.  This should only be called for scoring
-     * queries, not rewritten queries
-     */
-    public void pollLastQuery() {
-        queryTree.pollLast();
-    }
-
-    /**
-     * @return a hierarchical representation of the profiled query tree
-     */
-    public List<ProfileResult> getQueryTree() {
-        return queryTree.getQueryTree();
+        return ((InternalQueryProfileTree) profileTree).stopAndAddRewriteTime();
     }
 
     /**
      * @return total time taken to rewrite all queries in this profile
      */
     public long getRewriteTime() {
-        return queryTree.getRewriteTime();
+        return ((InternalQueryProfileTree) profileTree).getRewriteTime();
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index df68064f617..174a337f3d2 100644
--- a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -112,8 +112,8 @@ public class QueryPhase implements SearchPhase {
         aggregationPhase.execute(searchContext);
 
         if (searchContext.getProfilers() != null) {
-            List<ProfileShardResult> shardResults = SearchProfileShardResults
-                    .buildShardResults(searchContext.getProfilers().getQueryProfilers());
+            ProfileShardResult shardResults = SearchProfileShardResults
+                    .buildShardResults(searchContext.getProfilers());
             searchContext.queryResult().profileResults(shardResults);
         }
     }
@@ -385,8 +385,8 @@ public class QueryPhase implements SearchPhase {
             queryResult.topDocs(topDocsCallable.call(), sortValueFormats);
 
             if (searchContext.getProfilers() != null) {
-                List<ProfileShardResult> shardResults = SearchProfileShardResults
-                        .buildShardResults(searchContext.getProfilers().getQueryProfilers());
+                ProfileShardResult shardResults = SearchProfileShardResults
+                        .buildShardResults(searchContext.getProfilers());
                 searchContext.queryResult().profileResults(shardResults);
             }
 
diff --git a/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
index 1408ebe8359..be8c895eecd 100644
--- a/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
+++ b/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
@@ -59,7 +59,7 @@ public class QuerySearchResult extends QuerySearchResultProvider {
     private Suggest suggest;
     private boolean searchTimedOut;
     private Boolean terminatedEarly = null;
-    private List<ProfileShardResult> profileShardResults;
+    private ProfileShardResult profileShardResults;
 
     public QuerySearchResult() {
 
@@ -143,7 +143,7 @@ public class QuerySearchResult extends QuerySearchResultProvider {
      * Returns the profiled results for this search, or potentially null if result was empty
      * @return The profiled results, or null
      */
-    public @Nullable List<ProfileShardResult> profileResults() {
+    public @Nullable ProfileShardResult profileResults() {
         return profileShardResults;
     }
 
@@ -151,7 +151,7 @@ public class QuerySearchResult extends QuerySearchResultProvider {
      * Sets the finalized profiling results for this query
      * @param shardResults The finalized profile
      */
-    public void profileResults(List<ProfileShardResult> shardResults) {
+    public void profileResults(ProfileShardResult shardResults) {
         this.profileShardResults = shardResults;
     }
 
@@ -237,12 +237,7 @@ public class QuerySearchResult extends QuerySearchResultProvider {
         terminatedEarly = in.readOptionalBoolean();
 
         if (in.getVersion().onOrAfter(Version.V_2_2_0) && in.readBoolean()) {
-            int profileSize = in.readVInt();
-            profileShardResults = new ArrayList<>(profileSize);
-            for (int i = 0; i < profileSize; i++) {
-                ProfileShardResult result = new ProfileShardResult(in);
-                profileShardResults.add(result);
-            }
+            profileShardResults = new ProfileShardResult(in);
         }
     }
 
@@ -296,10 +291,7 @@ public class QuerySearchResult extends QuerySearchResultProvider {
                 out.writeBoolean(false);
             } else {
                 out.writeBoolean(true);
-                out.writeVInt(profileShardResults.size());
-                for (ProfileShardResult shardResult : profileShardResults) {
-                    shardResult.writeTo(out);
-                }
+                profileShardResults.writeTo(out);
             }
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java b/core/src/test/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java
new file mode 100644
index 00000000000..848b230b3fa
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/profile/aggregation/AggregationProfilerIT.java
@@ -0,0 +1,403 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.profile.aggregation;
+
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregator;
+import org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator;
+import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregator;
+import org.elasticsearch.search.aggregations.metrics.max.MaxAggregator;
+import org.elasticsearch.search.profile.ProfileResult;
+import org.elasticsearch.search.profile.ProfileShardResult;
+import org.elasticsearch.search.profile.aggregation.AggregationProfileShardResult;
+import org.elasticsearch.search.profile.aggregation.AggregationTimingType;
+import org.elasticsearch.test.ESIntegTestCase;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.avg;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.max;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.notNullValue;
+
+@ESIntegTestCase.SuiteScopeTestCase
+public class AggregationProfilerIT extends ESIntegTestCase {
+
+
+    private static final String NUMBER_FIELD = "number";
+    private static final String TAG_FIELD = "tag";
+    private static final String STRING_FIELD = "string_field";
+
+    @Override
+    protected int numberOfShards() {
+        return 1;
+    }
+
+    @Override
+    protected void setupSuiteScopeCluster() throws Exception {
+        assertAcked(client().admin().indices().prepareCreate("idx")
+                .addMapping("type", STRING_FIELD, "type=keyword", NUMBER_FIELD, "type=integer", TAG_FIELD, "type=keyword").get());
+        List<IndexRequestBuilder> builders = new ArrayList<>();
+
+        String[] randomStrings = new String[randomIntBetween(2, 10)];
+        for (int i = 0; i < randomStrings.length; i++) {
+            randomStrings[i] = randomAsciiOfLength(10);
+        }
+
+        for (int i = 0; i < 5; i++) {
+            builders.add(client().prepareIndex("idx", "type").setSource(
+                    jsonBuilder().startObject()
+                        .field(STRING_FIELD, randomFrom(randomStrings))
+                        .field(NUMBER_FIELD, randomIntBetween(0, 9))
+                        .field(TAG_FIELD, randomBoolean() ? "more" : "less")
+                        .endObject()));
+        }
+
+        indexRandom(true, builders);
+        createIndex("idx_unmapped");
+        ensureSearchable();
+    }
+
+    public void testSimpleProfile() {
+        SearchResponse response = client().prepareSearch("idx").setProfile(true)
+                .addAggregation(histogram("histo").field(NUMBER_FIELD).interval(1L)).get();
+        assertSearchResponse(response);
+        Map<String, ProfileShardResult> profileResults = response.getProfileResults();
+        assertThat(profileResults, notNullValue());
+        assertThat(profileResults.size(), equalTo(getNumShards("idx").numPrimaries));
+        for (ProfileShardResult profileShardResult : profileResults.values()) {
+            assertThat(profileShardResult, notNullValue());
+            AggregationProfileShardResult aggProfileResults = profileShardResult.getAggregationProfileResults();
+            assertThat(aggProfileResults, notNullValue());
+            List<ProfileResult> aggProfileResultsList = aggProfileResults.getProfileResults();
+            assertThat(aggProfileResultsList, notNullValue());
+            assertThat(aggProfileResultsList.size(), equalTo(1));
+            ProfileResult histoAggResult = aggProfileResultsList.get(0);
+            assertThat(histoAggResult, notNullValue());
+            assertThat(histoAggResult.getQueryName(), equalTo(HistogramAggregator.class.getName()));
+            assertThat(histoAggResult.getLuceneDescription(), equalTo("histo"));
+            assertThat(histoAggResult.getProfiledChildren().size(), equalTo(0));
+            assertThat(histoAggResult.getTime(), greaterThan(0L));
+            Map<String, Long> breakdown = histoAggResult.getTimeBreakdown();
+            assertThat(breakdown, notNullValue());
+            assertThat(breakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue());
+            assertThat(breakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L));
+            assertThat(breakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue());
+            assertThat(breakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L));
+            assertThat(breakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue());
+            assertThat(breakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L));
+            assertThat(breakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue());
+            assertThat(breakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L));
+
+        }
+    }
+
+    public void testMultiLevelProfile() {
+        SearchResponse response = client().prepareSearch("idx").setProfile(true)
+                .addAggregation(histogram("histo").field(NUMBER_FIELD).interval(1L)
+                        .subAggregation(terms("terms").field(TAG_FIELD)
+                                .subAggregation(avg("avg").field(NUMBER_FIELD)))).get();
+        assertSearchResponse(response);
+        Map<String, ProfileShardResult> profileResults = response.getProfileResults();
+        assertThat(profileResults, notNullValue());
+        assertThat(profileResults.size(), equalTo(getNumShards("idx").numPrimaries));
+        for (ProfileShardResult profileShardResult : profileResults.values()) {
+            assertThat(profileShardResult, notNullValue());
+            AggregationProfileShardResult aggProfileResults = profileShardResult.getAggregationProfileResults();
+            assertThat(aggProfileResults, notNullValue());
+            List<ProfileResult> aggProfileResultsList = aggProfileResults.getProfileResults();
+            assertThat(aggProfileResultsList, notNullValue());
+            assertThat(aggProfileResultsList.size(), equalTo(1));
+            ProfileResult histoAggResult = aggProfileResultsList.get(0);
+            assertThat(histoAggResult, notNullValue());
+            assertThat(histoAggResult.getQueryName(), equalTo(HistogramAggregator.class.getName()));
+            assertThat(histoAggResult.getLuceneDescription(), equalTo("histo"));
+            assertThat(histoAggResult.getTime(), greaterThan(0L));
+            Map<String, Long> histoBreakdown = histoAggResult.getTimeBreakdown();
+            assertThat(histoBreakdown, notNullValue());
+            assertThat(histoBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue());
+            assertThat(histoBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L));
+            assertThat(histoBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue());
+            assertThat(histoBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L));
+            assertThat(histoBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue());
+            assertThat(histoBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L));
+            assertThat(histoBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue());
+            assertThat(histoBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L));
+            assertThat(histoAggResult.getProfiledChildren().size(), equalTo(1));
+
+            ProfileResult termsAggResult = histoAggResult.getProfiledChildren().get(0);
+            assertThat(termsAggResult, notNullValue());
+            assertThat(termsAggResult.getQueryName(), equalTo(GlobalOrdinalsStringTermsAggregator.WithHash.class.getName()));
+            assertThat(termsAggResult.getLuceneDescription(), equalTo("terms"));
+            assertThat(termsAggResult.getTime(), greaterThan(0L));
+            Map<String, Long> termsBreakdown = termsAggResult.getTimeBreakdown();
+            assertThat(termsBreakdown, notNullValue());
+            assertThat(termsBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue());
+            assertThat(termsBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L));
+            assertThat(termsBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue());
+            assertThat(termsBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L));
+            assertThat(termsBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue());
+            assertThat(termsBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L));
+            assertThat(termsBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue());
+            assertThat(termsBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L));
+            assertThat(termsAggResult.getProfiledChildren().size(), equalTo(1));
+
+            ProfileResult avgAggResult = termsAggResult.getProfiledChildren().get(0);
+            assertThat(avgAggResult, notNullValue());
+            assertThat(avgAggResult.getQueryName(), equalTo(AvgAggregator.class.getName()));
+            assertThat(avgAggResult.getLuceneDescription(), equalTo("avg"));
+            assertThat(avgAggResult.getTime(), greaterThan(0L));
+            Map<String, Long> avgBreakdown = termsAggResult.getTimeBreakdown();
+            assertThat(avgBreakdown, notNullValue());
+            assertThat(avgBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue());
+            assertThat(avgBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L));
+            assertThat(avgBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue());
+            assertThat(avgBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L));
+            assertThat(avgBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue());
+            assertThat(avgBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L));
+            assertThat(avgBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue());
+            assertThat(avgBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L));
+            assertThat(avgAggResult.getProfiledChildren().size(), equalTo(0));
+        }
+    }
+
+    public void testComplexProfile() {
+        SearchResponse response = client().prepareSearch("idx").setProfile(true)
+                .addAggregation(histogram("histo").field(NUMBER_FIELD).interval(1L)
+                        .subAggregation(terms("tags").field(TAG_FIELD)
+                                .subAggregation(avg("avg").field(NUMBER_FIELD))
+                                .subAggregation(max("max").field(NUMBER_FIELD)))
+                        .subAggregation(terms("strings").field(STRING_FIELD)
+                                .subAggregation(avg("avg").field(NUMBER_FIELD))
+                                .subAggregation(max("max").field(NUMBER_FIELD))
+                                .subAggregation(terms("tags").field(TAG_FIELD)
+                                        .subAggregation(avg("avg").field(NUMBER_FIELD))
+                                        .subAggregation(max("max").field(NUMBER_FIELD)))))
+                .get();
+        assertSearchResponse(response);
+        Map<String, ProfileShardResult> profileResults = response.getProfileResults();
+        assertThat(profileResults, notNullValue());
+        assertThat(profileResults.size(), equalTo(getNumShards("idx").numPrimaries));
+        for (ProfileShardResult profileShardResult : profileResults.values()) {
+            assertThat(profileShardResult, notNullValue());
+            AggregationProfileShardResult aggProfileResults = profileShardResult.getAggregationProfileResults();
+            assertThat(aggProfileResults, notNullValue());
+            List<ProfileResult> aggProfileResultsList = aggProfileResults.getProfileResults();
+            assertThat(aggProfileResultsList, notNullValue());
+            assertThat(aggProfileResultsList.size(), equalTo(1));
+            ProfileResult histoAggResult = aggProfileResultsList.get(0);
+            assertThat(histoAggResult, notNullValue());
+            assertThat(histoAggResult.getQueryName(), equalTo(HistogramAggregator.class.getName()));
+            assertThat(histoAggResult.getLuceneDescription(), equalTo("histo"));
+            assertThat(histoAggResult.getTime(), greaterThan(0L));
+            Map<String, Long> histoBreakdown = histoAggResult.getTimeBreakdown();
+            assertThat(histoBreakdown, notNullValue());
+            assertThat(histoBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue());
+            assertThat(histoBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L));
+            assertThat(histoBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue());
+            assertThat(histoBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L));
+            assertThat(histoBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue());
+            assertThat(histoBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L));
+            assertThat(histoBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue());
+            assertThat(histoBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L));
+            assertThat(histoAggResult.getProfiledChildren().size(), equalTo(2));
+
+            ProfileResult tagsAggResult = histoAggResult.getProfiledChildren().get(0);
+            assertThat(tagsAggResult, notNullValue());
+            assertThat(tagsAggResult.getQueryName(), equalTo(GlobalOrdinalsStringTermsAggregator.WithHash.class.getName()));
+            assertThat(tagsAggResult.getLuceneDescription(), equalTo("tags"));
+            assertThat(tagsAggResult.getTime(), greaterThan(0L));
+            Map<String, Long> tagsBreakdown = tagsAggResult.getTimeBreakdown();
+            assertThat(tagsBreakdown, notNullValue());
+            assertThat(tagsBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue());
+            assertThat(tagsBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L));
+            assertThat(tagsBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue());
+            assertThat(tagsBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L));
+            assertThat(tagsBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue());
+            assertThat(tagsBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L));
+            assertThat(tagsBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue());
+            assertThat(tagsBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L));
+            assertThat(tagsAggResult.getProfiledChildren().size(), equalTo(2));
+
+            ProfileResult avgAggResult = tagsAggResult.getProfiledChildren().get(0);
+            assertThat(avgAggResult, notNullValue());
+            assertThat(avgAggResult.getQueryName(), equalTo(AvgAggregator.class.getName()));
+            assertThat(avgAggResult.getLuceneDescription(), equalTo("avg"));
+            assertThat(avgAggResult.getTime(), greaterThan(0L));
+            Map<String, Long> avgBreakdown = tagsAggResult.getTimeBreakdown();
+            assertThat(avgBreakdown, notNullValue());
+            assertThat(avgBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue());
+            assertThat(avgBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L));
+            assertThat(avgBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue());
+            assertThat(avgBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L));
+            assertThat(avgBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue());
+            assertThat(avgBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L));
+            assertThat(avgBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue());
+            assertThat(avgBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L));
+            assertThat(avgAggResult.getProfiledChildren().size(), equalTo(0));
+
+            ProfileResult maxAggResult = tagsAggResult.getProfiledChildren().get(1);
+            assertThat(maxAggResult, notNullValue());
+            assertThat(maxAggResult.getQueryName(), equalTo(MaxAggregator.class.getName()));
+            assertThat(maxAggResult.getLuceneDescription(), equalTo("max"));
+            assertThat(maxAggResult.getTime(), greaterThan(0L));
+            Map<String, Long> maxBreakdown = tagsAggResult.getTimeBreakdown();
+            assertThat(maxBreakdown, notNullValue());
+            assertThat(maxBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue());
+            assertThat(maxBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L));
+            assertThat(maxBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue());
+            assertThat(maxBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L));
+            assertThat(maxBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue());
+            assertThat(maxBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L));
+            assertThat(maxBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue());
+            assertThat(maxBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L));
+            assertThat(maxAggResult.getProfiledChildren().size(), equalTo(0));
+
+            ProfileResult stringsAggResult = histoAggResult.getProfiledChildren().get(1);
+            assertThat(stringsAggResult, notNullValue());
+            assertThat(stringsAggResult.getQueryName(), equalTo(GlobalOrdinalsStringTermsAggregator.WithHash.class.getName()));
+            assertThat(stringsAggResult.getLuceneDescription(), equalTo("strings"));
+            assertThat(stringsAggResult.getTime(), greaterThan(0L));
+            Map<String, Long> stringsBreakdown = stringsAggResult.getTimeBreakdown();
+            assertThat(stringsBreakdown, notNullValue());
+            assertThat(stringsBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue());
+            assertThat(stringsBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L));
+            assertThat(stringsBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue());
+            assertThat(stringsBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L));
+            assertThat(stringsBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue());
+            assertThat(stringsBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L));
+            assertThat(stringsBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue());
+            assertThat(stringsBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L));
+            assertThat(stringsAggResult.getProfiledChildren().size(), equalTo(3));
+
+            avgAggResult = stringsAggResult.getProfiledChildren().get(0);
+            assertThat(avgAggResult, notNullValue());
+            assertThat(avgAggResult.getQueryName(), equalTo(AvgAggregator.class.getName()));
+            assertThat(avgAggResult.getLuceneDescription(), equalTo("avg"));
+            assertThat(avgAggResult.getTime(), greaterThan(0L));
+            avgBreakdown = stringsAggResult.getTimeBreakdown();
+            assertThat(avgBreakdown, notNullValue());
+            assertThat(avgBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue());
+            assertThat(avgBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L));
+            assertThat(avgBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue());
+            assertThat(avgBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L));
+            assertThat(avgBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue());
+            assertThat(avgBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L));
+            assertThat(avgBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue());
+            assertThat(avgBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L));
+            assertThat(avgAggResult.getProfiledChildren().size(), equalTo(0));
+
+            maxAggResult = stringsAggResult.getProfiledChildren().get(1);
+            assertThat(maxAggResult, notNullValue());
+            assertThat(maxAggResult.getQueryName(), equalTo(MaxAggregator.class.getName()));
+            assertThat(maxAggResult.getLuceneDescription(), equalTo("max"));
+            assertThat(maxAggResult.getTime(), greaterThan(0L));
+            maxBreakdown = stringsAggResult.getTimeBreakdown();
+            assertThat(maxBreakdown, notNullValue());
+            assertThat(maxBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue());
+            assertThat(maxBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L));
+            assertThat(maxBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue());
+            assertThat(maxBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L));
+            assertThat(maxBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue());
+            assertThat(maxBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L));
+            assertThat(maxBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue());
+            assertThat(maxBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L));
+            assertThat(maxAggResult.getProfiledChildren().size(), equalTo(0));
+
+            tagsAggResult = stringsAggResult.getProfiledChildren().get(2);
+            assertThat(tagsAggResult, notNullValue());
+            assertThat(tagsAggResult.getQueryName(), equalTo(GlobalOrdinalsStringTermsAggregator.WithHash.class.getName()));
+            assertThat(tagsAggResult.getLuceneDescription(), equalTo("tags"));
+            assertThat(tagsAggResult.getTime(), greaterThan(0L));
+            tagsBreakdown = tagsAggResult.getTimeBreakdown();
+            assertThat(tagsBreakdown, notNullValue());
+            assertThat(tagsBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue());
+            assertThat(tagsBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L));
+            assertThat(tagsBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue());
+            assertThat(tagsBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L));
+            assertThat(tagsBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue());
+            assertThat(tagsBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L));
+            assertThat(tagsBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue());
+            assertThat(tagsBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L));
+            assertThat(tagsAggResult.getProfiledChildren().size(), equalTo(2));
+
+            avgAggResult = tagsAggResult.getProfiledChildren().get(0);
+            assertThat(avgAggResult, notNullValue());
+            assertThat(avgAggResult.getQueryName(), equalTo(AvgAggregator.class.getName()));
+            assertThat(avgAggResult.getLuceneDescription(), equalTo("avg"));
+            assertThat(avgAggResult.getTime(), greaterThan(0L));
+            avgBreakdown = tagsAggResult.getTimeBreakdown();
+            assertThat(avgBreakdown, notNullValue());
+            assertThat(avgBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue());
+            assertThat(avgBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L));
+            assertThat(avgBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue());
+            assertThat(avgBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L));
+            assertThat(avgBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue());
+            assertThat(avgBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L));
+            assertThat(avgBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue());
+            assertThat(avgBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L));
+            assertThat(avgAggResult.getProfiledChildren().size(), equalTo(0));
+
+            maxAggResult = tagsAggResult.getProfiledChildren().get(1);
+            assertThat(maxAggResult, notNullValue());
+            assertThat(maxAggResult.getQueryName(), equalTo(MaxAggregator.class.getName()));
+            assertThat(maxAggResult.getLuceneDescription(), equalTo("max"));
+            assertThat(maxAggResult.getTime(), greaterThan(0L));
+            maxBreakdown = tagsAggResult.getTimeBreakdown();
+            assertThat(maxBreakdown, notNullValue());
+            assertThat(maxBreakdown.get(AggregationTimingType.INITIALIZE.toString()), notNullValue());
+            assertThat(maxBreakdown.get(AggregationTimingType.INITIALIZE.toString()), greaterThan(0L));
+            assertThat(maxBreakdown.get(AggregationTimingType.COLLECT.toString()), notNullValue());
+            assertThat(maxBreakdown.get(AggregationTimingType.COLLECT.toString()), greaterThan(0L));
+            assertThat(maxBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), notNullValue());
+            assertThat(maxBreakdown.get(AggregationTimingType.BUILD_AGGREGATION.toString()), greaterThan(0L));
+            assertThat(maxBreakdown.get(AggregationTimingType.REDUCE.toString()), notNullValue());
+            assertThat(maxBreakdown.get(AggregationTimingType.REDUCE.toString()), equalTo(0L));
+            assertThat(maxAggResult.getProfiledChildren().size(), equalTo(0));
+        }
+    }
+
+    public void testNoProfile() {
+        SearchResponse response = client().prepareSearch("idx").setProfile(false)
+                .addAggregation(histogram("histo").field(NUMBER_FIELD).interval(1L)
+                        .subAggregation(terms("tags").field(TAG_FIELD)
+                                .subAggregation(avg("avg").field(NUMBER_FIELD))
+                                .subAggregation(max("max").field(NUMBER_FIELD)))
+                        .subAggregation(terms("strings").field(STRING_FIELD)
+                                .subAggregation(avg("avg").field(NUMBER_FIELD))
+                                .subAggregation(max("max").field(NUMBER_FIELD))
+                                .subAggregation(terms("tags").field(TAG_FIELD)
+                                        .subAggregation(avg("avg").field(NUMBER_FIELD))
+                                        .subAggregation(max("max").field(NUMBER_FIELD)))))
+                .get();
+        assertSearchResponse(response);
+        Map<String, ProfileShardResult> profileResults = response.getProfileResults();
+        assertThat(profileResults, notNullValue());
+        assertThat(profileResults.size(), equalTo(0));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java b/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java
index 371aaadd3a7..b6935f021d4 100644
--- a/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java
+++ b/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java
@@ -85,8 +85,8 @@ public class QueryProfilerIT extends ESIntegTestCase {
 
             assertNotNull("Profile response element should not be null", resp.getProfileResults());
             assertThat("Profile response should not be an empty array", resp.getProfileResults().size(), not(0));
-            for (Map.Entry<String, List<ProfileShardResult>> shard : resp.getProfileResults().entrySet()) {
-                for (ProfileShardResult searchProfiles : shard.getValue()) {
+            for (Map.Entry<String, ProfileShardResult> shard : resp.getProfileResults().entrySet()) {
+                for (QueryProfileShardResult searchProfiles : shard.getValue().getQueryProfileResults()) {
                     for (ProfileResult result : searchProfiles.getQueryResults()) {
                         assertNotNull(result.getQueryName());
                         assertNotNull(result.getLuceneDescription());
@@ -163,8 +163,9 @@ public class QueryProfilerIT extends ESIntegTestCase {
                         nearlyEqual(vanillaMaxScore, profileMaxScore, 0.001));
             }
 
-            assertThat("Profile totalHits of [" + profileResponse.getHits().totalHits() + "] is not close to Vanilla totalHits ["
-                    + vanillaResponse.getHits().totalHits() + "]",
+            assertThat(
+                    "Profile totalHits of [" + profileResponse.getHits().totalHits() + "] is not close to Vanilla totalHits ["
+                            + vanillaResponse.getHits().totalHits() + "]",
                     vanillaResponse.getHits().getTotalHits(), equalTo(profileResponse.getHits().getTotalHits()));
 
             SearchHit[] vanillaHits = vanillaResponse.getHits().getHits();
@@ -203,12 +204,12 @@ public class QueryProfilerIT extends ESIntegTestCase {
                 .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .execute().actionGet();
 
-        Map<String, List<ProfileShardResult>> p = resp.getProfileResults();
+        Map<String, ProfileShardResult> p = resp.getProfileResults();
         assertNotNull(p);
         assertThat("Profile response should not be an empty array", resp.getProfileResults().size(), not(0));
 
-        for (Map.Entry<String, List<ProfileShardResult>> shardResult : resp.getProfileResults().entrySet()) {
-            for (ProfileShardResult searchProfiles : shardResult.getValue()) {
+        for (Map.Entry<String, ProfileShardResult> shardResult : resp.getProfileResults().entrySet()) {
+            for (QueryProfileShardResult searchProfiles : shardResult.getValue().getQueryProfileResults()) {
                 for (ProfileResult result : searchProfiles.getQueryResults()) {
                     assertEquals(result.getQueryName(), "TermQuery");
                     assertEquals(result.getLuceneDescription(), "field1:one");
@@ -250,12 +251,12 @@ public class QueryProfilerIT extends ESIntegTestCase {
                 .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .execute().actionGet();
 
-        Map<String, List<ProfileShardResult>> p = resp.getProfileResults();
+        Map<String, ProfileShardResult> p = resp.getProfileResults();
         assertNotNull(p);
         assertThat("Profile response should not be an empty array", resp.getProfileResults().size(), not(0));
 
-        for (Map.Entry<String, List<ProfileShardResult>> shardResult : resp.getProfileResults().entrySet()) {
-            for (ProfileShardResult searchProfiles : shardResult.getValue()) {
+        for (Map.Entry<String, ProfileShardResult> shardResult : resp.getProfileResults().entrySet()) {
+            for (QueryProfileShardResult searchProfiles : shardResult.getValue().getQueryProfileResults()) {
                 for (ProfileResult result : searchProfiles.getQueryResults()) {
                     assertEquals(result.getQueryName(), "BooleanQuery");
                     assertEquals(result.getLuceneDescription(), "+field1:one +field1:two");
@@ -322,8 +323,8 @@ public class QueryProfilerIT extends ESIntegTestCase {
         assertNotNull("Profile response element should not be null", resp.getProfileResults());
         assertThat("Profile response should not be an empty array", resp.getProfileResults().size(), not(0));
 
-        for (Map.Entry<String, List<ProfileShardResult>> shardResult : resp.getProfileResults().entrySet()) {
-            for (ProfileShardResult searchProfiles : shardResult.getValue()) {
+        for (Map.Entry<String, ProfileShardResult> shardResult : resp.getProfileResults().entrySet()) {
+            for (QueryProfileShardResult searchProfiles : shardResult.getValue().getQueryProfileResults()) {
                 for (ProfileResult result : searchProfiles.getQueryResults()) {
                     assertNotNull(result.getQueryName());
                     assertNotNull(result.getLuceneDescription());
@@ -374,8 +375,8 @@ public class QueryProfilerIT extends ESIntegTestCase {
         assertNotNull("Profile response element should not be null", resp.getProfileResults());
         assertThat("Profile response should not be an empty array", resp.getProfileResults().size(), not(0));
 
-        for (Map.Entry<String, List<ProfileShardResult>> shardResult : resp.getProfileResults().entrySet()) {
-            for (ProfileShardResult searchProfiles : shardResult.getValue()) {
+        for (Map.Entry<String, ProfileShardResult> shardResult : resp.getProfileResults().entrySet()) {
+            for (QueryProfileShardResult searchProfiles : shardResult.getValue().getQueryProfileResults()) {
                 for (ProfileResult result : searchProfiles.getQueryResults()) {
                     assertNotNull(result.getQueryName());
                     assertNotNull(result.getLuceneDescription());
@@ -421,8 +422,8 @@ public class QueryProfilerIT extends ESIntegTestCase {
         assertNotNull("Profile response element should not be null", resp.getProfileResults());
         assertThat("Profile response should not be an empty array", resp.getProfileResults().size(), not(0));
 
-        for (Map.Entry<String, List<ProfileShardResult>> shardResult : resp.getProfileResults().entrySet()) {
-            for (ProfileShardResult searchProfiles : shardResult.getValue()) {
+        for (Map.Entry<String, ProfileShardResult> shardResult : resp.getProfileResults().entrySet()) {
+            for (QueryProfileShardResult searchProfiles : shardResult.getValue().getQueryProfileResults()) {
                 for (ProfileResult result : searchProfiles.getQueryResults()) {
                     assertNotNull(result.getQueryName());
                     assertNotNull(result.getLuceneDescription());
@@ -468,8 +469,8 @@ public class QueryProfilerIT extends ESIntegTestCase {
         assertNotNull("Profile response element should not be null", resp.getProfileResults());
         assertThat("Profile response should not be an empty array", resp.getProfileResults().size(), not(0));
 
-        for (Map.Entry<String, List<ProfileShardResult>> shardResult : resp.getProfileResults().entrySet()) {
-            for (ProfileShardResult searchProfiles : shardResult.getValue()) {
+        for (Map.Entry<String, ProfileShardResult> shardResult : resp.getProfileResults().entrySet()) {
+            for (QueryProfileShardResult searchProfiles : shardResult.getValue().getQueryProfileResults()) {
                 for (ProfileResult result : searchProfiles.getQueryResults()) {
                     assertNotNull(result.getQueryName());
                     assertNotNull(result.getLuceneDescription());
@@ -514,8 +515,8 @@ public class QueryProfilerIT extends ESIntegTestCase {
         assertNotNull("Profile response element should not be null", resp.getProfileResults());
         assertThat("Profile response should not be an empty array", resp.getProfileResults().size(), not(0));
 
-        for (Map.Entry<String, List<ProfileShardResult>> shardResult : resp.getProfileResults().entrySet()) {
-            for (ProfileShardResult searchProfiles : shardResult.getValue()) {
+        for (Map.Entry<String, ProfileShardResult> shardResult : resp.getProfileResults().entrySet()) {
+            for (QueryProfileShardResult searchProfiles : shardResult.getValue().getQueryProfileResults()) {
                 for (ProfileResult result : searchProfiles.getQueryResults()) {
                     assertNotNull(result.getQueryName());
                     assertNotNull(result.getLuceneDescription());
@@ -569,8 +570,8 @@ public class QueryProfilerIT extends ESIntegTestCase {
         assertNotNull("Profile response element should not be null", resp.getProfileResults());
         assertThat("Profile response should not be an empty array", resp.getProfileResults().size(), not(0));
 
-        for (Map.Entry<String, List<ProfileShardResult>> shardResult : resp.getProfileResults().entrySet()) {
-            for (ProfileShardResult searchProfiles : shardResult.getValue()) {
+        for (Map.Entry<String, ProfileShardResult> shardResult : resp.getProfileResults().entrySet()) {
+            for (QueryProfileShardResult searchProfiles : shardResult.getValue().getQueryProfileResults()) {
                 for (ProfileResult result : searchProfiles.getQueryResults()) {
                     assertNotNull(result.getQueryName());
                     assertNotNull(result.getLuceneDescription());
diff --git a/core/src/test/java/org/elasticsearch/search/profile/query/ProfileTests.java b/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java
similarity index 96%
rename from core/src/test/java/org/elasticsearch/search/profile/query/ProfileTests.java
rename to core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java
index 7488dbceee7..ffad39bc3f2 100644
--- a/core/src/test/java/org/elasticsearch/search/profile/query/ProfileTests.java
+++ b/core/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerTests.java
@@ -51,7 +51,7 @@ import java.util.Map;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 
-public class ProfileTests extends ESTestCase {
+public class QueryProfilerTests extends ESTestCase {
 
     static Directory dir;
     static IndexReader reader;
@@ -90,7 +90,7 @@ public class ProfileTests extends ESTestCase {
         searcher.setProfiler(profiler);
         Query query = new TermQuery(new Term("foo", "bar"));
         searcher.search(query, 1);
-        List<ProfileResult> results = profiler.getQueryTree();
+        List<ProfileResult> results = profiler.getTree();
         assertEquals(1, results.size());
         Map<String, Long> breakdown = results.get(0).getTimeBreakdown();
         assertThat(breakdown.get(QueryTimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L));
@@ -109,7 +109,7 @@ public class ProfileTests extends ESTestCase {
         searcher.setProfiler(profiler);
         Query query = new TermQuery(new Term("foo", "bar"));
         searcher.search(query, 1, Sort.INDEXORDER); // scores are not needed
-        List<ProfileResult> results = profiler.getQueryTree();
+        List<ProfileResult> results = profiler.getTree();
         assertEquals(1, results.size());
         Map<String, Long> breakdown = results.get(0).getTimeBreakdown();
         assertThat(breakdown.get(QueryTimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L));
@@ -128,7 +128,7 @@ public class ProfileTests extends ESTestCase {
         searcher.setProfiler(profiler);
         Query query = new TermQuery(new Term("foo", "bar"));
         searcher.count(query); // will use index stats
-        List<ProfileResult> results = profiler.getQueryTree();
+        List<ProfileResult> results = profiler.getTree();
         assertEquals(0, results.size());
 
         long rewriteTime = profiler.getRewriteTime();
@@ -144,7 +144,7 @@ public class ProfileTests extends ESTestCase {
         searcher.setProfiler(profiler);
         Query query = new RandomApproximationQuery(new TermQuery(new Term("foo", "bar")), random());
         searcher.count(query);
-        List<ProfileResult> results = profiler.getQueryTree();
+        List<ProfileResult> results = profiler.getTree();
         assertEquals(1, results.size());
         Map<String, Long> breakdown = results.get(0).getTimeBreakdown();
         assertThat(breakdown.get(QueryTimingType.CREATE_WEIGHT.toString()).longValue(), greaterThan(0L));

From a5e329e5634e2b6d438c8d9b21e21153c0aabd49 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 10 Jun 2016 10:11:21 +0200
Subject: [PATCH 0765/1311] [TEST] use animalsniffer annotation
 @IgnoreJRERequirement (similar to @SuppressForbidden) rather than exclusion
 pattern

---
 client-sniffer/build.gradle                                 | 6 +-----
 .../org/elasticsearch/client/sniff/HostsSnifferTests.java   | 3 +++
 client/build.gradle                                         | 6 +-----
 .../java/org/elasticsearch/client/RestClientIntegTests.java | 3 +++
 4 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/client-sniffer/build.gradle b/client-sniffer/build.gradle
index c772cfd084b..af04d85ac61 100644
--- a/client-sniffer/build.gradle
+++ b/client-sniffer/build.gradle
@@ -41,6 +41,7 @@ dependencies {
   testCompile "org.apache.lucene:lucene-core:${versions.lucene}"
   testCompile "org.apache.lucene:lucene-codecs:${versions.lucene}"
   testCompile "org.elasticsearch:securemock:${versions.securemock}"
+  testCompile "org.codehaus.mojo:animal-sniffer-annotations:1.15"
   signature "org.codehaus.mojo.signature:java17:1.0@signature"
 }
 
@@ -57,11 +58,6 @@ forbiddenApisTest {
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
 }
 
-animalsnifferTest {
-    //animalsniffer doesn't like sun classes usages (e.g. com.sun.net.httpserver.HttpServer)
-    exclude('org/elasticsearch/client/sniff/HostsSnifferTests.class')
-}
-
 //JarHell is part of es core, which we don't want to pull in
 jarHell.enabled=false
 //NamingConventionCheck is part of test-framework, which we don't want to pull in as it depends on es core
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
index 4ff11a24ea5..af1b878cfd4 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
@@ -32,6 +32,7 @@ import org.apache.http.HttpHost;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.SuppressForbidden;
+import org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
@@ -55,6 +56,7 @@ import java.util.Set;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
 
+@IgnoreJRERequirement
 @SuppressForbidden(reason = "uses sun HttpServer")
 public class HostsSnifferTests extends LuceneTestCase {
 
@@ -117,6 +119,7 @@ public class HostsSnifferTests extends LuceneTestCase {
         return httpServer;
     }
 
+    @IgnoreJRERequirement
     @SuppressForbidden(reason = "uses sun HttpServer")
     private static class ResponseHandler implements HttpHandler {
         private final int sniffTimeoutMillis;
diff --git a/client/build.gradle b/client/build.gradle
index 609e10e4abd..f905114fd0f 100644
--- a/client/build.gradle
+++ b/client/build.gradle
@@ -39,6 +39,7 @@ dependencies {
   testCompile "org.apache.lucene:lucene-core:${versions.lucene}"
   testCompile "org.apache.lucene:lucene-codecs:${versions.lucene}"
   testCompile "org.elasticsearch:securemock:${versions.securemock}"
+  testCompile "org.codehaus.mojo:animal-sniffer-annotations:1.15"
   signature "org.codehaus.mojo.signature:java17:1.0@signature"
 }
 
@@ -55,11 +56,6 @@ forbiddenApisTest {
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
 }
 
-animalsnifferTest {
-    //animalsniffer doesn't like sun classes usages (e.g. com.sun.net.httpserver.HttpServer)
-    exclude('org/elasticsearch/client/RestClientIntegTests.class')
-}
-
 //JarHell is part of es core, which we don't want to pull in
 jarHell.enabled=false
 //NamingConventionCheck is part of test-framework, which we don't want to pull in as it depends on es core
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java b/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
index 36a7e45b728..6ee2dc02844 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
@@ -33,6 +33,7 @@ import org.apache.http.message.BasicHeader;
 import org.apache.http.util.EntityUtils;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.SuppressForbidden;
+import org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
@@ -57,6 +58,7 @@ import static org.hamcrest.CoreMatchers.equalTo;
  * Integration test to check interaction between {@link RestClient} and {@link org.apache.http.client.HttpClient}.
  * Works against a real http server, one single host.
  */
+@IgnoreJRERequirement
 @SuppressForbidden(reason = "uses sun HttpServer")
 public class RestClientIntegTests extends LuceneTestCase {
 
@@ -87,6 +89,7 @@ public class RestClientIntegTests extends LuceneTestCase {
         httpServer.createContext("/" + statusCode, new ResponseHandler(statusCode));
     }
 
+    @IgnoreJRERequirement
     @SuppressForbidden(reason = "uses sun HttpServer")
     private static class ResponseHandler implements HttpHandler {
         private final int statusCode;

From 432efc75fbd46a86955d0d42fd4d38bfd799dbbd Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 10 Jun 2016 11:39:08 +0200
Subject: [PATCH 0766/1311] catch Exception rather than Throwable

---
 .../main/java/org/elasticsearch/client/sniff/Sniffer.java | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
index 37240741b3d..483b1498b57 100644
--- a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
+++ b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
@@ -96,8 +96,8 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
                     }
                     logger.debug("scheduling next sniff in " + delayMillis + " ms");
                     this.scheduledFuture = this.scheduledExecutorService.schedule(this, delayMillis, TimeUnit.MILLISECONDS);
-                } catch(Throwable t) {
-                    logger.error("error while scheduling next sniffer task", t);
+                } catch(Exception e) {
+                    logger.error("error while scheduling next sniffer task", e);
                 }
             }
         }
@@ -120,8 +120,8 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
                     }
                     logger.debug("sniffed nodes: " + sniffedNodes);
                     this.restClient.setHosts(sniffedNodes.toArray(new HttpHost[sniffedNodes.size()]));
-                } catch (Throwable t) {
-                    logger.error("error while sniffing nodes", t);
+                } catch (Exception e) {
+                    logger.error("error while sniffing nodes", e);
                 } finally {
                     scheduleNextRun(nextSniffDelayMillis);
                     running.set(false);

From 6db90da25e9c5c918116102662915b87cc464aec Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 10 Jun 2016 11:40:13 +0200
Subject: [PATCH 0767/1311] [TEST] Remove usage of @SuppressForbidden due to
 sun HttpServer usage, resolve some violations that were hidden by it

---
 client-sniffer/build.gradle                               | 3 +++
 .../org/elasticsearch/client/sniff/HostsSnifferTests.java | 8 +++-----
 client/build.gradle                                       | 3 +++
 .../org/elasticsearch/client/RestClientIntegTests.java    | 8 +++-----
 4 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/client-sniffer/build.gradle b/client-sniffer/build.gradle
index af04d85ac61..a8e4b684bf6 100644
--- a/client-sniffer/build.gradle
+++ b/client-sniffer/build.gradle
@@ -54,6 +54,9 @@ forbiddenApisMain {
 }
 
 forbiddenApisTest {
+  //we are excluding jdk-non-portable to allow for com.sun.net.httpserver.* usage
+  //TODO remove this line once https://github.com/policeman-tools/forbidden-apis/issues/103 gets solved
+  bundledSignatures = ['jdk-unsafe', 'jdk-deprecated', 'jdk-system-out']
   //client does not depend on core, so only jdk signatures should be checked
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
 }
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
index af1b878cfd4..06f8fd6940e 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
@@ -31,7 +31,6 @@ import org.apache.http.Consts;
 import org.apache.http.HttpHost;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.SuppressForbidden;
 import org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
@@ -42,6 +41,7 @@ import org.junit.Before;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.StringWriter;
+import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
@@ -57,7 +57,6 @@ import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
 
 @IgnoreJRERequirement
-@SuppressForbidden(reason = "uses sun HttpServer")
 public class HostsSnifferTests extends LuceneTestCase {
 
     private int sniffRequestTimeout;
@@ -84,7 +83,7 @@ public class HostsSnifferTests extends LuceneTestCase {
     }
 
     public void testSniffNodes() throws IOException, URISyntaxException {
-        HttpHost httpHost = new HttpHost(httpServer.getAddress().getHostName(), httpServer.getAddress().getPort());
+        HttpHost httpHost = new HttpHost(httpServer.getAddress().getHostString(), httpServer.getAddress().getPort());
         try (RestClient restClient = RestClient.builder(httpHost).build()) {
             HostsSniffer sniffer = new HostsSniffer(restClient, sniffRequestTimeout, scheme);
             try {
@@ -114,13 +113,12 @@ public class HostsSnifferTests extends LuceneTestCase {
     }
 
     private static HttpServer createHttpServer(final SniffResponse sniffResponse, final int sniffTimeoutMillis) throws IOException {
-        HttpServer httpServer = HttpServer.create(new InetSocketAddress(0), 0);
+        HttpServer httpServer = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
         httpServer.createContext("/_nodes/http", new ResponseHandler(sniffTimeoutMillis, sniffResponse));
         return httpServer;
     }
 
     @IgnoreJRERequirement
-    @SuppressForbidden(reason = "uses sun HttpServer")
     private static class ResponseHandler implements HttpHandler {
         private final int sniffTimeoutMillis;
         private final SniffResponse sniffResponse;
diff --git a/client/build.gradle b/client/build.gradle
index f905114fd0f..c2479b31d1e 100644
--- a/client/build.gradle
+++ b/client/build.gradle
@@ -52,6 +52,9 @@ forbiddenApisMain {
 }
 
 forbiddenApisTest {
+  //we are excluding jdk-non-portable to allow for com.sun.net.httpserver.* usage
+  //TODO remove this line once https://github.com/policeman-tools/forbidden-apis/issues/103 gets solved
+  bundledSignatures = ['jdk-unsafe', 'jdk-deprecated', 'jdk-system-out']
   //client does not depend on core, so only jdk signatures should be checked
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
 }
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java b/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
index 6ee2dc02844..7c4be56040b 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
@@ -32,7 +32,6 @@ import org.apache.http.entity.StringEntity;
 import org.apache.http.message.BasicHeader;
 import org.apache.http.util.EntityUtils;
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.SuppressForbidden;
 import org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -40,6 +39,7 @@ import org.junit.BeforeClass;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
+import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.util.Arrays;
 import java.util.Collections;
@@ -59,7 +59,6 @@ import static org.hamcrest.CoreMatchers.equalTo;
  * Works against a real http server, one single host.
  */
 @IgnoreJRERequirement
-@SuppressForbidden(reason = "uses sun HttpServer")
 public class RestClientIntegTests extends LuceneTestCase {
 
     private static HttpServer httpServer;
@@ -68,7 +67,7 @@ public class RestClientIntegTests extends LuceneTestCase {
 
     @BeforeClass
     public static void startHttpServer() throws Exception {
-        httpServer = HttpServer.create(new InetSocketAddress(0), 0);
+        httpServer = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
         httpServer.start();
         //returns a different status code depending on the path
         for (int statusCode : getAllStatusCodes()) {
@@ -81,7 +80,7 @@ public class RestClientIntegTests extends LuceneTestCase {
             String headerValue = RandomStrings.randomAsciiOfLengthBetween(random(), 3, 10);
             defaultHeaders[i] = new BasicHeader(headerName, headerValue);
         }
-        restClient = RestClient.builder(new HttpHost(httpServer.getAddress().getHostName(), httpServer.getAddress().getPort()))
+        restClient = RestClient.builder(new HttpHost(httpServer.getAddress().getHostString(), httpServer.getAddress().getPort()))
                 .setDefaultHeaders(defaultHeaders).build();
     }
 
@@ -90,7 +89,6 @@ public class RestClientIntegTests extends LuceneTestCase {
     }
 
     @IgnoreJRERequirement
-    @SuppressForbidden(reason = "uses sun HttpServer")
     private static class ResponseHandler implements HttpHandler {
         private final int statusCode;
 

From 3d7186c81fe260d54dacc12bb3573a6c28ce2155 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 10 Jun 2016 11:40:35 +0200
Subject: [PATCH 0768/1311] make DeadHostState final

---
 .../src/main/java/org/elasticsearch/client/DeadHostState.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/src/main/java/org/elasticsearch/client/DeadHostState.java b/client/src/main/java/org/elasticsearch/client/DeadHostState.java
index 89a1720d46e..44b7532d5de 100644
--- a/client/src/main/java/org/elasticsearch/client/DeadHostState.java
+++ b/client/src/main/java/org/elasticsearch/client/DeadHostState.java
@@ -26,7 +26,7 @@ import java.util.concurrent.TimeUnit;
  * when the host should be retried (based on number of previous failed attempts).
  * Class is immutable, a new copy of it should be created each time the state has to be changed.
  */
-class DeadHostState {
+final class DeadHostState {
 
     private static final long MIN_CONNECTION_TIMEOUT_NANOS = TimeUnit.MINUTES.toNanos(1);
     private static final long MAX_CONNECTION_TIMEOUT_NANOS = TimeUnit.MINUTES.toNanos(30);

From 656422cff62600d166dfbf4962dd9400ae50e7c9 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 10 Jun 2016 11:42:25 +0200
Subject: [PATCH 0769/1311] move shutdownNow within try block

---
 .../src/main/java/org/elasticsearch/client/sniff/Sniffer.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
index 483b1498b57..c1aeb3efe36 100644
--- a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
+++ b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
@@ -135,10 +135,10 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
                 if (scheduledExecutorService.awaitTermination(1000, TimeUnit.MILLISECONDS)) {
                     return;
                 }
+                scheduledExecutorService.shutdownNow();
             } catch (InterruptedException e) {
                 Thread.currentThread().interrupt();
             }
-            scheduledExecutorService.shutdownNow();
         }
     }
 

From 777d438f4861dbbf50cc191d58c4688ecd72324b Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 10 Jun 2016 12:41:49 +0200
Subject: [PATCH 0770/1311] [TEST] use jdk-internal bundled signature (rather
 than the previously removed jdk-non-portable)

---
 client-sniffer/build.gradle | 6 +++---
 client/build.gradle         | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/client-sniffer/build.gradle b/client-sniffer/build.gradle
index a8e4b684bf6..a29f07f1c42 100644
--- a/client-sniffer/build.gradle
+++ b/client-sniffer/build.gradle
@@ -54,9 +54,9 @@ forbiddenApisMain {
 }
 
 forbiddenApisTest {
-  //we are excluding jdk-non-portable to allow for com.sun.net.httpserver.* usage
-  //TODO remove this line once https://github.com/policeman-tools/forbidden-apis/issues/103 gets solved
-  bundledSignatures = ['jdk-unsafe', 'jdk-deprecated', 'jdk-system-out']
+  //we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
+  bundledSignatures -= 'jdk-non-portable'
+  bundledSignatures += 'jdk-internal'
   //client does not depend on core, so only jdk signatures should be checked
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
 }
diff --git a/client/build.gradle b/client/build.gradle
index c2479b31d1e..7a685fabfaf 100644
--- a/client/build.gradle
+++ b/client/build.gradle
@@ -52,9 +52,9 @@ forbiddenApisMain {
 }
 
 forbiddenApisTest {
-  //we are excluding jdk-non-portable to allow for com.sun.net.httpserver.* usage
-  //TODO remove this line once https://github.com/policeman-tools/forbidden-apis/issues/103 gets solved
-  bundledSignatures = ['jdk-unsafe', 'jdk-deprecated', 'jdk-system-out']
+  //we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
+  bundledSignatures -= 'jdk-non-portable'
+  bundledSignatures += 'jdk-internal'
   //client does not depend on core, so only jdk signatures should be checked
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
 }

From 6ea89004cded88ca4d0ab9cd96c7c44057e47b4f Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Fri, 10 Jun 2016 12:47:41 +0200
Subject: [PATCH 0771/1311] Make IndicesClusterStateService unit testable
 (#17270)

Testability of ICSS is achieved by introducing interfaces for IndicesService, IndexService and IndexShard. These interfaces extract all relevant methods used by ICSS (which do not deal directly with store) and give the possibility to easily mock all the store behavior away in the tests (and cuts down on dependencies).
---
 .../resources/checkstyle_suppressions.xml     |   1 -
 .../cluster/ClusterChangedEvent.java          |  13 +-
 .../index/NodeMappingRefreshAction.java       |   8 +-
 .../org/elasticsearch/index/IndexService.java |  14 +-
 .../elasticsearch/index/IndexSettings.java    |   2 +-
 .../index/mapper/MapperService.java           |  43 +
 .../elasticsearch/index/shard/IndexShard.java | 119 +--
 .../index/shard/ShadowIndexShard.java         |   6 +-
 .../elasticsearch/indices/IndicesService.java |  78 +-
 .../cluster/IndicesClusterStateService.java   | 890 ++++++++++--------
 .../TransportMasterNodeActionUtils.java       |  38 +
 .../cluster/ClusterChangedEventTests.java     |  14 +-
 .../RandomAllocationDeciderTests.java         |   2 +-
 .../index/shard/IndexShardTests.java          |  56 +-
 .../IndexingMemoryControllerTests.java        |   2 +-
 ...dicesLifecycleListenerSingleNodeTests.java |   4 +-
 ...actIndicesClusterStateServiceTestCase.java | 318 +++++++
 .../indices/cluster/ClusterStateChanges.java  | 234 +++++
 ...ClusterStateServiceRandomUpdatesTests.java | 281 ++++++
 19 files changed, 1556 insertions(+), 567 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionUtils.java
 create mode 100644 core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java
 create mode 100644 core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
 create mode 100644 core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 05ca294a9f0..d56bdeb537f 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -482,7 +482,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]analysis[/\\]PreBuiltCacheFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]analysis[/\\]PreBuiltTokenFilters.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]breaker[/\\]HierarchyCircuitBreakerService.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]cluster[/\\]IndicesClusterStateService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]fielddata[/\\]cache[/\\]IndicesFieldDataCache.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]fielddata[/\\]cache[/\\]IndicesFieldDataCacheListener.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]flush[/\\]ShardsSyncedFlushResult.java" checks="LineLength" />
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java b/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java
index d3a42a97ebb..efd525d313b 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterChangedEvent.java
@@ -148,18 +148,11 @@ public class ClusterChangedEvent {
      * has changed between the previous cluster state and the new cluster state.
      * Note that this is an object reference equality test, not an equals test.
      */
-    public boolean indexMetaDataChanged(IndexMetaData current) {
-        MetaData previousMetaData = previousState.metaData();
-        if (previousMetaData == null) {
-            return true;
-        }
-        IndexMetaData previousIndexMetaData = previousMetaData.index(current.getIndex());
+    public static boolean indexMetaDataChanged(IndexMetaData metaData1, IndexMetaData metaData2) {
+        assert metaData1 != null && metaData2 != null;
         // no need to check on version, since disco modules will make sure to use the
         // same instance if its a version match
-        if (previousIndexMetaData == current) {
-            return false;
-        }
-        return true;
+        return metaData1 != metaData2;
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java b/core/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java
index 0645accb42a..b1bf01018c9 100644
--- a/core/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java
+++ b/core/src/main/java/org/elasticsearch/cluster/action/index/NodeMappingRefreshAction.java
@@ -24,6 +24,7 @@ import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaDataMappingService;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
@@ -58,13 +59,12 @@ public class NodeMappingRefreshAction extends AbstractComponent {
         transportService.registerRequestHandler(ACTION_NAME, NodeMappingRefreshRequest::new, ThreadPool.Names.SAME, new NodeMappingRefreshTransportHandler());
     }
 
-    public void nodeMappingRefresh(final ClusterState state, final NodeMappingRefreshRequest request) {
-        final DiscoveryNodes nodes = state.nodes();
-        if (nodes.getMasterNode() == null) {
+    public void nodeMappingRefresh(final DiscoveryNode masterNode, final NodeMappingRefreshRequest request) {
+        if (masterNode == null) {
             logger.warn("can't send mapping refresh for [{}], no master known.", request.index());
             return;
         }
-        transportService.sendRequest(nodes.getMasterNode(), ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME);
+        transportService.sendRequest(masterNode, ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME);
     }
 
     private class NodeMappingRefreshTransportHandler implements TransportRequestHandler<NodeMappingRefreshRequest> {
diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java
index 5780dc256a5..1902f24c4b7 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexService.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexService.java
@@ -67,6 +67,7 @@ import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.indices.AliasFilterParsingException;
 import org.elasticsearch.indices.InvalidAliasNameException;
+import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.mapper.MapperRegistry;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -93,7 +94,7 @@ import static org.elasticsearch.common.collect.MapBuilder.newMapBuilder;
 /**
  *
  */
-public final class IndexService extends AbstractIndexComponent implements IndexComponent, Iterable<IndexShard> {
+public class IndexService extends AbstractIndexComponent implements IndicesClusterStateService.AllocatedIndex<IndexShard> {
 
     private final IndexEventListener eventListener;
     private final AnalysisService analysisService;
@@ -184,8 +185,8 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
     /**
      * Return the shard with the provided id, or null if there is no such shard.
      */
-    @Nullable
-    public IndexShard getShardOrNull(int shardId) {
+    @Override
+    public @Nullable IndexShard getShardOrNull(int shardId) {
         return shards.get(shardId);
     }
 
@@ -359,6 +360,7 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
         return primary == false && IndexMetaData.isIndexUsingShadowReplicas(indexSettings);
     }
 
+    @Override
     public synchronized void removeShard(int shardId, String reason) {
         final ShardId sId = new ShardId(index(), shardId);
         final IndexShard indexShard;
@@ -470,6 +472,11 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
         return searchOperationListeners;
     }
 
+    @Override
+    public boolean updateMapping(IndexMetaData indexMetaData) throws IOException {
+        return mapperService().updateMapping(indexMetaData);
+    }
+
     private class StoreCloseListener implements Store.OnClose {
         private final ShardId shardId;
         private final boolean ownsShard;
@@ -617,6 +624,7 @@ public final class IndexService extends AbstractIndexComponent implements IndexC
         return indexSettings.getIndexMetaData();
     }
 
+    @Override
     public synchronized void updateMetaData(final IndexMetaData metadata) {
         final Translog.Durability oldTranslogDurability = indexSettings.getTranslogDurability();
         if (indexSettings.updateIndexMetaData(metadata)) {
diff --git a/core/src/main/java/org/elasticsearch/index/IndexSettings.java b/core/src/main/java/org/elasticsearch/index/IndexSettings.java
index 6bd6df414af..2c20697d757 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexSettings.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexSettings.java
@@ -404,7 +404,7 @@ public final class IndexSettings {
      *
      * @return <code>true</code> iff any setting has been updated otherwise <code>false</code>.
      */
-    synchronized boolean updateIndexMetaData(IndexMetaData indexMetaData) {
+    public synchronized boolean updateIndexMetaData(IndexMetaData indexMetaData) {
         final Settings newSettings = indexMetaData.getSettings();
         if (version.equals(Version.indexCreated(newSettings)) == false) {
             throw new IllegalArgumentException("version mismatch on settings update expected: " + version + " but was: " + Version.indexCreated(newSettings));
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
index e49846ba6b2..34a0ead0596 100755
--- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
@@ -21,10 +21,13 @@ package org.elasticsearch.index.mapper;
 
 import com.carrotsearch.hppc.ObjectHashSet;
 
+import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;
 import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Setting;
@@ -32,6 +35,7 @@ import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.AbstractIndexComponent;
+import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.mapper.Mapper.BuilderContext;
@@ -183,6 +187,45 @@ public class MapperService extends AbstractIndexComponent {
         }
     }
 
+    public boolean updateMapping(IndexMetaData indexMetaData) throws IOException {
+        assert indexMetaData.getIndex().equals(index()) : "index mismatch: expected " + index() + " but was " + indexMetaData.getIndex();
+        // go over and add the relevant mappings (or update them)
+        boolean requireRefresh = false;
+        for (ObjectCursor<MappingMetaData> cursor : indexMetaData.getMappings().values()) {
+            MappingMetaData mappingMd = cursor.value;
+            String mappingType = mappingMd.type();
+            CompressedXContent mappingSource = mappingMd.source();
+            // refresh mapping can happen when the parsing/merging of the mapping from the metadata doesn't result in the same
+            // mapping, in this case, we send to the master to refresh its own version of the mappings (to conform with the
+            // merge version of it, which it does when refreshing the mappings), and warn log it.
+            try {
+                DocumentMapper existingMapper = documentMapper(mappingType);
+
+                if (existingMapper == null || mappingSource.equals(existingMapper.mappingSource()) == false) {
+                    String op = existingMapper == null ? "adding" : "updating";
+                    if (logger.isDebugEnabled() && mappingSource.compressed().length < 512) {
+                        logger.debug("[{}] {} mapping [{}], source [{}]", index(), op, mappingType, mappingSource.string());
+                    } else if (logger.isTraceEnabled()) {
+                        logger.trace("[{}] {} mapping [{}], source [{}]", index(), op, mappingType, mappingSource.string());
+                    } else {
+                        logger.debug("[{}] {} mapping [{}] (source suppressed due to length, use TRACE level if needed)", index(), op,
+                            mappingType);
+                    }
+                    merge(mappingType, mappingSource, MergeReason.MAPPING_RECOVERY, true);
+                    if (!documentMapper(mappingType).mappingSource().equals(mappingSource)) {
+                        logger.debug("[{}] parsed mapping [{}], and got different sources\noriginal:\n{}\nparsed:\n{}", index(),
+                            mappingType, mappingSource, documentMapper(mappingType).mappingSource());
+                        requireRefresh = true;
+                    }
+                }
+            } catch (Throwable e) {
+                logger.warn("[{}] failed to add mapping [{}], source [{}]", e, index(), mappingType, mappingSource);
+                throw e;
+            }
+        }
+        return requireRefresh;
+    }
+
     //TODO: make this atomic
     public void merge(Map<String, Map<String, Object>> mappings, boolean updateAllTypes) throws MapperParsingException {
         // first, add the default mapping
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 105d1e5b057..16ebd97d8c3 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -37,8 +37,6 @@ import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequest;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RestoreSource;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.common.Booleans;
@@ -108,6 +106,7 @@ import org.elasticsearch.index.warmer.ShardIndexWarmerService;
 import org.elasticsearch.index.warmer.WarmerStats;
 import org.elasticsearch.indices.IndexingMemoryController;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.recovery.RecoveryFailedException;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.indices.recovery.RecoveryTargetService;
@@ -136,7 +135,7 @@ import java.util.function.Consumer;
 import java.util.function.BiConsumer;
 import java.util.stream.Collectors;
 
-public class IndexShard extends AbstractIndexShardComponent {
+public class IndexShard extends AbstractIndexShardComponent implements IndicesClusterStateService.Shard {
 
     private final ThreadPool threadPool;
     private final MapperService mapperService;
@@ -338,6 +337,7 @@ public class IndexShard extends AbstractIndexShardComponent {
     /**
      * Returns the latest cluster routing entry received with this shard.
      */
+    @Override
     public ShardRouting routingEntry() {
         return this.shardRouting;
     }
@@ -348,13 +348,12 @@ public class IndexShard extends AbstractIndexShardComponent {
 
     /**
      * Updates the shards routing entry. This mutate the shards internal state depending
-     * on the changes that get introduced by the new routing value. This method will persist shard level metadata
-     * unless explicitly disabled.
+     * on the changes that get introduced by the new routing value. This method will persist shard level metadata.
      *
      * @throws IndexShardRelocatedException if shard is marked as relocated and relocation aborted
      * @throws IOException                  if shard state could not be persisted
      */
-    public void updateRoutingEntry(final ShardRouting newRouting, final boolean persistState) throws IOException {
+    public void updateRoutingEntry(final ShardRouting newRouting) throws IOException {
         final ShardRouting currentRouting = this.shardRouting;
         if (!newRouting.shardId().equals(shardId())) {
             throw new IllegalArgumentException("Trying to set a routing entry with shardId " + newRouting.shardId() + " on a shard with shardId " + shardId() + "");
@@ -408,9 +407,7 @@ public class IndexShard extends AbstractIndexShardComponent {
         }
         this.shardRouting = newRouting;
         indexEventListener.shardRoutingChanged(this, currentRouting, newRouting);
-        if (persistState) {
-            persistMetadata(newRouting, currentRouting);
-        }
+        persistMetadata(newRouting, currentRouting);
     }
 
     /**
@@ -589,7 +586,7 @@ public class IndexShard extends AbstractIndexShardComponent {
      */
     public void refresh(String source) {
         verifyNotClosed();
-        
+
         if (canIndex()) {
             long bytes = getEngine().getIndexBufferRAMBytesUsed();
             writingBytes.addAndGet(bytes);
@@ -1370,35 +1367,36 @@ public class IndexShard extends AbstractIndexShardComponent {
         return this.currentEngineReference.get();
     }
 
-    public void startRecovery(DiscoveryNode localNode, DiscoveryNode sourceNode, RecoveryTargetService recoveryTargetService,
+    public void startRecovery(RecoveryState recoveryState, RecoveryTargetService recoveryTargetService,
                               RecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService,
-                              BiConsumer<String, MappingMetaData> mappingUpdateConsumer, IndicesService indicesService) {
-        final RestoreSource restoreSource = shardRouting.restoreSource();
-
-        if (shardRouting.isPeerRecovery()) {
-            assert sourceNode != null : "peer recovery started but sourceNode is null";
-            // we don't mark this one as relocated at the end.
-            // For primaries: requests in any case are routed to both when its relocating and that way we handle
-            //    the edge case where its mark as relocated, and we might need to roll it back...
-            // For replicas: we are recovering a backup from a primary
-            RecoveryState.Type type = shardRouting.primary() ? RecoveryState.Type.PRIMARY_RELOCATION : RecoveryState.Type.REPLICA;
-            RecoveryState recoveryState = new RecoveryState(shardId(), shardRouting.primary(), type, sourceNode, localNode);
-            try {
-                markAsRecovering("from " + sourceNode, recoveryState);
-                recoveryTargetService.startRecovery(this, type, sourceNode, recoveryListener);
-            } catch (Throwable e) {
-                failShard("corrupted preexisting index", e);
-                recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(shardId, sourceNode, localNode, e), true);
-            }
-        } else if (restoreSource == null) {
-            // recover from filesystem store
-
-            IndexMetaData indexMetaData = indexSettings().getIndexMetaData();
-            Index mergeSourceIndex = indexMetaData.getMergeSourceIndex();
-            final boolean recoverFromLocalShards = mergeSourceIndex != null && shardRouting.allocatedPostIndexCreate(indexMetaData) == false && shardRouting.primary();
-            final RecoveryState recoveryState = new RecoveryState(shardId(), shardRouting.primary(),
-                recoverFromLocalShards ? RecoveryState.Type.LOCAL_SHARDS : RecoveryState.Type.STORE, localNode, localNode);
-            if (recoverFromLocalShards) {
+                              BiConsumer<String, MappingMetaData> mappingUpdateConsumer,
+                              IndicesService indicesService) {
+        switch (recoveryState.getType()) {
+            case PRIMARY_RELOCATION:
+            case REPLICA:
+                try {
+                    markAsRecovering("from " + recoveryState.getSourceNode(), recoveryState);
+                    recoveryTargetService.startRecovery(this, recoveryState.getType(), recoveryState.getSourceNode(), recoveryListener);
+                } catch (Throwable e) {
+                    failShard("corrupted preexisting index", e);
+                    recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, e), true);
+                }
+                break;
+            case STORE:
+                markAsRecovering("from store", recoveryState); // mark the shard as recovering on the cluster state thread
+                threadPool.generic().execute(() -> {
+                    try {
+                        if (recoverFromStore()) {
+                            recoveryListener.onRecoveryDone(recoveryState);
+                        }
+                    } catch (Throwable t) {
+                        recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, t), true);
+                    }
+                });
+                break;
+            case LOCAL_SHARDS:
+                final IndexMetaData indexMetaData = indexSettings().getIndexMetaData();
+                final Index mergeSourceIndex = indexMetaData.getMergeSourceIndex();
                 final List<IndexShard> startedShards = new ArrayList<>();
                 final IndexService sourceIndexService = indicesService.indexService(mergeSourceIndex);
                 final int numShards = sourceIndexService != null ? sourceIndexService.getIndexSettings().getNumberOfShards() : -1;
@@ -1414,14 +1412,14 @@ public class IndexShard extends AbstractIndexShardComponent {
                     threadPool.generic().execute(() -> {
                         try {
                             final Set<ShardId> shards = IndexMetaData.selectShrinkShards(shardId().id(), sourceIndexService.getMetaData(),
-                                indexMetaData.getNumberOfShards());
+                                +                                indexMetaData.getNumberOfShards());
                             if (recoverFromLocalShards(mappingUpdateConsumer, startedShards.stream()
                                 .filter((s) -> shards.contains(s.shardId())).collect(Collectors.toList()))) {
                                 recoveryListener.onRecoveryDone(recoveryState);
                             }
                         } catch (Throwable t) {
                             recoveryListener.onRecoveryFailure(recoveryState,
-                                new RecoveryFailedException(shardId, localNode, localNode, t), true);
+                                new RecoveryFailedException(recoveryState, null, t), true);
                         }
                     });
                 } else {
@@ -1433,36 +1431,25 @@ public class IndexShard extends AbstractIndexShardComponent {
                             + " are started yet, expected " + numShards + " found " + startedShards.size() + " can't recover shard "
                             + shardId());
                     }
-                    recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(shardId, localNode, localNode, t), true);
+                    recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, t), true);
                 }
-            } else {
-                markAsRecovering("from store", recoveryState); // mark the shard as recovering on the cluster state thread
+                break;
+            case SNAPSHOT:
+                markAsRecovering("from snapshot", recoveryState); // mark the shard as recovering on the cluster state thread
                 threadPool.generic().execute(() -> {
                     try {
-                        if (recoverFromStore()) {
+                        final IndexShardRepository indexShardRepository = repositoriesService.indexShardRepository(
+                            recoveryState.getRestoreSource().snapshot().getRepository());
+                        if (restoreFromRepository(indexShardRepository)) {
                             recoveryListener.onRecoveryDone(recoveryState);
                         }
-                    } catch (Throwable t) {
-                        recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(shardId, sourceNode, localNode, t), true);
+                    } catch (Throwable first) {
+                        recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, first), true);
                     }
-
                 });
-            }
-        } else {
-            // recover from a restore
-            final RecoveryState recoveryState = new RecoveryState(shardId(), shardRouting.primary(),
-                RecoveryState.Type.SNAPSHOT, shardRouting.restoreSource(), localNode);
-            markAsRecovering("from snapshot", recoveryState); // mark the shard as recovering on the cluster state thread
-            threadPool.generic().execute(() -> {
-                try {
-                    final IndexShardRepository indexShardRepository = repositoriesService.indexShardRepository(restoreSource.snapshot().getRepository());
-                    if (restoreFromRepository(indexShardRepository)) {
-                        recoveryListener.onRecoveryDone(recoveryState);
-                    }
-                } catch (Throwable first) {
-                    recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(shardId, sourceNode, localNode, first), true);
-                }
-            });
+                break;
+            default:
+                throw new IllegalArgumentException("Unknown recovery type " + recoveryState.getType());
         }
     }
 
@@ -1472,7 +1459,7 @@ public class IndexShard extends AbstractIndexShardComponent {
         // called by the current engine
         @Override
         public void onFailedEngine(String reason, @Nullable Throwable failure) {
-            final ShardFailure shardFailure = new ShardFailure(shardRouting, reason, failure, getIndexUUID());
+            final ShardFailure shardFailure = new ShardFailure(shardRouting, reason, failure);
             for (Callback<ShardFailure> listener : delegates) {
                 try {
                     listener.handle(shardFailure);
@@ -1661,13 +1648,11 @@ public class IndexShard extends AbstractIndexShardComponent {
         public final String reason;
         @Nullable
         public final Throwable cause;
-        public final String indexUUID;
 
-        public ShardFailure(ShardRouting routing, String reason, @Nullable Throwable cause, String indexUUID) {
+        public ShardFailure(ShardRouting routing, String reason, @Nullable Throwable cause) {
             this.routing = routing;
             this.reason = reason;
             this.cause = cause;
-            this.indexUUID = indexUUID;
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java
index e22f684637e..e35c95ae1f0 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java
@@ -59,16 +59,16 @@ public final class ShadowIndexShard extends IndexShard {
 
     /**
      * In addition to the regular accounting done in
-     * {@link IndexShard#updateRoutingEntry(ShardRouting, boolean)},
+     * {@link IndexShard#updateRoutingEntry(ShardRouting)},
      * if this shadow replica needs to be promoted to a primary, the shard is
      * failed in order to allow a new primary to be re-allocated.
      */
     @Override
-    public void updateRoutingEntry(ShardRouting newRouting, boolean persistState) throws IOException {
+    public void updateRoutingEntry(ShardRouting newRouting) throws IOException {
         if (newRouting.primary() == true) {// becoming a primary
             throw new IllegalStateException("can't promote shard to primary");
         }
-        super.updateRoutingEntry(newRouting, persistState);
+        super.updateRoutingEntry(newRouting);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index ba512379868..3ae02c7eadd 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -35,6 +35,7 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.breaker.CircuitBreaker;
@@ -55,6 +56,7 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.Callback;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.iterable.Iterables;
 import org.elasticsearch.env.NodeEnvironment;
@@ -86,10 +88,14 @@ import org.elasticsearch.index.shard.IndexingStats;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.store.IndexStoreConfig;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.mapper.MapperRegistry;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.indices.recovery.RecoveryState;
+import org.elasticsearch.indices.recovery.RecoveryTargetService;
 import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.query.QueryPhase;
@@ -124,7 +130,8 @@ import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList;
 /**
  *
  */
-public class IndicesService extends AbstractLifecycleComponent<IndicesService> implements Iterable<IndexService>, IndexService.ShardStoreDeleter {
+public class IndicesService extends AbstractLifecycleComponent<IndicesService>
+    implements IndicesClusterStateService.AllocatedIndices<IndexShard, IndexService>, IndexService.ShardStoreDeleter {
 
     public static final String INDICES_SHARDS_CLOSED_TIMEOUT = "indices.shards_closed_timeout";
     public static final Setting<TimeValue> INDICES_CACHE_CLEAN_INTERVAL_SETTING =
@@ -296,11 +303,14 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
     }
 
     /**
-     * Returns <tt>true</tt> if changes (adding / removing) indices, shards and so on are allowed.
+     * Checks if changes (adding / removing) indices, shards and so on are allowed.
+     *
+     * @throws IllegalStateException if no changes allowed.
      */
-    public boolean changesAllowed() {
-        // we check on stop here since we defined stop when we delete the indices
-        return lifecycle.started();
+    private void ensureChangesAllowed() {
+        if (lifecycle.started() == false) {
+            throw new IllegalStateException("Can't make changes to indices service, node is closed");
+        }
     }
 
     @Override
@@ -314,10 +324,9 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
 
     /**
      * Returns an IndexService for the specified index if exists otherwise returns <code>null</code>.
-     *
      */
-    @Nullable
-    public IndexService indexService(Index index) {
+    @Override
+    public @Nullable IndexService indexService(Index index) {
         return indices.get(index.getUUID());
     }
 
@@ -339,11 +348,9 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
      * @param builtInListeners a list of built-in lifecycle {@link IndexEventListener} that should should be used along side with the per-index listeners
      * @throws IndexAlreadyExistsException if the index already exists.
      */
+    @Override
     public synchronized IndexService createIndex(final NodeServicesProvider nodeServicesProvider, IndexMetaData indexMetaData, List<IndexEventListener> builtInListeners) throws IOException {
-
-        if (!lifecycle.started()) {
-            throw new IllegalStateException("Can't create an index [" + indexMetaData.getIndex() + "], node is closed");
-        }
+        ensureChangesAllowed();
         if (indexMetaData.getIndexUUID().equals(IndexMetaData.INDEX_UUID_NA_VALUE)) {
             throw new IllegalArgumentException("index must have a real UUID found value: [" + indexMetaData.getIndexUUID() + "]");
         }
@@ -424,14 +431,44 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
         }
     }
 
+    @Override
+    public IndexShard createShard(ShardRouting shardRouting, RecoveryState recoveryState, RecoveryTargetService recoveryTargetService,
+                RecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService,
+                NodeServicesProvider nodeServicesProvider, Callback<IndexShard.ShardFailure> onShardFailure) throws IOException {
+        ensureChangesAllowed();
+        IndexService indexService = indexService(shardRouting.index());
+        IndexShard indexShard = indexService.createShard(shardRouting);
+        indexShard.addShardFailureCallback(onShardFailure);
+        indexShard.startRecovery(recoveryState, recoveryTargetService, recoveryListener, repositoriesService,
+            (type, mapping) -> {
+                assert recoveryState.getType() == RecoveryState.Type.LOCAL_SHARDS :
+                    "mapping update consumer only required by local shards recovery";
+                try {
+                    nodeServicesProvider.getClient().admin().indices().preparePutMapping()
+                        .setConcreteIndex(shardRouting.index()) // concrete index - no name clash, it uses uuid
+                        .setType(type)
+                        .setSource(mapping.source().string())
+                        .get();
+                } catch (IOException ex) {
+                    throw new ElasticsearchException("failed to stringify mapping source", ex);
+                }
+            }, this);
+        return indexShard;
+    }
+
     /**
      * Removes the given index from this service and releases all associated resources. Persistent parts of the index
      * like the shards files, state and transaction logs are kept around in the case of a disaster recovery.
      * @param index the index to remove
      * @param reason  the high level reason causing this removal
      */
+    @Override
     public void removeIndex(Index index, String reason) {
-        removeIndex(index, reason, false);
+        try {
+            removeIndex(index, reason, false);
+        } catch (Throwable e) {
+            logger.warn("failed to remove index ({})", e, reason);
+        }
     }
 
     private void removeIndex(Index index, String reason, boolean delete) {
@@ -516,14 +553,20 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
      * @param index the index to delete
      * @param reason the high level reason causing this delete
      */
-    public void deleteIndex(Index index, String reason) throws IOException {
-        removeIndex(index, reason, true);
+    @Override
+    public void deleteIndex(Index index, String reason) {
+        try {
+            removeIndex(index, reason, true);
+        } catch (Throwable e) {
+            logger.warn("failed to delete index ({})", e, reason);
+        }
     }
 
     /**
      * Deletes an index that is not assigned to this node. This method cleans up all disk folders relating to the index
      * but does not deal with in-memory structures. For those call {@link #deleteIndex(Index, String)}
      */
+    @Override
     public void deleteUnassignedIndex(String reason, IndexMetaData metaData, ClusterState clusterState) {
         if (nodeEnv.hasNodeFile()) {
             String indexName = metaData.getIndex().getName();
@@ -683,8 +726,8 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
      * @param clusterState {@code ClusterState} to ensure the index is not part of it
      * @return IndexMetaData for the index loaded from disk
      */
-    @Nullable
-    public IndexMetaData verifyIndexIsDeleted(final Index index, final ClusterState clusterState) {
+    @Override
+    public @Nullable IndexMetaData verifyIndexIsDeleted(final Index index, final ClusterState clusterState) {
         // this method should only be called when we know the index (name + uuid) is not part of the cluster state
         if (clusterState.metaData().index(index) != null) {
             throw new IllegalStateException("Cannot delete index [" + index + "], it is still part of the cluster state.");
@@ -839,6 +882,7 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService> i
      * @param index the index to process the pending deletes for
      * @param timeout the timeout used for processing pending deletes
      */
+    @Override
     public void processPendingDeletes(Index index, IndexSettings indexSettings, TimeValue timeout) throws IOException, InterruptedException {
         logger.debug("{} processing pending deletes", index);
         final long startTimeNS = System.nanoTime();
diff --git a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
index 2c77f863c47..c0bfedc47ca 100644
--- a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
+++ b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
@@ -19,16 +19,13 @@
 
 package org.elasticsearch.indices.cluster;
 
-import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.apache.lucene.store.LockObtainFailedException;
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.action.index.NodeMappingRefreshAction;
 import org.elasticsearch.cluster.action.shard.ShardStateAction;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingNode;
@@ -37,7 +34,6 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.lucene.Lucene;
@@ -48,17 +44,18 @@ import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexComponent;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexShardAlreadyExistsException;
 import org.elasticsearch.index.NodeServicesProvider;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.IndexShardRelocatedException;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardNotFoundException;
+import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.flush.SyncedFlushService;
 import org.elasticsearch.indices.recovery.RecoveryFailedException;
@@ -73,7 +70,6 @@ import org.elasticsearch.threadpool.ThreadPool;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -82,14 +78,13 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
 /**
  *
  */
 public class IndicesClusterStateService extends AbstractLifecycleComponent<IndicesClusterStateService> implements ClusterStateListener {
 
-    private final IndicesService indicesService;
+    final AllocatedIndices<? extends Shard, ? extends AllocatedIndex<? extends Shard>> indicesService;
     private final ClusterService clusterService;
     private final ThreadPool threadPool;
     private final RecoveryTargetService recoveryTargetService;
@@ -102,11 +97,10 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
 
     // a list of shards that failed during recovery
     // we keep track of these shards in order to prevent repeated recovery of these shards on each cluster state update
-    private final ConcurrentMap<ShardId, ShardRouting> failedShards = ConcurrentCollections.newConcurrentMap();
+    final ConcurrentMap<ShardId, ShardRouting> failedShardsCache = ConcurrentCollections.newConcurrentMap();
     private final RestoreService restoreService;
     private final RepositoriesService repositoriesService;
 
-    private final Object mutex = new Object();
     private final FailedShardHandler failedShardHandler = new FailedShardHandler();
 
     private final boolean sendRefreshMapping;
@@ -120,6 +114,22 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
                                       RepositoriesService repositoriesService, RestoreService restoreService,
                                       SearchService searchService, SyncedFlushService syncedFlushService,
                                       RecoverySource recoverySource, NodeServicesProvider nodeServicesProvider) {
+        this(settings, (AllocatedIndices<? extends Shard, ? extends AllocatedIndex<? extends Shard>>) indicesService,
+            clusterService, threadPool, recoveryTargetService, shardStateAction,
+            nodeMappingRefreshAction, repositoriesService, restoreService, searchService, syncedFlushService, recoverySource,
+            nodeServicesProvider);
+    }
+
+    // for tests
+    IndicesClusterStateService(Settings settings,
+                               AllocatedIndices<? extends Shard, ? extends AllocatedIndex<? extends Shard>> indicesService,
+                               ClusterService clusterService,
+                               ThreadPool threadPool, RecoveryTargetService recoveryTargetService,
+                               ShardStateAction shardStateAction,
+                               NodeMappingRefreshAction nodeMappingRefreshAction,
+                               RepositoriesService repositoriesService, RestoreService restoreService,
+                               SearchService searchService, SyncedFlushService syncedFlushService,
+                               RecoverySource recoverySource, NodeServicesProvider nodeServicesProvider) {
         super(settings);
         this.buildInIndexListener = Arrays.asList(recoverySource, recoveryTargetService, searchService, syncedFlushService);
         this.indicesService = indicesService;
@@ -149,87 +159,97 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
     }
 
     @Override
-    public void clusterChanged(final ClusterChangedEvent event) {
-        if (!indicesService.changesAllowed()) {
-            return;
-        }
-
+    public synchronized void clusterChanged(final ClusterChangedEvent event) {
         if (!lifecycle.started()) {
             return;
         }
 
-        synchronized (mutex) {
-            // we need to clean the shards and indices we have on this node, since we
-            // are going to recover them again once state persistence is disabled (no master / not recovered)
-            // TODO: this feels a bit hacky here, a block disables state persistence, and then we clean the allocated shards, maybe another flag in blocks?
-            if (event.state().blocks().disableStatePersistence()) {
-                for (IndexService indexService : indicesService) {
-                    Index index = indexService.index();
-                    for (Integer shardId : indexService.shardIds()) {
-                        logger.debug("{}[{}] removing shard (disabled block persistence)", index, shardId);
-                        try {
-                            indexService.removeShard(shardId, "removing shard (disabled block persistence)");
-                        } catch (Throwable e) {
-                            logger.warn("{} failed to remove shard (disabled block persistence)", e, index);
-                        }
-                    }
-                    removeIndex(index, "cleaning index (disabled block persistence)");
-                }
-                return;
+        final ClusterState state = event.state();
+
+        // we need to clean the shards and indices we have on this node, since we
+        // are going to recover them again once state persistence is disabled (no master / not recovered)
+        // TODO: feels hacky, a block disables state persistence, and then we clean the allocated shards, maybe another flag in blocks?
+        if (state.blocks().disableStatePersistence()) {
+            for (AllocatedIndex<? extends Shard> indexService : indicesService) {
+                indicesService.removeIndex(indexService.index(), "cleaning index (disabled block persistence)"); // also cleans shards
             }
-
-            cleanFailedShards(event);
-
-            // cleaning up indices that are completely deleted so we won't need to worry about them
-            // when checking for shards
-            applyDeletedIndices(event);
-            applyDeletedShards(event);
-            // call after deleted shards so indices with no shards will be cleaned
-            applyCleanedIndices(event);
-            // make sure that newly created shards use the latest meta data
-            applyIndexMetaData(event);
-            applyNewIndices(event);
-            // apply mappings also updates new indices. TODO: make new indices good to begin with
-            applyMappings(event);
-            applyNewOrUpdatedShards(event);
-        }
-    }
-
-    private void cleanFailedShards(final ClusterChangedEvent event) {
-        RoutingNode routingNode = event.state().getRoutingNodes().node(event.state().nodes().getLocalNodeId());
-        if (routingNode == null) {
-            failedShards.clear();
             return;
         }
-        for (Iterator<Map.Entry<ShardId, ShardRouting>> iterator = failedShards.entrySet().iterator(); iterator.hasNext(); ) {
-            Map.Entry<ShardId, ShardRouting> entry = iterator.next();
-            ShardRouting failedShardRouting = entry.getValue();
-            ShardRouting matchedShardRouting = routingNode.getByShardId(failedShardRouting.shardId());
-            if (matchedShardRouting == null || matchedShardRouting.isSameAllocation(failedShardRouting) == false) {
+
+        updateFailedShardsCache(state);
+
+        deleteIndices(event); // also deletes shards of deleted indices
+
+        removeUnallocatedIndices(state); // also removes shards of removed indices
+
+        failMissingShards(state);
+
+        removeShards(state);
+
+        updateIndices(event); // can also fail shards, but these are then guaranteed to be in failedShardsCache
+
+        createIndices(state);
+
+        createOrUpdateShards(state);
+    }
+
+    /**
+     * Removes shard entries from the failed shards cache that are no longer allocated to this node by the master.
+     * Sends shard failures for shards that are marked as actively allocated to this node but don't actually exist on the node.
+     * Resends shard failures for shards that are still marked as allocated to this node but previously failed.
+     *
+     * @param state new cluster state
+     */
+    private void updateFailedShardsCache(final ClusterState state) {
+        RoutingNode localRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId());
+        if (localRoutingNode == null) {
+            failedShardsCache.clear();
+            return;
+        }
+
+        DiscoveryNode masterNode = state.nodes().getMasterNode();
+
+        // remove items from cache which are not in our routing table anymore and resend failures that have not executed on master yet
+        for (Iterator<Map.Entry<ShardId, ShardRouting>> iterator = failedShardsCache.entrySet().iterator(); iterator.hasNext(); ) {
+            ShardRouting failedShardRouting = iterator.next().getValue();
+            ShardRouting matchedRouting = localRoutingNode.getByShardId(failedShardRouting.shardId());
+            if (matchedRouting == null || matchedRouting.isSameAllocation(failedShardRouting) == false) {
                 iterator.remove();
+            } else {
+                if (masterNode != null) { // TODO: can we remove this? Is resending shard failures the responsibility of shardStateAction?
+                    String message = "master " + masterNode + " has not removed previously failed shard. resending shard failure";
+                    logger.trace("[{}] re-sending failed shard [{}], reason [{}]", matchedRouting.shardId(), matchedRouting, message);
+                    shardStateAction.shardFailed(matchedRouting, matchedRouting, message, null, SHARD_STATE_ACTION_LISTENER);
+                }
             }
         }
     }
 
-    private void applyDeletedIndices(final ClusterChangedEvent event) {
+    /**
+     * Deletes indices (with shard data).
+     *
+     * @param event cluster change event
+     */
+    private void deleteIndices(final ClusterChangedEvent event) {
         final ClusterState previousState = event.previousState();
-        final String localNodeId = event.state().nodes().getLocalNodeId();
+        final ClusterState state = event.state();
+        final String localNodeId = state.nodes().getLocalNodeId();
         assert localNodeId != null;
 
         for (Index index : event.indicesDeleted()) {
             if (logger.isDebugEnabled()) {
                 logger.debug("[{}] cleaning index, no longer part of the metadata", index);
             }
-            final IndexService idxService = indicesService.indexService(index);
+            AllocatedIndex<? extends Shard> indexService = indicesService.indexService(index);
             final IndexSettings indexSettings;
-            if (idxService != null) {
-                indexSettings = idxService.getIndexSettings();
-                deleteIndex(index, "index no longer part of the metadata");
+            if (indexService != null) {
+                indexSettings = indexService.getIndexSettings();
+                indicesService.deleteIndex(index, "index no longer part of the metadata");
             } else if (previousState.metaData().hasIndex(index.getName())) {
                 // The deleted index was part of the previous cluster state, but not loaded on the local node
                 final IndexMetaData metaData = previousState.metaData().index(index);
                 indexSettings = new IndexSettings(metaData, settings);
-                indicesService.deleteUnassignedIndex("deleted index was not assigned to local node", metaData, event.state());
+                indicesService.deleteUnassignedIndex("deleted index was not assigned to local node", metaData, state);
             } else {
                 // The previous cluster state's metadata also does not contain the index,
                 // which is what happens on node startup when an index was deleted while the
@@ -255,10 +275,10 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
                     @Override
                     protected void doRun() throws Exception {
                         try {
-                            // we are waiting until we can lock the index / all shards on the node and then we ack the delete of the store to the
-                            // master. If we can't acquire the locks here immediately there might be a shard of this index still holding on to the lock
-                            // due to a "currently canceled recovery" or so. The shard will delete itself BEFORE the lock is released so it's guaranteed to be
-                            // deleted by the time we get the lock
+                            // we are waiting until we can lock the index / all shards on the node and then we ack the delete of the store
+                            // to the master. If we can't acquire the locks here immediately there might be a shard of this index still
+                            // holding on to the lock due to a "currently canceled recovery" or so. The shard will delete itself BEFORE the
+                            // lock is released so it's guaranteed to be deleted by the time we get the lock
                             indicesService.processPendingDeletes(index, indexSettings, new TimeValue(30, TimeUnit.MINUTES));
                         } catch (LockObtainFailedException exc) {
                             logger.warn("[{}] failed to lock all shards for index - timed out after 30 seconds", index);
@@ -270,322 +290,223 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
             }
         }
 
-        for (IndexService indexService : indicesService) {
-            IndexMetaData indexMetaData = event.state().metaData().index(indexService.index());
+        // delete local indices that do neither exist in previous cluster state nor part of tombstones
+        for (AllocatedIndex<? extends Shard> indexService : indicesService) {
+            Index index = indexService.index();
+            IndexMetaData indexMetaData = event.state().metaData().index(index);
             if (indexMetaData == null) {
-                assert false : "index" + indexService.index() + " exists locally, doesn't have a metadata but is not part "
+                assert false : "index" + index + " exists locally, doesn't have a metadata but is not part"
                     + " of the delete index list. \nprevious state: " + event.previousState().prettyPrint()
                     + "\n current state:\n" + event.state().prettyPrint();
-                logger.warn("[{}] isn't part of metadata but is part of in memory structures. removing",
-                    indexService.index());
-                deleteIndex(indexService.index(), "isn't part of metadata (explicit check)");
+                logger.warn("[{}] isn't part of metadata but is part of in memory structures. removing", index);
+                indicesService.deleteIndex(index, "isn't part of metadata (explicit check)");
             }
         }
     }
 
-    private void applyDeletedShards(final ClusterChangedEvent event) {
-        RoutingNode routingNode = event.state().getRoutingNodes().node(event.state().nodes().getLocalNodeId());
-        if (routingNode == null) {
+    /**
+     * Removes indices that have no shards allocated to this node. This does not delete the shard data as we wait for enough
+     * shard copies to exist in the cluster before deleting shard data (triggered by {@link org.elasticsearch.indices.store.IndicesStore}).
+     *
+     * @param state new cluster state
+     */
+    private void removeUnallocatedIndices(final ClusterState state) {
+        final String localNodeId = state.nodes().getLocalNodeId();
+        assert localNodeId != null;
+
+        Set<Index> indicesWithShards = new HashSet<>();
+        RoutingNode localRoutingNode = state.getRoutingNodes().node(localNodeId);
+        if (localRoutingNode != null) { // null e.g. if we are not a data node
+            for (ShardRouting shardRouting : localRoutingNode) {
+                indicesWithShards.add(shardRouting.index());
+            }
+        }
+
+        for (AllocatedIndex<? extends Shard> indexService : indicesService) {
+            Index index = indexService.index();
+            if (indicesWithShards.contains(index) == false) {
+                logger.debug("{} removing index, no shards allocated", index);
+                indicesService.removeIndex(index, "removing index (no shards allocated)");
+            }
+        }
+    }
+
+    /**
+     * Notifies master about shards that don't exist but are supposed to be active on this node.
+     *
+     * @param state new cluster state
+     */
+    private void failMissingShards(final ClusterState state) {
+        RoutingNode localRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId());
+        if (localRoutingNode == null) {
             return;
         }
-
-        final Map<Index, Set<String>> shardsByIndex = new HashMap<>();
-        for (ShardRouting shard : routingNode) {
-            shardsByIndex.computeIfAbsent(shard.index(), k -> new HashSet<>()).add(shard.allocationId().getId());
+        for (final ShardRouting shardRouting : localRoutingNode) {
+            ShardId shardId = shardRouting.shardId();
+            if (shardRouting.initializing() == false &&
+                failedShardsCache.containsKey(shardId) == false &&
+                indicesService.getShardOrNull(shardId) == null) {
+                // the master thinks we are active, but we don't have this shard at all, mark it as failed
+                sendFailShard(shardRouting, "master marked shard as active, but shard has not been created, mark shard as failed", null);
+            }
         }
+    }
 
-        for (IndexService indexService : indicesService) {
-            Index index = indexService.index();
-            IndexMetaData indexMetaData = event.state().metaData().index(index);
-            assert indexMetaData != null : "local index doesn't have metadata, should have been cleaned up by applyDeletedIndices: " + index;
-            // now, go over and delete shards that needs to get deleted
-            Set<String> newShardAllocationIds = shardsByIndex.getOrDefault(index, Collections.emptySet());
-            for (IndexShard existingShard : indexService) {
-                if (newShardAllocationIds.contains(existingShard.routingEntry().allocationId().getId()) == false) {
-                    if (indexMetaData.getState() == IndexMetaData.State.CLOSE) {
-                        if (logger.isDebugEnabled()) {
-                            logger.debug("{} removing shard (index is closed)", existingShard.shardId());
+    /**
+     * Removes shards that are currently loaded by indicesService but have disappeared from the routing table of the current node.
+     * Also removes shards where the recovery source node has changed.
+     * This method does not delete the shard data.
+     *
+     * @param state new cluster state
+     */
+    private void removeShards(final ClusterState state) {
+        final RoutingTable routingTable = state.routingTable();
+        final DiscoveryNodes nodes = state.nodes();
+        final String localNodeId = state.nodes().getLocalNodeId();
+        assert localNodeId != null;
+
+        // remove shards based on routing nodes (no deletion of data)
+        RoutingNode localRoutingNode = state.getRoutingNodes().node(localNodeId);
+        for (AllocatedIndex<? extends Shard> indexService : indicesService) {
+            for (Shard shard : indexService) {
+                ShardRouting currentRoutingEntry = shard.routingEntry();
+                ShardId shardId = currentRoutingEntry.shardId();
+                ShardRouting newShardRouting = localRoutingNode == null ? null : localRoutingNode.getByShardId(shardId);
+                if (newShardRouting == null || newShardRouting.isSameAllocation(currentRoutingEntry) == false) {
+                    // we can just remove the shard without cleaning it locally, since we will clean it in IndicesStore
+                    // once all shards are allocated
+                    logger.debug("{} removing shard (not allocated)", shardId);
+                    indexService.removeShard(shardId.id(), "removing shard (not allocated)");
+                } else {
+                    // remove shards where recovery source has changed. This re-initializes shards later in createOrUpdateShards
+                    if (newShardRouting.isPeerRecovery()) {
+                        RecoveryState recoveryState = shard.recoveryState();
+                        final DiscoveryNode sourceNode = findSourceNodeForPeerRecovery(logger, routingTable, nodes, newShardRouting);
+                        if (recoveryState.getSourceNode().equals(sourceNode) == false) {
+                            if (recoveryTargetService.cancelRecoveriesForShard(shardId, "recovery source node changed")) {
+                                // getting here means that the shard was still recovering
+                                logger.debug("{} removing shard (recovery source changed), current [{}], global [{}])",
+                                    shardId, currentRoutingEntry, newShardRouting);
+                                indexService.removeShard(shardId.id(), "removing shard (recovery source node changed)");
+                            }
                         }
-                        indexService.removeShard(existingShard.shardId().id(), "removing shard (index is closed)");
-                    } else {
-                        // we can just remove the shard, without cleaning it locally, since we will clean it
-                        // when all shards are allocated in the IndicesStore
-                        if (logger.isDebugEnabled()) {
-                            logger.debug("{} removing shard (not allocated)", existingShard.shardId());
-                        }
-                        indexService.removeShard(existingShard.shardId().id(), "removing shard (not allocated)");
                     }
                 }
             }
         }
     }
 
-    private void applyCleanedIndices(final ClusterChangedEvent event) {
-        // handle closed indices, since they are not allocated on a node once they are closed
-        // so applyDeletedIndices might not take them into account
-        for (IndexService indexService : indicesService) {
-            Index index = indexService.index();
-            IndexMetaData indexMetaData = event.state().metaData().index(index);
-            if (indexMetaData != null && indexMetaData.getState() == IndexMetaData.State.CLOSE) {
-                for (Integer shardId : indexService.shardIds()) {
-                    logger.debug("{}[{}] removing shard (index is closed)", index, shardId);
-                    try {
-                        indexService.removeShard(shardId, "removing shard (index is closed)");
-                    } catch (Throwable e) {
-                        logger.warn("{} failed to remove shard (index is closed)", e, index);
-                    }
-                }
-            }
-        }
-
-        final Set<Index> hasAllocations = new HashSet<>();
-        final RoutingNode node = event.state().getRoutingNodes().node(event.state().nodes().getLocalNodeId());
-        // if no shards are allocated ie. if this node is a master-only node it can return nul
-        if (node != null) {
-            for (ShardRouting routing : node) {
-                hasAllocations.add(routing.index());
-            }
-        }
-        for (IndexService indexService : indicesService) {
-            Index index = indexService.index();
-            if (hasAllocations.contains(index) == false) {
-                assert indexService.shardIds().isEmpty() :
-                    "no locally assigned shards, but index wasn't emptied by applyDeletedShards."
-                        + " index " + index + ", shards: " + indexService.shardIds();
-                if (logger.isDebugEnabled()) {
-                    logger.debug("{} cleaning index (no shards allocated)", index);
-                }
-                // clean the index
-                removeIndex(index, "removing index (no shards allocated)");
-            }
-        }
-    }
-
-    private void applyIndexMetaData(ClusterChangedEvent event) {
-        if (!event.metaDataChanged()) {
-            return;
-        }
-        for (IndexMetaData indexMetaData : event.state().metaData()) {
-            if (!indicesService.hasIndex(indexMetaData.getIndex())) {
-                // we only create / update here
-                continue;
-            }
-            // if the index meta data didn't change, no need check for refreshed settings
-            if (!event.indexMetaDataChanged(indexMetaData)) {
-                continue;
-            }
-            Index index = indexMetaData.getIndex();
-            IndexService indexService = indicesService.indexService(index);
-            if (indexService == null) {
-                // already deleted on us, ignore it
-                continue;
-            }
-            indexService.updateMetaData(indexMetaData);
-        }
-    }
-
-    private void applyNewIndices(final ClusterChangedEvent event) {
+    private void createIndices(final ClusterState state) {
         // we only create indices for shards that are allocated
-        RoutingNode routingNode = event.state().getRoutingNodes().node(event.state().nodes().getLocalNodeId());
-        if (routingNode == null) {
+        RoutingNode localRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId());
+        if (localRoutingNode == null) {
             return;
         }
-        for (ShardRouting shard : routingNode) {
-            if (!indicesService.hasIndex(shard.index())) {
-                final IndexMetaData indexMetaData = event.state().metaData().getIndexSafe(shard.index());
-                if (logger.isDebugEnabled()) {
-                    logger.debug("[{}] creating index", indexMetaData.getIndex());
-                }
-                try {
-                    indicesService.createIndex(nodeServicesProvider, indexMetaData, buildInIndexListener);
-                } catch (Throwable e) {
-                    sendFailShard(shard, "failed to create index", e);
+        // create map of indices to create with shards to fail if index creation fails
+        final Map<Index, List<ShardRouting>> indicesToCreate = new HashMap<>();
+        for (ShardRouting shardRouting : localRoutingNode) {
+            if (failedShardsCache.containsKey(shardRouting.shardId()) == false) {
+                final Index index = shardRouting.index();
+                if (indicesService.indexService(index) == null) {
+                    indicesToCreate.computeIfAbsent(index, k -> new ArrayList<>()).add(shardRouting);
                 }
             }
         }
-    }
 
-    private void applyMappings(ClusterChangedEvent event) {
-        // go over and update mappings
-        for (IndexMetaData indexMetaData : event.state().metaData()) {
-            Index index = indexMetaData.getIndex();
-            if (!indicesService.hasIndex(index)) {
-                // we only create / update here
-                continue;
-            }
-            boolean requireRefresh = false;
-            IndexService indexService = indicesService.indexService(index);
-            if (indexService == null) {
-                // got deleted on us, ignore (closing the node)
-                return;
-            }
+        for (Map.Entry<Index, List<ShardRouting>> entry : indicesToCreate.entrySet()) {
+            final Index index = entry.getKey();
+            final IndexMetaData indexMetaData = state.metaData().index(index);
+            logger.debug("[{}] creating index", index);
+
+            AllocatedIndex<? extends Shard> indexService = null;
             try {
-                MapperService mapperService = indexService.mapperService();
-                // go over and add the relevant mappings (or update them)
-                for (ObjectCursor<MappingMetaData> cursor : indexMetaData.getMappings().values()) {
-                    MappingMetaData mappingMd = cursor.value;
-                    String mappingType = mappingMd.type();
-                    CompressedXContent mappingSource = mappingMd.source();
-                    requireRefresh |= processMapping(index.getName(), mapperService, mappingType, mappingSource);
-                }
-                if (requireRefresh && sendRefreshMapping) {
-                    nodeMappingRefreshAction.nodeMappingRefresh(event.state(),
-                        new NodeMappingRefreshAction.NodeMappingRefreshRequest(index.getName(), indexMetaData.getIndexUUID(),
-                            event.state().nodes().getLocalNodeId())
+                indexService = indicesService.createIndex(nodeServicesProvider, indexMetaData, buildInIndexListener);
+                if (indexService.updateMapping(indexMetaData) && sendRefreshMapping) {
+                    nodeMappingRefreshAction.nodeMappingRefresh(state.nodes().getMasterNode(),
+                        new NodeMappingRefreshAction.NodeMappingRefreshRequest(indexMetaData.getIndex().getName(),
+                            indexMetaData.getIndexUUID(), state.nodes().getLocalNodeId())
                     );
                 }
             } catch (Throwable t) {
-                // if we failed the mappings anywhere, we need to fail the shards for this index, note, we safeguard
-                // by creating the processing the mappings on the master, or on the node the mapping was introduced on,
-                // so this failure typically means wrong node level configuration or something similar
-                for (IndexShard indexShard : indexService) {
-                    ShardRouting shardRouting = indexShard.routingEntry();
-                    failAndRemoveShard(shardRouting, indexService, true, "failed to update mappings", t);
-                }
-            }
-        }
-    }
-
-    private boolean processMapping(String index, MapperService mapperService, String mappingType, CompressedXContent mappingSource) throws Throwable {
-        // refresh mapping can happen when the parsing/merging of the mapping from the metadata doesn't result in the same
-        // mapping, in this case, we send to the master to refresh its own version of the mappings (to conform with the
-        // merge version of it, which it does when refreshing the mappings), and warn log it.
-        boolean requiresRefresh = false;
-        try {
-            DocumentMapper existingMapper = mapperService.documentMapper(mappingType);
-
-            if (existingMapper == null || mappingSource.equals(existingMapper.mappingSource()) == false) {
-                String op = existingMapper == null ? "adding" : "updating";
-                if (logger.isDebugEnabled() && mappingSource.compressed().length < 512) {
-                    logger.debug("[{}] {} mapping [{}], source [{}]", index, op, mappingType, mappingSource.string());
-                } else if (logger.isTraceEnabled()) {
-                    logger.trace("[{}] {} mapping [{}], source [{}]", index, op, mappingType, mappingSource.string());
+                final String failShardReason;
+                if (indexService == null) {
+                    failShardReason = "failed to create index";
                 } else {
-                    logger.debug("[{}] {} mapping [{}] (source suppressed due to length, use TRACE level if needed)", index, op, mappingType);
+                    failShardReason = "failed to update mapping for index";
+                    indicesService.removeIndex(index, "removing index (mapping update failed)");
                 }
-                mapperService.merge(mappingType, mappingSource, MapperService.MergeReason.MAPPING_RECOVERY, true);
-                if (!mapperService.documentMapper(mappingType).mappingSource().equals(mappingSource)) {
-                    logger.debug("[{}] parsed mapping [{}], and got different sources\noriginal:\n{}\nparsed:\n{}", index, mappingType, mappingSource, mapperService.documentMapper(mappingType).mappingSource());
-                    requiresRefresh = true;
+                for (ShardRouting shardRouting : entry.getValue()) {
+                    sendFailShard(shardRouting, failShardReason, t);
                 }
             }
-        } catch (Throwable e) {
-            logger.warn("[{}] failed to add mapping [{}], source [{}]", e, index, mappingType, mappingSource);
-            throw e;
         }
-        return requiresRefresh;
     }
 
-
-    private void applyNewOrUpdatedShards(final ClusterChangedEvent event) {
-        if (!indicesService.changesAllowed()) {
+    private void updateIndices(ClusterChangedEvent event) {
+        if (!event.metaDataChanged()) {
             return;
         }
-
-        RoutingTable routingTable = event.state().routingTable();
-        RoutingNode routingNode = event.state().getRoutingNodes().node(event.state().nodes().getLocalNodeId());
-
-        if (routingNode == null) {
-            failedShards.clear();
-            return;
-        }
-
-        DiscoveryNodes nodes = event.state().nodes();
-        for (final ShardRouting shardRouting : routingNode) {
-            final IndexService indexService = indicesService.indexService(shardRouting.index());
-            if (indexService == null) {
-                // creation failed for some reasons
-                assert failedShards.containsKey(shardRouting.shardId()) :
-                    "index has local allocation but is not created by applyNewIndices and is not failed " + shardRouting;
-                continue;
-            }
-            final IndexMetaData indexMetaData = event.state().metaData().index(shardRouting.index());
-            assert indexMetaData != null : "index has local allocation but no meta data. " + shardRouting.index();
-
-            final int shardId = shardRouting.id();
-
-            if (!indexService.hasShard(shardId) && shardRouting.started()) {
-                if (failedShards.containsKey(shardRouting.shardId())) {
-                    if (nodes.getMasterNode() != null) {
-                        String message = "master " + nodes.getMasterNode() + " marked shard as started, but shard has previous failed. resending shard failure";
-                        logger.trace("[{}] re-sending failed shard [{}], reason [{}]", shardRouting.shardId(), shardRouting, message);
-                        shardStateAction.shardFailed(shardRouting, shardRouting, message, null, SHARD_STATE_ACTION_LISTENER);
+        final ClusterState state = event.state();
+        for (AllocatedIndex<? extends Shard> indexService : indicesService) {
+            final Index index = indexService.index();
+            final IndexMetaData currentIndexMetaData = indexService.getIndexSettings().getIndexMetaData();
+            final IndexMetaData newIndexMetaData = state.metaData().index(index);
+            assert newIndexMetaData != null : "index " + index + " should have been removed by deleteIndices";
+            if (ClusterChangedEvent.indexMetaDataChanged(currentIndexMetaData, newIndexMetaData)) {
+                indexService.updateMetaData(newIndexMetaData);
+                try {
+                    if (indexService.updateMapping(newIndexMetaData) && sendRefreshMapping) {
+                        nodeMappingRefreshAction.nodeMappingRefresh(state.nodes().getMasterNode(),
+                            new NodeMappingRefreshAction.NodeMappingRefreshRequest(newIndexMetaData.getIndex().getName(),
+                                newIndexMetaData.getIndexUUID(), state.nodes().getLocalNodeId())
+                        );
                     }
-                } else {
-                    // the master thinks we are started, but we don't have this shard at all, mark it as failed
-                    sendFailShard(shardRouting, "master [" + nodes.getMasterNode() + "] marked shard as started, but shard has not been created, mark shard as failed", null);
-                }
-                continue;
-            }
+                } catch (Throwable t) {
+                    indicesService.removeIndex(indexService.index(), "removing index (mapping update failed)");
 
-            IndexShard indexShard = indexService.getShardOrNull(shardId);
-            if (indexShard != null) {
-                ShardRouting currentRoutingEntry = indexShard.routingEntry();
-                // if the current and global routing are initializing, but are still not the same, its a different "shard" being allocated
-                // for example: a shard that recovers from one node and now needs to recover to another node,
-                //              or a replica allocated and then allocating a primary because the primary failed on another node
-                boolean shardHasBeenRemoved = false;
-                assert currentRoutingEntry.isSameAllocation(shardRouting) :
-                    "local shard has a different allocation id but wasn't cleaning by applyDeletedShards. "
-                        + "cluster state: " + shardRouting + " local: " + currentRoutingEntry;
-                if (shardRouting.isPeerRecovery()) {
-                    RecoveryState recoveryState = indexShard.recoveryState();
-                    final DiscoveryNode sourceNode = findSourceNodeForPeerRecovery(logger, routingTable, nodes, shardRouting);
-                    if (recoveryState.getSourceNode().equals(sourceNode) == false) {
-                        if (recoveryTargetService.cancelRecoveriesForShard(currentRoutingEntry.shardId(), "recovery source node changed")) {
-                            // getting here means that the shard was still recovering
-                            logger.debug("[{}][{}] removing shard (recovery source changed), current [{}], global [{}])", shardRouting.index(), shardRouting.id(), currentRoutingEntry, shardRouting);
-                            indexService.removeShard(shardRouting.id(), "removing shard (recovery source node changed)");
-                            shardHasBeenRemoved = true;
+                    // fail shards that would be created or updated by createOrUpdateShards
+                    RoutingNode localRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId());
+                    if (localRoutingNode != null) {
+                        for (final ShardRouting shardRouting : localRoutingNode) {
+                            if (shardRouting.index().equals(index) && failedShardsCache.containsKey(shardRouting.shardId()) == false) {
+                                sendFailShard(shardRouting, "failed to update mapping for index", t);
+                            }
                         }
                     }
                 }
-
-                if (shardHasBeenRemoved == false) {
-                    try {
-                        indexShard.updateRoutingEntry(shardRouting, event.state().blocks().disableStatePersistence() == false);
-                    } catch (Throwable e) {
-                        failAndRemoveShard(shardRouting, indexService, true, "failed updating shard routing entry", e);
-                    }
-                }
-            }
-
-            if (shardRouting.initializing()) {
-                applyInitializingShard(event.state(), indexService, shardRouting);
             }
         }
     }
 
-    private void applyInitializingShard(final ClusterState state, IndexService indexService, final ShardRouting shardRouting) {
-        final RoutingTable routingTable = state.routingTable();
-        final DiscoveryNodes nodes = state.getNodes();
-        final int shardId = shardRouting.id();
+    private void createOrUpdateShards(final ClusterState state) {
+        RoutingNode localRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId());
+        if (localRoutingNode == null) {
+            return;
+        }
 
-        if (indexService.hasShard(shardId)) {
-            IndexShard indexShard = indexService.getShard(shardId);
-            if (indexShard.state() == IndexShardState.STARTED || indexShard.state() == IndexShardState.POST_RECOVERY) {
-                // the master thinks we are initializing, but we are already started or on POST_RECOVERY and waiting
-                // for master to confirm a shard started message (either master failover, or a cluster event before
-                // we managed to tell the master we started), mark us as started
-                if (logger.isTraceEnabled()) {
-                    logger.trace("{} master marked shard as initializing, but shard has state [{}], resending shard started to {}",
-                        indexShard.shardId(), indexShard.state(), nodes.getMasterNode());
-                }
-                if (nodes.getMasterNode() != null) {
-                    shardStateAction.shardStarted(shardRouting,
-                        "master " + nodes.getMasterNode() + " marked shard as initializing, but shard state is [" + indexShard.state() + "], mark shard as started",
-                        SHARD_STATE_ACTION_LISTENER);
-                }
-                return;
-            } else {
-                if (indexShard.ignoreRecoveryAttempt()) {
-                    logger.trace("ignoring recovery instruction for an existing shard {} (shard state: [{}])", indexShard.shardId(), indexShard.state());
-                    return;
+        DiscoveryNodes nodes = state.nodes();
+        RoutingTable routingTable = state.routingTable();
+
+        for (final ShardRouting shardRouting : localRoutingNode) {
+            ShardId shardId = shardRouting.shardId();
+            if (failedShardsCache.containsKey(shardId) == false) {
+                AllocatedIndex<? extends Shard> indexService = indicesService.indexService(shardId.getIndex());
+                assert indexService != null : "index " + shardId.getIndex() + " should have been created by createIndices";
+                Shard shard = indexService.getShardOrNull(shardId.id());
+                if (shard == null) {
+                    assert shardRouting.initializing() : shardRouting + " should have been removed by failMissingShards";
+                    createShard(nodes, routingTable, shardRouting, indexService);
+                } else {
+                    updateShard(nodes, shardRouting, shard);
                 }
             }
         }
+    }
+
+    private void createShard(DiscoveryNodes nodes, RoutingTable routingTable, ShardRouting shardRouting,
+                             AllocatedIndex<? extends Shard> indexService) {
+        assert shardRouting.initializing() : "only allow shard creation for initializing shard but was " + shardRouting;
 
-        // if we're in peer recovery, try to find out the source node now so in case it fails, we will not create the index shard
         DiscoveryNode sourceNode = null;
         if (shardRouting.isPeerRecovery()) {
             sourceNode = findSourceNodeForPeerRecovery(logger, routingTable, nodes, shardRouting);
@@ -595,50 +516,49 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
             }
         }
 
-        // if there is no shard, create it
-        if (!indexService.hasShard(shardId)) {
-            if (failedShards.containsKey(shardRouting.shardId())) {
-                if (nodes.getMasterNode() != null) {
-                    String message = "master " + nodes.getMasterNode() + " marked shard as initializing, but shard is marked as failed, resend shard failure";
-                    logger.trace("[{}] re-sending failed shard [{}], reason [{}]", shardRouting.shardId(), shardRouting, message);
-                    shardStateAction.shardFailed(shardRouting, shardRouting, message, null, SHARD_STATE_ACTION_LISTENER);
-                }
-                return;
-            }
-            try {
-                if (logger.isDebugEnabled()) {
-                    logger.debug("[{}][{}] creating shard", shardRouting.index(), shardId);
-                }
-                IndexShard indexShard = indexService.createShard(shardRouting);
-                indexShard.addShardFailureCallback(failedShardHandler);
-            } catch (IndexShardAlreadyExistsException e) {
-                // ignore this, the method call can happen several times
-            } catch (Throwable e) {
-                failAndRemoveShard(shardRouting, indexService, true, "failed to create shard", e);
-                return;
-            }
+        try {
+            logger.debug("{} creating shard", shardRouting.shardId());
+            RecoveryState recoveryState = recoveryState(nodes.getLocalNode(), sourceNode, shardRouting,
+                indexService.getIndexSettings().getIndexMetaData());
+            indicesService.createShard(shardRouting, recoveryState, recoveryTargetService, new RecoveryListener(shardRouting),
+                repositoriesService, nodeServicesProvider, failedShardHandler);
+        } catch (IndexShardAlreadyExistsException e) {
+            // ignore this, the method call can happen several times
+            logger.debug("Trying to create shard that already exists", e);
+            assert false;
+        } catch (Throwable e) {
+            failAndRemoveShard(shardRouting, true, "failed to create shard", e);
         }
-        final IndexShard indexShard = indexService.getShard(shardId);
+    }
 
-        if (indexShard.ignoreRecoveryAttempt()) {
-            // we are already recovering (we can get to this state since the cluster event can happen several
-            // times while we recover)
-            logger.trace("ignoring recovery instruction for shard {} (shard state: [{}])", indexShard.shardId(), indexShard.state());
+    private void updateShard(DiscoveryNodes nodes, ShardRouting shardRouting, Shard shard) {
+        final ShardRouting currentRoutingEntry = shard.routingEntry();
+        assert currentRoutingEntry.isSameAllocation(shardRouting) :
+            "local shard has a different allocation id but wasn't cleaning by removeShards. "
+                + "cluster state: " + shardRouting + " local: " + currentRoutingEntry;
+
+        try {
+            shard.updateRoutingEntry(shardRouting);
+        } catch (Throwable e) {
+            failAndRemoveShard(shardRouting, true, "failed updating shard routing entry", e);
             return;
         }
 
-        indexShard.startRecovery(nodes.getLocalNode(), sourceNode, recoveryTargetService,
-            new RecoveryListener(shardRouting, indexService), repositoriesService, (type, mapping) -> {
-                try {
-                    nodeServicesProvider.getClient().admin().indices().preparePutMapping()
-                        .setConcreteIndex(indexService.index()) // concrete index - no name clash, it uses uuid
-                        .setType(type)
-                        .setSource(mapping.source().string())
-                        .get();
-                } catch (IOException ex) {
-                    throw new ElasticsearchException("failed to stringify mapping source", ex);
-                }
-            }, indicesService);
+        final IndexShardState state = shard.state();
+        if (shardRouting.initializing() && (state == IndexShardState.STARTED || state == IndexShardState.POST_RECOVERY)) {
+            // the master thinks we are initializing, but we are already started or on POST_RECOVERY and waiting
+            // for master to confirm a shard started message (either master failover, or a cluster event before
+            // we managed to tell the master we started), mark us as started
+            if (logger.isTraceEnabled()) {
+                logger.trace("{} master marked shard as initializing, but shard has state [{}], resending shard started to {}",
+                    shardRouting.shardId(), state, nodes.getMasterNode());
+            }
+            if (nodes.getMasterNode() != null) {
+                shardStateAction.shardStarted(shardRouting, "master " + nodes.getMasterNode() +
+                    " marked shard as initializing, but shard state is [" + state + "], mark shard as started",
+                    SHARD_STATE_ACTION_LISTENER);
+            }
+        }
     }
 
     /**
@@ -646,7 +566,8 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
      * routing to *require* peer recovery, use {@link ShardRouting#isPeerRecovery()} to
      * check if its needed or not.
      */
-    private static DiscoveryNode findSourceNodeForPeerRecovery(ESLogger logger, RoutingTable routingTable, DiscoveryNodes nodes, ShardRouting shardRouting) {
+    private static DiscoveryNode findSourceNodeForPeerRecovery(ESLogger logger, RoutingTable routingTable, DiscoveryNodes nodes,
+                                                               ShardRouting shardRouting) {
         DiscoveryNode sourceNode = null;
         if (!shardRouting.primary()) {
             ShardRouting primary = routingTable.shardRoutingTable(shardRouting.shardId()).primaryShard();
@@ -662,10 +583,12 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
         } else if (shardRouting.relocatingNodeId() != null) {
             sourceNode = nodes.get(shardRouting.relocatingNodeId());
             if (sourceNode == null) {
-                logger.trace("can't find relocation source node for shard {} because it is assigned to an unknown node [{}].", shardRouting.shardId(), shardRouting.relocatingNodeId());
+                logger.trace("can't find relocation source node for shard {} because it is assigned to an unknown node [{}].",
+                    shardRouting.shardId(), shardRouting.relocatingNodeId());
             }
         } else {
-            throw new IllegalStateException("trying to find source node for peer recovery when routing state means no peer recovery: " + shardRouting);
+            throw new IllegalStateException("trying to find source node for peer recovery when routing state means no peer recovery: " +
+                shardRouting);
         }
         return sourceNode;
     }
@@ -673,11 +596,9 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
     private class RecoveryListener implements RecoveryTargetService.RecoveryListener {
 
         private final ShardRouting shardRouting;
-        private final IndexService indexService;
 
-        private RecoveryListener(ShardRouting shardRouting, IndexService indexService) {
+        private RecoveryListener(ShardRouting shardRouting) {
             this.shardRouting = shardRouting;
-            this.indexService = indexService;
         }
 
         @Override
@@ -709,48 +630,50 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
                 } catch (Throwable inner) {
                     e.addSuppressed(inner);
                 } finally {
-                    handleRecoveryFailure(indexService, shardRouting, sendShardFailure, e);
+                    handleRecoveryFailure(shardRouting, sendShardFailure, e);
                 }
             } else {
-                handleRecoveryFailure(indexService, shardRouting, sendShardFailure, e);
+                handleRecoveryFailure(shardRouting, sendShardFailure, e);
             }
         }
     }
 
-    private void handleRecoveryFailure(IndexService indexService, ShardRouting shardRouting, boolean sendShardFailure, Throwable failure) {
-        synchronized (mutex) {
-            failAndRemoveShard(shardRouting, indexService, sendShardFailure, "failed recovery", failure);
+    private RecoveryState recoveryState(DiscoveryNode localNode, DiscoveryNode sourceNode, ShardRouting shardRouting,
+                                              IndexMetaData indexMetaData) {
+        assert shardRouting.initializing() : "only allow initializing shard routing to be recovered: " + shardRouting;
+        if (shardRouting.isPeerRecovery()) {
+            assert sourceNode != null : "peer recovery started but sourceNode is null for " + shardRouting;
+            RecoveryState.Type type = shardRouting.primary() ? RecoveryState.Type.PRIMARY_RELOCATION : RecoveryState.Type.REPLICA;
+            return new RecoveryState(shardRouting.shardId(), shardRouting.primary(), type, sourceNode, localNode);
+        } else if (shardRouting.restoreSource() == null) {
+            // recover from filesystem store
+            Index mergeSourceIndex = indexMetaData.getMergeSourceIndex();
+            final boolean recoverFromLocalShards = mergeSourceIndex != null && shardRouting.allocatedPostIndexCreate(indexMetaData) == false
+                && shardRouting.primary();
+            return new RecoveryState(shardRouting.shardId(), shardRouting.primary(),
+                recoverFromLocalShards ? RecoveryState.Type.LOCAL_SHARDS : RecoveryState.Type.STORE, localNode, localNode);
+        } else {
+            // recover from a snapshot
+            return new RecoveryState(shardRouting.shardId(), shardRouting.primary(),
+                RecoveryState.Type.SNAPSHOT, shardRouting.restoreSource(), localNode);
         }
     }
 
-    private void removeIndex(Index index, String reason) {
+    private synchronized void handleRecoveryFailure(ShardRouting shardRouting, boolean sendShardFailure, Throwable failure) {
+        failAndRemoveShard(shardRouting, sendShardFailure, "failed recovery", failure);
+    }
+
+    private void failAndRemoveShard(ShardRouting shardRouting, boolean sendShardFailure, String message, @Nullable Throwable failure) {
         try {
-            indicesService.removeIndex(index, reason);
-        } catch (Throwable e) {
-            logger.warn("failed to clean index ({})", e, reason);
-        }
-    }
-
-    private void deleteIndex(Index index, String reason) {
-        try {
-            indicesService.deleteIndex(index, reason);
-        } catch (Throwable e) {
-            logger.warn("failed to delete index ({})", e, reason);
-        }
-
-    }
-
-    private void failAndRemoveShard(ShardRouting shardRouting, @Nullable IndexService indexService, boolean sendShardFailure, String message, @Nullable Throwable failure) {
-        if (indexService != null && indexService.hasShard(shardRouting.getId())) {
-            // if the indexService is null we can't remove the shard, that's fine since we might have a failure
-            // when the index is remove and then we already removed the index service for that shard...
-            try {
-                indexService.removeShard(shardRouting.getId(), message);
-            } catch (ShardNotFoundException e) {
-                // the node got closed on us, ignore it
-            } catch (Throwable e1) {
-                logger.warn("[{}][{}] failed to remove shard after failure ([{}])", e1, shardRouting.getIndexName(), shardRouting.getId(), message);
+            AllocatedIndex<? extends Shard> indexService = indicesService.indexService(shardRouting.shardId().getIndex());
+            if (indexService != null) {
+                indexService.removeShard(shardRouting.shardId().id(), message);
             }
+        } catch (ShardNotFoundException e) {
+            // the node got closed on us, ignore it
+        } catch (Throwable e1) {
+            logger.warn("[{}][{}] failed to remove shard after failure ([{}])", e1, shardRouting.getIndexName(), shardRouting.getId(),
+                message);
         }
         if (sendShardFailure) {
             sendFailShard(shardRouting, message, failure);
@@ -760,23 +683,156 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
     private void sendFailShard(ShardRouting shardRouting, String message, @Nullable Throwable failure) {
         try {
             logger.warn("[{}] marking and sending shard failed due to [{}]", failure, shardRouting.shardId(), message);
-            failedShards.put(shardRouting.shardId(), shardRouting);
+            failedShardsCache.put(shardRouting.shardId(), shardRouting);
             shardStateAction.shardFailed(shardRouting, shardRouting, message, failure, SHARD_STATE_ACTION_LISTENER);
         } catch (Throwable e1) {
-            logger.warn("[{}][{}] failed to mark shard as failed (because of [{}])", e1, shardRouting.getIndexName(), shardRouting.getId(), message);
+            logger.warn("[{}][{}] failed to mark shard as failed (because of [{}])", e1, shardRouting.getIndexName(), shardRouting.getId(),
+                message);
         }
     }
 
     private class FailedShardHandler implements Callback<IndexShard.ShardFailure> {
         @Override
         public void handle(final IndexShard.ShardFailure shardFailure) {
-            final IndexService indexService = indicesService.indexService(shardFailure.routing.shardId().getIndex());
             final ShardRouting shardRouting = shardFailure.routing;
             threadPool.generic().execute(() -> {
-                synchronized (mutex) {
-                    failAndRemoveShard(shardRouting, indexService, true, "shard failure, reason [" + shardFailure.reason + "]", shardFailure.cause);
+                synchronized (IndicesClusterStateService.this) {
+                    failAndRemoveShard(shardRouting, true, "shard failure, reason [" + shardFailure.reason + "]", shardFailure.cause);
                 }
             });
         }
     }
+
+    public interface Shard {
+
+        /**
+         * Returns the shard id of this shard.
+         */
+        ShardId shardId();
+
+        /**
+         * Returns the latest cluster routing entry received with this shard.
+         */
+        ShardRouting routingEntry();
+
+        /**
+         * Returns the latest internal shard state.
+         */
+        IndexShardState state();
+
+        /**
+         * Returns the recovery state associated with this shard.
+         */
+        RecoveryState recoveryState();
+
+        /**
+         * Updates the shards routing entry. This mutate the shards internal state depending
+         * on the changes that get introduced by the new routing value. This method will persist shard level metadata.
+         *
+         * @throws IndexShardRelocatedException if shard is marked as relocated and relocation aborted
+         * @throws IOException                  if shard state could not be persisted
+         */
+        void updateRoutingEntry(ShardRouting shardRouting) throws IOException;
+    }
+
+    public interface AllocatedIndex<T extends Shard> extends Iterable<T>, IndexComponent {
+
+        /**
+         * Returns the index settings of this index.
+         */
+        IndexSettings getIndexSettings();
+
+        /**
+         * Updates the meta data of this index. Changes become visible through {@link #getIndexSettings()}
+         */
+        void updateMetaData(IndexMetaData indexMetaData);
+
+        /**
+         * Checks if index requires refresh from master.
+         */
+        boolean updateMapping(IndexMetaData indexMetaData) throws IOException;
+
+        /**
+         * Returns shard with given id.
+         */
+        @Nullable T getShardOrNull(int shardId);
+
+        /**
+         * Removes shard with given id.
+         */
+        void removeShard(int shardId, String message);
+    }
+
+    public interface AllocatedIndices<T extends Shard, U extends AllocatedIndex<T>> extends Iterable<U> {
+
+        /**
+         * Creates a new {@link IndexService} for the given metadata.
+         * @param indexMetaData the index metadata to create the index for
+         * @param builtInIndexListener a list of built-in lifecycle {@link IndexEventListener} that should should be used along side with
+         *                             the per-index listeners
+         * @throws IndexAlreadyExistsException if the index already exists.
+         */
+        U createIndex(NodeServicesProvider nodeServicesProvider, IndexMetaData indexMetaData,
+                         List<IndexEventListener> builtInIndexListener) throws IOException;
+
+        /**
+         * Verify that the contents on disk for the given index is deleted; if not, delete the contents.
+         * This method assumes that an index is already deleted in the cluster state and/or explicitly
+         * through index tombstones.
+         * @param index {@code Index} to make sure its deleted from disk
+         * @param clusterState {@code ClusterState} to ensure the index is not part of it
+         * @return IndexMetaData for the index loaded from disk
+         */
+        IndexMetaData verifyIndexIsDeleted(Index index, ClusterState clusterState);
+
+        /**
+         * Deletes the given index. Persistent parts of the index
+         * like the shards files, state and transaction logs are removed once all resources are released.
+         *
+         * Equivalent to {@link #removeIndex(Index, String)} but fires
+         * different lifecycle events to ensure pending resources of this index are immediately removed.
+         * @param index the index to delete
+         * @param reason the high level reason causing this delete
+         */
+        void deleteIndex(Index index, String reason);
+
+        /**
+         * Deletes an index that is not assigned to this node. This method cleans up all disk folders relating to the index
+         * but does not deal with in-memory structures. For those call {@link #deleteIndex(Index, String)}
+         */
+        void deleteUnassignedIndex(String reason, IndexMetaData metaData, ClusterState clusterState);
+
+        /**
+         * Removes the given index from this service and releases all associated resources. Persistent parts of the index
+         * like the shards files, state and transaction logs are kept around in the case of a disaster recovery.
+         * @param index the index to remove
+         * @param reason  the high level reason causing this removal
+         */
+        void removeIndex(Index index, String reason);
+
+        /**
+         * Returns an IndexService for the specified index if exists otherwise returns <code>null</code>.
+         */
+        @Nullable U indexService(Index index);
+
+        /**
+         * Creates shard for the specified shard routing and starts recovery,
+         */
+        T createShard(ShardRouting shardRouting, RecoveryState recoveryState, RecoveryTargetService recoveryTargetService,
+                      RecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService,
+                      NodeServicesProvider nodeServicesProvider, Callback<IndexShard.ShardFailure> onShardFailure) throws IOException;
+
+        /**
+         * Returns shard for the specified id if it exists otherwise returns <code>null</code>.
+         */
+        default T getShardOrNull(ShardId shardId) {
+            U indexRef = indexService(shardId.getIndex());
+            if (indexRef != null) {
+                return indexRef.getShardOrNull(shardId.id());
+            }
+            return null;
+        }
+
+        void processPendingDeletes(Index index, IndexSettings indexSettings, TimeValue timeValue) throws IOException, InterruptedException;
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionUtils.java b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionUtils.java
new file mode 100644
index 00000000000..be4a7b29703
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionUtils.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.support.master;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.cluster.ClusterState;
+
+public class TransportMasterNodeActionUtils {
+
+    /**
+     * Allows to directly call {@link TransportMasterNodeAction#masterOperation(MasterNodeRequest, ClusterState, ActionListener)} which is
+     * a protected method.
+     */
+    public static <Request extends MasterNodeRequest<Request>, Response extends ActionResponse> void runMasterOperation(
+        TransportMasterNodeAction<Request, Response> masterNodeAction, Request request, ClusterState clusterState,
+        ActionListener<Response> actionListener) throws Exception {
+        assert masterNodeAction.checkBlock(request, clusterState) == null;
+        masterNodeAction.masterOperation(request, clusterState, actionListener);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java b/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
index 731ecb859ee..72748a59986 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
@@ -140,24 +140,24 @@ public class ClusterChangedEventTests extends ESTestCase {
      */
     public void testIndexMetaDataChange() {
         final int numNodesInCluster = 3;
-        final ClusterState originalState = createState(numNodesInCluster, randomBoolean(), initialIndices);
-        final ClusterState newState = originalState; // doesn't matter for this test, just need a non-null value
-        final ClusterChangedEvent event = new ClusterChangedEvent("_na_", originalState, newState);
+        final ClusterState state = createState(numNodesInCluster, randomBoolean(), initialIndices);
 
         // test when its not the same IndexMetaData
         final Index index = initialIndices.get(0);
-        final IndexMetaData originalIndexMeta = originalState.metaData().index(index);
+        final IndexMetaData originalIndexMeta = state.metaData().index(index);
         // make sure the metadata is actually on the cluster state
         assertNotNull("IndexMetaData for " + index + " should exist on the cluster state", originalIndexMeta);
         IndexMetaData newIndexMeta = createIndexMetadata(index, originalIndexMeta.getVersion() + 1);
-        assertTrue("IndexMetaData with different version numbers must be considered changed", event.indexMetaDataChanged(newIndexMeta));
+        assertTrue("IndexMetaData with different version numbers must be considered changed",
+            ClusterChangedEvent.indexMetaDataChanged(originalIndexMeta, newIndexMeta));
 
         // test when it doesn't exist
         newIndexMeta = createIndexMetadata(new Index("doesntexist", UUIDs.randomBase64UUID()));
-        assertTrue("IndexMetaData that didn't previously exist should be considered changed", event.indexMetaDataChanged(newIndexMeta));
+        assertTrue("IndexMetaData that didn't previously exist should be considered changed",
+            ClusterChangedEvent.indexMetaDataChanged(originalIndexMeta, newIndexMeta));
 
         // test when its the same IndexMetaData
-        assertFalse("IndexMetaData should be the same", event.indexMetaDataChanged(originalIndexMeta));
+        assertFalse("IndexMetaData should be the same", ClusterChangedEvent.indexMetaDataChanged(originalIndexMeta, originalIndexMeta));
     }
 
     /**
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
index 307df91c302..1f39706e4f4 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
@@ -161,7 +161,7 @@ public class RandomAllocationDeciderTests extends ESAllocationTestCase {
         }
     }
 
-    private static final class RandomAllocationDecider extends AllocationDecider {
+    public static final class RandomAllocationDecider extends AllocationDecider {
 
         private final Random random;
 
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 8889ebfb910..40a23ee66cf 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -217,24 +217,14 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         ShardStateMetaData shardStateMetaData = load(logger, env.availableShardPaths(shard.shardId));
         assertEquals(getShardStateMetadata(shard), shardStateMetaData);
         ShardRouting routing = shard.shardRouting;
-        shard.updateRoutingEntry(routing, true);
+        shard.updateRoutingEntry(routing);
 
         shardStateMetaData = load(logger, env.availableShardPaths(shard.shardId));
         assertEquals(shardStateMetaData, getShardStateMetadata(shard));
         assertEquals(shardStateMetaData, new ShardStateMetaData(routing.primary(), shard.indexSettings().getUUID(), routing.allocationId()));
 
-        // check that we don't write shard state metadata if persist == false
-        ShardRouting updatedRouting = shard.shardRouting;
-        updatedRouting = TestShardRouting.relocate(shard.shardRouting, "some node", 42L);
-        shard.updateRoutingEntry(updatedRouting, false);
-        shardStateMetaData = load(logger, env.availableShardPaths(shard.shardId));
-        assertFalse("shard state persisted despite of persist=false", shardStateMetaData.equals(getShardStateMetadata(shard)));
-        assertEquals("shard state persisted despite of persist=false", shardStateMetaData, new ShardStateMetaData(routing.primary(), shard.indexSettings().getUUID(), routing.allocationId()));
-
-        // check that we write shard state metadata if persist == true
-        shard.updateRoutingEntry(routing, false); // move back state in IndexShard
-        routing = updatedRouting;
-        shard.updateRoutingEntry(routing, true);
+        routing = TestShardRouting.relocate(shard.shardRouting, "some node", 42L);
+        shard.updateRoutingEntry(routing);
         shardStateMetaData = load(logger, env.availableShardPaths(shard.shardId));
         assertEquals(shardStateMetaData, getShardStateMetadata(shard));
         assertEquals(shardStateMetaData, new ShardStateMetaData(routing.primary(), shard.indexSettings().getUUID(), routing.allocationId()));
@@ -336,7 +326,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
             // simulate promotion
             ShardRouting newReplicaShardRouting = TestShardRouting.newShardRouting(temp.shardId(), temp.currentNodeId(), null,
                 false, ShardRoutingState.STARTED, temp.allocationId());
-            indexShard.updateRoutingEntry(newReplicaShardRouting, false);
+            indexShard.updateRoutingEntry(newReplicaShardRouting);
             primaryTerm = primaryTerm + 1;
             indexShard.updatePrimaryTerm(primaryTerm);
             newPrimaryShardRouting = TestShardRouting.newShardRouting(temp.shardId(), temp.currentNodeId(), null,
@@ -344,7 +334,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         } else {
             newPrimaryShardRouting = temp;
         }
-        indexShard.updateRoutingEntry(newPrimaryShardRouting, false);
+        indexShard.updateRoutingEntry(newPrimaryShardRouting);
 
         assertEquals(0, indexShard.getActiveOperationsCount());
         if (newPrimaryShardRouting.isRelocationTarget() == false) {
@@ -381,7 +371,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
                 newShardRouting = TestShardRouting.newShardRouting(temp.shardId(), temp.currentNodeId(), null,
                     false, ShardRoutingState.STARTED, AllocationId.newRelocation(temp.allocationId()));
 
-                indexShard.updateRoutingEntry(newShardRouting, false);
+                indexShard.updateRoutingEntry(newShardRouting);
                 break;
             case 1:
                 // initializing replica / primary
@@ -391,13 +381,13 @@ public class IndexShardTests extends ESSingleNodeTestCase {
                     relocating ? randomBoolean() : false,
                     ShardRoutingState.INITIALIZING,
                     relocating ? AllocationId.newRelocation(temp.allocationId()) : temp.allocationId());
-                indexShard.updateRoutingEntry(newShardRouting, false);
+                indexShard.updateRoutingEntry(newShardRouting);
                 break;
             case 2:
                 // relocation source
                 newShardRouting = TestShardRouting.newShardRouting(temp.shardId(), temp.currentNodeId(), "otherNode",
                     false, ShardRoutingState.RELOCATING, AllocationId.newRelocation(temp.allocationId()));
-                indexShard.updateRoutingEntry(newShardRouting, false);
+                indexShard.updateRoutingEntry(newShardRouting);
                 indexShard.relocated("test");
                 break;
             default:
@@ -983,7 +973,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         test.removeShard(0, "b/c simon says so");
         routing = ShardRoutingHelper.reinit(routing);
         IndexShard newShard = test.createShard(routing);
-        newShard.updateRoutingEntry(routing, false);
+        newShard.updateRoutingEntry(routing);
         DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
         newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode));
         assertTrue(newShard.recoverFromStore());
@@ -991,7 +981,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         assertEquals(translogOps, newShard.recoveryState().getTranslog().totalOperations());
         assertEquals(translogOps, newShard.recoveryState().getTranslog().totalOperationsOnStart());
         assertEquals(100.0f, newShard.recoveryState().getTranslog().recoveredPercent(), 0.01f);
-        newShard.updateRoutingEntry(routing.moveToStarted(), true);
+        newShard.updateRoutingEntry(routing.moveToStarted());
         SearchResponse response = client().prepareSearch().get();
         assertHitCount(response, 1);
     }
@@ -1010,7 +1000,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         test.removeShard(0, "b/c simon says so");
         routing = ShardRoutingHelper.reinit(routing, UnassignedInfo.Reason.INDEX_CREATED);
         IndexShard newShard = test.createShard(routing);
-        newShard.updateRoutingEntry(routing, false);
+        newShard.updateRoutingEntry(routing);
         DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
         newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode,
             localNode));
@@ -1019,7 +1009,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         assertEquals(0, newShard.recoveryState().getTranslog().totalOperations());
         assertEquals(0, newShard.recoveryState().getTranslog().totalOperationsOnStart());
         assertEquals(100.0f, newShard.recoveryState().getTranslog().recoveredPercent(), 0.01f);
-        newShard.updateRoutingEntry(getInitializingShardRouting(routing).moveToStarted(), true);
+        newShard.updateRoutingEntry(getInitializingShardRouting(routing).moveToStarted());
         SearchResponse response = client().prepareSearch().get();
         assertHitCount(response, 0);
     }
@@ -1045,7 +1035,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         store.decRef();
         routing = ShardRoutingHelper.reinit(routing);
         IndexShard newShard = test.createShard(routing);
-        newShard.updateRoutingEntry(routing, false);
+        newShard.updateRoutingEntry(routing);
         newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode));
         try {
             newShard.recoverFromStore();
@@ -1065,11 +1055,11 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         }
         test.removeShard(0, "I broken it");
         newShard = test.createShard(routing);
-        newShard.updateRoutingEntry(routing, false);
+        newShard.updateRoutingEntry(routing);
         newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode));
         assertTrue("recover even if there is nothing to recover", newShard.recoverFromStore());
 
-        newShard.updateRoutingEntry(getInitializingShardRouting(routing).moveToStarted(), true);
+        newShard.updateRoutingEntry(getInitializingShardRouting(routing).moveToStarted());
         SearchResponse response = client().prepareSearch().get();
         assertHitCount(response, 0);
         // we can't issue this request through a client because of the inconsistencies we created with the cluster state
@@ -1090,11 +1080,11 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         ShardRouting origRouting = shard.routingEntry();
         assertThat(shard.state(), equalTo(IndexShardState.STARTED));
         ShardRouting inRecoveryRouting = ShardRoutingHelper.relocate(origRouting, "some_node");
-        shard.updateRoutingEntry(inRecoveryRouting, true);
+        shard.updateRoutingEntry(inRecoveryRouting);
         shard.relocated("simulate mark as relocated");
         assertThat(shard.state(), equalTo(IndexShardState.RELOCATED));
         try {
-            shard.updateRoutingEntry(origRouting, true);
+            shard.updateRoutingEntry(origRouting);
             fail("Expected IndexShardRelocatedException");
         } catch (IndexShardRelocatedException expected) {
         }
@@ -1123,7 +1113,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         Store sourceStore = test_shard.store();
         Store targetStore = test_target_shard.store();
 
-        test_target_shard.updateRoutingEntry(routing, false);
+        test_target_shard.updateRoutingEntry(routing);
         DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
         test_target_shard.markAsRecovering("store", new RecoveryState(routing.shardId(), routing.primary(), RecoveryState.Type.SNAPSHOT, routing.restoreSource(), localNode));
         assertTrue(test_target_shard.restoreFromRepository(new IndexShardRepository() {
@@ -1156,7 +1146,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
             }
         }));
 
-        test_target_shard.updateRoutingEntry(routing.moveToStarted(), true);
+        test_target_shard.updateRoutingEntry(routing.moveToStarted());
         assertHitCount(client().prepareSearch("test_target").get(), 1);
         assertSearchHits(client().prepareSearch("test_target").get(), "0");
     }
@@ -1411,7 +1401,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
         newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), newShard.routingEntry().primary(), RecoveryState.Type.STORE, localNode, localNode));
         assertTrue(newShard.recoverFromStore());
-        newShard.updateRoutingEntry(newShard.routingEntry().moveToStarted(), true);
+        newShard.updateRoutingEntry(newShard.routingEntry().moveToStarted());
         return newShard;
     }
 
@@ -1543,7 +1533,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
         {
             final IndexShard newShard = test.createShard(routing);
-            newShard.updateRoutingEntry(routing, false);
+            newShard.updateRoutingEntry(routing);
             newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.LOCAL_SHARDS, localNode, localNode));
 
             BiConsumer<String, MappingMetaData> mappingConsumer = (type, mapping) -> {
@@ -1575,7 +1565,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
                 }
             }
             routing = ShardRoutingHelper.moveToStarted(routing);
-            newShard.updateRoutingEntry(routing, true);
+            newShard.updateRoutingEntry(routing);
             assertHitCount(client().prepareSearch("index_1").get(), 2);
         }
         // now check that it's persistent ie. that the added shards are committed
@@ -1587,7 +1577,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
             newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.LOCAL_SHARDS, localNode, localNode));
             assertTrue(newShard.recoverFromStore());
             routing = ShardRoutingHelper.moveToStarted(routing);
-            newShard.updateRoutingEntry(routing, true);
+            newShard.updateRoutingEntry(routing);
             assertHitCount(client().prepareSearch("index_1").get(), 2);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
index 2722fc9d9d3..1f1b758f349 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
@@ -455,7 +455,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase {
             assertEquals(1, imc.availableShards().size());
             assertTrue(newShard.recoverFromStore());
             assertTrue("we should have flushed in IMC at least once but did: " + flushes.get(), flushes.get() >= 1);
-            newShard.updateRoutingEntry(routing.moveToStarted(), true);
+            newShard.updateRoutingEntry(routing.moveToStarted());
         } finally {
             newShard.close("simon says", false);
         }
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java
index 8d59da7da01..92a411a95de 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java
@@ -102,13 +102,13 @@ public class IndicesLifecycleListenerSingleNodeTests extends ESSingleNodeTestCas
             newRouting = ShardRoutingHelper.moveToUnassigned(newRouting, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "boom"));
             newRouting = ShardRoutingHelper.initialize(newRouting, nodeId);
             IndexShard shard = index.createShard(newRouting);
-            shard.updateRoutingEntry(newRouting, true);
+            shard.updateRoutingEntry(newRouting);
             final DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE,
                     emptyMap(), emptySet(), Version.CURRENT);
             shard.markAsRecovering("store", new RecoveryState(shard.shardId(), newRouting.primary(), RecoveryState.Type.SNAPSHOT, newRouting.restoreSource(), localNode));
             shard.recoverFromStore();
             newRouting = ShardRoutingHelper.moveToStarted(newRouting);
-            shard.updateRoutingEntry(newRouting, true);
+            shard.updateRoutingEntry(newRouting);
         } finally {
             indicesService.deleteIndex(idx, "simon says");
         }
diff --git a/core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java b/core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java
new file mode 100644
index 00000000000..69bee510710
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java
@@ -0,0 +1,318 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.indices.cluster;
+
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.RoutingNode;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.Callback;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.NodeServicesProvider;
+import org.elasticsearch.index.shard.IndexEventListener;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.IndexShardState;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndex;
+import org.elasticsearch.indices.cluster.IndicesClusterStateService.Shard;
+import org.elasticsearch.indices.cluster.IndicesClusterStateService.AllocatedIndices;
+import org.elasticsearch.indices.recovery.RecoveryState;
+import org.elasticsearch.indices.recovery.RecoveryTargetService;
+import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.unmodifiableMap;
+import static org.elasticsearch.common.collect.MapBuilder.newMapBuilder;
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * Abstract base class for tests against {@link IndicesClusterStateService}
+ */
+public abstract class AbstractIndicesClusterStateServiceTestCase extends ESTestCase {
+
+
+    protected void failRandomly() {
+        if (rarely()) {
+            throw new RuntimeException("dummy test failure");
+        }
+    }
+
+    /**
+     * Checks if cluster state matches internal state of IndicesClusterStateService instance
+     *
+     * @param state cluster state used for matching
+     */
+    public static void assertClusterStateMatchesNodeState(ClusterState state, IndicesClusterStateService indicesClusterStateService) {
+        AllocatedIndices<? extends Shard, ? extends AllocatedIndex<? extends Shard>> indicesService =
+            indicesClusterStateService.indicesService;
+        ConcurrentMap<ShardId, ShardRouting> failedShardsCache = indicesClusterStateService.failedShardsCache;
+        RoutingNode localRoutingNode = state.getRoutingNodes().node(state.getNodes().getLocalNodeId());
+        if (localRoutingNode != null) {
+            // check that all shards in local routing nodes have been allocated
+            for (ShardRouting shardRouting : localRoutingNode) {
+                Index index = shardRouting.index();
+                IndexMetaData indexMetaData = state.metaData().getIndexSafe(index);
+
+                Shard shard = indicesService.getShardOrNull(shardRouting.shardId());
+                ShardRouting failedShard = failedShardsCache.get(shardRouting.shardId());
+                if (shard == null && failedShard == null) {
+                    fail("Shard with id " + shardRouting + " expected but missing in indicesService and failedShardsCache");
+                }
+                if (failedShard != null && failedShard.isSameAllocation(shardRouting) == false) {
+                    fail("Shard cache has not been properly cleaned for " + failedShard);
+                }
+
+                if (shard != null) {
+                    AllocatedIndex<? extends Shard> indexService = indicesService.indexService(index);
+                    assertTrue("Index " + index + " expected but missing in indicesService", indexService != null);
+
+                    // index metadata has been updated
+                    assertThat(indexService.getIndexSettings().getIndexMetaData(), equalTo(indexMetaData));
+                    // shard has been created
+                    if (failedShard == null) {
+                        assertTrue("Shard with id " + shardRouting + " expected but missing in indexService",
+                            shard != null);
+                        // shard has latest shard routing
+                        assertThat(shard.routingEntry(), equalTo(shardRouting));
+                    }
+                }
+            }
+        }
+
+        // all other shards / indices have been cleaned up
+        for (AllocatedIndex<? extends Shard> indexService : indicesService) {
+            assertTrue(state.metaData().getIndexSafe(indexService.index()) != null);
+
+            boolean shardsFound = false;
+            for (Shard shard : indexService) {
+                shardsFound = true;
+                ShardRouting persistedShardRouting = shard.routingEntry();
+                boolean found = false;
+                for (ShardRouting shardRouting : localRoutingNode) {
+                    if (persistedShardRouting.equals(shardRouting)) {
+                        found = true;
+                    }
+                }
+                assertTrue(found);
+            }
+
+            if (shardsFound == false) {
+                // check if we have shards of that index in failedShardsCache
+                // if yes, we might not have cleaned the index as failedShardsCache can be populated by another thread
+                assertFalse(failedShardsCache.keySet().stream().noneMatch(shardId -> shardId.getIndex().equals(indexService.index())));
+            }
+
+        }
+    }
+
+    /**
+     * Mock for {@link IndicesService}
+     */
+    protected class MockIndicesService implements AllocatedIndices<MockIndexShard, MockIndexService> {
+        private volatile Map<String, MockIndexService> indices = emptyMap();
+
+        @Override
+        public synchronized MockIndexService createIndex(NodeServicesProvider nodeServicesProvider, IndexMetaData indexMetaData,
+                                                         List<IndexEventListener> buildInIndexListener) throws IOException {
+            MockIndexService indexService = new MockIndexService(new IndexSettings(indexMetaData, Settings.EMPTY));
+            indices = newMapBuilder(indices).put(indexMetaData.getIndexUUID(), indexService).immutableMap();
+            return indexService;
+        }
+
+        @Override
+        public IndexMetaData verifyIndexIsDeleted(Index index, ClusterState state) {
+            return null;
+        }
+
+        @Override
+        public void deleteUnassignedIndex(String reason, IndexMetaData metaData, ClusterState clusterState) {
+
+        }
+
+        @Override
+        public synchronized void deleteIndex(Index index, String reason) {
+            if (hasIndex(index) == false) {
+                return;
+            }
+            Map<String, MockIndexService> newIndices = new HashMap<>(indices);
+            newIndices.remove(index.getUUID());
+            indices = unmodifiableMap(newIndices);
+        }
+
+        @Override
+        public synchronized void removeIndex(Index index, String reason) {
+            if (hasIndex(index) == false) {
+                return;
+            }
+            Map<String, MockIndexService> newIndices = new HashMap<>(indices);
+            newIndices.remove(index.getUUID());
+            indices = unmodifiableMap(newIndices);
+        }
+
+        @Override
+        public @Nullable MockIndexService indexService(Index index) {
+            return indices.get(index.getUUID());
+        }
+
+        @Override
+        public MockIndexShard createShard(ShardRouting shardRouting, RecoveryState recoveryState,
+                                          RecoveryTargetService recoveryTargetService,
+                                          RecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService,
+                                          NodeServicesProvider nodeServicesProvider, Callback<IndexShard.ShardFailure> onShardFailure)
+            throws IOException {
+            failRandomly();
+            MockIndexService indexService = indexService(recoveryState.getShardId().getIndex());
+            MockIndexShard indexShard = indexService.createShard(shardRouting);
+            indexShard.recoveryState = recoveryState;
+            return indexShard;
+        }
+
+        @Override
+        public void processPendingDeletes(Index index, IndexSettings indexSettings, TimeValue timeValue) throws IOException,
+            InterruptedException {
+
+        }
+
+        private boolean hasIndex(Index index) {
+            return indices.containsKey(index.getUUID());
+        }
+
+        @Override
+        public Iterator<MockIndexService> iterator() {
+            return indices.values().iterator();
+        }
+    }
+
+    /**
+     * Mock for {@link IndexService}
+     */
+    protected class MockIndexService implements AllocatedIndex<MockIndexShard> {
+        private volatile Map<Integer, MockIndexShard> shards = emptyMap();
+
+        private final IndexSettings indexSettings;
+
+        public MockIndexService(IndexSettings indexSettings) {
+            this.indexSettings = indexSettings;
+        }
+
+        @Override
+        public IndexSettings getIndexSettings() {
+            return indexSettings;
+        }
+
+        @Override
+        public boolean updateMapping(IndexMetaData indexMetaData) throws IOException {
+            failRandomly();
+            return false;
+        }
+
+        @Override
+        public void updateMetaData(IndexMetaData indexMetaData) {
+            indexSettings.updateIndexMetaData(indexMetaData);
+        }
+
+        @Override
+        public MockIndexShard getShardOrNull(int shardId) {
+            return shards.get(shardId);
+        }
+
+        public synchronized MockIndexShard createShard(ShardRouting routing) throws IOException {
+            failRandomly();
+            MockIndexShard shard = new MockIndexShard(routing);
+            shards = newMapBuilder(shards).put(routing.id(), shard).immutableMap();
+            return shard;
+        }
+
+        @Override
+        public synchronized void removeShard(int shardId, String reason) {
+            if (shards.containsKey(shardId) == false) {
+                return;
+            }
+            HashMap<Integer, MockIndexShard> newShards = new HashMap<>(shards);
+            MockIndexShard indexShard = newShards.remove(shardId);
+            assert indexShard != null;
+            shards = unmodifiableMap(newShards);
+        }
+
+        @Override
+        public Iterator<MockIndexShard> iterator() {
+            return shards.values().iterator();
+        }
+
+        @Override
+        public Index index() {
+            return indexSettings.getIndex();
+        }
+    }
+
+    /**
+     * Mock for {@link IndexShard}
+     */
+    protected class MockIndexShard implements IndicesClusterStateService.Shard {
+        private volatile ShardRouting shardRouting;
+        private volatile RecoveryState recoveryState;
+
+        public MockIndexShard(ShardRouting shardRouting) {
+            this.shardRouting = shardRouting;
+        }
+
+        @Override
+        public ShardId shardId() {
+            return shardRouting.shardId();
+        }
+
+        @Override
+        public RecoveryState recoveryState() {
+            return recoveryState;
+        }
+
+        @Override
+        public ShardRouting routingEntry() {
+            return shardRouting;
+        }
+
+        @Override
+        public IndexShardState state() {
+            return null;
+        }
+
+        @Override
+        public void updateRoutingEntry(ShardRouting shardRouting) throws IOException {
+            failRandomly();
+            assert this.shardId().equals(shardRouting.shardId());
+            assert this.shardRouting.isSameAllocation(shardRouting);
+            this.shardRouting = shardRouting;
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
new file mode 100644
index 00000000000..84e83db6d1d
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.indices.cluster;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest;
+import org.elasticsearch.action.admin.cluster.reroute.TransportClusterRerouteAction;
+import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
+import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction;
+import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
+import org.elasticsearch.action.admin.indices.open.TransportOpenIndexAction;
+import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettingsAction;
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.DestructiveOperations;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.action.support.master.MasterNodeRequest;
+import org.elasticsearch.action.support.master.TransportMasterNodeAction;
+import org.elasticsearch.action.support.master.TransportMasterNodeActionUtils;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.metadata.AliasValidator;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
+import org.elasticsearch.cluster.metadata.MetaDataDeleteIndexService;
+import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
+import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
+import org.elasticsearch.cluster.metadata.MetaDataUpdateSettingsService;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
+import org.elasticsearch.cluster.routing.allocation.RandomAllocationDeciderTests;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.NodeServicesProvider;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.shard.IndexEventListener;
+import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.test.gateway.NoopGatewayAllocator;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.Transport;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+
+import static com.carrotsearch.randomizedtesting.RandomizedTest.getRandom;
+import static org.elasticsearch.env.Environment.PATH_HOME_SETTING;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Matchers.anyList;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ClusterStateChanges {
+
+    private final ClusterService clusterService;
+    private final AllocationService allocationService;
+
+    // transport actions
+    private final TransportCloseIndexAction transportCloseIndexAction;
+    private final TransportOpenIndexAction transportOpenIndexAction;
+    private final TransportDeleteIndexAction transportDeleteIndexAction;
+    private final TransportUpdateSettingsAction transportUpdateSettingsAction;
+    private final TransportClusterRerouteAction transportClusterRerouteAction;
+    private final TransportCreateIndexAction transportCreateIndexAction;
+
+    public ClusterStateChanges() {
+        Settings settings = Settings.builder().put(PATH_HOME_SETTING.getKey(), "dummy").build();
+
+        allocationService = new AllocationService(settings, new AllocationDeciders(settings,
+            new HashSet<>(Arrays.asList(new SameShardAllocationDecider(settings),
+                new ReplicaAfterPrimaryActiveAllocationDecider(settings),
+                new RandomAllocationDeciderTests.RandomAllocationDecider(getRandom())))),
+            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(settings),
+            EmptyClusterInfoService.INSTANCE);
+        ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        ActionFilters actionFilters = new ActionFilters(Collections.emptySet());
+        IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(settings);
+        DestructiveOperations destructiveOperations = new DestructiveOperations(settings, clusterSettings);
+        Environment environment = new Environment(settings);
+        ThreadPool threadPool = null; // it's not used
+        Transport transport = null; // it's not used
+
+        // mocks
+        clusterService = mock(ClusterService.class);
+        IndicesService indicesService = mock(IndicesService.class);
+        // MetaDataCreateIndexService creates indices using its IndicesService instance to check mappings -> fake it here
+        try {
+            when(indicesService.createIndex(any(NodeServicesProvider.class), any(IndexMetaData.class), anyList()))
+                .then(invocationOnMock -> {
+                    IndexService indexService = mock(IndexService.class);
+                    IndexMetaData indexMetaData = (IndexMetaData)invocationOnMock.getArguments()[1];
+                    when(indexService.index()).thenReturn(indexMetaData.getIndex());
+                    MapperService mapperService = mock(MapperService.class);
+                    when(indexService.mapperService()).thenReturn(mapperService);
+                    when(mapperService.docMappers(anyBoolean())).thenReturn(Collections.emptyList());
+                    when(indexService.getIndexEventListener()).thenReturn(new IndexEventListener() {});
+                    return indexService;
+            });
+        } catch (IOException e) {
+            throw new IllegalStateException(e);
+        }
+
+        // services
+        TransportService transportService = new TransportService(settings, transport, threadPool, null);
+        MetaDataIndexUpgradeService metaDataIndexUpgradeService = new MetaDataIndexUpgradeService(settings, null, null) {
+            // metaData upgrader should do nothing
+            @Override
+            public IndexMetaData upgradeIndexMetaData(IndexMetaData indexMetaData) {
+                return indexMetaData;
+            }
+        };
+        NodeServicesProvider nodeServicesProvider = new NodeServicesProvider(threadPool, null, null, null, null, null, clusterService);
+        MetaDataIndexStateService indexStateService = new MetaDataIndexStateService(settings, clusterService, allocationService,
+            metaDataIndexUpgradeService, nodeServicesProvider, indicesService);
+        MetaDataDeleteIndexService deleteIndexService = new MetaDataDeleteIndexService(settings, clusterService, allocationService);
+        MetaDataUpdateSettingsService metaDataUpdateSettingsService = new MetaDataUpdateSettingsService(settings, clusterService,
+            allocationService, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, new IndexNameExpressionResolver(settings));
+        MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(settings, clusterService, indicesService,
+            allocationService, Version.CURRENT, new AliasValidator(settings), Collections.emptySet(), environment,
+            nodeServicesProvider, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
+
+        transportCloseIndexAction = new TransportCloseIndexAction(settings, transportService, clusterService, threadPool,
+            indexStateService, clusterSettings, actionFilters, indexNameExpressionResolver, destructiveOperations);
+        transportOpenIndexAction = new TransportOpenIndexAction(settings, transportService,
+            clusterService, threadPool, indexStateService, actionFilters, indexNameExpressionResolver, destructiveOperations);
+        transportDeleteIndexAction = new TransportDeleteIndexAction(settings, transportService,
+            clusterService, threadPool, deleteIndexService, actionFilters, indexNameExpressionResolver, destructiveOperations);
+        transportUpdateSettingsAction = new TransportUpdateSettingsAction(settings,
+            transportService, clusterService, threadPool, metaDataUpdateSettingsService, actionFilters, indexNameExpressionResolver);
+        transportClusterRerouteAction = new TransportClusterRerouteAction(settings,
+            transportService, clusterService, threadPool, allocationService, actionFilters, indexNameExpressionResolver);
+        transportCreateIndexAction = new TransportCreateIndexAction(settings,
+            transportService, clusterService, threadPool, createIndexService, actionFilters, indexNameExpressionResolver);
+    }
+
+    public ClusterState createIndex(ClusterState state, CreateIndexRequest request) {
+        return execute(transportCreateIndexAction, request, state);
+    }
+
+    public ClusterState closeIndices(ClusterState state, CloseIndexRequest request) {
+        return execute(transportCloseIndexAction, request, state);
+    }
+
+    public ClusterState openIndices(ClusterState state, OpenIndexRequest request) {
+        return execute(transportOpenIndexAction, request, state);
+    }
+
+    public ClusterState deleteIndices(ClusterState state, DeleteIndexRequest request) {
+        return execute(transportDeleteIndexAction, request, state);
+    }
+
+    public ClusterState updateSettings(ClusterState state, UpdateSettingsRequest request) {
+        return execute(transportUpdateSettingsAction, request, state);
+    }
+
+    public ClusterState reroute(ClusterState state, ClusterRerouteRequest request) {
+        return execute(transportClusterRerouteAction, request, state);
+    }
+
+    public ClusterState applyFailedShards(ClusterState clusterState, List<FailedRerouteAllocation.FailedShard> failedShards) {
+        RoutingAllocation.Result rerouteResult = allocationService.applyFailedShards(clusterState, failedShards);
+        return ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+    }
+
+    public ClusterState applyStartedShards(ClusterState clusterState, List<ShardRouting> startedShards) {
+        RoutingAllocation.Result rerouteResult = allocationService.applyStartedShards(clusterState, startedShards);
+        return ClusterState.builder(clusterState).routingResult(rerouteResult).build();
+    }
+
+    private <Request extends MasterNodeRequest<Request>, Response extends ActionResponse> ClusterState execute(
+        TransportMasterNodeAction<Request, Response> masterNodeAction, Request request, ClusterState clusterState) {
+        return executeClusterStateUpdateTask(clusterState, () -> {
+            try {
+                TransportMasterNodeActionUtils.runMasterOperation(masterNodeAction, request, clusterState, new PlainActionFuture<>());
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        });
+    }
+
+    private ClusterState executeClusterStateUpdateTask(ClusterState state, Runnable runnable) {
+        ClusterState[] result = new ClusterState[1];
+        doAnswer(invocationOnMock -> {
+            ClusterStateUpdateTask task = (ClusterStateUpdateTask)invocationOnMock.getArguments()[1];
+            result[0] = task.execute(state);
+            return null;
+        }).when(clusterService).submitStateUpdateTask(anyString(), any(ClusterStateUpdateTask.class));
+        runnable.run();
+        assertThat(result[0], notNullValue());
+        return result[0];
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
new file mode 100644
index 00000000000..8c63c001a1e
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
@@ -0,0 +1,281 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.indices.cluster;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest;
+import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
+import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
+import org.elasticsearch.cluster.ClusterChangedEvent;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.action.shard.ShardStateAction;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation.FailedShard;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.indices.recovery.RecoveryTargetService;
+import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.Executor;
+
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
+import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class IndicesClusterStateServiceRandomUpdatesTests extends AbstractIndicesClusterStateServiceTestCase {
+
+    private final ClusterStateChanges cluster = new ClusterStateChanges();
+
+    public void testRandomClusterStateUpdates() {
+        // we have an IndicesClusterStateService per node in the cluster
+        final Map<DiscoveryNode, IndicesClusterStateService> clusterStateServiceMap = new HashMap<>();
+        ClusterState state = randomInitialClusterState(clusterStateServiceMap);
+
+        // each of the following iterations represents a new cluster state update processed on all nodes
+        for (int i = 0; i < 30; i++) {
+            logger.info("Iteration {}", i);
+            final ClusterState previousState = state;
+
+            // calculate new cluster state
+            for (int j = 0; j < randomInt(3); j++) { // multiple iterations to simulate batching of cluster states
+                state = randomlyUpdateClusterState(state, clusterStateServiceMap);
+            }
+
+            // apply cluster state to nodes (incl. master)
+            for (DiscoveryNode node : state.nodes()) {
+                IndicesClusterStateService indicesClusterStateService = clusterStateServiceMap.get(node);
+                ClusterState localState = adaptClusterStateToLocalNode(state, node);
+                ClusterState previousLocalState = adaptClusterStateToLocalNode(previousState, node);
+                indicesClusterStateService.clusterChanged(new ClusterChangedEvent("simulated change " + i, localState, previousLocalState));
+
+                // check that cluster state has been properly applied to node
+                assertClusterStateMatchesNodeState(localState, indicesClusterStateService);
+            }
+        }
+
+        // TODO: check if we can go to green by starting all shards and finishing all iterations
+        logger.info("Final cluster state: {}", state.prettyPrint());
+    }
+
+    public ClusterState randomInitialClusterState(Map<DiscoveryNode, IndicesClusterStateService> clusterStateServiceMap) {
+        List<DiscoveryNode> allNodes = new ArrayList<>();
+        DiscoveryNode localNode = createNode(DiscoveryNode.Role.MASTER); // local node is the master
+        allNodes.add(localNode);
+        // at least two nodes that have the data role so that we can allocate shards
+        allNodes.add(createNode(DiscoveryNode.Role.DATA));
+        allNodes.add(createNode(DiscoveryNode.Role.DATA));
+        for (int i = 0; i < randomIntBetween(2, 5); i++) {
+            allNodes.add(createNode());
+        }
+        ClusterState state = ClusterStateCreationUtils.state(localNode, localNode, allNodes.toArray(new DiscoveryNode[allNodes.size()]));
+        // add nodes to clusterStateServiceMap
+        updateNodes(state, clusterStateServiceMap);
+        return state;
+    }
+
+    private void updateNodes(ClusterState state, Map<DiscoveryNode, IndicesClusterStateService> clusterStateServiceMap) {
+        for (DiscoveryNode node : state.nodes()) {
+            clusterStateServiceMap.computeIfAbsent(node, discoveryNode -> {
+                IndicesClusterStateService ics = createIndicesClusterStateService();
+                ics.start();
+                return ics;
+            });
+        }
+
+        for (Iterator<Entry<DiscoveryNode, IndicesClusterStateService>> it = clusterStateServiceMap.entrySet().iterator(); it.hasNext(); ) {
+            DiscoveryNode node = it.next().getKey();
+            if (state.nodes().nodeExists(node.getId()) == false) {
+                it.remove();
+            }
+        }
+    }
+
+    public ClusterState randomlyUpdateClusterState(ClusterState state,
+                                                   Map<DiscoveryNode, IndicesClusterStateService> clusterStateServiceMap) {
+        // randomly create new indices (until we have 200 max)
+        for (int i = 0; i < randomInt(5); i++) {
+            if (state.metaData().indices().size() > 200) {
+                break;
+            }
+            String name = "index_" + randomAsciiOfLength(15).toLowerCase(Locale.ROOT);
+            CreateIndexRequest request = new CreateIndexRequest(name, Settings.builder()
+                .put(SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 3))
+                .put(SETTING_NUMBER_OF_REPLICAS, randomInt(2))
+                .build());
+            state = cluster.createIndex(state, request);
+            assertTrue(state.metaData().hasIndex(name));
+        }
+
+        // randomly delete indices
+        Set<String> indicesToDelete = new HashSet<>();
+        int numberOfIndicesToDelete = randomInt(Math.min(2, state.metaData().indices().size()));
+        for (String index : randomSubsetOf(numberOfIndicesToDelete, state.metaData().indices().keys().toArray(String.class))) {
+            indicesToDelete.add(state.metaData().index(index).getIndex().getName());
+        }
+        if (indicesToDelete.isEmpty() == false) {
+            DeleteIndexRequest deleteRequest = new DeleteIndexRequest(indicesToDelete.toArray(new String[indicesToDelete.size()]));
+            state = cluster.deleteIndices(state, deleteRequest);
+            for (String index : indicesToDelete) {
+                assertFalse(state.metaData().hasIndex(index));
+            }
+        }
+
+        // randomly close indices
+        int numberOfIndicesToClose = randomInt(Math.min(1, state.metaData().indices().size()));
+        for (String index : randomSubsetOf(numberOfIndicesToClose, state.metaData().indices().keys().toArray(String.class))) {
+            CloseIndexRequest closeIndexRequest = new CloseIndexRequest(state.metaData().index(index).getIndex().getName());
+            state = cluster.closeIndices(state, closeIndexRequest);
+        }
+
+        // randomly open indices
+        int numberOfIndicesToOpen = randomInt(Math.min(1, state.metaData().indices().size()));
+        for (String index : randomSubsetOf(numberOfIndicesToOpen, state.metaData().indices().keys().toArray(String.class))) {
+            OpenIndexRequest openIndexRequest = new OpenIndexRequest(state.metaData().index(index).getIndex().getName());
+            state = cluster.openIndices(state, openIndexRequest);
+        }
+
+        // randomly update settings
+        Set<String> indicesToUpdate = new HashSet<>();
+        boolean containsClosedIndex = false;
+        int numberOfIndicesToUpdate = randomInt(Math.min(2, state.metaData().indices().size()));
+        for (String index : randomSubsetOf(numberOfIndicesToUpdate, state.metaData().indices().keys().toArray(String.class))) {
+            indicesToUpdate.add(state.metaData().index(index).getIndex().getName());
+            if (state.metaData().index(index).getState() == IndexMetaData.State.CLOSE) {
+                containsClosedIndex = true;
+            }
+        }
+        if (indicesToUpdate.isEmpty() == false) {
+            UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(
+                indicesToUpdate.toArray(new String[indicesToUpdate.size()]));
+            Settings.Builder settings = Settings.builder();
+            if (containsClosedIndex == false) {
+                settings.put(SETTING_NUMBER_OF_REPLICAS, randomInt(2));
+            }
+            settings.put("index.refresh_interval", randomIntBetween(1, 5) + "s");
+            updateSettingsRequest.settings(settings.build());
+            state = cluster.updateSettings(state, updateSettingsRequest);
+        }
+
+        // randomly reroute
+        if (rarely()) {
+            state = cluster.reroute(state, new ClusterRerouteRequest());
+        }
+
+        // randomly start and fail allocated shards
+        List<ShardRouting> startedShards = new ArrayList<>();
+        List<FailedShard> failedShards = new ArrayList<>();
+        for (DiscoveryNode node : state.nodes()) {
+            IndicesClusterStateService indicesClusterStateService = clusterStateServiceMap.get(node);
+            MockIndicesService indicesService = (MockIndicesService) indicesClusterStateService.indicesService;
+            for (MockIndexService indexService : indicesService) {
+                for (MockIndexShard indexShard : indexService) {
+                    ShardRouting persistedShardRouting = indexShard.routingEntry();
+                    if (persistedShardRouting.initializing() && randomBoolean()) {
+                        startedShards.add(persistedShardRouting);
+                    } else if (rarely()) {
+                        failedShards.add(new FailedShard(persistedShardRouting, "fake shard failure", new Exception()));
+                    }
+                }
+            }
+        }
+        state = cluster.applyFailedShards(state, failedShards);
+        state = cluster.applyStartedShards(state, startedShards);
+
+        // randomly add and remove nodes (except current master)
+        if (rarely()) {
+            if (randomBoolean()) {
+                // add node
+                if (state.nodes().getSize() < 10) {
+                    DiscoveryNodes newNodes = DiscoveryNodes.builder(state.nodes()).put(createNode()).build();
+                    state = ClusterState.builder(state).nodes(newNodes).build();
+                    state = cluster.reroute(state, new ClusterRerouteRequest()); // always reroute after node leave
+                    updateNodes(state, clusterStateServiceMap);
+                }
+            } else {
+                // remove node
+                if (state.nodes().getDataNodes().size() > 3) {
+                    DiscoveryNode discoveryNode = randomFrom(state.nodes().getNodes().values().toArray(DiscoveryNode.class));
+                    if (discoveryNode.equals(state.nodes().getMasterNode()) == false) {
+                        DiscoveryNodes newNodes = DiscoveryNodes.builder(state.nodes()).remove(discoveryNode.getId()).build();
+                        state = ClusterState.builder(state).nodes(newNodes).build();
+                        state = cluster.reroute(state, new ClusterRerouteRequest()); // always reroute after node join
+                        updateNodes(state, clusterStateServiceMap);
+                    }
+                }
+            }
+        }
+
+        // TODO: go masterless?
+
+        return state;
+    }
+
+    protected DiscoveryNode createNode(DiscoveryNode.Role... mustHaveRoles) {
+        Set<DiscoveryNode.Role> roles = new HashSet<>(randomSubsetOf(Sets.newHashSet(DiscoveryNode.Role.values())));
+        for (DiscoveryNode.Role mustHaveRole : mustHaveRoles) {
+            roles.add(mustHaveRole);
+        }
+        return new DiscoveryNode("node_" + randomAsciiOfLength(8), DummyTransportAddress.INSTANCE, Collections.emptyMap(), roles,
+            Version.CURRENT);
+    }
+
+    private static ClusterState adaptClusterStateToLocalNode(ClusterState state, DiscoveryNode node) {
+        return ClusterState.builder(state).nodes(DiscoveryNodes.builder(state.nodes()).localNodeId(node.getId())).build();
+    }
+
+    private IndicesClusterStateService createIndicesClusterStateService() {
+        final ThreadPool threadPool = mock(ThreadPool.class);
+        final Executor executor = mock(Executor.class);
+        when(threadPool.generic()).thenReturn(executor);
+        final MockIndicesService indicesService = new MockIndicesService();
+        final TransportService transportService = new TransportService(Settings.EMPTY, null, threadPool, null);
+        final ClusterService clusterService = mock(ClusterService.class);
+        final RepositoriesService repositoriesService = new RepositoriesService(Settings.EMPTY, clusterService,
+            transportService, null, null);
+        final RecoveryTargetService recoveryTargetService = new RecoveryTargetService(Settings.EMPTY, threadPool,
+            transportService, null, clusterService);
+        final ShardStateAction shardStateAction = mock(ShardStateAction.class);
+        return new IndicesClusterStateService(Settings.EMPTY, indicesService, clusterService,
+            threadPool, recoveryTargetService, shardStateAction, null, repositoriesService, null, null, null, null, null);
+    }
+
+}

From a2c506acd3d841ac279668fbe52a54b847d0d5cd Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Fri, 10 Jun 2016 13:39:47 +0200
Subject: [PATCH 0772/1311] Fix sync flush total shards statistics (#18766)

---
 .../indices/flush/SyncedFlushService.java           | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java b/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
index f3cb76199dc..16e4d62a721 100644
--- a/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
+++ b/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
@@ -114,11 +114,9 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL
         final ClusterState state = clusterService.state();
         final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, indicesOptions, aliasesOrIndices);
         final Map<String, List<ShardsSyncedFlushResult>> results = ConcurrentCollections.newConcurrentMap();
-        int totalNumberOfShards = 0;
         int numberOfShards = 0;
         for (Index index : concreteIndices) {
             final IndexMetaData indexMetaData = state.metaData().getIndexSafe(index);
-            totalNumberOfShards += indexMetaData.getTotalNumberOfShards();
             numberOfShards += indexMetaData.getNumberOfShards();
             results.put(index.getName(), Collections.synchronizedList(new ArrayList<>()));
 
@@ -127,7 +125,6 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL
             listener.onResponse(new SyncedFlushResponse(results));
             return;
         }
-        final int finalTotalNumberOfShards = totalNumberOfShards;
         final CountDown countDown = new CountDown(numberOfShards);
 
         for (final Index concreteIndex : concreteIndices) {
@@ -136,7 +133,7 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL
             final int indexNumberOfShards = indexMetaData.getNumberOfShards();
             for (int shard = 0; shard < indexNumberOfShards; shard++) {
                 final ShardId shardId = new ShardId(indexMetaData.getIndex(), shard);
-                attemptSyncedFlush(shardId, new ActionListener<ShardsSyncedFlushResult>() {
+                innerAttemptSyncedFlush(shardId, state, new ActionListener<ShardsSyncedFlushResult>() {
                     @Override
                     public void onResponse(ShardsSyncedFlushResult syncedFlushResult) {
                         results.get(index).add(syncedFlushResult);
@@ -148,7 +145,8 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL
                     @Override
                     public void onFailure(Throwable e) {
                         logger.debug("{} unexpected error while executing synced flush", shardId);
-                        results.get(index).add(new ShardsSyncedFlushResult(shardId, finalTotalNumberOfShards, e.getMessage()));
+                        final int totalShards = indexMetaData.getNumberOfReplicas() + 1;
+                        results.get(index).add(new ShardsSyncedFlushResult(shardId, totalShards, e.getMessage()));
                         if (countDown.countDown()) {
                             listener.onResponse(new SyncedFlushResponse(results));
                         }
@@ -185,8 +183,11 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL
     * Synced flush is a best effort operation. The sync id may be written on all, some or none of the copies.
     **/
     public void attemptSyncedFlush(final ShardId shardId, final ActionListener<ShardsSyncedFlushResult> actionListener) {
+        innerAttemptSyncedFlush(shardId, clusterService.state(), actionListener);
+    }
+
+    private void innerAttemptSyncedFlush(final ShardId shardId, final ClusterState state, final ActionListener<ShardsSyncedFlushResult> actionListener) {
         try {
-            final ClusterState state = clusterService.state();
             final IndexShardRoutingTable shardRoutingTable = getShardRoutingTable(shardId, state);
             final List<ShardRouting> activeShards = shardRoutingTable.activeShards();
             final int totalShards = shardRoutingTable.getSize();

From d733fb689b1a7182a5f62d482ee1080b1779eae5 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 9 Jun 2016 15:05:37 -0400
Subject: [PATCH 0773/1311] Better error message when mapping configures null

Closes #18803
---
 .../index/mapper/core/TypeParsers.java           |  7 +++++++
 .../index/mapper/core/DateFieldMapperTests.java  | 15 +++++++++++++++
 .../index/mapper/core/TextFieldMapperTests.java  | 16 ++++++++++++++++
 3 files changed, 38 insertions(+)

diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java b/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java
index c72d746158b..8e6b538a2ae 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java
@@ -239,6 +239,13 @@ public class TypeParsers {
             Map.Entry<String, Object> entry = iterator.next();
             final String propName = entry.getKey();
             final Object propNode = entry.getValue();
+            if (false == propName.equals("null_value") && propNode == null) {
+                /*
+                 * No properties *except* null_value are allowed to have null. So we catch it here and tell the user something useful rather
+                 * than send them a null pointer exception later.
+                 */
+                throw new MapperParsingException("[" + propName + "] must not have a [null] value");
+            }
             if (propName.equals("store")) {
                 builder.store(parseStore(name, propNode.toString(), parserContext));
                 iterator.remove();
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/DateFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/DateFieldMapperTests.java
index ee19d094a3f..a3909637548 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/core/DateFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/core/DateFieldMapperTests.java
@@ -302,4 +302,19 @@ public class DateFieldMapperTests extends ESSingleNodeTestCase {
         assertEquals(1457654400000L, dvField.numericValue().longValue());
         assertFalse(dvField.fieldType().stored());
     }
+
+    public void testNullConfigValuesFail() throws MapperParsingException, IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("field")
+                            .field("type", "date")
+                            .field("format", (String) null)
+                        .endObject()
+                    .endObject()
+                .endObject().endObject().string();
+
+        Exception e = expectThrows(MapperParsingException.class, () -> parser.parse("type", new CompressedXContent(mapping)));
+        assertEquals("[format] must not have a [null] value", e.getMessage());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/TextFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/TextFieldMapperTests.java
index de14f38d6a9..224d512cb53 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/core/TextFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/core/TextFieldMapperTests.java
@@ -35,6 +35,7 @@ import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperParser;
+import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.index.mapper.core.TextFieldMapper.TextFieldType;
 import org.elasticsearch.index.mapper.ParsedDocument;
@@ -458,4 +459,19 @@ public class TextFieldMapperTests extends ESSingleNodeTestCase {
         assertThat(fieldType.fielddataMaxFrequency(), equalTo((double) Integer.MAX_VALUE));
         assertThat(fieldType.fielddataMinSegmentSize(), equalTo(1000));
     }
+
+    public void testNullConfigValuesFail() throws MapperParsingException, IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject()
+                .startObject("type")
+                    .startObject("properties")
+                        .startObject("field")
+                            .field("type", "text")
+                            .field("analyzer", (String) null)
+                        .endObject()
+                    .endObject()
+                .endObject().endObject().string();
+
+        Exception e = expectThrows(MapperParsingException.class, () -> parser.parse("type", new CompressedXContent(mapping)));
+        assertEquals("[analyzer] must not have a [null] value", e.getMessage());
+    }
 }

From 43e07c0c886223c65d26a4e3bdb7815ff15abe7d Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Fri, 10 Jun 2016 09:56:57 -0400
Subject: [PATCH 0774/1311] Better handling of an empty shard's segments_N file

When trying to restore a snapshot of an index created in a previous
version of Elasticsearch, it is possible that empty shards in the
snapshot have a segments_N file that has an unsupported Lucene version
and a missing checksum.  This leads to issues with restoring the
snapshot.  This commit handles this special case by avoiding a restore
of a shard that has no data, since there is nothing to restore anyway.

Closes #18707
---
 .../index/shard/CommitPoint.java              |   7 -
 .../BlobStoreIndexShardRepository.java        |  32 +++-
 .../BlobStoreIndexShardSnapshot.java          | 163 +++++++++++-------
 .../org/elasticsearch/index/store/Store.java  |  31 +---
 .../index/store/StoreFileMetaData.java        |  17 +-
 .../recovery/RecoveryFileChunkRequest.java    |  13 +-
 .../blobstore/BlobStoreFormat.java            |   1 +
 .../bwcompat/RestoreBackwardsCompatIT.java    |  40 ++++-
 .../snapshots/blobstore/FileInfoTests.java    |  12 +-
 .../bwc/missing-checksum-repo-2.3.4.zip       | Bin 0 -> 11371 bytes
 10 files changed, 187 insertions(+), 129 deletions(-)
 create mode 100644 core/src/test/resources/indices/bwc/missing-checksum-repo-2.3.4.zip

diff --git a/core/src/main/java/org/elasticsearch/index/shard/CommitPoint.java b/core/src/main/java/org/elasticsearch/index/shard/CommitPoint.java
index 916cf563fb8..9082fc072da 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/CommitPoint.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/CommitPoint.java
@@ -62,13 +62,6 @@ public class CommitPoint {
         public String checksum() {
             return checksum;
         }
-
-        public boolean isSame(StoreFileMetaData md) {
-            if (checksum == null || md.checksum() == null) {
-                return false;
-            }
-            return length == md.length() && checksum.equals(md.checksum());
-        }
     }
 
     public static enum Type {
diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java
index e0032fe503b..94337ecdbc5 100644
--- a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java
+++ b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java
@@ -23,6 +23,8 @@ import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.IndexFormatTooNewException;
 import org.apache.lucene.index.IndexFormatTooOldException;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
@@ -49,7 +51,6 @@ import org.elasticsearch.common.lucene.store.InputStreamIndexInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.iterable.Iterables;
-import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.snapshots.IndexShardRepository;
 import org.elasticsearch.index.snapshots.IndexShardRestoreFailedException;
@@ -458,7 +459,9 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements
             }
             if (latest >= 0) {
                 try {
-                    return new Tuple<>(indexShardSnapshotsFormat.read(blobContainer, Integer.toString(latest)), latest);
+                    final BlobStoreIndexShardSnapshots shardSnapshots =
+                        indexShardSnapshotsFormat.read(blobContainer, Integer.toString(latest));
+                    return new Tuple<>(shardSnapshots, latest);
                 } catch (IOException e) {
                     logger.warn("failed to read index file  [{}]", e, SNAPSHOT_INDEX_PREFIX + latest);
                 }
@@ -503,10 +506,8 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements
          */
         public SnapshotContext(SnapshotId snapshotId, ShardId shardId, IndexShardSnapshotStatus snapshotStatus) {
             super(snapshotId, Version.CURRENT, shardId);
-            IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
-            store = indexService.getShardOrNull(shardId.id()).store();
             this.snapshotStatus = snapshotStatus;
-
+            store = indicesService.indexServiceSafe(shardId.getIndex()).getShardOrNull(shardId.id()).store();
         }
 
         /**
@@ -788,8 +789,8 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements
          */
         public RestoreContext(SnapshotId snapshotId, Version version, ShardId shardId, ShardId snapshotShardId, RecoveryState recoveryState) {
             super(snapshotId, version, shardId, snapshotShardId);
-            store = indicesService.indexServiceSafe(shardId.getIndex()).getShardOrNull(shardId.id()).store();
             this.recoveryState = recoveryState;
+            store = indicesService.indexServiceSafe(shardId.getIndex()).getShardOrNull(shardId.id()).store();
         }
 
         /**
@@ -800,6 +801,25 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements
             try {
                 logger.debug("[{}] [{}] restoring to [{}] ...", snapshotId, repositoryName, shardId);
                 BlobStoreIndexShardSnapshot snapshot = loadSnapshot();
+
+                if (snapshot.indexFiles().size() == 1
+                    && snapshot.indexFiles().get(0).physicalName().startsWith("segments_")
+                    && snapshot.indexFiles().get(0).hasUnknownChecksum()) {
+                    // If the shard has no documents, it will only contain a single segments_N file for the
+                    // shard's snapshot.  If we are restoring a snapshot created by a previous supported version,
+                    // it is still possible that in that version, an empty shard has a segments_N file with an unsupported
+                    // version (and no checksum), because we don't know the Lucene version to assign segments_N until we
+                    // have written some data.  Since the segments_N for an empty shard could have an incompatible Lucene
+                    // version number and no checksum, even though the index itself is perfectly fine to restore, this
+                    // empty shard would cause exceptions to be thrown.  Since there is no data to restore from an empty
+                    // shard anyway, we just create the empty shard here and then exit.
+                    IndexWriter writer = new IndexWriter(store.directory(), new IndexWriterConfig(null)
+                                                                                .setOpenMode(IndexWriterConfig.OpenMode.CREATE)
+                                                                                .setCommitOnClose(true));
+                    writer.close();
+                    return;
+                }
+
                 SnapshotFiles snapshotFiles = new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles());
                 final Store.MetadataSnapshot recoveryTargetMetadata;
                 try {
diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java
index 60b7ec2112e..5bb0f728bc1 100644
--- a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java
+++ b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java
@@ -22,7 +22,6 @@ package org.elasticsearch.index.snapshots.blobstore;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Version;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
@@ -50,6 +49,8 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil
      * Information about snapshotted file
      */
     public static class FileInfo {
+        private static final String UNKNOWN_CHECKSUM = "_na_";
+
         private final String name;
         private final ByteSizeValue partSize;
         private final long partBytes;
@@ -207,27 +208,43 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil
          * @return true if file in a store this this file have the same checksum and length
          */
         public boolean isSame(FileInfo fileInfo) {
-            if (numberOfParts != fileInfo.numberOfParts) return false;
-            if (partBytes != fileInfo.partBytes) return false;
-            if (!name.equals(fileInfo.name)) return false;
+            if (numberOfParts != fileInfo.numberOfParts) {
+                return false;
+            }
+            if (partBytes != fileInfo.partBytes) {
+                return false;
+            }
+            if (!name.equals(fileInfo.name)) {
+                return false;
+            }
             if (partSize != null) {
-                if (!partSize.equals(fileInfo.partSize)) return false;
+                if (!partSize.equals(fileInfo.partSize)) {
+                    return false;
+                }
             } else {
-                if (fileInfo.partSize != null) return false;
+                if (fileInfo.partSize != null) {
+                    return false;
+                }
             }
             return metadata.isSame(fileInfo.metadata);
         }
 
-        static final class Fields {
-            static final String NAME = "name";
-            static final String PHYSICAL_NAME = "physical_name";
-            static final String LENGTH = "length";
-            static final String CHECKSUM = "checksum";
-            static final String PART_SIZE = "part_size";
-            static final String WRITTEN_BY = "written_by";
-            static final String META_HASH = "meta_hash";
+        /**
+         * Checks if the checksum for the file is unknown. This only is possible on an empty shard's
+         * segments_N file which was created in older Lucene versions.
+         */
+        public boolean hasUnknownChecksum() {
+            return metadata.checksum().equals(UNKNOWN_CHECKSUM);
         }
 
+        static final String NAME = "name";
+        static final String PHYSICAL_NAME = "physical_name";
+        static final String LENGTH = "length";
+        static final String CHECKSUM = "checksum";
+        static final String PART_SIZE = "part_size";
+        static final String WRITTEN_BY = "written_by";
+        static final String META_HASH = "meta_hash";
+
         /**
          * Serializes file info into JSON
          *
@@ -237,22 +254,22 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil
          */
         public static void toXContent(FileInfo file, XContentBuilder builder, ToXContent.Params params) throws IOException {
             builder.startObject();
-            builder.field(Fields.NAME, file.name);
-            builder.field(Fields.PHYSICAL_NAME, file.metadata.name());
-            builder.field(Fields.LENGTH, file.metadata.length());
-            if (file.metadata.checksum() != null) {
-                builder.field(Fields.CHECKSUM, file.metadata.checksum());
+            builder.field(NAME, file.name);
+            builder.field(PHYSICAL_NAME, file.metadata.name());
+            builder.field(LENGTH, file.metadata.length());
+            if (file.metadata.checksum().equals(UNKNOWN_CHECKSUM) == false) {
+                builder.field(CHECKSUM, file.metadata.checksum());
             }
             if (file.partSize != null) {
-                builder.field(Fields.PART_SIZE, file.partSize.bytes());
+                builder.field(PART_SIZE, file.partSize.bytes());
             }
 
             if (file.metadata.writtenBy() != null) {
-                builder.field(Fields.WRITTEN_BY, file.metadata.writtenBy());
+                builder.field(WRITTEN_BY, file.metadata.writtenBy());
             }
 
             if (file.metadata.hash() != null && file.metadata().hash().length > 0) {
-                builder.field(Fields.META_HASH, file.metadata.hash());
+                builder.field(META_HASH, file.metadata.hash());
             }
             builder.endObject();
         }
@@ -271,6 +288,7 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil
             String checksum = null;
             ByteSizeValue partSize = null;
             Version writtenBy = null;
+            String writtenByStr = null;
             BytesRef metaHash = new BytesRef();
             if (token == XContentParser.Token.START_OBJECT) {
                 while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -278,19 +296,20 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil
                         String currentFieldName = parser.currentName();
                         token = parser.nextToken();
                         if (token.isValue()) {
-                            if ("name".equals(currentFieldName)) {
+                            if (NAME.equals(currentFieldName)) {
                                 name = parser.text();
-                            } else if ("physical_name".equals(currentFieldName)) {
+                            } else if (PHYSICAL_NAME.equals(currentFieldName)) {
                                 physicalName = parser.text();
-                            } else if ("length".equals(currentFieldName)) {
+                            } else if (LENGTH.equals(currentFieldName)) {
                                 length = parser.longValue();
-                            } else if ("checksum".equals(currentFieldName)) {
+                            } else if (CHECKSUM.equals(currentFieldName)) {
                                 checksum = parser.text();
-                            } else if ("part_size".equals(currentFieldName)) {
+                            } else if (PART_SIZE.equals(currentFieldName)) {
                                 partSize = new ByteSizeValue(parser.longValue());
-                            } else if ("written_by".equals(currentFieldName)) {
-                                writtenBy = Lucene.parseVersionLenient(parser.text(), null);
-                            } else if ("meta_hash".equals(currentFieldName)) {
+                            } else if (WRITTEN_BY.equals(currentFieldName)) {
+                                writtenByStr = parser.text();
+                                writtenBy = Lucene.parseVersionLenient(writtenByStr, null);
+                            } else if (META_HASH.equals(currentFieldName)) {
                                 metaHash.bytes = parser.binaryValue();
                                 metaHash.offset = 0;
                                 metaHash.length = metaHash.bytes.length;
@@ -305,6 +324,7 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil
                     }
                 }
             }
+
             // Verify that file information is complete
             if (name == null || Strings.validFileName(name) == false) {
                 throw new ElasticsearchParseException("missing or invalid file name [" + name + "]");
@@ -312,10 +332,29 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil
                 throw new ElasticsearchParseException("missing or invalid physical file name [" + physicalName + "]");
             } else if (length < 0) {
                 throw new ElasticsearchParseException("missing or invalid file length");
+            } else if (writtenBy == null) {
+                throw new ElasticsearchParseException("missing or invalid written_by [" + writtenByStr + "]");
+            } else if (checksum == null) {
+                if (physicalName.startsWith("segments_")
+                        && writtenBy.onOrAfter(StoreFileMetaData.FIRST_LUCENE_CHECKSUM_VERSION) == false) {
+                    // its possible the checksum is null for segments_N files that belong to a shard with no data,
+                    // so we will assign it _na_ for now and try to get the checksum from the file itself later
+                    checksum = UNKNOWN_CHECKSUM;
+                } else {
+                    throw new ElasticsearchParseException("missing checksum for name [" + name + "]");
+                }
             }
             return new FileInfo(name, new StoreFileMetaData(physicalName, length, checksum, writtenBy, metaHash), partSize);
         }
 
+        @Override
+        public String toString() {
+            return "[name: " + name +
+                       ", numberOfParts: " + numberOfParts +
+                       ", partSize: " + partSize +
+                       ", partBytes: " + partBytes +
+                       ", metadata: " + metadata + "]";
+        }
     }
 
     private final String snapshot;
@@ -424,26 +463,21 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil
         return totalSize;
     }
 
-    static final class Fields {
-        static final String NAME = "name";
-        static final String INDEX_VERSION = "index_version";
-        static final String START_TIME = "start_time";
-        static final String TIME = "time";
-        static final String NUMBER_OF_FILES = "number_of_files";
-        static final String TOTAL_SIZE = "total_size";
-        static final String FILES = "files";
-    }
-
-    static final class ParseFields {
-        static final ParseField NAME = new ParseField("name");
-        static final ParseField INDEX_VERSION = new ParseField("index_version", "index-version");
-        static final ParseField START_TIME = new ParseField("start_time");
-        static final ParseField TIME = new ParseField("time");
-        static final ParseField NUMBER_OF_FILES = new ParseField("number_of_files");
-        static final ParseField TOTAL_SIZE = new ParseField("total_size");
-        static final ParseField FILES = new ParseField("files");
-    }
+    private static final String NAME = "name";
+    private static final String INDEX_VERSION = "index_version";
+    private static final String START_TIME = "start_time";
+    private static final String TIME = "time";
+    private static final String NUMBER_OF_FILES = "number_of_files";
+    private static final String TOTAL_SIZE = "total_size";
+    private static final String FILES = "files";
 
+    private static final ParseField PARSE_NAME = new ParseField("name");
+    private static final ParseField PARSE_INDEX_VERSION = new ParseField("index_version", "index-version");
+    private static final ParseField PARSE_START_TIME = new ParseField("start_time");
+    private static final ParseField PARSE_TIME = new ParseField("time");
+    private static final ParseField PARSE_NUMBER_OF_FILES = new ParseField("number_of_files");
+    private static final ParseField PARSE_TOTAL_SIZE = new ParseField("total_size");
+    private static final ParseField PARSE_FILES = new ParseField("files");
 
     /**
      * Serializes shard snapshot metadata info into JSON
@@ -453,13 +487,13 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil
      */
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(Fields.NAME, snapshot);
-        builder.field(Fields.INDEX_VERSION, indexVersion);
-        builder.field(Fields.START_TIME, startTime);
-        builder.field(Fields.TIME, time);
-        builder.field(Fields.NUMBER_OF_FILES, numberOfFiles);
-        builder.field(Fields.TOTAL_SIZE, totalSize);
-        builder.startArray(Fields.FILES);
+        builder.field(NAME, snapshot);
+        builder.field(INDEX_VERSION, indexVersion);
+        builder.field(START_TIME, startTime);
+        builder.field(TIME, time);
+        builder.field(NUMBER_OF_FILES, numberOfFiles);
+        builder.field(TOTAL_SIZE, totalSize);
+        builder.startArray(FILES);
         for (FileInfo fileInfo : indexFiles) {
             FileInfo.toXContent(fileInfo, builder, params);
         }
@@ -493,24 +527,24 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil
                     String currentFieldName = parser.currentName();
                     token = parser.nextToken();
                     if (token.isValue()) {
-                        if (parseFieldMatcher.match(currentFieldName, ParseFields.NAME)) {
+                        if (parseFieldMatcher.match(currentFieldName, PARSE_NAME)) {
                             snapshot = parser.text();
-                        } else if (parseFieldMatcher.match(currentFieldName, ParseFields.INDEX_VERSION)) {
+                        } else if (parseFieldMatcher.match(currentFieldName, PARSE_INDEX_VERSION)) {
                             // The index-version is needed for backward compatibility with v 1.0
                             indexVersion = parser.longValue();
-                        } else if (parseFieldMatcher.match(currentFieldName, ParseFields.START_TIME)) {
+                        } else if (parseFieldMatcher.match(currentFieldName, PARSE_START_TIME)) {
                             startTime = parser.longValue();
-                        } else if (parseFieldMatcher.match(currentFieldName, ParseFields.TIME)) {
+                        } else if (parseFieldMatcher.match(currentFieldName, PARSE_TIME)) {
                             time = parser.longValue();
-                        } else if (parseFieldMatcher.match(currentFieldName, ParseFields.NUMBER_OF_FILES)) {
+                        } else if (parseFieldMatcher.match(currentFieldName, PARSE_NUMBER_OF_FILES)) {
                             numberOfFiles = parser.intValue();
-                        } else if (parseFieldMatcher.match(currentFieldName, ParseFields.TOTAL_SIZE)) {
+                        } else if (parseFieldMatcher.match(currentFieldName, PARSE_TOTAL_SIZE)) {
                             totalSize = parser.longValue();
                         } else {
                             throw new ElasticsearchParseException("unknown parameter [{}]", currentFieldName);
                         }
                     } else if (token == XContentParser.Token.START_ARRAY) {
-                        if (parseFieldMatcher.match(currentFieldName, ParseFields.FILES)) {
+                        if (parseFieldMatcher.match(currentFieldName, PARSE_FILES)) {
                             while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                                 indexFiles.add(FileInfo.fromXContent(parser));
                             }
@@ -526,6 +560,7 @@ public class BlobStoreIndexShardSnapshot implements ToXContent, FromXContentBuil
             }
         }
         return new BlobStoreIndexShardSnapshot(snapshot, indexVersion, Collections.unmodifiableList(indexFiles),
-                startTime, time, numberOfFiles, totalSize);
+                                               startTime, time, numberOfFiles, totalSize);
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/index/store/Store.java b/core/src/main/java/org/elasticsearch/index/store/Store.java
index a720f5cb258..166f978a4db 100644
--- a/core/src/main/java/org/elasticsearch/index/store/Store.java
+++ b/core/src/main/java/org/elasticsearch/index/store/Store.java
@@ -41,7 +41,6 @@ import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.SimpleFSDirectory;
 import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.Version;
@@ -444,11 +443,9 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
     }
 
     /**
-     * The returned IndexOutput might validate the files checksum if the file has been written with a newer lucene version
-     * and the metadata holds the necessary information to detect that it was been written by Lucene 4.8 or newer. If it has only
-     * a legacy checksum, returned IndexOutput will not verify the checksum.
+     * The returned IndexOutput validates the files checksum.
      * <p>
-     * Note: Checksums are calculated nevertheless since lucene does it by default sicne version 4.8.0. This method only adds the
+     * Note: Checksums are calculated by default since version 4.8.0. This method only adds the
      * verification against the checksum in the given metadata and does not add any significant overhead.
      */
     public IndexOutput createVerifyingOutput(String fileName, final StoreFileMetaData metadata, final IOContext context) throws IOException {
@@ -652,17 +649,7 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
                     // different in the diff. That's why we have to double check here again if the rest of it matches.
 
                     // all is fine this file is just part of a commit or a segment that is different
-                    final boolean same = local.isSame(remote);
-
-                    // this check ensures that the two files are consistent ie. if we don't have checksums only the rest needs to match we are just
-                    // verifying that we are consistent on both ends source and target
-                    final boolean hashAndLengthEqual = (
-                            local.checksum() == null
-                                    && remote.checksum() == null
-                                    && local.hash().equals(remote.hash())
-                                    && local.length() == remote.length());
-                    final boolean consistent = hashAndLengthEqual || same;
-                    if (consistent == false) {
+                    if (local.isSame(remote) == false) {
                         logger.debug("Files are different on the recovery target: {} ", recoveryDiff);
                         throw new IllegalStateException("local version: " + local + " is different from remote version after recovery: " + remote, null);
                     }
@@ -898,18 +885,6 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
             }
         }
 
-        /**
-         * Computes a strong hash value for small files. Note that this method should only be used for files &lt; 1MB
-         */
-        public static BytesRef hashFile(Directory directory, String file) throws IOException {
-            final BytesRefBuilder fileHash = new BytesRefBuilder();
-            try (final IndexInput in = directory.openInput(file, IOContext.READONCE)) {
-                hashFile(fileHash, new InputStreamIndexInput(in, in.length()), in.length());
-            }
-            return fileHash.get();
-        }
-
-
         /**
          * Computes a strong hash value for small files. Note that this method should only be used for files &lt; 1MB
          */
diff --git a/core/src/main/java/org/elasticsearch/index/store/StoreFileMetaData.java b/core/src/main/java/org/elasticsearch/index/store/StoreFileMetaData.java
index e163b15f60e..2653f01c81d 100644
--- a/core/src/main/java/org/elasticsearch/index/store/StoreFileMetaData.java
+++ b/core/src/main/java/org/elasticsearch/index/store/StoreFileMetaData.java
@@ -21,10 +21,8 @@ package org.elasticsearch.index.store;
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Version;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.lucene.Lucene;
 
@@ -58,14 +56,15 @@ public class StoreFileMetaData implements Writeable {
     }
 
     public StoreFileMetaData(String name, long length, String checksum, Version writtenBy, BytesRef hash) {
-        assert writtenBy != null && writtenBy.onOrAfter(FIRST_LUCENE_CHECKSUM_VERSION) : "index version less that "
-            + FIRST_LUCENE_CHECKSUM_VERSION + " are not supported but got: " + writtenBy;
-        Objects.requireNonNull(writtenBy, "writtenBy must not be null");
-        Objects.requireNonNull(checksum, "checksum must not be null");
-        this.name = name;
+        // its possible here to have a _na_ checksum or an unsupported writtenBy version, if the
+        // file is a segments_N file, but that is fine in the case of a segments_N file because
+        // we handle that case upstream
+        assert name.startsWith("segments_") || (writtenBy != null && writtenBy.onOrAfter(FIRST_LUCENE_CHECKSUM_VERSION)) :
+            "index version less that " + FIRST_LUCENE_CHECKSUM_VERSION + " are not supported but got: " + writtenBy;
+        this.name = Objects.requireNonNull(name, "name must not be null");
         this.length = length;
-        this.checksum = checksum;
-        this.writtenBy = writtenBy;
+        this.checksum = Objects.requireNonNull(checksum, "checksum must not be null");
+        this.writtenBy = Objects.requireNonNull(writtenBy, "writtenBy must not be null");
         this.hash = hash == null ? new BytesRef() : hash;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryFileChunkRequest.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryFileChunkRequest.java
index 8fd08d9f8fb..49cdb737ed3 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryFileChunkRequest.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryFileChunkRequest.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.indices.recovery;
 
 import org.apache.lucene.util.Version;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -76,7 +75,6 @@ public final class RecoveryFileChunkRequest extends TransportRequest {
         return position;
     }
 
-    @Nullable
     public String checksum() {
         return metaData.checksum();
     }
@@ -105,11 +103,10 @@ public final class RecoveryFileChunkRequest extends TransportRequest {
         String name = in.readString();
         position = in.readVLong();
         long length = in.readVLong();
-        String checksum = in.readOptionalString();
+        String checksum = in.readString();
         content = in.readBytesReference();
-        Version writtenBy = null;
-        String versionString = in.readOptionalString();
-        writtenBy = Lucene.parseVersionLenient(versionString, null);
+        Version writtenBy = Lucene.parseVersionLenient(in.readString(), null);
+        assert writtenBy != null;
         metaData = new StoreFileMetaData(name, length, checksum, writtenBy);
         lastChunk = in.readBoolean();
         totalTranslogOps = in.readVInt();
@@ -124,9 +121,9 @@ public final class RecoveryFileChunkRequest extends TransportRequest {
         out.writeString(metaData.name());
         out.writeVLong(position);
         out.writeVLong(metaData.length());
-        out.writeOptionalString(metaData.checksum());
+        out.writeString(metaData.checksum());
         out.writeBytesReference(content);
-        out.writeOptionalString(metaData.writtenBy() == null ? null : metaData.writtenBy().toString());
+        out.writeString(metaData.writtenBy().toString());
         out.writeBoolean(lastChunk);
         out.writeVInt(totalTranslogOps);
         out.writeLong(sourceThrottleTimeInNanos);
diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreFormat.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreFormat.java
index 23d390dfcfe..04900705e0a 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreFormat.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreFormat.java
@@ -115,4 +115,5 @@ public abstract class BlobStoreFormat<T extends ToXContent> {
 
         }
     }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java
index 419104bfe34..494aa7d1095 100644
--- a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java
+++ b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java
@@ -1,4 +1,3 @@
-/*
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -46,6 +45,7 @@ import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
 import java.util.SortedSet;
@@ -127,6 +127,44 @@ public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase {
         }
     }
 
+    public void testRestoreSnapshotWithMissingChecksum() throws Exception {
+        final String repo = "test_repo";
+        final String snapshot = "test_1";
+        final String indexName = "index-2.3.4";
+        final String repoFileId = "missing-checksum-repo-2.3.4";
+        Path repoFile = getBwcIndicesPath().resolve(repoFileId + ".zip");
+        URI repoFileUri = repoFile.toUri();
+        URI repoJarUri = new URI("jar:" + repoFileUri.toString() + "!/repo/");
+        logger.info("-->  creating repository [{}] for repo file [{}]", repo, repoFileId);
+        assertAcked(client().admin().cluster().preparePutRepository(repo)
+                                              .setType("url")
+                                              .setSettings(Settings.builder().put("url", repoJarUri.toString())));
+
+        logger.info("--> get snapshot and check its indices");
+        GetSnapshotsResponse getSnapshotsResponse = client().admin().cluster().prepareGetSnapshots(repo).setSnapshots(snapshot).get();
+        assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1));
+        SnapshotInfo snapshotInfo = getSnapshotsResponse.getSnapshots().get(0);
+        assertThat(snapshotInfo.indices(), equalTo(Arrays.asList(indexName)));
+
+        logger.info("--> restoring snapshot");
+        RestoreSnapshotResponse response = client().admin().cluster().prepareRestoreSnapshot(repo, snapshot).setRestoreGlobalState(true).setWaitForCompletion(true).get();
+        assertThat(response.status(), equalTo(RestStatus.OK));
+        RestoreInfo restoreInfo = response.getRestoreInfo();
+        assertThat(restoreInfo.successfulShards(), greaterThan(0));
+        assertThat(restoreInfo.successfulShards(), equalTo(restoreInfo.totalShards()));
+        assertThat(restoreInfo.failedShards(), equalTo(0));
+        String index = restoreInfo.indices().get(0);
+        assertThat(index, equalTo(indexName));
+
+        logger.info("--> check search");
+        SearchResponse searchResponse = client().prepareSearch(index).get();
+        assertThat(searchResponse.getHits().totalHits(), greaterThan(0L));
+
+        logger.info("--> cleanup");
+        cluster().wipeIndices(restoreInfo.indices().toArray(new String[restoreInfo.indices().size()]));
+        cluster().wipeTemplates();
+    }
+
     private List<String> repoVersions() throws Exception {
         return listRepoVersions("repo");
     }
diff --git a/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java b/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java
index 67c431135a0..70eacaafedb 100644
--- a/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java
+++ b/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java
@@ -27,7 +27,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo.Fields;
+import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo;
 import org.elasticsearch.index.store.StoreFileMetaData;
 import org.elasticsearch.test.ESTestCase;
 
@@ -105,11 +105,11 @@ public class FileInfoTests extends ESTestCase {
 
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
             builder.startObject();
-            builder.field(Fields.NAME, name);
-            builder.field(Fields.PHYSICAL_NAME, physicalName);
-            builder.field(Fields.LENGTH, length);
-            builder.field(Fields.WRITTEN_BY, Version.LATEST.toString());
-            builder.field(Fields.CHECKSUM, "666");
+            builder.field(FileInfo.NAME, name);
+            builder.field(FileInfo.PHYSICAL_NAME, physicalName);
+            builder.field(FileInfo.LENGTH, length);
+            builder.field(FileInfo.WRITTEN_BY, Version.LATEST.toString());
+            builder.field(FileInfo.CHECKSUM, "666");
             builder.endObject();
             byte[] xContent = builder.bytes().toBytes();
 
diff --git a/core/src/test/resources/indices/bwc/missing-checksum-repo-2.3.4.zip b/core/src/test/resources/indices/bwc/missing-checksum-repo-2.3.4.zip
new file mode 100644
index 0000000000000000000000000000000000000000..9590f8dbd660fd56617c8e55b6bc73276aae4744
GIT binary patch
literal 11371
zcmdT~2{_bU`!*)V{@Rxi#+qzn%a$c3#=d4LYax>?k&HD<ipai<H8C+HWS5YgkV*=P
z?6lH?Lf=^WGQRhn#{2!R|9gFNT|Z-7=DE-Foaa90Irnq)wSfD`2sU1!_u4f!f7$*)
zM8HOH-rmhs)Wn#S0N5U5YN``!YI?z&oB*)z3xI%NA3f$D^|8Kw*5cX~X3SqV9<rZb
zcXByx@BMEZ8+LZt@CAFg*t&U~arN{7%bfsw+Ix6fL%^2)*tWMvwfRQ5!?x2YdyhQ~
zai<xU-DWn2Dl9G{AtL!VO|s%LDfU~V8zCjxoQdW8vpnZ8pMo*Z0o+Edt;KMKx2z$P
zyVoE+H8U;M)Udm;v2ozO(8_~}wG&?|MVwP)8qDQWAIBsn0?1NqDfnK;-E65FPhb7K
z{JFob|7J^E_o~7%RjcYVPEHOHX2u4`V+`VAJ=|<cTvMXaPEa=&<<l#Gs7_Iv>D(G^
zIB+u`emP-;s*<0_IItg+E@0eFY=!>!j+mz>o9U;=T5o!C2G!a6t&mq(g1=(9y?DB=
z<)v}nvUf;kaAenb_hd`Dcg$5scesxW+zRGiXM0u`Zsh|DBh%7yL>M^0U~nrxL)5kK
z%%Dsabr(3y&hMC=dz<wwKHnQqHnDTczAtK%d6m5D0t;UaT&AR(Ieb=KV*lN{;>rMt
z$Ok+p4HHsK)8n%aWM!r%TO^ppA+y@gbdWO%3-@T#Toddf6W0_!Pen!XmGTors)(eQ
z@~V*14af>nce-RU>|8^jSL2ayg%vy%)jY30CO7{P^C4h;eT~(8rREMd2(Xd)zoPiL
zc>N_CCW6|SM-X=uH;5#>MIe#Www`#T#a1+%Fln(BOj%LL;b&XpD&dz9<*fkM3svOH
zNGN^KcXWr@8NmEt4lV_PIZ0Kg)VgjuP|p}!TRFKH!VUZaEc`4Sbm0rxy>Y8?E$PeS
z-37)BNlA<)W6{=Bx(f@$M~GMwTalg0<}uLBR`gsA8o}GT5LP6MM$GZDD1XVAf+Bf&
z(TD|Dq6nW4u@5nO4VpK6fhD|^B@mhV_CaqL94MDo<ZHed(*C__{+#6tG53HnFUcIP
zLx+I$V$ICjO}x->M62w|+`=3hfrf_jDsw>gFE&B7v?PpLZbsY<j*N>;4*`bwH~VSp
z918XTf*<~IMmqo?!QW$VIQ${2A@=gu-l1ytU8@=<f&cx&$#2LW`2GINRC_JP>{-mD
z;^a3(6c2ujyf;csPfdoy$I`P-#8{^$k7Kx=fVAi)*f;Vpjzi&9gNf^#rm*l&X%!PJ
z)hiR(T{`2GpAEatvv4Z^p*ec_A?u~dPw`w$kuUXE6zEstwDSYnGzHd3p!J*$%-~)&
z%4ZynH<L}12pLXNGL$<!*3v>>B2x?@BhIOzF9>g?75orA9Y>h&2Qprl?&zZ>Nn@C&
z4AYJIqJOxoK8WlXfyuNA(csA&y&4J|954nngTGoH-L+UCoWdJ9Fwt<rCnAvFrMAf1
z?TAost$gFi<fKQKEPt_k@jD>U)=1XM+1Sz1nqb}iIxXe)7{_hu2N{WGN%^KpsmXEH
zJd@T5De3VAW9in$G1gmD2PTRL5}%i$_#*U~V!Wq&nd@ea7^yC|(GdxW5gt{KdbW~6
z8`|5qd~W#=zg=nd35RyFI5RLhEI9CiZAPFd1TV72Em1+NOyyPy!%Q+GZ{FQoQwfZY
zZh#A_45h65XIb{oBhLqWGA8p(Gk)Bknu6S)mWpI%WwA(=C~IY%<;_FgB}9*FljNls
z95`Uey8=ZYbU~tJ54O6jKx@KzeQH(?qEG7HdBC;s0C4Ay2uPBz;Zhh~NW>#1(5|qn
z$P~;P0|^K!NbyKSaa>{VG3n8xQ=uP4uE7BZv%rKFWFR`4;g1HFQv@ZY@&&?zN%<@X
zyoMwzEuV}Cs_CS<wP-{h7HtdRIsb_uoP)GWX{^)DMEfe&0U4)psAFZuh3t`}5FXtd
z_r6cxAN_tG!7(wcAt`mTGt_|WOU;$U$;dbuboDNugtxHJWu-vN6LVIGO#fBq_1u=_
z5-$&{d(VqQ`KQ!~(K#CN+>Cq-#wX5~Q7p+fq`Z_PUleVmE+J4RU@jox5}SJu;mGw3
zyB|G6U_I8RUL}vd<Sh3p%X=}3P{?`I&xxw{$gO+}1qLJ6+==~~6Z%?2Gf6M|2C3v_
ztf<d@jp!;+eRe%{sP<b}TeFUOYIJ3JCA+-KSFgF5H&Ww8E&|!1uArwRL<Vx`ZxgnZ
zADU=gK2wy_^P^8CWFv{{fL}kCod@><{G5-@vdJE4-^bq>9x8y=<YPZpm)q0BsoT}U
zEj>r2S?JE&+68GrP4dvHxR_jF{$vNuN4PB>rFi4?M1h@&k6(QK;j35mMvie+w~k%O
zVJfDa8~*xQGcUGUH*FZ{OV>Nx5yBd>d}VA{K=CZUr#n{(7Y8>9waeXVAsa;sgU~{@
z>0Hnax)!SeB8p2ZF-ingoyiRRhh!f;5T>Dn#P`I9JoF;etBe2PN9xnV@;>#OqUFuZ
zkETd9+dKz_;VXTj!KP`?XKM;#f=lO(%I5f|)nigB4L-W-z;ZM=uhY^<aTX3~2nIXA
z_7REYt`JBl552j=a(-cgR*RFB{8$hJ_fzIR1+@nhOKzu2G8&(KZ~=7{8Y@vtNQ)h4
zfhz#byy$4#d~;vIA?Ok-<628?K^KPjq2vqbi;@aeBvyUmh_h6}@}A#lO5qi9tE)i_
znf}t`oWSgh>ide8(nSnk+yUf~KTj(pSF(H7t$RVQ;gd?#g)(G#OSLKMbix!_e*dI{
z4BXfHhN}lrh=)aWCab-q@{#gbW8>nFO(o8rZPceu>CO;SWv;f(P7k%|TeRQIplb}Z
zFOn}%a_%326gVW<7~HoyBqK!^)#ex^MP<^L$5nGZvd}4u;vr&P-gW`OGWI>HP{Gwu
zmaTc+<JIKRF|M&66Opf3Rb(!86y^#HCU$fdi~|x{?vO4{kJPOUub|fFIIt^$=$cdI
z5tvP*8T0&h(~!W$)!!F|L65I)6ooH>QfNYhj%qIuN1lOC$ogr<8+|g=(OFjoXgm#+
zB#;h=Rp$yoie>nfjwrI{=LnSYNGL@=zRDzGVUdaiC*{8}H=nX)q&Gvdnq+1<APrq&
ztx^M&Pc4$CTuVtxHv{ZijZUQS$!HN05Wp}i63&VdB8o@FXm51H^{F!RbtBmxb#lXN
zHax1t)CKnPrD-MQJYH(K$BCLQnO!gRUl*tp{gT1j6iQD9Eg_&ibo4063+CFA#ttPL
zsNHLjSb2ECc(IxZ0>%0sF*;LoZ{~5)s;3D3RgiB>Q^IL+mg)H^u)L4+3o0-uCmUqh
zzfalwaT#&TzzrZ+4WOX@))Iu^s}Mw)>PeA1Ey2bL^gV@Gl-_)oMs)zwfY5QC>!Tht
zmm7my<m{A+LLEi!5ba-#X}K6X2afhq0IKIH{kXCAZS7pvaNyGH)bMKtYC9HbO9O~D
z<Qgo19X=K1vZCum{$AMygpm8nB<+_koBy&Y;E(SyLod>_h80N<BZ3M&6k?{zNauC;
z6B&%3MmQOnDNeLhrAoe7eqWa2*7yAO%aB@%$)TE*cc23mBXg54)#MF3Y1o$c+fnj!
z9*3OHQ!OKWL`uld0Orf?d)GrQWnQ3pzDSTPY6^MsOrZm(wAbN5>gn<RgAx9@Rp%yW
zbEEv340~JU1f`$7Ki79ZyP#`HtN2H*qP^hRK-2b|v$kG-MmnA+5}&Z5g!@5B)s)Y8
zQcmX%y^Y}k5w;xXT~afsjIBbURNv(wZRsrn#Odc}ba^jC(+t8ikb?r~Gne%lKmI_a
z>Wv!9>mCBLXBC(oyr<zmO9M)4q+o&Fj+9Y?hBOX5YDV>ry}qzA&=cfa6&)4!syHnO
z@#Om3?;ITQRS{_RTodyjxAUV?xxVDNu)Wv$(R3y`j0<2}?n9l%91xtsUVB02K(t;2
zQ$v8$4<I%X4>2g1yJ6DQ6_choOLB<l9!he1LmkWp%KSdb`5=)bO^&*3e@k@9pk=Mr
z0-+tbFbALrG;Yu4<#_MfJqde<Btu=kFV{4*`$N6OAbf2C7khm^Dz5^qbn2FY3IZB#
z4VonNoov&XgdId=Ikwl=%#=|y6PJ<dp$wZU9?H5GM`d5>VuMg6=z!ABP~HZ2u?06c
zi%}f}y<F&nzV7P&0`<7ZZdI??TwfI`z{yy`##A~&sVn0N@E9-%Sa}vgvU>lpBH&Wp
zV9AMYNU+OiE>5km%Li&np4Vi27{QRCqq^PC)$m}Y^93RoNp!PplMUMB36D<P5;wbj
z0lhxbsB@`4KPh{DNDg@&l<9p{)kf8U<0?mcFfYi#A84D~DP_DwJ#FF~>o%JAA?kyY
z;yVp_*AGg8PwUADg?Rb+g;Y9Fqz`*lPPx@fSHHMNVVYeYQ39JqBFHG2C|_}ws|ZX6
z9r1_BFA5fFKw-qS0dmLg>*lCG<FVwhFg&JP^SDrs65``~*p@aoEW`ZJys~I|)QkT1
z?4FFy`s-f#akeda#I(ZC9lb<s_*F+DgbqgJehKa7JY4*^nBMc8*;Df70&%$~iQySc
z+)NYXbRulBt)$TD+>n#;I<)gDB8V2Xn(*`Qr1OOn9-<y4TdiyRX-+u?Mzxpb8@b=S
z9hWtHCrO<=X(04zdkAZ0!Ce_*;(krVPx%%WD&B({VGdNaL#G}`BPoa$k#s5}492DL
z1_kmIf(n2H;>EyhhxkOYE3nJC&2$V0Q6QPiDK)uN^8jvL6aDYe`lA=SJ>2q#EQ)8V
zTq^72b?l;2d+bJvIWBUqxhyrWXRS^24K$5D|M7Q^IpTXb=5#(+)x}H35Xa}3gImUk
zz*ELZxm*~orEKo5Z=-zEDJ)hGny(Up@Y<F!n7b|5w`8qf2k=|f!6)#NF~srNls32C
ze^kbR;3;F2xk<T`3LK&$qtKA-5nDu)NNGSGiEPOjY$4`_z<?ibFZq5pJqv@o!|Gm(
zUg6*zU0YrIe)#U9qhFEhE*WFaCr}tK8AE(8Yvy(#^Iyvt+W)YOL64`5A#_sbBw<%X
z2qBPGM<UWs@F=BWciFa#(QY#^e`!=HkefVXLHHi|-{rT&9{By}ym}5^{FeCd`JIF(
zzs<b;bWui^0_|?t+IyEeskoWgm72oGwq#1}Ro_t3T}3D5-I0Jdn3DRpGNr`-1i#zG
z@#Hso^dqz<*+)GJ$=ej7(!wNnG=;uuQf^8#-Ra?h(PC@sgMg*k;(@sS@i^~hYbqt>
z`tK{>mt0Rzy?GumLb)r9GYZ1CcoAIUpCGu}KTL2TJPF=3;USnoxJn14Jk)$mhdj8D
z_S$_z;FeD5I6lg)-4M8bj8U$}u2bpn68!hgjbD4;#m}>fcoAIkzbE*0d<mY5cCXCQ
z3pLO%M0p(Z(Ko+)_O!w2ZGw{=uM^#Us(gHE#vZQ?T=IW|;P`If*CeHSy*`JFk--W)
zdo}(L)bffFilEpcINO7WCEma6L)1Kd+rPGp-veJA9^u7r$^UPDCt>;!8viiA+3@5y
zSSPBC-5(rzG)^!>;z>wXoF(@)@HW5IOUC^>H~SDrk4L=Pt<)Z^E4#A~BaX{p9syi^
zpmX+~wtMLUB_osLEV9$mEaFn)%;Gx!-3uZIyW~3Fc3dnh+HL`9Xkl7FumX<&6}XkJ
zI~)dcu<}_bhS{KeI-zJxy9K(S>|y9nq0_A$rPHGwNh#DVoTq9$nK;ddVr-qsYh^+)
zPM+rDXOw5wg(eI3Y6xi_^b&g)s*#Z7a4YoqlzP|-4?`QsVtFP-E)_JXcAT;HmR?Z)
zrCLh6KzKx2Y*1-XIBM#{*Ox89gfYOtRk#pj^_jW;K2I+Lm;-#F_*Tp3e(N6IL?U#r
z><VJ8A?*{bhf&~4En_l0Kx&mZIO|-vEHb=Rxk9;jqZnYmq3oyiCU94ldq=1H+GC0V
zXUs$YGs=67XzYK)63BCtoc7zZ<4G3A>By{$1@f|a<<&eD0`=vR61?S4Zb%M`_Ou=r
z04sDWDJm*-fNx83m*ntBRdMo{@>DE8|1jFHu?0LJAmUTfLF}rm18QT|P?+I1?HuwR
zbcesFq1pKs3wO2y%eJyWR8azYSyrjCNEp)|l#s1hzQPm}Ymt3+GgZn^)jXv<&xB?A
z2M48Os_8lo35JVycI%kf_b+-yYRsM0>(NqSeDTyW&N?oI&N@BTEWywZe$?GbysvM(
z+^|~T*TT+0*NLX@R#rnGM_<ZnFn~;)sc>{-MF$Wb0Fq%`{IjbH8#fL%UepBdR5sqX
zFD?*mU0m38;b6BZ>}v~KrZ5~M$PH5ZX=?it1Ag{42JrJmxp^64<I~2_(lGCvL&MnH
zzPPY=lPp*!$9J3D(nWt>V%WQRRxI;Um_T5Sc&GV|3k|pcew7i}*BO4!)<%dUa06^M
zKk>uBzHP7*Lk}a>;c#NhCCuFe{BW@Q@jG!6F>)Xdod2LCF1$a_bnMAtAi{yS(T&Ch
z@N06gd(k@`g5w6*!{lIhcXwj=5#w~i+8WdA#m^7yR_0EOPoUpmU>ckFVPLmlc4B0a
z{{{ookiicFy9u%rBZvYg#vT(HTgBdqBk%`Kod5GgUZuf-x1l%U7dULSai>EDv^W9w
zkjU6#;7$xiI-D4LoE&VqY$wi140Hc-mH89#3|H*%S!G@_;K17ui*Nz_T4k`MqMZ&w
z7;ysZVREpQoShh7abxUpa<HY2oj6A@AO3Q3{zUk|6+3(;=ROM#yp4lCE`VQ?gMG~3
z=@9LHoB(^69PCs0PK@WcG4?n)*yq%pIP7dVec98p_A7SS=dzu68JJeWUnawUoGI{|
tDeUvfPBb9LHrn5`jD3vQX}R*JWxOLqgxS<L?kF4}xQ*Es(VQEf{slJm3eNxl

literal 0
HcmV?d00001


From a25b8ee1bf2e70bc28f688731ef43962a3ef18ce Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 10 Jun 2016 10:18:06 -0400
Subject: [PATCH 0775/1311] Make random UUIDs reproducible in tests

Today we use a random source of UUIDs for assigning allocation IDs,
cluster IDs, etc. Yet, the source of randomness for this is not
reproducible in tests. Since allocation IDs end up as keys in hash maps,
this means allocation decisions and not reproducible in tests and this
leads to non-reproducible test failures. This commit modifies the
behavior of random UUIDs so that they are reproducible under tests. The
behavior for production code is not changed, we still use a true source
of secure randomness but under tests we just use a reproducible source
of non-secure randomness.

It is important to note that there is a test,
UUIDTests#testThreadedRandomUUID that relies on the UUIDs being truly
random. Thus, we have to modify the setup for this test to use a true
source of randomness. Thus, this is one test that will never be
reproducible but it is intentionally so.

Relates #18808
---
 .../common/RandomBasedUUIDGenerator.java      |  9 ++---
 .../org/elasticsearch/common/Randomness.java  | 39 +++++++++++++++++--
 .../cluster/ClusterChangedEventTests.java     | 17 +++++---
 .../org/elasticsearch/common/UUIDTests.java   | 17 +++++++-
 .../index/reindex/ReindexSameIndexTests.java  | 25 +++++++-----
 5 files changed, 83 insertions(+), 24 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java b/core/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java
index 9f5e5f34a1b..970adfc03fb 100644
--- a/core/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java
+++ b/core/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java
@@ -19,8 +19,6 @@
 
 package org.elasticsearch.common;
 
-
-import java.io.IOException;
 import java.util.Base64;
 import java.util.Random;
 
@@ -32,7 +30,7 @@ class RandomBasedUUIDGenerator implements UUIDGenerator {
      */
     @Override
     public String getBase64UUID() {
-        return getBase64UUID(SecureRandomHolder.INSTANCE);
+        return getBase64UUID(Randomness.getSecure());
     }
 
     /**
@@ -49,12 +47,13 @@ class RandomBasedUUIDGenerator implements UUIDGenerator {
          * stamp (bits 4 through 7 of the time_hi_and_version field).*/
         randomBytes[6] &= 0x0f;  /* clear the 4 most significant bits for the version  */
         randomBytes[6] |= 0x40;  /* set the version to 0100 / 0x40 */
-        
-        /* Set the variant: 
+
+        /* Set the variant:
          * The high field of th clock sequence multiplexed with the variant.
          * We set only the MSB of the variant*/
         randomBytes[8] &= 0x3f;  /* clear the 2 most significant bits */
         randomBytes[8] |= 0x80;  /* set the variant (MSB is set)*/
         return Base64.getUrlEncoder().withoutPadding().encodeToString(randomBytes);
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/common/Randomness.java b/core/src/main/java/org/elasticsearch/common/Randomness.java
index ddcd3ea90f7..bb700455be5 100644
--- a/core/src/main/java/org/elasticsearch/common/Randomness.java
+++ b/core/src/main/java/org/elasticsearch/common/Randomness.java
@@ -23,6 +23,9 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 
 import java.lang.reflect.Method;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.security.SecureRandom;
 import java.util.Collections;
 import java.util.List;
 import java.util.Random;
@@ -44,6 +47,7 @@ import java.util.concurrent.ThreadLocalRandom;
  * DiscoveryService#NODE_ID_SEED_SETTING)).
  */
 public final class Randomness {
+
     private static final Method currentMethod;
     private static final Method getRandomMethod;
 
@@ -72,7 +76,7 @@ public final class Randomness {
      * @param setting  the setting to access the seed
      * @return a reproducible source of randomness
      */
-    public static Random get(Settings settings, Setting<Long> setting) {
+    public static Random get(final Settings settings, final Setting<Long> setting) {
         if (setting.exists(settings)) {
             return new Random(setting.get(settings));
         } else {
@@ -98,7 +102,7 @@ public final class Randomness {
     public static Random get() {
         if (currentMethod != null && getRandomMethod != null) {
             try {
-                Object randomizedContext = currentMethod.invoke(null);
+                final Object randomizedContext = currentMethod.invoke(null);
                 return (Random) getRandomMethod.invoke(randomizedContext);
             } catch (ReflectiveOperationException e) {
                 // unexpected, bail
@@ -109,13 +113,42 @@ public final class Randomness {
         }
     }
 
+    /**
+     * Provides a source of randomness that is reproducible when
+     * running under the Elasticsearch test suite, and otherwise
+     * produces a non-reproducible source of secure randomness.
+     * Reproducible sources of randomness are created when the system
+     * property "tests.seed" is set and the security policy allows
+     * reading this system property. Otherwise, non-reproducible
+     * sources of secure randomness are created.
+     *
+     * @return a source of randomness
+     * @throws IllegalStateException if running tests but was not able
+     *                               to acquire an instance of Random from
+     *                               RandomizedContext or tests are
+     *                               running but tests.seed is not set
+     */
+    public static Random getSecure() {
+        if (currentMethod != null && getRandomMethod != null) {
+            return get();
+        } else {
+            return getSecureRandomWithoutSeed();
+        }
+    }
+
     @SuppressForbidden(reason = "ThreadLocalRandom is okay when not running tests")
     private static Random getWithoutSeed() {
         assert currentMethod == null && getRandomMethod == null : "running under tests but tried to create non-reproducible random";
         return ThreadLocalRandom.current();
     }
 
-    public static void shuffle(List<?> list) {
+    private static SecureRandom getSecureRandomWithoutSeed() {
+        assert currentMethod == null && getRandomMethod == null : "running under tests but tried to create non-reproducible random";
+        return SecureRandomHolder.INSTANCE;
+    }
+
+    public static void shuffle(final List<?> list) {
         Collections.shuffle(list, get());
     }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java b/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
index 72748a59986..7504c778d36 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
@@ -34,6 +34,7 @@ import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
+import org.junit.BeforeClass;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -54,11 +55,17 @@ public class ClusterChangedEventTests extends ESTestCase {
 
     private static final ClusterName TEST_CLUSTER_NAME = new ClusterName("test");
     private static final String NODE_ID_PREFIX = "node_";
-    private static final String INITIAL_CLUSTER_ID = UUIDs.randomBase64UUID();
-    // the initial indices which every cluster state test starts out with
-    private static final List<Index> initialIndices = Arrays.asList(new Index("idx1", UUIDs.randomBase64UUID()),
-                                                                    new Index("idx2", UUIDs.randomBase64UUID()),
-                                                                    new Index("idx3", UUIDs.randomBase64UUID()));
+    private static String INITIAL_CLUSTER_ID;
+    private static List<Index> initialIndices;
+
+    @BeforeClass
+    public static void beforeClass() {
+        INITIAL_CLUSTER_ID = UUIDs.randomBase64UUID();
+        // the initial indices which every cluster state test starts out with
+        initialIndices = Arrays.asList(new Index("idx1", UUIDs.randomBase64UUID()),
+            new Index("idx2", UUIDs.randomBase64UUID()),
+            new Index("idx3", UUIDs.randomBase64UUID()));
+    }
 
     /**
      * Test basic properties of the ClusterChangedEvent class:
diff --git a/core/src/test/java/org/elasticsearch/common/UUIDTests.java b/core/src/test/java/org/elasticsearch/common/UUIDTests.java
index f82e1a464d9..d963db2d6f6 100644
--- a/core/src/test/java/org/elasticsearch/common/UUIDTests.java
+++ b/core/src/test/java/org/elasticsearch/common/UUIDTests.java
@@ -20,7 +20,9 @@ package org.elasticsearch.common;
 
 import org.elasticsearch.test.ESTestCase;
 
+import java.security.SecureRandom;
 import java.util.HashSet;
+import java.util.Random;
 import java.util.Set;
 
 public class UUIDTests extends ESTestCase {
@@ -41,7 +43,18 @@ public class UUIDTests extends ESTestCase {
     }
 
     public void testThreadedRandomUUID() {
-        testUUIDThreaded(randomUUIDGen);
+        // we can not use a reproducible source of randomness for this
+        // test, the test explicitly relies on each thread having a
+        // unique source of randomness; thus, we fake what production
+        // code does when using a RandomBasedUUIDGenerator
+        testUUIDThreaded(new RandomBasedUUIDGenerator() {
+            private final SecureRandom sr = SecureRandomHolder.INSTANCE;
+
+            @Override
+            public String getBase64UUID() {
+                return getBase64UUID(sr);
+            }
+        });
     }
 
     Set<String> verifyUUIDSet(int count, UUIDGenerator uuidSource) {
@@ -98,6 +111,6 @@ public class UUIDTests extends ESTestCase {
         for (UUIDGenRunner runner : runners) {
             globalSet.addAll(runner.uuidSet);
         }
-        assertEquals(count*uuids, globalSet.size());
+        assertEquals(count * uuids, globalSet.size());
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSameIndexTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSameIndexTests.java
index f1218414af7..7bdb455cc33 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSameIndexTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSameIndexTests.java
@@ -32,6 +32,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
+import org.junit.BeforeClass;
 
 import static org.hamcrest.Matchers.containsString;
 
@@ -39,16 +40,22 @@ import static org.hamcrest.Matchers.containsString;
  * Tests that indexing from an index back into itself fails the request.
  */
 public class ReindexSameIndexTests extends ESTestCase {
-    private static final ClusterState STATE = ClusterState.builder(new ClusterName("test")).metaData(MetaData.builder()
-                .put(index("target", "target_alias", "target_multi"), true)
-                .put(index("target2", "target_multi"), true)
-                .put(index("foo"), true)
-                .put(index("bar"), true)
-                .put(index("baz"), true)
-                .put(index("source", "source_multi"), true)
-                .put(index("source2", "source_multi"), true)).build();
+
+    private static ClusterState STATE;
     private static final IndexNameExpressionResolver INDEX_NAME_EXPRESSION_RESOLVER = new IndexNameExpressionResolver(Settings.EMPTY);
-    private static final AutoCreateIndex AUTO_CREATE_INDEX = new AutoCreateIndex(Settings.EMPTY, INDEX_NAME_EXPRESSION_RESOLVER);
+    private static AutoCreateIndex AUTO_CREATE_INDEX = new AutoCreateIndex(Settings.EMPTY, INDEX_NAME_EXPRESSION_RESOLVER);
+
+    @BeforeClass
+    public static void beforeClass() {
+        STATE = ClusterState.builder(new ClusterName("test")).metaData(MetaData.builder()
+            .put(index("target", "target_alias", "target_multi"), true)
+            .put(index("target2", "target_multi"), true)
+            .put(index("foo"), true)
+            .put(index("bar"), true)
+            .put(index("baz"), true)
+            .put(index("source", "source_multi"), true)
+            .put(index("source2", "source_multi"), true)).build();
+    }
 
     public void testObviousCases() throws Exception {
         fails("target", "target");

From 44c653f5a87337a02daf4737bcb34150df509dbe Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Wed, 8 Jun 2016 15:40:46 +0200
Subject: [PATCH 0776/1311] Upgrade to lucene-6.1.0-snapshot-3a57bea.

---
 .../resources/forbidden/es-all-signatures.txt |   2 -
 buildSrc/version.properties                   |   2 +-
 .../lucene/document/XInetAddressPoint.java    | 117 ----------
 .../lucene/queries/BlendedTermQuery.java      |   4 +-
 .../apache/lucene/queries/MinDocQuery.java    |   4 +-
 .../analyzing/XAnalyzingSuggester.java        |  19 +-
 .../main/java/org/elasticsearch/Version.java  |   2 +-
 .../elasticsearch/bootstrap/JavaVersion.java  |  20 ++
 .../common/geo/GeoHashUtils.java              |  10 +-
 .../elasticsearch/common/geo/GeoPoint.java    |  13 +-
 .../elasticsearch/common/geo/GeoUtils.java    |  12 +-
 .../common/lucene/all/AllTermQuery.java       |  14 ++
 .../lucene/search/MatchNoDocsQuery.java       |  10 +
 .../lucene/search/MoreLikeThisQuery.java      |   4 +-
 .../lucene/search/MultiPhrasePrefixQuery.java |   4 +-
 .../function/FiltersFunctionScoreQuery.java   |   4 +-
 .../search/function/FunctionScoreQuery.java   |   4 +-
 .../common/settings/ClusterSettings.java      |   1 +
 .../fielddata/ordinals/OrdinalsBuilder.java   |   3 +-
 .../plain/AbstractIndexGeoPointFieldData.java |   3 +-
 .../index/mapper/core/DateFieldMapper.java    |   4 +-
 .../mapper/core/LegacyDateFieldMapper.java    |   4 +-
 .../mapper/internal/TypeFieldMapper.java      |   4 +-
 .../index/mapper/ip/IpFieldMapper.java        |  15 +-
 .../query/GeoDistanceRangeQueryBuilder.java   |   9 +-
 .../index/query/HasChildQueryBuilder.java     |   4 +-
 .../index/query/ScriptQueryBuilder.java       |   4 +-
 .../search/geo/GeoDistanceRangeQuery.java     |   4 +-
 .../index/search/geo/GeoPolygonQuery.java     |   4 +-
 .../geo/InMemoryGeoBoundingBoxQuery.java      |   4 +-
 .../index/shard/StoreRecovery.java            |  11 +-
 .../indices/IndicesQueryCache.java            | 221 +++++++++---------
 .../geocentroid/GeoCentroidAggregator.java    |  18 +-
 .../geocentroid/InternalGeoCentroid.java      |  11 +-
 .../fetch/innerhits/InnerHitsContext.java     |   4 +-
 .../search/slice/SliceQuery.java              |   4 +-
 .../search/slice/TermsSliceQuery.java         |   6 +-
 .../elasticsearch/bootstrap/security.policy   |   7 +-
 .../bootstrap/test-framework.policy           |   4 +-
 .../java/org/elasticsearch/VersionTests.java  |   3 +-
 .../bootstrap/JavaVersionTests.java           |   4 +
 .../AbstractGeoFieldDataTestCase.java         |   6 +-
 .../SimpleExternalMappingTests.java           |   8 +-
 .../mapper/geo/GeoPointFieldMapperTests.java  |  42 ++--
 .../geo/GeohashMappingGeoPointTests.java      |   6 +-
 .../index/mapper/ip/IpFieldTypeTests.java     |  13 +-
 .../query/GeoDistanceQueryBuilderTests.java   |   4 +-
 .../query/GeoDistanceRangeQueryTests.java     |   3 +-
 .../query/GeoPolygonQueryBuilderTests.java    |  10 +-
 .../query/plugin/DummyQueryParserPlugin.java  |  10 +
 .../index/shard/StoreRecoveryTests.java       |   7 +-
 .../indices/IndicesQueryCacheTests.java       |   8 +-
 .../indices/stats/IndexStatsIT.java           |   2 +
 .../search/geo/GeoBoundingBoxIT.java          |   2 +
 .../elasticsearch/search/geo/GeoFilterIT.java |   4 +-
 .../search/nested/SimpleNestedIT.java         |   8 +-
 .../lucene-analyzers-common-6.0.1.jar.sha1    |   1 -
 ...ers-common-6.1.0-snapshot-3a57bea.jar.sha1 |   1 +
 .../lucene-backward-codecs-6.0.1.jar.sha1     |   1 -
 ...ard-codecs-6.1.0-snapshot-3a57bea.jar.sha1 |   1 +
 .../licenses/lucene-core-6.0.1.jar.sha1       |   1 -
 ...ucene-core-6.1.0-snapshot-3a57bea.jar.sha1 |   1 +
 .../licenses/lucene-grouping-6.0.1.jar.sha1   |   1 -
 ...e-grouping-6.1.0-snapshot-3a57bea.jar.sha1 |   1 +
 .../lucene-highlighter-6.0.1.jar.sha1         |   1 -
 ...ighlighter-6.1.0-snapshot-3a57bea.jar.sha1 |   1 +
 .../licenses/lucene-join-6.0.1.jar.sha1       |   1 -
 ...ucene-join-6.1.0-snapshot-3a57bea.jar.sha1 |   1 +
 .../licenses/lucene-memory-6.0.1.jar.sha1     |   1 -
 ...ene-memory-6.1.0-snapshot-3a57bea.jar.sha1 |   1 +
 .../licenses/lucene-misc-6.0.1.jar.sha1       |   1 -
 ...ucene-misc-6.1.0-snapshot-3a57bea.jar.sha1 |   1 +
 .../licenses/lucene-queries-6.0.1.jar.sha1    |   1 -
 ...ne-queries-6.1.0-snapshot-3a57bea.jar.sha1 |   1 +
 .../lucene-queryparser-6.0.1.jar.sha1         |   1 -
 ...ueryparser-6.1.0-snapshot-3a57bea.jar.sha1 |   1 +
 .../licenses/lucene-sandbox-6.0.1.jar.sha1    |   1 -
 ...ne-sandbox-6.1.0-snapshot-3a57bea.jar.sha1 |   1 +
 .../licenses/lucene-spatial-6.0.1.jar.sha1    |   1 -
 ...ne-spatial-6.1.0-snapshot-3a57bea.jar.sha1 |   1 +
 .../lucene-spatial-extras-6.0.1.jar.sha1      |   1 -
 ...ial-extras-6.1.0-snapshot-3a57bea.jar.sha1 |   1 +
 .../licenses/lucene-spatial3d-6.0.1.jar.sha1  |   1 -
 ...-spatial3d-6.1.0-snapshot-3a57bea.jar.sha1 |   1 +
 .../licenses/lucene-suggest-6.0.1.jar.sha1    |   1 -
 ...ne-suggest-6.1.0-snapshot-3a57bea.jar.sha1 |   1 +
 .../lucene-expressions-6.0.1.jar.sha1         |   1 -
 ...xpressions-6.1.0-snapshot-3a57bea.jar.sha1 |   1 +
 .../messy/tests/SimpleSortTests.java          |   4 +-
 .../percolator/PercolateQuery.java            |   4 +-
 .../percolator/PercolateQueryTests.java       |  10 +
 .../lucene-analyzers-icu-6.0.1.jar.sha1       |   1 -
 ...lyzers-icu-6.1.0-snapshot-3a57bea.jar.sha1 |   1 +
 .../lucene-analyzers-kuromoji-6.0.1.jar.sha1  |   1 -
 ...s-kuromoji-6.1.0-snapshot-3a57bea.jar.sha1 |   1 +
 .../lucene-analyzers-phonetic-6.0.1.jar.sha1  |   1 -
 ...s-phonetic-6.1.0-snapshot-3a57bea.jar.sha1 |   1 +
 .../lucene-analyzers-smartcn-6.0.1.jar.sha1   |   1 -
 ...rs-smartcn-6.1.0-snapshot-3a57bea.jar.sha1 |   1 +
 .../lucene-analyzers-stempel-6.0.1.jar.sha1   |   1 -
 ...rs-stempel-6.1.0-snapshot-3a57bea.jar.sha1 |   1 +
 .../elasticsearch/test/ESIntegTestCase.java   |   7 +-
 102 files changed, 384 insertions(+), 417 deletions(-)
 delete mode 100644 core/src/main/java/org/apache/lucene/document/XInetAddressPoint.java
 delete mode 100644 distribution/licenses/lucene-analyzers-common-6.0.1.jar.sha1
 create mode 100644 distribution/licenses/lucene-analyzers-common-6.1.0-snapshot-3a57bea.jar.sha1
 delete mode 100644 distribution/licenses/lucene-backward-codecs-6.0.1.jar.sha1
 create mode 100644 distribution/licenses/lucene-backward-codecs-6.1.0-snapshot-3a57bea.jar.sha1
 delete mode 100644 distribution/licenses/lucene-core-6.0.1.jar.sha1
 create mode 100644 distribution/licenses/lucene-core-6.1.0-snapshot-3a57bea.jar.sha1
 delete mode 100644 distribution/licenses/lucene-grouping-6.0.1.jar.sha1
 create mode 100644 distribution/licenses/lucene-grouping-6.1.0-snapshot-3a57bea.jar.sha1
 delete mode 100644 distribution/licenses/lucene-highlighter-6.0.1.jar.sha1
 create mode 100644 distribution/licenses/lucene-highlighter-6.1.0-snapshot-3a57bea.jar.sha1
 delete mode 100644 distribution/licenses/lucene-join-6.0.1.jar.sha1
 create mode 100644 distribution/licenses/lucene-join-6.1.0-snapshot-3a57bea.jar.sha1
 delete mode 100644 distribution/licenses/lucene-memory-6.0.1.jar.sha1
 create mode 100644 distribution/licenses/lucene-memory-6.1.0-snapshot-3a57bea.jar.sha1
 delete mode 100644 distribution/licenses/lucene-misc-6.0.1.jar.sha1
 create mode 100644 distribution/licenses/lucene-misc-6.1.0-snapshot-3a57bea.jar.sha1
 delete mode 100644 distribution/licenses/lucene-queries-6.0.1.jar.sha1
 create mode 100644 distribution/licenses/lucene-queries-6.1.0-snapshot-3a57bea.jar.sha1
 delete mode 100644 distribution/licenses/lucene-queryparser-6.0.1.jar.sha1
 create mode 100644 distribution/licenses/lucene-queryparser-6.1.0-snapshot-3a57bea.jar.sha1
 delete mode 100644 distribution/licenses/lucene-sandbox-6.0.1.jar.sha1
 create mode 100644 distribution/licenses/lucene-sandbox-6.1.0-snapshot-3a57bea.jar.sha1
 delete mode 100644 distribution/licenses/lucene-spatial-6.0.1.jar.sha1
 create mode 100644 distribution/licenses/lucene-spatial-6.1.0-snapshot-3a57bea.jar.sha1
 delete mode 100644 distribution/licenses/lucene-spatial-extras-6.0.1.jar.sha1
 create mode 100644 distribution/licenses/lucene-spatial-extras-6.1.0-snapshot-3a57bea.jar.sha1
 delete mode 100644 distribution/licenses/lucene-spatial3d-6.0.1.jar.sha1
 create mode 100644 distribution/licenses/lucene-spatial3d-6.1.0-snapshot-3a57bea.jar.sha1
 delete mode 100644 distribution/licenses/lucene-suggest-6.0.1.jar.sha1
 create mode 100644 distribution/licenses/lucene-suggest-6.1.0-snapshot-3a57bea.jar.sha1
 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-6.0.1.jar.sha1
 create mode 100644 modules/lang-expression/licenses/lucene-expressions-6.1.0-snapshot-3a57bea.jar.sha1
 delete mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-6.0.1.jar.sha1
 create mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-6.1.0-snapshot-3a57bea.jar.sha1
 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.0.1.jar.sha1
 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.1.0-snapshot-3a57bea.jar.sha1
 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.0.1.jar.sha1
 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.1.0-snapshot-3a57bea.jar.sha1
 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.0.1.jar.sha1
 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.1.0-snapshot-3a57bea.jar.sha1
 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.0.1.jar.sha1
 create mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.1.0-snapshot-3a57bea.jar.sha1

diff --git a/buildSrc/src/main/resources/forbidden/es-all-signatures.txt b/buildSrc/src/main/resources/forbidden/es-all-signatures.txt
index 0e5ce884d9d..e31a7020282 100644
--- a/buildSrc/src/main/resources/forbidden/es-all-signatures.txt
+++ b/buildSrc/src/main/resources/forbidden/es-all-signatures.txt
@@ -31,5 +31,3 @@ org.apache.lucene.index.IndexReader#getCombinedCoreAndDeletesKey()
 
 @defaultMessage Soon to be removed
 org.apache.lucene.document.FieldType#numericType()
-
-org.apache.lucene.document.InetAddressPoint#newPrefixQuery(java.lang.String, java.net.InetAddress, int) @LUCENE-7232
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index d9e3908df22..f757eb8eef6 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 elasticsearch     = 5.0.0
-lucene            = 6.0.1
+lucene            = 6.1.0-snapshot-3a57bea
 
 # optional dependencies
 spatial4j         = 0.6
diff --git a/core/src/main/java/org/apache/lucene/document/XInetAddressPoint.java b/core/src/main/java/org/apache/lucene/document/XInetAddressPoint.java
deleted file mode 100644
index 580b875ce2c..00000000000
--- a/core/src/main/java/org/apache/lucene/document/XInetAddressPoint.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.document;
-
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.Arrays;
-
-import org.apache.lucene.search.Query;
-import org.apache.lucene.util.NumericUtils;
-import org.elasticsearch.common.SuppressForbidden;
-
-/**
- * Forked utility methods from Lucene's InetAddressPoint until LUCENE-7232 and
- * LUCENE-7234 are released.
- */
-// TODO: remove me when we upgrade to Lucene 6.1
-@SuppressForbidden(reason="uses InetAddress.getHostAddress")
-public final class XInetAddressPoint {
-
-    private XInetAddressPoint() {}
-
-    /** The minimum value that an ip address can hold. */
-    public static final InetAddress MIN_VALUE;
-    /** The maximum value that an ip address can hold. */
-    public static final InetAddress MAX_VALUE;
-    static {
-      MIN_VALUE = InetAddressPoint.decode(new byte[InetAddressPoint.BYTES]);
-      byte[] maxValueBytes = new byte[InetAddressPoint.BYTES];
-      Arrays.fill(maxValueBytes, (byte) 0xFF);
-      MAX_VALUE = InetAddressPoint.decode(maxValueBytes);
-    }
-
-    /**
-     * Return the {@link InetAddress} that compares immediately greater than
-     * {@code address}.
-     * @throws ArithmeticException if the provided address is the
-     *              {@link #MAX_VALUE maximum ip address}
-     */
-    public static InetAddress nextUp(InetAddress address) {
-      if (address.equals(MAX_VALUE)) {
-        throw new ArithmeticException("Overflow: there is no greater InetAddress than "
-            + address.getHostAddress());
-      }
-      byte[] delta = new byte[InetAddressPoint.BYTES];
-      delta[InetAddressPoint.BYTES-1] = 1;
-      byte[] nextUpBytes = new byte[InetAddressPoint.BYTES];
-      NumericUtils.add(InetAddressPoint.BYTES, 0, InetAddressPoint.encode(address), delta, nextUpBytes);
-      return InetAddressPoint.decode(nextUpBytes);
-    }
-
-    /**
-     * Return the {@link InetAddress} that compares immediately less than
-     * {@code address}.
-     * @throws ArithmeticException if the provided address is the
-     *              {@link #MIN_VALUE minimum ip address}
-     */
-    public static InetAddress nextDown(InetAddress address) {
-      if (address.equals(MIN_VALUE)) {
-        throw new ArithmeticException("Underflow: there is no smaller InetAddress than "
-            + address.getHostAddress());
-      }
-      byte[] delta = new byte[InetAddressPoint.BYTES];
-      delta[InetAddressPoint.BYTES-1] = 1;
-      byte[] nextDownBytes = new byte[InetAddressPoint.BYTES];
-      NumericUtils.subtract(InetAddressPoint.BYTES, 0, InetAddressPoint.encode(address), delta, nextDownBytes);
-      return InetAddressPoint.decode(nextDownBytes);
-    }
-
-    /** 
-     * Create a prefix query for matching a CIDR network range.
-     *
-     * @param field field name. must not be {@code null}.
-     * @param value any host address
-     * @param prefixLength the network prefix length for this address. This is also known as the subnet mask in the context of IPv4
-     * addresses.
-     * @throws IllegalArgumentException if {@code field} is null, or prefixLength is invalid.
-     * @return a query matching documents with addresses contained within this network
-     */
-    // TODO: remove me when we upgrade to Lucene 6.0.1
-    public static Query newPrefixQuery(String field, InetAddress value, int prefixLength) {
-      if (value == null) {
-        throw new IllegalArgumentException("InetAddress must not be null");
-      }
-      if (prefixLength < 0 || prefixLength > 8 * value.getAddress().length) {
-        throw new IllegalArgumentException("illegal prefixLength '" + prefixLength
-                + "'. Must be 0-32 for IPv4 ranges, 0-128 for IPv6 ranges");
-      }
-      // create the lower value by zeroing out the host portion, upper value by filling it with all ones.
-      byte lower[] = value.getAddress();
-      byte upper[] = value.getAddress();
-      for (int i = prefixLength; i < 8 * lower.length; i++) {
-        int m = 1 << (7 - (i & 7));
-        lower[i >> 3] &= ~m;
-        upper[i >> 3] |= m;
-      }
-      try {
-        return InetAddressPoint.newRangeQuery(field, InetAddress.getByAddress(lower), InetAddress.getByAddress(upper));
-      } catch (UnknownHostException e) {
-        throw new AssertionError(e); // values are coming from InetAddress
-      }
-    }
-}
diff --git a/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java b/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java
index 564f780b8ed..a4b94b007fd 100644
--- a/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java
+++ b/core/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java
@@ -283,7 +283,7 @@ public abstract class BlendedTermQuery extends Query {
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
-        if (!super.equals(o)) return false;
+        if (sameClassAs(o) == false) return false;
 
         BlendedTermQuery that = (BlendedTermQuery) o;
         return Arrays.equals(equalsTerms(), that.equalsTerms());
@@ -291,7 +291,7 @@ public abstract class BlendedTermQuery extends Query {
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), Arrays.hashCode(equalsTerms()));
+        return Objects.hash(classHash(), Arrays.hashCode(equalsTerms()));
     }
 
     public static BlendedTermQuery booleanBlendedQuery(Term[] terms, final boolean disableCoord) {
diff --git a/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java b/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java
index 86982bfc949..a8b7dc9299f 100644
--- a/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java
+++ b/core/src/main/java/org/apache/lucene/queries/MinDocQuery.java
@@ -44,12 +44,12 @@ public final class MinDocQuery extends Query {
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), minDoc);
+        return Objects.hash(classHash(), minDoc);
     }
 
     @Override
     public boolean equals(Object obj) {
-        if (super.equals(obj) == false) {
+        if (sameClassAs(obj) == false) {
             return false;
         }
         MinDocQuery that = (MinDocQuery) obj;
diff --git a/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java b/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java
index a9327d785e1..6017803b63d 100644
--- a/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java
+++ b/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java
@@ -63,9 +63,6 @@ import org.elasticsearch.common.io.PathUtils;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -622,8 +619,12 @@ public long ramBytesUsed() {
       Set<BytesRef> seenSurfaceForms = new HashSet<>();
 
       int dedup = 0;
-      while (reader.read(scratch)) {
-        input.reset(scratch.bytes(), 0, scratch.length());
+      while (true) {
+        BytesRef bytes = reader.next();
+        if (bytes == null) {
+          break;
+        }
+        input.reset(bytes.bytes, bytes.offset, bytes.length);
         short analyzedLength = input.readShort();
         analyzed.grow(analyzedLength+2);
         input.readBytes(analyzed.bytes(), 0, analyzedLength);
@@ -631,13 +632,13 @@ public long ramBytesUsed() {
 
         long cost = input.readInt();
 
-        surface.bytes = scratch.bytes();
+        surface.bytes = bytes.bytes;
         if (hasPayloads) {
           surface.length = input.readShort();
           surface.offset = input.getPosition();
         } else {
           surface.offset = input.getPosition();
-          surface.length = scratch.length() - surface.offset;
+          surface.length = bytes.length - surface.offset;
         }
 
         if (previousAnalyzed == null) {
@@ -679,11 +680,11 @@ public long ramBytesUsed() {
           builder.add(scratchInts.get(), outputs.newPair(cost, BytesRef.deepCopyOf(surface)));
         } else {
           int payloadOffset = input.getPosition() + surface.length;
-          int payloadLength = scratch.length() - payloadOffset;
+          int payloadLength = bytes.length - payloadOffset;
           BytesRef br = new BytesRef(surface.length + 1 + payloadLength);
           System.arraycopy(surface.bytes, surface.offset, br.bytes, 0, surface.length);
           br.bytes[surface.length] = (byte) payloadSep;
-          System.arraycopy(scratch.bytes(), payloadOffset, br.bytes, surface.length+1, payloadLength);
+          System.arraycopy(bytes.bytes, payloadOffset, br.bytes, surface.length+1, payloadLength);
           br.length = br.bytes.length;
           builder.add(scratchInts.get(), outputs.newPair(cost, br));
         }
diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java
index 5bdbf76265d..6e6e82b3fc5 100644
--- a/core/src/main/java/org/elasticsearch/Version.java
+++ b/core/src/main/java/org/elasticsearch/Version.java
@@ -77,7 +77,7 @@ public class Version {
     public static final int V_5_0_0_alpha3_ID = 5000003;
     public static final Version V_5_0_0_alpha3 = new Version(V_5_0_0_alpha3_ID, org.apache.lucene.util.Version.LUCENE_6_0_0);
     public static final int V_5_0_0_ID = 5000099;
-    public static final Version V_5_0_0 = new Version(V_5_0_0_ID, org.apache.lucene.util.Version.LUCENE_6_0_1);
+    public static final Version V_5_0_0 = new Version(V_5_0_0_ID, org.apache.lucene.util.Version.LUCENE_6_1_0);
     public static final Version CURRENT = V_5_0_0;
 
     static {
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/JavaVersion.java b/core/src/main/java/org/elasticsearch/bootstrap/JavaVersion.java
index afc2b77e211..4a434ebe6b7 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/JavaVersion.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/JavaVersion.java
@@ -33,6 +33,13 @@ public class JavaVersion implements Comparable<JavaVersion> {
     }
 
     private JavaVersion(List<Integer> version) {
+        if (version.size() >= 2
+                && version.get(0).intValue() == 1
+                && version.get(1).intValue() == 8) {
+            // for Java 8 there is ambiguity since both 1.8 and 8 are supported,
+            // so we rewrite the former to the latter
+            version = new ArrayList<>(version.subList(1, version.size()));
+        }
         this.version = Collections.unmodifiableList(version);
     }
 
@@ -75,6 +82,19 @@ public class JavaVersion implements Comparable<JavaVersion> {
         return 0;
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (o == null || o.getClass() != getClass()) {
+            return false;
+        }
+        return compareTo((JavaVersion) o) == 0;
+    }
+
+    @Override
+    public int hashCode() {
+        return version.hashCode();
+    }
+
     @Override
     public String toString() {
         return version.stream().map(v -> Integer.toString(v)).collect(Collectors.joining("."));
diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java b/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java
index 4087704d5cd..9982a08f17f 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java
@@ -19,7 +19,7 @@ package org.elasticsearch.common.geo;
 import java.util.ArrayList;
 import java.util.Collection;
 
-import org.apache.lucene.spatial.util.GeoEncodingUtils;
+import org.apache.lucene.spatial.geopoint.document.GeoPointField;
 import org.apache.lucene.util.BitUtil;
 
 /**
@@ -39,7 +39,7 @@ public class GeoHashUtils {
 
     /** maximum precision for geohash strings */
     public static final int PRECISION = 12;
-    private static final short MORTON_OFFSET = (GeoEncodingUtils.BITS<<1) - (PRECISION*5);
+    private static final short MORTON_OFFSET = (GeoPointField.BITS<<1) - (PRECISION*5);
 
     // No instance:
     private GeoHashUtils() {
@@ -51,7 +51,7 @@ public class GeoHashUtils {
     public static final long longEncode(final double lon, final double lat, final int level) {
         // shift to appropriate level
         final short msf = (short)(((12 - level) * 5) + MORTON_OFFSET);
-        return ((BitUtil.flipFlop(GeoEncodingUtils.mortonHash(lat, lon)) >>> msf) << 4) | level;
+        return ((BitUtil.flipFlop(GeoPointField.encodeLatLon(lat, lon)) >>> msf) << 4) | level;
     }
 
     /**
@@ -117,7 +117,7 @@ public class GeoHashUtils {
      */
     public static final String stringEncode(final double lon, final double lat, final int level) {
         // convert to geohashlong
-        final long ghLong = fromMorton(GeoEncodingUtils.mortonHash(lat, lon), level);
+        final long ghLong = fromMorton(GeoPointField.encodeLatLon(lat, lon), level);
         return stringEncode(ghLong);
 
     }
@@ -138,7 +138,7 @@ public class GeoHashUtils {
 
         StringBuilder geoHash = new StringBuilder();
         short precision = 0;
-        final short msf = (GeoEncodingUtils.BITS<<1)-5;
+        final short msf = (GeoPointField.BITS<<1)-5;
         long mask = 31L<<msf;
         do {
             geoHash.append(BASE_32[(int)((mask & hashedVal)>>>(msf-(precision*5)))]);
diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java b/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java
index 5d1250a5148..96fe2826da8 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java
@@ -19,12 +19,11 @@
 
 package org.elasticsearch.common.geo;
 
+import org.apache.lucene.spatial.geopoint.document.GeoPointField;
 import org.apache.lucene.util.BitUtil;
 
 import static org.elasticsearch.common.geo.GeoHashUtils.mortonEncode;
 import static org.elasticsearch.common.geo.GeoHashUtils.stringEncode;
-import static org.apache.lucene.spatial.util.GeoEncodingUtils.mortonUnhashLat;
-import static org.apache.lucene.spatial.util.GeoEncodingUtils.mortonUnhashLon;
 
 /**
  *
@@ -84,14 +83,14 @@ public final class GeoPoint {
     }
 
     public GeoPoint resetFromIndexHash(long hash) {
-        lon = mortonUnhashLon(hash);
-        lat = mortonUnhashLat(hash);
+        lon = GeoPointField.decodeLongitude(hash);
+        lat = GeoPointField.decodeLatitude(hash);
         return this;
     }
 
     public GeoPoint resetFromGeoHash(String geohash) {
         final long hash = mortonEncode(geohash);
-        return this.reset(mortonUnhashLat(hash), mortonUnhashLon(hash));
+        return this.reset(GeoPointField.decodeLatitude(hash), GeoPointField.decodeLongitude(hash));
     }
 
     public GeoPoint resetFromGeoHash(long geohashLong) {
@@ -164,8 +163,4 @@ public final class GeoPoint {
     public static GeoPoint fromGeohash(long geohashLong) {
         return new GeoPoint().resetFromGeoHash(geohashLong);
     }
-
-    public static GeoPoint fromIndexLong(long indexLong) {
-        return new GeoPoint().resetFromIndexHash(indexLong);
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java b/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java
index d5cc6846865..69ab2059ccf 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/GeoUtils.java
@@ -28,7 +28,6 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper;
 
-import static org.apache.lucene.spatial.util.GeoDistanceUtils.maxRadialDistanceMeters;
 
 import java.io.IOException;
 
@@ -67,6 +66,9 @@ public class GeoUtils {
     /** Earth ellipsoid polar distance in meters */
     public static final double EARTH_POLAR_DISTANCE = Math.PI * EARTH_SEMI_MINOR_AXIS;
 
+    /** rounding error for quantized latitude and longitude values */
+    public static final double TOLERANCE = 1E-6;
+
     /** Returns the minimum between the provided distance 'initialRadius' and the
      * maximum distance/radius from the point 'center' before overlapping
      **/
@@ -468,6 +470,14 @@ public class GeoUtils {
         }
     }
 
+    /** Returns the maximum distance/radius (in meters) from the point 'center' before overlapping */
+    public static double maxRadialDistanceMeters(final double centerLat, final double centerLon) {
+      if (Math.abs(centerLat) == MAX_LAT) {
+        return SloppyMath.haversinMeters(centerLat, centerLon, 0, centerLon);
+      }
+      return SloppyMath.haversinMeters(centerLat, centerLon, centerLat, (MAX_LON + centerLon) % 360);
+    }
+
     private GeoUtils() {
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java
index 9b995f423a3..75f400fdc9d 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java
@@ -45,6 +45,7 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.SmallFloat;
 
 import java.io.IOException;
+import java.util.Objects;
 import java.util.Set;
 
 /**
@@ -63,6 +64,19 @@ public final class AllTermQuery extends Query {
         this.term = term;
     }
 
+    @Override
+    public boolean equals(Object obj) {
+        if (sameClassAs(obj) == false) {
+            return false;
+        }
+        return Objects.equals(term, ((AllTermQuery) obj).term);
+    }
+
+    @Override
+    public int hashCode() {
+        return 31 * classHash() + term.hashCode();
+    }
+
     @Override
     public Query rewrite(IndexReader reader) throws IOException {
         Query rewritten = super.rewrite(reader);
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/MatchNoDocsQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/MatchNoDocsQuery.java
index a25b4c0aa29..9caf350926c 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/MatchNoDocsQuery.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/MatchNoDocsQuery.java
@@ -66,4 +66,14 @@ public class MatchNoDocsQuery extends Query {
     public String toString(String field) {
         return "MatchNoDocsQuery[\"" + reason + "\"]";
     }
+
+    @Override
+    public boolean equals(Object obj) {
+        return sameClassAs(obj);
+    }
+
+    @Override
+    public int hashCode() {
+        return classHash();
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java
index fbe0c28e341..06ab2b4a530 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/MoreLikeThisQuery.java
@@ -84,14 +84,14 @@ public class MoreLikeThisQuery extends Query {
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), boostTerms, boostTermsFactor, Arrays.hashCode(likeText),
+        return Objects.hash(classHash(), boostTerms, boostTermsFactor, Arrays.hashCode(likeText),
                 maxDocFreq, maxQueryTerms, maxWordLen, minDocFreq, minTermFrequency, minWordLen,
                 Arrays.hashCode(moreLikeFields), minimumShouldMatch, stopWords);
     }
 
     @Override
     public boolean equals(Object obj) {
-        if (super.equals(obj) == false) {
+        if (sameClassAs(obj) == false) {
             return false;
         }
         MoreLikeThisQuery other = (MoreLikeThisQuery) obj;
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java
index 05006ec0db7..87bfdacb1c7 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/MultiPhrasePrefixQuery.java
@@ -238,7 +238,7 @@ public class MultiPhrasePrefixQuery extends Query {
      */
     @Override
     public boolean equals(Object o) {
-        if (super.equals(o) == false) {
+        if (sameClassAs(o) == false) {
             return false;
         }
         MultiPhrasePrefixQuery other = (MultiPhrasePrefixQuery) o;
@@ -252,7 +252,7 @@ public class MultiPhrasePrefixQuery extends Query {
      */
     @Override
     public int hashCode() {
-        return super.hashCode()
+        return classHash()
                 ^ slop
                 ^ termArraysHashCode()
                 ^ positions.hashCode();
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java
index e62f3f6665a..3927dcd518e 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java
@@ -355,7 +355,7 @@ public class FiltersFunctionScoreQuery extends Query {
         if (this == o) {
             return true;
         }
-        if (super.equals(o) == false) {
+        if (sameClassAs(o) == false) {
             return false;
         }
         FiltersFunctionScoreQuery other = (FiltersFunctionScoreQuery) o;
@@ -367,6 +367,6 @@ public class FiltersFunctionScoreQuery extends Query {
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), subQuery, maxBoost, combineFunction, minScore, scoreMode, Arrays.hashCode(filterFunctions));
+        return Objects.hash(classHash(), subQuery, maxBoost, combineFunction, minScore, scoreMode, Arrays.hashCode(filterFunctions));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java
index 646076a3a17..be98a07a9c1 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java
@@ -210,7 +210,7 @@ public class FunctionScoreQuery extends Query {
         if (this == o) {
             return true;
         }
-        if (super.equals(o) == false) {
+        if (sameClassAs(o) == false) {
             return false;
         }
         FunctionScoreQuery other = (FunctionScoreQuery) o;
@@ -221,6 +221,6 @@ public class FunctionScoreQuery extends Query {
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), subQuery.hashCode(), function, combineFunction, minScore, maxBoost);
+        return Objects.hash(classHash(), subQuery.hashCode(), function, combineFunction, minScore, maxBoost);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index 6eb8df68242..8c2d4dc01bf 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -181,6 +181,7 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     IndexStoreConfig.INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC_SETTING,
                     IndicesQueryCache.INDICES_CACHE_QUERY_SIZE_SETTING,
                     IndicesQueryCache.INDICES_CACHE_QUERY_COUNT_SETTING,
+                    IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING,
                     IndicesTTLService.INDICES_TTL_INTERVAL_SETTING,
                     MappingUpdatedAction.INDICES_MAPPING_DYNAMIC_TIMEOUT_SETTING,
                     MetaData.SETTING_READ_ONLY_SETTING,
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java
index e3dc84a3477..967d07174b9 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/OrdinalsBuilder.java
@@ -24,7 +24,6 @@ import org.apache.lucene.index.PostingsEnum;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.spatial.geopoint.document.GeoPointField;
-import org.apache.lucene.spatial.util.GeoEncodingUtils;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BitSet;
 import org.apache.lucene.util.BytesRef;
@@ -426,7 +425,7 @@ public final class OrdinalsBuilder implements Closeable {
             protected AcceptStatus accept(BytesRef term) throws IOException {
                 // accept only the max resolution terms
                 // todo is this necessary?
-                return GeoEncodingUtils.getPrefixCodedShift(term) == GeoPointField.PRECISION_STEP * 4 ?
+                return GeoPointField.getPrefixCodedShift(term) == GeoPointField.PRECISION_STEP * 4 ?
                     AcceptStatus.YES : AcceptStatus.END;
             }
         };
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java
index c18f96c06b0..90554bd1308 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexGeoPointFieldData.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.index.fielddata.plain;
 
 import org.apache.lucene.spatial.geopoint.document.GeoPointField;
-import org.apache.lucene.spatial.util.GeoEncodingUtils;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefIterator;
 import org.apache.lucene.util.CharsRefBuilder;
@@ -58,7 +57,7 @@ abstract class AbstractIndexGeoPointFieldData extends AbstractIndexFieldData<Ato
                 return null;
             }
             if (termEncoding == GeoPointField.TermEncoding.PREFIX) {
-                return GeoEncodingUtils.prefixCodedToGeoCoded(term);
+                return GeoPointField.prefixCodedToGeoCoded(term);
             } else if (termEncoding == GeoPointField.TermEncoding.NUMERIC) {
                 return LegacyNumericUtils.prefixCodedToLong(term);
             }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
index a79631481d2..66cb7255fd6 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
@@ -204,7 +204,7 @@ public class DateFieldMapper extends FieldMapper implements AllFieldMapper.Inclu
             @Override
             public boolean equals(Object o) {
                 if (this == o) return true;
-                if (!super.equals(o)) return false;
+                if (sameClassAs(o) == false) return false;
 
                 LateParsingQuery that = (LateParsingQuery) o;
                 if (includeLower != that.includeLower) return false;
@@ -218,7 +218,7 @@ public class DateFieldMapper extends FieldMapper implements AllFieldMapper.Inclu
 
             @Override
             public int hashCode() {
-                return Objects.hash(super.hashCode(), lowerTerm, upperTerm, includeLower, includeUpper, timeZone);
+                return Objects.hash(classHash(), lowerTerm, upperTerm, includeLower, includeUpper, timeZone);
             }
 
             @Override
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDateFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDateFieldMapper.java
index a7e44ba3654..a3374153955 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDateFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyDateFieldMapper.java
@@ -213,7 +213,7 @@ public class LegacyDateFieldMapper extends LegacyNumberFieldMapper {
             @Override
             public boolean equals(Object o) {
                 if (this == o) return true;
-                if (!super.equals(o)) return false;
+                if (sameClassAs(o) == false) return false;
 
                 LateParsingQuery that = (LateParsingQuery) o;
                 if (includeLower != that.includeLower) return false;
@@ -227,7 +227,7 @@ public class LegacyDateFieldMapper extends LegacyNumberFieldMapper {
 
             @Override
             public int hashCode() {
-                return Objects.hash(super.hashCode(), lowerTerm, upperTerm, includeLower, includeUpper, timeZone);
+                return Objects.hash(classHash(), lowerTerm, upperTerm, includeLower, includeUpper, timeZone);
             }
 
             @Override
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
index f960ecaa977..d882be8e9d7 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java
@@ -195,7 +195,7 @@ public class TypeFieldMapper extends MetadataFieldMapper {
 
         @Override
         public boolean equals(Object obj) {
-            if (super.equals(obj) == false) {
+            if (sameClassAs(obj) == false) {
                 return false;
             }
             TypeQuery that = (TypeQuery) obj;
@@ -204,7 +204,7 @@ public class TypeFieldMapper extends MetadataFieldMapper {
 
         @Override
         public int hashCode() {
-            return 31 * super.hashCode() + type.hashCode();
+            return 31 * classHash() + type.hashCode();
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
index 1e2a078925f..a123f64c4d6 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java
@@ -23,7 +23,6 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.document.StoredField;
-import org.apache.lucene.document.XInetAddressPoint;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.XPointValues;
@@ -176,7 +175,7 @@ public class IpFieldMapper extends FieldMapper implements AllFieldMapper.Include
                     if (fields.length == 2) {
                         InetAddress address = InetAddresses.forString(fields[0]);
                         int prefixLength = Integer.parseInt(fields[1]);
-                        return XInetAddressPoint.newPrefixQuery(name(), address, prefixLength);
+                        return InetAddressPoint.newPrefixQuery(name(), address, prefixLength);
                     } else {
                         throw new IllegalArgumentException("Expected [ip/prefix] but was [" + term + "]");
                     }
@@ -191,27 +190,27 @@ public class IpFieldMapper extends FieldMapper implements AllFieldMapper.Include
             failIfNotIndexed();
             InetAddress lower;
             if (lowerTerm == null) {
-                lower = XInetAddressPoint.MIN_VALUE;
+                lower = InetAddressPoint.MIN_VALUE;
             } else {
                 lower = parse(lowerTerm);
                 if (includeLower == false) {
-                    if (lower.equals(XInetAddressPoint.MAX_VALUE)) {
+                    if (lower.equals(InetAddressPoint.MAX_VALUE)) {
                         return new MatchNoDocsQuery();
                     }
-                    lower = XInetAddressPoint.nextUp(lower);
+                    lower = InetAddressPoint.nextUp(lower);
                 }
             }
 
             InetAddress upper;
             if (upperTerm == null) {
-                upper = XInetAddressPoint.MAX_VALUE;
+                upper = InetAddressPoint.MAX_VALUE;
             } else {
                 upper = parse(upperTerm);
                 if (includeUpper == false) {
-                    if (upper.equals(XInetAddressPoint.MIN_VALUE)) {
+                    if (upper.equals(InetAddressPoint.MIN_VALUE)) {
                         return new MatchNoDocsQuery();
                     }
-                    upper = XInetAddressPoint.nextDown(upper);
+                    upper = InetAddressPoint.nextDown(upper);
                 }
             }
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java
index 06f30a3477e..cf18d5a4c10 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeQueryBuilder.java
@@ -23,7 +23,6 @@ import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.spatial.geopoint.document.GeoPointField;
 import org.apache.lucene.spatial.geopoint.search.XGeoPointDistanceRangeQuery;
-import org.apache.lucene.spatial.util.GeoDistanceUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
@@ -48,8 +47,6 @@ import java.util.Locale;
 import java.util.Objects;
 import java.util.Optional;
 
-import static org.apache.lucene.spatial.util.GeoEncodingUtils.TOLERANCE;
-
 public class GeoDistanceRangeQueryBuilder extends AbstractQueryBuilder<GeoDistanceRangeQueryBuilder> {
 
     public static final String NAME = "geo_distance_range";
@@ -354,7 +351,7 @@ public class GeoDistanceRangeQueryBuilder extends AbstractQueryBuilder<GeoDistan
                 toValue = geoDistance.normalize(toValue, DistanceUnit.DEFAULT);
             }
         } else {
-            toValue = GeoDistanceUtils.maxRadialDistanceMeters(point.lat(), point.lon());
+            toValue = GeoUtils.maxRadialDistanceMeters(point.lat(), point.lon());
         }
 
         final Version indexVersionCreated = context.indexVersionCreated();
@@ -371,8 +368,8 @@ public class GeoDistanceRangeQueryBuilder extends AbstractQueryBuilder<GeoDistan
             GeoPointField.TermEncoding.NUMERIC : GeoPointField.TermEncoding.PREFIX;
 
         return new XGeoPointDistanceRangeQuery(fieldType.name(), encoding, point.lat(), point.lon(),
-            (includeLower) ? fromValue : fromValue + TOLERANCE,
-            (includeUpper) ? toValue : toValue - TOLERANCE);
+            (includeLower) ? fromValue : fromValue + GeoUtils.TOLERANCE,
+            (includeUpper) ? toValue : toValue - GeoUtils.TOLERANCE);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
index 9fb44845f22..389e3812e33 100644
--- a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
@@ -411,7 +411,7 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
 
         @Override
         public boolean equals(Object o) {
-            if (!super.equals(o)) return false;
+            if (sameClassAs(o) == false) return false;
 
             LateParsingQuery that = (LateParsingQuery) o;
 
@@ -425,7 +425,7 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
 
         @Override
         public int hashCode() {
-            return Objects.hash(super.hashCode(), toQuery, innerQuery, minChildren, maxChildren, parentType, scoreMode);
+            return Objects.hash(classHash(), toQuery, innerQuery, minChildren, maxChildren, parentType, scoreMode);
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
index 6d563f22a06..778501f0132 100644
--- a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
@@ -184,7 +184,7 @@ public class ScriptQueryBuilder extends AbstractQueryBuilder<ScriptQueryBuilder>
         public boolean equals(Object obj) {
             if (this == obj)
                 return true;
-            if (!super.equals(obj))
+            if (sameClassAs(obj) == false)
                 return false;
             ScriptQuery other = (ScriptQuery) obj;
             return Objects.equals(script, other.script);
@@ -192,7 +192,7 @@ public class ScriptQueryBuilder extends AbstractQueryBuilder<ScriptQueryBuilder>
 
         @Override
         public int hashCode() {
-            return Objects.hash(super.hashCode(), script);
+            return Objects.hash(classHash(), script);
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceRangeQuery.java b/core/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceRangeQuery.java
index 6f92e411c00..6c4fd23e64c 100644
--- a/core/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceRangeQuery.java
+++ b/core/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceRangeQuery.java
@@ -190,7 +190,7 @@ public class GeoDistanceRangeQuery extends Query {
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
-        if (super.equals(o) == false) return false;
+        if (sameClassAs(o) == false) return false;
 
         GeoDistanceRangeQuery filter = (GeoDistanceRangeQuery) o;
 
@@ -212,7 +212,7 @@ public class GeoDistanceRangeQuery extends Query {
 
     @Override
     public int hashCode() {
-        int result = super.hashCode();
+        int result = classHash();
         long temp;
         temp = lat != +0.0d ? Double.doubleToLongBits(lat) : 0L;
         result = 31 * result + Long.hashCode(temp);
diff --git a/core/src/main/java/org/elasticsearch/index/search/geo/GeoPolygonQuery.java b/core/src/main/java/org/elasticsearch/index/search/geo/GeoPolygonQuery.java
index d62aa76efd9..c3a52cb114e 100644
--- a/core/src/main/java/org/elasticsearch/index/search/geo/GeoPolygonQuery.java
+++ b/core/src/main/java/org/elasticsearch/index/search/geo/GeoPolygonQuery.java
@@ -111,7 +111,7 @@ public class GeoPolygonQuery extends Query {
 
     @Override
     public boolean equals(Object obj) {
-        if (super.equals(obj) == false) {
+        if (sameClassAs(obj) == false) {
             return false;
         }
         GeoPolygonQuery that = (GeoPolygonQuery) obj;
@@ -121,7 +121,7 @@ public class GeoPolygonQuery extends Query {
 
     @Override
     public int hashCode() {
-        int h = super.hashCode();
+        int h = classHash();
         h = 31 * h + indexFieldData.getFieldName().hashCode();
         h = 31 * h + Arrays.hashCode(points);
         return h;
diff --git a/core/src/main/java/org/elasticsearch/index/search/geo/InMemoryGeoBoundingBoxQuery.java b/core/src/main/java/org/elasticsearch/index/search/geo/InMemoryGeoBoundingBoxQuery.java
index 2f2801a2abe..789ee25e1b5 100644
--- a/core/src/main/java/org/elasticsearch/index/search/geo/InMemoryGeoBoundingBoxQuery.java
+++ b/core/src/main/java/org/elasticsearch/index/search/geo/InMemoryGeoBoundingBoxQuery.java
@@ -84,7 +84,7 @@ public class InMemoryGeoBoundingBoxQuery extends Query {
 
     @Override
     public boolean equals(Object obj) {
-        if (super.equals(obj) == false) {
+        if (sameClassAs(obj) == false) {
             return false;
         }
         InMemoryGeoBoundingBoxQuery other = (InMemoryGeoBoundingBoxQuery) obj;
@@ -95,7 +95,7 @@ public class InMemoryGeoBoundingBoxQuery extends Query {
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), fieldName(), topLeft, bottomRight);
+        return Objects.hash(classHash(), fieldName(), topLeft, bottomRight);
     }
 
     private static class Meridian180GeoBoundingBoxBits implements Bits {
diff --git a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
index 62173f936c5..dbfcad6048a 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
@@ -131,16 +131,7 @@ final class StoreRecovery {
     }
 
     final void addIndices(RecoveryState.Index indexRecoveryStats, Directory target, Directory... sources) throws IOException {
-        /*
-         * TODO: once we upgraded to Lucene 6.1 use HardlinkCopyDirectoryWrapper to enable hardlinks if possible and enable it
-         * in the security.policy:
-         *
-         * grant codeBase "${codebase.lucene-misc-6.1.0.jar}" {
-         *  // needed to allow shard shrinking to use hard-links if possible via lucenes HardlinkCopyDirectoryWrapper
-         *  permission java.nio.file.LinkPermission "hard";
-         * };
-         * target = new org.apache.lucene.store.HardlinkCopyDirectoryWrapper(target);
-         */
+        target = new org.apache.lucene.store.HardlinkCopyDirectoryWrapper(target);
         try (IndexWriter writer = new IndexWriter(new StatsDirectoryWrapper(target, indexRecoveryStats),
             new IndexWriterConfig(null)
                 .setCommitOnClose(false)
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java b/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java
index bd01e7f0183..70b9443e043 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesQueryCache.java
@@ -45,6 +45,7 @@ import java.util.IdentityHashMap;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Predicate;
 
 public class IndicesQueryCache extends AbstractComponent implements QueryCache, Closeable {
 
@@ -52,6 +53,9 @@ public class IndicesQueryCache extends AbstractComponent implements QueryCache,
             "indices.queries.cache.size", "10%", Property.NodeScope);
     public static final Setting<Integer> INDICES_CACHE_QUERY_COUNT_SETTING = Setting.intSetting(
             "indices.queries.cache.count", 10000, 1, Property.NodeScope);
+    // enables caching on all segments instead of only the larger ones, for testing only
+    public static final Setting<Boolean> INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING = Setting.boolSetting(
+            "indices.queries.cache.all_segments", false, Property.NodeScope);
 
     private final LRUQueryCache cache;
     private final ShardCoreKeyMap shardKeyMap = new ShardCoreKeyMap();
@@ -69,111 +73,11 @@ public class IndicesQueryCache extends AbstractComponent implements QueryCache,
         final int count = INDICES_CACHE_QUERY_COUNT_SETTING.get(settings);
         logger.debug("using [node] query cache with size [{}] max filter count [{}]",
                 size, count);
-        cache = new LRUQueryCache(count, size.bytes()) {
-
-            private Stats getStats(Object coreKey) {
-                final ShardId shardId = shardKeyMap.getShardId(coreKey);
-                if (shardId == null) {
-                    return null;
-                }
-                return shardStats.get(shardId);
-            }
-
-            private Stats getOrCreateStats(Object coreKey) {
-                final ShardId shardId = shardKeyMap.getShardId(coreKey);
-                Stats stats = shardStats.get(shardId);
-                if (stats == null) {
-                    stats = new Stats();
-                    shardStats.put(shardId, stats);
-                }
-                return stats;
-            }
-
-            // It's ok to not protect these callbacks by a lock since it is
-            // done in LRUQueryCache
-            @Override
-            protected void onClear() {
-                assert Thread.holdsLock(this);
-                super.onClear();
-                for (Stats stats : shardStats.values()) {
-                    // don't throw away hit/miss
-                    stats.cacheSize = 0;
-                    stats.ramBytesUsed = 0;
-                }
-                sharedRamBytesUsed = 0;
-            }
-
-            @Override
-            protected void onQueryCache(Query filter, long ramBytesUsed) {
-                assert Thread.holdsLock(this);
-                super.onQueryCache(filter, ramBytesUsed);
-                sharedRamBytesUsed += ramBytesUsed;
-            }
-
-            @Override
-            protected void onQueryEviction(Query filter, long ramBytesUsed) {
-                assert Thread.holdsLock(this);
-                super.onQueryEviction(filter, ramBytesUsed);
-                sharedRamBytesUsed -= ramBytesUsed;
-            }
-
-            @Override
-            protected void onDocIdSetCache(Object readerCoreKey, long ramBytesUsed) {
-                assert Thread.holdsLock(this);
-                super.onDocIdSetCache(readerCoreKey, ramBytesUsed);
-                final Stats shardStats = getOrCreateStats(readerCoreKey);
-                shardStats.cacheSize += 1;
-                shardStats.cacheCount += 1;
-                shardStats.ramBytesUsed += ramBytesUsed;
-
-                StatsAndCount statsAndCount = stats2.get(readerCoreKey);
-                if (statsAndCount == null) {
-                    statsAndCount = new StatsAndCount(shardStats);
-                    stats2.put(readerCoreKey, statsAndCount);
-                }
-                statsAndCount.count += 1;
-            }
-
-            @Override
-            protected void onDocIdSetEviction(Object readerCoreKey, int numEntries, long sumRamBytesUsed) {
-                assert Thread.holdsLock(this);
-                super.onDocIdSetEviction(readerCoreKey, numEntries, sumRamBytesUsed);
-                // onDocIdSetEviction might sometimes be called with a number
-                // of entries equal to zero if the cache for the given segment
-                // was already empty when the close listener was called
-                if (numEntries > 0) {
-                    // We can't use ShardCoreKeyMap here because its core closed
-                    // listener is called before the listener of the cache which
-                    // triggers this eviction. So instead we use use stats2 that
-                    // we only evict when nothing is cached anymore on the segment
-                    // instead of relying on close listeners
-                    final StatsAndCount statsAndCount = stats2.get(readerCoreKey);
-                    final Stats shardStats = statsAndCount.stats;
-                    shardStats.cacheSize -= numEntries;
-                    shardStats.ramBytesUsed -= sumRamBytesUsed;
-                    statsAndCount.count -= numEntries;
-                    if (statsAndCount.count == 0) {
-                        stats2.remove(readerCoreKey);
-                    }
-                }
-            }
-
-            @Override
-            protected void onHit(Object readerCoreKey, Query filter) {
-                assert Thread.holdsLock(this);
-                super.onHit(readerCoreKey, filter);
-                final Stats shardStats = getStats(readerCoreKey);
-                shardStats.hitCount += 1;
-            }
-
-            @Override
-            protected void onMiss(Object readerCoreKey, Query filter) {
-                assert Thread.holdsLock(this);
-                super.onMiss(readerCoreKey, filter);
-                final Stats shardStats = getOrCreateStats(readerCoreKey);
-                shardStats.missCount += 1;
-            }
-        };
+        if (INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING.get(settings)) {
+            cache = new ElasticsearchLRUQueryCache(count, size.bytes(), context -> true);
+        } else {
+            cache = new ElasticsearchLRUQueryCache(count, size.bytes());
+        }
         sharedRamBytesUsed = 0;
     }
 
@@ -316,4 +220,111 @@ public class IndicesQueryCache extends AbstractComponent implements QueryCache,
         assert empty(shardStats.get(shardId));
         shardStats.remove(shardId);
     }
+
+    private class ElasticsearchLRUQueryCache extends LRUQueryCache {
+
+        ElasticsearchLRUQueryCache(int maxSize, long maxRamBytesUsed, Predicate<LeafReaderContext> leavesToCache) {
+            super(maxSize, maxRamBytesUsed, leavesToCache);
+        }
+
+        ElasticsearchLRUQueryCache(int maxSize, long maxRamBytesUsed) {
+            super(maxSize, maxRamBytesUsed);
+        }
+
+        private Stats getStats(Object coreKey) {
+            final ShardId shardId = shardKeyMap.getShardId(coreKey);
+            if (shardId == null) {
+                return null;
+            }
+            return shardStats.get(shardId);
+        }
+
+        private Stats getOrCreateStats(Object coreKey) {
+            final ShardId shardId = shardKeyMap.getShardId(coreKey);
+            Stats stats = shardStats.get(shardId);
+            if (stats == null) {
+                stats = new Stats();
+                shardStats.put(shardId, stats);
+            }
+            return stats;
+        }
+
+        // It's ok to not protect these callbacks by a lock since it is
+        // done in LRUQueryCache
+        @Override
+        protected void onClear() {
+            super.onClear();
+            for (Stats stats : shardStats.values()) {
+                // don't throw away hit/miss
+                stats.cacheSize = 0;
+                stats.ramBytesUsed = 0;
+            }
+            sharedRamBytesUsed = 0;
+        }
+
+        @Override
+        protected void onQueryCache(Query filter, long ramBytesUsed) {
+            super.onQueryCache(filter, ramBytesUsed);
+            sharedRamBytesUsed += ramBytesUsed;
+        }
+
+        @Override
+        protected void onQueryEviction(Query filter, long ramBytesUsed) {
+            super.onQueryEviction(filter, ramBytesUsed);
+            sharedRamBytesUsed -= ramBytesUsed;
+        }
+
+        @Override
+        protected void onDocIdSetCache(Object readerCoreKey, long ramBytesUsed) {
+            super.onDocIdSetCache(readerCoreKey, ramBytesUsed);
+            final Stats shardStats = getOrCreateStats(readerCoreKey);
+            shardStats.cacheSize += 1;
+            shardStats.cacheCount += 1;
+            shardStats.ramBytesUsed += ramBytesUsed;
+
+            StatsAndCount statsAndCount = stats2.get(readerCoreKey);
+            if (statsAndCount == null) {
+                statsAndCount = new StatsAndCount(shardStats);
+                stats2.put(readerCoreKey, statsAndCount);
+            }
+            statsAndCount.count += 1;
+        }
+
+        @Override
+        protected void onDocIdSetEviction(Object readerCoreKey, int numEntries, long sumRamBytesUsed) {
+            super.onDocIdSetEviction(readerCoreKey, numEntries, sumRamBytesUsed);
+            // onDocIdSetEviction might sometimes be called with a number
+            // of entries equal to zero if the cache for the given segment
+            // was already empty when the close listener was called
+            if (numEntries > 0) {
+                // We can't use ShardCoreKeyMap here because its core closed
+                // listener is called before the listener of the cache which
+                // triggers this eviction. So instead we use use stats2 that
+                // we only evict when nothing is cached anymore on the segment
+                // instead of relying on close listeners
+                final StatsAndCount statsAndCount = stats2.get(readerCoreKey);
+                final Stats shardStats = statsAndCount.stats;
+                shardStats.cacheSize -= numEntries;
+                shardStats.ramBytesUsed -= sumRamBytesUsed;
+                statsAndCount.count -= numEntries;
+                if (statsAndCount.count == 0) {
+                    stats2.remove(readerCoreKey);
+                }
+            }
+        }
+
+        @Override
+        protected void onHit(Object readerCoreKey, Query filter) {
+            super.onHit(readerCoreKey, filter);
+            final Stats shardStats = getStats(readerCoreKey);
+            shardStats.hitCount += 1;
+        }
+
+        @Override
+        protected void onMiss(Object readerCoreKey, Query filter) {
+            super.onMiss(readerCoreKey, filter);
+            final Stats shardStats = getOrCreateStats(readerCoreKey);
+            shardStats.missCount += 1;
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java
index 192ad6c28dc..ec838e7dd41 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregator.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.search.aggregations.metrics.geocentroid;
 
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.spatial.util.GeoEncodingUtils;
+import org.apache.lucene.spatial.geopoint.document.GeoPointField;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.util.BigArrays;
@@ -82,9 +82,9 @@ public final class GeoCentroidAggregator extends MetricsAggregator {
                     counts.increment(bucket, valueCount);
                     // get the previous GeoPoint if a moving avg was computed
                     if (prevCounts > 0) {
-                        final GeoPoint centroid = GeoPoint.fromIndexLong(centroids.get(bucket));
-                        pt[0] = centroid.lon();
-                        pt[1] = centroid.lat();
+                        final long mortonCode = centroids.get(bucket);
+                        pt[0] = GeoPointField.decodeLongitude(mortonCode);
+                        pt[1] = GeoPointField.decodeLatitude(mortonCode);
                     }
                     // update the moving average
                     for (int i = 0; i < valueCount; ++i) {
@@ -92,7 +92,9 @@ public final class GeoCentroidAggregator extends MetricsAggregator {
                         pt[0] = pt[0] + (value.getLon() - pt[0]) / ++prevCounts;
                         pt[1] = pt[1] + (value.getLat() - pt[1]) / prevCounts;
                     }
-                    centroids.set(bucket, GeoEncodingUtils.mortonHash(pt[1], pt[0]));
+                    // TODO: we do not need to interleave the lat and lon bits here
+                    // should we just store them contiguously?
+                    centroids.set(bucket, GeoPointField.encodeLatLon(pt[1], pt[0]));
                 }
             }
         };
@@ -104,8 +106,10 @@ public final class GeoCentroidAggregator extends MetricsAggregator {
             return buildEmptyAggregation();
         }
         final long bucketCount = counts.get(bucket);
-        final GeoPoint bucketCentroid = (bucketCount > 0) ? GeoPoint.fromIndexLong(centroids.get(bucket)) :
-                new GeoPoint(Double.NaN, Double.NaN);
+        final long mortonCode = centroids.get(bucket);
+        final GeoPoint bucketCentroid = (bucketCount > 0)
+                ? new GeoPoint(GeoPointField.decodeLatitude(mortonCode), GeoPointField.decodeLongitude(mortonCode))
+                : null;
         return new InternalGeoCentroid(name, bucketCentroid , bucketCount, pipelineAggregators(), metaData());
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java
index 2798169b699..2bb3056ca66 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.search.aggregations.metrics.geocentroid;
 
-import org.apache.lucene.spatial.util.GeoEncodingUtils;
+import org.apache.lucene.spatial.geopoint.document.GeoPointField;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -61,6 +61,7 @@ public class InternalGeoCentroid extends InternalMetricsAggregation implements G
     public InternalGeoCentroid(String name, GeoPoint centroid, long count, List<PipelineAggregator>
             pipelineAggregators, Map<String, Object> metaData) {
         super(name, pipelineAggregators, metaData);
+        assert (centroid == null) == (count == 0);
         this.centroid = centroid;
         assert count >= 0;
         this.count = count;
@@ -68,7 +69,7 @@ public class InternalGeoCentroid extends InternalMetricsAggregation implements G
 
     @Override
     public GeoPoint centroid() {
-        return (centroid == null || Double.isNaN(centroid.lon()) ? null : centroid);
+        return centroid;
     }
 
     @Override
@@ -128,7 +129,8 @@ public class InternalGeoCentroid extends InternalMetricsAggregation implements G
     protected void doReadFrom(StreamInput in) throws IOException {
         count = in.readVLong();
         if (in.readBoolean()) {
-            centroid = GeoPoint.fromIndexLong(in.readLong());
+            final long hash = in.readLong();
+            centroid = new GeoPoint(GeoPointField.decodeLatitude(hash), GeoPointField.decodeLongitude(hash));
         } else {
             centroid = null;
         }
@@ -139,7 +141,8 @@ public class InternalGeoCentroid extends InternalMetricsAggregation implements G
         out.writeVLong(count);
         if (centroid != null) {
             out.writeBoolean(true);
-            out.writeLong(GeoEncodingUtils.mortonHash(centroid.lat(), centroid.lon()));
+            // should we just write lat and lon separately?
+            out.writeLong(GeoPointField.encodeLatLon(centroid.lat(), centroid.lon()));
         } else {
             out.writeBoolean(false);
         }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java
index 31921457207..f34da5301d5 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java
@@ -175,7 +175,7 @@ public final class InnerHitsContext {
 
             @Override
             public boolean equals(Object obj) {
-                if (super.equals(obj) == false) {
+                if (sameClassAs(obj) == false) {
                     return false;
                 }
                 NestedChildrenQuery other = (NestedChildrenQuery) obj;
@@ -187,7 +187,7 @@ public final class InnerHitsContext {
 
             @Override
             public int hashCode() {
-                int hash = super.hashCode();
+                int hash = classHash();
                 hash = 31 * hash + parentFilter.hashCode();
                 hash = 31 * hash + childFilter.hashCode();
                 hash = 31 * hash + docId;
diff --git a/core/src/main/java/org/elasticsearch/search/slice/SliceQuery.java b/core/src/main/java/org/elasticsearch/search/slice/SliceQuery.java
index 0d87b275403..2b8040ebd28 100644
--- a/core/src/main/java/org/elasticsearch/search/slice/SliceQuery.java
+++ b/core/src/main/java/org/elasticsearch/search/slice/SliceQuery.java
@@ -61,7 +61,7 @@ public abstract class SliceQuery extends Query {
 
     @Override
     public boolean equals(Object o) {
-        if (super.equals(o) == false) {
+        if (sameClassAs(o) == false) {
             return false;
         }
         SliceQuery that = (SliceQuery) o;
@@ -70,7 +70,7 @@ public abstract class SliceQuery extends Query {
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), field, id, max);
+        return Objects.hash(classHash(), field, id, max);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java b/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java
index b967a6b6e71..429a3ebe892 100644
--- a/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java
+++ b/core/src/main/java/org/elasticsearch/search/slice/TermsSliceQuery.java
@@ -74,11 +74,7 @@ public final class TermsSliceQuery extends SliceQuery {
             int hashCode = term.hashCode();
             if (contains(hashCode)) {
                 docsEnum = te.postings(docsEnum, PostingsEnum.NONE);
-                int docId = docsEnum.nextDoc();
-                while (docId != DocIdSetIterator.NO_MORE_DOCS) {
-                    builder.add(docId);
-                    docId = docsEnum.nextDoc();
-                }
+                builder.add(docsEnum);
             }
         }
         return builder.build();
diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
index ff0ea773667..b185289b58d 100644
--- a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
+++ b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
@@ -31,7 +31,7 @@ grant codeBase "${codebase.securesm-1.0.jar}" {
 //// Very special jar permissions:
 //// These are dangerous permissions that we don't want to grant to everything.
 
-grant codeBase "${codebase.lucene-core-6.0.1.jar}" {
+grant codeBase "${codebase.lucene-core-6.1.0-snapshot-3a57bea.jar}" {
   // needed to allow MMapDirectory's "unmap hack" (die unmap hack, die)
   // java 8 package
   permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
@@ -42,6 +42,11 @@ grant codeBase "${codebase.lucene-core-6.0.1.jar}" {
   permission java.lang.RuntimePermission "accessDeclaredMembers";
 };
 
+grant codeBase "${codebase.lucene-misc-6.1.0-snapshot-3a57bea.jar}" {
+  // needed to allow shard shrinking to use hard-links if possible via lucenes HardlinkCopyDirectoryWrapper
+  permission java.nio.file.LinkPermission "hard";
+};
+
 //// Everything else:
 
 grant {
diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
index d4ab6e01ab9..d7faab5eeda 100644
--- a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
+++ b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
@@ -31,9 +31,11 @@ grant codeBase "${codebase.securemock-1.2.jar}" {
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
 };
 
-grant codeBase "${codebase.lucene-test-framework-6.0.1.jar}" {
+grant codeBase "${codebase.lucene-test-framework-6.1.0-snapshot-3a57bea.jar}" {
   // needed by RamUsageTester
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+  // needed for testing hardlinks in StoreRecoveryTests since we install MockFS
+  permission java.nio.file.LinkPermission "hard";
 };
 
 grant codeBase "${codebase.randomizedtesting-runner-2.3.2.jar}" {
diff --git a/core/src/test/java/org/elasticsearch/VersionTests.java b/core/src/test/java/org/elasticsearch/VersionTests.java
index 65c91f5daab..862cccab318 100644
--- a/core/src/test/java/org/elasticsearch/VersionTests.java
+++ b/core/src/test/java/org/elasticsearch/VersionTests.java
@@ -270,7 +270,8 @@ public class VersionTests extends ESTestCase {
                     assertTrue("lucene versions must be "  + other + " >= " + version,
                         other.luceneVersion.onOrAfter(version.luceneVersion));
                 }
-                if (other.major == version.major && other.minor == version.minor) {
+                if (other.isAlpha() == false && version.isAlpha() == false
+                        && other.major == version.major && other.minor == version.minor) {
                     assertEquals(other.luceneVersion.major, version.luceneVersion.major);
                     assertEquals(other.luceneVersion.minor, version.luceneVersion.minor);
                     // should we also assert the lucene bugfix version?
diff --git a/core/src/test/java/org/elasticsearch/bootstrap/JavaVersionTests.java b/core/src/test/java/org/elasticsearch/bootstrap/JavaVersionTests.java
index d2ef349625e..a6e74a47706 100644
--- a/core/src/test/java/org/elasticsearch/bootstrap/JavaVersionTests.java
+++ b/core/src/test/java/org/elasticsearch/bootstrap/JavaVersionTests.java
@@ -72,4 +72,8 @@ public class JavaVersionTests extends ESTestCase {
             assertFalse(JavaVersion.isValid(version));
         }
     }
+
+    public void testJava8Compat() {
+        assertEquals(JavaVersion.parse("1.8"), JavaVersion.parse("8"));
+    }
 }
\ No newline at end of file
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java
index 062774bf2f7..4e4d638d355 100644
--- a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java
@@ -22,9 +22,9 @@ import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.spatial.geopoint.document.GeoPointField;
-import org.apache.lucene.spatial.util.GeoUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.geo.GeoUtils;
 
 import static org.elasticsearch.test.geo.RandomShapeGenerator.randomPoint;
 import static org.hamcrest.Matchers.allOf;
@@ -105,8 +105,8 @@ public abstract class AbstractGeoFieldDataTestCase extends AbstractFieldDataImpl
             assertThat(docCount, greaterThan(0));
             for (int i = 0; i < docCount; ++i) {
                 final GeoPoint point = values.valueAt(i);
-                assertThat(point.lat(), allOf(greaterThanOrEqualTo(GeoUtils.MIN_LAT_INCL), lessThanOrEqualTo(GeoUtils.MAX_LAT_INCL)));
-                assertThat(point.lon(), allOf(greaterThanOrEqualTo(GeoUtils.MIN_LON_INCL), lessThanOrEqualTo(GeoUtils.MAX_LON_INCL)));
+                assertThat(point.lat(), allOf(greaterThanOrEqualTo(GeoUtils.MIN_LAT), lessThanOrEqualTo(GeoUtils.MAX_LAT)));
+                assertThat(point.lon(), allOf(greaterThanOrEqualTo(GeoUtils.MIN_LON), lessThanOrEqualTo(GeoUtils.MAX_LON)));
             }
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java
index 177d3b7b0f7..b2e1989454c 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.index.mapper.externalvalues;
 
-import org.apache.lucene.spatial.util.GeoEncodingUtils;
+import org.apache.lucene.spatial.geopoint.document.GeoPointField;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.compress.CompressedXContent;
@@ -88,7 +88,7 @@ public class SimpleExternalMappingTests extends ESSingleNodeTestCase {
         if (version.before(Version.V_2_2_0)) {
             assertThat(doc.rootDoc().getField("field.point").stringValue(), is("42.0,51.0"));
         } else {
-            assertThat(Long.parseLong(doc.rootDoc().getField("field.point").stringValue()), is(GeoEncodingUtils.mortonHash(42.0, 51.0)));
+            assertThat(Long.parseLong(doc.rootDoc().getField("field.point").stringValue()), is(GeoPointField.encodeLatLon(42.0, 51.0)));
         }
 
         assertThat(doc.rootDoc().getField("field.shape"), notNullValue());
@@ -146,7 +146,7 @@ public class SimpleExternalMappingTests extends ESSingleNodeTestCase {
         if (version.before(Version.V_2_2_0)) {
             assertThat(doc.rootDoc().getField("field.point").stringValue(), is("42.0,51.0"));
         } else {
-            assertThat(Long.parseLong(doc.rootDoc().getField("field.point").stringValue()), is(GeoEncodingUtils.mortonHash(42.0, 51.0)));
+            assertThat(Long.parseLong(doc.rootDoc().getField("field.point").stringValue()), is(GeoPointField.encodeLatLon(42.0, 51.0)));
         }
 
         assertThat(doc.rootDoc().getField("field.shape"), notNullValue());
@@ -208,7 +208,7 @@ public class SimpleExternalMappingTests extends ESSingleNodeTestCase {
         if (version.before(Version.V_2_2_0)) {
             assertThat(doc.rootDoc().getField("field.point").stringValue(), is("42.0,51.0"));
         } else {
-            assertThat(Long.parseLong(doc.rootDoc().getField("field.point").stringValue()), is(GeoEncodingUtils.mortonHash(42.0, 51.0)));
+            assertThat(Long.parseLong(doc.rootDoc().getField("field.point").stringValue()), is(GeoPointField.encodeLatLon(42.0, 51.0)));
         }
 
         assertThat(doc.rootDoc().getField("field.shape"), notNullValue());
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java
index a1fdb7ec60f..202afd7a4b1 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.index.mapper.geo;
 
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.spatial.geopoint.document.GeoPointField;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
@@ -46,7 +47,6 @@ import java.util.List;
 import java.util.Map;
 import java.lang.NumberFormatException;
 
-import static org.apache.lucene.spatial.util.GeoEncodingUtils.mortonHash;
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.geo.GeoHashUtils.stringEncode;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -90,7 +90,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
         if (indexCreatedBefore22 == true) {
             assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
         } else {
-            assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(mortonHash(1.2, 1.3)));
+            assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
         }
     }
 
@@ -197,7 +197,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
         if (version.before(Version.V_2_2_0)) {
             assertThat(doc.rootDoc().get("point"), equalTo("89.0,1.0"));
         } else {
-            assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(mortonHash(89.0, 1.0)));
+            assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoPointField.encodeLatLon(89.0, 1.0)));
         }
 
         doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
@@ -209,7 +209,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
         if (version.before(Version.V_2_2_0)) {
             assertThat(doc.rootDoc().get("point"), equalTo("-89.0,-1.0"));
         } else {
-            assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(mortonHash(-89.0, -1.0)));
+            assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoPointField.encodeLatLon(-89.0, -1.0)));
         }
 
         doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
@@ -221,7 +221,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
         if (version.before(Version.V_2_2_0)) {
             assertThat(doc.rootDoc().get("point"), equalTo("-1.0,-179.0"));
         } else {
-            assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(mortonHash(-1.0, -179.0)));
+            assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoPointField.encodeLatLon(-1.0, -179.0)));
         }
     }
 
@@ -408,7 +408,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
         if (version.before(Version.V_2_2_0)) {
             assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
         } else {
-            assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(mortonHash(1.2, 1.3)));
+            assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
         }
     }
 
@@ -441,7 +441,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
             assertThat(doc.rootDoc().getFields("point.lat")[1].numericValue().doubleValue(), equalTo(1.2));
             assertThat(doc.rootDoc().getFields("point.lon")[1].numericValue().doubleValue(), equalTo(1.3));
             // indexed hash
-            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(mortonHash(1.2, 1.3)));
+            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
 
             // point field for 2nd value
             assertThat(doc.rootDoc().getFields("point.lat")[2].numericValue().doubleValue(), equalTo(1.4));
@@ -450,7 +450,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
             assertThat(doc.rootDoc().getFields("point.lat")[3].numericValue().doubleValue(), equalTo(1.4));
             assertThat(doc.rootDoc().getFields("point.lon")[3].numericValue().doubleValue(), equalTo(1.5));
             // indexed hash
-            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(mortonHash(1.4, 1.5)));
+            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(GeoPointField.encodeLatLon(1.4, 1.5)));
         } else {
             assertThat(doc.rootDoc().getFields("point.lat").length, equalTo(2));
             assertThat(doc.rootDoc().getFields("point.lon").length, equalTo(2));
@@ -459,14 +459,14 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
             if (version.before(Version.V_2_2_0)) {
                 assertThat(doc.rootDoc().getFields("point")[0].stringValue(), equalTo("1.2,1.3"));
             } else {
-                assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(mortonHash(1.2, 1.3)));
+                assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
             }
             assertThat(doc.rootDoc().getFields("point.lat")[1].numericValue().doubleValue(), equalTo(1.4));
             assertThat(doc.rootDoc().getFields("point.lon")[1].numericValue().doubleValue(), equalTo(1.5));
             if (version.before(Version.V_2_2_0)) {
                 assertThat(doc.rootDoc().getFields("point")[1].stringValue(), equalTo("1.4,1.5"));
             } else {
-                assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(mortonHash(1.4, 1.5)));
+                assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(GeoPointField.encodeLatLon(1.4, 1.5)));
             }
         }
     }
@@ -491,7 +491,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
         if (version.before(Version.V_2_2_0)) {
             assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
         } else {
-            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(mortonHash(1.2, 1.3)));
+            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
         }
     }
 
@@ -517,7 +517,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
         if (version.before(Version.V_2_2_0)) {
             assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
         } else {
-            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(mortonHash(1.2, 1.3)));
+            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
         }
     }
 
@@ -559,12 +559,12 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
         if (version.before(Version.V_2_2_0)) {
             assertThat(doc.rootDoc().getFields("point")[0].stringValue(), equalTo("1.2,1.3"));
         } else {
-            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(mortonHash(1.2, 1.3)));
+            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
         }
         if (version.before(Version.V_2_2_0)) {
             assertThat(doc.rootDoc().getFields("point")[1].stringValue(), equalTo("1.4,1.5"));
         } else {
-            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(mortonHash(1.4, 1.5)));
+            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(GeoPointField.encodeLatLon(1.4, 1.5)));
         }
     }
 
@@ -588,7 +588,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
         if (version.before(Version.V_2_2_0)) {
             assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
         } else {
-            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(mortonHash(1.2, 1.3)));
+            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
         }
     }
 
@@ -613,7 +613,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
         if (version.before(Version.V_2_2_0)) {
             assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
         } else {
-            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(mortonHash(1.2, 1.3)));
+            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
         }
     }
 
@@ -639,7 +639,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
         if (version.before(Version.V_2_2_0)) {
             assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
         } else {
-            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(mortonHash(1.2, 1.3)));
+            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
         }
     }
 
@@ -669,14 +669,14 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
             if (version.before(Version.V_2_2_0)) {
                 assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
             } else {
-                assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(mortonHash(1.2, 1.3)));
+                assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
             }
             assertThat(doc.rootDoc().getFields("point.lat")[1].numericValue().doubleValue(), equalTo(1.4));
             assertThat(doc.rootDoc().getFields("point.lon")[1].numericValue().doubleValue(), equalTo(1.5));
             if (version.before(Version.V_2_2_0)) {
                 assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
             } else {
-                assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(mortonHash(1.4, 1.5)));
+                assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(GeoPointField.encodeLatLon(1.4, 1.5)));
             }
         } else {
             assertThat(doc.rootDoc().getFields("point.lat").length, equalTo(4));
@@ -685,12 +685,12 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
             assertThat(doc.rootDoc().getFields("point.lat")[1].numericValue().doubleValue(), equalTo(1.2));
             assertThat(doc.rootDoc().getFields("point.lon")[0].numericValue().doubleValue(), equalTo(1.3));
             assertThat(doc.rootDoc().getFields("point.lon")[1].numericValue().doubleValue(), equalTo(1.3));
-            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(mortonHash(1.2, 1.3)));
+            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[0].stringValue()), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
             assertThat(doc.rootDoc().getFields("point.lat")[2].numericValue().doubleValue(), equalTo(1.4));
             assertThat(doc.rootDoc().getFields("point.lat")[3].numericValue().doubleValue(), equalTo(1.4));
             assertThat(doc.rootDoc().getFields("point.lon")[2].numericValue().doubleValue(), equalTo(1.5));
             assertThat(doc.rootDoc().getFields("point.lon")[3].numericValue().doubleValue(), equalTo(1.5));
-            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(mortonHash(1.4, 1.5)));
+            assertThat(Long.parseLong(doc.rootDoc().getFields("point")[1].stringValue()), equalTo(GeoPointField.encodeLatLon(1.4, 1.5)));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeohashMappingGeoPointTests.java b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeohashMappingGeoPointTests.java
index 837cef6a17c..90528c9a8f4 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeohashMappingGeoPointTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeohashMappingGeoPointTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.mapper.geo;
 
+import org.apache.lucene.spatial.geopoint.document.GeoPointField;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.compress.CompressedXContent;
@@ -35,7 +36,6 @@ import org.elasticsearch.test.VersionUtils;
 import java.util.Collection;
 
 import static org.elasticsearch.common.geo.GeoHashUtils.stringEncode;
-import static org.apache.lucene.spatial.util.GeoEncodingUtils.mortonHash;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
@@ -72,7 +72,7 @@ public class GeohashMappingGeoPointTests extends ESSingleNodeTestCase {
         if (version.before(Version.V_2_2_0)) {
             assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
         } else {
-            assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(mortonHash(1.2, 1.3)));
+            assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
         }
     }
 
@@ -96,7 +96,7 @@ public class GeohashMappingGeoPointTests extends ESSingleNodeTestCase {
         if (version.before(Version.V_2_2_0)) {
             assertThat(doc.rootDoc().get("point"), equalTo("1.2,1.3"));
         } else {
-            assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(mortonHash(1.2, 1.3)));
+            assertThat(Long.parseLong(doc.rootDoc().get("point")), equalTo(GeoPointField.encodeLatLon(1.2, 1.3)));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ip/IpFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ip/IpFieldTypeTests.java
index 522a35ccd5d..884f52cc0ed 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/ip/IpFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ip/IpFieldTypeTests.java
@@ -21,7 +21,6 @@ package org.elasticsearch.index.mapper.ip;
 import java.net.InetAddress;
 
 import org.apache.lucene.document.InetAddressPoint;
-import org.apache.lucene.document.XInetAddressPoint;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.util.BytesRef;
@@ -69,11 +68,11 @@ public class IpFieldTypeTests extends FieldTypeTestCase {
 
         ip = "2001:db8::2:1";
         String prefix = ip + "/64";
-        assertEquals(XInetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 64), ft.termQuery(prefix, null));
+        assertEquals(InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 64), ft.termQuery(prefix, null));
 
         ip = "192.168.1.7";
         prefix = ip + "/16";
-        assertEquals(XInetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 16), ft.termQuery(prefix, null));
+        assertEquals(InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 16), ft.termQuery(prefix, null));
 
         ft.setIndexOptions(IndexOptions.NONE);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
@@ -88,7 +87,7 @@ public class IpFieldTypeTests extends FieldTypeTestCase {
         assertEquals(
                 InetAddressPoint.newRangeQuery("field",
                         InetAddresses.forString("::"),
-                        XInetAddressPoint.MAX_VALUE),
+                        InetAddressPoint.MAX_VALUE),
                 ft.rangeQuery(null, null, randomBoolean(), randomBoolean()));
 
         assertEquals(
@@ -106,13 +105,13 @@ public class IpFieldTypeTests extends FieldTypeTestCase {
         assertEquals(
                 InetAddressPoint.newRangeQuery("field",
                         InetAddresses.forString("2001:db8::"),
-                        XInetAddressPoint.MAX_VALUE),
+                        InetAddressPoint.MAX_VALUE),
                 ft.rangeQuery("2001:db8::", null, true, randomBoolean()));
 
         assertEquals(
                 InetAddressPoint.newRangeQuery("field",
                         InetAddresses.forString("2001:db8::1"),
-                        XInetAddressPoint.MAX_VALUE),
+                        InetAddressPoint.MAX_VALUE),
                 ft.rangeQuery("2001:db8::", null, false, randomBoolean()));
 
         assertEquals(
@@ -152,7 +151,7 @@ public class IpFieldTypeTests extends FieldTypeTestCase {
         assertEquals(
                 InetAddressPoint.newRangeQuery("field",
                         InetAddresses.forString("::1:0:0:0"),
-                        XInetAddressPoint.MAX_VALUE),
+                        InetAddressPoint.MAX_VALUE),
                 // same lo/hi values but inclusive=false so this won't match anything
                 ft.rangeQuery("255.255.255.255", "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", false, true));
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java
index 7780d218b52..387df7ac3ca 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceQueryBuilderTests.java
@@ -22,10 +22,10 @@ package org.elasticsearch.index.query;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.spatial.geopoint.search.GeoPointDistanceQuery;
-import org.apache.lucene.spatial.util.GeoEncodingUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.geo.GeoDistance;
 import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.index.search.geo.GeoDistanceRangeQuery;
 import org.elasticsearch.test.AbstractQueryTestCase;
@@ -213,7 +213,7 @@ public class GeoDistanceQueryBuilderTests extends AbstractQueryTestCase<GeoDista
         if (queryBuilder.geoDistance() != null) {
             distance = queryBuilder.geoDistance().normalize(distance, DistanceUnit.DEFAULT);
             distance = org.elasticsearch.common.geo.GeoUtils.maxRadialDistance(queryBuilder.point(), distance);
-            assertThat(geoQuery.getRadiusMeters(), closeTo(distance, GeoEncodingUtils.TOLERANCE));
+            assertThat(geoQuery.getRadiusMeters(), closeTo(distance, GeoUtils.TOLERANCE));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java
index a14bf7e0f31..3e10eda34e2 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoDistanceRangeQueryTests.java
@@ -22,7 +22,6 @@ package org.elasticsearch.index.query;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.spatial.geopoint.search.XGeoPointDistanceRangeQuery;
-import org.apache.lucene.spatial.util.GeoDistanceUtils;
 import org.apache.lucene.util.NumericUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.compress.CompressedXContent;
@@ -61,7 +60,7 @@ public class GeoDistanceRangeQueryTests extends AbstractQueryTestCase<GeoDistanc
             }
         }
         GeoPoint point = builder.point();
-        final double maxRadius = GeoDistanceUtils.maxRadialDistanceMeters(point.lat(), point.lon());
+        final double maxRadius = GeoUtils.maxRadialDistanceMeters(point.lat(), point.lon());
         final int fromValueMeters = randomInt((int)(maxRadius*0.5));
         final int toValueMeters = randomIntBetween(fromValueMeters + 1, (int)maxRadius);
         DistanceUnit fromToUnits = randomFrom(DistanceUnit.values());
diff --git a/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
index b41d8efafdf..e6fd5abd05e 100644
--- a/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/GeoPolygonQueryBuilderTests.java
@@ -100,8 +100,9 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase<GeoPolygo
         GeoPointInPolygonQuery geoQuery = (GeoPointInPolygonQuery) query;
         assertThat(geoQuery.getField(), equalTo(queryBuilder.fieldName()));
         List<GeoPoint> queryBuilderPoints = queryBuilder.points();
-        double[] lats = geoQuery.getLats();
-        double[] lons = geoQuery.getLons();
+        assertEquals(1, geoQuery.getPolygons().length);
+        double[] lats = geoQuery.getPolygons()[0].getPolyLats();
+        double[] lons = geoQuery.getPolygons()[0].getPolyLons();
         assertThat(lats.length, equalTo(queryBuilderPoints.size()));
         assertThat(lons.length, equalTo(queryBuilderPoints.size()));
         for (int i=0; i < queryBuilderPoints.size(); ++i) {
@@ -321,8 +322,9 @@ public class GeoPolygonQueryBuilderTests extends AbstractQueryTestCase<GeoPolygo
         } else {
             GeoPointInPolygonQuery q = (GeoPointInPolygonQuery) parsedQuery;
             assertThat(q.getField(), equalTo(GEO_POINT_FIELD_NAME));
-            final double[] lats = q.getLats();
-            final double[] lons = q.getLons();
+            assertEquals(1, q.getPolygons().length);
+            final double[] lats = q.getPolygons()[0].getPolyLats();
+            final double[] lons = q.getPolygons()[0].getPolyLons();
             assertThat(lats.length, equalTo(4));
             assertThat(lons.length, equalTo(4));
             assertThat(lats[0], closeTo(40, 1E-5));
diff --git a/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java b/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java
index fbb744f799c..d8021ad36bb 100644
--- a/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java
+++ b/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java
@@ -61,5 +61,15 @@ public class DummyQueryParserPlugin extends Plugin {
         public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
             return matchAllDocsQuery.createWeight(searcher, needsScores);
         }
+
+        @Override
+        public boolean equals(Object obj) {
+            return sameClassAs(obj);
+        }
+
+        @Override
+        public int hashCode() {
+            return classHash();
+        }
     }
 }
\ No newline at end of file
diff --git a/core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java b/core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java
index ffb64f991cc..f31733dc477 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java
@@ -31,7 +31,6 @@ import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.Version;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.test.ESTestCase;
 
@@ -74,11 +73,9 @@ public class StoreRecoveryTests extends ESTestCase {
         assertEquals(numFiles, targetNumFiles);
         assertEquals(indexStats.totalFileCount(), targetNumFiles);
         if (hardLinksSupported(createTempDir())) {
-            assertEquals("upgrade to HardlinkCopyDirectoryWrapper in Lucene 6.1", Version.LATEST, Version.LUCENE_6_0_1);
-            // assertEquals(indexStats.reusedFileCount(), targetNumFiles); -- uncomment this once upgraded to Lucene 6.1
-            assertEquals(indexStats.reusedFileCount(), 0);
+            assertEquals(targetNumFiles, indexStats.reusedFileCount());
         } else {
-            assertEquals(indexStats.reusedFileCount(), 0);
+            assertEquals(0, indexStats.reusedFileCount(), 0);
         }
         DirectoryReader reader = DirectoryReader.open(target);
         SegmentInfos segmentCommitInfos = SegmentInfos.readLatestCommit(target);
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java
index 5a4aa2e6b24..cd94ee0f8e9 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesQueryCacheTests.java
@@ -35,6 +35,7 @@ import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.cache.query.QueryCacheStats;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesQueryCache;
@@ -54,12 +55,12 @@ public class IndicesQueryCacheTests extends ESTestCase {
 
         @Override
         public boolean equals(Object obj) {
-            return super.equals(obj) && id == ((DummyQuery) obj).id;
+            return sameClassAs(obj) && id == ((DummyQuery) obj).id;
         }
 
         @Override
         public int hashCode() {
-            return 31 * super.hashCode() + id;
+            return 31 * classHash() + id;
         }
 
         @Override
@@ -93,6 +94,7 @@ public class IndicesQueryCacheTests extends ESTestCase {
 
         Settings settings = Settings.builder()
                 .put(IndicesQueryCache.INDICES_CACHE_QUERY_COUNT_SETTING.getKey(), 10)
+                .put(IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING.getKey(), true)
                 .build();
         IndicesQueryCache cache = new IndicesQueryCache(settings);
         s.setQueryCache(cache);
@@ -173,6 +175,7 @@ public class IndicesQueryCacheTests extends ESTestCase {
 
         Settings settings = Settings.builder()
                 .put(IndicesQueryCache.INDICES_CACHE_QUERY_COUNT_SETTING.getKey(), 10)
+                .put(IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING.getKey(), true)
                 .build();
         IndicesQueryCache cache = new IndicesQueryCache(settings);
         s1.setQueryCache(cache);
@@ -298,6 +301,7 @@ public class IndicesQueryCacheTests extends ESTestCase {
 
         Settings settings = Settings.builder()
                 .put(IndicesQueryCache.INDICES_CACHE_QUERY_COUNT_SETTING.getKey(), 10)
+                .put(IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING.getKey(), true)
                 .build();
         IndicesQueryCache cache = new IndicesQueryCache(settings);
         s1.setQueryCache(cache);
diff --git a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
index 3565cf0147d..a4096fde9da 100644
--- a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
@@ -47,6 +47,7 @@ import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.store.IndexStore;
 import org.elasticsearch.index.translog.Translog;
+import org.elasticsearch.indices.IndicesQueryCache;
 import org.elasticsearch.indices.IndicesRequestCache;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.search.sort.SortOrder;
@@ -78,6 +79,7 @@ public class IndexStatsIT extends ESIntegTestCase {
         //Filter/Query cache is cleaned periodically, default is 60s, so make sure it runs often. Thread.sleep for 60s is bad
         return Settings.builder().put(super.nodeSettings(nodeOrdinal))
                 .put(IndicesService.INDICES_CACHE_CLEAN_INTERVAL_SETTING.getKey(), "1ms")
+                .put(IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING.getKey(), true)
                 .build();
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java b/core/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java
index 7e01f575822..398ef64bc92 100644
--- a/core/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java
+++ b/core/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java
@@ -126,6 +126,8 @@ public class GeoBoundingBoxIT extends ESIntegTestCase {
         }
     }
 
+    // norelease
+    @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-7325")
     public void testLimitsBoundingBox() throws Exception {
         Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java
index b8b04a8bc33..0debdb263af 100644
--- a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java
+++ b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java
@@ -30,7 +30,6 @@ import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree;
 import org.apache.lucene.spatial.query.SpatialArgs;
 import org.apache.lucene.spatial.query.SpatialOperation;
 import org.apache.lucene.spatial.query.UnsupportedSpatialOperation;
-import org.apache.lucene.spatial.util.GeoProjectionUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.bulk.BulkItemResponse;
@@ -42,6 +41,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.geo.GeoHashUtils;
 import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.geo.builders.CoordinatesBuilder;
 import org.elasticsearch.common.geo.builders.LineStringBuilder;
 import org.elasticsearch.common.geo.builders.MultiPolygonBuilder;
@@ -540,7 +540,7 @@ public class GeoFilterIT extends ESIntegTestCase {
     }
 
     public static double distance(double lat1, double lon1, double lat2, double lon2) {
-        return GeoProjectionUtils.SEMIMAJOR_AXIS * DistanceUtils.distHaversineRAD(
+        return GeoUtils.EARTH_SEMI_MAJOR_AXIS * DistanceUtils.distHaversineRAD(
                 DistanceUtils.toRadians(lat1),
                 DistanceUtils.toRadians(lon1),
                 DistanceUtils.toRadians(lat2),
diff --git a/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java b/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
index ba378a3c404..e0aec941487 100644
--- a/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
+++ b/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
@@ -318,13 +318,7 @@ public class SimpleNestedIT extends ESIntegTestCase {
         assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
         Explanation explanation = searchResponse.getHits().hits()[0].explanation();
         assertThat(explanation.getValue(), equalTo(2f));
-        assertThat(explanation.toString(), startsWith("2.0 = sum of:\n  2.0 = Score based on child doc range from 0 to 1\n"));
-        // TODO: Enable when changes from BlockJoinQuery#explain are added to Lucene (Most likely version 4.2)
-//        assertThat(explanation.getDetails().length, equalTo(2));
-//        assertThat(explanation.getDetails()[0].getValue(), equalTo(1f));
-//        assertThat(explanation.getDetails()[0].getDescription(), equalTo("Child[0]"));
-//        assertThat(explanation.getDetails()[1].getValue(), equalTo(1f));
-//        assertThat(explanation.getDetails()[1].getDescription(), equalTo("Child[1]"));
+        assertThat(explanation.toString(), startsWith("2.0 = sum of:\n  2.0 = Score based on 2 child docs in range from 0 to 1"));
     }
 
     public void testSimpleNestedSorting() throws Exception {
diff --git a/distribution/licenses/lucene-analyzers-common-6.0.1.jar.sha1 b/distribution/licenses/lucene-analyzers-common-6.0.1.jar.sha1
deleted file mode 100644
index b581809a004..00000000000
--- a/distribution/licenses/lucene-analyzers-common-6.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-53953c1a9b097f83209c84a422cf8f9d271f47c1
\ No newline at end of file
diff --git a/distribution/licenses/lucene-analyzers-common-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-analyzers-common-6.1.0-snapshot-3a57bea.jar.sha1
new file mode 100644
index 00000000000..bb4cc98e068
--- /dev/null
+++ b/distribution/licenses/lucene-analyzers-common-6.1.0-snapshot-3a57bea.jar.sha1
@@ -0,0 +1 @@
+bf73c03e6b83f8e696133f40b9b1fc3381750149
\ No newline at end of file
diff --git a/distribution/licenses/lucene-backward-codecs-6.0.1.jar.sha1 b/distribution/licenses/lucene-backward-codecs-6.0.1.jar.sha1
deleted file mode 100644
index 5433f09c993..00000000000
--- a/distribution/licenses/lucene-backward-codecs-6.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3647088603be84b8f4916ef86954e3336b98d254
\ No newline at end of file
diff --git a/distribution/licenses/lucene-backward-codecs-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-backward-codecs-6.1.0-snapshot-3a57bea.jar.sha1
new file mode 100644
index 00000000000..a8eb4883e15
--- /dev/null
+++ b/distribution/licenses/lucene-backward-codecs-6.1.0-snapshot-3a57bea.jar.sha1
@@ -0,0 +1 @@
+8bc384f55faf99b6d6cee6f34df4fbd3145afb4d
\ No newline at end of file
diff --git a/distribution/licenses/lucene-core-6.0.1.jar.sha1 b/distribution/licenses/lucene-core-6.0.1.jar.sha1
deleted file mode 100644
index 6bcd7fc87f4..00000000000
--- a/distribution/licenses/lucene-core-6.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-40ccd40bec54266a10aa1f81c565914ede8c0ca0
\ No newline at end of file
diff --git a/distribution/licenses/lucene-core-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-core-6.1.0-snapshot-3a57bea.jar.sha1
new file mode 100644
index 00000000000..012fea19fdc
--- /dev/null
+++ b/distribution/licenses/lucene-core-6.1.0-snapshot-3a57bea.jar.sha1
@@ -0,0 +1 @@
+fe19e7558440e10db4bd7150931dff6a7cf73243
\ No newline at end of file
diff --git a/distribution/licenses/lucene-grouping-6.0.1.jar.sha1 b/distribution/licenses/lucene-grouping-6.0.1.jar.sha1
deleted file mode 100644
index b132acc9112..00000000000
--- a/distribution/licenses/lucene-grouping-6.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-010daaae60227fbe719ca95e9b6fcdb5c38d4eba
\ No newline at end of file
diff --git a/distribution/licenses/lucene-grouping-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-grouping-6.1.0-snapshot-3a57bea.jar.sha1
new file mode 100644
index 00000000000..3b3338ffd2f
--- /dev/null
+++ b/distribution/licenses/lucene-grouping-6.1.0-snapshot-3a57bea.jar.sha1
@@ -0,0 +1 @@
+c3f0de4cdd185d23bce66c580d9c12adb98182a5
\ No newline at end of file
diff --git a/distribution/licenses/lucene-highlighter-6.0.1.jar.sha1 b/distribution/licenses/lucene-highlighter-6.0.1.jar.sha1
deleted file mode 100644
index 95d4b3edab9..00000000000
--- a/distribution/licenses/lucene-highlighter-6.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-65d74c3642e6a86ba905045473b17cc84826527e
\ No newline at end of file
diff --git a/distribution/licenses/lucene-highlighter-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-highlighter-6.1.0-snapshot-3a57bea.jar.sha1
new file mode 100644
index 00000000000..255812ed5fb
--- /dev/null
+++ b/distribution/licenses/lucene-highlighter-6.1.0-snapshot-3a57bea.jar.sha1
@@ -0,0 +1 @@
+ffb7087267bb6076b00c90f97ee36ebe23ea0662
\ No newline at end of file
diff --git a/distribution/licenses/lucene-join-6.0.1.jar.sha1 b/distribution/licenses/lucene-join-6.0.1.jar.sha1
deleted file mode 100644
index 07392cf260f..00000000000
--- a/distribution/licenses/lucene-join-6.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2105e2826ce93d1f764e5a0a3afa9ee461d556c1
\ No newline at end of file
diff --git a/distribution/licenses/lucene-join-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-join-6.1.0-snapshot-3a57bea.jar.sha1
new file mode 100644
index 00000000000..4231857b4e7
--- /dev/null
+++ b/distribution/licenses/lucene-join-6.1.0-snapshot-3a57bea.jar.sha1
@@ -0,0 +1 @@
+d071ad17bed58b3267f6fa0b2a8211f8fe18c912
\ No newline at end of file
diff --git a/distribution/licenses/lucene-memory-6.0.1.jar.sha1 b/distribution/licenses/lucene-memory-6.0.1.jar.sha1
deleted file mode 100644
index b9820103d3f..00000000000
--- a/distribution/licenses/lucene-memory-6.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e2cde0688e487a27d08df0c2d81d492b1f4cdc2a
\ No newline at end of file
diff --git a/distribution/licenses/lucene-memory-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-memory-6.1.0-snapshot-3a57bea.jar.sha1
new file mode 100644
index 00000000000..19aa64ccd80
--- /dev/null
+++ b/distribution/licenses/lucene-memory-6.1.0-snapshot-3a57bea.jar.sha1
@@ -0,0 +1 @@
+f5e9b6eefe580a7f65276aca3192ca5796332509
\ No newline at end of file
diff --git a/distribution/licenses/lucene-misc-6.0.1.jar.sha1 b/distribution/licenses/lucene-misc-6.0.1.jar.sha1
deleted file mode 100644
index 2670ab628df..00000000000
--- a/distribution/licenses/lucene-misc-6.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e6e59996fc324319d695e41cf25e30e5f1e4c182
\ No newline at end of file
diff --git a/distribution/licenses/lucene-misc-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-misc-6.1.0-snapshot-3a57bea.jar.sha1
new file mode 100644
index 00000000000..8480fcc3a49
--- /dev/null
+++ b/distribution/licenses/lucene-misc-6.1.0-snapshot-3a57bea.jar.sha1
@@ -0,0 +1 @@
+6b84a79c37b01197130cceb65e5573794f073df1
\ No newline at end of file
diff --git a/distribution/licenses/lucene-queries-6.0.1.jar.sha1 b/distribution/licenses/lucene-queries-6.0.1.jar.sha1
deleted file mode 100644
index acaa53f1f8e..00000000000
--- a/distribution/licenses/lucene-queries-6.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-09b0e5862a676ff9e55a1bc6ca37ad578a25cb38
\ No newline at end of file
diff --git a/distribution/licenses/lucene-queries-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-queries-6.1.0-snapshot-3a57bea.jar.sha1
new file mode 100644
index 00000000000..a3ed70c12af
--- /dev/null
+++ b/distribution/licenses/lucene-queries-6.1.0-snapshot-3a57bea.jar.sha1
@@ -0,0 +1 @@
+a9d51b77395dfdd7e6c4cf8c8506ebca5e1bb374
\ No newline at end of file
diff --git a/distribution/licenses/lucene-queryparser-6.0.1.jar.sha1 b/distribution/licenses/lucene-queryparser-6.0.1.jar.sha1
deleted file mode 100644
index 48c91d68f44..00000000000
--- a/distribution/licenses/lucene-queryparser-6.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-24f7ba0707aa01be2dd7749adff1659262be8f33
\ No newline at end of file
diff --git a/distribution/licenses/lucene-queryparser-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-queryparser-6.1.0-snapshot-3a57bea.jar.sha1
new file mode 100644
index 00000000000..6bf59bc00e7
--- /dev/null
+++ b/distribution/licenses/lucene-queryparser-6.1.0-snapshot-3a57bea.jar.sha1
@@ -0,0 +1 @@
+e322f004e574df119ba08dd8751a743422a46724
\ No newline at end of file
diff --git a/distribution/licenses/lucene-sandbox-6.0.1.jar.sha1 b/distribution/licenses/lucene-sandbox-6.0.1.jar.sha1
deleted file mode 100644
index ef843328aa0..00000000000
--- a/distribution/licenses/lucene-sandbox-6.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0faf4c0d7e0adb6fccd830a2d5797d4176b579fe
\ No newline at end of file
diff --git a/distribution/licenses/lucene-sandbox-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-sandbox-6.1.0-snapshot-3a57bea.jar.sha1
new file mode 100644
index 00000000000..79567427105
--- /dev/null
+++ b/distribution/licenses/lucene-sandbox-6.1.0-snapshot-3a57bea.jar.sha1
@@ -0,0 +1 @@
+c7cb119652c906adcdf7fe64445c76d057329d63
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial-6.0.1.jar.sha1 b/distribution/licenses/lucene-spatial-6.0.1.jar.sha1
deleted file mode 100644
index 25e7232ac14..00000000000
--- a/distribution/licenses/lucene-spatial-6.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4d94d006251c904de3f1503c64746400877d6fa3
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-spatial-6.1.0-snapshot-3a57bea.jar.sha1
new file mode 100644
index 00000000000..2b06178aaad
--- /dev/null
+++ b/distribution/licenses/lucene-spatial-6.1.0-snapshot-3a57bea.jar.sha1
@@ -0,0 +1 @@
+ca6c17fe31884e968ae63fd475ce6532b767c7fa
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial-extras-6.0.1.jar.sha1 b/distribution/licenses/lucene-spatial-extras-6.0.1.jar.sha1
deleted file mode 100644
index d421e1b053c..00000000000
--- a/distribution/licenses/lucene-spatial-extras-6.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3de19dbdb889fe87791dae291ac3b340586854c4
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial-extras-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-spatial-extras-6.1.0-snapshot-3a57bea.jar.sha1
new file mode 100644
index 00000000000..9c487b7746f
--- /dev/null
+++ b/distribution/licenses/lucene-spatial-extras-6.1.0-snapshot-3a57bea.jar.sha1
@@ -0,0 +1 @@
+49235405e40757474aaa9e8e54946b67fe2a01d9
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial3d-6.0.1.jar.sha1 b/distribution/licenses/lucene-spatial3d-6.0.1.jar.sha1
deleted file mode 100644
index 348f501bb52..00000000000
--- a/distribution/licenses/lucene-spatial3d-6.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5b1b7a754e83e2d58a819afa279b20b08b48c9c1
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial3d-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-spatial3d-6.1.0-snapshot-3a57bea.jar.sha1
new file mode 100644
index 00000000000..1eaab9f9955
--- /dev/null
+++ b/distribution/licenses/lucene-spatial3d-6.1.0-snapshot-3a57bea.jar.sha1
@@ -0,0 +1 @@
+39f6b29c428327860c1a342bd57800e79ad92ef5
\ No newline at end of file
diff --git a/distribution/licenses/lucene-suggest-6.0.1.jar.sha1 b/distribution/licenses/lucene-suggest-6.0.1.jar.sha1
deleted file mode 100644
index 2cb6272d826..00000000000
--- a/distribution/licenses/lucene-suggest-6.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-55886bdaf16ecc6948e94b527837eaa1f16fe988
\ No newline at end of file
diff --git a/distribution/licenses/lucene-suggest-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-suggest-6.1.0-snapshot-3a57bea.jar.sha1
new file mode 100644
index 00000000000..948dfc4c2b9
--- /dev/null
+++ b/distribution/licenses/lucene-suggest-6.1.0-snapshot-3a57bea.jar.sha1
@@ -0,0 +1 @@
+4e9f713d34fd4208bf308ac59132216f96521f13
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-6.0.1.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-6.0.1.jar.sha1
deleted file mode 100644
index 7b3f5a1cef9..00000000000
--- a/modules/lang-expression/licenses/lucene-expressions-6.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2b76056dbd40fb51dc5e8ef71e1919ad23e635a1
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-6.1.0-snapshot-3a57bea.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-6.1.0-snapshot-3a57bea.jar.sha1
new file mode 100644
index 00000000000..271088e86c9
--- /dev/null
+++ b/modules/lang-expression/licenses/lucene-expressions-6.1.0-snapshot-3a57bea.jar.sha1
@@ -0,0 +1 @@
+e5a4b673918f448006c0531799706abebe9a1db0
\ No newline at end of file
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java
index 11f0e4c191a..78d95edecff 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SimpleSortTests.java
@@ -23,6 +23,7 @@ package org.elasticsearch.messy.tests;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
@@ -41,7 +42,6 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Random;
 
-import static org.apache.lucene.spatial.util.GeoEncodingUtils.TOLERANCE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
@@ -218,7 +218,7 @@ public class SimpleSortTests extends ESIntegTestCase {
 
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(20L));
         for (int i = 0; i < 10; i++) {
-            assertThat("res: " + i + " id: " + searchResponse.getHits().getAt(i).getId(), (Double) searchResponse.getHits().getAt(i).field("min").value(), closeTo(i, TOLERANCE));
+            assertThat("res: " + i + " id: " + searchResponse.getHits().getAt(i).getId(), (Double) searchResponse.getHits().getAt(i).field("min").value(), closeTo(i, GeoUtils.TOLERANCE));
         }
     }
 
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
index 2f3108d6298..fbf06468e75 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
@@ -243,7 +243,7 @@ public final class PercolateQuery extends Query implements Accountable {
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
-        if (!super.equals(o)) return false;
+        if (sameClassAs(o) == false) return false;
 
         PercolateQuery that = (PercolateQuery) o;
 
@@ -254,7 +254,7 @@ public final class PercolateQuery extends Query implements Accountable {
 
     @Override
     public int hashCode() {
-        int result = super.hashCode();
+        int result = classHash();
         result = 31 * result + documentType.hashCode();
         result = 31 * result + documentSource.hashCode();
         return result;
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java
index 07959db1ff1..4879badc7d3 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java
@@ -379,6 +379,16 @@ public class PercolateQueryTests extends ESTestCase {
         public String toString(String field) {
             return "custom{" + field + "}";
         }
+
+        @Override
+        public boolean equals(Object obj) {
+            return sameClassAs(obj);
+        }
+
+        @Override
+        public int hashCode() {
+            return classHash();
+        }
     }
 
 }
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.0.1.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.0.1.jar.sha1
deleted file mode 100644
index 95dab25e74a..00000000000
--- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-da08d9919f54efd2e09968d49fe05f6ce3f0c7ce
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.1.0-snapshot-3a57bea.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.1.0-snapshot-3a57bea.jar.sha1
new file mode 100644
index 00000000000..013def114d4
--- /dev/null
+++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.1.0-snapshot-3a57bea.jar.sha1
@@ -0,0 +1 @@
+9cd8cbea5baef18a36bee86846a9ba026d2a02e0
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.0.1.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.0.1.jar.sha1
deleted file mode 100644
index 70f83bf52cc..00000000000
--- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-77905f563c47994a764a6ab3d5ec198c174567a7
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.1.0-snapshot-3a57bea.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.1.0-snapshot-3a57bea.jar.sha1
new file mode 100644
index 00000000000..12c861c24ab
--- /dev/null
+++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.1.0-snapshot-3a57bea.jar.sha1
@@ -0,0 +1 @@
+86c6d6a367ed658351bd8c8828d6ed647ac79b7e
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.0.1.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.0.1.jar.sha1
deleted file mode 100644
index 8e2f7ab8b98..00000000000
--- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3ee5d909c269e5da7a92715f41ead88943b38123
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.1.0-snapshot-3a57bea.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.1.0-snapshot-3a57bea.jar.sha1
new file mode 100644
index 00000000000..6f571d75537
--- /dev/null
+++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.1.0-snapshot-3a57bea.jar.sha1
@@ -0,0 +1 @@
+6553bf764a69cd15e4fe1e55661382872795b853
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.0.1.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.0.1.jar.sha1
deleted file mode 100644
index 981855d5a97..00000000000
--- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3b7bdbf9efa84f8d8875bd7f1d8734276930b9c3
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.1.0-snapshot-3a57bea.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.1.0-snapshot-3a57bea.jar.sha1
new file mode 100644
index 00000000000..2ea2d6b9622
--- /dev/null
+++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.1.0-snapshot-3a57bea.jar.sha1
@@ -0,0 +1 @@
+979817950bc806400d8fa12a609ef215b5bdebd6
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.0.1.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.0.1.jar.sha1
deleted file mode 100644
index 4ff0afee687..00000000000
--- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.0.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e80e912621276e1009b72c06d5def188976c5426
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.1.0-snapshot-3a57bea.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.1.0-snapshot-3a57bea.jar.sha1
new file mode 100644
index 00000000000..6677cfd3fc4
--- /dev/null
+++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.1.0-snapshot-3a57bea.jar.sha1
@@ -0,0 +1 @@
+2a720b647b6a202ec1d8d91db02006ae9539670b
\ No newline at end of file
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index 2a3eecf4cc8..4e066bc7635 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -108,6 +108,7 @@ import org.elasticsearch.index.codec.CodecService;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
 import org.elasticsearch.index.translog.Translog;
+import org.elasticsearch.indices.IndicesQueryCache;
 import org.elasticsearch.indices.IndicesRequestCache;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.store.IndicesStore;
@@ -1633,7 +1634,11 @@ public abstract class ESIntegTestCase extends ESTestCase {
                 .put(DiskThresholdDecider.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "1b")
                 .put("script.stored", "true")
                 .put("script.inline", "true")
-                        // wait short time for other active shards before actually deleting, default 30s not needed in tests
+                // by default we never cache below 10k docs in a segment,
+                // bypass this limit so that caching gets some testing in
+                // integration tests that usually create few documents
+                .put(IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING.getKey(), nodeOrdinal % 2 == 0)
+                // wait short time for other active shards before actually deleting, default 30s not needed in tests
                 .put(IndicesStore.INDICES_STORE_DELETE_SHARD_TIMEOUT.getKey(), new TimeValue(1, TimeUnit.SECONDS));
         return builder.build();
     }

From 6d8692576e49ed9d4d570e1631a25f015f271cb8 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 10 Jun 2016 10:19:47 -0400
Subject: [PATCH 0777/1311] Mark field in ReindexSameIndexTests as final

This commit restores a final modifier on the field
AutoCreateIndex#AUTO_CREATE_INDEX that was inadvertently removed in
a25b8ee1bf2e70bc28f688731ef43962a3ef18ce.
---
 .../org/elasticsearch/index/reindex/ReindexSameIndexTests.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSameIndexTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSameIndexTests.java
index 7bdb455cc33..2af05a8455b 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSameIndexTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSameIndexTests.java
@@ -43,7 +43,7 @@ public class ReindexSameIndexTests extends ESTestCase {
 
     private static ClusterState STATE;
     private static final IndexNameExpressionResolver INDEX_NAME_EXPRESSION_RESOLVER = new IndexNameExpressionResolver(Settings.EMPTY);
-    private static AutoCreateIndex AUTO_CREATE_INDEX = new AutoCreateIndex(Settings.EMPTY, INDEX_NAME_EXPRESSION_RESOLVER);
+    private static final AutoCreateIndex AUTO_CREATE_INDEX = new AutoCreateIndex(Settings.EMPTY, INDEX_NAME_EXPRESSION_RESOLVER);
 
     @BeforeClass
     public static void beforeClass() {

From 25fde039fd570b27c44ee1e2ea15e072baaccc8a Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 10 Jun 2016 10:42:23 -0400
Subject: [PATCH 0778/1311] [docs] Flow the refresh docs

They were making multiple pages but that is silly. They should all
be one page.
---
 docs/reference/docs/refresh.asciidoc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/reference/docs/refresh.asciidoc b/docs/reference/docs/refresh.asciidoc
index 3e5153341c8..dd829e19bc3 100644
--- a/docs/reference/docs/refresh.asciidoc
+++ b/docs/reference/docs/refresh.asciidoc
@@ -28,6 +28,7 @@ APIs that support it.
 Take no refresh related actions. The changes made by this request will be made
 visible at some point after the request returns.
 
+[float]
 === Choosing which setting to use
 
 Unless you have a good reason to wait for the change to become visible always
@@ -60,6 +61,7 @@ refresh immediately, `refresh=true` will affect other ongoing request. In
 general, if you have a running system you don't wish to disturb then
 `refresh=wait_for` is a smaller modification.
 
+[float]
 === `refresh=wait_for` Can Force a Refresh
 
 If a `refresh=wait_for` request comes in when there are already
@@ -74,6 +76,7 @@ contain `"forced_refresh": true`.
 Bulk requests only take up one slot on each shard that they touch no matter how
 many times they modify the shard.
 
+[float]
 === Examples
 
 These will create a document and immediately refresh the index so it is visible:

From 62f98767eb1ac37c70465770562a89302670be56 Mon Sep 17 00:00:00 2001
From: Areek Zillur <areek.zillur@elasticsearch.com>
Date: Fri, 10 Jun 2016 12:02:36 -0300
Subject: [PATCH 0779/1311] removed redundant Fields class

---
 .../indices/rollover/RolloverResponse.java    | 24 +++++++++----------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
index d3a36f7d561..0f254e825da 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverResponse.java
@@ -34,6 +34,12 @@ import java.util.stream.Collectors;
 
 public final class RolloverResponse extends ActionResponse implements ToXContent {
 
+    private static final String NEW_INDEX = "new_index";
+    private static final String OLD_INDEX = "old_index";
+    private static final String DRY_RUN = "dry_run";
+    private static final String ROLLED_OVER = "rolled_over";
+    private static final String CONDITIONS = "conditions";
+
     private String oldIndex;
     private String newIndex;
     private Set<Map.Entry<String, Boolean>> conditionStatus;
@@ -122,23 +128,15 @@ public final class RolloverResponse extends ActionResponse implements ToXContent
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(Fields.OLD_INDEX, oldIndex);
-        builder.field(Fields.NEW_INDEX, newIndex);
-        builder.field(Fields.ROLLED_OVER, rolledOver);
-        builder.field(Fields.DRY_RUN, dryRun);
-        builder.startObject(Fields.CONDITIONS);
+        builder.field(OLD_INDEX, oldIndex);
+        builder.field(NEW_INDEX, newIndex);
+        builder.field(ROLLED_OVER, rolledOver);
+        builder.field(DRY_RUN, dryRun);
+        builder.startObject(CONDITIONS);
         for (Map.Entry<String, Boolean> entry : conditionStatus) {
             builder.field(entry.getKey(), entry.getValue());
         }
         builder.endObject();
         return builder;
     }
-
-    static final class Fields {
-        static final String NEW_INDEX = "new_index";
-        static final String OLD_INDEX = "old_index";
-        static final String DRY_RUN = "dry_run";
-        static final String ROLLED_OVER = "rolled_over";
-        static final String CONDITIONS = "conditions";
-    }
 }

From df4a959d6c7b1bca1fe9ce7561635f835c519d24 Mon Sep 17 00:00:00 2001
From: Areek Zillur <areek.zillur@elasticsearch.com>
Date: Fri, 10 Jun 2016 12:06:50 -0300
Subject: [PATCH 0780/1311] removed support for customs from create index
 request

---
 .../action/admin/indices/rollover/TransportRolloverAction.java   | 1 -
 1 file changed, 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
index 533e24a2ad5..952448af5e7 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -217,7 +217,6 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
             .masterNodeTimeout(createIndexRequest.masterNodeTimeout())
             .settings(createIndexRequest.settings())
             .aliases(createIndexRequest.aliases())
-            .customs(createIndexRequest.customs())
             .mappings(createIndexRequest.mappings());
     }
 

From edb0fd3b888eb36c18669543a2933e1a5f0f908b Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Fri, 10 Jun 2016 14:24:58 -0400
Subject: [PATCH 0781/1311] painless: add capturing method references

---
 .../src/main/antlr/PainlessParser.g4          |   1 +
 .../java/org/elasticsearch/painless/Def.java  | 104 ++-
 .../elasticsearch/painless/DefBootstrap.java  |  50 +-
 .../elasticsearch/painless/Definition.java    |   3 +-
 .../elasticsearch/painless/FeatureTest.java   |   7 +
 .../elasticsearch/painless/FunctionRef.java   |  24 +-
 .../elasticsearch/painless/MethodWriter.java  |   1 -
 .../elasticsearch/painless/ScriptImpl.java    |   2 +-
 .../painless/WriterConstants.java             |   2 +-
 .../painless/antlr/PainlessParser.java        | 637 +++++++++---------
 .../elasticsearch/painless/antlr/Walker.java  |  19 +-
 .../painless/node/ECapturingFunctionRef.java  | 118 ++++
 .../painless/node/EFunctionRef.java           |   2 +-
 .../painless/node/LDefArray.java              |   4 +-
 .../elasticsearch/painless/node/LDefCall.java |  10 +-
 .../painless/node/LDefField.java              |   4 +-
 .../elasticsearch/painless/node/SEach.java    |   2 +-
 .../painless/org.elasticsearch.txt            |   1 +
 .../painless/DefBootstrapTests.java           |   6 +-
 .../painless/FunctionRefTests.java            |  66 +-
 20 files changed, 698 insertions(+), 365 deletions(-)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java

diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index 141ff25e318..bf141b53cbd 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -75,6 +75,7 @@ decltype
 
 funcref
     : TYPE REF ( ID | NEW )
+    | ID REF ID
     ;
 
 declvar
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 1edaddea2c1..e60cbef79cf 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -212,6 +212,8 @@ public final class Def {
      * until it finds a matching whitelisted method. If one is not found, it throws an exception.
      * Otherwise it returns a handle to the matching method.
      * <p>
+     * @param lookup caller's lookup
+     * @param callSiteType callsite's type
      * @param receiverClass Class of the object to invoke the method on.
      * @param name Name of the method.
      * @param args args passed to callsite
@@ -220,31 +222,103 @@ public final class Def {
      * @throws LambdaConversionException if a method reference cannot be converted to an functional interface
      * @throws IllegalArgumentException if no matching whitelisted method was found.
      */
-     static MethodHandle lookupMethod(Lookup lookup, Class<?> receiverClass, String name,
-             Object args[], long recipe) throws LambdaConversionException {
-         Method method = lookupMethodInternal(receiverClass, name, args.length - 1);
+     static MethodHandle lookupMethod(Lookup lookup, MethodType callSiteType, 
+             Class<?> receiverClass, String name, Object args[], long recipe) throws LambdaConversionException {
+         // simple case: no lambdas
+         if (recipe == 0) {
+             return lookupMethodInternal(receiverClass, name, args.length - 1).handle;
+         }
+         
+         // otherwise: first we have to compute the "real" arity. This is because we have extra arguments:
+         // e.g. f(a, g(x), b, h(y), i()) looks like f(a, g, x, b, h, y, i). 
+         int arity = args.length - 1;
+         for (int i = 0; i < args.length; i++) {
+             if ((recipe & (1L << (i - 1))) != 0) {
+                 String signature = (String) args[i];
+                 int numCaptures = Integer.parseInt(signature.substring(signature.indexOf(',')+1));
+                 arity -= numCaptures;
+             }
+         }
+         
+         // lookup the method with the proper arity, then we know everything (e.g. interface types of parameters).
+         // based on these we can finally link any remaining lambdas that were deferred.
+         Method method = lookupMethodInternal(receiverClass, name, arity);
          MethodHandle handle = method.handle;
 
-         if (recipe != 0) {
-             MethodHandle filters[] = new MethodHandle[args.length];
-             for (int i = 0; i < args.length; i++) {
-                 // its a functional reference, replace the argument with an impl
-                 if ((recipe & (1L << (i - 1))) != 0) {
-                     filters[i] = lookupReference(lookup, method.arguments.get(i - 1), (String) args[i]);
+         int replaced = 0;
+         for (int i = 1; i < args.length; i++) {
+             // its a functional reference, replace the argument with an impl
+             if ((recipe & (1L << (i - 1))) != 0) {
+                 // decode signature of form 'type.call,2' 
+                 String signature = (String) args[i];
+                 int separator = signature.indexOf('.');
+                 int separator2 = signature.indexOf(',');
+                 String type = signature.substring(1, separator);
+                 String call = signature.substring(separator+1, separator2);
+                 int numCaptures = Integer.parseInt(signature.substring(separator2+1));
+                 Class<?> captures[] = new Class<?>[numCaptures];
+                 for (int capture = 0; capture < captures.length; capture++) {
+                     captures[capture] = callSiteType.parameterType(i + 1 + capture);
                  }
+                 MethodHandle filter;
+                 Definition.Type interfaceType = method.arguments.get(i - 1 - replaced);
+                 if (signature.charAt(0) == 'S') {
+                     // the implementation is strongly typed, now that we know the interface type,
+                     // we have everything.
+                     filter = lookupReferenceInternal(lookup,
+                                                      interfaceType,
+                                                      type,
+                                                      call,
+                                                      captures);
+                 } else if (signature.charAt(0) == 'D') {
+                     // the interface type is now known, but we need to get the implementation.
+                     // this is dynamically based on the receiver type (and cached separately, underneath
+                     // this cache). It won't blow up since we never nest here (just references)
+                     MethodType nestedType = MethodType.methodType(interfaceType.clazz, captures);
+                     CallSite nested = DefBootstrap.bootstrap(lookup, 
+                                                              call,
+                                                              nestedType, 
+                                                              DefBootstrap.REFERENCE,
+                                                              interfaceType.name);
+                     filter = nested.dynamicInvoker();
+                 } else {
+                     throw new AssertionError();
+                 }
+                 // the filter now ignores the signature (placeholder) on the stack
+                 filter = MethodHandles.dropArguments(filter, 0, String.class);
+                 handle = MethodHandles.collectArguments(handle, i, filter);
+                 i += numCaptures;
+                 replaced += numCaptures;
              }
-             handle = MethodHandles.filterArguments(handle, 0, filters);
          }
          
          return handle;
      }
      
+     /**
+      * Returns an implementation of interfaceClass that calls receiverClass.name
+      * <p>
+      * This is just like LambdaMetaFactory, only with a dynamic type. The interface type is known,
+      * so we simply need to lookup the matching implementation method based on receiver type.
+      */
+     static MethodHandle lookupReference(Lookup lookup, String interfaceClass, 
+                                         Class<?> receiverClass, String name) throws LambdaConversionException {
+         Definition.Type interfaceType = Definition.getType(interfaceClass);
+         Method interfaceMethod = interfaceType.struct.getFunctionalMethod();
+         if (interfaceMethod == null) {
+             throw new IllegalArgumentException("Class [" + interfaceClass + "] is not a functional interface");
+         }
+         int arity = interfaceMethod.arguments.size();
+         Method implMethod = lookupMethodInternal(receiverClass, name, arity);
+         return lookupReferenceInternal(lookup, interfaceType, implMethod.owner.name, implMethod.name, receiverClass);
+     }
+     
      /** Returns a method handle to an implementation of clazz, given method reference signature 
       * @throws LambdaConversionException if a method reference cannot be converted to an functional interface
       */
-     private static MethodHandle lookupReference(Lookup lookup, Definition.Type clazz, String signature) throws LambdaConversionException {
-         int separator = signature.indexOf('.');
-         FunctionRef ref = new FunctionRef(clazz, signature.substring(0, separator), signature.substring(separator+1));
+     private static MethodHandle lookupReferenceInternal(Lookup lookup, Definition.Type clazz, String type,
+                                                         String call, Class<?>... captures) throws LambdaConversionException {
+         FunctionRef ref = new FunctionRef(clazz, type, call, captures);
          final CallSite callSite;
          if (ref.needsBridges()) {
              callSite = LambdaMetafactory.altMetafactory(lookup, 
@@ -265,9 +339,7 @@ public final class Def {
                      ref.samMethodType,
                      0);
          }
-         // we could actually invoke and cache here (in non-capturing cases), but this is not a speedup.
-         MethodHandle factory = callSite.dynamicInvoker().asType(MethodType.methodType(clazz.clazz));
-         return MethodHandles.dropArguments(factory, 0, Object.class);
+         return callSite.dynamicInvoker().asType(MethodType.methodType(clazz.clazz, captures));
      }
      
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index 5dc773672a7..040008c0d62 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -60,6 +60,8 @@ public final class DefBootstrap {
     public static final int ARRAY_STORE = 4;
     /** static bootstrap parameter indicating a dynamic iteration, e.g. for (x : y) */
     public static final int ITERATOR = 5;
+    /** static bootstrap parameter indicating a dynamic method reference, e.g. foo::bar */
+    public static final int REFERENCE = 6;
 
     /**
      * CallSite that implements the polymorphic inlining cache (PIC).
@@ -71,17 +73,15 @@ public final class DefBootstrap {
         private final Lookup lookup;
         private final String name;
         private final int flavor;
-        private final long recipe;
+        private final Object[] args;
         int depth; // pkg-protected for testing
 
-        PIC(Lookup lookup, String name, MethodType type, int flavor, long recipe) {
+        PIC(Lookup lookup, String name, MethodType type, int flavor, Object[] args) {
             super(type);
             this.lookup = lookup;
             this.name = name;
             this.flavor = flavor;
-            this.recipe = recipe;
-            assert recipe == 0 || flavor == METHOD_CALL;
-            assert Long.bitCount(recipe) <= type.parameterCount();
+            this.args = args;
 
             final MethodHandle fallback = FALLBACK.bindTo(this)
               .asCollector(Object[].class, type.parameterCount())
@@ -101,10 +101,10 @@ public final class DefBootstrap {
         /**
          * Does a slow lookup against the whitelist.
          */
-        private MethodHandle lookup(int flavor, Class<?> clazz, String name, Object[] args, long recipe) throws Throwable {
+        private MethodHandle lookup(int flavor, Class<?> clazz, String name, Object[] args) throws Throwable {
             switch(flavor) {
                 case METHOD_CALL:
-                    return Def.lookupMethod(lookup, clazz, name, args, recipe);
+                    return Def.lookupMethod(lookup, type(), clazz, name, args, (Long) this.args[0]);
                 case LOAD:
                     return Def.lookupGetter(clazz, name);
                 case STORE:
@@ -115,6 +115,8 @@ public final class DefBootstrap {
                     return Def.lookupArrayStore(clazz);
                 case ITERATOR:
                     return Def.lookupIterator(clazz);
+                case REFERENCE:
+                    return Def.lookupReference(lookup, (String) this.args[0], clazz, name);
                 default: throw new AssertionError();
             }
         }
@@ -128,7 +130,7 @@ public final class DefBootstrap {
             final MethodType type = type();
             final Object receiver = args[0];
             final Class<?> receiverClass = receiver.getClass();
-            final MethodHandle target = lookup(flavor, receiverClass, name, args, recipe).asType(type);
+            final MethodHandle target = lookup(flavor, receiverClass, name, args).asType(type);
 
             if (depth >= MAX_DEPTH) {
                 // revert to a vtable call
@@ -170,8 +172,36 @@ public final class DefBootstrap {
      * <p>
      * see https://docs.oracle.com/javase/specs/jvms/se7/html/jvms-6.html#jvms-6.5.invokedynamic
      */
-    public static CallSite bootstrap(Lookup lookup, String name, MethodType type, int flavor, long recipe) {
-        return new PIC(lookup, name, type, flavor, recipe);
+    public static CallSite bootstrap(Lookup lookup, String name, MethodType type, int flavor, Object... args) {
+        // validate arguments
+        switch(flavor) {
+            case METHOD_CALL:
+                if (args.length != 1) {
+                    throw new BootstrapMethodError("Invalid number of parameters for method call");
+                }
+                if (args[0] instanceof Long == false) {
+                    throw new BootstrapMethodError("Illegal parameter for method call: " + args[0]);
+                }
+                long recipe = (Long) args[0];
+                if (Long.bitCount(recipe) > type.parameterCount()) {
+                    throw new BootstrapMethodError("Illegal recipe for method call: too many bits");
+                }
+                break;
+            case REFERENCE:
+                if (args.length != 1) {
+                    throw new BootstrapMethodError("Invalid number of parameters for reference call");
+                }
+                if (args[0] instanceof String == false) {
+                    throw new BootstrapMethodError("Illegal parameter for reference call: " + args[0]);
+                }
+                break;
+            default:
+                if (args.length > 0) {
+                    throw new BootstrapMethodError("Illegal static bootstrap parameters for flavor: " + flavor);
+                }
+                break;
+        }
+        return new PIC(lookup, name, type, flavor, args);
     }
 
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index dbfc9993c31..1285b4cfb5a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -890,8 +890,7 @@ public final class Definition {
                             throw new AssertionError(e);
                         }
                     }
-                    owner.methods.put(methodKey,
-                        new Method(method.name, owner, method.rtn, method.arguments, method.method, method.modifiers, method.handle));
+                    owner.methods.put(methodKey, method);
                 }
             }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java
index bef83daaad3..603023e61fe 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java
@@ -1,5 +1,7 @@
 package org.elasticsearch.painless;
 
+import java.util.function.Function;
+
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -63,4 +65,9 @@ public class FeatureTest {
     public static boolean overloadedStatic(boolean whatToReturn) {
         return whatToReturn;
     }
+    
+    /** method taking two functions! */
+    public Object twoFunctionsOfX(Function<Object,Object> f, Function<Object,Object> g) {
+        return f.apply(g.apply(x));
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
index 03b6cc604c9..fb0175cc97e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
@@ -53,8 +53,9 @@ public class FunctionRef {
      * @param expected interface type to implement.
      * @param type the left hand side of a method reference expression
      * @param call the right hand side of a method reference expression
+     * @param captures captured arguments
      */
-    public FunctionRef(Definition.Type expected, String type, String call) {
+    public FunctionRef(Definition.Type expected, String type, String call, Class<?>... captures) {
         boolean isCtorReference = "new".equals(call);
         // check its really a functional interface
         // for e.g. Comparable
@@ -66,7 +67,7 @@ public class FunctionRef {
         // e.g. compareTo
         invokedName = method.name;
         // e.g. (Object)Comparator
-        invokedType = MethodType.methodType(expected.clazz);
+        invokedType = MethodType.methodType(expected.clazz, captures);
         // e.g. (Object,Object)int
         interfaceMethodType = method.handle.type().dropParameterTypes(0, 1);
         // lookup requested method
@@ -80,7 +81,15 @@ public class FunctionRef {
             Definition.Method staticImpl = struct.staticMethods.get(new Definition.MethodKey(call, method.arguments.size()));
             if (staticImpl == null) {
                 // otherwise a virtual impl
-                impl = struct.methods.get(new Definition.MethodKey(call, method.arguments.size()-1));
+                final int arity;
+                if (captures.length > 0) {
+                    // receiver captured
+                    arity = method.arguments.size();
+                } else {
+                    // receiver passed
+                    arity = method.arguments.size() - 1;
+                }
+                impl = struct.methods.get(new Definition.MethodKey(call, arity));
             } else {
                 impl = staticImpl;
             }
@@ -98,12 +107,19 @@ public class FunctionRef {
         } else {
             tag = Opcodes.H_INVOKEVIRTUAL;
         }
-        implMethodASM = new Handle(tag, struct.type.getInternalName(), impl.name, impl.method.getDescriptor());
+        if (impl.owner.clazz.isInterface()) {
+            implMethodASM = new Handle(tag, struct.type.getInternalName(), impl.name, impl.method.getDescriptor());
+        } else {
+            implMethodASM = new Handle(tag, impl.owner.type.getInternalName(), impl.name, impl.method.getDescriptor());
+        }
         implMethod = impl.handle;
         if (isCtorReference) {
             samMethodType = MethodType.methodType(interfaceMethodType.returnType(), impl.handle.type().parameterArray());
         } else if (Modifier.isStatic(impl.modifiers)) {
             samMethodType = impl.handle.type();
+        } else if (captures.length > 0) {
+            // drop the receiver, we capture it
+            samMethodType = impl.handle.type().dropParameterTypes(0, 1);
         } else {
             // ensure the receiver type is exact and not a superclass type
             samMethodType = impl.handle.type().changeParameterType(0, struct.clazz);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index 7b7caec320e..d7c09a9c122 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -22,7 +22,6 @@ package org.elasticsearch.painless;
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
-import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.ClassWriter;
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
index 8fdc48e19ba..b303089b339 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptImpl.java
@@ -119,7 +119,7 @@ final class ScriptImpl implements ExecutableScript, LeafSearchScript {
     public Object run() {
         try {
             return executable.execute(variables, scorer, doc, aggregationValue);
-        } catch (PainlessError | BootstrapMethodError | Exception t) {
+        } catch (PainlessError | BootstrapMethodError | IllegalAccessError | Exception t) {
             throw convertToScriptException(t);
         }
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index 80082c4487b..7129df80a08 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -68,7 +68,7 @@ public final class WriterConstants {
 
     /** dynamic callsite bootstrap signature */
     public final static MethodType DEF_BOOTSTRAP_TYPE =
-        MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, int.class, long.class);
+        MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, int.class, Object[].class);
     public final static Handle DEF_BOOTSTRAP_HANDLE =
         new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(DefBootstrap.class),
             "bootstrap", DEF_BOOTSTRAP_TYPE.toMethodDescriptorString());
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
index 540773fbf70..a3ccc415028 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
@@ -1029,7 +1029,10 @@ class PainlessParser extends Parser {
   public static class FuncrefContext extends ParserRuleContext {
     public TerminalNode TYPE() { return getToken(PainlessParser.TYPE, 0); }
     public TerminalNode REF() { return getToken(PainlessParser.REF, 0); }
-    public TerminalNode ID() { return getToken(PainlessParser.ID, 0); }
+    public List<TerminalNode> ID() { return getTokens(PainlessParser.ID); }
+    public TerminalNode ID(int i) {
+      return getToken(PainlessParser.ID, i);
+    }
     public TerminalNode NEW() { return getToken(PainlessParser.NEW, 0); }
     public FuncrefContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
@@ -1047,19 +1050,37 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 18, RULE_funcref);
     int _la;
     try {
-      enterOuterAlt(_localctx, 1);
-      {
-      setState(170);
-      match(TYPE);
-      setState(171);
-      match(REF);
-      setState(172);
-      _la = _input.LA(1);
-      if ( !(_la==NEW || _la==ID) ) {
-      _errHandler.recoverInline(this);
-      } else {
-        consume();
-      }
+      setState(176);
+      switch (_input.LA(1)) {
+      case TYPE:
+        enterOuterAlt(_localctx, 1);
+        {
+        setState(170);
+        match(TYPE);
+        setState(171);
+        match(REF);
+        setState(172);
+        _la = _input.LA(1);
+        if ( !(_la==NEW || _la==ID) ) {
+        _errHandler.recoverInline(this);
+        } else {
+          consume();
+        }
+        }
+        break;
+      case ID:
+        enterOuterAlt(_localctx, 2);
+        {
+        setState(173);
+        match(ID);
+        setState(174);
+        match(REF);
+        setState(175);
+        match(ID);
+        }
+        break;
+      default:
+        throw new NoViableAltException(this);
       }
     }
     catch (RecognitionException re) {
@@ -1097,15 +1118,15 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(174);
+      setState(178);
       match(ID);
-      setState(177);
+      setState(181);
       _la = _input.LA(1);
       if (_la==ASSIGN) {
         {
-        setState(175);
+        setState(179);
         match(ASSIGN);
-        setState(176);
+        setState(180);
         expression(0);
         }
       }
@@ -1149,17 +1170,17 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(179);
-      match(CATCH);
-      setState(180);
-      match(LP);
-      setState(181);
-      match(TYPE);
-      setState(182);
-      match(ID);
       setState(183);
-      match(RP);
+      match(CATCH);
       setState(184);
+      match(LP);
+      setState(185);
+      match(TYPE);
+      setState(186);
+      match(ID);
+      setState(187);
+      match(RP);
+      setState(188);
       block();
       }
     }
@@ -1195,7 +1216,7 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(186);
+      setState(190);
       _la = _input.LA(1);
       if ( !(_la==EOF || _la==SEMICOLON) ) {
       _errHandler.recoverInline(this);
@@ -1364,24 +1385,24 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(197);
-      switch ( getInterpreter().adaptivePredict(_input,15,_ctx) ) {
+      setState(201);
+      switch ( getInterpreter().adaptivePredict(_input,16,_ctx) ) {
       case 1:
         {
         _localctx = new AssignmentContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(189);
+        setState(193);
         chain(true);
-        setState(190);
+        setState(194);
         _la = _input.LA(1);
         if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ASSIGN) | (1L << AADD) | (1L << ASUB) | (1L << AMUL) | (1L << ADIV) | (1L << AREM) | (1L << AAND) | (1L << AXOR) | (1L << AOR) | (1L << ALSH) | (1L << ARSH) | (1L << AUSH))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(191);
+        setState(195);
         expression(1);
          ((AssignmentContext)_localctx).s =  false; 
         }
@@ -1391,37 +1412,37 @@ class PainlessParser extends Parser {
         _localctx = new SingleContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(194);
+        setState(198);
         ((SingleContext)_localctx).u = unary(false);
          ((SingleContext)_localctx).s =  ((SingleContext)_localctx).u.s; 
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(258);
+      setState(262);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,17,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,18,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(256);
-          switch ( getInterpreter().adaptivePredict(_input,16,_ctx) ) {
+          setState(260);
+          switch ( getInterpreter().adaptivePredict(_input,17,_ctx) ) {
           case 1:
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(199);
+            setState(203);
             if (!(precpred(_ctx, 12))) throw new FailedPredicateException(this, "precpred(_ctx, 12)");
-            setState(200);
+            setState(204);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << MUL) | (1L << DIV) | (1L << REM))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(201);
+            setState(205);
             expression(13);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1430,16 +1451,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(204);
+            setState(208);
             if (!(precpred(_ctx, 11))) throw new FailedPredicateException(this, "precpred(_ctx, 11)");
-            setState(205);
+            setState(209);
             _la = _input.LA(1);
             if ( !(_la==ADD || _la==SUB) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(206);
+            setState(210);
             expression(12);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1448,16 +1469,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(209);
+            setState(213);
             if (!(precpred(_ctx, 10))) throw new FailedPredicateException(this, "precpred(_ctx, 10)");
-            setState(210);
+            setState(214);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LSH) | (1L << RSH) | (1L << USH))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(211);
+            setState(215);
             expression(11);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1466,16 +1487,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(214);
+            setState(218);
             if (!(precpred(_ctx, 9))) throw new FailedPredicateException(this, "precpred(_ctx, 9)");
-            setState(215);
+            setState(219);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LT) | (1L << LTE) | (1L << GT) | (1L << GTE))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(216);
+            setState(220);
             expression(10);
              ((CompContext)_localctx).s =  false; 
             }
@@ -1484,16 +1505,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(219);
+            setState(223);
             if (!(precpred(_ctx, 8))) throw new FailedPredicateException(this, "precpred(_ctx, 8)");
-            setState(220);
+            setState(224);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << EQ) | (1L << EQR) | (1L << NE) | (1L << NER))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(221);
+            setState(225);
             expression(9);
              ((CompContext)_localctx).s =  false; 
             }
@@ -1502,11 +1523,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(224);
+            setState(228);
             if (!(precpred(_ctx, 7))) throw new FailedPredicateException(this, "precpred(_ctx, 7)");
-            setState(225);
+            setState(229);
             match(BWAND);
-            setState(226);
+            setState(230);
             expression(8);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1515,11 +1536,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(229);
+            setState(233);
             if (!(precpred(_ctx, 6))) throw new FailedPredicateException(this, "precpred(_ctx, 6)");
-            setState(230);
+            setState(234);
             match(XOR);
-            setState(231);
+            setState(235);
             expression(7);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1528,11 +1549,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(234);
+            setState(238);
             if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)");
-            setState(235);
+            setState(239);
             match(BWOR);
-            setState(236);
+            setState(240);
             expression(6);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1541,11 +1562,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(239);
+            setState(243);
             if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)");
-            setState(240);
+            setState(244);
             match(BOOLAND);
-            setState(241);
+            setState(245);
             expression(5);
              ((BoolContext)_localctx).s =  false; 
             }
@@ -1554,11 +1575,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(244);
+            setState(248);
             if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)");
-            setState(245);
+            setState(249);
             match(BOOLOR);
-            setState(246);
+            setState(250);
             expression(4);
              ((BoolContext)_localctx).s =  false; 
             }
@@ -1567,15 +1588,15 @@ class PainlessParser extends Parser {
             {
             _localctx = new ConditionalContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(249);
-            if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
-            setState(250);
-            match(COND);
-            setState(251);
-            ((ConditionalContext)_localctx).e0 = expression(0);
-            setState(252);
-            match(COLON);
             setState(253);
+            if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
+            setState(254);
+            match(COND);
+            setState(255);
+            ((ConditionalContext)_localctx).e0 = expression(0);
+            setState(256);
+            match(COLON);
+            setState(257);
             ((ConditionalContext)_localctx).e1 = expression(2);
              ((ConditionalContext)_localctx).s =  ((ConditionalContext)_localctx).e0.s && ((ConditionalContext)_localctx).e1.s; 
             }
@@ -1583,9 +1604,9 @@ class PainlessParser extends Parser {
           }
           } 
         }
-        setState(260);
+        setState(264);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,17,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,18,_ctx);
       }
       }
     }
@@ -1730,22 +1751,22 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 28, RULE_unary);
     int _la;
     try {
-      setState(290);
-      switch ( getInterpreter().adaptivePredict(_input,18,_ctx) ) {
+      setState(294);
+      switch ( getInterpreter().adaptivePredict(_input,19,_ctx) ) {
       case 1:
         _localctx = new PreContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(261);
+        setState(265);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(262);
+        setState(266);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(263);
+        setState(267);
         chain(true);
         }
         break;
@@ -1753,11 +1774,11 @@ class PainlessParser extends Parser {
         _localctx = new PostContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(264);
+        setState(268);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(265);
+        setState(269);
         chain(true);
-        setState(266);
+        setState(270);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
@@ -1770,9 +1791,9 @@ class PainlessParser extends Parser {
         _localctx = new ReadContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(268);
+        setState(272);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(269);
+        setState(273);
         chain(false);
         }
         break;
@@ -1780,9 +1801,9 @@ class PainlessParser extends Parser {
         _localctx = new NumericContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(270);
+        setState(274);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(271);
+        setState(275);
         _la = _input.LA(1);
         if ( !(((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (OCTAL - 64)) | (1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)))) != 0)) ) {
         _errHandler.recoverInline(this);
@@ -1796,9 +1817,9 @@ class PainlessParser extends Parser {
         _localctx = new TrueContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(273);
+        setState(277);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(274);
+        setState(278);
         match(TRUE);
          ((TrueContext)_localctx).s =  false; 
         }
@@ -1807,9 +1828,9 @@ class PainlessParser extends Parser {
         _localctx = new FalseContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(276);
+        setState(280);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(277);
+        setState(281);
         match(FALSE);
          ((FalseContext)_localctx).s =  false; 
         }
@@ -1818,9 +1839,9 @@ class PainlessParser extends Parser {
         _localctx = new NullContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(279);
+        setState(283);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(280);
+        setState(284);
         match(NULL);
          ((NullContext)_localctx).s =  false; 
         }
@@ -1829,16 +1850,16 @@ class PainlessParser extends Parser {
         _localctx = new OperatorContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(282);
+        setState(286);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(283);
+        setState(287);
         _la = _input.LA(1);
         if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(284);
+        setState(288);
         unary(false);
         }
         break;
@@ -1846,13 +1867,13 @@ class PainlessParser extends Parser {
         _localctx = new CastContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(285);
+        setState(289);
         match(LP);
-        setState(286);
+        setState(290);
         decltype();
-        setState(287);
+        setState(291);
         match(RP);
-        setState(288);
+        setState(292);
         unary(_localctx.c);
         }
         break;
@@ -1961,29 +1982,29 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 30, RULE_chain);
     try {
       int _alt;
-      setState(326);
-      switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) {
+      setState(330);
+      switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) {
       case 1:
         _localctx = new DynamicContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(292);
-        ((DynamicContext)_localctx).p = primary(_localctx.c);
         setState(296);
+        ((DynamicContext)_localctx).p = primary(_localctx.c);
+        setState(300);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,19,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(293);
+            setState(297);
             secondary(((DynamicContext)_localctx).p.s);
             }
             } 
           }
-          setState(298);
+          setState(302);
           _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,19,_ctx);
+          _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
         }
         }
         break;
@@ -1991,25 +2012,25 @@ class PainlessParser extends Parser {
         _localctx = new StaticContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(299);
+        setState(303);
         decltype();
-        setState(300);
-        dot();
         setState(304);
+        dot();
+        setState(308);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(301);
+            setState(305);
             secondary(true);
             }
             } 
           }
-          setState(306);
+          setState(310);
           _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
+          _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
         }
         }
         break;
@@ -2017,11 +2038,11 @@ class PainlessParser extends Parser {
         _localctx = new NewarrayContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(307);
+        setState(311);
         match(NEW);
-        setState(308);
+        setState(312);
         match(TYPE);
-        setState(313); 
+        setState(317); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -2029,11 +2050,11 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(309);
+            setState(313);
             match(LBRACE);
-            setState(310);
+            setState(314);
             expression(0);
-            setState(311);
+            setState(315);
             match(RBRACE);
             }
             }
@@ -2041,31 +2062,31 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(315); 
-          _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
-        } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
-        setState(324);
-        switch ( getInterpreter().adaptivePredict(_input,23,_ctx) ) {
-        case 1:
-          {
-          setState(317);
-          dot();
-          setState(321);
+          setState(319); 
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
+        } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
+        setState(328);
+        switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) {
+        case 1:
+          {
+          setState(321);
+          dot();
+          setState(325);
+          _errHandler.sync(this);
+          _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
           while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
             if ( _alt==1 ) {
               {
               {
-              setState(318);
+              setState(322);
               secondary(true);
               }
               } 
             }
-            setState(323);
+            setState(327);
             _errHandler.sync(this);
-            _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
+            _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
           }
           }
           break;
@@ -2165,19 +2186,19 @@ class PainlessParser extends Parser {
     PrimaryContext _localctx = new PrimaryContext(_ctx, getState(), c);
     enterRule(_localctx, 32, RULE_primary);
     try {
-      setState(344);
-      switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) {
+      setState(348);
+      switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) {
       case 1:
         _localctx = new ExprprecContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(328);
+        setState(332);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(329);
+        setState(333);
         match(LP);
-        setState(330);
+        setState(334);
         ((ExprprecContext)_localctx).e = expression(0);
-        setState(331);
+        setState(335);
         match(RP);
          ((ExprprecContext)_localctx).s =  ((ExprprecContext)_localctx).e.s; 
         }
@@ -2186,13 +2207,13 @@ class PainlessParser extends Parser {
         _localctx = new ChainprecContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(334);
+        setState(338);
         if (!( _localctx.c )) throw new FailedPredicateException(this, " $c ");
-        setState(335);
+        setState(339);
         match(LP);
-        setState(336);
+        setState(340);
         unary(true);
-        setState(337);
+        setState(341);
         match(RP);
         }
         break;
@@ -2200,7 +2221,7 @@ class PainlessParser extends Parser {
         _localctx = new StringContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(339);
+        setState(343);
         match(STRING);
         }
         break;
@@ -2208,7 +2229,7 @@ class PainlessParser extends Parser {
         _localctx = new VariableContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(340);
+        setState(344);
         match(ID);
         }
         break;
@@ -2216,11 +2237,11 @@ class PainlessParser extends Parser {
         _localctx = new NewobjectContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(341);
+        setState(345);
         match(NEW);
-        setState(342);
+        setState(346);
         match(TYPE);
-        setState(343);
+        setState(347);
         arguments();
         }
         break;
@@ -2262,23 +2283,23 @@ class PainlessParser extends Parser {
     SecondaryContext _localctx = new SecondaryContext(_ctx, getState(), s);
     enterRule(_localctx, 34, RULE_secondary);
     try {
-      setState(350);
-      switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) {
+      setState(354);
+      switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(346);
+        setState(350);
         if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
-        setState(347);
+        setState(351);
         dot();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(348);
+        setState(352);
         if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
-        setState(349);
+        setState(353);
         brace();
         }
         break;
@@ -2336,17 +2357,17 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 36, RULE_dot);
     int _la;
     try {
-      setState(357);
-      switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) {
+      setState(361);
+      switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) {
       case 1:
         _localctx = new CallinvokeContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(352);
+        setState(356);
         match(DOT);
-        setState(353);
+        setState(357);
         match(DOTID);
-        setState(354);
+        setState(358);
         arguments();
         }
         break;
@@ -2354,9 +2375,9 @@ class PainlessParser extends Parser {
         _localctx = new FieldaccessContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(355);
+        setState(359);
         match(DOT);
-        setState(356);
+        setState(360);
         _la = _input.LA(1);
         if ( !(_la==DOTINTEGER || _la==DOTID) ) {
         _errHandler.recoverInline(this);
@@ -2410,11 +2431,11 @@ class PainlessParser extends Parser {
       _localctx = new BraceaccessContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(359);
+      setState(363);
       match(LBRACE);
-      setState(360);
+      setState(364);
       expression(0);
-      setState(361);
+      setState(365);
       match(RBRACE);
       }
     }
@@ -2461,34 +2482,34 @@ class PainlessParser extends Parser {
       enterOuterAlt(_localctx, 1);
       {
       {
-      setState(363);
+      setState(367);
       match(LP);
-      setState(372);
-      switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
+      setState(376);
+      switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
       case 1:
         {
-        setState(364);
+        setState(368);
         argument();
-        setState(369);
+        setState(373);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(365);
+          setState(369);
           match(COMMA);
-          setState(366);
+          setState(370);
           argument();
           }
           }
-          setState(371);
+          setState(375);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
         break;
       }
-      setState(374);
+      setState(378);
       match(RP);
       }
       }
@@ -2526,19 +2547,19 @@ class PainlessParser extends Parser {
     ArgumentContext _localctx = new ArgumentContext(_ctx, getState());
     enterRule(_localctx, 42, RULE_argument);
     try {
-      setState(378);
-      switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
+      setState(382);
+      switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(376);
+        setState(380);
         expression(0);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(377);
+        setState(381);
         funcref();
         }
         break;
@@ -2645,7 +2666,7 @@ class PainlessParser extends Parser {
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3M\u017f\4\2\t\2\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3M\u0183\4\2\t\2\4"+
     "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
     "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
     "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\3\2\7\2\60\n\2\f\2"+
@@ -2657,134 +2678,136 @@ class PainlessParser extends Parser {
     "\u0085\n\3\3\4\3\4\5\4\u0089\n\4\3\5\3\5\7\5\u008d\n\5\f\5\16\5\u0090"+
     "\13\5\3\5\3\5\3\6\3\6\3\7\3\7\5\7\u0098\n\7\3\b\3\b\3\t\3\t\3\t\3\t\7"+
     "\t\u00a0\n\t\f\t\16\t\u00a3\13\t\3\n\3\n\3\n\7\n\u00a8\n\n\f\n\16\n\u00ab"+
-    "\13\n\3\13\3\13\3\13\3\13\3\f\3\f\3\f\5\f\u00b4\n\f\3\r\3\r\3\r\3\r\3"+
-    "\r\3\r\3\r\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\5\17"+
-    "\u00c8\n\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
+    "\13\n\3\13\3\13\3\13\3\13\3\13\3\13\5\13\u00b3\n\13\3\f\3\f\3\f\5\f\u00b8"+
+    "\n\f\3\r\3\r\3\r\3\r\3\r\3\r\3\r\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3"+
+    "\17\3\17\3\17\3\17\5\17\u00cc\n\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
     "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
     "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
     "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
-    "\3\17\3\17\3\17\7\17\u0103\n\17\f\17\16\17\u0106\13\17\3\20\3\20\3\20"+
+    "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\7\17\u0107\n\17\f\17\16\17\u010a"+
+    "\13\17\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
-    "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\5\20\u0125"+
-    "\n\20\3\21\3\21\7\21\u0129\n\21\f\21\16\21\u012c\13\21\3\21\3\21\3\21"+
-    "\7\21\u0131\n\21\f\21\16\21\u0134\13\21\3\21\3\21\3\21\3\21\3\21\3\21"+
-    "\6\21\u013c\n\21\r\21\16\21\u013d\3\21\3\21\7\21\u0142\n\21\f\21\16\21"+
-    "\u0145\13\21\5\21\u0147\n\21\5\21\u0149\n\21\3\22\3\22\3\22\3\22\3\22"+
-    "\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\5\22\u015b\n\22"+
-    "\3\23\3\23\3\23\3\23\5\23\u0161\n\23\3\24\3\24\3\24\3\24\3\24\5\24\u0168"+
-    "\n\24\3\25\3\25\3\25\3\25\3\26\3\26\3\26\3\26\7\26\u0172\n\26\f\26\16"+
-    "\26\u0175\13\26\5\26\u0177\n\26\3\26\3\26\3\27\3\27\5\27\u017d\n\27\3"+
-    "\27\2\3\34\30\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,\2\16\4\2"+
-    "\26\26KK\3\3\r\r\3\2\66A\3\2\34\36\3\2\37 \3\2!#\3\2$\'\3\2(+\3\2\64\65"+
-    "\3\2BE\4\2\32\33\37 \3\2LM\u01a5\2\61\3\2\2\2\4\u0084\3\2\2\2\6\u0088"+
-    "\3\2\2\2\b\u008a\3\2\2\2\n\u0093\3\2\2\2\f\u0097\3\2\2\2\16\u0099\3\2"+
-    "\2\2\20\u009b\3\2\2\2\22\u00a4\3\2\2\2\24\u00ac\3\2\2\2\26\u00b0\3\2\2"+
-    "\2\30\u00b5\3\2\2\2\32\u00bc\3\2\2\2\34\u00c7\3\2\2\2\36\u0124\3\2\2\2"+
-    " \u0148\3\2\2\2\"\u015a\3\2\2\2$\u0160\3\2\2\2&\u0167\3\2\2\2(\u0169\3"+
-    "\2\2\2*\u016d\3\2\2\2,\u017c\3\2\2\2.\60\5\4\3\2/.\3\2\2\2\60\63\3\2\2"+
-    "\2\61/\3\2\2\2\61\62\3\2\2\2\62\64\3\2\2\2\63\61\3\2\2\2\64\65\7\2\2\3"+
-    "\65\3\3\2\2\2\66\67\7\16\2\2\678\7\t\2\289\5\34\17\29:\7\n\2\2:>\5\6\4"+
-    "\2;<\7\17\2\2<?\5\6\4\2=?\6\3\2\2>;\3\2\2\2>=\3\2\2\2?\u0085\3\2\2\2@"+
-    "A\7\20\2\2AB\7\t\2\2BC\5\34\17\2CF\7\n\2\2DG\5\6\4\2EG\5\n\6\2FD\3\2\2"+
-    "\2FE\3\2\2\2G\u0085\3\2\2\2HI\7\21\2\2IJ\5\b\5\2JK\7\20\2\2KL\7\t\2\2"+
-    "LM\5\34\17\2MN\7\n\2\2NO\5\32\16\2O\u0085\3\2\2\2PQ\7\22\2\2QS\7\t\2\2"+
-    "RT\5\f\7\2SR\3\2\2\2ST\3\2\2\2TU\3\2\2\2UW\7\r\2\2VX\5\34\17\2WV\3\2\2"+
-    "\2WX\3\2\2\2XY\3\2\2\2Y[\7\r\2\2Z\\\5\16\b\2[Z\3\2\2\2[\\\3\2\2\2\\]\3"+
-    "\2\2\2]`\7\n\2\2^a\5\6\4\2_a\5\n\6\2`^\3\2\2\2`_\3\2\2\2a\u0085\3\2\2"+
-    "\2bc\7\22\2\2cd\7\t\2\2de\5\22\n\2ef\7K\2\2fg\7\62\2\2gh\5\34\17\2hi\7"+
-    "\n\2\2ij\5\6\4\2j\u0085\3\2\2\2kl\5\20\t\2lm\5\32\16\2m\u0085\3\2\2\2"+
-    "no\7\23\2\2o\u0085\5\32\16\2pq\7\24\2\2q\u0085\5\32\16\2rs\7\25\2\2st"+
-    "\5\34\17\2tu\5\32\16\2u\u0085\3\2\2\2vw\7\27\2\2wy\5\b\5\2xz\5\30\r\2"+
-    "yx\3\2\2\2z{\3\2\2\2{y\3\2\2\2{|\3\2\2\2|\u0085\3\2\2\2}~\7\31\2\2~\177"+
-    "\5\34\17\2\177\u0080\5\32\16\2\u0080\u0085\3\2\2\2\u0081\u0082\5\34\17"+
-    "\2\u0082\u0083\5\32\16\2\u0083\u0085\3\2\2\2\u0084\66\3\2\2\2\u0084@\3"+
-    "\2\2\2\u0084H\3\2\2\2\u0084P\3\2\2\2\u0084b\3\2\2\2\u0084k\3\2\2\2\u0084"+
-    "n\3\2\2\2\u0084p\3\2\2\2\u0084r\3\2\2\2\u0084v\3\2\2\2\u0084}\3\2\2\2"+
-    "\u0084\u0081\3\2\2\2\u0085\5\3\2\2\2\u0086\u0089\5\b\5\2\u0087\u0089\5"+
-    "\4\3\2\u0088\u0086\3\2\2\2\u0088\u0087\3\2\2\2\u0089\7\3\2\2\2\u008a\u008e"+
-    "\7\5\2\2\u008b\u008d\5\4\3\2\u008c\u008b\3\2\2\2\u008d\u0090\3\2\2\2\u008e"+
-    "\u008c\3\2\2\2\u008e\u008f\3\2\2\2\u008f\u0091\3\2\2\2\u0090\u008e\3\2"+
-    "\2\2\u0091\u0092\7\6\2\2\u0092\t\3\2\2\2\u0093\u0094\7\r\2\2\u0094\13"+
-    "\3\2\2\2\u0095\u0098\5\20\t\2\u0096\u0098\5\34\17\2\u0097\u0095\3\2\2"+
-    "\2\u0097\u0096\3\2\2\2\u0098\r\3\2\2\2\u0099\u009a\5\34\17\2\u009a\17"+
-    "\3\2\2\2\u009b\u009c\5\22\n\2\u009c\u00a1\5\26\f\2\u009d\u009e\7\f\2\2"+
-    "\u009e\u00a0\5\26\f\2\u009f\u009d\3\2\2\2\u00a0\u00a3\3\2\2\2\u00a1\u009f"+
-    "\3\2\2\2\u00a1\u00a2\3\2\2\2\u00a2\21\3\2\2\2\u00a3\u00a1\3\2\2\2\u00a4"+
-    "\u00a9\7J\2\2\u00a5\u00a6\7\7\2\2\u00a6\u00a8\7\b\2\2\u00a7\u00a5\3\2"+
-    "\2\2\u00a8\u00ab\3\2\2\2\u00a9\u00a7\3\2\2\2\u00a9\u00aa\3\2\2\2\u00aa"+
-    "\23\3\2\2\2\u00ab\u00a9\3\2\2\2\u00ac\u00ad\7J\2\2\u00ad\u00ae\7\63\2"+
-    "\2\u00ae\u00af\t\2\2\2\u00af\25\3\2\2\2\u00b0\u00b3\7K\2\2\u00b1\u00b2"+
-    "\7\66\2\2\u00b2\u00b4\5\34\17\2\u00b3\u00b1\3\2\2\2\u00b3\u00b4\3\2\2"+
-    "\2\u00b4\27\3\2\2\2\u00b5\u00b6\7\30\2\2\u00b6\u00b7\7\t\2\2\u00b7\u00b8"+
-    "\7J\2\2\u00b8\u00b9\7K\2\2\u00b9\u00ba\7\n\2\2\u00ba\u00bb\5\b\5\2\u00bb"+
-    "\31\3\2\2\2\u00bc\u00bd\t\3\2\2\u00bd\33\3\2\2\2\u00be\u00bf\b\17\1\2"+
-    "\u00bf\u00c0\5 \21\2\u00c0\u00c1\t\4\2\2\u00c1\u00c2\5\34\17\3\u00c2\u00c3"+
-    "\b\17\1\2\u00c3\u00c8\3\2\2\2\u00c4\u00c5\5\36\20\2\u00c5\u00c6\b\17\1"+
-    "\2\u00c6\u00c8\3\2\2\2\u00c7\u00be\3\2\2\2\u00c7\u00c4\3\2\2\2\u00c8\u0104"+
-    "\3\2\2\2\u00c9\u00ca\f\16\2\2\u00ca\u00cb\t\5\2\2\u00cb\u00cc\5\34\17"+
-    "\17\u00cc\u00cd\b\17\1\2\u00cd\u0103\3\2\2\2\u00ce\u00cf\f\r\2\2\u00cf"+
-    "\u00d0\t\6\2\2\u00d0\u00d1\5\34\17\16\u00d1\u00d2\b\17\1\2\u00d2\u0103"+
-    "\3\2\2\2\u00d3\u00d4\f\f\2\2\u00d4\u00d5\t\7\2\2\u00d5\u00d6\5\34\17\r"+
-    "\u00d6\u00d7\b\17\1\2\u00d7\u0103\3\2\2\2\u00d8\u00d9\f\13\2\2\u00d9\u00da"+
-    "\t\b\2\2\u00da\u00db\5\34\17\f\u00db\u00dc\b\17\1\2\u00dc\u0103\3\2\2"+
-    "\2\u00dd\u00de\f\n\2\2\u00de\u00df\t\t\2\2\u00df\u00e0\5\34\17\13\u00e0"+
-    "\u00e1\b\17\1\2\u00e1\u0103\3\2\2\2\u00e2\u00e3\f\t\2\2\u00e3\u00e4\7"+
-    ",\2\2\u00e4\u00e5\5\34\17\n\u00e5\u00e6\b\17\1\2\u00e6\u0103\3\2\2\2\u00e7"+
-    "\u00e8\f\b\2\2\u00e8\u00e9\7-\2\2\u00e9\u00ea\5\34\17\t\u00ea\u00eb\b"+
-    "\17\1\2\u00eb\u0103\3\2\2\2\u00ec\u00ed\f\7\2\2\u00ed\u00ee\7.\2\2\u00ee"+
-    "\u00ef\5\34\17\b\u00ef\u00f0\b\17\1\2\u00f0\u0103\3\2\2\2\u00f1\u00f2"+
-    "\f\6\2\2\u00f2\u00f3\7/\2\2\u00f3\u00f4\5\34\17\7\u00f4\u00f5\b\17\1\2"+
-    "\u00f5\u0103\3\2\2\2\u00f6\u00f7\f\5\2\2\u00f7\u00f8\7\60\2\2\u00f8\u00f9"+
-    "\5\34\17\6\u00f9\u00fa\b\17\1\2\u00fa\u0103\3\2\2\2\u00fb\u00fc\f\4\2"+
-    "\2\u00fc\u00fd\7\61\2\2\u00fd\u00fe\5\34\17\2\u00fe\u00ff\7\62\2\2\u00ff"+
-    "\u0100\5\34\17\4\u0100\u0101\b\17\1\2\u0101\u0103\3\2\2\2\u0102\u00c9"+
-    "\3\2\2\2\u0102\u00ce\3\2\2\2\u0102\u00d3\3\2\2\2\u0102\u00d8\3\2\2\2\u0102"+
-    "\u00dd\3\2\2\2\u0102\u00e2\3\2\2\2\u0102\u00e7\3\2\2\2\u0102\u00ec\3\2"+
-    "\2\2\u0102\u00f1\3\2\2\2\u0102\u00f6\3\2\2\2\u0102\u00fb\3\2\2\2\u0103"+
-    "\u0106\3\2\2\2\u0104\u0102\3\2\2\2\u0104\u0105\3\2\2\2\u0105\35\3\2\2"+
-    "\2\u0106\u0104\3\2\2\2\u0107\u0108\6\20\16\3\u0108\u0109\t\n\2\2\u0109"+
-    "\u0125\5 \21\2\u010a\u010b\6\20\17\3\u010b\u010c\5 \21\2\u010c\u010d\t"+
-    "\n\2\2\u010d\u0125\3\2\2\2\u010e\u010f\6\20\20\3\u010f\u0125\5 \21\2\u0110"+
-    "\u0111\6\20\21\3\u0111\u0112\t\13\2\2\u0112\u0125\b\20\1\2\u0113\u0114"+
-    "\6\20\22\3\u0114\u0115\7G\2\2\u0115\u0125\b\20\1\2\u0116\u0117\6\20\23"+
-    "\3\u0117\u0118\7H\2\2\u0118\u0125\b\20\1\2\u0119\u011a\6\20\24\3\u011a"+
-    "\u011b\7I\2\2\u011b\u0125\b\20\1\2\u011c\u011d\6\20\25\3\u011d\u011e\t"+
-    "\f\2\2\u011e\u0125\5\36\20\2\u011f\u0120\7\t\2\2\u0120\u0121\5\22\n\2"+
-    "\u0121\u0122\7\n\2\2\u0122\u0123\5\36\20\2\u0123\u0125\3\2\2\2\u0124\u0107"+
-    "\3\2\2\2\u0124\u010a\3\2\2\2\u0124\u010e\3\2\2\2\u0124\u0110\3\2\2\2\u0124"+
-    "\u0113\3\2\2\2\u0124\u0116\3\2\2\2\u0124\u0119\3\2\2\2\u0124\u011c\3\2"+
-    "\2\2\u0124\u011f\3\2\2\2\u0125\37\3\2\2\2\u0126\u012a\5\"\22\2\u0127\u0129"+
-    "\5$\23\2\u0128\u0127\3\2\2\2\u0129\u012c\3\2\2\2\u012a\u0128\3\2\2\2\u012a"+
-    "\u012b\3\2\2\2\u012b\u0149\3\2\2\2\u012c\u012a\3\2\2\2\u012d\u012e\5\22"+
-    "\n\2\u012e\u0132\5&\24\2\u012f\u0131\5$\23\2\u0130\u012f\3\2\2\2\u0131"+
-    "\u0134\3\2\2\2\u0132\u0130\3\2\2\2\u0132\u0133\3\2\2\2\u0133\u0149\3\2"+
-    "\2\2\u0134\u0132\3\2\2\2\u0135\u0136\7\26\2\2\u0136\u013b\7J\2\2\u0137"+
-    "\u0138\7\7\2\2\u0138\u0139\5\34\17\2\u0139\u013a\7\b\2\2\u013a\u013c\3"+
-    "\2\2\2\u013b\u0137\3\2\2\2\u013c\u013d\3\2\2\2\u013d\u013b\3\2\2\2\u013d"+
-    "\u013e\3\2\2\2\u013e\u0146\3\2\2\2\u013f\u0143\5&\24\2\u0140\u0142\5$"+
-    "\23\2\u0141\u0140\3\2\2\2\u0142\u0145\3\2\2\2\u0143\u0141\3\2\2\2\u0143"+
-    "\u0144\3\2\2\2\u0144\u0147\3\2\2\2\u0145\u0143\3\2\2\2\u0146\u013f\3\2"+
-    "\2\2\u0146\u0147\3\2\2\2\u0147\u0149\3\2\2\2\u0148\u0126\3\2\2\2\u0148"+
-    "\u012d\3\2\2\2\u0148\u0135\3\2\2\2\u0149!\3\2\2\2\u014a\u014b\6\22\26"+
-    "\3\u014b\u014c\7\t\2\2\u014c\u014d\5\34\17\2\u014d\u014e\7\n\2\2\u014e"+
-    "\u014f\b\22\1\2\u014f\u015b\3\2\2\2\u0150\u0151\6\22\27\3\u0151\u0152"+
-    "\7\t\2\2\u0152\u0153\5\36\20\2\u0153\u0154\7\n\2\2\u0154\u015b\3\2\2\2"+
-    "\u0155\u015b\7F\2\2\u0156\u015b\7K\2\2\u0157\u0158\7\26\2\2\u0158\u0159"+
-    "\7J\2\2\u0159\u015b\5*\26\2\u015a\u014a\3\2\2\2\u015a\u0150\3\2\2\2\u015a"+
-    "\u0155\3\2\2\2\u015a\u0156\3\2\2\2\u015a\u0157\3\2\2\2\u015b#\3\2\2\2"+
-    "\u015c\u015d\6\23\30\3\u015d\u0161\5&\24\2\u015e\u015f\6\23\31\3\u015f"+
-    "\u0161\5(\25\2\u0160\u015c\3\2\2\2\u0160\u015e\3\2\2\2\u0161%\3\2\2\2"+
-    "\u0162\u0163\7\13\2\2\u0163\u0164\7M\2\2\u0164\u0168\5*\26\2\u0165\u0166"+
-    "\7\13\2\2\u0166\u0168\t\r\2\2\u0167\u0162\3\2\2\2\u0167\u0165\3\2\2\2"+
-    "\u0168\'\3\2\2\2\u0169\u016a\7\7\2\2\u016a\u016b\5\34\17\2\u016b\u016c"+
-    "\7\b\2\2\u016c)\3\2\2\2\u016d\u0176\7\t\2\2\u016e\u0173\5,\27\2\u016f"+
-    "\u0170\7\f\2\2\u0170\u0172\5,\27\2\u0171\u016f\3\2\2\2\u0172\u0175\3\2"+
-    "\2\2\u0173\u0171\3\2\2\2\u0173\u0174\3\2\2\2\u0174\u0177\3\2\2\2\u0175"+
-    "\u0173\3\2\2\2\u0176\u016e\3\2\2\2\u0176\u0177\3\2\2\2\u0177\u0178\3\2"+
-    "\2\2\u0178\u0179\7\n\2\2\u0179+\3\2\2\2\u017a\u017d\5\34\17\2\u017b\u017d"+
-    "\5\24\13\2\u017c\u017a\3\2\2\2\u017c\u017b\3\2\2\2\u017d-\3\2\2\2!\61"+
-    ">FSW[`{\u0084\u0088\u008e\u0097\u00a1\u00a9\u00b3\u00c7\u0102\u0104\u0124"+
-    "\u012a\u0132\u013d\u0143\u0146\u0148\u015a\u0160\u0167\u0173\u0176\u017c";
+    "\3\20\3\20\5\20\u0129\n\20\3\21\3\21\7\21\u012d\n\21\f\21\16\21\u0130"+
+    "\13\21\3\21\3\21\3\21\7\21\u0135\n\21\f\21\16\21\u0138\13\21\3\21\3\21"+
+    "\3\21\3\21\3\21\3\21\6\21\u0140\n\21\r\21\16\21\u0141\3\21\3\21\7\21\u0146"+
+    "\n\21\f\21\16\21\u0149\13\21\5\21\u014b\n\21\5\21\u014d\n\21\3\22\3\22"+
+    "\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22"+
+    "\5\22\u015f\n\22\3\23\3\23\3\23\3\23\5\23\u0165\n\23\3\24\3\24\3\24\3"+
+    "\24\3\24\5\24\u016c\n\24\3\25\3\25\3\25\3\25\3\26\3\26\3\26\3\26\7\26"+
+    "\u0176\n\26\f\26\16\26\u0179\13\26\5\26\u017b\n\26\3\26\3\26\3\27\3\27"+
+    "\5\27\u0181\n\27\3\27\2\3\34\30\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36"+
+    " \"$&(*,\2\16\4\2\26\26KK\3\3\r\r\3\2\66A\3\2\34\36\3\2\37 \3\2!#\3\2"+
+    "$\'\3\2(+\3\2\64\65\3\2BE\4\2\32\33\37 \3\2LM\u01aa\2\61\3\2\2\2\4\u0084"+
+    "\3\2\2\2\6\u0088\3\2\2\2\b\u008a\3\2\2\2\n\u0093\3\2\2\2\f\u0097\3\2\2"+
+    "\2\16\u0099\3\2\2\2\20\u009b\3\2\2\2\22\u00a4\3\2\2\2\24\u00b2\3\2\2\2"+
+    "\26\u00b4\3\2\2\2\30\u00b9\3\2\2\2\32\u00c0\3\2\2\2\34\u00cb\3\2\2\2\36"+
+    "\u0128\3\2\2\2 \u014c\3\2\2\2\"\u015e\3\2\2\2$\u0164\3\2\2\2&\u016b\3"+
+    "\2\2\2(\u016d\3\2\2\2*\u0171\3\2\2\2,\u0180\3\2\2\2.\60\5\4\3\2/.\3\2"+
+    "\2\2\60\63\3\2\2\2\61/\3\2\2\2\61\62\3\2\2\2\62\64\3\2\2\2\63\61\3\2\2"+
+    "\2\64\65\7\2\2\3\65\3\3\2\2\2\66\67\7\16\2\2\678\7\t\2\289\5\34\17\29"+
+    ":\7\n\2\2:>\5\6\4\2;<\7\17\2\2<?\5\6\4\2=?\6\3\2\2>;\3\2\2\2>=\3\2\2\2"+
+    "?\u0085\3\2\2\2@A\7\20\2\2AB\7\t\2\2BC\5\34\17\2CF\7\n\2\2DG\5\6\4\2E"+
+    "G\5\n\6\2FD\3\2\2\2FE\3\2\2\2G\u0085\3\2\2\2HI\7\21\2\2IJ\5\b\5\2JK\7"+
+    "\20\2\2KL\7\t\2\2LM\5\34\17\2MN\7\n\2\2NO\5\32\16\2O\u0085\3\2\2\2PQ\7"+
+    "\22\2\2QS\7\t\2\2RT\5\f\7\2SR\3\2\2\2ST\3\2\2\2TU\3\2\2\2UW\7\r\2\2VX"+
+    "\5\34\17\2WV\3\2\2\2WX\3\2\2\2XY\3\2\2\2Y[\7\r\2\2Z\\\5\16\b\2[Z\3\2\2"+
+    "\2[\\\3\2\2\2\\]\3\2\2\2]`\7\n\2\2^a\5\6\4\2_a\5\n\6\2`^\3\2\2\2`_\3\2"+
+    "\2\2a\u0085\3\2\2\2bc\7\22\2\2cd\7\t\2\2de\5\22\n\2ef\7K\2\2fg\7\62\2"+
+    "\2gh\5\34\17\2hi\7\n\2\2ij\5\6\4\2j\u0085\3\2\2\2kl\5\20\t\2lm\5\32\16"+
+    "\2m\u0085\3\2\2\2no\7\23\2\2o\u0085\5\32\16\2pq\7\24\2\2q\u0085\5\32\16"+
+    "\2rs\7\25\2\2st\5\34\17\2tu\5\32\16\2u\u0085\3\2\2\2vw\7\27\2\2wy\5\b"+
+    "\5\2xz\5\30\r\2yx\3\2\2\2z{\3\2\2\2{y\3\2\2\2{|\3\2\2\2|\u0085\3\2\2\2"+
+    "}~\7\31\2\2~\177\5\34\17\2\177\u0080\5\32\16\2\u0080\u0085\3\2\2\2\u0081"+
+    "\u0082\5\34\17\2\u0082\u0083\5\32\16\2\u0083\u0085\3\2\2\2\u0084\66\3"+
+    "\2\2\2\u0084@\3\2\2\2\u0084H\3\2\2\2\u0084P\3\2\2\2\u0084b\3\2\2\2\u0084"+
+    "k\3\2\2\2\u0084n\3\2\2\2\u0084p\3\2\2\2\u0084r\3\2\2\2\u0084v\3\2\2\2"+
+    "\u0084}\3\2\2\2\u0084\u0081\3\2\2\2\u0085\5\3\2\2\2\u0086\u0089\5\b\5"+
+    "\2\u0087\u0089\5\4\3\2\u0088\u0086\3\2\2\2\u0088\u0087\3\2\2\2\u0089\7"+
+    "\3\2\2\2\u008a\u008e\7\5\2\2\u008b\u008d\5\4\3\2\u008c\u008b\3\2\2\2\u008d"+
+    "\u0090\3\2\2\2\u008e\u008c\3\2\2\2\u008e\u008f\3\2\2\2\u008f\u0091\3\2"+
+    "\2\2\u0090\u008e\3\2\2\2\u0091\u0092\7\6\2\2\u0092\t\3\2\2\2\u0093\u0094"+
+    "\7\r\2\2\u0094\13\3\2\2\2\u0095\u0098\5\20\t\2\u0096\u0098\5\34\17\2\u0097"+
+    "\u0095\3\2\2\2\u0097\u0096\3\2\2\2\u0098\r\3\2\2\2\u0099\u009a\5\34\17"+
+    "\2\u009a\17\3\2\2\2\u009b\u009c\5\22\n\2\u009c\u00a1\5\26\f\2\u009d\u009e"+
+    "\7\f\2\2\u009e\u00a0\5\26\f\2\u009f\u009d\3\2\2\2\u00a0\u00a3\3\2\2\2"+
+    "\u00a1\u009f\3\2\2\2\u00a1\u00a2\3\2\2\2\u00a2\21\3\2\2\2\u00a3\u00a1"+
+    "\3\2\2\2\u00a4\u00a9\7J\2\2\u00a5\u00a6\7\7\2\2\u00a6\u00a8\7\b\2\2\u00a7"+
+    "\u00a5\3\2\2\2\u00a8\u00ab\3\2\2\2\u00a9\u00a7\3\2\2\2\u00a9\u00aa\3\2"+
+    "\2\2\u00aa\23\3\2\2\2\u00ab\u00a9\3\2\2\2\u00ac\u00ad\7J\2\2\u00ad\u00ae"+
+    "\7\63\2\2\u00ae\u00b3\t\2\2\2\u00af\u00b0\7K\2\2\u00b0\u00b1\7\63\2\2"+
+    "\u00b1\u00b3\7K\2\2\u00b2\u00ac\3\2\2\2\u00b2\u00af\3\2\2\2\u00b3\25\3"+
+    "\2\2\2\u00b4\u00b7\7K\2\2\u00b5\u00b6\7\66\2\2\u00b6\u00b8\5\34\17\2\u00b7"+
+    "\u00b5\3\2\2\2\u00b7\u00b8\3\2\2\2\u00b8\27\3\2\2\2\u00b9\u00ba\7\30\2"+
+    "\2\u00ba\u00bb\7\t\2\2\u00bb\u00bc\7J\2\2\u00bc\u00bd\7K\2\2\u00bd\u00be"+
+    "\7\n\2\2\u00be\u00bf\5\b\5\2\u00bf\31\3\2\2\2\u00c0\u00c1\t\3\2\2\u00c1"+
+    "\33\3\2\2\2\u00c2\u00c3\b\17\1\2\u00c3\u00c4\5 \21\2\u00c4\u00c5\t\4\2"+
+    "\2\u00c5\u00c6\5\34\17\3\u00c6\u00c7\b\17\1\2\u00c7\u00cc\3\2\2\2\u00c8"+
+    "\u00c9\5\36\20\2\u00c9\u00ca\b\17\1\2\u00ca\u00cc\3\2\2\2\u00cb\u00c2"+
+    "\3\2\2\2\u00cb\u00c8\3\2\2\2\u00cc\u0108\3\2\2\2\u00cd\u00ce\f\16\2\2"+
+    "\u00ce\u00cf\t\5\2\2\u00cf\u00d0\5\34\17\17\u00d0\u00d1\b\17\1\2\u00d1"+
+    "\u0107\3\2\2\2\u00d2\u00d3\f\r\2\2\u00d3\u00d4\t\6\2\2\u00d4\u00d5\5\34"+
+    "\17\16\u00d5\u00d6\b\17\1\2\u00d6\u0107\3\2\2\2\u00d7\u00d8\f\f\2\2\u00d8"+
+    "\u00d9\t\7\2\2\u00d9\u00da\5\34\17\r\u00da\u00db\b\17\1\2\u00db\u0107"+
+    "\3\2\2\2\u00dc\u00dd\f\13\2\2\u00dd\u00de\t\b\2\2\u00de\u00df\5\34\17"+
+    "\f\u00df\u00e0\b\17\1\2\u00e0\u0107\3\2\2\2\u00e1\u00e2\f\n\2\2\u00e2"+
+    "\u00e3\t\t\2\2\u00e3\u00e4\5\34\17\13\u00e4\u00e5\b\17\1\2\u00e5\u0107"+
+    "\3\2\2\2\u00e6\u00e7\f\t\2\2\u00e7\u00e8\7,\2\2\u00e8\u00e9\5\34\17\n"+
+    "\u00e9\u00ea\b\17\1\2\u00ea\u0107\3\2\2\2\u00eb\u00ec\f\b\2\2\u00ec\u00ed"+
+    "\7-\2\2\u00ed\u00ee\5\34\17\t\u00ee\u00ef\b\17\1\2\u00ef\u0107\3\2\2\2"+
+    "\u00f0\u00f1\f\7\2\2\u00f1\u00f2\7.\2\2\u00f2\u00f3\5\34\17\b\u00f3\u00f4"+
+    "\b\17\1\2\u00f4\u0107\3\2\2\2\u00f5\u00f6\f\6\2\2\u00f6\u00f7\7/\2\2\u00f7"+
+    "\u00f8\5\34\17\7\u00f8\u00f9\b\17\1\2\u00f9\u0107\3\2\2\2\u00fa\u00fb"+
+    "\f\5\2\2\u00fb\u00fc\7\60\2\2\u00fc\u00fd\5\34\17\6\u00fd\u00fe\b\17\1"+
+    "\2\u00fe\u0107\3\2\2\2\u00ff\u0100\f\4\2\2\u0100\u0101\7\61\2\2\u0101"+
+    "\u0102\5\34\17\2\u0102\u0103\7\62\2\2\u0103\u0104\5\34\17\4\u0104\u0105"+
+    "\b\17\1\2\u0105\u0107\3\2\2\2\u0106\u00cd\3\2\2\2\u0106\u00d2\3\2\2\2"+
+    "\u0106\u00d7\3\2\2\2\u0106\u00dc\3\2\2\2\u0106\u00e1\3\2\2\2\u0106\u00e6"+
+    "\3\2\2\2\u0106\u00eb\3\2\2\2\u0106\u00f0\3\2\2\2\u0106\u00f5\3\2\2\2\u0106"+
+    "\u00fa\3\2\2\2\u0106\u00ff\3\2\2\2\u0107\u010a\3\2\2\2\u0108\u0106\3\2"+
+    "\2\2\u0108\u0109\3\2\2\2\u0109\35\3\2\2\2\u010a\u0108\3\2\2\2\u010b\u010c"+
+    "\6\20\16\3\u010c\u010d\t\n\2\2\u010d\u0129\5 \21\2\u010e\u010f\6\20\17"+
+    "\3\u010f\u0110\5 \21\2\u0110\u0111\t\n\2\2\u0111\u0129\3\2\2\2\u0112\u0113"+
+    "\6\20\20\3\u0113\u0129\5 \21\2\u0114\u0115\6\20\21\3\u0115\u0116\t\13"+
+    "\2\2\u0116\u0129\b\20\1\2\u0117\u0118\6\20\22\3\u0118\u0119\7G\2\2\u0119"+
+    "\u0129\b\20\1\2\u011a\u011b\6\20\23\3\u011b\u011c\7H\2\2\u011c\u0129\b"+
+    "\20\1\2\u011d\u011e\6\20\24\3\u011e\u011f\7I\2\2\u011f\u0129\b\20\1\2"+
+    "\u0120\u0121\6\20\25\3\u0121\u0122\t\f\2\2\u0122\u0129\5\36\20\2\u0123"+
+    "\u0124\7\t\2\2\u0124\u0125\5\22\n\2\u0125\u0126\7\n\2\2\u0126\u0127\5"+
+    "\36\20\2\u0127\u0129\3\2\2\2\u0128\u010b\3\2\2\2\u0128\u010e\3\2\2\2\u0128"+
+    "\u0112\3\2\2\2\u0128\u0114\3\2\2\2\u0128\u0117\3\2\2\2\u0128\u011a\3\2"+
+    "\2\2\u0128\u011d\3\2\2\2\u0128\u0120\3\2\2\2\u0128\u0123\3\2\2\2\u0129"+
+    "\37\3\2\2\2\u012a\u012e\5\"\22\2\u012b\u012d\5$\23\2\u012c\u012b\3\2\2"+
+    "\2\u012d\u0130\3\2\2\2\u012e\u012c\3\2\2\2\u012e\u012f\3\2\2\2\u012f\u014d"+
+    "\3\2\2\2\u0130\u012e\3\2\2\2\u0131\u0132\5\22\n\2\u0132\u0136\5&\24\2"+
+    "\u0133\u0135\5$\23\2\u0134\u0133\3\2\2\2\u0135\u0138\3\2\2\2\u0136\u0134"+
+    "\3\2\2\2\u0136\u0137\3\2\2\2\u0137\u014d\3\2\2\2\u0138\u0136\3\2\2\2\u0139"+
+    "\u013a\7\26\2\2\u013a\u013f\7J\2\2\u013b\u013c\7\7\2\2\u013c\u013d\5\34"+
+    "\17\2\u013d\u013e\7\b\2\2\u013e\u0140\3\2\2\2\u013f\u013b\3\2\2\2\u0140"+
+    "\u0141\3\2\2\2\u0141\u013f\3\2\2\2\u0141\u0142\3\2\2\2\u0142\u014a\3\2"+
+    "\2\2\u0143\u0147\5&\24\2\u0144\u0146\5$\23\2\u0145\u0144\3\2\2\2\u0146"+
+    "\u0149\3\2\2\2\u0147\u0145\3\2\2\2\u0147\u0148\3\2\2\2\u0148\u014b\3\2"+
+    "\2\2\u0149\u0147\3\2\2\2\u014a\u0143\3\2\2\2\u014a\u014b\3\2\2\2\u014b"+
+    "\u014d\3\2\2\2\u014c\u012a\3\2\2\2\u014c\u0131\3\2\2\2\u014c\u0139\3\2"+
+    "\2\2\u014d!\3\2\2\2\u014e\u014f\6\22\26\3\u014f\u0150\7\t\2\2\u0150\u0151"+
+    "\5\34\17\2\u0151\u0152\7\n\2\2\u0152\u0153\b\22\1\2\u0153\u015f\3\2\2"+
+    "\2\u0154\u0155\6\22\27\3\u0155\u0156\7\t\2\2\u0156\u0157\5\36\20\2\u0157"+
+    "\u0158\7\n\2\2\u0158\u015f\3\2\2\2\u0159\u015f\7F\2\2\u015a\u015f\7K\2"+
+    "\2\u015b\u015c\7\26\2\2\u015c\u015d\7J\2\2\u015d\u015f\5*\26\2\u015e\u014e"+
+    "\3\2\2\2\u015e\u0154\3\2\2\2\u015e\u0159\3\2\2\2\u015e\u015a\3\2\2\2\u015e"+
+    "\u015b\3\2\2\2\u015f#\3\2\2\2\u0160\u0161\6\23\30\3\u0161\u0165\5&\24"+
+    "\2\u0162\u0163\6\23\31\3\u0163\u0165\5(\25\2\u0164\u0160\3\2\2\2\u0164"+
+    "\u0162\3\2\2\2\u0165%\3\2\2\2\u0166\u0167\7\13\2\2\u0167\u0168\7M\2\2"+
+    "\u0168\u016c\5*\26\2\u0169\u016a\7\13\2\2\u016a\u016c\t\r\2\2\u016b\u0166"+
+    "\3\2\2\2\u016b\u0169\3\2\2\2\u016c\'\3\2\2\2\u016d\u016e\7\7\2\2\u016e"+
+    "\u016f\5\34\17\2\u016f\u0170\7\b\2\2\u0170)\3\2\2\2\u0171\u017a\7\t\2"+
+    "\2\u0172\u0177\5,\27\2\u0173\u0174\7\f\2\2\u0174\u0176\5,\27\2\u0175\u0173"+
+    "\3\2\2\2\u0176\u0179\3\2\2\2\u0177\u0175\3\2\2\2\u0177\u0178\3\2\2\2\u0178"+
+    "\u017b\3\2\2\2\u0179\u0177\3\2\2\2\u017a\u0172\3\2\2\2\u017a\u017b\3\2"+
+    "\2\2\u017b\u017c\3\2\2\2\u017c\u017d\7\n\2\2\u017d+\3\2\2\2\u017e\u0181"+
+    "\5\34\17\2\u017f\u0181\5\24\13\2\u0180\u017e\3\2\2\2\u0180\u017f\3\2\2"+
+    "\2\u0181-\3\2\2\2\"\61>FSW[`{\u0084\u0088\u008e\u0097\u00a1\u00a9\u00b2"+
+    "\u00b7\u00cb\u0106\u0108\u0128\u012e\u0136\u0141\u0147\u014a\u014c\u015e"+
+    "\u0164\u016b\u0177\u017a\u0180";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index 471cb7d64ea..82ccf4083f1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -94,6 +94,7 @@ import org.elasticsearch.painless.node.AStatement;
 import org.elasticsearch.painless.node.EBinary;
 import org.elasticsearch.painless.node.EBool;
 import org.elasticsearch.painless.node.EBoolean;
+import org.elasticsearch.painless.node.ECapturingFunctionRef;
 import org.elasticsearch.painless.node.EChain;
 import org.elasticsearch.painless.node.EComp;
 import org.elasticsearch.painless.node.EConditional;
@@ -405,15 +406,19 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
 
     @Override
     public Object visitFuncref(FuncrefContext ctx) {
-        final String methodText;
-        if (ctx.ID() != null) {
-            methodText = ctx.ID().getText();
-        } else if (ctx.NEW() != null ){
-            methodText = ctx.NEW().getText();
+        if (ctx.TYPE() != null) {
+            // non-capturing Type::method or Type::new
+            final String methodText;
+            if (ctx.NEW() != null) {
+                methodText = ctx.NEW().getText();
+            } else {
+                methodText = ctx.ID(0).getText();
+            }
+            return new EFunctionRef(location(ctx), ctx.TYPE().getText(), methodText);
         } else {
-            throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
+            // capturing object::method
+            return new ECapturingFunctionRef(location(ctx), ctx.ID(0).getText(), ctx.ID(1).getText());
         }
-        return new EFunctionRef(location(ctx), ctx.TYPE().getText(), methodText);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
new file mode 100644
index 00000000000..d6c16b6ffc2
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.DefBootstrap;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.FunctionRef;
+import org.elasticsearch.painless.Location;
+import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Variables.Variable;
+import org.objectweb.asm.Type;
+
+import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
+import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE;
+
+import java.lang.invoke.LambdaMetafactory;
+
+/**
+ * Represents a capturing function reference.
+ */
+public class ECapturingFunctionRef extends AExpression {
+    public final String type;
+    public final String call;
+    
+    private FunctionRef ref;
+    Variable captured;
+    private boolean defInterface;
+
+    public ECapturingFunctionRef(Location location, String type, String call) {
+        super(location);
+
+        this.type = type;
+        this.call = call;
+    }
+
+    @Override
+    void analyze(Variables variables) {
+        captured = variables.getVariable(location, type);
+        if (expected == null) {
+            defInterface = true;
+            actual = Definition.getType("String");
+        } else {
+            defInterface = false;
+            // static case
+            if (captured.type.sort != Definition.Sort.DEF) {
+                try {
+                    ref = new FunctionRef(expected, captured.type.name, call, captured.type.clazz);
+                } catch (IllegalArgumentException e) {
+                    throw createError(e);
+                }
+            }
+            actual = expected;
+        }
+    }
+
+    @Override
+    void write(MethodWriter writer) {
+        writer.writeDebugInfo(location);
+        if (defInterface && captured.type.sort == Definition.Sort.DEF) {
+            // dynamic interface, dynamic implementation
+            writer.push("D" + type + "." + call + ",1");
+            writer.loadLocal(captured.slot);
+        } else if (defInterface) {
+            // dynamic interface, typed implementation
+            writer.push("S" + captured.type.name + "." + call + ",1");
+            writer.loadLocal(captured.slot);
+        } else if (ref == null) {
+            // typed interface, dynamic implementation
+            writer.loadLocal(captured.slot);
+            String descriptor = Type.getMethodType(expected.type, captured.type.type).getDescriptor();
+            writer.invokeDynamic(call, descriptor, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.REFERENCE, expected.name);
+        } else {
+            // typed interface, typed implementation
+            writer.loadLocal(captured.slot);
+            // convert MethodTypes to asm Type for the constant pool.
+            String invokedType = ref.invokedType.toMethodDescriptorString();
+            Type samMethodType = Type.getMethodType(ref.samMethodType.toMethodDescriptorString());
+            Type interfaceType = Type.getMethodType(ref.interfaceMethodType.toMethodDescriptorString());
+            if (ref.needsBridges()) {
+                writer.invokeDynamic(ref.invokedName, 
+                                     invokedType, 
+                                     LAMBDA_BOOTSTRAP_HANDLE, 
+                                     samMethodType, 
+                                     ref.implMethodASM, 
+                                     samMethodType, 
+                                     LambdaMetafactory.FLAG_BRIDGES, 
+                                     1, 
+                                     interfaceType);
+            } else {
+                writer.invokeDynamic(ref.invokedName, 
+                                     invokedType, 
+                                     LAMBDA_BOOTSTRAP_HANDLE, 
+                                     samMethodType, 
+                                     ref.implMethodASM, 
+                                     samMethodType, 
+                                     0);
+            }
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index 38b7a1b8ddb..26be001b211 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -64,7 +64,7 @@ public class EFunctionRef extends AExpression {
     @Override
     void write(MethodWriter writer) {
         if (ref == null) {
-            writer.push(type + "." + call);
+            writer.push("S" + type + "." + call + ",0");
         } else {
             writer.writeDebugInfo(location);
             // convert MethodTypes to asm Type for the constant pool.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index 5e28cd31b98..40e95879634 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -62,7 +62,7 @@ final class LDefArray extends ALink implements IDefLink {
         writer.writeDebugInfo(location);
 
         String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type, index.actual.type);
-        writer.invokeDynamic("arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ARRAY_LOAD, 0);
+        writer.invokeDynamic("arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ARRAY_LOAD);
     }
 
     @Override
@@ -70,6 +70,6 @@ final class LDefArray extends ALink implements IDefLink {
         writer.writeDebugInfo(location);
 
         String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, index.actual.type, after.type);
-        writer.invokeDynamic("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ARRAY_STORE, 0);
+        writer.invokeDynamic("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ARRAY_STORE);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index 28c57744860..3a06b9c899e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -55,11 +55,15 @@ final class LDefCall extends ALink implements IDefLink {
         }
         
         recipe = 0;
+        int totalCaptures = 0;
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
 
             if (expression instanceof EFunctionRef) {
-                recipe |= (1L << argument); // mark argument as deferred reference
+                recipe |= (1L << (argument + totalCaptures)); // mark argument as deferred reference
+            } else if (expression instanceof ECapturingFunctionRef) {
+                recipe |= (1L << (argument + totalCaptures)); // mark argument as deferred reference
+                totalCaptures++;
             }
             expression.internal = true;
             expression.analyze(variables);
@@ -90,6 +94,10 @@ final class LDefCall extends ALink implements IDefLink {
 
         for (AExpression argument : arguments) {
             signature.append(argument.actual.type.getDescriptor());
+            if (argument instanceof ECapturingFunctionRef) {
+                ECapturingFunctionRef capturingRef = (ECapturingFunctionRef) argument;
+                signature.append(capturingRef.captured.type.type.getDescriptor());
+            }
             argument.write(writer);
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index ec9d5bbe9e7..aad11d2e91a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -59,7 +59,7 @@ final class LDefField extends ALink implements IDefLink {
         writer.writeDebugInfo(location);
 
         String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type);
-        writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.LOAD, 0);
+        writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.LOAD);
     }
 
     @Override
@@ -67,6 +67,6 @@ final class LDefField extends ALink implements IDefLink {
         writer.writeDebugInfo(location);
 
         String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, after.type);
-        writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.STORE, 0);
+        writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.STORE);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index 9a6fe798957..2a8fb0e4063 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -197,7 +197,7 @@ public class SEach extends AStatement {
         if (method == null) {
             Type itr = Definition.getType("Iterator");
             String desc = org.objectweb.asm.Type.getMethodDescriptor(itr.type, Definition.DEF_TYPE.type);
-            writer.invokeDynamic("iterator", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ITERATOR, 0);
+            writer.invokeDynamic("iterator", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ITERATOR);
         } else if (java.lang.reflect.Modifier.isInterface(method.owner.clazz.getModifiers())) {
             writer.invokeInterface(method.owner.type, method.method);
         } else {
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt
index 7fff9ac0aa4..308f2661ebc 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.txt
@@ -125,4 +125,5 @@ class org.elasticsearch.painless.FeatureTest -> org.elasticsearch.painless.Featu
   void setY(int)
   boolean overloadedStatic()
   boolean overloadedStatic(boolean)
+  Object twoFunctionsOfX(Function,Function)
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
index 7a55844a224..4330c613e14 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
@@ -33,7 +33,7 @@ public class DefBootstrapTests extends ESTestCase {
         CallSite site = DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                   "toString", 
                                                   MethodType.methodType(String.class, Object.class), 
-                                                  DefBootstrap.METHOD_CALL, 0);
+                                                  DefBootstrap.METHOD_CALL, 0L);
         MethodHandle handle = site.dynamicInvoker();
         assertDepthEquals(site, 0);
 
@@ -50,7 +50,7 @@ public class DefBootstrapTests extends ESTestCase {
         CallSite site = DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                   "toString", 
                                                   MethodType.methodType(String.class, Object.class), 
-                                                  DefBootstrap.METHOD_CALL, 0);
+                                                  DefBootstrap.METHOD_CALL, 0L);
         MethodHandle handle = site.dynamicInvoker();
         assertDepthEquals(site, 0);
 
@@ -72,7 +72,7 @@ public class DefBootstrapTests extends ESTestCase {
         CallSite site = DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                   "toString", 
                                                   MethodType.methodType(String.class, Object.class), 
-                                                  DefBootstrap.METHOD_CALL, 0);
+                                                  DefBootstrap.METHOD_CALL, 0L);
         MethodHandle handle = site.dynamicInvoker();
         assertDepthEquals(site, 0);
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
index 12265f161ab..d3055ac228b 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.painless;
 
+import java.util.Comparator;
+
 public class FunctionRefTests extends ScriptTestCase {
 
     public void testStaticMethodReference() {
@@ -39,12 +41,12 @@ public class FunctionRefTests extends ScriptTestCase {
 
     public void testCtorMethodReference() {
         assertEquals(3.0D, 
-            exec("List l = new ArrayList(); l.add(1.0); l.add(2.0); " + 
-                 "DoubleStream doubleStream = l.stream().mapToDouble(Double::doubleValue);" + 
-                 "DoubleSummaryStatistics stats = doubleStream.collect(DoubleSummaryStatistics::new, " +
-                                                                      "DoubleSummaryStatistics::accept, " +
-                                                                      "DoubleSummaryStatistics::combine); " + 
-                 "return stats.getSum()"));
+                exec("List l = new ArrayList(); l.add(1.0); l.add(2.0); " + 
+                        "DoubleStream doubleStream = l.stream().mapToDouble(Double::doubleValue);" + 
+                        "DoubleSummaryStatistics stats = doubleStream.collect(DoubleSummaryStatistics::new, " +
+                        "DoubleSummaryStatistics::accept, " +
+                        "DoubleSummaryStatistics::combine); " + 
+                        "return stats.getSum()"));
     }
     
     public void testCtorMethodReferenceDef() {
@@ -56,6 +58,58 @@ public class FunctionRefTests extends ScriptTestCase {
                                                   "DoubleSummaryStatistics::combine); " + 
                  "return stats.getSum()"));
     }
+    
+    public void testCapturingMethodReference() {
+        assertEquals("5", exec("Integer x = Integer.valueOf(5); return Optional.empty().orElseGet(x::toString);"));
+        assertEquals("[]", exec("List l = new ArrayList(); return Optional.empty().orElseGet(l::toString);"));
+    }
+    
+    public void testCapturingMethodReferenceDefImpl() {
+        assertEquals("5", exec("def x = Integer.valueOf(5); return Optional.empty().orElseGet(x::toString);"));
+        assertEquals("[]", exec("def l = new ArrayList(); return Optional.empty().orElseGet(l::toString);"));
+    }
+    
+    public void testCapturingMethodReferenceDefInterface() {
+        assertEquals("5", exec("Integer x = Integer.valueOf(5); def opt = Optional.empty(); return opt.orElseGet(x::toString);"));
+        assertEquals("[]", exec("List l = new ArrayList(); def opt = Optional.empty(); return opt.orElseGet(l::toString);"));
+    }
+    
+    public void testCapturingMethodReferenceDefEverywhere() {
+        assertEquals("5", exec("def x = Integer.valueOf(5); def opt = Optional.empty(); return opt.orElseGet(x::toString);"));
+        assertEquals("[]", exec("def l = new ArrayList(); def opt = Optional.empty(); return opt.orElseGet(l::toString);"));
+    }
+    
+    public void testCapturingMethodReferenceMultipleLambdas() {
+        assertEquals("testingcdefg", exec(
+                "String x = 'testing';" +
+                "String y = 'abcdefg';" + 
+                "org.elasticsearch.painless.FeatureTest test = new org.elasticsearch.painless.FeatureTest(2,3);" + 
+                "return test.twoFunctionsOfX(x::concat, y::substring);"));
+    }
+    
+    public void testCapturingMethodReferenceMultipleLambdasDefImpls() {
+        assertEquals("testingcdefg", exec(
+                "def x = 'testing';" +
+                "def y = 'abcdefg';" + 
+                "org.elasticsearch.painless.FeatureTest test = new org.elasticsearch.painless.FeatureTest(2,3);" + 
+                "return test.twoFunctionsOfX(x::concat, y::substring);"));
+    }
+    
+    public void testCapturingMethodReferenceMultipleLambdasDefInterface() {
+        assertEquals("testingcdefg", exec(
+                "String x = 'testing';" +
+                "String y = 'abcdefg';" + 
+                "def test = new org.elasticsearch.painless.FeatureTest(2,3);" + 
+                "return test.twoFunctionsOfX(x::concat, y::substring);"));
+    }
+    
+    public void testCapturingMethodReferenceMultipleLambdasDefEverywhere() {
+        assertEquals("testingcdefg", exec(
+                "def x = 'testing';" +
+                "def y = 'abcdefg';" + 
+                "def test = new org.elasticsearch.painless.FeatureTest(2,3);" + 
+                "return test.twoFunctionsOfX(x::concat, y::substring);"));
+    }
 
     public void testMethodMissing() {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {

From 86f1bedaab683d77b86daa92d7ddde5aee4cedd7 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 10 Jun 2016 15:25:53 -0400
Subject: [PATCH 0782/1311] Rename NettyTransportChannel#close

This commit renames the NettyTransportChannel#close method to
NettyTransportChannel#release to clarify the semantics.
---
 .../transport/netty/NettyTransportChannel.java | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java
index 91b6bc120ad..03856017c36 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java
@@ -42,9 +42,6 @@ import org.jboss.netty.channel.ChannelFutureListener;
 import java.io.IOException;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-/**
- *
- */
 public class NettyTransportChannel implements TransportChannel {
 
     private final NettyTransport transport;
@@ -55,7 +52,7 @@ public class NettyTransportChannel implements TransportChannel {
     private final long requestId;
     private final String profileName;
     private final long reservedBytes;
-    private final AtomicBoolean closed = new AtomicBoolean();
+    private final AtomicBoolean released = new AtomicBoolean();
 
     public NettyTransportChannel(NettyTransport transport, TransportServiceAdapter transportServiceAdapter, String action, Channel channel,
                                  long requestId, Version version, String profileName, long reservedBytes) {
@@ -86,7 +83,7 @@ public class NettyTransportChannel implements TransportChannel {
 
     @Override
     public void sendResponse(TransportResponse response, TransportResponseOptions options) throws IOException {
-        close();
+        release();
         if (transport.compress) {
             options = TransportResponseOptions.builder(options).withCompress(transport.compress).build();
         }
@@ -128,7 +125,7 @@ public class NettyTransportChannel implements TransportChannel {
 
     @Override
     public void sendResponse(Throwable error) throws IOException {
-        close();
+        release();
         BytesStreamOutput stream = new BytesStreamOutput();
         stream.skip(NettyHeader.HEADER_SIZE);
         RemoteTransportException tx = new RemoteTransportException(
@@ -147,10 +144,10 @@ public class NettyTransportChannel implements TransportChannel {
         future.addListener(onResponseSentListener);
     }
 
-    private void close() {
-        // attempt to close once atomically
-        if (closed.compareAndSet(false, true) == false) {
-            throw new IllegalStateException("Channel is already closed");
+    private void release() {
+        // attempt to release once atomically
+        if (released.compareAndSet(false, true) == false) {
+            throw new IllegalStateException("reserved bytes are already released");
         }
         transport.inFlightRequestsBreaker().addWithoutBreaking(-reservedBytes);
     }
@@ -174,4 +171,5 @@ public class NettyTransportChannel implements TransportChannel {
     public Channel getChannel() {
         return channel;
     }
+
 }

From f98be6fa062795a5421102d90f3dd4cff58bb3a1 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Fri, 10 Jun 2016 13:59:59 -0700
Subject: [PATCH 0783/1311] Add functions to Painless.

Closes #18810
---
 .../src/main/antlr/PainlessParser.g4          |   11 +-
 .../org/elasticsearch/painless/Analyzer.java  |   37 -
 .../org/elasticsearch/painless/Compiler.java  |   26 +-
 .../elasticsearch/painless/Definition.java    |   30 +-
 .../painless/{Variables.java => Locals.java}  |  175 ++-
 .../elasticsearch/painless/MethodWriter.java  |    8 +-
 .../org/elasticsearch/painless/Writer.java    |  144 ---
 .../painless/WriterConstants.java             |    8 +-
 .../painless/antlr/PainlessParser.java        | 1072 ++++++++++-------
 .../antlr/PainlessParserBaseVisitor.java      |   21 +
 .../painless/antlr/PainlessParserVisitor.java |   19 +
 .../elasticsearch/painless/antlr/Walker.java  |  111 +-
 .../painless/node/AExpression.java            |   12 +-
 .../elasticsearch/painless/node/ALink.java    |    4 +-
 .../painless/node/AStatement.java             |    4 +-
 .../elasticsearch/painless/node/EBinary.java  |   48 +-
 .../elasticsearch/painless/node/EBool.java    |   12 +-
 .../elasticsearch/painless/node/EBoolean.java |    4 +-
 .../elasticsearch/painless/node/ECast.java    |    4 +-
 .../elasticsearch/painless/node/EChain.java   |   16 +-
 .../elasticsearch/painless/node/EComp.java    |   36 +-
 .../painless/node/EConditional.java           |   16 +-
 .../painless/node/EConstant.java              |    4 +-
 .../elasticsearch/painless/node/EDecimal.java |    4 +-
 .../painless/node/EExplicit.java              |   12 +-
 .../painless/node/EFunctionRef.java           |   34 +-
 .../elasticsearch/painless/node/ENull.java    |    4 +-
 .../elasticsearch/painless/node/ENumeric.java |    4 +-
 .../elasticsearch/painless/node/EUnary.java   |   20 +-
 .../painless/node/LArrayLength.java           |    4 +-
 .../elasticsearch/painless/node/LBrace.java   |   14 +-
 .../node/{LCall.java => LCallInvoke.java}     |   26 +-
 .../painless/node/LCallLocal.java             |   99 ++
 .../elasticsearch/painless/node/LCast.java    |    4 +-
 .../painless/node/LDefArray.java              |    8 +-
 .../elasticsearch/painless/node/LDefCall.java |   10 +-
 .../painless/node/LDefField.java              |    4 +-
 .../elasticsearch/painless/node/LField.java   |   16 +-
 .../painless/node/LListShortcut.java          |    8 +-
 .../painless/node/LMapShortcut.java           |    8 +-
 .../painless/node/LNewArray.java              |    8 +-
 .../elasticsearch/painless/node/LNewObj.java  |    8 +-
 .../painless/node/LShortcut.java              |    4 +-
 .../elasticsearch/painless/node/LStatic.java  |    4 +-
 .../elasticsearch/painless/node/LString.java  |    4 +-
 .../painless/node/LVariable.java              |    8 +-
 .../elasticsearch/painless/node/SBlock.java   |    6 +-
 .../elasticsearch/painless/node/SBreak.java   |    4 +-
 .../elasticsearch/painless/node/SCatch.java   |   10 +-
 .../painless/node/SContinue.java              |    4 +-
 .../painless/node/SDeclBlock.java             |    6 +-
 .../painless/node/SDeclaration.java           |   12 +-
 .../org/elasticsearch/painless/node/SDo.java  |   22 +-
 .../elasticsearch/painless/node/SEach.java    |   44 +-
 .../painless/node/SExpression.java            |   18 +-
 .../org/elasticsearch/painless/node/SFor.java |   29 +-
 .../painless/node/SFunction.java              |  163 +++
 .../org/elasticsearch/painless/node/SIf.java  |   14 +-
 .../elasticsearch/painless/node/SIfElse.java  |   20 +-
 .../elasticsearch/painless/node/SReturn.java  |   11 +-
 .../elasticsearch/painless/node/SSource.java  |  151 ++-
 .../elasticsearch/painless/node/SThrow.java   |    8 +-
 .../org/elasticsearch/painless/node/STry.java |   16 +-
 .../elasticsearch/painless/node/SWhile.java   |   22 +-
 .../painless/node/package-info.java           |    4 +-
 .../elasticsearch/painless/FunctionTests.java |   69 ++
 .../painless/ReservedWordTests.java           |    8 +-
 67 files changed, 1723 insertions(+), 1055 deletions(-)
 delete mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
 rename modules/lang-painless/src/main/java/org/elasticsearch/painless/{Variables.java => Locals.java} (52%)
 delete mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
 rename modules/lang-painless/src/main/java/org/elasticsearch/painless/node/{LCall.java => LCallInvoke.java} (79%)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallLocal.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
 create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionTests.java

diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index 141ff25e318..31b6a9445ea 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -22,7 +22,15 @@ parser grammar PainlessParser;
 options { tokenVocab=PainlessLexer; }
 
 source
-    : statement* EOF
+    : function* statement* EOF
+    ;
+
+function
+    : decltype ID parameters block
+    ;
+
+parameters
+    : LP ( decltype ID ( COMMA decltype ID )* )? RP
     ;
 
 // Note we use a predicate on the if/else case here to prevent the
@@ -143,6 +151,7 @@ primary[boolean c] returns [boolean s = true]
     | { $c }?  LP unary[true] RP                   # chainprec
     |          STRING                              # string
     |          ID                                  # variable
+    |          ID arguments                        # calllocal
     |          NEW TYPE arguments                  # newobject
     ;
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
deleted file mode 100644
index a06bcdf9840..00000000000
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Analyzer.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless;
-
-import org.elasticsearch.painless.Variables.Reserved;
-import org.elasticsearch.painless.node.SSource;
-
-/**
- * Runs the analysis phase of compilation using the Painless AST.
- */
-final class Analyzer {
-    static Variables analyze(Reserved shortcut, SSource root) {
-        Variables variables = new Variables(shortcut);
-        root.analyze(variables);
-
-        return variables;
-    }
-
-    private Analyzer() {}
-}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
index d0a17f64a48..8fa9f5d583e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.painless;
 
 import org.elasticsearch.bootstrap.BootstrapInfo;
-import org.elasticsearch.painless.Variables.Reserved;
 import org.elasticsearch.painless.antlr.Walker;
 import org.elasticsearch.painless.node.SSource;
 
@@ -36,8 +35,7 @@ import static org.elasticsearch.painless.WriterConstants.CLASS_NAME;
 /**
  * The Compiler is the entry point for generating a Painless script.  The compiler will receive a Painless
  * tree based on the type of input passed in (currently only ANTLR).  Two passes will then be run over the tree,
- * one for analysis using the {@link Analyzer} and another to generate the actual byte code using ASM in
- * the {@link Writer}.
+ * one for analysis and another to generate the actual byte code using ASM using the root of the tree {@link SSource}.
  */
 final class Compiler {
 
@@ -100,18 +98,17 @@ final class Compiler {
                 " plugin if a script longer than this length is a requirement.");
         }
 
-        Reserved reserved = new Reserved();
-        SSource root = Walker.buildPainlessTree(name, source, reserved, settings);
-        Variables variables = Analyzer.analyze(reserved, root);
-        BitSet expressions = new BitSet(source.length());
-        byte[] bytes = Writer.write(settings, name, source, variables, root, expressions);
+        SSource root = Walker.buildPainlessTree(name, source, settings);
+
+        root.analyze();
+        root.write();
 
         try {
-            Class<? extends Executable> clazz = loader.define(CLASS_NAME, bytes);
+            Class<? extends Executable> clazz = loader.define(CLASS_NAME, root.getBytes());
             java.lang.reflect.Constructor<? extends Executable> constructor =
                     clazz.getConstructor(String.class, String.class, BitSet.class);
 
-            return constructor.newInstance(name, source, expressions);
+            return constructor.newInstance(name, source, root.getExpressions());
         } catch (Exception exception) { // Catch everything to let the user know this is something caused internally.
             throw new IllegalStateException("An internal error occurred attempting to define the script [" + name + "].", exception);
         }
@@ -130,11 +127,12 @@ final class Compiler {
                 " plugin if a script longer than this length is a requirement.");
         }
 
-        Reserved reserved = new Reserved();
-        SSource root = Walker.buildPainlessTree(name, source, reserved, settings);
-        Variables variables = Analyzer.analyze(reserved, root);
+        SSource root = Walker.buildPainlessTree(name, source, settings);
 
-        return Writer.write(settings, name, source, variables, root, new BitSet(source.length()));
+        root.analyze();
+        root.write();
+
+        return root.getBytes();
     }
 
     /**
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index dbfc9993c31..a8d4a803361 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -45,9 +45,9 @@ import java.util.Spliterator;
  * methods and fields during at both compile-time and runtime.
  */
 public final class Definition {
-    
+
     private static final List<String> DEFINITION_FILES = Collections.unmodifiableList(
-        Arrays.asList("org.elasticsearch.txt", 
+        Arrays.asList("org.elasticsearch.txt",
                       "java.lang.txt",
                       "java.math.txt",
                       "java.text.txt",
@@ -188,8 +188,8 @@ public final class Definition {
         public final int modifiers;
         public final MethodHandle handle;
 
-        private Method(String name, Struct owner, Type rtn, List<Type> arguments,
-                       org.objectweb.asm.commons.Method method, int modifiers, MethodHandle handle) {
+        public Method(String name, Struct owner, Type rtn, List<Type> arguments,
+                      org.objectweb.asm.commons.Method method, int modifiers, MethodHandle handle) {
             this.name = name;
             this.owner = owner;
             this.rtn = rtn;
@@ -286,7 +286,7 @@ public final class Definition {
 
         public final Map<String, Field> staticMembers;
         public final Map<String, Field> members;
-        
+
         private final SetOnce<Method> functionalMethod;
 
         private Struct(final String name, final Class<?> clazz, final org.objectweb.asm.Type type) {
@@ -300,8 +300,8 @@ public final class Definition {
 
             staticMembers = new HashMap<>();
             members = new HashMap<>();
-            
-            functionalMethod = new SetOnce<Method>();
+
+            functionalMethod = new SetOnce<>();
         }
 
         private Struct(final Struct struct) {
@@ -315,7 +315,7 @@ public final class Definition {
 
             staticMembers = Collections.unmodifiableMap(struct.staticMembers);
             members = Collections.unmodifiableMap(struct.members);
-            
+
             functionalMethod = struct.functionalMethod;
         }
 
@@ -342,8 +342,8 @@ public final class Definition {
         public int hashCode() {
             return name.hashCode();
         }
-        
-        /** 
+
+        /**
          * If this class is a functional interface according to JLS, returns its method.
          * Otherwise returns null.
          */
@@ -637,7 +637,7 @@ public final class Definition {
         final org.objectweb.asm.commons.Method asm = org.objectweb.asm.commons.Method.getMethod(reflect);
         final Type returnType = getTypeInternal("void");
         final MethodHandle handle;
-        
+
         try {
             handle = MethodHandles.publicLookup().in(owner.clazz).unreflectConstructor(reflect);
         } catch (final IllegalAccessException exception) {
@@ -645,7 +645,7 @@ public final class Definition {
                 " not found for class [" + owner.clazz.getName() + "]" +
                 " with arguments " + Arrays.toString(classes) + ".");
         }
-        
+
         final Method constructor = new Method(name, owner, returnType, Arrays.asList(args), asm, reflect.getModifiers(), handle);
 
         owner.constructors.put(methodKey, constructor);
@@ -755,7 +755,7 @@ public final class Definition {
                 " method [" + name + "]" +
                 " within the struct [" + owner.name + "].");
         }
-        
+
         final org.objectweb.asm.commons.Method asm = org.objectweb.asm.commons.Method.getMethod(reflect);
 
         MethodHandle handle;
@@ -856,7 +856,7 @@ public final class Definition {
                 throw new ClassCastException("Child struct [" + child.name + "]" +
                     " is not a super type of owner struct [" + owner.name + "] in copy.");
             }
-            
+
             for (Map.Entry<MethodKey,Method> kvPair : child.methods.entrySet()) {
                 MethodKey methodKey = kvPair.getKey();
                 Method method = kvPair.getValue();
@@ -954,7 +954,7 @@ public final class Definition {
 
         runtimeMap.put(struct.clazz, new RuntimeClass(methods, getters, setters));
     }
-    
+
     /** computes the functional interface method for a class, or returns null */
     private Method computeFunctionalInterfaceMethod(Struct clazz) {
         if (!clazz.clazz.isInterface()) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
similarity index 52%
rename from modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
rename to modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
index 4905011520a..d102114bdeb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Variables.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
@@ -19,16 +19,21 @@
 
 package org.elasticsearch.painless;
 
+import org.elasticsearch.painless.Definition.Method;
+import org.elasticsearch.painless.Definition.MethodKey;
 import org.elasticsearch.painless.Definition.Type;
 
 import java.util.ArrayDeque;
+import java.util.Collections;
 import java.util.Deque;
 import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
 
 /**
- * Tracks variables across compilation phases.
+ * Tracks user defined methods and variables across compilation phases.
  */
-public final class Variables {
+public final class Locals {
 
     /**
      * Tracks reserved variables.  Must be given to any source of input
@@ -36,7 +41,15 @@ public final class Variables {
      * are known ahead of time to assign appropriate slots without
      * being wasteful.
      */
-    public static final class Reserved {
+    public interface Reserved {
+        void markReserved(String name);
+        boolean isReserved(String name);
+
+        void setMaxLoopCounter(int max);
+        int getMaxLoopCounter();
+    }
+
+    public static final class ExecuteReserved implements Reserved {
         public static final String THIS   = "#this";
         public static final String PARAMS = "params";
         public static final String SCORER = "#scorer";
@@ -46,10 +59,11 @@ public final class Variables {
         public static final String CTX    = "ctx";
         public static final String LOOP   = "#loop";
 
-        boolean score = false;
-        boolean ctx = false;
-        boolean loop = false;
+        private boolean score = false;
+        private boolean ctx = false;
+        private int maxLoopCounter = 0;
 
+        @Override
         public void markReserved(String name) {
             if (SCORE.equals(name)) {
                 score = true;
@@ -58,13 +72,53 @@ public final class Variables {
             }
         }
 
+        @Override
         public boolean isReserved(String name) {
             return name.equals(THIS) || name.equals(PARAMS) || name.equals(SCORER) || name.equals(DOC) ||
                 name.equals(VALUE) || name.equals(SCORE) || name.equals(CTX) || name.equals(LOOP);
-         }
+        }
 
-        public void usesLoop() {
-            loop = true;
+        public boolean usesScore() {
+            return score;
+        }
+
+        public boolean usesCtx() {
+            return ctx;
+        }
+
+        @Override
+        public void setMaxLoopCounter(int max) {
+            maxLoopCounter = max;
+        }
+
+        @Override
+        public int getMaxLoopCounter() {
+            return maxLoopCounter;
+        }
+    }
+
+    public static final class FunctionReserved implements Reserved {
+        public static final String THIS = "#this";
+        public static final String LOOP = "#loop";
+
+        private int maxLoopCounter = 0;
+
+        public void markReserved(String name) {
+            // Do nothing.
+        }
+
+        public boolean isReserved(String name) {
+            return name.equals(THIS) || name.equals(LOOP);
+        }
+
+        @Override
+        public void setMaxLoopCounter(int max) {
+            maxLoopCounter = max;
+        }
+
+        @Override
+        public int getMaxLoopCounter() {
+            return maxLoopCounter;
         }
     }
 
@@ -86,52 +140,97 @@ public final class Variables {
         }
     }
 
-    final Reserved reserved;
+    public static final class Parameter {
+        public final Location location;
+        public final String name;
+        public final Type type;
+
+        public Parameter(Location location, String name, Type type) {
+            this.location = location;
+            this.name = name;
+            this.type = type;
+        }
+    }
+
+    private final Reserved reserved;
+    private final Map<MethodKey, Method> methods;
+    private final Type rtnType;
 
     // TODO: this datastructure runs in linear time for nearly all operations. use linkedhashset instead?
     private final Deque<Integer> scopes = new ArrayDeque<>();
     private final Deque<Variable> variables = new ArrayDeque<>();
 
-    public Variables(Reserved reserved) {
+    public Locals(ExecuteReserved reserved, Map<MethodKey, Method> methods) {
         this.reserved = reserved;
+        this.methods = Collections.unmodifiableMap(methods);
+        this.rtnType = Definition.OBJECT_TYPE;
 
         incrementScope();
 
         // Method variables.
 
         // This reference.  Internal use only.
-        addVariable(null, Definition.getType("Object"), Reserved.THIS, true, true);
+        addVariable(null, Definition.getType("Object"), ExecuteReserved.THIS, true, true);
 
         // Input map of variables passed to the script.
-        addVariable(null, Definition.getType("Map"), Reserved.PARAMS, true, true);
+        addVariable(null, Definition.getType("Map"), ExecuteReserved.PARAMS, true, true);
 
         // Scorer parameter passed to the script.  Internal use only.
-        addVariable(null, Definition.DEF_TYPE, Reserved.SCORER, true, true);
+        addVariable(null, Definition.DEF_TYPE, ExecuteReserved.SCORER, true, true);
 
         // Doc parameter passed to the script. TODO: Currently working as a Map, we can do better?
-        addVariable(null, Definition.getType("Map"), Reserved.DOC, true, true);
+        addVariable(null, Definition.getType("Map"), ExecuteReserved.DOC, true, true);
 
         // Aggregation _value parameter passed to the script.
-        addVariable(null, Definition.DEF_TYPE, Reserved.VALUE, true, true);
+        addVariable(null, Definition.DEF_TYPE, ExecuteReserved.VALUE, true, true);
 
         // Shortcut variables.
 
         // Document's score as a read-only double.
-        if (reserved.score) {
-            addVariable(null, Definition.DOUBLE_TYPE, Reserved.SCORE, true, true);
+        if (reserved.usesScore()) {
+            addVariable(null, Definition.DOUBLE_TYPE, ExecuteReserved.SCORE, true, true);
         }
 
         // The ctx map set by executable scripts as a read-only map.
-        if (reserved.ctx) {
-            addVariable(null, Definition.getType("Map"), Reserved.CTX, true, true);
+        if (reserved.usesCtx()) {
+            addVariable(null, Definition.getType("Map"), ExecuteReserved.CTX, true, true);
         }
 
         // Loop counter to catch infinite loops.  Internal use only.
-        if (reserved.loop) {
-            addVariable(null, Definition.INT_TYPE, Reserved.LOOP, true, true);
+        if (reserved.getMaxLoopCounter() > 0) {
+            addVariable(null, Definition.INT_TYPE, ExecuteReserved.LOOP, true, true);
         }
     }
 
+    public Locals(FunctionReserved reserved, Locals locals, Type rtnType, List<Parameter> parameters) {
+        this.reserved = reserved;
+        this.methods = locals.methods;
+        this.rtnType = rtnType;
+
+        incrementScope();
+
+        for (Parameter parameter : parameters) {
+            addVariable(parameter.location, parameter.type, parameter.name, false, false);
+        }
+
+        // Loop counter to catch infinite loops.  Internal use only.
+        if (reserved.getMaxLoopCounter() > 0) {
+            addVariable(null, Definition.INT_TYPE, ExecuteReserved.LOOP, true, true);
+        }
+    }
+
+    public int getMaxLoopCounter() {
+        return reserved.getMaxLoopCounter();
+    }
+
+    public Method getMethod(MethodKey key) {
+        return methods.get(key);
+    }
+
+    public Type getReturnType() {
+        return rtnType;
+    }
+
     public void incrementScope() {
         scopes.push(0);
     }
@@ -142,9 +241,9 @@ public final class Variables {
         while (remove > 0) {
             Variable variable = variables.pop();
 
-            // TODO: is this working? the code reads backwards...
+            // This checks whether or not a variable is used when exiting a local scope.
             if (variable.read) {
-                throw variable.location.createError(new IllegalArgumentException("Variable [" + variable.name + "] never used."));
+                throw variable.location.createError(new IllegalArgumentException("Variable [" + variable.name + "] is never used."));
             }
 
             --remove;
@@ -162,26 +261,30 @@ public final class Variables {
             }
         }
 
-        throw location.createError(new IllegalArgumentException("Variable [" + name + "] not defined."));
+        throw location.createError(new IllegalArgumentException("Variable [" + name + "] is not defined."));
     }
 
-    private boolean variableExists(String name) {
-        return variables.contains(name);
+    public boolean isVariable(String name) {
+        Iterator<Variable> itr = variables.iterator();
+
+        while (itr.hasNext()) {
+            Variable variable = itr.next();
+
+            if (variable.name.equals(name)) {
+                return true;
+            }
+        }
+
+        return false;
     }
 
     public Variable addVariable(Location location, Type type, String name, boolean readonly, boolean reserved) {
         if (!reserved && this.reserved.isReserved(name)) {
-            throw location.createError(new IllegalArgumentException("Variable name [" + name + "] is reserved."));
+            throw location.createError(new IllegalArgumentException("Variable [" + name + "] is reserved."));
         }
 
-        if (variableExists(name)) {
-            throw new IllegalArgumentException("Variable name [" + name + "] already defined.");
-        }
-
-        try {
-            Definition.getType(name);
-        } catch (IllegalArgumentException exception) {
-            // Do nothing.
+        if (isVariable(name)) {
+            throw location.createError(new IllegalArgumentException("Variable [" + name + "] is already defined."));
         }
 
         Variable previous = variables.peekFirst();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index 7b7caec320e..923c2e3f1e8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -91,10 +91,10 @@ public final class MethodWriter extends GeneratorAdapter {
     private final ClassWriter parent;
     private final BitSet statements;
 
-    private final Deque<List<org.objectweb.asm.Type>> stringConcatArgs = (INDY_STRING_CONCAT_BOOTSTRAP_HANDLE == null) ?
-            null : new ArrayDeque<>();
+    private final Deque<List<org.objectweb.asm.Type>> stringConcatArgs =
+        (INDY_STRING_CONCAT_BOOTSTRAP_HANDLE == null) ?  null : new ArrayDeque<>();
 
-    MethodWriter(int access, Method method, ClassWriter cw, BitSet statements) {
+    public MethodWriter(int access, Method method, ClassWriter cw, BitSet statements) {
         super(Opcodes.ASM5, cw.visitMethod(access, method.getName(), method.getDescriptor(), null, null),
                 access, method.getName(), method.getDescriptor());
 
@@ -105,7 +105,7 @@ public final class MethodWriter extends GeneratorAdapter {
     /**
      * @return A new {@link MethodWriter} with the specified access and signature.
      */
-    MethodWriter newMethodWriter(int access, Method method) {
+    public MethodWriter newMethodWriter(int access, Method method) {
         return new MethodWriter(access, method, parent, statements);
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
deleted file mode 100644
index 4d4a778d558..00000000000
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Writer.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless;
-
-import org.elasticsearch.painless.Variables.Reserved;
-import org.elasticsearch.painless.Variables.Variable;
-import org.elasticsearch.painless.node.SSource;
-import org.objectweb.asm.ClassWriter;
-import org.objectweb.asm.Opcodes;
-import org.objectweb.asm.commons.GeneratorAdapter;
-
-import static org.elasticsearch.painless.WriterConstants.BASE_CLASS_TYPE;
-import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE;
-import static org.elasticsearch.painless.WriterConstants.CONSTRUCTOR;
-import static org.elasticsearch.painless.WriterConstants.EXECUTE;
-import static org.elasticsearch.painless.WriterConstants.MAP_GET;
-import static org.elasticsearch.painless.WriterConstants.MAP_TYPE;
-
-import java.util.BitSet;
-
-/**
- * Runs the writing phase of compilation using the Painless AST.
- */
-final class Writer {
-
-    static byte[] write(CompilerSettings settings, String name, String source, Variables variables, SSource root, BitSet expressions) {
-        return new Writer(settings, name, source, variables, root, expressions).getBytes();
-    }
-
-    private final CompilerSettings settings;
-    private final String scriptName;
-    private final String source;
-    private final Variables variables;
-    private final SSource root;
-
-    private final ClassWriter writer;
-    private final MethodWriter adapter;
-
-    private Writer(CompilerSettings settings, String name, String source, Variables variables, SSource root, BitSet expressions) {
-        this.settings = settings;
-        this.scriptName = name;
-        this.source = source;
-        this.variables = variables;
-        this.root = root;
-
-        writer = new ClassWriter(ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS);
-
-        writeBegin();
-        writeConstructor();
-
-        adapter = new MethodWriter(Opcodes.ACC_PUBLIC, EXECUTE, writer, expressions);
-
-        writeExecute();
-        writeEnd();
-    }
-
-    private void writeBegin() {
-        final int version = Opcodes.V1_8;
-        final int access = Opcodes.ACC_PUBLIC | Opcodes.ACC_SUPER | Opcodes.ACC_FINAL;
-        final String base = BASE_CLASS_TYPE.getInternalName();
-        final String name = CLASS_TYPE.getInternalName();
-
-        // apply marker interface NeedsScore if we use the score!
-        final String interfaces[] = variables.reserved.score ?
-            new String[] { WriterConstants.NEEDS_SCORE_TYPE.getInternalName() } : null;
-
-        writer.visit(version, access, name, null, base, interfaces);
-        writer.visitSource(Location.computeSourceName(scriptName,source), null);
-    }
-
-    private void writeConstructor() {
-        final GeneratorAdapter constructor = new GeneratorAdapter(Opcodes.ACC_PUBLIC, CONSTRUCTOR, null, null, writer);
-        constructor.loadThis();
-        constructor.loadArgs();
-        constructor.invokeConstructor(org.objectweb.asm.Type.getType(Executable.class), CONSTRUCTOR);
-        constructor.returnValue();
-        constructor.endMethod();
-    }
-
-    private void writeExecute() {
-        if (variables.reserved.score) {
-            // if the _score value is used, we do this once:
-            // final double _score = scorer.score();
-            final Variable scorer = variables.getVariable(null, Reserved.SCORER);
-            final Variable score = variables.getVariable(null, Reserved.SCORE);
-
-            adapter.visitVarInsn(Opcodes.ALOAD, scorer.slot);
-            adapter.invokeVirtual(WriterConstants.SCORER_TYPE, WriterConstants.SCORER_SCORE);
-            adapter.visitInsn(Opcodes.F2D);
-            adapter.visitVarInsn(Opcodes.DSTORE, score.slot);
-        }
-
-        if (variables.reserved.ctx) {
-            // if the _ctx value is used, we do this once:
-            // final Map<String,Object> ctx = input.get("ctx");
-
-            final Variable input = variables.getVariable(null, Reserved.PARAMS);
-            final Variable ctx = variables.getVariable(null, Reserved.CTX);
-
-            adapter.visitVarInsn(Opcodes.ALOAD, input.slot);
-            adapter.push(Reserved.CTX);
-            adapter.invokeInterface(MAP_TYPE, MAP_GET);
-            adapter.visitVarInsn(Opcodes.ASTORE, ctx.slot);
-        }
-
-        if (variables.reserved.loop) {
-            // if there is infinite loop protection, we do this once:
-            // int #loop = settings.getMaxLoopCounter()
-
-            final Variable loop = variables.getVariable(null, Reserved.LOOP);
-
-            adapter.push(settings.getMaxLoopCounter());
-            adapter.visitVarInsn(Opcodes.ISTORE, loop.slot);
-        }
-
-        root.write(adapter);
-        adapter.endMethod();
-    }
-
-    private void writeEnd() {
-        writer.visitEnd();
-    }
-
-    private byte[] getBytes() {
-        return writer.toByteArray();
-    }
-}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index 80082c4487b..03879a0a3a0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -107,15 +107,15 @@ public final class WriterConstants {
     public final static Method DEF_LTE_CALL = getAsmMethod(boolean.class, "lte", Object.class, Object.class);
     public final static Method DEF_GT_CALL  = getAsmMethod(boolean.class, "gt" , Object.class, Object.class);
     public final static Method DEF_GTE_CALL = getAsmMethod(boolean.class, "gte", Object.class, Object.class);
-    
+
     /** invokedynamic bootstrap for lambda expression/method references */
     public final static MethodType LAMBDA_BOOTSTRAP_TYPE =
-            MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, 
+            MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class,
                                   MethodType.class, Object[].class);
     public final static Handle LAMBDA_BOOTSTRAP_HANDLE =
             new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(LambdaMetafactory.class),
                 "altMetafactory", LAMBDA_BOOTSTRAP_TYPE.toMethodDescriptorString());
-        
+
     /** dynamic invokedynamic bootstrap for indy string concats (Java 9+) */
     public final static Handle INDY_STRING_CONCAT_BOOTSTRAP_HANDLE;
     static {
@@ -152,7 +152,7 @@ public final class WriterConstants {
 
     public final static Method CHECKEQUALS = getAsmMethod(boolean.class, "checkEquals", Object.class, Object.class);
 
-    public static Method getAsmMethod(final Class<?> rtype, final String name, final Class<?>... ptypes) {
+    private static Method getAsmMethod(final Class<?> rtype, final String name, final Class<?>... ptypes) {
         return new Method(name, MethodType.methodType(rtype, ptypes).toMethodDescriptorString());
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
index 540773fbf70..acaf00a5765 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
@@ -28,17 +28,17 @@ class PainlessParser extends Parser {
     STRING=68, TRUE=69, FALSE=70, NULL=71, TYPE=72, ID=73, DOTINTEGER=74, 
     DOTID=75;
   public static final int
-    RULE_source = 0, RULE_statement = 1, RULE_trailer = 2, RULE_block = 3, 
-    RULE_empty = 4, RULE_initializer = 5, RULE_afterthought = 6, RULE_declaration = 7, 
-    RULE_decltype = 8, RULE_funcref = 9, RULE_declvar = 10, RULE_trap = 11, 
-    RULE_delimiter = 12, RULE_expression = 13, RULE_unary = 14, RULE_chain = 15, 
-    RULE_primary = 16, RULE_secondary = 17, RULE_dot = 18, RULE_brace = 19, 
-    RULE_arguments = 20, RULE_argument = 21;
+    RULE_source = 0, RULE_function = 1, RULE_parameters = 2, RULE_statement = 3, 
+    RULE_trailer = 4, RULE_block = 5, RULE_empty = 6, RULE_initializer = 7, 
+    RULE_afterthought = 8, RULE_declaration = 9, RULE_decltype = 10, RULE_funcref = 11, 
+    RULE_declvar = 12, RULE_trap = 13, RULE_delimiter = 14, RULE_expression = 15, 
+    RULE_unary = 16, RULE_chain = 17, RULE_primary = 18, RULE_secondary = 19, 
+    RULE_dot = 20, RULE_brace = 21, RULE_arguments = 22, RULE_argument = 23;
   public static final String[] ruleNames = {
-    "source", "statement", "trailer", "block", "empty", "initializer", "afterthought", 
-    "declaration", "decltype", "funcref", "declvar", "trap", "delimiter", 
-    "expression", "unary", "chain", "primary", "secondary", "dot", "brace", 
-    "arguments", "argument"
+    "source", "function", "parameters", "statement", "trailer", "block", "empty", 
+    "initializer", "afterthought", "declaration", "decltype", "funcref", "declvar", 
+    "trap", "delimiter", "expression", "unary", "chain", "primary", "secondary", 
+    "dot", "brace", "arguments", "argument"
   };
 
   private static final String[] _LITERAL_NAMES = {
@@ -113,6 +113,12 @@ class PainlessParser extends Parser {
   }
   public static class SourceContext extends ParserRuleContext {
     public TerminalNode EOF() { return getToken(PainlessParser.EOF, 0); }
+    public List<FunctionContext> function() {
+      return getRuleContexts(FunctionContext.class);
+    }
+    public FunctionContext function(int i) {
+      return getRuleContext(FunctionContext.class,i);
+    }
     public List<StatementContext> statement() {
       return getRuleContexts(StatementContext.class);
     }
@@ -137,23 +143,39 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(47);
+      setState(51);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,0,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(44);
+          setState(48);
+          function();
+          }
+          } 
+        }
+        setState(53);
+        _errHandler.sync(this);
+        _alt = getInterpreter().adaptivePredict(_input,0,_ctx);
+      }
+      setState(57);
+      _errHandler.sync(this);
+      _alt = getInterpreter().adaptivePredict(_input,1,_ctx);
+      while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
+        if ( _alt==1 ) {
+          {
+          {
+          setState(54);
           statement();
           }
           } 
         }
-        setState(49);
+        setState(59);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,0,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,1,_ctx);
       }
-      setState(50);
+      setState(60);
       match(EOF);
       }
     }
@@ -168,6 +190,136 @@ class PainlessParser extends Parser {
     return _localctx;
   }
 
+  public static class FunctionContext extends ParserRuleContext {
+    public DecltypeContext decltype() {
+      return getRuleContext(DecltypeContext.class,0);
+    }
+    public TerminalNode ID() { return getToken(PainlessParser.ID, 0); }
+    public ParametersContext parameters() {
+      return getRuleContext(ParametersContext.class,0);
+    }
+    public BlockContext block() {
+      return getRuleContext(BlockContext.class,0);
+    }
+    public FunctionContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_function; }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitFunction(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final FunctionContext function() throws RecognitionException {
+    FunctionContext _localctx = new FunctionContext(_ctx, getState());
+    enterRule(_localctx, 2, RULE_function);
+    try {
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(62);
+      decltype();
+      setState(63);
+      match(ID);
+      setState(64);
+      parameters();
+      setState(65);
+      block();
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
+  public static class ParametersContext extends ParserRuleContext {
+    public TerminalNode LP() { return getToken(PainlessParser.LP, 0); }
+    public TerminalNode RP() { return getToken(PainlessParser.RP, 0); }
+    public List<DecltypeContext> decltype() {
+      return getRuleContexts(DecltypeContext.class);
+    }
+    public DecltypeContext decltype(int i) {
+      return getRuleContext(DecltypeContext.class,i);
+    }
+    public List<TerminalNode> ID() { return getTokens(PainlessParser.ID); }
+    public TerminalNode ID(int i) {
+      return getToken(PainlessParser.ID, i);
+    }
+    public List<TerminalNode> COMMA() { return getTokens(PainlessParser.COMMA); }
+    public TerminalNode COMMA(int i) {
+      return getToken(PainlessParser.COMMA, i);
+    }
+    public ParametersContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_parameters; }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitParameters(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final ParametersContext parameters() throws RecognitionException {
+    ParametersContext _localctx = new ParametersContext(_ctx, getState());
+    enterRule(_localctx, 4, RULE_parameters);
+    int _la;
+    try {
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(67);
+      match(LP);
+      setState(79);
+      _la = _input.LA(1);
+      if (_la==TYPE) {
+        {
+        setState(68);
+        decltype();
+        setState(69);
+        match(ID);
+        setState(76);
+        _errHandler.sync(this);
+        _la = _input.LA(1);
+        while (_la==COMMA) {
+          {
+          {
+          setState(70);
+          match(COMMA);
+          setState(71);
+          decltype();
+          setState(72);
+          match(ID);
+          }
+          }
+          setState(78);
+          _errHandler.sync(this);
+          _la = _input.LA(1);
+        }
+        }
+      }
+
+      setState(81);
+      match(RP);
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
   public static class StatementContext extends ParserRuleContext {
     public StatementContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
@@ -396,38 +548,38 @@ class PainlessParser extends Parser {
 
   public final StatementContext statement() throws RecognitionException {
     StatementContext _localctx = new StatementContext(_ctx, getState());
-    enterRule(_localctx, 2, RULE_statement);
+    enterRule(_localctx, 6, RULE_statement);
     try {
       int _alt;
-      setState(130);
-      switch ( getInterpreter().adaptivePredict(_input,8,_ctx) ) {
+      setState(161);
+      switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) {
       case 1:
         _localctx = new IfContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(52);
+        setState(83);
         match(IF);
-        setState(53);
+        setState(84);
         match(LP);
-        setState(54);
+        setState(85);
         expression(0);
-        setState(55);
+        setState(86);
         match(RP);
-        setState(56);
+        setState(87);
         trailer();
-        setState(60);
-        switch ( getInterpreter().adaptivePredict(_input,1,_ctx) ) {
+        setState(91);
+        switch ( getInterpreter().adaptivePredict(_input,4,_ctx) ) {
         case 1:
           {
-          setState(57);
+          setState(88);
           match(ELSE);
-          setState(58);
+          setState(89);
           trailer();
           }
           break;
         case 2:
           {
-          setState(59);
+          setState(90);
           if (!( _input.LA(1) != ELSE )) throw new FailedPredicateException(this, " _input.LA(1) != ELSE ");
           }
           break;
@@ -438,25 +590,25 @@ class PainlessParser extends Parser {
         _localctx = new WhileContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(62);
+        setState(93);
         match(WHILE);
-        setState(63);
+        setState(94);
         match(LP);
-        setState(64);
+        setState(95);
         expression(0);
-        setState(65);
+        setState(96);
         match(RP);
-        setState(68);
-        switch ( getInterpreter().adaptivePredict(_input,2,_ctx) ) {
+        setState(99);
+        switch ( getInterpreter().adaptivePredict(_input,5,_ctx) ) {
         case 1:
           {
-          setState(66);
+          setState(97);
           trailer();
           }
           break;
         case 2:
           {
-          setState(67);
+          setState(98);
           empty();
           }
           break;
@@ -467,19 +619,19 @@ class PainlessParser extends Parser {
         _localctx = new DoContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(70);
+        setState(101);
         match(DO);
-        setState(71);
+        setState(102);
         block();
-        setState(72);
+        setState(103);
         match(WHILE);
-        setState(73);
+        setState(104);
         match(LP);
-        setState(74);
+        setState(105);
         expression(0);
-        setState(75);
+        setState(106);
         match(RP);
-        setState(76);
+        setState(107);
         delimiter();
         }
         break;
@@ -487,54 +639,54 @@ class PainlessParser extends Parser {
         _localctx = new ForContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(78);
+        setState(109);
         match(FOR);
-        setState(79);
+        setState(110);
         match(LP);
-        setState(81);
-        switch ( getInterpreter().adaptivePredict(_input,3,_ctx) ) {
+        setState(112);
+        switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) {
         case 1:
           {
-          setState(80);
+          setState(111);
           initializer();
           }
           break;
         }
-        setState(83);
+        setState(114);
         match(SEMICOLON);
-        setState(85);
-        switch ( getInterpreter().adaptivePredict(_input,4,_ctx) ) {
+        setState(116);
+        switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) {
         case 1:
           {
-          setState(84);
+          setState(115);
           expression(0);
           }
           break;
         }
-        setState(87);
+        setState(118);
         match(SEMICOLON);
-        setState(89);
-        switch ( getInterpreter().adaptivePredict(_input,5,_ctx) ) {
+        setState(120);
+        switch ( getInterpreter().adaptivePredict(_input,8,_ctx) ) {
         case 1:
           {
-          setState(88);
+          setState(119);
           afterthought();
           }
           break;
         }
-        setState(91);
+        setState(122);
         match(RP);
-        setState(94);
-        switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) {
+        setState(125);
+        switch ( getInterpreter().adaptivePredict(_input,9,_ctx) ) {
         case 1:
           {
-          setState(92);
+          setState(123);
           trailer();
           }
           break;
         case 2:
           {
-          setState(93);
+          setState(124);
           empty();
           }
           break;
@@ -545,21 +697,21 @@ class PainlessParser extends Parser {
         _localctx = new EachContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(96);
+        setState(127);
         match(FOR);
-        setState(97);
+        setState(128);
         match(LP);
-        setState(98);
+        setState(129);
         decltype();
-        setState(99);
+        setState(130);
         match(ID);
-        setState(100);
+        setState(131);
         match(COLON);
-        setState(101);
+        setState(132);
         expression(0);
-        setState(102);
+        setState(133);
         match(RP);
-        setState(103);
+        setState(134);
         trailer();
         }
         break;
@@ -567,9 +719,9 @@ class PainlessParser extends Parser {
         _localctx = new DeclContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(105);
+        setState(136);
         declaration();
-        setState(106);
+        setState(137);
         delimiter();
         }
         break;
@@ -577,9 +729,9 @@ class PainlessParser extends Parser {
         _localctx = new ContinueContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(108);
+        setState(139);
         match(CONTINUE);
-        setState(109);
+        setState(140);
         delimiter();
         }
         break;
@@ -587,9 +739,9 @@ class PainlessParser extends Parser {
         _localctx = new BreakContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(110);
+        setState(141);
         match(BREAK);
-        setState(111);
+        setState(142);
         delimiter();
         }
         break;
@@ -597,11 +749,11 @@ class PainlessParser extends Parser {
         _localctx = new ReturnContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(112);
+        setState(143);
         match(RETURN);
-        setState(113);
+        setState(144);
         expression(0);
-        setState(114);
+        setState(145);
         delimiter();
         }
         break;
@@ -609,11 +761,11 @@ class PainlessParser extends Parser {
         _localctx = new TryContext(_localctx);
         enterOuterAlt(_localctx, 10);
         {
-        setState(116);
+        setState(147);
         match(TRY);
-        setState(117);
+        setState(148);
         block();
-        setState(119); 
+        setState(150); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -621,7 +773,7 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(118);
+            setState(149);
             trap();
             }
             }
@@ -629,9 +781,9 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(121); 
+          setState(152); 
           _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,7,_ctx);
+          _alt = getInterpreter().adaptivePredict(_input,10,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
         }
         break;
@@ -639,11 +791,11 @@ class PainlessParser extends Parser {
         _localctx = new ThrowContext(_localctx);
         enterOuterAlt(_localctx, 11);
         {
-        setState(123);
+        setState(154);
         match(THROW);
-        setState(124);
+        setState(155);
         expression(0);
-        setState(125);
+        setState(156);
         delimiter();
         }
         break;
@@ -651,9 +803,9 @@ class PainlessParser extends Parser {
         _localctx = new ExprContext(_localctx);
         enterOuterAlt(_localctx, 12);
         {
-        setState(127);
+        setState(158);
         expression(0);
-        setState(128);
+        setState(159);
         delimiter();
         }
         break;
@@ -690,21 +842,21 @@ class PainlessParser extends Parser {
 
   public final TrailerContext trailer() throws RecognitionException {
     TrailerContext _localctx = new TrailerContext(_ctx, getState());
-    enterRule(_localctx, 4, RULE_trailer);
+    enterRule(_localctx, 8, RULE_trailer);
     try {
-      setState(134);
-      switch ( getInterpreter().adaptivePredict(_input,9,_ctx) ) {
+      setState(165);
+      switch ( getInterpreter().adaptivePredict(_input,12,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(132);
+        setState(163);
         block();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(133);
+        setState(164);
         statement();
         }
         break;
@@ -743,30 +895,30 @@ class PainlessParser extends Parser {
 
   public final BlockContext block() throws RecognitionException {
     BlockContext _localctx = new BlockContext(_ctx, getState());
-    enterRule(_localctx, 6, RULE_block);
+    enterRule(_localctx, 10, RULE_block);
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(136);
+      setState(167);
       match(LBRACK);
-      setState(140);
+      setState(171);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,10,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,13,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(137);
+          setState(168);
           statement();
           }
           } 
         }
-        setState(142);
+        setState(173);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,10,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,13,_ctx);
       }
-      setState(143);
+      setState(174);
       match(RBRACK);
       }
     }
@@ -796,11 +948,11 @@ class PainlessParser extends Parser {
 
   public final EmptyContext empty() throws RecognitionException {
     EmptyContext _localctx = new EmptyContext(_ctx, getState());
-    enterRule(_localctx, 8, RULE_empty);
+    enterRule(_localctx, 12, RULE_empty);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(145);
+      setState(176);
       match(SEMICOLON);
       }
     }
@@ -835,21 +987,21 @@ class PainlessParser extends Parser {
 
   public final InitializerContext initializer() throws RecognitionException {
     InitializerContext _localctx = new InitializerContext(_ctx, getState());
-    enterRule(_localctx, 10, RULE_initializer);
+    enterRule(_localctx, 14, RULE_initializer);
     try {
-      setState(149);
-      switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) {
+      setState(180);
+      switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(147);
+        setState(178);
         declaration();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(148);
+        setState(179);
         expression(0);
         }
         break;
@@ -883,11 +1035,11 @@ class PainlessParser extends Parser {
 
   public final AfterthoughtContext afterthought() throws RecognitionException {
     AfterthoughtContext _localctx = new AfterthoughtContext(_ctx, getState());
-    enterRule(_localctx, 12, RULE_afterthought);
+    enterRule(_localctx, 16, RULE_afterthought);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(151);
+      setState(182);
       expression(0);
       }
     }
@@ -929,28 +1081,28 @@ class PainlessParser extends Parser {
 
   public final DeclarationContext declaration() throws RecognitionException {
     DeclarationContext _localctx = new DeclarationContext(_ctx, getState());
-    enterRule(_localctx, 14, RULE_declaration);
+    enterRule(_localctx, 18, RULE_declaration);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(153);
+      setState(184);
       decltype();
-      setState(154);
+      setState(185);
       declvar();
-      setState(159);
+      setState(190);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==COMMA) {
         {
         {
-        setState(155);
+        setState(186);
         match(COMMA);
-        setState(156);
+        setState(187);
         declvar();
         }
         }
-        setState(161);
+        setState(192);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
@@ -990,26 +1142,26 @@ class PainlessParser extends Parser {
 
   public final DecltypeContext decltype() throws RecognitionException {
     DecltypeContext _localctx = new DecltypeContext(_ctx, getState());
-    enterRule(_localctx, 16, RULE_decltype);
+    enterRule(_localctx, 20, RULE_decltype);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(162);
+      setState(193);
       match(TYPE);
-      setState(167);
+      setState(198);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==LBRACE) {
         {
         {
-        setState(163);
+        setState(194);
         match(LBRACE);
-        setState(164);
+        setState(195);
         match(RBRACE);
         }
         }
-        setState(169);
+        setState(200);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
@@ -1044,16 +1196,16 @@ class PainlessParser extends Parser {
 
   public final FuncrefContext funcref() throws RecognitionException {
     FuncrefContext _localctx = new FuncrefContext(_ctx, getState());
-    enterRule(_localctx, 18, RULE_funcref);
+    enterRule(_localctx, 22, RULE_funcref);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(170);
+      setState(201);
       match(TYPE);
-      setState(171);
+      setState(202);
       match(REF);
-      setState(172);
+      setState(203);
       _la = _input.LA(1);
       if ( !(_la==NEW || _la==ID) ) {
       _errHandler.recoverInline(this);
@@ -1092,20 +1244,20 @@ class PainlessParser extends Parser {
 
   public final DeclvarContext declvar() throws RecognitionException {
     DeclvarContext _localctx = new DeclvarContext(_ctx, getState());
-    enterRule(_localctx, 20, RULE_declvar);
+    enterRule(_localctx, 24, RULE_declvar);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(174);
+      setState(205);
       match(ID);
-      setState(177);
+      setState(208);
       _la = _input.LA(1);
       if (_la==ASSIGN) {
         {
-        setState(175);
+        setState(206);
         match(ASSIGN);
-        setState(176);
+        setState(207);
         expression(0);
         }
       }
@@ -1145,21 +1297,21 @@ class PainlessParser extends Parser {
 
   public final TrapContext trap() throws RecognitionException {
     TrapContext _localctx = new TrapContext(_ctx, getState());
-    enterRule(_localctx, 22, RULE_trap);
+    enterRule(_localctx, 26, RULE_trap);
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(179);
+      setState(210);
       match(CATCH);
-      setState(180);
+      setState(211);
       match(LP);
-      setState(181);
+      setState(212);
       match(TYPE);
-      setState(182);
+      setState(213);
       match(ID);
-      setState(183);
+      setState(214);
       match(RP);
-      setState(184);
+      setState(215);
       block();
       }
     }
@@ -1190,12 +1342,12 @@ class PainlessParser extends Parser {
 
   public final DelimiterContext delimiter() throws RecognitionException {
     DelimiterContext _localctx = new DelimiterContext(_ctx, getState());
-    enterRule(_localctx, 24, RULE_delimiter);
+    enterRule(_localctx, 28, RULE_delimiter);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(186);
+      setState(217);
       _la = _input.LA(1);
       if ( !(_la==EOF || _la==SEMICOLON) ) {
       _errHandler.recoverInline(this);
@@ -1357,31 +1509,31 @@ class PainlessParser extends Parser {
     int _parentState = getState();
     ExpressionContext _localctx = new ExpressionContext(_ctx, _parentState);
     ExpressionContext _prevctx = _localctx;
-    int _startState = 26;
-    enterRecursionRule(_localctx, 26, RULE_expression, _p);
+    int _startState = 30;
+    enterRecursionRule(_localctx, 30, RULE_expression, _p);
     int _la;
     try {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(197);
-      switch ( getInterpreter().adaptivePredict(_input,15,_ctx) ) {
+      setState(228);
+      switch ( getInterpreter().adaptivePredict(_input,18,_ctx) ) {
       case 1:
         {
         _localctx = new AssignmentContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(189);
+        setState(220);
         chain(true);
-        setState(190);
+        setState(221);
         _la = _input.LA(1);
         if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ASSIGN) | (1L << AADD) | (1L << ASUB) | (1L << AMUL) | (1L << ADIV) | (1L << AREM) | (1L << AAND) | (1L << AXOR) | (1L << AOR) | (1L << ALSH) | (1L << ARSH) | (1L << AUSH))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(191);
+        setState(222);
         expression(1);
          ((AssignmentContext)_localctx).s =  false; 
         }
@@ -1391,37 +1543,37 @@ class PainlessParser extends Parser {
         _localctx = new SingleContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(194);
+        setState(225);
         ((SingleContext)_localctx).u = unary(false);
          ((SingleContext)_localctx).s =  ((SingleContext)_localctx).u.s; 
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(258);
+      setState(289);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,17,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(256);
-          switch ( getInterpreter().adaptivePredict(_input,16,_ctx) ) {
+          setState(287);
+          switch ( getInterpreter().adaptivePredict(_input,19,_ctx) ) {
           case 1:
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(199);
+            setState(230);
             if (!(precpred(_ctx, 12))) throw new FailedPredicateException(this, "precpred(_ctx, 12)");
-            setState(200);
+            setState(231);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << MUL) | (1L << DIV) | (1L << REM))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(201);
+            setState(232);
             expression(13);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1430,16 +1582,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(204);
+            setState(235);
             if (!(precpred(_ctx, 11))) throw new FailedPredicateException(this, "precpred(_ctx, 11)");
-            setState(205);
+            setState(236);
             _la = _input.LA(1);
             if ( !(_la==ADD || _la==SUB) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(206);
+            setState(237);
             expression(12);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1448,16 +1600,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(209);
+            setState(240);
             if (!(precpred(_ctx, 10))) throw new FailedPredicateException(this, "precpred(_ctx, 10)");
-            setState(210);
+            setState(241);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LSH) | (1L << RSH) | (1L << USH))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(211);
+            setState(242);
             expression(11);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1466,16 +1618,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(214);
+            setState(245);
             if (!(precpred(_ctx, 9))) throw new FailedPredicateException(this, "precpred(_ctx, 9)");
-            setState(215);
+            setState(246);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LT) | (1L << LTE) | (1L << GT) | (1L << GTE))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(216);
+            setState(247);
             expression(10);
              ((CompContext)_localctx).s =  false; 
             }
@@ -1484,16 +1636,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(219);
+            setState(250);
             if (!(precpred(_ctx, 8))) throw new FailedPredicateException(this, "precpred(_ctx, 8)");
-            setState(220);
+            setState(251);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << EQ) | (1L << EQR) | (1L << NE) | (1L << NER))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(221);
+            setState(252);
             expression(9);
              ((CompContext)_localctx).s =  false; 
             }
@@ -1502,11 +1654,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(224);
+            setState(255);
             if (!(precpred(_ctx, 7))) throw new FailedPredicateException(this, "precpred(_ctx, 7)");
-            setState(225);
+            setState(256);
             match(BWAND);
-            setState(226);
+            setState(257);
             expression(8);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1515,11 +1667,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(229);
+            setState(260);
             if (!(precpred(_ctx, 6))) throw new FailedPredicateException(this, "precpred(_ctx, 6)");
-            setState(230);
+            setState(261);
             match(XOR);
-            setState(231);
+            setState(262);
             expression(7);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1528,11 +1680,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(234);
+            setState(265);
             if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)");
-            setState(235);
+            setState(266);
             match(BWOR);
-            setState(236);
+            setState(267);
             expression(6);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1541,11 +1693,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(239);
+            setState(270);
             if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)");
-            setState(240);
+            setState(271);
             match(BOOLAND);
-            setState(241);
+            setState(272);
             expression(5);
              ((BoolContext)_localctx).s =  false; 
             }
@@ -1554,11 +1706,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(244);
+            setState(275);
             if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)");
-            setState(245);
+            setState(276);
             match(BOOLOR);
-            setState(246);
+            setState(277);
             expression(4);
              ((BoolContext)_localctx).s =  false; 
             }
@@ -1567,15 +1719,15 @@ class PainlessParser extends Parser {
             {
             _localctx = new ConditionalContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(249);
+            setState(280);
             if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
-            setState(250);
+            setState(281);
             match(COND);
-            setState(251);
+            setState(282);
             ((ConditionalContext)_localctx).e0 = expression(0);
-            setState(252);
+            setState(283);
             match(COLON);
-            setState(253);
+            setState(284);
             ((ConditionalContext)_localctx).e1 = expression(2);
              ((ConditionalContext)_localctx).s =  ((ConditionalContext)_localctx).e0.s && ((ConditionalContext)_localctx).e1.s; 
             }
@@ -1583,9 +1735,9 @@ class PainlessParser extends Parser {
           }
           } 
         }
-        setState(260);
+        setState(291);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,17,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
       }
       }
     }
@@ -1727,25 +1879,25 @@ class PainlessParser extends Parser {
 
   public final UnaryContext unary(boolean c) throws RecognitionException {
     UnaryContext _localctx = new UnaryContext(_ctx, getState(), c);
-    enterRule(_localctx, 28, RULE_unary);
+    enterRule(_localctx, 32, RULE_unary);
     int _la;
     try {
-      setState(290);
-      switch ( getInterpreter().adaptivePredict(_input,18,_ctx) ) {
+      setState(321);
+      switch ( getInterpreter().adaptivePredict(_input,21,_ctx) ) {
       case 1:
         _localctx = new PreContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(261);
+        setState(292);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(262);
+        setState(293);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(263);
+        setState(294);
         chain(true);
         }
         break;
@@ -1753,11 +1905,11 @@ class PainlessParser extends Parser {
         _localctx = new PostContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(264);
+        setState(295);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(265);
+        setState(296);
         chain(true);
-        setState(266);
+        setState(297);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
@@ -1770,9 +1922,9 @@ class PainlessParser extends Parser {
         _localctx = new ReadContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(268);
+        setState(299);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(269);
+        setState(300);
         chain(false);
         }
         break;
@@ -1780,9 +1932,9 @@ class PainlessParser extends Parser {
         _localctx = new NumericContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(270);
+        setState(301);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(271);
+        setState(302);
         _la = _input.LA(1);
         if ( !(((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (OCTAL - 64)) | (1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)))) != 0)) ) {
         _errHandler.recoverInline(this);
@@ -1796,9 +1948,9 @@ class PainlessParser extends Parser {
         _localctx = new TrueContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(273);
+        setState(304);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(274);
+        setState(305);
         match(TRUE);
          ((TrueContext)_localctx).s =  false; 
         }
@@ -1807,9 +1959,9 @@ class PainlessParser extends Parser {
         _localctx = new FalseContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(276);
+        setState(307);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(277);
+        setState(308);
         match(FALSE);
          ((FalseContext)_localctx).s =  false; 
         }
@@ -1818,9 +1970,9 @@ class PainlessParser extends Parser {
         _localctx = new NullContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(279);
+        setState(310);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(280);
+        setState(311);
         match(NULL);
          ((NullContext)_localctx).s =  false; 
         }
@@ -1829,16 +1981,16 @@ class PainlessParser extends Parser {
         _localctx = new OperatorContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(282);
+        setState(313);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(283);
+        setState(314);
         _la = _input.LA(1);
         if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(284);
+        setState(315);
         unary(false);
         }
         break;
@@ -1846,13 +1998,13 @@ class PainlessParser extends Parser {
         _localctx = new CastContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(285);
+        setState(316);
         match(LP);
-        setState(286);
+        setState(317);
         decltype();
-        setState(287);
+        setState(318);
         match(RP);
-        setState(288);
+        setState(319);
         unary(_localctx.c);
         }
         break;
@@ -1958,32 +2110,32 @@ class PainlessParser extends Parser {
 
   public final ChainContext chain(boolean c) throws RecognitionException {
     ChainContext _localctx = new ChainContext(_ctx, getState(), c);
-    enterRule(_localctx, 30, RULE_chain);
+    enterRule(_localctx, 34, RULE_chain);
     try {
       int _alt;
-      setState(326);
-      switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) {
+      setState(357);
+      switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) {
       case 1:
         _localctx = new DynamicContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(292);
+        setState(323);
         ((DynamicContext)_localctx).p = primary(_localctx.c);
-        setState(296);
+        setState(327);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,19,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(293);
+            setState(324);
             secondary(((DynamicContext)_localctx).p.s);
             }
             } 
           }
-          setState(298);
+          setState(329);
           _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,19,_ctx);
+          _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
         }
         }
         break;
@@ -1991,25 +2143,25 @@ class PainlessParser extends Parser {
         _localctx = new StaticContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(299);
+        setState(330);
         decltype();
-        setState(300);
+        setState(331);
         dot();
-        setState(304);
+        setState(335);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(301);
+            setState(332);
             secondary(true);
             }
             } 
           }
-          setState(306);
+          setState(337);
           _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
+          _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
         }
         }
         break;
@@ -2017,11 +2169,11 @@ class PainlessParser extends Parser {
         _localctx = new NewarrayContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(307);
+        setState(338);
         match(NEW);
-        setState(308);
+        setState(339);
         match(TYPE);
-        setState(313); 
+        setState(344); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -2029,11 +2181,11 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(309);
+            setState(340);
             match(LBRACE);
-            setState(310);
+            setState(341);
             expression(0);
-            setState(311);
+            setState(342);
             match(RBRACE);
             }
             }
@@ -2041,31 +2193,31 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(315); 
+          setState(346); 
           _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
+          _alt = getInterpreter().adaptivePredict(_input,24,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
-        setState(324);
-        switch ( getInterpreter().adaptivePredict(_input,23,_ctx) ) {
+        setState(355);
+        switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) {
         case 1:
           {
-          setState(317);
+          setState(348);
           dot();
-          setState(321);
+          setState(352);
           _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
+          _alt = getInterpreter().adaptivePredict(_input,25,_ctx);
           while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
             if ( _alt==1 ) {
               {
               {
-              setState(318);
+              setState(349);
               secondary(true);
               }
               } 
             }
-            setState(323);
+            setState(354);
             _errHandler.sync(this);
-            _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
+            _alt = getInterpreter().adaptivePredict(_input,25,_ctx);
           }
           }
           break;
@@ -2111,6 +2263,18 @@ class PainlessParser extends Parser {
       else return visitor.visitChildren(this);
     }
   }
+  public static class CalllocalContext extends PrimaryContext {
+    public TerminalNode ID() { return getToken(PainlessParser.ID, 0); }
+    public ArgumentsContext arguments() {
+      return getRuleContext(ArgumentsContext.class,0);
+    }
+    public CalllocalContext(PrimaryContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitCalllocal(this);
+      else return visitor.visitChildren(this);
+    }
+  }
   public static class VariableContext extends PrimaryContext {
     public TerminalNode ID() { return getToken(PainlessParser.ID, 0); }
     public VariableContext(PrimaryContext ctx) { copyFrom(ctx); }
@@ -2163,21 +2327,21 @@ class PainlessParser extends Parser {
 
   public final PrimaryContext primary(boolean c) throws RecognitionException {
     PrimaryContext _localctx = new PrimaryContext(_ctx, getState(), c);
-    enterRule(_localctx, 32, RULE_primary);
+    enterRule(_localctx, 36, RULE_primary);
     try {
-      setState(344);
-      switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) {
+      setState(377);
+      switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) {
       case 1:
         _localctx = new ExprprecContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(328);
+        setState(359);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(329);
+        setState(360);
         match(LP);
-        setState(330);
+        setState(361);
         ((ExprprecContext)_localctx).e = expression(0);
-        setState(331);
+        setState(362);
         match(RP);
          ((ExprprecContext)_localctx).s =  ((ExprprecContext)_localctx).e.s; 
         }
@@ -2186,13 +2350,13 @@ class PainlessParser extends Parser {
         _localctx = new ChainprecContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(334);
+        setState(365);
         if (!( _localctx.c )) throw new FailedPredicateException(this, " $c ");
-        setState(335);
+        setState(366);
         match(LP);
-        setState(336);
+        setState(367);
         unary(true);
-        setState(337);
+        setState(368);
         match(RP);
         }
         break;
@@ -2200,7 +2364,7 @@ class PainlessParser extends Parser {
         _localctx = new StringContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(339);
+        setState(370);
         match(STRING);
         }
         break;
@@ -2208,19 +2372,29 @@ class PainlessParser extends Parser {
         _localctx = new VariableContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(340);
+        setState(371);
         match(ID);
         }
         break;
       case 5:
-        _localctx = new NewobjectContext(_localctx);
+        _localctx = new CalllocalContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(341);
+        setState(372);
+        match(ID);
+        setState(373);
+        arguments();
+        }
+        break;
+      case 6:
+        _localctx = new NewobjectContext(_localctx);
+        enterOuterAlt(_localctx, 6);
+        {
+        setState(374);
         match(NEW);
-        setState(342);
+        setState(375);
         match(TYPE);
-        setState(343);
+        setState(376);
         arguments();
         }
         break;
@@ -2260,25 +2434,25 @@ class PainlessParser extends Parser {
 
   public final SecondaryContext secondary(boolean s) throws RecognitionException {
     SecondaryContext _localctx = new SecondaryContext(_ctx, getState(), s);
-    enterRule(_localctx, 34, RULE_secondary);
+    enterRule(_localctx, 38, RULE_secondary);
     try {
-      setState(350);
-      switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) {
+      setState(383);
+      switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(346);
+        setState(379);
         if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
-        setState(347);
+        setState(380);
         dot();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(348);
+        setState(381);
         if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
-        setState(349);
+        setState(382);
         brace();
         }
         break;
@@ -2333,20 +2507,20 @@ class PainlessParser extends Parser {
 
   public final DotContext dot() throws RecognitionException {
     DotContext _localctx = new DotContext(_ctx, getState());
-    enterRule(_localctx, 36, RULE_dot);
+    enterRule(_localctx, 40, RULE_dot);
     int _la;
     try {
-      setState(357);
-      switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) {
+      setState(390);
+      switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
       case 1:
         _localctx = new CallinvokeContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(352);
+        setState(385);
         match(DOT);
-        setState(353);
+        setState(386);
         match(DOTID);
-        setState(354);
+        setState(387);
         arguments();
         }
         break;
@@ -2354,9 +2528,9 @@ class PainlessParser extends Parser {
         _localctx = new FieldaccessContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(355);
+        setState(388);
         match(DOT);
-        setState(356);
+        setState(389);
         _la = _input.LA(1);
         if ( !(_la==DOTINTEGER || _la==DOTID) ) {
         _errHandler.recoverInline(this);
@@ -2405,16 +2579,16 @@ class PainlessParser extends Parser {
 
   public final BraceContext brace() throws RecognitionException {
     BraceContext _localctx = new BraceContext(_ctx, getState());
-    enterRule(_localctx, 38, RULE_brace);
+    enterRule(_localctx, 42, RULE_brace);
     try {
       _localctx = new BraceaccessContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(359);
+      setState(392);
       match(LBRACE);
-      setState(360);
+      setState(393);
       expression(0);
-      setState(361);
+      setState(394);
       match(RBRACE);
       }
     }
@@ -2455,40 +2629,40 @@ class PainlessParser extends Parser {
 
   public final ArgumentsContext arguments() throws RecognitionException {
     ArgumentsContext _localctx = new ArgumentsContext(_ctx, getState());
-    enterRule(_localctx, 40, RULE_arguments);
+    enterRule(_localctx, 44, RULE_arguments);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
       {
       {
-      setState(363);
+      setState(396);
       match(LP);
-      setState(372);
-      switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
+      setState(405);
+      switch ( getInterpreter().adaptivePredict(_input,32,_ctx) ) {
       case 1:
         {
-        setState(364);
+        setState(397);
         argument();
-        setState(369);
+        setState(402);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(365);
+          setState(398);
           match(COMMA);
-          setState(366);
+          setState(399);
           argument();
           }
           }
-          setState(371);
+          setState(404);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
         break;
       }
-      setState(374);
+      setState(407);
       match(RP);
       }
       }
@@ -2524,21 +2698,21 @@ class PainlessParser extends Parser {
 
   public final ArgumentContext argument() throws RecognitionException {
     ArgumentContext _localctx = new ArgumentContext(_ctx, getState());
-    enterRule(_localctx, 42, RULE_argument);
+    enterRule(_localctx, 46, RULE_argument);
     try {
-      setState(378);
-      switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
+      setState(411);
+      switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(376);
+        setState(409);
         expression(0);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(377);
+        setState(410);
         funcref();
         }
         break;
@@ -2557,15 +2731,15 @@ class PainlessParser extends Parser {
 
   public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) {
     switch (ruleIndex) {
-    case 1:
+    case 3:
       return statement_sempred((StatementContext)_localctx, predIndex);
-    case 13:
+    case 15:
       return expression_sempred((ExpressionContext)_localctx, predIndex);
-    case 14:
-      return unary_sempred((UnaryContext)_localctx, predIndex);
     case 16:
+      return unary_sempred((UnaryContext)_localctx, predIndex);
+    case 18:
       return primary_sempred((PrimaryContext)_localctx, predIndex);
-    case 17:
+    case 19:
       return secondary_sempred((SecondaryContext)_localctx, predIndex);
     }
     return true;
@@ -2645,146 +2819,158 @@ class PainlessParser extends Parser {
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3M\u017f\4\2\t\2\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3M\u01a0\4\2\t\2\4"+
     "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
     "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
-    "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\3\2\7\2\60\n\2\f\2"+
-    "\16\2\63\13\2\3\2\3\2\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3?\n\3\3\3\3\3"+
-    "\3\3\3\3\3\3\3\3\5\3G\n\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3"+
-    "\5\3T\n\3\3\3\3\3\5\3X\n\3\3\3\3\3\5\3\\\n\3\3\3\3\3\3\3\5\3a\n\3\3\3"+
-    "\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3\3"+
-    "\3\3\3\3\3\3\3\3\3\6\3z\n\3\r\3\16\3{\3\3\3\3\3\3\3\3\3\3\3\3\3\3\5\3"+
-    "\u0085\n\3\3\4\3\4\5\4\u0089\n\4\3\5\3\5\7\5\u008d\n\5\f\5\16\5\u0090"+
-    "\13\5\3\5\3\5\3\6\3\6\3\7\3\7\5\7\u0098\n\7\3\b\3\b\3\t\3\t\3\t\3\t\7"+
-    "\t\u00a0\n\t\f\t\16\t\u00a3\13\t\3\n\3\n\3\n\7\n\u00a8\n\n\f\n\16\n\u00ab"+
-    "\13\n\3\13\3\13\3\13\3\13\3\f\3\f\3\f\5\f\u00b4\n\f\3\r\3\r\3\r\3\r\3"+
-    "\r\3\r\3\r\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\5\17"+
-    "\u00c8\n\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
-    "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
-    "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
-    "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\17"+
-    "\3\17\3\17\3\17\7\17\u0103\n\17\f\17\16\17\u0106\13\17\3\20\3\20\3\20"+
-    "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
-    "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\5\20\u0125"+
-    "\n\20\3\21\3\21\7\21\u0129\n\21\f\21\16\21\u012c\13\21\3\21\3\21\3\21"+
-    "\7\21\u0131\n\21\f\21\16\21\u0134\13\21\3\21\3\21\3\21\3\21\3\21\3\21"+
-    "\6\21\u013c\n\21\r\21\16\21\u013d\3\21\3\21\7\21\u0142\n\21\f\21\16\21"+
-    "\u0145\13\21\5\21\u0147\n\21\5\21\u0149\n\21\3\22\3\22\3\22\3\22\3\22"+
-    "\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\5\22\u015b\n\22"+
-    "\3\23\3\23\3\23\3\23\5\23\u0161\n\23\3\24\3\24\3\24\3\24\3\24\5\24\u0168"+
-    "\n\24\3\25\3\25\3\25\3\25\3\26\3\26\3\26\3\26\7\26\u0172\n\26\f\26\16"+
-    "\26\u0175\13\26\5\26\u0177\n\26\3\26\3\26\3\27\3\27\5\27\u017d\n\27\3"+
-    "\27\2\3\34\30\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,\2\16\4\2"+
-    "\26\26KK\3\3\r\r\3\2\66A\3\2\34\36\3\2\37 \3\2!#\3\2$\'\3\2(+\3\2\64\65"+
-    "\3\2BE\4\2\32\33\37 \3\2LM\u01a5\2\61\3\2\2\2\4\u0084\3\2\2\2\6\u0088"+
-    "\3\2\2\2\b\u008a\3\2\2\2\n\u0093\3\2\2\2\f\u0097\3\2\2\2\16\u0099\3\2"+
-    "\2\2\20\u009b\3\2\2\2\22\u00a4\3\2\2\2\24\u00ac\3\2\2\2\26\u00b0\3\2\2"+
-    "\2\30\u00b5\3\2\2\2\32\u00bc\3\2\2\2\34\u00c7\3\2\2\2\36\u0124\3\2\2\2"+
-    " \u0148\3\2\2\2\"\u015a\3\2\2\2$\u0160\3\2\2\2&\u0167\3\2\2\2(\u0169\3"+
-    "\2\2\2*\u016d\3\2\2\2,\u017c\3\2\2\2.\60\5\4\3\2/.\3\2\2\2\60\63\3\2\2"+
-    "\2\61/\3\2\2\2\61\62\3\2\2\2\62\64\3\2\2\2\63\61\3\2\2\2\64\65\7\2\2\3"+
-    "\65\3\3\2\2\2\66\67\7\16\2\2\678\7\t\2\289\5\34\17\29:\7\n\2\2:>\5\6\4"+
-    "\2;<\7\17\2\2<?\5\6\4\2=?\6\3\2\2>;\3\2\2\2>=\3\2\2\2?\u0085\3\2\2\2@"+
-    "A\7\20\2\2AB\7\t\2\2BC\5\34\17\2CF\7\n\2\2DG\5\6\4\2EG\5\n\6\2FD\3\2\2"+
-    "\2FE\3\2\2\2G\u0085\3\2\2\2HI\7\21\2\2IJ\5\b\5\2JK\7\20\2\2KL\7\t\2\2"+
-    "LM\5\34\17\2MN\7\n\2\2NO\5\32\16\2O\u0085\3\2\2\2PQ\7\22\2\2QS\7\t\2\2"+
-    "RT\5\f\7\2SR\3\2\2\2ST\3\2\2\2TU\3\2\2\2UW\7\r\2\2VX\5\34\17\2WV\3\2\2"+
-    "\2WX\3\2\2\2XY\3\2\2\2Y[\7\r\2\2Z\\\5\16\b\2[Z\3\2\2\2[\\\3\2\2\2\\]\3"+
-    "\2\2\2]`\7\n\2\2^a\5\6\4\2_a\5\n\6\2`^\3\2\2\2`_\3\2\2\2a\u0085\3\2\2"+
-    "\2bc\7\22\2\2cd\7\t\2\2de\5\22\n\2ef\7K\2\2fg\7\62\2\2gh\5\34\17\2hi\7"+
-    "\n\2\2ij\5\6\4\2j\u0085\3\2\2\2kl\5\20\t\2lm\5\32\16\2m\u0085\3\2\2\2"+
-    "no\7\23\2\2o\u0085\5\32\16\2pq\7\24\2\2q\u0085\5\32\16\2rs\7\25\2\2st"+
-    "\5\34\17\2tu\5\32\16\2u\u0085\3\2\2\2vw\7\27\2\2wy\5\b\5\2xz\5\30\r\2"+
-    "yx\3\2\2\2z{\3\2\2\2{y\3\2\2\2{|\3\2\2\2|\u0085\3\2\2\2}~\7\31\2\2~\177"+
-    "\5\34\17\2\177\u0080\5\32\16\2\u0080\u0085\3\2\2\2\u0081\u0082\5\34\17"+
-    "\2\u0082\u0083\5\32\16\2\u0083\u0085\3\2\2\2\u0084\66\3\2\2\2\u0084@\3"+
-    "\2\2\2\u0084H\3\2\2\2\u0084P\3\2\2\2\u0084b\3\2\2\2\u0084k\3\2\2\2\u0084"+
-    "n\3\2\2\2\u0084p\3\2\2\2\u0084r\3\2\2\2\u0084v\3\2\2\2\u0084}\3\2\2\2"+
-    "\u0084\u0081\3\2\2\2\u0085\5\3\2\2\2\u0086\u0089\5\b\5\2\u0087\u0089\5"+
-    "\4\3\2\u0088\u0086\3\2\2\2\u0088\u0087\3\2\2\2\u0089\7\3\2\2\2\u008a\u008e"+
-    "\7\5\2\2\u008b\u008d\5\4\3\2\u008c\u008b\3\2\2\2\u008d\u0090\3\2\2\2\u008e"+
-    "\u008c\3\2\2\2\u008e\u008f\3\2\2\2\u008f\u0091\3\2\2\2\u0090\u008e\3\2"+
-    "\2\2\u0091\u0092\7\6\2\2\u0092\t\3\2\2\2\u0093\u0094\7\r\2\2\u0094\13"+
-    "\3\2\2\2\u0095\u0098\5\20\t\2\u0096\u0098\5\34\17\2\u0097\u0095\3\2\2"+
-    "\2\u0097\u0096\3\2\2\2\u0098\r\3\2\2\2\u0099\u009a\5\34\17\2\u009a\17"+
-    "\3\2\2\2\u009b\u009c\5\22\n\2\u009c\u00a1\5\26\f\2\u009d\u009e\7\f\2\2"+
-    "\u009e\u00a0\5\26\f\2\u009f\u009d\3\2\2\2\u00a0\u00a3\3\2\2\2\u00a1\u009f"+
-    "\3\2\2\2\u00a1\u00a2\3\2\2\2\u00a2\21\3\2\2\2\u00a3\u00a1\3\2\2\2\u00a4"+
-    "\u00a9\7J\2\2\u00a5\u00a6\7\7\2\2\u00a6\u00a8\7\b\2\2\u00a7\u00a5\3\2"+
-    "\2\2\u00a8\u00ab\3\2\2\2\u00a9\u00a7\3\2\2\2\u00a9\u00aa\3\2\2\2\u00aa"+
-    "\23\3\2\2\2\u00ab\u00a9\3\2\2\2\u00ac\u00ad\7J\2\2\u00ad\u00ae\7\63\2"+
-    "\2\u00ae\u00af\t\2\2\2\u00af\25\3\2\2\2\u00b0\u00b3\7K\2\2\u00b1\u00b2"+
-    "\7\66\2\2\u00b2\u00b4\5\34\17\2\u00b3\u00b1\3\2\2\2\u00b3\u00b4\3\2\2"+
-    "\2\u00b4\27\3\2\2\2\u00b5\u00b6\7\30\2\2\u00b6\u00b7\7\t\2\2\u00b7\u00b8"+
-    "\7J\2\2\u00b8\u00b9\7K\2\2\u00b9\u00ba\7\n\2\2\u00ba\u00bb\5\b\5\2\u00bb"+
-    "\31\3\2\2\2\u00bc\u00bd\t\3\2\2\u00bd\33\3\2\2\2\u00be\u00bf\b\17\1\2"+
-    "\u00bf\u00c0\5 \21\2\u00c0\u00c1\t\4\2\2\u00c1\u00c2\5\34\17\3\u00c2\u00c3"+
-    "\b\17\1\2\u00c3\u00c8\3\2\2\2\u00c4\u00c5\5\36\20\2\u00c5\u00c6\b\17\1"+
-    "\2\u00c6\u00c8\3\2\2\2\u00c7\u00be\3\2\2\2\u00c7\u00c4\3\2\2\2\u00c8\u0104"+
-    "\3\2\2\2\u00c9\u00ca\f\16\2\2\u00ca\u00cb\t\5\2\2\u00cb\u00cc\5\34\17"+
-    "\17\u00cc\u00cd\b\17\1\2\u00cd\u0103\3\2\2\2\u00ce\u00cf\f\r\2\2\u00cf"+
-    "\u00d0\t\6\2\2\u00d0\u00d1\5\34\17\16\u00d1\u00d2\b\17\1\2\u00d2\u0103"+
-    "\3\2\2\2\u00d3\u00d4\f\f\2\2\u00d4\u00d5\t\7\2\2\u00d5\u00d6\5\34\17\r"+
-    "\u00d6\u00d7\b\17\1\2\u00d7\u0103\3\2\2\2\u00d8\u00d9\f\13\2\2\u00d9\u00da"+
-    "\t\b\2\2\u00da\u00db\5\34\17\f\u00db\u00dc\b\17\1\2\u00dc\u0103\3\2\2"+
-    "\2\u00dd\u00de\f\n\2\2\u00de\u00df\t\t\2\2\u00df\u00e0\5\34\17\13\u00e0"+
-    "\u00e1\b\17\1\2\u00e1\u0103\3\2\2\2\u00e2\u00e3\f\t\2\2\u00e3\u00e4\7"+
-    ",\2\2\u00e4\u00e5\5\34\17\n\u00e5\u00e6\b\17\1\2\u00e6\u0103\3\2\2\2\u00e7"+
-    "\u00e8\f\b\2\2\u00e8\u00e9\7-\2\2\u00e9\u00ea\5\34\17\t\u00ea\u00eb\b"+
-    "\17\1\2\u00eb\u0103\3\2\2\2\u00ec\u00ed\f\7\2\2\u00ed\u00ee\7.\2\2\u00ee"+
-    "\u00ef\5\34\17\b\u00ef\u00f0\b\17\1\2\u00f0\u0103\3\2\2\2\u00f1\u00f2"+
-    "\f\6\2\2\u00f2\u00f3\7/\2\2\u00f3\u00f4\5\34\17\7\u00f4\u00f5\b\17\1\2"+
-    "\u00f5\u0103\3\2\2\2\u00f6\u00f7\f\5\2\2\u00f7\u00f8\7\60\2\2\u00f8\u00f9"+
-    "\5\34\17\6\u00f9\u00fa\b\17\1\2\u00fa\u0103\3\2\2\2\u00fb\u00fc\f\4\2"+
-    "\2\u00fc\u00fd\7\61\2\2\u00fd\u00fe\5\34\17\2\u00fe\u00ff\7\62\2\2\u00ff"+
-    "\u0100\5\34\17\4\u0100\u0101\b\17\1\2\u0101\u0103\3\2\2\2\u0102\u00c9"+
-    "\3\2\2\2\u0102\u00ce\3\2\2\2\u0102\u00d3\3\2\2\2\u0102\u00d8\3\2\2\2\u0102"+
-    "\u00dd\3\2\2\2\u0102\u00e2\3\2\2\2\u0102\u00e7\3\2\2\2\u0102\u00ec\3\2"+
-    "\2\2\u0102\u00f1\3\2\2\2\u0102\u00f6\3\2\2\2\u0102\u00fb\3\2\2\2\u0103"+
-    "\u0106\3\2\2\2\u0104\u0102\3\2\2\2\u0104\u0105\3\2\2\2\u0105\35\3\2\2"+
-    "\2\u0106\u0104\3\2\2\2\u0107\u0108\6\20\16\3\u0108\u0109\t\n\2\2\u0109"+
-    "\u0125\5 \21\2\u010a\u010b\6\20\17\3\u010b\u010c\5 \21\2\u010c\u010d\t"+
-    "\n\2\2\u010d\u0125\3\2\2\2\u010e\u010f\6\20\20\3\u010f\u0125\5 \21\2\u0110"+
-    "\u0111\6\20\21\3\u0111\u0112\t\13\2\2\u0112\u0125\b\20\1\2\u0113\u0114"+
-    "\6\20\22\3\u0114\u0115\7G\2\2\u0115\u0125\b\20\1\2\u0116\u0117\6\20\23"+
-    "\3\u0117\u0118\7H\2\2\u0118\u0125\b\20\1\2\u0119\u011a\6\20\24\3\u011a"+
-    "\u011b\7I\2\2\u011b\u0125\b\20\1\2\u011c\u011d\6\20\25\3\u011d\u011e\t"+
-    "\f\2\2\u011e\u0125\5\36\20\2\u011f\u0120\7\t\2\2\u0120\u0121\5\22\n\2"+
-    "\u0121\u0122\7\n\2\2\u0122\u0123\5\36\20\2\u0123\u0125\3\2\2\2\u0124\u0107"+
-    "\3\2\2\2\u0124\u010a\3\2\2\2\u0124\u010e\3\2\2\2\u0124\u0110\3\2\2\2\u0124"+
-    "\u0113\3\2\2\2\u0124\u0116\3\2\2\2\u0124\u0119\3\2\2\2\u0124\u011c\3\2"+
-    "\2\2\u0124\u011f\3\2\2\2\u0125\37\3\2\2\2\u0126\u012a\5\"\22\2\u0127\u0129"+
-    "\5$\23\2\u0128\u0127\3\2\2\2\u0129\u012c\3\2\2\2\u012a\u0128\3\2\2\2\u012a"+
-    "\u012b\3\2\2\2\u012b\u0149\3\2\2\2\u012c\u012a\3\2\2\2\u012d\u012e\5\22"+
-    "\n\2\u012e\u0132\5&\24\2\u012f\u0131\5$\23\2\u0130\u012f\3\2\2\2\u0131"+
-    "\u0134\3\2\2\2\u0132\u0130\3\2\2\2\u0132\u0133\3\2\2\2\u0133\u0149\3\2"+
-    "\2\2\u0134\u0132\3\2\2\2\u0135\u0136\7\26\2\2\u0136\u013b\7J\2\2\u0137"+
-    "\u0138\7\7\2\2\u0138\u0139\5\34\17\2\u0139\u013a\7\b\2\2\u013a\u013c\3"+
-    "\2\2\2\u013b\u0137\3\2\2\2\u013c\u013d\3\2\2\2\u013d\u013b\3\2\2\2\u013d"+
-    "\u013e\3\2\2\2\u013e\u0146\3\2\2\2\u013f\u0143\5&\24\2\u0140\u0142\5$"+
-    "\23\2\u0141\u0140\3\2\2\2\u0142\u0145\3\2\2\2\u0143\u0141\3\2\2\2\u0143"+
-    "\u0144\3\2\2\2\u0144\u0147\3\2\2\2\u0145\u0143\3\2\2\2\u0146\u013f\3\2"+
-    "\2\2\u0146\u0147\3\2\2\2\u0147\u0149\3\2\2\2\u0148\u0126\3\2\2\2\u0148"+
-    "\u012d\3\2\2\2\u0148\u0135\3\2\2\2\u0149!\3\2\2\2\u014a\u014b\6\22\26"+
-    "\3\u014b\u014c\7\t\2\2\u014c\u014d\5\34\17\2\u014d\u014e\7\n\2\2\u014e"+
-    "\u014f\b\22\1\2\u014f\u015b\3\2\2\2\u0150\u0151\6\22\27\3\u0151\u0152"+
-    "\7\t\2\2\u0152\u0153\5\36\20\2\u0153\u0154\7\n\2\2\u0154\u015b\3\2\2\2"+
-    "\u0155\u015b\7F\2\2\u0156\u015b\7K\2\2\u0157\u0158\7\26\2\2\u0158\u0159"+
-    "\7J\2\2\u0159\u015b\5*\26\2\u015a\u014a\3\2\2\2\u015a\u0150\3\2\2\2\u015a"+
-    "\u0155\3\2\2\2\u015a\u0156\3\2\2\2\u015a\u0157\3\2\2\2\u015b#\3\2\2\2"+
-    "\u015c\u015d\6\23\30\3\u015d\u0161\5&\24\2\u015e\u015f\6\23\31\3\u015f"+
-    "\u0161\5(\25\2\u0160\u015c\3\2\2\2\u0160\u015e\3\2\2\2\u0161%\3\2\2\2"+
-    "\u0162\u0163\7\13\2\2\u0163\u0164\7M\2\2\u0164\u0168\5*\26\2\u0165\u0166"+
-    "\7\13\2\2\u0166\u0168\t\r\2\2\u0167\u0162\3\2\2\2\u0167\u0165\3\2\2\2"+
-    "\u0168\'\3\2\2\2\u0169\u016a\7\7\2\2\u016a\u016b\5\34\17\2\u016b\u016c"+
-    "\7\b\2\2\u016c)\3\2\2\2\u016d\u0176\7\t\2\2\u016e\u0173\5,\27\2\u016f"+
-    "\u0170\7\f\2\2\u0170\u0172\5,\27\2\u0171\u016f\3\2\2\2\u0172\u0175\3\2"+
-    "\2\2\u0173\u0171\3\2\2\2\u0173\u0174\3\2\2\2\u0174\u0177\3\2\2\2\u0175"+
-    "\u0173\3\2\2\2\u0176\u016e\3\2\2\2\u0176\u0177\3\2\2\2\u0177\u0178\3\2"+
-    "\2\2\u0178\u0179\7\n\2\2\u0179+\3\2\2\2\u017a\u017d\5\34\17\2\u017b\u017d"+
-    "\5\24\13\2\u017c\u017a\3\2\2\2\u017c\u017b\3\2\2\2\u017d-\3\2\2\2!\61"+
-    ">FSW[`{\u0084\u0088\u008e\u0097\u00a1\u00a9\u00b3\u00c7\u0102\u0104\u0124"+
-    "\u012a\u0132\u013d\u0143\u0146\u0148\u015a\u0160\u0167\u0173\u0176\u017c";
+    "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+
+    "\3\2\7\2\64\n\2\f\2\16\2\67\13\2\3\2\7\2:\n\2\f\2\16\2=\13\2\3\2\3\2\3"+
+    "\3\3\3\3\3\3\3\3\3\3\4\3\4\3\4\3\4\3\4\3\4\3\4\7\4M\n\4\f\4\16\4P\13\4"+
+    "\5\4R\n\4\3\4\3\4\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\5\5^\n\5\3\5\3\5\3\5"+
+    "\3\5\3\5\3\5\5\5f\n\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\5\5"+
+    "s\n\5\3\5\3\5\5\5w\n\5\3\5\3\5\5\5{\n\5\3\5\3\5\3\5\5\5\u0080\n\5\3\5"+
+    "\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3"+
+    "\5\3\5\3\5\3\5\3\5\6\5\u0099\n\5\r\5\16\5\u009a\3\5\3\5\3\5\3\5\3\5\3"+
+    "\5\3\5\5\5\u00a4\n\5\3\6\3\6\5\6\u00a8\n\6\3\7\3\7\7\7\u00ac\n\7\f\7\16"+
+    "\7\u00af\13\7\3\7\3\7\3\b\3\b\3\t\3\t\5\t\u00b7\n\t\3\n\3\n\3\13\3\13"+
+    "\3\13\3\13\7\13\u00bf\n\13\f\13\16\13\u00c2\13\13\3\f\3\f\3\f\7\f\u00c7"+
+    "\n\f\f\f\16\f\u00ca\13\f\3\r\3\r\3\r\3\r\3\16\3\16\3\16\5\16\u00d3\n\16"+
+    "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\20\3\20\3\21\3\21\3\21\3\21\3\21"+
+    "\3\21\3\21\3\21\3\21\5\21\u00e7\n\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
+    "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
+    "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
+    "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
+    "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\7\21\u0122\n\21\f\21\16\21\u0125"+
+    "\13\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22"+
+    "\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22"+
+    "\3\22\3\22\5\22\u0144\n\22\3\23\3\23\7\23\u0148\n\23\f\23\16\23\u014b"+
+    "\13\23\3\23\3\23\3\23\7\23\u0150\n\23\f\23\16\23\u0153\13\23\3\23\3\23"+
+    "\3\23\3\23\3\23\3\23\6\23\u015b\n\23\r\23\16\23\u015c\3\23\3\23\7\23\u0161"+
+    "\n\23\f\23\16\23\u0164\13\23\5\23\u0166\n\23\5\23\u0168\n\23\3\24\3\24"+
+    "\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\24"+
+    "\3\24\3\24\5\24\u017c\n\24\3\25\3\25\3\25\3\25\5\25\u0182\n\25\3\26\3"+
+    "\26\3\26\3\26\3\26\5\26\u0189\n\26\3\27\3\27\3\27\3\27\3\30\3\30\3\30"+
+    "\3\30\7\30\u0193\n\30\f\30\16\30\u0196\13\30\5\30\u0198\n\30\3\30\3\30"+
+    "\3\31\3\31\5\31\u019e\n\31\3\31\2\3 \32\2\4\6\b\n\f\16\20\22\24\26\30"+
+    "\32\34\36 \"$&(*,.\60\2\16\4\2\26\26KK\3\3\r\r\3\2\66A\3\2\34\36\3\2\37"+
+    " \3\2!#\3\2$\'\3\2(+\3\2\64\65\3\2BE\4\2\32\33\37 \3\2LM\u01c8\2\65\3"+
+    "\2\2\2\4@\3\2\2\2\6E\3\2\2\2\b\u00a3\3\2\2\2\n\u00a7\3\2\2\2\f\u00a9\3"+
+    "\2\2\2\16\u00b2\3\2\2\2\20\u00b6\3\2\2\2\22\u00b8\3\2\2\2\24\u00ba\3\2"+
+    "\2\2\26\u00c3\3\2\2\2\30\u00cb\3\2\2\2\32\u00cf\3\2\2\2\34\u00d4\3\2\2"+
+    "\2\36\u00db\3\2\2\2 \u00e6\3\2\2\2\"\u0143\3\2\2\2$\u0167\3\2\2\2&\u017b"+
+    "\3\2\2\2(\u0181\3\2\2\2*\u0188\3\2\2\2,\u018a\3\2\2\2.\u018e\3\2\2\2\60"+
+    "\u019d\3\2\2\2\62\64\5\4\3\2\63\62\3\2\2\2\64\67\3\2\2\2\65\63\3\2\2\2"+
+    "\65\66\3\2\2\2\66;\3\2\2\2\67\65\3\2\2\28:\5\b\5\298\3\2\2\2:=\3\2\2\2"+
+    ";9\3\2\2\2;<\3\2\2\2<>\3\2\2\2=;\3\2\2\2>?\7\2\2\3?\3\3\2\2\2@A\5\26\f"+
+    "\2AB\7K\2\2BC\5\6\4\2CD\5\f\7\2D\5\3\2\2\2EQ\7\t\2\2FG\5\26\f\2GN\7K\2"+
+    "\2HI\7\f\2\2IJ\5\26\f\2JK\7K\2\2KM\3\2\2\2LH\3\2\2\2MP\3\2\2\2NL\3\2\2"+
+    "\2NO\3\2\2\2OR\3\2\2\2PN\3\2\2\2QF\3\2\2\2QR\3\2\2\2RS\3\2\2\2ST\7\n\2"+
+    "\2T\7\3\2\2\2UV\7\16\2\2VW\7\t\2\2WX\5 \21\2XY\7\n\2\2Y]\5\n\6\2Z[\7\17"+
+    "\2\2[^\5\n\6\2\\^\6\5\2\2]Z\3\2\2\2]\\\3\2\2\2^\u00a4\3\2\2\2_`\7\20\2"+
+    "\2`a\7\t\2\2ab\5 \21\2be\7\n\2\2cf\5\n\6\2df\5\16\b\2ec\3\2\2\2ed\3\2"+
+    "\2\2f\u00a4\3\2\2\2gh\7\21\2\2hi\5\f\7\2ij\7\20\2\2jk\7\t\2\2kl\5 \21"+
+    "\2lm\7\n\2\2mn\5\36\20\2n\u00a4\3\2\2\2op\7\22\2\2pr\7\t\2\2qs\5\20\t"+
+    "\2rq\3\2\2\2rs\3\2\2\2st\3\2\2\2tv\7\r\2\2uw\5 \21\2vu\3\2\2\2vw\3\2\2"+
+    "\2wx\3\2\2\2xz\7\r\2\2y{\5\22\n\2zy\3\2\2\2z{\3\2\2\2{|\3\2\2\2|\177\7"+
+    "\n\2\2}\u0080\5\n\6\2~\u0080\5\16\b\2\177}\3\2\2\2\177~\3\2\2\2\u0080"+
+    "\u00a4\3\2\2\2\u0081\u0082\7\22\2\2\u0082\u0083\7\t\2\2\u0083\u0084\5"+
+    "\26\f\2\u0084\u0085\7K\2\2\u0085\u0086\7\62\2\2\u0086\u0087\5 \21\2\u0087"+
+    "\u0088\7\n\2\2\u0088\u0089\5\n\6\2\u0089\u00a4\3\2\2\2\u008a\u008b\5\24"+
+    "\13\2\u008b\u008c\5\36\20\2\u008c\u00a4\3\2\2\2\u008d\u008e\7\23\2\2\u008e"+
+    "\u00a4\5\36\20\2\u008f\u0090\7\24\2\2\u0090\u00a4\5\36\20\2\u0091\u0092"+
+    "\7\25\2\2\u0092\u0093\5 \21\2\u0093\u0094\5\36\20\2\u0094\u00a4\3\2\2"+
+    "\2\u0095\u0096\7\27\2\2\u0096\u0098\5\f\7\2\u0097\u0099\5\34\17\2\u0098"+
+    "\u0097\3\2\2\2\u0099\u009a\3\2\2\2\u009a\u0098\3\2\2\2\u009a\u009b\3\2"+
+    "\2\2\u009b\u00a4\3\2\2\2\u009c\u009d\7\31\2\2\u009d\u009e\5 \21\2\u009e"+
+    "\u009f\5\36\20\2\u009f\u00a4\3\2\2\2\u00a0\u00a1\5 \21\2\u00a1\u00a2\5"+
+    "\36\20\2\u00a2\u00a4\3\2\2\2\u00a3U\3\2\2\2\u00a3_\3\2\2\2\u00a3g\3\2"+
+    "\2\2\u00a3o\3\2\2\2\u00a3\u0081\3\2\2\2\u00a3\u008a\3\2\2\2\u00a3\u008d"+
+    "\3\2\2\2\u00a3\u008f\3\2\2\2\u00a3\u0091\3\2\2\2\u00a3\u0095\3\2\2\2\u00a3"+
+    "\u009c\3\2\2\2\u00a3\u00a0\3\2\2\2\u00a4\t\3\2\2\2\u00a5\u00a8\5\f\7\2"+
+    "\u00a6\u00a8\5\b\5\2\u00a7\u00a5\3\2\2\2\u00a7\u00a6\3\2\2\2\u00a8\13"+
+    "\3\2\2\2\u00a9\u00ad\7\5\2\2\u00aa\u00ac\5\b\5\2\u00ab\u00aa\3\2\2\2\u00ac"+
+    "\u00af\3\2\2\2\u00ad\u00ab\3\2\2\2\u00ad\u00ae\3\2\2\2\u00ae\u00b0\3\2"+
+    "\2\2\u00af\u00ad\3\2\2\2\u00b0\u00b1\7\6\2\2\u00b1\r\3\2\2\2\u00b2\u00b3"+
+    "\7\r\2\2\u00b3\17\3\2\2\2\u00b4\u00b7\5\24\13\2\u00b5\u00b7\5 \21\2\u00b6"+
+    "\u00b4\3\2\2\2\u00b6\u00b5\3\2\2\2\u00b7\21\3\2\2\2\u00b8\u00b9\5 \21"+
+    "\2\u00b9\23\3\2\2\2\u00ba\u00bb\5\26\f\2\u00bb\u00c0\5\32\16\2\u00bc\u00bd"+
+    "\7\f\2\2\u00bd\u00bf\5\32\16\2\u00be\u00bc\3\2\2\2\u00bf\u00c2\3\2\2\2"+
+    "\u00c0\u00be\3\2\2\2\u00c0\u00c1\3\2\2\2\u00c1\25\3\2\2\2\u00c2\u00c0"+
+    "\3\2\2\2\u00c3\u00c8\7J\2\2\u00c4\u00c5\7\7\2\2\u00c5\u00c7\7\b\2\2\u00c6"+
+    "\u00c4\3\2\2\2\u00c7\u00ca\3\2\2\2\u00c8\u00c6\3\2\2\2\u00c8\u00c9\3\2"+
+    "\2\2\u00c9\27\3\2\2\2\u00ca\u00c8\3\2\2\2\u00cb\u00cc\7J\2\2\u00cc\u00cd"+
+    "\7\63\2\2\u00cd\u00ce\t\2\2\2\u00ce\31\3\2\2\2\u00cf\u00d2\7K\2\2\u00d0"+
+    "\u00d1\7\66\2\2\u00d1\u00d3\5 \21\2\u00d2\u00d0\3\2\2\2\u00d2\u00d3\3"+
+    "\2\2\2\u00d3\33\3\2\2\2\u00d4\u00d5\7\30\2\2\u00d5\u00d6\7\t\2\2\u00d6"+
+    "\u00d7\7J\2\2\u00d7\u00d8\7K\2\2\u00d8\u00d9\7\n\2\2\u00d9\u00da\5\f\7"+
+    "\2\u00da\35\3\2\2\2\u00db\u00dc\t\3\2\2\u00dc\37\3\2\2\2\u00dd\u00de\b"+
+    "\21\1\2\u00de\u00df\5$\23\2\u00df\u00e0\t\4\2\2\u00e0\u00e1\5 \21\3\u00e1"+
+    "\u00e2\b\21\1\2\u00e2\u00e7\3\2\2\2\u00e3\u00e4\5\"\22\2\u00e4\u00e5\b"+
+    "\21\1\2\u00e5\u00e7\3\2\2\2\u00e6\u00dd\3\2\2\2\u00e6\u00e3\3\2\2\2\u00e7"+
+    "\u0123\3\2\2\2\u00e8\u00e9\f\16\2\2\u00e9\u00ea\t\5\2\2\u00ea\u00eb\5"+
+    " \21\17\u00eb\u00ec\b\21\1\2\u00ec\u0122\3\2\2\2\u00ed\u00ee\f\r\2\2\u00ee"+
+    "\u00ef\t\6\2\2\u00ef\u00f0\5 \21\16\u00f0\u00f1\b\21\1\2\u00f1\u0122\3"+
+    "\2\2\2\u00f2\u00f3\f\f\2\2\u00f3\u00f4\t\7\2\2\u00f4\u00f5\5 \21\r\u00f5"+
+    "\u00f6\b\21\1\2\u00f6\u0122\3\2\2\2\u00f7\u00f8\f\13\2\2\u00f8\u00f9\t"+
+    "\b\2\2\u00f9\u00fa\5 \21\f\u00fa\u00fb\b\21\1\2\u00fb\u0122\3\2\2\2\u00fc"+
+    "\u00fd\f\n\2\2\u00fd\u00fe\t\t\2\2\u00fe\u00ff\5 \21\13\u00ff\u0100\b"+
+    "\21\1\2\u0100\u0122\3\2\2\2\u0101\u0102\f\t\2\2\u0102\u0103\7,\2\2\u0103"+
+    "\u0104\5 \21\n\u0104\u0105\b\21\1\2\u0105\u0122\3\2\2\2\u0106\u0107\f"+
+    "\b\2\2\u0107\u0108\7-\2\2\u0108\u0109\5 \21\t\u0109\u010a\b\21\1\2\u010a"+
+    "\u0122\3\2\2\2\u010b\u010c\f\7\2\2\u010c\u010d\7.\2\2\u010d\u010e\5 \21"+
+    "\b\u010e\u010f\b\21\1\2\u010f\u0122\3\2\2\2\u0110\u0111\f\6\2\2\u0111"+
+    "\u0112\7/\2\2\u0112\u0113\5 \21\7\u0113\u0114\b\21\1\2\u0114\u0122\3\2"+
+    "\2\2\u0115\u0116\f\5\2\2\u0116\u0117\7\60\2\2\u0117\u0118\5 \21\6\u0118"+
+    "\u0119\b\21\1\2\u0119\u0122\3\2\2\2\u011a\u011b\f\4\2\2\u011b\u011c\7"+
+    "\61\2\2\u011c\u011d\5 \21\2\u011d\u011e\7\62\2\2\u011e\u011f\5 \21\4\u011f"+
+    "\u0120\b\21\1\2\u0120\u0122\3\2\2\2\u0121\u00e8\3\2\2\2\u0121\u00ed\3"+
+    "\2\2\2\u0121\u00f2\3\2\2\2\u0121\u00f7\3\2\2\2\u0121\u00fc\3\2\2\2\u0121"+
+    "\u0101\3\2\2\2\u0121\u0106\3\2\2\2\u0121\u010b\3\2\2\2\u0121\u0110\3\2"+
+    "\2\2\u0121\u0115\3\2\2\2\u0121\u011a\3\2\2\2\u0122\u0125\3\2\2\2\u0123"+
+    "\u0121\3\2\2\2\u0123\u0124\3\2\2\2\u0124!\3\2\2\2\u0125\u0123\3\2\2\2"+
+    "\u0126\u0127\6\22\16\3\u0127\u0128\t\n\2\2\u0128\u0144\5$\23\2\u0129\u012a"+
+    "\6\22\17\3\u012a\u012b\5$\23\2\u012b\u012c\t\n\2\2\u012c\u0144\3\2\2\2"+
+    "\u012d\u012e\6\22\20\3\u012e\u0144\5$\23\2\u012f\u0130\6\22\21\3\u0130"+
+    "\u0131\t\13\2\2\u0131\u0144\b\22\1\2\u0132\u0133\6\22\22\3\u0133\u0134"+
+    "\7G\2\2\u0134\u0144\b\22\1\2\u0135\u0136\6\22\23\3\u0136\u0137\7H\2\2"+
+    "\u0137\u0144\b\22\1\2\u0138\u0139\6\22\24\3\u0139\u013a\7I\2\2\u013a\u0144"+
+    "\b\22\1\2\u013b\u013c\6\22\25\3\u013c\u013d\t\f\2\2\u013d\u0144\5\"\22"+
+    "\2\u013e\u013f\7\t\2\2\u013f\u0140\5\26\f\2\u0140\u0141\7\n\2\2\u0141"+
+    "\u0142\5\"\22\2\u0142\u0144\3\2\2\2\u0143\u0126\3\2\2\2\u0143\u0129\3"+
+    "\2\2\2\u0143\u012d\3\2\2\2\u0143\u012f\3\2\2\2\u0143\u0132\3\2\2\2\u0143"+
+    "\u0135\3\2\2\2\u0143\u0138\3\2\2\2\u0143\u013b\3\2\2\2\u0143\u013e\3\2"+
+    "\2\2\u0144#\3\2\2\2\u0145\u0149\5&\24\2\u0146\u0148\5(\25\2\u0147\u0146"+
+    "\3\2\2\2\u0148\u014b\3\2\2\2\u0149\u0147\3\2\2\2\u0149\u014a\3\2\2\2\u014a"+
+    "\u0168\3\2\2\2\u014b\u0149\3\2\2\2\u014c\u014d\5\26\f\2\u014d\u0151\5"+
+    "*\26\2\u014e\u0150\5(\25\2\u014f\u014e\3\2\2\2\u0150\u0153\3\2\2\2\u0151"+
+    "\u014f\3\2\2\2\u0151\u0152\3\2\2\2\u0152\u0168\3\2\2\2\u0153\u0151\3\2"+
+    "\2\2\u0154\u0155\7\26\2\2\u0155\u015a\7J\2\2\u0156\u0157\7\7\2\2\u0157"+
+    "\u0158\5 \21\2\u0158\u0159\7\b\2\2\u0159\u015b\3\2\2\2\u015a\u0156\3\2"+
+    "\2\2\u015b\u015c\3\2\2\2\u015c\u015a\3\2\2\2\u015c\u015d\3\2\2\2\u015d"+
+    "\u0165\3\2\2\2\u015e\u0162\5*\26\2\u015f\u0161\5(\25\2\u0160\u015f\3\2"+
+    "\2\2\u0161\u0164\3\2\2\2\u0162\u0160\3\2\2\2\u0162\u0163\3\2\2\2\u0163"+
+    "\u0166\3\2\2\2\u0164\u0162\3\2\2\2\u0165\u015e\3\2\2\2\u0165\u0166\3\2"+
+    "\2\2\u0166\u0168\3\2\2\2\u0167\u0145\3\2\2\2\u0167\u014c\3\2\2\2\u0167"+
+    "\u0154\3\2\2\2\u0168%\3\2\2\2\u0169\u016a\6\24\26\3\u016a\u016b\7\t\2"+
+    "\2\u016b\u016c\5 \21\2\u016c\u016d\7\n\2\2\u016d\u016e\b\24\1\2\u016e"+
+    "\u017c\3\2\2\2\u016f\u0170\6\24\27\3\u0170\u0171\7\t\2\2\u0171\u0172\5"+
+    "\"\22\2\u0172\u0173\7\n\2\2\u0173\u017c\3\2\2\2\u0174\u017c\7F\2\2\u0175"+
+    "\u017c\7K\2\2\u0176\u0177\7K\2\2\u0177\u017c\5.\30\2\u0178\u0179\7\26"+
+    "\2\2\u0179\u017a\7J\2\2\u017a\u017c\5.\30\2\u017b\u0169\3\2\2\2\u017b"+
+    "\u016f\3\2\2\2\u017b\u0174\3\2\2\2\u017b\u0175\3\2\2\2\u017b\u0176\3\2"+
+    "\2\2\u017b\u0178\3\2\2\2\u017c\'\3\2\2\2\u017d\u017e\6\25\30\3\u017e\u0182"+
+    "\5*\26\2\u017f\u0180\6\25\31\3\u0180\u0182\5,\27\2\u0181\u017d\3\2\2\2"+
+    "\u0181\u017f\3\2\2\2\u0182)\3\2\2\2\u0183\u0184\7\13\2\2\u0184\u0185\7"+
+    "M\2\2\u0185\u0189\5.\30\2\u0186\u0187\7\13\2\2\u0187\u0189\t\r\2\2\u0188"+
+    "\u0183\3\2\2\2\u0188\u0186\3\2\2\2\u0189+\3\2\2\2\u018a\u018b\7\7\2\2"+
+    "\u018b\u018c\5 \21\2\u018c\u018d\7\b\2\2\u018d-\3\2\2\2\u018e\u0197\7"+
+    "\t\2\2\u018f\u0194\5\60\31\2\u0190\u0191\7\f\2\2\u0191\u0193\5\60\31\2"+
+    "\u0192\u0190\3\2\2\2\u0193\u0196\3\2\2\2\u0194\u0192\3\2\2\2\u0194\u0195"+
+    "\3\2\2\2\u0195\u0198\3\2\2\2\u0196\u0194\3\2\2\2\u0197\u018f\3\2\2\2\u0197"+
+    "\u0198\3\2\2\2\u0198\u0199\3\2\2\2\u0199\u019a\7\n\2\2\u019a/\3\2\2\2"+
+    "\u019b\u019e\5 \21\2\u019c\u019e\5\30\r\2\u019d\u019b\3\2\2\2\u019d\u019c"+
+    "\3\2\2\2\u019e\61\3\2\2\2$\65;NQ]ervz\177\u009a\u00a3\u00a7\u00ad\u00b6"+
+    "\u00c0\u00c8\u00d2\u00e6\u0121\u0123\u0143\u0149\u0151\u015c\u0162\u0165"+
+    "\u0167\u017b\u0181\u0188\u0194\u0197\u019d";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
index f116f087c5c..08358384fca 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
@@ -18,6 +18,20 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitSource(PainlessParser.SourceContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitFunction(PainlessParser.FunctionContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitParameters(PainlessParser.ParametersContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
@@ -333,6 +347,13 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitVariable(PainlessParser.VariableContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitCalllocal(PainlessParser.CalllocalContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
index f0943743ef8..5c0adb5076a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
@@ -16,6 +16,18 @@ interface PainlessParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitSource(PainlessParser.SourceContext ctx);
+  /**
+   * Visit a parse tree produced by {@link PainlessParser#function}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitFunction(PainlessParser.FunctionContext ctx);
+  /**
+   * Visit a parse tree produced by {@link PainlessParser#parameters}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitParameters(PainlessParser.ParametersContext ctx);
   /**
    * Visit a parse tree produced by the {@code if}
    * labeled alternative in {@link PainlessParser#statement}.
@@ -320,6 +332,13 @@ interface PainlessParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitVariable(PainlessParser.VariableContext ctx);
+  /**
+   * Visit a parse tree produced by the {@code calllocal}
+   * labeled alternative in {@link PainlessParser#primary}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitCalllocal(PainlessParser.CalllocalContext ctx);
   /**
    * Visit a parse tree produced by the {@code newobject}
    * labeled alternative in {@link PainlessParser#primary}.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index 471cb7d64ea..c1fe02f9388 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -27,10 +27,13 @@ import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.RecognitionException;
 import org.antlr.v4.runtime.Recognizer;
 import org.antlr.v4.runtime.atn.PredictionMode;
+import org.antlr.v4.runtime.tree.TerminalNode;
 import org.elasticsearch.painless.CompilerSettings;
+import org.elasticsearch.painless.Locals.ExecuteReserved;
+import org.elasticsearch.painless.Locals.FunctionReserved;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Operation;
-import org.elasticsearch.painless.Variables.Reserved;
+import org.elasticsearch.painless.Locals.Reserved;
 import org.elasticsearch.painless.antlr.PainlessParser.AfterthoughtContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ArgumentContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ArgumentsContext;
@@ -41,6 +44,7 @@ import org.elasticsearch.painless.antlr.PainlessParser.BoolContext;
 import org.elasticsearch.painless.antlr.PainlessParser.BraceaccessContext;
 import org.elasticsearch.painless.antlr.PainlessParser.BreakContext;
 import org.elasticsearch.painless.antlr.PainlessParser.CallinvokeContext;
+import org.elasticsearch.painless.antlr.PainlessParser.CalllocalContext;
 import org.elasticsearch.painless.antlr.PainlessParser.CastContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ChainprecContext;
 import org.elasticsearch.painless.antlr.PainlessParser.CompContext;
@@ -62,6 +66,7 @@ import org.elasticsearch.painless.antlr.PainlessParser.FalseContext;
 import org.elasticsearch.painless.antlr.PainlessParser.FieldaccessContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ForContext;
 import org.elasticsearch.painless.antlr.PainlessParser.FuncrefContext;
+import org.elasticsearch.painless.antlr.PainlessParser.FunctionContext;
 import org.elasticsearch.painless.antlr.PainlessParser.IfContext;
 import org.elasticsearch.painless.antlr.PainlessParser.InitializerContext;
 import org.elasticsearch.painless.antlr.PainlessParser.NewarrayContext;
@@ -69,6 +74,7 @@ import org.elasticsearch.painless.antlr.PainlessParser.NewobjectContext;
 import org.elasticsearch.painless.antlr.PainlessParser.NullContext;
 import org.elasticsearch.painless.antlr.PainlessParser.NumericContext;
 import org.elasticsearch.painless.antlr.PainlessParser.OperatorContext;
+import org.elasticsearch.painless.antlr.PainlessParser.ParametersContext;
 import org.elasticsearch.painless.antlr.PainlessParser.PostContext;
 import org.elasticsearch.painless.antlr.PainlessParser.PreContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ReadContext;
@@ -104,7 +110,8 @@ import org.elasticsearch.painless.node.ENull;
 import org.elasticsearch.painless.node.ENumeric;
 import org.elasticsearch.painless.node.EUnary;
 import org.elasticsearch.painless.node.LBrace;
-import org.elasticsearch.painless.node.LCall;
+import org.elasticsearch.painless.node.LCallInvoke;
+import org.elasticsearch.painless.node.LCallLocal;
 import org.elasticsearch.painless.node.LCast;
 import org.elasticsearch.painless.node.LField;
 import org.elasticsearch.painless.node.LNewArray;
@@ -122,6 +129,7 @@ import org.elasticsearch.painless.node.SDo;
 import org.elasticsearch.painless.node.SEach;
 import org.elasticsearch.painless.node.SExpression;
 import org.elasticsearch.painless.node.SFor;
+import org.elasticsearch.painless.node.SFunction;
 import org.elasticsearch.painless.node.SIf;
 import org.elasticsearch.painless.node.SIfElse;
 import org.elasticsearch.painless.node.SReturn;
@@ -130,7 +138,9 @@ import org.elasticsearch.painless.node.SThrow;
 import org.elasticsearch.painless.node.STry;
 import org.elasticsearch.painless.node.SWhile;
 
+import java.util.ArrayDeque;
 import java.util.ArrayList;
+import java.util.Deque;
 import java.util.List;
 
 /**
@@ -138,19 +148,21 @@ import java.util.List;
  */
 public final class Walker extends PainlessParserBaseVisitor<Object> {
 
-    public static SSource buildPainlessTree(String name, String sourceText, Reserved reserved, CompilerSettings settings) {
-        return new Walker(name, sourceText, reserved, settings).source;
+    public static SSource buildPainlessTree(String sourceName, String sourceText, CompilerSettings settings) {
+        return new Walker(sourceName, sourceText, settings).source;
     }
 
-    private final Reserved reserved;
     private final SSource source;
     private final CompilerSettings settings;
     private final String sourceName;
+    private final String sourceText;
 
-    private Walker(String name, String sourceText, Reserved reserved, CompilerSettings settings) {
-        this.reserved = reserved;
+    private final Deque<Reserved> reserved = new ArrayDeque<>();
+
+    private Walker(String sourceName, String sourceText, CompilerSettings settings) {
         this.settings = settings;
-        this.sourceName = Location.computeSourceName(name, sourceText);
+        this.sourceName = Location.computeSourceName(sourceName, sourceText);
+        this.sourceText = sourceText;
         this.source = (SSource)visit(buildAntlrTree(sourceText));
     }
 
@@ -196,13 +208,51 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
 
     @Override
     public Object visitSource(SourceContext ctx) {
+        reserved.push(new ExecuteReserved());
+
+        List<SFunction> functions = new ArrayList<>();
+
+        for (FunctionContext function : ctx.function()) {
+            functions.add((SFunction)visit(function));
+        }
+
         List<AStatement> statements = new ArrayList<>();
 
         for (StatementContext statement : ctx.statement()) {
             statements.add((AStatement)visit(statement));
         }
 
-        return new SSource(location(ctx), statements);
+        return new SSource(sourceName, sourceText, (ExecuteReserved)reserved.pop(), location(ctx), functions, statements);
+    }
+
+    @Override
+    public Object visitFunction(FunctionContext ctx) {
+        reserved.push(new FunctionReserved());
+
+        String rtnType = ctx.decltype().getText();
+        String name = ctx.ID().getText();
+        List<String> paramTypes = new ArrayList<>();
+        List<String> paramNames = new ArrayList<>();
+        List<AStatement> statements = new ArrayList<>();
+
+        for (DecltypeContext decltype : ctx.parameters().decltype()) {
+            paramTypes.add(decltype.getText());
+        }
+
+        for (TerminalNode id : ctx.parameters().ID()) {
+            paramNames.add(id.getText());
+        }
+
+        for (StatementContext statement : ctx.block().statement()) {
+            statements.add((AStatement)visit(statement));
+        }
+
+        return new SFunction((FunctionReserved)reserved.pop(), location(ctx), rtnType, name, paramTypes, paramNames, statements);
+    }
+
+    @Override
+    public Object visitParameters(ParametersContext ctx) {
+        throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
     }
 
     @Override
@@ -221,18 +271,16 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
 
     @Override
     public Object visitWhile(WhileContext ctx) {
-        if (settings.getMaxLoopCounter() > 0) {
-            reserved.usesLoop();
-        }
+        reserved.peek().setMaxLoopCounter(settings.getMaxLoopCounter());
 
         AExpression expression = (AExpression)visitExpression(ctx.expression());
 
         if (ctx.trailer() != null) {
             SBlock block = (SBlock)visit(ctx.trailer());
 
-            return new SWhile(location(ctx), settings.getMaxLoopCounter(), expression, block);
+            return new SWhile(location(ctx), expression, block);
         } else if (ctx.empty() != null) {
-            return new SWhile(location(ctx), settings.getMaxLoopCounter(), expression, null);
+            return new SWhile(location(ctx), expression, null);
         } else {
             throw location(ctx).createError(new IllegalStateException(" Illegal tree structure."));
         }
@@ -240,21 +288,17 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
 
     @Override
     public Object visitDo(DoContext ctx) {
-        if (settings.getMaxLoopCounter() > 0) {
-            reserved.usesLoop();
-        }
+        reserved.peek().setMaxLoopCounter(settings.getMaxLoopCounter());
 
         AExpression expression = (AExpression)visitExpression(ctx.expression());
         SBlock block = (SBlock)visit(ctx.block());
 
-        return new SDo(location(ctx), settings.getMaxLoopCounter(), block, expression);
+        return new SDo(location(ctx), block, expression);
     }
 
     @Override
     public Object visitFor(ForContext ctx) {
-        if (settings.getMaxLoopCounter() > 0) {
-            reserved.usesLoop();
-        }
+        reserved.peek().setMaxLoopCounter(settings.getMaxLoopCounter());
 
         ANode initializer = ctx.initializer() == null ? null : (ANode)visit(ctx.initializer());
         AExpression expression = ctx.expression() == null ? null : (AExpression)visitExpression(ctx.expression());
@@ -263,9 +307,9 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         if (ctx.trailer() != null) {
             SBlock block = (SBlock)visit(ctx.trailer());
 
-            return new SFor(location(ctx), settings.getMaxLoopCounter(), initializer, expression, afterthought, block);
+            return new SFor(location(ctx), initializer, expression, afterthought, block);
         } else if (ctx.empty() != null) {
-            return new SFor(location(ctx), settings.getMaxLoopCounter(), initializer, expression, afterthought, null);
+            return new SFor(location(ctx), initializer, expression, afterthought, null);
         } else {
             throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
         }
@@ -273,16 +317,14 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
 
     @Override
     public Object visitEach(EachContext ctx) {
-        if (settings.getMaxLoopCounter() > 0) {
-            reserved.usesLoop();
-        }
+        reserved.peek().setMaxLoopCounter(settings.getMaxLoopCounter());
 
         String type = ctx.decltype().getText();
         String name = ctx.ID().getText();
         AExpression expression = (AExpression)visitExpression(ctx.expression());
         SBlock block = (SBlock)visit(ctx.trailer());
 
-        return new SEach(location(ctx), settings.getMaxLoopCounter(), type, name, expression, block);
+        return new SEach(location(ctx), type, name, expression, block);
     }
 
     @Override
@@ -791,7 +833,18 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         List<ALink> links = new ArrayList<>();
         links.add(new LVariable(location(ctx), name));
 
-        reserved.markReserved(name);
+        reserved.peek().markReserved(name);
+
+        return links;
+    }
+
+    @Override
+    public Object visitCalllocal(CalllocalContext ctx) {
+        String name = ctx.ID().getText();
+        @SuppressWarnings("unchecked")
+        List<AExpression> arguments = (List<AExpression>)visit(ctx.arguments());
+        List<ALink> links = new ArrayList<>();
+        links.add(new LCallLocal(location(ctx), name, arguments));
 
         return links;
     }
@@ -825,7 +878,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         @SuppressWarnings("unchecked")
         List<AExpression> arguments = (List<AExpression>)visit(ctx.arguments());
 
-        return new LCall(location(ctx), name, arguments);
+        return new LCallInvoke(location(ctx), name, arguments);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
index c5494b978d8..fbaaa83a6ad 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
@@ -23,7 +23,7 @@ import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.AnalyzerCaster;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -106,7 +106,7 @@ public abstract class AExpression extends ANode {
     /**
      * Checks for errors and collects data for the writing phase.
      */
-    abstract void analyze(Variables variables);
+    abstract void analyze(Locals locals);
 
     /**
      * Writes ASM based on the data collected during the analysis phase.
@@ -118,7 +118,7 @@ public abstract class AExpression extends ANode {
      * nodes with the constant variable set to a non-null value with {@link EConstant}.
      * @return The new child node for the parent node calling this method.
      */
-    AExpression cast(Variables variables) {
+    AExpression cast(Locals locals) {
         final Cast cast = AnalyzerCaster.getLegalCast(location, actual, expected, explicit, internal);
 
         if (cast == null) {
@@ -136,7 +136,7 @@ public abstract class AExpression extends ANode {
                 // will already be the same.
 
                 EConstant econstant = new EConstant(location, constant);
-                econstant.analyze(variables);
+                econstant.analyze(locals);
 
                 if (!expected.equals(econstant.actual)) {
                     throw createError(new IllegalStateException("Illegal tree structure."));
@@ -170,7 +170,7 @@ public abstract class AExpression extends ANode {
                     constant = AnalyzerCaster.constCast(location, constant, cast);
 
                     EConstant econstant = new EConstant(location, constant);
-                    econstant.analyze(variables);
+                    econstant.analyze(locals);
 
                     if (!expected.equals(econstant.actual)) {
                         throw createError(new IllegalStateException("Illegal tree structure."));
@@ -201,7 +201,7 @@ public abstract class AExpression extends ANode {
                     // the EConstant will already be the same.
 
                     EConstant econstant = new EConstant(location, constant);
-                    econstant.analyze(variables);
+                    econstant.analyze(locals);
 
                     if (!actual.equals(econstant.actual)) {
                         throw createError(new IllegalStateException("Illegal tree structure."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
index 19875ebeec5..6d52bfe0de1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
@@ -21,7 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -86,7 +86,7 @@ public abstract class ALink extends ANode {
      * def or a shortcut is used. Otherwise, returns itself.  This will be
      * updated into the {@link EChain} node's list of links.
      */
-    abstract ALink analyze(Variables variables);
+    abstract ALink analyze(Locals locals);
 
     /**
      * Write values before a load/store occurs such as an array index.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
index b9e6679f630..e34d174ac43 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
@@ -115,7 +115,7 @@ public abstract class AStatement extends ANode {
     /**
      * Checks for errors and collects data for the writing phase.
      */
-    abstract void analyze(Variables variables);
+    abstract void analyze(Locals locals);
 
     /**
      * Writes ASM based on the data collected during the analysis phase.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 32a6e2382f2..55c2cc18210 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -26,7 +26,7 @@ import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.Operation;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 
 /**
  * Represents a binary math expression.
@@ -48,35 +48,35 @@ public final class EBinary extends AExpression {
     }
 
     @Override
-    void analyze(Variables variables) {
+    void analyze(Locals locals) {
         if (operation == Operation.MUL) {
-            analyzeMul(variables);
+            analyzeMul(locals);
         } else if (operation == Operation.DIV) {
-            analyzeDiv(variables);
+            analyzeDiv(locals);
         } else if (operation == Operation.REM) {
-            analyzeRem(variables);
+            analyzeRem(locals);
         } else if (operation == Operation.ADD) {
-            analyzeAdd(variables);
+            analyzeAdd(locals);
         } else if (operation == Operation.SUB) {
-            analyzeSub(variables);
+            analyzeSub(locals);
         } else if (operation == Operation.LSH) {
-            analyzeLSH(variables);
+            analyzeLSH(locals);
         } else if (operation == Operation.RSH) {
-            analyzeRSH(variables);
+            analyzeRSH(locals);
         } else if (operation == Operation.USH) {
-            analyzeUSH(variables);
+            analyzeUSH(locals);
         } else if (operation == Operation.BWAND) {
-            analyzeBWAnd(variables);
+            analyzeBWAnd(locals);
         } else if (operation == Operation.XOR) {
-            analyzeXor(variables);
+            analyzeXor(locals);
         } else if (operation == Operation.BWOR) {
-            analyzeBWOr(variables);
+            analyzeBWOr(locals);
         } else {
             throw createError(new IllegalStateException("Illegal tree structure."));
         }
     }
 
-    private void analyzeMul(Variables variables) {
+    private void analyzeMul(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
@@ -112,7 +112,7 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeDiv(Variables variables) {
+    private void analyzeDiv(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
@@ -152,7 +152,7 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeRem(Variables variables) {
+    private void analyzeRem(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
@@ -192,7 +192,7 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeAdd(Variables variables) {
+    private void analyzeAdd(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
@@ -244,7 +244,7 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeSub(Variables variables) {
+    private void analyzeSub(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
@@ -280,7 +280,7 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeLSH(Variables variables) {
+    private void analyzeLSH(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
@@ -313,7 +313,7 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeRSH(Variables variables) {
+    private void analyzeRSH(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
@@ -346,7 +346,7 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeUSH(Variables variables) {
+    private void analyzeUSH(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
@@ -379,7 +379,7 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeBWAnd(Variables variables) {
+    private void analyzeBWAnd(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
@@ -411,7 +411,7 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeXor(Variables variables) {
+    private void analyzeXor(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
@@ -445,7 +445,7 @@ public final class EBinary extends AExpression {
         actual = promote;
     }
 
-    private void analyzeBWOr(Variables variables) {
+    private void analyzeBWOr(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
index dd221f040c4..5aa2daeeb34 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Operation;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -44,14 +44,14 @@ public final class EBool extends AExpression {
     }
 
     @Override
-    void analyze(Variables variables) {
+    void analyze(Locals locals) {
         left.expected = Definition.BOOLEAN_TYPE;
-        left.analyze(variables);
-        left = left.cast(variables);
+        left.analyze(locals);
+        left = left.cast(locals);
 
         right.expected = Definition.BOOLEAN_TYPE;
-        right.analyze(variables);
-        right = right.cast(variables);
+        right.analyze(locals);
+        right = right.cast(locals);
 
         if (left.constant != null && right.constant != null) {
             if (operation == Operation.AND) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
index 5065989600f..877e79549af 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
@@ -21,7 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -36,7 +36,7 @@ public final class EBoolean extends AExpression {
     }
 
     @Override
-    void analyze(Variables variables) {
+    void analyze(Locals locals) {
         actual = Definition.BOOLEAN_TYPE;
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
index 27974240125..7892b918ed1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
@@ -21,7 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -45,7 +45,7 @@ final class ECast extends AExpression {
     }
 
     @Override
-    void analyze(Variables variables) {
+    void analyze(Locals locals) {
         throw createError(new IllegalStateException("Illegal tree structure."));
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index 9b2968db565..e21a1dd3134 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -26,7 +26,7 @@ import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.Operation;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 import java.util.List;
@@ -59,20 +59,20 @@ public final class EChain extends AExpression {
     }
 
     @Override
-    void analyze(Variables variables) {
-        analyzeLinks(variables);
+    void analyze(Locals locals) {
+        analyzeLinks(locals);
         analyzeIncrDecr();
 
         if (operation != null) {
-            analyzeCompound(variables);
+            analyzeCompound(locals);
         } else if (expression != null) {
-            analyzeWrite(variables);
+            analyzeWrite(locals);
         } else {
             analyzeRead();
         }
     }
 
-    private void analyzeLinks(Variables variables) {
+    private void analyzeLinks(Locals variables) {
         ALink previous = null;
         int index = 0;
 
@@ -153,7 +153,7 @@ public final class EChain extends AExpression {
         }
     }
 
-    private void analyzeCompound(Variables variables) {
+    private void analyzeCompound(Locals variables) {
         ALink last = links.get(links.size() - 1);
 
         expression.analyze(variables);
@@ -214,7 +214,7 @@ public final class EChain extends AExpression {
         this.actual = read ? last.after : Definition.VOID_TYPE;
     }
 
-    private void analyzeWrite(Variables variables) {
+    private void analyzeWrite(Locals variables) {
         ALink last = links.get(links.size() - 1);
 
         // If the store node is a def node, we remove the cast to def from the expression
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index 407b59f92cf..ec84600d323 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -25,7 +25,7 @@ import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.Operation;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -56,29 +56,29 @@ public final class EComp extends AExpression {
     }
 
     @Override
-    void analyze(Variables variables) {
+    void analyze(Locals locals) {
         if (operation == Operation.EQ) {
-            analyzeEq(variables);
+            analyzeEq(locals);
         } else if (operation == Operation.EQR) {
-            analyzeEqR(variables);
+            analyzeEqR(locals);
         } else if (operation == Operation.NE) {
-            analyzeNE(variables);
+            analyzeNE(locals);
         } else if (operation == Operation.NER) {
-            analyzeNER(variables);
+            analyzeNER(locals);
         } else if (operation == Operation.GTE) {
-            analyzeGTE(variables);
+            analyzeGTE(locals);
         } else if (operation == Operation.GT) {
-            analyzeGT(variables);
+            analyzeGT(locals);
         } else if (operation == Operation.LTE) {
-            analyzeLTE(variables);
+            analyzeLTE(locals);
         } else if (operation == Operation.LT) {
-            analyzeLT(variables);
+            analyzeLT(locals);
         } else {
             throw createError(new IllegalStateException("Illegal tree structure."));
         }
     }
 
-    private void analyzeEq(Variables variables) {
+    private void analyzeEq(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
@@ -124,7 +124,7 @@ public final class EComp extends AExpression {
         actual = Definition.BOOLEAN_TYPE;
     }
 
-    private void analyzeEqR(Variables variables) {
+    private void analyzeEqR(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
@@ -166,7 +166,7 @@ public final class EComp extends AExpression {
         actual = Definition.BOOLEAN_TYPE;
     }
 
-    private void analyzeNE(Variables variables) {
+    private void analyzeNE(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
@@ -212,7 +212,7 @@ public final class EComp extends AExpression {
         actual = Definition.BOOLEAN_TYPE;
     }
 
-    private void analyzeNER(Variables variables) {
+    private void analyzeNER(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
@@ -254,7 +254,7 @@ public final class EComp extends AExpression {
         actual = Definition.BOOLEAN_TYPE;
     }
 
-    private void analyzeGTE(Variables variables) {
+    private void analyzeGTE(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
@@ -290,7 +290,7 @@ public final class EComp extends AExpression {
         actual = Definition.BOOLEAN_TYPE;
     }
 
-    private void analyzeGT(Variables variables) {
+    private void analyzeGT(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
@@ -326,7 +326,7 @@ public final class EComp extends AExpression {
         actual = Definition.BOOLEAN_TYPE;
     }
 
-    private void analyzeLTE(Variables variables) {
+    private void analyzeLTE(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
@@ -362,7 +362,7 @@ public final class EComp extends AExpression {
         actual = Definition.BOOLEAN_TYPE;
     }
 
-    private void analyzeLT(Variables variables) {
+    private void analyzeLT(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
index 65231c45ed3..024e0c800fc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
@@ -23,7 +23,7 @@ import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.AnalyzerCaster;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -45,10 +45,10 @@ public final class EConditional extends AExpression {
     }
 
     @Override
-    void analyze(Variables variables) {
+    void analyze(Locals locals) {
         condition.expected = Definition.BOOLEAN_TYPE;
-        condition.analyze(variables);
-        condition = condition.cast(variables);
+        condition.analyze(locals);
+        condition = condition.cast(locals);
 
         if (condition.constant != null) {
             throw createError(new IllegalArgumentException("Extraneous conditional statement."));
@@ -62,8 +62,8 @@ public final class EConditional extends AExpression {
         right.internal = internal;
         actual = expected;
 
-        left.analyze(variables);
-        right.analyze(variables);
+        left.analyze(locals);
+        right.analyze(locals);
 
         if (expected == null) {
             final Type promote = AnalyzerCaster.promoteConditional(left.actual, right.actual, left.constant, right.constant);
@@ -73,8 +73,8 @@ public final class EConditional extends AExpression {
             actual = promote;
         }
 
-        left = left.cast(variables);
-        right = right.cast(variables);
+        left = left.cast(locals);
+        right = right.cast(locals);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
index 8cb032c5c3f..267d32983b9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -38,7 +38,7 @@ final class EConstant extends AExpression {
     }
 
     @Override
-    void analyze(Variables variables) {
+    void analyze(Locals locals) {
         if (constant instanceof String) {
             actual = Definition.STRING_TYPE;
         } else if (constant instanceof Double) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
index e59002c2a3b..0fd7fe46b51 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
@@ -21,7 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -38,7 +38,7 @@ public final class EDecimal extends AExpression {
     }
 
     @Override
-    void analyze(Variables variables) {
+    void analyze(Locals locals) {
         if (value.endsWith("f") || value.endsWith("F")) {
             try {
                 constant = Float.parseFloat(value.substring(0, value.length() - 1));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
index 0a55727b5e9..007a5d59e59 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
@@ -21,7 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -40,7 +40,7 @@ public final class EExplicit extends AExpression {
     }
 
     @Override
-    void analyze(Variables variables) {
+    void analyze(Locals locals) {
         try {
             actual = Definition.getType(this.type);
         } catch (IllegalArgumentException exception) {
@@ -49,8 +49,8 @@ public final class EExplicit extends AExpression {
 
         child.expected = actual;
         child.explicit = true;
-        child.analyze(variables);
-        child = child.cast(variables);
+        child.analyze(locals);
+        child = child.cast(locals);
     }
 
     @Override
@@ -58,11 +58,11 @@ public final class EExplicit extends AExpression {
         throw createError(new IllegalStateException("Illegal tree structure."));
     }
 
-    AExpression cast(Variables variables) {
+    AExpression cast(Locals locals) {
         child.expected = expected;
         child.explicit = explicit;
         child.internal = internal;
 
-        return child.cast(variables);
+        return child.cast(locals);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index 38b7a1b8ddb..ec45f89868a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -23,7 +23,7 @@ import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.FunctionRef;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Type;
 
 import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE;
@@ -36,7 +36,7 @@ import java.lang.invoke.LambdaMetafactory;
 public class EFunctionRef extends AExpression {
     public final String type;
     public final String call;
-    
+
     private FunctionRef ref;
 
     public EFunctionRef(Location location, String type, String call) {
@@ -47,7 +47,7 @@ public class EFunctionRef extends AExpression {
     }
 
     @Override
-    void analyze(Variables variables) {
+    void analyze(Locals locals) {
         if (expected == null) {
             ref = null;
             actual = Definition.getType("String");
@@ -72,22 +72,22 @@ public class EFunctionRef extends AExpression {
             Type samMethodType = Type.getMethodType(ref.samMethodType.toMethodDescriptorString());
             Type interfaceType = Type.getMethodType(ref.interfaceMethodType.toMethodDescriptorString());
             if (ref.needsBridges()) {
-                writer.invokeDynamic(ref.invokedName, 
-                                     invokedType, 
-                                     LAMBDA_BOOTSTRAP_HANDLE, 
-                                     samMethodType, 
-                                     ref.implMethodASM, 
-                                     samMethodType, 
-                                     LambdaMetafactory.FLAG_BRIDGES, 
-                                     1, 
+                writer.invokeDynamic(ref.invokedName,
+                                     invokedType,
+                                     LAMBDA_BOOTSTRAP_HANDLE,
+                                     samMethodType,
+                                     ref.implMethodASM,
+                                     samMethodType,
+                                     LambdaMetafactory.FLAG_BRIDGES,
+                                     1,
                                      interfaceType);
             } else {
-                writer.invokeDynamic(ref.invokedName, 
-                                     invokedType, 
-                                     LAMBDA_BOOTSTRAP_HANDLE, 
-                                     samMethodType, 
-                                     ref.implMethodASM, 
-                                     samMethodType, 
+                writer.invokeDynamic(ref.invokedName,
+                                     invokedType,
+                                     LAMBDA_BOOTSTRAP_HANDLE,
+                                     samMethodType,
+                                     ref.implMethodASM,
+                                     samMethodType,
                                      0);
             }
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
index 3f934da4968..eefc09c5946 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
@@ -21,7 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Opcodes;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -35,7 +35,7 @@ public final class ENull extends AExpression {
     }
 
     @Override
-    void analyze(Variables variables) {
+    void analyze(Locals locals) {
         isNull = true;
 
         if (expected != null) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
index 53f3559c294..9abdc8f6a12 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Sort;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -41,7 +41,7 @@ public final class ENumeric extends AExpression {
     }
 
     @Override
-    void analyze(Variables variables) {
+    void analyze(Locals locals) {
         if (value.endsWith("d") || value.endsWith("D")) {
             if (radix != 10) {
                 throw createError(new IllegalStateException("Illegal tree structure."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index 89d654cdf1f..2d7d8d4fd49 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -25,7 +25,7 @@ import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.Operation;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -49,21 +49,21 @@ public final class EUnary extends AExpression {
     }
 
     @Override
-    void analyze(Variables variables) {
+    void analyze(Locals locals) {
         if (operation == Operation.NOT) {
-            analyzeNot(variables);
+            analyzeNot(locals);
         } else if (operation == Operation.BWNOT) {
-            analyzeBWNot(variables);
+            analyzeBWNot(locals);
         } else if (operation == Operation.ADD) {
-            analyzerAdd(variables);
+            analyzerAdd(locals);
         } else if (operation == Operation.SUB) {
-            analyzerSub(variables);
+            analyzerSub(locals);
         } else {
             throw createError(new IllegalStateException("Illegal tree structure."));
         }
     }
 
-    void analyzeNot(Variables variables) {
+    void analyzeNot(Locals variables) {
         child.expected = Definition.BOOLEAN_TYPE;
         child.analyze(variables);
         child = child.cast(variables);
@@ -75,7 +75,7 @@ public final class EUnary extends AExpression {
         actual = Definition.BOOLEAN_TYPE;
     }
 
-    void analyzeBWNot(Variables variables) {
+    void analyzeBWNot(Locals variables) {
         child.analyze(variables);
 
         Type promote = AnalyzerCaster.promoteNumeric(child.actual, false);
@@ -102,7 +102,7 @@ public final class EUnary extends AExpression {
         actual = promote;
     }
 
-    void analyzerAdd(Variables variables) {
+    void analyzerAdd(Locals variables) {
         child.analyze(variables);
 
         Type promote = AnalyzerCaster.promoteNumeric(child.actual, true);
@@ -133,7 +133,7 @@ public final class EUnary extends AExpression {
         actual = promote;
     }
 
-    void analyzerSub(Variables variables) {
+    void analyzerSub(Locals variables) {
         child.analyze(variables);
 
         Type promote = AnalyzerCaster.promoteNumeric(child.actual, true);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
index c80cc8b8ada..38c1ae43907 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
@@ -21,7 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -38,7 +38,7 @@ public final class LArrayLength extends ALink {
     }
 
     @Override
-    ALink analyze(Variables variables) {
+    ALink analyze(Locals locals) {
         if ("length".equals(value)) {
             if (!load) {
                 throw createError(new IllegalArgumentException("Must read array field [length]."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
index b0816540c5e..91346b7a208 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Sort;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 import java.util.List;
@@ -42,7 +42,7 @@ public final class LBrace extends ALink {
     }
 
     @Override
-    ALink analyze(Variables variables) {
+    ALink analyze(Locals locals) {
         if (before == null) {
             throw createError(new IllegalArgumentException("Illegal array access made without target."));
         }
@@ -51,18 +51,18 @@ public final class LBrace extends ALink {
 
         if (sort == Sort.ARRAY) {
             index.expected = Definition.INT_TYPE;
-            index.analyze(variables);
-            index = index.cast(variables);
+            index.analyze(locals);
+            index = index.cast(locals);
 
             after = Definition.getType(before.struct, before.dimensions - 1);
 
             return this;
         } else if (sort == Sort.DEF) {
-            return new LDefArray(location, index).copy(this).analyze(variables);
+            return new LDefArray(location, index).copy(this).analyze(locals);
         } else if (Map.class.isAssignableFrom(before.clazz)) {
-            return new LMapShortcut(location, index).copy(this).analyze(variables);
+            return new LMapShortcut(location, index).copy(this).analyze(locals);
         } else if (List.class.isAssignableFrom(before.clazz)) {
-            return new LListShortcut(location, index).copy(this).analyze(variables);
+            return new LListShortcut(location, index).copy(this).analyze(locals);
         }
 
         throw createError(new IllegalArgumentException("Illegal array access on type [" + before.name + "]."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallInvoke.java
similarity index 79%
rename from modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
rename to modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallInvoke.java
index 53e66ee68b1..1f6e899b1dd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallInvoke.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.MethodKey;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Struct;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 import java.util.List;
@@ -32,14 +32,14 @@ import java.util.List;
 /**
  * Represents a method call or deferes to a def call.
  */
-public final class LCall extends ALink {
+public final class LCallInvoke extends ALink {
 
     final String name;
     final List<AExpression> arguments;
 
     Method method = null;
 
-    public LCall(Location location, String name, List<AExpression> arguments) {
+    public LCallInvoke(Location location, String name, List<AExpression> arguments) {
         super(location, -1);
 
         this.name = name;
@@ -47,7 +47,7 @@ public final class LCall extends ALink {
     }
 
     @Override
-    ALink analyze(Variables variables) {
+    ALink analyze(Locals locals) {
         if (before == null) {
             throw createError(new IllegalArgumentException("Illegal call [" + name + "] made without target."));
         } else if (before.sort == Sort.ARRAY) {
@@ -56,7 +56,7 @@ public final class LCall extends ALink {
             throw createError(new IllegalArgumentException("Cannot assign a value to a call [" + name + "]."));
         }
 
-        Definition.MethodKey methodKey = new Definition.MethodKey(name, arguments.size());
+        MethodKey methodKey = new MethodKey(name, arguments.size());
         Struct struct = before.struct;
         method = statik ? struct.staticMethods.get(methodKey) : struct.methods.get(methodKey);
 
@@ -66,8 +66,8 @@ public final class LCall extends ALink {
 
                 expression.expected = method.arguments.get(argument);
                 expression.internal = true;
-                expression.analyze(variables);
-                arguments.set(argument, expression.cast(variables));
+                expression.analyze(locals);
+                arguments.set(argument, expression.cast(locals));
             }
 
             statement = true;
@@ -78,11 +78,11 @@ public final class LCall extends ALink {
             ALink link = new LDefCall(location, name, arguments);
             link.copy(this);
 
-            return link.analyze(variables);
+            return link.analyze(locals);
         }
 
-        throw createError(new IllegalArgumentException("Unknown call [" + name + "] with [" + arguments.size() +
-                                                       "] arguments on type [" + struct.name + "]."));
+        throw createError(new IllegalArgumentException(
+            "Unknown call [" + name + "] with [" + arguments.size() + "] arguments on type [" + struct.name + "]."));
     }
 
     @Override
@@ -105,10 +105,6 @@ public final class LCall extends ALink {
         } else {
             writer.invokeVirtual(method.owner.type, method.method);
         }
-
-        if (!method.rtn.clazz.equals(method.handle.type().returnType())) {
-            writer.checkCast(method.rtn.type);
-        }
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallLocal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallLocal.java
new file mode 100644
index 00000000000..31085171bf6
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallLocal.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.Definition.Method;
+import org.elasticsearch.painless.Definition.MethodKey;
+import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Location;
+import org.elasticsearch.painless.MethodWriter;
+
+import java.util.List;
+
+import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE;
+
+/**
+ * Represents a user-defined call.
+ */
+public class LCallLocal extends ALink {
+
+    final String name;
+    final List<AExpression> arguments;
+
+    Method method = null;
+
+    public LCallLocal(Location location, String name, List<AExpression> arguments) {
+        super(location, -1);
+
+        this.name = name;
+        this.arguments = arguments;
+    }
+
+    @Override
+    ALink analyze(Locals locals) {
+        if (before != null) {
+            throw createError(new IllegalArgumentException("Illegal call [" + name + "] against an existing target."));
+        } else if (store) {
+            throw createError(new IllegalArgumentException("Cannot assign a value to a call [" + name + "]."));
+        }
+
+        MethodKey methodKey = new MethodKey(name, arguments.size());
+        method = locals.getMethod(methodKey);
+
+        if (method != null) {
+            for (int argument = 0; argument < arguments.size(); ++argument) {
+                AExpression expression = arguments.get(argument);
+
+                expression.expected = method.arguments.get(argument);
+                expression.internal = true;
+                expression.analyze(locals);
+                arguments.set(argument, expression.cast(locals));
+            }
+
+            statement = true;
+            after = method.rtn;
+
+            return this;
+        }
+
+        throw createError(new IllegalArgumentException("Unknown call [" + name + "] with [" + arguments.size() + "] arguments."));
+    }
+
+    @Override
+    void write(MethodWriter writer) {
+        // Do nothing.
+    }
+
+    @Override
+    void load(MethodWriter writer) {
+        writer.writeDebugInfo(location);
+
+        for (AExpression argument : arguments) {
+            argument.write(writer);
+        }
+
+        writer.invokeStatic(CLASS_TYPE, method.method);
+    }
+
+    @Override
+    void store(MethodWriter writer) {
+        throw createError(new IllegalStateException("Illegal tree structure."));
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
index de4a22a5ad8..d2b4f83a823 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
@@ -23,7 +23,7 @@ import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.AnalyzerCaster;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -42,7 +42,7 @@ public final class LCast extends ALink {
     }
 
     @Override
-    ALink analyze(Variables variables) {
+    ALink analyze(Locals locals) {
         if (before == null) {
             throw createError(new IllegalStateException("Illegal cast without a target."));
         } else if (store) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index 5e28cd31b98..524f62eea15 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.DefBootstrap;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Type;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -42,10 +42,10 @@ final class LDefArray extends ALink implements IDefLink {
     }
 
     @Override
-    ALink analyze(Variables variables) {
-        index.analyze(variables);
+    ALink analyze(Locals locals) {
+        index.analyze(locals);
         index.expected = index.actual;
-        index = index.cast(variables);
+        index = index.cast(locals);
 
         after = Definition.DEF_TYPE;
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index 28c57744860..871bd2273db 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.DefBootstrap;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 import java.util.List;
@@ -46,14 +46,14 @@ final class LDefCall extends ALink implements IDefLink {
     }
 
     @Override
-    ALink analyze(Variables variables) {
+    ALink analyze(Locals locals) {
         if (arguments.size() > 63) {
             // technically, the limitation is just methods with > 63 params, containing method references.
             // this is because we are lazy and use a long as a bitset. we can always change to a "string" if need be.
             // but NEED NOT BE. nothing with this many parameters is in the whitelist and we do not support varargs.
             throw new UnsupportedOperationException("methods with > 63 arguments are currently not supported");
         }
-        
+
         recipe = 0;
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
@@ -62,9 +62,9 @@ final class LDefCall extends ALink implements IDefLink {
                 recipe |= (1L << argument); // mark argument as deferred reference
             }
             expression.internal = true;
-            expression.analyze(variables);
+            expression.analyze(locals);
             expression.expected = expression.actual;
-            arguments.set(argument, expression.cast(variables));
+            arguments.set(argument, expression.cast(locals));
         }
 
         statement = true;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index ec9d5bbe9e7..861dfe0df53 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.DefBootstrap;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Type;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -43,7 +43,7 @@ final class LDefField extends ALink implements IDefLink {
 
 
     @Override
-    ALink analyze(Variables variables) {
+    ALink analyze(Locals locals) {
         after = Definition.DEF_TYPE;
 
         return this;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
index 8f34692d666..049dd2d8524 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
@@ -24,7 +24,7 @@ import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Field;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Struct;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 import java.util.List;
@@ -46,7 +46,7 @@ public final class LField extends ALink {
     }
 
     @Override
-    ALink analyze(Variables variables) {
+    ALink analyze(Locals locals) {
         if (before == null) {
             throw createError(new IllegalArgumentException("Illegal field [" + value + "] access made without target."));
         }
@@ -54,9 +54,9 @@ public final class LField extends ALink {
         Sort sort = before.sort;
 
         if (sort == Sort.ARRAY) {
-            return new LArrayLength(location, value).copy(this).analyze(variables);
+            return new LArrayLength(location, value).copy(this).analyze(locals);
         } else if (sort == Sort.DEF) {
-            return new LDefField(location, value).copy(this).analyze(variables);
+            return new LDefField(location, value).copy(this).analyze(locals);
         }
 
         Struct struct = before.struct;
@@ -81,17 +81,17 @@ public final class LField extends ALink {
                     Character.toUpperCase(value.charAt(0)) + value.substring(1), 1));
 
             if (shortcut) {
-                return new LShortcut(location, value).copy(this).analyze(variables);
+                return new LShortcut(location, value).copy(this).analyze(locals);
             } else {
                 EConstant index = new EConstant(location, value);
-                index.analyze(variables);
+                index.analyze(locals);
 
                 if (Map.class.isAssignableFrom(before.clazz)) {
-                    return new LMapShortcut(location, index).copy(this).analyze(variables);
+                    return new LMapShortcut(location, index).copy(this).analyze(locals);
                 }
 
                 if (List.class.isAssignableFrom(before.clazz)) {
-                    return new LListShortcut(location, index).copy(this).analyze(variables);
+                    return new LListShortcut(location, index).copy(this).analyze(locals);
                 }
             }
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
index a8252b40770..7b0feb094e1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
@@ -23,7 +23,7 @@ import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -42,7 +42,7 @@ final class LListShortcut extends ALink {
     }
 
     @Override
-    ALink analyze(Variables variables) {
+    ALink analyze(Locals locals) {
         getter = before.struct.methods.get(new Definition.MethodKey("get", 1));
         setter = before.struct.methods.get(new Definition.MethodKey("set", 2));
 
@@ -62,8 +62,8 @@ final class LListShortcut extends ALink {
 
         if ((load || store) && (!load || getter != null) && (!store || setter != null)) {
             index.expected = Definition.INT_TYPE;
-            index.analyze(variables);
-            index = index.cast(variables);
+            index.analyze(locals);
+            index = index.cast(locals);
 
             after = setter != null ? setter.arguments.get(1) : getter.rtn;
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
index f31179d135a..5d4c5a9e50a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
@@ -23,7 +23,7 @@ import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -42,7 +42,7 @@ final class LMapShortcut extends ALink {
     }
 
     @Override
-    ALink analyze(Variables variables) {
+    ALink analyze(Locals locals) {
         getter = before.struct.methods.get(new Definition.MethodKey("get", 1));
         setter = before.struct.methods.get(new Definition.MethodKey("put", 2));
 
@@ -61,8 +61,8 @@ final class LMapShortcut extends ALink {
 
         if ((load || store) && (!load || getter != null) && (!store || setter != null)) {
             index.expected = setter != null ? setter.arguments.get(0) : getter.arguments.get(0);
-            index.analyze(variables);
-            index = index.cast(variables);
+            index.analyze(locals);
+            index = index.cast(locals);
 
             after = setter != null ? setter.arguments.get(1) : getter.rtn;
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
index 75a4aecbe19..15aa01c1d26 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Type;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 import java.util.List;
@@ -43,7 +43,7 @@ public final class LNewArray extends ALink {
     }
 
     @Override
-    ALink analyze(Variables variables) {
+    ALink analyze(Locals locals) {
         if (before != null) {
             throw createError(new IllegalArgumentException("Cannot create a new array with a target already defined."));
         } else if (store) {
@@ -64,8 +64,8 @@ public final class LNewArray extends ALink {
             AExpression expression = arguments.get(argument);
 
             expression.expected = Definition.INT_TYPE;
-            expression.analyze(variables);
-            arguments.set(argument, expression.cast(variables));
+            expression.analyze(locals);
+            arguments.set(argument, expression.cast(locals));
         }
 
         after = Definition.getType(type.struct, arguments.size());
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
index 2f80b254350..aeb6f64f9db 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
@@ -24,7 +24,7 @@ import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Struct;
 import org.elasticsearch.painless.Definition.Type;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 import java.util.List;
@@ -47,7 +47,7 @@ public final class LNewObj extends ALink {
     }
 
     @Override
-    ALink analyze(Variables variables) {
+    ALink analyze(Locals locals) {
         if (before != null) {
             throw createError(new IllegalArgumentException("Illegal new call with a target already defined."));
         } else if (store) {
@@ -79,8 +79,8 @@ public final class LNewObj extends ALink {
 
                 expression.expected = types[argument];
                 expression.internal = true;
-                expression.analyze(variables);
-                arguments.set(argument, expression.cast(variables));
+                expression.analyze(locals);
+                arguments.set(argument, expression.cast(locals));
             }
 
             statement = true;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
index a91970fa577..73cbc201db4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
@@ -24,7 +24,7 @@ import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Struct;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -44,7 +44,7 @@ final class LShortcut extends ALink {
     }
 
     @Override
-    ALink analyze(Variables variables) {
+    ALink analyze(Locals locals) {
         Struct struct = before.struct;
 
         getter = struct.methods.get(new Definition.MethodKey("get" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java
index 98774513188..6f04e9a22c4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 
 /**
  * Represents a static type target.
@@ -38,7 +38,7 @@ public final class LStatic extends ALink {
     }
 
     @Override
-    ALink analyze(Variables variables) {
+    ALink analyze(Locals locals) {
         if (before != null) {
             throw createError(new IllegalArgumentException("Illegal static type [" + type + "] after target already defined."));
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
index 6afe62d02ba..41eb027e3ab 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
@@ -21,7 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -36,7 +36,7 @@ public final class LString extends ALink {
     }
 
     @Override
-    ALink analyze(Variables variables) {
+    ALink analyze(Locals locals) {
         if (before != null) {
             throw createError(new IllegalArgumentException("Illegal String constant [" + string + "]."));
         } else if (store) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
index 8fe6f17b0b5..64dc852117d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
@@ -21,8 +21,8 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
-import org.elasticsearch.painless.Variables;
-import org.elasticsearch.painless.Variables.Variable;
+import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Locals.Variable;
 import org.objectweb.asm.Opcodes;
 
 /**
@@ -41,12 +41,12 @@ public final class LVariable extends ALink {
     }
 
     @Override
-    ALink analyze(Variables variables) {
+    ALink analyze(Locals locals) {
         if (before != null) {
             throw createError(new IllegalArgumentException("Illegal variable [" + name + "] access with target already defined."));
         }
 
-        Variable variable = variables.getVariable(location, name);
+        Variable variable = locals.getVariable(location, name);
 
         if (store && variable.readonly) {
             throw createError(new IllegalArgumentException("Variable [" + variable.name + "] is read-only."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
index 4dbbd80de54..5fec362ec17 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -40,7 +40,7 @@ public final class SBlock extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
+    void analyze(Locals locals) {
         if (statements == null || statements.isEmpty()) {
             throw createError(new IllegalArgumentException("A block must contain at least one statement."));
         }
@@ -58,7 +58,7 @@ public final class SBlock extends AStatement {
             statement.lastSource = lastSource && statement == last;
             statement.lastLoop = (beginLoop || lastLoop) && statement == last;
 
-            statement.analyze(variables);
+            statement.analyze(locals);
 
             methodEscape = statement.methodEscape;
             loopEscape = statement.loopEscape;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
index ca72dd0b55b..d4f8bfff4e4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -33,7 +33,7 @@ public final class SBreak extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
+    void analyze(Locals locals) {
         if (!inLoop) {
             throw createError(new IllegalArgumentException("Break statement outside of a loop."));
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
index 8bcaf9d22cf..42de28e48f5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
@@ -22,8 +22,8 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Type;
-import org.elasticsearch.painless.Variables;
-import org.elasticsearch.painless.Variables.Variable;
+import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Locals.Variable;
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
 import org.elasticsearch.painless.MethodWriter;
@@ -52,7 +52,7 @@ public final class SCatch extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
+    void analyze(Locals locals) {
         final Type type;
 
         try {
@@ -65,14 +65,14 @@ public final class SCatch extends AStatement {
             throw createError(new ClassCastException("Not an exception type [" + this.type + "]."));
         }
 
-        variable = variables.addVariable(location, type, name, true, false);
+        variable = locals.addVariable(location, type, name, true, false);
 
         if (block != null) {
             block.lastSource = lastSource;
             block.inLoop = inLoop;
             block.lastLoop = lastLoop;
 
-            block.analyze(variables);
+            block.analyze(locals);
 
             methodEscape = block.methodEscape;
             loopEscape = block.loopEscape;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
index ef80766bdd1..802cf90087b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -33,7 +33,7 @@ public final class SContinue extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
+    void analyze(Locals locals) {
         if (!inLoop) {
             throw createError(new IllegalArgumentException("Continue statement outside of a loop."));
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
index 1ff187afe29..5365e72888a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -40,9 +40,9 @@ public final class SDeclBlock extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
+    void analyze(Locals locals) {
         for (SDeclaration declaration : declarations) {
-            declaration.analyze(variables);
+            declaration.analyze(locals);
         }
 
         statementCount = declarations.size();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
index 5f039184cb6..a62a287df9c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
@@ -22,8 +22,8 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Type;
-import org.elasticsearch.painless.Variables;
-import org.elasticsearch.painless.Variables.Variable;
+import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Locals.Variable;
 import org.objectweb.asm.Opcodes;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -47,7 +47,7 @@ public final class SDeclaration extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
+    void analyze(Locals locals) {
         final Type type;
 
         try {
@@ -58,11 +58,11 @@ public final class SDeclaration extends AStatement {
 
         if (expression != null) {
             expression.expected = type;
-            expression.analyze(variables);
-            expression = expression.cast(variables);
+            expression.analyze(locals);
+            expression = expression.cast(locals);
         }
 
-        variable = variables.addVariable(location, type, name, false, false);
+        variable = locals.addVariable(location, type, name, false, false);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
index 4989fb77a79..b25d903dc41 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -21,7 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -30,21 +30,19 @@ import org.elasticsearch.painless.MethodWriter;
  */
 public final class SDo extends AStatement {
 
-    final int maxLoopCounter;
     final SBlock block;
     AExpression condition;
 
-    public SDo(Location location, int maxLoopCounter, SBlock block, AExpression condition) {
+    public SDo(Location location, SBlock block, AExpression condition) {
         super(location);
 
         this.condition = condition;
         this.block = block;
-        this.maxLoopCounter = maxLoopCounter;
     }
 
     @Override
-    void analyze(Variables variables) {
-        variables.incrementScope();
+    void analyze(Locals locals) {
+        locals.incrementScope();
 
         if (block == null) {
             throw createError(new IllegalArgumentException("Extraneous do while loop."));
@@ -53,15 +51,15 @@ public final class SDo extends AStatement {
         block.beginLoop = true;
         block.inLoop = true;
 
-        block.analyze(variables);
+        block.analyze(locals);
 
         if (block.loopEscape && !block.anyContinue) {
             throw createError(new IllegalArgumentException("Extraneous do while loop."));
         }
 
         condition.expected = Definition.BOOLEAN_TYPE;
-        condition.analyze(variables);
-        condition = condition.cast(variables);
+        condition.analyze(locals);
+        condition = condition.cast(locals);
 
         if (condition.constant != null) {
             final boolean continuous = (boolean)condition.constant;
@@ -78,11 +76,11 @@ public final class SDo extends AStatement {
 
         statementCount = 1;
 
-        if (maxLoopCounter > 0) {
-            loopCounterSlot = variables.getVariable(location, "#loop").slot;
+        if (locals.getMaxLoopCounter() > 0) {
+            loopCounterSlot = locals.getVariable(location, "#loop").slot;
         }
 
-        variables.decrementScope();
+        locals.decrementScope();
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index 9a6fe798957..b42bd1aff5a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -29,8 +29,8 @@ import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
-import org.elasticsearch.painless.Variables;
-import org.elasticsearch.painless.Variables.Variable;
+import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Locals.Variable;
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
 
@@ -44,7 +44,6 @@ import static org.elasticsearch.painless.WriterConstants.ITERATOR_TYPE;
  */
 public class SEach extends AStatement {
 
-    final int maxLoopCounter;
     final String type;
     final String name;
     AExpression expression;
@@ -63,10 +62,9 @@ public class SEach extends AStatement {
     Variable iterator = null;
     Method method = null;
 
-    public SEach(Location location, int maxLoopCounter, String type, String name, AExpression expression, SBlock block) {
+    public SEach(Location location, String type, String name, AExpression expression, SBlock block) {
         super(location);
 
-        this.maxLoopCounter = maxLoopCounter;
         this.type = type;
         this.name = name;
         this.expression = expression;
@@ -74,10 +72,10 @@ public class SEach extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
-        expression.analyze(variables);
+    void analyze(Locals locals) {
+        expression.analyze(locals);
         expression.expected = expression.actual;
-        expression = expression.cast(variables);
+        expression = expression.cast(locals);
 
         final Type type;
 
@@ -87,25 +85,25 @@ public class SEach extends AStatement {
             throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
 
-        variables.incrementScope();
+        locals.incrementScope();
 
-        variable = variables.addVariable(location, type, name, true, false);
+        variable = locals.addVariable(location, type, name, true, false);
 
         if (expression.actual.sort == Sort.ARRAY) {
-            analyzeArray(variables, type);
+            analyzeArray(locals, type);
         } else if (expression.actual.sort == Sort.DEF || Iterable.class.isAssignableFrom(expression.actual.clazz)) {
-            analyzeIterable(variables, type);
+            analyzeIterable(locals, type);
         } else {
-            throw location.createError(new IllegalArgumentException("Illegal for each type [" + expression.actual.name + "]."));
+            throw createError(new IllegalArgumentException("Illegal for each type [" + expression.actual.name + "]."));
         }
 
         if (block == null) {
-            throw location.createError(new IllegalArgumentException("Extraneous for each loop."));
+            throw createError(new IllegalArgumentException("Extraneous for each loop."));
         }
 
         block.beginLoop = true;
         block.inLoop = true;
-        block.analyze(variables);
+        block.analyze(locals);
         block.statementCount = Math.max(1, block.statementCount);
 
         if (block.loopEscape && !block.anyContinue) {
@@ -114,14 +112,14 @@ public class SEach extends AStatement {
 
         statementCount = 1;
 
-        if (maxLoopCounter > 0) {
-            loopCounterSlot = variables.getVariable(location, "#loop").slot;
+        if (locals.getMaxLoopCounter() > 0) {
+            loopCounterSlot = locals.getVariable(location, "#loop").slot;
         }
 
-        variables.decrementScope();
+        locals.decrementScope();
     }
 
-    void analyzeArray(Variables variables, Type type) {
+    void analyzeArray(Locals variables, Type type) {
         // We must store the array and index as variables for securing slots on the stack, and
         // also add the location offset to make the names unique in case of nested for each loops.
         array = variables.addVariable(location, expression.actual, "#array" + location.getOffset(), true, false);
@@ -130,7 +128,7 @@ public class SEach extends AStatement {
         cast = AnalyzerCaster.getLegalCast(location, indexed, type, true, true);
     }
 
-    void analyzeIterable(Variables variables, Type type) {
+    void analyzeIterable(Locals variables, Type type) {
         // We must store the iterator as a variable for securing a slot on the stack, and
         // also add the location offset to make the name unique in case of nested for each loops.
         iterator = variables.addVariable(location, Definition.getType("Iterator"), "#itr" + location.getOffset(), true, false);
@@ -141,7 +139,7 @@ public class SEach extends AStatement {
             method = expression.actual.struct.methods.get(new MethodKey("iterator", 0));
 
             if (method == null) {
-                throw location.createError(new IllegalArgumentException(
+                throw createError(new IllegalArgumentException(
                     "Unable to create iterator for the type [" + expression.actual.name + "]."));
             }
         }
@@ -158,7 +156,7 @@ public class SEach extends AStatement {
         } else if (iterator != null) {
             writeIterable(writer);
         } else {
-            throw location.createError(new IllegalStateException("Illegal tree structure."));
+            throw createError(new IllegalStateException("Illegal tree structure."));
         }
     }
 
@@ -197,7 +195,7 @@ public class SEach extends AStatement {
         if (method == null) {
             Type itr = Definition.getType("Iterator");
             String desc = org.objectweb.asm.Type.getMethodDescriptor(itr.type, Definition.DEF_TYPE.type);
-            writer.invokeDynamic("iterator", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ITERATOR, 0);
+            writer.invokeDynamic("iterator", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ITERATOR, (Object)0);
         } else if (java.lang.reflect.Modifier.isInterface(method.owner.clazz.getModifiers())) {
             writer.invokeInterface(method.owner.type, method.method);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index 1bea07d5599..0a65664fa74 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -20,9 +20,10 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Sort;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -39,19 +40,22 @@ public final class SExpression extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
-        expression.read = lastSource;
-        expression.analyze(variables);
+    void analyze(Locals locals) {
+        Type rtnType = locals.getReturnType();
+        boolean isVoid = rtnType.sort == Sort.VOID;
+
+        expression.read = lastSource && !isVoid;
+        expression.analyze(locals);
 
         if (!lastSource && !expression.statement) {
             throw createError(new IllegalArgumentException("Not a statement."));
         }
 
-        final boolean rtn = lastSource && expression.actual.sort != Sort.VOID;
+        boolean rtn = lastSource && !isVoid && expression.actual.sort != Sort.VOID;
 
-        expression.expected = rtn ? Definition.OBJECT_TYPE : expression.actual;
+        expression.expected = rtn ? rtnType : expression.actual;
         expression.internal = rtn;
-        expression = expression.cast(variables);
+        expression = expression.cast(locals);
 
         methodEscape = rtn;
         loopEscape = rtn;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index 04475a91a1a..cc9dee20138 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -21,7 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -30,17 +30,14 @@ import org.elasticsearch.painless.MethodWriter;
  */
 public final class SFor extends AStatement {
 
-    final int maxLoopCounter;
     ANode initializer;
     AExpression condition;
     AExpression afterthought;
     final SBlock block;
 
-    public SFor(Location location, int maxLoopCounter,
-                ANode initializer, AExpression condition, AExpression afterthought, SBlock block) {
+    public SFor(Location location, ANode initializer, AExpression condition, AExpression afterthought, SBlock block) {
         super(location);
 
-        this.maxLoopCounter = maxLoopCounter;
         this.initializer = initializer;
         this.condition = condition;
         this.afterthought = afterthought;
@@ -48,19 +45,19 @@ public final class SFor extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
-        variables.incrementScope();
+    void analyze(Locals locals) {
+        locals.incrementScope();
 
         boolean continuous = false;
 
         if (initializer != null) {
             if (initializer instanceof AStatement) {
-                ((AStatement)initializer).analyze(variables);
+                ((AStatement)initializer).analyze(locals);
             } else if (initializer instanceof AExpression) {
                 AExpression initializer = (AExpression)this.initializer;
 
                 initializer.read = false;
-                initializer.analyze(variables);
+                initializer.analyze(locals);
 
                 if (!initializer.statement) {
                     throw createError(new IllegalArgumentException("Not a statement."));
@@ -72,8 +69,8 @@ public final class SFor extends AStatement {
 
         if (condition != null) {
             condition.expected = Definition.BOOLEAN_TYPE;
-            condition.analyze(variables);
-            condition = condition.cast(variables);
+            condition.analyze(locals);
+            condition = condition.cast(locals);
 
             if (condition.constant != null) {
                 continuous = (boolean)condition.constant;
@@ -92,7 +89,7 @@ public final class SFor extends AStatement {
 
         if (afterthought != null) {
             afterthought.read = false;
-            afterthought.analyze(variables);
+            afterthought.analyze(locals);
 
             if (!afterthought.statement) {
                 throw createError(new IllegalArgumentException("Not a statement."));
@@ -103,7 +100,7 @@ public final class SFor extends AStatement {
             block.beginLoop = true;
             block.inLoop = true;
 
-            block.analyze(variables);
+            block.analyze(locals);
 
             if (block.loopEscape && !block.anyContinue) {
                 throw createError(new IllegalArgumentException("Extraneous for loop."));
@@ -119,11 +116,11 @@ public final class SFor extends AStatement {
 
         statementCount = 1;
 
-        if (maxLoopCounter > 0) {
-            loopCounterSlot = variables.getVariable(location, "#loop").slot;
+        if (locals.getMaxLoopCounter() > 0) {
+            loopCounterSlot = locals.getVariable(location, "#loop").slot;
         }
 
-        variables.decrementScope();
+        locals.decrementScope();
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
new file mode 100644
index 00000000000..16d53f23bc6
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Method;
+import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Locals.Parameter;
+import org.elasticsearch.painless.Locals.FunctionReserved;
+import org.elasticsearch.painless.Locals.Variable;
+import org.elasticsearch.painless.Location;
+import org.elasticsearch.painless.MethodWriter;
+import org.objectweb.asm.Opcodes;
+
+import java.lang.invoke.MethodType;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Represents a user-defined function.
+ */
+public class SFunction extends AStatement {
+    final FunctionReserved reserved;
+    final String rtnTypeStr;
+    final String name;
+    final List<String> paramTypeStrs;
+    final List<String> paramNameStrs;
+    final List<AStatement> statements;
+
+    Type rtnType = null;
+    List<Parameter> parameters = new ArrayList<>();
+    Method method = null;
+
+    Locals locals = null;
+
+    public SFunction(FunctionReserved reserved, Location location,
+                     String rtnType, String name, List<String> paramTypes, List<String> paramNames, List<AStatement> statements) {
+        super(location);
+
+        this.reserved = reserved;
+        this.rtnTypeStr = rtnType;
+        this.name = name;
+        this.paramTypeStrs = Collections.unmodifiableList(paramTypes);
+        this.paramNameStrs = Collections.unmodifiableList(paramNames);
+        this.statements = Collections.unmodifiableList(statements);
+    }
+
+    void generate() {
+        try {
+            rtnType = Definition.getType(rtnTypeStr);
+        } catch (IllegalArgumentException exception) {
+            throw createError(new IllegalArgumentException("Illegal return type [" + rtnTypeStr + "] for function [" + name + "]."));
+        }
+
+        if (paramTypeStrs.size() != paramNameStrs.size()) {
+            throw createError(new IllegalStateException("Illegal tree structure."));
+        }
+
+        Class<?>[] paramClasses = new Class<?>[this.paramTypeStrs.size()];
+        List<Type> paramTypes = new ArrayList<>();
+
+        for (int param = 0; param < this.paramTypeStrs.size(); ++param) {
+            try {
+                Type paramType = Definition.getType(this.paramTypeStrs.get(param));
+
+                paramClasses[param] = paramType.clazz;
+                paramTypes.add(paramType);
+                parameters.add(new Parameter(location, paramNameStrs.get(param), paramType));
+            } catch (IllegalArgumentException exception) {
+                throw createError(new IllegalArgumentException(
+                    "Illegal parameter type [" + this.paramTypeStrs.get(param) + "] for function [" + name + "]."));
+            }
+        }
+
+        org.objectweb.asm.commons.Method method =
+            new org.objectweb.asm.commons.Method(name, MethodType.methodType(rtnType.clazz, paramClasses).toMethodDescriptorString());
+        this.method = new Method(name, null, rtnType, paramTypes, method, 0, null);
+    }
+
+    @Override
+    void analyze(Locals locals) {
+        if (statements == null || statements.isEmpty()) {
+            throw createError(new IllegalArgumentException("Cannot generate an empty function [" + name + "]."));
+        }
+
+        this.locals = new Locals(reserved, locals, rtnType, parameters);
+        locals = this.locals;
+
+        locals.incrementScope();
+
+        AStatement last = statements.get(statements.size() - 1);
+
+        for (AStatement statement : statements) {
+            // Note that we do not need to check after the last statement because
+            // there is no statement that can be unreachable after the last.
+            if (allEscape) {
+                throw createError(new IllegalArgumentException("Unreachable statement."));
+            }
+
+            statement.lastSource = statement == last;
+
+            statement.analyze(locals);
+
+            methodEscape = statement.methodEscape;
+            allEscape = statement.allEscape;
+        }
+
+        if (!methodEscape && rtnType.sort != Sort.VOID) {
+            throw createError(new IllegalArgumentException("Not all paths provide a return value for method [" + name + "]."));
+        }
+
+        locals.decrementScope();
+    }
+
+    @Override
+    void write(MethodWriter writer) {
+        MethodWriter function = writer.newMethodWriter(Opcodes.ACC_PRIVATE | Opcodes.ACC_STATIC, method.method);
+
+        if (reserved.getMaxLoopCounter() > 0) {
+            // if there is infinite loop protection, we do this once:
+            // int #loop = settings.getMaxLoopCounter()
+
+            Variable loop = locals.getVariable(null, FunctionReserved.LOOP);
+
+            function.push(reserved.getMaxLoopCounter());
+            function.visitVarInsn(Opcodes.ISTORE, loop.slot);
+        }
+
+        for (AStatement statement : statements) {
+            statement.write(function);
+        }
+
+        if (!methodEscape) {
+            if (rtnType.sort == Sort.VOID) {
+                function.returnValue();
+            } else {
+                throw createError(new IllegalStateException("Illegal tree structure."));
+            }
+        }
+
+        function.endMethod();
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
index 954ddac9c6a..a46075af9a1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
@@ -21,7 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -41,10 +41,10 @@ public final class SIf extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
+    void analyze(Locals locals) {
         condition.expected = Definition.BOOLEAN_TYPE;
-        condition.analyze(variables);
-        condition = condition.cast(variables);
+        condition.analyze(locals);
+        condition = condition.cast(locals);
 
         if (condition.constant != null) {
             throw createError(new IllegalArgumentException("Extraneous if statement."));
@@ -58,9 +58,9 @@ public final class SIf extends AStatement {
         ifblock.inLoop = inLoop;
         ifblock.lastLoop = lastLoop;
 
-        variables.incrementScope();
-        ifblock.analyze(variables);
-        variables.decrementScope();
+        locals.incrementScope();
+        ifblock.analyze(locals);
+        locals.decrementScope();
 
         anyContinue = ifblock.anyContinue;
         anyBreak = ifblock.anyBreak;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
index 1d801267054..22cbfe25614 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
@@ -21,7 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -43,10 +43,10 @@ public final class SIfElse extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
+    void analyze(Locals locals) {
         condition.expected = Definition.BOOLEAN_TYPE;
-        condition.analyze(variables);
-        condition = condition.cast(variables);
+        condition.analyze(locals);
+        condition = condition.cast(locals);
 
         if (condition.constant != null) {
             throw createError(new IllegalArgumentException("Extraneous if statement."));
@@ -60,9 +60,9 @@ public final class SIfElse extends AStatement {
         ifblock.inLoop = inLoop;
         ifblock.lastLoop = lastLoop;
 
-        variables.incrementScope();
-        ifblock.analyze(variables);
-        variables.decrementScope();
+        locals.incrementScope();
+        ifblock.analyze(locals);
+        locals.decrementScope();
 
         anyContinue = ifblock.anyContinue;
         anyBreak = ifblock.anyBreak;
@@ -76,9 +76,9 @@ public final class SIfElse extends AStatement {
         elseblock.inLoop = inLoop;
         elseblock.lastLoop = lastLoop;
 
-        variables.incrementScope();
-        elseblock.analyze(variables);
-        variables.decrementScope();
+        locals.incrementScope();
+        elseblock.analyze(locals);
+        locals.decrementScope();
 
         methodEscape = ifblock.methodEscape && elseblock.methodEscape;
         loopEscape = ifblock.loopEscape && elseblock.loopEscape;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
index bd9707eb6a4..e789f3372e3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -19,9 +19,8 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -38,11 +37,11 @@ public final class SReturn extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
-        expression.expected = Definition.OBJECT_TYPE;
+    void analyze(Locals locals) {
+        expression.expected = locals.getReturnType();
         expression.internal = true;
-        expression.analyze(variables);
-        expression = expression.cast(variables);
+        expression.analyze(locals);
+        expression = expression.cast(locals);
 
         methodEscape = true;
         loopEscape = true;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index 22fd2e3e28c..c7764b3d60a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -19,34 +19,85 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Definition.Method;
+import org.elasticsearch.painless.Definition.MethodKey;
+import org.elasticsearch.painless.Executable;
+import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Locals.ExecuteReserved;
+import org.elasticsearch.painless.Locals.Variable;
+import org.elasticsearch.painless.WriterConstants;
+import org.objectweb.asm.ClassWriter;
 import org.objectweb.asm.Opcodes;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
+import java.util.BitSet;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.painless.WriterConstants.BASE_CLASS_TYPE;
+import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE;
+import static org.elasticsearch.painless.WriterConstants.CONSTRUCTOR;
+import static org.elasticsearch.painless.WriterConstants.EXECUTE;
+import static org.elasticsearch.painless.WriterConstants.MAP_GET;
+import static org.elasticsearch.painless.WriterConstants.MAP_TYPE;
 
 /**
  * The root of all Painless trees.  Contains a series of statements.
  */
 public final class SSource extends AStatement {
 
+    final String name;
+    final String source;
+    final ExecuteReserved reserved;
+    final List<SFunction> functions;
     final List<AStatement> statements;
 
-    public SSource(Location location, List<AStatement> statements) {
+    private Locals locals;
+    private BitSet expressions;
+    private byte[] bytes;
+
+    public SSource(String name, String source, ExecuteReserved reserved, Location location,
+                   List<SFunction> functions, List<AStatement> statements) {
         super(location);
 
+        this.name = name;
+        this.source = source;
+        this.reserved = reserved;
+        this.functions = Collections.unmodifiableList(functions);
         this.statements = Collections.unmodifiableList(statements);
     }
 
+    public void analyze() {
+        Map<MethodKey, Method> methods = new HashMap<>();
+
+        for (SFunction function : functions) {
+            function.generate();
+
+            MethodKey key = new MethodKey(function.name, function.parameters.size());
+
+            if (methods.put(key, function.method) != null) {
+                throw createError(new IllegalArgumentException("Duplicate functions with name [" + function.name + "]."));
+            }
+        }
+
+        locals = new Locals(reserved, methods);
+        analyze(locals);
+    }
+
     @Override
-    public void analyze(Variables variables) {
+    void analyze(Locals locals) {
+        for (SFunction function : functions) {
+            function.analyze(locals);
+        }
+
         if (statements == null || statements.isEmpty()) {
             throw createError(new IllegalArgumentException("Cannot generate an empty script."));
         }
 
-        variables.incrementScope();
+        locals.incrementScope();
 
         AStatement last = statements.get(statements.size() - 1);
 
@@ -59,17 +110,95 @@ public final class SSource extends AStatement {
 
             statement.lastSource = statement == last;
 
-            statement.analyze(variables);
+            statement.analyze(locals);
 
             methodEscape = statement.methodEscape;
             allEscape = statement.allEscape;
         }
 
-        variables.decrementScope();
+        locals.decrementScope();
+    }
+
+    public void write() {
+        // Create the ClassWriter.
+
+        int classFrames = ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS;
+        int classVersion = Opcodes.V1_8;
+        int classAccess = Opcodes.ACC_PUBLIC | Opcodes.ACC_SUPER | Opcodes.ACC_FINAL;
+        String classBase = BASE_CLASS_TYPE.getInternalName();
+        String className = CLASS_TYPE.getInternalName();
+        String classInterfaces[] = reserved.usesScore() ? new String[] { WriterConstants.NEEDS_SCORE_TYPE.getInternalName() } : null;
+
+        ClassWriter writer = new ClassWriter(classFrames);
+        writer.visit(classVersion, classAccess, className, null, classBase, classInterfaces);
+        writer.visitSource(Location.computeSourceName(name, source), null);
+
+        // Create the execute MethodWriter.
+
+        expressions = new BitSet(source.length());
+        MethodWriter execute = new MethodWriter(Opcodes.ACC_PUBLIC, EXECUTE, writer, expressions);
+
+        // Write the constructor.
+
+        MethodWriter constructor = execute.newMethodWriter(Opcodes.ACC_PUBLIC, CONSTRUCTOR);
+        constructor.loadThis();
+        constructor.loadArgs();
+        constructor.invokeConstructor(org.objectweb.asm.Type.getType(Executable.class), CONSTRUCTOR);
+        constructor.returnValue();
+        constructor.endMethod();
+
+        // Write the execute method.
+
+        write(execute);
+        execute.endMethod();
+
+        // End writing the class and store the generated bytes.
+
+        writer.visitEnd();
+        bytes = writer.toByteArray();
     }
 
     @Override
-    public void write(MethodWriter writer) {
+    void write(MethodWriter writer) {
+        for (SFunction function : functions) {
+            function.write(writer);
+        }
+
+        if (reserved.usesScore()) {
+            // if the _score value is used, we do this once:
+            // final double _score = scorer.score();
+            Variable scorer = locals.getVariable(null, ExecuteReserved.SCORER);
+            Variable score = locals.getVariable(null, ExecuteReserved.SCORE);
+
+            writer.visitVarInsn(Opcodes.ALOAD, scorer.slot);
+            writer.invokeVirtual(WriterConstants.SCORER_TYPE, WriterConstants.SCORER_SCORE);
+            writer.visitInsn(Opcodes.F2D);
+            writer.visitVarInsn(Opcodes.DSTORE, score.slot);
+        }
+
+        if (reserved.usesCtx()) {
+            // if the _ctx value is used, we do this once:
+            // final Map<String,Object> ctx = input.get("ctx");
+
+            Variable input = locals.getVariable(null, ExecuteReserved.PARAMS);
+            Variable ctx = locals.getVariable(null, ExecuteReserved.CTX);
+
+            writer.visitVarInsn(Opcodes.ALOAD, input.slot);
+            writer.push(ExecuteReserved.CTX);
+            writer.invokeInterface(MAP_TYPE, MAP_GET);
+            writer.visitVarInsn(Opcodes.ASTORE, ctx.slot);
+        }
+
+        if (reserved.getMaxLoopCounter() > 0) {
+            // if there is infinite loop protection, we do this once:
+            // int #loop = settings.getMaxLoopCounter()
+
+            Variable loop = locals.getVariable(null, ExecuteReserved.LOOP);
+
+            writer.push(reserved.getMaxLoopCounter());
+            writer.visitVarInsn(Opcodes.ISTORE, loop.slot);
+        }
+
         for (AStatement statement : statements) {
             statement.write(writer);
         }
@@ -79,4 +208,12 @@ public final class SSource extends AStatement {
             writer.returnValue();
         }
     }
+
+    public BitSet getExpressions() {
+        return expressions;
+    }
+
+    public byte[] getBytes() {
+        return bytes;
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
index af9d7a65990..db04d622839 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
@@ -21,7 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -38,10 +38,10 @@ public final class SThrow extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
+    void analyze(Locals locals) {
         expression.expected = Definition.EXCEPTION_TYPE;
-        expression.analyze(variables);
-        expression = expression.cast(variables);
+        expression.analyze(locals);
+        expression = expression.cast(locals);
 
         methodEscape = true;
         loopEscape = true;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
index c24c8273dba..42fffc759ce 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
@@ -43,7 +43,7 @@ public final class STry extends AStatement {
     }
 
     @Override
-    void analyze(Variables variables) {
+    void analyze(Locals locals) {
         if (block == null) {
             throw createError(new IllegalArgumentException("Extraneous try statement."));
         }
@@ -52,9 +52,9 @@ public final class STry extends AStatement {
         block.inLoop = inLoop;
         block.lastLoop = lastLoop;
 
-        variables.incrementScope();
-        block.analyze(variables);
-        variables.decrementScope();
+        locals.incrementScope();
+        block.analyze(locals);
+        locals.decrementScope();
 
         methodEscape = block.methodEscape;
         loopEscape = block.loopEscape;
@@ -69,9 +69,9 @@ public final class STry extends AStatement {
             catc.inLoop = inLoop;
             catc.lastLoop = lastLoop;
 
-            variables.incrementScope();
-            catc.analyze(variables);
-            variables.decrementScope();
+            locals.incrementScope();
+            catc.analyze(locals);
+            locals.decrementScope();
 
             methodEscape &= catc.methodEscape;
             loopEscape &= catc.loopEscape;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
index 59c1bb75ee8..20478a55aa0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -21,7 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Location;
-import org.elasticsearch.painless.Variables;
+import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -30,25 +30,23 @@ import org.elasticsearch.painless.MethodWriter;
  */
 public final class SWhile extends AStatement {
 
-    final int maxLoopCounter;
     AExpression condition;
     final SBlock block;
 
-    public SWhile(Location location, int maxLoopCounter, AExpression condition, SBlock block) {
+    public SWhile(Location location, AExpression condition, SBlock block) {
         super(location);
 
-        this.maxLoopCounter = maxLoopCounter;
         this.condition = condition;
         this.block = block;
     }
 
     @Override
-    void analyze(Variables variables) {
-        variables.incrementScope();
+    void analyze(Locals locals) {
+        locals.incrementScope();
 
         condition.expected = Definition.BOOLEAN_TYPE;
-        condition.analyze(variables);
-        condition = condition.cast(variables);
+        condition.analyze(locals);
+        condition = condition.cast(locals);
 
         boolean continuous = false;
 
@@ -68,7 +66,7 @@ public final class SWhile extends AStatement {
             block.beginLoop = true;
             block.inLoop = true;
 
-            block.analyze(variables);
+            block.analyze(locals);
 
             if (block.loopEscape && !block.anyContinue) {
                 throw createError(new IllegalArgumentException("Extraneous while loop."));
@@ -84,11 +82,11 @@ public final class SWhile extends AStatement {
 
         statementCount = 1;
 
-        if (maxLoopCounter > 0) {
-            loopCounterSlot = variables.getVariable(location, "#loop").slot;
+        if (locals.getMaxLoopCounter() > 0) {
+            loopCounterSlot = locals.getVariable(location, "#loop").slot;
         }
 
-        variables.decrementScope();
+        locals.decrementScope();
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
index 71cde33e979..8ab69366f24 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
@@ -49,7 +49,8 @@
  * {@link org.elasticsearch.painless.node.IDefLink} - A marker interface for all LDef* (link) nodes.
  * {@link org.elasticsearch.painless.node.LArrayLength} - Represents an array length field load.
  * {@link org.elasticsearch.painless.node.LBrace} - Represents an array load/store or defers to possible shortcuts.
- * {@link org.elasticsearch.painless.node.LCall} - Represents a method call or defers to a def call.
+ * {@link org.elasticsearch.painless.node.LCallInvoke} - Represents a method call or defers to a def call.
+ * {@link org.elasticsearch.painless.node.LCallLocal} - Represents a user-defined call.
  * {@link org.elasticsearch.painless.node.LCast} - Represents a cast made in a variable/method chain.
  * {@link org.elasticsearch.painless.node.LDefArray} - Represents an array load/store or shortcut on a def type.  (Internal only.)
  * {@link org.elasticsearch.painless.node.LDefCall} - Represents a method call made on a def type. (Internal only.)
@@ -73,6 +74,7 @@
  * {@link org.elasticsearch.painless.node.SEach} - Represents a for each loop shortcut for iterables.
  * {@link org.elasticsearch.painless.node.SExpression} - Represents the top-level node for an expression as a statement.
  * {@link org.elasticsearch.painless.node.SFor} - Represents a for loop.
+ * {@link org.elasticsearch.painless.node.SFunction} - Represents a user-defined function.
  * {@link org.elasticsearch.painless.node.SIf} - Represents an if block.
  * {@link org.elasticsearch.painless.node.SIfElse} - Represents an if/else block.
  * {@link org.elasticsearch.painless.node.SReturn} - Represents a return statement.
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionTests.java
new file mode 100644
index 00000000000..48c09cd4025
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionTests.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+public class FunctionTests extends ScriptTestCase {
+    public void testBasic() {
+        assertEquals(5, exec("int get() {5;} get()"));
+    }
+
+    public void testReference() {
+        assertEquals(5, exec("void get(int[] x) {x[0] = 5;} int[] y = new int[1]; y[0] = 1; get(y); y[0]"));
+    }
+
+    public void testConcat() {
+        assertEquals("xyxy", exec("String catcat(String single) {single + single;} catcat('xy')"));
+    }
+
+    public void testMultiArgs() {
+        assertEquals(5, exec("int add(int x, int y) {return x + y;} int x = 1, y = 2; add(add(x, x), add(x, y))"));
+    }
+
+    public void testMultiFuncs() {
+        assertEquals(1, exec("int add(int x, int y) {return x + y;} int sub(int x, int y) {return x - y;} add(2, sub(3, 4))"));
+        assertEquals(3, exec("int sub2(int x, int y) {sub(x, y) - y;} int sub(int x, int y) {return x - y;} sub2(5, 1)"));
+    }
+
+    public void testRecursion() {
+        assertEquals(55, exec("int fib(int n) {if (n <= 1) return n; else return fib(n-1) + fib(n-2);} fib(10)"));
+    }
+
+    public void testEmpty() {
+        Exception expected = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("void test(int x) {} test()");
+        });
+        assertTrue(expected.getMessage().contains("Cannot generate an empty function"));
+    }
+
+    public void testDuplicates() {
+        Exception expected = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("void test(int x) {x = 2;} void test(def y) {y = 3;} test()");
+        });
+        assertTrue(expected.getMessage().contains("Duplicate functions"));
+    }
+
+    public void testInfiniteLoop() {
+        Error expected = expectScriptThrows(PainlessError.class, () -> {
+            exec("void test() {boolean x = true; while (x) {}} test()");
+        });
+        assertTrue(expected.getMessage().contains(
+            "The maximum number of statements that can be executed in a loop has been reached."));
+    }
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java
index 9136261c164..e1dbe9db0f7 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java
@@ -30,7 +30,7 @@ public class ReservedWordTests extends ScriptTestCase {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("int _score = 5; return _score;");
         });
-        assertTrue(expected.getMessage().contains("Variable name [_score] is reserved"));
+        assertTrue(expected.getMessage().contains("Variable [_score] is reserved"));
     }
 
     /** check that we can't write to _score, its read-only! */
@@ -46,7 +46,7 @@ public class ReservedWordTests extends ScriptTestCase {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("int doc = 5; return doc;");
         });
-        assertTrue(expected.getMessage().contains("Variable name [doc] is reserved"));
+        assertTrue(expected.getMessage().contains("Variable [doc] is reserved"));
     }
 
     /** check that we can't write to doc, its read-only! */
@@ -62,7 +62,7 @@ public class ReservedWordTests extends ScriptTestCase {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("int ctx = 5; return ctx;");
         });
-        assertTrue(expected.getMessage().contains("Variable name [ctx] is reserved"));
+        assertTrue(expected.getMessage().contains("Variable [ctx] is reserved"));
     }
 
     /** check that we can't write to ctx, its read-only! */
@@ -83,7 +83,7 @@ public class ReservedWordTests extends ScriptTestCase {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("int _value = 5; return _value;");
         });
-        assertTrue(expected.getMessage().contains("Variable name [_value] is reserved"));
+        assertTrue(expected.getMessage().contains("Variable [_value] is reserved"));
     }
 
     /** check that we can't write to _value, its read-only! */

From 16d17351f77ccae519a7f0263201056179e39d97 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Fri, 10 Jun 2016 17:37:57 -0400
Subject: [PATCH 0784/1311] don't use loadLocal, add new node to package-info

---
 .../painless/node/ECapturingFunctionRef.java             | 9 +++++----
 .../org/elasticsearch/painless/node/package-info.java    | 3 ++-
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
index 4e125a55de6..3e35602a3a2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
@@ -26,6 +26,7 @@ import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Locals.Variable;
+import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.Type;
 
 import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
@@ -77,19 +78,19 @@ public class ECapturingFunctionRef extends AExpression {
         if (defInterface && captured.type.sort == Definition.Sort.DEF) {
             // dynamic interface, dynamic implementation
             writer.push("D" + type + "." + call + ",1");
-            writer.loadLocal(captured.slot);
+            writer.visitVarInsn(captured.type.type.getOpcode(Opcodes.ILOAD), captured.slot);
         } else if (defInterface) {
             // dynamic interface, typed implementation
             writer.push("S" + captured.type.name + "." + call + ",1");
-            writer.loadLocal(captured.slot);
+            writer.visitVarInsn(captured.type.type.getOpcode(Opcodes.ILOAD), captured.slot);
         } else if (ref == null) {
             // typed interface, dynamic implementation
-            writer.loadLocal(captured.slot);
+            writer.visitVarInsn(captured.type.type.getOpcode(Opcodes.ILOAD), captured.slot);
             String descriptor = Type.getMethodType(expected.type, captured.type.type).getDescriptor();
             writer.invokeDynamic(call, descriptor, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.REFERENCE, expected.name);
         } else {
             // typed interface, typed implementation
-            writer.loadLocal(captured.slot);
+            writer.visitVarInsn(captured.type.type.getOpcode(Opcodes.ILOAD), captured.slot);
             // convert MethodTypes to asm Type for the constant pool.
             String invokedType = ref.invokedType.toMethodDescriptorString();
             Type samMethodType = Type.getMethodType(ref.samMethodType.toMethodDescriptorString());
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
index 8ab69366f24..d98e5f68bcf 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
@@ -35,6 +35,7 @@
  * {@link org.elasticsearch.painless.node.EBinary} - Represents a binary math expression.
  * {@link org.elasticsearch.painless.node.EBool} - Represents a boolean expression.
  * {@link org.elasticsearch.painless.node.EBoolean} - Represents a boolean constant.
+ * {@link org.elasticsearch.painless.node.ECapturingFunctionRef} - Represents a function reference (capturing).
  * {@link org.elasticsearch.painless.node.ECast} - Represents an implicit cast in most cases.  (Internal only.)
  * {@link org.elasticsearch.painless.node.EChain} - Represents the entirety of a variable/method chain for read/write operations.
  * {@link org.elasticsearch.painless.node.EComp} - Represents a comparison expression.
@@ -42,7 +43,7 @@
  * {@link org.elasticsearch.painless.node.EConstant} - Represents a constant.  (Internal only.)
  * {@link org.elasticsearch.painless.node.EDecimal} - Represents a decimal constant.
  * {@link org.elasticsearch.painless.node.EExplicit} - Represents an explicit cast.
- * {@link org.elasticsearch.painless.node.EFunctionRef} - Represents a function reference.
+ * {@link org.elasticsearch.painless.node.EFunctionRef} - Represents a function reference (non-capturing).
  * {@link org.elasticsearch.painless.node.ENull} - Represents a null constant.
  * {@link org.elasticsearch.painless.node.ENumeric} - Represents a non-decimal numeric constant.
  * {@link org.elasticsearch.painless.node.EUnary} - Represents a unary math expression.

From 387155559ead75fbc6d6f330528229be47e75781 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 10 Jun 2016 15:32:30 -0400
Subject: [PATCH 0785/1311] Make TimeValue Writeable instead of Streamable

Writeable is better for immutable objects like TimeValue.

Switch to writeZLong which takes up less space than the original
writeLong in the majority of cases. Since we expect negative
TimeValues we shouldn't use
writeVLong.
---
 .../cluster/health/ClusterHealthRequest.java  |  4 +-
 .../cluster/health/ClusterHealthResponse.java |  2 +-
 .../hotthreads/NodesHotThreadsRequest.java    |  2 +-
 .../action/bulk/BulkRequest.java              |  2 +-
 .../action/index/IndexRequest.java            | 10 +---
 .../support/master/AcknowledgedRequest.java   |  3 +-
 .../support/master/MasterNodeRequest.java     |  2 +-
 .../support/nodes/BaseNodesRequest.java       | 11 +----
 .../replication/ReplicationRequest.java       |  2 +-
 .../InstanceShardOperationRequest.java        |  2 +-
 .../support/tasks/BaseTasksRequest.java       |  6 +--
 .../common/io/stream/Streamable.java          |  2 +
 .../common/io/stream/Writeable.java           |  2 -
 .../elasticsearch/common/unit/TimeValue.java  | 46 +++++++------------
 .../zen/ping/unicast/UnicastZenPing.java      |  4 +-
 .../java/org/elasticsearch/search/Scroll.java | 13 +-----
 .../elasticsearch/threadpool/ThreadPool.java  |  2 +-
 .../common/unit/TimeValueTests.java           | 14 ++++--
 .../reindex/AbstractBulkByScrollRequest.java  |  4 +-
 .../index/reindex/BulkByScrollTask.java       |  4 +-
 .../reindex/BulkIndexByScrollResponse.java    |  2 +-
 21 files changed, 52 insertions(+), 87 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java
index 59b426d8c31..27970f332fc 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java
@@ -33,8 +33,6 @@ import org.elasticsearch.common.unit.TimeValue;
 import java.io.IOException;
 import java.util.concurrent.TimeUnit;
 
-import static org.elasticsearch.common.unit.TimeValue.readTimeValue;
-
 /**
  *
  */
@@ -160,7 +158,7 @@ public class ClusterHealthRequest extends MasterNodeReadRequest<ClusterHealthReq
                 indices[i] = in.readString();
             }
         }
-        timeout = readTimeValue(in);
+        timeout = new TimeValue(in);
         if (in.readBoolean()) {
             waitForStatus = ClusterHealthStatus.fromValue(in.readByte());
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java
index 0d3bfb970af..d483ae86bf7 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java
@@ -187,7 +187,7 @@ public class ClusterHealthResponse extends ActionResponse implements StatusToXCo
         timedOut = in.readBoolean();
         numberOfInFlightFetch = in.readInt();
         delayedUnassignedShards= in.readInt();
-        taskMaxWaitingTime = TimeValue.readTimeValue(in);
+        taskMaxWaitingTime = new TimeValue(in);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java
index 7c8f797fdcb..e3df7f57312 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java
@@ -101,7 +101,7 @@ public class NodesHotThreadsRequest extends BaseNodesRequest<NodesHotThreadsRequ
         threads = in.readInt();
         ignoreIdleThreads = in.readBoolean();
         type = in.readString();
-        interval = TimeValue.readTimeValue(in);
+        interval = new TimeValue(in);
         snapshots = in.readInt();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
index 85d7147ada0..e0572344656 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java
@@ -544,7 +544,7 @@ public class BulkRequest extends ActionRequest<BulkRequest> implements Composite
             }
         }
         refreshPolicy = RefreshPolicy.readFrom(in);
-        timeout = TimeValue.readTimeValue(in);
+        timeout = new TimeValue(in);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java
index bc1e631e559..a79f6d83204 100644
--- a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java
@@ -633,9 +633,8 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
         routing = in.readOptionalString();
         parent = in.readOptionalString();
         timestamp = in.readOptionalString();
-        ttl = in.readBoolean() ? TimeValue.readTimeValue(in) : null;
+        ttl = in.readOptionalWriteable(TimeValue::new);
         source = in.readBytesReference();
-
         opType = OpType.fromId(in.readByte());
         version = in.readLong();
         versionType = VersionType.fromValue(in.readByte());
@@ -650,12 +649,7 @@ public class IndexRequest extends ReplicatedWriteRequest<IndexRequest> implement
         out.writeOptionalString(routing);
         out.writeOptionalString(parent);
         out.writeOptionalString(timestamp);
-        if (ttl == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            ttl.writeTo(out);
-        }
+        out.writeOptionalWriteable(ttl);
         out.writeBytesReference(source);
         out.writeByte(opType.id());
         out.writeLong(version);
diff --git a/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java b/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java
index 9e45bccc547..e3f32543bf2 100644
--- a/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/master/AcknowledgedRequest.java
@@ -25,7 +25,6 @@ import org.elasticsearch.common.unit.TimeValue;
 
 import java.io.IOException;
 
-import static org.elasticsearch.common.unit.TimeValue.readTimeValue;
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 
 /**
@@ -75,7 +74,7 @@ public abstract class AcknowledgedRequest<Request extends MasterNodeRequest<Requ
      * Reads the timeout value
      */
     protected void readTimeout(StreamInput in) throws IOException {
-        timeout = readTimeValue(in);
+        timeout = new TimeValue(in);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java b/core/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java
index a964a44a140..efbcadf445f 100644
--- a/core/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/master/MasterNodeRequest.java
@@ -61,7 +61,7 @@ public abstract class MasterNodeRequest<Request extends MasterNodeRequest<Reques
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        masterNodeTimeout = TimeValue.readTimeValue(in);
+        masterNodeTimeout = new TimeValue(in);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java b/core/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java
index 5176ae52516..d5375ef9ef9 100644
--- a/core/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java
@@ -82,20 +82,13 @@ public abstract class BaseNodesRequest<Request extends BaseNodesRequest<Request>
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         nodesIds = in.readStringArray();
-        if (in.readBoolean()) {
-            timeout = TimeValue.readTimeValue(in);
-        }
+        timeout = in.readOptionalWriteable(TimeValue::new);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeStringArrayNullable(nodesIds);
-        if (timeout == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            timeout.writeTo(out);
-        }
+        out.writeOptionalWriteable(timeout);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java
index 44c420598b5..c444fd2cf39 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java
@@ -181,7 +181,7 @@ public abstract class ReplicationRequest<Request extends ReplicationRequest<Requ
             shardId = null;
         }
         consistencyLevel = WriteConsistencyLevel.fromId(in.readByte());
-        timeout = TimeValue.readTimeValue(in);
+        timeout = new TimeValue(in);
         index = in.readString();
         routedBasedOnClusterVersion = in.readVLong();
         primaryTerm = in.readVLong();
diff --git a/core/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java b/core/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java
index 186eb536b08..cb9a6ab9f69 100644
--- a/core/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/single/instance/InstanceShardOperationRequest.java
@@ -121,7 +121,7 @@ public abstract class InstanceShardOperationRequest<Request extends InstanceShar
         } else {
             shardId = null;
         }
-        timeout = TimeValue.readTimeValue(in);
+        timeout = new TimeValue(in);
         concreteIndex = in.readOptionalString();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java b/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java
index 3ab5bd65fe1..abc081dbc07 100644
--- a/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/tasks/BaseTasksRequest.java
@@ -144,9 +144,7 @@ public class BaseTasksRequest<Request extends BaseTasksRequest<Request>> extends
         parentTaskId = TaskId.readFromStream(in);
         nodesIds = in.readStringArray();
         actions = in.readStringArray();
-        if (in.readBoolean()) {
-            timeout = TimeValue.readTimeValue(in);
-        }
+        timeout = in.readOptionalWriteable(TimeValue::new);
     }
 
     @Override
@@ -156,7 +154,7 @@ public class BaseTasksRequest<Request extends BaseTasksRequest<Request>> extends
         parentTaskId.writeTo(out);
         out.writeStringArrayNullable(nodesIds);
         out.writeStringArrayNullable(actions);
-        out.writeOptionalStreamable(timeout);
+        out.writeOptionalWriteable(timeout);
     }
 
     public boolean match(Task task) {
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/Streamable.java b/core/src/main/java/org/elasticsearch/common/io/stream/Streamable.java
index 4added7df31..a37c6371482 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/Streamable.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/Streamable.java
@@ -26,6 +26,8 @@ import java.io.IOException;
  * across the wire" using Elasticsearch's internal protocol. If the implementer also implements equals and hashCode then a copy made by
  * serializing and deserializing must be equal and have the same hashCode. It isn't required that such a copy be entirely unchanged. For
  * example, {@link org.elasticsearch.common.unit.TimeValue} converts the time to nanoseconds for serialization.
+ * {@linkplain org.elasticsearch.common.unit.TimeValue} actually implements {@linkplain Writeable} not {@linkplain Streamable} but it has
+ * the same contract.
  *
  * Prefer implementing {@link Writeable} over implementing this interface where possible. Lots of code depends on this interface so this
  * isn't always possible.
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java b/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java
index 55ab419881f..cf127e5b968 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java
@@ -26,8 +26,6 @@ import java.io.IOException;
  * across the wire" using Elasticsearch's internal protocol. If the implementer also implements equals and hashCode then a copy made by
  * serializing and deserializing must be equal and have the same hashCode. It isn't required that such a copy be entirely unchanged. For
  * example, {@link org.elasticsearch.common.unit.TimeValue} converts the time to nanoseconds for serialization.
- * {@linkplain org.elasticsearch.common.unit.TimeValue} actually implements {@linkplain Streamable} not {@linkplain Writeable} but it has
- * the same contract.
  *
  * Prefer implementing this interface over implementing {@link Streamable} where possible. Lots of code depends on {@linkplain Streamable}
  * so this isn't always possible.
diff --git a/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java b/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java
index c467a0c18a8..2058355d300 100644
--- a/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java
+++ b/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java
@@ -23,7 +23,7 @@ import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.joda.time.Period;
 import org.joda.time.PeriodType;
 import org.joda.time.format.PeriodFormat;
@@ -34,7 +34,7 @@ import java.util.Locale;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 
-public class TimeValue implements Streamable {
+public class TimeValue implements Writeable {
 
     /** How many nano-seconds in one milli-second */
     public static final long NSEC_PER_MSEC = 1000000;
@@ -59,13 +59,8 @@ public class TimeValue implements Streamable {
         return new TimeValue(hours, TimeUnit.HOURS);
     }
 
-    private long duration;
-
-    private TimeUnit timeUnit;
-
-    private TimeValue() {
-
-    }
+    private final long duration;
+    private final TimeUnit timeUnit;
 
     public TimeValue(long millis) {
         this(millis, TimeUnit.MILLISECONDS);
@@ -76,6 +71,19 @@ public class TimeValue implements Streamable {
         this.timeUnit = timeUnit;
     }
 
+    /**
+     * Read from a stream.
+     */
+    public TimeValue(StreamInput in) throws IOException {
+        duration = in.readZLong();
+        timeUnit = TimeUnit.NANOSECONDS;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeZLong(nanos());
+    }
+
     public long nanos() {
         return timeUnit.toNanos(duration);
     }
@@ -304,26 +312,6 @@ public class TimeValue implements Streamable {
     static final long C5 = C4 * 60L;
     static final long C6 = C5 * 24L;
 
-    public static TimeValue readTimeValue(StreamInput in) throws IOException {
-        TimeValue timeValue = new TimeValue();
-        timeValue.readFrom(in);
-        return timeValue;
-    }
-
-    /**
-     * serialization converts TimeValue internally to NANOSECONDS
-     */
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        duration = in.readLong();
-        timeUnit = TimeUnit.NANOSECONDS;
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeLong(nanos());
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
index d200ca5b07b..1b4f1ba75b0 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.discovery.zen.ping.unicast;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
+
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
@@ -79,7 +80,6 @@ import java.util.function.Function;
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
-import static org.elasticsearch.common.unit.TimeValue.readTimeValue;
 import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap;
 import static org.elasticsearch.discovery.zen.ping.ZenPing.PingResponse.readPingResponse;
 
@@ -545,7 +545,7 @@ public class UnicastZenPing extends AbstractLifecycleComponent<ZenPing> implemen
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
             id = in.readInt();
-            timeout = readTimeValue(in);
+            timeout = new TimeValue(in);
             pingResponse = readPingResponse(in);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/search/Scroll.java b/core/src/main/java/org/elasticsearch/search/Scroll.java
index 2d703bdae90..13298b2cbbe 100644
--- a/core/src/main/java/org/elasticsearch/search/Scroll.java
+++ b/core/src/main/java/org/elasticsearch/search/Scroll.java
@@ -26,8 +26,6 @@ import org.elasticsearch.common.unit.TimeValue;
 
 import java.io.IOException;
 
-import static org.elasticsearch.common.unit.TimeValue.readTimeValue;
-
 /**
  * A scroll enables scrolling of search request. It holds a {@link #keepAlive()} time that
  * will control how long to keep the scrolling resources open.
@@ -64,18 +62,11 @@ public class Scroll implements Streamable {
 
     @Override
     public void readFrom(StreamInput in) throws IOException {
-        if (in.readBoolean()) {
-            keepAlive = readTimeValue(in);
-        }
+        in.readOptionalWriteable(TimeValue::new);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (keepAlive == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            keepAlive.writeTo(out);
-        }
+        out.writeOptionalWriteable(keepAlive);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
index 5c31323b3d8..f8cb882b083 100644
--- a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
+++ b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
@@ -579,7 +579,7 @@ public class ThreadPool extends AbstractComponent implements Closeable {
             min = in.readInt();
             max = in.readInt();
             if (in.readBoolean()) {
-                keepAlive = TimeValue.readTimeValue(in);
+                keepAlive = new TimeValue(in);
             }
             if (in.readBoolean()) {
                 queueSize = SizeValue.readSizeValue(in);
diff --git a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
index 3cd68bea038..cc36625e68f 100644
--- a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
+++ b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
@@ -28,6 +28,8 @@ import org.joda.time.PeriodType;
 import java.io.IOException;
 import java.util.concurrent.TimeUnit;
 
+import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
+import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.lessThan;
@@ -123,20 +125,22 @@ public class TimeValueTests extends ESTestCase {
                      TimeValue.parseTimeValue("10W", null, "test"));
     }
 
-    private void assertEqualityAfterSerialize(TimeValue value) throws IOException {
+    private void assertEqualityAfterSerialize(TimeValue value, int expectedSize) throws IOException {
         BytesStreamOutput out = new BytesStreamOutput();
         value.writeTo(out);
+        assertEquals(expectedSize, out.size());
 
         StreamInput in = StreamInput.wrap(out.bytes());
-        TimeValue inValue = TimeValue.readTimeValue(in);
+        TimeValue inValue = new TimeValue(in);
 
         assertThat(inValue, equalTo(value));
     }
 
     public void testSerialize() throws Exception {
-        assertEqualityAfterSerialize(new TimeValue(100, TimeUnit.DAYS));
-        assertEqualityAfterSerialize(new TimeValue(-1));
-        assertEqualityAfterSerialize(new TimeValue(1, TimeUnit.NANOSECONDS));
+        assertEqualityAfterSerialize(new TimeValue(100, TimeUnit.DAYS), 8);
+        assertEqualityAfterSerialize(timeValueNanos(-1), 1);
+        assertEqualityAfterSerialize(timeValueNanos(1), 1);
+        assertEqualityAfterSerialize(timeValueSeconds(30), 6);
     }
 
     public void testFailOnUnknownUnits() {
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
index 7505f490f45..499dfc82457 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
@@ -299,9 +299,9 @@ public abstract class AbstractBulkByScrollRequest<Self extends AbstractBulkByScr
         abortOnVersionConflict = in.readBoolean();
         size = in.readVInt();
         refresh = in.readBoolean();
-        timeout = TimeValue.readTimeValue(in);
+        timeout = new TimeValue(in);
         consistency = WriteConsistencyLevel.fromId(in.readByte());
-        retryBackoffInitialTime = TimeValue.readTimeValue(in);
+        retryBackoffInitialTime = new TimeValue(in);
         maxRetries = in.readVInt();
         requestsPerSecond = in.readFloat();
     }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
index 7af32740be2..b778210611e 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
@@ -168,10 +168,10 @@ public class BulkByScrollTask extends CancellableTask {
             noops = in.readVLong();
             bulkRetries = in.readVLong();
             searchRetries = in.readVLong();
-            throttled = TimeValue.readTimeValue(in);
+            throttled = new TimeValue(in);
             requestsPerSecond = in.readFloat();
             reasonCancelled = in.readOptionalString();
-            throttledUntil = TimeValue.readTimeValue(in);
+            throttledUntil = new TimeValue(in);
         }
 
         @Override
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java
index 8e20375d9a7..bfd139746ca 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java
@@ -153,7 +153,7 @@ public class BulkIndexByScrollResponse extends ActionResponse implements ToXCont
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        took = TimeValue.readTimeValue(in);
+        took = new TimeValue(in);
         status = new BulkByScrollTask.Status(in);
         int indexingFailuresCount = in.readVInt();
         List<Failure> indexingFailures = new ArrayList<>(indexingFailuresCount);

From bd94c62adf42e2bdde88ce13b1be7a09774c2423 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Fri, 10 Jun 2016 16:18:36 -0700
Subject: [PATCH 0786/1311] Added a stub node for lambda expressions.

---
 .../src/main/antlr/PainlessLexer.g4           |   1 +
 .../src/main/antlr/PainlessLexer.tokens       |  88 +-
 .../src/main/antlr/PainlessParser.g4          |  17 +-
 .../src/main/antlr/PainlessParser.tokens      |  88 +-
 .../painless/antlr/PainlessLexer.java         | 385 ++++----
 .../painless/antlr/PainlessParser.java        | 890 +++++++++++-------
 .../antlr/PainlessParserBaseVisitor.java      |  28 +-
 .../painless/antlr/PainlessParserVisitor.java |  24 +-
 .../elasticsearch/painless/antlr/Walker.java  |  61 +-
 .../elasticsearch/painless/node/ELambda.java  |  57 ++
 .../painless/BasicStatementTests.java         |  11 +-
 .../painless/antlr/ParserTests.java           |  13 +
 12 files changed, 992 insertions(+), 671 deletions(-)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java

diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.g4 b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
index 4ca5778ad3c..2ae77efd4d0 100644
--- a/modules/lang-painless/src/main/antlr/PainlessLexer.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
@@ -78,6 +78,7 @@ BOOLOR:  '||';
 COND:    '?';
 COLON:   ':';
 REF:     '::';
+ARROW:   '->';
 INCR:    '++';
 DECR:    '--';
 
diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.tokens b/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
index 567fb8fdddd..f485246aa67 100644
--- a/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
@@ -47,32 +47,33 @@ BOOLOR=46
 COND=47
 COLON=48
 REF=49
-INCR=50
-DECR=51
-ASSIGN=52
-AADD=53
-ASUB=54
-AMUL=55
-ADIV=56
-AREM=57
-AAND=58
-AXOR=59
-AOR=60
-ALSH=61
-ARSH=62
-AUSH=63
-OCTAL=64
-HEX=65
-INTEGER=66
-DECIMAL=67
-STRING=68
-TRUE=69
-FALSE=70
-NULL=71
-TYPE=72
-ID=73
-DOTINTEGER=74
-DOTID=75
+ARROW=50
+INCR=51
+DECR=52
+ASSIGN=53
+AADD=54
+ASUB=55
+AMUL=56
+ADIV=57
+AREM=58
+AAND=59
+AXOR=60
+AOR=61
+ALSH=62
+ARSH=63
+AUSH=64
+OCTAL=65
+HEX=66
+INTEGER=67
+DECIMAL=68
+STRING=69
+TRUE=70
+FALSE=71
+NULL=72
+TYPE=73
+ID=74
+DOTINTEGER=75
+DOTID=76
 '{'=3
 '}'=4
 '['=5
@@ -120,20 +121,21 @@ DOTID=75
 '?'=47
 ':'=48
 '::'=49
-'++'=50
-'--'=51
-'='=52
-'+='=53
-'-='=54
-'*='=55
-'/='=56
-'%='=57
-'&='=58
-'^='=59
-'|='=60
-'<<='=61
-'>>='=62
-'>>>='=63
-'true'=69
-'false'=70
-'null'=71
+'->'=50
+'++'=51
+'--'=52
+'='=53
+'+='=54
+'-='=55
+'*='=56
+'/='=57
+'%='=58
+'&='=59
+'^='=60
+'|='=61
+'<<='=62
+'>>='=63
+'>>>='=64
+'true'=70
+'false'=71
+'null'=72
diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index 31b6a9445ea..bd968268e38 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -81,10 +81,6 @@ decltype
     : TYPE (LBRACE RBRACE)*
     ;
 
-funcref
-    : TYPE REF ( ID | NEW )
-    ;
-
 declvar
     : ID ( ASSIGN expression )?
     ;
@@ -175,5 +171,18 @@ arguments
 
 argument
     : expression
+    | lambda
     | funcref
     ;
+
+lambda
+    : ( lamtype | LP ( lamtype ( COMMA lamtype )* )? RP ) ARROW block
+    ;
+
+lamtype
+    : decltype? ID
+    ;
+
+funcref
+    : TYPE REF ( ID | NEW )
+    ;
\ No newline at end of file
diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.tokens b/modules/lang-painless/src/main/antlr/PainlessParser.tokens
index 567fb8fdddd..f485246aa67 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.tokens
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.tokens
@@ -47,32 +47,33 @@ BOOLOR=46
 COND=47
 COLON=48
 REF=49
-INCR=50
-DECR=51
-ASSIGN=52
-AADD=53
-ASUB=54
-AMUL=55
-ADIV=56
-AREM=57
-AAND=58
-AXOR=59
-AOR=60
-ALSH=61
-ARSH=62
-AUSH=63
-OCTAL=64
-HEX=65
-INTEGER=66
-DECIMAL=67
-STRING=68
-TRUE=69
-FALSE=70
-NULL=71
-TYPE=72
-ID=73
-DOTINTEGER=74
-DOTID=75
+ARROW=50
+INCR=51
+DECR=52
+ASSIGN=53
+AADD=54
+ASUB=55
+AMUL=56
+ADIV=57
+AREM=58
+AAND=59
+AXOR=60
+AOR=61
+ALSH=62
+ARSH=63
+AUSH=64
+OCTAL=65
+HEX=66
+INTEGER=67
+DECIMAL=68
+STRING=69
+TRUE=70
+FALSE=71
+NULL=72
+TYPE=73
+ID=74
+DOTINTEGER=75
+DOTID=76
 '{'=3
 '}'=4
 '['=5
@@ -120,20 +121,21 @@ DOTID=75
 '?'=47
 ':'=48
 '::'=49
-'++'=50
-'--'=51
-'='=52
-'+='=53
-'-='=54
-'*='=55
-'/='=56
-'%='=57
-'&='=58
-'^='=59
-'|='=60
-'<<='=61
-'>>='=62
-'>>>='=63
-'true'=69
-'false'=70
-'null'=71
+'->'=50
+'++'=51
+'--'=52
+'='=53
+'+='=54
+'-='=55
+'*='=56
+'/='=57
+'%='=58
+'&='=59
+'^='=60
+'|='=61
+'<<='=62
+'>>='=63
+'>>>='=64
+'true'=70
+'false'=71
+'null'=72
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
index 9eb5730abc0..566db679ef9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
@@ -25,11 +25,11 @@ class PainlessLexer extends Lexer {
     BREAK=18, RETURN=19, NEW=20, TRY=21, CATCH=22, THROW=23, BOOLNOT=24, BWNOT=25, 
     MUL=26, DIV=27, REM=28, ADD=29, SUB=30, LSH=31, RSH=32, USH=33, LT=34, 
     LTE=35, GT=36, GTE=37, EQ=38, EQR=39, NE=40, NER=41, BWAND=42, XOR=43, 
-    BWOR=44, BOOLAND=45, BOOLOR=46, COND=47, COLON=48, REF=49, INCR=50, DECR=51, 
-    ASSIGN=52, AADD=53, ASUB=54, AMUL=55, ADIV=56, AREM=57, AAND=58, AXOR=59, 
-    AOR=60, ALSH=61, ARSH=62, AUSH=63, OCTAL=64, HEX=65, INTEGER=66, DECIMAL=67, 
-    STRING=68, TRUE=69, FALSE=70, NULL=71, TYPE=72, ID=73, DOTINTEGER=74, 
-    DOTID=75;
+    BWOR=44, BOOLAND=45, BOOLOR=46, COND=47, COLON=48, REF=49, ARROW=50, INCR=51, 
+    DECR=52, ASSIGN=53, AADD=54, ASUB=55, AMUL=56, ADIV=57, AREM=58, AAND=59, 
+    AXOR=60, AOR=61, ALSH=62, ARSH=63, AUSH=64, OCTAL=65, HEX=66, INTEGER=67, 
+    DECIMAL=68, STRING=69, TRUE=70, FALSE=71, NULL=72, TYPE=73, ID=74, DOTINTEGER=75, 
+    DOTID=76;
   public static final int AFTER_DOT = 1;
   public static String[] modeNames = {
     "DEFAULT_MODE", "AFTER_DOT"
@@ -41,10 +41,10 @@ class PainlessLexer extends Lexer {
     "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "BOOLNOT", "BWNOT", 
     "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", "LTE", "GT", 
     "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", "BOOLAND", "BOOLOR", 
-    "COND", "COLON", "REF", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", 
-    "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", 
-    "HEX", "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "TYPE", 
-    "ID", "DOTINTEGER", "DOTID"
+    "COND", "COLON", "REF", "ARROW", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", 
+    "AMUL", "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", 
+    "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", 
+    "TYPE", "ID", "DOTINTEGER", "DOTID"
   };
 
   private static final String[] _LITERAL_NAMES = {
@@ -53,9 +53,9 @@ class PainlessLexer extends Lexer {
     "'return'", "'new'", "'try'", "'catch'", "'throw'", "'!'", "'~'", "'*'", 
     "'/'", "'%'", "'+'", "'-'", "'<<'", "'>>'", "'>>>'", "'<'", "'<='", "'>'", 
     "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", "'^'", "'|'", "'&&'", 
-    "'||'", "'?'", "':'", "'::'", "'++'", "'--'", "'='", "'+='", "'-='", "'*='", 
-    "'/='", "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", "'>>>='", null, 
-    null, null, null, null, "'true'", "'false'", "'null'"
+    "'||'", "'?'", "':'", "'::'", "'->'", "'++'", "'--'", "'='", "'+='", "'-='", 
+    "'*='", "'/='", "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", "'>>>='", 
+    null, null, null, null, null, "'true'", "'false'", "'null'"
   };
   private static final String[] _SYMBOLIC_NAMES = {
     null, "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", 
@@ -63,10 +63,10 @@ class PainlessLexer extends Lexer {
     "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "BOOLNOT", "BWNOT", 
     "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", "LTE", "GT", 
     "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", "BOOLAND", "BOOLOR", 
-    "COND", "COLON", "REF", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", 
-    "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", 
-    "HEX", "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "TYPE", 
-    "ID", "DOTINTEGER", "DOTID"
+    "COND", "COLON", "REF", "ARROW", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", 
+    "AMUL", "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", 
+    "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", 
+    "TYPE", "ID", "DOTINTEGER", "DOTID"
   };
   public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
 
@@ -125,7 +125,7 @@ class PainlessLexer extends Lexer {
   @Override
   public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) {
     switch (ruleIndex) {
-    case 71:
+    case 72:
       return TYPE_sempred((RuleContext)_localctx, predIndex);
     }
     return true;
@@ -139,7 +139,7 @@ class PainlessLexer extends Lexer {
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2M\u020e\b\1\b\1\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2N\u0213\b\1\b\1\4"+
     "\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n"+
     "\4\13\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22"+
     "\t\22\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31"+
@@ -148,180 +148,181 @@ class PainlessLexer extends Lexer {
     "+\4,\t,\4-\t-\4.\t.\4/\t/\4\60\t\60\4\61\t\61\4\62\t\62\4\63\t\63\4\64"+
     "\t\64\4\65\t\65\4\66\t\66\4\67\t\67\48\t8\49\t9\4:\t:\4;\t;\4<\t<\4=\t"+
     "=\4>\t>\4?\t?\4@\t@\4A\tA\4B\tB\4C\tC\4D\tD\4E\tE\4F\tF\4G\tG\4H\tH\4"+
-    "I\tI\4J\tJ\4K\tK\4L\tL\3\2\6\2\u009c\n\2\r\2\16\2\u009d\3\2\3\2\3\3\3"+
-    "\3\3\3\3\3\7\3\u00a6\n\3\f\3\16\3\u00a9\13\3\3\3\3\3\3\3\3\3\3\3\7\3\u00b0"+
-    "\n\3\f\3\16\3\u00b3\13\3\3\3\3\3\5\3\u00b7\n\3\3\3\3\3\3\4\3\4\3\5\3\5"+
-    "\3\6\3\6\3\7\3\7\3\b\3\b\3\t\3\t\3\n\3\n\3\n\3\n\3\13\3\13\3\f\3\f\3\r"+
-    "\3\r\3\r\3\16\3\16\3\16\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3\17\3\20\3"+
-    "\20\3\20\3\21\3\21\3\21\3\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3"+
-    "\22\3\23\3\23\3\23\3\23\3\23\3\23\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3"+
-    "\25\3\25\3\25\3\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\27\3\27\3\27\3"+
-    "\30\3\30\3\30\3\30\3\30\3\30\3\31\3\31\3\32\3\32\3\33\3\33\3\34\3\34\3"+
-    "\35\3\35\3\36\3\36\3\37\3\37\3 \3 \3 \3!\3!\3!\3\"\3\"\3\"\3\"\3#\3#\3"+
-    "$\3$\3$\3%\3%\3&\3&\3&\3\'\3\'\3\'\3(\3(\3(\3(\3)\3)\3)\3*\3*\3*\3*\3"+
-    "+\3+\3,\3,\3-\3-\3.\3.\3.\3/\3/\3/\3\60\3\60\3\61\3\61\3\62\3\62\3\62"+
-    "\3\63\3\63\3\63\3\64\3\64\3\64\3\65\3\65\3\66\3\66\3\66\3\67\3\67\3\67"+
-    "\38\38\38\39\39\39\3:\3:\3:\3;\3;\3;\3<\3<\3<\3=\3=\3=\3>\3>\3>\3>\3?"+
-    "\3?\3?\3?\3@\3@\3@\3@\3@\3A\3A\6A\u0180\nA\rA\16A\u0181\3A\5A\u0185\n"+
-    "A\3B\3B\3B\6B\u018a\nB\rB\16B\u018b\3B\5B\u018f\nB\3C\3C\3C\7C\u0194\n"+
-    "C\fC\16C\u0197\13C\5C\u0199\nC\3C\5C\u019c\nC\3D\3D\3D\7D\u01a1\nD\fD"+
-    "\16D\u01a4\13D\5D\u01a6\nD\3D\3D\6D\u01aa\nD\rD\16D\u01ab\5D\u01ae\nD"+
-    "\3D\3D\5D\u01b2\nD\3D\6D\u01b5\nD\rD\16D\u01b6\5D\u01b9\nD\3D\5D\u01bc"+
-    "\nD\3E\3E\3E\3E\3E\3E\7E\u01c4\nE\fE\16E\u01c7\13E\3E\3E\3E\3E\3E\3E\3"+
-    "E\7E\u01d0\nE\fE\16E\u01d3\13E\3E\5E\u01d6\nE\3F\3F\3F\3F\3F\3G\3G\3G"+
-    "\3G\3G\3G\3H\3H\3H\3H\3H\3I\3I\3I\3I\7I\u01ec\nI\fI\16I\u01ef\13I\3I\3"+
-    "I\3J\3J\7J\u01f5\nJ\fJ\16J\u01f8\13J\3K\3K\3K\7K\u01fd\nK\fK\16K\u0200"+
-    "\13K\5K\u0202\nK\3K\3K\3L\3L\7L\u0208\nL\fL\16L\u020b\13L\3L\3L\6\u00a7"+
-    "\u00b1\u01c5\u01d1\2M\4\3\6\4\b\5\n\6\f\7\16\b\20\t\22\n\24\13\26\f\30"+
-    "\r\32\16\34\17\36\20 \21\"\22$\23&\24(\25*\26,\27.\30\60\31\62\32\64\33"+
-    "\66\348\35:\36<\37> @!B\"D#F$H%J&L\'N(P)R*T+V,X-Z.\\/^\60`\61b\62d\63"+
-    "f\64h\65j\66l\67n8p9r:t;v<x=z>|?~@\u0080A\u0082B\u0084C\u0086D\u0088E"+
-    "\u008aF\u008cG\u008eH\u0090I\u0092J\u0094K\u0096L\u0098M\4\2\3\21\5\2"+
-    "\13\f\17\17\"\"\4\2\f\f\17\17\3\2\629\4\2NNnn\4\2ZZzz\5\2\62;CHch\3\2"+
-    "\63;\3\2\62;\b\2FFHHNNffhhnn\4\2GGgg\4\2--//\4\2HHhh\4\2$$^^\5\2C\\aa"+
-    "c|\6\2\62;C\\aac|\u022b\2\4\3\2\2\2\2\6\3\2\2\2\2\b\3\2\2\2\2\n\3\2\2"+
-    "\2\2\f\3\2\2\2\2\16\3\2\2\2\2\20\3\2\2\2\2\22\3\2\2\2\2\24\3\2\2\2\2\26"+
-    "\3\2\2\2\2\30\3\2\2\2\2\32\3\2\2\2\2\34\3\2\2\2\2\36\3\2\2\2\2 \3\2\2"+
-    "\2\2\"\3\2\2\2\2$\3\2\2\2\2&\3\2\2\2\2(\3\2\2\2\2*\3\2\2\2\2,\3\2\2\2"+
-    "\2.\3\2\2\2\2\60\3\2\2\2\2\62\3\2\2\2\2\64\3\2\2\2\2\66\3\2\2\2\28\3\2"+
-    "\2\2\2:\3\2\2\2\2<\3\2\2\2\2>\3\2\2\2\2@\3\2\2\2\2B\3\2\2\2\2D\3\2\2\2"+
-    "\2F\3\2\2\2\2H\3\2\2\2\2J\3\2\2\2\2L\3\2\2\2\2N\3\2\2\2\2P\3\2\2\2\2R"+
-    "\3\2\2\2\2T\3\2\2\2\2V\3\2\2\2\2X\3\2\2\2\2Z\3\2\2\2\2\\\3\2\2\2\2^\3"+
-    "\2\2\2\2`\3\2\2\2\2b\3\2\2\2\2d\3\2\2\2\2f\3\2\2\2\2h\3\2\2\2\2j\3\2\2"+
-    "\2\2l\3\2\2\2\2n\3\2\2\2\2p\3\2\2\2\2r\3\2\2\2\2t\3\2\2\2\2v\3\2\2\2\2"+
-    "x\3\2\2\2\2z\3\2\2\2\2|\3\2\2\2\2~\3\2\2\2\2\u0080\3\2\2\2\2\u0082\3\2"+
-    "\2\2\2\u0084\3\2\2\2\2\u0086\3\2\2\2\2\u0088\3\2\2\2\2\u008a\3\2\2\2\2"+
-    "\u008c\3\2\2\2\2\u008e\3\2\2\2\2\u0090\3\2\2\2\2\u0092\3\2\2\2\2\u0094"+
-    "\3\2\2\2\3\u0096\3\2\2\2\3\u0098\3\2\2\2\4\u009b\3\2\2\2\6\u00b6\3\2\2"+
-    "\2\b\u00ba\3\2\2\2\n\u00bc\3\2\2\2\f\u00be\3\2\2\2\16\u00c0\3\2\2\2\20"+
-    "\u00c2\3\2\2\2\22\u00c4\3\2\2\2\24\u00c6\3\2\2\2\26\u00ca\3\2\2\2\30\u00cc"+
-    "\3\2\2\2\32\u00ce\3\2\2\2\34\u00d1\3\2\2\2\36\u00d6\3\2\2\2 \u00dc\3\2"+
-    "\2\2\"\u00df\3\2\2\2$\u00e3\3\2\2\2&\u00ec\3\2\2\2(\u00f2\3\2\2\2*\u00f9"+
-    "\3\2\2\2,\u00fd\3\2\2\2.\u0101\3\2\2\2\60\u0107\3\2\2\2\62\u010d\3\2\2"+
-    "\2\64\u010f\3\2\2\2\66\u0111\3\2\2\28\u0113\3\2\2\2:\u0115\3\2\2\2<\u0117"+
-    "\3\2\2\2>\u0119\3\2\2\2@\u011b\3\2\2\2B\u011e\3\2\2\2D\u0121\3\2\2\2F"+
-    "\u0125\3\2\2\2H\u0127\3\2\2\2J\u012a\3\2\2\2L\u012c\3\2\2\2N\u012f\3\2"+
-    "\2\2P\u0132\3\2\2\2R\u0136\3\2\2\2T\u0139\3\2\2\2V\u013d\3\2\2\2X\u013f"+
-    "\3\2\2\2Z\u0141\3\2\2\2\\\u0143\3\2\2\2^\u0146\3\2\2\2`\u0149\3\2\2\2"+
-    "b\u014b\3\2\2\2d\u014d\3\2\2\2f\u0150\3\2\2\2h\u0153\3\2\2\2j\u0156\3"+
-    "\2\2\2l\u0158\3\2\2\2n\u015b\3\2\2\2p\u015e\3\2\2\2r\u0161\3\2\2\2t\u0164"+
-    "\3\2\2\2v\u0167\3\2\2\2x\u016a\3\2\2\2z\u016d\3\2\2\2|\u0170\3\2\2\2~"+
-    "\u0174\3\2\2\2\u0080\u0178\3\2\2\2\u0082\u017d\3\2\2\2\u0084\u0186\3\2"+
-    "\2\2\u0086\u0198\3\2\2\2\u0088\u01a5\3\2\2\2\u008a\u01d5\3\2\2\2\u008c"+
-    "\u01d7\3\2\2\2\u008e\u01dc\3\2\2\2\u0090\u01e2\3\2\2\2\u0092\u01e7\3\2"+
-    "\2\2\u0094\u01f2\3\2\2\2\u0096\u0201\3\2\2\2\u0098\u0205\3\2\2\2\u009a"+
-    "\u009c\t\2\2\2\u009b\u009a\3\2\2\2\u009c\u009d\3\2\2\2\u009d\u009b\3\2"+
-    "\2\2\u009d\u009e\3\2\2\2\u009e\u009f\3\2\2\2\u009f\u00a0\b\2\2\2\u00a0"+
-    "\5\3\2\2\2\u00a1\u00a2\7\61\2\2\u00a2\u00a3\7\61\2\2\u00a3\u00a7\3\2\2"+
-    "\2\u00a4\u00a6\13\2\2\2\u00a5\u00a4\3\2\2\2\u00a6\u00a9\3\2\2\2\u00a7"+
-    "\u00a8\3\2\2\2\u00a7\u00a5\3\2\2\2\u00a8\u00aa\3\2\2\2\u00a9\u00a7\3\2"+
-    "\2\2\u00aa\u00b7\t\3\2\2\u00ab\u00ac\7\61\2\2\u00ac\u00ad\7,\2\2\u00ad"+
-    "\u00b1\3\2\2\2\u00ae\u00b0\13\2\2\2\u00af\u00ae\3\2\2\2\u00b0\u00b3\3"+
-    "\2\2\2\u00b1\u00b2\3\2\2\2\u00b1\u00af\3\2\2\2\u00b2\u00b4\3\2\2\2\u00b3"+
-    "\u00b1\3\2\2\2\u00b4\u00b5\7,\2\2\u00b5\u00b7\7\61\2\2\u00b6\u00a1\3\2"+
-    "\2\2\u00b6\u00ab\3\2\2\2\u00b7\u00b8\3\2\2\2\u00b8\u00b9\b\3\2\2\u00b9"+
-    "\7\3\2\2\2\u00ba\u00bb\7}\2\2\u00bb\t\3\2\2\2\u00bc\u00bd\7\177\2\2\u00bd"+
-    "\13\3\2\2\2\u00be\u00bf\7]\2\2\u00bf\r\3\2\2\2\u00c0\u00c1\7_\2\2\u00c1"+
-    "\17\3\2\2\2\u00c2\u00c3\7*\2\2\u00c3\21\3\2\2\2\u00c4\u00c5\7+\2\2\u00c5"+
-    "\23\3\2\2\2\u00c6\u00c7\7\60\2\2\u00c7\u00c8\3\2\2\2\u00c8\u00c9\b\n\3"+
-    "\2\u00c9\25\3\2\2\2\u00ca\u00cb\7.\2\2\u00cb\27\3\2\2\2\u00cc\u00cd\7"+
-    "=\2\2\u00cd\31\3\2\2\2\u00ce\u00cf\7k\2\2\u00cf\u00d0\7h\2\2\u00d0\33"+
-    "\3\2\2\2\u00d1\u00d2\7g\2\2\u00d2\u00d3\7n\2\2\u00d3\u00d4\7u\2\2\u00d4"+
-    "\u00d5\7g\2\2\u00d5\35\3\2\2\2\u00d6\u00d7\7y\2\2\u00d7\u00d8\7j\2\2\u00d8"+
-    "\u00d9\7k\2\2\u00d9\u00da\7n\2\2\u00da\u00db\7g\2\2\u00db\37\3\2\2\2\u00dc"+
-    "\u00dd\7f\2\2\u00dd\u00de\7q\2\2\u00de!\3\2\2\2\u00df\u00e0\7h\2\2\u00e0"+
-    "\u00e1\7q\2\2\u00e1\u00e2\7t\2\2\u00e2#\3\2\2\2\u00e3\u00e4\7e\2\2\u00e4"+
-    "\u00e5\7q\2\2\u00e5\u00e6\7p\2\2\u00e6\u00e7\7v\2\2\u00e7\u00e8\7k\2\2"+
-    "\u00e8\u00e9\7p\2\2\u00e9\u00ea\7w\2\2\u00ea\u00eb\7g\2\2\u00eb%\3\2\2"+
-    "\2\u00ec\u00ed\7d\2\2\u00ed\u00ee\7t\2\2\u00ee\u00ef\7g\2\2\u00ef\u00f0"+
-    "\7c\2\2\u00f0\u00f1\7m\2\2\u00f1\'\3\2\2\2\u00f2\u00f3\7t\2\2\u00f3\u00f4"+
-    "\7g\2\2\u00f4\u00f5\7v\2\2\u00f5\u00f6\7w\2\2\u00f6\u00f7\7t\2\2\u00f7"+
-    "\u00f8\7p\2\2\u00f8)\3\2\2\2\u00f9\u00fa\7p\2\2\u00fa\u00fb\7g\2\2\u00fb"+
-    "\u00fc\7y\2\2\u00fc+\3\2\2\2\u00fd\u00fe\7v\2\2\u00fe\u00ff\7t\2\2\u00ff"+
-    "\u0100\7{\2\2\u0100-\3\2\2\2\u0101\u0102\7e\2\2\u0102\u0103\7c\2\2\u0103"+
-    "\u0104\7v\2\2\u0104\u0105\7e\2\2\u0105\u0106\7j\2\2\u0106/\3\2\2\2\u0107"+
-    "\u0108\7v\2\2\u0108\u0109\7j\2\2\u0109\u010a\7t\2\2\u010a\u010b\7q\2\2"+
-    "\u010b\u010c\7y\2\2\u010c\61\3\2\2\2\u010d\u010e\7#\2\2\u010e\63\3\2\2"+
-    "\2\u010f\u0110\7\u0080\2\2\u0110\65\3\2\2\2\u0111\u0112\7,\2\2\u0112\67"+
-    "\3\2\2\2\u0113\u0114\7\61\2\2\u01149\3\2\2\2\u0115\u0116\7\'\2\2\u0116"+
-    ";\3\2\2\2\u0117\u0118\7-\2\2\u0118=\3\2\2\2\u0119\u011a\7/\2\2\u011a?"+
-    "\3\2\2\2\u011b\u011c\7>\2\2\u011c\u011d\7>\2\2\u011dA\3\2\2\2\u011e\u011f"+
-    "\7@\2\2\u011f\u0120\7@\2\2\u0120C\3\2\2\2\u0121\u0122\7@\2\2\u0122\u0123"+
-    "\7@\2\2\u0123\u0124\7@\2\2\u0124E\3\2\2\2\u0125\u0126\7>\2\2\u0126G\3"+
-    "\2\2\2\u0127\u0128\7>\2\2\u0128\u0129\7?\2\2\u0129I\3\2\2\2\u012a\u012b"+
-    "\7@\2\2\u012bK\3\2\2\2\u012c\u012d\7@\2\2\u012d\u012e\7?\2\2\u012eM\3"+
-    "\2\2\2\u012f\u0130\7?\2\2\u0130\u0131\7?\2\2\u0131O\3\2\2\2\u0132\u0133"+
-    "\7?\2\2\u0133\u0134\7?\2\2\u0134\u0135\7?\2\2\u0135Q\3\2\2\2\u0136\u0137"+
-    "\7#\2\2\u0137\u0138\7?\2\2\u0138S\3\2\2\2\u0139\u013a\7#\2\2\u013a\u013b"+
-    "\7?\2\2\u013b\u013c\7?\2\2\u013cU\3\2\2\2\u013d\u013e\7(\2\2\u013eW\3"+
-    "\2\2\2\u013f\u0140\7`\2\2\u0140Y\3\2\2\2\u0141\u0142\7~\2\2\u0142[\3\2"+
-    "\2\2\u0143\u0144\7(\2\2\u0144\u0145\7(\2\2\u0145]\3\2\2\2\u0146\u0147"+
-    "\7~\2\2\u0147\u0148\7~\2\2\u0148_\3\2\2\2\u0149\u014a\7A\2\2\u014aa\3"+
-    "\2\2\2\u014b\u014c\7<\2\2\u014cc\3\2\2\2\u014d\u014e\7<\2\2\u014e\u014f"+
-    "\7<\2\2\u014fe\3\2\2\2\u0150\u0151\7-\2\2\u0151\u0152\7-\2\2\u0152g\3"+
-    "\2\2\2\u0153\u0154\7/\2\2\u0154\u0155\7/\2\2\u0155i\3\2\2\2\u0156\u0157"+
-    "\7?\2\2\u0157k\3\2\2\2\u0158\u0159\7-\2\2\u0159\u015a\7?\2\2\u015am\3"+
-    "\2\2\2\u015b\u015c\7/\2\2\u015c\u015d\7?\2\2\u015do\3\2\2\2\u015e\u015f"+
-    "\7,\2\2\u015f\u0160\7?\2\2\u0160q\3\2\2\2\u0161\u0162\7\61\2\2\u0162\u0163"+
-    "\7?\2\2\u0163s\3\2\2\2\u0164\u0165\7\'\2\2\u0165\u0166\7?\2\2\u0166u\3"+
-    "\2\2\2\u0167\u0168\7(\2\2\u0168\u0169\7?\2\2\u0169w\3\2\2\2\u016a\u016b"+
-    "\7`\2\2\u016b\u016c\7?\2\2\u016cy\3\2\2\2\u016d\u016e\7~\2\2\u016e\u016f"+
-    "\7?\2\2\u016f{\3\2\2\2\u0170\u0171\7>\2\2\u0171\u0172\7>\2\2\u0172\u0173"+
-    "\7?\2\2\u0173}\3\2\2\2\u0174\u0175\7@\2\2\u0175\u0176\7@\2\2\u0176\u0177"+
-    "\7?\2\2\u0177\177\3\2\2\2\u0178\u0179\7@\2\2\u0179\u017a\7@\2\2\u017a"+
-    "\u017b\7@\2\2\u017b\u017c\7?\2\2\u017c\u0081\3\2\2\2\u017d\u017f\7\62"+
-    "\2\2\u017e\u0180\t\4\2\2\u017f\u017e\3\2\2\2\u0180\u0181\3\2\2\2\u0181"+
-    "\u017f\3\2\2\2\u0181\u0182\3\2\2\2\u0182\u0184\3\2\2\2\u0183\u0185\t\5"+
-    "\2\2\u0184\u0183\3\2\2\2\u0184\u0185\3\2\2\2\u0185\u0083\3\2\2\2\u0186"+
-    "\u0187\7\62\2\2\u0187\u0189\t\6\2\2\u0188\u018a\t\7\2\2\u0189\u0188\3"+
-    "\2\2\2\u018a\u018b\3\2\2\2\u018b\u0189\3\2\2\2\u018b\u018c\3\2\2\2\u018c"+
-    "\u018e\3\2\2\2\u018d\u018f\t\5\2\2\u018e\u018d\3\2\2\2\u018e\u018f\3\2"+
-    "\2\2\u018f\u0085\3\2\2\2\u0190\u0199\7\62\2\2\u0191\u0195\t\b\2\2\u0192"+
-    "\u0194\t\t\2\2\u0193\u0192\3\2\2\2\u0194\u0197\3\2\2\2\u0195\u0193\3\2"+
-    "\2\2\u0195\u0196\3\2\2\2\u0196\u0199\3\2\2\2\u0197\u0195\3\2\2\2\u0198"+
-    "\u0190\3\2\2\2\u0198\u0191\3\2\2\2\u0199\u019b\3\2\2\2\u019a\u019c\t\n"+
-    "\2\2\u019b\u019a\3\2\2\2\u019b\u019c\3\2\2\2\u019c\u0087\3\2\2\2\u019d"+
-    "\u01a6\7\62\2\2\u019e\u01a2\t\b\2\2\u019f\u01a1\t\t\2\2\u01a0\u019f\3"+
-    "\2\2\2\u01a1\u01a4\3\2\2\2\u01a2\u01a0\3\2\2\2\u01a2\u01a3\3\2\2\2\u01a3"+
-    "\u01a6\3\2\2\2\u01a4\u01a2\3\2\2\2\u01a5\u019d\3\2\2\2\u01a5\u019e\3\2"+
-    "\2\2\u01a6\u01ad\3\2\2\2\u01a7\u01a9\5\24\n\2\u01a8\u01aa\t\t\2\2\u01a9"+
-    "\u01a8\3\2\2\2\u01aa\u01ab\3\2\2\2\u01ab\u01a9\3\2\2\2\u01ab\u01ac\3\2"+
-    "\2\2\u01ac\u01ae\3\2\2\2\u01ad\u01a7\3\2\2\2\u01ad\u01ae\3\2\2\2\u01ae"+
-    "\u01b8\3\2\2\2\u01af\u01b1\t\13\2\2\u01b0\u01b2\t\f\2\2\u01b1\u01b0\3"+
-    "\2\2\2\u01b1\u01b2\3\2\2\2\u01b2\u01b4\3\2\2\2\u01b3\u01b5\t\t\2\2\u01b4"+
-    "\u01b3\3\2\2\2\u01b5\u01b6\3\2\2\2\u01b6\u01b4\3\2\2\2\u01b6\u01b7\3\2"+
-    "\2\2\u01b7\u01b9\3\2\2\2\u01b8\u01af\3\2\2\2\u01b8\u01b9\3\2\2\2\u01b9"+
-    "\u01bb\3\2\2\2\u01ba\u01bc\t\r\2\2\u01bb\u01ba\3\2\2\2\u01bb\u01bc\3\2"+
-    "\2\2\u01bc\u0089\3\2\2\2\u01bd\u01c5\7$\2\2\u01be\u01bf\7^\2\2\u01bf\u01c4"+
-    "\7$\2\2\u01c0\u01c1\7^\2\2\u01c1\u01c4\7^\2\2\u01c2\u01c4\n\16\2\2\u01c3"+
-    "\u01be\3\2\2\2\u01c3\u01c0\3\2\2\2\u01c3\u01c2\3\2\2\2\u01c4\u01c7\3\2"+
-    "\2\2\u01c5\u01c6\3\2\2\2\u01c5\u01c3\3\2\2\2\u01c6\u01c8\3\2\2\2\u01c7"+
-    "\u01c5\3\2\2\2\u01c8\u01d6\7$\2\2\u01c9\u01d1\7)\2\2\u01ca\u01cb\7^\2"+
-    "\2\u01cb\u01d0\7)\2\2\u01cc\u01cd\7^\2\2\u01cd\u01d0\7^\2\2\u01ce\u01d0"+
-    "\n\16\2\2\u01cf\u01ca\3\2\2\2\u01cf\u01cc\3\2\2\2\u01cf\u01ce\3\2\2\2"+
-    "\u01d0\u01d3\3\2\2\2\u01d1\u01d2\3\2\2\2\u01d1\u01cf\3\2\2\2\u01d2\u01d4"+
-    "\3\2\2\2\u01d3\u01d1\3\2\2\2\u01d4\u01d6\7)\2\2\u01d5\u01bd\3\2\2\2\u01d5"+
-    "\u01c9\3\2\2\2\u01d6\u008b\3\2\2\2\u01d7\u01d8\7v\2\2\u01d8\u01d9\7t\2"+
-    "\2\u01d9\u01da\7w\2\2\u01da\u01db\7g\2\2\u01db\u008d\3\2\2\2\u01dc\u01dd"+
-    "\7h\2\2\u01dd\u01de\7c\2\2\u01de\u01df\7n\2\2\u01df\u01e0\7u\2\2\u01e0"+
-    "\u01e1\7g\2\2\u01e1\u008f\3\2\2\2\u01e2\u01e3\7p\2\2\u01e3\u01e4\7w\2"+
-    "\2\u01e4\u01e5\7n\2\2\u01e5\u01e6\7n\2\2\u01e6\u0091\3\2\2\2\u01e7\u01ed"+
-    "\5\u0094J\2\u01e8\u01e9\5\24\n\2\u01e9\u01ea\5\u0094J\2\u01ea\u01ec\3"+
-    "\2\2\2\u01eb\u01e8\3\2\2\2\u01ec\u01ef\3\2\2\2\u01ed\u01eb\3\2\2\2\u01ed"+
-    "\u01ee\3\2\2\2\u01ee\u01f0\3\2\2\2\u01ef\u01ed\3\2\2\2\u01f0\u01f1\6I"+
-    "\2\2\u01f1\u0093\3\2\2\2\u01f2\u01f6\t\17\2\2\u01f3\u01f5\t\20\2\2\u01f4"+
-    "\u01f3\3\2\2\2\u01f5\u01f8\3\2\2\2\u01f6\u01f4\3\2\2\2\u01f6\u01f7\3\2"+
-    "\2\2\u01f7\u0095\3\2\2\2\u01f8\u01f6\3\2\2\2\u01f9\u0202\7\62\2\2\u01fa"+
-    "\u01fe\t\b\2\2\u01fb\u01fd\t\t\2\2\u01fc\u01fb\3\2\2\2\u01fd\u0200\3\2"+
-    "\2\2\u01fe\u01fc\3\2\2\2\u01fe\u01ff\3\2\2\2\u01ff\u0202\3\2\2\2\u0200"+
-    "\u01fe\3\2\2\2\u0201\u01f9\3\2\2\2\u0201\u01fa\3\2\2\2\u0202\u0203\3\2"+
-    "\2\2\u0203\u0204\bK\4\2\u0204\u0097\3\2\2\2\u0205\u0209\t\17\2\2\u0206"+
-    "\u0208\t\20\2\2\u0207\u0206\3\2\2\2\u0208\u020b\3\2\2\2\u0209\u0207\3"+
-    "\2\2\2\u0209\u020a\3\2\2\2\u020a\u020c\3\2\2\2\u020b\u0209\3\2\2\2\u020c"+
-    "\u020d\bL\4\2\u020d\u0099\3\2\2\2!\2\3\u009d\u00a7\u00b1\u00b6\u0181\u0184"+
-    "\u018b\u018e\u0195\u0198\u019b\u01a2\u01a5\u01ab\u01ad\u01b1\u01b6\u01b8"+
-    "\u01bb\u01c3\u01c5\u01cf\u01d1\u01d5\u01ed\u01f6\u01fe\u0201\u0209\5\b"+
-    "\2\2\4\3\2\4\2\2";
+    "I\tI\4J\tJ\4K\tK\4L\tL\4M\tM\3\2\6\2\u009e\n\2\r\2\16\2\u009f\3\2\3\2"+
+    "\3\3\3\3\3\3\3\3\7\3\u00a8\n\3\f\3\16\3\u00ab\13\3\3\3\3\3\3\3\3\3\3\3"+
+    "\7\3\u00b2\n\3\f\3\16\3\u00b5\13\3\3\3\3\3\5\3\u00b9\n\3\3\3\3\3\3\4\3"+
+    "\4\3\5\3\5\3\6\3\6\3\7\3\7\3\b\3\b\3\t\3\t\3\n\3\n\3\n\3\n\3\13\3\13\3"+
+    "\f\3\f\3\r\3\r\3\r\3\16\3\16\3\16\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3"+
+    "\17\3\20\3\20\3\20\3\21\3\21\3\21\3\21\3\22\3\22\3\22\3\22\3\22\3\22\3"+
+    "\22\3\22\3\22\3\23\3\23\3\23\3\23\3\23\3\23\3\24\3\24\3\24\3\24\3\24\3"+
+    "\24\3\24\3\25\3\25\3\25\3\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\27\3"+
+    "\27\3\27\3\30\3\30\3\30\3\30\3\30\3\30\3\31\3\31\3\32\3\32\3\33\3\33\3"+
+    "\34\3\34\3\35\3\35\3\36\3\36\3\37\3\37\3 \3 \3 \3!\3!\3!\3\"\3\"\3\"\3"+
+    "\"\3#\3#\3$\3$\3$\3%\3%\3&\3&\3&\3\'\3\'\3\'\3(\3(\3(\3(\3)\3)\3)\3*\3"+
+    "*\3*\3*\3+\3+\3,\3,\3-\3-\3.\3.\3.\3/\3/\3/\3\60\3\60\3\61\3\61\3\62\3"+
+    "\62\3\62\3\63\3\63\3\63\3\64\3\64\3\64\3\65\3\65\3\65\3\66\3\66\3\67\3"+
+    "\67\3\67\38\38\38\39\39\39\3:\3:\3:\3;\3;\3;\3<\3<\3<\3=\3=\3=\3>\3>\3"+
+    ">\3?\3?\3?\3?\3@\3@\3@\3@\3A\3A\3A\3A\3A\3B\3B\6B\u0185\nB\rB\16B\u0186"+
+    "\3B\5B\u018a\nB\3C\3C\3C\6C\u018f\nC\rC\16C\u0190\3C\5C\u0194\nC\3D\3"+
+    "D\3D\7D\u0199\nD\fD\16D\u019c\13D\5D\u019e\nD\3D\5D\u01a1\nD\3E\3E\3E"+
+    "\7E\u01a6\nE\fE\16E\u01a9\13E\5E\u01ab\nE\3E\3E\6E\u01af\nE\rE\16E\u01b0"+
+    "\5E\u01b3\nE\3E\3E\5E\u01b7\nE\3E\6E\u01ba\nE\rE\16E\u01bb\5E\u01be\n"+
+    "E\3E\5E\u01c1\nE\3F\3F\3F\3F\3F\3F\7F\u01c9\nF\fF\16F\u01cc\13F\3F\3F"+
+    "\3F\3F\3F\3F\3F\7F\u01d5\nF\fF\16F\u01d8\13F\3F\5F\u01db\nF\3G\3G\3G\3"+
+    "G\3G\3H\3H\3H\3H\3H\3H\3I\3I\3I\3I\3I\3J\3J\3J\3J\7J\u01f1\nJ\fJ\16J\u01f4"+
+    "\13J\3J\3J\3K\3K\7K\u01fa\nK\fK\16K\u01fd\13K\3L\3L\3L\7L\u0202\nL\fL"+
+    "\16L\u0205\13L\5L\u0207\nL\3L\3L\3M\3M\7M\u020d\nM\fM\16M\u0210\13M\3"+
+    "M\3M\6\u00a9\u00b3\u01ca\u01d6\2N\4\3\6\4\b\5\n\6\f\7\16\b\20\t\22\n\24"+
+    "\13\26\f\30\r\32\16\34\17\36\20 \21\"\22$\23&\24(\25*\26,\27.\30\60\31"+
+    "\62\32\64\33\66\348\35:\36<\37> @!B\"D#F$H%J&L\'N(P)R*T+V,X-Z.\\/^\60"+
+    "`\61b\62d\63f\64h\65j\66l\67n8p9r:t;v<x=z>|?~@\u0080A\u0082B\u0084C\u0086"+
+    "D\u0088E\u008aF\u008cG\u008eH\u0090I\u0092J\u0094K\u0096L\u0098M\u009a"+
+    "N\4\2\3\21\5\2\13\f\17\17\"\"\4\2\f\f\17\17\3\2\629\4\2NNnn\4\2ZZzz\5"+
+    "\2\62;CHch\3\2\63;\3\2\62;\b\2FFHHNNffhhnn\4\2GGgg\4\2--//\4\2HHhh\4\2"+
+    "$$^^\5\2C\\aac|\6\2\62;C\\aac|\u0230\2\4\3\2\2\2\2\6\3\2\2\2\2\b\3\2\2"+
+    "\2\2\n\3\2\2\2\2\f\3\2\2\2\2\16\3\2\2\2\2\20\3\2\2\2\2\22\3\2\2\2\2\24"+
+    "\3\2\2\2\2\26\3\2\2\2\2\30\3\2\2\2\2\32\3\2\2\2\2\34\3\2\2\2\2\36\3\2"+
+    "\2\2\2 \3\2\2\2\2\"\3\2\2\2\2$\3\2\2\2\2&\3\2\2\2\2(\3\2\2\2\2*\3\2\2"+
+    "\2\2,\3\2\2\2\2.\3\2\2\2\2\60\3\2\2\2\2\62\3\2\2\2\2\64\3\2\2\2\2\66\3"+
+    "\2\2\2\28\3\2\2\2\2:\3\2\2\2\2<\3\2\2\2\2>\3\2\2\2\2@\3\2\2\2\2B\3\2\2"+
+    "\2\2D\3\2\2\2\2F\3\2\2\2\2H\3\2\2\2\2J\3\2\2\2\2L\3\2\2\2\2N\3\2\2\2\2"+
+    "P\3\2\2\2\2R\3\2\2\2\2T\3\2\2\2\2V\3\2\2\2\2X\3\2\2\2\2Z\3\2\2\2\2\\\3"+
+    "\2\2\2\2^\3\2\2\2\2`\3\2\2\2\2b\3\2\2\2\2d\3\2\2\2\2f\3\2\2\2\2h\3\2\2"+
+    "\2\2j\3\2\2\2\2l\3\2\2\2\2n\3\2\2\2\2p\3\2\2\2\2r\3\2\2\2\2t\3\2\2\2\2"+
+    "v\3\2\2\2\2x\3\2\2\2\2z\3\2\2\2\2|\3\2\2\2\2~\3\2\2\2\2\u0080\3\2\2\2"+
+    "\2\u0082\3\2\2\2\2\u0084\3\2\2\2\2\u0086\3\2\2\2\2\u0088\3\2\2\2\2\u008a"+
+    "\3\2\2\2\2\u008c\3\2\2\2\2\u008e\3\2\2\2\2\u0090\3\2\2\2\2\u0092\3\2\2"+
+    "\2\2\u0094\3\2\2\2\2\u0096\3\2\2\2\3\u0098\3\2\2\2\3\u009a\3\2\2\2\4\u009d"+
+    "\3\2\2\2\6\u00b8\3\2\2\2\b\u00bc\3\2\2\2\n\u00be\3\2\2\2\f\u00c0\3\2\2"+
+    "\2\16\u00c2\3\2\2\2\20\u00c4\3\2\2\2\22\u00c6\3\2\2\2\24\u00c8\3\2\2\2"+
+    "\26\u00cc\3\2\2\2\30\u00ce\3\2\2\2\32\u00d0\3\2\2\2\34\u00d3\3\2\2\2\36"+
+    "\u00d8\3\2\2\2 \u00de\3\2\2\2\"\u00e1\3\2\2\2$\u00e5\3\2\2\2&\u00ee\3"+
+    "\2\2\2(\u00f4\3\2\2\2*\u00fb\3\2\2\2,\u00ff\3\2\2\2.\u0103\3\2\2\2\60"+
+    "\u0109\3\2\2\2\62\u010f\3\2\2\2\64\u0111\3\2\2\2\66\u0113\3\2\2\28\u0115"+
+    "\3\2\2\2:\u0117\3\2\2\2<\u0119\3\2\2\2>\u011b\3\2\2\2@\u011d\3\2\2\2B"+
+    "\u0120\3\2\2\2D\u0123\3\2\2\2F\u0127\3\2\2\2H\u0129\3\2\2\2J\u012c\3\2"+
+    "\2\2L\u012e\3\2\2\2N\u0131\3\2\2\2P\u0134\3\2\2\2R\u0138\3\2\2\2T\u013b"+
+    "\3\2\2\2V\u013f\3\2\2\2X\u0141\3\2\2\2Z\u0143\3\2\2\2\\\u0145\3\2\2\2"+
+    "^\u0148\3\2\2\2`\u014b\3\2\2\2b\u014d\3\2\2\2d\u014f\3\2\2\2f\u0152\3"+
+    "\2\2\2h\u0155\3\2\2\2j\u0158\3\2\2\2l\u015b\3\2\2\2n\u015d\3\2\2\2p\u0160"+
+    "\3\2\2\2r\u0163\3\2\2\2t\u0166\3\2\2\2v\u0169\3\2\2\2x\u016c\3\2\2\2z"+
+    "\u016f\3\2\2\2|\u0172\3\2\2\2~\u0175\3\2\2\2\u0080\u0179\3\2\2\2\u0082"+
+    "\u017d\3\2\2\2\u0084\u0182\3\2\2\2\u0086\u018b\3\2\2\2\u0088\u019d\3\2"+
+    "\2\2\u008a\u01aa\3\2\2\2\u008c\u01da\3\2\2\2\u008e\u01dc\3\2\2\2\u0090"+
+    "\u01e1\3\2\2\2\u0092\u01e7\3\2\2\2\u0094\u01ec\3\2\2\2\u0096\u01f7\3\2"+
+    "\2\2\u0098\u0206\3\2\2\2\u009a\u020a\3\2\2\2\u009c\u009e\t\2\2\2\u009d"+
+    "\u009c\3\2\2\2\u009e\u009f\3\2\2\2\u009f\u009d\3\2\2\2\u009f\u00a0\3\2"+
+    "\2\2\u00a0\u00a1\3\2\2\2\u00a1\u00a2\b\2\2\2\u00a2\5\3\2\2\2\u00a3\u00a4"+
+    "\7\61\2\2\u00a4\u00a5\7\61\2\2\u00a5\u00a9\3\2\2\2\u00a6\u00a8\13\2\2"+
+    "\2\u00a7\u00a6\3\2\2\2\u00a8\u00ab\3\2\2\2\u00a9\u00aa\3\2\2\2\u00a9\u00a7"+
+    "\3\2\2\2\u00aa\u00ac\3\2\2\2\u00ab\u00a9\3\2\2\2\u00ac\u00b9\t\3\2\2\u00ad"+
+    "\u00ae\7\61\2\2\u00ae\u00af\7,\2\2\u00af\u00b3\3\2\2\2\u00b0\u00b2\13"+
+    "\2\2\2\u00b1\u00b0\3\2\2\2\u00b2\u00b5\3\2\2\2\u00b3\u00b4\3\2\2\2\u00b3"+
+    "\u00b1\3\2\2\2\u00b4\u00b6\3\2\2\2\u00b5\u00b3\3\2\2\2\u00b6\u00b7\7,"+
+    "\2\2\u00b7\u00b9\7\61\2\2\u00b8\u00a3\3\2\2\2\u00b8\u00ad\3\2\2\2\u00b9"+
+    "\u00ba\3\2\2\2\u00ba\u00bb\b\3\2\2\u00bb\7\3\2\2\2\u00bc\u00bd\7}\2\2"+
+    "\u00bd\t\3\2\2\2\u00be\u00bf\7\177\2\2\u00bf\13\3\2\2\2\u00c0\u00c1\7"+
+    "]\2\2\u00c1\r\3\2\2\2\u00c2\u00c3\7_\2\2\u00c3\17\3\2\2\2\u00c4\u00c5"+
+    "\7*\2\2\u00c5\21\3\2\2\2\u00c6\u00c7\7+\2\2\u00c7\23\3\2\2\2\u00c8\u00c9"+
+    "\7\60\2\2\u00c9\u00ca\3\2\2\2\u00ca\u00cb\b\n\3\2\u00cb\25\3\2\2\2\u00cc"+
+    "\u00cd\7.\2\2\u00cd\27\3\2\2\2\u00ce\u00cf\7=\2\2\u00cf\31\3\2\2\2\u00d0"+
+    "\u00d1\7k\2\2\u00d1\u00d2\7h\2\2\u00d2\33\3\2\2\2\u00d3\u00d4\7g\2\2\u00d4"+
+    "\u00d5\7n\2\2\u00d5\u00d6\7u\2\2\u00d6\u00d7\7g\2\2\u00d7\35\3\2\2\2\u00d8"+
+    "\u00d9\7y\2\2\u00d9\u00da\7j\2\2\u00da\u00db\7k\2\2\u00db\u00dc\7n\2\2"+
+    "\u00dc\u00dd\7g\2\2\u00dd\37\3\2\2\2\u00de\u00df\7f\2\2\u00df\u00e0\7"+
+    "q\2\2\u00e0!\3\2\2\2\u00e1\u00e2\7h\2\2\u00e2\u00e3\7q\2\2\u00e3\u00e4"+
+    "\7t\2\2\u00e4#\3\2\2\2\u00e5\u00e6\7e\2\2\u00e6\u00e7\7q\2\2\u00e7\u00e8"+
+    "\7p\2\2\u00e8\u00e9\7v\2\2\u00e9\u00ea\7k\2\2\u00ea\u00eb\7p\2\2\u00eb"+
+    "\u00ec\7w\2\2\u00ec\u00ed\7g\2\2\u00ed%\3\2\2\2\u00ee\u00ef\7d\2\2\u00ef"+
+    "\u00f0\7t\2\2\u00f0\u00f1\7g\2\2\u00f1\u00f2\7c\2\2\u00f2\u00f3\7m\2\2"+
+    "\u00f3\'\3\2\2\2\u00f4\u00f5\7t\2\2\u00f5\u00f6\7g\2\2\u00f6\u00f7\7v"+
+    "\2\2\u00f7\u00f8\7w\2\2\u00f8\u00f9\7t\2\2\u00f9\u00fa\7p\2\2\u00fa)\3"+
+    "\2\2\2\u00fb\u00fc\7p\2\2\u00fc\u00fd\7g\2\2\u00fd\u00fe\7y\2\2\u00fe"+
+    "+\3\2\2\2\u00ff\u0100\7v\2\2\u0100\u0101\7t\2\2\u0101\u0102\7{\2\2\u0102"+
+    "-\3\2\2\2\u0103\u0104\7e\2\2\u0104\u0105\7c\2\2\u0105\u0106\7v\2\2\u0106"+
+    "\u0107\7e\2\2\u0107\u0108\7j\2\2\u0108/\3\2\2\2\u0109\u010a\7v\2\2\u010a"+
+    "\u010b\7j\2\2\u010b\u010c\7t\2\2\u010c\u010d\7q\2\2\u010d\u010e\7y\2\2"+
+    "\u010e\61\3\2\2\2\u010f\u0110\7#\2\2\u0110\63\3\2\2\2\u0111\u0112\7\u0080"+
+    "\2\2\u0112\65\3\2\2\2\u0113\u0114\7,\2\2\u0114\67\3\2\2\2\u0115\u0116"+
+    "\7\61\2\2\u01169\3\2\2\2\u0117\u0118\7\'\2\2\u0118;\3\2\2\2\u0119\u011a"+
+    "\7-\2\2\u011a=\3\2\2\2\u011b\u011c\7/\2\2\u011c?\3\2\2\2\u011d\u011e\7"+
+    ">\2\2\u011e\u011f\7>\2\2\u011fA\3\2\2\2\u0120\u0121\7@\2\2\u0121\u0122"+
+    "\7@\2\2\u0122C\3\2\2\2\u0123\u0124\7@\2\2\u0124\u0125\7@\2\2\u0125\u0126"+
+    "\7@\2\2\u0126E\3\2\2\2\u0127\u0128\7>\2\2\u0128G\3\2\2\2\u0129\u012a\7"+
+    ">\2\2\u012a\u012b\7?\2\2\u012bI\3\2\2\2\u012c\u012d\7@\2\2\u012dK\3\2"+
+    "\2\2\u012e\u012f\7@\2\2\u012f\u0130\7?\2\2\u0130M\3\2\2\2\u0131\u0132"+
+    "\7?\2\2\u0132\u0133\7?\2\2\u0133O\3\2\2\2\u0134\u0135\7?\2\2\u0135\u0136"+
+    "\7?\2\2\u0136\u0137\7?\2\2\u0137Q\3\2\2\2\u0138\u0139\7#\2\2\u0139\u013a"+
+    "\7?\2\2\u013aS\3\2\2\2\u013b\u013c\7#\2\2\u013c\u013d\7?\2\2\u013d\u013e"+
+    "\7?\2\2\u013eU\3\2\2\2\u013f\u0140\7(\2\2\u0140W\3\2\2\2\u0141\u0142\7"+
+    "`\2\2\u0142Y\3\2\2\2\u0143\u0144\7~\2\2\u0144[\3\2\2\2\u0145\u0146\7("+
+    "\2\2\u0146\u0147\7(\2\2\u0147]\3\2\2\2\u0148\u0149\7~\2\2\u0149\u014a"+
+    "\7~\2\2\u014a_\3\2\2\2\u014b\u014c\7A\2\2\u014ca\3\2\2\2\u014d\u014e\7"+
+    "<\2\2\u014ec\3\2\2\2\u014f\u0150\7<\2\2\u0150\u0151\7<\2\2\u0151e\3\2"+
+    "\2\2\u0152\u0153\7/\2\2\u0153\u0154\7@\2\2\u0154g\3\2\2\2\u0155\u0156"+
+    "\7-\2\2\u0156\u0157\7-\2\2\u0157i\3\2\2\2\u0158\u0159\7/\2\2\u0159\u015a"+
+    "\7/\2\2\u015ak\3\2\2\2\u015b\u015c\7?\2\2\u015cm\3\2\2\2\u015d\u015e\7"+
+    "-\2\2\u015e\u015f\7?\2\2\u015fo\3\2\2\2\u0160\u0161\7/\2\2\u0161\u0162"+
+    "\7?\2\2\u0162q\3\2\2\2\u0163\u0164\7,\2\2\u0164\u0165\7?\2\2\u0165s\3"+
+    "\2\2\2\u0166\u0167\7\61\2\2\u0167\u0168\7?\2\2\u0168u\3\2\2\2\u0169\u016a"+
+    "\7\'\2\2\u016a\u016b\7?\2\2\u016bw\3\2\2\2\u016c\u016d\7(\2\2\u016d\u016e"+
+    "\7?\2\2\u016ey\3\2\2\2\u016f\u0170\7`\2\2\u0170\u0171\7?\2\2\u0171{\3"+
+    "\2\2\2\u0172\u0173\7~\2\2\u0173\u0174\7?\2\2\u0174}\3\2\2\2\u0175\u0176"+
+    "\7>\2\2\u0176\u0177\7>\2\2\u0177\u0178\7?\2\2\u0178\177\3\2\2\2\u0179"+
+    "\u017a\7@\2\2\u017a\u017b\7@\2\2\u017b\u017c\7?\2\2\u017c\u0081\3\2\2"+
+    "\2\u017d\u017e\7@\2\2\u017e\u017f\7@\2\2\u017f\u0180\7@\2\2\u0180\u0181"+
+    "\7?\2\2\u0181\u0083\3\2\2\2\u0182\u0184\7\62\2\2\u0183\u0185\t\4\2\2\u0184"+
+    "\u0183\3\2\2\2\u0185\u0186\3\2\2\2\u0186\u0184\3\2\2\2\u0186\u0187\3\2"+
+    "\2\2\u0187\u0189\3\2\2\2\u0188\u018a\t\5\2\2\u0189\u0188\3\2\2\2\u0189"+
+    "\u018a\3\2\2\2\u018a\u0085\3\2\2\2\u018b\u018c\7\62\2\2\u018c\u018e\t"+
+    "\6\2\2\u018d\u018f\t\7\2\2\u018e\u018d\3\2\2\2\u018f\u0190\3\2\2\2\u0190"+
+    "\u018e\3\2\2\2\u0190\u0191\3\2\2\2\u0191\u0193\3\2\2\2\u0192\u0194\t\5"+
+    "\2\2\u0193\u0192\3\2\2\2\u0193\u0194\3\2\2\2\u0194\u0087\3\2\2\2\u0195"+
+    "\u019e\7\62\2\2\u0196\u019a\t\b\2\2\u0197\u0199\t\t\2\2\u0198\u0197\3"+
+    "\2\2\2\u0199\u019c\3\2\2\2\u019a\u0198\3\2\2\2\u019a\u019b\3\2\2\2\u019b"+
+    "\u019e\3\2\2\2\u019c\u019a\3\2\2\2\u019d\u0195\3\2\2\2\u019d\u0196\3\2"+
+    "\2\2\u019e\u01a0\3\2\2\2\u019f\u01a1\t\n\2\2\u01a0\u019f\3\2\2\2\u01a0"+
+    "\u01a1\3\2\2\2\u01a1\u0089\3\2\2\2\u01a2\u01ab\7\62\2\2\u01a3\u01a7\t"+
+    "\b\2\2\u01a4\u01a6\t\t\2\2\u01a5\u01a4\3\2\2\2\u01a6\u01a9\3\2\2\2\u01a7"+
+    "\u01a5\3\2\2\2\u01a7\u01a8\3\2\2\2\u01a8\u01ab\3\2\2\2\u01a9\u01a7\3\2"+
+    "\2\2\u01aa\u01a2\3\2\2\2\u01aa\u01a3\3\2\2\2\u01ab\u01b2\3\2\2\2\u01ac"+
+    "\u01ae\5\24\n\2\u01ad\u01af\t\t\2\2\u01ae\u01ad\3\2\2\2\u01af\u01b0\3"+
+    "\2\2\2\u01b0\u01ae\3\2\2\2\u01b0\u01b1\3\2\2\2\u01b1\u01b3\3\2\2\2\u01b2"+
+    "\u01ac\3\2\2\2\u01b2\u01b3\3\2\2\2\u01b3\u01bd\3\2\2\2\u01b4\u01b6\t\13"+
+    "\2\2\u01b5\u01b7\t\f\2\2\u01b6\u01b5\3\2\2\2\u01b6\u01b7\3\2\2\2\u01b7"+
+    "\u01b9\3\2\2\2\u01b8\u01ba\t\t\2\2\u01b9\u01b8\3\2\2\2\u01ba\u01bb\3\2"+
+    "\2\2\u01bb\u01b9\3\2\2\2\u01bb\u01bc\3\2\2\2\u01bc\u01be\3\2\2\2\u01bd"+
+    "\u01b4\3\2\2\2\u01bd\u01be\3\2\2\2\u01be\u01c0\3\2\2\2\u01bf\u01c1\t\r"+
+    "\2\2\u01c0\u01bf\3\2\2\2\u01c0\u01c1\3\2\2\2\u01c1\u008b\3\2\2\2\u01c2"+
+    "\u01ca\7$\2\2\u01c3\u01c4\7^\2\2\u01c4\u01c9\7$\2\2\u01c5\u01c6\7^\2\2"+
+    "\u01c6\u01c9\7^\2\2\u01c7\u01c9\n\16\2\2\u01c8\u01c3\3\2\2\2\u01c8\u01c5"+
+    "\3\2\2\2\u01c8\u01c7\3\2\2\2\u01c9\u01cc\3\2\2\2\u01ca\u01cb\3\2\2\2\u01ca"+
+    "\u01c8\3\2\2\2\u01cb\u01cd\3\2\2\2\u01cc\u01ca\3\2\2\2\u01cd\u01db\7$"+
+    "\2\2\u01ce\u01d6\7)\2\2\u01cf\u01d0\7^\2\2\u01d0\u01d5\7)\2\2\u01d1\u01d2"+
+    "\7^\2\2\u01d2\u01d5\7^\2\2\u01d3\u01d5\n\16\2\2\u01d4\u01cf\3\2\2\2\u01d4"+
+    "\u01d1\3\2\2\2\u01d4\u01d3\3\2\2\2\u01d5\u01d8\3\2\2\2\u01d6\u01d7\3\2"+
+    "\2\2\u01d6\u01d4\3\2\2\2\u01d7\u01d9\3\2\2\2\u01d8\u01d6\3\2\2\2\u01d9"+
+    "\u01db\7)\2\2\u01da\u01c2\3\2\2\2\u01da\u01ce\3\2\2\2\u01db\u008d\3\2"+
+    "\2\2\u01dc\u01dd\7v\2\2\u01dd\u01de\7t\2\2\u01de\u01df\7w\2\2\u01df\u01e0"+
+    "\7g\2\2\u01e0\u008f\3\2\2\2\u01e1\u01e2\7h\2\2\u01e2\u01e3\7c\2\2\u01e3"+
+    "\u01e4\7n\2\2\u01e4\u01e5\7u\2\2\u01e5\u01e6\7g\2\2\u01e6\u0091\3\2\2"+
+    "\2\u01e7\u01e8\7p\2\2\u01e8\u01e9\7w\2\2\u01e9\u01ea\7n\2\2\u01ea\u01eb"+
+    "\7n\2\2\u01eb\u0093\3\2\2\2\u01ec\u01f2\5\u0096K\2\u01ed\u01ee\5\24\n"+
+    "\2\u01ee\u01ef\5\u0096K\2\u01ef\u01f1\3\2\2\2\u01f0\u01ed\3\2\2\2\u01f1"+
+    "\u01f4\3\2\2\2\u01f2\u01f0\3\2\2\2\u01f2\u01f3\3\2\2\2\u01f3\u01f5\3\2"+
+    "\2\2\u01f4\u01f2\3\2\2\2\u01f5\u01f6\6J\2\2\u01f6\u0095\3\2\2\2\u01f7"+
+    "\u01fb\t\17\2\2\u01f8\u01fa\t\20\2\2\u01f9\u01f8\3\2\2\2\u01fa\u01fd\3"+
+    "\2\2\2\u01fb\u01f9\3\2\2\2\u01fb\u01fc\3\2\2\2\u01fc\u0097\3\2\2\2\u01fd"+
+    "\u01fb\3\2\2\2\u01fe\u0207\7\62\2\2\u01ff\u0203\t\b\2\2\u0200\u0202\t"+
+    "\t\2\2\u0201\u0200\3\2\2\2\u0202\u0205\3\2\2\2\u0203\u0201\3\2\2\2\u0203"+
+    "\u0204\3\2\2\2\u0204\u0207\3\2\2\2\u0205\u0203\3\2\2\2\u0206\u01fe\3\2"+
+    "\2\2\u0206\u01ff\3\2\2\2\u0207\u0208\3\2\2\2\u0208\u0209\bL\4\2\u0209"+
+    "\u0099\3\2\2\2\u020a\u020e\t\17\2\2\u020b\u020d\t\20\2\2\u020c\u020b\3"+
+    "\2\2\2\u020d\u0210\3\2\2\2\u020e\u020c\3\2\2\2\u020e\u020f\3\2\2\2\u020f"+
+    "\u0211\3\2\2\2\u0210\u020e\3\2\2\2\u0211\u0212\bM\4\2\u0212\u009b\3\2"+
+    "\2\2!\2\3\u009f\u00a9\u00b3\u00b8\u0186\u0189\u0190\u0193\u019a\u019d"+
+    "\u01a0\u01a7\u01aa\u01b0\u01b2\u01b6\u01bb\u01bd\u01c0\u01c8\u01ca\u01d4"+
+    "\u01d6\u01da\u01f2\u01fb\u0203\u0206\u020e\5\b\2\2\4\3\2\4\2\2";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
index acaf00a5765..0e1c9a96340 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
@@ -22,23 +22,24 @@ class PainlessParser extends Parser {
     BREAK=18, RETURN=19, NEW=20, TRY=21, CATCH=22, THROW=23, BOOLNOT=24, BWNOT=25, 
     MUL=26, DIV=27, REM=28, ADD=29, SUB=30, LSH=31, RSH=32, USH=33, LT=34, 
     LTE=35, GT=36, GTE=37, EQ=38, EQR=39, NE=40, NER=41, BWAND=42, XOR=43, 
-    BWOR=44, BOOLAND=45, BOOLOR=46, COND=47, COLON=48, REF=49, INCR=50, DECR=51, 
-    ASSIGN=52, AADD=53, ASUB=54, AMUL=55, ADIV=56, AREM=57, AAND=58, AXOR=59, 
-    AOR=60, ALSH=61, ARSH=62, AUSH=63, OCTAL=64, HEX=65, INTEGER=66, DECIMAL=67, 
-    STRING=68, TRUE=69, FALSE=70, NULL=71, TYPE=72, ID=73, DOTINTEGER=74, 
-    DOTID=75;
+    BWOR=44, BOOLAND=45, BOOLOR=46, COND=47, COLON=48, REF=49, ARROW=50, INCR=51, 
+    DECR=52, ASSIGN=53, AADD=54, ASUB=55, AMUL=56, ADIV=57, AREM=58, AAND=59, 
+    AXOR=60, AOR=61, ALSH=62, ARSH=63, AUSH=64, OCTAL=65, HEX=66, INTEGER=67, 
+    DECIMAL=68, STRING=69, TRUE=70, FALSE=71, NULL=72, TYPE=73, ID=74, DOTINTEGER=75, 
+    DOTID=76;
   public static final int
     RULE_source = 0, RULE_function = 1, RULE_parameters = 2, RULE_statement = 3, 
     RULE_trailer = 4, RULE_block = 5, RULE_empty = 6, RULE_initializer = 7, 
-    RULE_afterthought = 8, RULE_declaration = 9, RULE_decltype = 10, RULE_funcref = 11, 
-    RULE_declvar = 12, RULE_trap = 13, RULE_delimiter = 14, RULE_expression = 15, 
-    RULE_unary = 16, RULE_chain = 17, RULE_primary = 18, RULE_secondary = 19, 
-    RULE_dot = 20, RULE_brace = 21, RULE_arguments = 22, RULE_argument = 23;
+    RULE_afterthought = 8, RULE_declaration = 9, RULE_decltype = 10, RULE_declvar = 11, 
+    RULE_trap = 12, RULE_delimiter = 13, RULE_expression = 14, RULE_unary = 15, 
+    RULE_chain = 16, RULE_primary = 17, RULE_secondary = 18, RULE_dot = 19, 
+    RULE_brace = 20, RULE_arguments = 21, RULE_argument = 22, RULE_lambda = 23, 
+    RULE_lamtype = 24, RULE_funcref = 25;
   public static final String[] ruleNames = {
     "source", "function", "parameters", "statement", "trailer", "block", "empty", 
-    "initializer", "afterthought", "declaration", "decltype", "funcref", "declvar", 
-    "trap", "delimiter", "expression", "unary", "chain", "primary", "secondary", 
-    "dot", "brace", "arguments", "argument"
+    "initializer", "afterthought", "declaration", "decltype", "declvar", "trap", 
+    "delimiter", "expression", "unary", "chain", "primary", "secondary", "dot", 
+    "brace", "arguments", "argument", "lambda", "lamtype", "funcref"
   };
 
   private static final String[] _LITERAL_NAMES = {
@@ -47,9 +48,9 @@ class PainlessParser extends Parser {
     "'return'", "'new'", "'try'", "'catch'", "'throw'", "'!'", "'~'", "'*'", 
     "'/'", "'%'", "'+'", "'-'", "'<<'", "'>>'", "'>>>'", "'<'", "'<='", "'>'", 
     "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", "'^'", "'|'", "'&&'", 
-    "'||'", "'?'", "':'", "'::'", "'++'", "'--'", "'='", "'+='", "'-='", "'*='", 
-    "'/='", "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", "'>>>='", null, 
-    null, null, null, null, "'true'", "'false'", "'null'"
+    "'||'", "'?'", "':'", "'::'", "'->'", "'++'", "'--'", "'='", "'+='", "'-='", 
+    "'*='", "'/='", "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", "'>>>='", 
+    null, null, null, null, null, "'true'", "'false'", "'null'"
   };
   private static final String[] _SYMBOLIC_NAMES = {
     null, "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", 
@@ -57,10 +58,10 @@ class PainlessParser extends Parser {
     "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "BOOLNOT", "BWNOT", 
     "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", "LTE", "GT", 
     "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", "BOOLAND", "BOOLOR", 
-    "COND", "COLON", "REF", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", 
-    "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", 
-    "HEX", "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", "TYPE", 
-    "ID", "DOTINTEGER", "DOTID"
+    "COND", "COLON", "REF", "ARROW", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", 
+    "AMUL", "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", 
+    "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", 
+    "TYPE", "ID", "DOTINTEGER", "DOTID"
   };
   public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
 
@@ -143,39 +144,39 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(51);
+      setState(55);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,0,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(48);
+          setState(52);
           function();
           }
           } 
         }
-        setState(53);
+        setState(57);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,0,_ctx);
       }
-      setState(57);
+      setState(61);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,1,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(54);
+          setState(58);
           statement();
           }
           } 
         }
-        setState(59);
+        setState(63);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,1,_ctx);
       }
-      setState(60);
+      setState(64);
       match(EOF);
       }
     }
@@ -218,13 +219,13 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(62);
+      setState(66);
       decltype();
-      setState(63);
+      setState(67);
       match(ID);
-      setState(64);
+      setState(68);
       parameters();
-      setState(65);
+      setState(69);
       block();
       }
     }
@@ -274,38 +275,38 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(67);
+      setState(71);
       match(LP);
-      setState(79);
+      setState(83);
       _la = _input.LA(1);
       if (_la==TYPE) {
         {
-        setState(68);
+        setState(72);
         decltype();
-        setState(69);
+        setState(73);
         match(ID);
-        setState(76);
+        setState(80);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(70);
+          setState(74);
           match(COMMA);
-          setState(71);
+          setState(75);
           decltype();
-          setState(72);
+          setState(76);
           match(ID);
           }
           }
-          setState(78);
+          setState(82);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
       }
 
-      setState(81);
+      setState(85);
       match(RP);
       }
     }
@@ -551,35 +552,35 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 6, RULE_statement);
     try {
       int _alt;
-      setState(161);
+      setState(165);
       switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) {
       case 1:
         _localctx = new IfContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(83);
-        match(IF);
-        setState(84);
-        match(LP);
-        setState(85);
-        expression(0);
-        setState(86);
-        match(RP);
         setState(87);
-        trailer();
+        match(IF);
+        setState(88);
+        match(LP);
+        setState(89);
+        expression(0);
+        setState(90);
+        match(RP);
         setState(91);
+        trailer();
+        setState(95);
         switch ( getInterpreter().adaptivePredict(_input,4,_ctx) ) {
         case 1:
           {
-          setState(88);
+          setState(92);
           match(ELSE);
-          setState(89);
+          setState(93);
           trailer();
           }
           break;
         case 2:
           {
-          setState(90);
+          setState(94);
           if (!( _input.LA(1) != ELSE )) throw new FailedPredicateException(this, " _input.LA(1) != ELSE ");
           }
           break;
@@ -590,25 +591,25 @@ class PainlessParser extends Parser {
         _localctx = new WhileContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(93);
+        setState(97);
         match(WHILE);
-        setState(94);
+        setState(98);
         match(LP);
-        setState(95);
-        expression(0);
-        setState(96);
-        match(RP);
         setState(99);
+        expression(0);
+        setState(100);
+        match(RP);
+        setState(103);
         switch ( getInterpreter().adaptivePredict(_input,5,_ctx) ) {
         case 1:
           {
-          setState(97);
+          setState(101);
           trailer();
           }
           break;
         case 2:
           {
-          setState(98);
+          setState(102);
           empty();
           }
           break;
@@ -619,19 +620,19 @@ class PainlessParser extends Parser {
         _localctx = new DoContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(101);
-        match(DO);
-        setState(102);
-        block();
-        setState(103);
-        match(WHILE);
-        setState(104);
-        match(LP);
         setState(105);
-        expression(0);
+        match(DO);
         setState(106);
-        match(RP);
+        block();
         setState(107);
+        match(WHILE);
+        setState(108);
+        match(LP);
+        setState(109);
+        expression(0);
+        setState(110);
+        match(RP);
+        setState(111);
         delimiter();
         }
         break;
@@ -639,54 +640,54 @@ class PainlessParser extends Parser {
         _localctx = new ForContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(109);
+        setState(113);
         match(FOR);
-        setState(110);
+        setState(114);
         match(LP);
-        setState(112);
+        setState(116);
         switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) {
-        case 1:
-          {
-          setState(111);
-          initializer();
-          }
-          break;
-        }
-        setState(114);
-        match(SEMICOLON);
-        setState(116);
-        switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) {
         case 1:
           {
           setState(115);
-          expression(0);
+          initializer();
           }
           break;
         }
         setState(118);
         match(SEMICOLON);
         setState(120);
-        switch ( getInterpreter().adaptivePredict(_input,8,_ctx) ) {
+        switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) {
         case 1:
           {
           setState(119);
-          afterthought();
+          expression(0);
           }
           break;
         }
         setState(122);
-        match(RP);
-        setState(125);
-        switch ( getInterpreter().adaptivePredict(_input,9,_ctx) ) {
+        match(SEMICOLON);
+        setState(124);
+        switch ( getInterpreter().adaptivePredict(_input,8,_ctx) ) {
         case 1:
           {
           setState(123);
+          afterthought();
+          }
+          break;
+        }
+        setState(126);
+        match(RP);
+        setState(129);
+        switch ( getInterpreter().adaptivePredict(_input,9,_ctx) ) {
+        case 1:
+          {
+          setState(127);
           trailer();
           }
           break;
         case 2:
           {
-          setState(124);
+          setState(128);
           empty();
           }
           break;
@@ -697,21 +698,21 @@ class PainlessParser extends Parser {
         _localctx = new EachContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(127);
-        match(FOR);
-        setState(128);
-        match(LP);
-        setState(129);
-        decltype();
-        setState(130);
-        match(ID);
         setState(131);
-        match(COLON);
+        match(FOR);
         setState(132);
-        expression(0);
+        match(LP);
         setState(133);
-        match(RP);
+        decltype();
         setState(134);
+        match(ID);
+        setState(135);
+        match(COLON);
+        setState(136);
+        expression(0);
+        setState(137);
+        match(RP);
+        setState(138);
         trailer();
         }
         break;
@@ -719,9 +720,9 @@ class PainlessParser extends Parser {
         _localctx = new DeclContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(136);
+        setState(140);
         declaration();
-        setState(137);
+        setState(141);
         delimiter();
         }
         break;
@@ -729,9 +730,9 @@ class PainlessParser extends Parser {
         _localctx = new ContinueContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(139);
+        setState(143);
         match(CONTINUE);
-        setState(140);
+        setState(144);
         delimiter();
         }
         break;
@@ -739,9 +740,9 @@ class PainlessParser extends Parser {
         _localctx = new BreakContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(141);
+        setState(145);
         match(BREAK);
-        setState(142);
+        setState(146);
         delimiter();
         }
         break;
@@ -749,11 +750,11 @@ class PainlessParser extends Parser {
         _localctx = new ReturnContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(143);
+        setState(147);
         match(RETURN);
-        setState(144);
+        setState(148);
         expression(0);
-        setState(145);
+        setState(149);
         delimiter();
         }
         break;
@@ -761,11 +762,11 @@ class PainlessParser extends Parser {
         _localctx = new TryContext(_localctx);
         enterOuterAlt(_localctx, 10);
         {
-        setState(147);
+        setState(151);
         match(TRY);
-        setState(148);
+        setState(152);
         block();
-        setState(150); 
+        setState(154); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -773,7 +774,7 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(149);
+            setState(153);
             trap();
             }
             }
@@ -781,7 +782,7 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(152); 
+          setState(156); 
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,10,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
@@ -791,11 +792,11 @@ class PainlessParser extends Parser {
         _localctx = new ThrowContext(_localctx);
         enterOuterAlt(_localctx, 11);
         {
-        setState(154);
+        setState(158);
         match(THROW);
-        setState(155);
+        setState(159);
         expression(0);
-        setState(156);
+        setState(160);
         delimiter();
         }
         break;
@@ -803,9 +804,9 @@ class PainlessParser extends Parser {
         _localctx = new ExprContext(_localctx);
         enterOuterAlt(_localctx, 12);
         {
-        setState(158);
+        setState(162);
         expression(0);
-        setState(159);
+        setState(163);
         delimiter();
         }
         break;
@@ -844,19 +845,19 @@ class PainlessParser extends Parser {
     TrailerContext _localctx = new TrailerContext(_ctx, getState());
     enterRule(_localctx, 8, RULE_trailer);
     try {
-      setState(165);
+      setState(169);
       switch ( getInterpreter().adaptivePredict(_input,12,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(163);
+        setState(167);
         block();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(164);
+        setState(168);
         statement();
         }
         break;
@@ -900,25 +901,25 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(167);
-      match(LBRACK);
       setState(171);
+      match(LBRACK);
+      setState(175);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,13,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(168);
+          setState(172);
           statement();
           }
           } 
         }
-        setState(173);
+        setState(177);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,13,_ctx);
       }
-      setState(174);
+      setState(178);
       match(RBRACK);
       }
     }
@@ -952,7 +953,7 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(176);
+      setState(180);
       match(SEMICOLON);
       }
     }
@@ -989,19 +990,19 @@ class PainlessParser extends Parser {
     InitializerContext _localctx = new InitializerContext(_ctx, getState());
     enterRule(_localctx, 14, RULE_initializer);
     try {
-      setState(180);
+      setState(184);
       switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(178);
+        setState(182);
         declaration();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(179);
+        setState(183);
         expression(0);
         }
         break;
@@ -1039,7 +1040,7 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(182);
+      setState(186);
       expression(0);
       }
     }
@@ -1086,23 +1087,23 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(184);
+      setState(188);
       decltype();
-      setState(185);
+      setState(189);
       declvar();
-      setState(190);
+      setState(194);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==COMMA) {
         {
         {
-        setState(186);
+        setState(190);
         match(COMMA);
-        setState(187);
+        setState(191);
         declvar();
         }
         }
-        setState(192);
+        setState(196);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
@@ -1147,21 +1148,21 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(193);
+      setState(197);
       match(TYPE);
-      setState(198);
+      setState(202);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==LBRACE) {
         {
         {
-        setState(194);
+        setState(198);
         match(LBRACE);
-        setState(195);
+        setState(199);
         match(RBRACE);
         }
         }
-        setState(200);
+        setState(204);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
@@ -1178,53 +1179,6 @@ class PainlessParser extends Parser {
     return _localctx;
   }
 
-  public static class FuncrefContext extends ParserRuleContext {
-    public TerminalNode TYPE() { return getToken(PainlessParser.TYPE, 0); }
-    public TerminalNode REF() { return getToken(PainlessParser.REF, 0); }
-    public TerminalNode ID() { return getToken(PainlessParser.ID, 0); }
-    public TerminalNode NEW() { return getToken(PainlessParser.NEW, 0); }
-    public FuncrefContext(ParserRuleContext parent, int invokingState) {
-      super(parent, invokingState);
-    }
-    @Override public int getRuleIndex() { return RULE_funcref; }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitFuncref(this);
-      else return visitor.visitChildren(this);
-    }
-  }
-
-  public final FuncrefContext funcref() throws RecognitionException {
-    FuncrefContext _localctx = new FuncrefContext(_ctx, getState());
-    enterRule(_localctx, 22, RULE_funcref);
-    int _la;
-    try {
-      enterOuterAlt(_localctx, 1);
-      {
-      setState(201);
-      match(TYPE);
-      setState(202);
-      match(REF);
-      setState(203);
-      _la = _input.LA(1);
-      if ( !(_la==NEW || _la==ID) ) {
-      _errHandler.recoverInline(this);
-      } else {
-        consume();
-      }
-      }
-    }
-    catch (RecognitionException re) {
-      _localctx.exception = re;
-      _errHandler.reportError(this, re);
-      _errHandler.recover(this, re);
-    }
-    finally {
-      exitRule();
-    }
-    return _localctx;
-  }
-
   public static class DeclvarContext extends ParserRuleContext {
     public TerminalNode ID() { return getToken(PainlessParser.ID, 0); }
     public TerminalNode ASSIGN() { return getToken(PainlessParser.ASSIGN, 0); }
@@ -1244,7 +1198,7 @@ class PainlessParser extends Parser {
 
   public final DeclvarContext declvar() throws RecognitionException {
     DeclvarContext _localctx = new DeclvarContext(_ctx, getState());
-    enterRule(_localctx, 24, RULE_declvar);
+    enterRule(_localctx, 22, RULE_declvar);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
@@ -1297,7 +1251,7 @@ class PainlessParser extends Parser {
 
   public final TrapContext trap() throws RecognitionException {
     TrapContext _localctx = new TrapContext(_ctx, getState());
-    enterRule(_localctx, 26, RULE_trap);
+    enterRule(_localctx, 24, RULE_trap);
     try {
       enterOuterAlt(_localctx, 1);
       {
@@ -1342,7 +1296,7 @@ class PainlessParser extends Parser {
 
   public final DelimiterContext delimiter() throws RecognitionException {
     DelimiterContext _localctx = new DelimiterContext(_ctx, getState());
-    enterRule(_localctx, 28, RULE_delimiter);
+    enterRule(_localctx, 26, RULE_delimiter);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
@@ -1509,8 +1463,8 @@ class PainlessParser extends Parser {
     int _parentState = getState();
     ExpressionContext _localctx = new ExpressionContext(_ctx, _parentState);
     ExpressionContext _prevctx = _localctx;
-    int _startState = 30;
-    enterRecursionRule(_localctx, 30, RULE_expression, _p);
+    int _startState = 28;
+    enterRecursionRule(_localctx, 28, RULE_expression, _p);
     int _la;
     try {
       int _alt;
@@ -1528,7 +1482,7 @@ class PainlessParser extends Parser {
         chain(true);
         setState(221);
         _la = _input.LA(1);
-        if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ASSIGN) | (1L << AADD) | (1L << ASUB) | (1L << AMUL) | (1L << ADIV) | (1L << AREM) | (1L << AAND) | (1L << AXOR) | (1L << AOR) | (1L << ALSH) | (1L << ARSH) | (1L << AUSH))) != 0)) ) {
+        if ( !(((((_la - 53)) & ~0x3f) == 0 && ((1L << (_la - 53)) & ((1L << (ASSIGN - 53)) | (1L << (AADD - 53)) | (1L << (ASUB - 53)) | (1L << (AMUL - 53)) | (1L << (ADIV - 53)) | (1L << (AREM - 53)) | (1L << (AAND - 53)) | (1L << (AXOR - 53)) | (1L << (AOR - 53)) | (1L << (ALSH - 53)) | (1L << (ARSH - 53)) | (1L << (AUSH - 53)))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
@@ -1879,7 +1833,7 @@ class PainlessParser extends Parser {
 
   public final UnaryContext unary(boolean c) throws RecognitionException {
     UnaryContext _localctx = new UnaryContext(_ctx, getState(), c);
-    enterRule(_localctx, 32, RULE_unary);
+    enterRule(_localctx, 30, RULE_unary);
     int _la;
     try {
       setState(321);
@@ -1936,7 +1890,7 @@ class PainlessParser extends Parser {
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
         setState(302);
         _la = _input.LA(1);
-        if ( !(((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (OCTAL - 64)) | (1L << (HEX - 64)) | (1L << (INTEGER - 64)) | (1L << (DECIMAL - 64)))) != 0)) ) {
+        if ( !(((((_la - 65)) & ~0x3f) == 0 && ((1L << (_la - 65)) & ((1L << (OCTAL - 65)) | (1L << (HEX - 65)) | (1L << (INTEGER - 65)) | (1L << (DECIMAL - 65)))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
@@ -2110,7 +2064,7 @@ class PainlessParser extends Parser {
 
   public final ChainContext chain(boolean c) throws RecognitionException {
     ChainContext _localctx = new ChainContext(_ctx, getState(), c);
-    enterRule(_localctx, 34, RULE_chain);
+    enterRule(_localctx, 32, RULE_chain);
     try {
       int _alt;
       setState(357);
@@ -2327,7 +2281,7 @@ class PainlessParser extends Parser {
 
   public final PrimaryContext primary(boolean c) throws RecognitionException {
     PrimaryContext _localctx = new PrimaryContext(_ctx, getState(), c);
-    enterRule(_localctx, 36, RULE_primary);
+    enterRule(_localctx, 34, RULE_primary);
     try {
       setState(377);
       switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) {
@@ -2434,7 +2388,7 @@ class PainlessParser extends Parser {
 
   public final SecondaryContext secondary(boolean s) throws RecognitionException {
     SecondaryContext _localctx = new SecondaryContext(_ctx, getState(), s);
-    enterRule(_localctx, 38, RULE_secondary);
+    enterRule(_localctx, 36, RULE_secondary);
     try {
       setState(383);
       switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
@@ -2507,7 +2461,7 @@ class PainlessParser extends Parser {
 
   public final DotContext dot() throws RecognitionException {
     DotContext _localctx = new DotContext(_ctx, getState());
-    enterRule(_localctx, 40, RULE_dot);
+    enterRule(_localctx, 38, RULE_dot);
     int _la;
     try {
       setState(390);
@@ -2579,7 +2533,7 @@ class PainlessParser extends Parser {
 
   public final BraceContext brace() throws RecognitionException {
     BraceContext _localctx = new BraceContext(_ctx, getState());
-    enterRule(_localctx, 42, RULE_brace);
+    enterRule(_localctx, 40, RULE_brace);
     try {
       _localctx = new BraceaccessContext(_localctx);
       enterOuterAlt(_localctx, 1);
@@ -2629,7 +2583,7 @@ class PainlessParser extends Parser {
 
   public final ArgumentsContext arguments() throws RecognitionException {
     ArgumentsContext _localctx = new ArgumentsContext(_ctx, getState());
-    enterRule(_localctx, 44, RULE_arguments);
+    enterRule(_localctx, 42, RULE_arguments);
     int _la;
     try {
       enterOuterAlt(_localctx, 1);
@@ -2682,6 +2636,9 @@ class PainlessParser extends Parser {
     public ExpressionContext expression() {
       return getRuleContext(ExpressionContext.class,0);
     }
+    public LambdaContext lambda() {
+      return getRuleContext(LambdaContext.class,0);
+    }
     public FuncrefContext funcref() {
       return getRuleContext(FuncrefContext.class,0);
     }
@@ -2698,9 +2655,9 @@ class PainlessParser extends Parser {
 
   public final ArgumentContext argument() throws RecognitionException {
     ArgumentContext _localctx = new ArgumentContext(_ctx, getState());
-    enterRule(_localctx, 46, RULE_argument);
+    enterRule(_localctx, 44, RULE_argument);
     try {
-      setState(411);
+      setState(412);
       switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
@@ -2713,6 +2670,13 @@ class PainlessParser extends Parser {
         enterOuterAlt(_localctx, 2);
         {
         setState(410);
+        lambda();
+        }
+        break;
+      case 3:
+        enterOuterAlt(_localctx, 3);
+        {
+        setState(411);
         funcref();
         }
         break;
@@ -2729,17 +2693,208 @@ class PainlessParser extends Parser {
     return _localctx;
   }
 
+  public static class LambdaContext extends ParserRuleContext {
+    public TerminalNode ARROW() { return getToken(PainlessParser.ARROW, 0); }
+    public BlockContext block() {
+      return getRuleContext(BlockContext.class,0);
+    }
+    public List<LamtypeContext> lamtype() {
+      return getRuleContexts(LamtypeContext.class);
+    }
+    public LamtypeContext lamtype(int i) {
+      return getRuleContext(LamtypeContext.class,i);
+    }
+    public TerminalNode LP() { return getToken(PainlessParser.LP, 0); }
+    public TerminalNode RP() { return getToken(PainlessParser.RP, 0); }
+    public List<TerminalNode> COMMA() { return getTokens(PainlessParser.COMMA); }
+    public TerminalNode COMMA(int i) {
+      return getToken(PainlessParser.COMMA, i);
+    }
+    public LambdaContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_lambda; }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitLambda(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final LambdaContext lambda() throws RecognitionException {
+    LambdaContext _localctx = new LambdaContext(_ctx, getState());
+    enterRule(_localctx, 46, RULE_lambda);
+    int _la;
+    try {
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(427);
+      switch (_input.LA(1)) {
+      case TYPE:
+      case ID:
+        {
+        setState(414);
+        lamtype();
+        }
+        break;
+      case LP:
+        {
+        setState(415);
+        match(LP);
+        setState(424);
+        _la = _input.LA(1);
+        if (_la==TYPE || _la==ID) {
+          {
+          setState(416);
+          lamtype();
+          setState(421);
+          _errHandler.sync(this);
+          _la = _input.LA(1);
+          while (_la==COMMA) {
+            {
+            {
+            setState(417);
+            match(COMMA);
+            setState(418);
+            lamtype();
+            }
+            }
+            setState(423);
+            _errHandler.sync(this);
+            _la = _input.LA(1);
+          }
+          }
+        }
+
+        setState(426);
+        match(RP);
+        }
+        break;
+      default:
+        throw new NoViableAltException(this);
+      }
+      setState(429);
+      match(ARROW);
+      setState(430);
+      block();
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
+  public static class LamtypeContext extends ParserRuleContext {
+    public TerminalNode ID() { return getToken(PainlessParser.ID, 0); }
+    public DecltypeContext decltype() {
+      return getRuleContext(DecltypeContext.class,0);
+    }
+    public LamtypeContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_lamtype; }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitLamtype(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final LamtypeContext lamtype() throws RecognitionException {
+    LamtypeContext _localctx = new LamtypeContext(_ctx, getState());
+    enterRule(_localctx, 48, RULE_lamtype);
+    int _la;
+    try {
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(433);
+      _la = _input.LA(1);
+      if (_la==TYPE) {
+        {
+        setState(432);
+        decltype();
+        }
+      }
+
+      setState(435);
+      match(ID);
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
+  public static class FuncrefContext extends ParserRuleContext {
+    public TerminalNode TYPE() { return getToken(PainlessParser.TYPE, 0); }
+    public TerminalNode REF() { return getToken(PainlessParser.REF, 0); }
+    public TerminalNode ID() { return getToken(PainlessParser.ID, 0); }
+    public TerminalNode NEW() { return getToken(PainlessParser.NEW, 0); }
+    public FuncrefContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_funcref; }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitFuncref(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final FuncrefContext funcref() throws RecognitionException {
+    FuncrefContext _localctx = new FuncrefContext(_ctx, getState());
+    enterRule(_localctx, 50, RULE_funcref);
+    int _la;
+    try {
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(437);
+      match(TYPE);
+      setState(438);
+      match(REF);
+      setState(439);
+      _la = _input.LA(1);
+      if ( !(_la==NEW || _la==ID) ) {
+      _errHandler.recoverInline(this);
+      } else {
+        consume();
+      }
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
   public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) {
     switch (ruleIndex) {
     case 3:
       return statement_sempred((StatementContext)_localctx, predIndex);
-    case 15:
+    case 14:
       return expression_sempred((ExpressionContext)_localctx, predIndex);
-    case 16:
+    case 15:
       return unary_sempred((UnaryContext)_localctx, predIndex);
-    case 18:
+    case 17:
       return primary_sempred((PrimaryContext)_localctx, predIndex);
-    case 19:
+    case 18:
       return secondary_sempred((SecondaryContext)_localctx, predIndex);
     }
     return true;
@@ -2819,158 +2974,171 @@ class PainlessParser extends Parser {
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3M\u01a0\4\2\t\2\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3N\u01bc\4\2\t\2\4"+
     "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
     "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
     "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+
-    "\3\2\7\2\64\n\2\f\2\16\2\67\13\2\3\2\7\2:\n\2\f\2\16\2=\13\2\3\2\3\2\3"+
-    "\3\3\3\3\3\3\3\3\3\3\4\3\4\3\4\3\4\3\4\3\4\3\4\7\4M\n\4\f\4\16\4P\13\4"+
-    "\5\4R\n\4\3\4\3\4\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\5\5^\n\5\3\5\3\5\3\5"+
-    "\3\5\3\5\3\5\5\5f\n\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\5\5"+
-    "s\n\5\3\5\3\5\5\5w\n\5\3\5\3\5\5\5{\n\5\3\5\3\5\3\5\5\5\u0080\n\5\3\5"+
-    "\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3"+
-    "\5\3\5\3\5\3\5\3\5\6\5\u0099\n\5\r\5\16\5\u009a\3\5\3\5\3\5\3\5\3\5\3"+
-    "\5\3\5\5\5\u00a4\n\5\3\6\3\6\5\6\u00a8\n\6\3\7\3\7\7\7\u00ac\n\7\f\7\16"+
-    "\7\u00af\13\7\3\7\3\7\3\b\3\b\3\t\3\t\5\t\u00b7\n\t\3\n\3\n\3\13\3\13"+
-    "\3\13\3\13\7\13\u00bf\n\13\f\13\16\13\u00c2\13\13\3\f\3\f\3\f\7\f\u00c7"+
-    "\n\f\f\f\16\f\u00ca\13\f\3\r\3\r\3\r\3\r\3\16\3\16\3\16\5\16\u00d3\n\16"+
-    "\3\17\3\17\3\17\3\17\3\17\3\17\3\17\3\20\3\20\3\21\3\21\3\21\3\21\3\21"+
-    "\3\21\3\21\3\21\3\21\5\21\u00e7\n\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
+    "\4\32\t\32\4\33\t\33\3\2\7\28\n\2\f\2\16\2;\13\2\3\2\7\2>\n\2\f\2\16\2"+
+    "A\13\2\3\2\3\2\3\3\3\3\3\3\3\3\3\3\3\4\3\4\3\4\3\4\3\4\3\4\3\4\7\4Q\n"+
+    "\4\f\4\16\4T\13\4\5\4V\n\4\3\4\3\4\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\5\5"+
+    "b\n\5\3\5\3\5\3\5\3\5\3\5\3\5\5\5j\n\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5"+
+    "\3\5\3\5\3\5\5\5w\n\5\3\5\3\5\5\5{\n\5\3\5\3\5\5\5\177\n\5\3\5\3\5\3\5"+
+    "\5\5\u0084\n\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5"+
+    "\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\6\5\u009d\n\5\r\5\16\5\u009e\3\5"+
+    "\3\5\3\5\3\5\3\5\3\5\3\5\5\5\u00a8\n\5\3\6\3\6\5\6\u00ac\n\6\3\7\3\7\7"+
+    "\7\u00b0\n\7\f\7\16\7\u00b3\13\7\3\7\3\7\3\b\3\b\3\t\3\t\5\t\u00bb\n\t"+
+    "\3\n\3\n\3\13\3\13\3\13\3\13\7\13\u00c3\n\13\f\13\16\13\u00c6\13\13\3"+
+    "\f\3\f\3\f\7\f\u00cb\n\f\f\f\16\f\u00ce\13\f\3\r\3\r\3\r\5\r\u00d3\n\r"+
+    "\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\17\3\17\3\20\3\20\3\20\3\20\3\20"+
+    "\3\20\3\20\3\20\3\20\5\20\u00e7\n\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
+    "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
+    "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
+    "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
+    "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\7\20\u0122\n\20\f\20\16\20\u0125"+
+    "\13\20\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
     "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
-    "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
-    "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
-    "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\7\21\u0122\n\21\f\21\16\21\u0125"+
-    "\13\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22"+
-    "\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22"+
-    "\3\22\3\22\5\22\u0144\n\22\3\23\3\23\7\23\u0148\n\23\f\23\16\23\u014b"+
-    "\13\23\3\23\3\23\3\23\7\23\u0150\n\23\f\23\16\23\u0153\13\23\3\23\3\23"+
-    "\3\23\3\23\3\23\3\23\6\23\u015b\n\23\r\23\16\23\u015c\3\23\3\23\7\23\u0161"+
-    "\n\23\f\23\16\23\u0164\13\23\5\23\u0166\n\23\5\23\u0168\n\23\3\24\3\24"+
-    "\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\24"+
-    "\3\24\3\24\5\24\u017c\n\24\3\25\3\25\3\25\3\25\5\25\u0182\n\25\3\26\3"+
-    "\26\3\26\3\26\3\26\5\26\u0189\n\26\3\27\3\27\3\27\3\27\3\30\3\30\3\30"+
-    "\3\30\7\30\u0193\n\30\f\30\16\30\u0196\13\30\5\30\u0198\n\30\3\30\3\30"+
-    "\3\31\3\31\5\31\u019e\n\31\3\31\2\3 \32\2\4\6\b\n\f\16\20\22\24\26\30"+
-    "\32\34\36 \"$&(*,.\60\2\16\4\2\26\26KK\3\3\r\r\3\2\66A\3\2\34\36\3\2\37"+
-    " \3\2!#\3\2$\'\3\2(+\3\2\64\65\3\2BE\4\2\32\33\37 \3\2LM\u01c8\2\65\3"+
-    "\2\2\2\4@\3\2\2\2\6E\3\2\2\2\b\u00a3\3\2\2\2\n\u00a7\3\2\2\2\f\u00a9\3"+
-    "\2\2\2\16\u00b2\3\2\2\2\20\u00b6\3\2\2\2\22\u00b8\3\2\2\2\24\u00ba\3\2"+
-    "\2\2\26\u00c3\3\2\2\2\30\u00cb\3\2\2\2\32\u00cf\3\2\2\2\34\u00d4\3\2\2"+
-    "\2\36\u00db\3\2\2\2 \u00e6\3\2\2\2\"\u0143\3\2\2\2$\u0167\3\2\2\2&\u017b"+
-    "\3\2\2\2(\u0181\3\2\2\2*\u0188\3\2\2\2,\u018a\3\2\2\2.\u018e\3\2\2\2\60"+
-    "\u019d\3\2\2\2\62\64\5\4\3\2\63\62\3\2\2\2\64\67\3\2\2\2\65\63\3\2\2\2"+
-    "\65\66\3\2\2\2\66;\3\2\2\2\67\65\3\2\2\28:\5\b\5\298\3\2\2\2:=\3\2\2\2"+
-    ";9\3\2\2\2;<\3\2\2\2<>\3\2\2\2=;\3\2\2\2>?\7\2\2\3?\3\3\2\2\2@A\5\26\f"+
-    "\2AB\7K\2\2BC\5\6\4\2CD\5\f\7\2D\5\3\2\2\2EQ\7\t\2\2FG\5\26\f\2GN\7K\2"+
-    "\2HI\7\f\2\2IJ\5\26\f\2JK\7K\2\2KM\3\2\2\2LH\3\2\2\2MP\3\2\2\2NL\3\2\2"+
-    "\2NO\3\2\2\2OR\3\2\2\2PN\3\2\2\2QF\3\2\2\2QR\3\2\2\2RS\3\2\2\2ST\7\n\2"+
-    "\2T\7\3\2\2\2UV\7\16\2\2VW\7\t\2\2WX\5 \21\2XY\7\n\2\2Y]\5\n\6\2Z[\7\17"+
-    "\2\2[^\5\n\6\2\\^\6\5\2\2]Z\3\2\2\2]\\\3\2\2\2^\u00a4\3\2\2\2_`\7\20\2"+
-    "\2`a\7\t\2\2ab\5 \21\2be\7\n\2\2cf\5\n\6\2df\5\16\b\2ec\3\2\2\2ed\3\2"+
-    "\2\2f\u00a4\3\2\2\2gh\7\21\2\2hi\5\f\7\2ij\7\20\2\2jk\7\t\2\2kl\5 \21"+
-    "\2lm\7\n\2\2mn\5\36\20\2n\u00a4\3\2\2\2op\7\22\2\2pr\7\t\2\2qs\5\20\t"+
-    "\2rq\3\2\2\2rs\3\2\2\2st\3\2\2\2tv\7\r\2\2uw\5 \21\2vu\3\2\2\2vw\3\2\2"+
-    "\2wx\3\2\2\2xz\7\r\2\2y{\5\22\n\2zy\3\2\2\2z{\3\2\2\2{|\3\2\2\2|\177\7"+
-    "\n\2\2}\u0080\5\n\6\2~\u0080\5\16\b\2\177}\3\2\2\2\177~\3\2\2\2\u0080"+
-    "\u00a4\3\2\2\2\u0081\u0082\7\22\2\2\u0082\u0083\7\t\2\2\u0083\u0084\5"+
-    "\26\f\2\u0084\u0085\7K\2\2\u0085\u0086\7\62\2\2\u0086\u0087\5 \21\2\u0087"+
-    "\u0088\7\n\2\2\u0088\u0089\5\n\6\2\u0089\u00a4\3\2\2\2\u008a\u008b\5\24"+
-    "\13\2\u008b\u008c\5\36\20\2\u008c\u00a4\3\2\2\2\u008d\u008e\7\23\2\2\u008e"+
-    "\u00a4\5\36\20\2\u008f\u0090\7\24\2\2\u0090\u00a4\5\36\20\2\u0091\u0092"+
-    "\7\25\2\2\u0092\u0093\5 \21\2\u0093\u0094\5\36\20\2\u0094\u00a4\3\2\2"+
-    "\2\u0095\u0096\7\27\2\2\u0096\u0098\5\f\7\2\u0097\u0099\5\34\17\2\u0098"+
-    "\u0097\3\2\2\2\u0099\u009a\3\2\2\2\u009a\u0098\3\2\2\2\u009a\u009b\3\2"+
-    "\2\2\u009b\u00a4\3\2\2\2\u009c\u009d\7\31\2\2\u009d\u009e\5 \21\2\u009e"+
-    "\u009f\5\36\20\2\u009f\u00a4\3\2\2\2\u00a0\u00a1\5 \21\2\u00a1\u00a2\5"+
-    "\36\20\2\u00a2\u00a4\3\2\2\2\u00a3U\3\2\2\2\u00a3_\3\2\2\2\u00a3g\3\2"+
-    "\2\2\u00a3o\3\2\2\2\u00a3\u0081\3\2\2\2\u00a3\u008a\3\2\2\2\u00a3\u008d"+
-    "\3\2\2\2\u00a3\u008f\3\2\2\2\u00a3\u0091\3\2\2\2\u00a3\u0095\3\2\2\2\u00a3"+
-    "\u009c\3\2\2\2\u00a3\u00a0\3\2\2\2\u00a4\t\3\2\2\2\u00a5\u00a8\5\f\7\2"+
-    "\u00a6\u00a8\5\b\5\2\u00a7\u00a5\3\2\2\2\u00a7\u00a6\3\2\2\2\u00a8\13"+
-    "\3\2\2\2\u00a9\u00ad\7\5\2\2\u00aa\u00ac\5\b\5\2\u00ab\u00aa\3\2\2\2\u00ac"+
-    "\u00af\3\2\2\2\u00ad\u00ab\3\2\2\2\u00ad\u00ae\3\2\2\2\u00ae\u00b0\3\2"+
-    "\2\2\u00af\u00ad\3\2\2\2\u00b0\u00b1\7\6\2\2\u00b1\r\3\2\2\2\u00b2\u00b3"+
-    "\7\r\2\2\u00b3\17\3\2\2\2\u00b4\u00b7\5\24\13\2\u00b5\u00b7\5 \21\2\u00b6"+
-    "\u00b4\3\2\2\2\u00b6\u00b5\3\2\2\2\u00b7\21\3\2\2\2\u00b8\u00b9\5 \21"+
-    "\2\u00b9\23\3\2\2\2\u00ba\u00bb\5\26\f\2\u00bb\u00c0\5\32\16\2\u00bc\u00bd"+
-    "\7\f\2\2\u00bd\u00bf\5\32\16\2\u00be\u00bc\3\2\2\2\u00bf\u00c2\3\2\2\2"+
-    "\u00c0\u00be\3\2\2\2\u00c0\u00c1\3\2\2\2\u00c1\25\3\2\2\2\u00c2\u00c0"+
-    "\3\2\2\2\u00c3\u00c8\7J\2\2\u00c4\u00c5\7\7\2\2\u00c5\u00c7\7\b\2\2\u00c6"+
-    "\u00c4\3\2\2\2\u00c7\u00ca\3\2\2\2\u00c8\u00c6\3\2\2\2\u00c8\u00c9\3\2"+
-    "\2\2\u00c9\27\3\2\2\2\u00ca\u00c8\3\2\2\2\u00cb\u00cc\7J\2\2\u00cc\u00cd"+
-    "\7\63\2\2\u00cd\u00ce\t\2\2\2\u00ce\31\3\2\2\2\u00cf\u00d2\7K\2\2\u00d0"+
-    "\u00d1\7\66\2\2\u00d1\u00d3\5 \21\2\u00d2\u00d0\3\2\2\2\u00d2\u00d3\3"+
-    "\2\2\2\u00d3\33\3\2\2\2\u00d4\u00d5\7\30\2\2\u00d5\u00d6\7\t\2\2\u00d6"+
-    "\u00d7\7J\2\2\u00d7\u00d8\7K\2\2\u00d8\u00d9\7\n\2\2\u00d9\u00da\5\f\7"+
-    "\2\u00da\35\3\2\2\2\u00db\u00dc\t\3\2\2\u00dc\37\3\2\2\2\u00dd\u00de\b"+
-    "\21\1\2\u00de\u00df\5$\23\2\u00df\u00e0\t\4\2\2\u00e0\u00e1\5 \21\3\u00e1"+
-    "\u00e2\b\21\1\2\u00e2\u00e7\3\2\2\2\u00e3\u00e4\5\"\22\2\u00e4\u00e5\b"+
-    "\21\1\2\u00e5\u00e7\3\2\2\2\u00e6\u00dd\3\2\2\2\u00e6\u00e3\3\2\2\2\u00e7"+
-    "\u0123\3\2\2\2\u00e8\u00e9\f\16\2\2\u00e9\u00ea\t\5\2\2\u00ea\u00eb\5"+
-    " \21\17\u00eb\u00ec\b\21\1\2\u00ec\u0122\3\2\2\2\u00ed\u00ee\f\r\2\2\u00ee"+
-    "\u00ef\t\6\2\2\u00ef\u00f0\5 \21\16\u00f0\u00f1\b\21\1\2\u00f1\u0122\3"+
-    "\2\2\2\u00f2\u00f3\f\f\2\2\u00f3\u00f4\t\7\2\2\u00f4\u00f5\5 \21\r\u00f5"+
-    "\u00f6\b\21\1\2\u00f6\u0122\3\2\2\2\u00f7\u00f8\f\13\2\2\u00f8\u00f9\t"+
-    "\b\2\2\u00f9\u00fa\5 \21\f\u00fa\u00fb\b\21\1\2\u00fb\u0122\3\2\2\2\u00fc"+
-    "\u00fd\f\n\2\2\u00fd\u00fe\t\t\2\2\u00fe\u00ff\5 \21\13\u00ff\u0100\b"+
-    "\21\1\2\u0100\u0122\3\2\2\2\u0101\u0102\f\t\2\2\u0102\u0103\7,\2\2\u0103"+
-    "\u0104\5 \21\n\u0104\u0105\b\21\1\2\u0105\u0122\3\2\2\2\u0106\u0107\f"+
-    "\b\2\2\u0107\u0108\7-\2\2\u0108\u0109\5 \21\t\u0109\u010a\b\21\1\2\u010a"+
-    "\u0122\3\2\2\2\u010b\u010c\f\7\2\2\u010c\u010d\7.\2\2\u010d\u010e\5 \21"+
-    "\b\u010e\u010f\b\21\1\2\u010f\u0122\3\2\2\2\u0110\u0111\f\6\2\2\u0111"+
-    "\u0112\7/\2\2\u0112\u0113\5 \21\7\u0113\u0114\b\21\1\2\u0114\u0122\3\2"+
-    "\2\2\u0115\u0116\f\5\2\2\u0116\u0117\7\60\2\2\u0117\u0118\5 \21\6\u0118"+
-    "\u0119\b\21\1\2\u0119\u0122\3\2\2\2\u011a\u011b\f\4\2\2\u011b\u011c\7"+
-    "\61\2\2\u011c\u011d\5 \21\2\u011d\u011e\7\62\2\2\u011e\u011f\5 \21\4\u011f"+
-    "\u0120\b\21\1\2\u0120\u0122\3\2\2\2\u0121\u00e8\3\2\2\2\u0121\u00ed\3"+
-    "\2\2\2\u0121\u00f2\3\2\2\2\u0121\u00f7\3\2\2\2\u0121\u00fc\3\2\2\2\u0121"+
-    "\u0101\3\2\2\2\u0121\u0106\3\2\2\2\u0121\u010b\3\2\2\2\u0121\u0110\3\2"+
-    "\2\2\u0121\u0115\3\2\2\2\u0121\u011a\3\2\2\2\u0122\u0125\3\2\2\2\u0123"+
-    "\u0121\3\2\2\2\u0123\u0124\3\2\2\2\u0124!\3\2\2\2\u0125\u0123\3\2\2\2"+
-    "\u0126\u0127\6\22\16\3\u0127\u0128\t\n\2\2\u0128\u0144\5$\23\2\u0129\u012a"+
-    "\6\22\17\3\u012a\u012b\5$\23\2\u012b\u012c\t\n\2\2\u012c\u0144\3\2\2\2"+
-    "\u012d\u012e\6\22\20\3\u012e\u0144\5$\23\2\u012f\u0130\6\22\21\3\u0130"+
-    "\u0131\t\13\2\2\u0131\u0144\b\22\1\2\u0132\u0133\6\22\22\3\u0133\u0134"+
-    "\7G\2\2\u0134\u0144\b\22\1\2\u0135\u0136\6\22\23\3\u0136\u0137\7H\2\2"+
-    "\u0137\u0144\b\22\1\2\u0138\u0139\6\22\24\3\u0139\u013a\7I\2\2\u013a\u0144"+
-    "\b\22\1\2\u013b\u013c\6\22\25\3\u013c\u013d\t\f\2\2\u013d\u0144\5\"\22"+
-    "\2\u013e\u013f\7\t\2\2\u013f\u0140\5\26\f\2\u0140\u0141\7\n\2\2\u0141"+
-    "\u0142\5\"\22\2\u0142\u0144\3\2\2\2\u0143\u0126\3\2\2\2\u0143\u0129\3"+
-    "\2\2\2\u0143\u012d\3\2\2\2\u0143\u012f\3\2\2\2\u0143\u0132\3\2\2\2\u0143"+
-    "\u0135\3\2\2\2\u0143\u0138\3\2\2\2\u0143\u013b\3\2\2\2\u0143\u013e\3\2"+
-    "\2\2\u0144#\3\2\2\2\u0145\u0149\5&\24\2\u0146\u0148\5(\25\2\u0147\u0146"+
-    "\3\2\2\2\u0148\u014b\3\2\2\2\u0149\u0147\3\2\2\2\u0149\u014a\3\2\2\2\u014a"+
-    "\u0168\3\2\2\2\u014b\u0149\3\2\2\2\u014c\u014d\5\26\f\2\u014d\u0151\5"+
-    "*\26\2\u014e\u0150\5(\25\2\u014f\u014e\3\2\2\2\u0150\u0153\3\2\2\2\u0151"+
-    "\u014f\3\2\2\2\u0151\u0152\3\2\2\2\u0152\u0168\3\2\2\2\u0153\u0151\3\2"+
-    "\2\2\u0154\u0155\7\26\2\2\u0155\u015a\7J\2\2\u0156\u0157\7\7\2\2\u0157"+
-    "\u0158\5 \21\2\u0158\u0159\7\b\2\2\u0159\u015b\3\2\2\2\u015a\u0156\3\2"+
-    "\2\2\u015b\u015c\3\2\2\2\u015c\u015a\3\2\2\2\u015c\u015d\3\2\2\2\u015d"+
-    "\u0165\3\2\2\2\u015e\u0162\5*\26\2\u015f\u0161\5(\25\2\u0160\u015f\3\2"+
-    "\2\2\u0161\u0164\3\2\2\2\u0162\u0160\3\2\2\2\u0162\u0163\3\2\2\2\u0163"+
-    "\u0166\3\2\2\2\u0164\u0162\3\2\2\2\u0165\u015e\3\2\2\2\u0165\u0166\3\2"+
-    "\2\2\u0166\u0168\3\2\2\2\u0167\u0145\3\2\2\2\u0167\u014c\3\2\2\2\u0167"+
-    "\u0154\3\2\2\2\u0168%\3\2\2\2\u0169\u016a\6\24\26\3\u016a\u016b\7\t\2"+
-    "\2\u016b\u016c\5 \21\2\u016c\u016d\7\n\2\2\u016d\u016e\b\24\1\2\u016e"+
-    "\u017c\3\2\2\2\u016f\u0170\6\24\27\3\u0170\u0171\7\t\2\2\u0171\u0172\5"+
-    "\"\22\2\u0172\u0173\7\n\2\2\u0173\u017c\3\2\2\2\u0174\u017c\7F\2\2\u0175"+
-    "\u017c\7K\2\2\u0176\u0177\7K\2\2\u0177\u017c\5.\30\2\u0178\u0179\7\26"+
-    "\2\2\u0179\u017a\7J\2\2\u017a\u017c\5.\30\2\u017b\u0169\3\2\2\2\u017b"+
-    "\u016f\3\2\2\2\u017b\u0174\3\2\2\2\u017b\u0175\3\2\2\2\u017b\u0176\3\2"+
-    "\2\2\u017b\u0178\3\2\2\2\u017c\'\3\2\2\2\u017d\u017e\6\25\30\3\u017e\u0182"+
-    "\5*\26\2\u017f\u0180\6\25\31\3\u0180\u0182\5,\27\2\u0181\u017d\3\2\2\2"+
-    "\u0181\u017f\3\2\2\2\u0182)\3\2\2\2\u0183\u0184\7\13\2\2\u0184\u0185\7"+
-    "M\2\2\u0185\u0189\5.\30\2\u0186\u0187\7\13\2\2\u0187\u0189\t\r\2\2\u0188"+
-    "\u0183\3\2\2\2\u0188\u0186\3\2\2\2\u0189+\3\2\2\2\u018a\u018b\7\7\2\2"+
-    "\u018b\u018c\5 \21\2\u018c\u018d\7\b\2\2\u018d-\3\2\2\2\u018e\u0197\7"+
-    "\t\2\2\u018f\u0194\5\60\31\2\u0190\u0191\7\f\2\2\u0191\u0193\5\60\31\2"+
-    "\u0192\u0190\3\2\2\2\u0193\u0196\3\2\2\2\u0194\u0192\3\2\2\2\u0194\u0195"+
-    "\3\2\2\2\u0195\u0198\3\2\2\2\u0196\u0194\3\2\2\2\u0197\u018f\3\2\2\2\u0197"+
-    "\u0198\3\2\2\2\u0198\u0199\3\2\2\2\u0199\u019a\7\n\2\2\u019a/\3\2\2\2"+
-    "\u019b\u019e\5 \21\2\u019c\u019e\5\30\r\2\u019d\u019b\3\2\2\2\u019d\u019c"+
-    "\3\2\2\2\u019e\61\3\2\2\2$\65;NQ]ervz\177\u009a\u00a3\u00a7\u00ad\u00b6"+
-    "\u00c0\u00c8\u00d2\u00e6\u0121\u0123\u0143\u0149\u0151\u015c\u0162\u0165"+
-    "\u0167\u017b\u0181\u0188\u0194\u0197\u019d";
+    "\3\21\3\21\5\21\u0144\n\21\3\22\3\22\7\22\u0148\n\22\f\22\16\22\u014b"+
+    "\13\22\3\22\3\22\3\22\7\22\u0150\n\22\f\22\16\22\u0153\13\22\3\22\3\22"+
+    "\3\22\3\22\3\22\3\22\6\22\u015b\n\22\r\22\16\22\u015c\3\22\3\22\7\22\u0161"+
+    "\n\22\f\22\16\22\u0164\13\22\5\22\u0166\n\22\5\22\u0168\n\22\3\23\3\23"+
+    "\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23"+
+    "\3\23\3\23\5\23\u017c\n\23\3\24\3\24\3\24\3\24\5\24\u0182\n\24\3\25\3"+
+    "\25\3\25\3\25\3\25\5\25\u0189\n\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27"+
+    "\3\27\7\27\u0193\n\27\f\27\16\27\u0196\13\27\5\27\u0198\n\27\3\27\3\27"+
+    "\3\30\3\30\3\30\5\30\u019f\n\30\3\31\3\31\3\31\3\31\3\31\7\31\u01a6\n"+
+    "\31\f\31\16\31\u01a9\13\31\5\31\u01ab\n\31\3\31\5\31\u01ae\n\31\3\31\3"+
+    "\31\3\31\3\32\5\32\u01b4\n\32\3\32\3\32\3\33\3\33\3\33\3\33\3\33\2\3\36"+
+    "\34\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,.\60\62\64\2\16\3\3"+
+    "\r\r\3\2\67B\3\2\34\36\3\2\37 \3\2!#\3\2$\'\3\2(+\3\2\65\66\3\2CF\4\2"+
+    "\32\33\37 \3\2MN\4\2\26\26LL\u01e7\29\3\2\2\2\4D\3\2\2\2\6I\3\2\2\2\b"+
+    "\u00a7\3\2\2\2\n\u00ab\3\2\2\2\f\u00ad\3\2\2\2\16\u00b6\3\2\2\2\20\u00ba"+
+    "\3\2\2\2\22\u00bc\3\2\2\2\24\u00be\3\2\2\2\26\u00c7\3\2\2\2\30\u00cf\3"+
+    "\2\2\2\32\u00d4\3\2\2\2\34\u00db\3\2\2\2\36\u00e6\3\2\2\2 \u0143\3\2\2"+
+    "\2\"\u0167\3\2\2\2$\u017b\3\2\2\2&\u0181\3\2\2\2(\u0188\3\2\2\2*\u018a"+
+    "\3\2\2\2,\u018e\3\2\2\2.\u019e\3\2\2\2\60\u01ad\3\2\2\2\62\u01b3\3\2\2"+
+    "\2\64\u01b7\3\2\2\2\668\5\4\3\2\67\66\3\2\2\28;\3\2\2\29\67\3\2\2\29:"+
+    "\3\2\2\2:?\3\2\2\2;9\3\2\2\2<>\5\b\5\2=<\3\2\2\2>A\3\2\2\2?=\3\2\2\2?"+
+    "@\3\2\2\2@B\3\2\2\2A?\3\2\2\2BC\7\2\2\3C\3\3\2\2\2DE\5\26\f\2EF\7L\2\2"+
+    "FG\5\6\4\2GH\5\f\7\2H\5\3\2\2\2IU\7\t\2\2JK\5\26\f\2KR\7L\2\2LM\7\f\2"+
+    "\2MN\5\26\f\2NO\7L\2\2OQ\3\2\2\2PL\3\2\2\2QT\3\2\2\2RP\3\2\2\2RS\3\2\2"+
+    "\2SV\3\2\2\2TR\3\2\2\2UJ\3\2\2\2UV\3\2\2\2VW\3\2\2\2WX\7\n\2\2X\7\3\2"+
+    "\2\2YZ\7\16\2\2Z[\7\t\2\2[\\\5\36\20\2\\]\7\n\2\2]a\5\n\6\2^_\7\17\2\2"+
+    "_b\5\n\6\2`b\6\5\2\2a^\3\2\2\2a`\3\2\2\2b\u00a8\3\2\2\2cd\7\20\2\2de\7"+
+    "\t\2\2ef\5\36\20\2fi\7\n\2\2gj\5\n\6\2hj\5\16\b\2ig\3\2\2\2ih\3\2\2\2"+
+    "j\u00a8\3\2\2\2kl\7\21\2\2lm\5\f\7\2mn\7\20\2\2no\7\t\2\2op\5\36\20\2"+
+    "pq\7\n\2\2qr\5\34\17\2r\u00a8\3\2\2\2st\7\22\2\2tv\7\t\2\2uw\5\20\t\2"+
+    "vu\3\2\2\2vw\3\2\2\2wx\3\2\2\2xz\7\r\2\2y{\5\36\20\2zy\3\2\2\2z{\3\2\2"+
+    "\2{|\3\2\2\2|~\7\r\2\2}\177\5\22\n\2~}\3\2\2\2~\177\3\2\2\2\177\u0080"+
+    "\3\2\2\2\u0080\u0083\7\n\2\2\u0081\u0084\5\n\6\2\u0082\u0084\5\16\b\2"+
+    "\u0083\u0081\3\2\2\2\u0083\u0082\3\2\2\2\u0084\u00a8\3\2\2\2\u0085\u0086"+
+    "\7\22\2\2\u0086\u0087\7\t\2\2\u0087\u0088\5\26\f\2\u0088\u0089\7L\2\2"+
+    "\u0089\u008a\7\62\2\2\u008a\u008b\5\36\20\2\u008b\u008c\7\n\2\2\u008c"+
+    "\u008d\5\n\6\2\u008d\u00a8\3\2\2\2\u008e\u008f\5\24\13\2\u008f\u0090\5"+
+    "\34\17\2\u0090\u00a8\3\2\2\2\u0091\u0092\7\23\2\2\u0092\u00a8\5\34\17"+
+    "\2\u0093\u0094\7\24\2\2\u0094\u00a8\5\34\17\2\u0095\u0096\7\25\2\2\u0096"+
+    "\u0097\5\36\20\2\u0097\u0098\5\34\17\2\u0098\u00a8\3\2\2\2\u0099\u009a"+
+    "\7\27\2\2\u009a\u009c\5\f\7\2\u009b\u009d\5\32\16\2\u009c\u009b\3\2\2"+
+    "\2\u009d\u009e\3\2\2\2\u009e\u009c\3\2\2\2\u009e\u009f\3\2\2\2\u009f\u00a8"+
+    "\3\2\2\2\u00a0\u00a1\7\31\2\2\u00a1\u00a2\5\36\20\2\u00a2\u00a3\5\34\17"+
+    "\2\u00a3\u00a8\3\2\2\2\u00a4\u00a5\5\36\20\2\u00a5\u00a6\5\34\17\2\u00a6"+
+    "\u00a8\3\2\2\2\u00a7Y\3\2\2\2\u00a7c\3\2\2\2\u00a7k\3\2\2\2\u00a7s\3\2"+
+    "\2\2\u00a7\u0085\3\2\2\2\u00a7\u008e\3\2\2\2\u00a7\u0091\3\2\2\2\u00a7"+
+    "\u0093\3\2\2\2\u00a7\u0095\3\2\2\2\u00a7\u0099\3\2\2\2\u00a7\u00a0\3\2"+
+    "\2\2\u00a7\u00a4\3\2\2\2\u00a8\t\3\2\2\2\u00a9\u00ac\5\f\7\2\u00aa\u00ac"+
+    "\5\b\5\2\u00ab\u00a9\3\2\2\2\u00ab\u00aa\3\2\2\2\u00ac\13\3\2\2\2\u00ad"+
+    "\u00b1\7\5\2\2\u00ae\u00b0\5\b\5\2\u00af\u00ae\3\2\2\2\u00b0\u00b3\3\2"+
+    "\2\2\u00b1\u00af\3\2\2\2\u00b1\u00b2\3\2\2\2\u00b2\u00b4\3\2\2\2\u00b3"+
+    "\u00b1\3\2\2\2\u00b4\u00b5\7\6\2\2\u00b5\r\3\2\2\2\u00b6\u00b7\7\r\2\2"+
+    "\u00b7\17\3\2\2\2\u00b8\u00bb\5\24\13\2\u00b9\u00bb\5\36\20\2\u00ba\u00b8"+
+    "\3\2\2\2\u00ba\u00b9\3\2\2\2\u00bb\21\3\2\2\2\u00bc\u00bd\5\36\20\2\u00bd"+
+    "\23\3\2\2\2\u00be\u00bf\5\26\f\2\u00bf\u00c4\5\30\r\2\u00c0\u00c1\7\f"+
+    "\2\2\u00c1\u00c3\5\30\r\2\u00c2\u00c0\3\2\2\2\u00c3\u00c6\3\2\2\2\u00c4"+
+    "\u00c2\3\2\2\2\u00c4\u00c5\3\2\2\2\u00c5\25\3\2\2\2\u00c6\u00c4\3\2\2"+
+    "\2\u00c7\u00cc\7K\2\2\u00c8\u00c9\7\7\2\2\u00c9\u00cb\7\b\2\2\u00ca\u00c8"+
+    "\3\2\2\2\u00cb\u00ce\3\2\2\2\u00cc\u00ca\3\2\2\2\u00cc\u00cd\3\2\2\2\u00cd"+
+    "\27\3\2\2\2\u00ce\u00cc\3\2\2\2\u00cf\u00d2\7L\2\2\u00d0\u00d1\7\67\2"+
+    "\2\u00d1\u00d3\5\36\20\2\u00d2\u00d0\3\2\2\2\u00d2\u00d3\3\2\2\2\u00d3"+
+    "\31\3\2\2\2\u00d4\u00d5\7\30\2\2\u00d5\u00d6\7\t\2\2\u00d6\u00d7\7K\2"+
+    "\2\u00d7\u00d8\7L\2\2\u00d8\u00d9\7\n\2\2\u00d9\u00da\5\f\7\2\u00da\33"+
+    "\3\2\2\2\u00db\u00dc\t\2\2\2\u00dc\35\3\2\2\2\u00dd\u00de\b\20\1\2\u00de"+
+    "\u00df\5\"\22\2\u00df\u00e0\t\3\2\2\u00e0\u00e1\5\36\20\3\u00e1\u00e2"+
+    "\b\20\1\2\u00e2\u00e7\3\2\2\2\u00e3\u00e4\5 \21\2\u00e4\u00e5\b\20\1\2"+
+    "\u00e5\u00e7\3\2\2\2\u00e6\u00dd\3\2\2\2\u00e6\u00e3\3\2\2\2\u00e7\u0123"+
+    "\3\2\2\2\u00e8\u00e9\f\16\2\2\u00e9\u00ea\t\4\2\2\u00ea\u00eb\5\36\20"+
+    "\17\u00eb\u00ec\b\20\1\2\u00ec\u0122\3\2\2\2\u00ed\u00ee\f\r\2\2\u00ee"+
+    "\u00ef\t\5\2\2\u00ef\u00f0\5\36\20\16\u00f0\u00f1\b\20\1\2\u00f1\u0122"+
+    "\3\2\2\2\u00f2\u00f3\f\f\2\2\u00f3\u00f4\t\6\2\2\u00f4\u00f5\5\36\20\r"+
+    "\u00f5\u00f6\b\20\1\2\u00f6\u0122\3\2\2\2\u00f7\u00f8\f\13\2\2\u00f8\u00f9"+
+    "\t\7\2\2\u00f9\u00fa\5\36\20\f\u00fa\u00fb\b\20\1\2\u00fb\u0122\3\2\2"+
+    "\2\u00fc\u00fd\f\n\2\2\u00fd\u00fe\t\b\2\2\u00fe\u00ff\5\36\20\13\u00ff"+
+    "\u0100\b\20\1\2\u0100\u0122\3\2\2\2\u0101\u0102\f\t\2\2\u0102\u0103\7"+
+    ",\2\2\u0103\u0104\5\36\20\n\u0104\u0105\b\20\1\2\u0105\u0122\3\2\2\2\u0106"+
+    "\u0107\f\b\2\2\u0107\u0108\7-\2\2\u0108\u0109\5\36\20\t\u0109\u010a\b"+
+    "\20\1\2\u010a\u0122\3\2\2\2\u010b\u010c\f\7\2\2\u010c\u010d\7.\2\2\u010d"+
+    "\u010e\5\36\20\b\u010e\u010f\b\20\1\2\u010f\u0122\3\2\2\2\u0110\u0111"+
+    "\f\6\2\2\u0111\u0112\7/\2\2\u0112\u0113\5\36\20\7\u0113\u0114\b\20\1\2"+
+    "\u0114\u0122\3\2\2\2\u0115\u0116\f\5\2\2\u0116\u0117\7\60\2\2\u0117\u0118"+
+    "\5\36\20\6\u0118\u0119\b\20\1\2\u0119\u0122\3\2\2\2\u011a\u011b\f\4\2"+
+    "\2\u011b\u011c\7\61\2\2\u011c\u011d\5\36\20\2\u011d\u011e\7\62\2\2\u011e"+
+    "\u011f\5\36\20\4\u011f\u0120\b\20\1\2\u0120\u0122\3\2\2\2\u0121\u00e8"+
+    "\3\2\2\2\u0121\u00ed\3\2\2\2\u0121\u00f2\3\2\2\2\u0121\u00f7\3\2\2\2\u0121"+
+    "\u00fc\3\2\2\2\u0121\u0101\3\2\2\2\u0121\u0106\3\2\2\2\u0121\u010b\3\2"+
+    "\2\2\u0121\u0110\3\2\2\2\u0121\u0115\3\2\2\2\u0121\u011a\3\2\2\2\u0122"+
+    "\u0125\3\2\2\2\u0123\u0121\3\2\2\2\u0123\u0124\3\2\2\2\u0124\37\3\2\2"+
+    "\2\u0125\u0123\3\2\2\2\u0126\u0127\6\21\16\3\u0127\u0128\t\t\2\2\u0128"+
+    "\u0144\5\"\22\2\u0129\u012a\6\21\17\3\u012a\u012b\5\"\22\2\u012b\u012c"+
+    "\t\t\2\2\u012c\u0144\3\2\2\2\u012d\u012e\6\21\20\3\u012e\u0144\5\"\22"+
+    "\2\u012f\u0130\6\21\21\3\u0130\u0131\t\n\2\2\u0131\u0144\b\21\1\2\u0132"+
+    "\u0133\6\21\22\3\u0133\u0134\7H\2\2\u0134\u0144\b\21\1\2\u0135\u0136\6"+
+    "\21\23\3\u0136\u0137\7I\2\2\u0137\u0144\b\21\1\2\u0138\u0139\6\21\24\3"+
+    "\u0139\u013a\7J\2\2\u013a\u0144\b\21\1\2\u013b\u013c\6\21\25\3\u013c\u013d"+
+    "\t\13\2\2\u013d\u0144\5 \21\2\u013e\u013f\7\t\2\2\u013f\u0140\5\26\f\2"+
+    "\u0140\u0141\7\n\2\2\u0141\u0142\5 \21\2\u0142\u0144\3\2\2\2\u0143\u0126"+
+    "\3\2\2\2\u0143\u0129\3\2\2\2\u0143\u012d\3\2\2\2\u0143\u012f\3\2\2\2\u0143"+
+    "\u0132\3\2\2\2\u0143\u0135\3\2\2\2\u0143\u0138\3\2\2\2\u0143\u013b\3\2"+
+    "\2\2\u0143\u013e\3\2\2\2\u0144!\3\2\2\2\u0145\u0149\5$\23\2\u0146\u0148"+
+    "\5&\24\2\u0147\u0146\3\2\2\2\u0148\u014b\3\2\2\2\u0149\u0147\3\2\2\2\u0149"+
+    "\u014a\3\2\2\2\u014a\u0168\3\2\2\2\u014b\u0149\3\2\2\2\u014c\u014d\5\26"+
+    "\f\2\u014d\u0151\5(\25\2\u014e\u0150\5&\24\2\u014f\u014e\3\2\2\2\u0150"+
+    "\u0153\3\2\2\2\u0151\u014f\3\2\2\2\u0151\u0152\3\2\2\2\u0152\u0168\3\2"+
+    "\2\2\u0153\u0151\3\2\2\2\u0154\u0155\7\26\2\2\u0155\u015a\7K\2\2\u0156"+
+    "\u0157\7\7\2\2\u0157\u0158\5\36\20\2\u0158\u0159\7\b\2\2\u0159\u015b\3"+
+    "\2\2\2\u015a\u0156\3\2\2\2\u015b\u015c\3\2\2\2\u015c\u015a\3\2\2\2\u015c"+
+    "\u015d\3\2\2\2\u015d\u0165\3\2\2\2\u015e\u0162\5(\25\2\u015f\u0161\5&"+
+    "\24\2\u0160\u015f\3\2\2\2\u0161\u0164\3\2\2\2\u0162\u0160\3\2\2\2\u0162"+
+    "\u0163\3\2\2\2\u0163\u0166\3\2\2\2\u0164\u0162\3\2\2\2\u0165\u015e\3\2"+
+    "\2\2\u0165\u0166\3\2\2\2\u0166\u0168\3\2\2\2\u0167\u0145\3\2\2\2\u0167"+
+    "\u014c\3\2\2\2\u0167\u0154\3\2\2\2\u0168#\3\2\2\2\u0169\u016a\6\23\26"+
+    "\3\u016a\u016b\7\t\2\2\u016b\u016c\5\36\20\2\u016c\u016d\7\n\2\2\u016d"+
+    "\u016e\b\23\1\2\u016e\u017c\3\2\2\2\u016f\u0170\6\23\27\3\u0170\u0171"+
+    "\7\t\2\2\u0171\u0172\5 \21\2\u0172\u0173\7\n\2\2\u0173\u017c\3\2\2\2\u0174"+
+    "\u017c\7G\2\2\u0175\u017c\7L\2\2\u0176\u0177\7L\2\2\u0177\u017c\5,\27"+
+    "\2\u0178\u0179\7\26\2\2\u0179\u017a\7K\2\2\u017a\u017c\5,\27\2\u017b\u0169"+
+    "\3\2\2\2\u017b\u016f\3\2\2\2\u017b\u0174\3\2\2\2\u017b\u0175\3\2\2\2\u017b"+
+    "\u0176\3\2\2\2\u017b\u0178\3\2\2\2\u017c%\3\2\2\2\u017d\u017e\6\24\30"+
+    "\3\u017e\u0182\5(\25\2\u017f\u0180\6\24\31\3\u0180\u0182\5*\26\2\u0181"+
+    "\u017d\3\2\2\2\u0181\u017f\3\2\2\2\u0182\'\3\2\2\2\u0183\u0184\7\13\2"+
+    "\2\u0184\u0185\7N\2\2\u0185\u0189\5,\27\2\u0186\u0187\7\13\2\2\u0187\u0189"+
+    "\t\f\2\2\u0188\u0183\3\2\2\2\u0188\u0186\3\2\2\2\u0189)\3\2\2\2\u018a"+
+    "\u018b\7\7\2\2\u018b\u018c\5\36\20\2\u018c\u018d\7\b\2\2\u018d+\3\2\2"+
+    "\2\u018e\u0197\7\t\2\2\u018f\u0194\5.\30\2\u0190\u0191\7\f\2\2\u0191\u0193"+
+    "\5.\30\2\u0192\u0190\3\2\2\2\u0193\u0196\3\2\2\2\u0194\u0192\3\2\2\2\u0194"+
+    "\u0195\3\2\2\2\u0195\u0198\3\2\2\2\u0196\u0194\3\2\2\2\u0197\u018f\3\2"+
+    "\2\2\u0197\u0198\3\2\2\2\u0198\u0199\3\2\2\2\u0199\u019a\7\n\2\2\u019a"+
+    "-\3\2\2\2\u019b\u019f\5\36\20\2\u019c\u019f\5\60\31\2\u019d\u019f\5\64"+
+    "\33\2\u019e\u019b\3\2\2\2\u019e\u019c\3\2\2\2\u019e\u019d\3\2\2\2\u019f"+
+    "/\3\2\2\2\u01a0\u01ae\5\62\32\2\u01a1\u01aa\7\t\2\2\u01a2\u01a7\5\62\32"+
+    "\2\u01a3\u01a4\7\f\2\2\u01a4\u01a6\5\62\32\2\u01a5\u01a3\3\2\2\2\u01a6"+
+    "\u01a9\3\2\2\2\u01a7\u01a5\3\2\2\2\u01a7\u01a8\3\2\2\2\u01a8\u01ab\3\2"+
+    "\2\2\u01a9\u01a7\3\2\2\2\u01aa\u01a2\3\2\2\2\u01aa\u01ab\3\2\2\2\u01ab"+
+    "\u01ac\3\2\2\2\u01ac\u01ae\7\n\2\2\u01ad\u01a0\3\2\2\2\u01ad\u01a1\3\2"+
+    "\2\2\u01ae\u01af\3\2\2\2\u01af\u01b0\7\64\2\2\u01b0\u01b1\5\f\7\2\u01b1"+
+    "\61\3\2\2\2\u01b2\u01b4\5\26\f\2\u01b3\u01b2\3\2\2\2\u01b3\u01b4\3\2\2"+
+    "\2\u01b4\u01b5\3\2\2\2\u01b5\u01b6\7L\2\2\u01b6\63\3\2\2\2\u01b7\u01b8"+
+    "\7K\2\2\u01b8\u01b9\7\63\2\2\u01b9\u01ba\t\r\2\2\u01ba\65\3\2\2\2(9?R"+
+    "Uaivz~\u0083\u009e\u00a7\u00ab\u00b1\u00ba\u00c4\u00cc\u00d2\u00e6\u0121"+
+    "\u0123\u0143\u0149\u0151\u015c\u0162\u0165\u0167\u017b\u0181\u0188\u0194"+
+    "\u0197\u019e\u01a7\u01aa\u01ad\u01b3";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
index 08358384fca..d98ba899bfa 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
@@ -165,13 +165,6 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitDecltype(PainlessParser.DecltypeContext ctx) { return visitChildren(ctx); }
-  /**
-   * {@inheritDoc}
-   *
-   * <p>The default implementation returns the result of calling
-   * {@link #visitChildren} on {@code ctx}.</p>
-   */
-  @Override public T visitFuncref(PainlessParser.FuncrefContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
@@ -403,4 +396,25 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitArgument(PainlessParser.ArgumentContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitLambda(PainlessParser.LambdaContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitLamtype(PainlessParser.LamtypeContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitFuncref(PainlessParser.FuncrefContext ctx) { return visitChildren(ctx); }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
index 5c0adb5076a..514e22ccdc3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
@@ -154,12 +154,6 @@ interface PainlessParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitDecltype(PainlessParser.DecltypeContext ctx);
-  /**
-   * Visit a parse tree produced by {@link PainlessParser#funcref}.
-   * @param ctx the parse tree
-   * @return the visitor result
-   */
-  T visitFuncref(PainlessParser.FuncrefContext ctx);
   /**
    * Visit a parse tree produced by {@link PainlessParser#declvar}.
    * @param ctx the parse tree
@@ -385,4 +379,22 @@ interface PainlessParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitArgument(PainlessParser.ArgumentContext ctx);
+  /**
+   * Visit a parse tree produced by {@link PainlessParser#lambda}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitLambda(PainlessParser.LambdaContext ctx);
+  /**
+   * Visit a parse tree produced by {@link PainlessParser#lamtype}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitLamtype(PainlessParser.LamtypeContext ctx);
+  /**
+   * Visit a parse tree produced by {@link PainlessParser#funcref}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitFuncref(PainlessParser.FuncrefContext ctx);
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index c1fe02f9388..adeea30f339 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -69,6 +69,8 @@ import org.elasticsearch.painless.antlr.PainlessParser.FuncrefContext;
 import org.elasticsearch.painless.antlr.PainlessParser.FunctionContext;
 import org.elasticsearch.painless.antlr.PainlessParser.IfContext;
 import org.elasticsearch.painless.antlr.PainlessParser.InitializerContext;
+import org.elasticsearch.painless.antlr.PainlessParser.LambdaContext;
+import org.elasticsearch.painless.antlr.PainlessParser.LamtypeContext;
 import org.elasticsearch.painless.antlr.PainlessParser.NewarrayContext;
 import org.elasticsearch.painless.antlr.PainlessParser.NewobjectContext;
 import org.elasticsearch.painless.antlr.PainlessParser.NullContext;
@@ -106,6 +108,7 @@ import org.elasticsearch.painless.node.EConditional;
 import org.elasticsearch.painless.node.EDecimal;
 import org.elasticsearch.painless.node.EExplicit;
 import org.elasticsearch.painless.node.EFunctionRef;
+import org.elasticsearch.painless.node.ELambda;
 import org.elasticsearch.painless.node.ENull;
 import org.elasticsearch.painless.node.ENumeric;
 import org.elasticsearch.painless.node.EUnary;
@@ -445,19 +448,6 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
     }
 
-    @Override
-    public Object visitFuncref(FuncrefContext ctx) {
-        final String methodText;
-        if (ctx.ID() != null) {
-            methodText = ctx.ID().getText();
-        } else if (ctx.NEW() != null ){
-            methodText = ctx.NEW().getText();
-        } else {
-            throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
-        }
-        return new EFunctionRef(location(ctx), ctx.TYPE().getText(), methodText);
-    }
-
     @Override
     public Object visitDeclvar(DeclvarContext ctx) {
         throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
@@ -918,10 +908,55 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
     public Object visitArgument(ArgumentContext ctx) {
         if (ctx.expression() != null) {
             return visitExpression(ctx.expression());
+        } else if (ctx.lambda() != null) {
+            return visit(ctx.lambda());
         } else if (ctx.funcref() != null) {
             return visit(ctx.funcref());
         } else {
             throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
         }
     }
+
+    @Override
+    public Object visitLambda(LambdaContext ctx) {
+        reserved.push(new FunctionReserved());
+
+        List<String> paramTypes = new ArrayList<>();
+        List<String> paramNames = new ArrayList<>();
+        List<AStatement> statements = new ArrayList<>();
+
+        for (LamtypeContext lamtype : ctx.lamtype()) {
+            if (lamtype.decltype() == null) {
+                paramTypes.add(null);
+            } else {
+                paramTypes.add(lamtype.decltype().getText());
+            }
+
+            paramNames.add(lamtype.ID().getText());
+        }
+
+        for (StatementContext statement : ctx.block().statement()) {
+            statements.add((AStatement)visit(statement));
+        }
+
+        return new ELambda((FunctionReserved)reserved.pop(), location(ctx), paramTypes, paramNames, statements);
+    }
+
+    @Override
+    public Object visitLamtype(LamtypeContext ctx) {
+        throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
+    }
+
+    @Override
+    public Object visitFuncref(FuncrefContext ctx) {
+        final String methodText;
+        if (ctx.ID() != null) {
+            methodText = ctx.ID().getText();
+        } else if (ctx.NEW() != null ){
+            methodText = ctx.NEW().getText();
+        } else {
+            throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
+        }
+        return new EFunctionRef(location(ctx), ctx.TYPE().getText(), methodText);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
new file mode 100644
index 00000000000..3fac3cdfec8
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Locals.FunctionReserved;
+import org.elasticsearch.painless.Location;
+import org.elasticsearch.painless.MethodWriter;
+
+import java.util.Collections;
+import java.util.List;
+
+public class ELambda extends AExpression {
+    final FunctionReserved reserved;
+    final List<String> paramTypeStrs;
+    final List<String> paramNameStrs;
+    final List<AStatement> statements;
+
+    Locals locals = null;
+
+    public ELambda(FunctionReserved reserved, Location location,
+                   List<String> paramTypes, List<String> paramNames, List<AStatement> statements) {
+        super(location);
+
+        this.reserved = reserved;
+        this.paramTypeStrs = Collections.unmodifiableList(paramTypes);
+        this.paramNameStrs = Collections.unmodifiableList(paramNames);
+        this.statements = Collections.unmodifiableList(statements);
+    }
+
+    @Override
+    void analyze(Locals locals) {
+        throw createError(new UnsupportedOperationException("Lambda functions are not supported."));
+    }
+
+    @Override
+    void write(MethodWriter writer) {
+        throw createError(new IllegalStateException("Illegal tree structure."));
+    }
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
index c392e86806e..447b2139779 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
@@ -134,7 +134,7 @@ public class BasicStatementTests extends ScriptTestCase {
             " String cat = ''; int total = 0;" +
             " for (Map.Entry e : m.entrySet()) { cat += e.getKey(); total += e.getValue(); } return cat + total"));
     }
-    
+
     public void testIterableForEachStatementDef() {
         assertEquals(6, exec("def l = new ArrayList(); l.add(1); l.add(2); l.add(3); int total = 0;" +
             " for (int x : l) total += x; return total"));
@@ -153,7 +153,7 @@ public class BasicStatementTests extends ScriptTestCase {
         assertEquals(6, exec("int[][] i = new int[3][1]; i[0][0] = 1; i[1][0] = 2; i[2][0] = 3; int total = 0;" +
             " for (int[] j : i) total += j[0]; return total"));
     }
-    
+
     public void testArrayForEachStatementDef() {
         assertEquals(6, exec("def a = new int[3]; a[0] = 1; a[1] = 2; a[2] = 3; int total = 0;" +
             " for (int x : a) total += x; return total"));
@@ -214,4 +214,11 @@ public class BasicStatementTests extends ScriptTestCase {
         assertEquals(5, ((short[])exec("short[] s = new short[3]; s[1] = 5; return s;"))[1]);
         assertEquals(10, ((Map)exec("Map s = new HashMap(); s.put(\"x\", 10); return s;")).get("x"));
     }
+
+    public void testLambdas() {
+        Exception exception = expectThrows(Exception.class, () -> {
+            exec("Math.max(2, p -> {p.doSomething();})");
+        });
+        assertTrue(exception.getCause().getMessage().contains("Lambda functions are not supported."));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/antlr/ParserTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/antlr/ParserTests.java
index 1a75c970fc4..d1852ada27b 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/antlr/ParserTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/antlr/ParserTests.java
@@ -78,4 +78,17 @@ public class ParserTests extends ScriptTestCase {
         exception = expectThrows(RuntimeException.class, () -> buildAntlrTree("((Map)x.-x)"));
         assertTrue(exception.getMessage().contains("unexpected character"));
     }
+
+    public void testLambdaSyntax() {
+        buildAntlrTree("call(p -> {p.doSomething();});");
+        buildAntlrTree("call(int p -> {p.doSomething();});");
+        buildAntlrTree("call((p, u, v) -> {p.doSomething(); blah = 1;});");
+        buildAntlrTree("call(1, (p, u, v) -> {p.doSomething(); blah = 1;}, 3);");
+        buildAntlrTree("call((p, u, v) -> {p.doSomething(); blah = 1;});");
+        buildAntlrTree("call(x, y, z, (int p, int u, int v) -> {p.doSomething(); blah = 1;});");
+        buildAntlrTree("call(x, y, z, (long p, List u, String v) -> {p.doSomething(); blah = 1;});");
+        buildAntlrTree("call(x, y, z, (int p, u, int v) -> {p.doSomething(); blah = 1;});");
+        buildAntlrTree("call(x, (int p, u, int v) -> {p.doSomething(); blah = 1;}, z," +
+            " (int p, u, int v) -> {p.doSomething(); blah = 1;}, 'test');");
+    }
 }

From 4a265d027952b2cb0e46898d00fca357d8089d94 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Sat, 11 Jun 2016 10:15:14 -0400
Subject: [PATCH 0787/1311] Painless: Add } as a delimiter. Kindof.

Add `}` is statement delimiter but only in places where it is
otherwise a valid part of the syntax, specificall the end of a block.
We do this by matching but not consuming it. Antlr 4 doesn't have
syntax for this so we have to kind of hack it together by actually
matching the `}` and then seeking backwards in the token stream to
"unmatch" it. This looks reasonably efficient. Not perfect, but way
better than the alternatives.

I tried and rejected a few options:
1. Actually consuming the `}` and piping a boolean all through the
grammar from the last statement in a block to the delimiter. This
ended up being a rather large change and made the grammar way more
complicated.
2. Adding a semantic predicate to delimiter that just does the
lookahead. This doesn't work out well because it doesn't work (I
never figured out why) and because it generates an *amazing*
`adaptivePredict` which makes a super huge DFA. It looks super
inefficient.

Closes #18821
---
 .../src/main/antlr/PainlessParser.g4          |  11 +
 .../painless/antlr/PainlessParser.java        | 692 +++++++++---------
 .../painless/BasicStatementTests.java         |   9 +
 .../painless/WhenThingsGoWrongTests.java      |  14 +
 4 files changed, 390 insertions(+), 336 deletions(-)

diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index 8c6baf2b197..f4ba05d2367 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -92,6 +92,17 @@ trap
 delimiter
     : SEMICOLON
     | EOF
+    // RBRACK is a delimiter but we don't consume it because it is only valid
+    // in places where RBRACK can follow the statement. It is simpler to not
+    // consume it here then it is to consume it here. Unfortunately, they
+    // obvious syntax to do this `| { _input.LA(1) == RBRACK }?` generates an
+    // amazingly intense `adaptivePredict` call that doesn't actually work
+    // and builds a serious DFA. Huge. So instead we use standard ANTLR syntax
+    // to consume the token and then undo the consumption. This looks hairy but
+    // it is better than the alternatives.
+    |   { int mark = _input.mark(); int index = _input.index(); }
+            RBRACK
+        { _input.seek(index); _input.release(mark); }
     ;
 
 // Note we return the boolean s.  This is returned as true
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
index 089dea06107..73101d22bc4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
@@ -1283,6 +1283,7 @@ class PainlessParser extends Parser {
   public static class DelimiterContext extends ParserRuleContext {
     public TerminalNode SEMICOLON() { return getToken(PainlessParser.SEMICOLON, 0); }
     public TerminalNode EOF() { return getToken(PainlessParser.EOF, 0); }
+    public TerminalNode RBRACK() { return getToken(PainlessParser.RBRACK, 0); }
     public DelimiterContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
     }
@@ -1297,17 +1298,34 @@ class PainlessParser extends Parser {
   public final DelimiterContext delimiter() throws RecognitionException {
     DelimiterContext _localctx = new DelimiterContext(_ctx, getState());
     enterRule(_localctx, 26, RULE_delimiter);
-    int _la;
     try {
-      enterOuterAlt(_localctx, 1);
-      {
-      setState(217);
-      _la = _input.LA(1);
-      if ( !(_la==EOF || _la==SEMICOLON) ) {
-      _errHandler.recoverInline(this);
-      } else {
-        consume();
-      }
+      setState(222);
+      switch (_input.LA(1)) {
+      case SEMICOLON:
+        enterOuterAlt(_localctx, 1);
+        {
+        setState(217);
+        match(SEMICOLON);
+        }
+        break;
+      case EOF:
+        enterOuterAlt(_localctx, 2);
+        {
+        setState(218);
+        match(EOF);
+        }
+        break;
+      case RBRACK:
+        enterOuterAlt(_localctx, 3);
+        {
+         int mark = _input.mark(); int index = _input.index(); 
+        setState(220);
+        match(RBRACK);
+         _input.seek(index); _input.release(mark); 
+        }
+        break;
+      default:
+        throw new NoViableAltException(this);
       }
     }
     catch (RecognitionException re) {
@@ -1470,24 +1488,24 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(228);
-      switch ( getInterpreter().adaptivePredict(_input,18,_ctx) ) {
+      setState(233);
+      switch ( getInterpreter().adaptivePredict(_input,19,_ctx) ) {
       case 1:
         {
         _localctx = new AssignmentContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(220);
+        setState(225);
         chain(true);
-        setState(221);
+        setState(226);
         _la = _input.LA(1);
         if ( !(((((_la - 53)) & ~0x3f) == 0 && ((1L << (_la - 53)) & ((1L << (ASSIGN - 53)) | (1L << (AADD - 53)) | (1L << (ASUB - 53)) | (1L << (AMUL - 53)) | (1L << (ADIV - 53)) | (1L << (AREM - 53)) | (1L << (AAND - 53)) | (1L << (AXOR - 53)) | (1L << (AOR - 53)) | (1L << (ALSH - 53)) | (1L << (ARSH - 53)) | (1L << (AUSH - 53)))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(222);
+        setState(227);
         expression(1);
          ((AssignmentContext)_localctx).s =  false; 
         }
@@ -1497,37 +1515,37 @@ class PainlessParser extends Parser {
         _localctx = new SingleContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(225);
+        setState(230);
         ((SingleContext)_localctx).u = unary(false);
          ((SingleContext)_localctx).s =  ((SingleContext)_localctx).u.s; 
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(289);
+      setState(294);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(287);
-          switch ( getInterpreter().adaptivePredict(_input,19,_ctx) ) {
+          setState(292);
+          switch ( getInterpreter().adaptivePredict(_input,20,_ctx) ) {
           case 1:
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(230);
+            setState(235);
             if (!(precpred(_ctx, 12))) throw new FailedPredicateException(this, "precpred(_ctx, 12)");
-            setState(231);
+            setState(236);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << MUL) | (1L << DIV) | (1L << REM))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(232);
+            setState(237);
             expression(13);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1536,16 +1554,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(235);
+            setState(240);
             if (!(precpred(_ctx, 11))) throw new FailedPredicateException(this, "precpred(_ctx, 11)");
-            setState(236);
+            setState(241);
             _la = _input.LA(1);
             if ( !(_la==ADD || _la==SUB) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(237);
+            setState(242);
             expression(12);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1554,16 +1572,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(240);
+            setState(245);
             if (!(precpred(_ctx, 10))) throw new FailedPredicateException(this, "precpred(_ctx, 10)");
-            setState(241);
+            setState(246);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LSH) | (1L << RSH) | (1L << USH))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(242);
+            setState(247);
             expression(11);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1572,16 +1590,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(245);
+            setState(250);
             if (!(precpred(_ctx, 9))) throw new FailedPredicateException(this, "precpred(_ctx, 9)");
-            setState(246);
+            setState(251);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LT) | (1L << LTE) | (1L << GT) | (1L << GTE))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(247);
+            setState(252);
             expression(10);
              ((CompContext)_localctx).s =  false; 
             }
@@ -1590,16 +1608,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(250);
+            setState(255);
             if (!(precpred(_ctx, 8))) throw new FailedPredicateException(this, "precpred(_ctx, 8)");
-            setState(251);
+            setState(256);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << EQ) | (1L << EQR) | (1L << NE) | (1L << NER))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(252);
+            setState(257);
             expression(9);
              ((CompContext)_localctx).s =  false; 
             }
@@ -1608,11 +1626,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(255);
+            setState(260);
             if (!(precpred(_ctx, 7))) throw new FailedPredicateException(this, "precpred(_ctx, 7)");
-            setState(256);
+            setState(261);
             match(BWAND);
-            setState(257);
+            setState(262);
             expression(8);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1621,11 +1639,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(260);
+            setState(265);
             if (!(precpred(_ctx, 6))) throw new FailedPredicateException(this, "precpred(_ctx, 6)");
-            setState(261);
+            setState(266);
             match(XOR);
-            setState(262);
+            setState(267);
             expression(7);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1634,11 +1652,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(265);
+            setState(270);
             if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)");
-            setState(266);
+            setState(271);
             match(BWOR);
-            setState(267);
+            setState(272);
             expression(6);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1647,11 +1665,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(270);
+            setState(275);
             if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)");
-            setState(271);
+            setState(276);
             match(BOOLAND);
-            setState(272);
+            setState(277);
             expression(5);
              ((BoolContext)_localctx).s =  false; 
             }
@@ -1660,11 +1678,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(275);
+            setState(280);
             if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)");
-            setState(276);
+            setState(281);
             match(BOOLOR);
-            setState(277);
+            setState(282);
             expression(4);
              ((BoolContext)_localctx).s =  false; 
             }
@@ -1673,15 +1691,15 @@ class PainlessParser extends Parser {
             {
             _localctx = new ConditionalContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(280);
+            setState(285);
             if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
-            setState(281);
+            setState(286);
             match(COND);
-            setState(282);
+            setState(287);
             ((ConditionalContext)_localctx).e0 = expression(0);
-            setState(283);
+            setState(288);
             match(COLON);
-            setState(284);
+            setState(289);
             ((ConditionalContext)_localctx).e1 = expression(2);
              ((ConditionalContext)_localctx).s =  ((ConditionalContext)_localctx).e0.s && ((ConditionalContext)_localctx).e1.s; 
             }
@@ -1689,9 +1707,9 @@ class PainlessParser extends Parser {
           }
           } 
         }
-        setState(291);
+        setState(296);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
       }
       }
     }
@@ -1836,22 +1854,22 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 30, RULE_unary);
     int _la;
     try {
-      setState(321);
-      switch ( getInterpreter().adaptivePredict(_input,21,_ctx) ) {
+      setState(326);
+      switch ( getInterpreter().adaptivePredict(_input,22,_ctx) ) {
       case 1:
         _localctx = new PreContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(292);
+        setState(297);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(293);
+        setState(298);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(294);
+        setState(299);
         chain(true);
         }
         break;
@@ -1859,11 +1877,11 @@ class PainlessParser extends Parser {
         _localctx = new PostContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(295);
+        setState(300);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(296);
+        setState(301);
         chain(true);
-        setState(297);
+        setState(302);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
@@ -1876,9 +1894,9 @@ class PainlessParser extends Parser {
         _localctx = new ReadContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(299);
+        setState(304);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(300);
+        setState(305);
         chain(false);
         }
         break;
@@ -1886,9 +1904,9 @@ class PainlessParser extends Parser {
         _localctx = new NumericContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(301);
+        setState(306);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(302);
+        setState(307);
         _la = _input.LA(1);
         if ( !(((((_la - 65)) & ~0x3f) == 0 && ((1L << (_la - 65)) & ((1L << (OCTAL - 65)) | (1L << (HEX - 65)) | (1L << (INTEGER - 65)) | (1L << (DECIMAL - 65)))) != 0)) ) {
         _errHandler.recoverInline(this);
@@ -1902,9 +1920,9 @@ class PainlessParser extends Parser {
         _localctx = new TrueContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(304);
+        setState(309);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(305);
+        setState(310);
         match(TRUE);
          ((TrueContext)_localctx).s =  false; 
         }
@@ -1913,9 +1931,9 @@ class PainlessParser extends Parser {
         _localctx = new FalseContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(307);
+        setState(312);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(308);
+        setState(313);
         match(FALSE);
          ((FalseContext)_localctx).s =  false; 
         }
@@ -1924,9 +1942,9 @@ class PainlessParser extends Parser {
         _localctx = new NullContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(310);
+        setState(315);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(311);
+        setState(316);
         match(NULL);
          ((NullContext)_localctx).s =  false; 
         }
@@ -1935,16 +1953,16 @@ class PainlessParser extends Parser {
         _localctx = new OperatorContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(313);
+        setState(318);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(314);
+        setState(319);
         _la = _input.LA(1);
         if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(315);
+        setState(320);
         unary(false);
         }
         break;
@@ -1952,13 +1970,13 @@ class PainlessParser extends Parser {
         _localctx = new CastContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(316);
+        setState(321);
         match(LP);
-        setState(317);
+        setState(322);
         decltype();
-        setState(318);
+        setState(323);
         match(RP);
-        setState(319);
+        setState(324);
         unary(_localctx.c);
         }
         break;
@@ -2067,29 +2085,29 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 32, RULE_chain);
     try {
       int _alt;
-      setState(357);
-      switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) {
+      setState(362);
+      switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) {
       case 1:
         _localctx = new DynamicContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(323);
+        setState(328);
         ((DynamicContext)_localctx).p = primary(_localctx.c);
-        setState(327);
+        setState(332);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(324);
+            setState(329);
             secondary(((DynamicContext)_localctx).p.s);
             }
             } 
           }
-          setState(329);
+          setState(334);
           _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
+          _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
         }
         }
         break;
@@ -2097,25 +2115,25 @@ class PainlessParser extends Parser {
         _localctx = new StaticContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(330);
-        decltype();
-        setState(331);
-        dot();
         setState(335);
+        decltype();
+        setState(336);
+        dot();
+        setState(340);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,24,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(332);
+            setState(337);
             secondary(true);
             }
             } 
           }
-          setState(337);
+          setState(342);
           _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
+          _alt = getInterpreter().adaptivePredict(_input,24,_ctx);
         }
         }
         break;
@@ -2123,11 +2141,11 @@ class PainlessParser extends Parser {
         _localctx = new NewarrayContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(338);
+        setState(343);
         match(NEW);
-        setState(339);
+        setState(344);
         match(TYPE);
-        setState(344); 
+        setState(349); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -2135,11 +2153,11 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(340);
+            setState(345);
             match(LBRACE);
-            setState(341);
+            setState(346);
             expression(0);
-            setState(342);
+            setState(347);
             match(RBRACE);
             }
             }
@@ -2147,31 +2165,31 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(346); 
-          _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,24,_ctx);
-        } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
-        setState(355);
-        switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) {
-        case 1:
-          {
-          setState(348);
-          dot();
-          setState(352);
+          setState(351); 
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,25,_ctx);
+        } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
+        setState(360);
+        switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) {
+        case 1:
+          {
+          setState(353);
+          dot();
+          setState(357);
+          _errHandler.sync(this);
+          _alt = getInterpreter().adaptivePredict(_input,26,_ctx);
           while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
             if ( _alt==1 ) {
               {
               {
-              setState(349);
+              setState(354);
               secondary(true);
               }
               } 
             }
-            setState(354);
+            setState(359);
             _errHandler.sync(this);
-            _alt = getInterpreter().adaptivePredict(_input,25,_ctx);
+            _alt = getInterpreter().adaptivePredict(_input,26,_ctx);
           }
           }
           break;
@@ -2283,19 +2301,19 @@ class PainlessParser extends Parser {
     PrimaryContext _localctx = new PrimaryContext(_ctx, getState(), c);
     enterRule(_localctx, 34, RULE_primary);
     try {
-      setState(377);
-      switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) {
+      setState(382);
+      switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
       case 1:
         _localctx = new ExprprecContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(359);
+        setState(364);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(360);
+        setState(365);
         match(LP);
-        setState(361);
+        setState(366);
         ((ExprprecContext)_localctx).e = expression(0);
-        setState(362);
+        setState(367);
         match(RP);
          ((ExprprecContext)_localctx).s =  ((ExprprecContext)_localctx).e.s; 
         }
@@ -2304,13 +2322,13 @@ class PainlessParser extends Parser {
         _localctx = new ChainprecContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(365);
+        setState(370);
         if (!( _localctx.c )) throw new FailedPredicateException(this, " $c ");
-        setState(366);
+        setState(371);
         match(LP);
-        setState(367);
+        setState(372);
         unary(true);
-        setState(368);
+        setState(373);
         match(RP);
         }
         break;
@@ -2318,7 +2336,7 @@ class PainlessParser extends Parser {
         _localctx = new StringContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(370);
+        setState(375);
         match(STRING);
         }
         break;
@@ -2326,7 +2344,7 @@ class PainlessParser extends Parser {
         _localctx = new VariableContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(371);
+        setState(376);
         match(ID);
         }
         break;
@@ -2334,9 +2352,9 @@ class PainlessParser extends Parser {
         _localctx = new CalllocalContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(372);
+        setState(377);
         match(ID);
-        setState(373);
+        setState(378);
         arguments();
         }
         break;
@@ -2344,11 +2362,11 @@ class PainlessParser extends Parser {
         _localctx = new NewobjectContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(374);
+        setState(379);
         match(NEW);
-        setState(375);
+        setState(380);
         match(TYPE);
-        setState(376);
+        setState(381);
         arguments();
         }
         break;
@@ -2390,23 +2408,23 @@ class PainlessParser extends Parser {
     SecondaryContext _localctx = new SecondaryContext(_ctx, getState(), s);
     enterRule(_localctx, 36, RULE_secondary);
     try {
-      setState(383);
-      switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
+      setState(388);
+      switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(379);
+        setState(384);
         if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
-        setState(380);
+        setState(385);
         dot();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(381);
+        setState(386);
         if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
-        setState(382);
+        setState(387);
         brace();
         }
         break;
@@ -2464,17 +2482,17 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 38, RULE_dot);
     int _la;
     try {
-      setState(390);
-      switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
+      setState(395);
+      switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) {
       case 1:
         _localctx = new CallinvokeContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(385);
+        setState(390);
         match(DOT);
-        setState(386);
+        setState(391);
         match(DOTID);
-        setState(387);
+        setState(392);
         arguments();
         }
         break;
@@ -2482,9 +2500,9 @@ class PainlessParser extends Parser {
         _localctx = new FieldaccessContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(388);
+        setState(393);
         match(DOT);
-        setState(389);
+        setState(394);
         _la = _input.LA(1);
         if ( !(_la==DOTINTEGER || _la==DOTID) ) {
         _errHandler.recoverInline(this);
@@ -2538,11 +2556,11 @@ class PainlessParser extends Parser {
       _localctx = new BraceaccessContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(392);
+      setState(397);
       match(LBRACE);
-      setState(393);
+      setState(398);
       expression(0);
-      setState(394);
+      setState(399);
       match(RBRACE);
       }
     }
@@ -2589,34 +2607,34 @@ class PainlessParser extends Parser {
       enterOuterAlt(_localctx, 1);
       {
       {
-      setState(396);
+      setState(401);
       match(LP);
-      setState(405);
-      switch ( getInterpreter().adaptivePredict(_input,32,_ctx) ) {
+      setState(410);
+      switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) {
       case 1:
         {
-        setState(397);
-        argument();
         setState(402);
+        argument();
+        setState(407);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(398);
+          setState(403);
           match(COMMA);
-          setState(399);
+          setState(404);
           argument();
           }
           }
-          setState(404);
+          setState(409);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
         break;
       }
-      setState(407);
+      setState(412);
       match(RP);
       }
       }
@@ -2657,26 +2675,26 @@ class PainlessParser extends Parser {
     ArgumentContext _localctx = new ArgumentContext(_ctx, getState());
     enterRule(_localctx, 44, RULE_argument);
     try {
-      setState(412);
-      switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) {
+      setState(417);
+      switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(409);
+        setState(414);
         expression(0);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(410);
+        setState(415);
         lambda();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(411);
+        setState(416);
         funcref();
         }
         break;
@@ -2728,54 +2746,54 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(427);
+      setState(432);
       switch (_input.LA(1)) {
       case TYPE:
       case ID:
         {
-        setState(414);
+        setState(419);
         lamtype();
         }
         break;
       case LP:
         {
-        setState(415);
+        setState(420);
         match(LP);
-        setState(424);
+        setState(429);
         _la = _input.LA(1);
         if (_la==TYPE || _la==ID) {
           {
-          setState(416);
-          lamtype();
           setState(421);
+          lamtype();
+          setState(426);
           _errHandler.sync(this);
           _la = _input.LA(1);
           while (_la==COMMA) {
             {
             {
-            setState(417);
+            setState(422);
             match(COMMA);
-            setState(418);
+            setState(423);
             lamtype();
             }
             }
-            setState(423);
+            setState(428);
             _errHandler.sync(this);
             _la = _input.LA(1);
           }
           }
         }
 
-        setState(426);
+        setState(431);
         match(RP);
         }
         break;
       default:
         throw new NoViableAltException(this);
       }
-      setState(429);
+      setState(434);
       match(ARROW);
-      setState(430);
+      setState(435);
       block();
       }
     }
@@ -2813,16 +2831,16 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(433);
+      setState(438);
       _la = _input.LA(1);
       if (_la==TYPE) {
         {
-        setState(432);
+        setState(437);
         decltype();
         }
       }
 
-      setState(435);
+      setState(440);
       match(ID);
       }
     }
@@ -2861,16 +2879,16 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 50, RULE_funcref);
     int _la;
     try {
-      setState(443);
+      setState(448);
       switch (_input.LA(1)) {
       case TYPE:
         enterOuterAlt(_localctx, 1);
         {
-        setState(437);
+        setState(442);
         match(TYPE);
-        setState(438);
+        setState(443);
         match(REF);
-        setState(439);
+        setState(444);
         _la = _input.LA(1);
         if ( !(_la==NEW || _la==ID) ) {
         _errHandler.recoverInline(this);
@@ -2882,11 +2900,11 @@ class PainlessParser extends Parser {
       case ID:
         enterOuterAlt(_localctx, 2);
         {
-        setState(440);
+        setState(445);
         match(ID);
-        setState(441);
+        setState(446);
         match(REF);
-        setState(442);
+        setState(447);
         match(ID);
         }
         break;
@@ -2995,7 +3013,7 @@ class PainlessParser extends Parser {
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3N\u01c0\4\2\t\2\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3N\u01c5\4\2\t\2\4"+
     "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
     "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
     "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+
@@ -3010,158 +3028,160 @@ class PainlessParser extends Parser {
     "\7\u00b0\n\7\f\7\16\7\u00b3\13\7\3\7\3\7\3\b\3\b\3\t\3\t\5\t\u00bb\n\t"+
     "\3\n\3\n\3\13\3\13\3\13\3\13\7\13\u00c3\n\13\f\13\16\13\u00c6\13\13\3"+
     "\f\3\f\3\f\7\f\u00cb\n\f\f\f\16\f\u00ce\13\f\3\r\3\r\3\r\5\r\u00d3\n\r"+
-    "\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\17\3\17\3\20\3\20\3\20\3\20\3\20"+
-    "\3\20\3\20\3\20\3\20\5\20\u00e7\n\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
+    "\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\17\3\17\3\17\3\17\3\17\5\17\u00e1"+
+    "\n\17\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\5\20\u00ec\n\20\3\20"+
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
-    "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\7\20\u0122\n\20\f\20\16\20\u0125"+
-    "\13\20\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
+    "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
+    "\7\20\u0127\n\20\f\20\16\20\u012a\13\20\3\21\3\21\3\21\3\21\3\21\3\21"+
     "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
-    "\3\21\3\21\5\21\u0144\n\21\3\22\3\22\7\22\u0148\n\22\f\22\16\22\u014b"+
-    "\13\22\3\22\3\22\3\22\7\22\u0150\n\22\f\22\16\22\u0153\13\22\3\22\3\22"+
-    "\3\22\3\22\3\22\3\22\6\22\u015b\n\22\r\22\16\22\u015c\3\22\3\22\7\22\u0161"+
-    "\n\22\f\22\16\22\u0164\13\22\5\22\u0166\n\22\5\22\u0168\n\22\3\23\3\23"+
-    "\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23"+
-    "\3\23\3\23\5\23\u017c\n\23\3\24\3\24\3\24\3\24\5\24\u0182\n\24\3\25\3"+
-    "\25\3\25\3\25\3\25\5\25\u0189\n\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27"+
-    "\3\27\7\27\u0193\n\27\f\27\16\27\u0196\13\27\5\27\u0198\n\27\3\27\3\27"+
-    "\3\30\3\30\3\30\5\30\u019f\n\30\3\31\3\31\3\31\3\31\3\31\7\31\u01a6\n"+
-    "\31\f\31\16\31\u01a9\13\31\5\31\u01ab\n\31\3\31\5\31\u01ae\n\31\3\31\3"+
-    "\31\3\31\3\32\5\32\u01b4\n\32\3\32\3\32\3\33\3\33\3\33\3\33\3\33\3\33"+
-    "\5\33\u01be\n\33\3\33\2\3\36\34\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36"+
-    " \"$&(*,.\60\62\64\2\16\3\3\r\r\3\2\67B\3\2\34\36\3\2\37 \3\2!#\3\2$\'"+
-    "\3\2(+\3\2\65\66\3\2CF\4\2\32\33\37 \3\2MN\4\2\26\26LL\u01ec\29\3\2\2"+
-    "\2\4D\3\2\2\2\6I\3\2\2\2\b\u00a7\3\2\2\2\n\u00ab\3\2\2\2\f\u00ad\3\2\2"+
-    "\2\16\u00b6\3\2\2\2\20\u00ba\3\2\2\2\22\u00bc\3\2\2\2\24\u00be\3\2\2\2"+
-    "\26\u00c7\3\2\2\2\30\u00cf\3\2\2\2\32\u00d4\3\2\2\2\34\u00db\3\2\2\2\36"+
-    "\u00e6\3\2\2\2 \u0143\3\2\2\2\"\u0167\3\2\2\2$\u017b\3\2\2\2&\u0181\3"+
-    "\2\2\2(\u0188\3\2\2\2*\u018a\3\2\2\2,\u018e\3\2\2\2.\u019e\3\2\2\2\60"+
-    "\u01ad\3\2\2\2\62\u01b3\3\2\2\2\64\u01bd\3\2\2\2\668\5\4\3\2\67\66\3\2"+
-    "\2\28;\3\2\2\29\67\3\2\2\29:\3\2\2\2:?\3\2\2\2;9\3\2\2\2<>\5\b\5\2=<\3"+
-    "\2\2\2>A\3\2\2\2?=\3\2\2\2?@\3\2\2\2@B\3\2\2\2A?\3\2\2\2BC\7\2\2\3C\3"+
-    "\3\2\2\2DE\5\26\f\2EF\7L\2\2FG\5\6\4\2GH\5\f\7\2H\5\3\2\2\2IU\7\t\2\2"+
-    "JK\5\26\f\2KR\7L\2\2LM\7\f\2\2MN\5\26\f\2NO\7L\2\2OQ\3\2\2\2PL\3\2\2\2"+
-    "QT\3\2\2\2RP\3\2\2\2RS\3\2\2\2SV\3\2\2\2TR\3\2\2\2UJ\3\2\2\2UV\3\2\2\2"+
-    "VW\3\2\2\2WX\7\n\2\2X\7\3\2\2\2YZ\7\16\2\2Z[\7\t\2\2[\\\5\36\20\2\\]\7"+
-    "\n\2\2]a\5\n\6\2^_\7\17\2\2_b\5\n\6\2`b\6\5\2\2a^\3\2\2\2a`\3\2\2\2b\u00a8"+
-    "\3\2\2\2cd\7\20\2\2de\7\t\2\2ef\5\36\20\2fi\7\n\2\2gj\5\n\6\2hj\5\16\b"+
-    "\2ig\3\2\2\2ih\3\2\2\2j\u00a8\3\2\2\2kl\7\21\2\2lm\5\f\7\2mn\7\20\2\2"+
-    "no\7\t\2\2op\5\36\20\2pq\7\n\2\2qr\5\34\17\2r\u00a8\3\2\2\2st\7\22\2\2"+
-    "tv\7\t\2\2uw\5\20\t\2vu\3\2\2\2vw\3\2\2\2wx\3\2\2\2xz\7\r\2\2y{\5\36\20"+
-    "\2zy\3\2\2\2z{\3\2\2\2{|\3\2\2\2|~\7\r\2\2}\177\5\22\n\2~}\3\2\2\2~\177"+
-    "\3\2\2\2\177\u0080\3\2\2\2\u0080\u0083\7\n\2\2\u0081\u0084\5\n\6\2\u0082"+
-    "\u0084\5\16\b\2\u0083\u0081\3\2\2\2\u0083\u0082\3\2\2\2\u0084\u00a8\3"+
-    "\2\2\2\u0085\u0086\7\22\2\2\u0086\u0087\7\t\2\2\u0087\u0088\5\26\f\2\u0088"+
-    "\u0089\7L\2\2\u0089\u008a\7\62\2\2\u008a\u008b\5\36\20\2\u008b\u008c\7"+
-    "\n\2\2\u008c\u008d\5\n\6\2\u008d\u00a8\3\2\2\2\u008e\u008f\5\24\13\2\u008f"+
-    "\u0090\5\34\17\2\u0090\u00a8\3\2\2\2\u0091\u0092\7\23\2\2\u0092\u00a8"+
-    "\5\34\17\2\u0093\u0094\7\24\2\2\u0094\u00a8\5\34\17\2\u0095\u0096\7\25"+
-    "\2\2\u0096\u0097\5\36\20\2\u0097\u0098\5\34\17\2\u0098\u00a8\3\2\2\2\u0099"+
-    "\u009a\7\27\2\2\u009a\u009c\5\f\7\2\u009b\u009d\5\32\16\2\u009c\u009b"+
-    "\3\2\2\2\u009d\u009e\3\2\2\2\u009e\u009c\3\2\2\2\u009e\u009f\3\2\2\2\u009f"+
-    "\u00a8\3\2\2\2\u00a0\u00a1\7\31\2\2\u00a1\u00a2\5\36\20\2\u00a2\u00a3"+
-    "\5\34\17\2\u00a3\u00a8\3\2\2\2\u00a4\u00a5\5\36\20\2\u00a5\u00a6\5\34"+
-    "\17\2\u00a6\u00a8\3\2\2\2\u00a7Y\3\2\2\2\u00a7c\3\2\2\2\u00a7k\3\2\2\2"+
-    "\u00a7s\3\2\2\2\u00a7\u0085\3\2\2\2\u00a7\u008e\3\2\2\2\u00a7\u0091\3"+
-    "\2\2\2\u00a7\u0093\3\2\2\2\u00a7\u0095\3\2\2\2\u00a7\u0099\3\2\2\2\u00a7"+
-    "\u00a0\3\2\2\2\u00a7\u00a4\3\2\2\2\u00a8\t\3\2\2\2\u00a9\u00ac\5\f\7\2"+
-    "\u00aa\u00ac\5\b\5\2\u00ab\u00a9\3\2\2\2\u00ab\u00aa\3\2\2\2\u00ac\13"+
-    "\3\2\2\2\u00ad\u00b1\7\5\2\2\u00ae\u00b0\5\b\5\2\u00af\u00ae\3\2\2\2\u00b0"+
-    "\u00b3\3\2\2\2\u00b1\u00af\3\2\2\2\u00b1\u00b2\3\2\2\2\u00b2\u00b4\3\2"+
-    "\2\2\u00b3\u00b1\3\2\2\2\u00b4\u00b5\7\6\2\2\u00b5\r\3\2\2\2\u00b6\u00b7"+
-    "\7\r\2\2\u00b7\17\3\2\2\2\u00b8\u00bb\5\24\13\2\u00b9\u00bb\5\36\20\2"+
-    "\u00ba\u00b8\3\2\2\2\u00ba\u00b9\3\2\2\2\u00bb\21\3\2\2\2\u00bc\u00bd"+
-    "\5\36\20\2\u00bd\23\3\2\2\2\u00be\u00bf\5\26\f\2\u00bf\u00c4\5\30\r\2"+
-    "\u00c0\u00c1\7\f\2\2\u00c1\u00c3\5\30\r\2\u00c2\u00c0\3\2\2\2\u00c3\u00c6"+
-    "\3\2\2\2\u00c4\u00c2\3\2\2\2\u00c4\u00c5\3\2\2\2\u00c5\25\3\2\2\2\u00c6"+
-    "\u00c4\3\2\2\2\u00c7\u00cc\7K\2\2\u00c8\u00c9\7\7\2\2\u00c9\u00cb\7\b"+
-    "\2\2\u00ca\u00c8\3\2\2\2\u00cb\u00ce\3\2\2\2\u00cc\u00ca\3\2\2\2\u00cc"+
-    "\u00cd\3\2\2\2\u00cd\27\3\2\2\2\u00ce\u00cc\3\2\2\2\u00cf\u00d2\7L\2\2"+
-    "\u00d0\u00d1\7\67\2\2\u00d1\u00d3\5\36\20\2\u00d2\u00d0\3\2\2\2\u00d2"+
-    "\u00d3\3\2\2\2\u00d3\31\3\2\2\2\u00d4\u00d5\7\30\2\2\u00d5\u00d6\7\t\2"+
-    "\2\u00d6\u00d7\7K\2\2\u00d7\u00d8\7L\2\2\u00d8\u00d9\7\n\2\2\u00d9\u00da"+
-    "\5\f\7\2\u00da\33\3\2\2\2\u00db\u00dc\t\2\2\2\u00dc\35\3\2\2\2\u00dd\u00de"+
-    "\b\20\1\2\u00de\u00df\5\"\22\2\u00df\u00e0\t\3\2\2\u00e0\u00e1\5\36\20"+
-    "\3\u00e1\u00e2\b\20\1\2\u00e2\u00e7\3\2\2\2\u00e3\u00e4\5 \21\2\u00e4"+
-    "\u00e5\b\20\1\2\u00e5\u00e7\3\2\2\2\u00e6\u00dd\3\2\2\2\u00e6\u00e3\3"+
-    "\2\2\2\u00e7\u0123\3\2\2\2\u00e8\u00e9\f\16\2\2\u00e9\u00ea\t\4\2\2\u00ea"+
-    "\u00eb\5\36\20\17\u00eb\u00ec\b\20\1\2\u00ec\u0122\3\2\2\2\u00ed\u00ee"+
-    "\f\r\2\2\u00ee\u00ef\t\5\2\2\u00ef\u00f0\5\36\20\16\u00f0\u00f1\b\20\1"+
-    "\2\u00f1\u0122\3\2\2\2\u00f2\u00f3\f\f\2\2\u00f3\u00f4\t\6\2\2\u00f4\u00f5"+
-    "\5\36\20\r\u00f5\u00f6\b\20\1\2\u00f6\u0122\3\2\2\2\u00f7\u00f8\f\13\2"+
-    "\2\u00f8\u00f9\t\7\2\2\u00f9\u00fa\5\36\20\f\u00fa\u00fb\b\20\1\2\u00fb"+
-    "\u0122\3\2\2\2\u00fc\u00fd\f\n\2\2\u00fd\u00fe\t\b\2\2\u00fe\u00ff\5\36"+
-    "\20\13\u00ff\u0100\b\20\1\2\u0100\u0122\3\2\2\2\u0101\u0102\f\t\2\2\u0102"+
-    "\u0103\7,\2\2\u0103\u0104\5\36\20\n\u0104\u0105\b\20\1\2\u0105\u0122\3"+
-    "\2\2\2\u0106\u0107\f\b\2\2\u0107\u0108\7-\2\2\u0108\u0109\5\36\20\t\u0109"+
-    "\u010a\b\20\1\2\u010a\u0122\3\2\2\2\u010b\u010c\f\7\2\2\u010c\u010d\7"+
-    ".\2\2\u010d\u010e\5\36\20\b\u010e\u010f\b\20\1\2\u010f\u0122\3\2\2\2\u0110"+
-    "\u0111\f\6\2\2\u0111\u0112\7/\2\2\u0112\u0113\5\36\20\7\u0113\u0114\b"+
-    "\20\1\2\u0114\u0122\3\2\2\2\u0115\u0116\f\5\2\2\u0116\u0117\7\60\2\2\u0117"+
-    "\u0118\5\36\20\6\u0118\u0119\b\20\1\2\u0119\u0122\3\2\2\2\u011a\u011b"+
-    "\f\4\2\2\u011b\u011c\7\61\2\2\u011c\u011d\5\36\20\2\u011d\u011e\7\62\2"+
-    "\2\u011e\u011f\5\36\20\4\u011f\u0120\b\20\1\2\u0120\u0122\3\2\2\2\u0121"+
-    "\u00e8\3\2\2\2\u0121\u00ed\3\2\2\2\u0121\u00f2\3\2\2\2\u0121\u00f7\3\2"+
-    "\2\2\u0121\u00fc\3\2\2\2\u0121\u0101\3\2\2\2\u0121\u0106\3\2\2\2\u0121"+
-    "\u010b\3\2\2\2\u0121\u0110\3\2\2\2\u0121\u0115\3\2\2\2\u0121\u011a\3\2"+
-    "\2\2\u0122\u0125\3\2\2\2\u0123\u0121\3\2\2\2\u0123\u0124\3\2\2\2\u0124"+
-    "\37\3\2\2\2\u0125\u0123\3\2\2\2\u0126\u0127\6\21\16\3\u0127\u0128\t\t"+
-    "\2\2\u0128\u0144\5\"\22\2\u0129\u012a\6\21\17\3\u012a\u012b\5\"\22\2\u012b"+
-    "\u012c\t\t\2\2\u012c\u0144\3\2\2\2\u012d\u012e\6\21\20\3\u012e\u0144\5"+
-    "\"\22\2\u012f\u0130\6\21\21\3\u0130\u0131\t\n\2\2\u0131\u0144\b\21\1\2"+
-    "\u0132\u0133\6\21\22\3\u0133\u0134\7H\2\2\u0134\u0144\b\21\1\2\u0135\u0136"+
-    "\6\21\23\3\u0136\u0137\7I\2\2\u0137\u0144\b\21\1\2\u0138\u0139\6\21\24"+
-    "\3\u0139\u013a\7J\2\2\u013a\u0144\b\21\1\2\u013b\u013c\6\21\25\3\u013c"+
-    "\u013d\t\13\2\2\u013d\u0144\5 \21\2\u013e\u013f\7\t\2\2\u013f\u0140\5"+
-    "\26\f\2\u0140\u0141\7\n\2\2\u0141\u0142\5 \21\2\u0142\u0144\3\2\2\2\u0143"+
-    "\u0126\3\2\2\2\u0143\u0129\3\2\2\2\u0143\u012d\3\2\2\2\u0143\u012f\3\2"+
-    "\2\2\u0143\u0132\3\2\2\2\u0143\u0135\3\2\2\2\u0143\u0138\3\2\2\2\u0143"+
-    "\u013b\3\2\2\2\u0143\u013e\3\2\2\2\u0144!\3\2\2\2\u0145\u0149\5$\23\2"+
-    "\u0146\u0148\5&\24\2\u0147\u0146\3\2\2\2\u0148\u014b\3\2\2\2\u0149\u0147"+
-    "\3\2\2\2\u0149\u014a\3\2\2\2\u014a\u0168\3\2\2\2\u014b\u0149\3\2\2\2\u014c"+
-    "\u014d\5\26\f\2\u014d\u0151\5(\25\2\u014e\u0150\5&\24\2\u014f\u014e\3"+
-    "\2\2\2\u0150\u0153\3\2\2\2\u0151\u014f\3\2\2\2\u0151\u0152\3\2\2\2\u0152"+
-    "\u0168\3\2\2\2\u0153\u0151\3\2\2\2\u0154\u0155\7\26\2\2\u0155\u015a\7"+
-    "K\2\2\u0156\u0157\7\7\2\2\u0157\u0158\5\36\20\2\u0158\u0159\7\b\2\2\u0159"+
-    "\u015b\3\2\2\2\u015a\u0156\3\2\2\2\u015b\u015c\3\2\2\2\u015c\u015a\3\2"+
-    "\2\2\u015c\u015d\3\2\2\2\u015d\u0165\3\2\2\2\u015e\u0162\5(\25\2\u015f"+
-    "\u0161\5&\24\2\u0160\u015f\3\2\2\2\u0161\u0164\3\2\2\2\u0162\u0160\3\2"+
-    "\2\2\u0162\u0163\3\2\2\2\u0163\u0166\3\2\2\2\u0164\u0162\3\2\2\2\u0165"+
-    "\u015e\3\2\2\2\u0165\u0166\3\2\2\2\u0166\u0168\3\2\2\2\u0167\u0145\3\2"+
-    "\2\2\u0167\u014c\3\2\2\2\u0167\u0154\3\2\2\2\u0168#\3\2\2\2\u0169\u016a"+
-    "\6\23\26\3\u016a\u016b\7\t\2\2\u016b\u016c\5\36\20\2\u016c\u016d\7\n\2"+
-    "\2\u016d\u016e\b\23\1\2\u016e\u017c\3\2\2\2\u016f\u0170\6\23\27\3\u0170"+
-    "\u0171\7\t\2\2\u0171\u0172\5 \21\2\u0172\u0173\7\n\2\2\u0173\u017c\3\2"+
-    "\2\2\u0174\u017c\7G\2\2\u0175\u017c\7L\2\2\u0176\u0177\7L\2\2\u0177\u017c"+
-    "\5,\27\2\u0178\u0179\7\26\2\2\u0179\u017a\7K\2\2\u017a\u017c\5,\27\2\u017b"+
-    "\u0169\3\2\2\2\u017b\u016f\3\2\2\2\u017b\u0174\3\2\2\2\u017b\u0175\3\2"+
-    "\2\2\u017b\u0176\3\2\2\2\u017b\u0178\3\2\2\2\u017c%\3\2\2\2\u017d\u017e"+
-    "\6\24\30\3\u017e\u0182\5(\25\2\u017f\u0180\6\24\31\3\u0180\u0182\5*\26"+
-    "\2\u0181\u017d\3\2\2\2\u0181\u017f\3\2\2\2\u0182\'\3\2\2\2\u0183\u0184"+
-    "\7\13\2\2\u0184\u0185\7N\2\2\u0185\u0189\5,\27\2\u0186\u0187\7\13\2\2"+
-    "\u0187\u0189\t\f\2\2\u0188\u0183\3\2\2\2\u0188\u0186\3\2\2\2\u0189)\3"+
-    "\2\2\2\u018a\u018b\7\7\2\2\u018b\u018c\5\36\20\2\u018c\u018d\7\b\2\2\u018d"+
-    "+\3\2\2\2\u018e\u0197\7\t\2\2\u018f\u0194\5.\30\2\u0190\u0191\7\f\2\2"+
-    "\u0191\u0193\5.\30\2\u0192\u0190\3\2\2\2\u0193\u0196\3\2\2\2\u0194\u0192"+
-    "\3\2\2\2\u0194\u0195\3\2\2\2\u0195\u0198\3\2\2\2\u0196\u0194\3\2\2\2\u0197"+
-    "\u018f\3\2\2\2\u0197\u0198\3\2\2\2\u0198\u0199\3\2\2\2\u0199\u019a\7\n"+
-    "\2\2\u019a-\3\2\2\2\u019b\u019f\5\36\20\2\u019c\u019f\5\60\31\2\u019d"+
-    "\u019f\5\64\33\2\u019e\u019b\3\2\2\2\u019e\u019c\3\2\2\2\u019e\u019d\3"+
-    "\2\2\2\u019f/\3\2\2\2\u01a0\u01ae\5\62\32\2\u01a1\u01aa\7\t\2\2\u01a2"+
-    "\u01a7\5\62\32\2\u01a3\u01a4\7\f\2\2\u01a4\u01a6\5\62\32\2\u01a5\u01a3"+
-    "\3\2\2\2\u01a6\u01a9\3\2\2\2\u01a7\u01a5\3\2\2\2\u01a7\u01a8\3\2\2\2\u01a8"+
-    "\u01ab\3\2\2\2\u01a9\u01a7\3\2\2\2\u01aa\u01a2\3\2\2\2\u01aa\u01ab\3\2"+
-    "\2\2\u01ab\u01ac\3\2\2\2\u01ac\u01ae\7\n\2\2\u01ad\u01a0\3\2\2\2\u01ad"+
-    "\u01a1\3\2\2\2\u01ae\u01af\3\2\2\2\u01af\u01b0\7\64\2\2\u01b0\u01b1\5"+
-    "\f\7\2\u01b1\61\3\2\2\2\u01b2\u01b4\5\26\f\2\u01b3\u01b2\3\2\2\2\u01b3"+
-    "\u01b4\3\2\2\2\u01b4\u01b5\3\2\2\2\u01b5\u01b6\7L\2\2\u01b6\63\3\2\2\2"+
-    "\u01b7\u01b8\7K\2\2\u01b8\u01b9\7\63\2\2\u01b9\u01be\t\r\2\2\u01ba\u01bb"+
-    "\7L\2\2\u01bb\u01bc\7\63\2\2\u01bc\u01be\7L\2\2\u01bd\u01b7\3\2\2\2\u01bd"+
-    "\u01ba\3\2\2\2\u01be\65\3\2\2\2)9?RUaivz~\u0083\u009e\u00a7\u00ab\u00b1"+
-    "\u00ba\u00c4\u00cc\u00d2\u00e6\u0121\u0123\u0143\u0149\u0151\u015c\u0162"+
-    "\u0165\u0167\u017b\u0181\u0188\u0194\u0197\u019e\u01a7\u01aa\u01ad\u01b3"+
-    "\u01bd";
+    "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\5\21\u0149\n\21\3\22\3\22"+
+    "\7\22\u014d\n\22\f\22\16\22\u0150\13\22\3\22\3\22\3\22\7\22\u0155\n\22"+
+    "\f\22\16\22\u0158\13\22\3\22\3\22\3\22\3\22\3\22\3\22\6\22\u0160\n\22"+
+    "\r\22\16\22\u0161\3\22\3\22\7\22\u0166\n\22\f\22\16\22\u0169\13\22\5\22"+
+    "\u016b\n\22\5\22\u016d\n\22\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3"+
+    "\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\5\23\u0181\n\23\3\24"+
+    "\3\24\3\24\3\24\5\24\u0187\n\24\3\25\3\25\3\25\3\25\3\25\5\25\u018e\n"+
+    "\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\27\7\27\u0198\n\27\f\27\16\27"+
+    "\u019b\13\27\5\27\u019d\n\27\3\27\3\27\3\30\3\30\3\30\5\30\u01a4\n\30"+
+    "\3\31\3\31\3\31\3\31\3\31\7\31\u01ab\n\31\f\31\16\31\u01ae\13\31\5\31"+
+    "\u01b0\n\31\3\31\5\31\u01b3\n\31\3\31\3\31\3\31\3\32\5\32\u01b9\n\32\3"+
+    "\32\3\32\3\33\3\33\3\33\3\33\3\33\3\33\5\33\u01c3\n\33\3\33\2\3\36\34"+
+    "\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,.\60\62\64\2\r\3\2\67B"+
+    "\3\2\34\36\3\2\37 \3\2!#\3\2$\'\3\2(+\3\2\65\66\3\2CF\4\2\32\33\37 \3"+
+    "\2MN\4\2\26\26LL\u01f3\29\3\2\2\2\4D\3\2\2\2\6I\3\2\2\2\b\u00a7\3\2\2"+
+    "\2\n\u00ab\3\2\2\2\f\u00ad\3\2\2\2\16\u00b6\3\2\2\2\20\u00ba\3\2\2\2\22"+
+    "\u00bc\3\2\2\2\24\u00be\3\2\2\2\26\u00c7\3\2\2\2\30\u00cf\3\2\2\2\32\u00d4"+
+    "\3\2\2\2\34\u00e0\3\2\2\2\36\u00eb\3\2\2\2 \u0148\3\2\2\2\"\u016c\3\2"+
+    "\2\2$\u0180\3\2\2\2&\u0186\3\2\2\2(\u018d\3\2\2\2*\u018f\3\2\2\2,\u0193"+
+    "\3\2\2\2.\u01a3\3\2\2\2\60\u01b2\3\2\2\2\62\u01b8\3\2\2\2\64\u01c2\3\2"+
+    "\2\2\668\5\4\3\2\67\66\3\2\2\28;\3\2\2\29\67\3\2\2\29:\3\2\2\2:?\3\2\2"+
+    "\2;9\3\2\2\2<>\5\b\5\2=<\3\2\2\2>A\3\2\2\2?=\3\2\2\2?@\3\2\2\2@B\3\2\2"+
+    "\2A?\3\2\2\2BC\7\2\2\3C\3\3\2\2\2DE\5\26\f\2EF\7L\2\2FG\5\6\4\2GH\5\f"+
+    "\7\2H\5\3\2\2\2IU\7\t\2\2JK\5\26\f\2KR\7L\2\2LM\7\f\2\2MN\5\26\f\2NO\7"+
+    "L\2\2OQ\3\2\2\2PL\3\2\2\2QT\3\2\2\2RP\3\2\2\2RS\3\2\2\2SV\3\2\2\2TR\3"+
+    "\2\2\2UJ\3\2\2\2UV\3\2\2\2VW\3\2\2\2WX\7\n\2\2X\7\3\2\2\2YZ\7\16\2\2Z"+
+    "[\7\t\2\2[\\\5\36\20\2\\]\7\n\2\2]a\5\n\6\2^_\7\17\2\2_b\5\n\6\2`b\6\5"+
+    "\2\2a^\3\2\2\2a`\3\2\2\2b\u00a8\3\2\2\2cd\7\20\2\2de\7\t\2\2ef\5\36\20"+
+    "\2fi\7\n\2\2gj\5\n\6\2hj\5\16\b\2ig\3\2\2\2ih\3\2\2\2j\u00a8\3\2\2\2k"+
+    "l\7\21\2\2lm\5\f\7\2mn\7\20\2\2no\7\t\2\2op\5\36\20\2pq\7\n\2\2qr\5\34"+
+    "\17\2r\u00a8\3\2\2\2st\7\22\2\2tv\7\t\2\2uw\5\20\t\2vu\3\2\2\2vw\3\2\2"+
+    "\2wx\3\2\2\2xz\7\r\2\2y{\5\36\20\2zy\3\2\2\2z{\3\2\2\2{|\3\2\2\2|~\7\r"+
+    "\2\2}\177\5\22\n\2~}\3\2\2\2~\177\3\2\2\2\177\u0080\3\2\2\2\u0080\u0083"+
+    "\7\n\2\2\u0081\u0084\5\n\6\2\u0082\u0084\5\16\b\2\u0083\u0081\3\2\2\2"+
+    "\u0083\u0082\3\2\2\2\u0084\u00a8\3\2\2\2\u0085\u0086\7\22\2\2\u0086\u0087"+
+    "\7\t\2\2\u0087\u0088\5\26\f\2\u0088\u0089\7L\2\2\u0089\u008a\7\62\2\2"+
+    "\u008a\u008b\5\36\20\2\u008b\u008c\7\n\2\2\u008c\u008d\5\n\6\2\u008d\u00a8"+
+    "\3\2\2\2\u008e\u008f\5\24\13\2\u008f\u0090\5\34\17\2\u0090\u00a8\3\2\2"+
+    "\2\u0091\u0092\7\23\2\2\u0092\u00a8\5\34\17\2\u0093\u0094\7\24\2\2\u0094"+
+    "\u00a8\5\34\17\2\u0095\u0096\7\25\2\2\u0096\u0097\5\36\20\2\u0097\u0098"+
+    "\5\34\17\2\u0098\u00a8\3\2\2\2\u0099\u009a\7\27\2\2\u009a\u009c\5\f\7"+
+    "\2\u009b\u009d\5\32\16\2\u009c\u009b\3\2\2\2\u009d\u009e\3\2\2\2\u009e"+
+    "\u009c\3\2\2\2\u009e\u009f\3\2\2\2\u009f\u00a8\3\2\2\2\u00a0\u00a1\7\31"+
+    "\2\2\u00a1\u00a2\5\36\20\2\u00a2\u00a3\5\34\17\2\u00a3\u00a8\3\2\2\2\u00a4"+
+    "\u00a5\5\36\20\2\u00a5\u00a6\5\34\17\2\u00a6\u00a8\3\2\2\2\u00a7Y\3\2"+
+    "\2\2\u00a7c\3\2\2\2\u00a7k\3\2\2\2\u00a7s\3\2\2\2\u00a7\u0085\3\2\2\2"+
+    "\u00a7\u008e\3\2\2\2\u00a7\u0091\3\2\2\2\u00a7\u0093\3\2\2\2\u00a7\u0095"+
+    "\3\2\2\2\u00a7\u0099\3\2\2\2\u00a7\u00a0\3\2\2\2\u00a7\u00a4\3\2\2\2\u00a8"+
+    "\t\3\2\2\2\u00a9\u00ac\5\f\7\2\u00aa\u00ac\5\b\5\2\u00ab\u00a9\3\2\2\2"+
+    "\u00ab\u00aa\3\2\2\2\u00ac\13\3\2\2\2\u00ad\u00b1\7\5\2\2\u00ae\u00b0"+
+    "\5\b\5\2\u00af\u00ae\3\2\2\2\u00b0\u00b3\3\2\2\2\u00b1\u00af\3\2\2\2\u00b1"+
+    "\u00b2\3\2\2\2\u00b2\u00b4\3\2\2\2\u00b3\u00b1\3\2\2\2\u00b4\u00b5\7\6"+
+    "\2\2\u00b5\r\3\2\2\2\u00b6\u00b7\7\r\2\2\u00b7\17\3\2\2\2\u00b8\u00bb"+
+    "\5\24\13\2\u00b9\u00bb\5\36\20\2\u00ba\u00b8\3\2\2\2\u00ba\u00b9\3\2\2"+
+    "\2\u00bb\21\3\2\2\2\u00bc\u00bd\5\36\20\2\u00bd\23\3\2\2\2\u00be\u00bf"+
+    "\5\26\f\2\u00bf\u00c4\5\30\r\2\u00c0\u00c1\7\f\2\2\u00c1\u00c3\5\30\r"+
+    "\2\u00c2\u00c0\3\2\2\2\u00c3\u00c6\3\2\2\2\u00c4\u00c2\3\2\2\2\u00c4\u00c5"+
+    "\3\2\2\2\u00c5\25\3\2\2\2\u00c6\u00c4\3\2\2\2\u00c7\u00cc\7K\2\2\u00c8"+
+    "\u00c9\7\7\2\2\u00c9\u00cb\7\b\2\2\u00ca\u00c8\3\2\2\2\u00cb\u00ce\3\2"+
+    "\2\2\u00cc\u00ca\3\2\2\2\u00cc\u00cd\3\2\2\2\u00cd\27\3\2\2\2\u00ce\u00cc"+
+    "\3\2\2\2\u00cf\u00d2\7L\2\2\u00d0\u00d1\7\67\2\2\u00d1\u00d3\5\36\20\2"+
+    "\u00d2\u00d0\3\2\2\2\u00d2\u00d3\3\2\2\2\u00d3\31\3\2\2\2\u00d4\u00d5"+
+    "\7\30\2\2\u00d5\u00d6\7\t\2\2\u00d6\u00d7\7K\2\2\u00d7\u00d8\7L\2\2\u00d8"+
+    "\u00d9\7\n\2\2\u00d9\u00da\5\f\7\2\u00da\33\3\2\2\2\u00db\u00e1\7\r\2"+
+    "\2\u00dc\u00e1\7\2\2\3\u00dd\u00de\b\17\1\2\u00de\u00df\7\6\2\2\u00df"+
+    "\u00e1\b\17\1\2\u00e0\u00db\3\2\2\2\u00e0\u00dc\3\2\2\2\u00e0\u00dd\3"+
+    "\2\2\2\u00e1\35\3\2\2\2\u00e2\u00e3\b\20\1\2\u00e3\u00e4\5\"\22\2\u00e4"+
+    "\u00e5\t\2\2\2\u00e5\u00e6\5\36\20\3\u00e6\u00e7\b\20\1\2\u00e7\u00ec"+
+    "\3\2\2\2\u00e8\u00e9\5 \21\2\u00e9\u00ea\b\20\1\2\u00ea\u00ec\3\2\2\2"+
+    "\u00eb\u00e2\3\2\2\2\u00eb\u00e8\3\2\2\2\u00ec\u0128\3\2\2\2\u00ed\u00ee"+
+    "\f\16\2\2\u00ee\u00ef\t\3\2\2\u00ef\u00f0\5\36\20\17\u00f0\u00f1\b\20"+
+    "\1\2\u00f1\u0127\3\2\2\2\u00f2\u00f3\f\r\2\2\u00f3\u00f4\t\4\2\2\u00f4"+
+    "\u00f5\5\36\20\16\u00f5\u00f6\b\20\1\2\u00f6\u0127\3\2\2\2\u00f7\u00f8"+
+    "\f\f\2\2\u00f8\u00f9\t\5\2\2\u00f9\u00fa\5\36\20\r\u00fa\u00fb\b\20\1"+
+    "\2\u00fb\u0127\3\2\2\2\u00fc\u00fd\f\13\2\2\u00fd\u00fe\t\6\2\2\u00fe"+
+    "\u00ff\5\36\20\f\u00ff\u0100\b\20\1\2\u0100\u0127\3\2\2\2\u0101\u0102"+
+    "\f\n\2\2\u0102\u0103\t\7\2\2\u0103\u0104\5\36\20\13\u0104\u0105\b\20\1"+
+    "\2\u0105\u0127\3\2\2\2\u0106\u0107\f\t\2\2\u0107\u0108\7,\2\2\u0108\u0109"+
+    "\5\36\20\n\u0109\u010a\b\20\1\2\u010a\u0127\3\2\2\2\u010b\u010c\f\b\2"+
+    "\2\u010c\u010d\7-\2\2\u010d\u010e\5\36\20\t\u010e\u010f\b\20\1\2\u010f"+
+    "\u0127\3\2\2\2\u0110\u0111\f\7\2\2\u0111\u0112\7.\2\2\u0112\u0113\5\36"+
+    "\20\b\u0113\u0114\b\20\1\2\u0114\u0127\3\2\2\2\u0115\u0116\f\6\2\2\u0116"+
+    "\u0117\7/\2\2\u0117\u0118\5\36\20\7\u0118\u0119\b\20\1\2\u0119\u0127\3"+
+    "\2\2\2\u011a\u011b\f\5\2\2\u011b\u011c\7\60\2\2\u011c\u011d\5\36\20\6"+
+    "\u011d\u011e\b\20\1\2\u011e\u0127\3\2\2\2\u011f\u0120\f\4\2\2\u0120\u0121"+
+    "\7\61\2\2\u0121\u0122\5\36\20\2\u0122\u0123\7\62\2\2\u0123\u0124\5\36"+
+    "\20\4\u0124\u0125\b\20\1\2\u0125\u0127\3\2\2\2\u0126\u00ed\3\2\2\2\u0126"+
+    "\u00f2\3\2\2\2\u0126\u00f7\3\2\2\2\u0126\u00fc\3\2\2\2\u0126\u0101\3\2"+
+    "\2\2\u0126\u0106\3\2\2\2\u0126\u010b\3\2\2\2\u0126\u0110\3\2\2\2\u0126"+
+    "\u0115\3\2\2\2\u0126\u011a\3\2\2\2\u0126\u011f\3\2\2\2\u0127\u012a\3\2"+
+    "\2\2\u0128\u0126\3\2\2\2\u0128\u0129\3\2\2\2\u0129\37\3\2\2\2\u012a\u0128"+
+    "\3\2\2\2\u012b\u012c\6\21\16\3\u012c\u012d\t\b\2\2\u012d\u0149\5\"\22"+
+    "\2\u012e\u012f\6\21\17\3\u012f\u0130\5\"\22\2\u0130\u0131\t\b\2\2\u0131"+
+    "\u0149\3\2\2\2\u0132\u0133\6\21\20\3\u0133\u0149\5\"\22\2\u0134\u0135"+
+    "\6\21\21\3\u0135\u0136\t\t\2\2\u0136\u0149\b\21\1\2\u0137\u0138\6\21\22"+
+    "\3\u0138\u0139\7H\2\2\u0139\u0149\b\21\1\2\u013a\u013b\6\21\23\3\u013b"+
+    "\u013c\7I\2\2\u013c\u0149\b\21\1\2\u013d\u013e\6\21\24\3\u013e\u013f\7"+
+    "J\2\2\u013f\u0149\b\21\1\2\u0140\u0141\6\21\25\3\u0141\u0142\t\n\2\2\u0142"+
+    "\u0149\5 \21\2\u0143\u0144\7\t\2\2\u0144\u0145\5\26\f\2\u0145\u0146\7"+
+    "\n\2\2\u0146\u0147\5 \21\2\u0147\u0149\3\2\2\2\u0148\u012b\3\2\2\2\u0148"+
+    "\u012e\3\2\2\2\u0148\u0132\3\2\2\2\u0148\u0134\3\2\2\2\u0148\u0137\3\2"+
+    "\2\2\u0148\u013a\3\2\2\2\u0148\u013d\3\2\2\2\u0148\u0140\3\2\2\2\u0148"+
+    "\u0143\3\2\2\2\u0149!\3\2\2\2\u014a\u014e\5$\23\2\u014b\u014d\5&\24\2"+
+    "\u014c\u014b\3\2\2\2\u014d\u0150\3\2\2\2\u014e\u014c\3\2\2\2\u014e\u014f"+
+    "\3\2\2\2\u014f\u016d\3\2\2\2\u0150\u014e\3\2\2\2\u0151\u0152\5\26\f\2"+
+    "\u0152\u0156\5(\25\2\u0153\u0155\5&\24\2\u0154\u0153\3\2\2\2\u0155\u0158"+
+    "\3\2\2\2\u0156\u0154\3\2\2\2\u0156\u0157\3\2\2\2\u0157\u016d\3\2\2\2\u0158"+
+    "\u0156\3\2\2\2\u0159\u015a\7\26\2\2\u015a\u015f\7K\2\2\u015b\u015c\7\7"+
+    "\2\2\u015c\u015d\5\36\20\2\u015d\u015e\7\b\2\2\u015e\u0160\3\2\2\2\u015f"+
+    "\u015b\3\2\2\2\u0160\u0161\3\2\2\2\u0161\u015f\3\2\2\2\u0161\u0162\3\2"+
+    "\2\2\u0162\u016a\3\2\2\2\u0163\u0167\5(\25\2\u0164\u0166\5&\24\2\u0165"+
+    "\u0164\3\2\2\2\u0166\u0169\3\2\2\2\u0167\u0165\3\2\2\2\u0167\u0168\3\2"+
+    "\2\2\u0168\u016b\3\2\2\2\u0169\u0167\3\2\2\2\u016a\u0163\3\2\2\2\u016a"+
+    "\u016b\3\2\2\2\u016b\u016d\3\2\2\2\u016c\u014a\3\2\2\2\u016c\u0151\3\2"+
+    "\2\2\u016c\u0159\3\2\2\2\u016d#\3\2\2\2\u016e\u016f\6\23\26\3\u016f\u0170"+
+    "\7\t\2\2\u0170\u0171\5\36\20\2\u0171\u0172\7\n\2\2\u0172\u0173\b\23\1"+
+    "\2\u0173\u0181\3\2\2\2\u0174\u0175\6\23\27\3\u0175\u0176\7\t\2\2\u0176"+
+    "\u0177\5 \21\2\u0177\u0178\7\n\2\2\u0178\u0181\3\2\2\2\u0179\u0181\7G"+
+    "\2\2\u017a\u0181\7L\2\2\u017b\u017c\7L\2\2\u017c\u0181\5,\27\2\u017d\u017e"+
+    "\7\26\2\2\u017e\u017f\7K\2\2\u017f\u0181\5,\27\2\u0180\u016e\3\2\2\2\u0180"+
+    "\u0174\3\2\2\2\u0180\u0179\3\2\2\2\u0180\u017a\3\2\2\2\u0180\u017b\3\2"+
+    "\2\2\u0180\u017d\3\2\2\2\u0181%\3\2\2\2\u0182\u0183\6\24\30\3\u0183\u0187"+
+    "\5(\25\2\u0184\u0185\6\24\31\3\u0185\u0187\5*\26\2\u0186\u0182\3\2\2\2"+
+    "\u0186\u0184\3\2\2\2\u0187\'\3\2\2\2\u0188\u0189\7\13\2\2\u0189\u018a"+
+    "\7N\2\2\u018a\u018e\5,\27\2\u018b\u018c\7\13\2\2\u018c\u018e\t\13\2\2"+
+    "\u018d\u0188\3\2\2\2\u018d\u018b\3\2\2\2\u018e)\3\2\2\2\u018f\u0190\7"+
+    "\7\2\2\u0190\u0191\5\36\20\2\u0191\u0192\7\b\2\2\u0192+\3\2\2\2\u0193"+
+    "\u019c\7\t\2\2\u0194\u0199\5.\30\2\u0195\u0196\7\f\2\2\u0196\u0198\5."+
+    "\30\2\u0197\u0195\3\2\2\2\u0198\u019b\3\2\2\2\u0199\u0197\3\2\2\2\u0199"+
+    "\u019a\3\2\2\2\u019a\u019d\3\2\2\2\u019b\u0199\3\2\2\2\u019c\u0194\3\2"+
+    "\2\2\u019c\u019d\3\2\2\2\u019d\u019e\3\2\2\2\u019e\u019f\7\n\2\2\u019f"+
+    "-\3\2\2\2\u01a0\u01a4\5\36\20\2\u01a1\u01a4\5\60\31\2\u01a2\u01a4\5\64"+
+    "\33\2\u01a3\u01a0\3\2\2\2\u01a3\u01a1\3\2\2\2\u01a3\u01a2\3\2\2\2\u01a4"+
+    "/\3\2\2\2\u01a5\u01b3\5\62\32\2\u01a6\u01af\7\t\2\2\u01a7\u01ac\5\62\32"+
+    "\2\u01a8\u01a9\7\f\2\2\u01a9\u01ab\5\62\32\2\u01aa\u01a8\3\2\2\2\u01ab"+
+    "\u01ae\3\2\2\2\u01ac\u01aa\3\2\2\2\u01ac\u01ad\3\2\2\2\u01ad\u01b0\3\2"+
+    "\2\2\u01ae\u01ac\3\2\2\2\u01af\u01a7\3\2\2\2\u01af\u01b0\3\2\2\2\u01b0"+
+    "\u01b1\3\2\2\2\u01b1\u01b3\7\n\2\2\u01b2\u01a5\3\2\2\2\u01b2\u01a6\3\2"+
+    "\2\2\u01b3\u01b4\3\2\2\2\u01b4\u01b5\7\64\2\2\u01b5\u01b6\5\f\7\2\u01b6"+
+    "\61\3\2\2\2\u01b7\u01b9\5\26\f\2\u01b8\u01b7\3\2\2\2\u01b8\u01b9\3\2\2"+
+    "\2\u01b9\u01ba\3\2\2\2\u01ba\u01bb\7L\2\2\u01bb\63\3\2\2\2\u01bc\u01bd"+
+    "\7K\2\2\u01bd\u01be\7\63\2\2\u01be\u01c3\t\f\2\2\u01bf\u01c0\7L\2\2\u01c0"+
+    "\u01c1\7\63\2\2\u01c1\u01c3\7L\2\2\u01c2\u01bc\3\2\2\2\u01c2\u01bf\3\2"+
+    "\2\2\u01c3\65\3\2\2\2*9?RUaivz~\u0083\u009e\u00a7\u00ab\u00b1\u00ba\u00c4"+
+    "\u00cc\u00d2\u00e0\u00eb\u0126\u0128\u0148\u014e\u0156\u0161\u0167\u016a"+
+    "\u016c\u0180\u0186\u018d\u0199\u019c\u01a3\u01ac\u01af\u01b2\u01b8\u01c2";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
index 447b2139779..49b9ac21970 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
@@ -221,4 +221,13 @@ public class BasicStatementTests extends ScriptTestCase {
         });
         assertTrue(exception.getCause().getMessage().contains("Lambda functions are not supported."));
     }
+
+    public void testLastInBlockDoesntNeedSemi() {
+        // One statement in the block in case that is a special case
+        assertEquals(10, exec("def i = 1; if (i == 1) {return 10}"));
+        assertEquals(10, exec("def i = 1; if (i == 1) {return 10} else {return 12}"));
+        // Two statements in the block, in case that is the general case
+        assertEquals(10, exec("def i = 1; if (i == 1) {i = 2; return 10}"));
+        assertEquals(10, exec("def i = 1; if (i == 1) {i = 2; return 10} else {return 12}"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
index ee88c0000bd..1c54f2727b5 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
@@ -23,6 +23,8 @@ import java.lang.invoke.WrongMethodTypeException;
 import java.util.Arrays;
 import java.util.Collections;
 
+import static java.util.Collections.emptyMap;
+
 public class WhenThingsGoWrongTests extends ScriptTestCase {
     public void testNullPointer() {
         expectScriptThrows(NullPointerException.class, () -> {
@@ -197,4 +199,16 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
             exec("def x = new ArrayList(); x.add('foo'); return x['bogus'];");
         });
     }
+
+    /**
+     * Makes sure that we present a useful error message with a misplaced right-curly. This is important because we do some funky things in
+     * the parser with right-curly brackets to allow statements to be delimited by them at the end of blocks.
+     */
+    public void testRCurlyNotDelim() {
+        IllegalArgumentException e = expectScriptThrows(IllegalArgumentException.class, () -> {
+            // We don't want PICKY here so we get the normal error message
+            exec("def i = 1} return 1", emptyMap(), emptyMap(), null);
+        });
+        assertEquals("invalid sequence of tokens near ['}'].", e.getMessage());
+    }
 }

From 0bfb166eeba0ba8f292550cb9100708c88e4a4c9 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Sat, 11 Jun 2016 13:23:17 -0400
Subject: [PATCH 0788/1311] merge master

---
 modules/lang-painless/build.gradle            |   5 +-
 .../lang-painless/licenses/asm-5.0.4.jar.sha1 |   1 -
 .../licenses/asm-commons-5.0.4.jar.sha1       |   1 -
 .../licenses/asm-debug-all-5.1.jar.sha1       |   1 +
 .../licenses/asm-tree-5.0.4.jar.sha1          |   1 -
 .../src/main/antlr/PainlessLexer.g4           |   1 +
 .../src/main/antlr/PainlessLexer.tokens       | 196 +++++----
 .../src/main/antlr/PainlessParser.g4          |   4 +-
 .../src/main/antlr/PainlessParser.tokens      | 196 +++++----
 .../java/org/elasticsearch/painless/Def.java  |  16 +-
 .../elasticsearch/painless/Definition.java    |  36 ++
 .../elasticsearch/painless/FunctionRef.java   |  94 ++--
 .../elasticsearch/painless/MethodWriter.java  |   4 +-
 .../painless/WriterConstants.java             |   6 +-
 .../painless/antlr/PainlessLexer.java         | 413 +++++++++---------
 .../painless/antlr/PainlessParser.java        | 203 ++++-----
 .../elasticsearch/painless/antlr/Walker.java  |   3 +
 .../painless/node/EFunctionRef.java           |  21 +-
 .../painless/node/SExpression.java            |   1 -
 .../painless/node/SFunction.java              |   3 +-
 .../painless/FunctionRefTests.java            |  16 +-
 21 files changed, 667 insertions(+), 555 deletions(-)
 delete mode 100644 modules/lang-painless/licenses/asm-5.0.4.jar.sha1
 delete mode 100644 modules/lang-painless/licenses/asm-commons-5.0.4.jar.sha1
 create mode 100644 modules/lang-painless/licenses/asm-debug-all-5.1.jar.sha1
 delete mode 100644 modules/lang-painless/licenses/asm-tree-5.0.4.jar.sha1

diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle
index 9f617c4e436..25a348927d7 100644
--- a/modules/lang-painless/build.gradle
+++ b/modules/lang-painless/build.gradle
@@ -26,10 +26,7 @@ esplugin {
 
 dependencies {
   compile 'org.antlr:antlr4-runtime:4.5.1-1'
-  compile 'org.ow2.asm:asm:5.0.4'
-  compile 'org.ow2.asm:asm-commons:5.0.4'
-  compile 'org.ow2.asm:asm-tree:5.0.4'
-  testCompile 'org.ow2.asm:asm-util:5.0.4'
+  compile 'org.ow2.asm:asm-debug-all:5.1'
 }
 
 dependencyLicenses {
diff --git a/modules/lang-painless/licenses/asm-5.0.4.jar.sha1 b/modules/lang-painless/licenses/asm-5.0.4.jar.sha1
deleted file mode 100644
index 9223dba380f..00000000000
--- a/modules/lang-painless/licenses/asm-5.0.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0da08b8cce7bbf903602a25a3a163ae252435795
diff --git a/modules/lang-painless/licenses/asm-commons-5.0.4.jar.sha1 b/modules/lang-painless/licenses/asm-commons-5.0.4.jar.sha1
deleted file mode 100644
index 94fe0cd92c9..00000000000
--- a/modules/lang-painless/licenses/asm-commons-5.0.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-5a556786086c23cd689a0328f8519db93821c04c
diff --git a/modules/lang-painless/licenses/asm-debug-all-5.1.jar.sha1 b/modules/lang-painless/licenses/asm-debug-all-5.1.jar.sha1
new file mode 100644
index 00000000000..926910cbc68
--- /dev/null
+++ b/modules/lang-painless/licenses/asm-debug-all-5.1.jar.sha1
@@ -0,0 +1 @@
+dde860d586960d55a87cd52c4fc1f5059c89cbc6
\ No newline at end of file
diff --git a/modules/lang-painless/licenses/asm-tree-5.0.4.jar.sha1 b/modules/lang-painless/licenses/asm-tree-5.0.4.jar.sha1
deleted file mode 100644
index 5822a485a61..00000000000
--- a/modules/lang-painless/licenses/asm-tree-5.0.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-396ce0c07ba2b481f25a70195c7c94922f0d1b0b
\ No newline at end of file
diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.g4 b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
index 2ae77efd4d0..8a5c2a5d061 100644
--- a/modules/lang-painless/src/main/antlr/PainlessLexer.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
@@ -51,6 +51,7 @@ NEW:       'new';
 TRY:       'try';
 CATCH:     'catch';
 THROW:     'throw';
+THIS:      'this';
 
 BOOLNOT: '!';
 BWNOT:   '~';
diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.tokens b/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
index f485246aa67..d81d2b48c08 100644
--- a/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
@@ -21,59 +21,60 @@ NEW=20
 TRY=21
 CATCH=22
 THROW=23
-BOOLNOT=24
-BWNOT=25
-MUL=26
-DIV=27
-REM=28
-ADD=29
-SUB=30
-LSH=31
-RSH=32
-USH=33
-LT=34
-LTE=35
-GT=36
-GTE=37
-EQ=38
-EQR=39
-NE=40
-NER=41
-BWAND=42
-XOR=43
-BWOR=44
-BOOLAND=45
-BOOLOR=46
-COND=47
-COLON=48
-REF=49
-ARROW=50
-INCR=51
-DECR=52
-ASSIGN=53
-AADD=54
-ASUB=55
-AMUL=56
-ADIV=57
-AREM=58
-AAND=59
-AXOR=60
-AOR=61
-ALSH=62
-ARSH=63
-AUSH=64
-OCTAL=65
-HEX=66
-INTEGER=67
-DECIMAL=68
-STRING=69
-TRUE=70
-FALSE=71
-NULL=72
-TYPE=73
-ID=74
-DOTINTEGER=75
-DOTID=76
+THIS=24
+BOOLNOT=25
+BWNOT=26
+MUL=27
+DIV=28
+REM=29
+ADD=30
+SUB=31
+LSH=32
+RSH=33
+USH=34
+LT=35
+LTE=36
+GT=37
+GTE=38
+EQ=39
+EQR=40
+NE=41
+NER=42
+BWAND=43
+XOR=44
+BWOR=45
+BOOLAND=46
+BOOLOR=47
+COND=48
+COLON=49
+REF=50
+ARROW=51
+INCR=52
+DECR=53
+ASSIGN=54
+AADD=55
+ASUB=56
+AMUL=57
+ADIV=58
+AREM=59
+AAND=60
+AXOR=61
+AOR=62
+ALSH=63
+ARSH=64
+AUSH=65
+OCTAL=66
+HEX=67
+INTEGER=68
+DECIMAL=69
+STRING=70
+TRUE=71
+FALSE=72
+NULL=73
+TYPE=74
+ID=75
+DOTINTEGER=76
+DOTID=77
 '{'=3
 '}'=4
 '['=5
@@ -95,47 +96,48 @@ DOTID=76
 'try'=21
 'catch'=22
 'throw'=23
-'!'=24
-'~'=25
-'*'=26
-'/'=27
-'%'=28
-'+'=29
-'-'=30
-'<<'=31
-'>>'=32
-'>>>'=33
-'<'=34
-'<='=35
-'>'=36
-'>='=37
-'=='=38
-'==='=39
-'!='=40
-'!=='=41
-'&'=42
-'^'=43
-'|'=44
-'&&'=45
-'||'=46
-'?'=47
-':'=48
-'::'=49
-'->'=50
-'++'=51
-'--'=52
-'='=53
-'+='=54
-'-='=55
-'*='=56
-'/='=57
-'%='=58
-'&='=59
-'^='=60
-'|='=61
-'<<='=62
-'>>='=63
-'>>>='=64
-'true'=70
-'false'=71
-'null'=72
+'this'=24
+'!'=25
+'~'=26
+'*'=27
+'/'=28
+'%'=29
+'+'=30
+'-'=31
+'<<'=32
+'>>'=33
+'>>>'=34
+'<'=35
+'<='=36
+'>'=37
+'>='=38
+'=='=39
+'==='=40
+'!='=41
+'!=='=42
+'&'=43
+'^'=44
+'|'=45
+'&&'=46
+'||'=47
+'?'=48
+':'=49
+'::'=50
+'->'=51
+'++'=52
+'--'=53
+'='=54
+'+='=55
+'-='=56
+'*='=57
+'/='=58
+'%='=59
+'&='=60
+'^='=61
+'|='=62
+'<<='=63
+'>>='=64
+'>>>='=65
+'true'=71
+'false'=72
+'null'=73
diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index f4ba05d2367..4ee78d011f3 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -196,5 +196,5 @@ lamtype
 
 funcref
     : TYPE REF ( ID | NEW )
-    | ID REF ID
-    ;
\ No newline at end of file
+    | ( ID | THIS ) REF ID
+    ;
diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.tokens b/modules/lang-painless/src/main/antlr/PainlessParser.tokens
index f485246aa67..d81d2b48c08 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.tokens
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.tokens
@@ -21,59 +21,60 @@ NEW=20
 TRY=21
 CATCH=22
 THROW=23
-BOOLNOT=24
-BWNOT=25
-MUL=26
-DIV=27
-REM=28
-ADD=29
-SUB=30
-LSH=31
-RSH=32
-USH=33
-LT=34
-LTE=35
-GT=36
-GTE=37
-EQ=38
-EQR=39
-NE=40
-NER=41
-BWAND=42
-XOR=43
-BWOR=44
-BOOLAND=45
-BOOLOR=46
-COND=47
-COLON=48
-REF=49
-ARROW=50
-INCR=51
-DECR=52
-ASSIGN=53
-AADD=54
-ASUB=55
-AMUL=56
-ADIV=57
-AREM=58
-AAND=59
-AXOR=60
-AOR=61
-ALSH=62
-ARSH=63
-AUSH=64
-OCTAL=65
-HEX=66
-INTEGER=67
-DECIMAL=68
-STRING=69
-TRUE=70
-FALSE=71
-NULL=72
-TYPE=73
-ID=74
-DOTINTEGER=75
-DOTID=76
+THIS=24
+BOOLNOT=25
+BWNOT=26
+MUL=27
+DIV=28
+REM=29
+ADD=30
+SUB=31
+LSH=32
+RSH=33
+USH=34
+LT=35
+LTE=36
+GT=37
+GTE=38
+EQ=39
+EQR=40
+NE=41
+NER=42
+BWAND=43
+XOR=44
+BWOR=45
+BOOLAND=46
+BOOLOR=47
+COND=48
+COLON=49
+REF=50
+ARROW=51
+INCR=52
+DECR=53
+ASSIGN=54
+AADD=55
+ASUB=56
+AMUL=57
+ADIV=58
+AREM=59
+AAND=60
+AXOR=61
+AOR=62
+ALSH=63
+ARSH=64
+AUSH=65
+OCTAL=66
+HEX=67
+INTEGER=68
+DECIMAL=69
+STRING=70
+TRUE=71
+FALSE=72
+NULL=73
+TYPE=74
+ID=75
+DOTINTEGER=76
+DOTID=77
 '{'=3
 '}'=4
 '['=5
@@ -95,47 +96,48 @@ DOTID=76
 'try'=21
 'catch'=22
 'throw'=23
-'!'=24
-'~'=25
-'*'=26
-'/'=27
-'%'=28
-'+'=29
-'-'=30
-'<<'=31
-'>>'=32
-'>>>'=33
-'<'=34
-'<='=35
-'>'=36
-'>='=37
-'=='=38
-'==='=39
-'!='=40
-'!=='=41
-'&'=42
-'^'=43
-'|'=44
-'&&'=45
-'||'=46
-'?'=47
-':'=48
-'::'=49
-'->'=50
-'++'=51
-'--'=52
-'='=53
-'+='=54
-'-='=55
-'*='=56
-'/='=57
-'%='=58
-'&='=59
-'^='=60
-'|='=61
-'<<='=62
-'>>='=63
-'>>>='=64
-'true'=70
-'false'=71
-'null'=72
+'this'=24
+'!'=25
+'~'=26
+'*'=27
+'/'=28
+'%'=29
+'+'=30
+'-'=31
+'<<'=32
+'>>'=33
+'>>>'=34
+'<'=35
+'<='=36
+'>'=37
+'>='=38
+'=='=39
+'==='=40
+'!='=41
+'!=='=42
+'&'=43
+'^'=44
+'|'=45
+'&&'=46
+'||'=47
+'?'=48
+':'=49
+'::'=50
+'->'=51
+'++'=52
+'--'=53
+'='=54
+'+='=55
+'-='=56
+'*='=57
+'/='=58
+'%='=59
+'&='=60
+'^='=61
+'|='=62
+'<<='=63
+'>>='=64
+'>>>='=65
+'true'=71
+'false'=72
+'null'=73
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index e60cbef79cf..102441effb1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -29,6 +29,7 @@ import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodHandles.Lookup;
 import java.lang.invoke.MethodType;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
@@ -318,7 +319,20 @@ public final class Def {
       */
      private static MethodHandle lookupReferenceInternal(Lookup lookup, Definition.Type clazz, String type,
                                                          String call, Class<?>... captures) throws LambdaConversionException {
-         FunctionRef ref = new FunctionRef(clazz, type, call, captures);
+         final FunctionRef ref;
+         if ("this".equals(type)) {
+             Method interfaceMethod = clazz.struct.getFunctionalMethod();
+             if (interfaceMethod == null) {
+                 throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
+                                                    "to [" + clazz.name + "], not a functional interface");
+             }
+             int arity = interfaceMethod.arguments.size();
+             // user written method
+             ref = null;
+         } else {
+             // whitelist lookup
+             ref = new FunctionRef(clazz, type, call, captures);
+         }
          final CallSite callSite;
          if (ref.needsBridges()) {
              callSite = LambdaMetafactory.altMetafactory(lookup, 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 8a91bd7d7fd..1c258fd971a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -27,6 +27,7 @@ import java.io.InputStreamReader;
 import java.io.LineNumberReader;
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
 import java.lang.reflect.Modifier;
 import java.nio.charset.StandardCharsets;
 import java.time.LocalDate;
@@ -198,6 +199,41 @@ public final class Definition {
             this.modifiers = modifiers;
             this.handle = handle;
         }
+        
+        public MethodType getMethodType() {
+            // we have a methodhandle already (e.g. whitelisted class)
+            // just return its type
+            if (handle != null) {
+                return handle.type();
+            }
+            // otherwise compute it
+            final Class<?> params[];
+            final Class<?> returnValue;
+            if (Modifier.isStatic(modifiers)) {
+                // static method: straightforward copy
+                params = new Class<?>[arguments.size()];
+                for (int i = 0; i < arguments.size(); i++) {
+                    params[i] = arguments.get(i).clazz;
+                }
+                returnValue = rtn.clazz;
+            } else if ("<init>".equals(name)) {
+                // constructor: returns the owner class
+                params = new Class<?>[arguments.size()];
+                for (int i = 0; i < arguments.size(); i++) {
+                    params[i] = arguments.get(i).clazz;
+                }
+                returnValue = owner.clazz;
+            } else {
+                // virtual/interface method: add receiver class
+                params = new Class<?>[1 + arguments.size()];
+                params[0] = owner.clazz;
+                for (int i = 0; i < arguments.size(); i++) {
+                    params[i + 1] = arguments.get(i).clazz;
+                }
+                returnValue = rtn.clazz;
+            }
+            return MethodType.methodType(returnValue, params);
+        }
     }
 
     public static final class Field {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
index fb0175cc97e..7e0fd520b75 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
@@ -49,14 +49,62 @@ public class FunctionRef {
     public final Handle implMethodASM;
     
     /**
-     * Creates a new FunctionRef.
+     * Creates a new FunctionRef, which will resolve {@code type::call} from the whitelist.
      * @param expected interface type to implement.
      * @param type the left hand side of a method reference expression
      * @param call the right hand side of a method reference expression
      * @param captures captured arguments
-     */
+     */    
     public FunctionRef(Definition.Type expected, String type, String call, Class<?>... captures) {
-        boolean isCtorReference = "new".equals(call);
+        this(expected, expected.struct.getFunctionalMethod(), lookup(expected, type, call, captures.length > 0), captures);
+    }
+
+    /**
+     * Creates a new FunctionRef (already resolved)
+     * @param expected interface type to implement
+     * @param method functional interface method
+     * @param impl implementation method
+     * @param captures captured arguments
+     */   
+    public FunctionRef(Definition.Type expected, Definition.Method method, Definition.Method impl, Class<?>... captures) {
+        // e.g. compareTo
+        invokedName = method.name;
+        // e.g. (Object)Comparator
+        invokedType = MethodType.methodType(expected.clazz, captures);
+        // e.g. (Object,Object)int
+        interfaceMethodType = method.getMethodType().dropParameterTypes(0, 1);
+
+        final int tag;
+        if ("<init>".equals(impl.name)) {
+            tag = Opcodes.H_NEWINVOKESPECIAL;
+        } else if (Modifier.isStatic(impl.modifiers)) {
+            tag = Opcodes.H_INVOKESTATIC;
+        } else if (impl.owner.clazz.isInterface()) {
+            tag = Opcodes.H_INVOKEINTERFACE;
+        } else {
+            tag = Opcodes.H_INVOKEVIRTUAL;
+        }
+        final String owner;
+        final boolean ownerIsInterface;
+        if (impl.owner == null) {
+            // owner == null: script class itself
+            ownerIsInterface = false;
+            owner = WriterConstants.CLASS_TYPE.getInternalName();
+        } else {
+            ownerIsInterface = impl.owner.clazz.isInterface();
+            owner = impl.owner.type.getInternalName();
+        }
+        implMethodASM = new Handle(tag, owner, impl.name, impl.method.getDescriptor(), ownerIsInterface);
+        implMethod = impl.handle;
+        
+        // remove any prepended captured arguments for the 'natural' signature.
+        samMethodType = impl.getMethodType().dropParameterTypes(0, captures.length);
+    }
+
+    /** 
+     * Looks up {@code type::call} from the whitelist, and returns a matching method.
+     */
+    private static Definition.Method lookup(Definition.Type expected, String type, String call, boolean receiverCaptured) {
         // check its really a functional interface
         // for e.g. Comparable
         Method method = expected.struct.getFunctionalMethod();
@@ -64,17 +112,12 @@ public class FunctionRef {
             throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
                                                "to [" + expected.name + "], not a functional interface");
         }
-        // e.g. compareTo
-        invokedName = method.name;
-        // e.g. (Object)Comparator
-        invokedType = MethodType.methodType(expected.clazz, captures);
-        // e.g. (Object,Object)int
-        interfaceMethodType = method.handle.type().dropParameterTypes(0, 1);
+
         // lookup requested method
         Definition.Struct struct = Definition.getType(type).struct;
         final Definition.Method impl;
         // ctor ref
-        if (isCtorReference) {
+        if ("new".equals(call)) {
             impl = struct.constructors.get(new Definition.MethodKey("<init>", method.arguments.size()));
         } else {
             // look for a static impl first
@@ -82,7 +125,7 @@ public class FunctionRef {
             if (staticImpl == null) {
                 // otherwise a virtual impl
                 final int arity;
-                if (captures.length > 0) {
+                if (receiverCaptured) {
                     // receiver captured
                     arity = method.arguments.size();
                 } else {
@@ -98,34 +141,9 @@ public class FunctionRef {
             throw new IllegalArgumentException("Unknown reference [" + type + "::" + call + "] matching " +
                                                "[" + expected + "]");
         }
-        
-        final int tag;
-        if (isCtorReference) {
-            tag = Opcodes.H_NEWINVOKESPECIAL;
-        } else if (Modifier.isStatic(impl.modifiers)) {
-            tag = Opcodes.H_INVOKESTATIC;
-        } else {
-            tag = Opcodes.H_INVOKEVIRTUAL;
-        }
-        if (impl.owner.clazz.isInterface()) {
-            implMethodASM = new Handle(tag, struct.type.getInternalName(), impl.name, impl.method.getDescriptor());
-        } else {
-            implMethodASM = new Handle(tag, impl.owner.type.getInternalName(), impl.name, impl.method.getDescriptor());
-        }
-        implMethod = impl.handle;
-        if (isCtorReference) {
-            samMethodType = MethodType.methodType(interfaceMethodType.returnType(), impl.handle.type().parameterArray());
-        } else if (Modifier.isStatic(impl.modifiers)) {
-            samMethodType = impl.handle.type();
-        } else if (captures.length > 0) {
-            // drop the receiver, we capture it
-            samMethodType = impl.handle.type().dropParameterTypes(0, 1);
-        } else {
-            // ensure the receiver type is exact and not a superclass type
-            samMethodType = impl.handle.type().changeParameterType(0, struct.clazz);
-        }
+        return impl;
     }
-    
+
     /** Returns true if you should ask LambdaMetaFactory to construct a bridge for the interface signature */
     public boolean needsBridges() {
         // currently if the interface differs, we ask for a bridge, but maybe we should do smarter checking?
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index 641a5582ab8..5f82cfad56c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -355,11 +355,11 @@ public final class MethodWriter extends GeneratorAdapter {
     }
 
     @Override
-    public void visitEnd() {
+    public void endMethod() {
         if (stringConcatArgs != null && !stringConcatArgs.isEmpty()) {
             throw new IllegalStateException("String concat bytecode not completed.");
         }
-        super.visitEnd();
+        super.endMethod();
     }
 
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index 4ec835b02c5..025e1433e93 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -71,7 +71,7 @@ public final class WriterConstants {
         MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, int.class, Object[].class);
     public final static Handle DEF_BOOTSTRAP_HANDLE =
         new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(DefBootstrap.class),
-            "bootstrap", DEF_BOOTSTRAP_TYPE.toMethodDescriptorString());
+            "bootstrap", DEF_BOOTSTRAP_TYPE.toMethodDescriptorString(), false);
 
     public final static Type DEF_UTIL_TYPE = Type.getType(Def.class);
     public final static Method DEF_TO_BOOLEAN         = getAsmMethod(boolean.class, "DefToboolean"       , Object.class);
@@ -114,7 +114,7 @@ public final class WriterConstants {
                                   MethodType.class, Object[].class);
     public final static Handle LAMBDA_BOOTSTRAP_HANDLE =
             new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(LambdaMetafactory.class),
-                "altMetafactory", LAMBDA_BOOTSTRAP_TYPE.toMethodDescriptorString());
+                "altMetafactory", LAMBDA_BOOTSTRAP_TYPE.toMethodDescriptorString(), false);
 
     /** dynamic invokedynamic bootstrap for indy string concats (Java 9+) */
     public final static Handle INDY_STRING_CONCAT_BOOTSTRAP_HANDLE;
@@ -126,7 +126,7 @@ public final class WriterConstants {
             final MethodType type = MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class);
             // ensure it is there:
             MethodHandles.publicLookup().findStatic(factory, methodName, type);
-            bs = new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(factory), methodName, type.toMethodDescriptorString());
+            bs = new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(factory), methodName, type.toMethodDescriptorString(), false);
         } catch (ReflectiveOperationException e) {
             // not Java 9 - we set it null, so MethodWriter uses StringBuilder:
             bs = null;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
index 566db679ef9..92f8e9a8f1d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
@@ -22,14 +22,14 @@ class PainlessLexer extends Lexer {
   public static final int
     WS=1, COMMENT=2, LBRACK=3, RBRACK=4, LBRACE=5, RBRACE=6, LP=7, RP=8, DOT=9, 
     COMMA=10, SEMICOLON=11, IF=12, ELSE=13, WHILE=14, DO=15, FOR=16, CONTINUE=17, 
-    BREAK=18, RETURN=19, NEW=20, TRY=21, CATCH=22, THROW=23, BOOLNOT=24, BWNOT=25, 
-    MUL=26, DIV=27, REM=28, ADD=29, SUB=30, LSH=31, RSH=32, USH=33, LT=34, 
-    LTE=35, GT=36, GTE=37, EQ=38, EQR=39, NE=40, NER=41, BWAND=42, XOR=43, 
-    BWOR=44, BOOLAND=45, BOOLOR=46, COND=47, COLON=48, REF=49, ARROW=50, INCR=51, 
-    DECR=52, ASSIGN=53, AADD=54, ASUB=55, AMUL=56, ADIV=57, AREM=58, AAND=59, 
-    AXOR=60, AOR=61, ALSH=62, ARSH=63, AUSH=64, OCTAL=65, HEX=66, INTEGER=67, 
-    DECIMAL=68, STRING=69, TRUE=70, FALSE=71, NULL=72, TYPE=73, ID=74, DOTINTEGER=75, 
-    DOTID=76;
+    BREAK=18, RETURN=19, NEW=20, TRY=21, CATCH=22, THROW=23, THIS=24, BOOLNOT=25, 
+    BWNOT=26, MUL=27, DIV=28, REM=29, ADD=30, SUB=31, LSH=32, RSH=33, USH=34, 
+    LT=35, LTE=36, GT=37, GTE=38, EQ=39, EQR=40, NE=41, NER=42, BWAND=43, 
+    XOR=44, BWOR=45, BOOLAND=46, BOOLOR=47, COND=48, COLON=49, REF=50, ARROW=51, 
+    INCR=52, DECR=53, ASSIGN=54, AADD=55, ASUB=56, AMUL=57, ADIV=58, AREM=59, 
+    AAND=60, AXOR=61, AOR=62, ALSH=63, ARSH=64, AUSH=65, OCTAL=66, HEX=67, 
+    INTEGER=68, DECIMAL=69, STRING=70, TRUE=71, FALSE=72, NULL=73, TYPE=74, 
+    ID=75, DOTINTEGER=76, DOTID=77;
   public static final int AFTER_DOT = 1;
   public static String[] modeNames = {
     "DEFAULT_MODE", "AFTER_DOT"
@@ -38,35 +38,35 @@ class PainlessLexer extends Lexer {
   public static final String[] ruleNames = {
     "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", "DOT", 
     "COMMA", "SEMICOLON", "IF", "ELSE", "WHILE", "DO", "FOR", "CONTINUE", 
-    "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "BOOLNOT", "BWNOT", 
-    "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", "LTE", "GT", 
-    "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", "BOOLAND", "BOOLOR", 
-    "COND", "COLON", "REF", "ARROW", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", 
-    "AMUL", "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", 
-    "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", 
-    "TYPE", "ID", "DOTINTEGER", "DOTID"
+    "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", "BOOLNOT", 
+    "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", 
+    "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", 
+    "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "INCR", "DECR", 
+    "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", "AAND", "AXOR", "AOR", 
+    "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", 
+    "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", "DOTID"
   };
 
   private static final String[] _LITERAL_NAMES = {
     null, null, null, "'{'", "'}'", "'['", "']'", "'('", "')'", "'.'", "','", 
     "';'", "'if'", "'else'", "'while'", "'do'", "'for'", "'continue'", "'break'", 
-    "'return'", "'new'", "'try'", "'catch'", "'throw'", "'!'", "'~'", "'*'", 
-    "'/'", "'%'", "'+'", "'-'", "'<<'", "'>>'", "'>>>'", "'<'", "'<='", "'>'", 
-    "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", "'^'", "'|'", "'&&'", 
-    "'||'", "'?'", "':'", "'::'", "'->'", "'++'", "'--'", "'='", "'+='", "'-='", 
-    "'*='", "'/='", "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", "'>>>='", 
-    null, null, null, null, null, "'true'", "'false'", "'null'"
+    "'return'", "'new'", "'try'", "'catch'", "'throw'", "'this'", "'!'", "'~'", 
+    "'*'", "'/'", "'%'", "'+'", "'-'", "'<<'", "'>>'", "'>>>'", "'<'", "'<='", 
+    "'>'", "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", "'^'", "'|'", 
+    "'&&'", "'||'", "'?'", "':'", "'::'", "'->'", "'++'", "'--'", "'='", "'+='", 
+    "'-='", "'*='", "'/='", "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", 
+    "'>>>='", null, null, null, null, null, "'true'", "'false'", "'null'"
   };
   private static final String[] _SYMBOLIC_NAMES = {
     null, "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", 
     "DOT", "COMMA", "SEMICOLON", "IF", "ELSE", "WHILE", "DO", "FOR", "CONTINUE", 
-    "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "BOOLNOT", "BWNOT", 
-    "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", "LTE", "GT", 
-    "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", "BOOLAND", "BOOLOR", 
-    "COND", "COLON", "REF", "ARROW", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", 
-    "AMUL", "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", 
-    "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", 
-    "TYPE", "ID", "DOTINTEGER", "DOTID"
+    "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", "BOOLNOT", 
+    "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", 
+    "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", 
+    "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "INCR", "DECR", 
+    "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", "AAND", "AXOR", "AOR", 
+    "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", 
+    "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", "DOTID"
   };
   public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
 
@@ -125,7 +125,7 @@ class PainlessLexer extends Lexer {
   @Override
   public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) {
     switch (ruleIndex) {
-    case 72:
+    case 73:
       return TYPE_sempred((RuleContext)_localctx, predIndex);
     }
     return true;
@@ -139,7 +139,7 @@ class PainlessLexer extends Lexer {
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2N\u0213\b\1\b\1\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2O\u021a\b\1\b\1\4"+
     "\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n"+
     "\4\13\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22"+
     "\t\22\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31"+
@@ -148,181 +148,184 @@ class PainlessLexer extends Lexer {
     "+\4,\t,\4-\t-\4.\t.\4/\t/\4\60\t\60\4\61\t\61\4\62\t\62\4\63\t\63\4\64"+
     "\t\64\4\65\t\65\4\66\t\66\4\67\t\67\48\t8\49\t9\4:\t:\4;\t;\4<\t<\4=\t"+
     "=\4>\t>\4?\t?\4@\t@\4A\tA\4B\tB\4C\tC\4D\tD\4E\tE\4F\tF\4G\tG\4H\tH\4"+
-    "I\tI\4J\tJ\4K\tK\4L\tL\4M\tM\3\2\6\2\u009e\n\2\r\2\16\2\u009f\3\2\3\2"+
-    "\3\3\3\3\3\3\3\3\7\3\u00a8\n\3\f\3\16\3\u00ab\13\3\3\3\3\3\3\3\3\3\3\3"+
-    "\7\3\u00b2\n\3\f\3\16\3\u00b5\13\3\3\3\3\3\5\3\u00b9\n\3\3\3\3\3\3\4\3"+
-    "\4\3\5\3\5\3\6\3\6\3\7\3\7\3\b\3\b\3\t\3\t\3\n\3\n\3\n\3\n\3\13\3\13\3"+
-    "\f\3\f\3\r\3\r\3\r\3\16\3\16\3\16\3\16\3\16\3\17\3\17\3\17\3\17\3\17\3"+
-    "\17\3\20\3\20\3\20\3\21\3\21\3\21\3\21\3\22\3\22\3\22\3\22\3\22\3\22\3"+
-    "\22\3\22\3\22\3\23\3\23\3\23\3\23\3\23\3\23\3\24\3\24\3\24\3\24\3\24\3"+
-    "\24\3\24\3\25\3\25\3\25\3\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\27\3"+
-    "\27\3\27\3\30\3\30\3\30\3\30\3\30\3\30\3\31\3\31\3\32\3\32\3\33\3\33\3"+
-    "\34\3\34\3\35\3\35\3\36\3\36\3\37\3\37\3 \3 \3 \3!\3!\3!\3\"\3\"\3\"\3"+
-    "\"\3#\3#\3$\3$\3$\3%\3%\3&\3&\3&\3\'\3\'\3\'\3(\3(\3(\3(\3)\3)\3)\3*\3"+
-    "*\3*\3*\3+\3+\3,\3,\3-\3-\3.\3.\3.\3/\3/\3/\3\60\3\60\3\61\3\61\3\62\3"+
-    "\62\3\62\3\63\3\63\3\63\3\64\3\64\3\64\3\65\3\65\3\65\3\66\3\66\3\67\3"+
-    "\67\3\67\38\38\38\39\39\39\3:\3:\3:\3;\3;\3;\3<\3<\3<\3=\3=\3=\3>\3>\3"+
-    ">\3?\3?\3?\3?\3@\3@\3@\3@\3A\3A\3A\3A\3A\3B\3B\6B\u0185\nB\rB\16B\u0186"+
-    "\3B\5B\u018a\nB\3C\3C\3C\6C\u018f\nC\rC\16C\u0190\3C\5C\u0194\nC\3D\3"+
-    "D\3D\7D\u0199\nD\fD\16D\u019c\13D\5D\u019e\nD\3D\5D\u01a1\nD\3E\3E\3E"+
-    "\7E\u01a6\nE\fE\16E\u01a9\13E\5E\u01ab\nE\3E\3E\6E\u01af\nE\rE\16E\u01b0"+
-    "\5E\u01b3\nE\3E\3E\5E\u01b7\nE\3E\6E\u01ba\nE\rE\16E\u01bb\5E\u01be\n"+
-    "E\3E\5E\u01c1\nE\3F\3F\3F\3F\3F\3F\7F\u01c9\nF\fF\16F\u01cc\13F\3F\3F"+
-    "\3F\3F\3F\3F\3F\7F\u01d5\nF\fF\16F\u01d8\13F\3F\5F\u01db\nF\3G\3G\3G\3"+
-    "G\3G\3H\3H\3H\3H\3H\3H\3I\3I\3I\3I\3I\3J\3J\3J\3J\7J\u01f1\nJ\fJ\16J\u01f4"+
-    "\13J\3J\3J\3K\3K\7K\u01fa\nK\fK\16K\u01fd\13K\3L\3L\3L\7L\u0202\nL\fL"+
-    "\16L\u0205\13L\5L\u0207\nL\3L\3L\3M\3M\7M\u020d\nM\fM\16M\u0210\13M\3"+
-    "M\3M\6\u00a9\u00b3\u01ca\u01d6\2N\4\3\6\4\b\5\n\6\f\7\16\b\20\t\22\n\24"+
-    "\13\26\f\30\r\32\16\34\17\36\20 \21\"\22$\23&\24(\25*\26,\27.\30\60\31"+
-    "\62\32\64\33\66\348\35:\36<\37> @!B\"D#F$H%J&L\'N(P)R*T+V,X-Z.\\/^\60"+
-    "`\61b\62d\63f\64h\65j\66l\67n8p9r:t;v<x=z>|?~@\u0080A\u0082B\u0084C\u0086"+
-    "D\u0088E\u008aF\u008cG\u008eH\u0090I\u0092J\u0094K\u0096L\u0098M\u009a"+
-    "N\4\2\3\21\5\2\13\f\17\17\"\"\4\2\f\f\17\17\3\2\629\4\2NNnn\4\2ZZzz\5"+
-    "\2\62;CHch\3\2\63;\3\2\62;\b\2FFHHNNffhhnn\4\2GGgg\4\2--//\4\2HHhh\4\2"+
-    "$$^^\5\2C\\aac|\6\2\62;C\\aac|\u0230\2\4\3\2\2\2\2\6\3\2\2\2\2\b\3\2\2"+
-    "\2\2\n\3\2\2\2\2\f\3\2\2\2\2\16\3\2\2\2\2\20\3\2\2\2\2\22\3\2\2\2\2\24"+
-    "\3\2\2\2\2\26\3\2\2\2\2\30\3\2\2\2\2\32\3\2\2\2\2\34\3\2\2\2\2\36\3\2"+
-    "\2\2\2 \3\2\2\2\2\"\3\2\2\2\2$\3\2\2\2\2&\3\2\2\2\2(\3\2\2\2\2*\3\2\2"+
-    "\2\2,\3\2\2\2\2.\3\2\2\2\2\60\3\2\2\2\2\62\3\2\2\2\2\64\3\2\2\2\2\66\3"+
-    "\2\2\2\28\3\2\2\2\2:\3\2\2\2\2<\3\2\2\2\2>\3\2\2\2\2@\3\2\2\2\2B\3\2\2"+
-    "\2\2D\3\2\2\2\2F\3\2\2\2\2H\3\2\2\2\2J\3\2\2\2\2L\3\2\2\2\2N\3\2\2\2\2"+
-    "P\3\2\2\2\2R\3\2\2\2\2T\3\2\2\2\2V\3\2\2\2\2X\3\2\2\2\2Z\3\2\2\2\2\\\3"+
-    "\2\2\2\2^\3\2\2\2\2`\3\2\2\2\2b\3\2\2\2\2d\3\2\2\2\2f\3\2\2\2\2h\3\2\2"+
-    "\2\2j\3\2\2\2\2l\3\2\2\2\2n\3\2\2\2\2p\3\2\2\2\2r\3\2\2\2\2t\3\2\2\2\2"+
-    "v\3\2\2\2\2x\3\2\2\2\2z\3\2\2\2\2|\3\2\2\2\2~\3\2\2\2\2\u0080\3\2\2\2"+
-    "\2\u0082\3\2\2\2\2\u0084\3\2\2\2\2\u0086\3\2\2\2\2\u0088\3\2\2\2\2\u008a"+
-    "\3\2\2\2\2\u008c\3\2\2\2\2\u008e\3\2\2\2\2\u0090\3\2\2\2\2\u0092\3\2\2"+
-    "\2\2\u0094\3\2\2\2\2\u0096\3\2\2\2\3\u0098\3\2\2\2\3\u009a\3\2\2\2\4\u009d"+
-    "\3\2\2\2\6\u00b8\3\2\2\2\b\u00bc\3\2\2\2\n\u00be\3\2\2\2\f\u00c0\3\2\2"+
-    "\2\16\u00c2\3\2\2\2\20\u00c4\3\2\2\2\22\u00c6\3\2\2\2\24\u00c8\3\2\2\2"+
-    "\26\u00cc\3\2\2\2\30\u00ce\3\2\2\2\32\u00d0\3\2\2\2\34\u00d3\3\2\2\2\36"+
-    "\u00d8\3\2\2\2 \u00de\3\2\2\2\"\u00e1\3\2\2\2$\u00e5\3\2\2\2&\u00ee\3"+
-    "\2\2\2(\u00f4\3\2\2\2*\u00fb\3\2\2\2,\u00ff\3\2\2\2.\u0103\3\2\2\2\60"+
-    "\u0109\3\2\2\2\62\u010f\3\2\2\2\64\u0111\3\2\2\2\66\u0113\3\2\2\28\u0115"+
-    "\3\2\2\2:\u0117\3\2\2\2<\u0119\3\2\2\2>\u011b\3\2\2\2@\u011d\3\2\2\2B"+
-    "\u0120\3\2\2\2D\u0123\3\2\2\2F\u0127\3\2\2\2H\u0129\3\2\2\2J\u012c\3\2"+
-    "\2\2L\u012e\3\2\2\2N\u0131\3\2\2\2P\u0134\3\2\2\2R\u0138\3\2\2\2T\u013b"+
-    "\3\2\2\2V\u013f\3\2\2\2X\u0141\3\2\2\2Z\u0143\3\2\2\2\\\u0145\3\2\2\2"+
-    "^\u0148\3\2\2\2`\u014b\3\2\2\2b\u014d\3\2\2\2d\u014f\3\2\2\2f\u0152\3"+
-    "\2\2\2h\u0155\3\2\2\2j\u0158\3\2\2\2l\u015b\3\2\2\2n\u015d\3\2\2\2p\u0160"+
-    "\3\2\2\2r\u0163\3\2\2\2t\u0166\3\2\2\2v\u0169\3\2\2\2x\u016c\3\2\2\2z"+
-    "\u016f\3\2\2\2|\u0172\3\2\2\2~\u0175\3\2\2\2\u0080\u0179\3\2\2\2\u0082"+
-    "\u017d\3\2\2\2\u0084\u0182\3\2\2\2\u0086\u018b\3\2\2\2\u0088\u019d\3\2"+
-    "\2\2\u008a\u01aa\3\2\2\2\u008c\u01da\3\2\2\2\u008e\u01dc\3\2\2\2\u0090"+
-    "\u01e1\3\2\2\2\u0092\u01e7\3\2\2\2\u0094\u01ec\3\2\2\2\u0096\u01f7\3\2"+
-    "\2\2\u0098\u0206\3\2\2\2\u009a\u020a\3\2\2\2\u009c\u009e\t\2\2\2\u009d"+
-    "\u009c\3\2\2\2\u009e\u009f\3\2\2\2\u009f\u009d\3\2\2\2\u009f\u00a0\3\2"+
-    "\2\2\u00a0\u00a1\3\2\2\2\u00a1\u00a2\b\2\2\2\u00a2\5\3\2\2\2\u00a3\u00a4"+
-    "\7\61\2\2\u00a4\u00a5\7\61\2\2\u00a5\u00a9\3\2\2\2\u00a6\u00a8\13\2\2"+
-    "\2\u00a7\u00a6\3\2\2\2\u00a8\u00ab\3\2\2\2\u00a9\u00aa\3\2\2\2\u00a9\u00a7"+
-    "\3\2\2\2\u00aa\u00ac\3\2\2\2\u00ab\u00a9\3\2\2\2\u00ac\u00b9\t\3\2\2\u00ad"+
-    "\u00ae\7\61\2\2\u00ae\u00af\7,\2\2\u00af\u00b3\3\2\2\2\u00b0\u00b2\13"+
-    "\2\2\2\u00b1\u00b0\3\2\2\2\u00b2\u00b5\3\2\2\2\u00b3\u00b4\3\2\2\2\u00b3"+
-    "\u00b1\3\2\2\2\u00b4\u00b6\3\2\2\2\u00b5\u00b3\3\2\2\2\u00b6\u00b7\7,"+
-    "\2\2\u00b7\u00b9\7\61\2\2\u00b8\u00a3\3\2\2\2\u00b8\u00ad\3\2\2\2\u00b9"+
-    "\u00ba\3\2\2\2\u00ba\u00bb\b\3\2\2\u00bb\7\3\2\2\2\u00bc\u00bd\7}\2\2"+
-    "\u00bd\t\3\2\2\2\u00be\u00bf\7\177\2\2\u00bf\13\3\2\2\2\u00c0\u00c1\7"+
-    "]\2\2\u00c1\r\3\2\2\2\u00c2\u00c3\7_\2\2\u00c3\17\3\2\2\2\u00c4\u00c5"+
-    "\7*\2\2\u00c5\21\3\2\2\2\u00c6\u00c7\7+\2\2\u00c7\23\3\2\2\2\u00c8\u00c9"+
-    "\7\60\2\2\u00c9\u00ca\3\2\2\2\u00ca\u00cb\b\n\3\2\u00cb\25\3\2\2\2\u00cc"+
-    "\u00cd\7.\2\2\u00cd\27\3\2\2\2\u00ce\u00cf\7=\2\2\u00cf\31\3\2\2\2\u00d0"+
-    "\u00d1\7k\2\2\u00d1\u00d2\7h\2\2\u00d2\33\3\2\2\2\u00d3\u00d4\7g\2\2\u00d4"+
-    "\u00d5\7n\2\2\u00d5\u00d6\7u\2\2\u00d6\u00d7\7g\2\2\u00d7\35\3\2\2\2\u00d8"+
-    "\u00d9\7y\2\2\u00d9\u00da\7j\2\2\u00da\u00db\7k\2\2\u00db\u00dc\7n\2\2"+
-    "\u00dc\u00dd\7g\2\2\u00dd\37\3\2\2\2\u00de\u00df\7f\2\2\u00df\u00e0\7"+
-    "q\2\2\u00e0!\3\2\2\2\u00e1\u00e2\7h\2\2\u00e2\u00e3\7q\2\2\u00e3\u00e4"+
-    "\7t\2\2\u00e4#\3\2\2\2\u00e5\u00e6\7e\2\2\u00e6\u00e7\7q\2\2\u00e7\u00e8"+
-    "\7p\2\2\u00e8\u00e9\7v\2\2\u00e9\u00ea\7k\2\2\u00ea\u00eb\7p\2\2\u00eb"+
-    "\u00ec\7w\2\2\u00ec\u00ed\7g\2\2\u00ed%\3\2\2\2\u00ee\u00ef\7d\2\2\u00ef"+
-    "\u00f0\7t\2\2\u00f0\u00f1\7g\2\2\u00f1\u00f2\7c\2\2\u00f2\u00f3\7m\2\2"+
-    "\u00f3\'\3\2\2\2\u00f4\u00f5\7t\2\2\u00f5\u00f6\7g\2\2\u00f6\u00f7\7v"+
-    "\2\2\u00f7\u00f8\7w\2\2\u00f8\u00f9\7t\2\2\u00f9\u00fa\7p\2\2\u00fa)\3"+
-    "\2\2\2\u00fb\u00fc\7p\2\2\u00fc\u00fd\7g\2\2\u00fd\u00fe\7y\2\2\u00fe"+
-    "+\3\2\2\2\u00ff\u0100\7v\2\2\u0100\u0101\7t\2\2\u0101\u0102\7{\2\2\u0102"+
-    "-\3\2\2\2\u0103\u0104\7e\2\2\u0104\u0105\7c\2\2\u0105\u0106\7v\2\2\u0106"+
-    "\u0107\7e\2\2\u0107\u0108\7j\2\2\u0108/\3\2\2\2\u0109\u010a\7v\2\2\u010a"+
-    "\u010b\7j\2\2\u010b\u010c\7t\2\2\u010c\u010d\7q\2\2\u010d\u010e\7y\2\2"+
-    "\u010e\61\3\2\2\2\u010f\u0110\7#\2\2\u0110\63\3\2\2\2\u0111\u0112\7\u0080"+
-    "\2\2\u0112\65\3\2\2\2\u0113\u0114\7,\2\2\u0114\67\3\2\2\2\u0115\u0116"+
-    "\7\61\2\2\u01169\3\2\2\2\u0117\u0118\7\'\2\2\u0118;\3\2\2\2\u0119\u011a"+
-    "\7-\2\2\u011a=\3\2\2\2\u011b\u011c\7/\2\2\u011c?\3\2\2\2\u011d\u011e\7"+
-    ">\2\2\u011e\u011f\7>\2\2\u011fA\3\2\2\2\u0120\u0121\7@\2\2\u0121\u0122"+
-    "\7@\2\2\u0122C\3\2\2\2\u0123\u0124\7@\2\2\u0124\u0125\7@\2\2\u0125\u0126"+
-    "\7@\2\2\u0126E\3\2\2\2\u0127\u0128\7>\2\2\u0128G\3\2\2\2\u0129\u012a\7"+
-    ">\2\2\u012a\u012b\7?\2\2\u012bI\3\2\2\2\u012c\u012d\7@\2\2\u012dK\3\2"+
-    "\2\2\u012e\u012f\7@\2\2\u012f\u0130\7?\2\2\u0130M\3\2\2\2\u0131\u0132"+
-    "\7?\2\2\u0132\u0133\7?\2\2\u0133O\3\2\2\2\u0134\u0135\7?\2\2\u0135\u0136"+
-    "\7?\2\2\u0136\u0137\7?\2\2\u0137Q\3\2\2\2\u0138\u0139\7#\2\2\u0139\u013a"+
-    "\7?\2\2\u013aS\3\2\2\2\u013b\u013c\7#\2\2\u013c\u013d\7?\2\2\u013d\u013e"+
-    "\7?\2\2\u013eU\3\2\2\2\u013f\u0140\7(\2\2\u0140W\3\2\2\2\u0141\u0142\7"+
-    "`\2\2\u0142Y\3\2\2\2\u0143\u0144\7~\2\2\u0144[\3\2\2\2\u0145\u0146\7("+
-    "\2\2\u0146\u0147\7(\2\2\u0147]\3\2\2\2\u0148\u0149\7~\2\2\u0149\u014a"+
-    "\7~\2\2\u014a_\3\2\2\2\u014b\u014c\7A\2\2\u014ca\3\2\2\2\u014d\u014e\7"+
-    "<\2\2\u014ec\3\2\2\2\u014f\u0150\7<\2\2\u0150\u0151\7<\2\2\u0151e\3\2"+
-    "\2\2\u0152\u0153\7/\2\2\u0153\u0154\7@\2\2\u0154g\3\2\2\2\u0155\u0156"+
-    "\7-\2\2\u0156\u0157\7-\2\2\u0157i\3\2\2\2\u0158\u0159\7/\2\2\u0159\u015a"+
-    "\7/\2\2\u015ak\3\2\2\2\u015b\u015c\7?\2\2\u015cm\3\2\2\2\u015d\u015e\7"+
-    "-\2\2\u015e\u015f\7?\2\2\u015fo\3\2\2\2\u0160\u0161\7/\2\2\u0161\u0162"+
-    "\7?\2\2\u0162q\3\2\2\2\u0163\u0164\7,\2\2\u0164\u0165\7?\2\2\u0165s\3"+
-    "\2\2\2\u0166\u0167\7\61\2\2\u0167\u0168\7?\2\2\u0168u\3\2\2\2\u0169\u016a"+
-    "\7\'\2\2\u016a\u016b\7?\2\2\u016bw\3\2\2\2\u016c\u016d\7(\2\2\u016d\u016e"+
-    "\7?\2\2\u016ey\3\2\2\2\u016f\u0170\7`\2\2\u0170\u0171\7?\2\2\u0171{\3"+
-    "\2\2\2\u0172\u0173\7~\2\2\u0173\u0174\7?\2\2\u0174}\3\2\2\2\u0175\u0176"+
-    "\7>\2\2\u0176\u0177\7>\2\2\u0177\u0178\7?\2\2\u0178\177\3\2\2\2\u0179"+
-    "\u017a\7@\2\2\u017a\u017b\7@\2\2\u017b\u017c\7?\2\2\u017c\u0081\3\2\2"+
-    "\2\u017d\u017e\7@\2\2\u017e\u017f\7@\2\2\u017f\u0180\7@\2\2\u0180\u0181"+
-    "\7?\2\2\u0181\u0083\3\2\2\2\u0182\u0184\7\62\2\2\u0183\u0185\t\4\2\2\u0184"+
-    "\u0183\3\2\2\2\u0185\u0186\3\2\2\2\u0186\u0184\3\2\2\2\u0186\u0187\3\2"+
-    "\2\2\u0187\u0189\3\2\2\2\u0188\u018a\t\5\2\2\u0189\u0188\3\2\2\2\u0189"+
-    "\u018a\3\2\2\2\u018a\u0085\3\2\2\2\u018b\u018c\7\62\2\2\u018c\u018e\t"+
-    "\6\2\2\u018d\u018f\t\7\2\2\u018e\u018d\3\2\2\2\u018f\u0190\3\2\2\2\u0190"+
-    "\u018e\3\2\2\2\u0190\u0191\3\2\2\2\u0191\u0193\3\2\2\2\u0192\u0194\t\5"+
-    "\2\2\u0193\u0192\3\2\2\2\u0193\u0194\3\2\2\2\u0194\u0087\3\2\2\2\u0195"+
-    "\u019e\7\62\2\2\u0196\u019a\t\b\2\2\u0197\u0199\t\t\2\2\u0198\u0197\3"+
-    "\2\2\2\u0199\u019c\3\2\2\2\u019a\u0198\3\2\2\2\u019a\u019b\3\2\2\2\u019b"+
-    "\u019e\3\2\2\2\u019c\u019a\3\2\2\2\u019d\u0195\3\2\2\2\u019d\u0196\3\2"+
-    "\2\2\u019e\u01a0\3\2\2\2\u019f\u01a1\t\n\2\2\u01a0\u019f\3\2\2\2\u01a0"+
-    "\u01a1\3\2\2\2\u01a1\u0089\3\2\2\2\u01a2\u01ab\7\62\2\2\u01a3\u01a7\t"+
-    "\b\2\2\u01a4\u01a6\t\t\2\2\u01a5\u01a4\3\2\2\2\u01a6\u01a9\3\2\2\2\u01a7"+
-    "\u01a5\3\2\2\2\u01a7\u01a8\3\2\2\2\u01a8\u01ab\3\2\2\2\u01a9\u01a7\3\2"+
-    "\2\2\u01aa\u01a2\3\2\2\2\u01aa\u01a3\3\2\2\2\u01ab\u01b2\3\2\2\2\u01ac"+
-    "\u01ae\5\24\n\2\u01ad\u01af\t\t\2\2\u01ae\u01ad\3\2\2\2\u01af\u01b0\3"+
-    "\2\2\2\u01b0\u01ae\3\2\2\2\u01b0\u01b1\3\2\2\2\u01b1\u01b3\3\2\2\2\u01b2"+
-    "\u01ac\3\2\2\2\u01b2\u01b3\3\2\2\2\u01b3\u01bd\3\2\2\2\u01b4\u01b6\t\13"+
-    "\2\2\u01b5\u01b7\t\f\2\2\u01b6\u01b5\3\2\2\2\u01b6\u01b7\3\2\2\2\u01b7"+
-    "\u01b9\3\2\2\2\u01b8\u01ba\t\t\2\2\u01b9\u01b8\3\2\2\2\u01ba\u01bb\3\2"+
-    "\2\2\u01bb\u01b9\3\2\2\2\u01bb\u01bc\3\2\2\2\u01bc\u01be\3\2\2\2\u01bd"+
-    "\u01b4\3\2\2\2\u01bd\u01be\3\2\2\2\u01be\u01c0\3\2\2\2\u01bf\u01c1\t\r"+
-    "\2\2\u01c0\u01bf\3\2\2\2\u01c0\u01c1\3\2\2\2\u01c1\u008b\3\2\2\2\u01c2"+
-    "\u01ca\7$\2\2\u01c3\u01c4\7^\2\2\u01c4\u01c9\7$\2\2\u01c5\u01c6\7^\2\2"+
-    "\u01c6\u01c9\7^\2\2\u01c7\u01c9\n\16\2\2\u01c8\u01c3\3\2\2\2\u01c8\u01c5"+
-    "\3\2\2\2\u01c8\u01c7\3\2\2\2\u01c9\u01cc\3\2\2\2\u01ca\u01cb\3\2\2\2\u01ca"+
-    "\u01c8\3\2\2\2\u01cb\u01cd\3\2\2\2\u01cc\u01ca\3\2\2\2\u01cd\u01db\7$"+
-    "\2\2\u01ce\u01d6\7)\2\2\u01cf\u01d0\7^\2\2\u01d0\u01d5\7)\2\2\u01d1\u01d2"+
-    "\7^\2\2\u01d2\u01d5\7^\2\2\u01d3\u01d5\n\16\2\2\u01d4\u01cf\3\2\2\2\u01d4"+
-    "\u01d1\3\2\2\2\u01d4\u01d3\3\2\2\2\u01d5\u01d8\3\2\2\2\u01d6\u01d7\3\2"+
-    "\2\2\u01d6\u01d4\3\2\2\2\u01d7\u01d9\3\2\2\2\u01d8\u01d6\3\2\2\2\u01d9"+
-    "\u01db\7)\2\2\u01da\u01c2\3\2\2\2\u01da\u01ce\3\2\2\2\u01db\u008d\3\2"+
-    "\2\2\u01dc\u01dd\7v\2\2\u01dd\u01de\7t\2\2\u01de\u01df\7w\2\2\u01df\u01e0"+
-    "\7g\2\2\u01e0\u008f\3\2\2\2\u01e1\u01e2\7h\2\2\u01e2\u01e3\7c\2\2\u01e3"+
-    "\u01e4\7n\2\2\u01e4\u01e5\7u\2\2\u01e5\u01e6\7g\2\2\u01e6\u0091\3\2\2"+
-    "\2\u01e7\u01e8\7p\2\2\u01e8\u01e9\7w\2\2\u01e9\u01ea\7n\2\2\u01ea\u01eb"+
-    "\7n\2\2\u01eb\u0093\3\2\2\2\u01ec\u01f2\5\u0096K\2\u01ed\u01ee\5\24\n"+
-    "\2\u01ee\u01ef\5\u0096K\2\u01ef\u01f1\3\2\2\2\u01f0\u01ed\3\2\2\2\u01f1"+
-    "\u01f4\3\2\2\2\u01f2\u01f0\3\2\2\2\u01f2\u01f3\3\2\2\2\u01f3\u01f5\3\2"+
-    "\2\2\u01f4\u01f2\3\2\2\2\u01f5\u01f6\6J\2\2\u01f6\u0095\3\2\2\2\u01f7"+
-    "\u01fb\t\17\2\2\u01f8\u01fa\t\20\2\2\u01f9\u01f8\3\2\2\2\u01fa\u01fd\3"+
-    "\2\2\2\u01fb\u01f9\3\2\2\2\u01fb\u01fc\3\2\2\2\u01fc\u0097\3\2\2\2\u01fd"+
-    "\u01fb\3\2\2\2\u01fe\u0207\7\62\2\2\u01ff\u0203\t\b\2\2\u0200\u0202\t"+
-    "\t\2\2\u0201\u0200\3\2\2\2\u0202\u0205\3\2\2\2\u0203\u0201\3\2\2\2\u0203"+
-    "\u0204\3\2\2\2\u0204\u0207\3\2\2\2\u0205\u0203\3\2\2\2\u0206\u01fe\3\2"+
-    "\2\2\u0206\u01ff\3\2\2\2\u0207\u0208\3\2\2\2\u0208\u0209\bL\4\2\u0209"+
-    "\u0099\3\2\2\2\u020a\u020e\t\17\2\2\u020b\u020d\t\20\2\2\u020c\u020b\3"+
-    "\2\2\2\u020d\u0210\3\2\2\2\u020e\u020c\3\2\2\2\u020e\u020f\3\2\2\2\u020f"+
-    "\u0211\3\2\2\2\u0210\u020e\3\2\2\2\u0211\u0212\bM\4\2\u0212\u009b\3\2"+
-    "\2\2!\2\3\u009f\u00a9\u00b3\u00b8\u0186\u0189\u0190\u0193\u019a\u019d"+
-    "\u01a0\u01a7\u01aa\u01b0\u01b2\u01b6\u01bb\u01bd\u01c0\u01c8\u01ca\u01d4"+
-    "\u01d6\u01da\u01f2\u01fb\u0203\u0206\u020e\5\b\2\2\4\3\2\4\2\2";
+    "I\tI\4J\tJ\4K\tK\4L\tL\4M\tM\4N\tN\3\2\6\2\u00a0\n\2\r\2\16\2\u00a1\3"+
+    "\2\3\2\3\3\3\3\3\3\3\3\7\3\u00aa\n\3\f\3\16\3\u00ad\13\3\3\3\3\3\3\3\3"+
+    "\3\3\3\7\3\u00b4\n\3\f\3\16\3\u00b7\13\3\3\3\3\3\5\3\u00bb\n\3\3\3\3\3"+
+    "\3\4\3\4\3\5\3\5\3\6\3\6\3\7\3\7\3\b\3\b\3\t\3\t\3\n\3\n\3\n\3\n\3\13"+
+    "\3\13\3\f\3\f\3\r\3\r\3\r\3\16\3\16\3\16\3\16\3\16\3\17\3\17\3\17\3\17"+
+    "\3\17\3\17\3\20\3\20\3\20\3\21\3\21\3\21\3\21\3\22\3\22\3\22\3\22\3\22"+
+    "\3\22\3\22\3\22\3\22\3\23\3\23\3\23\3\23\3\23\3\23\3\24\3\24\3\24\3\24"+
+    "\3\24\3\24\3\24\3\25\3\25\3\25\3\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27"+
+    "\3\27\3\27\3\27\3\30\3\30\3\30\3\30\3\30\3\30\3\31\3\31\3\31\3\31\3\31"+
+    "\3\32\3\32\3\33\3\33\3\34\3\34\3\35\3\35\3\36\3\36\3\37\3\37\3 \3 \3!"+
+    "\3!\3!\3\"\3\"\3\"\3#\3#\3#\3#\3$\3$\3%\3%\3%\3&\3&\3\'\3\'\3\'\3(\3("+
+    "\3(\3)\3)\3)\3)\3*\3*\3*\3+\3+\3+\3+\3,\3,\3-\3-\3.\3.\3/\3/\3/\3\60\3"+
+    "\60\3\60\3\61\3\61\3\62\3\62\3\63\3\63\3\63\3\64\3\64\3\64\3\65\3\65\3"+
+    "\65\3\66\3\66\3\66\3\67\3\67\38\38\38\39\39\39\3:\3:\3:\3;\3;\3;\3<\3"+
+    "<\3<\3=\3=\3=\3>\3>\3>\3?\3?\3?\3@\3@\3@\3@\3A\3A\3A\3A\3B\3B\3B\3B\3"+
+    "B\3C\3C\6C\u018c\nC\rC\16C\u018d\3C\5C\u0191\nC\3D\3D\3D\6D\u0196\nD\r"+
+    "D\16D\u0197\3D\5D\u019b\nD\3E\3E\3E\7E\u01a0\nE\fE\16E\u01a3\13E\5E\u01a5"+
+    "\nE\3E\5E\u01a8\nE\3F\3F\3F\7F\u01ad\nF\fF\16F\u01b0\13F\5F\u01b2\nF\3"+
+    "F\3F\6F\u01b6\nF\rF\16F\u01b7\5F\u01ba\nF\3F\3F\5F\u01be\nF\3F\6F\u01c1"+
+    "\nF\rF\16F\u01c2\5F\u01c5\nF\3F\5F\u01c8\nF\3G\3G\3G\3G\3G\3G\7G\u01d0"+
+    "\nG\fG\16G\u01d3\13G\3G\3G\3G\3G\3G\3G\3G\7G\u01dc\nG\fG\16G\u01df\13"+
+    "G\3G\5G\u01e2\nG\3H\3H\3H\3H\3H\3I\3I\3I\3I\3I\3I\3J\3J\3J\3J\3J\3K\3"+
+    "K\3K\3K\7K\u01f8\nK\fK\16K\u01fb\13K\3K\3K\3L\3L\7L\u0201\nL\fL\16L\u0204"+
+    "\13L\3M\3M\3M\7M\u0209\nM\fM\16M\u020c\13M\5M\u020e\nM\3M\3M\3N\3N\7N"+
+    "\u0214\nN\fN\16N\u0217\13N\3N\3N\6\u00ab\u00b5\u01d1\u01dd\2O\4\3\6\4"+
+    "\b\5\n\6\f\7\16\b\20\t\22\n\24\13\26\f\30\r\32\16\34\17\36\20 \21\"\22"+
+    "$\23&\24(\25*\26,\27.\30\60\31\62\32\64\33\66\348\35:\36<\37> @!B\"D#"+
+    "F$H%J&L\'N(P)R*T+V,X-Z.\\/^\60`\61b\62d\63f\64h\65j\66l\67n8p9r:t;v<x"+
+    "=z>|?~@\u0080A\u0082B\u0084C\u0086D\u0088E\u008aF\u008cG\u008eH\u0090"+
+    "I\u0092J\u0094K\u0096L\u0098M\u009aN\u009cO\4\2\3\21\5\2\13\f\17\17\""+
+    "\"\4\2\f\f\17\17\3\2\629\4\2NNnn\4\2ZZzz\5\2\62;CHch\3\2\63;\3\2\62;\b"+
+    "\2FFHHNNffhhnn\4\2GGgg\4\2--//\4\2HHhh\4\2$$^^\5\2C\\aac|\6\2\62;C\\a"+
+    "ac|\u0237\2\4\3\2\2\2\2\6\3\2\2\2\2\b\3\2\2\2\2\n\3\2\2\2\2\f\3\2\2\2"+
+    "\2\16\3\2\2\2\2\20\3\2\2\2\2\22\3\2\2\2\2\24\3\2\2\2\2\26\3\2\2\2\2\30"+
+    "\3\2\2\2\2\32\3\2\2\2\2\34\3\2\2\2\2\36\3\2\2\2\2 \3\2\2\2\2\"\3\2\2\2"+
+    "\2$\3\2\2\2\2&\3\2\2\2\2(\3\2\2\2\2*\3\2\2\2\2,\3\2\2\2\2.\3\2\2\2\2\60"+
+    "\3\2\2\2\2\62\3\2\2\2\2\64\3\2\2\2\2\66\3\2\2\2\28\3\2\2\2\2:\3\2\2\2"+
+    "\2<\3\2\2\2\2>\3\2\2\2\2@\3\2\2\2\2B\3\2\2\2\2D\3\2\2\2\2F\3\2\2\2\2H"+
+    "\3\2\2\2\2J\3\2\2\2\2L\3\2\2\2\2N\3\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2T\3\2"+
+    "\2\2\2V\3\2\2\2\2X\3\2\2\2\2Z\3\2\2\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3\2\2"+
+    "\2\2b\3\2\2\2\2d\3\2\2\2\2f\3\2\2\2\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2\2\2"+
+    "n\3\2\2\2\2p\3\2\2\2\2r\3\2\2\2\2t\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2z\3"+
+    "\2\2\2\2|\3\2\2\2\2~\3\2\2\2\2\u0080\3\2\2\2\2\u0082\3\2\2\2\2\u0084\3"+
+    "\2\2\2\2\u0086\3\2\2\2\2\u0088\3\2\2\2\2\u008a\3\2\2\2\2\u008c\3\2\2\2"+
+    "\2\u008e\3\2\2\2\2\u0090\3\2\2\2\2\u0092\3\2\2\2\2\u0094\3\2\2\2\2\u0096"+
+    "\3\2\2\2\2\u0098\3\2\2\2\3\u009a\3\2\2\2\3\u009c\3\2\2\2\4\u009f\3\2\2"+
+    "\2\6\u00ba\3\2\2\2\b\u00be\3\2\2\2\n\u00c0\3\2\2\2\f\u00c2\3\2\2\2\16"+
+    "\u00c4\3\2\2\2\20\u00c6\3\2\2\2\22\u00c8\3\2\2\2\24\u00ca\3\2\2\2\26\u00ce"+
+    "\3\2\2\2\30\u00d0\3\2\2\2\32\u00d2\3\2\2\2\34\u00d5\3\2\2\2\36\u00da\3"+
+    "\2\2\2 \u00e0\3\2\2\2\"\u00e3\3\2\2\2$\u00e7\3\2\2\2&\u00f0\3\2\2\2(\u00f6"+
+    "\3\2\2\2*\u00fd\3\2\2\2,\u0101\3\2\2\2.\u0105\3\2\2\2\60\u010b\3\2\2\2"+
+    "\62\u0111\3\2\2\2\64\u0116\3\2\2\2\66\u0118\3\2\2\28\u011a\3\2\2\2:\u011c"+
+    "\3\2\2\2<\u011e\3\2\2\2>\u0120\3\2\2\2@\u0122\3\2\2\2B\u0124\3\2\2\2D"+
+    "\u0127\3\2\2\2F\u012a\3\2\2\2H\u012e\3\2\2\2J\u0130\3\2\2\2L\u0133\3\2"+
+    "\2\2N\u0135\3\2\2\2P\u0138\3\2\2\2R\u013b\3\2\2\2T\u013f\3\2\2\2V\u0142"+
+    "\3\2\2\2X\u0146\3\2\2\2Z\u0148\3\2\2\2\\\u014a\3\2\2\2^\u014c\3\2\2\2"+
+    "`\u014f\3\2\2\2b\u0152\3\2\2\2d\u0154\3\2\2\2f\u0156\3\2\2\2h\u0159\3"+
+    "\2\2\2j\u015c\3\2\2\2l\u015f\3\2\2\2n\u0162\3\2\2\2p\u0164\3\2\2\2r\u0167"+
+    "\3\2\2\2t\u016a\3\2\2\2v\u016d\3\2\2\2x\u0170\3\2\2\2z\u0173\3\2\2\2|"+
+    "\u0176\3\2\2\2~\u0179\3\2\2\2\u0080\u017c\3\2\2\2\u0082\u0180\3\2\2\2"+
+    "\u0084\u0184\3\2\2\2\u0086\u0189\3\2\2\2\u0088\u0192\3\2\2\2\u008a\u01a4"+
+    "\3\2\2\2\u008c\u01b1\3\2\2\2\u008e\u01e1\3\2\2\2\u0090\u01e3\3\2\2\2\u0092"+
+    "\u01e8\3\2\2\2\u0094\u01ee\3\2\2\2\u0096\u01f3\3\2\2\2\u0098\u01fe\3\2"+
+    "\2\2\u009a\u020d\3\2\2\2\u009c\u0211\3\2\2\2\u009e\u00a0\t\2\2\2\u009f"+
+    "\u009e\3\2\2\2\u00a0\u00a1\3\2\2\2\u00a1\u009f\3\2\2\2\u00a1\u00a2\3\2"+
+    "\2\2\u00a2\u00a3\3\2\2\2\u00a3\u00a4\b\2\2\2\u00a4\5\3\2\2\2\u00a5\u00a6"+
+    "\7\61\2\2\u00a6\u00a7\7\61\2\2\u00a7\u00ab\3\2\2\2\u00a8\u00aa\13\2\2"+
+    "\2\u00a9\u00a8\3\2\2\2\u00aa\u00ad\3\2\2\2\u00ab\u00ac\3\2\2\2\u00ab\u00a9"+
+    "\3\2\2\2\u00ac\u00ae\3\2\2\2\u00ad\u00ab\3\2\2\2\u00ae\u00bb\t\3\2\2\u00af"+
+    "\u00b0\7\61\2\2\u00b0\u00b1\7,\2\2\u00b1\u00b5\3\2\2\2\u00b2\u00b4\13"+
+    "\2\2\2\u00b3\u00b2\3\2\2\2\u00b4\u00b7\3\2\2\2\u00b5\u00b6\3\2\2\2\u00b5"+
+    "\u00b3\3\2\2\2\u00b6\u00b8\3\2\2\2\u00b7\u00b5\3\2\2\2\u00b8\u00b9\7,"+
+    "\2\2\u00b9\u00bb\7\61\2\2\u00ba\u00a5\3\2\2\2\u00ba\u00af\3\2\2\2\u00bb"+
+    "\u00bc\3\2\2\2\u00bc\u00bd\b\3\2\2\u00bd\7\3\2\2\2\u00be\u00bf\7}\2\2"+
+    "\u00bf\t\3\2\2\2\u00c0\u00c1\7\177\2\2\u00c1\13\3\2\2\2\u00c2\u00c3\7"+
+    "]\2\2\u00c3\r\3\2\2\2\u00c4\u00c5\7_\2\2\u00c5\17\3\2\2\2\u00c6\u00c7"+
+    "\7*\2\2\u00c7\21\3\2\2\2\u00c8\u00c9\7+\2\2\u00c9\23\3\2\2\2\u00ca\u00cb"+
+    "\7\60\2\2\u00cb\u00cc\3\2\2\2\u00cc\u00cd\b\n\3\2\u00cd\25\3\2\2\2\u00ce"+
+    "\u00cf\7.\2\2\u00cf\27\3\2\2\2\u00d0\u00d1\7=\2\2\u00d1\31\3\2\2\2\u00d2"+
+    "\u00d3\7k\2\2\u00d3\u00d4\7h\2\2\u00d4\33\3\2\2\2\u00d5\u00d6\7g\2\2\u00d6"+
+    "\u00d7\7n\2\2\u00d7\u00d8\7u\2\2\u00d8\u00d9\7g\2\2\u00d9\35\3\2\2\2\u00da"+
+    "\u00db\7y\2\2\u00db\u00dc\7j\2\2\u00dc\u00dd\7k\2\2\u00dd\u00de\7n\2\2"+
+    "\u00de\u00df\7g\2\2\u00df\37\3\2\2\2\u00e0\u00e1\7f\2\2\u00e1\u00e2\7"+
+    "q\2\2\u00e2!\3\2\2\2\u00e3\u00e4\7h\2\2\u00e4\u00e5\7q\2\2\u00e5\u00e6"+
+    "\7t\2\2\u00e6#\3\2\2\2\u00e7\u00e8\7e\2\2\u00e8\u00e9\7q\2\2\u00e9\u00ea"+
+    "\7p\2\2\u00ea\u00eb\7v\2\2\u00eb\u00ec\7k\2\2\u00ec\u00ed\7p\2\2\u00ed"+
+    "\u00ee\7w\2\2\u00ee\u00ef\7g\2\2\u00ef%\3\2\2\2\u00f0\u00f1\7d\2\2\u00f1"+
+    "\u00f2\7t\2\2\u00f2\u00f3\7g\2\2\u00f3\u00f4\7c\2\2\u00f4\u00f5\7m\2\2"+
+    "\u00f5\'\3\2\2\2\u00f6\u00f7\7t\2\2\u00f7\u00f8\7g\2\2\u00f8\u00f9\7v"+
+    "\2\2\u00f9\u00fa\7w\2\2\u00fa\u00fb\7t\2\2\u00fb\u00fc\7p\2\2\u00fc)\3"+
+    "\2\2\2\u00fd\u00fe\7p\2\2\u00fe\u00ff\7g\2\2\u00ff\u0100\7y\2\2\u0100"+
+    "+\3\2\2\2\u0101\u0102\7v\2\2\u0102\u0103\7t\2\2\u0103\u0104\7{\2\2\u0104"+
+    "-\3\2\2\2\u0105\u0106\7e\2\2\u0106\u0107\7c\2\2\u0107\u0108\7v\2\2\u0108"+
+    "\u0109\7e\2\2\u0109\u010a\7j\2\2\u010a/\3\2\2\2\u010b\u010c\7v\2\2\u010c"+
+    "\u010d\7j\2\2\u010d\u010e\7t\2\2\u010e\u010f\7q\2\2\u010f\u0110\7y\2\2"+
+    "\u0110\61\3\2\2\2\u0111\u0112\7v\2\2\u0112\u0113\7j\2\2\u0113\u0114\7"+
+    "k\2\2\u0114\u0115\7u\2\2\u0115\63\3\2\2\2\u0116\u0117\7#\2\2\u0117\65"+
+    "\3\2\2\2\u0118\u0119\7\u0080\2\2\u0119\67\3\2\2\2\u011a\u011b\7,\2\2\u011b"+
+    "9\3\2\2\2\u011c\u011d\7\61\2\2\u011d;\3\2\2\2\u011e\u011f\7\'\2\2\u011f"+
+    "=\3\2\2\2\u0120\u0121\7-\2\2\u0121?\3\2\2\2\u0122\u0123\7/\2\2\u0123A"+
+    "\3\2\2\2\u0124\u0125\7>\2\2\u0125\u0126\7>\2\2\u0126C\3\2\2\2\u0127\u0128"+
+    "\7@\2\2\u0128\u0129\7@\2\2\u0129E\3\2\2\2\u012a\u012b\7@\2\2\u012b\u012c"+
+    "\7@\2\2\u012c\u012d\7@\2\2\u012dG\3\2\2\2\u012e\u012f\7>\2\2\u012fI\3"+
+    "\2\2\2\u0130\u0131\7>\2\2\u0131\u0132\7?\2\2\u0132K\3\2\2\2\u0133\u0134"+
+    "\7@\2\2\u0134M\3\2\2\2\u0135\u0136\7@\2\2\u0136\u0137\7?\2\2\u0137O\3"+
+    "\2\2\2\u0138\u0139\7?\2\2\u0139\u013a\7?\2\2\u013aQ\3\2\2\2\u013b\u013c"+
+    "\7?\2\2\u013c\u013d\7?\2\2\u013d\u013e\7?\2\2\u013eS\3\2\2\2\u013f\u0140"+
+    "\7#\2\2\u0140\u0141\7?\2\2\u0141U\3\2\2\2\u0142\u0143\7#\2\2\u0143\u0144"+
+    "\7?\2\2\u0144\u0145\7?\2\2\u0145W\3\2\2\2\u0146\u0147\7(\2\2\u0147Y\3"+
+    "\2\2\2\u0148\u0149\7`\2\2\u0149[\3\2\2\2\u014a\u014b\7~\2\2\u014b]\3\2"+
+    "\2\2\u014c\u014d\7(\2\2\u014d\u014e\7(\2\2\u014e_\3\2\2\2\u014f\u0150"+
+    "\7~\2\2\u0150\u0151\7~\2\2\u0151a\3\2\2\2\u0152\u0153\7A\2\2\u0153c\3"+
+    "\2\2\2\u0154\u0155\7<\2\2\u0155e\3\2\2\2\u0156\u0157\7<\2\2\u0157\u0158"+
+    "\7<\2\2\u0158g\3\2\2\2\u0159\u015a\7/\2\2\u015a\u015b\7@\2\2\u015bi\3"+
+    "\2\2\2\u015c\u015d\7-\2\2\u015d\u015e\7-\2\2\u015ek\3\2\2\2\u015f\u0160"+
+    "\7/\2\2\u0160\u0161\7/\2\2\u0161m\3\2\2\2\u0162\u0163\7?\2\2\u0163o\3"+
+    "\2\2\2\u0164\u0165\7-\2\2\u0165\u0166\7?\2\2\u0166q\3\2\2\2\u0167\u0168"+
+    "\7/\2\2\u0168\u0169\7?\2\2\u0169s\3\2\2\2\u016a\u016b\7,\2\2\u016b\u016c"+
+    "\7?\2\2\u016cu\3\2\2\2\u016d\u016e\7\61\2\2\u016e\u016f\7?\2\2\u016fw"+
+    "\3\2\2\2\u0170\u0171\7\'\2\2\u0171\u0172\7?\2\2\u0172y\3\2\2\2\u0173\u0174"+
+    "\7(\2\2\u0174\u0175\7?\2\2\u0175{\3\2\2\2\u0176\u0177\7`\2\2\u0177\u0178"+
+    "\7?\2\2\u0178}\3\2\2\2\u0179\u017a\7~\2\2\u017a\u017b\7?\2\2\u017b\177"+
+    "\3\2\2\2\u017c\u017d\7>\2\2\u017d\u017e\7>\2\2\u017e\u017f\7?\2\2\u017f"+
+    "\u0081\3\2\2\2\u0180\u0181\7@\2\2\u0181\u0182\7@\2\2\u0182\u0183\7?\2"+
+    "\2\u0183\u0083\3\2\2\2\u0184\u0185\7@\2\2\u0185\u0186\7@\2\2\u0186\u0187"+
+    "\7@\2\2\u0187\u0188\7?\2\2\u0188\u0085\3\2\2\2\u0189\u018b\7\62\2\2\u018a"+
+    "\u018c\t\4\2\2\u018b\u018a\3\2\2\2\u018c\u018d\3\2\2\2\u018d\u018b\3\2"+
+    "\2\2\u018d\u018e\3\2\2\2\u018e\u0190\3\2\2\2\u018f\u0191\t\5\2\2\u0190"+
+    "\u018f\3\2\2\2\u0190\u0191\3\2\2\2\u0191\u0087\3\2\2\2\u0192\u0193\7\62"+
+    "\2\2\u0193\u0195\t\6\2\2\u0194\u0196\t\7\2\2\u0195\u0194\3\2\2\2\u0196"+
+    "\u0197\3\2\2\2\u0197\u0195\3\2\2\2\u0197\u0198\3\2\2\2\u0198\u019a\3\2"+
+    "\2\2\u0199\u019b\t\5\2\2\u019a\u0199\3\2\2\2\u019a\u019b\3\2\2\2\u019b"+
+    "\u0089\3\2\2\2\u019c\u01a5\7\62\2\2\u019d\u01a1\t\b\2\2\u019e\u01a0\t"+
+    "\t\2\2\u019f\u019e\3\2\2\2\u01a0\u01a3\3\2\2\2\u01a1\u019f\3\2\2\2\u01a1"+
+    "\u01a2\3\2\2\2\u01a2\u01a5\3\2\2\2\u01a3\u01a1\3\2\2\2\u01a4\u019c\3\2"+
+    "\2\2\u01a4\u019d\3\2\2\2\u01a5\u01a7\3\2\2\2\u01a6\u01a8\t\n\2\2\u01a7"+
+    "\u01a6\3\2\2\2\u01a7\u01a8\3\2\2\2\u01a8\u008b\3\2\2\2\u01a9\u01b2\7\62"+
+    "\2\2\u01aa\u01ae\t\b\2\2\u01ab\u01ad\t\t\2\2\u01ac\u01ab\3\2\2\2\u01ad"+
+    "\u01b0\3\2\2\2\u01ae\u01ac\3\2\2\2\u01ae\u01af\3\2\2\2\u01af\u01b2\3\2"+
+    "\2\2\u01b0\u01ae\3\2\2\2\u01b1\u01a9\3\2\2\2\u01b1\u01aa\3\2\2\2\u01b2"+
+    "\u01b9\3\2\2\2\u01b3\u01b5\5\24\n\2\u01b4\u01b6\t\t\2\2\u01b5\u01b4\3"+
+    "\2\2\2\u01b6\u01b7\3\2\2\2\u01b7\u01b5\3\2\2\2\u01b7\u01b8\3\2\2\2\u01b8"+
+    "\u01ba\3\2\2\2\u01b9\u01b3\3\2\2\2\u01b9\u01ba\3\2\2\2\u01ba\u01c4\3\2"+
+    "\2\2\u01bb\u01bd\t\13\2\2\u01bc\u01be\t\f\2\2\u01bd\u01bc\3\2\2\2\u01bd"+
+    "\u01be\3\2\2\2\u01be\u01c0\3\2\2\2\u01bf\u01c1\t\t\2\2\u01c0\u01bf\3\2"+
+    "\2\2\u01c1\u01c2\3\2\2\2\u01c2\u01c0\3\2\2\2\u01c2\u01c3\3\2\2\2\u01c3"+
+    "\u01c5\3\2\2\2\u01c4\u01bb\3\2\2\2\u01c4\u01c5\3\2\2\2\u01c5\u01c7\3\2"+
+    "\2\2\u01c6\u01c8\t\r\2\2\u01c7\u01c6\3\2\2\2\u01c7\u01c8\3\2\2\2\u01c8"+
+    "\u008d\3\2\2\2\u01c9\u01d1\7$\2\2\u01ca\u01cb\7^\2\2\u01cb\u01d0\7$\2"+
+    "\2\u01cc\u01cd\7^\2\2\u01cd\u01d0\7^\2\2\u01ce\u01d0\n\16\2\2\u01cf\u01ca"+
+    "\3\2\2\2\u01cf\u01cc\3\2\2\2\u01cf\u01ce\3\2\2\2\u01d0\u01d3\3\2\2\2\u01d1"+
+    "\u01d2\3\2\2\2\u01d1\u01cf\3\2\2\2\u01d2\u01d4\3\2\2\2\u01d3\u01d1\3\2"+
+    "\2\2\u01d4\u01e2\7$\2\2\u01d5\u01dd\7)\2\2\u01d6\u01d7\7^\2\2\u01d7\u01dc"+
+    "\7)\2\2\u01d8\u01d9\7^\2\2\u01d9\u01dc\7^\2\2\u01da\u01dc\n\16\2\2\u01db"+
+    "\u01d6\3\2\2\2\u01db\u01d8\3\2\2\2\u01db\u01da\3\2\2\2\u01dc\u01df\3\2"+
+    "\2\2\u01dd\u01de\3\2\2\2\u01dd\u01db\3\2\2\2\u01de\u01e0\3\2\2\2\u01df"+
+    "\u01dd\3\2\2\2\u01e0\u01e2\7)\2\2\u01e1\u01c9\3\2\2\2\u01e1\u01d5\3\2"+
+    "\2\2\u01e2\u008f\3\2\2\2\u01e3\u01e4\7v\2\2\u01e4\u01e5\7t\2\2\u01e5\u01e6"+
+    "\7w\2\2\u01e6\u01e7\7g\2\2\u01e7\u0091\3\2\2\2\u01e8\u01e9\7h\2\2\u01e9"+
+    "\u01ea\7c\2\2\u01ea\u01eb\7n\2\2\u01eb\u01ec\7u\2\2\u01ec\u01ed\7g\2\2"+
+    "\u01ed\u0093\3\2\2\2\u01ee\u01ef\7p\2\2\u01ef\u01f0\7w\2\2\u01f0\u01f1"+
+    "\7n\2\2\u01f1\u01f2\7n\2\2\u01f2\u0095\3\2\2\2\u01f3\u01f9\5\u0098L\2"+
+    "\u01f4\u01f5\5\24\n\2\u01f5\u01f6\5\u0098L\2\u01f6\u01f8\3\2\2\2\u01f7"+
+    "\u01f4\3\2\2\2\u01f8\u01fb\3\2\2\2\u01f9\u01f7\3\2\2\2\u01f9\u01fa\3\2"+
+    "\2\2\u01fa\u01fc\3\2\2\2\u01fb\u01f9\3\2\2\2\u01fc\u01fd\6K\2\2\u01fd"+
+    "\u0097\3\2\2\2\u01fe\u0202\t\17\2\2\u01ff\u0201\t\20\2\2\u0200\u01ff\3"+
+    "\2\2\2\u0201\u0204\3\2\2\2\u0202\u0200\3\2\2\2\u0202\u0203\3\2\2\2\u0203"+
+    "\u0099\3\2\2\2\u0204\u0202\3\2\2\2\u0205\u020e\7\62\2\2\u0206\u020a\t"+
+    "\b\2\2\u0207\u0209\t\t\2\2\u0208\u0207\3\2\2\2\u0209\u020c\3\2\2\2\u020a"+
+    "\u0208\3\2\2\2\u020a\u020b\3\2\2\2\u020b\u020e\3\2\2\2\u020c\u020a\3\2"+
+    "\2\2\u020d\u0205\3\2\2\2\u020d\u0206\3\2\2\2\u020e\u020f\3\2\2\2\u020f"+
+    "\u0210\bM\4\2\u0210\u009b\3\2\2\2\u0211\u0215\t\17\2\2\u0212\u0214\t\20"+
+    "\2\2\u0213\u0212\3\2\2\2\u0214\u0217\3\2\2\2\u0215\u0213\3\2\2\2\u0215"+
+    "\u0216\3\2\2\2\u0216\u0218\3\2\2\2\u0217\u0215\3\2\2\2\u0218\u0219\bN"+
+    "\4\2\u0219\u009d\3\2\2\2!\2\3\u00a1\u00ab\u00b5\u00ba\u018d\u0190\u0197"+
+    "\u019a\u01a1\u01a4\u01a7\u01ae\u01b1\u01b7\u01b9\u01bd\u01c2\u01c4\u01c7"+
+    "\u01cf\u01d1\u01db\u01dd\u01e1\u01f9\u0202\u020a\u020d\u0215\5\b\2\2\4"+
+    "\3\2\4\2\2";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
index 73101d22bc4..51f4efeb94a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
@@ -19,14 +19,14 @@ class PainlessParser extends Parser {
   public static final int
     WS=1, COMMENT=2, LBRACK=3, RBRACK=4, LBRACE=5, RBRACE=6, LP=7, RP=8, DOT=9, 
     COMMA=10, SEMICOLON=11, IF=12, ELSE=13, WHILE=14, DO=15, FOR=16, CONTINUE=17, 
-    BREAK=18, RETURN=19, NEW=20, TRY=21, CATCH=22, THROW=23, BOOLNOT=24, BWNOT=25, 
-    MUL=26, DIV=27, REM=28, ADD=29, SUB=30, LSH=31, RSH=32, USH=33, LT=34, 
-    LTE=35, GT=36, GTE=37, EQ=38, EQR=39, NE=40, NER=41, BWAND=42, XOR=43, 
-    BWOR=44, BOOLAND=45, BOOLOR=46, COND=47, COLON=48, REF=49, ARROW=50, INCR=51, 
-    DECR=52, ASSIGN=53, AADD=54, ASUB=55, AMUL=56, ADIV=57, AREM=58, AAND=59, 
-    AXOR=60, AOR=61, ALSH=62, ARSH=63, AUSH=64, OCTAL=65, HEX=66, INTEGER=67, 
-    DECIMAL=68, STRING=69, TRUE=70, FALSE=71, NULL=72, TYPE=73, ID=74, DOTINTEGER=75, 
-    DOTID=76;
+    BREAK=18, RETURN=19, NEW=20, TRY=21, CATCH=22, THROW=23, THIS=24, BOOLNOT=25, 
+    BWNOT=26, MUL=27, DIV=28, REM=29, ADD=30, SUB=31, LSH=32, RSH=33, USH=34, 
+    LT=35, LTE=36, GT=37, GTE=38, EQ=39, EQR=40, NE=41, NER=42, BWAND=43, 
+    XOR=44, BWOR=45, BOOLAND=46, BOOLOR=47, COND=48, COLON=49, REF=50, ARROW=51, 
+    INCR=52, DECR=53, ASSIGN=54, AADD=55, ASUB=56, AMUL=57, ADIV=58, AREM=59, 
+    AAND=60, AXOR=61, AOR=62, ALSH=63, ARSH=64, AUSH=65, OCTAL=66, HEX=67, 
+    INTEGER=68, DECIMAL=69, STRING=70, TRUE=71, FALSE=72, NULL=73, TYPE=74, 
+    ID=75, DOTINTEGER=76, DOTID=77;
   public static final int
     RULE_source = 0, RULE_function = 1, RULE_parameters = 2, RULE_statement = 3, 
     RULE_trailer = 4, RULE_block = 5, RULE_empty = 6, RULE_initializer = 7, 
@@ -45,23 +45,23 @@ class PainlessParser extends Parser {
   private static final String[] _LITERAL_NAMES = {
     null, null, null, "'{'", "'}'", "'['", "']'", "'('", "')'", "'.'", "','", 
     "';'", "'if'", "'else'", "'while'", "'do'", "'for'", "'continue'", "'break'", 
-    "'return'", "'new'", "'try'", "'catch'", "'throw'", "'!'", "'~'", "'*'", 
-    "'/'", "'%'", "'+'", "'-'", "'<<'", "'>>'", "'>>>'", "'<'", "'<='", "'>'", 
-    "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", "'^'", "'|'", "'&&'", 
-    "'||'", "'?'", "':'", "'::'", "'->'", "'++'", "'--'", "'='", "'+='", "'-='", 
-    "'*='", "'/='", "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", "'>>>='", 
-    null, null, null, null, null, "'true'", "'false'", "'null'"
+    "'return'", "'new'", "'try'", "'catch'", "'throw'", "'this'", "'!'", "'~'", 
+    "'*'", "'/'", "'%'", "'+'", "'-'", "'<<'", "'>>'", "'>>>'", "'<'", "'<='", 
+    "'>'", "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", "'^'", "'|'", 
+    "'&&'", "'||'", "'?'", "':'", "'::'", "'->'", "'++'", "'--'", "'='", "'+='", 
+    "'-='", "'*='", "'/='", "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", 
+    "'>>>='", null, null, null, null, null, "'true'", "'false'", "'null'"
   };
   private static final String[] _SYMBOLIC_NAMES = {
     null, "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", 
     "DOT", "COMMA", "SEMICOLON", "IF", "ELSE", "WHILE", "DO", "FOR", "CONTINUE", 
-    "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "BOOLNOT", "BWNOT", 
-    "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", "LTE", "GT", 
-    "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", "BOOLAND", "BOOLOR", 
-    "COND", "COLON", "REF", "ARROW", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", 
-    "AMUL", "ADIV", "AREM", "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", 
-    "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", "TRUE", "FALSE", "NULL", 
-    "TYPE", "ID", "DOTINTEGER", "DOTID"
+    "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", "BOOLNOT", 
+    "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", 
+    "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", 
+    "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "INCR", "DECR", 
+    "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", "AAND", "AXOR", "AOR", 
+    "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", 
+    "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", "DOTID"
   };
   public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
 
@@ -1500,7 +1500,7 @@ class PainlessParser extends Parser {
         chain(true);
         setState(226);
         _la = _input.LA(1);
-        if ( !(((((_la - 53)) & ~0x3f) == 0 && ((1L << (_la - 53)) & ((1L << (ASSIGN - 53)) | (1L << (AADD - 53)) | (1L << (ASUB - 53)) | (1L << (AMUL - 53)) | (1L << (ADIV - 53)) | (1L << (AREM - 53)) | (1L << (AAND - 53)) | (1L << (AXOR - 53)) | (1L << (AOR - 53)) | (1L << (ALSH - 53)) | (1L << (ARSH - 53)) | (1L << (AUSH - 53)))) != 0)) ) {
+        if ( !(((((_la - 54)) & ~0x3f) == 0 && ((1L << (_la - 54)) & ((1L << (ASSIGN - 54)) | (1L << (AADD - 54)) | (1L << (ASUB - 54)) | (1L << (AMUL - 54)) | (1L << (ADIV - 54)) | (1L << (AREM - 54)) | (1L << (AAND - 54)) | (1L << (AXOR - 54)) | (1L << (AOR - 54)) | (1L << (ALSH - 54)) | (1L << (ARSH - 54)) | (1L << (AUSH - 54)))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
@@ -1908,7 +1908,7 @@ class PainlessParser extends Parser {
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
         setState(307);
         _la = _input.LA(1);
-        if ( !(((((_la - 65)) & ~0x3f) == 0 && ((1L << (_la - 65)) & ((1L << (OCTAL - 65)) | (1L << (HEX - 65)) | (1L << (INTEGER - 65)) | (1L << (DECIMAL - 65)))) != 0)) ) {
+        if ( !(((((_la - 66)) & ~0x3f) == 0 && ((1L << (_la - 66)) & ((1L << (OCTAL - 66)) | (1L << (HEX - 66)) | (1L << (INTEGER - 66)) | (1L << (DECIMAL - 66)))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
@@ -2863,6 +2863,7 @@ class PainlessParser extends Parser {
       return getToken(PainlessParser.ID, i);
     }
     public TerminalNode NEW() { return getToken(PainlessParser.NEW, 0); }
+    public TerminalNode THIS() { return getToken(PainlessParser.THIS, 0); }
     public FuncrefContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
     }
@@ -2897,11 +2898,17 @@ class PainlessParser extends Parser {
         }
         }
         break;
+      case THIS:
       case ID:
         enterOuterAlt(_localctx, 2);
         {
         setState(445);
-        match(ID);
+        _la = _input.LA(1);
+        if ( !(_la==THIS || _la==ID) ) {
+        _errHandler.recoverInline(this);
+        } else {
+          consume();
+        }
         setState(446);
         match(REF);
         setState(447);
@@ -3013,7 +3020,7 @@ class PainlessParser extends Parser {
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3N\u01c5\4\2\t\2\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3O\u01c5\4\2\t\2\4"+
     "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
     "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
     "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+
@@ -3048,57 +3055,57 @@ class PainlessParser extends Parser {
     "\3\31\3\31\3\31\3\31\3\31\7\31\u01ab\n\31\f\31\16\31\u01ae\13\31\5\31"+
     "\u01b0\n\31\3\31\5\31\u01b3\n\31\3\31\3\31\3\31\3\32\5\32\u01b9\n\32\3"+
     "\32\3\32\3\33\3\33\3\33\3\33\3\33\3\33\5\33\u01c3\n\33\3\33\2\3\36\34"+
-    "\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,.\60\62\64\2\r\3\2\67B"+
-    "\3\2\34\36\3\2\37 \3\2!#\3\2$\'\3\2(+\3\2\65\66\3\2CF\4\2\32\33\37 \3"+
-    "\2MN\4\2\26\26LL\u01f3\29\3\2\2\2\4D\3\2\2\2\6I\3\2\2\2\b\u00a7\3\2\2"+
-    "\2\n\u00ab\3\2\2\2\f\u00ad\3\2\2\2\16\u00b6\3\2\2\2\20\u00ba\3\2\2\2\22"+
-    "\u00bc\3\2\2\2\24\u00be\3\2\2\2\26\u00c7\3\2\2\2\30\u00cf\3\2\2\2\32\u00d4"+
-    "\3\2\2\2\34\u00e0\3\2\2\2\36\u00eb\3\2\2\2 \u0148\3\2\2\2\"\u016c\3\2"+
-    "\2\2$\u0180\3\2\2\2&\u0186\3\2\2\2(\u018d\3\2\2\2*\u018f\3\2\2\2,\u0193"+
-    "\3\2\2\2.\u01a3\3\2\2\2\60\u01b2\3\2\2\2\62\u01b8\3\2\2\2\64\u01c2\3\2"+
-    "\2\2\668\5\4\3\2\67\66\3\2\2\28;\3\2\2\29\67\3\2\2\29:\3\2\2\2:?\3\2\2"+
-    "\2;9\3\2\2\2<>\5\b\5\2=<\3\2\2\2>A\3\2\2\2?=\3\2\2\2?@\3\2\2\2@B\3\2\2"+
-    "\2A?\3\2\2\2BC\7\2\2\3C\3\3\2\2\2DE\5\26\f\2EF\7L\2\2FG\5\6\4\2GH\5\f"+
-    "\7\2H\5\3\2\2\2IU\7\t\2\2JK\5\26\f\2KR\7L\2\2LM\7\f\2\2MN\5\26\f\2NO\7"+
-    "L\2\2OQ\3\2\2\2PL\3\2\2\2QT\3\2\2\2RP\3\2\2\2RS\3\2\2\2SV\3\2\2\2TR\3"+
-    "\2\2\2UJ\3\2\2\2UV\3\2\2\2VW\3\2\2\2WX\7\n\2\2X\7\3\2\2\2YZ\7\16\2\2Z"+
-    "[\7\t\2\2[\\\5\36\20\2\\]\7\n\2\2]a\5\n\6\2^_\7\17\2\2_b\5\n\6\2`b\6\5"+
-    "\2\2a^\3\2\2\2a`\3\2\2\2b\u00a8\3\2\2\2cd\7\20\2\2de\7\t\2\2ef\5\36\20"+
-    "\2fi\7\n\2\2gj\5\n\6\2hj\5\16\b\2ig\3\2\2\2ih\3\2\2\2j\u00a8\3\2\2\2k"+
-    "l\7\21\2\2lm\5\f\7\2mn\7\20\2\2no\7\t\2\2op\5\36\20\2pq\7\n\2\2qr\5\34"+
-    "\17\2r\u00a8\3\2\2\2st\7\22\2\2tv\7\t\2\2uw\5\20\t\2vu\3\2\2\2vw\3\2\2"+
-    "\2wx\3\2\2\2xz\7\r\2\2y{\5\36\20\2zy\3\2\2\2z{\3\2\2\2{|\3\2\2\2|~\7\r"+
-    "\2\2}\177\5\22\n\2~}\3\2\2\2~\177\3\2\2\2\177\u0080\3\2\2\2\u0080\u0083"+
-    "\7\n\2\2\u0081\u0084\5\n\6\2\u0082\u0084\5\16\b\2\u0083\u0081\3\2\2\2"+
-    "\u0083\u0082\3\2\2\2\u0084\u00a8\3\2\2\2\u0085\u0086\7\22\2\2\u0086\u0087"+
-    "\7\t\2\2\u0087\u0088\5\26\f\2\u0088\u0089\7L\2\2\u0089\u008a\7\62\2\2"+
-    "\u008a\u008b\5\36\20\2\u008b\u008c\7\n\2\2\u008c\u008d\5\n\6\2\u008d\u00a8"+
-    "\3\2\2\2\u008e\u008f\5\24\13\2\u008f\u0090\5\34\17\2\u0090\u00a8\3\2\2"+
-    "\2\u0091\u0092\7\23\2\2\u0092\u00a8\5\34\17\2\u0093\u0094\7\24\2\2\u0094"+
-    "\u00a8\5\34\17\2\u0095\u0096\7\25\2\2\u0096\u0097\5\36\20\2\u0097\u0098"+
-    "\5\34\17\2\u0098\u00a8\3\2\2\2\u0099\u009a\7\27\2\2\u009a\u009c\5\f\7"+
-    "\2\u009b\u009d\5\32\16\2\u009c\u009b\3\2\2\2\u009d\u009e\3\2\2\2\u009e"+
-    "\u009c\3\2\2\2\u009e\u009f\3\2\2\2\u009f\u00a8\3\2\2\2\u00a0\u00a1\7\31"+
-    "\2\2\u00a1\u00a2\5\36\20\2\u00a2\u00a3\5\34\17\2\u00a3\u00a8\3\2\2\2\u00a4"+
-    "\u00a5\5\36\20\2\u00a5\u00a6\5\34\17\2\u00a6\u00a8\3\2\2\2\u00a7Y\3\2"+
-    "\2\2\u00a7c\3\2\2\2\u00a7k\3\2\2\2\u00a7s\3\2\2\2\u00a7\u0085\3\2\2\2"+
-    "\u00a7\u008e\3\2\2\2\u00a7\u0091\3\2\2\2\u00a7\u0093\3\2\2\2\u00a7\u0095"+
-    "\3\2\2\2\u00a7\u0099\3\2\2\2\u00a7\u00a0\3\2\2\2\u00a7\u00a4\3\2\2\2\u00a8"+
-    "\t\3\2\2\2\u00a9\u00ac\5\f\7\2\u00aa\u00ac\5\b\5\2\u00ab\u00a9\3\2\2\2"+
-    "\u00ab\u00aa\3\2\2\2\u00ac\13\3\2\2\2\u00ad\u00b1\7\5\2\2\u00ae\u00b0"+
-    "\5\b\5\2\u00af\u00ae\3\2\2\2\u00b0\u00b3\3\2\2\2\u00b1\u00af\3\2\2\2\u00b1"+
-    "\u00b2\3\2\2\2\u00b2\u00b4\3\2\2\2\u00b3\u00b1\3\2\2\2\u00b4\u00b5\7\6"+
-    "\2\2\u00b5\r\3\2\2\2\u00b6\u00b7\7\r\2\2\u00b7\17\3\2\2\2\u00b8\u00bb"+
-    "\5\24\13\2\u00b9\u00bb\5\36\20\2\u00ba\u00b8\3\2\2\2\u00ba\u00b9\3\2\2"+
-    "\2\u00bb\21\3\2\2\2\u00bc\u00bd\5\36\20\2\u00bd\23\3\2\2\2\u00be\u00bf"+
-    "\5\26\f\2\u00bf\u00c4\5\30\r\2\u00c0\u00c1\7\f\2\2\u00c1\u00c3\5\30\r"+
-    "\2\u00c2\u00c0\3\2\2\2\u00c3\u00c6\3\2\2\2\u00c4\u00c2\3\2\2\2\u00c4\u00c5"+
-    "\3\2\2\2\u00c5\25\3\2\2\2\u00c6\u00c4\3\2\2\2\u00c7\u00cc\7K\2\2\u00c8"+
-    "\u00c9\7\7\2\2\u00c9\u00cb\7\b\2\2\u00ca\u00c8\3\2\2\2\u00cb\u00ce\3\2"+
-    "\2\2\u00cc\u00ca\3\2\2\2\u00cc\u00cd\3\2\2\2\u00cd\27\3\2\2\2\u00ce\u00cc"+
-    "\3\2\2\2\u00cf\u00d2\7L\2\2\u00d0\u00d1\7\67\2\2\u00d1\u00d3\5\36\20\2"+
-    "\u00d2\u00d0\3\2\2\2\u00d2\u00d3\3\2\2\2\u00d3\31\3\2\2\2\u00d4\u00d5"+
-    "\7\30\2\2\u00d5\u00d6\7\t\2\2\u00d6\u00d7\7K\2\2\u00d7\u00d8\7L\2\2\u00d8"+
+    "\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,.\60\62\64\2\16\3\28C\3"+
+    "\2\35\37\3\2 !\3\2\"$\3\2%(\3\2),\3\2\66\67\3\2DG\4\2\33\34 !\3\2NO\4"+
+    "\2\26\26MM\4\2\32\32MM\u01f3\29\3\2\2\2\4D\3\2\2\2\6I\3\2\2\2\b\u00a7"+
+    "\3\2\2\2\n\u00ab\3\2\2\2\f\u00ad\3\2\2\2\16\u00b6\3\2\2\2\20\u00ba\3\2"+
+    "\2\2\22\u00bc\3\2\2\2\24\u00be\3\2\2\2\26\u00c7\3\2\2\2\30\u00cf\3\2\2"+
+    "\2\32\u00d4\3\2\2\2\34\u00e0\3\2\2\2\36\u00eb\3\2\2\2 \u0148\3\2\2\2\""+
+    "\u016c\3\2\2\2$\u0180\3\2\2\2&\u0186\3\2\2\2(\u018d\3\2\2\2*\u018f\3\2"+
+    "\2\2,\u0193\3\2\2\2.\u01a3\3\2\2\2\60\u01b2\3\2\2\2\62\u01b8\3\2\2\2\64"+
+    "\u01c2\3\2\2\2\668\5\4\3\2\67\66\3\2\2\28;\3\2\2\29\67\3\2\2\29:\3\2\2"+
+    "\2:?\3\2\2\2;9\3\2\2\2<>\5\b\5\2=<\3\2\2\2>A\3\2\2\2?=\3\2\2\2?@\3\2\2"+
+    "\2@B\3\2\2\2A?\3\2\2\2BC\7\2\2\3C\3\3\2\2\2DE\5\26\f\2EF\7M\2\2FG\5\6"+
+    "\4\2GH\5\f\7\2H\5\3\2\2\2IU\7\t\2\2JK\5\26\f\2KR\7M\2\2LM\7\f\2\2MN\5"+
+    "\26\f\2NO\7M\2\2OQ\3\2\2\2PL\3\2\2\2QT\3\2\2\2RP\3\2\2\2RS\3\2\2\2SV\3"+
+    "\2\2\2TR\3\2\2\2UJ\3\2\2\2UV\3\2\2\2VW\3\2\2\2WX\7\n\2\2X\7\3\2\2\2YZ"+
+    "\7\16\2\2Z[\7\t\2\2[\\\5\36\20\2\\]\7\n\2\2]a\5\n\6\2^_\7\17\2\2_b\5\n"+
+    "\6\2`b\6\5\2\2a^\3\2\2\2a`\3\2\2\2b\u00a8\3\2\2\2cd\7\20\2\2de\7\t\2\2"+
+    "ef\5\36\20\2fi\7\n\2\2gj\5\n\6\2hj\5\16\b\2ig\3\2\2\2ih\3\2\2\2j\u00a8"+
+    "\3\2\2\2kl\7\21\2\2lm\5\f\7\2mn\7\20\2\2no\7\t\2\2op\5\36\20\2pq\7\n\2"+
+    "\2qr\5\34\17\2r\u00a8\3\2\2\2st\7\22\2\2tv\7\t\2\2uw\5\20\t\2vu\3\2\2"+
+    "\2vw\3\2\2\2wx\3\2\2\2xz\7\r\2\2y{\5\36\20\2zy\3\2\2\2z{\3\2\2\2{|\3\2"+
+    "\2\2|~\7\r\2\2}\177\5\22\n\2~}\3\2\2\2~\177\3\2\2\2\177\u0080\3\2\2\2"+
+    "\u0080\u0083\7\n\2\2\u0081\u0084\5\n\6\2\u0082\u0084\5\16\b\2\u0083\u0081"+
+    "\3\2\2\2\u0083\u0082\3\2\2\2\u0084\u00a8\3\2\2\2\u0085\u0086\7\22\2\2"+
+    "\u0086\u0087\7\t\2\2\u0087\u0088\5\26\f\2\u0088\u0089\7M\2\2\u0089\u008a"+
+    "\7\63\2\2\u008a\u008b\5\36\20\2\u008b\u008c\7\n\2\2\u008c\u008d\5\n\6"+
+    "\2\u008d\u00a8\3\2\2\2\u008e\u008f\5\24\13\2\u008f\u0090\5\34\17\2\u0090"+
+    "\u00a8\3\2\2\2\u0091\u0092\7\23\2\2\u0092\u00a8\5\34\17\2\u0093\u0094"+
+    "\7\24\2\2\u0094\u00a8\5\34\17\2\u0095\u0096\7\25\2\2\u0096\u0097\5\36"+
+    "\20\2\u0097\u0098\5\34\17\2\u0098\u00a8\3\2\2\2\u0099\u009a\7\27\2\2\u009a"+
+    "\u009c\5\f\7\2\u009b\u009d\5\32\16\2\u009c\u009b\3\2\2\2\u009d\u009e\3"+
+    "\2\2\2\u009e\u009c\3\2\2\2\u009e\u009f\3\2\2\2\u009f\u00a8\3\2\2\2\u00a0"+
+    "\u00a1\7\31\2\2\u00a1\u00a2\5\36\20\2\u00a2\u00a3\5\34\17\2\u00a3\u00a8"+
+    "\3\2\2\2\u00a4\u00a5\5\36\20\2\u00a5\u00a6\5\34\17\2\u00a6\u00a8\3\2\2"+
+    "\2\u00a7Y\3\2\2\2\u00a7c\3\2\2\2\u00a7k\3\2\2\2\u00a7s\3\2\2\2\u00a7\u0085"+
+    "\3\2\2\2\u00a7\u008e\3\2\2\2\u00a7\u0091\3\2\2\2\u00a7\u0093\3\2\2\2\u00a7"+
+    "\u0095\3\2\2\2\u00a7\u0099\3\2\2\2\u00a7\u00a0\3\2\2\2\u00a7\u00a4\3\2"+
+    "\2\2\u00a8\t\3\2\2\2\u00a9\u00ac\5\f\7\2\u00aa\u00ac\5\b\5\2\u00ab\u00a9"+
+    "\3\2\2\2\u00ab\u00aa\3\2\2\2\u00ac\13\3\2\2\2\u00ad\u00b1\7\5\2\2\u00ae"+
+    "\u00b0\5\b\5\2\u00af\u00ae\3\2\2\2\u00b0\u00b3\3\2\2\2\u00b1\u00af\3\2"+
+    "\2\2\u00b1\u00b2\3\2\2\2\u00b2\u00b4\3\2\2\2\u00b3\u00b1\3\2\2\2\u00b4"+
+    "\u00b5\7\6\2\2\u00b5\r\3\2\2\2\u00b6\u00b7\7\r\2\2\u00b7\17\3\2\2\2\u00b8"+
+    "\u00bb\5\24\13\2\u00b9\u00bb\5\36\20\2\u00ba\u00b8\3\2\2\2\u00ba\u00b9"+
+    "\3\2\2\2\u00bb\21\3\2\2\2\u00bc\u00bd\5\36\20\2\u00bd\23\3\2\2\2\u00be"+
+    "\u00bf\5\26\f\2\u00bf\u00c4\5\30\r\2\u00c0\u00c1\7\f\2\2\u00c1\u00c3\5"+
+    "\30\r\2\u00c2\u00c0\3\2\2\2\u00c3\u00c6\3\2\2\2\u00c4\u00c2\3\2\2\2\u00c4"+
+    "\u00c5\3\2\2\2\u00c5\25\3\2\2\2\u00c6\u00c4\3\2\2\2\u00c7\u00cc\7L\2\2"+
+    "\u00c8\u00c9\7\7\2\2\u00c9\u00cb\7\b\2\2\u00ca\u00c8\3\2\2\2\u00cb\u00ce"+
+    "\3\2\2\2\u00cc\u00ca\3\2\2\2\u00cc\u00cd\3\2\2\2\u00cd\27\3\2\2\2\u00ce"+
+    "\u00cc\3\2\2\2\u00cf\u00d2\7M\2\2\u00d0\u00d1\78\2\2\u00d1\u00d3\5\36"+
+    "\20\2\u00d2\u00d0\3\2\2\2\u00d2\u00d3\3\2\2\2\u00d3\31\3\2\2\2\u00d4\u00d5"+
+    "\7\30\2\2\u00d5\u00d6\7\t\2\2\u00d6\u00d7\7L\2\2\u00d7\u00d8\7M\2\2\u00d8"+
     "\u00d9\7\n\2\2\u00d9\u00da\5\f\7\2\u00da\33\3\2\2\2\u00db\u00e1\7\r\2"+
     "\2\u00dc\u00e1\7\2\2\3\u00dd\u00de\b\17\1\2\u00de\u00df\7\6\2\2\u00df"+
     "\u00e1\b\17\1\2\u00e0\u00db\3\2\2\2\u00e0\u00dc\3\2\2\2\u00e0\u00dd\3"+
@@ -3113,27 +3120,27 @@ class PainlessParser extends Parser {
     "\2\u00fb\u0127\3\2\2\2\u00fc\u00fd\f\13\2\2\u00fd\u00fe\t\6\2\2\u00fe"+
     "\u00ff\5\36\20\f\u00ff\u0100\b\20\1\2\u0100\u0127\3\2\2\2\u0101\u0102"+
     "\f\n\2\2\u0102\u0103\t\7\2\2\u0103\u0104\5\36\20\13\u0104\u0105\b\20\1"+
-    "\2\u0105\u0127\3\2\2\2\u0106\u0107\f\t\2\2\u0107\u0108\7,\2\2\u0108\u0109"+
+    "\2\u0105\u0127\3\2\2\2\u0106\u0107\f\t\2\2\u0107\u0108\7-\2\2\u0108\u0109"+
     "\5\36\20\n\u0109\u010a\b\20\1\2\u010a\u0127\3\2\2\2\u010b\u010c\f\b\2"+
-    "\2\u010c\u010d\7-\2\2\u010d\u010e\5\36\20\t\u010e\u010f\b\20\1\2\u010f"+
-    "\u0127\3\2\2\2\u0110\u0111\f\7\2\2\u0111\u0112\7.\2\2\u0112\u0113\5\36"+
+    "\2\u010c\u010d\7.\2\2\u010d\u010e\5\36\20\t\u010e\u010f\b\20\1\2\u010f"+
+    "\u0127\3\2\2\2\u0110\u0111\f\7\2\2\u0111\u0112\7/\2\2\u0112\u0113\5\36"+
     "\20\b\u0113\u0114\b\20\1\2\u0114\u0127\3\2\2\2\u0115\u0116\f\6\2\2\u0116"+
-    "\u0117\7/\2\2\u0117\u0118\5\36\20\7\u0118\u0119\b\20\1\2\u0119\u0127\3"+
-    "\2\2\2\u011a\u011b\f\5\2\2\u011b\u011c\7\60\2\2\u011c\u011d\5\36\20\6"+
-    "\u011d\u011e\b\20\1\2\u011e\u0127\3\2\2\2\u011f\u0120\f\4\2\2\u0120\u0121"+
-    "\7\61\2\2\u0121\u0122\5\36\20\2\u0122\u0123\7\62\2\2\u0123\u0124\5\36"+
-    "\20\4\u0124\u0125\b\20\1\2\u0125\u0127\3\2\2\2\u0126\u00ed\3\2\2\2\u0126"+
-    "\u00f2\3\2\2\2\u0126\u00f7\3\2\2\2\u0126\u00fc\3\2\2\2\u0126\u0101\3\2"+
-    "\2\2\u0126\u0106\3\2\2\2\u0126\u010b\3\2\2\2\u0126\u0110\3\2\2\2\u0126"+
+    "\u0117\7\60\2\2\u0117\u0118\5\36\20\7\u0118\u0119\b\20\1\2\u0119\u0127"+
+    "\3\2\2\2\u011a\u011b\f\5\2\2\u011b\u011c\7\61\2\2\u011c\u011d\5\36\20"+
+    "\6\u011d\u011e\b\20\1\2\u011e\u0127\3\2\2\2\u011f\u0120\f\4\2\2\u0120"+
+    "\u0121\7\62\2\2\u0121\u0122\5\36\20\2\u0122\u0123\7\63\2\2\u0123\u0124"+
+    "\5\36\20\4\u0124\u0125\b\20\1\2\u0125\u0127\3\2\2\2\u0126\u00ed\3\2\2"+
+    "\2\u0126\u00f2\3\2\2\2\u0126\u00f7\3\2\2\2\u0126\u00fc\3\2\2\2\u0126\u0101"+
+    "\3\2\2\2\u0126\u0106\3\2\2\2\u0126\u010b\3\2\2\2\u0126\u0110\3\2\2\2\u0126"+
     "\u0115\3\2\2\2\u0126\u011a\3\2\2\2\u0126\u011f\3\2\2\2\u0127\u012a\3\2"+
     "\2\2\u0128\u0126\3\2\2\2\u0128\u0129\3\2\2\2\u0129\37\3\2\2\2\u012a\u0128"+
     "\3\2\2\2\u012b\u012c\6\21\16\3\u012c\u012d\t\b\2\2\u012d\u0149\5\"\22"+
     "\2\u012e\u012f\6\21\17\3\u012f\u0130\5\"\22\2\u0130\u0131\t\b\2\2\u0131"+
     "\u0149\3\2\2\2\u0132\u0133\6\21\20\3\u0133\u0149\5\"\22\2\u0134\u0135"+
     "\6\21\21\3\u0135\u0136\t\t\2\2\u0136\u0149\b\21\1\2\u0137\u0138\6\21\22"+
-    "\3\u0138\u0139\7H\2\2\u0139\u0149\b\21\1\2\u013a\u013b\6\21\23\3\u013b"+
-    "\u013c\7I\2\2\u013c\u0149\b\21\1\2\u013d\u013e\6\21\24\3\u013e\u013f\7"+
-    "J\2\2\u013f\u0149\b\21\1\2\u0140\u0141\6\21\25\3\u0141\u0142\t\n\2\2\u0142"+
+    "\3\u0138\u0139\7I\2\2\u0139\u0149\b\21\1\2\u013a\u013b\6\21\23\3\u013b"+
+    "\u013c\7J\2\2\u013c\u0149\b\21\1\2\u013d\u013e\6\21\24\3\u013e\u013f\7"+
+    "K\2\2\u013f\u0149\b\21\1\2\u0140\u0141\6\21\25\3\u0141\u0142\t\n\2\2\u0142"+
     "\u0149\5 \21\2\u0143\u0144\7\t\2\2\u0144\u0145\5\26\f\2\u0145\u0146\7"+
     "\n\2\2\u0146\u0147\5 \21\2\u0147\u0149\3\2\2\2\u0148\u012b\3\2\2\2\u0148"+
     "\u012e\3\2\2\2\u0148\u0132\3\2\2\2\u0148\u0134\3\2\2\2\u0148\u0137\3\2"+
@@ -3143,7 +3150,7 @@ class PainlessParser extends Parser {
     "\3\2\2\2\u014f\u016d\3\2\2\2\u0150\u014e\3\2\2\2\u0151\u0152\5\26\f\2"+
     "\u0152\u0156\5(\25\2\u0153\u0155\5&\24\2\u0154\u0153\3\2\2\2\u0155\u0158"+
     "\3\2\2\2\u0156\u0154\3\2\2\2\u0156\u0157\3\2\2\2\u0157\u016d\3\2\2\2\u0158"+
-    "\u0156\3\2\2\2\u0159\u015a\7\26\2\2\u015a\u015f\7K\2\2\u015b\u015c\7\7"+
+    "\u0156\3\2\2\2\u0159\u015a\7\26\2\2\u015a\u015f\7L\2\2\u015b\u015c\7\7"+
     "\2\2\u015c\u015d\5\36\20\2\u015d\u015e\7\b\2\2\u015e\u0160\3\2\2\2\u015f"+
     "\u015b\3\2\2\2\u0160\u0161\3\2\2\2\u0161\u015f\3\2\2\2\u0161\u0162\3\2"+
     "\2\2\u0162\u016a\3\2\2\2\u0163\u0167\5(\25\2\u0164\u0166\5&\24\2\u0165"+
@@ -3153,14 +3160,14 @@ class PainlessParser extends Parser {
     "\2\2\u016c\u0159\3\2\2\2\u016d#\3\2\2\2\u016e\u016f\6\23\26\3\u016f\u0170"+
     "\7\t\2\2\u0170\u0171\5\36\20\2\u0171\u0172\7\n\2\2\u0172\u0173\b\23\1"+
     "\2\u0173\u0181\3\2\2\2\u0174\u0175\6\23\27\3\u0175\u0176\7\t\2\2\u0176"+
-    "\u0177\5 \21\2\u0177\u0178\7\n\2\2\u0178\u0181\3\2\2\2\u0179\u0181\7G"+
-    "\2\2\u017a\u0181\7L\2\2\u017b\u017c\7L\2\2\u017c\u0181\5,\27\2\u017d\u017e"+
-    "\7\26\2\2\u017e\u017f\7K\2\2\u017f\u0181\5,\27\2\u0180\u016e\3\2\2\2\u0180"+
+    "\u0177\5 \21\2\u0177\u0178\7\n\2\2\u0178\u0181\3\2\2\2\u0179\u0181\7H"+
+    "\2\2\u017a\u0181\7M\2\2\u017b\u017c\7M\2\2\u017c\u0181\5,\27\2\u017d\u017e"+
+    "\7\26\2\2\u017e\u017f\7L\2\2\u017f\u0181\5,\27\2\u0180\u016e\3\2\2\2\u0180"+
     "\u0174\3\2\2\2\u0180\u0179\3\2\2\2\u0180\u017a\3\2\2\2\u0180\u017b\3\2"+
     "\2\2\u0180\u017d\3\2\2\2\u0181%\3\2\2\2\u0182\u0183\6\24\30\3\u0183\u0187"+
     "\5(\25\2\u0184\u0185\6\24\31\3\u0185\u0187\5*\26\2\u0186\u0182\3\2\2\2"+
     "\u0186\u0184\3\2\2\2\u0187\'\3\2\2\2\u0188\u0189\7\13\2\2\u0189\u018a"+
-    "\7N\2\2\u018a\u018e\5,\27\2\u018b\u018c\7\13\2\2\u018c\u018e\t\13\2\2"+
+    "\7O\2\2\u018a\u018e\5,\27\2\u018b\u018c\7\13\2\2\u018c\u018e\t\13\2\2"+
     "\u018d\u0188\3\2\2\2\u018d\u018b\3\2\2\2\u018e)\3\2\2\2\u018f\u0190\7"+
     "\7\2\2\u0190\u0191\5\36\20\2\u0191\u0192\7\b\2\2\u0192+\3\2\2\2\u0193"+
     "\u019c\7\t\2\2\u0194\u0199\5.\30\2\u0195\u0196\7\f\2\2\u0196\u0198\5."+
@@ -3174,11 +3181,11 @@ class PainlessParser extends Parser {
     "\u01ae\3\2\2\2\u01ac\u01aa\3\2\2\2\u01ac\u01ad\3\2\2\2\u01ad\u01b0\3\2"+
     "\2\2\u01ae\u01ac\3\2\2\2\u01af\u01a7\3\2\2\2\u01af\u01b0\3\2\2\2\u01b0"+
     "\u01b1\3\2\2\2\u01b1\u01b3\7\n\2\2\u01b2\u01a5\3\2\2\2\u01b2\u01a6\3\2"+
-    "\2\2\u01b3\u01b4\3\2\2\2\u01b4\u01b5\7\64\2\2\u01b5\u01b6\5\f\7\2\u01b6"+
+    "\2\2\u01b3\u01b4\3\2\2\2\u01b4\u01b5\7\65\2\2\u01b5\u01b6\5\f\7\2\u01b6"+
     "\61\3\2\2\2\u01b7\u01b9\5\26\f\2\u01b8\u01b7\3\2\2\2\u01b8\u01b9\3\2\2"+
-    "\2\u01b9\u01ba\3\2\2\2\u01ba\u01bb\7L\2\2\u01bb\63\3\2\2\2\u01bc\u01bd"+
-    "\7K\2\2\u01bd\u01be\7\63\2\2\u01be\u01c3\t\f\2\2\u01bf\u01c0\7L\2\2\u01c0"+
-    "\u01c1\7\63\2\2\u01c1\u01c3\7L\2\2\u01c2\u01bc\3\2\2\2\u01c2\u01bf\3\2"+
+    "\2\u01b9\u01ba\3\2\2\2\u01ba\u01bb\7M\2\2\u01bb\63\3\2\2\2\u01bc\u01bd"+
+    "\7L\2\2\u01bd\u01be\7\64\2\2\u01be\u01c3\t\f\2\2\u01bf\u01c0\t\r\2\2\u01c0"+
+    "\u01c1\7\64\2\2\u01c1\u01c3\7M\2\2\u01c2\u01bc\3\2\2\2\u01c2\u01bf\3\2"+
     "\2\2\u01c3\65\3\2\2\2*9?RUaivz~\u0083\u009e\u00a7\u00ab\u00b1\u00ba\u00c4"+
     "\u00cc\u00d2\u00e0\u00eb\u0126\u0128\u0148\u014e\u0156\u0161\u0167\u016a"+
     "\u016c\u0180\u0186\u018d\u0199\u019c\u01a3\u01ac\u01af\u01b2\u01b8\u01c2";
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index ec523d9f2ba..d3bb65fa06b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -449,6 +449,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
     }
 
+    @Override
     public Object visitDeclvar(DeclvarContext ctx) {
         throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
     }
@@ -958,6 +959,8 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
                 methodText = ctx.ID(0).getText();
             }
             return new EFunctionRef(location(ctx), ctx.TYPE().getText(), methodText);
+        } else if (ctx.THIS() != null) {
+            return new EFunctionRef(location(ctx), ctx.THIS().getText(), ctx.ID(0).getText());
         } else {
             // capturing object::method
             return new ECapturingFunctionRef(location(ctx), ctx.ID(0).getText(), ctx.ID(1).getText());
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index 9d96809bf25..30b4fc37901 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -23,12 +23,15 @@ import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.FunctionRef;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.Definition.Method;
+import org.elasticsearch.painless.Definition.MethodKey;
 import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Type;
 
 import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE;
 
 import java.lang.invoke.LambdaMetafactory;
+import java.lang.reflect.Modifier;
 
 /**
  * Represents a function reference.
@@ -53,7 +56,23 @@ public class EFunctionRef extends AExpression {
             actual = Definition.getType("String");
         } else {
             try {
-                ref = new FunctionRef(expected, type, call);
+                if ("this".equals(type)) {
+                    // user's own function
+                    Method interfaceMethod = expected.struct.getFunctionalMethod();
+                    if (interfaceMethod == null) {
+                        throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
+                                                           "to [" + expected.name + "], not a functional interface");
+                    }
+                    Method implMethod = locals.getMethod(new MethodKey(call, interfaceMethod.arguments.size()));
+                    if (implMethod == null) {
+                        throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
+                                                           "to [" + expected.name + "], function not found");
+                    }
+                    ref = new FunctionRef(expected, interfaceMethod, implMethod);
+                } else {
+                    // whitelist lookup
+                    ref = new FunctionRef(expected, type, call);
+                }
             } catch (IllegalArgumentException e) {
                 throw createError(e);
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index 0a65664fa74..8817db95be6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless.node;
 
-import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Sort;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index 16d53f23bc6..5f3368357bc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -32,6 +32,7 @@ import org.elasticsearch.painless.MethodWriter;
 import org.objectweb.asm.Opcodes;
 
 import java.lang.invoke.MethodType;
+import java.lang.reflect.Modifier;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -94,7 +95,7 @@ public class SFunction extends AStatement {
 
         org.objectweb.asm.commons.Method method =
             new org.objectweb.asm.commons.Method(name, MethodType.methodType(rtnType.clazz, paramClasses).toMethodDescriptorString());
-        this.method = new Method(name, null, rtnType, paramTypes, method, 0, null);
+        this.method = new Method(name, null, rtnType, paramTypes, method, Modifier.STATIC | Modifier.PRIVATE, null);
     }
 
     @Override
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
index d3055ac228b..cd474c898d9 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
@@ -19,8 +19,6 @@
 
 package org.elasticsearch.painless;
 
-import java.util.Comparator;
-
 public class FunctionRefTests extends ScriptTestCase {
 
     public void testStaticMethodReference() {
@@ -110,6 +108,20 @@ public class FunctionRefTests extends ScriptTestCase {
                 "def test = new org.elasticsearch.painless.FeatureTest(2,3);" + 
                 "return test.twoFunctionsOfX(x::concat, y::substring);"));
     }
+    
+    public void testOwnStaticMethodReference() {
+        //System.out.println(Debugger.toString("int mycompare(int i, int j) { return j - i; } " +
+        //                     "List l = new ArrayList(); l.add(2); l.add(1); l.sort(this::mycompare); return l.get(0);"));
+        assertEquals(2, exec("int mycompare(int i, int j) { return j - i; } " +
+                             "List l = new ArrayList(); l.add(2); l.add(1); l.sort(this::mycompare); return l.get(0);"));
+    }
+    
+    @AwaitsFix(bugUrl = "working on it")
+    public void testOwnStaticMethodReferenceDef() {
+        assertEquals(2, exec("int mycompare(int i, int j) { return j - i; } " +
+                             "def l = new ArrayList(); l.add(2); l.add(1); l.sort(this::mycompare); return l.get(0);"));
+    }
+
 
     public void testMethodMissing() {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {

From ab457be9d80aa147259cebc3e83a7ada425b74c2 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Sat, 11 Jun 2016 13:48:04 -0400
Subject: [PATCH 0789/1311] remove unnecessary semicolon and return

---
 .../java/org/elasticsearch/painless/FunctionRefTests.java     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
index cd474c898d9..df05e50ae40 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
@@ -112,13 +112,13 @@ public class FunctionRefTests extends ScriptTestCase {
     public void testOwnStaticMethodReference() {
         //System.out.println(Debugger.toString("int mycompare(int i, int j) { return j - i; } " +
         //                     "List l = new ArrayList(); l.add(2); l.add(1); l.sort(this::mycompare); return l.get(0);"));
-        assertEquals(2, exec("int mycompare(int i, int j) { return j - i; } " +
+        assertEquals(2, exec("int mycompare(int i, int j) { j - i } " +
                              "List l = new ArrayList(); l.add(2); l.add(1); l.sort(this::mycompare); return l.get(0);"));
     }
     
     @AwaitsFix(bugUrl = "working on it")
     public void testOwnStaticMethodReferenceDef() {
-        assertEquals(2, exec("int mycompare(int i, int j) { return j - i; } " +
+        assertEquals(2, exec("int mycompare(int i, int j) { j - i } " +
                              "def l = new ArrayList(); l.add(2); l.add(1); l.sort(this::mycompare); return l.get(0);"));
     }
 

From 8d5a70200f3fc9634351165eb09e7d1432615b8b Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Sat, 11 Jun 2016 14:29:15 -0400
Subject: [PATCH 0790/1311] write refs for functions

---
 .../painless/WriterConstants.java             |  1 +
 .../elasticsearch/painless/node/SSource.java  | 29 ++++++++++++++++++-
 .../painless/FunctionRefTests.java            |  4 +--
 3 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index 025e1433e93..bb7ebd690ed 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -46,6 +46,7 @@ public final class WriterConstants {
     public final static Type CLASS_TYPE        = Type.getObjectType(CLASS_NAME.replace('.', '/'));
 
     public final static Method CONSTRUCTOR = getAsmMethod(void.class, "<init>", String.class, String.class, BitSet.class);
+    public final static Method CLINIT      = getAsmMethod(void.class, "<clinit>");
     public final static Method EXECUTE     =
         getAsmMethod(Object.class, "execute", Map.class, Scorer.class, LeafDocLookup.class, Object.class);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index c7764b3d60a..72496abdb79 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -27,10 +27,13 @@ import org.elasticsearch.painless.Locals.ExecuteReserved;
 import org.elasticsearch.painless.Locals.Variable;
 import org.elasticsearch.painless.WriterConstants;
 import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.Handle;
 import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.Type;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
+import java.lang.invoke.MethodHandle;
 import java.util.BitSet;
 import java.util.Collections;
 import java.util.HashMap;
@@ -151,7 +154,31 @@ public final class SSource extends AStatement {
 
         write(execute);
         execute.endMethod();
-
+        
+        if (!functions.isEmpty()) {
+            // write a reference to each function
+            for (SFunction function : functions) {
+                // XXX: Opcodes.ACC_FINAL too
+                writer.visitField(Opcodes.ACC_PRIVATE | Opcodes.ACC_STATIC, 
+                        "handle$" + function.name + "$" + function.parameters.size(), 
+                        Type.getDescriptor(MethodHandle.class), 
+                        null, 
+                        null).visitEnd();
+            }
+            MethodWriter clinit = new MethodWriter(Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC, 
+                                                   WriterConstants.CLINIT, writer, expressions);
+            for (SFunction function : functions) {
+                Handle handle = new Handle(Opcodes.H_INVOKESTATIC, 
+                                           CLASS_TYPE.getInternalName(), 
+                                           function.name, 
+                                           function.method.method.getDescriptor(), 
+                                           false);
+                clinit.push(handle);
+                clinit.putStatic(CLASS_TYPE, "handle$" + function.name + "$" + function.parameters.size(), Type.getType(MethodHandle.class));
+            }
+            clinit.returnValue();
+            clinit.endMethod();
+        }
         // End writing the class and store the generated bytes.
 
         writer.visitEnd();
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
index df05e50ae40..fcc06e01f1a 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
@@ -110,8 +110,8 @@ public class FunctionRefTests extends ScriptTestCase {
     }
     
     public void testOwnStaticMethodReference() {
-        //System.out.println(Debugger.toString("int mycompare(int i, int j) { return j - i; } " +
-        //                     "List l = new ArrayList(); l.add(2); l.add(1); l.sort(this::mycompare); return l.get(0);"));
+        System.out.println(Debugger.toString("int mycompare(int i, int j) { return j - i; } " +
+                             "List l = new ArrayList(); l.add(2); l.add(1); l.sort(this::mycompare); return l.get(0);"));
         assertEquals(2, exec("int mycompare(int i, int j) { j - i } " +
                              "List l = new ArrayList(); l.add(2); l.add(1); l.sort(this::mycompare); return l.get(0);"));
     }

From e2f999b02553236d6670466fcf6877eaedb9b421 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Sat, 11 Jun 2016 14:33:48 -0400
Subject: [PATCH 0791/1311] make this static final

---
 .../src/main/java/org/elasticsearch/painless/node/SSource.java | 3 +--
 .../test/java/org/elasticsearch/painless/FunctionRefTests.java | 2 --
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index 72496abdb79..1795e4a607d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -158,8 +158,7 @@ public final class SSource extends AStatement {
         if (!functions.isEmpty()) {
             // write a reference to each function
             for (SFunction function : functions) {
-                // XXX: Opcodes.ACC_FINAL too
-                writer.visitField(Opcodes.ACC_PRIVATE | Opcodes.ACC_STATIC, 
+                writer.visitField(Opcodes.ACC_FINAL | Opcodes.ACC_PRIVATE | Opcodes.ACC_STATIC, 
                         "handle$" + function.name + "$" + function.parameters.size(), 
                         Type.getDescriptor(MethodHandle.class), 
                         null, 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
index fcc06e01f1a..8790ba61e40 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
@@ -110,8 +110,6 @@ public class FunctionRefTests extends ScriptTestCase {
     }
     
     public void testOwnStaticMethodReference() {
-        System.out.println(Debugger.toString("int mycompare(int i, int j) { return j - i; } " +
-                             "List l = new ArrayList(); l.add(2); l.add(1); l.sort(this::mycompare); return l.get(0);"));
         assertEquals(2, exec("int mycompare(int i, int j) { j - i } " +
                              "List l = new ArrayList(); l.add(2); l.add(1); l.sort(this::mycompare); return l.get(0);"));
     }

From 876f0a613ef6b12e212a5c80a9e4d5ae2cc74b6d Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Sat, 11 Jun 2016 21:00:06 +0200
Subject: [PATCH 0792/1311] Unfuck the MethodWriter nesting. All MethodWriters
 should only live separately from each other, no nesting

---
 .../elasticsearch/painless/MethodWriter.java  | 14 +++++--------
 .../painless/node/SFunction.java              | 15 +++++++++-----
 .../elasticsearch/painless/node/SSource.java  | 20 +++++++++----------
 3 files changed, 25 insertions(+), 24 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index 5f82cfad56c..2abcace5d6d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -87,7 +87,6 @@ import static org.elasticsearch.painless.WriterConstants.UTILITY_TYPE;
  * shared by the nodes of the Painless tree.
  */
 public final class MethodWriter extends GeneratorAdapter {
-    private final ClassWriter parent;
     private final BitSet statements;
 
     private final Deque<List<org.objectweb.asm.Type>> stringConcatArgs =
@@ -97,17 +96,9 @@ public final class MethodWriter extends GeneratorAdapter {
         super(Opcodes.ASM5, cw.visitMethod(access, method.getName(), method.getDescriptor(), null, null),
                 access, method.getName(), method.getDescriptor());
 
-        this.parent = cw;
         this.statements = statements;
     }
 
-    /**
-     * @return A new {@link MethodWriter} with the specified access and signature.
-     */
-    public MethodWriter newMethodWriter(int access, Method method) {
-        return new MethodWriter(access, method, parent, statements);
-    }
-
     /**
      * Marks a new statement boundary.
      * <p>
@@ -362,4 +353,9 @@ public final class MethodWriter extends GeneratorAdapter {
         super.endMethod();
     }
 
+    @Override
+    public void visitEnd() {
+        throw new AssertionError("Should never call this method on MethodWriter, use endMethod() instead");
+    }
+
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index 5f3368357bc..d9fc6a66ccf 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -29,11 +29,13 @@ import org.elasticsearch.painless.Locals.FunctionReserved;
 import org.elasticsearch.painless.Locals.Variable;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.objectweb.asm.ClassWriter;
 import org.objectweb.asm.Opcodes;
 
 import java.lang.invoke.MethodType;
 import java.lang.reflect.Modifier;
 import java.util.ArrayList;
+import java.util.BitSet;
 import java.util.Collections;
 import java.util.List;
 
@@ -132,11 +134,16 @@ public class SFunction extends AStatement {
 
         locals.decrementScope();
     }
+    
+    /** Writes the function to given ClassWriter. */
+    void write (ClassWriter writer, BitSet statements) {
+        final MethodWriter function = new MethodWriter(Opcodes.ACC_PRIVATE | Opcodes.ACC_STATIC, method.method, writer, statements);
+        write(function);
+        function.endMethod();
+    }
 
     @Override
-    void write(MethodWriter writer) {
-        MethodWriter function = writer.newMethodWriter(Opcodes.ACC_PRIVATE | Opcodes.ACC_STATIC, method.method);
-
+    void write(MethodWriter function) {
         if (reserved.getMaxLoopCounter() > 0) {
             // if there is infinite loop protection, we do this once:
             // int #loop = settings.getMaxLoopCounter()
@@ -158,7 +165,5 @@ public class SFunction extends AStatement {
                 throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
-
-        function.endMethod();
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index 1795e4a607d..54824705f15 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -136,25 +136,29 @@ public final class SSource extends AStatement {
         writer.visit(classVersion, classAccess, className, null, classBase, classInterfaces);
         writer.visitSource(Location.computeSourceName(name, source), null);
 
-        // Create the execute MethodWriter.
-
         expressions = new BitSet(source.length());
-        MethodWriter execute = new MethodWriter(Opcodes.ACC_PUBLIC, EXECUTE, writer, expressions);
 
-        // Write the constructor.
+        // Write the constructor:
 
-        MethodWriter constructor = execute.newMethodWriter(Opcodes.ACC_PUBLIC, CONSTRUCTOR);
+        MethodWriter constructor = new MethodWriter(Opcodes.ACC_PUBLIC, CONSTRUCTOR, writer, expressions);
         constructor.loadThis();
         constructor.loadArgs();
         constructor.invokeConstructor(org.objectweb.asm.Type.getType(Executable.class), CONSTRUCTOR);
         constructor.returnValue();
         constructor.endMethod();
 
-        // Write the execute method.
+        // Write the execute method:
 
+        MethodWriter execute = new MethodWriter(Opcodes.ACC_PUBLIC, EXECUTE, writer, expressions);
         write(execute);
         execute.endMethod();
         
+        // Write all functions:
+        
+        for (SFunction function : functions) {
+            function.write(writer, expressions);
+        }
+
         if (!functions.isEmpty()) {
             // write a reference to each function
             for (SFunction function : functions) {
@@ -186,10 +190,6 @@ public final class SSource extends AStatement {
 
     @Override
     void write(MethodWriter writer) {
-        for (SFunction function : functions) {
-            function.write(writer);
-        }
-
         if (reserved.usesScore()) {
             // if the _score value is used, we do this once:
             // final double _score = scorer.score();

From 592042601fbfc67cb3d99aba428ce43ca580d8de Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Sat, 11 Jun 2016 15:02:21 -0400
Subject: [PATCH 0793/1311] def case working

---
 .../java/org/elasticsearch/painless/Def.java  | 19 +++++++++++++----
 .../elasticsearch/painless/FunctionRef.java   | 21 +++++++++++++++++++
 .../painless/node/EFunctionRef.java           |  1 -
 .../painless/FunctionRefTests.java            |  1 -
 4 files changed, 36 insertions(+), 6 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 102441effb1..c18b724c350 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -29,7 +29,6 @@ import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodHandles.Lookup;
 import java.lang.invoke.MethodType;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
@@ -321,14 +320,26 @@ public final class Def {
                                                          String call, Class<?>... captures) throws LambdaConversionException {
          final FunctionRef ref;
          if ("this".equals(type)) {
+             // user written method
              Method interfaceMethod = clazz.struct.getFunctionalMethod();
              if (interfaceMethod == null) {
                  throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
                                                     "to [" + clazz.name + "], not a functional interface");
              }
-             int arity = interfaceMethod.arguments.size();
-             // user written method
-             ref = null;
+             int arity = interfaceMethod.arguments.size() + captures.length;
+             final MethodHandle handle;
+             try {
+                 MethodHandle accessor = lookup.findStaticGetter(lookup.lookupClass(), 
+                                                                 "handle$" + call + "$" + arity, 
+                                                                 MethodHandle.class);
+                 handle = (MethodHandle) accessor.invokeExact();
+             } catch (NoSuchFieldException | IllegalAccessException e) {
+                 throw new IllegalArgumentException("Unknown call [" + call + "] with [" + arity + "] arguments.");
+             } catch (Throwable t) {
+                 rethrow(t);
+                 throw new AssertionError();
+             }
+             ref = new FunctionRef(clazz, interfaceMethod, handle, captures);
          } else {
              // whitelist lookup
              ref = new FunctionRef(clazz, type, call, captures);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
index 7e0fd520b75..9015b4dee61 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
@@ -101,6 +101,27 @@ public class FunctionRef {
         samMethodType = impl.getMethodType().dropParameterTypes(0, captures.length);
     }
 
+    /**
+     * Creates a new FunctionRef (low level). 
+     * <p>
+     * This will <b>not</b> set implMethodASM. It is for runtime use only.
+     */
+    public FunctionRef(Definition.Type expected, Definition.Method method, MethodHandle impl, Class<?>... captures) {
+        // e.g. compareTo
+        invokedName = method.name;
+        // e.g. (Object)Comparator
+        invokedType = MethodType.methodType(expected.clazz, captures);
+        // e.g. (Object,Object)int
+        interfaceMethodType = method.getMethodType().dropParameterTypes(0, 1);
+
+        implMethod = impl;
+        
+        implMethodASM = null;
+        
+        // remove any prepended captured arguments for the 'natural' signature.
+        samMethodType = impl.type().dropParameterTypes(0, captures.length);
+    }
+
     /** 
      * Looks up {@code type::call} from the whitelist, and returns a matching method.
      */
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index 30b4fc37901..00a69e0b7b9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -31,7 +31,6 @@ import org.objectweb.asm.Type;
 import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE;
 
 import java.lang.invoke.LambdaMetafactory;
-import java.lang.reflect.Modifier;
 
 /**
  * Represents a function reference.
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
index 8790ba61e40..00e52f056d9 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
@@ -114,7 +114,6 @@ public class FunctionRefTests extends ScriptTestCase {
                              "List l = new ArrayList(); l.add(2); l.add(1); l.sort(this::mycompare); return l.get(0);"));
     }
     
-    @AwaitsFix(bugUrl = "working on it")
     public void testOwnStaticMethodReferenceDef() {
         assertEquals(2, exec("int mycompare(int i, int j) { j - i } " +
                              "def l = new ArrayList(); l.add(2); l.add(1); l.sort(this::mycompare); return l.get(0);"));

From 65352face2395130583d69f691b0d6b6ae758f1e Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Sat, 11 Jun 2016 15:04:49 -0400
Subject: [PATCH 0794/1311] fix too long line

---
 .../main/java/org/elasticsearch/painless/node/SSource.java    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index 54824705f15..dfe3f58c821 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -177,7 +177,9 @@ public final class SSource extends AStatement {
                                            function.method.method.getDescriptor(), 
                                            false);
                 clinit.push(handle);
-                clinit.putStatic(CLASS_TYPE, "handle$" + function.name + "$" + function.parameters.size(), Type.getType(MethodHandle.class));
+                clinit.putStatic(CLASS_TYPE, 
+                                 "handle$" + function.name + "$" + function.parameters.size(), 
+                                 Type.getType(MethodHandle.class));
             }
             clinit.returnValue();
             clinit.endMethod();

From c08b45a277a2e2c85966821e61c0ae8c996cea2b Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Sat, 11 Jun 2016 21:20:39 +0200
Subject: [PATCH 0795/1311] More cleanup

---
 .../elasticsearch/painless/node/SFunction.java  |  4 ++++
 .../elasticsearch/painless/node/SSource.java    | 17 ++++++++---------
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index d9fc6a66ccf..4884e73ddce 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -166,4 +166,8 @@ public class SFunction extends AStatement {
             }
         }
     }
+    
+    String getHandleStaticFieldName() {
+        return "handle$" + name + "$" + parameters.size();
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index dfe3f58c821..63fe3f5c377 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -154,42 +154,41 @@ public final class SSource extends AStatement {
         execute.endMethod();
         
         // Write all functions:
-        
         for (SFunction function : functions) {
             function.write(writer, expressions);
         }
 
+        // Write a static field with Handle (function reference) for every function:
         if (!functions.isEmpty()) {
-            // write a reference to each function
             for (SFunction function : functions) {
                 writer.visitField(Opcodes.ACC_FINAL | Opcodes.ACC_PRIVATE | Opcodes.ACC_STATIC, 
-                        "handle$" + function.name + "$" + function.parameters.size(), 
+                        function.getHandleStaticFieldName(), 
                         Type.getDescriptor(MethodHandle.class), 
                         null, 
                         null).visitEnd();
             }
-            MethodWriter clinit = new MethodWriter(Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC, 
+            final MethodWriter clinit = new MethodWriter(Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC, 
                                                    WriterConstants.CLINIT, writer, expressions);
             for (SFunction function : functions) {
-                Handle handle = new Handle(Opcodes.H_INVOKESTATIC, 
+                final Handle handle = new Handle(Opcodes.H_INVOKESTATIC, 
                                            CLASS_TYPE.getInternalName(), 
                                            function.name, 
                                            function.method.method.getDescriptor(), 
                                            false);
                 clinit.push(handle);
                 clinit.putStatic(CLASS_TYPE, 
-                                 "handle$" + function.name + "$" + function.parameters.size(), 
+                                 function.getHandleStaticFieldName(), 
                                  Type.getType(MethodHandle.class));
             }
             clinit.returnValue();
             clinit.endMethod();
         }
-        // End writing the class and store the generated bytes.
-
+        
+        // End writing the class and store the generated bytes:
         writer.visitEnd();
         bytes = writer.toByteArray();
     }
-
+    
     @Override
     void write(MethodWriter writer) {
         if (reserved.usesScore()) {

From 454881eeb4c9c667646408621dfc1ebee13a816c Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Sat, 11 Jun 2016 15:51:32 -0400
Subject: [PATCH 0796/1311] test interface default methods

---
 .../org/elasticsearch/painless/FunctionRefTests.java     | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
index 00e52f056d9..a332ed7dd30 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
@@ -119,6 +119,15 @@ public class FunctionRefTests extends ScriptTestCase {
                              "def l = new ArrayList(); l.add(2); l.add(1); l.sort(this::mycompare); return l.get(0);"));
     }
 
+    public void testInterfaceDefaultMethod() {
+        assertEquals("bar", exec("String f(BiFunction function) { function.apply('foo', 'bar') }" + 
+                                 "Map map = new HashMap(); f(map::getOrDefault)"));
+    }
+    
+    public void testInterfaceDefaultMethodDef() {
+        assertEquals("bar", exec("String f(BiFunction function) { function.apply('foo', 'bar') }" + 
+                                 "def map = new HashMap(); f(map::getOrDefault)"));
+    }
 
     public void testMethodMissing() {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {

From fd9b0f90870823b24512dbca8d8e1f38048e5e9e Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Sat, 11 Jun 2016 21:53:15 +0200
Subject: [PATCH 0797/1311] Remove some hardcoded strings, fix exception
 handling (remove rethrow), add some utility methods around the "handle$"
 fields

---
 .../java/org/elasticsearch/painless/Def.java  | 22 +++++++++----------
 .../painless/WriterConstants.java             |  3 +++
 .../painless/node/SFunction.java              |  5 +++--
 .../elasticsearch/painless/node/SSource.java  | 19 +++++++---------
 4 files changed, 24 insertions(+), 25 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index c18b724c350..48775a88a20 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -23,7 +23,6 @@ import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.RuntimeClass;
 
 import java.lang.invoke.CallSite;
-import java.lang.invoke.LambdaConversionException;
 import java.lang.invoke.LambdaMetafactory;
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
@@ -219,11 +218,11 @@ public final class Def {
      * @param args args passed to callsite
      * @param recipe bitset marking functional parameters
      * @return pointer to matching method to invoke. never returns null.
-     * @throws LambdaConversionException if a method reference cannot be converted to an functional interface
      * @throws IllegalArgumentException if no matching whitelisted method was found.
+     * @throws Throwable if a method reference cannot be converted to an functional interface
      */
      static MethodHandle lookupMethod(Lookup lookup, MethodType callSiteType, 
-             Class<?> receiverClass, String name, Object args[], long recipe) throws LambdaConversionException {
+             Class<?> receiverClass, String name, Object args[], long recipe) throws Throwable {
          // simple case: no lambdas
          if (recipe == 0) {
              return lookupMethodInternal(receiverClass, name, args.length - 1).handle;
@@ -302,7 +301,7 @@ public final class Def {
       * so we simply need to lookup the matching implementation method based on receiver type.
       */
      static MethodHandle lookupReference(Lookup lookup, String interfaceClass, 
-                                         Class<?> receiverClass, String name) throws LambdaConversionException {
+                                         Class<?> receiverClass, String name) throws Throwable {
          Definition.Type interfaceType = Definition.getType(interfaceClass);
          Method interfaceMethod = interfaceType.struct.getFunctionalMethod();
          if (interfaceMethod == null) {
@@ -313,11 +312,9 @@ public final class Def {
          return lookupReferenceInternal(lookup, interfaceType, implMethod.owner.name, implMethod.name, receiverClass);
      }
      
-     /** Returns a method handle to an implementation of clazz, given method reference signature 
-      * @throws LambdaConversionException if a method reference cannot be converted to an functional interface
-      */
+     /** Returns a method handle to an implementation of clazz, given method reference signature. */
      private static MethodHandle lookupReferenceInternal(Lookup lookup, Definition.Type clazz, String type,
-                                                         String call, Class<?>... captures) throws LambdaConversionException {
+                                                         String call, Class<?>... captures) throws Throwable {
          final FunctionRef ref;
          if ("this".equals(type)) {
              // user written method
@@ -330,14 +327,11 @@ public final class Def {
              final MethodHandle handle;
              try {
                  MethodHandle accessor = lookup.findStaticGetter(lookup.lookupClass(), 
-                                                                 "handle$" + call + "$" + arity, 
+                                                                 getUserFunctionHandleFieldName(call, arity), 
                                                                  MethodHandle.class);
                  handle = (MethodHandle) accessor.invokeExact();
              } catch (NoSuchFieldException | IllegalAccessException e) {
                  throw new IllegalArgumentException("Unknown call [" + call + "] with [" + arity + "] arguments.");
-             } catch (Throwable t) {
-                 rethrow(t);
-                 throw new AssertionError();
              }
              ref = new FunctionRef(clazz, interfaceMethod, handle, captures);
          } else {
@@ -367,6 +361,10 @@ public final class Def {
          return callSite.dynamicInvoker().asType(MethodType.methodType(clazz.clazz, captures));
      }
      
+     /** gets the field name used to lookup up the MethodHandle for a function. */
+     public static String getUserFunctionHandleFieldName(String name, int arity) {
+         return "handle$" + name + "$" + arity;
+     }
 
     /**
      * Looks up handle for a dynamic field getter (field load)
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index bb7ebd690ed..038096da99e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -28,6 +28,7 @@ import org.objectweb.asm.commons.Method;
 
 import java.lang.invoke.CallSite;
 import java.lang.invoke.LambdaMetafactory;
+import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodType;
 import java.util.BitSet;
@@ -66,6 +67,8 @@ public final class WriterConstants {
     public final static Type UTILITY_TYPE = Type.getType(Utility.class);
     public final static Method STRING_TO_CHAR = getAsmMethod(char.class, "StringTochar", String.class);
     public final static Method CHAR_TO_STRING = getAsmMethod(String.class, "charToString", char.class);
+    
+    public final static Type METHOD_HANDLE_TYPE = Type.getType(MethodHandle.class);
 
     /** dynamic callsite bootstrap signature */
     public final static MethodType DEF_BOOTSTRAP_TYPE =
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index 4884e73ddce..a422e3b9d3d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.painless.node;
 
+import org.elasticsearch.painless.Def;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
@@ -167,7 +168,7 @@ public class SFunction extends AStatement {
         }
     }
     
-    String getHandleStaticFieldName() {
-        return "handle$" + name + "$" + parameters.size();
+    String getStaticHandleFieldName() {
+        return Def.getUserFunctionHandleFieldName(name, parameters.size());
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index 63fe3f5c377..07b5871f928 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -26,14 +26,13 @@ import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Locals.ExecuteReserved;
 import org.elasticsearch.painless.Locals.Variable;
 import org.elasticsearch.painless.WriterConstants;
-import org.objectweb.asm.ClassWriter;
-import org.objectweb.asm.Handle;
-import org.objectweb.asm.Opcodes;
-import org.objectweb.asm.Type;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
-import java.lang.invoke.MethodHandle;
+import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.Handle;
+import org.objectweb.asm.Opcodes;
+
 import java.util.BitSet;
 import java.util.Collections;
 import java.util.HashMap;
@@ -139,7 +138,6 @@ public final class SSource extends AStatement {
         expressions = new BitSet(source.length());
 
         // Write the constructor:
-
         MethodWriter constructor = new MethodWriter(Opcodes.ACC_PUBLIC, CONSTRUCTOR, writer, expressions);
         constructor.loadThis();
         constructor.loadArgs();
@@ -148,7 +146,6 @@ public final class SSource extends AStatement {
         constructor.endMethod();
 
         // Write the execute method:
-
         MethodWriter execute = new MethodWriter(Opcodes.ACC_PUBLIC, EXECUTE, writer, expressions);
         write(execute);
         execute.endMethod();
@@ -162,8 +159,8 @@ public final class SSource extends AStatement {
         if (!functions.isEmpty()) {
             for (SFunction function : functions) {
                 writer.visitField(Opcodes.ACC_FINAL | Opcodes.ACC_PRIVATE | Opcodes.ACC_STATIC, 
-                        function.getHandleStaticFieldName(), 
-                        Type.getDescriptor(MethodHandle.class), 
+                        function.getStaticHandleFieldName(), 
+                        WriterConstants.METHOD_HANDLE_TYPE.getDescriptor(), 
                         null, 
                         null).visitEnd();
             }
@@ -177,8 +174,8 @@ public final class SSource extends AStatement {
                                            false);
                 clinit.push(handle);
                 clinit.putStatic(CLASS_TYPE, 
-                                 function.getHandleStaticFieldName(), 
-                                 Type.getType(MethodHandle.class));
+                                 function.getStaticHandleFieldName(), 
+                                 WriterConstants.METHOD_HANDLE_TYPE);
             }
             clinit.returnValue();
             clinit.endMethod();

From 07baefac32825628644540a97faae7960765a0f2 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Sat, 11 Jun 2016 16:01:29 -0400
Subject: [PATCH 0798/1311] Add comments

---
 .../src/main/java/org/elasticsearch/painless/Definition.java | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 1c258fd971a..5a66b2154be 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -200,6 +200,11 @@ public final class Definition {
             this.handle = handle;
         }
         
+        /** 
+         * Returns MethodType for this method.
+         * <p>
+         * This works even for user-defined Methods (where the MethodHandle is null).
+         */
         public MethodType getMethodType() {
             // we have a methodhandle already (e.g. whitelisted class)
             // just return its type

From 5c352a063a35654b36963b062d495813b96c5ae8 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Sat, 11 Jun 2016 16:03:45 -0400
Subject: [PATCH 0799/1311] need not be public, should not be public

---
 .../src/main/java/org/elasticsearch/painless/node/SSource.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index 07b5871f928..338652bd56b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -164,7 +164,7 @@ public final class SSource extends AStatement {
                         null, 
                         null).visitEnd();
             }
-            final MethodWriter clinit = new MethodWriter(Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC, 
+            final MethodWriter clinit = new MethodWriter(Opcodes.ACC_STATIC, 
                                                    WriterConstants.CLINIT, writer, expressions);
             for (SFunction function : functions) {
                 final Handle handle = new Handle(Opcodes.H_INVOKESTATIC, 

From 60cc152cafe5ccc8a048d77f234881ca87527e04 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Sat, 11 Jun 2016 22:34:17 +0200
Subject: [PATCH 0800/1311] Remove unused field

---
 .../src/main/java/org/elasticsearch/painless/node/ELambda.java  | 2 --
 1 file changed, 2 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
index 3fac3cdfec8..b513e3813c4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
@@ -33,8 +33,6 @@ public class ELambda extends AExpression {
     final List<String> paramNameStrs;
     final List<AStatement> statements;
 
-    Locals locals = null;
-
     public ELambda(FunctionReserved reserved, Location location,
                    List<String> paramTypes, List<String> paramNames, List<AStatement> statements) {
         super(location);

From 5be9211761ec69d79d6d936e9f0c992574c8dac4 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Sun, 12 Jun 2016 11:25:52 -0400
Subject: [PATCH 0801/1311] cleanup grammar

---
 .../src/main/antlr/PainlessParser.g4          |   28 +-
 .../painless/antlr/PainlessParser.java        | 1122 ++++++++++-------
 .../antlr/PainlessParserBaseVisitor.java      |   28 +
 .../painless/antlr/PainlessParserVisitor.java |   24 +
 .../elasticsearch/painless/antlr/Walker.java  |   46 +-
 5 files changed, 756 insertions(+), 492 deletions(-)

diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index 4ee78d011f3..7aec0db02c7 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -195,6 +195,30 @@ lamtype
     ;
 
 funcref
-    : TYPE REF ( ID | NEW )
-    | ( ID | THIS ) REF ID
+    : classFuncref
+    | constructorFuncref
+    | capturingFuncref
+    | localFuncref
+    ;
+
+// reference to a static or instance method, e.g. ArrayList::size or Integer::compare
+classFuncref
+    : TYPE REF ID
+	;
+
+// reference to a constructor, e.g. ArrayList::new
+// currently limited to simple non-array types
+constructorFuncref
+    : TYPE REF NEW
+    ;
+
+// reference to an instance method, e.g. object::toString
+// currently limited to capture of a simple variable (id).
+capturingFuncref
+    : ID REF ID
+    ;
+
+// reference to a local function, e.g. this::myfunc
+localFuncref
+    : THIS REF ID
     ;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
index 51f4efeb94a..d8a07970afe 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
@@ -34,12 +34,14 @@ class PainlessParser extends Parser {
     RULE_trap = 12, RULE_delimiter = 13, RULE_expression = 14, RULE_unary = 15, 
     RULE_chain = 16, RULE_primary = 17, RULE_secondary = 18, RULE_dot = 19, 
     RULE_brace = 20, RULE_arguments = 21, RULE_argument = 22, RULE_lambda = 23, 
-    RULE_lamtype = 24, RULE_funcref = 25;
+    RULE_lamtype = 24, RULE_funcref = 25, RULE_classFuncref = 26, RULE_constructorFuncref = 27, 
+    RULE_capturingFuncref = 28, RULE_localFuncref = 29;
   public static final String[] ruleNames = {
     "source", "function", "parameters", "statement", "trailer", "block", "empty", 
     "initializer", "afterthought", "declaration", "decltype", "declvar", "trap", 
     "delimiter", "expression", "unary", "chain", "primary", "secondary", "dot", 
-    "brace", "arguments", "argument", "lambda", "lamtype", "funcref"
+    "brace", "arguments", "argument", "lambda", "lamtype", "funcref", "classFuncref", 
+    "constructorFuncref", "capturingFuncref", "localFuncref"
   };
 
   private static final String[] _LITERAL_NAMES = {
@@ -144,39 +146,39 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(55);
+      setState(63);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,0,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(52);
+          setState(60);
           function();
           }
           } 
         }
-        setState(57);
+        setState(65);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,0,_ctx);
       }
-      setState(61);
+      setState(69);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,1,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(58);
+          setState(66);
           statement();
           }
           } 
         }
-        setState(63);
+        setState(71);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,1,_ctx);
       }
-      setState(64);
+      setState(72);
       match(EOF);
       }
     }
@@ -219,13 +221,13 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(66);
+      setState(74);
       decltype();
-      setState(67);
+      setState(75);
       match(ID);
-      setState(68);
+      setState(76);
       parameters();
-      setState(69);
+      setState(77);
       block();
       }
     }
@@ -275,38 +277,38 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(71);
+      setState(79);
       match(LP);
-      setState(83);
+      setState(91);
       _la = _input.LA(1);
       if (_la==TYPE) {
         {
-        setState(72);
-        decltype();
-        setState(73);
-        match(ID);
         setState(80);
+        decltype();
+        setState(81);
+        match(ID);
+        setState(88);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(74);
+          setState(82);
           match(COMMA);
-          setState(75);
+          setState(83);
           decltype();
-          setState(76);
+          setState(84);
           match(ID);
           }
           }
-          setState(82);
+          setState(90);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
       }
 
-      setState(85);
+      setState(93);
       match(RP);
       }
     }
@@ -552,35 +554,35 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 6, RULE_statement);
     try {
       int _alt;
-      setState(165);
+      setState(173);
       switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) {
       case 1:
         _localctx = new IfContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(87);
-        match(IF);
-        setState(88);
-        match(LP);
-        setState(89);
-        expression(0);
-        setState(90);
-        match(RP);
-        setState(91);
-        trailer();
         setState(95);
+        match(IF);
+        setState(96);
+        match(LP);
+        setState(97);
+        expression(0);
+        setState(98);
+        match(RP);
+        setState(99);
+        trailer();
+        setState(103);
         switch ( getInterpreter().adaptivePredict(_input,4,_ctx) ) {
         case 1:
           {
-          setState(92);
+          setState(100);
           match(ELSE);
-          setState(93);
+          setState(101);
           trailer();
           }
           break;
         case 2:
           {
-          setState(94);
+          setState(102);
           if (!( _input.LA(1) != ELSE )) throw new FailedPredicateException(this, " _input.LA(1) != ELSE ");
           }
           break;
@@ -591,25 +593,25 @@ class PainlessParser extends Parser {
         _localctx = new WhileContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(97);
+        setState(105);
         match(WHILE);
-        setState(98);
+        setState(106);
         match(LP);
-        setState(99);
+        setState(107);
         expression(0);
-        setState(100);
+        setState(108);
         match(RP);
-        setState(103);
+        setState(111);
         switch ( getInterpreter().adaptivePredict(_input,5,_ctx) ) {
         case 1:
           {
-          setState(101);
+          setState(109);
           trailer();
           }
           break;
         case 2:
           {
-          setState(102);
+          setState(110);
           empty();
           }
           break;
@@ -620,19 +622,19 @@ class PainlessParser extends Parser {
         _localctx = new DoContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(105);
+        setState(113);
         match(DO);
-        setState(106);
+        setState(114);
         block();
-        setState(107);
+        setState(115);
         match(WHILE);
-        setState(108);
+        setState(116);
         match(LP);
-        setState(109);
+        setState(117);
         expression(0);
-        setState(110);
+        setState(118);
         match(RP);
-        setState(111);
+        setState(119);
         delimiter();
         }
         break;
@@ -640,54 +642,54 @@ class PainlessParser extends Parser {
         _localctx = new ForContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(113);
+        setState(121);
         match(FOR);
-        setState(114);
+        setState(122);
         match(LP);
-        setState(116);
+        setState(124);
         switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) {
         case 1:
           {
-          setState(115);
+          setState(123);
           initializer();
           }
           break;
         }
-        setState(118);
+        setState(126);
         match(SEMICOLON);
-        setState(120);
+        setState(128);
         switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) {
         case 1:
           {
-          setState(119);
+          setState(127);
           expression(0);
           }
           break;
         }
-        setState(122);
+        setState(130);
         match(SEMICOLON);
-        setState(124);
+        setState(132);
         switch ( getInterpreter().adaptivePredict(_input,8,_ctx) ) {
         case 1:
           {
-          setState(123);
+          setState(131);
           afterthought();
           }
           break;
         }
-        setState(126);
+        setState(134);
         match(RP);
-        setState(129);
+        setState(137);
         switch ( getInterpreter().adaptivePredict(_input,9,_ctx) ) {
         case 1:
           {
-          setState(127);
+          setState(135);
           trailer();
           }
           break;
         case 2:
           {
-          setState(128);
+          setState(136);
           empty();
           }
           break;
@@ -698,21 +700,21 @@ class PainlessParser extends Parser {
         _localctx = new EachContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(131);
+        setState(139);
         match(FOR);
-        setState(132);
+        setState(140);
         match(LP);
-        setState(133);
+        setState(141);
         decltype();
-        setState(134);
+        setState(142);
         match(ID);
-        setState(135);
+        setState(143);
         match(COLON);
-        setState(136);
+        setState(144);
         expression(0);
-        setState(137);
+        setState(145);
         match(RP);
-        setState(138);
+        setState(146);
         trailer();
         }
         break;
@@ -720,9 +722,9 @@ class PainlessParser extends Parser {
         _localctx = new DeclContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(140);
+        setState(148);
         declaration();
-        setState(141);
+        setState(149);
         delimiter();
         }
         break;
@@ -730,9 +732,9 @@ class PainlessParser extends Parser {
         _localctx = new ContinueContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(143);
+        setState(151);
         match(CONTINUE);
-        setState(144);
+        setState(152);
         delimiter();
         }
         break;
@@ -740,9 +742,9 @@ class PainlessParser extends Parser {
         _localctx = new BreakContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(145);
+        setState(153);
         match(BREAK);
-        setState(146);
+        setState(154);
         delimiter();
         }
         break;
@@ -750,11 +752,11 @@ class PainlessParser extends Parser {
         _localctx = new ReturnContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(147);
+        setState(155);
         match(RETURN);
-        setState(148);
+        setState(156);
         expression(0);
-        setState(149);
+        setState(157);
         delimiter();
         }
         break;
@@ -762,11 +764,11 @@ class PainlessParser extends Parser {
         _localctx = new TryContext(_localctx);
         enterOuterAlt(_localctx, 10);
         {
-        setState(151);
+        setState(159);
         match(TRY);
-        setState(152);
+        setState(160);
         block();
-        setState(154); 
+        setState(162); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -774,7 +776,7 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(153);
+            setState(161);
             trap();
             }
             }
@@ -782,7 +784,7 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(156); 
+          setState(164); 
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,10,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
@@ -792,11 +794,11 @@ class PainlessParser extends Parser {
         _localctx = new ThrowContext(_localctx);
         enterOuterAlt(_localctx, 11);
         {
-        setState(158);
+        setState(166);
         match(THROW);
-        setState(159);
+        setState(167);
         expression(0);
-        setState(160);
+        setState(168);
         delimiter();
         }
         break;
@@ -804,9 +806,9 @@ class PainlessParser extends Parser {
         _localctx = new ExprContext(_localctx);
         enterOuterAlt(_localctx, 12);
         {
-        setState(162);
+        setState(170);
         expression(0);
-        setState(163);
+        setState(171);
         delimiter();
         }
         break;
@@ -845,19 +847,19 @@ class PainlessParser extends Parser {
     TrailerContext _localctx = new TrailerContext(_ctx, getState());
     enterRule(_localctx, 8, RULE_trailer);
     try {
-      setState(169);
+      setState(177);
       switch ( getInterpreter().adaptivePredict(_input,12,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(167);
+        setState(175);
         block();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(168);
+        setState(176);
         statement();
         }
         break;
@@ -901,25 +903,25 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(171);
+      setState(179);
       match(LBRACK);
-      setState(175);
+      setState(183);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,13,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(172);
+          setState(180);
           statement();
           }
           } 
         }
-        setState(177);
+        setState(185);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,13,_ctx);
       }
-      setState(178);
+      setState(186);
       match(RBRACK);
       }
     }
@@ -953,7 +955,7 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(180);
+      setState(188);
       match(SEMICOLON);
       }
     }
@@ -990,19 +992,19 @@ class PainlessParser extends Parser {
     InitializerContext _localctx = new InitializerContext(_ctx, getState());
     enterRule(_localctx, 14, RULE_initializer);
     try {
-      setState(184);
+      setState(192);
       switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(182);
+        setState(190);
         declaration();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(183);
+        setState(191);
         expression(0);
         }
         break;
@@ -1040,7 +1042,7 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(186);
+      setState(194);
       expression(0);
       }
     }
@@ -1087,23 +1089,23 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(188);
+      setState(196);
       decltype();
-      setState(189);
+      setState(197);
       declvar();
-      setState(194);
+      setState(202);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==COMMA) {
         {
         {
-        setState(190);
+        setState(198);
         match(COMMA);
-        setState(191);
+        setState(199);
         declvar();
         }
         }
-        setState(196);
+        setState(204);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
@@ -1148,21 +1150,21 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(197);
+      setState(205);
       match(TYPE);
-      setState(202);
+      setState(210);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==LBRACE) {
         {
         {
-        setState(198);
+        setState(206);
         match(LBRACE);
-        setState(199);
+        setState(207);
         match(RBRACE);
         }
         }
-        setState(204);
+        setState(212);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
@@ -1203,15 +1205,15 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(205);
+      setState(213);
       match(ID);
-      setState(208);
+      setState(216);
       _la = _input.LA(1);
       if (_la==ASSIGN) {
         {
-        setState(206);
+        setState(214);
         match(ASSIGN);
-        setState(207);
+        setState(215);
         expression(0);
         }
       }
@@ -1255,17 +1257,17 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(210);
+      setState(218);
       match(CATCH);
-      setState(211);
+      setState(219);
       match(LP);
-      setState(212);
+      setState(220);
       match(TYPE);
-      setState(213);
+      setState(221);
       match(ID);
-      setState(214);
+      setState(222);
       match(RP);
-      setState(215);
+      setState(223);
       block();
       }
     }
@@ -1299,19 +1301,19 @@ class PainlessParser extends Parser {
     DelimiterContext _localctx = new DelimiterContext(_ctx, getState());
     enterRule(_localctx, 26, RULE_delimiter);
     try {
-      setState(222);
+      setState(230);
       switch (_input.LA(1)) {
       case SEMICOLON:
         enterOuterAlt(_localctx, 1);
         {
-        setState(217);
+        setState(225);
         match(SEMICOLON);
         }
         break;
       case EOF:
         enterOuterAlt(_localctx, 2);
         {
-        setState(218);
+        setState(226);
         match(EOF);
         }
         break;
@@ -1319,7 +1321,7 @@ class PainlessParser extends Parser {
         enterOuterAlt(_localctx, 3);
         {
          int mark = _input.mark(); int index = _input.index(); 
-        setState(220);
+        setState(228);
         match(RBRACK);
          _input.seek(index); _input.release(mark); 
         }
@@ -1488,7 +1490,7 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(233);
+      setState(241);
       switch ( getInterpreter().adaptivePredict(_input,19,_ctx) ) {
       case 1:
         {
@@ -1496,16 +1498,16 @@ class PainlessParser extends Parser {
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(225);
+        setState(233);
         chain(true);
-        setState(226);
+        setState(234);
         _la = _input.LA(1);
         if ( !(((((_la - 54)) & ~0x3f) == 0 && ((1L << (_la - 54)) & ((1L << (ASSIGN - 54)) | (1L << (AADD - 54)) | (1L << (ASUB - 54)) | (1L << (AMUL - 54)) | (1L << (ADIV - 54)) | (1L << (AREM - 54)) | (1L << (AAND - 54)) | (1L << (AXOR - 54)) | (1L << (AOR - 54)) | (1L << (ALSH - 54)) | (1L << (ARSH - 54)) | (1L << (AUSH - 54)))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(227);
+        setState(235);
         expression(1);
          ((AssignmentContext)_localctx).s =  false; 
         }
@@ -1515,14 +1517,14 @@ class PainlessParser extends Parser {
         _localctx = new SingleContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(230);
+        setState(238);
         ((SingleContext)_localctx).u = unary(false);
          ((SingleContext)_localctx).s =  ((SingleContext)_localctx).u.s; 
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(294);
+      setState(302);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -1530,22 +1532,22 @@ class PainlessParser extends Parser {
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(292);
+          setState(300);
           switch ( getInterpreter().adaptivePredict(_input,20,_ctx) ) {
           case 1:
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(235);
+            setState(243);
             if (!(precpred(_ctx, 12))) throw new FailedPredicateException(this, "precpred(_ctx, 12)");
-            setState(236);
+            setState(244);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << MUL) | (1L << DIV) | (1L << REM))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(237);
+            setState(245);
             expression(13);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1554,16 +1556,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(240);
+            setState(248);
             if (!(precpred(_ctx, 11))) throw new FailedPredicateException(this, "precpred(_ctx, 11)");
-            setState(241);
+            setState(249);
             _la = _input.LA(1);
             if ( !(_la==ADD || _la==SUB) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(242);
+            setState(250);
             expression(12);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1572,16 +1574,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(245);
+            setState(253);
             if (!(precpred(_ctx, 10))) throw new FailedPredicateException(this, "precpred(_ctx, 10)");
-            setState(246);
+            setState(254);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LSH) | (1L << RSH) | (1L << USH))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(247);
+            setState(255);
             expression(11);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1590,16 +1592,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(250);
+            setState(258);
             if (!(precpred(_ctx, 9))) throw new FailedPredicateException(this, "precpred(_ctx, 9)");
-            setState(251);
+            setState(259);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LT) | (1L << LTE) | (1L << GT) | (1L << GTE))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(252);
+            setState(260);
             expression(10);
              ((CompContext)_localctx).s =  false; 
             }
@@ -1608,16 +1610,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(255);
+            setState(263);
             if (!(precpred(_ctx, 8))) throw new FailedPredicateException(this, "precpred(_ctx, 8)");
-            setState(256);
+            setState(264);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << EQ) | (1L << EQR) | (1L << NE) | (1L << NER))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(257);
+            setState(265);
             expression(9);
              ((CompContext)_localctx).s =  false; 
             }
@@ -1626,11 +1628,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(260);
+            setState(268);
             if (!(precpred(_ctx, 7))) throw new FailedPredicateException(this, "precpred(_ctx, 7)");
-            setState(261);
+            setState(269);
             match(BWAND);
-            setState(262);
+            setState(270);
             expression(8);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1639,11 +1641,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(265);
+            setState(273);
             if (!(precpred(_ctx, 6))) throw new FailedPredicateException(this, "precpred(_ctx, 6)");
-            setState(266);
+            setState(274);
             match(XOR);
-            setState(267);
+            setState(275);
             expression(7);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1652,11 +1654,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(270);
+            setState(278);
             if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)");
-            setState(271);
+            setState(279);
             match(BWOR);
-            setState(272);
+            setState(280);
             expression(6);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1665,11 +1667,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(275);
+            setState(283);
             if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)");
-            setState(276);
+            setState(284);
             match(BOOLAND);
-            setState(277);
+            setState(285);
             expression(5);
              ((BoolContext)_localctx).s =  false; 
             }
@@ -1678,11 +1680,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(280);
+            setState(288);
             if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)");
-            setState(281);
+            setState(289);
             match(BOOLOR);
-            setState(282);
+            setState(290);
             expression(4);
              ((BoolContext)_localctx).s =  false; 
             }
@@ -1691,15 +1693,15 @@ class PainlessParser extends Parser {
             {
             _localctx = new ConditionalContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(285);
+            setState(293);
             if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
-            setState(286);
+            setState(294);
             match(COND);
-            setState(287);
+            setState(295);
             ((ConditionalContext)_localctx).e0 = expression(0);
-            setState(288);
+            setState(296);
             match(COLON);
-            setState(289);
+            setState(297);
             ((ConditionalContext)_localctx).e1 = expression(2);
              ((ConditionalContext)_localctx).s =  ((ConditionalContext)_localctx).e0.s && ((ConditionalContext)_localctx).e1.s; 
             }
@@ -1707,7 +1709,7 @@ class PainlessParser extends Parser {
           }
           } 
         }
-        setState(296);
+        setState(304);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
       }
@@ -1854,22 +1856,22 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 30, RULE_unary);
     int _la;
     try {
-      setState(326);
+      setState(334);
       switch ( getInterpreter().adaptivePredict(_input,22,_ctx) ) {
       case 1:
         _localctx = new PreContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(297);
+        setState(305);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(298);
+        setState(306);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(299);
+        setState(307);
         chain(true);
         }
         break;
@@ -1877,11 +1879,11 @@ class PainlessParser extends Parser {
         _localctx = new PostContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(300);
+        setState(308);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(301);
+        setState(309);
         chain(true);
-        setState(302);
+        setState(310);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
@@ -1894,9 +1896,9 @@ class PainlessParser extends Parser {
         _localctx = new ReadContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(304);
+        setState(312);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(305);
+        setState(313);
         chain(false);
         }
         break;
@@ -1904,9 +1906,9 @@ class PainlessParser extends Parser {
         _localctx = new NumericContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(306);
+        setState(314);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(307);
+        setState(315);
         _la = _input.LA(1);
         if ( !(((((_la - 66)) & ~0x3f) == 0 && ((1L << (_la - 66)) & ((1L << (OCTAL - 66)) | (1L << (HEX - 66)) | (1L << (INTEGER - 66)) | (1L << (DECIMAL - 66)))) != 0)) ) {
         _errHandler.recoverInline(this);
@@ -1920,9 +1922,9 @@ class PainlessParser extends Parser {
         _localctx = new TrueContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(309);
+        setState(317);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(310);
+        setState(318);
         match(TRUE);
          ((TrueContext)_localctx).s =  false; 
         }
@@ -1931,9 +1933,9 @@ class PainlessParser extends Parser {
         _localctx = new FalseContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(312);
+        setState(320);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(313);
+        setState(321);
         match(FALSE);
          ((FalseContext)_localctx).s =  false; 
         }
@@ -1942,9 +1944,9 @@ class PainlessParser extends Parser {
         _localctx = new NullContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(315);
+        setState(323);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(316);
+        setState(324);
         match(NULL);
          ((NullContext)_localctx).s =  false; 
         }
@@ -1953,16 +1955,16 @@ class PainlessParser extends Parser {
         _localctx = new OperatorContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(318);
+        setState(326);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(319);
+        setState(327);
         _la = _input.LA(1);
         if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(320);
+        setState(328);
         unary(false);
         }
         break;
@@ -1970,13 +1972,13 @@ class PainlessParser extends Parser {
         _localctx = new CastContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(321);
+        setState(329);
         match(LP);
-        setState(322);
+        setState(330);
         decltype();
-        setState(323);
+        setState(331);
         match(RP);
-        setState(324);
+        setState(332);
         unary(_localctx.c);
         }
         break;
@@ -2085,27 +2087,27 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 32, RULE_chain);
     try {
       int _alt;
-      setState(362);
+      setState(370);
       switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) {
       case 1:
         _localctx = new DynamicContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(328);
+        setState(336);
         ((DynamicContext)_localctx).p = primary(_localctx.c);
-        setState(332);
+        setState(340);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(329);
+            setState(337);
             secondary(((DynamicContext)_localctx).p.s);
             }
             } 
           }
-          setState(334);
+          setState(342);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
         }
@@ -2115,23 +2117,23 @@ class PainlessParser extends Parser {
         _localctx = new StaticContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(335);
+        setState(343);
         decltype();
-        setState(336);
+        setState(344);
         dot();
-        setState(340);
+        setState(348);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,24,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(337);
+            setState(345);
             secondary(true);
             }
             } 
           }
-          setState(342);
+          setState(350);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,24,_ctx);
         }
@@ -2141,11 +2143,11 @@ class PainlessParser extends Parser {
         _localctx = new NewarrayContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(343);
+        setState(351);
         match(NEW);
-        setState(344);
+        setState(352);
         match(TYPE);
-        setState(349); 
+        setState(357); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -2153,11 +2155,11 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(345);
+            setState(353);
             match(LBRACE);
-            setState(346);
+            setState(354);
             expression(0);
-            setState(347);
+            setState(355);
             match(RBRACE);
             }
             }
@@ -2165,29 +2167,29 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(351); 
+          setState(359); 
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,25,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
-        setState(360);
+        setState(368);
         switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) {
         case 1:
           {
-          setState(353);
+          setState(361);
           dot();
-          setState(357);
+          setState(365);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,26,_ctx);
           while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
             if ( _alt==1 ) {
               {
               {
-              setState(354);
+              setState(362);
               secondary(true);
               }
               } 
             }
-            setState(359);
+            setState(367);
             _errHandler.sync(this);
             _alt = getInterpreter().adaptivePredict(_input,26,_ctx);
           }
@@ -2301,19 +2303,19 @@ class PainlessParser extends Parser {
     PrimaryContext _localctx = new PrimaryContext(_ctx, getState(), c);
     enterRule(_localctx, 34, RULE_primary);
     try {
-      setState(382);
+      setState(390);
       switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
       case 1:
         _localctx = new ExprprecContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(364);
+        setState(372);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(365);
+        setState(373);
         match(LP);
-        setState(366);
+        setState(374);
         ((ExprprecContext)_localctx).e = expression(0);
-        setState(367);
+        setState(375);
         match(RP);
          ((ExprprecContext)_localctx).s =  ((ExprprecContext)_localctx).e.s; 
         }
@@ -2322,13 +2324,13 @@ class PainlessParser extends Parser {
         _localctx = new ChainprecContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(370);
+        setState(378);
         if (!( _localctx.c )) throw new FailedPredicateException(this, " $c ");
-        setState(371);
+        setState(379);
         match(LP);
-        setState(372);
+        setState(380);
         unary(true);
-        setState(373);
+        setState(381);
         match(RP);
         }
         break;
@@ -2336,7 +2338,7 @@ class PainlessParser extends Parser {
         _localctx = new StringContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(375);
+        setState(383);
         match(STRING);
         }
         break;
@@ -2344,7 +2346,7 @@ class PainlessParser extends Parser {
         _localctx = new VariableContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(376);
+        setState(384);
         match(ID);
         }
         break;
@@ -2352,9 +2354,9 @@ class PainlessParser extends Parser {
         _localctx = new CalllocalContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(377);
+        setState(385);
         match(ID);
-        setState(378);
+        setState(386);
         arguments();
         }
         break;
@@ -2362,11 +2364,11 @@ class PainlessParser extends Parser {
         _localctx = new NewobjectContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(379);
+        setState(387);
         match(NEW);
-        setState(380);
+        setState(388);
         match(TYPE);
-        setState(381);
+        setState(389);
         arguments();
         }
         break;
@@ -2408,23 +2410,23 @@ class PainlessParser extends Parser {
     SecondaryContext _localctx = new SecondaryContext(_ctx, getState(), s);
     enterRule(_localctx, 36, RULE_secondary);
     try {
-      setState(388);
+      setState(396);
       switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(384);
+        setState(392);
         if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
-        setState(385);
+        setState(393);
         dot();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(386);
+        setState(394);
         if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
-        setState(387);
+        setState(395);
         brace();
         }
         break;
@@ -2482,17 +2484,17 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 38, RULE_dot);
     int _la;
     try {
-      setState(395);
+      setState(403);
       switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) {
       case 1:
         _localctx = new CallinvokeContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(390);
+        setState(398);
         match(DOT);
-        setState(391);
+        setState(399);
         match(DOTID);
-        setState(392);
+        setState(400);
         arguments();
         }
         break;
@@ -2500,9 +2502,9 @@ class PainlessParser extends Parser {
         _localctx = new FieldaccessContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(393);
+        setState(401);
         match(DOT);
-        setState(394);
+        setState(402);
         _la = _input.LA(1);
         if ( !(_la==DOTINTEGER || _la==DOTID) ) {
         _errHandler.recoverInline(this);
@@ -2556,11 +2558,11 @@ class PainlessParser extends Parser {
       _localctx = new BraceaccessContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(397);
+      setState(405);
       match(LBRACE);
-      setState(398);
+      setState(406);
       expression(0);
-      setState(399);
+      setState(407);
       match(RBRACE);
       }
     }
@@ -2607,34 +2609,34 @@ class PainlessParser extends Parser {
       enterOuterAlt(_localctx, 1);
       {
       {
-      setState(401);
+      setState(409);
       match(LP);
-      setState(410);
+      setState(418);
       switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) {
       case 1:
         {
-        setState(402);
+        setState(410);
         argument();
-        setState(407);
+        setState(415);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(403);
+          setState(411);
           match(COMMA);
-          setState(404);
+          setState(412);
           argument();
           }
           }
-          setState(409);
+          setState(417);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
         break;
       }
-      setState(412);
+      setState(420);
       match(RP);
       }
       }
@@ -2675,26 +2677,26 @@ class PainlessParser extends Parser {
     ArgumentContext _localctx = new ArgumentContext(_ctx, getState());
     enterRule(_localctx, 44, RULE_argument);
     try {
-      setState(417);
+      setState(425);
       switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(414);
+        setState(422);
         expression(0);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(415);
+        setState(423);
         lambda();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(416);
+        setState(424);
         funcref();
         }
         break;
@@ -2746,54 +2748,54 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(432);
+      setState(440);
       switch (_input.LA(1)) {
       case TYPE:
       case ID:
         {
-        setState(419);
+        setState(427);
         lamtype();
         }
         break;
       case LP:
         {
-        setState(420);
+        setState(428);
         match(LP);
-        setState(429);
+        setState(437);
         _la = _input.LA(1);
         if (_la==TYPE || _la==ID) {
           {
-          setState(421);
+          setState(429);
           lamtype();
-          setState(426);
+          setState(434);
           _errHandler.sync(this);
           _la = _input.LA(1);
           while (_la==COMMA) {
             {
             {
-            setState(422);
+            setState(430);
             match(COMMA);
-            setState(423);
+            setState(431);
             lamtype();
             }
             }
-            setState(428);
+            setState(436);
             _errHandler.sync(this);
             _la = _input.LA(1);
           }
           }
         }
 
-        setState(431);
+        setState(439);
         match(RP);
         }
         break;
       default:
         throw new NoViableAltException(this);
       }
-      setState(434);
+      setState(442);
       match(ARROW);
-      setState(435);
+      setState(443);
       block();
       }
     }
@@ -2831,16 +2833,16 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(438);
+      setState(446);
       _la = _input.LA(1);
       if (_la==TYPE) {
         {
-        setState(437);
+        setState(445);
         decltype();
         }
       }
 
-      setState(440);
+      setState(448);
       match(ID);
       }
     }
@@ -2856,14 +2858,18 @@ class PainlessParser extends Parser {
   }
 
   public static class FuncrefContext extends ParserRuleContext {
-    public TerminalNode TYPE() { return getToken(PainlessParser.TYPE, 0); }
-    public TerminalNode REF() { return getToken(PainlessParser.REF, 0); }
-    public List<TerminalNode> ID() { return getTokens(PainlessParser.ID); }
-    public TerminalNode ID(int i) {
-      return getToken(PainlessParser.ID, i);
+    public ClassFuncrefContext classFuncref() {
+      return getRuleContext(ClassFuncrefContext.class,0);
+    }
+    public ConstructorFuncrefContext constructorFuncref() {
+      return getRuleContext(ConstructorFuncrefContext.class,0);
+    }
+    public CapturingFuncrefContext capturingFuncref() {
+      return getRuleContext(CapturingFuncrefContext.class,0);
+    }
+    public LocalFuncrefContext localFuncref() {
+      return getRuleContext(LocalFuncrefContext.class,0);
     }
-    public TerminalNode NEW() { return getToken(PainlessParser.NEW, 0); }
-    public TerminalNode THIS() { return getToken(PainlessParser.THIS, 0); }
     public FuncrefContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
     }
@@ -2878,45 +2884,199 @@ class PainlessParser extends Parser {
   public final FuncrefContext funcref() throws RecognitionException {
     FuncrefContext _localctx = new FuncrefContext(_ctx, getState());
     enterRule(_localctx, 50, RULE_funcref);
-    int _la;
     try {
-      setState(448);
-      switch (_input.LA(1)) {
-      case TYPE:
+      setState(454);
+      switch ( getInterpreter().adaptivePredict(_input,39,_ctx) ) {
+      case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(442);
-        match(TYPE);
-        setState(443);
-        match(REF);
-        setState(444);
-        _la = _input.LA(1);
-        if ( !(_la==NEW || _la==ID) ) {
-        _errHandler.recoverInline(this);
-        } else {
-          consume();
-        }
+        setState(450);
+        classFuncref();
         }
         break;
-      case THIS:
-      case ID:
+      case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(445);
-        _la = _input.LA(1);
-        if ( !(_la==THIS || _la==ID) ) {
-        _errHandler.recoverInline(this);
-        } else {
-          consume();
-        }
-        setState(446);
-        match(REF);
-        setState(447);
-        match(ID);
+        setState(451);
+        constructorFuncref();
         }
         break;
-      default:
-        throw new NoViableAltException(this);
+      case 3:
+        enterOuterAlt(_localctx, 3);
+        {
+        setState(452);
+        capturingFuncref();
+        }
+        break;
+      case 4:
+        enterOuterAlt(_localctx, 4);
+        {
+        setState(453);
+        localFuncref();
+        }
+        break;
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
+  public static class ClassFuncrefContext extends ParserRuleContext {
+    public TerminalNode TYPE() { return getToken(PainlessParser.TYPE, 0); }
+    public TerminalNode REF() { return getToken(PainlessParser.REF, 0); }
+    public TerminalNode ID() { return getToken(PainlessParser.ID, 0); }
+    public ClassFuncrefContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_classFuncref; }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitClassFuncref(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final ClassFuncrefContext classFuncref() throws RecognitionException {
+    ClassFuncrefContext _localctx = new ClassFuncrefContext(_ctx, getState());
+    enterRule(_localctx, 52, RULE_classFuncref);
+    try {
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(456);
+      match(TYPE);
+      setState(457);
+      match(REF);
+      setState(458);
+      match(ID);
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
+  public static class ConstructorFuncrefContext extends ParserRuleContext {
+    public TerminalNode TYPE() { return getToken(PainlessParser.TYPE, 0); }
+    public TerminalNode REF() { return getToken(PainlessParser.REF, 0); }
+    public TerminalNode NEW() { return getToken(PainlessParser.NEW, 0); }
+    public ConstructorFuncrefContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_constructorFuncref; }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitConstructorFuncref(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final ConstructorFuncrefContext constructorFuncref() throws RecognitionException {
+    ConstructorFuncrefContext _localctx = new ConstructorFuncrefContext(_ctx, getState());
+    enterRule(_localctx, 54, RULE_constructorFuncref);
+    try {
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(460);
+      match(TYPE);
+      setState(461);
+      match(REF);
+      setState(462);
+      match(NEW);
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
+  public static class CapturingFuncrefContext extends ParserRuleContext {
+    public List<TerminalNode> ID() { return getTokens(PainlessParser.ID); }
+    public TerminalNode ID(int i) {
+      return getToken(PainlessParser.ID, i);
+    }
+    public TerminalNode REF() { return getToken(PainlessParser.REF, 0); }
+    public CapturingFuncrefContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_capturingFuncref; }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitCapturingFuncref(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final CapturingFuncrefContext capturingFuncref() throws RecognitionException {
+    CapturingFuncrefContext _localctx = new CapturingFuncrefContext(_ctx, getState());
+    enterRule(_localctx, 56, RULE_capturingFuncref);
+    try {
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(464);
+      match(ID);
+      setState(465);
+      match(REF);
+      setState(466);
+      match(ID);
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
+  public static class LocalFuncrefContext extends ParserRuleContext {
+    public TerminalNode THIS() { return getToken(PainlessParser.THIS, 0); }
+    public TerminalNode REF() { return getToken(PainlessParser.REF, 0); }
+    public TerminalNode ID() { return getToken(PainlessParser.ID, 0); }
+    public LocalFuncrefContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_localFuncref; }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitLocalFuncref(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final LocalFuncrefContext localFuncref() throws RecognitionException {
+    LocalFuncrefContext _localctx = new LocalFuncrefContext(_ctx, getState());
+    enterRule(_localctx, 58, RULE_localFuncref);
+    try {
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(468);
+      match(THIS);
+      setState(469);
+      match(REF);
+      setState(470);
+      match(ID);
       }
     }
     catch (RecognitionException re) {
@@ -3020,175 +3180,183 @@ class PainlessParser extends Parser {
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3O\u01c5\4\2\t\2\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3O\u01db\4\2\t\2\4"+
     "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
     "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
     "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+
-    "\4\32\t\32\4\33\t\33\3\2\7\28\n\2\f\2\16\2;\13\2\3\2\7\2>\n\2\f\2\16\2"+
-    "A\13\2\3\2\3\2\3\3\3\3\3\3\3\3\3\3\3\4\3\4\3\4\3\4\3\4\3\4\3\4\7\4Q\n"+
-    "\4\f\4\16\4T\13\4\5\4V\n\4\3\4\3\4\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\5\5"+
-    "b\n\5\3\5\3\5\3\5\3\5\3\5\3\5\5\5j\n\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5"+
-    "\3\5\3\5\3\5\5\5w\n\5\3\5\3\5\5\5{\n\5\3\5\3\5\5\5\177\n\5\3\5\3\5\3\5"+
-    "\5\5\u0084\n\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5"+
-    "\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\6\5\u009d\n\5\r\5\16\5\u009e\3\5"+
-    "\3\5\3\5\3\5\3\5\3\5\3\5\5\5\u00a8\n\5\3\6\3\6\5\6\u00ac\n\6\3\7\3\7\7"+
-    "\7\u00b0\n\7\f\7\16\7\u00b3\13\7\3\7\3\7\3\b\3\b\3\t\3\t\5\t\u00bb\n\t"+
-    "\3\n\3\n\3\13\3\13\3\13\3\13\7\13\u00c3\n\13\f\13\16\13\u00c6\13\13\3"+
-    "\f\3\f\3\f\7\f\u00cb\n\f\f\f\16\f\u00ce\13\f\3\r\3\r\3\r\5\r\u00d3\n\r"+
-    "\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\17\3\17\3\17\3\17\3\17\5\17\u00e1"+
-    "\n\17\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\5\20\u00ec\n\20\3\20"+
+    "\4\32\t\32\4\33\t\33\4\34\t\34\4\35\t\35\4\36\t\36\4\37\t\37\3\2\7\2@"+
+    "\n\2\f\2\16\2C\13\2\3\2\7\2F\n\2\f\2\16\2I\13\2\3\2\3\2\3\3\3\3\3\3\3"+
+    "\3\3\3\3\4\3\4\3\4\3\4\3\4\3\4\3\4\7\4Y\n\4\f\4\16\4\\\13\4\5\4^\n\4\3"+
+    "\4\3\4\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\5\5j\n\5\3\5\3\5\3\5\3\5\3\5\3"+
+    "\5\5\5r\n\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\5\5\177\n\5\3"+
+    "\5\3\5\5\5\u0083\n\5\3\5\3\5\5\5\u0087\n\5\3\5\3\5\3\5\5\5\u008c\n\5\3"+
+    "\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5"+
+    "\3\5\3\5\3\5\3\5\3\5\6\5\u00a5\n\5\r\5\16\5\u00a6\3\5\3\5\3\5\3\5\3\5"+
+    "\3\5\3\5\5\5\u00b0\n\5\3\6\3\6\5\6\u00b4\n\6\3\7\3\7\7\7\u00b8\n\7\f\7"+
+    "\16\7\u00bb\13\7\3\7\3\7\3\b\3\b\3\t\3\t\5\t\u00c3\n\t\3\n\3\n\3\13\3"+
+    "\13\3\13\3\13\7\13\u00cb\n\13\f\13\16\13\u00ce\13\13\3\f\3\f\3\f\7\f\u00d3"+
+    "\n\f\f\f\16\f\u00d6\13\f\3\r\3\r\3\r\5\r\u00db\n\r\3\16\3\16\3\16\3\16"+
+    "\3\16\3\16\3\16\3\17\3\17\3\17\3\17\3\17\5\17\u00e9\n\17\3\20\3\20\3\20"+
+    "\3\20\3\20\3\20\3\20\3\20\3\20\5\20\u00f4\n\20\3\20\3\20\3\20\3\20\3\20"+
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
-    "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
-    "\7\20\u0127\n\20\f\20\16\20\u012a\13\20\3\21\3\21\3\21\3\21\3\21\3\21"+
-    "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
-    "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\5\21\u0149\n\21\3\22\3\22"+
-    "\7\22\u014d\n\22\f\22\16\22\u0150\13\22\3\22\3\22\3\22\7\22\u0155\n\22"+
-    "\f\22\16\22\u0158\13\22\3\22\3\22\3\22\3\22\3\22\3\22\6\22\u0160\n\22"+
-    "\r\22\16\22\u0161\3\22\3\22\7\22\u0166\n\22\f\22\16\22\u0169\13\22\5\22"+
-    "\u016b\n\22\5\22\u016d\n\22\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3"+
-    "\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\5\23\u0181\n\23\3\24"+
-    "\3\24\3\24\3\24\5\24\u0187\n\24\3\25\3\25\3\25\3\25\3\25\5\25\u018e\n"+
-    "\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\27\7\27\u0198\n\27\f\27\16\27"+
-    "\u019b\13\27\5\27\u019d\n\27\3\27\3\27\3\30\3\30\3\30\5\30\u01a4\n\30"+
-    "\3\31\3\31\3\31\3\31\3\31\7\31\u01ab\n\31\f\31\16\31\u01ae\13\31\5\31"+
-    "\u01b0\n\31\3\31\5\31\u01b3\n\31\3\31\3\31\3\31\3\32\5\32\u01b9\n\32\3"+
-    "\32\3\32\3\33\3\33\3\33\3\33\3\33\3\33\5\33\u01c3\n\33\3\33\2\3\36\34"+
-    "\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,.\60\62\64\2\16\3\28C\3"+
-    "\2\35\37\3\2 !\3\2\"$\3\2%(\3\2),\3\2\66\67\3\2DG\4\2\33\34 !\3\2NO\4"+
-    "\2\26\26MM\4\2\32\32MM\u01f3\29\3\2\2\2\4D\3\2\2\2\6I\3\2\2\2\b\u00a7"+
-    "\3\2\2\2\n\u00ab\3\2\2\2\f\u00ad\3\2\2\2\16\u00b6\3\2\2\2\20\u00ba\3\2"+
-    "\2\2\22\u00bc\3\2\2\2\24\u00be\3\2\2\2\26\u00c7\3\2\2\2\30\u00cf\3\2\2"+
-    "\2\32\u00d4\3\2\2\2\34\u00e0\3\2\2\2\36\u00eb\3\2\2\2 \u0148\3\2\2\2\""+
-    "\u016c\3\2\2\2$\u0180\3\2\2\2&\u0186\3\2\2\2(\u018d\3\2\2\2*\u018f\3\2"+
-    "\2\2,\u0193\3\2\2\2.\u01a3\3\2\2\2\60\u01b2\3\2\2\2\62\u01b8\3\2\2\2\64"+
-    "\u01c2\3\2\2\2\668\5\4\3\2\67\66\3\2\2\28;\3\2\2\29\67\3\2\2\29:\3\2\2"+
-    "\2:?\3\2\2\2;9\3\2\2\2<>\5\b\5\2=<\3\2\2\2>A\3\2\2\2?=\3\2\2\2?@\3\2\2"+
-    "\2@B\3\2\2\2A?\3\2\2\2BC\7\2\2\3C\3\3\2\2\2DE\5\26\f\2EF\7M\2\2FG\5\6"+
-    "\4\2GH\5\f\7\2H\5\3\2\2\2IU\7\t\2\2JK\5\26\f\2KR\7M\2\2LM\7\f\2\2MN\5"+
-    "\26\f\2NO\7M\2\2OQ\3\2\2\2PL\3\2\2\2QT\3\2\2\2RP\3\2\2\2RS\3\2\2\2SV\3"+
-    "\2\2\2TR\3\2\2\2UJ\3\2\2\2UV\3\2\2\2VW\3\2\2\2WX\7\n\2\2X\7\3\2\2\2YZ"+
-    "\7\16\2\2Z[\7\t\2\2[\\\5\36\20\2\\]\7\n\2\2]a\5\n\6\2^_\7\17\2\2_b\5\n"+
-    "\6\2`b\6\5\2\2a^\3\2\2\2a`\3\2\2\2b\u00a8\3\2\2\2cd\7\20\2\2de\7\t\2\2"+
-    "ef\5\36\20\2fi\7\n\2\2gj\5\n\6\2hj\5\16\b\2ig\3\2\2\2ih\3\2\2\2j\u00a8"+
-    "\3\2\2\2kl\7\21\2\2lm\5\f\7\2mn\7\20\2\2no\7\t\2\2op\5\36\20\2pq\7\n\2"+
-    "\2qr\5\34\17\2r\u00a8\3\2\2\2st\7\22\2\2tv\7\t\2\2uw\5\20\t\2vu\3\2\2"+
-    "\2vw\3\2\2\2wx\3\2\2\2xz\7\r\2\2y{\5\36\20\2zy\3\2\2\2z{\3\2\2\2{|\3\2"+
-    "\2\2|~\7\r\2\2}\177\5\22\n\2~}\3\2\2\2~\177\3\2\2\2\177\u0080\3\2\2\2"+
-    "\u0080\u0083\7\n\2\2\u0081\u0084\5\n\6\2\u0082\u0084\5\16\b\2\u0083\u0081"+
-    "\3\2\2\2\u0083\u0082\3\2\2\2\u0084\u00a8\3\2\2\2\u0085\u0086\7\22\2\2"+
-    "\u0086\u0087\7\t\2\2\u0087\u0088\5\26\f\2\u0088\u0089\7M\2\2\u0089\u008a"+
-    "\7\63\2\2\u008a\u008b\5\36\20\2\u008b\u008c\7\n\2\2\u008c\u008d\5\n\6"+
-    "\2\u008d\u00a8\3\2\2\2\u008e\u008f\5\24\13\2\u008f\u0090\5\34\17\2\u0090"+
-    "\u00a8\3\2\2\2\u0091\u0092\7\23\2\2\u0092\u00a8\5\34\17\2\u0093\u0094"+
-    "\7\24\2\2\u0094\u00a8\5\34\17\2\u0095\u0096\7\25\2\2\u0096\u0097\5\36"+
-    "\20\2\u0097\u0098\5\34\17\2\u0098\u00a8\3\2\2\2\u0099\u009a\7\27\2\2\u009a"+
-    "\u009c\5\f\7\2\u009b\u009d\5\32\16\2\u009c\u009b\3\2\2\2\u009d\u009e\3"+
-    "\2\2\2\u009e\u009c\3\2\2\2\u009e\u009f\3\2\2\2\u009f\u00a8\3\2\2\2\u00a0"+
-    "\u00a1\7\31\2\2\u00a1\u00a2\5\36\20\2\u00a2\u00a3\5\34\17\2\u00a3\u00a8"+
-    "\3\2\2\2\u00a4\u00a5\5\36\20\2\u00a5\u00a6\5\34\17\2\u00a6\u00a8\3\2\2"+
-    "\2\u00a7Y\3\2\2\2\u00a7c\3\2\2\2\u00a7k\3\2\2\2\u00a7s\3\2\2\2\u00a7\u0085"+
-    "\3\2\2\2\u00a7\u008e\3\2\2\2\u00a7\u0091\3\2\2\2\u00a7\u0093\3\2\2\2\u00a7"+
-    "\u0095\3\2\2\2\u00a7\u0099\3\2\2\2\u00a7\u00a0\3\2\2\2\u00a7\u00a4\3\2"+
-    "\2\2\u00a8\t\3\2\2\2\u00a9\u00ac\5\f\7\2\u00aa\u00ac\5\b\5\2\u00ab\u00a9"+
-    "\3\2\2\2\u00ab\u00aa\3\2\2\2\u00ac\13\3\2\2\2\u00ad\u00b1\7\5\2\2\u00ae"+
-    "\u00b0\5\b\5\2\u00af\u00ae\3\2\2\2\u00b0\u00b3\3\2\2\2\u00b1\u00af\3\2"+
-    "\2\2\u00b1\u00b2\3\2\2\2\u00b2\u00b4\3\2\2\2\u00b3\u00b1\3\2\2\2\u00b4"+
-    "\u00b5\7\6\2\2\u00b5\r\3\2\2\2\u00b6\u00b7\7\r\2\2\u00b7\17\3\2\2\2\u00b8"+
-    "\u00bb\5\24\13\2\u00b9\u00bb\5\36\20\2\u00ba\u00b8\3\2\2\2\u00ba\u00b9"+
-    "\3\2\2\2\u00bb\21\3\2\2\2\u00bc\u00bd\5\36\20\2\u00bd\23\3\2\2\2\u00be"+
-    "\u00bf\5\26\f\2\u00bf\u00c4\5\30\r\2\u00c0\u00c1\7\f\2\2\u00c1\u00c3\5"+
-    "\30\r\2\u00c2\u00c0\3\2\2\2\u00c3\u00c6\3\2\2\2\u00c4\u00c2\3\2\2\2\u00c4"+
-    "\u00c5\3\2\2\2\u00c5\25\3\2\2\2\u00c6\u00c4\3\2\2\2\u00c7\u00cc\7L\2\2"+
-    "\u00c8\u00c9\7\7\2\2\u00c9\u00cb\7\b\2\2\u00ca\u00c8\3\2\2\2\u00cb\u00ce"+
-    "\3\2\2\2\u00cc\u00ca\3\2\2\2\u00cc\u00cd\3\2\2\2\u00cd\27\3\2\2\2\u00ce"+
-    "\u00cc\3\2\2\2\u00cf\u00d2\7M\2\2\u00d0\u00d1\78\2\2\u00d1\u00d3\5\36"+
-    "\20\2\u00d2\u00d0\3\2\2\2\u00d2\u00d3\3\2\2\2\u00d3\31\3\2\2\2\u00d4\u00d5"+
-    "\7\30\2\2\u00d5\u00d6\7\t\2\2\u00d6\u00d7\7L\2\2\u00d7\u00d8\7M\2\2\u00d8"+
-    "\u00d9\7\n\2\2\u00d9\u00da\5\f\7\2\u00da\33\3\2\2\2\u00db\u00e1\7\r\2"+
-    "\2\u00dc\u00e1\7\2\2\3\u00dd\u00de\b\17\1\2\u00de\u00df\7\6\2\2\u00df"+
-    "\u00e1\b\17\1\2\u00e0\u00db\3\2\2\2\u00e0\u00dc\3\2\2\2\u00e0\u00dd\3"+
-    "\2\2\2\u00e1\35\3\2\2\2\u00e2\u00e3\b\20\1\2\u00e3\u00e4\5\"\22\2\u00e4"+
-    "\u00e5\t\2\2\2\u00e5\u00e6\5\36\20\3\u00e6\u00e7\b\20\1\2\u00e7\u00ec"+
-    "\3\2\2\2\u00e8\u00e9\5 \21\2\u00e9\u00ea\b\20\1\2\u00ea\u00ec\3\2\2\2"+
-    "\u00eb\u00e2\3\2\2\2\u00eb\u00e8\3\2\2\2\u00ec\u0128\3\2\2\2\u00ed\u00ee"+
-    "\f\16\2\2\u00ee\u00ef\t\3\2\2\u00ef\u00f0\5\36\20\17\u00f0\u00f1\b\20"+
-    "\1\2\u00f1\u0127\3\2\2\2\u00f2\u00f3\f\r\2\2\u00f3\u00f4\t\4\2\2\u00f4"+
-    "\u00f5\5\36\20\16\u00f5\u00f6\b\20\1\2\u00f6\u0127\3\2\2\2\u00f7\u00f8"+
-    "\f\f\2\2\u00f8\u00f9\t\5\2\2\u00f9\u00fa\5\36\20\r\u00fa\u00fb\b\20\1"+
-    "\2\u00fb\u0127\3\2\2\2\u00fc\u00fd\f\13\2\2\u00fd\u00fe\t\6\2\2\u00fe"+
-    "\u00ff\5\36\20\f\u00ff\u0100\b\20\1\2\u0100\u0127\3\2\2\2\u0101\u0102"+
-    "\f\n\2\2\u0102\u0103\t\7\2\2\u0103\u0104\5\36\20\13\u0104\u0105\b\20\1"+
-    "\2\u0105\u0127\3\2\2\2\u0106\u0107\f\t\2\2\u0107\u0108\7-\2\2\u0108\u0109"+
-    "\5\36\20\n\u0109\u010a\b\20\1\2\u010a\u0127\3\2\2\2\u010b\u010c\f\b\2"+
-    "\2\u010c\u010d\7.\2\2\u010d\u010e\5\36\20\t\u010e\u010f\b\20\1\2\u010f"+
-    "\u0127\3\2\2\2\u0110\u0111\f\7\2\2\u0111\u0112\7/\2\2\u0112\u0113\5\36"+
-    "\20\b\u0113\u0114\b\20\1\2\u0114\u0127\3\2\2\2\u0115\u0116\f\6\2\2\u0116"+
-    "\u0117\7\60\2\2\u0117\u0118\5\36\20\7\u0118\u0119\b\20\1\2\u0119\u0127"+
-    "\3\2\2\2\u011a\u011b\f\5\2\2\u011b\u011c\7\61\2\2\u011c\u011d\5\36\20"+
-    "\6\u011d\u011e\b\20\1\2\u011e\u0127\3\2\2\2\u011f\u0120\f\4\2\2\u0120"+
-    "\u0121\7\62\2\2\u0121\u0122\5\36\20\2\u0122\u0123\7\63\2\2\u0123\u0124"+
-    "\5\36\20\4\u0124\u0125\b\20\1\2\u0125\u0127\3\2\2\2\u0126\u00ed\3\2\2"+
-    "\2\u0126\u00f2\3\2\2\2\u0126\u00f7\3\2\2\2\u0126\u00fc\3\2\2\2\u0126\u0101"+
-    "\3\2\2\2\u0126\u0106\3\2\2\2\u0126\u010b\3\2\2\2\u0126\u0110\3\2\2\2\u0126"+
-    "\u0115\3\2\2\2\u0126\u011a\3\2\2\2\u0126\u011f\3\2\2\2\u0127\u012a\3\2"+
-    "\2\2\u0128\u0126\3\2\2\2\u0128\u0129\3\2\2\2\u0129\37\3\2\2\2\u012a\u0128"+
-    "\3\2\2\2\u012b\u012c\6\21\16\3\u012c\u012d\t\b\2\2\u012d\u0149\5\"\22"+
-    "\2\u012e\u012f\6\21\17\3\u012f\u0130\5\"\22\2\u0130\u0131\t\b\2\2\u0131"+
-    "\u0149\3\2\2\2\u0132\u0133\6\21\20\3\u0133\u0149\5\"\22\2\u0134\u0135"+
-    "\6\21\21\3\u0135\u0136\t\t\2\2\u0136\u0149\b\21\1\2\u0137\u0138\6\21\22"+
-    "\3\u0138\u0139\7I\2\2\u0139\u0149\b\21\1\2\u013a\u013b\6\21\23\3\u013b"+
-    "\u013c\7J\2\2\u013c\u0149\b\21\1\2\u013d\u013e\6\21\24\3\u013e\u013f\7"+
-    "K\2\2\u013f\u0149\b\21\1\2\u0140\u0141\6\21\25\3\u0141\u0142\t\n\2\2\u0142"+
-    "\u0149\5 \21\2\u0143\u0144\7\t\2\2\u0144\u0145\5\26\f\2\u0145\u0146\7"+
-    "\n\2\2\u0146\u0147\5 \21\2\u0147\u0149\3\2\2\2\u0148\u012b\3\2\2\2\u0148"+
-    "\u012e\3\2\2\2\u0148\u0132\3\2\2\2\u0148\u0134\3\2\2\2\u0148\u0137\3\2"+
-    "\2\2\u0148\u013a\3\2\2\2\u0148\u013d\3\2\2\2\u0148\u0140\3\2\2\2\u0148"+
-    "\u0143\3\2\2\2\u0149!\3\2\2\2\u014a\u014e\5$\23\2\u014b\u014d\5&\24\2"+
-    "\u014c\u014b\3\2\2\2\u014d\u0150\3\2\2\2\u014e\u014c\3\2\2\2\u014e\u014f"+
-    "\3\2\2\2\u014f\u016d\3\2\2\2\u0150\u014e\3\2\2\2\u0151\u0152\5\26\f\2"+
-    "\u0152\u0156\5(\25\2\u0153\u0155\5&\24\2\u0154\u0153\3\2\2\2\u0155\u0158"+
-    "\3\2\2\2\u0156\u0154\3\2\2\2\u0156\u0157\3\2\2\2\u0157\u016d\3\2\2\2\u0158"+
-    "\u0156\3\2\2\2\u0159\u015a\7\26\2\2\u015a\u015f\7L\2\2\u015b\u015c\7\7"+
-    "\2\2\u015c\u015d\5\36\20\2\u015d\u015e\7\b\2\2\u015e\u0160\3\2\2\2\u015f"+
-    "\u015b\3\2\2\2\u0160\u0161\3\2\2\2\u0161\u015f\3\2\2\2\u0161\u0162\3\2"+
-    "\2\2\u0162\u016a\3\2\2\2\u0163\u0167\5(\25\2\u0164\u0166\5&\24\2\u0165"+
-    "\u0164\3\2\2\2\u0166\u0169\3\2\2\2\u0167\u0165\3\2\2\2\u0167\u0168\3\2"+
-    "\2\2\u0168\u016b\3\2\2\2\u0169\u0167\3\2\2\2\u016a\u0163\3\2\2\2\u016a"+
-    "\u016b\3\2\2\2\u016b\u016d\3\2\2\2\u016c\u014a\3\2\2\2\u016c\u0151\3\2"+
-    "\2\2\u016c\u0159\3\2\2\2\u016d#\3\2\2\2\u016e\u016f\6\23\26\3\u016f\u0170"+
-    "\7\t\2\2\u0170\u0171\5\36\20\2\u0171\u0172\7\n\2\2\u0172\u0173\b\23\1"+
-    "\2\u0173\u0181\3\2\2\2\u0174\u0175\6\23\27\3\u0175\u0176\7\t\2\2\u0176"+
-    "\u0177\5 \21\2\u0177\u0178\7\n\2\2\u0178\u0181\3\2\2\2\u0179\u0181\7H"+
-    "\2\2\u017a\u0181\7M\2\2\u017b\u017c\7M\2\2\u017c\u0181\5,\27\2\u017d\u017e"+
-    "\7\26\2\2\u017e\u017f\7L\2\2\u017f\u0181\5,\27\2\u0180\u016e\3\2\2\2\u0180"+
-    "\u0174\3\2\2\2\u0180\u0179\3\2\2\2\u0180\u017a\3\2\2\2\u0180\u017b\3\2"+
-    "\2\2\u0180\u017d\3\2\2\2\u0181%\3\2\2\2\u0182\u0183\6\24\30\3\u0183\u0187"+
-    "\5(\25\2\u0184\u0185\6\24\31\3\u0185\u0187\5*\26\2\u0186\u0182\3\2\2\2"+
-    "\u0186\u0184\3\2\2\2\u0187\'\3\2\2\2\u0188\u0189\7\13\2\2\u0189\u018a"+
-    "\7O\2\2\u018a\u018e\5,\27\2\u018b\u018c\7\13\2\2\u018c\u018e\t\13\2\2"+
-    "\u018d\u0188\3\2\2\2\u018d\u018b\3\2\2\2\u018e)\3\2\2\2\u018f\u0190\7"+
-    "\7\2\2\u0190\u0191\5\36\20\2\u0191\u0192\7\b\2\2\u0192+\3\2\2\2\u0193"+
-    "\u019c\7\t\2\2\u0194\u0199\5.\30\2\u0195\u0196\7\f\2\2\u0196\u0198\5."+
-    "\30\2\u0197\u0195\3\2\2\2\u0198\u019b\3\2\2\2\u0199\u0197\3\2\2\2\u0199"+
-    "\u019a\3\2\2\2\u019a\u019d\3\2\2\2\u019b\u0199\3\2\2\2\u019c\u0194\3\2"+
-    "\2\2\u019c\u019d\3\2\2\2\u019d\u019e\3\2\2\2\u019e\u019f\7\n\2\2\u019f"+
-    "-\3\2\2\2\u01a0\u01a4\5\36\20\2\u01a1\u01a4\5\60\31\2\u01a2\u01a4\5\64"+
-    "\33\2\u01a3\u01a0\3\2\2\2\u01a3\u01a1\3\2\2\2\u01a3\u01a2\3\2\2\2\u01a4"+
-    "/\3\2\2\2\u01a5\u01b3\5\62\32\2\u01a6\u01af\7\t\2\2\u01a7\u01ac\5\62\32"+
-    "\2\u01a8\u01a9\7\f\2\2\u01a9\u01ab\5\62\32\2\u01aa\u01a8\3\2\2\2\u01ab"+
-    "\u01ae\3\2\2\2\u01ac\u01aa\3\2\2\2\u01ac\u01ad\3\2\2\2\u01ad\u01b0\3\2"+
-    "\2\2\u01ae\u01ac\3\2\2\2\u01af\u01a7\3\2\2\2\u01af\u01b0\3\2\2\2\u01b0"+
-    "\u01b1\3\2\2\2\u01b1\u01b3\7\n\2\2\u01b2\u01a5\3\2\2\2\u01b2\u01a6\3\2"+
-    "\2\2\u01b3\u01b4\3\2\2\2\u01b4\u01b5\7\65\2\2\u01b5\u01b6\5\f\7\2\u01b6"+
-    "\61\3\2\2\2\u01b7\u01b9\5\26\f\2\u01b8\u01b7\3\2\2\2\u01b8\u01b9\3\2\2"+
-    "\2\u01b9\u01ba\3\2\2\2\u01ba\u01bb\7M\2\2\u01bb\63\3\2\2\2\u01bc\u01bd"+
-    "\7L\2\2\u01bd\u01be\7\64\2\2\u01be\u01c3\t\f\2\2\u01bf\u01c0\t\r\2\2\u01c0"+
-    "\u01c1\7\64\2\2\u01c1\u01c3\7M\2\2\u01c2\u01bc\3\2\2\2\u01c2\u01bf\3\2"+
-    "\2\2\u01c3\65\3\2\2\2*9?RUaivz~\u0083\u009e\u00a7\u00ab\u00b1\u00ba\u00c4"+
-    "\u00cc\u00d2\u00e0\u00eb\u0126\u0128\u0148\u014e\u0156\u0161\u0167\u016a"+
-    "\u016c\u0180\u0186\u018d\u0199\u019c\u01a3\u01ac\u01af\u01b2\u01b8\u01c2";
+    "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\7\20\u012f\n\20\f\20"+
+    "\16\20\u0132\13\20\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3"+
+    "\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3"+
+    "\21\3\21\3\21\3\21\3\21\5\21\u0151\n\21\3\22\3\22\7\22\u0155\n\22\f\22"+
+    "\16\22\u0158\13\22\3\22\3\22\3\22\7\22\u015d\n\22\f\22\16\22\u0160\13"+
+    "\22\3\22\3\22\3\22\3\22\3\22\3\22\6\22\u0168\n\22\r\22\16\22\u0169\3\22"+
+    "\3\22\7\22\u016e\n\22\f\22\16\22\u0171\13\22\5\22\u0173\n\22\5\22\u0175"+
+    "\n\22\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23"+
+    "\3\23\3\23\3\23\3\23\3\23\5\23\u0189\n\23\3\24\3\24\3\24\3\24\5\24\u018f"+
+    "\n\24\3\25\3\25\3\25\3\25\3\25\5\25\u0196\n\25\3\26\3\26\3\26\3\26\3\27"+
+    "\3\27\3\27\3\27\7\27\u01a0\n\27\f\27\16\27\u01a3\13\27\5\27\u01a5\n\27"+
+    "\3\27\3\27\3\30\3\30\3\30\5\30\u01ac\n\30\3\31\3\31\3\31\3\31\3\31\7\31"+
+    "\u01b3\n\31\f\31\16\31\u01b6\13\31\5\31\u01b8\n\31\3\31\5\31\u01bb\n\31"+
+    "\3\31\3\31\3\31\3\32\5\32\u01c1\n\32\3\32\3\32\3\33\3\33\3\33\3\33\5\33"+
+    "\u01c9\n\33\3\34\3\34\3\34\3\34\3\35\3\35\3\35\3\35\3\36\3\36\3\36\3\36"+
+    "\3\37\3\37\3\37\3\37\3\37\2\3\36 \2\4\6\b\n\f\16\20\22\24\26\30\32\34"+
+    "\36 \"$&(*,.\60\62\64\668:<\2\f\3\28C\3\2\35\37\3\2 !\3\2\"$\3\2%(\3\2"+
+    "),\3\2\66\67\3\2DG\4\2\33\34 !\3\2NO\u0207\2A\3\2\2\2\4L\3\2\2\2\6Q\3"+
+    "\2\2\2\b\u00af\3\2\2\2\n\u00b3\3\2\2\2\f\u00b5\3\2\2\2\16\u00be\3\2\2"+
+    "\2\20\u00c2\3\2\2\2\22\u00c4\3\2\2\2\24\u00c6\3\2\2\2\26\u00cf\3\2\2\2"+
+    "\30\u00d7\3\2\2\2\32\u00dc\3\2\2\2\34\u00e8\3\2\2\2\36\u00f3\3\2\2\2 "+
+    "\u0150\3\2\2\2\"\u0174\3\2\2\2$\u0188\3\2\2\2&\u018e\3\2\2\2(\u0195\3"+
+    "\2\2\2*\u0197\3\2\2\2,\u019b\3\2\2\2.\u01ab\3\2\2\2\60\u01ba\3\2\2\2\62"+
+    "\u01c0\3\2\2\2\64\u01c8\3\2\2\2\66\u01ca\3\2\2\28\u01ce\3\2\2\2:\u01d2"+
+    "\3\2\2\2<\u01d6\3\2\2\2>@\5\4\3\2?>\3\2\2\2@C\3\2\2\2A?\3\2\2\2AB\3\2"+
+    "\2\2BG\3\2\2\2CA\3\2\2\2DF\5\b\5\2ED\3\2\2\2FI\3\2\2\2GE\3\2\2\2GH\3\2"+
+    "\2\2HJ\3\2\2\2IG\3\2\2\2JK\7\2\2\3K\3\3\2\2\2LM\5\26\f\2MN\7M\2\2NO\5"+
+    "\6\4\2OP\5\f\7\2P\5\3\2\2\2Q]\7\t\2\2RS\5\26\f\2SZ\7M\2\2TU\7\f\2\2UV"+
+    "\5\26\f\2VW\7M\2\2WY\3\2\2\2XT\3\2\2\2Y\\\3\2\2\2ZX\3\2\2\2Z[\3\2\2\2"+
+    "[^\3\2\2\2\\Z\3\2\2\2]R\3\2\2\2]^\3\2\2\2^_\3\2\2\2_`\7\n\2\2`\7\3\2\2"+
+    "\2ab\7\16\2\2bc\7\t\2\2cd\5\36\20\2de\7\n\2\2ei\5\n\6\2fg\7\17\2\2gj\5"+
+    "\n\6\2hj\6\5\2\2if\3\2\2\2ih\3\2\2\2j\u00b0\3\2\2\2kl\7\20\2\2lm\7\t\2"+
+    "\2mn\5\36\20\2nq\7\n\2\2or\5\n\6\2pr\5\16\b\2qo\3\2\2\2qp\3\2\2\2r\u00b0"+
+    "\3\2\2\2st\7\21\2\2tu\5\f\7\2uv\7\20\2\2vw\7\t\2\2wx\5\36\20\2xy\7\n\2"+
+    "\2yz\5\34\17\2z\u00b0\3\2\2\2{|\7\22\2\2|~\7\t\2\2}\177\5\20\t\2~}\3\2"+
+    "\2\2~\177\3\2\2\2\177\u0080\3\2\2\2\u0080\u0082\7\r\2\2\u0081\u0083\5"+
+    "\36\20\2\u0082\u0081\3\2\2\2\u0082\u0083\3\2\2\2\u0083\u0084\3\2\2\2\u0084"+
+    "\u0086\7\r\2\2\u0085\u0087\5\22\n\2\u0086\u0085\3\2\2\2\u0086\u0087\3"+
+    "\2\2\2\u0087\u0088\3\2\2\2\u0088\u008b\7\n\2\2\u0089\u008c\5\n\6\2\u008a"+
+    "\u008c\5\16\b\2\u008b\u0089\3\2\2\2\u008b\u008a\3\2\2\2\u008c\u00b0\3"+
+    "\2\2\2\u008d\u008e\7\22\2\2\u008e\u008f\7\t\2\2\u008f\u0090\5\26\f\2\u0090"+
+    "\u0091\7M\2\2\u0091\u0092\7\63\2\2\u0092\u0093\5\36\20\2\u0093\u0094\7"+
+    "\n\2\2\u0094\u0095\5\n\6\2\u0095\u00b0\3\2\2\2\u0096\u0097\5\24\13\2\u0097"+
+    "\u0098\5\34\17\2\u0098\u00b0\3\2\2\2\u0099\u009a\7\23\2\2\u009a\u00b0"+
+    "\5\34\17\2\u009b\u009c\7\24\2\2\u009c\u00b0\5\34\17\2\u009d\u009e\7\25"+
+    "\2\2\u009e\u009f\5\36\20\2\u009f\u00a0\5\34\17\2\u00a0\u00b0\3\2\2\2\u00a1"+
+    "\u00a2\7\27\2\2\u00a2\u00a4\5\f\7\2\u00a3\u00a5\5\32\16\2\u00a4\u00a3"+
+    "\3\2\2\2\u00a5\u00a6\3\2\2\2\u00a6\u00a4\3\2\2\2\u00a6\u00a7\3\2\2\2\u00a7"+
+    "\u00b0\3\2\2\2\u00a8\u00a9\7\31\2\2\u00a9\u00aa\5\36\20\2\u00aa\u00ab"+
+    "\5\34\17\2\u00ab\u00b0\3\2\2\2\u00ac\u00ad\5\36\20\2\u00ad\u00ae\5\34"+
+    "\17\2\u00ae\u00b0\3\2\2\2\u00afa\3\2\2\2\u00afk\3\2\2\2\u00afs\3\2\2\2"+
+    "\u00af{\3\2\2\2\u00af\u008d\3\2\2\2\u00af\u0096\3\2\2\2\u00af\u0099\3"+
+    "\2\2\2\u00af\u009b\3\2\2\2\u00af\u009d\3\2\2\2\u00af\u00a1\3\2\2\2\u00af"+
+    "\u00a8\3\2\2\2\u00af\u00ac\3\2\2\2\u00b0\t\3\2\2\2\u00b1\u00b4\5\f\7\2"+
+    "\u00b2\u00b4\5\b\5\2\u00b3\u00b1\3\2\2\2\u00b3\u00b2\3\2\2\2\u00b4\13"+
+    "\3\2\2\2\u00b5\u00b9\7\5\2\2\u00b6\u00b8\5\b\5\2\u00b7\u00b6\3\2\2\2\u00b8"+
+    "\u00bb\3\2\2\2\u00b9\u00b7\3\2\2\2\u00b9\u00ba\3\2\2\2\u00ba\u00bc\3\2"+
+    "\2\2\u00bb\u00b9\3\2\2\2\u00bc\u00bd\7\6\2\2\u00bd\r\3\2\2\2\u00be\u00bf"+
+    "\7\r\2\2\u00bf\17\3\2\2\2\u00c0\u00c3\5\24\13\2\u00c1\u00c3\5\36\20\2"+
+    "\u00c2\u00c0\3\2\2\2\u00c2\u00c1\3\2\2\2\u00c3\21\3\2\2\2\u00c4\u00c5"+
+    "\5\36\20\2\u00c5\23\3\2\2\2\u00c6\u00c7\5\26\f\2\u00c7\u00cc\5\30\r\2"+
+    "\u00c8\u00c9\7\f\2\2\u00c9\u00cb\5\30\r\2\u00ca\u00c8\3\2\2\2\u00cb\u00ce"+
+    "\3\2\2\2\u00cc\u00ca\3\2\2\2\u00cc\u00cd\3\2\2\2\u00cd\25\3\2\2\2\u00ce"+
+    "\u00cc\3\2\2\2\u00cf\u00d4\7L\2\2\u00d0\u00d1\7\7\2\2\u00d1\u00d3\7\b"+
+    "\2\2\u00d2\u00d0\3\2\2\2\u00d3\u00d6\3\2\2\2\u00d4\u00d2\3\2\2\2\u00d4"+
+    "\u00d5\3\2\2\2\u00d5\27\3\2\2\2\u00d6\u00d4\3\2\2\2\u00d7\u00da\7M\2\2"+
+    "\u00d8\u00d9\78\2\2\u00d9\u00db\5\36\20\2\u00da\u00d8\3\2\2\2\u00da\u00db"+
+    "\3\2\2\2\u00db\31\3\2\2\2\u00dc\u00dd\7\30\2\2\u00dd\u00de\7\t\2\2\u00de"+
+    "\u00df\7L\2\2\u00df\u00e0\7M\2\2\u00e0\u00e1\7\n\2\2\u00e1\u00e2\5\f\7"+
+    "\2\u00e2\33\3\2\2\2\u00e3\u00e9\7\r\2\2\u00e4\u00e9\7\2\2\3\u00e5\u00e6"+
+    "\b\17\1\2\u00e6\u00e7\7\6\2\2\u00e7\u00e9\b\17\1\2\u00e8\u00e3\3\2\2\2"+
+    "\u00e8\u00e4\3\2\2\2\u00e8\u00e5\3\2\2\2\u00e9\35\3\2\2\2\u00ea\u00eb"+
+    "\b\20\1\2\u00eb\u00ec\5\"\22\2\u00ec\u00ed\t\2\2\2\u00ed\u00ee\5\36\20"+
+    "\3\u00ee\u00ef\b\20\1\2\u00ef\u00f4\3\2\2\2\u00f0\u00f1\5 \21\2\u00f1"+
+    "\u00f2\b\20\1\2\u00f2\u00f4\3\2\2\2\u00f3\u00ea\3\2\2\2\u00f3\u00f0\3"+
+    "\2\2\2\u00f4\u0130\3\2\2\2\u00f5\u00f6\f\16\2\2\u00f6\u00f7\t\3\2\2\u00f7"+
+    "\u00f8\5\36\20\17\u00f8\u00f9\b\20\1\2\u00f9\u012f\3\2\2\2\u00fa\u00fb"+
+    "\f\r\2\2\u00fb\u00fc\t\4\2\2\u00fc\u00fd\5\36\20\16\u00fd\u00fe\b\20\1"+
+    "\2\u00fe\u012f\3\2\2\2\u00ff\u0100\f\f\2\2\u0100\u0101\t\5\2\2\u0101\u0102"+
+    "\5\36\20\r\u0102\u0103\b\20\1\2\u0103\u012f\3\2\2\2\u0104\u0105\f\13\2"+
+    "\2\u0105\u0106\t\6\2\2\u0106\u0107\5\36\20\f\u0107\u0108\b\20\1\2\u0108"+
+    "\u012f\3\2\2\2\u0109\u010a\f\n\2\2\u010a\u010b\t\7\2\2\u010b\u010c\5\36"+
+    "\20\13\u010c\u010d\b\20\1\2\u010d\u012f\3\2\2\2\u010e\u010f\f\t\2\2\u010f"+
+    "\u0110\7-\2\2\u0110\u0111\5\36\20\n\u0111\u0112\b\20\1\2\u0112\u012f\3"+
+    "\2\2\2\u0113\u0114\f\b\2\2\u0114\u0115\7.\2\2\u0115\u0116\5\36\20\t\u0116"+
+    "\u0117\b\20\1\2\u0117\u012f\3\2\2\2\u0118\u0119\f\7\2\2\u0119\u011a\7"+
+    "/\2\2\u011a\u011b\5\36\20\b\u011b\u011c\b\20\1\2\u011c\u012f\3\2\2\2\u011d"+
+    "\u011e\f\6\2\2\u011e\u011f\7\60\2\2\u011f\u0120\5\36\20\7\u0120\u0121"+
+    "\b\20\1\2\u0121\u012f\3\2\2\2\u0122\u0123\f\5\2\2\u0123\u0124\7\61\2\2"+
+    "\u0124\u0125\5\36\20\6\u0125\u0126\b\20\1\2\u0126\u012f\3\2\2\2\u0127"+
+    "\u0128\f\4\2\2\u0128\u0129\7\62\2\2\u0129\u012a\5\36\20\2\u012a\u012b"+
+    "\7\63\2\2\u012b\u012c\5\36\20\4\u012c\u012d\b\20\1\2\u012d\u012f\3\2\2"+
+    "\2\u012e\u00f5\3\2\2\2\u012e\u00fa\3\2\2\2\u012e\u00ff\3\2\2\2\u012e\u0104"+
+    "\3\2\2\2\u012e\u0109\3\2\2\2\u012e\u010e\3\2\2\2\u012e\u0113\3\2\2\2\u012e"+
+    "\u0118\3\2\2\2\u012e\u011d\3\2\2\2\u012e\u0122\3\2\2\2\u012e\u0127\3\2"+
+    "\2\2\u012f\u0132\3\2\2\2\u0130\u012e\3\2\2\2\u0130\u0131\3\2\2\2\u0131"+
+    "\37\3\2\2\2\u0132\u0130\3\2\2\2\u0133\u0134\6\21\16\3\u0134\u0135\t\b"+
+    "\2\2\u0135\u0151\5\"\22\2\u0136\u0137\6\21\17\3\u0137\u0138\5\"\22\2\u0138"+
+    "\u0139\t\b\2\2\u0139\u0151\3\2\2\2\u013a\u013b\6\21\20\3\u013b\u0151\5"+
+    "\"\22\2\u013c\u013d\6\21\21\3\u013d\u013e\t\t\2\2\u013e\u0151\b\21\1\2"+
+    "\u013f\u0140\6\21\22\3\u0140\u0141\7I\2\2\u0141\u0151\b\21\1\2\u0142\u0143"+
+    "\6\21\23\3\u0143\u0144\7J\2\2\u0144\u0151\b\21\1\2\u0145\u0146\6\21\24"+
+    "\3\u0146\u0147\7K\2\2\u0147\u0151\b\21\1\2\u0148\u0149\6\21\25\3\u0149"+
+    "\u014a\t\n\2\2\u014a\u0151\5 \21\2\u014b\u014c\7\t\2\2\u014c\u014d\5\26"+
+    "\f\2\u014d\u014e\7\n\2\2\u014e\u014f\5 \21\2\u014f\u0151\3\2\2\2\u0150"+
+    "\u0133\3\2\2\2\u0150\u0136\3\2\2\2\u0150\u013a\3\2\2\2\u0150\u013c\3\2"+
+    "\2\2\u0150\u013f\3\2\2\2\u0150\u0142\3\2\2\2\u0150\u0145\3\2\2\2\u0150"+
+    "\u0148\3\2\2\2\u0150\u014b\3\2\2\2\u0151!\3\2\2\2\u0152\u0156\5$\23\2"+
+    "\u0153\u0155\5&\24\2\u0154\u0153\3\2\2\2\u0155\u0158\3\2\2\2\u0156\u0154"+
+    "\3\2\2\2\u0156\u0157\3\2\2\2\u0157\u0175\3\2\2\2\u0158\u0156\3\2\2\2\u0159"+
+    "\u015a\5\26\f\2\u015a\u015e\5(\25\2\u015b\u015d\5&\24\2\u015c\u015b\3"+
+    "\2\2\2\u015d\u0160\3\2\2\2\u015e\u015c\3\2\2\2\u015e\u015f\3\2\2\2\u015f"+
+    "\u0175\3\2\2\2\u0160\u015e\3\2\2\2\u0161\u0162\7\26\2\2\u0162\u0167\7"+
+    "L\2\2\u0163\u0164\7\7\2\2\u0164\u0165\5\36\20\2\u0165\u0166\7\b\2\2\u0166"+
+    "\u0168\3\2\2\2\u0167\u0163\3\2\2\2\u0168\u0169\3\2\2\2\u0169\u0167\3\2"+
+    "\2\2\u0169\u016a\3\2\2\2\u016a\u0172\3\2\2\2\u016b\u016f\5(\25\2\u016c"+
+    "\u016e\5&\24\2\u016d\u016c\3\2\2\2\u016e\u0171\3\2\2\2\u016f\u016d\3\2"+
+    "\2\2\u016f\u0170\3\2\2\2\u0170\u0173\3\2\2\2\u0171\u016f\3\2\2\2\u0172"+
+    "\u016b\3\2\2\2\u0172\u0173\3\2\2\2\u0173\u0175\3\2\2\2\u0174\u0152\3\2"+
+    "\2\2\u0174\u0159\3\2\2\2\u0174\u0161\3\2\2\2\u0175#\3\2\2\2\u0176\u0177"+
+    "\6\23\26\3\u0177\u0178\7\t\2\2\u0178\u0179\5\36\20\2\u0179\u017a\7\n\2"+
+    "\2\u017a\u017b\b\23\1\2\u017b\u0189\3\2\2\2\u017c\u017d\6\23\27\3\u017d"+
+    "\u017e\7\t\2\2\u017e\u017f\5 \21\2\u017f\u0180\7\n\2\2\u0180\u0189\3\2"+
+    "\2\2\u0181\u0189\7H\2\2\u0182\u0189\7M\2\2\u0183\u0184\7M\2\2\u0184\u0189"+
+    "\5,\27\2\u0185\u0186\7\26\2\2\u0186\u0187\7L\2\2\u0187\u0189\5,\27\2\u0188"+
+    "\u0176\3\2\2\2\u0188\u017c\3\2\2\2\u0188\u0181\3\2\2\2\u0188\u0182\3\2"+
+    "\2\2\u0188\u0183\3\2\2\2\u0188\u0185\3\2\2\2\u0189%\3\2\2\2\u018a\u018b"+
+    "\6\24\30\3\u018b\u018f\5(\25\2\u018c\u018d\6\24\31\3\u018d\u018f\5*\26"+
+    "\2\u018e\u018a\3\2\2\2\u018e\u018c\3\2\2\2\u018f\'\3\2\2\2\u0190\u0191"+
+    "\7\13\2\2\u0191\u0192\7O\2\2\u0192\u0196\5,\27\2\u0193\u0194\7\13\2\2"+
+    "\u0194\u0196\t\13\2\2\u0195\u0190\3\2\2\2\u0195\u0193\3\2\2\2\u0196)\3"+
+    "\2\2\2\u0197\u0198\7\7\2\2\u0198\u0199\5\36\20\2\u0199\u019a\7\b\2\2\u019a"+
+    "+\3\2\2\2\u019b\u01a4\7\t\2\2\u019c\u01a1\5.\30\2\u019d\u019e\7\f\2\2"+
+    "\u019e\u01a0\5.\30\2\u019f\u019d\3\2\2\2\u01a0\u01a3\3\2\2\2\u01a1\u019f"+
+    "\3\2\2\2\u01a1\u01a2\3\2\2\2\u01a2\u01a5\3\2\2\2\u01a3\u01a1\3\2\2\2\u01a4"+
+    "\u019c\3\2\2\2\u01a4\u01a5\3\2\2\2\u01a5\u01a6\3\2\2\2\u01a6\u01a7\7\n"+
+    "\2\2\u01a7-\3\2\2\2\u01a8\u01ac\5\36\20\2\u01a9\u01ac\5\60\31\2\u01aa"+
+    "\u01ac\5\64\33\2\u01ab\u01a8\3\2\2\2\u01ab\u01a9\3\2\2\2\u01ab\u01aa\3"+
+    "\2\2\2\u01ac/\3\2\2\2\u01ad\u01bb\5\62\32\2\u01ae\u01b7\7\t\2\2\u01af"+
+    "\u01b4\5\62\32\2\u01b0\u01b1\7\f\2\2\u01b1\u01b3\5\62\32\2\u01b2\u01b0"+
+    "\3\2\2\2\u01b3\u01b6\3\2\2\2\u01b4\u01b2\3\2\2\2\u01b4\u01b5\3\2\2\2\u01b5"+
+    "\u01b8\3\2\2\2\u01b6\u01b4\3\2\2\2\u01b7\u01af\3\2\2\2\u01b7\u01b8\3\2"+
+    "\2\2\u01b8\u01b9\3\2\2\2\u01b9\u01bb\7\n\2\2\u01ba\u01ad\3\2\2\2\u01ba"+
+    "\u01ae\3\2\2\2\u01bb\u01bc\3\2\2\2\u01bc\u01bd\7\65\2\2\u01bd\u01be\5"+
+    "\f\7\2\u01be\61\3\2\2\2\u01bf\u01c1\5\26\f\2\u01c0\u01bf\3\2\2\2\u01c0"+
+    "\u01c1\3\2\2\2\u01c1\u01c2\3\2\2\2\u01c2\u01c3\7M\2\2\u01c3\63\3\2\2\2"+
+    "\u01c4\u01c9\5\66\34\2\u01c5\u01c9\58\35\2\u01c6\u01c9\5:\36\2\u01c7\u01c9"+
+    "\5<\37\2\u01c8\u01c4\3\2\2\2\u01c8\u01c5\3\2\2\2\u01c8\u01c6\3\2\2\2\u01c8"+
+    "\u01c7\3\2\2\2\u01c9\65\3\2\2\2\u01ca\u01cb\7L\2\2\u01cb\u01cc\7\64\2"+
+    "\2\u01cc\u01cd\7M\2\2\u01cd\67\3\2\2\2\u01ce\u01cf\7L\2\2\u01cf\u01d0"+
+    "\7\64\2\2\u01d0\u01d1\7\26\2\2\u01d19\3\2\2\2\u01d2\u01d3\7M\2\2\u01d3"+
+    "\u01d4\7\64\2\2\u01d4\u01d5\7M\2\2\u01d5;\3\2\2\2\u01d6\u01d7\7\32\2\2"+
+    "\u01d7\u01d8\7\64\2\2\u01d8\u01d9\7M\2\2\u01d9=\3\2\2\2*AGZ]iq~\u0082"+
+    "\u0086\u008b\u00a6\u00af\u00b3\u00b9\u00c2\u00cc\u00d4\u00da\u00e8\u00f3"+
+    "\u012e\u0130\u0150\u0156\u015e\u0169\u016f\u0172\u0174\u0188\u018e\u0195"+
+    "\u01a1\u01a4\u01ab\u01b4\u01b7\u01ba\u01c0\u01c8";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
index d98ba899bfa..4608a74847e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
@@ -417,4 +417,32 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitFuncref(PainlessParser.FuncrefContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitClassFuncref(PainlessParser.ClassFuncrefContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitConstructorFuncref(PainlessParser.ConstructorFuncrefContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitCapturingFuncref(PainlessParser.CapturingFuncrefContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitLocalFuncref(PainlessParser.LocalFuncrefContext ctx) { return visitChildren(ctx); }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
index 514e22ccdc3..bd2249617a0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
@@ -397,4 +397,28 @@ interface PainlessParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitFuncref(PainlessParser.FuncrefContext ctx);
+  /**
+   * Visit a parse tree produced by {@link PainlessParser#classFuncref}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitClassFuncref(PainlessParser.ClassFuncrefContext ctx);
+  /**
+   * Visit a parse tree produced by {@link PainlessParser#constructorFuncref}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitConstructorFuncref(PainlessParser.ConstructorFuncrefContext ctx);
+  /**
+   * Visit a parse tree produced by {@link PainlessParser#capturingFuncref}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitCapturingFuncref(PainlessParser.CapturingFuncrefContext ctx);
+  /**
+   * Visit a parse tree produced by {@link PainlessParser#localFuncref}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitLocalFuncref(PainlessParser.LocalFuncrefContext ctx);
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index d3bb65fa06b..3b6e0d47dee 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -45,10 +45,13 @@ import org.elasticsearch.painless.antlr.PainlessParser.BraceaccessContext;
 import org.elasticsearch.painless.antlr.PainlessParser.BreakContext;
 import org.elasticsearch.painless.antlr.PainlessParser.CallinvokeContext;
 import org.elasticsearch.painless.antlr.PainlessParser.CalllocalContext;
+import org.elasticsearch.painless.antlr.PainlessParser.CapturingFuncrefContext;
 import org.elasticsearch.painless.antlr.PainlessParser.CastContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ChainprecContext;
+import org.elasticsearch.painless.antlr.PainlessParser.ClassFuncrefContext;
 import org.elasticsearch.painless.antlr.PainlessParser.CompContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ConditionalContext;
+import org.elasticsearch.painless.antlr.PainlessParser.ConstructorFuncrefContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ContinueContext;
 import org.elasticsearch.painless.antlr.PainlessParser.DeclContext;
 import org.elasticsearch.painless.antlr.PainlessParser.DeclarationContext;
@@ -71,6 +74,7 @@ import org.elasticsearch.painless.antlr.PainlessParser.IfContext;
 import org.elasticsearch.painless.antlr.PainlessParser.InitializerContext;
 import org.elasticsearch.painless.antlr.PainlessParser.LambdaContext;
 import org.elasticsearch.painless.antlr.PainlessParser.LamtypeContext;
+import org.elasticsearch.painless.antlr.PainlessParser.LocalFuncrefContext;
 import org.elasticsearch.painless.antlr.PainlessParser.NewarrayContext;
 import org.elasticsearch.painless.antlr.PainlessParser.NewobjectContext;
 import org.elasticsearch.painless.antlr.PainlessParser.NullContext;
@@ -950,20 +954,36 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
 
     @Override
     public Object visitFuncref(FuncrefContext ctx) {
-        if (ctx.TYPE() != null) {
-            // non-capturing Type::method or Type::new
-            final String methodText;
-            if (ctx.NEW() != null) {
-                methodText = ctx.NEW().getText();
-            } else {
-                methodText = ctx.ID(0).getText();
-            }
-            return new EFunctionRef(location(ctx), ctx.TYPE().getText(), methodText);
-        } else if (ctx.THIS() != null) {
-            return new EFunctionRef(location(ctx), ctx.THIS().getText(), ctx.ID(0).getText());
+        if (ctx.classFuncref() != null) {
+            return visit(ctx.classFuncref());
+        } else if (ctx.constructorFuncref() != null) {
+            return visit(ctx.constructorFuncref());
+        } else if (ctx.capturingFuncref() != null) {
+            return visit(ctx.capturingFuncref());
+        } else if (ctx.localFuncref() != null) {
+            return visit(ctx.localFuncref());
         } else {
-            // capturing object::method
-            return new ECapturingFunctionRef(location(ctx), ctx.ID(0).getText(), ctx.ID(1).getText());
+            throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
         }
     }
+
+    @Override
+    public Object visitClassFuncref(ClassFuncrefContext ctx) {
+        return new EFunctionRef(location(ctx), ctx.TYPE().getText(), ctx.ID().getText());
+    }
+
+    @Override
+    public Object visitConstructorFuncref(ConstructorFuncrefContext ctx) {
+        return new EFunctionRef(location(ctx), ctx.TYPE().getText(), ctx.NEW().getText());
+    }
+
+    @Override
+    public Object visitCapturingFuncref(CapturingFuncrefContext ctx) {
+        return new ECapturingFunctionRef(location(ctx), ctx.ID(0).getText(), ctx.ID(1).getText());
+    }
+
+    @Override
+    public Object visitLocalFuncref(LocalFuncrefContext ctx) {
+        return new EFunctionRef(location(ctx), ctx.THIS().getText(), ctx.ID().getText());
+    }
 }

From faee2323ab538bbae5e2f10513b26ebc5d50c797 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Sun, 12 Jun 2016 12:53:37 -0400
Subject: [PATCH 0802/1311] add array ctor refs

---
 .../src/main/antlr/PainlessParser.g4          |  2 +-
 .../painless/antlr/PainlessParser.java        |  8 ++++---
 .../elasticsearch/painless/antlr/Walker.java  | 23 +++++++++++++++++--
 .../elasticsearch/painless/node/EChain.java   |  6 +++++
 .../painless/node/SFunction.java              | 11 +++++++--
 .../elasticsearch/painless/ArrayTests.java    |  4 ++++
 .../painless/FunctionRefTests.java            | 16 ++++++++++++-
 7 files changed, 61 insertions(+), 9 deletions(-)

diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index 7aec0db02c7..74d400da431 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -209,7 +209,7 @@ classFuncref
 // reference to a constructor, e.g. ArrayList::new
 // currently limited to simple non-array types
 constructorFuncref
-    : TYPE REF NEW
+    : decltype REF NEW
     ;
 
 // reference to an instance method, e.g. object::toString
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
index d8a07970afe..ae080fbb8aa 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
@@ -2969,7 +2969,9 @@ class PainlessParser extends Parser {
   }
 
   public static class ConstructorFuncrefContext extends ParserRuleContext {
-    public TerminalNode TYPE() { return getToken(PainlessParser.TYPE, 0); }
+    public DecltypeContext decltype() {
+      return getRuleContext(DecltypeContext.class,0);
+    }
     public TerminalNode REF() { return getToken(PainlessParser.REF, 0); }
     public TerminalNode NEW() { return getToken(PainlessParser.NEW, 0); }
     public ConstructorFuncrefContext(ParserRuleContext parent, int invokingState) {
@@ -2990,7 +2992,7 @@ class PainlessParser extends Parser {
       enterOuterAlt(_localctx, 1);
       {
       setState(460);
-      match(TYPE);
+      decltype();
       setState(461);
       match(REF);
       setState(462);
@@ -3350,7 +3352,7 @@ class PainlessParser extends Parser {
     "\u01c4\u01c9\5\66\34\2\u01c5\u01c9\58\35\2\u01c6\u01c9\5:\36\2\u01c7\u01c9"+
     "\5<\37\2\u01c8\u01c4\3\2\2\2\u01c8\u01c5\3\2\2\2\u01c8\u01c6\3\2\2\2\u01c8"+
     "\u01c7\3\2\2\2\u01c9\65\3\2\2\2\u01ca\u01cb\7L\2\2\u01cb\u01cc\7\64\2"+
-    "\2\u01cc\u01cd\7M\2\2\u01cd\67\3\2\2\2\u01ce\u01cf\7L\2\2\u01cf\u01d0"+
+    "\2\u01cc\u01cd\7M\2\2\u01cd\67\3\2\2\2\u01ce\u01cf\5\26\f\2\u01cf\u01d0"+
     "\7\64\2\2\u01d0\u01d1\7\26\2\2\u01d19\3\2\2\2\u01d2\u01d3\7M\2\2\u01d3"+
     "\u01d4\7\64\2\2\u01d4\u01d5\7M\2\2\u01d5;\3\2\2\2\u01d6\u01d7\7\32\2\2"+
     "\u01d7\u01d8\7\64\2\2\u01d8\u01d9\7M\2\2\u01d9=\3\2\2\2*AGZ]iq~\u0082"+
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index 3b6e0d47dee..6f00a337386 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -148,6 +148,7 @@ import org.elasticsearch.painless.node.SWhile;
 
 import java.util.ArrayDeque;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Deque;
 import java.util.List;
 
@@ -166,6 +167,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
     private final String sourceText;
 
     private final Deque<Reserved> reserved = new ArrayDeque<>();
+    private final List<SFunction> synthetic = new ArrayList<>();
 
     private Walker(String sourceName, String sourceText, CompilerSettings settings) {
         this.settings = settings;
@@ -229,6 +231,8 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         for (StatementContext statement : ctx.statement()) {
             statements.add((AStatement)visit(statement));
         }
+        
+        functions.addAll(synthetic);
 
         return new SSource(sourceName, sourceText, (ExecuteReserved)reserved.pop(), location(ctx), functions, statements);
     }
@@ -255,7 +259,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
             statements.add((AStatement)visit(statement));
         }
 
-        return new SFunction((FunctionReserved)reserved.pop(), location(ctx), rtnType, name, paramTypes, paramNames, statements);
+        return new SFunction((FunctionReserved)reserved.pop(), location(ctx), rtnType, name, paramTypes, paramNames, statements, false);
     }
 
     @Override
@@ -974,7 +978,22 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
 
     @Override
     public Object visitConstructorFuncref(ConstructorFuncrefContext ctx) {
-        return new EFunctionRef(location(ctx), ctx.TYPE().getText(), ctx.NEW().getText());
+        if (!ctx.decltype().LBRACE().isEmpty()) {
+            // array constructors are special: we need to make a synthetic method
+            // taking integer as argument and returning a new instance, and return a ref to that.
+            Location location = location(ctx);
+            String arrayType = ctx.decltype().getText();
+            SReturn code = new SReturn(location, 
+                           new EChain(location,
+                           new LNewArray(location, arrayType, Arrays.asList(
+                           new EChain(location, 
+                           new LVariable(location, "size"))))));
+            String name = "lambda$" + synthetic.size();
+            synthetic.add(new SFunction(new FunctionReserved(), location, arrayType, name, 
+                          Arrays.asList("int"), Arrays.asList("size"), Arrays.asList(code), true));
+            return new EFunctionRef(location(ctx), "this", name);
+        }
+        return new EFunctionRef(location(ctx), ctx.decltype().getText(), ctx.NEW().getText());
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index e21a1dd3134..4c5a44d1fac 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -29,6 +29,7 @@ import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
+import java.util.Arrays;
 import java.util.List;
 
 /**
@@ -46,6 +47,11 @@ public final class EChain extends AExpression {
     Type promote = null;
     Cast there = null;
     Cast back = null;
+    
+    /** Creates a new simple EChain */
+    public EChain(Location location, ALink link) {
+        this(location, Arrays.asList(link), false, false, null, null);
+    }
 
     public EChain(Location location, List<ALink> links,
                   boolean pre, boolean post, Operation operation, AExpression expression) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index a422e3b9d3d..57d80a51ea9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -50,6 +50,7 @@ public class SFunction extends AStatement {
     final List<String> paramTypeStrs;
     final List<String> paramNameStrs;
     final List<AStatement> statements;
+    final boolean synthetic;
 
     Type rtnType = null;
     List<Parameter> parameters = new ArrayList<>();
@@ -58,7 +59,8 @@ public class SFunction extends AStatement {
     Locals locals = null;
 
     public SFunction(FunctionReserved reserved, Location location,
-                     String rtnType, String name, List<String> paramTypes, List<String> paramNames, List<AStatement> statements) {
+                     String rtnType, String name, List<String> paramTypes, 
+                     List<String> paramNames, List<AStatement> statements, boolean synthetic) {
         super(location);
 
         this.reserved = reserved;
@@ -67,6 +69,7 @@ public class SFunction extends AStatement {
         this.paramTypeStrs = Collections.unmodifiableList(paramTypes);
         this.paramNameStrs = Collections.unmodifiableList(paramNames);
         this.statements = Collections.unmodifiableList(statements);
+        this.synthetic = synthetic;
     }
 
     void generate() {
@@ -138,7 +141,11 @@ public class SFunction extends AStatement {
     
     /** Writes the function to given ClassWriter. */
     void write (ClassWriter writer, BitSet statements) {
-        final MethodWriter function = new MethodWriter(Opcodes.ACC_PRIVATE | Opcodes.ACC_STATIC, method.method, writer, statements);
+        int access = Opcodes.ACC_PRIVATE | Opcodes.ACC_STATIC;
+        if (synthetic) {
+            access |= Opcodes.ACC_SYNTHETIC;
+        }
+        final MethodWriter function = new MethodWriter(access, method.method, writer, statements);
         write(function);
         function.endMethod();
     }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
index 8dabacdd5f9..397a8856231 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
@@ -70,6 +70,10 @@ public class ArrayTests extends ScriptTestCase {
         assertEquals(1, exec("int x; def y = new def[1]; x = y[0] = 1; return x;"));
     }
 
+    public void testArrayVariable() {
+        assertEquals(1, exec("int x = 1; int[] y = new int[x]; return y.length"));
+    }
+
     public void testForLoop() {
         assertEquals(999*1000/2, exec("def a = new int[1000]; for (int x = 0; x < a.length; x++) { a[x] = x; } "+
             "int total = 0; for (int x = 0; x < a.length; x++) { total += a[x]; } return total;"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
index a332ed7dd30..8ed2f3261cf 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
@@ -56,7 +56,21 @@ public class FunctionRefTests extends ScriptTestCase {
                                                   "DoubleSummaryStatistics::combine); " + 
                  "return stats.getSum()"));
     }
-    
+
+    public void testArrayCtorMethodRef() {
+        assertEquals(1.0D, 
+                exec("List l = new ArrayList(); l.add(1.0); l.add(2.0); " + 
+                     "def[] array = l.stream().toArray(Double[]::new);" + 
+                     "return array[0];"));
+    }
+
+    public void testArrayCtorMethodRefDef() {
+        assertEquals(1.0D, 
+                exec("def l = new ArrayList(); l.add(1.0); l.add(2.0); " + 
+                     "def[] array = l.stream().toArray(Double[]::new);" + 
+                     "return array[0];"));
+    }
+
     public void testCapturingMethodReference() {
         assertEquals("5", exec("Integer x = Integer.valueOf(5); return Optional.empty().orElseGet(x::toString);"));
         assertEquals("[]", exec("List l = new ArrayList(); return Optional.empty().orElseGet(l::toString);"));

From 50b6f4c02fe3f995c152c02cdb47fa65111460f4 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Mon, 13 Jun 2016 09:20:39 +0200
Subject: [PATCH 0803/1311] Build: changed forbidden-apis targetCompatibility
 to 1.7 for client and client-sniffer

---
 client-sniffer/build.gradle | 2 ++
 client/build.gradle         | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/client-sniffer/build.gradle b/client-sniffer/build.gradle
index a29f07f1c42..af46a5a452c 100644
--- a/client-sniffer/build.gradle
+++ b/client-sniffer/build.gradle
@@ -51,6 +51,7 @@ compileTestJava.options.compilerArgs << '-target' << '1.7' << '-source' << '1.7'
 forbiddenApisMain {
   //client does not depend on core, so only jdk signatures should be checked
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
+  targetCompatibility = 1.7
 }
 
 forbiddenApisTest {
@@ -59,6 +60,7 @@ forbiddenApisTest {
   bundledSignatures += 'jdk-internal'
   //client does not depend on core, so only jdk signatures should be checked
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
+  targetCompatibility = 1.7
 }
 
 //JarHell is part of es core, which we don't want to pull in
diff --git a/client/build.gradle b/client/build.gradle
index 7a685fabfaf..c9c619f80ea 100644
--- a/client/build.gradle
+++ b/client/build.gradle
@@ -49,6 +49,7 @@ compileTestJava.options.compilerArgs << '-target' << '1.7' << '-source' << '1.7'
 forbiddenApisMain {
   //client does not depend on core, so only jdk signatures should be checked
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
+  targetCompatibility = 1.7
 }
 
 forbiddenApisTest {
@@ -57,6 +58,7 @@ forbiddenApisTest {
   bundledSignatures += 'jdk-internal'
   //client does not depend on core, so only jdk signatures should be checked
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
+  targetCompatibility = 1.7
 }
 
 //JarHell is part of es core, which we don't want to pull in

From 3b96055b23cd4e5e744430a7ef72c53c633c467b Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Fri, 3 Jun 2016 14:47:42 +0200
Subject: [PATCH 0804/1311] msearch: Cap the number of searches the msearch api
 will concurrently execute

By default the number of searches msearch executes is capped by the number of
nodes multiplied with the default size of the search threadpool. This default can be
overwritten by using the newly added `max_concurrent_searches` parameter.

Before the msearch api would concurrently execute all searches concurrently. If many large
msearch requests would be executed this could lead to some searches being rejected
while other searches in the msearch request would succeed.

The goal of this change is to avoid this exhausting of the search TP.

Closes #17926
---
 .../action/search/MultiSearchRequest.java     |  22 +++
 .../search/MultiSearchRequestBuilder.java     |   8 +
 .../search/TransportMultiSearchAction.java    | 116 +++++++++++----
 .../action/search/RestMultiSearchAction.java  |   3 +
 .../elasticsearch/threadpool/ThreadPool.java  |   6 +-
 .../search/MultiSearchRequestTests.java       |  10 +-
 .../TransportMultiSearchActionTests.java      | 137 ++++++++++++++++++
 ...eMultiSearchIT.java => MultiSearchIT.java} |  32 +++-
 docs/reference/search/multi-search.asciidoc   |   4 +
 .../resources/rest-api-spec/api/msearch.json  |   4 +
 10 files changed, 308 insertions(+), 34 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java
 rename core/src/test/java/org/elasticsearch/search/msearch/{SimpleMultiSearchIT.java => MultiSearchIT.java} (70%)

diff --git a/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java b/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
index a3236e9653f..08a1ec5b3de 100644
--- a/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java
@@ -38,6 +38,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError;
  */
 public class MultiSearchRequest extends ActionRequest<MultiSearchRequest> implements CompositeIndicesRequest {
 
+    private int maxConcurrentSearchRequests = 0;
     private List<SearchRequest> requests = new ArrayList<>();
 
     private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed();
@@ -60,6 +61,25 @@ public class MultiSearchRequest extends ActionRequest<MultiSearchRequest> implem
         return this;
     }
 
+    /**
+     * Returns the amount of search requests specified in this multi search requests are allowed to be ran concurrently.
+     */
+    public int maxConcurrentSearchRequests() {
+        return maxConcurrentSearchRequests;
+    }
+
+    /**
+     * Sets how many search requests specified in this multi search requests are allowed to be ran concurrently.
+     */
+    public MultiSearchRequest maxConcurrentSearchRequests(int maxConcurrentSearchRequests) {
+        if (maxConcurrentSearchRequests < 1) {
+            throw new IllegalArgumentException("maxConcurrentSearchRequests must be positive");
+        }
+
+        this.maxConcurrentSearchRequests = maxConcurrentSearchRequests;
+        return this;
+    }
+
     public List<SearchRequest> requests() {
         return this.requests;
     }
@@ -100,6 +120,7 @@ public class MultiSearchRequest extends ActionRequest<MultiSearchRequest> implem
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
+        maxConcurrentSearchRequests = in.readVInt();
         int size = in.readVInt();
         for (int i = 0; i < size; i++) {
             SearchRequest request = new SearchRequest();
@@ -111,6 +132,7 @@ public class MultiSearchRequest extends ActionRequest<MultiSearchRequest> implem
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
+        out.writeVInt(maxConcurrentSearchRequests);
         out.writeVInt(requests.size());
         for (SearchRequest request : requests) {
             request.writeTo(out);
diff --git a/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java
index a0d1e4fb5c5..6cebb73fb4f 100644
--- a/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java
@@ -71,4 +71,12 @@ public class MultiSearchRequestBuilder extends ActionRequestBuilder<MultiSearchR
         request().indicesOptions(indicesOptions);
         return this;
     }
+
+    /**
+     * Sets how many search requests specified in this multi search requests are allowed to be ran concurrently.
+     */
+    public MultiSearchRequestBuilder setMaxConcurrentSearchRequests(int maxConcurrentSearchRequests) {
+        request().maxConcurrentSearchRequests(maxConcurrentSearchRequests);
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java b/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
index c0428cd531f..bd1751856b2 100644
--- a/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
@@ -22,6 +22,7 @@ package org.elasticsearch.action.search;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -29,57 +30,118 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.atomic.AtomicInteger;
 
-/**
- */
 public class TransportMultiSearchAction extends HandledTransportAction<MultiSearchRequest, MultiSearchResponse> {
 
+    private final int availableProcessors;
     private final ClusterService clusterService;
-    private final TransportSearchAction searchAction;
+    private final TransportAction<SearchRequest, SearchResponse> searchAction;
 
     @Inject
     public TransportMultiSearchAction(Settings settings, ThreadPool threadPool, TransportService transportService,
-                                      ClusterService clusterService, TransportSearchAction searchAction,
-                                      ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+                                ClusterService clusterService, TransportSearchAction searchAction,
+                                ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
         super(settings, MultiSearchAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, MultiSearchRequest::new);
         this.clusterService = clusterService;
         this.searchAction = searchAction;
+        this.availableProcessors = EsExecutors.boundedNumberOfProcessors(settings);
+    }
+
+    // For testing only:
+    TransportMultiSearchAction(ThreadPool threadPool, ActionFilters actionFilters, TransportService transportService,
+                               ClusterService clusterService, TransportAction<SearchRequest, SearchResponse> searchAction,
+                               IndexNameExpressionResolver indexNameExpressionResolver, int availableProcessors) {
+        super(Settings.EMPTY, MultiSearchAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, MultiSearchRequest::new);
+        this.clusterService = clusterService;
+        this.searchAction = searchAction;
+        this.availableProcessors = availableProcessors;
     }
 
     @Override
-    protected void doExecute(final MultiSearchRequest request, final ActionListener<MultiSearchResponse> listener) {
+    protected void doExecute(MultiSearchRequest request, ActionListener<MultiSearchResponse> listener) {
         ClusterState clusterState = clusterService.state();
         clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ);
 
-        final AtomicArray<MultiSearchResponse.Item> responses = new AtomicArray<>(request.requests().size());
-        final AtomicInteger counter = new AtomicInteger(responses.length());
-        for (int i = 0; i < responses.length(); i++) {
-            final int index = i;
-            searchAction.execute(request.requests().get(i), new ActionListener<SearchResponse>() {
-                @Override
-                public void onResponse(SearchResponse searchResponse) {
-                    responses.set(index, new MultiSearchResponse.Item(searchResponse, null));
-                    if (counter.decrementAndGet() == 0) {
-                        finishHim();
-                    }
-                }
+        int maxConcurrentSearches = request.maxConcurrentSearchRequests();
+        if (maxConcurrentSearches == 0) {
+            maxConcurrentSearches = defaultMaxConcurrentSearches(availableProcessors, clusterState);
+        }
 
-                @Override
-                public void onFailure(Throwable e) {
-                    responses.set(index, new MultiSearchResponse.Item(null, e));
-                    if (counter.decrementAndGet() == 0) {
-                        finishHim();
-                    }
-                }
+        Queue<SearchRequestSlot> searchRequestSlots = new ConcurrentLinkedQueue<>();
+        for (int i = 0; i < request.requests().size(); i++) {
+            SearchRequest searchRequest = request.requests().get(i);
+            searchRequestSlots.add(new SearchRequestSlot(searchRequest, i));
+        }
 
-                private void finishHim() {
+        int numRequests = request.requests().size();
+        final AtomicArray<MultiSearchResponse.Item> responses = new AtomicArray<>(numRequests);
+        final AtomicInteger responseCounter = new AtomicInteger(numRequests);
+        int numConcurrentSearches = Math.min(numRequests, maxConcurrentSearches);
+        for (int i = 0; i < numConcurrentSearches; i++) {
+            executeSearch(searchRequestSlots, responses, responseCounter, listener);
+        }
+    }
+
+    /*
+     * This is not perfect and makes a big assumption, that all nodes have the same thread pool size / have the number
+     * of processors and that shard of the indices the search requests go to are more or less evenly distributed across
+     * all nodes in the cluster. But I think it is a good enough default for most cases, if not then the default should be
+     * overwritten in the request itself.
+     */
+    static int defaultMaxConcurrentSearches(int availableProcessors, ClusterState state) {
+        int numDateNodes = state.getNodes().getDataNodes().size();
+        // availableProcessors will never be larger than 32, so max defaultMaxConcurrentSearches will never be larger than 49,
+        // but we don't know about about other search requests that are being executed so lets cap at 10 per node
+        int defaultSearchThreadPoolSize = Math.min(ThreadPool.searchThreadPoolSize(availableProcessors), 10);
+        return Math.max(1, numDateNodes * defaultSearchThreadPoolSize);
+    }
+
+    void executeSearch(Queue<SearchRequestSlot> requests, AtomicArray<MultiSearchResponse.Item> responses,
+                       AtomicInteger responseCounter, ActionListener<MultiSearchResponse> listener) {
+        SearchRequestSlot request = requests.poll();
+        if (request == null) {
+            // Ok... so there're no more requests then this is ok, we're then waiting for running requests to complete
+            return;
+        }
+        searchAction.execute(request.request, new ActionListener<SearchResponse>() {
+            @Override
+            public void onResponse(SearchResponse searchResponse) {
+                responses.set(request.responseSlot, new MultiSearchResponse.Item(searchResponse, null));
+                handleResponse();
+            }
+
+            @Override
+            public void onFailure(Throwable e) {
+                responses.set(request.responseSlot, new MultiSearchResponse.Item(null, e));
+                handleResponse();
+            }
+
+            private void handleResponse() {
+                if (responseCounter.decrementAndGet() == 0) {
                     listener.onResponse(new MultiSearchResponse(responses.toArray(new MultiSearchResponse.Item[responses.length()])));
+                } else {
+                    executeSearch(requests, responses, responseCounter, listener);
                 }
-            });
+            }
+        });
+    }
+
+    final static class SearchRequestSlot {
+
+        final SearchRequest request;
+        final int responseSlot;
+
+        SearchRequestSlot(SearchRequest request, int responseSlot) {
+            this.request = request;
+            this.responseSlot = responseSlot;
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
index 8c76525c857..2050360a9d5 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
@@ -91,6 +91,9 @@ public class RestMultiSearchAction extends BaseRestHandler {
     @Override
     public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws Exception {
         MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
+        if (request.hasParam("max_concurrent_searches")) {
+            multiSearchRequest.maxConcurrentSearchRequests(request.paramAsInt("max_concurrent_searches", 0));
+        }
 
         String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         String[] types = Strings.splitStringByCommaToArray(request.param("type"));
diff --git a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
index f8cb882b083..3f6a65c82c5 100644
--- a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
+++ b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
@@ -167,7 +167,7 @@ public class ThreadPool extends AbstractComponent implements Closeable {
         builders.put(Names.INDEX, new FixedExecutorBuilder(settings, Names.INDEX, availableProcessors, 200));
         builders.put(Names.BULK, new FixedExecutorBuilder(settings, Names.BULK, availableProcessors, 50));
         builders.put(Names.GET, new FixedExecutorBuilder(settings, Names.GET, availableProcessors, 1000));
-        builders.put(Names.SEARCH, new FixedExecutorBuilder(settings, Names.SEARCH, ((availableProcessors * 3) / 2) + 1, 1000));
+        builders.put(Names.SEARCH, new FixedExecutorBuilder(settings, Names.SEARCH, searchThreadPoolSize(availableProcessors), 1000));
         builders.put(Names.MANAGEMENT, new ScalingExecutorBuilder(Names.MANAGEMENT, 1, 5, TimeValue.timeValueMinutes(5)));
         // no queue as this means clients will need to handle rejections on listener queue even if the operation succeeded
         // the assumption here is that the listeners should be very lightweight on the listeners side
@@ -389,6 +389,10 @@ public class ThreadPool extends AbstractComponent implements Closeable {
         return boundedBy(2 * numberOfProcessors, 2, Integer.MAX_VALUE);
     }
 
+    public static int searchThreadPoolSize(int availableProcessors) {
+        return ((availableProcessors * 3) / 2) + 1;
+    }
+
     class LoggingRunnable implements Runnable {
 
         private final Runnable runnable;
diff --git a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
index 2df6f798b8e..2ae23774bdf 100644
--- a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
@@ -22,8 +22,6 @@ package org.elasticsearch.action.search;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -37,7 +35,6 @@ import org.elasticsearch.test.StreamsUtils;
 
 import java.io.IOException;
 
-import static java.util.Collections.singletonMap;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -167,6 +164,13 @@ public class MultiSearchRequestTests extends ESTestCase {
                 builder.string());
     }
 
+    public void testMaxConcurrentSearchRequests() {
+        MultiSearchRequest request = new MultiSearchRequest();
+        request.maxConcurrentSearchRequests(randomIntBetween(1, Integer.MAX_VALUE));
+        expectThrows(IllegalArgumentException.class, () ->
+                request.maxConcurrentSearchRequests(randomIntBetween(Integer.MIN_VALUE, 0)));
+    }
+
     private IndicesQueriesRegistry registry() {
         IndicesQueriesRegistry registry = new IndicesQueriesRegistry();
         QueryParser<MatchAllQueryBuilder> parser = MatchAllQueryBuilder::fromXContent;
diff --git a/core/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java b/core/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java
new file mode 100644
index 00000000000..d751424ef72
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilter;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.TransportAction;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.LocalTransportAddress;
+import org.elasticsearch.tasks.TaskManager;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TransportMultiSearchActionTests extends ESTestCase {
+
+    public void testBatchExecute() throws Exception {
+        // Initialize depedencies of TransportMultiSearchAction
+        Settings settings = Settings.builder()
+                .put("node.name", TransportMultiSearchActionTests.class.getSimpleName())
+                .build();
+        ActionFilters actionFilters = mock(ActionFilters.class);
+        when(actionFilters.filters()).thenReturn(new ActionFilter[0]);
+        ThreadPool threadPool = new ThreadPool(settings);
+        TaskManager taskManager = mock(TaskManager.class);
+        TransportService transportService = mock(TransportService.class);
+        when(transportService.getTaskManager()).thenReturn(taskManager);
+        ClusterService clusterService = mock(ClusterService.class);
+        when(clusterService.state()).thenReturn(ClusterState.builder(new ClusterName("test")).build());
+        IndexNameExpressionResolver resolver = new IndexNameExpressionResolver(Settings.EMPTY);
+
+        // Keep track of the number of concurrent searches started by multi search api,
+        // and if there are more searches than is allowed create an error and remember that.
+        int maxAllowedConcurrentSearches = scaledRandomIntBetween(1, 20);
+        AtomicInteger counter = new AtomicInteger();
+        AtomicReference<AssertionError> errorHolder = new AtomicReference<>();
+        TransportAction<SearchRequest, SearchResponse> searchAction = new TransportAction<SearchRequest, SearchResponse>
+                (Settings.EMPTY, "action", threadPool, actionFilters, resolver, taskManager) {
+            @Override
+            protected void doExecute(SearchRequest request, ActionListener<SearchResponse> listener) {
+                int currentConcurrentSearches = counter.incrementAndGet();
+                if (currentConcurrentSearches > maxAllowedConcurrentSearches) {
+                    errorHolder.set(new AssertionError("Current concurrent search [" + currentConcurrentSearches +
+                            "] is higher than is allowed [" + maxAllowedConcurrentSearches + "]"));
+                }
+                threadPool.executor(ThreadPool.Names.GENERIC).execute(
+                        () -> {
+                            try {
+                                Thread.sleep(scaledRandomIntBetween(10, 1000));
+                            } catch (InterruptedException e) {
+                            }
+                            counter.decrementAndGet();
+                            listener.onResponse(new SearchResponse());
+                        }
+                );
+            }
+        };
+        TransportMultiSearchAction action =
+                new TransportMultiSearchAction(threadPool, actionFilters, transportService, clusterService, searchAction, resolver, 10);
+
+        // Execute the multi search api and fail if we find an error after executing:
+        try {
+            int numSearchRequests = randomIntBetween(16, 128);
+            MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
+            multiSearchRequest.maxConcurrentSearchRequests(maxAllowedConcurrentSearches);
+            for (int i = 0; i < numSearchRequests; i++) {
+                multiSearchRequest.add(new SearchRequest());
+            }
+
+            MultiSearchResponse response = action.execute(multiSearchRequest).actionGet();
+            assertThat(response.getResponses().length, equalTo(numSearchRequests));
+            assertThat(errorHolder.get(), nullValue());
+        } finally {
+            assertTrue(ESTestCase.terminate(threadPool));
+        }
+    }
+
+    public void testDefaultMaxConcurrentSearches() {
+        int numDataNodes = randomIntBetween(1, 10);
+        DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
+        for (int i = 0; i < numDataNodes; i++) {
+            builder.put(new DiscoveryNode("_id" + i, new LocalTransportAddress("_id" + i), Collections.emptyMap(),
+                    Collections.singleton(DiscoveryNode.Role.DATA), Version.CURRENT));
+        }
+        builder.put(new DiscoveryNode("master", new LocalTransportAddress("mater"), Collections.emptyMap(),
+                Collections.singleton(DiscoveryNode.Role.MASTER), Version.CURRENT));
+        builder.put(new DiscoveryNode("ingest", new LocalTransportAddress("ingest"), Collections.emptyMap(),
+                Collections.singleton(DiscoveryNode.Role.INGEST), Version.CURRENT));
+
+        ClusterState state = ClusterState.builder(new ClusterName("_name")).nodes(builder).build();
+        int result = TransportMultiSearchAction.defaultMaxConcurrentSearches(10, state);
+        assertThat(result, equalTo(10 * numDataNodes));
+
+        state = ClusterState.builder(new ClusterName("_name")).build();
+        result = TransportMultiSearchAction.defaultMaxConcurrentSearches(10, state);
+        assertThat(result, equalTo(1));
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/search/msearch/SimpleMultiSearchIT.java b/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java
similarity index 70%
rename from core/src/test/java/org/elasticsearch/search/msearch/SimpleMultiSearchIT.java
rename to core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java
index d3ee811be23..72306fd99d4 100644
--- a/core/src/test/java/org/elasticsearch/search/msearch/SimpleMultiSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.search.msearch;
 
+import org.elasticsearch.action.search.MultiSearchRequest;
 import org.elasticsearch.action.search.MultiSearchResponse;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -29,9 +30,8 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFa
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId;
 import static org.hamcrest.Matchers.equalTo;
 
-/**
- */
-public class SimpleMultiSearchIT extends ESIntegTestCase {
+public class MultiSearchIT extends ESIntegTestCase {
+
     public void testSimpleMultiSearch() {
         createIndex("test");
         ensureGreen();
@@ -54,4 +54,30 @@ public class SimpleMultiSearchIT extends ESIntegTestCase {
         assertFirstHit(response.getResponses()[0].getResponse(), hasId("1"));
         assertFirstHit(response.getResponses()[1].getResponse(), hasId("2"));
     }
+
+    public void testSimpleMultiSearchMoreRequests() {
+        createIndex("test");
+        int numDocs = randomIntBetween(0, 16);
+        for (int i = 0; i < numDocs; i++) {
+            client().prepareIndex("test", "type", Integer.toString(i)).setSource("{}").get();
+        }
+        refresh();
+
+        int numSearchRequests = randomIntBetween(0, 64);
+        MultiSearchRequest request = new MultiSearchRequest();
+        if (randomBoolean()) {
+            request.maxConcurrentSearchRequests(randomIntBetween(1, numSearchRequests));
+        }
+        for (int i = 0; i < numSearchRequests; i++) {
+            request.add(client().prepareSearch("test"));
+        }
+
+        MultiSearchResponse response = client().multiSearch(request).actionGet();
+        assertThat(response.getResponses().length, equalTo(numSearchRequests));
+        for (MultiSearchResponse.Item item : response) {
+            assertNoFailures(item.getResponse());
+            assertHitCount(item.getResponse(), numDocs);
+        }
+    }
+
 }
diff --git a/docs/reference/search/multi-search.asciidoc b/docs/reference/search/multi-search.asciidoc
index 699a9b37123..7ecc0f6554c 100644
--- a/docs/reference/search/multi-search.asciidoc
+++ b/docs/reference/search/multi-search.asciidoc
@@ -71,6 +71,10 @@ against the `test2` index.
 The `search_type` can be set in a similar manner to globally apply to
 all search requests.
 
+The msearch's `max_concurrent_searches` request parameter can be used to control
+the maximum number of concurrent searches the multi search api will execute.
+This default is based on the number of data nodes and the default search thread pool size.
+
 [float]
 [[msearch-security]]
 === Security
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json
index 87a86fb1298..0344702ecfe 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json
@@ -20,6 +20,10 @@
           "type" : "enum",
           "options" : ["query_then_fetch", "query_and_fetch", "dfs_query_then_fetch", "dfs_query_and_fetch"],
           "description" : "Search operation type"
+        },
+        "max_concurrent_searches" : {
+          "type" : "number",
+          "description" : "Controls the maximum number of concurrent searches the multi search api will execute"
         }
       }
     },

From f20928b1461ec6a6cf91712d5c8a4a4b4321164d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Fri, 10 Jun 2016 16:12:45 +0200
Subject: [PATCH 0805/1311] Remove redundant parseElementst() method in
 RescorePhase and SuggestPhase

The default implementation in SearchPhase does the same.
---
 .../elasticsearch/search/rescore/RescorePhase.java   | 12 ------------
 .../elasticsearch/search/suggest/SuggestPhase.java   | 10 ----------
 2 files changed, 22 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java b/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java
index b82ed941e1c..06f7cf5b1cd 100644
--- a/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java
+++ b/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java
@@ -24,13 +24,10 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.SearchPhase;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
-import java.util.Collections;
-import java.util.Map;
 
 /**
  */
@@ -41,15 +38,6 @@ public class RescorePhase extends AbstractComponent implements SearchPhase {
         super(settings);
     }
 
-    /**
-     * rescorers do not have a parse element, they use
-     * {@link RescoreBuilder#parseFromXContent(org.elasticsearch.index.query.QueryParseContext)} for parsing instead.
-     */
-    @Override
-    public Map<String, ? extends SearchParseElement> parseElements() {
-        return Collections.emptyMap();
-    }
-
     @Override
     public void preProcess(SearchContext context) {
     }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java
index 520322587bb..3aedb6e6604 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java
@@ -23,7 +23,6 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.SearchPhase;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.suggest.Suggest.Suggestion;
@@ -36,8 +35,6 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
-import static java.util.Collections.emptyMap;
-
 /**
  */
 public class SuggestPhase extends AbstractComponent implements SearchPhase {
@@ -47,13 +44,6 @@ public class SuggestPhase extends AbstractComponent implements SearchPhase {
         super(settings);
     }
 
-    @Override
-    public Map<String, ? extends SearchParseElement> parseElements() {
-        // this is used to parse SearchSourceBuilder.ext() bytes
-        // we don't allow any suggestion parsing for the extension
-        return emptyMap();
-    }
-
     @Override
     public void preProcess(SearchContext context) {
     }

From f1d5fd72c88a0667c3b38d35f5c401a203fc2139 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Mon, 13 Jun 2016 10:58:23 +0200
Subject: [PATCH 0806/1311] Revert "Mark field in ReindexSameIndexTests as
 final"

This reverts commit 6d8692576e49ed9d4d570e1631a25f015f271cb8.
---
 .../org/elasticsearch/index/reindex/ReindexSameIndexTests.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSameIndexTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSameIndexTests.java
index 2af05a8455b..7bdb455cc33 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSameIndexTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSameIndexTests.java
@@ -43,7 +43,7 @@ public class ReindexSameIndexTests extends ESTestCase {
 
     private static ClusterState STATE;
     private static final IndexNameExpressionResolver INDEX_NAME_EXPRESSION_RESOLVER = new IndexNameExpressionResolver(Settings.EMPTY);
-    private static final AutoCreateIndex AUTO_CREATE_INDEX = new AutoCreateIndex(Settings.EMPTY, INDEX_NAME_EXPRESSION_RESOLVER);
+    private static AutoCreateIndex AUTO_CREATE_INDEX = new AutoCreateIndex(Settings.EMPTY, INDEX_NAME_EXPRESSION_RESOLVER);
 
     @BeforeClass
     public static void beforeClass() {

From 7379b17e619aba89eb86d0dda84e8b26883efac4 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Mon, 13 Jun 2016 10:58:34 +0200
Subject: [PATCH 0807/1311] Revert "Make random UUIDs reproducible in tests"

This reverts commit a25b8ee1bf2e70bc28f688731ef43962a3ef18ce.
---
 .../common/RandomBasedUUIDGenerator.java      |  9 +++--
 .../org/elasticsearch/common/Randomness.java  | 39 ++-----------------
 .../cluster/ClusterChangedEventTests.java     | 17 +++-----
 .../org/elasticsearch/common/UUIDTests.java   | 17 +-------
 .../index/reindex/ReindexSameIndexTests.java  | 25 +++++-------
 5 files changed, 24 insertions(+), 83 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java b/core/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java
index 970adfc03fb..9f5e5f34a1b 100644
--- a/core/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java
+++ b/core/src/main/java/org/elasticsearch/common/RandomBasedUUIDGenerator.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.common;
 
+
+import java.io.IOException;
 import java.util.Base64;
 import java.util.Random;
 
@@ -30,7 +32,7 @@ class RandomBasedUUIDGenerator implements UUIDGenerator {
      */
     @Override
     public String getBase64UUID() {
-        return getBase64UUID(Randomness.getSecure());
+        return getBase64UUID(SecureRandomHolder.INSTANCE);
     }
 
     /**
@@ -47,13 +49,12 @@ class RandomBasedUUIDGenerator implements UUIDGenerator {
          * stamp (bits 4 through 7 of the time_hi_and_version field).*/
         randomBytes[6] &= 0x0f;  /* clear the 4 most significant bits for the version  */
         randomBytes[6] |= 0x40;  /* set the version to 0100 / 0x40 */
-
-        /* Set the variant:
+        
+        /* Set the variant: 
          * The high field of th clock sequence multiplexed with the variant.
          * We set only the MSB of the variant*/
         randomBytes[8] &= 0x3f;  /* clear the 2 most significant bits */
         randomBytes[8] |= 0x80;  /* set the variant (MSB is set)*/
         return Base64.getUrlEncoder().withoutPadding().encodeToString(randomBytes);
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/common/Randomness.java b/core/src/main/java/org/elasticsearch/common/Randomness.java
index bb700455be5..ddcd3ea90f7 100644
--- a/core/src/main/java/org/elasticsearch/common/Randomness.java
+++ b/core/src/main/java/org/elasticsearch/common/Randomness.java
@@ -23,9 +23,6 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 
 import java.lang.reflect.Method;
-import java.security.NoSuchAlgorithmException;
-import java.security.NoSuchProviderException;
-import java.security.SecureRandom;
 import java.util.Collections;
 import java.util.List;
 import java.util.Random;
@@ -47,7 +44,6 @@ import java.util.concurrent.ThreadLocalRandom;
  * DiscoveryService#NODE_ID_SEED_SETTING)).
  */
 public final class Randomness {
-
     private static final Method currentMethod;
     private static final Method getRandomMethod;
 
@@ -76,7 +72,7 @@ public final class Randomness {
      * @param setting  the setting to access the seed
      * @return a reproducible source of randomness
      */
-    public static Random get(final Settings settings, final Setting<Long> setting) {
+    public static Random get(Settings settings, Setting<Long> setting) {
         if (setting.exists(settings)) {
             return new Random(setting.get(settings));
         } else {
@@ -102,7 +98,7 @@ public final class Randomness {
     public static Random get() {
         if (currentMethod != null && getRandomMethod != null) {
             try {
-                final Object randomizedContext = currentMethod.invoke(null);
+                Object randomizedContext = currentMethod.invoke(null);
                 return (Random) getRandomMethod.invoke(randomizedContext);
             } catch (ReflectiveOperationException e) {
                 // unexpected, bail
@@ -113,42 +109,13 @@ public final class Randomness {
         }
     }
 
-    /**
-     * Provides a source of randomness that is reproducible when
-     * running under the Elasticsearch test suite, and otherwise
-     * produces a non-reproducible source of secure randomness.
-     * Reproducible sources of randomness are created when the system
-     * property "tests.seed" is set and the security policy allows
-     * reading this system property. Otherwise, non-reproducible
-     * sources of secure randomness are created.
-     *
-     * @return a source of randomness
-     * @throws IllegalStateException if running tests but was not able
-     *                               to acquire an instance of Random from
-     *                               RandomizedContext or tests are
-     *                               running but tests.seed is not set
-     */
-    public static Random getSecure() {
-        if (currentMethod != null && getRandomMethod != null) {
-            return get();
-        } else {
-            return getSecureRandomWithoutSeed();
-        }
-    }
-
     @SuppressForbidden(reason = "ThreadLocalRandom is okay when not running tests")
     private static Random getWithoutSeed() {
         assert currentMethod == null && getRandomMethod == null : "running under tests but tried to create non-reproducible random";
         return ThreadLocalRandom.current();
     }
 
-    private static SecureRandom getSecureRandomWithoutSeed() {
-        assert currentMethod == null && getRandomMethod == null : "running under tests but tried to create non-reproducible random";
-        return SecureRandomHolder.INSTANCE;
-    }
-
-    public static void shuffle(final List<?> list) {
+    public static void shuffle(List<?> list) {
         Collections.shuffle(list, get());
     }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java b/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
index 7504c778d36..72748a59986 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
@@ -34,7 +34,6 @@ import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
-import org.junit.BeforeClass;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -55,17 +54,11 @@ public class ClusterChangedEventTests extends ESTestCase {
 
     private static final ClusterName TEST_CLUSTER_NAME = new ClusterName("test");
     private static final String NODE_ID_PREFIX = "node_";
-    private static String INITIAL_CLUSTER_ID;
-    private static List<Index> initialIndices;
-
-    @BeforeClass
-    public static void beforeClass() {
-        INITIAL_CLUSTER_ID = UUIDs.randomBase64UUID();
-        // the initial indices which every cluster state test starts out with
-        initialIndices = Arrays.asList(new Index("idx1", UUIDs.randomBase64UUID()),
-            new Index("idx2", UUIDs.randomBase64UUID()),
-            new Index("idx3", UUIDs.randomBase64UUID()));
-    }
+    private static final String INITIAL_CLUSTER_ID = UUIDs.randomBase64UUID();
+    // the initial indices which every cluster state test starts out with
+    private static final List<Index> initialIndices = Arrays.asList(new Index("idx1", UUIDs.randomBase64UUID()),
+                                                                    new Index("idx2", UUIDs.randomBase64UUID()),
+                                                                    new Index("idx3", UUIDs.randomBase64UUID()));
 
     /**
      * Test basic properties of the ClusterChangedEvent class:
diff --git a/core/src/test/java/org/elasticsearch/common/UUIDTests.java b/core/src/test/java/org/elasticsearch/common/UUIDTests.java
index d963db2d6f6..f82e1a464d9 100644
--- a/core/src/test/java/org/elasticsearch/common/UUIDTests.java
+++ b/core/src/test/java/org/elasticsearch/common/UUIDTests.java
@@ -20,9 +20,7 @@ package org.elasticsearch.common;
 
 import org.elasticsearch.test.ESTestCase;
 
-import java.security.SecureRandom;
 import java.util.HashSet;
-import java.util.Random;
 import java.util.Set;
 
 public class UUIDTests extends ESTestCase {
@@ -43,18 +41,7 @@ public class UUIDTests extends ESTestCase {
     }
 
     public void testThreadedRandomUUID() {
-        // we can not use a reproducible source of randomness for this
-        // test, the test explicitly relies on each thread having a
-        // unique source of randomness; thus, we fake what production
-        // code does when using a RandomBasedUUIDGenerator
-        testUUIDThreaded(new RandomBasedUUIDGenerator() {
-            private final SecureRandom sr = SecureRandomHolder.INSTANCE;
-
-            @Override
-            public String getBase64UUID() {
-                return getBase64UUID(sr);
-            }
-        });
+        testUUIDThreaded(randomUUIDGen);
     }
 
     Set<String> verifyUUIDSet(int count, UUIDGenerator uuidSource) {
@@ -111,6 +98,6 @@ public class UUIDTests extends ESTestCase {
         for (UUIDGenRunner runner : runners) {
             globalSet.addAll(runner.uuidSet);
         }
-        assertEquals(count * uuids, globalSet.size());
+        assertEquals(count*uuids, globalSet.size());
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSameIndexTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSameIndexTests.java
index 7bdb455cc33..f1218414af7 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSameIndexTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSameIndexTests.java
@@ -32,7 +32,6 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
-import org.junit.BeforeClass;
 
 import static org.hamcrest.Matchers.containsString;
 
@@ -40,22 +39,16 @@ import static org.hamcrest.Matchers.containsString;
  * Tests that indexing from an index back into itself fails the request.
  */
 public class ReindexSameIndexTests extends ESTestCase {
-
-    private static ClusterState STATE;
+    private static final ClusterState STATE = ClusterState.builder(new ClusterName("test")).metaData(MetaData.builder()
+                .put(index("target", "target_alias", "target_multi"), true)
+                .put(index("target2", "target_multi"), true)
+                .put(index("foo"), true)
+                .put(index("bar"), true)
+                .put(index("baz"), true)
+                .put(index("source", "source_multi"), true)
+                .put(index("source2", "source_multi"), true)).build();
     private static final IndexNameExpressionResolver INDEX_NAME_EXPRESSION_RESOLVER = new IndexNameExpressionResolver(Settings.EMPTY);
-    private static AutoCreateIndex AUTO_CREATE_INDEX = new AutoCreateIndex(Settings.EMPTY, INDEX_NAME_EXPRESSION_RESOLVER);
-
-    @BeforeClass
-    public static void beforeClass() {
-        STATE = ClusterState.builder(new ClusterName("test")).metaData(MetaData.builder()
-            .put(index("target", "target_alias", "target_multi"), true)
-            .put(index("target2", "target_multi"), true)
-            .put(index("foo"), true)
-            .put(index("bar"), true)
-            .put(index("baz"), true)
-            .put(index("source", "source_multi"), true)
-            .put(index("source2", "source_multi"), true)).build();
-    }
+    private static final AutoCreateIndex AUTO_CREATE_INDEX = new AutoCreateIndex(Settings.EMPTY, INDEX_NAME_EXPRESSION_RESOLVER);
 
     public void testObviousCases() throws Exception {
         fails("target", "target");

From 3cd201e67e49e1fd6cd36ab77777eb49737c8e1b Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Mon, 13 Jun 2016 12:02:09 +0200
Subject: [PATCH 0808/1311] [TEST] add comment on using animal-sniffer suppress
 annotation

---
 .../java/org/elasticsearch/client/sniff/HostsSnifferTests.java  | 2 ++
 .../java/org/elasticsearch/client/RestClientIntegTests.java     | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
index 06f8fd6940e..c6daf9c6f40 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
@@ -56,6 +56,7 @@ import java.util.Set;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
 
+//animal-sniffer doesn't like our usage of com.sun.net.httpserver.* classes
 @IgnoreJRERequirement
 public class HostsSnifferTests extends LuceneTestCase {
 
@@ -118,6 +119,7 @@ public class HostsSnifferTests extends LuceneTestCase {
         return httpServer;
     }
 
+    //animal-sniffer doesn't like our usage of com.sun.net.httpserver.* classes
     @IgnoreJRERequirement
     private static class ResponseHandler implements HttpHandler {
         private final int sniffTimeoutMillis;
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java b/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
index 7c4be56040b..7bc6dc76c91 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
@@ -58,6 +58,7 @@ import static org.hamcrest.CoreMatchers.equalTo;
  * Integration test to check interaction between {@link RestClient} and {@link org.apache.http.client.HttpClient}.
  * Works against a real http server, one single host.
  */
+//animal-sniffer doesn't like our usage of com.sun.net.httpserver.* classes
 @IgnoreJRERequirement
 public class RestClientIntegTests extends LuceneTestCase {
 
@@ -88,6 +89,7 @@ public class RestClientIntegTests extends LuceneTestCase {
         httpServer.createContext("/" + statusCode, new ResponseHandler(statusCode));
     }
 
+    //animal-sniffer doesn't like our usage of com.sun.net.httpserver.* classes
     @IgnoreJRERequirement
     private static class ResponseHandler implements HttpHandler {
         private final int statusCode;

From 8f7b7fb81312fbf43d4e6767bd5639e5fecd59b7 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Mon, 13 Jun 2016 12:59:30 +0200
Subject: [PATCH 0809/1311] added comments to clarify RequestLogger and
 DeadHostState

---
 .../main/java/org/elasticsearch/client/DeadHostState.java    | 5 +++++
 .../main/java/org/elasticsearch/client/RequestLogger.java    | 2 ++
 2 files changed, 7 insertions(+)

diff --git a/client/src/main/java/org/elasticsearch/client/DeadHostState.java b/client/src/main/java/org/elasticsearch/client/DeadHostState.java
index 44b7532d5de..a7b222da70e 100644
--- a/client/src/main/java/org/elasticsearch/client/DeadHostState.java
+++ b/client/src/main/java/org/elasticsearch/client/DeadHostState.java
@@ -41,6 +41,11 @@ final class DeadHostState {
         this.deadUntilNanos = System.nanoTime() + MIN_CONNECTION_TIMEOUT_NANOS;
     }
 
+    /**
+     * We keep track of how many times a certain node fails consecutively. The higher that number is the longer we will wait
+     * to retry that same node again. Minimum is 1 minute (for a node the only failed once), maximum is 30 minutes (for a node
+     * that failed many consecutive times).
+     */
     DeadHostState(DeadHostState previousDeadHostState) {
         long timeoutNanos = (long)Math.min(MIN_CONNECTION_TIMEOUT_NANOS * 2 * Math.pow(2, previousDeadHostState.failedAttempts * 0.5 - 1),
                 MAX_CONNECTION_TIMEOUT_NANOS);
diff --git a/client/src/main/java/org/elasticsearch/client/RequestLogger.java b/client/src/main/java/org/elasticsearch/client/RequestLogger.java
index f1b406b5cf3..c08e0c69941 100644
--- a/client/src/main/java/org/elasticsearch/client/RequestLogger.java
+++ b/client/src/main/java/org/elasticsearch/client/RequestLogger.java
@@ -40,6 +40,8 @@ import java.nio.charset.StandardCharsets;
 /**
  * Helper class that exposes static methods to unify the way requests are logged.
  * Includes trace logging to log complete requests and responses in curl format.
+ * Useful for debugging, manually sending logged requests via curl and checking their responses.
+ * Trace logging is a feature that all the language clients provide.
  */
 final class RequestLogger {
 

From 116805b28b8eaf386e39ca1c132704b7ea94fab7 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Mon, 13 Jun 2016 14:02:01 +0200
Subject: [PATCH 0810/1311] remove TODO around copying hosts when rotating the
 collection, it's not a problem for now

---
 client/src/main/java/org/elasticsearch/client/RestClient.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 350b750415d..b871e2e1346 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -244,7 +244,6 @@ public final class RestClient implements Closeable {
         }
 
         List<HttpHost> rotatedHosts = new ArrayList<>(filteredHosts);
-        //TODO is it possible to make this O(1)? (rotate is O(n))
         Collections.rotate(rotatedHosts, rotatedHosts.size() - lastHostIndex.getAndIncrement());
         return rotatedHosts;
     }

From 5e3f2ce7aa47710267d509c9335eb6e143ad138d Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Mon, 13 Jun 2016 15:29:20 +0200
Subject: [PATCH 0811/1311] test: Added extra tests for percolating with nested
 documents.

---
 .../percolator/PercolateQueryBuilder.java     |  2 +-
 .../PercolateQueryBuilderTests.java           | 36 ++++++++++++
 .../percolator/PercolatorQuerySearchIT.java   | 57 +++++++++++++++++++
 3 files changed, 94 insertions(+), 1 deletion(-)

diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
index aadcabda006..9955375b183 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
@@ -447,7 +447,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
         return document;
     }
 
-    private static IndexSearcher createMultiDocumentSearcher(Analyzer analyzer, ParsedDocument doc) {
+    static IndexSearcher createMultiDocumentSearcher(Analyzer analyzer, ParsedDocument doc) {
         IndexReader[] memoryIndices = new IndexReader[doc.docs().size()];
         List<ParseContext.Document> docs = doc.docs();
         int rootDocIndex = docs.size() - 1;
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
index c090ebf6dbf..adf8323b9b7 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
@@ -20,7 +20,15 @@
 package org.elasticsearch.percolator;
 
 import com.fasterxml.jackson.core.JsonParseException;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
@@ -30,10 +38,14 @@ import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -44,11 +56,14 @@ import org.hamcrest.Matchers;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.sameInstance;
 
 public class PercolateQueryBuilderTests extends AbstractQueryTestCase<PercolateQueryBuilder> {
 
@@ -233,6 +248,27 @@ public class PercolateQueryBuilderTests extends AbstractQueryTestCase<PercolateQ
         assertThat(e.getMessage(), equalTo("[percolate] query is missing required [document_type] parameter"));
     }
 
+    public void testCreateMultiDocumentSearcher() throws Exception {
+        int numDocs = randomIntBetween(1, 8);
+        List<ParseContext.Document> docs = new ArrayList<>(numDocs);
+        for (int i = 0; i < numDocs; i++) {
+            docs.add(new ParseContext.Document());
+        }
+
+        Analyzer analyzer = new WhitespaceAnalyzer();
+        ParsedDocument parsedDocument = new ParsedDocument(null, "_id", "_type", null, -1L, -1L, docs, null, null);
+        IndexSearcher indexSearcher = PercolateQueryBuilder.createMultiDocumentSearcher(analyzer, parsedDocument);
+        assertThat(indexSearcher.getIndexReader().numDocs(), equalTo(numDocs));
+
+        // ensure that any query get modified so that the nested docs are never included as hits:
+        Query query = new MatchAllDocsQuery();
+        BooleanQuery result = (BooleanQuery) indexSearcher.createNormalizedWeight(query, true).getQuery();
+        assertThat(result.clauses().size(), equalTo(2));
+        assertThat(result.clauses().get(0).getQuery(), sameInstance(query));
+        assertThat(result.clauses().get(0).getOccur(), equalTo(BooleanClause.Occur.MUST));
+        assertThat(result.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.MUST_NOT));
+    }
+
     private static BytesReference randomSource() {
         try {
             XContentBuilder xContent = XContentFactory.jsonBuilder();
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
index a277549f48e..e7690e23df4 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
@@ -18,12 +18,18 @@
  */
 package org.elasticsearch.percolator;
 
+import org.apache.lucene.search.join.ScoreMode;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.query.MultiMatchQueryBuilder;
+import org.elasticsearch.index.query.Operator;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.highlight.HighlightBuilder;
 import org.elasticsearch.search.sort.SortOrder;
@@ -32,6 +38,7 @@ import org.elasticsearch.test.ESSingleNodeTestCase;
 import java.util.Collection;
 import java.util.Collections;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
 import static org.elasticsearch.index.query.QueryBuilders.commonTermsQuery;
@@ -42,6 +49,7 @@ import static org.elasticsearch.index.query.QueryBuilders.spanNearQuery;
 import static org.elasticsearch.index.query.QueryBuilders.spanNotQuery;
 import static org.elasticsearch.index.query.QueryBuilders.spanTermQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.instanceOf;
@@ -382,4 +390,53 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
         assertThat(e.getCause().getMessage(), equalTo("a document can only contain one percolator query"));
     }
 
+    public void testPercolateQueryWithNestedDocuments() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder();
+        mapping.startObject().startObject("properties").startObject("companyname").field("type", "text").endObject()
+                .startObject("employee").field("type", "nested").startObject("properties")
+                .startObject("name").field("type", "text").endObject().endObject().endObject().endObject()
+                .endObject();
+        createIndex("test", client().admin().indices().prepareCreate("test")
+                .addMapping("employee", mapping)
+                .addMapping("queries", "query", "type=percolator")
+        );
+        client().prepareIndex("test", "queries", "q").setSource(jsonBuilder().startObject()
+                .field("query", QueryBuilders.nestedQuery("employee",
+                        QueryBuilders.matchQuery("employee.name", "virginia potts").operator(Operator.AND), ScoreMode.Avg)
+                ).endObject())
+                .setRefreshPolicy(IMMEDIATE)
+                .get();
+
+        SearchResponse response = client().prepareSearch()
+                .setQuery(new PercolateQueryBuilder("query", "employee",
+                        XContentFactory.jsonBuilder()
+                            .startObject().field("companyname", "stark")
+                                .startArray("employee")
+                                    .startObject().field("name", "virginia potts").endObject()
+                                    .startObject().field("name", "tony stark").endObject()
+                                .endArray()
+                            .endObject().bytes()))
+                .get();
+        assertHitCount(response, 1);
+        assertThat(response.getHits().getAt(0).getId(), equalTo("q"));
+
+        response = client().prepareSearch()
+                .setQuery(new PercolateQueryBuilder("query", "employee",
+                        XContentFactory.jsonBuilder()
+                            .startObject().field("companyname", "notstark")
+                                .startArray("employee")
+                                    .startObject().field("name", "virginia stark").endObject()
+                                    .startObject().field("name", "tony stark").endObject()
+                                .endArray()
+                            .endObject().bytes()))
+                .get();
+        assertHitCount(response, 0);
+
+        response = client().prepareSearch()
+                .setQuery(new PercolateQueryBuilder("query", "employee",
+                        XContentFactory.jsonBuilder().startObject().field("companyname", "notstark").endObject().bytes()))
+                .get();
+        assertHitCount(response, 0);
+    }
+
 }

From 8287e13616217488c78e8828f77f2ed6e332ea44 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Mon, 13 Jun 2016 15:58:36 +0200
Subject: [PATCH 0812/1311] test: fixed test bug

---
 .../elasticsearch/percolator/PercolateQueryBuilderTests.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
index adf8323b9b7..67024150fb7 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
@@ -249,7 +249,7 @@ public class PercolateQueryBuilderTests extends AbstractQueryTestCase<PercolateQ
     }
 
     public void testCreateMultiDocumentSearcher() throws Exception {
-        int numDocs = randomIntBetween(1, 8);
+        int numDocs = randomIntBetween(2, 8);
         List<ParseContext.Document> docs = new ArrayList<>(numDocs);
         for (int i = 0; i < numDocs; i++) {
             docs.add(new ParseContext.Document());

From 30b8a51eb9d92418ca4a89646431ea347e20aae6 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 13 Jun 2016 10:35:10 -0400
Subject: [PATCH 0813/1311] Fix lang-painless packaging test

This commit fixes a failing lang-painless packaging test after a change
to the dependencies was made in commit
0bfb166eeba0ba8f292550cb9100708c88e4a4c9.
---
 .../packaging/scripts/module_and_plugin_test_cases.bash      | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
index ac54bf3b664..c81f84cb778 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
@@ -248,10 +248,7 @@ fi
 }
 
 @test "[$GROUP] check lang-painless module" {
-    # we specify the version on the asm-5.0.4.jar so that the test does
-    # not spuriously pass if the jar is missing but the other asm jars
-    # are present
-    check_secure_module lang-painless antlr4-runtime-*.jar asm-5.0.4.jar asm-commons-*.jar asm-tree-*.jar
+    check_secure_module lang-painless antlr4-runtime-*.jar asm-debug-all-*.jar
 }
 
 @test "[$GROUP] install javascript plugin" {

From 991c2221a1fab3eb52da4c44d6e458cdc1501651 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Mon, 13 Jun 2016 09:26:45 -0700
Subject: [PATCH 0814/1311] Set next version back to alpha4

---
 buildSrc/version.properties                       |  2 +-
 core/src/main/java/org/elasticsearch/Version.java | 10 +++++-----
 qa/backwards-5.0/build.gradle                     |  2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index f757eb8eef6..4ee6febdfa5 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,4 +1,4 @@
-elasticsearch     = 5.0.0
+elasticsearch     = 5.0.0-alpha4
 lucene            = 6.1.0-snapshot-3a57bea
 
 # optional dependencies
diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java
index 6e6e82b3fc5..e77b0f450dd 100644
--- a/core/src/main/java/org/elasticsearch/Version.java
+++ b/core/src/main/java/org/elasticsearch/Version.java
@@ -76,9 +76,9 @@ public class Version {
     public static final Version V_5_0_0_alpha2 = new Version(V_5_0_0_alpha2_ID, org.apache.lucene.util.Version.LUCENE_6_0_0);
     public static final int V_5_0_0_alpha3_ID = 5000003;
     public static final Version V_5_0_0_alpha3 = new Version(V_5_0_0_alpha3_ID, org.apache.lucene.util.Version.LUCENE_6_0_0);
-    public static final int V_5_0_0_ID = 5000099;
-    public static final Version V_5_0_0 = new Version(V_5_0_0_ID, org.apache.lucene.util.Version.LUCENE_6_1_0);
-    public static final Version CURRENT = V_5_0_0;
+    public static final int V_5_0_0_alpha4_ID = 5000004;
+    public static final Version V_5_0_0_alpha4 = new Version(V_5_0_0_alpha4_ID, org.apache.lucene.util.Version.LUCENE_6_1_0);
+    public static final Version CURRENT = V_5_0_0_alpha4;
 
     static {
         assert CURRENT.luceneVersion.equals(org.apache.lucene.util.Version.LATEST) : "Version must be upgraded to ["
@@ -91,8 +91,8 @@ public class Version {
 
     public static Version fromId(int id) {
         switch (id) {
-            case V_5_0_0_ID:
-                return V_5_0_0;
+            case V_5_0_0_alpha4_ID:
+                return V_5_0_0_alpha4;
             case V_5_0_0_alpha3_ID:
                 return V_5_0_0_alpha3;
             case V_5_0_0_alpha2_ID:
diff --git a/qa/backwards-5.0/build.gradle b/qa/backwards-5.0/build.gradle
index 93d361c989c..fbce12f8126 100644
--- a/qa/backwards-5.0/build.gradle
+++ b/qa/backwards-5.0/build.gradle
@@ -18,6 +18,6 @@ integTest {
   cluster {
     numNodes = 2
     numBwcNodes = 1
-    bwcVersion = "5.0.0-SNAPSHOT" // this is the same as the current version until we released the first RC
+    bwcVersion = "5.0.0-alpha4-SNAPSHOT" // this is the same as the current version until we released the first RC
   }
 }

From 4c15d143d91112f4484ef2256c3bd5a97358f6b1 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 10 Jun 2016 17:20:04 -0400
Subject: [PATCH 0815/1311] Port reindex's script tests to painless

All but the timeout ones work. Those are skipped for now.
---
 .../build.gradle                              |  6 ----
 .../SmokeTestReindexWithPainlessIT.java}      |  4 +--
 .../rest-api-spec/test/reindex/10_script.yaml | 11 ++++++++
 .../rest-api-spec/test/reindex/20_broken.yaml |  3 +-
 .../test/reindex/30_timeout.yaml              |  6 +++-
 .../test/reindex/40_search_failures.yaml      | 11 ++++----
 .../test/update_by_query/10_script.yaml       | 28 +++++++++++--------
 .../test/update_by_query/20_broken.yaml       |  3 +-
 .../test/update_by_query/30_timeout.yaml      |  6 +++-
 .../update_by_query/40_search_failure.yaml    | 11 ++++----
 settings.gradle                               |  6 ++--
 11 files changed, 59 insertions(+), 36 deletions(-)
 rename qa/{smoke-test-reindex-with-groovy => smoke-test-reindex-with-painless}/build.gradle (92%)
 rename qa/{smoke-test-reindex-with-groovy/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithGroovyIT.java => smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessIT.java} (89%)
 rename qa/{smoke-test-reindex-with-groovy => smoke-test-reindex-with-painless}/src/test/resources/rest-api-spec/test/reindex/10_script.yaml (97%)
 rename qa/{smoke-test-reindex-with-groovy => smoke-test-reindex-with-painless}/src/test/resources/rest-api-spec/test/reindex/20_broken.yaml (80%)
 rename qa/{smoke-test-reindex-with-groovy => smoke-test-reindex-with-painless}/src/test/resources/rest-api-spec/test/reindex/30_timeout.yaml (75%)
 rename qa/{smoke-test-reindex-with-groovy => smoke-test-reindex-with-painless}/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml (62%)
 rename qa/{smoke-test-reindex-with-groovy => smoke-test-reindex-with-painless}/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml (92%)
 rename qa/{smoke-test-reindex-with-groovy => smoke-test-reindex-with-painless}/src/test/resources/rest-api-spec/test/update_by_query/20_broken.yaml (78%)
 rename qa/{smoke-test-reindex-with-groovy => smoke-test-reindex-with-painless}/src/test/resources/rest-api-spec/test/update_by_query/30_timeout.yaml (74%)
 rename qa/{smoke-test-reindex-with-groovy => smoke-test-reindex-with-painless}/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yaml (58%)

diff --git a/qa/smoke-test-reindex-with-groovy/build.gradle b/qa/smoke-test-reindex-with-painless/build.gradle
similarity index 92%
rename from qa/smoke-test-reindex-with-groovy/build.gradle
rename to qa/smoke-test-reindex-with-painless/build.gradle
index c4b462ce45a..51a2647f444 100644
--- a/qa/smoke-test-reindex-with-groovy/build.gradle
+++ b/qa/smoke-test-reindex-with-painless/build.gradle
@@ -18,9 +18,3 @@
  */
 
 apply plugin: 'elasticsearch.rest-test'
-
-integTest {
-  cluster {
-    setting 'script.inline', 'true'
-  }
-}
diff --git a/qa/smoke-test-reindex-with-groovy/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithGroovyIT.java b/qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessIT.java
similarity index 89%
rename from qa/smoke-test-reindex-with-groovy/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithGroovyIT.java
rename to qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessIT.java
index 975bf596eaa..7857259a60d 100644
--- a/qa/smoke-test-reindex-with-groovy/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithGroovyIT.java
+++ b/qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessIT.java
@@ -27,8 +27,8 @@ import org.elasticsearch.test.rest.parser.RestTestParseException;
 
 import java.io.IOException;
 
-public class SmokeTestReindexWithGroovyIT extends ESRestTestCase {
-    public SmokeTestReindexWithGroovyIT(@Name("yaml") RestTestCandidate testCandidate) {
+public class SmokeTestReindexWithPainlessIT extends ESRestTestCase {
+    public SmokeTestReindexWithPainlessIT(@Name("yaml") RestTestCandidate testCandidate) {
         super(testCandidate);
     }
 
diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/10_script.yaml b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/10_script.yaml
similarity index 97%
rename from qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/10_script.yaml
rename to qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/10_script.yaml
index 783838d7f8e..0e99d862fd1 100644
--- a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/10_script.yaml
+++ b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/10_script.yaml
@@ -18,6 +18,7 @@
           dest:
             index: new_twitter
           script:
+            lang: painless
             inline: ctx._source.user = "other" + ctx._source.user
   - match: {created: 1}
   - match: {noops: 0}
@@ -57,6 +58,7 @@
           dest:
             index: new_twitter
           script:
+            lang: painless
             inline: if (ctx._id == "1") {ctx._source.user = "other" + ctx._source.user}
   - match: {created: 2}
   - match: {noops: 0}
@@ -116,6 +118,7 @@
           dest:
             index: new_twitter
           script:
+            lang: painless
             inline: ctx._parent = ctx._source.user
   - match: {created: 1}
   - match: {noops: 0}
@@ -159,6 +162,7 @@
           dest:
             index: new_twitter
           script:
+            lang: painless
             inline: ctx._routing = ctx._source.user
   - match: {created: 2}
   - match: {noops: 0}
@@ -217,6 +221,7 @@
           dest:
             index: new_twitter
           script:
+            lang: painless
             inline: ctx._parent = ctx._source.user; ctx._routing = "cat"
   - match: {created: 1}
   - match: {noops: 0}
@@ -262,6 +267,7 @@
           dest:
             index: new_twitter
           script:
+            lang: painless
             inline: if (ctx._source.user == "kimchy") {ctx._source.user = "not" + ctx._source.user} else {ctx.op = "noop"}
   - match: {created: 1}
   - match: {noops: 1}
@@ -314,6 +320,7 @@
           dest:
             index: new_twitter
           script:
+            lang: painless
             inline: ctx.op = "noop"
   - match: {updated: 0}
   - match: {noops: 2}
@@ -354,6 +361,7 @@
             index: new_twitter
             version_type: external
           script:
+            lang: painless
             inline: ctx._source.user = "other" + ctx._source.user; ctx._version = null
   - match: {updated: 1}
   - match: {noops: 0}
@@ -393,6 +401,7 @@
           dest:
             index: new_twitter
           script:
+            lang: painless
             inline: ctx._source.user = "other" + ctx._source.user; ctx._id = null
   - match: {created: 1}
   - match: {noops: 0}
@@ -432,6 +441,7 @@
           dest:
             index: new_twitter
           script:
+            lang: painless
             inline: if (ctx._source.user == "kimchy") {ctx._index = 'other_new_twitter'}
   - match: {created: 2}
   - match: {noops: 0}
@@ -504,6 +514,7 @@
             index: index2
             type: type2
           script:
+            lang: painless
             inline: "ctx._id = ctx._source.lang + '_' + ctx._source.id;
                      if (ctx._source.lang != \"en\" ) {ctx.op = 'delete'}"
   - match: {created: 1}
diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/20_broken.yaml b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/20_broken.yaml
similarity index 80%
rename from qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/20_broken.yaml
rename to qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/20_broken.yaml
index 6fba2c9bd49..5ec35b4c9dc 100644
--- a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/20_broken.yaml
+++ b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/20_broken.yaml
@@ -19,5 +19,6 @@
           dest:
             index: new_twitter
           script:
+            lang: painless
             inline: syntax errors are fun!
-  - match: {error.reason: 'Failed to compile inline script [syntax errors are fun!] using lang [groovy]'}
+  - match: {error.reason: 'compile error'}
diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/30_timeout.yaml b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/30_timeout.yaml
similarity index 75%
rename from qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/30_timeout.yaml
rename to qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/30_timeout.yaml
index ddd22246717..df514c61ceb 100644
--- a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/30_timeout.yaml
+++ b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/30_timeout.yaml
@@ -1,5 +1,8 @@
 ---
 "Timeout":
+  - skip:
+      version: "all"
+      reason:  painless doesn't support thread.sleep so we need to come up with a better way
   - do:
       index:
         index:  twitter
@@ -19,9 +22,10 @@
             timeout: 10
             query:
               script:
+                lang: painless
                 # Sleep 100x longer than the timeout. That should cause a timeout!
                 # Return true causes the document to try to be collected which is what actually triggers the timeout.
-                script: sleep(1000); return true
+                script: java.lang.Thread.sleep(1000); return true
           dest:
             index: new_twitter
   - is_true: timed_out
diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml
similarity index 62%
rename from qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml
rename to qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml
index c43374e7de2..17e915fdc19 100644
--- a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml
+++ b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/reindex/40_search_failures.yaml
@@ -17,7 +17,8 @@
             index:   source
             query:
               script:
-                script: 1/0 # Divide by 0 to cause a search time exception
+                lang: painless
+                script: throw new IllegalArgumentException("Cats!")
           dest:
             index:   dest
   - match: {created: 0}
@@ -27,8 +28,8 @@
   - is_true: failures.0.shard
   - match: {failures.0.index:  source}
   - is_true: failures.0.node
-  - match: {failures.0.reason.type:   general_script_exception}
-  - match: {failures.0.reason.reason: "failed to run inline script [1/0] using lang [groovy]"}
-  - match: {failures.0.reason.caused_by.type:   arithmetic_exception}
-  - match: {failures.0.reason.caused_by.reason: Division by zero}
+  - match: {failures.0.reason.type:   script_exception}
+  - match: {failures.0.reason.reason: runtime error}
+  - match: {failures.0.reason.caused_by.type:   illegal_argument_exception}
+  - match: {failures.0.reason.caused_by.reason: Cats!}
   - gte: { took: 0 }
diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml
similarity index 92%
rename from qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml
rename to qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml
index e4fef86b1d1..a4580b97334 100644
--- a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml
+++ b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/10_script.yaml
@@ -15,6 +15,7 @@
         refresh: true
         body:
           script:
+            lang: painless
             inline: ctx._source.user = "not" + ctx._source.user
   - match: {updated: 1}
   - match: {noops: 0}
@@ -51,6 +52,7 @@
         index:   twitter
         body:
           script:
+            lang: painless
             inline: if (ctx._source.user == "kimchy") {ctx._source.user = "not" + ctx._source.user} else {ctx.op = "noop"}
   - match: {updated: 1}
   - match: {noops: 1}
@@ -96,6 +98,7 @@
         index:   twitter
         body:
           script:
+            lang: painless
             inline: ctx.op = "noop"
   - match: {updated: 0}
   - match: {noops: 2}
@@ -118,6 +121,7 @@
         index: twitter
         body:
           script:
+            lang: painless
             inline: ctx.junk = "stuff"
 
 ---
@@ -137,6 +141,7 @@
         index: twitter
         body:
           script:
+            lang: painless
             inline: ctx._id = "stuff"
 
 ---
@@ -174,6 +179,7 @@
         index:   twitter
         body:
           script:
+            lang: painless
             inline: if (ctx._source.level != 11) {ctx._source.last_updated = "2016-01-02T00:00:00Z"} else {ctx.op = "delete"}
   - match: {updated: 3}
   - match: {deleted: 1}
@@ -237,17 +243,16 @@
         index:   twitter
         body:
           script:
-            inline: "switch (ctx._source.level % 3) {
-                      case 0:
-                        ctx._source.last_updated = \"2016-01-02T00:00:00Z\";
-                        break;
-                      case 1:
-                        ctx.op = \"noop\";
-                        break;
-                      case 2:
-                        ctx.op = \"delete\";
-                        break;
-                    }"
+            lang: painless
+            inline: |
+              int choice = ctx._source.level % 3;
+              if (choice == 0) {
+                ctx._source.last_updated = "2016-01-02T00:00:00Z";
+              } else if (choice == 1) {
+                ctx.op = "noop";
+              } else {
+                ctx.op = "delete";
+              }
   - match: {updated: 2}
   - match: {deleted: 1}
   - match: {noops: 1}
@@ -303,6 +308,7 @@
         index:   twitter
         body:
           script:
+            lang: painless
             inline: if (ctx._source.user == "kimchy") {ctx.op = "update"} else {ctx.op = "junk"}
 
   - match: { error.reason: 'Operation type [junk] not allowed, only [noop, index, delete] are allowed' }
diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/20_broken.yaml b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/20_broken.yaml
similarity index 78%
rename from qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/20_broken.yaml
rename to qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/20_broken.yaml
index 66f9cfe5456..aa497bff77e 100644
--- a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/20_broken.yaml
+++ b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/20_broken.yaml
@@ -16,5 +16,6 @@
         refresh: true
         body:
           script:
+            lang: painless
             inline: syntax errors are fun!
-  - match: {error.reason: 'Failed to compile inline script [syntax errors are fun!] using lang [groovy]'}
+  - match: {error.reason: 'compile error'}
diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/30_timeout.yaml b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/30_timeout.yaml
similarity index 74%
rename from qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/30_timeout.yaml
rename to qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/30_timeout.yaml
index e6bdd73757b..ac1ed14a1f8 100644
--- a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/30_timeout.yaml
+++ b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/30_timeout.yaml
@@ -1,5 +1,8 @@
 ---
 "Timeout":
+  - skip:
+      version: "all"
+      reason:  painless doesn't support thread.sleep so we need to come up with a better way
   - do:
       index:
         index:  twitter
@@ -18,9 +21,10 @@
         body:
           query:
             script:
+              lang: painless
               # Sleep 100x longer than the timeout. That should cause a timeout!
               # Return true causes the document to try to be collected which is what actually triggers the timeout.
-              script: sleep(1000); return true
+              script: java.lang.Thread.sleep(1000); return true
   - is_true: timed_out
   - match: {updated: 0}
   - match: {noops: 0}
diff --git a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yaml b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yaml
similarity index 58%
rename from qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yaml
rename to qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yaml
index 3aecca73332..d1ba9808fba 100644
--- a/qa/smoke-test-reindex-with-groovy/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yaml
+++ b/qa/smoke-test-reindex-with-painless/src/test/resources/rest-api-spec/test/update_by_query/40_search_failure.yaml
@@ -16,15 +16,16 @@
         body:
           query:
             script:
-              script: 1/0 # Divide by 0 to cause a search time exception
+              lang: painless
+              script: throw new IllegalArgumentException("Cats!")
   - match: {updated: 0}
   - match: {version_conflicts: 0}
   - match: {batches: 0}
   - is_true: failures.0.shard
   - match: {failures.0.index:  source}
   - is_true: failures.0.node
-  - match: {failures.0.reason.type:   general_script_exception}
-  - match: {failures.0.reason.reason: "failed to run inline script [1/0] using lang [groovy]"}
-  - match: {failures.0.reason.caused_by.type:   arithmetic_exception}
-  - match: {failures.0.reason.caused_by.reason: Division by zero}
+  - match: {failures.0.reason.type:   script_exception}
+  - match: {failures.0.reason.reason: runtime error}
+  - match: {failures.0.reason.caused_by.type:   illegal_argument_exception}
+  - match: {failures.0.reason.caused_by.reason: Cats!}
   - gte: { took: 0 }
diff --git a/settings.gradle b/settings.gradle
index 457e89c906f..5aa675eac78 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -46,11 +46,11 @@ List projects = [
   'qa:backwards-5.0',
   'qa:evil-tests',
   'qa:smoke-test-client',
-  'qa:smoke-test-multinode',
-  'qa:smoke-test-reindex-with-groovy',
-  'qa:smoke-test-plugins',
   'qa:smoke-test-ingest-with-all-dependencies',
   'qa:smoke-test-ingest-disabled',
+  'qa:smoke-test-multinode',
+  'qa:smoke-test-plugins',
+  'qa:smoke-test-reindex-with-painless',
   'qa:vagrant',
 ]
 

From 26aacfff722b312327f140e9eca1a087ca9f51bd Mon Sep 17 00:00:00 2001
From: eratio08 <eratio@gmx.net>
Date: Mon, 13 Jun 2016 18:57:01 +0200
Subject: [PATCH 0816/1311] default values for BM25 Similarity (#18778)

assuming elasticsearch uses the lucene default values
---
 docs/reference/index-modules/similarity.asciidoc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/reference/index-modules/similarity.asciidoc b/docs/reference/index-modules/similarity.asciidoc
index 8173b98d505..f1250ddf6c1 100644
--- a/docs/reference/index-modules/similarity.asciidoc
+++ b/docs/reference/index-modules/similarity.asciidoc
@@ -73,10 +73,11 @@ This similarity has the following options:
 [horizontal]
 `k1`::
     Controls non-linear term frequency normalization
-    (saturation).
+    (saturation). The default value is `1.2`.
 
 `b`::
     Controls to what degree document length normalizes tf values.
+    The default value is `0.75`.
 
 `discount_overlaps`::
     Determines whether overlap tokens (Tokens with

From c6e144f669a72791d8db907ac395b76f40b2e21e Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 13 Jun 2016 13:35:11 -0400
Subject: [PATCH 0817/1311] remove stray tab, fix comment

---
 modules/lang-painless/src/main/antlr/PainlessParser.g4          | 2 +-
 .../src/main/java/org/elasticsearch/painless/node/EChain.java   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index 74d400da431..af1f12d3d96 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -204,7 +204,7 @@ funcref
 // reference to a static or instance method, e.g. ArrayList::size or Integer::compare
 classFuncref
     : TYPE REF ID
-	;
+    ;
 
 // reference to a constructor, e.g. ArrayList::new
 // currently limited to simple non-array types
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index 4c5a44d1fac..e7ee7697da7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -48,7 +48,7 @@ public final class EChain extends AExpression {
     Cast there = null;
     Cast back = null;
     
-    /** Creates a new simple EChain */
+    /** Creates a new RHS-only EChain */
     public EChain(Location location, ALink link) {
         this(location, Arrays.asList(link), false, false, null, null);
     }

From 41810bd63cf7bde50a4a1910219f8df32be5b3c2 Mon Sep 17 00:00:00 2001
From: Aaron Mildenstein <aaron@mildensteins.com>
Date: Mon, 13 Jun 2016 12:04:20 -0600
Subject: [PATCH 0818/1311] Pluralize "index" (#18811)

This doesn't just happen to "an index" unless you're restoring just one.  It reads better this way, IMO.
---
 docs/reference/modules/snapshots.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc
index 2d8106e981c..d000805e5e9 100644
--- a/docs/reference/modules/snapshots.asciidoc
+++ b/docs/reference/modules/snapshots.asciidoc
@@ -328,7 +328,7 @@ By default, all indices in the snapshot are restored, and the cluster state is
 as to allow the global cluster state from being restored by using `indices` and
 `include_global_state` options in the restore request body. The list of indices
 supports <<search-multi-index-type,multi index syntax>>. The `rename_pattern`
-and `rename_replacement` options can be also used to rename index on restore
+and `rename_replacement` options can be also used to rename indices on restore
 using regular expression that supports referencing the original text as
 explained
 http://docs.oracle.com/javase/6/docs/api/java/util/regex/Matcher.html#appendReplacement(java.lang.StringBuffer,%20java.lang.String)[here].

From b8309b143fed5dacb1f5a7c1f2c0bb68ca48c1bf Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Mon, 13 Jun 2016 20:14:35 +0200
Subject: [PATCH 0819/1311] percolator: add query extraction support for dismax
 query

---
 .../percolator/ExtractQueryTermsService.java  |  8 +++++++
 .../ExtractQueryTermsServiceTests.java        | 23 +++++++++++++++++++
 2 files changed, 31 insertions(+)

diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java
index f9f1aa575b7..c451a245754 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java
@@ -34,6 +34,7 @@ import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
@@ -192,6 +193,13 @@ public final class ExtractQueryTermsService {
         } else if (query instanceof BlendedTermQuery) {
             List<Term> terms = ((BlendedTermQuery) query).getTerms();
             return new HashSet<>(terms);
+        } else if (query instanceof DisjunctionMaxQuery) {
+            List<Query> disjuncts = ((DisjunctionMaxQuery) query).getDisjuncts();
+            Set<Term> terms = new HashSet<>();
+            for (Query disjunct : disjuncts) {
+                terms.addAll(extractQueryTerms(disjunct));
+            }
+            return terms;
         } else if (query instanceof SpanTermQuery) {
             return Collections.singleton(((SpanTermQuery) query).getTerm());
         } else if (query instanceof SpanNearQuery) {
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java
index 444e47d90ce..b9430a32425 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java
@@ -32,6 +32,7 @@ import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TermRangeQuery;
@@ -362,6 +363,28 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         assertThat(e.getUnsupportedQuery(), sameInstance(unsupportedQuery));
     }
 
+    public void testExtractQueryMetadata_disjunctionMaxQuery() {
+        TermQuery termQuery1 = new TermQuery(new Term("_field", "_term1"));
+        TermQuery termQuery2 = new TermQuery(new Term("_field", "_term2"));
+        TermQuery termQuery3 = new TermQuery(new Term("_field", "_term3"));
+        TermQuery termQuery4 = new TermQuery(new Term("_field", "_term4"));
+        DisjunctionMaxQuery disjunctionMaxQuery = new DisjunctionMaxQuery(
+                Arrays.asList(termQuery1, termQuery2, termQuery3, termQuery4), 0.1f
+        );
+
+        List<Term> terms = new ArrayList<>(extractQueryTerms(disjunctionMaxQuery));
+        Collections.sort(terms);
+        assertThat(terms.size(), equalTo(4));
+        assertThat(terms.get(0).field(), equalTo(termQuery1.getTerm().field()));
+        assertThat(terms.get(0).bytes(), equalTo(termQuery1.getTerm().bytes()));
+        assertThat(terms.get(1).field(), equalTo(termQuery2.getTerm().field()));
+        assertThat(terms.get(1).bytes(), equalTo(termQuery2.getTerm().bytes()));
+        assertThat(terms.get(2).field(), equalTo(termQuery3.getTerm().field()));
+        assertThat(terms.get(2).bytes(), equalTo(termQuery3.getTerm().bytes()));
+        assertThat(terms.get(3).field(), equalTo(termQuery4.getTerm().field()));
+        assertThat(terms.get(3).bytes(), equalTo(termQuery4.getTerm().bytes()));
+    }
+
     public void testCreateQueryMetadataQuery() throws Exception {
         MemoryIndex memoryIndex = new MemoryIndex(false);
         memoryIndex.addField("field1", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer());

From 6617b5342261993d0cf8ae63d237d8e176e49c0a Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Sun, 12 Jun 2016 23:07:13 -0400
Subject: [PATCH 0820/1311] Painless: Add support for /regex/

Adds `/regex/` as a regex constructor. A couple of fun points:
1. This makes generic the idea of arbitrary stuff adding a constant.
Both SFunction and LRegex create a statically initialized constant.
Both go through Locals to do this because they LRegex isn't directly
iterable from SScript.
2. Differentiating `/` as-in-division from `/` as-in-start-of-regex
is hard. See:
http://www-archive.mozilla.org/js/language/js20-2002-04/rationale/syntax.html#regular-expressions
The javascript folks have a way, way tougher time of it then we do
because they have semicolon insertion. We have the much simpler
delimiter rules. Even with our simpler life we still have to add
a hack to get lexing `/regex/` to work properly. I chose to add
token-level lookbehind because it seems to be a pretty contained hack.
I considered and rejected lexer modes, a lexer member variable,
having the parser set variables on the lexer (this is a fairly common
solution for js, I believe), and moving regex parsing to the parser
level.
3. I've only added a very small subset of java.util.regex to the
whitelist because it is the subset I needed to test LRegex sanely.
More deserves to be added, and maybe more regex syntax like `=~` and
`==~`. Those can probably be added without too much pain.
---
 .../src/main/antlr/PainlessLexer.g4           |   3 +-
 .../src/main/antlr/PainlessLexer.tokens       |  21 +-
 .../src/main/antlr/PainlessParser.g4          |   1 +
 .../src/main/antlr/PainlessParser.tokens      |  21 +-
 .../elasticsearch/painless/Definition.java    |   1 +
 .../org/elasticsearch/painless/Locals.java    |  58 +++
 .../painless/WriterConstants.java             |   9 +
 .../painless/antlr/ErrorHandlingLexer.java    |   2 +
 .../painless/antlr/PainlessLexer.java         | 386 ++++++++++--------
 .../painless/antlr/PainlessParser.java        | 288 +++++++------
 .../antlr/PainlessParserBaseVisitor.java      |   7 +
 .../painless/antlr/PainlessParserVisitor.java |   7 +
 .../painless/antlr/SlashStrategy.java         |  50 +++
 .../painless/antlr/StashingTokenFactory.java  |  62 +++
 .../elasticsearch/painless/antlr/Walker.java  |  11 +
 .../elasticsearch/painless/node/LRegex.java   |  89 ++++
 .../painless/node/SFunction.java              |  18 +-
 .../elasticsearch/painless/node/SSource.java  |  38 +-
 .../painless/java.util.regex.txt              |  33 ++
 .../elasticsearch/painless/ArrayTests.java    |   6 +
 .../elasticsearch/painless/RegexTests.java    |  71 ++++
 .../painless/WhenThingsGoWrongTests.java      |  16 +
 22 files changed, 838 insertions(+), 360 deletions(-)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SlashStrategy.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/StashingTokenFactory.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LRegex.java
 create mode 100644 modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt
 create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java

diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.g4 b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
index 8a5c2a5d061..fac36030120 100644
--- a/modules/lang-painless/src/main/antlr/PainlessLexer.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
@@ -56,7 +56,7 @@ THIS:      'this';
 BOOLNOT: '!';
 BWNOT:   '~';
 MUL:     '*';
-DIV:     '/';
+DIV:     '/' { false == SlashStrategy.slashIsRegex(_factory) }?;
 REM:     '%';
 ADD:     '+';
 SUB:     '-';
@@ -102,6 +102,7 @@ INTEGER: ( '0' | [1-9] [0-9]* ) [lLfFdD]?;
 DECIMAL: ( '0' | [1-9] [0-9]* ) (DOT [0-9]+)? ( [eE] [+\-]? [0-9]+ )? [fF]?;
 
 STRING: ( '"' ( '\\"' | '\\\\' | ~[\\"] )*? '"' ) | ( '\'' ( '\\\'' | '\\\\' | ~[\\"] )*? '\'' );
+REGEX: '/' ( ~('/' | '\n') | '\\' ~'\n' )+ '/' { SlashStrategy.slashIsRegex(_factory) }?;
 
 TRUE:  'true';
 FALSE: 'false';
diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.tokens b/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
index d81d2b48c08..b6df2d97a51 100644
--- a/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
@@ -68,13 +68,14 @@ HEX=67
 INTEGER=68
 DECIMAL=69
 STRING=70
-TRUE=71
-FALSE=72
-NULL=73
-TYPE=74
-ID=75
-DOTINTEGER=76
-DOTID=77
+REGEX=71
+TRUE=72
+FALSE=73
+NULL=74
+TYPE=75
+ID=76
+DOTINTEGER=77
+DOTID=78
 '{'=3
 '}'=4
 '['=5
@@ -138,6 +139,6 @@ DOTID=77
 '<<='=63
 '>>='=64
 '>>>='=65
-'true'=71
-'false'=72
-'null'=73
+'true'=72
+'false'=73
+'null'=74
diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index af1f12d3d96..cc2ca89047f 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -157,6 +157,7 @@ primary[boolean c] returns [boolean s = true]
     : { !$c }? LP e = expression RP { $s = $e.s; } # exprprec
     | { $c }?  LP unary[true] RP                   # chainprec
     |          STRING                              # string
+    |          REGEX                               # regex
     |          ID                                  # variable
     |          ID arguments                        # calllocal
     |          NEW TYPE arguments                  # newobject
diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.tokens b/modules/lang-painless/src/main/antlr/PainlessParser.tokens
index d81d2b48c08..b6df2d97a51 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.tokens
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.tokens
@@ -68,13 +68,14 @@ HEX=67
 INTEGER=68
 DECIMAL=69
 STRING=70
-TRUE=71
-FALSE=72
-NULL=73
-TYPE=74
-ID=75
-DOTINTEGER=76
-DOTID=77
+REGEX=71
+TRUE=72
+FALSE=73
+NULL=74
+TYPE=75
+ID=76
+DOTINTEGER=77
+DOTID=78
 '{'=3
 '}'=4
 '['=5
@@ -138,6 +139,6 @@ DOTID=77
 '<<='=63
 '>>='=64
 '>>>='=65
-'true'=71
-'false'=72
-'null'=73
+'true'=72
+'false'=73
+'null'=74
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 5a66b2154be..7188791d971 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -59,6 +59,7 @@ public final class Definition {
                       "java.time.zone.txt",
                       "java.util.txt",
                       "java.util.function.txt",
+                      "java.util.regex.txt",
                       "java.util.stream.txt",
                       "joda.time.txt"));
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
index d102114bdeb..30f15c42dde 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
@@ -24,11 +24,14 @@ import org.elasticsearch.painless.Definition.MethodKey;
 import org.elasticsearch.painless.Definition.Type;
 
 import java.util.ArrayDeque;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Deque;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Consumer;
 
 /**
  * Tracks user defined methods and variables across compilation phases.
@@ -140,6 +143,20 @@ public final class Locals {
         }
     }
 
+    public static final class Constant {
+        public final Location location;
+        public final String name;
+        public final org.objectweb.asm.Type type;
+        public final Consumer<MethodWriter> initializer;
+
+        private Constant(Location location, String name, org.objectweb.asm.Type type, Consumer<MethodWriter> initializer) {
+            this.location = location;
+            this.name = name;
+            this.type = type;
+            this.initializer = initializer;
+        }
+    }
+
     public static final class Parameter {
         public final Location location;
         public final String name;
@@ -154,6 +171,7 @@ public final class Locals {
 
     private final Reserved reserved;
     private final Map<MethodKey, Method> methods;
+    private final Map<String, Constant> constants;
     private final Type rtnType;
 
     // TODO: this datastructure runs in linear time for nearly all operations. use linkedhashset instead?
@@ -163,6 +181,7 @@ public final class Locals {
     public Locals(ExecuteReserved reserved, Map<MethodKey, Method> methods) {
         this.reserved = reserved;
         this.methods = Collections.unmodifiableMap(methods);
+        this.constants = new HashMap<>();
         this.rtnType = Definition.OBJECT_TYPE;
 
         incrementScope();
@@ -205,6 +224,7 @@ public final class Locals {
     public Locals(FunctionReserved reserved, Locals locals, Type rtnType, List<Parameter> parameters) {
         this.reserved = reserved;
         this.methods = locals.methods;
+        this.constants = locals.constants;
         this.rtnType = rtnType;
 
         incrementScope();
@@ -302,5 +322,43 @@ public final class Locals {
 
         return variable;
     }
+
+    /**
+     * Create a new constant.
+     *
+     * @param location the location in the script that is creating it
+     * @param type the type of the constant
+     * @param name the name of the constant
+     * @param initializer code to initialize the constant. It will be called when generating the clinit method and is expected to leave the
+     *        value of the constant on the stack. Generating the load instruction is managed by the caller.
+     * @return the constant
+     */
+    public Constant addConstant(Location location, org.objectweb.asm.Type type, String name, Consumer<MethodWriter> initializer) {
+        if (constants.containsKey(name)) {
+            throw location.createError(new IllegalArgumentException("Constant [" + name + "] is already defined."));
+        }
+
+        Constant constant = new Constant(location, name, type, initializer);
+        constants.put(name, constant);
+        return constant;
+    }
+
+    /**
+     * Create a new constant.
+     *
+     * @param location the location in the script that is creating it
+     * @param type the type of the constant
+     * @param name the name of the constant
+     * @param initializer code to initialize the constant. It will be called when generating the clinit method and is expected to leave the
+     *        value of the constant on the stack. Generating the load instruction is managed by the caller.
+     * @return the constant
+     */
+    public Constant addConstant(Location location, Type type, String name, Consumer<MethodWriter> initializer) {
+        return addConstant(location, type.type, name, initializer);
+    }
+
+    public Collection<Constant> getConstants() {
+        return constants.values();
+    }
 }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index 038096da99e..521287a02d7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -34,6 +34,7 @@ import java.lang.invoke.MethodType;
 import java.util.BitSet;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.regex.Pattern;
 
 /**
  * General pool of constants used during the writing phase of compilation.
@@ -70,6 +71,14 @@ public final class WriterConstants {
     
     public final static Type METHOD_HANDLE_TYPE = Type.getType(MethodHandle.class);
 
+    /**
+     * A Method instance for {@linkplain Pattern#compile}. This isn't looked up from Definition because we intentionally don't add it there
+     * so that the script can't create regexes without this syntax. Essentially, our static regex syntax has a monopoly on building regexes
+     * because it can do it statically. This is both faster and prevents the script from doing something super slow like building a regex
+     * per time it is run.
+     */
+    public final static Method PATTERN_COMPILE = getAsmMethod(Pattern.class, "compile", String.class);
+
     /** dynamic callsite bootstrap signature */
     public final static MethodType DEF_BOOTSTRAP_TYPE =
         MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, int.class, Object[].class);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ErrorHandlingLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ErrorHandlingLexer.java
index 99453b648a8..d490426c59c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ErrorHandlingLexer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/ErrorHandlingLexer.java
@@ -33,6 +33,8 @@ final class ErrorHandlingLexer extends PainlessLexer {
     ErrorHandlingLexer(CharStream charStream, String sourceName) {
         super(charStream);
         this.sourceName = sourceName;
+        // Replace the TokenFactory with a stashing wrapper so we can do token-level lookbehind for regex detection
+        _factory = new StashingTokenFactory<>(_factory);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
index 92f8e9a8f1d..143d2a1ef3c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
@@ -28,8 +28,8 @@ class PainlessLexer extends Lexer {
     XOR=44, BWOR=45, BOOLAND=46, BOOLOR=47, COND=48, COLON=49, REF=50, ARROW=51, 
     INCR=52, DECR=53, ASSIGN=54, AADD=55, ASUB=56, AMUL=57, ADIV=58, AREM=59, 
     AAND=60, AXOR=61, AOR=62, ALSH=63, ARSH=64, AUSH=65, OCTAL=66, HEX=67, 
-    INTEGER=68, DECIMAL=69, STRING=70, TRUE=71, FALSE=72, NULL=73, TYPE=74, 
-    ID=75, DOTINTEGER=76, DOTID=77;
+    INTEGER=68, DECIMAL=69, STRING=70, REGEX=71, TRUE=72, FALSE=73, NULL=74, 
+    TYPE=75, ID=76, DOTINTEGER=77, DOTID=78;
   public static final int AFTER_DOT = 1;
   public static String[] modeNames = {
     "DEFAULT_MODE", "AFTER_DOT"
@@ -44,7 +44,7 @@ class PainlessLexer extends Lexer {
     "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "INCR", "DECR", 
     "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", "AAND", "AXOR", "AOR", 
     "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", 
-    "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", "DOTID"
+    "REGEX", "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", "DOTID"
   };
 
   private static final String[] _LITERAL_NAMES = {
@@ -55,7 +55,7 @@ class PainlessLexer extends Lexer {
     "'>'", "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", "'^'", "'|'", 
     "'&&'", "'||'", "'?'", "':'", "'::'", "'->'", "'++'", "'--'", "'='", "'+='", 
     "'-='", "'*='", "'/='", "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", 
-    "'>>>='", null, null, null, null, null, "'true'", "'false'", "'null'"
+    "'>>>='", null, null, null, null, null, null, "'true'", "'false'", "'null'"
   };
   private static final String[] _SYMBOLIC_NAMES = {
     null, "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", 
@@ -66,7 +66,7 @@ class PainlessLexer extends Lexer {
     "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "INCR", "DECR", 
     "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", "AAND", "AXOR", "AOR", 
     "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", 
-    "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", "DOTID"
+    "REGEX", "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", "DOTID"
   };
   public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
 
@@ -125,21 +125,39 @@ class PainlessLexer extends Lexer {
   @Override
   public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) {
     switch (ruleIndex) {
-    case 73:
+    case 27:
+      return DIV_sempred((RuleContext)_localctx, predIndex);
+    case 70:
+      return REGEX_sempred((RuleContext)_localctx, predIndex);
+    case 74:
       return TYPE_sempred((RuleContext)_localctx, predIndex);
     }
     return true;
   }
+  private boolean DIV_sempred(RuleContext _localctx, int predIndex) {
+    switch (predIndex) {
+    case 0:
+      return  false == SlashStrategy.slashIsRegex(_factory) ;
+    }
+    return true;
+  }
+  private boolean REGEX_sempred(RuleContext _localctx, int predIndex) {
+    switch (predIndex) {
+    case 1:
+      return  SlashStrategy.slashIsRegex(_factory) ;
+    }
+    return true;
+  }
   private boolean TYPE_sempred(RuleContext _localctx, int predIndex) {
     switch (predIndex) {
-    case 0:
+    case 2:
       return  Definition.isSimpleType(getText()) ;
     }
     return true;
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2O\u021a\b\1\b\1\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2P\u0228\b\1\b\1\4"+
     "\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n"+
     "\4\13\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22"+
     "\t\22\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31"+
@@ -148,184 +166,190 @@ class PainlessLexer extends Lexer {
     "+\4,\t,\4-\t-\4.\t.\4/\t/\4\60\t\60\4\61\t\61\4\62\t\62\4\63\t\63\4\64"+
     "\t\64\4\65\t\65\4\66\t\66\4\67\t\67\48\t8\49\t9\4:\t:\4;\t;\4<\t<\4=\t"+
     "=\4>\t>\4?\t?\4@\t@\4A\tA\4B\tB\4C\tC\4D\tD\4E\tE\4F\tF\4G\tG\4H\tH\4"+
-    "I\tI\4J\tJ\4K\tK\4L\tL\4M\tM\4N\tN\3\2\6\2\u00a0\n\2\r\2\16\2\u00a1\3"+
-    "\2\3\2\3\3\3\3\3\3\3\3\7\3\u00aa\n\3\f\3\16\3\u00ad\13\3\3\3\3\3\3\3\3"+
-    "\3\3\3\7\3\u00b4\n\3\f\3\16\3\u00b7\13\3\3\3\3\3\5\3\u00bb\n\3\3\3\3\3"+
-    "\3\4\3\4\3\5\3\5\3\6\3\6\3\7\3\7\3\b\3\b\3\t\3\t\3\n\3\n\3\n\3\n\3\13"+
+    "I\tI\4J\tJ\4K\tK\4L\tL\4M\tM\4N\tN\4O\tO\3\2\6\2\u00a2\n\2\r\2\16\2\u00a3"+
+    "\3\2\3\2\3\3\3\3\3\3\3\3\7\3\u00ac\n\3\f\3\16\3\u00af\13\3\3\3\3\3\3\3"+
+    "\3\3\3\3\7\3\u00b6\n\3\f\3\16\3\u00b9\13\3\3\3\3\3\5\3\u00bd\n\3\3\3\3"+
+    "\3\3\4\3\4\3\5\3\5\3\6\3\6\3\7\3\7\3\b\3\b\3\t\3\t\3\n\3\n\3\n\3\n\3\13"+
     "\3\13\3\f\3\f\3\r\3\r\3\r\3\16\3\16\3\16\3\16\3\16\3\17\3\17\3\17\3\17"+
     "\3\17\3\17\3\20\3\20\3\20\3\21\3\21\3\21\3\21\3\22\3\22\3\22\3\22\3\22"+
     "\3\22\3\22\3\22\3\22\3\23\3\23\3\23\3\23\3\23\3\23\3\24\3\24\3\24\3\24"+
     "\3\24\3\24\3\24\3\25\3\25\3\25\3\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27"+
     "\3\27\3\27\3\27\3\30\3\30\3\30\3\30\3\30\3\30\3\31\3\31\3\31\3\31\3\31"+
-    "\3\32\3\32\3\33\3\33\3\34\3\34\3\35\3\35\3\36\3\36\3\37\3\37\3 \3 \3!"+
-    "\3!\3!\3\"\3\"\3\"\3#\3#\3#\3#\3$\3$\3%\3%\3%\3&\3&\3\'\3\'\3\'\3(\3("+
-    "\3(\3)\3)\3)\3)\3*\3*\3*\3+\3+\3+\3+\3,\3,\3-\3-\3.\3.\3/\3/\3/\3\60\3"+
-    "\60\3\60\3\61\3\61\3\62\3\62\3\63\3\63\3\63\3\64\3\64\3\64\3\65\3\65\3"+
-    "\65\3\66\3\66\3\66\3\67\3\67\38\38\38\39\39\39\3:\3:\3:\3;\3;\3;\3<\3"+
-    "<\3<\3=\3=\3=\3>\3>\3>\3?\3?\3?\3@\3@\3@\3@\3A\3A\3A\3A\3B\3B\3B\3B\3"+
-    "B\3C\3C\6C\u018c\nC\rC\16C\u018d\3C\5C\u0191\nC\3D\3D\3D\6D\u0196\nD\r"+
-    "D\16D\u0197\3D\5D\u019b\nD\3E\3E\3E\7E\u01a0\nE\fE\16E\u01a3\13E\5E\u01a5"+
-    "\nE\3E\5E\u01a8\nE\3F\3F\3F\7F\u01ad\nF\fF\16F\u01b0\13F\5F\u01b2\nF\3"+
-    "F\3F\6F\u01b6\nF\rF\16F\u01b7\5F\u01ba\nF\3F\3F\5F\u01be\nF\3F\6F\u01c1"+
-    "\nF\rF\16F\u01c2\5F\u01c5\nF\3F\5F\u01c8\nF\3G\3G\3G\3G\3G\3G\7G\u01d0"+
-    "\nG\fG\16G\u01d3\13G\3G\3G\3G\3G\3G\3G\3G\7G\u01dc\nG\fG\16G\u01df\13"+
-    "G\3G\5G\u01e2\nG\3H\3H\3H\3H\3H\3I\3I\3I\3I\3I\3I\3J\3J\3J\3J\3J\3K\3"+
-    "K\3K\3K\7K\u01f8\nK\fK\16K\u01fb\13K\3K\3K\3L\3L\7L\u0201\nL\fL\16L\u0204"+
-    "\13L\3M\3M\3M\7M\u0209\nM\fM\16M\u020c\13M\5M\u020e\nM\3M\3M\3N\3N\7N"+
-    "\u0214\nN\fN\16N\u0217\13N\3N\3N\6\u00ab\u00b5\u01d1\u01dd\2O\4\3\6\4"+
-    "\b\5\n\6\f\7\16\b\20\t\22\n\24\13\26\f\30\r\32\16\34\17\36\20 \21\"\22"+
-    "$\23&\24(\25*\26,\27.\30\60\31\62\32\64\33\66\348\35:\36<\37> @!B\"D#"+
-    "F$H%J&L\'N(P)R*T+V,X-Z.\\/^\60`\61b\62d\63f\64h\65j\66l\67n8p9r:t;v<x"+
-    "=z>|?~@\u0080A\u0082B\u0084C\u0086D\u0088E\u008aF\u008cG\u008eH\u0090"+
-    "I\u0092J\u0094K\u0096L\u0098M\u009aN\u009cO\4\2\3\21\5\2\13\f\17\17\""+
-    "\"\4\2\f\f\17\17\3\2\629\4\2NNnn\4\2ZZzz\5\2\62;CHch\3\2\63;\3\2\62;\b"+
-    "\2FFHHNNffhhnn\4\2GGgg\4\2--//\4\2HHhh\4\2$$^^\5\2C\\aac|\6\2\62;C\\a"+
-    "ac|\u0237\2\4\3\2\2\2\2\6\3\2\2\2\2\b\3\2\2\2\2\n\3\2\2\2\2\f\3\2\2\2"+
-    "\2\16\3\2\2\2\2\20\3\2\2\2\2\22\3\2\2\2\2\24\3\2\2\2\2\26\3\2\2\2\2\30"+
-    "\3\2\2\2\2\32\3\2\2\2\2\34\3\2\2\2\2\36\3\2\2\2\2 \3\2\2\2\2\"\3\2\2\2"+
-    "\2$\3\2\2\2\2&\3\2\2\2\2(\3\2\2\2\2*\3\2\2\2\2,\3\2\2\2\2.\3\2\2\2\2\60"+
-    "\3\2\2\2\2\62\3\2\2\2\2\64\3\2\2\2\2\66\3\2\2\2\28\3\2\2\2\2:\3\2\2\2"+
-    "\2<\3\2\2\2\2>\3\2\2\2\2@\3\2\2\2\2B\3\2\2\2\2D\3\2\2\2\2F\3\2\2\2\2H"+
-    "\3\2\2\2\2J\3\2\2\2\2L\3\2\2\2\2N\3\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2T\3\2"+
-    "\2\2\2V\3\2\2\2\2X\3\2\2\2\2Z\3\2\2\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3\2\2"+
-    "\2\2b\3\2\2\2\2d\3\2\2\2\2f\3\2\2\2\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2\2\2"+
-    "n\3\2\2\2\2p\3\2\2\2\2r\3\2\2\2\2t\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2z\3"+
-    "\2\2\2\2|\3\2\2\2\2~\3\2\2\2\2\u0080\3\2\2\2\2\u0082\3\2\2\2\2\u0084\3"+
-    "\2\2\2\2\u0086\3\2\2\2\2\u0088\3\2\2\2\2\u008a\3\2\2\2\2\u008c\3\2\2\2"+
-    "\2\u008e\3\2\2\2\2\u0090\3\2\2\2\2\u0092\3\2\2\2\2\u0094\3\2\2\2\2\u0096"+
-    "\3\2\2\2\2\u0098\3\2\2\2\3\u009a\3\2\2\2\3\u009c\3\2\2\2\4\u009f\3\2\2"+
-    "\2\6\u00ba\3\2\2\2\b\u00be\3\2\2\2\n\u00c0\3\2\2\2\f\u00c2\3\2\2\2\16"+
-    "\u00c4\3\2\2\2\20\u00c6\3\2\2\2\22\u00c8\3\2\2\2\24\u00ca\3\2\2\2\26\u00ce"+
-    "\3\2\2\2\30\u00d0\3\2\2\2\32\u00d2\3\2\2\2\34\u00d5\3\2\2\2\36\u00da\3"+
-    "\2\2\2 \u00e0\3\2\2\2\"\u00e3\3\2\2\2$\u00e7\3\2\2\2&\u00f0\3\2\2\2(\u00f6"+
-    "\3\2\2\2*\u00fd\3\2\2\2,\u0101\3\2\2\2.\u0105\3\2\2\2\60\u010b\3\2\2\2"+
-    "\62\u0111\3\2\2\2\64\u0116\3\2\2\2\66\u0118\3\2\2\28\u011a\3\2\2\2:\u011c"+
-    "\3\2\2\2<\u011e\3\2\2\2>\u0120\3\2\2\2@\u0122\3\2\2\2B\u0124\3\2\2\2D"+
-    "\u0127\3\2\2\2F\u012a\3\2\2\2H\u012e\3\2\2\2J\u0130\3\2\2\2L\u0133\3\2"+
-    "\2\2N\u0135\3\2\2\2P\u0138\3\2\2\2R\u013b\3\2\2\2T\u013f\3\2\2\2V\u0142"+
-    "\3\2\2\2X\u0146\3\2\2\2Z\u0148\3\2\2\2\\\u014a\3\2\2\2^\u014c\3\2\2\2"+
-    "`\u014f\3\2\2\2b\u0152\3\2\2\2d\u0154\3\2\2\2f\u0156\3\2\2\2h\u0159\3"+
-    "\2\2\2j\u015c\3\2\2\2l\u015f\3\2\2\2n\u0162\3\2\2\2p\u0164\3\2\2\2r\u0167"+
-    "\3\2\2\2t\u016a\3\2\2\2v\u016d\3\2\2\2x\u0170\3\2\2\2z\u0173\3\2\2\2|"+
-    "\u0176\3\2\2\2~\u0179\3\2\2\2\u0080\u017c\3\2\2\2\u0082\u0180\3\2\2\2"+
-    "\u0084\u0184\3\2\2\2\u0086\u0189\3\2\2\2\u0088\u0192\3\2\2\2\u008a\u01a4"+
-    "\3\2\2\2\u008c\u01b1\3\2\2\2\u008e\u01e1\3\2\2\2\u0090\u01e3\3\2\2\2\u0092"+
-    "\u01e8\3\2\2\2\u0094\u01ee\3\2\2\2\u0096\u01f3\3\2\2\2\u0098\u01fe\3\2"+
-    "\2\2\u009a\u020d\3\2\2\2\u009c\u0211\3\2\2\2\u009e\u00a0\t\2\2\2\u009f"+
-    "\u009e\3\2\2\2\u00a0\u00a1\3\2\2\2\u00a1\u009f\3\2\2\2\u00a1\u00a2\3\2"+
-    "\2\2\u00a2\u00a3\3\2\2\2\u00a3\u00a4\b\2\2\2\u00a4\5\3\2\2\2\u00a5\u00a6"+
-    "\7\61\2\2\u00a6\u00a7\7\61\2\2\u00a7\u00ab\3\2\2\2\u00a8\u00aa\13\2\2"+
-    "\2\u00a9\u00a8\3\2\2\2\u00aa\u00ad\3\2\2\2\u00ab\u00ac\3\2\2\2\u00ab\u00a9"+
-    "\3\2\2\2\u00ac\u00ae\3\2\2\2\u00ad\u00ab\3\2\2\2\u00ae\u00bb\t\3\2\2\u00af"+
-    "\u00b0\7\61\2\2\u00b0\u00b1\7,\2\2\u00b1\u00b5\3\2\2\2\u00b2\u00b4\13"+
-    "\2\2\2\u00b3\u00b2\3\2\2\2\u00b4\u00b7\3\2\2\2\u00b5\u00b6\3\2\2\2\u00b5"+
-    "\u00b3\3\2\2\2\u00b6\u00b8\3\2\2\2\u00b7\u00b5\3\2\2\2\u00b8\u00b9\7,"+
-    "\2\2\u00b9\u00bb\7\61\2\2\u00ba\u00a5\3\2\2\2\u00ba\u00af\3\2\2\2\u00bb"+
-    "\u00bc\3\2\2\2\u00bc\u00bd\b\3\2\2\u00bd\7\3\2\2\2\u00be\u00bf\7}\2\2"+
-    "\u00bf\t\3\2\2\2\u00c0\u00c1\7\177\2\2\u00c1\13\3\2\2\2\u00c2\u00c3\7"+
-    "]\2\2\u00c3\r\3\2\2\2\u00c4\u00c5\7_\2\2\u00c5\17\3\2\2\2\u00c6\u00c7"+
-    "\7*\2\2\u00c7\21\3\2\2\2\u00c8\u00c9\7+\2\2\u00c9\23\3\2\2\2\u00ca\u00cb"+
-    "\7\60\2\2\u00cb\u00cc\3\2\2\2\u00cc\u00cd\b\n\3\2\u00cd\25\3\2\2\2\u00ce"+
-    "\u00cf\7.\2\2\u00cf\27\3\2\2\2\u00d0\u00d1\7=\2\2\u00d1\31\3\2\2\2\u00d2"+
-    "\u00d3\7k\2\2\u00d3\u00d4\7h\2\2\u00d4\33\3\2\2\2\u00d5\u00d6\7g\2\2\u00d6"+
-    "\u00d7\7n\2\2\u00d7\u00d8\7u\2\2\u00d8\u00d9\7g\2\2\u00d9\35\3\2\2\2\u00da"+
-    "\u00db\7y\2\2\u00db\u00dc\7j\2\2\u00dc\u00dd\7k\2\2\u00dd\u00de\7n\2\2"+
-    "\u00de\u00df\7g\2\2\u00df\37\3\2\2\2\u00e0\u00e1\7f\2\2\u00e1\u00e2\7"+
-    "q\2\2\u00e2!\3\2\2\2\u00e3\u00e4\7h\2\2\u00e4\u00e5\7q\2\2\u00e5\u00e6"+
-    "\7t\2\2\u00e6#\3\2\2\2\u00e7\u00e8\7e\2\2\u00e8\u00e9\7q\2\2\u00e9\u00ea"+
-    "\7p\2\2\u00ea\u00eb\7v\2\2\u00eb\u00ec\7k\2\2\u00ec\u00ed\7p\2\2\u00ed"+
-    "\u00ee\7w\2\2\u00ee\u00ef\7g\2\2\u00ef%\3\2\2\2\u00f0\u00f1\7d\2\2\u00f1"+
-    "\u00f2\7t\2\2\u00f2\u00f3\7g\2\2\u00f3\u00f4\7c\2\2\u00f4\u00f5\7m\2\2"+
-    "\u00f5\'\3\2\2\2\u00f6\u00f7\7t\2\2\u00f7\u00f8\7g\2\2\u00f8\u00f9\7v"+
-    "\2\2\u00f9\u00fa\7w\2\2\u00fa\u00fb\7t\2\2\u00fb\u00fc\7p\2\2\u00fc)\3"+
-    "\2\2\2\u00fd\u00fe\7p\2\2\u00fe\u00ff\7g\2\2\u00ff\u0100\7y\2\2\u0100"+
-    "+\3\2\2\2\u0101\u0102\7v\2\2\u0102\u0103\7t\2\2\u0103\u0104\7{\2\2\u0104"+
-    "-\3\2\2\2\u0105\u0106\7e\2\2\u0106\u0107\7c\2\2\u0107\u0108\7v\2\2\u0108"+
-    "\u0109\7e\2\2\u0109\u010a\7j\2\2\u010a/\3\2\2\2\u010b\u010c\7v\2\2\u010c"+
-    "\u010d\7j\2\2\u010d\u010e\7t\2\2\u010e\u010f\7q\2\2\u010f\u0110\7y\2\2"+
-    "\u0110\61\3\2\2\2\u0111\u0112\7v\2\2\u0112\u0113\7j\2\2\u0113\u0114\7"+
-    "k\2\2\u0114\u0115\7u\2\2\u0115\63\3\2\2\2\u0116\u0117\7#\2\2\u0117\65"+
-    "\3\2\2\2\u0118\u0119\7\u0080\2\2\u0119\67\3\2\2\2\u011a\u011b\7,\2\2\u011b"+
-    "9\3\2\2\2\u011c\u011d\7\61\2\2\u011d;\3\2\2\2\u011e\u011f\7\'\2\2\u011f"+
-    "=\3\2\2\2\u0120\u0121\7-\2\2\u0121?\3\2\2\2\u0122\u0123\7/\2\2\u0123A"+
-    "\3\2\2\2\u0124\u0125\7>\2\2\u0125\u0126\7>\2\2\u0126C\3\2\2\2\u0127\u0128"+
-    "\7@\2\2\u0128\u0129\7@\2\2\u0129E\3\2\2\2\u012a\u012b\7@\2\2\u012b\u012c"+
-    "\7@\2\2\u012c\u012d\7@\2\2\u012dG\3\2\2\2\u012e\u012f\7>\2\2\u012fI\3"+
-    "\2\2\2\u0130\u0131\7>\2\2\u0131\u0132\7?\2\2\u0132K\3\2\2\2\u0133\u0134"+
-    "\7@\2\2\u0134M\3\2\2\2\u0135\u0136\7@\2\2\u0136\u0137\7?\2\2\u0137O\3"+
-    "\2\2\2\u0138\u0139\7?\2\2\u0139\u013a\7?\2\2\u013aQ\3\2\2\2\u013b\u013c"+
-    "\7?\2\2\u013c\u013d\7?\2\2\u013d\u013e\7?\2\2\u013eS\3\2\2\2\u013f\u0140"+
-    "\7#\2\2\u0140\u0141\7?\2\2\u0141U\3\2\2\2\u0142\u0143\7#\2\2\u0143\u0144"+
-    "\7?\2\2\u0144\u0145\7?\2\2\u0145W\3\2\2\2\u0146\u0147\7(\2\2\u0147Y\3"+
-    "\2\2\2\u0148\u0149\7`\2\2\u0149[\3\2\2\2\u014a\u014b\7~\2\2\u014b]\3\2"+
-    "\2\2\u014c\u014d\7(\2\2\u014d\u014e\7(\2\2\u014e_\3\2\2\2\u014f\u0150"+
-    "\7~\2\2\u0150\u0151\7~\2\2\u0151a\3\2\2\2\u0152\u0153\7A\2\2\u0153c\3"+
-    "\2\2\2\u0154\u0155\7<\2\2\u0155e\3\2\2\2\u0156\u0157\7<\2\2\u0157\u0158"+
-    "\7<\2\2\u0158g\3\2\2\2\u0159\u015a\7/\2\2\u015a\u015b\7@\2\2\u015bi\3"+
-    "\2\2\2\u015c\u015d\7-\2\2\u015d\u015e\7-\2\2\u015ek\3\2\2\2\u015f\u0160"+
-    "\7/\2\2\u0160\u0161\7/\2\2\u0161m\3\2\2\2\u0162\u0163\7?\2\2\u0163o\3"+
-    "\2\2\2\u0164\u0165\7-\2\2\u0165\u0166\7?\2\2\u0166q\3\2\2\2\u0167\u0168"+
-    "\7/\2\2\u0168\u0169\7?\2\2\u0169s\3\2\2\2\u016a\u016b\7,\2\2\u016b\u016c"+
-    "\7?\2\2\u016cu\3\2\2\2\u016d\u016e\7\61\2\2\u016e\u016f\7?\2\2\u016fw"+
-    "\3\2\2\2\u0170\u0171\7\'\2\2\u0171\u0172\7?\2\2\u0172y\3\2\2\2\u0173\u0174"+
-    "\7(\2\2\u0174\u0175\7?\2\2\u0175{\3\2\2\2\u0176\u0177\7`\2\2\u0177\u0178"+
-    "\7?\2\2\u0178}\3\2\2\2\u0179\u017a\7~\2\2\u017a\u017b\7?\2\2\u017b\177"+
-    "\3\2\2\2\u017c\u017d\7>\2\2\u017d\u017e\7>\2\2\u017e\u017f\7?\2\2\u017f"+
-    "\u0081\3\2\2\2\u0180\u0181\7@\2\2\u0181\u0182\7@\2\2\u0182\u0183\7?\2"+
-    "\2\u0183\u0083\3\2\2\2\u0184\u0185\7@\2\2\u0185\u0186\7@\2\2\u0186\u0187"+
-    "\7@\2\2\u0187\u0188\7?\2\2\u0188\u0085\3\2\2\2\u0189\u018b\7\62\2\2\u018a"+
-    "\u018c\t\4\2\2\u018b\u018a\3\2\2\2\u018c\u018d\3\2\2\2\u018d\u018b\3\2"+
-    "\2\2\u018d\u018e\3\2\2\2\u018e\u0190\3\2\2\2\u018f\u0191\t\5\2\2\u0190"+
-    "\u018f\3\2\2\2\u0190\u0191\3\2\2\2\u0191\u0087\3\2\2\2\u0192\u0193\7\62"+
-    "\2\2\u0193\u0195\t\6\2\2\u0194\u0196\t\7\2\2\u0195\u0194\3\2\2\2\u0196"+
-    "\u0197\3\2\2\2\u0197\u0195\3\2\2\2\u0197\u0198\3\2\2\2\u0198\u019a\3\2"+
-    "\2\2\u0199\u019b\t\5\2\2\u019a\u0199\3\2\2\2\u019a\u019b\3\2\2\2\u019b"+
-    "\u0089\3\2\2\2\u019c\u01a5\7\62\2\2\u019d\u01a1\t\b\2\2\u019e\u01a0\t"+
-    "\t\2\2\u019f\u019e\3\2\2\2\u01a0\u01a3\3\2\2\2\u01a1\u019f\3\2\2\2\u01a1"+
-    "\u01a2\3\2\2\2\u01a2\u01a5\3\2\2\2\u01a3\u01a1\3\2\2\2\u01a4\u019c\3\2"+
-    "\2\2\u01a4\u019d\3\2\2\2\u01a5\u01a7\3\2\2\2\u01a6\u01a8\t\n\2\2\u01a7"+
-    "\u01a6\3\2\2\2\u01a7\u01a8\3\2\2\2\u01a8\u008b\3\2\2\2\u01a9\u01b2\7\62"+
-    "\2\2\u01aa\u01ae\t\b\2\2\u01ab\u01ad\t\t\2\2\u01ac\u01ab\3\2\2\2\u01ad"+
-    "\u01b0\3\2\2\2\u01ae\u01ac\3\2\2\2\u01ae\u01af\3\2\2\2\u01af\u01b2\3\2"+
-    "\2\2\u01b0\u01ae\3\2\2\2\u01b1\u01a9\3\2\2\2\u01b1\u01aa\3\2\2\2\u01b2"+
-    "\u01b9\3\2\2\2\u01b3\u01b5\5\24\n\2\u01b4\u01b6\t\t\2\2\u01b5\u01b4\3"+
-    "\2\2\2\u01b6\u01b7\3\2\2\2\u01b7\u01b5\3\2\2\2\u01b7\u01b8\3\2\2\2\u01b8"+
-    "\u01ba\3\2\2\2\u01b9\u01b3\3\2\2\2\u01b9\u01ba\3\2\2\2\u01ba\u01c4\3\2"+
-    "\2\2\u01bb\u01bd\t\13\2\2\u01bc\u01be\t\f\2\2\u01bd\u01bc\3\2\2\2\u01bd"+
-    "\u01be\3\2\2\2\u01be\u01c0\3\2\2\2\u01bf\u01c1\t\t\2\2\u01c0\u01bf\3\2"+
-    "\2\2\u01c1\u01c2\3\2\2\2\u01c2\u01c0\3\2\2\2\u01c2\u01c3\3\2\2\2\u01c3"+
-    "\u01c5\3\2\2\2\u01c4\u01bb\3\2\2\2\u01c4\u01c5\3\2\2\2\u01c5\u01c7\3\2"+
-    "\2\2\u01c6\u01c8\t\r\2\2\u01c7\u01c6\3\2\2\2\u01c7\u01c8\3\2\2\2\u01c8"+
-    "\u008d\3\2\2\2\u01c9\u01d1\7$\2\2\u01ca\u01cb\7^\2\2\u01cb\u01d0\7$\2"+
-    "\2\u01cc\u01cd\7^\2\2\u01cd\u01d0\7^\2\2\u01ce\u01d0\n\16\2\2\u01cf\u01ca"+
-    "\3\2\2\2\u01cf\u01cc\3\2\2\2\u01cf\u01ce\3\2\2\2\u01d0\u01d3\3\2\2\2\u01d1"+
-    "\u01d2\3\2\2\2\u01d1\u01cf\3\2\2\2\u01d2\u01d4\3\2\2\2\u01d3\u01d1\3\2"+
-    "\2\2\u01d4\u01e2\7$\2\2\u01d5\u01dd\7)\2\2\u01d6\u01d7\7^\2\2\u01d7\u01dc"+
-    "\7)\2\2\u01d8\u01d9\7^\2\2\u01d9\u01dc\7^\2\2\u01da\u01dc\n\16\2\2\u01db"+
-    "\u01d6\3\2\2\2\u01db\u01d8\3\2\2\2\u01db\u01da\3\2\2\2\u01dc\u01df\3\2"+
-    "\2\2\u01dd\u01de\3\2\2\2\u01dd\u01db\3\2\2\2\u01de\u01e0\3\2\2\2\u01df"+
-    "\u01dd\3\2\2\2\u01e0\u01e2\7)\2\2\u01e1\u01c9\3\2\2\2\u01e1\u01d5\3\2"+
-    "\2\2\u01e2\u008f\3\2\2\2\u01e3\u01e4\7v\2\2\u01e4\u01e5\7t\2\2\u01e5\u01e6"+
-    "\7w\2\2\u01e6\u01e7\7g\2\2\u01e7\u0091\3\2\2\2\u01e8\u01e9\7h\2\2\u01e9"+
-    "\u01ea\7c\2\2\u01ea\u01eb\7n\2\2\u01eb\u01ec\7u\2\2\u01ec\u01ed\7g\2\2"+
-    "\u01ed\u0093\3\2\2\2\u01ee\u01ef\7p\2\2\u01ef\u01f0\7w\2\2\u01f0\u01f1"+
-    "\7n\2\2\u01f1\u01f2\7n\2\2\u01f2\u0095\3\2\2\2\u01f3\u01f9\5\u0098L\2"+
-    "\u01f4\u01f5\5\24\n\2\u01f5\u01f6\5\u0098L\2\u01f6\u01f8\3\2\2\2\u01f7"+
-    "\u01f4\3\2\2\2\u01f8\u01fb\3\2\2\2\u01f9\u01f7\3\2\2\2\u01f9\u01fa\3\2"+
-    "\2\2\u01fa\u01fc\3\2\2\2\u01fb\u01f9\3\2\2\2\u01fc\u01fd\6K\2\2\u01fd"+
-    "\u0097\3\2\2\2\u01fe\u0202\t\17\2\2\u01ff\u0201\t\20\2\2\u0200\u01ff\3"+
-    "\2\2\2\u0201\u0204\3\2\2\2\u0202\u0200\3\2\2\2\u0202\u0203\3\2\2\2\u0203"+
-    "\u0099\3\2\2\2\u0204\u0202\3\2\2\2\u0205\u020e\7\62\2\2\u0206\u020a\t"+
-    "\b\2\2\u0207\u0209\t\t\2\2\u0208\u0207\3\2\2\2\u0209\u020c\3\2\2\2\u020a"+
-    "\u0208\3\2\2\2\u020a\u020b\3\2\2\2\u020b\u020e\3\2\2\2\u020c\u020a\3\2"+
-    "\2\2\u020d\u0205\3\2\2\2\u020d\u0206\3\2\2\2\u020e\u020f\3\2\2\2\u020f"+
-    "\u0210\bM\4\2\u0210\u009b\3\2\2\2\u0211\u0215\t\17\2\2\u0212\u0214\t\20"+
-    "\2\2\u0213\u0212\3\2\2\2\u0214\u0217\3\2\2\2\u0215\u0213\3\2\2\2\u0215"+
-    "\u0216\3\2\2\2\u0216\u0218\3\2\2\2\u0217\u0215\3\2\2\2\u0218\u0219\bN"+
-    "\4\2\u0219\u009d\3\2\2\2!\2\3\u00a1\u00ab\u00b5\u00ba\u018d\u0190\u0197"+
-    "\u019a\u01a1\u01a4\u01a7\u01ae\u01b1\u01b7\u01b9\u01bd\u01c2\u01c4\u01c7"+
-    "\u01cf\u01d1\u01db\u01dd\u01e1\u01f9\u0202\u020a\u020d\u0215\5\b\2\2\4"+
-    "\3\2\4\2\2";
+    "\3\32\3\32\3\33\3\33\3\34\3\34\3\35\3\35\3\35\3\36\3\36\3\37\3\37\3 \3"+
+    " \3!\3!\3!\3\"\3\"\3\"\3#\3#\3#\3#\3$\3$\3%\3%\3%\3&\3&\3\'\3\'\3\'\3"+
+    "(\3(\3(\3)\3)\3)\3)\3*\3*\3*\3+\3+\3+\3+\3,\3,\3-\3-\3.\3.\3/\3/\3/\3"+
+    "\60\3\60\3\60\3\61\3\61\3\62\3\62\3\63\3\63\3\63\3\64\3\64\3\64\3\65\3"+
+    "\65\3\65\3\66\3\66\3\66\3\67\3\67\38\38\38\39\39\39\3:\3:\3:\3;\3;\3;"+
+    "\3<\3<\3<\3=\3=\3=\3>\3>\3>\3?\3?\3?\3@\3@\3@\3@\3A\3A\3A\3A\3B\3B\3B"+
+    "\3B\3B\3C\3C\6C\u018f\nC\rC\16C\u0190\3C\5C\u0194\nC\3D\3D\3D\6D\u0199"+
+    "\nD\rD\16D\u019a\3D\5D\u019e\nD\3E\3E\3E\7E\u01a3\nE\fE\16E\u01a6\13E"+
+    "\5E\u01a8\nE\3E\5E\u01ab\nE\3F\3F\3F\7F\u01b0\nF\fF\16F\u01b3\13F\5F\u01b5"+
+    "\nF\3F\3F\6F\u01b9\nF\rF\16F\u01ba\5F\u01bd\nF\3F\3F\5F\u01c1\nF\3F\6"+
+    "F\u01c4\nF\rF\16F\u01c5\5F\u01c8\nF\3F\5F\u01cb\nF\3G\3G\3G\3G\3G\3G\7"+
+    "G\u01d3\nG\fG\16G\u01d6\13G\3G\3G\3G\3G\3G\3G\3G\7G\u01df\nG\fG\16G\u01e2"+
+    "\13G\3G\5G\u01e5\nG\3H\3H\3H\3H\6H\u01eb\nH\rH\16H\u01ec\3H\3H\3H\3I\3"+
+    "I\3I\3I\3I\3J\3J\3J\3J\3J\3J\3K\3K\3K\3K\3K\3L\3L\3L\3L\7L\u0206\nL\f"+
+    "L\16L\u0209\13L\3L\3L\3M\3M\7M\u020f\nM\fM\16M\u0212\13M\3N\3N\3N\7N\u0217"+
+    "\nN\fN\16N\u021a\13N\5N\u021c\nN\3N\3N\3O\3O\7O\u0222\nO\fO\16O\u0225"+
+    "\13O\3O\3O\6\u00ad\u00b7\u01d4\u01e0\2P\4\3\6\4\b\5\n\6\f\7\16\b\20\t"+
+    "\22\n\24\13\26\f\30\r\32\16\34\17\36\20 \21\"\22$\23&\24(\25*\26,\27."+
+    "\30\60\31\62\32\64\33\66\348\35:\36<\37> @!B\"D#F$H%J&L\'N(P)R*T+V,X-"+
+    "Z.\\/^\60`\61b\62d\63f\64h\65j\66l\67n8p9r:t;v<x=z>|?~@\u0080A\u0082B"+
+    "\u0084C\u0086D\u0088E\u008aF\u008cG\u008eH\u0090I\u0092J\u0094K\u0096"+
+    "L\u0098M\u009aN\u009cO\u009eP\4\2\3\23\5\2\13\f\17\17\"\"\4\2\f\f\17\17"+
+    "\3\2\629\4\2NNnn\4\2ZZzz\5\2\62;CHch\3\2\63;\3\2\62;\b\2FFHHNNffhhnn\4"+
+    "\2GGgg\4\2--//\4\2HHhh\4\2$$^^\4\2\f\f\61\61\3\2\f\f\5\2C\\aac|\6\2\62"+
+    ";C\\aac|\u0247\2\4\3\2\2\2\2\6\3\2\2\2\2\b\3\2\2\2\2\n\3\2\2\2\2\f\3\2"+
+    "\2\2\2\16\3\2\2\2\2\20\3\2\2\2\2\22\3\2\2\2\2\24\3\2\2\2\2\26\3\2\2\2"+
+    "\2\30\3\2\2\2\2\32\3\2\2\2\2\34\3\2\2\2\2\36\3\2\2\2\2 \3\2\2\2\2\"\3"+
+    "\2\2\2\2$\3\2\2\2\2&\3\2\2\2\2(\3\2\2\2\2*\3\2\2\2\2,\3\2\2\2\2.\3\2\2"+
+    "\2\2\60\3\2\2\2\2\62\3\2\2\2\2\64\3\2\2\2\2\66\3\2\2\2\28\3\2\2\2\2:\3"+
+    "\2\2\2\2<\3\2\2\2\2>\3\2\2\2\2@\3\2\2\2\2B\3\2\2\2\2D\3\2\2\2\2F\3\2\2"+
+    "\2\2H\3\2\2\2\2J\3\2\2\2\2L\3\2\2\2\2N\3\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2"+
+    "T\3\2\2\2\2V\3\2\2\2\2X\3\2\2\2\2Z\3\2\2\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3"+
+    "\2\2\2\2b\3\2\2\2\2d\3\2\2\2\2f\3\2\2\2\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2"+
+    "\2\2n\3\2\2\2\2p\3\2\2\2\2r\3\2\2\2\2t\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2"+
+    "z\3\2\2\2\2|\3\2\2\2\2~\3\2\2\2\2\u0080\3\2\2\2\2\u0082\3\2\2\2\2\u0084"+
+    "\3\2\2\2\2\u0086\3\2\2\2\2\u0088\3\2\2\2\2\u008a\3\2\2\2\2\u008c\3\2\2"+
+    "\2\2\u008e\3\2\2\2\2\u0090\3\2\2\2\2\u0092\3\2\2\2\2\u0094\3\2\2\2\2\u0096"+
+    "\3\2\2\2\2\u0098\3\2\2\2\2\u009a\3\2\2\2\3\u009c\3\2\2\2\3\u009e\3\2\2"+
+    "\2\4\u00a1\3\2\2\2\6\u00bc\3\2\2\2\b\u00c0\3\2\2\2\n\u00c2\3\2\2\2\f\u00c4"+
+    "\3\2\2\2\16\u00c6\3\2\2\2\20\u00c8\3\2\2\2\22\u00ca\3\2\2\2\24\u00cc\3"+
+    "\2\2\2\26\u00d0\3\2\2\2\30\u00d2\3\2\2\2\32\u00d4\3\2\2\2\34\u00d7\3\2"+
+    "\2\2\36\u00dc\3\2\2\2 \u00e2\3\2\2\2\"\u00e5\3\2\2\2$\u00e9\3\2\2\2&\u00f2"+
+    "\3\2\2\2(\u00f8\3\2\2\2*\u00ff\3\2\2\2,\u0103\3\2\2\2.\u0107\3\2\2\2\60"+
+    "\u010d\3\2\2\2\62\u0113\3\2\2\2\64\u0118\3\2\2\2\66\u011a\3\2\2\28\u011c"+
+    "\3\2\2\2:\u011e\3\2\2\2<\u0121\3\2\2\2>\u0123\3\2\2\2@\u0125\3\2\2\2B"+
+    "\u0127\3\2\2\2D\u012a\3\2\2\2F\u012d\3\2\2\2H\u0131\3\2\2\2J\u0133\3\2"+
+    "\2\2L\u0136\3\2\2\2N\u0138\3\2\2\2P\u013b\3\2\2\2R\u013e\3\2\2\2T\u0142"+
+    "\3\2\2\2V\u0145\3\2\2\2X\u0149\3\2\2\2Z\u014b\3\2\2\2\\\u014d\3\2\2\2"+
+    "^\u014f\3\2\2\2`\u0152\3\2\2\2b\u0155\3\2\2\2d\u0157\3\2\2\2f\u0159\3"+
+    "\2\2\2h\u015c\3\2\2\2j\u015f\3\2\2\2l\u0162\3\2\2\2n\u0165\3\2\2\2p\u0167"+
+    "\3\2\2\2r\u016a\3\2\2\2t\u016d\3\2\2\2v\u0170\3\2\2\2x\u0173\3\2\2\2z"+
+    "\u0176\3\2\2\2|\u0179\3\2\2\2~\u017c\3\2\2\2\u0080\u017f\3\2\2\2\u0082"+
+    "\u0183\3\2\2\2\u0084\u0187\3\2\2\2\u0086\u018c\3\2\2\2\u0088\u0195\3\2"+
+    "\2\2\u008a\u01a7\3\2\2\2\u008c\u01b4\3\2\2\2\u008e\u01e4\3\2\2\2\u0090"+
+    "\u01e6\3\2\2\2\u0092\u01f1\3\2\2\2\u0094\u01f6\3\2\2\2\u0096\u01fc\3\2"+
+    "\2\2\u0098\u0201\3\2\2\2\u009a\u020c\3\2\2\2\u009c\u021b\3\2\2\2\u009e"+
+    "\u021f\3\2\2\2\u00a0\u00a2\t\2\2\2\u00a1\u00a0\3\2\2\2\u00a2\u00a3\3\2"+
+    "\2\2\u00a3\u00a1\3\2\2\2\u00a3\u00a4\3\2\2\2\u00a4\u00a5\3\2\2\2\u00a5"+
+    "\u00a6\b\2\2\2\u00a6\5\3\2\2\2\u00a7\u00a8\7\61\2\2\u00a8\u00a9\7\61\2"+
+    "\2\u00a9\u00ad\3\2\2\2\u00aa\u00ac\13\2\2\2\u00ab\u00aa\3\2\2\2\u00ac"+
+    "\u00af\3\2\2\2\u00ad\u00ae\3\2\2\2\u00ad\u00ab\3\2\2\2\u00ae\u00b0\3\2"+
+    "\2\2\u00af\u00ad\3\2\2\2\u00b0\u00bd\t\3\2\2\u00b1\u00b2\7\61\2\2\u00b2"+
+    "\u00b3\7,\2\2\u00b3\u00b7\3\2\2\2\u00b4\u00b6\13\2\2\2\u00b5\u00b4\3\2"+
+    "\2\2\u00b6\u00b9\3\2\2\2\u00b7\u00b8\3\2\2\2\u00b7\u00b5\3\2\2\2\u00b8"+
+    "\u00ba\3\2\2\2\u00b9\u00b7\3\2\2\2\u00ba\u00bb\7,\2\2\u00bb\u00bd\7\61"+
+    "\2\2\u00bc\u00a7\3\2\2\2\u00bc\u00b1\3\2\2\2\u00bd\u00be\3\2\2\2\u00be"+
+    "\u00bf\b\3\2\2\u00bf\7\3\2\2\2\u00c0\u00c1\7}\2\2\u00c1\t\3\2\2\2\u00c2"+
+    "\u00c3\7\177\2\2\u00c3\13\3\2\2\2\u00c4\u00c5\7]\2\2\u00c5\r\3\2\2\2\u00c6"+
+    "\u00c7\7_\2\2\u00c7\17\3\2\2\2\u00c8\u00c9\7*\2\2\u00c9\21\3\2\2\2\u00ca"+
+    "\u00cb\7+\2\2\u00cb\23\3\2\2\2\u00cc\u00cd\7\60\2\2\u00cd\u00ce\3\2\2"+
+    "\2\u00ce\u00cf\b\n\3\2\u00cf\25\3\2\2\2\u00d0\u00d1\7.\2\2\u00d1\27\3"+
+    "\2\2\2\u00d2\u00d3\7=\2\2\u00d3\31\3\2\2\2\u00d4\u00d5\7k\2\2\u00d5\u00d6"+
+    "\7h\2\2\u00d6\33\3\2\2\2\u00d7\u00d8\7g\2\2\u00d8\u00d9\7n\2\2\u00d9\u00da"+
+    "\7u\2\2\u00da\u00db\7g\2\2\u00db\35\3\2\2\2\u00dc\u00dd\7y\2\2\u00dd\u00de"+
+    "\7j\2\2\u00de\u00df\7k\2\2\u00df\u00e0\7n\2\2\u00e0\u00e1\7g\2\2\u00e1"+
+    "\37\3\2\2\2\u00e2\u00e3\7f\2\2\u00e3\u00e4\7q\2\2\u00e4!\3\2\2\2\u00e5"+
+    "\u00e6\7h\2\2\u00e6\u00e7\7q\2\2\u00e7\u00e8\7t\2\2\u00e8#\3\2\2\2\u00e9"+
+    "\u00ea\7e\2\2\u00ea\u00eb\7q\2\2\u00eb\u00ec\7p\2\2\u00ec\u00ed\7v\2\2"+
+    "\u00ed\u00ee\7k\2\2\u00ee\u00ef\7p\2\2\u00ef\u00f0\7w\2\2\u00f0\u00f1"+
+    "\7g\2\2\u00f1%\3\2\2\2\u00f2\u00f3\7d\2\2\u00f3\u00f4\7t\2\2\u00f4\u00f5"+
+    "\7g\2\2\u00f5\u00f6\7c\2\2\u00f6\u00f7\7m\2\2\u00f7\'\3\2\2\2\u00f8\u00f9"+
+    "\7t\2\2\u00f9\u00fa\7g\2\2\u00fa\u00fb\7v\2\2\u00fb\u00fc\7w\2\2\u00fc"+
+    "\u00fd\7t\2\2\u00fd\u00fe\7p\2\2\u00fe)\3\2\2\2\u00ff\u0100\7p\2\2\u0100"+
+    "\u0101\7g\2\2\u0101\u0102\7y\2\2\u0102+\3\2\2\2\u0103\u0104\7v\2\2\u0104"+
+    "\u0105\7t\2\2\u0105\u0106\7{\2\2\u0106-\3\2\2\2\u0107\u0108\7e\2\2\u0108"+
+    "\u0109\7c\2\2\u0109\u010a\7v\2\2\u010a\u010b\7e\2\2\u010b\u010c\7j\2\2"+
+    "\u010c/\3\2\2\2\u010d\u010e\7v\2\2\u010e\u010f\7j\2\2\u010f\u0110\7t\2"+
+    "\2\u0110\u0111\7q\2\2\u0111\u0112\7y\2\2\u0112\61\3\2\2\2\u0113\u0114"+
+    "\7v\2\2\u0114\u0115\7j\2\2\u0115\u0116\7k\2\2\u0116\u0117\7u\2\2\u0117"+
+    "\63\3\2\2\2\u0118\u0119\7#\2\2\u0119\65\3\2\2\2\u011a\u011b\7\u0080\2"+
+    "\2\u011b\67\3\2\2\2\u011c\u011d\7,\2\2\u011d9\3\2\2\2\u011e\u011f\7\61"+
+    "\2\2\u011f\u0120\6\35\2\2\u0120;\3\2\2\2\u0121\u0122\7\'\2\2\u0122=\3"+
+    "\2\2\2\u0123\u0124\7-\2\2\u0124?\3\2\2\2\u0125\u0126\7/\2\2\u0126A\3\2"+
+    "\2\2\u0127\u0128\7>\2\2\u0128\u0129\7>\2\2\u0129C\3\2\2\2\u012a\u012b"+
+    "\7@\2\2\u012b\u012c\7@\2\2\u012cE\3\2\2\2\u012d\u012e\7@\2\2\u012e\u012f"+
+    "\7@\2\2\u012f\u0130\7@\2\2\u0130G\3\2\2\2\u0131\u0132\7>\2\2\u0132I\3"+
+    "\2\2\2\u0133\u0134\7>\2\2\u0134\u0135\7?\2\2\u0135K\3\2\2\2\u0136\u0137"+
+    "\7@\2\2\u0137M\3\2\2\2\u0138\u0139\7@\2\2\u0139\u013a\7?\2\2\u013aO\3"+
+    "\2\2\2\u013b\u013c\7?\2\2\u013c\u013d\7?\2\2\u013dQ\3\2\2\2\u013e\u013f"+
+    "\7?\2\2\u013f\u0140\7?\2\2\u0140\u0141\7?\2\2\u0141S\3\2\2\2\u0142\u0143"+
+    "\7#\2\2\u0143\u0144\7?\2\2\u0144U\3\2\2\2\u0145\u0146\7#\2\2\u0146\u0147"+
+    "\7?\2\2\u0147\u0148\7?\2\2\u0148W\3\2\2\2\u0149\u014a\7(\2\2\u014aY\3"+
+    "\2\2\2\u014b\u014c\7`\2\2\u014c[\3\2\2\2\u014d\u014e\7~\2\2\u014e]\3\2"+
+    "\2\2\u014f\u0150\7(\2\2\u0150\u0151\7(\2\2\u0151_\3\2\2\2\u0152\u0153"+
+    "\7~\2\2\u0153\u0154\7~\2\2\u0154a\3\2\2\2\u0155\u0156\7A\2\2\u0156c\3"+
+    "\2\2\2\u0157\u0158\7<\2\2\u0158e\3\2\2\2\u0159\u015a\7<\2\2\u015a\u015b"+
+    "\7<\2\2\u015bg\3\2\2\2\u015c\u015d\7/\2\2\u015d\u015e\7@\2\2\u015ei\3"+
+    "\2\2\2\u015f\u0160\7-\2\2\u0160\u0161\7-\2\2\u0161k\3\2\2\2\u0162\u0163"+
+    "\7/\2\2\u0163\u0164\7/\2\2\u0164m\3\2\2\2\u0165\u0166\7?\2\2\u0166o\3"+
+    "\2\2\2\u0167\u0168\7-\2\2\u0168\u0169\7?\2\2\u0169q\3\2\2\2\u016a\u016b"+
+    "\7/\2\2\u016b\u016c\7?\2\2\u016cs\3\2\2\2\u016d\u016e\7,\2\2\u016e\u016f"+
+    "\7?\2\2\u016fu\3\2\2\2\u0170\u0171\7\61\2\2\u0171\u0172\7?\2\2\u0172w"+
+    "\3\2\2\2\u0173\u0174\7\'\2\2\u0174\u0175\7?\2\2\u0175y\3\2\2\2\u0176\u0177"+
+    "\7(\2\2\u0177\u0178\7?\2\2\u0178{\3\2\2\2\u0179\u017a\7`\2\2\u017a\u017b"+
+    "\7?\2\2\u017b}\3\2\2\2\u017c\u017d\7~\2\2\u017d\u017e\7?\2\2\u017e\177"+
+    "\3\2\2\2\u017f\u0180\7>\2\2\u0180\u0181\7>\2\2\u0181\u0182\7?\2\2\u0182"+
+    "\u0081\3\2\2\2\u0183\u0184\7@\2\2\u0184\u0185\7@\2\2\u0185\u0186\7?\2"+
+    "\2\u0186\u0083\3\2\2\2\u0187\u0188\7@\2\2\u0188\u0189\7@\2\2\u0189\u018a"+
+    "\7@\2\2\u018a\u018b\7?\2\2\u018b\u0085\3\2\2\2\u018c\u018e\7\62\2\2\u018d"+
+    "\u018f\t\4\2\2\u018e\u018d\3\2\2\2\u018f\u0190\3\2\2\2\u0190\u018e\3\2"+
+    "\2\2\u0190\u0191\3\2\2\2\u0191\u0193\3\2\2\2\u0192\u0194\t\5\2\2\u0193"+
+    "\u0192\3\2\2\2\u0193\u0194\3\2\2\2\u0194\u0087\3\2\2\2\u0195\u0196\7\62"+
+    "\2\2\u0196\u0198\t\6\2\2\u0197\u0199\t\7\2\2\u0198\u0197\3\2\2\2\u0199"+
+    "\u019a\3\2\2\2\u019a\u0198\3\2\2\2\u019a\u019b\3\2\2\2\u019b\u019d\3\2"+
+    "\2\2\u019c\u019e\t\5\2\2\u019d\u019c\3\2\2\2\u019d\u019e\3\2\2\2\u019e"+
+    "\u0089\3\2\2\2\u019f\u01a8\7\62\2\2\u01a0\u01a4\t\b\2\2\u01a1\u01a3\t"+
+    "\t\2\2\u01a2\u01a1\3\2\2\2\u01a3\u01a6\3\2\2\2\u01a4\u01a2\3\2\2\2\u01a4"+
+    "\u01a5\3\2\2\2\u01a5\u01a8\3\2\2\2\u01a6\u01a4\3\2\2\2\u01a7\u019f\3\2"+
+    "\2\2\u01a7\u01a0\3\2\2\2\u01a8\u01aa\3\2\2\2\u01a9\u01ab\t\n\2\2\u01aa"+
+    "\u01a9\3\2\2\2\u01aa\u01ab\3\2\2\2\u01ab\u008b\3\2\2\2\u01ac\u01b5\7\62"+
+    "\2\2\u01ad\u01b1\t\b\2\2\u01ae\u01b0\t\t\2\2\u01af\u01ae\3\2\2\2\u01b0"+
+    "\u01b3\3\2\2\2\u01b1\u01af\3\2\2\2\u01b1\u01b2\3\2\2\2\u01b2\u01b5\3\2"+
+    "\2\2\u01b3\u01b1\3\2\2\2\u01b4\u01ac\3\2\2\2\u01b4\u01ad\3\2\2\2\u01b5"+
+    "\u01bc\3\2\2\2\u01b6\u01b8\5\24\n\2\u01b7\u01b9\t\t\2\2\u01b8\u01b7\3"+
+    "\2\2\2\u01b9\u01ba\3\2\2\2\u01ba\u01b8\3\2\2\2\u01ba\u01bb\3\2\2\2\u01bb"+
+    "\u01bd\3\2\2\2\u01bc\u01b6\3\2\2\2\u01bc\u01bd\3\2\2\2\u01bd\u01c7\3\2"+
+    "\2\2\u01be\u01c0\t\13\2\2\u01bf\u01c1\t\f\2\2\u01c0\u01bf\3\2\2\2\u01c0"+
+    "\u01c1\3\2\2\2\u01c1\u01c3\3\2\2\2\u01c2\u01c4\t\t\2\2\u01c3\u01c2\3\2"+
+    "\2\2\u01c4\u01c5\3\2\2\2\u01c5\u01c3\3\2\2\2\u01c5\u01c6\3\2\2\2\u01c6"+
+    "\u01c8\3\2\2\2\u01c7\u01be\3\2\2\2\u01c7\u01c8\3\2\2\2\u01c8\u01ca\3\2"+
+    "\2\2\u01c9\u01cb\t\r\2\2\u01ca\u01c9\3\2\2\2\u01ca\u01cb\3\2\2\2\u01cb"+
+    "\u008d\3\2\2\2\u01cc\u01d4\7$\2\2\u01cd\u01ce\7^\2\2\u01ce\u01d3\7$\2"+
+    "\2\u01cf\u01d0\7^\2\2\u01d0\u01d3\7^\2\2\u01d1\u01d3\n\16\2\2\u01d2\u01cd"+
+    "\3\2\2\2\u01d2\u01cf\3\2\2\2\u01d2\u01d1\3\2\2\2\u01d3\u01d6\3\2\2\2\u01d4"+
+    "\u01d5\3\2\2\2\u01d4\u01d2\3\2\2\2\u01d5\u01d7\3\2\2\2\u01d6\u01d4\3\2"+
+    "\2\2\u01d7\u01e5\7$\2\2\u01d8\u01e0\7)\2\2\u01d9\u01da\7^\2\2\u01da\u01df"+
+    "\7)\2\2\u01db\u01dc\7^\2\2\u01dc\u01df\7^\2\2\u01dd\u01df\n\16\2\2\u01de"+
+    "\u01d9\3\2\2\2\u01de\u01db\3\2\2\2\u01de\u01dd\3\2\2\2\u01df\u01e2\3\2"+
+    "\2\2\u01e0\u01e1\3\2\2\2\u01e0\u01de\3\2\2\2\u01e1\u01e3\3\2\2\2\u01e2"+
+    "\u01e0\3\2\2\2\u01e3\u01e5\7)\2\2\u01e4\u01cc\3\2\2\2\u01e4\u01d8\3\2"+
+    "\2\2\u01e5\u008f\3\2\2\2\u01e6\u01ea\7\61\2\2\u01e7\u01eb\n\17\2\2\u01e8"+
+    "\u01e9\7^\2\2\u01e9\u01eb\n\20\2\2\u01ea\u01e7\3\2\2\2\u01ea\u01e8\3\2"+
+    "\2\2\u01eb\u01ec\3\2\2\2\u01ec\u01ea\3\2\2\2\u01ec\u01ed\3\2\2\2\u01ed"+
+    "\u01ee\3\2\2\2\u01ee\u01ef\7\61\2\2\u01ef\u01f0\6H\3\2\u01f0\u0091\3\2"+
+    "\2\2\u01f1\u01f2\7v\2\2\u01f2\u01f3\7t\2\2\u01f3\u01f4\7w\2\2\u01f4\u01f5"+
+    "\7g\2\2\u01f5\u0093\3\2\2\2\u01f6\u01f7\7h\2\2\u01f7\u01f8\7c\2\2\u01f8"+
+    "\u01f9\7n\2\2\u01f9\u01fa\7u\2\2\u01fa\u01fb\7g\2\2\u01fb\u0095\3\2\2"+
+    "\2\u01fc\u01fd\7p\2\2\u01fd\u01fe\7w\2\2\u01fe\u01ff\7n\2\2\u01ff\u0200"+
+    "\7n\2\2\u0200\u0097\3\2\2\2\u0201\u0207\5\u009aM\2\u0202\u0203\5\24\n"+
+    "\2\u0203\u0204\5\u009aM\2\u0204\u0206\3\2\2\2\u0205\u0202\3\2\2\2\u0206"+
+    "\u0209\3\2\2\2\u0207\u0205\3\2\2\2\u0207\u0208\3\2\2\2\u0208\u020a\3\2"+
+    "\2\2\u0209\u0207\3\2\2\2\u020a\u020b\6L\4\2\u020b\u0099\3\2\2\2\u020c"+
+    "\u0210\t\21\2\2\u020d\u020f\t\22\2\2\u020e\u020d\3\2\2\2\u020f\u0212\3"+
+    "\2\2\2\u0210\u020e\3\2\2\2\u0210\u0211\3\2\2\2\u0211\u009b\3\2\2\2\u0212"+
+    "\u0210\3\2\2\2\u0213\u021c\7\62\2\2\u0214\u0218\t\b\2\2\u0215\u0217\t"+
+    "\t\2\2\u0216\u0215\3\2\2\2\u0217\u021a\3\2\2\2\u0218\u0216\3\2\2\2\u0218"+
+    "\u0219\3\2\2\2\u0219\u021c\3\2\2\2\u021a\u0218\3\2\2\2\u021b\u0213\3\2"+
+    "\2\2\u021b\u0214\3\2\2\2\u021c\u021d\3\2\2\2\u021d\u021e\bN\4\2\u021e"+
+    "\u009d\3\2\2\2\u021f\u0223\t\21\2\2\u0220\u0222\t\22\2\2\u0221\u0220\3"+
+    "\2\2\2\u0222\u0225\3\2\2\2\u0223\u0221\3\2\2\2\u0223\u0224\3\2\2\2\u0224"+
+    "\u0226\3\2\2\2\u0225\u0223\3\2\2\2\u0226\u0227\bO\4\2\u0227\u009f\3\2"+
+    "\2\2#\2\3\u00a3\u00ad\u00b7\u00bc\u0190\u0193\u019a\u019d\u01a4\u01a7"+
+    "\u01aa\u01b1\u01b4\u01ba\u01bc\u01c0\u01c5\u01c7\u01ca\u01d2\u01d4\u01de"+
+    "\u01e0\u01e4\u01ea\u01ec\u0207\u0210\u0218\u021b\u0223\5\b\2\2\4\3\2\4"+
+    "\2\2";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
index ae080fbb8aa..ae9508f7969 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
@@ -25,8 +25,8 @@ class PainlessParser extends Parser {
     XOR=44, BWOR=45, BOOLAND=46, BOOLOR=47, COND=48, COLON=49, REF=50, ARROW=51, 
     INCR=52, DECR=53, ASSIGN=54, AADD=55, ASUB=56, AMUL=57, ADIV=58, AREM=59, 
     AAND=60, AXOR=61, AOR=62, ALSH=63, ARSH=64, AUSH=65, OCTAL=66, HEX=67, 
-    INTEGER=68, DECIMAL=69, STRING=70, TRUE=71, FALSE=72, NULL=73, TYPE=74, 
-    ID=75, DOTINTEGER=76, DOTID=77;
+    INTEGER=68, DECIMAL=69, STRING=70, REGEX=71, TRUE=72, FALSE=73, NULL=74, 
+    TYPE=75, ID=76, DOTINTEGER=77, DOTID=78;
   public static final int
     RULE_source = 0, RULE_function = 1, RULE_parameters = 2, RULE_statement = 3, 
     RULE_trailer = 4, RULE_block = 5, RULE_empty = 6, RULE_initializer = 7, 
@@ -52,7 +52,7 @@ class PainlessParser extends Parser {
     "'>'", "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", "'^'", "'|'", 
     "'&&'", "'||'", "'?'", "':'", "'::'", "'->'", "'++'", "'--'", "'='", "'+='", 
     "'-='", "'*='", "'/='", "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", 
-    "'>>>='", null, null, null, null, null, "'true'", "'false'", "'null'"
+    "'>>>='", null, null, null, null, null, null, "'true'", "'false'", "'null'"
   };
   private static final String[] _SYMBOLIC_NAMES = {
     null, "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", 
@@ -63,7 +63,7 @@ class PainlessParser extends Parser {
     "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "INCR", "DECR", 
     "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", "AAND", "AXOR", "AOR", 
     "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", 
-    "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", "DOTID"
+    "REGEX", "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", "DOTID"
   };
   public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
 
@@ -2228,6 +2228,15 @@ class PainlessParser extends Parser {
       this.s = ctx.s;
     }
   }
+  public static class RegexContext extends PrimaryContext {
+    public TerminalNode REGEX() { return getToken(PainlessParser.REGEX, 0); }
+    public RegexContext(PrimaryContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitRegex(this);
+      else return visitor.visitChildren(this);
+    }
+  }
   public static class StringContext extends PrimaryContext {
     public TerminalNode STRING() { return getToken(PainlessParser.STRING, 0); }
     public StringContext(PrimaryContext ctx) { copyFrom(ctx); }
@@ -2303,7 +2312,7 @@ class PainlessParser extends Parser {
     PrimaryContext _localctx = new PrimaryContext(_ctx, getState(), c);
     enterRule(_localctx, 34, RULE_primary);
     try {
-      setState(390);
+      setState(391);
       switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
       case 1:
         _localctx = new ExprprecContext(_localctx);
@@ -2343,32 +2352,40 @@ class PainlessParser extends Parser {
         }
         break;
       case 4:
-        _localctx = new VariableContext(_localctx);
+        _localctx = new RegexContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
         setState(384);
-        match(ID);
+        match(REGEX);
         }
         break;
       case 5:
-        _localctx = new CalllocalContext(_localctx);
+        _localctx = new VariableContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
         setState(385);
         match(ID);
-        setState(386);
-        arguments();
         }
         break;
       case 6:
-        _localctx = new NewobjectContext(_localctx);
+        _localctx = new CalllocalContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
+        setState(386);
+        match(ID);
         setState(387);
-        match(NEW);
+        arguments();
+        }
+        break;
+      case 7:
+        _localctx = new NewobjectContext(_localctx);
+        enterOuterAlt(_localctx, 7);
+        {
         setState(388);
-        match(TYPE);
+        match(NEW);
         setState(389);
+        match(TYPE);
+        setState(390);
         arguments();
         }
         break;
@@ -2410,23 +2427,23 @@ class PainlessParser extends Parser {
     SecondaryContext _localctx = new SecondaryContext(_ctx, getState(), s);
     enterRule(_localctx, 36, RULE_secondary);
     try {
-      setState(396);
+      setState(397);
       switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(392);
-        if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
         setState(393);
+        if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
+        setState(394);
         dot();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(394);
-        if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
         setState(395);
+        if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
+        setState(396);
         brace();
         }
         break;
@@ -2484,17 +2501,17 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 38, RULE_dot);
     int _la;
     try {
-      setState(403);
+      setState(404);
       switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) {
       case 1:
         _localctx = new CallinvokeContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(398);
-        match(DOT);
         setState(399);
-        match(DOTID);
+        match(DOT);
         setState(400);
+        match(DOTID);
+        setState(401);
         arguments();
         }
         break;
@@ -2502,9 +2519,9 @@ class PainlessParser extends Parser {
         _localctx = new FieldaccessContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(401);
-        match(DOT);
         setState(402);
+        match(DOT);
+        setState(403);
         _la = _input.LA(1);
         if ( !(_la==DOTINTEGER || _la==DOTID) ) {
         _errHandler.recoverInline(this);
@@ -2558,11 +2575,11 @@ class PainlessParser extends Parser {
       _localctx = new BraceaccessContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(405);
-      match(LBRACE);
       setState(406);
-      expression(0);
+      match(LBRACE);
       setState(407);
+      expression(0);
+      setState(408);
       match(RBRACE);
       }
     }
@@ -2609,34 +2626,34 @@ class PainlessParser extends Parser {
       enterOuterAlt(_localctx, 1);
       {
       {
-      setState(409);
+      setState(410);
       match(LP);
-      setState(418);
+      setState(419);
       switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) {
       case 1:
         {
-        setState(410);
+        setState(411);
         argument();
-        setState(415);
+        setState(416);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(411);
-          match(COMMA);
           setState(412);
+          match(COMMA);
+          setState(413);
           argument();
           }
           }
-          setState(417);
+          setState(418);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
         break;
       }
-      setState(420);
+      setState(421);
       match(RP);
       }
       }
@@ -2677,26 +2694,26 @@ class PainlessParser extends Parser {
     ArgumentContext _localctx = new ArgumentContext(_ctx, getState());
     enterRule(_localctx, 44, RULE_argument);
     try {
-      setState(425);
+      setState(426);
       switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(422);
+        setState(423);
         expression(0);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(423);
+        setState(424);
         lambda();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(424);
+        setState(425);
         funcref();
         }
         break;
@@ -2748,54 +2765,54 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(440);
+      setState(441);
       switch (_input.LA(1)) {
       case TYPE:
       case ID:
         {
-        setState(427);
+        setState(428);
         lamtype();
         }
         break;
       case LP:
         {
-        setState(428);
+        setState(429);
         match(LP);
-        setState(437);
+        setState(438);
         _la = _input.LA(1);
         if (_la==TYPE || _la==ID) {
           {
-          setState(429);
+          setState(430);
           lamtype();
-          setState(434);
+          setState(435);
           _errHandler.sync(this);
           _la = _input.LA(1);
           while (_la==COMMA) {
             {
             {
-            setState(430);
-            match(COMMA);
             setState(431);
+            match(COMMA);
+            setState(432);
             lamtype();
             }
             }
-            setState(436);
+            setState(437);
             _errHandler.sync(this);
             _la = _input.LA(1);
           }
           }
         }
 
-        setState(439);
+        setState(440);
         match(RP);
         }
         break;
       default:
         throw new NoViableAltException(this);
       }
-      setState(442);
-      match(ARROW);
       setState(443);
+      match(ARROW);
+      setState(444);
       block();
       }
     }
@@ -2833,16 +2850,16 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(446);
+      setState(447);
       _la = _input.LA(1);
       if (_la==TYPE) {
         {
-        setState(445);
+        setState(446);
         decltype();
         }
       }
 
-      setState(448);
+      setState(449);
       match(ID);
       }
     }
@@ -2885,33 +2902,33 @@ class PainlessParser extends Parser {
     FuncrefContext _localctx = new FuncrefContext(_ctx, getState());
     enterRule(_localctx, 50, RULE_funcref);
     try {
-      setState(454);
+      setState(455);
       switch ( getInterpreter().adaptivePredict(_input,39,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(450);
+        setState(451);
         classFuncref();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(451);
+        setState(452);
         constructorFuncref();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(452);
+        setState(453);
         capturingFuncref();
         }
         break;
       case 4:
         enterOuterAlt(_localctx, 4);
         {
-        setState(453);
+        setState(454);
         localFuncref();
         }
         break;
@@ -2949,11 +2966,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(456);
-      match(TYPE);
       setState(457);
-      match(REF);
+      match(TYPE);
       setState(458);
+      match(REF);
+      setState(459);
       match(ID);
       }
     }
@@ -2991,11 +3008,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(460);
-      decltype();
       setState(461);
-      match(REF);
+      decltype();
       setState(462);
+      match(REF);
+      setState(463);
       match(NEW);
       }
     }
@@ -3033,11 +3050,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(464);
-      match(ID);
       setState(465);
-      match(REF);
+      match(ID);
       setState(466);
+      match(REF);
+      setState(467);
       match(ID);
       }
     }
@@ -3073,11 +3090,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(468);
-      match(THIS);
       setState(469);
-      match(REF);
+      match(THIS);
       setState(470);
+      match(REF);
+      setState(471);
       match(ID);
       }
     }
@@ -3182,7 +3199,7 @@ class PainlessParser extends Parser {
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3O\u01db\4\2\t\2\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3P\u01dc\4\2\t\2\4"+
     "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
     "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
     "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+
@@ -3211,27 +3228,27 @@ class PainlessParser extends Parser {
     "\22\3\22\3\22\3\22\3\22\3\22\3\22\6\22\u0168\n\22\r\22\16\22\u0169\3\22"+
     "\3\22\7\22\u016e\n\22\f\22\16\22\u0171\13\22\5\22\u0173\n\22\5\22\u0175"+
     "\n\22\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23"+
-    "\3\23\3\23\3\23\3\23\3\23\5\23\u0189\n\23\3\24\3\24\3\24\3\24\5\24\u018f"+
-    "\n\24\3\25\3\25\3\25\3\25\3\25\5\25\u0196\n\25\3\26\3\26\3\26\3\26\3\27"+
-    "\3\27\3\27\3\27\7\27\u01a0\n\27\f\27\16\27\u01a3\13\27\5\27\u01a5\n\27"+
-    "\3\27\3\27\3\30\3\30\3\30\5\30\u01ac\n\30\3\31\3\31\3\31\3\31\3\31\7\31"+
-    "\u01b3\n\31\f\31\16\31\u01b6\13\31\5\31\u01b8\n\31\3\31\5\31\u01bb\n\31"+
-    "\3\31\3\31\3\31\3\32\5\32\u01c1\n\32\3\32\3\32\3\33\3\33\3\33\3\33\5\33"+
-    "\u01c9\n\33\3\34\3\34\3\34\3\34\3\35\3\35\3\35\3\35\3\36\3\36\3\36\3\36"+
-    "\3\37\3\37\3\37\3\37\3\37\2\3\36 \2\4\6\b\n\f\16\20\22\24\26\30\32\34"+
-    "\36 \"$&(*,.\60\62\64\668:<\2\f\3\28C\3\2\35\37\3\2 !\3\2\"$\3\2%(\3\2"+
-    "),\3\2\66\67\3\2DG\4\2\33\34 !\3\2NO\u0207\2A\3\2\2\2\4L\3\2\2\2\6Q\3"+
-    "\2\2\2\b\u00af\3\2\2\2\n\u00b3\3\2\2\2\f\u00b5\3\2\2\2\16\u00be\3\2\2"+
-    "\2\20\u00c2\3\2\2\2\22\u00c4\3\2\2\2\24\u00c6\3\2\2\2\26\u00cf\3\2\2\2"+
-    "\30\u00d7\3\2\2\2\32\u00dc\3\2\2\2\34\u00e8\3\2\2\2\36\u00f3\3\2\2\2 "+
-    "\u0150\3\2\2\2\"\u0174\3\2\2\2$\u0188\3\2\2\2&\u018e\3\2\2\2(\u0195\3"+
-    "\2\2\2*\u0197\3\2\2\2,\u019b\3\2\2\2.\u01ab\3\2\2\2\60\u01ba\3\2\2\2\62"+
-    "\u01c0\3\2\2\2\64\u01c8\3\2\2\2\66\u01ca\3\2\2\28\u01ce\3\2\2\2:\u01d2"+
-    "\3\2\2\2<\u01d6\3\2\2\2>@\5\4\3\2?>\3\2\2\2@C\3\2\2\2A?\3\2\2\2AB\3\2"+
+    "\3\23\3\23\3\23\3\23\3\23\3\23\5\23\u018a\n\23\3\24\3\24\3\24\3\24\5\24"+
+    "\u0190\n\24\3\25\3\25\3\25\3\25\3\25\5\25\u0197\n\25\3\26\3\26\3\26\3"+
+    "\26\3\27\3\27\3\27\3\27\7\27\u01a1\n\27\f\27\16\27\u01a4\13\27\5\27\u01a6"+
+    "\n\27\3\27\3\27\3\30\3\30\3\30\5\30\u01ad\n\30\3\31\3\31\3\31\3\31\3\31"+
+    "\7\31\u01b4\n\31\f\31\16\31\u01b7\13\31\5\31\u01b9\n\31\3\31\5\31\u01bc"+
+    "\n\31\3\31\3\31\3\31\3\32\5\32\u01c2\n\32\3\32\3\32\3\33\3\33\3\33\3\33"+
+    "\5\33\u01ca\n\33\3\34\3\34\3\34\3\34\3\35\3\35\3\35\3\35\3\36\3\36\3\36"+
+    "\3\36\3\37\3\37\3\37\3\37\3\37\2\3\36 \2\4\6\b\n\f\16\20\22\24\26\30\32"+
+    "\34\36 \"$&(*,.\60\62\64\668:<\2\f\3\28C\3\2\35\37\3\2 !\3\2\"$\3\2%("+
+    "\3\2),\3\2\66\67\3\2DG\4\2\33\34 !\3\2OP\u0209\2A\3\2\2\2\4L\3\2\2\2\6"+
+    "Q\3\2\2\2\b\u00af\3\2\2\2\n\u00b3\3\2\2\2\f\u00b5\3\2\2\2\16\u00be\3\2"+
+    "\2\2\20\u00c2\3\2\2\2\22\u00c4\3\2\2\2\24\u00c6\3\2\2\2\26\u00cf\3\2\2"+
+    "\2\30\u00d7\3\2\2\2\32\u00dc\3\2\2\2\34\u00e8\3\2\2\2\36\u00f3\3\2\2\2"+
+    " \u0150\3\2\2\2\"\u0174\3\2\2\2$\u0189\3\2\2\2&\u018f\3\2\2\2(\u0196\3"+
+    "\2\2\2*\u0198\3\2\2\2,\u019c\3\2\2\2.\u01ac\3\2\2\2\60\u01bb\3\2\2\2\62"+
+    "\u01c1\3\2\2\2\64\u01c9\3\2\2\2\66\u01cb\3\2\2\28\u01cf\3\2\2\2:\u01d3"+
+    "\3\2\2\2<\u01d7\3\2\2\2>@\5\4\3\2?>\3\2\2\2@C\3\2\2\2A?\3\2\2\2AB\3\2"+
     "\2\2BG\3\2\2\2CA\3\2\2\2DF\5\b\5\2ED\3\2\2\2FI\3\2\2\2GE\3\2\2\2GH\3\2"+
-    "\2\2HJ\3\2\2\2IG\3\2\2\2JK\7\2\2\3K\3\3\2\2\2LM\5\26\f\2MN\7M\2\2NO\5"+
-    "\6\4\2OP\5\f\7\2P\5\3\2\2\2Q]\7\t\2\2RS\5\26\f\2SZ\7M\2\2TU\7\f\2\2UV"+
-    "\5\26\f\2VW\7M\2\2WY\3\2\2\2XT\3\2\2\2Y\\\3\2\2\2ZX\3\2\2\2Z[\3\2\2\2"+
+    "\2\2HJ\3\2\2\2IG\3\2\2\2JK\7\2\2\3K\3\3\2\2\2LM\5\26\f\2MN\7N\2\2NO\5"+
+    "\6\4\2OP\5\f\7\2P\5\3\2\2\2Q]\7\t\2\2RS\5\26\f\2SZ\7N\2\2TU\7\f\2\2UV"+
+    "\5\26\f\2VW\7N\2\2WY\3\2\2\2XT\3\2\2\2Y\\\3\2\2\2ZX\3\2\2\2Z[\3\2\2\2"+
     "[^\3\2\2\2\\Z\3\2\2\2]R\3\2\2\2]^\3\2\2\2^_\3\2\2\2_`\7\n\2\2`\7\3\2\2"+
     "\2ab\7\16\2\2bc\7\t\2\2cd\5\36\20\2de\7\n\2\2ei\5\n\6\2fg\7\17\2\2gj\5"+
     "\n\6\2hj\6\5\2\2if\3\2\2\2ih\3\2\2\2j\u00b0\3\2\2\2kl\7\20\2\2lm\7\t\2"+
@@ -3244,7 +3261,7 @@ class PainlessParser extends Parser {
     "\2\2\2\u0087\u0088\3\2\2\2\u0088\u008b\7\n\2\2\u0089\u008c\5\n\6\2\u008a"+
     "\u008c\5\16\b\2\u008b\u0089\3\2\2\2\u008b\u008a\3\2\2\2\u008c\u00b0\3"+
     "\2\2\2\u008d\u008e\7\22\2\2\u008e\u008f\7\t\2\2\u008f\u0090\5\26\f\2\u0090"+
-    "\u0091\7M\2\2\u0091\u0092\7\63\2\2\u0092\u0093\5\36\20\2\u0093\u0094\7"+
+    "\u0091\7N\2\2\u0091\u0092\7\63\2\2\u0092\u0093\5\36\20\2\u0093\u0094\7"+
     "\n\2\2\u0094\u0095\5\n\6\2\u0095\u00b0\3\2\2\2\u0096\u0097\5\24\13\2\u0097"+
     "\u0098\5\34\17\2\u0098\u00b0\3\2\2\2\u0099\u009a\7\23\2\2\u009a\u00b0"+
     "\5\34\17\2\u009b\u009c\7\24\2\2\u009c\u00b0\5\34\17\2\u009d\u009e\7\25"+
@@ -3266,12 +3283,12 @@ class PainlessParser extends Parser {
     "\5\36\20\2\u00c5\23\3\2\2\2\u00c6\u00c7\5\26\f\2\u00c7\u00cc\5\30\r\2"+
     "\u00c8\u00c9\7\f\2\2\u00c9\u00cb\5\30\r\2\u00ca\u00c8\3\2\2\2\u00cb\u00ce"+
     "\3\2\2\2\u00cc\u00ca\3\2\2\2\u00cc\u00cd\3\2\2\2\u00cd\25\3\2\2\2\u00ce"+
-    "\u00cc\3\2\2\2\u00cf\u00d4\7L\2\2\u00d0\u00d1\7\7\2\2\u00d1\u00d3\7\b"+
+    "\u00cc\3\2\2\2\u00cf\u00d4\7M\2\2\u00d0\u00d1\7\7\2\2\u00d1\u00d3\7\b"+
     "\2\2\u00d2\u00d0\3\2\2\2\u00d3\u00d6\3\2\2\2\u00d4\u00d2\3\2\2\2\u00d4"+
-    "\u00d5\3\2\2\2\u00d5\27\3\2\2\2\u00d6\u00d4\3\2\2\2\u00d7\u00da\7M\2\2"+
+    "\u00d5\3\2\2\2\u00d5\27\3\2\2\2\u00d6\u00d4\3\2\2\2\u00d7\u00da\7N\2\2"+
     "\u00d8\u00d9\78\2\2\u00d9\u00db\5\36\20\2\u00da\u00d8\3\2\2\2\u00da\u00db"+
     "\3\2\2\2\u00db\31\3\2\2\2\u00dc\u00dd\7\30\2\2\u00dd\u00de\7\t\2\2\u00de"+
-    "\u00df\7L\2\2\u00df\u00e0\7M\2\2\u00e0\u00e1\7\n\2\2\u00e1\u00e2\5\f\7"+
+    "\u00df\7M\2\2\u00df\u00e0\7N\2\2\u00e0\u00e1\7\n\2\2\u00e1\u00e2\5\f\7"+
     "\2\u00e2\33\3\2\2\2\u00e3\u00e9\7\r\2\2\u00e4\u00e9\7\2\2\3\u00e5\u00e6"+
     "\b\17\1\2\u00e6\u00e7\7\6\2\2\u00e7\u00e9\b\17\1\2\u00e8\u00e3\3\2\2\2"+
     "\u00e8\u00e4\3\2\2\2\u00e8\u00e5\3\2\2\2\u00e9\35\3\2\2\2\u00ea\u00eb"+
@@ -3303,9 +3320,9 @@ class PainlessParser extends Parser {
     "\2\2\u0135\u0151\5\"\22\2\u0136\u0137\6\21\17\3\u0137\u0138\5\"\22\2\u0138"+
     "\u0139\t\b\2\2\u0139\u0151\3\2\2\2\u013a\u013b\6\21\20\3\u013b\u0151\5"+
     "\"\22\2\u013c\u013d\6\21\21\3\u013d\u013e\t\t\2\2\u013e\u0151\b\21\1\2"+
-    "\u013f\u0140\6\21\22\3\u0140\u0141\7I\2\2\u0141\u0151\b\21\1\2\u0142\u0143"+
-    "\6\21\23\3\u0143\u0144\7J\2\2\u0144\u0151\b\21\1\2\u0145\u0146\6\21\24"+
-    "\3\u0146\u0147\7K\2\2\u0147\u0151\b\21\1\2\u0148\u0149\6\21\25\3\u0149"+
+    "\u013f\u0140\6\21\22\3\u0140\u0141\7J\2\2\u0141\u0151\b\21\1\2\u0142\u0143"+
+    "\6\21\23\3\u0143\u0144\7K\2\2\u0144\u0151\b\21\1\2\u0145\u0146\6\21\24"+
+    "\3\u0146\u0147\7L\2\2\u0147\u0151\b\21\1\2\u0148\u0149\6\21\25\3\u0149"+
     "\u014a\t\n\2\2\u014a\u0151\5 \21\2\u014b\u014c\7\t\2\2\u014c\u014d\5\26"+
     "\f\2\u014d\u014e\7\n\2\2\u014e\u014f\5 \21\2\u014f\u0151\3\2\2\2\u0150"+
     "\u0133\3\2\2\2\u0150\u0136\3\2\2\2\u0150\u013a\3\2\2\2\u0150\u013c\3\2"+
@@ -3316,7 +3333,7 @@ class PainlessParser extends Parser {
     "\u015a\5\26\f\2\u015a\u015e\5(\25\2\u015b\u015d\5&\24\2\u015c\u015b\3"+
     "\2\2\2\u015d\u0160\3\2\2\2\u015e\u015c\3\2\2\2\u015e\u015f\3\2\2\2\u015f"+
     "\u0175\3\2\2\2\u0160\u015e\3\2\2\2\u0161\u0162\7\26\2\2\u0162\u0167\7"+
-    "L\2\2\u0163\u0164\7\7\2\2\u0164\u0165\5\36\20\2\u0165\u0166\7\b\2\2\u0166"+
+    "M\2\2\u0163\u0164\7\7\2\2\u0164\u0165\5\36\20\2\u0165\u0166\7\b\2\2\u0166"+
     "\u0168\3\2\2\2\u0167\u0163\3\2\2\2\u0168\u0169\3\2\2\2\u0169\u0167\3\2"+
     "\2\2\u0169\u016a\3\2\2\2\u016a\u0172\3\2\2\2\u016b\u016f\5(\25\2\u016c"+
     "\u016e\5&\24\2\u016d\u016c\3\2\2\2\u016e\u0171\3\2\2\2\u016f\u016d\3\2"+
@@ -3324,41 +3341,42 @@ class PainlessParser extends Parser {
     "\u016b\3\2\2\2\u0172\u0173\3\2\2\2\u0173\u0175\3\2\2\2\u0174\u0152\3\2"+
     "\2\2\u0174\u0159\3\2\2\2\u0174\u0161\3\2\2\2\u0175#\3\2\2\2\u0176\u0177"+
     "\6\23\26\3\u0177\u0178\7\t\2\2\u0178\u0179\5\36\20\2\u0179\u017a\7\n\2"+
-    "\2\u017a\u017b\b\23\1\2\u017b\u0189\3\2\2\2\u017c\u017d\6\23\27\3\u017d"+
-    "\u017e\7\t\2\2\u017e\u017f\5 \21\2\u017f\u0180\7\n\2\2\u0180\u0189\3\2"+
-    "\2\2\u0181\u0189\7H\2\2\u0182\u0189\7M\2\2\u0183\u0184\7M\2\2\u0184\u0189"+
-    "\5,\27\2\u0185\u0186\7\26\2\2\u0186\u0187\7L\2\2\u0187\u0189\5,\27\2\u0188"+
-    "\u0176\3\2\2\2\u0188\u017c\3\2\2\2\u0188\u0181\3\2\2\2\u0188\u0182\3\2"+
-    "\2\2\u0188\u0183\3\2\2\2\u0188\u0185\3\2\2\2\u0189%\3\2\2\2\u018a\u018b"+
-    "\6\24\30\3\u018b\u018f\5(\25\2\u018c\u018d\6\24\31\3\u018d\u018f\5*\26"+
-    "\2\u018e\u018a\3\2\2\2\u018e\u018c\3\2\2\2\u018f\'\3\2\2\2\u0190\u0191"+
-    "\7\13\2\2\u0191\u0192\7O\2\2\u0192\u0196\5,\27\2\u0193\u0194\7\13\2\2"+
-    "\u0194\u0196\t\13\2\2\u0195\u0190\3\2\2\2\u0195\u0193\3\2\2\2\u0196)\3"+
-    "\2\2\2\u0197\u0198\7\7\2\2\u0198\u0199\5\36\20\2\u0199\u019a\7\b\2\2\u019a"+
-    "+\3\2\2\2\u019b\u01a4\7\t\2\2\u019c\u01a1\5.\30\2\u019d\u019e\7\f\2\2"+
-    "\u019e\u01a0\5.\30\2\u019f\u019d\3\2\2\2\u01a0\u01a3\3\2\2\2\u01a1\u019f"+
-    "\3\2\2\2\u01a1\u01a2\3\2\2\2\u01a2\u01a5\3\2\2\2\u01a3\u01a1\3\2\2\2\u01a4"+
-    "\u019c\3\2\2\2\u01a4\u01a5\3\2\2\2\u01a5\u01a6\3\2\2\2\u01a6\u01a7\7\n"+
-    "\2\2\u01a7-\3\2\2\2\u01a8\u01ac\5\36\20\2\u01a9\u01ac\5\60\31\2\u01aa"+
-    "\u01ac\5\64\33\2\u01ab\u01a8\3\2\2\2\u01ab\u01a9\3\2\2\2\u01ab\u01aa\3"+
-    "\2\2\2\u01ac/\3\2\2\2\u01ad\u01bb\5\62\32\2\u01ae\u01b7\7\t\2\2\u01af"+
-    "\u01b4\5\62\32\2\u01b0\u01b1\7\f\2\2\u01b1\u01b3\5\62\32\2\u01b2\u01b0"+
-    "\3\2\2\2\u01b3\u01b6\3\2\2\2\u01b4\u01b2\3\2\2\2\u01b4\u01b5\3\2\2\2\u01b5"+
-    "\u01b8\3\2\2\2\u01b6\u01b4\3\2\2\2\u01b7\u01af\3\2\2\2\u01b7\u01b8\3\2"+
-    "\2\2\u01b8\u01b9\3\2\2\2\u01b9\u01bb\7\n\2\2\u01ba\u01ad\3\2\2\2\u01ba"+
-    "\u01ae\3\2\2\2\u01bb\u01bc\3\2\2\2\u01bc\u01bd\7\65\2\2\u01bd\u01be\5"+
-    "\f\7\2\u01be\61\3\2\2\2\u01bf\u01c1\5\26\f\2\u01c0\u01bf\3\2\2\2\u01c0"+
-    "\u01c1\3\2\2\2\u01c1\u01c2\3\2\2\2\u01c2\u01c3\7M\2\2\u01c3\63\3\2\2\2"+
-    "\u01c4\u01c9\5\66\34\2\u01c5\u01c9\58\35\2\u01c6\u01c9\5:\36\2\u01c7\u01c9"+
-    "\5<\37\2\u01c8\u01c4\3\2\2\2\u01c8\u01c5\3\2\2\2\u01c8\u01c6\3\2\2\2\u01c8"+
-    "\u01c7\3\2\2\2\u01c9\65\3\2\2\2\u01ca\u01cb\7L\2\2\u01cb\u01cc\7\64\2"+
-    "\2\u01cc\u01cd\7M\2\2\u01cd\67\3\2\2\2\u01ce\u01cf\5\26\f\2\u01cf\u01d0"+
-    "\7\64\2\2\u01d0\u01d1\7\26\2\2\u01d19\3\2\2\2\u01d2\u01d3\7M\2\2\u01d3"+
-    "\u01d4\7\64\2\2\u01d4\u01d5\7M\2\2\u01d5;\3\2\2\2\u01d6\u01d7\7\32\2\2"+
-    "\u01d7\u01d8\7\64\2\2\u01d8\u01d9\7M\2\2\u01d9=\3\2\2\2*AGZ]iq~\u0082"+
-    "\u0086\u008b\u00a6\u00af\u00b3\u00b9\u00c2\u00cc\u00d4\u00da\u00e8\u00f3"+
-    "\u012e\u0130\u0150\u0156\u015e\u0169\u016f\u0172\u0174\u0188\u018e\u0195"+
-    "\u01a1\u01a4\u01ab\u01b4\u01b7\u01ba\u01c0\u01c8";
+    "\2\u017a\u017b\b\23\1\2\u017b\u018a\3\2\2\2\u017c\u017d\6\23\27\3\u017d"+
+    "\u017e\7\t\2\2\u017e\u017f\5 \21\2\u017f\u0180\7\n\2\2\u0180\u018a\3\2"+
+    "\2\2\u0181\u018a\7H\2\2\u0182\u018a\7I\2\2\u0183\u018a\7N\2\2\u0184\u0185"+
+    "\7N\2\2\u0185\u018a\5,\27\2\u0186\u0187\7\26\2\2\u0187\u0188\7M\2\2\u0188"+
+    "\u018a\5,\27\2\u0189\u0176\3\2\2\2\u0189\u017c\3\2\2\2\u0189\u0181\3\2"+
+    "\2\2\u0189\u0182\3\2\2\2\u0189\u0183\3\2\2\2\u0189\u0184\3\2\2\2\u0189"+
+    "\u0186\3\2\2\2\u018a%\3\2\2\2\u018b\u018c\6\24\30\3\u018c\u0190\5(\25"+
+    "\2\u018d\u018e\6\24\31\3\u018e\u0190\5*\26\2\u018f\u018b\3\2\2\2\u018f"+
+    "\u018d\3\2\2\2\u0190\'\3\2\2\2\u0191\u0192\7\13\2\2\u0192\u0193\7P\2\2"+
+    "\u0193\u0197\5,\27\2\u0194\u0195\7\13\2\2\u0195\u0197\t\13\2\2\u0196\u0191"+
+    "\3\2\2\2\u0196\u0194\3\2\2\2\u0197)\3\2\2\2\u0198\u0199\7\7\2\2\u0199"+
+    "\u019a\5\36\20\2\u019a\u019b\7\b\2\2\u019b+\3\2\2\2\u019c\u01a5\7\t\2"+
+    "\2\u019d\u01a2\5.\30\2\u019e\u019f\7\f\2\2\u019f\u01a1\5.\30\2\u01a0\u019e"+
+    "\3\2\2\2\u01a1\u01a4\3\2\2\2\u01a2\u01a0\3\2\2\2\u01a2\u01a3\3\2\2\2\u01a3"+
+    "\u01a6\3\2\2\2\u01a4\u01a2\3\2\2\2\u01a5\u019d\3\2\2\2\u01a5\u01a6\3\2"+
+    "\2\2\u01a6\u01a7\3\2\2\2\u01a7\u01a8\7\n\2\2\u01a8-\3\2\2\2\u01a9\u01ad"+
+    "\5\36\20\2\u01aa\u01ad\5\60\31\2\u01ab\u01ad\5\64\33\2\u01ac\u01a9\3\2"+
+    "\2\2\u01ac\u01aa\3\2\2\2\u01ac\u01ab\3\2\2\2\u01ad/\3\2\2\2\u01ae\u01bc"+
+    "\5\62\32\2\u01af\u01b8\7\t\2\2\u01b0\u01b5\5\62\32\2\u01b1\u01b2\7\f\2"+
+    "\2\u01b2\u01b4\5\62\32\2\u01b3\u01b1\3\2\2\2\u01b4\u01b7\3\2\2\2\u01b5"+
+    "\u01b3\3\2\2\2\u01b5\u01b6\3\2\2\2\u01b6\u01b9\3\2\2\2\u01b7\u01b5\3\2"+
+    "\2\2\u01b8\u01b0\3\2\2\2\u01b8\u01b9\3\2\2\2\u01b9\u01ba\3\2\2\2\u01ba"+
+    "\u01bc\7\n\2\2\u01bb\u01ae\3\2\2\2\u01bb\u01af\3\2\2\2\u01bc\u01bd\3\2"+
+    "\2\2\u01bd\u01be\7\65\2\2\u01be\u01bf\5\f\7\2\u01bf\61\3\2\2\2\u01c0\u01c2"+
+    "\5\26\f\2\u01c1\u01c0\3\2\2\2\u01c1\u01c2\3\2\2\2\u01c2\u01c3\3\2\2\2"+
+    "\u01c3\u01c4\7N\2\2\u01c4\63\3\2\2\2\u01c5\u01ca\5\66\34\2\u01c6\u01ca"+
+    "\58\35\2\u01c7\u01ca\5:\36\2\u01c8\u01ca\5<\37\2\u01c9\u01c5\3\2\2\2\u01c9"+
+    "\u01c6\3\2\2\2\u01c9\u01c7\3\2\2\2\u01c9\u01c8\3\2\2\2\u01ca\65\3\2\2"+
+    "\2\u01cb\u01cc\7M\2\2\u01cc\u01cd\7\64\2\2\u01cd\u01ce\7N\2\2\u01ce\67"+
+    "\3\2\2\2\u01cf\u01d0\5\26\f\2\u01d0\u01d1\7\64\2\2\u01d1\u01d2\7\26\2"+
+    "\2\u01d29\3\2\2\2\u01d3\u01d4\7N\2\2\u01d4\u01d5\7\64\2\2\u01d5\u01d6"+
+    "\7N\2\2\u01d6;\3\2\2\2\u01d7\u01d8\7\32\2\2\u01d8\u01d9\7\64\2\2\u01d9"+
+    "\u01da\7N\2\2\u01da=\3\2\2\2*AGZ]iq~\u0082\u0086\u008b\u00a6\u00af\u00b3"+
+    "\u00b9\u00c2\u00cc\u00d4\u00da\u00e8\u00f3\u012e\u0130\u0150\u0156\u015e"+
+    "\u0169\u016f\u0172\u0174\u0189\u018f\u0196\u01a2\u01a5\u01ac\u01b5\u01b8"+
+    "\u01bb\u01c1\u01c9";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
index 4608a74847e..4b7daa213b3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
@@ -333,6 +333,13 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitString(PainlessParser.StringContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitRegex(PainlessParser.RegexContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
index bd2249617a0..f39bc84a7af 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
@@ -319,6 +319,13 @@ interface PainlessParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitString(PainlessParser.StringContext ctx);
+  /**
+   * Visit a parse tree produced by the {@code regex}
+   * labeled alternative in {@link PainlessParser#primary}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitRegex(PainlessParser.RegexContext ctx);
   /**
    * Visit a parse tree produced by the {@code variable}
    * labeled alternative in {@link PainlessParser#primary}.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SlashStrategy.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SlashStrategy.java
new file mode 100644
index 00000000000..720259d5316
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SlashStrategy.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.antlr;
+
+import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.TokenFactory;
+
+/**
+ * Utility to figure out if a {@code /} is division or the start of a regex literal.
+ */
+public class SlashStrategy {
+    public static boolean slashIsRegex(TokenFactory<?> factory) {
+        StashingTokenFactory<?> stashingFactory = (StashingTokenFactory<?>) factory;
+        Token lastToken = stashingFactory.getLastToken();
+        if (lastToken == null) {
+            return true;
+        }
+        switch (lastToken.getType()) {
+        case PainlessLexer.RBRACE:
+        case PainlessLexer.RP:
+        case PainlessLexer.OCTAL:
+        case PainlessLexer.HEX:
+        case PainlessLexer.INTEGER:
+        case PainlessLexer.DECIMAL:
+        case PainlessLexer.ID:
+        case PainlessLexer.DOTINTEGER:
+        case PainlessLexer.DOTID:
+            return false;
+        default:
+            return true;
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/StashingTokenFactory.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/StashingTokenFactory.java
new file mode 100644
index 00000000000..4f22137e5fb
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/StashingTokenFactory.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.antlr;
+
+import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.Lexer;
+import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.TokenFactory;
+import org.antlr.v4.runtime.TokenSource;
+import org.antlr.v4.runtime.misc.Pair;
+
+/**
+ * Token factory that preseres that last non-whitespace token so you can do token level lookbehind in the lexer.
+ */
+public class StashingTokenFactory<T extends Token> implements TokenFactory<T> {
+    private final TokenFactory<T> delegate;
+
+    private T lastToken;
+
+    public StashingTokenFactory(TokenFactory<T> delegate) {
+        this.delegate = delegate;
+    }
+
+    public T getLastToken() {
+        return lastToken;
+    }
+
+    @Override
+    public T create(Pair<TokenSource, CharStream> source, int type, String text, int channel, int start, int stop, int line,
+            int charPositionInLine) {
+        return maybeStash(delegate.create(source, type, text, channel, start, stop, line, charPositionInLine));
+    }
+
+    @Override
+    public T create(int type, String text) {
+        return maybeStash(delegate.create(type, text));
+    }
+
+    private T maybeStash(T token) {
+        if (token.getChannel() == Lexer.DEFAULT_TOKEN_CHANNEL) {
+            lastToken = token;
+        }
+        return token;
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index 6f00a337386..0f6f7c8f2d5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -84,6 +84,7 @@ import org.elasticsearch.painless.antlr.PainlessParser.ParametersContext;
 import org.elasticsearch.painless.antlr.PainlessParser.PostContext;
 import org.elasticsearch.painless.antlr.PainlessParser.PreContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ReadContext;
+import org.elasticsearch.painless.antlr.PainlessParser.RegexContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ReturnContext;
 import org.elasticsearch.painless.antlr.PainlessParser.SecondaryContext;
 import org.elasticsearch.painless.antlr.PainlessParser.SingleContext;
@@ -124,6 +125,7 @@ import org.elasticsearch.painless.node.LCast;
 import org.elasticsearch.painless.node.LField;
 import org.elasticsearch.painless.node.LNewArray;
 import org.elasticsearch.painless.node.LNewObj;
+import org.elasticsearch.painless.node.LRegex;
 import org.elasticsearch.painless.node.LStatic;
 import org.elasticsearch.painless.node.LString;
 import org.elasticsearch.painless.node.LVariable;
@@ -826,6 +828,15 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         return links;
     }
 
+    @Override
+    public Object visitRegex(RegexContext ctx) {
+        String pattern = ctx.REGEX().getText().substring(1, ctx.REGEX().getText().length() - 1);
+        List<ALink> links = new ArrayList<>();
+        links.add(new LRegex(location(ctx), pattern));
+
+        return links;
+    }
+
     @Override
     public Object visitVariable(VariableContext ctx) {
         String name = ctx.ID().getText();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LRegex.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LRegex.java
new file mode 100644
index 00000000000..d91bc3394ee
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LRegex.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Locals.Constant;
+
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+import org.elasticsearch.painless.Location;
+import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.WriterConstants;
+
+/**
+ * Represents a regex constant. All regexes are constants.
+ */
+public final class LRegex extends ALink {
+    private static final Definition.Type PATTERN_TYPE = Definition.getType("Pattern");
+
+    private final String pattern;
+    private Constant constant;
+
+    public LRegex(Location location, String pattern) {
+        super(location, 1);
+        this.pattern = pattern;
+        try {
+            // Compile the pattern early after parsing so we can throw an error to the user with the location
+            Pattern.compile(pattern);
+        } catch (PatternSyntaxException e) {
+            throw createError(e);
+        }
+    }
+
+    @Override
+    ALink analyze(Locals locals) {
+        if (before != null) {
+            throw createError(new IllegalArgumentException("Illegal Regex constant [" + pattern + "]."));
+        } else if (store) {
+            throw createError(new IllegalArgumentException("Cannot write to Regex constant [" + pattern + "]."));
+        } else if (!load) {
+            throw createError(new IllegalArgumentException("Regex constant may only be read [" + pattern + "]."));
+        }
+
+        constant = locals.addConstant(location, PATTERN_TYPE, "regexAt$" + location.getOffset(), this::initializeConstant);
+        after = PATTERN_TYPE;
+
+        return this;
+    }
+
+    @Override
+    void write(MethodWriter writer) {
+        // Do nothing.
+    }
+
+    @Override
+    void load(MethodWriter writer) {
+        writer.writeDebugInfo(location);
+        writer.getStatic(WriterConstants.CLASS_TYPE, constant.name, PATTERN_TYPE.type);
+    }
+
+    @Override
+    void store(MethodWriter writer) {
+        throw createError(new IllegalStateException("Illegal tree structure."));
+    }
+
+    private void initializeConstant(MethodWriter writer) {
+        writer.push(pattern);
+        writer.invokeStatic(PATTERN_TYPE.type, WriterConstants.PATTERN_COMPILE);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index 57d80a51ea9..0cdde13fce7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -30,7 +30,9 @@ import org.elasticsearch.painless.Locals.FunctionReserved;
 import org.elasticsearch.painless.Locals.Variable;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.WriterConstants;
 import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.Handle;
 import org.objectweb.asm.Opcodes;
 
 import java.lang.invoke.MethodType;
@@ -40,6 +42,8 @@ import java.util.BitSet;
 import java.util.Collections;
 import java.util.List;
 
+import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE;
+
 /**
  * Represents a user-defined function.
  */
@@ -137,6 +141,9 @@ public class SFunction extends AStatement {
         }
 
         locals.decrementScope();
+
+        String staticHandleFieldName = Def.getUserFunctionHandleFieldName(name, parameters.size());
+        locals.addConstant(location, WriterConstants.METHOD_HANDLE_TYPE, staticHandleFieldName, this::initializeConstant);
     }
     
     /** Writes the function to given ClassWriter. */
@@ -174,8 +181,13 @@ public class SFunction extends AStatement {
             }
         }
     }
-    
-    String getStaticHandleFieldName() {
-        return Def.getUserFunctionHandleFieldName(name, parameters.size());
+
+    private void initializeConstant(MethodWriter writer) {
+        final Handle handle = new Handle(Opcodes.H_INVOKESTATIC,
+                CLASS_TYPE.getInternalName(),
+                name,
+                method.method.getDescriptor(),
+                false);
+        writer.push(handle);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index 338652bd56b..aeda0387297 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -23,6 +23,7 @@ import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.MethodKey;
 import org.elasticsearch.painless.Executable;
 import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Locals.Constant;
 import org.elasticsearch.painless.Locals.ExecuteReserved;
 import org.elasticsearch.painless.Locals.Variable;
 import org.elasticsearch.painless.WriterConstants;
@@ -30,7 +31,6 @@ import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
 import org.objectweb.asm.ClassWriter;
-import org.objectweb.asm.Handle;
 import org.objectweb.asm.Opcodes;
 
 import java.util.BitSet;
@@ -155,33 +155,31 @@ public final class SSource extends AStatement {
             function.write(writer, expressions);
         }
 
-        // Write a static field with Handle (function reference) for every function:
-        if (!functions.isEmpty()) {
-            for (SFunction function : functions) {
-                writer.visitField(Opcodes.ACC_FINAL | Opcodes.ACC_PRIVATE | Opcodes.ACC_STATIC, 
-                        function.getStaticHandleFieldName(), 
-                        WriterConstants.METHOD_HANDLE_TYPE.getDescriptor(), 
-                        null, 
+        // Write the constants
+        if (false == locals.getConstants().isEmpty()) {
+            // Fields
+            for (Constant constant : locals.getConstants()) {
+                writer.visitField(
+                        Opcodes.ACC_FINAL | Opcodes.ACC_PRIVATE | Opcodes.ACC_STATIC,
+                        constant.name,
+                        constant.type.getDescriptor(),
+                        null,
                         null).visitEnd();
             }
+
+            // Initialize the constants in a static initializer
             final MethodWriter clinit = new MethodWriter(Opcodes.ACC_STATIC, 
-                                                   WriterConstants.CLINIT, writer, expressions);
-            for (SFunction function : functions) {
-                final Handle handle = new Handle(Opcodes.H_INVOKESTATIC, 
-                                           CLASS_TYPE.getInternalName(), 
-                                           function.name, 
-                                           function.method.method.getDescriptor(), 
-                                           false);
-                clinit.push(handle);
-                clinit.putStatic(CLASS_TYPE, 
-                                 function.getStaticHandleFieldName(), 
-                                 WriterConstants.METHOD_HANDLE_TYPE);
+                    WriterConstants.CLINIT, writer, expressions);
+            for (Constant constant : locals.getConstants()) {
+                constant.initializer.accept(clinit);
+                clinit.putStatic(CLASS_TYPE, constant.name, constant.type);
             }
             clinit.returnValue();
             clinit.endMethod();
         }
         
-        // End writing the class and store the generated bytes:
+        // End writing the class and store the generated bytes.
+
         writer.visitEnd();
         bytes = writer.toByteArray();
     }
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt
new file mode 100644
index 00000000000..b66793dcf5a
--- /dev/null
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt
@@ -0,0 +1,33 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#
+# Painless definition file. This defines the hierarchy of classes,
+# what methods and fields they have, etc.
+#
+
+class Pattern -> java.util.regex.Pattern extends Object {
+# Pattern compile(String)   Intentionally not included. We don't want dynamic patterns because they allow regexes to be generated per time
+#                           the script is run which is super slow. LRegex generates code that calls this method but it skips these checks.
+  Matcher matcher(CharSequence)
+}
+
+class Matcher -> java.util.regex.Matcher extends Object {
+  boolean matches()
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
index 397a8856231..e6a5537194d 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
@@ -79,4 +79,10 @@ public class ArrayTests extends ScriptTestCase {
             "int total = 0; for (int x = 0; x < a.length; x++) { total += a[x]; } return total;"));
     }
 
+    /**
+     * Make sure we don't try and convert the {@code /} after the {@code ]} into a regex....
+     */
+    public void testDivideArray() {
+        assertEquals(1, exec("def[] x = new def[1]; x[0] = 2; return x[0] / 2"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java
new file mode 100644
index 00000000000..69a81e0d77e
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+public class RegexTests extends ScriptTestCase {
+    public void testPatternAfterReturn() {
+        assertEquals(true, exec("return /foo/.matcher(\"foo\").matches()"));
+        assertEquals(false, exec("return /foo/.matcher(\"bar\").matches()"));
+    }
+
+    public void testSlashesEscapePattern() {
+        assertEquals(true, exec("return /\\/\\//.matcher(\"//\").matches()"));
+    }
+
+    public void testPatternAfterAssignment() {
+        assertEquals(true, exec("def a = /foo/; return a.matcher(\"foo\").matches()"));
+    }
+
+    public void testPatternInIfStement() {
+        assertEquals(true, exec("if (/foo/.matcher(\"foo\").matches()) { return true } else { return false }"));
+    }
+
+    public void testPatternAfterInfixBoolean() {
+        assertEquals(true, exec("return false || /foo/.matcher(\"foo\").matches()"));
+        assertEquals(true, exec("return true && /foo/.matcher(\"foo\").matches()"));
+    }
+
+    public void testPatternAfterUnaryNotBoolean() {
+        assertEquals(false, exec("return !/foo/.matcher(\"foo\").matches()"));
+        assertEquals(true, exec("return !/foo/.matcher(\"bar\").matches()"));
+    }
+
+    public void testInTernaryCondition()  {
+        assertEquals(true, exec("return /foo/.matcher(\"foo\").matches() ? true : false"));
+        assertEquals(1, exec("def i = 0; i += /foo/.matcher(\"foo\").matches() ? 1 : 1; return i"));
+    }
+
+    public void testInTernaryTrueArm()  {
+        assertEquals(true, exec("def i = true; return i ? /foo/.matcher(\"foo\").matches() : false"));
+    }
+
+    public void testInTernaryFalseArm()  {
+        assertEquals(true, exec("def i = false; return i ? false : /foo/.matcher(\"foo\").matches()"));
+    }
+
+    public void testRegexInFunction() {
+        assertEquals(true, exec("boolean m(String s) {/foo/.matcher(s).matches()} m(\"foo\")"));
+    }
+
+    public void testReturnRegexFromFunction() {
+        assertEquals(true, exec("Pattern m(boolean a) {a ? /foo/ : /bar/} m(true).matcher(\"foo\").matches()"));
+    }
+
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
index 1c54f2727b5..587e3e9fd09 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
@@ -22,6 +22,7 @@ package org.elasticsearch.painless;
 import java.lang.invoke.WrongMethodTypeException;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.regex.PatternSyntaxException;
 
 import static java.util.Collections.emptyMap;
 
@@ -211,4 +212,19 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
         });
         assertEquals("invalid sequence of tokens near ['}'].", e.getMessage());
     }
+
+    public void testCantUsePatternCompile() {
+        IllegalArgumentException e = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("Pattern.compile(\"aa\")");
+        });
+        assertEquals("Unknown call [compile] with [1] arguments on type [Pattern].", e.getMessage());
+    }
+
+    public void testBadRegexPattern() {
+        PatternSyntaxException e = expectScriptThrows(PatternSyntaxException.class, () -> {
+            exec("/\\ujjjj/"); // Invalid unicode
+        });
+        assertEquals("Illegal Unicode escape sequence near index 2\n\\ujjjj\n  ^", e.getMessage());
+    }
+
 }

From 7a226122e3fddac018e9475cfc32cc02f75dd552 Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Mon, 13 Jun 2016 23:04:03 +0300
Subject: [PATCH 0821/1311] MasterFaultDetection can leak an exception during
 shutdown

---
 .../discovery/zen/fd/MasterFaultDetection.java       | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
index 5f101cf3a1f..d6d523508e1 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
@@ -45,6 +45,7 @@ import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
 import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
@@ -196,14 +197,15 @@ public class MasterFaultDetection extends FaultDetection {
 
     private void notifyMasterFailure(final DiscoveryNode masterNode, final Throwable cause, final String reason) {
         if (notifiedMasterFailure.compareAndSet(false, true)) {
-            threadPool.generic().execute(new Runnable() {
-                @Override
-                public void run() {
+            try {
+                threadPool.generic().execute(() -> {
                     for (Listener listener : listeners) {
                         listener.onMasterFailure(masterNode, cause, reason);
                     }
-                }
-            });
+                });
+            } catch (RejectedExecutionException e) {
+                logger.error("master failure notification was rejected, it's highly likely the node is shutting down", e);
+            }
             stop("master failure, " + reason);
         }
     }

From 7ecb4ca4e99ef2b1fed12284a135b1c7fd113ad9 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 13 Jun 2016 18:56:28 -0400
Subject: [PATCH 0822/1311] Refactor def math

---
 .../java/org/elasticsearch/painless/Def.java  |  619 ---------
 .../elasticsearch/painless/DefBootstrap.java  |  139 +-
 .../org/elasticsearch/painless/DefMath.java   | 1149 +++++++++++++++++
 .../elasticsearch/painless/MethodWriter.java  |   60 +-
 .../org/elasticsearch/painless/Utility.java   |    8 -
 .../painless/WriterConstants.java             |   22 +-
 .../elasticsearch/painless/node/EBinary.java  |   57 +-
 .../painless/node/ECapturingFunctionRef.java  |    2 +-
 .../elasticsearch/painless/node/EComp.java    |   32 +-
 .../elasticsearch/painless/node/EUnary.java   |   18 +-
 .../painless/node/LDefArray.java              |    4 +-
 .../elasticsearch/painless/node/LDefCall.java |    2 +-
 .../painless/node/LDefField.java              |    4 +-
 .../elasticsearch/painless/node/SEach.java    |    2 +-
 .../org/elasticsearch/painless/AndTests.java  |    9 +
 .../painless/BinaryOperatorTests.java         |   37 +-
 .../painless/DefBootstrapTests.java           |   15 +
 .../painless/DefOperationTests.java           |  293 ++---
 .../org/elasticsearch/painless/OrTests.java   |    9 +
 .../elasticsearch/painless/UnaryTests.java    |   10 +
 .../org/elasticsearch/painless/XorTests.java  |    9 +
 21 files changed, 1587 insertions(+), 913 deletions(-)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 48775a88a20..158086ab429 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -655,625 +655,6 @@ public final class Def {
         }
     }
 
-    // NOTE: Below methods are not cached, instead invoked directly because they are performant.
-    //       We also check for Long values first when possible since the type is more
-    //       likely to be a Long than a Float.
-
-    public static Object not(final Object unary) {
-        if (unary instanceof Double || unary instanceof Long || unary instanceof Float) {
-            return ~((Number)unary).longValue();
-        } else if (unary instanceof Number) {
-            return ~((Number)unary).intValue();
-        } else if (unary instanceof Character) {
-            return ~(int)(char)unary;
-        }
-
-        throw new ClassCastException("Cannot apply [~] operation to type " +
-                "[" + unary.getClass().getCanonicalName() + "].");
-    }
-
-    public static Object neg(final Object unary) {
-        if (unary instanceof Double) {
-            return -(double)unary;
-        } else if (unary instanceof Float) {
-            return -(float)unary;
-        } else if (unary instanceof Long) {
-            return -(long)unary;
-        } else if (unary instanceof Number) {
-            return -((Number)unary).intValue();
-        } else if (unary instanceof Character) {
-            return -(char)unary;
-        }
-
-        throw new ClassCastException("Cannot apply [-] operation to type " +
-                "[" + unary.getClass().getCanonicalName() + "].");
-    }
-
-    public static Object mul(final Object left, final Object right) {
-        if (left instanceof Number) {
-            if (right instanceof Number) {
-                if (left instanceof Double || right instanceof Double) {
-                    return ((Number)left).doubleValue() * ((Number)right).doubleValue();
-                } else if (left instanceof Float || right instanceof Float) {
-                    return ((Number)left).floatValue() * ((Number)right).floatValue();
-                } else if (left instanceof Long || right instanceof Long) {
-                    return ((Number)left).longValue() * ((Number)right).longValue();
-                } else {
-                    return ((Number)left).intValue() * ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                if (left instanceof Double) {
-                    return ((Number)left).doubleValue() * (char)right;
-                } else if (left instanceof Long) {
-                    return ((Number)left).longValue() * (char)right;
-                } else if (left instanceof Float) {
-                    return ((Number)left).floatValue() * (char)right;
-                } else {
-                    return ((Number)left).intValue() * (char)right;
-                }
-            }
-        } else if (left instanceof Character) {
-            if (right instanceof Number) {
-                if (right instanceof Double) {
-                    return (char)left * ((Number)right).doubleValue();
-                } else if (right instanceof Long) {
-                    return (char)left * ((Number)right).longValue();
-                } else if (right instanceof Float) {
-                    return (char)left * ((Number)right).floatValue();
-                } else {
-                    return (char)left * ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                return (char)left * (char)right;
-            }
-        }
-
-        throw new ClassCastException("Cannot apply [*] operation to types " +
-                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
-    }
-
-    public static Object div(final Object left, final Object right) {
-        if (left instanceof Number) {
-            if (right instanceof Number) {
-                if (left instanceof Double || right instanceof Double) {
-                    return ((Number)left).doubleValue() / ((Number)right).doubleValue();
-                } else if (left instanceof Float || right instanceof Float) {
-                    return ((Number)left).floatValue() / ((Number)right).floatValue();
-                } else if (left instanceof Long || right instanceof Long) {
-                    return ((Number)left).longValue() / ((Number)right).longValue();
-                } else {
-                    return ((Number)left).intValue() / ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                if (left instanceof Double) {
-                    return ((Number)left).doubleValue() / (char)right;
-                } else if (left instanceof Long) {
-                    return ((Number)left).longValue() / (char)right;
-                } else if (left instanceof Float) {
-                    return ((Number)left).floatValue() / (char)right;
-                } else {
-                    return ((Number)left).intValue() / (char)right;
-                }
-            }
-        } else if (left instanceof Character) {
-            if (right instanceof Number) {
-                if (right instanceof Double) {
-                    return (char)left / ((Number)right).doubleValue();
-                } else if (right instanceof Long) {
-                    return (char)left / ((Number)right).longValue();
-                } else if (right instanceof Float) {
-                    return (char)left / ((Number)right).floatValue();
-                } else {
-                    return (char)left / ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                return (char)left / (char)right;
-            }
-        }
-
-        throw new ClassCastException("Cannot apply [/] operation to types " +
-                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
-    }
-
-    public static Object rem(final Object left, final Object right) {
-        if (left instanceof Number) {
-            if (right instanceof Number) {
-                if (left instanceof Double || right instanceof Double) {
-                    return ((Number)left).doubleValue() % ((Number)right).doubleValue();
-                } else if (left instanceof Float || right instanceof Float) {
-                    return ((Number)left).floatValue() % ((Number)right).floatValue();
-                } else if (left instanceof Long || right instanceof Long) {
-                    return ((Number)left).longValue() % ((Number)right).longValue();
-                } else {
-                    return ((Number)left).intValue() % ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                if (left instanceof Double) {
-                    return ((Number)left).doubleValue() % (char)right;
-                } else if (left instanceof Long) {
-                    return ((Number)left).longValue() % (char)right;
-                } else if (left instanceof Float) {
-                    return ((Number)left).floatValue() % (char)right;
-                } else {
-                    return ((Number)left).intValue() % (char)right;
-                }
-            }
-        } else if (left instanceof Character) {
-            if (right instanceof Number) {
-                if (right instanceof Double) {
-                    return (char)left % ((Number)right).doubleValue();
-                } else if (right instanceof Long) {
-                    return (char)left % ((Number)right).longValue();
-                } else if (right instanceof Float) {
-                    return (char)left % ((Number)right).floatValue();
-                } else {
-                    return (char)left % ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                return (char)left % (char)right;
-            }
-        }
-
-        throw new ClassCastException("Cannot apply [%] operation to types " +
-                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
-    }
-
-    public static Object add(final Object left, final Object right) {
-        if (left instanceof String || right instanceof String) {
-            return "" + left + right;
-        } else if (left instanceof Number) {
-            if (right instanceof Number) {
-                if (left instanceof Double || right instanceof Double) {
-                    return ((Number)left).doubleValue() + ((Number)right).doubleValue();
-                } else if (left instanceof Float || right instanceof Float) {
-                    return ((Number)left).floatValue() + ((Number)right).floatValue();
-                } else if (left instanceof Long || right instanceof Long) {
-                    return ((Number)left).longValue() + ((Number)right).longValue();
-                } else {
-                    return ((Number)left).intValue() + ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                if (left instanceof Double) {
-                    return ((Number)left).doubleValue() + (char)right;
-                } else if (left instanceof Long) {
-                    return ((Number)left).longValue() + (char)right;
-                } else if (left instanceof Float) {
-                    return ((Number)left).floatValue() + (char)right;
-                } else {
-                    return ((Number)left).intValue() + (char)right;
-                }
-            }
-        } else if (left instanceof Character) {
-            if (right instanceof Number) {
-                if (right instanceof Double) {
-                    return (char)left + ((Number)right).doubleValue();
-                } else if (right instanceof Long) {
-                    return (char)left + ((Number)right).longValue();
-                } else if (right instanceof Float) {
-                    return (char)left + ((Number)right).floatValue();
-                } else {
-                    return (char)left + ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                return (char)left + (char)right;
-            }
-        }
-
-        throw new ClassCastException("Cannot apply [+] operation to types " +
-                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
-    }
-
-    public static Object sub(final Object left, final Object right) {
-        if (left instanceof Number) {
-            if (right instanceof Number) {
-                if (left instanceof Double || right instanceof Double) {
-                    return ((Number)left).doubleValue() - ((Number)right).doubleValue();
-                } else if (left instanceof Float || right instanceof Float) {
-                    return ((Number)left).floatValue() - ((Number)right).floatValue();
-                } else if (left instanceof Long || right instanceof Long) {
-                    return ((Number)left).longValue() - ((Number)right).longValue();
-                } else {
-                    return ((Number)left).intValue() - ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                if (left instanceof Double) {
-                    return ((Number)left).doubleValue() - (char)right;
-                } else if (left instanceof Long) {
-                    return ((Number)left).longValue() - (char)right;
-                } else if (left instanceof Float) {
-                    return ((Number)left).floatValue() - (char)right;
-                } else {
-                    return ((Number)left).intValue() - (char)right;
-                }
-            }
-        } else if (left instanceof Character) {
-            if (right instanceof Number) {
-                if (right instanceof Double) {
-                    return (char)left - ((Number)right).doubleValue();
-                } else if (right instanceof Long) {
-                    return (char)left - ((Number)right).longValue();
-                } else if (right instanceof Float) {
-                    return (char)left - ((Number)right).floatValue();
-                } else {
-                    return (char)left - ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                return (char)left - (char)right;
-            }
-        }
-
-        throw new ClassCastException("Cannot apply [-] operation to types " +
-                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
-    }
-
-    public static Object lsh(final Object left, final int right) {
-        if (left instanceof Double || left instanceof Long || left instanceof Float) {
-            return ((Number)left).longValue() << right;
-        } else if (left instanceof Number) {
-            return ((Number)left).intValue() << right;
-        } else if (left instanceof Character) {
-            return (char)left << right;
-        }
-
-        throw new ClassCastException("Cannot apply [<<] operation to types [" + left.getClass().getCanonicalName() + "] and [int].");
-    }
-
-    public static Object rsh(final Object left, final int right) {
-        if (left instanceof Double || left instanceof Long || left instanceof Float) {
-            return ((Number)left).longValue() >> right;
-        } else if (left instanceof Number) {
-            return ((Number)left).intValue() >> right;
-        } else if (left instanceof Character) {
-            return (char)left >> right;
-        }
-
-        throw new ClassCastException("Cannot apply [>>] operation to types [" + left.getClass().getCanonicalName() + "] and [int].");
-    }
-
-    public static Object ush(final Object left, final int right) {
-        if (left instanceof Double || left instanceof Long || left instanceof Float) {
-            return ((Number)left).longValue() >>> right;
-        } else if (left instanceof Number) {
-            return ((Number)left).intValue() >>> right;
-        } else if (left instanceof Character) {
-            return (char)left >>> right;
-        }
-
-        throw new ClassCastException("Cannot apply [>>>] operation to types [" + left.getClass().getCanonicalName() + "] and [int].");
-    }
-
-    public static Object and(final Object left, final Object right) {
-        if (left instanceof Boolean && right instanceof Boolean) {
-            return (boolean)left && (boolean)right;
-        } else if (left instanceof Number) {
-            if (right instanceof Number) {
-                if (left instanceof Double || right instanceof Double ||
-                    left instanceof Long || right instanceof Long ||
-                    left instanceof Float || right instanceof Float) {
-                    return ((Number)left).longValue() & ((Number)right).longValue();
-                } else {
-                    return ((Number)left).intValue() & ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                if (left instanceof Double || left instanceof Long || left instanceof Float) {
-                    return ((Number)left).longValue() & (char)right;
-                } else {
-                    return ((Number)left).intValue() & (char)right;
-                }
-            }
-        } else if (left instanceof Character) {
-            if (right instanceof Number) {
-                if (right instanceof Double || right instanceof Long || right instanceof Float) {
-                    return (char)left & ((Number)right).longValue();
-                } else {
-                    return (char)left & ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                return (char)left & (char)right;
-            }
-        }
-
-        throw new ClassCastException("Cannot apply [&] operation to types " +
-                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
-    }
-
-    public static Object xor(final Object left, final Object right) {
-        if (left instanceof Boolean && right instanceof Boolean) {
-            return (boolean)left ^ (boolean)right;
-        } else if (left instanceof Number) {
-            if (right instanceof Number) {
-                if (left instanceof Double || right instanceof Double ||
-                    left instanceof Long || right instanceof Long ||
-                    left instanceof Float || right instanceof Float) {
-                    return ((Number)left).longValue() ^ ((Number)right).longValue();
-                } else {
-                    return ((Number)left).intValue() ^ ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                if (left instanceof Double || left instanceof Long || left instanceof Float) {
-                    return ((Number)left).longValue() ^ (char)right;
-                } else {
-                    return ((Number)left).intValue() ^ (char)right;
-                }
-            }
-        } else if (left instanceof Character) {
-            if (right instanceof Number) {
-                if (right instanceof Double || right instanceof Long || right instanceof Float) {
-                    return (char)left ^ ((Number)right).longValue();
-                } else {
-                    return (char)left ^ ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                return (char)left ^ (char)right;
-            }
-        }
-
-        throw new ClassCastException("Cannot apply [^] operation to types " +
-                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
-    }
-
-    public static Object or(final Object left, final Object right) {
-        if (left instanceof Boolean && right instanceof Boolean) {
-            return (boolean)left || (boolean)right;
-        } else if (left instanceof Number) {
-            if (right instanceof Number) {
-                if (left instanceof Double || right instanceof Double ||
-                    left instanceof Long || right instanceof Long ||
-                    left instanceof Float || right instanceof Float) {
-                    return ((Number)left).longValue() | ((Number)right).longValue();
-                } else {
-                    return ((Number)left).intValue() | ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                if (left instanceof Double || left instanceof Long || left instanceof Float) {
-                    return ((Number)left).longValue() | (char)right;
-                } else {
-                    return ((Number)left).intValue() | (char)right;
-                }
-            }
-        } else if (left instanceof Character) {
-            if (right instanceof Number) {
-                if (right instanceof Double || right instanceof Long || right instanceof Float) {
-                    return (char)left | ((Number)right).longValue();
-                } else {
-                    return (char)left | ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                return (char)left | (char)right;
-            }
-        }
-
-        throw new ClassCastException("Cannot apply [|] operation to types " +
-                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
-    }
-
-    public static boolean eq(final Object left, final Object right) {
-        if (left != null && right != null) {
-            if (left instanceof Double) {
-                if (right instanceof Number) {
-                    return (double)left == ((Number)right).doubleValue();
-                } else if (right instanceof Character) {
-                    return (double)left == (char)right;
-                }
-            } else if (right instanceof Double) {
-                if (left instanceof Number) {
-                    return ((Number)left).doubleValue() == (double)right;
-                } else if (left instanceof Character) {
-                    return (char)left == ((Number)right).doubleValue();
-                }
-            } else if (left instanceof Float) {
-                if (right instanceof Number) {
-                    return (float)left == ((Number)right).floatValue();
-                } else if (right instanceof Character) {
-                    return (float)left == (char)right;
-                }
-            } else if (right instanceof Float) {
-                if (left instanceof Number) {
-                    return ((Number)left).floatValue() == (float)right;
-                } else if (left instanceof Character) {
-                    return (char)left == ((Number)right).floatValue();
-                }
-            } else if (left instanceof Long) {
-                if (right instanceof Number) {
-                    return (long)left == ((Number)right).longValue();
-                } else if (right instanceof Character) {
-                    return (long)left == (char)right;
-                }
-            } else if (right instanceof Long) {
-                if (left instanceof Number) {
-                    return ((Number)left).longValue() == (long)right;
-                } else if (left instanceof Character) {
-                    return (char)left == ((Number)right).longValue();
-                }
-            } else if (left instanceof Number) {
-                if (right instanceof Number) {
-                    return ((Number)left).intValue() == ((Number)right).intValue();
-                } else if (right instanceof Character) {
-                    return ((Number)left).intValue() == (char)right;
-                }
-            } else if (right instanceof Number && left instanceof Character) {
-                return (char)left == ((Number)right).intValue();
-            } else if (left instanceof Character && right instanceof Character) {
-                return (char)left == (char)right;
-            }
-
-            return left.equals(right);
-        }
-
-        return left == null && right == null;
-    }
-
-    public static boolean lt(final Object left, final Object right) {
-        if (left instanceof Number) {
-            if (right instanceof Number) {
-                if (left instanceof Double || right instanceof Double) {
-                    return ((Number)left).doubleValue() < ((Number)right).doubleValue();
-                } else if (left instanceof Float || right instanceof Float) {
-                    return ((Number)left).floatValue() < ((Number)right).floatValue();
-                } else if (left instanceof Long || right instanceof Long) {
-                    return ((Number)left).longValue() < ((Number)right).longValue();
-                } else {
-                    return ((Number)left).intValue() < ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                if (left instanceof Double) {
-                    return ((Number)left).doubleValue() < (char)right;
-                } else if (left instanceof Long) {
-                    return ((Number)left).longValue() < (char)right;
-                } else if (left instanceof Float) {
-                    return ((Number)left).floatValue() < (char)right;
-                } else {
-                    return ((Number)left).intValue() < (char)right;
-                }
-            }
-        } else if (left instanceof Character) {
-            if (right instanceof Number) {
-                if (right instanceof Double) {
-                    return (char)left < ((Number)right).doubleValue();
-                } else if (right instanceof Long) {
-                    return (char)left < ((Number)right).longValue();
-                } else if (right instanceof Float) {
-                    return (char)left < ((Number)right).floatValue();
-                } else {
-                    return (char)left < ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                return (char)left < (char)right;
-            }
-        }
-
-        throw new ClassCastException("Cannot apply [<] operation to types " +
-                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
-    }
-
-    public static boolean lte(final Object left, final Object right) {
-        if (left instanceof Number) {
-            if (right instanceof Number) {
-                if (left instanceof Double || right instanceof Double) {
-                    return ((Number)left).doubleValue() <= ((Number)right).doubleValue();
-                } else if (left instanceof Float || right instanceof Float) {
-                    return ((Number)left).floatValue() <= ((Number)right).floatValue();
-                } else if (left instanceof Long || right instanceof Long) {
-                    return ((Number)left).longValue() <= ((Number)right).longValue();
-                } else {
-                    return ((Number)left).intValue() <= ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                if (left instanceof Double) {
-                    return ((Number)left).doubleValue() <= (char)right;
-                } else if (left instanceof Long) {
-                    return ((Number)left).longValue() <= (char)right;
-                } else if (left instanceof Float) {
-                    return ((Number)left).floatValue() <= (char)right;
-                } else {
-                    return ((Number)left).intValue() <= (char)right;
-                }
-            }
-        } else if (left instanceof Character) {
-            if (right instanceof Number) {
-                if (right instanceof Double) {
-                    return (char)left <= ((Number)right).doubleValue();
-                } else if (right instanceof Long) {
-                    return (char)left <= ((Number)right).longValue();
-                } else if (right instanceof Float) {
-                    return (char)left <= ((Number)right).floatValue();
-                } else {
-                    return (char)left <= ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                return (char)left <= (char)right;
-            }
-        }
-
-        throw new ClassCastException("Cannot apply [<=] operation to types " +
-                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
-    }
-
-    public static boolean gt(final Object left, final Object right) {
-        if (left instanceof Number) {
-            if (right instanceof Number) {
-                if (left instanceof Double || right instanceof Double) {
-                    return ((Number)left).doubleValue() > ((Number)right).doubleValue();
-                } else if (left instanceof Float || right instanceof Float) {
-                    return ((Number)left).floatValue() > ((Number)right).floatValue();
-                } else if (left instanceof Long || right instanceof Long) {
-                    return ((Number)left).longValue() > ((Number)right).longValue();
-                } else {
-                    return ((Number)left).intValue() > ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                if (left instanceof Double) {
-                    return ((Number)left).doubleValue() > (char)right;
-                } else if (left instanceof Long) {
-                    return ((Number)left).longValue() > (char)right;
-                } else if (left instanceof Float) {
-                    return ((Number)left).floatValue() > (char)right;
-                } else {
-                    return ((Number)left).intValue() > (char)right;
-                }
-            }
-        } else if (left instanceof Character) {
-            if (right instanceof Number) {
-                if (right instanceof Double) {
-                    return (char)left > ((Number)right).doubleValue();
-                } else if (right instanceof Long) {
-                    return (char)left > ((Number)right).longValue();
-                } else if (right instanceof Float) {
-                    return (char)left > ((Number)right).floatValue();
-                } else {
-                    return (char)left > ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                return (char)left > (char)right;
-            }
-        }
-
-        throw new ClassCastException("Cannot apply [>] operation to types " +
-                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
-    }
-
-    public static boolean gte(final Object left, final Object right) {
-        if (left instanceof Number) {
-            if (right instanceof Number) {
-                if (left instanceof Double || right instanceof Double) {
-                    return ((Number)left).doubleValue() >= ((Number)right).doubleValue();
-                } else if (left instanceof Float || right instanceof Float) {
-                    return ((Number)left).floatValue() >= ((Number)right).floatValue();
-                } else if (left instanceof Long || right instanceof Long) {
-                    return ((Number)left).longValue() >= ((Number)right).longValue();
-                } else {
-                    return ((Number)left).intValue() >= ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                if (left instanceof Double) {
-                    return ((Number)left).doubleValue() >= (char)right;
-                } else if (left instanceof Long) {
-                    return ((Number)left).longValue() >= (char)right;
-                } else if (left instanceof Float) {
-                    return ((Number)left).floatValue() >= (char)right;
-                } else {
-                    return ((Number)left).intValue() >= (char)right;
-                }
-            }
-        } else if (left instanceof Character) {
-            if (right instanceof Number) {
-                if (right instanceof Double) {
-                    return (char)left >= ((Number)right).doubleValue();
-                } else if (right instanceof Long) {
-                    return (char)left >= ((Number)right).longValue();
-                } else if (right instanceof Float) {
-                    return (char)left >= ((Number)right).floatValue();
-                } else {
-                    return (char)left >= ((Number)right).intValue();
-                }
-            } else if (right instanceof Character) {
-                return (char)left >= (char)right;
-            }
-        }
-
-        throw new ClassCastException("Cannot apply [>] operation to types " +
-                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
-    }
 
     // Conversion methods for Def to primitive types.
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index 040008c0d62..c688b314243 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -31,9 +31,9 @@ import java.lang.invoke.MutableCallSite;
 /**
  * Painless invokedynamic bootstrap for the call site.
  * <p>
- * Has 5 flavors (passed as static bootstrap parameters): dynamic method call,
+ * Has 7 flavors (passed as static bootstrap parameters): dynamic method call,
  * dynamic field load (getter), and dynamic field store (setter), dynamic array load,
- * and dynamic array store.
+ * dynamic array store, iterator, and method reference.
  * <p>
  * When a new type is encountered at the call site, we lookup from the appropriate
  * whitelist, and cache with a guard. If we encounter too many types, we stop caching.
@@ -62,6 +62,12 @@ public final class DefBootstrap {
     public static final int ITERATOR = 5;
     /** static bootstrap parameter indicating a dynamic method reference, e.g. foo::bar */
     public static final int REFERENCE = 6;
+    /** static bootstrap parameter indicating a unary math operator, e.g. ~foo */
+    public static final int UNARY_OPERATOR = 7;
+    /** static bootstrap parameter indicating a binary math operator, e.g. foo / bar */
+    public static final int BINARY_OPERATOR = 8;
+    /** static bootstrap parameter indicating a shift operator, e.g. foo &gt;&gt; bar */
+    public static final int SHIFT_OPERATOR = 9;
 
     /**
      * CallSite that implements the polymorphic inlining cache (PIC).
@@ -82,6 +88,12 @@ public final class DefBootstrap {
             this.name = name;
             this.flavor = flavor;
             this.args = args;
+            
+            // For operators use a monomorphic cache, fallback is fast.
+            // Just start with a depth of MAX-1, to keep it a constant.
+            if (flavor == UNARY_OPERATOR || flavor == BINARY_OPERATOR || flavor == SHIFT_OPERATOR) {
+                depth = MAX_DEPTH - 1;
+            }
 
             final MethodHandle fallback = FALLBACK.bindTo(this)
               .asCollector(Object[].class, type.parameterCount())
@@ -97,29 +109,69 @@ public final class DefBootstrap {
         static boolean checkClass(Class<?> clazz, Object receiver) {
             return receiver.getClass() == clazz;
         }
+        
+        /**
+         * guard method for inline caching: checks the receiver's class and the first argument
+         * are the same as the cached receiver and first argument.
+         */
+        static boolean checkBinary(Class<?> left, Class<?> right, Object leftObject, Object rightObject) {
+            return leftObject.getClass() == left && rightObject.getClass() == right;
+        }
+        
+        /**
+         * guard method for inline caching: checks the first argument is the same
+         * as the cached first argument.
+         */
+        static boolean checkBinaryArg(Class<?> left, Class<?> right, Object leftObject, Object rightObject) {
+            return rightObject.getClass() == right;
+        }
 
         /**
          * Does a slow lookup against the whitelist.
          */
-        private MethodHandle lookup(int flavor, Class<?> clazz, String name, Object[] args) throws Throwable {
+        private MethodHandle lookup(int flavor, String name, Object[] args) throws Throwable {
             switch(flavor) {
                 case METHOD_CALL:
-                    return Def.lookupMethod(lookup, type(), clazz, name, args, (Long) this.args[0]);
+                    return Def.lookupMethod(lookup, type(), args[0].getClass(), name, args, (Long) this.args[0]);
                 case LOAD:
-                    return Def.lookupGetter(clazz, name);
+                    return Def.lookupGetter(args[0].getClass(), name);
                 case STORE:
-                    return Def.lookupSetter(clazz, name);
+                    return Def.lookupSetter(args[0].getClass(), name);
                 case ARRAY_LOAD:
-                    return Def.lookupArrayLoad(clazz);
+                    return Def.lookupArrayLoad(args[0].getClass());
                 case ARRAY_STORE:
-                    return Def.lookupArrayStore(clazz);
+                    return Def.lookupArrayStore(args[0].getClass());
                 case ITERATOR:
-                    return Def.lookupIterator(clazz);
+                    return Def.lookupIterator(args[0].getClass());
                 case REFERENCE:
-                    return Def.lookupReference(lookup, (String) this.args[0], clazz, name);
+                    return Def.lookupReference(lookup, (String) this.args[0], args[0].getClass(), name);
+                case UNARY_OPERATOR:
+                case SHIFT_OPERATOR:
+                    // shifts are treated as unary, as java allows long arguments without a cast (but bits are ignored)
+                    return DefMath.lookupUnary(args[0].getClass(), name);
+                case BINARY_OPERATOR:
+                    if (args[0] == null || args[1] == null) {
+                        return getGeneric(flavor, name); // can handle nulls
+                    } else {
+                        return DefMath.lookupBinary(args[0].getClass(), args[1].getClass(), name);
+                    }
                 default: throw new AssertionError();
             }
         }
+        
+        /**
+         * Installs a permanent, generic solution that works with any parameter types, if possible.
+         */
+        private MethodHandle getGeneric(int flavor, String name) throws Throwable {
+            switch(flavor) {
+                case UNARY_OPERATOR:
+                case BINARY_OPERATOR:
+                case SHIFT_OPERATOR:
+                    return DefMath.lookupGeneric(name);
+                default:
+                    return null;
+            }
+        }
 
         /**
          * Called when a new type is encountered (or, when we have encountered more than {@code MAX_DEPTH}
@@ -127,21 +179,56 @@ public final class DefBootstrap {
          */
         @SuppressForbidden(reason = "slow path")
         Object fallback(Object[] args) throws Throwable {
-            final MethodType type = type();
-            final Object receiver = args[0];
-            final Class<?> receiverClass = receiver.getClass();
-            final MethodHandle target = lookup(flavor, receiverClass, name, args).asType(type);
-
             if (depth >= MAX_DEPTH) {
-                // revert to a vtable call
-                setTarget(target);
-                return target.invokeWithArguments(args);
+                // caching defeated
+                MethodHandle generic = getGeneric(flavor, name);
+                if (generic != null) {
+                    setTarget(generic.asType(type()));
+                    return generic.invokeWithArguments(args);
+                } else {
+                    return lookup(flavor, name, args).invokeWithArguments(args);
+                }
+            }
+            
+            final MethodType type = type();
+            final MethodHandle target = lookup(flavor, name, args).asType(type);
+
+            final MethodHandle test;
+            if (flavor == BINARY_OPERATOR || flavor == SHIFT_OPERATOR) {
+                // some binary operators support nulls, we handle them separate
+                Class<?> clazz0 = args[0] == null ? null : args[0].getClass();
+                Class<?> clazz1 = args[1] == null ? null : args[1].getClass();
+                if (type.parameterType(1) != Object.class) {
+                    // case 1: only the receiver is unknown, just check that
+                    MethodHandle unaryTest = CHECK_CLASS.bindTo(clazz0);
+                    test = unaryTest.asType(unaryTest.type()
+                                            .changeParameterType(0, type.parameterType(0)));
+                } else if (type.parameterType(0) != Object.class) {
+                    // case 2: only the argument is unknown, just check that
+                    MethodHandle unaryTest = CHECK_BINARY_ARG.bindTo(clazz0).bindTo(clazz1);
+                    test = unaryTest.asType(unaryTest.type()
+                                            .changeParameterType(0, type.parameterType(0))
+                                            .changeParameterType(1, type.parameterType(1)));
+                } else {
+                    // case 3: check both receiver and argument
+                    MethodHandle binaryTest = CHECK_BINARY.bindTo(clazz0).bindTo(clazz1);
+                    test = binaryTest.asType(binaryTest.type()
+                                            .changeParameterType(0, type.parameterType(0))
+                                            .changeParameterType(1, type.parameterType(1)));
+                }
+            } else {
+                MethodHandle receiverTest = CHECK_CLASS.bindTo(args[0].getClass());
+                test = receiverTest.asType(receiverTest.type()
+                                        .changeParameterType(0, type.parameterType(0)));
             }
 
-            MethodHandle test = CHECK_CLASS.bindTo(receiverClass);
-            test = test.asType(test.type().changeParameterType(0, type.parameterType(0)));
-
-            final MethodHandle guard = MethodHandles.guardWithTest(test, target, getTarget());
+            MethodHandle guard = MethodHandles.guardWithTest(test, target, getTarget());
+            // very special cases, where even the receiver can be null (see JLS rules for string concat)
+            // we wrap + with an NPE catcher, and use our generic method in that case.
+            if (flavor == BINARY_OPERATOR && "add".equals(name) || "eq".equals(name)) {
+                MethodHandle handler = MethodHandles.dropArguments(getGeneric(flavor, name).asType(type()), 0, NullPointerException.class);
+                guard = MethodHandles.catchException(guard, NullPointerException.class, handler);
+            }
             
             depth++;
 
@@ -150,12 +237,18 @@ public final class DefBootstrap {
         }
 
         private static final MethodHandle CHECK_CLASS;
+        private static final MethodHandle CHECK_BINARY;
+        private static final MethodHandle CHECK_BINARY_ARG;
         private static final MethodHandle FALLBACK;
         static {
             final Lookup lookup = MethodHandles.lookup();
             try {
                 CHECK_CLASS = lookup.findStatic(lookup.lookupClass(), "checkClass",
-                                                MethodType.methodType(boolean.class, Class.class, Object.class));
+                                              MethodType.methodType(boolean.class, Class.class, Object.class));
+                CHECK_BINARY = lookup.findStatic(lookup.lookupClass(), "checkBinary",
+                                              MethodType.methodType(boolean.class, Class.class, Class.class, Object.class, Object.class));
+                CHECK_BINARY_ARG = lookup.findStatic(lookup.lookupClass(), "checkBinaryArg",
+                                              MethodType.methodType(boolean.class, Class.class, Class.class, Object.class, Object.class));
                 FALLBACK = lookup.findVirtual(lookup.lookupClass(), "fallback",
                                               MethodType.methodType(Object.class, Object[].class));
             } catch (ReflectiveOperationException e) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java
new file mode 100644
index 00000000000..e6b3a8c6003
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java
@@ -0,0 +1,1149 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
+import java.lang.invoke.MethodHandles.Lookup;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Dynamic operators for painless. 
+ * <p>
+ * Each operator must "support" the following types:
+ * {@code int,long,float,double,boolean,Object}. Operators can throw exceptions if 
+ * the type is illegal. The {@code Object} type must be a "generic" handler that
+ * handles all legal types: it must be convertible to every possible legal signature.
+ */
+@SuppressWarnings("unused")
+public class DefMath {
+    
+    // Unary not: only applicable to integral types
+
+    private static int not(int v) {
+        return ~v;
+    }
+    
+    private static long not(long v) {
+        return ~v;
+    }
+    
+    private static float not(float v) {
+        throw new ClassCastException("Cannot apply not [~] to type [float]");
+    }
+    
+    private static double not(double v) {
+        throw new ClassCastException("Cannot apply not [~] to type [double]");
+    }
+    
+    private static boolean not(boolean v) {
+        throw new ClassCastException("Cannot apply not [~] to type [boolean]");
+    }
+    
+    private static Object not(Object unary) {
+        if (unary instanceof Long) {
+            return ~(Long)unary;
+        } else if (unary instanceof Integer) {
+            return ~(Integer)unary;
+        } else if (unary instanceof Short) {
+            return ~(Short)unary;
+        } else if (unary instanceof Character) {
+            return ~(Character)unary;
+        } else if (unary instanceof Byte) {
+            return ~(Byte)unary;
+        }
+
+        throw new ClassCastException("Cannot apply [~] operation to type " +
+                "[" + unary.getClass().getCanonicalName() + "].");
+    }
+    
+    // unary negation and plus: applicable to all numeric types
+
+    private static int neg(int v) {
+        return -v;
+    }
+    
+    private static long neg(long v) {
+        return -v;
+    }
+    
+    private static float neg(float v) {
+        return -v;
+    }
+    
+    private static double neg(double v) {
+        return -v;
+    }
+    
+    private static boolean neg(boolean v) {
+        throw new ClassCastException("Cannot apply [-] operation to type [boolean]");
+    }
+    
+    private static Object neg(final Object unary) {
+        if (unary instanceof Double) {
+            return -(double)unary;
+        } else if (unary instanceof Long) {
+            return -(long)unary;
+        } else if (unary instanceof Integer) {
+            return -(int)unary;
+        } else if (unary instanceof Float) {
+            return -(float)unary;
+        } else if (unary instanceof Short) {
+            return -(short)unary;
+        } else if (unary instanceof Character) {
+            return -(char)unary;
+        } else if (unary instanceof Byte) {
+            return -(byte)unary;
+        }
+
+        throw new ClassCastException("Cannot apply [-] operation to type " +
+                "[" + unary.getClass().getCanonicalName() + "].");
+    }
+    
+    private static int plus(int v) {
+        return +v;
+    }
+    
+    private static long plus(long v) {
+        return +v;
+    }
+    
+    private static float plus(float v) {
+        return +v;
+    }
+    
+    private static double plus(double v) {
+        return +v;
+    }
+    
+    private static boolean plus(boolean v) {
+        throw new ClassCastException("Cannot apply [+] operation to type [boolean]");
+    }
+    
+    private static Object plus(final Object unary) {
+        if (unary instanceof Double) {
+            return +(double)unary;
+        } else if (unary instanceof Long) {
+            return +(long)unary;
+        } else if (unary instanceof Integer) {
+            return +(int)unary;
+        } else if (unary instanceof Float) {
+            return +(float)unary;
+        } else if (unary instanceof Short) {
+            return +(short)unary;
+        } else if (unary instanceof Character) {
+            return +(char)unary;
+        } else if (unary instanceof Byte) {
+            return +(byte)unary;
+        }
+
+        throw new ClassCastException("Cannot apply [+] operation to type " +
+                "[" + unary.getClass().getCanonicalName() + "].");
+    }
+    
+    // multiplication/division/remainder/subtraction: applicable to all integer types
+    
+    private static int mul(int a, int b) {
+        return a * b;
+    }
+    
+    private static long mul(long a, long b) {
+        return a * b;
+    }
+    
+    private static float mul(float a, float b) {
+        return a * b;
+    }
+    
+    private static double mul(double a, double b) {
+        return a * b;
+    }
+    
+    private static boolean mul(boolean a, boolean b) {
+        throw new ClassCastException("Cannot apply [*] operation to type [boolean]");
+    }
+    
+    private static Object mul(Object left, Object right) {
+        if (left instanceof Number) {
+            if (right instanceof Number) {
+                if (left instanceof Double || right instanceof Double) {
+                    return ((Number)left).doubleValue() * ((Number)right).doubleValue();
+                } else if (left instanceof Float || right instanceof Float) {
+                    return ((Number)left).floatValue() * ((Number)right).floatValue();
+                } else if (left instanceof Long || right instanceof Long) {
+                    return ((Number)left).longValue() * ((Number)right).longValue();
+                } else {
+                    return ((Number)left).intValue() * ((Number)right).intValue();
+                }
+            } else if (right instanceof Character) {
+                if (left instanceof Double) {
+                    return ((Number)left).doubleValue() * (char)right;
+                } else if (left instanceof Long) {
+                    return ((Number)left).longValue() * (char)right;
+                } else if (left instanceof Float) {
+                    return ((Number)left).floatValue() * (char)right;
+                } else {
+                    return ((Number)left).intValue() * (char)right;
+                }
+            }
+        } else if (left instanceof Character) {
+            if (right instanceof Number) {
+                if (right instanceof Double) {
+                    return (char)left * ((Number)right).doubleValue();
+                } else if (right instanceof Long) {
+                    return (char)left * ((Number)right).longValue();
+                } else if (right instanceof Float) {
+                    return (char)left * ((Number)right).floatValue();
+                } else {
+                    return (char)left * ((Number)right).intValue();
+                }
+            } else if (right instanceof Character) {
+                return (char)left * (char)right;
+            }
+        }
+
+        throw new ClassCastException("Cannot apply [*] operation to types " +
+                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
+    }
+    
+    private static int div(int a, int b) {
+        return a / b;
+    }
+    
+    private static long div(long a, long b) {
+        return a / b;
+    }
+    
+    private static float div(float a, float b) {
+        return a / b;
+    }
+    
+    private static double div(double a, double b) {
+        return a / b;
+    }
+    
+    private static boolean div(boolean a, boolean b) {
+        throw new ClassCastException("Cannot apply [/] operation to type [boolean]");
+    }
+    
+    private static Object div(Object left, Object right) {
+        if (left instanceof Number) {
+            if (right instanceof Number) {
+                if (left instanceof Double || right instanceof Double) {
+                    return ((Number)left).doubleValue() / ((Number)right).doubleValue();
+                } else if (left instanceof Float || right instanceof Float) {
+                    return ((Number)left).floatValue() / ((Number)right).floatValue();
+                } else if (left instanceof Long || right instanceof Long) {
+                    return ((Number)left).longValue() / ((Number)right).longValue();
+                } else {
+                    return ((Number)left).intValue() / ((Number)right).intValue();
+                }
+            } else if (right instanceof Character) {
+                if (left instanceof Double) {
+                    return ((Number)left).doubleValue() / (char)right;
+                } else if (left instanceof Long) {
+                    return ((Number)left).longValue() / (char)right;
+                } else if (left instanceof Float) {
+                    return ((Number)left).floatValue() / (char)right;
+                } else {
+                    return ((Number)left).intValue() / (char)right;
+                }
+            }
+        } else if (left instanceof Character) {
+            if (right instanceof Number) {
+                if (right instanceof Double) {
+                    return (char)left / ((Number)right).doubleValue();
+                } else if (right instanceof Long) {
+                    return (char)left / ((Number)right).longValue();
+                } else if (right instanceof Float) {
+                    return (char)left / ((Number)right).floatValue();
+                } else {
+                    return (char)left / ((Number)right).intValue();
+                }
+            } else if (right instanceof Character) {
+                return (char)left / (char)right;
+            }
+        }
+
+        throw new ClassCastException("Cannot apply [/] operation to types " +
+                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
+    }
+    
+    private static int rem(int a, int b) {
+        return a % b;
+    }
+    
+    private static long rem(long a, long b) {
+        return a % b;
+    }
+    
+    private static float rem(float a, float b) {
+        return a % b;
+    }
+    
+    private static double rem(double a, double b) {
+        return a % b;
+    }
+    
+    private static boolean rem(boolean a, boolean b) {
+        throw new ClassCastException("Cannot apply [%] operation to type [boolean]");
+    }
+    
+    private static Object rem(Object left, Object right) {
+        if (left instanceof Number) {
+            if (right instanceof Number) {
+                if (left instanceof Double || right instanceof Double) {
+                    return ((Number)left).doubleValue() % ((Number)right).doubleValue();
+                } else if (left instanceof Float || right instanceof Float) {
+                    return ((Number)left).floatValue() % ((Number)right).floatValue();
+                } else if (left instanceof Long || right instanceof Long) {
+                    return ((Number)left).longValue() % ((Number)right).longValue();
+                } else {
+                    return ((Number)left).intValue() % ((Number)right).intValue();
+                }
+            } else if (right instanceof Character) {
+                if (left instanceof Double) {
+                    return ((Number)left).doubleValue() % (char)right;
+                } else if (left instanceof Long) {
+                    return ((Number)left).longValue() % (char)right;
+                } else if (left instanceof Float) {
+                    return ((Number)left).floatValue() % (char)right;
+                } else {
+                    return ((Number)left).intValue() % (char)right;
+                }
+            }
+        } else if (left instanceof Character) {
+            if (right instanceof Number) {
+                if (right instanceof Double) {
+                    return (char)left % ((Number)right).doubleValue();
+                } else if (right instanceof Long) {
+                    return (char)left % ((Number)right).longValue();
+                } else if (right instanceof Float) {
+                    return (char)left % ((Number)right).floatValue();
+                } else {
+                    return (char)left % ((Number)right).intValue();
+                }
+            } else if (right instanceof Character) {
+                return (char)left % (char)right;
+            }
+        }
+
+        throw new ClassCastException("Cannot apply [%] operation to types " +
+                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
+    }
+    
+    // addition: applicable to all numeric types.
+    // additionally, if either type is a string, the other type can be any arbitrary type (including null)
+    
+    private static int add(int a, int b) {
+        return a + b;
+    }
+    
+    private static long add(long a, long b) {
+        return a + b;
+    }
+    
+    private static float add(float a, float b) {
+        return a + b;
+    }
+    
+    private static double add(double a, double b) {
+        return a + b;
+    }
+    
+    private static boolean add(boolean a, boolean b) {
+        throw new ClassCastException("Cannot apply [+] operation to type [boolean]");
+    }
+    
+    private static Object add(Object left, Object right) {
+        if (left instanceof String || right instanceof String) {
+            return "" + left + right;
+        } else if (left instanceof Number) {
+            if (right instanceof Number) {
+                if (left instanceof Double || right instanceof Double) {
+                    return ((Number)left).doubleValue() + ((Number)right).doubleValue();
+                } else if (left instanceof Float || right instanceof Float) {
+                    return ((Number)left).floatValue() + ((Number)right).floatValue();
+                } else if (left instanceof Long || right instanceof Long) {
+                    return ((Number)left).longValue() + ((Number)right).longValue();
+                } else {
+                    return ((Number)left).intValue() + ((Number)right).intValue();
+                }
+            } else if (right instanceof Character) {
+                if (left instanceof Double) {
+                    return ((Number)left).doubleValue() + (char)right;
+                } else if (left instanceof Long) {
+                    return ((Number)left).longValue() + (char)right;
+                } else if (left instanceof Float) {
+                    return ((Number)left).floatValue() + (char)right;
+                } else {
+                    return ((Number)left).intValue() + (char)right;
+                }
+            }
+        } else if (left instanceof Character) {
+            if (right instanceof Number) {
+                if (right instanceof Double) {
+                    return (char)left + ((Number)right).doubleValue();
+                } else if (right instanceof Long) {
+                    return (char)left + ((Number)right).longValue();
+                } else if (right instanceof Float) {
+                    return (char)left + ((Number)right).floatValue();
+                } else {
+                    return (char)left + ((Number)right).intValue();
+                }
+            } else if (right instanceof Character) {
+                return (char)left + (char)right;
+            }
+        }
+
+        throw new ClassCastException("Cannot apply [+] operation to types " +
+                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
+    }
+    
+    private static int sub(int a, int b) {
+        return a - b;
+    }
+    
+    private static long sub(long a, long b) {
+        return a - b;
+    }
+    
+    private static float sub(float a, float b) {
+        return a - b;
+    }
+    
+    private static double sub(double a, double b) {
+        return a - b;
+    }
+    
+    private static boolean sub(boolean a, boolean b) {
+        throw new ClassCastException("Cannot apply [-] operation to type [boolean]");
+    }
+    
+    private static Object sub(Object left, Object right) {
+        if (left instanceof Number) {
+            if (right instanceof Number) {
+                if (left instanceof Double || right instanceof Double) {
+                    return ((Number)left).doubleValue() - ((Number)right).doubleValue();
+                } else if (left instanceof Float || right instanceof Float) {
+                    return ((Number)left).floatValue() - ((Number)right).floatValue();
+                } else if (left instanceof Long || right instanceof Long) {
+                    return ((Number)left).longValue() - ((Number)right).longValue();
+                } else {
+                    return ((Number)left).intValue() - ((Number)right).intValue();
+                }
+            } else if (right instanceof Character) {
+                if (left instanceof Double) {
+                    return ((Number)left).doubleValue() - (char)right;
+                } else if (left instanceof Long) {
+                    return ((Number)left).longValue() - (char)right;
+                } else if (left instanceof Float) {
+                    return ((Number)left).floatValue() - (char)right;
+                } else {
+                    return ((Number)left).intValue() - (char)right;
+                }
+            }
+        } else if (left instanceof Character) {
+            if (right instanceof Number) {
+                if (right instanceof Double) {
+                    return (char)left - ((Number)right).doubleValue();
+                } else if (right instanceof Long) {
+                    return (char)left - ((Number)right).longValue();
+                } else if (right instanceof Float) {
+                    return (char)left - ((Number)right).floatValue();
+                } else {
+                    return (char)left - ((Number)right).intValue();
+                }
+            } else if (right instanceof Character) {
+                return (char)left - (char)right;
+            }
+        }
+
+        throw new ClassCastException("Cannot apply [-] operation to types " +
+                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
+    }
+    
+    // eq: applicable to any arbitrary type, including nulls for both arguments!!!
+
+    private static boolean eq(int a, int b) {
+        return a == b;
+    }
+    
+    private static boolean eq(long a, long b) {
+        return a == b;
+    }
+    
+    private static boolean eq(float a, float b) {
+        return a == b;
+    }
+    
+    private static boolean eq(double a, double b) {
+        return a == b;
+    }
+    
+    private static boolean eq(boolean a, boolean b) {
+        return a == b;
+    }
+    
+    private static boolean eq(Object left, Object right) {
+        if (left != null && right != null) {
+            if (left instanceof Double) {
+                if (right instanceof Number) {
+                    return (double)left == ((Number)right).doubleValue();
+                } else if (right instanceof Character) {
+                    return (double)left == (char)right;
+                }
+            } else if (right instanceof Double) {
+                if (left instanceof Number) {
+                    return ((Number)left).doubleValue() == (double)right;
+                } else if (left instanceof Character) {
+                    return (char)left == ((Number)right).doubleValue();
+                }
+            } else if (left instanceof Float) {
+                if (right instanceof Number) {
+                    return (float)left == ((Number)right).floatValue();
+                } else if (right instanceof Character) {
+                    return (float)left == (char)right;
+                }
+            } else if (right instanceof Float) {
+                if (left instanceof Number) {
+                    return ((Number)left).floatValue() == (float)right;
+                } else if (left instanceof Character) {
+                    return (char)left == ((Number)right).floatValue();
+                }
+            } else if (left instanceof Long) {
+                if (right instanceof Number) {
+                    return (long)left == ((Number)right).longValue();
+                } else if (right instanceof Character) {
+                    return (long)left == (char)right;
+                }
+            } else if (right instanceof Long) {
+                if (left instanceof Number) {
+                    return ((Number)left).longValue() == (long)right;
+                } else if (left instanceof Character) {
+                    return (char)left == ((Number)right).longValue();
+                }
+            } else if (left instanceof Number) {
+                if (right instanceof Number) {
+                    return ((Number)left).intValue() == ((Number)right).intValue();
+                } else if (right instanceof Character) {
+                    return ((Number)left).intValue() == (char)right;
+                }
+            } else if (right instanceof Number && left instanceof Character) {
+                return (char)left == ((Number)right).intValue();
+            } else if (left instanceof Character && right instanceof Character) {
+                return (char)left == (char)right;
+            }
+
+            return left.equals(right);
+        }
+
+        return left == null && right == null;
+    }
+    
+    // comparison operators: applicable for any numeric type
+
+    private static boolean lt(int a, int b) {
+        return a < b;
+    }
+    
+    private static boolean lt(long a, long b) {
+        return a < b;
+    }
+    
+    private static boolean lt(float a, float b) {
+        return a < b;
+    }
+    
+    private static boolean lt(double a, double b) {
+        return a < b;
+    }
+    
+    private static boolean lt(boolean a, boolean b) {
+        throw new ClassCastException("Cannot apply [<] operation to type [boolean]");    
+    }
+    
+    private static boolean lt(Object left, Object right) {
+        if (left instanceof Number) {
+            if (right instanceof Number) {
+                if (left instanceof Double || right instanceof Double) {
+                    return ((Number)left).doubleValue() < ((Number)right).doubleValue();
+                } else if (left instanceof Float || right instanceof Float) {
+                    return ((Number)left).floatValue() < ((Number)right).floatValue();
+                } else if (left instanceof Long || right instanceof Long) {
+                    return ((Number)left).longValue() < ((Number)right).longValue();
+                } else {
+                    return ((Number)left).intValue() < ((Number)right).intValue();
+                }
+            } else if (right instanceof Character) {
+                if (left instanceof Double) {
+                    return ((Number)left).doubleValue() < (char)right;
+                } else if (left instanceof Long) {
+                    return ((Number)left).longValue() < (char)right;
+                } else if (left instanceof Float) {
+                    return ((Number)left).floatValue() < (char)right;
+                } else {
+                    return ((Number)left).intValue() < (char)right;
+                }
+            }
+        } else if (left instanceof Character) {
+            if (right instanceof Number) {
+                if (right instanceof Double) {
+                    return (char)left < ((Number)right).doubleValue();
+                } else if (right instanceof Long) {
+                    return (char)left < ((Number)right).longValue();
+                } else if (right instanceof Float) {
+                    return (char)left < ((Number)right).floatValue();
+                } else {
+                    return (char)left < ((Number)right).intValue();
+                }
+            } else if (right instanceof Character) {
+                return (char)left < (char)right;
+            }
+        }
+
+        throw new ClassCastException("Cannot apply [<] operation to types " +
+                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
+    }
+
+    private static boolean lte(int a, int b) {
+        return a <= b;
+    }
+    
+    private static boolean lte(long a, long b) {
+        return a <= b;
+    }
+    
+    private static boolean lte(float a, float b) {
+        return a <= b;
+    }
+    
+    private static boolean lte(double a, double b) {
+        return a <= b;
+    }
+    
+    private static boolean lte(boolean a, boolean b) {
+        throw new ClassCastException("Cannot apply [<=] operation to type [boolean]");    
+    }
+    
+    private static boolean lte(Object left, Object right) {
+        if (left instanceof Number) {
+            if (right instanceof Number) {
+                if (left instanceof Double || right instanceof Double) {
+                    return ((Number)left).doubleValue() <= ((Number)right).doubleValue();
+                } else if (left instanceof Float || right instanceof Float) {
+                    return ((Number)left).floatValue() <= ((Number)right).floatValue();
+                } else if (left instanceof Long || right instanceof Long) {
+                    return ((Number)left).longValue() <= ((Number)right).longValue();
+                } else {
+                    return ((Number)left).intValue() <= ((Number)right).intValue();
+                }
+            } else if (right instanceof Character) {
+                if (left instanceof Double) {
+                    return ((Number)left).doubleValue() <= (char)right;
+                } else if (left instanceof Long) {
+                    return ((Number)left).longValue() <= (char)right;
+                } else if (left instanceof Float) {
+                    return ((Number)left).floatValue() <= (char)right;
+                } else {
+                    return ((Number)left).intValue() <= (char)right;
+                }
+            }
+        } else if (left instanceof Character) {
+            if (right instanceof Number) {
+                if (right instanceof Double) {
+                    return (char)left <= ((Number)right).doubleValue();
+                } else if (right instanceof Long) {
+                    return (char)left <= ((Number)right).longValue();
+                } else if (right instanceof Float) {
+                    return (char)left <= ((Number)right).floatValue();
+                } else {
+                    return (char)left <= ((Number)right).intValue();
+                }
+            } else if (right instanceof Character) {
+                return (char)left <= (char)right;
+            }
+        }
+
+        throw new ClassCastException("Cannot apply [<=] operation to types " +
+                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
+    }
+
+    private static boolean gt(int a, int b) {
+        return a > b;
+    }
+    
+    private static boolean gt(long a, long b) {
+        return a > b;
+    }
+    
+    private static boolean gt(float a, float b) {
+        return a > b;
+    }
+    
+    private static boolean gt(double a, double b) {
+        return a > b;
+    }
+    
+    private static boolean gt(boolean a, boolean b) {
+        throw new ClassCastException("Cannot apply [>] operation to type [boolean]");    
+    }
+    
+    private static boolean gt(Object left, Object right) {
+        if (left instanceof Number) {
+            if (right instanceof Number) {
+                if (left instanceof Double || right instanceof Double) {
+                    return ((Number)left).doubleValue() > ((Number)right).doubleValue();
+                } else if (left instanceof Float || right instanceof Float) {
+                    return ((Number)left).floatValue() > ((Number)right).floatValue();
+                } else if (left instanceof Long || right instanceof Long) {
+                    return ((Number)left).longValue() > ((Number)right).longValue();
+                } else {
+                    return ((Number)left).intValue() > ((Number)right).intValue();
+                }
+            } else if (right instanceof Character) {
+                if (left instanceof Double) {
+                    return ((Number)left).doubleValue() > (char)right;
+                } else if (left instanceof Long) {
+                    return ((Number)left).longValue() > (char)right;
+                } else if (left instanceof Float) {
+                    return ((Number)left).floatValue() > (char)right;
+                } else {
+                    return ((Number)left).intValue() > (char)right;
+                }
+            }
+        } else if (left instanceof Character) {
+            if (right instanceof Number) {
+                if (right instanceof Double) {
+                    return (char)left > ((Number)right).doubleValue();
+                } else if (right instanceof Long) {
+                    return (char)left > ((Number)right).longValue();
+                } else if (right instanceof Float) {
+                    return (char)left > ((Number)right).floatValue();
+                } else {
+                    return (char)left > ((Number)right).intValue();
+                }
+            } else if (right instanceof Character) {
+                return (char)left > (char)right;
+            }
+        }
+
+        throw new ClassCastException("Cannot apply [>] operation to types " +
+                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
+    }
+    
+    private static boolean gte(int a, int b) {
+        return a >= b;
+    }
+    
+    private static boolean gte(long a, long b) {
+        return a >= b;
+    }
+    
+    private static boolean gte(float a, float b) {
+        return a >= b;
+    }
+    
+    private static boolean gte(double a, double b) {
+        return a >= b;
+    }
+    
+    private static boolean gte(boolean a, boolean b) {
+        throw new ClassCastException("Cannot apply [>=] operation to type [boolean]");    
+    }
+
+    private static boolean gte(Object left, Object right) {
+        if (left instanceof Number) {
+            if (right instanceof Number) {
+                if (left instanceof Double || right instanceof Double) {
+                    return ((Number)left).doubleValue() >= ((Number)right).doubleValue();
+                } else if (left instanceof Float || right instanceof Float) {
+                    return ((Number)left).floatValue() >= ((Number)right).floatValue();
+                } else if (left instanceof Long || right instanceof Long) {
+                    return ((Number)left).longValue() >= ((Number)right).longValue();
+                } else {
+                    return ((Number)left).intValue() >= ((Number)right).intValue();
+                }
+            } else if (right instanceof Character) {
+                if (left instanceof Double) {
+                    return ((Number)left).doubleValue() >= (char)right;
+                } else if (left instanceof Long) {
+                    return ((Number)left).longValue() >= (char)right;
+                } else if (left instanceof Float) {
+                    return ((Number)left).floatValue() >= (char)right;
+                } else {
+                    return ((Number)left).intValue() >= (char)right;
+                }
+            }
+        } else if (left instanceof Character) {
+            if (right instanceof Number) {
+                if (right instanceof Double) {
+                    return (char)left >= ((Number)right).doubleValue();
+                } else if (right instanceof Long) {
+                    return (char)left >= ((Number)right).longValue();
+                } else if (right instanceof Float) {
+                    return (char)left >= ((Number)right).floatValue();
+                } else {
+                    return (char)left >= ((Number)right).intValue();
+                }
+            } else if (right instanceof Character) {
+                return (char)left >= (char)right;
+            }
+        }
+
+        throw new ClassCastException("Cannot apply [>] operation to types " +
+                "[" + left.getClass().getCanonicalName() + "] and [" + right.getClass().getCanonicalName() + "].");
+    }
+    
+    // helper methods to convert an integral according to numeric promotion
+    // this is used by the generic code for bitwise and shift operators
+    
+    private static long longIntegralValue(Object o) {
+        if (o instanceof Long) {
+            return (long)o;
+        } else if (o instanceof Integer || o instanceof Short || o instanceof Byte) {
+            return ((Number)o).longValue();
+        } else if (o instanceof Character) {
+            return (char)o;
+        } else {
+            throw new ClassCastException("Cannot convert [" + o.getClass().getCanonicalName() + "] to an integral value.");
+        }
+    }
+    
+    private static int intIntegralValue(Object o) {
+        if (o instanceof Integer || o instanceof Short || o instanceof Byte) {
+            return ((Number)o).intValue();
+        } else if (o instanceof Character) {
+            return (char)o;
+        } else {
+            throw new ClassCastException("Cannot convert [" + o.getClass().getCanonicalName() + "] to an integral value.");
+        }
+    }
+    
+    // bitwise operators: valid only for integral types
+
+    private static int and(int a, int b) {
+        return a & b;
+    }
+    
+    private static long and(long a, long b) {
+        return a & b;
+    }
+    
+    private static float and(float a, float b) {
+        throw new ClassCastException("Cannot apply [&] operation to type [float]");    
+    }
+    
+    private static double and(double a, double b) {
+        throw new ClassCastException("Cannot apply [&] operation to type [float]");    
+    }
+    
+    private static boolean and(boolean a, boolean b) {
+        return a & b;
+    }
+    
+    private static Object and(Object left, Object right) {
+        if (left instanceof Boolean && right instanceof Boolean) {
+            return (boolean)left & (boolean)right;
+        } else if (left instanceof Long || right instanceof Long) {
+            return longIntegralValue(left) & longIntegralValue(right);
+        } else {
+            return intIntegralValue(left) & intIntegralValue(right);
+        }
+    }
+    
+    private static int xor(int a, int b) {
+        return a ^ b;
+    }
+    
+    private static long xor(long a, long b) {
+        return a ^ b;
+    }
+    
+    private static float xor(float a, float b) {
+        throw new ClassCastException("Cannot apply [^] operation to type [float]");    
+    }
+    
+    private static double xor(double a, double b) {
+        throw new ClassCastException("Cannot apply [^] operation to type [float]");    
+    }
+    
+    private static boolean xor(boolean a, boolean b) {
+        return a ^ b;
+    }
+
+    private static Object xor(Object left, Object right) {
+        if (left instanceof Boolean && right instanceof Boolean) {
+            return (boolean)left ^ (boolean)right;
+        } else if (left instanceof Long || right instanceof Long) {
+            return longIntegralValue(left) ^ longIntegralValue(right);
+        } else {
+            return intIntegralValue(left) ^ intIntegralValue(right);
+        }
+    }
+
+    private static int or(int a, int b) {
+        return a | b;
+    }
+    
+    private static long or(long a, long b) {
+        return a | b;
+    }
+    
+    private static float or(float a, float b) {
+        throw new ClassCastException("Cannot apply [|] operation to type [float]");    
+    }
+    
+    private static double or(double a, double b) {
+        throw new ClassCastException("Cannot apply [|] operation to type [float]");    
+    }
+    
+    private static boolean or(boolean a, boolean b) {
+        return a | b;
+    }
+    
+    private static Object or(Object left, Object right) {
+        if (left instanceof Boolean && right instanceof Boolean) {
+            return (boolean)left | (boolean)right;
+        } else if (left instanceof Long || right instanceof Long) {
+            return longIntegralValue(left) | longIntegralValue(right);
+        } else {
+            return intIntegralValue(left) | intIntegralValue(right);
+        }
+    }
+    
+    // shift operators, valid for any integral types, but does not promote.
+    // we implement all shifts as long shifts, because the extra bits are ignored anyway.
+    
+    private static int lsh(int a, long b) {
+        return a << b;
+    }
+    
+    private static long lsh(long a, long b) {
+        return a << b;
+    }
+    
+    private static float lsh(float a, long b) {
+        throw new ClassCastException("Cannot apply [<<] operation to type [float]");    
+    }
+    
+    private static double lsh(double a, long b) {
+        throw new ClassCastException("Cannot apply [<<] operation to type [double]");    
+    }
+    
+    private static boolean lsh(boolean a, long b) {
+        throw new ClassCastException("Cannot apply [<<] operation to type [boolean]");    
+    }
+    
+    public static Object lsh(Object left, long right) {
+        if (left instanceof Long) {
+            return (long)(left) << right;
+        } else {
+            return intIntegralValue(left) << right;
+        }
+    }
+    
+    private static int rsh(int a, long b) {
+        return a >> b;
+    }
+    
+    private static long rsh(long a, long b) {
+        return a >> b;
+    }
+    
+    private static float rsh(float a, long b) {
+        throw new ClassCastException("Cannot apply [>>] operation to type [float]");    
+    }
+    
+    private static double rsh(double a, long b) {
+        throw new ClassCastException("Cannot apply [>>] operation to type [double]");    
+    }
+    
+    private static boolean rsh(boolean a, long b) {
+        throw new ClassCastException("Cannot apply [>>] operation to type [boolean]");    
+    }
+
+    public static Object rsh(Object left, long right) {
+        if (left instanceof Long) {
+            return (long)left >> right;
+        } else {
+            return intIntegralValue(left) >> right;
+        }
+    }
+    
+    private static int ush(int a, long b) {
+        return a >>> b;
+    }
+    
+    private static long ush(long a, long b) {
+        return a >>> b;
+    }
+    
+    private static float ush(float a, long b) {
+        throw new ClassCastException("Cannot apply [>>>] operation to type [float]");    
+    }
+    
+    private static double ush(double a, long b) {
+        throw new ClassCastException("Cannot apply [>>>] operation to type [double]");    
+    }
+    
+    private static boolean ush(boolean a, long b) {
+        throw new ClassCastException("Cannot apply [>>>] operation to type [boolean]");    
+    }
+
+    public static Object ush(Object left, long right) {
+        if (left instanceof Long) {
+            return (long)(left) >>> right;
+        } else {
+            return intIntegralValue(left) >>> right;
+        }
+    }
+    
+    /** 
+     * unboxes a class to its primitive type, or returns the original 
+     * class if its not a boxed type.
+     */
+    private static Class<?> unbox(Class<?> clazz) {
+        if (clazz == Boolean.class) {
+            return boolean.class;
+        } else if (clazz == Byte.class) { 
+            return byte.class; 
+        } else if (clazz == Short.class) { 
+            return short.class; 
+        } else if (clazz == Character.class) {
+            return char.class;
+        } else if (clazz == Integer.class) {
+            return int.class;
+        } else if (clazz == Long.class) {
+            return long.class;
+        } else if (clazz == Float.class) {
+            return float.class;
+        } else if (clazz == Double.class) {
+            return double.class;
+        } else {
+            return clazz;
+        }
+    }
+    
+    /** Unary promotion. All Objects are promoted to Object. */
+    private static Class<?> promote(Class<?> clazz) {
+        // if either is a non-primitive type -> Object.
+        if (clazz.isPrimitive() == false) {
+            return Object.class;
+        }
+        // always promoted to integer
+        if (clazz == byte.class || clazz == short.class || clazz == char.class || clazz == int.class) { 
+            return int.class; 
+        } else { 
+            return clazz; 
+        } 
+    }
+    
+    /** Binary promotion. */
+    private static Class<?> promote(Class<?> a, Class<?> b) {
+        // if either is a non-primitive type -> Object.
+        if (a.isPrimitive() == false || b.isPrimitive() == false) {
+            return Object.class;
+        }
+        
+        // boolean -> boolean
+        if (a == boolean.class && b == boolean.class) {
+            return boolean.class;
+        }
+        
+        // ordinary numeric promotion
+        if (a == double.class || b == double.class) {
+            return double.class;
+        } else if (a == float.class || b == float.class) {
+            return float.class;
+        } else if (a == long.class || b == long.class) {
+            return long.class;
+        } else {
+            return int.class;
+        }
+    }
+    
+    private static final Lookup PRIV_LOOKUP = MethodHandles.lookup();
+
+    private static final Map<Class<?>,Map<String,MethodHandle>> TYPE_OP_MAPPING = Collections.unmodifiableMap(
+        Stream.of(boolean.class, int.class, long.class, float.class, double.class, Object.class)
+            .collect(Collectors.toMap(Function.identity(), type -> {
+                try {
+                    Map<String,MethodHandle> map = new HashMap<>();
+                    MethodType unary = MethodType.methodType(type, type);
+                    MethodType binary = MethodType.methodType(type, type, type);
+                    MethodType comparison = MethodType.methodType(boolean.class, type, type);
+                    MethodType shift = MethodType.methodType(type, type, long.class);
+                    Class<?> clazz = PRIV_LOOKUP.lookupClass();
+                    map.put("not",   PRIV_LOOKUP.findStatic(clazz, "not",  unary));
+                    map.put("neg",   PRIV_LOOKUP.findStatic(clazz, "neg",  unary));
+                    map.put("plus",  PRIV_LOOKUP.findStatic(clazz, "plus", unary));
+                    map.put("mul",   PRIV_LOOKUP.findStatic(clazz, "mul",  binary));
+                    map.put("div",   PRIV_LOOKUP.findStatic(clazz, "div",  binary));
+                    map.put("rem",   PRIV_LOOKUP.findStatic(clazz, "rem",  binary));
+                    map.put("add",   PRIV_LOOKUP.findStatic(clazz, "add",  binary));
+                    map.put("sub",   PRIV_LOOKUP.findStatic(clazz, "sub",  binary));
+                    map.put("and",   PRIV_LOOKUP.findStatic(clazz, "and",  binary));
+                    map.put("or",    PRIV_LOOKUP.findStatic(clazz, "or",   binary));
+                    map.put("xor",   PRIV_LOOKUP.findStatic(clazz, "xor",  binary));
+                    map.put("eq",    PRIV_LOOKUP.findStatic(clazz, "eq",   comparison));
+                    map.put("lt",    PRIV_LOOKUP.findStatic(clazz, "lt",   comparison));
+                    map.put("lte",   PRIV_LOOKUP.findStatic(clazz, "lte",  comparison));
+                    map.put("gt",    PRIV_LOOKUP.findStatic(clazz, "gt",   comparison));
+                    map.put("gte",   PRIV_LOOKUP.findStatic(clazz, "gte",  comparison));
+                    map.put("lsh",   PRIV_LOOKUP.findStatic(clazz, "lsh",  shift));
+                    map.put("rsh",   PRIV_LOOKUP.findStatic(clazz, "rsh",  shift));
+                    map.put("ush",   PRIV_LOOKUP.findStatic(clazz, "ush",  shift));
+                    return map;
+                } catch (ReflectiveOperationException e) {
+                    throw new AssertionError(e);
+                }
+            }))
+    );
+    
+    /** Returns an appropriate method handle for a unary or shift operator, based only on the receiver (LHS) */
+    public static MethodHandle lookupUnary(Class<?> receiverClass, String name) {
+        MethodHandle handle = TYPE_OP_MAPPING.get(promote(unbox(receiverClass))).get(name);
+        if (handle == null) {
+            throw new ClassCastException("Cannot apply operator [" + name + "] to type [" + receiverClass + "]");
+        }
+        return handle;
+    }
+    
+    /** Returns an appropriate method handle for a binary operator, based only promotion of the LHS and RHS arguments */
+    public static MethodHandle lookupBinary(Class<?> classA, Class<?> classB, String name) {
+        MethodHandle handle = TYPE_OP_MAPPING.get(promote(promote(unbox(classA)), promote(unbox(classB)))).get(name);
+        if (handle == null) {
+            throw new ClassCastException("Cannot apply operator [" + name + "] to types [" + classA + "] and [" + classB + "]");
+        }
+        return handle;
+    }
+    
+    /** Returns a generic method handle for any operator, that can handle all valid signatures, nulls, corner cases */
+    public static MethodHandle lookupGeneric(String name) {
+        return TYPE_OP_MAPPING.get(Object.class).get(name);
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index 2abcace5d6d..ae46ee54f35 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -35,15 +35,7 @@ import java.util.Deque;
 import java.util.List;
 
 import static org.elasticsearch.painless.WriterConstants.CHAR_TO_STRING;
-import static org.elasticsearch.painless.WriterConstants.DEF_ADD_CALL;
-import static org.elasticsearch.painless.WriterConstants.DEF_AND_CALL;
-import static org.elasticsearch.painless.WriterConstants.DEF_DIV_CALL;
-import static org.elasticsearch.painless.WriterConstants.DEF_LSH_CALL;
-import static org.elasticsearch.painless.WriterConstants.DEF_MUL_CALL;
-import static org.elasticsearch.painless.WriterConstants.DEF_OR_CALL;
-import static org.elasticsearch.painless.WriterConstants.DEF_REM_CALL;
-import static org.elasticsearch.painless.WriterConstants.DEF_RSH_CALL;
-import static org.elasticsearch.painless.WriterConstants.DEF_SUB_CALL;
+import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
 import static org.elasticsearch.painless.WriterConstants.DEF_TO_BOOLEAN;
 import static org.elasticsearch.painless.WriterConstants.DEF_TO_BYTE_EXPLICIT;
 import static org.elasticsearch.painless.WriterConstants.DEF_TO_BYTE_IMPLICIT;
@@ -59,9 +51,7 @@ import static org.elasticsearch.painless.WriterConstants.DEF_TO_LONG_EXPLICIT;
 import static org.elasticsearch.painless.WriterConstants.DEF_TO_LONG_IMPLICIT;
 import static org.elasticsearch.painless.WriterConstants.DEF_TO_SHORT_EXPLICIT;
 import static org.elasticsearch.painless.WriterConstants.DEF_TO_SHORT_IMPLICIT;
-import static org.elasticsearch.painless.WriterConstants.DEF_USH_CALL;
 import static org.elasticsearch.painless.WriterConstants.DEF_UTIL_TYPE;
-import static org.elasticsearch.painless.WriterConstants.DEF_XOR_CALL;
 import static org.elasticsearch.painless.WriterConstants.INDY_STRING_CONCAT_BOOTSTRAP_HANDLE;
 import static org.elasticsearch.painless.WriterConstants.MAX_INDY_STRING_CONCAT_ARGS;
 import static org.elasticsearch.painless.WriterConstants.PAINLESS_ERROR_TYPE;
@@ -283,18 +273,44 @@ public final class MethodWriter extends GeneratorAdapter {
         }
 
         if (sort == Sort.DEF) {
+            // XXX: move this out, so we can populate descriptor with what we really have (instead of casts/boxing!)
+            org.objectweb.asm.Type objectType = org.objectweb.asm.Type.getType(Object.class);
+            org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(objectType, objectType, objectType);
+            
             switch (operation) {
-                case MUL:   invokeStatic(DEF_UTIL_TYPE, DEF_MUL_CALL); break;
-                case DIV:   invokeStatic(DEF_UTIL_TYPE, DEF_DIV_CALL); break;
-                case REM:   invokeStatic(DEF_UTIL_TYPE, DEF_REM_CALL); break;
-                case ADD:   invokeStatic(DEF_UTIL_TYPE, DEF_ADD_CALL); break;
-                case SUB:   invokeStatic(DEF_UTIL_TYPE, DEF_SUB_CALL); break;
-                case LSH:   invokeStatic(DEF_UTIL_TYPE, DEF_LSH_CALL); break;
-                case USH:   invokeStatic(DEF_UTIL_TYPE, DEF_RSH_CALL); break;
-                case RSH:   invokeStatic(DEF_UTIL_TYPE, DEF_USH_CALL); break;
-                case BWAND: invokeStatic(DEF_UTIL_TYPE, DEF_AND_CALL); break;
-                case XOR:   invokeStatic(DEF_UTIL_TYPE, DEF_XOR_CALL); break;
-                case BWOR:  invokeStatic(DEF_UTIL_TYPE, DEF_OR_CALL);  break;
+                case MUL:
+                    invokeDynamic("mul", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
+                    break;
+                case DIV:
+                    invokeDynamic("div", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
+                    break;
+                case REM:
+                    invokeDynamic("rem", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
+                    break;
+                case ADD:
+                    invokeDynamic("add", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
+                    break;
+                case SUB:
+                    invokeDynamic("sub", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
+                    break;
+                case LSH:
+                    invokeDynamic("lsh", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR); 
+                    break;
+                case USH:
+                    invokeDynamic("ush", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR); 
+                    break;
+                case RSH:
+                    invokeDynamic("rsh", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR); 
+                    break;
+                case BWAND: 
+                    invokeDynamic("and", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
+                    break;
+                case XOR:   
+                    invokeDynamic("xor", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
+                    break;
+                case BWOR:  
+                    invokeDynamic("or", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
+                    break;
                 default:
                     throw location.createError(new IllegalStateException("Illegal tree structure."));
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java
index 5ab3450db7e..b965f25bf0e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Utility.java
@@ -37,13 +37,5 @@ public class Utility {
         return value.charAt(0);
     }
 
-    public static boolean checkEquals(final Object left, final Object right) {
-        if (left != null) {
-            return left.equals(right);
-        }
-
-        return right == null || right.equals(null);
-    }
-
     private Utility() {}
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index 521287a02d7..68814a2e1a6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -34,6 +34,7 @@ import java.lang.invoke.MethodType;
 import java.util.BitSet;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.Objects;
 import java.util.regex.Pattern;
 
 /**
@@ -102,24 +103,6 @@ public final class WriterConstants {
     public final static Method DEF_TO_LONG_EXPLICIT   = getAsmMethod(long.class   , "DefTolongExplicit"  , Object.class);
     public final static Method DEF_TO_FLOAT_EXPLICIT  = getAsmMethod(float.class  , "DefTofloatExplicit" , Object.class);
     public final static Method DEF_TO_DOUBLE_EXPLICIT = getAsmMethod(double.class , "DefTodoubleExplicit", Object.class);
-    public final static Method DEF_NOT_CALL = getAsmMethod(Object.class , "not", Object.class);
-    public final static Method DEF_NEG_CALL = getAsmMethod(Object.class , "neg", Object.class);
-    public final static Method DEF_MUL_CALL = getAsmMethod(Object.class , "mul", Object.class, Object.class);
-    public final static Method DEF_DIV_CALL = getAsmMethod(Object.class , "div", Object.class, Object.class);
-    public final static Method DEF_REM_CALL = getAsmMethod(Object.class , "rem", Object.class, Object.class);
-    public final static Method DEF_ADD_CALL = getAsmMethod(Object.class , "add", Object.class, Object.class);
-    public final static Method DEF_SUB_CALL = getAsmMethod(Object.class , "sub", Object.class, Object.class);
-    public final static Method DEF_LSH_CALL = getAsmMethod(Object.class , "lsh", Object.class, int.class);
-    public final static Method DEF_RSH_CALL = getAsmMethod(Object.class , "rsh", Object.class, int.class);
-    public final static Method DEF_USH_CALL = getAsmMethod(Object.class , "ush", Object.class, int.class);
-    public final static Method DEF_AND_CALL = getAsmMethod(Object.class , "and", Object.class, Object.class);
-    public final static Method DEF_XOR_CALL = getAsmMethod(Object.class , "xor", Object.class, Object.class);
-    public final static Method DEF_OR_CALL  = getAsmMethod(Object.class , "or" , Object.class, Object.class);
-    public final static Method DEF_EQ_CALL  = getAsmMethod(boolean.class, "eq" , Object.class, Object.class);
-    public final static Method DEF_LT_CALL  = getAsmMethod(boolean.class, "lt" , Object.class, Object.class);
-    public final static Method DEF_LTE_CALL = getAsmMethod(boolean.class, "lte", Object.class, Object.class);
-    public final static Method DEF_GT_CALL  = getAsmMethod(boolean.class, "gt" , Object.class, Object.class);
-    public final static Method DEF_GTE_CALL = getAsmMethod(boolean.class, "gte", Object.class, Object.class);
 
     /** invokedynamic bootstrap for lambda expression/method references */
     public final static MethodType LAMBDA_BOOTSTRAP_TYPE =
@@ -163,7 +146,8 @@ public final class WriterConstants {
     public final static Method STRINGBUILDER_APPEND_OBJECT  = getAsmMethod(StringBuilder.class, "append", Object.class);
     public final static Method STRINGBUILDER_TOSTRING       = getAsmMethod(String.class, "toString");
 
-    public final static Method CHECKEQUALS = getAsmMethod(boolean.class, "checkEquals", Object.class, Object.class);
+    public final static Type OBJECTS_TYPE = Type.getType(Objects.class);
+    public final static Method EQUALS = getAsmMethod(boolean.class, "equals", Object.class, Object.class);
 
     private static Method getAsmMethod(final Class<?> rtype, final String name, final Class<?>... ptypes) {
         return new Method(name, MethodType.methodType(rtype, ptypes).toMethodDescriptorString());
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 55c2cc18210..9f8f1553770 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -284,22 +284,27 @@ public final class EBinary extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        Type promote = AnalyzerCaster.promoteNumeric(left.actual, false);
+        Type lhspromote = AnalyzerCaster.promoteNumeric(left.actual, false);
+        Type rhspromote = AnalyzerCaster.promoteNumeric(right.actual, false);
 
-        if (promote == null) {
+        if (lhspromote == null || rhspromote == null) {
             throw createError(new ClassCastException("Cannot apply left shift [<<] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        left.expected = promote;
-        right.expected = Definition.INT_TYPE;
-        right.explicit = true;
+        left.expected = lhspromote;
+        if (rhspromote.sort == Sort.LONG) {
+            right.expected = Definition.INT_TYPE;
+            right.explicit = true;
+        } else {
+            right.expected = rhspromote;
+        }
 
         left = left.cast(variables);
         right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
-            Sort sort = promote.sort;
+            Sort sort = lhspromote.sort;
 
             if (sort == Sort.INT) {
                 constant = (int)left.constant << (int)right.constant;
@@ -310,29 +315,34 @@ public final class EBinary extends AExpression {
             }
         }
 
-        actual = promote;
+        actual = lhspromote;
     }
 
     private void analyzeRSH(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
-        Type promote = AnalyzerCaster.promoteNumeric(left.actual, false);
+        Type lhspromote = AnalyzerCaster.promoteNumeric(left.actual, false);
+        Type rhspromote = AnalyzerCaster.promoteNumeric(right.actual, false);
 
-        if (promote == null) {
+        if (lhspromote == null || rhspromote == null) {
             throw createError(new ClassCastException("Cannot apply right shift [>>] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        left.expected = promote;
-        right.expected = Definition.INT_TYPE;
-        right.explicit = true;
+        left.expected = lhspromote;
+        if (rhspromote.sort == Sort.LONG) { 
+            right.expected = Definition.INT_TYPE;
+            right.explicit = true;
+        } else {
+            right.expected = rhspromote;
+        }
 
         left = left.cast(variables);
         right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
-            Sort sort = promote.sort;
+            Sort sort = lhspromote.sort;
 
             if (sort == Sort.INT) {
                 constant = (int)left.constant >> (int)right.constant;
@@ -343,29 +353,34 @@ public final class EBinary extends AExpression {
             }
         }
 
-        actual = promote;
+        actual = lhspromote;
     }
 
     private void analyzeUSH(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
-        Type promote = AnalyzerCaster.promoteNumeric(left.actual, false);
+        Type lhspromote = AnalyzerCaster.promoteNumeric(left.actual, false);
+        Type rhspromote = AnalyzerCaster.promoteNumeric(right.actual, false);
 
-        if (promote == null) {
+        if (lhspromote == null || rhspromote == null) {
             throw createError(new ClassCastException("Cannot apply unsigned shift [>>>] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        left.expected = promote;
-        right.expected = Definition.INT_TYPE;
-        right.explicit = true;
+        left.expected = lhspromote;
+        if (rhspromote.sort == Sort.LONG) { 
+            right.expected = Definition.INT_TYPE;
+            right.explicit = true;
+        } else {
+            right.expected = rhspromote;
+        }
 
         left = left.cast(variables);
         right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
-            Sort sort = promote.sort;
+            Sort sort = lhspromote.sort;
 
             if (sort == Sort.INT) {
                 constant = (int)left.constant >>> (int)right.constant;
@@ -376,7 +391,7 @@ public final class EBinary extends AExpression {
             }
         }
 
-        actual = promote;
+        actual = lhspromote;
     }
 
     private void analyzeBWAnd(Locals variables) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
index 3e35602a3a2..ac316f07fbd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
@@ -87,7 +87,7 @@ public class ECapturingFunctionRef extends AExpression {
             // typed interface, dynamic implementation
             writer.visitVarInsn(captured.type.type.getOpcode(Opcodes.ILOAD), captured.slot);
             String descriptor = Type.getMethodType(expected.type, captured.type.type).getDescriptor();
-            writer.invokeDynamic(call, descriptor, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.REFERENCE, expected.name);
+            writer.invokeDynamic(call, descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.REFERENCE, expected.name);
         } else {
             // typed interface, typed implementation
             writer.visitVarInsn(captured.type.type.getOpcode(Opcodes.ILOAD), captured.slot);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index ec84600d323..a126cd7d936 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -24,19 +24,15 @@ import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.AnalyzerCaster;
+import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.MethodWriter;
 
-import static org.elasticsearch.painless.WriterConstants.CHECKEQUALS;
-import static org.elasticsearch.painless.WriterConstants.DEF_EQ_CALL;
-import static org.elasticsearch.painless.WriterConstants.DEF_GTE_CALL;
-import static org.elasticsearch.painless.WriterConstants.DEF_GT_CALL;
-import static org.elasticsearch.painless.WriterConstants.DEF_LTE_CALL;
-import static org.elasticsearch.painless.WriterConstants.DEF_LT_CALL;
-import static org.elasticsearch.painless.WriterConstants.DEF_UTIL_TYPE;
-import static org.elasticsearch.painless.WriterConstants.UTILITY_TYPE;
+import static org.elasticsearch.painless.WriterConstants.OBJECTS_TYPE;
+import static org.elasticsearch.painless.WriterConstants.EQUALS;
+import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
 
 /**
  * Represents a comparison expression.
@@ -456,11 +452,15 @@ public final class EComp extends AExpression {
 
                 break;
             case DEF:
+                // XXX: move this out, so we can populate descriptor with what we really have (instead of casts/boxing!)
+                org.objectweb.asm.Type booleanType = org.objectweb.asm.Type.getType(boolean.class);
+                org.objectweb.asm.Type objectType = org.objectweb.asm.Type.getType(Object.class);
+                org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(booleanType, objectType, objectType);
                 if (eq) {
                     if (right.isNull) {
                         writer.ifNull(jump);
                     } else if (!left.isNull && (operation == Operation.EQ || operation == Operation.NE)) {
-                        writer.invokeStatic(DEF_UTIL_TYPE, DEF_EQ_CALL);
+                        writer.invokeDynamic("eq", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
                         writejump = false;
                     } else {
                         writer.ifCmp(rtype, MethodWriter.EQ, jump);
@@ -469,22 +469,22 @@ public final class EComp extends AExpression {
                     if (right.isNull) {
                         writer.ifNonNull(jump);
                     } else if (!left.isNull && (operation == Operation.EQ || operation == Operation.NE)) {
-                        writer.invokeStatic(DEF_UTIL_TYPE, DEF_EQ_CALL);
+                        writer.invokeDynamic("eq", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
                         writer.ifZCmp(MethodWriter.EQ, jump);
                     } else {
                         writer.ifCmp(rtype, MethodWriter.NE, jump);
                     }
                 } else if (lt) {
-                    writer.invokeStatic(DEF_UTIL_TYPE, DEF_LT_CALL);
+                    writer.invokeDynamic("lt", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
                     writejump = false;
                 } else if (lte) {
-                    writer.invokeStatic(DEF_UTIL_TYPE, DEF_LTE_CALL);
+                    writer.invokeDynamic("lte", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
                     writejump = false;
                 } else if (gt) {
-                    writer.invokeStatic(DEF_UTIL_TYPE, DEF_GT_CALL);
+                    writer.invokeDynamic("gt", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
                     writejump = false;
                 } else if (gte) {
-                    writer.invokeStatic(DEF_UTIL_TYPE, DEF_GTE_CALL);
+                    writer.invokeDynamic("gte", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
                     writejump = false;
                 } else {
                     throw createError(new IllegalStateException("Illegal tree structure."));
@@ -500,7 +500,7 @@ public final class EComp extends AExpression {
                     if (right.isNull) {
                         writer.ifNull(jump);
                     } else if (operation == Operation.EQ || operation == Operation.NE) {
-                        writer.invokeStatic(UTILITY_TYPE, CHECKEQUALS);
+                        writer.invokeStatic(OBJECTS_TYPE, EQUALS);
 
                         if (branch) {
                             writer.ifZCmp(MethodWriter.NE, jump);
@@ -514,7 +514,7 @@ public final class EComp extends AExpression {
                     if (right.isNull) {
                         writer.ifNonNull(jump);
                     } else if (operation == Operation.EQ || operation == Operation.NE) {
-                        writer.invokeStatic(UTILITY_TYPE, CHECKEQUALS);
+                        writer.invokeStatic(OBJECTS_TYPE, EQUALS);
                         writer.ifZCmp(MethodWriter.EQ, jump);
                     } else {
                         writer.ifCmp(rtype, MethodWriter.NE, jump);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index 2d7d8d4fd49..d184961fd28 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -24,14 +24,13 @@ import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.AnalyzerCaster;
+import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.MethodWriter;
 
-import static org.elasticsearch.painless.WriterConstants.DEF_NEG_CALL;
-import static org.elasticsearch.painless.WriterConstants.DEF_NOT_CALL;
-import static org.elasticsearch.painless.WriterConstants.DEF_UTIL_TYPE;
+import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
 
 /**
  * Represents a unary math expression.
@@ -194,7 +193,8 @@ public final class EUnary extends AExpression {
 
             if (operation == Operation.BWNOT) {
                 if (sort == Sort.DEF) {
-                    writer.invokeStatic(DEF_UTIL_TYPE, DEF_NOT_CALL);
+                    org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(expected.type, child.actual.type);
+                    writer.invokeDynamic("not", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.UNARY_OPERATOR);
                 } else {
                     if (sort == Sort.INT) {
                         writer.push(-1);
@@ -208,11 +208,17 @@ public final class EUnary extends AExpression {
                 }
             } else if (operation == Operation.SUB) {
                 if (sort == Sort.DEF) {
-                    writer.invokeStatic(DEF_UTIL_TYPE, DEF_NEG_CALL);
+                    org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(expected.type, child.actual.type);
+                    writer.invokeDynamic("neg", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.UNARY_OPERATOR);
                 } else {
                     writer.math(MethodWriter.NEG, type);
                 }
-            } else if (operation != Operation.ADD) {
+            } else if (operation == Operation.ADD) {
+                if (sort == Sort.DEF) {
+                    org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(expected.type, child.actual.type);
+                    writer.invokeDynamic("plus", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.UNARY_OPERATOR);
+                } 
+            } else {
                 throw createError(new IllegalStateException("Illegal tree structure."));
             }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index dc8890e5122..5a5333aabe8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -62,7 +62,7 @@ final class LDefArray extends ALink implements IDefLink {
         writer.writeDebugInfo(location);
 
         String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type, index.actual.type);
-        writer.invokeDynamic("arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ARRAY_LOAD);
+        writer.invokeDynamic("arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_LOAD);
     }
 
     @Override
@@ -70,6 +70,6 @@ final class LDefArray extends ALink implements IDefLink {
         writer.writeDebugInfo(location);
 
         String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, index.actual.type, after.type);
-        writer.invokeDynamic("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ARRAY_STORE);
+        writer.invokeDynamic("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_STORE);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index 554144c2999..5301dd2b08d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -105,7 +105,7 @@ final class LDefCall extends ALink implements IDefLink {
         // return value
         signature.append(after.type.getDescriptor());
 
-        writer.invokeDynamic(name, signature.toString(), DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.METHOD_CALL, recipe);
+        writer.invokeDynamic(name, signature.toString(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.METHOD_CALL, recipe);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index 91ee8e0f03d..09b48a94f86 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -59,7 +59,7 @@ final class LDefField extends ALink implements IDefLink {
         writer.writeDebugInfo(location);
 
         String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type);
-        writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.LOAD);
+        writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.LOAD);
     }
 
     @Override
@@ -67,6 +67,6 @@ final class LDefField extends ALink implements IDefLink {
         writer.writeDebugInfo(location);
 
         String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, after.type);
-        writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.STORE);
+        writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.STORE);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index a6156324873..48a3f6e3eae 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -195,7 +195,7 @@ public class SEach extends AStatement {
         if (method == null) {
             Type itr = Definition.getType("Iterator");
             String desc = org.objectweb.asm.Type.getMethodDescriptor(itr.type, Definition.DEF_TYPE.type);
-            writer.invokeDynamic("iterator", desc, DEF_BOOTSTRAP_HANDLE, (Object)DefBootstrap.ITERATOR);
+            writer.invokeDynamic("iterator", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ITERATOR);
         } else if (java.lang.reflect.Modifier.isInterface(method.owner.clazz.getModifiers())) {
             writer.invokeInterface(method.owner.type, method.method);
         } else {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java
index 2c86250da83..6be068a10d1 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java
@@ -45,4 +45,13 @@ public class AndTests extends ScriptTestCase {
         assertEquals(5L & -12L, exec("return 5L & -12L;"));
         assertEquals(7L & 15L & 3L, exec("return 7L & 15L & 3L;"));
     }
+    
+    public void testIllegal() throws Exception {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("float x = (float)4; int y = 1; return x & y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("double x = (double)4; int y = 1; return x & y");
+        });
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BinaryOperatorTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BinaryOperatorTests.java
index 54a36234fd8..61bac8bd6b5 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BinaryOperatorTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BinaryOperatorTests.java
@@ -45,13 +45,12 @@ public class BinaryOperatorTests extends ScriptTestCase {
     }
 
     public void testLongShifts() {
-        // note: we always promote the results of shifts too (unlike java)
         assertEquals(1L << 2, exec("long x = 1L; int y = 2; return x << y;"));
-        assertEquals(1L << 2L, exec("long x = 1L; long y = 2L; return x << y;"));
-        assertEquals(4L >> 2L, exec("long x = 4L; long y = 2L; return x >> y;"));
+        assertEquals(1 << 2L, exec("int x = 1; long y = 2L; return x << y;"));
+        assertEquals(4 >> 2L, exec("int x = 4; long y = 2L; return x >> y;"));
         assertEquals(4L >> 2, exec("long x = 4L; int y = 2; return x >> y;"));
         assertEquals(-1L >>> 29, exec("long x = -1L; int y = 29; return x >>> y;"));
-        assertEquals(-1L >>> 29L, exec("long x = -1L; long y = 29L; return x >>> y;"));
+        assertEquals(-1 >>> 29L, exec("int x = -1; long y = 29L; return x >>> y;"));
     }
 
     public void testLongShiftsConst() {
@@ -62,6 +61,36 @@ public class BinaryOperatorTests extends ScriptTestCase {
         assertEquals(-1L >>> 29, exec("return -1L >>> 29;"));
         assertEquals(-1 >>> 29L, exec("return -1 >>> 29L;"));
     }
+    
+    public void testBogusShifts() {
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("long x = 1L; float y = 2; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("int x = 1; double y = 2L; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("float x = 1F; int y = 2; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("double x = 1D; int y = 2L; return x << y;");
+        });
+    }
+
+    public void testBogusShiftsConst() {
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("return 1L << 2F;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("return 1L << 2.0;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("return 1F << 2;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("return 1D << 2L");
+        });
+    }
 
     public void testMixedTypes() {
         assertEquals(8, exec("int x = 4; char y = 2; return x*y;"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
index 4330c613e14..12104844e7c 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
@@ -23,6 +23,8 @@ import java.lang.invoke.CallSite;
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodType;
+import java.util.Arrays;
+import java.util.Collections;
 
 import org.elasticsearch.test.ESTestCase;
 
@@ -90,6 +92,19 @@ public class DefBootstrapTests extends ESTestCase {
         assertDepthEquals(site, 5);
     }
     
+    /** test that we really revert to a "generic" method that can handle any receiver types */
+    public void testMegamorphic() throws Throwable {
+        DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
+                                                                          "size", 
+                                                                          MethodType.methodType(int.class, Object.class), 
+                                                                          DefBootstrap.METHOD_CALL, 0L);
+        site.depth = DefBootstrap.PIC.MAX_DEPTH; // mark megamorphic
+        MethodHandle handle = site.dynamicInvoker();
+        // arguments are cast to object here, or IDE compilers eat it :)
+        assertEquals(2, handle.invoke((Object) Arrays.asList("1", "2")));
+        assertEquals(1, handle.invoke((Object) Collections.singletonMap("a", "b")));
+    }
+    
     static void assertDepthEquals(CallSite site, int expected) {
         DefBootstrap.PIC dsite = (DefBootstrap.PIC) site;
         assertEquals(expected, dsite.depth);
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
index ca611760f07..bb720943f9d 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
@@ -49,6 +49,16 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(-1.0F, exec("def x = 1F; return -x"));
         assertEquals(-1.0, exec("def x = 1.0; return -x"));
     }
+    
+    public void testPlus() {
+        assertEquals(-1, exec("def x = (byte)-1; return +x"));
+        assertEquals(-1, exec("def x = (short)-1; return +x"));
+        assertEquals(65535, exec("def x = (char)-1; return +x"));
+        assertEquals(-1, exec("def x = -1; return +x"));
+        assertEquals(-1L, exec("def x = -1L; return +x"));
+        assertEquals(-1.0F, exec("def x = -1F; return +x"));
+        assertEquals(-1.0D, exec("def x = -1.0; return +x"));
+    }
 
     public void testMul() {
         assertEquals(4, exec("def x = (byte)2; def y = (byte)2; return x * y"));
@@ -313,6 +323,29 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(2F, exec("def x = (float)1; def y = (float)1; return x + y"));
         assertEquals(2D, exec("def x = (double)1; def y = (double)1; return x + y"));
     }
+    
+    public void testAddConcat() {
+        assertEquals("a" + (byte)2, exec("def x = 'a'; def y = (byte)2; return x + y"));
+        assertEquals("a" + (short)2, exec("def x = 'a'; def y = (short)2; return x + y"));
+        assertEquals("a" + (char)2, exec("def x = 'a'; def y = (char)2; return x + y"));
+        assertEquals("a" + 2, exec("def x = 'a'; def y = (int)2; return x + y"));
+        assertEquals("a" + 2L, exec("def x = 'a'; def y = (long)2; return x + y"));
+        assertEquals("a" + 2F, exec("def x = 'a'; def y = (float)2; return x + y"));
+        assertEquals("a" + 2D, exec("def x = 'a'; def y = (double)2; return x + y"));
+        assertEquals("ab", exec("def x = 'a'; def y = 'b'; return x + y"));
+        assertEquals((byte)2 + "a", exec("def x = 'a'; def y = (byte)2; return y + x"));
+        assertEquals((short)2 + "a", exec("def x = 'a'; def y = (short)2; return y + x"));
+        assertEquals((char)2 + "a", exec("def x = 'a'; def y = (char)2; return y + x"));
+        assertEquals(2 + "a", exec("def x = 'a'; def y = (int)2; return y + x"));
+        assertEquals(2L + "a", exec("def x = 'a'; def y = (long)2; return y + x"));
+        assertEquals(2F + "a", exec("def x = 'a'; def y = (float)2; return y + x"));
+        assertEquals(2D + "a", exec("def x = 'a'; def y = (double)2; return y + x"));
+        assertEquals("anull", exec("def x = 'a'; def y = null; return x + y"));
+        assertEquals("nullb", exec("def x = null; def y = 'b'; return x + y"));
+        expectScriptThrows(NullPointerException.class, () -> {
+            exec("def x = null; def y = null; return x + y");
+        });
+    }
 
     public void testSub() {
         assertEquals(0, exec("def x = (byte)1; def y = (byte)1; return x - y"));
@@ -386,64 +419,36 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(2, exec("def x = (char)1; def y = (byte)1; return x << y"));
         assertEquals(2, exec("def x = (int)1; def y = (byte)1; return x << y"));
         assertEquals(2L, exec("def x = (long)1; def y = (byte)1; return x << y"));
-        assertEquals(2L, exec("def x = (float)1; def y = (byte)1; return x << y"));
-        assertEquals(2L, exec("def x = (double)1; def y = (byte)1; return x << y"));
 
         assertEquals(2, exec("def x = (byte)1; def y = (short)1; return x << y"));
         assertEquals(2, exec("def x = (short)1; def y = (short)1; return x << y"));
         assertEquals(2, exec("def x = (char)1; def y = (short)1; return x << y"));
         assertEquals(2, exec("def x = (int)1; def y = (short)1; return x << y"));
         assertEquals(2L, exec("def x = (long)1; def y = (short)1; return x << y"));
-        assertEquals(2L, exec("def x = (float)1; def y = (short)1; return x << y"));
-        assertEquals(2L, exec("def x = (double)1; def y = (short)1; return x << y"));
 
         assertEquals(2, exec("def x = (byte)1; def y = (char)1; return x << y"));
         assertEquals(2, exec("def x = (short)1; def y = (char)1; return x << y"));
         assertEquals(2, exec("def x = (char)1; def y = (char)1; return x << y"));
         assertEquals(2, exec("def x = (int)1; def y = (char)1; return x << y"));
         assertEquals(2L, exec("def x = (long)1; def y = (char)1; return x << y"));
-        assertEquals(2L, exec("def x = (float)1; def y = (char)1; return x << y"));
-        assertEquals(2L, exec("def x = (double)1; def y = (char)1; return x << y"));
 
         assertEquals(2, exec("def x = (byte)1; def y = (int)1; return x << y"));
         assertEquals(2, exec("def x = (short)1; def y = (int)1; return x << y"));
         assertEquals(2, exec("def x = (char)1; def y = (int)1; return x << y"));
         assertEquals(2, exec("def x = (int)1; def y = (int)1; return x << y"));
         assertEquals(2L, exec("def x = (long)1; def y = (int)1; return x << y"));
-        assertEquals(2L, exec("def x = (float)1; def y = (int)1; return x << y"));
-        assertEquals(2L, exec("def x = (double)1; def y = (int)1; return x << y"));
 
         assertEquals(2, exec("def x = (byte)1; def y = (long)1; return x << y"));
         assertEquals(2, exec("def x = (short)1; def y = (long)1; return x << y"));
         assertEquals(2, exec("def x = (char)1; def y = (long)1; return x << y"));
         assertEquals(2, exec("def x = (int)1; def y = (long)1; return x << y"));
         assertEquals(2L, exec("def x = (long)1; def y = (long)1; return x << y"));
-        assertEquals(2L, exec("def x = (float)1; def y = (long)1; return x << y"));
-        assertEquals(2L, exec("def x = (double)1; def y = (long)1; return x << y"));
-
-        assertEquals(2, exec("def x = (byte)1; def y = (float)1; return x << y"));
-        assertEquals(2, exec("def x = (short)1; def y = (float)1; return x << y"));
-        assertEquals(2, exec("def x = (char)1; def y = (float)1; return x << y"));
-        assertEquals(2, exec("def x = (int)1; def y = (float)1; return x << y"));
-        assertEquals(2L, exec("def x = (long)1; def y = (float)1; return x << y"));
-        assertEquals(2L, exec("def x = (float)1; def y = (float)1; return x << y"));
-        assertEquals(2L, exec("def x = (double)1; def y = (float)1; return x << y"));
-
-        assertEquals(2, exec("def x = (byte)1; def y = (double)1; return x << y"));
-        assertEquals(2, exec("def x = (short)1; def y = (double)1; return x << y"));
-        assertEquals(2, exec("def x = (char)1; def y = (double)1; return x << y"));
-        assertEquals(2, exec("def x = (int)1; def y = (double)1; return x << y"));
-        assertEquals(2L, exec("def x = (long)1; def y = (double)1; return x << y"));
-        assertEquals(2L, exec("def x = (float)1; def y = (double)1; return x << y"));
-        assertEquals(2L, exec("def x = (double)1; def y = (double)1; return x << y"));
 
         assertEquals(2, exec("def x = (byte)1; def y = (byte)1; return x << y"));
         assertEquals(2, exec("def x = (short)1; def y = (short)1; return x << y"));
         assertEquals(2, exec("def x = (char)1; def y = (char)1; return x << y"));
         assertEquals(2, exec("def x = (int)1; def y = (int)1; return x << y"));
         assertEquals(2L, exec("def x = (long)1; def y = (long)1; return x << y"));
-        assertEquals(2L, exec("def x = (float)1; def y = (float)1; return x << y"));
-        assertEquals(2L, exec("def x = (double)1; def y = (double)1; return x << y"));
     }
 
     public void testRsh() {
@@ -452,64 +457,36 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(2, exec("def x = (char)4; def y = (byte)1; return x >> y"));
         assertEquals(2, exec("def x = (int)4; def y = (byte)1; return x >> y"));
         assertEquals(2L, exec("def x = (long)4; def y = (byte)1; return x >> y"));
-        assertEquals(2L, exec("def x = (float)4; def y = (byte)1; return x >> y"));
-        assertEquals(2L, exec("def x = (double)4; def y = (byte)1; return x >> y"));
 
         assertEquals(2, exec("def x = (byte)4; def y = (short)1; return x >> y"));
         assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >> y"));
         assertEquals(2, exec("def x = (char)4; def y = (short)1; return x >> y"));
         assertEquals(2, exec("def x = (int)4; def y = (short)1; return x >> y"));
         assertEquals(2L, exec("def x = (long)4; def y = (short)1; return x >> y"));
-        assertEquals(2L, exec("def x = (float)4; def y = (short)1; return x >> y"));
-        assertEquals(2L, exec("def x = (double)4; def y = (short)1; return x >> y"));
 
         assertEquals(2, exec("def x = (byte)4; def y = (char)1; return x >> y"));
         assertEquals(2, exec("def x = (short)4; def y = (char)1; return x >> y"));
         assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >> y"));
         assertEquals(2, exec("def x = (int)4; def y = (char)1; return x >> y"));
         assertEquals(2L, exec("def x = (long)4; def y = (char)1; return x >> y"));
-        assertEquals(2L, exec("def x = (float)4; def y = (char)1; return x >> y"));
-        assertEquals(2L, exec("def x = (double)4; def y = (char)1; return x >> y"));
 
         assertEquals(2, exec("def x = (byte)4; def y = (int)1; return x >> y"));
         assertEquals(2, exec("def x = (short)4; def y = (int)1; return x >> y"));
         assertEquals(2, exec("def x = (char)4; def y = (int)1; return x >> y"));
         assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >> y"));
         assertEquals(2L, exec("def x = (long)4; def y = (int)1; return x >> y"));
-        assertEquals(2L, exec("def x = (float)4; def y = (int)1; return x >> y"));
-        assertEquals(2L, exec("def x = (double)4; def y = (int)1; return x >> y"));
 
         assertEquals(2, exec("def x = (byte)4; def y = (long)1; return x >> y"));
         assertEquals(2, exec("def x = (short)4; def y = (long)1; return x >> y"));
         assertEquals(2, exec("def x = (char)4; def y = (long)1; return x >> y"));
         assertEquals(2, exec("def x = (int)4; def y = (long)1; return x >> y"));
         assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >> y"));
-        assertEquals(2L, exec("def x = (float)4; def y = (long)1; return x >> y"));
-        assertEquals(2L, exec("def x = (double)4; def y = (long)1; return x >> y"));
-
-        assertEquals(2, exec("def x = (byte)4; def y = (float)1; return x >> y"));
-        assertEquals(2, exec("def x = (short)4; def y = (float)1; return x >> y"));
-        assertEquals(2, exec("def x = (char)4; def y = (float)1; return x >> y"));
-        assertEquals(2, exec("def x = (int)4; def y = (float)1; return x >> y"));
-        assertEquals(2L, exec("def x = (long)4; def y = (float)1; return x >> y"));
-        assertEquals(2L, exec("def x = (float)4; def y = (float)1; return x >> y"));
-        assertEquals(2L, exec("def x = (double)4; def y = (float)1; return x >> y"));
-
-        assertEquals(2, exec("def x = (byte)4; def y = (double)1; return x >> y"));
-        assertEquals(2, exec("def x = (short)4; def y = (double)1; return x >> y"));
-        assertEquals(2, exec("def x = (char)4; def y = (double)1; return x >> y"));
-        assertEquals(2, exec("def x = (int)4; def y = (double)1; return x >> y"));
-        assertEquals(2L, exec("def x = (long)4; def y = (double)1; return x >> y"));
-        assertEquals(2L, exec("def x = (float)4; def y = (double)1; return x >> y"));
-        assertEquals(2L, exec("def x = (double)4; def y = (double)1; return x >> y"));
 
         assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >> y"));
         assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >> y"));
         assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >> y"));
         assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >> y"));
         assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >> y"));
-        assertEquals(2L, exec("def x = (float)4; def y = (float)1; return x >> y"));
-        assertEquals(2L, exec("def x = (double)4; def y = (double)1; return x >> y"));
     }
 
     public void testUsh() {
@@ -518,262 +495,224 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(2, exec("def x = (char)4; def y = (byte)1; return x >>> y"));
         assertEquals(2, exec("def x = (int)4; def y = (byte)1; return x >>> y"));
         assertEquals(2L, exec("def x = (long)4; def y = (byte)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (float)4; def y = (byte)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (double)4; def y = (byte)1; return x >>> y"));
 
         assertEquals(2, exec("def x = (byte)4; def y = (short)1; return x >>> y"));
         assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >>> y"));
         assertEquals(2, exec("def x = (char)4; def y = (short)1; return x >>> y"));
         assertEquals(2, exec("def x = (int)4; def y = (short)1; return x >>> y"));
         assertEquals(2L, exec("def x = (long)4; def y = (short)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (float)4; def y = (short)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (double)4; def y = (short)1; return x >>> y"));
 
         assertEquals(2, exec("def x = (byte)4; def y = (char)1; return x >>> y"));
         assertEquals(2, exec("def x = (short)4; def y = (char)1; return x >>> y"));
         assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >>> y"));
         assertEquals(2, exec("def x = (int)4; def y = (char)1; return x >>> y"));
         assertEquals(2L, exec("def x = (long)4; def y = (char)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (float)4; def y = (char)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (double)4; def y = (char)1; return x >>> y"));
 
         assertEquals(2, exec("def x = (byte)4; def y = (int)1; return x >>> y"));
         assertEquals(2, exec("def x = (short)4; def y = (int)1; return x >>> y"));
         assertEquals(2, exec("def x = (char)4; def y = (int)1; return x >>> y"));
         assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >>> y"));
         assertEquals(2L, exec("def x = (long)4; def y = (int)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (float)4; def y = (int)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (double)4; def y = (int)1; return x >>> y"));
 
         assertEquals(2, exec("def x = (byte)4; def y = (long)1; return x >>> y"));
         assertEquals(2, exec("def x = (short)4; def y = (long)1; return x >>> y"));
         assertEquals(2, exec("def x = (char)4; def y = (long)1; return x >>> y"));
         assertEquals(2, exec("def x = (int)4; def y = (long)1; return x >>> y"));
         assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (float)4; def y = (long)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (double)4; def y = (long)1; return x >>> y"));
-
-        assertEquals(2, exec("def x = (byte)4; def y = (float)1; return x >>> y"));
-        assertEquals(2, exec("def x = (short)4; def y = (float)1; return x >>> y"));
-        assertEquals(2, exec("def x = (char)4; def y = (float)1; return x >>> y"));
-        assertEquals(2, exec("def x = (int)4; def y = (float)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (long)4; def y = (float)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (float)4; def y = (float)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (double)4; def y = (float)1; return x >>> y"));
-
-        assertEquals(2, exec("def x = (byte)4; def y = (double)1; return x >>> y"));
-        assertEquals(2, exec("def x = (short)4; def y = (double)1; return x >>> y"));
-        assertEquals(2, exec("def x = (char)4; def y = (double)1; return x >>> y"));
-        assertEquals(2, exec("def x = (int)4; def y = (double)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (long)4; def y = (double)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (float)4; def y = (double)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (double)4; def y = (double)1; return x >>> y"));
 
         assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >>> y"));
         assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >>> y"));
         assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >>> y"));
         assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >>> y"));
         assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (float)4; def y = (float)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (double)4; def y = (double)1; return x >>> y"));
+    }
+    
+    public void testBogusShifts() {
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1L; def y = 2F; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1; def y = 2D; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1F; def y = 2; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1D; def y = 2L; return x << y;");
+        });
+        
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1L; def y = 2F; return x >> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1; def y = 2D; return x >> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1F; def y = 2; return x >> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1D; def y = 2L; return x >> y;");
+        });
+        
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1L; def y = 2F; return x >>> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1; def y = 2D; return x >>> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1F; def y = 2; return x >>> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1D; def y = 2L; return x >>> y;");
+        });
     }
 
     public void testAnd() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (float)4; def y = (byte)1; return x & y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (double)4; def y = (byte)1; return x & y");
+        });
         assertEquals(0, exec("def x = (byte)4; def y = (byte)1; return x & y"));
         assertEquals(0, exec("def x = (short)4; def y = (byte)1; return x & y"));
         assertEquals(0, exec("def x = (char)4; def y = (byte)1; return x & y"));
         assertEquals(0, exec("def x = (int)4; def y = (byte)1; return x & y"));
         assertEquals(0L, exec("def x = (long)4; def y = (byte)1; return x & y"));
-        assertEquals(0L, exec("def x = (float)4; def y = (byte)1; return x & y"));
-        assertEquals(0L, exec("def x = (double)4; def y = (byte)1; return x & y"));
 
         assertEquals(0, exec("def x = (byte)4; def y = (short)1; return x & y"));
         assertEquals(0, exec("def x = (short)4; def y = (short)1; return x & y"));
         assertEquals(0, exec("def x = (char)4; def y = (short)1; return x & y"));
         assertEquals(0, exec("def x = (int)4; def y = (short)1; return x & y"));
         assertEquals(0L, exec("def x = (long)4; def y = (short)1; return x & y"));
-        assertEquals(0L, exec("def x = (float)4; def y = (short)1; return x & y"));
-        assertEquals(0L, exec("def x = (double)4; def y = (short)1; return x & y"));
 
         assertEquals(0, exec("def x = (byte)4; def y = (char)1; return x & y"));
         assertEquals(0, exec("def x = (short)4; def y = (char)1; return x & y"));
         assertEquals(0, exec("def x = (char)4; def y = (char)1; return x & y"));
         assertEquals(0, exec("def x = (int)4; def y = (char)1; return x & y"));
         assertEquals(0L, exec("def x = (long)4; def y = (char)1; return x & y"));
-        assertEquals(0L, exec("def x = (float)4; def y = (char)1; return x & y"));
-        assertEquals(0L, exec("def x = (double)4; def y = (char)1; return x & y"));
 
         assertEquals(0, exec("def x = (byte)4; def y = (int)1; return x & y"));
         assertEquals(0, exec("def x = (short)4; def y = (int)1; return x & y"));
         assertEquals(0, exec("def x = (char)4; def y = (int)1; return x & y"));
         assertEquals(0, exec("def x = (int)4; def y = (int)1; return x & y"));
         assertEquals(0L, exec("def x = (long)4; def y = (int)1; return x & y"));
-        assertEquals(0L, exec("def x = (float)4; def y = (int)1; return x & y"));
-        assertEquals(0L, exec("def x = (double)4; def y = (int)1; return x & y"));
 
         assertEquals(0L, exec("def x = (byte)4; def y = (long)1; return x & y"));
         assertEquals(0L, exec("def x = (short)4; def y = (long)1; return x & y"));
         assertEquals(0L, exec("def x = (char)4; def y = (long)1; return x & y"));
         assertEquals(0L, exec("def x = (int)4; def y = (long)1; return x & y"));
         assertEquals(0L, exec("def x = (long)4; def y = (long)1; return x & y"));
-        assertEquals(0L, exec("def x = (float)4; def y = (long)1; return x & y"));
-        assertEquals(0L, exec("def x = (double)4; def y = (long)1; return x & y"));
-
-        assertEquals(0L, exec("def x = (byte)4; def y = (float)1; return x & y"));
-        assertEquals(0L, exec("def x = (short)4; def y = (float)1; return x & y"));
-        assertEquals(0L, exec("def x = (char)4; def y = (float)1; return x & y"));
-        assertEquals(0L, exec("def x = (int)4; def y = (float)1; return x & y"));
-        assertEquals(0L, exec("def x = (long)4; def y = (float)1; return x & y"));
-        assertEquals(0L, exec("def x = (float)4; def y = (float)1; return x & y"));
-        assertEquals(0L, exec("def x = (double)4; def y = (float)1; return x & y"));
-
-        assertEquals(0L, exec("def x = (byte)4; def y = (double)1; return x & y"));
-        assertEquals(0L, exec("def x = (short)4; def y = (double)1; return x & y"));
-        assertEquals(0L, exec("def x = (char)4; def y = (double)1; return x & y"));
-        assertEquals(0L, exec("def x = (int)4; def y = (double)1; return x & y"));
-        assertEquals(0L, exec("def x = (long)4; def y = (double)1; return x & y"));
-        assertEquals(0L, exec("def x = (float)4; def y = (double)1; return x & y"));
-        assertEquals(0L, exec("def x = (double)4; def y = (double)1; return x & y"));
 
         assertEquals(0, exec("def x = (byte)4; def y = (byte)1; return x & y"));
         assertEquals(0, exec("def x = (short)4; def y = (short)1; return x & y"));
         assertEquals(0, exec("def x = (char)4; def y = (char)1; return x & y"));
         assertEquals(0, exec("def x = (int)4; def y = (int)1; return x & y"));
         assertEquals(0L, exec("def x = (long)4; def y = (long)1; return x & y"));
-        assertEquals(0L, exec("def x = (float)4; def y = (float)1; return x & y"));
-        assertEquals(0L, exec("def x = (double)4; def y = (double)1; return x & y"));
+
+        assertEquals(true,  exec("def x = true;  def y = true; return x & y"));
+        assertEquals(false, exec("def x = true;  def y = false; return x & y"));
+        assertEquals(false, exec("def x = false; def y = true; return x & y"));
+        assertEquals(false, exec("def x = false; def y = false; return x & y"));
     }
 
     public void testXor() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (float)4; def y = (byte)1; return x ^ y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (double)4; def y = (byte)1; return x ^ y");
+        });
         assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x ^ y"));
         assertEquals(5, exec("def x = (short)4; def y = (byte)1; return x ^ y"));
         assertEquals(5, exec("def x = (char)4; def y = (byte)1; return x ^ y"));
         assertEquals(5, exec("def x = (int)4; def y = (byte)1; return x ^ y"));
         assertEquals(5L, exec("def x = (long)4; def y = (byte)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (float)4; def y = (byte)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (double)4; def y = (byte)1; return x ^ y"));
 
         assertEquals(5, exec("def x = (byte)4; def y = (short)1; return x ^ y"));
         assertEquals(5, exec("def x = (short)4; def y = (short)1; return x ^ y"));
         assertEquals(5, exec("def x = (char)4; def y = (short)1; return x ^ y"));
         assertEquals(5, exec("def x = (int)4; def y = (short)1; return x ^ y"));
         assertEquals(5L, exec("def x = (long)4; def y = (short)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (float)4; def y = (short)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (double)4; def y = (short)1; return x ^ y"));
 
         assertEquals(5, exec("def x = (byte)4; def y = (char)1; return x ^ y"));
         assertEquals(5, exec("def x = (short)4; def y = (char)1; return x ^ y"));
         assertEquals(5, exec("def x = (char)4; def y = (char)1; return x ^ y"));
         assertEquals(5, exec("def x = (int)4; def y = (char)1; return x ^ y"));
         assertEquals(5L, exec("def x = (long)4; def y = (char)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (float)4; def y = (char)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (double)4; def y = (char)1; return x ^ y"));
 
         assertEquals(5, exec("def x = (byte)4; def y = (int)1; return x ^ y"));
         assertEquals(5, exec("def x = (short)4; def y = (int)1; return x ^ y"));
         assertEquals(5, exec("def x = (char)4; def y = (int)1; return x ^ y"));
         assertEquals(5, exec("def x = (int)4; def y = (int)1; return x ^ y"));
         assertEquals(5L, exec("def x = (long)4; def y = (int)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (float)4; def y = (int)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (double)4; def y = (int)1; return x ^ y"));
 
         assertEquals(5L, exec("def x = (byte)4; def y = (long)1; return x ^ y"));
         assertEquals(5L, exec("def x = (short)4; def y = (long)1; return x ^ y"));
         assertEquals(5L, exec("def x = (char)4; def y = (long)1; return x ^ y"));
         assertEquals(5L, exec("def x = (int)4; def y = (long)1; return x ^ y"));
         assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (float)4; def y = (long)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (double)4; def y = (long)1; return x ^ y"));
-
-        assertEquals(5L, exec("def x = (byte)4; def y = (float)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (short)4; def y = (float)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (char)4; def y = (float)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (int)4; def y = (float)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4; def y = (float)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (float)4; def y = (float)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (double)4; def y = (float)1; return x ^ y"));
-
-        assertEquals(5L, exec("def x = (byte)4; def y = (double)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (short)4; def y = (double)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (char)4; def y = (double)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (int)4; def y = (double)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4; def y = (double)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (float)4; def y = (double)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (double)4; def y = (double)1; return x ^ y"));
 
         assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x ^ y"));
         assertEquals(5, exec("def x = (short)4; def y = (short)1; return x ^ y"));
         assertEquals(5, exec("def x = (char)4; def y = (char)1; return x ^ y"));
         assertEquals(5, exec("def x = (int)4; def y = (int)1; return x ^ y"));
         assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (float)4; def y = (float)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (double)4; def y = (double)1; return x ^ y"));
+        
+        assertEquals(false, exec("def x = true;  def y = true; return x ^ y"));
+        assertEquals(true,  exec("def x = true;  def y = false; return x ^ y"));
+        assertEquals(true,  exec("def x = false; def y = true; return x ^ y"));
+        assertEquals(false, exec("def x = false; def y = false; return x ^ y"));
     }
 
     public void testOr() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (float)4; def y = (byte)1; return x | y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (double)4; def y = (byte)1; return x | y");
+        });
         assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x | y"));
         assertEquals(5, exec("def x = (short)4; def y = (byte)1; return x | y"));
         assertEquals(5, exec("def x = (char)4; def y = (byte)1; return x | y"));
         assertEquals(5, exec("def x = (int)4; def y = (byte)1; return x | y"));
         assertEquals(5L, exec("def x = (long)4; def y = (byte)1; return x | y"));
-        assertEquals(5L, exec("def x = (float)4; def y = (byte)1; return x | y"));
-        assertEquals(5L, exec("def x = (double)4; def y = (byte)1; return x | y"));
 
         assertEquals(5, exec("def x = (byte)4; def y = (short)1; return x | y"));
         assertEquals(5, exec("def x = (short)4; def y = (short)1; return x | y"));
         assertEquals(5, exec("def x = (char)4; def y = (short)1; return x | y"));
         assertEquals(5, exec("def x = (int)4; def y = (short)1; return x | y"));
         assertEquals(5L, exec("def x = (long)4; def y = (short)1; return x | y"));
-        assertEquals(5L, exec("def x = (float)4; def y = (short)1; return x | y"));
-        assertEquals(5L, exec("def x = (double)4; def y = (short)1; return x | y"));
 
         assertEquals(5, exec("def x = (byte)4; def y = (char)1; return x | y"));
         assertEquals(5, exec("def x = (short)4; def y = (char)1; return x | y"));
         assertEquals(5, exec("def x = (char)4; def y = (char)1; return x | y"));
         assertEquals(5, exec("def x = (int)4; def y = (char)1; return x | y"));
         assertEquals(5L, exec("def x = (long)4; def y = (char)1; return x | y"));
-        assertEquals(5L, exec("def x = (float)4; def y = (char)1; return x | y"));
-        assertEquals(5L, exec("def x = (double)4; def y = (char)1; return x | y"));
 
         assertEquals(5, exec("def x = (byte)4; def y = (int)1; return x | y"));
         assertEquals(5, exec("def x = (short)4; def y = (int)1; return x | y"));
         assertEquals(5, exec("def x = (char)4; def y = (int)1; return x | y"));
         assertEquals(5, exec("def x = (int)4; def y = (int)1; return x | y"));
         assertEquals(5L, exec("def x = (long)4; def y = (int)1; return x | y"));
-        assertEquals(5L, exec("def x = (float)4; def y = (int)1; return x | y"));
-        assertEquals(5L, exec("def x = (double)4; def y = (int)1; return x | y"));
 
         assertEquals(5L, exec("def x = (byte)4; def y = (long)1; return x | y"));
         assertEquals(5L, exec("def x = (short)4; def y = (long)1; return x | y"));
         assertEquals(5L, exec("def x = (char)4; def y = (long)1; return x | y"));
         assertEquals(5L, exec("def x = (int)4; def y = (long)1; return x | y"));
         assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x | y"));
-        assertEquals(5L, exec("def x = (float)4; def y = (long)1; return x | y"));
-        assertEquals(5L, exec("def x = (double)4; def y = (long)1; return x | y"));
-
-        assertEquals(5L, exec("def x = (byte)4; def y = (float)1; return x | y"));
-        assertEquals(5L, exec("def x = (short)4; def y = (float)1; return x | y"));
-        assertEquals(5L, exec("def x = (char)4; def y = (float)1; return x | y"));
-        assertEquals(5L, exec("def x = (int)4; def y = (float)1; return x | y"));
-        assertEquals(5L, exec("def x = (long)4; def y = (float)1; return x | y"));
-        assertEquals(5L, exec("def x = (float)4; def y = (float)1; return x | y"));
-        assertEquals(5L, exec("def x = (double)4; def y = (float)1; return x | y"));
-
-        assertEquals(5L, exec("def x = (byte)4; def y = (double)1; return x | y"));
-        assertEquals(5L, exec("def x = (short)4; def y = (double)1; return x | y"));
-        assertEquals(5L, exec("def x = (char)4; def y = (double)1; return x | y"));
-        assertEquals(5L, exec("def x = (int)4; def y = (double)1; return x | y"));
-        assertEquals(5L, exec("def x = (long)4; def y = (double)1; return x | y"));
-        assertEquals(5L, exec("def x = (float)4; def y = (double)1; return x | y"));
-        assertEquals(5L, exec("def x = (double)4; def y = (double)1; return x | y"));
 
         assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x | y"));
         assertEquals(5, exec("def x = (short)4; def y = (short)1; return x | y"));
         assertEquals(5, exec("def x = (char)4; def y = (char)1; return x | y"));
         assertEquals(5, exec("def x = (int)4; def y = (int)1; return x | y"));
         assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x | y"));
-        assertEquals(5L, exec("def x = (float)4; def y = (float)1; return x | y"));
-        assertEquals(5L, exec("def x = (double)4; def y = (double)1; return x | y"));
+        
+        assertEquals(true,  exec("def x = true;  def y = true; return x | y"));
+        assertEquals(true,  exec("def x = true;  def y = false; return x | y"));
+        assertEquals(true,  exec("def x = false; def y = true; return x | y"));
+        assertEquals(false, exec("def x = false; def y = false; return x | y"));
     }
 
     public void testEq() {
@@ -792,11 +731,23 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(false, exec("def x = (long)5; def y = (double)3; return x == y"));
         assertEquals(false, exec("def x = (float)6; def y = (double)2; return x == y"));
         assertEquals(false, exec("def x = (double)7; def y = (double)1; return x == y"));
+        
+        assertEquals(false, exec("def x = false; def y = true; return x == y"));
+        assertEquals(false, exec("def x = true; def y = false; return x == y"));
+        assertEquals(false, exec("def x = true; def y = null; return x == y"));
+        assertEquals(false, exec("def x = null; def y = true; return x == y"));
+        assertEquals(true, exec("def x = true; def y = true; return x == y"));
+        assertEquals(true, exec("def x = false; def y = false; return x == y"));
 
         assertEquals(true, exec("def x = new HashMap(); def y = new HashMap(); return x == y"));
         assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x == y"));
         assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x == y"));
         assertEquals(true, exec("def x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x == y"));
+        
+        assertEquals(true,  exec("def x = true;  def y = true; return x == y"));
+        assertEquals(false, exec("def x = true;  def y = false; return x == y"));
+        assertEquals(false, exec("def x = false; def y = true; return x == y"));
+        assertEquals(true,  exec("def x = false; def y = false; return x == y"));
     }
 
     public void testEqr() {
@@ -807,6 +758,7 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(false, exec("def x = (long)5; def y = (int)3; return x === y"));
         assertEquals(false, exec("def x = (float)6; def y = (int)2; return x === y"));
         assertEquals(false, exec("def x = (double)7; def y = (int)1; return x === y"));
+        assertEquals(false, exec("def x = false; def y = true; return x === y"));
 
         assertEquals(false, exec("def x = new HashMap(); def y = new HashMap(); return x === y"));
         assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x === y"));
@@ -835,6 +787,11 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x != y"));
         assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x != y"));
         assertEquals(false, exec("def x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x != y"));
+        
+        assertEquals(false,  exec("def x = true;  def y = true; return x != y"));
+        assertEquals(true,   exec("def x = true;  def y = false; return x != y"));
+        assertEquals(true,   exec("def x = false; def y = true; return x != y"));
+        assertEquals(false,  exec("def x = false; def y = false; return x != y"));
     }
 
     public void testNer() {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java
index f287b1e4cf4..d27c019a798 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java
@@ -45,4 +45,13 @@ public class OrTests extends ScriptTestCase {
         assertEquals(5L | -12L, exec("return 5L | -12L;"));
         assertEquals(7L | 15L | 3L, exec("return 7L | 15L | 3L;"));
     }
+    
+    public void testIllegal() throws Exception {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("float x = (float)4; int y = 1; return x | y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("double x = (double)4; int y = 1; return x | y");
+        });
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/UnaryTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/UnaryTests.java
index e670e23925b..ad2876778db 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/UnaryTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/UnaryTests.java
@@ -39,4 +39,14 @@ public class UnaryTests extends ScriptTestCase {
         assertEquals(1, exec("return -(-1);"));
         assertEquals(0, exec("return -0;"));
     }
+    
+    public void testPlus() {
+        assertEquals(-1, exec("byte x = (byte)-1; return +x"));
+        assertEquals(-1, exec("short x = (short)-1; return +x"));
+        assertEquals(65535, exec("char x = (char)-1; return +x"));
+        assertEquals(-1, exec("int x = -1; return +x"));
+        assertEquals(-1L, exec("long x = -1L; return +x"));
+        assertEquals(-1.0F, exec("float x = -1F; return +x"));
+        assertEquals(-1.0, exec("double x = -1.0; return +x"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java
index f5dd0a92011..16436240459 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java
@@ -59,4 +59,13 @@ public class XorTests extends ScriptTestCase {
         assertEquals(true, exec("return false ^ true;"));
         assertEquals(false, exec("return false ^ false;"));
     }
+    
+    public void testIllegal() throws Exception {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("float x = (float)4; int y = 1; return x ^ y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("double x = (double)4; int y = 1; return x ^ y");
+        });
+    }
 }

From a12937d548bf886bc4ce96f6af2d1a04cd2eba93 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 14 Jun 2016 00:49:13 -0400
Subject: [PATCH 0823/1311] painless: remove casts and boxing for dynamic math

---
 .../elasticsearch/painless/MethodWriter.java  |  116 +-
 .../elasticsearch/painless/node/EBinary.java  |  215 ++-
 .../elasticsearch/painless/node/EChain.java   |   10 +-
 .../elasticsearch/painless/node/EComp.java    |  148 +-
 .../painless/DefOperationTests.java           | 1505 ++++++++++++++++-
 .../painless/DefOptimizationTests.java        |  225 +++
 6 files changed, 2027 insertions(+), 192 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index ae46ee54f35..97502384d22 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -262,6 +262,50 @@ public final class MethodWriter extends GeneratorAdapter {
         }
     }
 
+    /** Writes a dynamic binary instruction: returnType, lhs, and rhs can be different */
+    public void writeDynamicBinaryInstruction(Location location, Type returnType, Type lhs, Type rhs, Operation operation) {
+        org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(returnType.type, lhs.type, rhs.type);
+        
+        switch (operation) {
+            case MUL:
+                invokeDynamic("mul", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
+                break;
+            case DIV:
+                invokeDynamic("div", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
+                break;
+            case REM:
+                invokeDynamic("rem", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
+                break;
+            case ADD:
+                invokeDynamic("add", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
+                break;
+            case SUB:
+                invokeDynamic("sub", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
+                break;
+            case LSH:
+                invokeDynamic("lsh", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR); 
+                break;
+            case USH:
+                invokeDynamic("ush", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR); 
+                break;
+            case RSH:
+                invokeDynamic("rsh", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR); 
+                break;
+            case BWAND: 
+                invokeDynamic("and", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
+                break;
+            case XOR:   
+                invokeDynamic("xor", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
+                break;
+            case BWOR:  
+                invokeDynamic("or", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
+                break;
+            default:
+                throw location.createError(new IllegalStateException("Illegal tree structure."));
+        }
+    }
+    
+    /** Writes a static binary instruction */
     public void writeBinaryInstruction(Location location, Type type, Operation operation) {
         final Sort sort = type.sort;
 
@@ -272,64 +316,20 @@ public final class MethodWriter extends GeneratorAdapter {
             throw location.createError(new IllegalStateException("Illegal tree structure."));
         }
 
-        if (sort == Sort.DEF) {
-            // XXX: move this out, so we can populate descriptor with what we really have (instead of casts/boxing!)
-            org.objectweb.asm.Type objectType = org.objectweb.asm.Type.getType(Object.class);
-            org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(objectType, objectType, objectType);
-            
-            switch (operation) {
-                case MUL:
-                    invokeDynamic("mul", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
-                    break;
-                case DIV:
-                    invokeDynamic("div", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
-                    break;
-                case REM:
-                    invokeDynamic("rem", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
-                    break;
-                case ADD:
-                    invokeDynamic("add", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
-                    break;
-                case SUB:
-                    invokeDynamic("sub", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
-                    break;
-                case LSH:
-                    invokeDynamic("lsh", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR); 
-                    break;
-                case USH:
-                    invokeDynamic("ush", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR); 
-                    break;
-                case RSH:
-                    invokeDynamic("rsh", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR); 
-                    break;
-                case BWAND: 
-                    invokeDynamic("and", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
-                    break;
-                case XOR:   
-                    invokeDynamic("xor", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
-                    break;
-                case BWOR:  
-                    invokeDynamic("or", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
-                    break;
-                default:
-                    throw location.createError(new IllegalStateException("Illegal tree structure."));
-            }
-        } else {
-            switch (operation) {
-                case MUL:   math(GeneratorAdapter.MUL,  type.type); break;
-                case DIV:   math(GeneratorAdapter.DIV,  type.type); break;
-                case REM:   math(GeneratorAdapter.REM,  type.type); break;
-                case ADD:   math(GeneratorAdapter.ADD,  type.type); break;
-                case SUB:   math(GeneratorAdapter.SUB,  type.type); break;
-                case LSH:   math(GeneratorAdapter.SHL,  type.type); break;
-                case USH:   math(GeneratorAdapter.USHR, type.type); break;
-                case RSH:   math(GeneratorAdapter.SHR,  type.type); break;
-                case BWAND: math(GeneratorAdapter.AND,  type.type); break;
-                case XOR:   math(GeneratorAdapter.XOR,  type.type); break;
-                case BWOR:  math(GeneratorAdapter.OR,   type.type); break;
-                default:
-                    throw location.createError(new IllegalStateException("Illegal tree structure."));
-            }
+        switch (operation) {
+            case MUL:   math(GeneratorAdapter.MUL,  type.type); break;
+            case DIV:   math(GeneratorAdapter.DIV,  type.type); break;
+            case REM:   math(GeneratorAdapter.REM,  type.type); break;
+            case ADD:   math(GeneratorAdapter.ADD,  type.type); break;
+            case SUB:   math(GeneratorAdapter.SUB,  type.type); break;
+            case LSH:   math(GeneratorAdapter.SHL,  type.type); break;
+            case USH:   math(GeneratorAdapter.USHR, type.type); break;
+            case RSH:   math(GeneratorAdapter.SHR,  type.type); break;
+            case BWAND: math(GeneratorAdapter.AND,  type.type); break;
+            case XOR:   math(GeneratorAdapter.XOR,  type.type); break;
+            case BWOR:  math(GeneratorAdapter.OR,   type.type); break;
+            default:
+                throw location.createError(new IllegalStateException("Illegal tree structure."));
         }
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 9f8f1553770..23f9a614338 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -36,6 +36,8 @@ public final class EBinary extends AExpression {
     final Operation operation;
     AExpression left;
     AExpression right;
+    Type promote;       // promoted type
+    Type shiftDistance; // for shifts, the RHS is promoted independently
 
     boolean cat = false;
 
@@ -80,15 +82,25 @@ public final class EBinary extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+        promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply multiply [*] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        left.expected = promote;
-        right.expected = promote;
+        actual = promote;
+
+        if (promote.sort == Sort.DEF) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+            if (expected != null) {
+                actual = expected;
+            }
+        } else {
+            left.expected = promote;
+            right.expected = promote;
+        }
 
         left = left.cast(variables);
         right = right.cast(variables);
@@ -108,23 +120,30 @@ public final class EBinary extends AExpression {
                 throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
-
-        actual = promote;
     }
 
     private void analyzeDiv(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
-        Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+        promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply divide [/] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        left.expected = promote;
-        right.expected = promote;
+        actual = promote;
+        if (promote.sort == Sort.DEF) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+            if (expected != null) {
+                actual = expected;
+            }
+        } else {
+            left.expected = promote;
+            right.expected = promote;
+        }
 
         left = left.cast(variables);
         right = right.cast(variables);
@@ -148,23 +167,31 @@ public final class EBinary extends AExpression {
                 throw createError(e);
             }
         }
-
-        actual = promote;
     }
 
     private void analyzeRem(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
-        Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+        promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply remainder [%] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        left.expected = promote;
-        right.expected = promote;
+        actual = promote;
+
+        if (promote.sort == Sort.DEF) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+            if (expected != null) {
+                actual = expected;
+            }
+        } else {
+            left.expected = promote;
+            right.expected = promote;
+        }
 
         left = left.cast(variables);
         right = right.cast(variables);
@@ -188,15 +215,13 @@ public final class EBinary extends AExpression {
                 throw createError(e);
             }
         }
-
-        actual = promote;
     }
 
     private void analyzeAdd(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
-        Type promote = AnalyzerCaster.promoteAdd(left.actual, right.actual);
+        promote = AnalyzerCaster.promoteAdd(left.actual, right.actual);
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply add [+] to types " +
@@ -205,6 +230,8 @@ public final class EBinary extends AExpression {
 
         Sort sort = promote.sort;
 
+        actual = promote;
+
         if (sort == Sort.STRING) {
             left.expected = left.actual;
 
@@ -217,6 +244,12 @@ public final class EBinary extends AExpression {
             if (right instanceof EBinary && ((EBinary)right).operation == Operation.ADD && right.actual.sort == Sort.STRING) {
                 ((EBinary)right).cat = true;
             }
+        } else if (sort == Sort.DEF) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+            if (expected != null) {
+                actual = expected;
+            }
         } else {
             left.expected = promote;
             right.expected = promote;
@@ -241,22 +274,31 @@ public final class EBinary extends AExpression {
             }
         }
 
-        actual = promote;
     }
 
     private void analyzeSub(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
-        Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+        promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply subtract [-] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        left.expected = promote;
-        right.expected = promote;
+        actual = promote;
+
+        if (promote.sort == Sort.DEF) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+            if (expected != null) {
+                actual = expected;
+            }
+        } else {
+            left.expected = promote;
+            right.expected = promote;
+        }
 
         left = left.cast(variables);
         right = right.cast(variables);
@@ -276,8 +318,6 @@ public final class EBinary extends AExpression {
                 throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
-
-        actual = promote;
     }
 
     private void analyzeLSH(Locals variables) {
@@ -292,12 +332,23 @@ public final class EBinary extends AExpression {
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        left.expected = lhspromote;
-        if (rhspromote.sort == Sort.LONG) {
-            right.expected = Definition.INT_TYPE;
-            right.explicit = true;
+        actual = promote = lhspromote;
+        shiftDistance = rhspromote;
+
+        if (lhspromote.sort == Sort.DEF || rhspromote.sort == Sort.DEF) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+            if (expected != null) {
+                actual = expected;
+            }
         } else {
-            right.expected = rhspromote;
+            left.expected = lhspromote;
+            if (rhspromote.sort == Sort.LONG) {
+                right.expected = Definition.INT_TYPE;
+                right.explicit = true;
+            } else {
+                right.expected = rhspromote;
+            }
         }
 
         left = left.cast(variables);
@@ -314,8 +365,6 @@ public final class EBinary extends AExpression {
                 throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
-
-        actual = lhspromote;
     }
 
     private void analyzeRSH(Locals variables) {
@@ -330,12 +379,23 @@ public final class EBinary extends AExpression {
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        left.expected = lhspromote;
-        if (rhspromote.sort == Sort.LONG) { 
-            right.expected = Definition.INT_TYPE;
-            right.explicit = true;
+        actual = promote = lhspromote;
+        shiftDistance = rhspromote;
+
+        if (lhspromote.sort == Sort.DEF || rhspromote.sort == Sort.DEF) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+            if (expected != null) {
+                actual = expected;
+            }
         } else {
-            right.expected = rhspromote;
+            left.expected = lhspromote;
+            if (rhspromote.sort == Sort.LONG) { 
+                right.expected = Definition.INT_TYPE;
+                right.explicit = true;
+            } else {
+                right.expected = rhspromote;
+            }
         }
 
         left = left.cast(variables);
@@ -352,8 +412,6 @@ public final class EBinary extends AExpression {
                 throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
-
-        actual = lhspromote;
     }
 
     private void analyzeUSH(Locals variables) {
@@ -363,17 +421,28 @@ public final class EBinary extends AExpression {
         Type lhspromote = AnalyzerCaster.promoteNumeric(left.actual, false);
         Type rhspromote = AnalyzerCaster.promoteNumeric(right.actual, false);
 
+        actual = promote = lhspromote;
+        shiftDistance = rhspromote;
+
         if (lhspromote == null || rhspromote == null) {
             throw createError(new ClassCastException("Cannot apply unsigned shift [>>>] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        left.expected = lhspromote;
-        if (rhspromote.sort == Sort.LONG) { 
-            right.expected = Definition.INT_TYPE;
-            right.explicit = true;
+        if (lhspromote.sort == Sort.DEF || rhspromote.sort == Sort.DEF) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+            if (expected != null) {
+                actual = expected;
+            }
         } else {
-            right.expected = rhspromote;
+            left.expected = lhspromote;
+            if (rhspromote.sort == Sort.LONG) { 
+                right.expected = Definition.INT_TYPE;
+                right.explicit = true;
+            } else {
+                right.expected = rhspromote;
+            }
         }
 
         left = left.cast(variables);
@@ -390,23 +459,31 @@ public final class EBinary extends AExpression {
                 throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
-
-        actual = lhspromote;
     }
 
     private void analyzeBWAnd(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
-        Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false);
+        promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false);
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply and [&] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        left.expected = promote;
-        right.expected = promote;
+        actual = promote;
+
+        if (promote.sort == Sort.DEF) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+            if (expected != null) {
+                actual = expected;
+            }
+        } else {
+            left.expected = promote;
+            right.expected = promote;
+        }
 
         left = left.cast(variables);
         right = right.cast(variables);
@@ -422,23 +499,31 @@ public final class EBinary extends AExpression {
                 throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
-
-        actual = promote;
     }
 
     private void analyzeXor(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
-        Type promote = AnalyzerCaster.promoteXor(left.actual, right.actual);
+        promote = AnalyzerCaster.promoteXor(left.actual, right.actual);
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply xor [^] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        left.expected = promote;
-        right.expected = promote;
+        actual = promote;
+
+        if (promote.sort == Sort.DEF) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+            if (expected != null) {
+                actual = expected;
+            }
+        } else {
+            left.expected = promote;
+            right.expected = promote;
+        }
 
         left = left.cast(variables);
         right = right.cast(variables);
@@ -456,23 +541,31 @@ public final class EBinary extends AExpression {
                 throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
-
-        actual = promote;
     }
 
     private void analyzeBWOr(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
 
-        Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false);
+        promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, false);
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply or [|] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        left.expected = promote;
-        right.expected = promote;
+        actual = promote;
+
+        if (promote.sort == Sort.DEF) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+            if (expected != null) {
+                actual = expected;
+            }
+        } else {
+            left.expected = promote;
+            right.expected = promote;
+        }
 
         left = left.cast(variables);
         right = right.cast(variables);
@@ -488,15 +581,13 @@ public final class EBinary extends AExpression {
                 throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
-
-        actual = promote;
     }
 
     @Override
     void write(MethodWriter writer) {
         writer.writeDebugInfo(location);
 
-        if (actual.sort == Sort.STRING && operation == Operation.ADD) {
+        if (promote.sort == Sort.STRING && operation == Operation.ADD) {
             if (!cat) {
                 writer.writeNewStrings();
             }
@@ -520,7 +611,11 @@ public final class EBinary extends AExpression {
             left.write(writer);
             right.write(writer);
 
-            writer.writeBinaryInstruction(location, actual, operation);
+            if (promote.sort == Sort.DEF || (shiftDistance != null && shiftDistance.sort == Sort.DEF)) {
+                writer.writeDynamicBinaryInstruction(location, expected, left.actual, right.actual, operation);
+            } else {
+                writer.writeBinaryInstruction(location, actual, operation);
+            }
         }
 
         writer.writeBranch(tru, fals);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index e7ee7697da7..2d47c520b7b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -334,7 +334,15 @@ public final class EChain extends AExpression {
                     writer.writeCast(there);                                     // if necessary cast the current link's value
                                                                                  // to the promotion type between the lhs and rhs types
                     expression.write(writer);                                    // write the bytecode for the rhs expression
-                    writer.writeBinaryInstruction(location, promote, operation); // write the operation instruction for compound assignment
+                    // XXX: fix these types, but first we need def compound assignment tests.
+                    // (and also corner cases such as shifts). its tricky here as there are possibly explicit casts, too.
+                    // write the operation instruction for compound assignment
+                    if (promote.sort == Sort.DEF) {
+                        writer.writeDynamicBinaryInstruction(location, promote, 
+                            Definition.DEF_TYPE, Definition.DEF_TYPE, operation);
+                    } else {
+                        writer.writeBinaryInstruction(location, promote, operation);
+                    }
 
                     writer.writeCast(back); // if necessary cast the promotion type value back to the link's type
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index a126cd7d936..9d9c6c03812 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -42,6 +42,7 @@ public final class EComp extends AExpression {
     final Operation operation;
     AExpression left;
     AExpression right;
+    Type promotedType;
 
     public EComp(Location location, Operation operation, AExpression left, AExpression right) {
         super(location);
@@ -78,15 +79,20 @@ public final class EComp extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual);
+        promotedType = AnalyzerCaster.promoteEquality(left.actual, right.actual);
 
-        if (promote == null) {
+        if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply equals [==] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        left.expected = promote;
-        right.expected = promote;
+        if (promotedType.sort == Sort.DEF) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+        } else {
+            left.expected = promotedType;
+            right.expected = promotedType;
+        }
 
         left = left.cast(variables);
         right = right.cast(variables);
@@ -96,7 +102,7 @@ public final class EComp extends AExpression {
         }
 
         if ((left.constant != null || left.isNull) && (right.constant != null || right.isNull)) {
-            Sort sort = promote.sort;
+            Sort sort = promotedType.sort;
 
             if (sort == Sort.BOOL) {
                 constant = (boolean)left.constant == (boolean)right.constant;
@@ -124,15 +130,20 @@ public final class EComp extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual);
+        promotedType = AnalyzerCaster.promoteEquality(left.actual, right.actual);
 
-        if (promote == null) {
+        if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply reference equals [===] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        left.expected = promote;
-        right.expected = promote;
+        if (promotedType.sort == Sort.DEF) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+        } else {
+            left.expected = promotedType;
+            right.expected = promotedType;
+        }
 
         left = left.cast(variables);
         right = right.cast(variables);
@@ -142,7 +153,7 @@ public final class EComp extends AExpression {
         }
 
         if ((left.constant != null || left.isNull) && (right.constant != null || right.isNull)) {
-            Sort sort = promote.sort;
+            Sort sort = promotedType.sort;
 
             if (sort == Sort.BOOL) {
                 constant = (boolean)left.constant == (boolean)right.constant;
@@ -166,15 +177,20 @@ public final class EComp extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual);
+        promotedType = AnalyzerCaster.promoteEquality(left.actual, right.actual);
 
-        if (promote == null) {
+        if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply not equals [!=] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        left.expected = promote;
-        right.expected = promote;
+        if (promotedType.sort == Sort.DEF) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+        } else {
+            left.expected = promotedType;
+            right.expected = promotedType;
+        }
 
         left = left.cast(variables);
         right = right.cast(variables);
@@ -184,7 +200,7 @@ public final class EComp extends AExpression {
         }
 
         if ((left.constant != null || left.isNull) && (right.constant != null || right.isNull)) {
-            Sort sort = promote.sort;
+            Sort sort = promotedType.sort;
 
             if (sort == Sort.BOOL) {
                 constant = (boolean)left.constant != (boolean)right.constant;
@@ -212,15 +228,20 @@ public final class EComp extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        Type promote = AnalyzerCaster.promoteEquality(left.actual, right.actual);
+        promotedType = AnalyzerCaster.promoteEquality(left.actual, right.actual);
 
-        if (promote == null) {
+        if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply reference not equals [!==] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        left.expected = promote;
-        right.expected = promote;
+        if (promotedType.sort == Sort.DEF) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+        } else {
+            left.expected = promotedType;
+            right.expected = promotedType;
+        }
 
         left = left.cast(variables);
         right = right.cast(variables);
@@ -230,7 +251,7 @@ public final class EComp extends AExpression {
         }
 
         if ((left.constant != null || left.isNull) && (right.constant != null || right.isNull)) {
-            Sort sort = promote.sort;
+            Sort sort = promotedType.sort;
 
             if (sort == Sort.BOOL) {
                 constant = (boolean)left.constant != (boolean)right.constant;
@@ -254,21 +275,26 @@ public final class EComp extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+        promotedType = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
-        if (promote == null) {
+        if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply greater than or equals [>=] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        left.expected = promote;
-        right.expected = promote;
+        if (promotedType.sort == Sort.DEF) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+        } else {
+            left.expected = promotedType;
+            right.expected = promotedType;
+        }
 
         left = left.cast(variables);
         right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
-            Sort sort = promote.sort;
+            Sort sort = promotedType.sort;
 
             if (sort == Sort.INT) {
                 constant = (int)left.constant >= (int)right.constant;
@@ -290,21 +316,26 @@ public final class EComp extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+        promotedType = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
-        if (promote == null) {
+        if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply greater than [>] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        left.expected = promote;
-        right.expected = promote;
+        if (promotedType.sort == Sort.DEF) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+        } else {
+            left.expected = promotedType;
+            right.expected = promotedType;
+        }
 
         left = left.cast(variables);
         right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
-            Sort sort = promote.sort;
+            Sort sort = promotedType.sort;
 
             if (sort == Sort.INT) {
                 constant = (int)left.constant > (int)right.constant;
@@ -326,21 +357,26 @@ public final class EComp extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+        promotedType = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
-        if (promote == null) {
+        if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply less than or equals [<=] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        left.expected = promote;
-        right.expected = promote;
+        if (promotedType.sort == Sort.DEF) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+        } else {
+            left.expected = promotedType;
+            right.expected = promotedType;
+        }
 
         left = left.cast(variables);
         right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
-            Sort sort = promote.sort;
+            Sort sort = promotedType.sort;
 
             if (sort == Sort.INT) {
                 constant = (int)left.constant <= (int)right.constant;
@@ -362,21 +398,26 @@ public final class EComp extends AExpression {
         left.analyze(variables);
         right.analyze(variables);
 
-        Type promote = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
+        promotedType = AnalyzerCaster.promoteNumeric(left.actual, right.actual, true);
 
-        if (promote == null) {
+        if (promotedType == null) {
             throw createError(new ClassCastException("Cannot apply less than [>=] to types " +
                 "[" + left.actual.name + "] and [" + right.actual.name + "]."));
         }
 
-        left.expected = promote;
-        right.expected = promote;
+        if (promotedType.sort == Sort.DEF) {
+            left.expected = left.actual;
+            right.expected = right.actual;
+        } else {
+            left.expected = promotedType;
+            right.expected = promotedType;
+        }
 
         left = left.cast(variables);
         right = right.cast(variables);
 
         if (left.constant != null && right.constant != null) {
-            Sort sort = promote.sort;
+            Sort sort = promotedType.sort;
 
             if (sort == Sort.INT) {
                 constant = (int)left.constant < (int)right.constant;
@@ -399,8 +440,6 @@ public final class EComp extends AExpression {
         writer.writeDebugInfo(location);
 
         boolean branch = tru != null || fals != null;
-        org.objectweb.asm.Type rtype = right.actual.type;
-        Sort rsort = right.actual.sort;
 
         left.write(writer);
 
@@ -422,7 +461,7 @@ public final class EComp extends AExpression {
 
         boolean writejump = true;
 
-        switch (rsort) {
+        switch (promotedType.sort) {
             case VOID:
             case BYTE:
             case SHORT:
@@ -440,12 +479,12 @@ public final class EComp extends AExpression {
             case LONG:
             case FLOAT:
             case DOUBLE:
-                if      (eq)  writer.ifCmp(rtype, MethodWriter.EQ, jump);
-                else if (ne)  writer.ifCmp(rtype, MethodWriter.NE, jump);
-                else if (lt)  writer.ifCmp(rtype, MethodWriter.LT, jump);
-                else if (lte) writer.ifCmp(rtype, MethodWriter.LE, jump);
-                else if (gt)  writer.ifCmp(rtype, MethodWriter.GT, jump);
-                else if (gte) writer.ifCmp(rtype, MethodWriter.GE, jump);
+                if      (eq)  writer.ifCmp(promotedType.type, MethodWriter.EQ, jump);
+                else if (ne)  writer.ifCmp(promotedType.type, MethodWriter.NE, jump);
+                else if (lt)  writer.ifCmp(promotedType.type, MethodWriter.LT, jump);
+                else if (lte) writer.ifCmp(promotedType.type, MethodWriter.LE, jump);
+                else if (gt)  writer.ifCmp(promotedType.type, MethodWriter.GT, jump);
+                else if (gte) writer.ifCmp(promotedType.type, MethodWriter.GE, jump);
                 else {
                     throw createError(new IllegalStateException("Illegal tree structure."));
                 }
@@ -454,8 +493,7 @@ public final class EComp extends AExpression {
             case DEF:
                 // XXX: move this out, so we can populate descriptor with what we really have (instead of casts/boxing!)
                 org.objectweb.asm.Type booleanType = org.objectweb.asm.Type.getType(boolean.class);
-                org.objectweb.asm.Type objectType = org.objectweb.asm.Type.getType(Object.class);
-                org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(booleanType, objectType, objectType);
+                org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(booleanType, left.actual.type, right.actual.type);
                 if (eq) {
                     if (right.isNull) {
                         writer.ifNull(jump);
@@ -463,7 +501,7 @@ public final class EComp extends AExpression {
                         writer.invokeDynamic("eq", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
                         writejump = false;
                     } else {
-                        writer.ifCmp(rtype, MethodWriter.EQ, jump);
+                        writer.ifCmp(promotedType.type, MethodWriter.EQ, jump);
                     }
                 } else if (ne) {
                     if (right.isNull) {
@@ -472,7 +510,7 @@ public final class EComp extends AExpression {
                         writer.invokeDynamic("eq", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
                         writer.ifZCmp(MethodWriter.EQ, jump);
                     } else {
-                        writer.ifCmp(rtype, MethodWriter.NE, jump);
+                        writer.ifCmp(promotedType.type, MethodWriter.NE, jump);
                     }
                 } else if (lt) {
                     writer.invokeDynamic("lt", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
@@ -508,7 +546,7 @@ public final class EComp extends AExpression {
 
                         writejump = false;
                     } else {
-                        writer.ifCmp(rtype, MethodWriter.EQ, jump);
+                        writer.ifCmp(promotedType.type, MethodWriter.EQ, jump);
                     }
                 } else if (ne) {
                     if (right.isNull) {
@@ -517,7 +555,7 @@ public final class EComp extends AExpression {
                         writer.invokeStatic(OBJECTS_TYPE, EQUALS);
                         writer.ifZCmp(MethodWriter.EQ, jump);
                     } else {
-                        writer.ifCmp(rtype, MethodWriter.NE, jump);
+                        writer.ifCmp(promotedType.type, MethodWriter.NE, jump);
                     }
                 } else {
                     throw createError(new IllegalStateException("Illegal tree structure."));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
index bb720943f9d..9217b9e1210 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
@@ -125,6 +125,138 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(4F, exec("def x = (float)2; def y = (float)2; return x * y"));
         assertEquals(4D, exec("def x = (double)2; def y = (double)2; return x * y"));
     }
+    
+    public void testMulTypedLHS() {
+        assertEquals(4, exec("byte x = (byte)2; def y = (byte)2; return x * y"));
+        assertEquals(4, exec("short x = (short)2; def y = (byte)2; return x * y"));
+        assertEquals(4, exec("char x = (char)2; def y = (byte)2; return x * y"));
+        assertEquals(4, exec("int x = (int)2; def y = (byte)2; return x * y"));
+        assertEquals(4L, exec("long x = (long)2; def y = (byte)2; return x * y"));
+        assertEquals(4F, exec("float x = (float)2; def y = (byte)2; return x * y"));
+        assertEquals(4D, exec("double x = (double)2; def y = (byte)2; return x * y"));
+
+        assertEquals(4, exec("byte x = (byte)2; def y = (short)2; return x * y"));
+        assertEquals(4, exec("short x = (short)2; def y = (short)2; return x * y"));
+        assertEquals(4, exec("char x = (char)2; def y = (short)2; return x * y"));
+        assertEquals(4, exec("int x = (int)2; def y = (short)2; return x * y"));
+        assertEquals(4L, exec("long x = (long)2; def y = (short)2; return x * y"));
+        assertEquals(4F, exec("float x = (float)2; def y = (short)2; return x * y"));
+        assertEquals(4D, exec("double x = (double)2; def y = (short)2; return x * y"));
+
+        assertEquals(4, exec("byte x = (byte)2; def y = (char)2; return x * y"));
+        assertEquals(4, exec("short x = (short)2; def y = (char)2; return x * y"));
+        assertEquals(4, exec("char x = (char)2; def y = (char)2; return x * y"));
+        assertEquals(4, exec("int x = (int)2; def y = (char)2; return x * y"));
+        assertEquals(4L, exec("long x = (long)2; def y = (char)2; return x * y"));
+        assertEquals(4F, exec("float x = (float)2; def y = (char)2; return x * y"));
+        assertEquals(4D, exec("double x = (double)2; def y = (char)2; return x * y"));
+
+        assertEquals(4, exec("byte x = (byte)2; def y = (int)2; return x * y"));
+        assertEquals(4, exec("short x = (short)2; def y = (int)2; return x * y"));
+        assertEquals(4, exec("char x = (char)2; def y = (int)2; return x * y"));
+        assertEquals(4, exec("int x = (int)2; def y = (int)2; return x * y"));
+        assertEquals(4L, exec("long x = (long)2; def y = (int)2; return x * y"));
+        assertEquals(4F, exec("float x = (float)2; def y = (int)2; return x * y"));
+        assertEquals(4D, exec("double x = (double)2; def y = (int)2; return x * y"));
+
+        assertEquals(4L, exec("byte x = (byte)2; def y = (long)2; return x * y"));
+        assertEquals(4L, exec("short x = (short)2; def y = (long)2; return x * y"));
+        assertEquals(4L, exec("char x = (char)2; def y = (long)2; return x * y"));
+        assertEquals(4L, exec("int x = (int)2; def y = (long)2; return x * y"));
+        assertEquals(4L, exec("long x = (long)2; def y = (long)2; return x * y"));
+        assertEquals(4F, exec("float x = (float)2; def y = (long)2; return x * y"));
+        assertEquals(4D, exec("double x = (double)2; def y = (long)2; return x * y"));
+
+        assertEquals(4F, exec("byte x = (byte)2; def y = (float)2; return x * y"));
+        assertEquals(4F, exec("short x = (short)2; def y = (float)2; return x * y"));
+        assertEquals(4F, exec("char x = (char)2; def y = (float)2; return x * y"));
+        assertEquals(4F, exec("int x = (int)2; def y = (float)2; return x * y"));
+        assertEquals(4F, exec("long x = (long)2; def y = (float)2; return x * y"));
+        assertEquals(4F, exec("float x = (float)2; def y = (float)2; return x * y"));
+        assertEquals(4D, exec("double x = (double)2; def y = (float)2; return x * y"));
+
+        assertEquals(4D, exec("byte x = (byte)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("short x = (short)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("char x = (char)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("int x = (int)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("long x = (long)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("float x = (float)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("double x = (double)2; def y = (double)2; return x * y"));
+
+        assertEquals(4, exec("byte x = (byte)2; def y = (byte)2; return x * y"));
+        assertEquals(4, exec("short x = (short)2; def y = (short)2; return x * y"));
+        assertEquals(4, exec("char x = (char)2; def y = (char)2; return x * y"));
+        assertEquals(4, exec("int x = (int)2; def y = (int)2; return x * y"));
+        assertEquals(4L, exec("long x = (long)2; def y = (long)2; return x * y"));
+        assertEquals(4F, exec("float x = (float)2; def y = (float)2; return x * y"));
+        assertEquals(4D, exec("double x = (double)2; def y = (double)2; return x * y"));
+    }
+    
+    public void testMulTypedRHS() {
+        assertEquals(4, exec("def x = (byte)2; byte y = (byte)2; return x * y"));
+        assertEquals(4, exec("def x = (short)2; byte y = (byte)2; return x * y"));
+        assertEquals(4, exec("def x = (char)2; byte y = (byte)2; return x * y"));
+        assertEquals(4, exec("def x = (int)2; byte y = (byte)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; byte y = (byte)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; byte y = (byte)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; byte y = (byte)2; return x * y"));
+
+        assertEquals(4, exec("def x = (byte)2; short y = (short)2; return x * y"));
+        assertEquals(4, exec("def x = (short)2; short y = (short)2; return x * y"));
+        assertEquals(4, exec("def x = (char)2; short y = (short)2; return x * y"));
+        assertEquals(4, exec("def x = (int)2; short y = (short)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; short y = (short)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; short y = (short)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; short y = (short)2; return x * y"));
+
+        assertEquals(4, exec("def x = (byte)2; char y = (char)2; return x * y"));
+        assertEquals(4, exec("def x = (short)2; char y = (char)2; return x * y"));
+        assertEquals(4, exec("def x = (char)2; char y = (char)2; return x * y"));
+        assertEquals(4, exec("def x = (int)2; char y = (char)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; char y = (char)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; char y = (char)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; char y = (char)2; return x * y"));
+
+        assertEquals(4, exec("def x = (byte)2; int y = (int)2; return x * y"));
+        assertEquals(4, exec("def x = (short)2; int y = (int)2; return x * y"));
+        assertEquals(4, exec("def x = (char)2; int y = (int)2; return x * y"));
+        assertEquals(4, exec("def x = (int)2; int y = (int)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; int y = (int)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; int y = (int)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; int y = (int)2; return x * y"));
+
+        assertEquals(4L, exec("def x = (byte)2; long y = (long)2; return x * y"));
+        assertEquals(4L, exec("def x = (short)2; long y = (long)2; return x * y"));
+        assertEquals(4L, exec("def x = (char)2; long y = (long)2; return x * y"));
+        assertEquals(4L, exec("def x = (int)2; long y = (long)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; long y = (long)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; long y = (long)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; long y = (long)2; return x * y"));
+
+        assertEquals(4F, exec("def x = (byte)2; float y = (float)2; return x * y"));
+        assertEquals(4F, exec("def x = (short)2; float y = (float)2; return x * y"));
+        assertEquals(4F, exec("def x = (char)2; float y = (float)2; return x * y"));
+        assertEquals(4F, exec("def x = (int)2; float y = (float)2; return x * y"));
+        assertEquals(4F, exec("def x = (long)2; float y = (float)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; float y = (float)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; float y = (float)2; return x * y"));
+
+        assertEquals(4D, exec("def x = (byte)2; double y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (short)2; double y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (char)2; double y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (int)2; double y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (long)2; double y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (float)2; double y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; double y = (double)2; return x * y"));
+
+        assertEquals(4, exec("def x = (byte)2; byte y = (byte)2; return x * y"));
+        assertEquals(4, exec("def x = (short)2; short y = (short)2; return x * y"));
+        assertEquals(4, exec("def x = (char)2; char y = (char)2; return x * y"));
+        assertEquals(4, exec("def x = (int)2; int y = (int)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; long y = (long)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; float y = (float)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; double y = (double)2; return x * y"));
+    }
 
     public void testDiv() {
         assertEquals(1, exec("def x = (byte)2; def y = (byte)2; return x / y"));
@@ -191,6 +323,138 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(1F, exec("def x = (float)2; def y = (float)2; return x / y"));
         assertEquals(1D, exec("def x = (double)2; def y = (double)2; return x / y"));
     }
+    
+    public void testDivTypedLHS() {
+        assertEquals(1, exec("byte x = (byte)2; def y = (byte)2; return x / y"));
+        assertEquals(1, exec("short x = (short)2; def y = (byte)2; return x / y"));
+        assertEquals(1, exec("char x = (char)2; def y = (byte)2; return x / y"));
+        assertEquals(1, exec("int x = (int)2; def y = (byte)2; return x / y"));
+        assertEquals(1L, exec("long x = (long)2; def y = (byte)2; return x / y"));
+        assertEquals(1F, exec("float x = (float)2; def y = (byte)2; return x / y"));
+        assertEquals(1D, exec("double x = (double)2; def y = (byte)2; return x / y"));
+
+        assertEquals(1, exec("byte x = (byte)2; def y = (short)2; return x / y"));
+        assertEquals(1, exec("short x = (short)2; def y = (short)2; return x / y"));
+        assertEquals(1, exec("char x = (char)2; def y = (short)2; return x / y"));
+        assertEquals(1, exec("int x = (int)2; def y = (short)2; return x / y"));
+        assertEquals(1L, exec("long x = (long)2; def y = (short)2; return x / y"));
+        assertEquals(1F, exec("float x = (float)2; def y = (short)2; return x / y"));
+        assertEquals(1D, exec("double x = (double)2; def y = (short)2; return x / y"));
+
+        assertEquals(1, exec("byte x = (byte)2; def y = (char)2; return x / y"));
+        assertEquals(1, exec("short x = (short)2; def y = (char)2; return x / y"));
+        assertEquals(1, exec("char x = (char)2; def y = (char)2; return x / y"));
+        assertEquals(1, exec("int x = (int)2; def y = (char)2; return x / y"));
+        assertEquals(1L, exec("long x = (long)2; def y = (char)2; return x / y"));
+        assertEquals(1F, exec("float x = (float)2; def y = (char)2; return x / y"));
+        assertEquals(1D, exec("double x = (double)2; def y = (char)2; return x / y"));
+
+        assertEquals(1, exec("byte x = (byte)2; def y = (int)2; return x / y"));
+        assertEquals(1, exec("short x = (short)2; def y = (int)2; return x / y"));
+        assertEquals(1, exec("char x = (char)2; def y = (int)2; return x / y"));
+        assertEquals(1, exec("int x = (int)2; def y = (int)2; return x / y"));
+        assertEquals(1L, exec("long x = (long)2; def y = (int)2; return x / y"));
+        assertEquals(1F, exec("float x = (float)2; def y = (int)2; return x / y"));
+        assertEquals(1D, exec("double x = (double)2; def y = (int)2; return x / y"));
+
+        assertEquals(1L, exec("byte x = (byte)2; def y = (long)2; return x / y"));
+        assertEquals(1L, exec("short x = (short)2; def y = (long)2; return x / y"));
+        assertEquals(1L, exec("char x = (char)2; def y = (long)2; return x / y"));
+        assertEquals(1L, exec("int x = (int)2; def y = (long)2; return x / y"));
+        assertEquals(1L, exec("long x = (long)2; def y = (long)2; return x / y"));
+        assertEquals(1F, exec("float x = (float)2; def y = (long)2; return x / y"));
+        assertEquals(1D, exec("double x = (double)2; def y = (long)2; return x / y"));
+
+        assertEquals(1F, exec("byte x = (byte)2; def y = (float)2; return x / y"));
+        assertEquals(1F, exec("short x = (short)2; def y = (float)2; return x / y"));
+        assertEquals(1F, exec("char x = (char)2; def y = (float)2; return x / y"));
+        assertEquals(1F, exec("int x = (int)2; def y = (float)2; return x / y"));
+        assertEquals(1F, exec("long x = (long)2; def y = (float)2; return x / y"));
+        assertEquals(1F, exec("float x = (float)2; def y = (float)2; return x / y"));
+        assertEquals(1D, exec("double x = (double)2; def y = (float)2; return x / y"));
+
+        assertEquals(1D, exec("byte x = (byte)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("short x = (short)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("char x = (char)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("int x = (int)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("long x = (long)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("float x = (float)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("double x = (double)2; def y = (double)2; return x / y"));
+
+        assertEquals(1, exec("byte x = (byte)2; def y = (byte)2; return x / y"));
+        assertEquals(1, exec("short x = (short)2; def y = (short)2; return x / y"));
+        assertEquals(1, exec("char x = (char)2; def y = (char)2; return x / y"));
+        assertEquals(1, exec("int x = (int)2; def y = (int)2; return x / y"));
+        assertEquals(1L, exec("long x = (long)2; def y = (long)2; return x / y"));
+        assertEquals(1F, exec("float x = (float)2; def y = (float)2; return x / y"));
+        assertEquals(1D, exec("double x = (double)2; def y = (double)2; return x / y"));
+    }
+    
+    public void testDivTypedRHS() {
+        assertEquals(1, exec("def x = (byte)2; byte y = (byte)2; return x / y"));
+        assertEquals(1, exec("def x = (short)2; byte y = (byte)2; return x / y"));
+        assertEquals(1, exec("def x = (char)2; byte y = (byte)2; return x / y"));
+        assertEquals(1, exec("def x = (int)2; byte y = (byte)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; byte y = (byte)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; byte y = (byte)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; byte y = (byte)2; return x / y"));
+
+        assertEquals(1, exec("def x = (byte)2; short y = (short)2; return x / y"));
+        assertEquals(1, exec("def x = (short)2; short y = (short)2; return x / y"));
+        assertEquals(1, exec("def x = (char)2; short y = (short)2; return x / y"));
+        assertEquals(1, exec("def x = (int)2; short y = (short)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; short y = (short)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; short y = (short)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; short y = (short)2; return x / y"));
+
+        assertEquals(1, exec("def x = (byte)2; char y = (char)2; return x / y"));
+        assertEquals(1, exec("def x = (short)2; char y = (char)2; return x / y"));
+        assertEquals(1, exec("def x = (char)2; char y = (char)2; return x / y"));
+        assertEquals(1, exec("def x = (int)2; char y = (char)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; char y = (char)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; char y = (char)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; char y = (char)2; return x / y"));
+
+        assertEquals(1, exec("def x = (byte)2; int y = (int)2; return x / y"));
+        assertEquals(1, exec("def x = (short)2; int y = (int)2; return x / y"));
+        assertEquals(1, exec("def x = (char)2; int y = (int)2; return x / y"));
+        assertEquals(1, exec("def x = (int)2; int y = (int)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; int y = (int)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; int y = (int)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; int y = (int)2; return x / y"));
+
+        assertEquals(1L, exec("def x = (byte)2; long y = (long)2; return x / y"));
+        assertEquals(1L, exec("def x = (short)2; long y = (long)2; return x / y"));
+        assertEquals(1L, exec("def x = (char)2; long y = (long)2; return x / y"));
+        assertEquals(1L, exec("def x = (int)2; long y = (long)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; long y = (long)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; long y = (long)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; long y = (long)2; return x / y"));
+
+        assertEquals(1F, exec("def x = (byte)2; float y = (float)2; return x / y"));
+        assertEquals(1F, exec("def x = (short)2; float y = (float)2; return x / y"));
+        assertEquals(1F, exec("def x = (char)2; float y = (float)2; return x / y"));
+        assertEquals(1F, exec("def x = (int)2; float y = (float)2; return x / y"));
+        assertEquals(1F, exec("def x = (long)2; float y = (float)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; float y = (float)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; float y = (float)2; return x / y"));
+
+        assertEquals(1D, exec("def x = (byte)2; double y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (short)2; double y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (char)2; double y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (int)2; double y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (long)2; double y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (float)2; double y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; double y = (double)2; return x / y"));
+
+        assertEquals(1, exec("def x = (byte)2; byte y = (byte)2; return x / y"));
+        assertEquals(1, exec("def x = (short)2; short y = (short)2; return x / y"));
+        assertEquals(1, exec("def x = (char)2; char y = (char)2; return x / y"));
+        assertEquals(1, exec("def x = (int)2; int y = (int)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; long y = (long)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; float y = (float)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; double y = (double)2; return x / y"));
+    }
 
     public void testRem() {
         assertEquals(0, exec("def x = (byte)2; def y = (byte)2; return x % y"));
@@ -257,6 +521,138 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(0F, exec("def x = (float)2; def y = (float)2; return x % y"));
         assertEquals(0D, exec("def x = (double)2; def y = (double)2; return x % y"));
     }
+    
+    public void testRemTypedLHS() {
+        assertEquals(0, exec("byte x = (byte)2; def y = (byte)2; return x % y"));
+        assertEquals(0, exec("short x = (short)2; def y = (byte)2; return x % y"));
+        assertEquals(0, exec("char x = (char)2; def y = (byte)2; return x % y"));
+        assertEquals(0, exec("int x = (int)2; def y = (byte)2; return x % y"));
+        assertEquals(0L, exec("long x = (long)2; def y = (byte)2; return x % y"));
+        assertEquals(0F, exec("float x = (float)2; def y = (byte)2; return x % y"));
+        assertEquals(0D, exec("double x = (double)2; def y = (byte)2; return x % y"));
+
+        assertEquals(0, exec("byte x = (byte)2; def y = (short)2; return x % y"));
+        assertEquals(0, exec("short x = (short)2; def y = (short)2; return x % y"));
+        assertEquals(0, exec("char x = (char)2; def y = (short)2; return x % y"));
+        assertEquals(0, exec("int x = (int)2; def y = (short)2; return x % y"));
+        assertEquals(0L, exec("long x = (long)2; def y = (short)2; return x % y"));
+        assertEquals(0F, exec("float x = (float)2; def y = (short)2; return x % y"));
+        assertEquals(0D, exec("double x = (double)2; def y = (short)2; return x % y"));
+
+        assertEquals(0, exec("byte x = (byte)2; def y = (char)2; return x % y"));
+        assertEquals(0, exec("short x = (short)2; def y = (char)2; return x % y"));
+        assertEquals(0, exec("char x = (char)2; def y = (char)2; return x % y"));
+        assertEquals(0, exec("int x = (int)2; def y = (char)2; return x % y"));
+        assertEquals(0L, exec("long x = (long)2; def y = (char)2; return x % y"));
+        assertEquals(0F, exec("float x = (float)2; def y = (char)2; return x % y"));
+        assertEquals(0D, exec("double x = (double)2; def y = (char)2; return x % y"));
+
+        assertEquals(0, exec("byte x = (byte)2; def y = (int)2; return x % y"));
+        assertEquals(0, exec("short x = (short)2; def y = (int)2; return x % y"));
+        assertEquals(0, exec("char x = (char)2; def y = (int)2; return x % y"));
+        assertEquals(0, exec("int x = (int)2; def y = (int)2; return x % y"));
+        assertEquals(0L, exec("long x = (long)2; def y = (int)2; return x % y"));
+        assertEquals(0F, exec("float x = (float)2; def y = (int)2; return x % y"));
+        assertEquals(0D, exec("double x = (double)2; def y = (int)2; return x % y"));
+
+        assertEquals(0L, exec("byte x = (byte)2; def y = (long)2; return x % y"));
+        assertEquals(0L, exec("short x = (short)2; def y = (long)2; return x % y"));
+        assertEquals(0L, exec("char x = (char)2; def y = (long)2; return x % y"));
+        assertEquals(0L, exec("int x = (int)2; def y = (long)2; return x % y"));
+        assertEquals(0L, exec("long x = (long)2; def y = (long)2; return x % y"));
+        assertEquals(0F, exec("float x = (float)2; def y = (long)2; return x % y"));
+        assertEquals(0D, exec("double x = (double)2; def y = (long)2; return x % y"));
+
+        assertEquals(0F, exec("byte x = (byte)2; def y = (float)2; return x % y"));
+        assertEquals(0F, exec("short x = (short)2; def y = (float)2; return x % y"));
+        assertEquals(0F, exec("char x = (char)2; def y = (float)2; return x % y"));
+        assertEquals(0F, exec("int x = (int)2; def y = (float)2; return x % y"));
+        assertEquals(0F, exec("long x = (long)2; def y = (float)2; return x % y"));
+        assertEquals(0F, exec("float x = (float)2; def y = (float)2; return x % y"));
+        assertEquals(0D, exec("double x = (double)2; def y = (float)2; return x % y"));
+
+        assertEquals(0D, exec("byte x = (byte)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("short x = (short)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("char x = (char)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("int x = (int)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("long x = (long)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("float x = (float)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("double x = (double)2; def y = (double)2; return x % y"));
+
+        assertEquals(0, exec("byte x = (byte)2; def y = (byte)2; return x % y"));
+        assertEquals(0, exec("short x = (short)2; def y = (short)2; return x % y"));
+        assertEquals(0, exec("char x = (char)2; def y = (char)2; return x % y"));
+        assertEquals(0, exec("int x = (int)2; def y = (int)2; return x % y"));
+        assertEquals(0L, exec("long x = (long)2; def y = (long)2; return x % y"));
+        assertEquals(0F, exec("float x = (float)2; def y = (float)2; return x % y"));
+        assertEquals(0D, exec("double x = (double)2; def y = (double)2; return x % y"));
+    }
+
+    public void testRemTypedRHS() {
+        assertEquals(0, exec("def x = (byte)2; byte y = (byte)2; return x % y"));
+        assertEquals(0, exec("def x = (short)2; byte y = (byte)2; return x % y"));
+        assertEquals(0, exec("def x = (char)2; byte y = (byte)2; return x % y"));
+        assertEquals(0, exec("def x = (int)2; byte y = (byte)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; byte y = (byte)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; byte y = (byte)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; byte y = (byte)2; return x % y"));
+
+        assertEquals(0, exec("def x = (byte)2; short y = (short)2; return x % y"));
+        assertEquals(0, exec("def x = (short)2; short y = (short)2; return x % y"));
+        assertEquals(0, exec("def x = (char)2; short y = (short)2; return x % y"));
+        assertEquals(0, exec("def x = (int)2; short y = (short)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; short y = (short)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; short y = (short)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; short y = (short)2; return x % y"));
+
+        assertEquals(0, exec("def x = (byte)2; char y = (char)2; return x % y"));
+        assertEquals(0, exec("def x = (short)2; char y = (char)2; return x % y"));
+        assertEquals(0, exec("def x = (char)2; char y = (char)2; return x % y"));
+        assertEquals(0, exec("def x = (int)2; char y = (char)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; char y = (char)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; char y = (char)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; char y = (char)2; return x % y"));
+
+        assertEquals(0, exec("def x = (byte)2; int y = (int)2; return x % y"));
+        assertEquals(0, exec("def x = (short)2; int y = (int)2; return x % y"));
+        assertEquals(0, exec("def x = (char)2; int y = (int)2; return x % y"));
+        assertEquals(0, exec("def x = (int)2; int y = (int)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; int y = (int)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; int y = (int)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; int y = (int)2; return x % y"));
+
+        assertEquals(0L, exec("def x = (byte)2; long y = (long)2; return x % y"));
+        assertEquals(0L, exec("def x = (short)2; long y = (long)2; return x % y"));
+        assertEquals(0L, exec("def x = (char)2; long y = (long)2; return x % y"));
+        assertEquals(0L, exec("def x = (int)2; long y = (long)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; long y = (long)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; long y = (long)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; long y = (long)2; return x % y"));
+
+        assertEquals(0F, exec("def x = (byte)2; float y = (float)2; return x % y"));
+        assertEquals(0F, exec("def x = (short)2; float y = (float)2; return x % y"));
+        assertEquals(0F, exec("def x = (char)2; float y = (float)2; return x % y"));
+        assertEquals(0F, exec("def x = (int)2; float y = (float)2; return x % y"));
+        assertEquals(0F, exec("def x = (long)2; float y = (float)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; float y = (float)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; float y = (float)2; return x % y"));
+
+        assertEquals(0D, exec("def x = (byte)2; double y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (short)2; double y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (char)2; double y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (int)2; double y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (long)2; double y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (float)2; double y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; double y = (double)2; return x % y"));
+
+        assertEquals(0, exec("def x = (byte)2; byte y = (byte)2; return x % y"));
+        assertEquals(0, exec("def x = (short)2; short y = (short)2; return x % y"));
+        assertEquals(0, exec("def x = (char)2; char y = (char)2; return x % y"));
+        assertEquals(0, exec("def x = (int)2; int y = (int)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; long y = (long)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; float y = (float)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; double y = (double)2; return x % y"));
+    }
 
     public void testAdd() {
         assertEquals(2, exec("def x = (byte)1; def y = (byte)1; return x + y"));
@@ -314,14 +710,122 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(2D, exec("def x = (long)1; def y = (double)1; return x + y"));
         assertEquals(2D, exec("def x = (float)1; def y = (double)1; return x + y"));
         assertEquals(2D, exec("def x = (double)1; def y = (double)1; return x + y"));
+    }
+    
+    public void testAddTypedLHS() {
+        assertEquals(2, exec("byte x = (byte)1; def y = (byte)1; return x + y"));
+        assertEquals(2, exec("short x = (short)1; def y = (byte)1; return x + y"));
+        assertEquals(2, exec("char x = (char)1; def y = (byte)1; return x + y"));
+        assertEquals(2, exec("int x = (int)1; def y = (byte)1; return x + y"));
+        assertEquals(2L, exec("long x = (long)1; def y = (byte)1; return x + y"));
+        assertEquals(2F, exec("float x = (float)1; def y = (byte)1; return x + y"));
+        assertEquals(2D, exec("double x = (double)1; def y = (byte)1; return x + y"));
 
-        assertEquals(2, exec("def x = (byte)1; def y = (byte)1; return x + y"));
-        assertEquals(2, exec("def x = (short)1; def y = (short)1; return x + y"));
-        assertEquals(2, exec("def x = (char)1; def y = (char)1; return x + y"));
-        assertEquals(2, exec("def x = (int)1; def y = (int)1; return x + y"));
-        assertEquals(2L, exec("def x = (long)1; def y = (long)1; return x + y"));
-        assertEquals(2F, exec("def x = (float)1; def y = (float)1; return x + y"));
-        assertEquals(2D, exec("def x = (double)1; def y = (double)1; return x + y"));
+        assertEquals(2, exec("byte x = (byte)1; def y = (short)1; return x + y"));
+        assertEquals(2, exec("short x = (short)1; def y = (short)1; return x + y"));
+        assertEquals(2, exec("char x = (char)1; def y = (short)1; return x + y"));
+        assertEquals(2, exec("int x = (int)1; def y = (short)1; return x + y"));
+        assertEquals(2L, exec("long x = (long)1; def y = (short)1; return x + y"));
+        assertEquals(2F, exec("float x = (float)1; def y = (short)1; return x + y"));
+        assertEquals(2D, exec("double x = (double)1; def y = (short)1; return x + y"));
+
+        assertEquals(2, exec("byte x = (byte)1; def y = (char)1; return x + y"));
+        assertEquals(2, exec("short x = (short)1; def y = (char)1; return x + y"));
+        assertEquals(2, exec("char x = (char)1; def y = (char)1; return x + y"));
+        assertEquals(2, exec("int x = (int)1; def y = (char)1; return x + y"));
+        assertEquals(2L, exec("long x = (long)1; def y = (char)1; return x + y"));
+        assertEquals(2F, exec("float x = (float)1; def y = (char)1; return x + y"));
+        assertEquals(2D, exec("double x = (double)1; def y = (char)1; return x + y"));
+
+        assertEquals(2, exec("byte x = (byte)1; def y = (int)1; return x + y"));
+        assertEquals(2, exec("short x = (short)1; def y = (int)1; return x + y"));
+        assertEquals(2, exec("char x = (char)1; def y = (int)1; return x + y"));
+        assertEquals(2, exec("int x = (int)1; def y = (int)1; return x + y"));
+        assertEquals(2L, exec("long x = (long)1; def y = (int)1; return x + y"));
+        assertEquals(2F, exec("float x = (float)1; def y = (int)1; return x + y"));
+        assertEquals(2D, exec("double x = (double)1; def y = (int)1; return x + y"));
+
+        assertEquals(2L, exec("byte x = (byte)1; def y = (long)1; return x + y"));
+        assertEquals(2L, exec("short x = (short)1; def y = (long)1; return x + y"));
+        assertEquals(2L, exec("char x = (char)1; def y = (long)1; return x + y"));
+        assertEquals(2L, exec("int x = (int)1; def y = (long)1; return x + y"));
+        assertEquals(2L, exec("long x = (long)1; def y = (long)1; return x + y"));
+        assertEquals(2F, exec("float x = (float)1; def y = (long)1; return x + y"));
+        assertEquals(2D, exec("double x = (double)1; def y = (long)1; return x + y"));
+
+        assertEquals(2F, exec("byte x = (byte)1; def y = (float)1; return x + y"));
+        assertEquals(2F, exec("short x = (short)1; def y = (float)1; return x + y"));
+        assertEquals(2F, exec("char x = (char)1; def y = (float)1; return x + y"));
+        assertEquals(2F, exec("int x = (int)1; def y = (float)1; return x + y"));
+        assertEquals(2F, exec("long x = (long)1; def y = (float)1; return x + y"));
+        assertEquals(2F, exec("float x = (float)1; def y = (float)1; return x + y"));
+        assertEquals(2D, exec("double x = (double)1; def y = (float)1; return x + y"));
+
+        assertEquals(2D, exec("byte x = (byte)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("short x = (short)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("char x = (char)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("int x = (int)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("long x = (long)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("float x = (float)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("double x = (double)1; def y = (double)1; return x + y"));
+    }
+    
+    public void testAddTypedRHS() {
+        assertEquals(2, exec("def x = (byte)1; byte y = (byte)1; return x + y"));
+        assertEquals(2, exec("def x = (short)1; byte y = (byte)1; return x + y"));
+        assertEquals(2, exec("def x = (char)1; byte y = (byte)1; return x + y"));
+        assertEquals(2, exec("def x = (int)1; byte y = (byte)1; return x + y"));
+        assertEquals(2L, exec("def x = (long)1; byte y = (byte)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; byte y = (byte)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; byte y = (byte)1; return x + y"));
+
+        assertEquals(2, exec("def x = (byte)1; short y = (short)1; return x + y"));
+        assertEquals(2, exec("def x = (short)1; short y = (short)1; return x + y"));
+        assertEquals(2, exec("def x = (char)1; short y = (short)1; return x + y"));
+        assertEquals(2, exec("def x = (int)1; short y = (short)1; return x + y"));
+        assertEquals(2L, exec("def x = (long)1; short y = (short)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; short y = (short)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; short y = (short)1; return x + y"));
+
+        assertEquals(2, exec("def x = (byte)1; char y = (char)1; return x + y"));
+        assertEquals(2, exec("def x = (short)1; char y = (char)1; return x + y"));
+        assertEquals(2, exec("def x = (char)1; char y = (char)1; return x + y"));
+        assertEquals(2, exec("def x = (int)1; char y = (char)1; return x + y"));
+        assertEquals(2L, exec("def x = (long)1; char y = (char)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; char y = (char)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; char y = (char)1; return x + y"));
+
+        assertEquals(2, exec("def x = (byte)1; int y = (int)1; return x + y"));
+        assertEquals(2, exec("def x = (short)1; int y = (int)1; return x + y"));
+        assertEquals(2, exec("def x = (char)1; int y = (int)1; return x + y"));
+        assertEquals(2, exec("def x = (int)1; int y = (int)1; return x + y"));
+        assertEquals(2L, exec("def x = (long)1; int y = (int)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; int y = (int)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; int y = (int)1; return x + y"));
+
+        assertEquals(2L, exec("def x = (byte)1; long y = (long)1; return x + y"));
+        assertEquals(2L, exec("def x = (short)1; long y = (long)1; return x + y"));
+        assertEquals(2L, exec("def x = (char)1; long y = (long)1; return x + y"));
+        assertEquals(2L, exec("def x = (int)1; long y = (long)1; return x + y"));
+        assertEquals(2L, exec("def x = (long)1; long y = (long)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; long y = (long)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; long y = (long)1; return x + y"));
+
+        assertEquals(2F, exec("def x = (byte)1; float y = (float)1; return x + y"));
+        assertEquals(2F, exec("def x = (short)1; float y = (float)1; return x + y"));
+        assertEquals(2F, exec("def x = (char)1; float y = (float)1; return x + y"));
+        assertEquals(2F, exec("def x = (int)1; float y = (float)1; return x + y"));
+        assertEquals(2F, exec("def x = (long)1; float y = (float)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; float y = (float)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; float y = (float)1; return x + y"));
+
+        assertEquals(2D, exec("def x = (byte)1; double y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (short)1; double y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (char)1; double y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (int)1; double y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (long)1; double y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (float)1; double y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; double y = (double)1; return x + y"));
     }
     
     public void testAddConcat() {
@@ -403,14 +907,122 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(0D, exec("def x = (long)1; def y = (double)1; return x - y"));
         assertEquals(0D, exec("def x = (float)1; def y = (double)1; return x - y"));
         assertEquals(0D, exec("def x = (double)1; def y = (double)1; return x - y"));
+    }
+    
+    public void testSubTypedLHS() {
+        assertEquals(0, exec("byte x = (byte)1; def y = (byte)1; return x - y"));
+        assertEquals(0, exec("short x = (short)1; def y = (byte)1; return x - y"));
+        assertEquals(0, exec("char x = (char)1; def y = (byte)1; return x - y"));
+        assertEquals(0, exec("int x = (int)1; def y = (byte)1; return x - y"));
+        assertEquals(0L, exec("long x = (long)1; def y = (byte)1; return x - y"));
+        assertEquals(0F, exec("float x = (float)1; def y = (byte)1; return x - y"));
+        assertEquals(0D, exec("double x = (double)1; def y = (byte)1; return x - y"));
 
-        assertEquals(0, exec("def x = (byte)1; def y = (byte)1; return x - y"));
-        assertEquals(0, exec("def x = (short)1; def y = (short)1; return x - y"));
-        assertEquals(0, exec("def x = (char)1; def y = (char)1; return x - y"));
-        assertEquals(0, exec("def x = (int)1; def y = (int)1; return x - y"));
-        assertEquals(0L, exec("def x = (long)1; def y = (long)1; return x - y"));
-        assertEquals(0F, exec("def x = (float)1; def y = (float)1; return x - y"));
-        assertEquals(0D, exec("def x = (double)1; def y = (double)1; return x - y"));
+        assertEquals(0, exec("byte x = (byte)1; def y = (short)1; return x - y"));
+        assertEquals(0, exec("short x = (short)1; def y = (short)1; return x - y"));
+        assertEquals(0, exec("char x = (char)1; def y = (short)1; return x - y"));
+        assertEquals(0, exec("int x = (int)1; def y = (short)1; return x - y"));
+        assertEquals(0L, exec("long x = (long)1; def y = (short)1; return x - y"));
+        assertEquals(0F, exec("float x = (float)1; def y = (short)1; return x - y"));
+        assertEquals(0D, exec("double x = (double)1; def y = (short)1; return x - y"));
+
+        assertEquals(0, exec("byte x = (byte)1; def y = (char)1; return x - y"));
+        assertEquals(0, exec("short x = (short)1; def y = (char)1; return x - y"));
+        assertEquals(0, exec("char x = (char)1; def y = (char)1; return x - y"));
+        assertEquals(0, exec("int x = (int)1; def y = (char)1; return x - y"));
+        assertEquals(0L, exec("long x = (long)1; def y = (char)1; return x - y"));
+        assertEquals(0F, exec("float x = (float)1; def y = (char)1; return x - y"));
+        assertEquals(0D, exec("double x = (double)1; def y = (char)1; return x - y"));
+
+        assertEquals(0, exec("byte x = (byte)1; def y = (int)1; return x - y"));
+        assertEquals(0, exec("short x = (short)1; def y = (int)1; return x - y"));
+        assertEquals(0, exec("char x = (char)1; def y = (int)1; return x - y"));
+        assertEquals(0, exec("int x = (int)1; def y = (int)1; return x - y"));
+        assertEquals(0L, exec("long x = (long)1; def y = (int)1; return x - y"));
+        assertEquals(0F, exec("float x = (float)1; def y = (int)1; return x - y"));
+        assertEquals(0D, exec("double x = (double)1; def y = (int)1; return x - y"));
+
+        assertEquals(0L, exec("byte x = (byte)1; def y = (long)1; return x - y"));
+        assertEquals(0L, exec("short x = (short)1; def y = (long)1; return x - y"));
+        assertEquals(0L, exec("char x = (char)1; def y = (long)1; return x - y"));
+        assertEquals(0L, exec("int x = (int)1; def y = (long)1; return x - y"));
+        assertEquals(0L, exec("long x = (long)1; def y = (long)1; return x - y"));
+        assertEquals(0F, exec("float x = (float)1; def y = (long)1; return x - y"));
+        assertEquals(0D, exec("double x = (double)1; def y = (long)1; return x - y"));
+
+        assertEquals(0F, exec("byte x = (byte)1; def y = (float)1; return x - y"));
+        assertEquals(0F, exec("short x = (short)1; def y = (float)1; return x - y"));
+        assertEquals(0F, exec("char x = (char)1; def y = (float)1; return x - y"));
+        assertEquals(0F, exec("int x = (int)1; def y = (float)1; return x - y"));
+        assertEquals(0F, exec("long x = (long)1; def y = (float)1; return x - y"));
+        assertEquals(0F, exec("float x = (float)1; def y = (float)1; return x - y"));
+        assertEquals(0D, exec("double x = (double)1; def y = (float)1; return x - y"));
+
+        assertEquals(0D, exec("byte x = (byte)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("short x = (short)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("char x = (char)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("int x = (int)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("long x = (long)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("float x = (float)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("double x = (double)1; def y = (double)1; return x - y"));
+    }
+    
+    public void testSubTypedRHS() {
+        assertEquals(0, exec("def x = (byte)1; byte y = (byte)1; return x - y"));
+        assertEquals(0, exec("def x = (short)1; byte y = (byte)1; return x - y"));
+        assertEquals(0, exec("def x = (char)1; byte y = (byte)1; return x - y"));
+        assertEquals(0, exec("def x = (int)1; byte y = (byte)1; return x - y"));
+        assertEquals(0L, exec("def x = (long)1; byte y = (byte)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; byte y = (byte)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; byte y = (byte)1; return x - y"));
+
+        assertEquals(0, exec("def x = (byte)1; short y = (short)1; return x - y"));
+        assertEquals(0, exec("def x = (short)1; short y = (short)1; return x - y"));
+        assertEquals(0, exec("def x = (char)1; short y = (short)1; return x - y"));
+        assertEquals(0, exec("def x = (int)1; short y = (short)1; return x - y"));
+        assertEquals(0L, exec("def x = (long)1; short y = (short)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; short y = (short)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; short y = (short)1; return x - y"));
+
+        assertEquals(0, exec("def x = (byte)1; char y = (char)1; return x - y"));
+        assertEquals(0, exec("def x = (short)1; char y = (char)1; return x - y"));
+        assertEquals(0, exec("def x = (char)1; char y = (char)1; return x - y"));
+        assertEquals(0, exec("def x = (int)1; char y = (char)1; return x - y"));
+        assertEquals(0L, exec("def x = (long)1; char y = (char)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; char y = (char)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; char y = (char)1; return x - y"));
+
+        assertEquals(0, exec("def x = (byte)1; int y = (int)1; return x - y"));
+        assertEquals(0, exec("def x = (short)1; int y = (int)1; return x - y"));
+        assertEquals(0, exec("def x = (char)1; int y = (int)1; return x - y"));
+        assertEquals(0, exec("def x = (int)1; int y = (int)1; return x - y"));
+        assertEquals(0L, exec("def x = (long)1; int y = (int)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; int y = (int)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; int y = (int)1; return x - y"));
+
+        assertEquals(0L, exec("def x = (byte)1; long y = (long)1; return x - y"));
+        assertEquals(0L, exec("def x = (short)1; long y = (long)1; return x - y"));
+        assertEquals(0L, exec("def x = (char)1; long y = (long)1; return x - y"));
+        assertEquals(0L, exec("def x = (int)1; long y = (long)1; return x - y"));
+        assertEquals(0L, exec("def x = (long)1; long y = (long)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; long y = (long)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; long y = (long)1; return x - y"));
+
+        assertEquals(0F, exec("def x = (byte)1; float y = (float)1; return x - y"));
+        assertEquals(0F, exec("def x = (short)1; float y = (float)1; return x - y"));
+        assertEquals(0F, exec("def x = (char)1; float y = (float)1; return x - y"));
+        assertEquals(0F, exec("def x = (int)1; float y = (float)1; return x - y"));
+        assertEquals(0F, exec("def x = (long)1; float y = (float)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; float y = (float)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; float y = (float)1; return x - y"));
+
+        assertEquals(0D, exec("def x = (byte)1; double y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (short)1; double y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (char)1; double y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (int)1; double y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (long)1; double y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (float)1; double y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; double y = (double)1; return x - y"));
     }
 
     public void testLsh() {
@@ -450,6 +1062,82 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(2, exec("def x = (int)1; def y = (int)1; return x << y"));
         assertEquals(2L, exec("def x = (long)1; def y = (long)1; return x << y"));
     }
+    
+    public void testLshTypedLHS() {
+        assertEquals(2, exec("byte x = (byte)1; def y = (byte)1; return x << y"));
+        assertEquals(2, exec("short x = (short)1; def y = (byte)1; return x << y"));
+        assertEquals(2, exec("char x = (char)1; def y = (byte)1; return x << y"));
+        assertEquals(2, exec("int x = (int)1; def y = (byte)1; return x << y"));
+        assertEquals(2L, exec("long x = (long)1; def y = (byte)1; return x << y"));
+
+        assertEquals(2, exec("byte x = (byte)1; def y = (short)1; return x << y"));
+        assertEquals(2, exec("short x = (short)1; def y = (short)1; return x << y"));
+        assertEquals(2, exec("char x = (char)1; def y = (short)1; return x << y"));
+        assertEquals(2, exec("int x = (int)1; def y = (short)1; return x << y"));
+        assertEquals(2L, exec("long x = (long)1; def y = (short)1; return x << y"));
+
+        assertEquals(2, exec("byte x = (byte)1; def y = (char)1; return x << y"));
+        assertEquals(2, exec("short x = (short)1; def y = (char)1; return x << y"));
+        assertEquals(2, exec("char x = (char)1; def y = (char)1; return x << y"));
+        assertEquals(2, exec("int x = (int)1; def y = (char)1; return x << y"));
+        assertEquals(2L, exec("long x = (long)1; def y = (char)1; return x << y"));
+
+        assertEquals(2, exec("byte x = (byte)1; def y = (int)1; return x << y"));
+        assertEquals(2, exec("short x = (short)1; def y = (int)1; return x << y"));
+        assertEquals(2, exec("char x = (char)1; def y = (int)1; return x << y"));
+        assertEquals(2, exec("int x = (int)1; def y = (int)1; return x << y"));
+        assertEquals(2L, exec("long x = (long)1; def y = (int)1; return x << y"));
+
+        assertEquals(2, exec("byte x = (byte)1; def y = (long)1; return x << y"));
+        assertEquals(2, exec("short x = (short)1; def y = (long)1; return x << y"));
+        assertEquals(2, exec("char x = (char)1; def y = (long)1; return x << y"));
+        assertEquals(2, exec("int x = (int)1; def y = (long)1; return x << y"));
+        assertEquals(2L, exec("long x = (long)1; def y = (long)1; return x << y"));
+
+        assertEquals(2, exec("byte x = (byte)1; def y = (byte)1; return x << y"));
+        assertEquals(2, exec("short x = (short)1; def y = (short)1; return x << y"));
+        assertEquals(2, exec("char x = (char)1; def y = (char)1; return x << y"));
+        assertEquals(2, exec("int x = (int)1; def y = (int)1; return x << y"));
+        assertEquals(2L, exec("long x = (long)1; def y = (long)1; return x << y"));
+    }
+    
+    public void testLshTypedRHS() {
+        assertEquals(2, exec("def x = (byte)1; byte y = (byte)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; byte y = (byte)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; byte y = (byte)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; byte y = (byte)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; byte y = (byte)1; return x << y"));
+
+        assertEquals(2, exec("def x = (byte)1; short y = (short)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; short y = (short)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; short y = (short)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; short y = (short)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; short y = (short)1; return x << y"));
+
+        assertEquals(2, exec("def x = (byte)1; char y = (char)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; char y = (char)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; char y = (char)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; char y = (char)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; char y = (char)1; return x << y"));
+
+        assertEquals(2, exec("def x = (byte)1; int y = (int)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; int y = (int)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; int y = (int)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; int y = (int)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; int y = (int)1; return x << y"));
+
+        assertEquals(2, exec("def x = (byte)1; long y = (long)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; long y = (long)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; long y = (long)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; long y = (long)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; long y = (long)1; return x << y"));
+
+        assertEquals(2, exec("def x = (byte)1; byte y = (byte)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; short y = (short)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; char y = (char)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; int y = (int)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; long y = (long)1; return x << y"));
+    }
 
     public void testRsh() {
         assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >> y"));
@@ -488,6 +1176,82 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >> y"));
         assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >> y"));
     }
+    
+    public void testRshTypeLHS() {
+        assertEquals(2, exec("byte x = (byte)4; def y = (byte)1; return x >> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (byte)1; return x >> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (byte)1; return x >> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (byte)1; return x >> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (byte)1; return x >> y"));
+
+        assertEquals(2, exec("byte x = (byte)4; def y = (short)1; return x >> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (short)1; return x >> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (short)1; return x >> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (short)1; return x >> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (short)1; return x >> y"));
+
+        assertEquals(2, exec("byte x = (byte)4; def y = (char)1; return x >> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (char)1; return x >> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (char)1; return x >> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (char)1; return x >> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (char)1; return x >> y"));
+
+        assertEquals(2, exec("byte x = (byte)4; def y = (int)1; return x >> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (int)1; return x >> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (int)1; return x >> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (int)1; return x >> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (int)1; return x >> y"));
+
+        assertEquals(2, exec("byte x = (byte)4; def y = (long)1; return x >> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (long)1; return x >> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (long)1; return x >> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (long)1; return x >> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (long)1; return x >> y"));
+
+        assertEquals(2, exec("byte x = (byte)4; def y = (byte)1; return x >> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (short)1; return x >> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (char)1; return x >> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (int)1; return x >> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (long)1; return x >> y"));
+    }
+    
+    public void testRshTypedLHS() {
+        assertEquals(2, exec("def x = (byte)4; byte y = (byte)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; byte y = (byte)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; byte y = (byte)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; byte y = (byte)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; byte y = (byte)1; return x >> y"));
+
+        assertEquals(2, exec("def x = (byte)4; short y = (short)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; short y = (short)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; short y = (short)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; short y = (short)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; short y = (short)1; return x >> y"));
+
+        assertEquals(2, exec("def x = (byte)4; char y = (char)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; char y = (char)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; char y = (char)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; char y = (char)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; char y = (char)1; return x >> y"));
+
+        assertEquals(2, exec("def x = (byte)4; int y = (int)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; int y = (int)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; int y = (int)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; int y = (int)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; int y = (int)1; return x >> y"));
+
+        assertEquals(2, exec("def x = (byte)4; long y = (long)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; long y = (long)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; long y = (long)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; long y = (long)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; long y = (long)1; return x >> y"));
+
+        assertEquals(2, exec("def x = (byte)4; byte y = (byte)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; short y = (short)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; char y = (char)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; int y = (int)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; long y = (long)1; return x >> y"));
+    }
 
     public void testUsh() {
         assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >>> y"));
@@ -527,6 +1291,82 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >>> y"));
     }
     
+    public void testUshTypedLHS() {
+        assertEquals(2, exec("byte x = (byte)4; def y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (byte)1; return x >>> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (byte)1; return x >>> y"));
+
+        assertEquals(2, exec("byte x = (byte)4; def y = (short)1; return x >>> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (short)1; return x >>> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (short)1; return x >>> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (short)1; return x >>> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (short)1; return x >>> y"));
+
+        assertEquals(2, exec("byte x = (byte)4; def y = (char)1; return x >>> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (char)1; return x >>> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (char)1; return x >>> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (char)1; return x >>> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (char)1; return x >>> y"));
+
+        assertEquals(2, exec("byte x = (byte)4; def y = (int)1; return x >>> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (int)1; return x >>> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (int)1; return x >>> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (int)1; return x >>> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (int)1; return x >>> y"));
+
+        assertEquals(2, exec("byte x = (byte)4; def y = (long)1; return x >>> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (long)1; return x >>> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (long)1; return x >>> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (long)1; return x >>> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (long)1; return x >>> y"));
+
+        assertEquals(2, exec("byte x = (byte)4; def y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (short)1; return x >>> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (char)1; return x >>> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (int)1; return x >>> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (long)1; return x >>> y"));
+    }
+    
+    public void testUshTypedRHS() {
+        assertEquals(2, exec("def x = (byte)4; byte y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; byte y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; byte y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; byte y = (byte)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; byte y = (byte)1; return x >>> y"));
+
+        assertEquals(2, exec("def x = (byte)4; short y = (short)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; short y = (short)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; short y = (short)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; short y = (short)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; short y = (short)1; return x >>> y"));
+
+        assertEquals(2, exec("def x = (byte)4; char y = (char)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; char y = (char)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; char y = (char)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; char y = (char)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; char y = (char)1; return x >>> y"));
+
+        assertEquals(2, exec("def x = (byte)4; int y = (int)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; int y = (int)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; int y = (int)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; int y = (int)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; int y = (int)1; return x >>> y"));
+
+        assertEquals(2, exec("def x = (byte)4; long y = (long)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; long y = (long)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; long y = (long)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; long y = (long)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; long y = (long)1; return x >>> y"));
+
+        assertEquals(2, exec("def x = (byte)4; byte y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; short y = (short)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; char y = (char)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; int y = (int)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; long y = (long)1; return x >>> y"));
+    }
+    
     public void testBogusShifts() {
         expectScriptThrows(ClassCastException.class, ()-> {
             exec("def x = 1L; def y = 2F; return x << y;");
@@ -567,6 +1407,88 @@ public class DefOperationTests extends ScriptTestCase {
             exec("def x = 1D; def y = 2L; return x >>> y;");
         });
     }
+    
+    public void testBogusShiftsTypedLHS() {
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("long x = 1L; def y = 2F; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("int x = 1; def y = 2D; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("float x = 1F; def y = 2; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("double x = 1D; def y = 2L; return x << y;");
+        });
+        
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("long x = 1L; def y = 2F; return x >> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("int x = 1; def y = 2D; return x >> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("float x = 1F; def y = 2; return x >> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("double x = 1D; def y = 2L; return x >> y;");
+        });
+        
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("long x = 1L; def y = 2F; return x >>> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("int x = 1; def y = 2D; return x >>> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("float x = 1F; def y = 2; return x >>> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("double x = 1D; def y = 2L; return x >>> y;");
+        });
+    }
+    
+    public void testBogusShiftsTypedRHS() {
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1L; float y = 2F; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1; double y = 2D; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1F; int y = 2; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1D; long y = 2L; return x << y;");
+        });
+        
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1L; float y = 2F; return x >> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1; double y = 2D; return x >> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1F; int y = 2; return x >> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1D; long y = 2L; return x >> y;");
+        });
+        
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1L; float y = 2F; return x >>> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1; double y = 2D; return x >>> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1F; int y = 2; return x >>> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1D; long y = 2L; return x >>> y;");
+        });
+    }
 
     public void testAnd() {
         expectScriptThrows(ClassCastException.class, () -> {
@@ -616,6 +1538,104 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(false, exec("def x = false; def y = true; return x & y"));
         assertEquals(false, exec("def x = false; def y = false; return x & y"));
     }
+    
+    public void testAndTypedLHS() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("float x = (float)4; def y = (byte)1; return x & y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("double x = (double)4; def y = (byte)1; return x & y");
+        });
+        assertEquals(0, exec("byte x = (byte)4; def y = (byte)1; return x & y"));
+        assertEquals(0, exec("short x = (short)4; def y = (byte)1; return x & y"));
+        assertEquals(0, exec("char x = (char)4; def y = (byte)1; return x & y"));
+        assertEquals(0, exec("int x = (int)4; def y = (byte)1; return x & y"));
+        assertEquals(0L, exec("long x = (long)4; def y = (byte)1; return x & y"));
+
+        assertEquals(0, exec("byte x = (byte)4; def y = (short)1; return x & y"));
+        assertEquals(0, exec("short x = (short)4; def y = (short)1; return x & y"));
+        assertEquals(0, exec("char x = (char)4; def y = (short)1; return x & y"));
+        assertEquals(0, exec("int x = (int)4; def y = (short)1; return x & y"));
+        assertEquals(0L, exec("long x = (long)4; def y = (short)1; return x & y"));
+
+        assertEquals(0, exec("byte x = (byte)4; def y = (char)1; return x & y"));
+        assertEquals(0, exec("short x = (short)4; def y = (char)1; return x & y"));
+        assertEquals(0, exec("char x = (char)4; def y = (char)1; return x & y"));
+        assertEquals(0, exec("int x = (int)4; def y = (char)1; return x & y"));
+        assertEquals(0L, exec("long x = (long)4; def y = (char)1; return x & y"));
+
+        assertEquals(0, exec("byte x = (byte)4; def y = (int)1; return x & y"));
+        assertEquals(0, exec("short x = (short)4; def y = (int)1; return x & y"));
+        assertEquals(0, exec("char x = (char)4; def y = (int)1; return x & y"));
+        assertEquals(0, exec("int x = (int)4; def y = (int)1; return x & y"));
+        assertEquals(0L, exec("long x = (long)4; def y = (int)1; return x & y"));
+
+        assertEquals(0L, exec("byte x = (byte)4; def y = (long)1; return x & y"));
+        assertEquals(0L, exec("short x = (short)4; def y = (long)1; return x & y"));
+        assertEquals(0L, exec("char x = (char)4; def y = (long)1; return x & y"));
+        assertEquals(0L, exec("int x = (int)4; def y = (long)1; return x & y"));
+        assertEquals(0L, exec("long x = (long)4; def y = (long)1; return x & y"));
+
+        assertEquals(0, exec("byte x = (byte)4; def y = (byte)1; return x & y"));
+        assertEquals(0, exec("short x = (short)4; def y = (short)1; return x & y"));
+        assertEquals(0, exec("char x = (char)4; def y = (char)1; return x & y"));
+        assertEquals(0, exec("int x = (int)4; def y = (int)1; return x & y"));
+        assertEquals(0L, exec("long x = (long)4; def y = (long)1; return x & y"));
+
+        assertEquals(true,  exec("boolean x = true;  def y = true; return x & y"));
+        assertEquals(false, exec("boolean x = true;  def y = false; return x & y"));
+        assertEquals(false, exec("boolean x = false; def y = true; return x & y"));
+        assertEquals(false, exec("boolean x = false; def y = false; return x & y"));
+    }
+    
+    public void testAndTypedRHS() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (float)4; byte y = (byte)1; return x & y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (double)4; byte y = (byte)1; return x & y");
+        });
+        assertEquals(0, exec("def x = (byte)4; byte y = (byte)1; return x & y"));
+        assertEquals(0, exec("def x = (short)4; byte y = (byte)1; return x & y"));
+        assertEquals(0, exec("def x = (char)4; byte y = (byte)1; return x & y"));
+        assertEquals(0, exec("def x = (int)4; byte y = (byte)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; byte y = (byte)1; return x & y"));
+
+        assertEquals(0, exec("def x = (byte)4; short y = (short)1; return x & y"));
+        assertEquals(0, exec("def x = (short)4; short y = (short)1; return x & y"));
+        assertEquals(0, exec("def x = (char)4; short y = (short)1; return x & y"));
+        assertEquals(0, exec("def x = (int)4; short y = (short)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; short y = (short)1; return x & y"));
+
+        assertEquals(0, exec("def x = (byte)4; char y = (char)1; return x & y"));
+        assertEquals(0, exec("def x = (short)4; char y = (char)1; return x & y"));
+        assertEquals(0, exec("def x = (char)4; char y = (char)1; return x & y"));
+        assertEquals(0, exec("def x = (int)4; char y = (char)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; char y = (char)1; return x & y"));
+
+        assertEquals(0, exec("def x = (byte)4; int y = (int)1; return x & y"));
+        assertEquals(0, exec("def x = (short)4; int y = (int)1; return x & y"));
+        assertEquals(0, exec("def x = (char)4; int y = (int)1; return x & y"));
+        assertEquals(0, exec("def x = (int)4; int y = (int)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; int y = (int)1; return x & y"));
+
+        assertEquals(0L, exec("def x = (byte)4; long y = (long)1; return x & y"));
+        assertEquals(0L, exec("def x = (short)4; long y = (long)1; return x & y"));
+        assertEquals(0L, exec("def x = (char)4; long y = (long)1; return x & y"));
+        assertEquals(0L, exec("def x = (int)4; long y = (long)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; long y = (long)1; return x & y"));
+
+        assertEquals(0, exec("def x = (byte)4; byte y = (byte)1; return x & y"));
+        assertEquals(0, exec("def x = (short)4; short y = (short)1; return x & y"));
+        assertEquals(0, exec("def x = (char)4; char y = (char)1; return x & y"));
+        assertEquals(0, exec("def x = (int)4; int y = (int)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; long y = (long)1; return x & y"));
+
+        assertEquals(true,  exec("def x = true;  boolean y = true; return x & y"));
+        assertEquals(false, exec("def x = true;  boolean y = false; return x & y"));
+        assertEquals(false, exec("def x = false; boolean y = true; return x & y"));
+        assertEquals(false, exec("def x = false; boolean y = false; return x & y"));
+    }
 
     public void testXor() {
         expectScriptThrows(ClassCastException.class, () -> {
@@ -665,6 +1685,104 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(true,  exec("def x = false; def y = true; return x ^ y"));
         assertEquals(false, exec("def x = false; def y = false; return x ^ y"));
     }
+    
+    public void testXorTypedLHS() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("float x = (float)4; def y = (byte)1; return x ^ y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("double x = (double)4; def y = (byte)1; return x ^ y");
+        });
+        assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; def y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; def y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; def y = (byte)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (byte)1; return x ^ y"));
+
+        assertEquals(5, exec("def x = (byte)4; def y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; def y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; def y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; def y = (short)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (short)1; return x ^ y"));
+
+        assertEquals(5, exec("def x = (byte)4; def y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; def y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; def y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; def y = (char)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (char)1; return x ^ y"));
+
+        assertEquals(5, exec("def x = (byte)4; def y = (int)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; def y = (int)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; def y = (int)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; def y = (int)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (int)1; return x ^ y"));
+
+        assertEquals(5L, exec("def x = (byte)4; def y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (short)4; def y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (char)4; def y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (int)4; def y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x ^ y"));
+
+        assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; def y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; def y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; def y = (int)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x ^ y"));
+        
+        assertEquals(false, exec("def x = true;  def y = true; return x ^ y"));
+        assertEquals(true,  exec("def x = true;  def y = false; return x ^ y"));
+        assertEquals(true,  exec("def x = false; def y = true; return x ^ y"));
+        assertEquals(false, exec("def x = false; def y = false; return x ^ y"));
+    }
+    
+    public void testXorTypedRHS() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (float)4; byte y = (byte)1; return x ^ y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (double)4; byte y = (byte)1; return x ^ y");
+        });
+        assertEquals(5, exec("def x = (byte)4; byte y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; byte y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; byte y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; byte y = (byte)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; byte y = (byte)1; return x ^ y"));
+
+        assertEquals(5, exec("def x = (byte)4; short y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; short y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; short y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; short y = (short)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; short y = (short)1; return x ^ y"));
+
+        assertEquals(5, exec("def x = (byte)4; char y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; char y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; char y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; char y = (char)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; char y = (char)1; return x ^ y"));
+
+        assertEquals(5, exec("def x = (byte)4; int y = (int)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; int y = (int)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; int y = (int)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; int y = (int)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; int y = (int)1; return x ^ y"));
+
+        assertEquals(5L, exec("def x = (byte)4; long y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (short)4; long y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (char)4; long y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (int)4; long y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; long y = (long)1; return x ^ y"));
+
+        assertEquals(5, exec("def x = (byte)4; byte y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; short y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; char y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; int y = (int)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; long y = (long)1; return x ^ y"));
+        
+        assertEquals(false, exec("def x = true;  boolean y = true; return x ^ y"));
+        assertEquals(true,  exec("def x = true;  boolean y = false; return x ^ y"));
+        assertEquals(true,  exec("def x = false; boolean y = true; return x ^ y"));
+        assertEquals(false, exec("def x = false; boolean y = false; return x ^ y"));
+    }
 
     public void testOr() {
         expectScriptThrows(ClassCastException.class, () -> {
@@ -714,6 +1832,104 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(true,  exec("def x = false; def y = true; return x | y"));
         assertEquals(false, exec("def x = false; def y = false; return x | y"));
     }
+    
+    public void testOrTypedLHS() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("float x = (float)4; def y = (byte)1; return x | y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("double x = (double)4; def y = (byte)1; return x | y");
+        });
+        assertEquals(5, exec("byte x = (byte)4; def y = (byte)1; return x | y"));
+        assertEquals(5, exec("short x = (short)4; def y = (byte)1; return x | y"));
+        assertEquals(5, exec("char x = (char)4; def y = (byte)1; return x | y"));
+        assertEquals(5, exec("int x = (int)4; def y = (byte)1; return x | y"));
+        assertEquals(5L, exec("long x = (long)4; def y = (byte)1; return x | y"));
+
+        assertEquals(5, exec("byte x = (byte)4; def y = (short)1; return x | y"));
+        assertEquals(5, exec("short x = (short)4; def y = (short)1; return x | y"));
+        assertEquals(5, exec("char x = (char)4; def y = (short)1; return x | y"));
+        assertEquals(5, exec("int x = (int)4; def y = (short)1; return x | y"));
+        assertEquals(5L, exec("long x = (long)4; def y = (short)1; return x | y"));
+
+        assertEquals(5, exec("byte x = (byte)4; def y = (char)1; return x | y"));
+        assertEquals(5, exec("short x = (short)4; def y = (char)1; return x | y"));
+        assertEquals(5, exec("char x = (char)4; def y = (char)1; return x | y"));
+        assertEquals(5, exec("int x = (int)4; def y = (char)1; return x | y"));
+        assertEquals(5L, exec("long x = (long)4; def y = (char)1; return x | y"));
+
+        assertEquals(5, exec("byte x = (byte)4; def y = (int)1; return x | y"));
+        assertEquals(5, exec("short x = (short)4; def y = (int)1; return x | y"));
+        assertEquals(5, exec("char x = (char)4; def y = (int)1; return x | y"));
+        assertEquals(5, exec("int x = (int)4; def y = (int)1; return x | y"));
+        assertEquals(5L, exec("long x = (long)4; def y = (int)1; return x | y"));
+
+        assertEquals(5L, exec("byte x = (byte)4; def y = (long)1; return x | y"));
+        assertEquals(5L, exec("short x = (short)4; def y = (long)1; return x | y"));
+        assertEquals(5L, exec("char x = (char)4; def y = (long)1; return x | y"));
+        assertEquals(5L, exec("int x = (int)4; def y = (long)1; return x | y"));
+        assertEquals(5L, exec("long x = (long)4; def y = (long)1; return x | y"));
+
+        assertEquals(5, exec("byte x = (byte)4; def y = (byte)1; return x | y"));
+        assertEquals(5, exec("short x = (short)4; def y = (short)1; return x | y"));
+        assertEquals(5, exec("char x = (char)4; def y = (char)1; return x | y"));
+        assertEquals(5, exec("int x = (int)4; def y = (int)1; return x | y"));
+        assertEquals(5L, exec("long x = (long)4; def y = (long)1; return x | y"));
+        
+        assertEquals(true,  exec("boolean x = true;  def y = true; return x | y"));
+        assertEquals(true,  exec("boolean x = true;  def y = false; return x | y"));
+        assertEquals(true,  exec("boolean x = false; def y = true; return x | y"));
+        assertEquals(false, exec("boolean x = false; def y = false; return x | y"));
+    }
+    
+    public void testOrTypedRHS() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (float)4; byte y = (byte)1; return x | y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (double)4; byte y = (byte)1; return x | y");
+        });
+        assertEquals(5, exec("def x = (byte)4; byte y = (byte)1; return x | y"));
+        assertEquals(5, exec("def x = (short)4; byte y = (byte)1; return x | y"));
+        assertEquals(5, exec("def x = (char)4; byte y = (byte)1; return x | y"));
+        assertEquals(5, exec("def x = (int)4; byte y = (byte)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; byte y = (byte)1; return x | y"));
+
+        assertEquals(5, exec("def x = (byte)4; short y = (short)1; return x | y"));
+        assertEquals(5, exec("def x = (short)4; short y = (short)1; return x | y"));
+        assertEquals(5, exec("def x = (char)4; short y = (short)1; return x | y"));
+        assertEquals(5, exec("def x = (int)4; short y = (short)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; short y = (short)1; return x | y"));
+
+        assertEquals(5, exec("def x = (byte)4; char y = (char)1; return x | y"));
+        assertEquals(5, exec("def x = (short)4; char y = (char)1; return x | y"));
+        assertEquals(5, exec("def x = (char)4; char y = (char)1; return x | y"));
+        assertEquals(5, exec("def x = (int)4; char y = (char)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; char y = (char)1; return x | y"));
+
+        assertEquals(5, exec("def x = (byte)4; int y = (int)1; return x | y"));
+        assertEquals(5, exec("def x = (short)4; int y = (int)1; return x | y"));
+        assertEquals(5, exec("def x = (char)4; int y = (int)1; return x | y"));
+        assertEquals(5, exec("def x = (int)4; int y = (int)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; int y = (int)1; return x | y"));
+
+        assertEquals(5L, exec("def x = (byte)4; long y = (long)1; return x | y"));
+        assertEquals(5L, exec("def x = (short)4; long y = (long)1; return x | y"));
+        assertEquals(5L, exec("def x = (char)4; long y = (long)1; return x | y"));
+        assertEquals(5L, exec("def x = (int)4; long y = (long)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; long y = (long)1; return x | y"));
+
+        assertEquals(5, exec("def x = (byte)4; byte y = (byte)1; return x | y"));
+        assertEquals(5, exec("def x = (short)4; short y = (short)1; return x | y"));
+        assertEquals(5, exec("def x = (char)4; char y = (char)1; return x | y"));
+        assertEquals(5, exec("def x = (int)4; int y = (int)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; long y = (long)1; return x | y"));
+        
+        assertEquals(true,  exec("def x = true;  boolean y = true; return x | y"));
+        assertEquals(true,  exec("def x = true;  boolean y = false; return x | y"));
+        assertEquals(true,  exec("def x = false; boolean y = true; return x | y"));
+        assertEquals(false, exec("def x = false; boolean y = false; return x | y"));
+    }
 
     public void testEq() {
         assertEquals(true, exec("def x = (byte)7; def y = (int)7; return x == y"));
@@ -743,11 +1959,64 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x == y"));
         assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x == y"));
         assertEquals(true, exec("def x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x == y"));
+    }
+    
+    public void testEqTypedLHS() {
+        assertEquals(true, exec("byte x = (byte)7; def y = (int)7; return x == y"));
+        assertEquals(true, exec("short x = (short)6; def y = (int)6; return x == y"));
+        assertEquals(true, exec("char x = (char)5; def y = (int)5; return x == y"));
+        assertEquals(true, exec("int x = (int)4; def y = (int)4; return x == y"));
+        assertEquals(false, exec("long x = (long)5; def y = (int)3; return x == y"));
+        assertEquals(false, exec("float x = (float)6; def y = (int)2; return x == y"));
+        assertEquals(false, exec("double x = (double)7; def y = (int)1; return x == y"));
+
+        assertEquals(true, exec("byte x = (byte)7; def y = (double)7; return x == y"));
+        assertEquals(true, exec("short x = (short)6; def y = (double)6; return x == y"));
+        assertEquals(true, exec("char x = (char)5; def y = (double)5; return x == y"));
+        assertEquals(true, exec("int x = (int)4; def y = (double)4; return x == y"));
+        assertEquals(false, exec("long x = (long)5; def y = (double)3; return x == y"));
+        assertEquals(false, exec("float x = (float)6; def y = (double)2; return x == y"));
+        assertEquals(false, exec("double x = (double)7; def y = (double)1; return x == y"));
         
-        assertEquals(true,  exec("def x = true;  def y = true; return x == y"));
-        assertEquals(false, exec("def x = true;  def y = false; return x == y"));
-        assertEquals(false, exec("def x = false; def y = true; return x == y"));
-        assertEquals(true,  exec("def x = false; def y = false; return x == y"));
+        assertEquals(false, exec("boolean x = false; def y = true; return x == y"));
+        assertEquals(false, exec("boolean x = true; def y = false; return x == y"));
+        assertEquals(false, exec("boolean x = true; def y = null; return x == y"));
+        assertEquals(true, exec("boolean x = true; def y = true; return x == y"));
+        assertEquals(true, exec("boolean x = false; def y = false; return x == y"));
+
+        assertEquals(true, exec("Map x = new HashMap(); def y = new HashMap(); return x == y"));
+        assertEquals(false, exec("Map x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x == y"));
+        assertEquals(true, exec("Map x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x == y"));
+        assertEquals(true, exec("Map x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x == y"));
+    }
+    
+    public void testEqTypedRHS() {
+        assertEquals(true, exec("def x = (byte)7; int y = (int)7; return x == y"));
+        assertEquals(true, exec("def x = (short)6; int y = (int)6; return x == y"));
+        assertEquals(true, exec("def x = (char)5; int y = (int)5; return x == y"));
+        assertEquals(true, exec("def x = (int)4; int y = (int)4; return x == y"));
+        assertEquals(false, exec("def x = (long)5; int y = (int)3; return x == y"));
+        assertEquals(false, exec("def x = (float)6; int y = (int)2; return x == y"));
+        assertEquals(false, exec("def x = (double)7; int y = (int)1; return x == y"));
+
+        assertEquals(true, exec("def x = (byte)7; double y = (double)7; return x == y"));
+        assertEquals(true, exec("def x = (short)6; double y = (double)6; return x == y"));
+        assertEquals(true, exec("def x = (char)5; double y = (double)5; return x == y"));
+        assertEquals(true, exec("def x = (int)4; double y = (double)4; return x == y"));
+        assertEquals(false, exec("def x = (long)5; double y = (double)3; return x == y"));
+        assertEquals(false, exec("def x = (float)6; double y = (double)2; return x == y"));
+        assertEquals(false, exec("def x = (double)7; double y = (double)1; return x == y"));
+        
+        assertEquals(false, exec("def x = false; boolean y = true; return x == y"));
+        assertEquals(false, exec("def x = true; boolean y = false; return x == y"));
+        assertEquals(false, exec("def x = null; boolean y = true; return x == y"));
+        assertEquals(true, exec("def x = true; boolean y = true; return x == y"));
+        assertEquals(true, exec("def x = false; boolean y = false; return x == y"));
+
+        assertEquals(true, exec("def x = new HashMap(); Map y = new HashMap(); return x == y"));
+        assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); Map y = new HashMap(); return x == y"));
+        assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); Map y = new HashMap(); y.put(3, 3); return x == y"));
+        assertEquals(true, exec("def x = new HashMap(); Map y = x; x.put(3, 3); y.put(3, 3); return x == y"));
     }
 
     public void testEqr() {
@@ -793,6 +2062,62 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(true,   exec("def x = false; def y = true; return x != y"));
         assertEquals(false,  exec("def x = false; def y = false; return x != y"));
     }
+    
+    public void testNeTypedLHS() {
+        assertEquals(false, exec("byte x = (byte)7; def y = (int)7; return x != y"));
+        assertEquals(false, exec("short x = (short)6; def y = (int)6; return x != y"));
+        assertEquals(false, exec("char x = (char)5; def y = (int)5; return x != y"));
+        assertEquals(false, exec("int x = (int)4; def y = (int)4; return x != y"));
+        assertEquals(true, exec("long x = (long)5; def y = (int)3; return x != y"));
+        assertEquals(true, exec("float x = (float)6; def y = (int)2; return x != y"));
+        assertEquals(true, exec("double x = (double)7; def y = (int)1; return x != y"));
+
+        assertEquals(false, exec("byte x = (byte)7; def y = (double)7; return x != y"));
+        assertEquals(false, exec("short x = (short)6; def y = (double)6; return x != y"));
+        assertEquals(false, exec("char x = (char)5; def y = (double)5; return x != y"));
+        assertEquals(false, exec("int x = (int)4; def y = (double)4; return x != y"));
+        assertEquals(true, exec("long x = (long)5; def y = (double)3; return x != y"));
+        assertEquals(true, exec("float x = (float)6; def y = (double)2; return x != y"));
+        assertEquals(true, exec("double x = (double)7; def y = (double)1; return x != y"));
+
+        assertEquals(false, exec("Map x = new HashMap(); def y = new HashMap(); return x != y"));
+        assertEquals(true, exec("Map x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x != y"));
+        assertEquals(false, exec("Map x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x != y"));
+        assertEquals(false, exec("Map x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x != y"));
+        
+        assertEquals(false,  exec("boolean x = true;  def y = true; return x != y"));
+        assertEquals(true,   exec("boolean x = true;  def y = false; return x != y"));
+        assertEquals(true,   exec("boolean x = false; def y = true; return x != y"));
+        assertEquals(false,  exec("boolean x = false; def y = false; return x != y"));
+    }
+    
+    public void testNeTypedRHS() {
+        assertEquals(false, exec("def x = (byte)7; int y = (int)7; return x != y"));
+        assertEquals(false, exec("def x = (short)6; int y = (int)6; return x != y"));
+        assertEquals(false, exec("def x = (char)5; int y = (int)5; return x != y"));
+        assertEquals(false, exec("def x = (int)4; int y = (int)4; return x != y"));
+        assertEquals(true, exec("def x = (long)5; int y = (int)3; return x != y"));
+        assertEquals(true, exec("def x = (float)6; int y = (int)2; return x != y"));
+        assertEquals(true, exec("def x = (double)7; int y = (int)1; return x != y"));
+
+        assertEquals(false, exec("def x = (byte)7; double y = (double)7; return x != y"));
+        assertEquals(false, exec("def x = (short)6; double y = (double)6; return x != y"));
+        assertEquals(false, exec("def x = (char)5; double y = (double)5; return x != y"));
+        assertEquals(false, exec("def x = (int)4; double y = (double)4; return x != y"));
+        assertEquals(true, exec("def x = (long)5; double y = (double)3; return x != y"));
+        assertEquals(true, exec("def x = (float)6; double y = (double)2; return x != y"));
+        assertEquals(true, exec("def x = (double)7; double y = (double)1; return x != y"));
+
+        assertEquals(false, exec("def x = new HashMap(); Map y = new HashMap(); return x != y"));
+        assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); Map y = new HashMap(); return x != y"));
+        assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); Map y = new HashMap(); y.put(3, 3); return x != y"));
+        assertEquals(false, exec("def x = new HashMap(); Map y = x; x.put(3, 3); y.put(3, 3); return x != y"));
+        
+        assertEquals(false,  exec("def x = true;  boolean y = true; return x != y"));
+        assertEquals(true,   exec("def x = true;  boolean y = false; return x != y"));
+        assertEquals(true,   exec("def x = false; boolean y = true; return x != y"));
+        assertEquals(false,  exec("def x = false; boolean y = false; return x != y"));
+    }
 
     public void testNer() {
         assertEquals(true, exec("def x = (byte)7; def y = (int)7; return x !== y"));
@@ -826,6 +2151,42 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(false, exec("def x = (float)6; def y = (double)2; return x < y"));
         assertEquals(false, exec("def x = (double)7; def y = (double)1; return x < y"));
     }
+    
+    public void testLtTypedLHS() {
+        assertEquals(true, exec("byte x = (byte)1; def y = (int)7; return x < y"));
+        assertEquals(true, exec("short x = (short)2; def y = (int)6; return x < y"));
+        assertEquals(true, exec("char x = (char)3; def y = (int)5; return x < y"));
+        assertEquals(false, exec("int x = (int)4; def y = (int)4; return x < y"));
+        assertEquals(false, exec("long x = (long)5; def y = (int)3; return x < y"));
+        assertEquals(false, exec("float x = (float)6; def y = (int)2; return x < y"));
+        assertEquals(false, exec("double x = (double)7; def y = (int)1; return x < y"));
+
+        assertEquals(true, exec("byte x = (byte)1; def y = (double)7; return x < y"));
+        assertEquals(true, exec("short x = (short)2; def y = (double)6; return x < y"));
+        assertEquals(true, exec("char x = (char)3; def y = (double)5; return x < y"));
+        assertEquals(false, exec("int x = (int)4; def y = (double)4; return x < y"));
+        assertEquals(false, exec("long x = (long)5; def y = (double)3; return x < y"));
+        assertEquals(false, exec("float x = (float)6; def y = (double)2; return x < y"));
+        assertEquals(false, exec("double x = (double)7; def y = (double)1; return x < y"));
+    }
+    
+    public void testLtTypedRHS() {
+        assertEquals(true, exec("def x = (byte)1; int y = (int)7; return x < y"));
+        assertEquals(true, exec("def x = (short)2; int y = (int)6; return x < y"));
+        assertEquals(true, exec("def x = (char)3; int y = (int)5; return x < y"));
+        assertEquals(false, exec("def x = (int)4; int y = (int)4; return x < y"));
+        assertEquals(false, exec("def x = (long)5; int y = (int)3; return x < y"));
+        assertEquals(false, exec("def x = (float)6; int y = (int)2; return x < y"));
+        assertEquals(false, exec("def x = (double)7; int y = (int)1; return x < y"));
+
+        assertEquals(true, exec("def x = (byte)1; double y = (double)7; return x < y"));
+        assertEquals(true, exec("def x = (short)2; double y = (double)6; return x < y"));
+        assertEquals(true, exec("def x = (char)3; double y = (double)5; return x < y"));
+        assertEquals(false, exec("def x = (int)4; double y = (double)4; return x < y"));
+        assertEquals(false, exec("def x = (long)5; double y = (double)3; return x < y"));
+        assertEquals(false, exec("def x = (float)6; double y = (double)2; return x < y"));
+        assertEquals(false, exec("def x = (double)7; double y = (double)1; return x < y"));
+    }
 
     public void testLte() {
         assertEquals(true, exec("def x = (byte)1; def y = (int)7; return x <= y"));
@@ -844,6 +2205,42 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(false, exec("def x = (float)6; def y = (double)2; return x <= y"));
         assertEquals(false, exec("def x = (double)7; def y = (double)1; return x <= y"));
     }
+    
+    public void testLteTypedLHS() {
+        assertEquals(true, exec("byte x = (byte)1; def y = (int)7; return x <= y"));
+        assertEquals(true, exec("short x = (short)2; def y = (int)6; return x <= y"));
+        assertEquals(true, exec("char x = (char)3; def y = (int)5; return x <= y"));
+        assertEquals(true, exec("int x = (int)4; def y = (int)4; return x <= y"));
+        assertEquals(false, exec("long x = (long)5; def y = (int)3; return x <= y"));
+        assertEquals(false, exec("float x = (float)6; def y = (int)2; return x <= y"));
+        assertEquals(false, exec("double x = (double)7; def y = (int)1; return x <= y"));
+
+        assertEquals(true, exec("byte x = (byte)1; def y = (double)7; return x <= y"));
+        assertEquals(true, exec("short x = (short)2; def y = (double)6; return x <= y"));
+        assertEquals(true, exec("char x = (char)3; def y = (double)5; return x <= y"));
+        assertEquals(true, exec("int x = (int)4; def y = (double)4; return x <= y"));
+        assertEquals(false, exec("long x = (long)5; def y = (double)3; return x <= y"));
+        assertEquals(false, exec("float x = (float)6; def y = (double)2; return x <= y"));
+        assertEquals(false, exec("double x = (double)7; def y = (double)1; return x <= y"));
+    }
+    
+    public void testLteTypedRHS() {
+        assertEquals(true, exec("def x = (byte)1; int y = (int)7; return x <= y"));
+        assertEquals(true, exec("def x = (short)2; int y = (int)6; return x <= y"));
+        assertEquals(true, exec("def x = (char)3; int y = (int)5; return x <= y"));
+        assertEquals(true, exec("def x = (int)4; int y = (int)4; return x <= y"));
+        assertEquals(false, exec("def x = (long)5; int y = (int)3; return x <= y"));
+        assertEquals(false, exec("def x = (float)6; int y = (int)2; return x <= y"));
+        assertEquals(false, exec("def x = (double)7; int y = (int)1; return x <= y"));
+
+        assertEquals(true, exec("def x = (byte)1; double y = (double)7; return x <= y"));
+        assertEquals(true, exec("def x = (short)2; double y = (double)6; return x <= y"));
+        assertEquals(true, exec("def x = (char)3; double y = (double)5; return x <= y"));
+        assertEquals(true, exec("def x = (int)4; double y = (double)4; return x <= y"));
+        assertEquals(false, exec("def x = (long)5; double y = (double)3; return x <= y"));
+        assertEquals(false, exec("def x = (float)6; double y = (double)2; return x <= y"));
+        assertEquals(false, exec("def x = (double)7; double y = (double)1; return x <= y"));
+    }
 
     public void testGt() {
         assertEquals(false, exec("def x = (byte)1; def y = (int)7; return x > y"));
@@ -862,6 +2259,42 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(true, exec("def x = (float)6; def y = (double)2; return x > y"));
         assertEquals(true, exec("def x = (double)7; def y = (double)1; return x > y"));
     }
+    
+    public void testGtTypedLHS() {
+        assertEquals(false, exec("byte x = (byte)1; def y = (int)7; return x > y"));
+        assertEquals(false, exec("short x = (short)2; def y = (int)6; return x > y"));
+        assertEquals(false, exec("char x = (char)3; def y = (int)5; return x > y"));
+        assertEquals(false, exec("int x = (int)4; def y = (int)4; return x > y"));
+        assertEquals(true, exec("long x = (long)5; def y = (int)3; return x > y"));
+        assertEquals(true, exec("float x = (float)6; def y = (int)2; return x > y"));
+        assertEquals(true, exec("double x = (double)7; def y = (int)1; return x > y"));
+
+        assertEquals(false, exec("byte x = (byte)1; def y = (double)7; return x > y"));
+        assertEquals(false, exec("short x = (short)2; def y = (double)6; return x > y"));
+        assertEquals(false, exec("char x = (char)3; def y = (double)5; return x > y"));
+        assertEquals(false, exec("int x = (int)4; def y = (double)4; return x > y"));
+        assertEquals(true, exec("long x = (long)5; def y = (double)3; return x > y"));
+        assertEquals(true, exec("float x = (float)6; def y = (double)2; return x > y"));
+        assertEquals(true, exec("double x = (double)7; def y = (double)1; return x > y"));
+    }
+    
+    public void testGtTypedRHS() {
+        assertEquals(false, exec("def x = (byte)1; int y = (int)7; return x > y"));
+        assertEquals(false, exec("def x = (short)2; int y = (int)6; return x > y"));
+        assertEquals(false, exec("def x = (char)3; int y = (int)5; return x > y"));
+        assertEquals(false, exec("def x = (int)4; int y = (int)4; return x > y"));
+        assertEquals(true, exec("def x = (long)5; int y = (int)3; return x > y"));
+        assertEquals(true, exec("def x = (float)6; int y = (int)2; return x > y"));
+        assertEquals(true, exec("def x = (double)7; int y = (int)1; return x > y"));
+
+        assertEquals(false, exec("def x = (byte)1; double y = (double)7; return x > y"));
+        assertEquals(false, exec("def x = (short)2; double y = (double)6; return x > y"));
+        assertEquals(false, exec("def x = (char)3; double y = (double)5; return x > y"));
+        assertEquals(false, exec("def x = (int)4; double y = (double)4; return x > y"));
+        assertEquals(true, exec("def x = (long)5; double y = (double)3; return x > y"));
+        assertEquals(true, exec("def x = (float)6; double y = (double)2; return x > y"));
+        assertEquals(true, exec("def x = (double)7; double y = (double)1; return x > y"));
+    }
 
     public void testGte() {
         assertEquals(false, exec("def x = (byte)1; def y = (int)7; return x >= y"));
@@ -880,4 +2313,40 @@ public class DefOperationTests extends ScriptTestCase {
         assertEquals(true, exec("def x = (float)6; def y = (double)2; return x >= y"));
         assertEquals(true, exec("def x = (double)7; def y = (double)1; return x >= y"));
     }
+    
+    public void testGteTypedLHS() {
+        assertEquals(false, exec("byte x = (byte)1; def y = (int)7; return x >= y"));
+        assertEquals(false, exec("short x = (short)2; def y = (int)6; return x >= y"));
+        assertEquals(false, exec("char x = (char)3; def y = (int)5; return x >= y"));
+        assertEquals(true, exec("int x = (int)4; def y = (int)4; return x >= y"));
+        assertEquals(true, exec("long x = (long)5; def y = (int)3; return x >= y"));
+        assertEquals(true, exec("float x = (float)6; def y = (int)2; return x >= y"));
+        assertEquals(true, exec("double x = (double)7; def y = (int)1; return x >= y"));
+
+        assertEquals(false, exec("byte x = (byte)1; def y = (double)7; return x >= y"));
+        assertEquals(false, exec("short x = (short)2; def y = (double)6; return x >= y"));
+        assertEquals(false, exec("char x = (char)3; def y = (double)5; return x >= y"));
+        assertEquals(true, exec("int x = (int)4; def y = (double)4; return x >= y"));
+        assertEquals(true, exec("long x = (long)5; def y = (double)3; return x >= y"));
+        assertEquals(true, exec("float x = (float)6; def y = (double)2; return x >= y"));
+        assertEquals(true, exec("double x = (double)7; def y = (double)1; return x >= y"));
+    }
+    
+    public void testGteTypedRHS() {
+        assertEquals(false, exec("def x = (byte)1; int y = (int)7; return x >= y"));
+        assertEquals(false, exec("def x = (short)2; int y = (int)6; return x >= y"));
+        assertEquals(false, exec("def x = (char)3; int y = (int)5; return x >= y"));
+        assertEquals(true, exec("def x = (int)4; int y = (int)4; return x >= y"));
+        assertEquals(true, exec("def x = (long)5; int y = (int)3; return x >= y"));
+        assertEquals(true, exec("def x = (float)6; int y = (int)2; return x >= y"));
+        assertEquals(true, exec("def x = (double)7; int y = (int)1; return x >= y"));
+
+        assertEquals(false, exec("def x = (byte)1; double y = (double)7; return x >= y"));
+        assertEquals(false, exec("def x = (short)2; double y = (double)6; return x >= y"));
+        assertEquals(false, exec("def x = (char)3; double y = (double)5; return x >= y"));
+        assertEquals(true, exec("def x = (int)4; double y = (double)4; return x >= y"));
+        assertEquals(true, exec("def x = (long)5; double y = (double)3; return x >= y"));
+        assertEquals(true, exec("def x = (float)6; double y = (double)2; return x >= y"));
+        assertEquals(true, exec("def x = (double)7; double y = (double)1; return x >= y"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
index f9771862335..3a7196f9074 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
@@ -178,4 +178,229 @@ public class DefOptimizationTests extends ScriptTestCase {
         });
         assertTrue(exception.getMessage().contains("Cannot cast java.lang.Double to java.lang.Integer"));
     }
+    
+    public void testMulOptLHS() {
+        assertBytecodeExists("int x = 1; def y = 2; return x * y", 
+                             "INVOKEDYNAMIC mul(ILjava/lang/Object;)Ljava/lang/Object;");
+    }
+    
+    public void testMulOptRHS() {
+        assertBytecodeExists("def x = 1; int y = 2; return x * y", 
+                             "INVOKEDYNAMIC mul(Ljava/lang/Object;I)Ljava/lang/Object;");
+    }
+    
+    public void testMulOptRet() {
+        assertBytecodeExists("def x = 1; def y = 2; double d = x * y", 
+                             "INVOKEDYNAMIC mul(Ljava/lang/Object;Ljava/lang/Object;)D");
+    }
+    
+    public void testDivOptLHS() {
+        assertBytecodeExists("int x = 1; def y = 2; return x / y", 
+                             "INVOKEDYNAMIC div(ILjava/lang/Object;)Ljava/lang/Object;");
+    }
+    
+    public void testDivOptRHS() {
+        assertBytecodeExists("def x = 1; int y = 2; return x / y", 
+                             "INVOKEDYNAMIC div(Ljava/lang/Object;I)Ljava/lang/Object;");
+    }
+    
+    public void testDivOptRet() {
+        assertBytecodeExists("def x = 1; def y = 2; double d = x / y", 
+                             "INVOKEDYNAMIC div(Ljava/lang/Object;Ljava/lang/Object;)D");
+    }
+    
+    public void testRemOptLHS() {
+        assertBytecodeExists("int x = 1; def y = 2; return x % y", 
+                             "INVOKEDYNAMIC rem(ILjava/lang/Object;)Ljava/lang/Object;");
+    }
+    
+    public void testRemOptRHS() {
+        assertBytecodeExists("def x = 1; int y = 2; return x % y", 
+                             "INVOKEDYNAMIC rem(Ljava/lang/Object;I)Ljava/lang/Object;");
+    }
+    
+    public void testRemOptRet() {
+        assertBytecodeExists("def x = 1; def y = 2; double d = x % y", 
+                             "INVOKEDYNAMIC rem(Ljava/lang/Object;Ljava/lang/Object;)D");
+    }
+    
+    public void testAddOptLHS() {
+        assertBytecodeExists("int x = 1; def y = 2; return x + y", 
+                             "INVOKEDYNAMIC add(ILjava/lang/Object;)Ljava/lang/Object;");
+    }
+    
+    public void testAddOptRHS() {
+        assertBytecodeExists("def x = 1; int y = 2; return x + y", 
+                             "INVOKEDYNAMIC add(Ljava/lang/Object;I)Ljava/lang/Object;");
+    }
+    
+    public void testAddOptRet() {
+        assertBytecodeExists("def x = 1; def y = 2; double d = x + y", 
+                             "INVOKEDYNAMIC add(Ljava/lang/Object;Ljava/lang/Object;)D");
+    }
+    
+    public void testSubOptLHS() {
+        assertBytecodeExists("int x = 1; def y = 2; return x - y", 
+                             "INVOKEDYNAMIC sub(ILjava/lang/Object;)Ljava/lang/Object;");
+    }
+    
+    public void testSubOptRHS() {
+        assertBytecodeExists("def x = 1; int y = 2; return x - y", 
+                             "INVOKEDYNAMIC sub(Ljava/lang/Object;I)Ljava/lang/Object;");
+    }
+    
+    public void testSubOptRet() {
+        assertBytecodeExists("def x = 1; def y = 2; double d = x - y", 
+                             "INVOKEDYNAMIC sub(Ljava/lang/Object;Ljava/lang/Object;)D");
+    }
+    
+    public void testLshOptLHS() {
+        assertBytecodeExists("int x = 1; def y = 2; return x << y", 
+                             "INVOKEDYNAMIC lsh(ILjava/lang/Object;)Ljava/lang/Object;");
+    }
+    
+    public void testLshOptRHS() {
+        assertBytecodeExists("def x = 1; int y = 2; return x << y", 
+                             "INVOKEDYNAMIC lsh(Ljava/lang/Object;I)Ljava/lang/Object;");
+    }
+    
+    public void testLshOptRet() {
+        assertBytecodeExists("def x = 1; def y = 2; double d = x << y", 
+                             "INVOKEDYNAMIC lsh(Ljava/lang/Object;Ljava/lang/Object;)D");
+    }
+    
+    public void testRshOptLHS() {
+        assertBytecodeExists("int x = 1; def y = 2; return x >> y", 
+                             "INVOKEDYNAMIC rsh(ILjava/lang/Object;)Ljava/lang/Object;");
+    }
+    
+    public void testRshOptRHS() {
+        assertBytecodeExists("def x = 1; int y = 2; return x >> y", 
+                             "INVOKEDYNAMIC rsh(Ljava/lang/Object;I)Ljava/lang/Object;");
+    }
+    
+    public void testRshOptRet() {
+        assertBytecodeExists("def x = 1; def y = 2; double d = x >> y", 
+                             "INVOKEDYNAMIC rsh(Ljava/lang/Object;Ljava/lang/Object;)D");
+    }
+    
+    public void testUshOptLHS() {
+        assertBytecodeExists("int x = 1; def y = 2; return x >>> y", 
+                             "INVOKEDYNAMIC ush(ILjava/lang/Object;)Ljava/lang/Object;");
+    }
+    
+    public void testUshOptRHS() {
+        assertBytecodeExists("def x = 1; int y = 2; return x >>> y", 
+                             "INVOKEDYNAMIC ush(Ljava/lang/Object;I)Ljava/lang/Object;");
+    }
+    
+    public void testUshOptRet() {
+        assertBytecodeExists("def x = 1; def y = 2; double d = x >>> y", 
+                             "INVOKEDYNAMIC ush(Ljava/lang/Object;Ljava/lang/Object;)D");
+    }
+    
+    public void testAndOptLHS() {
+        assertBytecodeExists("int x = 1; def y = 2; return x & y", 
+                             "INVOKEDYNAMIC and(ILjava/lang/Object;)Ljava/lang/Object;");
+    }
+    
+    public void testAndOptRHS() {
+        assertBytecodeExists("def x = 1; int y = 2; return x & y", 
+                             "INVOKEDYNAMIC and(Ljava/lang/Object;I)Ljava/lang/Object;");
+    }
+    
+    public void testAndOptRet() {
+        assertBytecodeExists("def x = 1; def y = 2; double d = x & y", 
+                             "INVOKEDYNAMIC and(Ljava/lang/Object;Ljava/lang/Object;)D");
+    }
+    
+    public void testOrOptLHS() {
+        assertBytecodeExists("int x = 1; def y = 2; return x | y", 
+                             "INVOKEDYNAMIC or(ILjava/lang/Object;)Ljava/lang/Object;");
+    }
+    
+    public void testOrOptRHS() {
+        assertBytecodeExists("def x = 1; int y = 2; return x | y", 
+                             "INVOKEDYNAMIC or(Ljava/lang/Object;I)Ljava/lang/Object;");
+    }
+    
+    public void testOrOptRet() {
+        assertBytecodeExists("def x = 1; def y = 2; double d = x | y", 
+                             "INVOKEDYNAMIC or(Ljava/lang/Object;Ljava/lang/Object;)D");
+    }
+    
+    public void testXorOptLHS() {
+        assertBytecodeExists("int x = 1; def y = 2; return x ^ y", 
+                             "INVOKEDYNAMIC xor(ILjava/lang/Object;)Ljava/lang/Object;");
+    }
+    
+    public void testXorOptRHS() {
+        assertBytecodeExists("def x = 1; int y = 2; return x ^ y", 
+                             "INVOKEDYNAMIC xor(Ljava/lang/Object;I)Ljava/lang/Object;");
+    }
+    
+    public void testXorOptRet() {
+        assertBytecodeExists("def x = 1; def y = 2; double d = x ^ y", 
+                             "INVOKEDYNAMIC xor(Ljava/lang/Object;Ljava/lang/Object;)D");
+    }
+    
+    public void testLtOptLHS() {
+        assertBytecodeExists("int x = 1; def y = 2; return x < y", 
+                             "INVOKEDYNAMIC lt(ILjava/lang/Object;)Z");
+    }
+    
+    public void testLtOptRHS() {
+        assertBytecodeExists("def x = 1; int y = 2; return x < y", 
+                             "INVOKEDYNAMIC lt(Ljava/lang/Object;I)Z");
+    }
+    
+    public void testLteOptLHS() {
+        assertBytecodeExists("int x = 1; def y = 2; return x <= y", 
+                             "INVOKEDYNAMIC lte(ILjava/lang/Object;)Z");
+    }
+    
+    public void testLteOptRHS() {
+        assertBytecodeExists("def x = 1; int y = 2; return x <= y", 
+                             "INVOKEDYNAMIC lte(Ljava/lang/Object;I)Z");
+    }
+    
+    public void testEqOptLHS() {
+        assertBytecodeExists("int x = 1; def y = 2; return x == y", 
+                             "INVOKEDYNAMIC eq(ILjava/lang/Object;)Z");
+    }
+    
+    public void testEqOptRHS() {
+        assertBytecodeExists("def x = 1; int y = 2; return x == y", 
+                             "INVOKEDYNAMIC eq(Ljava/lang/Object;I)Z");
+    }
+    
+    public void testNeqOptLHS() {
+        assertBytecodeExists("int x = 1; def y = 2; return x != y", 
+                             "INVOKEDYNAMIC eq(ILjava/lang/Object;)Z");
+    }
+    
+    public void testNeqOptRHS() {
+        assertBytecodeExists("def x = 1; int y = 2; return x != y", 
+                             "INVOKEDYNAMIC eq(Ljava/lang/Object;I)Z");
+    }
+    
+    public void testGteOptLHS() {
+        assertBytecodeExists("int x = 1; def y = 2; return x >= y", 
+                             "INVOKEDYNAMIC gte(ILjava/lang/Object;)Z");
+    }
+    
+    public void testGteOptRHS() {
+        assertBytecodeExists("def x = 1; int y = 2; return x >= y", 
+                             "INVOKEDYNAMIC gte(Ljava/lang/Object;I)Z");
+    }
+    
+    public void testGtOptLHS() {
+        assertBytecodeExists("int x = 1; def y = 2; return x > y", 
+                             "INVOKEDYNAMIC gt(ILjava/lang/Object;)Z");
+    }
+    
+    public void testGtOptRHS() {
+        assertBytecodeExists("def x = 1; int y = 2; return x > y", 
+                             "INVOKEDYNAMIC gt(Ljava/lang/Object;I)Z");
+    }
 }

From caa6c962596af6c1eb54ed636440aab320ce227f Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Tue, 14 Jun 2016 09:43:31 +0200
Subject: [PATCH 0824/1311] Build: make client and client-sniffer depend on
 lucene-test version 5, last 1.7 compatible version

---
 client-sniffer/build.gradle | 7 ++++---
 client/build.gradle         | 7 ++++---
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/client-sniffer/build.gradle b/client-sniffer/build.gradle
index af46a5a452c..5b224168492 100644
--- a/client-sniffer/build.gradle
+++ b/client-sniffer/build.gradle
@@ -37,9 +37,10 @@ dependencies {
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
-  testCompile "org.apache.lucene:lucene-test-framework:${versions.lucene}"
-  testCompile "org.apache.lucene:lucene-core:${versions.lucene}"
-  testCompile "org.apache.lucene:lucene-codecs:${versions.lucene}"
+  //we use the last lucene-test version that is compatible with java 1.7
+  testCompile "org.apache.lucene:lucene-test-framework:5.5.1"
+  testCompile "org.apache.lucene:lucene-core:5.5.1"
+  testCompile "org.apache.lucene:lucene-codecs:5.5.1"
   testCompile "org.elasticsearch:securemock:${versions.securemock}"
   testCompile "org.codehaus.mojo:animal-sniffer-annotations:1.15"
   signature "org.codehaus.mojo.signature:java17:1.0@signature"
diff --git a/client/build.gradle b/client/build.gradle
index c9c619f80ea..83c96d07d34 100644
--- a/client/build.gradle
+++ b/client/build.gradle
@@ -35,9 +35,10 @@ dependencies {
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
-  testCompile "org.apache.lucene:lucene-test-framework:${versions.lucene}"
-  testCompile "org.apache.lucene:lucene-core:${versions.lucene}"
-  testCompile "org.apache.lucene:lucene-codecs:${versions.lucene}"
+  //we use the last lucene-test version that is compatible with java 1.7
+  testCompile "org.apache.lucene:lucene-test-framework:5.5.1"
+  testCompile "org.apache.lucene:lucene-core:5.5.1"
+  testCompile "org.apache.lucene:lucene-codecs:5.5.1"
   testCompile "org.elasticsearch:securemock:${versions.securemock}"
   testCompile "org.codehaus.mojo:animal-sniffer-annotations:1.15"
   signature "org.codehaus.mojo.signature:java17:1.0@signature"

From 1932f6bc7c4c5d77f79ad27320e3dcdc3162ccfc Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Tue, 14 Jun 2016 13:45:38 +0200
Subject: [PATCH 0825/1311] Rename RequestLogger#log methods to distinguish
 between the two

One method is to log a request that yielded a response, the other one for a failed request
---
 .../main/java/org/elasticsearch/client/RequestLogger.java   | 4 ++--
 .../src/main/java/org/elasticsearch/client/RestClient.java  | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/RequestLogger.java b/client/src/main/java/org/elasticsearch/client/RequestLogger.java
index c08e0c69941..7268a449981 100644
--- a/client/src/main/java/org/elasticsearch/client/RequestLogger.java
+++ b/client/src/main/java/org/elasticsearch/client/RequestLogger.java
@@ -53,7 +53,7 @@ final class RequestLogger {
     /**
      * Logs a request that yielded a response
      */
-    static void log(Log logger, String message, HttpUriRequest request, HttpHost host, HttpResponse httpResponse) {
+    static void logResponse(Log logger, String message, HttpUriRequest request, HttpHost host, HttpResponse httpResponse) {
         if (logger.isDebugEnabled()) {
             logger.debug(message + " [" + request.getMethod() + " " + host + request.getRequestLine().getUri() +
                     "] [" + httpResponse.getStatusLine() + "]");
@@ -81,7 +81,7 @@ final class RequestLogger {
     /**
      * Logs a request that failed
      */
-    static void log(Log logger, String message, HttpUriRequest request, HttpHost host, IOException e) {
+    static void logFailedRequest(Log logger, String message, HttpUriRequest request, HttpHost host, IOException e) {
         logger.debug(message + " [" + request.getMethod() + " " + host + request.getRequestLine().getUri() + "]", e);
         if (logger.isTraceEnabled()) {
             String traceRequest;
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index b871e2e1346..b2bd6030681 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -160,7 +160,7 @@ public final class RestClient implements Closeable {
             try {
                 httpResponse = client.execute(host, request);
             } catch(IOException e) {
-                RequestLogger.log(logger, "request failed", request, host, e);
+                RequestLogger.logFailedRequest(logger, "request failed", request, host, e);
                 onFailure(host);
                 lastSeenException = addSuppressedException(lastSeenException, e);
                 continue;
@@ -168,11 +168,11 @@ public final class RestClient implements Closeable {
             Response response = new Response(request.getRequestLine(), host, httpResponse);
             int statusCode = response.getStatusLine().getStatusCode();
             if (statusCode < 300 || (request.getMethod().equals(HttpHead.METHOD_NAME) && statusCode == 404) ) {
-                RequestLogger.log(logger, "request succeeded", request, host, httpResponse);
+                RequestLogger.logResponse(logger, "request succeeded", request, host, httpResponse);
                 onSuccess(host);
                 return response;
             }
-            RequestLogger.log(logger, "request failed", request, host, httpResponse);
+            RequestLogger.logResponse(logger, "request failed", request, host, httpResponse);
             String responseBody;
             try {
                 if (response.getEntity() == null) {

From cfd3356ee39b56c68fb1ca7e0449b44df6bdd909 Mon Sep 17 00:00:00 2001
From: Colin Goodheart-Smithe <colings86@users.noreply.github.com>
Date: Tue, 14 Jun 2016 11:37:10 +0100
Subject: [PATCH 0826/1311] Remove size 0 options in aggregations

This removes the ability to set `size: 0` in the `terms`, `significant_terms` and `geohash_grid` aggregations for the reasons described in https://github.com/elastic/elasticsearch/issues/18838

Closes #18838
---
 .../geogrid/GeoGridAggregationBuilder.java    | 23 +++----
 .../bucket/terms/TermsAggregator.java         | 15 ++---
 .../aggregations/bucket/ChildrenIT.java       |  7 +--
 .../aggregations/bucket/GeoHashGridIT.java    | 35 ++++++-----
 .../aggregations/bucket/GeoHashGridTests.java | 27 +-------
 .../aggregations/bucket/ReverseNestedIT.java  |  2 +-
 .../bucket/SignificantTermsTests.java         | 30 +--------
 .../bucket/TermsDocCountErrorIT.java          | 63 +++++++------------
 .../aggregations/bucket/TermsTests.java       | 31 +--------
 .../bucket/geohashgrid-aggregation.asciidoc   |  6 --
 .../significantterms-aggregation.asciidoc     | 52 +++++++--------
 .../bucket/terms-aggregation.asciidoc         | 14 ++---
 .../migrate_5_0/aggregations.asciidoc         |  6 ++
 .../messy/tests/DoubleTermsTests.java         | 22 +++----
 .../messy/tests/LongTermsTests.java           | 15 ++---
 .../messy/tests/StringTermsTests.java         | 14 ++---
 .../bucket/AbstractTermsTestCase.java         |  2 +-
 17 files changed, 122 insertions(+), 242 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
index 68e93f41991..9ba4b2210fa 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
@@ -21,6 +21,7 @@ package org.elasticsearch.search.aggregations.bucket.geogrid;
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.SortedNumericDocValues;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.geo.GeoHashUtils;
 import org.elasticsearch.common.geo.GeoPoint;
@@ -84,9 +85,9 @@ public class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder<Va
     }
 
     public GeoGridAggregationBuilder size(int size) {
-        if (size < -1) {
+        if (size <= 0) {
             throw new IllegalArgumentException(
-                    "[size] must be greater than or equal to 0. Found [" + shardSize + "] in [" + name + "]");
+                    "[size] must be greater than 0. Found [" + size + "] in [" + name + "]");
         }
         this.requiredSize = size;
         return this;
@@ -97,9 +98,9 @@ public class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder<Va
     }
 
     public GeoGridAggregationBuilder shardSize(int shardSize) {
-        if (shardSize < -1) {
+        if (shardSize < -1 || shardSize == 0) {
             throw new IllegalArgumentException(
-                    "[shardSize] must be greater than or equal to 0. Found [" + shardSize + "] in [" + name + "]");
+                    "[shardSize] must be greater than 0. Found [" + shardSize + "] in [" + name + "]");
             }
         this.shardSize = shardSize;
         return this;
@@ -114,20 +115,20 @@ public class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder<Va
             ValuesSourceConfig<ValuesSource.GeoPoint> config, AggregatorFactory<?> parent, Builder subFactoriesBuilder)
                     throws IOException {
         int shardSize = this.shardSize;
-        if (shardSize == 0) {
-            shardSize = Integer.MAX_VALUE;
-        }
 
         int requiredSize = this.requiredSize;
-        if (requiredSize == 0) {
-            requiredSize = Integer.MAX_VALUE;
-        }
 
         if (shardSize < 0) {
-            // Use default heuristic to avoid any wrong-ranking caused by distributed counting
+            // Use default heuristic to avoid any wrong-ranking caused by
+            // distributed counting
             shardSize = BucketUtils.suggestShardSideQueueSize(requiredSize, context.searchContext().numberOfShards());
         }
 
+        if (requiredSize <= 0 || shardSize <= 0) {
+            throw new ElasticsearchException(
+                    "parameters [required_size] and [shard_size] must be >0 in geohash_grid aggregation [" + name + "].");
+        }
+
         if (shardSize < requiredSize) {
             shardSize = requiredSize;
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java
index bd4fe874de7..4b8ec77c467 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java
@@ -83,13 +83,6 @@ public abstract class TermsAggregator extends BucketsAggregator {
 
         public void ensureValidity() {
 
-            if (shardSize == 0) {
-                setShardSize(Integer.MAX_VALUE);
-            }
-
-            if (requiredSize == 0) {
-                setRequiredSize(Integer.MAX_VALUE);
-            }
             // shard_size cannot be smaller than size as we need to at least fetch <size> entries from every shards in order to return <size>
             if (shardSize < requiredSize) {
                 setShardSize(requiredSize);
@@ -100,8 +93,12 @@ public abstract class TermsAggregator extends BucketsAggregator {
                 setShardMinDocCount(minDocCount);
             }
 
-            if (requiredSize < 0 || minDocCount < 0) {
-                throw new ElasticsearchException("parameters [requiredSize] and [minDocCount] must be >=0 in terms aggregation.");
+            if (requiredSize <= 0 || shardSize <= 0) {
+                throw new ElasticsearchException("parameters [required_size] and [shard_size] must be >0 in terms aggregation.");
+            }
+
+            if (minDocCount < 0 || shardMinDocCount < 0) {
+                throw new ElasticsearchException("parameter [min_doc_count] and [shardMinDocCount] must be >=0 in terms aggregation.");
             }
         }
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java
index 6f2d7c8fb93..b22f7524e32 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java
@@ -134,10 +134,9 @@ public class ChildrenIT extends ESIntegTestCase {
         SearchResponse searchResponse = client().prepareSearch("test")
                 .setQuery(matchQuery("randomized", true))
                 .addAggregation(
-                        terms("category").field("category").size(0).subAggregation(
-children("to_comment", "comment")
+                        terms("category").field("category").size(10000).subAggregation(children("to_comment", "comment")
                                 .subAggregation(
-                                        terms("commenters").field("commenter").size(0).subAggregation(
+                                        terms("commenters").field("commenter").size(10000).subAggregation(
                                                 topHits("top_comments")
                                         ))
                         )
@@ -176,7 +175,7 @@ children("to_comment", "comment")
         SearchResponse searchResponse = client().prepareSearch("test")
                 .setQuery(matchQuery("randomized", false))
                 .addAggregation(
-                        terms("category").field("category").size(0).subAggregation(
+                        terms("category").field("category").size(10000).subAggregation(
                         children("to_comment", "comment").subAggregation(topHits("top_comments").sort("_uid", SortOrder.ASC))
                         )
                 ).get();
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java
index 269d22914c1..711d3373590 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java
@@ -21,6 +21,7 @@ package org.elasticsearch.search.aggregations.bucket;
 import com.carrotsearch.hppc.ObjectIntHashMap;
 import com.carrotsearch.hppc.ObjectIntMap;
 import com.carrotsearch.hppc.cursors.ObjectIntCursor;
+
 import org.elasticsearch.Version;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
@@ -52,8 +53,8 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.geohashGrid;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 @ESIntegTestCase.SuiteScopeTestCase
 public class GeoHashGridIT extends ESIntegTestCase {
@@ -305,24 +306,24 @@ public class GeoHashGridIT extends ESIntegTestCase {
         }
     }
 
-    // making sure this doesn't runs into an OOME
     public void testSizeIsZero() {
-        for (int precision = 1; precision <= PRECISION; precision++) {
-            final int size = randomBoolean() ? 0 : randomIntBetween(1, Integer.MAX_VALUE);
-            final int shardSize = randomBoolean() ? -1 : 0;
-            SearchResponse response = client().prepareSearch("idx")
-                    .addAggregation(geohashGrid("geohashgrid")
-                            .field("location")
-                            .size(size)
-                            .shardSize(shardSize)
-                            .precision(precision)
-                    )
-                    .execute().actionGet();
+        final int size = 0;
+        final int shardSize = 10000;
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+                () -> client().prepareSearch("idx")
+                        .addAggregation(geohashGrid("geohashgrid").field("location").size(size).shardSize(shardSize)).execute()
+                        .actionGet());
+        assertThat(exception.getMessage(), containsString("[size] must be greater than 0. Found [0] in [geohashgrid]"));
+    }
 
-            assertSearchResponse(response);
-            GeoHashGrid geoGrid = response.getAggregations().get("geohashgrid");
-            assertThat(geoGrid.getBuckets().size(), greaterThanOrEqualTo(1));
-        }
+    public void testShardSizeIsZero() {
+        final int size = 100;
+        final int shardSize = 0;
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+                () -> client().prepareSearch("idx")
+                        .addAggregation(geohashGrid("geohashgrid").field("location").size(size).shardSize(shardSize))
+                        .execute().actionGet());
+        assertThat(exception.getMessage(), containsString("[shardSize] must be greater than 0. Found [0] in [geohashgrid]"));
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridTests.java
index c3c8f6902b3..688289f02dd 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridTests.java
@@ -33,33 +33,10 @@ public class GeoHashGridTests extends BaseAggregationTestCase<GeoGridAggregation
             factory.precision(precision);
         }
         if (randomBoolean()) {
-            int size = randomInt(5);
-            switch (size) {
-            case 0:
-                break;
-            case 1:
-            case 2:
-            case 3:
-            case 4:
-                size = randomIntBetween(0, Integer.MAX_VALUE);
-                break;
-            }
-            factory.size(size);
-
+            factory.size(randomIntBetween(1, Integer.MAX_VALUE));
         }
         if (randomBoolean()) {
-            int shardSize = randomInt(5);
-            switch (shardSize) {
-            case 0:
-                break;
-            case 1:
-            case 2:
-            case 3:
-            case 4:
-                shardSize = randomIntBetween(0, Integer.MAX_VALUE);
-                break;
-            }
-            factory.shardSize(shardSize);
+            factory.shardSize(randomIntBetween(1, Integer.MAX_VALUE));
         }
         return factory;
     }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java
index 21b4a74fe41..1d74c03048a 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ReverseNestedIT.java
@@ -353,7 +353,7 @@ public class ReverseNestedIT extends ESIntegTestCase {
                                 .subAggregation(
                                         terms("field2").field("nested1.nested2.field2").order(Terms.Order.term(true))
                                                 .collectMode(randomFrom(SubAggCollectionMode.values()))
-                                                .size(0)
+                                                .size(10000)
                                                 .subAggregation(
                                                         reverseNested("nested1_to_field1").path("nested1")
                                                                 .subAggregation(
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsTests.java
index 897125ee2fa..c8434924a38 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsTests.java
@@ -73,37 +73,11 @@ public class SignificantTermsTests extends BaseAggregationTestCase<SignificantTe
             factory.missing("MISSING");
         }
         if (randomBoolean()) {
-            int size = randomInt(4);
-            switch (size) {
-            case 0:
-                break;
-            case 1:
-            case 2:
-            case 3:
-            case 4:
-                size = randomInt();
-                break;
-            default:
-                fail();
-            }
-            factory.bucketCountThresholds().setRequiredSize(size);
+            factory.bucketCountThresholds().setRequiredSize(randomIntBetween(1, Integer.MAX_VALUE));
 
         }
         if (randomBoolean()) {
-            int shardSize = randomInt(4);
-            switch (shardSize) {
-            case 0:
-                break;
-            case 1:
-            case 2:
-            case 3:
-            case 4:
-                shardSize = randomInt();
-                break;
-            default:
-                fail();
-            }
-            factory.bucketCountThresholds().setShardSize(shardSize);
+            factory.bucketCountThresholds().setShardSize(randomIntBetween(1, Integer.MAX_VALUE));
         }
         if (randomBoolean()) {
             int minDocCount = randomInt(4);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java
index e112e6d44b4..6bbe1dc22b2 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java
@@ -202,8 +202,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
                         .executionHint(randomExecutionHint())
                         .field(STRING_FIELD_NAME)
                         .showTermDocCountError(true)
-                        .size(0)
-                        .shardSize(0)
+                        .size(10000).shardSize(10000)
                         .collectMode(randomFrom(SubAggCollectionMode.values())))
                 .execute().actionGet();
 
@@ -232,8 +231,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
                         .executionHint(randomExecutionHint())
                         .field(STRING_FIELD_NAME)
                         .showTermDocCountError(true)
-                        .size(0)
-                        .shardSize(0)
+                        .size(10000).shardSize(10000)
                         .collectMode(randomFrom(SubAggCollectionMode.values())))
                 .execute().actionGet();
 
@@ -281,8 +279,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
                         .executionHint(randomExecutionHint())
                         .field(STRING_FIELD_NAME)
                         .showTermDocCountError(true)
-                        .size(0)
-                        .shardSize(0)
+                        .size(10000).shardSize(10000)
                         .order(Order.count(true))
                         .collectMode(randomFrom(SubAggCollectionMode.values())))
                 .execute().actionGet();
@@ -313,8 +310,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
                         .executionHint(randomExecutionHint())
                         .field(STRING_FIELD_NAME)
                         .showTermDocCountError(true)
-                        .size(0)
-                        .shardSize(0)
+                        .size(10000).shardSize(10000)
                         .order(Order.term(true))
                         .collectMode(randomFrom(SubAggCollectionMode.values())))
                 .execute().actionGet();
@@ -345,8 +341,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
                         .executionHint(randomExecutionHint())
                         .field(STRING_FIELD_NAME)
                         .showTermDocCountError(true)
-                        .size(0)
-                        .shardSize(0)
+                        .size(10000).shardSize(10000)
                         .order(Order.term(false))
                         .collectMode(randomFrom(SubAggCollectionMode.values())))
                 .execute().actionGet();
@@ -377,8 +372,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
                         .executionHint(randomExecutionHint())
                         .field(STRING_FIELD_NAME)
                         .showTermDocCountError(true)
-                        .size(0)
-                        .shardSize(0)
+                        .size(10000).shardSize(10000)
                         .order(Order.aggregation("sortAgg", true))
                         .collectMode(randomFrom(SubAggCollectionMode.values()))
                         .subAggregation(sum("sortAgg").field(LONG_FIELD_NAME)))
@@ -411,8 +405,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
                         .executionHint(randomExecutionHint())
                         .field(STRING_FIELD_NAME)
                         .showTermDocCountError(true)
-                        .size(0)
-                        .shardSize(0)
+                        .size(10000).shardSize(10000)
                         .order(Order.aggregation("sortAgg", false))
                         .collectMode(randomFrom(SubAggCollectionMode.values()))
                         .subAggregation(sum("sortAgg").field(LONG_FIELD_NAME)))
@@ -445,8 +438,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
                         .executionHint(randomExecutionHint())
                         .field(LONG_FIELD_NAME)
                         .showTermDocCountError(true)
-                        .size(0)
-                        .shardSize(0)
+                        .size(10000).shardSize(10000)
                         .collectMode(randomFrom(SubAggCollectionMode.values())))
                 .execute().actionGet();
 
@@ -475,8 +467,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
                         .executionHint(randomExecutionHint())
                         .field(LONG_FIELD_NAME)
                         .showTermDocCountError(true)
-                        .size(0)
-                        .shardSize(0)
+                        .size(10000).shardSize(10000)
                         .collectMode(randomFrom(SubAggCollectionMode.values())))
                 .execute().actionGet();
 
@@ -524,8 +515,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
                         .executionHint(randomExecutionHint())
                         .field(LONG_FIELD_NAME)
                         .showTermDocCountError(true)
-                        .size(0)
-                        .shardSize(0)
+                        .size(10000).shardSize(10000)
                         .order(Order.count(true))
                         .collectMode(randomFrom(SubAggCollectionMode.values())))
                 .execute().actionGet();
@@ -556,8 +546,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
                         .executionHint(randomExecutionHint())
                         .field(LONG_FIELD_NAME)
                         .showTermDocCountError(true)
-                        .size(0)
-                        .shardSize(0)
+                        .size(10000).shardSize(10000)
                         .order(Order.term(true))
                         .collectMode(randomFrom(SubAggCollectionMode.values())))
                 .execute().actionGet();
@@ -588,8 +577,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
                         .executionHint(randomExecutionHint())
                         .field(LONG_FIELD_NAME)
                         .showTermDocCountError(true)
-                        .size(0)
-                        .shardSize(0)
+                        .size(10000).shardSize(10000)
                         .order(Order.term(false))
                         .collectMode(randomFrom(SubAggCollectionMode.values())))
                 .execute().actionGet();
@@ -620,8 +608,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
                         .executionHint(randomExecutionHint())
                         .field(LONG_FIELD_NAME)
                         .showTermDocCountError(true)
-                        .size(0)
-                        .shardSize(0)
+                        .size(10000).shardSize(10000)
                         .order(Order.aggregation("sortAgg", true))
                         .collectMode(randomFrom(SubAggCollectionMode.values()))
                         .subAggregation(sum("sortAgg").field(LONG_FIELD_NAME)))
@@ -654,8 +641,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
                         .executionHint(randomExecutionHint())
                         .field(LONG_FIELD_NAME)
                         .showTermDocCountError(true)
-                        .size(0)
-                        .shardSize(0)
+                        .size(10000).shardSize(10000)
                         .order(Order.aggregation("sortAgg", false))
                         .collectMode(randomFrom(SubAggCollectionMode.values()))
                         .subAggregation(sum("sortAgg").field(DOUBLE_FIELD_NAME)))
@@ -688,8 +674,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
                         .executionHint(randomExecutionHint())
                         .field(DOUBLE_FIELD_NAME)
                         .showTermDocCountError(true)
-                        .size(0)
-                        .shardSize(0)
+                        .size(10000).shardSize(10000)
                         .collectMode(randomFrom(SubAggCollectionMode.values())))
                 .execute().actionGet();
 
@@ -718,8 +703,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
                         .executionHint(randomExecutionHint())
                         .field(DOUBLE_FIELD_NAME)
                         .showTermDocCountError(true)
-                        .size(0)
-                        .shardSize(0)
+                        .size(10000).shardSize(10000)
                         .collectMode(randomFrom(SubAggCollectionMode.values())))
                 .execute().actionGet();
 
@@ -767,8 +751,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
                         .executionHint(randomExecutionHint())
                         .field(DOUBLE_FIELD_NAME)
                         .showTermDocCountError(true)
-                        .size(0)
-                        .shardSize(0)
+                        .size(10000).shardSize(10000)
                         .order(Order.count(true))
                         .collectMode(randomFrom(SubAggCollectionMode.values())))
                 .execute().actionGet();
@@ -799,8 +782,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
                         .executionHint(randomExecutionHint())
                         .field(DOUBLE_FIELD_NAME)
                         .showTermDocCountError(true)
-                        .size(0)
-                        .shardSize(0)
+                        .size(10000).shardSize(10000)
                         .order(Order.term(true))
                         .collectMode(randomFrom(SubAggCollectionMode.values())))
                 .execute().actionGet();
@@ -831,8 +813,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
                         .executionHint(randomExecutionHint())
                         .field(DOUBLE_FIELD_NAME)
                         .showTermDocCountError(true)
-                        .size(0)
-                        .shardSize(0)
+                        .size(10000).shardSize(10000)
                         .order(Order.term(false))
                         .collectMode(randomFrom(SubAggCollectionMode.values())))
                 .execute().actionGet();
@@ -863,8 +844,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
                         .executionHint(randomExecutionHint())
                         .field(DOUBLE_FIELD_NAME)
                         .showTermDocCountError(true)
-                        .size(0)
-                        .shardSize(0)
+                        .size(10000).shardSize(10000)
                         .order(Order.aggregation("sortAgg", true))
                         .collectMode(randomFrom(SubAggCollectionMode.values()))
                         .subAggregation(sum("sortAgg").field(LONG_FIELD_NAME)))
@@ -897,8 +877,7 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
                         .executionHint(randomExecutionHint())
                         .field(DOUBLE_FIELD_NAME)
                         .showTermDocCountError(true)
-                        .size(0)
-                        .shardSize(0)
+                        .size(10000).shardSize(10000)
                         .order(Order.aggregation("sortAgg", false))
                         .collectMode(randomFrom(SubAggCollectionMode.values()))
                         .subAggregation(sum("sortAgg").field(LONG_FIELD_NAME)))
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsTests.java
index d8600758f29..c8a14e82bf4 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsTests.java
@@ -69,37 +69,10 @@ public class TermsTests extends BaseAggregationTestCase<TermsAggregationBuilder>
             factory.missing("MISSING");
         }
         if (randomBoolean()) {
-            int size = randomInt(4);
-            switch (size) {
-            case 0:
-                break;
-            case 1:
-            case 2:
-            case 3:
-            case 4:
-                size = randomInt();
-                break;
-            default:
-                fail();
-            }
-            factory.bucketCountThresholds().setRequiredSize(size);
-
+            factory.bucketCountThresholds().setRequiredSize(randomIntBetween(1, Integer.MAX_VALUE));
         }
         if (randomBoolean()) {
-            int shardSize = randomInt(4);
-            switch (shardSize) {
-            case 0:
-                break;
-            case 1:
-            case 2:
-            case 3:
-            case 4:
-                shardSize = randomInt();
-                break;
-            default:
-                fail();
-            }
-            factory.bucketCountThresholds().setShardSize(shardSize);
+            factory.bucketCountThresholds().setShardSize(randomIntBetween(1, Integer.MAX_VALUE));
         }
         if (randomBoolean()) {
             int minDocCount = randomInt(4);
diff --git a/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc b/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc
index b550b240b75..9963b48fb04 100644
--- a/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/geohashgrid-aggregation.asciidoc
@@ -117,15 +117,9 @@ precision::     Optional. The string length of the geohashes used to define
 size::          Optional. The maximum number of geohash buckets to return
                 (defaults to 10,000). When results are trimmed, buckets are
                 prioritised based on the volumes of documents they contain.
-                A value of `0` will return all buckets that
-                contain a hit, use with caution as this could use a lot of CPU
-                and network bandwidth if there are many buckets.
 
 shard_size::    Optional. To allow for more accurate counting of the top cells
                 returned in the final result the aggregation defaults to
                 returning `max(10,(size x number-of-shards))` buckets from each
                 shard. If this heuristic is undesirable, the number considered
                 from each shard can be over-ridden using this parameter.
-                A value of `0` makes the shard size unlimited.
-
-
diff --git a/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc b/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc
index 80c747e61a5..61deffeccd2 100644
--- a/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/significantterms-aggregation.asciidoc
@@ -224,12 +224,12 @@ are presented unstemmed, highlighted, with the right case, in the right order an
 ==== Custom background sets
 
 Ordinarily, the foreground set of documents is "diffed" against a background set of all the documents in your index.
-However, sometimes it may prove useful to use a narrower background set as the basis for comparisons. 
-For example, a query on documents relating to "Madrid" in an index with content from all over the world might reveal that "Spanish" 
-was a significant term. This may be true but if you want some more focused terms you could use a `background_filter` 
-on the term 'spain' to establish a narrower set of documents as context. With this as a background "Spanish" would now 
-be seen as commonplace and therefore not as significant as words like "capital" that relate more strongly with Madrid.  
-Note that using a background filter will slow things down - each term's background frequency must now be derived on-the-fly from filtering posting lists rather than reading the index's pre-computed count for a term.  
+However, sometimes it may prove useful to use a narrower background set as the basis for comparisons.
+For example, a query on documents relating to "Madrid" in an index with content from all over the world might reveal that "Spanish"
+was a significant term. This may be true but if you want some more focused terms you could use a `background_filter`
+on the term 'spain' to establish a narrower set of documents as context. With this as a background "Spanish" would now
+be seen as commonplace and therefore not as significant as words like "capital" that relate more strongly with Madrid.
+Note that using a background filter will slow things down - each term's background frequency must now be derived on-the-fly from filtering posting lists rather than reading the index's pre-computed count for a term.
 
 ==== Limitations
 
@@ -274,7 +274,7 @@ The scores are derived from the doc frequencies in _foreground_ and _background_
 
 ===== mutual information
 Mutual information as described in "Information Retrieval", Manning et al., Chapter 13.5.1 can be used as significance score by adding the parameter
-	
+
 [source,js]
 --------------------------------------------------
 
@@ -283,9 +283,9 @@ Mutual information as described in "Information Retrieval", Manning et al., Chap
 	 }
 --------------------------------------------------
 
-Mutual information does not differentiate between terms that are descriptive for the subset or for documents outside the subset. The significant terms therefore can contain terms that appear more or less frequent in the subset than outside the subset. To filter out the terms that appear less often in the subset than in documents outside the subset, `include_negatives` can be set to `false`. 
+Mutual information does not differentiate between terms that are descriptive for the subset or for documents outside the subset. The significant terms therefore can contain terms that appear more or less frequent in the subset than outside the subset. To filter out the terms that appear less often in the subset than in documents outside the subset, `include_negatives` can be set to `false`.
 
-Per default, the assumption is that the documents in the bucket are also contained in the background. If instead you defined a custom background filter that represents a different set of documents that you want to compare to, set 
+Per default, the assumption is that the documents in the bucket are also contained in the background. If instead you defined a custom background filter that represents a different set of documents that you want to compare to, set
 
 [source,js]
 --------------------------------------------------
@@ -296,7 +296,7 @@ Per default, the assumption is that the documents in the bucket are also contain
 
 ===== Chi square
 Chi square as described in "Information Retrieval", Manning et al., Chapter 13.5.2 can be used as significance score by adding the parameter
-	
+
 [source,js]
 --------------------------------------------------
 
@@ -309,7 +309,7 @@ Chi square behaves like mutual information and can be configured with the same p
 
 ===== google normalized distance
 Google normalized distance  as described in "The Google Similarity Distance", Cilibrasi and Vitanyi, 2007 (http://arxiv.org/pdf/cs/0412098v3.pdf) can be used as significance score by adding the parameter
-	
+
 [source,js]
 --------------------------------------------------
 
@@ -317,7 +317,7 @@ Google normalized distance  as described in "The Google Similarity Distance", Ci
 	 }
 --------------------------------------------------
 
-`gnd` also accepts the `background_is_superset` parameter. 
+`gnd` also accepts the `background_is_superset` parameter.
 
 
 ===== Percentage
@@ -328,7 +328,7 @@ The benefit of this heuristic is that the scoring logic is simple to explain to
 
 It would be hard for a seasoned boxer to win a championship if the prize was awarded purely on the basis of percentage of fights won - by these rules a newcomer with only one fight under his belt would be impossible to beat.
 Multiple observations are typically required to reinforce a view so it is recommended in these cases to set both `min_doc_count` and `shard_min_doc_count` to a higher value such as 10 in order to filter out the low-frequency terms that otherwise take precedence.
-	
+
 [source,js]
 --------------------------------------------------
 
@@ -348,7 +348,7 @@ If none of the above measures suits your usecase than another option is to imple
 
 ===== scripted
 Customized scores can be implemented via a script:
-	
+
 [source,js]
 --------------------------------------------------
 
@@ -357,7 +357,7 @@ Customized scores can be implemented via a script:
             }
 --------------------------------------------------
 
-Scripts can be inline (as in above example), indexed or stored on disk. For details on the options, see <<modules-scripting, script documentation>>. 
+Scripts can be inline (as in above example), indexed or stored on disk. For details on the options, see <<modules-scripting, script documentation>>.
 
 Available parameters in the script are
 
@@ -374,9 +374,7 @@ default, the node coordinating the search process will request each shard to pro
 and once all shards respond, it will reduce the results to the final list that will then be returned to the client.
 If the number of unique terms is greater than `size`, the returned list can be slightly off and not accurate
 (it could be that the term counts are slightly off and it could even be that a term that should have been in the top
-size buckets was not returned). 
-
-If set to `0`, the `size` will be set to `Integer.MAX_VALUE`.
+size buckets was not returned).
 
 To ensure better accuracy a multiple of the final `size` is used as the number of terms to request from each shard
 using a heuristic based on the number of shards. To take manual control of this setting the `shard_size` parameter
@@ -386,12 +384,9 @@ Low-frequency terms can turn out to be the most interesting ones once all result
 significant_terms aggregation can produce higher-quality results when the `shard_size` parameter is set to
 values significantly higher than the `size` setting. This ensures that a bigger volume of promising candidate terms are given
 a consolidated review by the reducing node before the final selection. Obviously large candidate term lists
-will cause extra network traffic and RAM usage so this is  quality/cost trade off that needs to be balanced.  If `shard_size` is set to -1 (the default) then `shard_size` will be automatically estimated based on the number of shards and the `size` parameter. 
+will cause extra network traffic and RAM usage so this is  quality/cost trade off that needs to be balanced.  If `shard_size` is set to -1 (the default) then `shard_size` will be automatically estimated based on the number of shards and the `size` parameter.
 
 
-If set to `0`, the `shard_size` will be set to `Integer.MAX_VALUE`.
- 
-
 NOTE:   `shard_size` cannot be smaller than `size` (as it doesn't make much sense). When it is, elasticsearch will
         override it and reset it to be equal to `size`.
 
@@ -439,7 +434,7 @@ WARNING: Setting `min_doc_count` to `1` is generally not advised as it tends to
 
 The default source of statistical information for background term frequencies is the entire index and this
 scope can be narrowed through the use of a `background_filter` to focus in on significant terms within a narrower
-context: 
+context:
 
 [source,js]
 --------------------------------------------------
@@ -449,7 +444,7 @@ context:
     },
     "aggs" : {
         "tags" : {
-            "significant_terms" : { 
+            "significant_terms" : {
                 "field" : "tag",
                 "background_filter": {
                 	"term" : { "text" : "spain"}
@@ -460,9 +455,9 @@ context:
 }
 --------------------------------------------------
 
-The above filter would help focus in on terms that were peculiar to the city of Madrid rather than revealing 
-terms like "Spanish" that are unusual in the full index's worldwide context but commonplace in the subset of documents containing the 
-word "Spain".   
+The above filter would help focus in on terms that were peculiar to the city of Madrid rather than revealing
+terms like "Spanish" that are unusual in the full index's worldwide context but commonplace in the subset of documents containing the
+word "Spain".
 
 WARNING: Use of background filters will slow the query as each term's postings must be filtered to determine a frequency
 
@@ -482,7 +477,7 @@ There are different mechanisms by which terms aggregations can be executed:
  - by using field values directly in order to aggregate data per-bucket (`map`)
  - by using ordinals of the field and preemptively allocating one bucket per ordinal value (`global_ordinals`)
  - by using ordinals of the field and dynamically allocating one bucket per ordinal value (`global_ordinals_hash`)
- 
+
 Elasticsearch tries to have sensible defaults so this is something that generally doesn't need to be configured.
 
 `map` should only be considered when very few documents match a query. Otherwise the ordinals-based execution modes
@@ -514,4 +509,3 @@ in inner aggregations.
 <1> the possible values are `map`, `global_ordinals` and `global_ordinals_hash`
 
 Please note that Elasticsearch will ignore this execution hint if it is not applicable.
-
diff --git a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
index 1d9a65a7428..1c88f0a06d8 100644
--- a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
@@ -56,7 +56,7 @@ default, the node coordinating the search process will request each shard to pro
 and once all shards respond, it will reduce the results to the final list that will then be returned to the client.
 This means that if the number of unique terms is greater than `size`, the returned list is slightly off and not accurate
 (it could be that the term counts are slightly off and it could even be that a term that should have been in the top
-size buckets was not returned). If set to `0`, the `size` will be set to `Integer.MAX_VALUE`.
+size buckets was not returned).
 
 [[search-aggregations-bucket-terms-aggregation-approximate-counts]]
 ==== Document counts are approximate
@@ -149,14 +149,12 @@ The `shard_size` parameter can be  used to minimize the extra work that comes wi
 it will determine how many terms the coordinating node will request from each shard. Once all the shards responded, the
 coordinating node will then reduce them to a final result which will be based on the `size` parameter - this way,
 one can increase the accuracy of the returned terms and avoid the overhead of streaming a big list of buckets back to
-the client. If set to `0`, the `shard_size` will be set to `Integer.MAX_VALUE`.
+the client.
 
 
 NOTE:   `shard_size` cannot be smaller than `size` (as it doesn't make much sense). When it is, elasticsearch will
         override it and reset it to be equal to `size`.
 
-It is possible to not limit the number of terms that are returned by setting `size` to `0`. Don't use this
-on high-cardinality fields as this will kill both your CPU since terms need to be return sorted, and your network.
 
 The default `shard_size` is a multiple of the `size` parameter which is dependant on the number of shards.
 
@@ -443,7 +441,7 @@ Generating the terms using a script:
     "aggs" : {
         "genders" : {
             "terms" : {
-                "script" : { 
+                "script" : {
                     "inline": "doc['gender'].value"
                     "lang": "painless"
                 }
@@ -485,9 +483,9 @@ TIP: for indexed scripts replace the `file` parameter with an `id` parameter.
         "genders" : {
             "terms" : {
                 "field" : "gender",
-                "script" : { 
+                "script" : {
                     "inline" : "'Gender: ' +_value"
-                    "lang" : "painless" 
+                    "lang" : "painless"
                 }
             }
         }
@@ -710,4 +708,4 @@ had a value.
 }
 --------------------------------------------------
 
-<1> Documents without a value in the `tags` field will fall into the same bucket as documents that have the value `N/A`. 
+<1> Documents without a value in the `tags` field will fall into the same bucket as documents that have the value `N/A`.
diff --git a/docs/reference/migration/migrate_5_0/aggregations.asciidoc b/docs/reference/migration/migrate_5_0/aggregations.asciidoc
index d90e429732b..d9227e91385 100644
--- a/docs/reference/migration/migrate_5_0/aggregations.asciidoc
+++ b/docs/reference/migration/migrate_5_0/aggregations.asciidoc
@@ -20,3 +20,9 @@ Now that Elasticsearch supports `ipv6`, `ip` addresses are encoded in the index
 using a binary representation rather than a numeric representation. As a
 consequence, the output of `ip_range` aggregations does not give numeric values
 for `from` and `to` anymore.
+
+==== `size: 0` on Terms, Significant Terms and Geohash Grid Aggregations
+
+`size: 0` is no longer valid for the terms, significant terms and geohash grid
+aggregations. Instead a size should be explicitly specified with a number greater
+than zero.
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java
index ced13b8e53a..2012a232b2b 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java
@@ -63,6 +63,7 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.core.IsNull.notNullValue;
@@ -235,20 +236,13 @@ public class DoubleTermsTests extends AbstractTermsTestCase {
 
     // the main purpose of this test is to make sure we're not allocating 2GB of memory per shard
     public void testSizeIsZero() {
-        SearchResponse response = client().prepareSearch("idx").setTypes("high_card_type")
-                .addAggregation(terms("terms")
-                        .field(SINGLE_VALUED_FIELD_NAME)
-                        .minDocCount(randomInt(1))
-                        .size(0)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())))
-                .execute().actionGet();
-
-        assertSearchResponse(response);
-
-        Terms terms = response.getAggregations().get("terms");
-        assertThat(terms, notNullValue());
-        assertThat(terms.getName(), equalTo("terms"));
-        assertThat(terms.getBuckets().size(), equalTo(100));
+        ElasticsearchException exception = expectThrows(ElasticsearchException.class,
+                () -> client()
+                        .prepareSearch("idx").setTypes("high_card_type").addAggregation(terms("terms").field(SINGLE_VALUED_FIELD_NAME)
+                                .minDocCount(randomInt(1)).size(0).collectMode(randomFrom(SubAggCollectionMode.values())))
+                        .execute().actionGet());
+        assertThat(exception.getDetailedMessage(),
+                containsString("parameters [required_size] and [shard_size] must be >0 in terms aggregation."));
     }
 
     public void testSingleValueField() throws Exception {
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java
index 65f288113ab..dc3fea592a2 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java
@@ -61,6 +61,7 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.core.IsNull.notNullValue;
@@ -237,20 +238,16 @@ public class LongTermsTests extends AbstractTermsTestCase {
 
     // the main purpose of this test is to make sure we're not allocating 2GB of memory per shard
     public void testSizeIsZero() {
-        SearchResponse response = client().prepareSearch("idx").setTypes("high_card_type")
+        ElasticsearchException exception = expectThrows(ElasticsearchException.class,
+                () -> client().prepareSearch("idx").setTypes("high_card_type")
                 .addAggregation(terms("terms")
                         .field(SINGLE_VALUED_FIELD_NAME)
                         .collectMode(randomFrom(SubAggCollectionMode.values()))
                         .minDocCount(randomInt(1))
                         .size(0))
-                .execute().actionGet();
-
-        assertSearchResponse(response);
-
-        Terms terms = response.getAggregations().get("terms");
-        assertThat(terms, notNullValue());
-        assertThat(terms.getName(), equalTo("terms"));
-        assertThat(terms.getBuckets().size(), equalTo(100));
+                        .execute().actionGet());
+        assertThat(exception.getDetailedMessage(),
+                containsString("parameters [required_size] and [shard_size] must be >0 in terms aggregation."));
     }
 
     public void testSingleValueField() throws Exception {
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java
index deeeb2911b6..aa31ae2a118 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java
@@ -70,6 +70,7 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.core.IsNull.notNullValue;
@@ -202,20 +203,15 @@ public class StringTermsTests extends AbstractTermsTestCase {
     // the main purpose of this test is to make sure we're not allocating 2GB of memory per shard
     public void testSizeIsZero() {
         final int minDocCount = randomInt(1);
-        SearchResponse response = client()
+        ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> client()
                 .prepareSearch("idx")
                 .setTypes("high_card_type")
                 .addAggregation(
                         terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
                                 .collectMode(randomFrom(SubAggCollectionMode.values())).minDocCount(minDocCount).size(0)).execute()
-                .actionGet();
-
-        assertSearchResponse(response);
-
-        Terms terms = response.getAggregations().get("terms");
-        assertThat(terms, notNullValue());
-        assertThat(terms.getName(), equalTo("terms"));
-        assertThat(terms.getBuckets().size(), equalTo(minDocCount == 0 ? 105 : 100)); // 105 because of the other type
+                .actionGet());
+        assertThat(exception.getDetailedMessage(),
+                containsString("parameters [required_size] and [shard_size] must be >0 in terms aggregation."));
     }
 
     public void testSingleValueField() throws Exception {
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/AbstractTermsTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/AbstractTermsTestCase.java
index 4d5ff10f9cd..e1d6ab6dfaa 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/AbstractTermsTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/AbstractTermsTestCase.java
@@ -48,7 +48,7 @@ public abstract class AbstractTermsTestCase extends ESIntegTestCase {
                     .addAggregation(terms("terms")
                             .executionHint(randomExecutionHint())
                             .field(fieldName)
-                            .size(0)
+                            .size(10000)
                             .collectMode(randomFrom(SubAggCollectionMode.values())))
                     .get();
             assertSearchResponse(allTerms);

From cf93e904e32966e860d94d7e9a3192f72b764131 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Tue, 14 Jun 2016 14:14:03 +0200
Subject: [PATCH 0827/1311] remove message parameter from RequestLogger methods

This prevents useless string allocation.
---
 .../java/org/elasticsearch/client/RequestLogger.java  | 11 +++++------
 .../java/org/elasticsearch/client/RestClient.java     |  6 +++---
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/RequestLogger.java b/client/src/main/java/org/elasticsearch/client/RequestLogger.java
index 7268a449981..ad8204be2bb 100644
--- a/client/src/main/java/org/elasticsearch/client/RequestLogger.java
+++ b/client/src/main/java/org/elasticsearch/client/RequestLogger.java
@@ -53,12 +53,11 @@ final class RequestLogger {
     /**
      * Logs a request that yielded a response
      */
-    static void logResponse(Log logger, String message, HttpUriRequest request, HttpHost host, HttpResponse httpResponse) {
+    static void logResponse(Log logger, HttpUriRequest request, HttpHost host, HttpResponse httpResponse) {
         if (logger.isDebugEnabled()) {
-            logger.debug(message + " [" + request.getMethod() + " " + host + request.getRequestLine().getUri() +
-                    "] [" + httpResponse.getStatusLine() + "]");
+            logger.debug("request [" + request.getMethod() + " " + host + request.getRequestLine().getUri() +
+                    "] returned [" + httpResponse.getStatusLine() + "]");
         }
-
         if (tracer.isTraceEnabled()) {
             String requestLine;
             try {
@@ -81,8 +80,8 @@ final class RequestLogger {
     /**
      * Logs a request that failed
      */
-    static void logFailedRequest(Log logger, String message, HttpUriRequest request, HttpHost host, IOException e) {
-        logger.debug(message + " [" + request.getMethod() + " " + host + request.getRequestLine().getUri() + "]", e);
+    static void logFailedRequest(Log logger, HttpUriRequest request, HttpHost host, IOException e) {
+        logger.debug("request [" + request.getMethod() + " " + host + request.getRequestLine().getUri() + "] failed", e);
         if (logger.isTraceEnabled()) {
             String traceRequest;
             try {
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index b2bd6030681..2d1c50af3cb 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -160,7 +160,7 @@ public final class RestClient implements Closeable {
             try {
                 httpResponse = client.execute(host, request);
             } catch(IOException e) {
-                RequestLogger.logFailedRequest(logger, "request failed", request, host, e);
+                RequestLogger.logFailedRequest(logger, request, host, e);
                 onFailure(host);
                 lastSeenException = addSuppressedException(lastSeenException, e);
                 continue;
@@ -168,11 +168,11 @@ public final class RestClient implements Closeable {
             Response response = new Response(request.getRequestLine(), host, httpResponse);
             int statusCode = response.getStatusLine().getStatusCode();
             if (statusCode < 300 || (request.getMethod().equals(HttpHead.METHOD_NAME) && statusCode == 404) ) {
-                RequestLogger.logResponse(logger, "request succeeded", request, host, httpResponse);
+                RequestLogger.logResponse(logger, request, host, httpResponse);
                 onSuccess(host);
                 return response;
             }
-            RequestLogger.logResponse(logger, "request failed", request, host, httpResponse);
+            RequestLogger.logResponse(logger, request, host, httpResponse);
             String responseBody;
             try {
                 if (response.getEntity() == null) {

From 5a9303dec2c3a1ba9678b22a926bc50e1bdd1b38 Mon Sep 17 00:00:00 2001
From: Itamar Syn-Hershko <itamar@code972.com>
Date: Tue, 14 Jun 2016 15:22:19 +0300
Subject: [PATCH 0828/1311] Fixing typos (#18851)

---
 docs/reference/search/profile.asciidoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc
index 48b9de5ce60..25820d04800 100644
--- a/docs/reference/search/profile.asciidoc
+++ b/docs/reference/search/profile.asciidoc
@@ -324,8 +324,8 @@ Looking at the previous example:
 --------------------------------------------------
 
 We see a single collector named `SimpleTopScoreDocCollector`.  This is the default "scoring and sorting" Collector
-used by Elasticsearch.  The `"reason"` field attempts to give an plain english description of the class name.  The
-`"time` is similar to the time in the Query tree: a wall-clock time inclusive of all children.  Similarly, `children` lists
+used by Elasticsearch.  The `"reason"` field attempts to give a plain english description of the class name.  The
+`"time"` is similar to the time in the Query tree: a wall-clock time inclusive of all children.  Similarly, `children` lists
 all sub-collectors.
 
 It should be noted that Collector times are **independent** from the Query times.  They are calculated, combined

From d5f71f9e85fdf864e8fa063232566a3b04e507dc Mon Sep 17 00:00:00 2001
From: Vladimir Kovpak <cn007b@gmail.com>
Date: Tue, 14 Jun 2016 15:32:45 +0300
Subject: [PATCH 0829/1311] Updated from parameter description. (#18852)

Not sure that my description better but origin description looks very weird,
and i try to make emphasize to offset...
---
 docs/reference/search/request-body.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/search/request-body.asciidoc b/docs/reference/search/request-body.asciidoc
index 8207c6577fe..958320ea110 100644
--- a/docs/reference/search/request-body.asciidoc
+++ b/docs/reference/search/request-body.asciidoc
@@ -55,7 +55,7 @@ And here is a sample response:
 
 `from`::
 
-    The starting from index of the hits to return. Defaults to `0`.
+    To retrieve hits from a certain offset. Defaults to `0`.
 
 `size`::
 

From f5836951f8f44e9b70eb66a85e18256cc0c802cc Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Tue, 14 Jun 2016 14:34:11 +0200
Subject: [PATCH 0830/1311] Fixed link in PR template to CONTRIBUTING

---
 .github/PULL_REQUEST_TEMPLATE.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index ae61fd9f8aa..92b35e97baa 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -7,7 +7,7 @@ attention.
 -->
 
 - Have you signed the [contributor license agreement](https://www.elastic.co/contributor-agreement)?
-- Have you followed the [contributor guidelines](https://github.com/elastic/elasticsearch/blob/master/.github/CONTRIBUTING.md)?
+- Have you followed the [contributor guidelines](https://github.com/elastic/elasticsearch/blob/master/CONTRIBUTING.md)?
 - If submitting code, have you built your formula locally prior to submission with `gradle check`?
 - If submitting code, is your pull request against master? Unless there is a good reason otherwise, we prefer pull requests against master and will backport as needed.
 - If submitting code, have you checked that your submission is for an [OS that we support](https://www.elastic.co/support/matrix#show_os)?

From bec621d46f3fe9eb1be172d3e60738c4645dbc24 Mon Sep 17 00:00:00 2001
From: Colin Goodheart-Smithe <colings86@users.noreply.github.com>
Date: Tue, 14 Jun 2016 13:39:20 +0100
Subject: [PATCH 0831/1311] changes from review

---
 .../bucket/geogrid/GeoGridAggregationBuilder.java         | 6 ++++--
 .../significant/SignificantTermsAggregationBuilder.java   | 8 ++++----
 .../bucket/terms/TermsAggregationBuilder.java             | 8 ++++----
 .../org/elasticsearch/messy/tests/DoubleTermsTests.java   | 7 +++----
 .../org/elasticsearch/messy/tests/LongTermsTests.java     | 7 +++----
 .../org/elasticsearch/messy/tests/StringTermsTests.java   | 7 +++----
 6 files changed, 21 insertions(+), 22 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
index 9ba4b2210fa..4f00a49a1d9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java
@@ -98,7 +98,7 @@ public class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder<Va
     }
 
     public GeoGridAggregationBuilder shardSize(int shardSize) {
-        if (shardSize < -1 || shardSize == 0) {
+        if (shardSize <= 0) {
             throw new IllegalArgumentException(
                     "[shardSize] must be greater than 0. Found [" + shardSize + "] in [" + name + "]");
             }
@@ -140,7 +140,9 @@ public class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder<Va
     protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         builder.field(GeoHashGridParams.FIELD_PRECISION.getPreferredName(), precision);
         builder.field(GeoHashGridParams.FIELD_SIZE.getPreferredName(), requiredSize);
-        builder.field(GeoHashGridParams.FIELD_SHARD_SIZE.getPreferredName(), shardSize);
+        if (shardSize > -1) {
+            builder.field(GeoHashGridParams.FIELD_SHARD_SIZE.getPreferredName(), shardSize);
+        }
         return builder;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java
index 64a9c607729..e3886bee563 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java
@@ -113,8 +113,8 @@ public class SignificantTermsAggregationBuilder extends ValuesSourceAggregationB
      * (defaults to 10)
      */
     public SignificantTermsAggregationBuilder size(int size) {
-        if (size < 0) {
-            throw new IllegalArgumentException("[size] must be greater than or equal to 0. Found [" + size + "] in [" + name + "]");
+        if (size <= 0) {
+            throw new IllegalArgumentException("[size] must be greater than 0. Found [" + size + "] in [" + name + "]");
         }
         bucketCountThresholds.setRequiredSize(size);
         return this;
@@ -127,9 +127,9 @@ public class SignificantTermsAggregationBuilder extends ValuesSourceAggregationB
      * results are.
      */
     public SignificantTermsAggregationBuilder shardSize(int shardSize) {
-        if (shardSize < 0) {
+        if (shardSize <= 0) {
             throw new IllegalArgumentException(
-                    "[shardSize] must be greater than or equal to 0. Found [" + shardSize + "] in [" + name + "]");
+                    "[shardSize] must be greater than  0. Found [" + shardSize + "] in [" + name + "]");
         }
         bucketCountThresholds.setShardSize(shardSize);
         return this;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
index f4cb133c499..1dd59edf6d9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
@@ -110,8 +110,8 @@ public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder<Valu
      * (defaults to 10)
      */
     public TermsAggregationBuilder size(int size) {
-        if (size < 0) {
-            throw new IllegalArgumentException("[size] must be greater than or equal to 0. Found [" + size + "] in [" + name + "]");
+        if (size <= 0) {
+            throw new IllegalArgumentException("[size] must be greater than 0. Found [" + size + "] in [" + name + "]");
         }
         bucketCountThresholds.setRequiredSize(size);
         return this;
@@ -124,9 +124,9 @@ public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder<Valu
      * results are.
      */
     public TermsAggregationBuilder shardSize(int shardSize) {
-        if (shardSize < 0) {
+        if (shardSize <= 0) {
             throw new IllegalArgumentException(
-                    "[shardSize] must be greater than or equal to 0. Found [" + shardSize + "] in [" + name + "]");
+                    "[shardSize] must be greater than 0. Found [" + shardSize + "] in [" + name + "]");
         }
         bucketCountThresholds.setShardSize(shardSize);
         return this;
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java
index 2012a232b2b..6ed77fffcc8 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java
@@ -236,13 +236,12 @@ public class DoubleTermsTests extends AbstractTermsTestCase {
 
     // the main purpose of this test is to make sure we're not allocating 2GB of memory per shard
     public void testSizeIsZero() {
-        ElasticsearchException exception = expectThrows(ElasticsearchException.class,
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
                 () -> client()
                         .prepareSearch("idx").setTypes("high_card_type").addAggregation(terms("terms").field(SINGLE_VALUED_FIELD_NAME)
                                 .minDocCount(randomInt(1)).size(0).collectMode(randomFrom(SubAggCollectionMode.values())))
                         .execute().actionGet());
-        assertThat(exception.getDetailedMessage(),
-                containsString("parameters [required_size] and [shard_size] must be >0 in terms aggregation."));
+        assertThat(exception.getMessage(), containsString("[size] must be greater than 0. Found [0] in [terms]"));
     }
 
     public void testSingleValueField() throws Exception {
@@ -588,7 +587,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase {
         SearchResponse response = client().prepareSearch("idx_unmapped").setTypes("type")
                 .addAggregation(terms("terms")
                         .field(SINGLE_VALUED_FIELD_NAME)
-                        .size(randomInt(5))
+                        .size(randomIntBetween(1, 5))
                         .collectMode(randomFrom(SubAggCollectionMode.values())))
                 .execute().actionGet();
 
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java
index dc3fea592a2..fa049429a3a 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java
@@ -238,7 +238,7 @@ public class LongTermsTests extends AbstractTermsTestCase {
 
     // the main purpose of this test is to make sure we're not allocating 2GB of memory per shard
     public void testSizeIsZero() {
-        ElasticsearchException exception = expectThrows(ElasticsearchException.class,
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
                 () -> client().prepareSearch("idx").setTypes("high_card_type")
                 .addAggregation(terms("terms")
                         .field(SINGLE_VALUED_FIELD_NAME)
@@ -246,8 +246,7 @@ public class LongTermsTests extends AbstractTermsTestCase {
                         .minDocCount(randomInt(1))
                         .size(0))
                         .execute().actionGet());
-        assertThat(exception.getDetailedMessage(),
-                containsString("parameters [required_size] and [shard_size] must be >0 in terms aggregation."));
+        assertThat(exception.getMessage(), containsString("[size] must be greater than 0. Found [0] in [terms]"));
     }
 
     public void testSingleValueField() throws Exception {
@@ -593,7 +592,7 @@ public class LongTermsTests extends AbstractTermsTestCase {
         SearchResponse response = client().prepareSearch("idx_unmapped").setTypes("type")
                 .addAggregation(terms("terms")
                         .field(SINGLE_VALUED_FIELD_NAME)
-                        .size(randomInt(5))
+                        .size(randomIntBetween(1, 5))
                         .collectMode(randomFrom(SubAggCollectionMode.values())))
                 .execute().actionGet();
 
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java
index aa31ae2a118..b098db154da 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java
@@ -203,15 +203,14 @@ public class StringTermsTests extends AbstractTermsTestCase {
     // the main purpose of this test is to make sure we're not allocating 2GB of memory per shard
     public void testSizeIsZero() {
         final int minDocCount = randomInt(1);
-        ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> client()
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> client()
                 .prepareSearch("idx")
                 .setTypes("high_card_type")
                 .addAggregation(
                         terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME)
                                 .collectMode(randomFrom(SubAggCollectionMode.values())).minDocCount(minDocCount).size(0)).execute()
                 .actionGet());
-        assertThat(exception.getDetailedMessage(),
-                containsString("parameters [required_size] and [shard_size] must be >0 in terms aggregation."));
+        assertThat(exception.getMessage(), containsString("[size] must be greater than 0. Found [0] in [terms]"));
     }
 
     public void testSingleValueField() throws Exception {
@@ -756,7 +755,7 @@ public class StringTermsTests extends AbstractTermsTestCase {
                 .prepareSearch("idx_unmapped")
                 .setTypes("type")
                 .addAggregation(
-                        terms("terms").executionHint(randomExecutionHint()).size(randomInt(5)).field(SINGLE_VALUED_FIELD_NAME)
+                        terms("terms").executionHint(randomExecutionHint()).size(randomIntBetween(1, 5)).field(SINGLE_VALUED_FIELD_NAME)
                                 .collectMode(randomFrom(SubAggCollectionMode.values()))).execute().actionGet();
 
         assertSearchResponse(response);

From 5abe1f7bb2051e0611d7bba21c76652bbf16dfac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Wed, 8 Jun 2016 16:01:35 +0200
Subject: [PATCH 0832/1311] Fix invalid rounding value for TimeIntervalRounding
 close to DST transition

There are edge cases where rounding a date to a certain interval using a time
zone with DST shifts can currently cause the rounded date to be bigger than the
original date. This happens when rounding a date closely after a DST start and
the rounded date falls into the DST gap.

Here is an example for CET time zone, where local time is set forward by one
hour at 2016-03-27T02:00:00+01:00 to 2016-03-27T03:00:00.000+02:00:

The date 2016-03-27T03:01:00.000+02:00 (1459040460000) which is just after the
DST change is first converted to local time (1459047660000). If we then apply
interval rounding for a 14m interval in local time, this  takes us to
1459047240000, which unfortunately falls into the DST gap.  When converting
this back to UTC, joda provides options to throw exceptions on illegal dates
like this, or correct this by adjusting the date to the new time zone offset.
We currently do the later, but this leads to converting this illegal date back
to 2016-03-27T03:54:00.000+02:00 (1459043640000), giving us a date that is
larger than the original date we wanted to round.

This change fixes this by using the "strict" option of 'convertLocalToUTC()'
to detect rounded dates that fall into the DST gap. If this happens, we can use
the time of the DST change instead as the interval start.

Even before this change, intervals around DST shifts like this can be shorter
than the desired interval.  This, for example, happens when the requested
interval width doesn't completely fit into the remaining time span when the DST
shift happens. For example, using a 14m interval in UTC+1 (CET before DST
starts) leads to the following valid rounding values around the time where DST
happens:

2016-03-27T01:30:00+01:00
2016-03-27T01:44:00+01:00
2016-03-27T01:58:00+01:00
2016-03-27T02:12:00+01:00
2016-03-27T02:26:00+01:00
...

while the rounding values in UTC+2 (CET after DST start) are placed like this
around the same time:

2016-03-27T02:40:00+02:00
2016-03-27T02:54:00+02:00
2016-03-27T03:08:00+02:00
2016-03-27T03:22:00+02:00
...

From this we can see then when we switch from UTC+1 to UTC+2 at 02:00 the last
rounding value in UTC+1 is at 01:58 and the first valid one in UTC+2 is at
03:08, so even if we decide to put all the dates in between into one rounding
interval, it will only cover 10 minutes. With this change we choose to use the
moment of DST shift as an aditional interval separator, leaving us with a 2min
interval from [01:58,02:00) before the shift and an 8min interval from
[03:00,03:08) after the shift.

This change also adds tests for the above example and adds randomization to the
existing TimeIntervalRounding tests.
---
 .../common/rounding/TimeZoneRounding.java     | 15 ++++-
 .../rounding/TimeZoneRoundingTests.java       | 58 ++++++++++++++-----
 2 files changed, 58 insertions(+), 15 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java b/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java
index c78d6c730b6..67bd42bb73c 100644
--- a/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java
+++ b/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
 import org.joda.time.DateTimeField;
 import org.joda.time.DateTimeZone;
+import org.joda.time.IllegalInstantException;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -218,7 +219,19 @@ public abstract class TimeZoneRounding extends Rounding {
         public long roundKey(long utcMillis) {
             long timeLocal = timeZone.convertUTCToLocal(utcMillis);
             long rounded = Rounding.Interval.roundValue(Rounding.Interval.roundKey(timeLocal, interval), interval);
-            return timeZone.convertLocalToUTC(rounded, false, utcMillis);
+            try {
+                return timeZone.convertLocalToUTC(rounded, true, utcMillis);
+            } catch (IllegalInstantException e) {
+                /*
+                 * The rounded local time is illegal and landed in a DST gap. In
+                 * this case, we choose 1ms tick after the transition date. We
+                 * don't want the transition date itself because those dates,
+                 * when rounded themselves, fall into the previous interval.
+                 * This would violate the invariant that the rounding operation
+                 * should be idempotent.
+                 */
+                return timeZone.previousTransition(utcMillis) + 1;
+            }
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java b/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
index 49ba26231da..6302f9c67b1 100644
--- a/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
+++ b/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
@@ -309,24 +309,54 @@ public class TimeZoneRoundingTests extends ESTestCase {
         assertThat(time("1986-01-01T00:40:00+05:45") - time("1986-01-01T00:20:00+05:45"), equalTo(TimeUnit.MINUTES.toMillis(20)));
     }
 
+    /**
+     * Special test for intervals that don't fit evenly into rounding interval.
+     * In this case, when interval crosses DST transition point, rounding in local
+     * time can land in a DST gap which results in wrong UTC rounding values.
+     */
+    public void testIntervalRounding_NotDivisibleInteval() {
+        DateTimeZone tz = DateTimeZone.forID("CET");
+        long interval = TimeUnit.MINUTES.toMillis(14);
+        TimeZoneRounding rounding = new TimeZoneRounding.TimeIntervalRounding(interval, tz);
+
+        assertThat(rounding.round(time("2016-03-27T01:41:00+01:00")), equalTo(time("2016-03-27T01:30:00+01:00")));
+        assertThat(rounding.round(time("2016-03-27T01:51:00+01:00")), equalTo(time("2016-03-27T01:44:00+01:00")));
+        assertThat(rounding.round(time("2016-03-27T01:59:00+01:00")), equalTo(time("2016-03-27T01:58:00+01:00")));
+        assertThat(rounding.round(time("2016-03-27T03:05:00+02:00")), equalTo(time("2016-03-27T03:00:00+02:00")));
+        assertThat(rounding.round(time("2016-03-27T03:12:00+02:00")), equalTo(time("2016-03-27T03:08:00+02:00")));
+        assertThat(rounding.round(time("2016-03-27T03:25:00+02:00")), equalTo(time("2016-03-27T03:22:00+02:00")));
+        assertThat(rounding.round(time("2016-03-27T03:39:00+02:00")), equalTo(time("2016-03-27T03:36:00+02:00")));
+    }
+
     /**
      * randomized test on {@link TimeIntervalRounding} with random interval and time zone offsets
      */
     public void testIntervalRoundingRandom() {
-        for (int i = 0; i < 1000; ++i) {
-            // max random interval is a year, can be negative
-            long interval = Math.abs(randomLong() % (TimeUnit.DAYS.toMillis(365)));
-            TimeZoneRounding rounding;
-            int timezoneOffset = randomIntBetween(-23, 23);
-            rounding = new TimeZoneRounding.TimeIntervalRounding(interval, DateTimeZone.forOffsetHours(timezoneOffset));
-            long date = Math.abs(randomLong() % ((long) 10e11));
-            final long roundedDate = rounding.round(date);
-            final long nextRoundingValue = rounding.nextRoundingValue(roundedDate);
-            assertThat("Rounding should be idempotent", roundedDate, equalTo(rounding.round(roundedDate)));
-            assertThat("Rounded value smaller or equal than unrounded, regardless of timezone", roundedDate, lessThanOrEqualTo(date));
-            assertThat("NextRounding value should be greater than date", nextRoundingValue, greaterThan(roundedDate));
-            assertThat("NextRounding value should be interval from rounded value", nextRoundingValue - roundedDate, equalTo(interval));
-            assertThat("NextRounding value should be a rounded date", nextRoundingValue, equalTo(rounding.round(nextRoundingValue)));
+        for (int i = 0; i < 1000; i++) {
+            TimeUnit unit = randomFrom(new TimeUnit[] {TimeUnit.MINUTES, TimeUnit.HOURS, TimeUnit.DAYS});
+            long interval = unit.toMillis(randomIntBetween(1, 365));
+            DateTimeZone tz = randomDateTimeZone();
+            TimeZoneRounding rounding = new TimeZoneRounding.TimeIntervalRounding(interval, tz);
+            long date = Math.abs(randomLong() % (2 * (long) 10e11)); // 1970-01-01T00:00:00Z - 2033-05-18T05:33:20.000+02:00
+            try {
+                final long roundedDate = rounding.round(date);
+                final long nextRoundingValue = rounding.nextRoundingValue(roundedDate);
+                assertThat("Rounding should be idempotent", roundedDate, equalTo(rounding.round(roundedDate)));
+                assertThat("Rounded value smaller or equal than unrounded", roundedDate, lessThanOrEqualTo(date));
+                assertThat("Values smaller than rounded value should round further down", rounding.round(roundedDate - 1),
+                        lessThan(roundedDate));
+
+                if (tz.isFixed()) {
+                    assertThat("NextRounding value should be greater than date", nextRoundingValue, greaterThan(roundedDate));
+                    assertThat("NextRounding value should be interval from rounded value", nextRoundingValue - roundedDate,
+                            equalTo(interval));
+                    assertThat("NextRounding value should be a rounded date", nextRoundingValue,
+                            equalTo(rounding.round(nextRoundingValue)));
+                }
+            } catch (AssertionError e) {
+                logger.error("Rounding error at {}, timezone {}, interval: {},", new DateTime(date, tz), tz, interval);
+                throw e;
+            }
         }
     }
 

From 4d78f280ed11aaa6515fca232c2b1230ae0b47dc Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Tue, 14 Jun 2016 15:25:44 +0200
Subject: [PATCH 0833/1311] Remove dead code and dead parameters (#18855)

---
 .../action/bulk/TransportBulkAction.java      |  6 +--
 .../explain/TransportExplainAction.java       |  2 +-
 .../action/get/TransportGetAction.java        |  2 +-
 .../action/get/TransportMultiGetAction.java   |  2 +-
 .../TransportTermVectorsAction.java           |  3 +-
 .../action/update/TransportUpdateAction.java  |  2 +-
 .../cluster/routing/OperationRouting.java     |  4 +-
 .../org/elasticsearch/index/mapper/Uid.java   | 39 -------------------
 .../RoutingBackwardCompatibilityTests.java    |  2 +-
 .../elasticsearch/index/mapper/UidTests.java  | 25 +++++++++++-
 .../test/InternalTestCluster.java             |  2 +-
 11 files changed, 35 insertions(+), 54 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
index 4cbebd0739a..c6b046a4758 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
@@ -304,7 +304,7 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
             if (request instanceof IndexRequest) {
                 IndexRequest indexRequest = (IndexRequest) request;
                 String concreteIndex = concreteIndices.getConcreteIndex(indexRequest.index()).getName();
-                ShardId shardId = clusterService.operationRouting().indexShards(clusterState, concreteIndex, indexRequest.type(), indexRequest.id(), indexRequest.routing()).shardId();
+                ShardId shardId = clusterService.operationRouting().indexShards(clusterState, concreteIndex, indexRequest.id(), indexRequest.routing()).shardId();
                 List<BulkItemRequest> list = requestsByShard.get(shardId);
                 if (list == null) {
                     list = new ArrayList<>();
@@ -314,7 +314,7 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
             } else if (request instanceof DeleteRequest) {
                 DeleteRequest deleteRequest = (DeleteRequest) request;
                 String concreteIndex = concreteIndices.getConcreteIndex(deleteRequest.index()).getName();
-                ShardId shardId = clusterService.operationRouting().indexShards(clusterState, concreteIndex, deleteRequest.type(), deleteRequest.id(), deleteRequest.routing()).shardId();
+                ShardId shardId = clusterService.operationRouting().indexShards(clusterState, concreteIndex, deleteRequest.id(), deleteRequest.routing()).shardId();
                 List<BulkItemRequest> list = requestsByShard.get(shardId);
                 if (list == null) {
                     list = new ArrayList<>();
@@ -324,7 +324,7 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
             } else if (request instanceof UpdateRequest) {
                 UpdateRequest updateRequest = (UpdateRequest) request;
                 String concreteIndex = concreteIndices.getConcreteIndex(updateRequest.index()).getName();
-                ShardId shardId = clusterService.operationRouting().indexShards(clusterState, concreteIndex, updateRequest.type(), updateRequest.id(), updateRequest.routing()).shardId();
+                ShardId shardId = clusterService.operationRouting().indexShards(clusterState, concreteIndex, updateRequest.id(), updateRequest.routing()).shardId();
                 List<BulkItemRequest> list = requestsByShard.get(shardId);
                 if (list == null) {
                     list = new ArrayList<>();
diff --git a/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java b/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
index 88e7c66c1b1..1084cf6c237 100644
--- a/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
+++ b/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
@@ -152,7 +152,7 @@ public class TransportExplainAction extends TransportSingleShardAction<ExplainRe
     @Override
     protected ShardIterator shards(ClusterState state, InternalRequest request) {
         return clusterService.operationRouting().getShards(
-                clusterService.state(), request.concreteIndex(), request.request().type(), request.request().id(), request.request().routing(), request.request().preference()
+                clusterService.state(), request.concreteIndex(), request.request().id(), request.request().routing(), request.request().preference()
         );
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java b/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
index 528546d5b31..240035aee2a 100644
--- a/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
+++ b/core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java
@@ -62,7 +62,7 @@ public class TransportGetAction extends TransportSingleShardAction<GetRequest, G
     @Override
     protected ShardIterator shards(ClusterState state, InternalRequest request) {
         return clusterService.operationRouting()
-                .getShards(clusterService.state(), request.concreteIndex(), request.request().type(), request.request().id(), request.request().routing(), request.request().preference());
+                .getShards(clusterService.state(), request.concreteIndex(), request.request().id(), request.request().routing(), request.request().preference());
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java b/core/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java
index 1858ac8ba71..f4037e40863 100644
--- a/core/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java
+++ b/core/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java
@@ -76,7 +76,7 @@ public class TransportMultiGetAction extends HandledTransportAction<MultiGetRequ
                 continue;
             }
             ShardId shardId = clusterService.operationRouting()
-                    .getShards(clusterState, concreteSingleIndex, item.type(), item.id(), item.routing(), null).shardId();
+                    .getShards(clusterState, concreteSingleIndex, item.id(), item.routing(), null).shardId();
             MultiGetShardRequest shardRequest = shardRequests.get(shardId);
             if (shardRequest == null) {
                 shardRequest = new MultiGetShardRequest(request, shardId.getIndexName(), shardId.id());
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java b/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java
index 47eae68f2e8..55885b03696 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.action.termvectors;
 
-import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.RoutingMissingException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
@@ -56,7 +55,7 @@ public class TransportTermVectorsAction extends TransportSingleShardAction<TermV
 
     @Override
     protected ShardIterator shards(ClusterState state, InternalRequest request) {
-        return clusterService.operationRouting().getShards(state, request.concreteIndex(), request.request().type(), request.request().id(),
+        return clusterService.operationRouting().getShards(state, request.concreteIndex(), request.request().id(),
                 request.request().routing(), request.request().preference());
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
index ca55a63c1d6..ae6a15411f5 100644
--- a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
+++ b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
@@ -153,7 +153,7 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
             return clusterState.routingTable().index(request.concreteIndex()).shard(request.getShardId().getId()).primaryShardIt();
         }
         ShardIterator shardIterator = clusterService.operationRouting()
-                .indexShards(clusterState, request.concreteIndex(), request.type(), request.id(), request.routing());
+                .indexShards(clusterState, request.concreteIndex(), request.id(), request.routing());
         ShardRouting shard;
         while ((shard = shardIterator.nextOrNull()) != null) {
             if (shard.primary()) {
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
index 9b1c82e7c06..f95f3003a4a 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
@@ -52,11 +52,11 @@ public class OperationRouting extends AbstractComponent {
         this.awarenessAllocationDecider = awarenessAllocationDecider;
     }
 
-    public ShardIterator indexShards(ClusterState clusterState, String index, String type, String id, @Nullable String routing) {
+    public ShardIterator indexShards(ClusterState clusterState, String index, String id, @Nullable String routing) {
         return shards(clusterState, index, id, routing).shardsIt();
     }
 
-    public ShardIterator getShards(ClusterState clusterState, String index, String type, String id, @Nullable String routing, @Nullable String preference) {
+    public ShardIterator getShards(ClusterState clusterState, String index, String id, @Nullable String routing, @Nullable String preference) {
         return preferenceActiveShardIterator(shards(clusterState, index, id, routing), clusterState.nodes().getLocalNodeId(), clusterState.nodes(), preference);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/Uid.java b/core/src/main/java/org/elasticsearch/index/mapper/Uid.java
index 831f4575f3d..2a8938b4ab7 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/Uid.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/Uid.java
@@ -35,7 +35,6 @@ public final class Uid {
 
     public static final char DELIMITER = '#';
     public static final byte DELIMITER_BYTE = 0x23;
-    public static final BytesRef DELIMITER_BYTES = new BytesRef(new byte[]{DELIMITER_BYTE});
 
     private final String type;
 
@@ -83,13 +82,6 @@ public final class Uid {
         return createUidAsBytes(type, id);
     }
 
-    public static BytesRef typePrefixAsBytes(BytesRef type) {
-        BytesRefBuilder bytesRef = new BytesRefBuilder();
-        bytesRef.append(type);
-        bytesRef.append(DELIMITER_BYTES);
-        return bytesRef.toBytesRef();
-    }
-
     public static Uid createUid(String uid) {
         int delimiterIndex = uid.indexOf(DELIMITER); // type is not allowed to have # in it..., ids can
         return new Uid(uid.substring(0, delimiterIndex), uid.substring(delimiterIndex + 1));
@@ -136,35 +128,4 @@ public final class Uid {
         return type + DELIMITER + id;
     }
 
-    public static boolean hasDelimiter(BytesRef uid) {
-        final int limit = uid.offset + uid.length;
-        for (int i = uid.offset; i < limit; i++) {
-            if (uid.bytes[i] == DELIMITER_BYTE) { // 0x23 is equal to '#'
-                return true;
-            }
-        }
-        return false;
-    }
-
-    public static BytesRef[] splitUidIntoTypeAndId(BytesRef uid) {
-        int loc = -1;
-        final int limit = uid.offset + uid.length;
-        for (int i = uid.offset; i < limit; i++) {
-            if (uid.bytes[i] == DELIMITER_BYTE) { // 0x23 is equal to '#'
-                loc = i;
-                break;
-            }
-        }
-
-        if (loc == -1) {
-            return null;
-        }
-
-        int idStart = loc + 1;
-        return new BytesRef[] {
-                new BytesRef(uid.bytes, uid.offset, loc - uid.offset),
-                new BytesRef(uid.bytes, idStart, limit - idStart)
-        };
-    }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingBackwardCompatibilityTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingBackwardCompatibilityTests.java
index 5ff4a328ef6..d8dedb1e8e8 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingBackwardCompatibilityTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingBackwardCompatibilityTests.java
@@ -62,7 +62,7 @@ public class RoutingBackwardCompatibilityTests extends ESTestCase {
                     MetaData.Builder metaData = MetaData.builder().put(indexMetaData, false);
                     RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetaData).build();
                     ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
-                    final int shardId = operationRouting.indexShards(clusterState, index, type, id, routing).shardId().getId();
+                    final int shardId = operationRouting.indexShards(clusterState, index, id, routing).shardId().getId();
                     assertEquals(currentExpectedShard, shardId);
                 }
             }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/UidTests.java b/core/src/test/java/org/elasticsearch/index/mapper/UidTests.java
index 860c66863ff..7a800d469a1 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/UidTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/UidTests.java
@@ -26,7 +26,7 @@ import static org.hamcrest.Matchers.equalTo;
 public class UidTests extends ESTestCase {
     public void testCreateAndSplitId() {
         BytesRef createUid = Uid.createUidAsBytes("foo", "bar");
-        BytesRef[] splitUidIntoTypeAndId = Uid.splitUidIntoTypeAndId(createUid);
+        BytesRef[] splitUidIntoTypeAndId = splitUidIntoTypeAndId(createUid);
         assertThat("foo", equalTo(splitUidIntoTypeAndId[0].utf8ToString()));
         assertThat("bar", equalTo(splitUidIntoTypeAndId[1].utf8ToString()));
         // split also with an offset
@@ -34,8 +34,29 @@ public class UidTests extends ESTestCase {
         ref.offset = 9;
         ref.length = createUid.length;
         System.arraycopy(createUid.bytes, createUid.offset, ref.bytes, ref.offset, ref.length);
-        splitUidIntoTypeAndId = Uid.splitUidIntoTypeAndId(ref);
+        splitUidIntoTypeAndId = splitUidIntoTypeAndId(ref);
         assertThat("foo", equalTo(splitUidIntoTypeAndId[0].utf8ToString()));
         assertThat("bar", equalTo(splitUidIntoTypeAndId[1].utf8ToString()));
     }
+
+    public static BytesRef[] splitUidIntoTypeAndId(BytesRef uid) {
+        int loc = -1;
+        final int limit = uid.offset + uid.length;
+        for (int i = uid.offset; i < limit; i++) {
+            if (uid.bytes[i] == Uid.DELIMITER_BYTE) { // 0x23 is equal to '#'
+                loc = i;
+                break;
+            }
+        }
+
+        if (loc == -1) {
+            return null;
+        }
+
+        int idStart = loc + 1;
+        return new BytesRef[] {
+            new BytesRef(uid.bytes, uid.offset, loc - uid.offset),
+            new BytesRef(uid.bytes, idStart, limit - idStart)
+        };
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index 1217662480a..0dee5f75948 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -1773,7 +1773,7 @@ public final class InternalTestCluster extends TestCluster {
                 OperationRouting operationRouting = getInstanceFromNode(OperationRouting.class, node);
                 while (true) {
                     String routing = RandomStrings.randomAsciiOfLength(random, 10);
-                    final int targetShard = operationRouting.indexShards(clusterService.state(), index.getName(), type, null, routing).shardId().getId();
+                    final int targetShard = operationRouting.indexShards(clusterService.state(), index.getName(), null, routing).shardId().getId();
                     if (shard == targetShard) {
                         return routing;
                     }

From 03f5aa8ea01ec3c76494ffdae03f7149739beb0c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Tue, 14 Jun 2016 15:08:17 +0200
Subject: [PATCH 0834/1311] Don't throw IllegalInstantException to determine
 DST gap

By taking the logic from DateTimeZone#convertLocalToUTC(long, boolean) we
can avoid throwing the exception.
---
 .../common/rounding/TimeZoneRounding.java     | 57 +++++++++++++++----
 1 file changed, 47 insertions(+), 10 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java b/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java
index 67bd42bb73c..593b0484800 100644
--- a/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java
+++ b/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java
@@ -219,19 +219,56 @@ public abstract class TimeZoneRounding extends Rounding {
         public long roundKey(long utcMillis) {
             long timeLocal = timeZone.convertUTCToLocal(utcMillis);
             long rounded = Rounding.Interval.roundValue(Rounding.Interval.roundKey(timeLocal, interval), interval);
-            try {
-                return timeZone.convertLocalToUTC(rounded, true, utcMillis);
-            } catch (IllegalInstantException e) {
+            long roundedUTC;
+            if (isInDSTGap(rounded) == false) {
+                roundedUTC  = timeZone.convertLocalToUTC(rounded, true, utcMillis);
+            } else {
                 /*
-                 * The rounded local time is illegal and landed in a DST gap. In
-                 * this case, we choose 1ms tick after the transition date. We
-                 * don't want the transition date itself because those dates,
-                 * when rounded themselves, fall into the previous interval.
-                 * This would violate the invariant that the rounding operation
-                 * should be idempotent.
+                 * Edge case where the rounded local time is illegal and landed
+                 * in a DST gap. In this case, we choose 1ms tick after the
+                 * transition date. We don't want the transition date itself
+                 * because those dates, when rounded themselves, fall into the
+                 * previous interval. This would violate the invariant that the
+                 * rounding operation should be idempotent.
                  */
-                return timeZone.previousTransition(utcMillis) + 1;
+                roundedUTC = timeZone.previousTransition(utcMillis) + 1;
             }
+            return roundedUTC;
+        }
+
+        /**
+         * Determine whether the local instant is a valid instant in the given
+         * time zone. The logic for this is taken from
+         * {@link DateTimeZone#convertLocalToUTC(long, boolean)} for the
+         * `strict` mode case, but instead of throwing an
+         * {@link IllegalInstantException}, which is costly, we want to return a
+         * flag indicating that the value is illegal in that time zone.
+         */
+        private boolean isInDSTGap(long instantLocal) {
+            if (timeZone.isFixed()) {
+                return false;
+            }
+            // get the offset at instantLocal (first estimate)
+            int offsetLocal = timeZone.getOffset(instantLocal);
+            // adjust instantLocal using the estimate and recalc the offset
+            int offset = timeZone.getOffset(instantLocal - offsetLocal);
+            // if the offsets differ, we must be near a DST boundary
+            if (offsetLocal != offset) {
+                // determine if we are in the DST gap
+                long nextLocal = timeZone.nextTransition(instantLocal - offsetLocal);
+                if (nextLocal == (instantLocal - offsetLocal)) {
+                    nextLocal = Long.MAX_VALUE;
+                }
+                long nextAdjusted = timeZone.nextTransition(instantLocal - offset);
+                if (nextAdjusted == (instantLocal - offset)) {
+                    nextAdjusted = Long.MAX_VALUE;
+                }
+                if (nextLocal != nextAdjusted) {
+                    // we are in the DST gap
+                    return true;
+                }
+            }
+            return false;
         }
 
         @Override

From 49c09f1527ff0560c432b07c2e8c3ca72a21adc9 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 14 Jun 2016 09:56:49 -0400
Subject: [PATCH 0835/1311] Test: Fix error message check for Windows

Betrayal! CRLF strikes again. Weakened the test from equals to
containsString.
---
 .../org/elasticsearch/painless/WhenThingsGoWrongTests.java    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
index 587e3e9fd09..7a597132227 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
@@ -25,6 +25,7 @@ import java.util.Collections;
 import java.util.regex.PatternSyntaxException;
 
 import static java.util.Collections.emptyMap;
+import static org.hamcrest.Matchers.containsString;
 
 public class WhenThingsGoWrongTests extends ScriptTestCase {
     public void testNullPointer() {
@@ -224,7 +225,8 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
         PatternSyntaxException e = expectScriptThrows(PatternSyntaxException.class, () -> {
             exec("/\\ujjjj/"); // Invalid unicode
         });
-        assertEquals("Illegal Unicode escape sequence near index 2\n\\ujjjj\n  ^", e.getMessage());
+        assertThat(e.getMessage(), containsString("Illegal Unicode escape sequence near index 2"));
+        assertThat(e.getMessage(), containsString("\\ujjjj"));
     }
 
 }

From ee2ba13cce21382ea81ab204ab9cb71e30cc4580 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Tue, 14 Jun 2016 17:04:58 +0200
Subject: [PATCH 0836/1311] Register Highlighter instances instead of classes
 (#18859)

This change detaches highlighter registration from Guice. It's just a
small step into the right direction.
---
 .../elasticsearch/search/Highlighters.java    | 63 +++++++++++++++++
 .../elasticsearch/search/SearchModule.java    | 24 ++-----
 .../search/highlight/HighlightPhase.java      |  1 +
 .../search/highlight/Highlighters.java        | 69 -------------------
 .../search/SearchModuleTests.java             | 24 ++++---
 .../highlight/CustomHighlighterPlugin.java    |  2 +-
 6 files changed, 85 insertions(+), 98 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/search/Highlighters.java
 delete mode 100644 core/src/main/java/org/elasticsearch/search/highlight/Highlighters.java

diff --git a/core/src/main/java/org/elasticsearch/search/Highlighters.java b/core/src/main/java/org/elasticsearch/search/Highlighters.java
new file mode 100644
index 00000000000..38fed839d6e
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/Highlighters.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.search.highlight.FastVectorHighlighter;
+import org.elasticsearch.search.highlight.Highlighter;
+import org.elasticsearch.search.highlight.PlainHighlighter;
+import org.elasticsearch.search.highlight.PostingsHighlighter;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * An extensions point and registry for all the highlighters a node supports.
+ */
+public final class Highlighters {
+
+    private final Map<String, Highlighter> parsers = new HashMap<>();
+
+    Highlighters(Settings settings) {
+        registerHighlighter("fvh",  new FastVectorHighlighter(settings));
+        registerHighlighter("plain", new PlainHighlighter());
+        registerHighlighter("postings", new PostingsHighlighter());
+    }
+
+    /**
+     * Returns the highlighter for the given key or <code>null</code> if there is no highlighter registered for that key.
+     */
+    public Highlighter get(String key) {
+        return parsers.get(key);
+    }
+
+    /**
+     * Registers a highlighter for the given key
+     * @param key the key the highlighter should be referenced by in the search request
+     * @param highlighter the highlighter instance
+     */
+    void registerHighlighter(String key, Highlighter highlighter) {
+        if (highlighter == null) {
+            throw new IllegalArgumentException("Can't register null highlighter for key: [" + key + "]");
+        }
+        if (parsers.putIfAbsent(key, highlighter) != null) {
+            throw new IllegalArgumentException("Can't register the same [highlighter] more than once for [" + key + "]");
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java
index 8afda85c58f..2f626a82496 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -28,7 +28,6 @@ import org.elasticsearch.common.inject.multibindings.Multibinder;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.common.lucene.search.function.ScoreFunction;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ParseFieldRegistry;
@@ -250,7 +249,6 @@ import org.elasticsearch.search.fetch.source.FetchSourceSubPhase;
 import org.elasticsearch.search.fetch.version.VersionFetchSubPhase;
 import org.elasticsearch.search.highlight.HighlightPhase;
 import org.elasticsearch.search.highlight.Highlighter;
-import org.elasticsearch.search.highlight.Highlighters;
 import org.elasticsearch.search.query.QueryPhase;
 import org.elasticsearch.search.rescore.QueryRescorerBuilder;
 import org.elasticsearch.search.rescore.RescoreBuilder;
@@ -270,9 +268,8 @@ import java.util.Set;
  */
 public class SearchModule extends AbstractModule {
 
-    private final Highlighters highlighters = new Highlighters();
+    private final Highlighters highlighters;
     private final Suggesters suggesters;
-
     private final ParseFieldRegistry<ScoreFunctionParser<?>> scoreFunctionParserRegistry = new ParseFieldRegistry<>("score_function");
     private final IndicesQueriesRegistry queryParserRegistry = new IndicesQueriesRegistry();
     private final ParseFieldRegistry<Aggregator.Parser> aggregationParserRegistry = new ParseFieldRegistry<>("aggregation");
@@ -298,7 +295,7 @@ public class SearchModule extends AbstractModule {
         this.settings = settings;
         this.namedWriteableRegistry = namedWriteableRegistry;
         suggesters = new Suggesters(namedWriteableRegistry);
-
+        highlighters = new Highlighters(settings);
         registerBuiltinScoreFunctionParsers();
         registerBuiltinQueryParsers();
         registerBuiltinRescorers();
@@ -308,8 +305,8 @@ public class SearchModule extends AbstractModule {
         registerBuiltinMovingAverageModels();
     }
 
-    public void registerHighlighter(String key, Class<? extends Highlighter> clazz) {
-        highlighters.registerExtension(key, clazz);
+    public void registerHighlighter(String key, Highlighter highligher) {
+        highlighters.registerHighlighter(key, highligher);
     }
 
     public void registerSuggester(String key, Suggester<?> suggester) {
@@ -331,13 +328,6 @@ public class SearchModule extends AbstractModule {
         namedWriteableRegistry.register(ScoreFunctionBuilder.class, functionName.getPreferredName(), reader);
     }
 
-    /**
-     * Fetch the registry of {@linkplain ScoreFunction}s. This is public so extensions can access the score functions.
-     */
-    public ParseFieldRegistry<ScoreFunctionParser<?>> getScoreFunctionParserRegistry() {
-        return scoreFunctionParserRegistry;
-    }
-
     /**
      * Register a new ValueFormat.
      */
@@ -441,10 +431,6 @@ public class SearchModule extends AbstractModule {
         namedWriteableRegistry.register(PipelineAggregatorBuilder.class, aggregationName.getPreferredName(), reader);
     }
 
-    public AggregatorParsers getAggregatorParsers() {
-        return aggregatorParsers;
-    }
-
     @Override
     protected void configure() {
         bind(IndicesQueriesRegistry.class).toInstance(queryParserRegistry);
@@ -473,7 +459,7 @@ public class SearchModule extends AbstractModule {
     }
 
     protected void configureHighlighters() {
-       highlighters.bind(binder());
+       binder().bind(Highlighters.class).toInstance(highlighters);
     }
 
     protected void configureAggs() {
diff --git a/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java b/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java
index 92011fd77e7..701cd588892 100644
--- a/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java
@@ -30,6 +30,7 @@ import org.elasticsearch.index.mapper.core.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.core.StringFieldMapper;
 import org.elasticsearch.index.mapper.core.TextFieldMapper;
 import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
+import org.elasticsearch.search.Highlighters;
 import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.internal.InternalSearchHit;
diff --git a/core/src/main/java/org/elasticsearch/search/highlight/Highlighters.java b/core/src/main/java/org/elasticsearch/search/highlight/Highlighters.java
deleted file mode 100644
index 30b8d15d93d..00000000000
--- a/core/src/main/java/org/elasticsearch/search/highlight/Highlighters.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.search.highlight;
-
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.ExtensionPoint;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-
-/**
- * An extensions point and registry for all the highlighters a node supports.
- */
-public class Highlighters extends ExtensionPoint.ClassMap<Highlighter> {
-
-    private static final String FVH = "fvh";
-    private static final String PLAIN = "plain";
-    private static final String POSTINGS = "postings";
-
-    private final Map<String, Highlighter> parsers;
-
-    public Highlighters(){
-        this(Collections.emptyMap());
-    }
-
-    private Highlighters(Map<String, Highlighter> parsers) {
-        super("highlighter", Highlighter.class, new HashSet<>(Arrays.asList(FVH, PLAIN, POSTINGS)),
-                Highlighters.class);
-        this.parsers = Collections.unmodifiableMap(parsers);
-    }
-
-    @Inject
-    public Highlighters(Settings settings, Map<String, Highlighter> parsers) {
-        this(addBuiltIns(settings, parsers));
-    }
-
-    private static Map<String, Highlighter> addBuiltIns(Settings settings, Map<String, Highlighter> parsers) {
-        Map<String, Highlighter> map = new HashMap<>();
-        map.put(FVH,  new FastVectorHighlighter(settings));
-        map.put(PLAIN, new PlainHighlighter());
-        map.put(POSTINGS, new PostingsHighlighter());
-        map.putAll(parsers);
-        return map;
-    }
-
-    public Highlighter get(String type) {
-        return parsers.get(type);
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java b/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java
index 384e181c899..2cfb249f533 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java
@@ -29,8 +29,9 @@ import org.elasticsearch.index.query.QueryParser;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.highlight.CustomHighlighter;
-import org.elasticsearch.search.highlight.Highlighter;
+import org.elasticsearch.search.highlight.FastVectorHighlighter;
 import org.elasticsearch.search.highlight.PlainHighlighter;
+import org.elasticsearch.search.highlight.PostingsHighlighter;
 import org.elasticsearch.search.suggest.CustomSuggester;
 import org.elasticsearch.search.suggest.phrase.PhraseSuggester;
 
@@ -48,7 +49,7 @@ public class SearchModuleTests extends ModuleTestCase {
    public void testDoubleRegister() {
        SearchModule module = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry());
        try {
-           module.registerHighlighter("fvh", PlainHighlighter.class);
+           module.registerHighlighter("fvh", new PlainHighlighter());
        } catch (IllegalArgumentException e) {
            assertEquals(e.getMessage(), "Can't register the same [highlighter] more than once for [fvh]");
        }
@@ -70,13 +71,18 @@ public class SearchModuleTests extends ModuleTestCase {
 
     public void testRegisterHighlighter() {
         SearchModule module = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry());
-        module.registerHighlighter("custom", CustomHighlighter.class);
-        try {
-            module.registerHighlighter("custom", CustomHighlighter.class);
-        } catch (IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "Can't register the same [highlighter] more than once for [custom]");
-        }
-        assertMapMultiBinding(module, Highlighter.class, CustomHighlighter.class);
+        module.registerHighlighter("custom",  new CustomHighlighter());
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
+            () -> module.registerHighlighter("custom", new CustomHighlighter()));
+        assertEquals("Can't register the same [highlighter] more than once for [custom]", exception.getMessage());
+
+        exception = expectThrows(IllegalArgumentException.class,
+            () -> module.registerHighlighter("custom", null));
+        assertEquals("Can't register null highlighter for key: [custom]", exception.getMessage());
+        assertInstanceBinding(module, Highlighters.class, (h) -> h.get("custom").getClass() == CustomHighlighter.class
+        && h.get("fvh").getClass() == FastVectorHighlighter.class
+        && h.get("plain").getClass() == PlainHighlighter.class
+        && h.get("postings").getClass() == PostingsHighlighter.class);
     }
 
     public void testRegisterQueryParserDuplicate() {
diff --git a/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterPlugin.java b/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterPlugin.java
index 80e39a2a6dd..b6c42b73677 100644
--- a/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterPlugin.java
+++ b/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterPlugin.java
@@ -35,6 +35,6 @@ public class CustomHighlighterPlugin extends Plugin {
     }
 
     public void onModule(SearchModule highlightModule) {
-        highlightModule.registerHighlighter("test-custom", CustomHighlighter.class);
+        highlightModule.registerHighlighter("test-custom", new CustomHighlighter());
     }
 }

From 79f0ab2fc6f9242dcdcd199fb61edc2a2fce850f Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 14 Jun 2016 12:51:29 -0400
Subject: [PATCH 0837/1311] rename this variable

---
 .../elasticsearch/painless/MethodWriter.java  | 25 ++++++++++---------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index 97502384d22..3280c3bbcfe 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -264,41 +264,42 @@ public final class MethodWriter extends GeneratorAdapter {
 
     /** Writes a dynamic binary instruction: returnType, lhs, and rhs can be different */
     public void writeDynamicBinaryInstruction(Location location, Type returnType, Type lhs, Type rhs, Operation operation) {
-        org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(returnType.type, lhs.type, rhs.type);
+        org.objectweb.asm.Type methodType = org.objectweb.asm.Type.getMethodType(returnType.type, lhs.type, rhs.type);
+        String descriptor = methodType.getDescriptor();
         
         switch (operation) {
             case MUL:
-                invokeDynamic("mul", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
+                invokeDynamic("mul", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
                 break;
             case DIV:
-                invokeDynamic("div", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
+                invokeDynamic("div", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
                 break;
             case REM:
-                invokeDynamic("rem", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
+                invokeDynamic("rem", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
                 break;
             case ADD:
-                invokeDynamic("add", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
+                invokeDynamic("add", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
                 break;
             case SUB:
-                invokeDynamic("sub", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
+                invokeDynamic("sub", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
                 break;
             case LSH:
-                invokeDynamic("lsh", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR); 
+                invokeDynamic("lsh", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR); 
                 break;
             case USH:
-                invokeDynamic("ush", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR); 
+                invokeDynamic("ush", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR); 
                 break;
             case RSH:
-                invokeDynamic("rsh", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR); 
+                invokeDynamic("rsh", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR); 
                 break;
             case BWAND: 
-                invokeDynamic("and", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
+                invokeDynamic("and", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
                 break;
             case XOR:   
-                invokeDynamic("xor", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
+                invokeDynamic("xor", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
                 break;
             case BWOR:  
-                invokeDynamic("or", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
+                invokeDynamic("or", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
                 break;
             default:
                 throw location.createError(new IllegalStateException("Illegal tree structure."));

From e392e0b1dfe553324c313666b0d5a844429e976b Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 31 May 2016 19:44:01 -0400
Subject: [PATCH 0838/1311] Create get task API that falls back to the .tasks
 index

This adds a get task API that supports GET /_tasks/${taskId} and
removes that responsibility from the list tasks API. The get task
API supports wait_for_complation just as the list tasks API does
but doesn't support any of the list task API's filters. In exchange,
it supports falling back to the .results index when the task isn't
running any more. Like any good GET API it 404s when it doesn't
find the task.

Then we change reindex, update-by-query, and delete-by-query to
persist the task result when wait_for_completion=false. The leads
to the neat behavior that, once you start a reindex with
wait_for_completion=false, you can fetch the result of the task by
using the get task API and see the result when it has finished.

Also rename the .results index to .tasks.
---
 .../elasticsearch/action/ActionModule.java    |   3 +
 .../elasticsearch/action/ActionRequest.java   |   3 +
 .../tasks/cancel/CancelTasksResponse.java     |   2 +-
 .../cancel/TransportCancelTasksAction.java    |   2 +-
 .../cluster/node/tasks/get/GetTaskAction.java |  46 +++
 .../node/tasks/get/GetTaskRequest.java        | 119 ++++++++
 .../node/tasks/get/GetTaskRequestBuilder.java |  58 ++++
 .../node/tasks/get/GetTaskResponse.java       |  75 +++++
 .../tasks/get/TransportGetTaskAction.java     | 216 ++++++++++++++
 .../node/tasks/list/ListTasksResponse.java    |  89 +++---
 .../cluster/node/tasks/list/TaskGroup.java    |   8 +-
 .../tasks/list/TransportListTasksAction.java  |  55 ++--
 .../support/replication/ReplicationTask.java  |  15 +
 .../support/tasks/TasksRequestBuilder.java    |  10 +
 .../client/ClusterAdminClient.java            |  32 ++
 .../org/elasticsearch/client/Requests.java    |  11 +
 .../client/support/AbstractClient.java        |  48 ++-
 .../elasticsearch/cluster/ClusterModule.java  |   4 +-
 .../common/network/NetworkModule.java         |   4 +
 .../common/xcontent/AbstractObjectParser.java |  13 +
 .../xcontent/json/JsonXContentGenerator.java  |   4 +
 .../java/org/elasticsearch/node/Node.java     |   4 +-
 .../node/tasks/RestCancelTasksAction.java     |  12 +-
 .../cluster/node/tasks/RestGetTaskAction.java |  55 ++++
 .../node/tasks/RestListTasksAction.java       |  33 ++-
 .../rest/action/cat/RestTasksAction.java      |  29 +-
 .../tasks/PersistedTaskInfo.java              | 224 ++++++++++++++
 .../elasticsearch/tasks/RawTaskStatus.java    |  96 ++++++
 .../java/org/elasticsearch/tasks/Task.java    |  13 +-
 .../node/tasks/list => tasks}/TaskInfo.java   | 102 +++++--
 .../org/elasticsearch/tasks/TaskManager.java  |  12 +-
 ...rvice.java => TaskPersistenceService.java} |  64 ++--
 .../org/elasticsearch/tasks/TaskResult.java   |  90 ------
 ...x-mapping.json => task-index-mapping.json} |   2 +-
 .../node/tasks/CancellableTasksTests.java     |   2 +-
 .../tasks/RecordingTaskManagerListener.java   |   2 +-
 .../node/tasks/TaskManagerTestCase.java       |   1 -
 .../admin/cluster/node/tasks/TasksIT.java     | 278 ++++++++++++++----
 .../node/tasks/TransportTasksActionTests.java |  24 +-
 .../common/xcontent/BaseXContentTestCase.java |  23 +-
 .../tasks/PersistedTaskInfoTests.java         | 137 +++++++++
 docs/reference/cluster/tasks.asciidoc         |  29 +-
 docs/reference/docs/delete-by-query.asciidoc  |  27 +-
 docs/reference/docs/reindex.asciidoc          |  27 +-
 docs/reference/docs/update-by-query.asciidoc  |  27 +-
 .../AbstractBaseReindexRestHandler.java       |   2 +
 .../reindex/AbstractBulkByScrollRequest.java  |  18 ++
 .../AbstractBulkByScrollRequestBuilder.java   |   8 +
 .../index/reindex/RestRethrottleAction.java   |  12 +-
 .../reindex/TransportRethrottleAction.java    |   2 +-
 .../index/reindex/CancelTests.java            |   2 +-
 .../test/delete_by_query/10_basic.yaml        |   5 +-
 .../test/delete_by_query/70_throttle.yaml     |   4 +-
 .../rest-api-spec/test/reindex/10_basic.yaml  |   5 +-
 .../test/reindex/70_throttle.yaml             |   4 +-
 .../test/update_by_query/10_basic.yaml        |   5 +-
 .../test/update_by_query/60_throttle.yaml     |   4 +-
 .../resources/rest-api-spec/api/task.get.json |  23 ++
 .../rest-api-spec/api/tasks.list.json         |   7 +-
 .../rest-api-spec/test/task.get/10_basic.yaml |  10 +
 60 files changed, 1863 insertions(+), 378 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskAction.java
 create mode 100644 core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequest.java
 create mode 100644 core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequestBuilder.java
 create mode 100644 core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java
 create mode 100644 core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
 create mode 100644 core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestGetTaskAction.java
 create mode 100644 core/src/main/java/org/elasticsearch/tasks/PersistedTaskInfo.java
 create mode 100644 core/src/main/java/org/elasticsearch/tasks/RawTaskStatus.java
 rename core/src/main/java/org/elasticsearch/{action/admin/cluster/node/tasks/list => tasks}/TaskInfo.java (54%)
 rename core/src/main/java/org/elasticsearch/tasks/{TaskResultsService.java => TaskPersistenceService.java} (71%)
 delete mode 100644 core/src/main/java/org/elasticsearch/tasks/TaskResult.java
 rename core/src/main/resources/org/elasticsearch/tasks/{task-results-index-mapping.json => task-index-mapping.json} (98%)
 create mode 100644 core/src/test/java/org/elasticsearch/tasks/PersistedTaskInfoTests.java
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/task.get.json
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/task.get/10_basic.yaml

diff --git a/core/src/main/java/org/elasticsearch/action/ActionModule.java b/core/src/main/java/org/elasticsearch/action/ActionModule.java
index 7e975b922d4..7c62c171073 100644
--- a/core/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/core/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -32,6 +32,8 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsAction;
 import org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksAction;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.TransportCancelTasksAction;
+import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction;
+import org.elasticsearch.action.admin.cluster.node.tasks.get.TransportGetTaskAction;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction;
 import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryAction;
@@ -264,6 +266,7 @@ public class ActionModule extends AbstractModule {
         registerAction(NodesStatsAction.INSTANCE, TransportNodesStatsAction.class);
         registerAction(NodesHotThreadsAction.INSTANCE, TransportNodesHotThreadsAction.class);
         registerAction(ListTasksAction.INSTANCE, TransportListTasksAction.class);
+        registerAction(GetTaskAction.INSTANCE, TransportGetTaskAction.class);
         registerAction(CancelTasksAction.INSTANCE, TransportCancelTasksAction.class);
 
         registerAction(ClusterAllocationExplainAction.INSTANCE, TransportClusterAllocationExplainAction.class);
diff --git a/core/src/main/java/org/elasticsearch/action/ActionRequest.java b/core/src/main/java/org/elasticsearch/action/ActionRequest.java
index dac3b4f2a21..bc052895a6f 100644
--- a/core/src/main/java/org/elasticsearch/action/ActionRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/ActionRequest.java
@@ -39,6 +39,9 @@ public abstract class ActionRequest<Request extends ActionRequest<Request>> exte
 
     public abstract ActionRequestValidationException validate();
 
+    /**
+     * Should this task persist its result after it has finished?
+     */
     public boolean getShouldPersistResult() {
         return false;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksResponse.java
index 716e9027bf0..9bfeaecd78b 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/CancelTasksResponse.java
@@ -22,7 +22,7 @@ package org.elasticsearch.action.admin.cluster.node.tasks.cancel;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
-import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo;
+import org.elasticsearch.tasks.TaskInfo;
 
 import java.util.List;
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java
index a52a0358983..bc69228055a 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java
@@ -22,7 +22,6 @@ package org.elasticsearch.action.admin.cluster.node.tasks.cancel;
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.TaskOperationFailure;
-import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
 import org.elasticsearch.cluster.ClusterName;
@@ -36,6 +35,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.tasks.TaskInfo;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.EmptyTransportResponseHandler;
 import org.elasticsearch.transport.TransportChannel;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskAction.java
new file mode 100644
index 00000000000..c30f5e4091b
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskAction.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.node.tasks.get;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.client.ElasticsearchClient;
+
+/**
+ * Action for retrieving a list of currently running tasks
+ */
+public class GetTaskAction extends Action<GetTaskRequest, GetTaskResponse, GetTaskRequestBuilder> {
+
+    public static final GetTaskAction INSTANCE = new GetTaskAction();
+    public static final String NAME = "cluster:monitor/task/get";
+
+    private GetTaskAction() {
+        super(NAME);
+    }
+
+    @Override
+    public GetTaskResponse newResponse() {
+        return new GetTaskResponse();
+    }
+
+    @Override
+    public GetTaskRequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new GetTaskRequestBuilder(client, this);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequest.java
new file mode 100644
index 00000000000..efbc9679e71
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.node.tasks.get;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.tasks.TaskId;
+
+import java.io.IOException;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * A request to get node tasks
+ */
+public class GetTaskRequest extends ActionRequest<GetTaskRequest> {
+    private TaskId taskId = TaskId.EMPTY_TASK_ID;
+    private boolean waitForCompletion = false;
+    private TimeValue timeout = null;
+
+    /**
+     * Get the TaskId to look up.
+     */
+    public TaskId getTaskId() {
+        return taskId;
+    }
+
+    /**
+     * Set the TaskId to look up. Required.
+     */
+    public GetTaskRequest setTaskId(TaskId taskId) {
+        this.taskId = taskId;
+        return this;
+    }
+
+    /**
+     * Should this request wait for all found tasks to complete?
+     */
+    public boolean getWaitForCompletion() {
+        return waitForCompletion;
+    }
+
+    /**
+     * Should this request wait for all found tasks to complete?
+     */
+    public GetTaskRequest setWaitForCompletion(boolean waitForCompletion) {
+        this.waitForCompletion = waitForCompletion;
+        return this;
+    }
+
+    /**
+     * Timeout to wait for any async actions this request must take. It must take anywhere from 0 to 2.
+     */
+    public TimeValue getTimeout() {
+        return timeout;
+    }
+
+    /**
+     * Timeout to wait for any async actions this request must take. It must take anywhere from 0 to 2.
+     */
+    public GetTaskRequest setTimeout(TimeValue timeout) {
+        this.timeout = timeout;
+        return this;
+    }
+
+    GetTaskRequest nodeRequest(String thisNodeId, long thisTaskId) {
+        GetTaskRequest copy = new GetTaskRequest();
+        copy.setParentTask(thisNodeId, thisTaskId);
+        copy.setTaskId(taskId);
+        copy.setTimeout(timeout);
+        copy.setWaitForCompletion(waitForCompletion);
+        return copy;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (false == getTaskId().isSet()) {
+            validationException = addValidationError("task id is required", validationException);
+        }
+        return validationException;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        taskId = TaskId.readFromStream(in);
+        timeout = in.readOptionalWriteable(TimeValue::new);
+        waitForCompletion = in.readBoolean();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        taskId.writeTo(out);
+        out.writeOptionalWriteable(timeout);
+        out.writeBoolean(waitForCompletion);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequestBuilder.java
new file mode 100644
index 00000000000..e1042df2ac3
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskRequestBuilder.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.node.tasks.get;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.tasks.TaskId;
+
+/**
+ * Builder for the request to retrieve the list of tasks running on the specified nodes
+ */
+public class GetTaskRequestBuilder extends ActionRequestBuilder<GetTaskRequest, GetTaskResponse, GetTaskRequestBuilder> {
+    public GetTaskRequestBuilder(ElasticsearchClient client, GetTaskAction action) {
+        super(client, action, new GetTaskRequest());
+    }
+
+    /**
+     * Set the TaskId to look up. Required.
+     */
+    public final GetTaskRequestBuilder setTaskId(TaskId taskId) {
+        request.setTaskId(taskId);
+        return this;
+    }
+
+    /**
+     * Should this request wait for all found tasks to complete?
+     */
+    public final GetTaskRequestBuilder setWaitForCompletion(boolean waitForCompletion) {
+        request.setWaitForCompletion(waitForCompletion);
+        return this;
+    }
+
+    /**
+     * Timeout to wait for any async actions this request must take. It must take anywhere from 0 to 2.
+     */
+    public final GetTaskRequestBuilder setTimeout(TimeValue timeout) {
+        request.setTimeout(timeout);
+        return this;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java
new file mode 100644
index 00000000000..afb03a7c9dc
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/GetTaskResponse.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.node.tasks.get;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.tasks.PersistedTaskInfo;
+
+import java.io.IOException;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * Returns the list of tasks currently running on the nodes
+ */
+public class GetTaskResponse extends ActionResponse implements ToXContent {
+    private PersistedTaskInfo task;
+
+    public GetTaskResponse() {
+    }
+
+    public GetTaskResponse(PersistedTaskInfo task) {
+        this.task = requireNonNull(task, "task is required");
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        task = in.readOptionalWriteable(PersistedTaskInfo::new);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeOptionalWriteable(task);
+    }
+
+    /**
+     * Get the actual result of the fetch.
+     */
+    public PersistedTaskInfo getTask() {
+        return task;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return task.innerToXContent(builder, params);
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
new file mode 100644
index 00000000000..769676bcd6d
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.cluster.node.tasks.get;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.tasks.PersistedTaskInfo;
+import org.elasticsearch.tasks.TaskPersistenceService;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.BaseTransportResponseHandler;
+import org.elasticsearch.transport.TransportException;
+import org.elasticsearch.transport.TransportRequestOptions;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.IOException;
+
+import static org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction.waitForCompletionTimeout;
+import static org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction.waitForTaskCompletion;
+
+/**
+ * Action to get a single task. If the task isn't running then it'll try to request the status from request index.
+ *
+ * The general flow is:
+ * <ul>
+ * <li>If this isn't being executed on the node to which the requested TaskId belongs then move to that node.
+ * <li>Look up the task and return it if it exists
+ * <li>If it doesn't then look up the task from the results index
+ * </ul>
+ */
+public class TransportGetTaskAction extends HandledTransportAction<GetTaskRequest, GetTaskResponse> {
+    private final ClusterService clusterService;
+    private final TransportService transportService;
+    private final Client client;
+
+    @Inject
+    public TransportGetTaskAction(Settings settings, ThreadPool threadPool, TransportService transportService, ActionFilters actionFilters,
+            IndexNameExpressionResolver indexNameExpressionResolver, ClusterService clusterService, Client client) {
+        super(settings, GetTaskAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, GetTaskRequest::new);
+        this.clusterService = clusterService;
+        this.transportService = transportService;
+        this.client = client;
+    }
+
+    @Override
+    protected void doExecute(GetTaskRequest request, ActionListener<GetTaskResponse> listener) {
+        throw new UnsupportedOperationException("Task is required");
+    }
+
+    @Override
+    protected void doExecute(Task thisTask, GetTaskRequest request, ActionListener<GetTaskResponse> listener) {
+        if (clusterService.localNode().getId().equals(request.getTaskId().getNodeId())) {
+            getRunningTaskFromNode(thisTask, request, listener);
+        } else {
+            runOnNodeWithTaskIfPossible(thisTask, request, listener);
+        }
+    }
+
+    /**
+     * Executed on the coordinating node to forward execution of the remaining work to the node that matches that requested
+     * {@link TaskId#getNodeId()}. If the node isn't in the cluster then this will just proceed to
+     * {@link #getFinishedTaskFromIndex(Task, GetTaskRequest, ActionListener)} on this node.
+     */
+    private void runOnNodeWithTaskIfPossible(Task thisTask, GetTaskRequest request, ActionListener<GetTaskResponse> listener) {
+        TransportRequestOptions.Builder builder = TransportRequestOptions.builder();
+        if (request.getTimeout() != null) {
+            builder.withTimeout(request.getTimeout());
+        }
+        builder.withCompress(false);
+        DiscoveryNode node = clusterService.state().nodes().get(request.getTaskId().getNodeId());
+        if (node == null) {
+            // Node is no longer part of the cluster! Try and look the task up from the results index.
+            getFinishedTaskFromIndex(thisTask, request, listener);
+            return;
+        }
+        GetTaskRequest nodeRequest = request.nodeRequest(clusterService.localNode().getId(), thisTask.getId());
+        taskManager.registerChildTask(thisTask, node.getId());
+        transportService.sendRequest(node, GetTaskAction.NAME, nodeRequest, builder.build(),
+                new BaseTransportResponseHandler<GetTaskResponse>() {
+                    @Override
+                    public GetTaskResponse newInstance() {
+                        return new GetTaskResponse();
+                    }
+
+                    @Override
+                    public void handleResponse(GetTaskResponse response) {
+                        listener.onResponse(response);
+                    }
+
+                    @Override
+                    public void handleException(TransportException exp) {
+                        listener.onFailure(exp);
+                    }
+
+                    @Override
+                    public String executor() {
+                        return ThreadPool.Names.SAME;
+                    }
+                });
+    }
+
+    /**
+     * Executed on the node that should be running the task to find and return the running task. Falls back to
+     * {@link #getFinishedTaskFromIndex(Task, GetTaskRequest, ActionListener)} if the task isn't still running.
+     */
+    void getRunningTaskFromNode(Task thisTask, GetTaskRequest request, ActionListener<GetTaskResponse> listener) {
+        Task runningTask = taskManager.getTask(request.getTaskId().getId());
+        if (runningTask == null) {
+            getFinishedTaskFromIndex(thisTask, request, listener);
+        } else {
+            if (request.getWaitForCompletion()) {
+                // Shift to the generic thread pool and let it wait for the task to complete so we don't block any important threads.
+                threadPool.generic().execute(new AbstractRunnable() {
+                    @Override
+                    protected void doRun() throws Exception {
+                        waitForTaskCompletion(taskManager, runningTask, waitForCompletionTimeout(request.getTimeout()));
+                        // TODO look up the task's result from the .tasks index now that it is done
+                        listener.onResponse(
+                                new GetTaskResponse(new PersistedTaskInfo(runningTask.taskInfo(clusterService.localNode(), true))));
+                    }
+
+                    @Override
+                    public void onFailure(Throwable t) {
+                        listener.onFailure(t);
+                    }
+                });
+            } else {
+                listener.onResponse(new GetTaskResponse(new PersistedTaskInfo(runningTask.taskInfo(clusterService.localNode(), true))));
+            }
+        }
+    }
+
+    /**
+     * Send a {@link GetRequest} to the results index looking for the results of the task. It'll only be found only if the task's result was
+     * persisted. Called on the node that once had the task if that node is part of the cluster or on the coordinating node if the node
+     * wasn't part of the cluster.
+     */
+    void getFinishedTaskFromIndex(Task thisTask, GetTaskRequest request, ActionListener<GetTaskResponse> listener) {
+        GetRequest get = new GetRequest(TaskPersistenceService.TASK_INDEX, TaskPersistenceService.TASK_TYPE,
+                request.getTaskId().toString());
+        get.setParentTask(clusterService.localNode().getId(), thisTask.getId());
+        client.get(get, new ActionListener<GetResponse>() {
+            @Override
+            public void onResponse(GetResponse getResponse) {
+                try {
+                    onGetFinishedTaskFromIndex(getResponse, listener);
+                } catch (Throwable e) {
+                    listener.onFailure(e);
+                }
+            }
+
+            @Override
+            public void onFailure(Throwable e) {
+                if (ExceptionsHelper.unwrap(e, IndexNotFoundException.class) != null) {
+                    // We haven't yet created the index for the task results so it can't be found.
+                    listener.onFailure(new ResourceNotFoundException("task [{}] isn't running or persisted", e, request.getTaskId()));
+                } else {
+                    listener.onFailure(e);
+                }
+            }
+        });
+    }
+
+    /**
+     * Called with the {@linkplain GetResponse} from loading the task from the results index. Called on the node that once had the task if
+     * that node is part of the cluster or on the coordinating node if the node wasn't part of the cluster.
+     */
+    void onGetFinishedTaskFromIndex(GetResponse response, ActionListener<GetTaskResponse> listener) throws IOException {
+        if (false == response.isExists()) {
+            listener.onFailure(new ResourceNotFoundException("task [{}] isn't running or persisted", response.getId()));
+        }
+        if (response.isSourceEmpty()) {
+            listener.onFailure(new ElasticsearchException("Stored task status for [{}] didn't contain any source!", response.getId()));
+            return;
+        }
+        try (XContentParser parser = XContentHelper.createParser(response.getSourceAsBytesRef())) {
+            PersistedTaskInfo result = PersistedTaskInfo.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT);
+            listener.onResponse(new GetTaskResponse(result));
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
index bad4001e211..6ab0bafb2fb 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/ListTasksResponse.java
@@ -23,21 +23,21 @@ import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.support.tasks.BaseTasksResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.tasks.TaskInfo;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.stream.Collectors;
 
 /**
@@ -47,10 +47,12 @@ public class ListTasksResponse extends BaseTasksResponse implements ToXContent {
 
     private List<TaskInfo> tasks;
 
-    private Map<DiscoveryNode, List<TaskInfo>> nodes;
+    private Map<String, List<TaskInfo>> perNodeTasks;
 
     private List<TaskGroup> groups;
 
+    private DiscoveryNodes discoveryNodes;
+
     public ListTasksResponse() {
     }
 
@@ -75,28 +77,11 @@ public class ListTasksResponse extends BaseTasksResponse implements ToXContent {
     /**
      * Returns the list of tasks by node
      */
-    public Map<DiscoveryNode, List<TaskInfo>> getPerNodeTasks() {
-        if (nodes != null) {
-            return nodes;
+    public Map<String, List<TaskInfo>> getPerNodeTasks() {
+        if (perNodeTasks == null) {
+            perNodeTasks = tasks.stream().collect(Collectors.groupingBy(t -> t.getTaskId().getNodeId()));
         }
-        Map<DiscoveryNode, List<TaskInfo>> nodeTasks = new HashMap<>();
-
-        Set<DiscoveryNode> nodes = new HashSet<>();
-        for (TaskInfo shard : tasks) {
-            nodes.add(shard.getNode());
-        }
-
-        for (DiscoveryNode node : nodes) {
-            List<TaskInfo> tasks = new ArrayList<>();
-            for (TaskInfo taskInfo : this.tasks) {
-                if (taskInfo.getNode().equals(node)) {
-                    tasks.add(taskInfo);
-                }
-            }
-            nodeTasks.put(node, tasks);
-        }
-        this.nodes = nodeTasks;
-        return nodeTasks;
+        return perNodeTasks;
     }
 
     public List<TaskGroup> getTaskGroups() {
@@ -138,6 +123,14 @@ public class ListTasksResponse extends BaseTasksResponse implements ToXContent {
         return tasks;
     }
 
+    /**
+     * Set a reference to the {@linkplain DiscoveryNodes}. Used for calling {@link #toXContent(XContentBuilder, ToXContent.Params)} with
+     * {@code group_by=nodes}.
+     */
+    public void setDiscoveryNodes(DiscoveryNodes discoveryNodes) {
+        this.discoveryNodes = discoveryNodes;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         if (getTaskFailures() != null && getTaskFailures().size() > 0) {
@@ -161,33 +154,38 @@ public class ListTasksResponse extends BaseTasksResponse implements ToXContent {
         }
         String groupBy = params.param("group_by", "nodes");
         if ("nodes".equals(groupBy)) {
+            if (discoveryNodes == null) {
+                throw new IllegalStateException("discoveryNodes must be set before calling toXContent with group_by=nodes");
+            }
             builder.startObject("nodes");
-            for (Map.Entry<DiscoveryNode, List<TaskInfo>> entry : getPerNodeTasks().entrySet()) {
-                DiscoveryNode node = entry.getKey();
-                builder.startObject(node.getId());
-                builder.field("name", node.getName());
-                builder.field("transport_address", node.getAddress().toString());
-                builder.field("host", node.getHostName());
-                builder.field("ip", node.getAddress());
+            for (Map.Entry<String, List<TaskInfo>> entry : getPerNodeTasks().entrySet()) {
+                DiscoveryNode node = discoveryNodes.get(entry.getKey());
+                builder.startObject(entry.getKey());
+                if (node != null) {
+                    // If the node is no longer part of the cluster, oh well, we'll just skip it's useful information.
+                    builder.field("name", node.getName());
+                    builder.field("transport_address", node.getAddress().toString());
+                    builder.field("host", node.getHostName());
+                    builder.field("ip", node.getAddress());
 
-                builder.startArray("roles");
-                for (DiscoveryNode.Role role : node.getRoles()) {
-                    builder.value(role.getRoleName());
-                }
-                builder.endArray();
-
-                if (!node.getAttributes().isEmpty()) {
-                    builder.startObject("attributes");
-                    for (Map.Entry<String, String> attrEntry : node.getAttributes().entrySet()) {
-                        builder.field(attrEntry.getKey(), attrEntry.getValue());
+                    builder.startArray("roles");
+                    for (DiscoveryNode.Role role : node.getRoles()) {
+                        builder.value(role.getRoleName());
+                    }
+                    builder.endArray();
+
+                    if (!node.getAttributes().isEmpty()) {
+                        builder.startObject("attributes");
+                        for (Map.Entry<String, String> attrEntry : node.getAttributes().entrySet()) {
+                            builder.field(attrEntry.getKey(), attrEntry.getValue());
+                        }
+                        builder.endObject();
                     }
-                    builder.endObject();
                 }
                 builder.startObject("tasks");
                 for(TaskInfo task : entry.getValue()) {
-                    builder.startObject(task.getTaskId().toString());
+                    builder.field(task.getTaskId().toString());
                     task.toXContent(builder, params);
-                    builder.endObject();
                 }
                 builder.endObject();
                 builder.endObject();
@@ -196,9 +194,8 @@ public class ListTasksResponse extends BaseTasksResponse implements ToXContent {
         } else if ("parents".equals(groupBy)) {
             builder.startObject("tasks");
             for (TaskGroup group : getTaskGroups()) {
-                builder.startObject(group.getTaskInfo().getTaskId().toString());
+                builder.field(group.getTaskInfo().getTaskId().toString());
                 group.toXContent(builder, params);
-                builder.endObject();
             }
             builder.endObject();
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskGroup.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskGroup.java
index aa9bfd6b720..b254137163d 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskGroup.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskGroup.java
@@ -21,6 +21,7 @@ package org.elasticsearch.action.admin.cluster.node.tasks.list;
 
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.tasks.TaskInfo;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -79,16 +80,15 @@ public class TaskGroup implements ToXContent {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        task.toXContent(builder, params);
+        builder.startObject();
+        task.innerToXContent(builder, params);
         if (childTasks.isEmpty() == false) {
             builder.startArray("children");
             for (TaskGroup taskGroup : childTasks) {
-                builder.startObject();
                 taskGroup.toXContent(builder, params);
-                builder.endObject();
             }
             builder.endArray();
         }
-        return builder;
+        return builder.endObject();
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
index b05049f6776..7d6d122bbb7 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
@@ -33,6 +33,8 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskInfo;
+import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -47,6 +49,26 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
  *
  */
 public class TransportListTasksAction extends TransportTasksAction<Task, ListTasksRequest, ListTasksResponse, TaskInfo> {
+    public static void waitForTaskCompletion(TaskManager taskManager, Task task, long untilInNanos) {
+        while (System.nanoTime() - untilInNanos < 0) {
+            if (taskManager.getTask(task.getId()) == null) {
+                return;
+            }
+            try {
+                Thread.sleep(WAIT_FOR_COMPLETION_POLL.millis());
+            } catch (InterruptedException e) {
+                throw new ElasticsearchException("Interrupted waiting for completion of [{}]", e, task);
+            }
+        }
+        throw new ElasticsearchTimeoutException("Timed out waiting for completion of [{}]", task);
+    }
+    public static long waitForCompletionTimeout(TimeValue timeout) {
+        if (timeout == null) {
+            timeout = DEFAULT_WAIT_FOR_COMPLETION_TIMEOUT;
+        }
+        return System.nanoTime() + timeout.nanos();
+    }
+
     private static final TimeValue WAIT_FOR_COMPLETION_POLL = timeValueMillis(100);
     private static final TimeValue DEFAULT_WAIT_FOR_COMPLETION_TIMEOUT = timeValueSeconds(30);
 
@@ -75,35 +97,18 @@ public class TransportListTasksAction extends TransportTasksAction<Task, ListTas
 
     @Override
     protected void processTasks(ListTasksRequest request, Consumer<Task> operation) {
-        if (false == request.getWaitForCompletion()) {
-            super.processTasks(request, operation);
-            return;
-        }
-        // If we should wait for completion then we have to intercept every found task and wait for it to leave the manager.
-        TimeValue timeout = request.getTimeout();
-        if (timeout == null) {
-            timeout = DEFAULT_WAIT_FOR_COMPLETION_TIMEOUT;
-        }
-        long timeoutTime = System.nanoTime() + timeout.nanos();
-        super.processTasks(request, operation.andThen((Task t) -> {
-            while (System.nanoTime() - timeoutTime < 0) {
-                Task task = taskManager.getTask(t.getId());
-                if (task == null) {
-                    return;
-                }
+        if (request.getWaitForCompletion()) {
+            long timeoutNanos = waitForCompletionTimeout(request.getTimeout());
+            operation = operation.andThen(task -> {
                 if (task.getAction().startsWith(ListTasksAction.NAME)) {
                     // It doesn't make sense to wait for List Tasks and it can cause an infinite loop of the task waiting
-                    // for itself of one of its child tasks
+                    // for itself or one of its child tasks
                     return;
                 }
-                try {
-                    Thread.sleep(WAIT_FOR_COMPLETION_POLL.millis());
-                } catch (InterruptedException e) {
-                    throw new ElasticsearchException("Interrupted waiting for completion of [{}]", e, t);
-                }
-            }
-            throw new ElasticsearchTimeoutException("Timed out waiting for completion of [{}]", t);
-        }));
+                waitForTaskCompletion(taskManager, task, timeoutNanos);
+            });
+        }
+        super.processTasks(request, operation);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationTask.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationTask.java
index 9fe3da59a1f..b31c2439aaf 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationTask.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationTask.java
@@ -88,5 +88,20 @@ public class ReplicationTask extends Task {
         public void writeTo(StreamOutput out) throws IOException {
             out.writeString(phase);
         }
+
+        // Implements equals and hashcode for testing
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null || obj.getClass() != ReplicationTask.Status.class) {
+                return false;
+            }
+            ReplicationTask.Status other = (Status) obj;
+            return phase.equals(other.phase);
+        }
+
+        @Override
+        public int hashCode() {
+            return phase.hashCode();
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java
index 78a2de20a89..a3528cb75c4 100644
--- a/core/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/support/tasks/TasksRequestBuilder.java
@@ -22,6 +22,7 @@ import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.tasks.TaskId;
 
 /**
  * Builder for task-based requests
@@ -36,6 +37,15 @@ public class TasksRequestBuilder<
         super(client, action, request);
     }
 
+    /**
+     * Set the task to lookup.
+     */
+    @SuppressWarnings("unchecked")
+    public final RequestBuilder setTaskId(TaskId taskId) {
+        request.setTaskId(taskId);
+        return (RequestBuilder) this;
+    }
+
     @SuppressWarnings("unchecked")
     public final RequestBuilder setNodesIds(String... nodesIds) {
         request.setNodesIds(nodesIds);
diff --git a/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java b/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java
index 37886239195..953ca578579 100644
--- a/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java
+++ b/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java
@@ -39,6 +39,9 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequestBuilder;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
+import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequest;
+import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequestBuilder;
+import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequestBuilder;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
@@ -112,6 +115,7 @@ import org.elasticsearch.action.ingest.SimulatePipelineResponse;
 import org.elasticsearch.action.ingest.WritePipelineResponse;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.tasks.TaskId;
 
 /**
  * Administrative actions/operations against indices.
@@ -303,6 +307,34 @@ public interface ClusterAdminClient extends ElasticsearchClient {
      */
     ListTasksRequestBuilder prepareListTasks(String... nodesIds);
 
+    /**
+     * Get a task.
+     *
+     * @param request the request
+     * @return the result future
+     * @see org.elasticsearch.client.Requests#getTaskRequest()
+     */
+    ActionFuture<GetTaskResponse> getTask(GetTaskRequest request);
+
+    /**
+     * Get a task.
+     *
+     * @param request the request
+     * @param listener A listener to be notified with the result
+     * @see org.elasticsearch.client.Requests#getTaskRequest()
+     */
+    void getTask(GetTaskRequest request, ActionListener<GetTaskResponse> listener);
+
+    /**
+     * Fetch a task by id.
+     */
+    GetTaskRequestBuilder prepareGetTask(String taskId);
+
+    /**
+     * Fetch a task by id.
+     */
+    GetTaskRequestBuilder prepareGetTask(TaskId taskId);
+
     /**
      * Cancel tasks
      *
diff --git a/core/src/main/java/org/elasticsearch/client/Requests.java b/core/src/main/java/org/elasticsearch/client/Requests.java
index 276bd9d9062..6d652bf39d0 100644
--- a/core/src/main/java/org/elasticsearch/client/Requests.java
+++ b/core/src/main/java/org/elasticsearch/client/Requests.java
@@ -23,6 +23,7 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
+import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest;
 import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest;
 import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
@@ -406,6 +407,16 @@ public class Requests {
         return new ListTasksRequest();
     }
 
+    /**
+     * Creates a get task request.
+     *
+     * @return The nodes tasks request
+     * @see org.elasticsearch.client.ClusterAdminClient#getTask(GetTaskRequest)
+     */
+    public static GetTaskRequest getTaskRequest() {
+        return new GetTaskRequest();
+    }
+
     /**
      * Creates a nodes tasks request against one or more nodes. Pass <tt>null</tt> or an empty array for all nodes.
      *
diff --git a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
index 6481bec1b83..8ca6d502e3c 100644
--- a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
+++ b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
@@ -49,6 +49,10 @@ import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksActio
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequestBuilder;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
+import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskAction;
+import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequest;
+import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequestBuilder;
+import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequestBuilder;
@@ -109,6 +113,18 @@ import org.elasticsearch.action.admin.cluster.stats.ClusterStatsAction;
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequest;
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequestBuilder;
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
+import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptAction;
+import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
+import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequestBuilder;
+import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptResponse;
+import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptAction;
+import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
+import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequestBuilder;
+import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse;
+import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptAction;
+import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest;
+import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequestBuilder;
+import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptResponse;
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksAction;
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequestBuilder;
@@ -272,18 +288,6 @@ import org.elasticsearch.action.index.IndexAction;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptAction;
-import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
-import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequestBuilder;
-import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptResponse;
-import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptAction;
-import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
-import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequestBuilder;
-import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse;
-import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptAction;
-import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest;
-import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequestBuilder;
-import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptResponse;
 import org.elasticsearch.action.ingest.DeletePipelineAction;
 import org.elasticsearch.action.ingest.DeletePipelineRequest;
 import org.elasticsearch.action.ingest.DeletePipelineRequestBuilder;
@@ -339,6 +343,7 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Map;
@@ -851,6 +856,25 @@ public abstract class AbstractClient extends AbstractComponent implements Client
             return new ListTasksRequestBuilder(this, ListTasksAction.INSTANCE).setNodesIds(nodesIds);
         }
 
+        @Override
+        public ActionFuture<GetTaskResponse> getTask(final GetTaskRequest request) {
+            return execute(GetTaskAction.INSTANCE, request);
+        }
+
+        @Override
+        public void getTask(final GetTaskRequest request, final ActionListener<GetTaskResponse> listener) {
+            execute(GetTaskAction.INSTANCE, request, listener);
+        }
+
+        @Override
+        public GetTaskRequestBuilder prepareGetTask(String taskId) {
+            return prepareGetTask(new TaskId(taskId));
+        }
+
+        @Override
+        public GetTaskRequestBuilder prepareGetTask(TaskId taskId) {
+            return new GetTaskRequestBuilder(this, GetTaskAction.INSTANCE).setTaskId(taskId);
+        }
 
         @Override
         public ActionFuture<CancelTasksResponse> cancelTasks(CancelTasksRequest request) {
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index b9084d52a91..c90dbcd3c54 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -63,7 +63,7 @@ import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.ExtensionPoint;
 import org.elasticsearch.gateway.GatewayAllocator;
-import org.elasticsearch.tasks.TaskResultsService;
+import org.elasticsearch.tasks.TaskPersistenceService;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -156,6 +156,6 @@ public class ClusterModule extends AbstractModule {
         bind(ShardStateAction.class).asEagerSingleton();
         bind(NodeMappingRefreshAction.class).asEagerSingleton();
         bind(MappingUpdatedAction.class).asEagerSingleton();
-        bind(TaskResultsService.class).asEagerSingleton();
+        bind(TaskPersistenceService.class).asEagerSingleton();
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
index fd9191fd713..1484ad64018 100644
--- a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
+++ b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
@@ -49,6 +49,7 @@ import org.elasticsearch.rest.action.admin.cluster.node.hotthreads.RestNodesHotT
 import org.elasticsearch.rest.action.admin.cluster.node.info.RestNodesInfoAction;
 import org.elasticsearch.rest.action.admin.cluster.node.stats.RestNodesStatsAction;
 import org.elasticsearch.rest.action.admin.cluster.node.tasks.RestCancelTasksAction;
+import org.elasticsearch.rest.action.admin.cluster.node.tasks.RestGetTaskAction;
 import org.elasticsearch.rest.action.admin.cluster.node.tasks.RestListTasksAction;
 import org.elasticsearch.rest.action.admin.cluster.repositories.delete.RestDeleteRepositoryAction;
 import org.elasticsearch.rest.action.admin.cluster.repositories.get.RestGetRepositoriesAction;
@@ -146,6 +147,7 @@ import org.elasticsearch.rest.action.suggest.RestSuggestAction;
 import org.elasticsearch.rest.action.termvectors.RestMultiTermVectorsAction;
 import org.elasticsearch.rest.action.termvectors.RestTermVectorsAction;
 import org.elasticsearch.rest.action.update.RestUpdateAction;
+import org.elasticsearch.tasks.RawTaskStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportService;
@@ -277,6 +279,7 @@ public class NetworkModule extends AbstractModule {
 
         // Tasks API
         RestListTasksAction.class,
+        RestGetTaskAction.class,
         RestCancelTasksAction.class,
 
         // Ingest API
@@ -339,6 +342,7 @@ public class NetworkModule extends AbstractModule {
         registerTransport(LOCAL_TRANSPORT, LocalTransport.class);
         registerTransport(NETTY_TRANSPORT, NettyTransport.class);
         registerTaskStatus(ReplicationTask.Status.NAME, ReplicationTask.Status::new);
+        registerTaskStatus(RawTaskStatus.NAME, RawTaskStatus::new);
         registerBuiltinAllocationCommands();
 
         if (transportClient == false) {
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java
index 0fb20aa519b..6f8a606d9a6 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/AbstractObjectParser.java
@@ -21,7 +21,9 @@ package org.elasticsearch.common.xcontent;
 
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParseFieldMatcherSupplier;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -123,6 +125,17 @@ public abstract class AbstractObjectParser<Value, Context extends ParseFieldMatc
         declareField(consumer, (p, c) -> parseArray(p, p::intValue), field, ValueType.INT_ARRAY);
     }
 
+    public void declareRawObject(BiConsumer<Value, BytesReference> consumer, ParseField field) {
+        NoContextParser<BytesReference> bytesParser = p -> {
+            try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+                builder.prettyPrint();
+                builder.copyCurrentStructure(p);
+                return builder.bytes();
+            }
+        };
+        declareField(consumer, bytesParser, field, ValueType.OBJECT);
+    }
+
     private interface IOSupplier<T> {
         T get() throws IOException;
     }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java
index 56baf30c9e0..f86b44b4e6f 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java
@@ -328,6 +328,10 @@ public class JsonXContentGenerator implements XContentGenerator {
         if (mayWriteRawData(contentType) == false) {
             copyRawValue(content, contentType.xContent());
         } else {
+            if (generator.getOutputContext().getCurrentName() != null) {
+                // If we've just started a field we'll need to add the separator
+                generator.writeRaw(':');
+            }
             flush();
             content.writeTo(os);
             writeEndRaw();
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index 04063ce5864..b6bfba04ff5 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -97,7 +97,7 @@ import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.snapshots.SnapshotShardsService;
 import org.elasticsearch.snapshots.SnapshotsService;
-import org.elasticsearch.tasks.TaskResultsService;
+import org.elasticsearch.tasks.TaskPersistenceService;
 import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.threadpool.ThreadPoolModule;
@@ -340,7 +340,7 @@ public class Node implements Closeable {
 
         // Start the transport service now so the publish address will be added to the local disco node in ClusterService
         TransportService transportService = injector.getInstance(TransportService.class);
-        transportService.getTaskManager().setTaskResultsService(injector.getInstance(TaskResultsService.class));
+        transportService.getTaskManager().setTaskResultsService(injector.getInstance(TaskPersistenceService.class));
         transportService.start();
 
         validateNodeBeforeAcceptingRequests(settings, transportService.boundAddress());
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestCancelTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestCancelTasksAction.java
index 658090bb6db..e84dfbfc24c 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestCancelTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestCancelTasksAction.java
@@ -19,9 +19,11 @@
 
 package org.elasticsearch.rest.action.admin.cluster.node.tasks;
 
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
-import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest;
+import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -33,13 +35,16 @@ import org.elasticsearch.rest.action.support.RestToXContentListener;
 import org.elasticsearch.tasks.TaskId;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.rest.action.admin.cluster.node.tasks.RestListTasksAction.nodeSettingListener;
 
 
 public class RestCancelTasksAction extends BaseRestHandler {
+    private final ClusterService clusterService;
 
     @Inject
-    public RestCancelTasksAction(Settings settings, RestController controller, Client client) {
+    public RestCancelTasksAction(Settings settings, RestController controller, Client client, ClusterService clusterService) {
         super(settings, client);
+        this.clusterService = clusterService;
         controller.registerHandler(POST, "/_tasks/_cancel", this);
         controller.registerHandler(POST, "/_tasks/{taskId}/_cancel", this);
     }
@@ -56,6 +61,7 @@ public class RestCancelTasksAction extends BaseRestHandler {
         cancelTasksRequest.setNodesIds(nodesIds);
         cancelTasksRequest.setActions(actions);
         cancelTasksRequest.setParentTaskId(parentTaskId);
-        client.admin().cluster().cancelTasks(cancelTasksRequest, new RestToXContentListener<>(channel));
+        ActionListener<CancelTasksResponse> listener = nodeSettingListener(clusterService, new RestToXContentListener<>(channel));
+        client.admin().cluster().cancelTasks(cancelTasksRequest, listener);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestGetTaskAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestGetTaskAction.java
new file mode 100644
index 00000000000..e5617711014
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestGetTaskAction.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.action.admin.cluster.node.tasks;
+
+import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.support.RestToXContentListener;
+import org.elasticsearch.tasks.TaskId;
+
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+
+public class RestGetTaskAction extends BaseRestHandler {
+    @Inject
+    public RestGetTaskAction(Settings settings, RestController controller, Client client) {
+        super(settings, client);
+        controller.registerHandler(GET, "/_tasks/{taskId}", this);
+    }
+
+    @Override
+    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+        TaskId taskId = new TaskId(request.param("taskId"));
+        boolean waitForCompletion = request.paramAsBoolean("wait_for_completion", false);
+        TimeValue timeout = request.paramAsTime("timeout", null);
+
+        GetTaskRequest getTaskRequest = new GetTaskRequest();
+        getTaskRequest.setTaskId(taskId);
+        getTaskRequest.setWaitForCompletion(waitForCompletion);
+        getTaskRequest.setTimeout(timeout);
+        client.admin().cluster().getTask(getTaskRequest, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java
index 1227e7c10d5..4e20ca980c6 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java
@@ -19,8 +19,11 @@
 
 package org.elasticsearch.rest.action.admin.cluster.node.tasks;
 
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -36,25 +39,24 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 
 public class RestListTasksAction extends BaseRestHandler {
+    private final ClusterService clusterService;
 
     @Inject
-    public RestListTasksAction(Settings settings, RestController controller, Client client) {
+    public RestListTasksAction(Settings settings, RestController controller, Client client, ClusterService clusterService) {
         super(settings, client);
+        this.clusterService = clusterService;
         controller.registerHandler(GET, "/_tasks", this);
-        controller.registerHandler(GET, "/_tasks/{taskId}", this);
     }
 
     public static ListTasksRequest generateListTasksRequest(RestRequest request) {
         boolean detailed = request.paramAsBoolean("detailed", false);
         String[] nodesIds = Strings.splitStringByCommaToArray(request.param("node_id"));
-        TaskId taskId = new TaskId(request.param("taskId", request.param("task_id")));
         String[] actions = Strings.splitStringByCommaToArray(request.param("actions"));
         TaskId parentTaskId = new TaskId(request.param("parent_task_id"));
         boolean waitForCompletion = request.paramAsBoolean("wait_for_completion", false);
         TimeValue timeout = request.paramAsTime("timeout", null);
 
         ListTasksRequest listTasksRequest = new ListTasksRequest();
-        listTasksRequest.setTaskId(taskId);
         listTasksRequest.setNodesIds(nodesIds);
         listTasksRequest.setDetailed(detailed);
         listTasksRequest.setActions(actions);
@@ -66,6 +68,27 @@ public class RestListTasksAction extends BaseRestHandler {
 
     @Override
     public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
-        client.admin().cluster().listTasks(generateListTasksRequest(request), new RestToXContentListener<>(channel));
+        ActionListener<ListTasksResponse> listener = nodeSettingListener(clusterService, new RestToXContentListener<>(channel));
+        client.admin().cluster().listTasks(generateListTasksRequest(request), listener);
+    }
+
+    /**
+     * Wrap the normal channel listener in one that sets the discovery nodes on the response so we can support all of it's toXContent
+     * formats.
+     */
+    public static <T extends ListTasksResponse> ActionListener<T> nodeSettingListener(ClusterService clusterService,
+            ActionListener<T> channelListener) {
+        return new ActionListener<T>() {
+            @Override
+            public void onResponse(T response) {
+                response.setDiscoveryNodes(clusterService.state().nodes());
+                channelListener.onResponse(response);
+            }
+
+            @Override
+            public void onFailure(Throwable e) {
+                channelListener.onFailure(e);
+            }
+        };
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java
index 9e150e6178a..9bb01a7b166 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.rest.action.cat;
 
-import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskGroup;
-import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.inject.Inject;
@@ -37,7 +37,7 @@ import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.action.support.RestResponseListener;
 import org.elasticsearch.rest.action.support.RestTable;
-import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.tasks.TaskInfo;
 import org.joda.time.format.DateTimeFormat;
 import org.joda.time.format.DateTimeFormatter;
 
@@ -48,11 +48,13 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.action.admin.cluster.node.tasks.RestListTasksAction.generateListTasksRequest;
 
 public class RestTasksAction extends AbstractCatAction {
+    private final ClusterService clusterService;
 
     @Inject
-    public RestTasksAction(Settings settings, RestController controller, Client client) {
+    public RestTasksAction(Settings settings, RestController controller, Client client, ClusterService clusterService) {
         super(settings, controller, client);
         controller.registerHandler(GET, "/_cat/tasks", this);
+        this.clusterService = clusterService;
     }
 
     @Override
@@ -104,9 +106,10 @@ public class RestTasksAction extends AbstractCatAction {
 
     private DateTimeFormatter dateFormat = DateTimeFormat.forPattern("HH:mm:ss");
 
-    private void buildRow(Table table, boolean fullId, boolean detailed, TaskInfo taskInfo) {
+    private void buildRow(Table table, boolean fullId, boolean detailed, DiscoveryNodes discoveryNodes, TaskInfo taskInfo) {
         table.startRow();
-        DiscoveryNode node = taskInfo.getNode();
+        String nodeId = taskInfo.getTaskId().getNodeId();
+        DiscoveryNode node = discoveryNodes.get(nodeId);
 
         table.addCell(taskInfo.getId());
         table.addCell(taskInfo.getAction());
@@ -122,15 +125,16 @@ public class RestTasksAction extends AbstractCatAction {
         table.addCell(taskInfo.getRunningTimeNanos());
         table.addCell(TimeValue.timeValueNanos(taskInfo.getRunningTimeNanos()).toString());
 
-        table.addCell(fullId ? node.getId() : Strings.substring(node.getId(), 0, 4));
-        table.addCell(node.getHostAddress());
-        if (node.getAddress() instanceof InetSocketTransportAddress) {
+        // Node information. Note that the node may be null because it has left the cluster between when we got this response and now.
+        table.addCell(fullId ? nodeId : Strings.substring(nodeId, 0, 4));
+        table.addCell(node == null ? "-" : node.getHostAddress());
+        if (node != null && node.getAddress() instanceof InetSocketTransportAddress) {
             table.addCell(((InetSocketTransportAddress) node.getAddress()).address().getPort());
         } else {
             table.addCell("-");
         }
-        table.addCell(node.getName());
-        table.addCell(node.getVersion().toString());
+        table.addCell(node == null ? "-" : node.getName());
+        table.addCell(node == null ? "-" : node.getVersion().toString());
 
         if (detailed) {
             table.addCell(taskInfo.getDescription());
@@ -139,10 +143,11 @@ public class RestTasksAction extends AbstractCatAction {
     }
 
     private void buildGroups(Table table, boolean detailed, boolean fullId, List<TaskGroup> taskGroups) {
+        DiscoveryNodes discoveryNodes = clusterService.state().nodes();
         List<TaskGroup> sortedGroups = new ArrayList<>(taskGroups);
         sortedGroups.sort((o1, o2) -> Long.compare(o1.getTaskInfo().getStartTime(), o2.getTaskInfo().getStartTime()));
         for (TaskGroup taskGroup : sortedGroups) {
-            buildRow(table, fullId, detailed, taskGroup.getTaskInfo());
+            buildRow(table, fullId, detailed, discoveryNodes, taskGroup.getTaskInfo());
             buildGroups(table, fullId, detailed, taskGroup.getChildTasks());
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/tasks/PersistedTaskInfo.java b/core/src/main/java/org/elasticsearch/tasks/PersistedTaskInfo.java
new file mode 100644
index 00000000000..7d551ce195c
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/tasks/PersistedTaskInfo.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.tasks;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.client.Requests;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcherSupplier;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+
+import static java.util.Objects.requireNonNull;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+import static org.elasticsearch.common.xcontent.XContentHelper.convertToMap;
+
+/**
+ * Information about a persisted or running task. Running tasks just have a {@link #getTask()} while persisted tasks will have either a
+ * {@link #getError()} or {@link #getResult()}.
+ */
+public final class PersistedTaskInfo implements Writeable, ToXContent {
+    private final TaskInfo task;
+    @Nullable
+    private final BytesReference error;
+    @Nullable
+    private final BytesReference result;
+
+    /**
+     * Construct a {@linkplain PersistedTaskInfo} for a running task.
+     */
+    public PersistedTaskInfo(TaskInfo task) {
+        this(task, null, null);
+    }
+
+    /**
+     * Construct a {@linkplain PersistedTaskInfo} for a task that completed with an error.
+     */
+    public PersistedTaskInfo(TaskInfo task, Throwable error) throws IOException {
+        this(task, toXContent(error), null);
+    }
+
+    /**
+     * Construct a {@linkplain PersistedTaskInfo} for a task that completed successfully.
+     */
+    public PersistedTaskInfo(TaskInfo task, ToXContent result) throws IOException {
+        this(task, null, toXContent(result));
+    }
+
+    private PersistedTaskInfo(TaskInfo task, @Nullable BytesReference error, @Nullable BytesReference result) {
+        this.task = requireNonNull(task, "task is required");
+        this.error = error;
+        this.result = result;
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public PersistedTaskInfo(StreamInput in) throws IOException {
+        task = new TaskInfo(in);
+        error = in.readOptionalBytesReference();
+        result = in.readOptionalBytesReference();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        task.writeTo(out);
+        out.writeOptionalBytesReference(error);
+        out.writeOptionalBytesReference(result);
+    }
+
+    /**
+     * Get the task that this wraps.
+     */
+    public TaskInfo getTask() {
+        return task;
+    }
+
+    /**
+     * Get the error that finished this task. Will return null if the task didn't finish with an error or it hasn't yet finished.
+     */
+    public BytesReference getError() {
+        return error;
+    }
+
+    /**
+     * Convert {@link #getError()} from XContent to a Map for easy processing. Will return null if the task didn't finish with an error or
+     * hasn't yet finished.
+     */
+    public Map<String, Object> getErrorAsMap() {
+        if (error == null) {
+            return null;
+        }
+        return convertToMap(error, false).v2();
+    }
+
+    /**
+     * Get the result that this task finished with. Will return null if the task was finished by an error or it hasn't yet finished.
+     */
+    public BytesReference getResult() {
+        return result;
+    }
+
+    /**
+     * Convert {@link #getResult()} from XContent to a Map for easy processing. Will return null if the task was finished with an error or
+     * hasn't yet finished.
+     */
+    public Map<String, Object> getResultAsMap() {
+        if (result == null) {
+            return null;
+        }
+        return convertToMap(result, false).v2();
+    }
+
+    /**
+     * Was the task completed before returned?
+     */
+    public boolean isCompleted() {
+        return error != null || result != null;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        innerToXContent(builder, params);
+        return builder.endObject();
+    }
+
+    public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field("task", task);
+        if (error != null) {
+            XContentHelper.writeRawField("error", error, builder, params);
+        }
+        if (result != null) {
+            XContentHelper.writeRawField("result", result, builder, params);
+        }
+        return builder;
+    }
+
+    public static final ConstructingObjectParser<PersistedTaskInfo, ParseFieldMatcherSupplier> PARSER = new ConstructingObjectParser<>(
+            "persisted_task_info", a -> new PersistedTaskInfo((TaskInfo) a[0], (BytesReference) a[1], (BytesReference) a[2]));
+    static {
+        PARSER.declareObject(constructorArg(), TaskInfo.PARSER, new ParseField("task"));
+        PARSER.declareRawObject(optionalConstructorArg(), new ParseField("error"));
+        PARSER.declareRawObject(optionalConstructorArg(), new ParseField("result"));
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+    // Implements equals and hashcode for testing
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || obj.getClass() != PersistedTaskInfo.class) {
+            return false;
+        }
+        PersistedTaskInfo other = (PersistedTaskInfo) obj;
+        /*
+         * Equality of error and result is done by converting them to a map first. Not efficient but ignores field order and spacing
+         * differences so perfect for testing.
+         */
+        return Objects.equals(task, other.task)
+                && Objects.equals(getErrorAsMap(), other.getErrorAsMap())
+                && Objects.equals(getResultAsMap(), other.getResultAsMap());
+    }
+
+    @Override
+    public int hashCode() {
+        /*
+         * Hashing of error and result is done by converting them to a map first. Not efficient but ignores field order and spacing
+         * differences so perfect for testing.
+         */
+        return Objects.hash(task, getErrorAsMap(), getResultAsMap());
+    }
+
+    private static BytesReference toXContent(ToXContent result) throws IOException {
+        try (XContentBuilder builder = XContentFactory.contentBuilder(Requests.INDEX_CONTENT_TYPE)) {
+            // Elasticsearch's Response object never emit starting or ending objects. Most other implementers of ToXContent do....
+            builder.startObject();
+            result.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            builder.endObject();
+            return builder.bytes();
+        }
+    }
+
+    private static BytesReference toXContent(Throwable error) throws IOException {
+        try (XContentBuilder builder = XContentFactory.contentBuilder(Requests.INDEX_CONTENT_TYPE)) {
+            builder.startObject();
+            ElasticsearchException.toXContent(builder, ToXContent.EMPTY_PARAMS, error);
+            builder.endObject();
+            return builder.bytes(); 
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/tasks/RawTaskStatus.java b/core/src/main/java/org/elasticsearch/tasks/RawTaskStatus.java
new file mode 100644
index 00000000000..a540dbd8e3c
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/tasks/RawTaskStatus.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.tasks;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static java.util.Objects.requireNonNull;
+import static org.elasticsearch.common.xcontent.XContentHelper.convertToMap;
+
+/**
+ * Raw, unparsed status from the task results index.
+ */
+public class RawTaskStatus implements Task.Status {
+    public static final String NAME = "raw";
+
+    private final BytesReference status;
+
+    public RawTaskStatus(BytesReference status) {
+        this.status = requireNonNull(status, "status may not be null");
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public RawTaskStatus(StreamInput in) throws IOException {
+        status = in.readOptionalBytesReference();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeOptionalBytesReference(status);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return builder.rawValue(status);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+    /**
+     * Convert the from XContent to a Map for easy reading.
+     */
+    public Map<String, Object> toMap() {
+        return convertToMap(status, false).v2();
+    }
+
+    // Implements equals and hashcode for testing
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || obj.getClass() != RawTaskStatus.class) {
+            return false;
+        }
+        RawTaskStatus other = (RawTaskStatus) obj;
+        // Totally not efficient, but ok for testing because it ignores order and spacing differences
+        return toMap().equals(other.toMap());
+    }
+
+    @Override
+    public int hashCode() {
+        // Totally not efficient, but ok for testing because consistent with equals
+        return toMap().hashCode();
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/tasks/Task.java b/core/src/main/java/org/elasticsearch/tasks/Task.java
index 9c3f4cc842d..8c0b599e933 100644
--- a/core/src/main/java/org/elasticsearch/tasks/Task.java
+++ b/core/src/main/java/org/elasticsearch/tasks/Task.java
@@ -21,7 +21,6 @@
 package org.elasticsearch.tasks;
 
 import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -78,8 +77,8 @@ public class Task {
             description = getDescription();
             status = getStatus();
         }
-        return new TaskInfo(node, getId(), getType(), getAction(), description, status, startTime, System.nanoTime() - startTimeNanos,
-            this instanceof CancellableTask, parentTask);
+        return new TaskInfo(new TaskId(node.getId(), getId()), getType(), getAction(), description, status, startTime,
+                System.nanoTime() - startTimeNanos, this instanceof CancellableTask, parentTask);
     }
 
     /**
@@ -136,13 +135,13 @@ public class Task {
 
     public interface Status extends ToXContent, NamedWriteable {}
 
-    public TaskResult result(DiscoveryNode node, Throwable error) throws IOException {
-        return new TaskResult(taskInfo(node, true), error);
+    public PersistedTaskInfo result(DiscoveryNode node, Throwable error) throws IOException {
+        return new PersistedTaskInfo(taskInfo(node, true), error);
     }
 
-    public TaskResult result(DiscoveryNode node, ActionResponse response) throws IOException {
+    public PersistedTaskInfo result(DiscoveryNode node, ActionResponse response) throws IOException {
         if (response instanceof ToXContent) {
-            return new TaskResult(taskInfo(node, true), (ToXContent) response);
+            return new PersistedTaskInfo(taskInfo(node, true), (ToXContent) response);
         } else {
             throw new IllegalStateException("response has to implement ToXContent for persistence");
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskInfo.java b/core/src/main/java/org/elasticsearch/tasks/TaskInfo.java
similarity index 54%
rename from core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskInfo.java
rename to core/src/main/java/org/elasticsearch/tasks/TaskInfo.java
index adc3afb1bcd..1bd5f1bfc6a 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TaskInfo.java
+++ b/core/src/main/java/org/elasticsearch/tasks/TaskInfo.java
@@ -17,20 +17,27 @@
  * under the License.
  */
 
-package org.elasticsearch.action.admin.cluster.node.tasks.list;
+package org.elasticsearch.tasks;
 
-import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcherSupplier;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.tasks.Task;
-import org.elasticsearch.tasks.TaskId;
 
 import java.io.IOException;
+import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+
 /**
  * Information about a currently running task.
  * <p>
@@ -39,10 +46,7 @@ import java.util.concurrent.TimeUnit;
  * and use in APIs. Instead, immutable and streamable TaskInfo objects are used to represent
  * snapshot information about currently running tasks.
  */
-public class TaskInfo implements Writeable, ToXContent {
-
-    private final DiscoveryNode node;
-
+public final class TaskInfo implements Writeable, ToXContent {
     private final TaskId taskId;
 
     private final String type;
@@ -61,10 +65,9 @@ public class TaskInfo implements Writeable, ToXContent {
 
     private final TaskId parentTaskId;
 
-    public TaskInfo(DiscoveryNode node, long id, String type, String action, String description, Task.Status status, long startTime,
+    public TaskInfo(TaskId taskId, String type, String action, String description, Task.Status status, long startTime,
                     long runningTimeNanos, boolean cancellable, TaskId parentTaskId) {
-        this.node = node;
-        this.taskId = new TaskId(node.getId(), id);
+        this.taskId = taskId;
         this.type = type;
         this.action = action;
         this.description = description;
@@ -79,8 +82,7 @@ public class TaskInfo implements Writeable, ToXContent {
      * Read from a stream.
      */
     public TaskInfo(StreamInput in) throws IOException {
-        node = new DiscoveryNode(in);
-        taskId = new TaskId(node.getId(), in.readLong());
+        taskId = TaskId.readFromStream(in);
         type = in.readString();
         action = in.readString();
         description = in.readOptionalString();
@@ -93,8 +95,7 @@ public class TaskInfo implements Writeable, ToXContent {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        node.writeTo(out);
-        out.writeLong(taskId.getId());
+        taskId.writeTo(out);
         out.writeString(type);
         out.writeString(action);
         out.writeOptionalString(description);
@@ -109,10 +110,6 @@ public class TaskInfo implements Writeable, ToXContent {
         return taskId;
     }
 
-    public DiscoveryNode getNode() {
-        return node;
-    }
-
     public long getId() {
         return taskId.getId();
     }
@@ -167,7 +164,13 @@ public class TaskInfo implements Writeable, ToXContent {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field("node", node.getId());
+        builder.startObject();
+        innerToXContent(builder, params);
+        return builder.endObject();
+    }
+
+    public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field("node", taskId.getNodeId());
         builder.field("id", taskId.getId());
         builder.field("type", type);
         builder.field("action", action);
@@ -185,4 +188,63 @@ public class TaskInfo implements Writeable, ToXContent {
         }
         return builder;
     }
+
+    public static final ConstructingObjectParser<TaskInfo, ParseFieldMatcherSupplier> PARSER = new ConstructingObjectParser<>(
+            "task_info", a -> {
+                int i = 0;
+                TaskId id = new TaskId((String) a[i++], (Long) a[i++]);
+                String type = (String) a[i++];
+                String action = (String) a[i++];
+                String description = (String) a[i++];
+                BytesReference statusBytes = (BytesReference) a[i++];
+                long startTime = (Long) a[i++];
+                long runningTimeNanos = (Long) a[i++];
+                boolean cancellable = (Boolean) a[i++];
+                String parentTaskIdString = (String) a[i++];
+
+                RawTaskStatus status = statusBytes == null ? null : new RawTaskStatus(statusBytes);
+                TaskId parentTaskId = parentTaskIdString == null ? TaskId.EMPTY_TASK_ID : new TaskId((String) parentTaskIdString);
+                return new TaskInfo(id, type, action, description, status, startTime, runningTimeNanos, cancellable, parentTaskId);
+            });
+    static {
+        // Note for the future: this has to be backwards compatible with all changes to the task persistence format
+        PARSER.declareString(constructorArg(), new ParseField("node"));
+        PARSER.declareLong(constructorArg(), new ParseField("id"));
+        PARSER.declareString(constructorArg(), new ParseField("type"));
+        PARSER.declareString(constructorArg(), new ParseField("action"));
+        PARSER.declareString(optionalConstructorArg(), new ParseField("description"));
+        PARSER.declareRawObject(optionalConstructorArg(), new ParseField("status"));
+        PARSER.declareLong(constructorArg(), new ParseField("start_time_in_millis"));
+        PARSER.declareLong(constructorArg(), new ParseField("running_time_in_nanos"));
+        PARSER.declareBoolean(constructorArg(), new ParseField("cancellable"));
+        PARSER.declareString(optionalConstructorArg(), new ParseField("parent_task_id"));
+    }
+
+    @Override
+    public String toString() {
+        return Strings.toString(this);
+    }
+
+    // Implements equals and hashCode for testing
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || obj.getClass() != TaskInfo.class) {
+            return false;
+        }
+        TaskInfo other = (TaskInfo) obj;
+        return Objects.equals(taskId, other.taskId)
+                && Objects.equals(type, other.type)
+                && Objects.equals(action, other.action)
+                && Objects.equals(description, other.description)
+                && Objects.equals(startTime, other.startTime)
+                && Objects.equals(runningTimeNanos, other.runningTimeNanos)
+                && Objects.equals(parentTaskId, other.parentTaskId)
+                && Objects.equals(cancellable, other.cancellable)
+                && Objects.equals(status, other.status);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(taskId, type, action, description, startTime, runningTimeNanos, parentTaskId, cancellable, status);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskManager.java b/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
index c44eb65301a..045522f3288 100644
--- a/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
+++ b/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
@@ -57,7 +57,7 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen
 
     private final Map<TaskId, String> banedParents = new ConcurrentHashMap<>();
 
-    private TaskResultsService taskResultsService;
+    private TaskPersistenceService taskResultsService;
 
     private DiscoveryNodes lastDiscoveryNodes = DiscoveryNodes.EMPTY_NODES;
 
@@ -65,7 +65,7 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen
         super(settings);
     }
 
-    public void setTaskResultsService(TaskResultsService taskResultsService) {
+    public void setTaskResultsService(TaskPersistenceService taskResultsService) {
         assert this.taskResultsService == null;
         this.taskResultsService = taskResultsService;
     }
@@ -145,14 +145,14 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen
     /**
      * Stores the task failure
      */
-    public <Response extends  ActionResponse> void persistResult(Task task, Throwable error, ActionListener<Response> listener) {
+    public <Response extends ActionResponse> void persistResult(Task task, Throwable error, ActionListener<Response> listener) {
         DiscoveryNode localNode = lastDiscoveryNodes.getLocalNode();
         if (localNode == null) {
             // too early to persist anything, shouldn't really be here - just pass the error along
             listener.onFailure(error);
             return;
         }
-        final TaskResult taskResult;
+        final PersistedTaskInfo taskResult;
         try {
             taskResult = task.result(localNode, error);
         } catch (IOException ex) {
@@ -177,7 +177,7 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen
     /**
      * Stores the task result
      */
-    public <Response extends  ActionResponse> void persistResult(Task task, Response response, ActionListener<Response> listener) {
+    public <Response extends ActionResponse> void persistResult(Task task, Response response, ActionListener<Response> listener) {
         DiscoveryNode localNode = lastDiscoveryNodes.getLocalNode();
         if (localNode == null) {
             // too early to persist anything, shouldn't really be here - just pass the response along
@@ -185,7 +185,7 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen
             listener.onResponse(response);
             return;
         }
-        final TaskResult taskResult;
+        final PersistedTaskInfo taskResult;
         try {
             taskResult = task.result(localNode, response);
         } catch (IOException ex) {
diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskResultsService.java b/core/src/main/java/org/elasticsearch/tasks/TaskPersistenceService.java
similarity index 71%
rename from core/src/main/java/org/elasticsearch/tasks/TaskResultsService.java
rename to core/src/main/java/org/elasticsearch/tasks/TaskPersistenceService.java
index c0f63868d3d..e1a26663e7b 100644
--- a/core/src/main/java/org/elasticsearch/tasks/TaskResultsService.java
+++ b/core/src/main/java/org/elasticsearch/tasks/TaskPersistenceService.java
@@ -19,14 +19,17 @@
 package org.elasticsearch.tasks;
 
 import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
+import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -34,21 +37,25 @@ import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.indices.IndexAlreadyExistsException;
 
 import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.io.InputStream;
 
 /**
- * Service that can persist task results
+ * Service that can persist tasks and their results.
  */
-public class TaskResultsService extends AbstractComponent {
+public class TaskPersistenceService extends AbstractComponent {
 
-    public static final String TASK_RESULT_INDEX = ".results";
+    public static final String TASK_INDEX = ".tasks";
 
-    public static final String TASK_RESULT_TYPE = "result";
+    public static final String TASK_TYPE = "task";
 
-    public static final String TASK_RESULT_INDEX_MAPPING_FILE = "task-results-index-mapping.json";
+    public static final String TASK_RESULT_INDEX_MAPPING_FILE = "task-index-mapping.json";
 
     private final Client client;
 
@@ -57,7 +64,7 @@ public class TaskResultsService extends AbstractComponent {
     private final TransportCreateIndexAction createIndexAction;
 
     @Inject
-    public TaskResultsService(Settings settings, Client client, ClusterService clusterService,
+    public TaskPersistenceService(Settings settings, Client client, ClusterService clusterService,
                               TransportCreateIndexAction createIndexAction) {
         super(settings);
         this.client = client;
@@ -65,15 +72,15 @@ public class TaskResultsService extends AbstractComponent {
         this.createIndexAction = createIndexAction;
     }
 
-    public void persist(TaskResult taskResult, ActionListener<Void> listener) {
+    public void persist(PersistedTaskInfo taskResult, ActionListener<Void> listener) {
 
         ClusterState state = clusterService.state();
 
-        if (state.routingTable().hasIndex(TASK_RESULT_INDEX) == false) {
+        if (state.routingTable().hasIndex(TASK_INDEX) == false) {
             CreateIndexRequest createIndexRequest = new CreateIndexRequest();
             createIndexRequest.settings(taskResultIndexSettings());
-            createIndexRequest.index(TASK_RESULT_INDEX);
-            createIndexRequest.mapping(TASK_RESULT_TYPE, taskResultIndexMapping());
+            createIndexRequest.index(TASK_INDEX);
+            createIndexRequest.mapping(TASK_TYPE, taskResultIndexMapping());
             createIndexRequest.cause("auto(task api)");
 
             createIndexAction.execute(null, createIndexRequest, new ActionListener<CreateIndexResponse>() {
@@ -97,10 +104,10 @@ public class TaskResultsService extends AbstractComponent {
                 }
             });
         } else {
-            IndexMetaData metaData = state.getMetaData().index(TASK_RESULT_INDEX);
-            if (metaData.getMappings().containsKey(TASK_RESULT_TYPE) == false) {
+            IndexMetaData metaData = state.getMetaData().index(TASK_INDEX);
+            if (metaData.getMappings().containsKey(TASK_TYPE) == false) {
                 // The index already exists but doesn't have our mapping
-                client.admin().indices().preparePutMapping(TASK_RESULT_INDEX).setType(TASK_RESULT_TYPE).setSource(taskResultIndexMapping())
+                client.admin().indices().preparePutMapping(TASK_INDEX).setType(TASK_TYPE).setSource(taskResultIndexMapping())
                     .execute(new ActionListener<PutMappingResponse>() {
                                  @Override
                                  public void onResponse(PutMappingResponse putMappingResponse) {
@@ -120,20 +127,25 @@ public class TaskResultsService extends AbstractComponent {
     }
 
 
-    private void doPersist(TaskResult taskResult, ActionListener<Void> listener) {
-        client.prepareIndex(TASK_RESULT_INDEX, TASK_RESULT_TYPE, taskResult.getTaskId().toString()).setSource(taskResult.getResult())
-            .execute(new ActionListener<IndexResponse>() {
-                @Override
-                public void onResponse(IndexResponse indexResponse) {
-                    listener.onResponse(null);
-                }
-
-                @Override
-                public void onFailure(Throwable e) {
-                    listener.onFailure(e);
-                }
-            });
+    private void doPersist(PersistedTaskInfo taskResult, ActionListener<Void> listener) {
+        IndexRequestBuilder index = client.prepareIndex(TASK_INDEX, TASK_TYPE, taskResult.getTask().getTaskId().toString());
+        try (XContentBuilder builder = XContentFactory.contentBuilder(Requests.INDEX_CONTENT_TYPE)) {
+            taskResult.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            index.setSource(builder);
+        } catch (IOException e) {
+            throw new ElasticsearchException("Couldn't convert task result to XContent for [{}]", e, taskResult.getTask());
+        }
+        index.execute(new ActionListener<IndexResponse>() {
+            @Override
+            public void onResponse(IndexResponse indexResponse) {
+                listener.onResponse(null);
+            }
 
+            @Override
+            public void onFailure(Throwable e) {
+                listener.onFailure(e);
+            }
+        });
     }
 
     private Settings taskResultIndexSettings() {
diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskResult.java b/core/src/main/java/org/elasticsearch/tasks/TaskResult.java
deleted file mode 100644
index a33a7842c27..00000000000
--- a/core/src/main/java/org/elasticsearch/tasks/TaskResult.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.tasks;
-
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo;
-import org.elasticsearch.client.Requests;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentType;
-
-import java.io.IOException;
-
-/**
- * Represents the result or failure of a running task
- */
-public class TaskResult {
-
-    private final BytesReference result;
-
-    private final TaskId taskId;
-
-    public TaskResult(TaskInfo taskInfo, Throwable e) throws IOException {
-        ToXContent.Params params = ToXContent.EMPTY_PARAMS;
-        XContentBuilder builder = XContentFactory.contentBuilder(Requests.INDEX_CONTENT_TYPE);
-        builder.startObject();
-        {
-            builder.startObject("task");
-            {
-                taskInfo.toXContent(builder, params);
-            }
-            builder.endObject();
-            builder.startObject("error");
-            {
-                ElasticsearchException.toXContent(builder, params, e);
-            }
-            builder.endObject();
-        }
-        builder.endObject();
-        result = builder.bytes();
-        taskId = taskInfo.getTaskId();
-    }
-
-    public TaskResult(TaskInfo taskInfo, ToXContent toXContent) throws IOException {
-        ToXContent.Params params = ToXContent.EMPTY_PARAMS;
-        XContentBuilder builder = XContentFactory.contentBuilder(Requests.INDEX_CONTENT_TYPE);
-        builder.startObject();
-        {
-            builder.startObject("task");
-            {
-                taskInfo.toXContent(builder, params);
-            }
-            builder.endObject();
-            builder.startObject("result");
-            {
-                toXContent.toXContent(builder, params);
-            }
-            builder.endObject();
-        }
-        builder.endObject();
-        result = builder.bytes();
-        taskId = taskInfo.getTaskId();
-    }
-
-    public TaskId getTaskId() {
-        return taskId;
-    }
-
-    public BytesReference getResult() {
-        return result;
-    }
-}
diff --git a/core/src/main/resources/org/elasticsearch/tasks/task-results-index-mapping.json b/core/src/main/resources/org/elasticsearch/tasks/task-index-mapping.json
similarity index 98%
rename from core/src/main/resources/org/elasticsearch/tasks/task-results-index-mapping.json
rename to core/src/main/resources/org/elasticsearch/tasks/task-index-mapping.json
index 0310418bb43..30476b1fe3a 100644
--- a/core/src/main/resources/org/elasticsearch/tasks/task-results-index-mapping.json
+++ b/core/src/main/resources/org/elasticsearch/tasks/task-index-mapping.json
@@ -1,5 +1,5 @@
 {
-  "result" : {
+  "task" : {
     "dynamic" : "strict",
     "properties" : {
       "task" : {
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
index c35a9e91314..b5e113dfcd3 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
@@ -25,7 +25,6 @@ import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksReque
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
-import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo;
 import org.elasticsearch.action.support.nodes.BaseNodeRequest;
 import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
@@ -38,6 +37,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.tasks.TaskInfo;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/RecordingTaskManagerListener.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/RecordingTaskManagerListener.java
index 025d2b39f6d..2aad88c99a9 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/RecordingTaskManagerListener.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/RecordingTaskManagerListener.java
@@ -19,11 +19,11 @@
 
 package org.elasticsearch.action.admin.cluster.node.tasks;
 
-import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskInfo;
 import org.elasticsearch.test.tasks.MockTaskManagerListener;
 
 import java.util.ArrayList;
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
index a83a05b8ad8..95e8cde0a22 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
@@ -233,5 +233,4 @@ public abstract class TaskManagerTestCase extends ESTestCase {
         }
         return listeners;
     }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
index 270059b09dc..88c7f1ae210 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
@@ -19,14 +19,17 @@
 package org.elasticsearch.action.admin.cluster.node.tasks;
 
 import org.elasticsearch.ElasticsearchTimeoutException;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.ListenableActionFuture;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
+import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
-import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo;
 import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
 import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeAction;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryAction;
@@ -43,8 +46,11 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.tasks.PersistedTaskInfo;
 import org.elasticsearch.tasks.Task;
-import org.elasticsearch.tasks.TaskResultsService;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.tasks.TaskInfo;
+import org.elasticsearch.tasks.TaskPersistenceService;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.tasks.MockTaskManager;
 import org.elasticsearch.test.tasks.MockTaskManagerListener;
@@ -59,22 +65,26 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.BrokenBarrierException;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Consumer;
 import java.util.function.Function;
 
+import static java.util.Collections.singleton;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
 import static org.hamcrest.Matchers.allOf;
-import static org.hamcrest.Matchers.either;
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.emptyCollectionOf;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 
@@ -199,7 +209,7 @@ public class TasksIT extends ESIntegTestCase {
 
         logger.debug("number of shards, total: [{}], primaries: [{}] ", numberOfShards.totalNumShards, numberOfShards.numPrimaries);
         logger.debug("main events {}", numberOfEvents(RefreshAction.NAME, Tuple::v1));
-        logger.debug("main event node {}", findEvents(RefreshAction.NAME, Tuple::v1).get(0).getNode().getName());
+        logger.debug("main event node {}", findEvents(RefreshAction.NAME, Tuple::v1).get(0).getTaskId().getNodeId());
         logger.debug("[s] events {}", numberOfEvents(RefreshAction.NAME + "[s]", Tuple::v1));
         logger.debug("[s][*] events {}", numberOfEvents(RefreshAction.NAME + "[s][*]", Tuple::v1));
         logger.debug("nodes with the index {}", internalCluster().nodesInclude("test"));
@@ -219,15 +229,16 @@ public class TasksIT extends ESIntegTestCase {
         TaskInfo mainTask = findEvents(RefreshAction.NAME, Tuple::v1).get(0);
         List<TaskInfo> sTasks = findEvents(RefreshAction.NAME + "[s]", Tuple::v1);
         for (TaskInfo taskInfo : sTasks) {
-            if (mainTask.getNode().equals(taskInfo.getNode())) {
+            if (mainTask.getTaskId().getNodeId().equals(taskInfo.getTaskId().getNodeId())) {
                 // This shard level task runs on the same node as a parent task - it should have the main task as a direct parent
                 assertParentTask(Collections.singletonList(taskInfo), mainTask);
             } else {
                 String description = taskInfo.getDescription();
                 // This shard level task runs on another node - it should have a corresponding shard level task on the node where main task
                 // is running
-                List<TaskInfo> sTasksOnRequestingNode = findEvents(RefreshAction.NAME + "[s]", event -> event.v1()
-                        && mainTask.getNode().equals(event.v2().getNode()) && description.equals(event.v2().getDescription()));
+                List<TaskInfo> sTasksOnRequestingNode = findEvents(RefreshAction.NAME + "[s]",
+                        event -> event.v1() && mainTask.getTaskId().getNodeId().equals(event.v2().getTaskId().getNodeId())
+                                && description.equals(event.v2().getDescription()));
                 // There should be only one parent task
                 assertEquals(1, sTasksOnRequestingNode.size());
                 assertParentTask(Collections.singletonList(taskInfo), sTasksOnRequestingNode.get(0));
@@ -243,12 +254,13 @@ public class TasksIT extends ESIntegTestCase {
             List<TaskInfo> sTask;
             if (taskInfo.getAction().endsWith("[s][p]")) {
                 // A [s][p] level task should have a corresponding [s] level task on the same node
-                sTask = findEvents(RefreshAction.NAME + "[s]", event -> event.v1() && taskInfo.getNode().equals(event.v2().getNode())
-                        && taskInfo.getDescription().equals(event.v2().getDescription()));
+                sTask = findEvents(RefreshAction.NAME + "[s]",
+                        event -> event.v1() && taskInfo.getTaskId().getNodeId().equals(event.v2().getTaskId().getNodeId())
+                                && taskInfo.getDescription().equals(event.v2().getDescription()));
             } else {
                 // A [s][r] level task should have a corresponding [s] level task on the a different node (where primary is located)
                 sTask = findEvents(RefreshAction.NAME + "[s]",
-                    event -> event.v1() && taskInfo.getParentTaskId().getNodeId().equals(event.v2().getNode().getId()) && taskInfo
+                    event -> event.v1() && taskInfo.getParentTaskId().getNodeId().equals(event.v2().getTaskId().getNodeId()) && taskInfo
                         .getDescription()
                         .equals(event.v2().getDescription()));
             }
@@ -305,7 +317,7 @@ public class TasksIT extends ESIntegTestCase {
         // they all should have the same shard task as a parent
         assertEquals(getNumShards("test").numReplicas, numberOfEvents(BulkAction.NAME + "[s][r]", Tuple::v1));
         assertParentTask(findEvents(BulkAction.NAME + "[s][r]", Tuple::v1), shardTask);
-}
+    }
 
     /**
      * Very basic "is it plugged in" style test that indexes a document and
@@ -353,14 +365,28 @@ public class TasksIT extends ESIntegTestCase {
         }
         ListenableActionFuture<?> indexFuture = client().prepareIndex("test", "test").setSource("test", "test").execute();
         taskRegistered.await(10, TimeUnit.SECONDS); // waiting for at least one task to be registered
-        ListTasksResponse tasks = client().admin().cluster().prepareListTasks().setActions("indices:data/write/index*").setDetailed(true)
-                .get();
+
+        ListTasksResponse listResponse = client().admin().cluster().prepareListTasks().setActions("indices:data/write/index*")
+                .setDetailed(true).get();
+        assertThat(listResponse.getTasks(), not(empty()));
+        for (TaskInfo task : listResponse.getTasks()) {
+            assertNotNull(task.getStatus());
+            GetTaskResponse getResponse = client().admin().cluster().prepareGetTask(task.getTaskId()).get();
+            assertFalse("task should still be running", getResponse.getTask().isCompleted());
+            TaskInfo fetchedWithGet = getResponse.getTask().getTask();
+            assertEquals(task.getId(), fetchedWithGet.getId());
+            assertEquals(task.getType(), fetchedWithGet.getType());
+            assertEquals(task.getAction(), fetchedWithGet.getAction());
+            assertEquals(task.getDescription(), fetchedWithGet.getDescription());
+            assertEquals(task.getStatus(), fetchedWithGet.getStatus());
+            assertEquals(task.getStartTime(), fetchedWithGet.getStartTime());
+            assertThat(fetchedWithGet.getRunningTimeNanos(), greaterThanOrEqualTo(task.getRunningTimeNanos()));
+            assertEquals(task.isCancellable(), fetchedWithGet.isCancellable());
+            assertEquals(task.getParentTaskId(), fetchedWithGet.getParentTaskId());
+        }
+
         taskFinishLock.unlock();
         indexFuture.get();
-        assertThat(tasks.getTasks(), not(emptyCollectionOf(TaskInfo.class)));
-        for (TaskInfo task : tasks.getTasks()) {
-            assertNotNull(task.getStatus());
-        }
     }
 
     public void testTasksCancellation() throws Exception {
@@ -401,60 +427,91 @@ public class TasksIT extends ESIntegTestCase {
                 .getTasks().size());
     }
 
-    public void testTasksListWaitForCompletion() throws Exception {
+    public void testListTasksWaitForCompletion() throws Exception {
+        waitForCompletionTestCase(id -> {
+            return client().admin().cluster().prepareListTasks().setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]")
+                    .setWaitForCompletion(true).execute();
+        }, response -> {
+            assertThat(response.getNodeFailures(), empty());
+            assertThat(response.getTaskFailures(), empty());
+        });
+    }
+
+    public void testGetTaskWaitForCompletion() throws Exception {
+        waitForCompletionTestCase(id -> {
+            return client().admin().cluster().prepareGetTask(id).setWaitForCompletion(true).execute();
+        }, response -> {
+            // Really we're just happy we didn't get any exceptions
+            assertNotNull(response.getTask().getTask());
+        });
+    }
+
+    /**
+     * Test wait for completion.
+     * @param wait start waiting for a task. Accepts that id of the task to wait for and returns a future waiting for it.
+     * @param validator validate the response and return the task ids that were found
+     */
+    private <T> void waitForCompletionTestCase(Function<TaskId, ListenableActionFuture<T>> wait, Consumer<T> validator)
+            throws Exception {
         // Start blocking test task
         ListenableActionFuture<TestTaskPlugin.NodesResponse> future = TestTaskPlugin.TestTaskAction.INSTANCE.newRequestBuilder(client())
                 .execute();
 
-        ListenableActionFuture<ListTasksResponse> waitResponseFuture;
+        ListenableActionFuture<T> waitResponseFuture;
+        TaskId taskId;
         try {
-            // Wait for the task to start on all nodes
-            assertBusy(() -> assertEquals(internalCluster().size(), client().admin().cluster().prepareListTasks()
-                    .setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").get().getTasks().size()));
+            taskId = waitForTestTaskStartOnAllNodes();
 
             // Spin up a request to wait for that task to finish
-            waitResponseFuture = client().admin().cluster().prepareListTasks()
-                    .setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").setWaitForCompletion(true).execute();
+            waitResponseFuture = wait.apply(taskId);
         } finally {
             // Unblock the request so the wait for completion request can finish
             TestTaskPlugin.UnblockTestTasksAction.INSTANCE.newRequestBuilder(client()).get();
         }
 
         // Now that the task is unblocked the list response will come back
-        ListTasksResponse waitResponse = waitResponseFuture.get();
-        // If any tasks come back then they are the tasks we asked for - it'd be super weird if this wasn't true
-        for (TaskInfo task: waitResponse.getTasks()) {
-            assertEquals(task.getAction(), TestTaskPlugin.TestTaskAction.NAME + "[n]");
-        }
-        // See the next test to cover the timeout case
+        T waitResponse = waitResponseFuture.get();
+        validator.accept(waitResponse);
 
         future.get();
     }
 
-    public void testTasksListWaitForTimeout() throws Exception {
+    public void testListTasksWaitForTimeout() throws Exception {
+        waitForTimeoutTestCase(id -> {
+            ListTasksResponse response = client().admin().cluster().prepareListTasks()
+                    .setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").setWaitForCompletion(true).setTimeout(timeValueMillis(100))
+                    .get();
+            assertThat(response.getNodeFailures(), not(empty()));
+            return response.getNodeFailures();
+        });
+    }
+
+    public void testGetTaskWaitForTimeout() throws Exception {
+        waitForTimeoutTestCase(id -> {
+            Exception e = expectThrows(Exception.class,
+                    () -> client().admin().cluster().prepareGetTask(id).setWaitForCompletion(true).setTimeout(timeValueMillis(100)).get());
+            return singleton(e);
+        });
+    }
+
+    /**
+     * Test waiting for a task that times out.
+     * @param wait wait for the running task and return all the failures you accumulated waiting for it
+     */
+    private void waitForTimeoutTestCase(Function<TaskId, ? extends Iterable<? extends Throwable>> wait) throws Exception {
         // Start blocking test task
         ListenableActionFuture<TestTaskPlugin.NodesResponse> future = TestTaskPlugin.TestTaskAction.INSTANCE.newRequestBuilder(client())
                 .execute();
         try {
-            // Wait for the task to start on all nodes
-            assertBusy(() -> assertEquals(internalCluster().size(), client().admin().cluster().prepareListTasks()
-                    .setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").get().getTasks().size()));
+            TaskId taskId = waitForTestTaskStartOnAllNodes();
 
             // Spin up a request that should wait for those tasks to finish
             // It will timeout because we haven't unblocked the tasks
-            ListTasksResponse waitResponse = client().admin().cluster().prepareListTasks()
-                    .setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").setWaitForCompletion(true).setTimeout(timeValueMillis(100))
-                    .get();
+            Iterable<? extends Throwable> failures = wait.apply(taskId);
 
-            assertFalse(waitResponse.getNodeFailures().isEmpty());
-            for (FailedNodeException failure : waitResponse.getNodeFailures()) {
-                Throwable timeoutException = failure.getCause();
-                // The exception sometimes comes back wrapped depending on the client
-                if (timeoutException.getCause() != null) {
-                    timeoutException = timeoutException.getCause();
-                }
-                assertThat(timeoutException,
-                        either(instanceOf(ElasticsearchTimeoutException.class)).or(instanceOf(ReceiveTimeoutTransportException.class)));
+            for (Throwable failure : failures) {
+                assertNotNull(
+                        ExceptionsHelper.unwrap(failure, ElasticsearchTimeoutException.class, ReceiveTimeoutTransportException.class));
             }
         } finally {
             // Now we can unblock those requests
@@ -463,6 +520,17 @@ public class TasksIT extends ESIntegTestCase {
         future.get();
     }
 
+    private TaskId waitForTestTaskStartOnAllNodes() throws Exception {
+        AtomicReference<TaskId> result = new AtomicReference<>();
+        assertBusy(() -> {
+            List<TaskInfo> tasks = client().admin().cluster().prepareListTasks().setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]")
+                    .get().getTasks();
+            assertEquals(internalCluster().size(), tasks.size());
+            result.set(tasks.get(0).getTaskId());
+        });
+        return result.get();
+    }
+
     public void testTasksListWaitForNoTask() throws Exception {
         // Spin up a request to wait for no matching tasks
         ListenableActionFuture<ListTasksResponse> waitResponseFuture = client().admin().cluster().prepareListTasks()
@@ -470,7 +538,17 @@ public class TasksIT extends ESIntegTestCase {
                 .execute();
 
         // It should finish quickly and without complaint
-        assertThat(waitResponseFuture.get().getTasks(), emptyCollectionOf(TaskInfo.class));
+        assertThat(waitResponseFuture.get().getTasks(), empty());
+    }
+
+    public void testTasksGetWaitForNoTask() throws Exception {
+        // Spin up a request to wait for no matching tasks
+        ListenableActionFuture<GetTaskResponse> waitResponseFuture = client().admin().cluster().prepareGetTask("notfound:1")
+                .setWaitForCompletion(true).setTimeout(timeValueMillis(10))
+                .execute();
+
+        // It should finish quickly and without complaint
+        expectNotFound(() -> waitResponseFuture.get());
     }
 
     public void testTasksWaitForAllTask() throws Exception {
@@ -488,7 +566,7 @@ public class TasksIT extends ESIntegTestCase {
         // Randomly create an empty index to make sure the type is created automatically
         if (randomBoolean()) {
             logger.info("creating an empty results index with custom settings");
-            assertAcked(client().admin().indices().prepareCreate(TaskResultsService.TASK_RESULT_INDEX));
+            assertAcked(client().admin().indices().prepareCreate(TaskPersistenceService.TASK_INDEX));
         }
 
         registerTaskManageListeners(TestTaskPlugin.TestTaskAction.NAME);  // we need this to get task id of the process
@@ -500,34 +578,43 @@ public class TasksIT extends ESIntegTestCase {
 
         assertEquals(1, events.size());
         TaskInfo taskInfo = events.get(0);
-        String taskId = taskInfo.getTaskId().toString();
+        TaskId taskId = taskInfo.getTaskId();
 
-        GetResponse resultDoc = client().prepareGet(TaskResultsService.TASK_RESULT_INDEX, TaskResultsService.TASK_RESULT_TYPE, taskId)
-            .get();
+        GetResponse resultDoc = client()
+                .prepareGet(TaskPersistenceService.TASK_INDEX, TaskPersistenceService.TASK_TYPE, taskId.toString()).get();
         assertTrue(resultDoc.isExists());
 
         Map<String, Object> source = resultDoc.getSource();
+        @SuppressWarnings("unchecked")
         Map<String, Object> task = (Map<String, Object>) source.get("task");
-        assertEquals(taskInfo.getNode().getId(), task.get("node"));
+        assertEquals(taskInfo.getTaskId().getNodeId(), task.get("node"));
         assertEquals(taskInfo.getAction(), task.get("action"));
         assertEquals(Long.toString(taskInfo.getId()), task.get("id").toString());
 
+        @SuppressWarnings("unchecked")
         Map<String, Object> result = (Map<String, Object>) source.get("result");
         assertEquals("0", result.get("failure_count").toString());
 
-        assertNoFailures(client().admin().indices().prepareRefresh(TaskResultsService.TASK_RESULT_INDEX).get());
+        assertNull(source.get("failure"));
 
-        SearchResponse searchResponse = client().prepareSearch(TaskResultsService.TASK_RESULT_INDEX)
-            .setTypes(TaskResultsService.TASK_RESULT_TYPE)
+        assertNoFailures(client().admin().indices().prepareRefresh(TaskPersistenceService.TASK_INDEX).get());
+
+        SearchResponse searchResponse = client().prepareSearch(TaskPersistenceService.TASK_INDEX)
+            .setTypes(TaskPersistenceService.TASK_TYPE)
             .setSource(SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("task.action", taskInfo.getAction())))
             .get();
 
         assertEquals(1L, searchResponse.getHits().totalHits());
 
-        searchResponse = client().prepareSearch(TaskResultsService.TASK_RESULT_INDEX).setTypes(TaskResultsService.TASK_RESULT_TYPE)
-            .setSource(SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("task.node", taskInfo.getNode().getId()))).get();
+        searchResponse = client().prepareSearch(TaskPersistenceService.TASK_INDEX).setTypes(TaskPersistenceService.TASK_TYPE)
+                .setSource(SearchSourceBuilder.searchSource().query(QueryBuilders.termQuery("task.node", taskInfo.getTaskId().getNodeId())))
+                .get();
 
         assertEquals(1L, searchResponse.getHits().totalHits());
+
+        GetTaskResponse getResponse = expectFinishedTask(taskId);
+        assertEquals(result, getResponse.getTask().getResultAsMap());
+        assertNull(getResponse.getTask().getError());
     }
 
     public void testTaskFailurePersistence() throws Exception {
@@ -545,22 +632,69 @@ public class TasksIT extends ESIntegTestCase {
         List<TaskInfo> events = findEvents(TestTaskPlugin.TestTaskAction.NAME, Tuple::v1);
         assertEquals(1, events.size());
         TaskInfo failedTaskInfo = events.get(0);
-        String failedTaskId = failedTaskInfo.getTaskId().toString();
+        TaskId failedTaskId = failedTaskInfo.getTaskId();
 
         GetResponse failedResultDoc = client()
-            .prepareGet(TaskResultsService.TASK_RESULT_INDEX, TaskResultsService.TASK_RESULT_TYPE, failedTaskId)
+            .prepareGet(TaskPersistenceService.TASK_INDEX, TaskPersistenceService.TASK_TYPE, failedTaskId.toString())
             .get();
         assertTrue(failedResultDoc.isExists());
 
         Map<String, Object> source = failedResultDoc.getSource();
+        @SuppressWarnings("unchecked")
         Map<String, Object> task = (Map<String, Object>) source.get("task");
-        assertEquals(failedTaskInfo.getNode().getId(), task.get("node"));
+        assertEquals(failedTaskInfo.getTaskId().getNodeId(), task.get("node"));
         assertEquals(failedTaskInfo.getAction(), task.get("action"));
         assertEquals(Long.toString(failedTaskInfo.getId()), task.get("id").toString());
 
+        @SuppressWarnings("unchecked")
         Map<String, Object> error = (Map<String, Object>) source.get("error");
         assertEquals("Simulating operation failure", error.get("reason"));
         assertEquals("illegal_state_exception", error.get("type"));
+
+        assertNull(source.get("result"));
+
+        GetTaskResponse getResponse = expectFinishedTask(failedTaskId);
+        assertNull(getResponse.getTask().getResult());
+        assertEquals(error, getResponse.getTask().getErrorAsMap());
+    }
+
+    public void testGetTaskNotFound() throws Exception {
+        // Node isn't found, tasks index doesn't even exist
+        expectNotFound(() -> client().admin().cluster().prepareGetTask("not_a_node:1").get());
+
+        // Node exists but the task still isn't found
+        expectNotFound(() -> client().admin().cluster().prepareGetTask(new TaskId(internalCluster().getNodeNames()[0], 1)).get());
+    }
+
+    public void testNodeNotFoundButTaskFound() throws Exception {
+        // Save a fake task that looks like it is from a node that isn't part of the cluster
+        CyclicBarrier b = new CyclicBarrier(2);
+        TaskPersistenceService resultsService = internalCluster().getInstance(TaskPersistenceService.class);
+        resultsService.persist(
+                new PersistedTaskInfo(new TaskInfo(new TaskId("fake", 1), "test", "test", "", null, 0, 0, false, TaskId.EMPTY_TASK_ID),
+                        new RuntimeException("test")),
+                new ActionListener<Void>() {
+                    @Override
+                    public void onResponse(Void response) {
+                        try {
+                            b.await();
+                        } catch (InterruptedException | BrokenBarrierException e) {
+                            onFailure(e);
+                        }
+                    }
+
+                    @Override
+                    public void onFailure(Throwable e) {
+                        throw new RuntimeException(e);
+                    }
+                });
+        b.await();
+ 
+        // Now we can find it!
+        GetTaskResponse response = expectFinishedTask(new TaskId("fake:1"));
+        assertEquals("test", response.getTask().getTask().getAction());
+        assertNotNull(response.getTask().getError());
+        assertNull(response.getTask().getResult());
     }
 
     @Override
@@ -638,8 +772,28 @@ public class TasksIT extends ESIntegTestCase {
 
     private void assertParentTask(TaskInfo task, TaskInfo parentTask) {
         assertTrue(task.getParentTaskId().isSet());
-        assertEquals(parentTask.getNode().getId(), task.getParentTaskId().getNodeId());
+        assertEquals(parentTask.getTaskId().getNodeId(), task.getParentTaskId().getNodeId());
         assertTrue(Strings.hasLength(task.getParentTaskId().getNodeId()));
         assertEquals(parentTask.getId(), task.getParentTaskId().getId());
     }
+
+    private ResourceNotFoundException expectNotFound(ThrowingRunnable r) {
+        Exception e = expectThrows(Exception.class, r);
+        ResourceNotFoundException notFound = (ResourceNotFoundException) ExceptionsHelper.unwrap(e, ResourceNotFoundException.class);
+        if (notFound == null) throw new RuntimeException("Expected ResourceNotFoundException", e);
+        return notFound;
+    }
+
+    /**
+     * Fetch the task status from the list tasks API using it's "fallback to get from the task index" behavior. Asserts some obvious stuff
+     * about the fetched task and returns a map of it's status.
+     */
+    private GetTaskResponse expectFinishedTask(TaskId taskId) throws IOException {
+        GetTaskResponse response = client().admin().cluster().prepareGetTask(taskId).get();
+        assertTrue("the task should have been completed before fetching", response.getTask().isCompleted());
+        TaskInfo info = response.getTask().getTask();
+        assertEquals(taskId, info.getTaskId());
+        assertNull(info.getStatus()); // The test task doesn't have any status
+        return response;
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
index 3e996421441..326e909b3ff 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
@@ -28,7 +28,6 @@ import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskGroup;
-import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.action.support.nodes.BaseNodeRequest;
@@ -38,9 +37,8 @@ import org.elasticsearch.action.support.tasks.BaseTasksResponse;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -52,6 +50,7 @@ import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.tasks.TaskInfo;
 import org.elasticsearch.test.tasks.MockTaskManager;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -371,10 +370,10 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
         assertEquals(testNodes.length, response.getPerNodeTasks().size());
 
         // Coordinating node
-        assertEquals(2, response.getPerNodeTasks().get(testNodes[0].discoveryNode).size());
+        assertEquals(2, response.getPerNodeTasks().get(testNodes[0].discoveryNode.getId()).size());
         // Other nodes node
         for (int i = 1; i < testNodes.length; i++) {
-            assertEquals(1, response.getPerNodeTasks().get(testNodes[i].discoveryNode).size());
+            assertEquals(1, response.getPerNodeTasks().get(testNodes[i].discoveryNode.getId()).size());
         }
         // There should be a single main task when grouped by tasks
         assertEquals(1, response.getTaskGroups().size());
@@ -387,7 +386,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
         listTasksRequest.setActions("testAction[n]"); // only pick node actions
         response = testNode.transportListTasksAction.execute(listTasksRequest).get();
         assertEquals(testNodes.length, response.getPerNodeTasks().size());
-        for (Map.Entry<DiscoveryNode, List<TaskInfo>> entry : response.getPerNodeTasks().entrySet()) {
+        for (Map.Entry<String, List<TaskInfo>> entry : response.getPerNodeTasks().entrySet()) {
             assertEquals(1, entry.getValue().size());
             assertNull(entry.getValue().get(0).getDescription());
         }
@@ -401,7 +400,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
         listTasksRequest.setDetailed(true); // same request only with detailed description
         response = testNode.transportListTasksAction.execute(listTasksRequest).get();
         assertEquals(testNodes.length, response.getPerNodeTasks().size());
-        for (Map.Entry<DiscoveryNode, List<TaskInfo>> entry : response.getPerNodeTasks().entrySet()) {
+        for (Map.Entry<String, List<TaskInfo>> entry : response.getPerNodeTasks().entrySet()) {
             assertEquals(1, entry.getValue().size());
             assertEquals("CancellableNodeRequest[Test Request, true]", entry.getValue().get(0).getDescription());
         }
@@ -438,7 +437,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
         listTasksRequest.setActions("testAction");
         ListTasksResponse response = testNode.transportListTasksAction.execute(listTasksRequest).get();
         assertEquals(1, response.getTasks().size());
-        String parentNode = response.getTasks().get(0).getNode().getId();
+        String parentNode = response.getTasks().get(0).getTaskId().getNodeId();
         long parentTaskId = response.getTasks().get(0).getId();
 
         // Find tasks with common parent
@@ -493,7 +492,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
         listTasksRequest.setActions("testAction[n]"); // only pick node actions
         ListTasksResponse response = testNode.transportListTasksAction.execute(listTasksRequest).get();
         assertEquals(testNodes.length, response.getPerNodeTasks().size());
-        for (Map.Entry<DiscoveryNode, List<TaskInfo>> entry : response.getPerNodeTasks().entrySet()) {
+        for (Map.Entry<String, List<TaskInfo>> entry : response.getPerNodeTasks().entrySet()) {
             assertEquals(1, entry.getValue().size());
             assertNull(entry.getValue().get(0).getDescription());
         }
@@ -503,7 +502,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
         listTasksRequest.setDetailed(true); // same request only with detailed description
         response = testNode.transportListTasksAction.execute(listTasksRequest).get();
         assertEquals(testNodes.length, response.getPerNodeTasks().size());
-        for (Map.Entry<DiscoveryNode, List<TaskInfo>> entry : response.getPerNodeTasks().entrySet()) {
+        for (Map.Entry<String, List<TaskInfo>> entry : response.getPerNodeTasks().entrySet()) {
             assertEquals(1, entry.getValue().size());
             assertEquals("CancellableNodeRequest[Test Request, true]", entry.getValue().get(0).getDescription());
             assertThat(entry.getValue().get(0).getStartTime(), greaterThanOrEqualTo(minimalStartTime));
@@ -739,6 +738,11 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
         assertEquals(testNodes.length + 1, response.getTasks().size());
 
         // First group by node
+        DiscoveryNodes.Builder discoNodes = DiscoveryNodes.builder();
+        for (TestNode testNode : this.testNodes) {
+            discoNodes.put(testNode.discoveryNode);
+        }
+        response.setDiscoveryNodes(discoNodes.build());
         Map<String, Object> byNodes = serialize(response, new ToXContent.MapParams(Collections.singletonMap("group_by", "nodes")));
         byNodes = (Map<String, Object>) byNodes.get("nodes");
         // One element on the top level
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java b/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java
index b4ddd4ab589..461428581c5 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/BaseXContentTestCase.java
@@ -27,8 +27,6 @@ import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 
-import static org.hamcrest.Matchers.equalTo;
-
 public abstract class BaseXContentTestCase extends ESTestCase {
 
     public abstract XContentType xcontentType();
@@ -136,5 +134,26 @@ public abstract class BaseXContentTestCase extends ESTestCase {
         assertEquals(Token.VALUE_NULL, parser.nextToken());
         assertEquals(Token.END_OBJECT, parser.nextToken());
         assertNull(parser.nextToken());
+
+        os = new ByteArrayOutputStream();
+        try (XContentGenerator generator = xcontentType().xContent().createGenerator(os)) {
+            generator.writeStartObject();
+            generator.writeFieldName("test");
+            generator.writeRawValue(new BytesArray(rawData));
+            generator.writeEndObject();
+        }
+
+        parser = xcontentType().xContent().createParser(os.toByteArray());
+        assertEquals(Token.START_OBJECT, parser.nextToken());
+        assertEquals(Token.FIELD_NAME, parser.nextToken());
+        assertEquals("test", parser.currentName());
+        assertEquals(Token.START_OBJECT, parser.nextToken());
+        assertEquals(Token.FIELD_NAME, parser.nextToken());
+        assertEquals("foo", parser.currentName());
+        assertEquals(Token.VALUE_NULL, parser.nextToken());
+        assertEquals(Token.END_OBJECT, parser.nextToken());
+        assertEquals(Token.END_OBJECT, parser.nextToken());
+        assertNull(parser.nextToken());
+
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/tasks/PersistedTaskInfoTests.java b/core/src/test/java/org/elasticsearch/tasks/PersistedTaskInfoTests.java
new file mode 100644
index 00000000000..d35d6ce82c2
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/tasks/PersistedTaskInfoTests.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.tasks;
+
+import org.elasticsearch.client.Requests;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Round trip tests for {@link PersistedTaskInfo} and those classes that it includes like {@link TaskInfo} and {@link RawTaskStatus}. 
+ */
+public class PersistedTaskInfoTests extends ESTestCase {
+    public void testBinaryRoundTrip() throws IOException {
+        NamedWriteableRegistry registry = new NamedWriteableRegistry();
+        registry.register(Task.Status.class, RawTaskStatus.NAME, RawTaskStatus::new);
+        PersistedTaskInfo result = randomTaskResult();
+        PersistedTaskInfo read;
+        try (BytesStreamOutput out = new BytesStreamOutput()) {
+            result.writeTo(out);
+            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes()), registry)) {
+                read = new PersistedTaskInfo(in);
+            }
+        } catch (IOException e) {
+            throw new IOException("Error processing [" + result + "]", e);
+        }
+        assertEquals(result, read);
+    }
+
+    public void testXContentRoundTrip() throws IOException {
+        /*
+         * Note that this round trip isn't 100% perfect - status will always be read as RawTaskStatus. Since this test uses RawTaskStatus
+         * as the status we randomly generate then we can assert the round trip with .equals.
+         */
+        PersistedTaskInfo result = randomTaskResult();
+        PersistedTaskInfo read;
+        try (XContentBuilder builder = XContentBuilder.builder(randomFrom(XContentType.values()).xContent())) {
+            result.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            try (XContentBuilder shuffled = shuffleXContent(builder);
+                    XContentParser parser = XContentHelper.createParser(shuffled.bytes())) {
+                read = PersistedTaskInfo.PARSER.apply(parser, () -> ParseFieldMatcher.STRICT);
+            }
+        } catch (IOException e) {
+            throw new IOException("Error processing [" + result + "]", e);
+        }
+        assertEquals(result, read);
+    }
+
+    private static PersistedTaskInfo randomTaskResult() throws IOException {
+        switch (between(0, 2)) {
+        case 0:
+            return new PersistedTaskInfo(randomTaskInfo());
+        case 1:
+            return new PersistedTaskInfo(randomTaskInfo(), new RuntimeException("error"));
+        case 2:
+            return new PersistedTaskInfo(randomTaskInfo(), randomTaskActionResult());
+        default:
+            throw new UnsupportedOperationException("Unsupported random TaskResult constructor");
+        }
+    }
+
+    private static TaskInfo randomTaskInfo() throws IOException {
+        TaskId taskId = randomTaskId();
+        String type = randomAsciiOfLength(5);
+        String action = randomAsciiOfLength(5);
+        Task.Status status = randomBoolean() ? randomRawTaskStatus() : null;
+        String description = randomBoolean() ? randomAsciiOfLength(5) : null;
+        long startTime = randomLong();
+        long runningTimeNanos = randomLong();
+        boolean cancellable = randomBoolean();
+        TaskId parentTaskId = randomBoolean() ? TaskId.EMPTY_TASK_ID : randomTaskId();
+        return new TaskInfo(taskId, type, action, description, status, startTime, runningTimeNanos, cancellable, parentTaskId);
+    }
+
+    private static TaskId randomTaskId() {
+        return new TaskId(randomAsciiOfLength(5), randomLong());
+    }
+
+    private static RawTaskStatus randomRawTaskStatus() throws IOException {
+        try (XContentBuilder builder = XContentBuilder.builder(Requests.INDEX_CONTENT_TYPE.xContent())) {
+            builder.startObject();
+            int fields = between(0, 10);
+            for (int f = 0; f < fields; f++) {
+                builder.field(randomAsciiOfLength(5), randomAsciiOfLength(5));
+            }
+            builder.endObject();
+            return new RawTaskStatus(builder.bytes());
+        }
+    }
+
+    private static ToXContent randomTaskActionResult() {
+        Map<String, String> result = new TreeMap<>();
+        int fields = between(0, 10);
+        for (int f = 0; f < fields; f++) {
+            result.put(randomAsciiOfLength(5), randomAsciiOfLength(5));
+        }
+        return new ToXContent() {
+            @Override
+            public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+                // Results in Elasticsearch never output a leading startObject. There isn't really a good reason, they just don't.
+                for (Map.Entry<String, String> entry : result.entrySet()) {
+                    builder.field(entry.getKey(), entry.getValue());
+                }
+                return builder;
+            }
+        };
+    }
+}
diff --git a/docs/reference/cluster/tasks.asciidoc b/docs/reference/cluster/tasks.asciidoc
index b53d8de93a1..5720227f9f0 100644
--- a/docs/reference/cluster/tasks.asciidoc
+++ b/docs/reference/cluster/tasks.asciidoc
@@ -58,15 +58,26 @@ The result will look similar to the following:
 }
 --------------------------------------------------
 
-It is also possible to retrieve information for a particular task, or all children of a particular
-tasks using the following two commands:
+It is also possible to retrieve information for a particular task:
 
 [source,js]
 --------------------------------------------------
-GET _tasks/taskId:1
-GET _tasks?parent_task_id=parentTaskId:1
+GET _tasks/taskId:1 <1>
 --------------------------------------------------
 // CONSOLE
+// TEST[catch:missing]
+
+<1> This will return a 404 if the task isn't found.
+
+Or to retrieve all children of a particular task:
+
+[source,js]
+--------------------------------------------------
+GET _tasks?parent_task_id=parentTaskId:1 <1>
+--------------------------------------------------
+// CONSOLE
+
+<1> This won't return a 404 if the parent isn't found.
 
 The task API can be also used to wait for completion of a particular task. The following call will
 block for 10 seconds or until the task with id `oTUltX4IQMOUUVeiohTt8A:12345` is completed.
@@ -76,6 +87,16 @@ block for 10 seconds or until the task with id `oTUltX4IQMOUUVeiohTt8A:12345` is
 GET _tasks/oTUltX4IQMOUUVeiohTt8A:12345?wait_for_completion=true&timeout=10s
 --------------------------------------------------
 // CONSOLE
+// TEST[catch:missing]
+
+You can also wait for all tasks for certain action types to finish. This
+command will wait for all `reindex` tasks to finish:
+
+[source,js]
+--------------------------------------------------
+GET _tasks?actions=*reindex&wait_for_completion=true&timeout=10s
+--------------------------------------------------
+// CONSOLE
 
 Tasks can be also listed using _cat version of the list tasks command, which accepts the same arguments
 as the standard list tasks command.
diff --git a/docs/reference/docs/delete-by-query.asciidoc b/docs/reference/docs/delete-by-query.asciidoc
index 1562d8c515d..8b37e0a1220 100644
--- a/docs/reference/docs/delete-by-query.asciidoc
+++ b/docs/reference/docs/delete-by-query.asciidoc
@@ -149,10 +149,11 @@ to be refreshed.
 
 If the request contains `wait_for_completion=false` then Elasticsearch will
 perform some preflight checks, launch the request, and then return a `task`
-which can be used with <<docs-delete-by-query-task-api,Tasks APIs>> to cancel
-or get the status of the task. For now, once the request is finished the task
-is gone and the only place to look for the ultimate result of the task is in
-the Elasticsearch log file. This will be fixed soon.
+which can be used with <<docs-delete-by-query-task-api,Tasks APIs>>
+to cancel or get the status of the task. Elasticsearch will also create a
+record of this task as a document at `.tasks/task/${taskId}`. This is yours
+to keep or remove as you see fit. When you are done with it, delete it so
+Elasticsearch can reclaim the space it uses.
 
 `consistency` controls how many copies of a shard must respond to each write
 request. `timeout` controls how long each write request waits for unavailable
@@ -222,7 +223,7 @@ from aborting the operation.
 [[docs-delete-by-query-task-api]]
 === Works with the Task API
 
-While Delete By Query is running you can fetch their status using the
+You can fetch the status of any running delete-by-query requests with the
 <<tasks,Task API>>:
 
 [source,js]
@@ -277,6 +278,22 @@ of operations that the reindex expects to perform. You can estimate the
 progress by adding the `updated`, `created`, and `deleted` fields. The request
 will finish when their sum is equal to the `total` field.
 
+With the task id you can look up the task directly:
+
+[source,js]
+--------------------------------------------------
+GET /_tasks/taskId:1
+--------------------------------------------------
+// CONSOLE
+// TEST[catch:missing]
+
+The advantage of this API is that it integrates with `wait_for_completion=false`
+to transparently return the status of completed tasks. If the task is completed
+and `wait_for_completion=false` was set on it them it'll come back with a
+`results` or an `error` field. The cost of this feature is the document that
+`wait_for_completion=false` creates at `.tasks/task/${taskId}`. It is up to
+you to delete that document.
+
 
 [float]
 [[docs-delete-by-query-cancel-task-api]]
diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index 3311049c699..bb2a06c0234 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -375,10 +375,11 @@ parameter which causes just the shard that received the new data to be indexed.
 
 If the request contains `wait_for_completion=false` then Elasticsearch will
 perform some preflight checks, launch the request, and then return a `task`
-which can be used with <<docs-reindex-task-api,Tasks APIs>> to cancel or get
-the status of the task. For now, once the request is finished the task is gone
-and the only place to look for the ultimate result of the task is in the
-Elasticsearch log file. This will be fixed soon.
+which can be used with <<docs-reindex-task-api,Tasks APIs>>
+to cancel or get the status of the task. Elasticsearch will also create a
+record of this task as a document at `.tasks/task/${taskId}`. This is yours
+to keep or remove as you see fit. When you are done with it, delete it so
+Elasticsearch can reclaim the space it uses.
 
 `consistency` controls how many copies of a shard must respond to each write
 request. `timeout` controls how long each write request waits for unavailable
@@ -457,7 +458,7 @@ from aborting the operation.
 [[docs-reindex-task-api]]
 === Works with the Task API
 
-While Reindex is running you can fetch their status using the
+You can fetch the status of all running reindex requests with the
 <<tasks,Task API>>:
 
 [source,js]
@@ -515,6 +516,22 @@ of operations that the reindex expects to perform. You can estimate the
 progress by adding the `updated`, `created`, and `deleted` fields. The request
 will finish when their sum is equal to the `total` field.
 
+With the task id you can look up the task directly:
+
+[source,js]
+--------------------------------------------------
+GET /_tasks/taskId:1
+--------------------------------------------------
+// CONSOLE
+// TEST[catch:missing]
+
+The advantage of this API is that it integrates with `wait_for_completion=false`
+to transparently return the status of completed tasks. If the task is completed
+and `wait_for_completion=false` was set on it them it'll come back with a
+`results` or an `error` field. The cost of this feature is the document that
+`wait_for_completion=false` creates at `.tasks/task/${taskId}`. It is up to
+you to delete that document.
+
 
 [float]
 [[docs-reindex-cancel-task-api]]
diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc
index f3a147e36c1..6d6bfe64ecf 100644
--- a/docs/reference/docs/update-by-query.asciidoc
+++ b/docs/reference/docs/update-by-query.asciidoc
@@ -205,10 +205,11 @@ parameter which causes just the shard that received the new data to be indexed.
 
 If the request contains `wait_for_completion=false` then Elasticsearch will
 perform some preflight checks, launch the request, and then return a `task`
-which can be used with <<docs-update-by-query-task-api,Tasks APIs>> to cancel
-or get the status of the task. For now, once the request is finished the task
-is gone and the only place to look for the ultimate result of the task is in
-the Elasticsearch log file. This will be fixed soon.
+which can be used with <<docs-update-by-query-task-api,Tasks APIs>>
+to cancel or get the status of the task. Elasticsearch will also create a
+record of this task as a document at `.tasks/task/${taskId}`. This is yours
+to keep or remove as you see fit. When you are done with it, delete it so
+Elasticsearch can reclaim the space it uses.
 
 `consistency` controls how many copies of a shard must respond to each write
 request. `timeout` controls how long each write request waits for unavailable
@@ -283,7 +284,7 @@ from aborting the operation.
 [[docs-update-by-query-task-api]]
 === Works with the Task API
 
-While Update By Query is running you can fetch their status using the
+You can fetch the status of all running update-by-query requests with the
 <<tasks,Task API>>:
 
 [source,js]
@@ -341,6 +342,22 @@ of operations that the reindex expects to perform. You can estimate the
 progress by adding the `updated`, `created`, and `deleted` fields. The request
 will finish when their sum is equal to the `total` field.
 
+With the task id you can look up the task directly:
+
+[source,js]
+--------------------------------------------------
+GET /_tasks/taskId:1
+--------------------------------------------------
+// CONSOLE
+// TEST[catch:missing]
+
+The advantage of this API is that it integrates with `wait_for_completion=false`
+to transparently return the status of completed tasks. If the task is completed
+and `wait_for_completion=false` was set on it them it'll come back with a
+`results` or an `error` field. The cost of this feature is the document that
+`wait_for_completion=false` creates at `.tasks/task/${taskId}`. It is up to
+you to delete that document.
+
 
 [float]
 [[docs-update-by-query-cancel-task-api]]
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
index 3aea4dbce8a..284e51e054f 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
@@ -77,6 +77,8 @@ public abstract class AbstractBaseReindexRestHandler<
 
             action.execute(internal, new BulkIndexByScrollResponseContentListener<>(channel, params));
             return;
+        } else {
+            internal.setShouldPersistResult(true);
         }
 
         /*
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
index 499dfc82457..80a6ff891da 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
@@ -93,6 +93,11 @@ public abstract class AbstractBulkByScrollRequest<Self extends AbstractBulkByScr
      */
     private float requestsPerSecond = Float.POSITIVE_INFINITY;
 
+    /**
+     * Should this task persist its result?
+     */
+    private boolean shouldPersistResult;
+
     public AbstractBulkByScrollRequest() {
     }
 
@@ -286,6 +291,19 @@ public abstract class AbstractBulkByScrollRequest<Self extends AbstractBulkByScr
         return self();
     }
 
+    /**
+     * Should this task persist its result after it has finished?
+     */
+    public Self setShouldPersistResult(boolean shouldPersistResult) {
+        this.shouldPersistResult = shouldPersistResult;
+        return self();
+    }
+
+    @Override
+    public boolean getShouldPersistResult() {
+        return shouldPersistResult;
+    }
+
     @Override
     public Task createTask(long id, String type, String action, TaskId parentTaskId) {
         return new BulkByScrollTask(id, type, action, getDescription(), parentTaskId, requestsPerSecond);
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
index 3ce01201499..190c0c0bc33 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequestBuilder.java
@@ -131,4 +131,12 @@ public abstract class AbstractBulkByScrollRequestBuilder<
         request.setRequestsPerSecond(requestsPerSecond);
         return self();
     }
+
+    /**
+     * Should this task persist its result after it has finished?
+     */
+    public Self setShouldPersistResult(boolean shouldPersistResult) {
+        request.setShouldPersistResult(shouldPersistResult);
+        return self();
+    }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java
index 382f5b51726..9841794ca2a 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java
@@ -19,7 +19,10 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -30,14 +33,18 @@ import org.elasticsearch.rest.action.support.RestToXContentListener;
 import org.elasticsearch.tasks.TaskId;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
+import static org.elasticsearch.rest.action.admin.cluster.node.tasks.RestListTasksAction.nodeSettingListener;
 
 public class RestRethrottleAction extends BaseRestHandler {
     private final TransportRethrottleAction action;
+    private final ClusterService clusterService;
 
     @Inject
-    public RestRethrottleAction(Settings settings, RestController controller, Client client, TransportRethrottleAction action) {
+    public RestRethrottleAction(Settings settings, RestController controller, Client client, TransportRethrottleAction action,
+            ClusterService clusterService) {
         super(settings, client);
         this.action = action;
+        this.clusterService = clusterService;
         controller.registerHandler(POST, "/_update_by_query/{taskId}/_rethrottle", this);
         controller.registerHandler(POST, "/_delete_by_query/{taskId}/_rethrottle", this);
         controller.registerHandler(POST, "/_reindex/{taskId}/_rethrottle", this);
@@ -52,6 +59,7 @@ public class RestRethrottleAction extends BaseRestHandler {
             throw new IllegalArgumentException("requests_per_second is a required parameter");
         }
         internalRequest.setRequestsPerSecond(requestsPerSecond);
-        action.execute(internalRequest, new RestToXContentListener<>(channel));
+        ActionListener<ListTasksResponse> listener = nodeSettingListener(clusterService, new RestToXContentListener<>(channel));
+        action.execute(internalRequest, listener);
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java
index d083f7b2b14..b716050c036 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java
@@ -22,7 +22,6 @@ package org.elasticsearch.index.reindex;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
-import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
 import org.elasticsearch.cluster.ClusterName;
@@ -31,6 +30,7 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.tasks.TaskInfo;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
index 2bb7fd627e4..dcaa80a8169 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
@@ -22,7 +22,6 @@ package org.elasticsearch.index.reindex;
 import org.elasticsearch.action.ListenableActionFuture;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
-import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskInfo;
 import org.elasticsearch.action.ingest.DeletePipelineRequest;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -31,6 +30,7 @@ import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.Engine.Operation.Origin;
 import org.elasticsearch.index.shard.IndexingOperationListener;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.tasks.TaskInfo;
 import org.elasticsearch.ingest.IngestTestPlugin;
 import org.junit.BeforeClass;
 
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml
index bdad5f581bc..13a98921c05 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml
@@ -69,10 +69,13 @@
   - is_false: deleted
 
   - do:
-      tasks.list:
+      task.get:
         wait_for_completion: true
         task_id: $task
   - is_false: node_failures
+  # The task will be in the response even if it finished before we got here
+  # because of task persistence.
+  - is_true: task
 
 ---
 "Response for version conflict":
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yaml
index 0ff382ff751..65a22781550 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yaml
@@ -134,7 +134,7 @@
         task_id: $task
 
   - do:
-      tasks.list:
+      task.get:
         wait_for_completion: true
         task_id: $task
 
@@ -197,6 +197,6 @@
         task_id: $task
 
   - do:
-      tasks.list:
+      task.get:
         wait_for_completion: true
         task_id: $task
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml
index f4025383321..b54d8bd0e95 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml
@@ -93,10 +93,13 @@
   - is_false: deleted
 
   - do:
-      tasks.list:
+      task.get:
         wait_for_completion: true
         task_id: $task
   - is_false: node_failures
+  # The task will be in the response even if it finished before we got here
+  # because of task persistence.
+  - is_true: task
 
 ---
 "Response format for version conflict":
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yaml
index 597cfa4240f..73e1a3a3a94 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yaml
@@ -156,7 +156,7 @@
         task_id: $task
 
   - do:
-      tasks.list:
+      task.get:
         wait_for_completion: true
         task_id: $task
 
@@ -214,6 +214,6 @@
         task_id: $task
 
   - do:
-      tasks.list:
+      task.get:
         wait_for_completion: true
         task_id: $task
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yaml
index 843bb9b6eb5..8ba4013caa3 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yaml
@@ -53,10 +53,13 @@
   - is_false: deleted
 
   - do:
-      tasks.list:
+      task.get:
         wait_for_completion: true
         task_id: $task
   - is_false: node_failures
+  # The task will be in the response even if it finished before we got here
+  # because of task persistence.
+  - is_true: task
 
 ---
 "Response for version conflict":
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yaml
index 3179ce32a19..e13e29bc3f5 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yaml
@@ -122,7 +122,7 @@
         task_id: $task
 
   - do:
-      tasks.list:
+      task.get:
         wait_for_completion: true
         task_id: $task
 
@@ -172,6 +172,6 @@
         task_id: $task
 
   - do:
-      tasks.list:
+      task.get:
         wait_for_completion: true
         task_id: $task
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/task.get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/task.get.json
new file mode 100644
index 00000000000..8024f015e96
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/task.get.json
@@ -0,0 +1,23 @@
+{
+  "task.get": {
+    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html",
+    "methods": ["GET"],
+    "url": {
+      "path": "/_tasks/{task_id}",
+      "paths": ["/_tasks/{task_id}"],
+      "parts": {
+        "task_id": {
+          "type": "string",
+          "description": "Return the task with specified id (node_id:task_number)"
+        }
+      },
+      "params": {
+        "wait_for_completion": {
+          "type": "boolean",
+          "description": "Wait for the matching tasks to complete (default: false)"
+        }
+      }
+    },
+    "body": null
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json
index d9615a7fdab..a1913fbfc17 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json
@@ -5,12 +5,7 @@
     "url": {
       "path": "/_tasks",
       "paths": ["/_tasks", "/_tasks/{task_id}"],
-      "parts": {
-        "task_id": {
-          "type": "string",
-          "description": "Return the task with specified id (node_id:task_number)"
-        }
-      },
+      "parts": {},
       "params": {
         "node_id": {
           "type": "list",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/task.get/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/task.get/10_basic.yaml
new file mode 100644
index 00000000000..ba90e1541fe
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/task.get/10_basic.yaml
@@ -0,0 +1,10 @@
+---
+"get task test":
+  # Note that this gets much better testing in reindex's tests because it actually saves the task
+  - do:
+      cluster.state: {}
+
+  - do:
+      catch: missing
+      task.get:
+        task_id: foo:1

From 0508fc3bf14d26956c640a191ca4291e2ed5647b Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Tue, 14 Jun 2016 10:54:50 -0700
Subject: [PATCH 0839/1311] Test: Fix integ test extra config files to work
 with more than one

Groovy does some crazy capturing when using closures inside a loop. In
this case, it somehow decided the local loop variable would be
modified, and so each closure was getting a wrapped value that would be
updated on each loop iteration, until all the closures pointed at the
last value. This change fixes the loop to extract the object to be used by
the closures.
---
 .../elasticsearch/gradle/test/ClusterFormationTasks.groovy   | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
index 83dd1b7e4c5..c3004a64b86 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@@ -291,9 +291,10 @@ class ClusterFormationTasks {
         File configDir = new File(node.homeDir, 'config')
         copyConfig.into(configDir) // copy must always have a general dest dir, even though we don't use it
         for (Map.Entry<String,Object> extraConfigFile : node.config.extraConfigFiles.entrySet()) {
+            Object extraConfigFileValue = extraConfigFile.getValue()
             copyConfig.doFirst {
                 // make sure the copy won't be a no-op or act on a directory
-                File srcConfigFile = project.file(extraConfigFile.getValue())
+                File srcConfigFile = project.file(extraConfigFileValue)
                 if (srcConfigFile.isDirectory()) {
                     throw new GradleException("Source for extraConfigFile must be a file: ${srcConfigFile}")
                 }
@@ -303,7 +304,7 @@ class ClusterFormationTasks {
             }
             File destConfigFile = new File(node.homeDir, 'config/' + extraConfigFile.getKey())
             // wrap source file in closure to delay resolution to execution time
-            copyConfig.from({ extraConfigFile.getValue() }) {
+            copyConfig.from({ extraConfigFileValue }) {
                 // this must be in a closure so it is only applied to the single file specified in from above
                 into(configDir.toPath().relativize(destConfigFile.canonicalFile.parentFile.toPath()).toFile())
                 rename { destConfigFile.name }

From 3032a7c65373a0a11b6e7b1f78a9765435962944 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 6 Jun 2016 15:49:50 -0400
Subject: [PATCH 0840/1311] Cache FieldStats

This caches FieldStats at the field level. For one off requests or for
few indicies this doesn't save anything, but when there are 30 indices,
5 shards, 1 replica, 100 parallel requests this is about twice as fast
as not caching. I expect lots of usage won't see much benefit from this
but pointing kibana to a cluster with many indexes and shards, will be
faster.

Closes #18717
---
 .../elasticsearch/action/ActionModule.java    |   4 +-
 ...on.java => TransportFieldStatsAction.java} |  47 ++---
 .../cache/request/ShardRequestCache.java      |   7 +-
 .../elasticsearch/index/shard/IndexShard.java |   6 +-
 .../AbstractIndexShardCacheEntity.java        | 102 +++++++++++
 .../elasticsearch/indices/IndicesService.java | 110 ++++++------
 .../fieldstats/FieldStatsIntegrationIT.java   |  37 +++-
 .../indices/IndicesRequestCacheTests.java     | 169 ++++++------------
 8 files changed, 263 insertions(+), 219 deletions(-)
 rename core/src/main/java/org/elasticsearch/action/fieldstats/{TransportFieldStatsTransportAction.java => TransportFieldStatsAction.java} (90%)
 create mode 100644 core/src/main/java/org/elasticsearch/indices/AbstractIndexShardCacheEntity.java

diff --git a/core/src/main/java/org/elasticsearch/action/ActionModule.java b/core/src/main/java/org/elasticsearch/action/ActionModule.java
index 7c62c171073..c7e6d795afc 100644
--- a/core/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/core/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -143,7 +143,7 @@ import org.elasticsearch.action.delete.TransportDeleteAction;
 import org.elasticsearch.action.explain.ExplainAction;
 import org.elasticsearch.action.explain.TransportExplainAction;
 import org.elasticsearch.action.fieldstats.FieldStatsAction;
-import org.elasticsearch.action.fieldstats.TransportFieldStatsTransportAction;
+import org.elasticsearch.action.fieldstats.TransportFieldStatsAction;
 import org.elasticsearch.action.get.GetAction;
 import org.elasticsearch.action.get.MultiGetAction;
 import org.elasticsearch.action.get.TransportGetAction;
@@ -344,7 +344,7 @@ public class ActionModule extends AbstractModule {
         registerAction(GetStoredScriptAction.INSTANCE, TransportGetStoredScriptAction.class);
         registerAction(DeleteStoredScriptAction.INSTANCE, TransportDeleteStoredScriptAction.class);
 
-        registerAction(FieldStatsAction.INSTANCE, TransportFieldStatsTransportAction.class);
+        registerAction(FieldStatsAction.INSTANCE, TransportFieldStatsAction.class);
 
         registerAction(PutPipelineAction.INSTANCE, PutPipelineTransportAction.class);
         registerAction(GetPipelineAction.INSTANCE, GetPipelineTransportAction.class);
diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java b/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsAction.java
similarity index 90%
rename from core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java
rename to core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsAction.java
index 4d2d1db161d..c475b704d1a 100644
--- a/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsTransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsAction.java
@@ -33,7 +33,6 @@ import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
@@ -45,27 +44,23 @@ import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-import java.io.IOException;
-
-import java.util.Map;
-import java.util.HashMap;
-import java.util.List;
 import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.Set;
-import java.util.HashSet;
 import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicReferenceArray;
 
-public class TransportFieldStatsTransportAction extends
+public class TransportFieldStatsAction extends
     TransportBroadcastAction<FieldStatsRequest, FieldStatsResponse, FieldStatsShardRequest, FieldStatsShardResponse> {
 
     private final IndicesService indicesService;
 
     @Inject
-    public TransportFieldStatsTransportAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
+    public TransportFieldStatsAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
                                               TransportService transportService, ActionFilters actionFilters,
                                               IndexNameExpressionResolver indexNameExpressionResolver,
                                               IndicesService indicesService) {
@@ -195,26 +190,18 @@ public class TransportFieldStatsTransportAction extends
         MapperService mapperService = indexServices.mapperService();
         IndexShard shard = indexServices.getShard(shardId.id());
         try (Engine.Searcher searcher = shard.acquireSearcher("fieldstats")) {
+            // Resolve patterns and deduplicate
+            Set<String> fieldNames = new HashSet<>();
             for (String field : request.getFields()) {
-                Collection<String> matchFields;
-                if (Regex.isSimpleMatchPattern(field)) {
-                    matchFields = mapperService.simpleMatchToIndexNames(field);
-                } else {
-                    matchFields = Collections.singleton(field);
-                }
-                for (String matchField : matchFields) {
-                    MappedFieldType fieldType = mapperService.fullName(matchField);
-                    if (fieldType == null) {
-                        // ignore.
-                        continue;
-                    }
-                    FieldStats<?> stats = fieldType.stats(searcher.reader());
-                    if (stats != null) {
-                        fieldStats.put(matchField, stats);
-                    }
+                fieldNames.addAll(mapperService.simpleMatchToIndexNames(field));
+            }
+            for (String field : fieldNames) {
+                FieldStats<?> stats = indicesService.getFieldStats(shard, searcher, field);
+                if (stats != null) {
+                    fieldStats.put(field, stats);
                 }
             }
-        } catch (IOException e) {
+        } catch (Exception e) {
             throw ExceptionsHelper.convertToElastic(e);
         }
         return new FieldStatsShardResponse(shardId, fieldStats);
diff --git a/core/src/main/java/org/elasticsearch/index/cache/request/ShardRequestCache.java b/core/src/main/java/org/elasticsearch/index/cache/request/ShardRequestCache.java
index ef81b288f92..c955cbef75d 100644
--- a/core/src/main/java/org/elasticsearch/index/cache/request/ShardRequestCache.java
+++ b/core/src/main/java/org/elasticsearch/index/cache/request/ShardRequestCache.java
@@ -20,15 +20,10 @@
 package org.elasticsearch.index.cache.request;
 
 import org.apache.lucene.util.Accountable;
-import org.elasticsearch.common.cache.RemovalListener;
-import org.elasticsearch.common.cache.RemovalNotification;
 import org.elasticsearch.common.metrics.CounterMetric;
-import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.shard.AbstractIndexShardComponent;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.indices.IndicesRequestCache;
 
 /**
+ * Tracks the portion of the request cache in use for a particular shard.
  */
 public final class ShardRequestCache {
 
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 16ebd97d8c3..26701703ce1 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -145,7 +145,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
     private final ShardSearchStats searchStats = new ShardSearchStats();
     private final ShardGetService getService;
     private final ShardIndexWarmerService shardWarmerService;
-    private final ShardRequestCache shardQueryCache;
+    private final ShardRequestCache requestCacheStats;
     private final ShardFieldData shardFieldData;
     private final IndexFieldDataService indexFieldDataService;
     private final ShardBitsetFilterCache shardBitsetFilterCache;
@@ -241,7 +241,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         this.searchOperationListener = new SearchOperationListener.CompositeListener(searchListenersList, logger);
         this.getService = new ShardGetService(indexSettings, this, mapperService);
         this.shardWarmerService = new ShardIndexWarmerService(shardId, indexSettings);
-        this.shardQueryCache = new ShardRequestCache();
+        this.requestCacheStats = new ShardRequestCache();
         this.shardFieldData = new ShardFieldData();
         this.indexFieldDataService = indexFieldDataService;
         this.shardBitsetFilterCache = new ShardBitsetFilterCache(shardId, indexSettings);
@@ -303,7 +303,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
     }
 
     public ShardRequestCache requestCache() {
-        return this.shardQueryCache;
+        return this.requestCacheStats;
     }
 
     public ShardFieldData fieldData() {
diff --git a/core/src/main/java/org/elasticsearch/indices/AbstractIndexShardCacheEntity.java b/core/src/main/java/org/elasticsearch/indices/AbstractIndexShardCacheEntity.java
new file mode 100644
index 00000000000..c0d929d82f5
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/indices/AbstractIndexShardCacheEntity.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.indices;
+
+import org.apache.lucene.index.DirectoryReader;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.cache.RemovalNotification;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.index.cache.request.ShardRequestCache;
+import org.elasticsearch.index.shard.IndexShard;
+
+import java.io.IOException;
+
+/**
+ * Abstract base class for the an {@link IndexShard} level {@linkplain IndicesRequestCache.CacheEntity}.
+ */
+abstract class AbstractIndexShardCacheEntity implements IndicesRequestCache.CacheEntity {
+    @FunctionalInterface
+    public interface Loader {
+        void load(StreamOutput out) throws IOException;
+    }
+
+    private final Loader loader;
+    private boolean loadedFromCache = true;
+
+    protected AbstractIndexShardCacheEntity(Loader loader) {
+        this.loader = loader;
+    }
+
+    /**
+     * When called after passing this through
+     * {@link IndicesRequestCache#getOrCompute(IndicesRequestCache.CacheEntity, DirectoryReader, BytesReference)} this will return whether
+     * or not the result was loaded from the cache.
+     */
+    public final boolean loadedFromCache() {
+        return loadedFromCache;
+    }
+
+    /**
+     * Get the {@linkplain ShardRequestCache} used to track cache statistics.
+     */
+    protected abstract ShardRequestCache stats();
+
+    @Override
+    public final IndicesRequestCache.Value loadValue() throws IOException {
+        /* BytesStreamOutput allows to pass the expected size but by default uses
+         * BigArrays.PAGE_SIZE_IN_BYTES which is 16k. A common cached result ie.
+         * a date histogram with 3 buckets is ~100byte so 16k might be very wasteful
+         * since we don't shrink to the actual size once we are done serializing.
+         * By passing 512 as the expected size we will resize the byte array in the stream
+         * slowly until we hit the page size and don't waste too much memory for small query
+         * results.*/
+        final int expectedSizeInBytes = 512;
+        try (BytesStreamOutput out = new BytesStreamOutput(expectedSizeInBytes)) {
+            loader.load(out);
+            // for now, keep the paged data structure, which might have unused bytes to fill a page, but better to keep
+            // the memory properly paged instead of having varied sized bytes
+            final BytesReference reference = out.bytes();
+            loadedFromCache = false;
+            return new IndicesRequestCache.Value(reference, out.ramBytesUsed());
+        }
+    }
+
+    @Override
+    public final void onCached(IndicesRequestCache.Key key, IndicesRequestCache.Value value) {
+        stats().onCached(key, value);
+    }
+
+    @Override
+    public final void onHit() {
+        stats().onHit();
+    }
+
+    @Override
+    public final void onMiss() {
+        stats().onMiss();
+    }
+
+    @Override
+    public final void onRemoval(RemovalNotification<IndicesRequestCache.Key, IndicesRequestCache.Value> notification) {
+        stats().onRemoval(notification.getKey(), notification.getValue(),
+                notification.getRemovalReason() == RemovalNotification.RemovalReason.EVICTED);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index 3ae02c7eadd..155c60c7a68 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.indices;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
+
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.util.CollectionUtil;
@@ -28,6 +29,7 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
+import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.action.search.SearchType;
@@ -39,12 +41,11 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.cache.RemovalNotification;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.FileSystemUtils;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -71,9 +72,11 @@ import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.index.cache.request.ShardRequestCache;
+import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.flush.FlushStats;
 import org.elasticsearch.index.get.GetStats;
+import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.merge.MergeStats;
 import org.elasticsearch.index.recovery.RecoveryStats;
@@ -87,6 +90,7 @@ import org.elasticsearch.index.shard.IndexingOperationListener;
 import org.elasticsearch.index.shard.IndexingStats;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.store.IndexStoreConfig;
+import org.elasticsearch.indices.AbstractIndexShardCacheEntity.Loader;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
@@ -1085,9 +1089,10 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService>
         if (shard == null) {
             return;
         }
-        indicesRequestCache.clear(new IndexShardCacheEntity(shard));
+        indicesRequestCache.clear(new IndexShardCacheEntity(shard, null));
         logger.trace("{} explicit cache clear", shard.shardId());
     }
+
     /**
      * Loads the cache result, computing it if needed by executing the query phase and otherwise deserializing the cached
      * value into the {@link SearchContext#queryResult() context's query result}. The combination of load + compute allows
@@ -1096,10 +1101,13 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService>
      */
     public void loadIntoContext(ShardSearchRequest request, SearchContext context, QueryPhase queryPhase) throws Exception {
         assert canCache(request, context);
-        final IndexShardCacheEntity entity = new IndexShardCacheEntity(context.indexShard(), queryPhase, context);
+        final IndexShardCacheEntity entity = new IndexShardCacheEntity(context.indexShard(), out -> {
+            queryPhase.execute(context);
+            context.queryResult().writeToNoId(out);
+        });
         final DirectoryReader directoryReader = context.searcher().getDirectoryReader();
         final BytesReference bytesReference = indicesRequestCache.getOrCompute(entity, directoryReader, request.cacheKey());
-        if (entity.loaded == false) { // if we have loaded this we don't need to do anything
+        if (entity.loadedFromCache()) {
             // restore the cached query result into the context
             final QuerySearchResult result = context.queryResult();
             StreamInput in = new NamedWriteableAwareStreamInput(bytesReference.streamInput(), namedWriteableRegistry);
@@ -1108,51 +1116,53 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService>
         }
     }
 
-    static final class IndexShardCacheEntity implements IndicesRequestCache.CacheEntity {
-        private final QueryPhase queryPhase;
-        private final SearchContext context;
+    /**
+     * Fetch {@linkplain FieldStats} for a field. These stats are cached until the shard changes.
+     * @param shard the shard to use with the cache key
+     * @param searcher searcher to use to lookup the field stats
+     * @param field the actual field
+     */
+    public FieldStats<?> getFieldStats(IndexShard shard, Engine.Searcher searcher, String field) throws Exception {
+        MappedFieldType fieldType = shard.mapperService().fullName(field);
+        if (fieldType == null) {
+            return null;
+        }
+        BytesReference cacheKey = new BytesArray("fieldstats:" + field);
+        BytesReference statsRef = cacheShardLevelResult(shard, searcher.getDirectoryReader(), cacheKey, out -> {
+            out.writeOptionalWriteable(fieldType.stats(searcher.reader()));
+        });
+        try (StreamInput in = StreamInput.wrap(statsRef)) {
+            return in.readOptionalWriteable(FieldStats::readFrom);
+        }
+    }
+
+    /**
+     * Cache something calculated at the shard level.
+     * @param shard the shard this item is part of
+     * @param reader a reader for this shard. Used to invalidate the cache when there are changes.
+     * @param cacheKey key for the thing being cached within this shard
+     * @param loader loads the data into the cache if needed
+     * @return the contents of the cache or the result of calling the loader
+     */
+    private BytesReference cacheShardLevelResult(IndexShard shard, DirectoryReader reader, BytesReference cacheKey, Loader loader)
+            throws Exception {
+        IndexShardCacheEntity cacheEntity = new IndexShardCacheEntity(shard, loader);
+        return indicesRequestCache.getOrCompute(cacheEntity, reader, cacheKey);
+    }
+
+    final static class IndexShardCacheEntity extends AbstractIndexShardCacheEntity {
         private final IndexShard indexShard;
-        private final ShardRequestCache requestCache;
-        private boolean loaded = false;
 
-        IndexShardCacheEntity(IndexShard indexShard) {
-            this(indexShard, null, null);
-        }
-
-        public IndexShardCacheEntity(IndexShard indexShard, QueryPhase queryPhase, SearchContext context) {
-            this.queryPhase = queryPhase;
-            this.context = context;
+        protected IndexShardCacheEntity(IndexShard indexShard, Loader loader) {
+            super(loader);
             this.indexShard = indexShard;
-            this.requestCache = indexShard.requestCache();
         }
 
         @Override
-        public IndicesRequestCache.Value loadValue() throws IOException {
-            queryPhase.execute(context);
-            /* BytesStreamOutput allows to pass the expected size but by default uses
-             * BigArrays.PAGE_SIZE_IN_BYTES which is 16k. A common cached result ie.
-             * a date histogram with 3 buckets is ~100byte so 16k might be very wasteful
-             * since we don't shrink to the actual size once we are done serializing.
-             * By passing 512 as the expected size we will resize the byte array in the stream
-             * slowly until we hit the page size and don't waste too much memory for small query
-             * results.*/
-            final int expectedSizeInBytes = 512;
-            try (BytesStreamOutput out = new BytesStreamOutput(expectedSizeInBytes)) {
-                context.queryResult().writeToNoId(out);
-                // for now, keep the paged data structure, which might have unused bytes to fill a page, but better to keep
-                // the memory properly paged instead of having varied sized bytes
-                final BytesReference reference = out.bytes();
-                loaded = true;
-                return new IndicesRequestCache.Value(reference, out.ramBytesUsed());
-            }
+        protected ShardRequestCache stats() {
+            return indexShard.requestCache();
         }
 
-        @Override
-        public void onCached(IndicesRequestCache.Key key, IndicesRequestCache.Value value) {
-            requestCache.onCached(key, value);
-        }
-
-
         @Override
         public boolean isOpen() {
             return indexShard.state() != IndexShardState.CLOSED;
@@ -1162,22 +1172,6 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService>
         public Object getCacheIdentity() {
             return indexShard;
         }
-
-        @Override
-        public void onHit() {
-            requestCache.onHit();
-        }
-
-        @Override
-        public void onMiss() {
-            requestCache.onMiss();
-        }
-
-        @Override
-        public void onRemoval(RemovalNotification<IndicesRequestCache.Key, IndicesRequestCache.Value> notification) {
-            requestCache.onRemoval(notification.getKey(), notification.getValue(), notification.getRemovalReason() == RemovalNotification.RemovalReason.EVICTED);
-        }
-
     }
 
     @FunctionalInterface
diff --git a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java
index d04bc28936c..86fb4cc5c53 100644
--- a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java
@@ -20,14 +20,13 @@
 package org.elasticsearch.fieldstats;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.fieldstats.FieldStats;
+import org.elasticsearch.action.fieldstats.FieldStatsAction;
 import org.elasticsearch.action.fieldstats.FieldStatsResponse;
 import org.elasticsearch.action.fieldstats.IndexConstraint;
 import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.cache.request.RequestCacheStats;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.ArrayList;
@@ -40,11 +39,12 @@ import static org.elasticsearch.action.fieldstats.IndexConstraint.Property.MAX;
 import static org.elasticsearch.action.fieldstats.IndexConstraint.Property.MIN;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful;
-import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.nullValue;
 
 /**
+ * Tests for the {@link FieldStatsAction}.
  */
 public class FieldStatsIntegrationIT extends ESIntegTestCase {
 
@@ -150,7 +150,7 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
             .setFields("byte", "short", "integer", "long", "float", "double", "string").get();
         assertAllSuccessful(response);
 
-        for (FieldStats stats : response.getAllFieldStats().values()) {
+        for (FieldStats<?> stats : response.getAllFieldStats().values()) {
             assertThat(stats.getMaxDoc(), equalTo((long) numDocs));
             assertThat(stats.getDocCount(), equalTo((long) numDocs));
             assertThat(stats.getDensity(), equalTo(100));
@@ -462,6 +462,33 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
         assertThat(response.getIndicesMergedFieldStats().get("test2").get("foo").getMaxValue(), equalTo(100L));
     }
 
+    public void testCached() throws Exception {
+        assertAcked(client().admin().indices().prepareCreate("test").setSettings("index.number_of_replicas", 0));
+        indexRange("test", "value", 0, 99);
+
+        // First query should be a cache miss
+        FieldStatsResponse fieldStats = client().prepareFieldStats().setFields("value").get();
+        assertEquals(100, fieldStats.getAllFieldStats().get("value").getDocCount());
+        RequestCacheStats indexStats = client().admin().indices().prepareStats().get().getIndex("test").getTotal().getRequestCache();
+        assertEquals(0, indexStats.getHitCount());
+        assertThat(indexStats.getMemorySizeInBytes(), greaterThan(0L));
+
+        // Second query should be a cache hit
+        fieldStats = client().prepareFieldStats().setFields("value").get();
+        assertEquals(100, fieldStats.getAllFieldStats().get("value").getDocCount());
+        indexStats = client().admin().indices().prepareStats().get().getIndex("test").getTotal().getRequestCache();
+        assertThat(indexStats.getHitCount(), greaterThan(0L));
+        assertThat(indexStats.getMemorySizeInBytes(), greaterThan(0L));
+
+        // Indexing some new documents and refreshing should give you consistent data.
+        long oldHitCount = indexStats.getHitCount();
+        indexRange("test", "value", 100, 199);
+        fieldStats = client().prepareFieldStats().setFields("value").get();
+        assertEquals(200, fieldStats.getAllFieldStats().get("value").getDocCount());
+        // Because we refreshed the index we don't have any more hits in the cache. This is read from the index.
+        assertEquals(oldHitCount, indexStats.getHitCount());
+    }
+
     private void indexRange(String index, long from, long to) throws Exception {
         indexRange(index, "value", from, to);
     }
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java
index 646d9651436..1cca3bb7215 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java
@@ -30,9 +30,9 @@ import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.cache.RemovalNotification;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -58,29 +58,30 @@ public class IndicesRequestCacheTests extends ESTestCase {
             new ShardId("foo", "bar", 1));
         TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
         AtomicBoolean indexShard = new AtomicBoolean(true);
-        TestEntity entity = new TestEntity(requestCacheStats, reader, indexShard, 0);
 
         // initial cache
+        TestEntity entity = new TestEntity(requestCacheStats, reader, indexShard, 0);
         BytesReference value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes());
-        assertEquals("foo", value.toUtf8());
+        assertEquals("foo", StreamInput.wrap(value).readString());
         assertEquals(0, requestCacheStats.stats().getHitCount());
         assertEquals(1, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
-        assertEquals(1, entity.loaded);
+        assertFalse(entity.loadedFromCache());
         assertEquals(1, cache.count());
 
         // cache hit
+        entity = new TestEntity(requestCacheStats, reader, indexShard, 0);
         value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes());
-        assertEquals("foo", value.toUtf8());
+        assertEquals("foo", StreamInput.wrap(value).readString());
         assertEquals(1, requestCacheStats.stats().getHitCount());
         assertEquals(1, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
-        assertEquals(1, entity.loaded);
+        assertTrue(entity.loadedFromCache());
         assertEquals(1, cache.count());
         assertTrue(requestCacheStats.stats().getMemorySize().bytesAsInt() > value.length());
         assertEquals(1, cache.numRegisteredCloseListeners());
 
-        // release
+        // Closing the cache doesn't modify an already returned CacheEntity
         if (randomBoolean()) {
             reader.close();
         } else {
@@ -91,7 +92,7 @@ public class IndicesRequestCacheTests extends ESTestCase {
         assertEquals(1, requestCacheStats.stats().getHitCount());
         assertEquals(1, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
-        assertEquals(1, entity.loaded);
+        assertTrue(entity.loadedFromCache());
         assertEquals(0, cache.count());
         assertEquals(0, requestCacheStats.stats().getMemorySize().bytesAsInt());
 
@@ -99,43 +100,6 @@ public class IndicesRequestCacheTests extends ESTestCase {
         assertEquals(0, cache.numRegisteredCloseListeners());
     }
 
-    public void testCacheWithDifferentEntityInstance() throws Exception {
-        IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY);
-        AtomicBoolean indexShard =  new AtomicBoolean(true);
-        ShardRequestCache requestCacheStats = new ShardRequestCache();
-        Directory dir = newDirectory();
-        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
-
-        writer.addDocument(newDoc(0, "foo"));
-        DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer),
-            new ShardId("foo", "bar", 1));
-        TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
-        TestEntity entity = new TestEntity(requestCacheStats, reader, indexShard, 0);
-
-        // initial cache
-        BytesReference value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes());
-        assertEquals("foo", value.toUtf8());
-        assertEquals(0, requestCacheStats.stats().getHitCount());
-        assertEquals(1, requestCacheStats.stats().getMissCount());
-        assertEquals(0, requestCacheStats.stats().getEvictions());
-        assertEquals(1, entity.loaded);
-        assertEquals(1, cache.count());
-        assertEquals(1, cache.numRegisteredCloseListeners());
-        final int cacheSize = requestCacheStats.stats().getMemorySize().bytesAsInt();
-
-        value = cache.getOrCompute(new TestEntity(requestCacheStats, reader, indexShard, 0), reader, termQuery.buildAsBytes());
-        assertEquals("foo", value.toUtf8());
-        assertEquals(1, requestCacheStats.stats().getHitCount());
-        assertEquals(1, requestCacheStats.stats().getMissCount());
-        assertEquals(0, requestCacheStats.stats().getEvictions());
-        assertEquals(1, entity.loaded);
-        assertEquals(1, cache.count());
-        assertEquals(cacheSize, requestCacheStats.stats().getMemorySize().bytesAsInt());
-
-        assertEquals(1, cache.numRegisteredCloseListeners());
-        IOUtils.close(reader, writer, dir, cache);
-    }
-
     public void testCacheDifferentReaders() throws Exception {
         IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY);
         AtomicBoolean indexShard =  new AtomicBoolean(true);
@@ -146,62 +110,60 @@ public class IndicesRequestCacheTests extends ESTestCase {
         writer.addDocument(newDoc(0, "foo"));
         DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
         TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
-        TestEntity entity = new TestEntity(requestCacheStats, reader, indexShard, 0);
 
         writer.updateDocument(new Term("id", "0"), newDoc(0, "bar"));
         DirectoryReader secondReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
-        TestEntity secondEntity = new TestEntity(requestCacheStats, secondReader, indexShard, 0);
 
         // initial cache
+        TestEntity entity = new TestEntity(requestCacheStats, reader, indexShard, 0);
         BytesReference value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes());
-        assertEquals("foo", value.toUtf8());
+        assertEquals("foo", StreamInput.wrap(value).readString());
         assertEquals(0, requestCacheStats.stats().getHitCount());
         assertEquals(1, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
-        assertEquals(1, entity.loaded);
+        assertFalse(entity.loadedFromCache());
         assertEquals(1, cache.count());
         assertTrue(requestCacheStats.stats().getMemorySize().bytesAsInt() > value.length());
         final int cacheSize = requestCacheStats.stats().getMemorySize().bytesAsInt();
         assertEquals(1, cache.numRegisteredCloseListeners());
 
         // cache the second
+        TestEntity secondEntity = new TestEntity(requestCacheStats, secondReader, indexShard, 0);
         value = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes());
-        assertEquals("bar", value.toUtf8());
+        assertEquals("bar", StreamInput.wrap(value).readString());
         assertEquals(0, requestCacheStats.stats().getHitCount());
         assertEquals(2, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
-        assertEquals(1, entity.loaded);
-        assertEquals(1, secondEntity.loaded);
+        assertFalse(secondEntity.loadedFromCache());
         assertEquals(2, cache.count());
         assertTrue(requestCacheStats.stats().getMemorySize().bytesAsInt() > cacheSize + value.length());
         assertEquals(2, cache.numRegisteredCloseListeners());
 
-
-
+        secondEntity = new TestEntity(requestCacheStats, secondReader, indexShard, 0);
         value = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes());
-        assertEquals("bar", value.toUtf8());
+        assertEquals("bar", StreamInput.wrap(value).readString());
         assertEquals(1, requestCacheStats.stats().getHitCount());
         assertEquals(2, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
-        assertEquals(1, entity.loaded);
-        assertEquals(1, secondEntity.loaded);
+        assertTrue(secondEntity.loadedFromCache());
         assertEquals(2, cache.count());
 
+        entity = new TestEntity(requestCacheStats, reader, indexShard, 0);
         value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes());
-        assertEquals("foo", value.toUtf8());
+        assertEquals("foo", StreamInput.wrap(value).readString());
         assertEquals(2, requestCacheStats.stats().getHitCount());
         assertEquals(2, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
-        assertEquals(1, entity.loaded);
-        assertEquals(1, secondEntity.loaded);
+        assertTrue(entity.loadedFromCache());
         assertEquals(2, cache.count());
 
+        // Closing the cache doesn't change returned entities
         reader.close();
         cache.cleanCache();
         assertEquals(2, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
-        assertEquals(1, entity.loaded);
-        assertEquals(1, secondEntity.loaded);
+        assertTrue(entity.loadedFromCache());
+        assertTrue(secondEntity.loadedFromCache());
         assertEquals(1, cache.count());
         assertEquals(cacheSize, requestCacheStats.stats().getMemorySize().bytesAsInt());
         assertEquals(1, cache.numRegisteredCloseListeners());
@@ -217,14 +179,13 @@ public class IndicesRequestCacheTests extends ESTestCase {
         cache.cleanCache();
         assertEquals(2, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
-        assertEquals(1, entity.loaded);
-        assertEquals(1, secondEntity.loaded);
+        assertTrue(entity.loadedFromCache());
+        assertTrue(secondEntity.loadedFromCache());
         assertEquals(0, cache.count());
         assertEquals(0, requestCacheStats.stats().getMemorySize().bytesAsInt());
 
         IOUtils.close(secondReader, writer, dir, cache);
         assertEquals(0, cache.numRegisteredCloseListeners());
-
     }
 
     public void testEviction() throws Exception {
@@ -248,9 +209,9 @@ public class IndicesRequestCacheTests extends ESTestCase {
             TestEntity secondEntity = new TestEntity(requestCacheStats, secondReader, indexShard, 0);
 
             BytesReference value1 = cache.getOrCompute(entity, reader, termQuery.buildAsBytes());
-            assertEquals("foo", value1.toUtf8());
+            assertEquals("foo", StreamInput.wrap(value1).readString());
             BytesReference value2 = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes());
-            assertEquals("bar", value2.toUtf8());
+            assertEquals("bar", StreamInput.wrap(value2).readString());
             size = requestCacheStats.stats().getMemorySize();
             IOUtils.close(reader, secondReader, writer, dir, cache);
         }
@@ -279,12 +240,12 @@ public class IndicesRequestCacheTests extends ESTestCase {
         TestEntity thirddEntity = new TestEntity(requestCacheStats, thirdReader, indexShard, 0);
 
         BytesReference value1 = cache.getOrCompute(entity, reader, termQuery.buildAsBytes());
-        assertEquals("foo", value1.toUtf8());
+        assertEquals("foo", StreamInput.wrap(value1).readString());
         BytesReference value2 = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes());
-        assertEquals("bar", value2.toUtf8());
+        assertEquals("bar", StreamInput.wrap(value2).readString());
         logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize());
         BytesReference value3 = cache.getOrCompute(thirddEntity, thirdReader, termQuery.buildAsBytes());
-        assertEquals("baz", value3.toUtf8());
+        assertEquals("baz", StreamInput.wrap(value3).readString());
         assertEquals(2, cache.count());
         assertEquals(1, requestCacheStats.stats().getEvictions());
         IOUtils.close(reader, secondReader, thirdReader, writer, dir, cache);
@@ -316,12 +277,12 @@ public class IndicesRequestCacheTests extends ESTestCase {
         TestEntity thirddEntity = new TestEntity(requestCacheStats, thirdReader, differentIdentity, 0);
 
         BytesReference value1 = cache.getOrCompute(entity, reader, termQuery.buildAsBytes());
-        assertEquals("foo", value1.toUtf8());
+        assertEquals("foo", StreamInput.wrap(value1).readString());
         BytesReference value2 = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes());
-        assertEquals("bar", value2.toUtf8());
+        assertEquals("bar", StreamInput.wrap(value2).readString());
         logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize());
         BytesReference value3 = cache.getOrCompute(thirddEntity, thirdReader, termQuery.buildAsBytes());
-        assertEquals("baz", value3.toUtf8());
+        assertEquals("baz", StreamInput.wrap(value3).readString());
         assertEquals(3, cache.count());
         final long hitCount = requestCacheStats.stats().getHitCount();
         // clear all for the indexShard Idendity even though is't still open
@@ -331,7 +292,7 @@ public class IndicesRequestCacheTests extends ESTestCase {
         // third has not been validated since it's a different identity
         value3 = cache.getOrCompute(thirddEntity, thirdReader, termQuery.buildAsBytes());
         assertEquals(hitCount + 1, requestCacheStats.stats().getHitCount());
-        assertEquals("baz", value3.toUtf8());
+        assertEquals("baz", StreamInput.wrap(value3).readString());
 
 
         IOUtils.close(reader, secondReader, thirdReader, writer, dir, cache);
@@ -343,59 +304,37 @@ public class IndicesRequestCacheTests extends ESTestCase {
             StringField.TYPE_STORED));
     }
 
-    private class TestEntity implements IndicesRequestCache.CacheEntity {
-        private final DirectoryReader reader;
-        private final int id;
-        private final AtomicBoolean identity;
+    private class TestEntity extends AbstractIndexShardCacheEntity {
+        private final AtomicBoolean standInForIndexShard;
         private final ShardRequestCache shardRequestCache;
-        private int loaded;
-        private TestEntity(ShardRequestCache shardRequestCache, DirectoryReader reader, AtomicBoolean identity, int id) {
-            this.reader = reader;
-            this.id = id;
-            this.identity = identity;
+        private TestEntity(ShardRequestCache shardRequestCache, DirectoryReader reader, AtomicBoolean standInForIndexShard, int id) {
+            super(new Loader() {
+                @Override
+                public void load(StreamOutput out) throws IOException {
+                    IndexSearcher searcher = new IndexSearcher(reader);
+                    TopDocs topDocs = searcher.search(new TermQuery(new Term("id", Integer.toString(id))), 1);
+                    assertEquals(1, topDocs.totalHits);
+                    Document document = reader.document(topDocs.scoreDocs[0].doc);
+                    out.writeString(document.get("value"));
+                }
+            });
+            this.standInForIndexShard = standInForIndexShard;
             this.shardRequestCache = shardRequestCache;
         }
 
         @Override
-        public IndicesRequestCache.Value loadValue() throws IOException {
-            IndexSearcher searcher = new IndexSearcher(reader);
-            TopDocs topDocs = searcher.search(new TermQuery(new Term("id", Integer.toString(this.id))), 1);
-            assertEquals(1, topDocs.totalHits);
-            Document document = reader.document(topDocs.scoreDocs[0].doc);
-            BytesArray value = new BytesArray(document.get("value"));
-            loaded++;
-            return new IndicesRequestCache.Value(value, value.length());
-        }
-
-        @Override
-        public void onCached(IndicesRequestCache.Key key, IndicesRequestCache.Value value) {
-            shardRequestCache.onCached(key, value);
+        protected ShardRequestCache stats() {
+            return shardRequestCache;
         }
 
         @Override
         public boolean isOpen() {
-           return identity.get();
+           return standInForIndexShard.get();
         }
 
         @Override
         public Object getCacheIdentity() {
-            return identity;
-        }
-
-        @Override
-        public void onHit() {
-            shardRequestCache.onHit();
-        }
-
-        @Override
-        public void onMiss() {
-            shardRequestCache.onMiss();
-        }
-
-        @Override
-        public void onRemoval(RemovalNotification<IndicesRequestCache.Key, IndicesRequestCache.Value> notification) {
-            shardRequestCache.onRemoval(notification.getKey(), notification.getValue(),
-                notification.getRemovalReason() == RemovalNotification.RemovalReason.EVICTED);
+            return standInForIndexShard;
         }
     }
 }

From 9c855698836f6d324e22d5c6c330e356dcc58169 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 13 May 2016 16:15:51 -0400
Subject: [PATCH 0841/1311] Test docs for plugins

We weren't doing it before because we weren't starting the plugins.
Now we are.

The hardest part of this was handling the files the tests expect
to be on the filesystem. extraConfigFiles was broken.
---
 docs/build.gradle                             | 25 +++++-
 docs/plugins/analysis-icu.asciidoc            | 11 ++-
 docs/plugins/analysis-kuromoji.asciidoc       | 17 +++-
 docs/plugins/analysis-phonetic.asciidoc       |  4 +-
 docs/plugins/lang-javascript.asciidoc         | 11 +--
 docs/plugins/lang-python.asciidoc             |  8 --
 docs/plugins/mapper-attachments.asciidoc      | 90 ++++++++++++++-----
 docs/plugins/mapper-size.asciidoc             | 28 +-----
 docs/plugins/repository-azure.asciidoc        |  2 +-
 docs/plugins/repository-gcs.asciidoc          | 12 +--
 docs/plugins/repository-s3.asciidoc           |  1 +
 .../test/cluster/config/KeywordTokenizer.rbbi |  1 +
 .../test/cluster/config/scripts/my_script.js  |  1 +
 .../test/cluster/config/scripts/my_script.py  |  1 +
 docs/src/test/cluster/config/userdict_ja.txt  |  1 +
 15 files changed, 130 insertions(+), 83 deletions(-)
 create mode 100644 docs/src/test/cluster/config/KeywordTokenizer.rbbi
 create mode 100644 docs/src/test/cluster/config/scripts/my_script.js
 create mode 100644 docs/src/test/cluster/config/scripts/my_script.py
 create mode 100644 docs/src/test/cluster/config/userdict_ja.txt

diff --git a/docs/build.gradle b/docs/build.gradle
index 239eb81e820..660755a1c65 100644
--- a/docs/build.gradle
+++ b/docs/build.gradle
@@ -22,6 +22,29 @@ apply plugin: 'elasticsearch.docs-test'
 integTest {
   cluster {
     setting 'script.inline', 'true'
+    setting 'script.stored', 'true'
+    Closure configFile = {
+      extraConfigFile it, "src/test/cluster/config/$it"
+    }
+    configFile 'scripts/my_script.js'
+    configFile 'scripts/my_script.py'
+    configFile 'userdict_ja.txt'
+    configFile 'KeywordTokenizer.rbbi'
+  }
+}
+
+// Build the cluser with all plugins
+project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj ->
+  /* Skip repositories. We just aren't going to be able to test them so it
+   * doesn't make sense to waste time installing them. */
+  if (subproj.path.startsWith(':plugins:repository-')) {
+    return
+  }
+  integTest {
+    cluster {
+      // We need a non-decorated project object, so we lookup the project by path
+      plugin subproj.name, project(subproj.path)
+    }
   }
 }
 
@@ -30,8 +53,6 @@ buildRestTests.docs = fileTree(projectDir) {
   exclude 'build.gradle'
   // That is where the snippets go, not where they come from!
   exclude 'build'
-  // Remove plugins because they aren't installed during this test. Yet?
-  exclude 'plugins'
   // This file simply doesn't pass yet. We should figure out how to fix it.
   exclude 'reference/modules/snapshots.asciidoc'
 }
diff --git a/docs/plugins/analysis-icu.asciidoc b/docs/plugins/analysis-icu.asciidoc
index 815e6285235..3dbe7b2e354 100644
--- a/docs/plugins/analysis-icu.asciidoc
+++ b/docs/plugins/analysis-icu.asciidoc
@@ -161,6 +161,8 @@ PUT icu_sample
     }
 }
 
+GET _cluster/health?wait_for_status=yellow
+
 POST icu_sample/_analyze?analyzer=my_analyzer&text=Elasticsearch. Wow!
 --------------------------------------------------
 // CONSOLE
@@ -169,7 +171,6 @@ The above `analyze` request returns the following:
 
 [source,js]
 --------------------------------------------------
-# Result
 {
    "tokens": [
       {
@@ -182,7 +183,7 @@ The above `analyze` request returns the following:
    ]
 }
 --------------------------------------------------
-
+// TESTRESPONSE
 
 [[analysis-icu-normalization]]
 ==== ICU Normalization Token Filter
@@ -253,7 +254,7 @@ PUT icu_sample
       "analysis": {
         "analyzer": {
           "folded": {
-            "tokenizer": "icu",
+            "tokenizer": "icu_tokenizer",
             "filter": [
               "icu_folding"
             ]
@@ -359,6 +360,8 @@ PUT /my_index
   }
 }
 
+GET _cluster/health?wait_for_status=yellow
+
 GET _search <3>
 {
   "query": {
@@ -478,6 +481,8 @@ PUT icu_sample
   }
 }
 
+GET _cluster/health?wait_for_status=yellow
+
 GET icu_sample/_analyze?analyzer=latin
 {
   "text": "你好" <2>
diff --git a/docs/plugins/analysis-kuromoji.asciidoc b/docs/plugins/analysis-kuromoji.asciidoc
index f29d13c2867..b0d6c1eb962 100644
--- a/docs/plugins/analysis-kuromoji.asciidoc
+++ b/docs/plugins/analysis-kuromoji.asciidoc
@@ -172,6 +172,8 @@ PUT kuromoji_sample
   }
 }
 
+GET _cluster/health?wait_for_status=yellow
+
 POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=�京スカイツリー
 --------------------------------------------------
 // CONSOLE
@@ -224,6 +226,8 @@ PUT kuromoji_sample
   }
 }
 
+GET _cluster/health?wait_for_status=yellow
+
 POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=飲�
 --------------------------------------------------
 // CONSOLE
@@ -282,6 +286,8 @@ PUT kuromoji_sample
   }
 }
 
+GET _cluster/health?wait_for_status=yellow
+
 POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=寿�������
 
 --------------------------------------------------
@@ -354,6 +360,8 @@ PUT kuromoji_sample
     }
 }
 
+GET _cluster/health?wait_for_status=yellow
+
 POST kuromoji_sample/_analyze?analyzer=katakana_analyzer&text=寿� <1>
 
 POST kuromoji_sample/_analyze?analyzer=romaji_analyzer&text=寿� <2>
@@ -405,6 +413,8 @@ PUT kuromoji_sample
   }
 }
 
+GET _cluster/health?wait_for_status=yellow
+
 POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=コピー <1>
 
 POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=サー�ー <2>
@@ -454,7 +464,9 @@ PUT kuromoji_sample
   }
 }
 
-POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=ストップ�消�る
+GET _cluster/health?wait_for_status=yellow
+
+POST kuromoji_sample/_analyze?analyzer=analyzer_with_ja_stop&text=ストップ�消�る
 --------------------------------------------------
 // CONSOLE
 
@@ -500,6 +512,8 @@ PUT kuromoji_sample
   }
 }
 
+GET _cluster/health?wait_for_status=yellow
+
 POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=一〇〇〇
 
 --------------------------------------------------
@@ -518,4 +532,3 @@ POST kuromoji_sample/_analyze?analyzer=my_analyzer&text=一〇〇〇
   } ]
 }
 --------------------------------------------------
-
diff --git a/docs/plugins/analysis-phonetic.asciidoc b/docs/plugins/analysis-phonetic.asciidoc
index f37547d9f75..b44218b7521 100644
--- a/docs/plugins/analysis-phonetic.asciidoc
+++ b/docs/plugins/analysis-phonetic.asciidoc
@@ -79,6 +79,8 @@ PUT phonetic_sample
   }
 }
 
+GET _cluster/health?wait_for_status=yellow
+
 POST phonetic_sample/_analyze?analyzer=my_analyzer&text=Joe Bloggs <1>
 --------------------------------------------------
 // CONSOLE
@@ -116,5 +118,3 @@ supported:
     be guessed. Accepts: `any`, `comomon`, `cyrillic`, `english`, `french`,
     `german`, `hebrew`, `hungarian`, `polish`, `romanian`, `russian`,
     `spanish`.
-
-
diff --git a/docs/plugins/lang-javascript.asciidoc b/docs/plugins/lang-javascript.asciidoc
index bfee18a0a89..b7f858d6b4e 100644
--- a/docs/plugins/lang-javascript.asciidoc
+++ b/docs/plugins/lang-javascript.asciidoc
@@ -53,8 +53,6 @@ you can use JavaScript as follows:
 
 [source,js]
 ----
-DELETE test
-
 PUT test/doc/1
 {
   "num": 1.0
@@ -96,8 +94,6 @@ you can use JavaScript as follows:
 
 [source,js]
 ----
-DELETE test
-
 PUT test/doc/1
 {
   "num": 1.0
@@ -129,7 +125,6 @@ GET test/_search
     }
   }
 }
-
 ----
 // CONSOLE
 
@@ -145,7 +140,7 @@ You can save your scripts to a file in the `config/scripts/` directory on
 every node. The `.javascript` file suffix identifies the script as containing
 JavaScript:
 
-First, save this file as `config/scripts/my_script.javascript` on every node
+First, save this file as `config/scripts/my_script.js` on every node
 in the cluster:
 
 [source,js]
@@ -157,8 +152,6 @@ then use the script as follows:
 
 [source,js]
 ----
-DELETE test
-
 PUT test/doc/1
 {
   "num": 1.0
@@ -185,9 +178,7 @@ GET test/_search
     }
   }
 }
-
 ----
 // CONSOLE
 
 <1> The function score query retrieves the script with filename `my_script.javascript`.
-
diff --git a/docs/plugins/lang-python.asciidoc b/docs/plugins/lang-python.asciidoc
index af2fc8a3a39..3920ff0ab0a 100644
--- a/docs/plugins/lang-python.asciidoc
+++ b/docs/plugins/lang-python.asciidoc
@@ -52,8 +52,6 @@ you can use Python as follows:
 
 [source,js]
 ----
-DELETE test
-
 PUT test/doc/1
 {
   "num": 1.0
@@ -95,8 +93,6 @@ you can use Python as follows:
 
 [source,js]
 ----
-DELETE test
-
 PUT test/doc/1
 {
   "num": 1.0
@@ -156,8 +152,6 @@ then use the script as follows:
 
 [source,js]
 ----
-DELETE test
-
 PUT test/doc/1
 {
   "num": 1.0
@@ -184,9 +178,7 @@ GET test/_search
     }
   }
 }
-
 ----
 // CONSOLE
 
 <1> The function score query retrieves the script with filename `my_script.py`.
-
diff --git a/docs/plugins/mapper-attachments.asciidoc b/docs/plugins/mapper-attachments.asciidoc
index 570d05a7513..86e80cb1a26 100644
--- a/docs/plugins/mapper-attachments.asciidoc
+++ b/docs/plugins/mapper-attachments.asciidoc
@@ -57,12 +57,13 @@ Index a new document populated with a `base64`-encoded attachment:
 
 [source,js]
 --------------------------
-POST /trying-out-mapper-attachments/person/1
+POST /trying-out-mapper-attachments/person/1?refresh
 {
   "cv": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0="
 }
 --------------------------
 // CONSOLE
+// TEST[continued]
 
 Search for the document using words in the attachment:
 
@@ -76,8 +77,35 @@ POST /trying-out-mapper-attachments/person/_search
 }}}
 --------------------------
 // CONSOLE
+// TEST[continued]
 
-If you get a hit for your indexed document, the plugin should be installed and working.
+If you get a hit for your indexed document, the plugin should be installed and working. It'll look like:
+
+[source,js]
+--------------------------
+{
+  "timed_out": false,
+  "took": 53,
+  "hits": {
+    "total": 1,
+    "max_score": 0.3125,
+    "hits": [
+      {
+        "_score": 0.3125,
+        "_index": "trying-out-mapper-attachments",
+        "_type": "person",
+        "_id": "1",
+        "_source": {
+          "cv": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0="
+        }
+      }
+    ]
+  },
+  "_shards": ...
+}
+--------------------------
+// TESTRESPONSE[s/"took": 53/"took": "$body.took"/]
+// TESTRESPONSE[s/"_shards": \.\.\./"_shards": "$body._shards"/]
 
 [[mapper-attachments-usage]]
 ==== Usage
@@ -87,13 +115,14 @@ Using the attachment type is simple, in your mapping JSON, simply set a certain
 [source,js]
 --------------------------
 PUT /test
-PUT /test/person/_mapping
 {
+  "mappings": {
     "person" : {
-        "properties" : {
-            "my_attachment" : { "type" : "attachment" }
-        }
+      "properties" : {
+        "my_attachment" : { "type" : "attachment" }
+      }
     }
+  }
 }
 --------------------------
 // CONSOLE
@@ -146,25 +175,40 @@ in the mappings. For example:
 
 [source,js]
 --------------------------
-PUT /test/person/_mapping
+PUT /test
 {
-    "person" : {
-        "properties" : {
-            "file" : {
-                "type" : "attachment",
-                "fields" : {
-                    "content" : {"index" : "no"},
-                    "title" : {"store" : "yes"},
-                    "date" : {"store" : "yes"},
-                    "author" : {"analyzer" : "myAnalyzer"},
-                    "keywords" : {"store" : "yes"},
-                    "content_type" : {"store" : "yes"},
-                    "content_length" : {"store" : "yes"},
-                    "language" : {"store" : "yes"}
-                }
-            }
+  "settings": {
+    "index": {
+      "analysis": {
+        "analyzer": {
+          "my_analyzer": {
+            "type": "custom",
+            "tokenizer": "standard",
+            "filter": ["standard"]
+          }
         }
+      }
     }
+  },
+  "mappings": {
+    "person" : {
+      "properties" : {
+        "file" : {
+          "type" : "attachment",
+          "fields" : {
+            "content" : {"index" : "no"},
+            "title" : {"store" : "yes"},
+            "date" : {"store" : "yes"},
+            "author" : {"analyzer" : "my_analyzer"},
+            "keywords" : {"store" : "yes"},
+            "content_type" : {"store" : "yes"},
+            "content_length" : {"store" : "yes"},
+            "language" : {"store" : "yes"}
+          }
+        }
+      }
+    }
+  }
 }
 --------------------------
 // CONSOLE
@@ -179,7 +223,6 @@ If you need to query on metadata fields, use the attachment field name dot the m
 
 [source,js]
 --------------------------
-DELETE /test
 PUT /test
 PUT /test/person/_mapping
 {
@@ -300,7 +343,6 @@ If you want to highlight your attachment content, you will need to set `"store":
 
 [source,js]
 --------------------------
-DELETE /test
 PUT /test
 PUT /test/person/_mapping
 {
diff --git a/docs/plugins/mapper-size.asciidoc b/docs/plugins/mapper-size.asciidoc
index 4111a15ef6c..800a640890a 100644
--- a/docs/plugins/mapper-size.asciidoc
+++ b/docs/plugins/mapper-size.asciidoc
@@ -52,8 +52,7 @@ PUT my_index
 --------------------------
 // CONSOLE
 
-The value of the `_size` field is accessible in queries, aggregations, scripts,
-and when sorting:
+The value of the `_size` field is accessible in queries:
 
 [source,js]
 --------------------------
@@ -76,33 +75,10 @@ GET my_index/_search
         "gt": 10
       }
     }
-  },
-  "aggs": {
-    "Sizes": {
-      "terms": {
-        "field": "_size", <2>
-        "size": 10
-      }
-    }
-  },
-  "sort": [
-    {
-      "_size": { <3>
-        "order": "desc"
-      }
-    }
-  ],
-  "script_fields": {
-    "Size": {
-      "script": "doc['_size']"  <4>
-    }
   }
 }
 --------------------------
 // CONSOLE
+// TEST[continued]
 
 <1> Querying on the `_size` field
-<2> Aggregating on the `_size` field
-<3> Sorting on the `_size` field
-<4> Accessing the `_size` field in scripts (inline scripts must be modules-security-scripting.html#enable-dynamic-scripting[enabled] for this example to work)
-
diff --git a/docs/plugins/repository-azure.asciidoc b/docs/plugins/repository-azure.asciidoc
index f338728a35d..03466f0c643 100644
--- a/docs/plugins/repository-azure.asciidoc
+++ b/docs/plugins/repository-azure.asciidoc
@@ -168,6 +168,7 @@ PUT _snapshot/my_backup4
 }
 ----
 // CONSOLE
+// TEST[skip:we don't have azure setup while testing this]
 
 Example using Java:
 
@@ -207,4 +208,3 @@ be a valid DNS name, conforming to the following naming rules:
 permitted in container names.
 * All letters in a container name must be lowercase.
 * Container names must be from 3 through 63 characters long.
-
diff --git a/docs/plugins/repository-gcs.asciidoc b/docs/plugins/repository-gcs.asciidoc
index ea2cb99ee9b..5b2c633f23e 100644
--- a/docs/plugins/repository-gcs.asciidoc
+++ b/docs/plugins/repository-gcs.asciidoc
@@ -82,7 +82,7 @@ https://console.cloud.google.com/compute/[Compute Engine console].
 To indicate that a repository should use the built-in authentication,
 the repository `service_account` setting must be set to `_default_`:
 
-[source,json]
+[source,js]
 ----
 PUT _snapshot/my_gcs_repository_on_compute_engine
 {
@@ -94,6 +94,7 @@ PUT _snapshot/my_gcs_repository_on_compute_engine
 }
 ----
 // CONSOLE
+// TEST[skip:we don't have gcs setup while testing this]
 
 NOTE: The Compute Engine VM must be allowed to use the Storage service. This can be done only at VM
 creation time, when "Storage" access can be configured to "Read/Write" permission. Check your
@@ -115,7 +116,7 @@ To create a service account file:
 
 A service account file looks like this:
 
-[source,json]
+[source,js]
 ----
 {
   "type": "service_account",
@@ -136,7 +137,7 @@ every node of the cluster.
 
 To indicate that a repository should use a service account file:
 
-[source,json]
+[source,js]
 ----
 PUT _snapshot/my_gcs_repository
 {
@@ -148,7 +149,7 @@ PUT _snapshot/my_gcs_repository
 }
 ----
 // CONSOLE
-
+// TEST[skip:we don't have gcs setup while testing this]
 
 [[repository-gcs-bucket-permission]]
 ===== Set Bucket Permission
@@ -168,7 +169,7 @@ The service account used to access the bucket must have the "Writer" access to t
 Once everything is installed and every node is started, you can create a new repository that
 uses Google Cloud Storage to store snapshots:
 
-[source,json]
+[source,js]
 ----
 PUT _snapshot/my_gcs_repository
 {
@@ -180,6 +181,7 @@ PUT _snapshot/my_gcs_repository
 }
 ----
 // CONSOLE
+// TEST[skip:we don't have gcs setup while testing this]
 
 The following settings are supported:
 
diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc
index 82485b6d9d0..f086158954c 100644
--- a/docs/plugins/repository-s3.asciidoc
+++ b/docs/plugins/repository-s3.asciidoc
@@ -149,6 +149,7 @@ PUT _snapshot/my_s3_repository
 }
 ----
 // CONSOLE
+// TEST[skip:we don't have s3 set up while testing this]
 
 The following settings are supported:
 
diff --git a/docs/src/test/cluster/config/KeywordTokenizer.rbbi b/docs/src/test/cluster/config/KeywordTokenizer.rbbi
new file mode 100644
index 00000000000..b08617c8991
--- /dev/null
+++ b/docs/src/test/cluster/config/KeywordTokenizer.rbbi
@@ -0,0 +1 @@
+.+ {200};
diff --git a/docs/src/test/cluster/config/scripts/my_script.js b/docs/src/test/cluster/config/scripts/my_script.js
new file mode 100644
index 00000000000..cc985c18a94
--- /dev/null
+++ b/docs/src/test/cluster/config/scripts/my_script.js
@@ -0,0 +1 @@
+doc["num"].value * factor
diff --git a/docs/src/test/cluster/config/scripts/my_script.py b/docs/src/test/cluster/config/scripts/my_script.py
new file mode 100644
index 00000000000..cc985c18a94
--- /dev/null
+++ b/docs/src/test/cluster/config/scripts/my_script.py
@@ -0,0 +1 @@
+doc["num"].value * factor
diff --git a/docs/src/test/cluster/config/userdict_ja.txt b/docs/src/test/cluster/config/userdict_ja.txt
new file mode 100644
index 00000000000..7fe69d2b8d8
--- /dev/null
+++ b/docs/src/test/cluster/config/userdict_ja.txt
@@ -0,0 +1 @@
+�京スカイツリー,�京 スカイツリー,トウキョウ スカイツリー,カスタム�詞

From b69e8aadbdab0902d989e2a52d63acf055a25b9c Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 14 Jun 2016 15:07:56 -0400
Subject: [PATCH 0842/1311] refactor tests

---
 .../elasticsearch/painless/node/EBinary.java  |    2 +-
 .../elasticsearch/painless/node/EComp.java    |    1 -
 .../elasticsearch/painless/node/EUnary.java   |   39 +-
 .../elasticsearch/painless/AdditionTests.java |  178 ++
 .../org/elasticsearch/painless/AndTests.java  |  154 ++
 .../painless/BasicExpressionTests.java        |   12 +
 .../painless/ComparisonTests.java             |  442 ++++
 .../painless/DefOperationTests.java           | 2352 -----------------
 .../painless/DefOptimizationTests.java        |   15 +
 .../elasticsearch/painless/DivisionTests.java |  203 ++
 .../painless/MultiplicationTests.java         |  202 ++
 .../org/elasticsearch/painless/OrTests.java   |  154 ++
 ...OperatorTests.java => PromotionTests.java} |   88 +-
 .../painless/RemainderTests.java              |  203 ++
 .../painless/ScriptTestCase.java              |    2 +-
 .../elasticsearch/painless/ShiftTests.java    |  547 ++++
 .../elasticsearch/painless/StringTests.java   |   23 +
 .../painless/SubtractionTests.java            |  179 ++
 .../elasticsearch/painless/UnaryTests.java    |   49 +
 .../org/elasticsearch/painless/XorTests.java  |  154 ++
 20 files changed, 2543 insertions(+), 2456 deletions(-)
 create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/ComparisonTests.java
 delete mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
 rename modules/lang-painless/src/test/java/org/elasticsearch/painless/{BinaryOperatorTests.java => PromotionTests.java} (72%)
 create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/ShiftTests.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 23f9a614338..90d7b819902 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -612,7 +612,7 @@ public final class EBinary extends AExpression {
             right.write(writer);
 
             if (promote.sort == Sort.DEF || (shiftDistance != null && shiftDistance.sort == Sort.DEF)) {
-                writer.writeDynamicBinaryInstruction(location, expected, left.actual, right.actual, operation);
+                writer.writeDynamicBinaryInstruction(location, actual, left.actual, right.actual, operation);
             } else {
                 writer.writeBinaryInstruction(location, actual, operation);
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index 9d9c6c03812..1f4f691d4aa 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -491,7 +491,6 @@ public final class EComp extends AExpression {
 
                 break;
             case DEF:
-                // XXX: move this out, so we can populate descriptor with what we really have (instead of casts/boxing!)
                 org.objectweb.asm.Type booleanType = org.objectweb.asm.Type.getType(boolean.class);
                 org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(booleanType, left.actual.type, right.actual.type);
                 if (eq) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index d184961fd28..bb2ff4d1d47 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -39,6 +39,7 @@ public final class EUnary extends AExpression {
 
     final Operation operation;
     AExpression child;
+    Type promote;
 
     public EUnary(Location location, Operation operation, AExpression child) {
         super(location);
@@ -77,7 +78,7 @@ public final class EUnary extends AExpression {
     void analyzeBWNot(Locals variables) {
         child.analyze(variables);
 
-        Type promote = AnalyzerCaster.promoteNumeric(child.actual, false);
+        promote = AnalyzerCaster.promoteNumeric(child.actual, false);
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply not [~] to type [" + child.actual.name + "]."));
@@ -98,13 +99,17 @@ public final class EUnary extends AExpression {
             }
         }
 
-        actual = promote;
+        if (promote.sort == Sort.DEF && expected != null) {
+            actual = expected;
+        } else {
+            actual = promote;
+        }
     }
 
     void analyzerAdd(Locals variables) {
         child.analyze(variables);
 
-        Type promote = AnalyzerCaster.promoteNumeric(child.actual, true);
+        promote = AnalyzerCaster.promoteNumeric(child.actual, true);
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply positive [+] to type [" + child.actual.name + "]."));
@@ -129,13 +134,17 @@ public final class EUnary extends AExpression {
             }
         }
 
-        actual = promote;
+        if (promote.sort == Sort.DEF && expected != null) {
+            actual = expected;
+        } else {
+            actual = promote;
+        }
     }
 
     void analyzerSub(Locals variables) {
         child.analyze(variables);
 
-        Type promote = AnalyzerCaster.promoteNumeric(child.actual, true);
+        promote = AnalyzerCaster.promoteNumeric(child.actual, true);
 
         if (promote == null) {
             throw createError(new ClassCastException("Cannot apply negative [-] to type [" + child.actual.name + "]."));
@@ -160,7 +169,11 @@ public final class EUnary extends AExpression {
             }
         }
 
-        actual = promote;
+        if (promote.sort == Sort.DEF && expected != null) {
+            actual = expected;
+        } else {
+            actual = promote;
+        }
     }
 
     @Override
@@ -186,14 +199,12 @@ public final class EUnary extends AExpression {
                 child.write(writer);
             }
         } else {
-            org.objectweb.asm.Type type = actual.type;
-            Sort sort = actual.sort;
-
+            Sort sort = promote.sort;
             child.write(writer);
 
             if (operation == Operation.BWNOT) {
                 if (sort == Sort.DEF) {
-                    org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(expected.type, child.actual.type);
+                    org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(actual.type, child.actual.type);
                     writer.invokeDynamic("not", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.UNARY_OPERATOR);
                 } else {
                     if (sort == Sort.INT) {
@@ -204,18 +215,18 @@ public final class EUnary extends AExpression {
                         throw createError(new IllegalStateException("Illegal tree structure."));
                     }
 
-                    writer.math(MethodWriter.XOR, type);
+                    writer.math(MethodWriter.XOR, actual.type);
                 }
             } else if (operation == Operation.SUB) {
                 if (sort == Sort.DEF) {
-                    org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(expected.type, child.actual.type);
+                    org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(actual.type, child.actual.type);
                     writer.invokeDynamic("neg", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.UNARY_OPERATOR);
                 } else {
-                    writer.math(MethodWriter.NEG, type);
+                    writer.math(MethodWriter.NEG, actual.type);
                 }
             } else if (operation == Operation.ADD) {
                 if (sort == Sort.DEF) {
-                    org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(expected.type, child.actual.type);
+                    org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(actual.type, child.actual.type);
                     writer.invokeDynamic("plus", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.UNARY_OPERATOR);
                 } 
             } else {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java
index 2a8195ca66d..c41ec15bd35 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java
@@ -22,6 +22,10 @@ package org.elasticsearch.painless;
 /** Tests for addition operator across all types */
 //TODO: NaN/Inf/overflow/...
 public class AdditionTests extends ScriptTestCase {
+    
+    public void testBasics() throws Exception {
+        assertEquals(3.0, exec("double x = 1; byte y = 2; return x + y;"));
+    }
 
     public void testInt() throws Exception {
         assertEquals(1+1, exec("int x = 1; int y = 1; return x+y;"));
@@ -191,4 +195,178 @@ public class AdditionTests extends ScriptTestCase {
         assertEquals(1.0+0.0, exec("return 1.0+0.0;"));
         assertEquals(0.0+0.0, exec("return 0.0+0.0;"));
     }
+    
+    public void testDef() {
+        assertEquals(2, exec("def x = (byte)1; def y = (byte)1; return x + y"));
+        assertEquals(2, exec("def x = (short)1; def y = (byte)1; return x + y"));
+        assertEquals(2, exec("def x = (char)1; def y = (byte)1; return x + y"));
+        assertEquals(2, exec("def x = (int)1; def y = (byte)1; return x + y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (byte)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; def y = (byte)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; def y = (byte)1; return x + y"));
+
+        assertEquals(2, exec("def x = (byte)1; def y = (short)1; return x + y"));
+        assertEquals(2, exec("def x = (short)1; def y = (short)1; return x + y"));
+        assertEquals(2, exec("def x = (char)1; def y = (short)1; return x + y"));
+        assertEquals(2, exec("def x = (int)1; def y = (short)1; return x + y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (short)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; def y = (short)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; def y = (short)1; return x + y"));
+
+        assertEquals(2, exec("def x = (byte)1; def y = (char)1; return x + y"));
+        assertEquals(2, exec("def x = (short)1; def y = (char)1; return x + y"));
+        assertEquals(2, exec("def x = (char)1; def y = (char)1; return x + y"));
+        assertEquals(2, exec("def x = (int)1; def y = (char)1; return x + y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (char)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; def y = (char)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; def y = (char)1; return x + y"));
+
+        assertEquals(2, exec("def x = (byte)1; def y = (int)1; return x + y"));
+        assertEquals(2, exec("def x = (short)1; def y = (int)1; return x + y"));
+        assertEquals(2, exec("def x = (char)1; def y = (int)1; return x + y"));
+        assertEquals(2, exec("def x = (int)1; def y = (int)1; return x + y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (int)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; def y = (int)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; def y = (int)1; return x + y"));
+
+        assertEquals(2L, exec("def x = (byte)1; def y = (long)1; return x + y"));
+        assertEquals(2L, exec("def x = (short)1; def y = (long)1; return x + y"));
+        assertEquals(2L, exec("def x = (char)1; def y = (long)1; return x + y"));
+        assertEquals(2L, exec("def x = (int)1; def y = (long)1; return x + y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (long)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; def y = (long)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; def y = (long)1; return x + y"));
+
+        assertEquals(2F, exec("def x = (byte)1; def y = (float)1; return x + y"));
+        assertEquals(2F, exec("def x = (short)1; def y = (float)1; return x + y"));
+        assertEquals(2F, exec("def x = (char)1; def y = (float)1; return x + y"));
+        assertEquals(2F, exec("def x = (int)1; def y = (float)1; return x + y"));
+        assertEquals(2F, exec("def x = (long)1; def y = (float)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; def y = (float)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; def y = (float)1; return x + y"));
+
+        assertEquals(2D, exec("def x = (byte)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (short)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (char)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (int)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (long)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (float)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; def y = (double)1; return x + y"));
+    }
+    
+    public void testDefTypedLHS() {
+        assertEquals(2, exec("byte x = (byte)1; def y = (byte)1; return x + y"));
+        assertEquals(2, exec("short x = (short)1; def y = (byte)1; return x + y"));
+        assertEquals(2, exec("char x = (char)1; def y = (byte)1; return x + y"));
+        assertEquals(2, exec("int x = (int)1; def y = (byte)1; return x + y"));
+        assertEquals(2L, exec("long x = (long)1; def y = (byte)1; return x + y"));
+        assertEquals(2F, exec("float x = (float)1; def y = (byte)1; return x + y"));
+        assertEquals(2D, exec("double x = (double)1; def y = (byte)1; return x + y"));
+
+        assertEquals(2, exec("byte x = (byte)1; def y = (short)1; return x + y"));
+        assertEquals(2, exec("short x = (short)1; def y = (short)1; return x + y"));
+        assertEquals(2, exec("char x = (char)1; def y = (short)1; return x + y"));
+        assertEquals(2, exec("int x = (int)1; def y = (short)1; return x + y"));
+        assertEquals(2L, exec("long x = (long)1; def y = (short)1; return x + y"));
+        assertEquals(2F, exec("float x = (float)1; def y = (short)1; return x + y"));
+        assertEquals(2D, exec("double x = (double)1; def y = (short)1; return x + y"));
+
+        assertEquals(2, exec("byte x = (byte)1; def y = (char)1; return x + y"));
+        assertEquals(2, exec("short x = (short)1; def y = (char)1; return x + y"));
+        assertEquals(2, exec("char x = (char)1; def y = (char)1; return x + y"));
+        assertEquals(2, exec("int x = (int)1; def y = (char)1; return x + y"));
+        assertEquals(2L, exec("long x = (long)1; def y = (char)1; return x + y"));
+        assertEquals(2F, exec("float x = (float)1; def y = (char)1; return x + y"));
+        assertEquals(2D, exec("double x = (double)1; def y = (char)1; return x + y"));
+
+        assertEquals(2, exec("byte x = (byte)1; def y = (int)1; return x + y"));
+        assertEquals(2, exec("short x = (short)1; def y = (int)1; return x + y"));
+        assertEquals(2, exec("char x = (char)1; def y = (int)1; return x + y"));
+        assertEquals(2, exec("int x = (int)1; def y = (int)1; return x + y"));
+        assertEquals(2L, exec("long x = (long)1; def y = (int)1; return x + y"));
+        assertEquals(2F, exec("float x = (float)1; def y = (int)1; return x + y"));
+        assertEquals(2D, exec("double x = (double)1; def y = (int)1; return x + y"));
+
+        assertEquals(2L, exec("byte x = (byte)1; def y = (long)1; return x + y"));
+        assertEquals(2L, exec("short x = (short)1; def y = (long)1; return x + y"));
+        assertEquals(2L, exec("char x = (char)1; def y = (long)1; return x + y"));
+        assertEquals(2L, exec("int x = (int)1; def y = (long)1; return x + y"));
+        assertEquals(2L, exec("long x = (long)1; def y = (long)1; return x + y"));
+        assertEquals(2F, exec("float x = (float)1; def y = (long)1; return x + y"));
+        assertEquals(2D, exec("double x = (double)1; def y = (long)1; return x + y"));
+
+        assertEquals(2F, exec("byte x = (byte)1; def y = (float)1; return x + y"));
+        assertEquals(2F, exec("short x = (short)1; def y = (float)1; return x + y"));
+        assertEquals(2F, exec("char x = (char)1; def y = (float)1; return x + y"));
+        assertEquals(2F, exec("int x = (int)1; def y = (float)1; return x + y"));
+        assertEquals(2F, exec("long x = (long)1; def y = (float)1; return x + y"));
+        assertEquals(2F, exec("float x = (float)1; def y = (float)1; return x + y"));
+        assertEquals(2D, exec("double x = (double)1; def y = (float)1; return x + y"));
+
+        assertEquals(2D, exec("byte x = (byte)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("short x = (short)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("char x = (char)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("int x = (int)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("long x = (long)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("float x = (float)1; def y = (double)1; return x + y"));
+        assertEquals(2D, exec("double x = (double)1; def y = (double)1; return x + y"));
+    }
+    
+    public void testDefTypedRHS() {
+        assertEquals(2, exec("def x = (byte)1; byte y = (byte)1; return x + y"));
+        assertEquals(2, exec("def x = (short)1; byte y = (byte)1; return x + y"));
+        assertEquals(2, exec("def x = (char)1; byte y = (byte)1; return x + y"));
+        assertEquals(2, exec("def x = (int)1; byte y = (byte)1; return x + y"));
+        assertEquals(2L, exec("def x = (long)1; byte y = (byte)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; byte y = (byte)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; byte y = (byte)1; return x + y"));
+
+        assertEquals(2, exec("def x = (byte)1; short y = (short)1; return x + y"));
+        assertEquals(2, exec("def x = (short)1; short y = (short)1; return x + y"));
+        assertEquals(2, exec("def x = (char)1; short y = (short)1; return x + y"));
+        assertEquals(2, exec("def x = (int)1; short y = (short)1; return x + y"));
+        assertEquals(2L, exec("def x = (long)1; short y = (short)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; short y = (short)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; short y = (short)1; return x + y"));
+
+        assertEquals(2, exec("def x = (byte)1; char y = (char)1; return x + y"));
+        assertEquals(2, exec("def x = (short)1; char y = (char)1; return x + y"));
+        assertEquals(2, exec("def x = (char)1; char y = (char)1; return x + y"));
+        assertEquals(2, exec("def x = (int)1; char y = (char)1; return x + y"));
+        assertEquals(2L, exec("def x = (long)1; char y = (char)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; char y = (char)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; char y = (char)1; return x + y"));
+
+        assertEquals(2, exec("def x = (byte)1; int y = (int)1; return x + y"));
+        assertEquals(2, exec("def x = (short)1; int y = (int)1; return x + y"));
+        assertEquals(2, exec("def x = (char)1; int y = (int)1; return x + y"));
+        assertEquals(2, exec("def x = (int)1; int y = (int)1; return x + y"));
+        assertEquals(2L, exec("def x = (long)1; int y = (int)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; int y = (int)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; int y = (int)1; return x + y"));
+
+        assertEquals(2L, exec("def x = (byte)1; long y = (long)1; return x + y"));
+        assertEquals(2L, exec("def x = (short)1; long y = (long)1; return x + y"));
+        assertEquals(2L, exec("def x = (char)1; long y = (long)1; return x + y"));
+        assertEquals(2L, exec("def x = (int)1; long y = (long)1; return x + y"));
+        assertEquals(2L, exec("def x = (long)1; long y = (long)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; long y = (long)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; long y = (long)1; return x + y"));
+
+        assertEquals(2F, exec("def x = (byte)1; float y = (float)1; return x + y"));
+        assertEquals(2F, exec("def x = (short)1; float y = (float)1; return x + y"));
+        assertEquals(2F, exec("def x = (char)1; float y = (float)1; return x + y"));
+        assertEquals(2F, exec("def x = (int)1; float y = (float)1; return x + y"));
+        assertEquals(2F, exec("def x = (long)1; float y = (float)1; return x + y"));
+        assertEquals(2F, exec("def x = (float)1; float y = (float)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; float y = (float)1; return x + y"));
+
+        assertEquals(2D, exec("def x = (byte)1; double y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (short)1; double y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (char)1; double y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (int)1; double y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (long)1; double y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (float)1; double y = (double)1; return x + y"));
+        assertEquals(2D, exec("def x = (double)1; double y = (double)1; return x + y"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java
index 6be068a10d1..42cf3611219 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java
@@ -21,6 +21,13 @@ package org.elasticsearch.painless;
 
 /** Tests for and operator across all types */
 public class AndTests extends ScriptTestCase {
+    
+    public void testBasics() throws Exception {
+        assertEquals(5 & 3, exec("return 5 & 3;"));
+        assertEquals(5 & 3L, exec("return 5 & 3L;"));
+        assertEquals(5L & 3, exec("return 5L & 3;"));
+        assertEquals(1L, exec("int x = 5; long y = 3; return x & y;"));
+    }
 
     public void testInt() throws Exception {
         assertEquals(5 & 12, exec("int x = 5; int y = 12; return x & y;"));
@@ -54,4 +61,151 @@ public class AndTests extends ScriptTestCase {
             exec("double x = (double)4; int y = 1; return x & y");
         });
     }
+    
+    public void testDef() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (float)4; def y = (byte)1; return x & y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (double)4; def y = (byte)1; return x & y");
+        });
+        assertEquals(0, exec("def x = (byte)4; def y = (byte)1; return x & y"));
+        assertEquals(0, exec("def x = (short)4; def y = (byte)1; return x & y"));
+        assertEquals(0, exec("def x = (char)4; def y = (byte)1; return x & y"));
+        assertEquals(0, exec("def x = (int)4; def y = (byte)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; def y = (byte)1; return x & y"));
+
+        assertEquals(0, exec("def x = (byte)4; def y = (short)1; return x & y"));
+        assertEquals(0, exec("def x = (short)4; def y = (short)1; return x & y"));
+        assertEquals(0, exec("def x = (char)4; def y = (short)1; return x & y"));
+        assertEquals(0, exec("def x = (int)4; def y = (short)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; def y = (short)1; return x & y"));
+
+        assertEquals(0, exec("def x = (byte)4; def y = (char)1; return x & y"));
+        assertEquals(0, exec("def x = (short)4; def y = (char)1; return x & y"));
+        assertEquals(0, exec("def x = (char)4; def y = (char)1; return x & y"));
+        assertEquals(0, exec("def x = (int)4; def y = (char)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; def y = (char)1; return x & y"));
+
+        assertEquals(0, exec("def x = (byte)4; def y = (int)1; return x & y"));
+        assertEquals(0, exec("def x = (short)4; def y = (int)1; return x & y"));
+        assertEquals(0, exec("def x = (char)4; def y = (int)1; return x & y"));
+        assertEquals(0, exec("def x = (int)4; def y = (int)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; def y = (int)1; return x & y"));
+
+        assertEquals(0L, exec("def x = (byte)4; def y = (long)1; return x & y"));
+        assertEquals(0L, exec("def x = (short)4; def y = (long)1; return x & y"));
+        assertEquals(0L, exec("def x = (char)4; def y = (long)1; return x & y"));
+        assertEquals(0L, exec("def x = (int)4; def y = (long)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; def y = (long)1; return x & y"));
+
+        assertEquals(0, exec("def x = (byte)4; def y = (byte)1; return x & y"));
+        assertEquals(0, exec("def x = (short)4; def y = (short)1; return x & y"));
+        assertEquals(0, exec("def x = (char)4; def y = (char)1; return x & y"));
+        assertEquals(0, exec("def x = (int)4; def y = (int)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; def y = (long)1; return x & y"));
+
+        assertEquals(true,  exec("def x = true;  def y = true; return x & y"));
+        assertEquals(false, exec("def x = true;  def y = false; return x & y"));
+        assertEquals(false, exec("def x = false; def y = true; return x & y"));
+        assertEquals(false, exec("def x = false; def y = false; return x & y"));
+    }
+    
+    public void testDefTypedLHS() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("float x = (float)4; def y = (byte)1; return x & y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("double x = (double)4; def y = (byte)1; return x & y");
+        });
+        assertEquals(0, exec("byte x = (byte)4; def y = (byte)1; return x & y"));
+        assertEquals(0, exec("short x = (short)4; def y = (byte)1; return x & y"));
+        assertEquals(0, exec("char x = (char)4; def y = (byte)1; return x & y"));
+        assertEquals(0, exec("int x = (int)4; def y = (byte)1; return x & y"));
+        assertEquals(0L, exec("long x = (long)4; def y = (byte)1; return x & y"));
+
+        assertEquals(0, exec("byte x = (byte)4; def y = (short)1; return x & y"));
+        assertEquals(0, exec("short x = (short)4; def y = (short)1; return x & y"));
+        assertEquals(0, exec("char x = (char)4; def y = (short)1; return x & y"));
+        assertEquals(0, exec("int x = (int)4; def y = (short)1; return x & y"));
+        assertEquals(0L, exec("long x = (long)4; def y = (short)1; return x & y"));
+
+        assertEquals(0, exec("byte x = (byte)4; def y = (char)1; return x & y"));
+        assertEquals(0, exec("short x = (short)4; def y = (char)1; return x & y"));
+        assertEquals(0, exec("char x = (char)4; def y = (char)1; return x & y"));
+        assertEquals(0, exec("int x = (int)4; def y = (char)1; return x & y"));
+        assertEquals(0L, exec("long x = (long)4; def y = (char)1; return x & y"));
+
+        assertEquals(0, exec("byte x = (byte)4; def y = (int)1; return x & y"));
+        assertEquals(0, exec("short x = (short)4; def y = (int)1; return x & y"));
+        assertEquals(0, exec("char x = (char)4; def y = (int)1; return x & y"));
+        assertEquals(0, exec("int x = (int)4; def y = (int)1; return x & y"));
+        assertEquals(0L, exec("long x = (long)4; def y = (int)1; return x & y"));
+
+        assertEquals(0L, exec("byte x = (byte)4; def y = (long)1; return x & y"));
+        assertEquals(0L, exec("short x = (short)4; def y = (long)1; return x & y"));
+        assertEquals(0L, exec("char x = (char)4; def y = (long)1; return x & y"));
+        assertEquals(0L, exec("int x = (int)4; def y = (long)1; return x & y"));
+        assertEquals(0L, exec("long x = (long)4; def y = (long)1; return x & y"));
+
+        assertEquals(0, exec("byte x = (byte)4; def y = (byte)1; return x & y"));
+        assertEquals(0, exec("short x = (short)4; def y = (short)1; return x & y"));
+        assertEquals(0, exec("char x = (char)4; def y = (char)1; return x & y"));
+        assertEquals(0, exec("int x = (int)4; def y = (int)1; return x & y"));
+        assertEquals(0L, exec("long x = (long)4; def y = (long)1; return x & y"));
+
+        assertEquals(true,  exec("boolean x = true;  def y = true; return x & y"));
+        assertEquals(false, exec("boolean x = true;  def y = false; return x & y"));
+        assertEquals(false, exec("boolean x = false; def y = true; return x & y"));
+        assertEquals(false, exec("boolean x = false; def y = false; return x & y"));
+    }
+    
+    public void testDefTypedRHS() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (float)4; byte y = (byte)1; return x & y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (double)4; byte y = (byte)1; return x & y");
+        });
+        assertEquals(0, exec("def x = (byte)4; byte y = (byte)1; return x & y"));
+        assertEquals(0, exec("def x = (short)4; byte y = (byte)1; return x & y"));
+        assertEquals(0, exec("def x = (char)4; byte y = (byte)1; return x & y"));
+        assertEquals(0, exec("def x = (int)4; byte y = (byte)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; byte y = (byte)1; return x & y"));
+
+        assertEquals(0, exec("def x = (byte)4; short y = (short)1; return x & y"));
+        assertEquals(0, exec("def x = (short)4; short y = (short)1; return x & y"));
+        assertEquals(0, exec("def x = (char)4; short y = (short)1; return x & y"));
+        assertEquals(0, exec("def x = (int)4; short y = (short)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; short y = (short)1; return x & y"));
+
+        assertEquals(0, exec("def x = (byte)4; char y = (char)1; return x & y"));
+        assertEquals(0, exec("def x = (short)4; char y = (char)1; return x & y"));
+        assertEquals(0, exec("def x = (char)4; char y = (char)1; return x & y"));
+        assertEquals(0, exec("def x = (int)4; char y = (char)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; char y = (char)1; return x & y"));
+
+        assertEquals(0, exec("def x = (byte)4; int y = (int)1; return x & y"));
+        assertEquals(0, exec("def x = (short)4; int y = (int)1; return x & y"));
+        assertEquals(0, exec("def x = (char)4; int y = (int)1; return x & y"));
+        assertEquals(0, exec("def x = (int)4; int y = (int)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; int y = (int)1; return x & y"));
+
+        assertEquals(0L, exec("def x = (byte)4; long y = (long)1; return x & y"));
+        assertEquals(0L, exec("def x = (short)4; long y = (long)1; return x & y"));
+        assertEquals(0L, exec("def x = (char)4; long y = (long)1; return x & y"));
+        assertEquals(0L, exec("def x = (int)4; long y = (long)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; long y = (long)1; return x & y"));
+
+        assertEquals(0, exec("def x = (byte)4; byte y = (byte)1; return x & y"));
+        assertEquals(0, exec("def x = (short)4; short y = (short)1; return x & y"));
+        assertEquals(0, exec("def x = (char)4; char y = (char)1; return x & y"));
+        assertEquals(0, exec("def x = (int)4; int y = (int)1; return x & y"));
+        assertEquals(0L, exec("def x = (long)4; long y = (long)1; return x & y"));
+
+        assertEquals(true,  exec("def x = true;  boolean y = true; return x & y"));
+        assertEquals(false, exec("def x = true;  boolean y = false; return x & y"));
+        assertEquals(false, exec("def x = false; boolean y = true; return x & y"));
+        assertEquals(false, exec("def x = false; boolean y = false; return x & y"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java
index 2a8f3674ac1..f59a9209f4c 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java
@@ -68,6 +68,18 @@ public class BasicExpressionTests extends ScriptTestCase {
                 "((Map)y).put(2, 3);\n" +
                 "return x.get(2);\n"));
     }
+    
+    public void testIllegalDefCast() {
+        Exception exception = expectScriptThrows(ClassCastException.class, () -> { 
+            exec("def x = 1.0; int y = x; return y;");
+        });
+        assertTrue(exception.getMessage().contains("cannot be cast"));
+
+        exception = expectScriptThrows(ClassCastException.class, () -> { 
+            exec("def x = (short)1; byte y = x; return y;");
+        });
+        assertTrue(exception.getMessage().contains("cannot be cast"));
+    }
 
     public void testCat() {
         assertEquals("aaabbb", exec("return \"aaa\" + \"bbb\";"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ComparisonTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ComparisonTests.java
new file mode 100644
index 00000000000..355a1223273
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ComparisonTests.java
@@ -0,0 +1,442 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+public class ComparisonTests extends ScriptTestCase {
+
+    public void testDefEq() {
+        assertEquals(true, exec("def x = (byte)7; def y = (int)7; return x == y"));
+        assertEquals(true, exec("def x = (short)6; def y = (int)6; return x == y"));
+        assertEquals(true, exec("def x = (char)5; def y = (int)5; return x == y"));
+        assertEquals(true, exec("def x = (int)4; def y = (int)4; return x == y"));
+        assertEquals(false, exec("def x = (long)5; def y = (int)3; return x == y"));
+        assertEquals(false, exec("def x = (float)6; def y = (int)2; return x == y"));
+        assertEquals(false, exec("def x = (double)7; def y = (int)1; return x == y"));
+
+        assertEquals(true, exec("def x = (byte)7; def y = (double)7; return x == y"));
+        assertEquals(true, exec("def x = (short)6; def y = (double)6; return x == y"));
+        assertEquals(true, exec("def x = (char)5; def y = (double)5; return x == y"));
+        assertEquals(true, exec("def x = (int)4; def y = (double)4; return x == y"));
+        assertEquals(false, exec("def x = (long)5; def y = (double)3; return x == y"));
+        assertEquals(false, exec("def x = (float)6; def y = (double)2; return x == y"));
+        assertEquals(false, exec("def x = (double)7; def y = (double)1; return x == y"));
+        
+        assertEquals(false, exec("def x = false; def y = true; return x == y"));
+        assertEquals(false, exec("def x = true; def y = false; return x == y"));
+        assertEquals(false, exec("def x = true; def y = null; return x == y"));
+        assertEquals(false, exec("def x = null; def y = true; return x == y"));
+        assertEquals(true, exec("def x = true; def y = true; return x == y"));
+        assertEquals(true, exec("def x = false; def y = false; return x == y"));
+
+        assertEquals(true, exec("def x = new HashMap(); def y = new HashMap(); return x == y"));
+        assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x == y"));
+        assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x == y"));
+        assertEquals(true, exec("def x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x == y"));
+    }
+    
+    public void testDefEqTypedLHS() {
+        assertEquals(true, exec("byte x = (byte)7; def y = (int)7; return x == y"));
+        assertEquals(true, exec("short x = (short)6; def y = (int)6; return x == y"));
+        assertEquals(true, exec("char x = (char)5; def y = (int)5; return x == y"));
+        assertEquals(true, exec("int x = (int)4; def y = (int)4; return x == y"));
+        assertEquals(false, exec("long x = (long)5; def y = (int)3; return x == y"));
+        assertEquals(false, exec("float x = (float)6; def y = (int)2; return x == y"));
+        assertEquals(false, exec("double x = (double)7; def y = (int)1; return x == y"));
+
+        assertEquals(true, exec("byte x = (byte)7; def y = (double)7; return x == y"));
+        assertEquals(true, exec("short x = (short)6; def y = (double)6; return x == y"));
+        assertEquals(true, exec("char x = (char)5; def y = (double)5; return x == y"));
+        assertEquals(true, exec("int x = (int)4; def y = (double)4; return x == y"));
+        assertEquals(false, exec("long x = (long)5; def y = (double)3; return x == y"));
+        assertEquals(false, exec("float x = (float)6; def y = (double)2; return x == y"));
+        assertEquals(false, exec("double x = (double)7; def y = (double)1; return x == y"));
+        
+        assertEquals(false, exec("boolean x = false; def y = true; return x == y"));
+        assertEquals(false, exec("boolean x = true; def y = false; return x == y"));
+        assertEquals(false, exec("boolean x = true; def y = null; return x == y"));
+        assertEquals(true, exec("boolean x = true; def y = true; return x == y"));
+        assertEquals(true, exec("boolean x = false; def y = false; return x == y"));
+
+        assertEquals(true, exec("Map x = new HashMap(); def y = new HashMap(); return x == y"));
+        assertEquals(false, exec("Map x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x == y"));
+        assertEquals(true, exec("Map x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x == y"));
+        assertEquals(true, exec("Map x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x == y"));
+    }
+    
+    public void testDefEqTypedRHS() {
+        assertEquals(true, exec("def x = (byte)7; int y = (int)7; return x == y"));
+        assertEquals(true, exec("def x = (short)6; int y = (int)6; return x == y"));
+        assertEquals(true, exec("def x = (char)5; int y = (int)5; return x == y"));
+        assertEquals(true, exec("def x = (int)4; int y = (int)4; return x == y"));
+        assertEquals(false, exec("def x = (long)5; int y = (int)3; return x == y"));
+        assertEquals(false, exec("def x = (float)6; int y = (int)2; return x == y"));
+        assertEquals(false, exec("def x = (double)7; int y = (int)1; return x == y"));
+
+        assertEquals(true, exec("def x = (byte)7; double y = (double)7; return x == y"));
+        assertEquals(true, exec("def x = (short)6; double y = (double)6; return x == y"));
+        assertEquals(true, exec("def x = (char)5; double y = (double)5; return x == y"));
+        assertEquals(true, exec("def x = (int)4; double y = (double)4; return x == y"));
+        assertEquals(false, exec("def x = (long)5; double y = (double)3; return x == y"));
+        assertEquals(false, exec("def x = (float)6; double y = (double)2; return x == y"));
+        assertEquals(false, exec("def x = (double)7; double y = (double)1; return x == y"));
+        
+        assertEquals(false, exec("def x = false; boolean y = true; return x == y"));
+        assertEquals(false, exec("def x = true; boolean y = false; return x == y"));
+        assertEquals(false, exec("def x = null; boolean y = true; return x == y"));
+        assertEquals(true, exec("def x = true; boolean y = true; return x == y"));
+        assertEquals(true, exec("def x = false; boolean y = false; return x == y"));
+
+        assertEquals(true, exec("def x = new HashMap(); Map y = new HashMap(); return x == y"));
+        assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); Map y = new HashMap(); return x == y"));
+        assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); Map y = new HashMap(); y.put(3, 3); return x == y"));
+        assertEquals(true, exec("def x = new HashMap(); Map y = x; x.put(3, 3); y.put(3, 3); return x == y"));
+    }
+
+    public void testDefEqr() {
+        assertEquals(false, exec("def x = (byte)7; def y = (int)7; return x === y"));
+        assertEquals(false, exec("def x = (short)6; def y = (int)6; return x === y"));
+        assertEquals(false, exec("def x = (char)5; def y = (int)5; return x === y"));
+        assertEquals(true, exec("def x = (int)4; def y = (int)4; return x === y"));
+        assertEquals(false, exec("def x = (long)5; def y = (int)3; return x === y"));
+        assertEquals(false, exec("def x = (float)6; def y = (int)2; return x === y"));
+        assertEquals(false, exec("def x = (double)7; def y = (int)1; return x === y"));
+        assertEquals(false, exec("def x = false; def y = true; return x === y"));
+
+        assertEquals(false, exec("def x = new HashMap(); def y = new HashMap(); return x === y"));
+        assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x === y"));
+        assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x === y"));
+        assertEquals(true, exec("def x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x === y"));
+    }
+
+    public void testDefNe() {
+        assertEquals(false, exec("def x = (byte)7; def y = (int)7; return x != y"));
+        assertEquals(false, exec("def x = (short)6; def y = (int)6; return x != y"));
+        assertEquals(false, exec("def x = (char)5; def y = (int)5; return x != y"));
+        assertEquals(false, exec("def x = (int)4; def y = (int)4; return x != y"));
+        assertEquals(true, exec("def x = (long)5; def y = (int)3; return x != y"));
+        assertEquals(true, exec("def x = (float)6; def y = (int)2; return x != y"));
+        assertEquals(true, exec("def x = (double)7; def y = (int)1; return x != y"));
+
+        assertEquals(false, exec("def x = (byte)7; def y = (double)7; return x != y"));
+        assertEquals(false, exec("def x = (short)6; def y = (double)6; return x != y"));
+        assertEquals(false, exec("def x = (char)5; def y = (double)5; return x != y"));
+        assertEquals(false, exec("def x = (int)4; def y = (double)4; return x != y"));
+        assertEquals(true, exec("def x = (long)5; def y = (double)3; return x != y"));
+        assertEquals(true, exec("def x = (float)6; def y = (double)2; return x != y"));
+        assertEquals(true, exec("def x = (double)7; def y = (double)1; return x != y"));
+
+        assertEquals(false, exec("def x = new HashMap(); def y = new HashMap(); return x != y"));
+        assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x != y"));
+        assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x != y"));
+        assertEquals(false, exec("def x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x != y"));
+        
+        assertEquals(false,  exec("def x = true;  def y = true; return x != y"));
+        assertEquals(true,   exec("def x = true;  def y = false; return x != y"));
+        assertEquals(true,   exec("def x = false; def y = true; return x != y"));
+        assertEquals(false,  exec("def x = false; def y = false; return x != y"));
+    }
+    
+    public void testDefNeTypedLHS() {
+        assertEquals(false, exec("byte x = (byte)7; def y = (int)7; return x != y"));
+        assertEquals(false, exec("short x = (short)6; def y = (int)6; return x != y"));
+        assertEquals(false, exec("char x = (char)5; def y = (int)5; return x != y"));
+        assertEquals(false, exec("int x = (int)4; def y = (int)4; return x != y"));
+        assertEquals(true, exec("long x = (long)5; def y = (int)3; return x != y"));
+        assertEquals(true, exec("float x = (float)6; def y = (int)2; return x != y"));
+        assertEquals(true, exec("double x = (double)7; def y = (int)1; return x != y"));
+
+        assertEquals(false, exec("byte x = (byte)7; def y = (double)7; return x != y"));
+        assertEquals(false, exec("short x = (short)6; def y = (double)6; return x != y"));
+        assertEquals(false, exec("char x = (char)5; def y = (double)5; return x != y"));
+        assertEquals(false, exec("int x = (int)4; def y = (double)4; return x != y"));
+        assertEquals(true, exec("long x = (long)5; def y = (double)3; return x != y"));
+        assertEquals(true, exec("float x = (float)6; def y = (double)2; return x != y"));
+        assertEquals(true, exec("double x = (double)7; def y = (double)1; return x != y"));
+
+        assertEquals(false, exec("Map x = new HashMap(); def y = new HashMap(); return x != y"));
+        assertEquals(true, exec("Map x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x != y"));
+        assertEquals(false, exec("Map x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x != y"));
+        assertEquals(false, exec("Map x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x != y"));
+        
+        assertEquals(false,  exec("boolean x = true;  def y = true; return x != y"));
+        assertEquals(true,   exec("boolean x = true;  def y = false; return x != y"));
+        assertEquals(true,   exec("boolean x = false; def y = true; return x != y"));
+        assertEquals(false,  exec("boolean x = false; def y = false; return x != y"));
+    }
+    
+    public void testDefNeTypedRHS() {
+        assertEquals(false, exec("def x = (byte)7; int y = (int)7; return x != y"));
+        assertEquals(false, exec("def x = (short)6; int y = (int)6; return x != y"));
+        assertEquals(false, exec("def x = (char)5; int y = (int)5; return x != y"));
+        assertEquals(false, exec("def x = (int)4; int y = (int)4; return x != y"));
+        assertEquals(true, exec("def x = (long)5; int y = (int)3; return x != y"));
+        assertEquals(true, exec("def x = (float)6; int y = (int)2; return x != y"));
+        assertEquals(true, exec("def x = (double)7; int y = (int)1; return x != y"));
+
+        assertEquals(false, exec("def x = (byte)7; double y = (double)7; return x != y"));
+        assertEquals(false, exec("def x = (short)6; double y = (double)6; return x != y"));
+        assertEquals(false, exec("def x = (char)5; double y = (double)5; return x != y"));
+        assertEquals(false, exec("def x = (int)4; double y = (double)4; return x != y"));
+        assertEquals(true, exec("def x = (long)5; double y = (double)3; return x != y"));
+        assertEquals(true, exec("def x = (float)6; double y = (double)2; return x != y"));
+        assertEquals(true, exec("def x = (double)7; double y = (double)1; return x != y"));
+
+        assertEquals(false, exec("def x = new HashMap(); Map y = new HashMap(); return x != y"));
+        assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); Map y = new HashMap(); return x != y"));
+        assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); Map y = new HashMap(); y.put(3, 3); return x != y"));
+        assertEquals(false, exec("def x = new HashMap(); Map y = x; x.put(3, 3); y.put(3, 3); return x != y"));
+        
+        assertEquals(false,  exec("def x = true;  boolean y = true; return x != y"));
+        assertEquals(true,   exec("def x = true;  boolean y = false; return x != y"));
+        assertEquals(true,   exec("def x = false; boolean y = true; return x != y"));
+        assertEquals(false,  exec("def x = false; boolean y = false; return x != y"));
+    }
+
+    public void testDefNer() {
+        assertEquals(true, exec("def x = (byte)7; def y = (int)7; return x !== y"));
+        assertEquals(true, exec("def x = (short)6; def y = (int)6; return x !== y"));
+        assertEquals(true, exec("def x = (char)5; def y = (int)5; return x !== y"));
+        assertEquals(false, exec("def x = (int)4; def y = (int)4; return x !== y"));
+        assertEquals(true, exec("def x = (long)5; def y = (int)3; return x !== y"));
+        assertEquals(true, exec("def x = (float)6; def y = (int)2; return x !== y"));
+        assertEquals(true, exec("def x = (double)7; def y = (int)1; return x !== y"));
+
+        assertEquals(true, exec("def x = new HashMap(); def y = new HashMap(); return x !== y"));
+        assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x !== y"));
+        assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x !== y"));
+        assertEquals(false, exec("def x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x !== y"));
+    }
+
+    public void testDefLt() {
+        assertEquals(true, exec("def x = (byte)1; def y = (int)7; return x < y"));
+        assertEquals(true, exec("def x = (short)2; def y = (int)6; return x < y"));
+        assertEquals(true, exec("def x = (char)3; def y = (int)5; return x < y"));
+        assertEquals(false, exec("def x = (int)4; def y = (int)4; return x < y"));
+        assertEquals(false, exec("def x = (long)5; def y = (int)3; return x < y"));
+        assertEquals(false, exec("def x = (float)6; def y = (int)2; return x < y"));
+        assertEquals(false, exec("def x = (double)7; def y = (int)1; return x < y"));
+
+        assertEquals(true, exec("def x = (byte)1; def y = (double)7; return x < y"));
+        assertEquals(true, exec("def x = (short)2; def y = (double)6; return x < y"));
+        assertEquals(true, exec("def x = (char)3; def y = (double)5; return x < y"));
+        assertEquals(false, exec("def x = (int)4; def y = (double)4; return x < y"));
+        assertEquals(false, exec("def x = (long)5; def y = (double)3; return x < y"));
+        assertEquals(false, exec("def x = (float)6; def y = (double)2; return x < y"));
+        assertEquals(false, exec("def x = (double)7; def y = (double)1; return x < y"));
+    }
+    
+    public void testDefLtTypedLHS() {
+        assertEquals(true, exec("byte x = (byte)1; def y = (int)7; return x < y"));
+        assertEquals(true, exec("short x = (short)2; def y = (int)6; return x < y"));
+        assertEquals(true, exec("char x = (char)3; def y = (int)5; return x < y"));
+        assertEquals(false, exec("int x = (int)4; def y = (int)4; return x < y"));
+        assertEquals(false, exec("long x = (long)5; def y = (int)3; return x < y"));
+        assertEquals(false, exec("float x = (float)6; def y = (int)2; return x < y"));
+        assertEquals(false, exec("double x = (double)7; def y = (int)1; return x < y"));
+
+        assertEquals(true, exec("byte x = (byte)1; def y = (double)7; return x < y"));
+        assertEquals(true, exec("short x = (short)2; def y = (double)6; return x < y"));
+        assertEquals(true, exec("char x = (char)3; def y = (double)5; return x < y"));
+        assertEquals(false, exec("int x = (int)4; def y = (double)4; return x < y"));
+        assertEquals(false, exec("long x = (long)5; def y = (double)3; return x < y"));
+        assertEquals(false, exec("float x = (float)6; def y = (double)2; return x < y"));
+        assertEquals(false, exec("double x = (double)7; def y = (double)1; return x < y"));
+    }
+    
+    public void testDefLtTypedRHS() {
+        assertEquals(true, exec("def x = (byte)1; int y = (int)7; return x < y"));
+        assertEquals(true, exec("def x = (short)2; int y = (int)6; return x < y"));
+        assertEquals(true, exec("def x = (char)3; int y = (int)5; return x < y"));
+        assertEquals(false, exec("def x = (int)4; int y = (int)4; return x < y"));
+        assertEquals(false, exec("def x = (long)5; int y = (int)3; return x < y"));
+        assertEquals(false, exec("def x = (float)6; int y = (int)2; return x < y"));
+        assertEquals(false, exec("def x = (double)7; int y = (int)1; return x < y"));
+
+        assertEquals(true, exec("def x = (byte)1; double y = (double)7; return x < y"));
+        assertEquals(true, exec("def x = (short)2; double y = (double)6; return x < y"));
+        assertEquals(true, exec("def x = (char)3; double y = (double)5; return x < y"));
+        assertEquals(false, exec("def x = (int)4; double y = (double)4; return x < y"));
+        assertEquals(false, exec("def x = (long)5; double y = (double)3; return x < y"));
+        assertEquals(false, exec("def x = (float)6; double y = (double)2; return x < y"));
+        assertEquals(false, exec("def x = (double)7; double y = (double)1; return x < y"));
+    }
+
+    public void testDefLte() {
+        assertEquals(true, exec("def x = (byte)1; def y = (int)7; return x <= y"));
+        assertEquals(true, exec("def x = (short)2; def y = (int)6; return x <= y"));
+        assertEquals(true, exec("def x = (char)3; def y = (int)5; return x <= y"));
+        assertEquals(true, exec("def x = (int)4; def y = (int)4; return x <= y"));
+        assertEquals(false, exec("def x = (long)5; def y = (int)3; return x <= y"));
+        assertEquals(false, exec("def x = (float)6; def y = (int)2; return x <= y"));
+        assertEquals(false, exec("def x = (double)7; def y = (int)1; return x <= y"));
+
+        assertEquals(true, exec("def x = (byte)1; def y = (double)7; return x <= y"));
+        assertEquals(true, exec("def x = (short)2; def y = (double)6; return x <= y"));
+        assertEquals(true, exec("def x = (char)3; def y = (double)5; return x <= y"));
+        assertEquals(true, exec("def x = (int)4; def y = (double)4; return x <= y"));
+        assertEquals(false, exec("def x = (long)5; def y = (double)3; return x <= y"));
+        assertEquals(false, exec("def x = (float)6; def y = (double)2; return x <= y"));
+        assertEquals(false, exec("def x = (double)7; def y = (double)1; return x <= y"));
+    }
+    
+    public void testDefLteTypedLHS() {
+        assertEquals(true, exec("byte x = (byte)1; def y = (int)7; return x <= y"));
+        assertEquals(true, exec("short x = (short)2; def y = (int)6; return x <= y"));
+        assertEquals(true, exec("char x = (char)3; def y = (int)5; return x <= y"));
+        assertEquals(true, exec("int x = (int)4; def y = (int)4; return x <= y"));
+        assertEquals(false, exec("long x = (long)5; def y = (int)3; return x <= y"));
+        assertEquals(false, exec("float x = (float)6; def y = (int)2; return x <= y"));
+        assertEquals(false, exec("double x = (double)7; def y = (int)1; return x <= y"));
+
+        assertEquals(true, exec("byte x = (byte)1; def y = (double)7; return x <= y"));
+        assertEquals(true, exec("short x = (short)2; def y = (double)6; return x <= y"));
+        assertEquals(true, exec("char x = (char)3; def y = (double)5; return x <= y"));
+        assertEquals(true, exec("int x = (int)4; def y = (double)4; return x <= y"));
+        assertEquals(false, exec("long x = (long)5; def y = (double)3; return x <= y"));
+        assertEquals(false, exec("float x = (float)6; def y = (double)2; return x <= y"));
+        assertEquals(false, exec("double x = (double)7; def y = (double)1; return x <= y"));
+    }
+    
+    public void testDefLteTypedRHS() {
+        assertEquals(true, exec("def x = (byte)1; int y = (int)7; return x <= y"));
+        assertEquals(true, exec("def x = (short)2; int y = (int)6; return x <= y"));
+        assertEquals(true, exec("def x = (char)3; int y = (int)5; return x <= y"));
+        assertEquals(true, exec("def x = (int)4; int y = (int)4; return x <= y"));
+        assertEquals(false, exec("def x = (long)5; int y = (int)3; return x <= y"));
+        assertEquals(false, exec("def x = (float)6; int y = (int)2; return x <= y"));
+        assertEquals(false, exec("def x = (double)7; int y = (int)1; return x <= y"));
+
+        assertEquals(true, exec("def x = (byte)1; double y = (double)7; return x <= y"));
+        assertEquals(true, exec("def x = (short)2; double y = (double)6; return x <= y"));
+        assertEquals(true, exec("def x = (char)3; double y = (double)5; return x <= y"));
+        assertEquals(true, exec("def x = (int)4; double y = (double)4; return x <= y"));
+        assertEquals(false, exec("def x = (long)5; double y = (double)3; return x <= y"));
+        assertEquals(false, exec("def x = (float)6; double y = (double)2; return x <= y"));
+        assertEquals(false, exec("def x = (double)7; double y = (double)1; return x <= y"));
+    }
+
+    public void testDefGt() {
+        assertEquals(false, exec("def x = (byte)1; def y = (int)7; return x > y"));
+        assertEquals(false, exec("def x = (short)2; def y = (int)6; return x > y"));
+        assertEquals(false, exec("def x = (char)3; def y = (int)5; return x > y"));
+        assertEquals(false, exec("def x = (int)4; def y = (int)4; return x > y"));
+        assertEquals(true, exec("def x = (long)5; def y = (int)3; return x > y"));
+        assertEquals(true, exec("def x = (float)6; def y = (int)2; return x > y"));
+        assertEquals(true, exec("def x = (double)7; def y = (int)1; return x > y"));
+
+        assertEquals(false, exec("def x = (byte)1; def y = (double)7; return x > y"));
+        assertEquals(false, exec("def x = (short)2; def y = (double)6; return x > y"));
+        assertEquals(false, exec("def x = (char)3; def y = (double)5; return x > y"));
+        assertEquals(false, exec("def x = (int)4; def y = (double)4; return x > y"));
+        assertEquals(true, exec("def x = (long)5; def y = (double)3; return x > y"));
+        assertEquals(true, exec("def x = (float)6; def y = (double)2; return x > y"));
+        assertEquals(true, exec("def x = (double)7; def y = (double)1; return x > y"));
+    }
+    
+    public void testDefGtTypedLHS() {
+        assertEquals(false, exec("byte x = (byte)1; def y = (int)7; return x > y"));
+        assertEquals(false, exec("short x = (short)2; def y = (int)6; return x > y"));
+        assertEquals(false, exec("char x = (char)3; def y = (int)5; return x > y"));
+        assertEquals(false, exec("int x = (int)4; def y = (int)4; return x > y"));
+        assertEquals(true, exec("long x = (long)5; def y = (int)3; return x > y"));
+        assertEquals(true, exec("float x = (float)6; def y = (int)2; return x > y"));
+        assertEquals(true, exec("double x = (double)7; def y = (int)1; return x > y"));
+
+        assertEquals(false, exec("byte x = (byte)1; def y = (double)7; return x > y"));
+        assertEquals(false, exec("short x = (short)2; def y = (double)6; return x > y"));
+        assertEquals(false, exec("char x = (char)3; def y = (double)5; return x > y"));
+        assertEquals(false, exec("int x = (int)4; def y = (double)4; return x > y"));
+        assertEquals(true, exec("long x = (long)5; def y = (double)3; return x > y"));
+        assertEquals(true, exec("float x = (float)6; def y = (double)2; return x > y"));
+        assertEquals(true, exec("double x = (double)7; def y = (double)1; return x > y"));
+    }
+    
+    public void testDefGtTypedRHS() {
+        assertEquals(false, exec("def x = (byte)1; int y = (int)7; return x > y"));
+        assertEquals(false, exec("def x = (short)2; int y = (int)6; return x > y"));
+        assertEquals(false, exec("def x = (char)3; int y = (int)5; return x > y"));
+        assertEquals(false, exec("def x = (int)4; int y = (int)4; return x > y"));
+        assertEquals(true, exec("def x = (long)5; int y = (int)3; return x > y"));
+        assertEquals(true, exec("def x = (float)6; int y = (int)2; return x > y"));
+        assertEquals(true, exec("def x = (double)7; int y = (int)1; return x > y"));
+
+        assertEquals(false, exec("def x = (byte)1; double y = (double)7; return x > y"));
+        assertEquals(false, exec("def x = (short)2; double y = (double)6; return x > y"));
+        assertEquals(false, exec("def x = (char)3; double y = (double)5; return x > y"));
+        assertEquals(false, exec("def x = (int)4; double y = (double)4; return x > y"));
+        assertEquals(true, exec("def x = (long)5; double y = (double)3; return x > y"));
+        assertEquals(true, exec("def x = (float)6; double y = (double)2; return x > y"));
+        assertEquals(true, exec("def x = (double)7; double y = (double)1; return x > y"));
+    }
+
+    public void testDefGte() {
+        assertEquals(false, exec("def x = (byte)1; def y = (int)7; return x >= y"));
+        assertEquals(false, exec("def x = (short)2; def y = (int)6; return x >= y"));
+        assertEquals(false, exec("def x = (char)3; def y = (int)5; return x >= y"));
+        assertEquals(true, exec("def x = (int)4; def y = (int)4; return x >= y"));
+        assertEquals(true, exec("def x = (long)5; def y = (int)3; return x >= y"));
+        assertEquals(true, exec("def x = (float)6; def y = (int)2; return x >= y"));
+        assertEquals(true, exec("def x = (double)7; def y = (int)1; return x >= y"));
+
+        assertEquals(false, exec("def x = (byte)1; def y = (double)7; return x >= y"));
+        assertEquals(false, exec("def x = (short)2; def y = (double)6; return x >= y"));
+        assertEquals(false, exec("def x = (char)3; def y = (double)5; return x >= y"));
+        assertEquals(true, exec("def x = (int)4; def y = (double)4; return x >= y"));
+        assertEquals(true, exec("def x = (long)5; def y = (double)3; return x >= y"));
+        assertEquals(true, exec("def x = (float)6; def y = (double)2; return x >= y"));
+        assertEquals(true, exec("def x = (double)7; def y = (double)1; return x >= y"));
+    }
+    
+    public void testDefGteTypedLHS() {
+        assertEquals(false, exec("byte x = (byte)1; def y = (int)7; return x >= y"));
+        assertEquals(false, exec("short x = (short)2; def y = (int)6; return x >= y"));
+        assertEquals(false, exec("char x = (char)3; def y = (int)5; return x >= y"));
+        assertEquals(true, exec("int x = (int)4; def y = (int)4; return x >= y"));
+        assertEquals(true, exec("long x = (long)5; def y = (int)3; return x >= y"));
+        assertEquals(true, exec("float x = (float)6; def y = (int)2; return x >= y"));
+        assertEquals(true, exec("double x = (double)7; def y = (int)1; return x >= y"));
+
+        assertEquals(false, exec("byte x = (byte)1; def y = (double)7; return x >= y"));
+        assertEquals(false, exec("short x = (short)2; def y = (double)6; return x >= y"));
+        assertEquals(false, exec("char x = (char)3; def y = (double)5; return x >= y"));
+        assertEquals(true, exec("int x = (int)4; def y = (double)4; return x >= y"));
+        assertEquals(true, exec("long x = (long)5; def y = (double)3; return x >= y"));
+        assertEquals(true, exec("float x = (float)6; def y = (double)2; return x >= y"));
+        assertEquals(true, exec("double x = (double)7; def y = (double)1; return x >= y"));
+    }
+    
+    public void testDefGteTypedRHS() {
+        assertEquals(false, exec("def x = (byte)1; int y = (int)7; return x >= y"));
+        assertEquals(false, exec("def x = (short)2; int y = (int)6; return x >= y"));
+        assertEquals(false, exec("def x = (char)3; int y = (int)5; return x >= y"));
+        assertEquals(true, exec("def x = (int)4; int y = (int)4; return x >= y"));
+        assertEquals(true, exec("def x = (long)5; int y = (int)3; return x >= y"));
+        assertEquals(true, exec("def x = (float)6; int y = (int)2; return x >= y"));
+        assertEquals(true, exec("def x = (double)7; int y = (int)1; return x >= y"));
+
+        assertEquals(false, exec("def x = (byte)1; double y = (double)7; return x >= y"));
+        assertEquals(false, exec("def x = (short)2; double y = (double)6; return x >= y"));
+        assertEquals(false, exec("def x = (char)3; double y = (double)5; return x >= y"));
+        assertEquals(true, exec("def x = (int)4; double y = (double)4; return x >= y"));
+        assertEquals(true, exec("def x = (long)5; double y = (double)3; return x >= y"));
+        assertEquals(true, exec("def x = (float)6; double y = (double)2; return x >= y"));
+        assertEquals(true, exec("def x = (double)7; double y = (double)1; return x >= y"));
+    }
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
deleted file mode 100644
index 9217b9e1210..00000000000
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOperationTests.java
+++ /dev/null
@@ -1,2352 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless;
-
-public class DefOperationTests extends ScriptTestCase {
-    public void testIllegalCast() {
-        Exception exception = expectScriptThrows(ClassCastException.class, () -> { 
-            exec("def x = 1.0; int y = x; return y;");
-        });
-        assertTrue(exception.getMessage().contains("cannot be cast"));
-
-        exception = expectScriptThrows(ClassCastException.class, () -> { 
-            exec("def x = (short)1; byte y = x; return y;");
-        });
-        assertTrue(exception.getMessage().contains("cannot be cast"));
-    }
-
-    public void testNot() {
-        assertEquals(~1, exec("def x = (byte)1; return ~x"));
-        assertEquals(~1, exec("def x = (short)1; return ~x"));
-        assertEquals(~1, exec("def x = (char)1; return ~x"));
-        assertEquals(~1, exec("def x = 1; return ~x"));
-        assertEquals(~1L, exec("def x = 1L; return ~x"));
-    }
-
-    public void testNeg() {
-        assertEquals(-1, exec("def x = (byte)1; return -x"));
-        assertEquals(-1, exec("def x = (short)1; return -x"));
-        assertEquals(-1, exec("def x = (char)1; return -x"));
-        assertEquals(-1, exec("def x = 1; return -x"));
-        assertEquals(-1L, exec("def x = 1L; return -x"));
-        assertEquals(-1.0F, exec("def x = 1F; return -x"));
-        assertEquals(-1.0, exec("def x = 1.0; return -x"));
-    }
-    
-    public void testPlus() {
-        assertEquals(-1, exec("def x = (byte)-1; return +x"));
-        assertEquals(-1, exec("def x = (short)-1; return +x"));
-        assertEquals(65535, exec("def x = (char)-1; return +x"));
-        assertEquals(-1, exec("def x = -1; return +x"));
-        assertEquals(-1L, exec("def x = -1L; return +x"));
-        assertEquals(-1.0F, exec("def x = -1F; return +x"));
-        assertEquals(-1.0D, exec("def x = -1.0; return +x"));
-    }
-
-    public void testMul() {
-        assertEquals(4, exec("def x = (byte)2; def y = (byte)2; return x * y"));
-        assertEquals(4, exec("def x = (short)2; def y = (byte)2; return x * y"));
-        assertEquals(4, exec("def x = (char)2; def y = (byte)2; return x * y"));
-        assertEquals(4, exec("def x = (int)2; def y = (byte)2; return x * y"));
-        assertEquals(4L, exec("def x = (long)2; def y = (byte)2; return x * y"));
-        assertEquals(4F, exec("def x = (float)2; def y = (byte)2; return x * y"));
-        assertEquals(4D, exec("def x = (double)2; def y = (byte)2; return x * y"));
-
-        assertEquals(4, exec("def x = (byte)2; def y = (short)2; return x * y"));
-        assertEquals(4, exec("def x = (short)2; def y = (short)2; return x * y"));
-        assertEquals(4, exec("def x = (char)2; def y = (short)2; return x * y"));
-        assertEquals(4, exec("def x = (int)2; def y = (short)2; return x * y"));
-        assertEquals(4L, exec("def x = (long)2; def y = (short)2; return x * y"));
-        assertEquals(4F, exec("def x = (float)2; def y = (short)2; return x * y"));
-        assertEquals(4D, exec("def x = (double)2; def y = (short)2; return x * y"));
-
-        assertEquals(4, exec("def x = (byte)2; def y = (char)2; return x * y"));
-        assertEquals(4, exec("def x = (short)2; def y = (char)2; return x * y"));
-        assertEquals(4, exec("def x = (char)2; def y = (char)2; return x * y"));
-        assertEquals(4, exec("def x = (int)2; def y = (char)2; return x * y"));
-        assertEquals(4L, exec("def x = (long)2; def y = (char)2; return x * y"));
-        assertEquals(4F, exec("def x = (float)2; def y = (char)2; return x * y"));
-        assertEquals(4D, exec("def x = (double)2; def y = (char)2; return x * y"));
-
-        assertEquals(4, exec("def x = (byte)2; def y = (int)2; return x * y"));
-        assertEquals(4, exec("def x = (short)2; def y = (int)2; return x * y"));
-        assertEquals(4, exec("def x = (char)2; def y = (int)2; return x * y"));
-        assertEquals(4, exec("def x = (int)2; def y = (int)2; return x * y"));
-        assertEquals(4L, exec("def x = (long)2; def y = (int)2; return x * y"));
-        assertEquals(4F, exec("def x = (float)2; def y = (int)2; return x * y"));
-        assertEquals(4D, exec("def x = (double)2; def y = (int)2; return x * y"));
-
-        assertEquals(4L, exec("def x = (byte)2; def y = (long)2; return x * y"));
-        assertEquals(4L, exec("def x = (short)2; def y = (long)2; return x * y"));
-        assertEquals(4L, exec("def x = (char)2; def y = (long)2; return x * y"));
-        assertEquals(4L, exec("def x = (int)2; def y = (long)2; return x * y"));
-        assertEquals(4L, exec("def x = (long)2; def y = (long)2; return x * y"));
-        assertEquals(4F, exec("def x = (float)2; def y = (long)2; return x * y"));
-        assertEquals(4D, exec("def x = (double)2; def y = (long)2; return x * y"));
-
-        assertEquals(4F, exec("def x = (byte)2; def y = (float)2; return x * y"));
-        assertEquals(4F, exec("def x = (short)2; def y = (float)2; return x * y"));
-        assertEquals(4F, exec("def x = (char)2; def y = (float)2; return x * y"));
-        assertEquals(4F, exec("def x = (int)2; def y = (float)2; return x * y"));
-        assertEquals(4F, exec("def x = (long)2; def y = (float)2; return x * y"));
-        assertEquals(4F, exec("def x = (float)2; def y = (float)2; return x * y"));
-        assertEquals(4D, exec("def x = (double)2; def y = (float)2; return x * y"));
-
-        assertEquals(4D, exec("def x = (byte)2; def y = (double)2; return x * y"));
-        assertEquals(4D, exec("def x = (short)2; def y = (double)2; return x * y"));
-        assertEquals(4D, exec("def x = (char)2; def y = (double)2; return x * y"));
-        assertEquals(4D, exec("def x = (int)2; def y = (double)2; return x * y"));
-        assertEquals(4D, exec("def x = (long)2; def y = (double)2; return x * y"));
-        assertEquals(4D, exec("def x = (float)2; def y = (double)2; return x * y"));
-        assertEquals(4D, exec("def x = (double)2; def y = (double)2; return x * y"));
-
-        assertEquals(4, exec("def x = (byte)2; def y = (byte)2; return x * y"));
-        assertEquals(4, exec("def x = (short)2; def y = (short)2; return x * y"));
-        assertEquals(4, exec("def x = (char)2; def y = (char)2; return x * y"));
-        assertEquals(4, exec("def x = (int)2; def y = (int)2; return x * y"));
-        assertEquals(4L, exec("def x = (long)2; def y = (long)2; return x * y"));
-        assertEquals(4F, exec("def x = (float)2; def y = (float)2; return x * y"));
-        assertEquals(4D, exec("def x = (double)2; def y = (double)2; return x * y"));
-    }
-    
-    public void testMulTypedLHS() {
-        assertEquals(4, exec("byte x = (byte)2; def y = (byte)2; return x * y"));
-        assertEquals(4, exec("short x = (short)2; def y = (byte)2; return x * y"));
-        assertEquals(4, exec("char x = (char)2; def y = (byte)2; return x * y"));
-        assertEquals(4, exec("int x = (int)2; def y = (byte)2; return x * y"));
-        assertEquals(4L, exec("long x = (long)2; def y = (byte)2; return x * y"));
-        assertEquals(4F, exec("float x = (float)2; def y = (byte)2; return x * y"));
-        assertEquals(4D, exec("double x = (double)2; def y = (byte)2; return x * y"));
-
-        assertEquals(4, exec("byte x = (byte)2; def y = (short)2; return x * y"));
-        assertEquals(4, exec("short x = (short)2; def y = (short)2; return x * y"));
-        assertEquals(4, exec("char x = (char)2; def y = (short)2; return x * y"));
-        assertEquals(4, exec("int x = (int)2; def y = (short)2; return x * y"));
-        assertEquals(4L, exec("long x = (long)2; def y = (short)2; return x * y"));
-        assertEquals(4F, exec("float x = (float)2; def y = (short)2; return x * y"));
-        assertEquals(4D, exec("double x = (double)2; def y = (short)2; return x * y"));
-
-        assertEquals(4, exec("byte x = (byte)2; def y = (char)2; return x * y"));
-        assertEquals(4, exec("short x = (short)2; def y = (char)2; return x * y"));
-        assertEquals(4, exec("char x = (char)2; def y = (char)2; return x * y"));
-        assertEquals(4, exec("int x = (int)2; def y = (char)2; return x * y"));
-        assertEquals(4L, exec("long x = (long)2; def y = (char)2; return x * y"));
-        assertEquals(4F, exec("float x = (float)2; def y = (char)2; return x * y"));
-        assertEquals(4D, exec("double x = (double)2; def y = (char)2; return x * y"));
-
-        assertEquals(4, exec("byte x = (byte)2; def y = (int)2; return x * y"));
-        assertEquals(4, exec("short x = (short)2; def y = (int)2; return x * y"));
-        assertEquals(4, exec("char x = (char)2; def y = (int)2; return x * y"));
-        assertEquals(4, exec("int x = (int)2; def y = (int)2; return x * y"));
-        assertEquals(4L, exec("long x = (long)2; def y = (int)2; return x * y"));
-        assertEquals(4F, exec("float x = (float)2; def y = (int)2; return x * y"));
-        assertEquals(4D, exec("double x = (double)2; def y = (int)2; return x * y"));
-
-        assertEquals(4L, exec("byte x = (byte)2; def y = (long)2; return x * y"));
-        assertEquals(4L, exec("short x = (short)2; def y = (long)2; return x * y"));
-        assertEquals(4L, exec("char x = (char)2; def y = (long)2; return x * y"));
-        assertEquals(4L, exec("int x = (int)2; def y = (long)2; return x * y"));
-        assertEquals(4L, exec("long x = (long)2; def y = (long)2; return x * y"));
-        assertEquals(4F, exec("float x = (float)2; def y = (long)2; return x * y"));
-        assertEquals(4D, exec("double x = (double)2; def y = (long)2; return x * y"));
-
-        assertEquals(4F, exec("byte x = (byte)2; def y = (float)2; return x * y"));
-        assertEquals(4F, exec("short x = (short)2; def y = (float)2; return x * y"));
-        assertEquals(4F, exec("char x = (char)2; def y = (float)2; return x * y"));
-        assertEquals(4F, exec("int x = (int)2; def y = (float)2; return x * y"));
-        assertEquals(4F, exec("long x = (long)2; def y = (float)2; return x * y"));
-        assertEquals(4F, exec("float x = (float)2; def y = (float)2; return x * y"));
-        assertEquals(4D, exec("double x = (double)2; def y = (float)2; return x * y"));
-
-        assertEquals(4D, exec("byte x = (byte)2; def y = (double)2; return x * y"));
-        assertEquals(4D, exec("short x = (short)2; def y = (double)2; return x * y"));
-        assertEquals(4D, exec("char x = (char)2; def y = (double)2; return x * y"));
-        assertEquals(4D, exec("int x = (int)2; def y = (double)2; return x * y"));
-        assertEquals(4D, exec("long x = (long)2; def y = (double)2; return x * y"));
-        assertEquals(4D, exec("float x = (float)2; def y = (double)2; return x * y"));
-        assertEquals(4D, exec("double x = (double)2; def y = (double)2; return x * y"));
-
-        assertEquals(4, exec("byte x = (byte)2; def y = (byte)2; return x * y"));
-        assertEquals(4, exec("short x = (short)2; def y = (short)2; return x * y"));
-        assertEquals(4, exec("char x = (char)2; def y = (char)2; return x * y"));
-        assertEquals(4, exec("int x = (int)2; def y = (int)2; return x * y"));
-        assertEquals(4L, exec("long x = (long)2; def y = (long)2; return x * y"));
-        assertEquals(4F, exec("float x = (float)2; def y = (float)2; return x * y"));
-        assertEquals(4D, exec("double x = (double)2; def y = (double)2; return x * y"));
-    }
-    
-    public void testMulTypedRHS() {
-        assertEquals(4, exec("def x = (byte)2; byte y = (byte)2; return x * y"));
-        assertEquals(4, exec("def x = (short)2; byte y = (byte)2; return x * y"));
-        assertEquals(4, exec("def x = (char)2; byte y = (byte)2; return x * y"));
-        assertEquals(4, exec("def x = (int)2; byte y = (byte)2; return x * y"));
-        assertEquals(4L, exec("def x = (long)2; byte y = (byte)2; return x * y"));
-        assertEquals(4F, exec("def x = (float)2; byte y = (byte)2; return x * y"));
-        assertEquals(4D, exec("def x = (double)2; byte y = (byte)2; return x * y"));
-
-        assertEquals(4, exec("def x = (byte)2; short y = (short)2; return x * y"));
-        assertEquals(4, exec("def x = (short)2; short y = (short)2; return x * y"));
-        assertEquals(4, exec("def x = (char)2; short y = (short)2; return x * y"));
-        assertEquals(4, exec("def x = (int)2; short y = (short)2; return x * y"));
-        assertEquals(4L, exec("def x = (long)2; short y = (short)2; return x * y"));
-        assertEquals(4F, exec("def x = (float)2; short y = (short)2; return x * y"));
-        assertEquals(4D, exec("def x = (double)2; short y = (short)2; return x * y"));
-
-        assertEquals(4, exec("def x = (byte)2; char y = (char)2; return x * y"));
-        assertEquals(4, exec("def x = (short)2; char y = (char)2; return x * y"));
-        assertEquals(4, exec("def x = (char)2; char y = (char)2; return x * y"));
-        assertEquals(4, exec("def x = (int)2; char y = (char)2; return x * y"));
-        assertEquals(4L, exec("def x = (long)2; char y = (char)2; return x * y"));
-        assertEquals(4F, exec("def x = (float)2; char y = (char)2; return x * y"));
-        assertEquals(4D, exec("def x = (double)2; char y = (char)2; return x * y"));
-
-        assertEquals(4, exec("def x = (byte)2; int y = (int)2; return x * y"));
-        assertEquals(4, exec("def x = (short)2; int y = (int)2; return x * y"));
-        assertEquals(4, exec("def x = (char)2; int y = (int)2; return x * y"));
-        assertEquals(4, exec("def x = (int)2; int y = (int)2; return x * y"));
-        assertEquals(4L, exec("def x = (long)2; int y = (int)2; return x * y"));
-        assertEquals(4F, exec("def x = (float)2; int y = (int)2; return x * y"));
-        assertEquals(4D, exec("def x = (double)2; int y = (int)2; return x * y"));
-
-        assertEquals(4L, exec("def x = (byte)2; long y = (long)2; return x * y"));
-        assertEquals(4L, exec("def x = (short)2; long y = (long)2; return x * y"));
-        assertEquals(4L, exec("def x = (char)2; long y = (long)2; return x * y"));
-        assertEquals(4L, exec("def x = (int)2; long y = (long)2; return x * y"));
-        assertEquals(4L, exec("def x = (long)2; long y = (long)2; return x * y"));
-        assertEquals(4F, exec("def x = (float)2; long y = (long)2; return x * y"));
-        assertEquals(4D, exec("def x = (double)2; long y = (long)2; return x * y"));
-
-        assertEquals(4F, exec("def x = (byte)2; float y = (float)2; return x * y"));
-        assertEquals(4F, exec("def x = (short)2; float y = (float)2; return x * y"));
-        assertEquals(4F, exec("def x = (char)2; float y = (float)2; return x * y"));
-        assertEquals(4F, exec("def x = (int)2; float y = (float)2; return x * y"));
-        assertEquals(4F, exec("def x = (long)2; float y = (float)2; return x * y"));
-        assertEquals(4F, exec("def x = (float)2; float y = (float)2; return x * y"));
-        assertEquals(4D, exec("def x = (double)2; float y = (float)2; return x * y"));
-
-        assertEquals(4D, exec("def x = (byte)2; double y = (double)2; return x * y"));
-        assertEquals(4D, exec("def x = (short)2; double y = (double)2; return x * y"));
-        assertEquals(4D, exec("def x = (char)2; double y = (double)2; return x * y"));
-        assertEquals(4D, exec("def x = (int)2; double y = (double)2; return x * y"));
-        assertEquals(4D, exec("def x = (long)2; double y = (double)2; return x * y"));
-        assertEquals(4D, exec("def x = (float)2; double y = (double)2; return x * y"));
-        assertEquals(4D, exec("def x = (double)2; double y = (double)2; return x * y"));
-
-        assertEquals(4, exec("def x = (byte)2; byte y = (byte)2; return x * y"));
-        assertEquals(4, exec("def x = (short)2; short y = (short)2; return x * y"));
-        assertEquals(4, exec("def x = (char)2; char y = (char)2; return x * y"));
-        assertEquals(4, exec("def x = (int)2; int y = (int)2; return x * y"));
-        assertEquals(4L, exec("def x = (long)2; long y = (long)2; return x * y"));
-        assertEquals(4F, exec("def x = (float)2; float y = (float)2; return x * y"));
-        assertEquals(4D, exec("def x = (double)2; double y = (double)2; return x * y"));
-    }
-
-    public void testDiv() {
-        assertEquals(1, exec("def x = (byte)2; def y = (byte)2; return x / y"));
-        assertEquals(1, exec("def x = (short)2; def y = (byte)2; return x / y"));
-        assertEquals(1, exec("def x = (char)2; def y = (byte)2; return x / y"));
-        assertEquals(1, exec("def x = (int)2; def y = (byte)2; return x / y"));
-        assertEquals(1L, exec("def x = (long)2; def y = (byte)2; return x / y"));
-        assertEquals(1F, exec("def x = (float)2; def y = (byte)2; return x / y"));
-        assertEquals(1D, exec("def x = (double)2; def y = (byte)2; return x / y"));
-
-        assertEquals(1, exec("def x = (byte)2; def y = (short)2; return x / y"));
-        assertEquals(1, exec("def x = (short)2; def y = (short)2; return x / y"));
-        assertEquals(1, exec("def x = (char)2; def y = (short)2; return x / y"));
-        assertEquals(1, exec("def x = (int)2; def y = (short)2; return x / y"));
-        assertEquals(1L, exec("def x = (long)2; def y = (short)2; return x / y"));
-        assertEquals(1F, exec("def x = (float)2; def y = (short)2; return x / y"));
-        assertEquals(1D, exec("def x = (double)2; def y = (short)2; return x / y"));
-
-        assertEquals(1, exec("def x = (byte)2; def y = (char)2; return x / y"));
-        assertEquals(1, exec("def x = (short)2; def y = (char)2; return x / y"));
-        assertEquals(1, exec("def x = (char)2; def y = (char)2; return x / y"));
-        assertEquals(1, exec("def x = (int)2; def y = (char)2; return x / y"));
-        assertEquals(1L, exec("def x = (long)2; def y = (char)2; return x / y"));
-        assertEquals(1F, exec("def x = (float)2; def y = (char)2; return x / y"));
-        assertEquals(1D, exec("def x = (double)2; def y = (char)2; return x / y"));
-
-        assertEquals(1, exec("def x = (byte)2; def y = (int)2; return x / y"));
-        assertEquals(1, exec("def x = (short)2; def y = (int)2; return x / y"));
-        assertEquals(1, exec("def x = (char)2; def y = (int)2; return x / y"));
-        assertEquals(1, exec("def x = (int)2; def y = (int)2; return x / y"));
-        assertEquals(1L, exec("def x = (long)2; def y = (int)2; return x / y"));
-        assertEquals(1F, exec("def x = (float)2; def y = (int)2; return x / y"));
-        assertEquals(1D, exec("def x = (double)2; def y = (int)2; return x / y"));
-
-        assertEquals(1L, exec("def x = (byte)2; def y = (long)2; return x / y"));
-        assertEquals(1L, exec("def x = (short)2; def y = (long)2; return x / y"));
-        assertEquals(1L, exec("def x = (char)2; def y = (long)2; return x / y"));
-        assertEquals(1L, exec("def x = (int)2; def y = (long)2; return x / y"));
-        assertEquals(1L, exec("def x = (long)2; def y = (long)2; return x / y"));
-        assertEquals(1F, exec("def x = (float)2; def y = (long)2; return x / y"));
-        assertEquals(1D, exec("def x = (double)2; def y = (long)2; return x / y"));
-
-        assertEquals(1F, exec("def x = (byte)2; def y = (float)2; return x / y"));
-        assertEquals(1F, exec("def x = (short)2; def y = (float)2; return x / y"));
-        assertEquals(1F, exec("def x = (char)2; def y = (float)2; return x / y"));
-        assertEquals(1F, exec("def x = (int)2; def y = (float)2; return x / y"));
-        assertEquals(1F, exec("def x = (long)2; def y = (float)2; return x / y"));
-        assertEquals(1F, exec("def x = (float)2; def y = (float)2; return x / y"));
-        assertEquals(1D, exec("def x = (double)2; def y = (float)2; return x / y"));
-
-        assertEquals(1D, exec("def x = (byte)2; def y = (double)2; return x / y"));
-        assertEquals(1D, exec("def x = (short)2; def y = (double)2; return x / y"));
-        assertEquals(1D, exec("def x = (char)2; def y = (double)2; return x / y"));
-        assertEquals(1D, exec("def x = (int)2; def y = (double)2; return x / y"));
-        assertEquals(1D, exec("def x = (long)2; def y = (double)2; return x / y"));
-        assertEquals(1D, exec("def x = (float)2; def y = (double)2; return x / y"));
-        assertEquals(1D, exec("def x = (double)2; def y = (double)2; return x / y"));
-
-        assertEquals(1, exec("def x = (byte)2; def y = (byte)2; return x / y"));
-        assertEquals(1, exec("def x = (short)2; def y = (short)2; return x / y"));
-        assertEquals(1, exec("def x = (char)2; def y = (char)2; return x / y"));
-        assertEquals(1, exec("def x = (int)2; def y = (int)2; return x / y"));
-        assertEquals(1L, exec("def x = (long)2; def y = (long)2; return x / y"));
-        assertEquals(1F, exec("def x = (float)2; def y = (float)2; return x / y"));
-        assertEquals(1D, exec("def x = (double)2; def y = (double)2; return x / y"));
-    }
-    
-    public void testDivTypedLHS() {
-        assertEquals(1, exec("byte x = (byte)2; def y = (byte)2; return x / y"));
-        assertEquals(1, exec("short x = (short)2; def y = (byte)2; return x / y"));
-        assertEquals(1, exec("char x = (char)2; def y = (byte)2; return x / y"));
-        assertEquals(1, exec("int x = (int)2; def y = (byte)2; return x / y"));
-        assertEquals(1L, exec("long x = (long)2; def y = (byte)2; return x / y"));
-        assertEquals(1F, exec("float x = (float)2; def y = (byte)2; return x / y"));
-        assertEquals(1D, exec("double x = (double)2; def y = (byte)2; return x / y"));
-
-        assertEquals(1, exec("byte x = (byte)2; def y = (short)2; return x / y"));
-        assertEquals(1, exec("short x = (short)2; def y = (short)2; return x / y"));
-        assertEquals(1, exec("char x = (char)2; def y = (short)2; return x / y"));
-        assertEquals(1, exec("int x = (int)2; def y = (short)2; return x / y"));
-        assertEquals(1L, exec("long x = (long)2; def y = (short)2; return x / y"));
-        assertEquals(1F, exec("float x = (float)2; def y = (short)2; return x / y"));
-        assertEquals(1D, exec("double x = (double)2; def y = (short)2; return x / y"));
-
-        assertEquals(1, exec("byte x = (byte)2; def y = (char)2; return x / y"));
-        assertEquals(1, exec("short x = (short)2; def y = (char)2; return x / y"));
-        assertEquals(1, exec("char x = (char)2; def y = (char)2; return x / y"));
-        assertEquals(1, exec("int x = (int)2; def y = (char)2; return x / y"));
-        assertEquals(1L, exec("long x = (long)2; def y = (char)2; return x / y"));
-        assertEquals(1F, exec("float x = (float)2; def y = (char)2; return x / y"));
-        assertEquals(1D, exec("double x = (double)2; def y = (char)2; return x / y"));
-
-        assertEquals(1, exec("byte x = (byte)2; def y = (int)2; return x / y"));
-        assertEquals(1, exec("short x = (short)2; def y = (int)2; return x / y"));
-        assertEquals(1, exec("char x = (char)2; def y = (int)2; return x / y"));
-        assertEquals(1, exec("int x = (int)2; def y = (int)2; return x / y"));
-        assertEquals(1L, exec("long x = (long)2; def y = (int)2; return x / y"));
-        assertEquals(1F, exec("float x = (float)2; def y = (int)2; return x / y"));
-        assertEquals(1D, exec("double x = (double)2; def y = (int)2; return x / y"));
-
-        assertEquals(1L, exec("byte x = (byte)2; def y = (long)2; return x / y"));
-        assertEquals(1L, exec("short x = (short)2; def y = (long)2; return x / y"));
-        assertEquals(1L, exec("char x = (char)2; def y = (long)2; return x / y"));
-        assertEquals(1L, exec("int x = (int)2; def y = (long)2; return x / y"));
-        assertEquals(1L, exec("long x = (long)2; def y = (long)2; return x / y"));
-        assertEquals(1F, exec("float x = (float)2; def y = (long)2; return x / y"));
-        assertEquals(1D, exec("double x = (double)2; def y = (long)2; return x / y"));
-
-        assertEquals(1F, exec("byte x = (byte)2; def y = (float)2; return x / y"));
-        assertEquals(1F, exec("short x = (short)2; def y = (float)2; return x / y"));
-        assertEquals(1F, exec("char x = (char)2; def y = (float)2; return x / y"));
-        assertEquals(1F, exec("int x = (int)2; def y = (float)2; return x / y"));
-        assertEquals(1F, exec("long x = (long)2; def y = (float)2; return x / y"));
-        assertEquals(1F, exec("float x = (float)2; def y = (float)2; return x / y"));
-        assertEquals(1D, exec("double x = (double)2; def y = (float)2; return x / y"));
-
-        assertEquals(1D, exec("byte x = (byte)2; def y = (double)2; return x / y"));
-        assertEquals(1D, exec("short x = (short)2; def y = (double)2; return x / y"));
-        assertEquals(1D, exec("char x = (char)2; def y = (double)2; return x / y"));
-        assertEquals(1D, exec("int x = (int)2; def y = (double)2; return x / y"));
-        assertEquals(1D, exec("long x = (long)2; def y = (double)2; return x / y"));
-        assertEquals(1D, exec("float x = (float)2; def y = (double)2; return x / y"));
-        assertEquals(1D, exec("double x = (double)2; def y = (double)2; return x / y"));
-
-        assertEquals(1, exec("byte x = (byte)2; def y = (byte)2; return x / y"));
-        assertEquals(1, exec("short x = (short)2; def y = (short)2; return x / y"));
-        assertEquals(1, exec("char x = (char)2; def y = (char)2; return x / y"));
-        assertEquals(1, exec("int x = (int)2; def y = (int)2; return x / y"));
-        assertEquals(1L, exec("long x = (long)2; def y = (long)2; return x / y"));
-        assertEquals(1F, exec("float x = (float)2; def y = (float)2; return x / y"));
-        assertEquals(1D, exec("double x = (double)2; def y = (double)2; return x / y"));
-    }
-    
-    public void testDivTypedRHS() {
-        assertEquals(1, exec("def x = (byte)2; byte y = (byte)2; return x / y"));
-        assertEquals(1, exec("def x = (short)2; byte y = (byte)2; return x / y"));
-        assertEquals(1, exec("def x = (char)2; byte y = (byte)2; return x / y"));
-        assertEquals(1, exec("def x = (int)2; byte y = (byte)2; return x / y"));
-        assertEquals(1L, exec("def x = (long)2; byte y = (byte)2; return x / y"));
-        assertEquals(1F, exec("def x = (float)2; byte y = (byte)2; return x / y"));
-        assertEquals(1D, exec("def x = (double)2; byte y = (byte)2; return x / y"));
-
-        assertEquals(1, exec("def x = (byte)2; short y = (short)2; return x / y"));
-        assertEquals(1, exec("def x = (short)2; short y = (short)2; return x / y"));
-        assertEquals(1, exec("def x = (char)2; short y = (short)2; return x / y"));
-        assertEquals(1, exec("def x = (int)2; short y = (short)2; return x / y"));
-        assertEquals(1L, exec("def x = (long)2; short y = (short)2; return x / y"));
-        assertEquals(1F, exec("def x = (float)2; short y = (short)2; return x / y"));
-        assertEquals(1D, exec("def x = (double)2; short y = (short)2; return x / y"));
-
-        assertEquals(1, exec("def x = (byte)2; char y = (char)2; return x / y"));
-        assertEquals(1, exec("def x = (short)2; char y = (char)2; return x / y"));
-        assertEquals(1, exec("def x = (char)2; char y = (char)2; return x / y"));
-        assertEquals(1, exec("def x = (int)2; char y = (char)2; return x / y"));
-        assertEquals(1L, exec("def x = (long)2; char y = (char)2; return x / y"));
-        assertEquals(1F, exec("def x = (float)2; char y = (char)2; return x / y"));
-        assertEquals(1D, exec("def x = (double)2; char y = (char)2; return x / y"));
-
-        assertEquals(1, exec("def x = (byte)2; int y = (int)2; return x / y"));
-        assertEquals(1, exec("def x = (short)2; int y = (int)2; return x / y"));
-        assertEquals(1, exec("def x = (char)2; int y = (int)2; return x / y"));
-        assertEquals(1, exec("def x = (int)2; int y = (int)2; return x / y"));
-        assertEquals(1L, exec("def x = (long)2; int y = (int)2; return x / y"));
-        assertEquals(1F, exec("def x = (float)2; int y = (int)2; return x / y"));
-        assertEquals(1D, exec("def x = (double)2; int y = (int)2; return x / y"));
-
-        assertEquals(1L, exec("def x = (byte)2; long y = (long)2; return x / y"));
-        assertEquals(1L, exec("def x = (short)2; long y = (long)2; return x / y"));
-        assertEquals(1L, exec("def x = (char)2; long y = (long)2; return x / y"));
-        assertEquals(1L, exec("def x = (int)2; long y = (long)2; return x / y"));
-        assertEquals(1L, exec("def x = (long)2; long y = (long)2; return x / y"));
-        assertEquals(1F, exec("def x = (float)2; long y = (long)2; return x / y"));
-        assertEquals(1D, exec("def x = (double)2; long y = (long)2; return x / y"));
-
-        assertEquals(1F, exec("def x = (byte)2; float y = (float)2; return x / y"));
-        assertEquals(1F, exec("def x = (short)2; float y = (float)2; return x / y"));
-        assertEquals(1F, exec("def x = (char)2; float y = (float)2; return x / y"));
-        assertEquals(1F, exec("def x = (int)2; float y = (float)2; return x / y"));
-        assertEquals(1F, exec("def x = (long)2; float y = (float)2; return x / y"));
-        assertEquals(1F, exec("def x = (float)2; float y = (float)2; return x / y"));
-        assertEquals(1D, exec("def x = (double)2; float y = (float)2; return x / y"));
-
-        assertEquals(1D, exec("def x = (byte)2; double y = (double)2; return x / y"));
-        assertEquals(1D, exec("def x = (short)2; double y = (double)2; return x / y"));
-        assertEquals(1D, exec("def x = (char)2; double y = (double)2; return x / y"));
-        assertEquals(1D, exec("def x = (int)2; double y = (double)2; return x / y"));
-        assertEquals(1D, exec("def x = (long)2; double y = (double)2; return x / y"));
-        assertEquals(1D, exec("def x = (float)2; double y = (double)2; return x / y"));
-        assertEquals(1D, exec("def x = (double)2; double y = (double)2; return x / y"));
-
-        assertEquals(1, exec("def x = (byte)2; byte y = (byte)2; return x / y"));
-        assertEquals(1, exec("def x = (short)2; short y = (short)2; return x / y"));
-        assertEquals(1, exec("def x = (char)2; char y = (char)2; return x / y"));
-        assertEquals(1, exec("def x = (int)2; int y = (int)2; return x / y"));
-        assertEquals(1L, exec("def x = (long)2; long y = (long)2; return x / y"));
-        assertEquals(1F, exec("def x = (float)2; float y = (float)2; return x / y"));
-        assertEquals(1D, exec("def x = (double)2; double y = (double)2; return x / y"));
-    }
-
-    public void testRem() {
-        assertEquals(0, exec("def x = (byte)2; def y = (byte)2; return x % y"));
-        assertEquals(0, exec("def x = (short)2; def y = (byte)2; return x % y"));
-        assertEquals(0, exec("def x = (char)2; def y = (byte)2; return x % y"));
-        assertEquals(0, exec("def x = (int)2; def y = (byte)2; return x % y"));
-        assertEquals(0L, exec("def x = (long)2; def y = (byte)2; return x % y"));
-        assertEquals(0F, exec("def x = (float)2; def y = (byte)2; return x % y"));
-        assertEquals(0D, exec("def x = (double)2; def y = (byte)2; return x % y"));
-
-        assertEquals(0, exec("def x = (byte)2; def y = (short)2; return x % y"));
-        assertEquals(0, exec("def x = (short)2; def y = (short)2; return x % y"));
-        assertEquals(0, exec("def x = (char)2; def y = (short)2; return x % y"));
-        assertEquals(0, exec("def x = (int)2; def y = (short)2; return x % y"));
-        assertEquals(0L, exec("def x = (long)2; def y = (short)2; return x % y"));
-        assertEquals(0F, exec("def x = (float)2; def y = (short)2; return x % y"));
-        assertEquals(0D, exec("def x = (double)2; def y = (short)2; return x % y"));
-
-        assertEquals(0, exec("def x = (byte)2; def y = (char)2; return x % y"));
-        assertEquals(0, exec("def x = (short)2; def y = (char)2; return x % y"));
-        assertEquals(0, exec("def x = (char)2; def y = (char)2; return x % y"));
-        assertEquals(0, exec("def x = (int)2; def y = (char)2; return x % y"));
-        assertEquals(0L, exec("def x = (long)2; def y = (char)2; return x % y"));
-        assertEquals(0F, exec("def x = (float)2; def y = (char)2; return x % y"));
-        assertEquals(0D, exec("def x = (double)2; def y = (char)2; return x % y"));
-
-        assertEquals(0, exec("def x = (byte)2; def y = (int)2; return x % y"));
-        assertEquals(0, exec("def x = (short)2; def y = (int)2; return x % y"));
-        assertEquals(0, exec("def x = (char)2; def y = (int)2; return x % y"));
-        assertEquals(0, exec("def x = (int)2; def y = (int)2; return x % y"));
-        assertEquals(0L, exec("def x = (long)2; def y = (int)2; return x % y"));
-        assertEquals(0F, exec("def x = (float)2; def y = (int)2; return x % y"));
-        assertEquals(0D, exec("def x = (double)2; def y = (int)2; return x % y"));
-
-        assertEquals(0L, exec("def x = (byte)2; def y = (long)2; return x % y"));
-        assertEquals(0L, exec("def x = (short)2; def y = (long)2; return x % y"));
-        assertEquals(0L, exec("def x = (char)2; def y = (long)2; return x % y"));
-        assertEquals(0L, exec("def x = (int)2; def y = (long)2; return x % y"));
-        assertEquals(0L, exec("def x = (long)2; def y = (long)2; return x % y"));
-        assertEquals(0F, exec("def x = (float)2; def y = (long)2; return x % y"));
-        assertEquals(0D, exec("def x = (double)2; def y = (long)2; return x % y"));
-
-        assertEquals(0F, exec("def x = (byte)2; def y = (float)2; return x % y"));
-        assertEquals(0F, exec("def x = (short)2; def y = (float)2; return x % y"));
-        assertEquals(0F, exec("def x = (char)2; def y = (float)2; return x % y"));
-        assertEquals(0F, exec("def x = (int)2; def y = (float)2; return x % y"));
-        assertEquals(0F, exec("def x = (long)2; def y = (float)2; return x % y"));
-        assertEquals(0F, exec("def x = (float)2; def y = (float)2; return x % y"));
-        assertEquals(0D, exec("def x = (double)2; def y = (float)2; return x % y"));
-
-        assertEquals(0D, exec("def x = (byte)2; def y = (double)2; return x % y"));
-        assertEquals(0D, exec("def x = (short)2; def y = (double)2; return x % y"));
-        assertEquals(0D, exec("def x = (char)2; def y = (double)2; return x % y"));
-        assertEquals(0D, exec("def x = (int)2; def y = (double)2; return x % y"));
-        assertEquals(0D, exec("def x = (long)2; def y = (double)2; return x % y"));
-        assertEquals(0D, exec("def x = (float)2; def y = (double)2; return x % y"));
-        assertEquals(0D, exec("def x = (double)2; def y = (double)2; return x % y"));
-
-        assertEquals(0, exec("def x = (byte)2; def y = (byte)2; return x % y"));
-        assertEquals(0, exec("def x = (short)2; def y = (short)2; return x % y"));
-        assertEquals(0, exec("def x = (char)2; def y = (char)2; return x % y"));
-        assertEquals(0, exec("def x = (int)2; def y = (int)2; return x % y"));
-        assertEquals(0L, exec("def x = (long)2; def y = (long)2; return x % y"));
-        assertEquals(0F, exec("def x = (float)2; def y = (float)2; return x % y"));
-        assertEquals(0D, exec("def x = (double)2; def y = (double)2; return x % y"));
-    }
-    
-    public void testRemTypedLHS() {
-        assertEquals(0, exec("byte x = (byte)2; def y = (byte)2; return x % y"));
-        assertEquals(0, exec("short x = (short)2; def y = (byte)2; return x % y"));
-        assertEquals(0, exec("char x = (char)2; def y = (byte)2; return x % y"));
-        assertEquals(0, exec("int x = (int)2; def y = (byte)2; return x % y"));
-        assertEquals(0L, exec("long x = (long)2; def y = (byte)2; return x % y"));
-        assertEquals(0F, exec("float x = (float)2; def y = (byte)2; return x % y"));
-        assertEquals(0D, exec("double x = (double)2; def y = (byte)2; return x % y"));
-
-        assertEquals(0, exec("byte x = (byte)2; def y = (short)2; return x % y"));
-        assertEquals(0, exec("short x = (short)2; def y = (short)2; return x % y"));
-        assertEquals(0, exec("char x = (char)2; def y = (short)2; return x % y"));
-        assertEquals(0, exec("int x = (int)2; def y = (short)2; return x % y"));
-        assertEquals(0L, exec("long x = (long)2; def y = (short)2; return x % y"));
-        assertEquals(0F, exec("float x = (float)2; def y = (short)2; return x % y"));
-        assertEquals(0D, exec("double x = (double)2; def y = (short)2; return x % y"));
-
-        assertEquals(0, exec("byte x = (byte)2; def y = (char)2; return x % y"));
-        assertEquals(0, exec("short x = (short)2; def y = (char)2; return x % y"));
-        assertEquals(0, exec("char x = (char)2; def y = (char)2; return x % y"));
-        assertEquals(0, exec("int x = (int)2; def y = (char)2; return x % y"));
-        assertEquals(0L, exec("long x = (long)2; def y = (char)2; return x % y"));
-        assertEquals(0F, exec("float x = (float)2; def y = (char)2; return x % y"));
-        assertEquals(0D, exec("double x = (double)2; def y = (char)2; return x % y"));
-
-        assertEquals(0, exec("byte x = (byte)2; def y = (int)2; return x % y"));
-        assertEquals(0, exec("short x = (short)2; def y = (int)2; return x % y"));
-        assertEquals(0, exec("char x = (char)2; def y = (int)2; return x % y"));
-        assertEquals(0, exec("int x = (int)2; def y = (int)2; return x % y"));
-        assertEquals(0L, exec("long x = (long)2; def y = (int)2; return x % y"));
-        assertEquals(0F, exec("float x = (float)2; def y = (int)2; return x % y"));
-        assertEquals(0D, exec("double x = (double)2; def y = (int)2; return x % y"));
-
-        assertEquals(0L, exec("byte x = (byte)2; def y = (long)2; return x % y"));
-        assertEquals(0L, exec("short x = (short)2; def y = (long)2; return x % y"));
-        assertEquals(0L, exec("char x = (char)2; def y = (long)2; return x % y"));
-        assertEquals(0L, exec("int x = (int)2; def y = (long)2; return x % y"));
-        assertEquals(0L, exec("long x = (long)2; def y = (long)2; return x % y"));
-        assertEquals(0F, exec("float x = (float)2; def y = (long)2; return x % y"));
-        assertEquals(0D, exec("double x = (double)2; def y = (long)2; return x % y"));
-
-        assertEquals(0F, exec("byte x = (byte)2; def y = (float)2; return x % y"));
-        assertEquals(0F, exec("short x = (short)2; def y = (float)2; return x % y"));
-        assertEquals(0F, exec("char x = (char)2; def y = (float)2; return x % y"));
-        assertEquals(0F, exec("int x = (int)2; def y = (float)2; return x % y"));
-        assertEquals(0F, exec("long x = (long)2; def y = (float)2; return x % y"));
-        assertEquals(0F, exec("float x = (float)2; def y = (float)2; return x % y"));
-        assertEquals(0D, exec("double x = (double)2; def y = (float)2; return x % y"));
-
-        assertEquals(0D, exec("byte x = (byte)2; def y = (double)2; return x % y"));
-        assertEquals(0D, exec("short x = (short)2; def y = (double)2; return x % y"));
-        assertEquals(0D, exec("char x = (char)2; def y = (double)2; return x % y"));
-        assertEquals(0D, exec("int x = (int)2; def y = (double)2; return x % y"));
-        assertEquals(0D, exec("long x = (long)2; def y = (double)2; return x % y"));
-        assertEquals(0D, exec("float x = (float)2; def y = (double)2; return x % y"));
-        assertEquals(0D, exec("double x = (double)2; def y = (double)2; return x % y"));
-
-        assertEquals(0, exec("byte x = (byte)2; def y = (byte)2; return x % y"));
-        assertEquals(0, exec("short x = (short)2; def y = (short)2; return x % y"));
-        assertEquals(0, exec("char x = (char)2; def y = (char)2; return x % y"));
-        assertEquals(0, exec("int x = (int)2; def y = (int)2; return x % y"));
-        assertEquals(0L, exec("long x = (long)2; def y = (long)2; return x % y"));
-        assertEquals(0F, exec("float x = (float)2; def y = (float)2; return x % y"));
-        assertEquals(0D, exec("double x = (double)2; def y = (double)2; return x % y"));
-    }
-
-    public void testRemTypedRHS() {
-        assertEquals(0, exec("def x = (byte)2; byte y = (byte)2; return x % y"));
-        assertEquals(0, exec("def x = (short)2; byte y = (byte)2; return x % y"));
-        assertEquals(0, exec("def x = (char)2; byte y = (byte)2; return x % y"));
-        assertEquals(0, exec("def x = (int)2; byte y = (byte)2; return x % y"));
-        assertEquals(0L, exec("def x = (long)2; byte y = (byte)2; return x % y"));
-        assertEquals(0F, exec("def x = (float)2; byte y = (byte)2; return x % y"));
-        assertEquals(0D, exec("def x = (double)2; byte y = (byte)2; return x % y"));
-
-        assertEquals(0, exec("def x = (byte)2; short y = (short)2; return x % y"));
-        assertEquals(0, exec("def x = (short)2; short y = (short)2; return x % y"));
-        assertEquals(0, exec("def x = (char)2; short y = (short)2; return x % y"));
-        assertEquals(0, exec("def x = (int)2; short y = (short)2; return x % y"));
-        assertEquals(0L, exec("def x = (long)2; short y = (short)2; return x % y"));
-        assertEquals(0F, exec("def x = (float)2; short y = (short)2; return x % y"));
-        assertEquals(0D, exec("def x = (double)2; short y = (short)2; return x % y"));
-
-        assertEquals(0, exec("def x = (byte)2; char y = (char)2; return x % y"));
-        assertEquals(0, exec("def x = (short)2; char y = (char)2; return x % y"));
-        assertEquals(0, exec("def x = (char)2; char y = (char)2; return x % y"));
-        assertEquals(0, exec("def x = (int)2; char y = (char)2; return x % y"));
-        assertEquals(0L, exec("def x = (long)2; char y = (char)2; return x % y"));
-        assertEquals(0F, exec("def x = (float)2; char y = (char)2; return x % y"));
-        assertEquals(0D, exec("def x = (double)2; char y = (char)2; return x % y"));
-
-        assertEquals(0, exec("def x = (byte)2; int y = (int)2; return x % y"));
-        assertEquals(0, exec("def x = (short)2; int y = (int)2; return x % y"));
-        assertEquals(0, exec("def x = (char)2; int y = (int)2; return x % y"));
-        assertEquals(0, exec("def x = (int)2; int y = (int)2; return x % y"));
-        assertEquals(0L, exec("def x = (long)2; int y = (int)2; return x % y"));
-        assertEquals(0F, exec("def x = (float)2; int y = (int)2; return x % y"));
-        assertEquals(0D, exec("def x = (double)2; int y = (int)2; return x % y"));
-
-        assertEquals(0L, exec("def x = (byte)2; long y = (long)2; return x % y"));
-        assertEquals(0L, exec("def x = (short)2; long y = (long)2; return x % y"));
-        assertEquals(0L, exec("def x = (char)2; long y = (long)2; return x % y"));
-        assertEquals(0L, exec("def x = (int)2; long y = (long)2; return x % y"));
-        assertEquals(0L, exec("def x = (long)2; long y = (long)2; return x % y"));
-        assertEquals(0F, exec("def x = (float)2; long y = (long)2; return x % y"));
-        assertEquals(0D, exec("def x = (double)2; long y = (long)2; return x % y"));
-
-        assertEquals(0F, exec("def x = (byte)2; float y = (float)2; return x % y"));
-        assertEquals(0F, exec("def x = (short)2; float y = (float)2; return x % y"));
-        assertEquals(0F, exec("def x = (char)2; float y = (float)2; return x % y"));
-        assertEquals(0F, exec("def x = (int)2; float y = (float)2; return x % y"));
-        assertEquals(0F, exec("def x = (long)2; float y = (float)2; return x % y"));
-        assertEquals(0F, exec("def x = (float)2; float y = (float)2; return x % y"));
-        assertEquals(0D, exec("def x = (double)2; float y = (float)2; return x % y"));
-
-        assertEquals(0D, exec("def x = (byte)2; double y = (double)2; return x % y"));
-        assertEquals(0D, exec("def x = (short)2; double y = (double)2; return x % y"));
-        assertEquals(0D, exec("def x = (char)2; double y = (double)2; return x % y"));
-        assertEquals(0D, exec("def x = (int)2; double y = (double)2; return x % y"));
-        assertEquals(0D, exec("def x = (long)2; double y = (double)2; return x % y"));
-        assertEquals(0D, exec("def x = (float)2; double y = (double)2; return x % y"));
-        assertEquals(0D, exec("def x = (double)2; double y = (double)2; return x % y"));
-
-        assertEquals(0, exec("def x = (byte)2; byte y = (byte)2; return x % y"));
-        assertEquals(0, exec("def x = (short)2; short y = (short)2; return x % y"));
-        assertEquals(0, exec("def x = (char)2; char y = (char)2; return x % y"));
-        assertEquals(0, exec("def x = (int)2; int y = (int)2; return x % y"));
-        assertEquals(0L, exec("def x = (long)2; long y = (long)2; return x % y"));
-        assertEquals(0F, exec("def x = (float)2; float y = (float)2; return x % y"));
-        assertEquals(0D, exec("def x = (double)2; double y = (double)2; return x % y"));
-    }
-
-    public void testAdd() {
-        assertEquals(2, exec("def x = (byte)1; def y = (byte)1; return x + y"));
-        assertEquals(2, exec("def x = (short)1; def y = (byte)1; return x + y"));
-        assertEquals(2, exec("def x = (char)1; def y = (byte)1; return x + y"));
-        assertEquals(2, exec("def x = (int)1; def y = (byte)1; return x + y"));
-        assertEquals(2L, exec("def x = (long)1; def y = (byte)1; return x + y"));
-        assertEquals(2F, exec("def x = (float)1; def y = (byte)1; return x + y"));
-        assertEquals(2D, exec("def x = (double)1; def y = (byte)1; return x + y"));
-
-        assertEquals(2, exec("def x = (byte)1; def y = (short)1; return x + y"));
-        assertEquals(2, exec("def x = (short)1; def y = (short)1; return x + y"));
-        assertEquals(2, exec("def x = (char)1; def y = (short)1; return x + y"));
-        assertEquals(2, exec("def x = (int)1; def y = (short)1; return x + y"));
-        assertEquals(2L, exec("def x = (long)1; def y = (short)1; return x + y"));
-        assertEquals(2F, exec("def x = (float)1; def y = (short)1; return x + y"));
-        assertEquals(2D, exec("def x = (double)1; def y = (short)1; return x + y"));
-
-        assertEquals(2, exec("def x = (byte)1; def y = (char)1; return x + y"));
-        assertEquals(2, exec("def x = (short)1; def y = (char)1; return x + y"));
-        assertEquals(2, exec("def x = (char)1; def y = (char)1; return x + y"));
-        assertEquals(2, exec("def x = (int)1; def y = (char)1; return x + y"));
-        assertEquals(2L, exec("def x = (long)1; def y = (char)1; return x + y"));
-        assertEquals(2F, exec("def x = (float)1; def y = (char)1; return x + y"));
-        assertEquals(2D, exec("def x = (double)1; def y = (char)1; return x + y"));
-
-        assertEquals(2, exec("def x = (byte)1; def y = (int)1; return x + y"));
-        assertEquals(2, exec("def x = (short)1; def y = (int)1; return x + y"));
-        assertEquals(2, exec("def x = (char)1; def y = (int)1; return x + y"));
-        assertEquals(2, exec("def x = (int)1; def y = (int)1; return x + y"));
-        assertEquals(2L, exec("def x = (long)1; def y = (int)1; return x + y"));
-        assertEquals(2F, exec("def x = (float)1; def y = (int)1; return x + y"));
-        assertEquals(2D, exec("def x = (double)1; def y = (int)1; return x + y"));
-
-        assertEquals(2L, exec("def x = (byte)1; def y = (long)1; return x + y"));
-        assertEquals(2L, exec("def x = (short)1; def y = (long)1; return x + y"));
-        assertEquals(2L, exec("def x = (char)1; def y = (long)1; return x + y"));
-        assertEquals(2L, exec("def x = (int)1; def y = (long)1; return x + y"));
-        assertEquals(2L, exec("def x = (long)1; def y = (long)1; return x + y"));
-        assertEquals(2F, exec("def x = (float)1; def y = (long)1; return x + y"));
-        assertEquals(2D, exec("def x = (double)1; def y = (long)1; return x + y"));
-
-        assertEquals(2F, exec("def x = (byte)1; def y = (float)1; return x + y"));
-        assertEquals(2F, exec("def x = (short)1; def y = (float)1; return x + y"));
-        assertEquals(2F, exec("def x = (char)1; def y = (float)1; return x + y"));
-        assertEquals(2F, exec("def x = (int)1; def y = (float)1; return x + y"));
-        assertEquals(2F, exec("def x = (long)1; def y = (float)1; return x + y"));
-        assertEquals(2F, exec("def x = (float)1; def y = (float)1; return x + y"));
-        assertEquals(2D, exec("def x = (double)1; def y = (float)1; return x + y"));
-
-        assertEquals(2D, exec("def x = (byte)1; def y = (double)1; return x + y"));
-        assertEquals(2D, exec("def x = (short)1; def y = (double)1; return x + y"));
-        assertEquals(2D, exec("def x = (char)1; def y = (double)1; return x + y"));
-        assertEquals(2D, exec("def x = (int)1; def y = (double)1; return x + y"));
-        assertEquals(2D, exec("def x = (long)1; def y = (double)1; return x + y"));
-        assertEquals(2D, exec("def x = (float)1; def y = (double)1; return x + y"));
-        assertEquals(2D, exec("def x = (double)1; def y = (double)1; return x + y"));
-    }
-    
-    public void testAddTypedLHS() {
-        assertEquals(2, exec("byte x = (byte)1; def y = (byte)1; return x + y"));
-        assertEquals(2, exec("short x = (short)1; def y = (byte)1; return x + y"));
-        assertEquals(2, exec("char x = (char)1; def y = (byte)1; return x + y"));
-        assertEquals(2, exec("int x = (int)1; def y = (byte)1; return x + y"));
-        assertEquals(2L, exec("long x = (long)1; def y = (byte)1; return x + y"));
-        assertEquals(2F, exec("float x = (float)1; def y = (byte)1; return x + y"));
-        assertEquals(2D, exec("double x = (double)1; def y = (byte)1; return x + y"));
-
-        assertEquals(2, exec("byte x = (byte)1; def y = (short)1; return x + y"));
-        assertEquals(2, exec("short x = (short)1; def y = (short)1; return x + y"));
-        assertEquals(2, exec("char x = (char)1; def y = (short)1; return x + y"));
-        assertEquals(2, exec("int x = (int)1; def y = (short)1; return x + y"));
-        assertEquals(2L, exec("long x = (long)1; def y = (short)1; return x + y"));
-        assertEquals(2F, exec("float x = (float)1; def y = (short)1; return x + y"));
-        assertEquals(2D, exec("double x = (double)1; def y = (short)1; return x + y"));
-
-        assertEquals(2, exec("byte x = (byte)1; def y = (char)1; return x + y"));
-        assertEquals(2, exec("short x = (short)1; def y = (char)1; return x + y"));
-        assertEquals(2, exec("char x = (char)1; def y = (char)1; return x + y"));
-        assertEquals(2, exec("int x = (int)1; def y = (char)1; return x + y"));
-        assertEquals(2L, exec("long x = (long)1; def y = (char)1; return x + y"));
-        assertEquals(2F, exec("float x = (float)1; def y = (char)1; return x + y"));
-        assertEquals(2D, exec("double x = (double)1; def y = (char)1; return x + y"));
-
-        assertEquals(2, exec("byte x = (byte)1; def y = (int)1; return x + y"));
-        assertEquals(2, exec("short x = (short)1; def y = (int)1; return x + y"));
-        assertEquals(2, exec("char x = (char)1; def y = (int)1; return x + y"));
-        assertEquals(2, exec("int x = (int)1; def y = (int)1; return x + y"));
-        assertEquals(2L, exec("long x = (long)1; def y = (int)1; return x + y"));
-        assertEquals(2F, exec("float x = (float)1; def y = (int)1; return x + y"));
-        assertEquals(2D, exec("double x = (double)1; def y = (int)1; return x + y"));
-
-        assertEquals(2L, exec("byte x = (byte)1; def y = (long)1; return x + y"));
-        assertEquals(2L, exec("short x = (short)1; def y = (long)1; return x + y"));
-        assertEquals(2L, exec("char x = (char)1; def y = (long)1; return x + y"));
-        assertEquals(2L, exec("int x = (int)1; def y = (long)1; return x + y"));
-        assertEquals(2L, exec("long x = (long)1; def y = (long)1; return x + y"));
-        assertEquals(2F, exec("float x = (float)1; def y = (long)1; return x + y"));
-        assertEquals(2D, exec("double x = (double)1; def y = (long)1; return x + y"));
-
-        assertEquals(2F, exec("byte x = (byte)1; def y = (float)1; return x + y"));
-        assertEquals(2F, exec("short x = (short)1; def y = (float)1; return x + y"));
-        assertEquals(2F, exec("char x = (char)1; def y = (float)1; return x + y"));
-        assertEquals(2F, exec("int x = (int)1; def y = (float)1; return x + y"));
-        assertEquals(2F, exec("long x = (long)1; def y = (float)1; return x + y"));
-        assertEquals(2F, exec("float x = (float)1; def y = (float)1; return x + y"));
-        assertEquals(2D, exec("double x = (double)1; def y = (float)1; return x + y"));
-
-        assertEquals(2D, exec("byte x = (byte)1; def y = (double)1; return x + y"));
-        assertEquals(2D, exec("short x = (short)1; def y = (double)1; return x + y"));
-        assertEquals(2D, exec("char x = (char)1; def y = (double)1; return x + y"));
-        assertEquals(2D, exec("int x = (int)1; def y = (double)1; return x + y"));
-        assertEquals(2D, exec("long x = (long)1; def y = (double)1; return x + y"));
-        assertEquals(2D, exec("float x = (float)1; def y = (double)1; return x + y"));
-        assertEquals(2D, exec("double x = (double)1; def y = (double)1; return x + y"));
-    }
-    
-    public void testAddTypedRHS() {
-        assertEquals(2, exec("def x = (byte)1; byte y = (byte)1; return x + y"));
-        assertEquals(2, exec("def x = (short)1; byte y = (byte)1; return x + y"));
-        assertEquals(2, exec("def x = (char)1; byte y = (byte)1; return x + y"));
-        assertEquals(2, exec("def x = (int)1; byte y = (byte)1; return x + y"));
-        assertEquals(2L, exec("def x = (long)1; byte y = (byte)1; return x + y"));
-        assertEquals(2F, exec("def x = (float)1; byte y = (byte)1; return x + y"));
-        assertEquals(2D, exec("def x = (double)1; byte y = (byte)1; return x + y"));
-
-        assertEquals(2, exec("def x = (byte)1; short y = (short)1; return x + y"));
-        assertEquals(2, exec("def x = (short)1; short y = (short)1; return x + y"));
-        assertEquals(2, exec("def x = (char)1; short y = (short)1; return x + y"));
-        assertEquals(2, exec("def x = (int)1; short y = (short)1; return x + y"));
-        assertEquals(2L, exec("def x = (long)1; short y = (short)1; return x + y"));
-        assertEquals(2F, exec("def x = (float)1; short y = (short)1; return x + y"));
-        assertEquals(2D, exec("def x = (double)1; short y = (short)1; return x + y"));
-
-        assertEquals(2, exec("def x = (byte)1; char y = (char)1; return x + y"));
-        assertEquals(2, exec("def x = (short)1; char y = (char)1; return x + y"));
-        assertEquals(2, exec("def x = (char)1; char y = (char)1; return x + y"));
-        assertEquals(2, exec("def x = (int)1; char y = (char)1; return x + y"));
-        assertEquals(2L, exec("def x = (long)1; char y = (char)1; return x + y"));
-        assertEquals(2F, exec("def x = (float)1; char y = (char)1; return x + y"));
-        assertEquals(2D, exec("def x = (double)1; char y = (char)1; return x + y"));
-
-        assertEquals(2, exec("def x = (byte)1; int y = (int)1; return x + y"));
-        assertEquals(2, exec("def x = (short)1; int y = (int)1; return x + y"));
-        assertEquals(2, exec("def x = (char)1; int y = (int)1; return x + y"));
-        assertEquals(2, exec("def x = (int)1; int y = (int)1; return x + y"));
-        assertEquals(2L, exec("def x = (long)1; int y = (int)1; return x + y"));
-        assertEquals(2F, exec("def x = (float)1; int y = (int)1; return x + y"));
-        assertEquals(2D, exec("def x = (double)1; int y = (int)1; return x + y"));
-
-        assertEquals(2L, exec("def x = (byte)1; long y = (long)1; return x + y"));
-        assertEquals(2L, exec("def x = (short)1; long y = (long)1; return x + y"));
-        assertEquals(2L, exec("def x = (char)1; long y = (long)1; return x + y"));
-        assertEquals(2L, exec("def x = (int)1; long y = (long)1; return x + y"));
-        assertEquals(2L, exec("def x = (long)1; long y = (long)1; return x + y"));
-        assertEquals(2F, exec("def x = (float)1; long y = (long)1; return x + y"));
-        assertEquals(2D, exec("def x = (double)1; long y = (long)1; return x + y"));
-
-        assertEquals(2F, exec("def x = (byte)1; float y = (float)1; return x + y"));
-        assertEquals(2F, exec("def x = (short)1; float y = (float)1; return x + y"));
-        assertEquals(2F, exec("def x = (char)1; float y = (float)1; return x + y"));
-        assertEquals(2F, exec("def x = (int)1; float y = (float)1; return x + y"));
-        assertEquals(2F, exec("def x = (long)1; float y = (float)1; return x + y"));
-        assertEquals(2F, exec("def x = (float)1; float y = (float)1; return x + y"));
-        assertEquals(2D, exec("def x = (double)1; float y = (float)1; return x + y"));
-
-        assertEquals(2D, exec("def x = (byte)1; double y = (double)1; return x + y"));
-        assertEquals(2D, exec("def x = (short)1; double y = (double)1; return x + y"));
-        assertEquals(2D, exec("def x = (char)1; double y = (double)1; return x + y"));
-        assertEquals(2D, exec("def x = (int)1; double y = (double)1; return x + y"));
-        assertEquals(2D, exec("def x = (long)1; double y = (double)1; return x + y"));
-        assertEquals(2D, exec("def x = (float)1; double y = (double)1; return x + y"));
-        assertEquals(2D, exec("def x = (double)1; double y = (double)1; return x + y"));
-    }
-    
-    public void testAddConcat() {
-        assertEquals("a" + (byte)2, exec("def x = 'a'; def y = (byte)2; return x + y"));
-        assertEquals("a" + (short)2, exec("def x = 'a'; def y = (short)2; return x + y"));
-        assertEquals("a" + (char)2, exec("def x = 'a'; def y = (char)2; return x + y"));
-        assertEquals("a" + 2, exec("def x = 'a'; def y = (int)2; return x + y"));
-        assertEquals("a" + 2L, exec("def x = 'a'; def y = (long)2; return x + y"));
-        assertEquals("a" + 2F, exec("def x = 'a'; def y = (float)2; return x + y"));
-        assertEquals("a" + 2D, exec("def x = 'a'; def y = (double)2; return x + y"));
-        assertEquals("ab", exec("def x = 'a'; def y = 'b'; return x + y"));
-        assertEquals((byte)2 + "a", exec("def x = 'a'; def y = (byte)2; return y + x"));
-        assertEquals((short)2 + "a", exec("def x = 'a'; def y = (short)2; return y + x"));
-        assertEquals((char)2 + "a", exec("def x = 'a'; def y = (char)2; return y + x"));
-        assertEquals(2 + "a", exec("def x = 'a'; def y = (int)2; return y + x"));
-        assertEquals(2L + "a", exec("def x = 'a'; def y = (long)2; return y + x"));
-        assertEquals(2F + "a", exec("def x = 'a'; def y = (float)2; return y + x"));
-        assertEquals(2D + "a", exec("def x = 'a'; def y = (double)2; return y + x"));
-        assertEquals("anull", exec("def x = 'a'; def y = null; return x + y"));
-        assertEquals("nullb", exec("def x = null; def y = 'b'; return x + y"));
-        expectScriptThrows(NullPointerException.class, () -> {
-            exec("def x = null; def y = null; return x + y");
-        });
-    }
-
-    public void testSub() {
-        assertEquals(0, exec("def x = (byte)1; def y = (byte)1; return x - y"));
-        assertEquals(0, exec("def x = (short)1; def y = (byte)1; return x - y"));
-        assertEquals(0, exec("def x = (char)1; def y = (byte)1; return x - y"));
-        assertEquals(0, exec("def x = (int)1; def y = (byte)1; return x - y"));
-        assertEquals(0L, exec("def x = (long)1; def y = (byte)1; return x - y"));
-        assertEquals(0F, exec("def x = (float)1; def y = (byte)1; return x - y"));
-        assertEquals(0D, exec("def x = (double)1; def y = (byte)1; return x - y"));
-
-        assertEquals(0, exec("def x = (byte)1; def y = (short)1; return x - y"));
-        assertEquals(0, exec("def x = (short)1; def y = (short)1; return x - y"));
-        assertEquals(0, exec("def x = (char)1; def y = (short)1; return x - y"));
-        assertEquals(0, exec("def x = (int)1; def y = (short)1; return x - y"));
-        assertEquals(0L, exec("def x = (long)1; def y = (short)1; return x - y"));
-        assertEquals(0F, exec("def x = (float)1; def y = (short)1; return x - y"));
-        assertEquals(0D, exec("def x = (double)1; def y = (short)1; return x - y"));
-
-        assertEquals(0, exec("def x = (byte)1; def y = (char)1; return x - y"));
-        assertEquals(0, exec("def x = (short)1; def y = (char)1; return x - y"));
-        assertEquals(0, exec("def x = (char)1; def y = (char)1; return x - y"));
-        assertEquals(0, exec("def x = (int)1; def y = (char)1; return x - y"));
-        assertEquals(0L, exec("def x = (long)1; def y = (char)1; return x - y"));
-        assertEquals(0F, exec("def x = (float)1; def y = (char)1; return x - y"));
-        assertEquals(0D, exec("def x = (double)1; def y = (char)1; return x - y"));
-
-        assertEquals(0, exec("def x = (byte)1; def y = (int)1; return x - y"));
-        assertEquals(0, exec("def x = (short)1; def y = (int)1; return x - y"));
-        assertEquals(0, exec("def x = (char)1; def y = (int)1; return x - y"));
-        assertEquals(0, exec("def x = (int)1; def y = (int)1; return x - y"));
-        assertEquals(0L, exec("def x = (long)1; def y = (int)1; return x - y"));
-        assertEquals(0F, exec("def x = (float)1; def y = (int)1; return x - y"));
-        assertEquals(0D, exec("def x = (double)1; def y = (int)1; return x - y"));
-
-        assertEquals(0L, exec("def x = (byte)1; def y = (long)1; return x - y"));
-        assertEquals(0L, exec("def x = (short)1; def y = (long)1; return x - y"));
-        assertEquals(0L, exec("def x = (char)1; def y = (long)1; return x - y"));
-        assertEquals(0L, exec("def x = (int)1; def y = (long)1; return x - y"));
-        assertEquals(0L, exec("def x = (long)1; def y = (long)1; return x - y"));
-        assertEquals(0F, exec("def x = (float)1; def y = (long)1; return x - y"));
-        assertEquals(0D, exec("def x = (double)1; def y = (long)1; return x - y"));
-
-        assertEquals(0F, exec("def x = (byte)1; def y = (float)1; return x - y"));
-        assertEquals(0F, exec("def x = (short)1; def y = (float)1; return x - y"));
-        assertEquals(0F, exec("def x = (char)1; def y = (float)1; return x - y"));
-        assertEquals(0F, exec("def x = (int)1; def y = (float)1; return x - y"));
-        assertEquals(0F, exec("def x = (long)1; def y = (float)1; return x - y"));
-        assertEquals(0F, exec("def x = (float)1; def y = (float)1; return x - y"));
-        assertEquals(0D, exec("def x = (double)1; def y = (float)1; return x - y"));
-
-        assertEquals(0D, exec("def x = (byte)1; def y = (double)1; return x - y"));
-        assertEquals(0D, exec("def x = (short)1; def y = (double)1; return x - y"));
-        assertEquals(0D, exec("def x = (char)1; def y = (double)1; return x - y"));
-        assertEquals(0D, exec("def x = (int)1; def y = (double)1; return x - y"));
-        assertEquals(0D, exec("def x = (long)1; def y = (double)1; return x - y"));
-        assertEquals(0D, exec("def x = (float)1; def y = (double)1; return x - y"));
-        assertEquals(0D, exec("def x = (double)1; def y = (double)1; return x - y"));
-    }
-    
-    public void testSubTypedLHS() {
-        assertEquals(0, exec("byte x = (byte)1; def y = (byte)1; return x - y"));
-        assertEquals(0, exec("short x = (short)1; def y = (byte)1; return x - y"));
-        assertEquals(0, exec("char x = (char)1; def y = (byte)1; return x - y"));
-        assertEquals(0, exec("int x = (int)1; def y = (byte)1; return x - y"));
-        assertEquals(0L, exec("long x = (long)1; def y = (byte)1; return x - y"));
-        assertEquals(0F, exec("float x = (float)1; def y = (byte)1; return x - y"));
-        assertEquals(0D, exec("double x = (double)1; def y = (byte)1; return x - y"));
-
-        assertEquals(0, exec("byte x = (byte)1; def y = (short)1; return x - y"));
-        assertEquals(0, exec("short x = (short)1; def y = (short)1; return x - y"));
-        assertEquals(0, exec("char x = (char)1; def y = (short)1; return x - y"));
-        assertEquals(0, exec("int x = (int)1; def y = (short)1; return x - y"));
-        assertEquals(0L, exec("long x = (long)1; def y = (short)1; return x - y"));
-        assertEquals(0F, exec("float x = (float)1; def y = (short)1; return x - y"));
-        assertEquals(0D, exec("double x = (double)1; def y = (short)1; return x - y"));
-
-        assertEquals(0, exec("byte x = (byte)1; def y = (char)1; return x - y"));
-        assertEquals(0, exec("short x = (short)1; def y = (char)1; return x - y"));
-        assertEquals(0, exec("char x = (char)1; def y = (char)1; return x - y"));
-        assertEquals(0, exec("int x = (int)1; def y = (char)1; return x - y"));
-        assertEquals(0L, exec("long x = (long)1; def y = (char)1; return x - y"));
-        assertEquals(0F, exec("float x = (float)1; def y = (char)1; return x - y"));
-        assertEquals(0D, exec("double x = (double)1; def y = (char)1; return x - y"));
-
-        assertEquals(0, exec("byte x = (byte)1; def y = (int)1; return x - y"));
-        assertEquals(0, exec("short x = (short)1; def y = (int)1; return x - y"));
-        assertEquals(0, exec("char x = (char)1; def y = (int)1; return x - y"));
-        assertEquals(0, exec("int x = (int)1; def y = (int)1; return x - y"));
-        assertEquals(0L, exec("long x = (long)1; def y = (int)1; return x - y"));
-        assertEquals(0F, exec("float x = (float)1; def y = (int)1; return x - y"));
-        assertEquals(0D, exec("double x = (double)1; def y = (int)1; return x - y"));
-
-        assertEquals(0L, exec("byte x = (byte)1; def y = (long)1; return x - y"));
-        assertEquals(0L, exec("short x = (short)1; def y = (long)1; return x - y"));
-        assertEquals(0L, exec("char x = (char)1; def y = (long)1; return x - y"));
-        assertEquals(0L, exec("int x = (int)1; def y = (long)1; return x - y"));
-        assertEquals(0L, exec("long x = (long)1; def y = (long)1; return x - y"));
-        assertEquals(0F, exec("float x = (float)1; def y = (long)1; return x - y"));
-        assertEquals(0D, exec("double x = (double)1; def y = (long)1; return x - y"));
-
-        assertEquals(0F, exec("byte x = (byte)1; def y = (float)1; return x - y"));
-        assertEquals(0F, exec("short x = (short)1; def y = (float)1; return x - y"));
-        assertEquals(0F, exec("char x = (char)1; def y = (float)1; return x - y"));
-        assertEquals(0F, exec("int x = (int)1; def y = (float)1; return x - y"));
-        assertEquals(0F, exec("long x = (long)1; def y = (float)1; return x - y"));
-        assertEquals(0F, exec("float x = (float)1; def y = (float)1; return x - y"));
-        assertEquals(0D, exec("double x = (double)1; def y = (float)1; return x - y"));
-
-        assertEquals(0D, exec("byte x = (byte)1; def y = (double)1; return x - y"));
-        assertEquals(0D, exec("short x = (short)1; def y = (double)1; return x - y"));
-        assertEquals(0D, exec("char x = (char)1; def y = (double)1; return x - y"));
-        assertEquals(0D, exec("int x = (int)1; def y = (double)1; return x - y"));
-        assertEquals(0D, exec("long x = (long)1; def y = (double)1; return x - y"));
-        assertEquals(0D, exec("float x = (float)1; def y = (double)1; return x - y"));
-        assertEquals(0D, exec("double x = (double)1; def y = (double)1; return x - y"));
-    }
-    
-    public void testSubTypedRHS() {
-        assertEquals(0, exec("def x = (byte)1; byte y = (byte)1; return x - y"));
-        assertEquals(0, exec("def x = (short)1; byte y = (byte)1; return x - y"));
-        assertEquals(0, exec("def x = (char)1; byte y = (byte)1; return x - y"));
-        assertEquals(0, exec("def x = (int)1; byte y = (byte)1; return x - y"));
-        assertEquals(0L, exec("def x = (long)1; byte y = (byte)1; return x - y"));
-        assertEquals(0F, exec("def x = (float)1; byte y = (byte)1; return x - y"));
-        assertEquals(0D, exec("def x = (double)1; byte y = (byte)1; return x - y"));
-
-        assertEquals(0, exec("def x = (byte)1; short y = (short)1; return x - y"));
-        assertEquals(0, exec("def x = (short)1; short y = (short)1; return x - y"));
-        assertEquals(0, exec("def x = (char)1; short y = (short)1; return x - y"));
-        assertEquals(0, exec("def x = (int)1; short y = (short)1; return x - y"));
-        assertEquals(0L, exec("def x = (long)1; short y = (short)1; return x - y"));
-        assertEquals(0F, exec("def x = (float)1; short y = (short)1; return x - y"));
-        assertEquals(0D, exec("def x = (double)1; short y = (short)1; return x - y"));
-
-        assertEquals(0, exec("def x = (byte)1; char y = (char)1; return x - y"));
-        assertEquals(0, exec("def x = (short)1; char y = (char)1; return x - y"));
-        assertEquals(0, exec("def x = (char)1; char y = (char)1; return x - y"));
-        assertEquals(0, exec("def x = (int)1; char y = (char)1; return x - y"));
-        assertEquals(0L, exec("def x = (long)1; char y = (char)1; return x - y"));
-        assertEquals(0F, exec("def x = (float)1; char y = (char)1; return x - y"));
-        assertEquals(0D, exec("def x = (double)1; char y = (char)1; return x - y"));
-
-        assertEquals(0, exec("def x = (byte)1; int y = (int)1; return x - y"));
-        assertEquals(0, exec("def x = (short)1; int y = (int)1; return x - y"));
-        assertEquals(0, exec("def x = (char)1; int y = (int)1; return x - y"));
-        assertEquals(0, exec("def x = (int)1; int y = (int)1; return x - y"));
-        assertEquals(0L, exec("def x = (long)1; int y = (int)1; return x - y"));
-        assertEquals(0F, exec("def x = (float)1; int y = (int)1; return x - y"));
-        assertEquals(0D, exec("def x = (double)1; int y = (int)1; return x - y"));
-
-        assertEquals(0L, exec("def x = (byte)1; long y = (long)1; return x - y"));
-        assertEquals(0L, exec("def x = (short)1; long y = (long)1; return x - y"));
-        assertEquals(0L, exec("def x = (char)1; long y = (long)1; return x - y"));
-        assertEquals(0L, exec("def x = (int)1; long y = (long)1; return x - y"));
-        assertEquals(0L, exec("def x = (long)1; long y = (long)1; return x - y"));
-        assertEquals(0F, exec("def x = (float)1; long y = (long)1; return x - y"));
-        assertEquals(0D, exec("def x = (double)1; long y = (long)1; return x - y"));
-
-        assertEquals(0F, exec("def x = (byte)1; float y = (float)1; return x - y"));
-        assertEquals(0F, exec("def x = (short)1; float y = (float)1; return x - y"));
-        assertEquals(0F, exec("def x = (char)1; float y = (float)1; return x - y"));
-        assertEquals(0F, exec("def x = (int)1; float y = (float)1; return x - y"));
-        assertEquals(0F, exec("def x = (long)1; float y = (float)1; return x - y"));
-        assertEquals(0F, exec("def x = (float)1; float y = (float)1; return x - y"));
-        assertEquals(0D, exec("def x = (double)1; float y = (float)1; return x - y"));
-
-        assertEquals(0D, exec("def x = (byte)1; double y = (double)1; return x - y"));
-        assertEquals(0D, exec("def x = (short)1; double y = (double)1; return x - y"));
-        assertEquals(0D, exec("def x = (char)1; double y = (double)1; return x - y"));
-        assertEquals(0D, exec("def x = (int)1; double y = (double)1; return x - y"));
-        assertEquals(0D, exec("def x = (long)1; double y = (double)1; return x - y"));
-        assertEquals(0D, exec("def x = (float)1; double y = (double)1; return x - y"));
-        assertEquals(0D, exec("def x = (double)1; double y = (double)1; return x - y"));
-    }
-
-    public void testLsh() {
-        assertEquals(2, exec("def x = (byte)1; def y = (byte)1; return x << y"));
-        assertEquals(2, exec("def x = (short)1; def y = (byte)1; return x << y"));
-        assertEquals(2, exec("def x = (char)1; def y = (byte)1; return x << y"));
-        assertEquals(2, exec("def x = (int)1; def y = (byte)1; return x << y"));
-        assertEquals(2L, exec("def x = (long)1; def y = (byte)1; return x << y"));
-
-        assertEquals(2, exec("def x = (byte)1; def y = (short)1; return x << y"));
-        assertEquals(2, exec("def x = (short)1; def y = (short)1; return x << y"));
-        assertEquals(2, exec("def x = (char)1; def y = (short)1; return x << y"));
-        assertEquals(2, exec("def x = (int)1; def y = (short)1; return x << y"));
-        assertEquals(2L, exec("def x = (long)1; def y = (short)1; return x << y"));
-
-        assertEquals(2, exec("def x = (byte)1; def y = (char)1; return x << y"));
-        assertEquals(2, exec("def x = (short)1; def y = (char)1; return x << y"));
-        assertEquals(2, exec("def x = (char)1; def y = (char)1; return x << y"));
-        assertEquals(2, exec("def x = (int)1; def y = (char)1; return x << y"));
-        assertEquals(2L, exec("def x = (long)1; def y = (char)1; return x << y"));
-
-        assertEquals(2, exec("def x = (byte)1; def y = (int)1; return x << y"));
-        assertEquals(2, exec("def x = (short)1; def y = (int)1; return x << y"));
-        assertEquals(2, exec("def x = (char)1; def y = (int)1; return x << y"));
-        assertEquals(2, exec("def x = (int)1; def y = (int)1; return x << y"));
-        assertEquals(2L, exec("def x = (long)1; def y = (int)1; return x << y"));
-
-        assertEquals(2, exec("def x = (byte)1; def y = (long)1; return x << y"));
-        assertEquals(2, exec("def x = (short)1; def y = (long)1; return x << y"));
-        assertEquals(2, exec("def x = (char)1; def y = (long)1; return x << y"));
-        assertEquals(2, exec("def x = (int)1; def y = (long)1; return x << y"));
-        assertEquals(2L, exec("def x = (long)1; def y = (long)1; return x << y"));
-
-        assertEquals(2, exec("def x = (byte)1; def y = (byte)1; return x << y"));
-        assertEquals(2, exec("def x = (short)1; def y = (short)1; return x << y"));
-        assertEquals(2, exec("def x = (char)1; def y = (char)1; return x << y"));
-        assertEquals(2, exec("def x = (int)1; def y = (int)1; return x << y"));
-        assertEquals(2L, exec("def x = (long)1; def y = (long)1; return x << y"));
-    }
-    
-    public void testLshTypedLHS() {
-        assertEquals(2, exec("byte x = (byte)1; def y = (byte)1; return x << y"));
-        assertEquals(2, exec("short x = (short)1; def y = (byte)1; return x << y"));
-        assertEquals(2, exec("char x = (char)1; def y = (byte)1; return x << y"));
-        assertEquals(2, exec("int x = (int)1; def y = (byte)1; return x << y"));
-        assertEquals(2L, exec("long x = (long)1; def y = (byte)1; return x << y"));
-
-        assertEquals(2, exec("byte x = (byte)1; def y = (short)1; return x << y"));
-        assertEquals(2, exec("short x = (short)1; def y = (short)1; return x << y"));
-        assertEquals(2, exec("char x = (char)1; def y = (short)1; return x << y"));
-        assertEquals(2, exec("int x = (int)1; def y = (short)1; return x << y"));
-        assertEquals(2L, exec("long x = (long)1; def y = (short)1; return x << y"));
-
-        assertEquals(2, exec("byte x = (byte)1; def y = (char)1; return x << y"));
-        assertEquals(2, exec("short x = (short)1; def y = (char)1; return x << y"));
-        assertEquals(2, exec("char x = (char)1; def y = (char)1; return x << y"));
-        assertEquals(2, exec("int x = (int)1; def y = (char)1; return x << y"));
-        assertEquals(2L, exec("long x = (long)1; def y = (char)1; return x << y"));
-
-        assertEquals(2, exec("byte x = (byte)1; def y = (int)1; return x << y"));
-        assertEquals(2, exec("short x = (short)1; def y = (int)1; return x << y"));
-        assertEquals(2, exec("char x = (char)1; def y = (int)1; return x << y"));
-        assertEquals(2, exec("int x = (int)1; def y = (int)1; return x << y"));
-        assertEquals(2L, exec("long x = (long)1; def y = (int)1; return x << y"));
-
-        assertEquals(2, exec("byte x = (byte)1; def y = (long)1; return x << y"));
-        assertEquals(2, exec("short x = (short)1; def y = (long)1; return x << y"));
-        assertEquals(2, exec("char x = (char)1; def y = (long)1; return x << y"));
-        assertEquals(2, exec("int x = (int)1; def y = (long)1; return x << y"));
-        assertEquals(2L, exec("long x = (long)1; def y = (long)1; return x << y"));
-
-        assertEquals(2, exec("byte x = (byte)1; def y = (byte)1; return x << y"));
-        assertEquals(2, exec("short x = (short)1; def y = (short)1; return x << y"));
-        assertEquals(2, exec("char x = (char)1; def y = (char)1; return x << y"));
-        assertEquals(2, exec("int x = (int)1; def y = (int)1; return x << y"));
-        assertEquals(2L, exec("long x = (long)1; def y = (long)1; return x << y"));
-    }
-    
-    public void testLshTypedRHS() {
-        assertEquals(2, exec("def x = (byte)1; byte y = (byte)1; return x << y"));
-        assertEquals(2, exec("def x = (short)1; byte y = (byte)1; return x << y"));
-        assertEquals(2, exec("def x = (char)1; byte y = (byte)1; return x << y"));
-        assertEquals(2, exec("def x = (int)1; byte y = (byte)1; return x << y"));
-        assertEquals(2L, exec("def x = (long)1; byte y = (byte)1; return x << y"));
-
-        assertEquals(2, exec("def x = (byte)1; short y = (short)1; return x << y"));
-        assertEquals(2, exec("def x = (short)1; short y = (short)1; return x << y"));
-        assertEquals(2, exec("def x = (char)1; short y = (short)1; return x << y"));
-        assertEquals(2, exec("def x = (int)1; short y = (short)1; return x << y"));
-        assertEquals(2L, exec("def x = (long)1; short y = (short)1; return x << y"));
-
-        assertEquals(2, exec("def x = (byte)1; char y = (char)1; return x << y"));
-        assertEquals(2, exec("def x = (short)1; char y = (char)1; return x << y"));
-        assertEquals(2, exec("def x = (char)1; char y = (char)1; return x << y"));
-        assertEquals(2, exec("def x = (int)1; char y = (char)1; return x << y"));
-        assertEquals(2L, exec("def x = (long)1; char y = (char)1; return x << y"));
-
-        assertEquals(2, exec("def x = (byte)1; int y = (int)1; return x << y"));
-        assertEquals(2, exec("def x = (short)1; int y = (int)1; return x << y"));
-        assertEquals(2, exec("def x = (char)1; int y = (int)1; return x << y"));
-        assertEquals(2, exec("def x = (int)1; int y = (int)1; return x << y"));
-        assertEquals(2L, exec("def x = (long)1; int y = (int)1; return x << y"));
-
-        assertEquals(2, exec("def x = (byte)1; long y = (long)1; return x << y"));
-        assertEquals(2, exec("def x = (short)1; long y = (long)1; return x << y"));
-        assertEquals(2, exec("def x = (char)1; long y = (long)1; return x << y"));
-        assertEquals(2, exec("def x = (int)1; long y = (long)1; return x << y"));
-        assertEquals(2L, exec("def x = (long)1; long y = (long)1; return x << y"));
-
-        assertEquals(2, exec("def x = (byte)1; byte y = (byte)1; return x << y"));
-        assertEquals(2, exec("def x = (short)1; short y = (short)1; return x << y"));
-        assertEquals(2, exec("def x = (char)1; char y = (char)1; return x << y"));
-        assertEquals(2, exec("def x = (int)1; int y = (int)1; return x << y"));
-        assertEquals(2L, exec("def x = (long)1; long y = (long)1; return x << y"));
-    }
-
-    public void testRsh() {
-        assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >> y"));
-        assertEquals(2, exec("def x = (short)4; def y = (byte)1; return x >> y"));
-        assertEquals(2, exec("def x = (char)4; def y = (byte)1; return x >> y"));
-        assertEquals(2, exec("def x = (int)4; def y = (byte)1; return x >> y"));
-        assertEquals(2L, exec("def x = (long)4; def y = (byte)1; return x >> y"));
-
-        assertEquals(2, exec("def x = (byte)4; def y = (short)1; return x >> y"));
-        assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >> y"));
-        assertEquals(2, exec("def x = (char)4; def y = (short)1; return x >> y"));
-        assertEquals(2, exec("def x = (int)4; def y = (short)1; return x >> y"));
-        assertEquals(2L, exec("def x = (long)4; def y = (short)1; return x >> y"));
-
-        assertEquals(2, exec("def x = (byte)4; def y = (char)1; return x >> y"));
-        assertEquals(2, exec("def x = (short)4; def y = (char)1; return x >> y"));
-        assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >> y"));
-        assertEquals(2, exec("def x = (int)4; def y = (char)1; return x >> y"));
-        assertEquals(2L, exec("def x = (long)4; def y = (char)1; return x >> y"));
-
-        assertEquals(2, exec("def x = (byte)4; def y = (int)1; return x >> y"));
-        assertEquals(2, exec("def x = (short)4; def y = (int)1; return x >> y"));
-        assertEquals(2, exec("def x = (char)4; def y = (int)1; return x >> y"));
-        assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >> y"));
-        assertEquals(2L, exec("def x = (long)4; def y = (int)1; return x >> y"));
-
-        assertEquals(2, exec("def x = (byte)4; def y = (long)1; return x >> y"));
-        assertEquals(2, exec("def x = (short)4; def y = (long)1; return x >> y"));
-        assertEquals(2, exec("def x = (char)4; def y = (long)1; return x >> y"));
-        assertEquals(2, exec("def x = (int)4; def y = (long)1; return x >> y"));
-        assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >> y"));
-
-        assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >> y"));
-        assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >> y"));
-        assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >> y"));
-        assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >> y"));
-        assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >> y"));
-    }
-    
-    public void testRshTypeLHS() {
-        assertEquals(2, exec("byte x = (byte)4; def y = (byte)1; return x >> y"));
-        assertEquals(2, exec("short x = (short)4; def y = (byte)1; return x >> y"));
-        assertEquals(2, exec("char x = (char)4; def y = (byte)1; return x >> y"));
-        assertEquals(2, exec("int x = (int)4; def y = (byte)1; return x >> y"));
-        assertEquals(2L, exec("long x = (long)4; def y = (byte)1; return x >> y"));
-
-        assertEquals(2, exec("byte x = (byte)4; def y = (short)1; return x >> y"));
-        assertEquals(2, exec("short x = (short)4; def y = (short)1; return x >> y"));
-        assertEquals(2, exec("char x = (char)4; def y = (short)1; return x >> y"));
-        assertEquals(2, exec("int x = (int)4; def y = (short)1; return x >> y"));
-        assertEquals(2L, exec("long x = (long)4; def y = (short)1; return x >> y"));
-
-        assertEquals(2, exec("byte x = (byte)4; def y = (char)1; return x >> y"));
-        assertEquals(2, exec("short x = (short)4; def y = (char)1; return x >> y"));
-        assertEquals(2, exec("char x = (char)4; def y = (char)1; return x >> y"));
-        assertEquals(2, exec("int x = (int)4; def y = (char)1; return x >> y"));
-        assertEquals(2L, exec("long x = (long)4; def y = (char)1; return x >> y"));
-
-        assertEquals(2, exec("byte x = (byte)4; def y = (int)1; return x >> y"));
-        assertEquals(2, exec("short x = (short)4; def y = (int)1; return x >> y"));
-        assertEquals(2, exec("char x = (char)4; def y = (int)1; return x >> y"));
-        assertEquals(2, exec("int x = (int)4; def y = (int)1; return x >> y"));
-        assertEquals(2L, exec("long x = (long)4; def y = (int)1; return x >> y"));
-
-        assertEquals(2, exec("byte x = (byte)4; def y = (long)1; return x >> y"));
-        assertEquals(2, exec("short x = (short)4; def y = (long)1; return x >> y"));
-        assertEquals(2, exec("char x = (char)4; def y = (long)1; return x >> y"));
-        assertEquals(2, exec("int x = (int)4; def y = (long)1; return x >> y"));
-        assertEquals(2L, exec("long x = (long)4; def y = (long)1; return x >> y"));
-
-        assertEquals(2, exec("byte x = (byte)4; def y = (byte)1; return x >> y"));
-        assertEquals(2, exec("short x = (short)4; def y = (short)1; return x >> y"));
-        assertEquals(2, exec("char x = (char)4; def y = (char)1; return x >> y"));
-        assertEquals(2, exec("int x = (int)4; def y = (int)1; return x >> y"));
-        assertEquals(2L, exec("long x = (long)4; def y = (long)1; return x >> y"));
-    }
-    
-    public void testRshTypedLHS() {
-        assertEquals(2, exec("def x = (byte)4; byte y = (byte)1; return x >> y"));
-        assertEquals(2, exec("def x = (short)4; byte y = (byte)1; return x >> y"));
-        assertEquals(2, exec("def x = (char)4; byte y = (byte)1; return x >> y"));
-        assertEquals(2, exec("def x = (int)4; byte y = (byte)1; return x >> y"));
-        assertEquals(2L, exec("def x = (long)4; byte y = (byte)1; return x >> y"));
-
-        assertEquals(2, exec("def x = (byte)4; short y = (short)1; return x >> y"));
-        assertEquals(2, exec("def x = (short)4; short y = (short)1; return x >> y"));
-        assertEquals(2, exec("def x = (char)4; short y = (short)1; return x >> y"));
-        assertEquals(2, exec("def x = (int)4; short y = (short)1; return x >> y"));
-        assertEquals(2L, exec("def x = (long)4; short y = (short)1; return x >> y"));
-
-        assertEquals(2, exec("def x = (byte)4; char y = (char)1; return x >> y"));
-        assertEquals(2, exec("def x = (short)4; char y = (char)1; return x >> y"));
-        assertEquals(2, exec("def x = (char)4; char y = (char)1; return x >> y"));
-        assertEquals(2, exec("def x = (int)4; char y = (char)1; return x >> y"));
-        assertEquals(2L, exec("def x = (long)4; char y = (char)1; return x >> y"));
-
-        assertEquals(2, exec("def x = (byte)4; int y = (int)1; return x >> y"));
-        assertEquals(2, exec("def x = (short)4; int y = (int)1; return x >> y"));
-        assertEquals(2, exec("def x = (char)4; int y = (int)1; return x >> y"));
-        assertEquals(2, exec("def x = (int)4; int y = (int)1; return x >> y"));
-        assertEquals(2L, exec("def x = (long)4; int y = (int)1; return x >> y"));
-
-        assertEquals(2, exec("def x = (byte)4; long y = (long)1; return x >> y"));
-        assertEquals(2, exec("def x = (short)4; long y = (long)1; return x >> y"));
-        assertEquals(2, exec("def x = (char)4; long y = (long)1; return x >> y"));
-        assertEquals(2, exec("def x = (int)4; long y = (long)1; return x >> y"));
-        assertEquals(2L, exec("def x = (long)4; long y = (long)1; return x >> y"));
-
-        assertEquals(2, exec("def x = (byte)4; byte y = (byte)1; return x >> y"));
-        assertEquals(2, exec("def x = (short)4; short y = (short)1; return x >> y"));
-        assertEquals(2, exec("def x = (char)4; char y = (char)1; return x >> y"));
-        assertEquals(2, exec("def x = (int)4; int y = (int)1; return x >> y"));
-        assertEquals(2L, exec("def x = (long)4; long y = (long)1; return x >> y"));
-    }
-
-    public void testUsh() {
-        assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >>> y"));
-        assertEquals(2, exec("def x = (short)4; def y = (byte)1; return x >>> y"));
-        assertEquals(2, exec("def x = (char)4; def y = (byte)1; return x >>> y"));
-        assertEquals(2, exec("def x = (int)4; def y = (byte)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (long)4; def y = (byte)1; return x >>> y"));
-
-        assertEquals(2, exec("def x = (byte)4; def y = (short)1; return x >>> y"));
-        assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >>> y"));
-        assertEquals(2, exec("def x = (char)4; def y = (short)1; return x >>> y"));
-        assertEquals(2, exec("def x = (int)4; def y = (short)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (long)4; def y = (short)1; return x >>> y"));
-
-        assertEquals(2, exec("def x = (byte)4; def y = (char)1; return x >>> y"));
-        assertEquals(2, exec("def x = (short)4; def y = (char)1; return x >>> y"));
-        assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >>> y"));
-        assertEquals(2, exec("def x = (int)4; def y = (char)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (long)4; def y = (char)1; return x >>> y"));
-
-        assertEquals(2, exec("def x = (byte)4; def y = (int)1; return x >>> y"));
-        assertEquals(2, exec("def x = (short)4; def y = (int)1; return x >>> y"));
-        assertEquals(2, exec("def x = (char)4; def y = (int)1; return x >>> y"));
-        assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (long)4; def y = (int)1; return x >>> y"));
-
-        assertEquals(2, exec("def x = (byte)4; def y = (long)1; return x >>> y"));
-        assertEquals(2, exec("def x = (short)4; def y = (long)1; return x >>> y"));
-        assertEquals(2, exec("def x = (char)4; def y = (long)1; return x >>> y"));
-        assertEquals(2, exec("def x = (int)4; def y = (long)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >>> y"));
-
-        assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >>> y"));
-        assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >>> y"));
-        assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >>> y"));
-        assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >>> y"));
-    }
-    
-    public void testUshTypedLHS() {
-        assertEquals(2, exec("byte x = (byte)4; def y = (byte)1; return x >>> y"));
-        assertEquals(2, exec("short x = (short)4; def y = (byte)1; return x >>> y"));
-        assertEquals(2, exec("char x = (char)4; def y = (byte)1; return x >>> y"));
-        assertEquals(2, exec("int x = (int)4; def y = (byte)1; return x >>> y"));
-        assertEquals(2L, exec("long x = (long)4; def y = (byte)1; return x >>> y"));
-
-        assertEquals(2, exec("byte x = (byte)4; def y = (short)1; return x >>> y"));
-        assertEquals(2, exec("short x = (short)4; def y = (short)1; return x >>> y"));
-        assertEquals(2, exec("char x = (char)4; def y = (short)1; return x >>> y"));
-        assertEquals(2, exec("int x = (int)4; def y = (short)1; return x >>> y"));
-        assertEquals(2L, exec("long x = (long)4; def y = (short)1; return x >>> y"));
-
-        assertEquals(2, exec("byte x = (byte)4; def y = (char)1; return x >>> y"));
-        assertEquals(2, exec("short x = (short)4; def y = (char)1; return x >>> y"));
-        assertEquals(2, exec("char x = (char)4; def y = (char)1; return x >>> y"));
-        assertEquals(2, exec("int x = (int)4; def y = (char)1; return x >>> y"));
-        assertEquals(2L, exec("long x = (long)4; def y = (char)1; return x >>> y"));
-
-        assertEquals(2, exec("byte x = (byte)4; def y = (int)1; return x >>> y"));
-        assertEquals(2, exec("short x = (short)4; def y = (int)1; return x >>> y"));
-        assertEquals(2, exec("char x = (char)4; def y = (int)1; return x >>> y"));
-        assertEquals(2, exec("int x = (int)4; def y = (int)1; return x >>> y"));
-        assertEquals(2L, exec("long x = (long)4; def y = (int)1; return x >>> y"));
-
-        assertEquals(2, exec("byte x = (byte)4; def y = (long)1; return x >>> y"));
-        assertEquals(2, exec("short x = (short)4; def y = (long)1; return x >>> y"));
-        assertEquals(2, exec("char x = (char)4; def y = (long)1; return x >>> y"));
-        assertEquals(2, exec("int x = (int)4; def y = (long)1; return x >>> y"));
-        assertEquals(2L, exec("long x = (long)4; def y = (long)1; return x >>> y"));
-
-        assertEquals(2, exec("byte x = (byte)4; def y = (byte)1; return x >>> y"));
-        assertEquals(2, exec("short x = (short)4; def y = (short)1; return x >>> y"));
-        assertEquals(2, exec("char x = (char)4; def y = (char)1; return x >>> y"));
-        assertEquals(2, exec("int x = (int)4; def y = (int)1; return x >>> y"));
-        assertEquals(2L, exec("long x = (long)4; def y = (long)1; return x >>> y"));
-    }
-    
-    public void testUshTypedRHS() {
-        assertEquals(2, exec("def x = (byte)4; byte y = (byte)1; return x >>> y"));
-        assertEquals(2, exec("def x = (short)4; byte y = (byte)1; return x >>> y"));
-        assertEquals(2, exec("def x = (char)4; byte y = (byte)1; return x >>> y"));
-        assertEquals(2, exec("def x = (int)4; byte y = (byte)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (long)4; byte y = (byte)1; return x >>> y"));
-
-        assertEquals(2, exec("def x = (byte)4; short y = (short)1; return x >>> y"));
-        assertEquals(2, exec("def x = (short)4; short y = (short)1; return x >>> y"));
-        assertEquals(2, exec("def x = (char)4; short y = (short)1; return x >>> y"));
-        assertEquals(2, exec("def x = (int)4; short y = (short)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (long)4; short y = (short)1; return x >>> y"));
-
-        assertEquals(2, exec("def x = (byte)4; char y = (char)1; return x >>> y"));
-        assertEquals(2, exec("def x = (short)4; char y = (char)1; return x >>> y"));
-        assertEquals(2, exec("def x = (char)4; char y = (char)1; return x >>> y"));
-        assertEquals(2, exec("def x = (int)4; char y = (char)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (long)4; char y = (char)1; return x >>> y"));
-
-        assertEquals(2, exec("def x = (byte)4; int y = (int)1; return x >>> y"));
-        assertEquals(2, exec("def x = (short)4; int y = (int)1; return x >>> y"));
-        assertEquals(2, exec("def x = (char)4; int y = (int)1; return x >>> y"));
-        assertEquals(2, exec("def x = (int)4; int y = (int)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (long)4; int y = (int)1; return x >>> y"));
-
-        assertEquals(2, exec("def x = (byte)4; long y = (long)1; return x >>> y"));
-        assertEquals(2, exec("def x = (short)4; long y = (long)1; return x >>> y"));
-        assertEquals(2, exec("def x = (char)4; long y = (long)1; return x >>> y"));
-        assertEquals(2, exec("def x = (int)4; long y = (long)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (long)4; long y = (long)1; return x >>> y"));
-
-        assertEquals(2, exec("def x = (byte)4; byte y = (byte)1; return x >>> y"));
-        assertEquals(2, exec("def x = (short)4; short y = (short)1; return x >>> y"));
-        assertEquals(2, exec("def x = (char)4; char y = (char)1; return x >>> y"));
-        assertEquals(2, exec("def x = (int)4; int y = (int)1; return x >>> y"));
-        assertEquals(2L, exec("def x = (long)4; long y = (long)1; return x >>> y"));
-    }
-    
-    public void testBogusShifts() {
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1L; def y = 2F; return x << y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1; def y = 2D; return x << y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1F; def y = 2; return x << y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1D; def y = 2L; return x << y;");
-        });
-        
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1L; def y = 2F; return x >> y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1; def y = 2D; return x >> y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1F; def y = 2; return x >> y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1D; def y = 2L; return x >> y;");
-        });
-        
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1L; def y = 2F; return x >>> y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1; def y = 2D; return x >>> y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1F; def y = 2; return x >>> y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1D; def y = 2L; return x >>> y;");
-        });
-    }
-    
-    public void testBogusShiftsTypedLHS() {
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("long x = 1L; def y = 2F; return x << y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("int x = 1; def y = 2D; return x << y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("float x = 1F; def y = 2; return x << y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("double x = 1D; def y = 2L; return x << y;");
-        });
-        
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("long x = 1L; def y = 2F; return x >> y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("int x = 1; def y = 2D; return x >> y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("float x = 1F; def y = 2; return x >> y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("double x = 1D; def y = 2L; return x >> y;");
-        });
-        
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("long x = 1L; def y = 2F; return x >>> y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("int x = 1; def y = 2D; return x >>> y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("float x = 1F; def y = 2; return x >>> y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("double x = 1D; def y = 2L; return x >>> y;");
-        });
-    }
-    
-    public void testBogusShiftsTypedRHS() {
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1L; float y = 2F; return x << y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1; double y = 2D; return x << y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1F; int y = 2; return x << y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1D; long y = 2L; return x << y;");
-        });
-        
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1L; float y = 2F; return x >> y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1; double y = 2D; return x >> y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1F; int y = 2; return x >> y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1D; long y = 2L; return x >> y;");
-        });
-        
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1L; float y = 2F; return x >>> y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1; double y = 2D; return x >>> y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1F; int y = 2; return x >>> y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("def x = 1D; long y = 2L; return x >>> y;");
-        });
-    }
-
-    public void testAnd() {
-        expectScriptThrows(ClassCastException.class, () -> {
-            exec("def x = (float)4; def y = (byte)1; return x & y");
-        });
-        expectScriptThrows(ClassCastException.class, () -> {
-            exec("def x = (double)4; def y = (byte)1; return x & y");
-        });
-        assertEquals(0, exec("def x = (byte)4; def y = (byte)1; return x & y"));
-        assertEquals(0, exec("def x = (short)4; def y = (byte)1; return x & y"));
-        assertEquals(0, exec("def x = (char)4; def y = (byte)1; return x & y"));
-        assertEquals(0, exec("def x = (int)4; def y = (byte)1; return x & y"));
-        assertEquals(0L, exec("def x = (long)4; def y = (byte)1; return x & y"));
-
-        assertEquals(0, exec("def x = (byte)4; def y = (short)1; return x & y"));
-        assertEquals(0, exec("def x = (short)4; def y = (short)1; return x & y"));
-        assertEquals(0, exec("def x = (char)4; def y = (short)1; return x & y"));
-        assertEquals(0, exec("def x = (int)4; def y = (short)1; return x & y"));
-        assertEquals(0L, exec("def x = (long)4; def y = (short)1; return x & y"));
-
-        assertEquals(0, exec("def x = (byte)4; def y = (char)1; return x & y"));
-        assertEquals(0, exec("def x = (short)4; def y = (char)1; return x & y"));
-        assertEquals(0, exec("def x = (char)4; def y = (char)1; return x & y"));
-        assertEquals(0, exec("def x = (int)4; def y = (char)1; return x & y"));
-        assertEquals(0L, exec("def x = (long)4; def y = (char)1; return x & y"));
-
-        assertEquals(0, exec("def x = (byte)4; def y = (int)1; return x & y"));
-        assertEquals(0, exec("def x = (short)4; def y = (int)1; return x & y"));
-        assertEquals(0, exec("def x = (char)4; def y = (int)1; return x & y"));
-        assertEquals(0, exec("def x = (int)4; def y = (int)1; return x & y"));
-        assertEquals(0L, exec("def x = (long)4; def y = (int)1; return x & y"));
-
-        assertEquals(0L, exec("def x = (byte)4; def y = (long)1; return x & y"));
-        assertEquals(0L, exec("def x = (short)4; def y = (long)1; return x & y"));
-        assertEquals(0L, exec("def x = (char)4; def y = (long)1; return x & y"));
-        assertEquals(0L, exec("def x = (int)4; def y = (long)1; return x & y"));
-        assertEquals(0L, exec("def x = (long)4; def y = (long)1; return x & y"));
-
-        assertEquals(0, exec("def x = (byte)4; def y = (byte)1; return x & y"));
-        assertEquals(0, exec("def x = (short)4; def y = (short)1; return x & y"));
-        assertEquals(0, exec("def x = (char)4; def y = (char)1; return x & y"));
-        assertEquals(0, exec("def x = (int)4; def y = (int)1; return x & y"));
-        assertEquals(0L, exec("def x = (long)4; def y = (long)1; return x & y"));
-
-        assertEquals(true,  exec("def x = true;  def y = true; return x & y"));
-        assertEquals(false, exec("def x = true;  def y = false; return x & y"));
-        assertEquals(false, exec("def x = false; def y = true; return x & y"));
-        assertEquals(false, exec("def x = false; def y = false; return x & y"));
-    }
-    
-    public void testAndTypedLHS() {
-        expectScriptThrows(ClassCastException.class, () -> {
-            exec("float x = (float)4; def y = (byte)1; return x & y");
-        });
-        expectScriptThrows(ClassCastException.class, () -> {
-            exec("double x = (double)4; def y = (byte)1; return x & y");
-        });
-        assertEquals(0, exec("byte x = (byte)4; def y = (byte)1; return x & y"));
-        assertEquals(0, exec("short x = (short)4; def y = (byte)1; return x & y"));
-        assertEquals(0, exec("char x = (char)4; def y = (byte)1; return x & y"));
-        assertEquals(0, exec("int x = (int)4; def y = (byte)1; return x & y"));
-        assertEquals(0L, exec("long x = (long)4; def y = (byte)1; return x & y"));
-
-        assertEquals(0, exec("byte x = (byte)4; def y = (short)1; return x & y"));
-        assertEquals(0, exec("short x = (short)4; def y = (short)1; return x & y"));
-        assertEquals(0, exec("char x = (char)4; def y = (short)1; return x & y"));
-        assertEquals(0, exec("int x = (int)4; def y = (short)1; return x & y"));
-        assertEquals(0L, exec("long x = (long)4; def y = (short)1; return x & y"));
-
-        assertEquals(0, exec("byte x = (byte)4; def y = (char)1; return x & y"));
-        assertEquals(0, exec("short x = (short)4; def y = (char)1; return x & y"));
-        assertEquals(0, exec("char x = (char)4; def y = (char)1; return x & y"));
-        assertEquals(0, exec("int x = (int)4; def y = (char)1; return x & y"));
-        assertEquals(0L, exec("long x = (long)4; def y = (char)1; return x & y"));
-
-        assertEquals(0, exec("byte x = (byte)4; def y = (int)1; return x & y"));
-        assertEquals(0, exec("short x = (short)4; def y = (int)1; return x & y"));
-        assertEquals(0, exec("char x = (char)4; def y = (int)1; return x & y"));
-        assertEquals(0, exec("int x = (int)4; def y = (int)1; return x & y"));
-        assertEquals(0L, exec("long x = (long)4; def y = (int)1; return x & y"));
-
-        assertEquals(0L, exec("byte x = (byte)4; def y = (long)1; return x & y"));
-        assertEquals(0L, exec("short x = (short)4; def y = (long)1; return x & y"));
-        assertEquals(0L, exec("char x = (char)4; def y = (long)1; return x & y"));
-        assertEquals(0L, exec("int x = (int)4; def y = (long)1; return x & y"));
-        assertEquals(0L, exec("long x = (long)4; def y = (long)1; return x & y"));
-
-        assertEquals(0, exec("byte x = (byte)4; def y = (byte)1; return x & y"));
-        assertEquals(0, exec("short x = (short)4; def y = (short)1; return x & y"));
-        assertEquals(0, exec("char x = (char)4; def y = (char)1; return x & y"));
-        assertEquals(0, exec("int x = (int)4; def y = (int)1; return x & y"));
-        assertEquals(0L, exec("long x = (long)4; def y = (long)1; return x & y"));
-
-        assertEquals(true,  exec("boolean x = true;  def y = true; return x & y"));
-        assertEquals(false, exec("boolean x = true;  def y = false; return x & y"));
-        assertEquals(false, exec("boolean x = false; def y = true; return x & y"));
-        assertEquals(false, exec("boolean x = false; def y = false; return x & y"));
-    }
-    
-    public void testAndTypedRHS() {
-        expectScriptThrows(ClassCastException.class, () -> {
-            exec("def x = (float)4; byte y = (byte)1; return x & y");
-        });
-        expectScriptThrows(ClassCastException.class, () -> {
-            exec("def x = (double)4; byte y = (byte)1; return x & y");
-        });
-        assertEquals(0, exec("def x = (byte)4; byte y = (byte)1; return x & y"));
-        assertEquals(0, exec("def x = (short)4; byte y = (byte)1; return x & y"));
-        assertEquals(0, exec("def x = (char)4; byte y = (byte)1; return x & y"));
-        assertEquals(0, exec("def x = (int)4; byte y = (byte)1; return x & y"));
-        assertEquals(0L, exec("def x = (long)4; byte y = (byte)1; return x & y"));
-
-        assertEquals(0, exec("def x = (byte)4; short y = (short)1; return x & y"));
-        assertEquals(0, exec("def x = (short)4; short y = (short)1; return x & y"));
-        assertEquals(0, exec("def x = (char)4; short y = (short)1; return x & y"));
-        assertEquals(0, exec("def x = (int)4; short y = (short)1; return x & y"));
-        assertEquals(0L, exec("def x = (long)4; short y = (short)1; return x & y"));
-
-        assertEquals(0, exec("def x = (byte)4; char y = (char)1; return x & y"));
-        assertEquals(0, exec("def x = (short)4; char y = (char)1; return x & y"));
-        assertEquals(0, exec("def x = (char)4; char y = (char)1; return x & y"));
-        assertEquals(0, exec("def x = (int)4; char y = (char)1; return x & y"));
-        assertEquals(0L, exec("def x = (long)4; char y = (char)1; return x & y"));
-
-        assertEquals(0, exec("def x = (byte)4; int y = (int)1; return x & y"));
-        assertEquals(0, exec("def x = (short)4; int y = (int)1; return x & y"));
-        assertEquals(0, exec("def x = (char)4; int y = (int)1; return x & y"));
-        assertEquals(0, exec("def x = (int)4; int y = (int)1; return x & y"));
-        assertEquals(0L, exec("def x = (long)4; int y = (int)1; return x & y"));
-
-        assertEquals(0L, exec("def x = (byte)4; long y = (long)1; return x & y"));
-        assertEquals(0L, exec("def x = (short)4; long y = (long)1; return x & y"));
-        assertEquals(0L, exec("def x = (char)4; long y = (long)1; return x & y"));
-        assertEquals(0L, exec("def x = (int)4; long y = (long)1; return x & y"));
-        assertEquals(0L, exec("def x = (long)4; long y = (long)1; return x & y"));
-
-        assertEquals(0, exec("def x = (byte)4; byte y = (byte)1; return x & y"));
-        assertEquals(0, exec("def x = (short)4; short y = (short)1; return x & y"));
-        assertEquals(0, exec("def x = (char)4; char y = (char)1; return x & y"));
-        assertEquals(0, exec("def x = (int)4; int y = (int)1; return x & y"));
-        assertEquals(0L, exec("def x = (long)4; long y = (long)1; return x & y"));
-
-        assertEquals(true,  exec("def x = true;  boolean y = true; return x & y"));
-        assertEquals(false, exec("def x = true;  boolean y = false; return x & y"));
-        assertEquals(false, exec("def x = false; boolean y = true; return x & y"));
-        assertEquals(false, exec("def x = false; boolean y = false; return x & y"));
-    }
-
-    public void testXor() {
-        expectScriptThrows(ClassCastException.class, () -> {
-            exec("def x = (float)4; def y = (byte)1; return x ^ y");
-        });
-        expectScriptThrows(ClassCastException.class, () -> {
-            exec("def x = (double)4; def y = (byte)1; return x ^ y");
-        });
-        assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x ^ y"));
-        assertEquals(5, exec("def x = (short)4; def y = (byte)1; return x ^ y"));
-        assertEquals(5, exec("def x = (char)4; def y = (byte)1; return x ^ y"));
-        assertEquals(5, exec("def x = (int)4; def y = (byte)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4; def y = (byte)1; return x ^ y"));
-
-        assertEquals(5, exec("def x = (byte)4; def y = (short)1; return x ^ y"));
-        assertEquals(5, exec("def x = (short)4; def y = (short)1; return x ^ y"));
-        assertEquals(5, exec("def x = (char)4; def y = (short)1; return x ^ y"));
-        assertEquals(5, exec("def x = (int)4; def y = (short)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4; def y = (short)1; return x ^ y"));
-
-        assertEquals(5, exec("def x = (byte)4; def y = (char)1; return x ^ y"));
-        assertEquals(5, exec("def x = (short)4; def y = (char)1; return x ^ y"));
-        assertEquals(5, exec("def x = (char)4; def y = (char)1; return x ^ y"));
-        assertEquals(5, exec("def x = (int)4; def y = (char)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4; def y = (char)1; return x ^ y"));
-
-        assertEquals(5, exec("def x = (byte)4; def y = (int)1; return x ^ y"));
-        assertEquals(5, exec("def x = (short)4; def y = (int)1; return x ^ y"));
-        assertEquals(5, exec("def x = (char)4; def y = (int)1; return x ^ y"));
-        assertEquals(5, exec("def x = (int)4; def y = (int)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4; def y = (int)1; return x ^ y"));
-
-        assertEquals(5L, exec("def x = (byte)4; def y = (long)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (short)4; def y = (long)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (char)4; def y = (long)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (int)4; def y = (long)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x ^ y"));
-
-        assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x ^ y"));
-        assertEquals(5, exec("def x = (short)4; def y = (short)1; return x ^ y"));
-        assertEquals(5, exec("def x = (char)4; def y = (char)1; return x ^ y"));
-        assertEquals(5, exec("def x = (int)4; def y = (int)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x ^ y"));
-        
-        assertEquals(false, exec("def x = true;  def y = true; return x ^ y"));
-        assertEquals(true,  exec("def x = true;  def y = false; return x ^ y"));
-        assertEquals(true,  exec("def x = false; def y = true; return x ^ y"));
-        assertEquals(false, exec("def x = false; def y = false; return x ^ y"));
-    }
-    
-    public void testXorTypedLHS() {
-        expectScriptThrows(ClassCastException.class, () -> {
-            exec("float x = (float)4; def y = (byte)1; return x ^ y");
-        });
-        expectScriptThrows(ClassCastException.class, () -> {
-            exec("double x = (double)4; def y = (byte)1; return x ^ y");
-        });
-        assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x ^ y"));
-        assertEquals(5, exec("def x = (short)4; def y = (byte)1; return x ^ y"));
-        assertEquals(5, exec("def x = (char)4; def y = (byte)1; return x ^ y"));
-        assertEquals(5, exec("def x = (int)4; def y = (byte)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4; def y = (byte)1; return x ^ y"));
-
-        assertEquals(5, exec("def x = (byte)4; def y = (short)1; return x ^ y"));
-        assertEquals(5, exec("def x = (short)4; def y = (short)1; return x ^ y"));
-        assertEquals(5, exec("def x = (char)4; def y = (short)1; return x ^ y"));
-        assertEquals(5, exec("def x = (int)4; def y = (short)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4; def y = (short)1; return x ^ y"));
-
-        assertEquals(5, exec("def x = (byte)4; def y = (char)1; return x ^ y"));
-        assertEquals(5, exec("def x = (short)4; def y = (char)1; return x ^ y"));
-        assertEquals(5, exec("def x = (char)4; def y = (char)1; return x ^ y"));
-        assertEquals(5, exec("def x = (int)4; def y = (char)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4; def y = (char)1; return x ^ y"));
-
-        assertEquals(5, exec("def x = (byte)4; def y = (int)1; return x ^ y"));
-        assertEquals(5, exec("def x = (short)4; def y = (int)1; return x ^ y"));
-        assertEquals(5, exec("def x = (char)4; def y = (int)1; return x ^ y"));
-        assertEquals(5, exec("def x = (int)4; def y = (int)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4; def y = (int)1; return x ^ y"));
-
-        assertEquals(5L, exec("def x = (byte)4; def y = (long)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (short)4; def y = (long)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (char)4; def y = (long)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (int)4; def y = (long)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x ^ y"));
-
-        assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x ^ y"));
-        assertEquals(5, exec("def x = (short)4; def y = (short)1; return x ^ y"));
-        assertEquals(5, exec("def x = (char)4; def y = (char)1; return x ^ y"));
-        assertEquals(5, exec("def x = (int)4; def y = (int)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x ^ y"));
-        
-        assertEquals(false, exec("def x = true;  def y = true; return x ^ y"));
-        assertEquals(true,  exec("def x = true;  def y = false; return x ^ y"));
-        assertEquals(true,  exec("def x = false; def y = true; return x ^ y"));
-        assertEquals(false, exec("def x = false; def y = false; return x ^ y"));
-    }
-    
-    public void testXorTypedRHS() {
-        expectScriptThrows(ClassCastException.class, () -> {
-            exec("def x = (float)4; byte y = (byte)1; return x ^ y");
-        });
-        expectScriptThrows(ClassCastException.class, () -> {
-            exec("def x = (double)4; byte y = (byte)1; return x ^ y");
-        });
-        assertEquals(5, exec("def x = (byte)4; byte y = (byte)1; return x ^ y"));
-        assertEquals(5, exec("def x = (short)4; byte y = (byte)1; return x ^ y"));
-        assertEquals(5, exec("def x = (char)4; byte y = (byte)1; return x ^ y"));
-        assertEquals(5, exec("def x = (int)4; byte y = (byte)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4; byte y = (byte)1; return x ^ y"));
-
-        assertEquals(5, exec("def x = (byte)4; short y = (short)1; return x ^ y"));
-        assertEquals(5, exec("def x = (short)4; short y = (short)1; return x ^ y"));
-        assertEquals(5, exec("def x = (char)4; short y = (short)1; return x ^ y"));
-        assertEquals(5, exec("def x = (int)4; short y = (short)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4; short y = (short)1; return x ^ y"));
-
-        assertEquals(5, exec("def x = (byte)4; char y = (char)1; return x ^ y"));
-        assertEquals(5, exec("def x = (short)4; char y = (char)1; return x ^ y"));
-        assertEquals(5, exec("def x = (char)4; char y = (char)1; return x ^ y"));
-        assertEquals(5, exec("def x = (int)4; char y = (char)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4; char y = (char)1; return x ^ y"));
-
-        assertEquals(5, exec("def x = (byte)4; int y = (int)1; return x ^ y"));
-        assertEquals(5, exec("def x = (short)4; int y = (int)1; return x ^ y"));
-        assertEquals(5, exec("def x = (char)4; int y = (int)1; return x ^ y"));
-        assertEquals(5, exec("def x = (int)4; int y = (int)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4; int y = (int)1; return x ^ y"));
-
-        assertEquals(5L, exec("def x = (byte)4; long y = (long)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (short)4; long y = (long)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (char)4; long y = (long)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (int)4; long y = (long)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4; long y = (long)1; return x ^ y"));
-
-        assertEquals(5, exec("def x = (byte)4; byte y = (byte)1; return x ^ y"));
-        assertEquals(5, exec("def x = (short)4; short y = (short)1; return x ^ y"));
-        assertEquals(5, exec("def x = (char)4; char y = (char)1; return x ^ y"));
-        assertEquals(5, exec("def x = (int)4; int y = (int)1; return x ^ y"));
-        assertEquals(5L, exec("def x = (long)4; long y = (long)1; return x ^ y"));
-        
-        assertEquals(false, exec("def x = true;  boolean y = true; return x ^ y"));
-        assertEquals(true,  exec("def x = true;  boolean y = false; return x ^ y"));
-        assertEquals(true,  exec("def x = false; boolean y = true; return x ^ y"));
-        assertEquals(false, exec("def x = false; boolean y = false; return x ^ y"));
-    }
-
-    public void testOr() {
-        expectScriptThrows(ClassCastException.class, () -> {
-            exec("def x = (float)4; def y = (byte)1; return x | y");
-        });
-        expectScriptThrows(ClassCastException.class, () -> {
-            exec("def x = (double)4; def y = (byte)1; return x | y");
-        });
-        assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x | y"));
-        assertEquals(5, exec("def x = (short)4; def y = (byte)1; return x | y"));
-        assertEquals(5, exec("def x = (char)4; def y = (byte)1; return x | y"));
-        assertEquals(5, exec("def x = (int)4; def y = (byte)1; return x | y"));
-        assertEquals(5L, exec("def x = (long)4; def y = (byte)1; return x | y"));
-
-        assertEquals(5, exec("def x = (byte)4; def y = (short)1; return x | y"));
-        assertEquals(5, exec("def x = (short)4; def y = (short)1; return x | y"));
-        assertEquals(5, exec("def x = (char)4; def y = (short)1; return x | y"));
-        assertEquals(5, exec("def x = (int)4; def y = (short)1; return x | y"));
-        assertEquals(5L, exec("def x = (long)4; def y = (short)1; return x | y"));
-
-        assertEquals(5, exec("def x = (byte)4; def y = (char)1; return x | y"));
-        assertEquals(5, exec("def x = (short)4; def y = (char)1; return x | y"));
-        assertEquals(5, exec("def x = (char)4; def y = (char)1; return x | y"));
-        assertEquals(5, exec("def x = (int)4; def y = (char)1; return x | y"));
-        assertEquals(5L, exec("def x = (long)4; def y = (char)1; return x | y"));
-
-        assertEquals(5, exec("def x = (byte)4; def y = (int)1; return x | y"));
-        assertEquals(5, exec("def x = (short)4; def y = (int)1; return x | y"));
-        assertEquals(5, exec("def x = (char)4; def y = (int)1; return x | y"));
-        assertEquals(5, exec("def x = (int)4; def y = (int)1; return x | y"));
-        assertEquals(5L, exec("def x = (long)4; def y = (int)1; return x | y"));
-
-        assertEquals(5L, exec("def x = (byte)4; def y = (long)1; return x | y"));
-        assertEquals(5L, exec("def x = (short)4; def y = (long)1; return x | y"));
-        assertEquals(5L, exec("def x = (char)4; def y = (long)1; return x | y"));
-        assertEquals(5L, exec("def x = (int)4; def y = (long)1; return x | y"));
-        assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x | y"));
-
-        assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x | y"));
-        assertEquals(5, exec("def x = (short)4; def y = (short)1; return x | y"));
-        assertEquals(5, exec("def x = (char)4; def y = (char)1; return x | y"));
-        assertEquals(5, exec("def x = (int)4; def y = (int)1; return x | y"));
-        assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x | y"));
-        
-        assertEquals(true,  exec("def x = true;  def y = true; return x | y"));
-        assertEquals(true,  exec("def x = true;  def y = false; return x | y"));
-        assertEquals(true,  exec("def x = false; def y = true; return x | y"));
-        assertEquals(false, exec("def x = false; def y = false; return x | y"));
-    }
-    
-    public void testOrTypedLHS() {
-        expectScriptThrows(ClassCastException.class, () -> {
-            exec("float x = (float)4; def y = (byte)1; return x | y");
-        });
-        expectScriptThrows(ClassCastException.class, () -> {
-            exec("double x = (double)4; def y = (byte)1; return x | y");
-        });
-        assertEquals(5, exec("byte x = (byte)4; def y = (byte)1; return x | y"));
-        assertEquals(5, exec("short x = (short)4; def y = (byte)1; return x | y"));
-        assertEquals(5, exec("char x = (char)4; def y = (byte)1; return x | y"));
-        assertEquals(5, exec("int x = (int)4; def y = (byte)1; return x | y"));
-        assertEquals(5L, exec("long x = (long)4; def y = (byte)1; return x | y"));
-
-        assertEquals(5, exec("byte x = (byte)4; def y = (short)1; return x | y"));
-        assertEquals(5, exec("short x = (short)4; def y = (short)1; return x | y"));
-        assertEquals(5, exec("char x = (char)4; def y = (short)1; return x | y"));
-        assertEquals(5, exec("int x = (int)4; def y = (short)1; return x | y"));
-        assertEquals(5L, exec("long x = (long)4; def y = (short)1; return x | y"));
-
-        assertEquals(5, exec("byte x = (byte)4; def y = (char)1; return x | y"));
-        assertEquals(5, exec("short x = (short)4; def y = (char)1; return x | y"));
-        assertEquals(5, exec("char x = (char)4; def y = (char)1; return x | y"));
-        assertEquals(5, exec("int x = (int)4; def y = (char)1; return x | y"));
-        assertEquals(5L, exec("long x = (long)4; def y = (char)1; return x | y"));
-
-        assertEquals(5, exec("byte x = (byte)4; def y = (int)1; return x | y"));
-        assertEquals(5, exec("short x = (short)4; def y = (int)1; return x | y"));
-        assertEquals(5, exec("char x = (char)4; def y = (int)1; return x | y"));
-        assertEquals(5, exec("int x = (int)4; def y = (int)1; return x | y"));
-        assertEquals(5L, exec("long x = (long)4; def y = (int)1; return x | y"));
-
-        assertEquals(5L, exec("byte x = (byte)4; def y = (long)1; return x | y"));
-        assertEquals(5L, exec("short x = (short)4; def y = (long)1; return x | y"));
-        assertEquals(5L, exec("char x = (char)4; def y = (long)1; return x | y"));
-        assertEquals(5L, exec("int x = (int)4; def y = (long)1; return x | y"));
-        assertEquals(5L, exec("long x = (long)4; def y = (long)1; return x | y"));
-
-        assertEquals(5, exec("byte x = (byte)4; def y = (byte)1; return x | y"));
-        assertEquals(5, exec("short x = (short)4; def y = (short)1; return x | y"));
-        assertEquals(5, exec("char x = (char)4; def y = (char)1; return x | y"));
-        assertEquals(5, exec("int x = (int)4; def y = (int)1; return x | y"));
-        assertEquals(5L, exec("long x = (long)4; def y = (long)1; return x | y"));
-        
-        assertEquals(true,  exec("boolean x = true;  def y = true; return x | y"));
-        assertEquals(true,  exec("boolean x = true;  def y = false; return x | y"));
-        assertEquals(true,  exec("boolean x = false; def y = true; return x | y"));
-        assertEquals(false, exec("boolean x = false; def y = false; return x | y"));
-    }
-    
-    public void testOrTypedRHS() {
-        expectScriptThrows(ClassCastException.class, () -> {
-            exec("def x = (float)4; byte y = (byte)1; return x | y");
-        });
-        expectScriptThrows(ClassCastException.class, () -> {
-            exec("def x = (double)4; byte y = (byte)1; return x | y");
-        });
-        assertEquals(5, exec("def x = (byte)4; byte y = (byte)1; return x | y"));
-        assertEquals(5, exec("def x = (short)4; byte y = (byte)1; return x | y"));
-        assertEquals(5, exec("def x = (char)4; byte y = (byte)1; return x | y"));
-        assertEquals(5, exec("def x = (int)4; byte y = (byte)1; return x | y"));
-        assertEquals(5L, exec("def x = (long)4; byte y = (byte)1; return x | y"));
-
-        assertEquals(5, exec("def x = (byte)4; short y = (short)1; return x | y"));
-        assertEquals(5, exec("def x = (short)4; short y = (short)1; return x | y"));
-        assertEquals(5, exec("def x = (char)4; short y = (short)1; return x | y"));
-        assertEquals(5, exec("def x = (int)4; short y = (short)1; return x | y"));
-        assertEquals(5L, exec("def x = (long)4; short y = (short)1; return x | y"));
-
-        assertEquals(5, exec("def x = (byte)4; char y = (char)1; return x | y"));
-        assertEquals(5, exec("def x = (short)4; char y = (char)1; return x | y"));
-        assertEquals(5, exec("def x = (char)4; char y = (char)1; return x | y"));
-        assertEquals(5, exec("def x = (int)4; char y = (char)1; return x | y"));
-        assertEquals(5L, exec("def x = (long)4; char y = (char)1; return x | y"));
-
-        assertEquals(5, exec("def x = (byte)4; int y = (int)1; return x | y"));
-        assertEquals(5, exec("def x = (short)4; int y = (int)1; return x | y"));
-        assertEquals(5, exec("def x = (char)4; int y = (int)1; return x | y"));
-        assertEquals(5, exec("def x = (int)4; int y = (int)1; return x | y"));
-        assertEquals(5L, exec("def x = (long)4; int y = (int)1; return x | y"));
-
-        assertEquals(5L, exec("def x = (byte)4; long y = (long)1; return x | y"));
-        assertEquals(5L, exec("def x = (short)4; long y = (long)1; return x | y"));
-        assertEquals(5L, exec("def x = (char)4; long y = (long)1; return x | y"));
-        assertEquals(5L, exec("def x = (int)4; long y = (long)1; return x | y"));
-        assertEquals(5L, exec("def x = (long)4; long y = (long)1; return x | y"));
-
-        assertEquals(5, exec("def x = (byte)4; byte y = (byte)1; return x | y"));
-        assertEquals(5, exec("def x = (short)4; short y = (short)1; return x | y"));
-        assertEquals(5, exec("def x = (char)4; char y = (char)1; return x | y"));
-        assertEquals(5, exec("def x = (int)4; int y = (int)1; return x | y"));
-        assertEquals(5L, exec("def x = (long)4; long y = (long)1; return x | y"));
-        
-        assertEquals(true,  exec("def x = true;  boolean y = true; return x | y"));
-        assertEquals(true,  exec("def x = true;  boolean y = false; return x | y"));
-        assertEquals(true,  exec("def x = false; boolean y = true; return x | y"));
-        assertEquals(false, exec("def x = false; boolean y = false; return x | y"));
-    }
-
-    public void testEq() {
-        assertEquals(true, exec("def x = (byte)7; def y = (int)7; return x == y"));
-        assertEquals(true, exec("def x = (short)6; def y = (int)6; return x == y"));
-        assertEquals(true, exec("def x = (char)5; def y = (int)5; return x == y"));
-        assertEquals(true, exec("def x = (int)4; def y = (int)4; return x == y"));
-        assertEquals(false, exec("def x = (long)5; def y = (int)3; return x == y"));
-        assertEquals(false, exec("def x = (float)6; def y = (int)2; return x == y"));
-        assertEquals(false, exec("def x = (double)7; def y = (int)1; return x == y"));
-
-        assertEquals(true, exec("def x = (byte)7; def y = (double)7; return x == y"));
-        assertEquals(true, exec("def x = (short)6; def y = (double)6; return x == y"));
-        assertEquals(true, exec("def x = (char)5; def y = (double)5; return x == y"));
-        assertEquals(true, exec("def x = (int)4; def y = (double)4; return x == y"));
-        assertEquals(false, exec("def x = (long)5; def y = (double)3; return x == y"));
-        assertEquals(false, exec("def x = (float)6; def y = (double)2; return x == y"));
-        assertEquals(false, exec("def x = (double)7; def y = (double)1; return x == y"));
-        
-        assertEquals(false, exec("def x = false; def y = true; return x == y"));
-        assertEquals(false, exec("def x = true; def y = false; return x == y"));
-        assertEquals(false, exec("def x = true; def y = null; return x == y"));
-        assertEquals(false, exec("def x = null; def y = true; return x == y"));
-        assertEquals(true, exec("def x = true; def y = true; return x == y"));
-        assertEquals(true, exec("def x = false; def y = false; return x == y"));
-
-        assertEquals(true, exec("def x = new HashMap(); def y = new HashMap(); return x == y"));
-        assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x == y"));
-        assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x == y"));
-        assertEquals(true, exec("def x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x == y"));
-    }
-    
-    public void testEqTypedLHS() {
-        assertEquals(true, exec("byte x = (byte)7; def y = (int)7; return x == y"));
-        assertEquals(true, exec("short x = (short)6; def y = (int)6; return x == y"));
-        assertEquals(true, exec("char x = (char)5; def y = (int)5; return x == y"));
-        assertEquals(true, exec("int x = (int)4; def y = (int)4; return x == y"));
-        assertEquals(false, exec("long x = (long)5; def y = (int)3; return x == y"));
-        assertEquals(false, exec("float x = (float)6; def y = (int)2; return x == y"));
-        assertEquals(false, exec("double x = (double)7; def y = (int)1; return x == y"));
-
-        assertEquals(true, exec("byte x = (byte)7; def y = (double)7; return x == y"));
-        assertEquals(true, exec("short x = (short)6; def y = (double)6; return x == y"));
-        assertEquals(true, exec("char x = (char)5; def y = (double)5; return x == y"));
-        assertEquals(true, exec("int x = (int)4; def y = (double)4; return x == y"));
-        assertEquals(false, exec("long x = (long)5; def y = (double)3; return x == y"));
-        assertEquals(false, exec("float x = (float)6; def y = (double)2; return x == y"));
-        assertEquals(false, exec("double x = (double)7; def y = (double)1; return x == y"));
-        
-        assertEquals(false, exec("boolean x = false; def y = true; return x == y"));
-        assertEquals(false, exec("boolean x = true; def y = false; return x == y"));
-        assertEquals(false, exec("boolean x = true; def y = null; return x == y"));
-        assertEquals(true, exec("boolean x = true; def y = true; return x == y"));
-        assertEquals(true, exec("boolean x = false; def y = false; return x == y"));
-
-        assertEquals(true, exec("Map x = new HashMap(); def y = new HashMap(); return x == y"));
-        assertEquals(false, exec("Map x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x == y"));
-        assertEquals(true, exec("Map x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x == y"));
-        assertEquals(true, exec("Map x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x == y"));
-    }
-    
-    public void testEqTypedRHS() {
-        assertEquals(true, exec("def x = (byte)7; int y = (int)7; return x == y"));
-        assertEquals(true, exec("def x = (short)6; int y = (int)6; return x == y"));
-        assertEquals(true, exec("def x = (char)5; int y = (int)5; return x == y"));
-        assertEquals(true, exec("def x = (int)4; int y = (int)4; return x == y"));
-        assertEquals(false, exec("def x = (long)5; int y = (int)3; return x == y"));
-        assertEquals(false, exec("def x = (float)6; int y = (int)2; return x == y"));
-        assertEquals(false, exec("def x = (double)7; int y = (int)1; return x == y"));
-
-        assertEquals(true, exec("def x = (byte)7; double y = (double)7; return x == y"));
-        assertEquals(true, exec("def x = (short)6; double y = (double)6; return x == y"));
-        assertEquals(true, exec("def x = (char)5; double y = (double)5; return x == y"));
-        assertEquals(true, exec("def x = (int)4; double y = (double)4; return x == y"));
-        assertEquals(false, exec("def x = (long)5; double y = (double)3; return x == y"));
-        assertEquals(false, exec("def x = (float)6; double y = (double)2; return x == y"));
-        assertEquals(false, exec("def x = (double)7; double y = (double)1; return x == y"));
-        
-        assertEquals(false, exec("def x = false; boolean y = true; return x == y"));
-        assertEquals(false, exec("def x = true; boolean y = false; return x == y"));
-        assertEquals(false, exec("def x = null; boolean y = true; return x == y"));
-        assertEquals(true, exec("def x = true; boolean y = true; return x == y"));
-        assertEquals(true, exec("def x = false; boolean y = false; return x == y"));
-
-        assertEquals(true, exec("def x = new HashMap(); Map y = new HashMap(); return x == y"));
-        assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); Map y = new HashMap(); return x == y"));
-        assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); Map y = new HashMap(); y.put(3, 3); return x == y"));
-        assertEquals(true, exec("def x = new HashMap(); Map y = x; x.put(3, 3); y.put(3, 3); return x == y"));
-    }
-
-    public void testEqr() {
-        assertEquals(false, exec("def x = (byte)7; def y = (int)7; return x === y"));
-        assertEquals(false, exec("def x = (short)6; def y = (int)6; return x === y"));
-        assertEquals(false, exec("def x = (char)5; def y = (int)5; return x === y"));
-        assertEquals(true, exec("def x = (int)4; def y = (int)4; return x === y"));
-        assertEquals(false, exec("def x = (long)5; def y = (int)3; return x === y"));
-        assertEquals(false, exec("def x = (float)6; def y = (int)2; return x === y"));
-        assertEquals(false, exec("def x = (double)7; def y = (int)1; return x === y"));
-        assertEquals(false, exec("def x = false; def y = true; return x === y"));
-
-        assertEquals(false, exec("def x = new HashMap(); def y = new HashMap(); return x === y"));
-        assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x === y"));
-        assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x === y"));
-        assertEquals(true, exec("def x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x === y"));
-    }
-
-    public void testNe() {
-        assertEquals(false, exec("def x = (byte)7; def y = (int)7; return x != y"));
-        assertEquals(false, exec("def x = (short)6; def y = (int)6; return x != y"));
-        assertEquals(false, exec("def x = (char)5; def y = (int)5; return x != y"));
-        assertEquals(false, exec("def x = (int)4; def y = (int)4; return x != y"));
-        assertEquals(true, exec("def x = (long)5; def y = (int)3; return x != y"));
-        assertEquals(true, exec("def x = (float)6; def y = (int)2; return x != y"));
-        assertEquals(true, exec("def x = (double)7; def y = (int)1; return x != y"));
-
-        assertEquals(false, exec("def x = (byte)7; def y = (double)7; return x != y"));
-        assertEquals(false, exec("def x = (short)6; def y = (double)6; return x != y"));
-        assertEquals(false, exec("def x = (char)5; def y = (double)5; return x != y"));
-        assertEquals(false, exec("def x = (int)4; def y = (double)4; return x != y"));
-        assertEquals(true, exec("def x = (long)5; def y = (double)3; return x != y"));
-        assertEquals(true, exec("def x = (float)6; def y = (double)2; return x != y"));
-        assertEquals(true, exec("def x = (double)7; def y = (double)1; return x != y"));
-
-        assertEquals(false, exec("def x = new HashMap(); def y = new HashMap(); return x != y"));
-        assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x != y"));
-        assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x != y"));
-        assertEquals(false, exec("def x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x != y"));
-        
-        assertEquals(false,  exec("def x = true;  def y = true; return x != y"));
-        assertEquals(true,   exec("def x = true;  def y = false; return x != y"));
-        assertEquals(true,   exec("def x = false; def y = true; return x != y"));
-        assertEquals(false,  exec("def x = false; def y = false; return x != y"));
-    }
-    
-    public void testNeTypedLHS() {
-        assertEquals(false, exec("byte x = (byte)7; def y = (int)7; return x != y"));
-        assertEquals(false, exec("short x = (short)6; def y = (int)6; return x != y"));
-        assertEquals(false, exec("char x = (char)5; def y = (int)5; return x != y"));
-        assertEquals(false, exec("int x = (int)4; def y = (int)4; return x != y"));
-        assertEquals(true, exec("long x = (long)5; def y = (int)3; return x != y"));
-        assertEquals(true, exec("float x = (float)6; def y = (int)2; return x != y"));
-        assertEquals(true, exec("double x = (double)7; def y = (int)1; return x != y"));
-
-        assertEquals(false, exec("byte x = (byte)7; def y = (double)7; return x != y"));
-        assertEquals(false, exec("short x = (short)6; def y = (double)6; return x != y"));
-        assertEquals(false, exec("char x = (char)5; def y = (double)5; return x != y"));
-        assertEquals(false, exec("int x = (int)4; def y = (double)4; return x != y"));
-        assertEquals(true, exec("long x = (long)5; def y = (double)3; return x != y"));
-        assertEquals(true, exec("float x = (float)6; def y = (double)2; return x != y"));
-        assertEquals(true, exec("double x = (double)7; def y = (double)1; return x != y"));
-
-        assertEquals(false, exec("Map x = new HashMap(); def y = new HashMap(); return x != y"));
-        assertEquals(true, exec("Map x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x != y"));
-        assertEquals(false, exec("Map x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x != y"));
-        assertEquals(false, exec("Map x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x != y"));
-        
-        assertEquals(false,  exec("boolean x = true;  def y = true; return x != y"));
-        assertEquals(true,   exec("boolean x = true;  def y = false; return x != y"));
-        assertEquals(true,   exec("boolean x = false; def y = true; return x != y"));
-        assertEquals(false,  exec("boolean x = false; def y = false; return x != y"));
-    }
-    
-    public void testNeTypedRHS() {
-        assertEquals(false, exec("def x = (byte)7; int y = (int)7; return x != y"));
-        assertEquals(false, exec("def x = (short)6; int y = (int)6; return x != y"));
-        assertEquals(false, exec("def x = (char)5; int y = (int)5; return x != y"));
-        assertEquals(false, exec("def x = (int)4; int y = (int)4; return x != y"));
-        assertEquals(true, exec("def x = (long)5; int y = (int)3; return x != y"));
-        assertEquals(true, exec("def x = (float)6; int y = (int)2; return x != y"));
-        assertEquals(true, exec("def x = (double)7; int y = (int)1; return x != y"));
-
-        assertEquals(false, exec("def x = (byte)7; double y = (double)7; return x != y"));
-        assertEquals(false, exec("def x = (short)6; double y = (double)6; return x != y"));
-        assertEquals(false, exec("def x = (char)5; double y = (double)5; return x != y"));
-        assertEquals(false, exec("def x = (int)4; double y = (double)4; return x != y"));
-        assertEquals(true, exec("def x = (long)5; double y = (double)3; return x != y"));
-        assertEquals(true, exec("def x = (float)6; double y = (double)2; return x != y"));
-        assertEquals(true, exec("def x = (double)7; double y = (double)1; return x != y"));
-
-        assertEquals(false, exec("def x = new HashMap(); Map y = new HashMap(); return x != y"));
-        assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); Map y = new HashMap(); return x != y"));
-        assertEquals(false, exec("def x = new HashMap(); x.put(3, 3); Map y = new HashMap(); y.put(3, 3); return x != y"));
-        assertEquals(false, exec("def x = new HashMap(); Map y = x; x.put(3, 3); y.put(3, 3); return x != y"));
-        
-        assertEquals(false,  exec("def x = true;  boolean y = true; return x != y"));
-        assertEquals(true,   exec("def x = true;  boolean y = false; return x != y"));
-        assertEquals(true,   exec("def x = false; boolean y = true; return x != y"));
-        assertEquals(false,  exec("def x = false; boolean y = false; return x != y"));
-    }
-
-    public void testNer() {
-        assertEquals(true, exec("def x = (byte)7; def y = (int)7; return x !== y"));
-        assertEquals(true, exec("def x = (short)6; def y = (int)6; return x !== y"));
-        assertEquals(true, exec("def x = (char)5; def y = (int)5; return x !== y"));
-        assertEquals(false, exec("def x = (int)4; def y = (int)4; return x !== y"));
-        assertEquals(true, exec("def x = (long)5; def y = (int)3; return x !== y"));
-        assertEquals(true, exec("def x = (float)6; def y = (int)2; return x !== y"));
-        assertEquals(true, exec("def x = (double)7; def y = (int)1; return x !== y"));
-
-        assertEquals(true, exec("def x = new HashMap(); def y = new HashMap(); return x !== y"));
-        assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); return x !== y"));
-        assertEquals(true, exec("def x = new HashMap(); x.put(3, 3); def y = new HashMap(); y.put(3, 3); return x !== y"));
-        assertEquals(false, exec("def x = new HashMap(); def y = x; x.put(3, 3); y.put(3, 3); return x !== y"));
-    }
-
-    public void testLt() {
-        assertEquals(true, exec("def x = (byte)1; def y = (int)7; return x < y"));
-        assertEquals(true, exec("def x = (short)2; def y = (int)6; return x < y"));
-        assertEquals(true, exec("def x = (char)3; def y = (int)5; return x < y"));
-        assertEquals(false, exec("def x = (int)4; def y = (int)4; return x < y"));
-        assertEquals(false, exec("def x = (long)5; def y = (int)3; return x < y"));
-        assertEquals(false, exec("def x = (float)6; def y = (int)2; return x < y"));
-        assertEquals(false, exec("def x = (double)7; def y = (int)1; return x < y"));
-
-        assertEquals(true, exec("def x = (byte)1; def y = (double)7; return x < y"));
-        assertEquals(true, exec("def x = (short)2; def y = (double)6; return x < y"));
-        assertEquals(true, exec("def x = (char)3; def y = (double)5; return x < y"));
-        assertEquals(false, exec("def x = (int)4; def y = (double)4; return x < y"));
-        assertEquals(false, exec("def x = (long)5; def y = (double)3; return x < y"));
-        assertEquals(false, exec("def x = (float)6; def y = (double)2; return x < y"));
-        assertEquals(false, exec("def x = (double)7; def y = (double)1; return x < y"));
-    }
-    
-    public void testLtTypedLHS() {
-        assertEquals(true, exec("byte x = (byte)1; def y = (int)7; return x < y"));
-        assertEquals(true, exec("short x = (short)2; def y = (int)6; return x < y"));
-        assertEquals(true, exec("char x = (char)3; def y = (int)5; return x < y"));
-        assertEquals(false, exec("int x = (int)4; def y = (int)4; return x < y"));
-        assertEquals(false, exec("long x = (long)5; def y = (int)3; return x < y"));
-        assertEquals(false, exec("float x = (float)6; def y = (int)2; return x < y"));
-        assertEquals(false, exec("double x = (double)7; def y = (int)1; return x < y"));
-
-        assertEquals(true, exec("byte x = (byte)1; def y = (double)7; return x < y"));
-        assertEquals(true, exec("short x = (short)2; def y = (double)6; return x < y"));
-        assertEquals(true, exec("char x = (char)3; def y = (double)5; return x < y"));
-        assertEquals(false, exec("int x = (int)4; def y = (double)4; return x < y"));
-        assertEquals(false, exec("long x = (long)5; def y = (double)3; return x < y"));
-        assertEquals(false, exec("float x = (float)6; def y = (double)2; return x < y"));
-        assertEquals(false, exec("double x = (double)7; def y = (double)1; return x < y"));
-    }
-    
-    public void testLtTypedRHS() {
-        assertEquals(true, exec("def x = (byte)1; int y = (int)7; return x < y"));
-        assertEquals(true, exec("def x = (short)2; int y = (int)6; return x < y"));
-        assertEquals(true, exec("def x = (char)3; int y = (int)5; return x < y"));
-        assertEquals(false, exec("def x = (int)4; int y = (int)4; return x < y"));
-        assertEquals(false, exec("def x = (long)5; int y = (int)3; return x < y"));
-        assertEquals(false, exec("def x = (float)6; int y = (int)2; return x < y"));
-        assertEquals(false, exec("def x = (double)7; int y = (int)1; return x < y"));
-
-        assertEquals(true, exec("def x = (byte)1; double y = (double)7; return x < y"));
-        assertEquals(true, exec("def x = (short)2; double y = (double)6; return x < y"));
-        assertEquals(true, exec("def x = (char)3; double y = (double)5; return x < y"));
-        assertEquals(false, exec("def x = (int)4; double y = (double)4; return x < y"));
-        assertEquals(false, exec("def x = (long)5; double y = (double)3; return x < y"));
-        assertEquals(false, exec("def x = (float)6; double y = (double)2; return x < y"));
-        assertEquals(false, exec("def x = (double)7; double y = (double)1; return x < y"));
-    }
-
-    public void testLte() {
-        assertEquals(true, exec("def x = (byte)1; def y = (int)7; return x <= y"));
-        assertEquals(true, exec("def x = (short)2; def y = (int)6; return x <= y"));
-        assertEquals(true, exec("def x = (char)3; def y = (int)5; return x <= y"));
-        assertEquals(true, exec("def x = (int)4; def y = (int)4; return x <= y"));
-        assertEquals(false, exec("def x = (long)5; def y = (int)3; return x <= y"));
-        assertEquals(false, exec("def x = (float)6; def y = (int)2; return x <= y"));
-        assertEquals(false, exec("def x = (double)7; def y = (int)1; return x <= y"));
-
-        assertEquals(true, exec("def x = (byte)1; def y = (double)7; return x <= y"));
-        assertEquals(true, exec("def x = (short)2; def y = (double)6; return x <= y"));
-        assertEquals(true, exec("def x = (char)3; def y = (double)5; return x <= y"));
-        assertEquals(true, exec("def x = (int)4; def y = (double)4; return x <= y"));
-        assertEquals(false, exec("def x = (long)5; def y = (double)3; return x <= y"));
-        assertEquals(false, exec("def x = (float)6; def y = (double)2; return x <= y"));
-        assertEquals(false, exec("def x = (double)7; def y = (double)1; return x <= y"));
-    }
-    
-    public void testLteTypedLHS() {
-        assertEquals(true, exec("byte x = (byte)1; def y = (int)7; return x <= y"));
-        assertEquals(true, exec("short x = (short)2; def y = (int)6; return x <= y"));
-        assertEquals(true, exec("char x = (char)3; def y = (int)5; return x <= y"));
-        assertEquals(true, exec("int x = (int)4; def y = (int)4; return x <= y"));
-        assertEquals(false, exec("long x = (long)5; def y = (int)3; return x <= y"));
-        assertEquals(false, exec("float x = (float)6; def y = (int)2; return x <= y"));
-        assertEquals(false, exec("double x = (double)7; def y = (int)1; return x <= y"));
-
-        assertEquals(true, exec("byte x = (byte)1; def y = (double)7; return x <= y"));
-        assertEquals(true, exec("short x = (short)2; def y = (double)6; return x <= y"));
-        assertEquals(true, exec("char x = (char)3; def y = (double)5; return x <= y"));
-        assertEquals(true, exec("int x = (int)4; def y = (double)4; return x <= y"));
-        assertEquals(false, exec("long x = (long)5; def y = (double)3; return x <= y"));
-        assertEquals(false, exec("float x = (float)6; def y = (double)2; return x <= y"));
-        assertEquals(false, exec("double x = (double)7; def y = (double)1; return x <= y"));
-    }
-    
-    public void testLteTypedRHS() {
-        assertEquals(true, exec("def x = (byte)1; int y = (int)7; return x <= y"));
-        assertEquals(true, exec("def x = (short)2; int y = (int)6; return x <= y"));
-        assertEquals(true, exec("def x = (char)3; int y = (int)5; return x <= y"));
-        assertEquals(true, exec("def x = (int)4; int y = (int)4; return x <= y"));
-        assertEquals(false, exec("def x = (long)5; int y = (int)3; return x <= y"));
-        assertEquals(false, exec("def x = (float)6; int y = (int)2; return x <= y"));
-        assertEquals(false, exec("def x = (double)7; int y = (int)1; return x <= y"));
-
-        assertEquals(true, exec("def x = (byte)1; double y = (double)7; return x <= y"));
-        assertEquals(true, exec("def x = (short)2; double y = (double)6; return x <= y"));
-        assertEquals(true, exec("def x = (char)3; double y = (double)5; return x <= y"));
-        assertEquals(true, exec("def x = (int)4; double y = (double)4; return x <= y"));
-        assertEquals(false, exec("def x = (long)5; double y = (double)3; return x <= y"));
-        assertEquals(false, exec("def x = (float)6; double y = (double)2; return x <= y"));
-        assertEquals(false, exec("def x = (double)7; double y = (double)1; return x <= y"));
-    }
-
-    public void testGt() {
-        assertEquals(false, exec("def x = (byte)1; def y = (int)7; return x > y"));
-        assertEquals(false, exec("def x = (short)2; def y = (int)6; return x > y"));
-        assertEquals(false, exec("def x = (char)3; def y = (int)5; return x > y"));
-        assertEquals(false, exec("def x = (int)4; def y = (int)4; return x > y"));
-        assertEquals(true, exec("def x = (long)5; def y = (int)3; return x > y"));
-        assertEquals(true, exec("def x = (float)6; def y = (int)2; return x > y"));
-        assertEquals(true, exec("def x = (double)7; def y = (int)1; return x > y"));
-
-        assertEquals(false, exec("def x = (byte)1; def y = (double)7; return x > y"));
-        assertEquals(false, exec("def x = (short)2; def y = (double)6; return x > y"));
-        assertEquals(false, exec("def x = (char)3; def y = (double)5; return x > y"));
-        assertEquals(false, exec("def x = (int)4; def y = (double)4; return x > y"));
-        assertEquals(true, exec("def x = (long)5; def y = (double)3; return x > y"));
-        assertEquals(true, exec("def x = (float)6; def y = (double)2; return x > y"));
-        assertEquals(true, exec("def x = (double)7; def y = (double)1; return x > y"));
-    }
-    
-    public void testGtTypedLHS() {
-        assertEquals(false, exec("byte x = (byte)1; def y = (int)7; return x > y"));
-        assertEquals(false, exec("short x = (short)2; def y = (int)6; return x > y"));
-        assertEquals(false, exec("char x = (char)3; def y = (int)5; return x > y"));
-        assertEquals(false, exec("int x = (int)4; def y = (int)4; return x > y"));
-        assertEquals(true, exec("long x = (long)5; def y = (int)3; return x > y"));
-        assertEquals(true, exec("float x = (float)6; def y = (int)2; return x > y"));
-        assertEquals(true, exec("double x = (double)7; def y = (int)1; return x > y"));
-
-        assertEquals(false, exec("byte x = (byte)1; def y = (double)7; return x > y"));
-        assertEquals(false, exec("short x = (short)2; def y = (double)6; return x > y"));
-        assertEquals(false, exec("char x = (char)3; def y = (double)5; return x > y"));
-        assertEquals(false, exec("int x = (int)4; def y = (double)4; return x > y"));
-        assertEquals(true, exec("long x = (long)5; def y = (double)3; return x > y"));
-        assertEquals(true, exec("float x = (float)6; def y = (double)2; return x > y"));
-        assertEquals(true, exec("double x = (double)7; def y = (double)1; return x > y"));
-    }
-    
-    public void testGtTypedRHS() {
-        assertEquals(false, exec("def x = (byte)1; int y = (int)7; return x > y"));
-        assertEquals(false, exec("def x = (short)2; int y = (int)6; return x > y"));
-        assertEquals(false, exec("def x = (char)3; int y = (int)5; return x > y"));
-        assertEquals(false, exec("def x = (int)4; int y = (int)4; return x > y"));
-        assertEquals(true, exec("def x = (long)5; int y = (int)3; return x > y"));
-        assertEquals(true, exec("def x = (float)6; int y = (int)2; return x > y"));
-        assertEquals(true, exec("def x = (double)7; int y = (int)1; return x > y"));
-
-        assertEquals(false, exec("def x = (byte)1; double y = (double)7; return x > y"));
-        assertEquals(false, exec("def x = (short)2; double y = (double)6; return x > y"));
-        assertEquals(false, exec("def x = (char)3; double y = (double)5; return x > y"));
-        assertEquals(false, exec("def x = (int)4; double y = (double)4; return x > y"));
-        assertEquals(true, exec("def x = (long)5; double y = (double)3; return x > y"));
-        assertEquals(true, exec("def x = (float)6; double y = (double)2; return x > y"));
-        assertEquals(true, exec("def x = (double)7; double y = (double)1; return x > y"));
-    }
-
-    public void testGte() {
-        assertEquals(false, exec("def x = (byte)1; def y = (int)7; return x >= y"));
-        assertEquals(false, exec("def x = (short)2; def y = (int)6; return x >= y"));
-        assertEquals(false, exec("def x = (char)3; def y = (int)5; return x >= y"));
-        assertEquals(true, exec("def x = (int)4; def y = (int)4; return x >= y"));
-        assertEquals(true, exec("def x = (long)5; def y = (int)3; return x >= y"));
-        assertEquals(true, exec("def x = (float)6; def y = (int)2; return x >= y"));
-        assertEquals(true, exec("def x = (double)7; def y = (int)1; return x >= y"));
-
-        assertEquals(false, exec("def x = (byte)1; def y = (double)7; return x >= y"));
-        assertEquals(false, exec("def x = (short)2; def y = (double)6; return x >= y"));
-        assertEquals(false, exec("def x = (char)3; def y = (double)5; return x >= y"));
-        assertEquals(true, exec("def x = (int)4; def y = (double)4; return x >= y"));
-        assertEquals(true, exec("def x = (long)5; def y = (double)3; return x >= y"));
-        assertEquals(true, exec("def x = (float)6; def y = (double)2; return x >= y"));
-        assertEquals(true, exec("def x = (double)7; def y = (double)1; return x >= y"));
-    }
-    
-    public void testGteTypedLHS() {
-        assertEquals(false, exec("byte x = (byte)1; def y = (int)7; return x >= y"));
-        assertEquals(false, exec("short x = (short)2; def y = (int)6; return x >= y"));
-        assertEquals(false, exec("char x = (char)3; def y = (int)5; return x >= y"));
-        assertEquals(true, exec("int x = (int)4; def y = (int)4; return x >= y"));
-        assertEquals(true, exec("long x = (long)5; def y = (int)3; return x >= y"));
-        assertEquals(true, exec("float x = (float)6; def y = (int)2; return x >= y"));
-        assertEquals(true, exec("double x = (double)7; def y = (int)1; return x >= y"));
-
-        assertEquals(false, exec("byte x = (byte)1; def y = (double)7; return x >= y"));
-        assertEquals(false, exec("short x = (short)2; def y = (double)6; return x >= y"));
-        assertEquals(false, exec("char x = (char)3; def y = (double)5; return x >= y"));
-        assertEquals(true, exec("int x = (int)4; def y = (double)4; return x >= y"));
-        assertEquals(true, exec("long x = (long)5; def y = (double)3; return x >= y"));
-        assertEquals(true, exec("float x = (float)6; def y = (double)2; return x >= y"));
-        assertEquals(true, exec("double x = (double)7; def y = (double)1; return x >= y"));
-    }
-    
-    public void testGteTypedRHS() {
-        assertEquals(false, exec("def x = (byte)1; int y = (int)7; return x >= y"));
-        assertEquals(false, exec("def x = (short)2; int y = (int)6; return x >= y"));
-        assertEquals(false, exec("def x = (char)3; int y = (int)5; return x >= y"));
-        assertEquals(true, exec("def x = (int)4; int y = (int)4; return x >= y"));
-        assertEquals(true, exec("def x = (long)5; int y = (int)3; return x >= y"));
-        assertEquals(true, exec("def x = (float)6; int y = (int)2; return x >= y"));
-        assertEquals(true, exec("def x = (double)7; int y = (int)1; return x >= y"));
-
-        assertEquals(false, exec("def x = (byte)1; double y = (double)7; return x >= y"));
-        assertEquals(false, exec("def x = (short)2; double y = (double)6; return x >= y"));
-        assertEquals(false, exec("def x = (char)3; double y = (double)5; return x >= y"));
-        assertEquals(true, exec("def x = (int)4; double y = (double)4; return x >= y"));
-        assertEquals(true, exec("def x = (long)5; double y = (double)3; return x >= y"));
-        assertEquals(true, exec("def x = (float)6; double y = (double)2; return x >= y"));
-        assertEquals(true, exec("def x = (double)7; double y = (double)1; return x >= y"));
-    }
-}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
index 3a7196f9074..1dd0d09a580 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
@@ -403,4 +403,19 @@ public class DefOptimizationTests extends ScriptTestCase {
         assertBytecodeExists("def x = 1; int y = 2; return x > y", 
                              "INVOKEDYNAMIC gt(Ljava/lang/Object;I)Z");
     }
+    
+    public void testUnaryMinusOptRet() {
+        assertBytecodeExists("def x = 1; double y = -x; return y", 
+                             "INVOKEDYNAMIC neg(Ljava/lang/Object;)D");
+    }
+    
+    public void testUnaryNotOptRet() {
+        assertBytecodeExists("def x = 1; double y = ~x; return y", 
+                             "INVOKEDYNAMIC not(Ljava/lang/Object;)D");
+    }
+    
+    public void testUnaryPlusOptRet() {
+        assertBytecodeExists("def x = 1; double y = +x; return y", 
+                             "INVOKEDYNAMIC plus(Ljava/lang/Object;)D");
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java
index 834f998723e..226cf6f1710 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java
@@ -24,6 +24,11 @@ package org.elasticsearch.painless;
 public class DivisionTests extends ScriptTestCase {
 
     // TODO: byte,short,char
+    
+    public void testBasics() throws Exception {
+        assertEquals(2.25F / 1.5F, exec("return 2.25F / 1.5F;"));
+        assertEquals(0.5, exec("double x = 1; float y = 2; return x / y;"));
+    }
 
     public void testInt() throws Exception {
         assertEquals(1/1, exec("int x = 1; int y = 1; return x/y;"));
@@ -132,4 +137,202 @@ public class DivisionTests extends ScriptTestCase {
             exec("return 1L/0L;");
         });
     }
+    
+    public void testDef() {
+        assertEquals(1, exec("def x = (byte)2; def y = (byte)2; return x / y"));
+        assertEquals(1, exec("def x = (short)2; def y = (byte)2; return x / y"));
+        assertEquals(1, exec("def x = (char)2; def y = (byte)2; return x / y"));
+        assertEquals(1, exec("def x = (int)2; def y = (byte)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; def y = (byte)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; def y = (byte)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; def y = (byte)2; return x / y"));
+
+        assertEquals(1, exec("def x = (byte)2; def y = (short)2; return x / y"));
+        assertEquals(1, exec("def x = (short)2; def y = (short)2; return x / y"));
+        assertEquals(1, exec("def x = (char)2; def y = (short)2; return x / y"));
+        assertEquals(1, exec("def x = (int)2; def y = (short)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; def y = (short)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; def y = (short)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; def y = (short)2; return x / y"));
+
+        assertEquals(1, exec("def x = (byte)2; def y = (char)2; return x / y"));
+        assertEquals(1, exec("def x = (short)2; def y = (char)2; return x / y"));
+        assertEquals(1, exec("def x = (char)2; def y = (char)2; return x / y"));
+        assertEquals(1, exec("def x = (int)2; def y = (char)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; def y = (char)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; def y = (char)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; def y = (char)2; return x / y"));
+
+        assertEquals(1, exec("def x = (byte)2; def y = (int)2; return x / y"));
+        assertEquals(1, exec("def x = (short)2; def y = (int)2; return x / y"));
+        assertEquals(1, exec("def x = (char)2; def y = (int)2; return x / y"));
+        assertEquals(1, exec("def x = (int)2; def y = (int)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; def y = (int)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; def y = (int)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; def y = (int)2; return x / y"));
+
+        assertEquals(1L, exec("def x = (byte)2; def y = (long)2; return x / y"));
+        assertEquals(1L, exec("def x = (short)2; def y = (long)2; return x / y"));
+        assertEquals(1L, exec("def x = (char)2; def y = (long)2; return x / y"));
+        assertEquals(1L, exec("def x = (int)2; def y = (long)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; def y = (long)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; def y = (long)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; def y = (long)2; return x / y"));
+
+        assertEquals(1F, exec("def x = (byte)2; def y = (float)2; return x / y"));
+        assertEquals(1F, exec("def x = (short)2; def y = (float)2; return x / y"));
+        assertEquals(1F, exec("def x = (char)2; def y = (float)2; return x / y"));
+        assertEquals(1F, exec("def x = (int)2; def y = (float)2; return x / y"));
+        assertEquals(1F, exec("def x = (long)2; def y = (float)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; def y = (float)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; def y = (float)2; return x / y"));
+
+        assertEquals(1D, exec("def x = (byte)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (short)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (char)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (int)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (long)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (float)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; def y = (double)2; return x / y"));
+
+        assertEquals(1, exec("def x = (byte)2; def y = (byte)2; return x / y"));
+        assertEquals(1, exec("def x = (short)2; def y = (short)2; return x / y"));
+        assertEquals(1, exec("def x = (char)2; def y = (char)2; return x / y"));
+        assertEquals(1, exec("def x = (int)2; def y = (int)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; def y = (long)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; def y = (float)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; def y = (double)2; return x / y"));
+    }
+    
+    public void testDefTypedLHS() {
+        assertEquals(1, exec("byte x = (byte)2; def y = (byte)2; return x / y"));
+        assertEquals(1, exec("short x = (short)2; def y = (byte)2; return x / y"));
+        assertEquals(1, exec("char x = (char)2; def y = (byte)2; return x / y"));
+        assertEquals(1, exec("int x = (int)2; def y = (byte)2; return x / y"));
+        assertEquals(1L, exec("long x = (long)2; def y = (byte)2; return x / y"));
+        assertEquals(1F, exec("float x = (float)2; def y = (byte)2; return x / y"));
+        assertEquals(1D, exec("double x = (double)2; def y = (byte)2; return x / y"));
+
+        assertEquals(1, exec("byte x = (byte)2; def y = (short)2; return x / y"));
+        assertEquals(1, exec("short x = (short)2; def y = (short)2; return x / y"));
+        assertEquals(1, exec("char x = (char)2; def y = (short)2; return x / y"));
+        assertEquals(1, exec("int x = (int)2; def y = (short)2; return x / y"));
+        assertEquals(1L, exec("long x = (long)2; def y = (short)2; return x / y"));
+        assertEquals(1F, exec("float x = (float)2; def y = (short)2; return x / y"));
+        assertEquals(1D, exec("double x = (double)2; def y = (short)2; return x / y"));
+
+        assertEquals(1, exec("byte x = (byte)2; def y = (char)2; return x / y"));
+        assertEquals(1, exec("short x = (short)2; def y = (char)2; return x / y"));
+        assertEquals(1, exec("char x = (char)2; def y = (char)2; return x / y"));
+        assertEquals(1, exec("int x = (int)2; def y = (char)2; return x / y"));
+        assertEquals(1L, exec("long x = (long)2; def y = (char)2; return x / y"));
+        assertEquals(1F, exec("float x = (float)2; def y = (char)2; return x / y"));
+        assertEquals(1D, exec("double x = (double)2; def y = (char)2; return x / y"));
+
+        assertEquals(1, exec("byte x = (byte)2; def y = (int)2; return x / y"));
+        assertEquals(1, exec("short x = (short)2; def y = (int)2; return x / y"));
+        assertEquals(1, exec("char x = (char)2; def y = (int)2; return x / y"));
+        assertEquals(1, exec("int x = (int)2; def y = (int)2; return x / y"));
+        assertEquals(1L, exec("long x = (long)2; def y = (int)2; return x / y"));
+        assertEquals(1F, exec("float x = (float)2; def y = (int)2; return x / y"));
+        assertEquals(1D, exec("double x = (double)2; def y = (int)2; return x / y"));
+
+        assertEquals(1L, exec("byte x = (byte)2; def y = (long)2; return x / y"));
+        assertEquals(1L, exec("short x = (short)2; def y = (long)2; return x / y"));
+        assertEquals(1L, exec("char x = (char)2; def y = (long)2; return x / y"));
+        assertEquals(1L, exec("int x = (int)2; def y = (long)2; return x / y"));
+        assertEquals(1L, exec("long x = (long)2; def y = (long)2; return x / y"));
+        assertEquals(1F, exec("float x = (float)2; def y = (long)2; return x / y"));
+        assertEquals(1D, exec("double x = (double)2; def y = (long)2; return x / y"));
+
+        assertEquals(1F, exec("byte x = (byte)2; def y = (float)2; return x / y"));
+        assertEquals(1F, exec("short x = (short)2; def y = (float)2; return x / y"));
+        assertEquals(1F, exec("char x = (char)2; def y = (float)2; return x / y"));
+        assertEquals(1F, exec("int x = (int)2; def y = (float)2; return x / y"));
+        assertEquals(1F, exec("long x = (long)2; def y = (float)2; return x / y"));
+        assertEquals(1F, exec("float x = (float)2; def y = (float)2; return x / y"));
+        assertEquals(1D, exec("double x = (double)2; def y = (float)2; return x / y"));
+
+        assertEquals(1D, exec("byte x = (byte)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("short x = (short)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("char x = (char)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("int x = (int)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("long x = (long)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("float x = (float)2; def y = (double)2; return x / y"));
+        assertEquals(1D, exec("double x = (double)2; def y = (double)2; return x / y"));
+
+        assertEquals(1, exec("byte x = (byte)2; def y = (byte)2; return x / y"));
+        assertEquals(1, exec("short x = (short)2; def y = (short)2; return x / y"));
+        assertEquals(1, exec("char x = (char)2; def y = (char)2; return x / y"));
+        assertEquals(1, exec("int x = (int)2; def y = (int)2; return x / y"));
+        assertEquals(1L, exec("long x = (long)2; def y = (long)2; return x / y"));
+        assertEquals(1F, exec("float x = (float)2; def y = (float)2; return x / y"));
+        assertEquals(1D, exec("double x = (double)2; def y = (double)2; return x / y"));
+    }
+    
+    public void testDefTypedRHS() {
+        assertEquals(1, exec("def x = (byte)2; byte y = (byte)2; return x / y"));
+        assertEquals(1, exec("def x = (short)2; byte y = (byte)2; return x / y"));
+        assertEquals(1, exec("def x = (char)2; byte y = (byte)2; return x / y"));
+        assertEquals(1, exec("def x = (int)2; byte y = (byte)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; byte y = (byte)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; byte y = (byte)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; byte y = (byte)2; return x / y"));
+
+        assertEquals(1, exec("def x = (byte)2; short y = (short)2; return x / y"));
+        assertEquals(1, exec("def x = (short)2; short y = (short)2; return x / y"));
+        assertEquals(1, exec("def x = (char)2; short y = (short)2; return x / y"));
+        assertEquals(1, exec("def x = (int)2; short y = (short)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; short y = (short)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; short y = (short)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; short y = (short)2; return x / y"));
+
+        assertEquals(1, exec("def x = (byte)2; char y = (char)2; return x / y"));
+        assertEquals(1, exec("def x = (short)2; char y = (char)2; return x / y"));
+        assertEquals(1, exec("def x = (char)2; char y = (char)2; return x / y"));
+        assertEquals(1, exec("def x = (int)2; char y = (char)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; char y = (char)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; char y = (char)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; char y = (char)2; return x / y"));
+
+        assertEquals(1, exec("def x = (byte)2; int y = (int)2; return x / y"));
+        assertEquals(1, exec("def x = (short)2; int y = (int)2; return x / y"));
+        assertEquals(1, exec("def x = (char)2; int y = (int)2; return x / y"));
+        assertEquals(1, exec("def x = (int)2; int y = (int)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; int y = (int)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; int y = (int)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; int y = (int)2; return x / y"));
+
+        assertEquals(1L, exec("def x = (byte)2; long y = (long)2; return x / y"));
+        assertEquals(1L, exec("def x = (short)2; long y = (long)2; return x / y"));
+        assertEquals(1L, exec("def x = (char)2; long y = (long)2; return x / y"));
+        assertEquals(1L, exec("def x = (int)2; long y = (long)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; long y = (long)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; long y = (long)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; long y = (long)2; return x / y"));
+
+        assertEquals(1F, exec("def x = (byte)2; float y = (float)2; return x / y"));
+        assertEquals(1F, exec("def x = (short)2; float y = (float)2; return x / y"));
+        assertEquals(1F, exec("def x = (char)2; float y = (float)2; return x / y"));
+        assertEquals(1F, exec("def x = (int)2; float y = (float)2; return x / y"));
+        assertEquals(1F, exec("def x = (long)2; float y = (float)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; float y = (float)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; float y = (float)2; return x / y"));
+
+        assertEquals(1D, exec("def x = (byte)2; double y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (short)2; double y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (char)2; double y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (int)2; double y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (long)2; double y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (float)2; double y = (double)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; double y = (double)2; return x / y"));
+
+        assertEquals(1, exec("def x = (byte)2; byte y = (byte)2; return x / y"));
+        assertEquals(1, exec("def x = (short)2; short y = (short)2; return x / y"));
+        assertEquals(1, exec("def x = (char)2; char y = (char)2; return x / y"));
+        assertEquals(1, exec("def x = (int)2; int y = (int)2; return x / y"));
+        assertEquals(1L, exec("def x = (long)2; long y = (long)2; return x / y"));
+        assertEquals(1F, exec("def x = (float)2; float y = (float)2; return x / y"));
+        assertEquals(1D, exec("def x = (double)2; double y = (double)2; return x / y"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/MultiplicationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/MultiplicationTests.java
index 9c10d90bec2..2ccb51d7946 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/MultiplicationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/MultiplicationTests.java
@@ -24,6 +24,10 @@ package org.elasticsearch.painless;
 public class MultiplicationTests extends ScriptTestCase {
 
     // TODO: short,byte,char
+    
+    public void testBasics() throws Exception {
+        assertEquals(8, exec("int x = 4; char y = 2; return x*y;"));
+    }
 
     public void testInt() throws Exception {
         assertEquals(1*1, exec("int x = 1; int y = 1; return x*y;"));
@@ -123,4 +127,202 @@ public class MultiplicationTests extends ScriptTestCase {
         assertEquals(10.0*0.0, exec("return 10.0*0.0;"));
         assertEquals(0.0*0.0, exec("return 0.0*0.0;"));
     }
+    
+    public void testDef() {
+        assertEquals(4, exec("def x = (byte)2; def y = (byte)2; return x * y"));
+        assertEquals(4, exec("def x = (short)2; def y = (byte)2; return x * y"));
+        assertEquals(4, exec("def x = (char)2; def y = (byte)2; return x * y"));
+        assertEquals(4, exec("def x = (int)2; def y = (byte)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; def y = (byte)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; def y = (byte)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; def y = (byte)2; return x * y"));
+
+        assertEquals(4, exec("def x = (byte)2; def y = (short)2; return x * y"));
+        assertEquals(4, exec("def x = (short)2; def y = (short)2; return x * y"));
+        assertEquals(4, exec("def x = (char)2; def y = (short)2; return x * y"));
+        assertEquals(4, exec("def x = (int)2; def y = (short)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; def y = (short)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; def y = (short)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; def y = (short)2; return x * y"));
+
+        assertEquals(4, exec("def x = (byte)2; def y = (char)2; return x * y"));
+        assertEquals(4, exec("def x = (short)2; def y = (char)2; return x * y"));
+        assertEquals(4, exec("def x = (char)2; def y = (char)2; return x * y"));
+        assertEquals(4, exec("def x = (int)2; def y = (char)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; def y = (char)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; def y = (char)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; def y = (char)2; return x * y"));
+
+        assertEquals(4, exec("def x = (byte)2; def y = (int)2; return x * y"));
+        assertEquals(4, exec("def x = (short)2; def y = (int)2; return x * y"));
+        assertEquals(4, exec("def x = (char)2; def y = (int)2; return x * y"));
+        assertEquals(4, exec("def x = (int)2; def y = (int)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; def y = (int)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; def y = (int)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; def y = (int)2; return x * y"));
+
+        assertEquals(4L, exec("def x = (byte)2; def y = (long)2; return x * y"));
+        assertEquals(4L, exec("def x = (short)2; def y = (long)2; return x * y"));
+        assertEquals(4L, exec("def x = (char)2; def y = (long)2; return x * y"));
+        assertEquals(4L, exec("def x = (int)2; def y = (long)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; def y = (long)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; def y = (long)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; def y = (long)2; return x * y"));
+
+        assertEquals(4F, exec("def x = (byte)2; def y = (float)2; return x * y"));
+        assertEquals(4F, exec("def x = (short)2; def y = (float)2; return x * y"));
+        assertEquals(4F, exec("def x = (char)2; def y = (float)2; return x * y"));
+        assertEquals(4F, exec("def x = (int)2; def y = (float)2; return x * y"));
+        assertEquals(4F, exec("def x = (long)2; def y = (float)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; def y = (float)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; def y = (float)2; return x * y"));
+
+        assertEquals(4D, exec("def x = (byte)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (short)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (char)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (int)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (long)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (float)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; def y = (double)2; return x * y"));
+
+        assertEquals(4, exec("def x = (byte)2; def y = (byte)2; return x * y"));
+        assertEquals(4, exec("def x = (short)2; def y = (short)2; return x * y"));
+        assertEquals(4, exec("def x = (char)2; def y = (char)2; return x * y"));
+        assertEquals(4, exec("def x = (int)2; def y = (int)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; def y = (long)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; def y = (float)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; def y = (double)2; return x * y"));
+    }
+    
+    public void testDefTypedLHS() {
+        assertEquals(4, exec("byte x = (byte)2; def y = (byte)2; return x * y"));
+        assertEquals(4, exec("short x = (short)2; def y = (byte)2; return x * y"));
+        assertEquals(4, exec("char x = (char)2; def y = (byte)2; return x * y"));
+        assertEquals(4, exec("int x = (int)2; def y = (byte)2; return x * y"));
+        assertEquals(4L, exec("long x = (long)2; def y = (byte)2; return x * y"));
+        assertEquals(4F, exec("float x = (float)2; def y = (byte)2; return x * y"));
+        assertEquals(4D, exec("double x = (double)2; def y = (byte)2; return x * y"));
+
+        assertEquals(4, exec("byte x = (byte)2; def y = (short)2; return x * y"));
+        assertEquals(4, exec("short x = (short)2; def y = (short)2; return x * y"));
+        assertEquals(4, exec("char x = (char)2; def y = (short)2; return x * y"));
+        assertEquals(4, exec("int x = (int)2; def y = (short)2; return x * y"));
+        assertEquals(4L, exec("long x = (long)2; def y = (short)2; return x * y"));
+        assertEquals(4F, exec("float x = (float)2; def y = (short)2; return x * y"));
+        assertEquals(4D, exec("double x = (double)2; def y = (short)2; return x * y"));
+
+        assertEquals(4, exec("byte x = (byte)2; def y = (char)2; return x * y"));
+        assertEquals(4, exec("short x = (short)2; def y = (char)2; return x * y"));
+        assertEquals(4, exec("char x = (char)2; def y = (char)2; return x * y"));
+        assertEquals(4, exec("int x = (int)2; def y = (char)2; return x * y"));
+        assertEquals(4L, exec("long x = (long)2; def y = (char)2; return x * y"));
+        assertEquals(4F, exec("float x = (float)2; def y = (char)2; return x * y"));
+        assertEquals(4D, exec("double x = (double)2; def y = (char)2; return x * y"));
+
+        assertEquals(4, exec("byte x = (byte)2; def y = (int)2; return x * y"));
+        assertEquals(4, exec("short x = (short)2; def y = (int)2; return x * y"));
+        assertEquals(4, exec("char x = (char)2; def y = (int)2; return x * y"));
+        assertEquals(4, exec("int x = (int)2; def y = (int)2; return x * y"));
+        assertEquals(4L, exec("long x = (long)2; def y = (int)2; return x * y"));
+        assertEquals(4F, exec("float x = (float)2; def y = (int)2; return x * y"));
+        assertEquals(4D, exec("double x = (double)2; def y = (int)2; return x * y"));
+
+        assertEquals(4L, exec("byte x = (byte)2; def y = (long)2; return x * y"));
+        assertEquals(4L, exec("short x = (short)2; def y = (long)2; return x * y"));
+        assertEquals(4L, exec("char x = (char)2; def y = (long)2; return x * y"));
+        assertEquals(4L, exec("int x = (int)2; def y = (long)2; return x * y"));
+        assertEquals(4L, exec("long x = (long)2; def y = (long)2; return x * y"));
+        assertEquals(4F, exec("float x = (float)2; def y = (long)2; return x * y"));
+        assertEquals(4D, exec("double x = (double)2; def y = (long)2; return x * y"));
+
+        assertEquals(4F, exec("byte x = (byte)2; def y = (float)2; return x * y"));
+        assertEquals(4F, exec("short x = (short)2; def y = (float)2; return x * y"));
+        assertEquals(4F, exec("char x = (char)2; def y = (float)2; return x * y"));
+        assertEquals(4F, exec("int x = (int)2; def y = (float)2; return x * y"));
+        assertEquals(4F, exec("long x = (long)2; def y = (float)2; return x * y"));
+        assertEquals(4F, exec("float x = (float)2; def y = (float)2; return x * y"));
+        assertEquals(4D, exec("double x = (double)2; def y = (float)2; return x * y"));
+
+        assertEquals(4D, exec("byte x = (byte)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("short x = (short)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("char x = (char)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("int x = (int)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("long x = (long)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("float x = (float)2; def y = (double)2; return x * y"));
+        assertEquals(4D, exec("double x = (double)2; def y = (double)2; return x * y"));
+
+        assertEquals(4, exec("byte x = (byte)2; def y = (byte)2; return x * y"));
+        assertEquals(4, exec("short x = (short)2; def y = (short)2; return x * y"));
+        assertEquals(4, exec("char x = (char)2; def y = (char)2; return x * y"));
+        assertEquals(4, exec("int x = (int)2; def y = (int)2; return x * y"));
+        assertEquals(4L, exec("long x = (long)2; def y = (long)2; return x * y"));
+        assertEquals(4F, exec("float x = (float)2; def y = (float)2; return x * y"));
+        assertEquals(4D, exec("double x = (double)2; def y = (double)2; return x * y"));
+    }
+    
+    public void testDefTypedRHS() {
+        assertEquals(4, exec("def x = (byte)2; byte y = (byte)2; return x * y"));
+        assertEquals(4, exec("def x = (short)2; byte y = (byte)2; return x * y"));
+        assertEquals(4, exec("def x = (char)2; byte y = (byte)2; return x * y"));
+        assertEquals(4, exec("def x = (int)2; byte y = (byte)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; byte y = (byte)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; byte y = (byte)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; byte y = (byte)2; return x * y"));
+
+        assertEquals(4, exec("def x = (byte)2; short y = (short)2; return x * y"));
+        assertEquals(4, exec("def x = (short)2; short y = (short)2; return x * y"));
+        assertEquals(4, exec("def x = (char)2; short y = (short)2; return x * y"));
+        assertEquals(4, exec("def x = (int)2; short y = (short)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; short y = (short)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; short y = (short)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; short y = (short)2; return x * y"));
+
+        assertEquals(4, exec("def x = (byte)2; char y = (char)2; return x * y"));
+        assertEquals(4, exec("def x = (short)2; char y = (char)2; return x * y"));
+        assertEquals(4, exec("def x = (char)2; char y = (char)2; return x * y"));
+        assertEquals(4, exec("def x = (int)2; char y = (char)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; char y = (char)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; char y = (char)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; char y = (char)2; return x * y"));
+
+        assertEquals(4, exec("def x = (byte)2; int y = (int)2; return x * y"));
+        assertEquals(4, exec("def x = (short)2; int y = (int)2; return x * y"));
+        assertEquals(4, exec("def x = (char)2; int y = (int)2; return x * y"));
+        assertEquals(4, exec("def x = (int)2; int y = (int)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; int y = (int)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; int y = (int)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; int y = (int)2; return x * y"));
+
+        assertEquals(4L, exec("def x = (byte)2; long y = (long)2; return x * y"));
+        assertEquals(4L, exec("def x = (short)2; long y = (long)2; return x * y"));
+        assertEquals(4L, exec("def x = (char)2; long y = (long)2; return x * y"));
+        assertEquals(4L, exec("def x = (int)2; long y = (long)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; long y = (long)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; long y = (long)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; long y = (long)2; return x * y"));
+
+        assertEquals(4F, exec("def x = (byte)2; float y = (float)2; return x * y"));
+        assertEquals(4F, exec("def x = (short)2; float y = (float)2; return x * y"));
+        assertEquals(4F, exec("def x = (char)2; float y = (float)2; return x * y"));
+        assertEquals(4F, exec("def x = (int)2; float y = (float)2; return x * y"));
+        assertEquals(4F, exec("def x = (long)2; float y = (float)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; float y = (float)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; float y = (float)2; return x * y"));
+
+        assertEquals(4D, exec("def x = (byte)2; double y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (short)2; double y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (char)2; double y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (int)2; double y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (long)2; double y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (float)2; double y = (double)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; double y = (double)2; return x * y"));
+
+        assertEquals(4, exec("def x = (byte)2; byte y = (byte)2; return x * y"));
+        assertEquals(4, exec("def x = (short)2; short y = (short)2; return x * y"));
+        assertEquals(4, exec("def x = (char)2; char y = (char)2; return x * y"));
+        assertEquals(4, exec("def x = (int)2; int y = (int)2; return x * y"));
+        assertEquals(4L, exec("def x = (long)2; long y = (long)2; return x * y"));
+        assertEquals(4F, exec("def x = (float)2; float y = (float)2; return x * y"));
+        assertEquals(4D, exec("def x = (double)2; double y = (double)2; return x * y"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java
index d27c019a798..5cf9c186fd6 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java
@@ -22,6 +22,13 @@ package org.elasticsearch.painless;
 /** Tests for or operator across all types */
 public class OrTests extends ScriptTestCase {
 
+    public void testBasics() throws Exception {
+        assertEquals(5 | 3, exec("return 5 | 3;"));
+        assertEquals(5L | 3, exec("return 5L | 3;"));
+        assertEquals(5 | 3L, exec("return 5 | 3L;"));
+        assertEquals(7, exec("short x = 5; byte y = 3; return x | y;"));
+    }
+
     public void testInt() throws Exception {
         assertEquals(5 | 12, exec("int x = 5; int y = 12; return x | y;"));
         assertEquals(5 | -12, exec("int x = 5; int y = -12; return x | y;"));
@@ -54,4 +61,151 @@ public class OrTests extends ScriptTestCase {
             exec("double x = (double)4; int y = 1; return x | y");
         });
     }
+    
+    public void testDef() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (float)4; def y = (byte)1; return x | y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (double)4; def y = (byte)1; return x | y");
+        });
+        assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x | y"));
+        assertEquals(5, exec("def x = (short)4; def y = (byte)1; return x | y"));
+        assertEquals(5, exec("def x = (char)4; def y = (byte)1; return x | y"));
+        assertEquals(5, exec("def x = (int)4; def y = (byte)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (byte)1; return x | y"));
+
+        assertEquals(5, exec("def x = (byte)4; def y = (short)1; return x | y"));
+        assertEquals(5, exec("def x = (short)4; def y = (short)1; return x | y"));
+        assertEquals(5, exec("def x = (char)4; def y = (short)1; return x | y"));
+        assertEquals(5, exec("def x = (int)4; def y = (short)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (short)1; return x | y"));
+
+        assertEquals(5, exec("def x = (byte)4; def y = (char)1; return x | y"));
+        assertEquals(5, exec("def x = (short)4; def y = (char)1; return x | y"));
+        assertEquals(5, exec("def x = (char)4; def y = (char)1; return x | y"));
+        assertEquals(5, exec("def x = (int)4; def y = (char)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (char)1; return x | y"));
+
+        assertEquals(5, exec("def x = (byte)4; def y = (int)1; return x | y"));
+        assertEquals(5, exec("def x = (short)4; def y = (int)1; return x | y"));
+        assertEquals(5, exec("def x = (char)4; def y = (int)1; return x | y"));
+        assertEquals(5, exec("def x = (int)4; def y = (int)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (int)1; return x | y"));
+
+        assertEquals(5L, exec("def x = (byte)4; def y = (long)1; return x | y"));
+        assertEquals(5L, exec("def x = (short)4; def y = (long)1; return x | y"));
+        assertEquals(5L, exec("def x = (char)4; def y = (long)1; return x | y"));
+        assertEquals(5L, exec("def x = (int)4; def y = (long)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x | y"));
+
+        assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x | y"));
+        assertEquals(5, exec("def x = (short)4; def y = (short)1; return x | y"));
+        assertEquals(5, exec("def x = (char)4; def y = (char)1; return x | y"));
+        assertEquals(5, exec("def x = (int)4; def y = (int)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x | y"));
+        
+        assertEquals(true,  exec("def x = true;  def y = true; return x | y"));
+        assertEquals(true,  exec("def x = true;  def y = false; return x | y"));
+        assertEquals(true,  exec("def x = false; def y = true; return x | y"));
+        assertEquals(false, exec("def x = false; def y = false; return x | y"));
+    }
+    
+    public void testDefTypedLHS() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("float x = (float)4; def y = (byte)1; return x | y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("double x = (double)4; def y = (byte)1; return x | y");
+        });
+        assertEquals(5, exec("byte x = (byte)4; def y = (byte)1; return x | y"));
+        assertEquals(5, exec("short x = (short)4; def y = (byte)1; return x | y"));
+        assertEquals(5, exec("char x = (char)4; def y = (byte)1; return x | y"));
+        assertEquals(5, exec("int x = (int)4; def y = (byte)1; return x | y"));
+        assertEquals(5L, exec("long x = (long)4; def y = (byte)1; return x | y"));
+
+        assertEquals(5, exec("byte x = (byte)4; def y = (short)1; return x | y"));
+        assertEquals(5, exec("short x = (short)4; def y = (short)1; return x | y"));
+        assertEquals(5, exec("char x = (char)4; def y = (short)1; return x | y"));
+        assertEquals(5, exec("int x = (int)4; def y = (short)1; return x | y"));
+        assertEquals(5L, exec("long x = (long)4; def y = (short)1; return x | y"));
+
+        assertEquals(5, exec("byte x = (byte)4; def y = (char)1; return x | y"));
+        assertEquals(5, exec("short x = (short)4; def y = (char)1; return x | y"));
+        assertEquals(5, exec("char x = (char)4; def y = (char)1; return x | y"));
+        assertEquals(5, exec("int x = (int)4; def y = (char)1; return x | y"));
+        assertEquals(5L, exec("long x = (long)4; def y = (char)1; return x | y"));
+
+        assertEquals(5, exec("byte x = (byte)4; def y = (int)1; return x | y"));
+        assertEquals(5, exec("short x = (short)4; def y = (int)1; return x | y"));
+        assertEquals(5, exec("char x = (char)4; def y = (int)1; return x | y"));
+        assertEquals(5, exec("int x = (int)4; def y = (int)1; return x | y"));
+        assertEquals(5L, exec("long x = (long)4; def y = (int)1; return x | y"));
+
+        assertEquals(5L, exec("byte x = (byte)4; def y = (long)1; return x | y"));
+        assertEquals(5L, exec("short x = (short)4; def y = (long)1; return x | y"));
+        assertEquals(5L, exec("char x = (char)4; def y = (long)1; return x | y"));
+        assertEquals(5L, exec("int x = (int)4; def y = (long)1; return x | y"));
+        assertEquals(5L, exec("long x = (long)4; def y = (long)1; return x | y"));
+
+        assertEquals(5, exec("byte x = (byte)4; def y = (byte)1; return x | y"));
+        assertEquals(5, exec("short x = (short)4; def y = (short)1; return x | y"));
+        assertEquals(5, exec("char x = (char)4; def y = (char)1; return x | y"));
+        assertEquals(5, exec("int x = (int)4; def y = (int)1; return x | y"));
+        assertEquals(5L, exec("long x = (long)4; def y = (long)1; return x | y"));
+        
+        assertEquals(true,  exec("boolean x = true;  def y = true; return x | y"));
+        assertEquals(true,  exec("boolean x = true;  def y = false; return x | y"));
+        assertEquals(true,  exec("boolean x = false; def y = true; return x | y"));
+        assertEquals(false, exec("boolean x = false; def y = false; return x | y"));
+    }
+    
+    public void testDefTypedRHS() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (float)4; byte y = (byte)1; return x | y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (double)4; byte y = (byte)1; return x | y");
+        });
+        assertEquals(5, exec("def x = (byte)4; byte y = (byte)1; return x | y"));
+        assertEquals(5, exec("def x = (short)4; byte y = (byte)1; return x | y"));
+        assertEquals(5, exec("def x = (char)4; byte y = (byte)1; return x | y"));
+        assertEquals(5, exec("def x = (int)4; byte y = (byte)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; byte y = (byte)1; return x | y"));
+
+        assertEquals(5, exec("def x = (byte)4; short y = (short)1; return x | y"));
+        assertEquals(5, exec("def x = (short)4; short y = (short)1; return x | y"));
+        assertEquals(5, exec("def x = (char)4; short y = (short)1; return x | y"));
+        assertEquals(5, exec("def x = (int)4; short y = (short)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; short y = (short)1; return x | y"));
+
+        assertEquals(5, exec("def x = (byte)4; char y = (char)1; return x | y"));
+        assertEquals(5, exec("def x = (short)4; char y = (char)1; return x | y"));
+        assertEquals(5, exec("def x = (char)4; char y = (char)1; return x | y"));
+        assertEquals(5, exec("def x = (int)4; char y = (char)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; char y = (char)1; return x | y"));
+
+        assertEquals(5, exec("def x = (byte)4; int y = (int)1; return x | y"));
+        assertEquals(5, exec("def x = (short)4; int y = (int)1; return x | y"));
+        assertEquals(5, exec("def x = (char)4; int y = (int)1; return x | y"));
+        assertEquals(5, exec("def x = (int)4; int y = (int)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; int y = (int)1; return x | y"));
+
+        assertEquals(5L, exec("def x = (byte)4; long y = (long)1; return x | y"));
+        assertEquals(5L, exec("def x = (short)4; long y = (long)1; return x | y"));
+        assertEquals(5L, exec("def x = (char)4; long y = (long)1; return x | y"));
+        assertEquals(5L, exec("def x = (int)4; long y = (long)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; long y = (long)1; return x | y"));
+
+        assertEquals(5, exec("def x = (byte)4; byte y = (byte)1; return x | y"));
+        assertEquals(5, exec("def x = (short)4; short y = (short)1; return x | y"));
+        assertEquals(5, exec("def x = (char)4; char y = (char)1; return x | y"));
+        assertEquals(5, exec("def x = (int)4; int y = (int)1; return x | y"));
+        assertEquals(5L, exec("def x = (long)4; long y = (long)1; return x | y"));
+        
+        assertEquals(true,  exec("def x = true;  boolean y = true; return x | y"));
+        assertEquals(true,  exec("def x = true;  boolean y = false; return x | y"));
+        assertEquals(true,  exec("def x = false; boolean y = true; return x | y"));
+        assertEquals(false, exec("def x = false; boolean y = false; return x | y"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BinaryOperatorTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PromotionTests.java
similarity index 72%
rename from modules/lang-painless/src/test/java/org/elasticsearch/painless/BinaryOperatorTests.java
rename to modules/lang-painless/src/test/java/org/elasticsearch/painless/PromotionTests.java
index 61bac8bd6b5..9de618be4a7 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BinaryOperatorTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PromotionTests.java
@@ -19,93 +19,7 @@
 
 package org.elasticsearch.painless;
 
-/**
- * Tests binary operators across different types
- */
-// TODO: NaN/Inf/overflow/...
-public class BinaryOperatorTests extends ScriptTestCase {
-
-    // TODO: move to per-type tests and test for each type
-    public void testBasics() {
-        assertEquals(2.25F / 1.5F, exec("return 2.25F / 1.5F;"));
-        assertEquals(2.25F % 1.5F, exec("return 2.25F % 1.5F;"));
-        assertEquals(2 - 1, exec("return 2 - 1;"));
-        assertEquals(1 << 2, exec("return 1 << 2;"));
-        assertEquals(4 >> 2, exec("return 4 >> 2;"));
-        assertEquals(-1 >>> 29, exec("return -1 >>> 29;"));
-        assertEquals(5 & 3, exec("return 5 & 3;"));
-        assertEquals(5 & 3L, exec("return 5 & 3L;"));
-        assertEquals(5L & 3, exec("return 5L & 3;"));
-        assertEquals(5 | 3, exec("return 5 | 3;"));
-        assertEquals(5L | 3, exec("return 5L | 3;"));
-        assertEquals(5 | 3L, exec("return 5 | 3L;"));
-        assertEquals(9 ^ 3, exec("return 9 ^ 3;"));
-        assertEquals(9L ^ 3, exec("return 9L ^ 3;"));
-        assertEquals(9 ^ 3L, exec("return 9 ^ 3L;"));
-    }
-
-    public void testLongShifts() {
-        assertEquals(1L << 2, exec("long x = 1L; int y = 2; return x << y;"));
-        assertEquals(1 << 2L, exec("int x = 1; long y = 2L; return x << y;"));
-        assertEquals(4 >> 2L, exec("int x = 4; long y = 2L; return x >> y;"));
-        assertEquals(4L >> 2, exec("long x = 4L; int y = 2; return x >> y;"));
-        assertEquals(-1L >>> 29, exec("long x = -1L; int y = 29; return x >>> y;"));
-        assertEquals(-1 >>> 29L, exec("int x = -1; long y = 29L; return x >>> y;"));
-    }
-
-    public void testLongShiftsConst() {
-        assertEquals(1L << 2, exec("return 1L << 2;"));
-        assertEquals(1 << 2L, exec("return 1 << 2L;"));
-        assertEquals(4 >> 2L, exec("return 4 >> 2L;"));
-        assertEquals(4L >> 2, exec("return 4L >> 2;"));
-        assertEquals(-1L >>> 29, exec("return -1L >>> 29;"));
-        assertEquals(-1 >>> 29L, exec("return -1 >>> 29L;"));
-    }
-    
-    public void testBogusShifts() {
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("long x = 1L; float y = 2; return x << y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("int x = 1; double y = 2L; return x << y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("float x = 1F; int y = 2; return x << y;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("double x = 1D; int y = 2L; return x << y;");
-        });
-    }
-
-    public void testBogusShiftsConst() {
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("return 1L << 2F;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("return 1L << 2.0;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("return 1F << 2;");
-        });
-        expectScriptThrows(ClassCastException.class, ()-> {
-            exec("return 1D << 2L");
-        });
-    }
-
-    public void testMixedTypes() {
-        assertEquals(8, exec("int x = 4; char y = 2; return x*y;"));
-        assertEquals(0.5, exec("double x = 1; float y = 2; return x / y;"));
-        assertEquals(1, exec("int x = 3; int y = 2; return x % y;"));
-        assertEquals(3.0, exec("double x = 1; byte y = 2; return x + y;"));
-        assertEquals(-1, exec("int x = 1; char y = 2; return x - y;"));
-        assertEquals(4, exec("int x = 1; char y = 2; return x << y;"));
-        assertEquals(-1, exec("int x = -1; char y = 29; return x >> y;"));
-        assertEquals(3, exec("int x = -1; char y = 30; return x >>> y;"));
-        assertEquals(1L, exec("int x = 5; long y = 3; return x & y;"));
-        assertEquals(7, exec("short x = 5; byte y = 3; return x | y;"));
-        assertEquals(10, exec("short x = 9; char y = 3; return x ^ y;"));
-    }
-
+public class PromotionTests extends ScriptTestCase {
     public void testBinaryPromotion() throws Exception {
         // byte/byte
         assertEquals((byte)1 + (byte)1, exec("byte x = 1; byte y = 1; return x+y;"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java
index 448d4262879..d2c9406c551 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java
@@ -24,6 +24,11 @@ package org.elasticsearch.painless;
 public class RemainderTests extends ScriptTestCase {
 
     // TODO: byte,short,char
+    
+    public void testBasics() throws Exception {
+        assertEquals(2.25F % 1.5F, exec("return 2.25F % 1.5F;"));
+        assertEquals(1, exec("int x = 3; int y = 2; return x % y;"));
+    }
 
     public void testInt() throws Exception {
         assertEquals(1%1, exec("int x = 1; int y = 1; return x%y;"));
@@ -132,4 +137,202 @@ public class RemainderTests extends ScriptTestCase {
             exec("return 1L%0L;");
         });
     }
+    
+    public void testDef() {
+        assertEquals(0, exec("def x = (byte)2; def y = (byte)2; return x % y"));
+        assertEquals(0, exec("def x = (short)2; def y = (byte)2; return x % y"));
+        assertEquals(0, exec("def x = (char)2; def y = (byte)2; return x % y"));
+        assertEquals(0, exec("def x = (int)2; def y = (byte)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; def y = (byte)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; def y = (byte)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; def y = (byte)2; return x % y"));
+
+        assertEquals(0, exec("def x = (byte)2; def y = (short)2; return x % y"));
+        assertEquals(0, exec("def x = (short)2; def y = (short)2; return x % y"));
+        assertEquals(0, exec("def x = (char)2; def y = (short)2; return x % y"));
+        assertEquals(0, exec("def x = (int)2; def y = (short)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; def y = (short)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; def y = (short)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; def y = (short)2; return x % y"));
+
+        assertEquals(0, exec("def x = (byte)2; def y = (char)2; return x % y"));
+        assertEquals(0, exec("def x = (short)2; def y = (char)2; return x % y"));
+        assertEquals(0, exec("def x = (char)2; def y = (char)2; return x % y"));
+        assertEquals(0, exec("def x = (int)2; def y = (char)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; def y = (char)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; def y = (char)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; def y = (char)2; return x % y"));
+
+        assertEquals(0, exec("def x = (byte)2; def y = (int)2; return x % y"));
+        assertEquals(0, exec("def x = (short)2; def y = (int)2; return x % y"));
+        assertEquals(0, exec("def x = (char)2; def y = (int)2; return x % y"));
+        assertEquals(0, exec("def x = (int)2; def y = (int)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; def y = (int)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; def y = (int)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; def y = (int)2; return x % y"));
+
+        assertEquals(0L, exec("def x = (byte)2; def y = (long)2; return x % y"));
+        assertEquals(0L, exec("def x = (short)2; def y = (long)2; return x % y"));
+        assertEquals(0L, exec("def x = (char)2; def y = (long)2; return x % y"));
+        assertEquals(0L, exec("def x = (int)2; def y = (long)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; def y = (long)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; def y = (long)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; def y = (long)2; return x % y"));
+
+        assertEquals(0F, exec("def x = (byte)2; def y = (float)2; return x % y"));
+        assertEquals(0F, exec("def x = (short)2; def y = (float)2; return x % y"));
+        assertEquals(0F, exec("def x = (char)2; def y = (float)2; return x % y"));
+        assertEquals(0F, exec("def x = (int)2; def y = (float)2; return x % y"));
+        assertEquals(0F, exec("def x = (long)2; def y = (float)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; def y = (float)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; def y = (float)2; return x % y"));
+
+        assertEquals(0D, exec("def x = (byte)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (short)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (char)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (int)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (long)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (float)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; def y = (double)2; return x % y"));
+
+        assertEquals(0, exec("def x = (byte)2; def y = (byte)2; return x % y"));
+        assertEquals(0, exec("def x = (short)2; def y = (short)2; return x % y"));
+        assertEquals(0, exec("def x = (char)2; def y = (char)2; return x % y"));
+        assertEquals(0, exec("def x = (int)2; def y = (int)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; def y = (long)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; def y = (float)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; def y = (double)2; return x % y"));
+    }
+    
+    public void testDefTypedLHS() {
+        assertEquals(0, exec("byte x = (byte)2; def y = (byte)2; return x % y"));
+        assertEquals(0, exec("short x = (short)2; def y = (byte)2; return x % y"));
+        assertEquals(0, exec("char x = (char)2; def y = (byte)2; return x % y"));
+        assertEquals(0, exec("int x = (int)2; def y = (byte)2; return x % y"));
+        assertEquals(0L, exec("long x = (long)2; def y = (byte)2; return x % y"));
+        assertEquals(0F, exec("float x = (float)2; def y = (byte)2; return x % y"));
+        assertEquals(0D, exec("double x = (double)2; def y = (byte)2; return x % y"));
+
+        assertEquals(0, exec("byte x = (byte)2; def y = (short)2; return x % y"));
+        assertEquals(0, exec("short x = (short)2; def y = (short)2; return x % y"));
+        assertEquals(0, exec("char x = (char)2; def y = (short)2; return x % y"));
+        assertEquals(0, exec("int x = (int)2; def y = (short)2; return x % y"));
+        assertEquals(0L, exec("long x = (long)2; def y = (short)2; return x % y"));
+        assertEquals(0F, exec("float x = (float)2; def y = (short)2; return x % y"));
+        assertEquals(0D, exec("double x = (double)2; def y = (short)2; return x % y"));
+
+        assertEquals(0, exec("byte x = (byte)2; def y = (char)2; return x % y"));
+        assertEquals(0, exec("short x = (short)2; def y = (char)2; return x % y"));
+        assertEquals(0, exec("char x = (char)2; def y = (char)2; return x % y"));
+        assertEquals(0, exec("int x = (int)2; def y = (char)2; return x % y"));
+        assertEquals(0L, exec("long x = (long)2; def y = (char)2; return x % y"));
+        assertEquals(0F, exec("float x = (float)2; def y = (char)2; return x % y"));
+        assertEquals(0D, exec("double x = (double)2; def y = (char)2; return x % y"));
+
+        assertEquals(0, exec("byte x = (byte)2; def y = (int)2; return x % y"));
+        assertEquals(0, exec("short x = (short)2; def y = (int)2; return x % y"));
+        assertEquals(0, exec("char x = (char)2; def y = (int)2; return x % y"));
+        assertEquals(0, exec("int x = (int)2; def y = (int)2; return x % y"));
+        assertEquals(0L, exec("long x = (long)2; def y = (int)2; return x % y"));
+        assertEquals(0F, exec("float x = (float)2; def y = (int)2; return x % y"));
+        assertEquals(0D, exec("double x = (double)2; def y = (int)2; return x % y"));
+
+        assertEquals(0L, exec("byte x = (byte)2; def y = (long)2; return x % y"));
+        assertEquals(0L, exec("short x = (short)2; def y = (long)2; return x % y"));
+        assertEquals(0L, exec("char x = (char)2; def y = (long)2; return x % y"));
+        assertEquals(0L, exec("int x = (int)2; def y = (long)2; return x % y"));
+        assertEquals(0L, exec("long x = (long)2; def y = (long)2; return x % y"));
+        assertEquals(0F, exec("float x = (float)2; def y = (long)2; return x % y"));
+        assertEquals(0D, exec("double x = (double)2; def y = (long)2; return x % y"));
+
+        assertEquals(0F, exec("byte x = (byte)2; def y = (float)2; return x % y"));
+        assertEquals(0F, exec("short x = (short)2; def y = (float)2; return x % y"));
+        assertEquals(0F, exec("char x = (char)2; def y = (float)2; return x % y"));
+        assertEquals(0F, exec("int x = (int)2; def y = (float)2; return x % y"));
+        assertEquals(0F, exec("long x = (long)2; def y = (float)2; return x % y"));
+        assertEquals(0F, exec("float x = (float)2; def y = (float)2; return x % y"));
+        assertEquals(0D, exec("double x = (double)2; def y = (float)2; return x % y"));
+
+        assertEquals(0D, exec("byte x = (byte)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("short x = (short)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("char x = (char)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("int x = (int)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("long x = (long)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("float x = (float)2; def y = (double)2; return x % y"));
+        assertEquals(0D, exec("double x = (double)2; def y = (double)2; return x % y"));
+
+        assertEquals(0, exec("byte x = (byte)2; def y = (byte)2; return x % y"));
+        assertEquals(0, exec("short x = (short)2; def y = (short)2; return x % y"));
+        assertEquals(0, exec("char x = (char)2; def y = (char)2; return x % y"));
+        assertEquals(0, exec("int x = (int)2; def y = (int)2; return x % y"));
+        assertEquals(0L, exec("long x = (long)2; def y = (long)2; return x % y"));
+        assertEquals(0F, exec("float x = (float)2; def y = (float)2; return x % y"));
+        assertEquals(0D, exec("double x = (double)2; def y = (double)2; return x % y"));
+    }
+
+    public void testDefTypedRHS() {
+        assertEquals(0, exec("def x = (byte)2; byte y = (byte)2; return x % y"));
+        assertEquals(0, exec("def x = (short)2; byte y = (byte)2; return x % y"));
+        assertEquals(0, exec("def x = (char)2; byte y = (byte)2; return x % y"));
+        assertEquals(0, exec("def x = (int)2; byte y = (byte)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; byte y = (byte)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; byte y = (byte)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; byte y = (byte)2; return x % y"));
+
+        assertEquals(0, exec("def x = (byte)2; short y = (short)2; return x % y"));
+        assertEquals(0, exec("def x = (short)2; short y = (short)2; return x % y"));
+        assertEquals(0, exec("def x = (char)2; short y = (short)2; return x % y"));
+        assertEquals(0, exec("def x = (int)2; short y = (short)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; short y = (short)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; short y = (short)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; short y = (short)2; return x % y"));
+
+        assertEquals(0, exec("def x = (byte)2; char y = (char)2; return x % y"));
+        assertEquals(0, exec("def x = (short)2; char y = (char)2; return x % y"));
+        assertEquals(0, exec("def x = (char)2; char y = (char)2; return x % y"));
+        assertEquals(0, exec("def x = (int)2; char y = (char)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; char y = (char)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; char y = (char)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; char y = (char)2; return x % y"));
+
+        assertEquals(0, exec("def x = (byte)2; int y = (int)2; return x % y"));
+        assertEquals(0, exec("def x = (short)2; int y = (int)2; return x % y"));
+        assertEquals(0, exec("def x = (char)2; int y = (int)2; return x % y"));
+        assertEquals(0, exec("def x = (int)2; int y = (int)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; int y = (int)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; int y = (int)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; int y = (int)2; return x % y"));
+
+        assertEquals(0L, exec("def x = (byte)2; long y = (long)2; return x % y"));
+        assertEquals(0L, exec("def x = (short)2; long y = (long)2; return x % y"));
+        assertEquals(0L, exec("def x = (char)2; long y = (long)2; return x % y"));
+        assertEquals(0L, exec("def x = (int)2; long y = (long)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; long y = (long)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; long y = (long)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; long y = (long)2; return x % y"));
+
+        assertEquals(0F, exec("def x = (byte)2; float y = (float)2; return x % y"));
+        assertEquals(0F, exec("def x = (short)2; float y = (float)2; return x % y"));
+        assertEquals(0F, exec("def x = (char)2; float y = (float)2; return x % y"));
+        assertEquals(0F, exec("def x = (int)2; float y = (float)2; return x % y"));
+        assertEquals(0F, exec("def x = (long)2; float y = (float)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; float y = (float)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; float y = (float)2; return x % y"));
+
+        assertEquals(0D, exec("def x = (byte)2; double y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (short)2; double y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (char)2; double y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (int)2; double y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (long)2; double y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (float)2; double y = (double)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; double y = (double)2; return x % y"));
+
+        assertEquals(0, exec("def x = (byte)2; byte y = (byte)2; return x % y"));
+        assertEquals(0, exec("def x = (short)2; short y = (short)2; return x % y"));
+        assertEquals(0, exec("def x = (char)2; char y = (char)2; return x % y"));
+        assertEquals(0, exec("def x = (int)2; int y = (int)2; return x % y"));
+        assertEquals(0L, exec("def x = (long)2; long y = (long)2; return x % y"));
+        assertEquals(0F, exec("def x = (float)2; float y = (float)2; return x % y"));
+        assertEquals(0D, exec("def x = (double)2; double y = (double)2; return x % y"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
index b4e416ca614..cb35b53db5c 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
@@ -76,7 +76,7 @@ public abstract class ScriptTestCase extends ESTestCase {
      */
     public void assertBytecodeExists(String script, String bytecode) {
         final String asm = Debugger.toString(script);
-        assertTrue("bytecode not found", asm.contains(bytecode));
+        assertTrue("bytecode not found, got: \n" + asm , asm.contains(bytecode));
     }
     
     /** Checks a specific exception class is thrown (boxed inside ScriptException) and returns it. */
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ShiftTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ShiftTests.java
new file mode 100644
index 00000000000..d409c8c5f7f
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ShiftTests.java
@@ -0,0 +1,547 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+/** Tests for shift operator across all types */
+public class ShiftTests extends ScriptTestCase {
+
+    public void testBasics() {
+        assertEquals(1 << 2, exec("return 1 << 2;"));
+        assertEquals(4 >> 2, exec("return 4 >> 2;"));
+        assertEquals(-1 >>> 29, exec("return -1 >>> 29;"));
+        assertEquals(4, exec("int x = 1; char y = 2; return x << y;"));
+        assertEquals(-1, exec("int x = -1; char y = 29; return x >> y;"));
+        assertEquals(3, exec("int x = -1; char y = 30; return x >>> y;"));
+    }
+
+    public void testLongShifts() {
+        assertEquals(1L << 2, exec("long x = 1L; int y = 2; return x << y;"));
+        assertEquals(1 << 2L, exec("int x = 1; long y = 2L; return x << y;"));
+        assertEquals(4 >> 2L, exec("int x = 4; long y = 2L; return x >> y;"));
+        assertEquals(4L >> 2, exec("long x = 4L; int y = 2; return x >> y;"));
+        assertEquals(-1L >>> 29, exec("long x = -1L; int y = 29; return x >>> y;"));
+        assertEquals(-1 >>> 29L, exec("int x = -1; long y = 29L; return x >>> y;"));
+    }
+    
+    public void testLongShiftsConst() {
+        assertEquals(1L << 2, exec("return 1L << 2;"));
+        assertEquals(1 << 2L, exec("return 1 << 2L;"));
+        assertEquals(4 >> 2L, exec("return 4 >> 2L;"));
+        assertEquals(4L >> 2, exec("return 4L >> 2;"));
+        assertEquals(-1L >>> 29, exec("return -1L >>> 29;"));
+        assertEquals(-1 >>> 29L, exec("return -1 >>> 29L;"));
+    }
+    
+    public void testBogusShifts() {
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("long x = 1L; float y = 2; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("int x = 1; double y = 2L; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("float x = 1F; int y = 2; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("double x = 1D; int y = 2L; return x << y;");
+        });
+    }
+
+    public void testBogusShiftsConst() {
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("return 1L << 2F;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("return 1L << 2.0;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("return 1F << 2;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("return 1D << 2L");
+        });
+    }
+    
+    public void testLshDef() {
+        assertEquals(2, exec("def x = (byte)1; def y = (byte)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; def y = (byte)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; def y = (byte)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; def y = (byte)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (byte)1; return x << y"));
+
+        assertEquals(2, exec("def x = (byte)1; def y = (short)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; def y = (short)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; def y = (short)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; def y = (short)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (short)1; return x << y"));
+
+        assertEquals(2, exec("def x = (byte)1; def y = (char)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; def y = (char)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; def y = (char)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; def y = (char)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (char)1; return x << y"));
+
+        assertEquals(2, exec("def x = (byte)1; def y = (int)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; def y = (int)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; def y = (int)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; def y = (int)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (int)1; return x << y"));
+
+        assertEquals(2, exec("def x = (byte)1; def y = (long)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; def y = (long)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; def y = (long)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; def y = (long)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (long)1; return x << y"));
+
+        assertEquals(2, exec("def x = (byte)1; def y = (byte)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; def y = (short)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; def y = (char)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; def y = (int)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; def y = (long)1; return x << y"));
+    }
+    
+    public void testLshDefTypedLHS() {
+        assertEquals(2, exec("byte x = (byte)1; def y = (byte)1; return x << y"));
+        assertEquals(2, exec("short x = (short)1; def y = (byte)1; return x << y"));
+        assertEquals(2, exec("char x = (char)1; def y = (byte)1; return x << y"));
+        assertEquals(2, exec("int x = (int)1; def y = (byte)1; return x << y"));
+        assertEquals(2L, exec("long x = (long)1; def y = (byte)1; return x << y"));
+
+        assertEquals(2, exec("byte x = (byte)1; def y = (short)1; return x << y"));
+        assertEquals(2, exec("short x = (short)1; def y = (short)1; return x << y"));
+        assertEquals(2, exec("char x = (char)1; def y = (short)1; return x << y"));
+        assertEquals(2, exec("int x = (int)1; def y = (short)1; return x << y"));
+        assertEquals(2L, exec("long x = (long)1; def y = (short)1; return x << y"));
+
+        assertEquals(2, exec("byte x = (byte)1; def y = (char)1; return x << y"));
+        assertEquals(2, exec("short x = (short)1; def y = (char)1; return x << y"));
+        assertEquals(2, exec("char x = (char)1; def y = (char)1; return x << y"));
+        assertEquals(2, exec("int x = (int)1; def y = (char)1; return x << y"));
+        assertEquals(2L, exec("long x = (long)1; def y = (char)1; return x << y"));
+
+        assertEquals(2, exec("byte x = (byte)1; def y = (int)1; return x << y"));
+        assertEquals(2, exec("short x = (short)1; def y = (int)1; return x << y"));
+        assertEquals(2, exec("char x = (char)1; def y = (int)1; return x << y"));
+        assertEquals(2, exec("int x = (int)1; def y = (int)1; return x << y"));
+        assertEquals(2L, exec("long x = (long)1; def y = (int)1; return x << y"));
+
+        assertEquals(2, exec("byte x = (byte)1; def y = (long)1; return x << y"));
+        assertEquals(2, exec("short x = (short)1; def y = (long)1; return x << y"));
+        assertEquals(2, exec("char x = (char)1; def y = (long)1; return x << y"));
+        assertEquals(2, exec("int x = (int)1; def y = (long)1; return x << y"));
+        assertEquals(2L, exec("long x = (long)1; def y = (long)1; return x << y"));
+
+        assertEquals(2, exec("byte x = (byte)1; def y = (byte)1; return x << y"));
+        assertEquals(2, exec("short x = (short)1; def y = (short)1; return x << y"));
+        assertEquals(2, exec("char x = (char)1; def y = (char)1; return x << y"));
+        assertEquals(2, exec("int x = (int)1; def y = (int)1; return x << y"));
+        assertEquals(2L, exec("long x = (long)1; def y = (long)1; return x << y"));
+    }
+    
+    public void testLshDefTypedRHS() {
+        assertEquals(2, exec("def x = (byte)1; byte y = (byte)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; byte y = (byte)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; byte y = (byte)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; byte y = (byte)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; byte y = (byte)1; return x << y"));
+
+        assertEquals(2, exec("def x = (byte)1; short y = (short)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; short y = (short)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; short y = (short)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; short y = (short)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; short y = (short)1; return x << y"));
+
+        assertEquals(2, exec("def x = (byte)1; char y = (char)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; char y = (char)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; char y = (char)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; char y = (char)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; char y = (char)1; return x << y"));
+
+        assertEquals(2, exec("def x = (byte)1; int y = (int)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; int y = (int)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; int y = (int)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; int y = (int)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; int y = (int)1; return x << y"));
+
+        assertEquals(2, exec("def x = (byte)1; long y = (long)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; long y = (long)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; long y = (long)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; long y = (long)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; long y = (long)1; return x << y"));
+
+        assertEquals(2, exec("def x = (byte)1; byte y = (byte)1; return x << y"));
+        assertEquals(2, exec("def x = (short)1; short y = (short)1; return x << y"));
+        assertEquals(2, exec("def x = (char)1; char y = (char)1; return x << y"));
+        assertEquals(2, exec("def x = (int)1; int y = (int)1; return x << y"));
+        assertEquals(2L, exec("def x = (long)1; long y = (long)1; return x << y"));
+    }
+
+    public void testRshDef() {
+        assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (byte)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (byte)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (byte)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (byte)1; return x >> y"));
+
+        assertEquals(2, exec("def x = (byte)4; def y = (short)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (short)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (short)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (short)1; return x >> y"));
+
+        assertEquals(2, exec("def x = (byte)4; def y = (char)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (char)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (char)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (char)1; return x >> y"));
+
+        assertEquals(2, exec("def x = (byte)4; def y = (int)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (int)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (int)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (int)1; return x >> y"));
+
+        assertEquals(2, exec("def x = (byte)4; def y = (long)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (long)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (long)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (long)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >> y"));
+
+        assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >> y"));
+    }
+    
+    public void testRshDefTypeLHS() {
+        assertEquals(2, exec("byte x = (byte)4; def y = (byte)1; return x >> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (byte)1; return x >> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (byte)1; return x >> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (byte)1; return x >> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (byte)1; return x >> y"));
+
+        assertEquals(2, exec("byte x = (byte)4; def y = (short)1; return x >> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (short)1; return x >> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (short)1; return x >> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (short)1; return x >> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (short)1; return x >> y"));
+
+        assertEquals(2, exec("byte x = (byte)4; def y = (char)1; return x >> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (char)1; return x >> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (char)1; return x >> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (char)1; return x >> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (char)1; return x >> y"));
+
+        assertEquals(2, exec("byte x = (byte)4; def y = (int)1; return x >> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (int)1; return x >> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (int)1; return x >> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (int)1; return x >> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (int)1; return x >> y"));
+
+        assertEquals(2, exec("byte x = (byte)4; def y = (long)1; return x >> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (long)1; return x >> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (long)1; return x >> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (long)1; return x >> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (long)1; return x >> y"));
+
+        assertEquals(2, exec("byte x = (byte)4; def y = (byte)1; return x >> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (short)1; return x >> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (char)1; return x >> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (int)1; return x >> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (long)1; return x >> y"));
+    }
+    
+    public void testRshDefTypedLHS() {
+        assertEquals(2, exec("def x = (byte)4; byte y = (byte)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; byte y = (byte)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; byte y = (byte)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; byte y = (byte)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; byte y = (byte)1; return x >> y"));
+
+        assertEquals(2, exec("def x = (byte)4; short y = (short)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; short y = (short)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; short y = (short)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; short y = (short)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; short y = (short)1; return x >> y"));
+
+        assertEquals(2, exec("def x = (byte)4; char y = (char)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; char y = (char)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; char y = (char)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; char y = (char)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; char y = (char)1; return x >> y"));
+
+        assertEquals(2, exec("def x = (byte)4; int y = (int)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; int y = (int)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; int y = (int)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; int y = (int)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; int y = (int)1; return x >> y"));
+
+        assertEquals(2, exec("def x = (byte)4; long y = (long)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; long y = (long)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; long y = (long)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; long y = (long)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; long y = (long)1; return x >> y"));
+
+        assertEquals(2, exec("def x = (byte)4; byte y = (byte)1; return x >> y"));
+        assertEquals(2, exec("def x = (short)4; short y = (short)1; return x >> y"));
+        assertEquals(2, exec("def x = (char)4; char y = (char)1; return x >> y"));
+        assertEquals(2, exec("def x = (int)4; int y = (int)1; return x >> y"));
+        assertEquals(2L, exec("def x = (long)4; long y = (long)1; return x >> y"));
+    }
+
+    public void testUshDef() {
+        assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (byte)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (byte)1; return x >>> y"));
+
+        assertEquals(2, exec("def x = (byte)4; def y = (short)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (short)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (short)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (short)1; return x >>> y"));
+
+        assertEquals(2, exec("def x = (byte)4; def y = (char)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (char)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (char)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (char)1; return x >>> y"));
+
+        assertEquals(2, exec("def x = (byte)4; def y = (int)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (int)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (int)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (int)1; return x >>> y"));
+
+        assertEquals(2, exec("def x = (byte)4; def y = (long)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (long)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (long)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (long)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >>> y"));
+
+        assertEquals(2, exec("def x = (byte)4; def y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; def y = (short)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; def y = (char)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; def y = (int)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; def y = (long)1; return x >>> y"));
+    }
+    
+    public void testUshDefTypedLHS() {
+        assertEquals(2, exec("byte x = (byte)4; def y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (byte)1; return x >>> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (byte)1; return x >>> y"));
+
+        assertEquals(2, exec("byte x = (byte)4; def y = (short)1; return x >>> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (short)1; return x >>> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (short)1; return x >>> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (short)1; return x >>> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (short)1; return x >>> y"));
+
+        assertEquals(2, exec("byte x = (byte)4; def y = (char)1; return x >>> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (char)1; return x >>> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (char)1; return x >>> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (char)1; return x >>> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (char)1; return x >>> y"));
+
+        assertEquals(2, exec("byte x = (byte)4; def y = (int)1; return x >>> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (int)1; return x >>> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (int)1; return x >>> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (int)1; return x >>> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (int)1; return x >>> y"));
+
+        assertEquals(2, exec("byte x = (byte)4; def y = (long)1; return x >>> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (long)1; return x >>> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (long)1; return x >>> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (long)1; return x >>> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (long)1; return x >>> y"));
+
+        assertEquals(2, exec("byte x = (byte)4; def y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("short x = (short)4; def y = (short)1; return x >>> y"));
+        assertEquals(2, exec("char x = (char)4; def y = (char)1; return x >>> y"));
+        assertEquals(2, exec("int x = (int)4; def y = (int)1; return x >>> y"));
+        assertEquals(2L, exec("long x = (long)4; def y = (long)1; return x >>> y"));
+    }
+    
+    public void testUshDefTypedRHS() {
+        assertEquals(2, exec("def x = (byte)4; byte y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; byte y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; byte y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; byte y = (byte)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; byte y = (byte)1; return x >>> y"));
+
+        assertEquals(2, exec("def x = (byte)4; short y = (short)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; short y = (short)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; short y = (short)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; short y = (short)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; short y = (short)1; return x >>> y"));
+
+        assertEquals(2, exec("def x = (byte)4; char y = (char)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; char y = (char)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; char y = (char)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; char y = (char)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; char y = (char)1; return x >>> y"));
+
+        assertEquals(2, exec("def x = (byte)4; int y = (int)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; int y = (int)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; int y = (int)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; int y = (int)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; int y = (int)1; return x >>> y"));
+
+        assertEquals(2, exec("def x = (byte)4; long y = (long)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; long y = (long)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; long y = (long)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; long y = (long)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; long y = (long)1; return x >>> y"));
+
+        assertEquals(2, exec("def x = (byte)4; byte y = (byte)1; return x >>> y"));
+        assertEquals(2, exec("def x = (short)4; short y = (short)1; return x >>> y"));
+        assertEquals(2, exec("def x = (char)4; char y = (char)1; return x >>> y"));
+        assertEquals(2, exec("def x = (int)4; int y = (int)1; return x >>> y"));
+        assertEquals(2L, exec("def x = (long)4; long y = (long)1; return x >>> y"));
+    }
+    
+    public void testBogusDefShifts() {
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1L; def y = 2F; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1; def y = 2D; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1F; def y = 2; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1D; def y = 2L; return x << y;");
+        });
+        
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1L; def y = 2F; return x >> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1; def y = 2D; return x >> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1F; def y = 2; return x >> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1D; def y = 2L; return x >> y;");
+        });
+        
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1L; def y = 2F; return x >>> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1; def y = 2D; return x >>> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1F; def y = 2; return x >>> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1D; def y = 2L; return x >>> y;");
+        });
+    }
+    
+    public void testBogusDefShiftsTypedLHS() {
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("long x = 1L; def y = 2F; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("int x = 1; def y = 2D; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("float x = 1F; def y = 2; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("double x = 1D; def y = 2L; return x << y;");
+        });
+        
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("long x = 1L; def y = 2F; return x >> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("int x = 1; def y = 2D; return x >> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("float x = 1F; def y = 2; return x >> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("double x = 1D; def y = 2L; return x >> y;");
+        });
+        
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("long x = 1L; def y = 2F; return x >>> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("int x = 1; def y = 2D; return x >>> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("float x = 1F; def y = 2; return x >>> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("double x = 1D; def y = 2L; return x >>> y;");
+        });
+    }
+    
+    public void testBogusDefShiftsTypedRHS() {
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1L; float y = 2F; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1; double y = 2D; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1F; int y = 2; return x << y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1D; long y = 2L; return x << y;");
+        });
+        
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1L; float y = 2F; return x >> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1; double y = 2D; return x >> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1F; int y = 2; return x >> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1D; long y = 2L; return x >> y;");
+        });
+        
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1L; float y = 2F; return x >>> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1; double y = 2D; return x >>> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1F; int y = 2; return x >>> y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1D; long y = 2L; return x >>> y;");
+        });
+    }
+
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
index 12d275de6e5..f42dd5f7c98 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
@@ -182,4 +182,27 @@ public class StringTests extends ScriptTestCase {
         });
         assertTrue(expected.getMessage().contains("Cannot cast [String] with length greater than one to [char]."));
     }
+    
+    public void testDefConcat() {
+        assertEquals("a" + (byte)2, exec("def x = 'a'; def y = (byte)2; return x + y"));
+        assertEquals("a" + (short)2, exec("def x = 'a'; def y = (short)2; return x + y"));
+        assertEquals("a" + (char)2, exec("def x = 'a'; def y = (char)2; return x + y"));
+        assertEquals("a" + 2, exec("def x = 'a'; def y = (int)2; return x + y"));
+        assertEquals("a" + 2L, exec("def x = 'a'; def y = (long)2; return x + y"));
+        assertEquals("a" + 2F, exec("def x = 'a'; def y = (float)2; return x + y"));
+        assertEquals("a" + 2D, exec("def x = 'a'; def y = (double)2; return x + y"));
+        assertEquals("ab", exec("def x = 'a'; def y = 'b'; return x + y"));
+        assertEquals((byte)2 + "a", exec("def x = 'a'; def y = (byte)2; return y + x"));
+        assertEquals((short)2 + "a", exec("def x = 'a'; def y = (short)2; return y + x"));
+        assertEquals((char)2 + "a", exec("def x = 'a'; def y = (char)2; return y + x"));
+        assertEquals(2 + "a", exec("def x = 'a'; def y = (int)2; return y + x"));
+        assertEquals(2L + "a", exec("def x = 'a'; def y = (long)2; return y + x"));
+        assertEquals(2F + "a", exec("def x = 'a'; def y = (float)2; return y + x"));
+        assertEquals(2D + "a", exec("def x = 'a'; def y = (double)2; return y + x"));
+        assertEquals("anull", exec("def x = 'a'; def y = null; return x + y"));
+        assertEquals("nullb", exec("def x = null; def y = 'b'; return x + y"));
+        expectScriptThrows(NullPointerException.class, () -> {
+            exec("def x = null; def y = null; return x + y");
+        });
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/SubtractionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/SubtractionTests.java
index f6a14b175e6..4817b040326 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/SubtractionTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/SubtractionTests.java
@@ -22,6 +22,11 @@ package org.elasticsearch.painless;
 /** Tests for subtraction operator across all types */
 //TODO: NaN/Inf/overflow/...
 public class SubtractionTests extends ScriptTestCase {
+    
+    public void testBasics() throws Exception {
+        assertEquals(2 - 1, exec("return 2 - 1;"));
+        assertEquals(-1, exec("int x = 1; char y = 2; return x - y;"));
+    }
 
     public void testInt() throws Exception {
         assertEquals(1-1, exec("int x = 1; int y = 1; return x-y;"));
@@ -176,4 +181,178 @@ public class SubtractionTests extends ScriptTestCase {
         assertEquals(10.0-0.0, exec("return 10.0-0.0;"));
         assertEquals(0.0-0.0, exec("return 0.0-0.0;"));
     }
+    
+    public void testDef() {
+        assertEquals(0, exec("def x = (byte)1; def y = (byte)1; return x - y"));
+        assertEquals(0, exec("def x = (short)1; def y = (byte)1; return x - y"));
+        assertEquals(0, exec("def x = (char)1; def y = (byte)1; return x - y"));
+        assertEquals(0, exec("def x = (int)1; def y = (byte)1; return x - y"));
+        assertEquals(0L, exec("def x = (long)1; def y = (byte)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; def y = (byte)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; def y = (byte)1; return x - y"));
+
+        assertEquals(0, exec("def x = (byte)1; def y = (short)1; return x - y"));
+        assertEquals(0, exec("def x = (short)1; def y = (short)1; return x - y"));
+        assertEquals(0, exec("def x = (char)1; def y = (short)1; return x - y"));
+        assertEquals(0, exec("def x = (int)1; def y = (short)1; return x - y"));
+        assertEquals(0L, exec("def x = (long)1; def y = (short)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; def y = (short)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; def y = (short)1; return x - y"));
+
+        assertEquals(0, exec("def x = (byte)1; def y = (char)1; return x - y"));
+        assertEquals(0, exec("def x = (short)1; def y = (char)1; return x - y"));
+        assertEquals(0, exec("def x = (char)1; def y = (char)1; return x - y"));
+        assertEquals(0, exec("def x = (int)1; def y = (char)1; return x - y"));
+        assertEquals(0L, exec("def x = (long)1; def y = (char)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; def y = (char)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; def y = (char)1; return x - y"));
+
+        assertEquals(0, exec("def x = (byte)1; def y = (int)1; return x - y"));
+        assertEquals(0, exec("def x = (short)1; def y = (int)1; return x - y"));
+        assertEquals(0, exec("def x = (char)1; def y = (int)1; return x - y"));
+        assertEquals(0, exec("def x = (int)1; def y = (int)1; return x - y"));
+        assertEquals(0L, exec("def x = (long)1; def y = (int)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; def y = (int)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; def y = (int)1; return x - y"));
+
+        assertEquals(0L, exec("def x = (byte)1; def y = (long)1; return x - y"));
+        assertEquals(0L, exec("def x = (short)1; def y = (long)1; return x - y"));
+        assertEquals(0L, exec("def x = (char)1; def y = (long)1; return x - y"));
+        assertEquals(0L, exec("def x = (int)1; def y = (long)1; return x - y"));
+        assertEquals(0L, exec("def x = (long)1; def y = (long)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; def y = (long)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; def y = (long)1; return x - y"));
+
+        assertEquals(0F, exec("def x = (byte)1; def y = (float)1; return x - y"));
+        assertEquals(0F, exec("def x = (short)1; def y = (float)1; return x - y"));
+        assertEquals(0F, exec("def x = (char)1; def y = (float)1; return x - y"));
+        assertEquals(0F, exec("def x = (int)1; def y = (float)1; return x - y"));
+        assertEquals(0F, exec("def x = (long)1; def y = (float)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; def y = (float)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; def y = (float)1; return x - y"));
+
+        assertEquals(0D, exec("def x = (byte)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (short)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (char)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (int)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (long)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (float)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; def y = (double)1; return x - y"));
+    }
+    
+    public void testDefTypedLHS() {
+        assertEquals(0, exec("byte x = (byte)1; def y = (byte)1; return x - y"));
+        assertEquals(0, exec("short x = (short)1; def y = (byte)1; return x - y"));
+        assertEquals(0, exec("char x = (char)1; def y = (byte)1; return x - y"));
+        assertEquals(0, exec("int x = (int)1; def y = (byte)1; return x - y"));
+        assertEquals(0L, exec("long x = (long)1; def y = (byte)1; return x - y"));
+        assertEquals(0F, exec("float x = (float)1; def y = (byte)1; return x - y"));
+        assertEquals(0D, exec("double x = (double)1; def y = (byte)1; return x - y"));
+
+        assertEquals(0, exec("byte x = (byte)1; def y = (short)1; return x - y"));
+        assertEquals(0, exec("short x = (short)1; def y = (short)1; return x - y"));
+        assertEquals(0, exec("char x = (char)1; def y = (short)1; return x - y"));
+        assertEquals(0, exec("int x = (int)1; def y = (short)1; return x - y"));
+        assertEquals(0L, exec("long x = (long)1; def y = (short)1; return x - y"));
+        assertEquals(0F, exec("float x = (float)1; def y = (short)1; return x - y"));
+        assertEquals(0D, exec("double x = (double)1; def y = (short)1; return x - y"));
+
+        assertEquals(0, exec("byte x = (byte)1; def y = (char)1; return x - y"));
+        assertEquals(0, exec("short x = (short)1; def y = (char)1; return x - y"));
+        assertEquals(0, exec("char x = (char)1; def y = (char)1; return x - y"));
+        assertEquals(0, exec("int x = (int)1; def y = (char)1; return x - y"));
+        assertEquals(0L, exec("long x = (long)1; def y = (char)1; return x - y"));
+        assertEquals(0F, exec("float x = (float)1; def y = (char)1; return x - y"));
+        assertEquals(0D, exec("double x = (double)1; def y = (char)1; return x - y"));
+
+        assertEquals(0, exec("byte x = (byte)1; def y = (int)1; return x - y"));
+        assertEquals(0, exec("short x = (short)1; def y = (int)1; return x - y"));
+        assertEquals(0, exec("char x = (char)1; def y = (int)1; return x - y"));
+        assertEquals(0, exec("int x = (int)1; def y = (int)1; return x - y"));
+        assertEquals(0L, exec("long x = (long)1; def y = (int)1; return x - y"));
+        assertEquals(0F, exec("float x = (float)1; def y = (int)1; return x - y"));
+        assertEquals(0D, exec("double x = (double)1; def y = (int)1; return x - y"));
+
+        assertEquals(0L, exec("byte x = (byte)1; def y = (long)1; return x - y"));
+        assertEquals(0L, exec("short x = (short)1; def y = (long)1; return x - y"));
+        assertEquals(0L, exec("char x = (char)1; def y = (long)1; return x - y"));
+        assertEquals(0L, exec("int x = (int)1; def y = (long)1; return x - y"));
+        assertEquals(0L, exec("long x = (long)1; def y = (long)1; return x - y"));
+        assertEquals(0F, exec("float x = (float)1; def y = (long)1; return x - y"));
+        assertEquals(0D, exec("double x = (double)1; def y = (long)1; return x - y"));
+
+        assertEquals(0F, exec("byte x = (byte)1; def y = (float)1; return x - y"));
+        assertEquals(0F, exec("short x = (short)1; def y = (float)1; return x - y"));
+        assertEquals(0F, exec("char x = (char)1; def y = (float)1; return x - y"));
+        assertEquals(0F, exec("int x = (int)1; def y = (float)1; return x - y"));
+        assertEquals(0F, exec("long x = (long)1; def y = (float)1; return x - y"));
+        assertEquals(0F, exec("float x = (float)1; def y = (float)1; return x - y"));
+        assertEquals(0D, exec("double x = (double)1; def y = (float)1; return x - y"));
+
+        assertEquals(0D, exec("byte x = (byte)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("short x = (short)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("char x = (char)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("int x = (int)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("long x = (long)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("float x = (float)1; def y = (double)1; return x - y"));
+        assertEquals(0D, exec("double x = (double)1; def y = (double)1; return x - y"));
+    }
+    
+    public void testDefTypedRHS() {
+        assertEquals(0, exec("def x = (byte)1; byte y = (byte)1; return x - y"));
+        assertEquals(0, exec("def x = (short)1; byte y = (byte)1; return x - y"));
+        assertEquals(0, exec("def x = (char)1; byte y = (byte)1; return x - y"));
+        assertEquals(0, exec("def x = (int)1; byte y = (byte)1; return x - y"));
+        assertEquals(0L, exec("def x = (long)1; byte y = (byte)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; byte y = (byte)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; byte y = (byte)1; return x - y"));
+
+        assertEquals(0, exec("def x = (byte)1; short y = (short)1; return x - y"));
+        assertEquals(0, exec("def x = (short)1; short y = (short)1; return x - y"));
+        assertEquals(0, exec("def x = (char)1; short y = (short)1; return x - y"));
+        assertEquals(0, exec("def x = (int)1; short y = (short)1; return x - y"));
+        assertEquals(0L, exec("def x = (long)1; short y = (short)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; short y = (short)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; short y = (short)1; return x - y"));
+
+        assertEquals(0, exec("def x = (byte)1; char y = (char)1; return x - y"));
+        assertEquals(0, exec("def x = (short)1; char y = (char)1; return x - y"));
+        assertEquals(0, exec("def x = (char)1; char y = (char)1; return x - y"));
+        assertEquals(0, exec("def x = (int)1; char y = (char)1; return x - y"));
+        assertEquals(0L, exec("def x = (long)1; char y = (char)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; char y = (char)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; char y = (char)1; return x - y"));
+
+        assertEquals(0, exec("def x = (byte)1; int y = (int)1; return x - y"));
+        assertEquals(0, exec("def x = (short)1; int y = (int)1; return x - y"));
+        assertEquals(0, exec("def x = (char)1; int y = (int)1; return x - y"));
+        assertEquals(0, exec("def x = (int)1; int y = (int)1; return x - y"));
+        assertEquals(0L, exec("def x = (long)1; int y = (int)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; int y = (int)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; int y = (int)1; return x - y"));
+
+        assertEquals(0L, exec("def x = (byte)1; long y = (long)1; return x - y"));
+        assertEquals(0L, exec("def x = (short)1; long y = (long)1; return x - y"));
+        assertEquals(0L, exec("def x = (char)1; long y = (long)1; return x - y"));
+        assertEquals(0L, exec("def x = (int)1; long y = (long)1; return x - y"));
+        assertEquals(0L, exec("def x = (long)1; long y = (long)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; long y = (long)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; long y = (long)1; return x - y"));
+
+        assertEquals(0F, exec("def x = (byte)1; float y = (float)1; return x - y"));
+        assertEquals(0F, exec("def x = (short)1; float y = (float)1; return x - y"));
+        assertEquals(0F, exec("def x = (char)1; float y = (float)1; return x - y"));
+        assertEquals(0F, exec("def x = (int)1; float y = (float)1; return x - y"));
+        assertEquals(0F, exec("def x = (long)1; float y = (float)1; return x - y"));
+        assertEquals(0F, exec("def x = (float)1; float y = (float)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; float y = (float)1; return x - y"));
+
+        assertEquals(0D, exec("def x = (byte)1; double y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (short)1; double y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (char)1; double y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (int)1; double y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (long)1; double y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (float)1; double y = (double)1; return x - y"));
+        assertEquals(0D, exec("def x = (double)1; double y = (double)1; return x - y"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/UnaryTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/UnaryTests.java
index ad2876778db..f3e8dc2f430 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/UnaryTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/UnaryTests.java
@@ -49,4 +49,53 @@ public class UnaryTests extends ScriptTestCase {
         assertEquals(-1.0F, exec("float x = -1F; return +x"));
         assertEquals(-1.0, exec("double x = -1.0; return +x"));
     }
+    
+    public void testDefNot() {
+        assertEquals(~1, exec("def x = (byte)1; return ~x"));
+        assertEquals(~1, exec("def x = (short)1; return ~x"));
+        assertEquals(~1, exec("def x = (char)1; return ~x"));
+        assertEquals(~1, exec("def x = 1; return ~x"));
+        assertEquals(~1L, exec("def x = 1L; return ~x"));
+    }
+    
+    public void testDefNotTypedRet() {
+        assertEquals((double)~1, exec("def x = (byte)1; double y = ~x; return y;"));
+        assertEquals((float)~1, exec("def x = (short)1; float y = ~x; return y;"));
+        assertEquals((long)~1, exec("def x = (char)1; long y = ~x; return y;"));
+        assertEquals(~1, exec("def x = 1; int y = ~x; return y;"));
+    }
+
+    public void testDefNeg() {
+        assertEquals(-1, exec("def x = (byte)1; return -x"));
+        assertEquals(-1, exec("def x = (short)1; return -x"));
+        assertEquals(-1, exec("def x = (char)1; return -x"));
+        assertEquals(-1, exec("def x = 1; return -x"));
+        assertEquals(-1L, exec("def x = 1L; return -x"));
+        assertEquals(-1.0F, exec("def x = 1F; return -x"));
+        assertEquals(-1.0, exec("def x = 1.0; return -x"));
+    }
+    
+    public void testDefNegTypedRet() {
+        assertEquals((double)-1, exec("def x = (byte)1; double y = -x; return y;"));
+        assertEquals((float)-1, exec("def x = (short)1; float y = -x; return y;"));
+        assertEquals((long)-1, exec("def x = (char)1; long y = -x; return y;"));
+        assertEquals(-1, exec("def x = 1; int y = -x; return y;"));
+    }
+    
+    public void testDefPlus() {
+        assertEquals(-1, exec("def x = (byte)-1; return +x"));
+        assertEquals(-1, exec("def x = (short)-1; return +x"));
+        assertEquals(65535, exec("def x = (char)-1; return +x"));
+        assertEquals(-1, exec("def x = -1; return +x"));
+        assertEquals(-1L, exec("def x = -1L; return +x"));
+        assertEquals(-1.0F, exec("def x = -1F; return +x"));
+        assertEquals(-1.0D, exec("def x = -1.0; return +x"));
+    }
+    
+    public void testDefPlusTypedRet() {
+        assertEquals((double)-1, exec("def x = (byte)-1; double y = +x; return y;"));
+        assertEquals((float)-1, exec("def x = (short)-1; float y = +x; return y;"));
+        assertEquals((long)65535, exec("def x = (char)-1; long y = +x; return y;"));
+        assertEquals(-1, exec("def x = -1; int y = +x; return y;"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java
index 16436240459..1921539b0bf 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java
@@ -22,6 +22,13 @@ package org.elasticsearch.painless;
 /** Tests for xor operator across all types */
 public class XorTests extends ScriptTestCase {
 
+    public void testBasics() throws Exception {
+        assertEquals(9 ^ 3, exec("return 9 ^ 3;"));
+        assertEquals(9L ^ 3, exec("return 9L ^ 3;"));
+        assertEquals(9 ^ 3L, exec("return 9 ^ 3L;"));
+        assertEquals(10, exec("short x = 9; char y = 3; return x ^ y;"));
+    }
+
     public void testInt() throws Exception {
         assertEquals(5 ^ 12, exec("int x = 5; int y = 12; return x ^ y;"));
         assertEquals(5 ^ -12, exec("int x = 5; int y = -12; return x ^ y;"));
@@ -68,4 +75,151 @@ public class XorTests extends ScriptTestCase {
             exec("double x = (double)4; int y = 1; return x ^ y");
         });
     }
+    
+    public void testDef() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (float)4; def y = (byte)1; return x ^ y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (double)4; def y = (byte)1; return x ^ y");
+        });
+        assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; def y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; def y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; def y = (byte)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (byte)1; return x ^ y"));
+
+        assertEquals(5, exec("def x = (byte)4; def y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; def y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; def y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; def y = (short)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (short)1; return x ^ y"));
+
+        assertEquals(5, exec("def x = (byte)4; def y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; def y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; def y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; def y = (char)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (char)1; return x ^ y"));
+
+        assertEquals(5, exec("def x = (byte)4; def y = (int)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; def y = (int)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; def y = (int)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; def y = (int)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (int)1; return x ^ y"));
+
+        assertEquals(5L, exec("def x = (byte)4; def y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (short)4; def y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (char)4; def y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (int)4; def y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x ^ y"));
+
+        assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; def y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; def y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; def y = (int)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x ^ y"));
+        
+        assertEquals(false, exec("def x = true;  def y = true; return x ^ y"));
+        assertEquals(true,  exec("def x = true;  def y = false; return x ^ y"));
+        assertEquals(true,  exec("def x = false; def y = true; return x ^ y"));
+        assertEquals(false, exec("def x = false; def y = false; return x ^ y"));
+    }
+    
+    public void testDefTypedLHS() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("float x = (float)4; def y = (byte)1; return x ^ y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("double x = (double)4; def y = (byte)1; return x ^ y");
+        });
+        assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; def y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; def y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; def y = (byte)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (byte)1; return x ^ y"));
+
+        assertEquals(5, exec("def x = (byte)4; def y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; def y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; def y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; def y = (short)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (short)1; return x ^ y"));
+
+        assertEquals(5, exec("def x = (byte)4; def y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; def y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; def y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; def y = (char)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (char)1; return x ^ y"));
+
+        assertEquals(5, exec("def x = (byte)4; def y = (int)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; def y = (int)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; def y = (int)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; def y = (int)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (int)1; return x ^ y"));
+
+        assertEquals(5L, exec("def x = (byte)4; def y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (short)4; def y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (char)4; def y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (int)4; def y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x ^ y"));
+
+        assertEquals(5, exec("def x = (byte)4; def y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; def y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; def y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; def y = (int)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; def y = (long)1; return x ^ y"));
+        
+        assertEquals(false, exec("def x = true;  def y = true; return x ^ y"));
+        assertEquals(true,  exec("def x = true;  def y = false; return x ^ y"));
+        assertEquals(true,  exec("def x = false; def y = true; return x ^ y"));
+        assertEquals(false, exec("def x = false; def y = false; return x ^ y"));
+    }
+    
+    public void testDefTypedRHS() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (float)4; byte y = (byte)1; return x ^ y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = (double)4; byte y = (byte)1; return x ^ y");
+        });
+        assertEquals(5, exec("def x = (byte)4; byte y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; byte y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; byte y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; byte y = (byte)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; byte y = (byte)1; return x ^ y"));
+
+        assertEquals(5, exec("def x = (byte)4; short y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; short y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; short y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; short y = (short)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; short y = (short)1; return x ^ y"));
+
+        assertEquals(5, exec("def x = (byte)4; char y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; char y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; char y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; char y = (char)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; char y = (char)1; return x ^ y"));
+
+        assertEquals(5, exec("def x = (byte)4; int y = (int)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; int y = (int)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; int y = (int)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; int y = (int)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; int y = (int)1; return x ^ y"));
+
+        assertEquals(5L, exec("def x = (byte)4; long y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (short)4; long y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (char)4; long y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (int)4; long y = (long)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; long y = (long)1; return x ^ y"));
+
+        assertEquals(5, exec("def x = (byte)4; byte y = (byte)1; return x ^ y"));
+        assertEquals(5, exec("def x = (short)4; short y = (short)1; return x ^ y"));
+        assertEquals(5, exec("def x = (char)4; char y = (char)1; return x ^ y"));
+        assertEquals(5, exec("def x = (int)4; int y = (int)1; return x ^ y"));
+        assertEquals(5L, exec("def x = (long)4; long y = (long)1; return x ^ y"));
+        
+        assertEquals(false, exec("def x = true;  boolean y = true; return x ^ y"));
+        assertEquals(true,  exec("def x = true;  boolean y = false; return x ^ y"));
+        assertEquals(true,  exec("def x = false; boolean y = true; return x ^ y"));
+        assertEquals(false, exec("def x = false; boolean y = false; return x ^ y"));
+    }
 }

From c8931768ba0307d611f95ed2703e183d0704ba9c Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 14 Jun 2016 16:14:57 -0400
Subject: [PATCH 0843/1311] Clean up after test failure

If the test fails we properly clean up. Also add a toString
implementation so we get useful results on failure.
---
 .../support/replication/ReplicationTask.java  |  6 ++
 .../admin/cluster/node/tasks/TasksIT.java     | 93 ++++++++++---------
 2 files changed, 55 insertions(+), 44 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationTask.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationTask.java
index b31c2439aaf..2e0baa057b2 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationTask.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationTask.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.support.replication;
 
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -89,6 +90,11 @@ public class ReplicationTask extends Task {
             out.writeString(phase);
         }
 
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+
         // Implements equals and hashcode for testing
         @Override
         public boolean equals(Object obj) {
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
index 88c7f1ae210..ebb93ee7f21 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
@@ -338,55 +338,60 @@ public class TasksIT extends ESIntegTestCase {
          */
         ReentrantLock taskFinishLock = new ReentrantLock();
         taskFinishLock.lock();
-        CountDownLatch taskRegistered = new CountDownLatch(1);
-        for (TransportService transportService : internalCluster().getInstances(TransportService.class)) {
-            ((MockTaskManager) transportService.getTaskManager()).addListener(new MockTaskManagerListener() {
-                @Override
-                public void onTaskRegistered(Task task) {
-                    if (task.getAction().startsWith(IndexAction.NAME)) {
-                        taskRegistered.countDown();
+        ListenableActionFuture<?> indexFuture = null;
+        try {
+            CountDownLatch taskRegistered = new CountDownLatch(1);
+            for (TransportService transportService : internalCluster().getInstances(TransportService.class)) {
+                ((MockTaskManager) transportService.getTaskManager()).addListener(new MockTaskManagerListener() {
+                    @Override
+                    public void onTaskRegistered(Task task) {
+                        if (task.getAction().startsWith(IndexAction.NAME)) {
+                            taskRegistered.countDown();
+                        }
                     }
-                }
 
-                @Override
-                public void onTaskUnregistered(Task task) {
-                    /*
-                     * We can't block all tasks here or the task listing task
-                     * would never return.
-                     */
-                    if (false == task.getAction().startsWith(IndexAction.NAME)) {
-                        return;
+                    @Override
+                    public void onTaskUnregistered(Task task) {
+                        /*
+                         * We can't block all tasks here or the task listing task
+                         * would never return.
+                         */
+                        if (false == task.getAction().startsWith(IndexAction.NAME)) {
+                            return;
+                        }
+                        logger.debug("Blocking {} from being unregistered", task);
+                        taskFinishLock.lock();
+                        taskFinishLock.unlock();
                     }
-                    logger.debug("Blocking {} from being unregistered", task);
-                    taskFinishLock.lock();
-                    taskFinishLock.unlock();
-                }
-            });
-        }
-        ListenableActionFuture<?> indexFuture = client().prepareIndex("test", "test").setSource("test", "test").execute();
-        taskRegistered.await(10, TimeUnit.SECONDS); // waiting for at least one task to be registered
+                });
+            }
+            indexFuture = client().prepareIndex("test", "test").setSource("test", "test").execute();
+            taskRegistered.await(10, TimeUnit.SECONDS); // waiting for at least one task to be registered
 
-        ListTasksResponse listResponse = client().admin().cluster().prepareListTasks().setActions("indices:data/write/index*")
-                .setDetailed(true).get();
-        assertThat(listResponse.getTasks(), not(empty()));
-        for (TaskInfo task : listResponse.getTasks()) {
-            assertNotNull(task.getStatus());
-            GetTaskResponse getResponse = client().admin().cluster().prepareGetTask(task.getTaskId()).get();
-            assertFalse("task should still be running", getResponse.getTask().isCompleted());
-            TaskInfo fetchedWithGet = getResponse.getTask().getTask();
-            assertEquals(task.getId(), fetchedWithGet.getId());
-            assertEquals(task.getType(), fetchedWithGet.getType());
-            assertEquals(task.getAction(), fetchedWithGet.getAction());
-            assertEquals(task.getDescription(), fetchedWithGet.getDescription());
-            assertEquals(task.getStatus(), fetchedWithGet.getStatus());
-            assertEquals(task.getStartTime(), fetchedWithGet.getStartTime());
-            assertThat(fetchedWithGet.getRunningTimeNanos(), greaterThanOrEqualTo(task.getRunningTimeNanos()));
-            assertEquals(task.isCancellable(), fetchedWithGet.isCancellable());
-            assertEquals(task.getParentTaskId(), fetchedWithGet.getParentTaskId());
+            ListTasksResponse listResponse = client().admin().cluster().prepareListTasks().setActions("indices:data/write/index*")
+                    .setDetailed(true).get();
+            assertThat(listResponse.getTasks(), not(empty()));
+            for (TaskInfo task : listResponse.getTasks()) {
+                assertNotNull(task.getStatus());
+                GetTaskResponse getResponse = client().admin().cluster().prepareGetTask(task.getTaskId()).get();
+                assertFalse("task should still be running", getResponse.getTask().isCompleted());
+                TaskInfo fetchedWithGet = getResponse.getTask().getTask();
+                assertEquals(task.getId(), fetchedWithGet.getId());
+                assertEquals(task.getType(), fetchedWithGet.getType());
+                assertEquals(task.getAction(), fetchedWithGet.getAction());
+                assertEquals(task.getDescription(), fetchedWithGet.getDescription());
+                assertEquals(task.getStatus(), fetchedWithGet.getStatus());
+                assertEquals(task.getStartTime(), fetchedWithGet.getStartTime());
+                assertThat(fetchedWithGet.getRunningTimeNanos(), greaterThanOrEqualTo(task.getRunningTimeNanos()));
+                assertEquals(task.isCancellable(), fetchedWithGet.isCancellable());
+                assertEquals(task.getParentTaskId(), fetchedWithGet.getParentTaskId());
+            }
+        } finally {
+            taskFinishLock.unlock();
+            if (indexFuture != null) {
+                indexFuture.get();
+            }
         }
-
-        taskFinishLock.unlock();
-        indexFuture.get();
     }
 
     public void testTasksCancellation() throws Exception {

From 36f76322326013009fb59ac62edb3941c4f5a65b Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Tue, 14 Jun 2016 14:12:23 -0700
Subject: [PATCH 0844/1311] Correct type checking during casting related to
 Object.

---
 .../painless/AnalyzerCaster.java              | 71 +++++++++++++++++++
 1 file changed, 71 insertions(+)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
index e724fa9e810..a37e4a4b0aa 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
@@ -40,6 +40,10 @@ public final class AnalyzerCaster {
                     case DEF:
                         return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false);
                     case OBJECT:
+                        if (Definition.OBJECT_TYPE.equals(expected) && internal)
+                            return new Cast(actual, actual, explicit, false, false, false, true);
+
+                        break;
                     case BOOL_OBJ:
                         if (internal)
                             return new Cast(actual, actual, explicit, false, false, false, true);
@@ -62,6 +66,10 @@ public final class AnalyzerCaster {
                     case DEF:
                         return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false);
                     case OBJECT:
+                        if (Definition.OBJECT_TYPE.equals(expected) && internal)
+                            return new Cast(actual, actual, explicit, false, false, false, true);
+
+                        break;
                     case NUMBER:
                     case BYTE_OBJ:
                         if (internal)
@@ -117,6 +125,10 @@ public final class AnalyzerCaster {
                     case DEF:
                         return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false);
                     case OBJECT:
+                        if (Definition.OBJECT_TYPE.equals(expected) && internal)
+                            return new Cast(actual, actual, explicit, false, false, false, true);
+
+                        break;
                     case NUMBER:
                     case SHORT_OBJ:
                         if (internal)
@@ -172,6 +184,10 @@ public final class AnalyzerCaster {
                     case DEF:
                         return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false);
                     case OBJECT:
+                        if (Definition.OBJECT_TYPE.equals(expected) && internal)
+                            return new Cast(actual, actual, explicit, false, false, false, true);
+
+                        break;
                     case NUMBER:
                     case CHAR_OBJ:
                         if (internal)
@@ -229,6 +245,10 @@ public final class AnalyzerCaster {
                     case DEF:
                         return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false);
                     case OBJECT:
+                        if (Definition.OBJECT_TYPE.equals(expected) && internal)
+                            return new Cast(actual, actual, explicit, false, false, false, true);
+
+                        break;
                     case NUMBER:
                     case INT_OBJ:
                         if (internal)
@@ -284,6 +304,10 @@ public final class AnalyzerCaster {
                     case DEF:
                         return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false);
                     case OBJECT:
+                        if (Definition.OBJECT_TYPE.equals(expected) && internal)
+                            return new Cast(actual, actual, explicit, false, false, false, true);
+
+                        break;
                     case NUMBER:
                     case LONG_OBJ:
                         if (internal)
@@ -339,6 +363,10 @@ public final class AnalyzerCaster {
                     case DEF:
                         return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false);
                     case OBJECT:
+                        if (Definition.OBJECT_TYPE.equals(expected) && internal)
+                            return new Cast(actual, actual, explicit, false, false, false, true);
+
+                        break;
                     case NUMBER:
                     case FLOAT_OBJ:
                         if (internal)
@@ -392,6 +420,10 @@ public final class AnalyzerCaster {
                     case DEF:
                         return new Cast(actual, Definition.DEF_TYPE, explicit, false, false, true, false);
                     case OBJECT:
+                        if (Definition.OBJECT_TYPE.equals(expected) && internal)
+                            return new Cast(actual, actual, explicit, false, false, false, true);
+
+                        break;
                     case NUMBER:
                     case DOUBLE_OBJ:
                         if (internal)
@@ -432,6 +464,45 @@ public final class AnalyzerCaster {
 
                 break;
             case OBJECT:
+                if (Definition.OBJECT_TYPE.equals(actual))
+                    switch (expected.sort) {
+                        case BYTE:
+                            if (internal && explicit)
+                                return new Cast(actual, Definition.BYTE_OBJ_TYPE, true, false, true, false, false);
+
+                            break;
+                        case SHORT:
+                            if (internal && explicit)
+                                return new Cast(actual, Definition.SHORT_OBJ_TYPE, true, false, true, false, false);
+
+                            break;
+                        case CHAR:
+                            if (internal && explicit)
+                                return new Cast(actual, Definition.CHAR_OBJ_TYPE, true, false, true, false, false);
+
+                            break;
+                        case INT:
+                            if (internal && explicit)
+                                return new Cast(actual, Definition.INT_OBJ_TYPE, true, false, true, false, false);
+
+                            break;
+                        case LONG:
+                            if (internal && explicit)
+                                return new Cast(actual, Definition.LONG_OBJ_TYPE, true, false, true, false, false);
+
+                            break;
+                        case FLOAT:
+                            if (internal && explicit)
+                                return new Cast(actual, Definition.FLOAT_OBJ_TYPE, true, false, true, false, false);
+
+                            break;
+                        case DOUBLE:
+                            if (internal && explicit)
+                                return new Cast(actual, Definition.DOUBLE_OBJ_TYPE, true, false, true, false, false);
+
+                            break;
+                    }
+                break;
             case NUMBER:
                 switch (expected.sort) {
                     case BYTE:

From 70fb67cc7b1fd72b385b98bbf0eb425f7462bb83 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Tue, 14 Jun 2016 23:16:38 +0200
Subject: [PATCH 0845/1311] Build: targetCompatibility and sourceCompatibility
 are enough to make sure we compile client and client-sniffer with target and
 source 1.7

---
 client-sniffer/build.gradle | 3 ---
 client/build.gradle         | 3 ---
 2 files changed, 6 deletions(-)

diff --git a/client-sniffer/build.gradle b/client-sniffer/build.gradle
index 5b224168492..c01f730e094 100644
--- a/client-sniffer/build.gradle
+++ b/client-sniffer/build.gradle
@@ -46,9 +46,6 @@ dependencies {
   signature "org.codehaus.mojo.signature:java17:1.0@signature"
 }
 
-compileJava.options.compilerArgs << '-target' << '1.7' << '-source' << '1.7' << '-Xlint:all,-path,-serial,-options'
-compileTestJava.options.compilerArgs << '-target' << '1.7' << '-source' << '1.7'
-
 forbiddenApisMain {
   //client does not depend on core, so only jdk signatures should be checked
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
diff --git a/client/build.gradle b/client/build.gradle
index 83c96d07d34..30225382e41 100644
--- a/client/build.gradle
+++ b/client/build.gradle
@@ -44,9 +44,6 @@ dependencies {
   signature "org.codehaus.mojo.signature:java17:1.0@signature"
 }
 
-compileJava.options.compilerArgs << '-target' << '1.7' << '-source' << '1.7' << '-Xlint:all,-path,-serial,-options'
-compileTestJava.options.compilerArgs << '-target' << '1.7' << '-source' << '1.7'
-
 forbiddenApisMain {
   //client does not depend on core, so only jdk signatures should be checked
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]

From f7f377791f1c81ab567ccc55004db57e7feb40f3 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 14 Jun 2016 17:27:45 -0400
Subject: [PATCH 0846/1311] Test: don't use 0 size for terms aggregation

It is no longer allowed.
---
 .../search/aggregations/bucket/BooleanTermsIT.java              | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java
index aad2c9bb3ed..e38f33cfa2b 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/BooleanTermsIT.java
@@ -156,7 +156,7 @@ public class BooleanTermsIT extends ESIntegTestCase {
         SearchResponse response = client().prepareSearch("idx_unmapped").setTypes("type")
                 .addAggregation(terms("terms")
                         .field(SINGLE_VALUED_FIELD_NAME)
-                        .size(randomInt(5))
+                        .size(between(1, 5))
                         .collectMode(randomFrom(SubAggCollectionMode.values())))
                 .execute().actionGet();
 

From 5cef06fa3eff6de5ff7c62e99b7e5a0cfecac302 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Tue, 14 Jun 2016 14:28:47 -0700
Subject: [PATCH 0847/1311] Add a test.

---
 .../org/elasticsearch/painless/WhenThingsGoWrongTests.java   | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
index 7a597132227..a9e03095297 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
@@ -229,4 +229,9 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
         assertThat(e.getMessage(), containsString("\\ujjjj"));
     }
 
+    public void testBadBoxingCast() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("BitSet bs = new BitSet(); bs.and(2);");
+        });
+    }
 }

From 14d6b0494477d4a3418a6a4da5bc00ad92f8fd07 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Tue, 14 Jun 2016 23:26:10 +0200
Subject: [PATCH 0848/1311] test: don't return 0, at least one request must be
 added to msearch request also the maxConcurrentSearchRequests() setter only
 support positive values.

---
 .../java/org/elasticsearch/search/msearch/MultiSearchIT.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java b/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java
index 72306fd99d4..ccc3102f3a8 100644
--- a/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/msearch/MultiSearchIT.java
@@ -63,7 +63,7 @@ public class MultiSearchIT extends ESIntegTestCase {
         }
         refresh();
 
-        int numSearchRequests = randomIntBetween(0, 64);
+        int numSearchRequests = randomIntBetween(1, 64);
         MultiSearchRequest request = new MultiSearchRequest();
         if (randomBoolean()) {
             request.maxConcurrentSearchRequests(randomIntBetween(1, numSearchRequests));

From d0e4485d421360ff3967bc5fe20ca08086b742c8 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 3 Jun 2016 12:48:45 -0400
Subject: [PATCH 0849/1311] Move NamingConventionsCheck into buildSrc

This will let things that don't depend on :test:framework like the
client use it.

Also skip initializing the classes we check because we don't care
about their initialization behavior because we're not executing them.
This makes the naming conventions check pretty close to instant
from a "human eye" perspective.
---
 build.gradle                                  |  1 +
 buildSrc/build.gradle                         |  8 ++-
 .../precommit/NamingConventionsTask.groovy    | 27 +++++++-
 .../gradle/precommit/PrecommitTasks.groovy    | 15 ++++-
 .../test/NamingConventionsCheck.java          | 66 ++++++++++---------
 .../NamingConventionsCheckBadClasses.java     | 26 ++++++--
 6 files changed, 101 insertions(+), 42 deletions(-)
 rename {test/framework => buildSrc}/src/main/java/org/elasticsearch/test/NamingConventionsCheck.java (83%)
 rename {test/framework/src/test/java/org/elasticsearch/test => buildSrc/src/test/java/org/elasticsearch}/test/NamingConventionsCheckBadClasses.java (59%)

diff --git a/build.gradle b/build.gradle
index 5c6ec366930..e78876a84c7 100644
--- a/build.gradle
+++ b/build.gradle
@@ -160,6 +160,7 @@ subprojects {
     them as external dependencies so the build plugin that we use can be used
     to build elasticsearch plugins outside of the elasticsearch source tree. */
   ext.projectSubstitutions = [
+    "org.elasticsearch.gradle:build-tools:${version}": ':build-tools',
     "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec',
     "org.elasticsearch:elasticsearch:${version}": ':core',
     "org.elasticsearch.test:framework:${version}": ':test:framework',
diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
index 623fdab3e3e..a5a9b4209ba 100644
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -103,6 +103,7 @@ if (project == rootProject) {
       url "https://oss.sonatype.org/content/repositories/snapshots/"
     }
   }
+  test.exclude 'org/elasticsearch/test/NamingConventionsCheckBadClasses*'
 }
 
 /*****************************************************************************
@@ -122,8 +123,8 @@ if (project != rootProject) {
   // build-tools is not ready for primetime with these...
   dependencyLicenses.enabled = false
   forbiddenApisMain.enabled = false
+  forbiddenApisTest.enabled = false
   jarHell.enabled = false
-  loggerUsageCheck.enabled = false
   thirdPartyAudit.enabled = false
 
   // test for elasticsearch.build tries to run with ES...
@@ -137,4 +138,9 @@ if (project != rootProject) {
     // the file that actually defines nocommit
     exclude '**/ForbiddenPatternsTask.groovy'
   }
+
+  namingConventions {
+    testClass = 'org.elasticsearch.test.NamingConventionsCheckBadClasses$UnitTestCase'
+    integTestClass = 'org.elasticsearch.test.NamingConventionsCheckBadClasses$IntegTestCase'
+  }
 }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/NamingConventionsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/NamingConventionsTask.groovy
index 612bc568621..fe7f13f29e6 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/NamingConventionsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/NamingConventionsTask.groovy
@@ -21,6 +21,7 @@ package org.elasticsearch.gradle.precommit
 
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.VersionProperties
+import org.gradle.api.artifacts.Dependency
 import org.gradle.api.file.FileCollection
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.InputFiles
@@ -57,8 +58,27 @@ public class NamingConventionsTask extends LoggedExec {
     @Input
     boolean skipIntegTestInDisguise = false
 
+    /**
+     * Superclass for all tests.
+     */
+    @Input
+    String testClass = 'org.apache.lucene.util.LuceneTestCase'
+
+    /**
+     * Superclass for all integration tests.
+     */
+    @Input
+    String integTestClass = 'org.elasticsearch.test.ESIntegTestCase'
+
     public NamingConventionsTask() {
-        dependsOn(classpath)
+        // Extra classpath contains the actual test
+        project.configurations.create('namingConventions')
+        Dependency buildToolsDep = project.dependencies.add('namingConventions',
+                "org.elasticsearch.gradle:build-tools:${VersionProperties.elasticsearch}")
+        buildToolsDep.transitive = false // We don't need gradle in the classpath. It conflicts.
+        FileCollection extraClasspath = project.configurations.namingConventions
+        dependsOn(extraClasspath)
+
         description = "Runs NamingConventionsCheck on ${classpath}"
         executable = new File(project.javaHome, 'bin/java')
         onlyIf { project.sourceSets.test.output.classesDir.exists() }
@@ -69,7 +89,8 @@ public class NamingConventionsTask extends LoggedExec {
         project.afterEvaluate {
             doFirst {
                 args('-Djna.nosys=true')
-                args('-cp', classpath.asPath, 'org.elasticsearch.test.NamingConventionsCheck')
+                args('-cp', (classpath + extraClasspath).asPath, 'org.elasticsearch.test.NamingConventionsCheck')
+                args(testClass, integTestClass)
                 if (skipIntegTestInDisguise) {
                     args('--skip-integ-tests-in-disguise')
                 }
@@ -79,7 +100,7 @@ public class NamingConventionsTask extends LoggedExec {
                  * process of ignoring them lets us validate that they were found so this ignore parameter acts
                  * as the test for the NamingConventionsCheck.
                  */
-                if (':test:framework'.equals(project.path)) {
+                if (':build-tools'.equals(project.path)) {
                     args('--self-test')
                 }
                 args('--', project.sourceSets.test.output.classesDir.absolutePath)
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
index 48a4d7c26dc..a5e1e4c8932 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy
@@ -34,7 +34,6 @@ class PrecommitTasks {
             configureForbiddenApis(project),
             configureCheckstyle(project),
             configureNamingConventions(project),
-            configureLoggerUsage(project),
             project.tasks.create('forbiddenPatterns', ForbiddenPatternsTask.class),
             project.tasks.create('licenseHeaders', LicenseHeadersTask.class),
             project.tasks.create('jarHell', JarHellTask.class),
@@ -49,6 +48,20 @@ class PrecommitTasks {
             UpdateShasTask updateShas = project.tasks.create('updateShas', UpdateShasTask.class)
             updateShas.parentTask = dependencyLicenses
         }
+        if (project.path != ':build-tools') {
+            /*
+             * Sadly, build-tools can't have logger-usage-check because that
+             * would create a circular project dependency between build-tools
+             * (which provides NamingConventionsCheck) and :test:logger-usage
+             * which provides the logger usage check. Since the build tools
+             * don't use the logger usage check because they don't have any
+             * of Elaticsearch's loggers and :test:logger-usage actually does
+             * use the NamingConventionsCheck we break the circular dependency
+             * here.
+             */
+            precommitTasks.add(configureLoggerUsage(project))
+        }
+
 
         Map<String, Object> precommitOptions = [
             name: 'precommit',
diff --git a/test/framework/src/main/java/org/elasticsearch/test/NamingConventionsCheck.java b/buildSrc/src/main/java/org/elasticsearch/test/NamingConventionsCheck.java
similarity index 83%
rename from test/framework/src/main/java/org/elasticsearch/test/NamingConventionsCheck.java
rename to buildSrc/src/main/java/org/elasticsearch/test/NamingConventionsCheck.java
index 13163cee029..ed25d52739c 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/NamingConventionsCheck.java
+++ b/buildSrc/src/main/java/org/elasticsearch/test/NamingConventionsCheck.java
@@ -25,14 +25,11 @@ import java.nio.file.FileVisitResult;
 import java.nio.file.FileVisitor;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.HashSet;
 import java.util.Set;
 
-import org.apache.lucene.util.LuceneTestCase;
-import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.io.PathUtils;
-
 /**
  * Checks that all tests in a directory are named according to our naming conventions. This is important because tests that do not follow
  * our conventions aren't run by gradle. This was once a glorious unit test but now that Elasticsearch is a multi-module project it must be
@@ -46,11 +43,13 @@ import org.elasticsearch.common.io.PathUtils;
  * {@code --self-test} that is only run in the test:framework project.
  */
 public class NamingConventionsCheck {
-    public static void main(String[] args) throws IOException, ClassNotFoundException {
-        NamingConventionsCheck check = new NamingConventionsCheck();
+    public static void main(String[] args) throws IOException {
+        int i = 0;
+        NamingConventionsCheck check = new NamingConventionsCheck(
+                loadClassWithoutInitializing(args[i++]),
+                loadClassWithoutInitializing(args[i++]));
         boolean skipIntegTestsInDisguise = false;
         boolean selfTest = false;
-        int i = 0;
         while (true) {
             switch (args[i]) {
             case "--skip-integ-tests-in-disguise":
@@ -69,7 +68,7 @@ public class NamingConventionsCheck {
             }
             break;
         }
-        check.check(PathUtils.get(args[i]));
+        check.check(Paths.get(args[i]));
 
         if (selfTest) {
             assertViolation("WrongName", check.missingSuffix);
@@ -82,14 +81,12 @@ public class NamingConventionsCheck {
         }
 
         // Now we should have no violations
-        assertNoViolations("Not all subclasses of " + ESTestCase.class.getSimpleName()
+        assertNoViolations("Not all subclasses of " + check.testClass.getSimpleName()
                 + " match the naming convention. Concrete classes must end with [Tests]", check.missingSuffix);
         assertNoViolations("Classes ending with [Tests] are abstract or interfaces", check.notRunnable);
         assertNoViolations("Found inner classes that are tests, which are excluded from the test runner", check.innerClasses);
-        String classesToSubclass = String.join(",", ESTestCase.class.getSimpleName(), ESTestCase.class.getSimpleName(),
-                ESTokenStreamTestCase.class.getSimpleName(), LuceneTestCase.class.getSimpleName());
-        assertNoViolations("Pure Unit-Test found must subclass one of [" + classesToSubclass + "]", check.pureUnitTest);
-        assertNoViolations("Classes ending with [Tests] must subclass [" + classesToSubclass + "]", check.notImplementing);
+        assertNoViolations("Pure Unit-Test found must subclass [" + check.testClass.getSimpleName() + "]", check.pureUnitTest);
+        assertNoViolations("Classes ending with [Tests] must subclass [" + check.testClass.getSimpleName() + "]", check.notImplementing);
         if (!skipIntegTestsInDisguise) {
             assertNoViolations("Subclasses of ESIntegTestCase should end with IT as they are integration tests",
                     check.integTestsInDisguise);
@@ -103,6 +100,14 @@ public class NamingConventionsCheck {
     private final Set<Class<?>> notRunnable = new HashSet<>();
     private final Set<Class<?>> innerClasses = new HashSet<>();
 
+    private final Class<?> testClass;
+    private final Class<?> integTestClass;
+
+    public NamingConventionsCheck(Class<?> testClass, Class<?> integTestClass) {
+        this.testClass = testClass;
+        this.integTestClass = integTestClass;
+    }
+
     public void check(Path rootPath) throws IOException {
         Files.walkFileTree(rootPath, new FileVisitor<Path>() {
             /**
@@ -136,9 +141,9 @@ public class NamingConventionsCheck {
                 String filename = file.getFileName().toString();
                 if (filename.endsWith(".class")) {
                     String className = filename.substring(0, filename.length() - ".class".length());
-                    Class<?> clazz = loadClass(className);
+                    Class<?> clazz = loadClassWithoutInitializing(packageName + className);
                     if (clazz.getName().endsWith("Tests")) {
-                        if (ESIntegTestCase.class.isAssignableFrom(clazz)) {
+                        if (integTestClass.isAssignableFrom(clazz)) {
                             integTestsInDisguise.add(clazz);
                         }
                         if (Modifier.isAbstract(clazz.getModifiers()) || Modifier.isInterface(clazz.getModifiers())) {
@@ -164,15 +169,7 @@ public class NamingConventionsCheck {
             }
 
             private boolean isTestCase(Class<?> clazz) {
-                return LuceneTestCase.class.isAssignableFrom(clazz);
-            }
-
-            private Class<?> loadClass(String className) {
-                try {
-                    return Thread.currentThread().getContextClassLoader().loadClass(packageName + className);
-                } catch (ClassNotFoundException e) {
-                    throw new RuntimeException(e);
-                }
+                return testClass.isAssignableFrom(clazz);
             }
 
             @Override
@@ -186,7 +183,6 @@ public class NamingConventionsCheck {
      * Fail the process if there are any violations in the set. Named to look like a junit assertion even though it isn't because it is
      * similar enough.
      */
-    @SuppressForbidden(reason = "System.err/System.exit")
     private static void assertNoViolations(String message, Set<Class<?>> set) {
         if (false == set.isEmpty()) {
             System.err.println(message + ":");
@@ -201,10 +197,9 @@ public class NamingConventionsCheck {
      * Fail the process if we didn't detect a particular violation. Named to look like a junit assertion even though it isn't because it is
      * similar enough.
      */
-    @SuppressForbidden(reason = "System.err/System.exit")
-    private static void assertViolation(String className, Set<Class<?>> set) throws ClassNotFoundException {
-        className = "org.elasticsearch.test.test.NamingConventionsCheckBadClasses$" + className;
-        if (false == set.remove(Class.forName(className))) {
+    private static void assertViolation(String className, Set<Class<?>> set) {
+        className = "org.elasticsearch.test.NamingConventionsCheckBadClasses$" + className;
+        if (false == set.remove(loadClassWithoutInitializing(className))) {
             System.err.println("Error in NamingConventionsCheck! Expected [" + className + "] to be a violation but wasn't.");
             System.exit(1);
         }
@@ -213,9 +208,20 @@ public class NamingConventionsCheck {
     /**
      * Fail the process with the provided message.
      */
-    @SuppressForbidden(reason = "System.err/System.exit")
     private static void fail(String reason) {
         System.err.println(reason);
         System.exit(1);
     }
+
+    static Class<?> loadClassWithoutInitializing(String name) {
+        try {
+            return Class.forName(name,
+                    // Don't initialize the class to save time. Not needed for this test and this doesn't share a VM with any other tests.
+                    false,
+                    // Use our classloader rather than the bootstrap class loader.
+                    NamingConventionsCheck.class.getClassLoader());
+        } catch (ClassNotFoundException e) {
+            throw new RuntimeException(e);
+        }
+    }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/NamingConventionsCheckBadClasses.java b/buildSrc/src/test/java/org/elasticsearch/test/NamingConventionsCheckBadClasses.java
similarity index 59%
rename from test/framework/src/test/java/org/elasticsearch/test/test/NamingConventionsCheckBadClasses.java
rename to buildSrc/src/test/java/org/elasticsearch/test/NamingConventionsCheckBadClasses.java
index 233e9fe5975..87240534880 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/test/NamingConventionsCheckBadClasses.java
+++ b/buildSrc/src/test/java/org/elasticsearch/test/NamingConventionsCheckBadClasses.java
@@ -17,9 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.test.test;
-
-import org.elasticsearch.test.ESTestCase;
+package org.elasticsearch.test;
 
 import junit.framework.TestCase;
 
@@ -30,21 +28,35 @@ public class NamingConventionsCheckBadClasses {
     public static final class NotImplementingTests {
     }
 
-    public static final class WrongName extends ESTestCase {
+    public static final class WrongName extends UnitTestCase {
+        /*
+         * Dummy test so the tests pass. We do this *and* skip the tests so anyone who jumps back to a branch without these tests can still
+         * compile without a failure. That is because clean doesn't actually clean these....
+         */
+        public void testDummy() {}
     }
 
-    public static abstract class DummyAbstractTests extends ESTestCase {
+    public static abstract class DummyAbstractTests extends UnitTestCase {
     }
 
     public interface DummyInterfaceTests {
     }
 
-    public static final class InnerTests extends ESTestCase {
+    public static final class InnerTests extends UnitTestCase {
+        public void testDummy() {}
     }
 
-    public static final class WrongNameTheSecond extends ESTestCase {
+    public static final class WrongNameTheSecond extends UnitTestCase {
+        public void testDummy() {}
     }
 
     public static final class PlainUnit extends TestCase {
+        public void testDummy() {}
+    }
+
+    public abstract static class UnitTestCase extends TestCase {
+    }
+
+    public abstract static class IntegTestCase extends UnitTestCase {
     }
 }

From cd4f4594d61912682fa9c1626b90fecf55d3d713 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 14 Jun 2016 19:19:50 -0400
Subject: [PATCH 0850/1311] fix perf bug with xor promotion

---
 .../elasticsearch/painless/AnalyzerCaster.java    |  4 ++++
 .../painless/DefOptimizationTests.java            | 15 +++++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
index a37e4a4b0aa..864b44fb766 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java
@@ -826,6 +826,10 @@ public final class AnalyzerCaster {
         final Sort sort0 = from0.sort;
         final Sort sort1 = from1.sort;
 
+        if (sort0 == Sort.DEF || sort1 == Sort.DEF) {
+            return Definition.DEF_TYPE;
+        }
+
         if (sort0.bool || sort1.bool) {
             return Definition.BOOLEAN_TYPE;
         }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
index 1dd0d09a580..86461880a99 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
@@ -343,7 +343,22 @@ public class DefOptimizationTests extends ScriptTestCase {
         assertBytecodeExists("def x = 1; def y = 2; double d = x ^ y", 
                              "INVOKEDYNAMIC xor(Ljava/lang/Object;Ljava/lang/Object;)D");
     }
+
+    public void testBooleanXorOptLHS() {
+        assertBytecodeExists("boolean x = true; def y = true; return x ^ y", 
+                "INVOKEDYNAMIC xor(ZLjava/lang/Object;)Ljava/lang/Object;");
+    }
+
+    public void testBooleanXorOptRHS() {
+        assertBytecodeExists("def x = true; boolean y = true; return x ^ y", 
+                "INVOKEDYNAMIC xor(Ljava/lang/Object;Z)Ljava/lang/Object;");
+    }
     
+    public void testBooleanXorOptRet() {
+        assertBytecodeExists("def x = true; def y = true; boolean v = x ^ y", 
+                "INVOKEDYNAMIC xor(Ljava/lang/Object;Ljava/lang/Object;)Z");
+    }
+
     public void testLtOptLHS() {
         assertBytecodeExists("int x = 1; def y = 2; return x < y", 
                              "INVOKEDYNAMIC lt(ILjava/lang/Object;)Z");

From 5e4b2f281574f697a2c016415368ea3fb51ebdff Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 14 Jun 2016 20:40:16 -0400
Subject: [PATCH 0851/1311] clean up compound assignment tests, fix shift bug

---
 .../elasticsearch/painless/node/EChain.java   |  22 +-
 .../elasticsearch/painless/AdditionTests.java |  25 ++
 .../org/elasticsearch/painless/AndTests.java  |  46 +++
 .../painless/CompoundAssignmentTests.java     | 314 ------------------
 .../elasticsearch/painless/DivisionTests.java |  50 +++
 .../painless/MultiplicationTests.java         |  23 ++
 .../org/elasticsearch/painless/OrTests.java   |  46 +++
 .../painless/RemainderTests.java              |  23 ++
 .../elasticsearch/painless/ShiftTests.java    |  89 +++++
 .../painless/SubtractionTests.java            |  24 ++
 .../org/elasticsearch/painless/XorTests.java  |  46 +++
 11 files changed, 389 insertions(+), 319 deletions(-)
 delete mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/CompoundAssignmentTests.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index 2d47c520b7b..e3484159b50 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -45,6 +45,7 @@ public final class EChain extends AExpression {
 
     boolean cat = false;
     Type promote = null;
+    Type shiftDistance; // for shifts, the RHS is promoted independently
     Cast there = null;
     Cast back = null;
     
@@ -163,6 +164,7 @@ public final class EChain extends AExpression {
         ALink last = links.get(links.size() - 1);
 
         expression.analyze(variables);
+        boolean shift = false;
 
         if (operation == Operation.MUL) {
             promote = AnalyzerCaster.promoteNumeric(last.after, expression.actual, true);
@@ -176,10 +178,16 @@ public final class EChain extends AExpression {
             promote = AnalyzerCaster.promoteNumeric(last.after, expression.actual, true);
         } else if (operation == Operation.LSH) {
             promote = AnalyzerCaster.promoteNumeric(last.after, false);
+            shiftDistance = AnalyzerCaster.promoteNumeric(expression.actual, false);
+            shift = true;
         } else if (operation == Operation.RSH) {
             promote = AnalyzerCaster.promoteNumeric(last.after, false);
+            shiftDistance = AnalyzerCaster.promoteNumeric(expression.actual, false);
+            shift = true;
         } else if (operation == Operation.USH) {
             promote = AnalyzerCaster.promoteNumeric(last.after, false);
+            shiftDistance = AnalyzerCaster.promoteNumeric(expression.actual, false);
+            shift = true;
         } else if (operation == Operation.BWAND) {
             promote = AnalyzerCaster.promoteXor(last.after, expression.actual);
         } else if (operation == Operation.XOR) {
@@ -190,7 +198,7 @@ public final class EChain extends AExpression {
             throw createError(new IllegalStateException("Illegal tree structure."));
         }
 
-        if (promote == null) {
+        if (promote == null || (shift && shiftDistance == null)) {
             throw createError(new ClassCastException("Cannot apply compound assignment " +
                 "[" + operation.symbol + "=] to types [" + last.after + "] and [" + expression.actual + "]."));
         }
@@ -204,9 +212,13 @@ public final class EChain extends AExpression {
             }
 
             expression.expected = expression.actual;
-        } else if (operation == Operation.LSH || operation == Operation.RSH || operation == Operation.USH) {
-            expression.expected = Definition.INT_TYPE;
-            expression.explicit = true;
+        } else if (shift) {
+            if (shiftDistance.sort == Sort.LONG) {
+                expression.expected = Definition.INT_TYPE;
+                expression.explicit = true;   
+            } else {
+                expression.expected = shiftDistance;
+            }
         } else {
             expression.expected = promote;
         }
@@ -335,7 +347,7 @@ public final class EChain extends AExpression {
                                                                                  // to the promotion type between the lhs and rhs types
                     expression.write(writer);                                    // write the bytecode for the rhs expression
                     // XXX: fix these types, but first we need def compound assignment tests.
-                    // (and also corner cases such as shifts). its tricky here as there are possibly explicit casts, too.
+                    // its tricky here as there are possibly explicit casts, too.
                     // write the operation instruction for compound assignment
                     if (promote.sort == Sort.DEF) {
                         writer.writeDynamicBinaryInstruction(location, promote, 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java
index c41ec15bd35..21f3b87ca1d 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java
@@ -369,4 +369,29 @@ public class AdditionTests extends ScriptTestCase {
         assertEquals(2D, exec("def x = (float)1; double y = (double)1; return x + y"));
         assertEquals(2D, exec("def x = (double)1; double y = (double)1; return x + y"));
     }
+    
+    public void testCompoundAssignment() {
+        // byte
+        assertEquals((byte) 15, exec("byte x = 5; x += 10; return x;"));
+        assertEquals((byte) -5, exec("byte x = 5; x += -10; return x;"));
+
+        // short
+        assertEquals((short) 15, exec("short x = 5; x += 10; return x;"));
+        assertEquals((short) -5, exec("short x = 5; x += -10; return x;"));
+        // char
+        assertEquals((char) 15, exec("char x = 5; x += 10; return x;"));
+        assertEquals((char) 5, exec("char x = 10; x += -5; return x;"));
+        // int
+        assertEquals(15, exec("int x = 5; x += 10; return x;"));
+        assertEquals(-5, exec("int x = 5; x += -10; return x;"));
+        // long
+        assertEquals(15L, exec("long x = 5; x += 10; return x;"));
+        assertEquals(-5L, exec("long x = 5; x += -10; return x;"));
+        // float
+        assertEquals(15F, exec("float x = 5f; x += 10; return x;"));
+        assertEquals(-5F, exec("float x = 5f; x += -10; return x;"));
+        // double
+        assertEquals(15D, exec("double x = 5.0; x += 10; return x;"));
+        assertEquals(-5D, exec("double x = 5.0; x += -10; return x;"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java
index 42cf3611219..a5ffa3faa35 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java
@@ -208,4 +208,50 @@ public class AndTests extends ScriptTestCase {
         assertEquals(false, exec("def x = false; boolean y = true; return x & y"));
         assertEquals(false, exec("def x = false; boolean y = false; return x & y"));
     }
+    
+    public void testCompoundAssignment() {
+        // boolean
+        assertEquals(true, exec("boolean x = true; x &= true; return x;"));
+        assertEquals(false, exec("boolean x = true; x &= false; return x;"));
+        assertEquals(false, exec("boolean x = false; x &= true; return x;"));
+        assertEquals(false, exec("boolean x = false; x &= false; return x;"));
+        assertEquals(true, exec("def x = true; x &= true; return x;"));
+        assertEquals(false, exec("def x = true; x &= false; return x;"));
+        assertEquals(false, exec("def x = false; x &= true; return x;"));
+        assertEquals(false, exec("def x = false; x &= false; return x;"));
+        assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] &= true; return x[0];"));
+        assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] &= false; return x[0];"));
+        assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] &= true; return x[0];"));
+        assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] &= false; return x[0];"));
+        assertEquals(true, exec("def[] x = new def[1]; x[0] = true; x[0] &= true; return x[0];"));
+        assertEquals(false, exec("def[] x = new def[1]; x[0] = true; x[0] &= false; return x[0];"));
+        assertEquals(false, exec("def[] x = new def[1]; x[0] = false; x[0] &= true; return x[0];"));
+        assertEquals(false, exec("def[] x = new def[1]; x[0] = false; x[0] &= false; return x[0];"));
+
+        // byte
+        assertEquals((byte) (13 & 14), exec("byte x = 13; x &= 14; return x;"));
+        // short
+        assertEquals((short) (13 & 14), exec("short x = 13; x &= 14; return x;"));
+        // char
+        assertEquals((char) (13 & 14), exec("char x = 13; x &= 14; return x;"));
+        // int
+        assertEquals(13 & 14, exec("int x = 13; x &= 14; return x;"));
+        // long
+        assertEquals((long) (13 & 14), exec("long x = 13L; x &= 14; return x;"));
+    }
+    
+    public void testBogusCompoundAssignment() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("float x = 4; int y = 1; x &= y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("double x = 4; int y = 1; x &= y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("int x = 4; float y = 1; x &= y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("int x = 4; double y = 1; x &= y");
+        });
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/CompoundAssignmentTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/CompoundAssignmentTests.java
deleted file mode 100644
index 73a92edf668..00000000000
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/CompoundAssignmentTests.java
+++ /dev/null
@@ -1,314 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless;
-
-/**
- * Tests compound assignments (+=, etc) across all data types
- */
-public class CompoundAssignmentTests extends ScriptTestCase {
-    public void testAddition() {
-        // byte
-        assertEquals((byte) 15, exec("byte x = 5; x += 10; return x;"));
-        assertEquals((byte) -5, exec("byte x = 5; x += -10; return x;"));
-
-        // short
-        assertEquals((short) 15, exec("short x = 5; x += 10; return x;"));
-        assertEquals((short) -5, exec("short x = 5; x += -10; return x;"));
-        // char
-        assertEquals((char) 15, exec("char x = 5; x += 10; return x;"));
-        assertEquals((char) 5, exec("char x = 10; x += -5; return x;"));
-        // int
-        assertEquals(15, exec("int x = 5; x += 10; return x;"));
-        assertEquals(-5, exec("int x = 5; x += -10; return x;"));
-        // long
-        assertEquals(15L, exec("long x = 5; x += 10; return x;"));
-        assertEquals(-5L, exec("long x = 5; x += -10; return x;"));
-        // float
-        assertEquals(15F, exec("float x = 5f; x += 10; return x;"));
-        assertEquals(-5F, exec("float x = 5f; x += -10; return x;"));
-        // double
-        assertEquals(15D, exec("double x = 5.0; x += 10; return x;"));
-        assertEquals(-5D, exec("double x = 5.0; x += -10; return x;"));
-    }
-
-    public void testSubtraction() {
-        // byte
-        assertEquals((byte) 15, exec("byte x = 5; x -= -10; return x;"));
-        assertEquals((byte) -5, exec("byte x = 5; x -= 10; return x;"));
-        // short
-        assertEquals((short) 15, exec("short x = 5; x -= -10; return x;"));
-        assertEquals((short) -5, exec("short x = 5; x -= 10; return x;"));
-        // char
-        assertEquals((char) 15, exec("char x = 5; x -= -10; return x;"));
-        assertEquals((char) 5, exec("char x = 10; x -= 5; return x;"));
-        // int
-        assertEquals(15, exec("int x = 5; x -= -10; return x;"));
-        assertEquals(-5, exec("int x = 5; x -= 10; return x;"));
-        // long
-        assertEquals(15L, exec("long x = 5; x -= -10; return x;"));
-        assertEquals(-5L, exec("long x = 5; x -= 10; return x;"));
-        // float
-        assertEquals(15F, exec("float x = 5f; x -= -10; return x;"));
-        assertEquals(-5F, exec("float x = 5f; x -= 10; return x;"));
-        // double
-        assertEquals(15D, exec("double x = 5.0; x -= -10; return x;"));
-        assertEquals(-5D, exec("double x = 5.0; x -= 10; return x;"));
-    }
-
-    public void testMultiplication() {
-        // byte
-        assertEquals((byte) 15, exec("byte x = 5; x *= 3; return x;"));
-        assertEquals((byte) -5, exec("byte x = 5; x *= -1; return x;"));
-        // short
-        assertEquals((short) 15, exec("short x = 5; x *= 3; return x;"));
-        assertEquals((short) -5, exec("short x = 5; x *= -1; return x;"));
-        // char
-        assertEquals((char) 15, exec("char x = 5; x *= 3; return x;"));
-        // int
-        assertEquals(15, exec("int x = 5; x *= 3; return x;"));
-        assertEquals(-5, exec("int x = 5; x *= -1; return x;"));
-        // long
-        assertEquals(15L, exec("long x = 5; x *= 3; return x;"));
-        assertEquals(-5L, exec("long x = 5; x *= -1; return x;"));
-        // float
-        assertEquals(15F, exec("float x = 5f; x *= 3; return x;"));
-        assertEquals(-5F, exec("float x = 5f; x *= -1; return x;"));
-        // double
-        assertEquals(15D, exec("double x = 5.0; x *= 3; return x;"));
-        assertEquals(-5D, exec("double x = 5.0; x *= -1; return x;"));
-    }
-
-    public void testDivision() {
-        // byte
-        assertEquals((byte) 15, exec("byte x = 45; x /= 3; return x;"));
-        assertEquals((byte) -5, exec("byte x = 5; x /= -1; return x;"));
-        // short
-        assertEquals((short) 15, exec("short x = 45; x /= 3; return x;"));
-        assertEquals((short) -5, exec("short x = 5; x /= -1; return x;"));
-        // char
-        assertEquals((char) 15, exec("char x = 45; x /= 3; return x;"));
-        // int
-        assertEquals(15, exec("int x = 45; x /= 3; return x;"));
-        assertEquals(-5, exec("int x = 5; x /= -1; return x;"));
-        // long
-        assertEquals(15L, exec("long x = 45; x /= 3; return x;"));
-        assertEquals(-5L, exec("long x = 5; x /= -1; return x;"));
-        // float
-        assertEquals(15F, exec("float x = 45f; x /= 3; return x;"));
-        assertEquals(-5F, exec("float x = 5f; x /= -1; return x;"));
-        // double
-        assertEquals(15D, exec("double x = 45.0; x /= 3; return x;"));
-        assertEquals(-5D, exec("double x = 5.0; x /= -1; return x;"));
-    }
-
-    public void testDivisionByZero() {
-        // byte
-        expectScriptThrows(ArithmeticException.class, () -> {
-            exec("byte x = 1; x /= 0; return x;");
-        });
-
-        // short
-        expectScriptThrows(ArithmeticException.class, () -> {
-            exec("short x = 1; x /= 0; return x;");
-        });
-
-        // char
-        expectScriptThrows(ArithmeticException.class, () -> {
-            exec("char x = 1; x /= 0; return x;");
-        });
-
-        // int
-        expectScriptThrows(ArithmeticException.class, () -> {
-            exec("int x = 1; x /= 0; return x;");
-        });
-
-        // long
-        expectScriptThrows(ArithmeticException.class, () -> {
-            exec("long x = 1; x /= 0; return x;");
-        });
-    }
-
-    public void testRemainder() {
-        // byte
-        assertEquals((byte) 3, exec("byte x = 15; x %= 4; return x;"));
-        assertEquals((byte) -3, exec("byte x = (byte) -15; x %= 4; return x;"));
-        // short
-        assertEquals((short) 3, exec("short x = 15; x %= 4; return x;"));
-        assertEquals((short) -3, exec("short x = (short) -15; x %= 4; return x;"));
-        // char
-        assertEquals((char) 3, exec("char x = (char) 15; x %= 4; return x;"));
-        // int
-        assertEquals(3, exec("int x = 15; x %= 4; return x;"));
-        assertEquals(-3, exec("int x = -15; x %= 4; return x;"));
-        // long
-        assertEquals(3L, exec("long x = 15L; x %= 4; return x;"));
-        assertEquals(-3L, exec("long x = -15L; x %= 4; return x;"));
-        // float
-        assertEquals(3F, exec("float x = 15F; x %= 4; return x;"));
-        assertEquals(-3F, exec("float x = -15F; x %= 4; return x;"));
-        // double
-        assertEquals(3D, exec("double x = 15.0; x %= 4; return x;"));
-        assertEquals(-3D, exec("double x = -15.0; x %= 4; return x;"));
-    }
-
-    public void testLeftShift() {
-        // byte
-        assertEquals((byte) 60, exec("byte x = 15; x <<= 2; return x;"));
-        assertEquals((byte) -60, exec("byte x = (byte) -15; x <<= 2; return x;"));
-        // short
-        assertEquals((short) 60, exec("short x = 15; x <<= 2; return x;"));
-        assertEquals((short) -60, exec("short x = (short) -15; x <<= 2; return x;"));
-        // char
-        assertEquals((char) 60, exec("char x = (char) 15; x <<= 2; return x;"));
-        // int
-        assertEquals(60, exec("int x = 15; x <<= 2; return x;"));
-        assertEquals(-60, exec("int x = -15; x <<= 2; return x;"));
-        // long
-        assertEquals(60L, exec("long x = 15L; x <<= 2; return x;"));
-        assertEquals(-60L, exec("long x = -15L; x <<= 2; return x;"));
-    }
-
-    public void testRightShift() {
-        // byte
-        assertEquals((byte) 15, exec("byte x = 60; x >>= 2; return x;"));
-        assertEquals((byte) -15, exec("byte x = (byte) -60; x >>= 2; return x;"));
-        // short
-        assertEquals((short) 15, exec("short x = 60; x >>= 2; return x;"));
-        assertEquals((short) -15, exec("short x = (short) -60; x >>= 2; return x;"));
-        // char
-        assertEquals((char) 15, exec("char x = (char) 60; x >>= 2; return x;"));
-        // int
-        assertEquals(15, exec("int x = 60; x >>= 2; return x;"));
-        assertEquals(-15, exec("int x = -60; x >>= 2; return x;"));
-        // long
-        assertEquals(15L, exec("long x = 60L; x >>= 2; return x;"));
-        assertEquals(-15L, exec("long x = -60L; x >>= 2; return x;"));
-    }
-
-    public void testUnsignedRightShift() {
-        // byte
-        assertEquals((byte) 15, exec("byte x = 60; x >>>= 2; return x;"));
-        assertEquals((byte) -15, exec("byte x = (byte) -60; x >>>= 2; return x;"));
-        // short
-        assertEquals((short) 15, exec("short x = 60; x >>>= 2; return x;"));
-        assertEquals((short) -15, exec("short x = (short) -60; x >>>= 2; return x;"));
-        // char
-        assertEquals((char) 15, exec("char x = (char) 60; x >>>= 2; return x;"));
-        // int
-        assertEquals(15, exec("int x = 60; x >>>= 2; return x;"));
-        assertEquals(-60 >>> 2, exec("int x = -60; x >>>= 2; return x;"));
-        // long
-        assertEquals(15L, exec("long x = 60L; x >>>= 2; return x;"));
-        assertEquals(-60L >>> 2, exec("long x = -60L; x >>>= 2; return x;"));
-    }
-
-    public void testAnd() {
-        // boolean
-        assertEquals(true, exec("boolean x = true; x &= true; return x;"));
-        assertEquals(false, exec("boolean x = true; x &= false; return x;"));
-        assertEquals(false, exec("boolean x = false; x &= true; return x;"));
-        assertEquals(false, exec("boolean x = false; x &= false; return x;"));
-        assertEquals(true, exec("def x = true; x &= true; return x;"));
-        assertEquals(false, exec("def x = true; x &= false; return x;"));
-        assertEquals(false, exec("def x = false; x &= true; return x;"));
-        assertEquals(false, exec("def x = false; x &= false; return x;"));
-        assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] &= true; return x[0];"));
-        assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] &= false; return x[0];"));
-        assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] &= true; return x[0];"));
-        assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] &= false; return x[0];"));
-        assertEquals(true, exec("def[] x = new def[1]; x[0] = true; x[0] &= true; return x[0];"));
-        assertEquals(false, exec("def[] x = new def[1]; x[0] = true; x[0] &= false; return x[0];"));
-        assertEquals(false, exec("def[] x = new def[1]; x[0] = false; x[0] &= true; return x[0];"));
-        assertEquals(false, exec("def[] x = new def[1]; x[0] = false; x[0] &= false; return x[0];"));
-
-        // byte
-        assertEquals((byte) (13 & 14), exec("byte x = 13; x &= 14; return x;"));
-        // short
-        assertEquals((short) (13 & 14), exec("short x = 13; x &= 14; return x;"));
-        // char
-        assertEquals((char) (13 & 14), exec("char x = 13; x &= 14; return x;"));
-        // int
-        assertEquals(13 & 14, exec("int x = 13; x &= 14; return x;"));
-        // long
-        assertEquals((long) (13 & 14), exec("long x = 13L; x &= 14; return x;"));
-    }
-
-    public void testOr() {
-        // boolean
-        assertEquals(true, exec("boolean x = true; x |= true; return x;"));
-        assertEquals(true, exec("boolean x = true; x |= false; return x;"));
-        assertEquals(true, exec("boolean x = false; x |= true; return x;"));
-        assertEquals(false, exec("boolean x = false; x |= false; return x;"));
-        assertEquals(true, exec("def x = true; x |= true; return x;"));
-        assertEquals(true, exec("def x = true; x |= false; return x;"));
-        assertEquals(true, exec("def x = false; x |= true; return x;"));
-        assertEquals(false, exec("def x = false; x |= false; return x;"));
-        assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] |= true; return x[0];"));
-        assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] |= false; return x[0];"));
-        assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] |= true; return x[0];"));
-        assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] |= false; return x[0];"));
-        assertEquals(true, exec("def[] x = new def[1]; x[0] = true; x[0] |= true; return x[0];"));
-        assertEquals(true, exec("def[] x = new def[1]; x[0] = true; x[0] |= false; return x[0];"));
-        assertEquals(true, exec("def[] x = new def[1]; x[0] = false; x[0] |= true; return x[0];"));
-        assertEquals(false, exec("def[] x = new def[1]; x[0] = false; x[0] |= false; return x[0];"));
-
-        // byte
-        assertEquals((byte) (13 | 14), exec("byte x = 13; x |= 14; return x;"));
-        // short
-        assertEquals((short) (13 | 14), exec("short x = 13; x |= 14; return x;"));
-        // char
-        assertEquals((char) (13 | 14), exec("char x = 13; x |= 14; return x;"));
-        // int
-        assertEquals(13 | 14, exec("int x = 13; x |= 14; return x;"));
-        // long
-        assertEquals((long) (13 | 14), exec("long x = 13L; x |= 14; return x;"));
-    }
-
-    public void testXor() {
-        // boolean
-        assertEquals(false, exec("boolean x = true; x ^= true; return x;"));
-        assertEquals(true, exec("boolean x = true; x ^= false; return x;"));
-        assertEquals(true, exec("boolean x = false; x ^= true; return x;"));
-        assertEquals(false, exec("boolean x = false; x ^= false; return x;"));
-        assertEquals(false, exec("def x = true; x ^= true; return x;"));
-        assertEquals(true, exec("def x = true; x ^= false; return x;"));
-        assertEquals(true, exec("def x = false; x ^= true; return x;"));
-        assertEquals(false, exec("def x = false; x ^= false; return x;"));
-        assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] ^= true; return x[0];"));
-        assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] ^= false; return x[0];"));
-        assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] ^= true; return x[0];"));
-        assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] ^= false; return x[0];"));
-        assertEquals(false, exec("def[] x = new def[1]; x[0] = true; x[0] ^= true; return x[0];"));
-        assertEquals(true, exec("def[] x = new def[1]; x[0] = true; x[0] ^= false; return x[0];"));
-        assertEquals(true, exec("def[] x = new def[1]; x[0] = false; x[0] ^= true; return x[0];"));
-        assertEquals(false, exec("def[] x = new def[1]; x[0] = false; x[0] ^= false; return x[0];"));
-
-        // byte
-        assertEquals((byte) (13 ^ 14), exec("byte x = 13; x ^= 14; return x;"));
-        // short
-        assertEquals((short) (13 ^ 14), exec("short x = 13; x ^= 14; return x;"));
-        // char
-        assertEquals((char) (13 ^ 14), exec("char x = 13; x ^= 14; return x;"));
-        // int
-        assertEquals(13 ^ 14, exec("int x = 13; x ^= 14; return x;"));
-        // long
-        assertEquals((long) (13 ^ 14), exec("long x = 13L; x ^= 14; return x;"));
-    }
-}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java
index 226cf6f1710..dbe14ea2fc9 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java
@@ -335,4 +335,54 @@ public class DivisionTests extends ScriptTestCase {
         assertEquals(1F, exec("def x = (float)2; float y = (float)2; return x / y"));
         assertEquals(1D, exec("def x = (double)2; double y = (double)2; return x / y"));
     }
+    
+    public void testCompoundAssignment() {
+        // byte
+        assertEquals((byte) 15, exec("byte x = 45; x /= 3; return x;"));
+        assertEquals((byte) -5, exec("byte x = 5; x /= -1; return x;"));
+        // short
+        assertEquals((short) 15, exec("short x = 45; x /= 3; return x;"));
+        assertEquals((short) -5, exec("short x = 5; x /= -1; return x;"));
+        // char
+        assertEquals((char) 15, exec("char x = 45; x /= 3; return x;"));
+        // int
+        assertEquals(15, exec("int x = 45; x /= 3; return x;"));
+        assertEquals(-5, exec("int x = 5; x /= -1; return x;"));
+        // long
+        assertEquals(15L, exec("long x = 45; x /= 3; return x;"));
+        assertEquals(-5L, exec("long x = 5; x /= -1; return x;"));
+        // float
+        assertEquals(15F, exec("float x = 45f; x /= 3; return x;"));
+        assertEquals(-5F, exec("float x = 5f; x /= -1; return x;"));
+        // double
+        assertEquals(15D, exec("double x = 45.0; x /= 3; return x;"));
+        assertEquals(-5D, exec("double x = 5.0; x /= -1; return x;"));
+    }
+    
+    public void testCompoundAssignmentByZero() {
+        // byte
+        expectScriptThrows(ArithmeticException.class, () -> {
+            exec("byte x = 1; x /= 0; return x;");
+        });
+
+        // short
+        expectScriptThrows(ArithmeticException.class, () -> {
+            exec("short x = 1; x /= 0; return x;");
+        });
+
+        // char
+        expectScriptThrows(ArithmeticException.class, () -> {
+            exec("char x = 1; x /= 0; return x;");
+        });
+
+        // int
+        expectScriptThrows(ArithmeticException.class, () -> {
+            exec("int x = 1; x /= 0; return x;");
+        });
+
+        // long
+        expectScriptThrows(ArithmeticException.class, () -> {
+            exec("long x = 1; x /= 0; return x;");
+        });
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/MultiplicationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/MultiplicationTests.java
index 2ccb51d7946..44612ee0c76 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/MultiplicationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/MultiplicationTests.java
@@ -325,4 +325,27 @@ public class MultiplicationTests extends ScriptTestCase {
         assertEquals(4F, exec("def x = (float)2; float y = (float)2; return x * y"));
         assertEquals(4D, exec("def x = (double)2; double y = (double)2; return x * y"));
     }
+    
+    public void testCompoundAssignment() {
+        // byte
+        assertEquals((byte) 15, exec("byte x = 5; x *= 3; return x;"));
+        assertEquals((byte) -5, exec("byte x = 5; x *= -1; return x;"));
+        // short
+        assertEquals((short) 15, exec("short x = 5; x *= 3; return x;"));
+        assertEquals((short) -5, exec("short x = 5; x *= -1; return x;"));
+        // char
+        assertEquals((char) 15, exec("char x = 5; x *= 3; return x;"));
+        // int
+        assertEquals(15, exec("int x = 5; x *= 3; return x;"));
+        assertEquals(-5, exec("int x = 5; x *= -1; return x;"));
+        // long
+        assertEquals(15L, exec("long x = 5; x *= 3; return x;"));
+        assertEquals(-5L, exec("long x = 5; x *= -1; return x;"));
+        // float
+        assertEquals(15F, exec("float x = 5f; x *= 3; return x;"));
+        assertEquals(-5F, exec("float x = 5f; x *= -1; return x;"));
+        // double
+        assertEquals(15D, exec("double x = 5.0; x *= 3; return x;"));
+        assertEquals(-5D, exec("double x = 5.0; x *= -1; return x;"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java
index 5cf9c186fd6..5863ec60b38 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java
@@ -208,4 +208,50 @@ public class OrTests extends ScriptTestCase {
         assertEquals(true,  exec("def x = false; boolean y = true; return x | y"));
         assertEquals(false, exec("def x = false; boolean y = false; return x | y"));
     }
+    
+    public void testCompoundAssignment() {
+        // boolean
+        assertEquals(true, exec("boolean x = true; x |= true; return x;"));
+        assertEquals(true, exec("boolean x = true; x |= false; return x;"));
+        assertEquals(true, exec("boolean x = false; x |= true; return x;"));
+        assertEquals(false, exec("boolean x = false; x |= false; return x;"));
+        assertEquals(true, exec("def x = true; x |= true; return x;"));
+        assertEquals(true, exec("def x = true; x |= false; return x;"));
+        assertEquals(true, exec("def x = false; x |= true; return x;"));
+        assertEquals(false, exec("def x = false; x |= false; return x;"));
+        assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] |= true; return x[0];"));
+        assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] |= false; return x[0];"));
+        assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] |= true; return x[0];"));
+        assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] |= false; return x[0];"));
+        assertEquals(true, exec("def[] x = new def[1]; x[0] = true; x[0] |= true; return x[0];"));
+        assertEquals(true, exec("def[] x = new def[1]; x[0] = true; x[0] |= false; return x[0];"));
+        assertEquals(true, exec("def[] x = new def[1]; x[0] = false; x[0] |= true; return x[0];"));
+        assertEquals(false, exec("def[] x = new def[1]; x[0] = false; x[0] |= false; return x[0];"));
+
+        // byte
+        assertEquals((byte) (13 | 14), exec("byte x = 13; x |= 14; return x;"));
+        // short
+        assertEquals((short) (13 | 14), exec("short x = 13; x |= 14; return x;"));
+        // char
+        assertEquals((char) (13 | 14), exec("char x = 13; x |= 14; return x;"));
+        // int
+        assertEquals(13 | 14, exec("int x = 13; x |= 14; return x;"));
+        // long
+        assertEquals((long) (13 | 14), exec("long x = 13L; x |= 14; return x;"));
+    }
+    
+    public void testBogusCompoundAssignment() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("float x = 4; int y = 1; x |= y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("double x = 4; int y = 1; x |= y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("int x = 4; float y = 1; x |= y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("int x = 4; double y = 1; x |= y");
+        });
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java
index d2c9406c551..b42a3ca1d58 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java
@@ -335,4 +335,27 @@ public class RemainderTests extends ScriptTestCase {
         assertEquals(0F, exec("def x = (float)2; float y = (float)2; return x % y"));
         assertEquals(0D, exec("def x = (double)2; double y = (double)2; return x % y"));
     }
+    
+    public void testCompoundAssignment() {
+        // byte
+        assertEquals((byte) 3, exec("byte x = 15; x %= 4; return x;"));
+        assertEquals((byte) -3, exec("byte x = (byte) -15; x %= 4; return x;"));
+        // short
+        assertEquals((short) 3, exec("short x = 15; x %= 4; return x;"));
+        assertEquals((short) -3, exec("short x = (short) -15; x %= 4; return x;"));
+        // char
+        assertEquals((char) 3, exec("char x = (char) 15; x %= 4; return x;"));
+        // int
+        assertEquals(3, exec("int x = 15; x %= 4; return x;"));
+        assertEquals(-3, exec("int x = -15; x %= 4; return x;"));
+        // long
+        assertEquals(3L, exec("long x = 15L; x %= 4; return x;"));
+        assertEquals(-3L, exec("long x = -15L; x %= 4; return x;"));
+        // float
+        assertEquals(3F, exec("float x = 15F; x %= 4; return x;"));
+        assertEquals(-3F, exec("float x = -15F; x %= 4; return x;"));
+        // double
+        assertEquals(3D, exec("double x = 15.0; x %= 4; return x;"));
+        assertEquals(-3D, exec("double x = -15.0; x %= 4; return x;"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ShiftTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ShiftTests.java
index d409c8c5f7f..b6aa86a839d 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ShiftTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ShiftTests.java
@@ -544,4 +544,93 @@ public class ShiftTests extends ScriptTestCase {
         });
     }
 
+    public void testLshCompoundAssignment() {
+        // byte
+        assertEquals((byte) 60, exec("byte x = 15; x <<= 2; return x;"));
+        assertEquals((byte) -60, exec("byte x = (byte) -15; x <<= 2; return x;"));
+        // short
+        assertEquals((short) 60, exec("short x = 15; x <<= 2; return x;"));
+        assertEquals((short) -60, exec("short x = (short) -15; x <<= 2; return x;"));
+        // char
+        assertEquals((char) 60, exec("char x = (char) 15; x <<= 2; return x;"));
+        // int
+        assertEquals(60, exec("int x = 15; x <<= 2; return x;"));
+        assertEquals(-60, exec("int x = -15; x <<= 2; return x;"));
+        // long
+        assertEquals(60L, exec("long x = 15L; x <<= 2; return x;"));
+        assertEquals(-60L, exec("long x = -15L; x <<= 2; return x;"));
+        // long shift distance
+        assertEquals(60, exec("int x = 15; x <<= 2L; return x;"));
+        assertEquals(-60, exec("int x = -15; x <<= 2L; return x;"));
+    }
+
+    public void testRshCompoundAssignment() {
+        // byte
+        assertEquals((byte) 15, exec("byte x = 60; x >>= 2; return x;"));
+        assertEquals((byte) -15, exec("byte x = (byte) -60; x >>= 2; return x;"));
+        // short
+        assertEquals((short) 15, exec("short x = 60; x >>= 2; return x;"));
+        assertEquals((short) -15, exec("short x = (short) -60; x >>= 2; return x;"));
+        // char
+        assertEquals((char) 15, exec("char x = (char) 60; x >>= 2; return x;"));
+        // int
+        assertEquals(15, exec("int x = 60; x >>= 2; return x;"));
+        assertEquals(-15, exec("int x = -60; x >>= 2; return x;"));
+        // long
+        assertEquals(15L, exec("long x = 60L; x >>= 2; return x;"));
+        assertEquals(-15L, exec("long x = -60L; x >>= 2; return x;"));
+        // long shift distance
+        assertEquals(15, exec("int x = 60; x >>= 2L; return x;"));
+        assertEquals(-15, exec("int x = -60; x >>= 2L; return x;"));
+    }
+
+    public void testUshCompoundAssignment() {
+        // byte
+        assertEquals((byte) 15, exec("byte x = 60; x >>>= 2; return x;"));
+        assertEquals((byte) -15, exec("byte x = (byte) -60; x >>>= 2; return x;"));
+        // short
+        assertEquals((short) 15, exec("short x = 60; x >>>= 2; return x;"));
+        assertEquals((short) -15, exec("short x = (short) -60; x >>>= 2; return x;"));
+        // char
+        assertEquals((char) 15, exec("char x = (char) 60; x >>>= 2; return x;"));
+        // int
+        assertEquals(15, exec("int x = 60; x >>>= 2; return x;"));
+        assertEquals(-60 >>> 2, exec("int x = -60; x >>>= 2; return x;"));
+        // long
+        assertEquals(15L, exec("long x = 60L; x >>>= 2; return x;"));
+        assertEquals(-60L >>> 2, exec("long x = -60L; x >>>= 2; return x;"));
+        // long shift distance
+        assertEquals(15, exec("int x = 60; x >>>= 2L; return x;"));
+        assertEquals(-60 >>> 2, exec("int x = -60; x >>>= 2L; return x;"));
+    }
+    
+    public void testBogusCompoundAssignment() {
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("long x = 1L; float y = 2; x <<= y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("int x = 1; double y = 2L; x <<= y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("float x = 1F; int y = 2; x <<= y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("double x = 1D; int y = 2L; x <<= y;");
+        });
+    }
+
+    public void testBogusCompoundAssignmentConst() {
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("int x = 1L; x <<= 2F;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("int x = 1L; x <<= 2.0;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("float x = 1F; x <<= 2;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("double x = 1D; x <<= 2L;");
+        });
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/SubtractionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/SubtractionTests.java
index 4817b040326..447e164d205 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/SubtractionTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/SubtractionTests.java
@@ -355,4 +355,28 @@ public class SubtractionTests extends ScriptTestCase {
         assertEquals(0D, exec("def x = (float)1; double y = (double)1; return x - y"));
         assertEquals(0D, exec("def x = (double)1; double y = (double)1; return x - y"));
     }
+    
+    public void testCompoundAssignment() {
+        // byte
+        assertEquals((byte) 15, exec("byte x = 5; x -= -10; return x;"));
+        assertEquals((byte) -5, exec("byte x = 5; x -= 10; return x;"));
+        // short
+        assertEquals((short) 15, exec("short x = 5; x -= -10; return x;"));
+        assertEquals((short) -5, exec("short x = 5; x -= 10; return x;"));
+        // char
+        assertEquals((char) 15, exec("char x = 5; x -= -10; return x;"));
+        assertEquals((char) 5, exec("char x = 10; x -= 5; return x;"));
+        // int
+        assertEquals(15, exec("int x = 5; x -= -10; return x;"));
+        assertEquals(-5, exec("int x = 5; x -= 10; return x;"));
+        // long
+        assertEquals(15L, exec("long x = 5; x -= -10; return x;"));
+        assertEquals(-5L, exec("long x = 5; x -= 10; return x;"));
+        // float
+        assertEquals(15F, exec("float x = 5f; x -= -10; return x;"));
+        assertEquals(-5F, exec("float x = 5f; x -= 10; return x;"));
+        // double
+        assertEquals(15D, exec("double x = 5.0; x -= -10; return x;"));
+        assertEquals(-5D, exec("double x = 5.0; x -= 10; return x;"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java
index 1921539b0bf..eee49c309d3 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java
@@ -222,4 +222,50 @@ public class XorTests extends ScriptTestCase {
         assertEquals(true,  exec("def x = false; boolean y = true; return x ^ y"));
         assertEquals(false, exec("def x = false; boolean y = false; return x ^ y"));
     }
+    
+    public void testCompoundAssignment() {
+        // boolean
+        assertEquals(false, exec("boolean x = true; x ^= true; return x;"));
+        assertEquals(true, exec("boolean x = true; x ^= false; return x;"));
+        assertEquals(true, exec("boolean x = false; x ^= true; return x;"));
+        assertEquals(false, exec("boolean x = false; x ^= false; return x;"));
+        assertEquals(false, exec("def x = true; x ^= true; return x;"));
+        assertEquals(true, exec("def x = true; x ^= false; return x;"));
+        assertEquals(true, exec("def x = false; x ^= true; return x;"));
+        assertEquals(false, exec("def x = false; x ^= false; return x;"));
+        assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] ^= true; return x[0];"));
+        assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] ^= false; return x[0];"));
+        assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] ^= true; return x[0];"));
+        assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] ^= false; return x[0];"));
+        assertEquals(false, exec("def[] x = new def[1]; x[0] = true; x[0] ^= true; return x[0];"));
+        assertEquals(true, exec("def[] x = new def[1]; x[0] = true; x[0] ^= false; return x[0];"));
+        assertEquals(true, exec("def[] x = new def[1]; x[0] = false; x[0] ^= true; return x[0];"));
+        assertEquals(false, exec("def[] x = new def[1]; x[0] = false; x[0] ^= false; return x[0];"));
+
+        // byte
+        assertEquals((byte) (13 ^ 14), exec("byte x = 13; x ^= 14; return x;"));
+        // short
+        assertEquals((short) (13 ^ 14), exec("short x = 13; x ^= 14; return x;"));
+        // char
+        assertEquals((char) (13 ^ 14), exec("char x = 13; x ^= 14; return x;"));
+        // int
+        assertEquals(13 ^ 14, exec("int x = 13; x ^= 14; return x;"));
+        // long
+        assertEquals((long) (13 ^ 14), exec("long x = 13L; x ^= 14; return x;"));
+    }
+    
+    public void testBogusCompoundAssignment() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("float x = 4; int y = 1; x ^= y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("double x = 4; int y = 1; x ^= y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("int x = 4; float y = 1; x ^= y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("int x = 4; double y = 1; x ^= y");
+        });
+    }
 }

From e96722d91cadc81aca7989ca036a17ac39991205 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 14 Jun 2016 21:34:24 -0400
Subject: [PATCH 0852/1311] Add search preference to prefer multiple nodes

The search preference _prefer_node allows specifying a single node to
prefer when routing a request. This functionality can be enhanced by
permitting multiple nodes to be preferred. This commit replaces the
search preference _prefer_node with the search preference _prefer_nodes
which supplants the former by specifying a single node and otherwise
adds functionality.

Relates #18872
---
 .../routing/IndexShardRoutingTable.java       | 19 +++---
 .../cluster/routing/OperationRouting.java     | 16 ++---
 .../cluster/routing/Preference.java           |  9 +--
 .../routing/OperationRoutingTests.java        | 59 +++++++++++++++++++
 .../structure/RoutingIteratorTests.java       | 17 ++++--
 .../search/preference/SearchPreferenceIT.java |  2 +-
 .../migration/migrate_5_0/search.asciidoc     |  7 +++
 .../search/request/preference.asciidoc        |  6 +-
 8 files changed, 107 insertions(+), 28 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
index a1e891bce3d..ceea83dbcaa 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
@@ -375,21 +375,22 @@ public class IndexShardRoutingTable implements Iterable<ShardRouting> {
         return new PlainShardIterator(shardId, ordered);
     }
 
-    public ShardIterator preferNodeActiveInitializingShardsIt(String nodeId) {
-        ArrayList<ShardRouting> ordered = new ArrayList<>(activeShards.size() + allInitializingShards.size());
+    public ShardIterator preferNodeActiveInitializingShardsIt(Set<String> nodeIds) {
+        ArrayList<ShardRouting> preferred = new ArrayList<>(activeShards.size() + allInitializingShards.size());
+        ArrayList<ShardRouting> notPreferred = new ArrayList<>(activeShards.size() + allInitializingShards.size());
         // fill it in a randomized fashion
         for (ShardRouting shardRouting : shuffler.shuffle(activeShards)) {
-            ordered.add(shardRouting);
-            if (nodeId.equals(shardRouting.currentNodeId())) {
-                // switch, its the matching node id
-                ordered.set(ordered.size() - 1, ordered.get(0));
-                ordered.set(0, shardRouting);
+            if (nodeIds.contains(shardRouting.currentNodeId())) {
+                preferred.add(shardRouting);
+            } else {
+                notPreferred.add(shardRouting);
             }
         }
+        preferred.addAll(notPreferred);
         if (!allInitializingShards.isEmpty()) {
-            ordered.addAll(allInitializingShards);
+            preferred.addAll(allInitializingShards);
         }
-        return new PlainShardIterator(shardId, ordered);
+        return new PlainShardIterator(shardId, preferred);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
index f95f3003a4a..129650de991 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
@@ -33,17 +33,15 @@ import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardNotFoundException;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
-/**
- *
- */
 public class OperationRouting extends AbstractComponent {
 
-
     private final AwarenessAllocationDecider awarenessAllocationDecider;
 
     @Inject
@@ -158,10 +156,14 @@ public class OperationRouting extends AbstractComponent {
             }
             preferenceType = Preference.parse(preference);
             switch (preferenceType) {
-                case PREFER_NODE:
-                    return indexShard.preferNodeActiveInitializingShardsIt(preference.substring(Preference.PREFER_NODE.type().length() + 1));
+                case PREFER_NODES:
+                    final Set<String> nodesIds =
+                            Arrays.stream(
+                                    preference.substring(Preference.PREFER_NODES.type().length() + 1).split(",")
+                            ).collect(Collectors.toSet());
+                    return indexShard.preferNodeActiveInitializingShardsIt(nodesIds);
                 case LOCAL:
-                    return indexShard.preferNodeActiveInitializingShardsIt(localNodeId);
+                    return indexShard.preferNodeActiveInitializingShardsIt(Collections.singleton(localNodeId));
                 case PRIMARY:
                     return indexShard.primaryActiveInitializingShardIt();
                 case REPLICA:
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/Preference.java b/core/src/main/java/org/elasticsearch/cluster/routing/Preference.java
index 6de251b9d52..cf0dd6cc54d 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/Preference.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/Preference.java
@@ -30,9 +30,9 @@ public enum  Preference {
     SHARDS("_shards"),
 
     /**
-     * Route to preferred node, if possible
+     * Route to preferred nodes, if possible
      */
-    PREFER_NODE("_prefer_node"),
+    PREFER_NODES("_prefer_nodes"),
 
     /**
      * Route to local node, if possible
@@ -98,8 +98,8 @@ public enum  Preference {
         switch (preferenceType) {
             case "_shards":
                 return SHARDS;
-            case "_prefer_node":
-                return PREFER_NODE;
+            case "_prefer_nodes":
+                return PREFER_NODES;
             case "_only_node":
                 return ONLY_NODE;
             case "_local":
@@ -123,6 +123,7 @@ public enum  Preference {
                 throw new IllegalArgumentException("no Preference for [" + preferenceType + "]");
         }
     }
+
 }
 
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java
index e515e0f9bf4..0127702c5ea 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java
@@ -18,16 +18,31 @@
  */
 package org.elasticsearch.cluster.routing;
 
+import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 
+import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
 
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+
 public class OperationRoutingTests extends ESTestCase{
 
     public void testGenerateShardId() {
@@ -170,4 +185,48 @@ public class OperationRoutingTests extends ESTestCase{
             assertEquals(shard, entry.getValue().intValue());
         }
     }
+
+    public void testPreferNodes() throws InterruptedException, IOException {
+        TestThreadPool threadPool = null;
+        ClusterService clusterService = null;
+        try {
+            threadPool = new TestThreadPool("testPreferNodes");
+            clusterService = ClusterServiceUtils.createClusterService(threadPool);
+            final String indexName = "test";
+            ClusterServiceUtils.setState(clusterService, ClusterStateCreationUtils.stateWithActivePrimary(indexName, true, randomInt(8)));
+            final Index index = clusterService.state().metaData().index(indexName).getIndex();
+            final List<ShardRouting> shards = clusterService.state().getRoutingNodes().assignedShards(new ShardId(index, 0));
+            final int count = randomIntBetween(1, shards.size());
+            int position = 0;
+            final List<String> nodes = new ArrayList<>();
+            final List<ShardRouting> expected = new ArrayList<>();
+            for (int i = 0; i < count; i++) {
+                if (randomBoolean() && !shards.get(position).initializing()) {
+                    nodes.add(shards.get(position).currentNodeId());
+                    expected.add(shards.get(position));
+                    position++;
+                } else {
+                    nodes.add("missing_" + i);
+                }
+            }
+            final ShardIterator it =
+                    new OperationRouting(Settings.EMPTY, new AwarenessAllocationDecider())
+                            .getShards(clusterService.state(), indexName, 0, "_prefer_nodes:" + String.join(",", nodes));
+            final List<ShardRouting> all = new ArrayList<>();
+            ShardRouting shard;
+            while ((shard = it.nextOrNull()) != null) {
+                all.add(shard);
+            }
+            final Set<ShardRouting> preferred = new HashSet<>();
+            preferred.addAll(all.subList(0, expected.size()));
+            // the preferred shards should be at the front of the list
+            assertThat(preferred, containsInAnyOrder(expected.toArray()));
+            // verify all the shards are there
+            assertThat(all.size(), equalTo(shards.size()));
+        } finally {
+            IOUtils.close(clusterService);
+            terminate(threadPool);
+        }
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java b/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java
index c9dd0e26361..e9ec914d0dc 100644
--- a/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java
@@ -43,6 +43,7 @@ import org.elasticsearch.test.ESAllocationTestCase;
 
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
 
 import static java.util.Collections.singletonMap;
@@ -401,15 +402,23 @@ public class RoutingIteratorTests extends ESAllocationTestCase {
         assertThat(shardIterators.iterator().next().shardId().id(), equalTo(0));
         assertThat(shardIterators.iterator().next().nextOrNull().currentNodeId(), not(equalTo(firstRoundNodeId)));
 
-        shardIterators = operationRouting.searchShards(clusterState, new String[]{"test"}, null, "_shards:0;_prefer_node:node1");
+        shardIterators = operationRouting.searchShards(clusterState, new String[]{"test"}, null, "_shards:0;_prefer_nodes:node1");
         assertThat(shardIterators.size(), equalTo(1));
         assertThat(shardIterators.iterator().next().shardId().id(), equalTo(0));
         assertThat(shardIterators.iterator().next().nextOrNull().currentNodeId(), equalTo("node1"));
 
-        shardIterators = operationRouting.searchShards(clusterState, new String[]{"test"}, null, "_shards:0;_prefer_node:node1");
+        shardIterators = operationRouting.searchShards(clusterState, new String[]{"test"}, null, "_shards:0;_prefer_nodes:node1,node2");
         assertThat(shardIterators.size(), equalTo(1));
-        assertThat(shardIterators.iterator().next().shardId().id(), equalTo(0));
-        assertThat(shardIterators.iterator().next().nextOrNull().currentNodeId(), equalTo("node1"));
+        Iterator<ShardIterator> iterator = shardIterators.iterator();
+        final ShardIterator it = iterator.next();
+        assertThat(it.shardId().id(), equalTo(0));
+        final String firstNodeId = it.nextOrNull().currentNodeId();
+        assertThat(firstNodeId, anyOf(equalTo("node1"), equalTo("node2")));
+        if ("node1".equals(firstNodeId)) {
+            assertThat(it.nextOrNull().currentNodeId(), equalTo("node2"));
+        } else {
+            assertThat(it.nextOrNull().currentNodeId(), equalTo("node1"));
+        }
     }
 
     public void testReplicaShardPreferenceIters() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java
index 8bfadeb9443..49142988a46 100644
--- a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java
+++ b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java
@@ -56,7 +56,7 @@ public class SearchPreferenceIT extends ESIntegTestCase {
         refresh();
         internalCluster().stopRandomDataNode();
         client().admin().cluster().prepareHealth().setWaitForStatus(ClusterHealthStatus.RED).execute().actionGet();
-        String[] preferences = new String[] {"_primary", "_local", "_primary_first", "_prefer_node:somenode", "_prefer_node:server2"};
+        String[] preferences = new String[] {"_primary", "_local", "_primary_first", "_prefer_nodes:somenode", "_prefer_nodes:server2", "_prefer_nodes:somenode,server2"};
         for (String pref : preferences) {
             logger.info("--> Testing out preference={}", pref);
             SearchResponse searchResponse = client().prepareSearch().setSize(0).setPreference(pref).execute().actionGet();
diff --git a/docs/reference/migration/migrate_5_0/search.asciidoc b/docs/reference/migration/migrate_5_0/search.asciidoc
index 62384afa60a..c20df657504 100644
--- a/docs/reference/migration/migrate_5_0/search.asciidoc
+++ b/docs/reference/migration/migrate_5_0/search.asciidoc
@@ -185,3 +185,10 @@ This is now consistent for source filtering on other places in the search API.
 In the response for profiling queries, the `query_type` has been renamed to `type` and `lucene` has been renamed to
 `description`. These changes have been made so the response format is more friendly to supporting other types of profiling
 in the future.
+
+==== Search prefernce
+
+The <<search-request-preference,search preference>> `_prefer_node` has
+been superseded by `_prefer_nodes`. By specifying a single node,
+`_prefer_nodes` provides the same functionality as `_prefer_node` but
+also supports specifying multiple nodes.
diff --git a/docs/reference/search/request/preference.asciidoc b/docs/reference/search/request/preference.asciidoc
index 3d6c6b40cb9..761fdeaa369 100644
--- a/docs/reference/search/request/preference.asciidoc
+++ b/docs/reference/search/request/preference.asciidoc
@@ -31,9 +31,9 @@ The `preference` is a query string parameter which can be set to:
 	Restricts the search to execute only on a node with
 	the provided node id (`xyz` in this case).
 
-`_prefer_node:xyz`:: 
-	Prefers execution on the node with the provided
-	node id (`xyz` in this case) if applicable.
+`_prefer_nodes:abc,xyz`::
+	Prefers execution on the nodes with the provided
+	node ids (`abc` or `xyz` in this case) if applicable.
 
 `_shards:2,3`:: 
 	Restricts the operation to the specified shards. (`2`

From fa77d4d88527c64d3ff3751e4fe4338c00b37ed0 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Tue, 14 Jun 2016 18:45:07 -0700
Subject: [PATCH 0853/1311] Test: make secure mock setup work with ibm jdk

---
 .../resources/org/elasticsearch/bootstrap/test-framework.policy | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
index d7faab5eeda..7abebf01be1 100644
--- a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
+++ b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
@@ -24,6 +24,8 @@
 grant codeBase "${codebase.securemock-1.2.jar}" {
   // needed to access ReflectionFactory (see below)
   permission java.lang.RuntimePermission "accessClassInPackage.sun.reflect";
+  // needed for reflection in ibm jdk
+  permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
   // needed to support creation of mocks
   permission java.lang.RuntimePermission "reflectionFactoryAccess";
   // needed for spy interception, etc

From 2b1ebc55a9b1e9fefd10116b8b99047a86c96822 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 14 Jun 2016 23:52:21 -0400
Subject: [PATCH 0854/1311] remove null guards from +, if not needed.

improve DefBootstrapTests
---
 .../resources/forbidden/es-all-signatures.txt |  5 +
 .../forbidden/es-core-signatures.txt          |  5 -
 .../elasticsearch/painless/DefBootstrap.java  | 28 +++++-
 .../elasticsearch/painless/MethodWriter.java  | 27 ++---
 .../elasticsearch/painless/node/EComp.java    | 14 +--
 .../elasticsearch/painless/node/EUnary.java   |  6 +-
 .../elasticsearch/painless/AdditionTests.java | 12 +++
 .../painless/DefBootstrapTests.java           | 98 ++++++++++++++++---
 .../painless/DefOptimizationTests.java        | 19 ++++
 .../painless/ScriptTestCase.java              |  9 ++
 10 files changed, 181 insertions(+), 42 deletions(-)

diff --git a/buildSrc/src/main/resources/forbidden/es-all-signatures.txt b/buildSrc/src/main/resources/forbidden/es-all-signatures.txt
index e31a7020282..71eb1433521 100644
--- a/buildSrc/src/main/resources/forbidden/es-all-signatures.txt
+++ b/buildSrc/src/main/resources/forbidden/es-all-signatures.txt
@@ -31,3 +31,8 @@ org.apache.lucene.index.IndexReader#getCombinedCoreAndDeletesKey()
 
 @defaultMessage Soon to be removed
 org.apache.lucene.document.FieldType#numericType()
+
+@defaultMessage Don't use MethodHandles in slow ways, dont be lenient in tests.
+# unfortunately, invoke() cannot be banned, because forbidden apis does not support signature polymorphic methods
+java.lang.invoke.MethodHandle#invokeWithArguments(java.lang.Object[])
+java.lang.invoke.MethodHandle#invokeWithArguments(java.util.List)
diff --git a/buildSrc/src/main/resources/forbidden/es-core-signatures.txt b/buildSrc/src/main/resources/forbidden/es-core-signatures.txt
index 31084c7556c..059be403a67 100644
--- a/buildSrc/src/main/resources/forbidden/es-core-signatures.txt
+++ b/buildSrc/src/main/resources/forbidden/es-core-signatures.txt
@@ -92,8 +92,3 @@ org.joda.time.DateTime#<init>(int, int, int, int, int, int)
 org.joda.time.DateTime#<init>(int, int, int, int, int, int, int)
 org.joda.time.DateTime#now()
 org.joda.time.DateTimeZone#getDefault()
-
-@defaultMessage Don't use MethodHandles in slow ways, except in tests.
-java.lang.invoke.MethodHandle#invoke(java.lang.Object[])
-java.lang.invoke.MethodHandle#invokeWithArguments(java.lang.Object[])
-java.lang.invoke.MethodHandle#invokeWithArguments(java.util.List)
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index c688b314243..32428686cbf 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -68,6 +68,15 @@ public final class DefBootstrap {
     public static final int BINARY_OPERATOR = 8;
     /** static bootstrap parameter indicating a shift operator, e.g. foo &gt;&gt; bar */
     public static final int SHIFT_OPERATOR = 9;
+    
+    // constants for the flags parameter of operators
+    /** 
+     * static bootstrap parameter indicating the binary operator allows nulls (e.g. == and +) 
+     * <p>
+     * requires additional {@link MethodHandles#catchException} guard, which will invoke
+     * the fallback if a null is encountered.
+     */
+    public static final int OPERATOR_ALLOWS_NULL = 1 << 0;
 
     /**
      * CallSite that implements the polymorphic inlining cache (PIC).
@@ -151,7 +160,7 @@ public final class DefBootstrap {
                     return DefMath.lookupUnary(args[0].getClass(), name);
                 case BINARY_OPERATOR:
                     if (args[0] == null || args[1] == null) {
-                        return getGeneric(flavor, name); // can handle nulls
+                        return getGeneric(flavor, name); // can handle nulls, if supported
                     } else {
                         return DefMath.lookupBinary(args[0].getClass(), args[1].getClass(), name);
                     }
@@ -225,7 +234,7 @@ public final class DefBootstrap {
             MethodHandle guard = MethodHandles.guardWithTest(test, target, getTarget());
             // very special cases, where even the receiver can be null (see JLS rules for string concat)
             // we wrap + with an NPE catcher, and use our generic method in that case.
-            if (flavor == BINARY_OPERATOR && "add".equals(name) || "eq".equals(name)) {
+            if (flavor == BINARY_OPERATOR && ((int)this.args[0] & OPERATOR_ALLOWS_NULL) != 0) {
                 MethodHandle handler = MethodHandles.dropArguments(getGeneric(flavor, name).asType(type()), 0, NullPointerException.class);
                 guard = MethodHandles.catchException(guard, NullPointerException.class, handler);
             }
@@ -288,6 +297,21 @@ public final class DefBootstrap {
                     throw new BootstrapMethodError("Illegal parameter for reference call: " + args[0]);
                 }
                 break;
+            case UNARY_OPERATOR:
+            case SHIFT_OPERATOR:
+            case BINARY_OPERATOR:
+                if (args.length != 1) {
+                    throw new BootstrapMethodError("Invalid number of parameters for operator call");
+                }
+                if (args[0] instanceof Integer == false) {
+                    throw new BootstrapMethodError("Illegal parameter for reference call: " + args[0]);
+                }
+                int flags = (int)args[0];
+                if ((flags & OPERATOR_ALLOWS_NULL) != 0 && flavor != BINARY_OPERATOR) {
+                    // we just don't need it anywhere else.
+                    throw new BootstrapMethodError("This parameter is only supported for BINARY_OPERATORs");
+                }
+                break;
             default:
                 if (args.length > 0) {
                     throw new BootstrapMethodError("Illegal static bootstrap parameters for flavor: " + flavor);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index 3280c3bbcfe..b4923beabdc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -269,37 +269,42 @@ public final class MethodWriter extends GeneratorAdapter {
         
         switch (operation) {
             case MUL:
-                invokeDynamic("mul", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
+                invokeDynamic("mul", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, 0); 
                 break;
             case DIV:
-                invokeDynamic("div", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
+                invokeDynamic("div", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, 0); 
                 break;
             case REM:
-                invokeDynamic("rem", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
+                invokeDynamic("rem", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, 0); 
                 break;
             case ADD:
-                invokeDynamic("add", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
+                // if either side is primitive, then the + operator should always throw NPE on null,
+                // so we don't need a special NPE guard.
+                // otherwise, we need to allow nulls for possible string concatenation.
+                boolean hasPrimitiveArg = lhs.clazz.isPrimitive() || rhs.clazz.isPrimitive();
+                int flags = hasPrimitiveArg ? 0 : DefBootstrap.OPERATOR_ALLOWS_NULL;
+                invokeDynamic("add", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, flags);
                 break;
             case SUB:
-                invokeDynamic("sub", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR); 
+                invokeDynamic("sub", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, 0); 
                 break;
             case LSH:
-                invokeDynamic("lsh", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR); 
+                invokeDynamic("lsh", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR, 0); 
                 break;
             case USH:
-                invokeDynamic("ush", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR); 
+                invokeDynamic("ush", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR, 0); 
                 break;
             case RSH:
-                invokeDynamic("rsh", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR); 
+                invokeDynamic("rsh", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR, 0); 
                 break;
             case BWAND: 
-                invokeDynamic("and", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
+                invokeDynamic("and", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, 0);
                 break;
             case XOR:   
-                invokeDynamic("xor", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
+                invokeDynamic("xor", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, 0);
                 break;
             case BWOR:  
-                invokeDynamic("or", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
+                invokeDynamic("or", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, 0);
                 break;
             default:
                 throw location.createError(new IllegalStateException("Illegal tree structure."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index 1f4f691d4aa..03c6351e95c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -497,7 +497,8 @@ public final class EComp extends AExpression {
                     if (right.isNull) {
                         writer.ifNull(jump);
                     } else if (!left.isNull && (operation == Operation.EQ || operation == Operation.NE)) {
-                        writer.invokeDynamic("eq", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
+                        writer.invokeDynamic("eq", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR,
+                                                                                                     DefBootstrap.OPERATOR_ALLOWS_NULL);
                         writejump = false;
                     } else {
                         writer.ifCmp(promotedType.type, MethodWriter.EQ, jump);
@@ -506,22 +507,23 @@ public final class EComp extends AExpression {
                     if (right.isNull) {
                         writer.ifNonNull(jump);
                     } else if (!left.isNull && (operation == Operation.EQ || operation == Operation.NE)) {
-                        writer.invokeDynamic("eq", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
+                        writer.invokeDynamic("eq", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR,
+                                                                                                     DefBootstrap.OPERATOR_ALLOWS_NULL);
                         writer.ifZCmp(MethodWriter.EQ, jump);
                     } else {
                         writer.ifCmp(promotedType.type, MethodWriter.NE, jump);
                     }
                 } else if (lt) {
-                    writer.invokeDynamic("lt", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
+                    writer.invokeDynamic("lt", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, 0);
                     writejump = false;
                 } else if (lte) {
-                    writer.invokeDynamic("lte", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
+                    writer.invokeDynamic("lte", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, 0);
                     writejump = false;
                 } else if (gt) {
-                    writer.invokeDynamic("gt", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
+                    writer.invokeDynamic("gt", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, 0);
                     writejump = false;
                 } else if (gte) {
-                    writer.invokeDynamic("gte", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR);
+                    writer.invokeDynamic("gte", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, 0);
                     writejump = false;
                 } else {
                     throw createError(new IllegalStateException("Illegal tree structure."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index bb2ff4d1d47..f4f73e6101b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -205,7 +205,7 @@ public final class EUnary extends AExpression {
             if (operation == Operation.BWNOT) {
                 if (sort == Sort.DEF) {
                     org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(actual.type, child.actual.type);
-                    writer.invokeDynamic("not", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.UNARY_OPERATOR);
+                    writer.invokeDynamic("not", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.UNARY_OPERATOR, 0);
                 } else {
                     if (sort == Sort.INT) {
                         writer.push(-1);
@@ -220,14 +220,14 @@ public final class EUnary extends AExpression {
             } else if (operation == Operation.SUB) {
                 if (sort == Sort.DEF) {
                     org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(actual.type, child.actual.type);
-                    writer.invokeDynamic("neg", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.UNARY_OPERATOR);
+                    writer.invokeDynamic("neg", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.UNARY_OPERATOR, 0);
                 } else {
                     writer.math(MethodWriter.NEG, actual.type);
                 }
             } else if (operation == Operation.ADD) {
                 if (sort == Sort.DEF) {
                     org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(actual.type, child.actual.type);
-                    writer.invokeDynamic("plus", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.UNARY_OPERATOR);
+                    writer.invokeDynamic("plus", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.UNARY_OPERATOR, 0);
                 } 
             } else {
                 throw createError(new IllegalStateException("Illegal tree structure."));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java
index 21f3b87ca1d..59bf81b31f6 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java
@@ -370,6 +370,18 @@ public class AdditionTests extends ScriptTestCase {
         assertEquals(2D, exec("def x = (double)1; double y = (double)1; return x + y"));
     }
     
+    public void testDefNulls() {
+        expectScriptThrows(NullPointerException.class, () -> {
+            exec("def x = null; int y = 1; return x + y"); 
+        });
+        expectScriptThrows(NullPointerException.class, () -> {
+            exec("int x = 1; def y = null; return x + y"); 
+        });
+        expectScriptThrows(NullPointerException.class, () -> {
+            exec("def x = null; def y = 1; return x + y"); 
+        });
+    }
+    
     public void testCompoundAssignment() {
         // byte
         assertEquals((byte) 15, exec("byte x = 5; x += 10; return x;"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
index 12104844e7c..e551a89ea5a 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
@@ -40,11 +40,11 @@ public class DefBootstrapTests extends ESTestCase {
         assertDepthEquals(site, 0);
 
         // invoke with integer, needs lookup
-        assertEquals("5", handle.invoke(Integer.valueOf(5)));
+        assertEquals("5", (String)handle.invokeExact((Object)5));
         assertDepthEquals(site, 1);
 
         // invoked with integer again: should be cached
-        assertEquals("6", handle.invoke(Integer.valueOf(6)));
+        assertEquals("6", (String)handle.invokeExact((Object)6));
         assertDepthEquals(site, 1);
     }
     
@@ -56,15 +56,15 @@ public class DefBootstrapTests extends ESTestCase {
         MethodHandle handle = site.dynamicInvoker();
         assertDepthEquals(site, 0);
 
-        assertEquals("5", handle.invoke(Integer.valueOf(5)));
+        assertEquals("5", (String)handle.invokeExact((Object)5));
         assertDepthEquals(site, 1);
-        assertEquals("1.5", handle.invoke(Float.valueOf(1.5f)));
+        assertEquals("1.5", (String)handle.invokeExact((Object)1.5f));
         assertDepthEquals(site, 2);
 
         // both these should be cached
-        assertEquals("6", handle.invoke(Integer.valueOf(6)));
+        assertEquals("6", (String)handle.invokeExact((Object)6));
         assertDepthEquals(site, 2);
-        assertEquals("2.5", handle.invoke(Float.valueOf(2.5f)));
+        assertEquals("2.5", (String)handle.invokeExact((Object)2.5f));
         assertDepthEquals(site, 2);
     }
     
@@ -78,17 +78,17 @@ public class DefBootstrapTests extends ESTestCase {
         MethodHandle handle = site.dynamicInvoker();
         assertDepthEquals(site, 0);
 
-        assertEquals("5", handle.invoke(Integer.valueOf(5)));
+        assertEquals("5", (String)handle.invokeExact((Object)5));
         assertDepthEquals(site, 1);
-        assertEquals("1.5", handle.invoke(Float.valueOf(1.5f)));
+        assertEquals("1.5", (String)handle.invokeExact((Object)1.5f));
         assertDepthEquals(site, 2);
-        assertEquals("6", handle.invoke(Long.valueOf(6)));
+        assertEquals("6", (String)handle.invokeExact((Object)6L));
         assertDepthEquals(site, 3);
-        assertEquals("3.2", handle.invoke(Double.valueOf(3.2d)));
+        assertEquals("3.2", (String)handle.invokeExact((Object)3.2d));
         assertDepthEquals(site, 4);
-        assertEquals("foo", handle.invoke(new String("foo")));
+        assertEquals("foo", (String)handle.invokeExact((Object)"foo"));
         assertDepthEquals(site, 5);
-        assertEquals("c", handle.invoke(Character.valueOf('c')));
+        assertEquals("c", (String)handle.invokeExact((Object)'c'));
         assertDepthEquals(site, 5);
     }
     
@@ -100,9 +100,77 @@ public class DefBootstrapTests extends ESTestCase {
                                                                           DefBootstrap.METHOD_CALL, 0L);
         site.depth = DefBootstrap.PIC.MAX_DEPTH; // mark megamorphic
         MethodHandle handle = site.dynamicInvoker();
-        // arguments are cast to object here, or IDE compilers eat it :)
-        assertEquals(2, handle.invoke((Object) Arrays.asList("1", "2")));
-        assertEquals(1, handle.invoke((Object) Collections.singletonMap("a", "b")));
+        assertEquals(2, (int)handle.invokeExact((Object) Arrays.asList("1", "2")));
+        assertEquals(1, (int)handle.invokeExact((Object) Collections.singletonMap("a", "b")));
+    }
+    
+    // test operators with null guards
+
+    public void testNullGuardAdd() throws Throwable {
+        DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
+                                                               "add", 
+                                                               MethodType.methodType(Object.class, Object.class, Object.class),
+                                                               DefBootstrap.BINARY_OPERATOR, DefBootstrap.OPERATOR_ALLOWS_NULL);
+        MethodHandle handle = site.dynamicInvoker();
+        assertEquals("nulltest", (Object)handle.invokeExact((Object)null, (Object)"test"));
+    }
+    
+    public void testNullGuardAddWhenCached() throws Throwable {
+        DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
+                                                               "add", 
+                                                               MethodType.methodType(Object.class, Object.class, Object.class),
+                                                               DefBootstrap.BINARY_OPERATOR, DefBootstrap.OPERATOR_ALLOWS_NULL);
+        MethodHandle handle = site.dynamicInvoker();
+        assertEquals(2, (Object)handle.invokeExact((Object)1, (Object)1));
+        assertEquals("nulltest", (Object)handle.invoke((Object)null, (Object)"test"));
+    }
+    
+    public void testNullGuardEq() throws Throwable {
+        DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
+                                                               "eq", 
+                                                               MethodType.methodType(boolean.class, Object.class, Object.class),
+                                                               DefBootstrap.BINARY_OPERATOR, DefBootstrap.OPERATOR_ALLOWS_NULL);
+        MethodHandle handle = site.dynamicInvoker();
+        assertFalse((boolean) handle.invokeExact((Object)null, (Object)"test"));
+        assertTrue((boolean) handle.invokeExact((Object)null, (Object)null));
+    }
+    
+    public void testNullGuardEqWhenCached() throws Throwable {
+        DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
+                                                               "eq", 
+                                                               MethodType.methodType(boolean.class, Object.class, Object.class),
+                                                               DefBootstrap.BINARY_OPERATOR, DefBootstrap.OPERATOR_ALLOWS_NULL);
+        MethodHandle handle = site.dynamicInvoker();
+        assertTrue((boolean) handle.invokeExact((Object)1, (Object)1));
+        assertFalse((boolean) handle.invokeExact((Object)null, (Object)"test"));
+        assertTrue((boolean) handle.invokeExact((Object)null, (Object)null));
+    }
+    
+    // make sure these operators work without null guards too
+    // for example, nulls are only legal for + if the other parameter is a String,
+    // and can be disabled in some circumstances.
+    
+    public void testNoNullGuardAdd() throws Throwable {
+        DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
+                                                               "add", 
+                                                               MethodType.methodType(Object.class, int.class, Object.class),
+                                                               DefBootstrap.BINARY_OPERATOR, 0);
+        MethodHandle handle = site.dynamicInvoker();
+        expectThrows(NullPointerException.class, () -> {
+            assertNotNull((Object)handle.invokeExact(5, (Object)null));
+        });
+    }
+    
+    public void testNoNullGuardAddWhenCached() throws Throwable {
+        DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
+                                                               "add", 
+                                                               MethodType.methodType(Object.class, int.class, Object.class),
+                                                               DefBootstrap.BINARY_OPERATOR, 0);
+        MethodHandle handle = site.dynamicInvoker();
+        assertEquals(2, (Object)handle.invokeExact(1, (Object)1));
+        expectThrows(NullPointerException.class, () -> {
+            assertNotNull((Object)handle.invokeExact(5, (Object)null));
+        });
     }
     
     static void assertDepthEquals(CallSite site, int expected) {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
index 86461880a99..6fb9151cb0d 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
@@ -239,6 +239,25 @@ public class DefOptimizationTests extends ScriptTestCase {
                              "INVOKEDYNAMIC add(Ljava/lang/Object;Ljava/lang/Object;)D");
     }
     
+    // horrible, sorry
+    public void testAddOptNullGuards() {
+        // needs null guard
+        assertBytecodeHasPattern("def x = 1; def y = 2; return x + y", 
+                "(?s).*INVOKEDYNAMIC add.*arguments:\\s+" + DefBootstrap.BINARY_OPERATOR 
+                                                + ",\\s+" + DefBootstrap.OPERATOR_ALLOWS_NULL + ".*");
+        // still needs null guard, NPE is the wrong thing!
+        assertBytecodeHasPattern("def x = 1; def y = 2; double z = x + y", 
+                "(?s).*INVOKEDYNAMIC add.*arguments:\\s+" + DefBootstrap.BINARY_OPERATOR 
+                                                + ",\\s+" + DefBootstrap.OPERATOR_ALLOWS_NULL + ".*");
+        // a primitive argument is present: no null guard needed
+        assertBytecodeHasPattern("def x = 1; int y = 2; return x + y", 
+                "(?s).*INVOKEDYNAMIC add.*arguments:\\s+" + DefBootstrap.BINARY_OPERATOR 
+                                                + ",\\s+" + 0 + ".*");
+        assertBytecodeHasPattern("int x = 1; def y = 2; return x + y", 
+                "(?s).*INVOKEDYNAMIC add.*arguments:\\s+" + DefBootstrap.BINARY_OPERATOR 
+                                                + ",\\s+" + 0 + ".*");
+    }
+    
     public void testSubOptLHS() {
         assertBytecodeExists("int x = 1; def y = 2; return x - y", 
                              "INVOKEDYNAMIC sub(ILjava/lang/Object;)Ljava/lang/Object;");
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
index cb35b53db5c..40f59e91120 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
@@ -79,6 +79,15 @@ public abstract class ScriptTestCase extends ESTestCase {
         assertTrue("bytecode not found, got: \n" + asm , asm.contains(bytecode));
     }
     
+    /**
+     * Uses the {@link Debugger} to get the bytecode output for a script and compare
+     * it against an expected bytecode pattern as a regular expression (please try to avoid!)
+     */
+    public void assertBytecodeHasPattern(String script, String pattern) {
+        final String asm = Debugger.toString(script);
+        assertTrue("bytecode not found, got: \n" + asm , asm.matches(pattern));
+    }
+    
     /** Checks a specific exception class is thrown (boxed inside ScriptException) and returns it. */
     public static <T extends Throwable> T expectScriptThrows(Class<T> expectedType, ThrowingRunnable runnable) {
         try {

From 6db323164e5373a3d5bfb35a41c2cb3005aed241 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Tue, 14 Jun 2016 20:01:34 -0700
Subject: [PATCH 0855/1311] Plugins: Emit nicer error message when trying to
 install unknown plugin

When installing plugins, we first try the elastic download service for
official plugins, then try maven coordinates, and finally try the
argument as a url. This can lead to confusing error messages about
unknown protocols when eg an official plugin name is mispelled. This
change adds a heuristic for determining if the argument in the final
case is in fact a url that we should try, and gives a simplified error
message in the case it is definitely not a url.

closes #17226
---
 .../org/elasticsearch/plugins/InstallPluginCommand.java    | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
index 9e17eec2e93..75ab95414a2 100644
--- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
@@ -134,6 +134,9 @@ class InstallPluginCommand extends SettingCommand {
         }
     }
 
+    // protocols allowed for direct url installation
+    private static final List<String> URL_PROTOCOLS = Arrays.asList("http", "https", "file");
+
     private final OptionSpec<Void> batchOption;
     private final OptionSpec<String> arguments;
 
@@ -237,6 +240,10 @@ class InstallPluginCommand extends SettingCommand {
         }
 
         // fall back to plain old URL
+        if (pluginId.contains("://") == false) {
+            // definitely not a valid url, so assume it is a plugin name
+            throw new UserError(ExitCodes.USAGE, "Unknown plugin " + pluginId);
+        }
         terminal.println("-> Downloading " + URLDecoder.decode(pluginId, "UTF-8"));
         return downloadZip(terminal, pluginId, tmpDir);
     }

From 1ecf14cee052eb30e46003c37bf411fd63ef890e Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Tue, 14 Jun 2016 23:01:37 -0700
Subject: [PATCH 0856/1311] Add test for plugin install heuristic

---
 .../org/elasticsearch/plugins/InstallPluginCommand.java     | 3 ---
 .../elasticsearch/plugins/InstallPluginCommandTests.java    | 6 ++++++
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
index 75ab95414a2..aa224b3c7c3 100644
--- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
@@ -134,9 +134,6 @@ class InstallPluginCommand extends SettingCommand {
         }
     }
 
-    // protocols allowed for direct url installation
-    private static final List<String> URL_PROTOCOLS = Arrays.asList("http", "https", "file");
-
     private final OptionSpec<Void> batchOption;
     private final OptionSpec<String> arguments;
 
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
index 5c766018f13..fdf52da2634 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
@@ -307,6 +307,12 @@ public class InstallPluginCommandTests extends ESTestCase {
         assertTrue(e.getMessage(), e.getMessage().contains("no protocol"));
     }
 
+    public void testUnknownPlugin() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        UserError e = expectThrows(UserError.class, () -> installPlugin("foo", env.v1()));
+        assertTrue(e.getMessage(), e.getMessage().contains("Unknown plugin foo"));
+    }
+
     public void testPluginsDirMissing() throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
         Files.delete(env.v2().pluginsFile());

From 429dd3a87680be91265357786fd44ed5af7756b9 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Wed, 15 Jun 2016 09:13:02 +0200
Subject: [PATCH 0857/1311] Simplify FetchSubPhase registration and detach it
 from Guice (#18862)

this commit removes FetchSubPhrase registration by class to registration
by instance. No Guice binding needed anymore.
---
 .../elasticsearch/search/Highlighters.java    |  2 +-
 .../elasticsearch/search/SearchModule.java    | 64 ++++++++-----------
 .../elasticsearch/search/SearchService.java   | 16 ++---
 .../search/fetch/FetchPhase.java              |  7 +-
 .../innerhits/InnerHitsFetchSubPhase.java     | 12 +---
 .../search/query/QueryPhase.java              | 11 ++--
 .../search/rescore/RescorePhase.java          |  2 -
 .../search/suggest/SuggestPhase.java          |  2 -
 .../search/fetch/FetchSubPhasePluginIT.java   |  2 +-
 .../PercolatorHighlightSubFetchPhase.java     | 14 ++--
 .../percolator/PercolatorPlugin.java          |  8 ++-
 ...PercolatorHighlightSubFetchPhaseTests.java |  5 +-
 .../search/MockSearchService.java             |  9 +--
 13 files changed, 67 insertions(+), 87 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/Highlighters.java b/core/src/main/java/org/elasticsearch/search/Highlighters.java
index 38fed839d6e..b66a6bda640 100644
--- a/core/src/main/java/org/elasticsearch/search/Highlighters.java
+++ b/core/src/main/java/org/elasticsearch/search/Highlighters.java
@@ -34,7 +34,7 @@ public final class Highlighters {
 
     private final Map<String, Highlighter> parsers = new HashMap<>();
 
-    Highlighters(Settings settings) {
+    public Highlighters(Settings settings) {
         registerHighlighter("fvh",  new FastVectorHighlighter(settings));
         registerHighlighter("plain", new PlainHighlighter());
         registerHighlighter("postings", new PostingsHighlighter());
diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java
index 2f626a82496..2b212756528 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -24,7 +24,6 @@ import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.geo.ShapesAvailability;
 import org.elasticsearch.common.geo.builders.ShapeBuilders;
 import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.inject.multibindings.Multibinder;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -92,7 +91,6 @@ import org.elasticsearch.index.query.functionscore.ScriptScoreFunctionBuilder;
 import org.elasticsearch.index.query.functionscore.WeightBuilder;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.action.SearchTransportService;
-import org.elasticsearch.search.aggregations.AggregationPhase;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
@@ -236,12 +234,10 @@ import org.elasticsearch.search.aggregations.pipeline.movavg.models.SimpleModel;
 import org.elasticsearch.search.aggregations.pipeline.serialdiff.SerialDiffPipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.serialdiff.SerialDiffPipelineAggregatorBuilder;
 import org.elasticsearch.search.controller.SearchPhaseController;
-import org.elasticsearch.search.dfs.DfsPhase;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.fetch.explain.ExplainFetchSubPhase;
 import org.elasticsearch.search.fetch.fielddata.FieldDataFieldsFetchSubPhase;
-import org.elasticsearch.search.fetch.innerhits.InnerHitsFetchSubPhase;
 import org.elasticsearch.search.fetch.matchedqueries.MatchedQueriesFetchSubPhase;
 import org.elasticsearch.search.fetch.parent.ParentFieldSubFetchPhase;
 import org.elasticsearch.search.fetch.script.ScriptFieldsFetchSubPhase;
@@ -249,7 +245,6 @@ import org.elasticsearch.search.fetch.source.FetchSourceSubPhase;
 import org.elasticsearch.search.fetch.version.VersionFetchSubPhase;
 import org.elasticsearch.search.highlight.HighlightPhase;
 import org.elasticsearch.search.highlight.Highlighter;
-import org.elasticsearch.search.query.QueryPhase;
 import org.elasticsearch.search.rescore.QueryRescorerBuilder;
 import org.elasticsearch.search.rescore.RescoreBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
@@ -261,6 +256,7 @@ import org.elasticsearch.search.suggest.Suggester;
 import org.elasticsearch.search.suggest.Suggesters;
 
 import java.util.HashSet;
+import java.util.Objects;
 import java.util.Set;
 
 /**
@@ -281,7 +277,7 @@ public class SearchModule extends AbstractModule {
     private final ParseFieldRegistry<MovAvgModel.AbstractModelParser> movingAverageModelParserRegistry = new ParseFieldRegistry<>(
             "moving_avg_model");
 
-    private final Set<Class<? extends FetchSubPhase>> fetchSubPhases = new HashSet<>();
+    private final Set<FetchSubPhase> fetchSubPhases = new HashSet<>();
 
     private final Settings settings;
     private final NamedWriteableRegistry namedWriteableRegistry;
@@ -303,6 +299,7 @@ public class SearchModule extends AbstractModule {
         registerBuiltinValueFormats();
         registerBuiltinSignificanceHeuristics();
         registerBuiltinMovingAverageModels();
+        registerBuiltinSubFetchPhases();
     }
 
     public void registerHighlighter(String key, Highlighter highligher) {
@@ -357,8 +354,19 @@ public class SearchModule extends AbstractModule {
         return queryParserRegistry;
     }
 
-    public void registerFetchSubPhase(Class<? extends FetchSubPhase> subPhase) {
-        fetchSubPhases.add(subPhase);
+    /**
+     * Registers a {@link FetchSubPhase} instance. This sub phase is executed when docuemnts are fetched for instanced to highlight
+     * documents.
+     */
+    public void registerFetchSubPhase(FetchSubPhase subPhase) {
+        fetchSubPhases.add(Objects.requireNonNull(subPhase, "FetchSubPhase must not be null"));
+    }
+
+    /**
+     * Returns the {@link Highlighter} registry
+     */
+    public Highlighters getHighlighters() {
+        return highlighters;
     }
 
     /**
@@ -437,31 +445,9 @@ public class SearchModule extends AbstractModule {
         bind(Suggesters.class).toInstance(suggesters);
         configureSearch();
         configureAggs();
-        configureHighlighters();
-        configureFetchSubPhase();
         configureShapes();
     }
 
-    protected void configureFetchSubPhase() {
-        Multibinder<FetchSubPhase> fetchSubPhaseMultibinder = Multibinder.newSetBinder(binder(), FetchSubPhase.class);
-        fetchSubPhaseMultibinder.addBinding().to(ExplainFetchSubPhase.class);
-        fetchSubPhaseMultibinder.addBinding().to(FieldDataFieldsFetchSubPhase.class);
-        fetchSubPhaseMultibinder.addBinding().to(ScriptFieldsFetchSubPhase.class);
-        fetchSubPhaseMultibinder.addBinding().to(FetchSourceSubPhase.class);
-        fetchSubPhaseMultibinder.addBinding().to(VersionFetchSubPhase.class);
-        fetchSubPhaseMultibinder.addBinding().to(MatchedQueriesFetchSubPhase.class);
-        fetchSubPhaseMultibinder.addBinding().to(HighlightPhase.class);
-        fetchSubPhaseMultibinder.addBinding().to(ParentFieldSubFetchPhase.class);
-        for (Class<? extends FetchSubPhase> clazz : fetchSubPhases) {
-            fetchSubPhaseMultibinder.addBinding().to(clazz);
-        }
-        bind(InnerHitsFetchSubPhase.class).asEagerSingleton();
-    }
-
-    protected void configureHighlighters() {
-       binder().bind(Highlighters.class).toInstance(highlighters);
-    }
-
     protected void configureAggs() {
         registerAggregation(AvgAggregationBuilder::new, new AvgParser(), AvgAggregationBuilder.AGGREGATION_NAME_FIELD);
         registerAggregation(SumAggregationBuilder::new, new SumParser(), SumAggregationBuilder.AGGREGATION_NAME_FIELD);
@@ -541,18 +527,13 @@ public class SearchModule extends AbstractModule {
                 BucketSelectorPipelineAggregatorBuilder.AGGREGATION_NAME_FIELD);
         registerPipelineAggregation(SerialDiffPipelineAggregatorBuilder::new, SerialDiffPipelineAggregatorBuilder::parse,
                 SerialDiffPipelineAggregatorBuilder.AGGREGATION_NAME_FIELD);
-
-        AggregationPhase aggPhase = new AggregationPhase();
         bind(AggregatorParsers.class).toInstance(aggregatorParsers);
-        bind(AggregationPhase.class).toInstance(aggPhase);
     }
 
     protected void configureSearch() {
         // configure search private classes...
-        bind(DfsPhase.class).asEagerSingleton();
-        bind(QueryPhase.class).asEagerSingleton();
         bind(SearchPhaseController.class).asEagerSingleton();
-        bind(FetchPhase.class).asEagerSingleton();
+        bind(FetchPhase.class).toInstance(new FetchPhase(fetchSubPhases));
         bind(SearchTransportService.class).asEagerSingleton();
         if (searchServiceImpl == SearchService.class) {
             bind(SearchService.class).asEagerSingleton();
@@ -623,6 +604,17 @@ public class SearchModule extends AbstractModule {
         registerMovingAverageModel(HoltWintersModel.NAME_FIELD, HoltWintersModel::new, HoltWintersModel.PARSER);
     }
 
+    private void registerBuiltinSubFetchPhases() {
+        registerFetchSubPhase(new ExplainFetchSubPhase());
+        registerFetchSubPhase(new FieldDataFieldsFetchSubPhase());
+        registerFetchSubPhase(new ScriptFieldsFetchSubPhase());
+        registerFetchSubPhase(new FetchSourceSubPhase());
+        registerFetchSubPhase(new VersionFetchSubPhase());
+        registerFetchSubPhase(new MatchedQueriesFetchSubPhase());
+        registerFetchSubPhase(new HighlightPhase(settings, highlighters));
+        registerFetchSubPhase(new ParentFieldSubFetchPhase());
+    }
+
     private void registerBuiltinQueryParsers() {
         registerQuery(MatchQueryBuilder::new, MatchQueryBuilder::fromXContent, MatchQueryBuilder.QUERY_NAME_FIELD);
         registerQuery(MatchPhraseQueryBuilder::new, MatchPhraseQueryBuilder::fromXContent, MatchPhraseQueryBuilder.QUERY_NAME_FIELD);
diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java
index b7a6d714e63..dfbde3a6b19 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchService.java
@@ -134,7 +134,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
 
     private final BigArrays bigArrays;
 
-    private final DfsPhase dfsPhase;
+    private final DfsPhase dfsPhase = new DfsPhase();
 
     private final QueryPhase queryPhase;
 
@@ -158,8 +158,8 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
 
     @Inject
     public SearchService(Settings settings, ClusterSettings clusterSettings, ClusterService clusterService, IndicesService indicesService,
-                         ThreadPool threadPool, ScriptService scriptService, BigArrays bigArrays, DfsPhase dfsPhase,
-                         QueryPhase queryPhase, FetchPhase fetchPhase, AggregatorParsers aggParsers, Suggesters suggesters) {
+                         ThreadPool threadPool, ScriptService scriptService, BigArrays bigArrays,
+                          FetchPhase fetchPhase, AggregatorParsers aggParsers, Suggesters suggesters) {
         super(settings);
         this.aggParsers = aggParsers;
         this.suggesters = suggesters;
@@ -169,8 +169,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
         this.indicesService = indicesService;
         this.scriptService = scriptService;
         this.bigArrays = bigArrays;
-        this.dfsPhase = dfsPhase;
-        this.queryPhase = queryPhase;
+        this.queryPhase = new QueryPhase(settings);
         this.fetchPhase = fetchPhase;
 
         TimeValue keepAliveInterval = KEEPALIVE_INTERVAL_SETTING.get(settings);
@@ -257,8 +256,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
     /**
      * Try to load the query results from the cache or execute the query phase directly if the cache cannot be used.
      */
-    private void loadOrExecuteQueryPhase(final ShardSearchRequest request, final SearchContext context,
-            final QueryPhase queryPhase) throws Exception {
+    private void loadOrExecuteQueryPhase(final ShardSearchRequest request, final SearchContext context) throws Exception {
         final boolean canCache = indicesService.canCache(request, context);
         if (canCache) {
             indicesService.loadIntoContext(request, context, queryPhase);
@@ -275,7 +273,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
             long time = System.nanoTime();
             contextProcessing(context);
 
-            loadOrExecuteQueryPhase(request, context, queryPhase);
+            loadOrExecuteQueryPhase(request, context);
 
             if (context.queryResult().topDocs().scoreDocs.length == 0 && context.scrollContext() == null) {
                 freeContext(context.id());
@@ -367,7 +365,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
             operationListener.onPreQueryPhase(context);
             long time = System.nanoTime();
             try {
-                loadOrExecuteQueryPhase(request, context, queryPhase);
+                loadOrExecuteQueryPhase(request, context);
             } catch (Throwable e) {
                 operationListener.onFailedQueryPhase(context);
                 throw ExceptionsHelper.convertToRuntime(e);
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
index 1b08df76743..50f07d04f6f 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
@@ -29,7 +29,6 @@ import org.apache.lucene.util.BitSet;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.text.Text;
@@ -73,11 +72,9 @@ public class FetchPhase implements SearchPhase {
 
     private final FetchSubPhase[] fetchSubPhases;
 
-    @Inject
-    public FetchPhase(Set<FetchSubPhase> fetchSubPhases, InnerHitsFetchSubPhase innerHitsFetchSubPhase) {
-        innerHitsFetchSubPhase.setFetchPhase(this);
+    public FetchPhase(Set<FetchSubPhase> fetchSubPhases) {
         this.fetchSubPhases = fetchSubPhases.toArray(new FetchSubPhase[fetchSubPhases.size() + 1]);
-        this.fetchSubPhases[fetchSubPhases.size()] = innerHitsFetchSubPhase;
+        this.fetchSubPhases[fetchSubPhases.size()] = new InnerHitsFetchSubPhase(this);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java
index 993d1ccbb5b..1382e8e2194 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java
@@ -23,7 +23,6 @@ import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.fetch.FetchSearchResult;
@@ -41,10 +40,10 @@ import java.util.Map;
  */
 public class InnerHitsFetchSubPhase implements FetchSubPhase {
 
-    private FetchPhase fetchPhase;
+    private final FetchPhase fetchPhase;
 
-    @Inject
-    public InnerHitsFetchSubPhase() {
+    public InnerHitsFetchSubPhase(FetchPhase fetchPhase) {
+        this.fetchPhase = fetchPhase;
     }
 
     @Override
@@ -102,9 +101,4 @@ public class InnerHitsFetchSubPhase implements FetchSubPhase {
     @Override
     public void hitsExecute(SearchContext context, InternalSearchHit[] hits) {
     }
-
-    // To get around cyclic dependency issue
-    public void setFetchPhase(FetchPhase fetchPhase) {
-        this.fetchPhase = fetchPhase;
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index 174a337f3d2..4e3a642f694 100644
--- a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -42,10 +42,10 @@ import org.apache.lucene.search.TopScoreDocCollector;
 import org.apache.lucene.search.TotalHitCountCollector;
 import org.apache.lucene.search.Weight;
 import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.MinimumScoreCollector;
 import org.elasticsearch.common.lucene.search.FilteredCollector;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.SearchPhase;
 import org.elasticsearch.search.SearchService;
@@ -76,11 +76,10 @@ public class QueryPhase implements SearchPhase {
     private final SuggestPhase suggestPhase;
     private RescorePhase rescorePhase;
 
-    @Inject
-    public QueryPhase(AggregationPhase aggregationPhase, SuggestPhase suggestPhase, RescorePhase rescorePhase) {
-        this.aggregationPhase = aggregationPhase;
-        this.suggestPhase = suggestPhase;
-        this.rescorePhase = rescorePhase;
+    public QueryPhase(Settings settings) {
+        this.aggregationPhase = new AggregationPhase();
+        this.suggestPhase = new SuggestPhase(settings);
+        this.rescorePhase = new RescorePhase(settings);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java b/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java
index 06f7cf5b1cd..395db4cdcd8 100644
--- a/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java
+++ b/core/src/main/java/org/elasticsearch/search/rescore/RescorePhase.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.rescore;
 import org.apache.lucene.search.TopDocs;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.search.SearchPhase;
 import org.elasticsearch.search.internal.SearchContext;
@@ -33,7 +32,6 @@ import java.io.IOException;
  */
 public class RescorePhase extends AbstractComponent implements SearchPhase {
 
-    @Inject
     public RescorePhase(Settings settings) {
         super(settings);
     }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java
index 3aedb6e6604..c0567e59e8c 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestPhase.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.suggest;
 import org.apache.lucene.util.CharsRefBuilder;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.search.SearchPhase;
 import org.elasticsearch.search.internal.SearchContext;
@@ -39,7 +38,6 @@ import java.util.Map;
  */
 public class SuggestPhase extends AbstractComponent implements SearchPhase {
 
-    @Inject
     public SuggestPhase(Settings settings) {
         super(settings);
     }
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
index cf8830aa7ef..65da6b58d20 100644
--- a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
+++ b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
@@ -112,7 +112,7 @@ public class FetchSubPhasePluginIT extends ESIntegTestCase {
         }
 
         public void onModule(SearchModule searchModule) {
-            searchModule.registerFetchSubPhase(TermVectorsFetchSubPhase.class);
+            searchModule.registerFetchSubPhase(new TermVectorsFetchSubPhase());
         }
     }
 
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
index 41a43d39328..ccf33a69739 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
@@ -28,9 +28,10 @@ import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.index.query.ParsedQuery;
+import org.elasticsearch.search.Highlighters;
 import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.highlight.HighlightPhase;
@@ -46,13 +47,10 @@ import java.util.Map;
 
 // Highlighting in the case of the percolate query is a bit different, because the PercolateQuery itself doesn't get highlighted,
 // but the source of the PercolateQuery gets highlighted by each hit containing a query.
-public class PercolatorHighlightSubFetchPhase implements FetchSubPhase {
+public class PercolatorHighlightSubFetchPhase extends HighlightPhase {
 
-    private final HighlightPhase highlightPhase;
-
-    @Inject
-    public PercolatorHighlightSubFetchPhase(HighlightPhase highlightPhase) {
-        this.highlightPhase = highlightPhase;
+    public PercolatorHighlightSubFetchPhase(Settings settings, Highlighters highlighters) {
+        super(settings, highlighters);
     }
 
     @Override
@@ -93,7 +91,7 @@ public class PercolatorHighlightSubFetchPhase implements FetchSubPhase {
                         percolatorLeafReaderContext, 0, percolatorIndexSearcher
                 );
                 hitContext.cache().clear();
-                highlightPhase.hitExecute(subSearchContext, hitContext);
+                super.hitExecute(subSearchContext, hitContext);
                 hit.highlightFields().putAll(hitContext.hit().getHighlightFields());
             }
         }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
index 963f14041b0..dfc11804dd9 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
@@ -28,15 +28,21 @@ import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.fetch.FetchSubPhase;
+import org.elasticsearch.search.highlight.HighlightPhase;
+
+import java.util.Optional;
 
 public class PercolatorPlugin extends Plugin {
 
     public static final String NAME = "percolator";
 
     private final boolean transportClientMode;
+    private final Settings settings;
 
     public PercolatorPlugin(Settings settings) {
         this.transportClientMode = transportClientMode(settings);
+        this.settings = settings;
     }
 
     @Override
@@ -67,7 +73,7 @@ public class PercolatorPlugin extends Plugin {
 
     public void onModule(SearchModule module) {
         module.registerQuery(PercolateQueryBuilder::new, PercolateQueryBuilder::fromXContent, PercolateQueryBuilder.QUERY_NAME_FIELD);
-        module.registerFetchSubPhase(PercolatorHighlightSubFetchPhase.class);
+        module.registerFetchSubPhase(new PercolatorHighlightSubFetchPhase(settings, module.getHighlighters()));
     }
 
     public void onModule(SettingsModule module) {
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
index 3f7aaafc105..95f64132ea1 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
@@ -25,6 +25,8 @@ import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.search.Highlighters;
 import org.elasticsearch.search.highlight.SearchContextHighlight;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.ESTestCase;
@@ -44,7 +46,8 @@ public class PercolatorHighlightSubFetchPhaseTests extends ESTestCase {
                 Mockito.mock(IndexSearcher.class))
                 .build();
 
-        PercolatorHighlightSubFetchPhase subFetchPhase = new PercolatorHighlightSubFetchPhase(null);
+        PercolatorHighlightSubFetchPhase subFetchPhase = new PercolatorHighlightSubFetchPhase(Settings.EMPTY,
+            new Highlighters(Settings.EMPTY));
         SearchContext searchContext = Mockito.mock(SearchContext.class);
         Mockito.when(searchContext.highlight()).thenReturn(new SearchContextHighlight(Collections.emptyList()));
         Mockito.when(searchContext.query()).thenReturn(new MatchAllDocsQuery());
diff --git a/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java b/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
index 629a5df3fee..1ed90408f7d 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
@@ -28,10 +28,8 @@ import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
-import org.elasticsearch.search.dfs.DfsPhase;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.internal.SearchContext;
-import org.elasticsearch.search.query.QueryPhase;
 import org.elasticsearch.search.suggest.Suggesters;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -84,10 +82,9 @@ public class MockSearchService extends SearchService {
     @Inject
     public MockSearchService(Settings settings, ClusterSettings clusterSettings, ClusterService clusterService,
             IndicesService indicesService, ThreadPool threadPool, ScriptService scriptService,
-            BigArrays bigArrays, DfsPhase dfsPhase, QueryPhase queryPhase, FetchPhase fetchPhase,
-            AggregatorParsers aggParsers, Suggesters suggesters) {
-        super(settings, clusterSettings, clusterService, indicesService, threadPool, scriptService, bigArrays, dfsPhase,
-                queryPhase, fetchPhase, aggParsers, suggesters);
+            BigArrays bigArrays, FetchPhase fetchPhase, AggregatorParsers aggParsers, Suggesters suggesters) {
+        super(settings, clusterSettings, clusterService, indicesService, threadPool, scriptService, bigArrays,
+                fetchPhase, aggParsers, suggesters);
     }
 
     @Override

From 0f87afe2bf105591f5d36dfc0ce7484e45334732 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Wed, 15 Jun 2016 09:26:34 +0200
Subject: [PATCH 0858/1311] [TEST] Fix Highlighters assertion - not guice
 injected anymore

---
 .../org/elasticsearch/search/SearchModuleTests.java  | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java b/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java
index 2cfb249f533..72873bc0d48 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java
@@ -71,7 +71,8 @@ public class SearchModuleTests extends ModuleTestCase {
 
     public void testRegisterHighlighter() {
         SearchModule module = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry());
-        module.registerHighlighter("custom",  new CustomHighlighter());
+        CustomHighlighter customHighlighter = new CustomHighlighter();
+        module.registerHighlighter("custom",  customHighlighter);
         IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
             () -> module.registerHighlighter("custom", new CustomHighlighter()));
         assertEquals("Can't register the same [highlighter] more than once for [custom]", exception.getMessage());
@@ -79,10 +80,11 @@ public class SearchModuleTests extends ModuleTestCase {
         exception = expectThrows(IllegalArgumentException.class,
             () -> module.registerHighlighter("custom", null));
         assertEquals("Can't register null highlighter for key: [custom]", exception.getMessage());
-        assertInstanceBinding(module, Highlighters.class, (h) -> h.get("custom").getClass() == CustomHighlighter.class
-        && h.get("fvh").getClass() == FastVectorHighlighter.class
-        && h.get("plain").getClass() == PlainHighlighter.class
-        && h.get("postings").getClass() == PostingsHighlighter.class);
+        Highlighters highlighters = module.getHighlighters();
+        assertEquals(highlighters.get("fvh").getClass(), FastVectorHighlighter.class);
+        assertEquals(highlighters.get("plain").getClass(), PlainHighlighter.class);
+        assertEquals(highlighters.get("postings").getClass(), PostingsHighlighter.class);
+        assertSame(highlighters.get("custom"), customHighlighter);
     }
 
     public void testRegisterQueryParserDuplicate() {

From a2ad5c0282674d49d953065b2527415e1d1a78be Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Wed, 15 Jun 2016 10:56:46 +0200
Subject: [PATCH 0859/1311] docs: fix typo

Closes #18877
---
 docs/reference/ingest/ingest-node.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc
index ec641383beb..ce50f27451e 100644
--- a/docs/reference/ingest/ingest-node.asciidoc
+++ b/docs/reference/ingest/ingest-node.asciidoc
@@ -822,7 +822,7 @@ This is because the date is being rounded by month.
 | Name                   | Required  | Default                      | Description
 | `field`                | yes       | -                            | The field to get the date or timestamp from.
 | `index_name_prefix`    | no        | -                            | A prefix of the index name to be prepended before the printed date.
-| `date_rounding`        | yes       | -                            | How to round the date when formatting the date into the index name. Valid values are: `y` (year), `m` (month), `w` (week), `d` (day), `h` (hour), `m` (minute) and `s` (second).
+| `date_rounding`        | yes       | -                            | How to round the date when formatting the date into the index name. Valid values are: `y` (year), `M` (month), `w` (week), `d` (day), `h` (hour), `m` (minute) and `s` (second).
 | `date_formats `        | no        | yyyy-MM-dd'T'HH:mm:ss.SSSZ   | An array of the expected date formats for parsing dates / timestamps in the document being preprocessed. Can be a Joda pattern or one of the following formats: ISO8601, UNIX, UNIX_MS, or TAI64N.
 | `timezone`             | no        | UTC                          | The timezone to use when parsing the date and when date math index supports resolves expressions into concrete index names.
 | `locale`               | no        | ENGLISH                      | The locale to use when parsing the date from the document being preprocessed, relevant when parsing month names or week days.

From 7c8013f9fdfe7a7e6754d1ebe4f968a7d6c84506 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Wed, 15 Jun 2016 11:43:48 +0200
Subject: [PATCH 0860/1311] Build: remove explicit targetCompatibility from
 forbiddenapis config

targetCompatibility set to the project is enough.
---
 client-sniffer/build.gradle | 2 --
 client/build.gradle         | 2 --
 2 files changed, 4 deletions(-)

diff --git a/client-sniffer/build.gradle b/client-sniffer/build.gradle
index c01f730e094..368cdaa073f 100644
--- a/client-sniffer/build.gradle
+++ b/client-sniffer/build.gradle
@@ -49,7 +49,6 @@ dependencies {
 forbiddenApisMain {
   //client does not depend on core, so only jdk signatures should be checked
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
-  targetCompatibility = 1.7
 }
 
 forbiddenApisTest {
@@ -58,7 +57,6 @@ forbiddenApisTest {
   bundledSignatures += 'jdk-internal'
   //client does not depend on core, so only jdk signatures should be checked
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
-  targetCompatibility = 1.7
 }
 
 //JarHell is part of es core, which we don't want to pull in
diff --git a/client/build.gradle b/client/build.gradle
index 30225382e41..afc2e6e1e7b 100644
--- a/client/build.gradle
+++ b/client/build.gradle
@@ -47,7 +47,6 @@ dependencies {
 forbiddenApisMain {
   //client does not depend on core, so only jdk signatures should be checked
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
-  targetCompatibility = 1.7
 }
 
 forbiddenApisTest {
@@ -56,7 +55,6 @@ forbiddenApisTest {
   bundledSignatures += 'jdk-internal'
   //client does not depend on core, so only jdk signatures should be checked
   signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
-  targetCompatibility = 1.7
 }
 
 //JarHell is part of es core, which we don't want to pull in

From f32b70047241fe319cb37047cc2a47d1b56da6e1 Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@elastic.co>
Date: Wed, 15 Jun 2016 14:29:46 +0200
Subject: [PATCH 0861/1311] Exclude admin / diagnostic requests from HTTP
 request limiting

With this commit we exclude certain HTTP requests that are needed to inspect the cluster
from HTTP request limiting to ensure these commands are processed even in critical
memory conditions.

Relates #17951, relates #18145, closes #18833
---
 .../org/elasticsearch/http/HttpServer.java    |  6 +-
 .../elasticsearch/rest/RestController.java    | 59 +++++++++--------
 .../org/elasticsearch/rest/RestHandler.java   |  6 +-
 .../health/RestClusterHealthAction.java       |  5 ++
 .../hotthreads/RestNodesHotThreadsAction.java |  5 ++
 .../node/info/RestNodesInfoAction.java        |  5 ++
 .../node/stats/RestNodesStatsAction.java      |  5 ++
 .../node/tasks/RestCancelTasksAction.java     |  5 ++
 .../node/tasks/RestListTasksAction.java       |  5 ++
 .../RestClusterGetSettingsAction.java         |  5 ++
 .../RestClusterUpdateSettingsAction.java      |  5 ++
 .../cluster/state/RestClusterStateAction.java |  5 ++
 .../cluster/stats/RestClusterStatsAction.java |  5 ++
 .../clear/RestClearIndicesCacheAction.java    |  5 ++
 .../indices/stats/RestIndicesStatsAction.java |  5 ++
 .../reroute/ClusterRerouteRequestTests.java   |  7 +-
 .../http/netty/NettyHttpClient.java           | 14 +++-
 .../netty/NettyHttpRequestSizeLimitIT.java    | 34 ++++++++--
 .../rest/RestControllerTests.java             | 32 ++++++++-
 .../rest/action/main/RestMainActionTests.java |  3 +-
 .../rest/action/support/RestTableTests.java   |  2 +-
 .../test/rest/FakeRestRequest.java            | 66 +++++++++++++++----
 22 files changed, 235 insertions(+), 54 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/http/HttpServer.java b/core/src/main/java/org/elasticsearch/http/HttpServer.java
index ccc90d9149c..31bbd235009 100644
--- a/core/src/main/java/org/elasticsearch/http/HttpServer.java
+++ b/core/src/main/java/org/elasticsearch/http/HttpServer.java
@@ -108,7 +108,11 @@ public class HttpServer extends AbstractLifecycleComponent<HttpServer> implement
         RestChannel responseChannel = channel;
         try {
             int contentLength = request.content().length();
-            inFlightRequestsBreaker(circuitBreakerService).addEstimateBytesAndMaybeBreak(contentLength, "<http_request>");
+            if (restController.canTripCircuitBreaker(request)) {
+                inFlightRequestsBreaker(circuitBreakerService).addEstimateBytesAndMaybeBreak(contentLength, "<http_request>");
+            } else {
+                inFlightRequestsBreaker(circuitBreakerService).addWithoutBreaking(contentLength);
+            }
             // iff we could reserve bytes for the request we need to send the response also over this channel
             responseChannel = new ResourceHandlingHttpChannel(channel, circuitBreakerService, contentLength);
             restController.dispatchRequest(request, responseChannel, threadContext);
diff --git a/core/src/main/java/org/elasticsearch/rest/RestController.java b/core/src/main/java/org/elasticsearch/rest/RestController.java
index aa567af09f7..be72ed19c40 100644
--- a/core/src/main/java/org/elasticsearch/rest/RestController.java
+++ b/core/src/main/java/org/elasticsearch/rest/RestController.java
@@ -113,30 +113,14 @@ public class RestController extends AbstractLifecycleComponent<RestController> {
     }
 
     /**
-     * Registers a rest handler to be execute when the provided method and path match the request.
+     * Registers a rest handler to be executed when the provided method and path match the request.
      */
     public void registerHandler(RestRequest.Method method, String path, RestHandler handler) {
-        switch (method) {
-            case GET:
-                getHandlers.insert(path, handler);
-                break;
-            case DELETE:
-                deleteHandlers.insert(path, handler);
-                break;
-            case POST:
-                postHandlers.insert(path, handler);
-                break;
-            case PUT:
-                putHandlers.insert(path, handler);
-                break;
-            case OPTIONS:
-                optionsHandlers.insert(path, handler);
-                break;
-            case HEAD:
-                headHandlers.insert(path, handler);
-                break;
-            default:
-                throw new IllegalArgumentException("Can't handle [" + method + "] for path [" + path + "]");
+        PathTrie<RestHandler> handlers = getHandlersForMethod(method);
+        if (handlers != null) {
+            handlers.insert(path, handler);
+        } else {
+            throw new IllegalArgumentException("Can't handle [" + method + "] for path [" + path + "]");
         }
     }
 
@@ -159,6 +143,15 @@ public class RestController extends AbstractLifecycleComponent<RestController> {
         return new ControllerFilterChain(executionFilter);
     }
 
+    /**
+     * @param request The current request. Must not be null.
+     * @return true iff the circuit breaker limit must be enforced for processing this request.
+     */
+    public boolean canTripCircuitBreaker(RestRequest request) {
+        RestHandler handler = getHandler(request);
+        return (handler != null) ? handler.canTripCircuitBreaker() : true;
+    }
+
     public void dispatchRequest(final RestRequest request, final RestChannel channel, ThreadContext threadContext) throws Exception {
         if (!checkRequestParameters(request, channel)) {
             return;
@@ -226,19 +219,27 @@ public class RestController extends AbstractLifecycleComponent<RestController> {
 
     private RestHandler getHandler(RestRequest request) {
         String path = getPath(request);
-        RestRequest.Method method = request.method();
+        PathTrie<RestHandler> handlers = getHandlersForMethod(request.method());
+        if (handlers != null) {
+            return handlers.retrieve(path, request.params());
+        } else {
+            return null;
+        }
+    }
+
+    private PathTrie<RestHandler> getHandlersForMethod(RestRequest.Method method) {
         if (method == RestRequest.Method.GET) {
-            return getHandlers.retrieve(path, request.params());
+            return getHandlers;
         } else if (method == RestRequest.Method.POST) {
-            return postHandlers.retrieve(path, request.params());
+            return postHandlers;
         } else if (method == RestRequest.Method.PUT) {
-            return putHandlers.retrieve(path, request.params());
+            return putHandlers;
         } else if (method == RestRequest.Method.DELETE) {
-            return deleteHandlers.retrieve(path, request.params());
+            return deleteHandlers;
         } else if (method == RestRequest.Method.HEAD) {
-            return headHandlers.retrieve(path, request.params());
+            return headHandlers;
         } else if (method == RestRequest.Method.OPTIONS) {
-            return optionsHandlers.retrieve(path, request.params());
+            return optionsHandlers;
         } else {
             return null;
         }
diff --git a/core/src/main/java/org/elasticsearch/rest/RestHandler.java b/core/src/main/java/org/elasticsearch/rest/RestHandler.java
index 45a172e816d..31970441493 100644
--- a/core/src/main/java/org/elasticsearch/rest/RestHandler.java
+++ b/core/src/main/java/org/elasticsearch/rest/RestHandler.java
@@ -25,4 +25,8 @@ package org.elasticsearch.rest;
 public interface RestHandler {
 
     void handleRequest(RestRequest request, RestChannel channel) throws Exception;
-}
\ No newline at end of file
+
+    default boolean canTripCircuitBreaker() {
+        return true;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/health/RestClusterHealthAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/health/RestClusterHealthAction.java
index ccd0f982597..a2c6ffeaf14 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/health/RestClusterHealthAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/health/RestClusterHealthAction.java
@@ -64,4 +64,9 @@ public class RestClusterHealthAction extends BaseRestHandler {
         clusterHealthRequest.waitForNodes(request.param("wait_for_nodes", clusterHealthRequest.waitForNodes()));
         client.admin().cluster().health(clusterHealthRequest, new RestStatusToXContentListener<ClusterHealthResponse>(channel));
     }
+
+    @Override
+    public boolean canTripCircuitBreaker() {
+        return false;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/hotthreads/RestNodesHotThreadsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/hotthreads/RestNodesHotThreadsAction.java
index 19bee19509a..2030d5e7d92 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/hotthreads/RestNodesHotThreadsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/hotthreads/RestNodesHotThreadsAction.java
@@ -78,4 +78,9 @@ public class RestNodesHotThreadsAction extends BaseRestHandler {
             }
         });
     }
+
+    @Override
+    public boolean canTripCircuitBreaker() {
+        return false;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java
index 65b7715385f..0f1e8691770 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java
@@ -103,4 +103,9 @@ public class RestNodesInfoAction extends BaseRestHandler {
 
         client.admin().cluster().nodesInfo(nodesInfoRequest, new NodesResponseRestListener<>(channel));
     }
+
+    @Override
+    public boolean canTripCircuitBreaker() {
+        return false;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java
index 47cce5283a9..8019de0b4e5 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java
@@ -115,4 +115,9 @@ public class RestNodesStatsAction extends BaseRestHandler {
 
         client.admin().cluster().nodesStats(nodesStatsRequest, new NodesResponseRestListener<>(channel));
     }
+
+    @Override
+    public boolean canTripCircuitBreaker() {
+        return false;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestCancelTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestCancelTasksAction.java
index e84dfbfc24c..0602abe651f 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestCancelTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestCancelTasksAction.java
@@ -64,4 +64,9 @@ public class RestCancelTasksAction extends BaseRestHandler {
         ActionListener<CancelTasksResponse> listener = nodeSettingListener(clusterService, new RestToXContentListener<>(channel));
         client.admin().cluster().cancelTasks(cancelTasksRequest, listener);
     }
+
+    @Override
+    public boolean canTripCircuitBreaker() {
+        return false;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java
index 4e20ca980c6..8fa13e808ac 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java
@@ -91,4 +91,9 @@ public class RestListTasksAction extends BaseRestHandler {
             }
         };
     }
+
+    @Override
+    public boolean canTripCircuitBreaker() {
+        return false;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterGetSettingsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterGetSettingsAction.java
index 62779292ac1..44a7f2f714b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterGetSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterGetSettingsAction.java
@@ -72,6 +72,11 @@ public class RestClusterGetSettingsAction extends BaseRestHandler {
         });
     }
 
+    @Override
+    public boolean canTripCircuitBreaker() {
+        return false;
+    }
+
     private XContentBuilder renderResponse(ClusterState state, boolean renderDefaults, XContentBuilder builder, ToXContent.Params params) throws IOException {
         builder.startObject();
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterUpdateSettingsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterUpdateSettingsAction.java
index 64083f1e806..b25866d4520 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterUpdateSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterUpdateSettingsAction.java
@@ -76,4 +76,9 @@ public class RestClusterUpdateSettingsAction extends BaseRestHandler {
             }
         });
     }
+
+    @Override
+    public boolean canTripCircuitBreaker() {
+        return false;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java
index edce416072e..c756796446e 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java
@@ -96,6 +96,11 @@ public class RestClusterStateAction extends BaseRestHandler {
         });
     }
 
+    @Override
+    public boolean canTripCircuitBreaker() {
+        return false;
+    }
+
     static final class Fields {
         static final String CLUSTER_NAME = "cluster_name";
     }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/stats/RestClusterStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/stats/RestClusterStatsAction.java
index 29cc6377494..c83a5b7ae1b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/stats/RestClusterStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/stats/RestClusterStatsAction.java
@@ -47,4 +47,9 @@ public class RestClusterStatsAction extends BaseRestHandler {
         clusterStatsRequest.timeout(request.param("timeout"));
         client.admin().cluster().clusterStats(clusterStatsRequest, new NodesResponseRestListener<>(channel));
     }
+
+    @Override
+    public boolean canTripCircuitBreaker() {
+        return false;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/cache/clear/RestClearIndicesCacheAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/cache/clear/RestClearIndicesCacheAction.java
index 7adb6909532..97e424445d8 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/cache/clear/RestClearIndicesCacheAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/cache/clear/RestClearIndicesCacheAction.java
@@ -75,6 +75,11 @@ public class RestClearIndicesCacheAction extends BaseRestHandler {
         });
     }
 
+    @Override
+    public boolean canTripCircuitBreaker() {
+        return false;
+    }
+
     public static ClearIndicesCacheRequest fromRequest(final RestRequest request, ClearIndicesCacheRequest clearIndicesCacheRequest, ParseFieldMatcher parseFieldMatcher) {
 
         for (Map.Entry<String, String> entry : request.params().entrySet()) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java
index 55ed1d8dda4..623e60cb4b9 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java
@@ -117,4 +117,9 @@ public class RestIndicesStatsAction extends BaseRestHandler {
             }
         });
     }
+
+    @Override
+    public boolean canTripCircuitBreaker() {
+        return false;
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java
index 06ca3a4f869..b736751b781 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java
@@ -209,6 +209,11 @@ public class ClusterRerouteRequestTests extends ESTestCase {
         }
         builder.endObject();
 
-        return new FakeRestRequest(emptyMap(), params, hasBody ? builder.bytes() : null);
+        FakeRestRequest.Builder requestBuilder = new FakeRestRequest.Builder();
+        requestBuilder.withParams(params);
+        if (hasBody) {
+            requestBuilder.withContent(builder.bytes());
+        }
+        return requestBuilder.build();
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpClient.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpClient.java
index 6af06f605dc..264876b7963 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpClient.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpClient.java
@@ -94,10 +94,22 @@ public class NettyHttpClient implements Closeable {
     @SafeVarargs // Safe not because it doesn't do anything with the type parameters but because it won't leak them into other methods.
     public final Collection<HttpResponse> post(SocketAddress remoteAddress, Tuple<String, CharSequence>... urisAndBodies)
             throws InterruptedException {
+        return processRequestsWithBody(HttpMethod.POST, remoteAddress, urisAndBodies);
+    }
+
+    @SafeVarargs // Safe not because it doesn't do anything with the type parameters but because it won't leak them into other methods.
+    public final Collection<HttpResponse> put(SocketAddress remoteAddress, Tuple<String, CharSequence>... urisAndBodies)
+            throws InterruptedException {
+        return processRequestsWithBody(HttpMethod.PUT, remoteAddress, urisAndBodies);
+    }
+
+    @SafeVarargs // Safe not because it doesn't do anything with the type parameters but because it won't leak them into other methods.
+    private final Collection<HttpResponse> processRequestsWithBody(HttpMethod method, SocketAddress remoteAddress, Tuple<String,
+        CharSequence>... urisAndBodies) throws InterruptedException {
         Collection<HttpRequest> requests = new ArrayList<>(urisAndBodies.length);
         for (Tuple<String, CharSequence> uriAndBody : urisAndBodies) {
             ChannelBuffer content = ChannelBuffers.copiedBuffer(uriAndBody.v2(), StandardCharsets.UTF_8);
-            HttpRequest request = new DefaultHttpRequest(HTTP_1_1, HttpMethod.POST, uriAndBody.v1());
+            HttpRequest request = new DefaultHttpRequest(HTTP_1_1, method, uriAndBody.v1());
             request.headers().add(HOST, "localhost");
             request.headers().add(HttpHeaders.Names.CONTENT_LENGTH, content.readableBytes());
             request.setContent(content);
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java
index e1135f807cf..d633313b1e7 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java
@@ -29,12 +29,12 @@ import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
-import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.jboss.netty.handler.codec.http.HttpResponse;
 import org.jboss.netty.handler.codec.http.HttpResponseStatus;
 
 import java.util.Collection;
 
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasSize;
 
@@ -89,9 +89,35 @@ public class NettyHttpRequestSizeLimitIT extends ESIntegTestCase {
         }
     }
 
-    private void assertAtLeastOnceExpectedStatus(Collection<HttpResponse> responses, HttpResponseStatus expectedStatus) {
-        long countResponseErrors = responses.stream().filter(r -> r.getStatus().equals(expectedStatus)).count();
-        assertThat(countResponseErrors, greaterThan(0L));
+    public void testDoesNotLimitExcludedRequests() throws Exception {
+        ensureGreen();
 
+        @SuppressWarnings("unchecked")
+        Tuple<String, CharSequence>[] requestUris = new Tuple[1500];
+        for (int i = 0; i < requestUris.length; i++) {
+            requestUris[i] = Tuple.tuple("/_cluster/settings",
+                "{ \"transient\": {\"indices.ttl.interval\": \"40s\" } }");
+        }
+
+        HttpServerTransport httpServerTransport = internalCluster().getInstance(HttpServerTransport.class);
+        InetSocketTransportAddress inetSocketTransportAddress = (InetSocketTransportAddress) randomFrom(httpServerTransport.boundAddress
+            ().boundAddresses());
+
+        try (NettyHttpClient nettyHttpClient = new NettyHttpClient()) {
+            Collection<HttpResponse> responses = nettyHttpClient.put(inetSocketTransportAddress.address(), requestUris);
+            assertThat(responses, hasSize(requestUris.length));
+            assertAllInExpectedStatus(responses, HttpResponseStatus.OK);
+        }
+    }
+
+    private void assertAtLeastOnceExpectedStatus(Collection<HttpResponse> responses, HttpResponseStatus expectedStatus) {
+        long countExpectedStatus = responses.stream().filter(r -> r.getStatus().equals(expectedStatus)).count();
+        assertThat("Expected at least one request with status [" + expectedStatus + "]", countExpectedStatus, greaterThan(0L));
+    }
+
+    private void assertAllInExpectedStatus(Collection<HttpResponse> responses, HttpResponseStatus expectedStatus) {
+        long countUnexpectedStatus = responses.stream().filter(r -> r.getStatus().equals(expectedStatus) == false).count();
+        assertThat("Expected all requests with status [" + expectedStatus + "] but [" + countUnexpectedStatus +
+            "] requests had a different one", countUnexpectedStatus, equalTo(0L));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java
index 101d2d8e50d..9cade7aa513 100644
--- a/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java
@@ -91,9 +91,39 @@ public class RestControllerTests extends ESTestCase {
         restHeaders.put("header.1", "true");
         restHeaders.put("header.2", "true");
         restHeaders.put("header.3", "false");
-        restController.dispatchRequest(new FakeRestRequest(restHeaders), null, threadContext);
+        restController.dispatchRequest(new FakeRestRequest.Builder().withHeaders(restHeaders).build(), null, threadContext);
         assertNull(threadContext.getHeader("header.1"));
         assertNull(threadContext.getHeader("header.2"));
         assertEquals("true", threadContext.getHeader("header.3"));
     }
+
+    public void testCanTripCircuitBreaker() throws Exception {
+        RestController controller = new RestController(Settings.EMPTY);
+        // trip circuit breaker by default
+        controller.registerHandler(RestRequest.Method.GET, "/trip", new FakeRestHandler(true));
+        controller.registerHandler(RestRequest.Method.GET, "/do-not-trip", new FakeRestHandler(false));
+
+        assertTrue(controller.canTripCircuitBreaker(new FakeRestRequest.Builder().withPath("/trip").build()));
+        // assume trip even on unknown paths
+        assertTrue(controller.canTripCircuitBreaker(new FakeRestRequest.Builder().withPath("/unknown-path").build()));
+        assertFalse(controller.canTripCircuitBreaker(new FakeRestRequest.Builder().withPath("/do-not-trip").build()));
+    }
+
+    private static class FakeRestHandler implements RestHandler {
+        private final boolean canTripCircuitBreaker;
+
+        private FakeRestHandler(boolean canTripCircuitBreaker) {
+            this.canTripCircuitBreaker = canTripCircuitBreaker;
+        }
+
+        @Override
+        public void handleRequest(RestRequest request, RestChannel channel) throws Exception {
+            //no op
+        }
+
+        @Override
+        public boolean canTripCircuitBreaker() {
+            return canTripCircuitBreaker;
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionTests.java
index bb0497ed343..ebb7dd255aa 100644
--- a/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionTests.java
@@ -33,7 +33,6 @@ import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.FakeRestRequest;
 
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -82,7 +81,7 @@ public class RestMainActionTests extends ESTestCase {
         if (prettyPrint == false) {
             params.put("pretty", String.valueOf(prettyPrint));
         }
-        RestRequest restRequest = new FakeRestRequest(Collections.emptyMap(), params);
+        RestRequest restRequest = new FakeRestRequest.Builder().withParams(params).build();
 
         BytesRestResponse response = RestMainAction.convertMainResponse(mainResponse, restRequest, builder);
         assertNotNull(response);
diff --git a/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java b/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java
index 89b3b6fd239..a7e17785d48 100644
--- a/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java
@@ -146,7 +146,7 @@ public class RestTableTests extends ESTestCase {
     }
 
     private RestResponse assertResponseContentType(Map<String, String> headers, String mediaType) throws Exception {
-        FakeRestRequest requestWithAcceptHeader = new FakeRestRequest(headers);
+        FakeRestRequest requestWithAcceptHeader = new FakeRestRequest.Builder().withHeaders(headers).build();
         table.startRow();
         table.addCell("foo");
         table.addCell("foo");
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java
index 66167bc7ec8..3fc924176f0 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/FakeRestRequest.java
@@ -33,37 +33,35 @@ public class FakeRestRequest extends RestRequest {
 
     private final BytesReference content;
 
+    private final Method method;
+
+    private final String path;
+
     public FakeRestRequest() {
-        this(new HashMap<>());
+        this(new HashMap<>(), new HashMap<>(), null, Method.GET, "/");
     }
 
-    public FakeRestRequest(Map<String, String> headers) {
-        this(headers, new HashMap<>());
-    }
-
-    public FakeRestRequest(Map<String, String> headers, Map<String, String> params) {
-        this(headers, params, null);
-    }
-
-    public FakeRestRequest(Map<String, String> headers, Map<String, String> params, BytesReference content) {
+    private FakeRestRequest(Map<String, String> headers, Map<String, String> params, BytesReference content, Method method, String path) {
         this.headers = headers;
         this.params = params;
         this.content = content;
+        this.method = method;
+        this.path = path;
     }
 
     @Override
     public Method method() {
-        return Method.GET;
+        return method;
     }
 
     @Override
     public String uri() {
-        return "/";
+        return path;
     }
 
     @Override
     public String rawPath() {
-        return "/";
+        return path;
     }
 
     @Override
@@ -109,4 +107,46 @@ public class FakeRestRequest extends RestRequest {
     public Map<String, String> params() {
         return params;
     }
+
+    public static class Builder {
+        private Map<String, String> headers = new HashMap<>();
+
+        private Map<String, String> params = new HashMap<>();
+
+        private BytesReference content;
+
+        private String path = "/";
+
+        private Method method = Method.GET;
+
+        public Builder withHeaders(Map<String, String> headers) {
+            this.headers = headers;
+            return this;
+        }
+
+        public Builder withParams(Map<String, String> params) {
+            this.params = params;
+            return this;
+        }
+
+        public Builder withContent(BytesReference content) {
+            this.content = content;
+            return this;
+        }
+
+        public Builder withPath(String path) {
+            this.path = path;
+            return this;
+        }
+
+        public Builder withMethod(Method method) {
+            this.method = method;
+            return this;
+        }
+
+        public FakeRestRequest build() {
+            return new FakeRestRequest(headers, params, content, method, path);
+        }
+    }
+
 }

From e09b6d7ba172500d41347ae5c640e037f1cfff00 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 15 Jun 2016 09:02:36 -0400
Subject: [PATCH 0862/1311] Test: Remove and untrue assertion

Task status might change between list and get.
---
 .../elasticsearch/action/admin/cluster/node/tasks/TasksIT.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
index ebb93ee7f21..376bd7f3ffa 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
@@ -380,7 +380,7 @@ public class TasksIT extends ESIntegTestCase {
                 assertEquals(task.getType(), fetchedWithGet.getType());
                 assertEquals(task.getAction(), fetchedWithGet.getAction());
                 assertEquals(task.getDescription(), fetchedWithGet.getDescription());
-                assertEquals(task.getStatus(), fetchedWithGet.getStatus());
+                // The status won't always be equal - it might change between the list and the get.
                 assertEquals(task.getStartTime(), fetchedWithGet.getStartTime());
                 assertThat(fetchedWithGet.getRunningTimeNanos(), greaterThanOrEqualTo(task.getRunningTimeNanos()));
                 assertEquals(task.isCancellable(), fetchedWithGet.isCancellable());

From b71f42a6272edb4a0573a0e80f55e62df9a47488 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Wed, 15 Jun 2016 09:07:11 -0400
Subject: [PATCH 0863/1311] split MIC from PIC

---
 .../elasticsearch/painless/DefBootstrap.java  | 227 ++++++++++++------
 .../painless/DefBootstrapTests.java           |  12 +-
 2 files changed, 156 insertions(+), 83 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index 32428686cbf..ffcc295a33a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -77,6 +77,12 @@ public final class DefBootstrap {
      * the fallback if a null is encountered.
      */
     public static final int OPERATOR_ALLOWS_NULL = 1 << 0;
+    
+    /**
+     * static bootstrap parameter indicating the binary operator is part of compound assignment (e.g. +=).
+     * 
+     */
+    public static final int OPERATOR_COMPOUND_ASSIGNMENT = 1 << 1;
 
     /**
      * CallSite that implements the polymorphic inlining cache (PIC).
@@ -97,14 +103,8 @@ public final class DefBootstrap {
             this.name = name;
             this.flavor = flavor;
             this.args = args;
-            
-            // For operators use a monomorphic cache, fallback is fast.
-            // Just start with a depth of MAX-1, to keep it a constant.
-            if (flavor == UNARY_OPERATOR || flavor == BINARY_OPERATOR || flavor == SHIFT_OPERATOR) {
-                depth = MAX_DEPTH - 1;
-            }
 
-            final MethodHandle fallback = FALLBACK.bindTo(this)
+            MethodHandle fallback = FALLBACK.bindTo(this)
               .asCollector(Object[].class, type.parameterCount())
               .asType(type);
 
@@ -118,22 +118,6 @@ public final class DefBootstrap {
         static boolean checkClass(Class<?> clazz, Object receiver) {
             return receiver.getClass() == clazz;
         }
-        
-        /**
-         * guard method for inline caching: checks the receiver's class and the first argument
-         * are the same as the cached receiver and first argument.
-         */
-        static boolean checkBinary(Class<?> left, Class<?> right, Object leftObject, Object rightObject) {
-            return leftObject.getClass() == left && rightObject.getClass() == right;
-        }
-        
-        /**
-         * guard method for inline caching: checks the first argument is the same
-         * as the cached first argument.
-         */
-        static boolean checkBinaryArg(Class<?> left, Class<?> right, Object leftObject, Object rightObject) {
-            return rightObject.getClass() == right;
-        }
 
         /**
          * Does a slow lookup against the whitelist.
@@ -154,33 +138,9 @@ public final class DefBootstrap {
                     return Def.lookupIterator(args[0].getClass());
                 case REFERENCE:
                     return Def.lookupReference(lookup, (String) this.args[0], args[0].getClass(), name);
-                case UNARY_OPERATOR:
-                case SHIFT_OPERATOR:
-                    // shifts are treated as unary, as java allows long arguments without a cast (but bits are ignored)
-                    return DefMath.lookupUnary(args[0].getClass(), name);
-                case BINARY_OPERATOR:
-                    if (args[0] == null || args[1] == null) {
-                        return getGeneric(flavor, name); // can handle nulls, if supported
-                    } else {
-                        return DefMath.lookupBinary(args[0].getClass(), args[1].getClass(), name);
-                    }
                 default: throw new AssertionError();
             }
         }
-        
-        /**
-         * Installs a permanent, generic solution that works with any parameter types, if possible.
-         */
-        private MethodHandle getGeneric(int flavor, String name) throws Throwable {
-            switch(flavor) {
-                case UNARY_OPERATOR:
-                case BINARY_OPERATOR:
-                case SHIFT_OPERATOR:
-                    return DefMath.lookupGeneric(name);
-                default:
-                    return null;
-            }
-        }
 
         /**
          * Called when a new type is encountered (or, when we have encountered more than {@code MAX_DEPTH}
@@ -189,14 +149,92 @@ public final class DefBootstrap {
         @SuppressForbidden(reason = "slow path")
         Object fallback(Object[] args) throws Throwable {
             if (depth >= MAX_DEPTH) {
+                // XXX: caching defeated: lookups up every time(!)
+                return lookup(flavor, name, args).invokeWithArguments(args);
+            }
+            
+            final MethodType type = type();
+            final MethodHandle target = lookup(flavor, name, args).asType(type);
+
+            MethodHandle test = CHECK_CLASS.bindTo(args[0].getClass());
+            test = test.asType(test.type().changeParameterType(0, type.parameterType(0)));
+
+            MethodHandle guard = MethodHandles.guardWithTest(test, target, getTarget());
+            
+            depth++;
+
+            setTarget(guard);
+            return target.invokeWithArguments(args);
+        }
+
+        private static final MethodHandle CHECK_CLASS;
+        private static final MethodHandle FALLBACK;
+        static {
+            final Lookup lookup = MethodHandles.lookup();
+            try {
+                CHECK_CLASS = lookup.findStatic(lookup.lookupClass(), "checkClass",
+                                              MethodType.methodType(boolean.class, Class.class, Object.class));
+                FALLBACK = lookup.findVirtual(lookup.lookupClass(), "fallback",
+                                              MethodType.methodType(Object.class, Object[].class));
+            } catch (ReflectiveOperationException e) {
+                throw new AssertionError(e);
+            }
+        }
+    }
+    
+    /**
+     * CallSite that implements the monomorphic inlining cache (for operators).
+     */
+    static final class MIC extends MutableCallSite {
+        private boolean initialized;
+        
+        private final String name;
+        private final int flavor;
+        private final int flags;
+
+        MIC(String name, MethodType type, int flavor, int flags) {
+            super(type);
+            this.name = name;
+            this.flavor = flavor;
+            this.flags = flags;
+            
+            MethodHandle fallback = FALLBACK.bindTo(this)
+              .asCollector(Object[].class, type.parameterCount())
+              .asType(type);
+
+            setTarget(fallback);
+        }
+        
+        /**
+         * Does a slow lookup for the operator
+         */
+        private MethodHandle lookup(int flavor, String name, Object[] args) throws Throwable {
+            switch(flavor) {
+                case UNARY_OPERATOR:
+                case SHIFT_OPERATOR:
+                    // shifts are treated as unary, as java allows long arguments without a cast (but bits are ignored)
+                    return DefMath.lookupUnary(args[0].getClass(), name);
+                case BINARY_OPERATOR:
+                    if (args[0] == null || args[1] == null) {
+                        return DefMath.lookupGeneric(name); // can handle nulls, if supported
+                    } else {
+                        return DefMath.lookupBinary(args[0].getClass(), args[1].getClass(), name);
+                    }
+                default: throw new AssertionError();
+            }
+        }
+        
+        /**
+         * Called when a new type is encountered (or, when we have encountered more than {@code MAX_DEPTH}
+         * types at this call site and given up on caching).
+         */
+        @SuppressForbidden(reason = "slow path")
+        Object fallback(Object[] args) throws Throwable {
+            if (initialized) {
                 // caching defeated
-                MethodHandle generic = getGeneric(flavor, name);
-                if (generic != null) {
-                    setTarget(generic.asType(type()));
-                    return generic.invokeWithArguments(args);
-                } else {
-                    return lookup(flavor, name, args).invokeWithArguments(args);
-                }
+                MethodHandle generic = DefMath.lookupGeneric(name);
+                setTarget(generic.asType(type()));
+                return generic.invokeWithArguments(args);
             }
             
             final MethodType type = type();
@@ -209,24 +247,25 @@ public final class DefBootstrap {
                 Class<?> clazz1 = args[1] == null ? null : args[1].getClass();
                 if (type.parameterType(1) != Object.class) {
                     // case 1: only the receiver is unknown, just check that
-                    MethodHandle unaryTest = CHECK_CLASS.bindTo(clazz0);
+                    MethodHandle unaryTest = CHECK_LHS.bindTo(clazz0);
                     test = unaryTest.asType(unaryTest.type()
                                             .changeParameterType(0, type.parameterType(0)));
                 } else if (type.parameterType(0) != Object.class) {
                     // case 2: only the argument is unknown, just check that
-                    MethodHandle unaryTest = CHECK_BINARY_ARG.bindTo(clazz0).bindTo(clazz1);
+                    MethodHandle unaryTest = CHECK_RHS.bindTo(clazz0).bindTo(clazz1);
                     test = unaryTest.asType(unaryTest.type()
                                             .changeParameterType(0, type.parameterType(0))
                                             .changeParameterType(1, type.parameterType(1)));
                 } else {
                     // case 3: check both receiver and argument
-                    MethodHandle binaryTest = CHECK_BINARY.bindTo(clazz0).bindTo(clazz1);
+                    MethodHandle binaryTest = CHECK_BOTH.bindTo(clazz0).bindTo(clazz1);
                     test = binaryTest.asType(binaryTest.type()
                                             .changeParameterType(0, type.parameterType(0))
                                             .changeParameterType(1, type.parameterType(1)));
                 }
             } else {
-                MethodHandle receiverTest = CHECK_CLASS.bindTo(args[0].getClass());
+                // unary operator
+                MethodHandle receiverTest = CHECK_LHS.bindTo(args[0].getClass());
                 test = receiverTest.asType(receiverTest.type()
                                         .changeParameterType(0, type.parameterType(0)));
             }
@@ -234,29 +273,56 @@ public final class DefBootstrap {
             MethodHandle guard = MethodHandles.guardWithTest(test, target, getTarget());
             // very special cases, where even the receiver can be null (see JLS rules for string concat)
             // we wrap + with an NPE catcher, and use our generic method in that case.
-            if (flavor == BINARY_OPERATOR && ((int)this.args[0] & OPERATOR_ALLOWS_NULL) != 0) {
-                MethodHandle handler = MethodHandles.dropArguments(getGeneric(flavor, name).asType(type()), 0, NullPointerException.class);
+            if (flavor == BINARY_OPERATOR && (flags & OPERATOR_ALLOWS_NULL) != 0) {
+                MethodHandle handler = MethodHandles.dropArguments(DefMath.lookupGeneric(name).asType(type()), 
+                                                                   0, 
+                                                                   NullPointerException.class);
                 guard = MethodHandles.catchException(guard, NullPointerException.class, handler);
             }
             
-            depth++;
+            initialized = true;
 
             setTarget(guard);
             return target.invokeWithArguments(args);
         }
-
-        private static final MethodHandle CHECK_CLASS;
-        private static final MethodHandle CHECK_BINARY;
-        private static final MethodHandle CHECK_BINARY_ARG;
+        
+        /**
+         * guard method for inline caching: checks the receiver's class is the same
+         * as the cached class
+         */
+        static boolean checkLHS(Class<?> clazz, Object leftObject) {
+            return leftObject.getClass() == clazz;
+        }
+        
+        
+        /**
+         * guard method for inline caching: checks the first argument is the same
+         * as the cached first argument.
+         */
+        static boolean checkRHS(Class<?> left, Class<?> right, Object leftObject, Object rightObject) {
+            return rightObject.getClass() == right;
+        }
+        
+        /**
+         * guard method for inline caching: checks the receiver's class and the first argument
+         * are the same as the cached receiver and first argument.
+         */
+        static boolean checkBoth(Class<?> left, Class<?> right, Object leftObject, Object rightObject) {
+            return leftObject.getClass() == left && rightObject.getClass() == right;
+        }
+        
+        private static final MethodHandle CHECK_LHS;
+        private static final MethodHandle CHECK_RHS;
+        private static final MethodHandle CHECK_BOTH;
         private static final MethodHandle FALLBACK;
         static {
             final Lookup lookup = MethodHandles.lookup();
             try {
-                CHECK_CLASS = lookup.findStatic(lookup.lookupClass(), "checkClass",
+                CHECK_LHS = lookup.findStatic(lookup.lookupClass(), "checkLHS",
                                               MethodType.methodType(boolean.class, Class.class, Object.class));
-                CHECK_BINARY = lookup.findStatic(lookup.lookupClass(), "checkBinary",
+                CHECK_RHS = lookup.findStatic(lookup.lookupClass(), "checkRHS",
                                               MethodType.methodType(boolean.class, Class.class, Class.class, Object.class, Object.class));
-                CHECK_BINARY_ARG = lookup.findStatic(lookup.lookupClass(), "checkBinaryArg",
+                CHECK_BOTH = lookup.findStatic(lookup.lookupClass(), "checkBoth",
                                               MethodType.methodType(boolean.class, Class.class, Class.class, Object.class, Object.class));
                 FALLBACK = lookup.findVirtual(lookup.lookupClass(), "fallback",
                                               MethodType.methodType(Object.class, Object[].class));
@@ -277,6 +343,7 @@ public final class DefBootstrap {
     public static CallSite bootstrap(Lookup lookup, String name, MethodType type, int flavor, Object... args) {
         // validate arguments
         switch(flavor) {
+            // "function-call" like things get a polymorphic cache
             case METHOD_CALL:
                 if (args.length != 1) {
                     throw new BootstrapMethodError("Invalid number of parameters for method call");
@@ -288,7 +355,16 @@ public final class DefBootstrap {
                 if (Long.bitCount(recipe) > type.parameterCount()) {
                     throw new BootstrapMethodError("Illegal recipe for method call: too many bits");
                 }
-                break;
+                return new PIC(lookup, name, type, flavor, args);
+            case LOAD:
+            case STORE:
+            case ARRAY_LOAD:
+            case ARRAY_STORE:
+            case ITERATOR:
+                if (args.length > 0) {
+                    throw new BootstrapMethodError("Illegal static bootstrap parameters for flavor: " + flavor);
+                }
+                return new PIC(lookup, name, type, flavor, args);
             case REFERENCE:
                 if (args.length != 1) {
                     throw new BootstrapMethodError("Invalid number of parameters for reference call");
@@ -296,7 +372,9 @@ public final class DefBootstrap {
                 if (args[0] instanceof String == false) {
                     throw new BootstrapMethodError("Illegal parameter for reference call: " + args[0]);
                 }
-                break;
+                return new PIC(lookup, name, type, flavor, args);
+
+            // operators get monomorphic cache, with a generic impl for a fallback
             case UNARY_OPERATOR:
             case SHIFT_OPERATOR:
             case BINARY_OPERATOR:
@@ -311,14 +389,9 @@ public final class DefBootstrap {
                     // we just don't need it anywhere else.
                     throw new BootstrapMethodError("This parameter is only supported for BINARY_OPERATORs");
                 }
-                break;
+                return new MIC(name, type, flavor, flags);
             default:
-                if (args.length > 0) {
-                    throw new BootstrapMethodError("Illegal static bootstrap parameters for flavor: " + flavor);
-                }
-                break;
+                throw new BootstrapMethodError("Illegal static bootstrap parameter for flavor: " + flavor);
         }
-        return new PIC(lookup, name, type, flavor, args);
     }
-
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
index e551a89ea5a..59dc816b763 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
@@ -107,7 +107,7 @@ public class DefBootstrapTests extends ESTestCase {
     // test operators with null guards
 
     public void testNullGuardAdd() throws Throwable {
-        DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
+        DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                                "add", 
                                                                MethodType.methodType(Object.class, Object.class, Object.class),
                                                                DefBootstrap.BINARY_OPERATOR, DefBootstrap.OPERATOR_ALLOWS_NULL);
@@ -116,7 +116,7 @@ public class DefBootstrapTests extends ESTestCase {
     }
     
     public void testNullGuardAddWhenCached() throws Throwable {
-        DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
+        DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                                "add", 
                                                                MethodType.methodType(Object.class, Object.class, Object.class),
                                                                DefBootstrap.BINARY_OPERATOR, DefBootstrap.OPERATOR_ALLOWS_NULL);
@@ -126,7 +126,7 @@ public class DefBootstrapTests extends ESTestCase {
     }
     
     public void testNullGuardEq() throws Throwable {
-        DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
+        DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                                "eq", 
                                                                MethodType.methodType(boolean.class, Object.class, Object.class),
                                                                DefBootstrap.BINARY_OPERATOR, DefBootstrap.OPERATOR_ALLOWS_NULL);
@@ -136,7 +136,7 @@ public class DefBootstrapTests extends ESTestCase {
     }
     
     public void testNullGuardEqWhenCached() throws Throwable {
-        DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
+        DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                                "eq", 
                                                                MethodType.methodType(boolean.class, Object.class, Object.class),
                                                                DefBootstrap.BINARY_OPERATOR, DefBootstrap.OPERATOR_ALLOWS_NULL);
@@ -151,7 +151,7 @@ public class DefBootstrapTests extends ESTestCase {
     // and can be disabled in some circumstances.
     
     public void testNoNullGuardAdd() throws Throwable {
-        DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
+        DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                                "add", 
                                                                MethodType.methodType(Object.class, int.class, Object.class),
                                                                DefBootstrap.BINARY_OPERATOR, 0);
@@ -162,7 +162,7 @@ public class DefBootstrapTests extends ESTestCase {
     }
     
     public void testNoNullGuardAddWhenCached() throws Throwable {
-        DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
+        DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                                "add", 
                                                                MethodType.methodType(Object.class, int.class, Object.class),
                                                                DefBootstrap.BINARY_OPERATOR, 0);

From 7df5d05c62c8315e08e5c4dd36cb1fcc0537f986 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Wed, 15 Jun 2016 15:49:11 +0200
Subject: [PATCH 0864/1311] Simplify SubFetchPhase interface (#18881)

This interface used to have dedicated methods to prevent calling execute
methods. These methods are unnecessary as the checks can simply be
done inside the execute methods itself. This simplifies the interface
as well as its usage.
---
 .../search/fetch/FetchPhase.java              |  8 ++---
 .../search/fetch/FetchSubPhase.java           | 15 ++++----
 .../fetch/explain/ExplainFetchSubPhase.java   | 19 +++-------
 .../FieldDataFieldsFetchSubPhase.java         | 27 +++-----------
 .../innerhits/InnerHitsFetchSubPhase.java     | 30 +++-------------
 .../MatchedQueriesFetchSubPhase.java          | 36 ++++---------------
 .../parent/ParentFieldSubFetchPhase.java      | 24 +------------
 .../script/ScriptFieldsFetchSubPhase.java     | 32 +++++------------
 .../fetch/source/FetchSourceSubPhase.java     | 22 +++---------
 .../fetch/version/VersionFetchSubPhase.java   | 31 ++++------------
 .../search/highlight/HighlightPhase.java      | 33 ++---------------
 .../search/fetch/FetchSubPhasePluginIT.java   | 19 +++-------
 .../PercolatorHighlightSubFetchPhase.java     | 31 ++++++----------
 ...PercolatorHighlightSubFetchPhaseTests.java |  4 ---
 14 files changed, 63 insertions(+), 268 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
index 50f07d04f6f..e884d32f3e4 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
@@ -160,16 +160,12 @@ public class FetchPhase implements SearchPhase {
             hits[index] = searchHit;
             hitContext.reset(searchHit, subReaderContext, subDocId, context.searcher());
             for (FetchSubPhase fetchSubPhase : fetchSubPhases) {
-                if (fetchSubPhase.hitExecutionNeeded(context)) {
-                    fetchSubPhase.hitExecute(context, hitContext);
-                }
+                fetchSubPhase.hitExecute(context, hitContext);
             }
         }
 
         for (FetchSubPhase fetchSubPhase : fetchSubPhases) {
-            if (fetchSubPhase.hitsExecutionNeeded(context)) {
-                fetchSubPhase.hitsExecute(context, hits);
-            }
+            fetchSubPhase.hitsExecute(context, hits);
         }
 
         context.fetchResult().hits(new InternalSearchHits(hits, context.queryResult().topDocs().totalHits, context.queryResult().topDocs().getMaxScore()));
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java
index 5ba75b84fcc..b55afe642ec 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java
@@ -36,7 +36,7 @@ import java.util.Map;
  */
 public interface FetchSubPhase {
 
-    public static class HitContext {
+    class HitContext {
         private InternalSearchHit hit;
         private IndexSearcher searcher;
         private LeafReaderContext readerContext;
@@ -87,16 +87,13 @@ public interface FetchSubPhase {
         return Collections.emptyMap();
     }
 
-    boolean hitExecutionNeeded(SearchContext context);
-
     /**
      * Executes the hit level phase, with a reader and doc id (note, its a low level reader, and the matching doc).
      */
-    void hitExecute(SearchContext context, HitContext hitContext);
+    default void hitExecute(SearchContext context, HitContext hitContext) {}
 
-    boolean hitsExecutionNeeded(SearchContext context);
 
-    void hitsExecute(SearchContext context, InternalSearchHit[] hits);
+    default void hitsExecute(SearchContext context, InternalSearchHit[] hits) {}
 
     /**
      * This interface is in the fetch phase plugin mechanism.
@@ -104,16 +101,16 @@ public interface FetchSubPhase {
      * Fetch phases that use the plugin mechanism must provide a ContextFactory to the SearchContext that creates the fetch phase context and also associates them with a name.
      * See {@link SearchContext#getFetchSubPhaseContext(FetchSubPhase.ContextFactory)}
      */
-    public interface ContextFactory<SubPhaseContext extends FetchSubPhaseContext> {
+    interface ContextFactory<SubPhaseContext extends FetchSubPhaseContext> {
 
         /**
          * The name of the context.
          */
-        public String getName();
+        String getName();
 
         /**
          * Creates a new instance of a FetchSubPhaseContext that holds all information a FetchSubPhase needs to execute on hits.
          */
-        public SubPhaseContext newContextInstance();
+        SubPhaseContext newContextInstance();
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/explain/ExplainFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/explain/ExplainFetchSubPhase.java
index 0d8b4ef3271..e560b815d5a 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/explain/ExplainFetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/explain/ExplainFetchSubPhase.java
@@ -30,24 +30,13 @@ import java.io.IOException;
 /**
  *
  */
-public class ExplainFetchSubPhase implements FetchSubPhase {
-
-    @Override
-    public boolean hitsExecutionNeeded(SearchContext context) {
-        return false;
-    }
-
-    @Override
-    public void hitsExecute(SearchContext context, InternalSearchHit[] hits) {
-    }
-
-    @Override
-    public boolean hitExecutionNeeded(SearchContext context) {
-        return context.explain();
-    }
+public final class ExplainFetchSubPhase implements FetchSubPhase {
 
     @Override
     public void hitExecute(SearchContext context, HitContext hitContext) {
+        if (context.explain() == false) {
+            return;
+        }
         try {
             final int topLevelDocId = hitContext.hit().docId();
             Explanation explanation = context.searcher().explain(context.query(), topLevelDocId);
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/fielddata/FieldDataFieldsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/fielddata/FieldDataFieldsFetchSubPhase.java
index f8034719a1d..81907d0b906 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/fielddata/FieldDataFieldsFetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/fielddata/FieldDataFieldsFetchSubPhase.java
@@ -18,13 +18,11 @@
  */
 package org.elasticsearch.search.fetch.fielddata;
 
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.index.fielddata.AtomicFieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.search.fetch.FetchSubPhase;
-import org.elasticsearch.search.internal.InternalSearchHit;
 import org.elasticsearch.search.internal.InternalSearchHitField;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -37,7 +35,7 @@ import java.util.HashMap;
  *
  * Specifying {@code "fielddata_fields": ["field1", "field2"]}
  */
-public class FieldDataFieldsFetchSubPhase implements FetchSubPhase {
+public final class FieldDataFieldsFetchSubPhase implements FetchSubPhase {
 
     public static final String[] NAMES = {"fielddata_fields", "fielddataFields"};
     public static final ContextFactory<FieldDataFieldsContext> CONTEXT_FACTORY = new ContextFactory<FieldDataFieldsContext>() {
@@ -53,29 +51,14 @@ public class FieldDataFieldsFetchSubPhase implements FetchSubPhase {
         }
     };
 
-    @Inject
-    public FieldDataFieldsFetchSubPhase() {
-    }
-
-    @Override
-    public boolean hitsExecutionNeeded(SearchContext context) {
-        return false;
-    }
-
-    @Override
-    public void hitsExecute(SearchContext context, InternalSearchHit[] hits) {
-    }
-
-    @Override
-    public boolean hitExecutionNeeded(SearchContext context) {
-        return context.getFetchSubPhaseContext(CONTEXT_FACTORY).hitExecutionNeeded();
-    }
-
     @Override
     public void hitExecute(SearchContext context, HitContext hitContext) {
+        if (context.getFetchSubPhaseContext(CONTEXT_FACTORY).hitExecutionNeeded() == false) {
+            return;
+        }
         for (FieldDataFieldsContext.FieldDataField field : context.getFetchSubPhaseContext(CONTEXT_FACTORY).fields()) {
             if (hitContext.hit().fieldsOrNull() == null) {
-                hitContext.hit().fields(new HashMap<String, SearchHitField>(2));
+                hitContext.hit().fields(new HashMap<>(2));
             }
             SearchHitField hitField = hitContext.hit().fields().get(field.name());
             if (hitField == null) {
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java
index 1382e8e2194..35a25b522c0 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java
@@ -23,7 +23,6 @@ import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.fetch.FetchSearchResult;
 import org.elasticsearch.search.fetch.FetchSubPhase;
@@ -32,13 +31,10 @@ import org.elasticsearch.search.internal.InternalSearchHits;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
-/**
- */
-public class InnerHitsFetchSubPhase implements FetchSubPhase {
+public final class InnerHitsFetchSubPhase implements FetchSubPhase {
 
     private final FetchPhase fetchPhase;
 
@@ -46,20 +42,11 @@ public class InnerHitsFetchSubPhase implements FetchSubPhase {
         this.fetchPhase = fetchPhase;
     }
 
-    @Override
-    public Map<String, ? extends SearchParseElement> parseElements() {
-        // SearchParse elements needed because everything is parsed by InnerHitBuilder and eventually put
-        // into the search context.
-        return Collections.emptyMap();
-    }
-
-    @Override
-    public boolean hitExecutionNeeded(SearchContext context) {
-        return context.innerHits() != null && context.innerHits().getInnerHits().size() > 0;
-    }
-
     @Override
     public void hitExecute(SearchContext context, HitContext hitContext) {
+        if ((context.innerHits() != null && context.innerHits().getInnerHits().size() > 0) == false) {
+            return;
+        }
         Map<String, InternalSearchHits> results = new HashMap<>();
         for (Map.Entry<String, InnerHitsContext.BaseInnerHits> entry : context.innerHits().getInnerHits().entrySet()) {
             InnerHitsContext.BaseInnerHits innerHits = entry.getValue();
@@ -92,13 +79,4 @@ public class InnerHitsFetchSubPhase implements FetchSubPhase {
         }
         hitContext.hit().setInnerHits(results);
     }
-
-    @Override
-    public boolean hitsExecutionNeeded(SearchContext context) {
-        return false;
-    }
-
-    @Override
-    public void hitsExecute(SearchContext context, InternalSearchHit[] hits) {
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/matchedqueries/MatchedQueriesFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/matchedqueries/MatchedQueriesFetchSubPhase.java
index 983e131215d..59225f93a61 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/matchedqueries/MatchedQueriesFetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/matchedqueries/MatchedQueriesFetchSubPhase.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.search.fetch.matchedqueries;
 
+import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.ReaderUtil;
 import org.apache.lucene.search.Query;
@@ -26,7 +27,6 @@ import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.Bits;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.internal.InternalSearchHit;
 import org.elasticsearch.search.internal.SearchContext;
@@ -39,22 +39,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import static java.util.Collections.emptyMap;
-
-/**
- *
- */
-public class MatchedQueriesFetchSubPhase implements FetchSubPhase {
-
-    @Override
-    public Map<String, ? extends SearchParseElement> parseElements() {
-        return emptyMap();
-    }
-
-    @Override
-    public boolean hitsExecutionNeeded(SearchContext context) {
-        return true; // we short-circuit in hitsExecute
-    }
+public final class MatchedQueriesFetchSubPhase implements FetchSubPhase {
 
     @Override
     public void hitsExecute(SearchContext context, InternalSearchHit[] hits) {
@@ -82,12 +67,13 @@ public class MatchedQueriesFetchSubPhase implements FetchSubPhase {
                 int docBase = -1;
                 Weight weight = context.searcher().createNormalizedWeight(query, false);
                 Bits matchingDocs = null;
+                final IndexReader indexReader = context.searcher().getIndexReader();
                 for (int i = 0; i < hits.length; ++i) {
                     InternalSearchHit hit = hits[i];
-                    int hitReaderIndex = ReaderUtil.subIndex(hit.docId(), context.searcher().getIndexReader().leaves());
+                    int hitReaderIndex = ReaderUtil.subIndex(hit.docId(), indexReader.leaves());
                     if (readerIndex != hitReaderIndex) {
                         readerIndex = hitReaderIndex;
-                        LeafReaderContext ctx = context.searcher().getIndexReader().leaves().get(readerIndex);
+                        LeafReaderContext ctx = indexReader.leaves().get(readerIndex);
                         docBase = ctx.docBase;
                         // scorers can be costly to create, so reuse them across docs of the same segment
                         Scorer scorer = weight.scorer(ctx);
@@ -99,7 +85,7 @@ public class MatchedQueriesFetchSubPhase implements FetchSubPhase {
                 }
             }
             for (int i = 0; i < hits.length; ++i) {
-                hits[i].matchedQueries(matchedQueries[i].toArray(new String[0]));
+                hits[i].matchedQueries(matchedQueries[i].toArray(new String[matchedQueries[i].size()]));
             }
         } catch (IOException e) {
             throw ExceptionsHelper.convertToElastic(e);
@@ -107,14 +93,4 @@ public class MatchedQueriesFetchSubPhase implements FetchSubPhase {
             SearchContext.current().clearReleasables(Lifetime.COLLECTION);
         }
     }
-
-    @Override
-    public boolean hitExecutionNeeded(SearchContext context) {
-        return false;
-    }
-
-    @Override
-    public void hitExecute(SearchContext context, HitContext hitContext) {
-        // we do everything in hitsExecute
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/parent/ParentFieldSubFetchPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/parent/ParentFieldSubFetchPhase.java
index 41fe7176424..6ace9a86a3e 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/parent/ParentFieldSubFetchPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/parent/ParentFieldSubFetchPhase.java
@@ -25,10 +25,7 @@ import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
 import org.elasticsearch.search.SearchHitField;
-import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.fetch.FetchSubPhase;
-import org.elasticsearch.search.fetch.innerhits.InnerHitsContext;
-import org.elasticsearch.search.internal.InternalSearchHit;
 import org.elasticsearch.search.internal.InternalSearchHitField;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -37,17 +34,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
-public class ParentFieldSubFetchPhase implements FetchSubPhase {
-
-    @Override
-    public Map<String, ? extends SearchParseElement> parseElements() {
-        return Collections.emptyMap();
-    }
-
-    @Override
-    public boolean hitExecutionNeeded(SearchContext context) {
-        return true;
-    }
+public final class ParentFieldSubFetchPhase implements FetchSubPhase {
 
     @Override
     public void hitExecute(SearchContext context, HitContext hitContext) {
@@ -65,15 +52,6 @@ public class ParentFieldSubFetchPhase implements FetchSubPhase {
         fields.put(ParentFieldMapper.NAME, new InternalSearchHitField(ParentFieldMapper.NAME, Collections.singletonList(parentId)));
     }
 
-    @Override
-    public boolean hitsExecutionNeeded(SearchContext context) {
-        return false;
-    }
-
-    @Override
-    public void hitsExecute(SearchContext context, InternalSearchHit[] hits) {
-    }
-
     public static String getParentId(ParentFieldMapper fieldMapper, LeafReader reader, int docId) {
         try {
             SortedDocValues docValues = reader.getSortedDocValues(fieldMapper.name());
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/script/ScriptFieldsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/script/ScriptFieldsFetchSubPhase.java
index 540ce9aa66b..0838467bb6c 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/script/ScriptFieldsFetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/script/ScriptFieldsFetchSubPhase.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.fetch.script;
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.search.fetch.FetchSubPhase;
-import org.elasticsearch.search.internal.InternalSearchHit;
 import org.elasticsearch.search.internal.InternalSearchHitField;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -32,27 +31,13 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 
-/**
- *
- */
-public class ScriptFieldsFetchSubPhase implements FetchSubPhase {
-
-    @Override
-    public boolean hitsExecutionNeeded(SearchContext context) {
-        return false;
-    }
-
-    @Override
-    public void hitsExecute(SearchContext context, InternalSearchHit[] hits) {
-    }
-
-    @Override
-    public boolean hitExecutionNeeded(SearchContext context) {
-        return context.hasScriptFields();
-    }
+public final class ScriptFieldsFetchSubPhase implements FetchSubPhase {
 
     @Override
     public void hitExecute(SearchContext context, HitContext hitContext) {
+        if (context.hasScriptFields() == false) {
+            return;
+        }
         for (ScriptFieldsContext.ScriptField scriptField : context.scriptFields().fields()) {
             LeafSearchScript leafScript;
             try {
@@ -62,10 +47,9 @@ public class ScriptFieldsFetchSubPhase implements FetchSubPhase {
             }
             leafScript.setDocument(hitContext.docId());
 
-            Object value;
+            final Object value;
             try {
-                value = leafScript.run();
-                value = leafScript.unwrap(value);
+                value = leafScript.unwrap(leafScript.run());
             } catch (RuntimeException e) {
                 if (scriptField.ignoreException()) {
                     continue;
@@ -74,7 +58,7 @@ public class ScriptFieldsFetchSubPhase implements FetchSubPhase {
             }
 
             if (hitContext.hit().fieldsOrNull() == null) {
-                hitContext.hit().fields(new HashMap<String, SearchHitField>(2));
+                hitContext.hit().fields(new HashMap<>(2));
             }
 
             SearchHitField hitField = hitContext.hit().fields().get(scriptField.name());
@@ -84,7 +68,7 @@ public class ScriptFieldsFetchSubPhase implements FetchSubPhase {
                     values = Collections.emptyList();
                 } else if (value instanceof Collection) {
                     // TODO: use diamond operator once JI-9019884 is fixed
-                    values = new ArrayList<Object>((Collection<?>) value);
+                    values = new ArrayList<>((Collection<?>) value);
                 } else {
                     values = Collections.singletonList(value);
                 }
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceSubPhase.java
index 30220d12694..900402bda13 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/source/FetchSourceSubPhase.java
@@ -23,32 +23,18 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.fetch.FetchSubPhase;
-import org.elasticsearch.search.internal.InternalSearchHit;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.lookup.SourceLookup;
 
 import java.io.IOException;
 
-/**
- */
-public class FetchSourceSubPhase implements FetchSubPhase {
-
-    @Override
-    public boolean hitsExecutionNeeded(SearchContext context) {
-        return false;
-    }
-
-    @Override
-    public void hitsExecute(SearchContext context, InternalSearchHit[] hits) {
-    }
-
-    @Override
-    public boolean hitExecutionNeeded(SearchContext context) {
-        return context.sourceRequested();
-    }
+public final class FetchSourceSubPhase implements FetchSubPhase {
 
     @Override
     public void hitExecute(SearchContext context, HitContext hitContext) {
+        if (context.sourceRequested() == false) {
+            return;
+        }
         FetchSourceContext fetchSourceContext = context.fetchSourceContext();
         assert fetchSourceContext.fetchSource();
         if (fetchSourceContext.includes().length == 0 && fetchSourceContext.excludes().length == 0) {
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/version/VersionFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/version/VersionFetchSubPhase.java
index c5511352ec2..77a0e954b2d 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/version/VersionFetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/version/VersionFetchSubPhase.java
@@ -25,36 +25,21 @@ import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.mapper.internal.UidFieldMapper;
 import org.elasticsearch.search.fetch.FetchSubPhase;
-import org.elasticsearch.search.internal.InternalSearchHit;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.io.IOException;
 
-/**
- *
- */
-public class VersionFetchSubPhase implements FetchSubPhase {
-
-    @Override
-    public boolean hitsExecutionNeeded(SearchContext context) {
-        return false;
-    }
-
-    @Override
-    public void hitsExecute(SearchContext context, InternalSearchHit[] hits) {
-    }
-
-    @Override
-    public boolean hitExecutionNeeded(SearchContext context) {
-        return context.version();
-    }
+public final class VersionFetchSubPhase implements FetchSubPhase {
 
     @Override
     public void hitExecute(SearchContext context, HitContext hitContext) {
+        if (context.version() == false) {
+            return;
+        }
         // it might make sense to cache the TermDocs on a shared fetch context and just skip here)
         // it is going to mean we work on the high level multi reader and not the lower level reader as is
         // the case below...
-        long version;
+        final long version;
         try {
             BytesRef uid = Uid.createUidAsBytes(hitContext.hit().type(), hitContext.hit().id());
             version = Versions.loadVersion(
@@ -64,10 +49,6 @@ public class VersionFetchSubPhase implements FetchSubPhase {
         } catch (IOException e) {
             throw new ElasticsearchException("Could not query index for _version", e);
         }
-
-        if (version < 0) {
-            version = -1;
-        }
-        hitContext.hit().version(version);
+        hitContext.hit().version(version < 0 ? -1 : version);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java b/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java
index 701cd588892..7952addbe8c 100644
--- a/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.highlight;
 
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.DocumentMapper;
@@ -31,9 +30,7 @@ import org.elasticsearch.index.mapper.core.StringFieldMapper;
 import org.elasticsearch.index.mapper.core.TextFieldMapper;
 import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
 import org.elasticsearch.search.Highlighters;
-import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.fetch.FetchSubPhase;
-import org.elasticsearch.search.internal.InternalSearchHit;
 import org.elasticsearch.search.internal.SearchContext;
 
 import java.util.Arrays;
@@ -43,45 +40,21 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-/**
- *
- */
 public class HighlightPhase extends AbstractComponent implements FetchSubPhase {
     private static final List<String> STANDARD_HIGHLIGHTERS_BY_PRECEDENCE = Arrays.asList("fvh", "postings", "plain");
 
     private final Highlighters highlighters;
 
-    @Inject
     public HighlightPhase(Settings settings, Highlighters highlighters) {
         super(settings);
         this.highlighters = highlighters;
     }
 
-    /**
-     * highlighters do not have a parse element, they use
-     * {@link HighlightBuilder#fromXContent(org.elasticsearch.index.query.QueryParseContext)} for parsing instead.
-     */
-    @Override
-    public Map<String, ? extends SearchParseElement> parseElements() {
-        return Collections.emptyMap();
-    }
-
-    @Override
-    public boolean hitsExecutionNeeded(SearchContext context) {
-        return false;
-    }
-
-    @Override
-    public void hitsExecute(SearchContext context, InternalSearchHit[] hits) {
-    }
-
-    @Override
-    public boolean hitExecutionNeeded(SearchContext context) {
-        return context.highlight() != null;
-    }
-
     @Override
     public void hitExecute(SearchContext context, HitContext hitContext) {
+        if (context.highlight() == null) {
+            return;
+        }
         Map<String, HighlightField> highlightFields = new HashMap<>();
         for (SearchContextHighlight.Field field : context.highlight().fields()) {
             Collection<String> fieldNamesToHighlight;
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
index 65da6b58d20..81c923ccf6d 100644
--- a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
+++ b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
@@ -116,7 +116,7 @@ public class FetchSubPhasePluginIT extends ESIntegTestCase {
         }
     }
 
-    public static class TermVectorsFetchSubPhase implements FetchSubPhase {
+    public final static class TermVectorsFetchSubPhase implements FetchSubPhase {
 
         public static final ContextFactory<TermVectorsFetchContext> CONTEXT_FACTORY = new ContextFactory<TermVectorsFetchContext>() {
 
@@ -138,22 +138,11 @@ public class FetchSubPhasePluginIT extends ESIntegTestCase {
             return singletonMap("term_vectors_fetch", new TermVectorsFetchParseElement());
         }
 
-        @Override
-        public boolean hitsExecutionNeeded(SearchContext context) {
-            return false;
-        }
-
-        @Override
-        public void hitsExecute(SearchContext context, InternalSearchHit[] hits) {
-        }
-
-        @Override
-        public boolean hitExecutionNeeded(SearchContext context) {
-            return context.getFetchSubPhaseContext(CONTEXT_FACTORY).hitExecutionNeeded();
-        }
-
         @Override
         public void hitExecute(SearchContext context, HitContext hitContext) {
+            if (context.getFetchSubPhaseContext(CONTEXT_FACTORY).hitExecutionNeeded() == false) {
+                return;
+            }
             String field = context.getFetchSubPhaseContext(CONTEXT_FACTORY).getField();
 
             if (hitContext.hit().fieldsOrNull() == null) {
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
index ccf33a69739..4a110172d77 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
@@ -32,7 +32,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.search.Highlighters;
-import org.elasticsearch.search.SearchParseElement;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.highlight.HighlightPhase;
 import org.elasticsearch.search.highlight.SearchContextHighlight;
@@ -43,23 +42,27 @@ import org.elasticsearch.search.internal.SubSearchContext;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
-import java.util.Map;
 
-// Highlighting in the case of the percolate query is a bit different, because the PercolateQuery itself doesn't get highlighted,
-// but the source of the PercolateQuery gets highlighted by each hit containing a query.
-public class PercolatorHighlightSubFetchPhase extends HighlightPhase {
+/**
+ * Highlighting in the case of the percolate query is a bit different, because the PercolateQuery itself doesn't get highlighted,
+ * but the source of the PercolateQuery gets highlighted by each hit containing a query.
+ */
+public final class PercolatorHighlightSubFetchPhase extends HighlightPhase {
 
     public PercolatorHighlightSubFetchPhase(Settings settings, Highlighters highlighters) {
         super(settings, highlighters);
     }
 
-    @Override
-    public boolean hitsExecutionNeeded(SearchContext context) {
+
+    boolean hitsExecutionNeeded(SearchContext context) { // for testing
         return context.highlight() != null && locatePercolatorQuery(context.query()) != null;
     }
 
     @Override
     public void hitsExecute(SearchContext context, InternalSearchHit[] hits) {
+        if (hitsExecutionNeeded(context) == false) {
+            return;
+        }
         PercolateQuery percolateQuery = locatePercolatorQuery(context.query());
         if (percolateQuery == null) {
             // shouldn't happen as we checked for the existence of a percolator query in hitsExecutionNeeded(...)
@@ -97,20 +100,6 @@ public class PercolatorHighlightSubFetchPhase extends HighlightPhase {
         }
     }
 
-    @Override
-    public Map<String, ? extends SearchParseElement> parseElements() {
-        return Collections.emptyMap();
-    }
-
-    @Override
-    public boolean hitExecutionNeeded(SearchContext context) {
-        return false;
-    }
-
-    @Override
-    public void hitExecute(SearchContext context, HitContext hitContext) {
-    }
-
     static PercolateQuery locatePercolatorQuery(Query query) {
         if (query instanceof PercolateQuery) {
             return (PercolateQuery) query;
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
index 95f64132ea1..1dfd941e8e0 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
@@ -53,10 +53,6 @@ public class PercolatorHighlightSubFetchPhaseTests extends ESTestCase {
         Mockito.when(searchContext.query()).thenReturn(new MatchAllDocsQuery());
 
         assertThat(subFetchPhase.hitsExecutionNeeded(searchContext), is(false));
-        IllegalStateException exception = expectThrows(IllegalStateException.class,
-                () -> subFetchPhase.hitsExecute(searchContext, null));
-        assertThat(exception.getMessage(), equalTo("couldn't locate percolator query"));
-
         Mockito.when(searchContext.query()).thenReturn(percolateQuery);
         assertThat(subFetchPhase.hitsExecutionNeeded(searchContext), is(true));
     }

From 2c467fd9c21cc2f79a58b6c1d412a87b83be94f3 Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@gmail.com>
Date: Wed, 15 Jun 2016 16:48:02 +0200
Subject: [PATCH 0865/1311] Add microbenchmarking infrastructure (#18891)

With this commit we add a benchmarks project that contains the necessary build
infrastructure and an example benchmark. It is added as a separate project to avoid
interfering with the regular build too much (especially sanity checks) and to keep
the microbenchmarks isolated.

Microbenchmarks are generated with `gradle :benchmarks:jmhJar` and can be run with
` gradle :benchmarks:jmh`.

We intentionally do not use the
[jmh-gradle-plugin](https://github.com/melix/jmh-gradle-plugin) as it causes all
sorts of problems (dependencies are not properly excluded, not all JMH parameters
can be set) and it adds another abstraction layer that is not needed.

Closes #18242
---
 benchmarks/README.md                          |  63 +++++++
 benchmarks/build.gradle                       |  96 ++++++++++
 .../benchmark/DateBenchmark.java              |  67 +++++++
 .../benchmark/HelloBenchmark.java             |  29 +++
 .../allocation/AllocationBenchmark.java       | 170 ++++++++++++++++++
 .../routing/allocation/Allocators.java        | 108 +++++++++++
 .../src/main/resources/log4j.properties       |   8 +
 buildSrc/version.properties                   |   3 +
 settings.gradle                               |   1 +
 9 files changed, 545 insertions(+)
 create mode 100644 benchmarks/README.md
 create mode 100644 benchmarks/build.gradle
 create mode 100644 benchmarks/src/main/java/org/elasticsearch/benchmark/DateBenchmark.java
 create mode 100644 benchmarks/src/main/java/org/elasticsearch/benchmark/HelloBenchmark.java
 create mode 100644 benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java
 create mode 100644 benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java
 create mode 100644 benchmarks/src/main/resources/log4j.properties

diff --git a/benchmarks/README.md b/benchmarks/README.md
new file mode 100644
index 00000000000..afe60c89368
--- /dev/null
+++ b/benchmarks/README.md
@@ -0,0 +1,63 @@
+# Elasticsearch Microbenchmark Suite
+
+This directory contains the microbenchmark suite of Elasticsearch. It relies on [JMH](http://openjdk.java.net/projects/code-tools/jmh/).
+
+## Purpose
+
+We do not want to microbenchmark everything but the kitchen sink and should typically rely on our 
+[macrobenchmarks](https://elasticsearch-benchmarks.elastic.co/app/kibana#/dashboard/Nightly-Benchmark-Overview) with 
+[Rally](http://github.com/elastic/rally). Microbenchmarks are intended for performance-critical components to spot performance 
+regressions. The microbenchmark suite is also handy for ad-hoc microbenchmarks but please remove them again before merging your PR.
+
+## Getting Started
+
+Just run `gradle :benchmarks:jmh` from the project root directory. It will build all microbenchmarks, execute them and print the result.
+
+## Running Microbenchmarks
+
+Benchmarks are always run via Gradle with `gradle :benchmarks:jmh`.
+```
+ 
+Running via an IDE is not supported as the results are meaningless (we have no control over the JVM running the benchmarks).
+
+If you want to run a specific benchmark class, e.g. `org.elasticsearch.benchmark.MySampleBenchmark` or have any other special requirements 
+generate the uberjar with `gradle :benchmarks:jmhJar` and run the it directly with:
+
+```
+java -jar benchmarks/build/distributions/elasticsearch-benchmarks-*.jar
+```
+
+JMH supports lots of command line parameters. Add `-h` to the command above for more information about the available command line options.
+
+## Adding Microbenchmarks
+
+Before adding a new microbenchmark, make yourself familiar with the JMH API. You can check our existing microbenchmarks and also the 
+[JMH samples](http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/).
+
+In contrast to tests, the actual name of the benchmark class is not relevant to JMH. However, stick to the naming convention and 
+end the class name of a benchmark with `Benchmark`. To have JMH execute a benchmark, annotate the respective methods with `@Benchmark`.
+
+## Tips and Best Practices
+
+To get realistic results, you should exercise care when running your benchmarks. Here are a few tips:
+
+### Do
+
+* Ensure that the system executing your microbenchmarks has as little load as possible and shutdown every process that can cause unnecessary 
+  runtime jitter. Watch the `Error` column in the benchmark results to see the run-to-run variance.
+* Ensure to run enough warmup iterations to get into a stable state. If you are unsure, don't change the defaults.
+* Avoid CPU migrations by pinning your benchmarks to specific CPU cores. On Linux you can use `taskset`.
+* Fix the CPU frequency to avoid Turbo Boost from kicking in and skewing your results. On Linux you can use `cpufreq-set` and the 
+  `performance` CPU governor.
+* Vary problem input size with `@Param`.
+* Use the integrated profilers in JMH to dig deeper if benchmark results to not match your hypotheses:
+** Run the generated uberjar directly and use `-prof gc` to check whether the garbage collector runs during a microbenchmarks and skews 
+   your results. If so, try to force a GC between runs (`-gc true`).
+** Use `-prof perf` or `-prof perfasm` (both only available on Linux) to see hotspots.
+* Have your benchmarks peer-reviewed.
+
+### Don't
+
+* Blindly believe the numbers that your microbenchmark produces but verify them by measuring e.e. with `-prof perfasm`.
+* Run run more threads than your number of CPU cores (in case you run multi-threaded microbenchmarks).
+* Look only at the `Score` column and ignore `Error`. Instead take countermeasures to keep `Error` low / variance explainable.
\ No newline at end of file
diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle
new file mode 100644
index 00000000000..26fb5105af3
--- /dev/null
+++ b/benchmarks/build.gradle
@@ -0,0 +1,96 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+buildscript {
+    repositories {
+        maven {
+            url 'https://plugins.gradle.org/m2/'
+        }
+    }
+    dependencies {
+        classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.3'
+    }
+}
+
+apply plugin: 'elasticsearch.build'
+// build an uberjar with all benchmarks
+apply plugin: 'com.github.johnrengelman.shadow'
+// have the shadow plugin provide the runShadow task
+apply plugin: 'application'
+
+archivesBaseName = 'elasticsearch-benchmarks'
+mainClassName = 'org.openjdk.jmh.Main'
+
+// never try to invoke tests on the benchmark project - there aren't any
+check.dependsOn.remove(test)
+// explicitly override the test task too in case somebody invokes 'gradle test' so it won't trip
+task test(type: Test, overwrite: true)
+
+dependencies {
+    compile("org.elasticsearch:elasticsearch:${version}") {
+        // JMH ships with the conflicting version 4.6 (JMH will not update this dependency as it is Java 6 compatible and joptsimple is one
+        // of the most recent compatible version). This prevents us from using jopt-simple in benchmarks (which should be ok) but allows us
+        // to invoke the JMH uberjar as usual.
+        exclude group: 'net.sf.jopt-simple', module: 'jopt-simple'
+    }
+    compile "org.openjdk.jmh:jmh-core:$versions.jmh"
+    compile "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh"
+    //TODO: Transitive dependencies of JMH. Add them here explicitly for the time being and find out why they are not included implicitly.
+    runtime 'net.sf.jopt-simple:jopt-simple:4.6'
+    runtime 'org.apache.commons:commons-math3:3.2'
+}
+
+compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
+compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
+
+forbiddenApis {
+    // classes generated by JMH can use all sorts of forbidden APIs but we have no influence at all and cannot exclude these classes
+    ignoreFailures = true
+}
+
+// No licenses for our benchmark deps (we don't ship benchmarks)
+dependencyLicenses.enabled = false
+
+thirdPartyAudit.excludes = [
+        // these classes intentionally use JDK internal API (and this is ok since the project is maintained by Oracle employees)
+        'org.openjdk.jmh.profile.AbstractHotspotProfiler',
+        'org.openjdk.jmh.profile.HotspotThreadProfiler',
+        'org.openjdk.jmh.profile.HotspotClassloadingProfiler',
+        'org.openjdk.jmh.profile.HotspotCompilationProfiler',
+        'org.openjdk.jmh.profile.HotspotMemoryProfiler',
+        'org.openjdk.jmh.profile.HotspotRuntimeProfiler',
+        'org.openjdk.jmh.util.Utils'
+]
+
+shadowJar {
+    classifier = 'benchmarks'
+}
+
+// alias the shadowJar and runShadow tasks to abstract from the concrete plugin that we are using and provide a more consistent interface
+task jmhJar(
+        dependsOn: shadowJar,
+        description: 'Generates an uberjar with the microbenchmarks and all dependencies',
+        group: 'Benchmark'
+)
+
+task jmh(
+        dependsOn: runShadow,
+        description: 'Runs all microbenchmarks',
+        group: 'Benchmark'
+)
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/DateBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/DateBenchmark.java
new file mode 100644
index 00000000000..87f4f0ccbdf
--- /dev/null
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/DateBenchmark.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.benchmark;
+
+import org.joda.time.DateTimeZone;
+import org.joda.time.MutableDateTime;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.temporal.TemporalAmount;
+import java.util.Calendar;
+import java.util.Locale;
+import java.util.TimeZone;
+
+@SuppressWarnings("unused") //invoked by benchmarking framework
+@State(Scope.Benchmark)
+public class DateBenchmark {
+    private long instant;
+
+    private MutableDateTime jodaDate = new MutableDateTime(0, DateTimeZone.UTC);
+
+    private Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT);
+
+    private ZonedDateTime javaDateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(0L), ZoneOffset.UTC);
+
+    private TemporalAmount diff = Duration.ofMillis(1L);
+
+    @Benchmark
+    public int mutableDateTimeSetMillisGetDayOfMonth() {
+        jodaDate.setMillis(instant++);
+        return jodaDate.getDayOfMonth();
+    }
+
+    @Benchmark
+    public int calendarSetMillisGetDayOfMonth() {
+        calendar.setTimeInMillis(instant++);
+        return calendar.get(Calendar.DAY_OF_MONTH);
+    }
+
+    @Benchmark
+    public int javaDateTimeSetMillisGetDayOfMonth() {
+        // all classes in java.time are immutable, we have to use a new instance
+        javaDateTime = javaDateTime.plus(diff);
+        return javaDateTime.getDayOfMonth();
+    }
+}
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/HelloBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/HelloBenchmark.java
new file mode 100644
index 00000000000..0af853c18f7
--- /dev/null
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/HelloBenchmark.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.benchmark;
+
+import org.openjdk.jmh.annotations.Benchmark;
+
+@SuppressWarnings("unused") //invoked by benchmarking framework
+public class HelloBenchmark {
+    @Benchmark
+    public void benchmarkRuntimeOverhead() {
+        //intentionally left blank
+    }
+}
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java
new file mode 100644
index 00000000000..d5bb0916eca
--- /dev/null
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.benchmark.routing.allocation;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.common.settings.Settings;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.Collections;
+import java.util.concurrent.TimeUnit;
+
+@Fork(3)
+@Warmup(iterations = 10)
+@Measurement(iterations = 10)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+@SuppressWarnings("unused") //invoked by benchmarking framework
+public class AllocationBenchmark {
+    // Do NOT make any field final (even if it is not annotated with @Param)! See also
+    // http://hg.openjdk.java.net/code-tools/jmh/file/tip/jmh-samples/src/main/java/org/openjdk/jmh/samples/JMHSample_10_ConstantFold.java
+
+    // we cannot use individual @Params as some will lead to invalid combinations which do not let the benchmark terminate. JMH offers no
+    // support to constrain the combinations of benchmark parameters and we do not want to rely on OptionsBuilder as each benchmark would
+    // need its own main method and we cannot execute more than one class with a main method per JAR.
+    @Param({
+        // indices, shards, replicas, nodes
+        "       10,      1,        0,     1",
+        "       10,      3,        0,     1",
+        "       10,     10,        0,     1",
+        "      100,      1,        0,     1",
+        "      100,      3,        0,     1",
+        "      100,     10,        0,     1",
+
+        "       10,      1,        0,    10",
+        "       10,      3,        0,    10",
+        "       10,     10,        0,    10",
+        "      100,      1,        0,    10",
+        "      100,      3,        0,    10",
+        "      100,     10,        0,    10",
+
+        "       10,      1,        1,    10",
+        "       10,      3,        1,    10",
+        "       10,     10,        1,    10",
+        "      100,      1,        1,    10",
+        "      100,      3,        1,    10",
+        "      100,     10,        1,    10",
+
+        "       10,      1,        2,    10",
+        "       10,      3,        2,    10",
+        "       10,     10,        2,    10",
+        "      100,      1,        2,    10",
+        "      100,      3,        2,    10",
+        "      100,     10,        2,    10",
+
+        "       10,      1,        0,    50",
+        "       10,      3,        0,    50",
+        "       10,     10,        0,    50",
+        "      100,      1,        0,    50",
+        "      100,      3,        0,    50",
+        "      100,     10,        0,    50",
+
+        "       10,      1,        1,    50",
+        "       10,      3,        1,    50",
+        "       10,     10,        1,    50",
+        "      100,      1,        1,    50",
+        "      100,      3,        1,    50",
+        "      100,     10,        1,    50",
+
+        "       10,      1,        2,    50",
+        "       10,      3,        2,    50",
+        "       10,     10,        2,    50",
+        "      100,      1,        2,    50",
+        "      100,      3,        2,    50",
+        "      100,     10,        2,    50"
+    })
+    public String indicesShardsReplicasNodes = "10,1,0,1";
+
+    public int numTags = 2;
+
+    private AllocationService strategy;
+    private ClusterState initialClusterState;
+
+    @Setup
+    public void setUp() throws Exception {
+        final String[] params = indicesShardsReplicasNodes.split(",");
+
+        int numIndices = toInt(params[0]);
+        int numShards = toInt(params[1]);
+        int numReplicas = toInt(params[2]);
+        int numNodes = toInt(params[3]);
+
+        strategy = Allocators.createAllocationService(Settings.builder()
+                .put("cluster.routing.allocation.awareness.attributes", "tag")
+                .build());
+
+        MetaData.Builder mb = MetaData.builder();
+        for (int i = 1; i <= numIndices; i++) {
+            mb.put(IndexMetaData.builder("test_" + i)
+                    .settings(Settings.builder().put("index.version.created", Version.CURRENT))
+                    .numberOfShards(numShards)
+                    .numberOfReplicas(numReplicas)
+            );
+        }
+        MetaData metaData = mb.build();
+        RoutingTable.Builder rb = RoutingTable.builder();
+        for (int i = 1; i <= numIndices; i++) {
+            rb.addAsNew(metaData.index("test_" + i));
+        }
+        RoutingTable routingTable = rb.build();
+        DiscoveryNodes.Builder nb = DiscoveryNodes.builder();
+        for (int i = 1; i <= numNodes; i++) {
+            nb.put(Allocators.newNode("node" + i, Collections.singletonMap("tag", "tag_" + (i % numTags))));
+        }
+        initialClusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).nodes
+                (nb).build();
+    }
+
+    private int toInt(String v) {
+        return Integer.valueOf(v.trim());
+    }
+
+    @Benchmark
+    public ClusterState measureAllocation() {
+        ClusterState clusterState = initialClusterState;
+        while (clusterState.getRoutingNodes().hasUnassignedShards()) {
+            RoutingAllocation.Result result = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes()
+                    .shardsWithState(ShardRoutingState.INITIALIZING));
+            clusterState = ClusterState.builder(clusterState).routingResult(result).build();
+            result = strategy.reroute(clusterState, "reroute");
+            clusterState = ClusterState.builder(clusterState).routingResult(result).build();
+        }
+        return clusterState;
+    }
+}
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java
new file mode 100644
index 00000000000..aba7fda1021
--- /dev/null
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.benchmark.routing.allocation;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.cluster.EmptyClusterInfoService;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
+import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
+import org.elasticsearch.cluster.routing.allocation.StartedRerouteAllocation;
+import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.gateway.GatewayAllocator;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public final class Allocators {
+    private static class NoopGatewayAllocator extends GatewayAllocator {
+        public static final NoopGatewayAllocator INSTANCE = new NoopGatewayAllocator();
+
+        protected NoopGatewayAllocator() {
+            super(Settings.EMPTY, null, null);
+        }
+
+        @Override
+        public void applyStartedShards(StartedRerouteAllocation allocation) {
+            // noop
+        }
+
+        @Override
+        public void applyFailedShards(FailedRerouteAllocation allocation) {
+            // noop
+        }
+
+        @Override
+        public boolean allocateUnassigned(RoutingAllocation allocation) {
+            return false;
+        }
+    }
+
+    private Allocators() {
+        throw new AssertionError("Do not instantiate");
+    }
+
+
+    public static AllocationService createAllocationService(Settings settings) throws NoSuchMethodException, InstantiationException,
+        IllegalAccessException, InvocationTargetException {
+        return createAllocationService(settings, new ClusterSettings(Settings.Builder.EMPTY_SETTINGS, ClusterSettings
+            .BUILT_IN_CLUSTER_SETTINGS));
+    }
+
+    public static AllocationService createAllocationService(Settings settings, ClusterSettings clusterSettings) throws
+        InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
+        return new AllocationService(settings,
+            defaultAllocationDeciders(settings, clusterSettings),
+            NoopGatewayAllocator.INSTANCE, new BalancedShardsAllocator(settings), EmptyClusterInfoService.INSTANCE);
+    }
+
+    public static AllocationDeciders defaultAllocationDeciders(Settings settings, ClusterSettings clusterSettings) throws
+        IllegalAccessException, InvocationTargetException, InstantiationException, NoSuchMethodException {
+        List<AllocationDecider> list = new ArrayList<>();
+        // Keep a deterministic order of allocation deciders for the benchmark
+        for (Class<? extends AllocationDecider> deciderClass : ClusterModule.DEFAULT_ALLOCATION_DECIDERS) {
+            try {
+                Constructor<? extends AllocationDecider> constructor = deciderClass.getConstructor(Settings.class, ClusterSettings
+                    .class);
+                list.add(constructor.newInstance(settings, clusterSettings));
+            } catch (NoSuchMethodException e) {
+                Constructor<? extends AllocationDecider> constructor = deciderClass.getConstructor(Settings.class);
+                list.add(constructor.newInstance(settings));
+            }
+        }
+        return new AllocationDeciders(settings, list.toArray(new AllocationDecider[0]));
+
+    }
+
+    public static DiscoveryNode newNode(String nodeId, Map<String, String> attributes) {
+        return new DiscoveryNode("", nodeId, DummyTransportAddress.INSTANCE, attributes, Sets.newHashSet(DiscoveryNode.Role.MASTER,
+            DiscoveryNode.Role.DATA), Version.CURRENT);
+    }
+}
diff --git a/benchmarks/src/main/resources/log4j.properties b/benchmarks/src/main/resources/log4j.properties
new file mode 100644
index 00000000000..8ca1bc87295
--- /dev/null
+++ b/benchmarks/src/main/resources/log4j.properties
@@ -0,0 +1,8 @@
+# Do not log at all if it is not really critical - we're in a benchmark
+benchmarks.es.logger.level=ERROR
+log4j.rootLogger=${benchmarks.es.logger.level}, out
+
+log4j.appender.out=org.apache.log4j.ConsoleAppender
+log4j.appender.out.layout=org.apache.log4j.PatternLayout
+log4j.appender.out.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c] %m%n
+
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index 4ee6febdfa5..f8d8b5848c9 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -17,3 +17,6 @@ httpclient        = 4.5.2
 httpcore          = 4.4.4
 commonslogging    = 1.1.3
 commonscodec      = 1.10
+
+# benchmark dependencies
+jmh               = 1.12
\ No newline at end of file
diff --git a/settings.gradle b/settings.gradle
index 5aa675eac78..82cfa58af62 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -5,6 +5,7 @@ List projects = [
   'rest-api-spec',
   'core',
   'docs',
+  'benchmarks',
   'distribution:integ-test-zip',
   'distribution:zip',
   'distribution:tar',

From d56e4bc7b162a6467ee23d261844a88227716eb9 Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@elastic.co>
Date: Wed, 15 Jun 2016 16:54:54 +0200
Subject: [PATCH 0866/1311] Remove obsolete benchmarks / comments

---
 benchmarks/build.gradle                       |  2 +-
 .../benchmark/DateBenchmark.java              | 67 -------------------
 .../benchmark/HelloBenchmark.java             | 29 --------
 3 files changed, 1 insertion(+), 97 deletions(-)
 delete mode 100644 benchmarks/src/main/java/org/elasticsearch/benchmark/DateBenchmark.java
 delete mode 100644 benchmarks/src/main/java/org/elasticsearch/benchmark/HelloBenchmark.java

diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle
index 26fb5105af3..186fdca44ea 100644
--- a/benchmarks/build.gradle
+++ b/benchmarks/build.gradle
@@ -51,7 +51,7 @@ dependencies {
     }
     compile "org.openjdk.jmh:jmh-core:$versions.jmh"
     compile "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh"
-    //TODO: Transitive dependencies of JMH. Add them here explicitly for the time being and find out why they are not included implicitly.
+    // Dependencies of JMH
     runtime 'net.sf.jopt-simple:jopt-simple:4.6'
     runtime 'org.apache.commons:commons-math3:3.2'
 }
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/DateBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/DateBenchmark.java
deleted file mode 100644
index 87f4f0ccbdf..00000000000
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/DateBenchmark.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.benchmark;
-
-import org.joda.time.DateTimeZone;
-import org.joda.time.MutableDateTime;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.State;
-
-import java.time.Duration;
-import java.time.Instant;
-import java.time.ZoneOffset;
-import java.time.ZonedDateTime;
-import java.time.temporal.TemporalAmount;
-import java.util.Calendar;
-import java.util.Locale;
-import java.util.TimeZone;
-
-@SuppressWarnings("unused") //invoked by benchmarking framework
-@State(Scope.Benchmark)
-public class DateBenchmark {
-    private long instant;
-
-    private MutableDateTime jodaDate = new MutableDateTime(0, DateTimeZone.UTC);
-
-    private Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ROOT);
-
-    private ZonedDateTime javaDateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(0L), ZoneOffset.UTC);
-
-    private TemporalAmount diff = Duration.ofMillis(1L);
-
-    @Benchmark
-    public int mutableDateTimeSetMillisGetDayOfMonth() {
-        jodaDate.setMillis(instant++);
-        return jodaDate.getDayOfMonth();
-    }
-
-    @Benchmark
-    public int calendarSetMillisGetDayOfMonth() {
-        calendar.setTimeInMillis(instant++);
-        return calendar.get(Calendar.DAY_OF_MONTH);
-    }
-
-    @Benchmark
-    public int javaDateTimeSetMillisGetDayOfMonth() {
-        // all classes in java.time are immutable, we have to use a new instance
-        javaDateTime = javaDateTime.plus(diff);
-        return javaDateTime.getDayOfMonth();
-    }
-}
diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/HelloBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/HelloBenchmark.java
deleted file mode 100644
index 0af853c18f7..00000000000
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/HelloBenchmark.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.benchmark;
-
-import org.openjdk.jmh.annotations.Benchmark;
-
-@SuppressWarnings("unused") //invoked by benchmarking framework
-public class HelloBenchmark {
-    @Benchmark
-    public void benchmarkRuntimeOverhead() {
-        //intentionally left blank
-    }
-}

From cca4529b1cb8794e05fff6b0d1567058276a9483 Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@elastic.co>
Date: Wed, 15 Jun 2016 18:02:31 +0200
Subject: [PATCH 0867/1311] Mute NHRSLIT while investigating

---
 .../elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java
index d633313b1e7..8fdd02f8624 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java
@@ -89,6 +89,7 @@ public class NettyHttpRequestSizeLimitIT extends ESIntegTestCase {
         }
     }
 
+    @AwaitsFix(bugUrl = "muted while investigating")
     public void testDoesNotLimitExcludedRequests() throws Exception {
         ensureGreen();
 

From a473298f44869bb40436d874fbc4e50d1918f80e Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Wed, 15 Jun 2016 18:21:42 +0200
Subject: [PATCH 0868/1311] Add megamorphic cache

---
 .../elasticsearch/painless/DefBootstrap.java  | 61 +++++++++++++------
 1 file changed, 44 insertions(+), 17 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index ffcc295a33a..671b24d0dd0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -122,41 +122,62 @@ public final class DefBootstrap {
         /**
          * Does a slow lookup against the whitelist.
          */
-        private MethodHandle lookup(int flavor, String name, Object[] args) throws Throwable {
+        private MethodHandle lookup(int flavor, String name, Class<?> receiver, Object[] callArgs) throws Throwable {
             switch(flavor) {
                 case METHOD_CALL:
-                    return Def.lookupMethod(lookup, type(), args[0].getClass(), name, args, (Long) this.args[0]);
+                    return Def.lookupMethod(lookup, type(), receiver, name, callArgs, (Long) this.args[0]);
                 case LOAD:
-                    return Def.lookupGetter(args[0].getClass(), name);
+                    return Def.lookupGetter(receiver, name);
                 case STORE:
-                    return Def.lookupSetter(args[0].getClass(), name);
+                    return Def.lookupSetter(receiver, name);
                 case ARRAY_LOAD:
-                    return Def.lookupArrayLoad(args[0].getClass());
+                    return Def.lookupArrayLoad(receiver);
                 case ARRAY_STORE:
-                    return Def.lookupArrayStore(args[0].getClass());
+                    return Def.lookupArrayStore(receiver);
                 case ITERATOR:
-                    return Def.lookupIterator(args[0].getClass());
+                    return Def.lookupIterator(receiver);
                 case REFERENCE:
-                    return Def.lookupReference(lookup, (String) this.args[0], args[0].getClass(), name);
+                    return Def.lookupReference(lookup, (String) this.args[0], receiver, name);
                 default: throw new AssertionError();
             }
         }
 
         /**
          * Called when a new type is encountered (or, when we have encountered more than {@code MAX_DEPTH}
-         * types at this call site and given up on caching).
+         * types at this call site and given up on caching using this fallback wand we switch to a
+         * megamorphic cache using {@link ClassValue}).
          */
         @SuppressForbidden(reason = "slow path")
-        Object fallback(Object[] args) throws Throwable {
+        Object fallback(final Object[] callArgs) throws Throwable {
+            final Class<?> receiver = callArgs[0].getClass();
+            final MethodType type = type();
+            
             if (depth >= MAX_DEPTH) {
-                // XXX: caching defeated: lookups up every time(!)
-                return lookup(flavor, name, args).invokeWithArguments(args);
+                // we revert the whole cache and build a new megamorphic one
+                // using ClassValue and MethodHandles.exactInvoker():
+                final ClassValue<MethodHandle> megamorphicCache = new ClassValue<MethodHandle>() {
+                    @Override
+                    protected MethodHandle computeValue(Class<?> receiverType) {
+                        try {
+                            return lookup(flavor, name, receiverType, callArgs).asType(type);
+                        } catch (Throwable e) {
+                            // XXX: fix that
+                            throw new RuntimeException(e);
+                        }
+                    }
+                };
+                MethodHandle cacheLookup = MEGAMORPHIC_LOOKUP.bindTo(megamorphicCache);
+                cacheLookup = MethodHandles.dropArguments(cacheLookup,
+                        0, type.parameterList().subList(1, type.parameterCount()));
+                cacheLookup = cacheLookup.asType(type.changeReturnType(MethodHandle.class));
+                MethodHandle target = MethodHandles.foldArguments(MethodHandles.exactInvoker(type), cacheLookup);
+                setTarget(target);
+                return target.invokeWithArguments(callArgs);                    
             }
             
-            final MethodType type = type();
-            final MethodHandle target = lookup(flavor, name, args).asType(type);
+            final MethodHandle target = lookup(flavor, name, receiver, callArgs).asType(type);
 
-            MethodHandle test = CHECK_CLASS.bindTo(args[0].getClass());
+            MethodHandle test = CHECK_CLASS.bindTo(receiver);
             test = test.asType(test.type().changeParameterType(0, type.parameterType(0)));
 
             MethodHandle guard = MethodHandles.guardWithTest(test, target, getTarget());
@@ -164,18 +185,24 @@ public final class DefBootstrap {
             depth++;
 
             setTarget(guard);
-            return target.invokeWithArguments(args);
+            return target.invokeWithArguments(callArgs);
         }
 
         private static final MethodHandle CHECK_CLASS;
         private static final MethodHandle FALLBACK;
+        private static final MethodHandle MEGAMORPHIC_LOOKUP;
         static {
             final Lookup lookup = MethodHandles.lookup();
             try {
                 CHECK_CLASS = lookup.findStatic(lookup.lookupClass(), "checkClass",
                                               MethodType.methodType(boolean.class, Class.class, Object.class));
                 FALLBACK = lookup.findVirtual(lookup.lookupClass(), "fallback",
-                                              MethodType.methodType(Object.class, Object[].class));
+                        MethodType.methodType(Object.class, Object[].class));
+                MethodHandle mh = MethodHandles.publicLookup().findVirtual(ClassValue.class, "get",
+                        MethodType.methodType(Object.class, Class.class));
+                mh = MethodHandles.filterArguments(mh, 1, 
+                        MethodHandles.publicLookup().findVirtual(Object.class, "getClass", MethodType.methodType(Class.class)));
+                MEGAMORPHIC_LOOKUP = mh;
             } catch (ReflectiveOperationException e) {
                 throw new AssertionError(e);
             }

From 6487940a79ed6d4e4c8aad9bb30c1bc6652a1748 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Wed, 15 Jun 2016 18:26:11 +0200
Subject: [PATCH 0869/1311] Move type changes up to static part

---
 .../src/main/java/org/elasticsearch/painless/DefBootstrap.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index 671b24d0dd0..181aa10061f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -169,7 +169,6 @@ public final class DefBootstrap {
                 MethodHandle cacheLookup = MEGAMORPHIC_LOOKUP.bindTo(megamorphicCache);
                 cacheLookup = MethodHandles.dropArguments(cacheLookup,
                         0, type.parameterList().subList(1, type.parameterCount()));
-                cacheLookup = cacheLookup.asType(type.changeReturnType(MethodHandle.class));
                 MethodHandle target = MethodHandles.foldArguments(MethodHandles.exactInvoker(type), cacheLookup);
                 setTarget(target);
                 return target.invokeWithArguments(callArgs);                    
@@ -202,6 +201,7 @@ public final class DefBootstrap {
                         MethodType.methodType(Object.class, Class.class));
                 mh = MethodHandles.filterArguments(mh, 1, 
                         MethodHandles.publicLookup().findVirtual(Object.class, "getClass", MethodType.methodType(Class.class)));
+                mh = mh.asType(mh.type().changeReturnType(MethodHandle.class));
                 MEGAMORPHIC_LOOKUP = mh;
             } catch (ReflectiveOperationException e) {
                 throw new AssertionError(e);

From d4db8c9c8258d7e25befa7c7acbba86fb8ee9f77 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Wed, 15 Jun 2016 12:32:24 -0400
Subject: [PATCH 0870/1311] some cleanup

---
 .../elasticsearch/painless/DefBootstrap.java  | 56 ++++++++++++++++---
 .../elasticsearch/painless/MethodWriter.java  | 30 ++++++----
 .../elasticsearch/painless/node/EBinary.java  |  2 +-
 .../elasticsearch/painless/node/EChain.java   |  2 +-
 .../elasticsearch/painless/AdditionTests.java | 25 +++++++++
 5 files changed, 94 insertions(+), 21 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index 181aa10061f..c83a488e594 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -235,15 +235,21 @@ public final class DefBootstrap {
         /**
          * Does a slow lookup for the operator
          */
-        private MethodHandle lookup(int flavor, String name, Object[] args) throws Throwable {
+        private MethodHandle lookup(Object[] args) throws Throwable {
             switch(flavor) {
                 case UNARY_OPERATOR:
                 case SHIFT_OPERATOR:
+                    if ((flags & OPERATOR_COMPOUND_ASSIGNMENT) != 0) {
+                        return lookupGeneric(); // XXX: optimize better.
+                    }
                     // shifts are treated as unary, as java allows long arguments without a cast (but bits are ignored)
                     return DefMath.lookupUnary(args[0].getClass(), name);
                 case BINARY_OPERATOR:
+                    if ((flags & OPERATOR_COMPOUND_ASSIGNMENT) != 0) {
+                        return lookupGeneric(); // XXX: optimize better.
+                    }
                     if (args[0] == null || args[1] == null) {
-                        return DefMath.lookupGeneric(name); // can handle nulls, if supported
+                        return lookupGeneric(); // can handle nulls, if supported
                     } else {
                         return DefMath.lookupBinary(args[0].getClass(), args[1].getClass(), name);
                     }
@@ -251,6 +257,23 @@ public final class DefBootstrap {
             }
         }
         
+        private MethodHandle lookupGeneric() throws Throwable {
+            MethodHandle generic = DefMath.lookupGeneric(name);
+            if ((flags & OPERATOR_COMPOUND_ASSIGNMENT) != 0) {
+                assert flavor == BINARY_OPERATOR || flavor == SHIFT_OPERATOR;
+                // adapt dynamic cast to the generic method and the callsite's return value
+                MethodHandle cast = DYNAMIC_CAST.asType(MethodType.methodType(type().returnType(), 
+                                                                              generic.type().returnType(),
+                                                                              generic.type().parameterType(0)));
+                // drop the RHS parameter
+                cast = MethodHandles.dropArguments(cast, 2, generic.type().parameterType(1));
+                // combine: f(x,y) -> g(f(x,y), x, y);
+                return MethodHandles.foldArguments(cast, generic);
+            } else {
+                return generic;
+            }
+        }
+        
         /**
          * Called when a new type is encountered (or, when we have encountered more than {@code MAX_DEPTH}
          * types at this call site and given up on caching).
@@ -259,13 +282,13 @@ public final class DefBootstrap {
         Object fallback(Object[] args) throws Throwable {
             if (initialized) {
                 // caching defeated
-                MethodHandle generic = DefMath.lookupGeneric(name);
+                MethodHandle generic = lookupGeneric();
                 setTarget(generic.asType(type()));
                 return generic.invokeWithArguments(args);
             }
             
             final MethodType type = type();
-            final MethodHandle target = lookup(flavor, name, args).asType(type);
+            final MethodHandle target = lookup(args).asType(type);
 
             final MethodHandle test;
             if (flavor == BINARY_OPERATOR || flavor == SHIFT_OPERATOR) {
@@ -301,7 +324,7 @@ public final class DefBootstrap {
             // very special cases, where even the receiver can be null (see JLS rules for string concat)
             // we wrap + with an NPE catcher, and use our generic method in that case.
             if (flavor == BINARY_OPERATOR && (flags & OPERATOR_ALLOWS_NULL) != 0) {
-                MethodHandle handler = MethodHandles.dropArguments(DefMath.lookupGeneric(name).asType(type()), 
+                MethodHandle handler = MethodHandles.dropArguments(lookupGeneric().asType(type()), 
                                                                    0, 
                                                                    NullPointerException.class);
                 guard = MethodHandles.catchException(guard, NullPointerException.class, handler);
@@ -320,8 +343,7 @@ public final class DefBootstrap {
         static boolean checkLHS(Class<?> clazz, Object leftObject) {
             return leftObject.getClass() == clazz;
         }
-        
-        
+
         /**
          * guard method for inline caching: checks the first argument is the same
          * as the cached first argument.
@@ -338,9 +360,23 @@ public final class DefBootstrap {
             return leftObject.getClass() == left && rightObject.getClass() == right;
         }
         
+        /**
+         * Slow dynamic cast: casts {@code returnValue} to the runtime type of {@code lhs}
+         * based upon inspection. If {@code lhs} is null, no cast takes place.
+         * This is used for the generic fallback case of compound assignment.
+         */
+        static Object dynamicCast(Object returnValue, Object lhs) {
+            if (lhs != null) {
+                return lhs.getClass().cast(returnValue);
+            } else {
+                return returnValue;
+            }
+        }
+        
         private static final MethodHandle CHECK_LHS;
         private static final MethodHandle CHECK_RHS;
         private static final MethodHandle CHECK_BOTH;
+        private static final MethodHandle DYNAMIC_CAST;
         private static final MethodHandle FALLBACK;
         static {
             final Lookup lookup = MethodHandles.lookup();
@@ -351,6 +387,8 @@ public final class DefBootstrap {
                                               MethodType.methodType(boolean.class, Class.class, Class.class, Object.class, Object.class));
                 CHECK_BOTH = lookup.findStatic(lookup.lookupClass(), "checkBoth",
                                               MethodType.methodType(boolean.class, Class.class, Class.class, Object.class, Object.class));
+                DYNAMIC_CAST = lookup.findStatic(lookup.lookupClass(), "dynamicCast", 
+                                              MethodType.methodType(Object.class, Object.class, Object.class));
                 FALLBACK = lookup.findVirtual(lookup.lookupClass(), "fallback",
                                               MethodType.methodType(Object.class, Object[].class));
             } catch (ReflectiveOperationException e) {
@@ -416,6 +454,10 @@ public final class DefBootstrap {
                     // we just don't need it anywhere else.
                     throw new BootstrapMethodError("This parameter is only supported for BINARY_OPERATORs");
                 }
+                if ((flags & OPERATOR_COMPOUND_ASSIGNMENT) != 0 && flavor != BINARY_OPERATOR) {
+                    // we just don't need it anywhere else.
+                    throw new BootstrapMethodError("This parameter is only supported for BINARY_OPERATORs");
+                }
                 return new MIC(name, type, flavor, flags);
             default:
                 throw new BootstrapMethodError("Illegal static bootstrap parameter for flavor: " + flavor);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index b4923beabdc..79377ce1903 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -263,48 +263,54 @@ public final class MethodWriter extends GeneratorAdapter {
     }
 
     /** Writes a dynamic binary instruction: returnType, lhs, and rhs can be different */
-    public void writeDynamicBinaryInstruction(Location location, Type returnType, Type lhs, Type rhs, Operation operation) {
+    public void writeDynamicBinaryInstruction(Location location, Type returnType, Type lhs, Type rhs, Operation operation, boolean compoundAssignment) {
         org.objectweb.asm.Type methodType = org.objectweb.asm.Type.getMethodType(returnType.type, lhs.type, rhs.type);
         String descriptor = methodType.getDescriptor();
         
+        int flags = 0;
+        if (compoundAssignment) {
+            flags |= DefBootstrap.OPERATOR_COMPOUND_ASSIGNMENT;
+        }
         switch (operation) {
             case MUL:
-                invokeDynamic("mul", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, 0); 
+                invokeDynamic("mul", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, flags); 
                 break;
             case DIV:
-                invokeDynamic("div", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, 0); 
+                invokeDynamic("div", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, flags); 
                 break;
             case REM:
-                invokeDynamic("rem", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, 0); 
+                invokeDynamic("rem", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, flags); 
                 break;
             case ADD:
                 // if either side is primitive, then the + operator should always throw NPE on null,
                 // so we don't need a special NPE guard.
                 // otherwise, we need to allow nulls for possible string concatenation.
                 boolean hasPrimitiveArg = lhs.clazz.isPrimitive() || rhs.clazz.isPrimitive();
-                int flags = hasPrimitiveArg ? 0 : DefBootstrap.OPERATOR_ALLOWS_NULL;
+                if (!hasPrimitiveArg) {
+                    flags |= DefBootstrap.OPERATOR_ALLOWS_NULL;
+                }
                 invokeDynamic("add", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, flags);
                 break;
             case SUB:
-                invokeDynamic("sub", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, 0); 
+                invokeDynamic("sub", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, flags); 
                 break;
             case LSH:
-                invokeDynamic("lsh", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR, 0); 
+                invokeDynamic("lsh", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR, flags);
                 break;
             case USH:
-                invokeDynamic("ush", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR, 0); 
+                invokeDynamic("ush", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR, flags); 
                 break;
             case RSH:
-                invokeDynamic("rsh", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR, 0); 
+                invokeDynamic("rsh", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR, flags); 
                 break;
             case BWAND: 
-                invokeDynamic("and", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, 0);
+                invokeDynamic("and", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, flags);
                 break;
             case XOR:   
-                invokeDynamic("xor", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, 0);
+                invokeDynamic("xor", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, flags);
                 break;
             case BWOR:  
-                invokeDynamic("or", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, 0);
+                invokeDynamic("or", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, flags);
                 break;
             default:
                 throw location.createError(new IllegalStateException("Illegal tree structure."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 90d7b819902..acf9eb218b8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -612,7 +612,7 @@ public final class EBinary extends AExpression {
             right.write(writer);
 
             if (promote.sort == Sort.DEF || (shiftDistance != null && shiftDistance.sort == Sort.DEF)) {
-                writer.writeDynamicBinaryInstruction(location, actual, left.actual, right.actual, operation);
+                writer.writeDynamicBinaryInstruction(location, actual, left.actual, right.actual, operation, false);
             } else {
                 writer.writeBinaryInstruction(location, actual, operation);
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index e3484159b50..d0d26542f5c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -351,7 +351,7 @@ public final class EChain extends AExpression {
                     // write the operation instruction for compound assignment
                     if (promote.sort == Sort.DEF) {
                         writer.writeDynamicBinaryInstruction(location, promote, 
-                            Definition.DEF_TYPE, Definition.DEF_TYPE, operation);
+                            Definition.DEF_TYPE, Definition.DEF_TYPE, operation, true);
                     } else {
                         writer.writeBinaryInstruction(location, promote, operation);
                     }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java
index 59bf81b31f6..cce2be71a12 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java
@@ -406,4 +406,29 @@ public class AdditionTests extends ScriptTestCase {
         assertEquals(15D, exec("double x = 5.0; x += 10; return x;"));
         assertEquals(-5D, exec("double x = 5.0; x += -10; return x;"));
     }
+    
+    public void testDefCompoundAssignmentLHS() {
+        // byte
+        assertEquals((byte) 15, exec("def x = (byte)5; x += 10; return x;"));
+        assertEquals((byte) -5, exec("def x = (byte)5; x += -10; return x;"));
+
+        // short
+        assertEquals((short) 15, exec("def x = (short)5; x += 10; return x;"));
+        assertEquals((short) -5, exec("def x = (short)5; x += -10; return x;"));
+        // char
+        assertEquals((char) 15, exec("def x = (char)5; x += 10; return x;"));
+        assertEquals((char) 5, exec("def x = (char)10; x += -5; return x;"));
+        // int
+        assertEquals(15, exec("def x = 5; x += 10; return x;"));
+        assertEquals(-5, exec("def x = 5; x += -10; return x;"));
+        // long
+        assertEquals(15L, exec("def x = 5L; x += 10; return x;"));
+        assertEquals(-5L, exec("def x = 5L; x += -10; return x;"));
+        // float
+        assertEquals(15F, exec("def x = 5f; x += 10; return x;"));
+        assertEquals(-5F, exec("def x = 5f; x += -10; return x;"));
+        // double
+        assertEquals(15D, exec("def x = 5.0; x += 10; return x;"));
+        assertEquals(-5D, exec("def x = 5.0; x += -10; return x;"));
+    }
 }

From 04ab42cdf5b2a0d50704dc257aca68de7cc82669 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Wed, 15 Jun 2016 18:36:45 +0200
Subject: [PATCH 0871/1311] Use rethrow puzzler in the ClassValue

---
 .../src/main/java/org/elasticsearch/painless/Def.java      | 2 +-
 .../main/java/org/elasticsearch/painless/DefBootstrap.java | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 158086ab429..c24261dc5ac 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -138,7 +138,7 @@ public final class Def {
 
     /** Hack to rethrow unknown Exceptions from {@link MethodHandle#invokeExact}: */
     @SuppressWarnings("unchecked")
-    private static <T extends Throwable> void rethrow(Throwable t) throws T {
+    static <T extends Throwable> void rethrow(Throwable t) throws T {
         throw (T) t;
     }
     
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index 181aa10061f..f1651b5c869 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -158,11 +158,12 @@ public final class DefBootstrap {
                 final ClassValue<MethodHandle> megamorphicCache = new ClassValue<MethodHandle>() {
                     @Override
                     protected MethodHandle computeValue(Class<?> receiverType) {
+                        // it's too stupid that we cannot throw checked exceptions... (use rethrow puzzler):
                         try {
                             return lookup(flavor, name, receiverType, callArgs).asType(type);
-                        } catch (Throwable e) {
-                            // XXX: fix that
-                            throw new RuntimeException(e);
+                        } catch (Throwable t) {
+                            Def.rethrow(t);
+                            throw new AssertionError();
                         }
                     }
                 };

From 767bb49da846c8efc06b69fd48edeef6b3317865 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Wed, 15 Jun 2016 19:00:22 +0200
Subject: [PATCH 0872/1311] Fix bug with first param (receiver) on the cache
 lookup

---
 .../src/main/java/org/elasticsearch/painless/DefBootstrap.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index 312f4dd7c93..55948d9ce51 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -169,7 +169,7 @@ public final class DefBootstrap {
                 };
                 MethodHandle cacheLookup = MEGAMORPHIC_LOOKUP.bindTo(megamorphicCache);
                 cacheLookup = MethodHandles.dropArguments(cacheLookup,
-                        0, type.parameterList().subList(1, type.parameterCount()));
+                        1, type.parameterList().subList(1, type.parameterCount()));
                 MethodHandle target = MethodHandles.foldArguments(MethodHandles.exactInvoker(type), cacheLookup);
                 setTarget(target);
                 return target.invokeWithArguments(callArgs);                    

From ca2e0e16600db005eba262b6c3636a4e88e94504 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Wed, 15 Jun 2016 19:07:13 +0200
Subject: [PATCH 0873/1311] Some small cleanups around the lookup

---
 .../java/org/elasticsearch/painless/DefBootstrap.java     | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index 55948d9ce51..c40fc6a3234 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -193,17 +193,17 @@ public final class DefBootstrap {
         private static final MethodHandle MEGAMORPHIC_LOOKUP;
         static {
             final Lookup lookup = MethodHandles.lookup();
+            final Lookup publicLookup = MethodHandles.publicLookup();
             try {
                 CHECK_CLASS = lookup.findStatic(lookup.lookupClass(), "checkClass",
                                               MethodType.methodType(boolean.class, Class.class, Object.class));
                 FALLBACK = lookup.findVirtual(lookup.lookupClass(), "fallback",
                         MethodType.methodType(Object.class, Object[].class));
-                MethodHandle mh = MethodHandles.publicLookup().findVirtual(ClassValue.class, "get",
+                MethodHandle mh = publicLookup.findVirtual(ClassValue.class, "get",
                         MethodType.methodType(Object.class, Class.class));
                 mh = MethodHandles.filterArguments(mh, 1, 
-                        MethodHandles.publicLookup().findVirtual(Object.class, "getClass", MethodType.methodType(Class.class)));
-                mh = mh.asType(mh.type().changeReturnType(MethodHandle.class));
-                MEGAMORPHIC_LOOKUP = mh;
+                        publicLookup.findVirtual(Object.class, "getClass", MethodType.methodType(Class.class)));
+                MEGAMORPHIC_LOOKUP = mh.asType(mh.type().changeReturnType(MethodHandle.class));
             } catch (ReflectiveOperationException e) {
                 throw new AssertionError(e);
             }

From 27f8b6e6dbad8f1f3b4ae7c454ad73cab832e6cd Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Wed, 15 Jun 2016 13:30:21 -0400
Subject: [PATCH 0874/1311] get compound assignment working

---
 .../elasticsearch/painless/DefBootstrap.java  | 29 +-------
 .../org/elasticsearch/painless/DefMath.java   | 66 +++++++++++++++++++
 .../elasticsearch/painless/AdditionTests.java | 25 +++++++
 3 files changed, 93 insertions(+), 27 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index c40fc6a3234..d82b057b990 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -259,19 +259,10 @@ public final class DefBootstrap {
         }
         
         private MethodHandle lookupGeneric() throws Throwable {
-            MethodHandle generic = DefMath.lookupGeneric(name);
             if ((flags & OPERATOR_COMPOUND_ASSIGNMENT) != 0) {
-                assert flavor == BINARY_OPERATOR || flavor == SHIFT_OPERATOR;
-                // adapt dynamic cast to the generic method and the callsite's return value
-                MethodHandle cast = DYNAMIC_CAST.asType(MethodType.methodType(type().returnType(), 
-                                                                              generic.type().returnType(),
-                                                                              generic.type().parameterType(0)));
-                // drop the RHS parameter
-                cast = MethodHandles.dropArguments(cast, 2, generic.type().parameterType(1));
-                // combine: f(x,y) -> g(f(x,y), x, y);
-                return MethodHandles.foldArguments(cast, generic);
+                return DefMath.lookupGenericWithCast(name);
             } else {
-                return generic;
+                return DefMath.lookupGeneric(name);
             }
         }
         
@@ -361,23 +352,9 @@ public final class DefBootstrap {
             return leftObject.getClass() == left && rightObject.getClass() == right;
         }
         
-        /**
-         * Slow dynamic cast: casts {@code returnValue} to the runtime type of {@code lhs}
-         * based upon inspection. If {@code lhs} is null, no cast takes place.
-         * This is used for the generic fallback case of compound assignment.
-         */
-        static Object dynamicCast(Object returnValue, Object lhs) {
-            if (lhs != null) {
-                return lhs.getClass().cast(returnValue);
-            } else {
-                return returnValue;
-            }
-        }
-        
         private static final MethodHandle CHECK_LHS;
         private static final MethodHandle CHECK_RHS;
         private static final MethodHandle CHECK_BOTH;
-        private static final MethodHandle DYNAMIC_CAST;
         private static final MethodHandle FALLBACK;
         static {
             final Lookup lookup = MethodHandles.lookup();
@@ -388,8 +365,6 @@ public final class DefBootstrap {
                                               MethodType.methodType(boolean.class, Class.class, Class.class, Object.class, Object.class));
                 CHECK_BOTH = lookup.findStatic(lookup.lookupClass(), "checkBoth",
                                               MethodType.methodType(boolean.class, Class.class, Class.class, Object.class, Object.class));
-                DYNAMIC_CAST = lookup.findStatic(lookup.lookupClass(), "dynamicCast", 
-                                              MethodType.methodType(Object.class, Object.class, Object.class));
                 FALLBACK = lookup.findVirtual(lookup.lookupClass(), "fallback",
                                               MethodType.methodType(Object.class, Object[].class));
             } catch (ReflectiveOperationException e) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java
index e6b3a8c6003..665f5acbba3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java
@@ -1146,4 +1146,70 @@ public class DefMath {
     public static MethodHandle lookupGeneric(String name) {
         return TYPE_OP_MAPPING.get(Object.class).get(name);
     }
+    
+    
+    /**
+     * Slow dynamic cast: casts {@code returnValue} to the runtime type of {@code lhs}
+     * based upon inspection. If {@code lhs} is null, no cast takes place.
+     * This is used for the generic fallback case of compound assignment.
+     */
+    static Object dynamicCast(Object returnValue, Object lhs) {
+        if (lhs != null) {
+            Class<?> c = lhs.getClass();
+            if (c == Integer.class) {
+                return getNumber(returnValue).intValue();
+            } else if (c == Long.class) {
+                return getNumber(returnValue).longValue();
+            } else if (c == Double.class) {
+                return getNumber(returnValue).doubleValue();
+            } else if (c == Float.class) {
+                return getNumber(returnValue).floatValue();
+            } else if (c == Short.class) {
+                return getNumber(returnValue).shortValue();
+            } else if (c == Byte.class) {
+                return getNumber(returnValue).byteValue();
+            } else if (c == Character.class) {
+                return (char) getNumber(returnValue).intValue();
+            }
+            return lhs.getClass().cast(returnValue);
+        } else {
+            return returnValue;
+        }
+    }
+    
+    /** Slowly returns a Number for o. Just for supporting dynamicCast */
+    static Number getNumber(Object o) {
+        if (o instanceof Number) {
+            return (Number)o;
+        } else if (o instanceof Character) {
+            return Integer.valueOf((char)o);
+        } else {
+            throw new ClassCastException("Cannot convert [" + o.getClass() + "] to a Number");
+        }
+    }
+    
+    private static final MethodHandle DYNAMIC_CAST;
+    static {
+        final Lookup lookup = MethodHandles.lookup();
+        try {
+            DYNAMIC_CAST = lookup.findStatic(lookup.lookupClass(), 
+                                             "dynamicCast", 
+                                             MethodType.methodType(Object.class, Object.class, Object.class));
+        } catch (ReflectiveOperationException e) {
+            throw new AssertionError(e);
+        }
+    }
+
+    /** Looks up generic method, with a dynamic cast to the receiver's type. (compound assignment) */
+    public static MethodHandle lookupGenericWithCast(String name) {
+        MethodHandle generic = lookupGeneric(name);
+        // adapt dynamic cast to the generic method
+        MethodHandle cast = DYNAMIC_CAST.asType(MethodType.methodType(generic.type().returnType(), 
+                                                                      generic.type().returnType(),
+                                                                      generic.type().parameterType(0)));
+        // drop the RHS parameter
+        cast = MethodHandles.dropArguments(cast, 2, generic.type().parameterType(1));
+        // combine: f(x,y) -> g(f(x,y), x, y);
+        return MethodHandles.foldArguments(cast, generic);
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java
index cce2be71a12..554da280dda 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AdditionTests.java
@@ -431,4 +431,29 @@ public class AdditionTests extends ScriptTestCase {
         assertEquals(15D, exec("def x = 5.0; x += 10; return x;"));
         assertEquals(-5D, exec("def x = 5.0; x += -10; return x;"));
     }
+    
+    public void testDefCompoundAssignmentRHS() {
+        // byte
+        assertEquals((byte) 15, exec("byte x = 5; def y = 10; x += y; return x;"));
+        assertEquals((byte) -5, exec("byte x = 5; def y = -10; x += y; return x;"));
+
+        // short
+        assertEquals((short) 15, exec("short x = 5; def y = 10; x += y; return x;"));
+        assertEquals((short) -5, exec("short x = 5; def y = -10; x += y; return x;"));
+        // char
+        assertEquals((char) 15, exec("char x = 5; def y = 10; x += y; return x;"));
+        assertEquals((char) 5, exec("char x = 10; def y = -5; x += y; return x;"));
+        // int
+        assertEquals(15, exec("int x = 5; def y = 10; x += y; return x;"));
+        assertEquals(-5, exec("int x = 5; def y = -10; x += y; return x;"));
+        // long
+        assertEquals(15L, exec("long x = 5; def y = 10; x += y; return x;"));
+        assertEquals(-5L, exec("long x = 5; def y = -10; x += y; return x;"));
+        // float
+        assertEquals(15F, exec("float x = 5f; def y = 10; x += y; return x;"));
+        assertEquals(-5F, exec("float x = 5f; def y = -10; x += y; return x;"));
+        // double
+        assertEquals(15D, exec("double x = 5.0; def y = 10; x += y; return x;"));
+        assertEquals(-5D, exec("double x = 5.0; def y = -10; x += y; return x;"));
+    }
 }

From e3f34730aec80e70bda77321713e63e98a102154 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Wed, 15 Jun 2016 13:37:10 -0400
Subject: [PATCH 0875/1311] and compound assignment

---
 .../org/elasticsearch/painless/AndTests.java  | 46 +++++++++++++++----
 1 file changed, 38 insertions(+), 8 deletions(-)

diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java
index a5ffa3faa35..e7a764f3d0f 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AndTests.java
@@ -215,18 +215,10 @@ public class AndTests extends ScriptTestCase {
         assertEquals(false, exec("boolean x = true; x &= false; return x;"));
         assertEquals(false, exec("boolean x = false; x &= true; return x;"));
         assertEquals(false, exec("boolean x = false; x &= false; return x;"));
-        assertEquals(true, exec("def x = true; x &= true; return x;"));
-        assertEquals(false, exec("def x = true; x &= false; return x;"));
-        assertEquals(false, exec("def x = false; x &= true; return x;"));
-        assertEquals(false, exec("def x = false; x &= false; return x;"));
         assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] &= true; return x[0];"));
         assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] &= false; return x[0];"));
         assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] &= true; return x[0];"));
         assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] &= false; return x[0];"));
-        assertEquals(true, exec("def[] x = new def[1]; x[0] = true; x[0] &= true; return x[0];"));
-        assertEquals(false, exec("def[] x = new def[1]; x[0] = true; x[0] &= false; return x[0];"));
-        assertEquals(false, exec("def[] x = new def[1]; x[0] = false; x[0] &= true; return x[0];"));
-        assertEquals(false, exec("def[] x = new def[1]; x[0] = false; x[0] &= false; return x[0];"));
 
         // byte
         assertEquals((byte) (13 & 14), exec("byte x = 13; x &= 14; return x;"));
@@ -254,4 +246,42 @@ public class AndTests extends ScriptTestCase {
             exec("int x = 4; double y = 1; x &= y");
         });
     }
+    
+    public void testDefCompoundAssignment() {
+        // boolean
+        assertEquals(true, exec("def x = true; x &= true; return x;"));
+        assertEquals(false, exec("def x = true; x &= false; return x;"));
+        assertEquals(false, exec("def x = false; x &= true; return x;"));
+        assertEquals(false, exec("def x = false; x &= false; return x;"));
+        assertEquals(true, exec("def[] x = new def[1]; x[0] = true; x[0] &= true; return x[0];"));
+        assertEquals(false, exec("def[] x = new def[1]; x[0] = true; x[0] &= false; return x[0];"));
+        assertEquals(false, exec("def[] x = new def[1]; x[0] = false; x[0] &= true; return x[0];"));
+        assertEquals(false, exec("def[] x = new def[1]; x[0] = false; x[0] &= false; return x[0];"));
+
+        // byte
+        assertEquals((byte) (13 & 14), exec("def x = (byte)13; x &= 14; return x;"));
+        // short
+        assertEquals((short) (13 & 14), exec("def x = (short)13; x &= 14; return x;"));
+        // char
+        assertEquals((char) (13 & 14), exec("def x = (char)13; x &= 14; return x;"));
+        // int
+        assertEquals(13 & 14, exec("def x = 13; x &= 14; return x;"));
+        // long
+        assertEquals((long) (13 & 14), exec("def x = 13L; x &= 14; return x;"));
+    }
+    
+    public void testDefBogusCompoundAssignment() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = 4F; int y = 1; x &= y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = 4D; int y = 1; x &= y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("int x = 4; def y = 1F; x &= y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("int x = 4; def y = 1D; x &= y");
+        });
+    }
 }

From 8d3c411298e81695f1be712ccb0bd785592f348b Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Wed, 15 Jun 2016 19:48:38 +0200
Subject: [PATCH 0876/1311] Remove unneeded type adaption and add error message
 on violation

---
 .../main/java/org/elasticsearch/painless/DefBootstrap.java   | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index c40fc6a3234..7f79334b1e2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -99,6 +99,9 @@ public final class DefBootstrap {
 
         PIC(Lookup lookup, String name, MethodType type, int flavor, Object[] args) {
             super(type);
+            if (type.parameterType(0) != Object.class) {
+                throw new BootstrapMethodError("The receiver type (1st arg) of invokedynamic descriptor must be Object.");
+            }
             this.lookup = lookup;
             this.name = name;
             this.flavor = flavor;
@@ -178,8 +181,6 @@ public final class DefBootstrap {
             final MethodHandle target = lookup(flavor, name, receiver, callArgs).asType(type);
 
             MethodHandle test = CHECK_CLASS.bindTo(receiver);
-            test = test.asType(test.type().changeParameterType(0, type.parameterType(0)));
-
             MethodHandle guard = MethodHandles.guardWithTest(test, target, getTarget());
             
             depth++;

From a4036b80692bf9f30f8d546ae7977bb28811925d Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Wed, 15 Jun 2016 14:02:01 -0400
Subject: [PATCH 0877/1311] add compound assignment tests

---
 .../elasticsearch/painless/MethodWriter.java  |  3 +-
 .../elasticsearch/painless/DivisionTests.java | 28 +++++++++++
 .../painless/IncrementTests.java              | 31 +++++++++++++
 .../painless/MultiplicationTests.java         | 23 ++++++++++
 .../org/elasticsearch/painless/OrTests.java   | 46 +++++++++++++++----
 .../painless/RemainderTests.java              | 23 ++++++++++
 .../elasticsearch/painless/ShiftTests.java    | 15 ++++++
 .../elasticsearch/painless/StringTests.java   | 16 +++++++
 .../painless/SubtractionTests.java            | 24 ++++++++++
 .../org/elasticsearch/painless/XorTests.java  | 46 +++++++++++++++----
 10 files changed, 238 insertions(+), 17 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index 79377ce1903..3f3ad6b7781 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -263,7 +263,8 @@ public final class MethodWriter extends GeneratorAdapter {
     }
 
     /** Writes a dynamic binary instruction: returnType, lhs, and rhs can be different */
-    public void writeDynamicBinaryInstruction(Location location, Type returnType, Type lhs, Type rhs, Operation operation, boolean compoundAssignment) {
+    public void writeDynamicBinaryInstruction(Location location, Type returnType, Type lhs, Type rhs, 
+                                              Operation operation, boolean compoundAssignment) {
         org.objectweb.asm.Type methodType = org.objectweb.asm.Type.getMethodType(returnType.type, lhs.type, rhs.type);
         String descriptor = methodType.getDescriptor();
         
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java
index dbe14ea2fc9..e9880acebbc 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DivisionTests.java
@@ -359,6 +359,29 @@ public class DivisionTests extends ScriptTestCase {
         assertEquals(-5D, exec("double x = 5.0; x /= -1; return x;"));
     }
     
+    public void testDefCompoundAssignment() {
+        // byte
+        assertEquals((byte) 15, exec("def x = (byte)45; x /= 3; return x;"));
+        assertEquals((byte) -5, exec("def x = (byte)5; x /= -1; return x;"));
+        // short
+        assertEquals((short) 15, exec("def x = (short)45; x /= 3; return x;"));
+        assertEquals((short) -5, exec("def x = (short)5; x /= -1; return x;"));
+        // char
+        assertEquals((char) 15, exec("def x = (char)45; x /= 3; return x;"));
+        // int
+        assertEquals(15, exec("def x = 45; x /= 3; return x;"));
+        assertEquals(-5, exec("def x = 5; x /= -1; return x;"));
+        // long
+        assertEquals(15L, exec("def x = 45L; x /= 3; return x;"));
+        assertEquals(-5L, exec("def x = 5L; x /= -1; return x;"));
+        // float
+        assertEquals(15F, exec("def x = 45f; x /= 3; return x;"));
+        assertEquals(-5F, exec("def x = 5f; x /= -1; return x;"));
+        // double
+        assertEquals(15D, exec("def x = 45.0; x /= 3; return x;"));
+        assertEquals(-5D, exec("def x = 5.0; x /= -1; return x;"));
+    }
+    
     public void testCompoundAssignmentByZero() {
         // byte
         expectScriptThrows(ArithmeticException.class, () -> {
@@ -384,5 +407,10 @@ public class DivisionTests extends ScriptTestCase {
         expectScriptThrows(ArithmeticException.class, () -> {
             exec("long x = 1; x /= 0; return x;");
         });
+        
+        // def
+        expectScriptThrows(ArithmeticException.class, () -> {
+            exec("def x = 1; x /= 0; return x;");
+        });
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/IncrementTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/IncrementTests.java
index 13d94b9e9b3..de7fa173202 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/IncrementTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/IncrementTests.java
@@ -76,4 +76,35 @@ public class IncrementTests extends ScriptTestCase {
         assertEquals(1D, exec("double x = 0.0; return ++x;"));
         assertEquals(-1D, exec("double x = 0.0; return --x;"));
     }
+    
+    /** incrementing def values */
+    public void testIncrementDef() {
+        assertEquals((byte)0, exec("def x = (byte)0; return x++;"));
+        assertEquals((byte)0, exec("def x = (byte)0; return x--;"));
+        assertEquals((byte)1, exec("def x = (byte)0; return ++x;"));
+        assertEquals((byte)-1, exec("def x = (byte)0; return --x;"));
+        assertEquals((char)0, exec("def x = (char)0; return x++;"));
+        assertEquals((char)1, exec("def x = (char)1; return x--;"));
+        assertEquals((char)1, exec("def x = (char)0; return ++x;"));
+        assertEquals((short)0, exec("def x = (short)0; return x++;"));
+        assertEquals((short)0, exec("def x = (short)0; return x--;"));
+        assertEquals((short)1, exec("def x = (short)0; return ++x;"));
+        assertEquals((short)-1, exec("def x = (short)0; return --x;"));
+        assertEquals(0, exec("def x = 0; return x++;"));
+        assertEquals(0, exec("def x = 0; return x--;"));
+        assertEquals(1, exec("def x = 0; return ++x;"));
+        assertEquals(-1, exec("def x = 0; return --x;"));
+        assertEquals(0L, exec("def x = 0L; return x++;"));
+        assertEquals(0L, exec("def x = 0L; return x--;"));
+        assertEquals(1L, exec("def x = 0L; return ++x;"));
+        assertEquals(-1L, exec("def x = 0L; return --x;"));
+        assertEquals(0F, exec("def x = 0F; return x++;"));
+        assertEquals(0F, exec("def x = 0F; return x--;"));
+        assertEquals(1F, exec("def x = 0F; return ++x;"));
+        assertEquals(-1F, exec("def x = 0F; return --x;"));
+        assertEquals(0D, exec("def x = 0.0; return x++;"));
+        assertEquals(0D, exec("def x = 0.0; return x--;"));
+        assertEquals(1D, exec("def x = 0.0; return ++x;"));
+        assertEquals(-1D, exec("def x = 0.0; return --x;"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/MultiplicationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/MultiplicationTests.java
index 44612ee0c76..be9a929e5b2 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/MultiplicationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/MultiplicationTests.java
@@ -348,4 +348,27 @@ public class MultiplicationTests extends ScriptTestCase {
         assertEquals(15D, exec("double x = 5.0; x *= 3; return x;"));
         assertEquals(-5D, exec("double x = 5.0; x *= -1; return x;"));
     }
+    
+    public void testDefCompoundAssignment() {
+        // byte
+        assertEquals((byte) 15, exec("def x = (byte)5; x *= 3; return x;"));
+        assertEquals((byte) -5, exec("def x = (byte)5; x *= -1; return x;"));
+        // short
+        assertEquals((short) 15, exec("def x = (short)5; x *= 3; return x;"));
+        assertEquals((short) -5, exec("def x = (short)5; x *= -1; return x;"));
+        // char
+        assertEquals((char) 15, exec("def x = (char)5; x *= 3; return x;"));
+        // int
+        assertEquals(15, exec("def x = 5; x *= 3; return x;"));
+        assertEquals(-5, exec("def x = 5; x *= -1; return x;"));
+        // long
+        assertEquals(15L, exec("def x = 5L; x *= 3; return x;"));
+        assertEquals(-5L, exec("def x = 5L; x *= -1; return x;"));
+        // float
+        assertEquals(15F, exec("def x = 5f; x *= 3; return x;"));
+        assertEquals(-5F, exec("def x = 5f; x *= -1; return x;"));
+        // double
+        assertEquals(15D, exec("def x = 5.0; x *= 3; return x;"));
+        assertEquals(-5D, exec("def x = 5.0; x *= -1; return x;"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java
index 5863ec60b38..929d9486d88 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/OrTests.java
@@ -215,18 +215,10 @@ public class OrTests extends ScriptTestCase {
         assertEquals(true, exec("boolean x = true; x |= false; return x;"));
         assertEquals(true, exec("boolean x = false; x |= true; return x;"));
         assertEquals(false, exec("boolean x = false; x |= false; return x;"));
-        assertEquals(true, exec("def x = true; x |= true; return x;"));
-        assertEquals(true, exec("def x = true; x |= false; return x;"));
-        assertEquals(true, exec("def x = false; x |= true; return x;"));
-        assertEquals(false, exec("def x = false; x |= false; return x;"));
         assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] |= true; return x[0];"));
         assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] |= false; return x[0];"));
         assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] |= true; return x[0];"));
         assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] |= false; return x[0];"));
-        assertEquals(true, exec("def[] x = new def[1]; x[0] = true; x[0] |= true; return x[0];"));
-        assertEquals(true, exec("def[] x = new def[1]; x[0] = true; x[0] |= false; return x[0];"));
-        assertEquals(true, exec("def[] x = new def[1]; x[0] = false; x[0] |= true; return x[0];"));
-        assertEquals(false, exec("def[] x = new def[1]; x[0] = false; x[0] |= false; return x[0];"));
 
         // byte
         assertEquals((byte) (13 | 14), exec("byte x = 13; x |= 14; return x;"));
@@ -254,4 +246,42 @@ public class OrTests extends ScriptTestCase {
             exec("int x = 4; double y = 1; x |= y");
         });
     }
+    
+    public void testDefCompoundAssignment() {
+        // boolean
+        assertEquals(true, exec("def x = true; x |= true; return x;"));
+        assertEquals(true, exec("def x = true; x |= false; return x;"));
+        assertEquals(true, exec("def x = false; x |= true; return x;"));
+        assertEquals(false, exec("def x = false; x |= false; return x;"));
+        assertEquals(true, exec("def[] x = new def[1]; x[0] = true; x[0] |= true; return x[0];"));
+        assertEquals(true, exec("def[] x = new def[1]; x[0] = true; x[0] |= false; return x[0];"));
+        assertEquals(true, exec("def[] x = new def[1]; x[0] = false; x[0] |= true; return x[0];"));
+        assertEquals(false, exec("def[] x = new def[1]; x[0] = false; x[0] |= false; return x[0];"));
+
+        // byte
+        assertEquals((byte) (13 | 14), exec("def x = (byte)13; x |= 14; return x;"));
+        // short
+        assertEquals((short) (13 | 14), exec("def x = (short)13; x |= 14; return x;"));
+        // char
+        assertEquals((char) (13 | 14), exec("def x = (char)13; x |= 14; return x;"));
+        // int
+        assertEquals(13 | 14, exec("def x = 13; x |= 14; return x;"));
+        // long
+        assertEquals((long) (13 | 14), exec("def x = 13L; x |= 14; return x;"));
+    }
+    
+    public void testDefBogusCompoundAssignment() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = 4F; int y = 1; x |= y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = 4D; int y = 1; x |= y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = 4; float y = 1; x |= y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = 4; double y = 1; x |= y");
+        });
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java
index b42a3ca1d58..7380923c746 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RemainderTests.java
@@ -358,4 +358,27 @@ public class RemainderTests extends ScriptTestCase {
         assertEquals(3D, exec("double x = 15.0; x %= 4; return x;"));
         assertEquals(-3D, exec("double x = -15.0; x %= 4; return x;"));
     }
+    
+    public void testDefCompoundAssignment() {
+        // byte
+        assertEquals((byte) 3, exec("def x = (byte)15; x %= 4; return x;"));
+        assertEquals((byte) -3, exec("def x = (byte) -15; x %= 4; return x;"));
+        // short
+        assertEquals((short) 3, exec("def x = (short)15; x %= 4; return x;"));
+        assertEquals((short) -3, exec("def x = (short) -15; x %= 4; return x;"));
+        // char
+        assertEquals((char) 3, exec("def x = (char) 15; x %= 4; return x;"));
+        // int
+        assertEquals(3, exec("def x = 15; x %= 4; return x;"));
+        assertEquals(-3, exec("def x = -15; x %= 4; return x;"));
+        // long
+        assertEquals(3L, exec("def x = 15L; x %= 4; return x;"));
+        assertEquals(-3L, exec("def x = -15L; x %= 4; return x;"));
+        // float
+        assertEquals(3F, exec("def x = 15F; x %= 4; return x;"));
+        assertEquals(-3F, exec("def x = -15F; x %= 4; return x;"));
+        // double
+        assertEquals(3D, exec("def x = 15.0; x %= 4; return x;"));
+        assertEquals(-3D, exec("def x = -15.0; x %= 4; return x;"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ShiftTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ShiftTests.java
index b6aa86a839d..e969802e27c 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ShiftTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ShiftTests.java
@@ -633,4 +633,19 @@ public class ShiftTests extends ScriptTestCase {
             exec("double x = 1D; x <<= 2L;");
         });
     }
+    
+    public void testBogusCompoundAssignmentDef() {
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1L; float y = 2; x <<= y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("def x = 1; double y = 2L; x <<= y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("float x = 1F; def y = 2; x <<= y;");
+        });
+        expectScriptThrows(ClassCastException.class, ()-> {
+            exec("double x = 1D; def y = 2L; x <<= y;");
+        });
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
index f42dd5f7c98..80b5675a6e1 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
@@ -205,4 +205,20 @@ public class StringTests extends ScriptTestCase {
             exec("def x = null; def y = null; return x + y");
         });
     }
+    
+    public void testDefCompoundAssignment() {
+        assertEquals("a" + (byte)2, exec("def x = 'a'; x += (byte)2; return x"));
+        assertEquals("a" + (short)2, exec("def x = 'a'; x  += (short)2; return x"));
+        assertEquals("a" + (char)2, exec("def x = 'a'; x += (char)2; return x"));
+        assertEquals("a" + 2, exec("def x = 'a'; x += (int)2; return x"));
+        assertEquals("a" + 2L, exec("def x = 'a'; x += (long)2; return x"));
+        assertEquals("a" + 2F, exec("def x = 'a'; x += (float)2; return x"));
+        assertEquals("a" + 2D, exec("def x = 'a'; x += (double)2; return x"));
+        assertEquals("ab", exec("def x = 'a'; def y = 'b'; x += y; return x"));
+        assertEquals("anull", exec("def x = 'a'; x += null; return x"));
+        assertEquals("nullb", exec("def x = null; x += 'b'; return x"));
+        expectScriptThrows(NullPointerException.class, () -> {
+            exec("def x = null; def y = null; x += y");
+        });
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/SubtractionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/SubtractionTests.java
index 447e164d205..e70322af3af 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/SubtractionTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/SubtractionTests.java
@@ -379,4 +379,28 @@ public class SubtractionTests extends ScriptTestCase {
         assertEquals(15D, exec("double x = 5.0; x -= -10; return x;"));
         assertEquals(-5D, exec("double x = 5.0; x -= 10; return x;"));
     }
+    
+    public void testDefCompoundAssignment() {
+        // byte
+        assertEquals((byte) 15, exec("def x = (byte)5; x -= -10; return x;"));
+        assertEquals((byte) -5, exec("def x = (byte)5; x -= 10; return x;"));
+        // short
+        assertEquals((short) 15, exec("def x = (short)5; x -= -10; return x;"));
+        assertEquals((short) -5, exec("def x = (short)5; x -= 10; return x;"));
+        // char
+        assertEquals((char) 15, exec("def x = (char)5; x -= -10; return x;"));
+        assertEquals((char) 5, exec("def x = (char)10; x -= 5; return x;"));
+        // int
+        assertEquals(15, exec("def x = 5; x -= -10; return x;"));
+        assertEquals(-5, exec("def x = 5; x -= 10; return x;"));
+        // long
+        assertEquals(15L, exec("def x = 5L; x -= -10; return x;"));
+        assertEquals(-5L, exec("def x = 5L; x -= 10; return x;"));
+        // float
+        assertEquals(15F, exec("def x = 5f; x -= -10; return x;"));
+        assertEquals(-5F, exec("def x = 5f; x -= 10; return x;"));
+        // double
+        assertEquals(15D, exec("def x = 5.0; x -= -10; return x;"));
+        assertEquals(-5D, exec("def x = 5.0; x -= 10; return x;"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java
index eee49c309d3..bda23aef01f 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/XorTests.java
@@ -229,18 +229,10 @@ public class XorTests extends ScriptTestCase {
         assertEquals(true, exec("boolean x = true; x ^= false; return x;"));
         assertEquals(true, exec("boolean x = false; x ^= true; return x;"));
         assertEquals(false, exec("boolean x = false; x ^= false; return x;"));
-        assertEquals(false, exec("def x = true; x ^= true; return x;"));
-        assertEquals(true, exec("def x = true; x ^= false; return x;"));
-        assertEquals(true, exec("def x = false; x ^= true; return x;"));
-        assertEquals(false, exec("def x = false; x ^= false; return x;"));
         assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] ^= true; return x[0];"));
         assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = true; x[0] ^= false; return x[0];"));
         assertEquals(true, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] ^= true; return x[0];"));
         assertEquals(false, exec("boolean[] x = new boolean[1]; x[0] = false; x[0] ^= false; return x[0];"));
-        assertEquals(false, exec("def[] x = new def[1]; x[0] = true; x[0] ^= true; return x[0];"));
-        assertEquals(true, exec("def[] x = new def[1]; x[0] = true; x[0] ^= false; return x[0];"));
-        assertEquals(true, exec("def[] x = new def[1]; x[0] = false; x[0] ^= true; return x[0];"));
-        assertEquals(false, exec("def[] x = new def[1]; x[0] = false; x[0] ^= false; return x[0];"));
 
         // byte
         assertEquals((byte) (13 ^ 14), exec("byte x = 13; x ^= 14; return x;"));
@@ -268,4 +260,42 @@ public class XorTests extends ScriptTestCase {
             exec("int x = 4; double y = 1; x ^= y");
         });
     }
+    
+    public void testCompoundAssignmentDef() {
+        // boolean
+        assertEquals(false, exec("def x = true; x ^= true; return x;"));
+        assertEquals(true, exec("def x = true; x ^= false; return x;"));
+        assertEquals(true, exec("def x = false; x ^= true; return x;"));
+        assertEquals(false, exec("def x = false; x ^= false; return x;"));
+        assertEquals(false, exec("def[] x = new def[1]; x[0] = true; x[0] ^= true; return x[0];"));
+        assertEquals(true, exec("def[] x = new def[1]; x[0] = true; x[0] ^= false; return x[0];"));
+        assertEquals(true, exec("def[] x = new def[1]; x[0] = false; x[0] ^= true; return x[0];"));
+        assertEquals(false, exec("def[] x = new def[1]; x[0] = false; x[0] ^= false; return x[0];"));
+
+        // byte
+        assertEquals((byte) (13 ^ 14), exec("def x = (byte)13; x ^= 14; return x;"));
+        // short
+        assertEquals((short) (13 ^ 14), exec("def x = (short)13; x ^= 14; return x;"));
+        // char
+        assertEquals((char) (13 ^ 14), exec("def x = (char)13; x ^= 14; return x;"));
+        // int
+        assertEquals(13 ^ 14, exec("def x = 13; x ^= 14; return x;"));
+        // long
+        assertEquals((long) (13 ^ 14), exec("def x = 13L; x ^= 14; return x;"));
+    }
+    
+    public void testDefBogusCompoundAssignment() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = 4F; int y = 1; x ^= y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = 4D; int y = 1; x ^= y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("int x = 4; def y = (float)1; x ^= y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("int x = 4; def y = (double)1; x ^= y");
+        });
+    }
 }

From 786c3b0fa8e3972c86d83b309b184e26a31d0746 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Wed, 15 Jun 2016 14:38:37 -0400
Subject: [PATCH 0878/1311] speed up compound assignments

---
 .../elasticsearch/painless/DefBootstrap.java  | 18 ++++++++++--------
 .../org/elasticsearch/painless/DefMath.java   | 19 ++++++++++++++++++-
 2 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index 980225c900a..e3a33ba6331 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -241,19 +241,21 @@ public final class DefBootstrap {
             switch(flavor) {
                 case UNARY_OPERATOR:
                 case SHIFT_OPERATOR:
-                    if ((flags & OPERATOR_COMPOUND_ASSIGNMENT) != 0) {
-                        return lookupGeneric(); // XXX: optimize better.
-                    }
                     // shifts are treated as unary, as java allows long arguments without a cast (but bits are ignored)
-                    return DefMath.lookupUnary(args[0].getClass(), name);
-                case BINARY_OPERATOR:
+                    MethodHandle unary = DefMath.lookupUnary(args[0].getClass(), name);
                     if ((flags & OPERATOR_COMPOUND_ASSIGNMENT) != 0) {
-                        return lookupGeneric(); // XXX: optimize better.
+                        unary = DefMath.cast(args[0].getClass(), unary);
                     }
+                    return unary;
+                case BINARY_OPERATOR:
                     if (args[0] == null || args[1] == null) {
-                        return lookupGeneric(); // can handle nulls, if supported
+                        return lookupGeneric(); // can handle nulls, casts if supported
                     } else {
-                        return DefMath.lookupBinary(args[0].getClass(), args[1].getClass(), name);
+                        MethodHandle binary = DefMath.lookupBinary(args[0].getClass(), args[1].getClass(), name);
+                        if ((flags & OPERATOR_COMPOUND_ASSIGNMENT) != 0) {
+                            binary = DefMath.cast(args[0].getClass(), binary);
+                        }
+                        return binary;
                     }
                 default: throw new AssertionError();
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java
index 665f5acbba3..f56256f6cb2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java
@@ -1133,7 +1133,7 @@ public class DefMath {
         return handle;
     }
     
-    /** Returns an appropriate method handle for a binary operator, based only promotion of the LHS and RHS arguments */
+    /** Returns an appropriate method handle for a binary operator, based on promotion of the LHS and RHS arguments */
     public static MethodHandle lookupBinary(Class<?> classA, Class<?> classB, String name) {
         MethodHandle handle = TYPE_OP_MAPPING.get(promote(promote(unbox(classA)), promote(unbox(classB)))).get(name);
         if (handle == null) {
@@ -1156,6 +1156,9 @@ public class DefMath {
     static Object dynamicCast(Object returnValue, Object lhs) {
         if (lhs != null) {
             Class<?> c = lhs.getClass();
+            if (c == returnValue.getClass()) {
+                return returnValue;
+            }
             if (c == Integer.class) {
                 return getNumber(returnValue).intValue();
             } else if (c == Long.class) {
@@ -1212,4 +1215,18 @@ public class DefMath {
         // combine: f(x,y) -> g(f(x,y), x, y);
         return MethodHandles.foldArguments(cast, generic);
     }
+    
+    /** Forces a cast to class A for target (only if types differ) */
+    public static MethodHandle cast(Class<?> classA, MethodHandle target) {
+        MethodType newType = MethodType.methodType(classA).unwrap();
+        MethodType targetType = MethodType.methodType(target.type().returnType()).unwrap();
+        
+        if (newType.returnType() == targetType.returnType()) {
+            return target; // no conversion
+        }
+        
+        // this is safe for our uses of it here only, because we change just the return value,
+        // the original method itself does all the type checks correctly.
+        return MethodHandles.explicitCastArguments(target, target.type().changeReturnType(newType.returnType()));
+    }
 }

From 8de90a66a1e161a664c43699a309c4162b420876 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Wed, 15 Jun 2016 11:43:38 -0700
Subject: [PATCH 0879/1311] Relax plugin id url heuristic, since java uses
 single slash instead of double

---
 .../java/org/elasticsearch/plugins/InstallPluginCommand.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
index aa224b3c7c3..643998f19c2 100644
--- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
@@ -237,7 +237,7 @@ class InstallPluginCommand extends SettingCommand {
         }
 
         // fall back to plain old URL
-        if (pluginId.contains("://") == false) {
+        if (pluginId.contains(":/") == false) {
             // definitely not a valid url, so assume it is a plugin name
             throw new UserError(ExitCodes.USAGE, "Unknown plugin " + pluginId);
         }

From eb9b4437b2020d9789a5d725d9eff14124268105 Mon Sep 17 00:00:00 2001
From: Areek Zillur <areek.zillur@elasticsearch.com>
Date: Wed, 15 Jun 2016 14:57:17 -0400
Subject: [PATCH 0880/1311] update docs

---
 .../reference/indices/rollover-index.asciidoc | 163 +++++++++++-------
 1 file changed, 96 insertions(+), 67 deletions(-)

diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc
index b5c845f7360..35ad327f181 100644
--- a/docs/reference/indices/rollover-index.asciidoc
+++ b/docs/reference/indices/rollover-index.asciidoc
@@ -1,84 +1,113 @@
 [[indices-rollover-index]]
 == Rollover Index
 
-The rollover index API allows to switch the index pointed to by an alias given some predicates.
-In order to rollover an index, the provided alias has to point to a single index. Upon satisfying
-any of the predicates, the alias is switched to point to the rollover index, if the rollover index
-already exists, the rollover fails.
+The rollover index API rolls an alias over to a new index when the existing
+index is considered to be too large or too old.
 
-This API is syntactic sugar for changing the index pointed to by an alias given some predicate.
+The API accepts a single alias name and a list of `conditions`.  The alias
+must point to a single index only.  If the index satisfies the specified
+conditions then a new index is created and the alias is switched to point to
+the new alias.
 
-The rollover API must be used against an alias that points to a single index:
 
 [source,js]
 --------------------------------------------------
-$ curl -XPUT 'http://localhost:9200/index1' -d '{
-    "aliases" : {
-        "index_alias": {}
-    }
-}'
+PUT /logs-0001 <1>
+{
+  "aliases": {
+    "logs_write": {}
+  }
+}
+
+POST logs_write/_rollover <2>
+{
+  "conditions": {
+    "max_age":   "7d",
+    "max_docs":  1000
+  }
+}
 --------------------------------------------------
+// CONSOLE
+<1> Creates an index called `logs-0001` with the alias `logs_write`.
+<2> If the index pointed to by `logs_write` was created 7 or more days ago, or
+    contains 1,000 or more documents, then the `logs-0002` index is created
+    and the `logs_write` alias is updated to point to `logs-0002`.
 
-To rollover `index_alias` to point to a new index `index2`:
-
-[source,js]
---------------------------------------------------
-$ curl -XPOST 'http://localhost:9200/index_alias/_rollover/index2' -d '{
-    "conditions" : {
-        "max_age": "7d", <1>
-        "max_docs": 1000 <2>
-    }
-}'
---------------------------------------------------
-<1> Sets a condition that the index has to be at least 7 days old
-<2> Sets a condition that the index has to have at least a 1000 documents
-
-The API call above switches the index pointed to by `index_alias` from `index1` to `index2`, if any
-of the conditions are met. `index2` is created (using matching <<indices-templates>> if available).
-The API call returns immediately if none of the conditions are met.
-
-The rollover API can be used without specifying the name for the new index. In this case, the API requires
-the old concrete index name to have `{index_prefix}-{num}` format, as rollover index name is generated
-following `{index_prefix}-{num+1}` format.
-
-The `_rollover` API is similar to <<indices-create-index>> and accepts `settings`, `mappings` and
-`aliases` to override the index create request for the rollover index.
-
-[source,js]
---------------------------------------------------
-$ curl -XPOST 'http://localhost:9200/index_alias/_rollover' -d '{
-    "conditions" : {
-        "max_age": "7d",
-        "max_docs": 1000
-    },
-    "settings": { <1>
-        "index.number_of_shards": 2
-    }
-}'
---------------------------------------------------
-<1> Set settings to override matching index template, `mappings` and `aliases` can also be provided.
-
-The rollover API supports `dry_run` mode, where request conditions can be checked without performing the
-actual rollover. The `dry_run` mode can be enabled by setting `dry_run=true` as a request parameter.
-
-An example response for the index rollover API:
+The above request might return the following response:
 
 [source,js]
 --------------------------------------------------
 {
-	"old_index": "index-1", <1>
-	"new_index": "index-2", <2>
-	"rolled_over": true, <3>
-	"dry_run": false, <4>
-	"conditions": { <5>
-	    "[max_age: 7d]": true,
-	    "[max_docs: 1000]": true
-	}
+  "old_index": "logs-0001",
+  "new_index": "logs-0002",
+  "rolled_over": true, <1>
+  "dry_run": false, <2>
+  "conditions": { <3>
+    "[max_age: 7d]": false,
+    "[max_docs: 1000]": true
+  }
 }
 --------------------------------------------------
-<1> name of the index the alias was pointing to
-<2> name of the index the alias currently points to
-<3> whether the alias switch was successful
-<4> whether the rollover was dry run
-<5> status of the evaluated request conditions
+ <1> Whether the index was rolled over.
+ <2> Whether the rollover was dry run.
+ <3> The result of each condition.
+
+[float]
+=== Naming the new index
+
+If the name of the existing index ends with `-` and a number -- e.g.
+`logs-0001` -- then the name of the new index will follow the same pattern,
+just incrementing the number (`logs-0002`).
+
+If the old name doesn't match this pattern then you must specify the name for
+the new index as follows:
+
+[source,js]
+--------------------------------------------------
+POST my_alias/_rollover/my_new_index_name
+{...}
+--------------------------------------------------
+
+[float]
+=== Defining the new index
+
+The settings, mappings, and aliases for the new index will be taken from any
+matching <<indices-templates,index templates>>. Additionally, the body of the
+request accepts `settings`, `mappings`, and `aliases` just like the
+<<indices-create-index,create index>> API, which will override any values
+set in matching index templates:
+
+[source,js]
+--------------------------------------------------
+POST /logs_writes/_rollover
+{
+  "conditions" : {
+    "max_age": "7d",
+    "max_docs": 1000
+  },
+  "settings": { <1>
+    "index.number_of_shards": 2 <1>
+  }
+}
+--------------------------------------------------
+// CONSOLE
+<1> Set settings to override matching index template, `mappings` and `aliases` can also be provided.
+
+ [float]
+ === Dry run
+
+The rollover API supports `dry_run` mode, where request conditions can be
+checked without performing the actual rollover:
+
+[source,js]
+--------------------------------------------------
+POST /logs_writes/_rollover?dry_run
+{
+  "conditions" : {
+    "max_age": "7d",
+    "max_docs": 1000
+  }
+}
+--------------------------------------------------
+// CONSOLE
 

From 2fc328e10a3cab8dae3fd0cdf6f42d0bca537696 Mon Sep 17 00:00:00 2001
From: Alexander Kazakov <alexshadow007@gmail.com>
Date: Wed, 15 Jun 2016 22:06:50 +0300
Subject: [PATCH 0881/1311] Docs: Fix typo in RestIT

---
 TESTING.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/TESTING.asciidoc b/TESTING.asciidoc
index ce81f97548f..44eda08020a 100644
--- a/TESTING.asciidoc
+++ b/TESTING.asciidoc
@@ -296,7 +296,7 @@ gradle :distribution:integ-test-zip:integTest \
   -Dtests.method="test {p0=cat.shards/10_basic/Help}"
 ---------------------------------------------------------------------------
 
-`RestNIT` are the executable test classes that runs all the
+`RestIT` are the executable test classes that runs all the
 yaml suites available within the `rest-api-spec` folder.
 
 The REST tests support all the options provided by the randomized runner, plus the following:

From 8cc848f31cc40faca0fe10253f7cd8c1f859451a Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 15 Jun 2016 14:15:14 -0400
Subject: [PATCH 0882/1311] Allow FieldStatsRequest to disable cache

---
 .../snapshots/status/SnapshotIndexShardStatus.java    |  2 +-
 .../cluster/snapshots/status/SnapshotIndexStatus.java |  2 +-
 .../action/fieldstats/FieldStatsRequest.java          | 11 +++++++++++
 .../action/fieldstats/FieldStatsRequestBuilder.java   |  5 +++++
 .../action/fieldstats/FieldStatsShardRequest.java     |  8 ++++++++
 .../action/fieldstats/FieldStatsShardResponse.java    |  1 -
 .../action/fieldstats/TransportFieldStatsAction.java  |  5 ++---
 .../support/broadcast/BroadcastShardResponse.java     |  4 ++--
 .../org/elasticsearch/indices/IndicesService.java     |  6 +++++-
 .../fieldstats/FieldStatsIntegrationIT.java           |  5 +++++
 10 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java
index 3d287eee9d1..a7cebca0aa6 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexShardStatus.java
@@ -141,7 +141,7 @@ public class SnapshotIndexShardStatus extends BroadcastShardResponse implements
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject(Integer.toString(getShardId()));
+        builder.startObject(Integer.toString(getShardId().getId()));
         builder.field(Fields.STAGE, getStage());
         stats.toXContent(builder, params);
         if (getNodeId() != null) {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexStatus.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexStatus.java
index 16c361d5ca6..8f2f7ed5f50 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexStatus.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/SnapshotIndexStatus.java
@@ -49,7 +49,7 @@ public class SnapshotIndexStatus implements Iterable<SnapshotIndexShardStatus>,
         Map<Integer, SnapshotIndexShardStatus> indexShards = new HashMap<>();
         stats = new SnapshotStats();
         for (SnapshotIndexShardStatus shard : shards) {
-            indexShards.put(shard.getShardId(), shard);
+            indexShards.put(shard.getShardId().getId(), shard);
             stats.add(shard.getStats());
         }
         shardsStats = new SnapshotShardsStats(shards);
diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequest.java b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequest.java
index a1e1110f622..af5fface7df 100644
--- a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequest.java
@@ -44,6 +44,7 @@ public class FieldStatsRequest extends BroadcastRequest<FieldStatsRequest> {
     private String[] fields = Strings.EMPTY_ARRAY;
     private String level = DEFAULT_LEVEL;
     private IndexConstraint[] indexConstraints = new IndexConstraint[0];
+    private boolean useCache = true;
 
     public String[] getFields() {
         return fields;
@@ -56,6 +57,14 @@ public class FieldStatsRequest extends BroadcastRequest<FieldStatsRequest> {
         this.fields = fields;
     }
 
+    public void setUseCache(boolean useCache) {
+        this.useCache = useCache;
+    }
+
+    public boolean shouldUseCache() {
+        return useCache;
+    }
+
     public IndexConstraint[] getIndexConstraints() {
         return indexConstraints;
     }
@@ -184,6 +193,7 @@ public class FieldStatsRequest extends BroadcastRequest<FieldStatsRequest> {
             indexConstraints[i] = new IndexConstraint(in);
         }
         level = in.readString();
+        useCache = in.readBoolean();
     }
 
     @Override
@@ -201,6 +211,7 @@ public class FieldStatsRequest extends BroadcastRequest<FieldStatsRequest> {
             }
         }
         out.writeString(level);
+        out.writeBoolean(useCache);
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequestBuilder.java
index c0c4d78de9b..1a3a8070e46 100644
--- a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequestBuilder.java
@@ -45,4 +45,9 @@ public class FieldStatsRequestBuilder extends
         request().level(level);
         return this;
     }
+
+    public FieldStatsRequestBuilder setUseCache(boolean useCache) {
+        request().setUseCache(useCache);
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardRequest.java b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardRequest.java
index 6705bd0e0b5..85a0d469541 100644
--- a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardRequest.java
@@ -34,6 +34,7 @@ import java.util.Set;
 public class FieldStatsShardRequest extends BroadcastShardRequest {
 
     private String[] fields;
+    private boolean useCache;
 
     public FieldStatsShardRequest() {
     }
@@ -46,22 +47,29 @@ public class FieldStatsShardRequest extends BroadcastShardRequest {
             fields.add(indexConstraint.getField());
         }
         this.fields = fields.toArray(new String[fields.size()]);
+        useCache = request.shouldUseCache();
     }
 
     public String[] getFields() {
         return fields;
     }
 
+    public boolean shouldUseCache() {
+        return useCache;
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         fields = in.readStringArray();
+        useCache = in.readBoolean();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeStringArrayNullable(fields);
+        out.writeBoolean(useCache);
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardResponse.java b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardResponse.java
index a3043d3ae35..7cc298729f0 100644
--- a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsShardResponse.java
@@ -46,7 +46,6 @@ public class FieldStatsShardResponse extends BroadcastShardResponse {
         return fieldStats;
     }
 
-
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsAction.java b/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsAction.java
index c475b704d1a..f70e5dda114 100644
--- a/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsAction.java
@@ -187,16 +187,15 @@ public class TransportFieldStatsAction extends
         ShardId shardId = request.shardId();
         Map<String, FieldStats<?>> fieldStats = new HashMap<>();
         IndexService indexServices = indicesService.indexServiceSafe(shardId.getIndex());
-        MapperService mapperService = indexServices.mapperService();
         IndexShard shard = indexServices.getShard(shardId.id());
         try (Engine.Searcher searcher = shard.acquireSearcher("fieldstats")) {
             // Resolve patterns and deduplicate
             Set<String> fieldNames = new HashSet<>();
             for (String field : request.getFields()) {
-                fieldNames.addAll(mapperService.simpleMatchToIndexNames(field));
+                fieldNames.addAll(shard.mapperService().simpleMatchToIndexNames(field));
             }
             for (String field : fieldNames) {
-                FieldStats<?> stats = indicesService.getFieldStats(shard, searcher, field);
+                FieldStats<?> stats = indicesService.getFieldStats(shard, searcher, field, request.shouldUseCache());
                 if (stats != null) {
                     fieldStats.put(field, stats);
                 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastShardResponse.java b/core/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastShardResponse.java
index ad79285051a..398a8d6c905 100644
--- a/core/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastShardResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/support/broadcast/BroadcastShardResponse.java
@@ -45,8 +45,8 @@ public abstract class BroadcastShardResponse extends TransportResponse {
         return this.shardId.getIndexName();
     }
 
-    public int getShardId() {
-        return this.shardId.id();
+    public ShardId getShardId() {
+        return this.shardId;
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index 155c60c7a68..92bcfbe64a2 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -1121,12 +1121,16 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService>
      * @param shard the shard to use with the cache key
      * @param searcher searcher to use to lookup the field stats
      * @param field the actual field
+     * @param useCache should this request use the cache?
      */
-    public FieldStats<?> getFieldStats(IndexShard shard, Engine.Searcher searcher, String field) throws Exception {
+    public FieldStats<?> getFieldStats(IndexShard shard, Engine.Searcher searcher, String field, boolean useCache) throws Exception {
         MappedFieldType fieldType = shard.mapperService().fullName(field);
         if (fieldType == null) {
             return null;
         }
+        if (useCache == false) {
+            return fieldType.stats(searcher.reader());
+        }
         BytesReference cacheKey = new BytesArray("fieldstats:" + field);
         BytesReference statsRef = cacheShardLevelResult(shard, searcher.getDirectoryReader(), cacheKey, out -> {
             out.writeOptionalWriteable(fieldType.stats(searcher.reader()));
diff --git a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java
index 86fb4cc5c53..454c973cc34 100644
--- a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java
@@ -487,6 +487,11 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
         assertEquals(200, fieldStats.getAllFieldStats().get("value").getDocCount());
         // Because we refreshed the index we don't have any more hits in the cache. This is read from the index.
         assertEquals(oldHitCount, indexStats.getHitCount());
+
+        // We can also turn off the cache entirely
+        fieldStats = client().prepareFieldStats().setFields("value").get();
+        assertEquals(200, fieldStats.getAllFieldStats().get("value").getDocCount());
+        assertEquals(oldHitCount, indexStats.getHitCount());
     }
 
     private void indexRange(String index, long from, long to) throws Exception {

From ab2a4a0d72f45cb59b7e9169445397bd745b0a64 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 15 Jun 2016 15:29:42 -0400
Subject: [PATCH 0883/1311] Fix exception on task not found

Silly protected method....
---
 .../main/java/org/elasticsearch/ResourceNotFoundException.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/ResourceNotFoundException.java b/core/src/main/java/org/elasticsearch/ResourceNotFoundException.java
index d38de2e3bc1..d408fdef033 100644
--- a/core/src/main/java/org/elasticsearch/ResourceNotFoundException.java
+++ b/core/src/main/java/org/elasticsearch/ResourceNotFoundException.java
@@ -32,7 +32,7 @@ public class ResourceNotFoundException extends ElasticsearchException {
         super(msg, args);
     }
 
-    protected ResourceNotFoundException(String msg, Throwable cause, Object... args) {
+    public ResourceNotFoundException(String msg, Throwable cause, Object... args) {
         super(msg, cause, args);
     }
 

From 5b11b153003d36d0afa25be6fdb16d080bc9d443 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Wed, 15 Jun 2016 21:46:58 +0200
Subject: [PATCH 0884/1311] fix typo

---
 .../src/main/java/org/elasticsearch/painless/DefBootstrap.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index 980225c900a..8761523db6c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -147,7 +147,7 @@ public final class DefBootstrap {
 
         /**
          * Called when a new type is encountered (or, when we have encountered more than {@code MAX_DEPTH}
-         * types at this call site and given up on caching using this fallback wand we switch to a
+         * types at this call site and given up on caching using this fallback and we switch to a
          * megamorphic cache using {@link ClassValue}).
          */
         @SuppressForbidden(reason = "slow path")

From ae6ce1c42e0b9693f329f243806806276b278760 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Wed, 15 Jun 2016 22:27:30 +0200
Subject: [PATCH 0885/1311] Small refactoring and cleanup to separate the
 polymorphic and megamorphic case

---
 .../elasticsearch/painless/DefBootstrap.java  | 69 +++++++++++--------
 1 file changed, 39 insertions(+), 30 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index 165c9856bb1..7c9ee430c66 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -144,6 +144,32 @@ public final class DefBootstrap {
                 default: throw new AssertionError();
             }
         }
+        
+        /**
+         * Creates the {@link MethodHandle} for the megamorphic call site
+         * using {@link ClassValue} and {@link MethodHandles#exactInvoker(MethodType)}:
+         * <p>
+         * TODO: Remove the variable args and just use {@code type()}!
+         */
+        private MethodHandle createMegamorphicHandle(final Object[] callArgs) throws Throwable {
+            final MethodType type = type();
+            final ClassValue<MethodHandle> megamorphicCache = new ClassValue<MethodHandle>() {
+                @Override
+                protected MethodHandle computeValue(Class<?> receiverType) {
+                    // it's too stupid that we cannot throw checked exceptions... (use rethrow puzzler):
+                    try {
+                        return lookup(flavor, name, receiverType, callArgs).asType(type);
+                    } catch (Throwable t) {
+                        Def.rethrow(t);
+                        throw new AssertionError();
+                    }
+                }
+            };
+            MethodHandle cacheLookup = MEGAMORPHIC_LOOKUP.bindTo(megamorphicCache);
+            cacheLookup = MethodHandles.dropArguments(cacheLookup,
+                    1, type.parameterList().subList(1, type.parameterCount()));
+            return MethodHandles.foldArguments(MethodHandles.exactInvoker(type), cacheLookup);            
+        }
 
         /**
          * Called when a new type is encountered (or, when we have encountered more than {@code MAX_DEPTH}
@@ -152,41 +178,24 @@ public final class DefBootstrap {
          */
         @SuppressForbidden(reason = "slow path")
         Object fallback(final Object[] callArgs) throws Throwable {
-            final Class<?> receiver = callArgs[0].getClass();
-            final MethodType type = type();
-            
             if (depth >= MAX_DEPTH) {
                 // we revert the whole cache and build a new megamorphic one
-                // using ClassValue and MethodHandles.exactInvoker():
-                final ClassValue<MethodHandle> megamorphicCache = new ClassValue<MethodHandle>() {
-                    @Override
-                    protected MethodHandle computeValue(Class<?> receiverType) {
-                        // it's too stupid that we cannot throw checked exceptions... (use rethrow puzzler):
-                        try {
-                            return lookup(flavor, name, receiverType, callArgs).asType(type);
-                        } catch (Throwable t) {
-                            Def.rethrow(t);
-                            throw new AssertionError();
-                        }
-                    }
-                };
-                MethodHandle cacheLookup = MEGAMORPHIC_LOOKUP.bindTo(megamorphicCache);
-                cacheLookup = MethodHandles.dropArguments(cacheLookup,
-                        1, type.parameterList().subList(1, type.parameterCount()));
-                MethodHandle target = MethodHandles.foldArguments(MethodHandles.exactInvoker(type), cacheLookup);
+                final MethodHandle target = this.createMegamorphicHandle(callArgs);
+                
                 setTarget(target);
                 return target.invokeWithArguments(callArgs);                    
+            } else {
+                final Class<?> receiver = callArgs[0].getClass();
+                final MethodHandle target = lookup(flavor, name, receiver, callArgs).asType(type());
+    
+                MethodHandle test = CHECK_CLASS.bindTo(receiver);
+                MethodHandle guard = MethodHandles.guardWithTest(test, target, getTarget());
+                
+                depth++;
+    
+                setTarget(guard);
+                return target.invokeWithArguments(callArgs);
             }
-            
-            final MethodHandle target = lookup(flavor, name, receiver, callArgs).asType(type);
-
-            MethodHandle test = CHECK_CLASS.bindTo(receiver);
-            MethodHandle guard = MethodHandles.guardWithTest(test, target, getTarget());
-            
-            depth++;
-
-            setTarget(guard);
-            return target.invokeWithArguments(callArgs);
         }
 
         private static final MethodHandle CHECK_CLASS;

From 32dd8134368cac5e0aa01624b7b7d16fec5426be Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@elastic.co>
Date: Wed, 15 Jun 2016 22:45:47 +0200
Subject: [PATCH 0886/1311] Fix typo in benchmark README

---
 benchmarks/README.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/benchmarks/README.md b/benchmarks/README.md
index afe60c89368..285ac484d3c 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -16,7 +16,6 @@ Just run `gradle :benchmarks:jmh` from the project root directory. It will build
 ## Running Microbenchmarks
 
 Benchmarks are always run via Gradle with `gradle :benchmarks:jmh`.
-```
  
 Running via an IDE is not supported as the results are meaningless (we have no control over the JVM running the benchmarks).
 

From 889d802115336cc70f6c7f36df336f1f7aec5bb0 Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@elastic.co>
Date: Wed, 15 Jun 2016 23:01:34 +0200
Subject: [PATCH 0887/1311] Refine wording in benchmark README and correct
 typos

---
 benchmarks/README.md | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/benchmarks/README.md b/benchmarks/README.md
index 285ac484d3c..03aaac7f3c4 100644
--- a/benchmarks/README.md
+++ b/benchmarks/README.md
@@ -6,8 +6,8 @@ This directory contains the microbenchmark suite of Elasticsearch. It relies on
 
 We do not want to microbenchmark everything but the kitchen sink and should typically rely on our 
 [macrobenchmarks](https://elasticsearch-benchmarks.elastic.co/app/kibana#/dashboard/Nightly-Benchmark-Overview) with 
-[Rally](http://github.com/elastic/rally). Microbenchmarks are intended for performance-critical components to spot performance 
-regressions. The microbenchmark suite is also handy for ad-hoc microbenchmarks but please remove them again before merging your PR.
+[Rally](http://github.com/elastic/rally). Microbenchmarks are intended to spot performance regressions in performance-critical components. 
+The microbenchmark suite is also handy for ad-hoc microbenchmarks but please remove them again before merging your PR.
 
 ## Getting Started
 
@@ -19,14 +19,14 @@ Benchmarks are always run via Gradle with `gradle :benchmarks:jmh`.
  
 Running via an IDE is not supported as the results are meaningless (we have no control over the JVM running the benchmarks).
 
-If you want to run a specific benchmark class, e.g. `org.elasticsearch.benchmark.MySampleBenchmark` or have any other special requirements 
-generate the uberjar with `gradle :benchmarks:jmhJar` and run the it directly with:
+If you want to run a specific benchmark class, e.g. `org.elasticsearch.benchmark.MySampleBenchmark` or have special requirements 
+generate the uberjar with `gradle :benchmarks:jmhJar` and run it directly with:
 
 ```
 java -jar benchmarks/build/distributions/elasticsearch-benchmarks-*.jar
 ```
 
-JMH supports lots of command line parameters. Add `-h` to the command above for more information about the available command line options.
+JMH supports lots of command line parameters. Add `-h` to the command above to see the available command line options.
 
 ## Adding Microbenchmarks
 
@@ -38,25 +38,25 @@ end the class name of a benchmark with `Benchmark`. To have JMH execute a benchm
 
 ## Tips and Best Practices
 
-To get realistic results, you should exercise care when running your benchmarks. Here are a few tips:
+To get realistic results, you should exercise care when running benchmarks. Here are a few tips:
 
 ### Do
 
-* Ensure that the system executing your microbenchmarks has as little load as possible and shutdown every process that can cause unnecessary 
+* Ensure that the system executing your microbenchmarks has as little load as possible. Shutdown every process that can cause unnecessary 
   runtime jitter. Watch the `Error` column in the benchmark results to see the run-to-run variance.
-* Ensure to run enough warmup iterations to get into a stable state. If you are unsure, don't change the defaults.
+* Ensure to run enough warmup iterations to get the benchmark into a stable state. If you are unsure, don't change the defaults.
 * Avoid CPU migrations by pinning your benchmarks to specific CPU cores. On Linux you can use `taskset`.
 * Fix the CPU frequency to avoid Turbo Boost from kicking in and skewing your results. On Linux you can use `cpufreq-set` and the 
   `performance` CPU governor.
-* Vary problem input size with `@Param`.
+* Vary the problem input size with `@Param`.
 * Use the integrated profilers in JMH to dig deeper if benchmark results to not match your hypotheses:
-** Run the generated uberjar directly and use `-prof gc` to check whether the garbage collector runs during a microbenchmarks and skews 
-   your results. If so, try to force a GC between runs (`-gc true`).
-** Use `-prof perf` or `-prof perfasm` (both only available on Linux) to see hotspots.
+    * Run the generated uberjar directly and use `-prof gc` to check whether the garbage collector runs during a microbenchmarks and skews 
+   your results. If so, try to force a GC between runs (`-gc true`) but watch out for the caveats.
+    * Use `-prof perf` or `-prof perfasm` (both only available on Linux) to see hotspots.
 * Have your benchmarks peer-reviewed.
 
 ### Don't
 
-* Blindly believe the numbers that your microbenchmark produces but verify them by measuring e.e. with `-prof perfasm`.
-* Run run more threads than your number of CPU cores (in case you run multi-threaded microbenchmarks).
+* Blindly believe the numbers that your microbenchmark produces but verify them by measuring e.g. with `-prof perfasm`.
+* Run more threads than your number of CPU cores (in case you run multi-threaded microbenchmarks).
 * Look only at the `Score` column and ignore `Error`. Instead take countermeasures to keep `Error` low / variance explainable.
\ No newline at end of file

From a26260fb72071cffed52cfbb5f5f5225594a29a0 Mon Sep 17 00:00:00 2001
From: Tal Levy <JubBoy333@gmail.com>
Date: Wed, 15 Jun 2016 14:57:18 -0700
Subject: [PATCH 0888/1311] new ScriptProcessor for Ingest (#18193)

add new ScriptProcessor for executing ES Scripts within pipelines
---
 .../elasticsearch/ingest/IngestService.java   |   5 +-
 .../elasticsearch/ingest/PipelineStore.java   |   5 +-
 .../ingest/ProcessorsRegistry.java            |  41 ++++--
 .../org/elasticsearch/node/NodeModule.java    |   5 +-
 .../node/service/NodeService.java             |   4 +-
 .../SimulatePipelineRequestParsingTests.java  |   6 +-
 .../elasticsearch/ingest/IngestCloseIT.java   |   2 +-
 .../ingest/PipelineStoreTests.java            |   9 +-
 .../ingest/ProcessorsRegistryTests.java       |  11 +-
 .../ingest/core/ConfigurationUtilsTests.java  |   8 +-
 .../ingest/core/PipelineFactoryTests.java     |   9 +-
 docs/reference/ingest/ingest-node.asciidoc    |  40 ++++++
 .../modules/scripting/using.asciidoc          |  13 +-
 .../ingest/IngestCommonPlugin.java            |  36 ++---
 .../elasticsearch/ingest/ScriptProcessor.java | 126 ++++++++++++++++++
 .../ingest/ForEachProcessorFactoryTests.java  |   8 +-
 .../ingest/ScriptProcessorFactoryTests.java   |  73 ++++++++++
 .../ingest/ScriptProcessorTests.java          |  66 +++++++++
 .../rest-api-spec/test/ingest/10_basic.yaml   |  11 +-
 .../ingest/geoip/IngestGeoIpPlugin.java       |   2 +-
 .../build.gradle                              |   5 +-
 ...tCase.java => AbstractScriptTestCase.java} |  15 ++-
 .../ingest/IngestDocumentMustacheIT.java      |   2 +-
 .../ingest/TemplateServiceIT.java             |   2 +-
 .../ingest/ValueSourceMustacheIT.java         |   2 +-
 ...eIT.java => IngestWithDependenciesIT.java} |   4 +-
 .../50_script_processor_using_painless.yaml   | 120 +++++++++++++++++
 .../test/resources/scripts/master.painless    |   1 +
 .../ingest/IngestTestPlugin.java              |   2 +-
 29 files changed, 559 insertions(+), 74 deletions(-)
 create mode 100644 modules/ingest-common/src/main/java/org/elasticsearch/ingest/ScriptProcessor.java
 create mode 100644 modules/ingest-common/src/test/java/org/elasticsearch/ingest/ScriptProcessorFactoryTests.java
 create mode 100644 modules/ingest-common/src/test/java/org/elasticsearch/ingest/ScriptProcessorTests.java
 rename qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/{AbstractMustacheTestCase.java => AbstractScriptTestCase.java} (78%)
 rename qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/{IngestWithMustacheIT.java => IngestWithDependenciesIT.java} (90%)
 create mode 100644 qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/50_script_processor_using_painless.yaml
 create mode 100644 qa/smoke-test-ingest-with-all-dependencies/src/test/resources/scripts/master.painless

diff --git a/core/src/main/java/org/elasticsearch/ingest/IngestService.java b/core/src/main/java/org/elasticsearch/ingest/IngestService.java
index b38f7470e39..60fb3c26eea 100644
--- a/core/src/main/java/org/elasticsearch/ingest/IngestService.java
+++ b/core/src/main/java/org/elasticsearch/ingest/IngestService.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.ingest;
 
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.ingest.core.IngestInfo;
 import org.elasticsearch.ingest.core.Processor;
@@ -55,8 +56,8 @@ public class IngestService implements Closeable {
         return pipelineExecutionService;
     }
 
-    public void setScriptService(ScriptService scriptService) {
-        pipelineStore.buildProcessorFactoryRegistry(processorsRegistryBuilder, scriptService);
+    public void buildProcessorsFactoryRegistry(ScriptService scriptService, ClusterService clusterService) {
+        pipelineStore.buildProcessorFactoryRegistry(processorsRegistryBuilder, scriptService, clusterService);
     }
 
     public IngestInfo info() {
diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
index 7e0dc1b4ffa..54cd716f0e8 100644
--- a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
+++ b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
@@ -66,9 +66,8 @@ public class PipelineStore extends AbstractComponent implements Closeable, Clust
         super(settings);
     }
 
-    public void buildProcessorFactoryRegistry(ProcessorsRegistry.Builder processorsRegistryBuilder, ScriptService scriptService) {
-        TemplateService templateService = new InternalTemplateService(scriptService);
-        this.processorRegistry = processorsRegistryBuilder.build(templateService);
+    public void buildProcessorFactoryRegistry(ProcessorsRegistry.Builder processorsRegistryBuilder, ScriptService scriptService, ClusterService clusterService) {
+        this.processorRegistry = processorsRegistryBuilder.build(scriptService, clusterService);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java b/core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java
index e831d70702e..88a021d2059 100644
--- a/core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java
+++ b/core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java
@@ -20,9 +20,10 @@
 package org.elasticsearch.ingest;
 
 import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.ingest.core.Processor;
-import org.elasticsearch.ingest.core.ProcessorInfo;
 import org.elasticsearch.ingest.core.TemplateService;
+import org.elasticsearch.script.ScriptService;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -31,21 +32,39 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.function.BiFunction;
+import java.util.function.Function;
 
 public final class ProcessorsRegistry implements Closeable {
 
     private final Map<String, Processor.Factory> processorFactories;
+    private final TemplateService templateService;
+    private final ScriptService scriptService;
+    private final ClusterService clusterService;
 
-    private ProcessorsRegistry(TemplateService templateService,
-                               Map<String, BiFunction<TemplateService, ProcessorsRegistry, Processor.Factory<?>>> providers) {
+    private ProcessorsRegistry(ScriptService scriptService, ClusterService clusterService,
+                               Map<String, Function<ProcessorsRegistry, Processor.Factory<?>>> providers) {
+        this.templateService = new InternalTemplateService(scriptService);
+        this.scriptService = scriptService;
+        this.clusterService = clusterService;
         Map<String, Processor.Factory> processorFactories = new HashMap<>();
-        for (Map.Entry<String, BiFunction<TemplateService, ProcessorsRegistry, Processor.Factory<?>>> entry : providers.entrySet()) {
-            processorFactories.put(entry.getKey(), entry.getValue().apply(templateService, this));
+        for (Map.Entry<String, Function<ProcessorsRegistry, Processor.Factory<?>>> entry : providers.entrySet()) {
+            processorFactories.put(entry.getKey(), entry.getValue().apply(this));
         }
         this.processorFactories = Collections.unmodifiableMap(processorFactories);
     }
 
+    public TemplateService getTemplateService() {
+        return templateService;
+    }
+
+    public ScriptService getScriptService() {
+        return scriptService;
+    }
+
+    public ClusterService getClusterService() {
+        return clusterService;
+    }
+
     public Processor.Factory getProcessorFactory(String name) {
         return processorFactories.get(name);
     }
@@ -68,20 +87,20 @@ public final class ProcessorsRegistry implements Closeable {
 
     public static final class Builder {
 
-        private final Map<String, BiFunction<TemplateService, ProcessorsRegistry, Processor.Factory<?>>> providers = new HashMap<>();
+        private final Map<String, Function<ProcessorsRegistry, Processor.Factory<?>>> providers = new HashMap<>();
 
         /**
          * Adds a processor factory under a specific name.
          */
-        public void registerProcessor(String name, BiFunction<TemplateService, ProcessorsRegistry, Processor.Factory<?>> provider) {
-            BiFunction<TemplateService, ProcessorsRegistry, Processor.Factory<?>> previous = this.providers.putIfAbsent(name, provider);
+        public void registerProcessor(String name, Function<ProcessorsRegistry, Processor.Factory<?>> provider) {
+            Function<ProcessorsRegistry, Processor.Factory<?>> previous = this.providers.putIfAbsent(name, provider);
             if (previous != null) {
                 throw new IllegalArgumentException("Processor factory already registered for name [" + name + "]");
             }
         }
 
-        public ProcessorsRegistry build(TemplateService templateService) {
-            return new ProcessorsRegistry(templateService, providers);
+        public ProcessorsRegistry build(ScriptService scriptService, ClusterService clusterService) {
+            return new ProcessorsRegistry(scriptService, clusterService, providers);
         }
 
     }
diff --git a/core/src/main/java/org/elasticsearch/node/NodeModule.java b/core/src/main/java/org/elasticsearch/node/NodeModule.java
index a9696593ec4..c5d087c0ef2 100644
--- a/core/src/main/java/org/elasticsearch/node/NodeModule.java
+++ b/core/src/main/java/org/elasticsearch/node/NodeModule.java
@@ -23,11 +23,10 @@ import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.ingest.ProcessorsRegistry;
 import org.elasticsearch.ingest.core.Processor;
-import org.elasticsearch.ingest.core.TemplateService;
 import org.elasticsearch.monitor.MonitorService;
 import org.elasticsearch.node.service.NodeService;
 
-import java.util.function.BiFunction;
+import java.util.function.Function;
 
 /**
  *
@@ -71,7 +70,7 @@ public class NodeModule extends AbstractModule {
     /**
      * Adds a processor factory under a specific type name.
      */
-    public void registerProcessor(String type, BiFunction<TemplateService, ProcessorsRegistry, Processor.Factory<?>> provider) {
+    public void registerProcessor(String type, Function<ProcessorsRegistry, Processor.Factory<?>> provider) {
         processorsRegistryBuilder.registerProcessor(type, provider);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/node/service/NodeService.java b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
index cf2bb0f9bb7..1cbdfe12d2c 100644
--- a/core/src/main/java/org/elasticsearch/node/service/NodeService.java
+++ b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
@@ -62,6 +62,7 @@ public class NodeService extends AbstractComponent implements Closeable {
     private final CircuitBreakerService circuitBreakerService;
     private final IngestService ingestService;
     private final SettingsFilter settingsFilter;
+    private ClusterService clusterService;
     private ScriptService scriptService;
 
     @Nullable
@@ -87,6 +88,7 @@ public class NodeService extends AbstractComponent implements Closeable {
         this.version = version;
         this.pluginService = pluginService;
         this.circuitBreakerService = circuitBreakerService;
+        this.clusterService = clusterService;
         this.ingestService = new IngestService(settings, threadPool, processorsRegistryBuilder);
         this.settingsFilter = settingsFilter;
         clusterService.add(ingestService.getPipelineStore());
@@ -97,7 +99,7 @@ public class NodeService extends AbstractComponent implements Closeable {
     @Inject(optional = true)
     public void setScriptService(ScriptService scriptService) {
         this.scriptService = scriptService;
-        this.ingestService.setScriptService(scriptService);
+        this.ingestService.buildProcessorsFactoryRegistry(scriptService, clusterService);
     }
 
     public void setHttpServer(@Nullable HttpServer httpServer) {
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
index b7cd5ef0b92..4b50f69e4c8 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.ingest;
 
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.ingest.PipelineStore;
 import org.elasticsearch.ingest.ProcessorsRegistry;
 import org.elasticsearch.ingest.TestProcessor;
@@ -27,6 +28,7 @@ import org.elasticsearch.ingest.core.CompoundProcessor;
 import org.elasticsearch.ingest.core.IngestDocument;
 import org.elasticsearch.ingest.core.Pipeline;
 import org.elasticsearch.ingest.core.Processor;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
@@ -58,8 +60,8 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
         CompoundProcessor pipelineCompoundProcessor = new CompoundProcessor(processor);
         Pipeline pipeline = new Pipeline(SIMULATED_PIPELINE_ID, null, pipelineCompoundProcessor);
         ProcessorsRegistry.Builder processorRegistryBuilder = new ProcessorsRegistry.Builder();
-        processorRegistryBuilder.registerProcessor("mock_processor", ((templateService, registry) -> mock(Processor.Factory.class)));
-        ProcessorsRegistry processorRegistry = processorRegistryBuilder.build(TestTemplateService.instance());
+        processorRegistryBuilder.registerProcessor("mock_processor", ((registry) -> mock(Processor.Factory.class)));
+        ProcessorsRegistry processorRegistry = processorRegistryBuilder.build(mock(ScriptService.class), mock(ClusterService.class));
         store = mock(PipelineStore.class);
         when(store.get(SIMULATED_PIPELINE_ID)).thenReturn(pipeline);
         when(store.getProcessorRegistry()).thenReturn(processorRegistry);
diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestCloseIT.java b/core/src/test/java/org/elasticsearch/ingest/IngestCloseIT.java
index 502b2a9bf02..5e0706388ce 100644
--- a/core/src/test/java/org/elasticsearch/ingest/IngestCloseIT.java
+++ b/core/src/test/java/org/elasticsearch/ingest/IngestCloseIT.java
@@ -65,7 +65,7 @@ public class IngestCloseIT extends ESSingleNodeTestCase {
         }
 
         public void onModule(NodeModule nodeModule) {
-            nodeModule.registerProcessor("test", (templateService, registry) -> new Factory());
+            nodeModule.registerProcessor("test", (registry) -> new Factory());
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
index 26dc1957757..33992af0f67 100644
--- a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
@@ -28,6 +28,7 @@ import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
@@ -36,6 +37,7 @@ import org.elasticsearch.ingest.core.IngestInfo;
 import org.elasticsearch.ingest.core.Pipeline;
 import org.elasticsearch.ingest.core.Processor;
 import org.elasticsearch.ingest.core.ProcessorInfo;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
@@ -51,6 +53,7 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
 
 public class PipelineStoreTests extends ESTestCase {
 
@@ -60,7 +63,7 @@ public class PipelineStoreTests extends ESTestCase {
     public void init() throws Exception {
         store = new PipelineStore(Settings.EMPTY);
         ProcessorsRegistry.Builder registryBuilder = new ProcessorsRegistry.Builder();
-        registryBuilder.registerProcessor("set", (templateService, registry) -> config -> {
+        registryBuilder.registerProcessor("set", (registry) -> config -> {
             String field = (String) config.remove("field");
             String value = (String) config.remove("value");
             return new Processor() {
@@ -80,7 +83,7 @@ public class PipelineStoreTests extends ESTestCase {
                 }
             };
         });
-        registryBuilder.registerProcessor("remove", (templateService, registry) -> config -> {
+        registryBuilder.registerProcessor("remove", (registry) -> config -> {
             String field = (String) config.remove("field");
             return new Processor() {
                 @Override
@@ -99,7 +102,7 @@ public class PipelineStoreTests extends ESTestCase {
                 }
             };
         });
-        store.buildProcessorFactoryRegistry(registryBuilder, null);
+        store.buildProcessorFactoryRegistry(registryBuilder, mock(ScriptService.class), mock(ClusterService.class));
     }
 
     public void testUpdatePipelines() {
diff --git a/core/src/test/java/org/elasticsearch/ingest/ProcessorsRegistryTests.java b/core/src/test/java/org/elasticsearch/ingest/ProcessorsRegistryTests.java
index 26edb54e6fa..41701102584 100644
--- a/core/src/test/java/org/elasticsearch/ingest/ProcessorsRegistryTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/ProcessorsRegistryTests.java
@@ -19,28 +19,31 @@
 
 package org.elasticsearch.ingest;
 
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.sameInstance;
+import static org.mockito.Mockito.mock;
 
 public class ProcessorsRegistryTests extends ESTestCase {
 
     public void testBuildProcessorRegistry() {
         ProcessorsRegistry.Builder builder = new ProcessorsRegistry.Builder();
         TestProcessor.Factory factory1 = new TestProcessor.Factory();
-        builder.registerProcessor("1", (templateService, registry) -> factory1);
+        builder.registerProcessor("1", (registry) -> factory1);
         TestProcessor.Factory factory2 = new TestProcessor.Factory();
-        builder.registerProcessor("2", (templateService, registry) -> factory2);
+        builder.registerProcessor("2", (registry) -> factory2);
         TestProcessor.Factory factory3 = new TestProcessor.Factory();
         try {
-            builder.registerProcessor("1", (templateService, registry) -> factory3);
+            builder.registerProcessor("1", (registry) -> factory3);
             fail("addProcessor should have failed");
         } catch(IllegalArgumentException e) {
             assertThat(e.getMessage(), equalTo("Processor factory already registered for name [1]"));
         }
 
-        ProcessorsRegistry registry = builder.build(TestTemplateService.instance());
+        ProcessorsRegistry registry = builder.build(mock(ScriptService.class), mock(ClusterService.class));
         assertThat(registry.getProcessorFactories().size(), equalTo(2));
         assertThat(registry.getProcessorFactory("1"), sameInstance(factory1));
         assertThat(registry.getProcessorFactory("2"), sameInstance(factory2));
diff --git a/core/src/test/java/org/elasticsearch/ingest/core/ConfigurationUtilsTests.java b/core/src/test/java/org/elasticsearch/ingest/core/ConfigurationUtilsTests.java
index 32c2dd24881..ff3670b22b2 100644
--- a/core/src/test/java/org/elasticsearch/ingest/core/ConfigurationUtilsTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/core/ConfigurationUtilsTests.java
@@ -20,9 +20,13 @@
 package org.elasticsearch.ingest.core;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.ingest.ProcessorsRegistry;
 import org.elasticsearch.ingest.TestTemplateService;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 import org.junit.Before;
 
 import java.util.ArrayList;
@@ -96,8 +100,8 @@ public class ConfigurationUtilsTests extends ESTestCase {
     public void testReadProcessors() throws Exception {
         Processor processor = mock(Processor.class);
         ProcessorsRegistry.Builder builder = new ProcessorsRegistry.Builder();
-        builder.registerProcessor("test_processor", (templateService, registry) -> config -> processor);
-        ProcessorsRegistry registry = builder.build(TestTemplateService.instance());
+        builder.registerProcessor("test_processor", (registry) -> config -> processor);
+        ProcessorsRegistry registry = builder.build(mock(ScriptService.class), mock(ClusterService.class));
 
 
         List<Map<String, Map<String, Object>>> config = new ArrayList<>();
diff --git a/core/src/test/java/org/elasticsearch/ingest/core/PipelineFactoryTests.java b/core/src/test/java/org/elasticsearch/ingest/core/PipelineFactoryTests.java
index b6840b7a554..dd12d020247 100644
--- a/core/src/test/java/org/elasticsearch/ingest/core/PipelineFactoryTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/core/PipelineFactoryTests.java
@@ -20,10 +20,14 @@
 package org.elasticsearch.ingest.core;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.ingest.ProcessorsRegistry;
 import org.elasticsearch.ingest.TestProcessor;
 import org.elasticsearch.ingest.TestTemplateService;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -34,6 +38,7 @@ import java.util.Map;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
+import static org.mockito.Mockito.mock;
 
 public class PipelineFactoryTests extends ESTestCase {
 
@@ -152,8 +157,8 @@ public class PipelineFactoryTests extends ESTestCase {
     private ProcessorsRegistry createProcessorRegistry(Map<String, Processor.Factory> processorRegistry) {
         ProcessorsRegistry.Builder builder = new ProcessorsRegistry.Builder();
         for (Map.Entry<String, Processor.Factory> entry : processorRegistry.entrySet()) {
-            builder.registerProcessor(entry.getKey(), ((templateService, registry) -> entry.getValue()));
+            builder.registerProcessor(entry.getKey(), ((registry) -> entry.getValue()));
         }
-        return builder.build(TestTemplateService.instance());
+        return builder.build(mock(ScriptService.class), mock(ClusterService.class));
     }
 }
diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc
index ce50f27451e..c8a5a05002d 100644
--- a/docs/reference/ingest/ingest-node.asciidoc
+++ b/docs/reference/ingest/ingest-node.asciidoc
@@ -1258,6 +1258,46 @@ Renames an existing field. If the field doesn't exist or the new name is already
 }
 --------------------------------------------------
 
+[[script-processor]]
+=== Script Processor
+
+Allows inline, stored, and file scripts to be executed within ingest pipelines.
+
+See <<modules-scripting-using, How to use scripts>> to learn more about writing scripts. The Script Processor
+leverages caching of compiled scripts for improved performance. Since the
+script specified within the processor is potentially re-compiled per document, it is important
+to understand how script caching works. To learn more about
+caching see <<modules-scripting-using-caching, Script Caching>>.
+
+[[script-options]]
+.Script Options
+[options="header"]
+|======
+| Name                   | Required  | Default | Description
+| `field`                | yes       | -       | The field to set
+| `lang`                 | no        | -       | The scripting language
+| `file`                 | no        | -       | The script file to refer to
+| `id`                   | no        | -       | The stored script id to refer to
+| `inline`               | no        | -       | An inline script to be executed
+|======
+
+You can access the current ingest document from within the script context by using the `ctx` variable.
+
+The following example sets a new field called `field_a_plus_b` to be the sum of two existing
+numeric fields `field_a` and `field_b`:
+
+[source,js]
+--------------------------------------------------
+{
+  "script": {
+    "field": "field_a_plus_b",
+    "lang": "painless",
+    "inline": "return ctx.field_a + ctx.field_b"
+  }
+}
+--------------------------------------------------
+
+
 [[set-processor]]
 === Set Processor
 Sets one field and associates it with the specified value. If the field already exists,
diff --git a/docs/reference/modules/scripting/using.asciidoc b/docs/reference/modules/scripting/using.asciidoc
index f0ede243b78..90b8a12c3bf 100644
--- a/docs/reference/modules/scripting/using.asciidoc
+++ b/docs/reference/modules/scripting/using.asciidoc
@@ -43,7 +43,6 @@ GET my_index/_search
 -------------------------------------
 // CONSOLE
 
-
 [float]
 === Script Parameters
 
@@ -236,6 +235,18 @@ DELETE _scripts/groovy/calculate-score
 // CONSOLE
 // TEST[continued]
 
+[float]
+[[modules-scripting-using-caching]]
+=== Script Caching
+
+All scripts are cached by default so that they only need to be recompiled
+when updates occur. File scripts keep a static cache and will always reside
+in memory. Both inline and stored scripts are stored in a cache that can evict
+residing scripts. By default, scripts do not have a time-based expiration, but
+you can change this behavior by using the `script.cache.expire` setting.
+You can configure the size of this cache by using the `script.cache.max_size` setting.
+By default, the cache size is `100`.
+
 NOTE: The size of stored scripts is limited to 65,535 bytes. This can be
 changed by setting `script.max_size_in_bytes` setting to increase that soft
 limit, but if scripts are really large then alternatives like
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/IngestCommonPlugin.java
index 8e5a821b8a5..7e70c3cba49 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/IngestCommonPlugin.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/IngestCommonPlugin.java
@@ -52,23 +52,25 @@ public class IngestCommonPlugin extends Plugin {
     }
 
     public void onModule(NodeModule nodeModule) {
-        nodeModule.registerProcessor(DateProcessor.TYPE, (templateService, registry) -> new DateProcessor.Factory());
-        nodeModule.registerProcessor(SetProcessor.TYPE, (templateService, registry) -> new SetProcessor.Factory(templateService));
-        nodeModule.registerProcessor(AppendProcessor.TYPE, (templateService, registry) -> new AppendProcessor.Factory(templateService));
-        nodeModule.registerProcessor(RenameProcessor.TYPE, (templateService, registry) -> new RenameProcessor.Factory());
-        nodeModule.registerProcessor(RemoveProcessor.TYPE, (templateService, registry) -> new RemoveProcessor.Factory(templateService));
-        nodeModule.registerProcessor(SplitProcessor.TYPE, (templateService, registry) -> new SplitProcessor.Factory());
-        nodeModule.registerProcessor(JoinProcessor.TYPE, (templateService, registry) -> new JoinProcessor.Factory());
-        nodeModule.registerProcessor(UppercaseProcessor.TYPE, (templateService, registry) -> new UppercaseProcessor.Factory());
-        nodeModule.registerProcessor(LowercaseProcessor.TYPE, (templateService, registry) -> new LowercaseProcessor.Factory());
-        nodeModule.registerProcessor(TrimProcessor.TYPE, (templateService, registry) -> new TrimProcessor.Factory());
-        nodeModule.registerProcessor(ConvertProcessor.TYPE, (templateService, registry) -> new ConvertProcessor.Factory());
-        nodeModule.registerProcessor(GsubProcessor.TYPE, (templateService, registry) -> new GsubProcessor.Factory());
-        nodeModule.registerProcessor(FailProcessor.TYPE, (templateService, registry) -> new FailProcessor.Factory(templateService));
-        nodeModule.registerProcessor(ForEachProcessor.TYPE, (templateService, registry) -> new ForEachProcessor.Factory(registry));
-        nodeModule.registerProcessor(DateIndexNameProcessor.TYPE, (templateService, registry) -> new DateIndexNameProcessor.Factory());
-        nodeModule.registerProcessor(SortProcessor.TYPE, (templateService, registry) -> new SortProcessor.Factory());
-        nodeModule.registerProcessor(GrokProcessor.TYPE, (templateService, registry) -> new GrokProcessor.Factory(builtinPatterns));
+        nodeModule.registerProcessor(DateProcessor.TYPE, (registry) -> new DateProcessor.Factory());
+        nodeModule.registerProcessor(SetProcessor.TYPE, (registry) -> new SetProcessor.Factory(registry.getTemplateService()));
+        nodeModule.registerProcessor(AppendProcessor.TYPE, (registry) -> new AppendProcessor.Factory(registry.getTemplateService()));
+        nodeModule.registerProcessor(RenameProcessor.TYPE, (registry) -> new RenameProcessor.Factory());
+        nodeModule.registerProcessor(RemoveProcessor.TYPE, (registry) -> new RemoveProcessor.Factory(registry.getTemplateService()));
+        nodeModule.registerProcessor(SplitProcessor.TYPE, (registry) -> new SplitProcessor.Factory());
+        nodeModule.registerProcessor(JoinProcessor.TYPE, (registry) -> new JoinProcessor.Factory());
+        nodeModule.registerProcessor(UppercaseProcessor.TYPE, (registry) -> new UppercaseProcessor.Factory());
+        nodeModule.registerProcessor(LowercaseProcessor.TYPE, (registry) -> new LowercaseProcessor.Factory());
+        nodeModule.registerProcessor(TrimProcessor.TYPE, (registry) -> new TrimProcessor.Factory());
+        nodeModule.registerProcessor(ConvertProcessor.TYPE, (registry) -> new ConvertProcessor.Factory());
+        nodeModule.registerProcessor(GsubProcessor.TYPE, (registry) -> new GsubProcessor.Factory());
+        nodeModule.registerProcessor(FailProcessor.TYPE, (registry) -> new FailProcessor.Factory(registry.getTemplateService()));
+        nodeModule.registerProcessor(ForEachProcessor.TYPE, (registry) -> new ForEachProcessor.Factory(registry));
+        nodeModule.registerProcessor(DateIndexNameProcessor.TYPE, (registry) -> new DateIndexNameProcessor.Factory());
+        nodeModule.registerProcessor(SortProcessor.TYPE, (registry) -> new SortProcessor.Factory());
+        nodeModule.registerProcessor(GrokProcessor.TYPE, (registry) -> new GrokProcessor.Factory(builtinPatterns));
+        nodeModule.registerProcessor(ScriptProcessor.TYPE, (registry) ->
+            new ScriptProcessor.Factory(registry.getScriptService(), registry.getClusterService()));
     }
 
     // Code for loading built-in grok patterns packaged with the jar file:
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/ScriptProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/ScriptProcessor.java
new file mode 100644
index 00000000000..9051ffe408e
--- /dev/null
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/ScriptProcessor.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest;
+
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.ingest.core.AbstractProcessor;
+import org.elasticsearch.ingest.core.AbstractProcessorFactory;
+import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.script.CompiledScript;
+import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.script.ScriptService;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static java.util.Collections.emptyMap;
+import static org.elasticsearch.common.Strings.hasLength;
+import static org.elasticsearch.ingest.core.ConfigurationUtils.newConfigurationException;
+import static org.elasticsearch.ingest.core.ConfigurationUtils.readOptionalStringProperty;
+import static org.elasticsearch.ingest.core.ConfigurationUtils.readStringProperty;
+import static org.elasticsearch.script.ScriptService.ScriptType.FILE;
+import static org.elasticsearch.script.ScriptService.ScriptType.INLINE;
+import static org.elasticsearch.script.ScriptService.ScriptType.STORED;
+
+/**
+ * Processor that adds new fields with their corresponding values. If the field is already present, its value
+ * will be replaced with the provided one.
+ */
+public final class ScriptProcessor extends AbstractProcessor {
+
+    public static final String TYPE = "script";
+
+    private final Script script;
+    private final ScriptService scriptService;
+    private final ClusterService clusterService;
+    private final String field;
+
+    ScriptProcessor(String tag, Script script, ScriptService scriptService, ClusterService clusterService, String field)  {
+        super(tag);
+        this.script = script;
+        this.scriptService = scriptService;
+        this.clusterService = clusterService;
+        this.field = field;
+    }
+
+    @Override
+    public void execute(IngestDocument document) {
+        Map<String, Object> vars = new HashMap<>();
+        vars.put("ctx", document.getSourceAndMetadata());
+        CompiledScript compiledScript = scriptService.compile(script, ScriptContext.Standard.INGEST, emptyMap(), clusterService.state());
+        ExecutableScript executableScript = scriptService.executable(compiledScript, vars);
+        Object value = executableScript.run();
+        if (field != null) {
+            document.setFieldValue(field, value);
+        }
+    }
+
+    @Override
+    public String getType() {
+        return TYPE;
+    }
+
+    public static final class Factory extends AbstractProcessorFactory<ScriptProcessor> {
+
+        private final ScriptService scriptService;
+        private final ClusterService clusterService;
+
+        public Factory(ScriptService scriptService, ClusterService clusterService) {
+            this.scriptService = scriptService;
+            this.clusterService = clusterService;
+        }
+
+        @Override
+        public ScriptProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+            String field = readOptionalStringProperty(TYPE, processorTag, config, "field");
+            String lang = readStringProperty(TYPE, processorTag, config, "lang");
+            String inline = readOptionalStringProperty(TYPE, processorTag, config, "inline");
+            String file = readOptionalStringProperty(TYPE, processorTag, config, "file");
+            String id = readOptionalStringProperty(TYPE, processorTag, config, "id");
+
+            boolean containsNoScript = !hasLength(file) && !hasLength(id) && !hasLength(inline);
+            if (containsNoScript) {
+                throw newConfigurationException(TYPE, processorTag, null, "Need [file], [id], or [inline] parameter to refer to scripts");
+            }
+
+            boolean moreThanOneConfigured = (Strings.hasLength(file) && Strings.hasLength(id)) ||
+                (Strings.hasLength(file) && Strings.hasLength(inline)) || (Strings.hasLength(id) && Strings.hasLength(inline));
+            if (moreThanOneConfigured) {
+                throw newConfigurationException(TYPE, processorTag, null, "Only one of [file], [id], or [inline] may be configured");
+            }
+
+            final Script script;
+            if (Strings.hasLength(file)) {
+                script = new Script(file, FILE, lang, emptyMap());
+            } else if (Strings.hasLength(inline)) {
+                script = new Script(inline, INLINE, lang, emptyMap());
+            } else if (Strings.hasLength(id)) {
+                script = new Script(id, STORED, lang, emptyMap());
+            } else {
+                throw newConfigurationException(TYPE, processorTag, null, "Could not initialize script");
+            }
+
+            return new ScriptProcessor(processorTag, script, scriptService, clusterService, field);
+        }
+    }
+}
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ForEachProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ForEachProcessorFactoryTests.java
index 74771b5768e..161b50c5e4c 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ForEachProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ForEachProcessorFactoryTests.java
@@ -19,7 +19,9 @@
 
 package org.elasticsearch.ingest;
 
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.ingest.core.Processor;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
 
@@ -27,13 +29,15 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.mockito.Mockito.mock;
+
 public class ForEachProcessorFactoryTests extends ESTestCase {
 
     public void testCreate() throws Exception {
         ProcessorsRegistry.Builder builder = new ProcessorsRegistry.Builder();
         Processor processor = new TestProcessor(ingestDocument -> {});
-        builder.registerProcessor("_name", (templateService, registry) -> config -> processor);
-        ProcessorsRegistry registry = builder.build(TestTemplateService.instance());
+        builder.registerProcessor("_name", (registry) -> config -> processor);
+        ProcessorsRegistry registry = builder.build(mock(ScriptService.class), mock(ClusterService.class));
         ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(registry);
 
         Map<String, Object> config = new HashMap<>();
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ScriptProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ScriptProcessorFactoryTests.java
new file mode 100644
index 00000000000..e5cd4d3699e
--- /dev/null
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ScriptProcessorFactoryTests.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.mock;
+
+public class ScriptProcessorFactoryTests extends ESTestCase {
+
+    private ScriptProcessor.Factory factory;
+
+    @Before
+    public void init() {
+        factory = new ScriptProcessor.Factory(mock(ScriptService.class), mock(ClusterService.class));
+    }
+
+
+    public void testFactoryValidationForMultipleScriptingTypes() throws Exception {
+        Map<String, Object> configMap = new HashMap<>();
+        String randomType = randomFrom("id", "inline", "file");
+        String otherRandomType = randomFrom("id", "inline", "file");
+        while (randomType.equals(otherRandomType)) {
+            otherRandomType = randomFrom("id", "inline", "file");
+        }
+
+        configMap.put(randomType, "foo");
+        configMap.put(otherRandomType, "bar");
+        configMap.put("field", "my_field");
+        configMap.put("lang", "mockscript");
+
+        ElasticsearchException exception = expectThrows(ElasticsearchException.class,
+            () -> factory.doCreate(randomAsciiOfLength(10), configMap));
+
+        assertThat(exception.getMessage(), is("[null] Only one of [file], [id], or [inline] may be configured"));
+    }
+
+    public void testFactoryValidationAtLeastOneScriptingType() throws Exception {
+        Map<String, Object> configMap = new HashMap<>();
+        configMap.put("field", "my_field");
+        configMap.put("lang", "mockscript");
+
+        ElasticsearchException exception = expectThrows(ElasticsearchException.class,
+            () -> factory.doCreate(randomAsciiOfLength(10), configMap));
+
+        assertThat(exception.getMessage(), is("[null] Need [file], [id], or [inline] parameter to refer to scripts"));
+    }
+}
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ScriptProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ScriptProcessorTests.java
new file mode 100644
index 00000000000..377b93ef454
--- /dev/null
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ScriptProcessorTests.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest;
+
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.script.CompiledScript;
+import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.core.Is.is;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ScriptProcessorTests extends ESTestCase {
+
+    public void testScripting() throws Exception {
+        int randomInt = randomInt();
+        ScriptService scriptService = mock(ScriptService.class);
+        ClusterService clusterService = mock(ClusterService.class);
+        CompiledScript compiledScript = mock(CompiledScript.class);
+        Script script = mock(Script.class);
+        when(scriptService.compile(any(), any(), any(), any())).thenReturn(compiledScript);
+        ExecutableScript executableScript = mock(ExecutableScript.class);
+        when(scriptService.executable(any(), any())).thenReturn(executableScript);
+        when(executableScript.run()).thenReturn(randomInt);
+
+        ScriptProcessor processor = new ScriptProcessor(randomAsciiOfLength(10), script,
+            scriptService, clusterService, "bytes_total");
+
+        Map<String, Object> document = new HashMap<>();
+        document.put("bytes_in", 1234);
+        document.put("bytes_out", 4321);
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        processor.execute(ingestDocument);
+
+        assertThat(ingestDocument.getSourceAndMetadata(), hasKey("bytes_in"));
+        assertThat(ingestDocument.getSourceAndMetadata(), hasKey("bytes_out"));
+        assertThat(ingestDocument.getSourceAndMetadata(), hasKey("bytes_total"));
+        assertThat(ingestDocument.getSourceAndMetadata().get("bytes_total"), is(randomInt));
+    }
+}
diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/10_basic.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/10_basic.yaml
index 1f6962c6e2f..14f58369dfa 100644
--- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/10_basic.yaml
+++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/10_basic.yaml
@@ -21,8 +21,9 @@
     - match:  { nodes.$master.ingest.processors.9.type: lowercase }
     - match:  { nodes.$master.ingest.processors.10.type: remove }
     - match:  { nodes.$master.ingest.processors.11.type: rename }
-    - match:  { nodes.$master.ingest.processors.12.type: set }
-    - match:  { nodes.$master.ingest.processors.13.type: sort }
-    - match:  { nodes.$master.ingest.processors.14.type: split }
-    - match:  { nodes.$master.ingest.processors.15.type: trim }
-    - match:  { nodes.$master.ingest.processors.16.type: uppercase }
+    - match:  { nodes.$master.ingest.processors.12.type: script }
+    - match:  { nodes.$master.ingest.processors.13.type: set }
+    - match:  { nodes.$master.ingest.processors.14.type: sort }
+    - match:  { nodes.$master.ingest.processors.15.type: split }
+    - match:  { nodes.$master.ingest.processors.16.type: trim }
+    - match:  { nodes.$master.ingest.processors.17.type: uppercase }
diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
index 2c7edd7ee96..5d6196f274b 100644
--- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
+++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
@@ -51,7 +51,7 @@ public class IngestGeoIpPlugin extends Plugin {
     public void onModule(NodeModule nodeModule) throws IOException {
         Path geoIpConfigDirectory = nodeModule.getNode().getEnvironment().configFile().resolve("ingest-geoip");
         Map<String, DatabaseReader> databaseReaders = loadDatabaseReaders(geoIpConfigDirectory);
-        nodeModule.registerProcessor(GeoIpProcessor.TYPE, (templateService, registry) -> new GeoIpProcessor.Factory(databaseReaders));
+        nodeModule.registerProcessor(GeoIpProcessor.TYPE, (registry) -> new GeoIpProcessor.Factory(databaseReaders));
     }
 
     public static Map<String, DatabaseReader> loadDatabaseReaders(Path geoIpConfigDirectory) throws IOException {
diff --git a/qa/smoke-test-ingest-with-all-dependencies/build.gradle b/qa/smoke-test-ingest-with-all-dependencies/build.gradle
index 60cd9f80a84..9c9943a1712 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/build.gradle
+++ b/qa/smoke-test-ingest-with-all-dependencies/build.gradle
@@ -23,11 +23,14 @@ dependencies {
     testCompile project(path: ':modules:ingest-common', configuration: 'runtime')
     testCompile project(path: ':plugins:ingest-geoip', configuration: 'runtime')
     testCompile project(path: ':modules:lang-mustache', configuration: 'runtime')
+    testCompile project(path: ':modules:lang-painless', configuration: 'runtime')
     testCompile project(path: ':modules:reindex', configuration: 'runtime')
 }
 
 integTest {
     cluster {
         plugin 'ingest-geoip', project(':plugins:ingest-geoip')
+        setting 'script.inline', 'true'
+        setting 'path.scripts', "${project.buildDir}/resources/test/scripts"
     }
-}
\ No newline at end of file
+}
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractMustacheTestCase.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java
similarity index 78%
rename from qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractMustacheTestCase.java
rename to qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java
index 8a24ac2408f..b0cae6a693f 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractMustacheTestCase.java
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java
@@ -29,10 +29,12 @@ import org.elasticsearch.script.ScriptSettings;
 import org.elasticsearch.script.mustache.MustacheScriptEngineService;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
+import org.mockito.internal.util.collections.Sets;
 
+import java.util.Arrays;
 import java.util.Collections;
 
-public abstract class AbstractMustacheTestCase extends ESTestCase {
+public abstract class AbstractScriptTestCase extends ESTestCase {
 
     protected TemplateService templateService;
 
@@ -43,14 +45,13 @@ public abstract class AbstractMustacheTestCase extends ESTestCase {
             .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
             .build();
         MustacheScriptEngineService mustache = new MustacheScriptEngineService(settings);
-        ScriptEngineRegistry scriptEngineRegistry =
-                new ScriptEngineRegistry(Collections.singletonList(
-                                new ScriptEngineRegistry.ScriptEngineRegistration(MustacheScriptEngineService.class,
-                                                                                  MustacheScriptEngineService.NAME,
-                                                                                  true)));
+
+        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Arrays.asList(
+            new ScriptEngineRegistry.ScriptEngineRegistration(MustacheScriptEngineService.class, MustacheScriptEngineService.NAME, true)));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
-        ScriptService scriptService = new ScriptService(settings, new Environment(settings), Collections.singleton(mustache), null,
+
+        ScriptService scriptService = new ScriptService(settings, new Environment(settings), Sets.newSet(mustache), null,
                 scriptEngineRegistry, scriptContextRegistry, scriptSettings);
         templateService = new InternalTemplateService(scriptService);
     }
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java
index 2314e273fc6..112f44b9f2c 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java
@@ -31,7 +31,7 @@ import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
 
-public class IngestDocumentMustacheIT extends AbstractMustacheTestCase {
+public class IngestDocumentMustacheIT extends AbstractScriptTestCase {
 
     public void testAccessMetaDataViaTemplate() {
         Map<String, Object> document = new HashMap<>();
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/TemplateServiceIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/TemplateServiceIT.java
index d33976b2507..98b215110b1 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/TemplateServiceIT.java
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/TemplateServiceIT.java
@@ -27,7 +27,7 @@ import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
 
-public class TemplateServiceIT extends AbstractMustacheTestCase {
+public class TemplateServiceIT extends AbstractScriptTestCase {
 
     public void testTemplates() {
         Map<String, Object> model = new HashMap<>();
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/ValueSourceMustacheIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/ValueSourceMustacheIT.java
index 0d51418ecd4..854b615e9bb 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/ValueSourceMustacheIT.java
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/ValueSourceMustacheIT.java
@@ -32,7 +32,7 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 
-public class ValueSourceMustacheIT extends AbstractMustacheTestCase {
+public class ValueSourceMustacheIT extends AbstractScriptTestCase {
 
     public void testValueSourceWithTemplates() {
         Map<String, Object> model = new HashMap<>();
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/IngestWithMustacheIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/IngestWithDependenciesIT.java
similarity index 90%
rename from qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/IngestWithMustacheIT.java
rename to qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/IngestWithDependenciesIT.java
index 73f64d4433c..b9873214aa2 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/IngestWithMustacheIT.java
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/IngestWithDependenciesIT.java
@@ -27,9 +27,9 @@ import org.elasticsearch.test.rest.parser.RestTestParseException;
 
 import java.io.IOException;
 
-public class IngestWithMustacheIT extends ESRestTestCase {
+public class IngestWithDependenciesIT extends ESRestTestCase {
 
-    public IngestWithMustacheIT(@Name("yaml") RestTestCandidate testCandidate) {
+    public IngestWithDependenciesIT(@Name("yaml") RestTestCandidate testCandidate) {
         super(testCandidate);
     }
 
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/50_script_processor_using_painless.yaml b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/50_script_processor_using_painless.yaml
new file mode 100644
index 00000000000..69b18aecf82
--- /dev/null
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/50_script_processor_using_painless.yaml
@@ -0,0 +1,120 @@
+---
+"Test script processor with inline script":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "script" : {
+                  "field" : "bytes_total",
+                  "lang" : "painless",
+                  "inline": "return ctx.bytes_in + ctx.bytes_out"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: { bytes_in: 1234, bytes_out: 4321 }
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.bytes_in: 1234 }
+  - match: { _source.bytes_out: 4321 }
+  - match: { _source.bytes_total: 5555 }
+
+---
+"Test script processor with file script":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "script" : {
+                  "field" : "bytes_total",
+                  "lang" : "painless",
+                  "file": "master"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: { bytes_in: 1234, bytes_out: 4321 }
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.bytes_in: 1234 }
+  - match: { _source.bytes_out: 4321 }
+  - match: { _source.bytes_total: 5555 }
+
+---
+"Test script processor with stored script":
+  - do:
+      put_script:
+        id: "sum_bytes"
+        lang: "painless"
+        body: >
+          {
+            "script" : "return ctx.bytes_in + ctx.bytes_out"
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "script" : {
+                  "field" : "bytes_total",
+                  "lang" : "painless",
+                  "id" : "sum_bytes"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: { bytes_in: 1234, bytes_out: 4321 }
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.bytes_in: 1234 }
+  - match: { _source.bytes_out: 4321 }
+  - match: { _source.bytes_total: 5555 }
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/scripts/master.painless b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/scripts/master.painless
new file mode 100644
index 00000000000..ebac61e3879
--- /dev/null
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/scripts/master.painless
@@ -0,0 +1 @@
+return ctx.bytes_in + ctx.bytes_out
\ No newline at end of file
diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java b/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java
index 696799d31fb..df3036f8459 100644
--- a/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java
+++ b/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java
@@ -38,7 +38,7 @@ public class IngestTestPlugin extends Plugin {
     }
 
     public void onModule(NodeModule nodeModule) {
-        nodeModule.registerProcessor("test", (templateService, registry) -> config ->
+        nodeModule.registerProcessor("test", (registry) -> config ->
                 new TestProcessor("id", "test", doc -> {
                     doc.setFieldValue("processed", true);
                     if (doc.hasField("fail") && doc.getFieldValue("fail", Boolean.class)) {

From 8caaf9ad114de6d8c6af128837372ddbfa1cc575 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 15 Jun 2016 18:23:33 -0400
Subject: [PATCH 0889/1311] Fix thread pool docs regarding dynamic settings

Thread pool settings are no longer dynamically updatable since
da74323141a69bd06fc19a70bc072143e9f0e986. This commit removes a leftover
note from the thread pool module docs that incorrectly states that
thread pool settings are dynamically updatable.
---
 docs/reference/modules/threadpool.asciidoc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/docs/reference/modules/threadpool.asciidoc b/docs/reference/modules/threadpool.asciidoc
index d8f8545c213..374b2b2bd7b 100644
--- a/docs/reference/modules/threadpool.asciidoc
+++ b/docs/reference/modules/threadpool.asciidoc
@@ -66,8 +66,6 @@ thread_pool:
         size: 30
 --------------------------------------------------
 
-NOTE: you can update thread pool settings dynamically using <<cluster-update-settings>>.
-
 [float]
 [[types]]
 === Thread pool types

From dc2d7a2a6d33175066e2bf3c06a0b79ce3fed64f Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 15 Jun 2016 18:42:48 -0400
Subject: [PATCH 0890/1311] Test: wait for task to start before waiting for it
 to finish (#18902)

---
 .../elasticsearch/action/admin/cluster/node/tasks/TasksIT.java | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
index 376bd7f3ffa..a2938876942 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
@@ -467,6 +467,9 @@ public class TasksIT extends ESIntegTestCase {
         try {
             taskId = waitForTestTaskStartOnAllNodes();
 
+            // Wait for the task to start
+            assertBusy(() -> client().admin().cluster().prepareGetTask(taskId).get());
+
             // Spin up a request to wait for that task to finish
             waitResponseFuture = wait.apply(taskId);
         } finally {

From 54797b7d099c0ecf5f544d550d5d951b98600c2b Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Wed, 15 Jun 2016 19:33:59 -0400
Subject: [PATCH 0891/1311] don't let megamorphic cache "capture" arbitrary
 arguments.

pass the shit we need as bootstrap params
---
 .../java/org/elasticsearch/painless/Def.java  | 21 +++++++++--------
 .../elasticsearch/painless/DefBootstrap.java  | 20 ++++++++--------
 .../painless/node/ECapturingFunctionRef.java  | 23 +++++++++++--------
 .../painless/node/EFunctionRef.java           | 10 +++++---
 .../elasticsearch/painless/node/LDefCall.java | 16 ++++++++++---
 5 files changed, 56 insertions(+), 34 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index c24261dc5ac..a4033255b77 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -215,25 +215,27 @@ public final class Def {
      * @param callSiteType callsite's type
      * @param receiverClass Class of the object to invoke the method on.
      * @param name Name of the method.
-     * @param args args passed to callsite
-     * @param recipe bitset marking functional parameters
+     * @param args bootstrap args passed to callsite
      * @return pointer to matching method to invoke. never returns null.
      * @throws IllegalArgumentException if no matching whitelisted method was found.
      * @throws Throwable if a method reference cannot be converted to an functional interface
      */
      static MethodHandle lookupMethod(Lookup lookup, MethodType callSiteType, 
-             Class<?> receiverClass, String name, Object args[], long recipe) throws Throwable {
+             Class<?> receiverClass, String name, Object args[]) throws Throwable {
+         long recipe = (Long) args[0];
+         int numArguments = callSiteType.parameterCount();
          // simple case: no lambdas
          if (recipe == 0) {
-             return lookupMethodInternal(receiverClass, name, args.length - 1).handle;
+             return lookupMethodInternal(receiverClass, name, numArguments - 1).handle;
          }
          
          // otherwise: first we have to compute the "real" arity. This is because we have extra arguments:
          // e.g. f(a, g(x), b, h(y), i()) looks like f(a, g, x, b, h, y, i). 
-         int arity = args.length - 1;
-         for (int i = 0; i < args.length; i++) {
+         int arity = callSiteType.parameterCount() - 1;
+         int upTo = 1;
+         for (int i = 0; i < numArguments; i++) {
              if ((recipe & (1L << (i - 1))) != 0) {
-                 String signature = (String) args[i];
+                 String signature = (String) args[upTo++];
                  int numCaptures = Integer.parseInt(signature.substring(signature.indexOf(',')+1));
                  arity -= numCaptures;
              }
@@ -245,11 +247,12 @@ public final class Def {
          MethodHandle handle = method.handle;
 
          int replaced = 0;
-         for (int i = 1; i < args.length; i++) {
+         upTo = 1;
+         for (int i = 1; i < numArguments; i++) {
              // its a functional reference, replace the argument with an impl
              if ((recipe & (1L << (i - 1))) != 0) {
                  // decode signature of form 'type.call,2' 
-                 String signature = (String) args[i];
+                 String signature = (String) args[upTo++];
                  int separator = signature.indexOf('.');
                  int separator2 = signature.indexOf(',');
                  String type = signature.substring(1, separator);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index 7c9ee430c66..ad02f4bf338 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -125,10 +125,10 @@ public final class DefBootstrap {
         /**
          * Does a slow lookup against the whitelist.
          */
-        private MethodHandle lookup(int flavor, String name, Class<?> receiver, Object[] callArgs) throws Throwable {
+        private MethodHandle lookup(int flavor, String name, Class<?> receiver) throws Throwable {
             switch(flavor) {
                 case METHOD_CALL:
-                    return Def.lookupMethod(lookup, type(), receiver, name, callArgs, (Long) this.args[0]);
+                    return Def.lookupMethod(lookup, type(), receiver, name, args);
                 case LOAD:
                     return Def.lookupGetter(receiver, name);
                 case STORE:
@@ -140,7 +140,7 @@ public final class DefBootstrap {
                 case ITERATOR:
                     return Def.lookupIterator(receiver);
                 case REFERENCE:
-                    return Def.lookupReference(lookup, (String) this.args[0], receiver, name);
+                    return Def.lookupReference(lookup, (String) args[0], receiver, name);
                 default: throw new AssertionError();
             }
         }
@@ -148,8 +148,6 @@ public final class DefBootstrap {
         /**
          * Creates the {@link MethodHandle} for the megamorphic call site
          * using {@link ClassValue} and {@link MethodHandles#exactInvoker(MethodType)}:
-         * <p>
-         * TODO: Remove the variable args and just use {@code type()}!
          */
         private MethodHandle createMegamorphicHandle(final Object[] callArgs) throws Throwable {
             final MethodType type = type();
@@ -158,7 +156,7 @@ public final class DefBootstrap {
                 protected MethodHandle computeValue(Class<?> receiverType) {
                     // it's too stupid that we cannot throw checked exceptions... (use rethrow puzzler):
                     try {
-                        return lookup(flavor, name, receiverType, callArgs).asType(type);
+                        return lookup(flavor, name, receiverType).asType(type);
                     } catch (Throwable t) {
                         Def.rethrow(t);
                         throw new AssertionError();
@@ -186,7 +184,7 @@ public final class DefBootstrap {
                 return target.invokeWithArguments(callArgs);                    
             } else {
                 final Class<?> receiver = callArgs[0].getClass();
-                final MethodHandle target = lookup(flavor, name, receiver, callArgs).asType(type());
+                final MethodHandle target = lookup(flavor, name, receiver).asType(type());
     
                 MethodHandle test = CHECK_CLASS.bindTo(receiver);
                 MethodHandle guard = MethodHandles.guardWithTest(test, target, getTarget());
@@ -398,16 +396,20 @@ public final class DefBootstrap {
         switch(flavor) {
             // "function-call" like things get a polymorphic cache
             case METHOD_CALL:
-                if (args.length != 1) {
+                if (args.length == 0) {
                     throw new BootstrapMethodError("Invalid number of parameters for method call");
                 }
                 if (args[0] instanceof Long == false) {
                     throw new BootstrapMethodError("Illegal parameter for method call: " + args[0]);
                 }
                 long recipe = (Long) args[0];
-                if (Long.bitCount(recipe) > type.parameterCount()) {
+                int numLambdas = Long.bitCount(recipe);
+                if (numLambdas > type.parameterCount()) {
                     throw new BootstrapMethodError("Illegal recipe for method call: too many bits");
                 }
+                if (args.length != numLambdas + 1) {
+                    throw new BootstrapMethodError("Illegal number of parameters: expected " + numLambdas + " references");
+                }
                 return new PIC(lookup, name, type, flavor, args);
             case LOAD:
             case STORE:
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
index ac316f07fbd..cdbd17818c2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
@@ -43,7 +43,7 @@ public class ECapturingFunctionRef extends AExpression {
     
     private FunctionRef ref;
     Variable captured;
-    private boolean defInterface;
+    String defPointer;
 
     public ECapturingFunctionRef(Location location, String type, String call) {
         super(location);
@@ -56,10 +56,16 @@ public class ECapturingFunctionRef extends AExpression {
     void analyze(Locals variables) {
         captured = variables.getVariable(location, type);
         if (expected == null) {
-            defInterface = true;
+            if (captured.type.sort == Definition.Sort.DEF) {
+                // dynamic implementation
+                defPointer = "D" + type + "." + call + ",1";
+            } else {
+                // typed implementation
+                defPointer = "S" + captured.type.name + "." + call + ",1";
+            }
             actual = Definition.getType("String");
         } else {
-            defInterface = false;
+            defPointer = null;
             // static case
             if (captured.type.sort != Definition.Sort.DEF) {
                 try {
@@ -75,13 +81,10 @@ public class ECapturingFunctionRef extends AExpression {
     @Override
     void write(MethodWriter writer) {
         writer.writeDebugInfo(location);
-        if (defInterface && captured.type.sort == Definition.Sort.DEF) {
-            // dynamic interface, dynamic implementation
-            writer.push("D" + type + "." + call + ",1");
-            writer.visitVarInsn(captured.type.type.getOpcode(Opcodes.ILOAD), captured.slot);
-        } else if (defInterface) {
-            // dynamic interface, typed implementation
-            writer.push("S" + captured.type.name + "." + call + ",1");
+        if (defPointer != null) {
+            // dynamic interface: push captured parameter on stack
+            // TODO: don't do this: its just to cutover :)
+            writer.push(defPointer);
             writer.visitVarInsn(captured.type.type.getOpcode(Opcodes.ILOAD), captured.slot);
         } else if (ref == null) {
             // typed interface, dynamic implementation
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index 00a69e0b7b9..b41799883d7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -40,6 +40,7 @@ public class EFunctionRef extends AExpression {
     public final String call;
 
     private FunctionRef ref;
+    String defPointer;
 
     public EFunctionRef(Location location, String type, String call) {
         super(location);
@@ -53,7 +54,9 @@ public class EFunctionRef extends AExpression {
         if (expected == null) {
             ref = null;
             actual = Definition.getType("String");
+            defPointer = "S" + type + "." + call + ",0";
         } else {
+            defPointer = null;
             try {
                 if ("this".equals(type)) {
                     // user's own function
@@ -81,9 +84,7 @@ public class EFunctionRef extends AExpression {
 
     @Override
     void write(MethodWriter writer) {
-        if (ref == null) {
-            writer.push("S" + type + "." + call + ",0");
-        } else {
+        if (ref != null) {
             writer.writeDebugInfo(location);
             // convert MethodTypes to asm Type for the constant pool.
             String invokedType = ref.invokedType.toMethodDescriptorString();
@@ -108,6 +109,9 @@ public class EFunctionRef extends AExpression {
                                      samMethodType,
                                      0);
             }
+        } else {
+            // TODO: don't do this: its just to cutover :)
+            writer.push(defPointer);
         }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index 5301dd2b08d..a2c2f150840 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -25,6 +25,7 @@ import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
+import java.util.ArrayList;
 import java.util.List;
 
 import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
@@ -37,6 +38,7 @@ final class LDefCall extends ALink implements IDefLink {
     final String name;
     final List<AExpression> arguments;
     long recipe;
+    List<String> pointers = new ArrayList<>();
 
     LDefCall(Location location, String name, List<AExpression> arguments) {
         super(location, -1);
@@ -59,14 +61,18 @@ final class LDefCall extends ALink implements IDefLink {
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
 
+            expression.internal = true;
+            expression.analyze(locals);
+
             if (expression instanceof EFunctionRef) {
+                pointers.add(((EFunctionRef)expression).defPointer);
                 recipe |= (1L << (argument + totalCaptures)); // mark argument as deferred reference
             } else if (expression instanceof ECapturingFunctionRef) {
+                pointers.add(((ECapturingFunctionRef)expression).defPointer);
                 recipe |= (1L << (argument + totalCaptures)); // mark argument as deferred reference
                 totalCaptures++;
             }
-            expression.internal = true;
-            expression.analyze(locals);
+
             expression.expected = expression.actual;
             arguments.set(argument, expression.cast(locals));
         }
@@ -105,7 +111,11 @@ final class LDefCall extends ALink implements IDefLink {
         // return value
         signature.append(after.type.getDescriptor());
 
-        writer.invokeDynamic(name, signature.toString(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.METHOD_CALL, recipe);
+        List<Object> args = new ArrayList<>();
+        args.add(DefBootstrap.METHOD_CALL);
+        args.add(recipe);
+        args.addAll(pointers);
+        writer.invokeDynamic(name, signature.toString(), DEF_BOOTSTRAP_HANDLE, args.toArray());
     }
 
     @Override

From 7f6e0c6c0203fd67713ae0bc6dbbc20a18817df7 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Wed, 15 Jun 2016 16:57:01 -0700
Subject: [PATCH 0892/1311] fix compile for ingest plugin lambda

---
 .../elasticsearch/ingest/attachment/IngestAttachmentPlugin.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/IngestAttachmentPlugin.java b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/IngestAttachmentPlugin.java
index 8957fc2f841..6e0b3e9e1b3 100644
--- a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/IngestAttachmentPlugin.java
+++ b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/IngestAttachmentPlugin.java
@@ -38,6 +38,6 @@ public class IngestAttachmentPlugin extends Plugin {
 
     public void onModule(NodeModule nodeModule) throws IOException {
         nodeModule.registerProcessor(AttachmentProcessor.TYPE,
-            (templateService, registry) -> new AttachmentProcessor.Factory());
+            (registry) -> new AttachmentProcessor.Factory());
     }
 }

From a4503c2aed8f319ce85e07b7fa9848e523713ebc Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Wed, 15 Jun 2016 15:13:25 -0700
Subject: [PATCH 0893/1311] Plugins: Remove name() and description() from api

In 2.0 we added plugin descriptors which require defining a name and
description for the plugin. However, we still have name() and
description() which must be overriden from the Plugin class. This still
exists for classpath plugins. But classpath plugins are mainly for
tests, and even then, referring to classpath plugins with their class is
a better idea. This change removes name() and description(), replacing
the name for classpath plugins with the full class name.
---
 .../org/elasticsearch/plugins/Plugin.java     | 10 ----
 .../elasticsearch/plugins/PluginsService.java | 10 ++--
 .../cluster/node/tasks/TestTaskPlugin.java    | 11 ----
 .../TransportClientHeadersTests.java          |  8 ---
 .../cluster/ClusterInfoServiceIT.java         | 10 ----
 .../cluster/service/ClusterServiceIT.java     | 10 ----
 .../cluster/settings/SettingsFilteringIT.java | 16 ------
 .../index/SettingsListenerIT.java             | 15 -----
 .../index/WaitUntilRefreshIT.java             | 10 ----
 .../externalvalues/ExternalMapperPlugin.java  | 10 ----
 .../FunctionScoreQueryBuilderTests.java       | 11 ----
 .../query/plugin/DummyQueryParserPlugin.java  | 12 +---
 .../indices/IndicesOptionsIntegrationIT.java  |  9 ---
 .../indices/analysis/DummyAnalysisPlugin.java | 16 ------
 .../RandomExceptionCircuitBreakerIT.java      |  9 ---
 .../template/IndexTemplateFilteringIT.java    | 10 ----
 .../elasticsearch/ingest/IngestCloseIT.java   | 11 ----
 .../nodesinfo/plugin/dummy1/TestPlugin.java   | 40 -------------
 .../plugins/PluginsServiceTests.java          | 29 +---------
 .../TestResponseHeaderPlugin.java             | 10 ----
 .../elasticsearch/script/ScriptFieldIT.java   | 10 ----
 .../search/SearchServiceTests.java            | 10 ----
 .../elasticsearch/search/SearchTimeoutIT.java |  9 ---
 .../aggregations/bucket/DateScriptMocks.java  | 10 ----
 .../search/aggregations/bucket/IpRangeIT.java | 10 ----
 .../SignificantTermsSignificanceScoreIT.java  |  9 ---
 .../search/aggregations/metrics/AvgIT.java    | 22 --------
 .../search/aggregations/metrics/SumIT.java    | 22 --------
 .../aggregations/metrics/ValueCountIT.java    | 11 ----
 .../basic/SearchWithRandomExceptionsIT.java   |  8 ---
 .../search/fetch/FetchSubPhasePluginIT.java   | 11 ----
 .../ExplainableScriptPlugin.java              | 12 ----
 .../functionscore/FunctionScorePluginIT.java  | 11 ----
 .../highlight/CustomHighlighterPlugin.java    | 10 ----
 .../search/suggest/CustomSuggesterPlugin.java | 13 -----
 .../snapshots/mockstore/MockRepository.java   | 10 ----
 .../ContextAndHeaderTransportIT.java          | 10 ----
 .../transport/netty/NettyTransportIT.java     |  8 ---
 .../update/UpdateByNativeScriptIT.java        |  8 ---
 .../org/elasticsearch/update/UpdateIT.java    | 56 -------------------
 .../node2/dummy/_site/es-plugin.properties    | 21 -------
 .../nodesinfo/node2/dummy/_site/index.html    |  9 ---
 .../nodesinfo/node4/dummy/_site/index.html    |  9 ---
 .../test-no-version-plugin/_site/index.html   |  9 ---
 .../matrix/MatrixAggregationPlugin.java       | 13 -----
 .../ingest/IngestCommonPlugin.java            | 10 ----
 .../script/expression/ExpressionPlugin.java   | 10 ----
 .../script/groovy/GroovyPlugin.java           | 10 ----
 .../messy/tests/IndicesRequestTests.java      |  9 +--
 .../script/mustache/MustachePlugin.java       | 10 ----
 .../painless/PainlessPlugin.java              | 10 ----
 .../percolator/PercolatorPlugin.java          | 10 ----
 .../index/reindex/ReindexPlugin.java          | 10 ----
 .../index/reindex/CancelTests.java            | 10 ----
 .../analysis/icu/AnalysisICUPlugin.java       | 13 -----
 .../kuromoji/AnalysisKuromojiPlugin.java      | 11 ----
 .../analysis/AnalysisPhoneticPlugin.java      | 12 ----
 .../smartcn/AnalysisSmartChinesePlugin.java   | 13 -----
 .../stempel/AnalysisStempelPlugin.java        | 13 -----
 .../discovery/azure/AzureDiscoveryPlugin.java | 10 ----
 .../azure/AzureComputeServiceSimpleMock.java  |  8 ---
 .../AzureComputeServiceTwoNodesMock.java      |  8 ---
 .../AzureDiscoveryClusterFormationTests.java  | 11 ----
 .../discovery/ec2/Ec2DiscoveryPlugin.java     | 17 ------
 .../cloud/aws/AWSSignersTests.java            |  3 +-
 .../Ec2DiscoveryClusterFormationTests.java    | 15 +----
 .../discovery/gce/GceDiscoveryPlugin.java     | 17 ------
 .../discovery/gce/GceDiscoverTests.java       | 10 ----
 .../attachment/IngestAttachmentPlugin.java    | 10 ----
 .../ingest/geoip/IngestGeoIpPlugin.java       | 10 ----
 .../plugin/example/JvmExamplePlugin.java      | 10 ----
 .../plugin/javascript/JavaScriptPlugin.java   | 10 ----
 .../plugin/python/PythonPlugin.java           | 13 -----
 .../attachments/MapperAttachmentsPlugin.java  | 10 ----
 .../plugin/mapper/MapperMurmur3Plugin.java    | 10 ----
 .../plugin/mapper/MapperSizePlugin.java       | 10 ----
 .../azure/AzureRepositoryPlugin.java          | 10 ----
 ...ctAzureRepositoryServiceIntegTestCase.java |  8 ---
 .../gcs/GoogleCloudStoragePlugin.java         | 10 ----
 .../repositories/hdfs/HdfsPlugin.java         | 14 +----
 .../repository/s3/S3RepositoryPlugin.java     | 10 ----
 .../cloud/aws/TestAwsS3Service.java           |  8 ---
 .../plugin/store/smb/SMBStorePlugin.java      | 10 ----
 .../MockInternalClusterInfoService.java       |  8 ---
 .../index/MockEngineFactoryPlugin.java        |  8 ---
 .../ingest/IngestTestPlugin.java              | 10 ----
 .../elasticsearch/node/NodeMocksPlugin.java   | 10 ----
 .../script/MockScriptEngine.java              | 20 +------
 .../search/MockSearchService.java             |  8 ---
 .../elasticsearch/test/ESIntegTestCase.java   |  8 ---
 .../test/InternalSettingsPlugin.java          |  9 ---
 .../test/MockIndexEventListener.java          |  8 ---
 .../test/store/MockFSIndexStore.java          |  8 ---
 .../transport/AssertingLocalTransport.java    |  8 ---
 .../test/transport/MockTransportService.java  |  8 ---
 95 files changed, 16 insertions(+), 1098 deletions(-)
 delete mode 100644 core/src/test/java/org/elasticsearch/nodesinfo/plugin/dummy1/TestPlugin.java
 delete mode 100644 core/src/test/resources/org/elasticsearch/nodesinfo/node2/dummy/_site/es-plugin.properties
 delete mode 100644 core/src/test/resources/org/elasticsearch/nodesinfo/node2/dummy/_site/index.html
 delete mode 100644 core/src/test/resources/org/elasticsearch/nodesinfo/node4/dummy/_site/index.html
 delete mode 100644 core/src/test/resources/org/elasticsearch/nodesinfo/node4/test-no-version-plugin/_site/index.html

diff --git a/core/src/main/java/org/elasticsearch/plugins/Plugin.java b/core/src/main/java/org/elasticsearch/plugins/Plugin.java
index 0283567bf80..fac4af57c97 100644
--- a/core/src/main/java/org/elasticsearch/plugins/Plugin.java
+++ b/core/src/main/java/org/elasticsearch/plugins/Plugin.java
@@ -38,16 +38,6 @@ import java.util.List;
  */
 public abstract class Plugin {
 
-    /**
-     * The name of the plugin.
-     */
-    public abstract String name();
-
-    /**
-     * The description of the plugin.
-     */
-    public abstract String description();
-
     /**
      * Node level modules.
      */
diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
index 4f123625387..28a7da7c923 100644
--- a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
+++ b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
@@ -104,7 +104,7 @@ public class PluginsService extends AbstractComponent {
         // first we load plugins that are on the classpath. this is for tests and transport clients
         for (Class<? extends Plugin> pluginClass : classpathPlugins) {
             Plugin plugin = loadPlugin(pluginClass, settings);
-            PluginInfo pluginInfo = new PluginInfo(plugin.name(), plugin.description(), "NA", pluginClass.getName());
+            PluginInfo pluginInfo = new PluginInfo(pluginClass.getName(), "classpath plugin", "NA", pluginClass.getName());
             if (logger.isTraceEnabled()) {
                 logger.trace("plugin loaded from classpath [{}]", pluginInfo);
             }
@@ -188,12 +188,12 @@ public class PluginsService extends AbstractComponent {
                     continue;
                 }
                 if (method.getParameterTypes().length == 0 || method.getParameterTypes().length > 1) {
-                    logger.warn("Plugin: {} implementing onModule with no parameters or more than one parameter", plugin.name());
+                    logger.warn("Plugin: {} implementing onModule with no parameters or more than one parameter", pluginEntry.v1().getName());
                     continue;
                 }
                 Class moduleClass = method.getParameterTypes()[0];
                 if (!Module.class.isAssignableFrom(moduleClass)) {
-                    logger.warn("Plugin: {} implementing onModule by the type is not of Module type {}", plugin.name(), moduleClass);
+                    logger.warn("Plugin: {} implementing onModule by the type is not of Module type {}", pluginEntry.v1().getName(), moduleClass);
                     continue;
                 }
                 list.add(new OnModuleReference(moduleClass, method));
@@ -225,10 +225,10 @@ public class PluginsService extends AbstractComponent {
                         try {
                             reference.onModuleMethod.invoke(plugin.v2(), module);
                         } catch (IllegalAccessException | InvocationTargetException e) {
-                            logger.warn("plugin {}, failed to invoke custom onModule method", e, plugin.v2().name());
+                            logger.warn("plugin {}, failed to invoke custom onModule method", e, plugin.v1().getName());
                             throw new ElasticsearchException("failed to invoke onModule", e);
                         } catch (Exception e) {
-                            logger.warn("plugin {}, failed to invoke custom onModule method", e, plugin.v2().name());
+                            logger.warn("plugin {}, failed to invoke custom onModule method", e, plugin.v1().getName());
                             throw e;
                         }
                     }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
index 9874bdd44b1..11b83d67a8c 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
@@ -67,17 +67,6 @@ import static org.elasticsearch.test.ESTestCase.awaitBusy;
  */
 public class TestTaskPlugin extends Plugin {
 
-
-    @Override
-    public String name() {
-        return "test-task-plugin";
-    }
-
-    @Override
-    public String description() {
-        return "Test plugin for testing task management";
-    }
-
     public void onModule(ActionModule module) {
         module.registerAction(TestTaskAction.INSTANCE, TransportTestTaskAction.class);
         module.registerAction(UnblockTestTasksAction.INSTANCE, TransportUnblockTestTasksAction.class);
diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java
index ec0a9e5cc08..e26ebfefcc9 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java
@@ -103,14 +103,6 @@ public class TransportClientHeadersTests extends AbstractClientHeadersTestCase {
     public static class InternalTransportService extends TransportService {
 
         public static class TestPlugin extends Plugin {
-            @Override
-            public String name() {
-                return "mock-transport-service";
-            }
-            @Override
-            public String description() {
-                return "a mock transport service";
-            }
             public void onModule(NetworkModule transportModule) {
                 transportModule.registerTransportService("internal", InternalTransportService.class);
             }
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
index 8b50d125a40..27b9192006d 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
@@ -76,16 +76,6 @@ public class ClusterInfoServiceIT extends ESIntegTestCase {
 
     public static class TestPlugin extends Plugin {
 
-        @Override
-        public String name() {
-            return "ClusterInfoServiceIT";
-        }
-
-        @Override
-        public String description() {
-            return "ClusterInfoServiceIT";
-        }
-
         public void onModule(ActionModule module) {
             module.registerFilter(BlockingActionFilter.class);
         }
diff --git a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java
index 9db548e2f3f..991f11a4493 100644
--- a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java
@@ -582,16 +582,6 @@ public class ClusterServiceIT extends ESIntegTestCase {
 
     public static class TestPlugin extends Plugin {
 
-        @Override
-        public String name() {
-            return "test plugin";
-        }
-
-        @Override
-        public String description() {
-            return "test plugin";
-        }
-
         @Override
         public Collection<Class<? extends LifecycleComponent>> nodeServices() {
             List<Class<? extends LifecycleComponent>> services = new ArrayList<>(1);
diff --git a/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java b/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java
index 194b82620de..b2e89417511 100644
--- a/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java
@@ -51,22 +51,6 @@ public class SettingsFilteringIT extends ESIntegTestCase {
         public static final Setting<Boolean> SOME_OTHER_NODE_SETTING =
             Setting.boolSetting("some.other.node.setting", false, Property.NodeScope);
 
-        /**
-         * The name of the plugin.
-         */
-        @Override
-        public String name() {
-            return "settings-filtering";
-        }
-
-        /**
-         * The description of the plugin.
-         */
-        @Override
-        public String description() {
-            return "Settings Filtering Plugin";
-        }
-
         @Override
         public Settings additionalSettings() {
             return Settings.builder().put("some.node.setting", true).put("some.other.node.setting", true).build();
diff --git a/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java b/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java
index 07afef1d8ae..705fe32a415 100644
--- a/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java
+++ b/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java
@@ -46,21 +46,6 @@ public class SettingsListenerIT extends ESIntegTestCase {
         private final SettingsTestingService service = new SettingsTestingService();
         private static final Setting<Integer> SETTING = Setting.intSetting("index.test.new.setting", 0,
             Property.Dynamic, Property.IndexScope);
-        /**
-         * The name of the plugin.
-         */
-        @Override
-        public String name() {
-            return "settings-listener";
-        }
-
-        /**
-         * The description of the plugin.
-         */
-        @Override
-        public String description() {
-            return "Settings Listenern Plugin";
-        }
 
         public void onModule(SettingsModule settingsModule) {
             settingsModule.registerSetting(SettingsTestingService.VALUE);
diff --git a/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java b/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java
index b2cb2d96818..3e71b394a88 100644
--- a/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java
+++ b/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java
@@ -172,16 +172,6 @@ public class WaitUntilRefreshIT extends ESIntegTestCase {
     }
 
     public static class DeletePlzPlugin extends Plugin {
-        @Override
-        public String name() {
-            return "delete_please";
-        }
-
-        @Override
-        public String description() {
-            return "adds a script that converts any update into a delete for testing";
-        }
-
         public void onModule(ScriptModule scriptModule) {
             scriptModule.registerScript("delete_plz", DeletePlzFactory.class);
         }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java
index 87daaa58769..115faf2c6a1 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java
@@ -28,16 +28,6 @@ public class ExternalMapperPlugin extends Plugin {
     public static final String EXTERNAL_BIS = "external_bis";
     public static final String EXTERNAL_UPPER = "external_upper";
 
-    @Override
-    public String name() {
-        return "external-mappers";
-    }
-
-    @Override
-    public String description() {
-        return "External Mappers Plugin";
-    }
-
     public void onModule(IndicesModule indicesModule) {
         indicesModule.registerMetadataMapper(ExternalMetadataMapper.CONTENT_TYPE, new ExternalMetadataMapper.TypeParser());
         indicesModule.registerMapper(EXTERNAL, new ExternalMapper.TypeParser(EXTERNAL, "foo"));
diff --git a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
index 2597675095a..e0bf9be2c92 100644
--- a/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/functionscore/FunctionScoreQueryBuilderTests.java
@@ -761,17 +761,6 @@ public class FunctionScoreQueryBuilderTests extends AbstractQueryTestCase<Functi
     }
 
     public static class TestPlugin extends Plugin {
-
-        @Override
-        public String name() {
-            return "test-plugin";
-        }
-
-        @Override
-        public String description() {
-            return "Adds random function with fixed seed";
-        }
-
         public void onModule(SearchModule module) {
             module.registerScoreFunction(RandomScoreFunctionBuilderWithFixedSeed::new,
                     RandomScoreFunctionBuilderWithFixedSeed::fromXContent, RandomScoreFunctionBuilderWithFixedSeed.FUNCTION_NAME_FIELD);
diff --git a/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java b/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java
index d8021ad36bb..e61fe24d297 100644
--- a/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java
+++ b/core/src/test/java/org/elasticsearch/index/query/plugin/DummyQueryParserPlugin.java
@@ -30,16 +30,6 @@ import java.io.IOException;
 
 public class DummyQueryParserPlugin extends Plugin {
 
-    @Override
-    public String name() {
-        return "dummy";
-    }
-
-    @Override
-    public String description() {
-        return "dummy query";
-    }
-
     public void onModule(SearchModule module) {
         module.registerQuery(DummyQueryBuilder::new, DummyQueryBuilder::fromXContent, DummyQueryBuilder.QUERY_NAME_FIELD);
     }
@@ -72,4 +62,4 @@ public class DummyQueryParserPlugin extends Plugin {
             return classHash();
         }
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
index 0256fab1b7f..65d15ad9dc5 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
@@ -588,15 +588,6 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
     }
 
     public static final class TestPlugin extends Plugin {
-        @Override
-        public String name() {
-            return "index-a-setting";
-        }
-
-        @Override
-        public String description() {
-            return "a plugin that adds a dynamic tst setting";
-        }
 
         private static final Setting<String> INDEX_A =
             new Setting<>("index.a", "", Function.identity(), Property.Dynamic, Property.IndexScope);
diff --git a/core/src/test/java/org/elasticsearch/indices/analysis/DummyAnalysisPlugin.java b/core/src/test/java/org/elasticsearch/indices/analysis/DummyAnalysisPlugin.java
index 8e15a6f02f4..86c54f2ece9 100644
--- a/core/src/test/java/org/elasticsearch/indices/analysis/DummyAnalysisPlugin.java
+++ b/core/src/test/java/org/elasticsearch/indices/analysis/DummyAnalysisPlugin.java
@@ -22,22 +22,6 @@ package org.elasticsearch.indices.analysis;
 import org.elasticsearch.plugins.Plugin;
 
 public class DummyAnalysisPlugin extends Plugin {
-    /**
-     * The name of the plugin.
-     */
-    @Override
-    public String name() {
-        return "analysis-dummy";
-    }
-
-    /**
-     * The description of the plugin.
-     */
-    @Override
-    public String description() {
-        return "Analysis Dummy Plugin";
-    }
-
 
     public void onModule(AnalysisModule module) {
         module.registerAnalyzer("dummy", (a, b, c, d) -> new DummyAnalyzerProvider());
diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java
index 19242b1f90b..ca76fffedd4 100644
--- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java
@@ -200,15 +200,6 @@ public class RandomExceptionCircuitBreakerIT extends ESIntegTestCase {
         public static final Setting<Double> EXCEPTION_LOW_LEVEL_RATIO_SETTING =
             Setting.doubleSetting(EXCEPTION_LOW_LEVEL_RATIO_KEY, 0.1d, 0.0d, Property.IndexScope);
         public static class TestPlugin extends Plugin {
-            @Override
-            public String name() {
-                return "random-exception-reader-wrapper";
-            }
-            @Override
-            public String description() {
-                return "a mock reader wrapper that throws random exceptions for testing";
-            }
-
             public void onModule(SettingsModule module) {
                 module.registerSetting(EXCEPTION_TOP_LEVEL_RATIO_SETTING);
                 module.registerSetting(EXCEPTION_LOW_LEVEL_RATIO_SETTING);
diff --git a/core/src/test/java/org/elasticsearch/indices/template/IndexTemplateFilteringIT.java b/core/src/test/java/org/elasticsearch/indices/template/IndexTemplateFilteringIT.java
index 47742969501..300c7769fcd 100644
--- a/core/src/test/java/org/elasticsearch/indices/template/IndexTemplateFilteringIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/template/IndexTemplateFilteringIT.java
@@ -75,16 +75,6 @@ public class IndexTemplateFilteringIT extends ESIntegTestCase {
     }
 
     public static class TestPlugin extends Plugin {
-        @Override
-        public String name() {
-            return "test-plugin";
-        }
-
-        @Override
-        public String description() {
-            return "";
-        }
-
         public void onModule(ClusterModule module) {
             module.registerIndexTemplateFilter(TestFilter.class);
         }
diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestCloseIT.java b/core/src/test/java/org/elasticsearch/ingest/IngestCloseIT.java
index 5e0706388ce..007338cf918 100644
--- a/core/src/test/java/org/elasticsearch/ingest/IngestCloseIT.java
+++ b/core/src/test/java/org/elasticsearch/ingest/IngestCloseIT.java
@@ -53,17 +53,6 @@ public class IngestCloseIT extends ESSingleNodeTestCase {
     }
 
     public static class IngestPlugin extends Plugin {
-
-        @Override
-        public String name() {
-            return "ingest";
-        }
-
-        @Override
-        public String description() {
-            return "ingest mock";
-        }
-
         public void onModule(NodeModule nodeModule) {
             nodeModule.registerProcessor("test", (registry) -> new Factory());
         }
diff --git a/core/src/test/java/org/elasticsearch/nodesinfo/plugin/dummy1/TestPlugin.java b/core/src/test/java/org/elasticsearch/nodesinfo/plugin/dummy1/TestPlugin.java
deleted file mode 100644
index 8455a3bf562..00000000000
--- a/core/src/test/java/org/elasticsearch/nodesinfo/plugin/dummy1/TestPlugin.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.nodesinfo.plugin.dummy1;
-
-import org.elasticsearch.plugins.Plugin;
-
-public class TestPlugin extends Plugin {
-
-    static final public class Fields {
-        static public final String NAME = "test-plugin";
-        static public final String DESCRIPTION = NAME + " description";
-    }
-
-    @Override
-    public String name() {
-        return Fields.NAME;
-    }
-
-    @Override
-    public String description() {
-        return Fields.DESCRIPTION;
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java b/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java
index d3c1f1b8bad..659e11fe06a 100644
--- a/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java
@@ -32,28 +32,12 @@ import java.util.Arrays;
 
 public class PluginsServiceTests extends ESTestCase {
     public static class AdditionalSettingsPlugin1 extends Plugin {
-        @Override
-        public String name() {
-            return "additional-settings1";
-        }
-        @Override
-        public String description() {
-            return "adds additional setting 'foo.bar'";
-        }
         @Override
         public Settings additionalSettings() {
             return Settings.builder().put("foo.bar", "1").put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.MMAPFS.getSettingsKey()).build();
         }
     }
     public static class AdditionalSettingsPlugin2 extends Plugin {
-        @Override
-        public String name() {
-            return "additional-settings2";
-        }
-        @Override
-        public String description() {
-            return "adds additional setting 'foo.bar'";
-        }
         @Override
         public Settings additionalSettings() {
             return Settings.builder().put("foo.bar", "2").build();
@@ -61,15 +45,6 @@ public class PluginsServiceTests extends ESTestCase {
     }
 
     public static class FailOnModule extends Plugin {
-        @Override
-        public String name() {
-            return "fail-on-module";
-        }
-        @Override
-        public String description() {
-            return "fails in onModule";
-        }
-
         public void onModule(BrokenModule brokenModule) {
             throw new IllegalStateException("boom");
         }
@@ -108,8 +83,8 @@ public class PluginsServiceTests extends ESTestCase {
         } catch (IllegalArgumentException e) {
             String msg = e.getMessage();
             assertTrue(msg, msg.contains("Cannot have additional setting [foo.bar]"));
-            assertTrue(msg, msg.contains("plugin [additional-settings1]"));
-            assertTrue(msg, msg.contains("plugin [additional-settings2]"));
+            assertTrue(msg, msg.contains("plugin [" + AdditionalSettingsPlugin1.class.getName()));
+            assertTrue(msg, msg.contains("plugin [" + AdditionalSettingsPlugin2.class.getName()));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/plugins/responseheader/TestResponseHeaderPlugin.java b/core/src/test/java/org/elasticsearch/plugins/responseheader/TestResponseHeaderPlugin.java
index a16f318140f..701d1154587 100644
--- a/core/src/test/java/org/elasticsearch/plugins/responseheader/TestResponseHeaderPlugin.java
+++ b/core/src/test/java/org/elasticsearch/plugins/responseheader/TestResponseHeaderPlugin.java
@@ -24,16 +24,6 @@ import org.elasticsearch.plugins.Plugin;
 
 public class TestResponseHeaderPlugin extends Plugin {
 
-    @Override
-    public String name() {
-        return "test-plugin-custom-header";
-    }
-
-    @Override
-    public String description() {
-        return "test-plugin-custom-header-desc";
-    }
-
     public void onModule(NetworkModule module) {
         module.registerRestHandler(TestResponseHeaderRestAction.class);
     }
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptFieldIT.java b/core/src/test/java/org/elasticsearch/script/ScriptFieldIT.java
index 8f20d641169..3eba68c489a 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptFieldIT.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptFieldIT.java
@@ -153,16 +153,6 @@ public class ScriptFieldIT extends ESIntegTestCase {
 
     public static class CustomScriptPlugin extends Plugin {
 
-        @Override
-        public String name() {
-            return "custom_script";
-        }
-
-        @Override
-        public String description() {
-            return "script ";
-        }
-
         public void onModule(ScriptModule scriptModule) {
             scriptModule.registerScript("int", IntArrayScriptFactory.class);
             scriptModule.registerScript("long", LongArrayScriptFactory.class);
diff --git a/core/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/core/src/test/java/org/elasticsearch/search/SearchServiceTests.java
index b465d25043f..1c63ac7b62a 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchServiceTests.java
@@ -112,16 +112,6 @@ public class SearchServiceTests extends ESSingleNodeTestCase {
 
     public static class FailOnRewriteQueryPlugin extends Plugin {
 
-        @Override
-        public String name() {
-            return FailOnRewriteQueryPlugin.class.getSimpleName();
-        }
-
-        @Override
-        public String description() {
-            return "This plugin registers a query that always fails at rewrite phase";
-        }
-
         public void onModule(SearchModule module) {
             module.registerQuery(FailOnRewriteQueryBuilder::new, parseContext -> {
                 throw new UnsupportedOperationException("No query parser for this plugin");
diff --git a/core/src/test/java/org/elasticsearch/search/SearchTimeoutIT.java b/core/src/test/java/org/elasticsearch/search/SearchTimeoutIT.java
index b2eb12159f8..164261c1ed5 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchTimeoutIT.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchTimeoutIT.java
@@ -65,15 +65,6 @@ public class SearchTimeoutIT extends ESIntegTestCase {
     }
 
     public static class ScriptedTimeoutPlugin extends Plugin {
-        @Override
-        public String name() {
-            return "test-scripted-search-timeout";
-        }
-
-        @Override
-        public String description() {
-            return "Test for scripted timeouts on searches";
-        }
 
         public void onModule(ScriptModule module) {
             module.registerScript(NativeTestScriptedTimeout.TEST_NATIVE_SCRIPT_TIMEOUT, NativeTestScriptedTimeout.Factory.class);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateScriptMocks.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateScriptMocks.java
index f97b163c24c..034308464c4 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateScriptMocks.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateScriptMocks.java
@@ -41,16 +41,6 @@ public class DateScriptMocks {
      */
     public static class DateScriptsMockPlugin extends Plugin {
 
-        @Override
-        public String name() {
-            return "DateScriptMocks";
-        }
-
-        @Override
-        public String description() {
-            return "A mock script plugin.";
-        }
-
         public void onModule(ScriptModule module) {
             module.registerScript(ExtractFieldScript.NAME, ExtractFieldScriptFactory.class);
             module.registerScript(PlusOneMonthScript.NAME, PlusOneMonthScriptFactory.class);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java
index 3a2abda6aa7..c26d595c531 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java
@@ -218,16 +218,6 @@ public class IpRangeIT extends ESIntegTestCase {
 
     public static class DummyScriptPlugin extends Plugin {
 
-        @Override
-        public String name() {
-            return "DummyScriptPlugin";
-        }
-
-        @Override
-        public String description() {
-            return "A mock script plugin.";
-        }
-
         public void onModule(ScriptModule module) {
             module.registerScript(DummyScript.NAME, DummyScriptFactory.class);
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
index 0467992002f..c59e511f3cd 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
@@ -165,15 +165,6 @@ public class SignificantTermsSignificanceScoreIT extends ESIntegTestCase {
     }
 
     public static class CustomSignificanceHeuristicPlugin extends Plugin {
-        @Override
-        public String name() {
-            return "test-plugin-significance-heuristic";
-        }
-
-        @Override
-        public String description() {
-            return "Significance heuristic plugin";
-        }
 
         public void onModule(SearchModule searchModule) {
             searchModule.registerSignificanceHeuristic(SimpleHeuristic.NAMES_FIELD, SimpleHeuristic::new, SimpleHeuristic::parse);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
index a19b274c4db..fdc94908f9d 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
@@ -357,17 +357,6 @@ public class AvgIT extends AbstractNumericTestCase {
      * Mock plugin for the {@link ExtractFieldScriptEngine}
      */
     public static class ExtractFieldScriptPlugin extends Plugin {
-
-        @Override
-        public String name() {
-            return ExtractFieldScriptEngine.NAME;
-        }
-
-        @Override
-        public String description() {
-            return "Mock script engine for " + AvgIT.class;
-        }
-
         public void onModule(ScriptModule module) {
             module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractFieldScriptEngine.class, ExtractFieldScriptEngine.NAME, true));
         }
@@ -477,17 +466,6 @@ public class AvgIT extends AbstractNumericTestCase {
      * Mock plugin for the {@link FieldValueScriptEngine}
      */
     public static class FieldValueScriptPlugin extends Plugin {
-
-        @Override
-        public String name() {
-            return FieldValueScriptEngine.NAME;
-        }
-
-        @Override
-        public String description() {
-            return "Mock script engine for " + AvgIT.class;
-        }
-
         public void onModule(ScriptModule module) {
             module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.NAME, true));
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
index 641696722c9..e38fe44a045 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
@@ -352,17 +352,6 @@ public class SumIT extends AbstractNumericTestCase {
      * Mock plugin for the {@link ExtractFieldScriptEngine}
      */
     public static class ExtractFieldScriptPlugin extends Plugin {
-
-        @Override
-        public String name() {
-            return ExtractFieldScriptEngine.NAME;
-        }
-
-        @Override
-        public String description() {
-            return "Mock script engine for " + SumIT.class;
-        }
-
         public void onModule(ScriptModule module) {
             module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractFieldScriptEngine.class, ExtractFieldScriptEngine.NAME, true));
         }
@@ -474,17 +463,6 @@ public class SumIT extends AbstractNumericTestCase {
      * Mock plugin for the {@link FieldValueScriptEngine}
      */
     public static class FieldValueScriptPlugin extends Plugin {
-
-        @Override
-        public String name() {
-            return FieldValueScriptEngine.NAME;
-        }
-
-        @Override
-        public String description() {
-            return "Mock script engine for " + SumIT.class;
-        }
-
         public void onModule(ScriptModule module) {
             module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.NAME, true));
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
index 04bef292051..62ec2e64459 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
@@ -211,17 +211,6 @@ public class ValueCountIT extends ESIntegTestCase {
      * Mock plugin for the {@link FieldValueScriptEngine}
      */
     public static class FieldValueScriptPlugin extends Plugin {
-
-        @Override
-        public String name() {
-            return FieldValueScriptEngine.NAME;
-        }
-
-        @Override
-        public String description() {
-            return "Mock script engine for " + ValueCountIT.class;
-        }
-
         public void onModule(ScriptModule module) {
             module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.NAME, true));
         }
diff --git a/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java b/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java
index 7ddb6413364..d433eb48cb5 100644
--- a/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java
@@ -156,18 +156,10 @@ public class SearchWithRandomExceptionsIT extends ESIntegTestCase {
                 Setting.doubleSetting(EXCEPTION_TOP_LEVEL_RATIO_KEY, 0.1d, 0.0d, Property.IndexScope);
             public static final Setting<Double> EXCEPTION_LOW_LEVEL_RATIO_SETTING =
                 Setting.doubleSetting(EXCEPTION_LOW_LEVEL_RATIO_KEY, 0.1d, 0.0d, Property.IndexScope);
-            @Override
-            public String name() {
-                return "random-exception-reader-wrapper";
-            }
             public void onModule(SettingsModule module) {
                 module.registerSetting(EXCEPTION_TOP_LEVEL_RATIO_SETTING);
                 module.registerSetting(EXCEPTION_LOW_LEVEL_RATIO_SETTING);
             }
-            @Override
-            public String description() {
-                return "a mock reader wrapper that throws random exceptions for testing";
-            }
             public void onModule(MockEngineFactoryPlugin.MockEngineReaderModule module) {
                 module.setReaderClass(RandomExceptionDirectoryReaderWrapper.class);
             }
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
index 81c923ccf6d..39df053c862 100644
--- a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
+++ b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
@@ -100,17 +100,6 @@ public class FetchSubPhasePluginIT extends ESIntegTestCase {
     }
 
     public static class FetchTermVectorsPlugin extends Plugin {
-
-        @Override
-        public String name() {
-            return "fetch-term-vectors";
-        }
-
-        @Override
-        public String description() {
-            return "fetch plugin to test if the plugin mechanism works";
-        }
-
         public void onModule(SearchModule searchModule) {
             searchModule.registerFetchSubPhase(new TermVectorsFetchSubPhase());
         }
diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptPlugin.java b/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptPlugin.java
index a32baa526b5..5d58921b666 100644
--- a/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptPlugin.java
+++ b/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptPlugin.java
@@ -23,18 +23,6 @@ import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptModule;
 
 public class ExplainableScriptPlugin extends Plugin {
-
-    public ExplainableScriptPlugin() {}
-    @Override
-    public String name() {
-        return "native-explainable-script";
-    }
-
-    @Override
-    public String description() {
-        return "Native explainable script";
-    }
-
     public void onModule(ScriptModule module) {
         module.registerScript("native_explainable_script", ExplainableScriptIT.MyNativeScriptFactory.class);
     }
diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java
index 7fdeca4680e..e5be795d8f5 100644
--- a/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java
+++ b/core/src/test/java/org/elasticsearch/search/functionscore/FunctionScorePluginIT.java
@@ -95,17 +95,6 @@ public class FunctionScorePluginIT extends ESIntegTestCase {
     }
 
     public static class CustomDistanceScorePlugin extends Plugin {
-
-        @Override
-        public String name() {
-            return "test-plugin-distance-score";
-        }
-
-        @Override
-        public String description() {
-            return "Distance score plugin to test pluggable implementation";
-        }
-
         public void onModule(SearchModule scoreModule) {
             scoreModule.registerScoreFunction(CustomDistanceScoreBuilder::new, CustomDistanceScoreBuilder.PARSER,
                     CustomDistanceScoreBuilder.FUNCTION_NAME_FIELD);
diff --git a/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterPlugin.java b/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterPlugin.java
index b6c42b73677..f6d56b8b39c 100644
--- a/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterPlugin.java
+++ b/core/src/test/java/org/elasticsearch/search/highlight/CustomHighlighterPlugin.java
@@ -24,16 +24,6 @@ import org.elasticsearch.search.SearchModule;
 
 public class CustomHighlighterPlugin extends Plugin {
 
-    @Override
-    public String name() {
-        return "test-plugin-custom-highlighter";
-    }
-
-    @Override
-    public String description() {
-        return "Custom highlighter to test pluggable implementation";
-    }
-
     public void onModule(SearchModule highlightModule) {
         highlightModule.registerHighlighter("test-custom", new CustomHighlighter());
     }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterPlugin.java b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterPlugin.java
index eb12c4def70..dc78aec69cf 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterPlugin.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterPlugin.java
@@ -21,21 +21,8 @@ package org.elasticsearch.search.suggest;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchModule;
 
-/**
- *
- */
 public class CustomSuggesterPlugin extends Plugin {
 
-    @Override
-    public String name() {
-        return "test-plugin-custom-suggester";
-    }
-
-    @Override
-    public String description() {
-        return "Custom suggester to test pluggable implementation";
-    }
-
     public void onModule(SearchModule searchModule) {
         searchModule.registerSuggester("custom", CustomSuggester.INSTANCE);
     }
diff --git a/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java b/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
index 8b02f90f51c..ffe523b7a14 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
@@ -62,16 +62,6 @@ public class MockRepository extends FsRepository {
         public static final Setting<String> PASSWORD_SETTING =
             Setting.simpleString("secret.mock.password", Property.NodeScope, Property.Filtered);
 
-        @Override
-        public String name() {
-            return "mock-repository";
-        }
-
-        @Override
-        public String description() {
-            return "Mock Repository";
-        }
-
         public void onModule(RepositoriesModule repositoriesModule) {
             repositoriesModule.registerRepository("mock", MockRepository.class, BlobStoreIndexShardRepository.class);
         }
diff --git a/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java b/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
index eb13de34858..ea6db14b3b6 100644
--- a/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
+++ b/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
@@ -294,16 +294,6 @@ public class ContextAndHeaderTransportIT extends ESIntegTestCase {
 
     public static class ActionLoggingPlugin extends Plugin {
 
-        @Override
-        public String name() {
-            return "test-action-logging";
-        }
-
-        @Override
-        public String description() {
-            return "Test action logging";
-        }
-
         @Override
         public Collection<Module> nodeModules() {
             return Collections.<Module>singletonList(new ActionLoggingModule());
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java
index f01a4ca5e92..f70c7b27492 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java
@@ -86,14 +86,6 @@ public class NettyTransportIT extends ESIntegTestCase {
     public static final class ExceptionThrowingNettyTransport extends NettyTransport {
 
         public static class TestPlugin extends Plugin {
-            @Override
-            public String name() {
-                return "exception-throwing-netty-transport";
-            }
-            @Override
-            public String description() {
-                return "an exception throwing transport for testing";
-            }
             public void onModule(NetworkModule module) {
                 module.registerTransport("exception-throwing", ExceptionThrowingNettyTransport.class);
             }
diff --git a/core/src/test/java/org/elasticsearch/update/UpdateByNativeScriptIT.java b/core/src/test/java/org/elasticsearch/update/UpdateByNativeScriptIT.java
index dc10c9751cf..e86578ab438 100644
--- a/core/src/test/java/org/elasticsearch/update/UpdateByNativeScriptIT.java
+++ b/core/src/test/java/org/elasticsearch/update/UpdateByNativeScriptIT.java
@@ -67,14 +67,6 @@ public class UpdateByNativeScriptIT extends ESIntegTestCase {
 
     public static class CustomNativeScriptFactory implements NativeScriptFactory {
         public static class TestPlugin extends Plugin {
-            @Override
-            public String name() {
-                return "mock-native-script";
-            }
-            @Override
-            public String description() {
-                return "a mock native script for testing";
-            }
             public void onModule(ScriptModule scriptModule) {
                 scriptModule.registerScript("custom", CustomNativeScriptFactory.class);
             }
diff --git a/core/src/test/java/org/elasticsearch/update/UpdateIT.java b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
index 078bcd8cc3f..38eb7f4b59c 100644
--- a/core/src/test/java/org/elasticsearch/update/UpdateIT.java
+++ b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
@@ -79,20 +79,6 @@ import static org.hamcrest.Matchers.nullValue;
 public class UpdateIT extends ESIntegTestCase {
 
     public static class PutFieldValuesScriptPlugin extends Plugin {
-
-        public PutFieldValuesScriptPlugin() {
-        }
-
-        @Override
-        public String name() {
-            return PutFieldValuesScriptEngine.NAME;
-        }
-
-        @Override
-        public String description() {
-            return "Mock script engine for " + UpdateIT.class;
-        }
-
         public void onModule(ScriptModule module) {
             module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(PutFieldValuesScriptEngine.class, PutFieldValuesScriptEngine.NAME, true));
         }
@@ -166,20 +152,6 @@ public class UpdateIT extends ESIntegTestCase {
     }
 
     public static class FieldIncrementScriptPlugin extends Plugin {
-
-        public FieldIncrementScriptPlugin() {
-        }
-
-        @Override
-        public String name() {
-            return FieldIncrementScriptEngine.NAME;
-        }
-
-        @Override
-        public String description() {
-            return "Mock script engine for " + UpdateIT.class;
-        }
-
         public void onModule(ScriptModule module) {
             module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldIncrementScriptEngine.class, FieldIncrementScriptEngine.NAME, true));
         }
@@ -246,20 +218,6 @@ public class UpdateIT extends ESIntegTestCase {
     }
 
     public static class ScriptedUpsertScriptPlugin extends Plugin {
-
-        public ScriptedUpsertScriptPlugin() {
-        }
-
-        @Override
-        public String name() {
-            return ScriptedUpsertScriptEngine.NAME;
-        }
-
-        @Override
-        public String description() {
-            return "Mock script engine for " + UpdateIT.class + ".testScriptedUpsert";
-        }
-
         public void onModule(ScriptModule module) {
             module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ScriptedUpsertScriptEngine.class, ScriptedUpsertScriptEngine.NAME, true));
         }
@@ -326,20 +284,6 @@ public class UpdateIT extends ESIntegTestCase {
     }
 
     public static class ExtractContextInSourceScriptPlugin extends Plugin {
-
-        public ExtractContextInSourceScriptPlugin() {
-        }
-
-        @Override
-        public String name() {
-            return ExtractContextInSourceScriptEngine.NAME;
-        }
-
-        @Override
-        public String description() {
-            return "Mock script engine for " + UpdateIT.class;
-        }
-
         public void onModule(ScriptModule module) {
             module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractContextInSourceScriptEngine.class, ExtractContextInSourceScriptEngine.NAME, true));
         }
diff --git a/core/src/test/resources/org/elasticsearch/nodesinfo/node2/dummy/_site/es-plugin.properties b/core/src/test/resources/org/elasticsearch/nodesinfo/node2/dummy/_site/es-plugin.properties
deleted file mode 100644
index 4487d7c8de9..00000000000
--- a/core/src/test/resources/org/elasticsearch/nodesinfo/node2/dummy/_site/es-plugin.properties
+++ /dev/null
@@ -1,21 +0,0 @@
-################################################################
-# Licensed to Elasticsearch under one or more contributor
-# license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright
-# ownership. Elasticsearch licenses this file to you under
-# the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance  with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-################################################################
-description=This is a description for a dummy test site plugin.
-version=0.0.7-BOND-SITE
-
diff --git a/core/src/test/resources/org/elasticsearch/nodesinfo/node2/dummy/_site/index.html b/core/src/test/resources/org/elasticsearch/nodesinfo/node2/dummy/_site/index.html
deleted file mode 100644
index ceb8774542e..00000000000
--- a/core/src/test/resources/org/elasticsearch/nodesinfo/node2/dummy/_site/index.html
+++ /dev/null
@@ -1,9 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <title>Dummy Site Plugin on Node 3</title>
-</head>
-<body>
-<p>Welcome to this dummy elasticsearch plugin</p>
-</body>
-</html>
diff --git a/core/src/test/resources/org/elasticsearch/nodesinfo/node4/dummy/_site/index.html b/core/src/test/resources/org/elasticsearch/nodesinfo/node4/dummy/_site/index.html
deleted file mode 100644
index ceb8774542e..00000000000
--- a/core/src/test/resources/org/elasticsearch/nodesinfo/node4/dummy/_site/index.html
+++ /dev/null
@@ -1,9 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <title>Dummy Site Plugin on Node 3</title>
-</head>
-<body>
-<p>Welcome to this dummy elasticsearch plugin</p>
-</body>
-</html>
diff --git a/core/src/test/resources/org/elasticsearch/nodesinfo/node4/test-no-version-plugin/_site/index.html b/core/src/test/resources/org/elasticsearch/nodesinfo/node4/test-no-version-plugin/_site/index.html
deleted file mode 100644
index ceb8774542e..00000000000
--- a/core/src/test/resources/org/elasticsearch/nodesinfo/node4/test-no-version-plugin/_site/index.html
+++ /dev/null
@@ -1,9 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <title>Dummy Site Plugin on Node 3</title>
-</head>
-<body>
-<p>Welcome to this dummy elasticsearch plugin</p>
-</body>
-</html>
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/MatrixAggregationPlugin.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/MatrixAggregationPlugin.java
index 156ff05ad65..077c3d3d31a 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/MatrixAggregationPlugin.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/MatrixAggregationPlugin.java
@@ -29,19 +29,6 @@ import java.io.IOException;
 
 public class MatrixAggregationPlugin extends Plugin {
 
-    public MatrixAggregationPlugin() throws IOException {
-    }
-
-    @Override
-    public String name() {
-        return "aggs-matrix-stats";
-    }
-
-    @Override
-    public String description() {
-        return "Adds aggregations whose input are a list of numeric fields and output includes a matrix.";
-    }
-
     public void onModule(SearchModule searchModule) {
         InternalMatrixStats.registerStreams();
         searchModule.registerAggregation(MatrixStatsAggregationBuilder::new, new MatrixStatsParser(),
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/IngestCommonPlugin.java
index 7e70c3cba49..92112a37b96 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/IngestCommonPlugin.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/IngestCommonPlugin.java
@@ -41,16 +41,6 @@ public class IngestCommonPlugin extends Plugin {
         this.builtinPatterns = loadBuiltinPatterns();
     }
 
-    @Override
-    public String name() {
-        return NAME;
-    }
-
-    @Override
-    public String description() {
-        return "Module for ingest processors that do not require additional security permissions or have large dependencies and resources";
-    }
-
     public void onModule(NodeModule nodeModule) {
         nodeModule.registerProcessor(DateProcessor.TYPE, (registry) -> new DateProcessor.Factory());
         nodeModule.registerProcessor(SetProcessor.TYPE, (registry) -> new SetProcessor.Factory(registry.getTemplateService()));
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java
index cd95f13f63a..be4bb2c7157 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java
@@ -25,16 +25,6 @@ import org.elasticsearch.script.ScriptModule;
 
 public class ExpressionPlugin extends Plugin {
 
-    @Override
-    public String name() {
-        return "lang-expression";
-    }
-
-    @Override
-    public String description() {
-        return "Lucene expressions integration for Elasticsearch";
-    }
-
     public void onModule(ScriptModule module) {
         module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExpressionScriptEngineService.class,
                         ExpressionScriptEngineService.NAME, true));
diff --git a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyPlugin.java b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyPlugin.java
index e5ebcf12509..6f71190339f 100644
--- a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyPlugin.java
+++ b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyPlugin.java
@@ -25,16 +25,6 @@ import org.elasticsearch.script.ScriptModule;
 
 public class GroovyPlugin extends Plugin {
 
-    @Override
-    public String name() {
-        return "lang-groovy";
-    }
-
-    @Override
-    public String description() {
-        return "Groovy scripting integration for Elasticsearch";
-    }
-
     public void onModule(ScriptModule module) {
         module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(GroovyScriptEngineService.class, GroovyScriptEngineService.NAME));
     }
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java
index c0dfcb69ead..27bb7fdc23f 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java
@@ -708,14 +708,7 @@ public class IndicesRequestTests extends ESIntegTestCase {
     public static class InterceptingTransportService extends TransportService {
 
         public static class TestPlugin extends Plugin {
-            @Override
-            public String name() {
-                return "intercepting-transport-service";
-            }
-            @Override
-            public String description() {
-                return "an intercepting transport service for testing";
-            }
+
             public void onModule(NetworkModule module) {
                 module.registerTransportService("intercepting", InterceptingTransportService.class);
             }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
index 5ef3ad766e6..769bea8ac0e 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
@@ -25,16 +25,6 @@ import org.elasticsearch.script.ScriptModule;
 
 public class MustachePlugin extends Plugin {
 
-    @Override
-    public String name() {
-        return "lang-mustache";
-    }
-
-    @Override
-    public String description() {
-        return "Mustache scripting integration for Elasticsearch";
-    }
-
     public void onModule(ScriptModule module) {
         module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(MustacheScriptEngineService.class,
                         MustacheScriptEngineService.NAME, true));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
index 67f889b7a72..21864cb586c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
@@ -34,16 +34,6 @@ public final class PainlessPlugin extends Plugin {
         Definition.VOID_TYPE.hashCode();
     }
 
-    @Override
-    public String name() {
-        return "lang-painless";
-    }
-
-    @Override
-    public String description() {
-        return "Painless scripting language for Elasticsearch";
-    }
-
     public void onModule(final ScriptModule module) {
         module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(
                         PainlessScriptEngineService.class, PainlessScriptEngineService.NAME, true));
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
index dfc11804dd9..727d567105a 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
@@ -45,16 +45,6 @@ public class PercolatorPlugin extends Plugin {
         this.settings = settings;
     }
 
-    @Override
-    public String name() {
-        return NAME;
-    }
-
-    @Override
-    public String description() {
-        return "Percolator module adds capability to index queries and query these queries by specifying documents";
-    }
-
     public void onModule(ActionModule module) {
         module.registerAction(PercolateAction.INSTANCE, TransportPercolateAction.class);
         module.registerAction(MultiPercolateAction.INSTANCE, TransportMultiPercolateAction.class);
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
index e3a826dbdad..d4c81177571 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
@@ -26,16 +26,6 @@ import org.elasticsearch.plugins.Plugin;
 public class ReindexPlugin extends Plugin {
     public static final String NAME = "reindex";
 
-    @Override
-    public String name() {
-        return NAME;
-    }
-
-    @Override
-    public String description() {
-        return "The Reindex module adds APIs to reindex from one index to another or update documents in place.";
-    }
-
     public void onModule(ActionModule actionModule) {
         actionModule.registerAction(ReindexAction.INSTANCE, TransportReindexAction.class);
         actionModule.registerAction(UpdateByQueryAction.INSTANCE, TransportUpdateByQueryAction.class);
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
index dcaa80a8169..bbe7dd612e8 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
@@ -190,16 +190,6 @@ public class CancelTests extends ReindexTestCase {
 
     public static class ReindexCancellationPlugin extends Plugin {
 
-        @Override
-        public String name() {
-            return "reindex-cancellation";
-        }
-
-        @Override
-        public String description() {
-            return "See " + CancelTests.class.getName() + " documentation";
-        }
-
         @Override
         public void onIndexModule(IndexModule indexModule) {
             indexModule.addIndexOperationListener(new BlockingDeleteListener());
diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java
index 46b8d530f5f..e0e6c3c2e36 100644
--- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java
+++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java
@@ -28,21 +28,8 @@ import org.elasticsearch.index.analysis.IcuTransformTokenFilterFactory;
 import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.plugins.Plugin;
 
-/**
- *
- */
 public class AnalysisICUPlugin extends Plugin {
 
-    @Override
-    public String name() {
-        return "analysis-icu";
-    }
-
-    @Override
-    public String description() {
-        return "UTF related ICU analysis support";
-    }
-
     /**
      * Automatically called with the analysis module.
      */
diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java
index 4208b1be504..e92e782a96c 100644
--- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java
+++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java
@@ -36,17 +36,6 @@ import org.elasticsearch.plugins.Plugin;
  */
 public class AnalysisKuromojiPlugin extends Plugin {
 
-    @Override
-    public String name() {
-        return "analysis-kuromoji";
-    }
-
-    @Override
-    public String description() {
-        return "Kuromoji analysis support";
-    }
-
-
     public void onModule(AnalysisModule module) {
         module.registerCharFilter("kuromoji_iteration_mark", KuromojiIterationMarkCharFilterFactory::new);
         module.registerAnalyzer("kuromoji", KuromojiAnalyzerProvider::new);
diff --git a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/AnalysisPhoneticPlugin.java b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/AnalysisPhoneticPlugin.java
index 3156a6adfe3..3be97b444db 100644
--- a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/AnalysisPhoneticPlugin.java
+++ b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/AnalysisPhoneticPlugin.java
@@ -23,20 +23,8 @@ import org.elasticsearch.index.analysis.PhoneticTokenFilterFactory;
 import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.plugins.Plugin;
 
-/**
- */
 public class AnalysisPhoneticPlugin extends Plugin {
 
-    @Override
-    public String name() {
-        return "analysis-phonetic";
-    }
-
-    @Override
-    public String description() {
-        return "Phonetic analysis support";
-    }
-
     public void onModule(AnalysisModule module) {
         module.registerTokenFilter("phonetic", PhoneticTokenFilterFactory::new);
     }
diff --git a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java
index 801c51f8e81..05c779bb5a9 100644
--- a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java
+++ b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java
@@ -25,21 +25,8 @@ import org.elasticsearch.index.analysis.SmartChineseTokenizerTokenizerFactory;
 import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.plugins.Plugin;
 
-/**
- *
- */
 public class AnalysisSmartChinesePlugin extends Plugin {
 
-    @Override
-    public String name() {
-        return "analysis-smartcn";
-    }
-
-    @Override
-    public String description() {
-        return "Smart Chinese analysis support";
-    }
-
     public void onModule(AnalysisModule module) {
             module.registerAnalyzer("smartcn", SmartChineseAnalyzerProvider::new);
             module.registerTokenizer("smartcn_tokenizer", SmartChineseTokenizerTokenizerFactory::new);
diff --git a/plugins/analysis-stempel/src/main/java/org/elasticsearch/plugin/analysis/stempel/AnalysisStempelPlugin.java b/plugins/analysis-stempel/src/main/java/org/elasticsearch/plugin/analysis/stempel/AnalysisStempelPlugin.java
index 07f50ed7738..8549795f4b6 100644
--- a/plugins/analysis-stempel/src/main/java/org/elasticsearch/plugin/analysis/stempel/AnalysisStempelPlugin.java
+++ b/plugins/analysis-stempel/src/main/java/org/elasticsearch/plugin/analysis/stempel/AnalysisStempelPlugin.java
@@ -24,21 +24,8 @@ import org.elasticsearch.index.analysis.pl.PolishStemTokenFilterFactory;
 import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.plugins.Plugin;
 
-/**
- *
- */
 public class AnalysisStempelPlugin extends Plugin {
 
-    @Override
-    public String name() {
-        return "analysis-stempel";
-    }
-
-    @Override
-    public String description() {
-        return "Stempel (Polish) analysis support";
-    }
-
     public void onModule(AnalysisModule module) {
         module.registerAnalyzer("polish", PolishAnalyzerProvider::new);
         module.registerTokenFilter("polish_stem", PolishStemTokenFilterFactory::new);
diff --git a/plugins/discovery-azure/src/main/java/org/elasticsearch/plugin/discovery/azure/AzureDiscoveryPlugin.java b/plugins/discovery-azure/src/main/java/org/elasticsearch/plugin/discovery/azure/AzureDiscoveryPlugin.java
index 757ff359d42..ffc02bd3251 100644
--- a/plugins/discovery-azure/src/main/java/org/elasticsearch/plugin/discovery/azure/AzureDiscoveryPlugin.java
+++ b/plugins/discovery-azure/src/main/java/org/elasticsearch/plugin/discovery/azure/AzureDiscoveryPlugin.java
@@ -45,16 +45,6 @@ public class AzureDiscoveryPlugin extends Plugin {
         logger.trace("starting azure discovery plugin...");
     }
 
-    @Override
-    public String name() {
-        return "discovery-azure";
-    }
-
-    @Override
-    public String description() {
-        return "Azure Discovery Plugin";
-    }
-
     @Override
     public Collection<Module> nodeModules() {
         return Collections.singletonList((Module) new AzureDiscoveryModule(settings));
diff --git a/plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AzureComputeServiceSimpleMock.java b/plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AzureComputeServiceSimpleMock.java
index 5f43de3be7c..26843eba532 100644
--- a/plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AzureComputeServiceSimpleMock.java
+++ b/plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AzureComputeServiceSimpleMock.java
@@ -38,14 +38,6 @@ import java.net.InetAddress;
 public class AzureComputeServiceSimpleMock extends AzureComputeServiceAbstractMock {
 
     public static class TestPlugin extends Plugin {
-        @Override
-        public String name() {
-            return "mock-compute-service";
-        }
-        @Override
-        public String description() {
-            return "plugs in a mock compute service for testing";
-        }
         public void onModule(AzureDiscoveryModule azureDiscoveryModule) {
             azureDiscoveryModule.computeServiceImpl = AzureComputeServiceSimpleMock.class;
         }
diff --git a/plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AzureComputeServiceTwoNodesMock.java b/plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AzureComputeServiceTwoNodesMock.java
index 829c1dc21f8..bf7589c8836 100644
--- a/plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AzureComputeServiceTwoNodesMock.java
+++ b/plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AzureComputeServiceTwoNodesMock.java
@@ -42,14 +42,6 @@ import static org.elasticsearch.common.util.CollectionUtils.newSingletonArrayLis
  */
 public class AzureComputeServiceTwoNodesMock extends AzureComputeServiceAbstractMock {
     public static class TestPlugin extends Plugin {
-        @Override
-        public String name() {
-            return "mock-compute-service";
-        }
-        @Override
-        public String description() {
-            return "plugs in a mock compute service for testing";
-        }
         public void onModule(AzureDiscoveryModule azureDiscoveryModule) {
             azureDiscoveryModule.computeServiceImpl = AzureComputeServiceTwoNodesMock.class;
         }
diff --git a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryClusterFormationTests.java b/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryClusterFormationTests.java
index 489f7043b83..0aebe727e33 100644
--- a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryClusterFormationTests.java
+++ b/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryClusterFormationTests.java
@@ -72,17 +72,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTi
 public class AzureDiscoveryClusterFormationTests extends ESIntegTestCase {
 
     public static class TestPlugin extends Plugin {
-
-        @Override
-        public String name() {
-            return AzureDiscoveryClusterFormationTests.class.getName();
-        }
-
-        @Override
-        public String description() {
-            return AzureDiscoveryClusterFormationTests.class.getName();
-        }
-
         public void onModule(SettingsModule settingsModule) {
             settingsModule.registerSetting(AzureComputeService.Management.ENDPOINT_SETTING);
         }
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
index 21e31b9f076..648cfeb0279 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
@@ -67,23 +67,6 @@ public class Ec2DiscoveryPlugin extends Plugin {
         });
     }
 
-    private final Settings settings;
-    protected final ESLogger logger = Loggers.getLogger(Ec2DiscoveryPlugin.class);
-
-    public Ec2DiscoveryPlugin(Settings settings) {
-        this.settings = settings;
-    }
-
-    @Override
-    public String name() {
-        return "discovery-ec2";
-    }
-
-    @Override
-    public String description() {
-        return "EC2 Discovery Plugin";
-    }
-
     @Override
     public Collection<Module> nodeModules() {
         Collection<Module> modules = new ArrayList<>();
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AWSSignersTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AWSSignersTests.java
index 555e9f5c10a..c313f4bd195 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AWSSignersTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AWSSignersTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.cloud.aws;
 
 import com.amazonaws.ClientConfiguration;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugin.discovery.ec2.Ec2DiscoveryPlugin;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.BeforeClass;
@@ -35,7 +34,7 @@ public class AWSSignersTests extends ESTestCase {
      */
     @BeforeClass
     public static void instantiatePlugin() {
-        new Ec2DiscoveryPlugin(Settings.EMPTY);
+        new Ec2DiscoveryPlugin();
     }
 
     public void testSigners() {
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java
index 0271f522943..54f70d80cfb 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryClusterFormationTests.java
@@ -62,25 +62,12 @@ import static org.hamcrest.Matchers.equalTo;
 // TODO this should be a IT but currently all ITs in this project run against a real cluster
 public class Ec2DiscoveryClusterFormationTests extends ESIntegTestCase {
 
-    public static class TestPlugin extends Plugin {
-
-        @Override
-        public String name() {
-            return Ec2DiscoveryClusterFormationTests.class.getName();
-        }
-
-        @Override
-        public String description() {
-            return Ec2DiscoveryClusterFormationTests.class.getName();
-        }
-    }
-
     private static HttpServer httpServer;
     private static Path logDir;
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return pluginList(Ec2DiscoveryPlugin.class, TestPlugin.class);
+        return pluginList(Ec2DiscoveryPlugin.class);
     }
 
     @Override
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java
index 2b92c4fd8c1..87494e90b98 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java
@@ -66,23 +66,6 @@ public class GceDiscoveryPlugin extends Plugin {
         });
     }
 
-    private final Settings settings;
-    protected final ESLogger logger = Loggers.getLogger(GceDiscoveryPlugin.class);
-
-    public GceDiscoveryPlugin(Settings settings) {
-        this.settings = settings;
-    }
-
-    @Override
-    public String name() {
-        return "discovery-gce";
-    }
-
-    @Override
-    public String description() {
-        return "Cloud Google Compute Engine Discovery Plugin";
-    }
-
     @Override
     public Collection<Module> nodeModules() {
         return Collections.singletonList(new GceModule());
diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java
index f6a850d1d23..7f35f49bb4b 100644
--- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java
+++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java
@@ -64,16 +64,6 @@ public class GceDiscoverTests extends ESIntegTestCase {
 
     public static class TestPlugin extends Plugin {
 
-        @Override
-        public String name() {
-            return "GceDiscoverTests";
-        }
-
-        @Override
-        public String description() {
-            return "GceDiscoverTests";
-        }
-
         public void onModule(SettingsModule module) {
             module.registerSetting(GceComputeServiceImpl.GCE_HOST);
             module.registerSetting(GceComputeServiceImpl.GCE_ROOT_URL);
diff --git a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/IngestAttachmentPlugin.java b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/IngestAttachmentPlugin.java
index 6e0b3e9e1b3..3156fe381fd 100644
--- a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/IngestAttachmentPlugin.java
+++ b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/IngestAttachmentPlugin.java
@@ -26,16 +26,6 @@ import java.io.IOException;
 
 public class IngestAttachmentPlugin extends Plugin {
 
-    @Override
-    public String name() {
-        return "ingest-attachment";
-    }
-
-    @Override
-    public String description() {
-        return "Ingest processor that adds uses Tika to extract binary data";
-    }
-
     public void onModule(NodeModule nodeModule) throws IOException {
         nodeModule.registerProcessor(AttachmentProcessor.TYPE,
             (registry) -> new AttachmentProcessor.Factory());
diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
index 5d6196f274b..8339ff1fc60 100644
--- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
+++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
@@ -38,16 +38,6 @@ import java.util.zip.GZIPInputStream;
 
 public class IngestGeoIpPlugin extends Plugin {
 
-    @Override
-    public String name() {
-        return "ingest-geoip";
-    }
-
-    @Override
-    public String description() {
-        return "Ingest processor that adds information about the geographical location of ip addresses";
-    }
-
     public void onModule(NodeModule nodeModule) throws IOException {
         Path geoIpConfigDirectory = nodeModule.getNode().getEnvironment().configFile().resolve("ingest-geoip");
         Map<String, DatabaseReader> databaseReaders = loadDatabaseReaders(geoIpConfigDirectory);
diff --git a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/JvmExamplePlugin.java b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/JvmExamplePlugin.java
index 0c0e71dc1c0..163d832d22c 100644
--- a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/JvmExamplePlugin.java
+++ b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/JvmExamplePlugin.java
@@ -43,16 +43,6 @@ public class JvmExamplePlugin extends Plugin {
         this.settings = settings;
     }
 
-    @Override
-    public String name() {
-        return "jvm-example";
-    }
-
-    @Override
-    public String description() {
-        return "A plugin that extends all extension points";
-    }
-
     @Override
     public Collection<Module> nodeModules() {
         return Collections.<Module>singletonList(new ConfiguredExampleModule());
diff --git a/plugins/lang-javascript/src/main/java/org/elasticsearch/plugin/javascript/JavaScriptPlugin.java b/plugins/lang-javascript/src/main/java/org/elasticsearch/plugin/javascript/JavaScriptPlugin.java
index edd7c860244..111dd720c09 100644
--- a/plugins/lang-javascript/src/main/java/org/elasticsearch/plugin/javascript/JavaScriptPlugin.java
+++ b/plugins/lang-javascript/src/main/java/org/elasticsearch/plugin/javascript/JavaScriptPlugin.java
@@ -34,16 +34,6 @@ public class JavaScriptPlugin extends Plugin {
         JavaScriptScriptEngineService.init();
     }
 
-    @Override
-    public String name() {
-        return "lang-javascript";
-    }
-
-    @Override
-    public String description() {
-        return "JavaScript plugin allowing to add javascript scripting support";
-    }
-
     public void onModule(ScriptModule module) {
         module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(JavaScriptScriptEngineService.class, JavaScriptScriptEngineService.NAME));
     }
diff --git a/plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java b/plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java
index ca1d3436c11..d6debcf0c7c 100644
--- a/plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java
+++ b/plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java
@@ -24,21 +24,8 @@ import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.python.PythonScriptEngineService;
 
-/**
- *
- */
 public class PythonPlugin extends Plugin {
 
-    @Override
-    public String name() {
-        return "lang-python";
-    }
-
-    @Override
-    public String description() {
-        return "Adds support for writing scripts in Python";
-    }
-
     public void onModule(ScriptModule module) {
         module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(PythonScriptEngineService.class, PythonScriptEngineService.NAME));
     }
diff --git a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java
index 7a77510447d..01e43f94173 100644
--- a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java
+++ b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java
@@ -32,16 +32,6 @@ public class MapperAttachmentsPlugin extends Plugin {
     private static ESLogger logger = ESLoggerFactory.getLogger("mapper.attachment");
     private static DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
 
-    @Override
-    public String name() {
-        return "mapper-attachments";
-    }
-
-    @Override
-    public String description() {
-        return "Adds the attachment type allowing to parse difference attachment formats";
-    }
-
     public void onModule(SettingsModule settingsModule) {
         deprecationLogger.deprecated("[mapper-attachments] plugin has been deprecated and will be replaced by [ingest-attachment] plugin.");
         settingsModule.registerSetting(AttachmentMapper.INDEX_ATTACHMENT_DETECT_LANGUAGE_SETTING);
diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java
index 19b2f0fb9c6..384fc4272f6 100644
--- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java
+++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java
@@ -25,16 +25,6 @@ import org.elasticsearch.plugins.Plugin;
 
 public class MapperMurmur3Plugin extends Plugin {
 
-    @Override
-    public String name() {
-        return "mapper-murmur3";
-    }
-
-    @Override
-    public String description() {
-        return "A mapper that allows to precompute murmur3 hashes of values at index-time and store them in the index";
-    }
-
     public void onModule(IndicesModule indicesModule) {
         indicesModule.registerMapper(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser());
     }
diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java b/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java
index a4c34e97cd8..1aecfdf444d 100644
--- a/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java
+++ b/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java
@@ -25,16 +25,6 @@ import org.elasticsearch.plugins.Plugin;
 
 public class MapperSizePlugin extends Plugin {
 
-    @Override
-    public String name() {
-        return "mapper-size";
-    }
-
-    @Override
-    public String description() {
-        return "A mapper that allows document to record their uncompressed size";
-    }
-
     public void onModule(IndicesModule indicesModule) {
         indicesModule.registerMetadataMapper(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser());
     }
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java b/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
index 3ce043500ae..56b50322c90 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
@@ -47,16 +47,6 @@ public class AzureRepositoryPlugin extends Plugin {
         logger.trace("starting azure repository plugin...");
     }
 
-    @Override
-    public String name() {
-        return "repository-azure";
-    }
-
-    @Override
-    public String description() {
-        return "Azure Repository Plugin";
-    }
-
     @Override
     public Collection<Module> nodeModules() {
         return Collections.singletonList((Module) new AzureRepositoryModule(settings));
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceIntegTestCase.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceIntegTestCase.java
index ae1bdd19a54..82af834cd35 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceIntegTestCase.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceIntegTestCase.java
@@ -41,14 +41,6 @@ import java.util.Collection;
 public abstract class AbstractAzureRepositoryServiceIntegTestCase extends AbstractAzureIntegTestCase {
 
     public static class TestPlugin extends Plugin {
-        @Override
-        public String name() {
-            return "mock-storage-service";
-        }
-        @Override
-        public String description() {
-            return "plugs in a mock storage service for testing";
-        }
         public void onModule(AzureRepositoryModule azureRepositoryModule) {
             AzureRepositoryModule.storageServiceImpl = AzureStorageServiceMock.class;
         }
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStoragePlugin.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStoragePlugin.java
index 477a083fad5..77d8e23c9e5 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStoragePlugin.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStoragePlugin.java
@@ -109,16 +109,6 @@ public class GoogleCloudStoragePlugin extends Plugin {
         });
     }
 
-    @Override
-    public String name() {
-        return NAME;
-    }
-
-    @Override
-    public String description() {
-        return "Google Cloud Storage Repository Plugin";
-    }
-
     @Override
     public Collection<Module> nodeModules() {
         return Collections.singletonList(new GoogleCloudStorageModule());
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java
index ccd0b405ff2..ab539aeea1c 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java
@@ -30,9 +30,9 @@ import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.repositories.RepositoriesModule;
 
-// Code 
+// Code
 public final class HdfsPlugin extends Plugin {
-  
+
     // initialize some problematic classes with elevated privileges
     static {
         SecurityManager sm = System.getSecurityManager();
@@ -84,16 +84,6 @@ public final class HdfsPlugin extends Plugin {
         return null;
     }
 
-    @Override
-    public String name() {
-        return "repository-hdfs";
-    }
-
-    @Override
-    public String description() {
-        return "HDFS Repository Plugin";
-    }
-
     public void onModule(RepositoriesModule repositoriesModule) {
         repositoriesModule.registerRepository("hdfs", HdfsRepository.class, BlobStoreIndexShardRepository.class);
     }
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
index d07d8c174c5..23ee36fa3f0 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
@@ -62,16 +62,6 @@ public class S3RepositoryPlugin extends Plugin {
         });
     }
 
-    @Override
-    public String name() {
-        return "repository-s3";
-    }
-
-    @Override
-    public String description() {
-        return "S3 Repository Plugin";
-    }
-
     @Override
     public Collection<Module> nodeModules() {
         Collection<Module> modules = new ArrayList<>();
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
index 47e884d73bd..f83ad222b70 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
@@ -29,14 +29,6 @@ import java.util.IdentityHashMap;
 
 public class TestAwsS3Service extends InternalAwsS3Service {
     public static class TestPlugin extends Plugin {
-        @Override
-        public String name() {
-            return "mock-s3-service";
-        }
-        @Override
-        public String description() {
-            return "plugs in mock s3 service";
-        }
         public void onModule(S3Module s3Module) {
             S3Module.s3ServiceImpl = TestAwsS3Service.class;
         }
diff --git a/plugins/store-smb/src/main/java/org/elasticsearch/plugin/store/smb/SMBStorePlugin.java b/plugins/store-smb/src/main/java/org/elasticsearch/plugin/store/smb/SMBStorePlugin.java
index e93bc939e65..241f3d77e8b 100644
--- a/plugins/store-smb/src/main/java/org/elasticsearch/plugin/store/smb/SMBStorePlugin.java
+++ b/plugins/store-smb/src/main/java/org/elasticsearch/plugin/store/smb/SMBStorePlugin.java
@@ -26,16 +26,6 @@ import org.elasticsearch.plugins.Plugin;
 
 public class SMBStorePlugin extends Plugin {
 
-    @Override
-    public String name() {
-        return "store-smb";
-    }
-
-    @Override
-    public String description() {
-        return "SMB Store Plugin";
-    }
-
     @Override
     public void onIndexModule(IndexModule indexModule) {
         indexModule.addIndexStore("smb_mmap_fs", SmbMmapFsIndexStore::new);
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
index cf881f4cd66..3bb77d34f8d 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
@@ -52,14 +52,6 @@ import static java.util.Collections.emptySet;
 public class MockInternalClusterInfoService extends InternalClusterInfoService {
 
     public static class TestPlugin extends Plugin {
-        @Override
-        public String name() {
-            return "mock-cluster-info-service";
-        }
-        @Override
-        public String description() {
-            return "a mock cluster info service for testing";
-        }
         public void onModule(ClusterModule module) {
             module.clusterInfoServiceImpl = MockInternalClusterInfoService.class;
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/index/MockEngineFactoryPlugin.java b/test/framework/src/main/java/org/elasticsearch/index/MockEngineFactoryPlugin.java
index 3c72701f214..63b3361d48d 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/MockEngineFactoryPlugin.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/MockEngineFactoryPlugin.java
@@ -32,14 +32,6 @@ import java.util.Collections;
 
 // this must exist in the same package as IndexModule to allow access to setting the impl
 public class MockEngineFactoryPlugin extends Plugin {
-    @Override
-    public String name() {
-        return "mock-engine-factory";
-    }
-    @Override
-    public String description() {
-        return "a mock engine factory for testing";
-    }
 
     private Class<? extends FilterDirectoryReader> readerWrapper = AssertingDirectoryReader.class;
 
diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java b/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java
index df3036f8459..b32a2eab991 100644
--- a/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java
+++ b/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java
@@ -27,16 +27,6 @@ import org.elasticsearch.plugins.Plugin;
  */
 public class IngestTestPlugin extends Plugin {
 
-    @Override
-    public String name() {
-        return "ingest-test";
-    }
-
-    @Override
-    public String description() {
-        return "Contains an ingest processor to be used in tests";
-    }
-
     public void onModule(NodeModule nodeModule) {
         nodeModule.registerProcessor("test", (registry) -> config ->
                 new TestProcessor("id", "test", doc -> {
diff --git a/test/framework/src/main/java/org/elasticsearch/node/NodeMocksPlugin.java b/test/framework/src/main/java/org/elasticsearch/node/NodeMocksPlugin.java
index 598f5a7ee7c..a121ad46bcc 100644
--- a/test/framework/src/main/java/org/elasticsearch/node/NodeMocksPlugin.java
+++ b/test/framework/src/main/java/org/elasticsearch/node/NodeMocksPlugin.java
@@ -23,16 +23,6 @@ import org.elasticsearch.plugins.Plugin;
 
 public class NodeMocksPlugin extends Plugin {
 
-    @Override
-    public String name() {
-        return "node-mocks";
-    }
-
-    @Override
-    public String description() {
-        return "a plugin to setup mocks for node level classes";
-    }
-
     public void onModule(NodeModule module) {
         module.bigArraysImpl = MockBigArrays.class;
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
index 4e06bbe8b45..9aa6aeb88f6 100644
--- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
+++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
@@ -31,7 +31,7 @@ import java.util.List;
 import java.util.Map;
 
 /**
- * A dummy script engine used for testing. Scripts must be a number. Many 
+ * A dummy script engine used for testing. Scripts must be a number. Many
  * tests rely on the fact this thing returns a String as its compiled form.
  * they even try to serialize it over the network!
  */
@@ -51,22 +51,8 @@ public class MockScriptEngine implements ScriptEngineService {
             this.params = params;
         }
     }
-    
+
     public static class TestPlugin extends Plugin {
-
-        public TestPlugin() {
-        }
-
-        @Override
-        public String name() {
-            return NAME;
-        }
-
-        @Override
-        public String description() {
-            return "Mock script engine for integration tests";
-        }
-
         public void onModule(ScriptModule module) {
             module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
                             MockScriptEngine.NAME, true));
@@ -91,7 +77,7 @@ public class MockScriptEngine implements ScriptEngineService {
 
     @Override
     public ExecutableScript executable(CompiledScript compiledScript, @Nullable Map<String, Object> vars) {
-        assert compiledScript.compiled() instanceof MockCompiledScript 
+        assert compiledScript.compiled() instanceof MockCompiledScript
           : "do NOT pass compiled scripts from other engines to me, I will fail your test, got: " + compiledScript;
         return new AbstractExecutableScript() {
             @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java b/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
index 1ed90408f7d..cabb67a8771 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
@@ -39,14 +39,6 @@ import java.util.concurrent.ConcurrentHashMap;
 
 public class MockSearchService extends SearchService {
     public static class TestPlugin extends Plugin {
-        @Override
-        public String name() {
-            return "mock-search-service";
-        }
-        @Override
-        public String description() {
-            return "a mock search service for testing";
-        }
         public void onModule(SearchModule module) {
             module.searchServiceImpl = MockSearchService.class;
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index 4e066bc7635..0ea17b1f868 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -1798,14 +1798,6 @@ public abstract class ESIntegTestCase extends ESTestCase {
     }
 
     public static final class TestSeedPlugin extends Plugin {
-        @Override
-        public String name() {
-            return "test-seed-plugin";
-        }
-        @Override
-        public String description() {
-            return "a test plugin that registers index.tests.seed as an index setting";
-        }
         public void onModule(SettingsModule module) {
             module.registerSetting(INDEX_TEST_SEED_SETTING);
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java b/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java
index f76ae7b4b56..9d19fadcf2a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java
@@ -25,15 +25,6 @@ import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.plugins.Plugin;
 
 public final class InternalSettingsPlugin extends Plugin {
-    @Override
-    public String name() {
-        return "internal-settings-plugin";
-    }
-
-    @Override
-    public String description() {
-        return "a plugin that allows to set values for internal settings which are can't be set via the ordinary API without this plugin installed";
-    }
 
     public static final Setting<Integer> VERSION_CREATED =
         Setting.intSetting("index.version.created", 0, Property.IndexScope, Property.NodeScope);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java b/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java
index f17fe024f14..1d62ea489d8 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java
@@ -52,14 +52,6 @@ public final class MockIndexEventListener {
 
     public static class TestPlugin extends Plugin {
         private final TestEventListener listener = new TestEventListener();
-        @Override
-        public String name() {
-            return "mock-index-listener";
-        }
-        @Override
-        public String description() {
-            return "a mock index listener for testing only";
-        }
 
         /**
          * For tests to pass in to fail on listener invocation
diff --git a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java
index d44cf60e9e3..568a93a531a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java
@@ -49,14 +49,6 @@ public class MockFSIndexStore extends IndexStore {
         Setting.boolSetting("index.store.mock.check_index_on_close", true, Property.IndexScope, Property.NodeScope);
 
     public static class TestPlugin extends Plugin {
-        @Override
-        public String name() {
-            return "mock-index-store";
-        }
-        @Override
-        public String description() {
-            return "a mock index store for testing";
-        }
         @Override
         public Settings additionalSettings() {
             return Settings.builder().put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "mock").build();
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/AssertingLocalTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/AssertingLocalTransport.java
index a9687c27873..bdbbee0313a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/AssertingLocalTransport.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/AssertingLocalTransport.java
@@ -47,14 +47,6 @@ import java.util.Random;
 public class AssertingLocalTransport extends LocalTransport {
 
     public static class TestPlugin extends Plugin {
-        @Override
-        public String name() {
-            return "asserting-local-transport";
-        }
-        @Override
-        public String description() {
-            return "an asserting transport for testing";
-        }
         public void onModule(NetworkModule module) {
             module.registerTransport("mock", AssertingLocalTransport.class);
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
index 05b8b23e90c..308b804231c 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
@@ -79,14 +79,6 @@ import java.util.concurrent.CopyOnWriteArrayList;
 public class MockTransportService extends TransportService {
 
     public static class TestPlugin extends Plugin {
-        @Override
-        public String name() {
-            return "mock-transport-service";
-        }
-        @Override
-        public String description() {
-            return "a mock transport service for testing";
-        }
         public void onModule(NetworkModule module) {
             module.registerTransportService("mock", MockTransportService.class);
         }

From 60176afdde40fdf5ec13317f0fa2cf9bfb873620 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Wed, 15 Jun 2016 20:30:16 -0400
Subject: [PATCH 0894/1311] clean up a bit more

---
 .../main/java/org/elasticsearch/painless/DefBootstrap.java    | 4 ++--
 .../elasticsearch/painless/node/ECapturingFunctionRef.java    | 2 +-
 .../java/org/elasticsearch/painless/node/EFunctionRef.java    | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index ad02f4bf338..cbf9d8bdbe6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -149,7 +149,7 @@ public final class DefBootstrap {
          * Creates the {@link MethodHandle} for the megamorphic call site
          * using {@link ClassValue} and {@link MethodHandles#exactInvoker(MethodType)}:
          */
-        private MethodHandle createMegamorphicHandle(final Object[] callArgs) throws Throwable {
+        private MethodHandle createMegamorphicHandle() throws Throwable {
             final MethodType type = type();
             final ClassValue<MethodHandle> megamorphicCache = new ClassValue<MethodHandle>() {
                 @Override
@@ -178,7 +178,7 @@ public final class DefBootstrap {
         Object fallback(final Object[] callArgs) throws Throwable {
             if (depth >= MAX_DEPTH) {
                 // we revert the whole cache and build a new megamorphic one
-                final MethodHandle target = this.createMegamorphicHandle(callArgs);
+                final MethodHandle target = this.createMegamorphicHandle();
                 
                 setTarget(target);
                 return target.invokeWithArguments(callArgs);                    
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
index cdbd17818c2..af4e3779b65 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
@@ -84,7 +84,7 @@ public class ECapturingFunctionRef extends AExpression {
         if (defPointer != null) {
             // dynamic interface: push captured parameter on stack
             // TODO: don't do this: its just to cutover :)
-            writer.push(defPointer);
+            writer.push((String)null);
             writer.visitVarInsn(captured.type.type.getOpcode(Opcodes.ILOAD), captured.slot);
         } else if (ref == null) {
             // typed interface, dynamic implementation
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index b41799883d7..7ab7703d02e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -111,7 +111,7 @@ public class EFunctionRef extends AExpression {
             }
         } else {
             // TODO: don't do this: its just to cutover :)
-            writer.push(defPointer);
+            writer.push((String)null);
         }
     }
 }

From cc465a38bf390c231d263b004b9bbd801bf5ae32 Mon Sep 17 00:00:00 2001
From: Philips Kokoh <philipskokoh@users.noreply.github.com>
Date: Thu, 16 Jun 2016 11:32:49 +0800
Subject: [PATCH 0895/1311] TransportClient instead of Client in the code

---
 docs/java-api/client.asciidoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/java-api/client.asciidoc b/docs/java-api/client.asciidoc
index c71bd5f6ece..bb254e8b356 100644
--- a/docs/java-api/client.asciidoc
+++ b/docs/java-api/client.asciidoc
@@ -37,7 +37,7 @@ be "two hop" operations).
 --------------------------------------------------
 // on startup
 
-Client client = TransportClient.builder().build()
+TransportClient client = TransportClient.builder().build()
         .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("host1"), 9300))
         .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("host2"), 9300));
 
@@ -53,7 +53,7 @@ Note that you have to set the cluster name if you use one different than
 --------------------------------------------------
 Settings settings = Settings.builder()
         .put("cluster.name", "myClusterName").build();
-Client client = TransportClient.builder().settings(settings).build();
+TransportClient client = TransportClient.builder().settings(settings).build();
 //Add transport addresses and do something with the client...
 --------------------------------------------------
 

From b036d238f5561283049de38b0e9e7a6a256139bc Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Thu, 16 Jun 2016 05:51:39 +0200
Subject: [PATCH 0896/1311] Fix typo

---
 .../rest-api-spec/test/repository_s3/20_repository.yaml         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml
index eb909f91f1d..17294986d2d 100644
--- a/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml
+++ b/plugins/repository-s3/src/test/resources/rest-api-spec/test/repository_s3/20_repository.yaml
@@ -14,7 +14,7 @@
               canned_acl: "public-read"
               storage_class: "standard"
 
-    # Get repositry
+    # Get repository
     - do:
         snapshot.get_repository:
           repository: test_repo_s3_1

From ec7215e5f221f6d75f74ece322bb74510f17645f Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Thu, 16 Jun 2016 01:25:43 -0400
Subject: [PATCH 0897/1311] get non-capturing lambdas working

---
 .../elasticsearch/painless/FunctionRef.java   | 15 +++-
 .../elasticsearch/painless/antlr/Walker.java  | 19 +++--
 .../painless/BasicStatementTests.java         |  7 --
 .../elasticsearch/painless/LambdaTests.java   | 80 +++++++++++++++++++
 4 files changed, 107 insertions(+), 14 deletions(-)
 create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
index 9015b4dee61..6a39d99f4e3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
@@ -98,7 +98,7 @@ public class FunctionRef {
         implMethod = impl.handle;
         
         // remove any prepended captured arguments for the 'natural' signature.
-        samMethodType = impl.getMethodType().dropParameterTypes(0, captures.length);
+        samMethodType = adapt(interfaceMethodType, impl.getMethodType().dropParameterTypes(0, captures.length));
     }
 
     /**
@@ -119,7 +119,7 @@ public class FunctionRef {
         implMethodASM = null;
         
         // remove any prepended captured arguments for the 'natural' signature.
-        samMethodType = impl.type().dropParameterTypes(0, captures.length);
+        samMethodType = adapt(interfaceMethodType, impl.type().dropParameterTypes(0, captures.length));
     }
 
     /** 
@@ -171,4 +171,15 @@ public class FunctionRef {
         // either way, stuff will fail if its wrong :)
         return interfaceMethodType.equals(samMethodType) == false;
     }
+    
+    /** 
+     * If the interface expects a primitive type to be returned, we can't return Object,
+     * But we can set SAM to the wrapper version, and a cast will take place 
+     */
+    private static MethodType adapt(MethodType expected, MethodType actual) {
+        if (expected.returnType().isPrimitive() && actual.returnType() == Object.class) {
+            actual = actual.changeReturnType(MethodType.methodType(expected.returnType()).wrap().returnType());
+        }
+        return actual;
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index 0f6f7c8f2d5..2b50f571bb1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -114,7 +114,6 @@ import org.elasticsearch.painless.node.EConditional;
 import org.elasticsearch.painless.node.EDecimal;
 import org.elasticsearch.painless.node.EExplicit;
 import org.elasticsearch.painless.node.EFunctionRef;
-import org.elasticsearch.painless.node.ELambda;
 import org.elasticsearch.painless.node.ENull;
 import org.elasticsearch.painless.node.ENumeric;
 import org.elasticsearch.painless.node.EUnary;
@@ -170,6 +169,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
 
     private final Deque<Reserved> reserved = new ArrayDeque<>();
     private final List<SFunction> synthetic = new ArrayList<>();
+    private int syntheticCounter = 0;
 
     private Walker(String sourceName, String sourceText, CompilerSettings settings) {
         this.settings = settings;
@@ -947,7 +947,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
 
         for (LamtypeContext lamtype : ctx.lamtype()) {
             if (lamtype.decltype() == null) {
-                paramTypes.add(null);
+                paramTypes.add("def");
             } else {
                 paramTypes.add(lamtype.decltype().getText());
             }
@@ -958,8 +958,12 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         for (StatementContext statement : ctx.block().statement()) {
             statements.add((AStatement)visit(statement));
         }
-
-        return new ELambda((FunctionReserved)reserved.pop(), location(ctx), paramTypes, paramNames, statements);
+        
+        String name = nextLambda();
+        synthetic.add(new SFunction((FunctionReserved)reserved.pop(), location(ctx), "def", name, 
+                      paramTypes, paramNames, statements, true));
+        return new EFunctionRef(location(ctx), "this", name);
+        // TODO: use a real node for captures and shit
     }
 
     @Override
@@ -999,7 +1003,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
                            new LNewArray(location, arrayType, Arrays.asList(
                            new EChain(location, 
                            new LVariable(location, "size"))))));
-            String name = "lambda$" + synthetic.size();
+            String name = nextLambda();
             synthetic.add(new SFunction(new FunctionReserved(), location, arrayType, name, 
                           Arrays.asList("int"), Arrays.asList("size"), Arrays.asList(code), true));
             return new EFunctionRef(location(ctx), "this", name);
@@ -1016,4 +1020,9 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
     public Object visitLocalFuncref(LocalFuncrefContext ctx) {
         return new EFunctionRef(location(ctx), ctx.THIS().getText(), ctx.ID().getText());
     }
+    
+    /** Returns name of next lambda */
+    private String nextLambda() {
+        return "lambda$" + syntheticCounter++;
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
index 49b9ac21970..98921f88c38 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
@@ -215,13 +215,6 @@ public class BasicStatementTests extends ScriptTestCase {
         assertEquals(10, ((Map)exec("Map s = new HashMap(); s.put(\"x\", 10); return s;")).get("x"));
     }
 
-    public void testLambdas() {
-        Exception exception = expectThrows(Exception.class, () -> {
-            exec("Math.max(2, p -> {p.doSomething();})");
-        });
-        assertTrue(exception.getCause().getMessage().contains("Lambda functions are not supported."));
-    }
-
     public void testLastInBlockDoesntNeedSemi() {
         // One statement in the block in case that is a special case
         assertEquals(10, exec("def i = 1; if (i == 1) {return 10}"));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
new file mode 100644
index 00000000000..1e7eb78be73
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+public class LambdaTests extends ScriptTestCase {
+
+    public void testNoArgLambda() {
+        assertEquals(1, exec("Optional.empty().orElseGet(() -> { return 1; });"));
+    }
+    
+    public void testNoArgLambdaDef() {
+        assertEquals(1, exec("def x = Optional.empty(); x.orElseGet(() -> { return 1; });"));
+    }
+    
+    public void testLambdaWithArgs() {
+        assertEquals("short", exec("List l = new ArrayList(); l.add('looooong'); l.add('short'); " 
+                                 + "l.sort((a, b) -> { a.length() - b.length(); }); return l.get(0)"));
+
+    }
+    
+    public void testLambdaWithTypedArgs() {
+        assertEquals("short", exec("List l = new ArrayList(); l.add('looooong'); l.add('short'); " 
+                                 + "l.sort((String a, String b) -> { (a.length() - b.length()); }); return l.get(0)"));
+
+    }
+    
+    public void testPrimitiveLambdas() {
+        assertEquals(4, exec("List l = new ArrayList(); l.add(1); l.add(1); "
+                           + "return l.stream().mapToInt(x -> { x + 1; }).sum();"));
+    }
+    
+    public void testPrimitiveLambdasWithTypedArgs() {
+        assertEquals(4, exec("List l = new ArrayList(); l.add(1); l.add(1); "
+                           + "return l.stream().mapToInt(int x -> { x + 1; }).sum();"));
+    }
+    
+    public void testPrimitiveLambdasDef() {
+        assertEquals(4, exec("def l = new ArrayList(); l.add(1); l.add(1); "
+                           + "return l.stream().mapToInt(x -> { x + 1; }).sum();"));
+    }
+    
+    public void testPrimitiveLambdasWithTypedArgsDef() {
+        assertEquals(4, exec("def l = new ArrayList(); l.add(1); l.add(1); "
+                           + "return l.stream().mapToInt(int x -> { x + 1; }).sum();"));
+    }
+    
+    public void testPrimitiveLambdasConvertible() {
+        assertEquals(2, exec("List l = new ArrayList(); l.add(1); l.add(1); "
+                           + "return l.stream().mapToInt(byte x -> { return x; }).sum();"));
+    }
+    
+    public void testPrimitiveArgs() {
+        assertEquals(2, exec("int applyOne(IntFunction arg) { arg.apply(1) } applyOne(x -> { x + 1; })"));
+    }
+    
+    public void testPrimitiveArgsTyped() {
+        assertEquals(2, exec("int applyOne(IntFunction arg) { arg.apply(1) } applyOne(int x -> { x + 1; })"));
+    }
+    
+    public void testPrimitiveArgsTypedOddly() {
+        assertEquals(2L, exec("long applyOne(IntFunction arg) { arg.apply(1) } applyOne(long x -> { x + 1; })"));
+    }
+}

From edf010f87821079136b322dca1d0e8c463cf9c44 Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@elastic.co>
Date: Thu, 16 Jun 2016 07:30:46 +0200
Subject: [PATCH 0898/1311] Force single-node cluster in
 NettyHttpRequestSizeLimitIT

---
 .../http/netty/NettyHttpRequestSizeLimitIT.java             | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java
index 8fdd02f8624..bae3b8c8175 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java
@@ -39,9 +39,12 @@ import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasSize;
 
 /**
+ * This test checks that in-flight requests are limited on HTTP level and that requests that are excluded from limiting can pass.
  *
+ * As the same setting is also used to limit in-flight requests on transport level, we avoid transport messages by forcing
+ * a single node "cluster".
  */
-@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1)
+@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numClientNodes = 0, numDataNodes = 1)
 public class NettyHttpRequestSizeLimitIT extends ESIntegTestCase {
     private static final ByteSizeValue LIMIT = new ByteSizeValue(2, ByteSizeUnit.KB);
 
@@ -89,7 +92,6 @@ public class NettyHttpRequestSizeLimitIT extends ESIntegTestCase {
         }
     }
 
-    @AwaitsFix(bugUrl = "muted while investigating")
     public void testDoesNotLimitExcludedRequests() throws Exception {
         ensureGreen();
 

From 7d42e7e7163ba49e3e404e840092b4392bec4fc8 Mon Sep 17 00:00:00 2001
From: Alexander Lin <lin.alexander.2011@gmail.com>
Date: Wed, 25 May 2016 22:13:04 -0700
Subject: [PATCH 0899/1311] Closes #18013. Added status field to _msearch
 response bodies.

---
 .../action/search/MultiSearchResponse.java    |  5 +++++
 .../search/MultiSearchRequestTests.java       |  2 +-
 docs/reference/search/multi-search.asciidoc   |  8 ++++----
 .../rest-api-spec/test/msearch/11_status.yaml | 20 +++++++++++++++++++
 4 files changed, 30 insertions(+), 5 deletions(-)
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/msearch/11_status.yaml

diff --git a/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java b/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java
index bc1e697646f..bc23c11ceac 100644
--- a/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.action.search;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -28,6 +29,7 @@ import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -155,8 +157,10 @@ public class MultiSearchResponse extends ActionResponse implements Iterable<Mult
             builder.startObject();
             if (item.isFailure()) {
                 ElasticsearchException.renderThrowable(builder, params, item.getFailure());
+                builder.field(Fields.STATUS, ExceptionsHelper.status(item.getFailure()).getStatus());
             } else {
                 item.getResponse().toXContent(builder, params);
+                builder.field(Fields.STATUS, item.getResponse().status().getStatus());
             }
             builder.endObject();
         }
@@ -166,6 +170,7 @@ public class MultiSearchResponse extends ActionResponse implements Iterable<Mult
 
     static final class Fields {
         static final String RESPONSES = "responses";
+        static final String STATUS = "status";
         static final String ERROR = "error";
         static final String ROOT_CAUSE = "root_cause";
     }
diff --git a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
index 2ae23774bdf..1039df68645 100644
--- a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
@@ -160,7 +160,7 @@ public class MultiSearchRequestTests extends ESTestCase {
         MultiSearchResponse response = new MultiSearchResponse(new MultiSearchResponse.Item[]{new MultiSearchResponse.Item(null, new IllegalStateException("foobar")), new MultiSearchResponse.Item(null, new IllegalStateException("baaaaaazzzz"))});
         XContentBuilder builder = XContentFactory.jsonBuilder();
         response.toXContent(builder, ToXContent.EMPTY_PARAMS);
-        assertEquals("\"responses\"[{\"error\":{\"root_cause\":[{\"type\":\"illegal_state_exception\",\"reason\":\"foobar\"}],\"type\":\"illegal_state_exception\",\"reason\":\"foobar\"}},{\"error\":{\"root_cause\":[{\"type\":\"illegal_state_exception\",\"reason\":\"baaaaaazzzz\"}],\"type\":\"illegal_state_exception\",\"reason\":\"baaaaaazzzz\"}}]",
+        assertEquals("\"responses\"[{\"error\":{\"root_cause\":[{\"type\":\"illegal_state_exception\",\"reason\":\"foobar\"}],\"type\":\"illegal_state_exception\",\"reason\":\"foobar\"},\"status\":500},{\"error\":{\"root_cause\":[{\"type\":\"illegal_state_exception\",\"reason\":\"baaaaaazzzz\"}],\"type\":\"illegal_state_exception\",\"reason\":\"baaaaaazzzz\"},\"status\":500}]",
                 builder.string());
     }
 
diff --git a/docs/reference/search/multi-search.asciidoc b/docs/reference/search/multi-search.asciidoc
index 7ecc0f6554c..d32cd45d87d 100644
--- a/docs/reference/search/multi-search.asciidoc
+++ b/docs/reference/search/multi-search.asciidoc
@@ -42,10 +42,10 @@ Note, the above includes an example of an empty header (can also be just
 without any content) which is supported as well.
 
 The response returns a `responses` array, which includes the search
-response for each search request matching its order in the original
-multi search request. If there was a complete failure for that specific
-search request, an object with `error` message will be returned in place
-of the actual search response.
+response and status code for each search request matching its order in
+the original multi search request. If there was a complete failure for that
+specific search request, an object with `error` message and corresponding
+status code will be returned in place of the actual search response.
 
 The endpoint allows to also search against an index/indices and
 type/types in the URI itself, in which case it will be used as the
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/11_status.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/11_status.yaml
new file mode 100644
index 00000000000..525ea7ac421
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/11_status.yaml
@@ -0,0 +1,20 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: test_1
+
+---
+"Check Status":
+  - do:
+      msearch:
+        body:
+          - index: test_2
+          - query:
+              match_all: {}
+          - index: test_1
+          - query:
+              match_all: {}
+
+  - match: { responses.0.status: 404 }
+  - match: { responses.1.status: 200 }

From 18ff051ad508e5234d00429e0ae22f6e16758bb0 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Thu, 16 Jun 2016 09:35:13 +0200
Subject: [PATCH 0900/1311] Simplify ScriptModule and script registration
 (#18903)

Registering a script engine or native scripts still uses Guice today
and is much more complicated than needed. This change moves to a pull
based model where script plugins have to implement a dedicated interface
`ScriptPlugin` and defines simple getter returning instances rather than
classes.
---
 .../java/org/elasticsearch/node/Node.java     |  4 +-
 .../org/elasticsearch/plugins/Plugin.java     | 10 ++-
 .../elasticsearch/plugins/PluginsService.java | 10 +++
 .../elasticsearch/plugins/ScriptPlugin.java   | 55 +++++++++++++
 .../script/NativeScriptEngineService.java     |  9 ++-
 .../script/NativeScriptFactory.java           |  9 ++-
 .../script/ScriptContextRegistry.java         |  5 +-
 .../script/ScriptEngineRegistry.java          | 70 ++++-------------
 .../script/ScriptEngineService.java           |  9 ++-
 .../elasticsearch/script/ScriptModule.java    | 77 +++++++------------
 .../elasticsearch/script/ScriptService.java   |  7 +-
 .../elasticsearch/index/IndexModuleTests.java |  6 +-
 .../index/WaitUntilRefreshIT.java             | 15 +++-
 .../plugins/PluginsServiceTests.java          | 26 +++++++
 .../elasticsearch/script/FileScriptTests.java |  5 +-
 .../script/NativeScriptTests.java             | 16 ++--
 .../script/ScriptContextTests.java            |  7 +-
 .../elasticsearch/script/ScriptFieldIT.java   | 35 +++++++--
 .../script/ScriptModesTests.java              |  7 +-
 .../script/ScriptServiceTests.java            | 19 ++---
 .../script/ScriptSettingsTests.java           | 10 +--
 .../elasticsearch/search/SearchTimeoutIT.java | 14 +++-
 .../aggregations/AggregatorParsingTests.java  | 33 +-------
 .../aggregations/BaseAggregationTestCase.java | 33 +-------
 .../BasePipelineAggregationTestCase.java      | 32 +-------
 .../aggregations/bucket/DateScriptMocks.java  | 21 ++++-
 .../search/aggregations/bucket/IpRangeIT.java | 14 +++-
 .../SignificantTermsSignificanceScoreIT.java  | 11 ++-
 .../search/aggregations/metrics/AvgIT.java    | 25 +++---
 .../search/aggregations/metrics/SumIT.java    | 31 +++++---
 .../aggregations/metrics/ValueCountIT.java    | 15 +++-
 .../builder/SearchSourceBuilderTests.java     | 32 +-------
 .../functionscore/ExplainableScriptIT.java    |  5 ++
 .../ExplainableScriptPlugin.java              | 13 +++-
 .../search/sort/AbstractSortTestCase.java     |  5 +-
 .../update/UpdateByNativeScriptIT.java        | 17 +++-
 .../org/elasticsearch/update/UpdateIT.java    | 53 ++++++++-----
 .../script/expression/ExpressionPlugin.java   | 11 ++-
 .../ExpressionScriptEngineService.java        | 22 +++---
 .../script/groovy/GroovyPlugin.java           | 10 ++-
 .../groovy/GroovyScriptEngineService.java     |  4 -
 .../script/mustache/MustachePlugin.java       | 11 ++-
 .../mustache/MustacheScriptEngineService.java |  8 +-
 .../messy/tests/TemplateQueryParserTests.java |  5 +-
 .../painless/PainlessPlugin.java              | 11 ++-
 .../painless/PainlessScriptEngineService.java | 15 ++--
 .../plugin/javascript/JavaScriptPlugin.java   | 10 ++-
 .../JavaScriptScriptEngineService.java        |  4 -
 .../plugin/python/PythonPlugin.java           | 10 ++-
 .../python/PythonScriptEngineService.java     |  5 --
 .../ingest/AbstractScriptTestCase.java        |  7 +-
 .../script/MockScriptEngine.java              | 24 +++---
 ...NativeSignificanceScoreScriptNoParams.java |  5 ++
 ...tiveSignificanceScoreScriptWithParams.java |  5 ++
 .../test/AbstractQueryTestCase.java           | 33 +-------
 .../org/elasticsearch/test/ESTestCase.java    | 29 +++++++
 56 files changed, 527 insertions(+), 467 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/plugins/ScriptPlugin.java

diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index b6bfba04ff5..2b5fa53a8f9 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -89,6 +89,7 @@ import org.elasticsearch.node.service.NodeService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsModule;
 import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.repositories.RepositoriesModule;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.script.ScriptModule;
@@ -213,7 +214,7 @@ public class Node implements Closeable {
         final NetworkService networkService = new NetworkService(settings);
         final List<ExecutorBuilder<?>> executorBuilders = pluginsService.getExecutorBuilders(settings);
         final ThreadPool threadPool = new ThreadPool(settings, executorBuilders.toArray(new ExecutorBuilder[0]));
-
+        final ScriptModule scriptModule = ScriptModule.create(settings, pluginsService.filterPlugins(ScriptPlugin.class));
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
         boolean success = false;
         try {
@@ -231,7 +232,6 @@ public class Node implements Closeable {
             modules.add(new EnvironmentModule(environment));
             modules.add(new NodeModule(this, monitorService));
             modules.add(new NetworkModule(networkService, settings, false, namedWriteableRegistry));
-            ScriptModule scriptModule = new ScriptModule();
             modules.add(scriptModule);
             modules.add(new NodeEnvironmentModule(nodeEnvironment));
             modules.add(new ClusterNameModule(this.settings));
diff --git a/core/src/main/java/org/elasticsearch/plugins/Plugin.java b/core/src/main/java/org/elasticsearch/plugins/Plugin.java
index 0283567bf80..c619072a6ba 100644
--- a/core/src/main/java/org/elasticsearch/plugins/Plugin.java
+++ b/core/src/main/java/org/elasticsearch/plugins/Plugin.java
@@ -23,8 +23,8 @@ import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexModule;
+import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.threadpool.ExecutorBuilder;
-import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Collection;
 import java.util.Collections;
@@ -84,6 +84,14 @@ public abstract class Plugin {
     @Deprecated
     public final void onModule(IndexModule indexModule) {}
 
+    /**
+     * Old-style guice scripting extension point.
+     *
+     * @deprecated implement {@link ScriptPlugin} instead
+     */
+    @Deprecated
+    public final void onModule(ScriptModule module) {}
+
     /**
      * Provides the list of this plugin's custom thread pools, empty if
      * none.
diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
index 4f123625387..130870ea661 100644
--- a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
+++ b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
@@ -40,6 +40,9 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexModule;
+import org.elasticsearch.script.NativeScriptFactory;
+import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.script.ScriptEngineService;
 import org.elasticsearch.threadpool.ExecutorBuilder;
 
 import java.io.IOException;
@@ -60,6 +63,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.common.io.FileSystemUtils.isAccessibleDirectory;
 
@@ -440,4 +445,9 @@ public class PluginsService extends AbstractComponent {
             throw new ElasticsearchException("Failed to load plugin class [" + pluginClass.getName() + "]", e);
         }
     }
+
+    public <T> List<T> filterPlugins(Class<T> type) {
+        return plugins.stream().filter(x -> type.isAssignableFrom(x.v2().getClass()))
+            .map(p -> ((T)p.v2())).collect(Collectors.toList());
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/plugins/ScriptPlugin.java b/core/src/main/java/org/elasticsearch/plugins/ScriptPlugin.java
new file mode 100644
index 00000000000..7f85b8182a5
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/plugins/ScriptPlugin.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.plugins;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.script.NativeScriptFactory;
+import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.script.ScriptEngineService;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * An additional extension point to {@link Plugin}. Plugins extending the scripting functionality must implement this inteface
+ * to provide access to script engines or script factories.
+ */
+public interface ScriptPlugin {
+
+    /**
+     * Returns a {@link ScriptEngineService} instance or <code>null</code> if this plugin doesn't add a new script engine
+     */
+    default ScriptEngineService getScriptEngineService(Settings settings) {
+        return null;
+    }
+
+    /**
+     * Returns a list of {@link NativeScriptFactory} instances.
+     */
+    default List<NativeScriptFactory> getNativeScripts() {
+        return Collections.emptyList();
+    }
+
+    /**
+     * Returns a {@link ScriptContext.Plugin} instance or <code>null</code> if this plugin doesn't add a new script context plugin
+     */
+    default ScriptContext.Plugin getCustomScriptContexts() {
+        return null;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java b/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java
index 79b9725c338..c9bfdfc1670 100644
--- a/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java
+++ b/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java
@@ -22,13 +22,10 @@ package org.elasticsearch.script;
 import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
 import java.util.Map;
 
 import static java.util.Collections.unmodifiableMap;
@@ -42,7 +39,6 @@ public class NativeScriptEngineService extends AbstractComponent implements Scri
 
     private final Map<String, NativeScriptFactory> scripts;
 
-    @Inject
     public NativeScriptEngineService(Settings settings, Map<String, NativeScriptFactory> scripts) {
         super(settings);
         this.scripts = unmodifiableMap(scripts);
@@ -98,4 +94,9 @@ public class NativeScriptEngineService extends AbstractComponent implements Scri
     public void scriptRemoved(CompiledScript script) {
         // Nothing to do here
     }
+
+    @Override
+    public boolean isInlineScriptEnabled() {
+        return true;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/script/NativeScriptFactory.java b/core/src/main/java/org/elasticsearch/script/NativeScriptFactory.java
index 6768bc4f87a..7fca2501903 100644
--- a/core/src/main/java/org/elasticsearch/script/NativeScriptFactory.java
+++ b/core/src/main/java/org/elasticsearch/script/NativeScriptFactory.java
@@ -43,8 +43,13 @@ public interface NativeScriptFactory {
 
     /**
      * Indicates if document scores may be needed by the produced scripts.
-     * 
+     *
      * @return {@code true} if scores are needed.
      */
     boolean needsScores();
-}
\ No newline at end of file
+
+    /**
+     * Returns the name of the script factory
+     */
+    String getName();
+}
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptContextRegistry.java b/core/src/main/java/org/elasticsearch/script/ScriptContextRegistry.java
index 929575cc3aa..b4ed91faeba 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptContextRegistry.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptContextRegistry.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.script;
 
+import org.elasticsearch.common.settings.Settings;
+
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -30,7 +32,8 @@ import static java.util.Collections.unmodifiableSet;
 
 /**
  * Registry for operations that use scripts as part of their execution. Can be standard operations of custom defined ones (via plugin).
- * Allows plugins to register custom operations that they use scripts for, via {@link ScriptModule#registerScriptContext(org.elasticsearch.script.ScriptContext.Plugin)}.
+ * Allows plugins to register custom operations that they use scripts for,
+ * via {@link org.elasticsearch.plugins.ScriptPlugin}
  * Scripts can be enabled/disabled via fine-grained settings for each single registered operation.
  */
 public final class ScriptContextRegistry {
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptEngineRegistry.java b/core/src/main/java/org/elasticsearch/script/ScriptEngineRegistry.java
index 31ff9ac0b61..f65d694aa31 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptEngineRegistry.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptEngineRegistry.java
@@ -29,29 +29,29 @@ import org.elasticsearch.common.Strings;
 public class ScriptEngineRegistry {
 
     private final Map<Class<? extends ScriptEngineService>, String> registeredScriptEngineServices;
-    private final Map<String, Class<? extends ScriptEngineService>> registeredLanguages;
+    private final Map<String, ScriptEngineService> registeredLanguages;
     private final Map<String, Boolean> defaultInlineScriptEnableds;
 
-    public ScriptEngineRegistry(Iterable<ScriptEngineRegistration> registrations) {
+    public ScriptEngineRegistry(Iterable<ScriptEngineService> registrations) {
         Objects.requireNonNull(registrations);
         Map<Class<? extends ScriptEngineService>, String> registeredScriptEngineServices = new HashMap<>();
-        Map<String, Class<? extends ScriptEngineService>> registeredLanguages = new HashMap<>();
+        Map<String, ScriptEngineService> registeredLanguages = new HashMap<>();
         Map<String, Boolean> inlineScriptEnableds = new HashMap<>();
-        for (ScriptEngineRegistration registration : registrations) {
-            String oldLanguage = registeredScriptEngineServices.putIfAbsent(registration.getScriptEngineService(),
-                    registration.getScriptEngineLanguage());
+        for (ScriptEngineService service : registrations) {
+            String oldLanguage = registeredScriptEngineServices.putIfAbsent(service.getClass(),
+                    service.getType());
             if (oldLanguage != null) {
-                throw new IllegalArgumentException("script engine service [" + registration.getScriptEngineService() +
+                throw new IllegalArgumentException("script engine service [" + service.getClass() +
                                 "] already registered for language [" + oldLanguage + "]");
             }
-            String language = registration.getScriptEngineLanguage();
-            Class<? extends ScriptEngineService> scriptEngineServiceClazz =
-                    registeredLanguages.putIfAbsent(language, registration.getScriptEngineService());
-            if (scriptEngineServiceClazz != null) {
+            String language = service.getType();
+            ScriptEngineService scriptEngineService =
+                    registeredLanguages.putIfAbsent(language, service);
+            if (scriptEngineService != null) {
                 throw new IllegalArgumentException("scripting language [" + language + "] already registered for script engine service [" +
-                                scriptEngineServiceClazz.getCanonicalName() + "]");
+                    scriptEngineService.getClass().getCanonicalName() + "]");
             }
-            inlineScriptEnableds.put(language, registration.getDefaultInlineScriptEnabled());
+            inlineScriptEnableds.put(language, service.isInlineScriptEnabled());
         }
 
         this.registeredScriptEngineServices = Collections.unmodifiableMap(registeredScriptEngineServices);
@@ -68,52 +68,12 @@ public class ScriptEngineRegistry {
         return registeredScriptEngineServices.get(scriptEngineService);
     }
 
-    Map<String, Class<? extends ScriptEngineService>> getRegisteredLanguages() {
+    public Map<String, ScriptEngineService> getRegisteredLanguages() {
         return registeredLanguages;
     }
 
-    Map<String, Boolean> getDefaultInlineScriptEnableds() {
+    public Map<String, Boolean> getDefaultInlineScriptEnableds() {
         return this.defaultInlineScriptEnableds;
     }
 
-    public static class ScriptEngineRegistration {
-        private final Class<? extends ScriptEngineService> scriptEngineService;
-        private final String scriptEngineLanguage;
-        private final boolean defaultInlineScriptEnabled;
-
-        /**
-         * Register a script engine service with the default of inline scripts disabled
-         */
-        public ScriptEngineRegistration(Class<? extends ScriptEngineService> scriptEngineService, String scriptEngineLanguage) {
-            this(scriptEngineService, scriptEngineLanguage, false);
-        }
-
-        /**
-         * Register a script engine service with the given default mode for inline scripts
-         */
-        public ScriptEngineRegistration(Class<? extends ScriptEngineService> scriptEngineService, String scriptEngineLanguage,
-                                        boolean defaultInlineScriptEnabled) {
-            Objects.requireNonNull(scriptEngineService);
-            if (Strings.hasText(scriptEngineLanguage) == false) {
-                throw new IllegalArgumentException("languages for script engine service [" +
-                                scriptEngineService.getCanonicalName() + "] should be a non-empty string");
-            }
-            this.scriptEngineService = scriptEngineService;
-            this.scriptEngineLanguage = scriptEngineLanguage;
-            this.defaultInlineScriptEnabled = defaultInlineScriptEnabled;
-        }
-
-        Class<? extends ScriptEngineService> getScriptEngineService() {
-            return scriptEngineService;
-        }
-
-        String getScriptEngineLanguage() {
-            return scriptEngineLanguage;
-        }
-
-        boolean getDefaultInlineScriptEnabled() {
-            return defaultInlineScriptEnabled;
-        }
-    }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java b/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java
index 2a36cf645e5..ab49c3c8011 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java
@@ -37,7 +37,7 @@ public interface ScriptEngineService extends Closeable {
 
     /**
      * Compiles a script.
-     * @param scriptName name of the script. {@code null} if it is anonymous (inline). 
+     * @param scriptName name of the script. {@code null} if it is anonymous (inline).
      *                                        For a file script, its the file name (with extension).
      *                                        For a stored script, its the identifier.
      * @param scriptSource actual source of the script
@@ -55,4 +55,11 @@ public interface ScriptEngineService extends Closeable {
      * The passed script may be null if it has already been garbage collected.
      * */
     void scriptRemoved(@Nullable CompiledScript script);
+
+    /**
+     * Returns <code>true</code> if this scripting engine can safely accept inline scripts by default. The default is <code>false</code>
+     */
+    default boolean isInlineScriptEnabled() {
+        return false;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptModule.java b/core/src/main/java/org/elasticsearch/script/ScriptModule.java
index c08c31c9bf2..b3c8e46c753 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptModule.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptModule.java
@@ -20,16 +20,17 @@
 package org.elasticsearch.script;
 
 import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.inject.multibindings.MapBinder;
-import org.elasticsearch.common.inject.multibindings.Multibinder;
-import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
+import org.elasticsearch.plugins.ScriptPlugin;
 
-import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 /**
  * An {@link org.elasticsearch.common.inject.Module} which manages {@link ScriptEngineService}s, as well
@@ -37,33 +38,19 @@ import java.util.Objects;
  */
 public class ScriptModule extends AbstractModule {
 
-    private final List<ScriptEngineRegistry.ScriptEngineRegistration> scriptEngineRegistrations = new ArrayList<>();
+    protected final ScriptContextRegistry scriptContextRegistry;
+    protected final ScriptEngineRegistry scriptEngineRegistry;
+    protected final ScriptSettings scriptSettings;
 
-    {
-        scriptEngineRegistrations.add(new ScriptEngineRegistry.ScriptEngineRegistration(NativeScriptEngineService.class,
-                        NativeScriptEngineService.NAME, true));
+    public ScriptModule(ScriptEngineService... services) {
+        this(Arrays.asList(services), Collections.emptyList());
     }
 
-    private final Map<String, Class<? extends NativeScriptFactory>> scripts = new HashMap<>();
-
-    private final List<ScriptContext.Plugin> customScriptContexts = new ArrayList<>();
-
-
-    public void addScriptEngine(ScriptEngineRegistry.ScriptEngineRegistration scriptEngineRegistration) {
-        Objects.requireNonNull(scriptEngineRegistration);
-        scriptEngineRegistrations.add(scriptEngineRegistration);
-    }
-
-    public void registerScript(String name, Class<? extends NativeScriptFactory> script) {
-        scripts.put(name, script);
-    }
-
-    /**
-     * Registers a custom script context that can be used by plugins to categorize the different operations that they use scripts for.
-     * Fine-grained settings allow to enable/disable scripts per context.
-     */
-    public void registerScriptContext(ScriptContext.Plugin scriptContext) {
-        customScriptContexts.add(scriptContext);
+    public ScriptModule(List<ScriptEngineService> scriptEngineServices,
+                        List<ScriptContext.Plugin> customScriptContexts) {
+        this.scriptContextRegistry = new ScriptContextRegistry(customScriptContexts);
+        this.scriptEngineRegistry = new ScriptEngineRegistry(scriptEngineServices);
+        this.scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
     }
 
     /**
@@ -71,10 +58,6 @@ public class ScriptModule extends AbstractModule {
      * script extensions to add all their settings.
      */
     public void prepareSettings(SettingsModule settingsModule) {
-        ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customScriptContexts);
-        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(scriptEngineRegistrations);
-        ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
-
         scriptSettings.getScriptTypeSettings().forEach(settingsModule::registerSetting);
         scriptSettings.getScriptContextSettings().forEach(settingsModule::registerSetting);
         scriptSettings.getScriptLanguageSettings().forEach(settingsModule::registerSetting);
@@ -83,27 +66,21 @@ public class ScriptModule extends AbstractModule {
 
     @Override
     protected void configure() {
-        MapBinder<String, NativeScriptFactory> scriptsBinder
-                = MapBinder.newMapBinder(binder(), String.class, NativeScriptFactory.class);
-        for (Map.Entry<String, Class<? extends NativeScriptFactory>> entry : scripts.entrySet()) {
-            scriptsBinder.addBinding(entry.getKey()).to(entry.getValue()).asEagerSingleton();
-        }
-
-        Multibinder<ScriptEngineService> multibinder = Multibinder.newSetBinder(binder(), ScriptEngineService.class);
-        multibinder.addBinding().to(NativeScriptEngineService.class);
-
-        for (ScriptEngineRegistry.ScriptEngineRegistration scriptEngineRegistration : scriptEngineRegistrations) {
-            multibinder.addBinding().to(scriptEngineRegistration.getScriptEngineService()).asEagerSingleton();
-        }
-
-
-        ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customScriptContexts);
-        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(scriptEngineRegistrations);
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
-
         bind(ScriptContextRegistry.class).toInstance(scriptContextRegistry);
         bind(ScriptEngineRegistry.class).toInstance(scriptEngineRegistry);
         bind(ScriptSettings.class).toInstance(scriptSettings);
         bind(ScriptService.class).asEagerSingleton();
     }
+
+    public static ScriptModule create(Settings settings, List<ScriptPlugin> scriptPlugins) {
+        Map<String, NativeScriptFactory> factoryMap = scriptPlugins.stream().flatMap(x -> x.getNativeScripts().stream())
+            .collect(Collectors.toMap(NativeScriptFactory::getName, Function.identity()));
+        NativeScriptEngineService nativeScriptEngineService = new NativeScriptEngineService(settings, factoryMap);
+        List<ScriptEngineService> scriptEngineServices = scriptPlugins.stream().map(x -> x.getScriptEngineService(settings))
+            .filter(Objects::nonNull).collect(Collectors.toList());
+        scriptEngineServices.add(nativeScriptEngineService);
+        return new ScriptModule(scriptEngineServices, scriptPlugins.stream().map(x -> x.getCustomScriptContexts())
+            .filter(Objects::nonNull).collect(Collectors.toList()));
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java
index 646ad5e25d6..fe3eaa29142 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java
@@ -66,6 +66,7 @@ import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Locale;
@@ -91,7 +92,7 @@ public class ScriptService extends AbstractComponent implements Closeable {
 
     private final String defaultLang;
 
-    private final Set<ScriptEngineService> scriptEngines;
+    private final Collection<ScriptEngineService> scriptEngines;
     private final Map<String, ScriptEngineService> scriptEnginesByLang;
     private final Map<String, ScriptEngineService> scriptEnginesByExt;
 
@@ -132,7 +133,7 @@ public class ScriptService extends AbstractComponent implements Closeable {
     public static final ParseField SCRIPT_INLINE = new ParseField("script");
 
     @Inject
-    public ScriptService(Settings settings, Environment env, Set<ScriptEngineService> scriptEngines,
+    public ScriptService(Settings settings, Environment env,
                          ResourceWatcherService resourceWatcherService, ScriptEngineRegistry scriptEngineRegistry,
                          ScriptContextRegistry scriptContextRegistry, ScriptSettings scriptSettings) throws IOException {
         super(settings);
@@ -145,7 +146,7 @@ public class ScriptService extends AbstractComponent implements Closeable {
                     "Dynamic scripts can be enabled for all languages and all operations by replacing `script.disable_dynamic: false` with `script.inline: true` and `script.stored: true` in elasticsearch.yml");
         }
 
-        this.scriptEngines = scriptEngines;
+        this.scriptEngines = scriptEngineRegistry.getRegisteredLanguages().values();
         this.scriptContextRegistry = scriptContextRegistry;
         int cacheMaxSize = SCRIPT_CACHE_SIZE_SETTING.get(settings);
 
diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
index f552ab9cf17..3441ed58e6d 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
@@ -114,12 +114,10 @@ public class IndexModuleTests extends ESTestCase {
         ThreadPool threadPool = new TestThreadPool("test");
         CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService();
         BigArrays bigArrays = new BigArrays(settings, circuitBreakerService);
-        Set<ScriptEngineService> scriptEngines = Collections.emptySet();
-        scriptEngines.addAll(Arrays.asList(scriptEngineServices));
-        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.emptyList());
+        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Arrays.asList(scriptEngineServices));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
-        ScriptService scriptService = new ScriptService(settings, environment, scriptEngines, new ResourceWatcherService(settings, threadPool), scriptEngineRegistry, scriptContextRegistry, scriptSettings);
+        ScriptService scriptService = new ScriptService(settings, environment, new ResourceWatcherService(settings, threadPool), scriptEngineRegistry, scriptContextRegistry, scriptSettings);
         IndicesQueriesRegistry indicesQueriesRegistry = new IndicesQueriesRegistry();
         ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool);
         return new NodeServicesProvider(threadPool, bigArrays, client, scriptService, indicesQueriesRegistry, circuitBreakerService, clusterService);
diff --git a/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java b/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java
index b2cb2d96818..cb40daa4647 100644
--- a/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java
+++ b/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java
@@ -30,6 +30,7 @@ import org.elasticsearch.action.update.UpdateResponse;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.NativeScriptFactory;
@@ -40,6 +41,8 @@ import org.elasticsearch.test.ESIntegTestCase;
 import org.junit.Before;
 
 import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 
@@ -171,7 +174,7 @@ public class WaitUntilRefreshIT extends ESIntegTestCase {
         return singleton(DeletePlzPlugin.class);
     }
 
-    public static class DeletePlzPlugin extends Plugin {
+    public static class DeletePlzPlugin extends Plugin implements ScriptPlugin {
         @Override
         public String name() {
             return "delete_please";
@@ -182,8 +185,9 @@ public class WaitUntilRefreshIT extends ESIntegTestCase {
             return "adds a script that converts any update into a delete for testing";
         }
 
-        public void onModule(ScriptModule scriptModule) {
-            scriptModule.registerScript("delete_plz", DeletePlzFactory.class);
+        @Override
+        public List<NativeScriptFactory> getNativeScripts() {
+            return Collections.singletonList(new DeletePlzFactory());
         }
     }
 
@@ -213,5 +217,10 @@ public class WaitUntilRefreshIT extends ESIntegTestCase {
         public boolean needsScores() {
             return false;
         }
+
+        @Override
+        public String getName() {
+            return "delete_plz";
+        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java b/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java
index d3c1f1b8bad..9b08cb44b39 100644
--- a/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/plugins/PluginsServiceTests.java
@@ -29,6 +29,7 @@ import org.elasticsearch.test.ESTestCase;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Arrays;
+import java.util.List;
 
 public class PluginsServiceTests extends ESTestCase {
     public static class AdditionalSettingsPlugin1 extends Plugin {
@@ -75,6 +76,20 @@ public class PluginsServiceTests extends ESTestCase {
         }
     }
 
+    public static class FilterablePlugin extends Plugin implements ScriptPlugin {
+        @Override
+        public String name() {
+            return "filter-plugin";
+        }
+        @Override
+        public String description() {
+            return "is filterable";
+        }
+
+    }
+
+
+
     public static class BrokenModule extends AbstractModule {
 
         @Override
@@ -136,4 +151,15 @@ public class PluginsServiceTests extends ESTestCase {
             assertTrue(e.getMessage(), e.getMessage().contains("Could not load plugin descriptor for existing plugin"));
         }
     }
+
+    public void testFilterPlugins() {
+        Settings settings = Settings.builder()
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+            .put("my.setting", "test")
+            .put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.SIMPLEFS.getSettingsKey()).build();
+        PluginsService service = newPluginsService(settings, AdditionalSettingsPlugin1.class, FilterablePlugin.class);
+        List<ScriptPlugin> scriptPlugins = service.filterPlugins(ScriptPlugin.class);
+        assertEquals(1, scriptPlugins.size());
+        assertEquals(FilterablePlugin.class, scriptPlugins.get(0).getClass());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/script/FileScriptTests.java b/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
index fefe55b9ec3..1872ce8f050 100644
--- a/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
@@ -44,11 +44,10 @@ public class FileScriptTests extends ESTestCase {
             .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
             .put(settings)
             .build();
-        Set<ScriptEngineService> engines = new HashSet<>(Collections.singletonList(new MockScriptEngine()));
-        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class, MockScriptEngine.NAME, true)));
+        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singleton(new MockScriptEngine()));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
-        return new ScriptService(settings, new Environment(settings), engines, null, scriptEngineRegistry, scriptContextRegistry, scriptSettings);
+        return new ScriptService(settings, new Environment(settings), null, scriptEngineRegistry, scriptContextRegistry, scriptSettings);
     }
 
     public void testFileScriptFound() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
index e0eed8387a6..c35fb624e1e 100644
--- a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
@@ -52,9 +52,9 @@ public class NativeScriptTests extends ESTestCase {
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                 .build();
         SettingsModule settingsModule = new SettingsModule(settings);
-        ScriptModule scriptModule = new ScriptModule();
+        ScriptModule scriptModule = new ScriptModule(new NativeScriptEngineService(settings,
+            Collections.singletonMap("my", new MyNativeScriptFactory())));
         scriptModule.prepareSettings(settingsModule);
-        scriptModule.registerScript("my", MyNativeScriptFactory.class);
         final ThreadPool threadPool = new ThreadPool(settings);
         Injector injector = new ModulesBuilder().add(
                 new EnvironmentModule(new Environment(settings)),
@@ -85,11 +85,12 @@ public class NativeScriptTests extends ESTestCase {
         ResourceWatcherService resourceWatcherService = new ResourceWatcherService(settings, null);
         Map<String, NativeScriptFactory> nativeScriptFactoryMap = new HashMap<>();
         nativeScriptFactoryMap.put("my", new MyNativeScriptFactory());
-        Set<ScriptEngineService> scriptEngineServices = singleton(new NativeScriptEngineService(settings, nativeScriptFactoryMap));
-        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(NativeScriptEngineService.class, NativeScriptEngineService.NAME, true)));
+        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singleton(new NativeScriptEngineService(settings,
+            nativeScriptFactoryMap)));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(new ArrayList<>());
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
-        ScriptService scriptService = new ScriptService(settings, environment, scriptEngineServices, resourceWatcherService, scriptEngineRegistry, scriptContextRegistry, scriptSettings);
+        ScriptService scriptService = new ScriptService(settings, environment, resourceWatcherService, scriptEngineRegistry,
+            scriptContextRegistry, scriptSettings);
 
         for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) {
             assertThat(scriptService.compile(new Script("my", ScriptType.INLINE, NativeScriptEngineService.NAME, null), scriptContext,
@@ -107,6 +108,11 @@ public class NativeScriptTests extends ESTestCase {
         public boolean needsScores() {
             return false;
         }
+
+        @Override
+        public String getName() {
+            return "my";
+        }
     }
 
     static class MyScript extends AbstractExecutableScript {
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
index bbfdb720814..57bdfff3f7f 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
@@ -43,10 +43,7 @@ public class ScriptContextTests extends ESTestCase {
             .put("script." + PLUGIN_NAME + "_custom_globally_disabled_op", "false")
             .put("script.engine." + MockScriptEngine.NAME + ".inline." + PLUGIN_NAME + "_custom_exp_disabled_op", "false")
             .build();
-        Set<ScriptEngineService> engines = new HashSet<>(Collections.singletonList(new MockScriptEngine()));
-        ScriptEngineRegistry.ScriptEngineRegistration registration =
-                new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class, MockScriptEngine.NAME, true);
-        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(registration));
+        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new MockScriptEngine()));
         List<ScriptContext.Plugin> customContexts = Arrays.asList(
             new ScriptContext.Plugin(PLUGIN_NAME, "custom_op"),
             new ScriptContext.Plugin(PLUGIN_NAME, "custom_exp_disabled_op"),
@@ -54,7 +51,7 @@ public class ScriptContextTests extends ESTestCase {
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts);
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
 
-        return new ScriptService(settings, new Environment(settings), engines, null, scriptEngineRegistry, scriptContextRegistry, scriptSettings);
+        return new ScriptService(settings, new Environment(settings), null, scriptEngineRegistry, scriptContextRegistry, scriptSettings);
     }
 
     public void testCustomGlobalScriptContextSettings() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptFieldIT.java b/core/src/test/java/org/elasticsearch/script/ScriptFieldIT.java
index 8f20d641169..3716b0ce16b 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptFieldIT.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptFieldIT.java
@@ -23,13 +23,16 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 
+import java.util.Arrays;
 import java.util.Collection;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 
@@ -85,6 +88,11 @@ public class ScriptFieldIT extends ESIntegTestCase {
         public boolean needsScores() {
             return false;
         }
+
+        @Override
+        public String getName() {
+            return "int";
+        }
     }
 
     static class IntScript extends AbstractSearchScript {
@@ -104,6 +112,11 @@ public class ScriptFieldIT extends ESIntegTestCase {
         public boolean needsScores() {
             return false;
         }
+
+        @Override
+        public String getName() {
+            return "long";
+        }
     }
 
     static class LongScript extends AbstractSearchScript {
@@ -123,6 +136,11 @@ public class ScriptFieldIT extends ESIntegTestCase {
         public boolean needsScores() {
             return false;
         }
+
+        @Override
+        public String getName() {
+            return "float";
+        }
     }
 
     static class FloatScript extends AbstractSearchScript {
@@ -142,6 +160,11 @@ public class ScriptFieldIT extends ESIntegTestCase {
         public boolean needsScores() {
             return false;
         }
+
+        @Override
+        public String getName() {
+            return "double";
+        }
     }
 
     static class DoubleScript extends AbstractSearchScript {
@@ -151,7 +174,7 @@ public class ScriptFieldIT extends ESIntegTestCase {
         }
     }
 
-    public static class CustomScriptPlugin extends Plugin {
+    public static class CustomScriptPlugin extends Plugin implements ScriptPlugin {
 
         @Override
         public String name() {
@@ -163,12 +186,10 @@ public class ScriptFieldIT extends ESIntegTestCase {
             return "script ";
         }
 
-        public void onModule(ScriptModule scriptModule) {
-            scriptModule.registerScript("int", IntArrayScriptFactory.class);
-            scriptModule.registerScript("long", LongArrayScriptFactory.class);
-            scriptModule.registerScript("float", FloatArrayScriptFactory.class);
-            scriptModule.registerScript("double", DoubleArrayScriptFactory.class);
+        @Override
+        public List<NativeScriptFactory> getNativeScripts() {
+            return Arrays.asList(new IntArrayScriptFactory(), new LongArrayScriptFactory(), new FloatArrayScriptFactory(),
+                new DoubleArrayScriptFactory());
         }
-
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java b/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
index 8f02bee234d..2927b4747dc 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
@@ -27,16 +27,13 @@ import org.elasticsearch.test.ESTestCase;
 import org.junit.After;
 import org.junit.Before;
 
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
 import static java.util.Collections.unmodifiableMap;
-import static java.util.Collections.unmodifiableSet;
 import static org.elasticsearch.common.util.set.Sets.newHashSet;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.notNullValue;
@@ -71,9 +68,7 @@ public class ScriptModesTests extends ESTestCase {
                 //add the native engine just to make sure it gets filtered out
                 new NativeScriptEngineService(Settings.EMPTY, Collections.<String, NativeScriptFactory>emptyMap()),
                 new CustomScriptEngineService()));
-        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Arrays.asList(
-            new ScriptEngineRegistry.ScriptEngineRegistration(NativeScriptEngineService.class, NativeScriptEngineService.NAME),
-            new ScriptEngineRegistry.ScriptEngineRegistration(CustomScriptEngineService.class, CustomScriptEngineService.NAME)));
+        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(scriptEngines.values());
         scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
         checkedSettings = new HashSet<>();
         assertAllSettingsWereChecked = true;
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
index c81c8b10a80..db9e7d66731 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
@@ -41,14 +41,11 @@ import org.junit.Before;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -61,7 +58,6 @@ public class ScriptServiceTests extends ESTestCase {
     private ResourceWatcherService resourceWatcherService;
     private ScriptEngineService scriptEngineService;
     private ScriptEngineService dangerousScriptEngineService;
-    private Set<ScriptEngineService> services;
     private Map<String, ScriptEngineService> scriptEnginesByLangMap;
     private ScriptEngineRegistry scriptEngineRegistry;
     private ScriptContextRegistry scriptContextRegistry;
@@ -89,9 +85,6 @@ public class ScriptServiceTests extends ESTestCase {
         resourceWatcherService = new ResourceWatcherService(baseSettings, null);
         scriptEngineService = new TestEngineService();
         dangerousScriptEngineService = new TestDangerousEngineService();
-        services = new HashSet<>(2);
-        services.add(scriptEngineService);
-        services.add(dangerousScriptEngineService);
         scriptEnginesByLangMap = ScriptModesTests.buildScriptEnginesByLangMap(Collections.singleton(scriptEngineService));
         //randomly register custom script contexts
         int randomInt = randomIntBetween(0, 3);
@@ -109,10 +102,7 @@ public class ScriptServiceTests extends ESTestCase {
             String context = plugin + "_" + operation;
             contexts.put(context, new ScriptContext.Plugin(plugin, operation));
         }
-        List<ScriptEngineRegistry.ScriptEngineRegistration> registries = new ArrayList<>(2);
-        registries.add(new ScriptEngineRegistry.ScriptEngineRegistration(TestEngineService.class, TestEngineService.NAME, true));
-        registries.add(new ScriptEngineRegistry.ScriptEngineRegistration(TestDangerousEngineService.class, TestDangerousEngineService.NAME));
-        scriptEngineRegistry = new ScriptEngineRegistry(registries);
+        scriptEngineRegistry = new ScriptEngineRegistry(Arrays.asList(scriptEngineService, dangerousScriptEngineService));
         scriptContextRegistry = new ScriptContextRegistry(contexts.values());
         scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
         scriptContexts = scriptContextRegistry.scriptContexts().toArray(new ScriptContext[scriptContextRegistry.scriptContexts().size()]);
@@ -124,7 +114,7 @@ public class ScriptServiceTests extends ESTestCase {
     private void buildScriptService(Settings additionalSettings) throws IOException {
         Settings finalSettings = Settings.builder().put(baseSettings).put(additionalSettings).build();
         Environment environment = new Environment(finalSettings);
-        scriptService = new ScriptService(finalSettings, environment, services, resourceWatcherService, scriptEngineRegistry, scriptContextRegistry, scriptSettings) {
+        scriptService = new ScriptService(finalSettings, environment, resourceWatcherService, scriptEngineRegistry, scriptContextRegistry, scriptSettings) {
             @Override
             String getScriptFromClusterState(ClusterState state, String scriptLang, String id) {
                 //mock the script that gets retrieved from an index
@@ -532,6 +522,11 @@ public class ScriptServiceTests extends ESTestCase {
         public void scriptRemoved(CompiledScript script) {
             // Nothing to do here
         }
+
+        @Override
+        public boolean isInlineScriptEnabled() {
+            return true;
+        }
     }
 
     public static class TestDangerousEngineService implements ScriptEngineService {
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java b/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
index 92598ec8dd2..0e2bb384152 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
@@ -20,17 +20,13 @@
 package org.elasticsearch.script;
 
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.test.ESTestCase;
 
-import java.util.Arrays;
 import java.util.Collections;
-import java.util.List;
 import java.util.Map;
 
-import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -38,7 +34,7 @@ public class ScriptSettingsTests extends ESTestCase {
 
     public void testDefaultLanguageIsGroovy() {
         ScriptEngineRegistry scriptEngineRegistry =
-                new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(CustomScriptEngineService.class, CustomScriptEngineService.NAME, true)));
+                new ScriptEngineRegistry(Collections.singletonList(new CustomScriptEngineService()));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
         assertThat(scriptSettings.getDefaultScriptLanguageSetting().get(Settings.EMPTY), equalTo("groovy"));
@@ -46,7 +42,7 @@ public class ScriptSettingsTests extends ESTestCase {
 
     public void testCustomDefaultLanguage() {
         ScriptEngineRegistry scriptEngineRegistry =
-            new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(CustomScriptEngineService.class, CustomScriptEngineService.NAME, true)));
+            new ScriptEngineRegistry(Collections.singletonList(new CustomScriptEngineService()));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
         String defaultLanguage = CustomScriptEngineService.NAME;
@@ -56,7 +52,7 @@ public class ScriptSettingsTests extends ESTestCase {
 
     public void testInvalidDefaultLanguage() {
         ScriptEngineRegistry scriptEngineRegistry =
-            new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(CustomScriptEngineService.class, CustomScriptEngineService.NAME, true)));
+            new ScriptEngineRegistry(Collections.singletonList(new CustomScriptEngineService()));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
         Settings settings = Settings.builder().put("script.default_lang", "C++").build();
diff --git a/core/src/test/java/org/elasticsearch/search/SearchTimeoutIT.java b/core/src/test/java/org/elasticsearch/search/SearchTimeoutIT.java
index b2eb12159f8..f44f125aafc 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchTimeoutIT.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchTimeoutIT.java
@@ -23,6 +23,7 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.script.AbstractSearchScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.NativeScriptFactory;
@@ -33,6 +34,7 @@ import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
@@ -64,7 +66,7 @@ public class SearchTimeoutIT extends ESIntegTestCase {
         assertThat(searchResponse.isTimedOut(), equalTo(true));
     }
 
-    public static class ScriptedTimeoutPlugin extends Plugin {
+    public static class ScriptedTimeoutPlugin extends Plugin implements ScriptPlugin {
         @Override
         public String name() {
             return "test-scripted-search-timeout";
@@ -75,8 +77,9 @@ public class SearchTimeoutIT extends ESIntegTestCase {
             return "Test for scripted timeouts on searches";
         }
 
-        public void onModule(ScriptModule module) {
-            module.registerScript(NativeTestScriptedTimeout.TEST_NATIVE_SCRIPT_TIMEOUT, NativeTestScriptedTimeout.Factory.class);
+        @Override
+        public List<NativeScriptFactory> getNativeScripts() {
+            return Collections.singletonList(new NativeTestScriptedTimeout.Factory());
         }
     }
 
@@ -95,6 +98,11 @@ public class SearchTimeoutIT extends ESIntegTestCase {
             public boolean needsScores() {
                 return false;
             }
+
+            @Override
+            public String getName() {
+                return TEST_NATIVE_SCRIPT_TIMEOUT;
+            }
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
index 8801b3da45e..3b80cb5679f 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
@@ -117,38 +117,7 @@ public class AggregatorParsingTests extends ESTestCase {
                 .put(new IndexMetaData.Builder(index.getName()).settings(indexSettings).numberOfShards(1).numberOfReplicas(0))));
         SettingsModule settingsModule = new SettingsModule(settings);
         settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
-        ScriptModule scriptModule = new ScriptModule() {
-            @Override
-            protected void configure() {
-                Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-                        // no file watching, so we don't need a
-                        // ResourceWatcherService
-                        .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false).build();
-                MockScriptEngine mockScriptEngine = new MockScriptEngine();
-                Multibinder<ScriptEngineService> multibinder = Multibinder.newSetBinder(binder(), ScriptEngineService.class);
-                multibinder.addBinding().toInstance(mockScriptEngine);
-                Set<ScriptEngineService> engines = new HashSet<>();
-                engines.add(mockScriptEngine);
-                List<ScriptContext.Plugin> customContexts = new ArrayList<>();
-                ScriptEngineRegistry scriptEngineRegistry =
-                        new ScriptEngineRegistry(Collections
-                                .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
-                                                                                                 MockScriptEngine.NAME,
-                                                                                                 true)));
-                bind(ScriptEngineRegistry.class).toInstance(scriptEngineRegistry);
-                ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts);
-                bind(ScriptContextRegistry.class).toInstance(scriptContextRegistry);
-                ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
-                bind(ScriptSettings.class).toInstance(scriptSettings);
-                try {
-                    ScriptService scriptService = new ScriptService(settings, new Environment(settings), engines, null,
-                            scriptEngineRegistry, scriptContextRegistry, scriptSettings);
-                    bind(ScriptService.class).toInstance(scriptService);
-                } catch (IOException e) {
-                    throw new IllegalStateException("error while binding ScriptService", e);
-                }
-            }
-        };
+        ScriptModule scriptModule = newTestScriptModule();
         scriptModule.prepareSettings(settingsModule);
         injector = new ModulesBuilder().add(new EnvironmentModule(new Environment(settings)), settingsModule,
                 new ThreadPoolModule(threadPool), scriptModule, new IndicesModule() {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
index c81ee0bb9c2..c73df1bbf06 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
@@ -131,38 +131,7 @@ public abstract class BaseAggregationTestCase<AB extends AbstractAggregationBuil
                 .put(new IndexMetaData.Builder(index.getName()).settings(indexSettings).numberOfShards(1).numberOfReplicas(0))));
         SettingsModule settingsModule = new SettingsModule(settings);
         settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
-        ScriptModule scriptModule = new ScriptModule() {
-            @Override
-            protected void configure() {
-                Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-                        // no file watching, so we don't need a
-                        // ResourceWatcherService
-                        .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false).build();
-                MockScriptEngine mockScriptEngine = new MockScriptEngine();
-                Multibinder<ScriptEngineService> multibinder = Multibinder.newSetBinder(binder(), ScriptEngineService.class);
-                multibinder.addBinding().toInstance(mockScriptEngine);
-                Set<ScriptEngineService> engines = new HashSet<>();
-                engines.add(mockScriptEngine);
-                List<ScriptContext.Plugin> customContexts = new ArrayList<>();
-                ScriptEngineRegistry scriptEngineRegistry =
-                        new ScriptEngineRegistry(Collections
-                                .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
-                                                                                                 MockScriptEngine.NAME,
-                                                                                                 true)));
-                bind(ScriptEngineRegistry.class).toInstance(scriptEngineRegistry);
-                ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts);
-                bind(ScriptContextRegistry.class).toInstance(scriptContextRegistry);
-                ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
-                bind(ScriptSettings.class).toInstance(scriptSettings);
-                try {
-                    ScriptService scriptService = new ScriptService(settings, new Environment(settings), engines, null,
-                            scriptEngineRegistry, scriptContextRegistry, scriptSettings);
-                    bind(ScriptService.class).toInstance(scriptService);
-                } catch (IOException e) {
-                    throw new IllegalStateException("error while binding ScriptService", e);
-                }
-            }
-        };
+        ScriptModule scriptModule = newTestScriptModule();
         scriptModule.prepareSettings(settingsModule);
         injector = new ModulesBuilder().add(
                 new EnvironmentModule(new Environment(settings)),
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
index 078c04d5ffe..2f2354b73a1 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
@@ -131,37 +131,7 @@ public abstract class BasePipelineAggregationTestCase<AF extends AbstractPipelin
                 .put(new IndexMetaData.Builder(index.getName()).settings(indexSettings).numberOfShards(1).numberOfReplicas(0))));
         SettingsModule settingsModule = new SettingsModule(settings);
         settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
-        ScriptModule scriptModule = new ScriptModule() {
-            @Override
-            protected void configure() {
-                Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-                        // no file watching, so we don't need a
-                        // ResourceWatcherService
-                        .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false).build();
-                MockScriptEngine mockScriptEngine = new MockScriptEngine();
-                Multibinder<ScriptEngineService> multibinder = Multibinder.newSetBinder(binder(), ScriptEngineService.class);
-                multibinder.addBinding().toInstance(mockScriptEngine);
-                Set<ScriptEngineService> engines = new HashSet<>();
-                engines.add(mockScriptEngine);
-                List<ScriptContext.Plugin> customContexts = new ArrayList<>();
-                ScriptEngineRegistry scriptEngineRegistry =
-                        new ScriptEngineRegistry(Collections
-                                .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
-                                                                                                 MockScriptEngine.NAME, true)));
-                bind(ScriptEngineRegistry.class).toInstance(scriptEngineRegistry);
-                ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts);
-                bind(ScriptContextRegistry.class).toInstance(scriptContextRegistry);
-                ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
-                bind(ScriptSettings.class).toInstance(scriptSettings);
-                try {
-                    ScriptService scriptService = new ScriptService(settings, new Environment(settings), engines, null,
-                            scriptEngineRegistry, scriptContextRegistry, scriptSettings);
-                    bind(ScriptService.class).toInstance(scriptService);
-                } catch (IOException e) {
-                    throw new IllegalStateException("error while binding ScriptService", e);
-                }
-            }
-        };
+        ScriptModule scriptModule = newTestScriptModule();
         scriptModule.prepareSettings(settingsModule);
         injector = new ModulesBuilder().add(
                 new EnvironmentModule(new Environment(settings)),
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateScriptMocks.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateScriptMocks.java
index f97b163c24c..601632f0ee9 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateScriptMocks.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateScriptMocks.java
@@ -21,6 +21,7 @@ package org.elasticsearch.search.aggregations.bucket;
 
 import org.elasticsearch.common.inject.internal.Nullable;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.script.AbstractSearchScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.NativeScriptFactory;
@@ -28,7 +29,9 @@ import org.elasticsearch.script.ScriptModule;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -39,7 +42,7 @@ public class DateScriptMocks {
     /**
      * Mock plugin for the {@link DateScriptMocks.ExtractFieldScript} and {@link DateScriptMocks.PlusOneMonthScript}
      */
-    public static class DateScriptsMockPlugin extends Plugin {
+    public static class DateScriptsMockPlugin extends Plugin implements ScriptPlugin {
 
         @Override
         public String name() {
@@ -51,9 +54,9 @@ public class DateScriptMocks {
             return "A mock script plugin.";
         }
 
-        public void onModule(ScriptModule module) {
-            module.registerScript(ExtractFieldScript.NAME, ExtractFieldScriptFactory.class);
-            module.registerScript(PlusOneMonthScript.NAME, PlusOneMonthScriptFactory.class);
+        @Override
+        public List<NativeScriptFactory> getNativeScripts() {
+            return Arrays.asList(new ExtractFieldScriptFactory(), new PlusOneMonthScriptFactory());
         }
     }
 
@@ -66,6 +69,11 @@ public class DateScriptMocks {
         public boolean needsScores() {
             return false;
         }
+
+        @Override
+        public String getName() {
+            return ExtractFieldScript.NAME;
+        }
     }
 
     public static class ExtractFieldScript extends AbstractSearchScript {
@@ -94,6 +102,11 @@ public class DateScriptMocks {
         public boolean needsScores() {
             return false;
         }
+
+        @Override
+        public String getName() {
+            return PlusOneMonthScript.NAME;
+        }
     }
 
     /**
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java
index 3a2abda6aa7..44e00f18aef 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java
@@ -25,11 +25,13 @@ import static org.hamcrest.Matchers.containsString;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.inject.internal.Nullable;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.script.AbstractSearchScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.NativeScriptFactory;
@@ -216,7 +218,7 @@ public class IpRangeIT extends ESIntegTestCase {
         assertThat(e.getMessage(), containsString("[ip_range] does not support scripts"));
     }
 
-    public static class DummyScriptPlugin extends Plugin {
+    public static class DummyScriptPlugin extends Plugin implements ScriptPlugin {
 
         @Override
         public String name() {
@@ -228,8 +230,9 @@ public class IpRangeIT extends ESIntegTestCase {
             return "A mock script plugin.";
         }
 
-        public void onModule(ScriptModule module) {
-            module.registerScript(DummyScript.NAME, DummyScriptFactory.class);
+        @Override
+        public List<NativeScriptFactory> getNativeScripts() {
+            return Collections.singletonList(new DummyScriptFactory());
         }
     }
 
@@ -243,6 +246,11 @@ public class IpRangeIT extends ESIntegTestCase {
         public boolean needsScores() {
             return false;
         }
+
+        @Override
+        public String getName() {
+            return DummyScript.NAME;
+        }
     }
 
     private static class DummyScript extends AbstractSearchScript {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
index 0467992002f..0bac7cb40cc 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/SignificantTermsSignificanceScoreIT.java
@@ -30,6 +30,8 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ScriptPlugin;
+import org.elasticsearch.script.NativeScriptFactory;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService.ScriptType;
@@ -52,6 +54,7 @@ import org.elasticsearch.test.search.aggregations.bucket.SharedSignificantTermsT
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -164,7 +167,7 @@ public class SignificantTermsSignificanceScoreIT extends ESIntegTestCase {
         }
     }
 
-    public static class CustomSignificanceHeuristicPlugin extends Plugin {
+    public static class CustomSignificanceHeuristicPlugin extends Plugin implements ScriptPlugin {
         @Override
         public String name() {
             return "test-plugin-significance-heuristic";
@@ -179,9 +182,9 @@ public class SignificantTermsSignificanceScoreIT extends ESIntegTestCase {
             searchModule.registerSignificanceHeuristic(SimpleHeuristic.NAMES_FIELD, SimpleHeuristic::new, SimpleHeuristic::parse);
         }
 
-        public void onModule(ScriptModule module) {
-            module.registerScript(NativeSignificanceScoreScriptNoParams.NATIVE_SIGNIFICANCE_SCORE_SCRIPT_NO_PARAMS, NativeSignificanceScoreScriptNoParams.Factory.class);
-            module.registerScript(NativeSignificanceScoreScriptWithParams.NATIVE_SIGNIFICANCE_SCORE_SCRIPT_WITH_PARAMS, NativeSignificanceScoreScriptWithParams.Factory.class);
+        @Override
+        public List<NativeScriptFactory> getNativeScripts() {
+            return Arrays.asList(new NativeSignificanceScoreScriptNoParams.Factory(), new NativeSignificanceScoreScriptWithParams.Factory());
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
index a19b274c4db..566cd3a5385 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
@@ -21,14 +21,14 @@ package org.elasticsearch.search.aggregations.metrics;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.Scorer;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
-import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
@@ -356,7 +356,7 @@ public class AvgIT extends AbstractNumericTestCase {
     /**
      * Mock plugin for the {@link ExtractFieldScriptEngine}
      */
-    public static class ExtractFieldScriptPlugin extends Plugin {
+    public static class ExtractFieldScriptPlugin extends Plugin implements ScriptPlugin {
 
         @Override
         public String name() {
@@ -368,10 +368,10 @@ public class AvgIT extends AbstractNumericTestCase {
             return "Mock script engine for " + AvgIT.class;
         }
 
-        public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractFieldScriptEngine.class, ExtractFieldScriptEngine.NAME, true));
+        @Override
+        public ScriptEngineService getScriptEngineService(Settings settings) {
+            return new ExtractFieldScriptEngine();
         }
-
     }
 
     /**
@@ -476,7 +476,7 @@ public class AvgIT extends AbstractNumericTestCase {
     /**
      * Mock plugin for the {@link FieldValueScriptEngine}
      */
-    public static class FieldValueScriptPlugin extends Plugin {
+    public static class FieldValueScriptPlugin extends Plugin implements ScriptPlugin {
 
         @Override
         public String name() {
@@ -488,10 +488,10 @@ public class AvgIT extends AbstractNumericTestCase {
             return "Mock script engine for " + AvgIT.class;
         }
 
-        public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.NAME, true));
+        @Override
+        public ScriptEngineService getScriptEngineService(Settings settings) {
+            return new FieldValueScriptEngine();
         }
-
     }
 
     /**
@@ -589,5 +589,10 @@ public class AvgIT extends AbstractNumericTestCase {
         @Override
         public void scriptRemoved(CompiledScript script) {
         }
+
+        @Override
+        public boolean isInlineScriptEnabled() {
+            return true;
+        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
index 641696722c9..57a92d039c5 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
@@ -21,14 +21,14 @@ package org.elasticsearch.search.aggregations.metrics;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.Scorer;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
-import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
@@ -44,7 +44,6 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -351,7 +350,7 @@ public class SumIT extends AbstractNumericTestCase {
     /**
      * Mock plugin for the {@link ExtractFieldScriptEngine}
      */
-    public static class ExtractFieldScriptPlugin extends Plugin {
+    public static class ExtractFieldScriptPlugin extends Plugin implements ScriptPlugin {
 
         @Override
         public String name() {
@@ -363,10 +362,10 @@ public class SumIT extends AbstractNumericTestCase {
             return "Mock script engine for " + SumIT.class;
         }
 
-        public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractFieldScriptEngine.class, ExtractFieldScriptEngine.NAME, true));
+        @Override
+        public ScriptEngineService getScriptEngineService(Settings settings) {
+            return new ExtractFieldScriptEngine();
         }
-
     }
 
     /**
@@ -468,12 +467,17 @@ public class SumIT extends AbstractNumericTestCase {
         @Override
         public void scriptRemoved(CompiledScript script) {
         }
+
+        @Override
+        public boolean isInlineScriptEnabled() {
+            return true;
+        }
     }
 
     /**
      * Mock plugin for the {@link FieldValueScriptEngine}
      */
-    public static class FieldValueScriptPlugin extends Plugin {
+    public static class FieldValueScriptPlugin extends Plugin implements ScriptPlugin {
 
         @Override
         public String name() {
@@ -485,10 +489,10 @@ public class SumIT extends AbstractNumericTestCase {
             return "Mock script engine for " + SumIT.class;
         }
 
-        public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.NAME, true));
+        @Override
+        public ScriptEngineService getScriptEngineService(Settings settings) {
+            return new FieldValueScriptEngine();
         }
-
     }
 
     /**
@@ -594,5 +598,10 @@ public class SumIT extends AbstractNumericTestCase {
         @Override
         public void scriptRemoved(CompiledScript script) {
         }
+
+        @Override
+        public boolean isInlineScriptEnabled() {
+            return true;
+        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
index 04bef292051..35507fb9858 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
@@ -21,7 +21,9 @@ package org.elasticsearch.search.aggregations.metrics;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.Scorer;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.LeafSearchScript;
@@ -210,7 +212,7 @@ public class ValueCountIT extends ESIntegTestCase {
     /**
      * Mock plugin for the {@link FieldValueScriptEngine}
      */
-    public static class FieldValueScriptPlugin extends Plugin {
+    public static class FieldValueScriptPlugin extends Plugin implements ScriptPlugin {
 
         @Override
         public String name() {
@@ -222,10 +224,10 @@ public class ValueCountIT extends ESIntegTestCase {
             return "Mock script engine for " + ValueCountIT.class;
         }
 
-        public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldValueScriptEngine.class, FieldValueScriptEngine.NAME, true));
+        @Override
+        public ScriptEngineService getScriptEngineService(Settings settings) {
+            return new FieldValueScriptEngine();
         }
-
     }
 
     /**
@@ -330,5 +332,10 @@ public class ValueCountIT extends ESIntegTestCase {
         @Override
         public void scriptRemoved(CompiledScript script) {
         }
+
+        @Override
+        public boolean isInlineScriptEnabled() {
+            return true;
+        }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index 7ef3fa30e96..641cb2c1b85 100644
--- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -138,37 +138,7 @@ public class SearchSourceBuilderTests extends ESTestCase {
                 .put(new IndexMetaData.Builder(index.getName()).settings(indexSettings).numberOfShards(1).numberOfReplicas(0))));
         SettingsModule settingsModule = new SettingsModule(settings);
         settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
-        ScriptModule scriptModule = new ScriptModule() {
-            @Override
-            protected void configure() {
-                Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-                        // no file watching, so we don't need a
-                        // ResourceWatcherService
-                        .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false).build();
-                MockScriptEngine mockScriptEngine = new MockScriptEngine();
-                Multibinder<ScriptEngineService> multibinder = Multibinder.newSetBinder(binder(), ScriptEngineService.class);
-                multibinder.addBinding().toInstance(mockScriptEngine);
-                Set<ScriptEngineService> engines = new HashSet<>();
-                engines.add(mockScriptEngine);
-                List<ScriptContext.Plugin> customContexts = new ArrayList<>();
-                ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections
-                        .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
-                                                                                         MockScriptEngine.NAME,
-                                                                                         true)));
-                bind(ScriptEngineRegistry.class).toInstance(scriptEngineRegistry);
-                ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts);
-                bind(ScriptContextRegistry.class).toInstance(scriptContextRegistry);
-                ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
-                bind(ScriptSettings.class).toInstance(scriptSettings);
-                try {
-                    ScriptService scriptService = new ScriptService(settings, new Environment(settings), engines, null,
-                            scriptEngineRegistry, scriptContextRegistry, scriptSettings);
-                    bind(ScriptService.class).toInstance(scriptService);
-                } catch (IOException e) {
-                    throw new IllegalStateException("error while binding ScriptService", e);
-                }
-            }
-        };
+        ScriptModule scriptModule = newTestScriptModule();
         scriptModule.prepareSettings(settingsModule);
         injector = new ModulesBuilder().add(
                 new EnvironmentModule(new Environment(settings)), settingsModule,
diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java
index 9999f96e0b1..eabae8be055 100644
--- a/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java
+++ b/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java
@@ -101,6 +101,11 @@ public class ExplainableScriptIT extends ESIntegTestCase {
         public boolean needsScores() {
             return true;
         }
+
+        @Override
+        public String getName() {
+            return "native_explainable_script";
+        }
     }
 
     static class MyScript extends AbstractDoubleSearchScript implements ExplainableSearchScript, ExecutableScript {
diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptPlugin.java b/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptPlugin.java
index a32baa526b5..5911db53dbd 100644
--- a/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptPlugin.java
+++ b/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptPlugin.java
@@ -20,9 +20,14 @@
 package org.elasticsearch.search.functionscore;
 
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ScriptPlugin;
+import org.elasticsearch.script.NativeScriptFactory;
 import org.elasticsearch.script.ScriptModule;
 
-public class ExplainableScriptPlugin extends Plugin {
+import java.util.Collections;
+import java.util.List;
+
+public class ExplainableScriptPlugin extends Plugin implements ScriptPlugin {
 
     public ExplainableScriptPlugin() {}
     @Override
@@ -35,7 +40,9 @@ public class ExplainableScriptPlugin extends Plugin {
         return "Native explainable script";
     }
 
-    public void onModule(ScriptModule module) {
-        module.registerScript("native_explainable_script", ExplainableScriptIT.MyNativeScriptFactory.class);
+
+    @Override
+    public List<NativeScriptFactory> getNativeScripts() {
+        return Collections.singletonList(new ExplainableScriptIT.MyNativeScriptFactory());
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
index e964b975bb2..2ac7fd68e2e 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
@@ -95,10 +95,9 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST
                 .build();
         Environment environment = new Environment(baseSettings);
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
-        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry
-                .ScriptEngineRegistration(TestEngineService.class, TestEngineService.NAME)));
+        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new TestEngineService()));
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
-        scriptService = new ScriptService(baseSettings, environment, Collections.singleton(new TestEngineService()),
+        scriptService = new ScriptService(baseSettings, environment,
                 new ResourceWatcherService(baseSettings, null), scriptEngineRegistry, scriptContextRegistry, scriptSettings) {
             @Override
             public CompiledScript compile(Script script, ScriptContext scriptContext, Map<String, String> params, ClusterState state) {
diff --git a/core/src/test/java/org/elasticsearch/update/UpdateByNativeScriptIT.java b/core/src/test/java/org/elasticsearch/update/UpdateByNativeScriptIT.java
index dc10c9751cf..cf38658f9f2 100644
--- a/core/src/test/java/org/elasticsearch/update/UpdateByNativeScriptIT.java
+++ b/core/src/test/java/org/elasticsearch/update/UpdateByNativeScriptIT.java
@@ -20,6 +20,7 @@ package org.elasticsearch.update;
 
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.script.AbstractExecutableScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.NativeScriptEngineService;
@@ -32,7 +33,9 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.hasKey;
@@ -65,8 +68,8 @@ public class UpdateByNativeScriptIT extends ESIntegTestCase {
         assertThat(data.get("foo").toString(), is("SETVALUE"));
     }
 
-    public static class CustomNativeScriptFactory implements NativeScriptFactory {
-        public static class TestPlugin extends Plugin {
+    public static class CustomNativeScriptFactory implements NativeScriptFactory  {
+        public static class TestPlugin extends Plugin implements ScriptPlugin {
             @Override
             public String name() {
                 return "mock-native-script";
@@ -75,8 +78,9 @@ public class UpdateByNativeScriptIT extends ESIntegTestCase {
             public String description() {
                 return "a mock native script for testing";
             }
-            public void onModule(ScriptModule scriptModule) {
-                scriptModule.registerScript("custom", CustomNativeScriptFactory.class);
+            @Override
+            public List<NativeScriptFactory> getNativeScripts() {
+                return Collections.singletonList(new CustomNativeScriptFactory());
             }
         }
         @Override
@@ -87,6 +91,11 @@ public class UpdateByNativeScriptIT extends ESIntegTestCase {
         public boolean needsScores() {
             return false;
         }
+
+        @Override
+        public String getName() {
+            return "custom";
+        }
     }
 
     static class CustomScript extends AbstractExecutableScript {
diff --git a/core/src/test/java/org/elasticsearch/update/UpdateIT.java b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
index 078bcd8cc3f..bb47ce09535 100644
--- a/core/src/test/java/org/elasticsearch/update/UpdateIT.java
+++ b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
@@ -38,12 +38,11 @@ import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.engine.DocumentMissingException;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
-import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.lookup.SearchLookup;
@@ -78,7 +77,7 @@ import static org.hamcrest.Matchers.nullValue;
 
 public class UpdateIT extends ESIntegTestCase {
 
-    public static class PutFieldValuesScriptPlugin extends Plugin {
+    public static class PutFieldValuesScriptPlugin extends Plugin implements ScriptPlugin {
 
         public PutFieldValuesScriptPlugin() {
         }
@@ -93,10 +92,10 @@ public class UpdateIT extends ESIntegTestCase {
             return "Mock script engine for " + UpdateIT.class;
         }
 
-        public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(PutFieldValuesScriptEngine.class, PutFieldValuesScriptEngine.NAME, true));
+        @Override
+        public ScriptEngineService getScriptEngineService(Settings settings) {
+            return new PutFieldValuesScriptEngine();
         }
-
     }
 
     public static class PutFieldValuesScriptEngine implements ScriptEngineService {
@@ -163,9 +162,13 @@ public class UpdateIT extends ESIntegTestCase {
         public void scriptRemoved(CompiledScript script) {
         }
 
+        @Override
+        public boolean isInlineScriptEnabled() {
+            return true;
+        }
     }
 
-    public static class FieldIncrementScriptPlugin extends Plugin {
+    public static class FieldIncrementScriptPlugin extends Plugin implements ScriptPlugin {
 
         public FieldIncrementScriptPlugin() {
         }
@@ -180,10 +183,10 @@ public class UpdateIT extends ESIntegTestCase {
             return "Mock script engine for " + UpdateIT.class;
         }
 
-        public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(FieldIncrementScriptEngine.class, FieldIncrementScriptEngine.NAME, true));
+        @Override
+        public ScriptEngineService getScriptEngineService(Settings settings) {
+            return new FieldIncrementScriptEngine();
         }
-
     }
 
     public static class FieldIncrementScriptEngine implements ScriptEngineService {
@@ -243,9 +246,14 @@ public class UpdateIT extends ESIntegTestCase {
         public void scriptRemoved(CompiledScript script) {
         }
 
+        @Override
+        public boolean isInlineScriptEnabled() {
+            return true;
+        }
+
     }
 
-    public static class ScriptedUpsertScriptPlugin extends Plugin {
+    public static class ScriptedUpsertScriptPlugin extends Plugin implements ScriptPlugin {
 
         public ScriptedUpsertScriptPlugin() {
         }
@@ -260,10 +268,10 @@ public class UpdateIT extends ESIntegTestCase {
             return "Mock script engine for " + UpdateIT.class + ".testScriptedUpsert";
         }
 
-        public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ScriptedUpsertScriptEngine.class, ScriptedUpsertScriptEngine.NAME, true));
+        @Override
+        public ScriptEngineService getScriptEngineService(Settings settings) {
+            return new ScriptedUpsertScriptEngine();
         }
-
     }
 
     public static class ScriptedUpsertScriptEngine implements ScriptEngineService {
@@ -323,9 +331,14 @@ public class UpdateIT extends ESIntegTestCase {
         public void scriptRemoved(CompiledScript script) {
         }
 
+        @Override
+        public boolean isInlineScriptEnabled() {
+            return true;
+        }
+
     }
 
-    public static class ExtractContextInSourceScriptPlugin extends Plugin {
+    public static class ExtractContextInSourceScriptPlugin extends Plugin implements ScriptPlugin {
 
         public ExtractContextInSourceScriptPlugin() {
         }
@@ -340,10 +353,10 @@ public class UpdateIT extends ESIntegTestCase {
             return "Mock script engine for " + UpdateIT.class;
         }
 
-        public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExtractContextInSourceScriptEngine.class, ExtractContextInSourceScriptEngine.NAME, true));
+        @Override
+        public ScriptEngineService getScriptEngineService(Settings settings) {
+            return new ExtractContextInSourceScriptEngine();
         }
-
     }
 
     public static class ExtractContextInSourceScriptEngine implements ScriptEngineService {
@@ -404,6 +417,10 @@ public class UpdateIT extends ESIntegTestCase {
         public void scriptRemoved(CompiledScript script) {
         }
 
+        @Override
+        public boolean isInlineScriptEnabled() {
+            return true;
+        }
     }
 
     @Override
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java
index cd95f13f63a..d2def600f66 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionPlugin.java
@@ -19,11 +19,14 @@
 
 package org.elasticsearch.script.expression;
 
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.script.ScriptEngineRegistry;
+import org.elasticsearch.script.ScriptEngineService;
 import org.elasticsearch.script.ScriptModule;
 
-public class ExpressionPlugin extends Plugin {
+public class ExpressionPlugin extends Plugin implements ScriptPlugin {
 
     @Override
     public String name() {
@@ -35,8 +38,8 @@ public class ExpressionPlugin extends Plugin {
         return "Lucene expressions integration for Elasticsearch";
     }
 
-    public void onModule(ScriptModule module) {
-        module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(ExpressionScriptEngineService.class,
-                        ExpressionScriptEngineService.NAME, true));
+    @Override
+    public ScriptEngineService getScriptEngineService(Settings settings) {
+        return new ExpressionScriptEngineService(settings);
     }
 }
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
index dce8d8a7800..fdede1e77bc 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
@@ -29,7 +29,6 @@ import org.apache.lucene.search.SortField;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
@@ -51,7 +50,6 @@ import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.text.ParseException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
@@ -63,7 +61,6 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
 
     public static final String NAME = "expression";
 
-    @Inject
     public ExpressionScriptEngineService(Settings settings) {
         super(settings);
     }
@@ -122,7 +119,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
         // instead of complicating SimpleBindings (which should stay simple)
         SimpleBindings bindings = new SimpleBindings();
         ReplaceableConstValueSource specialValue = null;
-        
+
         for (String variable : expr.variables) {
             try {
                 if (variable.equals("_score")) {
@@ -191,10 +188,10 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                     }
 
                     IndexFieldData<?> fieldData = lookup.doc().fieldDataService().getForField(fieldType);
-                    
+
                     // delegate valuesource creation based on field's type
                     // there are three types of "fields" to expressions, and each one has a different "api" of variables and methods.
-                    
+
                     final ValueSource valueSource;
                     if (fieldType instanceof BaseGeoPointFieldMapper.GeoPointFieldType) {
                         // geo
@@ -203,7 +200,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                         } else {
                             valueSource = GeoField.getMethod(fieldData, fieldname, methodname);
                         }
-                    } else if (fieldType instanceof LegacyDateFieldMapper.DateFieldType || 
+                    } else if (fieldType instanceof LegacyDateFieldMapper.DateFieldType ||
                             fieldType instanceof DateFieldMapper.DateFieldType) {
                         if (dateAccessor) {
                             // date object
@@ -230,7 +227,7 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                     } else {
                         throw new ParseException("Field [" + fieldname + "] must be numeric, date, or geopoint", 5);
                     }
-                    
+
                     bindings.add(variable, valueSource);
                 }
             } catch (Exception e) {
@@ -238,11 +235,11 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
                 throw convertToScriptException("link error", expr.sourceText, variable, e);
             }
         }
-        
+
         final boolean needsScores = expr.getSortField(bindings, false).needsScores();
         return new ExpressionSearchScript(compiledScript, bindings, specialValue, needsScores);
     }
-    
+
     /**
      * converts a ParseException at compile-time or link-time to a ScriptException
      */
@@ -273,4 +270,9 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
     public void scriptRemoved(CompiledScript script) {
         // Nothing to do
     }
+
+    @Override
+    public boolean isInlineScriptEnabled() {
+        return true;
+    }
 }
diff --git a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyPlugin.java b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyPlugin.java
index e5ebcf12509..3f4a98b27fb 100644
--- a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyPlugin.java
+++ b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyPlugin.java
@@ -19,11 +19,14 @@
 
 package org.elasticsearch.script.groovy;
 
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.script.ScriptEngineRegistry;
+import org.elasticsearch.script.ScriptEngineService;
 import org.elasticsearch.script.ScriptModule;
 
-public class GroovyPlugin extends Plugin {
+public class GroovyPlugin extends Plugin implements ScriptPlugin {
 
     @Override
     public String name() {
@@ -35,7 +38,8 @@ public class GroovyPlugin extends Plugin {
         return "Groovy scripting integration for Elasticsearch";
     }
 
-    public void onModule(ScriptModule module) {
-        module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(GroovyScriptEngineService.class, GroovyScriptEngineService.NAME));
+    @Override
+    public ScriptEngineService getScriptEngineService(Settings settings) {
+        return new GroovyScriptEngineService(settings);
     }
 }
diff --git a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
index ef4f6fd76b6..017e51280c0 100644
--- a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
+++ b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
@@ -41,7 +41,6 @@ import org.elasticsearch.bootstrap.BootstrapInfo;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.hash.MessageDigests;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.ClassPermission;
@@ -61,9 +60,7 @@ import java.nio.charset.StandardCharsets;
 import java.security.AccessControlContext;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
-import java.util.Collections;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 /**
@@ -83,7 +80,6 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
 
     private final GroovyClassLoader loader;
 
-    @Inject
     public GroovyScriptEngineService(Settings settings) {
         super(settings);
 
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
index 5ef3ad766e6..ef7d7a64547 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
@@ -19,11 +19,14 @@
 
 package org.elasticsearch.script.mustache;
 
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.script.ScriptEngineRegistry;
+import org.elasticsearch.script.ScriptEngineService;
 import org.elasticsearch.script.ScriptModule;
 
-public class MustachePlugin extends Plugin {
+public class MustachePlugin extends Plugin implements ScriptPlugin {
 
     @Override
     public String name() {
@@ -35,8 +38,8 @@ public class MustachePlugin extends Plugin {
         return "Mustache scripting integration for Elasticsearch";
     }
 
-    public void onModule(ScriptModule module) {
-        module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(MustacheScriptEngineService.class,
-                        MustacheScriptEngineService.NAME, true));
+    @Override
+    public ScriptEngineService getScriptEngineService(Settings settings) {
+        return new MustacheScriptEngineService(settings);
     }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
index d565f66b8e2..283a72b02af 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
@@ -23,7 +23,6 @@ import com.github.mustachejava.Mustache;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.FastStringReader;
 import org.elasticsearch.common.io.UTF8StreamWriter;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -40,7 +39,6 @@ import java.lang.ref.SoftReference;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.Collections;
-import java.util.List;
 import java.util.Map;
 
 /**
@@ -77,7 +75,6 @@ public final class MustacheScriptEngineService extends AbstractComponent impleme
     /**
      * @param settings automatically wired by Guice.
      * */
-    @Inject
     public MustacheScriptEngineService(Settings settings) {
         super(settings);
     }
@@ -190,4 +187,9 @@ public final class MustacheScriptEngineService extends AbstractComponent impleme
             return result.bytes();
         }
     }
+
+    @Override
+    public boolean isInlineScriptEnabled() {
+        return true;
+    }
 }
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
index ddb330b01a7..54068d1921c 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
@@ -103,10 +103,9 @@ public class TemplateQueryParserTests extends ESTestCase {
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
         Index index = idxSettings.getIndex();
         SettingsModule settingsModule = new SettingsModule(settings);
-        ScriptModule scriptModule = new ScriptModule();
-        scriptModule.prepareSettings(settingsModule);
         // TODO: make this use a mock engine instead of mustache and it will no longer be messy!
-        scriptModule.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(MustacheScriptEngineService.class, MustacheScriptEngineService.NAME, true));
+        ScriptModule scriptModule = new ScriptModule(new MustacheScriptEngineService(settings));
+        scriptModule.prepareSettings(settingsModule);
         settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
         final ThreadPool threadPool = new ThreadPool(settings);
         injector = new ModulesBuilder().add(
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
index 67f889b7a72..eb92c865e9c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessPlugin.java
@@ -20,14 +20,17 @@
 package org.elasticsearch.painless;
 
 
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.script.ScriptEngineRegistry;
+import org.elasticsearch.script.ScriptEngineService;
 import org.elasticsearch.script.ScriptModule;
 
 /**
  * Registers Painless as a plugin.
  */
-public final class PainlessPlugin extends Plugin {
+public final class PainlessPlugin extends Plugin implements ScriptPlugin {
 
     // force to pare our definition at startup (not on the user's first script)
     static {
@@ -44,8 +47,8 @@ public final class PainlessPlugin extends Plugin {
         return "Painless scripting language for Elasticsearch";
     }
 
-    public void onModule(final ScriptModule module) {
-        module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(
-                        PainlessScriptEngineService.class, PainlessScriptEngineService.NAME, true));
+    @Override
+    public ScriptEngineService getScriptEngineService(Settings settings) {
+        return new PainlessScriptEngineService(settings);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
index 0d8d9fa4a8b..bb94785d339 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
@@ -22,7 +22,6 @@ package org.elasticsearch.painless;
 import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.painless.Compiler.Loader;
 import org.elasticsearch.script.CompiledScript;
@@ -79,7 +78,6 @@ public final class PainlessScriptEngineService extends AbstractComponent impleme
      * Constructor.
      * @param settings The settings to initialize the engine with.
      */
-    @Inject
     public PainlessScriptEngineService(final Settings settings) {
         super(settings);
     }
@@ -101,7 +99,7 @@ public final class PainlessScriptEngineService extends AbstractComponent impleme
     public String getExtension() {
         return NAME;
     }
-    
+
     /**
      * When a script is anonymous (inline), we give it this name.
      */
@@ -220,7 +218,7 @@ public final class PainlessScriptEngineService extends AbstractComponent impleme
     public void close() {
         // Nothing to do.
     }
-    
+
     private ScriptException convertToScriptException(String scriptName, String scriptSource, Throwable t) {
         // create a script stack: this is just the script portion
         List<String> scriptStack = new ArrayList<>();
@@ -258,13 +256,18 @@ public final class PainlessScriptEngineService extends AbstractComponent impleme
         }
         throw new ScriptException("compile error", t, scriptStack, scriptSource, PainlessScriptEngineService.NAME);
     }
-    
+
     // very simple heuristic: +/- 25 chars. can be improved later.
     private int getPreviousStatement(String scriptSource, int offset) {
         return Math.max(0, offset - 25);
     }
-    
+
     private int getNextStatement(String scriptSource, int offset) {
         return Math.min(scriptSource.length(), offset + 25);
     }
+
+    @Override
+    public boolean isInlineScriptEnabled() {
+        return true;
+    }
 }
diff --git a/plugins/lang-javascript/src/main/java/org/elasticsearch/plugin/javascript/JavaScriptPlugin.java b/plugins/lang-javascript/src/main/java/org/elasticsearch/plugin/javascript/JavaScriptPlugin.java
index edd7c860244..3643d216da8 100644
--- a/plugins/lang-javascript/src/main/java/org/elasticsearch/plugin/javascript/JavaScriptPlugin.java
+++ b/plugins/lang-javascript/src/main/java/org/elasticsearch/plugin/javascript/JavaScriptPlugin.java
@@ -19,15 +19,18 @@
 
 package org.elasticsearch.plugin.javascript;
 
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.script.ScriptEngineRegistry;
+import org.elasticsearch.script.ScriptEngineService;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.javascript.JavaScriptScriptEngineService;
 
 /**
  *
  */
-public class JavaScriptPlugin extends Plugin {
+public class JavaScriptPlugin extends Plugin implements ScriptPlugin {
 
     static {
         // install rhino policy on plugin init
@@ -44,7 +47,8 @@ public class JavaScriptPlugin extends Plugin {
         return "JavaScript plugin allowing to add javascript scripting support";
     }
 
-    public void onModule(ScriptModule module) {
-        module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(JavaScriptScriptEngineService.class, JavaScriptScriptEngineService.NAME));
+    @Override
+    public ScriptEngineService getScriptEngineService(Settings settings) {
+        return new JavaScriptScriptEngineService(settings);
     }
 }
diff --git a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
index 4dd24454b96..3ef030bdeb0 100644
--- a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
+++ b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
@@ -25,7 +25,6 @@ import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.bootstrap.BootstrapInfo;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.ClassPermission;
 import org.elasticsearch.script.CompiledScript;
@@ -57,8 +56,6 @@ import java.security.AccessController;
 import java.security.CodeSource;
 import java.security.PrivilegedAction;
 import java.security.cert.Certificate;
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
@@ -138,7 +135,6 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements
     /** ensures this engine is initialized */
     public static void init() {}
 
-    @Inject
     public JavaScriptScriptEngineService(Settings settings) {
         super(settings);
 
diff --git a/plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java b/plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java
index ca1d3436c11..9445c24232d 100644
--- a/plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java
+++ b/plugins/lang-python/src/main/java/org/elasticsearch/plugin/python/PythonPlugin.java
@@ -19,15 +19,18 @@
 
 package org.elasticsearch.plugin.python;
 
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.script.ScriptEngineRegistry;
+import org.elasticsearch.script.ScriptEngineService;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.python.PythonScriptEngineService;
 
 /**
  *
  */
-public class PythonPlugin extends Plugin {
+public class PythonPlugin extends Plugin implements ScriptPlugin {
 
     @Override
     public String name() {
@@ -39,7 +42,8 @@ public class PythonPlugin extends Plugin {
         return "Adds support for writing scripts in Python";
     }
 
-    public void onModule(ScriptModule module) {
-        module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(PythonScriptEngineService.class, PythonScriptEngineService.NAME));
+    @Override
+    public ScriptEngineService getScriptEngineService(Settings settings) {
+        return new PythonScriptEngineService(settings);
     }
 }
diff --git a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
index df7e9c07394..26bf1bed6fc 100644
--- a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
+++ b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
@@ -24,7 +24,6 @@ import org.apache.lucene.search.Scorer;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.ClassPermission;
 import org.elasticsearch.script.CompiledScript;
@@ -47,9 +46,6 @@ import java.security.AccessController;
 import java.security.Permissions;
 import java.security.PrivilegedAction;
 import java.security.ProtectionDomain;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
 import java.util.Map;
 
 /**
@@ -63,7 +59,6 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri
 
     private final PythonInterpreter interp;
 
-    @Inject
     public PythonScriptEngineService(Settings settings) {
         super(settings);
 
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java
index b0cae6a693f..77f09d4df37 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java
@@ -44,14 +44,11 @@ public abstract class AbstractScriptTestCase extends ESTestCase {
             .put("path.home", createTempDir())
             .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
             .build();
-        MustacheScriptEngineService mustache = new MustacheScriptEngineService(settings);
-
-        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Arrays.asList(
-            new ScriptEngineRegistry.ScriptEngineRegistration(MustacheScriptEngineService.class, MustacheScriptEngineService.NAME, true)));
+        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Arrays.asList(new MustacheScriptEngineService(settings)));
         ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList());
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
 
-        ScriptService scriptService = new ScriptService(settings, new Environment(settings), Sets.newSet(mustache), null,
+        ScriptService scriptService = new ScriptService(settings, new Environment(settings), null,
                 scriptEngineRegistry, scriptContextRegistry, scriptSettings);
         templateService = new InternalTemplateService(scriptService);
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
index 4e06bbe8b45..33111069ac7 100644
--- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
+++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
@@ -22,16 +22,16 @@ package org.elasticsearch.script;
 import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
 import java.util.Map;
 
 /**
- * A dummy script engine used for testing. Scripts must be a number. Many 
+ * A dummy script engine used for testing. Scripts must be a number. Many
  * tests rely on the fact this thing returns a String as its compiled form.
  * they even try to serialize it over the network!
  */
@@ -51,8 +51,8 @@ public class MockScriptEngine implements ScriptEngineService {
             this.params = params;
         }
     }
-    
-    public static class TestPlugin extends Plugin {
+
+    public static class TestPlugin extends Plugin implements ScriptPlugin {
 
         public TestPlugin() {
         }
@@ -67,11 +67,10 @@ public class MockScriptEngine implements ScriptEngineService {
             return "Mock script engine for integration tests";
         }
 
-        public void onModule(ScriptModule module) {
-            module.addScriptEngine(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
-                            MockScriptEngine.NAME, true));
+        @Override
+        public ScriptEngineService getScriptEngineService(Settings settings) {
+            return new MockScriptEngine();
         }
-
     }
 
     @Override
@@ -91,7 +90,7 @@ public class MockScriptEngine implements ScriptEngineService {
 
     @Override
     public ExecutableScript executable(CompiledScript compiledScript, @Nullable Map<String, Object> vars) {
-        assert compiledScript.compiled() instanceof MockCompiledScript 
+        assert compiledScript.compiled() instanceof MockCompiledScript
           : "do NOT pass compiled scripts from other engines to me, I will fail your test, got: " + compiledScript;
         return new AbstractExecutableScript() {
             @Override
@@ -132,4 +131,9 @@ public class MockScriptEngine implements ScriptEngineService {
     @Override
     public void close() throws IOException {
     }
+
+    @Override
+    public boolean isInlineScriptEnabled() {
+        return true;
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptNoParams.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptNoParams.java
index e770f98b2d5..d37d243e21f 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptNoParams.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptNoParams.java
@@ -40,6 +40,11 @@ public class NativeSignificanceScoreScriptNoParams extends TestScript {
         public boolean needsScores() {
             return false;
         }
+
+        @Override
+        public String getName() {
+            return NATIVE_SIGNIFICANCE_SCORE_SCRIPT_NO_PARAMS;
+        }
     }
 
     private NativeSignificanceScoreScriptNoParams() {
diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptWithParams.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptWithParams.java
index e4fda62c29d..1a79c647b99 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptWithParams.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/bucket/script/NativeSignificanceScoreScriptWithParams.java
@@ -42,6 +42,11 @@ public class NativeSignificanceScoreScriptWithParams extends TestScript {
         public boolean needsScores() {
             return false;
         }
+
+        @Override
+        public String getName() {
+            return NATIVE_SIGNIFICANCE_SCORE_SCRIPT_WITH_PARAMS;
+        }
     }
 
     private NativeSignificanceScoreScriptWithParams(Map<String, Object> params) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
index b7a0d5a16f6..34e673ac126 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
@@ -887,38 +887,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
                     new Class[]{Client.class},
                     clientInvocationHandler);
             NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
-            ScriptModule scriptModule = new ScriptModule() {
-                @Override
-                protected void configure() {
-                    Settings settings = Settings.builder()
-                            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-                            // no file watching, so we don't need a ResourceWatcherService
-                            .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
-                            .build();
-                    MockScriptEngine mockScriptEngine = new MockScriptEngine();
-                    Multibinder<ScriptEngineService> multibinder = Multibinder.newSetBinder(binder(), ScriptEngineService.class);
-                    multibinder.addBinding().toInstance(mockScriptEngine);
-                    Set<ScriptEngineService> engines = new HashSet<>();
-                    engines.add(mockScriptEngine);
-                    List<ScriptContext.Plugin> customContexts = new ArrayList<>();
-                    ScriptEngineRegistry scriptEngineRegistry =
-                            new ScriptEngineRegistry(Collections
-                                    .singletonList(new ScriptEngineRegistry.ScriptEngineRegistration(MockScriptEngine.class,
-                                            MockScriptEngine.NAME, true)));
-                    bind(ScriptEngineRegistry.class).toInstance(scriptEngineRegistry);
-                    ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customContexts);
-                    bind(ScriptContextRegistry.class).toInstance(scriptContextRegistry);
-                    ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
-                    bind(ScriptSettings.class).toInstance(scriptSettings);
-                    try {
-                        ScriptService scriptService = new ScriptService(settings, new Environment(settings), engines, null,
-                                scriptEngineRegistry, scriptContextRegistry, scriptSettings);
-                        bind(ScriptService.class).toInstance(scriptService);
-                    } catch (IOException e) {
-                        throw new IllegalStateException("error while binding ScriptService", e);
-                    }
-                }
-            };
+            ScriptModule scriptModule = newTestScriptModule();
             scriptModule.prepareSettings(settingsModule);
             searchModule = new SearchModule(settings, namedWriteableRegistry) {
                 @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 8f3fd5010e6..727ea93a659 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -57,6 +57,12 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.analysis.AnalysisModule;
+import org.elasticsearch.script.MockScriptEngine;
+import org.elasticsearch.script.ScriptContextRegistry;
+import org.elasticsearch.script.ScriptEngineRegistry;
+import org.elasticsearch.script.ScriptModule;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.ScriptSettings;
 import org.elasticsearch.search.MockSearchService;
 import org.elasticsearch.test.junit.listeners.LoggingListener;
 import org.elasticsearch.test.junit.listeners.ReproduceInfoPrinter;
@@ -772,4 +778,27 @@ public abstract class ESTestCase extends LuceneTestCase {
         final AnalysisService analysisService = analysisModule.buildRegistry().build(IndexSettingsModule.newIndexSettings(index, indexSettings));
         return analysisService;
     }
+
+    public static ScriptModule newTestScriptModule() {
+        return new ScriptModule(new MockScriptEngine()) {
+            @Override
+            protected void configure() {
+                Settings settings = Settings.builder()
+                    .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+                    // no file watching, so we don't need a ResourceWatcherService
+                    .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
+                    .build();
+                bind(ScriptEngineRegistry.class).toInstance(scriptEngineRegistry);
+                bind(ScriptContextRegistry.class).toInstance(scriptContextRegistry);
+                bind(ScriptSettings.class).toInstance(scriptSettings);
+                try {
+                    ScriptService scriptService = new ScriptService(settings, new Environment(settings), null,
+                        scriptEngineRegistry, scriptContextRegistry, scriptSettings);
+                    bind(ScriptService.class).toInstance(scriptService);
+                } catch (IOException e) {
+                    throw new IllegalStateException("error while binding ScriptService", e);
+                }
+            }
+        };
+    }
 }

From 9ffb2ff6ba9a4238b7ce2d2718454d2d7c9c717e Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Wed, 15 Jun 2016 10:35:55 +0200
Subject: [PATCH 0901/1311] Expose half-floats. #18887

They have been implemented in https://issues.apache.org/jira/browse/LUCENE-7289.
Ranges are implemented so that the accuracy loss only occurs at index time,
which means that if you are searching for values between A and B, the query will
match exactly all documents whose value rounded to the closest half-float point
is between A and B.
---
 .../fielddata/IndexNumericFieldData.java      |  1 +
 .../plain/SortedNumericDVIndexFieldData.java  | 93 +++++++++++++++++++
 .../index/mapper/core/NumberFieldMapper.java  | 81 ++++++++++++++++
 .../fieldstats/FieldStatsIntegrationIT.java   | 23 ++++-
 .../fieldstats/FieldStatsTests.java           | 18 ++++
 .../plain/HalfFloatFielddataTests.java        | 82 ++++++++++++++++
 .../mapper/core/NumberFieldTypeTests.java     | 45 ++++++++-
 docs/reference/mapping/types/numeric.asciidoc | 41 ++++++--
 8 files changed, 372 insertions(+), 12 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/index/fielddata/plain/HalfFloatFielddataTests.java

diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java
index 2acea212c4c..abfe0d8e96a 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java
@@ -29,6 +29,7 @@ public interface IndexNumericFieldData extends IndexFieldData<AtomicNumericField
         SHORT(false),
         INT(false),
         LONG(false),
+        HALF_FLOAT(true),
         FLOAT(true),
         DOUBLE(true);
 
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java
index 1577225f173..be877b9c68a 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.fielddata.plain;
 
+import org.apache.lucene.document.HalfFloatPoint;
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.LeafReader;
@@ -61,6 +62,7 @@ public class SortedNumericDVIndexFieldData extends DocValuesIndexFieldData imple
     @Override
     public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
         switch (numericType) {
+            case HALF_FLOAT:
             case FLOAT:
                 return new FloatValuesComparatorSource(this, missingValue, sortMode, nested);
             case DOUBLE:
@@ -87,6 +89,8 @@ public class SortedNumericDVIndexFieldData extends DocValuesIndexFieldData imple
         final String field = fieldName;
 
         switch (numericType) {
+            case HALF_FLOAT:
+                return new SortedNumericHalfFloatFieldData(reader, field);
             case FLOAT:
                 return new SortedNumericFloatFieldData(reader, field);
             case DOUBLE:
@@ -134,6 +138,95 @@ public class SortedNumericDVIndexFieldData extends DocValuesIndexFieldData imple
         }
     }
 
+    /**
+     * FieldData implementation for 16-bit float values.
+     * <p>
+     * Order of values within a document is consistent with
+     * {@link Float#compareTo(Float)}, hence the following reversible
+     * transformation is applied at both index and search:
+     * {@code bits ^ (bits >> 15) & 0x7fff}
+     * <p>
+     * Although the API is multi-valued, most codecs in Lucene specialize
+     * for the case where documents have at most one value. In this case
+     * {@link FieldData#unwrapSingleton(SortedNumericDoubleValues)} will return
+     * the underlying single-valued NumericDoubleValues representation, and
+     * {@link FieldData#unwrapSingletonBits(SortedNumericDoubleValues)} will return
+     * a Bits matching documents that have a real value (as opposed to missing).
+     */
+    static final class SortedNumericHalfFloatFieldData extends AtomicDoubleFieldData {
+        final LeafReader reader;
+        final String field;
+
+        SortedNumericHalfFloatFieldData(LeafReader reader, String field) {
+            super(0L);
+            this.reader = reader;
+            this.field = field;
+        }
+
+        @Override
+        public SortedNumericDoubleValues getDoubleValues() {
+            try {
+                SortedNumericDocValues raw = DocValues.getSortedNumeric(reader, field);
+
+                NumericDocValues single = DocValues.unwrapSingleton(raw);
+                if (single != null) {
+                    return FieldData.singleton(new SingleHalfFloatValues(single), DocValues.unwrapSingletonBits(raw));
+                } else {
+                    return new MultiHalfFloatValues(raw);
+                }
+            } catch (IOException e) {
+                throw new IllegalStateException("Cannot load doc values", e);
+            }
+        }
+
+        @Override
+        public Collection<Accountable> getChildResources() {
+            return Collections.emptyList();
+        }
+    }
+
+    /**
+     * Wraps a NumericDocValues and exposes a single 16-bit float per document.
+     */
+    static final class SingleHalfFloatValues extends NumericDoubleValues {
+        final NumericDocValues in;
+
+        SingleHalfFloatValues(NumericDocValues in) {
+            this.in = in;
+        }
+
+        @Override
+        public double get(int docID) {
+            return HalfFloatPoint.sortableShortToHalfFloat((short) in.get(docID));
+        }
+    }
+
+    /**
+     * Wraps a SortedNumericDocValues and exposes multiple 16-bit floats per document.
+     */
+    static final class MultiHalfFloatValues extends SortedNumericDoubleValues {
+        final SortedNumericDocValues in;
+
+        MultiHalfFloatValues(SortedNumericDocValues in) {
+            this.in = in;
+        }
+
+        @Override
+        public void setDocument(int doc) {
+            in.setDocument(doc);
+        }
+
+        @Override
+        public double valueAt(int index) {
+            return HalfFloatPoint.sortableShortToHalfFloat((short) in.valueAt(index));
+        }
+
+        @Override
+        public int count() {
+            return in.count();
+        }
+    }
+
     /**
      * FieldData implementation for 32-bit float values.
      * <p>
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
index 568b651912a..0f688919783 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
@@ -22,6 +22,7 @@ package org.elasticsearch.index.mapper.core;
 import org.apache.lucene.document.DoublePoint;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FloatPoint;
+import org.apache.lucene.document.HalfFloatPoint;
 import org.apache.lucene.document.IntPoint;
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.SortedNumericDocValuesField;
@@ -180,6 +181,86 @@ public class NumberFieldMapper extends FieldMapper implements AllFieldMapper.Inc
     }
 
     public enum NumberType {
+        HALF_FLOAT("half_float", NumericType.HALF_FLOAT) {
+            @Override
+            Float parse(Object value) {
+                return (Float) FLOAT.parse(value);
+            }
+
+            @Override
+            Float parse(XContentParser parser, boolean coerce) throws IOException {
+                return parser.floatValue(coerce);
+            }
+
+            @Override
+            Query termQuery(String field, Object value) {
+                float v = parse(value);
+                return HalfFloatPoint.newExactQuery(field, v);
+            }
+
+            @Override
+            Query termsQuery(String field, List<Object> values) {
+                float[] v = new float[values.size()];
+                for (int i = 0; i < values.size(); ++i) {
+                    v[i] = parse(values.get(i));
+                }
+                return HalfFloatPoint.newSetQuery(field, v);
+            }
+
+            @Override
+            Query rangeQuery(String field, Object lowerTerm, Object upperTerm,
+                             boolean includeLower, boolean includeUpper) {
+                float l = Float.NEGATIVE_INFINITY;
+                float u = Float.POSITIVE_INFINITY;
+                if (lowerTerm != null) {
+                    l = parse(lowerTerm);
+                    if (includeLower) {
+                        l = Math.nextDown(l);
+                    }
+                    l = HalfFloatPoint.nextUp(l);
+                }
+                if (upperTerm != null) {
+                    u = parse(upperTerm);
+                    if (includeUpper) {
+                        u = Math.nextUp(u);
+                    }
+                    u = HalfFloatPoint.nextDown(u);
+                }
+                return HalfFloatPoint.newRangeQuery(field, l, u);
+            }
+
+            @Override
+            public List<Field> createFields(String name, Number value,
+                                            boolean indexed, boolean docValued, boolean stored) {
+                List<Field> fields = new ArrayList<>();
+                if (indexed) {
+                    fields.add(new HalfFloatPoint(name, value.floatValue()));
+                }
+                if (docValued) {
+                    fields.add(new SortedNumericDocValuesField(name,
+                        HalfFloatPoint.halfFloatToSortableShort(value.floatValue())));
+                }
+                if (stored) {
+                    fields.add(new StoredField(name, value.floatValue()));
+                }
+                return fields;
+            }
+
+            @Override
+            FieldStats.Double stats(IndexReader reader, String fieldName,
+                                    boolean isSearchable, boolean isAggregatable) throws IOException {
+                long size = XPointValues.size(reader, fieldName);
+                if (size == 0) {
+                    return null;
+                }
+                int docCount = XPointValues.getDocCount(reader, fieldName);
+                byte[] min = XPointValues.getMinPackedValue(reader, fieldName);
+                byte[] max = XPointValues.getMaxPackedValue(reader, fieldName);
+                return new FieldStats.Double(reader.maxDoc(),docCount, -1L, size,
+                    isSearchable, isAggregatable,
+                    HalfFloatPoint.decodeDimension(min, 0), HalfFloatPoint.decodeDimension(max, 0));
+            }
+        },
         FLOAT("float", NumericType.FLOAT) {
             @Override
             Float parse(Object value) {
diff --git a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java
index 454c973cc34..e87cc5f6571 100644
--- a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.fieldstats;
 
+import org.apache.lucene.document.HalfFloatPoint;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.fieldstats.FieldStats;
@@ -54,6 +55,7 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
                     "string", "type=text",
                     "date", "type=date",
                     "double", "type=double",
+                    "half_float", "type=half_float",
                     "float", "type=float",
                     "long", "type=long",
                     "integer", "type=integer",
@@ -67,6 +69,7 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
             "string", "type=text,index=false",
             "date", "type=date,index=false",
             "double", "type=double,index=false",
+            "half_float", "type=half_float",
             "float", "type=float,index=false",
             "long", "type=long,index=false",
             "integer", "type=integer,index=false",
@@ -81,6 +84,7 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
             "string", "type=text,index=false",
             "date", "type=date,index=false",
             "double", "type=double,index=false",
+            "half_float", "type=half_float",
             "float", "type=float,index=false",
             "long", "type=long,index=false",
             "integer", "type=integer,index=false",
@@ -97,10 +101,12 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
         long maxInt = Integer.MIN_VALUE;
         long minLong = Long.MAX_VALUE;
         long maxLong = Long.MIN_VALUE;
-        double minFloat = Float.MAX_VALUE;
-        double maxFloat = Float.MIN_VALUE;
-        double minDouble = Double.MAX_VALUE;
-        double maxDouble = Double.MIN_VALUE;
+        double minHalfFloat = Double.POSITIVE_INFINITY;
+        double maxHalfFloat = Double.NEGATIVE_INFINITY;
+        double minFloat = Double.POSITIVE_INFINITY;
+        double maxFloat = Double.NEGATIVE_INFINITY;
+        double minDouble = Double.POSITIVE_INFINITY;
+        double maxDouble = Double.NEGATIVE_INFINITY;
         String minString = new String(Character.toChars(1114111));
         String maxString = "0";
 
@@ -119,6 +125,10 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
             long l = randomLong();
             minLong = Math.min(minLong, l);
             maxLong = Math.max(maxLong, l);
+            float hf = randomFloat();
+            hf = HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(hf));
+            minHalfFloat = Math.min(minHalfFloat, hf);
+            maxHalfFloat = Math.max(maxHalfFloat, hf);
             float f = randomFloat();
             minFloat = Math.min(minFloat, f);
             maxFloat = Math.max(maxFloat, f);
@@ -138,6 +148,7 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
                                 "short", s,
                                 "integer", i,
                                 "long", l,
+                                "half_float", hf,
                                 "float", f,
                                 "double", d,
                                 "string", str)
@@ -147,7 +158,7 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
 
         FieldStatsResponse response = client()
             .prepareFieldStats()
-            .setFields("byte", "short", "integer", "long", "float", "double", "string").get();
+            .setFields("byte", "short", "integer", "long", "half_float", "float", "double", "string").get();
         assertAllSuccessful(response);
 
         for (FieldStats<?> stats : response.getAllFieldStats().values()) {
@@ -164,6 +175,8 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
         assertThat(response.getAllFieldStats().get("integer").getMaxValue(), equalTo(maxInt));
         assertThat(response.getAllFieldStats().get("long").getMinValue(), equalTo(minLong));
         assertThat(response.getAllFieldStats().get("long").getMaxValue(), equalTo(maxLong));
+        assertThat(response.getAllFieldStats().get("half_float").getMinValue(), equalTo(minHalfFloat));
+        assertThat(response.getAllFieldStats().get("half_float").getMaxValue(), equalTo(maxHalfFloat));
         assertThat(response.getAllFieldStats().get("float").getMinValue(), equalTo(minFloat));
         assertThat(response.getAllFieldStats().get("float").getMaxValue(), equalTo(maxFloat));
         assertThat(response.getAllFieldStats().get("double").getMinValue(), equalTo(minDouble));
diff --git a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java
index 57f0f001a83..20db1bf20c4 100644
--- a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java
@@ -108,6 +108,24 @@ public class FieldStatsTests extends ESSingleNodeTestCase {
         assertThat(result.getAllFieldStats().get(fieldName).getMinValueAsString(), equalTo(Double.toString(-1)));
     }
 
+    public void testHalfFloat() {
+        String fieldName = "field";
+        createIndex("test", Settings.EMPTY, "test", fieldName, "type=half_float");
+        for (float value = -1; value <= 9; value++) {
+            client().prepareIndex("test", "test").setSource(fieldName, value).get();
+        }
+        client().admin().indices().prepareRefresh().get();
+
+        FieldStatsResponse result = client().prepareFieldStats().setFields(fieldName).get();
+        assertThat(result.getAllFieldStats().get(fieldName).getMaxDoc(), equalTo(11L));
+        assertThat(result.getAllFieldStats().get(fieldName).getDocCount(), equalTo(11L));
+        assertThat(result.getAllFieldStats().get(fieldName).getDensity(), equalTo(100));
+        assertThat(result.getAllFieldStats().get(fieldName).getMinValue(), equalTo(-1d));
+        assertThat(result.getAllFieldStats().get(fieldName).getMaxValue(), equalTo(9d));
+        assertThat(result.getAllFieldStats().get(fieldName).getMinValueAsString(), equalTo(Float.toString(-1)));
+        assertThat(result.getAllFieldStats().get(fieldName).getMaxValueAsString(), equalTo(Float.toString(9)));
+    }
+
     public void testFloat() {
         String fieldName = "field";
         createIndex("test", Settings.EMPTY, "test", fieldName, "type=float");
diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/plain/HalfFloatFielddataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/plain/HalfFloatFielddataTests.java
new file mode 100644
index 00000000000..8c2d9b684bc
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/fielddata/plain/HalfFloatFielddataTests.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.TestUtil;
+import org.elasticsearch.index.fielddata.FieldData;
+import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
+import org.elasticsearch.index.mapper.core.NumberFieldMapper;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+
+public class HalfFloatFielddataTests extends ESTestCase {
+
+    public void testSingleValued() throws IOException {
+        Directory dir = newDirectory();
+        // we need the default codec to check for singletons
+        IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null).setCodec(TestUtil.getDefaultCodec()));
+        Document doc = new Document();
+        for (IndexableField f : NumberFieldMapper.NumberType.HALF_FLOAT.createFields("half_float", 3f, false, true, false)) {
+            doc.add(f);
+        }
+        w.addDocument(doc);
+        final DirectoryReader dirReader = DirectoryReader.open(w);
+        LeafReader reader = getOnlyLeafReader(dirReader);
+        SortedNumericDoubleValues values = new SortedNumericDVIndexFieldData.SortedNumericHalfFloatFieldData(
+                reader, "half_float").getDoubleValues();
+        assertNotNull(FieldData.unwrapSingleton(values));
+        values.setDocument(0);
+        assertEquals(1, values.count());
+        assertEquals(3f, values.valueAt(0), 0f);
+        IOUtils.close(dirReader, w, dir);
+    }
+
+    public void testMultiValued() throws IOException {
+        Directory dir = newDirectory();
+        IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
+        Document doc = new Document();
+        for (IndexableField f : NumberFieldMapper.NumberType.HALF_FLOAT.createFields("half_float", 3f, false, true, false)) {
+            doc.add(f);
+        }
+        for (IndexableField f : NumberFieldMapper.NumberType.HALF_FLOAT.createFields("half_float", 2f, false, true, false)) {
+            doc.add(f);
+        }
+        w.addDocument(doc);
+        final DirectoryReader dirReader = DirectoryReader.open(w);
+        LeafReader reader = getOnlyLeafReader(dirReader);
+        SortedNumericDoubleValues values = new SortedNumericDVIndexFieldData.SortedNumericHalfFloatFieldData(
+                reader, "half_float").getDoubleValues();
+        assertNull(FieldData.unwrapSingleton(values));
+        values.setDocument(0);
+        assertEquals(2, values.count());
+        assertEquals(2f, values.valueAt(0), 0f);
+        assertEquals(3f, values.valueAt(1), 0f);
+        IOUtils.close(dirReader, w, dir);
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/NumberFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/NumberFieldTypeTests.java
index 22dec5bdde5..4b516434b77 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/core/NumberFieldTypeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/core/NumberFieldTypeTests.java
@@ -21,8 +21,18 @@ package org.elasticsearch.index.mapper.core;
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.FloatPoint;
+import org.apache.lucene.document.HalfFloatPoint;
 import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.index.mapper.FieldTypeTestCase;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType.Relation;
@@ -81,6 +91,7 @@ public class NumberFieldTypeTests extends FieldTypeTestCase {
         assertEquals((short) 3, NumberType.SHORT.parse(3d));
         assertEquals(3, NumberType.INTEGER.parse(3d));
         assertEquals(3L, NumberType.LONG.parse(3d));
+        assertEquals(3f, NumberType.HALF_FLOAT.parse(3d));
         assertEquals(3f, NumberType.FLOAT.parse(3d));
         assertEquals(3d, NumberType.DOUBLE.parse(3d));
 
@@ -103,7 +114,39 @@ public class NumberFieldTypeTests extends FieldTypeTestCase {
         assertEquals("Value [2147483648] is out of range for an integer", e.getMessage());
         e = expectThrows(IllegalArgumentException.class, () -> NumberType.LONG.parse(10000000000000000000d));
         assertEquals("Value [1.0E19] is out of range for a long", e.getMessage());
-        assertEquals(1.1f, NumberType.FLOAT.parse(1.1)); // accuracy loss is expected
+        assertEquals(1.1f, NumberType.HALF_FLOAT.parse(1.1));
+        assertEquals(1.1f, NumberType.FLOAT.parse(1.1));
         assertEquals(1.1d, NumberType.DOUBLE.parse(1.1));
     }
+
+    public void testHalfFloatRange() throws IOException {
+        // make sure the accuracy loss of half floats only occurs at index time
+        // this test checks that searching half floats yields the same results as
+        // searching floats that are rounded to the closest half float
+        Directory dir = newDirectory();
+        IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
+        final int numDocs = 10000;
+        for (int i = 0; i < numDocs; ++i) {
+            Document doc = new Document();
+            float value = (randomFloat() * 2 - 1) * 70000;
+            float rounded = HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(value));
+            doc.add(new HalfFloatPoint("half_float", value));
+            doc.add(new FloatPoint("float", rounded));
+            w.addDocument(doc);
+        }
+        final DirectoryReader reader = DirectoryReader.open(w);
+        w.close();
+        IndexSearcher searcher = newSearcher(reader);
+        final int numQueries = 1000;
+        for (int i = 0; i < numQueries; ++i) {
+            float l = (randomFloat() * 2 - 1) * 70000;
+            float u = (randomFloat() * 2 - 1) * 70000;
+            boolean includeLower = randomBoolean();
+            boolean includeUpper = randomBoolean();
+            Query floatQ = NumberFieldMapper.NumberType.FLOAT.rangeQuery("float", l, u, includeLower, includeUpper);
+            Query halfFloatQ = NumberFieldMapper.NumberType.HALF_FLOAT.rangeQuery("half_float", l, u, includeLower, includeUpper);
+            assertEquals(searcher.count(floatQ), searcher.count(halfFloatQ));
+        }
+        IOUtils.close(reader, dir);
+    }
 }
diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc
index 6982047db34..7148585faf1 100644
--- a/docs/reference/mapping/types/numeric.asciidoc
+++ b/docs/reference/mapping/types/numeric.asciidoc
@@ -4,12 +4,13 @@
 The following numeric types are supported:
 
 [horizontal]
-`long`::    A signed 64-bit integer with a minimum value of +-2^63^+ and a maximum value of +2^63^-1+.
-`integer`:: A signed 32-bit integer with a minimum value of +-2^31^+ and a maximum value of +2^31^-1+.
-`short`::   A signed 16-bit integer with a minimum value of +-32,768+ and a maximum value of +32,767+.
-`byte`::    A signed 8-bit integer with a minimum value of +-128+ and a maximum value of +127+.
-`double`::  A double-precision 64-bit IEEE 754 floating point.
-`float`::   A single-precision 32-bit IEEE 754 floating point.
+`long`::       A signed 64-bit integer with a minimum value of +-2^63^+ and a maximum value of +2^63^-1+.
+`integer`::    A signed 32-bit integer with a minimum value of +-2^31^+ and a maximum value of +2^31^-1+.
+`short`::      A signed 16-bit integer with a minimum value of +-32,768+ and a maximum value of +32,767+.
+`byte`::       A signed 8-bit integer with a minimum value of +-128+ and a maximum value of +127+.
+`double`::     A double-precision 64-bit IEEE 754 floating point.
+`float`::      A single-precision 32-bit IEEE 754 floating point.
+`half_float`:: A half-precision 16-bit IEEE 754 floating point.
 
 Below is an example of configuring a mapping with numeric fields:
 
@@ -33,6 +34,34 @@ PUT my_index
 --------------------------------------------------
 // CONSOLE
 
+==== Which type should I use?
+
+As far as integer types (`byte`, `short`, `integer` and `long`) are concerned,
+you should pick the smallest type which is enough for your use-case. This will
+help indexing and searching be more efficient. Note however that given that
+storage is optimized based on the actual values that are stored, picking one
+type over another one will have no impact on storage requirements.
+
+For floating-point types, picking the smallest type that is enough for the
+use-case will still help indexing and searching be more efficient. However,
+given that floating-point data is hard to compress, it might also have a
+significant impact on storage requirements. Here is a table that compares the
+3 floating-point types that are available in order to help make a decision.
+
+[cols="<,<,<,<",options="header",]
+|=======================================================================
+|Type |Minimum value |Maximum value |Significant bits / digits
+|`double`|+2^-1074^+ |+(2-2^-52^)·2^1023^+ |+53+ / +15.95+
+|`float`|+2^-149^+ |+(2-2^-23^)·2^127^+ |+24+ / +7.22+
+|`half_float`|+2^-24^+ |+65504+ |+11+ / +3.31+
+|=======================================================================
+
+When possible, it is often more efficient to store floating-point data into an
+integer using a scaling factor. For instance, it is more efficient to store
+percentages as integers between 0 and 100 than as floating-point numbers between 0
+and 1. Another example would be prices: it will be more efficient to store prices
+as a number of cents, which is an integer, than as a floating-point number.
+
 [[number-params]]
 ==== Parameters for numeric fields
 

From 01004c72ba306ac104cdd74ed20934dfb59222e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Wed, 15 Jun 2016 18:03:59 +0200
Subject: [PATCH 0902/1311] Improve TimeZoneRoundingTests error messages

Currently the error messages for failing tests in the TimeZoneRoundingTests test
suite are hard to read because they usually report the actual end expected date
in milliseconds utc (e.g. "Expected: <1414270860000L> but: was <1414270800000L>".
This makes failing tests hard to read.

This change introduces a new Matcher that can be used for equality checks for
long dates but reports the error both as a formated date string according to
some time zone and also as the actual long values, so you get messages like
"Expected: 2014-10-26T00:01:00.000+03:00 [1414270860000] but:  was
"2014-10-26T00:00:00.000+03:00 [1414270800000]".

Also clean cleaning up some helper methods and generally simplifying a few test
cases. Otherwise this change shouldn't affect either the scope of the test or
anything about the rounding implementation itself.
---
 .../rounding/TimeZoneRoundingTests.java       | 423 +++++++++---------
 1 file changed, 215 insertions(+), 208 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java b/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
index 6302f9c67b1..e82d37a5cf5 100644
--- a/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
+++ b/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
@@ -23,6 +23,9 @@ import org.elasticsearch.common.rounding.TimeZoneRounding.TimeIntervalRounding;
 import org.elasticsearch.common.rounding.TimeZoneRounding.TimeUnitRounding;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Description;
+import org.hamcrest.Matcher;
+import org.hamcrest.TypeSafeMatcher;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeConstants;
 import org.joda.time.DateTimeZone;
@@ -38,67 +41,70 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo;
 /**
  */
 public class TimeZoneRoundingTests extends ESTestCase {
-    final static DateTimeZone JERUSALEM_TIMEZONE = DateTimeZone.forID("Asia/Jerusalem");
 
     public void testUTCTimeUnitRounding() {
         Rounding tzRounding = TimeZoneRounding.builder(DateTimeUnit.MONTH_OF_YEAR).build();
-        assertThat(tzRounding.round(utc("2009-02-03T01:01:01")), equalTo(utc("2009-02-01T00:00:00.000Z")));
-        assertThat(tzRounding.nextRoundingValue(utc("2009-02-01T00:00:00.000Z")), equalTo(utc("2009-03-01T00:00:00.000Z")));
+        DateTimeZone tz = DateTimeZone.UTC;
+        assertThat(tzRounding.round(time("2009-02-03T01:01:01")), isDate(time("2009-02-01T00:00:00.000Z"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-01T00:00:00.000Z")), isDate(time("2009-03-01T00:00:00.000Z"), tz));
 
         tzRounding = TimeZoneRounding.builder(DateTimeUnit.WEEK_OF_WEEKYEAR).build();
-        assertThat(tzRounding.round(utc("2012-01-10T01:01:01")), equalTo(utc("2012-01-09T00:00:00.000Z")));
-        assertThat(tzRounding.nextRoundingValue(utc("2012-01-09T00:00:00.000Z")), equalTo(utc("2012-01-16T00:00:00.000Z")));
+        assertThat(tzRounding.round(time("2012-01-10T01:01:01")), isDate(time("2012-01-09T00:00:00.000Z"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2012-01-09T00:00:00.000Z")), isDate(time("2012-01-16T00:00:00.000Z"), tz));
 
         tzRounding = TimeZoneRounding.builder(DateTimeUnit.WEEK_OF_WEEKYEAR).offset(-TimeValue.timeValueHours(24).millis()).build();
-        assertThat(tzRounding.round(utc("2012-01-10T01:01:01")), equalTo(utc("2012-01-08T00:00:00.000Z")));
-        assertThat(tzRounding.nextRoundingValue(utc("2012-01-08T00:00:00.000Z")), equalTo(utc("2012-01-15T00:00:00.000Z")));
+        assertThat(tzRounding.round(time("2012-01-10T01:01:01")), isDate(time("2012-01-08T00:00:00.000Z"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2012-01-08T00:00:00.000Z")), isDate(time("2012-01-15T00:00:00.000Z"), tz));
     }
 
     public void testUTCIntervalRounding() {
         Rounding tzRounding = TimeZoneRounding.builder(TimeValue.timeValueHours(12)).build();
-        assertThat(tzRounding.round(utc("2009-02-03T01:01:01")), equalTo(utc("2009-02-03T00:00:00.000Z")));
-        long roundKey = tzRounding.roundKey(utc("2009-02-03T01:01:01"));
-        assertThat(roundKey, equalTo(tzRounding.roundKey(utc("2009-02-03T00:00:00.000Z"))));
-        assertThat(tzRounding.valueForKey(roundKey), equalTo(utc("2009-02-03T00:00:00.000Z")));
-        assertThat(tzRounding.nextRoundingValue(utc("2009-02-03T00:00:00.000Z")), equalTo(utc("2009-02-03T12:00:00.000Z")));
-        assertThat(tzRounding.round(utc("2009-02-03T13:01:01")), equalTo(utc("2009-02-03T12:00:00.000Z")));
-        assertThat(tzRounding.nextRoundingValue(utc("2009-02-03T12:00:00.000Z")), equalTo(utc("2009-02-04T00:00:00.000Z")));
+        DateTimeZone tz = DateTimeZone.UTC;
+        assertThat(tzRounding.round(time("2009-02-03T01:01:01")), isDate(time("2009-02-03T00:00:00.000Z"), tz));
+        long roundKey = tzRounding.roundKey(time("2009-02-03T01:01:01"));
+        assertThat(roundKey, isDate(tzRounding.roundKey(time("2009-02-03T00:00:00.000Z")), tz));
+        assertThat(tzRounding.valueForKey(roundKey), isDate(time("2009-02-03T00:00:00.000Z"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-03T00:00:00.000Z")), isDate(time("2009-02-03T12:00:00.000Z"), tz));
+        assertThat(tzRounding.round(time("2009-02-03T13:01:01")), isDate(time("2009-02-03T12:00:00.000Z"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-03T12:00:00.000Z")), isDate(time("2009-02-04T00:00:00.000Z"), tz));
 
         tzRounding = TimeZoneRounding.builder(TimeValue.timeValueHours(48)).build();
-        assertThat(tzRounding.round(utc("2009-02-03T01:01:01")), equalTo(utc("2009-02-03T00:00:00.000Z")));
-        assertThat(tzRounding.nextRoundingValue(utc("2009-02-03T00:00:00.000Z")), equalTo(utc("2009-02-05T00:00:00.000Z")));
-        assertThat(tzRounding.round(utc("2009-02-05T13:01:01")), equalTo(utc("2009-02-05T00:00:00.000Z")));
-        assertThat(tzRounding.nextRoundingValue(utc("2009-02-05T00:00:00.000Z")), equalTo(utc("2009-02-07T00:00:00.000Z")));
+        assertThat(tzRounding.round(time("2009-02-03T01:01:01")), isDate(time("2009-02-03T00:00:00.000Z"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-03T00:00:00.000Z")), isDate(time("2009-02-05T00:00:00.000Z"), tz));
+        assertThat(tzRounding.round(time("2009-02-05T13:01:01")), isDate(time("2009-02-05T00:00:00.000Z"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-05T00:00:00.000Z")), isDate(time("2009-02-07T00:00:00.000Z"), tz));
     }
 
     /**
      * test TimeIntervalTimeZoneRounding, (interval &lt; 12h) with time zone shift
      */
     public void testTimeIntervalTimeZoneRounding() {
-        Rounding tzRounding = TimeZoneRounding.builder(TimeValue.timeValueHours(6)).timeZone(DateTimeZone.forOffsetHours(-1)).build();
-        assertThat(tzRounding.round(utc("2009-02-03T00:01:01")), equalTo(utc("2009-02-02T19:00:00.000Z")));
-        long roundKey = tzRounding.roundKey(utc("2009-02-03T00:01:01"));
-        assertThat(roundKey, equalTo(tzRounding.roundKey(utc("2009-02-02T19:00:00.000Z"))));
-        assertThat(tzRounding.valueForKey(roundKey), equalTo(utc("2009-02-02T19:00:00.000Z")));
-        assertThat(tzRounding.nextRoundingValue(utc("2009-02-02T19:00:00.000Z")), equalTo(utc("2009-02-03T01:00:00.000Z")));
+        DateTimeZone tz = DateTimeZone.forOffsetHours(-1);
+        Rounding tzRounding = TimeZoneRounding.builder(TimeValue.timeValueHours(6)).timeZone(tz).build();
+        assertThat(tzRounding.round(time("2009-02-03T00:01:01")), isDate(time("2009-02-02T19:00:00.000Z"), tz));
+        long roundKey = tzRounding.roundKey(time("2009-02-03T00:01:01"));
+        assertThat(roundKey, equalTo(tzRounding.roundKey(time("2009-02-02T19:00:00.000Z"))));
+        assertThat(tzRounding.valueForKey(roundKey), isDate(time("2009-02-02T19:00:00.000Z"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-02T19:00:00.000Z")), isDate(time("2009-02-03T01:00:00.000Z"), tz));
 
-        assertThat(tzRounding.round(utc("2009-02-03T13:01:01")), equalTo(utc("2009-02-03T13:00:00.000Z")));
-        assertThat(tzRounding.nextRoundingValue(utc("2009-02-03T13:00:00.000Z")), equalTo(utc("2009-02-03T19:00:00.000Z")));
+        assertThat(tzRounding.round(time("2009-02-03T13:01:01")), isDate(time("2009-02-03T13:00:00.000Z"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-03T13:00:00.000Z")), isDate(time("2009-02-03T19:00:00.000Z"), tz));
     }
 
     /**
      * test DayIntervalTimeZoneRounding, (interval &gt;= 12h) with time zone shift
      */
     public void testDayIntervalTimeZoneRounding() {
-        Rounding tzRounding = TimeZoneRounding.builder(TimeValue.timeValueHours(12)).timeZone(DateTimeZone.forOffsetHours(-8)).build();
-        assertThat(tzRounding.round(utc("2009-02-03T00:01:01")), equalTo(utc("2009-02-02T20:00:00.000Z")));
-        long roundKey = tzRounding.roundKey(utc("2009-02-03T00:01:01"));
-        assertThat(roundKey, equalTo(tzRounding.roundKey(utc("2009-02-02T20:00:00.000Z"))));
-        assertThat(tzRounding.valueForKey(roundKey), equalTo(utc("2009-02-02T20:00:00.000Z")));
-        assertThat(tzRounding.nextRoundingValue(utc("2009-02-02T20:00:00.000Z")), equalTo(utc("2009-02-03T08:00:00.000Z")));
+        DateTimeZone tz = DateTimeZone.forOffsetHours(-8);
+        Rounding tzRounding = TimeZoneRounding.builder(TimeValue.timeValueHours(12)).timeZone(tz).build();
+        assertThat(tzRounding.round(time("2009-02-03T00:01:01")), isDate(time("2009-02-02T20:00:00.000Z"), tz));
+        long roundKey = tzRounding.roundKey(time("2009-02-03T00:01:01"));
+        assertThat(roundKey, isDate(tzRounding.roundKey(time("2009-02-02T20:00:00.000Z")), tz));
+        assertThat(tzRounding.valueForKey(roundKey), isDate(time("2009-02-02T20:00:00.000Z"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-02T20:00:00.000Z")), isDate(time("2009-02-03T08:00:00.000Z"), tz));
 
-        assertThat(tzRounding.round(utc("2009-02-03T13:01:01")), equalTo(utc("2009-02-03T08:00:00.000Z")));
-        assertThat(tzRounding.nextRoundingValue(utc("2009-02-03T08:00:00.000Z")), equalTo(utc("2009-02-03T20:00:00.000Z")));
+        assertThat(tzRounding.round(time("2009-02-03T13:01:01")), isDate(time("2009-02-03T08:00:00.000Z"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-03T08:00:00.000Z")), isDate(time("2009-02-03T20:00:00.000Z"), tz));
     }
 
     public void testDayTimeZoneRounding() {
@@ -109,106 +115,72 @@ public class TimeZoneRoundingTests extends ESTestCase {
         assertThat(tzRounding.nextRoundingValue(0L - TimeValue.timeValueHours(24 + timezoneOffset).millis()), equalTo(0L - TimeValue
                 .timeValueHours(timezoneOffset).millis()));
 
-        tzRounding = TimeZoneRounding.builder(DateTimeUnit.DAY_OF_MONTH).timeZone(DateTimeZone.forID("-08:00")).build();
-        assertThat(tzRounding.round(utc("2012-04-01T04:15:30Z")), equalTo(utc("2012-03-31T08:00:00Z")));
-        assertThat(toUTCDateString(tzRounding.nextRoundingValue(utc("2012-03-31T08:00:00Z"))),
-                equalTo(toUTCDateString(utc("2012-04-01T08:0:00Z"))));
+        DateTimeZone tz = DateTimeZone.forID("-08:00");
+        tzRounding = TimeZoneRounding.builder(DateTimeUnit.DAY_OF_MONTH).timeZone(tz).build();
+        assertThat(tzRounding.round(time("2012-04-01T04:15:30Z")), isDate(time("2012-03-31T08:00:00Z"), tz));
 
-        tzRounding = TimeZoneRounding.builder(DateTimeUnit.MONTH_OF_YEAR).timeZone(DateTimeZone.forID("-08:00")).build();
-        assertThat(tzRounding.round(utc("2012-04-01T04:15:30Z")), equalTo(utc("2012-03-01T08:00:00Z")));
-        assertThat(toUTCDateString(tzRounding.nextRoundingValue(utc("2012-03-01T08:00:00Z"))),
-                equalTo(toUTCDateString(utc("2012-04-01T08:0:00Z"))));
+        tzRounding = TimeZoneRounding.builder(DateTimeUnit.MONTH_OF_YEAR).timeZone(tz).build();
+        assertThat(tzRounding.round(time("2012-04-01T04:15:30Z")), equalTo(time("2012-03-01T08:00:00Z")));
 
         // date in Feb-3rd, but still in Feb-2nd in -02:00 timezone
-        tzRounding = TimeZoneRounding.builder(DateTimeUnit.DAY_OF_MONTH).timeZone(DateTimeZone.forID("-02:00")).build();
-        assertThat(tzRounding.round(utc("2009-02-03T01:01:01")), equalTo(utc("2009-02-02T02:00:00")));
-        long roundKey = tzRounding.roundKey(utc("2009-02-03T01:01:01"));
-        assertThat(roundKey, equalTo(tzRounding.roundKey(utc("2009-02-02T02:00:00.000Z"))));
-        assertThat(tzRounding.valueForKey(roundKey), equalTo(utc("2009-02-02T02:00:00.000Z")));
-        assertThat(tzRounding.nextRoundingValue(utc("2009-02-02T02:00:00")), equalTo(utc("2009-02-03T02:00:00")));
+        tz = DateTimeZone.forID("-02:00");
+        tzRounding = TimeZoneRounding.builder(DateTimeUnit.DAY_OF_MONTH).timeZone(tz).build();
+        assertThat(tzRounding.round(time("2009-02-03T01:01:01")), isDate(time("2009-02-02T02:00:00"), tz));
+        long roundKey = tzRounding.roundKey(time("2009-02-03T01:01:01"));
+        assertThat(roundKey, isDate(tzRounding.roundKey(time("2009-02-02T02:00:00.000Z")), tz));
+        assertThat(tzRounding.valueForKey(roundKey), isDate(time("2009-02-02T02:00:00.000Z"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-02T02:00:00")), isDate(time("2009-02-03T02:00:00"), tz));
 
         // date in Feb-3rd, also in -02:00 timezone
-        tzRounding = TimeZoneRounding.builder(DateTimeUnit.DAY_OF_MONTH).timeZone(DateTimeZone.forID("-02:00")).build();
-        assertThat(tzRounding.round(utc("2009-02-03T02:01:01")), equalTo(utc("2009-02-03T02:00:00")));
-        roundKey = tzRounding.roundKey(utc("2009-02-03T02:01:01"));
-        assertThat(roundKey, equalTo(tzRounding.roundKey(utc("2009-02-03T02:00:00.000Z"))));
-        assertThat(tzRounding.valueForKey(roundKey), equalTo(utc("2009-02-03T02:00:00.000Z")));
-        assertThat(tzRounding.nextRoundingValue(utc("2009-02-03T02:00:00")), equalTo(utc("2009-02-04T02:00:00")));
+        tzRounding = TimeZoneRounding.builder(DateTimeUnit.DAY_OF_MONTH).timeZone(tz).build();
+        assertThat(tzRounding.round(time("2009-02-03T02:01:01")), isDate(time("2009-02-03T02:00:00"), tz));
+        roundKey = tzRounding.roundKey(time("2009-02-03T02:01:01"));
+        assertThat(roundKey, isDate(tzRounding.roundKey(time("2009-02-03T02:00:00.000Z")), tz));
+        assertThat(tzRounding.valueForKey(roundKey), isDate(time("2009-02-03T02:00:00.000Z"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-03T02:00:00")), isDate(time("2009-02-04T02:00:00"), tz));
     }
 
     public void testTimeTimeZoneRounding() {
         // hour unit
-        Rounding tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(DateTimeZone.forOffsetHours(-2)).build();
+        DateTimeZone tz = DateTimeZone.forOffsetHours(-2);
+        Rounding tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(tz).build();
         assertThat(tzRounding.round(0), equalTo(0L));
         assertThat(tzRounding.nextRoundingValue(0L), equalTo(TimeValue.timeValueHours(1L).getMillis()));
 
-        tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(DateTimeZone.forOffsetHours(-2)).build();
-        assertThat(tzRounding.round(utc("2009-02-03T01:01:01")), equalTo(utc("2009-02-03T01:00:00")));
-        assertThat(tzRounding.nextRoundingValue(utc("2009-02-03T01:00:00")), equalTo(utc("2009-02-03T02:00:00")));
+        assertThat(tzRounding.round(time("2009-02-03T01:01:01")), isDate(time("2009-02-03T01:00:00"), tz));
+        assertThat(tzRounding.nextRoundingValue(time("2009-02-03T01:00:00")), isDate(time("2009-02-03T02:00:00"), tz));
     }
 
     public void testTimeUnitRoundingDST() {
         Rounding tzRounding;
         // testing savings to non savings switch
-        tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(DateTimeZone.forID("UTC")).build();
-        assertThat(tzRounding.round(time("2014-10-26T01:01:01", DateTimeZone.forOffsetHours(2))),  // CEST = UTC+2
-                equalTo(time("2014-10-26T01:00:00", DateTimeZone.forOffsetHours(2))));
-        assertThat(tzRounding.nextRoundingValue(time("2014-10-26T01:00:00", DateTimeZone.forOffsetHours(2))),
-                equalTo(time("2014-10-26T02:00:00", DateTimeZone.forOffsetHours(2))));
-        assertThat(tzRounding.nextRoundingValue(time("2014-10-26T02:00:00", DateTimeZone.forOffsetHours(2))),
-                equalTo(time("2014-10-26T03:00:00", DateTimeZone.forOffsetHours(2))));
-
-        tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(DateTimeZone.forID("CET")).build();
-        assertThat(tzRounding.round(time("2014-10-26T01:01:01", DateTimeZone.forOffsetHours(2))),  // CEST = UTC+2
-                equalTo(time("2014-10-26T01:00:00", DateTimeZone.forOffsetHours(2))));
-        assertThat(tzRounding.nextRoundingValue(time("2014-10-26T01:00:00", DateTimeZone.forOffsetHours(2))),
-                equalTo(time("2014-10-26T02:00:00", DateTimeZone.forOffsetHours(2))));
-        assertThat(tzRounding.nextRoundingValue(time("2014-10-26T02:00:00", DateTimeZone.forOffsetHours(2))),
-                equalTo(time("2014-10-26T03:00:00", DateTimeZone.forOffsetHours(2))));
+        DateTimeZone cet = DateTimeZone.forID("CET");
+        tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(cet).build();
+        assertThat(tzRounding.round(time("2014-10-26T01:01:01", cet)), isDate(time("2014-10-26T01:00:00+02:00"), cet));
+        assertThat(tzRounding.nextRoundingValue(time("2014-10-26T01:00:00", cet)),isDate(time("2014-10-26T02:00:00+02:00"), cet));
+        assertThat(tzRounding.nextRoundingValue(time("2014-10-26T02:00:00", cet)), isDate(time("2014-10-26T02:00:00+01:00"), cet));
 
         // testing non savings to savings switch
-        tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(DateTimeZone.forID("UTC")).build();
-        assertThat(tzRounding.round(time("2014-03-30T01:01:01", DateTimeZone.forOffsetHours(1))),  // CET = UTC+1
-                equalTo(time("2014-03-30T01:00:00", DateTimeZone.forOffsetHours(1))));
-        assertThat(tzRounding.nextRoundingValue(time("2014-03-30T01:00:00", DateTimeZone.forOffsetHours(1))),
-                equalTo(time("2014-03-30T02:00:00", DateTimeZone.forOffsetHours(1))));
-        assertThat(tzRounding.nextRoundingValue(time("2014-03-30T02:00:00", DateTimeZone.forOffsetHours(1))),
-                equalTo(time("2014-03-30T03:00:00", DateTimeZone.forOffsetHours(1))));
-
-        tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(DateTimeZone.forID("CET")).build();
-        assertThat(tzRounding.round(time("2014-03-30T01:01:01", DateTimeZone.forOffsetHours(1))),  // CET = UTC+1
-                equalTo(time("2014-03-30T01:00:00", DateTimeZone.forOffsetHours(1))));
-        assertThat(tzRounding.nextRoundingValue(time("2014-03-30T01:00:00", DateTimeZone.forOffsetHours(1))),
-                equalTo(time("2014-03-30T02:00:00", DateTimeZone.forOffsetHours(1))));
-        assertThat(tzRounding.nextRoundingValue(time("2014-03-30T02:00:00", DateTimeZone.forOffsetHours(1))),
-                equalTo(time("2014-03-30T03:00:00", DateTimeZone.forOffsetHours(1))));
+        tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(cet).build();
+        assertThat(tzRounding.round(time("2014-03-30T01:01:01", cet)), isDate(time("2014-03-30T01:00:00+01:00"), cet));
+        assertThat(tzRounding.nextRoundingValue(time("2014-03-30T01:00:00", cet)), isDate(time("2014-03-30T03:00:00", cet), cet));
+        assertThat(tzRounding.nextRoundingValue(time("2014-03-30T03:00:00", cet)), isDate(time("2014-03-30T04:00:00", cet), cet));
 
         // testing non savings to savings switch (America/Chicago)
-        tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(DateTimeZone.forID("UTC")).build();
-        assertThat(tzRounding.round(time("2014-03-09T03:01:01", DateTimeZone.forID("America/Chicago"))),
-                equalTo(time("2014-03-09T03:00:00", DateTimeZone.forID("America/Chicago"))));
+        DateTimeZone chg = DateTimeZone.forID("America/Chicago");
+        Rounding tzRounding_utc = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(DateTimeZone.UTC).build();
+        assertThat(tzRounding.round(time("2014-03-09T03:01:01", chg)), isDate(time("2014-03-09T03:00:00", chg), chg));
 
-        tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(DateTimeZone.forID("America/Chicago")).build();
-        assertThat(tzRounding.round(time("2014-03-09T03:01:01", DateTimeZone.forID("America/Chicago"))),
-                equalTo(time("2014-03-09T03:00:00", DateTimeZone.forID("America/Chicago"))));
+        Rounding tzRounding_chg = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(chg).build();
+        assertThat(tzRounding_chg.round(time("2014-03-09T03:01:01", chg)), isDate(time("2014-03-09T03:00:00", chg), chg));
 
         // testing savings to non savings switch 2013 (America/Chicago)
-        tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(DateTimeZone.forID("UTC")).build();
-        assertThat(tzRounding.round(time("2013-11-03T06:01:01", DateTimeZone.forID("America/Chicago"))),
-                equalTo(time("2013-11-03T06:00:00", DateTimeZone.forID("America/Chicago"))));
-
-        tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(DateTimeZone.forID("America/Chicago")).build();
-        assertThat(tzRounding.round(time("2013-11-03T06:01:01", DateTimeZone.forID("America/Chicago"))),
-                equalTo(time("2013-11-03T06:00:00", DateTimeZone.forID("America/Chicago"))));
+        assertThat(tzRounding_utc.round(time("2013-11-03T06:01:01", chg)), isDate(time("2013-11-03T06:00:00", chg), chg));
+        assertThat(tzRounding_chg.round(time("2013-11-03T06:01:01", chg)), isDate(time("2013-11-03T06:00:00", chg), chg));
 
         // testing savings to non savings switch 2014 (America/Chicago)
-        tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(DateTimeZone.forID("UTC")).build();
-        assertThat(tzRounding.round(time("2014-11-02T06:01:01", DateTimeZone.forID("America/Chicago"))),
-                equalTo(time("2014-11-02T06:00:00", DateTimeZone.forID("America/Chicago"))));
-
-        tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(DateTimeZone.forID("America/Chicago")).build();
-        assertThat(tzRounding.round(time("2014-11-02T06:01:01", DateTimeZone.forID("America/Chicago"))),
-                equalTo(time("2014-11-02T06:00:00", DateTimeZone.forID("America/Chicago"))));
+        assertThat(tzRounding_utc.round(time("2014-11-02T06:01:01", chg)), isDate(time("2014-11-02T06:00:00", chg), chg));
+        assertThat(tzRounding_chg.round(time("2014-11-02T06:01:01", chg)), isDate(time("2014-11-02T06:00:00", chg), chg));
     }
 
     /**
@@ -217,28 +189,28 @@ public class TimeZoneRoundingTests extends ESTestCase {
      * test dates that are exactly on or close to offset changes (e.g. DST) in the chosen time zone.
      *
      * It rounds the test date down and up and performs various checks on the rounding unit interval that is
-     * defined by this. Assumptions tested are described in {@link #assertInterval(long, long, long, TimeZoneRounding)}
+     * defined by this. Assumptions tested are described in {@link #assertInterval(long, long, long, TimeZoneRounding, DateTimeZone)}
      */
     public void testTimeZoneRoundingRandom() {
         for (int i = 0; i < 1000; ++i) {
             DateTimeUnit timeUnit = randomTimeUnit();
-            DateTimeZone timezone = randomDateTimeZone();
-            TimeZoneRounding rounding = new TimeZoneRounding.TimeUnitRounding(timeUnit, timezone);
+            DateTimeZone tz = randomDateTimeZone();
+            TimeZoneRounding rounding = new TimeZoneRounding.TimeUnitRounding(timeUnit, tz);
             long date = Math.abs(randomLong() % (2 * (long) 10e11)); // 1970-01-01T00:00:00Z - 2033-05-18T05:33:20.000+02:00
-            long unitMillis = timeUnit.field(timezone).getDurationField().getUnitMillis();
+            long unitMillis = timeUnit.field(tz).getDurationField().getUnitMillis();
             if (randomBoolean()) {
-                nastyDate(date, timezone, unitMillis);
+                nastyDate(date, tz, unitMillis);
             }
             final long roundedDate = rounding.round(date);
             final long nextRoundingValue = rounding.nextRoundingValue(roundedDate);
 
-            assertInterval(roundedDate, date, nextRoundingValue, rounding);
+            assertInterval(roundedDate, date, nextRoundingValue, rounding, tz);
 
             // check correct unit interval width for units smaller than a day, they should be fixed size except for transitions
             if (unitMillis <= DateTimeConstants.MILLIS_PER_DAY) {
                 // if the interval defined didn't cross timezone offset transition, it should cover unitMillis width
-                if (timezone.getOffset(roundedDate - 1) == timezone.getOffset(nextRoundingValue + 1)) {
-                    assertThat("unit interval width not as expected for [" + timeUnit + "], [" + timezone + "] at "
+                if (tz.getOffset(roundedDate - 1) == tz.getOffset(nextRoundingValue + 1)) {
+                    assertThat("unit interval width not as expected for [" + timeUnit + "], [" + tz + "] at "
                             + new DateTime(roundedDate), nextRoundingValue - roundedDate, equalTo(unitMillis));
                 }
             }
@@ -264,17 +236,18 @@ public class TimeZoneRoundingTests extends ESTestCase {
      */
     public void testTimeIntervalCET_DST_End() {
         long interval = TimeUnit.MINUTES.toMillis(20);
-        TimeZoneRounding rounding = new TimeIntervalRounding(interval, DateTimeZone.forID("CET"));
+        DateTimeZone tz = DateTimeZone.forID("CET");
+        TimeZoneRounding rounding = new TimeIntervalRounding(interval, tz);
 
-        assertThat(rounding.round(time("2015-10-25T01:55:00+02:00")), equalTo(time("2015-10-25T01:40:00+02:00")));
-        assertThat(rounding.round(time("2015-10-25T02:15:00+02:00")), equalTo(time("2015-10-25T02:00:00+02:00")));
-        assertThat(rounding.round(time("2015-10-25T02:35:00+02:00")), equalTo(time("2015-10-25T02:20:00+02:00")));
-        assertThat(rounding.round(time("2015-10-25T02:55:00+02:00")), equalTo(time("2015-10-25T02:40:00+02:00")));
+        assertThat(rounding.round(time("2015-10-25T01:55:00+02:00")), isDate(time("2015-10-25T01:40:00+02:00"), tz));
+        assertThat(rounding.round(time("2015-10-25T02:15:00+02:00")), isDate(time("2015-10-25T02:00:00+02:00"), tz));
+        assertThat(rounding.round(time("2015-10-25T02:35:00+02:00")), isDate(time("2015-10-25T02:20:00+02:00"), tz));
+        assertThat(rounding.round(time("2015-10-25T02:55:00+02:00")), isDate(time("2015-10-25T02:40:00+02:00"), tz));
         // after DST shift
-        assertThat(rounding.round(time("2015-10-25T02:15:00+01:00")), equalTo(time("2015-10-25T02:00:00+01:00")));
-        assertThat(rounding.round(time("2015-10-25T02:35:00+01:00")), equalTo(time("2015-10-25T02:20:00+01:00")));
-        assertThat(rounding.round(time("2015-10-25T02:55:00+01:00")), equalTo(time("2015-10-25T02:40:00+01:00")));
-        assertThat(rounding.round(time("2015-10-25T03:15:00+01:00")), equalTo(time("2015-10-25T03:00:00+01:00")));
+        assertThat(rounding.round(time("2015-10-25T02:15:00+01:00")), isDate(time("2015-10-25T02:00:00+01:00"), tz));
+        assertThat(rounding.round(time("2015-10-25T02:35:00+01:00")), isDate(time("2015-10-25T02:20:00+01:00"), tz));
+        assertThat(rounding.round(time("2015-10-25T02:55:00+01:00")), isDate(time("2015-10-25T02:40:00+01:00"), tz));
+        assertThat(rounding.round(time("2015-10-25T03:15:00+01:00")), isDate(time("2015-10-25T03:00:00+01:00"), tz));
     }
 
     /**
@@ -283,12 +256,13 @@ public class TimeZoneRoundingTests extends ESTestCase {
      */
     public void testTimeIntervalCET_DST_Start() {
         long interval = TimeUnit.MINUTES.toMillis(20);
-        TimeZoneRounding rounding = new TimeIntervalRounding(interval, DateTimeZone.forID("CET"));
+        DateTimeZone tz = DateTimeZone.forID("CET");
+        TimeZoneRounding rounding = new TimeIntervalRounding(interval, tz);
         // test DST start
-        assertThat(rounding.round(time("2016-03-27T01:55:00+01:00")), equalTo(time("2016-03-27T01:40:00+01:00")));
-        assertThat(rounding.round(time("2016-03-27T02:00:00+01:00")), equalTo(time("2016-03-27T03:00:00+02:00")));
-        assertThat(rounding.round(time("2016-03-27T03:15:00+02:00")), equalTo(time("2016-03-27T03:00:00+02:00")));
-        assertThat(rounding.round(time("2016-03-27T03:35:00+02:00")), equalTo(time("2016-03-27T03:20:00+02:00")));
+        assertThat(rounding.round(time("2016-03-27T01:55:00+01:00")), isDate(time("2016-03-27T01:40:00+01:00"), tz));
+        assertThat(rounding.round(time("2016-03-27T02:00:00+01:00")), isDate(time("2016-03-27T03:00:00+02:00"), tz));
+        assertThat(rounding.round(time("2016-03-27T03:15:00+02:00")), isDate(time("2016-03-27T03:00:00+02:00"), tz));
+        assertThat(rounding.round(time("2016-03-27T03:35:00+02:00")), isDate(time("2016-03-27T03:20:00+02:00"), tz));
     }
 
     /**
@@ -299,13 +273,14 @@ public class TimeZoneRoundingTests extends ESTestCase {
      */
     public void testTimeInterval_Kathmandu_DST_Start() {
         long interval = TimeUnit.MINUTES.toMillis(20);
-        TimeZoneRounding rounding = new TimeIntervalRounding(interval, DateTimeZone.forID("Asia/Kathmandu"));
-        assertThat(rounding.round(time("1985-12-31T23:55:00+05:30")), equalTo(time("1985-12-31T23:40:00+05:30")));
-        assertThat(rounding.round(time("1986-01-01T00:16:00+05:45")), equalTo(time("1986-01-01T00:15:00+05:45")));
+        DateTimeZone tz = DateTimeZone.forID("Asia/Kathmandu");
+        TimeZoneRounding rounding = new TimeIntervalRounding(interval, tz);
+        assertThat(rounding.round(time("1985-12-31T23:55:00+05:30")), isDate(time("1985-12-31T23:40:00+05:30"), tz));
+        assertThat(rounding.round(time("1986-01-01T00:16:00+05:45")), isDate(time("1986-01-01T00:15:00+05:45"), tz));
         assertThat(time("1986-01-01T00:15:00+05:45") - time("1985-12-31T23:40:00+05:30"), equalTo(TimeUnit.MINUTES.toMillis(20)));
-        assertThat(rounding.round(time("1986-01-01T00:26:00+05:45")), equalTo(time("1986-01-01T00:20:00+05:45")));
+        assertThat(rounding.round(time("1986-01-01T00:26:00+05:45")), isDate(time("1986-01-01T00:20:00+05:45"), tz));
         assertThat(time("1986-01-01T00:20:00+05:45") - time("1986-01-01T00:15:00+05:45"), equalTo(TimeUnit.MINUTES.toMillis(5)));
-        assertThat(rounding.round(time("1986-01-01T00:46:00+05:45")), equalTo(time("1986-01-01T00:40:00+05:45")));
+        assertThat(rounding.round(time("1986-01-01T00:46:00+05:45")), isDate(time("1986-01-01T00:40:00+05:45"), tz));
         assertThat(time("1986-01-01T00:40:00+05:45") - time("1986-01-01T00:20:00+05:45"), equalTo(TimeUnit.MINUTES.toMillis(20)));
     }
 
@@ -319,13 +294,29 @@ public class TimeZoneRoundingTests extends ESTestCase {
         long interval = TimeUnit.MINUTES.toMillis(14);
         TimeZoneRounding rounding = new TimeZoneRounding.TimeIntervalRounding(interval, tz);
 
-        assertThat(rounding.round(time("2016-03-27T01:41:00+01:00")), equalTo(time("2016-03-27T01:30:00+01:00")));
-        assertThat(rounding.round(time("2016-03-27T01:51:00+01:00")), equalTo(time("2016-03-27T01:44:00+01:00")));
-        assertThat(rounding.round(time("2016-03-27T01:59:00+01:00")), equalTo(time("2016-03-27T01:58:00+01:00")));
-        assertThat(rounding.round(time("2016-03-27T03:05:00+02:00")), equalTo(time("2016-03-27T03:00:00+02:00")));
-        assertThat(rounding.round(time("2016-03-27T03:12:00+02:00")), equalTo(time("2016-03-27T03:08:00+02:00")));
-        assertThat(rounding.round(time("2016-03-27T03:25:00+02:00")), equalTo(time("2016-03-27T03:22:00+02:00")));
-        assertThat(rounding.round(time("2016-03-27T03:39:00+02:00")), equalTo(time("2016-03-27T03:36:00+02:00")));
+        assertThat(rounding.round(time("2016-03-27T01:41:00+01:00")), isDate(time("2016-03-27T01:30:00+01:00"), tz));
+        assertThat(rounding.round(time("2016-03-27T01:51:00+01:00")), isDate(time("2016-03-27T01:44:00+01:00"), tz));
+        assertThat(rounding.round(time("2016-03-27T01:59:00+01:00")), isDate(time("2016-03-27T01:58:00+01:00"), tz));
+        assertThat(rounding.round(time("2016-03-27T03:05:00+02:00")), isDate(time("2016-03-27T03:00:00+02:00"), tz));
+        assertThat(rounding.round(time("2016-03-27T03:12:00+02:00")), isDate(time("2016-03-27T03:08:00+02:00"), tz));
+        assertThat(rounding.round(time("2016-03-27T03:25:00+02:00")), isDate(time("2016-03-27T03:22:00+02:00"), tz));
+        assertThat(rounding.round(time("2016-03-27T03:39:00+02:00")), isDate(time("2016-03-27T03:36:00+02:00"), tz));
+    }
+
+    /**
+     * Test for half day rounding intervals scrossing DST.
+     */
+    public void testIntervalRounding_HalfDay_DST() {
+        DateTimeZone tz = DateTimeZone.forID("CET");
+        long interval = TimeUnit.HOURS.toMillis(12);
+        TimeZoneRounding rounding = new TimeZoneRounding.TimeIntervalRounding(interval, tz);
+
+        assertThat(rounding.round(time("2016-03-26T01:00:00+01:00")), isDate(time("2016-03-26T00:00:00+01:00"), tz));
+        assertThat(rounding.round(time("2016-03-26T13:00:00+01:00")), isDate(time("2016-03-26T12:00:00+01:00"), tz));
+        assertThat(rounding.round(time("2016-03-27T01:00:00+01:00")), isDate(time("2016-03-27T00:00:00+01:00"), tz));
+        assertThat(rounding.round(time("2016-03-27T13:00:00+02:00")), isDate(time("2016-03-27T12:00:00+02:00"), tz));
+        assertThat(rounding.round(time("2016-03-28T01:00:00+02:00")), isDate(time("2016-03-28T00:00:00+02:00"), tz));
+        assertThat(rounding.round(time("2016-03-28T13:00:00+02:00")), isDate(time("2016-03-28T12:00:00+02:00"), tz));
     }
 
     /**
@@ -365,36 +356,39 @@ public class TimeZoneRoundingTests extends ESTestCase {
      */
     public void testAmbiguousHoursAfterDSTSwitch() {
         Rounding tzRounding;
-        tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(JERUSALEM_TIMEZONE).build();
-        // Both timestamps "2014-10-25T22:30:00Z" and "2014-10-25T23:30:00Z" are "2014-10-26T01:30:00" in local time because
-        // of DST switch between them. This test checks that they are both returned to their correct UTC time after rounding.
-        assertThat(tzRounding.round(time("2014-10-25T22:30:00", DateTimeZone.UTC)), equalTo(time("2014-10-25T22:00:00", DateTimeZone.UTC)));
-        assertThat(tzRounding.round(time("2014-10-25T23:30:00", DateTimeZone.UTC)), equalTo(time("2014-10-25T23:00:00", DateTimeZone.UTC)));
+        final DateTimeZone tz = DateTimeZone.forID("Asia/Jerusalem");
+        tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).timeZone(tz).build();
+        assertThat(tzRounding.round(time("2014-10-26T00:30:00+03:00")), isDate(time("2014-10-26T00:00:00+03:00"), tz));
+        assertThat(tzRounding.round(time("2014-10-26T01:30:00+03:00")), isDate(time("2014-10-26T01:00:00+03:00"), tz));
+        // the utc date for "2014-10-25T03:00:00+03:00" and "2014-10-25T03:00:00+02:00" is the same, local time turns back 1h here
+        assertThat(time("2014-10-26T03:00:00+03:00"), isDate(time("2014-10-26T02:00:00+02:00"), tz));
+        assertThat(tzRounding.round(time("2014-10-26T01:30:00+02:00")), isDate(time("2014-10-26T01:00:00+02:00"), tz));
+        assertThat(tzRounding.round(time("2014-10-26T02:30:00+02:00")), isDate(time("2014-10-26T02:00:00+02:00"), tz));
 
         // Day interval
-        tzRounding = TimeZoneRounding.builder(DateTimeUnit.DAY_OF_MONTH).timeZone(JERUSALEM_TIMEZONE).build();
-        assertThat(tzRounding.round(time("2014-11-11T17:00:00", JERUSALEM_TIMEZONE)), equalTo(time("2014-11-11T00:00:00", JERUSALEM_TIMEZONE)));
+        tzRounding = TimeZoneRounding.builder(DateTimeUnit.DAY_OF_MONTH).timeZone(tz).build();
+        assertThat(tzRounding.round(time("2014-11-11T17:00:00", tz)), isDate(time("2014-11-11T00:00:00", tz), tz));
         // DST on
-        assertThat(tzRounding.round(time("2014-08-11T17:00:00", JERUSALEM_TIMEZONE)), equalTo(time("2014-08-11T00:00:00", JERUSALEM_TIMEZONE)));
+        assertThat(tzRounding.round(time("2014-08-11T17:00:00", tz)), isDate(time("2014-08-11T00:00:00", tz), tz));
         // Day of switching DST on -> off
-        assertThat(tzRounding.round(time("2014-10-26T17:00:00", JERUSALEM_TIMEZONE)), equalTo(time("2014-10-26T00:00:00", JERUSALEM_TIMEZONE)));
+        assertThat(tzRounding.round(time("2014-10-26T17:00:00", tz)), isDate(time("2014-10-26T00:00:00", tz), tz));
         // Day of switching DST off -> on
-        assertThat(tzRounding.round(time("2015-03-27T17:00:00", JERUSALEM_TIMEZONE)), equalTo(time("2015-03-27T00:00:00", JERUSALEM_TIMEZONE)));
+        assertThat(tzRounding.round(time("2015-03-27T17:00:00", tz)), isDate(time("2015-03-27T00:00:00", tz), tz));
 
         // Month interval
-        tzRounding = TimeZoneRounding.builder(DateTimeUnit.MONTH_OF_YEAR).timeZone(JERUSALEM_TIMEZONE).build();
-        assertThat(tzRounding.round(time("2014-11-11T17:00:00", JERUSALEM_TIMEZONE)), equalTo(time("2014-11-01T00:00:00", JERUSALEM_TIMEZONE)));
+        tzRounding = TimeZoneRounding.builder(DateTimeUnit.MONTH_OF_YEAR).timeZone(tz).build();
+        assertThat(tzRounding.round(time("2014-11-11T17:00:00", tz)), isDate(time("2014-11-01T00:00:00", tz), tz));
         // DST on
-        assertThat(tzRounding.round(time("2014-10-10T17:00:00", JERUSALEM_TIMEZONE)), equalTo(time("2014-10-01T00:00:00", JERUSALEM_TIMEZONE)));
+        assertThat(tzRounding.round(time("2014-10-10T17:00:00", tz)), isDate(time("2014-10-01T00:00:00", tz), tz));
 
         // Year interval
-        tzRounding = TimeZoneRounding.builder(DateTimeUnit.YEAR_OF_CENTURY).timeZone(JERUSALEM_TIMEZONE).build();
-        assertThat(tzRounding.round(time("2014-11-11T17:00:00", JERUSALEM_TIMEZONE)), equalTo(time("2014-01-01T00:00:00", JERUSALEM_TIMEZONE)));
+        tzRounding = TimeZoneRounding.builder(DateTimeUnit.YEAR_OF_CENTURY).timeZone(tz).build();
+        assertThat(tzRounding.round(time("2014-11-11T17:00:00", tz)), isDate(time("2014-01-01T00:00:00", tz), tz));
 
         // Two timestamps in same year and different timezone offset ("Double buckets" issue - #9491)
-        tzRounding = TimeZoneRounding.builder(DateTimeUnit.YEAR_OF_CENTURY).timeZone(JERUSALEM_TIMEZONE).build();
-        assertThat(tzRounding.round(time("2014-11-11T17:00:00", JERUSALEM_TIMEZONE)),
-                equalTo(tzRounding.round(time("2014-08-11T17:00:00", JERUSALEM_TIMEZONE))));
+        tzRounding = TimeZoneRounding.builder(DateTimeUnit.YEAR_OF_CENTURY).timeZone(tz).build();
+        assertThat(tzRounding.round(time("2014-11-11T17:00:00", tz)),
+                isDate(tzRounding.round(time("2014-08-11T17:00:00", tz)), tz));
     }
 
     /**
@@ -417,20 +411,20 @@ public class TimeZoneRoundingTests extends ESTestCase {
         {
             // standard +/-1 hour DST transition, CET
             DateTimeUnit timeUnit = DateTimeUnit.HOUR_OF_DAY;
-            DateTimeZone timezone = DateTimeZone.forID("CET");
-            TimeZoneRounding rounding = new TimeZoneRounding.TimeUnitRounding(timeUnit, timezone);
+            DateTimeZone tz = DateTimeZone.forID("CET");
+            TimeZoneRounding rounding = new TimeZoneRounding.TimeUnitRounding(timeUnit, tz);
 
             // 29 Mar 2015 - Daylight Saving Time Started
             // at 02:00:00 clocks were turned forward 1 hour to 03:00:00
-            assertInterval(time("2015-03-29T00:00:00.000+01:00"), time("2015-03-29T01:00:00.000+01:00"), rounding, 60);
-            assertInterval(time("2015-03-29T01:00:00.000+01:00"), time("2015-03-29T03:00:00.000+02:00"), rounding, 60);
-            assertInterval(time("2015-03-29T03:00:00.000+02:00"), time("2015-03-29T04:00:00.000+02:00"), rounding, 60);
+            assertInterval(time("2015-03-29T00:00:00.000+01:00"), time("2015-03-29T01:00:00.000+01:00"), rounding, 60, tz);
+            assertInterval(time("2015-03-29T01:00:00.000+01:00"), time("2015-03-29T03:00:00.000+02:00"), rounding, 60, tz);
+            assertInterval(time("2015-03-29T03:00:00.000+02:00"), time("2015-03-29T04:00:00.000+02:00"), rounding, 60, tz);
 
             // 25 Oct 2015 - Daylight Saving Time Ended
             // at 03:00:00 clocks were turned backward 1 hour to 02:00:00
-            assertInterval(time("2015-10-25T01:00:00.000+02:00"), time("2015-10-25T02:00:00.000+02:00"), rounding, 60);
-            assertInterval(time("2015-10-25T02:00:00.000+02:00"), time("2015-10-25T02:00:00.000+01:00"), rounding, 60);
-            assertInterval(time("2015-10-25T02:00:00.000+01:00"), time("2015-10-25T03:00:00.000+01:00"), rounding, 60);
+            assertInterval(time("2015-10-25T01:00:00.000+02:00"), time("2015-10-25T02:00:00.000+02:00"), rounding, 60, tz);
+            assertInterval(time("2015-10-25T02:00:00.000+02:00"), time("2015-10-25T02:00:00.000+01:00"), rounding, 60, tz);
+            assertInterval(time("2015-10-25T02:00:00.000+01:00"), time("2015-10-25T03:00:00.000+01:00"), rounding, 60, tz);
         }
 
         {
@@ -441,12 +435,12 @@ public class TimeZoneRoundingTests extends ESTestCase {
             // the interval between is 105 minutes long because the hour after transition starts at 00:15
             // which is not a round value for hourly rounding
             DateTimeUnit timeUnit = DateTimeUnit.HOUR_OF_DAY;
-            DateTimeZone timezone = DateTimeZone.forID("Asia/Kathmandu");
-            TimeZoneRounding rounding = new TimeZoneRounding.TimeUnitRounding(timeUnit, timezone);
+            DateTimeZone tz = DateTimeZone.forID("Asia/Kathmandu");
+            TimeZoneRounding rounding = new TimeZoneRounding.TimeUnitRounding(timeUnit, tz);
 
-            assertInterval(time("1985-12-31T22:00:00.000+05:30"), time("1985-12-31T23:00:00.000+05:30"), rounding, 60);
-            assertInterval(time("1985-12-31T23:00:00.000+05:30"), time("1986-01-01T01:00:00.000+05:45"), rounding, 105);
-            assertInterval(time("1986-01-01T01:00:00.000+05:45"), time("1986-01-01T02:00:00.000+05:45"), rounding, 60);
+            assertInterval(time("1985-12-31T22:00:00.000+05:30"), time("1985-12-31T23:00:00.000+05:30"), rounding, 60, tz);
+            assertInterval(time("1985-12-31T23:00:00.000+05:30"), time("1986-01-01T01:00:00.000+05:45"), rounding, 105, tz);
+            assertInterval(time("1986-01-01T01:00:00.000+05:45"), time("1986-01-01T02:00:00.000+05:45"), rounding, 60, tz);
         }
 
         {
@@ -454,19 +448,19 @@ public class TimeZoneRoundingTests extends ESTestCase {
             // 3 Mar 1991 - Daylight Saving Time Ended
             // at 02:00:00 clocks were turned backward 0:30 hours to Sunday, 3 March 1991, 01:30:00
             DateTimeUnit timeUnit = DateTimeUnit.HOUR_OF_DAY;
-            DateTimeZone timezone = DateTimeZone.forID("Australia/Lord_Howe");
-            TimeZoneRounding rounding = new TimeZoneRounding.TimeUnitRounding(timeUnit, timezone);
+            DateTimeZone tz = DateTimeZone.forID("Australia/Lord_Howe");
+            TimeZoneRounding rounding = new TimeZoneRounding.TimeUnitRounding(timeUnit, tz);
 
-            assertInterval(time("1991-03-03T00:00:00.000+11:00"), time("1991-03-03T01:00:00.000+11:00"), rounding, 60);
-            assertInterval(time("1991-03-03T01:00:00.000+11:00"), time("1991-03-03T02:00:00.000+10:30"), rounding, 90);
-            assertInterval(time("1991-03-03T02:00:00.000+10:30"), time("1991-03-03T03:00:00.000+10:30"), rounding, 60);
+            assertInterval(time("1991-03-03T00:00:00.000+11:00"), time("1991-03-03T01:00:00.000+11:00"), rounding, 60, tz);
+            assertInterval(time("1991-03-03T01:00:00.000+11:00"), time("1991-03-03T02:00:00.000+10:30"), rounding, 90, tz);
+            assertInterval(time("1991-03-03T02:00:00.000+10:30"), time("1991-03-03T03:00:00.000+10:30"), rounding, 60, tz);
 
             // 27 Oct 1991 - Daylight Saving Time Started
             // at 02:00:00 clocks were turned forward 0:30 hours to 02:30:00
-            assertInterval(time("1991-10-27T00:00:00.000+10:30"), time("1991-10-27T01:00:00.000+10:30"), rounding, 60);
+            assertInterval(time("1991-10-27T00:00:00.000+10:30"), time("1991-10-27T01:00:00.000+10:30"), rounding, 60, tz);
             // the interval containing the switch time is 90 minutes long
-            assertInterval(time("1991-10-27T01:00:00.000+10:30"), time("1991-10-27T03:00:00.000+11:00"), rounding, 90);
-            assertInterval(time("1991-10-27T03:00:00.000+11:00"), time("1991-10-27T04:00:00.000+11:00"), rounding, 60);
+            assertInterval(time("1991-10-27T01:00:00.000+10:30"), time("1991-10-27T03:00:00.000+11:00"), rounding, 90, tz);
+            assertInterval(time("1991-10-27T03:00:00.000+11:00"), time("1991-10-27T04:00:00.000+11:00"), rounding, 60, tz);
         }
 
         {
@@ -474,24 +468,25 @@ public class TimeZoneRoundingTests extends ESTestCase {
             // 5 Apr 2015 - Daylight Saving Time Ended
             // at 03:45:00 clocks were turned backward 1 hour to 02:45:00
             DateTimeUnit timeUnit = DateTimeUnit.HOUR_OF_DAY;
-            DateTimeZone timezone = DateTimeZone.forID("Pacific/Chatham");
-            TimeZoneRounding rounding = new TimeZoneRounding.TimeUnitRounding(timeUnit, timezone);
+            DateTimeZone tz = DateTimeZone.forID("Pacific/Chatham");
+            TimeZoneRounding rounding = new TimeZoneRounding.TimeUnitRounding(timeUnit, tz);
 
-            assertInterval(time("2015-04-05T02:00:00.000+13:45"), time("2015-04-05T03:00:00.000+13:45"), rounding, 60);
-            assertInterval(time("2015-04-05T03:00:00.000+13:45"), time("2015-04-05T03:00:00.000+12:45"), rounding, 60);
-            assertInterval(time("2015-04-05T03:00:00.000+12:45"), time("2015-04-05T04:00:00.000+12:45"), rounding, 60);
+            assertInterval(time("2015-04-05T02:00:00.000+13:45"), time("2015-04-05T03:00:00.000+13:45"), rounding, 60, tz);
+            assertInterval(time("2015-04-05T03:00:00.000+13:45"), time("2015-04-05T03:00:00.000+12:45"), rounding, 60, tz);
+            assertInterval(time("2015-04-05T03:00:00.000+12:45"), time("2015-04-05T04:00:00.000+12:45"), rounding, 60, tz);
 
             // 27 Sep 2015 - Daylight Saving Time Started
             // at 02:45:00 clocks were turned forward 1 hour to 03:45:00
 
-            assertInterval(time("2015-09-27T01:00:00.000+12:45"), time("2015-09-27T02:00:00.000+12:45"), rounding, 60);
-            assertInterval(time("2015-09-27T02:00:00.000+12:45"), time("2015-09-27T04:00:00.000+13:45"), rounding, 60);
-            assertInterval(time("2015-09-27T04:00:00.000+13:45"), time("2015-09-27T05:00:00.000+13:45"), rounding, 60);
+            assertInterval(time("2015-09-27T01:00:00.000+12:45"), time("2015-09-27T02:00:00.000+12:45"), rounding, 60, tz);
+            assertInterval(time("2015-09-27T02:00:00.000+12:45"), time("2015-09-27T04:00:00.000+13:45"), rounding, 60, tz);
+            assertInterval(time("2015-09-27T04:00:00.000+13:45"), time("2015-09-27T05:00:00.000+13:45"), rounding, 60, tz);
         }
     }
 
-    private static void assertInterval(long rounded, long nextRoundingValue, TimeZoneRounding rounding, int minutes) {
-        assertInterval(rounded, dateBetween(rounded, nextRoundingValue), nextRoundingValue, rounding);
+    private static void assertInterval(long rounded, long nextRoundingValue, TimeZoneRounding rounding, int minutes,
+            DateTimeZone tz) {
+        assertInterval(rounded, dateBetween(rounded, nextRoundingValue), nextRoundingValue, rounding, tz);
         assertEquals(DateTimeConstants.MILLIS_PER_MINUTE * minutes, nextRoundingValue - rounded);
     }
 
@@ -502,20 +497,21 @@ public class TimeZoneRoundingTests extends ESTestCase {
      * @param nextRoundingValue the expected upper end of the rounding interval
      * @param rounding the rounding instance
      */
-    private static void assertInterval(long rounded, long unrounded, long nextRoundingValue, TimeZoneRounding rounding) {
+    private static void assertInterval(long rounded, long unrounded, long nextRoundingValue, TimeZoneRounding rounding,
+            DateTimeZone tz) {
         assert rounded <= unrounded && unrounded <= nextRoundingValue;
-        assertThat("rounding should be idempotent " + rounding, rounded, equalTo(rounding.round(rounded)));
+        assertThat("rounding should be idempotent ", rounding.round(rounded), isDate(rounded, tz));
         assertThat("rounded value smaller or equal than unrounded" + rounding, rounded, lessThanOrEqualTo(unrounded));
         assertThat("values less than rounded should round further down" + rounding, rounding.round(rounded - 1), lessThan(rounded));
         assertThat("nextRounding value should be greater than date" + rounding, nextRoundingValue, greaterThan(unrounded));
-        assertThat("nextRounding value should be a rounded date" + rounding, nextRoundingValue, equalTo(rounding.round(nextRoundingValue)));
-        assertThat("values above nextRounding should round down there" + rounding, rounding.round(nextRoundingValue + 1),
-                equalTo(nextRoundingValue));
+        assertThat("nextRounding value should be a rounded date", rounding.round(nextRoundingValue), isDate(nextRoundingValue, tz));
+        assertThat("values above nextRounding should round down there", rounding.round(nextRoundingValue + 1),
+                isDate(nextRoundingValue, tz));
 
         long dateBetween = dateBetween(rounded, nextRoundingValue);
-        assertThat("dateBetween should round down to roundedDate" + rounding, rounding.round(dateBetween), equalTo(rounded));
-        assertThat("dateBetween should round up to nextRoundingValue" + rounding, rounding.nextRoundingValue(dateBetween),
-                equalTo(nextRoundingValue));
+        assertThat("dateBetween should round down to roundedDate", rounding.round(dateBetween), isDate(rounded, tz));
+        assertThat("dateBetween should round up to nextRoundingValue", rounding.nextRoundingValue(dateBetween),
+                isDate(nextRoundingValue, tz));
     }
 
     private static long dateBetween(long lower, long upper) {
@@ -529,19 +525,30 @@ public class TimeZoneRoundingTests extends ESTestCase {
         return DateTimeUnit.resolve(id);
     }
 
-    private static String toUTCDateString(long time) {
-        return new DateTime(time, DateTimeZone.UTC).toString();
-    }
-
-    private static long utc(String time) {
-        return time(time, DateTimeZone.UTC);
-    }
-
     private static long time(String time) {
-        return ISODateTimeFormat.dateOptionalTimeParser().parseMillis(time);
+        return time(time, DateTimeZone.UTC);
     }
 
     private static long time(String time, DateTimeZone zone) {
         return ISODateTimeFormat.dateOptionalTimeParser().withZone(zone).parseMillis(time);
     }
+
+    private static Matcher<Long> isDate(final long expected, DateTimeZone tz) {
+        return new TypeSafeMatcher<Long>() {
+            @Override
+            public boolean matchesSafely(final Long item) {
+                return expected == item.longValue();
+            }
+
+            @Override
+            public void describeTo(Description description) {
+                description.appendText("Expected: " + new DateTime(expected, tz) + " [" + expected + "] ");
+            }
+
+            @Override
+            protected void describeMismatchSafely(final Long actual, final Description mismatchDescription) {
+                mismatchDescription.appendText(" was ").appendValue(new DateTime(actual, tz) + " [" + actual + "]");
+            }
+        };
+    }
 }

From 3f221bf7cbd9b2db7706302fe5e05562f3f1b685 Mon Sep 17 00:00:00 2001
From: Mike McCandless <mikemccand@apache.org>
Date: Thu, 16 Jun 2016 04:39:34 -0400
Subject: [PATCH 0903/1311] Add total_indexing_buffer/_in_bytes to nodes info
 API

---
 .../admin/cluster/node/info/NodeInfo.java     | 19 +++++++++-
 .../cluster/node/info/NodesInfoResponse.java  |  2 ++
 .../elasticsearch/indices/IndicesService.java | 11 ++++--
 .../node/service/NodeService.java             |  6 ++--
 .../nodesinfo/NodeInfoStreamingTests.java     |  2 +-
 .../nodesinfo/SimpleNodesInfoIT.java          | 26 +++++++++++++-
 docs/reference/cluster/nodes-info.asciidoc    | 36 ++++++++++++++++++-
 7 files changed, 93 insertions(+), 9 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
index 87ec2d052ab..cf4f3e812d4 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.http.HttpInfo;
 import org.elasticsearch.ingest.core.IngestInfo;
 import org.elasticsearch.monitor.jvm.JvmInfo;
@@ -78,12 +79,16 @@ public class NodeInfo extends BaseNodeResponse {
     @Nullable
     private IngestInfo ingest;
 
+    @Nullable
+    private ByteSizeValue totalIndexingBuffer;
+
     public NodeInfo() {
     }
 
     public NodeInfo(Version version, Build build, DiscoveryNode node, @Nullable Map<String, String> serviceAttributes, @Nullable Settings settings,
                     @Nullable OsInfo os, @Nullable ProcessInfo process, @Nullable JvmInfo jvm, @Nullable ThreadPoolInfo threadPool,
-                    @Nullable TransportInfo transport, @Nullable HttpInfo http, @Nullable PluginsAndModules plugins, @Nullable IngestInfo ingest) {
+                    @Nullable TransportInfo transport, @Nullable HttpInfo http, @Nullable PluginsAndModules plugins, @Nullable IngestInfo ingest,
+                    ByteSizeValue totalIndexingBuffer) {
         super(node);
         this.version = version;
         this.build = build;
@@ -97,6 +102,7 @@ public class NodeInfo extends BaseNodeResponse {
         this.http = http;
         this.plugins = plugins;
         this.ingest = ingest;
+        this.totalIndexingBuffer = totalIndexingBuffer;
     }
 
     /**
@@ -186,6 +192,11 @@ public class NodeInfo extends BaseNodeResponse {
         return ingest;
     }
 
+    @Nullable
+    public ByteSizeValue getTotalIndexingBuffer() {
+        return totalIndexingBuffer;
+    }
+
     public static NodeInfo readNodeInfo(StreamInput in) throws IOException {
         NodeInfo nodeInfo = new NodeInfo();
         nodeInfo.readFrom(in);
@@ -197,6 +208,7 @@ public class NodeInfo extends BaseNodeResponse {
         super.readFrom(in);
         version = Version.readVersion(in);
         build = Build.readBuild(in);
+        totalIndexingBuffer = new ByteSizeValue(in.readLong());
         if (in.readBoolean()) {
             Map<String, String> builder = new HashMap<>();
             int size = in.readVInt();
@@ -240,6 +252,11 @@ public class NodeInfo extends BaseNodeResponse {
         super.writeTo(out);
         out.writeVInt(version.id);
         Build.writeBuild(build, out);
+        if (totalIndexingBuffer == null) {
+            out.writeLong(0);
+        } else {
+            out.writeLong(totalIndexingBuffer.bytes());
+        }
         if (getServiceAttributes() == null) {
             out.writeBoolean(false);
         } else {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java
index d5a43eb030e..c42deeac703 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java
@@ -69,6 +69,8 @@ public class NodesInfoResponse extends BaseNodesResponse<NodeInfo> implements To
 
             builder.field("version", nodeInfo.getVersion());
             builder.field("build_hash", nodeInfo.getBuild().shortHash());
+            builder.field("total_indexing_buffer_in_bytes", nodeInfo.getTotalIndexingBuffer().bytes());
+            builder.field("total_indexing_buffer", nodeInfo.getTotalIndexingBuffer());
 
             if (nodeInfo.getServiceAttributes() != null) {
                 for (Map.Entry<String, String> nodeAttribute : nodeInfo.getServiceAttributes().entrySet()) {
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index 92bcfbe64a2..89db99be923 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -27,11 +27,11 @@ import org.apache.lucene.util.CollectionUtil;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
-import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
-import org.elasticsearch.action.fieldstats.FieldStats;
+import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
+import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -53,9 +53,10 @@ import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
-import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.Callback;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -1140,6 +1141,10 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService>
         }
     }
 
+    public ByteSizeValue getTotalIndexingBufferBytes() {
+        return indexingMemoryController.indexingBufferSize();
+    }
+
     /**
      * Cache something calculated at the shard level.
      * @param shard the shard this item is part of
diff --git a/core/src/main/java/org/elasticsearch/node/service/NodeService.java b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
index 1cbdfe12d2c..8280cb6d6ef 100644
--- a/core/src/main/java/org/elasticsearch/node/service/NodeService.java
+++ b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
@@ -135,7 +135,8 @@ public class NodeService extends AbstractComponent implements Closeable {
                 transportService.info(),
                 httpServer == null ? null : httpServer.info(),
                 pluginService == null ? null : pluginService.info(),
-                ingestService == null ? null : ingestService.info()
+                ingestService == null ? null : ingestService.info(),
+                indicesService == null ? null : indicesService.getTotalIndexingBufferBytes()
         );
     }
 
@@ -150,7 +151,8 @@ public class NodeService extends AbstractComponent implements Closeable {
                 transport ? transportService.info() : null,
                 http ? (httpServer == null ? null : httpServer.info()) : null,
                 plugin ? (pluginService == null ? null : pluginService.info()) : null,
-                ingest ? (ingestService == null ? null : ingestService.info()) : null
+                ingest ? (ingestService == null ? null : ingestService.info()) : null,
+                indicesService == null ? null : indicesService.getTotalIndexingBufferBytes()
         );
     }
 
diff --git a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
index 92dac4bd784..b84ffca106c 100644
--- a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
+++ b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
@@ -141,6 +141,6 @@ public class NodeInfoStreamingTests extends ESTestCase {
         plugins.addModule(DummyPluginInfo.INSTANCE);
         plugins.addPlugin(DummyPluginInfo.INSTANCE);
         IngestInfo ingestInfo = new IngestInfo(Collections.emptyList());
-        return new NodeInfo(VersionUtils.randomVersion(random()), build, node, serviceAttributes, settings, osInfo, process, jvm, threadPoolInfo, transport, htttpInfo, plugins, ingestInfo);
+        return new NodeInfo(VersionUtils.randomVersion(random()), build, node, serviceAttributes, settings, osInfo, process, jvm, threadPoolInfo, transport, htttpInfo, plugins, ingestInfo, null);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java b/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java
index b7dcf2872e2..73f0b99ecf4 100644
--- a/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java
+++ b/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java
@@ -24,14 +24,15 @@ import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
+import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.List;
 
 import static org.elasticsearch.client.Requests.nodesInfoRequest;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.notNullValue;
 
@@ -80,6 +81,29 @@ public class SimpleNodesInfoIT extends ESIntegTestCase {
         assertThat(response.getNodesMap().get(server2NodeId), notNullValue());
     }
 
+    public void testNodesInfosTotalIndexingBuffer() throws Exception {
+        List<String> nodesIds = internalCluster().startNodesAsync(2).get();
+        final String node_1 = nodesIds.get(0);
+        final String node_2 = nodesIds.get(1);
+
+        ClusterHealthResponse clusterHealth = client().admin().cluster().prepareHealth().setWaitForGreenStatus().setWaitForNodes("2").get();
+        logger.info("--> done cluster_health, status {}", clusterHealth.getStatus());
+
+        String server1NodeId = internalCluster().getInstance(ClusterService.class, node_1).state().nodes().getLocalNodeId();
+        String server2NodeId = internalCluster().getInstance(ClusterService.class, node_2).state().nodes().getLocalNodeId();
+        logger.info("--> started nodes: {} and {}", server1NodeId, server2NodeId);
+
+        NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().execute().actionGet();
+        assertThat(response.getNodes().size(), is(2));
+        assertThat(response.getNodesMap().get(server1NodeId), notNullValue());
+        assertNotNull(response.getNodesMap().get(server1NodeId).getTotalIndexingBuffer());
+        assertThat(response.getNodesMap().get(server1NodeId).getTotalIndexingBuffer().bytes(), greaterThan(0L));
+
+        assertThat(response.getNodesMap().get(server2NodeId), notNullValue());
+        assertNotNull(response.getNodesMap().get(server2NodeId).getTotalIndexingBuffer());
+        assertThat(response.getNodesMap().get(server2NodeId).getTotalIndexingBuffer().bytes(), greaterThan(0L));
+    }
+
     public void testAllocatedProcessors() throws Exception {
         List<String> nodesIds = internalCluster().
                 startNodesAsync(
diff --git a/docs/reference/cluster/nodes-info.asciidoc b/docs/reference/cluster/nodes-info.asciidoc
index 5ed979abd0d..8b505218664 100644
--- a/docs/reference/cluster/nodes-info.asciidoc
+++ b/docs/reference/cluster/nodes-info.asciidoc
@@ -15,7 +15,41 @@ The second command selectively retrieves nodes information of only
 `nodeId1` and `nodeId2`. All the nodes selective options are explained
 <<cluster-nodes,here>>.
 
-By default, it just returns all attributes and core settings for a node.
+By default, it just returns all attributes and core settings for a node:
+
+[float]
+[[core-info]]
+
+`build_hash`::
+	Short hash of the last git commit in this release.
+
+`host`::
+	The node's host name.
+
+`http_address`::
+	Host and port where primary HTTP connections are accepted.
+
+`ip`::
+	The node's IP address.
+
+`name`::
+	The node's name.
+
+`total_indexing_buffer`::
+	Total heap allowed to be used to hold recently indexed
+	documents before they must be written to disk.  This size is
+	a shared pool across all shards on this node, and is
+        controlled by <<indexing-buffer,Indexing Buffer settings>>.
+
+`total_indexing_buffer_in_bytes`::
+	Same as `total_indexing_buffer`, but expressed in bytes.
+
+`transport_address`::
+	Host and port where transport HTTP connections are accepted.
+
+`version`::
+	Elasticsearch version running on this node.
+
 It also allows to get only information on `settings`, `os`, `process`, `jvm`,
 `thread_pool`, `transport`, `http`, `plugins` and `ingest`:
 

From f2d62194262a9b1e12f3af53f17a015aeca06e62 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Thu, 16 Jun 2016 11:08:10 +0200
Subject: [PATCH 0904/1311] painless: remove useless dropArguments and throws
 statement in megamorphic cache; add tests

---
 .../elasticsearch/painless/DefBootstrap.java  |  8 +++-----
 .../painless/DefBootstrapTests.java           | 19 ++++++++++++++++++-
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index cbf9d8bdbe6..e5cf5a4311c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -149,7 +149,7 @@ public final class DefBootstrap {
          * Creates the {@link MethodHandle} for the megamorphic call site
          * using {@link ClassValue} and {@link MethodHandles#exactInvoker(MethodType)}:
          */
-        private MethodHandle createMegamorphicHandle() throws Throwable {
+        private MethodHandle createMegamorphicHandle() {
             final MethodType type = type();
             final ClassValue<MethodHandle> megamorphicCache = new ClassValue<MethodHandle>() {
                 @Override
@@ -163,10 +163,8 @@ public final class DefBootstrap {
                     }
                 }
             };
-            MethodHandle cacheLookup = MEGAMORPHIC_LOOKUP.bindTo(megamorphicCache);
-            cacheLookup = MethodHandles.dropArguments(cacheLookup,
-                    1, type.parameterList().subList(1, type.parameterCount()));
-            return MethodHandles.foldArguments(MethodHandles.exactInvoker(type), cacheLookup);            
+            return MethodHandles.foldArguments(MethodHandles.exactInvoker(type),
+                    MEGAMORPHIC_LOOKUP.bindTo(megamorphicCache));            
         }
 
         /**
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
index 59dc816b763..6529bed3a48 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
@@ -25,6 +25,7 @@ import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodType;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 
 import org.elasticsearch.test.ESTestCase;
 
@@ -92,7 +93,7 @@ public class DefBootstrapTests extends ESTestCase {
         assertDepthEquals(site, 5);
     }
     
-    /** test that we really revert to a "generic" method that can handle any receiver types */
+    /** test that we revert to the megamorphic classvalue cache and that it works as expected */
     public void testMegamorphic() throws Throwable {
         DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                                           "size", 
@@ -102,6 +103,22 @@ public class DefBootstrapTests extends ESTestCase {
         MethodHandle handle = site.dynamicInvoker();
         assertEquals(2, (int)handle.invokeExact((Object) Arrays.asList("1", "2")));
         assertEquals(1, (int)handle.invokeExact((Object) Collections.singletonMap("a", "b")));
+        assertEquals(3, (int)handle.invokeExact((Object) Arrays.asList("x", "y", "z")));
+        assertEquals(2, (int)handle.invokeExact((Object) Arrays.asList("u", "v")));
+        
+        final HashMap<String,String> map = new HashMap<String,String>();
+        map.put("x", "y");
+        map.put("a", "b");
+        assertEquals(2, (int)handle.invokeExact((Object) map));
+        
+        final IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> {
+            Integer.toString((int)handle.invokeExact(new Object()));
+        });
+        assertEquals("Unable to find dynamic method [size] with [0] arguments for class [java.lang.Object].", iae.getMessage());
+        assertTrue("Does not fail inside ClassValue.computeValue()", Arrays.stream(iae.getStackTrace()).anyMatch(e -> {
+            return e.getMethodName().equals("computeValue") &&
+                   e.getClassName().startsWith("org.elasticsearch.painless.DefBootstrap$PIC$");
+        }));
     }
     
     // test operators with null guards

From ad232aebbea681a9fa910f0da29e38b29a44c62c Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Wed, 8 Jun 2016 12:46:11 +0200
Subject: [PATCH 0905/1311] Set collection mode to breadth_first in the terms
 aggregation when the cardinality of the field is unknown or smaller than the
 requested size.

closes #9825
---
 .../bucket/BestBucketsDeferringCollector.java | 14 +++---
 .../bucket/terms/TermsAggregationBuilder.java | 10 +++--
 .../bucket/terms/TermsAggregatorFactory.java  | 38 ++++++++++++++--
 .../terms/TermsAggregatorFactoryTests.java    | 44 +++++++++++++++++++
 .../bucket/terms-aggregation.asciidoc         | 18 ++++----
 5 files changed, 101 insertions(+), 23 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactoryTests.java

diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java
index 43cefdca290..4dba4000da6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java
@@ -166,16 +166,16 @@ public class BestBucketsDeferringCollector extends DeferringBucketCollector {
             int doc = 0;
             for (long i = 0, end = entry.docDeltas.size(); i < end; ++i) {
                 doc += docDeltaIterator.next();
-                if (needsScores) {
-                    if (docIt.docID() < doc) {
-                        docIt.advance(doc);
-                    }
-                    // aggregations should only be replayed on matching documents
-                    assert docIt.docID() == doc;
-                }
                 final long bucket = buckets.next();
                 final long rebasedBucket = hash.find(bucket);
                 if (rebasedBucket != -1) {
+                    if (needsScores) {
+                        if (docIt.docID() < doc) {
+                            docIt.advance(doc);
+                        }
+                        // aggregations should only be replayed on matching documents
+                        assert docIt.docID() == doc;
+                    }
                     leafCollector.collect(doc, rebasedBucket);
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
index 1dd59edf6d9..87567d64df4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
@@ -56,7 +56,7 @@ public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder<Valu
     private Terms.Order order = Terms.Order.compound(Terms.Order.count(false), Terms.Order.term(true));
     private IncludeExclude includeExclude = null;
     private String executionHint = null;
-    private SubAggCollectionMode collectMode = SubAggCollectionMode.DEPTH_FIRST;
+    private SubAggCollectionMode collectMode = null;
     private TermsAggregator.BucketCountThresholds bucketCountThresholds = new TermsAggregator.BucketCountThresholds(
             DEFAULT_BUCKET_COUNT_THRESHOLDS);
     private boolean showTermDocCountError = false;
@@ -71,7 +71,7 @@ public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder<Valu
     public TermsAggregationBuilder(StreamInput in) throws IOException {
         super(in, StringTerms.TYPE, ValuesSourceType.ANY);
         bucketCountThresholds = new BucketCountThresholds(in);
-        collectMode = SubAggCollectionMode.readFromStream(in);
+        collectMode = in.readOptionalWriteable(SubAggCollectionMode::readFromStream);
         executionHint = in.readOptionalString();
         includeExclude = in.readOptionalWriteable(IncludeExclude::new);
         order = InternalOrder.Streams.readOrder(in);
@@ -86,7 +86,7 @@ public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder<Valu
     @Override
     protected void innerWriteTo(StreamOutput out) throws IOException {
         bucketCountThresholds.writeTo(out);
-        collectMode.writeTo(out);
+        out.writeOptionalWriteable(collectMode);
         out.writeOptionalString(executionHint);
         out.writeOptionalWriteable(includeExclude);
         InternalOrder.Streams.writeOrder(order, out);
@@ -266,7 +266,9 @@ public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder<Valu
         }
         builder.field(ORDER_FIELD.getPreferredName());
         order.toXContent(builder, params);
-        builder.field(SubAggCollectionMode.KEY.getPreferredName(), collectMode.parseField().getPreferredName());
+        if (collectMode != null) {
+            builder.field(SubAggCollectionMode.KEY.getPreferredName(), collectMode.parseField().getPreferredName());
+        }
         if (includeExclude != null) {
             includeExclude.toXContent(builder, params);
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
index 14df012d4b7..8d5ff30c031 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
@@ -150,14 +150,22 @@ public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory<Values
                     }
                 }
             }
+            SubAggCollectionMode cm = collectMode;
+            if (cm == null) {
+                cm = SubAggCollectionMode.DEPTH_FIRST;
+                if (factories != AggregatorFactories.EMPTY) {
+                    cm = subAggCollectionMode(bucketCountThresholds.getShardSize(), maxOrd);
+                }
+            }
+
             DocValueFormat format = config.format();
             if ((includeExclude != null) && (includeExclude.isRegexBased()) && format != DocValueFormat.RAW) {
                 throw new AggregationExecutionException("Aggregation [" + name + "] cannot support regular expression style include/exclude "
                         + "settings as they can only be applied to string fields. Use an array of values for include/exclude clauses");
             }
 
-            return execution.create(name, factories, valuesSource, order, format, bucketCountThresholds, includeExclude, context, parent, 
-                    collectMode, showTermDocCountError, pipelineAggregators, metaData);
+            return execution.create(name, factories, valuesSource, order, format, bucketCountThresholds, includeExclude, context, parent,
+                    cm, showTermDocCountError, pipelineAggregators, metaData);
         }
 
         if ((includeExclude != null) && (includeExclude.isRegexBased())) {
@@ -167,19 +175,27 @@ public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory<Values
 
         if (valuesSource instanceof ValuesSource.Numeric) {
             IncludeExclude.LongFilter longFilter = null;
+            SubAggCollectionMode cm = collectMode;
+            if (cm == null) {
+                if (factories != AggregatorFactories.EMPTY) {
+                    cm = subAggCollectionMode(bucketCountThresholds.getShardSize(), -1);
+                } else {
+                    cm = SubAggCollectionMode.DEPTH_FIRST;
+                }
+            }
             if (((ValuesSource.Numeric) valuesSource).isFloatingPoint()) {
                 if (includeExclude != null) {
                     longFilter = includeExclude.convertToDoubleFilter();
                 }
                 return new DoubleTermsAggregator(name, factories, (ValuesSource.Numeric) valuesSource, config.format(), order,
-                        bucketCountThresholds, context, parent, collectMode, showTermDocCountError, longFilter,
+                        bucketCountThresholds, context, parent, cm, showTermDocCountError, longFilter,
                         pipelineAggregators, metaData);
             }
             if (includeExclude != null) {
                 longFilter = includeExclude.convertToLongFilter(config.format());
             }
             return new LongTermsAggregator(name, factories, (ValuesSource.Numeric) valuesSource, config.format(), order,
-                    bucketCountThresholds, context, parent, collectMode, showTermDocCountError, longFilter, pipelineAggregators,
+                    bucketCountThresholds, context, parent, cm, showTermDocCountError, longFilter, pipelineAggregators,
                     metaData);
         }
 
@@ -187,6 +203,20 @@ public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory<Values
                 + "]. It can only be applied to numeric or string fields.");
     }
 
+    // return the SubAggCollectionMode that this aggregation should use based on the expected size
+    // and the cardinality of the field
+    static SubAggCollectionMode subAggCollectionMode(int expectedSize, long maxOrd) {
+        if (expectedSize == Integer.MAX_VALUE) {
+            // return all buckets
+            return SubAggCollectionMode.DEPTH_FIRST;
+        }
+        if (maxOrd == -1 || maxOrd > expectedSize) {
+            // use breadth_first if the cardinality is bigger than the expected size or unknown (-1)
+            return SubAggCollectionMode.BREADTH_FIRST;
+        }
+        return SubAggCollectionMode.DEPTH_FIRST;
+    }
+
     public enum ExecutionMode {
 
         MAP(new ParseField("map")) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactoryTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactoryTests.java
new file mode 100644
index 00000000000..7d7ffac4fb3
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactoryTests.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class TermsAggregatorFactoryTests extends ESSingleNodeTestCase {
+    public void testSubAggCollectMode() throws Exception {
+        assertThat(TermsAggregatorFactory.subAggCollectionMode(Integer.MAX_VALUE, -1),
+            equalTo(Aggregator.SubAggCollectionMode.DEPTH_FIRST));
+        assertThat(TermsAggregatorFactory.subAggCollectionMode(10, -1),
+            equalTo(Aggregator.SubAggCollectionMode.BREADTH_FIRST));
+        assertThat(TermsAggregatorFactory.subAggCollectionMode(10, 5),
+            equalTo(Aggregator.SubAggCollectionMode.DEPTH_FIRST));
+        assertThat(TermsAggregatorFactory.subAggCollectionMode(10, 10),
+            equalTo(Aggregator.SubAggCollectionMode.DEPTH_FIRST));
+        assertThat(TermsAggregatorFactory.subAggCollectionMode(10, 100),
+            equalTo(Aggregator.SubAggCollectionMode.BREADTH_FIRST));
+        assertThat(TermsAggregatorFactory.subAggCollectionMode(1, 2),
+            equalTo(Aggregator.SubAggCollectionMode.BREADTH_FIRST));
+        assertThat(TermsAggregatorFactory.subAggCollectionMode(1, 100),
+            equalTo(Aggregator.SubAggCollectionMode.BREADTH_FIRST));
+    }
+}
diff --git a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
index 1c88f0a06d8..3c1f4ae860a 100644
--- a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
@@ -577,7 +577,8 @@ Deferring calculation of child aggregations
 
 For fields with many unique terms and a small number of required results it can be more efficient to delay the calculation
 of child aggregations until the top parent-level aggs have been pruned. Ordinarily, all branches of the aggregation tree
-are expanded in one depth-first pass and only then any pruning occurs. In some rare scenarios this can be very wasteful and can hit memory constraints.
+are expanded in one depth-first pass and only then any pruning occurs.
+In some scenarios this can be very wasteful and can hit memory constraints.
 An example problem scenario is querying a movie database for the 10 most popular actors and their 5 most common co-stars:
 
 [source,js]
@@ -602,10 +603,13 @@ An example problem scenario is querying a movie database for the 10 most popular
 }
 --------------------------------------------------
 
-Even though the number of movies may be comparatively small and we want only 50 result buckets there is a combinatorial explosion of buckets
-during calculation - a single movie will produce n² buckets where n is the number of actors. The sane option would be to first determine
+Even though the number of actors may be comparatively small and we want only 50 result buckets there is a combinatorial explosion of buckets
+during calculation - a single actor can produce n² buckets where n is the number of actors. The sane option would be to first determine
 the 10 most popular actors and only then examine the top co-stars for these 10 actors. This alternative strategy is what we call the `breadth_first` collection
-mode as opposed to the default `depth_first` mode:
+mode as opposed to the `depth_first` mode.
+
+NOTE: The `breadth_first` is the default mode for fields with a cardinality bigger than the requested size or when the cardinality is unknown (numeric fields or scripts for instance).
+It is possible to override the default heuristic and to provide a collect mode directly in the request:
 
 [source,js]
 --------------------------------------------------
@@ -615,7 +619,7 @@ mode as opposed to the default `depth_first` mode:
              "terms" : {
                  "field" : "actors",
                  "size" : 10,
-                 "collect_mode" : "breadth_first"
+                 "collect_mode" : "breadth_first" <1>
              },
             "aggs" : {
                 "costars" : {
@@ -630,12 +634,10 @@ mode as opposed to the default `depth_first` mode:
 }
 --------------------------------------------------
 
+<1> the possible values are `breadth_first` and `depth_first`
 
 When using `breadth_first` mode the set of documents that fall into the uppermost buckets are
 cached for subsequent replay so there is a memory overhead in doing this which is linear with the number of matching documents.
-In most requests the volume of buckets generated is smaller than the number of documents that fall into them so the default `depth_first`
-collection mode is normally the best bet but occasionally the `breadth_first` strategy can be significantly more efficient. Currently
-elasticsearch will always use the `depth_first` collect_mode unless explicitly instructed to use `breadth_first` as in the above example.
 Note that the `order` parameter can still be used to refer to data from a child aggregation when using the `breadth_first` setting - the parent
 aggregation understands that this child aggregation will need to be called first before any of the other child aggregations.
 

From 4293030fd9d5be09712ce03e50b39e81fa15419e Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Thu, 16 Jun 2016 11:46:11 +0200
Subject: [PATCH 0906/1311] painless: Remove stale comment and remove declared
 exception

---
 .../main/java/org/elasticsearch/painless/DefBootstrap.java  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index e5cf5a4311c..a60985eec67 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -266,7 +266,7 @@ public final class DefBootstrap {
             }
         }
         
-        private MethodHandle lookupGeneric() throws Throwable {
+        private MethodHandle lookupGeneric() {
             if ((flags & OPERATOR_COMPOUND_ASSIGNMENT) != 0) {
                 return DefMath.lookupGenericWithCast(name);
             } else {
@@ -275,8 +275,8 @@ public final class DefBootstrap {
         }
         
         /**
-         * Called when a new type is encountered (or, when we have encountered more than {@code MAX_DEPTH}
-         * types at this call site and given up on caching).
+         * Called when a new type is encountered or if cached type does not match.
+         * In that case we revert to a generic, but slower operator handling.
          */
         @SuppressForbidden(reason = "slow path")
         Object fallback(Object[] args) throws Throwable {

From 8d3ef742db722f883e3a465fa45bb10cee26739c Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 13 Jun 2016 21:42:37 -0400
Subject: [PATCH 0907/1311] Painless: =~ and ==~ operators

Adds support for the find operator (=~) and the match operator (==~)
to painless's regexes. Also whitelists most of the Matcher class and
documents regex support in painless.

The find operator (=~) returns a boolean that is the result of building
a matcher on the lhs with the Pattern on the RHS and calling `find` on
it. Use it like this:

```
if (ctx._source.last =~ /b/)
```

The match operator (==~) returns boolean like find but instead of calling
`find` on the Matcher it calls `matches`.

```
if (ctx._source.last ==~ /[^aeiou].*[aeiou]/)
```

Finally, if you want the actual matcher you do:

```
Matcher m = /[aeiou]/.matcher(ctx._source.last)
```
---
 .../modules/scripting/painless.asciidoc       |  75 ++
 .../src/main/antlr/PainlessLexer.g4           |   2 +
 .../src/main/antlr/PainlessLexer.tokens       |  92 ++-
 .../src/main/antlr/PainlessParser.g4          |   1 +
 .../src/main/antlr/PainlessParser.tokens      |  92 ++-
 .../elasticsearch/painless/Definition.java    |   4 +-
 .../elasticsearch/painless/MethodWriter.java  |   1 -
 .../org/elasticsearch/painless/Operation.java |   2 +
 .../painless/WriterConstants.java             |   6 +-
 .../painless/antlr/PainlessLexer.java         | 410 +++++-----
 .../painless/antlr/PainlessParser.java        | 766 +++++++++---------
 .../elasticsearch/painless/antlr/Walker.java  |   4 +
 .../elasticsearch/painless/node/EBinary.java  |  32 +
 .../elasticsearch/painless/node/LRegex.java   |  10 +-
 .../painless/java.util.regex.txt              |  26 +
 .../elasticsearch/painless/RegexTests.java    | 108 ++-
 .../painless/WhenThingsGoWrongTests.java      |  16 -
 17 files changed, 947 insertions(+), 700 deletions(-)

diff --git a/docs/reference/modules/scripting/painless.asciidoc b/docs/reference/modules/scripting/painless.asciidoc
index d79b59cf0d6..cb28e735e5c 100644
--- a/docs/reference/modules/scripting/painless.asciidoc
+++ b/docs/reference/modules/scripting/painless.asciidoc
@@ -33,6 +33,8 @@ to `painless`.
 
 * Shortcuts for list, map access using the dot `.` operator
 
+* Native support for regular expressions with `/pattern/`, `=~`, and `==~`
+
 
 [[painless-examples]]
 [float]
@@ -199,6 +201,79 @@ POST hockey/player/1/_update
 ----------------------------------------------------------------
 // CONSOLE
 
+[float]
+=== Regular expressions
+
+Painless's native support for regular expressions has syntax constructs:
+
+* `/pattern/`: Pattern literals create patterns. This is the only way to create
+a pattern in painless.
+* `=~`: The find operator return a `boolean`, `true` if a subsequence of the
+text matches, `false` otherwise.
+* `==~`: The match operator returns a `boolean`, `true` if the text matches,
+`false` if it doesn't.
+
+Using the find operator (`=~`) you can update all hockey players with "b" in
+their last name:
+
+[source,js]
+----------------------------------------------------------------
+POST hockey/player/_update_by_query
+{
+  "script": {
+    "lang": "painless",
+    "inline": "if (ctx._source.last =~ /b/) {ctx._source.last += \"matched\"} else {ctx.op = 'noop'}"
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+
+Using the match operator (`==~`) you can update all the hockey players who's
+names start with a consonant and end with a vowel:
+
+[source,js]
+----------------------------------------------------------------
+POST hockey/player/_update_by_query
+{
+  "script": {
+    "lang": "painless",
+    "inline": "if (ctx._source.last ==~ /[^aeiou].*[aeiou]/) {ctx._source.last += \"matched\"} else {ctx.op = 'noop'}"
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+
+Or you can use the `Pattern.matcher` directory to get a `Matcher` instance and
+remove all of the vowels in all of their names:
+
+[source,js]
+----------------------------------------------------------------
+POST hockey/player/_update_by_query
+{
+  "script": {
+    "lang": "painless",
+    "inline": "ctx._source.last = /[aeiou]/.matcher(ctx._source.last).replaceAll('')"
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+
+
+Note: all of the `_update_by_query` examples above could really do with a
+`query` to limit the data that they pull back. While you *could* use a
+<<query-dsl-script-query>> it wouldn't be as efficient as using any other query
+because script queries aren't able to use the inverted index to limit the
+documents that they have to check.
+
+The pattern syntax is just
+http://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html[Java regular expressions].
+We intentionally don't allow scripts to call `Pattern.compile` to get a new
+pattern on the fly because building a `Pattern` is (comparatively) slow.
+Pattern literals (`/apattern/`) have fancy constant extraction so no matter
+where they show up in the painless script they are built only when the script
+is first used. It is fairly similar to how `String` literals work in Java.
+
+
 [[painless-api]]
 [float]
 == Painless API
diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.g4 b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
index fac36030120..a31c50ee198 100644
--- a/modules/lang-painless/src/main/antlr/PainlessLexer.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
@@ -80,6 +80,8 @@ COND:    '?';
 COLON:   ':';
 REF:     '::';
 ARROW:   '->';
+FIND:    '=~';
+MATCH:   '==~';
 INCR:    '++';
 DECR:    '--';
 
diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.tokens b/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
index b6df2d97a51..3968464f015 100644
--- a/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
@@ -49,33 +49,35 @@ COND=48
 COLON=49
 REF=50
 ARROW=51
-INCR=52
-DECR=53
-ASSIGN=54
-AADD=55
-ASUB=56
-AMUL=57
-ADIV=58
-AREM=59
-AAND=60
-AXOR=61
-AOR=62
-ALSH=63
-ARSH=64
-AUSH=65
-OCTAL=66
-HEX=67
-INTEGER=68
-DECIMAL=69
-STRING=70
-REGEX=71
-TRUE=72
-FALSE=73
-NULL=74
-TYPE=75
-ID=76
-DOTINTEGER=77
-DOTID=78
+FIND=52
+MATCH=53
+INCR=54
+DECR=55
+ASSIGN=56
+AADD=57
+ASUB=58
+AMUL=59
+ADIV=60
+AREM=61
+AAND=62
+AXOR=63
+AOR=64
+ALSH=65
+ARSH=66
+AUSH=67
+OCTAL=68
+HEX=69
+INTEGER=70
+DECIMAL=71
+STRING=72
+REGEX=73
+TRUE=74
+FALSE=75
+NULL=76
+TYPE=77
+ID=78
+DOTINTEGER=79
+DOTID=80
 '{'=3
 '}'=4
 '['=5
@@ -125,20 +127,22 @@ DOTID=78
 ':'=49
 '::'=50
 '->'=51
-'++'=52
-'--'=53
-'='=54
-'+='=55
-'-='=56
-'*='=57
-'/='=58
-'%='=59
-'&='=60
-'^='=61
-'|='=62
-'<<='=63
-'>>='=64
-'>>>='=65
-'true'=72
-'false'=73
-'null'=74
+'=~'=52
+'==~'=53
+'++'=54
+'--'=55
+'='=56
+'+='=57
+'-='=58
+'*='=59
+'/='=60
+'%='=61
+'&='=62
+'^='=63
+'|='=64
+'<<='=65
+'>>='=66
+'>>>='=67
+'true'=74
+'false'=75
+'null'=76
diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index cc2ca89047f..a020b0d7d73 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -113,6 +113,7 @@ expression returns [boolean s = true]
     :               u = unary[false]                                       { $s = $u.s; }           # single
     |               expression ( MUL | DIV | REM ) expression              { $s = false; }          # binary
     |               expression ( ADD | SUB ) expression                    { $s = false; }          # binary
+    |               expression ( FIND | MATCH ) expression                 { $s = false; }          # binary
     |               expression ( LSH | RSH | USH ) expression              { $s = false; }          # binary
     |               expression ( LT | LTE | GT | GTE ) expression          { $s = false; }          # comp
     |               expression ( EQ | EQR | NE | NER ) expression          { $s = false; }          # comp
diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.tokens b/modules/lang-painless/src/main/antlr/PainlessParser.tokens
index b6df2d97a51..3968464f015 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.tokens
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.tokens
@@ -49,33 +49,35 @@ COND=48
 COLON=49
 REF=50
 ARROW=51
-INCR=52
-DECR=53
-ASSIGN=54
-AADD=55
-ASUB=56
-AMUL=57
-ADIV=58
-AREM=59
-AAND=60
-AXOR=61
-AOR=62
-ALSH=63
-ARSH=64
-AUSH=65
-OCTAL=66
-HEX=67
-INTEGER=68
-DECIMAL=69
-STRING=70
-REGEX=71
-TRUE=72
-FALSE=73
-NULL=74
-TYPE=75
-ID=76
-DOTINTEGER=77
-DOTID=78
+FIND=52
+MATCH=53
+INCR=54
+DECR=55
+ASSIGN=56
+AADD=57
+ASUB=58
+AMUL=59
+ADIV=60
+AREM=61
+AAND=62
+AXOR=63
+AOR=64
+ALSH=65
+ARSH=66
+AUSH=67
+OCTAL=68
+HEX=69
+INTEGER=70
+DECIMAL=71
+STRING=72
+REGEX=73
+TRUE=74
+FALSE=75
+NULL=76
+TYPE=77
+ID=78
+DOTINTEGER=79
+DOTID=80
 '{'=3
 '}'=4
 '['=5
@@ -125,20 +127,22 @@ DOTID=78
 ':'=49
 '::'=50
 '->'=51
-'++'=52
-'--'=53
-'='=54
-'+='=55
-'-='=56
-'*='=57
-'/='=58
-'%='=59
-'&='=60
-'^='=61
-'|='=62
-'<<='=63
-'>>='=64
-'>>>='=65
-'true'=72
-'false'=73
-'null'=74
+'=~'=52
+'==~'=53
+'++'=54
+'--'=55
+'='=56
+'+='=57
+'-='=58
+'*='=59
+'/='=60
+'%='=61
+'&='=62
+'^='=63
+'|='=64
+'<<='=65
+'>>='=66
+'>>>='=67
+'true'=74
+'false'=75
+'null'=76
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 7188791d971..761a2afeeb1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -87,6 +87,8 @@ public final class Definition {
     public static final Type DEF_TYPE = getType("def");
     public static final Type STRING_TYPE = getType("String");
     public static final Type EXCEPTION_TYPE = getType("Exception");
+    public static final Type PATTERN_TYPE = getType("Pattern");
+    public static final Type MATCHER_TYPE = getType("Matcher");
 
     public enum Sort {
         VOID(       void.class      , 0 , true  , false , false , false ),
@@ -772,7 +774,7 @@ public final class Definition {
 
         if (owner.staticMethods.containsKey(methodKey) || owner.methods.containsKey(methodKey)) {
             throw new IllegalArgumentException(
-                "Duplicate  method signature [" + methodKey + "] found within the struct [" + owner.name + "].");
+                "Duplicate method signature [" + methodKey + "] found within the struct [" + owner.name + "].");
         }
 
         final Class<?>[] classes = new Class<?>[args.length];
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index 3f3ad6b7781..622e528f846 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -386,5 +386,4 @@ public final class MethodWriter extends GeneratorAdapter {
     public void visitEnd() {
         throw new AssertionError("Should never call this method on MethodWriter, use endMethod() instead");
     }
-
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java
index 9cafd1d33d7..e32a77022f6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Operation.java
@@ -32,6 +32,8 @@ public enum Operation {
     REM     ( "%"   ),
     ADD     ( "+"   ),
     SUB     ( "-"   ),
+    FIND    ( "=~"  ),
+    MATCH   ( "==~" ),
     LSH     ( "<<"  ),
     RSH     ( ">>"  ),
     USH     ( ">>>" ),
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index 68814a2e1a6..d798bb5ca68 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -35,6 +35,7 @@ import java.util.BitSet;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Objects;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 /**
@@ -73,12 +74,15 @@ public final class WriterConstants {
     public final static Type METHOD_HANDLE_TYPE = Type.getType(MethodHandle.class);
 
     /**
-     * A Method instance for {@linkplain Pattern#compile}. This isn't looked up from Definition because we intentionally don't add it there
+     * A Method instance for {@linkplain Pattern#compile}. This isn't available from Definition because we intentionally don't add it there
      * so that the script can't create regexes without this syntax. Essentially, our static regex syntax has a monopoly on building regexes
      * because it can do it statically. This is both faster and prevents the script from doing something super slow like building a regex
      * per time it is run.
      */
     public final static Method PATTERN_COMPILE = getAsmMethod(Pattern.class, "compile", String.class);
+    public final static Method PATTERN_MATCHER = getAsmMethod(Matcher.class, "matcher", CharSequence.class);
+    public final static Method MATCHER_MATCHES = getAsmMethod(boolean.class, "matches");
+    public final static Method MATCHER_FIND = getAsmMethod(boolean.class, "find");
 
     /** dynamic callsite bootstrap signature */
     public final static MethodType DEF_BOOTSTRAP_TYPE =
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
index 143d2a1ef3c..040d17c521b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
@@ -26,10 +26,10 @@ class PainlessLexer extends Lexer {
     BWNOT=26, MUL=27, DIV=28, REM=29, ADD=30, SUB=31, LSH=32, RSH=33, USH=34, 
     LT=35, LTE=36, GT=37, GTE=38, EQ=39, EQR=40, NE=41, NER=42, BWAND=43, 
     XOR=44, BWOR=45, BOOLAND=46, BOOLOR=47, COND=48, COLON=49, REF=50, ARROW=51, 
-    INCR=52, DECR=53, ASSIGN=54, AADD=55, ASUB=56, AMUL=57, ADIV=58, AREM=59, 
-    AAND=60, AXOR=61, AOR=62, ALSH=63, ARSH=64, AUSH=65, OCTAL=66, HEX=67, 
-    INTEGER=68, DECIMAL=69, STRING=70, REGEX=71, TRUE=72, FALSE=73, NULL=74, 
-    TYPE=75, ID=76, DOTINTEGER=77, DOTID=78;
+    FIND=52, MATCH=53, INCR=54, DECR=55, ASSIGN=56, AADD=57, ASUB=58, AMUL=59, 
+    ADIV=60, AREM=61, AAND=62, AXOR=63, AOR=64, ALSH=65, ARSH=66, AUSH=67, 
+    OCTAL=68, HEX=69, INTEGER=70, DECIMAL=71, STRING=72, REGEX=73, TRUE=74, 
+    FALSE=75, NULL=76, TYPE=77, ID=78, DOTINTEGER=79, DOTID=80;
   public static final int AFTER_DOT = 1;
   public static String[] modeNames = {
     "DEFAULT_MODE", "AFTER_DOT"
@@ -41,10 +41,11 @@ class PainlessLexer extends Lexer {
     "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", "BOOLNOT", 
     "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", 
     "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", 
-    "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "INCR", "DECR", 
-    "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", "AAND", "AXOR", "AOR", 
-    "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", 
-    "REGEX", "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", "DOTID"
+    "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "FIND", "MATCH", 
+    "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", "AAND", 
+    "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", "DECIMAL", 
+    "STRING", "REGEX", "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", 
+    "DOTID"
   };
 
   private static final String[] _LITERAL_NAMES = {
@@ -53,9 +54,10 @@ class PainlessLexer extends Lexer {
     "'return'", "'new'", "'try'", "'catch'", "'throw'", "'this'", "'!'", "'~'", 
     "'*'", "'/'", "'%'", "'+'", "'-'", "'<<'", "'>>'", "'>>>'", "'<'", "'<='", 
     "'>'", "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", "'^'", "'|'", 
-    "'&&'", "'||'", "'?'", "':'", "'::'", "'->'", "'++'", "'--'", "'='", "'+='", 
-    "'-='", "'*='", "'/='", "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", 
-    "'>>>='", null, null, null, null, null, null, "'true'", "'false'", "'null'"
+    "'&&'", "'||'", "'?'", "':'", "'::'", "'->'", "'=~'", "'==~'", "'++'", 
+    "'--'", "'='", "'+='", "'-='", "'*='", "'/='", "'%='", "'&='", "'^='", 
+    "'|='", "'<<='", "'>>='", "'>>>='", null, null, null, null, null, null, 
+    "'true'", "'false'", "'null'"
   };
   private static final String[] _SYMBOLIC_NAMES = {
     null, "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", 
@@ -63,10 +65,11 @@ class PainlessLexer extends Lexer {
     "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", "BOOLNOT", 
     "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", 
     "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", 
-    "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "INCR", "DECR", 
-    "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", "AAND", "AXOR", "AOR", 
-    "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", 
-    "REGEX", "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", "DOTID"
+    "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "FIND", "MATCH", 
+    "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", "AAND", 
+    "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", "DECIMAL", 
+    "STRING", "REGEX", "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", 
+    "DOTID"
   };
   public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
 
@@ -127,9 +130,9 @@ class PainlessLexer extends Lexer {
     switch (ruleIndex) {
     case 27:
       return DIV_sempred((RuleContext)_localctx, predIndex);
-    case 70:
+    case 72:
       return REGEX_sempred((RuleContext)_localctx, predIndex);
-    case 74:
+    case 76:
       return TYPE_sempred((RuleContext)_localctx, predIndex);
     }
     return true;
@@ -157,7 +160,7 @@ class PainlessLexer extends Lexer {
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2P\u0228\b\1\b\1\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2R\u0233\b\1\b\1\4"+
     "\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n"+
     "\4\13\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22"+
     "\t\22\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31"+
@@ -166,190 +169,193 @@ class PainlessLexer extends Lexer {
     "+\4,\t,\4-\t-\4.\t.\4/\t/\4\60\t\60\4\61\t\61\4\62\t\62\4\63\t\63\4\64"+
     "\t\64\4\65\t\65\4\66\t\66\4\67\t\67\48\t8\49\t9\4:\t:\4;\t;\4<\t<\4=\t"+
     "=\4>\t>\4?\t?\4@\t@\4A\tA\4B\tB\4C\tC\4D\tD\4E\tE\4F\tF\4G\tG\4H\tH\4"+
-    "I\tI\4J\tJ\4K\tK\4L\tL\4M\tM\4N\tN\4O\tO\3\2\6\2\u00a2\n\2\r\2\16\2\u00a3"+
-    "\3\2\3\2\3\3\3\3\3\3\3\3\7\3\u00ac\n\3\f\3\16\3\u00af\13\3\3\3\3\3\3\3"+
-    "\3\3\3\3\7\3\u00b6\n\3\f\3\16\3\u00b9\13\3\3\3\3\3\5\3\u00bd\n\3\3\3\3"+
-    "\3\3\4\3\4\3\5\3\5\3\6\3\6\3\7\3\7\3\b\3\b\3\t\3\t\3\n\3\n\3\n\3\n\3\13"+
-    "\3\13\3\f\3\f\3\r\3\r\3\r\3\16\3\16\3\16\3\16\3\16\3\17\3\17\3\17\3\17"+
-    "\3\17\3\17\3\20\3\20\3\20\3\21\3\21\3\21\3\21\3\22\3\22\3\22\3\22\3\22"+
-    "\3\22\3\22\3\22\3\22\3\23\3\23\3\23\3\23\3\23\3\23\3\24\3\24\3\24\3\24"+
-    "\3\24\3\24\3\24\3\25\3\25\3\25\3\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27"+
-    "\3\27\3\27\3\27\3\30\3\30\3\30\3\30\3\30\3\30\3\31\3\31\3\31\3\31\3\31"+
-    "\3\32\3\32\3\33\3\33\3\34\3\34\3\35\3\35\3\35\3\36\3\36\3\37\3\37\3 \3"+
-    " \3!\3!\3!\3\"\3\"\3\"\3#\3#\3#\3#\3$\3$\3%\3%\3%\3&\3&\3\'\3\'\3\'\3"+
-    "(\3(\3(\3)\3)\3)\3)\3*\3*\3*\3+\3+\3+\3+\3,\3,\3-\3-\3.\3.\3/\3/\3/\3"+
-    "\60\3\60\3\60\3\61\3\61\3\62\3\62\3\63\3\63\3\63\3\64\3\64\3\64\3\65\3"+
-    "\65\3\65\3\66\3\66\3\66\3\67\3\67\38\38\38\39\39\39\3:\3:\3:\3;\3;\3;"+
-    "\3<\3<\3<\3=\3=\3=\3>\3>\3>\3?\3?\3?\3@\3@\3@\3@\3A\3A\3A\3A\3B\3B\3B"+
-    "\3B\3B\3C\3C\6C\u018f\nC\rC\16C\u0190\3C\5C\u0194\nC\3D\3D\3D\6D\u0199"+
-    "\nD\rD\16D\u019a\3D\5D\u019e\nD\3E\3E\3E\7E\u01a3\nE\fE\16E\u01a6\13E"+
-    "\5E\u01a8\nE\3E\5E\u01ab\nE\3F\3F\3F\7F\u01b0\nF\fF\16F\u01b3\13F\5F\u01b5"+
-    "\nF\3F\3F\6F\u01b9\nF\rF\16F\u01ba\5F\u01bd\nF\3F\3F\5F\u01c1\nF\3F\6"+
-    "F\u01c4\nF\rF\16F\u01c5\5F\u01c8\nF\3F\5F\u01cb\nF\3G\3G\3G\3G\3G\3G\7"+
-    "G\u01d3\nG\fG\16G\u01d6\13G\3G\3G\3G\3G\3G\3G\3G\7G\u01df\nG\fG\16G\u01e2"+
-    "\13G\3G\5G\u01e5\nG\3H\3H\3H\3H\6H\u01eb\nH\rH\16H\u01ec\3H\3H\3H\3I\3"+
-    "I\3I\3I\3I\3J\3J\3J\3J\3J\3J\3K\3K\3K\3K\3K\3L\3L\3L\3L\7L\u0206\nL\f"+
-    "L\16L\u0209\13L\3L\3L\3M\3M\7M\u020f\nM\fM\16M\u0212\13M\3N\3N\3N\7N\u0217"+
-    "\nN\fN\16N\u021a\13N\5N\u021c\nN\3N\3N\3O\3O\7O\u0222\nO\fO\16O\u0225"+
-    "\13O\3O\3O\6\u00ad\u00b7\u01d4\u01e0\2P\4\3\6\4\b\5\n\6\f\7\16\b\20\t"+
-    "\22\n\24\13\26\f\30\r\32\16\34\17\36\20 \21\"\22$\23&\24(\25*\26,\27."+
-    "\30\60\31\62\32\64\33\66\348\35:\36<\37> @!B\"D#F$H%J&L\'N(P)R*T+V,X-"+
-    "Z.\\/^\60`\61b\62d\63f\64h\65j\66l\67n8p9r:t;v<x=z>|?~@\u0080A\u0082B"+
-    "\u0084C\u0086D\u0088E\u008aF\u008cG\u008eH\u0090I\u0092J\u0094K\u0096"+
-    "L\u0098M\u009aN\u009cO\u009eP\4\2\3\23\5\2\13\f\17\17\"\"\4\2\f\f\17\17"+
-    "\3\2\629\4\2NNnn\4\2ZZzz\5\2\62;CHch\3\2\63;\3\2\62;\b\2FFHHNNffhhnn\4"+
-    "\2GGgg\4\2--//\4\2HHhh\4\2$$^^\4\2\f\f\61\61\3\2\f\f\5\2C\\aac|\6\2\62"+
-    ";C\\aac|\u0247\2\4\3\2\2\2\2\6\3\2\2\2\2\b\3\2\2\2\2\n\3\2\2\2\2\f\3\2"+
-    "\2\2\2\16\3\2\2\2\2\20\3\2\2\2\2\22\3\2\2\2\2\24\3\2\2\2\2\26\3\2\2\2"+
-    "\2\30\3\2\2\2\2\32\3\2\2\2\2\34\3\2\2\2\2\36\3\2\2\2\2 \3\2\2\2\2\"\3"+
-    "\2\2\2\2$\3\2\2\2\2&\3\2\2\2\2(\3\2\2\2\2*\3\2\2\2\2,\3\2\2\2\2.\3\2\2"+
-    "\2\2\60\3\2\2\2\2\62\3\2\2\2\2\64\3\2\2\2\2\66\3\2\2\2\28\3\2\2\2\2:\3"+
-    "\2\2\2\2<\3\2\2\2\2>\3\2\2\2\2@\3\2\2\2\2B\3\2\2\2\2D\3\2\2\2\2F\3\2\2"+
-    "\2\2H\3\2\2\2\2J\3\2\2\2\2L\3\2\2\2\2N\3\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2"+
-    "T\3\2\2\2\2V\3\2\2\2\2X\3\2\2\2\2Z\3\2\2\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3"+
-    "\2\2\2\2b\3\2\2\2\2d\3\2\2\2\2f\3\2\2\2\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2"+
-    "\2\2n\3\2\2\2\2p\3\2\2\2\2r\3\2\2\2\2t\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2"+
-    "z\3\2\2\2\2|\3\2\2\2\2~\3\2\2\2\2\u0080\3\2\2\2\2\u0082\3\2\2\2\2\u0084"+
-    "\3\2\2\2\2\u0086\3\2\2\2\2\u0088\3\2\2\2\2\u008a\3\2\2\2\2\u008c\3\2\2"+
-    "\2\2\u008e\3\2\2\2\2\u0090\3\2\2\2\2\u0092\3\2\2\2\2\u0094\3\2\2\2\2\u0096"+
-    "\3\2\2\2\2\u0098\3\2\2\2\2\u009a\3\2\2\2\3\u009c\3\2\2\2\3\u009e\3\2\2"+
-    "\2\4\u00a1\3\2\2\2\6\u00bc\3\2\2\2\b\u00c0\3\2\2\2\n\u00c2\3\2\2\2\f\u00c4"+
-    "\3\2\2\2\16\u00c6\3\2\2\2\20\u00c8\3\2\2\2\22\u00ca\3\2\2\2\24\u00cc\3"+
-    "\2\2\2\26\u00d0\3\2\2\2\30\u00d2\3\2\2\2\32\u00d4\3\2\2\2\34\u00d7\3\2"+
-    "\2\2\36\u00dc\3\2\2\2 \u00e2\3\2\2\2\"\u00e5\3\2\2\2$\u00e9\3\2\2\2&\u00f2"+
-    "\3\2\2\2(\u00f8\3\2\2\2*\u00ff\3\2\2\2,\u0103\3\2\2\2.\u0107\3\2\2\2\60"+
-    "\u010d\3\2\2\2\62\u0113\3\2\2\2\64\u0118\3\2\2\2\66\u011a\3\2\2\28\u011c"+
-    "\3\2\2\2:\u011e\3\2\2\2<\u0121\3\2\2\2>\u0123\3\2\2\2@\u0125\3\2\2\2B"+
-    "\u0127\3\2\2\2D\u012a\3\2\2\2F\u012d\3\2\2\2H\u0131\3\2\2\2J\u0133\3\2"+
-    "\2\2L\u0136\3\2\2\2N\u0138\3\2\2\2P\u013b\3\2\2\2R\u013e\3\2\2\2T\u0142"+
-    "\3\2\2\2V\u0145\3\2\2\2X\u0149\3\2\2\2Z\u014b\3\2\2\2\\\u014d\3\2\2\2"+
-    "^\u014f\3\2\2\2`\u0152\3\2\2\2b\u0155\3\2\2\2d\u0157\3\2\2\2f\u0159\3"+
-    "\2\2\2h\u015c\3\2\2\2j\u015f\3\2\2\2l\u0162\3\2\2\2n\u0165\3\2\2\2p\u0167"+
-    "\3\2\2\2r\u016a\3\2\2\2t\u016d\3\2\2\2v\u0170\3\2\2\2x\u0173\3\2\2\2z"+
-    "\u0176\3\2\2\2|\u0179\3\2\2\2~\u017c\3\2\2\2\u0080\u017f\3\2\2\2\u0082"+
-    "\u0183\3\2\2\2\u0084\u0187\3\2\2\2\u0086\u018c\3\2\2\2\u0088\u0195\3\2"+
-    "\2\2\u008a\u01a7\3\2\2\2\u008c\u01b4\3\2\2\2\u008e\u01e4\3\2\2\2\u0090"+
-    "\u01e6\3\2\2\2\u0092\u01f1\3\2\2\2\u0094\u01f6\3\2\2\2\u0096\u01fc\3\2"+
-    "\2\2\u0098\u0201\3\2\2\2\u009a\u020c\3\2\2\2\u009c\u021b\3\2\2\2\u009e"+
-    "\u021f\3\2\2\2\u00a0\u00a2\t\2\2\2\u00a1\u00a0\3\2\2\2\u00a2\u00a3\3\2"+
-    "\2\2\u00a3\u00a1\3\2\2\2\u00a3\u00a4\3\2\2\2\u00a4\u00a5\3\2\2\2\u00a5"+
-    "\u00a6\b\2\2\2\u00a6\5\3\2\2\2\u00a7\u00a8\7\61\2\2\u00a8\u00a9\7\61\2"+
-    "\2\u00a9\u00ad\3\2\2\2\u00aa\u00ac\13\2\2\2\u00ab\u00aa\3\2\2\2\u00ac"+
-    "\u00af\3\2\2\2\u00ad\u00ae\3\2\2\2\u00ad\u00ab\3\2\2\2\u00ae\u00b0\3\2"+
-    "\2\2\u00af\u00ad\3\2\2\2\u00b0\u00bd\t\3\2\2\u00b1\u00b2\7\61\2\2\u00b2"+
-    "\u00b3\7,\2\2\u00b3\u00b7\3\2\2\2\u00b4\u00b6\13\2\2\2\u00b5\u00b4\3\2"+
-    "\2\2\u00b6\u00b9\3\2\2\2\u00b7\u00b8\3\2\2\2\u00b7\u00b5\3\2\2\2\u00b8"+
-    "\u00ba\3\2\2\2\u00b9\u00b7\3\2\2\2\u00ba\u00bb\7,\2\2\u00bb\u00bd\7\61"+
-    "\2\2\u00bc\u00a7\3\2\2\2\u00bc\u00b1\3\2\2\2\u00bd\u00be\3\2\2\2\u00be"+
-    "\u00bf\b\3\2\2\u00bf\7\3\2\2\2\u00c0\u00c1\7}\2\2\u00c1\t\3\2\2\2\u00c2"+
-    "\u00c3\7\177\2\2\u00c3\13\3\2\2\2\u00c4\u00c5\7]\2\2\u00c5\r\3\2\2\2\u00c6"+
-    "\u00c7\7_\2\2\u00c7\17\3\2\2\2\u00c8\u00c9\7*\2\2\u00c9\21\3\2\2\2\u00ca"+
-    "\u00cb\7+\2\2\u00cb\23\3\2\2\2\u00cc\u00cd\7\60\2\2\u00cd\u00ce\3\2\2"+
-    "\2\u00ce\u00cf\b\n\3\2\u00cf\25\3\2\2\2\u00d0\u00d1\7.\2\2\u00d1\27\3"+
-    "\2\2\2\u00d2\u00d3\7=\2\2\u00d3\31\3\2\2\2\u00d4\u00d5\7k\2\2\u00d5\u00d6"+
-    "\7h\2\2\u00d6\33\3\2\2\2\u00d7\u00d8\7g\2\2\u00d8\u00d9\7n\2\2\u00d9\u00da"+
-    "\7u\2\2\u00da\u00db\7g\2\2\u00db\35\3\2\2\2\u00dc\u00dd\7y\2\2\u00dd\u00de"+
-    "\7j\2\2\u00de\u00df\7k\2\2\u00df\u00e0\7n\2\2\u00e0\u00e1\7g\2\2\u00e1"+
-    "\37\3\2\2\2\u00e2\u00e3\7f\2\2\u00e3\u00e4\7q\2\2\u00e4!\3\2\2\2\u00e5"+
-    "\u00e6\7h\2\2\u00e6\u00e7\7q\2\2\u00e7\u00e8\7t\2\2\u00e8#\3\2\2\2\u00e9"+
-    "\u00ea\7e\2\2\u00ea\u00eb\7q\2\2\u00eb\u00ec\7p\2\2\u00ec\u00ed\7v\2\2"+
-    "\u00ed\u00ee\7k\2\2\u00ee\u00ef\7p\2\2\u00ef\u00f0\7w\2\2\u00f0\u00f1"+
-    "\7g\2\2\u00f1%\3\2\2\2\u00f2\u00f3\7d\2\2\u00f3\u00f4\7t\2\2\u00f4\u00f5"+
-    "\7g\2\2\u00f5\u00f6\7c\2\2\u00f6\u00f7\7m\2\2\u00f7\'\3\2\2\2\u00f8\u00f9"+
-    "\7t\2\2\u00f9\u00fa\7g\2\2\u00fa\u00fb\7v\2\2\u00fb\u00fc\7w\2\2\u00fc"+
-    "\u00fd\7t\2\2\u00fd\u00fe\7p\2\2\u00fe)\3\2\2\2\u00ff\u0100\7p\2\2\u0100"+
-    "\u0101\7g\2\2\u0101\u0102\7y\2\2\u0102+\3\2\2\2\u0103\u0104\7v\2\2\u0104"+
-    "\u0105\7t\2\2\u0105\u0106\7{\2\2\u0106-\3\2\2\2\u0107\u0108\7e\2\2\u0108"+
-    "\u0109\7c\2\2\u0109\u010a\7v\2\2\u010a\u010b\7e\2\2\u010b\u010c\7j\2\2"+
-    "\u010c/\3\2\2\2\u010d\u010e\7v\2\2\u010e\u010f\7j\2\2\u010f\u0110\7t\2"+
-    "\2\u0110\u0111\7q\2\2\u0111\u0112\7y\2\2\u0112\61\3\2\2\2\u0113\u0114"+
-    "\7v\2\2\u0114\u0115\7j\2\2\u0115\u0116\7k\2\2\u0116\u0117\7u\2\2\u0117"+
-    "\63\3\2\2\2\u0118\u0119\7#\2\2\u0119\65\3\2\2\2\u011a\u011b\7\u0080\2"+
-    "\2\u011b\67\3\2\2\2\u011c\u011d\7,\2\2\u011d9\3\2\2\2\u011e\u011f\7\61"+
-    "\2\2\u011f\u0120\6\35\2\2\u0120;\3\2\2\2\u0121\u0122\7\'\2\2\u0122=\3"+
-    "\2\2\2\u0123\u0124\7-\2\2\u0124?\3\2\2\2\u0125\u0126\7/\2\2\u0126A\3\2"+
-    "\2\2\u0127\u0128\7>\2\2\u0128\u0129\7>\2\2\u0129C\3\2\2\2\u012a\u012b"+
-    "\7@\2\2\u012b\u012c\7@\2\2\u012cE\3\2\2\2\u012d\u012e\7@\2\2\u012e\u012f"+
-    "\7@\2\2\u012f\u0130\7@\2\2\u0130G\3\2\2\2\u0131\u0132\7>\2\2\u0132I\3"+
-    "\2\2\2\u0133\u0134\7>\2\2\u0134\u0135\7?\2\2\u0135K\3\2\2\2\u0136\u0137"+
-    "\7@\2\2\u0137M\3\2\2\2\u0138\u0139\7@\2\2\u0139\u013a\7?\2\2\u013aO\3"+
-    "\2\2\2\u013b\u013c\7?\2\2\u013c\u013d\7?\2\2\u013dQ\3\2\2\2\u013e\u013f"+
-    "\7?\2\2\u013f\u0140\7?\2\2\u0140\u0141\7?\2\2\u0141S\3\2\2\2\u0142\u0143"+
-    "\7#\2\2\u0143\u0144\7?\2\2\u0144U\3\2\2\2\u0145\u0146\7#\2\2\u0146\u0147"+
-    "\7?\2\2\u0147\u0148\7?\2\2\u0148W\3\2\2\2\u0149\u014a\7(\2\2\u014aY\3"+
-    "\2\2\2\u014b\u014c\7`\2\2\u014c[\3\2\2\2\u014d\u014e\7~\2\2\u014e]\3\2"+
-    "\2\2\u014f\u0150\7(\2\2\u0150\u0151\7(\2\2\u0151_\3\2\2\2\u0152\u0153"+
-    "\7~\2\2\u0153\u0154\7~\2\2\u0154a\3\2\2\2\u0155\u0156\7A\2\2\u0156c\3"+
-    "\2\2\2\u0157\u0158\7<\2\2\u0158e\3\2\2\2\u0159\u015a\7<\2\2\u015a\u015b"+
-    "\7<\2\2\u015bg\3\2\2\2\u015c\u015d\7/\2\2\u015d\u015e\7@\2\2\u015ei\3"+
-    "\2\2\2\u015f\u0160\7-\2\2\u0160\u0161\7-\2\2\u0161k\3\2\2\2\u0162\u0163"+
-    "\7/\2\2\u0163\u0164\7/\2\2\u0164m\3\2\2\2\u0165\u0166\7?\2\2\u0166o\3"+
-    "\2\2\2\u0167\u0168\7-\2\2\u0168\u0169\7?\2\2\u0169q\3\2\2\2\u016a\u016b"+
-    "\7/\2\2\u016b\u016c\7?\2\2\u016cs\3\2\2\2\u016d\u016e\7,\2\2\u016e\u016f"+
-    "\7?\2\2\u016fu\3\2\2\2\u0170\u0171\7\61\2\2\u0171\u0172\7?\2\2\u0172w"+
-    "\3\2\2\2\u0173\u0174\7\'\2\2\u0174\u0175\7?\2\2\u0175y\3\2\2\2\u0176\u0177"+
-    "\7(\2\2\u0177\u0178\7?\2\2\u0178{\3\2\2\2\u0179\u017a\7`\2\2\u017a\u017b"+
-    "\7?\2\2\u017b}\3\2\2\2\u017c\u017d\7~\2\2\u017d\u017e\7?\2\2\u017e\177"+
-    "\3\2\2\2\u017f\u0180\7>\2\2\u0180\u0181\7>\2\2\u0181\u0182\7?\2\2\u0182"+
-    "\u0081\3\2\2\2\u0183\u0184\7@\2\2\u0184\u0185\7@\2\2\u0185\u0186\7?\2"+
-    "\2\u0186\u0083\3\2\2\2\u0187\u0188\7@\2\2\u0188\u0189\7@\2\2\u0189\u018a"+
-    "\7@\2\2\u018a\u018b\7?\2\2\u018b\u0085\3\2\2\2\u018c\u018e\7\62\2\2\u018d"+
-    "\u018f\t\4\2\2\u018e\u018d\3\2\2\2\u018f\u0190\3\2\2\2\u0190\u018e\3\2"+
-    "\2\2\u0190\u0191\3\2\2\2\u0191\u0193\3\2\2\2\u0192\u0194\t\5\2\2\u0193"+
-    "\u0192\3\2\2\2\u0193\u0194\3\2\2\2\u0194\u0087\3\2\2\2\u0195\u0196\7\62"+
-    "\2\2\u0196\u0198\t\6\2\2\u0197\u0199\t\7\2\2\u0198\u0197\3\2\2\2\u0199"+
-    "\u019a\3\2\2\2\u019a\u0198\3\2\2\2\u019a\u019b\3\2\2\2\u019b\u019d\3\2"+
-    "\2\2\u019c\u019e\t\5\2\2\u019d\u019c\3\2\2\2\u019d\u019e\3\2\2\2\u019e"+
-    "\u0089\3\2\2\2\u019f\u01a8\7\62\2\2\u01a0\u01a4\t\b\2\2\u01a1\u01a3\t"+
-    "\t\2\2\u01a2\u01a1\3\2\2\2\u01a3\u01a6\3\2\2\2\u01a4\u01a2\3\2\2\2\u01a4"+
-    "\u01a5\3\2\2\2\u01a5\u01a8\3\2\2\2\u01a6\u01a4\3\2\2\2\u01a7\u019f\3\2"+
-    "\2\2\u01a7\u01a0\3\2\2\2\u01a8\u01aa\3\2\2\2\u01a9\u01ab\t\n\2\2\u01aa"+
-    "\u01a9\3\2\2\2\u01aa\u01ab\3\2\2\2\u01ab\u008b\3\2\2\2\u01ac\u01b5\7\62"+
-    "\2\2\u01ad\u01b1\t\b\2\2\u01ae\u01b0\t\t\2\2\u01af\u01ae\3\2\2\2\u01b0"+
-    "\u01b3\3\2\2\2\u01b1\u01af\3\2\2\2\u01b1\u01b2\3\2\2\2\u01b2\u01b5\3\2"+
-    "\2\2\u01b3\u01b1\3\2\2\2\u01b4\u01ac\3\2\2\2\u01b4\u01ad\3\2\2\2\u01b5"+
-    "\u01bc\3\2\2\2\u01b6\u01b8\5\24\n\2\u01b7\u01b9\t\t\2\2\u01b8\u01b7\3"+
-    "\2\2\2\u01b9\u01ba\3\2\2\2\u01ba\u01b8\3\2\2\2\u01ba\u01bb\3\2\2\2\u01bb"+
-    "\u01bd\3\2\2\2\u01bc\u01b6\3\2\2\2\u01bc\u01bd\3\2\2\2\u01bd\u01c7\3\2"+
-    "\2\2\u01be\u01c0\t\13\2\2\u01bf\u01c1\t\f\2\2\u01c0\u01bf\3\2\2\2\u01c0"+
-    "\u01c1\3\2\2\2\u01c1\u01c3\3\2\2\2\u01c2\u01c4\t\t\2\2\u01c3\u01c2\3\2"+
-    "\2\2\u01c4\u01c5\3\2\2\2\u01c5\u01c3\3\2\2\2\u01c5\u01c6\3\2\2\2\u01c6"+
-    "\u01c8\3\2\2\2\u01c7\u01be\3\2\2\2\u01c7\u01c8\3\2\2\2\u01c8\u01ca\3\2"+
-    "\2\2\u01c9\u01cb\t\r\2\2\u01ca\u01c9\3\2\2\2\u01ca\u01cb\3\2\2\2\u01cb"+
-    "\u008d\3\2\2\2\u01cc\u01d4\7$\2\2\u01cd\u01ce\7^\2\2\u01ce\u01d3\7$\2"+
-    "\2\u01cf\u01d0\7^\2\2\u01d0\u01d3\7^\2\2\u01d1\u01d3\n\16\2\2\u01d2\u01cd"+
-    "\3\2\2\2\u01d2\u01cf\3\2\2\2\u01d2\u01d1\3\2\2\2\u01d3\u01d6\3\2\2\2\u01d4"+
-    "\u01d5\3\2\2\2\u01d4\u01d2\3\2\2\2\u01d5\u01d7\3\2\2\2\u01d6\u01d4\3\2"+
-    "\2\2\u01d7\u01e5\7$\2\2\u01d8\u01e0\7)\2\2\u01d9\u01da\7^\2\2\u01da\u01df"+
-    "\7)\2\2\u01db\u01dc\7^\2\2\u01dc\u01df\7^\2\2\u01dd\u01df\n\16\2\2\u01de"+
-    "\u01d9\3\2\2\2\u01de\u01db\3\2\2\2\u01de\u01dd\3\2\2\2\u01df\u01e2\3\2"+
-    "\2\2\u01e0\u01e1\3\2\2\2\u01e0\u01de\3\2\2\2\u01e1\u01e3\3\2\2\2\u01e2"+
-    "\u01e0\3\2\2\2\u01e3\u01e5\7)\2\2\u01e4\u01cc\3\2\2\2\u01e4\u01d8\3\2"+
-    "\2\2\u01e5\u008f\3\2\2\2\u01e6\u01ea\7\61\2\2\u01e7\u01eb\n\17\2\2\u01e8"+
-    "\u01e9\7^\2\2\u01e9\u01eb\n\20\2\2\u01ea\u01e7\3\2\2\2\u01ea\u01e8\3\2"+
-    "\2\2\u01eb\u01ec\3\2\2\2\u01ec\u01ea\3\2\2\2\u01ec\u01ed\3\2\2\2\u01ed"+
-    "\u01ee\3\2\2\2\u01ee\u01ef\7\61\2\2\u01ef\u01f0\6H\3\2\u01f0\u0091\3\2"+
-    "\2\2\u01f1\u01f2\7v\2\2\u01f2\u01f3\7t\2\2\u01f3\u01f4\7w\2\2\u01f4\u01f5"+
-    "\7g\2\2\u01f5\u0093\3\2\2\2\u01f6\u01f7\7h\2\2\u01f7\u01f8\7c\2\2\u01f8"+
-    "\u01f9\7n\2\2\u01f9\u01fa\7u\2\2\u01fa\u01fb\7g\2\2\u01fb\u0095\3\2\2"+
-    "\2\u01fc\u01fd\7p\2\2\u01fd\u01fe\7w\2\2\u01fe\u01ff\7n\2\2\u01ff\u0200"+
-    "\7n\2\2\u0200\u0097\3\2\2\2\u0201\u0207\5\u009aM\2\u0202\u0203\5\24\n"+
-    "\2\u0203\u0204\5\u009aM\2\u0204\u0206\3\2\2\2\u0205\u0202\3\2\2\2\u0206"+
-    "\u0209\3\2\2\2\u0207\u0205\3\2\2\2\u0207\u0208\3\2\2\2\u0208\u020a\3\2"+
-    "\2\2\u0209\u0207\3\2\2\2\u020a\u020b\6L\4\2\u020b\u0099\3\2\2\2\u020c"+
-    "\u0210\t\21\2\2\u020d\u020f\t\22\2\2\u020e\u020d\3\2\2\2\u020f\u0212\3"+
-    "\2\2\2\u0210\u020e\3\2\2\2\u0210\u0211\3\2\2\2\u0211\u009b\3\2\2\2\u0212"+
-    "\u0210\3\2\2\2\u0213\u021c\7\62\2\2\u0214\u0218\t\b\2\2\u0215\u0217\t"+
-    "\t\2\2\u0216\u0215\3\2\2\2\u0217\u021a\3\2\2\2\u0218\u0216\3\2\2\2\u0218"+
-    "\u0219\3\2\2\2\u0219\u021c\3\2\2\2\u021a\u0218\3\2\2\2\u021b\u0213\3\2"+
-    "\2\2\u021b\u0214\3\2\2\2\u021c\u021d\3\2\2\2\u021d\u021e\bN\4\2\u021e"+
-    "\u009d\3\2\2\2\u021f\u0223\t\21\2\2\u0220\u0222\t\22\2\2\u0221\u0220\3"+
-    "\2\2\2\u0222\u0225\3\2\2\2\u0223\u0221\3\2\2\2\u0223\u0224\3\2\2\2\u0224"+
-    "\u0226\3\2\2\2\u0225\u0223\3\2\2\2\u0226\u0227\bO\4\2\u0227\u009f\3\2"+
-    "\2\2#\2\3\u00a3\u00ad\u00b7\u00bc\u0190\u0193\u019a\u019d\u01a4\u01a7"+
-    "\u01aa\u01b1\u01b4\u01ba\u01bc\u01c0\u01c5\u01c7\u01ca\u01d2\u01d4\u01de"+
-    "\u01e0\u01e4\u01ea\u01ec\u0207\u0210\u0218\u021b\u0223\5\b\2\2\4\3\2\4"+
-    "\2\2";
+    "I\tI\4J\tJ\4K\tK\4L\tL\4M\tM\4N\tN\4O\tO\4P\tP\4Q\tQ\3\2\6\2\u00a6\n\2"+
+    "\r\2\16\2\u00a7\3\2\3\2\3\3\3\3\3\3\3\3\7\3\u00b0\n\3\f\3\16\3\u00b3\13"+
+    "\3\3\3\3\3\3\3\3\3\3\3\7\3\u00ba\n\3\f\3\16\3\u00bd\13\3\3\3\3\3\5\3\u00c1"+
+    "\n\3\3\3\3\3\3\4\3\4\3\5\3\5\3\6\3\6\3\7\3\7\3\b\3\b\3\t\3\t\3\n\3\n\3"+
+    "\n\3\n\3\13\3\13\3\f\3\f\3\r\3\r\3\r\3\16\3\16\3\16\3\16\3\16\3\17\3\17"+
+    "\3\17\3\17\3\17\3\17\3\20\3\20\3\20\3\21\3\21\3\21\3\21\3\22\3\22\3\22"+
+    "\3\22\3\22\3\22\3\22\3\22\3\22\3\23\3\23\3\23\3\23\3\23\3\23\3\24\3\24"+
+    "\3\24\3\24\3\24\3\24\3\24\3\25\3\25\3\25\3\25\3\26\3\26\3\26\3\26\3\27"+
+    "\3\27\3\27\3\27\3\27\3\27\3\30\3\30\3\30\3\30\3\30\3\30\3\31\3\31\3\31"+
+    "\3\31\3\31\3\32\3\32\3\33\3\33\3\34\3\34\3\35\3\35\3\35\3\36\3\36\3\37"+
+    "\3\37\3 \3 \3!\3!\3!\3\"\3\"\3\"\3#\3#\3#\3#\3$\3$\3%\3%\3%\3&\3&\3\'"+
+    "\3\'\3\'\3(\3(\3(\3)\3)\3)\3)\3*\3*\3*\3+\3+\3+\3+\3,\3,\3-\3-\3.\3.\3"+
+    "/\3/\3/\3\60\3\60\3\60\3\61\3\61\3\62\3\62\3\63\3\63\3\63\3\64\3\64\3"+
+    "\64\3\65\3\65\3\65\3\66\3\66\3\66\3\66\3\67\3\67\3\67\38\38\38\39\39\3"+
+    ":\3:\3:\3;\3;\3;\3<\3<\3<\3=\3=\3=\3>\3>\3>\3?\3?\3?\3@\3@\3@\3A\3A\3"+
+    "A\3B\3B\3B\3B\3C\3C\3C\3C\3D\3D\3D\3D\3D\3E\3E\6E\u019a\nE\rE\16E\u019b"+
+    "\3E\5E\u019f\nE\3F\3F\3F\6F\u01a4\nF\rF\16F\u01a5\3F\5F\u01a9\nF\3G\3"+
+    "G\3G\7G\u01ae\nG\fG\16G\u01b1\13G\5G\u01b3\nG\3G\5G\u01b6\nG\3H\3H\3H"+
+    "\7H\u01bb\nH\fH\16H\u01be\13H\5H\u01c0\nH\3H\3H\6H\u01c4\nH\rH\16H\u01c5"+
+    "\5H\u01c8\nH\3H\3H\5H\u01cc\nH\3H\6H\u01cf\nH\rH\16H\u01d0\5H\u01d3\n"+
+    "H\3H\5H\u01d6\nH\3I\3I\3I\3I\3I\3I\7I\u01de\nI\fI\16I\u01e1\13I\3I\3I"+
+    "\3I\3I\3I\3I\3I\7I\u01ea\nI\fI\16I\u01ed\13I\3I\5I\u01f0\nI\3J\3J\3J\3"+
+    "J\6J\u01f6\nJ\rJ\16J\u01f7\3J\3J\3J\3K\3K\3K\3K\3K\3L\3L\3L\3L\3L\3L\3"+
+    "M\3M\3M\3M\3M\3N\3N\3N\3N\7N\u0211\nN\fN\16N\u0214\13N\3N\3N\3O\3O\7O"+
+    "\u021a\nO\fO\16O\u021d\13O\3P\3P\3P\7P\u0222\nP\fP\16P\u0225\13P\5P\u0227"+
+    "\nP\3P\3P\3Q\3Q\7Q\u022d\nQ\fQ\16Q\u0230\13Q\3Q\3Q\6\u00b1\u00bb\u01df"+
+    "\u01eb\2R\4\3\6\4\b\5\n\6\f\7\16\b\20\t\22\n\24\13\26\f\30\r\32\16\34"+
+    "\17\36\20 \21\"\22$\23&\24(\25*\26,\27.\30\60\31\62\32\64\33\66\348\35"+
+    ":\36<\37> @!B\"D#F$H%J&L\'N(P)R*T+V,X-Z.\\/^\60`\61b\62d\63f\64h\65j\66"+
+    "l\67n8p9r:t;v<x=z>|?~@\u0080A\u0082B\u0084C\u0086D\u0088E\u008aF\u008c"+
+    "G\u008eH\u0090I\u0092J\u0094K\u0096L\u0098M\u009aN\u009cO\u009eP\u00a0"+
+    "Q\u00a2R\4\2\3\23\5\2\13\f\17\17\"\"\4\2\f\f\17\17\3\2\629\4\2NNnn\4\2"+
+    "ZZzz\5\2\62;CHch\3\2\63;\3\2\62;\b\2FFHHNNffhhnn\4\2GGgg\4\2--//\4\2H"+
+    "Hhh\4\2$$^^\4\2\f\f\61\61\3\2\f\f\5\2C\\aac|\6\2\62;C\\aac|\u0252\2\4"+
+    "\3\2\2\2\2\6\3\2\2\2\2\b\3\2\2\2\2\n\3\2\2\2\2\f\3\2\2\2\2\16\3\2\2\2"+
+    "\2\20\3\2\2\2\2\22\3\2\2\2\2\24\3\2\2\2\2\26\3\2\2\2\2\30\3\2\2\2\2\32"+
+    "\3\2\2\2\2\34\3\2\2\2\2\36\3\2\2\2\2 \3\2\2\2\2\"\3\2\2\2\2$\3\2\2\2\2"+
+    "&\3\2\2\2\2(\3\2\2\2\2*\3\2\2\2\2,\3\2\2\2\2.\3\2\2\2\2\60\3\2\2\2\2\62"+
+    "\3\2\2\2\2\64\3\2\2\2\2\66\3\2\2\2\28\3\2\2\2\2:\3\2\2\2\2<\3\2\2\2\2"+
+    ">\3\2\2\2\2@\3\2\2\2\2B\3\2\2\2\2D\3\2\2\2\2F\3\2\2\2\2H\3\2\2\2\2J\3"+
+    "\2\2\2\2L\3\2\2\2\2N\3\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2T\3\2\2\2\2V\3\2\2"+
+    "\2\2X\3\2\2\2\2Z\3\2\2\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3\2\2\2\2b\3\2\2\2"+
+    "\2d\3\2\2\2\2f\3\2\2\2\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2\2\2n\3\2\2\2\2p"+
+    "\3\2\2\2\2r\3\2\2\2\2t\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2z\3\2\2\2\2|\3\2"+
+    "\2\2\2~\3\2\2\2\2\u0080\3\2\2\2\2\u0082\3\2\2\2\2\u0084\3\2\2\2\2\u0086"+
+    "\3\2\2\2\2\u0088\3\2\2\2\2\u008a\3\2\2\2\2\u008c\3\2\2\2\2\u008e\3\2\2"+
+    "\2\2\u0090\3\2\2\2\2\u0092\3\2\2\2\2\u0094\3\2\2\2\2\u0096\3\2\2\2\2\u0098"+
+    "\3\2\2\2\2\u009a\3\2\2\2\2\u009c\3\2\2\2\2\u009e\3\2\2\2\3\u00a0\3\2\2"+
+    "\2\3\u00a2\3\2\2\2\4\u00a5\3\2\2\2\6\u00c0\3\2\2\2\b\u00c4\3\2\2\2\n\u00c6"+
+    "\3\2\2\2\f\u00c8\3\2\2\2\16\u00ca\3\2\2\2\20\u00cc\3\2\2\2\22\u00ce\3"+
+    "\2\2\2\24\u00d0\3\2\2\2\26\u00d4\3\2\2\2\30\u00d6\3\2\2\2\32\u00d8\3\2"+
+    "\2\2\34\u00db\3\2\2\2\36\u00e0\3\2\2\2 \u00e6\3\2\2\2\"\u00e9\3\2\2\2"+
+    "$\u00ed\3\2\2\2&\u00f6\3\2\2\2(\u00fc\3\2\2\2*\u0103\3\2\2\2,\u0107\3"+
+    "\2\2\2.\u010b\3\2\2\2\60\u0111\3\2\2\2\62\u0117\3\2\2\2\64\u011c\3\2\2"+
+    "\2\66\u011e\3\2\2\28\u0120\3\2\2\2:\u0122\3\2\2\2<\u0125\3\2\2\2>\u0127"+
+    "\3\2\2\2@\u0129\3\2\2\2B\u012b\3\2\2\2D\u012e\3\2\2\2F\u0131\3\2\2\2H"+
+    "\u0135\3\2\2\2J\u0137\3\2\2\2L\u013a\3\2\2\2N\u013c\3\2\2\2P\u013f\3\2"+
+    "\2\2R\u0142\3\2\2\2T\u0146\3\2\2\2V\u0149\3\2\2\2X\u014d\3\2\2\2Z\u014f"+
+    "\3\2\2\2\\\u0151\3\2\2\2^\u0153\3\2\2\2`\u0156\3\2\2\2b\u0159\3\2\2\2"+
+    "d\u015b\3\2\2\2f\u015d\3\2\2\2h\u0160\3\2\2\2j\u0163\3\2\2\2l\u0166\3"+
+    "\2\2\2n\u016a\3\2\2\2p\u016d\3\2\2\2r\u0170\3\2\2\2t\u0172\3\2\2\2v\u0175"+
+    "\3\2\2\2x\u0178\3\2\2\2z\u017b\3\2\2\2|\u017e\3\2\2\2~\u0181\3\2\2\2\u0080"+
+    "\u0184\3\2\2\2\u0082\u0187\3\2\2\2\u0084\u018a\3\2\2\2\u0086\u018e\3\2"+
+    "\2\2\u0088\u0192\3\2\2\2\u008a\u0197\3\2\2\2\u008c\u01a0\3\2\2\2\u008e"+
+    "\u01b2\3\2\2\2\u0090\u01bf\3\2\2\2\u0092\u01ef\3\2\2\2\u0094\u01f1\3\2"+
+    "\2\2\u0096\u01fc\3\2\2\2\u0098\u0201\3\2\2\2\u009a\u0207\3\2\2\2\u009c"+
+    "\u020c\3\2\2\2\u009e\u0217\3\2\2\2\u00a0\u0226\3\2\2\2\u00a2\u022a\3\2"+
+    "\2\2\u00a4\u00a6\t\2\2\2\u00a5\u00a4\3\2\2\2\u00a6\u00a7\3\2\2\2\u00a7"+
+    "\u00a5\3\2\2\2\u00a7\u00a8\3\2\2\2\u00a8\u00a9\3\2\2\2\u00a9\u00aa\b\2"+
+    "\2\2\u00aa\5\3\2\2\2\u00ab\u00ac\7\61\2\2\u00ac\u00ad\7\61\2\2\u00ad\u00b1"+
+    "\3\2\2\2\u00ae\u00b0\13\2\2\2\u00af\u00ae\3\2\2\2\u00b0\u00b3\3\2\2\2"+
+    "\u00b1\u00b2\3\2\2\2\u00b1\u00af\3\2\2\2\u00b2\u00b4\3\2\2\2\u00b3\u00b1"+
+    "\3\2\2\2\u00b4\u00c1\t\3\2\2\u00b5\u00b6\7\61\2\2\u00b6\u00b7\7,\2\2\u00b7"+
+    "\u00bb\3\2\2\2\u00b8\u00ba\13\2\2\2\u00b9\u00b8\3\2\2\2\u00ba\u00bd\3"+
+    "\2\2\2\u00bb\u00bc\3\2\2\2\u00bb\u00b9\3\2\2\2\u00bc\u00be\3\2\2\2\u00bd"+
+    "\u00bb\3\2\2\2\u00be\u00bf\7,\2\2\u00bf\u00c1\7\61\2\2\u00c0\u00ab\3\2"+
+    "\2\2\u00c0\u00b5\3\2\2\2\u00c1\u00c2\3\2\2\2\u00c2\u00c3\b\3\2\2\u00c3"+
+    "\7\3\2\2\2\u00c4\u00c5\7}\2\2\u00c5\t\3\2\2\2\u00c6\u00c7\7\177\2\2\u00c7"+
+    "\13\3\2\2\2\u00c8\u00c9\7]\2\2\u00c9\r\3\2\2\2\u00ca\u00cb\7_\2\2\u00cb"+
+    "\17\3\2\2\2\u00cc\u00cd\7*\2\2\u00cd\21\3\2\2\2\u00ce\u00cf\7+\2\2\u00cf"+
+    "\23\3\2\2\2\u00d0\u00d1\7\60\2\2\u00d1\u00d2\3\2\2\2\u00d2\u00d3\b\n\3"+
+    "\2\u00d3\25\3\2\2\2\u00d4\u00d5\7.\2\2\u00d5\27\3\2\2\2\u00d6\u00d7\7"+
+    "=\2\2\u00d7\31\3\2\2\2\u00d8\u00d9\7k\2\2\u00d9\u00da\7h\2\2\u00da\33"+
+    "\3\2\2\2\u00db\u00dc\7g\2\2\u00dc\u00dd\7n\2\2\u00dd\u00de\7u\2\2\u00de"+
+    "\u00df\7g\2\2\u00df\35\3\2\2\2\u00e0\u00e1\7y\2\2\u00e1\u00e2\7j\2\2\u00e2"+
+    "\u00e3\7k\2\2\u00e3\u00e4\7n\2\2\u00e4\u00e5\7g\2\2\u00e5\37\3\2\2\2\u00e6"+
+    "\u00e7\7f\2\2\u00e7\u00e8\7q\2\2\u00e8!\3\2\2\2\u00e9\u00ea\7h\2\2\u00ea"+
+    "\u00eb\7q\2\2\u00eb\u00ec\7t\2\2\u00ec#\3\2\2\2\u00ed\u00ee\7e\2\2\u00ee"+
+    "\u00ef\7q\2\2\u00ef\u00f0\7p\2\2\u00f0\u00f1\7v\2\2\u00f1\u00f2\7k\2\2"+
+    "\u00f2\u00f3\7p\2\2\u00f3\u00f4\7w\2\2\u00f4\u00f5\7g\2\2\u00f5%\3\2\2"+
+    "\2\u00f6\u00f7\7d\2\2\u00f7\u00f8\7t\2\2\u00f8\u00f9\7g\2\2\u00f9\u00fa"+
+    "\7c\2\2\u00fa\u00fb\7m\2\2\u00fb\'\3\2\2\2\u00fc\u00fd\7t\2\2\u00fd\u00fe"+
+    "\7g\2\2\u00fe\u00ff\7v\2\2\u00ff\u0100\7w\2\2\u0100\u0101\7t\2\2\u0101"+
+    "\u0102\7p\2\2\u0102)\3\2\2\2\u0103\u0104\7p\2\2\u0104\u0105\7g\2\2\u0105"+
+    "\u0106\7y\2\2\u0106+\3\2\2\2\u0107\u0108\7v\2\2\u0108\u0109\7t\2\2\u0109"+
+    "\u010a\7{\2\2\u010a-\3\2\2\2\u010b\u010c\7e\2\2\u010c\u010d\7c\2\2\u010d"+
+    "\u010e\7v\2\2\u010e\u010f\7e\2\2\u010f\u0110\7j\2\2\u0110/\3\2\2\2\u0111"+
+    "\u0112\7v\2\2\u0112\u0113\7j\2\2\u0113\u0114\7t\2\2\u0114\u0115\7q\2\2"+
+    "\u0115\u0116\7y\2\2\u0116\61\3\2\2\2\u0117\u0118\7v\2\2\u0118\u0119\7"+
+    "j\2\2\u0119\u011a\7k\2\2\u011a\u011b\7u\2\2\u011b\63\3\2\2\2\u011c\u011d"+
+    "\7#\2\2\u011d\65\3\2\2\2\u011e\u011f\7\u0080\2\2\u011f\67\3\2\2\2\u0120"+
+    "\u0121\7,\2\2\u01219\3\2\2\2\u0122\u0123\7\61\2\2\u0123\u0124\6\35\2\2"+
+    "\u0124;\3\2\2\2\u0125\u0126\7\'\2\2\u0126=\3\2\2\2\u0127\u0128\7-\2\2"+
+    "\u0128?\3\2\2\2\u0129\u012a\7/\2\2\u012aA\3\2\2\2\u012b\u012c\7>\2\2\u012c"+
+    "\u012d\7>\2\2\u012dC\3\2\2\2\u012e\u012f\7@\2\2\u012f\u0130\7@\2\2\u0130"+
+    "E\3\2\2\2\u0131\u0132\7@\2\2\u0132\u0133\7@\2\2\u0133\u0134\7@\2\2\u0134"+
+    "G\3\2\2\2\u0135\u0136\7>\2\2\u0136I\3\2\2\2\u0137\u0138\7>\2\2\u0138\u0139"+
+    "\7?\2\2\u0139K\3\2\2\2\u013a\u013b\7@\2\2\u013bM\3\2\2\2\u013c\u013d\7"+
+    "@\2\2\u013d\u013e\7?\2\2\u013eO\3\2\2\2\u013f\u0140\7?\2\2\u0140\u0141"+
+    "\7?\2\2\u0141Q\3\2\2\2\u0142\u0143\7?\2\2\u0143\u0144\7?\2\2\u0144\u0145"+
+    "\7?\2\2\u0145S\3\2\2\2\u0146\u0147\7#\2\2\u0147\u0148\7?\2\2\u0148U\3"+
+    "\2\2\2\u0149\u014a\7#\2\2\u014a\u014b\7?\2\2\u014b\u014c\7?\2\2\u014c"+
+    "W\3\2\2\2\u014d\u014e\7(\2\2\u014eY\3\2\2\2\u014f\u0150\7`\2\2\u0150["+
+    "\3\2\2\2\u0151\u0152\7~\2\2\u0152]\3\2\2\2\u0153\u0154\7(\2\2\u0154\u0155"+
+    "\7(\2\2\u0155_\3\2\2\2\u0156\u0157\7~\2\2\u0157\u0158\7~\2\2\u0158a\3"+
+    "\2\2\2\u0159\u015a\7A\2\2\u015ac\3\2\2\2\u015b\u015c\7<\2\2\u015ce\3\2"+
+    "\2\2\u015d\u015e\7<\2\2\u015e\u015f\7<\2\2\u015fg\3\2\2\2\u0160\u0161"+
+    "\7/\2\2\u0161\u0162\7@\2\2\u0162i\3\2\2\2\u0163\u0164\7?\2\2\u0164\u0165"+
+    "\7\u0080\2\2\u0165k\3\2\2\2\u0166\u0167\7?\2\2\u0167\u0168\7?\2\2\u0168"+
+    "\u0169\7\u0080\2\2\u0169m\3\2\2\2\u016a\u016b\7-\2\2\u016b\u016c\7-\2"+
+    "\2\u016co\3\2\2\2\u016d\u016e\7/\2\2\u016e\u016f\7/\2\2\u016fq\3\2\2\2"+
+    "\u0170\u0171\7?\2\2\u0171s\3\2\2\2\u0172\u0173\7-\2\2\u0173\u0174\7?\2"+
+    "\2\u0174u\3\2\2\2\u0175\u0176\7/\2\2\u0176\u0177\7?\2\2\u0177w\3\2\2\2"+
+    "\u0178\u0179\7,\2\2\u0179\u017a\7?\2\2\u017ay\3\2\2\2\u017b\u017c\7\61"+
+    "\2\2\u017c\u017d\7?\2\2\u017d{\3\2\2\2\u017e\u017f\7\'\2\2\u017f\u0180"+
+    "\7?\2\2\u0180}\3\2\2\2\u0181\u0182\7(\2\2\u0182\u0183\7?\2\2\u0183\177"+
+    "\3\2\2\2\u0184\u0185\7`\2\2\u0185\u0186\7?\2\2\u0186\u0081\3\2\2\2\u0187"+
+    "\u0188\7~\2\2\u0188\u0189\7?\2\2\u0189\u0083\3\2\2\2\u018a\u018b\7>\2"+
+    "\2\u018b\u018c\7>\2\2\u018c\u018d\7?\2\2\u018d\u0085\3\2\2\2\u018e\u018f"+
+    "\7@\2\2\u018f\u0190\7@\2\2\u0190\u0191\7?\2\2\u0191\u0087\3\2\2\2\u0192"+
+    "\u0193\7@\2\2\u0193\u0194\7@\2\2\u0194\u0195\7@\2\2\u0195\u0196\7?\2\2"+
+    "\u0196\u0089\3\2\2\2\u0197\u0199\7\62\2\2\u0198\u019a\t\4\2\2\u0199\u0198"+
+    "\3\2\2\2\u019a\u019b\3\2\2\2\u019b\u0199\3\2\2\2\u019b\u019c\3\2\2\2\u019c"+
+    "\u019e\3\2\2\2\u019d\u019f\t\5\2\2\u019e\u019d\3\2\2\2\u019e\u019f\3\2"+
+    "\2\2\u019f\u008b\3\2\2\2\u01a0\u01a1\7\62\2\2\u01a1\u01a3\t\6\2\2\u01a2"+
+    "\u01a4\t\7\2\2\u01a3\u01a2\3\2\2\2\u01a4\u01a5\3\2\2\2\u01a5\u01a3\3\2"+
+    "\2\2\u01a5\u01a6\3\2\2\2\u01a6\u01a8\3\2\2\2\u01a7\u01a9\t\5\2\2\u01a8"+
+    "\u01a7\3\2\2\2\u01a8\u01a9\3\2\2\2\u01a9\u008d\3\2\2\2\u01aa\u01b3\7\62"+
+    "\2\2\u01ab\u01af\t\b\2\2\u01ac\u01ae\t\t\2\2\u01ad\u01ac\3\2\2\2\u01ae"+
+    "\u01b1\3\2\2\2\u01af\u01ad\3\2\2\2\u01af\u01b0\3\2\2\2\u01b0\u01b3\3\2"+
+    "\2\2\u01b1\u01af\3\2\2\2\u01b2\u01aa\3\2\2\2\u01b2\u01ab\3\2\2\2\u01b3"+
+    "\u01b5\3\2\2\2\u01b4\u01b6\t\n\2\2\u01b5\u01b4\3\2\2\2\u01b5\u01b6\3\2"+
+    "\2\2\u01b6\u008f\3\2\2\2\u01b7\u01c0\7\62\2\2\u01b8\u01bc\t\b\2\2\u01b9"+
+    "\u01bb\t\t\2\2\u01ba\u01b9\3\2\2\2\u01bb\u01be\3\2\2\2\u01bc\u01ba\3\2"+
+    "\2\2\u01bc\u01bd\3\2\2\2\u01bd\u01c0\3\2\2\2\u01be\u01bc\3\2\2\2\u01bf"+
+    "\u01b7\3\2\2\2\u01bf\u01b8\3\2\2\2\u01c0\u01c7\3\2\2\2\u01c1\u01c3\5\24"+
+    "\n\2\u01c2\u01c4\t\t\2\2\u01c3\u01c2\3\2\2\2\u01c4\u01c5\3\2\2\2\u01c5"+
+    "\u01c3\3\2\2\2\u01c5\u01c6\3\2\2\2\u01c6\u01c8\3\2\2\2\u01c7\u01c1\3\2"+
+    "\2\2\u01c7\u01c8\3\2\2\2\u01c8\u01d2\3\2\2\2\u01c9\u01cb\t\13\2\2\u01ca"+
+    "\u01cc\t\f\2\2\u01cb\u01ca\3\2\2\2\u01cb\u01cc\3\2\2\2\u01cc\u01ce\3\2"+
+    "\2\2\u01cd\u01cf\t\t\2\2\u01ce\u01cd\3\2\2\2\u01cf\u01d0\3\2\2\2\u01d0"+
+    "\u01ce\3\2\2\2\u01d0\u01d1\3\2\2\2\u01d1\u01d3\3\2\2\2\u01d2\u01c9\3\2"+
+    "\2\2\u01d2\u01d3\3\2\2\2\u01d3\u01d5\3\2\2\2\u01d4\u01d6\t\r\2\2\u01d5"+
+    "\u01d4\3\2\2\2\u01d5\u01d6\3\2\2\2\u01d6\u0091\3\2\2\2\u01d7\u01df\7$"+
+    "\2\2\u01d8\u01d9\7^\2\2\u01d9\u01de\7$\2\2\u01da\u01db\7^\2\2\u01db\u01de"+
+    "\7^\2\2\u01dc\u01de\n\16\2\2\u01dd\u01d8\3\2\2\2\u01dd\u01da\3\2\2\2\u01dd"+
+    "\u01dc\3\2\2\2\u01de\u01e1\3\2\2\2\u01df\u01e0\3\2\2\2\u01df\u01dd\3\2"+
+    "\2\2\u01e0\u01e2\3\2\2\2\u01e1\u01df\3\2\2\2\u01e2\u01f0\7$\2\2\u01e3"+
+    "\u01eb\7)\2\2\u01e4\u01e5\7^\2\2\u01e5\u01ea\7)\2\2\u01e6\u01e7\7^\2\2"+
+    "\u01e7\u01ea\7^\2\2\u01e8\u01ea\n\16\2\2\u01e9\u01e4\3\2\2\2\u01e9\u01e6"+
+    "\3\2\2\2\u01e9\u01e8\3\2\2\2\u01ea\u01ed\3\2\2\2\u01eb\u01ec\3\2\2\2\u01eb"+
+    "\u01e9\3\2\2\2\u01ec\u01ee\3\2\2\2\u01ed\u01eb\3\2\2\2\u01ee\u01f0\7)"+
+    "\2\2\u01ef\u01d7\3\2\2\2\u01ef\u01e3\3\2\2\2\u01f0\u0093\3\2\2\2\u01f1"+
+    "\u01f5\7\61\2\2\u01f2\u01f6\n\17\2\2\u01f3\u01f4\7^\2\2\u01f4\u01f6\n"+
+    "\20\2\2\u01f5\u01f2\3\2\2\2\u01f5\u01f3\3\2\2\2\u01f6\u01f7\3\2\2\2\u01f7"+
+    "\u01f5\3\2\2\2\u01f7\u01f8\3\2\2\2\u01f8\u01f9\3\2\2\2\u01f9\u01fa\7\61"+
+    "\2\2\u01fa\u01fb\6J\3\2\u01fb\u0095\3\2\2\2\u01fc\u01fd\7v\2\2\u01fd\u01fe"+
+    "\7t\2\2\u01fe\u01ff\7w\2\2\u01ff\u0200\7g\2\2\u0200\u0097\3\2\2\2\u0201"+
+    "\u0202\7h\2\2\u0202\u0203\7c\2\2\u0203\u0204\7n\2\2\u0204\u0205\7u\2\2"+
+    "\u0205\u0206\7g\2\2\u0206\u0099\3\2\2\2\u0207\u0208\7p\2\2\u0208\u0209"+
+    "\7w\2\2\u0209\u020a\7n\2\2\u020a\u020b\7n\2\2\u020b\u009b\3\2\2\2\u020c"+
+    "\u0212\5\u009eO\2\u020d\u020e\5\24\n\2\u020e\u020f\5\u009eO\2\u020f\u0211"+
+    "\3\2\2\2\u0210\u020d\3\2\2\2\u0211\u0214\3\2\2\2\u0212\u0210\3\2\2\2\u0212"+
+    "\u0213\3\2\2\2\u0213\u0215\3\2\2\2\u0214\u0212\3\2\2\2\u0215\u0216\6N"+
+    "\4\2\u0216\u009d\3\2\2\2\u0217\u021b\t\21\2\2\u0218\u021a\t\22\2\2\u0219"+
+    "\u0218\3\2\2\2\u021a\u021d\3\2\2\2\u021b\u0219\3\2\2\2\u021b\u021c\3\2"+
+    "\2\2\u021c\u009f\3\2\2\2\u021d\u021b\3\2\2\2\u021e\u0227\7\62\2\2\u021f"+
+    "\u0223\t\b\2\2\u0220\u0222\t\t\2\2\u0221\u0220\3\2\2\2\u0222\u0225\3\2"+
+    "\2\2\u0223\u0221\3\2\2\2\u0223\u0224\3\2\2\2\u0224\u0227\3\2\2\2\u0225"+
+    "\u0223\3\2\2\2\u0226\u021e\3\2\2\2\u0226\u021f\3\2\2\2\u0227\u0228\3\2"+
+    "\2\2\u0228\u0229\bP\4\2\u0229\u00a1\3\2\2\2\u022a\u022e\t\21\2\2\u022b"+
+    "\u022d\t\22\2\2\u022c\u022b\3\2\2\2\u022d\u0230\3\2\2\2\u022e\u022c\3"+
+    "\2\2\2\u022e\u022f\3\2\2\2\u022f\u0231\3\2\2\2\u0230\u022e\3\2\2\2\u0231"+
+    "\u0232\bQ\4\2\u0232\u00a3\3\2\2\2#\2\3\u00a7\u00b1\u00bb\u00c0\u019b\u019e"+
+    "\u01a5\u01a8\u01af\u01b2\u01b5\u01bc\u01bf\u01c5\u01c7\u01cb\u01d0\u01d2"+
+    "\u01d5\u01dd\u01df\u01e9\u01eb\u01ef\u01f5\u01f7\u0212\u021b\u0223\u0226"+
+    "\u022e\5\b\2\2\4\3\2\4\2\2";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
index ae9508f7969..ba3798bb414 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
@@ -23,10 +23,10 @@ class PainlessParser extends Parser {
     BWNOT=26, MUL=27, DIV=28, REM=29, ADD=30, SUB=31, LSH=32, RSH=33, USH=34, 
     LT=35, LTE=36, GT=37, GTE=38, EQ=39, EQR=40, NE=41, NER=42, BWAND=43, 
     XOR=44, BWOR=45, BOOLAND=46, BOOLOR=47, COND=48, COLON=49, REF=50, ARROW=51, 
-    INCR=52, DECR=53, ASSIGN=54, AADD=55, ASUB=56, AMUL=57, ADIV=58, AREM=59, 
-    AAND=60, AXOR=61, AOR=62, ALSH=63, ARSH=64, AUSH=65, OCTAL=66, HEX=67, 
-    INTEGER=68, DECIMAL=69, STRING=70, REGEX=71, TRUE=72, FALSE=73, NULL=74, 
-    TYPE=75, ID=76, DOTINTEGER=77, DOTID=78;
+    FIND=52, MATCH=53, INCR=54, DECR=55, ASSIGN=56, AADD=57, ASUB=58, AMUL=59, 
+    ADIV=60, AREM=61, AAND=62, AXOR=63, AOR=64, ALSH=65, ARSH=66, AUSH=67, 
+    OCTAL=68, HEX=69, INTEGER=70, DECIMAL=71, STRING=72, REGEX=73, TRUE=74, 
+    FALSE=75, NULL=76, TYPE=77, ID=78, DOTINTEGER=79, DOTID=80;
   public static final int
     RULE_source = 0, RULE_function = 1, RULE_parameters = 2, RULE_statement = 3, 
     RULE_trailer = 4, RULE_block = 5, RULE_empty = 6, RULE_initializer = 7, 
@@ -50,9 +50,10 @@ class PainlessParser extends Parser {
     "'return'", "'new'", "'try'", "'catch'", "'throw'", "'this'", "'!'", "'~'", 
     "'*'", "'/'", "'%'", "'+'", "'-'", "'<<'", "'>>'", "'>>>'", "'<'", "'<='", 
     "'>'", "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", "'^'", "'|'", 
-    "'&&'", "'||'", "'?'", "':'", "'::'", "'->'", "'++'", "'--'", "'='", "'+='", 
-    "'-='", "'*='", "'/='", "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", 
-    "'>>>='", null, null, null, null, null, null, "'true'", "'false'", "'null'"
+    "'&&'", "'||'", "'?'", "':'", "'::'", "'->'", "'=~'", "'==~'", "'++'", 
+    "'--'", "'='", "'+='", "'-='", "'*='", "'/='", "'%='", "'&='", "'^='", 
+    "'|='", "'<<='", "'>>='", "'>>>='", null, null, null, null, null, null, 
+    "'true'", "'false'", "'null'"
   };
   private static final String[] _SYMBOLIC_NAMES = {
     null, "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", 
@@ -60,10 +61,11 @@ class PainlessParser extends Parser {
     "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", "BOOLNOT", 
     "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", 
     "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", 
-    "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "INCR", "DECR", 
-    "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", "AAND", "AXOR", "AOR", 
-    "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", "DECIMAL", "STRING", 
-    "REGEX", "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", "DOTID"
+    "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "FIND", "MATCH", 
+    "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", "AAND", 
+    "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", "DECIMAL", 
+    "STRING", "REGEX", "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", 
+    "DOTID"
   };
   public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
 
@@ -1460,6 +1462,8 @@ class PainlessParser extends Parser {
     public TerminalNode REM() { return getToken(PainlessParser.REM, 0); }
     public TerminalNode ADD() { return getToken(PainlessParser.ADD, 0); }
     public TerminalNode SUB() { return getToken(PainlessParser.SUB, 0); }
+    public TerminalNode FIND() { return getToken(PainlessParser.FIND, 0); }
+    public TerminalNode MATCH() { return getToken(PainlessParser.MATCH, 0); }
     public TerminalNode LSH() { return getToken(PainlessParser.LSH, 0); }
     public TerminalNode RSH() { return getToken(PainlessParser.RSH, 0); }
     public TerminalNode USH() { return getToken(PainlessParser.USH, 0); }
@@ -1502,7 +1506,7 @@ class PainlessParser extends Parser {
         chain(true);
         setState(234);
         _la = _input.LA(1);
-        if ( !(((((_la - 54)) & ~0x3f) == 0 && ((1L << (_la - 54)) & ((1L << (ASSIGN - 54)) | (1L << (AADD - 54)) | (1L << (ASUB - 54)) | (1L << (AMUL - 54)) | (1L << (ADIV - 54)) | (1L << (AREM - 54)) | (1L << (AAND - 54)) | (1L << (AXOR - 54)) | (1L << (AOR - 54)) | (1L << (ALSH - 54)) | (1L << (ARSH - 54)) | (1L << (AUSH - 54)))) != 0)) ) {
+        if ( !(((((_la - 56)) & ~0x3f) == 0 && ((1L << (_la - 56)) & ((1L << (ASSIGN - 56)) | (1L << (AADD - 56)) | (1L << (ASUB - 56)) | (1L << (AMUL - 56)) | (1L << (ADIV - 56)) | (1L << (AREM - 56)) | (1L << (AAND - 56)) | (1L << (AXOR - 56)) | (1L << (AOR - 56)) | (1L << (ALSH - 56)) | (1L << (ARSH - 56)) | (1L << (AUSH - 56)))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
@@ -1524,7 +1528,7 @@ class PainlessParser extends Parser {
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(302);
+      setState(307);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -1532,14 +1536,14 @@ class PainlessParser extends Parser {
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(300);
+          setState(305);
           switch ( getInterpreter().adaptivePredict(_input,20,_ctx) ) {
           case 1:
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(243);
-            if (!(precpred(_ctx, 12))) throw new FailedPredicateException(this, "precpred(_ctx, 12)");
+            if (!(precpred(_ctx, 13))) throw new FailedPredicateException(this, "precpred(_ctx, 13)");
             setState(244);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << MUL) | (1L << DIV) | (1L << REM))) != 0)) ) {
@@ -1548,7 +1552,7 @@ class PainlessParser extends Parser {
               consume();
             }
             setState(245);
-            expression(13);
+            expression(14);
              ((BinaryContext)_localctx).s =  false; 
             }
             break;
@@ -1557,7 +1561,7 @@ class PainlessParser extends Parser {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(248);
-            if (!(precpred(_ctx, 11))) throw new FailedPredicateException(this, "precpred(_ctx, 11)");
+            if (!(precpred(_ctx, 12))) throw new FailedPredicateException(this, "precpred(_ctx, 12)");
             setState(249);
             _la = _input.LA(1);
             if ( !(_la==ADD || _la==SUB) ) {
@@ -1566,7 +1570,7 @@ class PainlessParser extends Parser {
               consume();
             }
             setState(250);
-            expression(12);
+            expression(13);
              ((BinaryContext)_localctx).s =  false; 
             }
             break;
@@ -1575,78 +1579,83 @@ class PainlessParser extends Parser {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(253);
-            if (!(precpred(_ctx, 10))) throw new FailedPredicateException(this, "precpred(_ctx, 10)");
+            if (!(precpred(_ctx, 11))) throw new FailedPredicateException(this, "precpred(_ctx, 11)");
             setState(254);
             _la = _input.LA(1);
+            if ( !(_la==FIND || _la==MATCH) ) {
+            _errHandler.recoverInline(this);
+            } else {
+              consume();
+            }
+            setState(255);
+            expression(12);
+             ((BinaryContext)_localctx).s =  false; 
+            }
+            break;
+          case 4:
+            {
+            _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
+            pushNewRecursionContext(_localctx, _startState, RULE_expression);
+            setState(258);
+            if (!(precpred(_ctx, 10))) throw new FailedPredicateException(this, "precpred(_ctx, 10)");
+            setState(259);
+            _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LSH) | (1L << RSH) | (1L << USH))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(255);
+            setState(260);
             expression(11);
              ((BinaryContext)_localctx).s =  false; 
             }
             break;
-          case 4:
-            {
-            _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
-            pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(258);
-            if (!(precpred(_ctx, 9))) throw new FailedPredicateException(this, "precpred(_ctx, 9)");
-            setState(259);
-            _la = _input.LA(1);
-            if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LT) | (1L << LTE) | (1L << GT) | (1L << GTE))) != 0)) ) {
-            _errHandler.recoverInline(this);
-            } else {
-              consume();
-            }
-            setState(260);
-            expression(10);
-             ((CompContext)_localctx).s =  false; 
-            }
-            break;
           case 5:
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(263);
-            if (!(precpred(_ctx, 8))) throw new FailedPredicateException(this, "precpred(_ctx, 8)");
+            if (!(precpred(_ctx, 9))) throw new FailedPredicateException(this, "precpred(_ctx, 9)");
             setState(264);
             _la = _input.LA(1);
+            if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LT) | (1L << LTE) | (1L << GT) | (1L << GTE))) != 0)) ) {
+            _errHandler.recoverInline(this);
+            } else {
+              consume();
+            }
+            setState(265);
+            expression(10);
+             ((CompContext)_localctx).s =  false; 
+            }
+            break;
+          case 6:
+            {
+            _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
+            pushNewRecursionContext(_localctx, _startState, RULE_expression);
+            setState(268);
+            if (!(precpred(_ctx, 8))) throw new FailedPredicateException(this, "precpred(_ctx, 8)");
+            setState(269);
+            _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << EQ) | (1L << EQR) | (1L << NE) | (1L << NER))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(265);
+            setState(270);
             expression(9);
              ((CompContext)_localctx).s =  false; 
             }
             break;
-          case 6:
-            {
-            _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
-            pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(268);
-            if (!(precpred(_ctx, 7))) throw new FailedPredicateException(this, "precpred(_ctx, 7)");
-            setState(269);
-            match(BWAND);
-            setState(270);
-            expression(8);
-             ((BinaryContext)_localctx).s =  false; 
-            }
-            break;
           case 7:
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(273);
-            if (!(precpred(_ctx, 6))) throw new FailedPredicateException(this, "precpred(_ctx, 6)");
+            if (!(precpred(_ctx, 7))) throw new FailedPredicateException(this, "precpred(_ctx, 7)");
             setState(274);
-            match(XOR);
+            match(BWAND);
             setState(275);
-            expression(7);
+            expression(8);
              ((BinaryContext)_localctx).s =  false; 
             }
             break;
@@ -1655,25 +1664,25 @@ class PainlessParser extends Parser {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(278);
-            if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)");
+            if (!(precpred(_ctx, 6))) throw new FailedPredicateException(this, "precpred(_ctx, 6)");
             setState(279);
-            match(BWOR);
+            match(XOR);
             setState(280);
-            expression(6);
+            expression(7);
              ((BinaryContext)_localctx).s =  false; 
             }
             break;
           case 9:
             {
-            _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
+            _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(283);
-            if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)");
+            if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)");
             setState(284);
-            match(BOOLAND);
+            match(BWOR);
             setState(285);
-            expression(5);
-             ((BoolContext)_localctx).s =  false; 
+            expression(6);
+             ((BinaryContext)_localctx).s =  false; 
             }
             break;
           case 10:
@@ -1681,27 +1690,40 @@ class PainlessParser extends Parser {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(288);
-            if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)");
+            if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)");
             setState(289);
-            match(BOOLOR);
+            match(BOOLAND);
             setState(290);
-            expression(4);
+            expression(5);
              ((BoolContext)_localctx).s =  false; 
             }
             break;
           case 11:
             {
-            _localctx = new ConditionalContext(new ExpressionContext(_parentctx, _parentState));
+            _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(293);
-            if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
+            if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)");
             setState(294);
-            match(COND);
+            match(BOOLOR);
             setState(295);
+            expression(4);
+             ((BoolContext)_localctx).s =  false; 
+            }
+            break;
+          case 12:
+            {
+            _localctx = new ConditionalContext(new ExpressionContext(_parentctx, _parentState));
+            pushNewRecursionContext(_localctx, _startState, RULE_expression);
+            setState(298);
+            if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
+            setState(299);
+            match(COND);
+            setState(300);
             ((ConditionalContext)_localctx).e0 = expression(0);
-            setState(296);
+            setState(301);
             match(COLON);
-            setState(297);
+            setState(302);
             ((ConditionalContext)_localctx).e1 = expression(2);
              ((ConditionalContext)_localctx).s =  ((ConditionalContext)_localctx).e0.s && ((ConditionalContext)_localctx).e1.s; 
             }
@@ -1709,7 +1731,7 @@ class PainlessParser extends Parser {
           }
           } 
         }
-        setState(304);
+        setState(309);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
       }
@@ -1856,22 +1878,22 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 30, RULE_unary);
     int _la;
     try {
-      setState(334);
+      setState(339);
       switch ( getInterpreter().adaptivePredict(_input,22,_ctx) ) {
       case 1:
         _localctx = new PreContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(305);
+        setState(310);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(306);
+        setState(311);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(307);
+        setState(312);
         chain(true);
         }
         break;
@@ -1879,11 +1901,11 @@ class PainlessParser extends Parser {
         _localctx = new PostContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(308);
+        setState(313);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(309);
+        setState(314);
         chain(true);
-        setState(310);
+        setState(315);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
@@ -1896,9 +1918,9 @@ class PainlessParser extends Parser {
         _localctx = new ReadContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(312);
+        setState(317);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(313);
+        setState(318);
         chain(false);
         }
         break;
@@ -1906,11 +1928,11 @@ class PainlessParser extends Parser {
         _localctx = new NumericContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(314);
+        setState(319);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(315);
+        setState(320);
         _la = _input.LA(1);
-        if ( !(((((_la - 66)) & ~0x3f) == 0 && ((1L << (_la - 66)) & ((1L << (OCTAL - 66)) | (1L << (HEX - 66)) | (1L << (INTEGER - 66)) | (1L << (DECIMAL - 66)))) != 0)) ) {
+        if ( !(((((_la - 68)) & ~0x3f) == 0 && ((1L << (_la - 68)) & ((1L << (OCTAL - 68)) | (1L << (HEX - 68)) | (1L << (INTEGER - 68)) | (1L << (DECIMAL - 68)))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
@@ -1922,9 +1944,9 @@ class PainlessParser extends Parser {
         _localctx = new TrueContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(317);
+        setState(322);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(318);
+        setState(323);
         match(TRUE);
          ((TrueContext)_localctx).s =  false; 
         }
@@ -1933,9 +1955,9 @@ class PainlessParser extends Parser {
         _localctx = new FalseContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(320);
+        setState(325);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(321);
+        setState(326);
         match(FALSE);
          ((FalseContext)_localctx).s =  false; 
         }
@@ -1944,9 +1966,9 @@ class PainlessParser extends Parser {
         _localctx = new NullContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(323);
+        setState(328);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(324);
+        setState(329);
         match(NULL);
          ((NullContext)_localctx).s =  false; 
         }
@@ -1955,16 +1977,16 @@ class PainlessParser extends Parser {
         _localctx = new OperatorContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(326);
+        setState(331);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(327);
+        setState(332);
         _la = _input.LA(1);
         if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(328);
+        setState(333);
         unary(false);
         }
         break;
@@ -1972,13 +1994,13 @@ class PainlessParser extends Parser {
         _localctx = new CastContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(329);
+        setState(334);
         match(LP);
-        setState(330);
+        setState(335);
         decltype();
-        setState(331);
+        setState(336);
         match(RP);
-        setState(332);
+        setState(337);
         unary(_localctx.c);
         }
         break;
@@ -2087,27 +2109,27 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 32, RULE_chain);
     try {
       int _alt;
-      setState(370);
+      setState(375);
       switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) {
       case 1:
         _localctx = new DynamicContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(336);
+        setState(341);
         ((DynamicContext)_localctx).p = primary(_localctx.c);
-        setState(340);
+        setState(345);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(337);
+            setState(342);
             secondary(((DynamicContext)_localctx).p.s);
             }
             } 
           }
-          setState(342);
+          setState(347);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
         }
@@ -2117,23 +2139,23 @@ class PainlessParser extends Parser {
         _localctx = new StaticContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(343);
-        decltype();
-        setState(344);
-        dot();
         setState(348);
+        decltype();
+        setState(349);
+        dot();
+        setState(353);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,24,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(345);
+            setState(350);
             secondary(true);
             }
             } 
           }
-          setState(350);
+          setState(355);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,24,_ctx);
         }
@@ -2143,11 +2165,11 @@ class PainlessParser extends Parser {
         _localctx = new NewarrayContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(351);
+        setState(356);
         match(NEW);
-        setState(352);
+        setState(357);
         match(TYPE);
-        setState(357); 
+        setState(362); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -2155,11 +2177,11 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(353);
+            setState(358);
             match(LBRACE);
-            setState(354);
+            setState(359);
             expression(0);
-            setState(355);
+            setState(360);
             match(RBRACE);
             }
             }
@@ -2167,29 +2189,29 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(359); 
+          setState(364); 
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,25,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
-        setState(368);
+        setState(373);
         switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) {
         case 1:
           {
-          setState(361);
+          setState(366);
           dot();
-          setState(365);
+          setState(370);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,26,_ctx);
           while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
             if ( _alt==1 ) {
               {
               {
-              setState(362);
+              setState(367);
               secondary(true);
               }
               } 
             }
-            setState(367);
+            setState(372);
             _errHandler.sync(this);
             _alt = getInterpreter().adaptivePredict(_input,26,_ctx);
           }
@@ -2312,19 +2334,19 @@ class PainlessParser extends Parser {
     PrimaryContext _localctx = new PrimaryContext(_ctx, getState(), c);
     enterRule(_localctx, 34, RULE_primary);
     try {
-      setState(391);
+      setState(396);
       switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
       case 1:
         _localctx = new ExprprecContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(372);
+        setState(377);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(373);
+        setState(378);
         match(LP);
-        setState(374);
+        setState(379);
         ((ExprprecContext)_localctx).e = expression(0);
-        setState(375);
+        setState(380);
         match(RP);
          ((ExprprecContext)_localctx).s =  ((ExprprecContext)_localctx).e.s; 
         }
@@ -2333,13 +2355,13 @@ class PainlessParser extends Parser {
         _localctx = new ChainprecContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(378);
+        setState(383);
         if (!( _localctx.c )) throw new FailedPredicateException(this, " $c ");
-        setState(379);
+        setState(384);
         match(LP);
-        setState(380);
+        setState(385);
         unary(true);
-        setState(381);
+        setState(386);
         match(RP);
         }
         break;
@@ -2347,7 +2369,7 @@ class PainlessParser extends Parser {
         _localctx = new StringContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(383);
+        setState(388);
         match(STRING);
         }
         break;
@@ -2355,7 +2377,7 @@ class PainlessParser extends Parser {
         _localctx = new RegexContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(384);
+        setState(389);
         match(REGEX);
         }
         break;
@@ -2363,7 +2385,7 @@ class PainlessParser extends Parser {
         _localctx = new VariableContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(385);
+        setState(390);
         match(ID);
         }
         break;
@@ -2371,9 +2393,9 @@ class PainlessParser extends Parser {
         _localctx = new CalllocalContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(386);
+        setState(391);
         match(ID);
-        setState(387);
+        setState(392);
         arguments();
         }
         break;
@@ -2381,11 +2403,11 @@ class PainlessParser extends Parser {
         _localctx = new NewobjectContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(388);
+        setState(393);
         match(NEW);
-        setState(389);
+        setState(394);
         match(TYPE);
-        setState(390);
+        setState(395);
         arguments();
         }
         break;
@@ -2427,23 +2449,23 @@ class PainlessParser extends Parser {
     SecondaryContext _localctx = new SecondaryContext(_ctx, getState(), s);
     enterRule(_localctx, 36, RULE_secondary);
     try {
-      setState(397);
+      setState(402);
       switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(393);
+        setState(398);
         if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
-        setState(394);
+        setState(399);
         dot();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(395);
+        setState(400);
         if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
-        setState(396);
+        setState(401);
         brace();
         }
         break;
@@ -2501,17 +2523,17 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 38, RULE_dot);
     int _la;
     try {
-      setState(404);
+      setState(409);
       switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) {
       case 1:
         _localctx = new CallinvokeContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(399);
+        setState(404);
         match(DOT);
-        setState(400);
+        setState(405);
         match(DOTID);
-        setState(401);
+        setState(406);
         arguments();
         }
         break;
@@ -2519,9 +2541,9 @@ class PainlessParser extends Parser {
         _localctx = new FieldaccessContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(402);
+        setState(407);
         match(DOT);
-        setState(403);
+        setState(408);
         _la = _input.LA(1);
         if ( !(_la==DOTINTEGER || _la==DOTID) ) {
         _errHandler.recoverInline(this);
@@ -2575,11 +2597,11 @@ class PainlessParser extends Parser {
       _localctx = new BraceaccessContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(406);
+      setState(411);
       match(LBRACE);
-      setState(407);
+      setState(412);
       expression(0);
-      setState(408);
+      setState(413);
       match(RBRACE);
       }
     }
@@ -2626,34 +2648,34 @@ class PainlessParser extends Parser {
       enterOuterAlt(_localctx, 1);
       {
       {
-      setState(410);
+      setState(415);
       match(LP);
-      setState(419);
+      setState(424);
       switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) {
       case 1:
         {
-        setState(411);
-        argument();
         setState(416);
+        argument();
+        setState(421);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(412);
+          setState(417);
           match(COMMA);
-          setState(413);
+          setState(418);
           argument();
           }
           }
-          setState(418);
+          setState(423);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
         break;
       }
-      setState(421);
+      setState(426);
       match(RP);
       }
       }
@@ -2694,26 +2716,26 @@ class PainlessParser extends Parser {
     ArgumentContext _localctx = new ArgumentContext(_ctx, getState());
     enterRule(_localctx, 44, RULE_argument);
     try {
-      setState(426);
+      setState(431);
       switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(423);
+        setState(428);
         expression(0);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(424);
+        setState(429);
         lambda();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(425);
+        setState(430);
         funcref();
         }
         break;
@@ -2765,54 +2787,54 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(441);
+      setState(446);
       switch (_input.LA(1)) {
       case TYPE:
       case ID:
         {
-        setState(428);
+        setState(433);
         lamtype();
         }
         break;
       case LP:
         {
-        setState(429);
+        setState(434);
         match(LP);
-        setState(438);
+        setState(443);
         _la = _input.LA(1);
         if (_la==TYPE || _la==ID) {
           {
-          setState(430);
-          lamtype();
           setState(435);
+          lamtype();
+          setState(440);
           _errHandler.sync(this);
           _la = _input.LA(1);
           while (_la==COMMA) {
             {
             {
-            setState(431);
+            setState(436);
             match(COMMA);
-            setState(432);
+            setState(437);
             lamtype();
             }
             }
-            setState(437);
+            setState(442);
             _errHandler.sync(this);
             _la = _input.LA(1);
           }
           }
         }
 
-        setState(440);
+        setState(445);
         match(RP);
         }
         break;
       default:
         throw new NoViableAltException(this);
       }
-      setState(443);
+      setState(448);
       match(ARROW);
-      setState(444);
+      setState(449);
       block();
       }
     }
@@ -2850,16 +2872,16 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(447);
+      setState(452);
       _la = _input.LA(1);
       if (_la==TYPE) {
         {
-        setState(446);
+        setState(451);
         decltype();
         }
       }
 
-      setState(449);
+      setState(454);
       match(ID);
       }
     }
@@ -2902,33 +2924,33 @@ class PainlessParser extends Parser {
     FuncrefContext _localctx = new FuncrefContext(_ctx, getState());
     enterRule(_localctx, 50, RULE_funcref);
     try {
-      setState(455);
+      setState(460);
       switch ( getInterpreter().adaptivePredict(_input,39,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(451);
+        setState(456);
         classFuncref();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(452);
+        setState(457);
         constructorFuncref();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(453);
+        setState(458);
         capturingFuncref();
         }
         break;
       case 4:
         enterOuterAlt(_localctx, 4);
         {
-        setState(454);
+        setState(459);
         localFuncref();
         }
         break;
@@ -2966,11 +2988,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(457);
+      setState(462);
       match(TYPE);
-      setState(458);
+      setState(463);
       match(REF);
-      setState(459);
+      setState(464);
       match(ID);
       }
     }
@@ -3008,11 +3030,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(461);
+      setState(466);
       decltype();
-      setState(462);
+      setState(467);
       match(REF);
-      setState(463);
+      setState(468);
       match(NEW);
       }
     }
@@ -3050,11 +3072,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(465);
+      setState(470);
       match(ID);
-      setState(466);
+      setState(471);
       match(REF);
-      setState(467);
+      setState(472);
       match(ID);
       }
     }
@@ -3090,11 +3112,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(469);
+      setState(474);
       match(THIS);
-      setState(470);
+      setState(475);
       match(REF);
-      setState(471);
+      setState(476);
       match(ID);
       }
     }
@@ -3134,34 +3156,34 @@ class PainlessParser extends Parser {
   private boolean expression_sempred(ExpressionContext _localctx, int predIndex) {
     switch (predIndex) {
     case 1:
-      return precpred(_ctx, 12);
+      return precpred(_ctx, 13);
     case 2:
-      return precpred(_ctx, 11);
+      return precpred(_ctx, 12);
     case 3:
-      return precpred(_ctx, 10);
+      return precpred(_ctx, 11);
     case 4:
-      return precpred(_ctx, 9);
+      return precpred(_ctx, 10);
     case 5:
-      return precpred(_ctx, 8);
+      return precpred(_ctx, 9);
     case 6:
-      return precpred(_ctx, 7);
+      return precpred(_ctx, 8);
     case 7:
-      return precpred(_ctx, 6);
+      return precpred(_ctx, 7);
     case 8:
-      return precpred(_ctx, 5);
+      return precpred(_ctx, 6);
     case 9:
-      return precpred(_ctx, 4);
+      return precpred(_ctx, 5);
     case 10:
-      return precpred(_ctx, 3);
+      return precpred(_ctx, 4);
     case 11:
+      return precpred(_ctx, 3);
+    case 12:
       return precpred(_ctx, 2);
     }
     return true;
   }
   private boolean unary_sempred(UnaryContext _localctx, int predIndex) {
     switch (predIndex) {
-    case 12:
-      return  !_localctx.c ;
     case 13:
       return  !_localctx.c ;
     case 14:
@@ -3176,30 +3198,32 @@ class PainlessParser extends Parser {
       return  !_localctx.c ;
     case 19:
       return  !_localctx.c ;
+    case 20:
+      return  !_localctx.c ;
     }
     return true;
   }
   private boolean primary_sempred(PrimaryContext _localctx, int predIndex) {
     switch (predIndex) {
-    case 20:
-      return  !_localctx.c ;
     case 21:
+      return  !_localctx.c ;
+    case 22:
       return  _localctx.c ;
     }
     return true;
   }
   private boolean secondary_sempred(SecondaryContext _localctx, int predIndex) {
     switch (predIndex) {
-    case 22:
-      return  _localctx.s ;
     case 23:
       return  _localctx.s ;
+    case 24:
+      return  _localctx.s ;
     }
     return true;
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3P\u01dc\4\2\t\2\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3R\u01e1\4\2\t\2\4"+
     "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
     "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
     "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+
@@ -3220,163 +3244,165 @@ class PainlessParser extends Parser {
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
-    "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\7\20\u012f\n\20\f\20"+
-    "\16\20\u0132\13\20\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3"+
-    "\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3"+
-    "\21\3\21\3\21\3\21\3\21\5\21\u0151\n\21\3\22\3\22\7\22\u0155\n\22\f\22"+
-    "\16\22\u0158\13\22\3\22\3\22\3\22\7\22\u015d\n\22\f\22\16\22\u0160\13"+
-    "\22\3\22\3\22\3\22\3\22\3\22\3\22\6\22\u0168\n\22\r\22\16\22\u0169\3\22"+
-    "\3\22\7\22\u016e\n\22\f\22\16\22\u0171\13\22\5\22\u0173\n\22\5\22\u0175"+
-    "\n\22\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23"+
-    "\3\23\3\23\3\23\3\23\3\23\3\23\5\23\u018a\n\23\3\24\3\24\3\24\3\24\5\24"+
-    "\u0190\n\24\3\25\3\25\3\25\3\25\3\25\5\25\u0197\n\25\3\26\3\26\3\26\3"+
-    "\26\3\27\3\27\3\27\3\27\7\27\u01a1\n\27\f\27\16\27\u01a4\13\27\5\27\u01a6"+
-    "\n\27\3\27\3\27\3\30\3\30\3\30\5\30\u01ad\n\30\3\31\3\31\3\31\3\31\3\31"+
-    "\7\31\u01b4\n\31\f\31\16\31\u01b7\13\31\5\31\u01b9\n\31\3\31\5\31\u01bc"+
-    "\n\31\3\31\3\31\3\31\3\32\5\32\u01c2\n\32\3\32\3\32\3\33\3\33\3\33\3\33"+
-    "\5\33\u01ca\n\33\3\34\3\34\3\34\3\34\3\35\3\35\3\35\3\35\3\36\3\36\3\36"+
-    "\3\36\3\37\3\37\3\37\3\37\3\37\2\3\36 \2\4\6\b\n\f\16\20\22\24\26\30\32"+
-    "\34\36 \"$&(*,.\60\62\64\668:<\2\f\3\28C\3\2\35\37\3\2 !\3\2\"$\3\2%("+
-    "\3\2),\3\2\66\67\3\2DG\4\2\33\34 !\3\2OP\u0209\2A\3\2\2\2\4L\3\2\2\2\6"+
-    "Q\3\2\2\2\b\u00af\3\2\2\2\n\u00b3\3\2\2\2\f\u00b5\3\2\2\2\16\u00be\3\2"+
-    "\2\2\20\u00c2\3\2\2\2\22\u00c4\3\2\2\2\24\u00c6\3\2\2\2\26\u00cf\3\2\2"+
-    "\2\30\u00d7\3\2\2\2\32\u00dc\3\2\2\2\34\u00e8\3\2\2\2\36\u00f3\3\2\2\2"+
-    " \u0150\3\2\2\2\"\u0174\3\2\2\2$\u0189\3\2\2\2&\u018f\3\2\2\2(\u0196\3"+
-    "\2\2\2*\u0198\3\2\2\2,\u019c\3\2\2\2.\u01ac\3\2\2\2\60\u01bb\3\2\2\2\62"+
-    "\u01c1\3\2\2\2\64\u01c9\3\2\2\2\66\u01cb\3\2\2\28\u01cf\3\2\2\2:\u01d3"+
-    "\3\2\2\2<\u01d7\3\2\2\2>@\5\4\3\2?>\3\2\2\2@C\3\2\2\2A?\3\2\2\2AB\3\2"+
-    "\2\2BG\3\2\2\2CA\3\2\2\2DF\5\b\5\2ED\3\2\2\2FI\3\2\2\2GE\3\2\2\2GH\3\2"+
-    "\2\2HJ\3\2\2\2IG\3\2\2\2JK\7\2\2\3K\3\3\2\2\2LM\5\26\f\2MN\7N\2\2NO\5"+
-    "\6\4\2OP\5\f\7\2P\5\3\2\2\2Q]\7\t\2\2RS\5\26\f\2SZ\7N\2\2TU\7\f\2\2UV"+
-    "\5\26\f\2VW\7N\2\2WY\3\2\2\2XT\3\2\2\2Y\\\3\2\2\2ZX\3\2\2\2Z[\3\2\2\2"+
-    "[^\3\2\2\2\\Z\3\2\2\2]R\3\2\2\2]^\3\2\2\2^_\3\2\2\2_`\7\n\2\2`\7\3\2\2"+
-    "\2ab\7\16\2\2bc\7\t\2\2cd\5\36\20\2de\7\n\2\2ei\5\n\6\2fg\7\17\2\2gj\5"+
-    "\n\6\2hj\6\5\2\2if\3\2\2\2ih\3\2\2\2j\u00b0\3\2\2\2kl\7\20\2\2lm\7\t\2"+
-    "\2mn\5\36\20\2nq\7\n\2\2or\5\n\6\2pr\5\16\b\2qo\3\2\2\2qp\3\2\2\2r\u00b0"+
-    "\3\2\2\2st\7\21\2\2tu\5\f\7\2uv\7\20\2\2vw\7\t\2\2wx\5\36\20\2xy\7\n\2"+
-    "\2yz\5\34\17\2z\u00b0\3\2\2\2{|\7\22\2\2|~\7\t\2\2}\177\5\20\t\2~}\3\2"+
-    "\2\2~\177\3\2\2\2\177\u0080\3\2\2\2\u0080\u0082\7\r\2\2\u0081\u0083\5"+
-    "\36\20\2\u0082\u0081\3\2\2\2\u0082\u0083\3\2\2\2\u0083\u0084\3\2\2\2\u0084"+
-    "\u0086\7\r\2\2\u0085\u0087\5\22\n\2\u0086\u0085\3\2\2\2\u0086\u0087\3"+
-    "\2\2\2\u0087\u0088\3\2\2\2\u0088\u008b\7\n\2\2\u0089\u008c\5\n\6\2\u008a"+
-    "\u008c\5\16\b\2\u008b\u0089\3\2\2\2\u008b\u008a\3\2\2\2\u008c\u00b0\3"+
-    "\2\2\2\u008d\u008e\7\22\2\2\u008e\u008f\7\t\2\2\u008f\u0090\5\26\f\2\u0090"+
-    "\u0091\7N\2\2\u0091\u0092\7\63\2\2\u0092\u0093\5\36\20\2\u0093\u0094\7"+
-    "\n\2\2\u0094\u0095\5\n\6\2\u0095\u00b0\3\2\2\2\u0096\u0097\5\24\13\2\u0097"+
-    "\u0098\5\34\17\2\u0098\u00b0\3\2\2\2\u0099\u009a\7\23\2\2\u009a\u00b0"+
-    "\5\34\17\2\u009b\u009c\7\24\2\2\u009c\u00b0\5\34\17\2\u009d\u009e\7\25"+
-    "\2\2\u009e\u009f\5\36\20\2\u009f\u00a0\5\34\17\2\u00a0\u00b0\3\2\2\2\u00a1"+
-    "\u00a2\7\27\2\2\u00a2\u00a4\5\f\7\2\u00a3\u00a5\5\32\16\2\u00a4\u00a3"+
-    "\3\2\2\2\u00a5\u00a6\3\2\2\2\u00a6\u00a4\3\2\2\2\u00a6\u00a7\3\2\2\2\u00a7"+
-    "\u00b0\3\2\2\2\u00a8\u00a9\7\31\2\2\u00a9\u00aa\5\36\20\2\u00aa\u00ab"+
-    "\5\34\17\2\u00ab\u00b0\3\2\2\2\u00ac\u00ad\5\36\20\2\u00ad\u00ae\5\34"+
-    "\17\2\u00ae\u00b0\3\2\2\2\u00afa\3\2\2\2\u00afk\3\2\2\2\u00afs\3\2\2\2"+
-    "\u00af{\3\2\2\2\u00af\u008d\3\2\2\2\u00af\u0096\3\2\2\2\u00af\u0099\3"+
-    "\2\2\2\u00af\u009b\3\2\2\2\u00af\u009d\3\2\2\2\u00af\u00a1\3\2\2\2\u00af"+
-    "\u00a8\3\2\2\2\u00af\u00ac\3\2\2\2\u00b0\t\3\2\2\2\u00b1\u00b4\5\f\7\2"+
-    "\u00b2\u00b4\5\b\5\2\u00b3\u00b1\3\2\2\2\u00b3\u00b2\3\2\2\2\u00b4\13"+
-    "\3\2\2\2\u00b5\u00b9\7\5\2\2\u00b6\u00b8\5\b\5\2\u00b7\u00b6\3\2\2\2\u00b8"+
-    "\u00bb\3\2\2\2\u00b9\u00b7\3\2\2\2\u00b9\u00ba\3\2\2\2\u00ba\u00bc\3\2"+
-    "\2\2\u00bb\u00b9\3\2\2\2\u00bc\u00bd\7\6\2\2\u00bd\r\3\2\2\2\u00be\u00bf"+
-    "\7\r\2\2\u00bf\17\3\2\2\2\u00c0\u00c3\5\24\13\2\u00c1\u00c3\5\36\20\2"+
-    "\u00c2\u00c0\3\2\2\2\u00c2\u00c1\3\2\2\2\u00c3\21\3\2\2\2\u00c4\u00c5"+
-    "\5\36\20\2\u00c5\23\3\2\2\2\u00c6\u00c7\5\26\f\2\u00c7\u00cc\5\30\r\2"+
-    "\u00c8\u00c9\7\f\2\2\u00c9\u00cb\5\30\r\2\u00ca\u00c8\3\2\2\2\u00cb\u00ce"+
-    "\3\2\2\2\u00cc\u00ca\3\2\2\2\u00cc\u00cd\3\2\2\2\u00cd\25\3\2\2\2\u00ce"+
-    "\u00cc\3\2\2\2\u00cf\u00d4\7M\2\2\u00d0\u00d1\7\7\2\2\u00d1\u00d3\7\b"+
-    "\2\2\u00d2\u00d0\3\2\2\2\u00d3\u00d6\3\2\2\2\u00d4\u00d2\3\2\2\2\u00d4"+
-    "\u00d5\3\2\2\2\u00d5\27\3\2\2\2\u00d6\u00d4\3\2\2\2\u00d7\u00da\7N\2\2"+
-    "\u00d8\u00d9\78\2\2\u00d9\u00db\5\36\20\2\u00da\u00d8\3\2\2\2\u00da\u00db"+
-    "\3\2\2\2\u00db\31\3\2\2\2\u00dc\u00dd\7\30\2\2\u00dd\u00de\7\t\2\2\u00de"+
-    "\u00df\7M\2\2\u00df\u00e0\7N\2\2\u00e0\u00e1\7\n\2\2\u00e1\u00e2\5\f\7"+
-    "\2\u00e2\33\3\2\2\2\u00e3\u00e9\7\r\2\2\u00e4\u00e9\7\2\2\3\u00e5\u00e6"+
-    "\b\17\1\2\u00e6\u00e7\7\6\2\2\u00e7\u00e9\b\17\1\2\u00e8\u00e3\3\2\2\2"+
-    "\u00e8\u00e4\3\2\2\2\u00e8\u00e5\3\2\2\2\u00e9\35\3\2\2\2\u00ea\u00eb"+
-    "\b\20\1\2\u00eb\u00ec\5\"\22\2\u00ec\u00ed\t\2\2\2\u00ed\u00ee\5\36\20"+
-    "\3\u00ee\u00ef\b\20\1\2\u00ef\u00f4\3\2\2\2\u00f0\u00f1\5 \21\2\u00f1"+
-    "\u00f2\b\20\1\2\u00f2\u00f4\3\2\2\2\u00f3\u00ea\3\2\2\2\u00f3\u00f0\3"+
-    "\2\2\2\u00f4\u0130\3\2\2\2\u00f5\u00f6\f\16\2\2\u00f6\u00f7\t\3\2\2\u00f7"+
-    "\u00f8\5\36\20\17\u00f8\u00f9\b\20\1\2\u00f9\u012f\3\2\2\2\u00fa\u00fb"+
-    "\f\r\2\2\u00fb\u00fc\t\4\2\2\u00fc\u00fd\5\36\20\16\u00fd\u00fe\b\20\1"+
-    "\2\u00fe\u012f\3\2\2\2\u00ff\u0100\f\f\2\2\u0100\u0101\t\5\2\2\u0101\u0102"+
-    "\5\36\20\r\u0102\u0103\b\20\1\2\u0103\u012f\3\2\2\2\u0104\u0105\f\13\2"+
-    "\2\u0105\u0106\t\6\2\2\u0106\u0107\5\36\20\f\u0107\u0108\b\20\1\2\u0108"+
-    "\u012f\3\2\2\2\u0109\u010a\f\n\2\2\u010a\u010b\t\7\2\2\u010b\u010c\5\36"+
-    "\20\13\u010c\u010d\b\20\1\2\u010d\u012f\3\2\2\2\u010e\u010f\f\t\2\2\u010f"+
-    "\u0110\7-\2\2\u0110\u0111\5\36\20\n\u0111\u0112\b\20\1\2\u0112\u012f\3"+
-    "\2\2\2\u0113\u0114\f\b\2\2\u0114\u0115\7.\2\2\u0115\u0116\5\36\20\t\u0116"+
-    "\u0117\b\20\1\2\u0117\u012f\3\2\2\2\u0118\u0119\f\7\2\2\u0119\u011a\7"+
-    "/\2\2\u011a\u011b\5\36\20\b\u011b\u011c\b\20\1\2\u011c\u012f\3\2\2\2\u011d"+
-    "\u011e\f\6\2\2\u011e\u011f\7\60\2\2\u011f\u0120\5\36\20\7\u0120\u0121"+
-    "\b\20\1\2\u0121\u012f\3\2\2\2\u0122\u0123\f\5\2\2\u0123\u0124\7\61\2\2"+
-    "\u0124\u0125\5\36\20\6\u0125\u0126\b\20\1\2\u0126\u012f\3\2\2\2\u0127"+
-    "\u0128\f\4\2\2\u0128\u0129\7\62\2\2\u0129\u012a\5\36\20\2\u012a\u012b"+
-    "\7\63\2\2\u012b\u012c\5\36\20\4\u012c\u012d\b\20\1\2\u012d\u012f\3\2\2"+
-    "\2\u012e\u00f5\3\2\2\2\u012e\u00fa\3\2\2\2\u012e\u00ff\3\2\2\2\u012e\u0104"+
-    "\3\2\2\2\u012e\u0109\3\2\2\2\u012e\u010e\3\2\2\2\u012e\u0113\3\2\2\2\u012e"+
-    "\u0118\3\2\2\2\u012e\u011d\3\2\2\2\u012e\u0122\3\2\2\2\u012e\u0127\3\2"+
-    "\2\2\u012f\u0132\3\2\2\2\u0130\u012e\3\2\2\2\u0130\u0131\3\2\2\2\u0131"+
-    "\37\3\2\2\2\u0132\u0130\3\2\2\2\u0133\u0134\6\21\16\3\u0134\u0135\t\b"+
-    "\2\2\u0135\u0151\5\"\22\2\u0136\u0137\6\21\17\3\u0137\u0138\5\"\22\2\u0138"+
-    "\u0139\t\b\2\2\u0139\u0151\3\2\2\2\u013a\u013b\6\21\20\3\u013b\u0151\5"+
-    "\"\22\2\u013c\u013d\6\21\21\3\u013d\u013e\t\t\2\2\u013e\u0151\b\21\1\2"+
-    "\u013f\u0140\6\21\22\3\u0140\u0141\7J\2\2\u0141\u0151\b\21\1\2\u0142\u0143"+
-    "\6\21\23\3\u0143\u0144\7K\2\2\u0144\u0151\b\21\1\2\u0145\u0146\6\21\24"+
-    "\3\u0146\u0147\7L\2\2\u0147\u0151\b\21\1\2\u0148\u0149\6\21\25\3\u0149"+
-    "\u014a\t\n\2\2\u014a\u0151\5 \21\2\u014b\u014c\7\t\2\2\u014c\u014d\5\26"+
-    "\f\2\u014d\u014e\7\n\2\2\u014e\u014f\5 \21\2\u014f\u0151\3\2\2\2\u0150"+
-    "\u0133\3\2\2\2\u0150\u0136\3\2\2\2\u0150\u013a\3\2\2\2\u0150\u013c\3\2"+
-    "\2\2\u0150\u013f\3\2\2\2\u0150\u0142\3\2\2\2\u0150\u0145\3\2\2\2\u0150"+
-    "\u0148\3\2\2\2\u0150\u014b\3\2\2\2\u0151!\3\2\2\2\u0152\u0156\5$\23\2"+
-    "\u0153\u0155\5&\24\2\u0154\u0153\3\2\2\2\u0155\u0158\3\2\2\2\u0156\u0154"+
-    "\3\2\2\2\u0156\u0157\3\2\2\2\u0157\u0175\3\2\2\2\u0158\u0156\3\2\2\2\u0159"+
-    "\u015a\5\26\f\2\u015a\u015e\5(\25\2\u015b\u015d\5&\24\2\u015c\u015b\3"+
-    "\2\2\2\u015d\u0160\3\2\2\2\u015e\u015c\3\2\2\2\u015e\u015f\3\2\2\2\u015f"+
-    "\u0175\3\2\2\2\u0160\u015e\3\2\2\2\u0161\u0162\7\26\2\2\u0162\u0167\7"+
-    "M\2\2\u0163\u0164\7\7\2\2\u0164\u0165\5\36\20\2\u0165\u0166\7\b\2\2\u0166"+
-    "\u0168\3\2\2\2\u0167\u0163\3\2\2\2\u0168\u0169\3\2\2\2\u0169\u0167\3\2"+
-    "\2\2\u0169\u016a\3\2\2\2\u016a\u0172\3\2\2\2\u016b\u016f\5(\25\2\u016c"+
-    "\u016e\5&\24\2\u016d\u016c\3\2\2\2\u016e\u0171\3\2\2\2\u016f\u016d\3\2"+
-    "\2\2\u016f\u0170\3\2\2\2\u0170\u0173\3\2\2\2\u0171\u016f\3\2\2\2\u0172"+
-    "\u016b\3\2\2\2\u0172\u0173\3\2\2\2\u0173\u0175\3\2\2\2\u0174\u0152\3\2"+
-    "\2\2\u0174\u0159\3\2\2\2\u0174\u0161\3\2\2\2\u0175#\3\2\2\2\u0176\u0177"+
-    "\6\23\26\3\u0177\u0178\7\t\2\2\u0178\u0179\5\36\20\2\u0179\u017a\7\n\2"+
-    "\2\u017a\u017b\b\23\1\2\u017b\u018a\3\2\2\2\u017c\u017d\6\23\27\3\u017d"+
-    "\u017e\7\t\2\2\u017e\u017f\5 \21\2\u017f\u0180\7\n\2\2\u0180\u018a\3\2"+
-    "\2\2\u0181\u018a\7H\2\2\u0182\u018a\7I\2\2\u0183\u018a\7N\2\2\u0184\u0185"+
-    "\7N\2\2\u0185\u018a\5,\27\2\u0186\u0187\7\26\2\2\u0187\u0188\7M\2\2\u0188"+
-    "\u018a\5,\27\2\u0189\u0176\3\2\2\2\u0189\u017c\3\2\2\2\u0189\u0181\3\2"+
-    "\2\2\u0189\u0182\3\2\2\2\u0189\u0183\3\2\2\2\u0189\u0184\3\2\2\2\u0189"+
-    "\u0186\3\2\2\2\u018a%\3\2\2\2\u018b\u018c\6\24\30\3\u018c\u0190\5(\25"+
-    "\2\u018d\u018e\6\24\31\3\u018e\u0190\5*\26\2\u018f\u018b\3\2\2\2\u018f"+
-    "\u018d\3\2\2\2\u0190\'\3\2\2\2\u0191\u0192\7\13\2\2\u0192\u0193\7P\2\2"+
-    "\u0193\u0197\5,\27\2\u0194\u0195\7\13\2\2\u0195\u0197\t\13\2\2\u0196\u0191"+
-    "\3\2\2\2\u0196\u0194\3\2\2\2\u0197)\3\2\2\2\u0198\u0199\7\7\2\2\u0199"+
-    "\u019a\5\36\20\2\u019a\u019b\7\b\2\2\u019b+\3\2\2\2\u019c\u01a5\7\t\2"+
-    "\2\u019d\u01a2\5.\30\2\u019e\u019f\7\f\2\2\u019f\u01a1\5.\30\2\u01a0\u019e"+
-    "\3\2\2\2\u01a1\u01a4\3\2\2\2\u01a2\u01a0\3\2\2\2\u01a2\u01a3\3\2\2\2\u01a3"+
-    "\u01a6\3\2\2\2\u01a4\u01a2\3\2\2\2\u01a5\u019d\3\2\2\2\u01a5\u01a6\3\2"+
-    "\2\2\u01a6\u01a7\3\2\2\2\u01a7\u01a8\7\n\2\2\u01a8-\3\2\2\2\u01a9\u01ad"+
-    "\5\36\20\2\u01aa\u01ad\5\60\31\2\u01ab\u01ad\5\64\33\2\u01ac\u01a9\3\2"+
-    "\2\2\u01ac\u01aa\3\2\2\2\u01ac\u01ab\3\2\2\2\u01ad/\3\2\2\2\u01ae\u01bc"+
-    "\5\62\32\2\u01af\u01b8\7\t\2\2\u01b0\u01b5\5\62\32\2\u01b1\u01b2\7\f\2"+
-    "\2\u01b2\u01b4\5\62\32\2\u01b3\u01b1\3\2\2\2\u01b4\u01b7\3\2\2\2\u01b5"+
-    "\u01b3\3\2\2\2\u01b5\u01b6\3\2\2\2\u01b6\u01b9\3\2\2\2\u01b7\u01b5\3\2"+
-    "\2\2\u01b8\u01b0\3\2\2\2\u01b8\u01b9\3\2\2\2\u01b9\u01ba\3\2\2\2\u01ba"+
-    "\u01bc\7\n\2\2\u01bb\u01ae\3\2\2\2\u01bb\u01af\3\2\2\2\u01bc\u01bd\3\2"+
-    "\2\2\u01bd\u01be\7\65\2\2\u01be\u01bf\5\f\7\2\u01bf\61\3\2\2\2\u01c0\u01c2"+
-    "\5\26\f\2\u01c1\u01c0\3\2\2\2\u01c1\u01c2\3\2\2\2\u01c2\u01c3\3\2\2\2"+
-    "\u01c3\u01c4\7N\2\2\u01c4\63\3\2\2\2\u01c5\u01ca\5\66\34\2\u01c6\u01ca"+
-    "\58\35\2\u01c7\u01ca\5:\36\2\u01c8\u01ca\5<\37\2\u01c9\u01c5\3\2\2\2\u01c9"+
-    "\u01c6\3\2\2\2\u01c9\u01c7\3\2\2\2\u01c9\u01c8\3\2\2\2\u01ca\65\3\2\2"+
-    "\2\u01cb\u01cc\7M\2\2\u01cc\u01cd\7\64\2\2\u01cd\u01ce\7N\2\2\u01ce\67"+
-    "\3\2\2\2\u01cf\u01d0\5\26\f\2\u01d0\u01d1\7\64\2\2\u01d1\u01d2\7\26\2"+
-    "\2\u01d29\3\2\2\2\u01d3\u01d4\7N\2\2\u01d4\u01d5\7\64\2\2\u01d5\u01d6"+
-    "\7N\2\2\u01d6;\3\2\2\2\u01d7\u01d8\7\32\2\2\u01d8\u01d9\7\64\2\2\u01d9"+
-    "\u01da\7N\2\2\u01da=\3\2\2\2*AGZ]iq~\u0082\u0086\u008b\u00a6\u00af\u00b3"+
-    "\u00b9\u00c2\u00cc\u00d4\u00da\u00e8\u00f3\u012e\u0130\u0150\u0156\u015e"+
-    "\u0169\u016f\u0172\u0174\u0189\u018f\u0196\u01a2\u01a5\u01ac\u01b5\u01b8"+
-    "\u01bb\u01c1\u01c9";
+    "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
+    "\3\20\7\20\u0134\n\20\f\20\16\20\u0137\13\20\3\21\3\21\3\21\3\21\3\21"+
+    "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
+    "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\5\21\u0156\n\21\3\22"+
+    "\3\22\7\22\u015a\n\22\f\22\16\22\u015d\13\22\3\22\3\22\3\22\7\22\u0162"+
+    "\n\22\f\22\16\22\u0165\13\22\3\22\3\22\3\22\3\22\3\22\3\22\6\22\u016d"+
+    "\n\22\r\22\16\22\u016e\3\22\3\22\7\22\u0173\n\22\f\22\16\22\u0176\13\22"+
+    "\5\22\u0178\n\22\5\22\u017a\n\22\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3"+
+    "\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\5\23\u018f"+
+    "\n\23\3\24\3\24\3\24\3\24\5\24\u0195\n\24\3\25\3\25\3\25\3\25\3\25\5\25"+
+    "\u019c\n\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\27\7\27\u01a6\n\27\f"+
+    "\27\16\27\u01a9\13\27\5\27\u01ab\n\27\3\27\3\27\3\30\3\30\3\30\5\30\u01b2"+
+    "\n\30\3\31\3\31\3\31\3\31\3\31\7\31\u01b9\n\31\f\31\16\31\u01bc\13\31"+
+    "\5\31\u01be\n\31\3\31\5\31\u01c1\n\31\3\31\3\31\3\31\3\32\5\32\u01c7\n"+
+    "\32\3\32\3\32\3\33\3\33\3\33\3\33\5\33\u01cf\n\33\3\34\3\34\3\34\3\34"+
+    "\3\35\3\35\3\35\3\35\3\36\3\36\3\36\3\36\3\37\3\37\3\37\3\37\3\37\2\3"+
+    "\36 \2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,.\60\62\64\668:<\2"+
+    "\r\3\2:E\3\2\35\37\3\2 !\3\2\66\67\3\2\"$\3\2%(\3\2),\3\289\3\2FI\4\2"+
+    "\33\34 !\3\2QR\u020f\2A\3\2\2\2\4L\3\2\2\2\6Q\3\2\2\2\b\u00af\3\2\2\2"+
+    "\n\u00b3\3\2\2\2\f\u00b5\3\2\2\2\16\u00be\3\2\2\2\20\u00c2\3\2\2\2\22"+
+    "\u00c4\3\2\2\2\24\u00c6\3\2\2\2\26\u00cf\3\2\2\2\30\u00d7\3\2\2\2\32\u00dc"+
+    "\3\2\2\2\34\u00e8\3\2\2\2\36\u00f3\3\2\2\2 \u0155\3\2\2\2\"\u0179\3\2"+
+    "\2\2$\u018e\3\2\2\2&\u0194\3\2\2\2(\u019b\3\2\2\2*\u019d\3\2\2\2,\u01a1"+
+    "\3\2\2\2.\u01b1\3\2\2\2\60\u01c0\3\2\2\2\62\u01c6\3\2\2\2\64\u01ce\3\2"+
+    "\2\2\66\u01d0\3\2\2\28\u01d4\3\2\2\2:\u01d8\3\2\2\2<\u01dc\3\2\2\2>@\5"+
+    "\4\3\2?>\3\2\2\2@C\3\2\2\2A?\3\2\2\2AB\3\2\2\2BG\3\2\2\2CA\3\2\2\2DF\5"+
+    "\b\5\2ED\3\2\2\2FI\3\2\2\2GE\3\2\2\2GH\3\2\2\2HJ\3\2\2\2IG\3\2\2\2JK\7"+
+    "\2\2\3K\3\3\2\2\2LM\5\26\f\2MN\7P\2\2NO\5\6\4\2OP\5\f\7\2P\5\3\2\2\2Q"+
+    "]\7\t\2\2RS\5\26\f\2SZ\7P\2\2TU\7\f\2\2UV\5\26\f\2VW\7P\2\2WY\3\2\2\2"+
+    "XT\3\2\2\2Y\\\3\2\2\2ZX\3\2\2\2Z[\3\2\2\2[^\3\2\2\2\\Z\3\2\2\2]R\3\2\2"+
+    "\2]^\3\2\2\2^_\3\2\2\2_`\7\n\2\2`\7\3\2\2\2ab\7\16\2\2bc\7\t\2\2cd\5\36"+
+    "\20\2de\7\n\2\2ei\5\n\6\2fg\7\17\2\2gj\5\n\6\2hj\6\5\2\2if\3\2\2\2ih\3"+
+    "\2\2\2j\u00b0\3\2\2\2kl\7\20\2\2lm\7\t\2\2mn\5\36\20\2nq\7\n\2\2or\5\n"+
+    "\6\2pr\5\16\b\2qo\3\2\2\2qp\3\2\2\2r\u00b0\3\2\2\2st\7\21\2\2tu\5\f\7"+
+    "\2uv\7\20\2\2vw\7\t\2\2wx\5\36\20\2xy\7\n\2\2yz\5\34\17\2z\u00b0\3\2\2"+
+    "\2{|\7\22\2\2|~\7\t\2\2}\177\5\20\t\2~}\3\2\2\2~\177\3\2\2\2\177\u0080"+
+    "\3\2\2\2\u0080\u0082\7\r\2\2\u0081\u0083\5\36\20\2\u0082\u0081\3\2\2\2"+
+    "\u0082\u0083\3\2\2\2\u0083\u0084\3\2\2\2\u0084\u0086\7\r\2\2\u0085\u0087"+
+    "\5\22\n\2\u0086\u0085\3\2\2\2\u0086\u0087\3\2\2\2\u0087\u0088\3\2\2\2"+
+    "\u0088\u008b\7\n\2\2\u0089\u008c\5\n\6\2\u008a\u008c\5\16\b\2\u008b\u0089"+
+    "\3\2\2\2\u008b\u008a\3\2\2\2\u008c\u00b0\3\2\2\2\u008d\u008e\7\22\2\2"+
+    "\u008e\u008f\7\t\2\2\u008f\u0090\5\26\f\2\u0090\u0091\7P\2\2\u0091\u0092"+
+    "\7\63\2\2\u0092\u0093\5\36\20\2\u0093\u0094\7\n\2\2\u0094\u0095\5\n\6"+
+    "\2\u0095\u00b0\3\2\2\2\u0096\u0097\5\24\13\2\u0097\u0098\5\34\17\2\u0098"+
+    "\u00b0\3\2\2\2\u0099\u009a\7\23\2\2\u009a\u00b0\5\34\17\2\u009b\u009c"+
+    "\7\24\2\2\u009c\u00b0\5\34\17\2\u009d\u009e\7\25\2\2\u009e\u009f\5\36"+
+    "\20\2\u009f\u00a0\5\34\17\2\u00a0\u00b0\3\2\2\2\u00a1\u00a2\7\27\2\2\u00a2"+
+    "\u00a4\5\f\7\2\u00a3\u00a5\5\32\16\2\u00a4\u00a3\3\2\2\2\u00a5\u00a6\3"+
+    "\2\2\2\u00a6\u00a4\3\2\2\2\u00a6\u00a7\3\2\2\2\u00a7\u00b0\3\2\2\2\u00a8"+
+    "\u00a9\7\31\2\2\u00a9\u00aa\5\36\20\2\u00aa\u00ab\5\34\17\2\u00ab\u00b0"+
+    "\3\2\2\2\u00ac\u00ad\5\36\20\2\u00ad\u00ae\5\34\17\2\u00ae\u00b0\3\2\2"+
+    "\2\u00afa\3\2\2\2\u00afk\3\2\2\2\u00afs\3\2\2\2\u00af{\3\2\2\2\u00af\u008d"+
+    "\3\2\2\2\u00af\u0096\3\2\2\2\u00af\u0099\3\2\2\2\u00af\u009b\3\2\2\2\u00af"+
+    "\u009d\3\2\2\2\u00af\u00a1\3\2\2\2\u00af\u00a8\3\2\2\2\u00af\u00ac\3\2"+
+    "\2\2\u00b0\t\3\2\2\2\u00b1\u00b4\5\f\7\2\u00b2\u00b4\5\b\5\2\u00b3\u00b1"+
+    "\3\2\2\2\u00b3\u00b2\3\2\2\2\u00b4\13\3\2\2\2\u00b5\u00b9\7\5\2\2\u00b6"+
+    "\u00b8\5\b\5\2\u00b7\u00b6\3\2\2\2\u00b8\u00bb\3\2\2\2\u00b9\u00b7\3\2"+
+    "\2\2\u00b9\u00ba\3\2\2\2\u00ba\u00bc\3\2\2\2\u00bb\u00b9\3\2\2\2\u00bc"+
+    "\u00bd\7\6\2\2\u00bd\r\3\2\2\2\u00be\u00bf\7\r\2\2\u00bf\17\3\2\2\2\u00c0"+
+    "\u00c3\5\24\13\2\u00c1\u00c3\5\36\20\2\u00c2\u00c0\3\2\2\2\u00c2\u00c1"+
+    "\3\2\2\2\u00c3\21\3\2\2\2\u00c4\u00c5\5\36\20\2\u00c5\23\3\2\2\2\u00c6"+
+    "\u00c7\5\26\f\2\u00c7\u00cc\5\30\r\2\u00c8\u00c9\7\f\2\2\u00c9\u00cb\5"+
+    "\30\r\2\u00ca\u00c8\3\2\2\2\u00cb\u00ce\3\2\2\2\u00cc\u00ca\3\2\2\2\u00cc"+
+    "\u00cd\3\2\2\2\u00cd\25\3\2\2\2\u00ce\u00cc\3\2\2\2\u00cf\u00d4\7O\2\2"+
+    "\u00d0\u00d1\7\7\2\2\u00d1\u00d3\7\b\2\2\u00d2\u00d0\3\2\2\2\u00d3\u00d6"+
+    "\3\2\2\2\u00d4\u00d2\3\2\2\2\u00d4\u00d5\3\2\2\2\u00d5\27\3\2\2\2\u00d6"+
+    "\u00d4\3\2\2\2\u00d7\u00da\7P\2\2\u00d8\u00d9\7:\2\2\u00d9\u00db\5\36"+
+    "\20\2\u00da\u00d8\3\2\2\2\u00da\u00db\3\2\2\2\u00db\31\3\2\2\2\u00dc\u00dd"+
+    "\7\30\2\2\u00dd\u00de\7\t\2\2\u00de\u00df\7O\2\2\u00df\u00e0\7P\2\2\u00e0"+
+    "\u00e1\7\n\2\2\u00e1\u00e2\5\f\7\2\u00e2\33\3\2\2\2\u00e3\u00e9\7\r\2"+
+    "\2\u00e4\u00e9\7\2\2\3\u00e5\u00e6\b\17\1\2\u00e6\u00e7\7\6\2\2\u00e7"+
+    "\u00e9\b\17\1\2\u00e8\u00e3\3\2\2\2\u00e8\u00e4\3\2\2\2\u00e8\u00e5\3"+
+    "\2\2\2\u00e9\35\3\2\2\2\u00ea\u00eb\b\20\1\2\u00eb\u00ec\5\"\22\2\u00ec"+
+    "\u00ed\t\2\2\2\u00ed\u00ee\5\36\20\3\u00ee\u00ef\b\20\1\2\u00ef\u00f4"+
+    "\3\2\2\2\u00f0\u00f1\5 \21\2\u00f1\u00f2\b\20\1\2\u00f2\u00f4\3\2\2\2"+
+    "\u00f3\u00ea\3\2\2\2\u00f3\u00f0\3\2\2\2\u00f4\u0135\3\2\2\2\u00f5\u00f6"+
+    "\f\17\2\2\u00f6\u00f7\t\3\2\2\u00f7\u00f8\5\36\20\20\u00f8\u00f9\b\20"+
+    "\1\2\u00f9\u0134\3\2\2\2\u00fa\u00fb\f\16\2\2\u00fb\u00fc\t\4\2\2\u00fc"+
+    "\u00fd\5\36\20\17\u00fd\u00fe\b\20\1\2\u00fe\u0134\3\2\2\2\u00ff\u0100"+
+    "\f\r\2\2\u0100\u0101\t\5\2\2\u0101\u0102\5\36\20\16\u0102\u0103\b\20\1"+
+    "\2\u0103\u0134\3\2\2\2\u0104\u0105\f\f\2\2\u0105\u0106\t\6\2\2\u0106\u0107"+
+    "\5\36\20\r\u0107\u0108\b\20\1\2\u0108\u0134\3\2\2\2\u0109\u010a\f\13\2"+
+    "\2\u010a\u010b\t\7\2\2\u010b\u010c\5\36\20\f\u010c\u010d\b\20\1\2\u010d"+
+    "\u0134\3\2\2\2\u010e\u010f\f\n\2\2\u010f\u0110\t\b\2\2\u0110\u0111\5\36"+
+    "\20\13\u0111\u0112\b\20\1\2\u0112\u0134\3\2\2\2\u0113\u0114\f\t\2\2\u0114"+
+    "\u0115\7-\2\2\u0115\u0116\5\36\20\n\u0116\u0117\b\20\1\2\u0117\u0134\3"+
+    "\2\2\2\u0118\u0119\f\b\2\2\u0119\u011a\7.\2\2\u011a\u011b\5\36\20\t\u011b"+
+    "\u011c\b\20\1\2\u011c\u0134\3\2\2\2\u011d\u011e\f\7\2\2\u011e\u011f\7"+
+    "/\2\2\u011f\u0120\5\36\20\b\u0120\u0121\b\20\1\2\u0121\u0134\3\2\2\2\u0122"+
+    "\u0123\f\6\2\2\u0123\u0124\7\60\2\2\u0124\u0125\5\36\20\7\u0125\u0126"+
+    "\b\20\1\2\u0126\u0134\3\2\2\2\u0127\u0128\f\5\2\2\u0128\u0129\7\61\2\2"+
+    "\u0129\u012a\5\36\20\6\u012a\u012b\b\20\1\2\u012b\u0134\3\2\2\2\u012c"+
+    "\u012d\f\4\2\2\u012d\u012e\7\62\2\2\u012e\u012f\5\36\20\2\u012f\u0130"+
+    "\7\63\2\2\u0130\u0131\5\36\20\4\u0131\u0132\b\20\1\2\u0132\u0134\3\2\2"+
+    "\2\u0133\u00f5\3\2\2\2\u0133\u00fa\3\2\2\2\u0133\u00ff\3\2\2\2\u0133\u0104"+
+    "\3\2\2\2\u0133\u0109\3\2\2\2\u0133\u010e\3\2\2\2\u0133\u0113\3\2\2\2\u0133"+
+    "\u0118\3\2\2\2\u0133\u011d\3\2\2\2\u0133\u0122\3\2\2\2\u0133\u0127\3\2"+
+    "\2\2\u0133\u012c\3\2\2\2\u0134\u0137\3\2\2\2\u0135\u0133\3\2\2\2\u0135"+
+    "\u0136\3\2\2\2\u0136\37\3\2\2\2\u0137\u0135\3\2\2\2\u0138\u0139\6\21\17"+
+    "\3\u0139\u013a\t\t\2\2\u013a\u0156\5\"\22\2\u013b\u013c\6\21\20\3\u013c"+
+    "\u013d\5\"\22\2\u013d\u013e\t\t\2\2\u013e\u0156\3\2\2\2\u013f\u0140\6"+
+    "\21\21\3\u0140\u0156\5\"\22\2\u0141\u0142\6\21\22\3\u0142\u0143\t\n\2"+
+    "\2\u0143\u0156\b\21\1\2\u0144\u0145\6\21\23\3\u0145\u0146\7L\2\2\u0146"+
+    "\u0156\b\21\1\2\u0147\u0148\6\21\24\3\u0148\u0149\7M\2\2\u0149\u0156\b"+
+    "\21\1\2\u014a\u014b\6\21\25\3\u014b\u014c\7N\2\2\u014c\u0156\b\21\1\2"+
+    "\u014d\u014e\6\21\26\3\u014e\u014f\t\13\2\2\u014f\u0156\5 \21\2\u0150"+
+    "\u0151\7\t\2\2\u0151\u0152\5\26\f\2\u0152\u0153\7\n\2\2\u0153\u0154\5"+
+    " \21\2\u0154\u0156\3\2\2\2\u0155\u0138\3\2\2\2\u0155\u013b\3\2\2\2\u0155"+
+    "\u013f\3\2\2\2\u0155\u0141\3\2\2\2\u0155\u0144\3\2\2\2\u0155\u0147\3\2"+
+    "\2\2\u0155\u014a\3\2\2\2\u0155\u014d\3\2\2\2\u0155\u0150\3\2\2\2\u0156"+
+    "!\3\2\2\2\u0157\u015b\5$\23\2\u0158\u015a\5&\24\2\u0159\u0158\3\2\2\2"+
+    "\u015a\u015d\3\2\2\2\u015b\u0159\3\2\2\2\u015b\u015c\3\2\2\2\u015c\u017a"+
+    "\3\2\2\2\u015d\u015b\3\2\2\2\u015e\u015f\5\26\f\2\u015f\u0163\5(\25\2"+
+    "\u0160\u0162\5&\24\2\u0161\u0160\3\2\2\2\u0162\u0165\3\2\2\2\u0163\u0161"+
+    "\3\2\2\2\u0163\u0164\3\2\2\2\u0164\u017a\3\2\2\2\u0165\u0163\3\2\2\2\u0166"+
+    "\u0167\7\26\2\2\u0167\u016c\7O\2\2\u0168\u0169\7\7\2\2\u0169\u016a\5\36"+
+    "\20\2\u016a\u016b\7\b\2\2\u016b\u016d\3\2\2\2\u016c\u0168\3\2\2\2\u016d"+
+    "\u016e\3\2\2\2\u016e\u016c\3\2\2\2\u016e\u016f\3\2\2\2\u016f\u0177\3\2"+
+    "\2\2\u0170\u0174\5(\25\2\u0171\u0173\5&\24\2\u0172\u0171\3\2\2\2\u0173"+
+    "\u0176\3\2\2\2\u0174\u0172\3\2\2\2\u0174\u0175\3\2\2\2\u0175\u0178\3\2"+
+    "\2\2\u0176\u0174\3\2\2\2\u0177\u0170\3\2\2\2\u0177\u0178\3\2\2\2\u0178"+
+    "\u017a\3\2\2\2\u0179\u0157\3\2\2\2\u0179\u015e\3\2\2\2\u0179\u0166\3\2"+
+    "\2\2\u017a#\3\2\2\2\u017b\u017c\6\23\27\3\u017c\u017d\7\t\2\2\u017d\u017e"+
+    "\5\36\20\2\u017e\u017f\7\n\2\2\u017f\u0180\b\23\1\2\u0180\u018f\3\2\2"+
+    "\2\u0181\u0182\6\23\30\3\u0182\u0183\7\t\2\2\u0183\u0184\5 \21\2\u0184"+
+    "\u0185\7\n\2\2\u0185\u018f\3\2\2\2\u0186\u018f\7J\2\2\u0187\u018f\7K\2"+
+    "\2\u0188\u018f\7P\2\2\u0189\u018a\7P\2\2\u018a\u018f\5,\27\2\u018b\u018c"+
+    "\7\26\2\2\u018c\u018d\7O\2\2\u018d\u018f\5,\27\2\u018e\u017b\3\2\2\2\u018e"+
+    "\u0181\3\2\2\2\u018e\u0186\3\2\2\2\u018e\u0187\3\2\2\2\u018e\u0188\3\2"+
+    "\2\2\u018e\u0189\3\2\2\2\u018e\u018b\3\2\2\2\u018f%\3\2\2\2\u0190\u0191"+
+    "\6\24\31\3\u0191\u0195\5(\25\2\u0192\u0193\6\24\32\3\u0193\u0195\5*\26"+
+    "\2\u0194\u0190\3\2\2\2\u0194\u0192\3\2\2\2\u0195\'\3\2\2\2\u0196\u0197"+
+    "\7\13\2\2\u0197\u0198\7R\2\2\u0198\u019c\5,\27\2\u0199\u019a\7\13\2\2"+
+    "\u019a\u019c\t\f\2\2\u019b\u0196\3\2\2\2\u019b\u0199\3\2\2\2\u019c)\3"+
+    "\2\2\2\u019d\u019e\7\7\2\2\u019e\u019f\5\36\20\2\u019f\u01a0\7\b\2\2\u01a0"+
+    "+\3\2\2\2\u01a1\u01aa\7\t\2\2\u01a2\u01a7\5.\30\2\u01a3\u01a4\7\f\2\2"+
+    "\u01a4\u01a6\5.\30\2\u01a5\u01a3\3\2\2\2\u01a6\u01a9\3\2\2\2\u01a7\u01a5"+
+    "\3\2\2\2\u01a7\u01a8\3\2\2\2\u01a8\u01ab\3\2\2\2\u01a9\u01a7\3\2\2\2\u01aa"+
+    "\u01a2\3\2\2\2\u01aa\u01ab\3\2\2\2\u01ab\u01ac\3\2\2\2\u01ac\u01ad\7\n"+
+    "\2\2\u01ad-\3\2\2\2\u01ae\u01b2\5\36\20\2\u01af\u01b2\5\60\31\2\u01b0"+
+    "\u01b2\5\64\33\2\u01b1\u01ae\3\2\2\2\u01b1\u01af\3\2\2\2\u01b1\u01b0\3"+
+    "\2\2\2\u01b2/\3\2\2\2\u01b3\u01c1\5\62\32\2\u01b4\u01bd\7\t\2\2\u01b5"+
+    "\u01ba\5\62\32\2\u01b6\u01b7\7\f\2\2\u01b7\u01b9\5\62\32\2\u01b8\u01b6"+
+    "\3\2\2\2\u01b9\u01bc\3\2\2\2\u01ba\u01b8\3\2\2\2\u01ba\u01bb\3\2\2\2\u01bb"+
+    "\u01be\3\2\2\2\u01bc\u01ba\3\2\2\2\u01bd\u01b5\3\2\2\2\u01bd\u01be\3\2"+
+    "\2\2\u01be\u01bf\3\2\2\2\u01bf\u01c1\7\n\2\2\u01c0\u01b3\3\2\2\2\u01c0"+
+    "\u01b4\3\2\2\2\u01c1\u01c2\3\2\2\2\u01c2\u01c3\7\65\2\2\u01c3\u01c4\5"+
+    "\f\7\2\u01c4\61\3\2\2\2\u01c5\u01c7\5\26\f\2\u01c6\u01c5\3\2\2\2\u01c6"+
+    "\u01c7\3\2\2\2\u01c7\u01c8\3\2\2\2\u01c8\u01c9\7P\2\2\u01c9\63\3\2\2\2"+
+    "\u01ca\u01cf\5\66\34\2\u01cb\u01cf\58\35\2\u01cc\u01cf\5:\36\2\u01cd\u01cf"+
+    "\5<\37\2\u01ce\u01ca\3\2\2\2\u01ce\u01cb\3\2\2\2\u01ce\u01cc\3\2\2\2\u01ce"+
+    "\u01cd\3\2\2\2\u01cf\65\3\2\2\2\u01d0\u01d1\7O\2\2\u01d1\u01d2\7\64\2"+
+    "\2\u01d2\u01d3\7P\2\2\u01d3\67\3\2\2\2\u01d4\u01d5\5\26\f\2\u01d5\u01d6"+
+    "\7\64\2\2\u01d6\u01d7\7\26\2\2\u01d79\3\2\2\2\u01d8\u01d9\7P\2\2\u01d9"+
+    "\u01da\7\64\2\2\u01da\u01db\7P\2\2\u01db;\3\2\2\2\u01dc\u01dd\7\32\2\2"+
+    "\u01dd\u01de\7\64\2\2\u01de\u01df\7P\2\2\u01df=\3\2\2\2*AGZ]iq~\u0082"+
+    "\u0086\u008b\u00a6\u00af\u00b3\u00b9\u00c2\u00cc\u00d4\u00da\u00e8\u00f3"+
+    "\u0133\u0135\u0155\u015b\u0163\u016e\u0174\u0177\u0179\u018e\u0194\u019b"+
+    "\u01a7\u01aa\u01b1\u01ba\u01bd\u01c0\u01c6\u01ce";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index 0f6f7c8f2d5..4a22d016505 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -512,6 +512,10 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
             operation = Operation.ADD;
         } else if (ctx.SUB() != null) {
             operation = Operation.SUB;
+        } else if (ctx.FIND() != null) {
+            operation = Operation.FIND;
+        } else if (ctx.MATCH() != null) {
+            operation = Operation.MATCH;
         } else if (ctx.LSH() != null) {
             operation = Operation.LSH;
         } else if (ctx.RSH() != null) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index acf9eb218b8..d3b92d7eb08 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -26,6 +26,7 @@ import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.Operation;
+import org.elasticsearch.painless.WriterConstants;
 import org.elasticsearch.painless.Locals;
 
 /**
@@ -61,6 +62,10 @@ public final class EBinary extends AExpression {
             analyzeAdd(locals);
         } else if (operation == Operation.SUB) {
             analyzeSub(locals);
+        } else if (operation == Operation.FIND) {
+            analyzeRegexOp(locals);
+        } else if (operation == Operation.MATCH) {
+            analyzeRegexOp(locals);
         } else if (operation == Operation.LSH) {
             analyzeLSH(locals);
         } else if (operation == Operation.RSH) {
@@ -320,6 +325,21 @@ public final class EBinary extends AExpression {
         }
     }
 
+    private void analyzeRegexOp(Locals variables) {
+        left.analyze(variables);
+        right.analyze(variables);
+
+        left.expected = Definition.STRING_TYPE;
+        right.expected = Definition.PATTERN_TYPE;
+
+        left = left.cast(variables);
+        right = right.cast(variables);
+
+        // It'd be nice to be able to do constant folding here but we can't because constants aren't flowing through EChain
+        promote = Definition.BOOLEAN_TYPE;
+        actual = Definition.BOOLEAN_TYPE;
+    }
+
     private void analyzeLSH(Locals variables) {
         left.analyze(variables);
         right.analyze(variables);
@@ -607,6 +627,12 @@ public final class EBinary extends AExpression {
             if (!cat) {
                 writer.writeToStrings();
             }
+        } else if (operation == Operation.FIND) {
+            writeBuildMatcher(writer);
+            writer.invokeVirtual(Definition.MATCHER_TYPE.type, WriterConstants.MATCHER_FIND);
+        } else if (operation == Operation.MATCH) {
+            writeBuildMatcher(writer);
+            writer.invokeVirtual(Definition.MATCHER_TYPE.type, WriterConstants.MATCHER_MATCHES);
         } else {
             left.write(writer);
             right.write(writer);
@@ -620,4 +646,10 @@ public final class EBinary extends AExpression {
 
         writer.writeBranch(tru, fals);
     }
+
+    private void writeBuildMatcher(MethodWriter writer) {
+        right.write(writer);
+        left.write(writer);
+        writer.invokeVirtual(Definition.PATTERN_TYPE.type, WriterConstants.PATTERN_MATCHER);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LRegex.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LRegex.java
index d91bc3394ee..720e9d45969 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LRegex.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LRegex.java
@@ -34,8 +34,6 @@ import org.elasticsearch.painless.WriterConstants;
  * Represents a regex constant. All regexes are constants.
  */
 public final class LRegex extends ALink {
-    private static final Definition.Type PATTERN_TYPE = Definition.getType("Pattern");
-
     private final String pattern;
     private Constant constant;
 
@@ -60,8 +58,8 @@ public final class LRegex extends ALink {
             throw createError(new IllegalArgumentException("Regex constant may only be read [" + pattern + "]."));
         }
 
-        constant = locals.addConstant(location, PATTERN_TYPE, "regexAt$" + location.getOffset(), this::initializeConstant);
-        after = PATTERN_TYPE;
+        constant = locals.addConstant(location, Definition.PATTERN_TYPE, "regexAt$" + location.getOffset(), this::initializeConstant);
+        after = Definition.PATTERN_TYPE;
 
         return this;
     }
@@ -74,7 +72,7 @@ public final class LRegex extends ALink {
     @Override
     void load(MethodWriter writer) {
         writer.writeDebugInfo(location);
-        writer.getStatic(WriterConstants.CLASS_TYPE, constant.name, PATTERN_TYPE.type);
+        writer.getStatic(WriterConstants.CLASS_TYPE, constant.name, Definition.PATTERN_TYPE.type);
     }
 
     @Override
@@ -84,6 +82,6 @@ public final class LRegex extends ALink {
 
     private void initializeConstant(MethodWriter writer) {
         writer.push(pattern);
-        writer.invokeStatic(PATTERN_TYPE.type, WriterConstants.PATTERN_COMPILE);
+        writer.invokeStatic(Definition.PATTERN_TYPE.type, WriterConstants.PATTERN_COMPILE);
     }
 }
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt
index b66793dcf5a..a8982871a33 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt
@@ -29,5 +29,31 @@ class Pattern -> java.util.regex.Pattern extends Object {
 }
 
 class Matcher -> java.util.regex.Matcher extends Object {
+  int end()
+  int end(int)
+  boolean find()
+  boolean find(int)
+  String group()
+  String group(int)
+  String namedGroup/group(String)
+  int groupCount()
+  boolean hasAnchoringBounds()
+  boolean hasTransparentBounds()
+  boolean hitEnd()
+  boolean lookingAt()
   boolean matches()
+  Pattern pattern()
+  String quoteReplacement(String)
+  Matcher region(int,int)
+  int regionEnd()
+  int regionStart()
+  String replaceAll(String)
+  String replaceFirst(String)
+  boolean requireEnd()
+  Matcher reset()
+  int start()
+  int start(int)
+  Matcher useAnchoringBounds(boolean)
+  Matcher usePattern(Pattern)
+  Matcher useTransparentBounds(boolean)
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java
index 69a81e0d77e..ee281e4656b 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java
@@ -19,53 +19,131 @@
 
 package org.elasticsearch.painless;
 
+import java.util.regex.PatternSyntaxException;
+
+import static java.util.Collections.singletonMap;
+import static org.hamcrest.Matchers.containsString;
+
 public class RegexTests extends ScriptTestCase {
     public void testPatternAfterReturn() {
-        assertEquals(true, exec("return /foo/.matcher(\"foo\").matches()"));
-        assertEquals(false, exec("return /foo/.matcher(\"bar\").matches()"));
+        assertEquals(true, exec("return 'foo' ==~ /foo/"));
+        assertEquals(false, exec("return 'bar' ==~ /foo/"));
     }
 
     public void testSlashesEscapePattern() {
-        assertEquals(true, exec("return /\\/\\//.matcher(\"//\").matches()"));
+        assertEquals(true, exec("return '//' ==~ /\\/\\//"));
     }
 
     public void testPatternAfterAssignment() {
-        assertEquals(true, exec("def a = /foo/; return a.matcher(\"foo\").matches()"));
+        assertEquals(true, exec("def a = /foo/; return 'foo' ==~ a"));
     }
 
     public void testPatternInIfStement() {
-        assertEquals(true, exec("if (/foo/.matcher(\"foo\").matches()) { return true } else { return false }"));
+        assertEquals(true, exec("if (/foo/.matcher('foo').matches()) { return true } else { return false }"));
+        assertEquals(true, exec("if ('foo' ==~ /foo/) { return true } else { return false }"));
     }
 
     public void testPatternAfterInfixBoolean() {
-        assertEquals(true, exec("return false || /foo/.matcher(\"foo\").matches()"));
-        assertEquals(true, exec("return true && /foo/.matcher(\"foo\").matches()"));
+        assertEquals(true, exec("return false || /foo/.matcher('foo').matches()"));
+        assertEquals(true, exec("return true && /foo/.matcher('foo').matches()"));
+        assertEquals(true, exec("return false || 'foo' ==~ /foo/"));
+        assertEquals(true, exec("return true && 'foo' ==~ /foo/"));
     }
 
     public void testPatternAfterUnaryNotBoolean() {
-        assertEquals(false, exec("return !/foo/.matcher(\"foo\").matches()"));
-        assertEquals(true, exec("return !/foo/.matcher(\"bar\").matches()"));
+        assertEquals(false, exec("return !/foo/.matcher('foo').matches()"));
+        assertEquals(true, exec("return !/foo/.matcher('bar').matches()"));
     }
 
     public void testInTernaryCondition()  {
-        assertEquals(true, exec("return /foo/.matcher(\"foo\").matches() ? true : false"));
-        assertEquals(1, exec("def i = 0; i += /foo/.matcher(\"foo\").matches() ? 1 : 1; return i"));
+        assertEquals(true, exec("return /foo/.matcher('foo').matches() ? true : false"));
+        assertEquals(1, exec("def i = 0; i += /foo/.matcher('foo').matches() ? 1 : 1; return i"));
+        assertEquals(true, exec("return 'foo' ==~ /foo/ ? true : false"));
+        assertEquals(1, exec("def i = 0; i += 'foo' ==~ /foo/ ? 1 : 1; return i"));
     }
 
     public void testInTernaryTrueArm()  {
-        assertEquals(true, exec("def i = true; return i ? /foo/.matcher(\"foo\").matches() : false"));
+        assertEquals(true, exec("def i = true; return i ? /foo/.matcher('foo').matches() : false"));
+        assertEquals(true, exec("def i = true; return i ? 'foo' ==~ /foo/ : false"));
     }
 
     public void testInTernaryFalseArm()  {
-        assertEquals(true, exec("def i = false; return i ? false : /foo/.matcher(\"foo\").matches()"));
+        assertEquals(true, exec("def i = false; return i ? false : 'foo' ==~ /foo/"));
     }
 
     public void testRegexInFunction() {
-        assertEquals(true, exec("boolean m(String s) {/foo/.matcher(s).matches()} m(\"foo\")"));
+        assertEquals(true, exec("boolean m(String s) {/foo/.matcher(s).matches()} m('foo')"));
+        assertEquals(true, exec("boolean m(String s) {s ==~ /foo/} m('foo')"));
     }
 
     public void testReturnRegexFromFunction() {
-        assertEquals(true, exec("Pattern m(boolean a) {a ? /foo/ : /bar/} m(true).matcher(\"foo\").matches()"));
+        assertEquals(true, exec("Pattern m(boolean a) {a ? /foo/ : /bar/} m(true).matcher('foo').matches()"));
+        assertEquals(true, exec("Pattern m(boolean a) {a ? /foo/ : /bar/} 'foo' ==~ m(true)"));
+        assertEquals(false, exec("Pattern m(boolean a) {a ? /foo/ : /bar/} m(false).matcher('foo').matches()"));
+        assertEquals(false, exec("Pattern m(boolean a) {a ? /foo/ : /bar/} 'foo' ==~ m(false)"));
     }
 
+    public void testCallMatcherDirectly() {
+        assertEquals(true, exec("return /foo/.matcher('foo').matches()"));
+        assertEquals(false, exec("return /foo/.matcher('bar').matches()"));
+    }
+
+    public void testFindInIf() {
+        assertEquals(true, exec("if ('fooasdfbasdf' =~ /foo/) {return true} else {return false}"));
+        assertEquals(true, exec("if ('1fooasdfbasdf' =~ /foo/) {return true} else {return false}"));
+        assertEquals(false, exec("if ('1f11ooasdfbasdf' =~ /foo/) {return true} else {return false}"));
+    }
+
+    public void testFindCastToBoolean() {
+        assertEquals(true, exec("return (boolean)('fooasdfbasdf' =~ /foo/)"));
+        assertEquals(true, exec("return (boolean)('111fooasdfbasdf' =~ /foo/)"));
+        assertEquals(false, exec("return (boolean)('fo11oasdfbasdf' =~ /foo/)"));
+    }
+
+    public void testFindOrStringConcat() {
+        assertEquals(true, exec("return 'f' + 'o' + 'o' =~ /foo/"));
+    }
+
+    public void testFindOfDef() {
+        assertEquals(true, exec("def s = 'foo'; return s =~ /foo/"));
+    }
+
+    public void testFindOnInput() {
+        assertEquals(true, exec("return params.s =~ /foo/", singletonMap("s", "fooasdfdf")));
+        assertEquals(false, exec("return params.s =~ /foo/", singletonMap("s", "11f2ooasdfdf")));
+    }
+
+    public void testGroup() {
+        assertEquals("foo", exec("Matcher m = /foo/.matcher('foo'); m.find(); return m.group()"));
+    }
+
+    public void testNumberedGroup() {
+        assertEquals("o", exec("Matcher m = /(f)(o)o/.matcher('foo'); m.find(); return m.group(2)"));
+    }
+
+    public void testNamedGroup() {
+        assertEquals("o", exec("Matcher m = /(?<first>f)(?<second>o)o/.matcher('foo'); m.find(); return m.namedGroup('second')"));
+    }
+
+    public void testCantUsePatternCompile() {
+        IllegalArgumentException e = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("Pattern.compile('aa')");
+        });
+        assertEquals("Unknown call [compile] with [1] arguments on type [Pattern].", e.getMessage());
+    }
+
+    public void testBadRegexPattern() {
+        PatternSyntaxException e = expectScriptThrows(PatternSyntaxException.class, () -> {
+            exec("/\\ujjjj/"); // Invalid unicode
+        });
+        assertThat(e.getMessage(), containsString("Illegal Unicode escape sequence near index 2"));
+        assertThat(e.getMessage(), containsString("\\ujjjj"));
+    }
+
+    public void testRegexAgainstNumber() {
+        ClassCastException e = expectScriptThrows(ClassCastException.class, () -> {
+            exec("12 ==~ /cat/");
+        });
+        assertEquals("Cannot cast from [int] to [String].", e.getMessage());
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
index a9e03095297..fa1538e56aa 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/WhenThingsGoWrongTests.java
@@ -22,7 +22,6 @@ package org.elasticsearch.painless;
 import java.lang.invoke.WrongMethodTypeException;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.regex.PatternSyntaxException;
 
 import static java.util.Collections.emptyMap;
 import static org.hamcrest.Matchers.containsString;
@@ -214,21 +213,6 @@ public class WhenThingsGoWrongTests extends ScriptTestCase {
         assertEquals("invalid sequence of tokens near ['}'].", e.getMessage());
     }
 
-    public void testCantUsePatternCompile() {
-        IllegalArgumentException e = expectScriptThrows(IllegalArgumentException.class, () -> {
-            exec("Pattern.compile(\"aa\")");
-        });
-        assertEquals("Unknown call [compile] with [1] arguments on type [Pattern].", e.getMessage());
-    }
-
-    public void testBadRegexPattern() {
-        PatternSyntaxException e = expectScriptThrows(PatternSyntaxException.class, () -> {
-            exec("/\\ujjjj/"); // Invalid unicode
-        });
-        assertThat(e.getMessage(), containsString("Illegal Unicode escape sequence near index 2"));
-        assertThat(e.getMessage(), containsString("\\ujjjj"));
-    }
-
     public void testBadBoxingCast() {
         expectScriptThrows(ClassCastException.class, () -> {
             exec("BitSet bs = new BitSet(); bs.and(2);");

From e442f07460b521b757ef11330b689202764f38d2 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Thu, 16 Jun 2016 15:29:49 +0200
Subject: [PATCH 0908/1311] Remove dead code

---
 .../java/org/elasticsearch/index/query/QueryParser.java    | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryParser.java b/core/src/main/java/org/elasticsearch/index/query/QueryParser.java
index 222d3bd4963..c17b80460e3 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryParser.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryParser.java
@@ -38,11 +38,4 @@ public interface QueryParser<QB extends QueryBuilder> {
      * @return the new QueryBuilder
      */
     Optional<QB> fromXContent(QueryParseContext parseContext) throws IOException;
-
-    /**
-     * @return an empty {@link QueryBuilder} instance for this parser that can be used for deserialization
-     */
-    default QB getBuilderPrototype() { // TODO remove this when nothing implements it
-        throw new UnsupportedOperationException();
-    }
 }

From 0faa9409b31b929fc83a3bb98134b89a50fd54d1 Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@elastic.co>
Date: Thu, 16 Jun 2016 15:36:29 +0200
Subject: [PATCH 0909/1311] Force test infra to use node client in
 NettyHttpRequestSizeLimitIT

---
 .../elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java
index bae3b8c8175..47b34811df6 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpRequestSizeLimitIT.java
@@ -42,9 +42,9 @@ import static org.hamcrest.Matchers.hasSize;
  * This test checks that in-flight requests are limited on HTTP level and that requests that are excluded from limiting can pass.
  *
  * As the same setting is also used to limit in-flight requests on transport level, we avoid transport messages by forcing
- * a single node "cluster".
+ * a single node "cluster". We also force test infrastructure to use the node client instead of the transport client for the same reason.
  */
-@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numClientNodes = 0, numDataNodes = 1)
+@ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numClientNodes = 0, numDataNodes = 1, transportClientRatio = 0)
 public class NettyHttpRequestSizeLimitIT extends ESIntegTestCase {
     private static final ByteSizeValue LIMIT = new ByteSizeValue(2, ByteSizeUnit.KB);
 

From 5aa4769b257cb874e12b7497210de74f113d8947 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 16 Jun 2016 09:43:14 -0400
Subject: [PATCH 0910/1311] Move waitForTaskCompletion into TaskManager

This allows for listening for the waiting to start using
MockTaskManager. This allows us to work around a race condition
in the TasksIT.
---
 .../tasks/get/TransportGetTaskAction.java     |  5 ++--
 .../tasks/list/TransportListTasksAction.java  | 20 +------------
 .../org/elasticsearch/tasks/TaskManager.java  | 23 +++++++++++++++
 .../tasks/RecordingTaskManagerListener.java   |  4 +++
 .../admin/cluster/node/tasks/TasksIT.java     | 28 ++++++++++++++++++-
 .../test/tasks/MockTaskManager.java           | 12 ++++++++
 .../test/tasks/MockTaskManagerListener.java   |  2 ++
 7 files changed, 71 insertions(+), 23 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
index 769676bcd6d..0a869ff296a 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
@@ -38,9 +38,9 @@ import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.tasks.PersistedTaskInfo;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
-import org.elasticsearch.tasks.PersistedTaskInfo;
 import org.elasticsearch.tasks.TaskPersistenceService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.BaseTransportResponseHandler;
@@ -51,7 +51,6 @@ import org.elasticsearch.transport.TransportService;
 import java.io.IOException;
 
 import static org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction.waitForCompletionTimeout;
-import static org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction.waitForTaskCompletion;
 
 /**
  * Action to get a single task. If the task isn't running then it'll try to request the status from request index.
@@ -148,7 +147,7 @@ public class TransportGetTaskAction extends HandledTransportAction<GetTaskReques
                 threadPool.generic().execute(new AbstractRunnable() {
                     @Override
                     protected void doRun() throws Exception {
-                        waitForTaskCompletion(taskManager, runningTask, waitForCompletionTimeout(request.getTimeout()));
+                        taskManager.waitForTaskCompletion(runningTask, waitForCompletionTimeout(request.getTimeout()));
                         // TODO look up the task's result from the .tasks index now that it is done
                         listener.onResponse(
                                 new GetTaskResponse(new PersistedTaskInfo(runningTask.taskInfo(clusterService.localNode(), true))));
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
index 7d6d122bbb7..9acdcff81c9 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
@@ -19,8 +19,6 @@
 
 package org.elasticsearch.action.admin.cluster.node.tasks.list;
 
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.support.ActionFilters;
@@ -34,7 +32,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskInfo;
-import org.elasticsearch.tasks.TaskManager;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -42,26 +39,12 @@ import java.io.IOException;
 import java.util.List;
 import java.util.function.Consumer;
 
-import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 
 /**
  *
  */
 public class TransportListTasksAction extends TransportTasksAction<Task, ListTasksRequest, ListTasksResponse, TaskInfo> {
-    public static void waitForTaskCompletion(TaskManager taskManager, Task task, long untilInNanos) {
-        while (System.nanoTime() - untilInNanos < 0) {
-            if (taskManager.getTask(task.getId()) == null) {
-                return;
-            }
-            try {
-                Thread.sleep(WAIT_FOR_COMPLETION_POLL.millis());
-            } catch (InterruptedException e) {
-                throw new ElasticsearchException("Interrupted waiting for completion of [{}]", e, task);
-            }
-        }
-        throw new ElasticsearchTimeoutException("Timed out waiting for completion of [{}]", task);
-    }
     public static long waitForCompletionTimeout(TimeValue timeout) {
         if (timeout == null) {
             timeout = DEFAULT_WAIT_FOR_COMPLETION_TIMEOUT;
@@ -69,7 +52,6 @@ public class TransportListTasksAction extends TransportTasksAction<Task, ListTas
         return System.nanoTime() + timeout.nanos();
     }
 
-    private static final TimeValue WAIT_FOR_COMPLETION_POLL = timeValueMillis(100);
     private static final TimeValue DEFAULT_WAIT_FOR_COMPLETION_TIMEOUT = timeValueSeconds(30);
 
     @Inject
@@ -105,7 +87,7 @@ public class TransportListTasksAction extends TransportTasksAction<Task, ListTas
                     // for itself or one of its child tasks
                     return;
                 }
-                waitForTaskCompletion(taskManager, task, timeoutNanos);
+                taskManager.waitForTaskCompletion(task, timeoutNanos);
             });
         }
         super.processTasks(request, operation);
diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskManager.java b/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
index 045522f3288..95adc850f82 100644
--- a/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
+++ b/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.tasks;
 
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionResponse;
@@ -28,6 +30,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.ConcurrentMapLong;
 import org.elasticsearch.transport.TransportRequest;
@@ -43,10 +46,13 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Consumer;
 
+import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
+
 /**
  * Task Manager service for keeping track of currently running tasks on the nodes
  */
 public class TaskManager extends AbstractComponent implements ClusterStateListener {
+    private static final TimeValue WAIT_FOR_COMPLETION_POLL = timeValueMillis(100);
 
     private final ConcurrentMapLong<Task> tasks = ConcurrentCollections.newConcurrentMapLongWithAggressiveConcurrency();
 
@@ -341,6 +347,23 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen
         }
     }
 
+    /**
+     * Blocks the calling thread, waiting for the task to vanish from the TaskManager.
+     */
+    public void waitForTaskCompletion(Task task, long untilInNanos) {
+        while (System.nanoTime() - untilInNanos < 0) {
+            if (getTask(task.getId()) == null) {
+                return;
+            }
+            try {
+                Thread.sleep(WAIT_FOR_COMPLETION_POLL.millis());
+            } catch (InterruptedException e) {
+                throw new ElasticsearchException("Interrupted waiting for completion of [{}]", e, task);
+            }
+        }
+        throw new ElasticsearchTimeoutException("Timed out waiting for completion of [{}]", task);
+    }
+
     private static class CancellableTaskHolder {
 
         private static final String TASK_FINISHED_MARKER = "task finished";
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/RecordingTaskManagerListener.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/RecordingTaskManagerListener.java
index 2aad88c99a9..656a5ab9ec4 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/RecordingTaskManagerListener.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/RecordingTaskManagerListener.java
@@ -60,6 +60,10 @@ public class RecordingTaskManagerListener implements MockTaskManagerListener {
         }
     }
 
+    @Override
+    public void waitForTaskCompletion(Task task) {
+    }
+
     public synchronized List<Tuple<Boolean, TaskInfo>> getEvents() {
         return Collections.unmodifiableList(new ArrayList<>(events));
     }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
index a2938876942..4c720536d0c 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
@@ -363,6 +363,10 @@ public class TasksIT extends ESIntegTestCase {
                         taskFinishLock.lock();
                         taskFinishLock.unlock();
                     }
+
+                    @Override
+                    public void waitForTaskCompletion(Task task) {
+                    }
                 });
             }
             indexFuture = client().prepareIndex("test", "test").setSource("test", "test").execute();
@@ -470,8 +474,30 @@ public class TasksIT extends ESIntegTestCase {
             // Wait for the task to start
             assertBusy(() -> client().admin().cluster().prepareGetTask(taskId).get());
 
-            // Spin up a request to wait for that task to finish
+            // Register listeners so we can be sure the waiting started
+            CountDownLatch waitForWaitingToStart = new CountDownLatch(1);
+            for (TransportService transportService : internalCluster().getInstances(TransportService.class)) {
+                ((MockTaskManager) transportService.getTaskManager()).addListener(new MockTaskManagerListener() {
+                    @Override
+                    public void waitForTaskCompletion(Task task) {
+                    }
+
+                    @Override
+                    public void onTaskRegistered(Task task) {
+                    }
+
+                    @Override
+                    public void onTaskUnregistered(Task task) {
+                        waitForWaitingToStart.countDown();
+                    }
+                });
+            }
+
+            // Spin up a request to wait for the test task to finish
             waitResponseFuture = wait.apply(taskId);
+
+            // Wait for the wait to start
+            waitForWaitingToStart.await();
         } finally {
             // Unblock the request so the wait for completion request can finish
             TestTaskPlugin.UnblockTestTasksAction.INSTANCE.newRequestBuilder(client()).get();
diff --git a/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java b/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java
index fc090e151a3..b0d16d10c49 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java
@@ -75,6 +75,18 @@ public class MockTaskManager extends TaskManager {
         return removedTask;
     }
 
+    @Override
+    public void waitForTaskCompletion(Task task, long untilInNanos) {
+        for (MockTaskManagerListener listener : listeners) {
+            try {
+                listener.waitForTaskCompletion(task);
+            } catch (Throwable t) {
+                logger.warn("failed to notify task manager listener about waitForTaskCompletion the task with id {}", t, task.getId());
+            }
+        }
+        super.waitForTaskCompletion(task, untilInNanos);
+    }
+
     public void addListener(MockTaskManagerListener listener) {
         listeners.add(listener);
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManagerListener.java b/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManagerListener.java
index d10dd357999..ede880c8f54 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManagerListener.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManagerListener.java
@@ -28,4 +28,6 @@ public interface MockTaskManagerListener {
     void onTaskRegistered(Task task);
 
     void onTaskUnregistered(Task task);
+
+    void waitForTaskCompletion(Task task);
 }

From b22c526b34c5bba261ec79a083a41e40f6f1483d Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Thu, 16 Jun 2016 15:52:58 +0200
Subject: [PATCH 0911/1311] Cut over settings registration to a pull model
 (#18890)

Today we have a push model for registering basically anything. All our extension points
are defined on modules which we pass in to plugins. This is harder to maintain and adds
unnecessary dependencies on the modules itself. This change moves towards a pull model
where the plugin offers a getter kind of method to get the extensions. This will also
help in the future if we need to pass dependencies to the extension points which can
easily be defined on the method as arguments if a pull model is used.
---
 .../resources/checkstyle_suppressions.xml     |   1 -
 .../client/transport/TransportClient.java     |  22 +++-
 .../common/settings/ClusterSettings.java      |   3 +-
 .../common/settings/Setting.java              |  12 ++
 .../common/settings/SettingsModule.java       |  78 ++++++++-----
 .../elasticsearch/env/EnvironmentModule.java  |   6 +-
 .../indices/breaker/CircuitBreakerModule.java |  48 --------
 .../breaker/CircuitBreakerService.java        |   1 +
 .../HierarchyCircuitBreakerService.java       |   2 -
 .../java/org/elasticsearch/node/Node.java     |  93 ++++++++++-----
 .../org/elasticsearch/plugins/Plugin.java     |  21 ++++
 .../elasticsearch/plugins/PluginsService.java |  11 +-
 .../elasticsearch/script/ScriptModule.java    |  16 ++-
 .../threadpool/ExecutorBuilder.java           |   2 +-
 .../threadpool/FixedExecutorBuilder.java      |   2 +-
 .../threadpool/ScalingExecutorBuilder.java    |   2 +-
 .../threadpool/ThreadPoolModule.java          |  44 -------
 .../cluster/ClusterModuleTests.java           |  26 +++--
 .../cluster/settings/SettingsFilteringIT.java |  18 ++-
 .../common/settings/SettingsModuleTests.java  |  84 +++++++------
 .../index/SettingsListenerIT.java             |   8 +-
 .../indices/IndicesOptionsIntegrationIT.java  |   9 +-
 .../RandomExceptionCircuitBreakerIT.java      |   7 +-
 .../script/NativeScriptTests.java             |  12 +-
 .../aggregations/AggregatorParsingTests.java  |  22 +---
 .../aggregations/BaseAggregationTestCase.java |  21 +---
 .../BasePipelineAggregationTestCase.java      |  23 +---
 .../basic/SearchWithRandomExceptionsIT.java   |   8 +-
 .../builder/SearchSourceBuilderTests.java     |  20 +---
 .../snapshots/mockstore/MockRepository.java   |   7 +-
 .../messy/tests/TemplateQueryParserTests.java |  12 +-
 .../percolator/PercolatorPlugin.java          |   8 +-
 .../discovery/azure/AzureDiscoveryPlugin.java |  27 +++--
 .../AzureDiscoveryClusterFormationTests.java  |   8 +-
 .../cloud/aws/AwsEc2Service.java              |  19 +--
 .../discovery/ec2/Ec2DiscoveryPlugin.java     |  70 +++++------
 .../discovery/gce/GceDiscoveryPlugin.java     |  19 +--
 .../discovery/gce/GceDiscoverTests.java       |  10 +-
 .../attachments/MapperAttachmentsPlugin.java  |  14 ++-
 .../azure/AzureRepositoryPlugin.java          |  25 ++--
 .../AzureStorageSettingsFilterTests.java      |   3 +-
 .../elasticsearch/cloud/aws/AwsS3Service.java |  19 +--
 .../repository/s3/S3RepositoryPlugin.java     | 110 +++++++++---------
 .../index/MockEngineFactoryPlugin.java        |   9 +-
 .../test/AbstractQueryTestCase.java           |  12 +-
 .../elasticsearch/test/ESIntegTestCase.java   |   5 +-
 .../org/elasticsearch/test/ESTestCase.java    |   3 +-
 .../test/InternalSettingsPlugin.java          |  10 +-
 .../test/InternalTestCluster.java             |   2 +-
 .../test/MockIndexEventListener.java          |   7 +-
 .../test/store/MockFSIndexStore.java          |  18 +--
 .../transport/AssertingLocalTransport.java    |   8 +-
 .../test/transport/MockTransportService.java  |   6 +-
 53 files changed, 546 insertions(+), 507 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/indices/breaker/CircuitBreakerModule.java
 delete mode 100644 core/src/main/java/org/elasticsearch/threadpool/ThreadPoolModule.java

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index d56bdeb537f..0a4dbbf3ff9 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -753,7 +753,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]AbstractClientHeadersTestCase.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterHealthIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterInfoServiceIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterModuleTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterStateDiffIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterStateTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]DiskUsageTests.java" checks="LineLength" />
diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
index 76c4eafad78..56f541631ab 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
@@ -37,19 +37,21 @@ import org.elasticsearch.common.inject.ModulesBuilder;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.network.NetworkService;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.indices.breaker.CircuitBreakerModule;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.monitor.MonitorService;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsModule;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.threadpool.ThreadPoolModule;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.netty.NettyTransport;
 
@@ -134,10 +136,9 @@ public class TransportClient extends AbstractClient {
                     modules.add(pluginModule);
                 }
                 modules.add(new PluginsModule(pluginsService));
-                modules.add(new SettingsModule(settings));
                 modules.add(new NetworkModule(networkService, settings, true, namedWriteableRegistry));
                 modules.add(new ClusterNameModule(settings));
-                modules.add(new ThreadPoolModule(threadPool));
+                modules.add(b -> b.bind(ThreadPool.class).toInstance(threadPool));
                 modules.add(new SearchModule(settings, namedWriteableRegistry) {
                     @Override
                     protected void configure() {
@@ -145,9 +146,20 @@ public class TransportClient extends AbstractClient {
                     }
                 });
                 modules.add(new ActionModule(false, true));
-                modules.add(new CircuitBreakerModule(settings));
 
                 pluginsService.processModules(modules);
+                final List<Setting<?>> additionalSettings = new ArrayList<>();
+                final List<String> additionalSettingsFilter = new ArrayList<>();
+                additionalSettings.addAll(pluginsService.getPluginSettings());
+                additionalSettingsFilter.addAll(pluginsService.getPluginSettingsFilter());
+                for (final ExecutorBuilder<?> builder : threadPool.builders()) {
+                    additionalSettings.addAll(builder.getRegisteredSettings());
+                }
+                SettingsModule settingsModule = new SettingsModule(settings, additionalSettings, additionalSettingsFilter);
+                CircuitBreakerService circuitBreakerService = Node.createCircuitBreakerService(settingsModule.getSettings(),
+                    settingsModule.getClusterSettings());
+                modules.add(settingsModule);
+                modules.add((b -> b.bind(CircuitBreakerService.class).toInstance(circuitBreakerService)));
 
                 Injector injector = modules.createInjector();
                 final TransportService transportService = injector.getInstance(TransportService.class);
diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index 8c2d4dc01bf..de2d3b75df6 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -420,6 +420,7 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     ResourceWatcherService.RELOAD_INTERVAL_MEDIUM,
                     ResourceWatcherService.RELOAD_INTERVAL_LOW,
                     SearchModule.INDICES_MAX_CLAUSE_COUNT_SETTING,
-                    ThreadPool.ESTIMATED_TIME_INTERVAL_SETTING
+                    ThreadPool.ESTIMATED_TIME_INTERVAL_SETTING,
+                    Node.BREAKER_TYPE_KEY
             )));
 }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/Setting.java b/core/src/main/java/org/elasticsearch/common/settings/Setting.java
index da32468acc3..2c3aad38c63 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/Setting.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/Setting.java
@@ -81,6 +81,11 @@ public class Setting<T> extends ToXContentToBytes {
          */
         Filtered,
 
+        /**
+         * iff this setting is shared with more than one module ie. can be defined multiple times.
+         */
+        Shared,
+
         /**
          * iff this setting can be dynamically updateable
          */
@@ -247,6 +252,13 @@ public class Setting<T> extends ToXContentToBytes {
         return properties.contains(Property.Deprecated);
     }
 
+    /**
+     * Returns <code>true</code> if this setting is shared with more than one other module or plugin, otherwise <code>false</code>
+     */
+    public boolean isShared() {
+        return properties.contains(Property.Shared);
+    }
+
     /**
      * Returns <code>true</code> iff this setting is a group setting. Group settings represent a set of settings rather than a single value.
      * The key, see {@link #getKey()}, in contrast to non-group settings is a prefix like <tt>cluster.store.</tt> that matches all settings
diff --git a/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java b/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
index 2ed5ffd86cd..5fd19c4fc1f 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/SettingsModule.java
@@ -19,7 +19,8 @@
 
 package org.elasticsearch.common.settings;
 
-import org.elasticsearch.common.inject.AbstractModule;
+import org.elasticsearch.common.inject.Binder;
+import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -28,9 +29,11 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.tribe.TribeService;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Predicate;
@@ -40,7 +43,7 @@ import java.util.stream.IntStream;
 /**
  * A module that binds the provided settings to the {@link Settings} interface.
  */
-public class SettingsModule extends AbstractModule {
+public class SettingsModule implements Module {
 
     private final Settings settings;
     private final Set<String> settingsFilterPattern = new HashSet<>();
@@ -49,8 +52,14 @@ public class SettingsModule extends AbstractModule {
     private static final Predicate<String> TRIBE_CLIENT_NODE_SETTINGS_PREDICATE =  (s) -> s.startsWith("tribe.")
         && TribeService.TRIBE_SETTING_KEYS.contains(s) == false;
     private final ESLogger logger;
+    private final IndexScopedSettings indexScopedSettings;
+    private final ClusterSettings clusterSettings;
 
-    public SettingsModule(Settings settings) {
+    public SettingsModule(Settings settings, Setting<?>... additionalSettings) {
+        this(settings, Arrays.asList(additionalSettings), Collections.emptyList());
+    }
+
+    public SettingsModule(Settings settings, List<Setting<?>> additionalSettings, List<String> settingsFilter) {
         logger = Loggers.getLogger(getClass(), settings);
         this.settings = settings;
         for (Setting<?> setting : ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) {
@@ -59,12 +68,16 @@ public class SettingsModule extends AbstractModule {
         for (Setting<?> setting : IndexScopedSettings.BUILT_IN_INDEX_SETTINGS) {
             registerSetting(setting);
         }
-    }
 
-    @Override
-    protected void configure() {
-        final IndexScopedSettings indexScopedSettings = new IndexScopedSettings(settings, new HashSet<>(this.indexSettings.values()));
-        final ClusterSettings clusterSettings = new ClusterSettings(settings, new HashSet<>(this.nodeSettings.values()));
+        for (Setting<?> setting : additionalSettings) {
+            registerSetting(setting);
+        }
+
+        for (String filter : settingsFilter) {
+            registerSettingsFilter(filter);
+        }
+        this.indexScopedSettings = new IndexScopedSettings(settings, new HashSet<>(this.indexSettings.values()));
+        this.clusterSettings = new ClusterSettings(settings, new HashSet<>(this.nodeSettings.values()));
         Settings indexSettings = settings.filter((s) -> (s.startsWith("index.") &&
             // special case - we want to get Did you mean indices.query.bool.max_clause_count
             // which means we need to by-pass this check for this setting
@@ -87,7 +100,7 @@ public class SettingsModule extends AbstractModule {
                     "In order to upgrade all indices the settings must be updated via the /${index}/_settings API. " +
                     "Unless all settings are dynamic all indices must be closed in order to apply the upgrade" +
                     "Indices created in the future should use index templates to set default values."
-                    ).split(" ")) {
+                ).split(" ")) {
                     if (count + word.length() > 85) {
                         builder.append(System.lineSeparator());
                         count = 0;
@@ -124,19 +137,23 @@ public class SettingsModule extends AbstractModule {
         final Predicate<String> acceptOnlyClusterSettings = TRIBE_CLIENT_NODE_SETTINGS_PREDICATE.negate();
         clusterSettings.validate(settings.filter(acceptOnlyClusterSettings));
         validateTribeSettings(settings, clusterSettings);
-        bind(Settings.class).toInstance(settings);
-        bind(SettingsFilter.class).toInstance(new SettingsFilter(settings, settingsFilterPattern));
+     }
 
-        bind(ClusterSettings.class).toInstance(clusterSettings);
-        bind(IndexScopedSettings.class).toInstance(indexScopedSettings);
+    @Override
+    public void configure(Binder binder) {
+        binder.bind(Settings.class).toInstance(settings);
+        binder.bind(SettingsFilter.class).toInstance(new SettingsFilter(settings, settingsFilterPattern));
+        binder.bind(ClusterSettings.class).toInstance(clusterSettings);
+        binder.bind(IndexScopedSettings.class).toInstance(indexScopedSettings);
     }
 
+
     /**
      * Registers a new setting. This method should be used by plugins in order to expose any custom settings the plugin defines.
      * Unless a setting is registered the setting is unusable. If a setting is never the less specified the node will reject
      * the setting during startup.
      */
-    public void registerSetting(Setting<?> setting) {
+    private void registerSetting(Setting<?> setting) {
         if (setting.isFiltered()) {
             if (settingsFilterPattern.contains(setting.getKey()) == false) {
                 registerSettingsFilter(setting.getKey());
@@ -144,13 +161,15 @@ public class SettingsModule extends AbstractModule {
         }
         if (setting.hasNodeScope() || setting.hasIndexScope()) {
             if (setting.hasNodeScope()) {
-                if (nodeSettings.containsKey(setting.getKey())) {
+                Setting<?> existingSetting = nodeSettings.get(setting.getKey());
+                if (existingSetting != null && (setting.isShared() == false || existingSetting.isShared() == false)) {
                     throw new IllegalArgumentException("Cannot register setting [" + setting.getKey() + "] twice");
                 }
                 nodeSettings.put(setting.getKey(), setting);
             }
             if (setting.hasIndexScope()) {
-                if (indexSettings.containsKey(setting.getKey())) {
+                Setting<?> existingSetting = indexSettings.get(setting.getKey());
+                if (existingSetting != null && (setting.isShared() == false || existingSetting.isShared() == false)) {
                     throw new IllegalArgumentException("Cannot register setting [" + setting.getKey() + "] twice");
                 }
                 indexSettings.put(setting.getKey(), setting);
@@ -164,7 +183,7 @@ public class SettingsModule extends AbstractModule {
      * Registers a settings filter pattern that allows to filter out certain settings that for instance contain sensitive information
      * or if a setting is for internal purposes only. The given pattern must either be a valid settings key or a simple regexp pattern.
      */
-    public void registerSettingsFilter(String filter) {
+    private void registerSettingsFilter(String filter) {
         if (SettingsFilter.isValidPattern(filter) == false) {
             throw new IllegalArgumentException("filter [" + filter +"] is invalid must be either a key or a regex pattern");
         }
@@ -174,19 +193,6 @@ public class SettingsModule extends AbstractModule {
         settingsFilterPattern.add(filter);
     }
 
-    /**
-     * Check if a setting has already been registered
-     */
-    public boolean exists(Setting<?> setting) {
-        if (setting.hasNodeScope()) {
-            return nodeSettings.containsKey(setting.getKey());
-        }
-        if (setting.hasIndexScope()) {
-            return indexSettings.containsKey(setting.getKey());
-        }
-        throw new IllegalArgumentException("setting scope is unknown. This should never happen!");
-    }
-
     private void validateTribeSettings(Settings settings, ClusterSettings clusterSettings) {
         Map<String, Settings> groups = settings.filter(TRIBE_CLIENT_NODE_SETTINGS_PREDICATE).getGroups("tribe.", true);
         for (Map.Entry<String, Settings>  tribeSettings : groups.entrySet()) {
@@ -200,4 +206,16 @@ public class SettingsModule extends AbstractModule {
             }
         }
     }
+
+    public Settings getSettings() {
+        return settings;
+    }
+
+    public IndexScopedSettings getIndexScopedSettings() {
+        return indexScopedSettings;
+    }
+
+    public ClusterSettings getClusterSettings() {
+        return clusterSettings;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/env/EnvironmentModule.java b/core/src/main/java/org/elasticsearch/env/EnvironmentModule.java
index 8a4aa1fa885..6a893a73437 100644
--- a/core/src/main/java/org/elasticsearch/env/EnvironmentModule.java
+++ b/core/src/main/java/org/elasticsearch/env/EnvironmentModule.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.env;
 
 import org.elasticsearch.common.inject.AbstractModule;
+import org.elasticsearch.threadpool.ThreadPool;
 
 /**
  *
@@ -27,13 +28,16 @@ import org.elasticsearch.common.inject.AbstractModule;
 public class EnvironmentModule extends AbstractModule {
 
     private final Environment environment;
+    private final ThreadPool threadPool;
 
-    public EnvironmentModule(Environment environment) {
+    public EnvironmentModule(Environment environment, ThreadPool threadPool) {
+        this.threadPool = threadPool;
         this.environment = environment;
     }
 
     @Override
     protected void configure() {
+        bind(ThreadPool.class).toInstance(threadPool);
         bind(Environment.class).toInstance(environment);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/indices/breaker/CircuitBreakerModule.java b/core/src/main/java/org/elasticsearch/indices/breaker/CircuitBreakerModule.java
deleted file mode 100644
index 084d3b7c66a..00000000000
--- a/core/src/main/java/org/elasticsearch/indices/breaker/CircuitBreakerModule.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.indices.breaker;
-
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.settings.Settings;
-
-public class CircuitBreakerModule extends AbstractModule {
-
-    public static final String TYPE_KEY = "indices.breaker.type";
-
-    private final Settings settings;
-
-    public CircuitBreakerModule(Settings settings) {
-        this.settings = settings;
-    }
-
-    @Override
-    protected void configure() {
-        String type = settings.get(TYPE_KEY);
-        Class<? extends CircuitBreakerService> impl;
-        if (type == null || type.equals("hierarchy")) {
-            impl = HierarchyCircuitBreakerService.class;
-        } else if (type.equals("none")) {
-            impl = NoneCircuitBreakerService.class;
-        } else {
-            throw new IllegalArgumentException("Unknown circuit breaker type [" + type + "]");
-        }
-        bind(CircuitBreakerService.class).to(impl).asEagerSingleton();
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/indices/breaker/CircuitBreakerService.java b/core/src/main/java/org/elasticsearch/indices/breaker/CircuitBreakerService.java
index d8bf8f10695..05bf8472ac9 100644
--- a/core/src/main/java/org/elasticsearch/indices/breaker/CircuitBreakerService.java
+++ b/core/src/main/java/org/elasticsearch/indices/breaker/CircuitBreakerService.java
@@ -64,4 +64,5 @@ public abstract class CircuitBreakerService extends AbstractLifecycleComponent<C
     @Override
     protected void doClose() {
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java b/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java
index 45304d135cb..328b9564a5d 100644
--- a/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java
+++ b/core/src/main/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerService.java
@@ -23,7 +23,6 @@ import org.elasticsearch.common.breaker.ChildMemoryCircuitBreaker;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.breaker.CircuitBreakingException;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
@@ -79,7 +78,6 @@ public class HierarchyCircuitBreakerService extends CircuitBreakerService {
     // Tripped count for when redistribution was attempted but wasn't successful
     private final AtomicLong parentTripCount = new AtomicLong(0);
 
-    @Inject
     public HierarchyCircuitBreakerService(Settings settings, ClusterSettings clusterSettings) {
         super(settings);
         this.fielddataSettings = new BreakerSettings(CircuitBreaker.FIELDDATA,
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index 2b5fa53a8f9..87bfd99b900 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -78,7 +78,9 @@ import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.analysis.AnalysisModule;
-import org.elasticsearch.indices.breaker.CircuitBreakerModule;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.store.IndicesStore;
 import org.elasticsearch.indices.ttl.IndicesTTLService;
@@ -101,7 +103,6 @@ import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.tasks.TaskPersistenceService;
 import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.threadpool.ThreadPoolModule;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.tribe.TribeModule;
 import org.elasticsearch.tribe.TribeService;
@@ -133,6 +134,7 @@ import java.util.function.Function;
  */
 public class Node implements Closeable {
 
+
     public static final Setting<Boolean> WRITE_PORTS_FIELD_SETTING =
         Setting.boolSetting("node.portsfile", false, Property.NodeScope);
     public static final Setting<Boolean> NODE_DATA_SETTING = Setting.boolSetting("node.data", true, Property.NodeScope);
@@ -146,6 +148,16 @@ public class Node implements Closeable {
         Setting.boolSetting("node.ingest", true, Property.NodeScope);
     public static final Setting<String> NODE_NAME_SETTING = Setting.simpleString("node.name", Property.NodeScope);
     public static final Setting<Settings> NODE_ATTRIBUTES = Setting.groupSetting("node.attr.", Property.NodeScope);
+    public static final Setting<String> BREAKER_TYPE_KEY = new Setting<>("indices.breaker.type", "hierarchy", (s) -> {
+        switch (s) {
+            case "hierarchy":
+            case "none":
+                return s;
+            default:
+                throw new IllegalArgumentException("indices.breaker.type must be one of [hierarchy, none] but was: " + s);
+        }
+    }, Setting.Property.NodeScope);
+
 
 
     private static final String CLIENT_TYPE = "node";
@@ -168,8 +180,9 @@ public class Node implements Closeable {
     protected Node(Environment tmpEnv, Version version, Collection<Class<? extends Plugin>> classpathPlugins) {
         Settings tmpSettings = Settings.builder().put(tmpEnv.settings())
                 .put(Client.CLIENT_TYPE_SETTING_S.getKey(), CLIENT_TYPE).build();
-        tmpSettings = TribeService.processSettings(tmpSettings);
+        final List<Closeable> resourcesToClose = new ArrayList<>(); // register everything we need to release in the case of an error
 
+        tmpSettings = TribeService.processSettings(tmpSettings);
         ESLogger logger = Loggers.getLogger(Node.class, NODE_NAME_SETTING.get(tmpSettings));
         final String displayVersion = version + (Build.CURRENT.isSnapshot() ? "-SNAPSHOT" : "");
         final JvmInfo jvmInfo = JvmInfo.jvmInfo();
@@ -203,40 +216,48 @@ public class Node implements Closeable {
         this.pluginsService = new PluginsService(tmpSettings, tmpEnv.modulesFile(), tmpEnv.pluginsFile(), classpathPlugins);
         this.settings = pluginsService.updatedSettings();
         // create the environment based on the finalized (processed) view of the settings
-        this.environment = new Environment(this.settings());
-
-        final NodeEnvironment nodeEnvironment;
-        try {
-            nodeEnvironment = new NodeEnvironment(this.settings, this.environment);
-        } catch (IOException ex) {
-            throw new IllegalStateException("Failed to created node environment", ex);
-        }
-        final NetworkService networkService = new NetworkService(settings);
+        this.environment = new Environment(this.settings);
         final List<ExecutorBuilder<?>> executorBuilders = pluginsService.getExecutorBuilders(settings);
-        final ThreadPool threadPool = new ThreadPool(settings, executorBuilders.toArray(new ExecutorBuilder[0]));
-        final ScriptModule scriptModule = ScriptModule.create(settings, pluginsService.filterPlugins(ScriptPlugin.class));
-        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
+
         boolean success = false;
         try {
+            final ThreadPool threadPool = new ThreadPool(settings, executorBuilders.toArray(new ExecutorBuilder[0]));
+            resourcesToClose.add(() -> ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS));
+            final List<Setting<?>> additionalSettings = new ArrayList<>();
+            final List<String> additionalSettingsFilter = new ArrayList<>();
+            additionalSettings.addAll(pluginsService.getPluginSettings());
+            additionalSettingsFilter.addAll(pluginsService.getPluginSettingsFilter());
+            for (final ExecutorBuilder<?> builder : threadPool.builders()) {
+                additionalSettings.addAll(builder.getRegisteredSettings());
+            }
+            final ScriptModule scriptModule = ScriptModule.create(settings, pluginsService.filterPlugins(ScriptPlugin.class));
+            additionalSettings.addAll(scriptModule.getSettings());
+            // this is as early as we can validate settings at this point. we already pass them to ScriptModule as well as ThreadPool
+            // so we might be late here already
+            final SettingsModule settingsModule = new SettingsModule(this.settings, additionalSettings, additionalSettingsFilter);
+            final NodeEnvironment nodeEnvironment;
+            try {
+                nodeEnvironment = new NodeEnvironment(this.settings, this.environment);
+                resourcesToClose.add(nodeEnvironment);
+            } catch (IOException ex) {
+                throw new IllegalStateException("Failed to created node environment", ex);
+            }
+            final NetworkService networkService = new NetworkService(settings);
+            NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
             ModulesBuilder modules = new ModulesBuilder();
             modules.add(new Version.Module(version));
-            modules.add(new CircuitBreakerModule(settings));
             // plugin modules must be added here, before others or we can get crazy injection errors...
             for (Module pluginModule : pluginsService.nodeModules()) {
                 modules.add(pluginModule);
             }
             final MonitorService monitorService = new MonitorService(settings, nodeEnvironment, threadPool);
             modules.add(new PluginsModule(pluginsService));
-            SettingsModule settingsModule = new SettingsModule(this.settings);
-            modules.add(settingsModule);
-            modules.add(new EnvironmentModule(environment));
+            modules.add(new EnvironmentModule(environment, threadPool));
             modules.add(new NodeModule(this, monitorService));
             modules.add(new NetworkModule(networkService, settings, false, namedWriteableRegistry));
             modules.add(scriptModule);
             modules.add(new NodeEnvironmentModule(nodeEnvironment));
             modules.add(new ClusterNameModule(this.settings));
-            final ThreadPoolModule threadPoolModule = new ThreadPoolModule(threadPool);
-            modules.add(threadPoolModule);
             modules.add(new DiscoveryModule(this.settings));
             modules.add(new ClusterModule(this.settings));
             modules.add(new IndicesModule());
@@ -248,23 +269,20 @@ public class Node implements Closeable {
             modules.add(new RepositoriesModule());
             modules.add(new TribeModule());
             modules.add(new AnalysisModule(environment));
-
             pluginsService.processModules(modules);
-
-            scriptModule.prepareSettings(settingsModule);
-
-            threadPoolModule.prepareSettings(settingsModule);
-
+            CircuitBreakerService circuitBreakerService = createCircuitBreakerService(settingsModule.getSettings(),
+                settingsModule.getClusterSettings());
+            resourcesToClose.add(circuitBreakerService);
+            modules.add(settingsModule);
+            modules.add(b -> b.bind(CircuitBreakerService.class).toInstance(circuitBreakerService));
             injector = modules.createInjector();
-
             client = injector.getInstance(Client.class);
             success = true;
         } catch (IOException ex) {
             throw new ElasticsearchException("failed to bind service", ex);
         } finally {
             if (!success) {
-                nodeEnvironment.close();
-                ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
+                IOUtils.closeWhileHandlingException(resourcesToClose);
             }
         }
 
@@ -590,4 +608,19 @@ public class Node implements Closeable {
             throw new RuntimeException("Failed to rename ports file", e);
         }
     }
+
+    /**
+     * Creates a new {@link CircuitBreakerService} based on the settings provided.
+     * @see #BREAKER_TYPE_KEY
+     */
+    public static CircuitBreakerService createCircuitBreakerService(Settings settings, ClusterSettings clusterSettings) {
+        String type = BREAKER_TYPE_KEY.get(settings);
+        if (type.equals("hierarchy")) {
+            return new HierarchyCircuitBreakerService(settings, clusterSettings);
+        } else if (type.equals("none")) {
+            return new NoneCircuitBreakerService();
+        } else {
+            throw new IllegalArgumentException("Unknown circuit breaker type [" + type + "]");
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/plugins/Plugin.java b/core/src/main/java/org/elasticsearch/plugins/Plugin.java
index c619072a6ba..3907551a3cc 100644
--- a/core/src/main/java/org/elasticsearch/plugins/Plugin.java
+++ b/core/src/main/java/org/elasticsearch/plugins/Plugin.java
@@ -21,7 +21,9 @@ package org.elasticsearch.plugins;
 
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.threadpool.ExecutorBuilder;
@@ -76,6 +78,16 @@ public abstract class Plugin {
      */
     public void onIndexModule(IndexModule indexModule) {}
 
+    /**
+     * Returns a list of additional {@link Setting} definitions for this plugin.
+     */
+    public List<Setting<?>> getSettings() { return Collections.emptyList(); }
+
+    /**
+     * Returns a list of additional settings filter for this plugin
+     */
+    public List<String> getSettingsFilter() { return Collections.emptyList(); }
+
     /**
      * Old-style guice index level extension point.
      *
@@ -84,6 +96,15 @@ public abstract class Plugin {
     @Deprecated
     public final void onModule(IndexModule indexModule) {}
 
+
+    /**
+     * Old-style guice settings extension point.
+     *
+     * @deprecated use #getSettings and #getSettingsFilter instead
+     */
+    @Deprecated
+    public final void onModule(SettingsModule settingsModule) {}
+
     /**
      * Old-style guice scripting extension point.
      *
diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
index 130870ea661..6d5e0161142 100644
--- a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
+++ b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
@@ -39,6 +39,7 @@ import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.script.NativeScriptFactory;
 import org.elasticsearch.script.ScriptContext;
@@ -64,7 +65,6 @@ import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import static org.elasticsearch.common.io.FileSystemUtils.isAccessibleDirectory;
 
@@ -83,6 +83,14 @@ public class PluginsService extends AbstractComponent {
 
     private final Map<Plugin, List<OnModuleReference>> onModuleReferences;
 
+    public List<Setting<?>> getPluginSettings() {
+        return plugins.stream().flatMap(p -> p.v2().getSettings().stream()).collect(Collectors.toList());
+    }
+
+    public List<String> getPluginSettingsFilter() {
+        return plugins.stream().flatMap(p -> p.v2().getSettingsFilter().stream()).collect(Collectors.toList());
+    }
+
     static class OnModuleReference {
         public final Class<? extends Module> moduleClass;
         public final Method onModuleMethod;
@@ -288,6 +296,7 @@ public class PluginsService extends AbstractComponent {
             plugin.v2().onIndexModule(indexModule);
         }
     }
+
     /**
      * Get information about plugins and modules
      */
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptModule.java b/core/src/main/java/org/elasticsearch/script/ScriptModule.java
index b3c8e46c753..a66f642fa15 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptModule.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptModule.java
@@ -20,10 +20,11 @@
 package org.elasticsearch.script;
 
 import org.elasticsearch.common.inject.AbstractModule;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.plugins.ScriptPlugin;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
@@ -57,13 +58,16 @@ public class ScriptModule extends AbstractModule {
      * This method is called after all modules have been processed but before we actually validate all settings. This allows the
      * script extensions to add all their settings.
      */
-    public void prepareSettings(SettingsModule settingsModule) {
-        scriptSettings.getScriptTypeSettings().forEach(settingsModule::registerSetting);
-        scriptSettings.getScriptContextSettings().forEach(settingsModule::registerSetting);
-        scriptSettings.getScriptLanguageSettings().forEach(settingsModule::registerSetting);
-        settingsModule.registerSetting(scriptSettings.getDefaultScriptLanguageSetting());
+    public List<Setting<?>> getSettings() {
+        ArrayList<Setting<?>> settings = new ArrayList<>();
+        scriptSettings.getScriptTypeSettings().forEach(settings::add);
+        scriptSettings.getScriptContextSettings().forEach(settings::add);
+        scriptSettings.getScriptLanguageSettings().forEach(settings::add);
+        settings.add(scriptSettings.getDefaultScriptLanguageSetting());
+        return settings;
     }
 
+
     @Override
     protected void configure() {
         ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
diff --git a/core/src/main/java/org/elasticsearch/threadpool/ExecutorBuilder.java b/core/src/main/java/org/elasticsearch/threadpool/ExecutorBuilder.java
index 434e6fc509c..5f36d391eba 100644
--- a/core/src/main/java/org/elasticsearch/threadpool/ExecutorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/threadpool/ExecutorBuilder.java
@@ -51,7 +51,7 @@ public abstract class ExecutorBuilder<U extends ExecutorBuilder.ExecutorSettings
      *
      * @return the list of registered settings
      */
-    abstract List<Setting<?>> getRegisteredSettings();
+    public abstract List<Setting<?>> getRegisteredSettings();
 
     /**
      * Return an executor settings object from the node-level settings.
diff --git a/core/src/main/java/org/elasticsearch/threadpool/FixedExecutorBuilder.java b/core/src/main/java/org/elasticsearch/threadpool/FixedExecutorBuilder.java
index 0735774d972..de7dbbaefc9 100644
--- a/core/src/main/java/org/elasticsearch/threadpool/FixedExecutorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/threadpool/FixedExecutorBuilder.java
@@ -86,7 +86,7 @@ public final class FixedExecutorBuilder extends ExecutorBuilder<FixedExecutorBui
     }
 
     @Override
-    List<Setting<?>> getRegisteredSettings() {
+    public List<Setting<?>> getRegisteredSettings() {
         return Arrays.asList(sizeSetting, queueSizeSetting);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java b/core/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java
index 68c70c83c19..6ab108e94b4 100644
--- a/core/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java
@@ -77,7 +77,7 @@ public final class ScalingExecutorBuilder extends ExecutorBuilder<ScalingExecuto
     }
 
     @Override
-    List<Setting<?>> getRegisteredSettings() {
+    public List<Setting<?>> getRegisteredSettings() {
         return Arrays.asList(coreSetting, maxSetting, keepAliveSetting);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/threadpool/ThreadPoolModule.java b/core/src/main/java/org/elasticsearch/threadpool/ThreadPoolModule.java
deleted file mode 100644
index 843febfef8c..00000000000
--- a/core/src/main/java/org/elasticsearch/threadpool/ThreadPoolModule.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.threadpool;
-
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.settings.SettingsModule;
-
-public class ThreadPoolModule extends AbstractModule {
-
-    private final ThreadPool threadPool;
-
-    public ThreadPoolModule(final ThreadPool threadPool) {
-        this.threadPool = threadPool;
-    }
-
-    public void prepareSettings(SettingsModule settingsModule) {
-        for (final ExecutorBuilder<?> builder : threadPool.builders()) {
-            builder.getRegisteredSettings().forEach(settingsModule::registerSetting);
-        }
-    }
-
-    @Override
-    protected void configure() {
-        bind(ThreadPool.class).toInstance(threadPool);
-    }
-
-}
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java b/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
index fd924641859..98bb013f308 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
@@ -67,32 +67,32 @@ public class ClusterModuleTests extends ModuleTestCase {
     }
 
     public void testRegisterClusterDynamicSettingDuplicate() {
-        SettingsModule module = new SettingsModule(Settings.EMPTY);
         try {
-            module.registerSetting(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING);
+            new SettingsModule(Settings.EMPTY, EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING);
         } catch (IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "Cannot register setting [" + EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey() + "] twice");
+            assertEquals(e.getMessage(),
+                "Cannot register setting [" + EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey() + "] twice");
         }
     }
 
     public void testRegisterClusterDynamicSetting() {
-        SettingsModule module = new SettingsModule(Settings.EMPTY);
-        module.registerSetting(Setting.boolSetting("foo.bar", false, Property.Dynamic, Property.NodeScope));
+        SettingsModule module = new SettingsModule(Settings.EMPTY,
+            Setting.boolSetting("foo.bar", false, Property.Dynamic, Property.NodeScope));
         assertInstanceBinding(module, ClusterSettings.class, service -> service.hasDynamicSetting("foo.bar"));
     }
 
     public void testRegisterIndexDynamicSettingDuplicate() {
-        SettingsModule module = new SettingsModule(Settings.EMPTY);
         try {
-            module.registerSetting(EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING);
+            new SettingsModule(Settings.EMPTY, EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING);
         } catch (IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "Cannot register setting [" + EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey() + "] twice");
+            assertEquals(e.getMessage(),
+                "Cannot register setting [" + EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING.getKey() + "] twice");
         }
     }
 
     public void testRegisterIndexDynamicSetting() {
-        SettingsModule module = new SettingsModule(Settings.EMPTY);
-        module.registerSetting(Setting.boolSetting("index.foo.bar", false, Property.Dynamic, Property.IndexScope));
+        SettingsModule module = new SettingsModule(Settings.EMPTY,
+            Setting.boolSetting("index.foo.bar", false, Property.Dynamic, Property.IndexScope));
         assertInstanceBinding(module, IndexScopedSettings.class, service -> service.hasDynamicSetting("index.foo.bar"));
     }
 
@@ -101,7 +101,8 @@ public class ClusterModuleTests extends ModuleTestCase {
         try {
             module.registerAllocationDecider(EnableAllocationDecider.class);
         } catch (IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "Can't register the same [allocation_decider] more than once for [" + EnableAllocationDecider.class.getName() + "]");
+            assertEquals(e.getMessage(),
+                "Can't register the same [allocation_decider] more than once for [" + EnableAllocationDecider.class.getName() + "]");
         }
     }
 
@@ -146,7 +147,8 @@ public class ClusterModuleTests extends ModuleTestCase {
             module.registerIndexTemplateFilter(FakeIndexTemplateFilter.class);
             module.registerIndexTemplateFilter(FakeIndexTemplateFilter.class);
         } catch (IllegalArgumentException e) {
-            assertEquals(e.getMessage(), "Can't register the same [index_template_filter] more than once for [" + FakeIndexTemplateFilter.class.getName() + "]");
+            assertEquals(e.getMessage(),
+                "Can't register the same [index_template_filter] more than once for [" + FakeIndexTemplateFilter.class.getName() + "]");
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java b/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java
index 194b82620de..339ecf2bec6 100644
--- a/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/settings/SettingsFilteringIT.java
@@ -30,7 +30,9 @@ import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 
+import java.util.Arrays;
 import java.util.Collection;
+import java.util.List;
 
 import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -72,12 +74,16 @@ public class SettingsFilteringIT extends ESIntegTestCase {
             return Settings.builder().put("some.node.setting", true).put("some.other.node.setting", true).build();
         }
 
-        public void onModule(SettingsModule module) {
-            module.registerSetting(SOME_NODE_SETTING);
-            module.registerSetting(SOME_OTHER_NODE_SETTING);
-            module.registerSetting(Setting.groupSetting("index.filter_test.", Property.IndexScope));
-            module.registerSettingsFilter("index.filter_test.foo");
-            module.registerSettingsFilter("index.filter_test.bar*");
+        @Override
+        public List<Setting<?>> getSettings() {
+            return Arrays.asList(SOME_NODE_SETTING,
+            SOME_OTHER_NODE_SETTING,
+            Setting.groupSetting("index.filter_test.", Property.IndexScope));
+        }
+
+        @Override
+        public List<String> getSettingsFilter() {
+            return Arrays.asList("index.filter_test.foo", "index.filter_test.bar*");
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
index 5e992fc947c..353b7b61d6c 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
@@ -21,6 +21,10 @@ package org.elasticsearch.common.settings;
 
 import org.elasticsearch.common.inject.ModuleTestCase;
 import org.elasticsearch.common.settings.Setting.Property;
+import org.joda.time.MonthDay;
+
+import java.util.Arrays;
+import java.util.Collections;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.is;
@@ -35,18 +39,16 @@ public class SettingsModuleTests extends ModuleTestCase {
         }
         {
             Settings settings = Settings.builder().put("cluster.routing.allocation.balance.shard", "[2.0]").build();
-            SettingsModule module = new SettingsModule(settings);
             IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> assertInstanceBinding(module, Settings.class, (s) -> s == settings));
+                () ->  new SettingsModule(settings));
             assertEquals("Failed to parse value [[2.0]] for setting [cluster.routing.allocation.balance.shard]", ex.getMessage());
         }
 
         {
             Settings settings = Settings.builder().put("cluster.routing.allocation.balance.shard", "[2.0]")
                 .put("some.foo.bar", 1).build();
-            SettingsModule module = new SettingsModule(settings);
             IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> assertInstanceBinding(module, Settings.class, (s) -> s == settings));
+                () -> new SettingsModule(settings));
             assertEquals("Failed to parse value [[2.0]] for setting [cluster.routing.allocation.balance.shard]", ex.getMessage());
             assertEquals(1, ex.getSuppressed().length);
             assertEquals("unknown setting [some.foo.bar]", ex.getSuppressed()[0].getMessage());
@@ -55,9 +57,8 @@ public class SettingsModuleTests extends ModuleTestCase {
         {
             Settings settings = Settings.builder().put("index.codec", "default")
                 .put("index.foo.bar", 1).build();
-            SettingsModule module = new SettingsModule(settings);
             IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> assertInstanceBinding(module, Settings.class, (s) -> s == settings));
+                () -> new SettingsModule(settings));
             assertEquals("node settings must not contain any index level settings", ex.getMessage());
         }
 
@@ -71,16 +72,13 @@ public class SettingsModuleTests extends ModuleTestCase {
     public void testRegisterSettings() {
         {
             Settings settings = Settings.builder().put("some.custom.setting", "2.0").build();
-            SettingsModule module = new SettingsModule(settings);
-            module.registerSetting(Setting.floatSetting("some.custom.setting", 1.0f, Property.NodeScope));
+            SettingsModule module = new SettingsModule(settings, Setting.floatSetting("some.custom.setting", 1.0f, Property.NodeScope));
             assertInstanceBinding(module, Settings.class, (s) -> s == settings);
         }
         {
             Settings settings = Settings.builder().put("some.custom.setting", "false").build();
-            SettingsModule module = new SettingsModule(settings);
-            module.registerSetting(Setting.floatSetting("some.custom.setting", 1.0f, Property.NodeScope));
             try {
-                assertInstanceBinding(module, Settings.class, (s) -> s == settings);
+                new SettingsModule(settings, Setting.floatSetting("some.custom.setting", 1.0f, Property.NodeScope));
                 fail();
             } catch (IllegalArgumentException ex) {
                 assertEquals("Failed to parse value [false] for setting [some.custom.setting]", ex.getMessage());
@@ -96,9 +94,8 @@ public class SettingsModuleTests extends ModuleTestCase {
         }
         {
             Settings settings = Settings.builder().put("tribe.t1.cluster.routing.allocation.balance.shard", "[2.0]").build();
-            SettingsModule module = new SettingsModule(settings);
             try {
-                assertInstanceBinding(module, Settings.class, (s) -> s == settings);
+                new SettingsModule(settings);
                 fail();
             } catch (IllegalArgumentException ex) {
                 assertEquals(
@@ -116,9 +113,8 @@ public class SettingsModuleTests extends ModuleTestCase {
         }
         {
             Settings settings = Settings.builder().put("tribe.blocks.write", "BOOM").build();
-            SettingsModule module = new SettingsModule(settings);
             try {
-                assertInstanceBinding(module, Settings.class, (s) -> s == settings);
+                new SettingsModule(settings);
                 fail();
             } catch (IllegalArgumentException ex) {
                 assertEquals("Failed to parse value [BOOM] cannot be parsed to boolean [ true/1/on/yes OR false/0/off/no ]",
@@ -127,9 +123,8 @@ public class SettingsModuleTests extends ModuleTestCase {
         }
         {
             Settings settings = Settings.builder().put("tribe.blocks.wtf", "BOOM").build();
-            SettingsModule module = new SettingsModule(settings);
             try {
-                assertInstanceBinding(module, Settings.class, (s) -> s == settings);
+                new SettingsModule(settings);
                 fail();
             } catch (IllegalArgumentException ex) {
                 assertEquals("tribe.blocks validation failed: unknown setting [wtf]", ex.getMessage());
@@ -147,9 +142,8 @@ public class SettingsModuleTests extends ModuleTestCase {
 
         {
             Settings settings = Settings.builder().put("logger._root", "BOOM").put("logger.transport", "WOW").build();
-            SettingsModule module = new SettingsModule(settings);
             try {
-                assertInstanceBinding(module, Settings.class, (s) -> s == settings);
+                new SettingsModule(settings);
                 fail();
             } catch (IllegalArgumentException ex) {
                 assertEquals("No enum constant org.elasticsearch.common.logging.ESLoggerFactory.LogLevel.BOOM", ex.getMessage());
@@ -160,18 +154,17 @@ public class SettingsModuleTests extends ModuleTestCase {
 
     public void testRegisterSettingsFilter() {
         Settings settings = Settings.builder().put("foo.bar", "false").put("bar.foo", false).put("bar.baz", false).build();
-        SettingsModule module = new SettingsModule(settings);
-        module.registerSetting(Setting.boolSetting("foo.bar", true, Property.NodeScope));
-        module.registerSetting(Setting.boolSetting("bar.foo", true, Property.NodeScope, Property.Filtered));
-        module.registerSetting(Setting.boolSetting("bar.baz", true, Property.NodeScope));
-
-        module.registerSettingsFilter("foo.*");
         try {
-            module.registerSettingsFilter("bar.foo");
+            new SettingsModule(settings, Arrays.asList(Setting.boolSetting("foo.bar", true, Property.NodeScope),
+            Setting.boolSetting("bar.foo", true, Property.NodeScope, Property.Filtered),
+            Setting.boolSetting("bar.baz", true, Property.NodeScope)), Arrays.asList("foo.*", "bar.foo"));
             fail();
         } catch (IllegalArgumentException ex) {
             assertEquals("filter [bar.foo] has already been registered", ex.getMessage());
         }
+        SettingsModule module = new SettingsModule(settings, Arrays.asList(Setting.boolSetting("foo.bar", true, Property.NodeScope),
+            Setting.boolSetting("bar.foo", true, Property.NodeScope, Property.Filtered),
+            Setting.boolSetting("bar.baz", true, Property.NodeScope)), Arrays.asList("foo.*"));
         assertInstanceBinding(module, Settings.class, (s) -> s == settings);
         assertInstanceBinding(module, SettingsFilter.class, (s) -> s.filter(settings).getAsMap().size() == 1);
         assertInstanceBinding(module, SettingsFilter.class, (s) -> s.filter(settings).getAsMap().containsKey("bar.baz"));
@@ -180,29 +173,30 @@ public class SettingsModuleTests extends ModuleTestCase {
     }
 
     public void testMutuallyExclusiveScopes() {
-        new SettingsModule(Settings.EMPTY).registerSetting(Setting.simpleString("foo.bar", Property.NodeScope));
-        new SettingsModule(Settings.EMPTY).registerSetting(Setting.simpleString("foo.bar", Property.IndexScope));
+        new SettingsModule(Settings.EMPTY, Setting.simpleString("foo.bar", Property.NodeScope));
+        new SettingsModule(Settings.EMPTY, Setting.simpleString("index.foo.bar", Property.IndexScope));
 
         // Those should fail
         try {
-            new SettingsModule(Settings.EMPTY).registerSetting(Setting.simpleString("foo.bar"));
+            new SettingsModule(Settings.EMPTY, Setting.simpleString("foo.bar"));
             fail("No scope should fail");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), containsString("No scope found for setting"));
         }
         // Some settings have both scopes - that's fine too if they have per-node defaults
-        SettingsModule module = new SettingsModule(Settings.EMPTY);
-        module.registerSetting(Setting.simpleString("foo.bar", Property.IndexScope, Property.NodeScope));
-
         try {
-            module.registerSetting(Setting.simpleString("foo.bar", Property.NodeScope));
+            new SettingsModule(Settings.EMPTY,
+                Setting.simpleString("foo.bar", Property.IndexScope, Property.NodeScope),
+                Setting.simpleString("foo.bar", Property.NodeScope));
             fail("already registered");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), containsString("Cannot register setting [foo.bar] twice"));
         }
 
         try {
-            module.registerSetting(Setting.simpleString("foo.bar", Property.IndexScope));
+            new SettingsModule(Settings.EMPTY,
+                Setting.simpleString("foo.bar", Property.IndexScope, Property.NodeScope),
+                Setting.simpleString("foo.bar", Property.IndexScope));
             fail("already registered");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), containsString("Cannot register setting [foo.bar] twice"));
@@ -211,10 +205,28 @@ public class SettingsModuleTests extends ModuleTestCase {
 
     public void testOldMaxClauseCountSetting() {
             Settings settings = Settings.builder().put("index.query.bool.max_clause_count", 1024).build();
-            SettingsModule module = new SettingsModule(settings);
             IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
-                () -> assertInstanceBinding(module, Settings.class, (s) -> s == settings));
+                () -> new SettingsModule(settings));
             assertEquals("unknown setting [index.query.bool.max_clause_count] did you mean [indices.query.bool.max_clause_count]?",
                 ex.getMessage());
     }
+
+    public void testRegisterShared() {
+        Property scope = randomFrom(Property.NodeScope, Property.IndexScope);
+        expectThrows(IllegalArgumentException.class, () ->
+            new SettingsModule(Settings.EMPTY,
+                Setting.simpleString("index.foo.bar", scope), Setting.simpleString("index.foo.bar", scope))
+        );
+        expectThrows(IllegalArgumentException.class, () ->
+            new SettingsModule(Settings.EMPTY,
+                Setting.simpleString("index.foo.bar", scope, Property.Shared), Setting.simpleString("index.foo.bar", scope))
+        );
+        expectThrows(IllegalArgumentException.class, () ->
+            new SettingsModule(Settings.EMPTY,
+                Setting.simpleString("index.foo.bar", scope), Setting.simpleString("index.foo.bar", scope, Property.Shared))
+        );
+        new SettingsModule(Settings.EMPTY,
+            Setting.simpleString("index.foo.bar", scope, Property.Shared),
+            Setting.simpleString("index.foo.bar", scope, Property.Shared));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java b/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java
index 07afef1d8ae..17ee96e91a7 100644
--- a/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java
+++ b/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java
@@ -23,13 +23,14 @@ import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 
 import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -62,8 +63,9 @@ public class SettingsListenerIT extends ESIntegTestCase {
             return "Settings Listenern Plugin";
         }
 
-        public void onModule(SettingsModule settingsModule) {
-            settingsModule.registerSetting(SettingsTestingService.VALUE);
+        @Override
+        public List<Setting<?>> getSettings() {
+            return Arrays.asList(SettingsTestingService.VALUE);
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
index 0256fab1b7f..6eda8dbb0b1 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java
@@ -50,7 +50,9 @@ import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 
+import java.util.Arrays;
 import java.util.Collection;
+import java.util.List;
 import java.util.function.Function;
 
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
@@ -606,10 +608,9 @@ public class IndicesOptionsIntegrationIT extends ESIntegTestCase {
             new Setting<>("index.e", "", Function.identity(), Property.IndexScope);
 
 
-        public void onModule(SettingsModule module) {
-            module.registerSetting(INDEX_A);
-            module.registerSetting(INDEX_C);
-            module.registerSetting(INDEX_E);
+        @Override
+        public List<Setting<?>> getSettings() {
+            return Arrays.asList(INDEX_A, INDEX_C, INDEX_E);
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java
index 19242b1f90b..41e15a3a351 100644
--- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/RandomExceptionCircuitBreakerIT.java
@@ -49,6 +49,7 @@ import org.elasticsearch.test.engine.ThrowingLeafReaderWrapper;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.List;
 import java.util.Random;
 import java.util.concurrent.ExecutionException;
 
@@ -209,9 +210,9 @@ public class RandomExceptionCircuitBreakerIT extends ESIntegTestCase {
                 return "a mock reader wrapper that throws random exceptions for testing";
             }
 
-            public void onModule(SettingsModule module) {
-                module.registerSetting(EXCEPTION_TOP_LEVEL_RATIO_SETTING);
-                module.registerSetting(EXCEPTION_LOW_LEVEL_RATIO_SETTING);
+            @Override
+            public List<Setting<?>> getSettings() {
+                return Arrays.asList(EXCEPTION_TOP_LEVEL_RATIO_SETTING, EXCEPTION_LOW_LEVEL_RATIO_SETTING);
             }
 
             public void onModule(MockEngineFactoryPlugin.MockEngineReaderModule module) {
diff --git a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
index c35fb624e1e..65c0a4fddef 100644
--- a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
@@ -24,20 +24,22 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Injector;
 import org.elasticsearch.common.inject.ModulesBuilder;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.EnvironmentModule;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.threadpool.ThreadPoolModule;
 import org.elasticsearch.watcher.ResourceWatcherService;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -51,14 +53,14 @@ public class NativeScriptTests extends ESTestCase {
                 .put("node.name", "testNativeScript")
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                 .build();
-        SettingsModule settingsModule = new SettingsModule(settings);
         ScriptModule scriptModule = new ScriptModule(new NativeScriptEngineService(settings,
             Collections.singletonMap("my", new MyNativeScriptFactory())));
-        scriptModule.prepareSettings(settingsModule);
+        List<Setting<?>> scriptSettings = scriptModule.getSettings();
+        scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
+        SettingsModule settingsModule = new SettingsModule(settings, scriptSettings, Collections.emptyList());
         final ThreadPool threadPool = new ThreadPool(settings);
         Injector injector = new ModulesBuilder().add(
-                new EnvironmentModule(new Environment(settings)),
-                new ThreadPoolModule(threadPool),
+                new EnvironmentModule(new Environment(settings), threadPool),
                 new SettingsModule(settings),
                 scriptModule).createInjector();
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
index 3b80cb5679f..f052977873a 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
@@ -29,9 +29,9 @@ import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.Injector;
 import org.elasticsearch.common.inject.ModulesBuilder;
-import org.elasticsearch.common.inject.multibindings.Multibinder;
 import org.elasticsearch.common.inject.util.Providers;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -46,31 +46,21 @@ import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.script.MockScriptEngine;
-import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.ScriptContextRegistry;
-import org.elasticsearch.script.ScriptEngineRegistry;
-import org.elasticsearch.script.ScriptEngineService;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.script.ScriptSettings;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.threadpool.ThreadPoolModule;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Random;
-import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -115,12 +105,12 @@ public class AggregatorParsingTests extends ESTestCase {
         final ClusterService clusterService = createClusterService(threadPool);
         setState(clusterService, new ClusterState.Builder(clusterService.state()).metaData(new MetaData.Builder()
                 .put(new IndexMetaData.Builder(index.getName()).settings(indexSettings).numberOfShards(1).numberOfReplicas(0))));
-        SettingsModule settingsModule = new SettingsModule(settings);
-        settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
         ScriptModule scriptModule = newTestScriptModule();
-        scriptModule.prepareSettings(settingsModule);
-        injector = new ModulesBuilder().add(new EnvironmentModule(new Environment(settings)), settingsModule,
-                new ThreadPoolModule(threadPool), scriptModule, new IndicesModule() {
+        List<Setting<?>> scriptSettings = scriptModule.getSettings();
+        scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
+        SettingsModule settingsModule = new SettingsModule(settings, scriptSettings, Collections.emptyList());
+        injector = new ModulesBuilder().add(new EnvironmentModule(new Environment(settings), threadPool), settingsModule
+            , scriptModule, new IndicesModule() {
 
                     @Override
                     protected void configure() {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
index c73df1bbf06..5cb267315a5 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
@@ -28,12 +28,12 @@ import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.Injector;
 import org.elasticsearch.common.inject.ModulesBuilder;
-import org.elasticsearch.common.inject.multibindings.Multibinder;
 import org.elasticsearch.common.inject.util.Providers;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -50,30 +50,20 @@ import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.script.MockScriptEngine;
-import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.ScriptContextRegistry;
-import org.elasticsearch.script.ScriptEngineRegistry;
-import org.elasticsearch.script.ScriptEngineService;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.script.ScriptSettings;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.threadpool.ThreadPoolModule;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 
 import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
 import static org.elasticsearch.test.ClusterServiceUtils.setState;
@@ -129,14 +119,13 @@ public abstract class BaseAggregationTestCase<AB extends AbstractAggregationBuil
         final ClusterService clusterService = createClusterService(threadPool);
         setState(clusterService, new ClusterState.Builder(clusterService.state()).metaData(new MetaData.Builder()
                 .put(new IndexMetaData.Builder(index.getName()).settings(indexSettings).numberOfShards(1).numberOfReplicas(0))));
-        SettingsModule settingsModule = new SettingsModule(settings);
-        settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
         ScriptModule scriptModule = newTestScriptModule();
-        scriptModule.prepareSettings(settingsModule);
+        List<Setting<?>> scriptSettings = scriptModule.getSettings();
+        scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
+        SettingsModule settingsModule = new SettingsModule(settings, scriptSettings, Collections.emptyList());
         injector = new ModulesBuilder().add(
-                new EnvironmentModule(new Environment(settings)),
+                new EnvironmentModule(new Environment(settings), threadPool),
                 settingsModule,
-                new ThreadPoolModule(threadPool),
                 scriptModule,
                 new IndicesModule() {
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
index 2f2354b73a1..2d3a86fb4be 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
@@ -28,12 +28,12 @@ import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.Injector;
 import org.elasticsearch.common.inject.ModulesBuilder;
-import org.elasticsearch.common.inject.multibindings.Multibinder;
 import org.elasticsearch.common.inject.util.Providers;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -44,37 +44,27 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.EnvironmentModule;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.script.MockScriptEngine;
-import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.ScriptContextRegistry;
-import org.elasticsearch.script.ScriptEngineRegistry;
-import org.elasticsearch.script.ScriptEngineService;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.script.ScriptSettings;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregatorBuilder;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.threadpool.ThreadPoolModule;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 
 import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
 import static org.elasticsearch.test.ClusterServiceUtils.setState;
@@ -129,14 +119,13 @@ public abstract class BasePipelineAggregationTestCase<AF extends AbstractPipelin
         final ClusterService clusterService = createClusterService(threadPool);
         setState(clusterService, new ClusterState.Builder(clusterService.state()).metaData(new MetaData.Builder()
                 .put(new IndexMetaData.Builder(index.getName()).settings(indexSettings).numberOfShards(1).numberOfReplicas(0))));
-        SettingsModule settingsModule = new SettingsModule(settings);
-        settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
         ScriptModule scriptModule = newTestScriptModule();
-        scriptModule.prepareSettings(settingsModule);
+        List<Setting<?>> scriptSettings = scriptModule.getSettings();
+        scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
+        SettingsModule settingsModule = new SettingsModule(settings, scriptSettings, Collections.emptyList());
         injector = new ModulesBuilder().add(
-                new EnvironmentModule(new Environment(settings)),
+                new EnvironmentModule(new Environment(settings),threadPool),
                 settingsModule,
-                new ThreadPoolModule(threadPool),
                 scriptModule,
                 new IndicesModule() {
 
diff --git a/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java b/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java
index 7ddb6413364..0b3ccf165c3 100644
--- a/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/basic/SearchWithRandomExceptionsIT.java
@@ -44,7 +44,9 @@ import org.elasticsearch.test.engine.MockEngineSupport;
 import org.elasticsearch.test.engine.ThrowingLeafReaderWrapper;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collection;
+import java.util.List;
 import java.util.Random;
 import java.util.concurrent.ExecutionException;
 
@@ -160,9 +162,9 @@ public class SearchWithRandomExceptionsIT extends ESIntegTestCase {
             public String name() {
                 return "random-exception-reader-wrapper";
             }
-            public void onModule(SettingsModule module) {
-                module.registerSetting(EXCEPTION_TOP_LEVEL_RATIO_SETTING);
-                module.registerSetting(EXCEPTION_LOW_LEVEL_RATIO_SETTING);
+            @Override
+            public List<Setting<?>> getSettings() {
+                return Arrays.asList(EXCEPTION_TOP_LEVEL_RATIO_SETTING, EXCEPTION_LOW_LEVEL_RATIO_SETTING);
             }
             @Override
             public String description() {
diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index 641cb2c1b85..9bb39d00f4a 100644
--- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -30,12 +30,12 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.Injector;
 import org.elasticsearch.common.inject.ModulesBuilder;
-import org.elasticsearch.common.inject.multibindings.Multibinder;
 import org.elasticsearch.common.inject.util.Providers;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.text.Text;
@@ -54,15 +54,9 @@ import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.ScriptContextRegistry;
-import org.elasticsearch.script.ScriptEngineRegistry;
-import org.elasticsearch.script.ScriptEngineService;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.script.ScriptSettings;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
@@ -85,16 +79,13 @@ import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.threadpool.ThreadPoolModule;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
@@ -136,13 +127,12 @@ public class SearchSourceBuilderTests extends ESTestCase {
         final ClusterService clusterService = createClusterService(threadPool);
         setState(clusterService, new ClusterState.Builder(clusterService.state()).metaData(new MetaData.Builder()
                 .put(new IndexMetaData.Builder(index.getName()).settings(indexSettings).numberOfShards(1).numberOfReplicas(0))));
-        SettingsModule settingsModule = new SettingsModule(settings);
-        settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
         ScriptModule scriptModule = newTestScriptModule();
-        scriptModule.prepareSettings(settingsModule);
+        List<Setting<?>> scriptSettings = scriptModule.getSettings();
+        scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
+        SettingsModule settingsModule = new SettingsModule(settings, scriptSettings, Collections.emptyList());
         injector = new ModulesBuilder().add(
-                new EnvironmentModule(new Environment(settings)), settingsModule,
-                new ThreadPoolModule(threadPool),
+                new EnvironmentModule(new Environment(settings), threadPool), settingsModule,
                 scriptModule, new IndicesModule() {
                     @Override
                     protected void configure() {
diff --git a/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java b/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
index 8b02f90f51c..ceaaa8aa470 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
@@ -48,6 +48,7 @@ import java.io.UnsupportedEncodingException;
 import java.nio.file.Path;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -76,9 +77,9 @@ public class MockRepository extends FsRepository {
             repositoriesModule.registerRepository("mock", MockRepository.class, BlobStoreIndexShardRepository.class);
         }
 
-        public void onModule(SettingsModule module) {
-            module.registerSetting(USERNAME_SETTING);
-            module.registerSetting(PASSWORD_SETTING);
+        @Override
+        public List<Setting<?>> getSettings() {
+            return Arrays.asList(USERNAME_SETTING, PASSWORD_SETTING);
         }
     }
 
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
index 54068d1921c..a16175689ff 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
@@ -32,6 +32,7 @@ import org.elasticsearch.common.inject.ModulesBuilder;
 import org.elasticsearch.common.inject.multibindings.Multibinder;
 import org.elasticsearch.common.inject.util.Providers;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -67,13 +68,13 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.threadpool.ThreadPoolModule;
 import org.junit.After;
 import org.junit.Before;
 
 import java.io.IOException;
 import java.lang.reflect.Proxy;
 import java.util.Collections;
+import java.util.List;
 import java.util.function.Supplier;
 
 import static org.hamcrest.Matchers.containsString;
@@ -102,16 +103,15 @@ public class TemplateQueryParserTests extends ESTestCase {
                 });
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
         Index index = idxSettings.getIndex();
-        SettingsModule settingsModule = new SettingsModule(settings);
         // TODO: make this use a mock engine instead of mustache and it will no longer be messy!
         ScriptModule scriptModule = new ScriptModule(new MustacheScriptEngineService(settings));
-        scriptModule.prepareSettings(settingsModule);
-        settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
+        List<Setting<?>> scriptSettings = scriptModule.getSettings();
+        scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
+        SettingsModule settingsModule = new SettingsModule(settings, scriptSettings, Collections.emptyList());
         final ThreadPool threadPool = new ThreadPool(settings);
         injector = new ModulesBuilder().add(
-                new EnvironmentModule(new Environment(settings)),
+                new EnvironmentModule(new Environment(settings), threadPool),
                 settingsModule,
-                new ThreadPoolModule(threadPool),
                 new SearchModule(settings, new NamedWriteableRegistry()) {
                     @Override
                     protected void configureSearch() {
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
index dfc11804dd9..b2064506059 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
@@ -23,6 +23,7 @@ import org.elasticsearch.action.ActionModule;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.indices.IndicesModule;
@@ -31,6 +32,8 @@ import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.highlight.HighlightPhase;
 
+import java.util.Arrays;
+import java.util.List;
 import java.util.Optional;
 
 public class PercolatorPlugin extends Plugin {
@@ -76,8 +79,9 @@ public class PercolatorPlugin extends Plugin {
         module.registerFetchSubPhase(new PercolatorHighlightSubFetchPhase(settings, module.getHighlighters()));
     }
 
-    public void onModule(SettingsModule module) {
-        module.registerSetting(PercolatorFieldMapper.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING);
+    @Override
+    public List<Setting<?>> getSettings() {
+        return Arrays.asList(PercolatorFieldMapper.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING);
     }
 
     static boolean transportClientMode(Settings settings) {
diff --git a/plugins/discovery-azure/src/main/java/org/elasticsearch/plugin/discovery/azure/AzureDiscoveryPlugin.java b/plugins/discovery-azure/src/main/java/org/elasticsearch/plugin/discovery/azure/AzureDiscoveryPlugin.java
index 757ff359d42..7ee7d15d7ce 100644
--- a/plugins/discovery-azure/src/main/java/org/elasticsearch/plugin/discovery/azure/AzureDiscoveryPlugin.java
+++ b/plugins/discovery-azure/src/main/java/org/elasticsearch/plugin/discovery/azure/AzureDiscoveryPlugin.java
@@ -24,6 +24,7 @@ import org.elasticsearch.cloud.azure.management.AzureComputeService;
 import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.discovery.DiscoveryModule;
@@ -31,8 +32,10 @@ import org.elasticsearch.discovery.azure.AzureUnicastHostsProvider;
 import org.elasticsearch.discovery.zen.ZenDiscovery;
 import org.elasticsearch.plugins.Plugin;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 
 public class AzureDiscoveryPlugin extends Plugin {
 
@@ -67,16 +70,18 @@ public class AzureDiscoveryPlugin extends Plugin {
         }
     }
 
-    public void onModule(SettingsModule settingsModule) {
-        settingsModule.registerSetting(AzureComputeService.Discovery.REFRESH_SETTING);
-        settingsModule.registerSetting(AzureComputeService.Management.KEYSTORE_PASSWORD_SETTING);
-        settingsModule.registerSetting(AzureComputeService.Management.KEYSTORE_PATH_SETTING);
-        settingsModule.registerSetting(AzureComputeService.Management.KEYSTORE_TYPE_SETTING);
-        settingsModule.registerSetting(AzureComputeService.Management.SUBSCRIPTION_ID_SETTING);
-        settingsModule.registerSetting(AzureComputeService.Management.SERVICE_NAME_SETTING);
-        settingsModule.registerSetting(AzureComputeService.Discovery.HOST_TYPE_SETTING);
-        settingsModule.registerSetting(AzureComputeService.Discovery.DEPLOYMENT_NAME_SETTING);
-        settingsModule.registerSetting(AzureComputeService.Discovery.DEPLOYMENT_SLOT_SETTING);
-        settingsModule.registerSetting(AzureComputeService.Discovery.ENDPOINT_NAME_SETTING);
+    @Override
+    public List<Setting<?>> getSettings() {
+        return Arrays.asList(AzureComputeService.Discovery.REFRESH_SETTING,
+                            AzureComputeService.Management.KEYSTORE_PASSWORD_SETTING,
+                            AzureComputeService.Management.KEYSTORE_PATH_SETTING,
+                            AzureComputeService.Management.KEYSTORE_TYPE_SETTING,
+                            AzureComputeService.Management.SUBSCRIPTION_ID_SETTING,
+                            AzureComputeService.Management.SERVICE_NAME_SETTING,
+                            AzureComputeService.Discovery.HOST_TYPE_SETTING,
+                            AzureComputeService.Discovery.DEPLOYMENT_NAME_SETTING,
+                            AzureComputeService.Discovery.DEPLOYMENT_SLOT_SETTING,
+                            AzureComputeService.Discovery.ENDPOINT_NAME_SETTING);
     }
+
 }
diff --git a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryClusterFormationTests.java b/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryClusterFormationTests.java
index 489f7043b83..0a1973ddda3 100644
--- a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryClusterFormationTests.java
+++ b/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryClusterFormationTests.java
@@ -28,8 +28,8 @@ import org.elasticsearch.cloud.azure.management.AzureComputeService;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.node.Node;
@@ -57,6 +57,7 @@ import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.KeyStore;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -83,8 +84,9 @@ public class AzureDiscoveryClusterFormationTests extends ESIntegTestCase {
             return AzureDiscoveryClusterFormationTests.class.getName();
         }
 
-        public void onModule(SettingsModule settingsModule) {
-            settingsModule.registerSetting(AzureComputeService.Management.ENDPOINT_SETTING);
+        @Override
+        public List<Setting<?>> getSettings() {
+            return Arrays.asList(AzureComputeService.Management.ENDPOINT_SETTING);
         }
     }
 
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java
index 8cfe6c43108..139b78226cf 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java
@@ -42,43 +42,44 @@ public interface AwsEc2Service {
      * cloud.aws.access_key: AWS Access key. Shared with repository-s3 plugin
      */
     Setting<String> KEY_SETTING =
-        Setting.simpleString("cloud.aws.access_key", Property.NodeScope, Property.Filtered);
+        Setting.simpleString("cloud.aws.access_key", Property.NodeScope, Property.Filtered, Property.Shared);
     /**
      * cloud.aws.secret_key: AWS Secret key. Shared with repository-s3 plugin
      */
     Setting<String> SECRET_SETTING =
-        Setting.simpleString("cloud.aws.secret_key", Property.NodeScope, Property.Filtered);
+        Setting.simpleString("cloud.aws.secret_key", Property.NodeScope, Property.Filtered, Property.Shared);
     /**
      * cloud.aws.protocol: Protocol for AWS API: http or https. Defaults to https. Shared with repository-s3 plugin
      */
     Setting<Protocol> PROTOCOL_SETTING = new Setting<>("cloud.aws.protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)),
-        Property.NodeScope);
+        Property.NodeScope, Property.Shared);
     /**
      * cloud.aws.proxy.host: In case of proxy, define its hostname/IP. Shared with repository-s3 plugin
      */
-    Setting<String> PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host", Property.NodeScope);
+    Setting<String> PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host", Property.NodeScope, Property.Shared);
     /**
      * cloud.aws.proxy.port: In case of proxy, define its port. Defaults to 80. Shared with repository-s3 plugin
      */
-    Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16, Property.NodeScope);
+    Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16, Property.NodeScope,
+        Property.Shared);
     /**
      * cloud.aws.proxy.username: In case of proxy with auth, define the username. Shared with repository-s3 plugin
      */
-    Setting<String> PROXY_USERNAME_SETTING = Setting.simpleString("cloud.aws.proxy.username", Property.NodeScope);
+    Setting<String> PROXY_USERNAME_SETTING = Setting.simpleString("cloud.aws.proxy.username", Property.NodeScope, Property.Shared);
     /**
      * cloud.aws.proxy.password: In case of proxy with auth, define the password. Shared with repository-s3 plugin
      */
     Setting<String> PROXY_PASSWORD_SETTING =
-        Setting.simpleString("cloud.aws.proxy.password", Property.NodeScope, Property.Filtered);
+        Setting.simpleString("cloud.aws.proxy.password", Property.NodeScope, Property.Filtered, Property.Shared);
     /**
      * cloud.aws.signer: If you are using an old AWS API version, you can define a Signer. Shared with repository-s3 plugin
      */
-    Setting<String> SIGNER_SETTING = Setting.simpleString("cloud.aws.signer", Property.NodeScope);
+    Setting<String> SIGNER_SETTING = Setting.simpleString("cloud.aws.signer", Property.NodeScope, Property.Shared);
     /**
      * cloud.aws.region: Region. Shared with repository-s3 plugin
      */
     Setting<String> REGION_SETTING =
-        new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope);
+        new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Shared);
 
     /**
      * Defines specific ec2 settings starting with cloud.aws.ec2.
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
index 21e31b9f076..6a61b610ba6 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
@@ -38,7 +38,9 @@ import org.elasticsearch.plugins.Plugin;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
+import java.util.List;
 
 /**
  *
@@ -104,48 +106,38 @@ public class Ec2DiscoveryPlugin extends Plugin {
         discoveryModule.addUnicastHostProvider(EC2, AwsEc2UnicastHostsProvider.class);
     }
 
-    public void onModule(SettingsModule settingsModule) {
+    @Override
+    public List<Setting<?>> getSettings() {
+        return Arrays.asList(
         // Register global cloud aws settings: cloud.aws (might have been registered in ec2 plugin)
-        registerSettingIfMissing(settingsModule, AwsEc2Service.KEY_SETTING);
-        registerSettingIfMissing(settingsModule, AwsEc2Service.SECRET_SETTING);
-        registerSettingIfMissing(settingsModule, AwsEc2Service.PROTOCOL_SETTING);
-        registerSettingIfMissing(settingsModule, AwsEc2Service.PROXY_HOST_SETTING);
-        registerSettingIfMissing(settingsModule, AwsEc2Service.PROXY_PORT_SETTING);
-        registerSettingIfMissing(settingsModule, AwsEc2Service.PROXY_USERNAME_SETTING);
-        registerSettingIfMissing(settingsModule, AwsEc2Service.PROXY_PASSWORD_SETTING);
-        registerSettingIfMissing(settingsModule, AwsEc2Service.SIGNER_SETTING);
-        registerSettingIfMissing(settingsModule, AwsEc2Service.REGION_SETTING);
-
+        AwsEc2Service.KEY_SETTING,
+        AwsEc2Service.SECRET_SETTING,
+        AwsEc2Service.PROTOCOL_SETTING,
+        AwsEc2Service.PROXY_HOST_SETTING,
+        AwsEc2Service.PROXY_PORT_SETTING,
+        AwsEc2Service.PROXY_USERNAME_SETTING,
+        AwsEc2Service.PROXY_PASSWORD_SETTING,
+        AwsEc2Service.SIGNER_SETTING,
+        AwsEc2Service.REGION_SETTING,
         // Register EC2 specific settings: cloud.aws.ec2
-        settingsModule.registerSetting(AwsEc2Service.CLOUD_EC2.KEY_SETTING);
-        settingsModule.registerSetting(AwsEc2Service.CLOUD_EC2.SECRET_SETTING);
-        settingsModule.registerSetting(AwsEc2Service.CLOUD_EC2.PROTOCOL_SETTING);
-        settingsModule.registerSetting(AwsEc2Service.CLOUD_EC2.PROXY_HOST_SETTING);
-        settingsModule.registerSetting(AwsEc2Service.CLOUD_EC2.PROXY_PORT_SETTING);
-        settingsModule.registerSetting(AwsEc2Service.CLOUD_EC2.PROXY_USERNAME_SETTING);
-        settingsModule.registerSetting(AwsEc2Service.CLOUD_EC2.PROXY_PASSWORD_SETTING);
-        settingsModule.registerSetting(AwsEc2Service.CLOUD_EC2.SIGNER_SETTING);
-        settingsModule.registerSetting(AwsEc2Service.CLOUD_EC2.REGION_SETTING);
-        settingsModule.registerSetting(AwsEc2Service.CLOUD_EC2.ENDPOINT_SETTING);
-
+        AwsEc2Service.CLOUD_EC2.KEY_SETTING,
+        AwsEc2Service.CLOUD_EC2.SECRET_SETTING,
+        AwsEc2Service.CLOUD_EC2.PROTOCOL_SETTING,
+        AwsEc2Service.CLOUD_EC2.PROXY_HOST_SETTING,
+        AwsEc2Service.CLOUD_EC2.PROXY_PORT_SETTING,
+        AwsEc2Service.CLOUD_EC2.PROXY_USERNAME_SETTING,
+        AwsEc2Service.CLOUD_EC2.PROXY_PASSWORD_SETTING,
+        AwsEc2Service.CLOUD_EC2.SIGNER_SETTING,
+        AwsEc2Service.CLOUD_EC2.REGION_SETTING,
+        AwsEc2Service.CLOUD_EC2.ENDPOINT_SETTING,
         // Register EC2 discovery settings: discovery.ec2
-        settingsModule.registerSetting(AwsEc2Service.DISCOVERY_EC2.HOST_TYPE_SETTING);
-        settingsModule.registerSetting(AwsEc2Service.DISCOVERY_EC2.ANY_GROUP_SETTING);
-        settingsModule.registerSetting(AwsEc2Service.DISCOVERY_EC2.GROUPS_SETTING);
-        settingsModule.registerSetting(AwsEc2Service.DISCOVERY_EC2.AVAILABILITY_ZONES_SETTING);
-        settingsModule.registerSetting(AwsEc2Service.DISCOVERY_EC2.NODE_CACHE_TIME_SETTING);
-        settingsModule.registerSetting(AwsEc2Service.DISCOVERY_EC2.TAG_SETTING);
-
+        AwsEc2Service.DISCOVERY_EC2.HOST_TYPE_SETTING,
+        AwsEc2Service.DISCOVERY_EC2.ANY_GROUP_SETTING,
+        AwsEc2Service.DISCOVERY_EC2.GROUPS_SETTING,
+        AwsEc2Service.DISCOVERY_EC2.AVAILABILITY_ZONES_SETTING,
+        AwsEc2Service.DISCOVERY_EC2.NODE_CACHE_TIME_SETTING,
+        AwsEc2Service.DISCOVERY_EC2.TAG_SETTING,
         // Register cloud node settings: cloud.node
-        settingsModule.registerSetting(AwsEc2Service.AUTO_ATTRIBUTE_SETTING);
-    }
-
-    /**
-     * We manage potential duplicates between s3 and ec2 plugins (cloud.aws.xxx)
-     */
-    private void registerSettingIfMissing(SettingsModule settingsModule, Setting<?> setting) {
-        if (settingsModule.exists(setting) == false) {
-            settingsModule.registerSetting(setting);
-        }
+        AwsEc2Service.AUTO_ATTRIBUTE_SETTING);
     }
 }
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java
index 2b92c4fd8c1..c48a8b409e5 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/plugin/discovery/gce/GceDiscoveryPlugin.java
@@ -28,6 +28,7 @@ import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.discovery.DiscoveryModule;
@@ -37,8 +38,10 @@ import org.elasticsearch.plugins.Plugin;
 
 import java.security.AccessController;
 import java.security.PrivilegedAction;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 
 public class GceDiscoveryPlugin extends Plugin {
 
@@ -100,13 +103,15 @@ public class GceDiscoveryPlugin extends Plugin {
             discoveryModule.addUnicastHostProvider(GCE, GceUnicastHostsProvider.class);
     }
 
-    public void onModule(SettingsModule settingsModule) {
+    @Override
+    public List<Setting<?>> getSettings() {
+        return Arrays.asList(
         // Register GCE settings
-        settingsModule.registerSetting(GceComputeService.PROJECT_SETTING);
-        settingsModule.registerSetting(GceComputeService.ZONE_SETTING);
-        settingsModule.registerSetting(GceUnicastHostsProvider.TAGS_SETTING);
-        settingsModule.registerSetting(GceComputeService.REFRESH_SETTING);
-        settingsModule.registerSetting(GceComputeService.RETRY_SETTING);
-        settingsModule.registerSetting(GceComputeService.MAX_WAIT_SETTING);
+        GceComputeService.PROJECT_SETTING,
+        GceComputeService.ZONE_SETTING,
+        GceUnicastHostsProvider.TAGS_SETTING,
+        GceComputeService.REFRESH_SETTING,
+        GceComputeService.RETRY_SETTING,
+        GceComputeService.MAX_WAIT_SETTING);
     }
 }
diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java
index f6a850d1d23..0b977d7896e 100644
--- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java
+++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoverTests.java
@@ -28,6 +28,7 @@ import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.plugin.discovery.gce.GceDiscoveryPlugin;
@@ -48,6 +49,7 @@ import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.KeyStore;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -74,10 +76,10 @@ public class GceDiscoverTests extends ESIntegTestCase {
             return "GceDiscoverTests";
         }
 
-        public void onModule(SettingsModule module) {
-            module.registerSetting(GceComputeServiceImpl.GCE_HOST);
-            module.registerSetting(GceComputeServiceImpl.GCE_ROOT_URL);
-            module.registerSetting(GceComputeServiceImpl.GCE_VALIDATE_CERTIFICATES);
+        @Override
+        public List<Setting<?>> getSettings() {
+            return Arrays.asList(GceComputeServiceImpl.GCE_HOST, GceComputeServiceImpl.GCE_ROOT_URL,
+                GceComputeServiceImpl.GCE_VALIDATE_CERTIFICATES);
         }
     }
 
diff --git a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java
index 7a77510447d..13f10fcdbe0 100644
--- a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java
+++ b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java
@@ -22,10 +22,14 @@ package org.elasticsearch.mapper.attachments;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.plugins.Plugin;
 
+import java.util.Arrays;
+import java.util.List;
+
 public class MapperAttachmentsPlugin extends Plugin {
 
 
@@ -42,11 +46,13 @@ public class MapperAttachmentsPlugin extends Plugin {
         return "Adds the attachment type allowing to parse difference attachment formats";
     }
 
-    public void onModule(SettingsModule settingsModule) {
+    @Override
+    public List<Setting<?>> getSettings() {
         deprecationLogger.deprecated("[mapper-attachments] plugin has been deprecated and will be replaced by [ingest-attachment] plugin.");
-        settingsModule.registerSetting(AttachmentMapper.INDEX_ATTACHMENT_DETECT_LANGUAGE_SETTING);
-        settingsModule.registerSetting(AttachmentMapper.INDEX_ATTACHMENT_IGNORE_ERRORS_SETTING);
-        settingsModule.registerSetting(AttachmentMapper.INDEX_ATTACHMENT_INDEXED_CHARS_SETTING);
+
+        return Arrays.asList(AttachmentMapper.INDEX_ATTACHMENT_DETECT_LANGUAGE_SETTING,
+        AttachmentMapper.INDEX_ATTACHMENT_IGNORE_ERRORS_SETTING,
+        AttachmentMapper.INDEX_ATTACHMENT_INDEXED_CHARS_SETTING);
     }
 
     public void onModule(IndicesModule indicesModule) {
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java b/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
index 3ce043500ae..f3e0c267ada 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
@@ -24,6 +24,7 @@ import org.elasticsearch.cloud.azure.storage.AzureStorageService;
 import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository;
@@ -31,8 +32,10 @@ import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.repositories.RepositoriesModule;
 import org.elasticsearch.repositories.azure.AzureRepository;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 
 /**
  *
@@ -67,16 +70,20 @@ public class AzureRepositoryPlugin extends Plugin {
         module.registerRepository(AzureRepository.TYPE, AzureRepository.class, BlobStoreIndexShardRepository.class);
     }
 
-    public void onModule(SettingsModule module) {
-        module.registerSetting(AzureStorageService.Storage.ACCOUNT_SETTING);
-        module.registerSetting(AzureStorageService.Storage.COMPRESS_SETTING);
-        module.registerSetting(AzureStorageService.Storage.CONTAINER_SETTING);
-        module.registerSetting(AzureStorageService.Storage.BASE_PATH_SETTING);
-        module.registerSetting(AzureStorageService.Storage.CHUNK_SIZE_SETTING);
-        module.registerSetting(AzureStorageService.Storage.LOCATION_MODE_SETTING);
+    @Override
+    public List<Setting<?>> getSettings() {
+        return Arrays.asList(AzureStorageService.Storage.ACCOUNT_SETTING,
+        AzureStorageService.Storage.COMPRESS_SETTING,
+        AzureStorageService.Storage.CONTAINER_SETTING,
+        AzureStorageService.Storage.BASE_PATH_SETTING,
+        AzureStorageService.Storage.CHUNK_SIZE_SETTING,
+        AzureStorageService.Storage.LOCATION_MODE_SETTING);
 
+    }
+
+    @Override
+    public List<String> getSettingsFilter() {
         // Cloud storage API settings using a pattern needed to be hidden
-        module.registerSettingsFilter(AzureStorageService.Storage.PREFIX + "*.account");
-        module.registerSettingsFilter(AzureStorageService.Storage.PREFIX + "*.key");
+        return Arrays.asList(AzureStorageService.Storage.PREFIX + "*.account", AzureStorageService.Storage.PREFIX + "*.key");
     }
 }
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java
index 6f215069564..db3bd617da4 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java
@@ -47,8 +47,7 @@ public class AzureStorageSettingsFilterTests extends ESTestCase {
 
     public void testSettingsFiltering() throws IOException {
         AzureRepositoryPlugin p = new AzureRepositoryPlugin(Settings.EMPTY);
-        SettingsModule module = new SettingsModule(Settings.EMPTY);
-        p.onModule(module);
+        SettingsModule module = new SettingsModule(Settings.EMPTY, p.getSettings(), p.getSettingsFilter());
         SettingsFilter settingsFilter = ModuleTestCase.bindAndGetInstance(module, SettingsFilter.class);
 
         // Test using direct filtering
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
index 427c454fa28..87c37b52e72 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
@@ -40,43 +40,44 @@ public interface AwsS3Service extends LifecycleComponent<AwsS3Service> {
      * cloud.aws.access_key: AWS Access key. Shared with discovery-ec2 plugin
      */
     Setting<String> KEY_SETTING =
-        Setting.simpleString("cloud.aws.access_key", Property.NodeScope, Property.Filtered);
+        Setting.simpleString("cloud.aws.access_key", Property.NodeScope, Property.Filtered, Property.Shared);
     /**
      * cloud.aws.secret_key: AWS Secret key. Shared with discovery-ec2 plugin
      */
     Setting<String> SECRET_SETTING =
-        Setting.simpleString("cloud.aws.secret_key", Property.NodeScope, Property.Filtered);
+        Setting.simpleString("cloud.aws.secret_key", Property.NodeScope, Property.Filtered, Property.Shared);
     /**
      * cloud.aws.protocol: Protocol for AWS API: http or https. Defaults to https. Shared with discovery-ec2 plugin
      */
     Setting<Protocol> PROTOCOL_SETTING = new Setting<>("cloud.aws.protocol", "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)),
-        Property.NodeScope);
+        Property.NodeScope, Property.Shared);
     /**
      * cloud.aws.proxy.host: In case of proxy, define its hostname/IP. Shared with discovery-ec2 plugin
      */
-    Setting<String> PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host", Property.NodeScope);
+    Setting<String> PROXY_HOST_SETTING = Setting.simpleString("cloud.aws.proxy.host", Property.NodeScope, Property.Shared);
     /**
      * cloud.aws.proxy.port: In case of proxy, define its port. Defaults to 80. Shared with discovery-ec2 plugin
      */
-    Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16, Property.NodeScope);
+    Setting<Integer> PROXY_PORT_SETTING = Setting.intSetting("cloud.aws.proxy.port", 80, 0, 1<<16, Property.NodeScope,
+        Property.Shared);
     /**
      * cloud.aws.proxy.username: In case of proxy with auth, define the username. Shared with discovery-ec2 plugin
      */
-    Setting<String> PROXY_USERNAME_SETTING = Setting.simpleString("cloud.aws.proxy.username", Property.NodeScope);
+    Setting<String> PROXY_USERNAME_SETTING = Setting.simpleString("cloud.aws.proxy.username", Property.NodeScope, Property.Shared);
     /**
      * cloud.aws.proxy.password: In case of proxy with auth, define the password. Shared with discovery-ec2 plugin
      */
     Setting<String> PROXY_PASSWORD_SETTING =
-        Setting.simpleString("cloud.aws.proxy.password", Property.NodeScope, Property.Filtered);
+        Setting.simpleString("cloud.aws.proxy.password", Property.NodeScope, Property.Filtered, Property.Shared);
     /**
      * cloud.aws.signer: If you are using an old AWS API version, you can define a Signer. Shared with discovery-ec2 plugin
      */
-    Setting<String> SIGNER_SETTING = Setting.simpleString("cloud.aws.signer", Property.NodeScope);
+    Setting<String> SIGNER_SETTING = Setting.simpleString("cloud.aws.signer", Property.NodeScope, Property.Shared);
     /**
      * cloud.aws.region: Region. Shared with discovery-ec2 plugin
      */
     Setting<String> REGION_SETTING =
-        new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope);
+        new Setting<>("cloud.aws.region", "", s -> s.toLowerCase(Locale.ROOT), Property.NodeScope, Property.Shared);
 
     /**
      * Defines specific s3 settings starting with cloud.aws.s3.
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
index d07d8c174c5..6d02e149530 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
@@ -34,8 +34,10 @@ import org.elasticsearch.repositories.s3.S3Repository;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 
 /**
  *
@@ -89,69 +91,61 @@ public class S3RepositoryPlugin extends Plugin {
         repositoriesModule.registerRepository(S3Repository.TYPE, S3Repository.class, BlobStoreIndexShardRepository.class);
     }
 
-    public void onModule(SettingsModule settingsModule) {
-        // Register global cloud aws settings: cloud.aws (might have been registered in ec2 plugin)
-        registerSettingIfMissing(settingsModule, AwsS3Service.KEY_SETTING);
-        registerSettingIfMissing(settingsModule, AwsS3Service.SECRET_SETTING);
-        registerSettingIfMissing(settingsModule, AwsS3Service.PROTOCOL_SETTING);
-        registerSettingIfMissing(settingsModule, AwsS3Service.PROXY_HOST_SETTING);
-        registerSettingIfMissing(settingsModule, AwsS3Service.PROXY_PORT_SETTING);
-        registerSettingIfMissing(settingsModule, AwsS3Service.PROXY_USERNAME_SETTING);
-        registerSettingIfMissing(settingsModule, AwsS3Service.PROXY_PASSWORD_SETTING);
-        registerSettingIfMissing(settingsModule, AwsS3Service.SIGNER_SETTING);
-        registerSettingIfMissing(settingsModule, AwsS3Service.REGION_SETTING);
+    @Override
+    public List<Setting<?>> getSettings() {
+        return Arrays.asList(        // Register global cloud aws settings: cloud.aws (might have been registered in ec2 plugin)
+        AwsS3Service.KEY_SETTING,
+        AwsS3Service.SECRET_SETTING,
+        AwsS3Service.PROTOCOL_SETTING,
+        AwsS3Service.PROXY_HOST_SETTING,
+        AwsS3Service.PROXY_PORT_SETTING,
+        AwsS3Service.PROXY_USERNAME_SETTING,
+        AwsS3Service.PROXY_PASSWORD_SETTING,
+        AwsS3Service.SIGNER_SETTING,
+        AwsS3Service.REGION_SETTING,
 
         // Register S3 specific settings: cloud.aws.s3
-        settingsModule.registerSetting(AwsS3Service.CLOUD_S3.KEY_SETTING);
-        settingsModule.registerSetting(AwsS3Service.CLOUD_S3.SECRET_SETTING);
-        settingsModule.registerSetting(AwsS3Service.CLOUD_S3.PROTOCOL_SETTING);
-        settingsModule.registerSetting(AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING);
-        settingsModule.registerSetting(AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING);
-        settingsModule.registerSetting(AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING);
-        settingsModule.registerSetting(AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING);
-        settingsModule.registerSetting(AwsS3Service.CLOUD_S3.SIGNER_SETTING);
-        settingsModule.registerSetting(AwsS3Service.CLOUD_S3.REGION_SETTING);
-        settingsModule.registerSetting(AwsS3Service.CLOUD_S3.ENDPOINT_SETTING);
+        AwsS3Service.CLOUD_S3.KEY_SETTING,
+        AwsS3Service.CLOUD_S3.SECRET_SETTING,
+        AwsS3Service.CLOUD_S3.PROTOCOL_SETTING,
+        AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING,
+        AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING,
+        AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING,
+        AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING,
+        AwsS3Service.CLOUD_S3.SIGNER_SETTING,
+        AwsS3Service.CLOUD_S3.REGION_SETTING,
+        AwsS3Service.CLOUD_S3.ENDPOINT_SETTING,
 
         // Register S3 repositories settings: repositories.s3
-        settingsModule.registerSetting(S3Repository.Repositories.KEY_SETTING);
-        settingsModule.registerSetting(S3Repository.Repositories.SECRET_SETTING);
-        settingsModule.registerSetting(S3Repository.Repositories.BUCKET_SETTING);
-        settingsModule.registerSetting(S3Repository.Repositories.REGION_SETTING);
-        settingsModule.registerSetting(S3Repository.Repositories.ENDPOINT_SETTING);
-        settingsModule.registerSetting(S3Repository.Repositories.PROTOCOL_SETTING);
-        settingsModule.registerSetting(S3Repository.Repositories.SERVER_SIDE_ENCRYPTION_SETTING);
-        settingsModule.registerSetting(S3Repository.Repositories.BUFFER_SIZE_SETTING);
-        settingsModule.registerSetting(S3Repository.Repositories.MAX_RETRIES_SETTING);
-        settingsModule.registerSetting(S3Repository.Repositories.CHUNK_SIZE_SETTING);
-        settingsModule.registerSetting(S3Repository.Repositories.COMPRESS_SETTING);
-        settingsModule.registerSetting(S3Repository.Repositories.STORAGE_CLASS_SETTING);
-        settingsModule.registerSetting(S3Repository.Repositories.CANNED_ACL_SETTING);
-        settingsModule.registerSetting(S3Repository.Repositories.BASE_PATH_SETTING);
+        S3Repository.Repositories.KEY_SETTING,
+        S3Repository.Repositories.SECRET_SETTING,
+        S3Repository.Repositories.BUCKET_SETTING,
+        S3Repository.Repositories.REGION_SETTING,
+        S3Repository.Repositories.ENDPOINT_SETTING,
+        S3Repository.Repositories.PROTOCOL_SETTING,
+        S3Repository.Repositories.SERVER_SIDE_ENCRYPTION_SETTING,
+        S3Repository.Repositories.BUFFER_SIZE_SETTING,
+        S3Repository.Repositories.MAX_RETRIES_SETTING,
+        S3Repository.Repositories.CHUNK_SIZE_SETTING,
+        S3Repository.Repositories.COMPRESS_SETTING,
+        S3Repository.Repositories.STORAGE_CLASS_SETTING,
+        S3Repository.Repositories.CANNED_ACL_SETTING,
+        S3Repository.Repositories.BASE_PATH_SETTING,
 
         // Register S3 single repository settings
-        settingsModule.registerSetting(S3Repository.Repository.KEY_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.SECRET_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.BUCKET_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.ENDPOINT_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.PROTOCOL_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.REGION_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.SERVER_SIDE_ENCRYPTION_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.BUFFER_SIZE_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.MAX_RETRIES_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.CHUNK_SIZE_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.COMPRESS_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.STORAGE_CLASS_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.CANNED_ACL_SETTING);
-        settingsModule.registerSetting(S3Repository.Repository.BASE_PATH_SETTING);
-    }
-
-    /**
-     * We manage potential duplicates between s3 and ec2 plugins (cloud.aws.xxx)
-     */
-    private void registerSettingIfMissing(SettingsModule settingsModule, Setting<?> setting) {
-        if (settingsModule.exists(setting) == false) {
-            settingsModule.registerSetting(setting);
-        }
+        S3Repository.Repository.KEY_SETTING,
+        S3Repository.Repository.SECRET_SETTING,
+        S3Repository.Repository.BUCKET_SETTING,
+        S3Repository.Repository.ENDPOINT_SETTING,
+        S3Repository.Repository.PROTOCOL_SETTING,
+        S3Repository.Repository.REGION_SETTING,
+        S3Repository.Repository.SERVER_SIDE_ENCRYPTION_SETTING,
+        S3Repository.Repository.BUFFER_SIZE_SETTING,
+        S3Repository.Repository.MAX_RETRIES_SETTING,
+        S3Repository.Repository.CHUNK_SIZE_SETTING,
+        S3Repository.Repository.COMPRESS_SETTING,
+        S3Repository.Repository.STORAGE_CLASS_SETTING,
+        S3Repository.Repository.CANNED_ACL_SETTING,
+        S3Repository.Repository.BASE_PATH_SETTING);
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/index/MockEngineFactoryPlugin.java b/test/framework/src/main/java/org/elasticsearch/index/MockEngineFactoryPlugin.java
index 3c72701f214..5b6c150b729 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/MockEngineFactoryPlugin.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/MockEngineFactoryPlugin.java
@@ -22,13 +22,16 @@ import org.apache.lucene.index.AssertingDirectoryReader;
 import org.apache.lucene.index.FilterDirectoryReader;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.engine.MockEngineFactory;
 import org.elasticsearch.test.engine.MockEngineSupport;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 
 // this must exist in the same package as IndexModule to allow access to setting the impl
 public class MockEngineFactoryPlugin extends Plugin {
@@ -43,9 +46,9 @@ public class MockEngineFactoryPlugin extends Plugin {
 
     private Class<? extends FilterDirectoryReader> readerWrapper = AssertingDirectoryReader.class;
 
-    public void onModule(SettingsModule module) {
-        module.registerSetting(MockEngineSupport.DISABLE_FLUSH_ON_CLOSE);
-        module.registerSetting(MockEngineSupport.WRAP_READER_RATIO);
+    @Override
+    public List<Setting<?>> getSettings() {
+        return Arrays.asList(MockEngineSupport.DISABLE_FLUSH_ON_CLOSE, MockEngineSupport.WRAP_READER_RATIO);
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
index 34e673ac126..9200662d6e4 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
@@ -60,6 +60,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.unit.Fuzziness;
@@ -108,7 +109,6 @@ import org.elasticsearch.script.ScriptSettings;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.threadpool.ThreadPoolModule;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.junit.After;
@@ -880,15 +880,16 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             Environment env = InternalSettingsPreparer.prepareEnvironment(settings, null);
             PluginsService pluginsService =new PluginsService(settings, env.modulesFile(), env.pluginsFile(), plugins);
 
-            SettingsModule settingsModule = new SettingsModule(settings);
-            settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
             final Client proxy = (Client) Proxy.newProxyInstance(
                     Client.class.getClassLoader(),
                     new Class[]{Client.class},
                     clientInvocationHandler);
             NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
             ScriptModule scriptModule = newTestScriptModule();
-            scriptModule.prepareSettings(settingsModule);
+            List<Setting<?>> scriptSettings = scriptModule.getSettings();
+            scriptSettings.addAll(pluginsService.getPluginSettings());
+            scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
+            SettingsModule settingsModule = new SettingsModule(settings, scriptSettings, pluginsService.getPluginSettingsFilter());
             searchModule = new SearchModule(settings, namedWriteableRegistry) {
                 @Override
                 protected void configureSearch() {
@@ -901,9 +902,8 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             }
             modulesBuilder.add(new PluginsModule(pluginsService));
             modulesBuilder.add(
-                    new EnvironmentModule(new Environment(settings)),
+                    new EnvironmentModule(new Environment(settings), threadPool),
                     settingsModule,
-                    new ThreadPoolModule(threadPool),
                     new IndicesModule() {
                         @Override
                         public void configure() {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index 4e066bc7635..5480e1549cf 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -1806,8 +1806,9 @@ public abstract class ESIntegTestCase extends ESTestCase {
         public String description() {
             return "a test plugin that registers index.tests.seed as an index setting";
         }
-        public void onModule(SettingsModule module) {
-            module.registerSetting(INDEX_TEST_SEED_SETTING);
+        @Override
+        public List<Setting<?>> getSettings() {
+            return Arrays.asList(INDEX_TEST_SEED_SETTING);
         }
 
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 727ea93a659..e0ae77d3f56 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -773,8 +773,7 @@ public abstract class ESTestCase extends LuceneTestCase {
         for (Consumer<AnalysisModule> consumer : moduleConsumers) {
             consumer.accept(analysisModule);
         }
-        SettingsModule settingsModule = new SettingsModule(nodeSettings);
-        settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
+        SettingsModule settingsModule = new SettingsModule(nodeSettings, InternalSettingsPlugin.VERSION_CREATED);
         final AnalysisService analysisService = analysisModule.buildRegistry().build(IndexSettingsModule.newIndexSettings(index, indexSettings));
         return analysisService;
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java b/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java
index f76ae7b4b56..c7b86a6ac13 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalSettingsPlugin.java
@@ -24,6 +24,9 @@ import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.plugins.Plugin;
 
+import java.util.Arrays;
+import java.util.List;
+
 public final class InternalSettingsPlugin extends Plugin {
     @Override
     public String name() {
@@ -42,9 +45,8 @@ public final class InternalSettingsPlugin extends Plugin {
     public static final Setting<Long> INDEX_CREATION_DATE_SETTING =
         Setting.longSetting(IndexMetaData.SETTING_CREATION_DATE, -1, -1, Property.IndexScope, Property.NodeScope);
 
-    public void onModule(SettingsModule module) {
-        module.registerSetting(VERSION_CREATED);
-        module.registerSetting(MERGE_ENABLED);
-        module.registerSetting(INDEX_CREATION_DATE_SETTING);
+    @Override
+    public List<Setting<?>> getSettings() {
+        return Arrays.asList(VERSION_CREATED, MERGE_ENABLED, INDEX_CREATION_DATE_SETTING);
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index 0dee5f75948..66f02c8a376 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -1940,7 +1940,7 @@ public final class InternalTestCluster extends TestCluster {
     private void assertRequestsFinished() {
         if (size() > 0) {
             for (NodeAndClient nodeAndClient : nodes.values()) {
-                CircuitBreaker inFlightRequestsBreaker = getInstance(HierarchyCircuitBreakerService.class, nodeAndClient.name)
+                CircuitBreaker inFlightRequestsBreaker = getInstance(CircuitBreakerService.class, nodeAndClient.name)
                     .getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS);
                 try {
                     // see #ensureEstimatedStats()
diff --git a/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java b/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java
index f17fe024f14..4cf224c4544 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/MockIndexEventListener.java
@@ -34,8 +34,10 @@ import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.plugins.Plugin;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 
 /**
  * This is a testing plugin that registers a generic {@link org.elasticsearch.test.MockIndexEventListener.TestEventListener} as a node level service as well as a listener
@@ -65,8 +67,9 @@ public final class MockIndexEventListener {
          * For tests to pass in to fail on listener invocation
          */
         public static final Setting<Boolean> INDEX_FAIL = Setting.boolSetting("index.fail", false, Property.IndexScope);
-        public void onModule(SettingsModule module) {
-            module.registerSetting(INDEX_FAIL);
+        @Override
+        public List<Setting<?>> getSettings() {
+            return Arrays.asList(INDEX_FAIL);
         }
 
         @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java
index d44cf60e9e3..34dd613fb0c 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSIndexStore.java
@@ -38,9 +38,11 @@ import org.elasticsearch.index.store.IndexStore;
 import org.elasticsearch.index.store.IndexStoreConfig;
 import org.elasticsearch.plugins.Plugin;
 
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.IdentityHashMap;
+import java.util.List;
 import java.util.Map;
 
 public class MockFSIndexStore extends IndexStore {
@@ -62,14 +64,14 @@ public class MockFSIndexStore extends IndexStore {
             return Settings.builder().put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "mock").build();
         }
 
-        public void onModule(SettingsModule module) {
-
-            module.registerSetting(INDEX_CHECK_INDEX_ON_CLOSE_SETTING);
-            module.registerSetting(MockFSDirectoryService.CRASH_INDEX_SETTING);
-            module.registerSetting(MockFSDirectoryService.RANDOM_IO_EXCEPTION_RATE_SETTING);
-            module.registerSetting(MockFSDirectoryService.RANDOM_PREVENT_DOUBLE_WRITE_SETTING);
-            module.registerSetting(MockFSDirectoryService.RANDOM_NO_DELETE_OPEN_FILE_SETTING);
-            module.registerSetting(MockFSDirectoryService.RANDOM_IO_EXCEPTION_RATE_ON_OPEN_SETTING);
+        @Override
+        public List<Setting<?>> getSettings() {
+            return Arrays.asList(INDEX_CHECK_INDEX_ON_CLOSE_SETTING,
+            MockFSDirectoryService.CRASH_INDEX_SETTING,
+            MockFSDirectoryService.RANDOM_IO_EXCEPTION_RATE_SETTING,
+            MockFSDirectoryService.RANDOM_PREVENT_DOUBLE_WRITE_SETTING,
+            MockFSDirectoryService.RANDOM_NO_DELETE_OPEN_FILE_SETTING,
+            MockFSDirectoryService.RANDOM_IO_EXCEPTION_RATE_ON_OPEN_SETTING);
         }
 
         @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/AssertingLocalTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/AssertingLocalTransport.java
index a9687c27873..3b63eadcc0c 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/AssertingLocalTransport.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/AssertingLocalTransport.java
@@ -42,6 +42,8 @@ import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.local.LocalTransport;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Random;
 
 public class AssertingLocalTransport extends LocalTransport {
@@ -63,9 +65,9 @@ public class AssertingLocalTransport extends LocalTransport {
             return Settings.builder().put(NetworkModule.TRANSPORT_TYPE_KEY, "mock").build();
         }
 
-        public void onModule(SettingsModule module) {
-            module.registerSetting(ASSERTING_TRANSPORT_MIN_VERSION_KEY);
-            module.registerSetting(ASSERTING_TRANSPORT_MAX_VERSION_KEY);
+        @Override
+        public List<Setting<?>> getSettings() {
+            return Arrays.asList(ASSERTING_TRANSPORT_MIN_VERSION_KEY, ASSERTING_TRANSPORT_MAX_VERSION_KEY);
         }
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
index 05b8b23e90c..3df16d3bc0d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
@@ -21,6 +21,7 @@ package org.elasticsearch.test.transport;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.transport.TransportService;
 
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -91,8 +92,9 @@ public class MockTransportService extends TransportService {
             module.registerTransportService("mock", MockTransportService.class);
         }
 
-        public void onModule(SettingsModule module) {
-            module.registerSetting(MockTaskManager.USE_MOCK_TASK_MANAGER_SETTING);
+        @Override
+        public List<Setting<?>> getSettings() {
+            return Arrays.asList(MockTaskManager.USE_MOCK_TASK_MANAGER_SETTING);
         }
         @Override
         public Settings additionalSettings() {

From 9eea1b68339c19ef1563e8adda26d081e227e9a9 Mon Sep 17 00:00:00 2001
From: Alexander Kazakov <alexshadow007@gmail.com>
Date: Thu, 16 Jun 2016 17:03:51 +0300
Subject: [PATCH 0912/1311] Fix flat_settings REST parameter

* Get XContent params from request in Nodes rest actions
* Adding test for nodes info rest api
---
 .../rest/action/support/RestActions.java      |  2 +-
 .../test/nodes.info/30_settings.yaml          | 19 +++++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/nodes.info/30_settings.yaml

diff --git a/core/src/main/java/org/elasticsearch/rest/action/support/RestActions.java b/core/src/main/java/org/elasticsearch/rest/action/support/RestActions.java
index 51d5089ec6f..66998ee8727 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/support/RestActions.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/support/RestActions.java
@@ -275,7 +275,7 @@ public class RestActions {
 
         @Override
         public RestResponse buildResponse(NodesResponse response, XContentBuilder builder) throws Exception {
-            return RestActions.nodesResponse(builder, ToXContent.EMPTY_PARAMS, response);
+            return RestActions.nodesResponse(builder, channel.request(), response);
         }
 
     }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.info/30_settings.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.info/30_settings.yaml
new file mode 100644
index 00000000000..a63c246b603
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/nodes.info/30_settings.yaml
@@ -0,0 +1,19 @@
+---
+"node_info test flat_settings":
+  - do:
+      cluster.state: {}
+
+  - set: { master_node: master }
+
+  - do:
+      nodes.info:
+        metric: [ settings ]
+
+  - match : { nodes.$master.settings.client.type: node }
+
+  - do:
+      nodes.info:
+        metric: [ settings ]
+        flat_settings: true
+
+  - match : { nodes.$master.settings.client\.type: node }

From fdd1f152a73c91cf4f9d90f0365a15895b6417b3 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Thu, 16 Jun 2016 11:02:06 -0400
Subject: [PATCH 0913/1311] improve lambda syntax (allow single expression)

---
 .../src/main/antlr/PainlessParser.g4          |   2 +-
 .../painless/antlr/PainlessParser.java        | 367 +++++++++---------
 .../elasticsearch/painless/antlr/Walker.java  |  10 +-
 .../elasticsearch/painless/LambdaTests.java   |  28 +-
 4 files changed, 217 insertions(+), 190 deletions(-)

diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index a020b0d7d73..6dbaa50c73d 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -189,7 +189,7 @@ argument
     ;
 
 lambda
-    : ( lamtype | LP ( lamtype ( COMMA lamtype )* )? RP ) ARROW block
+    : ( lamtype | LP ( lamtype ( COMMA lamtype )* )? RP ) ARROW ( block | expression )
     ;
 
 lamtype
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
index ba3798bb414..7b994e699a2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
@@ -2754,9 +2754,6 @@ class PainlessParser extends Parser {
 
   public static class LambdaContext extends ParserRuleContext {
     public TerminalNode ARROW() { return getToken(PainlessParser.ARROW, 0); }
-    public BlockContext block() {
-      return getRuleContext(BlockContext.class,0);
-    }
     public List<LamtypeContext> lamtype() {
       return getRuleContexts(LamtypeContext.class);
     }
@@ -2765,6 +2762,12 @@ class PainlessParser extends Parser {
     }
     public TerminalNode LP() { return getToken(PainlessParser.LP, 0); }
     public TerminalNode RP() { return getToken(PainlessParser.RP, 0); }
+    public BlockContext block() {
+      return getRuleContext(BlockContext.class,0);
+    }
+    public ExpressionContext expression() {
+      return getRuleContext(ExpressionContext.class,0);
+    }
     public List<TerminalNode> COMMA() { return getTokens(PainlessParser.COMMA); }
     public TerminalNode COMMA(int i) {
       return getToken(PainlessParser.COMMA, i);
@@ -2834,8 +2837,21 @@ class PainlessParser extends Parser {
       }
       setState(448);
       match(ARROW);
-      setState(449);
-      block();
+      setState(451);
+      switch ( getInterpreter().adaptivePredict(_input,38,_ctx) ) {
+      case 1:
+        {
+        setState(449);
+        block();
+        }
+        break;
+      case 2:
+        {
+        setState(450);
+        expression(0);
+        }
+        break;
+      }
       }
     }
     catch (RecognitionException re) {
@@ -2872,16 +2888,16 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(452);
+      setState(454);
       _la = _input.LA(1);
       if (_la==TYPE) {
         {
-        setState(451);
+        setState(453);
         decltype();
         }
       }
 
-      setState(454);
+      setState(456);
       match(ID);
       }
     }
@@ -2924,33 +2940,33 @@ class PainlessParser extends Parser {
     FuncrefContext _localctx = new FuncrefContext(_ctx, getState());
     enterRule(_localctx, 50, RULE_funcref);
     try {
-      setState(460);
-      switch ( getInterpreter().adaptivePredict(_input,39,_ctx) ) {
+      setState(462);
+      switch ( getInterpreter().adaptivePredict(_input,40,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(456);
+        setState(458);
         classFuncref();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(457);
+        setState(459);
         constructorFuncref();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(458);
+        setState(460);
         capturingFuncref();
         }
         break;
       case 4:
         enterOuterAlt(_localctx, 4);
         {
-        setState(459);
+        setState(461);
         localFuncref();
         }
         break;
@@ -2988,11 +3004,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(462);
-      match(TYPE);
-      setState(463);
-      match(REF);
       setState(464);
+      match(TYPE);
+      setState(465);
+      match(REF);
+      setState(466);
       match(ID);
       }
     }
@@ -3030,11 +3046,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(466);
-      decltype();
-      setState(467);
-      match(REF);
       setState(468);
+      decltype();
+      setState(469);
+      match(REF);
+      setState(470);
       match(NEW);
       }
     }
@@ -3072,12 +3088,12 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(470);
-      match(ID);
-      setState(471);
-      match(REF);
       setState(472);
       match(ID);
+      setState(473);
+      match(REF);
+      setState(474);
+      match(ID);
       }
     }
     catch (RecognitionException re) {
@@ -3112,11 +3128,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(474);
-      match(THIS);
-      setState(475);
-      match(REF);
       setState(476);
+      match(THIS);
+      setState(477);
+      match(REF);
+      setState(478);
       match(ID);
       }
     }
@@ -3223,7 +3239,7 @@ class PainlessParser extends Parser {
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3R\u01e1\4\2\t\2\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3R\u01e3\4\2\t\2\4"+
     "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
     "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
     "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+
@@ -3257,152 +3273,153 @@ class PainlessParser extends Parser {
     "\u019c\n\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\27\7\27\u01a6\n\27\f"+
     "\27\16\27\u01a9\13\27\5\27\u01ab\n\27\3\27\3\27\3\30\3\30\3\30\5\30\u01b2"+
     "\n\30\3\31\3\31\3\31\3\31\3\31\7\31\u01b9\n\31\f\31\16\31\u01bc\13\31"+
-    "\5\31\u01be\n\31\3\31\5\31\u01c1\n\31\3\31\3\31\3\31\3\32\5\32\u01c7\n"+
-    "\32\3\32\3\32\3\33\3\33\3\33\3\33\5\33\u01cf\n\33\3\34\3\34\3\34\3\34"+
-    "\3\35\3\35\3\35\3\35\3\36\3\36\3\36\3\36\3\37\3\37\3\37\3\37\3\37\2\3"+
-    "\36 \2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,.\60\62\64\668:<\2"+
-    "\r\3\2:E\3\2\35\37\3\2 !\3\2\66\67\3\2\"$\3\2%(\3\2),\3\289\3\2FI\4\2"+
-    "\33\34 !\3\2QR\u020f\2A\3\2\2\2\4L\3\2\2\2\6Q\3\2\2\2\b\u00af\3\2\2\2"+
-    "\n\u00b3\3\2\2\2\f\u00b5\3\2\2\2\16\u00be\3\2\2\2\20\u00c2\3\2\2\2\22"+
-    "\u00c4\3\2\2\2\24\u00c6\3\2\2\2\26\u00cf\3\2\2\2\30\u00d7\3\2\2\2\32\u00dc"+
-    "\3\2\2\2\34\u00e8\3\2\2\2\36\u00f3\3\2\2\2 \u0155\3\2\2\2\"\u0179\3\2"+
-    "\2\2$\u018e\3\2\2\2&\u0194\3\2\2\2(\u019b\3\2\2\2*\u019d\3\2\2\2,\u01a1"+
-    "\3\2\2\2.\u01b1\3\2\2\2\60\u01c0\3\2\2\2\62\u01c6\3\2\2\2\64\u01ce\3\2"+
-    "\2\2\66\u01d0\3\2\2\28\u01d4\3\2\2\2:\u01d8\3\2\2\2<\u01dc\3\2\2\2>@\5"+
-    "\4\3\2?>\3\2\2\2@C\3\2\2\2A?\3\2\2\2AB\3\2\2\2BG\3\2\2\2CA\3\2\2\2DF\5"+
-    "\b\5\2ED\3\2\2\2FI\3\2\2\2GE\3\2\2\2GH\3\2\2\2HJ\3\2\2\2IG\3\2\2\2JK\7"+
-    "\2\2\3K\3\3\2\2\2LM\5\26\f\2MN\7P\2\2NO\5\6\4\2OP\5\f\7\2P\5\3\2\2\2Q"+
-    "]\7\t\2\2RS\5\26\f\2SZ\7P\2\2TU\7\f\2\2UV\5\26\f\2VW\7P\2\2WY\3\2\2\2"+
-    "XT\3\2\2\2Y\\\3\2\2\2ZX\3\2\2\2Z[\3\2\2\2[^\3\2\2\2\\Z\3\2\2\2]R\3\2\2"+
-    "\2]^\3\2\2\2^_\3\2\2\2_`\7\n\2\2`\7\3\2\2\2ab\7\16\2\2bc\7\t\2\2cd\5\36"+
-    "\20\2de\7\n\2\2ei\5\n\6\2fg\7\17\2\2gj\5\n\6\2hj\6\5\2\2if\3\2\2\2ih\3"+
-    "\2\2\2j\u00b0\3\2\2\2kl\7\20\2\2lm\7\t\2\2mn\5\36\20\2nq\7\n\2\2or\5\n"+
-    "\6\2pr\5\16\b\2qo\3\2\2\2qp\3\2\2\2r\u00b0\3\2\2\2st\7\21\2\2tu\5\f\7"+
-    "\2uv\7\20\2\2vw\7\t\2\2wx\5\36\20\2xy\7\n\2\2yz\5\34\17\2z\u00b0\3\2\2"+
-    "\2{|\7\22\2\2|~\7\t\2\2}\177\5\20\t\2~}\3\2\2\2~\177\3\2\2\2\177\u0080"+
-    "\3\2\2\2\u0080\u0082\7\r\2\2\u0081\u0083\5\36\20\2\u0082\u0081\3\2\2\2"+
-    "\u0082\u0083\3\2\2\2\u0083\u0084\3\2\2\2\u0084\u0086\7\r\2\2\u0085\u0087"+
-    "\5\22\n\2\u0086\u0085\3\2\2\2\u0086\u0087\3\2\2\2\u0087\u0088\3\2\2\2"+
-    "\u0088\u008b\7\n\2\2\u0089\u008c\5\n\6\2\u008a\u008c\5\16\b\2\u008b\u0089"+
-    "\3\2\2\2\u008b\u008a\3\2\2\2\u008c\u00b0\3\2\2\2\u008d\u008e\7\22\2\2"+
-    "\u008e\u008f\7\t\2\2\u008f\u0090\5\26\f\2\u0090\u0091\7P\2\2\u0091\u0092"+
-    "\7\63\2\2\u0092\u0093\5\36\20\2\u0093\u0094\7\n\2\2\u0094\u0095\5\n\6"+
-    "\2\u0095\u00b0\3\2\2\2\u0096\u0097\5\24\13\2\u0097\u0098\5\34\17\2\u0098"+
-    "\u00b0\3\2\2\2\u0099\u009a\7\23\2\2\u009a\u00b0\5\34\17\2\u009b\u009c"+
-    "\7\24\2\2\u009c\u00b0\5\34\17\2\u009d\u009e\7\25\2\2\u009e\u009f\5\36"+
-    "\20\2\u009f\u00a0\5\34\17\2\u00a0\u00b0\3\2\2\2\u00a1\u00a2\7\27\2\2\u00a2"+
-    "\u00a4\5\f\7\2\u00a3\u00a5\5\32\16\2\u00a4\u00a3\3\2\2\2\u00a5\u00a6\3"+
-    "\2\2\2\u00a6\u00a4\3\2\2\2\u00a6\u00a7\3\2\2\2\u00a7\u00b0\3\2\2\2\u00a8"+
-    "\u00a9\7\31\2\2\u00a9\u00aa\5\36\20\2\u00aa\u00ab\5\34\17\2\u00ab\u00b0"+
-    "\3\2\2\2\u00ac\u00ad\5\36\20\2\u00ad\u00ae\5\34\17\2\u00ae\u00b0\3\2\2"+
-    "\2\u00afa\3\2\2\2\u00afk\3\2\2\2\u00afs\3\2\2\2\u00af{\3\2\2\2\u00af\u008d"+
-    "\3\2\2\2\u00af\u0096\3\2\2\2\u00af\u0099\3\2\2\2\u00af\u009b\3\2\2\2\u00af"+
-    "\u009d\3\2\2\2\u00af\u00a1\3\2\2\2\u00af\u00a8\3\2\2\2\u00af\u00ac\3\2"+
-    "\2\2\u00b0\t\3\2\2\2\u00b1\u00b4\5\f\7\2\u00b2\u00b4\5\b\5\2\u00b3\u00b1"+
-    "\3\2\2\2\u00b3\u00b2\3\2\2\2\u00b4\13\3\2\2\2\u00b5\u00b9\7\5\2\2\u00b6"+
-    "\u00b8\5\b\5\2\u00b7\u00b6\3\2\2\2\u00b8\u00bb\3\2\2\2\u00b9\u00b7\3\2"+
-    "\2\2\u00b9\u00ba\3\2\2\2\u00ba\u00bc\3\2\2\2\u00bb\u00b9\3\2\2\2\u00bc"+
-    "\u00bd\7\6\2\2\u00bd\r\3\2\2\2\u00be\u00bf\7\r\2\2\u00bf\17\3\2\2\2\u00c0"+
-    "\u00c3\5\24\13\2\u00c1\u00c3\5\36\20\2\u00c2\u00c0\3\2\2\2\u00c2\u00c1"+
-    "\3\2\2\2\u00c3\21\3\2\2\2\u00c4\u00c5\5\36\20\2\u00c5\23\3\2\2\2\u00c6"+
-    "\u00c7\5\26\f\2\u00c7\u00cc\5\30\r\2\u00c8\u00c9\7\f\2\2\u00c9\u00cb\5"+
-    "\30\r\2\u00ca\u00c8\3\2\2\2\u00cb\u00ce\3\2\2\2\u00cc\u00ca\3\2\2\2\u00cc"+
-    "\u00cd\3\2\2\2\u00cd\25\3\2\2\2\u00ce\u00cc\3\2\2\2\u00cf\u00d4\7O\2\2"+
-    "\u00d0\u00d1\7\7\2\2\u00d1\u00d3\7\b\2\2\u00d2\u00d0\3\2\2\2\u00d3\u00d6"+
-    "\3\2\2\2\u00d4\u00d2\3\2\2\2\u00d4\u00d5\3\2\2\2\u00d5\27\3\2\2\2\u00d6"+
-    "\u00d4\3\2\2\2\u00d7\u00da\7P\2\2\u00d8\u00d9\7:\2\2\u00d9\u00db\5\36"+
-    "\20\2\u00da\u00d8\3\2\2\2\u00da\u00db\3\2\2\2\u00db\31\3\2\2\2\u00dc\u00dd"+
-    "\7\30\2\2\u00dd\u00de\7\t\2\2\u00de\u00df\7O\2\2\u00df\u00e0\7P\2\2\u00e0"+
-    "\u00e1\7\n\2\2\u00e1\u00e2\5\f\7\2\u00e2\33\3\2\2\2\u00e3\u00e9\7\r\2"+
-    "\2\u00e4\u00e9\7\2\2\3\u00e5\u00e6\b\17\1\2\u00e6\u00e7\7\6\2\2\u00e7"+
-    "\u00e9\b\17\1\2\u00e8\u00e3\3\2\2\2\u00e8\u00e4\3\2\2\2\u00e8\u00e5\3"+
-    "\2\2\2\u00e9\35\3\2\2\2\u00ea\u00eb\b\20\1\2\u00eb\u00ec\5\"\22\2\u00ec"+
-    "\u00ed\t\2\2\2\u00ed\u00ee\5\36\20\3\u00ee\u00ef\b\20\1\2\u00ef\u00f4"+
-    "\3\2\2\2\u00f0\u00f1\5 \21\2\u00f1\u00f2\b\20\1\2\u00f2\u00f4\3\2\2\2"+
-    "\u00f3\u00ea\3\2\2\2\u00f3\u00f0\3\2\2\2\u00f4\u0135\3\2\2\2\u00f5\u00f6"+
-    "\f\17\2\2\u00f6\u00f7\t\3\2\2\u00f7\u00f8\5\36\20\20\u00f8\u00f9\b\20"+
-    "\1\2\u00f9\u0134\3\2\2\2\u00fa\u00fb\f\16\2\2\u00fb\u00fc\t\4\2\2\u00fc"+
-    "\u00fd\5\36\20\17\u00fd\u00fe\b\20\1\2\u00fe\u0134\3\2\2\2\u00ff\u0100"+
-    "\f\r\2\2\u0100\u0101\t\5\2\2\u0101\u0102\5\36\20\16\u0102\u0103\b\20\1"+
-    "\2\u0103\u0134\3\2\2\2\u0104\u0105\f\f\2\2\u0105\u0106\t\6\2\2\u0106\u0107"+
-    "\5\36\20\r\u0107\u0108\b\20\1\2\u0108\u0134\3\2\2\2\u0109\u010a\f\13\2"+
-    "\2\u010a\u010b\t\7\2\2\u010b\u010c\5\36\20\f\u010c\u010d\b\20\1\2\u010d"+
-    "\u0134\3\2\2\2\u010e\u010f\f\n\2\2\u010f\u0110\t\b\2\2\u0110\u0111\5\36"+
-    "\20\13\u0111\u0112\b\20\1\2\u0112\u0134\3\2\2\2\u0113\u0114\f\t\2\2\u0114"+
-    "\u0115\7-\2\2\u0115\u0116\5\36\20\n\u0116\u0117\b\20\1\2\u0117\u0134\3"+
-    "\2\2\2\u0118\u0119\f\b\2\2\u0119\u011a\7.\2\2\u011a\u011b\5\36\20\t\u011b"+
-    "\u011c\b\20\1\2\u011c\u0134\3\2\2\2\u011d\u011e\f\7\2\2\u011e\u011f\7"+
-    "/\2\2\u011f\u0120\5\36\20\b\u0120\u0121\b\20\1\2\u0121\u0134\3\2\2\2\u0122"+
-    "\u0123\f\6\2\2\u0123\u0124\7\60\2\2\u0124\u0125\5\36\20\7\u0125\u0126"+
-    "\b\20\1\2\u0126\u0134\3\2\2\2\u0127\u0128\f\5\2\2\u0128\u0129\7\61\2\2"+
-    "\u0129\u012a\5\36\20\6\u012a\u012b\b\20\1\2\u012b\u0134\3\2\2\2\u012c"+
-    "\u012d\f\4\2\2\u012d\u012e\7\62\2\2\u012e\u012f\5\36\20\2\u012f\u0130"+
-    "\7\63\2\2\u0130\u0131\5\36\20\4\u0131\u0132\b\20\1\2\u0132\u0134\3\2\2"+
-    "\2\u0133\u00f5\3\2\2\2\u0133\u00fa\3\2\2\2\u0133\u00ff\3\2\2\2\u0133\u0104"+
-    "\3\2\2\2\u0133\u0109\3\2\2\2\u0133\u010e\3\2\2\2\u0133\u0113\3\2\2\2\u0133"+
-    "\u0118\3\2\2\2\u0133\u011d\3\2\2\2\u0133\u0122\3\2\2\2\u0133\u0127\3\2"+
-    "\2\2\u0133\u012c\3\2\2\2\u0134\u0137\3\2\2\2\u0135\u0133\3\2\2\2\u0135"+
-    "\u0136\3\2\2\2\u0136\37\3\2\2\2\u0137\u0135\3\2\2\2\u0138\u0139\6\21\17"+
-    "\3\u0139\u013a\t\t\2\2\u013a\u0156\5\"\22\2\u013b\u013c\6\21\20\3\u013c"+
-    "\u013d\5\"\22\2\u013d\u013e\t\t\2\2\u013e\u0156\3\2\2\2\u013f\u0140\6"+
-    "\21\21\3\u0140\u0156\5\"\22\2\u0141\u0142\6\21\22\3\u0142\u0143\t\n\2"+
-    "\2\u0143\u0156\b\21\1\2\u0144\u0145\6\21\23\3\u0145\u0146\7L\2\2\u0146"+
-    "\u0156\b\21\1\2\u0147\u0148\6\21\24\3\u0148\u0149\7M\2\2\u0149\u0156\b"+
-    "\21\1\2\u014a\u014b\6\21\25\3\u014b\u014c\7N\2\2\u014c\u0156\b\21\1\2"+
-    "\u014d\u014e\6\21\26\3\u014e\u014f\t\13\2\2\u014f\u0156\5 \21\2\u0150"+
-    "\u0151\7\t\2\2\u0151\u0152\5\26\f\2\u0152\u0153\7\n\2\2\u0153\u0154\5"+
-    " \21\2\u0154\u0156\3\2\2\2\u0155\u0138\3\2\2\2\u0155\u013b\3\2\2\2\u0155"+
-    "\u013f\3\2\2\2\u0155\u0141\3\2\2\2\u0155\u0144\3\2\2\2\u0155\u0147\3\2"+
-    "\2\2\u0155\u014a\3\2\2\2\u0155\u014d\3\2\2\2\u0155\u0150\3\2\2\2\u0156"+
-    "!\3\2\2\2\u0157\u015b\5$\23\2\u0158\u015a\5&\24\2\u0159\u0158\3\2\2\2"+
-    "\u015a\u015d\3\2\2\2\u015b\u0159\3\2\2\2\u015b\u015c\3\2\2\2\u015c\u017a"+
-    "\3\2\2\2\u015d\u015b\3\2\2\2\u015e\u015f\5\26\f\2\u015f\u0163\5(\25\2"+
-    "\u0160\u0162\5&\24\2\u0161\u0160\3\2\2\2\u0162\u0165\3\2\2\2\u0163\u0161"+
-    "\3\2\2\2\u0163\u0164\3\2\2\2\u0164\u017a\3\2\2\2\u0165\u0163\3\2\2\2\u0166"+
-    "\u0167\7\26\2\2\u0167\u016c\7O\2\2\u0168\u0169\7\7\2\2\u0169\u016a\5\36"+
-    "\20\2\u016a\u016b\7\b\2\2\u016b\u016d\3\2\2\2\u016c\u0168\3\2\2\2\u016d"+
-    "\u016e\3\2\2\2\u016e\u016c\3\2\2\2\u016e\u016f\3\2\2\2\u016f\u0177\3\2"+
-    "\2\2\u0170\u0174\5(\25\2\u0171\u0173\5&\24\2\u0172\u0171\3\2\2\2\u0173"+
-    "\u0176\3\2\2\2\u0174\u0172\3\2\2\2\u0174\u0175\3\2\2\2\u0175\u0178\3\2"+
-    "\2\2\u0176\u0174\3\2\2\2\u0177\u0170\3\2\2\2\u0177\u0178\3\2\2\2\u0178"+
-    "\u017a\3\2\2\2\u0179\u0157\3\2\2\2\u0179\u015e\3\2\2\2\u0179\u0166\3\2"+
-    "\2\2\u017a#\3\2\2\2\u017b\u017c\6\23\27\3\u017c\u017d\7\t\2\2\u017d\u017e"+
-    "\5\36\20\2\u017e\u017f\7\n\2\2\u017f\u0180\b\23\1\2\u0180\u018f\3\2\2"+
-    "\2\u0181\u0182\6\23\30\3\u0182\u0183\7\t\2\2\u0183\u0184\5 \21\2\u0184"+
-    "\u0185\7\n\2\2\u0185\u018f\3\2\2\2\u0186\u018f\7J\2\2\u0187\u018f\7K\2"+
-    "\2\u0188\u018f\7P\2\2\u0189\u018a\7P\2\2\u018a\u018f\5,\27\2\u018b\u018c"+
-    "\7\26\2\2\u018c\u018d\7O\2\2\u018d\u018f\5,\27\2\u018e\u017b\3\2\2\2\u018e"+
-    "\u0181\3\2\2\2\u018e\u0186\3\2\2\2\u018e\u0187\3\2\2\2\u018e\u0188\3\2"+
-    "\2\2\u018e\u0189\3\2\2\2\u018e\u018b\3\2\2\2\u018f%\3\2\2\2\u0190\u0191"+
-    "\6\24\31\3\u0191\u0195\5(\25\2\u0192\u0193\6\24\32\3\u0193\u0195\5*\26"+
-    "\2\u0194\u0190\3\2\2\2\u0194\u0192\3\2\2\2\u0195\'\3\2\2\2\u0196\u0197"+
-    "\7\13\2\2\u0197\u0198\7R\2\2\u0198\u019c\5,\27\2\u0199\u019a\7\13\2\2"+
-    "\u019a\u019c\t\f\2\2\u019b\u0196\3\2\2\2\u019b\u0199\3\2\2\2\u019c)\3"+
-    "\2\2\2\u019d\u019e\7\7\2\2\u019e\u019f\5\36\20\2\u019f\u01a0\7\b\2\2\u01a0"+
-    "+\3\2\2\2\u01a1\u01aa\7\t\2\2\u01a2\u01a7\5.\30\2\u01a3\u01a4\7\f\2\2"+
-    "\u01a4\u01a6\5.\30\2\u01a5\u01a3\3\2\2\2\u01a6\u01a9\3\2\2\2\u01a7\u01a5"+
-    "\3\2\2\2\u01a7\u01a8\3\2\2\2\u01a8\u01ab\3\2\2\2\u01a9\u01a7\3\2\2\2\u01aa"+
-    "\u01a2\3\2\2\2\u01aa\u01ab\3\2\2\2\u01ab\u01ac\3\2\2\2\u01ac\u01ad\7\n"+
-    "\2\2\u01ad-\3\2\2\2\u01ae\u01b2\5\36\20\2\u01af\u01b2\5\60\31\2\u01b0"+
-    "\u01b2\5\64\33\2\u01b1\u01ae\3\2\2\2\u01b1\u01af\3\2\2\2\u01b1\u01b0\3"+
-    "\2\2\2\u01b2/\3\2\2\2\u01b3\u01c1\5\62\32\2\u01b4\u01bd\7\t\2\2\u01b5"+
-    "\u01ba\5\62\32\2\u01b6\u01b7\7\f\2\2\u01b7\u01b9\5\62\32\2\u01b8\u01b6"+
-    "\3\2\2\2\u01b9\u01bc\3\2\2\2\u01ba\u01b8\3\2\2\2\u01ba\u01bb\3\2\2\2\u01bb"+
-    "\u01be\3\2\2\2\u01bc\u01ba\3\2\2\2\u01bd\u01b5\3\2\2\2\u01bd\u01be\3\2"+
-    "\2\2\u01be\u01bf\3\2\2\2\u01bf\u01c1\7\n\2\2\u01c0\u01b3\3\2\2\2\u01c0"+
-    "\u01b4\3\2\2\2\u01c1\u01c2\3\2\2\2\u01c2\u01c3\7\65\2\2\u01c3\u01c4\5"+
-    "\f\7\2\u01c4\61\3\2\2\2\u01c5\u01c7\5\26\f\2\u01c6\u01c5\3\2\2\2\u01c6"+
-    "\u01c7\3\2\2\2\u01c7\u01c8\3\2\2\2\u01c8\u01c9\7P\2\2\u01c9\63\3\2\2\2"+
-    "\u01ca\u01cf\5\66\34\2\u01cb\u01cf\58\35\2\u01cc\u01cf\5:\36\2\u01cd\u01cf"+
-    "\5<\37\2\u01ce\u01ca\3\2\2\2\u01ce\u01cb\3\2\2\2\u01ce\u01cc\3\2\2\2\u01ce"+
-    "\u01cd\3\2\2\2\u01cf\65\3\2\2\2\u01d0\u01d1\7O\2\2\u01d1\u01d2\7\64\2"+
-    "\2\u01d2\u01d3\7P\2\2\u01d3\67\3\2\2\2\u01d4\u01d5\5\26\f\2\u01d5\u01d6"+
-    "\7\64\2\2\u01d6\u01d7\7\26\2\2\u01d79\3\2\2\2\u01d8\u01d9\7P\2\2\u01d9"+
-    "\u01da\7\64\2\2\u01da\u01db\7P\2\2\u01db;\3\2\2\2\u01dc\u01dd\7\32\2\2"+
-    "\u01dd\u01de\7\64\2\2\u01de\u01df\7P\2\2\u01df=\3\2\2\2*AGZ]iq~\u0082"+
+    "\5\31\u01be\n\31\3\31\5\31\u01c1\n\31\3\31\3\31\3\31\5\31\u01c6\n\31\3"+
+    "\32\5\32\u01c9\n\32\3\32\3\32\3\33\3\33\3\33\3\33\5\33\u01d1\n\33\3\34"+
+    "\3\34\3\34\3\34\3\35\3\35\3\35\3\35\3\36\3\36\3\36\3\36\3\37\3\37\3\37"+
+    "\3\37\3\37\2\3\36 \2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,.\60"+
+    "\62\64\668:<\2\r\3\2:E\3\2\35\37\3\2 !\3\2\66\67\3\2\"$\3\2%(\3\2),\3"+
+    "\289\3\2FI\4\2\33\34 !\3\2QR\u0212\2A\3\2\2\2\4L\3\2\2\2\6Q\3\2\2\2\b"+
+    "\u00af\3\2\2\2\n\u00b3\3\2\2\2\f\u00b5\3\2\2\2\16\u00be\3\2\2\2\20\u00c2"+
+    "\3\2\2\2\22\u00c4\3\2\2\2\24\u00c6\3\2\2\2\26\u00cf\3\2\2\2\30\u00d7\3"+
+    "\2\2\2\32\u00dc\3\2\2\2\34\u00e8\3\2\2\2\36\u00f3\3\2\2\2 \u0155\3\2\2"+
+    "\2\"\u0179\3\2\2\2$\u018e\3\2\2\2&\u0194\3\2\2\2(\u019b\3\2\2\2*\u019d"+
+    "\3\2\2\2,\u01a1\3\2\2\2.\u01b1\3\2\2\2\60\u01c0\3\2\2\2\62\u01c8\3\2\2"+
+    "\2\64\u01d0\3\2\2\2\66\u01d2\3\2\2\28\u01d6\3\2\2\2:\u01da\3\2\2\2<\u01de"+
+    "\3\2\2\2>@\5\4\3\2?>\3\2\2\2@C\3\2\2\2A?\3\2\2\2AB\3\2\2\2BG\3\2\2\2C"+
+    "A\3\2\2\2DF\5\b\5\2ED\3\2\2\2FI\3\2\2\2GE\3\2\2\2GH\3\2\2\2HJ\3\2\2\2"+
+    "IG\3\2\2\2JK\7\2\2\3K\3\3\2\2\2LM\5\26\f\2MN\7P\2\2NO\5\6\4\2OP\5\f\7"+
+    "\2P\5\3\2\2\2Q]\7\t\2\2RS\5\26\f\2SZ\7P\2\2TU\7\f\2\2UV\5\26\f\2VW\7P"+
+    "\2\2WY\3\2\2\2XT\3\2\2\2Y\\\3\2\2\2ZX\3\2\2\2Z[\3\2\2\2[^\3\2\2\2\\Z\3"+
+    "\2\2\2]R\3\2\2\2]^\3\2\2\2^_\3\2\2\2_`\7\n\2\2`\7\3\2\2\2ab\7\16\2\2b"+
+    "c\7\t\2\2cd\5\36\20\2de\7\n\2\2ei\5\n\6\2fg\7\17\2\2gj\5\n\6\2hj\6\5\2"+
+    "\2if\3\2\2\2ih\3\2\2\2j\u00b0\3\2\2\2kl\7\20\2\2lm\7\t\2\2mn\5\36\20\2"+
+    "nq\7\n\2\2or\5\n\6\2pr\5\16\b\2qo\3\2\2\2qp\3\2\2\2r\u00b0\3\2\2\2st\7"+
+    "\21\2\2tu\5\f\7\2uv\7\20\2\2vw\7\t\2\2wx\5\36\20\2xy\7\n\2\2yz\5\34\17"+
+    "\2z\u00b0\3\2\2\2{|\7\22\2\2|~\7\t\2\2}\177\5\20\t\2~}\3\2\2\2~\177\3"+
+    "\2\2\2\177\u0080\3\2\2\2\u0080\u0082\7\r\2\2\u0081\u0083\5\36\20\2\u0082"+
+    "\u0081\3\2\2\2\u0082\u0083\3\2\2\2\u0083\u0084\3\2\2\2\u0084\u0086\7\r"+
+    "\2\2\u0085\u0087\5\22\n\2\u0086\u0085\3\2\2\2\u0086\u0087\3\2\2\2\u0087"+
+    "\u0088\3\2\2\2\u0088\u008b\7\n\2\2\u0089\u008c\5\n\6\2\u008a\u008c\5\16"+
+    "\b\2\u008b\u0089\3\2\2\2\u008b\u008a\3\2\2\2\u008c\u00b0\3\2\2\2\u008d"+
+    "\u008e\7\22\2\2\u008e\u008f\7\t\2\2\u008f\u0090\5\26\f\2\u0090\u0091\7"+
+    "P\2\2\u0091\u0092\7\63\2\2\u0092\u0093\5\36\20\2\u0093\u0094\7\n\2\2\u0094"+
+    "\u0095\5\n\6\2\u0095\u00b0\3\2\2\2\u0096\u0097\5\24\13\2\u0097\u0098\5"+
+    "\34\17\2\u0098\u00b0\3\2\2\2\u0099\u009a\7\23\2\2\u009a\u00b0\5\34\17"+
+    "\2\u009b\u009c\7\24\2\2\u009c\u00b0\5\34\17\2\u009d\u009e\7\25\2\2\u009e"+
+    "\u009f\5\36\20\2\u009f\u00a0\5\34\17\2\u00a0\u00b0\3\2\2\2\u00a1\u00a2"+
+    "\7\27\2\2\u00a2\u00a4\5\f\7\2\u00a3\u00a5\5\32\16\2\u00a4\u00a3\3\2\2"+
+    "\2\u00a5\u00a6\3\2\2\2\u00a6\u00a4\3\2\2\2\u00a6\u00a7\3\2\2\2\u00a7\u00b0"+
+    "\3\2\2\2\u00a8\u00a9\7\31\2\2\u00a9\u00aa\5\36\20\2\u00aa\u00ab\5\34\17"+
+    "\2\u00ab\u00b0\3\2\2\2\u00ac\u00ad\5\36\20\2\u00ad\u00ae\5\34\17\2\u00ae"+
+    "\u00b0\3\2\2\2\u00afa\3\2\2\2\u00afk\3\2\2\2\u00afs\3\2\2\2\u00af{\3\2"+
+    "\2\2\u00af\u008d\3\2\2\2\u00af\u0096\3\2\2\2\u00af\u0099\3\2\2\2\u00af"+
+    "\u009b\3\2\2\2\u00af\u009d\3\2\2\2\u00af\u00a1\3\2\2\2\u00af\u00a8\3\2"+
+    "\2\2\u00af\u00ac\3\2\2\2\u00b0\t\3\2\2\2\u00b1\u00b4\5\f\7\2\u00b2\u00b4"+
+    "\5\b\5\2\u00b3\u00b1\3\2\2\2\u00b3\u00b2\3\2\2\2\u00b4\13\3\2\2\2\u00b5"+
+    "\u00b9\7\5\2\2\u00b6\u00b8\5\b\5\2\u00b7\u00b6\3\2\2\2\u00b8\u00bb\3\2"+
+    "\2\2\u00b9\u00b7\3\2\2\2\u00b9\u00ba\3\2\2\2\u00ba\u00bc\3\2\2\2\u00bb"+
+    "\u00b9\3\2\2\2\u00bc\u00bd\7\6\2\2\u00bd\r\3\2\2\2\u00be\u00bf\7\r\2\2"+
+    "\u00bf\17\3\2\2\2\u00c0\u00c3\5\24\13\2\u00c1\u00c3\5\36\20\2\u00c2\u00c0"+
+    "\3\2\2\2\u00c2\u00c1\3\2\2\2\u00c3\21\3\2\2\2\u00c4\u00c5\5\36\20\2\u00c5"+
+    "\23\3\2\2\2\u00c6\u00c7\5\26\f\2\u00c7\u00cc\5\30\r\2\u00c8\u00c9\7\f"+
+    "\2\2\u00c9\u00cb\5\30\r\2\u00ca\u00c8\3\2\2\2\u00cb\u00ce\3\2\2\2\u00cc"+
+    "\u00ca\3\2\2\2\u00cc\u00cd\3\2\2\2\u00cd\25\3\2\2\2\u00ce\u00cc\3\2\2"+
+    "\2\u00cf\u00d4\7O\2\2\u00d0\u00d1\7\7\2\2\u00d1\u00d3\7\b\2\2\u00d2\u00d0"+
+    "\3\2\2\2\u00d3\u00d6\3\2\2\2\u00d4\u00d2\3\2\2\2\u00d4\u00d5\3\2\2\2\u00d5"+
+    "\27\3\2\2\2\u00d6\u00d4\3\2\2\2\u00d7\u00da\7P\2\2\u00d8\u00d9\7:\2\2"+
+    "\u00d9\u00db\5\36\20\2\u00da\u00d8\3\2\2\2\u00da\u00db\3\2\2\2\u00db\31"+
+    "\3\2\2\2\u00dc\u00dd\7\30\2\2\u00dd\u00de\7\t\2\2\u00de\u00df\7O\2\2\u00df"+
+    "\u00e0\7P\2\2\u00e0\u00e1\7\n\2\2\u00e1\u00e2\5\f\7\2\u00e2\33\3\2\2\2"+
+    "\u00e3\u00e9\7\r\2\2\u00e4\u00e9\7\2\2\3\u00e5\u00e6\b\17\1\2\u00e6\u00e7"+
+    "\7\6\2\2\u00e7\u00e9\b\17\1\2\u00e8\u00e3\3\2\2\2\u00e8\u00e4\3\2\2\2"+
+    "\u00e8\u00e5\3\2\2\2\u00e9\35\3\2\2\2\u00ea\u00eb\b\20\1\2\u00eb\u00ec"+
+    "\5\"\22\2\u00ec\u00ed\t\2\2\2\u00ed\u00ee\5\36\20\3\u00ee\u00ef\b\20\1"+
+    "\2\u00ef\u00f4\3\2\2\2\u00f0\u00f1\5 \21\2\u00f1\u00f2\b\20\1\2\u00f2"+
+    "\u00f4\3\2\2\2\u00f3\u00ea\3\2\2\2\u00f3\u00f0\3\2\2\2\u00f4\u0135\3\2"+
+    "\2\2\u00f5\u00f6\f\17\2\2\u00f6\u00f7\t\3\2\2\u00f7\u00f8\5\36\20\20\u00f8"+
+    "\u00f9\b\20\1\2\u00f9\u0134\3\2\2\2\u00fa\u00fb\f\16\2\2\u00fb\u00fc\t"+
+    "\4\2\2\u00fc\u00fd\5\36\20\17\u00fd\u00fe\b\20\1\2\u00fe\u0134\3\2\2\2"+
+    "\u00ff\u0100\f\r\2\2\u0100\u0101\t\5\2\2\u0101\u0102\5\36\20\16\u0102"+
+    "\u0103\b\20\1\2\u0103\u0134\3\2\2\2\u0104\u0105\f\f\2\2\u0105\u0106\t"+
+    "\6\2\2\u0106\u0107\5\36\20\r\u0107\u0108\b\20\1\2\u0108\u0134\3\2\2\2"+
+    "\u0109\u010a\f\13\2\2\u010a\u010b\t\7\2\2\u010b\u010c\5\36\20\f\u010c"+
+    "\u010d\b\20\1\2\u010d\u0134\3\2\2\2\u010e\u010f\f\n\2\2\u010f\u0110\t"+
+    "\b\2\2\u0110\u0111\5\36\20\13\u0111\u0112\b\20\1\2\u0112\u0134\3\2\2\2"+
+    "\u0113\u0114\f\t\2\2\u0114\u0115\7-\2\2\u0115\u0116\5\36\20\n\u0116\u0117"+
+    "\b\20\1\2\u0117\u0134\3\2\2\2\u0118\u0119\f\b\2\2\u0119\u011a\7.\2\2\u011a"+
+    "\u011b\5\36\20\t\u011b\u011c\b\20\1\2\u011c\u0134\3\2\2\2\u011d\u011e"+
+    "\f\7\2\2\u011e\u011f\7/\2\2\u011f\u0120\5\36\20\b\u0120\u0121\b\20\1\2"+
+    "\u0121\u0134\3\2\2\2\u0122\u0123\f\6\2\2\u0123\u0124\7\60\2\2\u0124\u0125"+
+    "\5\36\20\7\u0125\u0126\b\20\1\2\u0126\u0134\3\2\2\2\u0127\u0128\f\5\2"+
+    "\2\u0128\u0129\7\61\2\2\u0129\u012a\5\36\20\6\u012a\u012b\b\20\1\2\u012b"+
+    "\u0134\3\2\2\2\u012c\u012d\f\4\2\2\u012d\u012e\7\62\2\2\u012e\u012f\5"+
+    "\36\20\2\u012f\u0130\7\63\2\2\u0130\u0131\5\36\20\4\u0131\u0132\b\20\1"+
+    "\2\u0132\u0134\3\2\2\2\u0133\u00f5\3\2\2\2\u0133\u00fa\3\2\2\2\u0133\u00ff"+
+    "\3\2\2\2\u0133\u0104\3\2\2\2\u0133\u0109\3\2\2\2\u0133\u010e\3\2\2\2\u0133"+
+    "\u0113\3\2\2\2\u0133\u0118\3\2\2\2\u0133\u011d\3\2\2\2\u0133\u0122\3\2"+
+    "\2\2\u0133\u0127\3\2\2\2\u0133\u012c\3\2\2\2\u0134\u0137\3\2\2\2\u0135"+
+    "\u0133\3\2\2\2\u0135\u0136\3\2\2\2\u0136\37\3\2\2\2\u0137\u0135\3\2\2"+
+    "\2\u0138\u0139\6\21\17\3\u0139\u013a\t\t\2\2\u013a\u0156\5\"\22\2\u013b"+
+    "\u013c\6\21\20\3\u013c\u013d\5\"\22\2\u013d\u013e\t\t\2\2\u013e\u0156"+
+    "\3\2\2\2\u013f\u0140\6\21\21\3\u0140\u0156\5\"\22\2\u0141\u0142\6\21\22"+
+    "\3\u0142\u0143\t\n\2\2\u0143\u0156\b\21\1\2\u0144\u0145\6\21\23\3\u0145"+
+    "\u0146\7L\2\2\u0146\u0156\b\21\1\2\u0147\u0148\6\21\24\3\u0148\u0149\7"+
+    "M\2\2\u0149\u0156\b\21\1\2\u014a\u014b\6\21\25\3\u014b\u014c\7N\2\2\u014c"+
+    "\u0156\b\21\1\2\u014d\u014e\6\21\26\3\u014e\u014f\t\13\2\2\u014f\u0156"+
+    "\5 \21\2\u0150\u0151\7\t\2\2\u0151\u0152\5\26\f\2\u0152\u0153\7\n\2\2"+
+    "\u0153\u0154\5 \21\2\u0154\u0156\3\2\2\2\u0155\u0138\3\2\2\2\u0155\u013b"+
+    "\3\2\2\2\u0155\u013f\3\2\2\2\u0155\u0141\3\2\2\2\u0155\u0144\3\2\2\2\u0155"+
+    "\u0147\3\2\2\2\u0155\u014a\3\2\2\2\u0155\u014d\3\2\2\2\u0155\u0150\3\2"+
+    "\2\2\u0156!\3\2\2\2\u0157\u015b\5$\23\2\u0158\u015a\5&\24\2\u0159\u0158"+
+    "\3\2\2\2\u015a\u015d\3\2\2\2\u015b\u0159\3\2\2\2\u015b\u015c\3\2\2\2\u015c"+
+    "\u017a\3\2\2\2\u015d\u015b\3\2\2\2\u015e\u015f\5\26\f\2\u015f\u0163\5"+
+    "(\25\2\u0160\u0162\5&\24\2\u0161\u0160\3\2\2\2\u0162\u0165\3\2\2\2\u0163"+
+    "\u0161\3\2\2\2\u0163\u0164\3\2\2\2\u0164\u017a\3\2\2\2\u0165\u0163\3\2"+
+    "\2\2\u0166\u0167\7\26\2\2\u0167\u016c\7O\2\2\u0168\u0169\7\7\2\2\u0169"+
+    "\u016a\5\36\20\2\u016a\u016b\7\b\2\2\u016b\u016d\3\2\2\2\u016c\u0168\3"+
+    "\2\2\2\u016d\u016e\3\2\2\2\u016e\u016c\3\2\2\2\u016e\u016f\3\2\2\2\u016f"+
+    "\u0177\3\2\2\2\u0170\u0174\5(\25\2\u0171\u0173\5&\24\2\u0172\u0171\3\2"+
+    "\2\2\u0173\u0176\3\2\2\2\u0174\u0172\3\2\2\2\u0174\u0175\3\2\2\2\u0175"+
+    "\u0178\3\2\2\2\u0176\u0174\3\2\2\2\u0177\u0170\3\2\2\2\u0177\u0178\3\2"+
+    "\2\2\u0178\u017a\3\2\2\2\u0179\u0157\3\2\2\2\u0179\u015e\3\2\2\2\u0179"+
+    "\u0166\3\2\2\2\u017a#\3\2\2\2\u017b\u017c\6\23\27\3\u017c\u017d\7\t\2"+
+    "\2\u017d\u017e\5\36\20\2\u017e\u017f\7\n\2\2\u017f\u0180\b\23\1\2\u0180"+
+    "\u018f\3\2\2\2\u0181\u0182\6\23\30\3\u0182\u0183\7\t\2\2\u0183\u0184\5"+
+    " \21\2\u0184\u0185\7\n\2\2\u0185\u018f\3\2\2\2\u0186\u018f\7J\2\2\u0187"+
+    "\u018f\7K\2\2\u0188\u018f\7P\2\2\u0189\u018a\7P\2\2\u018a\u018f\5,\27"+
+    "\2\u018b\u018c\7\26\2\2\u018c\u018d\7O\2\2\u018d\u018f\5,\27\2\u018e\u017b"+
+    "\3\2\2\2\u018e\u0181\3\2\2\2\u018e\u0186\3\2\2\2\u018e\u0187\3\2\2\2\u018e"+
+    "\u0188\3\2\2\2\u018e\u0189\3\2\2\2\u018e\u018b\3\2\2\2\u018f%\3\2\2\2"+
+    "\u0190\u0191\6\24\31\3\u0191\u0195\5(\25\2\u0192\u0193\6\24\32\3\u0193"+
+    "\u0195\5*\26\2\u0194\u0190\3\2\2\2\u0194\u0192\3\2\2\2\u0195\'\3\2\2\2"+
+    "\u0196\u0197\7\13\2\2\u0197\u0198\7R\2\2\u0198\u019c\5,\27\2\u0199\u019a"+
+    "\7\13\2\2\u019a\u019c\t\f\2\2\u019b\u0196\3\2\2\2\u019b\u0199\3\2\2\2"+
+    "\u019c)\3\2\2\2\u019d\u019e\7\7\2\2\u019e\u019f\5\36\20\2\u019f\u01a0"+
+    "\7\b\2\2\u01a0+\3\2\2\2\u01a1\u01aa\7\t\2\2\u01a2\u01a7\5.\30\2\u01a3"+
+    "\u01a4\7\f\2\2\u01a4\u01a6\5.\30\2\u01a5\u01a3\3\2\2\2\u01a6\u01a9\3\2"+
+    "\2\2\u01a7\u01a5\3\2\2\2\u01a7\u01a8\3\2\2\2\u01a8\u01ab\3\2\2\2\u01a9"+
+    "\u01a7\3\2\2\2\u01aa\u01a2\3\2\2\2\u01aa\u01ab\3\2\2\2\u01ab\u01ac\3\2"+
+    "\2\2\u01ac\u01ad\7\n\2\2\u01ad-\3\2\2\2\u01ae\u01b2\5\36\20\2\u01af\u01b2"+
+    "\5\60\31\2\u01b0\u01b2\5\64\33\2\u01b1\u01ae\3\2\2\2\u01b1\u01af\3\2\2"+
+    "\2\u01b1\u01b0\3\2\2\2\u01b2/\3\2\2\2\u01b3\u01c1\5\62\32\2\u01b4\u01bd"+
+    "\7\t\2\2\u01b5\u01ba\5\62\32\2\u01b6\u01b7\7\f\2\2\u01b7\u01b9\5\62\32"+
+    "\2\u01b8\u01b6\3\2\2\2\u01b9\u01bc\3\2\2\2\u01ba\u01b8\3\2\2\2\u01ba\u01bb"+
+    "\3\2\2\2\u01bb\u01be\3\2\2\2\u01bc\u01ba\3\2\2\2\u01bd\u01b5\3\2\2\2\u01bd"+
+    "\u01be\3\2\2\2\u01be\u01bf\3\2\2\2\u01bf\u01c1\7\n\2\2\u01c0\u01b3\3\2"+
+    "\2\2\u01c0\u01b4\3\2\2\2\u01c1\u01c2\3\2\2\2\u01c2\u01c5\7\65\2\2\u01c3"+
+    "\u01c6\5\f\7\2\u01c4\u01c6\5\36\20\2\u01c5\u01c3\3\2\2\2\u01c5\u01c4\3"+
+    "\2\2\2\u01c6\61\3\2\2\2\u01c7\u01c9\5\26\f\2\u01c8\u01c7\3\2\2\2\u01c8"+
+    "\u01c9\3\2\2\2\u01c9\u01ca\3\2\2\2\u01ca\u01cb\7P\2\2\u01cb\63\3\2\2\2"+
+    "\u01cc\u01d1\5\66\34\2\u01cd\u01d1\58\35\2\u01ce\u01d1\5:\36\2\u01cf\u01d1"+
+    "\5<\37\2\u01d0\u01cc\3\2\2\2\u01d0\u01cd\3\2\2\2\u01d0\u01ce\3\2\2\2\u01d0"+
+    "\u01cf\3\2\2\2\u01d1\65\3\2\2\2\u01d2\u01d3\7O\2\2\u01d3\u01d4\7\64\2"+
+    "\2\u01d4\u01d5\7P\2\2\u01d5\67\3\2\2\2\u01d6\u01d7\5\26\f\2\u01d7\u01d8"+
+    "\7\64\2\2\u01d8\u01d9\7\26\2\2\u01d99\3\2\2\2\u01da\u01db\7P\2\2\u01db"+
+    "\u01dc\7\64\2\2\u01dc\u01dd\7P\2\2\u01dd;\3\2\2\2\u01de\u01df\7\32\2\2"+
+    "\u01df\u01e0\7\64\2\2\u01e0\u01e1\7P\2\2\u01e1=\3\2\2\2+AGZ]iq~\u0082"+
     "\u0086\u008b\u00a6\u00af\u00b3\u00b9\u00c2\u00cc\u00d4\u00da\u00e8\u00f3"+
     "\u0133\u0135\u0155\u015b\u0163\u016e\u0174\u0177\u0179\u018e\u0194\u019b"+
-    "\u01a7\u01aa\u01b1\u01ba\u01bd\u01c0\u01c6\u01ce";
+    "\u01a7\u01aa\u01b1\u01ba\u01bd\u01c0\u01c5\u01c8\u01d0";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index f57e3aef049..b1a5ed3a260 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -959,8 +959,14 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
             paramNames.add(lamtype.ID().getText());
         }
 
-        for (StatementContext statement : ctx.block().statement()) {
-            statements.add((AStatement)visit(statement));
+        if (ctx.expression() != null) {
+            // single expression
+            AExpression expression = (AExpression) visitExpression(ctx.expression());
+            statements.add(new SReturn(location(ctx), expression));
+        } else {
+            for (StatementContext statement : ctx.block().statement()) {
+                statements.add((AStatement)visit(statement));
+            }
         }
         
         String name = nextLambda();
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
index 1e7eb78be73..9db0e5c93ae 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
@@ -22,59 +22,63 @@ package org.elasticsearch.painless;
 public class LambdaTests extends ScriptTestCase {
 
     public void testNoArgLambda() {
-        assertEquals(1, exec("Optional.empty().orElseGet(() -> { return 1; });"));
+        assertEquals(1, exec("Optional.empty().orElseGet(() -> 1);"));
     }
     
     public void testNoArgLambdaDef() {
-        assertEquals(1, exec("def x = Optional.empty(); x.orElseGet(() -> { return 1; });"));
+        assertEquals(1, exec("def x = Optional.empty(); x.orElseGet(() -> 1);"));
     }
     
     public void testLambdaWithArgs() {
         assertEquals("short", exec("List l = new ArrayList(); l.add('looooong'); l.add('short'); " 
-                                 + "l.sort((a, b) -> { a.length() - b.length(); }); return l.get(0)"));
+                                 + "l.sort((a, b) -> a.length() - b.length()); return l.get(0)"));
 
     }
     
     public void testLambdaWithTypedArgs() {
         assertEquals("short", exec("List l = new ArrayList(); l.add('looooong'); l.add('short'); " 
-                                 + "l.sort((String a, String b) -> { (a.length() - b.length()); }); return l.get(0)"));
+                                 + "l.sort((String a, String b) -> a.length() - b.length()); return l.get(0)"));
 
     }
     
     public void testPrimitiveLambdas() {
         assertEquals(4, exec("List l = new ArrayList(); l.add(1); l.add(1); "
-                           + "return l.stream().mapToInt(x -> { x + 1; }).sum();"));
+                           + "return l.stream().mapToInt(x -> x + 1).sum();"));
     }
     
     public void testPrimitiveLambdasWithTypedArgs() {
         assertEquals(4, exec("List l = new ArrayList(); l.add(1); l.add(1); "
-                           + "return l.stream().mapToInt(int x -> { x + 1; }).sum();"));
+                           + "return l.stream().mapToInt(int x -> x + 1).sum();"));
     }
     
     public void testPrimitiveLambdasDef() {
         assertEquals(4, exec("def l = new ArrayList(); l.add(1); l.add(1); "
-                           + "return l.stream().mapToInt(x -> { x + 1; }).sum();"));
+                           + "return l.stream().mapToInt(x -> x + 1).sum();"));
     }
     
     public void testPrimitiveLambdasWithTypedArgsDef() {
         assertEquals(4, exec("def l = new ArrayList(); l.add(1); l.add(1); "
-                           + "return l.stream().mapToInt(int x -> { x + 1; }).sum();"));
+                           + "return l.stream().mapToInt(int x -> x + 1).sum();"));
     }
     
     public void testPrimitiveLambdasConvertible() {
         assertEquals(2, exec("List l = new ArrayList(); l.add(1); l.add(1); "
-                           + "return l.stream().mapToInt(byte x -> { return x; }).sum();"));
+                           + "return l.stream().mapToInt(byte x -> x).sum();"));
     }
     
     public void testPrimitiveArgs() {
-        assertEquals(2, exec("int applyOne(IntFunction arg) { arg.apply(1) } applyOne(x -> { x + 1; })"));
+        assertEquals(2, exec("int applyOne(IntFunction arg) { arg.apply(1) } applyOne(x -> x + 1)"));
     }
     
     public void testPrimitiveArgsTyped() {
-        assertEquals(2, exec("int applyOne(IntFunction arg) { arg.apply(1) } applyOne(int x -> { x + 1; })"));
+        assertEquals(2, exec("int applyOne(IntFunction arg) { arg.apply(1) } applyOne(int x -> x + 1)"));
     }
     
     public void testPrimitiveArgsTypedOddly() {
-        assertEquals(2L, exec("long applyOne(IntFunction arg) { arg.apply(1) } applyOne(long x -> { x + 1; })"));
+        assertEquals(2L, exec("long applyOne(IntFunction arg) { arg.apply(1) } applyOne(long x -> x + 1)"));
+    }
+
+    public void testMultipleStatements() {
+        assertEquals(2, exec("int applyOne(IntFunction arg) { arg.apply(1) } applyOne(x -> { x = x + 1; return x;})"));
     }
 }

From 251001e435c8d693f8835423f7f985c0bec4f462 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Thu, 16 Jun 2016 12:40:30 -0400
Subject: [PATCH 0914/1311] add more simple tests

---
 .../elasticsearch/painless/LambdaTests.java    | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
index 9db0e5c93ae..5001e28a954 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
@@ -81,4 +81,22 @@ public class LambdaTests extends ScriptTestCase {
     public void testMultipleStatements() {
         assertEquals(2, exec("int applyOne(IntFunction arg) { arg.apply(1) } applyOne(x -> { x = x + 1; return x;})"));
     }
+
+    public void testTwoLambdas() {
+        assertEquals("testingcdefg", exec(
+                "org.elasticsearch.painless.FeatureTest test = new org.elasticsearch.painless.FeatureTest(2,3);" +
+                "return test.twoFunctionsOfX(x -> 'testing'.concat(x), y -> 'abcdefg'.substring(y))"));
+    }
+
+    public void testNestedLambdas() {
+        assertEquals(1, exec("Optional.empty().orElseGet(() -> Optional.empty().orElseGet(() -> 1));"));
+    }
+
+    public void testLambdaInLoop() {
+        assertEquals(100, exec("int sum = 0; " +
+                               "for (int i = 0; i < 100; i++) {" +
+                               "  sum += Optional.empty().orElseGet(() -> 1);" +
+                               "}" +
+                               "return sum;"));
+    }
 }

From 96321d77490f8eba8ae010e5fe722870fe742a46 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Thu, 16 Jun 2016 10:18:10 -0700
Subject: [PATCH 0915/1311] Remove outtdated comment referring to
 name/description for Plugin class

---
 .../elasticsearch/indices/analysis/AnalysisModule.java | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
index 02aca17e03c..b5b480fc909 100644
--- a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
+++ b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
@@ -54,16 +54,6 @@ import java.util.Map;
  * The {@link org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider} is only a functional interface that allows to register factory constructors directly like the plugin example below:
  * <pre>
  *     public class MyAnalysisPlugin extends Plugin {
- *       \@Override
- *       public String name() {
- *         return "analysis-my-plugin";
- *       }
- *
- *       \@Override
- *       public String description() {
- *         return "my very fast and efficient analyzer";
- *       }
- *
  *       public void onModule(AnalysisModule module) {
  *         module.registerAnalyzer("my-analyzer-name", MyAnalyzer::new);
  *       }

From b665d8a18752aa3000c1a451472f5dec040eedcf Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 16 Jun 2016 11:07:09 -0400
Subject: [PATCH 0916/1311] Painless: Add flag support to regexes

Painless: Add support for //m
Painless: Add support for //s
Painless: Add support for //i
Painless: Add support for //u
Painless: Add support for //U
Painless: Add support for //l
  This means "literal" and is exposed for completeness sake with
  the java api.
Painless: Add support for //c
  c enables Java's CANON_EQ (canonical equivalence) flag which makes
  unicode characters that are canonically equal match. Java's javadoc
  gives "a\u030A" being equal to "\u00E5". That is that the "a" code
  point followed by the "combining ring above" code point is equal to
  the "a with combining ring above" code point.
Update docs and add multi-flag test
Whitelist most of the Pattern class.
---
 .../modules/scripting/painless.asciidoc       |  30 +-
 .../src/main/antlr/PainlessLexer.g4           |   2 +-
 .../painless/WriterConstants.java             |   2 +-
 .../painless/antlr/PainlessLexer.java         | 333 +++++++++---------
 .../elasticsearch/painless/antlr/Walker.java  |   7 +-
 .../elasticsearch/painless/node/LRegex.java   |  25 +-
 .../painless/java.util.regex.txt              |   7 +
 .../elasticsearch/painless/RegexTests.java    |  57 +++
 8 files changed, 289 insertions(+), 174 deletions(-)

diff --git a/docs/reference/modules/scripting/painless.asciidoc b/docs/reference/modules/scripting/painless.asciidoc
index cb28e735e5c..20b301bda31 100644
--- a/docs/reference/modules/scripting/painless.asciidoc
+++ b/docs/reference/modules/scripting/painless.asciidoc
@@ -202,12 +202,15 @@ POST hockey/player/1/_update
 // CONSOLE
 
 [float]
+[[modules-scripting-painless-regex]]
 === Regular expressions
 
 Painless's native support for regular expressions has syntax constructs:
 
 * `/pattern/`: Pattern literals create patterns. This is the only way to create
-a pattern in painless.
+a pattern in painless. The pattern inside the `/`s are just
+http://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html[Java regular expressions].
+See <<modules-scripting-painless-regex-flags>> for more.
 * `=~`: The find operator return a `boolean`, `true` if a subsequence of the
 text matches, `false` otherwise.
 * `==~`: The match operator returns a `boolean`, `true` if the text matches,
@@ -265,14 +268,35 @@ Note: all of the `_update_by_query` examples above could really do with a
 because script queries aren't able to use the inverted index to limit the
 documents that they have to check.
 
-The pattern syntax is just
-http://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html[Java regular expressions].
 We intentionally don't allow scripts to call `Pattern.compile` to get a new
 pattern on the fly because building a `Pattern` is (comparatively) slow.
 Pattern literals (`/apattern/`) have fancy constant extraction so no matter
 where they show up in the painless script they are built only when the script
 is first used. It is fairly similar to how `String` literals work in Java.
 
+[float]
+[[modules-scripting-painless-regex-flags]]
+==== Regular expression flags
+
+You can define flags on patterns in Painless by adding characters after the
+trailing `/` like `/foo/i` or `/foo \w #comment/iUx`. Painless exposes all the
+flags from
+https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html[Java's Pattern class]
+using these characters:
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+| Character | Java Constant | Example
+|`c` | CANON_EQ                  | `'Ã¥' ==~ /aÌŠ/c` (open in hex editor to see)
+|`i` | CASE_INSENSITIVE          | `'A' ==~ /a/i`
+|`l` | LITERAL                   | `'[a]' ==~ /[a]/l`
+|`m` | MULTILINE                 | `'a\nb\nc' =~ /^b$/m`
+|`s` | DOTALL (aka single line)  | `'a\nb\nc' =~ /.b./s`
+|`U` | UNICODE_CHARACTER_CLASS   | `'�' ==~ /\\w/U`
+|`u` | UNICODE_CASE              | `'�' ==~ /ɛ/iu`
+|`x` | COMMENTS (aka extended)   | `'a' ==~ /a #comment/x`
+|=======================================================================
+
 
 [[painless-api]]
 [float]
diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.g4 b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
index a31c50ee198..62a154db40c 100644
--- a/modules/lang-painless/src/main/antlr/PainlessLexer.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
@@ -104,7 +104,7 @@ INTEGER: ( '0' | [1-9] [0-9]* ) [lLfFdD]?;
 DECIMAL: ( '0' | [1-9] [0-9]* ) (DOT [0-9]+)? ( [eE] [+\-]? [0-9]+ )? [fF]?;
 
 STRING: ( '"' ( '\\"' | '\\\\' | ~[\\"] )*? '"' ) | ( '\'' ( '\\\'' | '\\\\' | ~[\\"] )*? '\'' );
-REGEX: '/' ( ~('/' | '\n') | '\\' ~'\n' )+ '/' { SlashStrategy.slashIsRegex(_factory) }?;
+REGEX: '/' ( ~('/' | '\n') | '\\' ~'\n' )+ '/' [cilmsUux]* { SlashStrategy.slashIsRegex(_factory) }?;
 
 TRUE:  'true';
 FALSE: 'false';
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index d798bb5ca68..6960e906339 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -79,7 +79,7 @@ public final class WriterConstants {
      * because it can do it statically. This is both faster and prevents the script from doing something super slow like building a regex
      * per time it is run.
      */
-    public final static Method PATTERN_COMPILE = getAsmMethod(Pattern.class, "compile", String.class);
+    public final static Method PATTERN_COMPILE = getAsmMethod(Pattern.class, "compile", String.class, int.class);
     public final static Method PATTERN_MATCHER = getAsmMethod(Matcher.class, "matcher", CharSequence.class);
     public final static Method MATCHER_MATCHES = getAsmMethod(boolean.class, "matches");
     public final static Method MATCHER_FIND = getAsmMethod(boolean.class, "find");
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
index 040d17c521b..cf0ac8605f0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
@@ -160,7 +160,7 @@ class PainlessLexer extends Lexer {
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2R\u0233\b\1\b\1\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2R\u0239\b\1\b\1\4"+
     "\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n"+
     "\4\13\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22"+
     "\t\22\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31"+
@@ -191,171 +191,174 @@ class PainlessLexer extends Lexer {
     "\5H\u01c8\nH\3H\3H\5H\u01cc\nH\3H\6H\u01cf\nH\rH\16H\u01d0\5H\u01d3\n"+
     "H\3H\5H\u01d6\nH\3I\3I\3I\3I\3I\3I\7I\u01de\nI\fI\16I\u01e1\13I\3I\3I"+
     "\3I\3I\3I\3I\3I\7I\u01ea\nI\fI\16I\u01ed\13I\3I\5I\u01f0\nI\3J\3J\3J\3"+
-    "J\6J\u01f6\nJ\rJ\16J\u01f7\3J\3J\3J\3K\3K\3K\3K\3K\3L\3L\3L\3L\3L\3L\3"+
-    "M\3M\3M\3M\3M\3N\3N\3N\3N\7N\u0211\nN\fN\16N\u0214\13N\3N\3N\3O\3O\7O"+
-    "\u021a\nO\fO\16O\u021d\13O\3P\3P\3P\7P\u0222\nP\fP\16P\u0225\13P\5P\u0227"+
-    "\nP\3P\3P\3Q\3Q\7Q\u022d\nQ\fQ\16Q\u0230\13Q\3Q\3Q\6\u00b1\u00bb\u01df"+
-    "\u01eb\2R\4\3\6\4\b\5\n\6\f\7\16\b\20\t\22\n\24\13\26\f\30\r\32\16\34"+
-    "\17\36\20 \21\"\22$\23&\24(\25*\26,\27.\30\60\31\62\32\64\33\66\348\35"+
-    ":\36<\37> @!B\"D#F$H%J&L\'N(P)R*T+V,X-Z.\\/^\60`\61b\62d\63f\64h\65j\66"+
-    "l\67n8p9r:t;v<x=z>|?~@\u0080A\u0082B\u0084C\u0086D\u0088E\u008aF\u008c"+
-    "G\u008eH\u0090I\u0092J\u0094K\u0096L\u0098M\u009aN\u009cO\u009eP\u00a0"+
-    "Q\u00a2R\4\2\3\23\5\2\13\f\17\17\"\"\4\2\f\f\17\17\3\2\629\4\2NNnn\4\2"+
-    "ZZzz\5\2\62;CHch\3\2\63;\3\2\62;\b\2FFHHNNffhhnn\4\2GGgg\4\2--//\4\2H"+
-    "Hhh\4\2$$^^\4\2\f\f\61\61\3\2\f\f\5\2C\\aac|\6\2\62;C\\aac|\u0252\2\4"+
-    "\3\2\2\2\2\6\3\2\2\2\2\b\3\2\2\2\2\n\3\2\2\2\2\f\3\2\2\2\2\16\3\2\2\2"+
-    "\2\20\3\2\2\2\2\22\3\2\2\2\2\24\3\2\2\2\2\26\3\2\2\2\2\30\3\2\2\2\2\32"+
-    "\3\2\2\2\2\34\3\2\2\2\2\36\3\2\2\2\2 \3\2\2\2\2\"\3\2\2\2\2$\3\2\2\2\2"+
-    "&\3\2\2\2\2(\3\2\2\2\2*\3\2\2\2\2,\3\2\2\2\2.\3\2\2\2\2\60\3\2\2\2\2\62"+
-    "\3\2\2\2\2\64\3\2\2\2\2\66\3\2\2\2\28\3\2\2\2\2:\3\2\2\2\2<\3\2\2\2\2"+
-    ">\3\2\2\2\2@\3\2\2\2\2B\3\2\2\2\2D\3\2\2\2\2F\3\2\2\2\2H\3\2\2\2\2J\3"+
-    "\2\2\2\2L\3\2\2\2\2N\3\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2T\3\2\2\2\2V\3\2\2"+
-    "\2\2X\3\2\2\2\2Z\3\2\2\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3\2\2\2\2b\3\2\2\2"+
-    "\2d\3\2\2\2\2f\3\2\2\2\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2\2\2n\3\2\2\2\2p"+
-    "\3\2\2\2\2r\3\2\2\2\2t\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2z\3\2\2\2\2|\3\2"+
-    "\2\2\2~\3\2\2\2\2\u0080\3\2\2\2\2\u0082\3\2\2\2\2\u0084\3\2\2\2\2\u0086"+
-    "\3\2\2\2\2\u0088\3\2\2\2\2\u008a\3\2\2\2\2\u008c\3\2\2\2\2\u008e\3\2\2"+
-    "\2\2\u0090\3\2\2\2\2\u0092\3\2\2\2\2\u0094\3\2\2\2\2\u0096\3\2\2\2\2\u0098"+
-    "\3\2\2\2\2\u009a\3\2\2\2\2\u009c\3\2\2\2\2\u009e\3\2\2\2\3\u00a0\3\2\2"+
-    "\2\3\u00a2\3\2\2\2\4\u00a5\3\2\2\2\6\u00c0\3\2\2\2\b\u00c4\3\2\2\2\n\u00c6"+
-    "\3\2\2\2\f\u00c8\3\2\2\2\16\u00ca\3\2\2\2\20\u00cc\3\2\2\2\22\u00ce\3"+
-    "\2\2\2\24\u00d0\3\2\2\2\26\u00d4\3\2\2\2\30\u00d6\3\2\2\2\32\u00d8\3\2"+
-    "\2\2\34\u00db\3\2\2\2\36\u00e0\3\2\2\2 \u00e6\3\2\2\2\"\u00e9\3\2\2\2"+
-    "$\u00ed\3\2\2\2&\u00f6\3\2\2\2(\u00fc\3\2\2\2*\u0103\3\2\2\2,\u0107\3"+
-    "\2\2\2.\u010b\3\2\2\2\60\u0111\3\2\2\2\62\u0117\3\2\2\2\64\u011c\3\2\2"+
-    "\2\66\u011e\3\2\2\28\u0120\3\2\2\2:\u0122\3\2\2\2<\u0125\3\2\2\2>\u0127"+
-    "\3\2\2\2@\u0129\3\2\2\2B\u012b\3\2\2\2D\u012e\3\2\2\2F\u0131\3\2\2\2H"+
-    "\u0135\3\2\2\2J\u0137\3\2\2\2L\u013a\3\2\2\2N\u013c\3\2\2\2P\u013f\3\2"+
-    "\2\2R\u0142\3\2\2\2T\u0146\3\2\2\2V\u0149\3\2\2\2X\u014d\3\2\2\2Z\u014f"+
-    "\3\2\2\2\\\u0151\3\2\2\2^\u0153\3\2\2\2`\u0156\3\2\2\2b\u0159\3\2\2\2"+
-    "d\u015b\3\2\2\2f\u015d\3\2\2\2h\u0160\3\2\2\2j\u0163\3\2\2\2l\u0166\3"+
-    "\2\2\2n\u016a\3\2\2\2p\u016d\3\2\2\2r\u0170\3\2\2\2t\u0172\3\2\2\2v\u0175"+
-    "\3\2\2\2x\u0178\3\2\2\2z\u017b\3\2\2\2|\u017e\3\2\2\2~\u0181\3\2\2\2\u0080"+
-    "\u0184\3\2\2\2\u0082\u0187\3\2\2\2\u0084\u018a\3\2\2\2\u0086\u018e\3\2"+
-    "\2\2\u0088\u0192\3\2\2\2\u008a\u0197\3\2\2\2\u008c\u01a0\3\2\2\2\u008e"+
-    "\u01b2\3\2\2\2\u0090\u01bf\3\2\2\2\u0092\u01ef\3\2\2\2\u0094\u01f1\3\2"+
-    "\2\2\u0096\u01fc\3\2\2\2\u0098\u0201\3\2\2\2\u009a\u0207\3\2\2\2\u009c"+
-    "\u020c\3\2\2\2\u009e\u0217\3\2\2\2\u00a0\u0226\3\2\2\2\u00a2\u022a\3\2"+
-    "\2\2\u00a4\u00a6\t\2\2\2\u00a5\u00a4\3\2\2\2\u00a6\u00a7\3\2\2\2\u00a7"+
-    "\u00a5\3\2\2\2\u00a7\u00a8\3\2\2\2\u00a8\u00a9\3\2\2\2\u00a9\u00aa\b\2"+
-    "\2\2\u00aa\5\3\2\2\2\u00ab\u00ac\7\61\2\2\u00ac\u00ad\7\61\2\2\u00ad\u00b1"+
-    "\3\2\2\2\u00ae\u00b0\13\2\2\2\u00af\u00ae\3\2\2\2\u00b0\u00b3\3\2\2\2"+
-    "\u00b1\u00b2\3\2\2\2\u00b1\u00af\3\2\2\2\u00b2\u00b4\3\2\2\2\u00b3\u00b1"+
-    "\3\2\2\2\u00b4\u00c1\t\3\2\2\u00b5\u00b6\7\61\2\2\u00b6\u00b7\7,\2\2\u00b7"+
-    "\u00bb\3\2\2\2\u00b8\u00ba\13\2\2\2\u00b9\u00b8\3\2\2\2\u00ba\u00bd\3"+
-    "\2\2\2\u00bb\u00bc\3\2\2\2\u00bb\u00b9\3\2\2\2\u00bc\u00be\3\2\2\2\u00bd"+
-    "\u00bb\3\2\2\2\u00be\u00bf\7,\2\2\u00bf\u00c1\7\61\2\2\u00c0\u00ab\3\2"+
-    "\2\2\u00c0\u00b5\3\2\2\2\u00c1\u00c2\3\2\2\2\u00c2\u00c3\b\3\2\2\u00c3"+
-    "\7\3\2\2\2\u00c4\u00c5\7}\2\2\u00c5\t\3\2\2\2\u00c6\u00c7\7\177\2\2\u00c7"+
-    "\13\3\2\2\2\u00c8\u00c9\7]\2\2\u00c9\r\3\2\2\2\u00ca\u00cb\7_\2\2\u00cb"+
-    "\17\3\2\2\2\u00cc\u00cd\7*\2\2\u00cd\21\3\2\2\2\u00ce\u00cf\7+\2\2\u00cf"+
-    "\23\3\2\2\2\u00d0\u00d1\7\60\2\2\u00d1\u00d2\3\2\2\2\u00d2\u00d3\b\n\3"+
-    "\2\u00d3\25\3\2\2\2\u00d4\u00d5\7.\2\2\u00d5\27\3\2\2\2\u00d6\u00d7\7"+
-    "=\2\2\u00d7\31\3\2\2\2\u00d8\u00d9\7k\2\2\u00d9\u00da\7h\2\2\u00da\33"+
-    "\3\2\2\2\u00db\u00dc\7g\2\2\u00dc\u00dd\7n\2\2\u00dd\u00de\7u\2\2\u00de"+
-    "\u00df\7g\2\2\u00df\35\3\2\2\2\u00e0\u00e1\7y\2\2\u00e1\u00e2\7j\2\2\u00e2"+
-    "\u00e3\7k\2\2\u00e3\u00e4\7n\2\2\u00e4\u00e5\7g\2\2\u00e5\37\3\2\2\2\u00e6"+
-    "\u00e7\7f\2\2\u00e7\u00e8\7q\2\2\u00e8!\3\2\2\2\u00e9\u00ea\7h\2\2\u00ea"+
-    "\u00eb\7q\2\2\u00eb\u00ec\7t\2\2\u00ec#\3\2\2\2\u00ed\u00ee\7e\2\2\u00ee"+
-    "\u00ef\7q\2\2\u00ef\u00f0\7p\2\2\u00f0\u00f1\7v\2\2\u00f1\u00f2\7k\2\2"+
-    "\u00f2\u00f3\7p\2\2\u00f3\u00f4\7w\2\2\u00f4\u00f5\7g\2\2\u00f5%\3\2\2"+
-    "\2\u00f6\u00f7\7d\2\2\u00f7\u00f8\7t\2\2\u00f8\u00f9\7g\2\2\u00f9\u00fa"+
-    "\7c\2\2\u00fa\u00fb\7m\2\2\u00fb\'\3\2\2\2\u00fc\u00fd\7t\2\2\u00fd\u00fe"+
-    "\7g\2\2\u00fe\u00ff\7v\2\2\u00ff\u0100\7w\2\2\u0100\u0101\7t\2\2\u0101"+
-    "\u0102\7p\2\2\u0102)\3\2\2\2\u0103\u0104\7p\2\2\u0104\u0105\7g\2\2\u0105"+
-    "\u0106\7y\2\2\u0106+\3\2\2\2\u0107\u0108\7v\2\2\u0108\u0109\7t\2\2\u0109"+
-    "\u010a\7{\2\2\u010a-\3\2\2\2\u010b\u010c\7e\2\2\u010c\u010d\7c\2\2\u010d"+
-    "\u010e\7v\2\2\u010e\u010f\7e\2\2\u010f\u0110\7j\2\2\u0110/\3\2\2\2\u0111"+
-    "\u0112\7v\2\2\u0112\u0113\7j\2\2\u0113\u0114\7t\2\2\u0114\u0115\7q\2\2"+
-    "\u0115\u0116\7y\2\2\u0116\61\3\2\2\2\u0117\u0118\7v\2\2\u0118\u0119\7"+
-    "j\2\2\u0119\u011a\7k\2\2\u011a\u011b\7u\2\2\u011b\63\3\2\2\2\u011c\u011d"+
-    "\7#\2\2\u011d\65\3\2\2\2\u011e\u011f\7\u0080\2\2\u011f\67\3\2\2\2\u0120"+
-    "\u0121\7,\2\2\u01219\3\2\2\2\u0122\u0123\7\61\2\2\u0123\u0124\6\35\2\2"+
-    "\u0124;\3\2\2\2\u0125\u0126\7\'\2\2\u0126=\3\2\2\2\u0127\u0128\7-\2\2"+
-    "\u0128?\3\2\2\2\u0129\u012a\7/\2\2\u012aA\3\2\2\2\u012b\u012c\7>\2\2\u012c"+
-    "\u012d\7>\2\2\u012dC\3\2\2\2\u012e\u012f\7@\2\2\u012f\u0130\7@\2\2\u0130"+
-    "E\3\2\2\2\u0131\u0132\7@\2\2\u0132\u0133\7@\2\2\u0133\u0134\7@\2\2\u0134"+
-    "G\3\2\2\2\u0135\u0136\7>\2\2\u0136I\3\2\2\2\u0137\u0138\7>\2\2\u0138\u0139"+
-    "\7?\2\2\u0139K\3\2\2\2\u013a\u013b\7@\2\2\u013bM\3\2\2\2\u013c\u013d\7"+
-    "@\2\2\u013d\u013e\7?\2\2\u013eO\3\2\2\2\u013f\u0140\7?\2\2\u0140\u0141"+
-    "\7?\2\2\u0141Q\3\2\2\2\u0142\u0143\7?\2\2\u0143\u0144\7?\2\2\u0144\u0145"+
-    "\7?\2\2\u0145S\3\2\2\2\u0146\u0147\7#\2\2\u0147\u0148\7?\2\2\u0148U\3"+
-    "\2\2\2\u0149\u014a\7#\2\2\u014a\u014b\7?\2\2\u014b\u014c\7?\2\2\u014c"+
-    "W\3\2\2\2\u014d\u014e\7(\2\2\u014eY\3\2\2\2\u014f\u0150\7`\2\2\u0150["+
-    "\3\2\2\2\u0151\u0152\7~\2\2\u0152]\3\2\2\2\u0153\u0154\7(\2\2\u0154\u0155"+
-    "\7(\2\2\u0155_\3\2\2\2\u0156\u0157\7~\2\2\u0157\u0158\7~\2\2\u0158a\3"+
-    "\2\2\2\u0159\u015a\7A\2\2\u015ac\3\2\2\2\u015b\u015c\7<\2\2\u015ce\3\2"+
-    "\2\2\u015d\u015e\7<\2\2\u015e\u015f\7<\2\2\u015fg\3\2\2\2\u0160\u0161"+
-    "\7/\2\2\u0161\u0162\7@\2\2\u0162i\3\2\2\2\u0163\u0164\7?\2\2\u0164\u0165"+
-    "\7\u0080\2\2\u0165k\3\2\2\2\u0166\u0167\7?\2\2\u0167\u0168\7?\2\2\u0168"+
-    "\u0169\7\u0080\2\2\u0169m\3\2\2\2\u016a\u016b\7-\2\2\u016b\u016c\7-\2"+
-    "\2\u016co\3\2\2\2\u016d\u016e\7/\2\2\u016e\u016f\7/\2\2\u016fq\3\2\2\2"+
-    "\u0170\u0171\7?\2\2\u0171s\3\2\2\2\u0172\u0173\7-\2\2\u0173\u0174\7?\2"+
-    "\2\u0174u\3\2\2\2\u0175\u0176\7/\2\2\u0176\u0177\7?\2\2\u0177w\3\2\2\2"+
-    "\u0178\u0179\7,\2\2\u0179\u017a\7?\2\2\u017ay\3\2\2\2\u017b\u017c\7\61"+
-    "\2\2\u017c\u017d\7?\2\2\u017d{\3\2\2\2\u017e\u017f\7\'\2\2\u017f\u0180"+
-    "\7?\2\2\u0180}\3\2\2\2\u0181\u0182\7(\2\2\u0182\u0183\7?\2\2\u0183\177"+
-    "\3\2\2\2\u0184\u0185\7`\2\2\u0185\u0186\7?\2\2\u0186\u0081\3\2\2\2\u0187"+
-    "\u0188\7~\2\2\u0188\u0189\7?\2\2\u0189\u0083\3\2\2\2\u018a\u018b\7>\2"+
-    "\2\u018b\u018c\7>\2\2\u018c\u018d\7?\2\2\u018d\u0085\3\2\2\2\u018e\u018f"+
-    "\7@\2\2\u018f\u0190\7@\2\2\u0190\u0191\7?\2\2\u0191\u0087\3\2\2\2\u0192"+
-    "\u0193\7@\2\2\u0193\u0194\7@\2\2\u0194\u0195\7@\2\2\u0195\u0196\7?\2\2"+
-    "\u0196\u0089\3\2\2\2\u0197\u0199\7\62\2\2\u0198\u019a\t\4\2\2\u0199\u0198"+
-    "\3\2\2\2\u019a\u019b\3\2\2\2\u019b\u0199\3\2\2\2\u019b\u019c\3\2\2\2\u019c"+
-    "\u019e\3\2\2\2\u019d\u019f\t\5\2\2\u019e\u019d\3\2\2\2\u019e\u019f\3\2"+
-    "\2\2\u019f\u008b\3\2\2\2\u01a0\u01a1\7\62\2\2\u01a1\u01a3\t\6\2\2\u01a2"+
-    "\u01a4\t\7\2\2\u01a3\u01a2\3\2\2\2\u01a4\u01a5\3\2\2\2\u01a5\u01a3\3\2"+
-    "\2\2\u01a5\u01a6\3\2\2\2\u01a6\u01a8\3\2\2\2\u01a7\u01a9\t\5\2\2\u01a8"+
-    "\u01a7\3\2\2\2\u01a8\u01a9\3\2\2\2\u01a9\u008d\3\2\2\2\u01aa\u01b3\7\62"+
-    "\2\2\u01ab\u01af\t\b\2\2\u01ac\u01ae\t\t\2\2\u01ad\u01ac\3\2\2\2\u01ae"+
-    "\u01b1\3\2\2\2\u01af\u01ad\3\2\2\2\u01af\u01b0\3\2\2\2\u01b0\u01b3\3\2"+
-    "\2\2\u01b1\u01af\3\2\2\2\u01b2\u01aa\3\2\2\2\u01b2\u01ab\3\2\2\2\u01b3"+
-    "\u01b5\3\2\2\2\u01b4\u01b6\t\n\2\2\u01b5\u01b4\3\2\2\2\u01b5\u01b6\3\2"+
-    "\2\2\u01b6\u008f\3\2\2\2\u01b7\u01c0\7\62\2\2\u01b8\u01bc\t\b\2\2\u01b9"+
-    "\u01bb\t\t\2\2\u01ba\u01b9\3\2\2\2\u01bb\u01be\3\2\2\2\u01bc\u01ba\3\2"+
-    "\2\2\u01bc\u01bd\3\2\2\2\u01bd\u01c0\3\2\2\2\u01be\u01bc\3\2\2\2\u01bf"+
-    "\u01b7\3\2\2\2\u01bf\u01b8\3\2\2\2\u01c0\u01c7\3\2\2\2\u01c1\u01c3\5\24"+
-    "\n\2\u01c2\u01c4\t\t\2\2\u01c3\u01c2\3\2\2\2\u01c4\u01c5\3\2\2\2\u01c5"+
-    "\u01c3\3\2\2\2\u01c5\u01c6\3\2\2\2\u01c6\u01c8\3\2\2\2\u01c7\u01c1\3\2"+
-    "\2\2\u01c7\u01c8\3\2\2\2\u01c8\u01d2\3\2\2\2\u01c9\u01cb\t\13\2\2\u01ca"+
-    "\u01cc\t\f\2\2\u01cb\u01ca\3\2\2\2\u01cb\u01cc\3\2\2\2\u01cc\u01ce\3\2"+
-    "\2\2\u01cd\u01cf\t\t\2\2\u01ce\u01cd\3\2\2\2\u01cf\u01d0\3\2\2\2\u01d0"+
-    "\u01ce\3\2\2\2\u01d0\u01d1\3\2\2\2\u01d1\u01d3\3\2\2\2\u01d2\u01c9\3\2"+
-    "\2\2\u01d2\u01d3\3\2\2\2\u01d3\u01d5\3\2\2\2\u01d4\u01d6\t\r\2\2\u01d5"+
-    "\u01d4\3\2\2\2\u01d5\u01d6\3\2\2\2\u01d6\u0091\3\2\2\2\u01d7\u01df\7$"+
-    "\2\2\u01d8\u01d9\7^\2\2\u01d9\u01de\7$\2\2\u01da\u01db\7^\2\2\u01db\u01de"+
-    "\7^\2\2\u01dc\u01de\n\16\2\2\u01dd\u01d8\3\2\2\2\u01dd\u01da\3\2\2\2\u01dd"+
-    "\u01dc\3\2\2\2\u01de\u01e1\3\2\2\2\u01df\u01e0\3\2\2\2\u01df\u01dd\3\2"+
-    "\2\2\u01e0\u01e2\3\2\2\2\u01e1\u01df\3\2\2\2\u01e2\u01f0\7$\2\2\u01e3"+
-    "\u01eb\7)\2\2\u01e4\u01e5\7^\2\2\u01e5\u01ea\7)\2\2\u01e6\u01e7\7^\2\2"+
-    "\u01e7\u01ea\7^\2\2\u01e8\u01ea\n\16\2\2\u01e9\u01e4\3\2\2\2\u01e9\u01e6"+
-    "\3\2\2\2\u01e9\u01e8\3\2\2\2\u01ea\u01ed\3\2\2\2\u01eb\u01ec\3\2\2\2\u01eb"+
-    "\u01e9\3\2\2\2\u01ec\u01ee\3\2\2\2\u01ed\u01eb\3\2\2\2\u01ee\u01f0\7)"+
-    "\2\2\u01ef\u01d7\3\2\2\2\u01ef\u01e3\3\2\2\2\u01f0\u0093\3\2\2\2\u01f1"+
-    "\u01f5\7\61\2\2\u01f2\u01f6\n\17\2\2\u01f3\u01f4\7^\2\2\u01f4\u01f6\n"+
-    "\20\2\2\u01f5\u01f2\3\2\2\2\u01f5\u01f3\3\2\2\2\u01f6\u01f7\3\2\2\2\u01f7"+
-    "\u01f5\3\2\2\2\u01f7\u01f8\3\2\2\2\u01f8\u01f9\3\2\2\2\u01f9\u01fa\7\61"+
-    "\2\2\u01fa\u01fb\6J\3\2\u01fb\u0095\3\2\2\2\u01fc\u01fd\7v\2\2\u01fd\u01fe"+
-    "\7t\2\2\u01fe\u01ff\7w\2\2\u01ff\u0200\7g\2\2\u0200\u0097\3\2\2\2\u0201"+
-    "\u0202\7h\2\2\u0202\u0203\7c\2\2\u0203\u0204\7n\2\2\u0204\u0205\7u\2\2"+
-    "\u0205\u0206\7g\2\2\u0206\u0099\3\2\2\2\u0207\u0208\7p\2\2\u0208\u0209"+
-    "\7w\2\2\u0209\u020a\7n\2\2\u020a\u020b\7n\2\2\u020b\u009b\3\2\2\2\u020c"+
-    "\u0212\5\u009eO\2\u020d\u020e\5\24\n\2\u020e\u020f\5\u009eO\2\u020f\u0211"+
-    "\3\2\2\2\u0210\u020d\3\2\2\2\u0211\u0214\3\2\2\2\u0212\u0210\3\2\2\2\u0212"+
-    "\u0213\3\2\2\2\u0213\u0215\3\2\2\2\u0214\u0212\3\2\2\2\u0215\u0216\6N"+
-    "\4\2\u0216\u009d\3\2\2\2\u0217\u021b\t\21\2\2\u0218\u021a\t\22\2\2\u0219"+
-    "\u0218\3\2\2\2\u021a\u021d\3\2\2\2\u021b\u0219\3\2\2\2\u021b\u021c\3\2"+
-    "\2\2\u021c\u009f\3\2\2\2\u021d\u021b\3\2\2\2\u021e\u0227\7\62\2\2\u021f"+
-    "\u0223\t\b\2\2\u0220\u0222\t\t\2\2\u0221\u0220\3\2\2\2\u0222\u0225\3\2"+
-    "\2\2\u0223\u0221\3\2\2\2\u0223\u0224\3\2\2\2\u0224\u0227\3\2\2\2\u0225"+
-    "\u0223\3\2\2\2\u0226\u021e\3\2\2\2\u0226\u021f\3\2\2\2\u0227\u0228\3\2"+
-    "\2\2\u0228\u0229\bP\4\2\u0229\u00a1\3\2\2\2\u022a\u022e\t\21\2\2\u022b"+
-    "\u022d\t\22\2\2\u022c\u022b\3\2\2\2\u022d\u0230\3\2\2\2\u022e\u022c\3"+
-    "\2\2\2\u022e\u022f\3\2\2\2\u022f\u0231\3\2\2\2\u0230\u022e\3\2\2\2\u0231"+
-    "\u0232\bQ\4\2\u0232\u00a3\3\2\2\2#\2\3\u00a7\u00b1\u00bb\u00c0\u019b\u019e"+
+    "J\6J\u01f6\nJ\rJ\16J\u01f7\3J\3J\7J\u01fc\nJ\fJ\16J\u01ff\13J\3J\3J\3"+
+    "K\3K\3K\3K\3K\3L\3L\3L\3L\3L\3L\3M\3M\3M\3M\3M\3N\3N\3N\3N\7N\u0217\n"+
+    "N\fN\16N\u021a\13N\3N\3N\3O\3O\7O\u0220\nO\fO\16O\u0223\13O\3P\3P\3P\7"+
+    "P\u0228\nP\fP\16P\u022b\13P\5P\u022d\nP\3P\3P\3Q\3Q\7Q\u0233\nQ\fQ\16"+
+    "Q\u0236\13Q\3Q\3Q\6\u00b1\u00bb\u01df\u01eb\2R\4\3\6\4\b\5\n\6\f\7\16"+
+    "\b\20\t\22\n\24\13\26\f\30\r\32\16\34\17\36\20 \21\"\22$\23&\24(\25*\26"+
+    ",\27.\30\60\31\62\32\64\33\66\348\35:\36<\37> @!B\"D#F$H%J&L\'N(P)R*T"+
+    "+V,X-Z.\\/^\60`\61b\62d\63f\64h\65j\66l\67n8p9r:t;v<x=z>|?~@\u0080A\u0082"+
+    "B\u0084C\u0086D\u0088E\u008aF\u008cG\u008eH\u0090I\u0092J\u0094K\u0096"+
+    "L\u0098M\u009aN\u009cO\u009eP\u00a0Q\u00a2R\4\2\3\24\5\2\13\f\17\17\""+
+    "\"\4\2\f\f\17\17\3\2\629\4\2NNnn\4\2ZZzz\5\2\62;CHch\3\2\63;\3\2\62;\b"+
+    "\2FFHHNNffhhnn\4\2GGgg\4\2--//\4\2HHhh\4\2$$^^\4\2\f\f\61\61\3\2\f\f\t"+
+    "\2WWeekknouuwwzz\5\2C\\aac|\6\2\62;C\\aac|\u0259\2\4\3\2\2\2\2\6\3\2\2"+
+    "\2\2\b\3\2\2\2\2\n\3\2\2\2\2\f\3\2\2\2\2\16\3\2\2\2\2\20\3\2\2\2\2\22"+
+    "\3\2\2\2\2\24\3\2\2\2\2\26\3\2\2\2\2\30\3\2\2\2\2\32\3\2\2\2\2\34\3\2"+
+    "\2\2\2\36\3\2\2\2\2 \3\2\2\2\2\"\3\2\2\2\2$\3\2\2\2\2&\3\2\2\2\2(\3\2"+
+    "\2\2\2*\3\2\2\2\2,\3\2\2\2\2.\3\2\2\2\2\60\3\2\2\2\2\62\3\2\2\2\2\64\3"+
+    "\2\2\2\2\66\3\2\2\2\28\3\2\2\2\2:\3\2\2\2\2<\3\2\2\2\2>\3\2\2\2\2@\3\2"+
+    "\2\2\2B\3\2\2\2\2D\3\2\2\2\2F\3\2\2\2\2H\3\2\2\2\2J\3\2\2\2\2L\3\2\2\2"+
+    "\2N\3\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2T\3\2\2\2\2V\3\2\2\2\2X\3\2\2\2\2Z"+
+    "\3\2\2\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3\2\2\2\2b\3\2\2\2\2d\3\2\2\2\2f\3"+
+    "\2\2\2\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2\2\2n\3\2\2\2\2p\3\2\2\2\2r\3\2\2"+
+    "\2\2t\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2z\3\2\2\2\2|\3\2\2\2\2~\3\2\2\2\2"+
+    "\u0080\3\2\2\2\2\u0082\3\2\2\2\2\u0084\3\2\2\2\2\u0086\3\2\2\2\2\u0088"+
+    "\3\2\2\2\2\u008a\3\2\2\2\2\u008c\3\2\2\2\2\u008e\3\2\2\2\2\u0090\3\2\2"+
+    "\2\2\u0092\3\2\2\2\2\u0094\3\2\2\2\2\u0096\3\2\2\2\2\u0098\3\2\2\2\2\u009a"+
+    "\3\2\2\2\2\u009c\3\2\2\2\2\u009e\3\2\2\2\3\u00a0\3\2\2\2\3\u00a2\3\2\2"+
+    "\2\4\u00a5\3\2\2\2\6\u00c0\3\2\2\2\b\u00c4\3\2\2\2\n\u00c6\3\2\2\2\f\u00c8"+
+    "\3\2\2\2\16\u00ca\3\2\2\2\20\u00cc\3\2\2\2\22\u00ce\3\2\2\2\24\u00d0\3"+
+    "\2\2\2\26\u00d4\3\2\2\2\30\u00d6\3\2\2\2\32\u00d8\3\2\2\2\34\u00db\3\2"+
+    "\2\2\36\u00e0\3\2\2\2 \u00e6\3\2\2\2\"\u00e9\3\2\2\2$\u00ed\3\2\2\2&\u00f6"+
+    "\3\2\2\2(\u00fc\3\2\2\2*\u0103\3\2\2\2,\u0107\3\2\2\2.\u010b\3\2\2\2\60"+
+    "\u0111\3\2\2\2\62\u0117\3\2\2\2\64\u011c\3\2\2\2\66\u011e\3\2\2\28\u0120"+
+    "\3\2\2\2:\u0122\3\2\2\2<\u0125\3\2\2\2>\u0127\3\2\2\2@\u0129\3\2\2\2B"+
+    "\u012b\3\2\2\2D\u012e\3\2\2\2F\u0131\3\2\2\2H\u0135\3\2\2\2J\u0137\3\2"+
+    "\2\2L\u013a\3\2\2\2N\u013c\3\2\2\2P\u013f\3\2\2\2R\u0142\3\2\2\2T\u0146"+
+    "\3\2\2\2V\u0149\3\2\2\2X\u014d\3\2\2\2Z\u014f\3\2\2\2\\\u0151\3\2\2\2"+
+    "^\u0153\3\2\2\2`\u0156\3\2\2\2b\u0159\3\2\2\2d\u015b\3\2\2\2f\u015d\3"+
+    "\2\2\2h\u0160\3\2\2\2j\u0163\3\2\2\2l\u0166\3\2\2\2n\u016a\3\2\2\2p\u016d"+
+    "\3\2\2\2r\u0170\3\2\2\2t\u0172\3\2\2\2v\u0175\3\2\2\2x\u0178\3\2\2\2z"+
+    "\u017b\3\2\2\2|\u017e\3\2\2\2~\u0181\3\2\2\2\u0080\u0184\3\2\2\2\u0082"+
+    "\u0187\3\2\2\2\u0084\u018a\3\2\2\2\u0086\u018e\3\2\2\2\u0088\u0192\3\2"+
+    "\2\2\u008a\u0197\3\2\2\2\u008c\u01a0\3\2\2\2\u008e\u01b2\3\2\2\2\u0090"+
+    "\u01bf\3\2\2\2\u0092\u01ef\3\2\2\2\u0094\u01f1\3\2\2\2\u0096\u0202\3\2"+
+    "\2\2\u0098\u0207\3\2\2\2\u009a\u020d\3\2\2\2\u009c\u0212\3\2\2\2\u009e"+
+    "\u021d\3\2\2\2\u00a0\u022c\3\2\2\2\u00a2\u0230\3\2\2\2\u00a4\u00a6\t\2"+
+    "\2\2\u00a5\u00a4\3\2\2\2\u00a6\u00a7\3\2\2\2\u00a7\u00a5\3\2\2\2\u00a7"+
+    "\u00a8\3\2\2\2\u00a8\u00a9\3\2\2\2\u00a9\u00aa\b\2\2\2\u00aa\5\3\2\2\2"+
+    "\u00ab\u00ac\7\61\2\2\u00ac\u00ad\7\61\2\2\u00ad\u00b1\3\2\2\2\u00ae\u00b0"+
+    "\13\2\2\2\u00af\u00ae\3\2\2\2\u00b0\u00b3\3\2\2\2\u00b1\u00b2\3\2\2\2"+
+    "\u00b1\u00af\3\2\2\2\u00b2\u00b4\3\2\2\2\u00b3\u00b1\3\2\2\2\u00b4\u00c1"+
+    "\t\3\2\2\u00b5\u00b6\7\61\2\2\u00b6\u00b7\7,\2\2\u00b7\u00bb\3\2\2\2\u00b8"+
+    "\u00ba\13\2\2\2\u00b9\u00b8\3\2\2\2\u00ba\u00bd\3\2\2\2\u00bb\u00bc\3"+
+    "\2\2\2\u00bb\u00b9\3\2\2\2\u00bc\u00be\3\2\2\2\u00bd\u00bb\3\2\2\2\u00be"+
+    "\u00bf\7,\2\2\u00bf\u00c1\7\61\2\2\u00c0\u00ab\3\2\2\2\u00c0\u00b5\3\2"+
+    "\2\2\u00c1\u00c2\3\2\2\2\u00c2\u00c3\b\3\2\2\u00c3\7\3\2\2\2\u00c4\u00c5"+
+    "\7}\2\2\u00c5\t\3\2\2\2\u00c6\u00c7\7\177\2\2\u00c7\13\3\2\2\2\u00c8\u00c9"+
+    "\7]\2\2\u00c9\r\3\2\2\2\u00ca\u00cb\7_\2\2\u00cb\17\3\2\2\2\u00cc\u00cd"+
+    "\7*\2\2\u00cd\21\3\2\2\2\u00ce\u00cf\7+\2\2\u00cf\23\3\2\2\2\u00d0\u00d1"+
+    "\7\60\2\2\u00d1\u00d2\3\2\2\2\u00d2\u00d3\b\n\3\2\u00d3\25\3\2\2\2\u00d4"+
+    "\u00d5\7.\2\2\u00d5\27\3\2\2\2\u00d6\u00d7\7=\2\2\u00d7\31\3\2\2\2\u00d8"+
+    "\u00d9\7k\2\2\u00d9\u00da\7h\2\2\u00da\33\3\2\2\2\u00db\u00dc\7g\2\2\u00dc"+
+    "\u00dd\7n\2\2\u00dd\u00de\7u\2\2\u00de\u00df\7g\2\2\u00df\35\3\2\2\2\u00e0"+
+    "\u00e1\7y\2\2\u00e1\u00e2\7j\2\2\u00e2\u00e3\7k\2\2\u00e3\u00e4\7n\2\2"+
+    "\u00e4\u00e5\7g\2\2\u00e5\37\3\2\2\2\u00e6\u00e7\7f\2\2\u00e7\u00e8\7"+
+    "q\2\2\u00e8!\3\2\2\2\u00e9\u00ea\7h\2\2\u00ea\u00eb\7q\2\2\u00eb\u00ec"+
+    "\7t\2\2\u00ec#\3\2\2\2\u00ed\u00ee\7e\2\2\u00ee\u00ef\7q\2\2\u00ef\u00f0"+
+    "\7p\2\2\u00f0\u00f1\7v\2\2\u00f1\u00f2\7k\2\2\u00f2\u00f3\7p\2\2\u00f3"+
+    "\u00f4\7w\2\2\u00f4\u00f5\7g\2\2\u00f5%\3\2\2\2\u00f6\u00f7\7d\2\2\u00f7"+
+    "\u00f8\7t\2\2\u00f8\u00f9\7g\2\2\u00f9\u00fa\7c\2\2\u00fa\u00fb\7m\2\2"+
+    "\u00fb\'\3\2\2\2\u00fc\u00fd\7t\2\2\u00fd\u00fe\7g\2\2\u00fe\u00ff\7v"+
+    "\2\2\u00ff\u0100\7w\2\2\u0100\u0101\7t\2\2\u0101\u0102\7p\2\2\u0102)\3"+
+    "\2\2\2\u0103\u0104\7p\2\2\u0104\u0105\7g\2\2\u0105\u0106\7y\2\2\u0106"+
+    "+\3\2\2\2\u0107\u0108\7v\2\2\u0108\u0109\7t\2\2\u0109\u010a\7{\2\2\u010a"+
+    "-\3\2\2\2\u010b\u010c\7e\2\2\u010c\u010d\7c\2\2\u010d\u010e\7v\2\2\u010e"+
+    "\u010f\7e\2\2\u010f\u0110\7j\2\2\u0110/\3\2\2\2\u0111\u0112\7v\2\2\u0112"+
+    "\u0113\7j\2\2\u0113\u0114\7t\2\2\u0114\u0115\7q\2\2\u0115\u0116\7y\2\2"+
+    "\u0116\61\3\2\2\2\u0117\u0118\7v\2\2\u0118\u0119\7j\2\2\u0119\u011a\7"+
+    "k\2\2\u011a\u011b\7u\2\2\u011b\63\3\2\2\2\u011c\u011d\7#\2\2\u011d\65"+
+    "\3\2\2\2\u011e\u011f\7\u0080\2\2\u011f\67\3\2\2\2\u0120\u0121\7,\2\2\u0121"+
+    "9\3\2\2\2\u0122\u0123\7\61\2\2\u0123\u0124\6\35\2\2\u0124;\3\2\2\2\u0125"+
+    "\u0126\7\'\2\2\u0126=\3\2\2\2\u0127\u0128\7-\2\2\u0128?\3\2\2\2\u0129"+
+    "\u012a\7/\2\2\u012aA\3\2\2\2\u012b\u012c\7>\2\2\u012c\u012d\7>\2\2\u012d"+
+    "C\3\2\2\2\u012e\u012f\7@\2\2\u012f\u0130\7@\2\2\u0130E\3\2\2\2\u0131\u0132"+
+    "\7@\2\2\u0132\u0133\7@\2\2\u0133\u0134\7@\2\2\u0134G\3\2\2\2\u0135\u0136"+
+    "\7>\2\2\u0136I\3\2\2\2\u0137\u0138\7>\2\2\u0138\u0139\7?\2\2\u0139K\3"+
+    "\2\2\2\u013a\u013b\7@\2\2\u013bM\3\2\2\2\u013c\u013d\7@\2\2\u013d\u013e"+
+    "\7?\2\2\u013eO\3\2\2\2\u013f\u0140\7?\2\2\u0140\u0141\7?\2\2\u0141Q\3"+
+    "\2\2\2\u0142\u0143\7?\2\2\u0143\u0144\7?\2\2\u0144\u0145\7?\2\2\u0145"+
+    "S\3\2\2\2\u0146\u0147\7#\2\2\u0147\u0148\7?\2\2\u0148U\3\2\2\2\u0149\u014a"+
+    "\7#\2\2\u014a\u014b\7?\2\2\u014b\u014c\7?\2\2\u014cW\3\2\2\2\u014d\u014e"+
+    "\7(\2\2\u014eY\3\2\2\2\u014f\u0150\7`\2\2\u0150[\3\2\2\2\u0151\u0152\7"+
+    "~\2\2\u0152]\3\2\2\2\u0153\u0154\7(\2\2\u0154\u0155\7(\2\2\u0155_\3\2"+
+    "\2\2\u0156\u0157\7~\2\2\u0157\u0158\7~\2\2\u0158a\3\2\2\2\u0159\u015a"+
+    "\7A\2\2\u015ac\3\2\2\2\u015b\u015c\7<\2\2\u015ce\3\2\2\2\u015d\u015e\7"+
+    "<\2\2\u015e\u015f\7<\2\2\u015fg\3\2\2\2\u0160\u0161\7/\2\2\u0161\u0162"+
+    "\7@\2\2\u0162i\3\2\2\2\u0163\u0164\7?\2\2\u0164\u0165\7\u0080\2\2\u0165"+
+    "k\3\2\2\2\u0166\u0167\7?\2\2\u0167\u0168\7?\2\2\u0168\u0169\7\u0080\2"+
+    "\2\u0169m\3\2\2\2\u016a\u016b\7-\2\2\u016b\u016c\7-\2\2\u016co\3\2\2\2"+
+    "\u016d\u016e\7/\2\2\u016e\u016f\7/\2\2\u016fq\3\2\2\2\u0170\u0171\7?\2"+
+    "\2\u0171s\3\2\2\2\u0172\u0173\7-\2\2\u0173\u0174\7?\2\2\u0174u\3\2\2\2"+
+    "\u0175\u0176\7/\2\2\u0176\u0177\7?\2\2\u0177w\3\2\2\2\u0178\u0179\7,\2"+
+    "\2\u0179\u017a\7?\2\2\u017ay\3\2\2\2\u017b\u017c\7\61\2\2\u017c\u017d"+
+    "\7?\2\2\u017d{\3\2\2\2\u017e\u017f\7\'\2\2\u017f\u0180\7?\2\2\u0180}\3"+
+    "\2\2\2\u0181\u0182\7(\2\2\u0182\u0183\7?\2\2\u0183\177\3\2\2\2\u0184\u0185"+
+    "\7`\2\2\u0185\u0186\7?\2\2\u0186\u0081\3\2\2\2\u0187\u0188\7~\2\2\u0188"+
+    "\u0189\7?\2\2\u0189\u0083\3\2\2\2\u018a\u018b\7>\2\2\u018b\u018c\7>\2"+
+    "\2\u018c\u018d\7?\2\2\u018d\u0085\3\2\2\2\u018e\u018f\7@\2\2\u018f\u0190"+
+    "\7@\2\2\u0190\u0191\7?\2\2\u0191\u0087\3\2\2\2\u0192\u0193\7@\2\2\u0193"+
+    "\u0194\7@\2\2\u0194\u0195\7@\2\2\u0195\u0196\7?\2\2\u0196\u0089\3\2\2"+
+    "\2\u0197\u0199\7\62\2\2\u0198\u019a\t\4\2\2\u0199\u0198\3\2\2\2\u019a"+
+    "\u019b\3\2\2\2\u019b\u0199\3\2\2\2\u019b\u019c\3\2\2\2\u019c\u019e\3\2"+
+    "\2\2\u019d\u019f\t\5\2\2\u019e\u019d\3\2\2\2\u019e\u019f\3\2\2\2\u019f"+
+    "\u008b\3\2\2\2\u01a0\u01a1\7\62\2\2\u01a1\u01a3\t\6\2\2\u01a2\u01a4\t"+
+    "\7\2\2\u01a3\u01a2\3\2\2\2\u01a4\u01a5\3\2\2\2\u01a5\u01a3\3\2\2\2\u01a5"+
+    "\u01a6\3\2\2\2\u01a6\u01a8\3\2\2\2\u01a7\u01a9\t\5\2\2\u01a8\u01a7\3\2"+
+    "\2\2\u01a8\u01a9\3\2\2\2\u01a9\u008d\3\2\2\2\u01aa\u01b3\7\62\2\2\u01ab"+
+    "\u01af\t\b\2\2\u01ac\u01ae\t\t\2\2\u01ad\u01ac\3\2\2\2\u01ae\u01b1\3\2"+
+    "\2\2\u01af\u01ad\3\2\2\2\u01af\u01b0\3\2\2\2\u01b0\u01b3\3\2\2\2\u01b1"+
+    "\u01af\3\2\2\2\u01b2\u01aa\3\2\2\2\u01b2\u01ab\3\2\2\2\u01b3\u01b5\3\2"+
+    "\2\2\u01b4\u01b6\t\n\2\2\u01b5\u01b4\3\2\2\2\u01b5\u01b6\3\2\2\2\u01b6"+
+    "\u008f\3\2\2\2\u01b7\u01c0\7\62\2\2\u01b8\u01bc\t\b\2\2\u01b9\u01bb\t"+
+    "\t\2\2\u01ba\u01b9\3\2\2\2\u01bb\u01be\3\2\2\2\u01bc\u01ba\3\2\2\2\u01bc"+
+    "\u01bd\3\2\2\2\u01bd\u01c0\3\2\2\2\u01be\u01bc\3\2\2\2\u01bf\u01b7\3\2"+
+    "\2\2\u01bf\u01b8\3\2\2\2\u01c0\u01c7\3\2\2\2\u01c1\u01c3\5\24\n\2\u01c2"+
+    "\u01c4\t\t\2\2\u01c3\u01c2\3\2\2\2\u01c4\u01c5\3\2\2\2\u01c5\u01c3\3\2"+
+    "\2\2\u01c5\u01c6\3\2\2\2\u01c6\u01c8\3\2\2\2\u01c7\u01c1\3\2\2\2\u01c7"+
+    "\u01c8\3\2\2\2\u01c8\u01d2\3\2\2\2\u01c9\u01cb\t\13\2\2\u01ca\u01cc\t"+
+    "\f\2\2\u01cb\u01ca\3\2\2\2\u01cb\u01cc\3\2\2\2\u01cc\u01ce\3\2\2\2\u01cd"+
+    "\u01cf\t\t\2\2\u01ce\u01cd\3\2\2\2\u01cf\u01d0\3\2\2\2\u01d0\u01ce\3\2"+
+    "\2\2\u01d0\u01d1\3\2\2\2\u01d1\u01d3\3\2\2\2\u01d2\u01c9\3\2\2\2\u01d2"+
+    "\u01d3\3\2\2\2\u01d3\u01d5\3\2\2\2\u01d4\u01d6\t\r\2\2\u01d5\u01d4\3\2"+
+    "\2\2\u01d5\u01d6\3\2\2\2\u01d6\u0091\3\2\2\2\u01d7\u01df\7$\2\2\u01d8"+
+    "\u01d9\7^\2\2\u01d9\u01de\7$\2\2\u01da\u01db\7^\2\2\u01db\u01de\7^\2\2"+
+    "\u01dc\u01de\n\16\2\2\u01dd\u01d8\3\2\2\2\u01dd\u01da\3\2\2\2\u01dd\u01dc"+
+    "\3\2\2\2\u01de\u01e1\3\2\2\2\u01df\u01e0\3\2\2\2\u01df\u01dd\3\2\2\2\u01e0"+
+    "\u01e2\3\2\2\2\u01e1\u01df\3\2\2\2\u01e2\u01f0\7$\2\2\u01e3\u01eb\7)\2"+
+    "\2\u01e4\u01e5\7^\2\2\u01e5\u01ea\7)\2\2\u01e6\u01e7\7^\2\2\u01e7\u01ea"+
+    "\7^\2\2\u01e8\u01ea\n\16\2\2\u01e9\u01e4\3\2\2\2\u01e9\u01e6\3\2\2\2\u01e9"+
+    "\u01e8\3\2\2\2\u01ea\u01ed\3\2\2\2\u01eb\u01ec\3\2\2\2\u01eb\u01e9\3\2"+
+    "\2\2\u01ec\u01ee\3\2\2\2\u01ed\u01eb\3\2\2\2\u01ee\u01f0\7)\2\2\u01ef"+
+    "\u01d7\3\2\2\2\u01ef\u01e3\3\2\2\2\u01f0\u0093\3\2\2\2\u01f1\u01f5\7\61"+
+    "\2\2\u01f2\u01f6\n\17\2\2\u01f3\u01f4\7^\2\2\u01f4\u01f6\n\20\2\2\u01f5"+
+    "\u01f2\3\2\2\2\u01f5\u01f3\3\2\2\2\u01f6\u01f7\3\2\2\2\u01f7\u01f5\3\2"+
+    "\2\2\u01f7\u01f8\3\2\2\2\u01f8\u01f9\3\2\2\2\u01f9\u01fd\7\61\2\2\u01fa"+
+    "\u01fc\t\21\2\2\u01fb\u01fa\3\2\2\2\u01fc\u01ff\3\2\2\2\u01fd\u01fb\3"+
+    "\2\2\2\u01fd\u01fe\3\2\2\2\u01fe\u0200\3\2\2\2\u01ff\u01fd\3\2\2\2\u0200"+
+    "\u0201\6J\3\2\u0201\u0095\3\2\2\2\u0202\u0203\7v\2\2\u0203\u0204\7t\2"+
+    "\2\u0204\u0205\7w\2\2\u0205\u0206\7g\2\2\u0206\u0097\3\2\2\2\u0207\u0208"+
+    "\7h\2\2\u0208\u0209\7c\2\2\u0209\u020a\7n\2\2\u020a\u020b\7u\2\2\u020b"+
+    "\u020c\7g\2\2\u020c\u0099\3\2\2\2\u020d\u020e\7p\2\2\u020e\u020f\7w\2"+
+    "\2\u020f\u0210\7n\2\2\u0210\u0211\7n\2\2\u0211\u009b\3\2\2\2\u0212\u0218"+
+    "\5\u009eO\2\u0213\u0214\5\24\n\2\u0214\u0215\5\u009eO\2\u0215\u0217\3"+
+    "\2\2\2\u0216\u0213\3\2\2\2\u0217\u021a\3\2\2\2\u0218\u0216\3\2\2\2\u0218"+
+    "\u0219\3\2\2\2\u0219\u021b\3\2\2\2\u021a\u0218\3\2\2\2\u021b\u021c\6N"+
+    "\4\2\u021c\u009d\3\2\2\2\u021d\u0221\t\22\2\2\u021e\u0220\t\23\2\2\u021f"+
+    "\u021e\3\2\2\2\u0220\u0223\3\2\2\2\u0221\u021f\3\2\2\2\u0221\u0222\3\2"+
+    "\2\2\u0222\u009f\3\2\2\2\u0223\u0221\3\2\2\2\u0224\u022d\7\62\2\2\u0225"+
+    "\u0229\t\b\2\2\u0226\u0228\t\t\2\2\u0227\u0226\3\2\2\2\u0228\u022b\3\2"+
+    "\2\2\u0229\u0227\3\2\2\2\u0229\u022a\3\2\2\2\u022a\u022d\3\2\2\2\u022b"+
+    "\u0229\3\2\2\2\u022c\u0224\3\2\2\2\u022c\u0225\3\2\2\2\u022d\u022e\3\2"+
+    "\2\2\u022e\u022f\bP\4\2\u022f\u00a1\3\2\2\2\u0230\u0234\t\22\2\2\u0231"+
+    "\u0233\t\23\2\2\u0232\u0231\3\2\2\2\u0233\u0236\3\2\2\2\u0234\u0232\3"+
+    "\2\2\2\u0234\u0235\3\2\2\2\u0235\u0237\3\2\2\2\u0236\u0234\3\2\2\2\u0237"+
+    "\u0238\bQ\4\2\u0238\u00a3\3\2\2\2$\2\3\u00a7\u00b1\u00bb\u00c0\u019b\u019e"+
     "\u01a5\u01a8\u01af\u01b2\u01b5\u01bc\u01bf\u01c5\u01c7\u01cb\u01d0\u01d2"+
-    "\u01d5\u01dd\u01df\u01e9\u01eb\u01ef\u01f5\u01f7\u0212\u021b\u0223\u0226"+
-    "\u022e\5\b\2\2\4\3\2\4\2\2";
+    "\u01d5\u01dd\u01df\u01e9\u01eb\u01ef\u01f5\u01f7\u01fd\u0218\u0221\u0229"+
+    "\u022c\u0234\5\b\2\2\4\3\2\4\2\2";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index b1a5ed3a260..474e47599a8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -834,9 +834,12 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
 
     @Override
     public Object visitRegex(RegexContext ctx) {
-        String pattern = ctx.REGEX().getText().substring(1, ctx.REGEX().getText().length() - 1);
+        String text = ctx.REGEX().getText();
+        int lastSlash = text.lastIndexOf('/');
+        String pattern = text.substring(1, lastSlash);
+        String flags = text.substring(lastSlash + 1);
         List<ALink> links = new ArrayList<>();
-        links.add(new LRegex(location(ctx), pattern));
+        links.add(new LRegex(location(ctx), pattern, flags));
 
         return links;
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LRegex.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LRegex.java
index 720e9d45969..324b6d72cf1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LRegex.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LRegex.java
@@ -35,14 +35,20 @@ import org.elasticsearch.painless.WriterConstants;
  */
 public final class LRegex extends ALink {
     private final String pattern;
+    private final int flags;
     private Constant constant;
 
-    public LRegex(Location location, String pattern) {
+    public LRegex(Location location, String pattern, String flagsString) {
         super(location, 1);
         this.pattern = pattern;
+        int flags = 0;
+        for (int c = 0; c < flagsString.length(); c++) {
+            flags |= flagForChar(flagsString.charAt(c));
+        }
+        this.flags = flags;
         try {
             // Compile the pattern early after parsing so we can throw an error to the user with the location
-            Pattern.compile(pattern);
+            Pattern.compile(pattern, flags);
         } catch (PatternSyntaxException e) {
             throw createError(e);
         }
@@ -82,6 +88,21 @@ public final class LRegex extends ALink {
 
     private void initializeConstant(MethodWriter writer) {
         writer.push(pattern);
+        writer.push(flags);
         writer.invokeStatic(Definition.PATTERN_TYPE.type, WriterConstants.PATTERN_COMPILE);
     }
+
+    private int flagForChar(char c) {
+        switch (c) {
+        case 'c': return Pattern.CANON_EQ;
+        case 'i': return Pattern.CASE_INSENSITIVE;
+        case 'l': return Pattern.LITERAL;
+        case 'm': return Pattern.MULTILINE;
+        case 's': return Pattern.DOTALL;
+        case 'U': return Pattern.UNICODE_CHARACTER_CLASS;
+        case 'u': return Pattern.UNICODE_CASE;
+        case 'x': return Pattern.COMMENTS;
+        default: throw new IllegalArgumentException("Unknown flag [" + c + "]");
+        }
+    }
 }
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt
index a8982871a33..6befc865731 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt
@@ -25,7 +25,14 @@
 class Pattern -> java.util.regex.Pattern extends Object {
 # Pattern compile(String)   Intentionally not included. We don't want dynamic patterns because they allow regexes to be generated per time
 #                           the script is run which is super slow. LRegex generates code that calls this method but it skips these checks.
+  Predicate asPredicate()
+  int flags()
   Matcher matcher(CharSequence)
+  String pattern()
+  String quote(String)
+  String[] split(CharSequence)
+  String[] split(CharSequence,int)
+  Stream splitAsStream(CharSequence)
 }
 
 class Matcher -> java.util.regex.Matcher extends Object {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java
index ee281e4656b..a36eedd7058 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java
@@ -19,8 +19,12 @@
 
 package org.elasticsearch.painless;
 
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
 
+import static java.util.Collections.emptyMap;
 import static java.util.Collections.singletonMap;
 import static org.hamcrest.Matchers.containsString;
 
@@ -125,6 +129,52 @@ public class RegexTests extends ScriptTestCase {
         assertEquals("o", exec("Matcher m = /(?<first>f)(?<second>o)o/.matcher('foo'); m.find(); return m.namedGroup('second')"));
     }
 
+    // Make sure some methods on Pattern are whitelisted
+    public void testSplit() {
+        assertArrayEquals(new String[] {"cat", "dog"}, (String[]) exec("/,/.split('cat,dog')"));
+    }
+
+    public void testSplitAsStream() {
+        assertEquals(new HashSet<>(Arrays.asList("cat", "dog")), exec("/,/.splitAsStream('cat,dog').collect(Collectors.toSet())"));
+    }
+
+    // Make sure the flags are set
+    public void testMultilineFlag() {
+        assertEquals(Pattern.MULTILINE, exec("/./m.flags()"));
+    }
+
+    public void testSinglelineFlag() {
+        assertEquals(Pattern.DOTALL, exec("/./s.flags()"));
+    }
+
+    public void testInsensitiveFlag() {
+        assertEquals(Pattern.CASE_INSENSITIVE, exec("/./i.flags()"));
+    }
+
+    public void testExtendedFlag() {
+        assertEquals(Pattern.COMMENTS, exec("/./x.flags()"));
+    }
+
+    public void testUnicodeCaseFlag() {
+        assertEquals(Pattern.UNICODE_CASE, exec("/./u.flags()"));
+    }
+
+    public void testUnicodeCharacterClassFlag() {
+        assertEquals(Pattern.UNICODE_CASE | Pattern.UNICODE_CHARACTER_CLASS, exec("/./U.flags()"));
+    }
+
+    public void testLiteralFlag() {
+        assertEquals(Pattern.LITERAL, exec("/./l.flags()"));
+    }
+
+    public void testCanonicalEquivalenceFlag() {
+        assertEquals(Pattern.CANON_EQ, exec("/./c.flags()"));
+    }
+
+    public void testManyFlags() {
+        assertEquals(Pattern.CANON_EQ | Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE | Pattern.COMMENTS, exec("/./ciux.flags()"));
+    }
+
     public void testCantUsePatternCompile() {
         IllegalArgumentException e = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("Pattern.compile('aa')");
@@ -146,4 +196,11 @@ public class RegexTests extends ScriptTestCase {
         });
         assertEquals("Cannot cast from [int] to [String].", e.getMessage());
     }
+
+    public void testBogusRegexFlag() {
+        IllegalArgumentException e = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("/asdf/b", emptyMap(), emptyMap(), null); // Not picky so we get a non-assertion error
+        });
+        assertEquals("invalid sequence of tokens near ['b'].", e.getMessage());
+    }
 }

From 13d16fbf41e0cf90c9051e015681cc2198bb0334 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 16 Jun 2016 15:06:01 -0400
Subject: [PATCH 0917/1311] Painless: Disable java-9 indy string thing

It is breaking some of the doc tests. Also add a unit test that
shows the failure.

Relates to #18929
---
 .../main/java/org/elasticsearch/painless/WriterConstants.java | 2 +-
 .../src/test/java/org/elasticsearch/painless/StringTests.java | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index 6960e906339..7906a125f35 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -131,7 +131,7 @@ public final class WriterConstants {
             // not Java 9 - we set it null, so MethodWriter uses StringBuilder:
             bs = null;
         }
-        INDY_STRING_CONCAT_BOOTSTRAP_HANDLE = bs;
+        INDY_STRING_CONCAT_BOOTSTRAP_HANDLE = null; // Disabled until https://github.com/elastic/elasticsearch/issues/18929
     }
 
     public final static int MAX_INDY_STRING_CONCAT_ARGS = 200;
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
index 80b5675a6e1..873e773b9a3 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/StringTests.java
@@ -221,4 +221,8 @@ public class StringTests extends ScriptTestCase {
             exec("def x = null; def y = null; x += y");
         });
     }
+
+    public void testAppendStringIntoMap() {
+        assertEquals("nullcat", exec("def a = new HashMap(); a.cat += 'cat'"));
+    }
 }

From 2a3184604e15da7c6ce8d89c5fda29c366149585 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Thu, 16 Jun 2016 17:34:50 -0400
Subject: [PATCH 0918/1311] improve Debugger to print code even if it hits
 exception

---
 .../org/elasticsearch/painless/Compiler.java  |  7 +++--
 .../elasticsearch/painless/MethodWriter.java  |  4 +--
 .../elasticsearch/painless/antlr/Walker.java  | 12 ++++---
 .../painless/node/SFunction.java              |  5 +--
 .../elasticsearch/painless/node/SSource.java  | 31 +++++++++++++------
 .../org/elasticsearch/painless/Debugger.java  | 23 ++++++++------
 6 files changed, 52 insertions(+), 30 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
index 8fa9f5d583e..664c567091c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
@@ -22,6 +22,7 @@ package org.elasticsearch.painless;
 import org.elasticsearch.bootstrap.BootstrapInfo;
 import org.elasticsearch.painless.antlr.Walker;
 import org.elasticsearch.painless.node.SSource;
+import org.objectweb.asm.util.Printer;
 
 import java.net.MalformedURLException;
 import java.net.URL;
@@ -98,7 +99,7 @@ final class Compiler {
                 " plugin if a script longer than this length is a requirement.");
         }
 
-        SSource root = Walker.buildPainlessTree(name, source, settings);
+        SSource root = Walker.buildPainlessTree(name, source, settings, null);
 
         root.analyze();
         root.write();
@@ -120,14 +121,14 @@ final class Compiler {
      * @param settings The CompilerSettings to be used during the compilation.
      * @return The bytes for compilation.
      */
-    static byte[] compile(String name, String source, CompilerSettings settings) {
+    static byte[] compile(String name, String source, CompilerSettings settings, Printer debugStream) {
         if (source.length() > MAXIMUM_SOURCE_LENGTH) {
             throw new IllegalArgumentException("Scripts may be no longer than " + MAXIMUM_SOURCE_LENGTH +
                 " characters.  The passed in script is " + source.length() + " characters.  Consider using a" +
                 " plugin if a script longer than this length is a requirement.");
         }
 
-        SSource root = Walker.buildPainlessTree(name, source, settings);
+        SSource root = Walker.buildPainlessTree(name, source, settings, debugStream);
 
         root.analyze();
         root.write();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index 622e528f846..1290633879a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -22,7 +22,7 @@ package org.elasticsearch.painless;
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
-import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.commons.GeneratorAdapter;
@@ -82,7 +82,7 @@ public final class MethodWriter extends GeneratorAdapter {
     private final Deque<List<org.objectweb.asm.Type>> stringConcatArgs =
         (INDY_STRING_CONCAT_BOOTSTRAP_HANDLE == null) ?  null : new ArrayDeque<>();
 
-    public MethodWriter(int access, Method method, ClassWriter cw, BitSet statements) {
+    public MethodWriter(int access, Method method, ClassVisitor cw, BitSet statements) {
         super(Opcodes.ASM5, cw.visitMethod(access, method.getName(), method.getDescriptor(), null, null),
                 access, method.getName(), method.getDescriptor());
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index b1a5ed3a260..a04024e4efa 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -146,6 +146,7 @@ import org.elasticsearch.painless.node.SSource;
 import org.elasticsearch.painless.node.SThrow;
 import org.elasticsearch.painless.node.STry;
 import org.elasticsearch.painless.node.SWhile;
+import org.objectweb.asm.util.Printer;
 
 import java.util.ArrayDeque;
 import java.util.ArrayList;
@@ -158,12 +159,13 @@ import java.util.List;
  */
 public final class Walker extends PainlessParserBaseVisitor<Object> {
 
-    public static SSource buildPainlessTree(String sourceName, String sourceText, CompilerSettings settings) {
-        return new Walker(sourceName, sourceText, settings).source;
+    public static SSource buildPainlessTree(String sourceName, String sourceText, CompilerSettings settings, Printer debugStream) {
+        return new Walker(sourceName, sourceText, settings, debugStream).source;
     }
 
     private final SSource source;
     private final CompilerSettings settings;
+    private final Printer debugStream;
     private final String sourceName;
     private final String sourceText;
 
@@ -171,7 +173,8 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
     private final List<SFunction> synthetic = new ArrayList<>();
     private int syntheticCounter = 0;
 
-    private Walker(String sourceName, String sourceText, CompilerSettings settings) {
+    private Walker(String sourceName, String sourceText, CompilerSettings settings, Printer debugStream) {
+        this.debugStream = debugStream;
         this.settings = settings;
         this.sourceName = Location.computeSourceName(sourceName, sourceText);
         this.sourceText = sourceText;
@@ -236,7 +239,8 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         
         functions.addAll(synthetic);
 
-        return new SSource(sourceName, sourceText, (ExecuteReserved)reserved.pop(), location(ctx), functions, statements);
+        return new SSource(sourceName, sourceText, debugStream, 
+                          (ExecuteReserved)reserved.pop(), location(ctx), functions, statements);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index 0cdde13fce7..60015844a96 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -31,6 +31,7 @@ import org.elasticsearch.painless.Locals.Variable;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.WriterConstants;
+import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.ClassWriter;
 import org.objectweb.asm.Handle;
 import org.objectweb.asm.Opcodes;
@@ -146,8 +147,8 @@ public class SFunction extends AStatement {
         locals.addConstant(location, WriterConstants.METHOD_HANDLE_TYPE, staticHandleFieldName, this::initializeConstant);
     }
     
-    /** Writes the function to given ClassWriter. */
-    void write (ClassWriter writer, BitSet statements) {
+    /** Writes the function to given ClassVisitor. */
+    void write (ClassVisitor writer, BitSet statements) {
         int access = Opcodes.ACC_PRIVATE | Opcodes.ACC_STATIC;
         if (synthetic) {
             access |= Opcodes.ACC_SYNTHETIC;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index aeda0387297..1d3bb74bc31 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -29,9 +29,11 @@ import org.elasticsearch.painless.Locals.Variable;
 import org.elasticsearch.painless.WriterConstants;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
-
+import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.ClassWriter;
 import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.util.Printer;
+import org.objectweb.asm.util.TraceClassVisitor;
 
 import java.util.BitSet;
 import java.util.Collections;
@@ -53,6 +55,7 @@ public final class SSource extends AStatement {
 
     final String name;
     final String source;
+    final Printer debugStream;
     final ExecuteReserved reserved;
     final List<SFunction> functions;
     final List<AStatement> statements;
@@ -61,12 +64,13 @@ public final class SSource extends AStatement {
     private BitSet expressions;
     private byte[] bytes;
 
-    public SSource(String name, String source, ExecuteReserved reserved, Location location,
+    public SSource(String name, String source, Printer debugStream, ExecuteReserved reserved, Location location, 
                    List<SFunction> functions, List<AStatement> statements) {
         super(location);
 
         this.name = name;
         this.source = source;
+        this.debugStream = debugStream;
         this.reserved = reserved;
         this.functions = Collections.unmodifiableList(functions);
         this.statements = Collections.unmodifiableList(statements);
@@ -132,13 +136,19 @@ public final class SSource extends AStatement {
         String classInterfaces[] = reserved.usesScore() ? new String[] { WriterConstants.NEEDS_SCORE_TYPE.getInternalName() } : null;
 
         ClassWriter writer = new ClassWriter(classFrames);
-        writer.visit(classVersion, classAccess, className, null, classBase, classInterfaces);
-        writer.visitSource(Location.computeSourceName(name, source), null);
+        ClassVisitor visitor = writer;
+        
+        if (debugStream != null) {
+            visitor = new TraceClassVisitor(visitor, debugStream, null);
+        }
+        visitor.visit(classVersion, classAccess, className, null, classBase, classInterfaces);
+        visitor.visitSource(Location.computeSourceName(name, source), null);
 
         expressions = new BitSet(source.length());
 
         // Write the constructor:
-        MethodWriter constructor = new MethodWriter(Opcodes.ACC_PUBLIC, CONSTRUCTOR, writer, expressions);
+        MethodWriter constructor = new MethodWriter(Opcodes.ACC_PUBLIC, CONSTRUCTOR, visitor, expressions);
+        constructor.visitCode();
         constructor.loadThis();
         constructor.loadArgs();
         constructor.invokeConstructor(org.objectweb.asm.Type.getType(Executable.class), CONSTRUCTOR);
@@ -146,20 +156,21 @@ public final class SSource extends AStatement {
         constructor.endMethod();
 
         // Write the execute method:
-        MethodWriter execute = new MethodWriter(Opcodes.ACC_PUBLIC, EXECUTE, writer, expressions);
+        MethodWriter execute = new MethodWriter(Opcodes.ACC_PUBLIC, EXECUTE, visitor, expressions);
+        execute.visitCode();
         write(execute);
         execute.endMethod();
         
         // Write all functions:
         for (SFunction function : functions) {
-            function.write(writer, expressions);
+            function.write(visitor, expressions);
         }
 
         // Write the constants
         if (false == locals.getConstants().isEmpty()) {
             // Fields
             for (Constant constant : locals.getConstants()) {
-                writer.visitField(
+                visitor.visitField(
                         Opcodes.ACC_FINAL | Opcodes.ACC_PRIVATE | Opcodes.ACC_STATIC,
                         constant.name,
                         constant.type.getDescriptor(),
@@ -169,7 +180,7 @@ public final class SSource extends AStatement {
 
             // Initialize the constants in a static initializer
             final MethodWriter clinit = new MethodWriter(Opcodes.ACC_STATIC, 
-                    WriterConstants.CLINIT, writer, expressions);
+                    WriterConstants.CLINIT, visitor, expressions);
             for (Constant constant : locals.getConstants()) {
                 constant.initializer.accept(clinit);
                 clinit.putStatic(CLASS_TYPE, constant.name, constant.type);
@@ -180,7 +191,7 @@ public final class SSource extends AStatement {
         
         // End writing the class and store the generated bytes.
 
-        writer.visitEnd();
+        visitor.visitEnd();
         bytes = writer.toByteArray();
     }
     
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
index 3e973997229..d8e6896f083 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.painless;
 
-import org.objectweb.asm.ClassReader;
-import org.objectweb.asm.util.TraceClassVisitor;
+import org.apache.lucene.util.IOUtils;
+import org.objectweb.asm.util.Textifier;
 
 import java.io.PrintWriter;
 import java.io.StringWriter;
@@ -35,14 +35,19 @@ final class Debugger {
 
     /** compiles to bytecode, and returns debugging output */
     static String toString(String source, CompilerSettings settings) {
-        final byte[] bytes = Compiler.compile("<debugging>", source, settings);
-        final StringWriter output = new StringWriter();
-        final PrintWriter outputWriter = new PrintWriter(output);
-        final ClassReader reader = new ClassReader(bytes);
-
-        reader.accept(new TraceClassVisitor(outputWriter), 0);
-        outputWriter.flush();
+        StringWriter output = new StringWriter();
+        PrintWriter outputWriter = new PrintWriter(output);
+        Textifier textifier = new Textifier();
+        try {
+            Compiler.compile("<debugging>", source, settings, textifier);
+        } catch (Exception e) {
+            textifier.print(outputWriter);
+            e.addSuppressed(new Exception("current bytecode: \n" + output));
+            IOUtils.reThrowUnchecked(e);
+            throw new AssertionError();
+        }
         
+        textifier.print(outputWriter);
         return output.toString();
     }
 }

From a7aedbe0a17700a11a94ce097d59c448f7539344 Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Fri, 17 Jun 2016 00:30:09 +0200
Subject: [PATCH 0919/1311] Fix compound assignment with string concats. in
 Java 9 there is no stringbuilder on stack! This closes #18929

---
 .../main/java/org/elasticsearch/painless/MethodWriter.java | 7 ++++++-
 .../java/org/elasticsearch/painless/WriterConstants.java   | 2 +-
 .../main/java/org/elasticsearch/painless/node/EChain.java  | 7 ++++---
 3 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index 1290633879a..6241b5b4207 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -209,15 +209,20 @@ public final class MethodWriter extends GeneratorAdapter {
         }
     }
 
-    public void writeNewStrings() {
+    /** Starts a new string concat.
+     * @return the size of arguments pushed to stack (the object that does string concats, e.g. a StringBuilder)
+     */
+    public int writeNewStrings() {
         if (INDY_STRING_CONCAT_BOOTSTRAP_HANDLE != null) {
             // Java 9+: we just push our argument collector onto deque
             stringConcatArgs.push(new ArrayList<>());
+            return 0; // nothing added to stack
         } else {
             // Java 8: create a StringBuilder in bytecode
             newInstance(STRINGBUILDER_TYPE);
             dup();
             invokeConstructor(STRINGBUILDER_TYPE, STRINGBUILDER_CONSTRUCTOR);
+            return 1; // StringBuilder on stack
         }
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index 7906a125f35..6960e906339 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -131,7 +131,7 @@ public final class WriterConstants {
             // not Java 9 - we set it null, so MethodWriter uses StringBuilder:
             bs = null;
         }
-        INDY_STRING_CONCAT_BOOTSTRAP_HANDLE = null; // Disabled until https://github.com/elastic/elasticsearch/issues/18929
+        INDY_STRING_CONCAT_BOOTSTRAP_HANDLE = bs;
     }
 
     public final static int MAX_INDY_STRING_CONCAT_ARGS = 200;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index d0d26542f5c..3be7f068261 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -294,8 +294,9 @@ public final class EChain extends AExpression {
         // track types going onto the stack.  This must be done before the
         // links in the chain are read because we need the StringBuilder to
         // be placed on the stack ahead of any potential concatenation arguments.
+        int catElementStackSize = 0;
         if (cat) {
-            writer.writeNewStrings();
+            catElementStackSize = writer.writeNewStrings();
         }
 
         ALink last = links.get(links.size() - 1);
@@ -312,7 +313,7 @@ public final class EChain extends AExpression {
                     // Handle the case where we are doing a compound assignment
                     // representing a String concatenation.
 
-                    writer.writeDup(link.size, 1);         // dup the StringBuilder
+                    writer.writeDup(link.size, catElementStackSize);  // dup the top element and insert it before concat helper on stack
                     link.load(writer);                     // read the current link's value
                     writer.writeAppendStrings(link.after); // append the link's value using the StringBuilder
 
@@ -323,7 +324,7 @@ public final class EChain extends AExpression {
                         writer.writeAppendStrings(expression.actual); // append the expression's value unless it's also a concatenation
                     }
 
-                    writer.writeToStrings(); // put the value of the StringBuilder on the stack
+                    writer.writeToStrings(); // put the value for string concat onto the stack
                     writer.writeCast(back);  // if necessary, cast the String to the lhs actual type
 
                     if (link.load) {

From 615920df2e121bc70d52f49e39e93a934c5c0e3c Mon Sep 17 00:00:00 2001
From: Areek Zillur <areek.zillur@elasticsearch.com>
Date: Fri, 17 Jun 2016 00:03:28 -0400
Subject: [PATCH 0920/1311] update docs

---
 .../reference/indices/rollover-index.asciidoc | 21 ++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc
index 35ad327f181..6879470267f 100644
--- a/docs/reference/indices/rollover-index.asciidoc
+++ b/docs/reference/indices/rollover-index.asciidoc
@@ -79,7 +79,14 @@ set in matching index templates:
 
 [source,js]
 --------------------------------------------------
-POST /logs_writes/_rollover
+PUT /logs-0001
+{
+  "aliases": {
+    "logs_write": {}
+  }
+}
+
+POST logs_write/_rollover
 {
   "conditions" : {
     "max_age": "7d",
@@ -91,7 +98,8 @@ POST /logs_writes/_rollover
 }
 --------------------------------------------------
 // CONSOLE
-<1> Set settings to override matching index template, `mappings` and `aliases` can also be provided.
+<1> Set settings to override matching index template, `mappings` and `aliases`
+    can also be provided.
 
  [float]
  === Dry run
@@ -101,7 +109,14 @@ checked without performing the actual rollover:
 
 [source,js]
 --------------------------------------------------
-POST /logs_writes/_rollover?dry_run
+PUT /logs-0001
+{
+  "aliases": {
+    "logs_write": {}
+  }
+}
+
+POST logs_write/_rollover?dry_run
 {
   "conditions" : {
     "max_age": "7d",

From 600cbb6ab0a1f06fd8ad71301849f859a3557049 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Thu, 16 Jun 2016 17:56:50 +0200
Subject: [PATCH 0921/1311] Upgrade to Lucene 6.1.0. #18926

---
 buildSrc/version.properties                   |   4 +-
 .../elasticsearch/bootstrap/security.policy   |   4 +-
 .../bootstrap/test-framework.policy           |   2 +-
 .../search/geo/GeoBoundingBoxIT.java          | 109 ------------------
 ...ers-common-6.1.0-snapshot-3a57bea.jar.sha1 |   1 -
 .../lucene-analyzers-common-6.1.0.jar.sha1    |   1 +
 ...ard-codecs-6.1.0-snapshot-3a57bea.jar.sha1 |   1 -
 .../lucene-backward-codecs-6.1.0.jar.sha1     |   1 +
 ...ucene-core-6.1.0-snapshot-3a57bea.jar.sha1 |   1 -
 .../licenses/lucene-core-6.1.0.jar.sha1       |   1 +
 ...e-grouping-6.1.0-snapshot-3a57bea.jar.sha1 |   1 -
 .../licenses/lucene-grouping-6.1.0.jar.sha1   |   1 +
 ...ighlighter-6.1.0-snapshot-3a57bea.jar.sha1 |   1 -
 .../lucene-highlighter-6.1.0.jar.sha1         |   1 +
 ...ucene-join-6.1.0-snapshot-3a57bea.jar.sha1 |   1 -
 .../licenses/lucene-join-6.1.0.jar.sha1       |   1 +
 ...ene-memory-6.1.0-snapshot-3a57bea.jar.sha1 |   1 -
 .../licenses/lucene-memory-6.1.0.jar.sha1     |   1 +
 ...ucene-misc-6.1.0-snapshot-3a57bea.jar.sha1 |   1 -
 .../licenses/lucene-misc-6.1.0.jar.sha1       |   1 +
 ...ne-queries-6.1.0-snapshot-3a57bea.jar.sha1 |   1 -
 .../licenses/lucene-queries-6.1.0.jar.sha1    |   1 +
 ...ueryparser-6.1.0-snapshot-3a57bea.jar.sha1 |   1 -
 .../lucene-queryparser-6.1.0.jar.sha1         |   1 +
 ...ne-sandbox-6.1.0-snapshot-3a57bea.jar.sha1 |   1 -
 .../licenses/lucene-sandbox-6.1.0.jar.sha1    |   1 +
 ...ne-spatial-6.1.0-snapshot-3a57bea.jar.sha1 |   1 -
 .../licenses/lucene-spatial-6.1.0.jar.sha1    |   1 +
 ...ial-extras-6.1.0-snapshot-3a57bea.jar.sha1 |   1 -
 .../lucene-spatial-extras-6.1.0.jar.sha1      |   1 +
 ...-spatial3d-6.1.0-snapshot-3a57bea.jar.sha1 |   1 -
 .../licenses/lucene-spatial3d-6.1.0.jar.sha1  |   1 +
 ...ne-suggest-6.1.0-snapshot-3a57bea.jar.sha1 |   1 -
 .../licenses/lucene-suggest-6.1.0.jar.sha1    |   1 +
 ...xpressions-6.1.0-snapshot-3a57bea.jar.sha1 |   1 -
 .../lucene-expressions-6.1.0.jar.sha1         |   1 +
 ...lyzers-icu-6.1.0-snapshot-3a57bea.jar.sha1 |   1 -
 .../lucene-analyzers-icu-6.1.0.jar.sha1       |   1 +
 ...s-kuromoji-6.1.0-snapshot-3a57bea.jar.sha1 |   1 -
 .../lucene-analyzers-kuromoji-6.1.0.jar.sha1  |   1 +
 ...s-phonetic-6.1.0-snapshot-3a57bea.jar.sha1 |   1 -
 .../lucene-analyzers-phonetic-6.1.0.jar.sha1  |   1 +
 ...rs-smartcn-6.1.0-snapshot-3a57bea.jar.sha1 |   1 -
 .../lucene-analyzers-smartcn-6.1.0.jar.sha1   |   1 +
 ...rs-stempel-6.1.0-snapshot-3a57bea.jar.sha1 |   1 -
 .../lucene-analyzers-stempel-6.1.0.jar.sha1   |   1 +
 46 files changed, 26 insertions(+), 135 deletions(-)
 delete mode 100644 distribution/licenses/lucene-analyzers-common-6.1.0-snapshot-3a57bea.jar.sha1
 create mode 100644 distribution/licenses/lucene-analyzers-common-6.1.0.jar.sha1
 delete mode 100644 distribution/licenses/lucene-backward-codecs-6.1.0-snapshot-3a57bea.jar.sha1
 create mode 100644 distribution/licenses/lucene-backward-codecs-6.1.0.jar.sha1
 delete mode 100644 distribution/licenses/lucene-core-6.1.0-snapshot-3a57bea.jar.sha1
 create mode 100644 distribution/licenses/lucene-core-6.1.0.jar.sha1
 delete mode 100644 distribution/licenses/lucene-grouping-6.1.0-snapshot-3a57bea.jar.sha1
 create mode 100644 distribution/licenses/lucene-grouping-6.1.0.jar.sha1
 delete mode 100644 distribution/licenses/lucene-highlighter-6.1.0-snapshot-3a57bea.jar.sha1
 create mode 100644 distribution/licenses/lucene-highlighter-6.1.0.jar.sha1
 delete mode 100644 distribution/licenses/lucene-join-6.1.0-snapshot-3a57bea.jar.sha1
 create mode 100644 distribution/licenses/lucene-join-6.1.0.jar.sha1
 delete mode 100644 distribution/licenses/lucene-memory-6.1.0-snapshot-3a57bea.jar.sha1
 create mode 100644 distribution/licenses/lucene-memory-6.1.0.jar.sha1
 delete mode 100644 distribution/licenses/lucene-misc-6.1.0-snapshot-3a57bea.jar.sha1
 create mode 100644 distribution/licenses/lucene-misc-6.1.0.jar.sha1
 delete mode 100644 distribution/licenses/lucene-queries-6.1.0-snapshot-3a57bea.jar.sha1
 create mode 100644 distribution/licenses/lucene-queries-6.1.0.jar.sha1
 delete mode 100644 distribution/licenses/lucene-queryparser-6.1.0-snapshot-3a57bea.jar.sha1
 create mode 100644 distribution/licenses/lucene-queryparser-6.1.0.jar.sha1
 delete mode 100644 distribution/licenses/lucene-sandbox-6.1.0-snapshot-3a57bea.jar.sha1
 create mode 100644 distribution/licenses/lucene-sandbox-6.1.0.jar.sha1
 delete mode 100644 distribution/licenses/lucene-spatial-6.1.0-snapshot-3a57bea.jar.sha1
 create mode 100644 distribution/licenses/lucene-spatial-6.1.0.jar.sha1
 delete mode 100644 distribution/licenses/lucene-spatial-extras-6.1.0-snapshot-3a57bea.jar.sha1
 create mode 100644 distribution/licenses/lucene-spatial-extras-6.1.0.jar.sha1
 delete mode 100644 distribution/licenses/lucene-spatial3d-6.1.0-snapshot-3a57bea.jar.sha1
 create mode 100644 distribution/licenses/lucene-spatial3d-6.1.0.jar.sha1
 delete mode 100644 distribution/licenses/lucene-suggest-6.1.0-snapshot-3a57bea.jar.sha1
 create mode 100644 distribution/licenses/lucene-suggest-6.1.0.jar.sha1
 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-6.1.0-snapshot-3a57bea.jar.sha1
 create mode 100644 modules/lang-expression/licenses/lucene-expressions-6.1.0.jar.sha1
 delete mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-6.1.0-snapshot-3a57bea.jar.sha1
 create mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-6.1.0.jar.sha1
 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.1.0-snapshot-3a57bea.jar.sha1
 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.1.0.jar.sha1
 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.1.0-snapshot-3a57bea.jar.sha1
 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.1.0.jar.sha1
 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.1.0-snapshot-3a57bea.jar.sha1
 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.1.0.jar.sha1
 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.1.0-snapshot-3a57bea.jar.sha1
 create mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.1.0.jar.sha1

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index f8d8b5848c9..4f5f5a49a47 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 elasticsearch     = 5.0.0-alpha4
-lucene            = 6.1.0-snapshot-3a57bea
+lucene            = 6.1.0
 
 # optional dependencies
 spatial4j         = 0.6
@@ -19,4 +19,4 @@ commonslogging    = 1.1.3
 commonscodec      = 1.10
 
 # benchmark dependencies
-jmh               = 1.12
\ No newline at end of file
+jmh               = 1.12
diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
index b185289b58d..dcf37d3dd5c 100644
--- a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
+++ b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
@@ -31,7 +31,7 @@ grant codeBase "${codebase.securesm-1.0.jar}" {
 //// Very special jar permissions:
 //// These are dangerous permissions that we don't want to grant to everything.
 
-grant codeBase "${codebase.lucene-core-6.1.0-snapshot-3a57bea.jar}" {
+grant codeBase "${codebase.lucene-core-6.1.0.jar}" {
   // needed to allow MMapDirectory's "unmap hack" (die unmap hack, die)
   // java 8 package
   permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
@@ -42,7 +42,7 @@ grant codeBase "${codebase.lucene-core-6.1.0-snapshot-3a57bea.jar}" {
   permission java.lang.RuntimePermission "accessDeclaredMembers";
 };
 
-grant codeBase "${codebase.lucene-misc-6.1.0-snapshot-3a57bea.jar}" {
+grant codeBase "${codebase.lucene-misc-6.1.0.jar}" {
   // needed to allow shard shrinking to use hard-links if possible via lucenes HardlinkCopyDirectoryWrapper
   permission java.nio.file.LinkPermission "hard";
 };
diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
index 7abebf01be1..f2c60923f61 100644
--- a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
+++ b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
@@ -33,7 +33,7 @@ grant codeBase "${codebase.securemock-1.2.jar}" {
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
 };
 
-grant codeBase "${codebase.lucene-test-framework-6.1.0-snapshot-3a57bea.jar}" {
+grant codeBase "${codebase.lucene-test-framework-6.1.0.jar}" {
   // needed by RamUsageTester
   permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
   // needed for testing hardlinks in StoreRecoveryTests since we install MockFS
diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java b/core/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java
index 398ef64bc92..c3ce3364a3a 100644
--- a/core/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java
+++ b/core/src/test/java/org/elasticsearch/search/geo/GeoBoundingBoxIT.java
@@ -126,115 +126,6 @@ public class GeoBoundingBoxIT extends ESIntegTestCase {
         }
     }
 
-    // norelease
-    @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-7325")
-    public void testLimitsBoundingBox() throws Exception {
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
-        Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type1")
-                .startObject("properties").startObject("location").field("type", "geo_point");
-        if (version.before(Version.V_2_2_0)) {
-            xContentBuilder.field("lat_lon", true);
-        }
-        xContentBuilder.endObject().endObject().endObject().endObject();
-        assertAcked(prepareCreate("test").setSettings(settings).addMapping("type1", xContentBuilder));
-        ensureGreen();
-
-        client().prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject()
-                .startObject("location").field("lat", 40).field("lon", -20).endObject()
-                .endObject()).execute().actionGet();
-
-        client().prepareIndex("test", "type1", "2").setSource(jsonBuilder().startObject()
-                .startObject("location").field("lat", 40).field("lon", -10).endObject()
-                .endObject()).execute().actionGet();
-
-        client().prepareIndex("test", "type1", "3").setSource(jsonBuilder().startObject()
-                .startObject("location").field("lat", 40).field("lon", 10).endObject()
-                .endObject()).execute().actionGet();
-
-        client().prepareIndex("test", "type1", "4").setSource(jsonBuilder().startObject()
-                .startObject("location").field("lat", 40).field("lon", 20).endObject()
-                .endObject()).execute().actionGet();
-
-        client().prepareIndex("test", "type1", "5").setSource(jsonBuilder().startObject()
-                .startObject("location").field("lat", 10).field("lon", -170).endObject()
-                .endObject()).execute().actionGet();
-
-        client().prepareIndex("test", "type1", "6").setSource(jsonBuilder().startObject()
-                .startObject("location").field("lat", 0).field("lon", -170).endObject()
-                .endObject()).execute().actionGet();
-
-        client().prepareIndex("test", "type1", "7").setSource(jsonBuilder().startObject()
-                .startObject("location").field("lat", -10).field("lon", -170).endObject()
-                .endObject()).execute().actionGet();
-
-        client().prepareIndex("test", "type1", "8").setSource(jsonBuilder().startObject()
-                .startObject("location").field("lat", 10).field("lon", 170).endObject()
-                .endObject()).execute().actionGet();
-
-        client().prepareIndex("test", "type1", "9").setSource(jsonBuilder().startObject()
-                .startObject("location").field("lat", 0).field("lon", 170).endObject()
-                .endObject()).execute().actionGet();
-
-        client().prepareIndex("test", "type1", "10").setSource(jsonBuilder().startObject()
-                .startObject("location").field("lat", -10).field("lon", 170).endObject()
-                .endObject()).execute().actionGet();
-
-        refresh();
-
-        SearchResponse searchResponse = client().prepareSearch()
-                .setQuery(geoBoundingBoxQuery("location").setCorners(41, -11, 40, 9))
-                .execute().actionGet();
-        assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().hits().length, equalTo(1));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("2"));
-        searchResponse = client().prepareSearch()
-                .setQuery(geoBoundingBoxQuery("location").setCorners(41, -11, 40, 9).type("indexed"))
-                .execute().actionGet();
-        assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().hits().length, equalTo(1));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("2"));
-
-        searchResponse = client().prepareSearch()
-                .setQuery(geoBoundingBoxQuery("location").setCorners(41, -9, 40, 11))
-                .execute().actionGet();
-        assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().hits().length, equalTo(1));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("3"));
-        searchResponse = client().prepareSearch()
-                .setQuery(geoBoundingBoxQuery("location").setCorners(41, -9, 40, 11).type("indexed"))
-                .execute().actionGet();
-        assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().hits().length, equalTo(1));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("3"));
-
-        searchResponse = client().prepareSearch()
-                .setQuery(geoBoundingBoxQuery("location").setCorners(11, 171, 1, -169))
-                .execute().actionGet();
-        assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().hits().length, equalTo(1));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("5"));
-        searchResponse = client().prepareSearch()
-                .setQuery(geoBoundingBoxQuery("location").setCorners(11, 171, 1, -169).type("indexed"))
-                .execute().actionGet();
-        assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().hits().length, equalTo(1));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("5"));
-
-        searchResponse = client().prepareSearch()
-                .setQuery(geoBoundingBoxQuery("location").setCorners(9, 169, -1, -171))
-                .execute().actionGet();
-        assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().hits().length, equalTo(1));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("9"));
-        searchResponse = client().prepareSearch()
-                .setQuery(geoBoundingBoxQuery("location").setCorners(9, 169, -1, -171).type("indexed"))
-                .execute().actionGet();
-        assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
-        assertThat(searchResponse.getHits().hits().length, equalTo(1));
-        assertThat(searchResponse.getHits().getAt(0).id(), equalTo("9"));
-    }
-
     public void testLimit2BoundingBox() throws Exception {
         Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
diff --git a/distribution/licenses/lucene-analyzers-common-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-analyzers-common-6.1.0-snapshot-3a57bea.jar.sha1
deleted file mode 100644
index bb4cc98e068..00000000000
--- a/distribution/licenses/lucene-analyzers-common-6.1.0-snapshot-3a57bea.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-bf73c03e6b83f8e696133f40b9b1fc3381750149
\ No newline at end of file
diff --git a/distribution/licenses/lucene-analyzers-common-6.1.0.jar.sha1 b/distribution/licenses/lucene-analyzers-common-6.1.0.jar.sha1
new file mode 100644
index 00000000000..382bf79e91f
--- /dev/null
+++ b/distribution/licenses/lucene-analyzers-common-6.1.0.jar.sha1
@@ -0,0 +1 @@
+20c3c3a717a225df0b462216e70a57922a8edd28
\ No newline at end of file
diff --git a/distribution/licenses/lucene-backward-codecs-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-backward-codecs-6.1.0-snapshot-3a57bea.jar.sha1
deleted file mode 100644
index a8eb4883e15..00000000000
--- a/distribution/licenses/lucene-backward-codecs-6.1.0-snapshot-3a57bea.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8bc384f55faf99b6d6cee6f34df4fbd3145afb4d
\ No newline at end of file
diff --git a/distribution/licenses/lucene-backward-codecs-6.1.0.jar.sha1 b/distribution/licenses/lucene-backward-codecs-6.1.0.jar.sha1
new file mode 100644
index 00000000000..f13099389d2
--- /dev/null
+++ b/distribution/licenses/lucene-backward-codecs-6.1.0.jar.sha1
@@ -0,0 +1 @@
+cbbba4d2d0c1469e0cc3358489b72922ba4963bf
\ No newline at end of file
diff --git a/distribution/licenses/lucene-core-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-core-6.1.0-snapshot-3a57bea.jar.sha1
deleted file mode 100644
index 012fea19fdc..00000000000
--- a/distribution/licenses/lucene-core-6.1.0-snapshot-3a57bea.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fe19e7558440e10db4bd7150931dff6a7cf73243
\ No newline at end of file
diff --git a/distribution/licenses/lucene-core-6.1.0.jar.sha1 b/distribution/licenses/lucene-core-6.1.0.jar.sha1
new file mode 100644
index 00000000000..5c0b798a771
--- /dev/null
+++ b/distribution/licenses/lucene-core-6.1.0.jar.sha1
@@ -0,0 +1 @@
+763361bd8cb48161ae28d67a7ca20c4abf194ecb
\ No newline at end of file
diff --git a/distribution/licenses/lucene-grouping-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-grouping-6.1.0-snapshot-3a57bea.jar.sha1
deleted file mode 100644
index 3b3338ffd2f..00000000000
--- a/distribution/licenses/lucene-grouping-6.1.0-snapshot-3a57bea.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c3f0de4cdd185d23bce66c580d9c12adb98182a5
\ No newline at end of file
diff --git a/distribution/licenses/lucene-grouping-6.1.0.jar.sha1 b/distribution/licenses/lucene-grouping-6.1.0.jar.sha1
new file mode 100644
index 00000000000..74a68a00f05
--- /dev/null
+++ b/distribution/licenses/lucene-grouping-6.1.0.jar.sha1
@@ -0,0 +1 @@
+c40e8a1904557f67e3d14462d64865cf4a838123
\ No newline at end of file
diff --git a/distribution/licenses/lucene-highlighter-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-highlighter-6.1.0-snapshot-3a57bea.jar.sha1
deleted file mode 100644
index 255812ed5fb..00000000000
--- a/distribution/licenses/lucene-highlighter-6.1.0-snapshot-3a57bea.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ffb7087267bb6076b00c90f97ee36ebe23ea0662
\ No newline at end of file
diff --git a/distribution/licenses/lucene-highlighter-6.1.0.jar.sha1 b/distribution/licenses/lucene-highlighter-6.1.0.jar.sha1
new file mode 100644
index 00000000000..b26247fef51
--- /dev/null
+++ b/distribution/licenses/lucene-highlighter-6.1.0.jar.sha1
@@ -0,0 +1 @@
+6ab7c27256d3cd51022fb7130eb3e92391f24cdc
\ No newline at end of file
diff --git a/distribution/licenses/lucene-join-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-join-6.1.0-snapshot-3a57bea.jar.sha1
deleted file mode 100644
index 4231857b4e7..00000000000
--- a/distribution/licenses/lucene-join-6.1.0-snapshot-3a57bea.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d071ad17bed58b3267f6fa0b2a8211f8fe18c912
\ No newline at end of file
diff --git a/distribution/licenses/lucene-join-6.1.0.jar.sha1 b/distribution/licenses/lucene-join-6.1.0.jar.sha1
new file mode 100644
index 00000000000..2198e453138
--- /dev/null
+++ b/distribution/licenses/lucene-join-6.1.0.jar.sha1
@@ -0,0 +1 @@
+63411cef454a282c9ead56b25e0ca87daecbcf77
\ No newline at end of file
diff --git a/distribution/licenses/lucene-memory-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-memory-6.1.0-snapshot-3a57bea.jar.sha1
deleted file mode 100644
index 19aa64ccd80..00000000000
--- a/distribution/licenses/lucene-memory-6.1.0-snapshot-3a57bea.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f5e9b6eefe580a7f65276aca3192ca5796332509
\ No newline at end of file
diff --git a/distribution/licenses/lucene-memory-6.1.0.jar.sha1 b/distribution/licenses/lucene-memory-6.1.0.jar.sha1
new file mode 100644
index 00000000000..7937bebc519
--- /dev/null
+++ b/distribution/licenses/lucene-memory-6.1.0.jar.sha1
@@ -0,0 +1 @@
+e6702402615fcca549457842a08d21c35375a632
\ No newline at end of file
diff --git a/distribution/licenses/lucene-misc-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-misc-6.1.0-snapshot-3a57bea.jar.sha1
deleted file mode 100644
index 8480fcc3a49..00000000000
--- a/distribution/licenses/lucene-misc-6.1.0-snapshot-3a57bea.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6b84a79c37b01197130cceb65e5573794f073df1
\ No newline at end of file
diff --git a/distribution/licenses/lucene-misc-6.1.0.jar.sha1 b/distribution/licenses/lucene-misc-6.1.0.jar.sha1
new file mode 100644
index 00000000000..87e8d94d463
--- /dev/null
+++ b/distribution/licenses/lucene-misc-6.1.0.jar.sha1
@@ -0,0 +1 @@
+9a4884f1eaa7ac8af4abb51b45d0175ef7a3e03f
\ No newline at end of file
diff --git a/distribution/licenses/lucene-queries-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-queries-6.1.0-snapshot-3a57bea.jar.sha1
deleted file mode 100644
index a3ed70c12af..00000000000
--- a/distribution/licenses/lucene-queries-6.1.0-snapshot-3a57bea.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a9d51b77395dfdd7e6c4cf8c8506ebca5e1bb374
\ No newline at end of file
diff --git a/distribution/licenses/lucene-queries-6.1.0.jar.sha1 b/distribution/licenses/lucene-queries-6.1.0.jar.sha1
new file mode 100644
index 00000000000..687a6fa1ce6
--- /dev/null
+++ b/distribution/licenses/lucene-queries-6.1.0.jar.sha1
@@ -0,0 +1 @@
+2239ddd94a44d627b667221bdf129681af85ba51
\ No newline at end of file
diff --git a/distribution/licenses/lucene-queryparser-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-queryparser-6.1.0-snapshot-3a57bea.jar.sha1
deleted file mode 100644
index 6bf59bc00e7..00000000000
--- a/distribution/licenses/lucene-queryparser-6.1.0-snapshot-3a57bea.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e322f004e574df119ba08dd8751a743422a46724
\ No newline at end of file
diff --git a/distribution/licenses/lucene-queryparser-6.1.0.jar.sha1 b/distribution/licenses/lucene-queryparser-6.1.0.jar.sha1
new file mode 100644
index 00000000000..69df3d6e695
--- /dev/null
+++ b/distribution/licenses/lucene-queryparser-6.1.0.jar.sha1
@@ -0,0 +1 @@
+6b9e5e3dfc5b3c2689dcdc63d115c06be06c3837
\ No newline at end of file
diff --git a/distribution/licenses/lucene-sandbox-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-sandbox-6.1.0-snapshot-3a57bea.jar.sha1
deleted file mode 100644
index 79567427105..00000000000
--- a/distribution/licenses/lucene-sandbox-6.1.0-snapshot-3a57bea.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c7cb119652c906adcdf7fe64445c76d057329d63
\ No newline at end of file
diff --git a/distribution/licenses/lucene-sandbox-6.1.0.jar.sha1 b/distribution/licenses/lucene-sandbox-6.1.0.jar.sha1
new file mode 100644
index 00000000000..5790e6e19b1
--- /dev/null
+++ b/distribution/licenses/lucene-sandbox-6.1.0.jar.sha1
@@ -0,0 +1 @@
+53d35813f366d70ae0aef99d4f567d007290bdd2
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-spatial-6.1.0-snapshot-3a57bea.jar.sha1
deleted file mode 100644
index 2b06178aaad..00000000000
--- a/distribution/licenses/lucene-spatial-6.1.0-snapshot-3a57bea.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ca6c17fe31884e968ae63fd475ce6532b767c7fa
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial-6.1.0.jar.sha1 b/distribution/licenses/lucene-spatial-6.1.0.jar.sha1
new file mode 100644
index 00000000000..3031c7fbd96
--- /dev/null
+++ b/distribution/licenses/lucene-spatial-6.1.0.jar.sha1
@@ -0,0 +1 @@
+03ce415700267e5d329f2d01e599d13291aaef97
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial-extras-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-spatial-extras-6.1.0-snapshot-3a57bea.jar.sha1
deleted file mode 100644
index 9c487b7746f..00000000000
--- a/distribution/licenses/lucene-spatial-extras-6.1.0-snapshot-3a57bea.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-49235405e40757474aaa9e8e54946b67fe2a01d9
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial-extras-6.1.0.jar.sha1 b/distribution/licenses/lucene-spatial-extras-6.1.0.jar.sha1
new file mode 100644
index 00000000000..5d5c9fc40d2
--- /dev/null
+++ b/distribution/licenses/lucene-spatial-extras-6.1.0.jar.sha1
@@ -0,0 +1 @@
+eed865fccebb3c0a1ec2bebba1eddaaf9295c385
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial3d-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-spatial3d-6.1.0-snapshot-3a57bea.jar.sha1
deleted file mode 100644
index 1eaab9f9955..00000000000
--- a/distribution/licenses/lucene-spatial3d-6.1.0-snapshot-3a57bea.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-39f6b29c428327860c1a342bd57800e79ad92ef5
\ No newline at end of file
diff --git a/distribution/licenses/lucene-spatial3d-6.1.0.jar.sha1 b/distribution/licenses/lucene-spatial3d-6.1.0.jar.sha1
new file mode 100644
index 00000000000..4735753ddc0
--- /dev/null
+++ b/distribution/licenses/lucene-spatial3d-6.1.0.jar.sha1
@@ -0,0 +1 @@
+365a48f8d019aeeea34de1e80b03344fe3d4401b
\ No newline at end of file
diff --git a/distribution/licenses/lucene-suggest-6.1.0-snapshot-3a57bea.jar.sha1 b/distribution/licenses/lucene-suggest-6.1.0-snapshot-3a57bea.jar.sha1
deleted file mode 100644
index 948dfc4c2b9..00000000000
--- a/distribution/licenses/lucene-suggest-6.1.0-snapshot-3a57bea.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4e9f713d34fd4208bf308ac59132216f96521f13
\ No newline at end of file
diff --git a/distribution/licenses/lucene-suggest-6.1.0.jar.sha1 b/distribution/licenses/lucene-suggest-6.1.0.jar.sha1
new file mode 100644
index 00000000000..548cba1d048
--- /dev/null
+++ b/distribution/licenses/lucene-suggest-6.1.0.jar.sha1
@@ -0,0 +1 @@
+e39f0e09ff1e3d9874a3b072294b80cf6567e431
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-6.1.0-snapshot-3a57bea.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-6.1.0-snapshot-3a57bea.jar.sha1
deleted file mode 100644
index 271088e86c9..00000000000
--- a/modules/lang-expression/licenses/lucene-expressions-6.1.0-snapshot-3a57bea.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e5a4b673918f448006c0531799706abebe9a1db0
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-6.1.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-6.1.0.jar.sha1
new file mode 100644
index 00000000000..ac5b533d138
--- /dev/null
+++ b/modules/lang-expression/licenses/lucene-expressions-6.1.0.jar.sha1
@@ -0,0 +1 @@
+e722f2e6b4838ede6bf4f1c088fe7b261a7b7571
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.1.0-snapshot-3a57bea.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.1.0-snapshot-3a57bea.jar.sha1
deleted file mode 100644
index 013def114d4..00000000000
--- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.1.0-snapshot-3a57bea.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9cd8cbea5baef18a36bee86846a9ba026d2a02e0
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.1.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.1.0.jar.sha1
new file mode 100644
index 00000000000..96fef36cf4e
--- /dev/null
+++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-6.1.0.jar.sha1
@@ -0,0 +1 @@
+f580ed2ea0dca073199daa1a190ac142b3426030
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.1.0-snapshot-3a57bea.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.1.0-snapshot-3a57bea.jar.sha1
deleted file mode 100644
index 12c861c24ab..00000000000
--- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.1.0-snapshot-3a57bea.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-86c6d6a367ed658351bd8c8828d6ed647ac79b7e
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.1.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.1.0.jar.sha1
new file mode 100644
index 00000000000..bb0e327d275
--- /dev/null
+++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-6.1.0.jar.sha1
@@ -0,0 +1 @@
+dc436a2a2324e95fb27678d85ca6fd5018a5cec6
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.1.0-snapshot-3a57bea.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.1.0-snapshot-3a57bea.jar.sha1
deleted file mode 100644
index 6f571d75537..00000000000
--- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.1.0-snapshot-3a57bea.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6553bf764a69cd15e4fe1e55661382872795b853
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.1.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.1.0.jar.sha1
new file mode 100644
index 00000000000..94556c24056
--- /dev/null
+++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-6.1.0.jar.sha1
@@ -0,0 +1 @@
+1debc0cb187cde2bb2bcb3fc8a468f820d25b440
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.1.0-snapshot-3a57bea.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.1.0-snapshot-3a57bea.jar.sha1
deleted file mode 100644
index 2ea2d6b9622..00000000000
--- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.1.0-snapshot-3a57bea.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-979817950bc806400d8fa12a609ef215b5bdebd6
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.1.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.1.0.jar.sha1
new file mode 100644
index 00000000000..5abfc22ee1a
--- /dev/null
+++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-6.1.0.jar.sha1
@@ -0,0 +1 @@
+8e898fbd5da085f7b041feb3537a34137d2db560
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.1.0-snapshot-3a57bea.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.1.0-snapshot-3a57bea.jar.sha1
deleted file mode 100644
index 6677cfd3fc4..00000000000
--- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.1.0-snapshot-3a57bea.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2a720b647b6a202ec1d8d91db02006ae9539670b
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.1.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.1.0.jar.sha1
new file mode 100644
index 00000000000..ac449309ee2
--- /dev/null
+++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-6.1.0.jar.sha1
@@ -0,0 +1 @@
+51a52b0cd4ba5e686201917e65393feb56afd3a7
\ No newline at end of file

From f25676917991c5827da9247a85f51334d96612e6 Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Fri, 17 Jun 2016 09:22:15 +0200
Subject: [PATCH 0922/1311] Simplify NodeJoinController to make use of new
 cluster state batching infra (#18832)

The NodeJoinController is responsible for processing joins from nodes, both normally and during master election. For both use cases, the class processes incoming joins in batches in order to be efficient and to accumulated enough joins (i.e., >= min_master_nodes) to seal an election and ensure the new cluster state can be committed. Since the class was written, we introduced a new infrastructure to support batch changes to the cluster state at the `ClusterService` level. This commit rewrites NodeJoinController to use that infra and be simpler.

The PR also introduces a new concept to ClusterService allowing to submit tasks in batches, guaranteeing that all tasks submitted in a batch will be processed together (potentially with more tasks).  On top of that I added some extra safety checks to the ClusterService, around potential double submission of task objects into the queue.

This is done in preparation to revive #17811
---
 .../cluster/ClusterStateTaskExecutor.java     |   7 +-
 .../action/shard/ShardStateAction.java        |   5 +-
 .../cluster/node/DiscoveryNodes.java          |   4 +
 .../cluster/service/ClusterService.java       |  59 +-
 .../discovery/zen/NodeJoinController.java     | 569 +++++++++---------
 .../discovery/zen/ZenDiscovery.java           |   4 +-
 .../cluster/service/ClusterServiceTests.java  | 310 +++++++---
 .../zen/NodeJoinControllerTests.java          |  47 +-
 8 files changed, 611 insertions(+), 394 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java b/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java
index be9381a7fe6..ced8dd00982 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java
@@ -40,8 +40,10 @@ public interface ClusterStateTaskExecutor<T> {
     /**
      * Callback invoked after new cluster state is published. Note that
      * this method is not invoked if the cluster state was not updated.
+     * @param clusterChangedEvent the change event for this cluster state change, containing
+     *                            both old and new states
      */
-    default void clusterStatePublished(ClusterState newClusterState) {
+    default void clusterStatePublished(ClusterChangedEvent clusterChangedEvent) {
     }
 
     /**
@@ -92,7 +94,8 @@ public interface ClusterStateTaskExecutor<T> {
             }
 
             private Builder<T> result(T task, TaskResult executionResult) {
-                executionResults.put(task, executionResult);
+                TaskResult existing = executionResults.put(task, executionResult);
+                assert existing == null : task + " already has result " + existing;
                 return this;
             }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
index edeaf95443f..630852361b4 100644
--- a/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
+++ b/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
@@ -21,6 +21,7 @@ package org.elasticsearch.cluster.action.shard;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.ClusterStateTaskConfig;
@@ -312,8 +313,8 @@ public class ShardStateAction extends AbstractComponent {
         }
 
         @Override
-        public void clusterStatePublished(ClusterState newClusterState) {
-            int numberOfUnassignedShards = newClusterState.getRoutingNodes().unassigned().size();
+        public void clusterStatePublished(ClusterChangedEvent clusterChangedEvent) {
+            int numberOfUnassignedShards = clusterChangedEvent.state().getRoutingNodes().unassigned().size();
             if (numberOfUnassignedShards > 0) {
                 String reason = String.format(Locale.ROOT, "[%d] unassigned shards after failing shards", numberOfUnassignedShards);
                 if (logger.isTraceEnabled()) {
diff --git a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
index ed4cfbd9134..16ec31be0b1 100644
--- a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
+++ b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
@@ -614,5 +614,9 @@ public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements
         public static DiscoveryNodes readFrom(StreamInput in, @Nullable DiscoveryNode localNode) throws IOException {
             return PROTO.readFrom(in, localNode);
         }
+
+        public boolean isLocalNodeElectedMaster() {
+            return masterNodeId != null && masterNodeId.equals(localNodeId);
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
index ac4830fe0ec..13bbccc863a 100644
--- a/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
@@ -66,7 +66,9 @@ import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.IdentityHashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
@@ -371,34 +373,61 @@ public class ClusterService extends AbstractLifecycleComponent<ClusterService> {
     public <T> void submitStateUpdateTask(final String source, final T task,
                                           final ClusterStateTaskConfig config,
                                           final ClusterStateTaskExecutor<T> executor,
-                                          final ClusterStateTaskListener listener
-    ) {
-        innerSubmitStateUpdateTask(source, task, config, executor, safe(listener, logger));
+                                          final ClusterStateTaskListener listener) {
+        submitStateUpdateTasks(source, Collections.singletonMap(task, listener), config, executor);
     }
 
-    private <T> void innerSubmitStateUpdateTask(final String source, final T task,
-                                                final ClusterStateTaskConfig config,
-                                                final ClusterStateTaskExecutor executor,
-                                                final SafeClusterStateTaskListener listener) {
+    /**
+     * Submits a batch of cluster state update tasks; submitted updates are guaranteed to be processed together,
+     * potentially with more tasks of the same executor.
+     *
+     * @param source   the source of the cluster state update task
+     * @param tasks    a map of update tasks and their corresponding listeners
+     * @param config   the cluster state update task configuration
+     * @param executor the cluster state update task executor; tasks
+     *                 that share the same executor will be executed
+     *                 batches on this executor
+     * @param <T>      the type of the cluster state update task state
+     */
+    public <T> void submitStateUpdateTasks(final String source,
+                                           final Map<T, ClusterStateTaskListener> tasks, final ClusterStateTaskConfig config,
+                                           final ClusterStateTaskExecutor<T> executor) {
         if (!lifecycle.started()) {
             return;
         }
+        if (tasks.isEmpty()) {
+            return;
+        }
         try {
-            final UpdateTask<T> updateTask = new UpdateTask<>(source, task, config, executor, listener);
+            // convert to an identity map to check for dups based on update tasks semantics of using identity instead of equal
+            final IdentityHashMap<T, ClusterStateTaskListener> tasksIdentity = new IdentityHashMap<>(tasks);
+            final List<UpdateTask<T>> updateTasks = tasksIdentity.entrySet().stream().map(
+                entry -> new UpdateTask<>(source, entry.getKey(), config, executor, safe(entry.getValue(), logger))
+            ).collect(Collectors.toList());
 
             synchronized (updateTasksPerExecutor) {
-                updateTasksPerExecutor.computeIfAbsent(executor, k -> new ArrayList<>()).add(updateTask);
+                List<UpdateTask> existingTasks = updateTasksPerExecutor.computeIfAbsent(executor, k -> new ArrayList<>());
+                for (@SuppressWarnings("unchecked") UpdateTask<T> existing : existingTasks) {
+                    if (tasksIdentity.containsKey(existing.task)) {
+                        throw new IllegalArgumentException("task [" + existing.task + "] is already queued");
+                    }
+                }
+                existingTasks.addAll(updateTasks);
             }
 
+            final UpdateTask<T> firstTask = updateTasks.get(0);
+
             if (config.timeout() != null) {
-                updateTasksExecutor.execute(updateTask, threadPool.scheduler(), config.timeout(), () -> threadPool.generic().execute(() -> {
-                    if (updateTask.processed.getAndSet(true) == false) {
-                        logger.debug("cluster state update task [{}] timed out after [{}]", source, config.timeout());
-                        listener.onFailure(source, new ProcessClusterEventTimeoutException(config.timeout(), source));
+                updateTasksExecutor.execute(firstTask, threadPool.scheduler(), config.timeout(), () -> threadPool.generic().execute(() -> {
+                    for (UpdateTask<T> task : updateTasks) {
+                        if (task.processed.getAndSet(true) == false) {
+                            logger.debug("cluster state update task [{}] timed out after [{}]", source, config.timeout());
+                            task.listener.onFailure(source, new ProcessClusterEventTimeoutException(config.timeout(), source));
+                        }
                     }
                 }));
             } else {
-                updateTasksExecutor.execute(updateTask);
+                updateTasksExecutor.execute(firstTask);
             }
         } catch (EsRejectedExecutionException e) {
             // ignore cases where we are shutting down..., there is really nothing interesting
@@ -681,7 +710,7 @@ public class ClusterService extends AbstractLifecycleComponent<ClusterService> {
             }
 
             try {
-                executor.clusterStatePublished(newClusterState);
+                executor.clusterStatePublished(clusterChangedEvent);
             } catch (Exception e) {
                 logger.error("exception thrown while notifying executor of new cluster state publication [{}]", e, source);
             }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
index fdf084639d1..9659434e03d 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
@@ -18,9 +18,13 @@
  */
 package org.elasticsearch.discovery.zen;
 
-import org.elasticsearch.ElasticsearchTimeoutException;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.ClusterStateUpdateTask;
+import org.elasticsearch.cluster.ClusterStateTaskConfig;
+import org.elasticsearch.cluster.ClusterStateTaskExecutor;
+import org.elasticsearch.cluster.ClusterStateTaskListener;
 import org.elasticsearch.cluster.NotMasterException;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -30,21 +34,22 @@ import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.component.AbstractComponent;
+import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.discovery.zen.elect.ElectMasterService;
 import org.elasticsearch.discovery.zen.membership.MembershipAction;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
 
 /**
  * This class processes incoming join request (passed zia {@link ZenDiscovery}). Incoming nodes
@@ -52,18 +57,17 @@ import java.util.concurrent.atomic.AtomicReference;
  */
 public class NodeJoinController extends AbstractComponent {
 
-    final ClusterService clusterService;
-    final RoutingService routingService;
-    final ElectMasterService electMaster;
-    final DiscoverySettings discoverySettings;
-    final AtomicBoolean accumulateJoins = new AtomicBoolean(false);
+    private final ClusterService clusterService;
+    private final RoutingService routingService;
+    private final ElectMasterService electMaster;
+    private final DiscoverySettings discoverySettings;
+    private final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor();
 
-    // this is site while trying to become a master
-    final AtomicReference<ElectionContext> electionContext = new AtomicReference<>();
+    // this is set while trying to become a master
+    // mutation should be done under lock
+    private ElectionContext electionContext = null;
 
 
-    protected final Map<DiscoveryNode, List<MembershipAction.JoinCallback>> pendingJoinRequests = new HashMap<>();
-
     public NodeJoinController(ClusterService clusterService, RoutingService routingService, ElectMasterService electMaster, DiscoverySettings discoverySettings, Settings settings) {
         super(settings);
         this.clusterService = clusterService;
@@ -75,7 +79,7 @@ public class NodeJoinController extends AbstractComponent {
     /**
      * waits for enough incoming joins from master eligible nodes to complete the master election
      * <p>
-     * You must start accumulating joins before calling this method. See {@link #startAccumulatingJoins()}
+     * You must start accumulating joins before calling this method. See {@link #startElectionContext()}
      * <p>
      * The method will return once the local node has been elected as master or some failure/timeout has happened.
      * The exact outcome is communicated via the callback parameter, which is guaranteed to be called.
@@ -86,29 +90,32 @@ public class NodeJoinController extends AbstractComponent {
      *                            object
      **/
     public void waitToBeElectedAsMaster(int requiredMasterJoins, TimeValue timeValue, final ElectionCallback callback) {
-        assert accumulateJoins.get() : "waitToBeElectedAsMaster is called we are not accumulating joins";
-
         final CountDownLatch done = new CountDownLatch(1);
-        final ElectionContext newContext = new ElectionContext(callback, requiredMasterJoins) {
+        final ElectionCallback wrapperCallback = new ElectionCallback() {
             @Override
-            void onClose() {
-                if (electionContext.compareAndSet(this, null)) {
-                    stopAccumulatingJoins("election closed");
-                } else {
-                    assert false : "failed to remove current election context";
-                }
+            public void onElectedAsMaster(ClusterState state) {
                 done.countDown();
+                callback.onElectedAsMaster(state);
+            }
+
+            @Override
+            public void onFailure(Throwable t) {
+                done.countDown();
+                callback.onFailure(t);
             }
         };
 
-        if (electionContext.compareAndSet(null, newContext) == false) {
-            // should never happen, but be conservative
-            failContext(newContext, new IllegalStateException("double waiting for election"));
-            return;
-        }
+        ElectionContext myElectionContext = null;
+
         try {
             // check what we have so far..
-            checkPendingJoinsAndElectIfNeeded();
+            // capture the context we add the callback to make sure we fail our own
+            synchronized (this) {
+                assert electionContext != null : "waitToBeElectedAsMaster is called we are not accumulating joins";
+                myElectionContext = electionContext;
+                electionContext.onAttemptToBeElected(requiredMasterJoins, wrapperCallback);
+                checkPendingJoinsAndElectIfNeeded();
+            }
 
             try {
                 if (done.await(timeValue.millis(), TimeUnit.MILLISECONDS)) {
@@ -119,69 +126,46 @@ public class NodeJoinController extends AbstractComponent {
 
             }
             if (logger.isTraceEnabled()) {
-                final int pendingNodes;
-                synchronized (pendingJoinRequests) {
-                    pendingNodes = pendingJoinRequests.size();
-                }
-                logger.trace("timed out waiting to be elected. waited [{}]. pending node joins [{}]", timeValue, pendingNodes);
+                final int pendingNodes = myElectionContext.getPendingMasterJoinsCount();
+                logger.trace("timed out waiting to be elected. waited [{}]. pending master node joins [{}]", timeValue, pendingNodes);
             }
-            // callback will clear the context, if it's active
-            failContext(newContext, new ElasticsearchTimeoutException("timed out waiting to be elected"));
+            failContextIfNeeded(myElectionContext, "timed out waiting to be elected");
         } catch (Throwable t) {
             logger.error("unexpected failure while waiting for incoming joins", t);
-            failContext(newContext, "unexpected failure while waiting for pending joins", t);
+            if (myElectionContext != null) {
+                failContextIfNeeded(myElectionContext, "unexpected failure while waiting for pending joins [" + t.getMessage() + "]");
+            }
         }
     }
 
-    private void failContext(final ElectionContext context, final Throwable throwable) {
-        failContext(context, throwable.getMessage(), throwable);
-    }
-
-    /** utility method to fail the given election context under the cluster state thread */
-    private void failContext(final ElectionContext context, final String reason, final Throwable throwable) {
-        clusterService.submitStateUpdateTask("zen-disco-join(failure [" + reason + "])", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
-
-            @Override
-            public boolean runOnlyOnMaster() {
-                return false;
-            }
-
-            @Override
-            public ClusterState execute(ClusterState currentState) throws Exception {
-                context.onFailure(throwable);
-                return currentState;
-            }
-
-            @Override
-            public void onFailure(String source, Throwable updateFailure) {
-                logger.warn("unexpected error while trying to fail election context due to [{}]. original exception [{}]", updateFailure, reason, throwable);
-                context.onFailure(updateFailure);
-            }
-        });
-
+    /**
+     * utility method to fail the given election context under the cluster state thread
+     */
+    private synchronized void failContextIfNeeded(final ElectionContext context, final String reason) {
+        if (electionContext == context) {
+            stopElectionContext(reason);
+        }
     }
 
     /**
      * Accumulates any future incoming join request. Pending join requests will be processed in the final steps of becoming a
-     * master or when {@link #stopAccumulatingJoins(String)} is called.
+     * master or when {@link #stopElectionContext(String)} is called.
      */
-    public void startAccumulatingJoins() {
-        logger.trace("starting to accumulate joins");
-        boolean b = accumulateJoins.getAndSet(true);
-        assert b == false : "double startAccumulatingJoins() calls";
-        assert electionContext.get() == null : "startAccumulatingJoins() called, but there is an ongoing election context";
+    public synchronized void startElectionContext() {
+        logger.trace("starting an election context, will accumulate joins");
+        assert electionContext == null : "double startElectionContext() calls";
+        electionContext = new ElectionContext();
     }
 
-    /** Stopped accumulating joins. All pending joins will be processed. Future joins will be processed immediately */
-    public void stopAccumulatingJoins(String reason) {
-        logger.trace("stopping join accumulation ([{}])", reason);
-        assert electionContext.get() == null : "stopAccumulatingJoins() called, but there is an ongoing election context";
-        boolean b = accumulateJoins.getAndSet(false);
-        assert b : "stopAccumulatingJoins() called but not accumulating";
-        synchronized (pendingJoinRequests) {
-            if (pendingJoinRequests.size() > 0) {
-                processJoins("pending joins after accumulation stop [" + reason + "]");
-            }
+    /**
+     * Stopped accumulating joins. All pending joins will be processed. Future joins will be processed immediately
+     */
+    public void stopElectionContext(String reason) {
+        logger.trace("stopping election ([{}])", reason);
+        synchronized (this) {
+            assert electionContext != null : "stopElectionContext() called but not accumulating";
+            electionContext.closeAndProcessPending(reason);
+            electionContext = null;
         }
     }
 
@@ -190,19 +174,14 @@ public class NodeJoinController extends AbstractComponent {
      * <p>
      * Note: doesn't do any validation. This should have been done before.
      */
-    public void handleJoinRequest(final DiscoveryNode node, final MembershipAction.JoinCallback callback) {
-        synchronized (pendingJoinRequests) {
-            List<MembershipAction.JoinCallback> nodeCallbacks = pendingJoinRequests.get(node);
-            if (nodeCallbacks == null) {
-                nodeCallbacks = new ArrayList<>();
-                pendingJoinRequests.put(node, nodeCallbacks);
-            }
-            nodeCallbacks.add(callback);
-        }
-        if (accumulateJoins.get() == false) {
-            processJoins("join from node[" + node + "]");
-        } else {
+    public synchronized void handleJoinRequest(final DiscoveryNode node, final MembershipAction.JoinCallback callback) {
+        if (electionContext != null) {
+            electionContext.addIncomingJoin(node, callback);
             checkPendingJoinsAndElectIfNeeded();
+        } else {
+            clusterService.submitStateUpdateTask("zen-disco-join(node " + node + "])",
+                node, ClusterStateTaskConfig.build(Priority.URGENT),
+                joinTaskExecutor, new JoinTaskListener(callback, logger));
         }
     }
 
@@ -210,86 +189,24 @@ public class NodeJoinController extends AbstractComponent {
      * checks if there is an on going request to become master and if it has enough pending joins. If so, the node will
      * become master via a ClusterState update task.
      */
-    private void checkPendingJoinsAndElectIfNeeded() {
-        assert accumulateJoins.get() : "election check requested but we are not accumulating joins";
-        final ElectionContext context = electionContext.get();
-        if (context == null) {
-            return;
+    private synchronized void checkPendingJoinsAndElectIfNeeded() {
+        assert electionContext != null : "election check requested but no active context";
+        final int pendingMasterJoins = electionContext.getPendingMasterJoinsCount();
+        if (electionContext.isEnoughPendingJoins(pendingMasterJoins) == false) {
+            if (logger.isTraceEnabled()) {
+                logger.trace("not enough joins for election. Got [{}], required [{}]", pendingMasterJoins,
+                    electionContext.requiredMasterJoins);
+            }
+        } else {
+            if (logger.isTraceEnabled()) {
+                logger.trace("have enough joins for election. Got [{}], required [{}]", pendingMasterJoins,
+                    electionContext.requiredMasterJoins);
+            }
+            electionContext.closeAndBecomeMaster();
+            electionContext = null; // clear this out so future joins won't be accumulated
         }
-
-        int pendingMasterJoins = 0;
-        synchronized (pendingJoinRequests) {
-            for (DiscoveryNode node : pendingJoinRequests.keySet()) {
-                if (node.isMasterNode()) {
-                    pendingMasterJoins++;
-                }
-            }
-        }
-        if (pendingMasterJoins < context.requiredMasterJoins) {
-            if (context.pendingSetAsMasterTask.get() == false) {
-                logger.trace("not enough joins for election. Got [{}], required [{}]", pendingMasterJoins, context.requiredMasterJoins);
-            }
-            return;
-        }
-        if (context.pendingSetAsMasterTask.getAndSet(true)) {
-            logger.trace("elected as master task already submitted, ignoring...");
-            return;
-        }
-
-        final String source = "zen-disco-join(elected_as_master, [" + pendingMasterJoins + "] joins received)";
-        clusterService.submitStateUpdateTask(source, new ProcessJoinsTask(Priority.IMMEDIATE) {
-            @Override
-            public ClusterState execute(ClusterState currentState) {
-                // Take into account the previous known nodes, if they happen not to be available
-                // then fault detection will remove these nodes.
-
-                if (currentState.nodes().getMasterNode() != null) {
-                    // TODO can we tie break here? we don't have a remote master cluster state version to decide on
-                    logger.trace("join thread elected local node as master, but there is already a master in place: {}", currentState.nodes().getMasterNode());
-                    throw new NotMasterException("Node [" + clusterService.localNode() + "] not master for join request");
-                }
-
-                DiscoveryNodes.Builder builder = new DiscoveryNodes.Builder(currentState.nodes()).masterNodeId(currentState.nodes().getLocalNode().getId());
-                // update the fact that we are the master...
-                ClusterBlocks clusterBlocks = ClusterBlocks.builder().blocks(currentState.blocks()).removeGlobalBlock(discoverySettings.getNoMasterBlock()).build();
-                currentState = ClusterState.builder(currentState).nodes(builder).blocks(clusterBlocks).build();
-
-                // reroute now to remove any dead nodes (master may have stepped down when they left and didn't update the routing table)
-                RoutingAllocation.Result result = routingService.getAllocationService().reroute(currentState, "nodes joined");
-                if (result.changed()) {
-                    currentState = ClusterState.builder(currentState).routingResult(result).build();
-                }
-
-                // Add the incoming join requests.
-                // Note: we only do this now (after the reroute) to avoid assigning shards to these nodes.
-                return super.execute(currentState);
-            }
-
-            @Override
-            public boolean runOnlyOnMaster() {
-                return false;
-            }
-
-            @Override
-            public void onFailure(String source, Throwable t) {
-                super.onFailure(source, t);
-                context.onFailure(t);
-            }
-
-            @Override
-            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                super.clusterStateProcessed(source, oldState, newState);
-                context.onElectedAsMaster(newState);
-            }
-        });
     }
 
-    /** process all pending joins */
-    private void processJoins(String reason) {
-        clusterService.submitStateUpdateTask("zen-disco-join(" + reason + ")", new ProcessJoinsTask(Priority.URGENT));
-    }
-
-
     public interface ElectionCallback {
         /**
          * called when the local node is successfully elected as master
@@ -304,119 +221,143 @@ public class NodeJoinController extends AbstractComponent {
         void onFailure(Throwable t);
     }
 
-    static abstract class ElectionContext implements ElectionCallback {
-        private final ElectionCallback callback;
-        private final int requiredMasterJoins;
+    class ElectionContext {
+        private ElectionCallback callback = null;
+        private int requiredMasterJoins = -1;
+        private final Map<DiscoveryNode, List<MembershipAction.JoinCallback>> joinRequestAccumulator = new HashMap<>();
 
-        /** set to true after enough joins have been seen and a cluster update task is submitted to become master */
-        final AtomicBoolean pendingSetAsMasterTask = new AtomicBoolean();
         final AtomicBoolean closed = new AtomicBoolean();
 
-        ElectionContext(ElectionCallback callback, int requiredMasterJoins) {
-            this.callback = callback;
+        public synchronized void onAttemptToBeElected(int requiredMasterJoins, ElectionCallback callback) {
+            ensureOpen();
+            assert this.requiredMasterJoins < 0;
+            assert this.callback == null;
             this.requiredMasterJoins = requiredMasterJoins;
+            this.callback = callback;
         }
 
-        abstract void onClose();
+        public synchronized void addIncomingJoin(DiscoveryNode node, MembershipAction.JoinCallback callback) {
+            ensureOpen();
+            joinRequestAccumulator.computeIfAbsent(node, n -> new ArrayList<>()).add(callback);
+        }
 
-        @Override
-        public void onElectedAsMaster(ClusterState state) {
-            assert pendingSetAsMasterTask.get() : "onElectedAsMaster called but pendingSetAsMasterTask is not set";
+
+        public synchronized boolean isEnoughPendingJoins(int pendingMasterJoins) {
+            final boolean hasEnough;
+            if (requiredMasterJoins < 0) {
+                // requiredMasterNodes is unknown yet, return false and keep on waiting
+                hasEnough = false;
+            } else {
+                assert callback != null : "requiredMasterJoins is set but not the callback";
+                hasEnough = pendingMasterJoins >= requiredMasterJoins;
+            }
+            return hasEnough;
+        }
+
+        private Map<DiscoveryNode, ClusterStateTaskListener> getPendingAsTasks() {
+            Map<DiscoveryNode, ClusterStateTaskListener> tasks = new HashMap<>();
+            joinRequestAccumulator.entrySet().stream().forEach(e -> tasks.put(e.getKey(), new JoinTaskListener(e.getValue(), logger)));
+            return tasks;
+        }
+
+        public synchronized int getPendingMasterJoinsCount() {
+            int pendingMasterJoins = 0;
+            for (DiscoveryNode node : joinRequestAccumulator.keySet()) {
+                if (node.isMasterNode()) {
+                    pendingMasterJoins++;
+                }
+            }
+            return pendingMasterJoins;
+        }
+
+        public synchronized void closeAndBecomeMaster() {
+            assert callback != null : "becoming a master but the callback is not yet set";
+            assert isEnoughPendingJoins(getPendingMasterJoinsCount()) : "becoming a master but pending joins of "
+                + getPendingMasterJoinsCount() + " are not enough. needs [" + requiredMasterJoins + "];";
+
+            innerClose();
+
+            Map<DiscoveryNode, ClusterStateTaskListener> tasks = getPendingAsTasks();
+            final String source = "zen-disco-join(elected_as_master, [" + tasks.size() + "] nodes joined)";
+
+            tasks.put(BECOME_MASTER_TASK, joinProcessedListener);
+            clusterService.submitStateUpdateTasks(source, tasks, ClusterStateTaskConfig.build(Priority.URGENT), joinTaskExecutor);
+        }
+
+        public synchronized void closeAndProcessPending(String reason) {
+            innerClose();
+            Map<DiscoveryNode, ClusterStateTaskListener> tasks = getPendingAsTasks();
+            final String source = "zen-disco-join(election stopped [" + reason + "] nodes joined";
+
+            tasks.put(FINISH_ELECTION_NOT_MASTER_TASK, joinProcessedListener);
+            clusterService.submitStateUpdateTasks(source, tasks, ClusterStateTaskConfig.build(Priority.URGENT), joinTaskExecutor);
+        }
+
+        private void innerClose() {
+            if (closed.getAndSet(true)) {
+                throw new AlreadyClosedException("election context is already closed");
+            }
+        }
+
+        private void ensureOpen() {
+            if (closed.get()) {
+                throw new AlreadyClosedException("election context is already closed");
+            }
+        }
+
+        private synchronized ElectionCallback getCallback() {
+            return callback;
+        }
+
+        private void onElectedAsMaster(ClusterState state) {
             ClusterService.assertClusterStateThread();
             assert state.nodes().isLocalNodeElectedMaster() : "onElectedAsMaster called but local node is not master";
-            if (closed.compareAndSet(false, true)) {
-                try {
-                    onClose();
-                } finally {
-                    callback.onElectedAsMaster(state);
-                }
+            ElectionCallback callback = getCallback(); // get under lock
+            if (callback != null) {
+                callback.onElectedAsMaster(state);
             }
         }
 
-        @Override
-        public void onFailure(Throwable t) {
+        private void onFailure(Throwable t) {
             ClusterService.assertClusterStateThread();
-            if (closed.compareAndSet(false, true)) {
-                try {
-                    onClose();
-                } finally {
-                    callback.onFailure(t);
-                }
+            ElectionCallback callback = getCallback(); // get under lock
+            if (callback != null) {
+                callback.onFailure(t);
             }
         }
+
+        private final ClusterStateTaskListener joinProcessedListener = new ClusterStateTaskListener() {
+
+            @Override
+            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                assert newState.nodes().isLocalNodeElectedMaster() : "should have become a master but isn't " + newState.prettyPrint();
+                onElectedAsMaster(newState);
+            }
+
+            @Override
+            public void onFailure(String source, Throwable t) {
+                ElectionContext.this.onFailure(t);
+            }
+        };
+
     }
 
+    static class JoinTaskListener implements ClusterStateTaskListener {
+        final List<MembershipAction.JoinCallback> callbacks;
+        final private ESLogger logger;
 
-    /**
-     * Processes any pending joins via a ClusterState update task.
-     * Note: this task automatically fails (and fails all pending joins) if the current node is not marked as master
-     */
-    class ProcessJoinsTask extends ClusterStateUpdateTask {
+        JoinTaskListener(MembershipAction.JoinCallback callback, ESLogger logger) {
+            this(Collections.singletonList(callback), logger);
+        }
 
-        private final List<MembershipAction.JoinCallback> joinCallbacksToRespondTo = new ArrayList<>();
-        private boolean nodeAdded = false;
-
-        public ProcessJoinsTask(Priority priority) {
-            super(priority);
+        JoinTaskListener(List<MembershipAction.JoinCallback> callbacks, ESLogger logger) {
+            this.callbacks = callbacks;
+            this.logger = logger;
         }
 
         @Override
-        public ClusterState execute(ClusterState currentState) {
-            DiscoveryNodes.Builder nodesBuilder;
-            synchronized (pendingJoinRequests) {
-                if (pendingJoinRequests.isEmpty()) {
-                    return currentState;
-                }
-
-                nodesBuilder = DiscoveryNodes.builder(currentState.nodes());
-                Iterator<Map.Entry<DiscoveryNode, List<MembershipAction.JoinCallback>>> iterator = pendingJoinRequests.entrySet().iterator();
-                while (iterator.hasNext()) {
-                    Map.Entry<DiscoveryNode, List<MembershipAction.JoinCallback>> entry = iterator.next();
-                    final DiscoveryNode node = entry.getKey();
-                    joinCallbacksToRespondTo.addAll(entry.getValue());
-                    iterator.remove();
-                    if (currentState.nodes().nodeExists(node.getId())) {
-                        logger.debug("received a join request for an existing node [{}]", node);
-                    } else {
-                        nodeAdded = true;
-                        nodesBuilder.put(node);
-                        for (DiscoveryNode existingNode : currentState.nodes()) {
-                            if (node.getAddress().equals(existingNode.getAddress())) {
-                                nodesBuilder.remove(existingNode.getId());
-                                logger.warn("received join request from node [{}], but found existing node {} with same address, removing existing node", node, existingNode);
-                            }
-                        }
-                    }
-                }
-            }
-
-            // we must return a new cluster state instance to force publishing. This is important
-            // for the joining node to finalize it's join and set us as a master
-            final ClusterState.Builder newState = ClusterState.builder(currentState);
-            if (nodeAdded) {
-                newState.nodes(nodesBuilder);
-            }
-
-            return newState.build();
-        }
-
-        @Override
-        public void onNoLongerMaster(String source) {
-            // we are rejected, so drain all pending task (execute never run)
-            synchronized (pendingJoinRequests) {
-                Iterator<Map.Entry<DiscoveryNode, List<MembershipAction.JoinCallback>>> iterator = pendingJoinRequests.entrySet().iterator();
-                while (iterator.hasNext()) {
-                    Map.Entry<DiscoveryNode, List<MembershipAction.JoinCallback>> entry = iterator.next();
-                    joinCallbacksToRespondTo.addAll(entry.getValue());
-                    iterator.remove();
-                }
-            }
-            Exception e = new NotMasterException("Node [" + clusterService.localNode() + "] not master for join request");
-            innerOnFailure(e);
-        }
-
-        void innerOnFailure(Throwable t) {
-            for (MembershipAction.JoinCallback callback : joinCallbacksToRespondTo) {
+        public void onFailure(String source, Throwable t) {
+            for (MembershipAction.JoinCallback callback : callbacks) {
                 try {
                     callback.onFailure(t);
                 } catch (Exception e) {
@@ -425,29 +366,111 @@ public class NodeJoinController extends AbstractComponent {
             }
         }
 
-        @Override
-        public void onFailure(String source, Throwable t) {
-            logger.error("unexpected failure during [{}]", t, source);
-            innerOnFailure(t);
-        }
-
         @Override
         public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-            if (nodeAdded) {
-                // we reroute not in the same cluster state update since in certain areas we rely on
-                // the node to be in the cluster state (sampled from ClusterService#state) to be there, also
-                // shard transitions need to better be handled in such cases
-                routingService.reroute("post_node_add");
-            }
-            for (MembershipAction.JoinCallback callback : joinCallbacksToRespondTo) {
+            for (MembershipAction.JoinCallback callback : callbacks) {
                 try {
                     callback.onSuccess();
                 } catch (Exception e) {
                     logger.error("unexpected error during [{}]", e, source);
                 }
             }
+        }
+    }
 
-            NodeJoinController.this.electMaster.logMinimumMasterNodesWarningIfNecessary(oldState, newState);
+    // a task indicated that the current node should become master, if no current master is known
+    private final static DiscoveryNode BECOME_MASTER_TASK = new DiscoveryNode("_BECOME_MASTER_TASK_", DummyTransportAddress.INSTANCE,
+        Collections.emptyMap(), Collections.emptySet(), Version.CURRENT);
+
+    // a task that is used to process pending joins without explicitly becoming a master and listening to the results
+    // this task is used when election is stop without the local node becoming a master per se (though it might
+    private final static DiscoveryNode FINISH_ELECTION_NOT_MASTER_TASK = new DiscoveryNode("_NOT_MASTER_TASK_",
+        DummyTransportAddress.INSTANCE, Collections.emptyMap(), Collections.emptySet(), Version.CURRENT);
+
+    class JoinTaskExecutor implements ClusterStateTaskExecutor<DiscoveryNode> {
+
+        @Override
+        public BatchResult<DiscoveryNode> execute(ClusterState currentState, List<DiscoveryNode> joiningNodes) throws Exception {
+            final DiscoveryNodes currentNodes = currentState.nodes();
+            final BatchResult.Builder<DiscoveryNode> results = BatchResult.builder();
+            boolean nodesChanged = false;
+            ClusterState.Builder newState = ClusterState.builder(currentState);
+            DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(currentNodes);
+
+            if (currentNodes.getMasterNode() == null && joiningNodes.contains(BECOME_MASTER_TASK)) {
+                // use these joins to try and become the master.
+                // Note that we don't have to do any validation of the amount of joining nodes - the commit
+                // during the cluster state publishing guarantees that we have enough
+
+                nodesBuilder.masterNodeId(currentNodes.getLocalNodeId());
+                ClusterBlocks clusterBlocks = ClusterBlocks.builder().blocks(currentState.blocks())
+                    .removeGlobalBlock(discoverySettings.getNoMasterBlock()).build();
+                newState.blocks(clusterBlocks);
+                newState.nodes(nodesBuilder);
+                nodesChanged = true;
+
+                // reroute now to remove any dead nodes (master may have stepped down when they left and didn't update the routing table)
+                // Note: also do it now to avoid assigning shards to these nodes. We will have another reroute after the cluster
+                // state is published.
+                // TODO: this publishing of a cluster state with no nodes assigned to joining nodes shouldn't be needed anymore. remove.
+
+                final ClusterState tmpState = newState.build();
+                RoutingAllocation.Result result = routingService.getAllocationService().reroute(tmpState, "nodes joined");
+                newState = ClusterState.builder(tmpState);
+                if (result.changed()) {
+                    newState.routingResult(result);
+                }
+                nodesBuilder = DiscoveryNodes.builder(tmpState.nodes());
+            }
+
+            if (nodesBuilder.isLocalNodeElectedMaster() == false) {
+                logger.trace("processing node joins, but we are not the master. current master: {}", currentNodes.getMasterNode());
+                throw new NotMasterException("Node [" + currentNodes.getLocalNode() + "] not master for join request");
+            }
+
+            for (final DiscoveryNode node : joiningNodes) {
+                if (node.equals(BECOME_MASTER_TASK) || node.equals(FINISH_ELECTION_NOT_MASTER_TASK)) {
+                    // noop
+                } else if (currentNodes.nodeExists(node.getId())) {
+                    logger.debug("received a join request for an existing node [{}]", node);
+                } else {
+                    nodesChanged = true;
+                    nodesBuilder.put(node);
+                    for (DiscoveryNode existingNode : currentNodes) {
+                        if (node.getAddress().equals(existingNode.getAddress())) {
+                            nodesBuilder.remove(existingNode.getId());
+                            logger.warn("received join request from node [{}], but found existing node {} with same address, removing existing node", node, existingNode);
+                        }
+                    }
+                }
+                results.success(node);
+            }
+
+            if (nodesChanged) {
+                newState.nodes(nodesBuilder);
+            }
+
+            // we must return a new cluster state instance to force publishing. This is important
+            // for the joining node to finalize its join and set us as a master
+            return results.build(newState.build());
+        }
+
+        @Override
+        public boolean runOnlyOnMaster() {
+            // we validate that we are allowed to change the cluster state during cluster state processing
+            return false;
+        }
+
+        @Override
+        public void clusterStatePublished(ClusterChangedEvent event) {
+            if (event.nodesDelta().hasChanges()) {
+                // we reroute not in the same cluster state update since in certain areas we rely on
+                // the node to be in the cluster state (sampled from ClusterService#state) to be there, also
+                // shard transitions need to better be handled in such cases
+                routingService.reroute("post_node_add");
+            }
+
+            NodeJoinController.this.electMaster.logMinimumMasterNodesWarningIfNecessary(event.previousState(), event.state());
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
index 4d2b9e7c5e7..3c949b768d1 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
@@ -372,7 +372,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
     private void innerJoinCluster() {
         DiscoveryNode masterNode = null;
         final Thread currentThread = Thread.currentThread();
-        nodeJoinController.startAccumulatingJoins();
+        nodeJoinController.startElectionContext();
         while (masterNode == null && joinThreadControl.joinThreadActive(currentThread)) {
             masterNode = findMaster();
         }
@@ -406,7 +406,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
             );
         } else {
             // process any incoming joins (they will fail because we are not the master)
-            nodeJoinController.stopAccumulatingJoins("not master");
+            nodeJoinController.stopElectionContext("not master");
 
             // send join request
             final boolean success = joinElectedMaster(masterNode);
diff --git a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
index bf85f67e48e..dd71fa6a4a7 100644
--- a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
@@ -20,6 +20,7 @@ package org.elasticsearch.cluster.service;
 
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterName;
@@ -35,6 +36,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.OperationRouting;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.DummyTransportAddress;
@@ -51,8 +53,10 @@ import org.junit.BeforeClass;
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.BrokenBarrierException;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -62,12 +66,15 @@ import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static org.elasticsearch.test.ClusterServiceUtils.setState;
+import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.is;
 
 public class ClusterServiceTests extends ESTestCase {
@@ -102,10 +109,10 @@ public class ClusterServiceTests extends ESTestCase {
 
     TimedClusterService createTimedClusterService(boolean makeMaster) throws InterruptedException {
         TimedClusterService timedClusterService = new TimedClusterService(Settings.EMPTY, null,
-                new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-                threadPool, new ClusterName("ClusterServiceTests"));
+            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+            threadPool, new ClusterName("ClusterServiceTests"));
         timedClusterService.setLocalNode(new DiscoveryNode("node1", DummyTransportAddress.INSTANCE, emptyMap(),
-                emptySet(), Version.CURRENT));
+            emptySet(), Version.CURRENT));
         timedClusterService.setNodeConnectionsService(new NodeConnectionsService(Settings.EMPTY, null, null) {
             @Override
             public void connectToAddedNodes(ClusterChangedEvent event) {
@@ -123,9 +130,9 @@ public class ClusterServiceTests extends ESTestCase {
         ClusterState state = timedClusterService.state();
         final DiscoveryNodes nodes = state.nodes();
         final DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(nodes)
-                .masterNodeId(makeMaster ? nodes.getLocalNodeId() : null);
+            .masterNodeId(makeMaster ? nodes.getLocalNodeId() : null);
         state = ClusterState.builder(state).blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK)
-                .nodes(nodesBuilder).build();
+            .nodes(nodesBuilder).build();
         setState(timedClusterService, state);
         return timedClusterService;
     }
@@ -253,37 +260,37 @@ public class ClusterServiceTests extends ESTestCase {
         AtomicBoolean published = new AtomicBoolean();
 
         clusterService.submitStateUpdateTask(
-                "testClusterStateTaskListenerThrowingExceptionIsOkay",
-                new Object(),
-                ClusterStateTaskConfig.build(Priority.NORMAL),
-                new ClusterStateTaskExecutor<Object>() {
-                    @Override
-                    public boolean runOnlyOnMaster() {
-                        return false;
-                    }
-
-                    @Override
-                    public BatchResult<Object> execute(ClusterState currentState, List<Object> tasks) throws Exception {
-                        ClusterState newClusterState = ClusterState.builder(currentState).build();
-                        return BatchResult.builder().successes(tasks).build(newClusterState);
-                    }
-
-                    @Override
-                    public void clusterStatePublished(ClusterState newClusterState) {
-                        published.set(true);
-                        latch.countDown();
-                    }
-                },
-                new ClusterStateTaskListener() {
-                    @Override
-                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                        throw new IllegalStateException(source);
-                    }
-
-                    @Override
-                    public void onFailure(String source, Throwable t) {
-                    }
+            "testClusterStateTaskListenerThrowingExceptionIsOkay",
+            new Object(),
+            ClusterStateTaskConfig.build(Priority.NORMAL),
+            new ClusterStateTaskExecutor<Object>() {
+                @Override
+                public boolean runOnlyOnMaster() {
+                    return false;
                 }
+
+                @Override
+                public BatchResult<Object> execute(ClusterState currentState, List<Object> tasks) throws Exception {
+                    ClusterState newClusterState = ClusterState.builder(currentState).build();
+                    return BatchResult.builder().successes(tasks).build(newClusterState);
+                }
+
+                @Override
+                public void clusterStatePublished(ClusterChangedEvent clusterChangedEvent) {
+                    published.set(true);
+                    latch.countDown();
+                }
+            },
+            new ClusterStateTaskListener() {
+                @Override
+                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                    throw new IllegalStateException(source);
+                }
+
+                @Override
+                public void onFailure(String source, Throwable t) {
+                }
+            }
         );
 
         latch.await();
@@ -342,7 +349,7 @@ public class ClusterServiceTests extends ESTestCase {
                     barrier.await();
                     for (int j = 0; j < tasksSubmittedPerThread; j++) {
                         clusterService.submitStateUpdateTask("[" + index + "][" + j + "]", j,
-                                ClusterStateTaskConfig.build(randomFrom(Priority.values())), executors[index], listener);
+                            ClusterStateTaskConfig.build(randomFrom(Priority.values())), executors[index], listener);
                     }
                     barrier.await();
                 } catch (InterruptedException | BrokenBarrierException e) {
@@ -370,10 +377,43 @@ public class ClusterServiceTests extends ESTestCase {
         }
     }
 
+    public void testSingleBatchSubmission() throws InterruptedException {
+        Map<Integer, ClusterStateTaskListener> tasks = new HashMap<>();
+        final int numOfTasks = randomInt(10);
+        final CountDownLatch latch = new CountDownLatch(numOfTasks);
+        for (int i = 0; i < numOfTasks; i++) {
+            tasks.put(randomInt(1024), new ClusterStateTaskListener() {
+                @Override
+                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                    latch.countDown();
+                }
+
+                @Override
+                public void onFailure(String source, Throwable t) {
+                    fail(ExceptionsHelper.detailedMessage(t));
+                }
+            });
+        }
+
+        clusterService.submitStateUpdateTasks("test", tasks, ClusterStateTaskConfig.build(Priority.LANGUID),
+            (currentState, taskList) -> {
+                assertThat(taskList.size(), equalTo(tasks.size()));
+                assertThat(taskList.stream().collect(Collectors.toSet()), equalTo(tasks.keySet()));
+                return ClusterStateTaskExecutor.BatchResult.<Integer>builder().successes(taskList).build(currentState);
+            });
+
+        latch.await();
+    }
+
     public void testClusterStateBatchedUpdates() throws BrokenBarrierException, InterruptedException {
         AtomicInteger counter = new AtomicInteger();
         class Task {
             private AtomicBoolean state = new AtomicBoolean();
+            private final int id;
+
+            Task(int id) {
+                this.id = id;
+            }
 
             public void execute() {
                 if (!state.compareAndSet(false, true)) {
@@ -382,21 +422,54 @@ public class ClusterServiceTests extends ESTestCase {
                     counter.incrementAndGet();
                 }
             }
+
+            @Override
+            public boolean equals(Object o) {
+                if (this == o) {
+                    return true;
+                }
+                if (o == null || getClass() != o.getClass()) {
+                    return false;
+                }
+                Task task = (Task) o;
+                return id == task.id;
+
+            }
+
+            @Override
+            public int hashCode() {
+                return id;
+            }
+
+            @Override
+            public String toString() {
+                return Integer.toString(id);
+            }
         }
 
         int numberOfThreads = randomIntBetween(2, 8);
-        int tasksSubmittedPerThread = randomIntBetween(1, 1024);
+        int taskSubmissionsPerThread = randomIntBetween(1, 64);
         int numberOfExecutors = Math.max(1, numberOfThreads / 4);
         final Semaphore semaphore = new Semaphore(numberOfExecutors);
 
         class TaskExecutor implements ClusterStateTaskExecutor<Task> {
+            private final List<Set<Task>> taskGroups;
             private AtomicInteger counter = new AtomicInteger();
             private AtomicInteger batches = new AtomicInteger();
             private AtomicInteger published = new AtomicInteger();
 
+            public TaskExecutor(List<Set<Task>> taskGroups) {
+                this.taskGroups = taskGroups;
+            }
+
             @Override
             public BatchResult<Task> execute(ClusterState currentState, List<Task> tasks) throws Exception {
-                tasks.forEach(task -> task.execute());
+                for (Set<Task> expectedSet : taskGroups) {
+                    long count = tasks.stream().filter(expectedSet::contains).count();
+                    assertThat("batched set should be executed together or not at all. Expected " + expectedSet + "s. Executing " + tasks,
+                        count, anyOf(equalTo(0L), equalTo((long) expectedSet.size())));
+                }
+                tasks.forEach(Task::execute);
                 counter.addAndGet(tasks.size());
                 ClusterState maybeUpdatedClusterState = currentState;
                 if (randomBoolean()) {
@@ -413,59 +486,85 @@ public class ClusterServiceTests extends ESTestCase {
             }
 
             @Override
-            public void clusterStatePublished(ClusterState newClusterState) {
+            public void clusterStatePublished(ClusterChangedEvent clusterChangedEvent) {
                 published.incrementAndGet();
                 semaphore.release();
             }
         }
 
-        ConcurrentMap<String, AtomicInteger> counters = new ConcurrentHashMap<>();
-        CountDownLatch updateLatch = new CountDownLatch(numberOfThreads * tasksSubmittedPerThread);
-        ClusterStateTaskListener listener = new ClusterStateTaskListener() {
-            @Override
-            public void onFailure(String source, Throwable t) {
-                assert false;
-            }
-
-            @Override
-            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                counters.computeIfAbsent(source, key -> new AtomicInteger()).incrementAndGet();
-                updateLatch.countDown();
-            }
-        };
+        ConcurrentMap<String, AtomicInteger> processedStates = new ConcurrentHashMap<>();
 
+        List<Set<Task>> taskGroups = new ArrayList<>();
         List<TaskExecutor> executors = new ArrayList<>();
         for (int i = 0; i < numberOfExecutors; i++) {
-            executors.add(new TaskExecutor());
+            executors.add(new TaskExecutor(taskGroups));
         }
 
         // randomly assign tasks to executors
-        List<TaskExecutor> assignments = new ArrayList<>();
+        List<Tuple<TaskExecutor, Set<Task>>> assignments = new ArrayList<>();
+        int taskId = 0;
         for (int i = 0; i < numberOfThreads; i++) {
-            for (int j = 0; j < tasksSubmittedPerThread; j++) {
-                assignments.add(randomFrom(executors));
+            for (int j = 0; j < taskSubmissionsPerThread; j++) {
+                TaskExecutor executor = randomFrom(executors);
+                Set<Task> tasks = new HashSet<>();
+                for (int t = randomInt(3); t >= 0; t--) {
+                    tasks.add(new Task(taskId++));
+                }
+                taskGroups.add(tasks);
+                assignments.add(Tuple.tuple(executor, tasks));
             }
         }
 
         Map<TaskExecutor, Integer> counts = new HashMap<>();
-        for (TaskExecutor executor : assignments) {
-            counts.merge(executor, 1, (previous, one) -> previous + one);
+        int totalTaskCount = 0;
+        for (Tuple<TaskExecutor, Set<Task>> assignment : assignments) {
+            final int taskCount = assignment.v2().size();
+            counts.merge(assignment.v1(), taskCount, (previous, count) -> previous + count);
+            totalTaskCount += taskCount;
         }
+        final CountDownLatch updateLatch = new CountDownLatch(totalTaskCount);
+        final ClusterStateTaskListener listener = new ClusterStateTaskListener() {
+            @Override
+            public void onFailure(String source, Throwable t) {
+                fail(ExceptionsHelper.detailedMessage(t));
+            }
 
+            @Override
+            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                processedStates.computeIfAbsent(source, key -> new AtomicInteger()).incrementAndGet();
+                updateLatch.countDown();
+            }
+        };
+
+        final ConcurrentMap<String, AtomicInteger> submittedTasksPerThread = new ConcurrentHashMap<>();
         CyclicBarrier barrier = new CyclicBarrier(1 + numberOfThreads);
         for (int i = 0; i < numberOfThreads; i++) {
             final int index = i;
             Thread thread = new Thread(() -> {
+                final String threadName = Thread.currentThread().getName();
                 try {
                     barrier.await();
-                    for (int j = 0; j < tasksSubmittedPerThread; j++) {
-                        ClusterStateTaskExecutor<Task> executor = assignments.get(index * tasksSubmittedPerThread + j);
-                        clusterService.submitStateUpdateTask(
-                                Thread.currentThread().getName(),
-                                new Task(),
+                    for (int j = 0; j < taskSubmissionsPerThread; j++) {
+                        Tuple<TaskExecutor, Set<Task>> assignment = assignments.get(index * taskSubmissionsPerThread + j);
+                        final Set<Task> tasks = assignment.v2();
+                        submittedTasksPerThread.computeIfAbsent(threadName, key -> new AtomicInteger()).addAndGet(tasks.size());
+                        final TaskExecutor executor = assignment.v1();
+                        if (tasks.size() == 1) {
+                            clusterService.submitStateUpdateTask(
+                                threadName,
+                                tasks.stream().findFirst().get(),
                                 ClusterStateTaskConfig.build(randomFrom(Priority.values())),
                                 executor,
                                 listener);
+                        } else {
+                            Map<Task, ClusterStateTaskListener> taskListeners = new HashMap<>();
+                            tasks.stream().forEach(t -> taskListeners.put(t, listener));
+                            clusterService.submitStateUpdateTasks(
+                                threadName,
+                                taskListeners, ClusterStateTaskConfig.build(randomFrom(Priority.values())),
+                                executor
+                            );
+                        }
                     }
                     barrier.await();
                 } catch (BrokenBarrierException | InterruptedException e) {
@@ -486,7 +585,7 @@ public class ClusterServiceTests extends ESTestCase {
         semaphore.acquire(numberOfExecutors);
 
         // assert the number of executed tasks is correct
-        assertEquals(numberOfThreads * tasksSubmittedPerThread, counter.get());
+        assertEquals(totalTaskCount, counter.get());
 
         // assert each executor executed the correct number of tasks
         for (TaskExecutor executor : executors) {
@@ -497,8 +596,10 @@ public class ClusterServiceTests extends ESTestCase {
         }
 
         // assert the correct number of clusterStateProcessed events were triggered
-        for (Map.Entry<String, AtomicInteger> entry : counters.entrySet()) {
-            assertEquals(entry.getValue().get(), tasksSubmittedPerThread);
+        for (Map.Entry<String, AtomicInteger> entry : processedStates.entrySet()) {
+            assertThat(submittedTasksPerThread, hasKey(entry.getKey()));
+            assertEquals("not all tasks submitted by " + entry.getKey() + " received a processed event",
+                entry.getValue().get(), submittedTasksPerThread.get(entry.getKey()).get());
         }
     }
 
@@ -506,9 +607,6 @@ public class ClusterServiceTests extends ESTestCase {
      * Note, this test can only work as long as we have a single thread executor executing the state update tasks!
      */
     public void testPrioritizedTasks() throws Exception {
-        Settings settings = Settings.builder()
-                .put("discovery.type", "local")
-                .build();
         BlockingTask block = new BlockingTask(Priority.IMMEDIATE);
         clusterService.submitStateUpdateTask("test", block);
         int taskCount = randomIntBetween(5, 20);
@@ -522,7 +620,7 @@ public class ClusterServiceTests extends ESTestCase {
             clusterService.submitStateUpdateTask("test", new PrioritizedTask(priority, latch, tasks));
         }
 
-        block.release();
+        block.close();
         latch.await();
 
         Priority prevPriority = null;
@@ -535,15 +633,49 @@ public class ClusterServiceTests extends ESTestCase {
         }
     }
 
+    public void testDuplicateSubmission() throws InterruptedException {
+        final CountDownLatch latch = new CountDownLatch(2);
+        try (BlockingTask blockingTask = new BlockingTask(Priority.IMMEDIATE)) {
+            clusterService.submitStateUpdateTask("blocking", blockingTask);
+
+            ClusterStateTaskExecutor<SimpleTask> executor = (currentState, tasks) ->
+                ClusterStateTaskExecutor.BatchResult.<SimpleTask>builder().successes(tasks).build(currentState);
+
+            SimpleTask task = new SimpleTask(1);
+            ClusterStateTaskListener listener = new ClusterStateTaskListener() {
+                @Override
+                public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                    latch.countDown();
+                }
+
+                @Override
+                public void onFailure(String source, Throwable t) {
+                    fail(ExceptionsHelper.detailedMessage(t));
+                }
+            };
+
+            clusterService.submitStateUpdateTask("first time", task, ClusterStateTaskConfig.build(Priority.NORMAL), executor, listener);
+
+            expectThrows(IllegalArgumentException.class, () -> clusterService.submitStateUpdateTask("second time", task,
+                ClusterStateTaskConfig.build(Priority.NORMAL), executor, listener));
+
+            clusterService.submitStateUpdateTask("third time a charm", new SimpleTask(1),
+                ClusterStateTaskConfig.build(Priority.NORMAL), executor, listener);
+
+            assertThat(latch.getCount(), equalTo(2L));
+        }
+        latch.await();
+    }
+
     @TestLogging("cluster:TRACE") // To ensure that we log cluster state events on TRACE level
     public void testClusterStateUpdateLogging() throws Exception {
         MockLogAppender mockAppender = new MockLogAppender();
         mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test1", "cluster.service", Level.DEBUG,
-                "*processing [test1]: took [1s] no change in cluster_state"));
+            "*processing [test1]: took [1s] no change in cluster_state"));
         mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test2", "cluster.service", Level.TRACE,
-                "*failed to execute cluster state update in [2s]*"));
+            "*failed to execute cluster state update in [2s]*"));
         mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test3", "cluster.service", Level.DEBUG,
-                "*processing [test3]: took [3s] done applying updated cluster_state (version: *, uuid: *)"));
+            "*processing [test3]: took [3s] done applying updated cluster_state (version: *, uuid: *)"));
 
         Logger rootLogger = Logger.getRootLogger();
         rootLogger.addAppender(mockAppender);
@@ -630,13 +762,13 @@ public class ClusterServiceTests extends ESTestCase {
     public void testLongClusterStateUpdateLogging() throws Exception {
         MockLogAppender mockAppender = new MockLogAppender();
         mockAppender.addExpectation(new MockLogAppender.UnseenEventExpectation("test1 shouldn't see because setting is too low",
-                "cluster.service", Level.WARN, "*cluster state update task [test1] took [*] above the warn threshold of *"));
+            "cluster.service", Level.WARN, "*cluster state update task [test1] took [*] above the warn threshold of *"));
         mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test2", "cluster.service", Level.WARN,
-                "*cluster state update task [test2] took [32s] above the warn threshold of *"));
+            "*cluster state update task [test2] took [32s] above the warn threshold of *"));
         mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test3", "cluster.service", Level.WARN,
-                "*cluster state update task [test3] took [33s] above the warn threshold of *"));
+            "*cluster state update task [test3] took [33s] above the warn threshold of *"));
         mockAppender.addExpectation(new MockLogAppender.SeenEventExpectation("test4", "cluster.service", Level.WARN,
-                "*cluster state update task [test4] took [34s] above the warn threshold of *"));
+            "*cluster state update task [test4] took [34s] above the warn threshold of *"));
 
         Logger rootLogger = Logger.getRootLogger();
         rootLogger.addAppender(mockAppender);
@@ -740,7 +872,25 @@ public class ClusterServiceTests extends ESTestCase {
         mockAppender.assertAllExpectationsMatched();
     }
 
-    private static class BlockingTask extends ClusterStateUpdateTask {
+    private static class SimpleTask {
+        private final int id;
+
+        private SimpleTask(int id) {
+            this.id = id;
+        }
+
+        @Override
+        public int hashCode() {
+            return super.hashCode();
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            return super.equals(obj);
+        }
+    }
+
+    private static class BlockingTask extends ClusterStateUpdateTask implements Releasable {
         private final CountDownLatch latch = new CountDownLatch(1);
 
         public BlockingTask(Priority priority) {
@@ -757,7 +907,7 @@ public class ClusterServiceTests extends ESTestCase {
         public void onFailure(String source, Throwable t) {
         }
 
-        public void release() {
+        public void close() {
             latch.countDown();
         }
 
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
index b7db9f9e609..27c38e66074 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.discovery.zen;
 
-import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
@@ -99,11 +98,11 @@ public class NodeJoinControllerTests extends ESTestCase {
         final DiscoveryNode localNode = initialNodes.getLocalNode();
         // make sure we have a master
         setState(clusterService, ClusterState.builder(clusterService.state()).nodes(
-                DiscoveryNodes.builder(initialNodes).masterNodeId(localNode.getId())));
+            DiscoveryNodes.builder(initialNodes).masterNodeId(localNode.getId())));
         nodeJoinController = new NodeJoinController(clusterService, new NoopRoutingService(Settings.EMPTY),
-                new ElectMasterService(Settings.EMPTY, Version.CURRENT),
-                new DiscoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)),
-                Settings.EMPTY);
+            new ElectMasterService(Settings.EMPTY, Version.CURRENT),
+            new DiscoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)),
+            Settings.EMPTY);
     }
 
     @After
@@ -122,14 +121,14 @@ public class NodeJoinControllerTests extends ESTestCase {
             nodes.add(node);
             joinNode(node);
         }
-        nodeJoinController.startAccumulatingJoins();
+        nodeJoinController.startElectionContext();
         ArrayList<Future<Void>> pendingJoins = new ArrayList<>();
         for (int i = randomInt(5); i > 0; i--) {
             DiscoveryNode node = newNode(nodeId++);
             nodes.add(node);
             pendingJoins.add(joinNodeAsync(node));
         }
-        nodeJoinController.stopAccumulatingJoins("test");
+        nodeJoinController.stopElectionContext("test");
         boolean hadSyncJoin = false;
         for (int i = randomInt(5); i > 0; i--) {
             DiscoveryNode node = newNode(nodeId++);
@@ -145,8 +144,6 @@ public class NodeJoinControllerTests extends ESTestCase {
         for (Future<Void> joinFuture : pendingJoins) {
             joinFuture.get();
         }
-
-        assertNodesInCurrentState(nodes);
     }
 
     public void testFailingJoinsWhenNotMaster() throws ExecutionException, InterruptedException {
@@ -163,14 +160,14 @@ public class NodeJoinControllerTests extends ESTestCase {
 
         logger.debug("--> testing joins fail post accumulation");
         ArrayList<Future<Void>> pendingJoins = new ArrayList<>();
-        nodeJoinController.startAccumulatingJoins();
+        nodeJoinController.startElectionContext();
         for (int i = 1 + randomInt(5); i > 0; i--) {
             DiscoveryNode node = newNode(nodeId++);
             final Future<Void> future = joinNodeAsync(node);
             pendingJoins.add(future);
             assertThat(future.isDone(), equalTo(false));
         }
-        nodeJoinController.stopAccumulatingJoins("test");
+        nodeJoinController.stopElectionContext("test");
         for (Future<Void> future : pendingJoins) {
             try {
                 future.get();
@@ -197,7 +194,7 @@ public class NodeJoinControllerTests extends ESTestCase {
             }
         }
 
-        nodeJoinController.startAccumulatingJoins();
+        nodeJoinController.startElectionContext();
         final SimpleFuture electionFuture = new SimpleFuture("master election");
         final Thread masterElection = new Thread(new AbstractRunnable() {
             @Override
@@ -245,7 +242,7 @@ public class NodeJoinControllerTests extends ESTestCase {
             }
         }
 
-        nodeJoinController.startAccumulatingJoins();
+        nodeJoinController.startElectionContext();
         final SimpleFuture electionFuture = new SimpleFuture("master election");
         final Thread masterElection = new Thread(new AbstractRunnable() {
             @Override
@@ -334,8 +331,8 @@ public class NodeJoinControllerTests extends ESTestCase {
         }
 
         logger.debug("--> testing accumulation stopped");
-        nodeJoinController.startAccumulatingJoins();
-        nodeJoinController.stopAccumulatingJoins("test");
+        nodeJoinController.startElectionContext();
+        nodeJoinController.stopElectionContext("test");
 
     }
 
@@ -356,7 +353,7 @@ public class NodeJoinControllerTests extends ESTestCase {
             }
         }
 
-        nodeJoinController.startAccumulatingJoins();
+        nodeJoinController.startElectionContext();
         final int initialJoins = randomIntBetween(0, requiredJoins - 1);
         final ArrayList<SimpleFuture> pendingJoins = new ArrayList<>();
         ArrayList<DiscoveryNode> nodesToJoin = new ArrayList<>();
@@ -389,7 +386,7 @@ public class NodeJoinControllerTests extends ESTestCase {
         });
         latch.await();
         logger.debug("--> verifying election timed out");
-        assertThat(failure.get(), instanceOf(ElasticsearchTimeoutException.class));
+        assertThat(failure.get(), instanceOf(NotMasterException.class));
 
         logger.debug("--> verifying all joins are failed");
         for (SimpleFuture future : pendingJoins) {
@@ -407,7 +404,7 @@ public class NodeJoinControllerTests extends ESTestCase {
         ClusterState state = clusterService.state();
         final DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(state.nodes());
         final DiscoveryNode other_node = new DiscoveryNode("other_node", DummyTransportAddress.INSTANCE,
-                emptyMap(), emptySet(), Version.CURRENT);
+            emptyMap(), emptySet(), Version.CURRENT);
         nodesBuilder.put(other_node);
         setState(clusterService, ClusterState.builder(state).nodes(nodesBuilder));
 
@@ -457,7 +454,7 @@ public class NodeJoinControllerTests extends ESTestCase {
         DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(clusterService.state().nodes()).masterNodeId(null);
         setState(clusterService, ClusterState.builder(clusterService.state()).nodes(nodesBuilder));
 
-        nodeJoinController.startAccumulatingJoins();
+        nodeJoinController.startElectionContext();
 
         Thread[] threads = new Thread[3 + randomInt(5)];
         final int requiredJoins = randomInt(threads.length);
@@ -589,7 +586,9 @@ public class NodeJoinControllerTests extends ESTestCase {
     private SimpleFuture joinNodeAsync(final DiscoveryNode node) throws InterruptedException {
         final SimpleFuture future = new SimpleFuture("join of " + node + " (id [" + joinId.incrementAndGet() + "]");
         logger.debug("starting {}", future);
-        nodeJoinController.handleJoinRequest(node, new MembershipAction.JoinCallback() {
+        // clone the node before submitting to simulate an incoming join, which is guaranteed to have a new
+        // disco node object serialized off the network
+        nodeJoinController.handleJoinRequest(cloneNode(node), new MembershipAction.JoinCallback() {
             @Override
             public void onSuccess() {
                 logger.debug("{} completed", future);
@@ -605,6 +604,14 @@ public class NodeJoinControllerTests extends ESTestCase {
         return future;
     }
 
+    /**
+     * creates an object clone of node, so it will be a different object instance
+     */
+    private DiscoveryNode cloneNode(DiscoveryNode node) {
+        return new DiscoveryNode(node.getName(), node.getId(), node.getHostName(), node.getHostAddress(), node.getAddress(),
+            node.getAttributes(), node.getRoles(), node.getVersion());
+    }
+
     private void joinNode(final DiscoveryNode node) throws InterruptedException, ExecutionException {
         joinNodeAsync(node).get();
     }

From 5284c5094d05c03649b70eb42ae4f0c7bc9c2ce6 Mon Sep 17 00:00:00 2001
From: Glen Smith <github@smithsrock.com>
Date: Fri, 17 Jun 2016 00:27:37 -0400
Subject: [PATCH 0923/1311] grammar

---
 docs/reference/aggregations/bucket/filters-aggregation.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/aggregations/bucket/filters-aggregation.asciidoc b/docs/reference/aggregations/bucket/filters-aggregation.asciidoc
index a7e07acc660..5f5d6c54b6a 100644
--- a/docs/reference/aggregations/bucket/filters-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/filters-aggregation.asciidoc
@@ -1,7 +1,7 @@
 [[search-aggregations-bucket-filters-aggregation]]
 === Filters Aggregation
 
-Defines a multi bucket aggregations where each bucket is associated with a
+Defines a multi bucket aggregation where each bucket is associated with a
 filter. Each bucket will collect all documents that match its associated
 filter.
 

From 529c2ca13f3e689dcd3e4f856070db5dc125c605 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Fri, 17 Jun 2016 12:17:48 +0200
Subject: [PATCH 0924/1311] Add did-you-mean for plugin cli This commit adds
 error messages like: `Unknown plugin xpack, did you mean [x-pack]?`

Closes #18896
---
 .../plugins/InstallPluginCommand.java         | 25 ++++++++++++++++++-
 .../plugins/InstallPluginCommandTests.java    | 16 ++++++++++++
 2 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
index 643998f19c2..51bb3b6b82f 100644
--- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
@@ -45,11 +45,14 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.TreeSet;
+import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
+import org.apache.lucene.search.spell.LevensteinDistance;
+import org.apache.lucene.util.CollectionUtil;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
 import org.elasticsearch.bootstrap.JarHell;
@@ -57,6 +60,7 @@ import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.SettingCommand;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserError;
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.hash.MessageDigests;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.settings.Settings;
@@ -239,12 +243,31 @@ class InstallPluginCommand extends SettingCommand {
         // fall back to plain old URL
         if (pluginId.contains(":/") == false) {
             // definitely not a valid url, so assume it is a plugin name
-            throw new UserError(ExitCodes.USAGE, "Unknown plugin " + pluginId);
+            List<String> plugins = checkMisspelledPlugin(pluginId);
+            String msg = "Unknown plugin " + pluginId;
+            if (plugins.isEmpty() == false) {
+                msg += ", did you mean " + (plugins.size() == 1 ? "[" + plugins.get(0) + "]": "any of " + plugins.toString()) + "?";
+            }
+            throw new UserError(ExitCodes.USAGE, msg);
         }
         terminal.println("-> Downloading " + URLDecoder.decode(pluginId, "UTF-8"));
         return downloadZip(terminal, pluginId, tmpDir);
     }
 
+    /** Returns all the official plugin names that look similar to pluginId. **/
+    private List<String> checkMisspelledPlugin(String pluginId) {
+        LevensteinDistance ld = new LevensteinDistance();
+        List<Tuple<Float, String>> scoredKeys = new ArrayList<>();
+        for (String officialPlugin : OFFICIAL_PLUGINS) {
+            float distance = ld.getDistance(pluginId, officialPlugin);
+            if (distance > 0.7f) {
+                scoredKeys.add(new Tuple<>(distance, officialPlugin));
+            }
+        }
+        CollectionUtil.timSort(scoredKeys, (a, b) -> b.v1().compareTo(a.v1()));
+        return scoredKeys.stream().map((a) -> a.v2()).collect(Collectors.toList());
+    }
+
     /** Downloads a zip from the url, into a temp file under the given temp dir. */
     private Path downloadZip(Terminal terminal, String urlString, Path tmpDir) throws IOException {
         terminal.println(VERBOSE, "Retrieving zip from " + urlString);
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
index fdf52da2634..2b9b6ec6ab9 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
@@ -67,6 +67,7 @@ import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
+import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 
@@ -567,6 +568,21 @@ public class InstallPluginCommandTests extends ESTestCase {
         assertTrue(terminal.getOutput(), terminal.getOutput().contains("x-pack"));
     }
 
+    public void testInstallMisspelledOfficialPlugins() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        UserError e = expectThrows(UserError.class, () -> installPlugin("xpack", env.v1()));
+        assertThat(e.getMessage(), containsString("Unknown plugin xpack, did you mean [x-pack]?"));
+
+        e = expectThrows(UserError.class, () -> installPlugin("analysis-smartnc", env.v1()));
+        assertThat(e.getMessage(), containsString("Unknown plugin analysis-smartnc, did you mean [analysis-smartcn]?"));
+
+        e = expectThrows(UserError.class, () -> installPlugin("repository", env.v1()));
+        assertThat(e.getMessage(), containsString("Unknown plugin repository, did you mean any of [repository-s3, repository-gcs]?"));
+
+        e = expectThrows(UserError.class, () -> installPlugin("unknown_plugin", env.v1()));
+        assertThat(e.getMessage(), containsString("Unknown plugin unknown_plugin"));
+    }
+
     // TODO: test batch flag?
     // TODO: test checksum (need maven/official below)
     // TODO: test maven, official, and staging install...need tests with fixtures...

From 8c6037428490f266b3a73133e37acd44949c192a Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 17 Jun 2016 11:37:01 +0200
Subject: [PATCH 0925/1311] Build: do not load integ test class if
 --skip-integ-tests-in-disguise is specified in NamingConventionsCheck

Projects that don't depend on elasticsearch-test fail otherwise because org.elasticsearch.test.EsIntegTestCase (default integ test class) is not in the classpath. They should provide their onw integ test base class, but having integration tests should not be mandatory. One can simply set skipIntegTestsInDisguise to true to prevent loading of integ test class.
---
 .../gradle/plugin/PluginBuildPlugin.groovy    |  4 +-
 .../precommit/NamingConventionsTask.groovy    |  5 +-
 .../test/NamingConventionsCheck.java          | 57 ++++++++++---------
 client-sniffer/build.gradle                   |  7 ++-
 client/build.gradle                           |  7 ++-
 5 files changed, 45 insertions(+), 35 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
index 7ff83f4eae3..ba013da31e9 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
@@ -19,7 +19,6 @@
 package org.elasticsearch.gradle.plugin
 
 import nebula.plugin.publishing.maven.MavenBasePublishPlugin
-import nebula.plugin.publishing.maven.MavenManifestPlugin
 import nebula.plugin.publishing.maven.MavenScmPlugin
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.test.RestIntegTestTask
@@ -27,7 +26,6 @@ import org.elasticsearch.gradle.test.RunTask
 import org.gradle.api.Project
 import org.gradle.api.tasks.SourceSet
 import org.gradle.api.tasks.bundling.Zip
-
 /**
  * Encapsulates build configuration for an Elasticsearch plugin.
  */
@@ -56,7 +54,7 @@ public class PluginBuildPlugin extends BuildPlugin {
             }
 
             project.namingConventions {
-                // Plugins decalare extensions of ESIntegTestCase as "Tests" instead of IT.
+                // Plugins declare integration tests as "Tests" instead of IT.
                 skipIntegTestInDisguise = true
             }
         }
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/NamingConventionsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/NamingConventionsTask.groovy
index fe7f13f29e6..52de7dac2d5 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/NamingConventionsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/NamingConventionsTask.groovy
@@ -26,7 +26,6 @@ import org.gradle.api.file.FileCollection
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.InputFiles
 import org.gradle.api.tasks.OutputFile
-
 /**
  * Runs NamingConventionsCheck on a classpath/directory combo to verify that
  * tests are named according to our conventions so they'll be picked up by
@@ -90,9 +89,11 @@ public class NamingConventionsTask extends LoggedExec {
             doFirst {
                 args('-Djna.nosys=true')
                 args('-cp', (classpath + extraClasspath).asPath, 'org.elasticsearch.test.NamingConventionsCheck')
-                args(testClass, integTestClass)
+                args('--test-class', testClass)
                 if (skipIntegTestInDisguise) {
                     args('--skip-integ-tests-in-disguise')
+                } else {
+                    args('--integ-test-class', integTestClass)
                 }
                 /*
                  * The test framework has classes that fail the checks to validate that the checks fail properly.
diff --git a/buildSrc/src/main/java/org/elasticsearch/test/NamingConventionsCheck.java b/buildSrc/src/main/java/org/elasticsearch/test/NamingConventionsCheck.java
index ed25d52739c..cbfa31d1aaf 100644
--- a/buildSrc/src/main/java/org/elasticsearch/test/NamingConventionsCheck.java
+++ b/buildSrc/src/main/java/org/elasticsearch/test/NamingConventionsCheck.java
@@ -44,31 +44,36 @@ import java.util.Set;
  */
 public class NamingConventionsCheck {
     public static void main(String[] args) throws IOException {
-        int i = 0;
-        NamingConventionsCheck check = new NamingConventionsCheck(
-                loadClassWithoutInitializing(args[i++]),
-                loadClassWithoutInitializing(args[i++]));
+        Class<?> testClass = null;
+        Class<?> integTestClass = null;
+        Path rootPath = null;
         boolean skipIntegTestsInDisguise = false;
         boolean selfTest = false;
-        while (true) {
-            switch (args[i]) {
-            case "--skip-integ-tests-in-disguise":
-                skipIntegTestsInDisguise = true;
-                i++;
-                continue;
-            case "--self-test":
-                selfTest = true;
-                i++;
-                continue;
-            case "--":
-                i++;
-                break;
-            default:
-                fail("Expected -- before a path.");
+        for (int i = 0; i < args.length; i++) {
+            String arg = args[i];
+            switch (arg) {
+                case "--test-class":
+                    testClass = loadClassWithoutInitializing(args[++i]);
+                    break;
+                case "--integ-test-class":
+                    integTestClass = loadClassWithoutInitializing(args[++i]);
+                    break;
+                case "--skip-integ-tests-in-disguise":
+                    skipIntegTestsInDisguise = true;
+                    break;
+                case "--self-test":
+                    selfTest = true;
+                    break;
+                case "--":
+                    rootPath = Paths.get(args[++i]);
+                    break;
+                default:
+                    fail("unsupported argument '" + arg + "'");
             }
-            break;
         }
-        check.check(Paths.get(args[i]));
+
+        NamingConventionsCheck check = new NamingConventionsCheck(testClass, integTestClass);
+        check.check(rootPath, skipIntegTestsInDisguise);
 
         if (selfTest) {
             assertViolation("WrongName", check.missingSuffix);
@@ -87,9 +92,9 @@ public class NamingConventionsCheck {
         assertNoViolations("Found inner classes that are tests, which are excluded from the test runner", check.innerClasses);
         assertNoViolations("Pure Unit-Test found must subclass [" + check.testClass.getSimpleName() + "]", check.pureUnitTest);
         assertNoViolations("Classes ending with [Tests] must subclass [" + check.testClass.getSimpleName() + "]", check.notImplementing);
-        if (!skipIntegTestsInDisguise) {
-            assertNoViolations("Subclasses of ESIntegTestCase should end with IT as they are integration tests",
-                    check.integTestsInDisguise);
+        if (skipIntegTestsInDisguise == false) {
+            assertNoViolations("Subclasses of " + check.integTestClass.getSimpleName() +
+                    " should end with IT as they are integration tests", check.integTestsInDisguise);
         }
     }
 
@@ -108,7 +113,7 @@ public class NamingConventionsCheck {
         this.integTestClass = integTestClass;
     }
 
-    public void check(Path rootPath) throws IOException {
+    public void check(Path rootPath, boolean skipTestsInDisguised) throws IOException {
         Files.walkFileTree(rootPath, new FileVisitor<Path>() {
             /**
              * The package name of the directory we are currently visiting. Kept as a string rather than something fancy because we load
@@ -143,7 +148,7 @@ public class NamingConventionsCheck {
                     String className = filename.substring(0, filename.length() - ".class".length());
                     Class<?> clazz = loadClassWithoutInitializing(packageName + className);
                     if (clazz.getName().endsWith("Tests")) {
-                        if (integTestClass.isAssignableFrom(clazz)) {
+                        if (skipTestsInDisguised == false && integTestClass.isAssignableFrom(clazz)) {
                             integTestsInDisguise.add(clazz);
                         }
                         if (Modifier.isAbstract(clazz.getModifiers()) || Modifier.isInterface(clazz.getModifiers())) {
diff --git a/client-sniffer/build.gradle b/client-sniffer/build.gradle
index 368cdaa073f..402bc2f61b4 100644
--- a/client-sniffer/build.gradle
+++ b/client-sniffer/build.gradle
@@ -61,8 +61,11 @@ forbiddenApisTest {
 
 //JarHell is part of es core, which we don't want to pull in
 jarHell.enabled=false
-//NamingConventionCheck is part of test-framework, which we don't want to pull in as it depends on es core
-namingConventions.enabled=false
+
+namingConventions {
+  //we don't have integration tests
+  skipIntegTestInDisguise = true
+}
 
 dependencyLicenses {
   dependencies = project.configurations.runtime.fileCollection {
diff --git a/client/build.gradle b/client/build.gradle
index afc2e6e1e7b..a91c224c745 100644
--- a/client/build.gradle
+++ b/client/build.gradle
@@ -59,8 +59,11 @@ forbiddenApisTest {
 
 //JarHell is part of es core, which we don't want to pull in
 jarHell.enabled=false
-//NamingConventionCheck is part of test-framework, which we don't want to pull in as it depends on es core
-namingConventions.enabled=false
+
+namingConventions {
+  //we don't have integration tests
+  skipIntegTestInDisguise = true
+}
 
 thirdPartyAudit.excludes = [
   //commons-logging optional dependencies

From 712e387058a85406ddc4e706dba7e2caed54ce2e Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Wed, 1 Jun 2016 10:44:16 +0200
Subject: [PATCH 0926/1311] Rename PipelineAggregatorBuilder to
 PipelineAggregationBuilder.

This is a follow-up to #18377.
---
 .../action/search/SearchRequestBuilder.java   |  4 +-
 .../elasticsearch/search/SearchModule.java    | 86 +++++++++----------
 .../AbstractAggregationBuilder.java           |  2 +-
 .../aggregations/AggregationBuilder.java      |  2 +-
 .../aggregations/AggregatorFactories.java     | 52 +++++------
 .../aggregations/AggregatorParsers.java       |  2 +-
 ...r.java => PipelineAggregationBuilder.java} | 10 +--
 ...> AbstractPipelineAggregationBuilder.java} | 20 ++---
 .../aggregations/pipeline/BucketHelpers.java  |  4 +-
 .../pipeline/PipelineAggregator.java          |  6 +-
 .../pipeline/PipelineAggregatorBuilders.java  | 86 +++++++++----------
 .../bucketmetrics/BucketMetricsParser.java    |  6 +-
 ...ketMetricsPipelineAggregationBuilder.java} | 18 ++--
 ... AvgBucketPipelineAggregationBuilder.java} | 18 ++--
 ... MaxBucketPipelineAggregationBuilder.java} | 18 ++--
 ... MinBucketPipelineAggregationBuilder.java} | 18 ++--
 ...ilesBucketPipelineAggregationBuilder.java} | 26 +++---
 ...tatsBucketPipelineAggregationBuilder.java} | 18 ++--
 .../extended/ExtendedStatsBucketParser.java   |  6 +-
 ...tatsBucketPipelineAggregationBuilder.java} | 20 ++---
 ... SumBucketPipelineAggregationBuilder.java} | 18 ++--
 ...cketScriptPipelineAggregationBuilder.java} | 22 ++---
 ...etSelectorPipelineAggregationBuilder.java} | 20 ++---
 ...ulativeSumPipelineAggregationBuilder.java} | 22 ++---
 ...DerivativePipelineAggregationBuilder.java} | 30 +++----
 ... => MovAvgPipelineAggregationBuilder.java} | 35 ++++----
 .../pipeline/movavg/models/EwmaModel.java     | 10 +--
 .../movavg/models/HoltLinearModel.java        | 10 +--
 .../movavg/models/HoltWintersModel.java       | 10 +--
 .../pipeline/movavg/models/LinearModel.java   |  6 +-
 .../pipeline/movavg/models/SimpleModel.java   |  6 +-
 ...SerialDiffPipelineAggregationBuilder.java} | 22 ++---
 .../search/builder/SearchSourceBuilder.java   |  4 +-
 .../BasePipelineAggregationTestCase.java      | 10 +--
 .../pipeline/BucketScriptTests.java           |  8 +-
 .../pipeline/BucketSelectorTests.java         |  8 +-
 .../pipeline/CumulativeSumTests.java          |  8 +-
 .../pipeline/DerivativeTests.java             |  8 +-
 .../pipeline/SerialDifferenceTests.java       |  8 +-
 .../AbstractBucketMetricsTestCase.java        |  2 +-
 .../bucketmetrics/AvgBucketTests.java         |  8 +-
 .../ExtendedStatsBucketTests.java             | 10 +--
 .../bucketmetrics/MaxBucketTests.java         |  8 +-
 .../bucketmetrics/MinBucketTests.java         |  8 +-
 .../bucketmetrics/PercentilesBucketTests.java | 10 +--
 .../bucketmetrics/StatsBucketTests.java       |  8 +-
 .../bucketmetrics/SumBucketTests.java         |  8 +-
 .../pipeline/moving/avg/MovAvgTests.java      | 14 +--
 .../percolator/PercolateRequestBuilder.java   |  6 +-
 .../percolator/PercolateSourceBuilder.java    |  8 +-
 50 files changed, 389 insertions(+), 388 deletions(-)
 rename core/src/main/java/org/elasticsearch/search/aggregations/{PipelineAggregatorBuilder.java => PipelineAggregationBuilder.java} (86%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/pipeline/{AbstractPipelineAggregatorBuilder.java => AbstractPipelineAggregationBuilder.java} (86%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/{BucketMetricsPipelineAggregatorBuilder.java => BucketMetricsPipelineAggregationBuilder.java} (86%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/avg/{AvgBucketPipelineAggregatorBuilder.java => AvgBucketPipelineAggregationBuilder.java} (78%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/max/{MaxBucketPipelineAggregatorBuilder.java => MaxBucketPipelineAggregationBuilder.java} (78%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/min/{MinBucketPipelineAggregatorBuilder.java => MinBucketPipelineAggregationBuilder.java} (78%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/{PercentilesBucketPipelineAggregatorBuilder.java => PercentilesBucketPipelineAggregationBuilder.java} (83%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/{StatsBucketPipelineAggregatorBuilder.java => StatsBucketPipelineAggregationBuilder.java} (79%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/{ExtendedStatsBucketPipelineAggregatorBuilder.java => ExtendedStatsBucketPipelineAggregationBuilder.java} (81%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/sum/{SumBucketPipelineAggregatorBuilder.java => SumBucketPipelineAggregationBuilder.java} (78%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/{BucketScriptPipelineAggregatorBuilder.java => BucketScriptPipelineAggregationBuilder.java} (91%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/{BucketSelectorPipelineAggregatorBuilder.java => BucketSelectorPipelineAggregationBuilder.java} (90%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/{CumulativeSumPipelineAggregatorBuilder.java => CumulativeSumPipelineAggregationBuilder.java} (88%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/{DerivativePipelineAggregatorBuilder.java => DerivativePipelineAggregationBuilder.java} (89%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/{MovAvgPipelineAggregatorBuilder.java => MovAvgPipelineAggregationBuilder.java} (92%)
 rename core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/{SerialDiffPipelineAggregatorBuilder.java => SerialDiffPipelineAggregationBuilder.java} (90%)

diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
index 173c15ec1a5..47be65521d6 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
@@ -29,7 +29,7 @@ import org.elasticsearch.script.Script;
 import org.elasticsearch.script.Template;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.slice.SliceBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.highlight.HighlightBuilder;
@@ -387,7 +387,7 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
     /**
      * Adds an aggregation to the search operation.
      */
-    public SearchRequestBuilder addAggregation(PipelineAggregatorBuilder aggregation) {
+    public SearchRequestBuilder addAggregation(PipelineAggregationBuilder aggregation) {
         sourceBuilder().aggregation(aggregation);
         return this;
     }
diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java
index 2b212756528..34a90894425 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -94,7 +94,7 @@ import org.elasticsearch.search.action.SearchTransportService;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
-import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.children.ChildrenAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.children.InternalChildren;
 import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
@@ -200,31 +200,31 @@ import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.InternalBucketMetricValue;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.avg.AvgBucketPipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.avg.AvgBucketPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.avg.AvgBucketPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.max.MaxBucketPipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.max.MaxBucketPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.max.MaxBucketPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.min.MinBucketPipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.min.MinBucketPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.min.MinBucketPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.PercentilesBucketPipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.PercentilesBucketPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.PercentilesBucketPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.StatsBucketPipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.StatsBucketPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.StatsBucketPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketParser;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.sum.SumBucketPipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.sum.SumBucketPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.sum.SumBucketPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.bucketscript.BucketScriptPipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.bucketscript.BucketScriptPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketscript.BucketScriptPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.bucketselector.BucketSelectorPipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.bucketselector.BucketSelectorPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketselector.BucketSelectorPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.cumulativesum.CumulativeSumPipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.cumulativesum.CumulativeSumPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.cumulativesum.CumulativeSumPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.derivative.DerivativePipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.derivative.DerivativePipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.derivative.DerivativePipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.derivative.InternalDerivative;
 import org.elasticsearch.search.aggregations.pipeline.movavg.MovAvgPipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.movavg.MovAvgPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.movavg.MovAvgPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.EwmaModel;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.HoltLinearModel;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.HoltWintersModel;
@@ -232,7 +232,7 @@ import org.elasticsearch.search.aggregations.pipeline.movavg.models.LinearModel;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.MovAvgModel;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.SimpleModel;
 import org.elasticsearch.search.aggregations.pipeline.serialdiff.SerialDiffPipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.serialdiff.SerialDiffPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.serialdiff.SerialDiffPipelineAggregationBuilder;
 import org.elasticsearch.search.controller.SearchPhaseController;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.fetch.FetchSubPhase;
@@ -433,10 +433,10 @@ public class SearchModule extends AbstractModule {
      * @param aggregationName names by which the aggregation may be parsed. The first name is special because it is the name that the reader
      *        is registered under.
      */
-    public void registerPipelineAggregation(Writeable.Reader<? extends PipelineAggregatorBuilder> reader,
+    public void registerPipelineAggregation(Writeable.Reader<? extends PipelineAggregationBuilder> reader,
             PipelineAggregator.Parser aggregationParser, ParseField aggregationName) {
         pipelineAggregationParserRegistry.register(aggregationParser, aggregationName);
-        namedWriteableRegistry.register(PipelineAggregatorBuilder.class, aggregationName.getPreferredName(), reader);
+        namedWriteableRegistry.register(PipelineAggregationBuilder.class, aggregationName.getPreferredName(), reader);
     }
 
     @Override
@@ -500,33 +500,33 @@ public class SearchModule extends AbstractModule {
                 ScriptedMetricAggregationBuilder.AGGREGATION_NAME_FIELD);
         registerAggregation(ChildrenAggregationBuilder::new, ChildrenAggregationBuilder::parse,
                 ChildrenAggregationBuilder.AGGREGATION_NAME_FIELD);
-        registerPipelineAggregation(DerivativePipelineAggregatorBuilder::new, DerivativePipelineAggregatorBuilder::parse,
-                DerivativePipelineAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerPipelineAggregation(MaxBucketPipelineAggregatorBuilder::new, MaxBucketPipelineAggregatorBuilder.PARSER,
-                MaxBucketPipelineAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerPipelineAggregation(MinBucketPipelineAggregatorBuilder::new, MinBucketPipelineAggregatorBuilder.PARSER,
-                MinBucketPipelineAggregatorBuilder.AGGREGATION_FIELD_NAME);
-        registerPipelineAggregation(AvgBucketPipelineAggregatorBuilder::new, AvgBucketPipelineAggregatorBuilder.PARSER,
-                AvgBucketPipelineAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerPipelineAggregation(SumBucketPipelineAggregatorBuilder::new, SumBucketPipelineAggregatorBuilder.PARSER,
-                SumBucketPipelineAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerPipelineAggregation(StatsBucketPipelineAggregatorBuilder::new, StatsBucketPipelineAggregatorBuilder.PARSER,
-                StatsBucketPipelineAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerPipelineAggregation(ExtendedStatsBucketPipelineAggregatorBuilder::new, new ExtendedStatsBucketParser(),
-                ExtendedStatsBucketPipelineAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerPipelineAggregation(PercentilesBucketPipelineAggregatorBuilder::new, PercentilesBucketPipelineAggregatorBuilder.PARSER,
-                PercentilesBucketPipelineAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerPipelineAggregation(MovAvgPipelineAggregatorBuilder::new,
-                (n, c) -> MovAvgPipelineAggregatorBuilder.parse(movingAverageModelParserRegistry, n, c),
-                MovAvgPipelineAggregatorBuilder.AGGREGATION_FIELD_NAME);
-        registerPipelineAggregation(CumulativeSumPipelineAggregatorBuilder::new, CumulativeSumPipelineAggregatorBuilder::parse,
-                CumulativeSumPipelineAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerPipelineAggregation(BucketScriptPipelineAggregatorBuilder::new, BucketScriptPipelineAggregatorBuilder::parse,
-                BucketScriptPipelineAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerPipelineAggregation(BucketSelectorPipelineAggregatorBuilder::new, BucketSelectorPipelineAggregatorBuilder::parse,
-                BucketSelectorPipelineAggregatorBuilder.AGGREGATION_NAME_FIELD);
-        registerPipelineAggregation(SerialDiffPipelineAggregatorBuilder::new, SerialDiffPipelineAggregatorBuilder::parse,
-                SerialDiffPipelineAggregatorBuilder.AGGREGATION_NAME_FIELD);
+        registerPipelineAggregation(DerivativePipelineAggregationBuilder::new, DerivativePipelineAggregationBuilder::parse,
+                DerivativePipelineAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerPipelineAggregation(MaxBucketPipelineAggregationBuilder::new, MaxBucketPipelineAggregationBuilder.PARSER,
+                MaxBucketPipelineAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerPipelineAggregation(MinBucketPipelineAggregationBuilder::new, MinBucketPipelineAggregationBuilder.PARSER,
+                MinBucketPipelineAggregationBuilder.AGGREGATION_FIELD_NAME);
+        registerPipelineAggregation(AvgBucketPipelineAggregationBuilder::new, AvgBucketPipelineAggregationBuilder.PARSER,
+                AvgBucketPipelineAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerPipelineAggregation(SumBucketPipelineAggregationBuilder::new, SumBucketPipelineAggregationBuilder.PARSER,
+                SumBucketPipelineAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerPipelineAggregation(StatsBucketPipelineAggregationBuilder::new, StatsBucketPipelineAggregationBuilder.PARSER,
+                StatsBucketPipelineAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerPipelineAggregation(ExtendedStatsBucketPipelineAggregationBuilder::new, new ExtendedStatsBucketParser(),
+                ExtendedStatsBucketPipelineAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerPipelineAggregation(PercentilesBucketPipelineAggregationBuilder::new, PercentilesBucketPipelineAggregationBuilder.PARSER,
+                PercentilesBucketPipelineAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerPipelineAggregation(MovAvgPipelineAggregationBuilder::new,
+                (n, c) -> MovAvgPipelineAggregationBuilder.parse(movingAverageModelParserRegistry, n, c),
+                MovAvgPipelineAggregationBuilder.AGGREGATION_FIELD_NAME);
+        registerPipelineAggregation(CumulativeSumPipelineAggregationBuilder::new, CumulativeSumPipelineAggregationBuilder::parse,
+                CumulativeSumPipelineAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerPipelineAggregation(BucketScriptPipelineAggregationBuilder::new, BucketScriptPipelineAggregationBuilder::parse,
+                BucketScriptPipelineAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerPipelineAggregation(BucketSelectorPipelineAggregationBuilder::new, BucketSelectorPipelineAggregationBuilder::parse,
+                BucketSelectorPipelineAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerPipelineAggregation(SerialDiffPipelineAggregationBuilder::new, SerialDiffPipelineAggregationBuilder::parse,
+                SerialDiffPipelineAggregationBuilder.AGGREGATION_NAME_FIELD);
         bind(AggregatorParsers.class).toInstance(aggregatorParsers);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java
index ef54cdf0ef5..e84d7684304 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AbstractAggregationBuilder.java
@@ -80,7 +80,7 @@ public abstract class AbstractAggregationBuilder<AB extends AbstractAggregationB
      */
     @SuppressWarnings("unchecked")
     @Override
-    public AB subAggregation(PipelineAggregatorBuilder aggregation) {
+    public AB subAggregation(PipelineAggregationBuilder aggregation) {
         if (aggregation == null) {
             throw new IllegalArgumentException("[aggregation] must not be null: [" + name + "]");
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
index d718227aae8..9417ec8a0e6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java
@@ -72,7 +72,7 @@ public abstract class AggregationBuilder
     public abstract AggregationBuilder subAggregation(AggregationBuilder aggregation);
 
     /** Add a sub aggregation to this builder. */
-    public abstract AggregationBuilder subAggregation(PipelineAggregatorBuilder aggregation);
+    public abstract AggregationBuilder subAggregation(PipelineAggregationBuilder aggregation);
 
     /**
      * Internal: Registers sub-factories with this factory. The sub-factory will be
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
index 2d8ea318fd6..616b0091fe8 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorFactories.java
@@ -46,18 +46,18 @@ import java.util.Set;
 public class AggregatorFactories {
 
     public static final AggregatorFactories EMPTY = new AggregatorFactories(null, new AggregatorFactory<?>[0],
-            new ArrayList<PipelineAggregatorBuilder>());
+            new ArrayList<PipelineAggregationBuilder>());
 
     private AggregatorFactory<?> parent;
     private AggregatorFactory<?>[] factories;
-    private List<PipelineAggregatorBuilder> pipelineAggregatorFactories;
+    private List<PipelineAggregationBuilder> pipelineAggregatorFactories;
 
     public static Builder builder() {
         return new Builder();
     }
 
     private AggregatorFactories(AggregatorFactory<?> parent, AggregatorFactory<?>[] factories,
-            List<PipelineAggregatorBuilder> pipelineAggregators) {
+            List<PipelineAggregationBuilder> pipelineAggregators) {
         this.parent = parent;
         this.factories = factories;
         this.pipelineAggregatorFactories = pipelineAggregators;
@@ -65,7 +65,7 @@ public class AggregatorFactories {
 
     public List<PipelineAggregator> createPipelineAggregators() throws IOException {
         List<PipelineAggregator> pipelineAggregators = new ArrayList<>();
-        for (PipelineAggregatorBuilder factory : this.pipelineAggregatorFactories) {
+        for (PipelineAggregationBuilder factory : this.pipelineAggregatorFactories) {
             pipelineAggregators.add(factory.create());
         }
         return pipelineAggregators;
@@ -128,7 +128,7 @@ public class AggregatorFactories {
         for (AggregatorFactory<?> factory : factories) {
             factory.validate();
         }
-        for (PipelineAggregatorBuilder factory : pipelineAggregatorFactories) {
+        for (PipelineAggregationBuilder factory : pipelineAggregatorFactories) {
             factory.validate(parent, factories, pipelineAggregatorFactories);
         }
     }
@@ -136,7 +136,7 @@ public class AggregatorFactories {
     public static class Builder extends ToXContentToBytes implements Writeable {
         private final Set<String> names = new HashSet<>();
         private final List<AggregationBuilder> aggregationBuilders = new ArrayList<>();
-        private final List<PipelineAggregatorBuilder> pipelineAggregatorBuilders = new ArrayList<>();
+        private final List<PipelineAggregationBuilder> pipelineAggregatorBuilders = new ArrayList<>();
         private boolean skipResolveOrder;
 
         /**
@@ -155,7 +155,7 @@ public class AggregatorFactories {
             }
             int pipelineFactoriesSize = in.readVInt();
             for (int i = 0; i < pipelineFactoriesSize; i++) {
-                addPipelineAggregator(in.readNamedWriteable(PipelineAggregatorBuilder.class));
+                addPipelineAggregator(in.readNamedWriteable(PipelineAggregationBuilder.class));
             }
         }
 
@@ -166,7 +166,7 @@ public class AggregatorFactories {
                 out.writeNamedWriteable(factory);
             }
             out.writeVInt(this.pipelineAggregatorBuilders.size());
-            for (PipelineAggregatorBuilder factory : pipelineAggregatorBuilders) {
+            for (PipelineAggregationBuilder factory : pipelineAggregatorBuilders) {
                 out.writeNamedWriteable(factory);
             }
         }
@@ -183,7 +183,7 @@ public class AggregatorFactories {
             return this;
         }
 
-        public Builder addPipelineAggregator(PipelineAggregatorBuilder pipelineAggregatorFactory) {
+        public Builder addPipelineAggregator(PipelineAggregationBuilder pipelineAggregatorFactory) {
             this.pipelineAggregatorBuilders.add(pipelineAggregatorFactory);
             return this;
         }
@@ -200,7 +200,7 @@ public class AggregatorFactories {
             if (aggregationBuilders.isEmpty() && pipelineAggregatorBuilders.isEmpty()) {
                 return EMPTY;
             }
-            List<PipelineAggregatorBuilder> orderedpipelineAggregators = null;
+            List<PipelineAggregationBuilder> orderedpipelineAggregators = null;
             if (skipResolveOrder) {
                 orderedpipelineAggregators = new ArrayList<>(pipelineAggregatorBuilders);
             } else {
@@ -213,21 +213,21 @@ public class AggregatorFactories {
             return new AggregatorFactories(parent, aggFactories, orderedpipelineAggregators);
         }
 
-        private List<PipelineAggregatorBuilder> resolvePipelineAggregatorOrder(
-                List<PipelineAggregatorBuilder> pipelineAggregatorBuilders, List<AggregationBuilder> aggBuilders) {
-            Map<String, PipelineAggregatorBuilder> pipelineAggregatorBuildersMap = new HashMap<>();
-            for (PipelineAggregatorBuilder builder : pipelineAggregatorBuilders) {
+        private List<PipelineAggregationBuilder> resolvePipelineAggregatorOrder(
+                List<PipelineAggregationBuilder> pipelineAggregatorBuilders, List<AggregationBuilder> aggBuilders) {
+            Map<String, PipelineAggregationBuilder> pipelineAggregatorBuildersMap = new HashMap<>();
+            for (PipelineAggregationBuilder builder : pipelineAggregatorBuilders) {
                 pipelineAggregatorBuildersMap.put(builder.getName(), builder);
             }
             Map<String, AggregationBuilder> aggBuildersMap = new HashMap<>();
             for (AggregationBuilder aggBuilder : aggBuilders) {
                 aggBuildersMap.put(aggBuilder.name, aggBuilder);
             }
-            List<PipelineAggregatorBuilder> orderedPipelineAggregatorrs = new LinkedList<>();
-            List<PipelineAggregatorBuilder> unmarkedBuilders = new ArrayList<PipelineAggregatorBuilder>(pipelineAggregatorBuilders);
-            Set<PipelineAggregatorBuilder> temporarilyMarked = new HashSet<PipelineAggregatorBuilder>();
+            List<PipelineAggregationBuilder> orderedPipelineAggregatorrs = new LinkedList<>();
+            List<PipelineAggregationBuilder> unmarkedBuilders = new ArrayList<PipelineAggregationBuilder>(pipelineAggregatorBuilders);
+            Set<PipelineAggregationBuilder> temporarilyMarked = new HashSet<PipelineAggregationBuilder>();
             while (!unmarkedBuilders.isEmpty()) {
-                PipelineAggregatorBuilder builder = unmarkedBuilders.get(0);
+                PipelineAggregationBuilder builder = unmarkedBuilders.get(0);
                 resolvePipelineAggregatorOrder(aggBuildersMap, pipelineAggregatorBuildersMap, orderedPipelineAggregatorrs, unmarkedBuilders,
                         temporarilyMarked, builder);
             }
@@ -235,9 +235,9 @@ public class AggregatorFactories {
         }
 
         private void resolvePipelineAggregatorOrder(Map<String, AggregationBuilder> aggBuildersMap,
-                Map<String, PipelineAggregatorBuilder> pipelineAggregatorBuildersMap,
-                List<PipelineAggregatorBuilder> orderedPipelineAggregators, List<PipelineAggregatorBuilder> unmarkedBuilders,
-                Set<PipelineAggregatorBuilder> temporarilyMarked, PipelineAggregatorBuilder builder) {
+                Map<String, PipelineAggregationBuilder> pipelineAggregatorBuildersMap,
+                List<PipelineAggregationBuilder> orderedPipelineAggregators, List<PipelineAggregationBuilder> unmarkedBuilders,
+                Set<PipelineAggregationBuilder> temporarilyMarked, PipelineAggregationBuilder builder) {
             if (temporarilyMarked.contains(builder)) {
                 throw new IllegalArgumentException("Cyclical dependency found with pipeline aggregator [" + builder.getName() + "]");
             } else if (unmarkedBuilders.contains(builder)) {
@@ -269,8 +269,8 @@ public class AggregatorFactories {
                                 }
                                 // Check the pipeline sub-aggregator factories
                                 if (!foundSubBuilder && (i == bucketsPathElements.size() - 1)) {
-                                    List<PipelineAggregatorBuilder> subPipelineBuilders = aggBuilder.factoriesBuilder.pipelineAggregatorBuilders;
-                                    for (PipelineAggregatorBuilder subFactory : subPipelineBuilders) {
+                                    List<PipelineAggregationBuilder> subPipelineBuilders = aggBuilder.factoriesBuilder.pipelineAggregatorBuilders;
+                                    for (PipelineAggregationBuilder subFactory : subPipelineBuilders) {
                                         if (aggName.equals(subFactory.getName())) {
                                             foundSubBuilder = true;
                                             break;
@@ -285,7 +285,7 @@ public class AggregatorFactories {
                         }
                         continue;
                     } else {
-                        PipelineAggregatorBuilder matchingBuilder = pipelineAggregatorBuildersMap.get(firstAggName);
+                        PipelineAggregationBuilder matchingBuilder = pipelineAggregatorBuildersMap.get(firstAggName);
                         if (matchingBuilder != null) {
                             resolvePipelineAggregatorOrder(aggBuildersMap, pipelineAggregatorBuildersMap, orderedPipelineAggregators,
                                     unmarkedBuilders, temporarilyMarked, matchingBuilder);
@@ -304,7 +304,7 @@ public class AggregatorFactories {
             return this.aggregationBuilders.toArray(new AggregationBuilder[this.aggregationBuilders.size()]);
         }
 
-        List<PipelineAggregatorBuilder> getPipelineAggregatorFactories() {
+        List<PipelineAggregationBuilder> getPipelineAggregatorFactories() {
             return this.pipelineAggregatorBuilders;
         }
 
@@ -321,7 +321,7 @@ public class AggregatorFactories {
                 }
             }
             if (pipelineAggregatorBuilders != null) {
-                for (PipelineAggregatorBuilder subAgg : pipelineAggregatorBuilders) {
+                for (PipelineAggregationBuilder subAgg : pipelineAggregatorBuilders) {
                     subAgg.toXContent(builder, params);
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorParsers.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorParsers.java
index 178c1666b18..df64d440dc0 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorParsers.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregatorParsers.java
@@ -104,7 +104,7 @@ public class AggregatorParsers {
             }
 
             AggregationBuilder aggFactory = null;
-            PipelineAggregatorBuilder pipelineAggregatorFactory = null;
+            PipelineAggregationBuilder pipelineAggregatorFactory = null;
             AggregatorFactories.Builder subFactories = null;
 
             Map<String, Object> metaData = null;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java
similarity index 86%
rename from core/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java
index 1931ec232d5..f9462325471 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregationBuilder.java
@@ -31,7 +31,7 @@ import java.util.Map;
  * A factory that knows how to create an {@link PipelineAggregator} of a
  * specific type.
  */
-public abstract class PipelineAggregatorBuilder extends ToXContentToBytes
+public abstract class PipelineAggregationBuilder extends ToXContentToBytes
         implements NamedWriteable {
 
     protected final String name;
@@ -43,7 +43,7 @@ public abstract class PipelineAggregatorBuilder extends ToXContentToBytes
      * @param name
      *            The aggregation name
      */
-    protected PipelineAggregatorBuilder(String name, String[] bucketsPaths) {
+    protected PipelineAggregationBuilder(String name, String[] bucketsPaths) {
         if (name == null) {
             throw new IllegalArgumentException("[name] must not be null: [" + name + "]");
         }
@@ -69,7 +69,7 @@ public abstract class PipelineAggregatorBuilder extends ToXContentToBytes
      * configured)
      */
     protected abstract void validate(AggregatorFactory<?> parent, AggregatorFactory<?>[] factories,
-            List<PipelineAggregatorBuilder> pipelineAggregatorFactories);
+            List<PipelineAggregationBuilder> pipelineAggregatorFactories);
 
     /**
      * Creates the pipeline aggregator
@@ -78,7 +78,7 @@ public abstract class PipelineAggregatorBuilder extends ToXContentToBytes
      */
     protected abstract PipelineAggregator create() throws IOException;
 
-    /** Associate metadata with this {@link PipelineAggregatorBuilder}. */
-    public abstract PipelineAggregatorBuilder setMetaData(Map<String, Object> metaData);
+    /** Associate metadata with this {@link PipelineAggregationBuilder}. */
+    public abstract PipelineAggregationBuilder setMetaData(Map<String, Object> metaData);
 
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java
similarity index 86%
rename from core/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java
index fc308b1bcdd..4fb2ed91401 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/AbstractPipelineAggregationBuilder.java
@@ -23,7 +23,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -32,10 +32,10 @@ import java.util.Map;
 import java.util.Objects;
 
 /**
- * Base implementation of a {@link PipelineAggregatorBuilder}.
+ * Base implementation of a {@link PipelineAggregationBuilder}.
  */
-public abstract class AbstractPipelineAggregatorBuilder<PAB extends AbstractPipelineAggregatorBuilder<PAB>>
-        extends PipelineAggregatorBuilder {
+public abstract class AbstractPipelineAggregationBuilder<PAB extends AbstractPipelineAggregationBuilder<PAB>>
+        extends PipelineAggregationBuilder {
 
     /**
      * Field shared by many parsers.
@@ -45,7 +45,7 @@ public abstract class AbstractPipelineAggregatorBuilder<PAB extends AbstractPipe
     protected final String type;
     protected Map<String, Object> metaData;
 
-    protected AbstractPipelineAggregatorBuilder(String name, String type, String[] bucketsPaths) {
+    protected AbstractPipelineAggregationBuilder(String name, String type, String[] bucketsPaths) {
         super(name, bucketsPaths);
         if (type == null) {
             throw new IllegalArgumentException("[type] must not be null: [" + name + "]");
@@ -56,7 +56,7 @@ public abstract class AbstractPipelineAggregatorBuilder<PAB extends AbstractPipe
     /**
      * Read from a stream.
      */
-    protected AbstractPipelineAggregatorBuilder(StreamInput in, String type) throws IOException {
+    protected AbstractPipelineAggregationBuilder(StreamInput in, String type) throws IOException {
         this(in.readString(), type, in.readStringArray());
         metaData = in.readMap();
     }
@@ -81,7 +81,7 @@ public abstract class AbstractPipelineAggregatorBuilder<PAB extends AbstractPipe
      */
     @Override
     public final void validate(AggregatorFactory<?> parent, AggregatorFactory<?>[] factories,
-            List<PipelineAggregatorBuilder> pipelineAggregatorFactories) {
+            List<PipelineAggregationBuilder> pipelineAggregatorFactories) {
         doValidate(parent, factories, pipelineAggregatorFactories);
     }
 
@@ -99,7 +99,7 @@ public abstract class AbstractPipelineAggregatorBuilder<PAB extends AbstractPipe
     }
 
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] factories,
-            List<PipelineAggregatorBuilder> pipelineAggregatorFactories) {
+            List<PipelineAggregationBuilder> pipelineAggregatorFactories) {
     }
 
     @SuppressWarnings("unchecked")
@@ -134,7 +134,7 @@ public abstract class AbstractPipelineAggregatorBuilder<PAB extends AbstractPipe
     }
 
     /**
-     * @return <code>true</code> if the {@link AbstractPipelineAggregatorBuilder}
+     * @return <code>true</code> if the {@link AbstractPipelineAggregationBuilder}
      *         overrides the XContent rendering of the bucketPath option.
      */
     protected boolean overrideBucketsPath() {
@@ -157,7 +157,7 @@ public abstract class AbstractPipelineAggregatorBuilder<PAB extends AbstractPipe
         if (getClass() != obj.getClass())
             return false;
         @SuppressWarnings("unchecked")
-        AbstractPipelineAggregatorBuilder<PAB> other = (AbstractPipelineAggregatorBuilder<PAB>) obj;
+        AbstractPipelineAggregationBuilder<PAB> other = (AbstractPipelineAggregationBuilder<PAB>) obj;
         if (!Objects.equals(name, other.name))
             return false;
         if (!Objects.equals(type, other.type))
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpers.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpers.java
index ea0973dd1a2..7e46105346d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpers.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/BucketHelpers.java
@@ -156,7 +156,7 @@ public class BucketHelpers {
         try {
             Object propertyValue = bucket.getProperty(agg.getName(), aggPathAsList);
             if (propertyValue == null) {
-                throw new AggregationExecutionException(AbstractPipelineAggregatorBuilder.BUCKETS_PATH_FIELD.getPreferredName()
+                throw new AggregationExecutionException(AbstractPipelineAggregationBuilder.BUCKETS_PATH_FIELD.getPreferredName()
                         + " must reference either a number value or a single value numeric metric aggregation");
             } else {
                 double value;
@@ -165,7 +165,7 @@ public class BucketHelpers {
                 } else if (propertyValue instanceof InternalNumericMetricsAggregation.SingleValue) {
                     value = ((InternalNumericMetricsAggregation.SingleValue) propertyValue).value();
                 } else {
-                    throw new AggregationExecutionException(AbstractPipelineAggregatorBuilder.BUCKETS_PATH_FIELD.getPreferredName()
+                    throw new AggregationExecutionException(AbstractPipelineAggregationBuilder.BUCKETS_PATH_FIELD.getPreferredName()
                             + " must reference either a number value or a single value numeric metric aggregation, got: "
                             + propertyValue.getClass().getCanonicalName());
                 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregator.java
index 4b0dea560a8..8f7ea955e91 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregator.java
@@ -28,7 +28,7 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
 import org.elasticsearch.search.aggregations.InternalAggregation.Type;
-import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 
 import java.io.IOException;
 import java.util.Map;
@@ -36,7 +36,7 @@ import java.util.Map;
 public abstract class PipelineAggregator implements Streamable {
 
     /**
-     * Parse the {@link PipelineAggregatorBuilder} from a {@link QueryParseContext}.
+     * Parse the {@link PipelineAggregationBuilder} from a {@link QueryParseContext}.
      */
     @FunctionalInterface
     public static interface Parser {
@@ -56,7 +56,7 @@ public abstract class PipelineAggregator implements Streamable {
          * @throws java.io.IOException
          *             When parsing fails
          */
-        PipelineAggregatorBuilder parse(String pipelineAggregatorName, QueryParseContext context)
+        PipelineAggregationBuilder parse(String pipelineAggregatorName, QueryParseContext context)
                 throws IOException;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregatorBuilders.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregatorBuilders.java
index 9d97fd6af8f..3dc33188495 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregatorBuilders.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregatorBuilders.java
@@ -20,19 +20,19 @@
 package org.elasticsearch.search.aggregations.pipeline;
 
 import org.elasticsearch.script.Script;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.avg.AvgBucketPipelineAggregatorBuilder;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.max.MaxBucketPipelineAggregatorBuilder;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.min.MinBucketPipelineAggregatorBuilder;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.PercentilesBucketPipelineAggregatorBuilder;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.StatsBucketPipelineAggregatorBuilder;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregatorBuilder;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.sum.SumBucketPipelineAggregatorBuilder;
-import org.elasticsearch.search.aggregations.pipeline.bucketscript.BucketScriptPipelineAggregatorBuilder;
-import org.elasticsearch.search.aggregations.pipeline.bucketselector.BucketSelectorPipelineAggregatorBuilder;
-import org.elasticsearch.search.aggregations.pipeline.cumulativesum.CumulativeSumPipelineAggregatorBuilder;
-import org.elasticsearch.search.aggregations.pipeline.derivative.DerivativePipelineAggregatorBuilder;
-import org.elasticsearch.search.aggregations.pipeline.movavg.MovAvgPipelineAggregatorBuilder;
-import org.elasticsearch.search.aggregations.pipeline.serialdiff.SerialDiffPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.avg.AvgBucketPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.max.MaxBucketPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.min.MinBucketPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.PercentilesBucketPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.StatsBucketPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.sum.SumBucketPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketscript.BucketScriptPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketselector.BucketSelectorPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.cumulativesum.CumulativeSumPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.derivative.DerivativePipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.movavg.MovAvgPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.serialdiff.SerialDiffPipelineAggregationBuilder;
 
 import java.util.Map;
 
@@ -41,70 +41,70 @@ public final class PipelineAggregatorBuilders {
     private PipelineAggregatorBuilders() {
     }
 
-    public static final DerivativePipelineAggregatorBuilder derivative(String name, String bucketsPath) {
-        return new DerivativePipelineAggregatorBuilder(name, bucketsPath);
+    public static final DerivativePipelineAggregationBuilder derivative(String name, String bucketsPath) {
+        return new DerivativePipelineAggregationBuilder(name, bucketsPath);
     }
 
-    public static final MaxBucketPipelineAggregatorBuilder maxBucket(String name, String bucketsPath) {
-        return new MaxBucketPipelineAggregatorBuilder(name, bucketsPath);
+    public static final MaxBucketPipelineAggregationBuilder maxBucket(String name, String bucketsPath) {
+        return new MaxBucketPipelineAggregationBuilder(name, bucketsPath);
     }
 
-    public static final MinBucketPipelineAggregatorBuilder minBucket(String name, String bucketsPath) {
-        return new MinBucketPipelineAggregatorBuilder(name, bucketsPath);
+    public static final MinBucketPipelineAggregationBuilder minBucket(String name, String bucketsPath) {
+        return new MinBucketPipelineAggregationBuilder(name, bucketsPath);
     }
 
-    public static final AvgBucketPipelineAggregatorBuilder avgBucket(String name, String bucketsPath) {
-        return new AvgBucketPipelineAggregatorBuilder(name, bucketsPath);
+    public static final AvgBucketPipelineAggregationBuilder avgBucket(String name, String bucketsPath) {
+        return new AvgBucketPipelineAggregationBuilder(name, bucketsPath);
     }
 
-    public static final SumBucketPipelineAggregatorBuilder sumBucket(String name, String bucketsPath) {
-        return new SumBucketPipelineAggregatorBuilder(name, bucketsPath);
+    public static final SumBucketPipelineAggregationBuilder sumBucket(String name, String bucketsPath) {
+        return new SumBucketPipelineAggregationBuilder(name, bucketsPath);
     }
 
-    public static final StatsBucketPipelineAggregatorBuilder statsBucket(String name, String bucketsPath) {
-        return new StatsBucketPipelineAggregatorBuilder(name, bucketsPath);
+    public static final StatsBucketPipelineAggregationBuilder statsBucket(String name, String bucketsPath) {
+        return new StatsBucketPipelineAggregationBuilder(name, bucketsPath);
     }
 
-    public static final ExtendedStatsBucketPipelineAggregatorBuilder extendedStatsBucket(String name,
+    public static final ExtendedStatsBucketPipelineAggregationBuilder extendedStatsBucket(String name,
             String bucketsPath) {
-        return new ExtendedStatsBucketPipelineAggregatorBuilder(name, bucketsPath);
+        return new ExtendedStatsBucketPipelineAggregationBuilder(name, bucketsPath);
     }
 
-    public static final PercentilesBucketPipelineAggregatorBuilder percentilesBucket(String name,
+    public static final PercentilesBucketPipelineAggregationBuilder percentilesBucket(String name,
             String bucketsPath) {
-        return new PercentilesBucketPipelineAggregatorBuilder(name, bucketsPath);
+        return new PercentilesBucketPipelineAggregationBuilder(name, bucketsPath);
     }
 
-    public static final MovAvgPipelineAggregatorBuilder movingAvg(String name, String bucketsPath) {
-        return new MovAvgPipelineAggregatorBuilder(name, bucketsPath);
+    public static final MovAvgPipelineAggregationBuilder movingAvg(String name, String bucketsPath) {
+        return new MovAvgPipelineAggregationBuilder(name, bucketsPath);
     }
 
-    public static final BucketScriptPipelineAggregatorBuilder bucketScript(String name,
+    public static final BucketScriptPipelineAggregationBuilder bucketScript(String name,
             Map<String, String> bucketsPathsMap, Script script) {
-        return new BucketScriptPipelineAggregatorBuilder(name, bucketsPathsMap, script);
+        return new BucketScriptPipelineAggregationBuilder(name, bucketsPathsMap, script);
     }
 
-    public static final BucketScriptPipelineAggregatorBuilder bucketScript(String name, Script script,
+    public static final BucketScriptPipelineAggregationBuilder bucketScript(String name, Script script,
             String... bucketsPaths) {
-        return new BucketScriptPipelineAggregatorBuilder(name, script, bucketsPaths);
+        return new BucketScriptPipelineAggregationBuilder(name, script, bucketsPaths);
     }
 
-    public static final BucketSelectorPipelineAggregatorBuilder bucketSelector(String name,
+    public static final BucketSelectorPipelineAggregationBuilder bucketSelector(String name,
             Map<String, String> bucketsPathsMap, Script script) {
-        return new BucketSelectorPipelineAggregatorBuilder(name, bucketsPathsMap, script);
+        return new BucketSelectorPipelineAggregationBuilder(name, bucketsPathsMap, script);
     }
 
-    public static final BucketSelectorPipelineAggregatorBuilder bucketSelector(String name, Script script,
+    public static final BucketSelectorPipelineAggregationBuilder bucketSelector(String name, Script script,
             String... bucketsPaths) {
-        return new BucketSelectorPipelineAggregatorBuilder(name, script, bucketsPaths);
+        return new BucketSelectorPipelineAggregationBuilder(name, script, bucketsPaths);
     }
 
-    public static final CumulativeSumPipelineAggregatorBuilder cumulativeSum(String name,
+    public static final CumulativeSumPipelineAggregationBuilder cumulativeSum(String name,
             String bucketsPath) {
-        return new CumulativeSumPipelineAggregatorBuilder(name, bucketsPath);
+        return new CumulativeSumPipelineAggregationBuilder(name, bucketsPath);
     }
 
-    public static final SerialDiffPipelineAggregatorBuilder diff(String name, String bucketsPath) {
-        return new SerialDiffPipelineAggregatorBuilder(name, bucketsPath);
+    public static final SerialDiffPipelineAggregationBuilder diff(String name, String bucketsPath) {
+        return new SerialDiffPipelineAggregationBuilder(name, bucketsPath);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsParser.java
index 8eddd83b7e2..9dee002ca29 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsParser.java
@@ -44,7 +44,7 @@ public abstract class BucketMetricsParser implements PipelineAggregator.Parser {
     }
 
     @Override
-    public final BucketMetricsPipelineAggregatorBuilder<?> parse(String pipelineAggregatorName, QueryParseContext context)
+    public final BucketMetricsPipelineAggregationBuilder<?> parse(String pipelineAggregatorName, QueryParseContext context)
             throws IOException {
         XContentParser parser = context.parser();
         XContentParser.Token token;
@@ -88,7 +88,7 @@ public abstract class BucketMetricsParser implements PipelineAggregator.Parser {
                     "Missing required field [" + BUCKETS_PATH.getPreferredName() + "] for aggregation [" + pipelineAggregatorName + "]");
         }
 
-        BucketMetricsPipelineAggregatorBuilder<?> factory  = buildFactory(pipelineAggregatorName, bucketsPaths[0], params);
+        BucketMetricsPipelineAggregationBuilder<?> factory  = buildFactory(pipelineAggregatorName, bucketsPaths[0], params);
         if (format != null) {
             factory.format(format);
         }
@@ -101,7 +101,7 @@ public abstract class BucketMetricsParser implements PipelineAggregator.Parser {
         return factory;
     }
 
-    protected abstract BucketMetricsPipelineAggregatorBuilder<?> buildFactory(String pipelineAggregatorName, String bucketsPaths,
+    protected abstract BucketMetricsPipelineAggregationBuilder<?> buildFactory(String pipelineAggregatorName, String bucketsPaths,
                                                                               Map<String, Object> params);
 
     protected boolean token(XContentParser parser, QueryParseContext context, String field,
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsPipelineAggregationBuilder.java
similarity index 86%
rename from core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsPipelineAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsPipelineAggregationBuilder.java
index 4945fd3a2e5..be34236d044 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsPipelineAggregationBuilder.java
@@ -24,9 +24,9 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
-import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
@@ -34,20 +34,20 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
-public abstract class BucketMetricsPipelineAggregatorBuilder<AF extends BucketMetricsPipelineAggregatorBuilder<AF>>
-        extends AbstractPipelineAggregatorBuilder<AF> {
+public abstract class BucketMetricsPipelineAggregationBuilder<AF extends BucketMetricsPipelineAggregationBuilder<AF>>
+        extends AbstractPipelineAggregationBuilder<AF> {
 
     private String format = null;
     private GapPolicy gapPolicy = GapPolicy.SKIP;
 
-    protected BucketMetricsPipelineAggregatorBuilder(String name, String type, String[] bucketsPaths) {
+    protected BucketMetricsPipelineAggregationBuilder(String name, String type, String[] bucketsPaths) {
         super(name, type, bucketsPaths);
     }
 
     /**
      * Read from a stream.
      */
-    protected BucketMetricsPipelineAggregatorBuilder(StreamInput in, String type) throws IOException {
+    protected BucketMetricsPipelineAggregationBuilder(StreamInput in, String type) throws IOException {
         super(in, type);
         format = in.readOptionalString();
         gapPolicy = GapPolicy.readFrom(in);
@@ -107,7 +107,7 @@ public abstract class BucketMetricsPipelineAggregatorBuilder<AF extends BucketMe
 
     @Override
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] aggFactories,
-            List<PipelineAggregatorBuilder> pipelineAggregatorFactories) {
+            List<PipelineAggregationBuilder> pipelineAggregatorFactories) {
         if (bucketsPaths.length != 1) {
             throw new IllegalStateException(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                     + " must contain a single entry for aggregation [" + name + "]");
@@ -138,12 +138,12 @@ public abstract class BucketMetricsPipelineAggregatorBuilder<AF extends BucketMe
     @Override
     protected final boolean doEquals(Object obj) {
         @SuppressWarnings("unchecked")
-        BucketMetricsPipelineAggregatorBuilder<AF> other = (BucketMetricsPipelineAggregatorBuilder<AF>) obj;
+        BucketMetricsPipelineAggregationBuilder<AF> other = (BucketMetricsPipelineAggregationBuilder<AF>) obj;
         return Objects.equals(format, other.format)
                 && Objects.equals(gapPolicy, other.gapPolicy)
                 && innerEquals(other);
     }
 
-    protected abstract boolean innerEquals(BucketMetricsPipelineAggregatorBuilder<AF> other);
+    protected abstract boolean innerEquals(BucketMetricsPipelineAggregationBuilder<AF> other);
 
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/avg/AvgBucketPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/avg/AvgBucketPipelineAggregationBuilder.java
similarity index 78%
rename from core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/avg/AvgBucketPipelineAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/avg/AvgBucketPipelineAggregationBuilder.java
index 509b45868b1..d7d7a1e0c57 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/avg/AvgBucketPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/avg/AvgBucketPipelineAggregationBuilder.java
@@ -24,27 +24,27 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsParser;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsPipelineAggregationBuilder;
 
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
-public class AvgBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAggregatorBuilder<AvgBucketPipelineAggregatorBuilder> {
+public class AvgBucketPipelineAggregationBuilder extends BucketMetricsPipelineAggregationBuilder<AvgBucketPipelineAggregationBuilder> {
     public static final String NAME = AvgBucketPipelineAggregator.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
-    public AvgBucketPipelineAggregatorBuilder(String name, String bucketsPath) {
+    public AvgBucketPipelineAggregationBuilder(String name, String bucketsPath) {
         super(name, AvgBucketPipelineAggregator.TYPE.name(), new String[] { bucketsPath });
     }
 
     /**
      * Read from a stream.
      */
-    public AvgBucketPipelineAggregatorBuilder(StreamInput in) throws IOException {
+    public AvgBucketPipelineAggregationBuilder(StreamInput in) throws IOException {
         super(in, NAME);
     }
 
@@ -60,7 +60,7 @@ public class AvgBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
 
     @Override
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] aggFactories,
-            List<PipelineAggregatorBuilder> pipelineAggregatorFactories) {
+            List<PipelineAggregationBuilder> pipelineAggregatorFactories) {
         if (bucketsPaths.length != 1) {
             throw new IllegalStateException(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                     + " must contain a single entry for aggregation [" + name + "]");
@@ -74,9 +74,9 @@ public class AvgBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
 
     public static final PipelineAggregator.Parser PARSER = new BucketMetricsParser() {
         @Override
-        protected AvgBucketPipelineAggregatorBuilder buildFactory(String pipelineAggregatorName,
+        protected AvgBucketPipelineAggregationBuilder buildFactory(String pipelineAggregatorName,
                 String bucketsPath, Map<String, Object> params) {
-            return new AvgBucketPipelineAggregatorBuilder(pipelineAggregatorName, bucketsPath);
+            return new AvgBucketPipelineAggregationBuilder(pipelineAggregatorName, bucketsPath);
         }
     };
 
@@ -86,7 +86,7 @@ public class AvgBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
     }
 
     @Override
-    protected boolean innerEquals(BucketMetricsPipelineAggregatorBuilder<AvgBucketPipelineAggregatorBuilder> other) {
+    protected boolean innerEquals(BucketMetricsPipelineAggregationBuilder<AvgBucketPipelineAggregationBuilder> other) {
         return true;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/max/MaxBucketPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/max/MaxBucketPipelineAggregationBuilder.java
similarity index 78%
rename from core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/max/MaxBucketPipelineAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/max/MaxBucketPipelineAggregationBuilder.java
index 2800a63661d..0c4fc632b95 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/max/MaxBucketPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/max/MaxBucketPipelineAggregationBuilder.java
@@ -24,27 +24,27 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsParser;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsPipelineAggregationBuilder;
 
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
-public class MaxBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAggregatorBuilder<MaxBucketPipelineAggregatorBuilder> {
+public class MaxBucketPipelineAggregationBuilder extends BucketMetricsPipelineAggregationBuilder<MaxBucketPipelineAggregationBuilder> {
     public static final String NAME = MaxBucketPipelineAggregator.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
-    public MaxBucketPipelineAggregatorBuilder(String name, String bucketsPath) {
+    public MaxBucketPipelineAggregationBuilder(String name, String bucketsPath) {
         super(name, MaxBucketPipelineAggregator.TYPE.name(), new String[] { bucketsPath });
     }
 
     /**
      * Read from a stream.
      */
-    public MaxBucketPipelineAggregatorBuilder(StreamInput in) throws IOException {
+    public MaxBucketPipelineAggregationBuilder(StreamInput in) throws IOException {
         super(in, NAME);
     }
 
@@ -60,7 +60,7 @@ public class MaxBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
 
     @Override
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] aggFactories,
-            List<PipelineAggregatorBuilder> pipelineAggregatorFactories) {
+            List<PipelineAggregationBuilder> pipelineAggregatorFactories) {
         if (bucketsPaths.length != 1) {
             throw new IllegalStateException(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                     + " must contain a single entry for aggregation [" + name + "]");
@@ -74,9 +74,9 @@ public class MaxBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
 
     public static final PipelineAggregator.Parser PARSER = new BucketMetricsParser() {
         @Override
-        protected MaxBucketPipelineAggregatorBuilder buildFactory(String pipelineAggregatorName,
+        protected MaxBucketPipelineAggregationBuilder buildFactory(String pipelineAggregatorName,
                 String bucketsPath, Map<String, Object> params) {
-            return new MaxBucketPipelineAggregatorBuilder(pipelineAggregatorName, bucketsPath);
+            return new MaxBucketPipelineAggregationBuilder(pipelineAggregatorName, bucketsPath);
         }
     };
 
@@ -86,7 +86,7 @@ public class MaxBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
     }
 
     @Override
-    protected boolean innerEquals(BucketMetricsPipelineAggregatorBuilder<MaxBucketPipelineAggregatorBuilder> other) {
+    protected boolean innerEquals(BucketMetricsPipelineAggregationBuilder<MaxBucketPipelineAggregationBuilder> other) {
         return true;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/min/MinBucketPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/min/MinBucketPipelineAggregationBuilder.java
similarity index 78%
rename from core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/min/MinBucketPipelineAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/min/MinBucketPipelineAggregationBuilder.java
index ee7ae9838cc..d20dbe9f7fc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/min/MinBucketPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/min/MinBucketPipelineAggregationBuilder.java
@@ -24,27 +24,27 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsParser;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsPipelineAggregationBuilder;
 
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
-public class MinBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAggregatorBuilder<MinBucketPipelineAggregatorBuilder> {
+public class MinBucketPipelineAggregationBuilder extends BucketMetricsPipelineAggregationBuilder<MinBucketPipelineAggregationBuilder> {
     public static final String NAME = MinBucketPipelineAggregator.TYPE.name();
     public static final ParseField AGGREGATION_FIELD_NAME = new ParseField(NAME);
 
-    public MinBucketPipelineAggregatorBuilder(String name, String bucketsPath) {
+    public MinBucketPipelineAggregationBuilder(String name, String bucketsPath) {
         super(name, MinBucketPipelineAggregator.TYPE.name(), new String[] { bucketsPath });
     }
 
     /**
      * Read from a stream.
      */
-    public MinBucketPipelineAggregatorBuilder(StreamInput in) throws IOException {
+    public MinBucketPipelineAggregationBuilder(StreamInput in) throws IOException {
         super(in, NAME);
     }
 
@@ -60,7 +60,7 @@ public class MinBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
 
     @Override
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] aggFactories,
-            List<PipelineAggregatorBuilder> pipelineAggregatorFactories) {
+            List<PipelineAggregationBuilder> pipelineAggregatorFactories) {
         if (bucketsPaths.length != 1) {
             throw new IllegalStateException(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                     + " must contain a single entry for aggregation [" + name + "]");
@@ -74,9 +74,9 @@ public class MinBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
 
     public static final PipelineAggregator.Parser PARSER = new BucketMetricsParser() {
         @Override
-        protected MinBucketPipelineAggregatorBuilder buildFactory(String pipelineAggregatorName,
+        protected MinBucketPipelineAggregationBuilder buildFactory(String pipelineAggregatorName,
                 String bucketsPath, Map<String, Object> params) {
-            return new MinBucketPipelineAggregatorBuilder(pipelineAggregatorName, bucketsPath);
+            return new MinBucketPipelineAggregationBuilder(pipelineAggregatorName, bucketsPath);
         }
     };
 
@@ -86,7 +86,7 @@ public class MinBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
     }
 
     @Override
-    protected boolean innerEquals(BucketMetricsPipelineAggregatorBuilder<MinBucketPipelineAggregatorBuilder> other) {
+    protected boolean innerEquals(BucketMetricsPipelineAggregationBuilder<MinBucketPipelineAggregationBuilder> other) {
         return true;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregationBuilder.java
similarity index 83%
rename from core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregationBuilder.java
index ddea1a65545..06a1b06dff1 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregationBuilder.java
@@ -27,10 +27,10 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsParser;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsPipelineAggregationBuilder;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -38,8 +38,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
-public class PercentilesBucketPipelineAggregatorBuilder
-        extends BucketMetricsPipelineAggregatorBuilder<PercentilesBucketPipelineAggregatorBuilder> {
+public class PercentilesBucketPipelineAggregationBuilder
+        extends BucketMetricsPipelineAggregationBuilder<PercentilesBucketPipelineAggregationBuilder> {
     public static final String NAME = PercentilesBucketPipelineAggregator.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
@@ -47,14 +47,14 @@ public class PercentilesBucketPipelineAggregatorBuilder
 
     private double[] percents = new double[] { 1.0, 5.0, 25.0, 50.0, 75.0, 95.0, 99.0 };
 
-    public PercentilesBucketPipelineAggregatorBuilder(String name, String bucketsPath) {
+    public PercentilesBucketPipelineAggregationBuilder(String name, String bucketsPath) {
         super(name, PercentilesBucketPipelineAggregator.TYPE.name(), new String[] { bucketsPath });
     }
 
     /**
      * Read from a stream.
      */
-    public PercentilesBucketPipelineAggregatorBuilder(StreamInput in)
+    public PercentilesBucketPipelineAggregationBuilder(StreamInput in)
             throws IOException {
         super(in, NAME);
         percents = in.readDoubleArray();
@@ -75,7 +75,7 @@ public class PercentilesBucketPipelineAggregatorBuilder
     /**
      * Set the percentages to calculate percentiles for in this aggregation
      */
-    public PercentilesBucketPipelineAggregatorBuilder percents(double[] percents) {
+    public PercentilesBucketPipelineAggregationBuilder percents(double[] percents) {
         if (percents == null) {
             throw new IllegalArgumentException("[percents] must not be null: [" + name + "]");
         }
@@ -96,7 +96,7 @@ public class PercentilesBucketPipelineAggregatorBuilder
 
     @Override
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] aggFactories,
-            List<PipelineAggregatorBuilder> pipelineAggregatorFactories) {
+            List<PipelineAggregationBuilder> pipelineAggregatorFactories) {
         if (bucketsPaths.length != 1) {
             throw new IllegalStateException(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                     + " must contain a single entry for aggregation [" + name + "]");
@@ -121,11 +121,11 @@ public class PercentilesBucketPipelineAggregatorBuilder
     public static final PipelineAggregator.Parser PARSER = new BucketMetricsParser() {
 
         @Override
-        protected PercentilesBucketPipelineAggregatorBuilder buildFactory(String pipelineAggregatorName,
+        protected PercentilesBucketPipelineAggregationBuilder buildFactory(String pipelineAggregatorName,
                                                                           String bucketsPath, Map<String, Object> params) {
 
-            PercentilesBucketPipelineAggregatorBuilder factory = new
-                PercentilesBucketPipelineAggregatorBuilder(pipelineAggregatorName, bucketsPath);
+            PercentilesBucketPipelineAggregationBuilder factory = new
+                PercentilesBucketPipelineAggregationBuilder(pipelineAggregatorName, bucketsPath);
 
             double[] percents = (double[]) params.get(PERCENTS_FIELD.getPreferredName());
             if (percents != null) {
@@ -157,8 +157,8 @@ public class PercentilesBucketPipelineAggregatorBuilder
     }
 
     @Override
-    protected boolean innerEquals(BucketMetricsPipelineAggregatorBuilder<PercentilesBucketPipelineAggregatorBuilder> obj) {
-        PercentilesBucketPipelineAggregatorBuilder other = (PercentilesBucketPipelineAggregatorBuilder) obj;
+    protected boolean innerEquals(BucketMetricsPipelineAggregationBuilder<PercentilesBucketPipelineAggregationBuilder> obj) {
+        PercentilesBucketPipelineAggregationBuilder other = (PercentilesBucketPipelineAggregationBuilder) obj;
         return Objects.deepEquals(percents, other.percents);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregationBuilder.java
similarity index 79%
rename from core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregationBuilder.java
index a3a37bfe9d6..d3418e897ef 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregationBuilder.java
@@ -24,28 +24,28 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsParser;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsPipelineAggregationBuilder;
 
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
-public class StatsBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAggregatorBuilder<StatsBucketPipelineAggregatorBuilder> {
+public class StatsBucketPipelineAggregationBuilder extends BucketMetricsPipelineAggregationBuilder<StatsBucketPipelineAggregationBuilder> {
     public static final String NAME = StatsBucketPipelineAggregator.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
-    public StatsBucketPipelineAggregatorBuilder(String name, String bucketsPath) {
+    public StatsBucketPipelineAggregationBuilder(String name, String bucketsPath) {
         super(name, StatsBucketPipelineAggregator.TYPE.name(), new String[] { bucketsPath });
     }
 
     /**
      * Read from a stream.
      */
-    public StatsBucketPipelineAggregatorBuilder(StreamInput in)
+    public StatsBucketPipelineAggregationBuilder(StreamInput in)
             throws IOException {
         super(in, StatsBucketPipelineAggregator.TYPE.name());
     }
@@ -62,7 +62,7 @@ public class StatsBucketPipelineAggregatorBuilder extends BucketMetricsPipelineA
 
     @Override
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] aggFactories,
-            List<PipelineAggregatorBuilder> pipelineAggregatorFactories) {
+            List<PipelineAggregationBuilder> pipelineAggregatorFactories) {
         if (bucketsPaths.length != 1) {
             throw new IllegalStateException(Parser.BUCKETS_PATH.getPreferredName()
                     + " must contain a single entry for aggregation [" + name + "]");
@@ -76,9 +76,9 @@ public class StatsBucketPipelineAggregatorBuilder extends BucketMetricsPipelineA
 
     public static final PipelineAggregator.Parser PARSER = new BucketMetricsParser() {
         @Override
-        protected StatsBucketPipelineAggregatorBuilder buildFactory(String pipelineAggregatorName,
+        protected StatsBucketPipelineAggregationBuilder buildFactory(String pipelineAggregatorName,
                 String bucketsPath, Map<String, Object> params) {
-            return new StatsBucketPipelineAggregatorBuilder(pipelineAggregatorName, bucketsPath);
+            return new StatsBucketPipelineAggregationBuilder(pipelineAggregatorName, bucketsPath);
         }
     };
 
@@ -88,7 +88,7 @@ public class StatsBucketPipelineAggregatorBuilder extends BucketMetricsPipelineA
     }
 
     @Override
-    protected boolean innerEquals(BucketMetricsPipelineAggregatorBuilder<StatsBucketPipelineAggregatorBuilder> other) {
+    protected boolean innerEquals(BucketMetricsPipelineAggregationBuilder<StatsBucketPipelineAggregationBuilder> other) {
         return true;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketParser.java
index 5ab1a6afae0..b7fa49267dc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketParser.java
@@ -31,10 +31,10 @@ public class ExtendedStatsBucketParser extends BucketMetricsParser {
     static final ParseField SIGMA = new ParseField("sigma");
 
     @Override
-    protected ExtendedStatsBucketPipelineAggregatorBuilder buildFactory(String pipelineAggregatorName,
+    protected ExtendedStatsBucketPipelineAggregationBuilder buildFactory(String pipelineAggregatorName,
             String bucketsPath, Map<String, Object> params) {
-        ExtendedStatsBucketPipelineAggregatorBuilder factory =
-            new ExtendedStatsBucketPipelineAggregatorBuilder(pipelineAggregatorName, bucketsPath);
+        ExtendedStatsBucketPipelineAggregationBuilder factory =
+            new ExtendedStatsBucketPipelineAggregationBuilder(pipelineAggregatorName, bucketsPath);
         Double sigma = (Double) params.get(SIGMA.getPreferredName());
         if (sigma != null) {
             factory.sigma(sigma);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketPipelineAggregationBuilder.java
similarity index 81%
rename from core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketPipelineAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketPipelineAggregationBuilder.java
index 8b17cf7db18..792b9d1f7f1 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketPipelineAggregationBuilder.java
@@ -24,31 +24,31 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsPipelineAggregationBuilder;
 
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
-public class ExtendedStatsBucketPipelineAggregatorBuilder
-        extends BucketMetricsPipelineAggregatorBuilder<ExtendedStatsBucketPipelineAggregatorBuilder> {
+public class ExtendedStatsBucketPipelineAggregationBuilder
+        extends BucketMetricsPipelineAggregationBuilder<ExtendedStatsBucketPipelineAggregationBuilder> {
     public static final String NAME = ExtendedStatsBucketPipelineAggregator.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     private double sigma = 2.0;
 
-    public ExtendedStatsBucketPipelineAggregatorBuilder(String name, String bucketsPath) {
+    public ExtendedStatsBucketPipelineAggregationBuilder(String name, String bucketsPath) {
         super(name, ExtendedStatsBucketPipelineAggregator.TYPE.name(), new String[] { bucketsPath });
     }
 
     /**
      * Read from a stream.
      */
-    public ExtendedStatsBucketPipelineAggregatorBuilder(StreamInput in) throws IOException {
+    public ExtendedStatsBucketPipelineAggregationBuilder(StreamInput in) throws IOException {
         super(in, ExtendedStatsBucketPipelineAggregator.TYPE.name());
         sigma = in.readDouble();
     }
@@ -62,7 +62,7 @@ public class ExtendedStatsBucketPipelineAggregatorBuilder
      * Set the value of sigma to use when calculating the standard deviation
      * bounds
      */
-    public ExtendedStatsBucketPipelineAggregatorBuilder sigma(double sigma) {
+    public ExtendedStatsBucketPipelineAggregationBuilder sigma(double sigma) {
         if (sigma < 0.0) {
             throw new IllegalArgumentException(ExtendedStatsBucketParser.SIGMA.getPreferredName() + " must be a non-negative double");
         }
@@ -85,7 +85,7 @@ public class ExtendedStatsBucketPipelineAggregatorBuilder
 
     @Override
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] aggFactories,
-            List<PipelineAggregatorBuilder> pipelineAggregatorFactories) {
+            List<PipelineAggregationBuilder> pipelineAggregatorFactories) {
         if (bucketsPaths.length != 1) {
             throw new IllegalStateException(Parser.BUCKETS_PATH.getPreferredName()
                     + " must contain a single entry for aggregation [" + name + "]");
@@ -109,8 +109,8 @@ public class ExtendedStatsBucketPipelineAggregatorBuilder
     }
 
     @Override
-    protected boolean innerEquals(BucketMetricsPipelineAggregatorBuilder<ExtendedStatsBucketPipelineAggregatorBuilder> obj) {
-        ExtendedStatsBucketPipelineAggregatorBuilder other = (ExtendedStatsBucketPipelineAggregatorBuilder) obj;
+    protected boolean innerEquals(BucketMetricsPipelineAggregationBuilder<ExtendedStatsBucketPipelineAggregationBuilder> obj) {
+        ExtendedStatsBucketPipelineAggregationBuilder other = (ExtendedStatsBucketPipelineAggregationBuilder) obj;
         return Objects.equals(sigma, other.sigma);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/sum/SumBucketPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/sum/SumBucketPipelineAggregationBuilder.java
similarity index 78%
rename from core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/sum/SumBucketPipelineAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/sum/SumBucketPipelineAggregationBuilder.java
index 72ba158a2b2..cf20eaf6e7d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/sum/SumBucketPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/sum/SumBucketPipelineAggregationBuilder.java
@@ -24,27 +24,27 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsParser;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsPipelineAggregationBuilder;
 
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
-public class SumBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAggregatorBuilder<SumBucketPipelineAggregatorBuilder> {
+public class SumBucketPipelineAggregationBuilder extends BucketMetricsPipelineAggregationBuilder<SumBucketPipelineAggregationBuilder> {
     public static final String NAME = SumBucketPipelineAggregator.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
-    public SumBucketPipelineAggregatorBuilder(String name, String bucketsPath) {
+    public SumBucketPipelineAggregationBuilder(String name, String bucketsPath) {
         super(name, SumBucketPipelineAggregator.TYPE.name(), new String[] { bucketsPath });
     }
 
     /**
      * Read from a stream.
      */
-    public SumBucketPipelineAggregatorBuilder(StreamInput in) throws IOException {
+    public SumBucketPipelineAggregationBuilder(StreamInput in) throws IOException {
         super(in, NAME);
     }
 
@@ -60,7 +60,7 @@ public class SumBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
 
     @Override
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] aggFactories,
-            List<PipelineAggregatorBuilder> pipelineAggregatorFactories) {
+            List<PipelineAggregationBuilder> pipelineAggregatorFactories) {
         if (bucketsPaths.length != 1) {
             throw new IllegalStateException(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                     + " must contain a single entry for aggregation [" + name + "]");
@@ -74,9 +74,9 @@ public class SumBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
 
     public static final PipelineAggregator.Parser PARSER = new BucketMetricsParser() {
         @Override
-        protected SumBucketPipelineAggregatorBuilder buildFactory(String pipelineAggregatorName,
+        protected SumBucketPipelineAggregationBuilder buildFactory(String pipelineAggregatorName,
                 String bucketsPath, Map<String, Object> params) {
-            return new SumBucketPipelineAggregatorBuilder(pipelineAggregatorName, bucketsPath);
+            return new SumBucketPipelineAggregationBuilder(pipelineAggregatorName, bucketsPath);
         }
     };
 
@@ -86,7 +86,7 @@ public class SumBucketPipelineAggregatorBuilder extends BucketMetricsPipelineAgg
     }
 
     @Override
-    protected boolean innerEquals(BucketMetricsPipelineAggregatorBuilder<SumBucketPipelineAggregatorBuilder> other) {
+    protected boolean innerEquals(BucketMetricsPipelineAggregationBuilder<SumBucketPipelineAggregationBuilder> other) {
         return true;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregationBuilder.java
similarity index 91%
rename from core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregationBuilder.java
index 946c48080db..1e285c24a1f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregationBuilder.java
@@ -29,7 +29,7 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.Script.ScriptField;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
@@ -46,7 +46,7 @@ import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.FORMAT;
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.GAP_POLICY;
 
-public class BucketScriptPipelineAggregatorBuilder extends AbstractPipelineAggregatorBuilder<BucketScriptPipelineAggregatorBuilder> {
+public class BucketScriptPipelineAggregationBuilder extends AbstractPipelineAggregationBuilder<BucketScriptPipelineAggregationBuilder> {
     public static final String NAME = BucketScriptPipelineAggregator.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
@@ -55,21 +55,21 @@ public class BucketScriptPipelineAggregatorBuilder extends AbstractPipelineAggre
     private String format = null;
     private GapPolicy gapPolicy = GapPolicy.SKIP;
 
-    public BucketScriptPipelineAggregatorBuilder(String name, Map<String, String> bucketsPathsMap, Script script) {
+    public BucketScriptPipelineAggregationBuilder(String name, Map<String, String> bucketsPathsMap, Script script) {
         super(name, BucketScriptPipelineAggregator.TYPE.name(), new TreeMap<>(bucketsPathsMap).values()
                 .toArray(new String[bucketsPathsMap.size()]));
         this.bucketsPathsMap = bucketsPathsMap;
         this.script = script;
     }
 
-    public BucketScriptPipelineAggregatorBuilder(String name, Script script, String... bucketsPaths) {
+    public BucketScriptPipelineAggregationBuilder(String name, Script script, String... bucketsPaths) {
         this(name, convertToBucketsPathMap(bucketsPaths), script);
     }
 
     /**
      * Read from a stream.
      */
-    public BucketScriptPipelineAggregatorBuilder(StreamInput in) throws IOException {
+    public BucketScriptPipelineAggregationBuilder(StreamInput in) throws IOException {
         super(in, BucketScriptPipelineAggregator.TYPE.name());
         int mapSize = in.readVInt();
         bucketsPathsMap = new HashMap<String, String>(mapSize);
@@ -104,7 +104,7 @@ public class BucketScriptPipelineAggregatorBuilder extends AbstractPipelineAggre
     /**
      * Sets the format to use on the output of this aggregation.
      */
-    public BucketScriptPipelineAggregatorBuilder format(String format) {
+    public BucketScriptPipelineAggregationBuilder format(String format) {
         if (format == null) {
             throw new IllegalArgumentException("[format] must not be null: [" + name + "]");
         }
@@ -130,7 +130,7 @@ public class BucketScriptPipelineAggregatorBuilder extends AbstractPipelineAggre
     /**
      * Sets the gap policy to use for this aggregation.
      */
-    public BucketScriptPipelineAggregatorBuilder gapPolicy(GapPolicy gapPolicy) {
+    public BucketScriptPipelineAggregationBuilder gapPolicy(GapPolicy gapPolicy) {
         if (gapPolicy == null) {
             throw new IllegalArgumentException("[gapPolicy] must not be null: [" + name + "]");
         }
@@ -161,7 +161,7 @@ public class BucketScriptPipelineAggregatorBuilder extends AbstractPipelineAggre
         return builder;
     }
 
-    public static BucketScriptPipelineAggregatorBuilder parse(String reducerName, QueryParseContext context) throws IOException {
+    public static BucketScriptPipelineAggregationBuilder parse(String reducerName, QueryParseContext context) throws IOException {
         XContentParser parser = context.parser();
         XContentParser.Token token;
         Script script = null;
@@ -230,8 +230,8 @@ public class BucketScriptPipelineAggregatorBuilder extends AbstractPipelineAggre
                     + "] for series_arithmetic aggregation [" + reducerName + "]");
         }
 
-        BucketScriptPipelineAggregatorBuilder factory =
-                new BucketScriptPipelineAggregatorBuilder(reducerName, bucketsPathsMap, script);
+        BucketScriptPipelineAggregationBuilder factory =
+                new BucketScriptPipelineAggregationBuilder(reducerName, bucketsPathsMap, script);
         if (format != null) {
             factory.format(format);
         }
@@ -254,7 +254,7 @@ public class BucketScriptPipelineAggregatorBuilder extends AbstractPipelineAggre
 
     @Override
     protected boolean doEquals(Object obj) {
-        BucketScriptPipelineAggregatorBuilder other = (BucketScriptPipelineAggregatorBuilder) obj;
+        BucketScriptPipelineAggregationBuilder other = (BucketScriptPipelineAggregationBuilder) obj;
         return Objects.equals(bucketsPathsMap, other.bucketsPathsMap) && Objects.equals(script, other.script)
                 && Objects.equals(format, other.format) && Objects.equals(gapPolicy, other.gapPolicy);
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregationBuilder.java
similarity index 90%
rename from core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregationBuilder.java
index 8e1b854b350..9054812ed3f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregationBuilder.java
@@ -28,7 +28,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.Script.ScriptField;
-import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
@@ -44,7 +44,7 @@ import java.util.TreeMap;
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.BUCKETS_PATH;
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.GAP_POLICY;
 
-public class BucketSelectorPipelineAggregatorBuilder extends AbstractPipelineAggregatorBuilder<BucketSelectorPipelineAggregatorBuilder> {
+public class BucketSelectorPipelineAggregationBuilder extends AbstractPipelineAggregationBuilder<BucketSelectorPipelineAggregationBuilder> {
     public static final String NAME = BucketSelectorPipelineAggregator.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
@@ -52,21 +52,21 @@ public class BucketSelectorPipelineAggregatorBuilder extends AbstractPipelineAgg
     private Script script;
     private GapPolicy gapPolicy = GapPolicy.SKIP;
 
-    public BucketSelectorPipelineAggregatorBuilder(String name, Map<String, String> bucketsPathsMap, Script script) {
+    public BucketSelectorPipelineAggregationBuilder(String name, Map<String, String> bucketsPathsMap, Script script) {
         super(name, BucketSelectorPipelineAggregator.TYPE.name(), new TreeMap<>(bucketsPathsMap).values()
                 .toArray(new String[bucketsPathsMap.size()]));
         this.bucketsPathsMap = bucketsPathsMap;
         this.script = script;
     }
 
-    public BucketSelectorPipelineAggregatorBuilder(String name, Script script, String... bucketsPaths) {
+    public BucketSelectorPipelineAggregationBuilder(String name, Script script, String... bucketsPaths) {
         this(name, convertToBucketsPathMap(bucketsPaths), script);
     }
 
     /**
      * Read from a stream.
      */
-    public BucketSelectorPipelineAggregatorBuilder(StreamInput in) throws IOException {
+    public BucketSelectorPipelineAggregationBuilder(StreamInput in) throws IOException {
         super(in, BucketSelectorPipelineAggregator.TYPE.name());
         int mapSize = in.readVInt();
         bucketsPathsMap = new HashMap<String, String>(mapSize);
@@ -99,7 +99,7 @@ public class BucketSelectorPipelineAggregatorBuilder extends AbstractPipelineAgg
     /**
      * Sets the gap policy to use for this aggregation.
      */
-    public BucketSelectorPipelineAggregatorBuilder gapPolicy(GapPolicy gapPolicy) {
+    public BucketSelectorPipelineAggregationBuilder gapPolicy(GapPolicy gapPolicy) {
         if (gapPolicy == null) {
             throw new IllegalArgumentException("[gapPolicy] must not be null: [" + name + "]");
         }
@@ -127,7 +127,7 @@ public class BucketSelectorPipelineAggregatorBuilder extends AbstractPipelineAgg
         return builder;
     }
 
-    public static BucketSelectorPipelineAggregatorBuilder parse(String reducerName, QueryParseContext context) throws IOException {
+    public static BucketSelectorPipelineAggregationBuilder parse(String reducerName, QueryParseContext context) throws IOException {
         XContentParser parser = context.parser();
         XContentParser.Token token;
         Script script = null;
@@ -193,8 +193,8 @@ public class BucketSelectorPipelineAggregatorBuilder extends AbstractPipelineAgg
                     + "] for bucket_selector aggregation [" + reducerName + "]");
         }
 
-        BucketSelectorPipelineAggregatorBuilder factory =
-                new BucketSelectorPipelineAggregatorBuilder(reducerName, bucketsPathsMap, script);
+        BucketSelectorPipelineAggregationBuilder factory =
+                new BucketSelectorPipelineAggregationBuilder(reducerName, bucketsPathsMap, script);
         if (gapPolicy != null) {
             factory.gapPolicy(gapPolicy);
         }
@@ -213,7 +213,7 @@ public class BucketSelectorPipelineAggregatorBuilder extends AbstractPipelineAgg
 
     @Override
     protected boolean doEquals(Object obj) {
-        BucketSelectorPipelineAggregatorBuilder other = (BucketSelectorPipelineAggregatorBuilder) obj;
+        BucketSelectorPipelineAggregationBuilder other = (BucketSelectorPipelineAggregationBuilder) obj;
         return Objects.equals(bucketsPathsMap, other.bucketsPathsMap) && Objects.equals(script, other.script)
                 && Objects.equals(gapPolicy, other.gapPolicy);
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregationBuilder.java
similarity index 88%
rename from core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregationBuilder.java
index a4b2178d0aa..1927317fc2c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregationBuilder.java
@@ -28,9 +28,9 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.AbstractHistogramAggregatorFactory;
-import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsParser;
 
@@ -43,20 +43,20 @@ import java.util.Objects;
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.BUCKETS_PATH;
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.FORMAT;
 
-public class CumulativeSumPipelineAggregatorBuilder extends AbstractPipelineAggregatorBuilder<CumulativeSumPipelineAggregatorBuilder> {
+public class CumulativeSumPipelineAggregationBuilder extends AbstractPipelineAggregationBuilder<CumulativeSumPipelineAggregationBuilder> {
     public static final String NAME = CumulativeSumPipelineAggregator.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     private String format;
 
-    public CumulativeSumPipelineAggregatorBuilder(String name, String bucketsPath) {
+    public CumulativeSumPipelineAggregationBuilder(String name, String bucketsPath) {
         super(name, CumulativeSumPipelineAggregator.TYPE.name(), new String[] { bucketsPath });
     }
 
     /**
      * Read from a stream.
      */
-    public CumulativeSumPipelineAggregatorBuilder(StreamInput in) throws IOException {
+    public CumulativeSumPipelineAggregationBuilder(StreamInput in) throws IOException {
         super(in, CumulativeSumPipelineAggregator.TYPE.name());
         format = in.readOptionalString();
     }
@@ -69,7 +69,7 @@ public class CumulativeSumPipelineAggregatorBuilder extends AbstractPipelineAggr
     /**
      * Sets the format to use on the output of this aggregation.
      */
-    public CumulativeSumPipelineAggregatorBuilder format(String format) {
+    public CumulativeSumPipelineAggregationBuilder format(String format) {
         if (format == null) {
             throw new IllegalArgumentException("[format] must not be null: [" + name + "]");
         }
@@ -99,7 +99,7 @@ public class CumulativeSumPipelineAggregatorBuilder extends AbstractPipelineAggr
 
     @Override
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] aggFactories,
-            List<PipelineAggregatorBuilder> pipelineAggregatorFactories) {
+            List<PipelineAggregationBuilder> pipelineAggregatorFactories) {
         if (bucketsPaths.length != 1) {
             throw new IllegalStateException(BUCKETS_PATH.getPreferredName()
                     + " must contain a single entry for aggregation [" + name + "]");
@@ -124,7 +124,7 @@ public class CumulativeSumPipelineAggregatorBuilder extends AbstractPipelineAggr
         return builder;
     }
 
-    public static CumulativeSumPipelineAggregatorBuilder parse(String pipelineAggregatorName, QueryParseContext context)
+    public static CumulativeSumPipelineAggregationBuilder parse(String pipelineAggregatorName, QueryParseContext context)
             throws IOException {
         XContentParser parser = context.parser();
         XContentParser.Token token;
@@ -167,8 +167,8 @@ public class CumulativeSumPipelineAggregatorBuilder extends AbstractPipelineAggr
                     + "] for derivative aggregation [" + pipelineAggregatorName + "]");
         }
 
-        CumulativeSumPipelineAggregatorBuilder factory =
-                new CumulativeSumPipelineAggregatorBuilder(pipelineAggregatorName, bucketsPaths[0]);
+        CumulativeSumPipelineAggregationBuilder factory =
+                new CumulativeSumPipelineAggregationBuilder(pipelineAggregatorName, bucketsPaths[0]);
         if (format != null) {
             factory.format(format);
         }
@@ -182,7 +182,7 @@ public class CumulativeSumPipelineAggregatorBuilder extends AbstractPipelineAggr
 
     @Override
     protected boolean doEquals(Object obj) {
-        CumulativeSumPipelineAggregatorBuilder other = (CumulativeSumPipelineAggregatorBuilder) obj;
+        CumulativeSumPipelineAggregationBuilder other = (CumulativeSumPipelineAggregationBuilder) obj;
         return Objects.equals(format, other.format);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregationBuilder.java
similarity index 89%
rename from core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregationBuilder.java
index 0aba20e37c4..5ad2f88b198 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregationBuilder.java
@@ -30,11 +30,11 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.AbstractHistogramAggregatorFactory;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregatorFactory;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
-import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.joda.time.DateTimeZone;
@@ -45,7 +45,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
-public class DerivativePipelineAggregatorBuilder extends AbstractPipelineAggregatorBuilder<DerivativePipelineAggregatorBuilder> {
+public class DerivativePipelineAggregationBuilder extends AbstractPipelineAggregationBuilder<DerivativePipelineAggregationBuilder> {
     public static final String NAME = DerivativePipelineAggregator.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
@@ -57,18 +57,18 @@ public class DerivativePipelineAggregatorBuilder extends AbstractPipelineAggrega
     private GapPolicy gapPolicy = GapPolicy.SKIP;
     private String units;
 
-    public DerivativePipelineAggregatorBuilder(String name, String bucketsPath) {
+    public DerivativePipelineAggregationBuilder(String name, String bucketsPath) {
         this(name, new String[] { bucketsPath });
     }
 
-    private DerivativePipelineAggregatorBuilder(String name, String[] bucketsPaths) {
+    private DerivativePipelineAggregationBuilder(String name, String[] bucketsPaths) {
         super(name, DerivativePipelineAggregator.TYPE.name(), bucketsPaths);
     }
 
     /**
      * Read from a stream.
      */
-    public DerivativePipelineAggregatorBuilder(StreamInput in) throws IOException {
+    public DerivativePipelineAggregationBuilder(StreamInput in) throws IOException {
         super(in, DerivativePipelineAggregator.TYPE.name());
         format = in.readOptionalString();
         if (in.readBoolean()) {
@@ -88,7 +88,7 @@ public class DerivativePipelineAggregatorBuilder extends AbstractPipelineAggrega
         out.writeOptionalString(units);
     }
 
-    public DerivativePipelineAggregatorBuilder format(String format) {
+    public DerivativePipelineAggregationBuilder format(String format) {
         if (format == null) {
             throw new IllegalArgumentException("[format] must not be null: [" + name + "]");
         }
@@ -100,7 +100,7 @@ public class DerivativePipelineAggregatorBuilder extends AbstractPipelineAggrega
         return format;
     }
 
-    public DerivativePipelineAggregatorBuilder gapPolicy(GapPolicy gapPolicy) {
+    public DerivativePipelineAggregationBuilder gapPolicy(GapPolicy gapPolicy) {
         if (gapPolicy == null) {
             throw new IllegalArgumentException("[gapPolicy] must not be null: [" + name + "]");
         }
@@ -112,7 +112,7 @@ public class DerivativePipelineAggregatorBuilder extends AbstractPipelineAggrega
         return gapPolicy;
     }
 
-    public DerivativePipelineAggregatorBuilder unit(String units) {
+    public DerivativePipelineAggregationBuilder unit(String units) {
         if (units == null) {
             throw new IllegalArgumentException("[units] must not be null: [" + name + "]");
         }
@@ -120,7 +120,7 @@ public class DerivativePipelineAggregatorBuilder extends AbstractPipelineAggrega
         return this;
     }
 
-    public DerivativePipelineAggregatorBuilder unit(DateHistogramInterval units) {
+    public DerivativePipelineAggregationBuilder unit(DateHistogramInterval units) {
         if (units == null) {
             throw new IllegalArgumentException("[units] must not be null: [" + name + "]");
         }
@@ -157,7 +157,7 @@ public class DerivativePipelineAggregatorBuilder extends AbstractPipelineAggrega
 
     @Override
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] aggFactories,
-            List<PipelineAggregatorBuilder> pipelineAggregatoractories) {
+            List<PipelineAggregationBuilder> pipelineAggregatoractories) {
         if (bucketsPaths.length != 1) {
             throw new IllegalStateException(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName()
                     + " must contain a single entry for aggregation [" + name + "]");
@@ -188,7 +188,7 @@ public class DerivativePipelineAggregatorBuilder extends AbstractPipelineAggrega
         return builder;
     }
 
-    public static DerivativePipelineAggregatorBuilder parse(String pipelineAggregatorName, QueryParseContext context) throws IOException {
+    public static DerivativePipelineAggregationBuilder parse(String pipelineAggregatorName, QueryParseContext context) throws IOException {
         XContentParser parser = context.parser();
         XContentParser.Token token;
         String currentFieldName = null;
@@ -236,8 +236,8 @@ public class DerivativePipelineAggregatorBuilder extends AbstractPipelineAggrega
                     + "] for derivative aggregation [" + pipelineAggregatorName + "]");
         }
 
-        DerivativePipelineAggregatorBuilder factory =
-                new DerivativePipelineAggregatorBuilder(pipelineAggregatorName, bucketsPaths[0]);
+        DerivativePipelineAggregationBuilder factory =
+                new DerivativePipelineAggregationBuilder(pipelineAggregatorName, bucketsPaths[0]);
         if (format != null) {
             factory.format(format);
         }
@@ -252,7 +252,7 @@ public class DerivativePipelineAggregatorBuilder extends AbstractPipelineAggrega
 
     @Override
     protected boolean doEquals(Object obj) {
-        DerivativePipelineAggregatorBuilder other = (DerivativePipelineAggregatorBuilder) obj;
+        DerivativePipelineAggregationBuilder other = (DerivativePipelineAggregationBuilder) obj;
         if (!Objects.equals(format, other.format)) {
             return false;
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregationBuilder.java
similarity index 92%
rename from core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregationBuilder.java
index 2cf8a1bd046..7d71af17b42 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregationBuilder.java
@@ -29,9 +29,9 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
-import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.AbstractHistogramAggregatorFactory;
-import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.MovAvgModel;
@@ -49,7 +49,7 @@ import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.FORMAT;
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.GAP_POLICY;
 
-public class MovAvgPipelineAggregatorBuilder extends AbstractPipelineAggregatorBuilder<MovAvgPipelineAggregatorBuilder> {
+public class MovAvgPipelineAggregationBuilder extends AbstractPipelineAggregationBuilder<MovAvgPipelineAggregationBuilder> {
     public static final String NAME = MovAvgPipelineAggregator.TYPE.name();
     public static final ParseField AGGREGATION_FIELD_NAME = new ParseField(NAME);
 
@@ -66,14 +66,14 @@ public class MovAvgPipelineAggregatorBuilder extends AbstractPipelineAggregatorB
     private int predict = 0;
     private Boolean minimize;
 
-    public MovAvgPipelineAggregatorBuilder(String name, String bucketsPath) {
+    public MovAvgPipelineAggregationBuilder(String name, String bucketsPath) {
         super(name, MovAvgPipelineAggregator.TYPE.name(), new String[] { bucketsPath });
     }
 
     /**
      * Read from a stream.
      */
-    public MovAvgPipelineAggregatorBuilder(StreamInput in) throws IOException {
+    public MovAvgPipelineAggregationBuilder(StreamInput in) throws IOException {
         super(in, MovAvgPipelineAggregator.TYPE.name());
         format = in.readOptionalString();
         gapPolicy = GapPolicy.readFrom(in);
@@ -96,7 +96,7 @@ public class MovAvgPipelineAggregatorBuilder extends AbstractPipelineAggregatorB
     /**
      * Sets the format to use on the output of this aggregation.
      */
-    public MovAvgPipelineAggregatorBuilder format(String format) {
+    public MovAvgPipelineAggregationBuilder format(String format) {
         if (format == null) {
             throw new IllegalArgumentException("[format] must not be null: [" + name + "]");
         }
@@ -114,7 +114,7 @@ public class MovAvgPipelineAggregatorBuilder extends AbstractPipelineAggregatorB
     /**
      * Sets the GapPolicy to use on the output of this aggregation.
      */
-    public MovAvgPipelineAggregatorBuilder gapPolicy(GapPolicy gapPolicy) {
+    public MovAvgPipelineAggregationBuilder gapPolicy(GapPolicy gapPolicy) {
         if (gapPolicy == null) {
             throw new IllegalArgumentException("[gapPolicy] must not be null: [" + name + "]");
         }
@@ -145,7 +145,7 @@ public class MovAvgPipelineAggregatorBuilder extends AbstractPipelineAggregatorB
      * @param window
      *            Size of window
      */
-    public MovAvgPipelineAggregatorBuilder window(int window) {
+    public MovAvgPipelineAggregationBuilder window(int window) {
         if (window <= 0) {
             throw new IllegalArgumentException("[window] must be a positive integer: [" + name + "]");
         }
@@ -169,7 +169,7 @@ public class MovAvgPipelineAggregatorBuilder extends AbstractPipelineAggregatorB
      * @param model
      *            A MovAvgModel which has been prepopulated with settings
      */
-    public MovAvgPipelineAggregatorBuilder modelBuilder(MovAvgModelBuilder model) {
+    public MovAvgPipelineAggregationBuilder modelBuilder(MovAvgModelBuilder model) {
         if (model == null) {
             throw new IllegalArgumentException("[model] must not be null: [" + name + "]");
         }
@@ -184,7 +184,7 @@ public class MovAvgPipelineAggregatorBuilder extends AbstractPipelineAggregatorB
      * @param model
      *            A MovAvgModel which has been prepopulated with settings
      */
-    public MovAvgPipelineAggregatorBuilder model(MovAvgModel model) {
+    public MovAvgPipelineAggregationBuilder model(MovAvgModel model) {
         if (model == null) {
             throw new IllegalArgumentException("[model] must not be null: [" + name + "]");
         }
@@ -209,7 +209,7 @@ public class MovAvgPipelineAggregatorBuilder extends AbstractPipelineAggregatorB
      * @param predict
      *            Number of predictions to make
      */
-    public MovAvgPipelineAggregatorBuilder predict(int predict) {
+    public MovAvgPipelineAggregationBuilder predict(int predict) {
         if (predict <= 0) {
             throw new IllegalArgumentException("predict must be greater than 0. Found [" + predict + "] in [" + name + "]");
         }
@@ -234,7 +234,7 @@ public class MovAvgPipelineAggregatorBuilder extends AbstractPipelineAggregatorB
      * @param minimize
      *            If the model should be fit to the underlying data
      */
-    public MovAvgPipelineAggregatorBuilder minimize(boolean minimize) {
+    public MovAvgPipelineAggregationBuilder minimize(boolean minimize) {
         this.minimize = minimize;
         return this;
     }
@@ -257,7 +257,7 @@ public class MovAvgPipelineAggregatorBuilder extends AbstractPipelineAggregatorB
 
     @Override
     public void doValidate(AggregatorFactory<?> parent, AggregatorFactory<?>[] aggFactories,
-            List<PipelineAggregatorBuilder> pipelineAggregatoractories) {
+            List<PipelineAggregationBuilder> pipelineAggregatoractories) {
         if (minimize != null && minimize && !model.canBeMinimized()) {
             // If the user asks to minimize, but this model doesn't support
             // it, throw exception
@@ -296,7 +296,8 @@ public class MovAvgPipelineAggregatorBuilder extends AbstractPipelineAggregatorB
         return builder;
     }
 
-    public static MovAvgPipelineAggregatorBuilder parse(ParseFieldRegistry<MovAvgModel.AbstractModelParser> movingAverageMdelParserRegistry,
+    public static MovAvgPipelineAggregationBuilder parse(
+            ParseFieldRegistry<MovAvgModel.AbstractModelParser> movingAverageMdelParserRegistry,
             String pipelineAggregatorName, QueryParseContext context) throws IOException {
         XContentParser parser = context.parser();
         XContentParser.Token token;
@@ -381,8 +382,8 @@ public class MovAvgPipelineAggregatorBuilder extends AbstractPipelineAggregatorB
                     + "] for movingAvg aggregation [" + pipelineAggregatorName + "]");
         }
 
-        MovAvgPipelineAggregatorBuilder factory =
-                new MovAvgPipelineAggregatorBuilder(pipelineAggregatorName, bucketsPaths[0]);
+        MovAvgPipelineAggregationBuilder factory =
+                new MovAvgPipelineAggregationBuilder(pipelineAggregatorName, bucketsPaths[0]);
         if (format != null) {
             factory.format(format);
         }
@@ -419,7 +420,7 @@ public class MovAvgPipelineAggregatorBuilder extends AbstractPipelineAggregatorB
 
     @Override
     protected boolean doEquals(Object obj) {
-        MovAvgPipelineAggregatorBuilder other = (MovAvgPipelineAggregatorBuilder) obj;
+        MovAvgPipelineAggregationBuilder other = (MovAvgPipelineAggregationBuilder) obj;
         return Objects.equals(format, other.format)
                 && Objects.equals(gapPolicy, other.gapPolicy)
                 && Objects.equals(window, other.window)
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/EwmaModel.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/EwmaModel.java
index 3248096488f..418930d2465 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/EwmaModel.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/EwmaModel.java
@@ -25,7 +25,7 @@ import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.search.aggregations.pipeline.movavg.MovAvgPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.movavg.MovAvgPipelineAggregationBuilder;
 
 import java.io.IOException;
 import java.text.ParseException;
@@ -120,8 +120,8 @@ public class EwmaModel extends MovAvgModel {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(MovAvgPipelineAggregatorBuilder.MODEL.getPreferredName(), NAME_FIELD.getPreferredName());
-        builder.startObject(MovAvgPipelineAggregatorBuilder.SETTINGS.getPreferredName());
+        builder.field(MovAvgPipelineAggregationBuilder.MODEL.getPreferredName(), NAME_FIELD.getPreferredName());
+        builder.startObject(MovAvgPipelineAggregationBuilder.SETTINGS.getPreferredName());
         builder.field("alpha", alpha);
         builder.endObject();
         return builder;
@@ -174,8 +174,8 @@ public class EwmaModel extends MovAvgModel {
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.field(MovAvgPipelineAggregatorBuilder.MODEL.getPreferredName(), NAME_FIELD.getPreferredName());
-            builder.startObject(MovAvgPipelineAggregatorBuilder.SETTINGS.getPreferredName());
+            builder.field(MovAvgPipelineAggregationBuilder.MODEL.getPreferredName(), NAME_FIELD.getPreferredName());
+            builder.startObject(MovAvgPipelineAggregationBuilder.SETTINGS.getPreferredName());
             builder.field("alpha", alpha);
 
             builder.endObject();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltLinearModel.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltLinearModel.java
index 23eb1fea036..3ebfeb2ce26 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltLinearModel.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltLinearModel.java
@@ -25,7 +25,7 @@ import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.search.aggregations.pipeline.movavg.MovAvgPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.movavg.MovAvgPipelineAggregationBuilder;
 
 import java.io.IOException;
 import java.text.ParseException;
@@ -183,8 +183,8 @@ public class HoltLinearModel extends MovAvgModel {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(MovAvgPipelineAggregatorBuilder.MODEL.getPreferredName(), NAME_FIELD.getPreferredName());
-        builder.startObject(MovAvgPipelineAggregatorBuilder.SETTINGS.getPreferredName());
+        builder.field(MovAvgPipelineAggregationBuilder.MODEL.getPreferredName(), NAME_FIELD.getPreferredName());
+        builder.startObject(MovAvgPipelineAggregationBuilder.SETTINGS.getPreferredName());
         builder.field("alpha", alpha);
         builder.field("beta", beta);
         builder.endObject();
@@ -255,8 +255,8 @@ public class HoltLinearModel extends MovAvgModel {
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.field(MovAvgPipelineAggregatorBuilder.MODEL.getPreferredName(), NAME_FIELD.getPreferredName());
-            builder.startObject(MovAvgPipelineAggregatorBuilder.SETTINGS.getPreferredName());
+            builder.field(MovAvgPipelineAggregationBuilder.MODEL.getPreferredName(), NAME_FIELD.getPreferredName());
+            builder.startObject(MovAvgPipelineAggregationBuilder.SETTINGS.getPreferredName());
             builder.field("alpha", alpha);
             builder.field("beta", beta);
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltWintersModel.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltWintersModel.java
index 43a0be951cc..9249a4c7116 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltWintersModel.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/HoltWintersModel.java
@@ -28,7 +28,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
-import org.elasticsearch.search.aggregations.pipeline.movavg.MovAvgPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.movavg.MovAvgPipelineAggregationBuilder;
 
 import java.io.IOException;
 import java.text.ParseException;
@@ -366,8 +366,8 @@ public class HoltWintersModel extends MovAvgModel {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(MovAvgPipelineAggregatorBuilder.MODEL.getPreferredName(), NAME_FIELD.getPreferredName());
-        builder.startObject(MovAvgPipelineAggregatorBuilder.SETTINGS.getPreferredName());
+        builder.field(MovAvgPipelineAggregationBuilder.MODEL.getPreferredName(), NAME_FIELD.getPreferredName());
+        builder.startObject(MovAvgPipelineAggregationBuilder.SETTINGS.getPreferredName());
         builder.field("alpha", alpha);
         builder.field("beta", beta);
         builder.field("gamma", gamma);
@@ -495,8 +495,8 @@ public class HoltWintersModel extends MovAvgModel {
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.field(MovAvgPipelineAggregatorBuilder.MODEL.getPreferredName(), NAME_FIELD.getPreferredName());
-            builder.startObject(MovAvgPipelineAggregatorBuilder.SETTINGS.getPreferredName());
+            builder.field(MovAvgPipelineAggregationBuilder.MODEL.getPreferredName(), NAME_FIELD.getPreferredName());
+            builder.startObject(MovAvgPipelineAggregationBuilder.SETTINGS.getPreferredName());
             builder.field("alpha", alpha);
             builder.field("beta", beta);
             builder.field("gamma", gamma);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/LinearModel.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/LinearModel.java
index 6ce24338917..01ca557ef32 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/LinearModel.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/LinearModel.java
@@ -26,7 +26,7 @@ import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.search.aggregations.pipeline.movavg.MovAvgPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.movavg.MovAvgPipelineAggregationBuilder;
 
 import java.io.IOException;
 import java.text.ParseException;
@@ -102,7 +102,7 @@ public class LinearModel extends MovAvgModel {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(MovAvgPipelineAggregatorBuilder.MODEL.getPreferredName(), NAME_FIELD.getPreferredName());
+        builder.field(MovAvgPipelineAggregationBuilder.MODEL.getPreferredName(), NAME_FIELD.getPreferredName());
         return builder;
     }
 
@@ -118,7 +118,7 @@ public class LinearModel extends MovAvgModel {
     public static class LinearModelBuilder implements MovAvgModelBuilder {
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.field(MovAvgPipelineAggregatorBuilder.MODEL.getPreferredName(), NAME_FIELD.getPreferredName());
+            builder.field(MovAvgPipelineAggregationBuilder.MODEL.getPreferredName(), NAME_FIELD.getPreferredName());
             return builder;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/SimpleModel.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/SimpleModel.java
index 4e9e2112947..7a6c72b00dc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/SimpleModel.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/SimpleModel.java
@@ -25,7 +25,7 @@ import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.search.aggregations.pipeline.movavg.MovAvgPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.movavg.MovAvgPipelineAggregationBuilder;
 
 import java.io.IOException;
 import java.text.ParseException;
@@ -95,7 +95,7 @@ public class SimpleModel extends MovAvgModel {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.field(MovAvgPipelineAggregatorBuilder.MODEL.getPreferredName(), NAME);
+        builder.field(MovAvgPipelineAggregationBuilder.MODEL.getPreferredName(), NAME);
         return builder;
     }
 
@@ -111,7 +111,7 @@ public class SimpleModel extends MovAvgModel {
     public static class SimpleModelBuilder implements MovAvgModelBuilder {
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-            builder.field(MovAvgPipelineAggregatorBuilder.MODEL.getPreferredName(), NAME_FIELD.getPreferredName());
+            builder.field(MovAvgPipelineAggregationBuilder.MODEL.getPreferredName(), NAME_FIELD.getPreferredName());
             return builder;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregatorBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregationBuilder.java
similarity index 90%
rename from core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregatorBuilder.java
rename to core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregationBuilder.java
index 5f9096542ea..d671e43045b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregatorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregationBuilder.java
@@ -27,7 +27,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
@@ -40,7 +40,7 @@ import java.util.Objects;
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.BUCKETS_PATH;
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.FORMAT;
 
-public class SerialDiffPipelineAggregatorBuilder extends AbstractPipelineAggregatorBuilder<SerialDiffPipelineAggregatorBuilder> {
+public class SerialDiffPipelineAggregationBuilder extends AbstractPipelineAggregationBuilder<SerialDiffPipelineAggregationBuilder> {
     public static final String NAME = SerialDiffPipelineAggregator.TYPE.name();
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
@@ -51,14 +51,14 @@ public class SerialDiffPipelineAggregatorBuilder extends AbstractPipelineAggrega
     private GapPolicy gapPolicy = GapPolicy.SKIP;
     private int lag = 1;
 
-    public SerialDiffPipelineAggregatorBuilder(String name, String bucketsPath) {
+    public SerialDiffPipelineAggregationBuilder(String name, String bucketsPath) {
         super(name, SerialDiffPipelineAggregator.TYPE.name(), new String[] { bucketsPath });
     }
 
     /**
      * Read from a stream.
      */
-    public SerialDiffPipelineAggregatorBuilder(StreamInput in) throws IOException {
+    public SerialDiffPipelineAggregationBuilder(StreamInput in) throws IOException {
         super(in, SerialDiffPipelineAggregator.TYPE.name());
         format = in.readOptionalString();
         gapPolicy = GapPolicy.readFrom(in);
@@ -75,7 +75,7 @@ public class SerialDiffPipelineAggregatorBuilder extends AbstractPipelineAggrega
     /**
      * Sets the lag to use when calculating the serial difference.
      */
-    public SerialDiffPipelineAggregatorBuilder lag(int lag) {
+    public SerialDiffPipelineAggregationBuilder lag(int lag) {
         if (lag <= 0) {
             throw new IllegalArgumentException("[lag] must be a positive integer: [" + name + "]");
         }
@@ -93,7 +93,7 @@ public class SerialDiffPipelineAggregatorBuilder extends AbstractPipelineAggrega
     /**
      * Sets the format to use on the output of this aggregation.
      */
-    public SerialDiffPipelineAggregatorBuilder format(String format) {
+    public SerialDiffPipelineAggregationBuilder format(String format) {
         if (format == null) {
             throw new IllegalArgumentException("[format] must not be null: [" + name + "]");
         }
@@ -111,7 +111,7 @@ public class SerialDiffPipelineAggregatorBuilder extends AbstractPipelineAggrega
     /**
      * Sets the GapPolicy to use on the output of this aggregation.
      */
-    public SerialDiffPipelineAggregatorBuilder gapPolicy(GapPolicy gapPolicy) {
+    public SerialDiffPipelineAggregationBuilder gapPolicy(GapPolicy gapPolicy) {
         if (gapPolicy == null) {
             throw new IllegalArgumentException("[gapPolicy] must not be null: [" + name + "]");
         }
@@ -149,7 +149,7 @@ public class SerialDiffPipelineAggregatorBuilder extends AbstractPipelineAggrega
         return builder;
     }
 
-    public static SerialDiffPipelineAggregatorBuilder parse(String reducerName, QueryParseContext context) throws IOException {
+    public static SerialDiffPipelineAggregationBuilder parse(String reducerName, QueryParseContext context) throws IOException {
         XContentParser parser = context.parser();
         XContentParser.Token token;
         String currentFieldName = null;
@@ -208,8 +208,8 @@ public class SerialDiffPipelineAggregatorBuilder extends AbstractPipelineAggrega
                     "Missing required field [" + BUCKETS_PATH.getPreferredName() + "] for derivative aggregation [" + reducerName + "]");
         }
 
-        SerialDiffPipelineAggregatorBuilder factory =
-                new SerialDiffPipelineAggregatorBuilder(reducerName, bucketsPaths[0]);
+        SerialDiffPipelineAggregationBuilder factory =
+                new SerialDiffPipelineAggregationBuilder(reducerName, bucketsPaths[0]);
         if (lag != null) {
             factory.lag(lag);
         }
@@ -228,7 +228,7 @@ public class SerialDiffPipelineAggregatorBuilder extends AbstractPipelineAggrega
     }
     @Override
     protected boolean doEquals(Object obj) {
-        SerialDiffPipelineAggregatorBuilder other = (SerialDiffPipelineAggregatorBuilder) obj;
+        SerialDiffPipelineAggregationBuilder other = (SerialDiffPipelineAggregationBuilder) obj;
         return Objects.equals(format, other.format)
                 && Objects.equals(gapPolicy, other.gapPolicy)
                 && Objects.equals(lag, other.lag);
diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index b1a5d2d03e6..34b2b536259 100644
--- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -45,7 +45,7 @@ import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.slice.SliceBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
-import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.fetch.source.FetchSourceContext;
 import org.elasticsearch.search.highlight.HighlightBuilder;
 import org.elasticsearch.search.internal.SearchContext;
@@ -591,7 +591,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
     /**
      * Add an aggregation to perform as part of the search.
      */
-    public SearchSourceBuilder aggregation(PipelineAggregatorBuilder aggregation) {
+    public SearchSourceBuilder aggregation(PipelineAggregationBuilder aggregation) {
             if (aggregations == null) {
             aggregations = AggregatorFactories.builder();
             }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
index 2d3a86fb4be..3e409f3d408 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
@@ -52,7 +52,7 @@ import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchModule;
-import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
@@ -70,7 +70,7 @@ import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
 import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.equalTo;
 
-public abstract class BasePipelineAggregationTestCase<AF extends AbstractPipelineAggregatorBuilder<AF>> extends ESTestCase {
+public abstract class BasePipelineAggregationTestCase<AF extends AbstractPipelineAggregationBuilder<AF>> extends ESTestCase {
 
     protected static final String STRING_FIELD_NAME = "mapped_string";
     protected static final String INT_FIELD_NAME = "mapped_int";
@@ -198,7 +198,7 @@ public abstract class BasePipelineAggregationTestCase<AF extends AbstractPipelin
         assertSame(XContentParser.Token.FIELD_NAME, parser.nextToken());
         assertEquals(testAgg.type(), parser.currentName());
         assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        PipelineAggregatorBuilder newAgg = aggParsers.pipelineParser(testAgg.getWriteableName(), ParseFieldMatcher.STRICT)
+        PipelineAggregationBuilder newAgg = aggParsers.pipelineParser(testAgg.getWriteableName(), ParseFieldMatcher.STRICT)
                 .parse(testAgg.getName(), parseContext);
         assertSame(XContentParser.Token.END_OBJECT, parser.currentToken());
         assertSame(XContentParser.Token.END_OBJECT, parser.nextToken());
@@ -219,7 +219,7 @@ public abstract class BasePipelineAggregationTestCase<AF extends AbstractPipelin
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             output.writeNamedWriteable(testAgg);
             try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
-                PipelineAggregatorBuilder deserializedQuery = in.readNamedWriteable(PipelineAggregatorBuilder.class);
+                PipelineAggregationBuilder deserializedQuery = in.readNamedWriteable(PipelineAggregationBuilder.class);
                 assertEquals(deserializedQuery, testAgg);
                 assertEquals(deserializedQuery.hashCode(), testAgg.hashCode());
                 assertNotSame(deserializedQuery, testAgg);
@@ -260,7 +260,7 @@ public abstract class BasePipelineAggregationTestCase<AF extends AbstractPipelin
             output.writeNamedWriteable(agg);
             try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
                 @SuppressWarnings("unchecked")
-                AF secondAgg = (AF) in.readNamedWriteable(PipelineAggregatorBuilder.class);
+                AF secondAgg = (AF) in.readNamedWriteable(PipelineAggregationBuilder.class);
                 return secondAgg;
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptTests.java
index 61b8117a0b0..252c1247a49 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptTests.java
@@ -23,15 +23,15 @@ import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
-import org.elasticsearch.search.aggregations.pipeline.bucketscript.BucketScriptPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketscript.BucketScriptPipelineAggregationBuilder;
 
 import java.util.HashMap;
 import java.util.Map;
 
-public class BucketScriptTests extends BasePipelineAggregationTestCase<BucketScriptPipelineAggregatorBuilder> {
+public class BucketScriptTests extends BasePipelineAggregationTestCase<BucketScriptPipelineAggregationBuilder> {
 
     @Override
-    protected BucketScriptPipelineAggregatorBuilder createTestAggregatorFactory() {
+    protected BucketScriptPipelineAggregationBuilder createTestAggregatorFactory() {
         String name = randomAsciiOfLengthBetween(3, 20);
         Map<String, String> bucketsPaths = new HashMap<>();
         int numBucketPaths = randomIntBetween(1, 10);
@@ -49,7 +49,7 @@ public class BucketScriptTests extends BasePipelineAggregationTestCase<BucketScr
             }
             script = new Script("script", randomFrom(ScriptType.values()), randomFrom("my_lang", null), params);
         }
-        BucketScriptPipelineAggregatorBuilder factory = new BucketScriptPipelineAggregatorBuilder(name, bucketsPaths, script);
+        BucketScriptPipelineAggregationBuilder factory = new BucketScriptPipelineAggregationBuilder(name, bucketsPaths, script);
         if (randomBoolean()) {
             factory.format(randomAsciiOfLengthBetween(1, 10));
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorTests.java
index 0f5e91f8d03..5ffbda13f9c 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorTests.java
@@ -23,15 +23,15 @@ import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
-import org.elasticsearch.search.aggregations.pipeline.bucketselector.BucketSelectorPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketselector.BucketSelectorPipelineAggregationBuilder;
 
 import java.util.HashMap;
 import java.util.Map;
 
-public class BucketSelectorTests extends BasePipelineAggregationTestCase<BucketSelectorPipelineAggregatorBuilder> {
+public class BucketSelectorTests extends BasePipelineAggregationTestCase<BucketSelectorPipelineAggregationBuilder> {
 
     @Override
-    protected BucketSelectorPipelineAggregatorBuilder createTestAggregatorFactory() {
+    protected BucketSelectorPipelineAggregationBuilder createTestAggregatorFactory() {
         String name = randomAsciiOfLengthBetween(3, 20);
         Map<String, String> bucketsPaths = new HashMap<>();
         int numBucketPaths = randomIntBetween(1, 10);
@@ -49,7 +49,7 @@ public class BucketSelectorTests extends BasePipelineAggregationTestCase<BucketS
             }
             script = new Script("script", randomFrom(ScriptType.values()), randomFrom("my_lang", null), params);
         }
-        BucketSelectorPipelineAggregatorBuilder factory = new BucketSelectorPipelineAggregatorBuilder(name, bucketsPaths, script);
+        BucketSelectorPipelineAggregationBuilder factory = new BucketSelectorPipelineAggregationBuilder(name, bucketsPaths, script);
         if (randomBoolean()) {
             factory.gapPolicy(randomFrom(GapPolicy.values()));
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumTests.java
index 6cb9bb936e6..07cd9010ed0 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/CumulativeSumTests.java
@@ -20,15 +20,15 @@
 package org.elasticsearch.search.aggregations.pipeline;
 
 import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
-import org.elasticsearch.search.aggregations.pipeline.cumulativesum.CumulativeSumPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.cumulativesum.CumulativeSumPipelineAggregationBuilder;
 
-public class CumulativeSumTests extends BasePipelineAggregationTestCase<CumulativeSumPipelineAggregatorBuilder> {
+public class CumulativeSumTests extends BasePipelineAggregationTestCase<CumulativeSumPipelineAggregationBuilder> {
 
     @Override
-    protected CumulativeSumPipelineAggregatorBuilder createTestAggregatorFactory() {
+    protected CumulativeSumPipelineAggregationBuilder createTestAggregatorFactory() {
         String name = randomAsciiOfLengthBetween(3, 20);
         String bucketsPath = randomAsciiOfLengthBetween(3, 20);
-        CumulativeSumPipelineAggregatorBuilder factory = new CumulativeSumPipelineAggregatorBuilder(name, bucketsPath);
+        CumulativeSumPipelineAggregationBuilder factory = new CumulativeSumPipelineAggregationBuilder(name, bucketsPath);
         if (randomBoolean()) {
             factory.format(randomAsciiOfLengthBetween(1, 10));
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeTests.java
index 9ceac7b73a2..678bb0b5c4d 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeTests.java
@@ -21,15 +21,15 @@ package org.elasticsearch.search.aggregations.pipeline;
 
 import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
-import org.elasticsearch.search.aggregations.pipeline.derivative.DerivativePipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.derivative.DerivativePipelineAggregationBuilder;
 
-public class DerivativeTests extends BasePipelineAggregationTestCase<DerivativePipelineAggregatorBuilder> {
+public class DerivativeTests extends BasePipelineAggregationTestCase<DerivativePipelineAggregationBuilder> {
 
     @Override
-    protected DerivativePipelineAggregatorBuilder createTestAggregatorFactory() {
+    protected DerivativePipelineAggregationBuilder createTestAggregatorFactory() {
         String name = randomAsciiOfLengthBetween(3, 20);
         String bucketsPath = randomAsciiOfLengthBetween(3, 20);
-        DerivativePipelineAggregatorBuilder factory = new DerivativePipelineAggregatorBuilder(name, bucketsPath);
+        DerivativePipelineAggregationBuilder factory = new DerivativePipelineAggregationBuilder(name, bucketsPath);
         if (randomBoolean()) {
             factory.format(randomAsciiOfLengthBetween(1, 10));
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDifferenceTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDifferenceTests.java
index f7449192d5e..01b31743828 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDifferenceTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/SerialDifferenceTests.java
@@ -21,15 +21,15 @@ package org.elasticsearch.search.aggregations.pipeline;
 
 import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
-import org.elasticsearch.search.aggregations.pipeline.serialdiff.SerialDiffPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.serialdiff.SerialDiffPipelineAggregationBuilder;
 
-public class SerialDifferenceTests extends BasePipelineAggregationTestCase<SerialDiffPipelineAggregatorBuilder> {
+public class SerialDifferenceTests extends BasePipelineAggregationTestCase<SerialDiffPipelineAggregationBuilder> {
 
     @Override
-    protected SerialDiffPipelineAggregatorBuilder createTestAggregatorFactory() {
+    protected SerialDiffPipelineAggregationBuilder createTestAggregatorFactory() {
         String name = randomAsciiOfLengthBetween(3, 20);
         String bucketsPath = randomAsciiOfLengthBetween(3, 20);
-        SerialDiffPipelineAggregatorBuilder factory = new SerialDiffPipelineAggregatorBuilder(name, bucketsPath);
+        SerialDiffPipelineAggregationBuilder factory = new SerialDiffPipelineAggregationBuilder(name, bucketsPath);
         if (randomBoolean()) {
             factory.format(randomAsciiOfLengthBetween(1, 10));
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/AbstractBucketMetricsTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/AbstractBucketMetricsTestCase.java
index 20c4a614ba2..13ed4027b3a 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/AbstractBucketMetricsTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/AbstractBucketMetricsTestCase.java
@@ -22,7 +22,7 @@ package org.elasticsearch.search.aggregations.pipeline.bucketmetrics;
 import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 
-public abstract class AbstractBucketMetricsTestCase<PAF extends BucketMetricsPipelineAggregatorBuilder<PAF>>
+public abstract class AbstractBucketMetricsTestCase<PAF extends BucketMetricsPipelineAggregationBuilder<PAF>>
         extends BasePipelineAggregationTestCase<PAF> {
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/AvgBucketTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/AvgBucketTests.java
index 86bf5d92aca..5ae06f671f1 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/AvgBucketTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/AvgBucketTests.java
@@ -19,13 +19,13 @@
 
 package org.elasticsearch.search.aggregations.pipeline.bucketmetrics;
 
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.avg.AvgBucketPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.avg.AvgBucketPipelineAggregationBuilder;
 
-public class AvgBucketTests extends AbstractBucketMetricsTestCase<AvgBucketPipelineAggregatorBuilder> {
+public class AvgBucketTests extends AbstractBucketMetricsTestCase<AvgBucketPipelineAggregationBuilder> {
 
     @Override
-    protected AvgBucketPipelineAggregatorBuilder doCreateTestAggregatorFactory(String name, String bucketsPath) {
-        return new AvgBucketPipelineAggregatorBuilder(name, bucketsPath);
+    protected AvgBucketPipelineAggregationBuilder doCreateTestAggregatorFactory(String name, String bucketsPath) {
+        return new AvgBucketPipelineAggregationBuilder(name, bucketsPath);
     }
 
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java
index 65e8156f71b..e1441b0dc54 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java
@@ -23,15 +23,15 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregationBuilder;
 
 import static org.hamcrest.Matchers.equalTo;
 
-public class ExtendedStatsBucketTests extends AbstractBucketMetricsTestCase<ExtendedStatsBucketPipelineAggregatorBuilder> {
+public class ExtendedStatsBucketTests extends AbstractBucketMetricsTestCase<ExtendedStatsBucketPipelineAggregationBuilder> {
 
     @Override
-    protected ExtendedStatsBucketPipelineAggregatorBuilder doCreateTestAggregatorFactory(String name, String bucketsPath) {
-        ExtendedStatsBucketPipelineAggregatorBuilder factory = new ExtendedStatsBucketPipelineAggregatorBuilder(name, bucketsPath);
+    protected ExtendedStatsBucketPipelineAggregationBuilder doCreateTestAggregatorFactory(String name, String bucketsPath) {
+        ExtendedStatsBucketPipelineAggregationBuilder factory = new ExtendedStatsBucketPipelineAggregationBuilder(name, bucketsPath);
         if (randomBoolean()) {
             factory.sigma(randomDoubleBetween(0.0, 10.0, false));
         }
@@ -50,7 +50,7 @@ public class ExtendedStatsBucketTests extends AbstractBucketMetricsTestCase<Exte
         QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
         parser.nextToken(); // skip object start
 
-        ExtendedStatsBucketPipelineAggregatorBuilder builder = (ExtendedStatsBucketPipelineAggregatorBuilder) aggParsers
+        ExtendedStatsBucketPipelineAggregationBuilder builder = (ExtendedStatsBucketPipelineAggregationBuilder) aggParsers
             .pipelineParser(ExtendedStatsBucketPipelineAggregator.TYPE.name(), parseFieldMatcher)
             .parse("test", parseContext);
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/MaxBucketTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/MaxBucketTests.java
index b7a531de748..43115c668a0 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/MaxBucketTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/MaxBucketTests.java
@@ -19,13 +19,13 @@
 
 package org.elasticsearch.search.aggregations.pipeline.bucketmetrics;
 
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.max.MaxBucketPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.max.MaxBucketPipelineAggregationBuilder;
 
-public class MaxBucketTests extends AbstractBucketMetricsTestCase<MaxBucketPipelineAggregatorBuilder> {
+public class MaxBucketTests extends AbstractBucketMetricsTestCase<MaxBucketPipelineAggregationBuilder> {
 
     @Override
-    protected MaxBucketPipelineAggregatorBuilder doCreateTestAggregatorFactory(String name, String bucketsPath) {
-        return new MaxBucketPipelineAggregatorBuilder(name, bucketsPath);
+    protected MaxBucketPipelineAggregationBuilder doCreateTestAggregatorFactory(String name, String bucketsPath) {
+        return new MaxBucketPipelineAggregationBuilder(name, bucketsPath);
     }
 
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/MinBucketTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/MinBucketTests.java
index 4e0befeb4a5..e577e733b4e 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/MinBucketTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/MinBucketTests.java
@@ -19,13 +19,13 @@
 
 package org.elasticsearch.search.aggregations.pipeline.bucketmetrics;
 
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.min.MinBucketPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.min.MinBucketPipelineAggregationBuilder;
 
-public class MinBucketTests extends AbstractBucketMetricsTestCase<MinBucketPipelineAggregatorBuilder> {
+public class MinBucketTests extends AbstractBucketMetricsTestCase<MinBucketPipelineAggregationBuilder> {
 
     @Override
-    protected MinBucketPipelineAggregatorBuilder doCreateTestAggregatorFactory(String name, String bucketsPath) {
-        return new MinBucketPipelineAggregatorBuilder(name, bucketsPath);
+    protected MinBucketPipelineAggregationBuilder doCreateTestAggregatorFactory(String name, String bucketsPath) {
+        return new MinBucketPipelineAggregationBuilder(name, bucketsPath);
     }
 
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/PercentilesBucketTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/PercentilesBucketTests.java
index 236f122bdd0..e7c478b23f2 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/PercentilesBucketTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/PercentilesBucketTests.java
@@ -23,15 +23,15 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.PercentilesBucketPipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.PercentilesBucketPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.PercentilesBucketPipelineAggregationBuilder;
 
 import static org.hamcrest.Matchers.equalTo;
 
-public class PercentilesBucketTests extends AbstractBucketMetricsTestCase<PercentilesBucketPipelineAggregatorBuilder> {
+public class PercentilesBucketTests extends AbstractBucketMetricsTestCase<PercentilesBucketPipelineAggregationBuilder> {
 
     @Override
-    protected PercentilesBucketPipelineAggregatorBuilder doCreateTestAggregatorFactory(String name, String bucketsPath) {
-        PercentilesBucketPipelineAggregatorBuilder factory = new PercentilesBucketPipelineAggregatorBuilder(name, bucketsPath);
+    protected PercentilesBucketPipelineAggregationBuilder doCreateTestAggregatorFactory(String name, String bucketsPath) {
+        PercentilesBucketPipelineAggregationBuilder factory = new PercentilesBucketPipelineAggregationBuilder(name, bucketsPath);
         if (randomBoolean()) {
             int numPercents = randomIntBetween(1, 20);
             double[] percents = new double[numPercents];
@@ -55,7 +55,7 @@ public class PercentilesBucketTests extends AbstractBucketMetricsTestCase<Percen
         QueryParseContext parseContext = new QueryParseContext(queriesRegistry, parser, parseFieldMatcher);
         parser.nextToken(); // skip object start
 
-        PercentilesBucketPipelineAggregatorBuilder builder = (PercentilesBucketPipelineAggregatorBuilder) aggParsers
+        PercentilesBucketPipelineAggregationBuilder builder = (PercentilesBucketPipelineAggregationBuilder) aggParsers
             .pipelineParser(PercentilesBucketPipelineAggregator.TYPE.name(), parseFieldMatcher)
             .parse("test", parseContext);
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/StatsBucketTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/StatsBucketTests.java
index f06d6621003..2be01b3fd68 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/StatsBucketTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/StatsBucketTests.java
@@ -19,14 +19,14 @@
 
 package org.elasticsearch.search.aggregations.pipeline.bucketmetrics;
 
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.StatsBucketPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.StatsBucketPipelineAggregationBuilder;
 
-public class StatsBucketTests extends AbstractBucketMetricsTestCase<StatsBucketPipelineAggregatorBuilder> {
+public class StatsBucketTests extends AbstractBucketMetricsTestCase<StatsBucketPipelineAggregationBuilder> {
 
     @Override
-    protected StatsBucketPipelineAggregatorBuilder doCreateTestAggregatorFactory(String name,
+    protected StatsBucketPipelineAggregationBuilder doCreateTestAggregatorFactory(String name,
             String bucketsPath) {
-        return new StatsBucketPipelineAggregatorBuilder(name, bucketsPath);
+        return new StatsBucketPipelineAggregationBuilder(name, bucketsPath);
     }
 
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/SumBucketTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/SumBucketTests.java
index 048d26bb612..01ce2277058 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/SumBucketTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/SumBucketTests.java
@@ -19,13 +19,13 @@
 
 package org.elasticsearch.search.aggregations.pipeline.bucketmetrics;
 
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.sum.SumBucketPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.sum.SumBucketPipelineAggregationBuilder;
 
-public class SumBucketTests extends AbstractBucketMetricsTestCase<SumBucketPipelineAggregatorBuilder> {
+public class SumBucketTests extends AbstractBucketMetricsTestCase<SumBucketPipelineAggregationBuilder> {
 
     @Override
-    protected SumBucketPipelineAggregatorBuilder doCreateTestAggregatorFactory(String name, String bucketsPath) {
-        return new SumBucketPipelineAggregatorBuilder(name, bucketsPath);
+    protected SumBucketPipelineAggregationBuilder doCreateTestAggregatorFactory(String name, String bucketsPath) {
+        return new SumBucketPipelineAggregationBuilder(name, bucketsPath);
     }
 
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgTests.java
index 8aa294e987f..e34a56c496a 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/moving/avg/MovAvgTests.java
@@ -24,9 +24,9 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase;
-import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
-import org.elasticsearch.search.aggregations.pipeline.movavg.MovAvgPipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.pipeline.movavg.MovAvgPipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.EwmaModel;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.HoltLinearModel;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.HoltWintersModel;
@@ -34,13 +34,13 @@ import org.elasticsearch.search.aggregations.pipeline.movavg.models.HoltWintersM
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.LinearModel;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.SimpleModel;;
 
-public class MovAvgTests extends BasePipelineAggregationTestCase<MovAvgPipelineAggregatorBuilder> {
+public class MovAvgTests extends BasePipelineAggregationTestCase<MovAvgPipelineAggregationBuilder> {
 
     @Override
-    protected MovAvgPipelineAggregatorBuilder createTestAggregatorFactory() {
+    protected MovAvgPipelineAggregationBuilder createTestAggregatorFactory() {
         String name = randomAsciiOfLengthBetween(3, 20);
         String bucketsPath = randomAsciiOfLengthBetween(3, 20);
-        MovAvgPipelineAggregatorBuilder factory = new MovAvgPipelineAggregatorBuilder(name, bucketsPath);
+        MovAvgPipelineAggregationBuilder factory = new MovAvgPipelineAggregationBuilder(name, bucketsPath);
         if (randomBoolean()) {
             factory.format(randomAsciiOfLengthBetween(1, 10));
         }
@@ -98,7 +98,7 @@ public class MovAvgTests extends BasePipelineAggregationTestCase<MovAvgPipelineA
     }
 
     public void testDefaultParsing() throws Exception {
-        MovAvgPipelineAggregatorBuilder expected = new MovAvgPipelineAggregatorBuilder("commits_moving_avg", "commits");
+        MovAvgPipelineAggregationBuilder expected = new MovAvgPipelineAggregationBuilder("commits_moving_avg", "commits");
         String json = "{" +
             "    \"commits_moving_avg\": {" +
             "        \"moving_avg\": {" +
@@ -115,7 +115,7 @@ public class MovAvgTests extends BasePipelineAggregationTestCase<MovAvgPipelineA
         assertSame(XContentParser.Token.FIELD_NAME, parser.nextToken());
         assertEquals(expected.type(), parser.currentName());
         assertSame(XContentParser.Token.START_OBJECT, parser.nextToken());
-        PipelineAggregatorBuilder newAgg = aggParsers.pipelineParser(expected.getWriteableName(), ParseFieldMatcher.STRICT)
+        PipelineAggregationBuilder newAgg = aggParsers.pipelineParser(expected.getWriteableName(), ParseFieldMatcher.STRICT)
                 .parse(expected.getName(), parseContext);
         assertSame(XContentParser.Token.END_OBJECT, parser.currentToken());
         assertSame(XContentParser.Token.END_OBJECT, parser.nextToken());
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequestBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequestBuilder.java
index 36eca70049a..e1e71c8e0e3 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequestBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateRequestBuilder.java
@@ -28,7 +28,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.highlight.HighlightBuilder;
 import org.elasticsearch.search.sort.SortBuilder;
 
@@ -177,9 +177,9 @@ public class PercolateRequestBuilder extends ActionRequestBuilder<PercolateReque
 
     /**
      * Delegates to
-     * {@link PercolateSourceBuilder#addAggregation(PipelineAggregatorBuilder)}
+     * {@link PercolateSourceBuilder#addAggregation(PipelineAggregationBuilder)}
      */
-    public PercolateRequestBuilder addAggregation(PipelineAggregatorBuilder aggregationBuilder) {
+    public PercolateRequestBuilder addAggregation(PipelineAggregationBuilder aggregationBuilder) {
         sourceBuilder().addAggregation(aggregationBuilder);
         return this;
     }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateSourceBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateSourceBuilder.java
index 8de722260f0..e61f9d68dab 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateSourceBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateSourceBuilder.java
@@ -30,7 +30,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.aggregations.PipelineAggregatorBuilder;
+import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.highlight.HighlightBuilder;
 import org.elasticsearch.search.sort.ScoreSortBuilder;
@@ -57,7 +57,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes {
     private Boolean trackScores;
     private HighlightBuilder highlightBuilder;
     private List<AggregationBuilder> aggregationBuilders;
-    private List<PipelineAggregatorBuilder> pipelineAggregationBuilders;
+    private List<PipelineAggregationBuilder> pipelineAggregationBuilders;
 
     /**
      * Sets the document to run the percolate queries against.
@@ -140,7 +140,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes {
     /**
      * Add an aggregation definition.
      */
-    public PercolateSourceBuilder addAggregation(PipelineAggregatorBuilder aggregationBuilder) {
+    public PercolateSourceBuilder addAggregation(PipelineAggregationBuilder aggregationBuilder) {
         if (pipelineAggregationBuilders == null) {
             pipelineAggregationBuilders = new ArrayList<>();
         }
@@ -183,7 +183,7 @@ public class PercolateSourceBuilder extends ToXContentToBytes {
                 }
             }
             if (pipelineAggregationBuilders != null) {
-                for (PipelineAggregatorBuilder aggregation : pipelineAggregationBuilders) {
+                for (PipelineAggregationBuilder aggregation : pipelineAggregationBuilders) {
                     aggregation.toXContent(builder, params);
                 }
             }

From 755721953bf5222ac4784572a1377d28825ef9f5 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Fri, 17 Jun 2016 10:43:13 +0200
Subject: [PATCH 0927/1311] Remove support for sorting terms aggregation by
 ascending count

closes #17614
---
 .../bucket/terms/InternalOrder.java           | 19 +---
 .../bucket/terms/InternalTerms.java           |  2 +-
 .../aggregations/bucket/terms/Terms.java      |  6 +-
 .../bucket/terms/TermsAggregationBuilder.java |  2 +-
 .../bucket/terms/TermsParser.java             | 20 +++-
 .../aggregations/bucket/ShardSizeTermsIT.java | 24 ++---
 .../bucket/TermsDocCountErrorIT.java          | 93 -------------------
 .../aggregations/bucket/TermsTests.java       |  2 +-
 .../bucket/terms-aggregation.asciidoc         | 13 ++-
 .../migrate_5_0/aggregations.asciidoc         |  4 +
 .../messy/tests/DoubleTermsTests.java         |  7 +-
 .../messy/tests/LongTermsTests.java           |  7 +-
 .../messy/tests/MinDocCountTests.java         | 48 ++--------
 .../messy/tests/StringTermsTests.java         |  7 +-
 .../test/search.aggregation/20_terms.yaml     | 15 ++-
 15 files changed, 75 insertions(+), 194 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalOrder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalOrder.java
index f3f87c09dca..5ebeb6ee7fb 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalOrder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalOrder.java
@@ -46,9 +46,8 @@ import java.util.Objects;
 class InternalOrder extends Terms.Order {
 
     private static final byte COUNT_DESC_ID = 1;
-    private static final byte COUNT_ASC_ID = 2;
-    private static final byte TERM_DESC_ID = 3;
-    private static final byte TERM_ASC_ID = 4;
+    private static final byte TERM_DESC_ID = 2;
+    private static final byte TERM_ASC_ID = 3;
 
     /**
      * Order by the (higher) count of each term.
@@ -60,17 +59,6 @@ class InternalOrder extends Terms.Order {
         }
     });
 
-    /**
-     * Order by the (lower) count of each term.
-     */
-    public static final InternalOrder COUNT_ASC = new InternalOrder(COUNT_ASC_ID, "_count", true, new Comparator<Terms.Bucket>() {
-
-        @Override
-        public int compare(Terms.Bucket o1, Terms.Bucket o2) {
-            return Long.compare(o1.getDocCount(), o2.getDocCount());
-        }
-    });
-
     /**
      * Order by the terms.
      */
@@ -93,7 +81,7 @@ class InternalOrder extends Terms.Order {
         }
     });
 
-    public static boolean isCountDesc(Terms.Order order) {
+    public static boolean isCountOrder(Terms.Order order) {
         if (order == COUNT_DESC) {
             return true;
         } else if (order instanceof CompoundOrder) {
@@ -348,7 +336,6 @@ class InternalOrder extends Terms.Order {
             byte id = in.readByte();
             switch (id) {
                 case COUNT_DESC_ID: return absoluteOrder ? new CompoundOrder(Collections.singletonList((Terms.Order) InternalOrder.COUNT_DESC)) : InternalOrder.COUNT_DESC;
-                case COUNT_ASC_ID: return absoluteOrder ? new CompoundOrder(Collections.singletonList((Terms.Order) InternalOrder.COUNT_ASC)) : InternalOrder.COUNT_ASC;
                 case TERM_DESC_ID: return InternalOrder.TERM_DESC;
                 case TERM_ASC_ID: return InternalOrder.TERM_ASC;
                 case Aggregation.ID:
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
index fcbd66dced6..5784066f85c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
@@ -192,7 +192,7 @@ public abstract class InternalTerms<A extends InternalTerms, B extends InternalT
             final long thisAggDocCountError;
             if (terms.buckets.size() < this.shardSize || InternalOrder.isTermOrder(order)) {
                 thisAggDocCountError = 0;
-            } else if (InternalOrder.isCountDesc(this.order)) {
+            } else if (InternalOrder.isCountOrder(this.order)) {
                 thisAggDocCountError = terms.buckets.get(terms.buckets.size() - 1).docCount;
             } else {
                 thisAggDocCountError = -1;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/Terms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/Terms.java
index 16fb7f1d81e..3dd186c80f6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/Terms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/Terms.java
@@ -100,10 +100,10 @@ public interface Terms extends MultiBucketsAggregation {
     static abstract class Order implements ToXContent {
 
         /**
-         * @return a bucket ordering strategy that sorts buckets by their document counts (ascending or descending)
+         * @return a bucket ordering strategy that sorts buckets by their document counts (descending)
          */
-        public static Order count(boolean asc) {
-            return asc ? InternalOrder.COUNT_ASC : InternalOrder.COUNT_DESC;
+        public static Order count() {
+            return InternalOrder.COUNT_DESC;
         }
 
         /**
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
index 87567d64df4..e64a89e3a0f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
@@ -53,7 +53,7 @@ public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder<Valu
     public static final ParseField SHOW_TERM_DOC_COUNT_ERROR = new ParseField("show_term_doc_count_error");
     public static final ParseField ORDER_FIELD = new ParseField("order");
 
-    private Terms.Order order = Terms.Order.compound(Terms.Order.count(false), Terms.Order.term(true));
+    private Terms.Order order = Terms.Order.compound(Terms.Order.count(), Terms.Order.term(true));
     private IncludeExclude includeExclude = null;
     private String executionHint = null;
     private SubAggCollectionMode collectMode = null;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsParser.java
index 2a67dbe2218..7b61abcdccd 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsParser.java
@@ -77,9 +77,19 @@ public class TermsParser extends AbstractTermsParser {
     @Override
     public boolean parseSpecial(String aggregationName, XContentParser parser, ParseFieldMatcher parseFieldMatcher, Token token,
             String currentFieldName, Map<ParseField, Object> otherOptions) throws IOException {
-        if (token == XContentParser.Token.START_OBJECT) {
+        if (token == Token.VALUE_STRING) {
             if (parseFieldMatcher.match(currentFieldName, TermsAggregationBuilder.ORDER_FIELD)) {
-                otherOptions.put(TermsAggregationBuilder.ORDER_FIELD, Collections.singletonList(parseOrderParam(aggregationName, parser)));
+                if ("_count".equals(parser.text())) {
+                    otherOptions.put(TermsAggregationBuilder.ORDER_FIELD,
+                        Collections.singletonList(new OrderElement(parser.text(), false)));
+                    return true;
+                }
+            }
+            return false;
+        } else if (token == XContentParser.Token.START_OBJECT) {
+            if (parseFieldMatcher.match(currentFieldName, TermsAggregationBuilder.ORDER_FIELD)) {
+                otherOptions.put(TermsAggregationBuilder.ORDER_FIELD,
+                    Collections.singletonList(parseOrderParam(aggregationName, parser)));
                 return true;
             }
         } else if (token == XContentParser.Token.START_ARRAY) {
@@ -117,6 +127,10 @@ public class TermsParser extends AbstractTermsParser {
             } else if (token == XContentParser.Token.VALUE_STRING) {
                 String dir = parser.text();
                 if ("asc".equalsIgnoreCase(dir)) {
+                    if ("_count".equals(orderKey)) {
+                        throw new ParsingException(parser.getTokenLocation(),
+                            "Sort by ascending _count is not supported in [" + aggregationName + "].");
+                    }
                     orderAsc = true;
                 } else if ("desc".equalsIgnoreCase(dir)) {
                     orderAsc = false;
@@ -167,7 +181,7 @@ public class TermsParser extends AbstractTermsParser {
             return Order.term(asc);
         }
         if ("_count".equals(key)) {
-            return Order.count(asc);
+            return Order.count();
         }
         return Order.aggregation(key, asc);
     }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java
index ecc16b85f13..82b60289c75 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java
@@ -39,7 +39,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count(false)))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count()))
                 .execute().actionGet();
 
         Terms  terms = response.getAggregations().get("keys");
@@ -62,7 +62,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3).shardSize(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count(false)))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count()))
                 .execute().actionGet();
 
         Terms  terms = response.getAggregations().get("keys");
@@ -86,7 +86,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count(false)))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count()))
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
@@ -110,7 +110,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type").setRouting(routing1)
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count(false)))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count()))
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
@@ -156,7 +156,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count(false)))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count()))
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
@@ -179,7 +179,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3).shardSize(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count(false)))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count()))
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
@@ -202,7 +202,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count(false)))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count()))
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
@@ -226,7 +226,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type").setRouting(routing1)
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count(false)))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count()))
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
@@ -272,7 +272,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count(false)))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count()))
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
@@ -295,7 +295,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3).shardSize(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count(false)))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count()))
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
@@ -318,7 +318,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count(false)))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count()))
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
@@ -341,7 +341,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type").setRouting(routing1)
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count(false)))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count()))
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java
index 6bbe1dc22b2..00c73b42246 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java
@@ -271,37 +271,6 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
         assertNoDocCountErrorSingleResponse(size, testResponse);
     }
 
-    public void testStringValueFieldDocCountAsc() throws Exception {
-        int size = randomIntBetween(1, 20);
-        int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
-                .addAggregation(terms("terms")
-                        .executionHint(randomExecutionHint())
-                        .field(STRING_FIELD_NAME)
-                        .showTermDocCountError(true)
-                        .size(10000).shardSize(10000)
-                        .order(Order.count(true))
-                        .collectMode(randomFrom(SubAggCollectionMode.values())))
-                .execute().actionGet();
-
-        assertSearchResponse(accurateResponse);
-
-        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
-                .addAggregation(terms("terms")
-                        .executionHint(randomExecutionHint())
-                        .field(STRING_FIELD_NAME)
-                        .showTermDocCountError(true)
-                        .size(size)
-                        .shardSize(shardSize)
-                        .order(Order.count(true))
-                        .collectMode(randomFrom(SubAggCollectionMode.values())))
-                .execute().actionGet();
-
-        assertSearchResponse(testResponse);
-
-        assertUnboundedDocCountError(size, accurateResponse, testResponse);
-    }
-
     public void testStringValueFieldTermSortAsc() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
@@ -507,37 +476,6 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
         assertNoDocCountErrorSingleResponse(size, testResponse);
     }
 
-    public void testLongValueFieldDocCountAsc() throws Exception {
-        int size = randomIntBetween(1, 20);
-        int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
-                .addAggregation(terms("terms")
-                        .executionHint(randomExecutionHint())
-                        .field(LONG_FIELD_NAME)
-                        .showTermDocCountError(true)
-                        .size(10000).shardSize(10000)
-                        .order(Order.count(true))
-                        .collectMode(randomFrom(SubAggCollectionMode.values())))
-                .execute().actionGet();
-
-        assertSearchResponse(accurateResponse);
-
-        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
-                .addAggregation(terms("terms")
-                        .executionHint(randomExecutionHint())
-                        .field(LONG_FIELD_NAME)
-                        .showTermDocCountError(true)
-                        .size(size)
-                        .shardSize(shardSize)
-                        .order(Order.count(true))
-                        .collectMode(randomFrom(SubAggCollectionMode.values())))
-                .execute().actionGet();
-
-        assertSearchResponse(testResponse);
-
-        assertUnboundedDocCountError(size, accurateResponse, testResponse);
-    }
-
     public void testLongValueFieldTermSortAsc() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
@@ -743,37 +681,6 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
         assertNoDocCountErrorSingleResponse(size, testResponse);
     }
 
-    public void testDoubleValueFieldDocCountAsc() throws Exception {
-        int size = randomIntBetween(1, 20);
-        int shardSize = randomIntBetween(size, size * 2);
-        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
-                .addAggregation(terms("terms")
-                        .executionHint(randomExecutionHint())
-                        .field(DOUBLE_FIELD_NAME)
-                        .showTermDocCountError(true)
-                        .size(10000).shardSize(10000)
-                        .order(Order.count(true))
-                        .collectMode(randomFrom(SubAggCollectionMode.values())))
-                .execute().actionGet();
-
-        assertSearchResponse(accurateResponse);
-
-        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
-                .addAggregation(terms("terms")
-                        .executionHint(randomExecutionHint())
-                        .field(DOUBLE_FIELD_NAME)
-                        .showTermDocCountError(true)
-                        .size(size)
-                        .shardSize(shardSize)
-                        .order(Order.count(true))
-                        .collectMode(randomFrom(SubAggCollectionMode.values())))
-                .execute().actionGet();
-
-        assertSearchResponse(testResponse);
-
-        assertUnboundedDocCountError(size, accurateResponse, testResponse);
-    }
-
     public void testDoubleValueFieldTermSortAsc() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsTests.java
index c8a14e82bf4..055258bb4cc 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsTests.java
@@ -180,7 +180,7 @@ public class TermsTests extends BaseAggregationTestCase<TermsAggregationBuilder>
             orders.add(Terms.Order.term(randomBoolean()));
             break;
         case 1:
-            orders.add(Terms.Order.count(randomBoolean()));
+            orders.add(Terms.Order.count());
             break;
         case 2:
             orders.add(Terms.Order.aggregation(randomAsciiOfLengthBetween(3, 20), randomBoolean()));
diff --git a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
index 3c1f4ae860a..30e3d649179 100644
--- a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
@@ -239,9 +239,7 @@ determined and is given a value of -1 to indicate this.
 ==== Order
 
 The order of the buckets can be customized by setting the `order` parameter. By default, the buckets are ordered by
-their `doc_count` descending. It is also possible to change this behaviour as follows:
-
-Ordering the buckets by their `doc_count` in an ascending manner:
+their `doc_count` descending:
 
 [source,js]
 --------------------------------------------------
@@ -250,12 +248,15 @@ Ordering the buckets by their `doc_count` in an ascending manner:
         "genders" : {
             "terms" : {
                 "field" : "gender",
-                "order" : { "_count" : "asc" }
+                "order" : "_count" <1>
             }
         }
     }
 }
 --------------------------------------------------
+<1> This is the default if the order is missing.
+
+It is also possible to change this behaviour as follows:
 
 Ordering the buckets alphabetically by their terms in an ascending manner:
 
@@ -312,7 +313,9 @@ Ordering the buckets by multi value metrics sub-aggregation (identified by the a
 }
 --------------------------------------------------
 
-WARNING: Sorting by ascending `_count` or by sub aggregation is discouraged as it increases the
+Ordering the buckets by their `doc_count` in an ascending manner is not supported.
+
+WARNING: Sorting by sub aggregation is discouraged as it increases the
 <<search-aggregations-bucket-terms-aggregation-approximate-counts,error>> on document counts.
 It is fine when a single shard is queried, or when the field that is being aggregated was used
 as a routing key at index time: in these cases results will be accurate since shards have disjoint
diff --git a/docs/reference/migration/migrate_5_0/aggregations.asciidoc b/docs/reference/migration/migrate_5_0/aggregations.asciidoc
index d9227e91385..57afd44719b 100644
--- a/docs/reference/migration/migrate_5_0/aggregations.asciidoc
+++ b/docs/reference/migration/migrate_5_0/aggregations.asciidoc
@@ -26,3 +26,7 @@ for `from` and `to` anymore.
 `size: 0` is no longer valid for the terms, significant terms and geohash grid
 aggregations. Instead a size should be explicitly specified with a number greater
 than zero.
+
+==== `order: {"count": "asc"}` on Terms
+
+The ability to sort the terms aggregation by ascending count has been removed.
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java
index 6ed77fffcc8..5e1dec0402f 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java
@@ -1048,11 +1048,6 @@ public class DoubleTermsTests extends AbstractTermsTestCase {
         assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("avg_l", false), Terms.Order.term(true));
     }
 
-    public void testSingleValuedFieldOrderedByCountAscAndSingleValueSubAggregationAsc() throws Exception {
-        double[] expectedKeys = new double[] { 6, 7, 3, 4, 5, 1, 2 };
-        assertMultiSortResponse(expectedKeys, Terms.Order.count(true), Terms.Order.aggregation("avg_l", true));
-    }
-
     public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscSingleValueSubAggregationAsc() throws Exception {
         double[] expectedKeys = new double[] { 6, 7, 3, 5, 4, 1, 2 };
         assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("sum_d", true), Terms.Order.aggregation("avg_l", true));
@@ -1060,7 +1055,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase {
 
     public void testSingleValuedFieldOrderedByThreeCriteria() throws Exception {
         double[] expectedKeys = new double[] { 2, 1, 4, 5, 3, 6, 7 };
-        assertMultiSortResponse(expectedKeys, Terms.Order.count(false), Terms.Order.aggregation("sum_d", false), Terms.Order.aggregation("avg_l", false));
+        assertMultiSortResponse(expectedKeys, Terms.Order.count(), Terms.Order.aggregation("sum_d", false), Terms.Order.aggregation("avg_l", false));
     }
 
     public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscAsCompound() throws Exception {
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java
index fa049429a3a..efc9833a33d 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java
@@ -1025,11 +1025,6 @@ public class LongTermsTests extends AbstractTermsTestCase {
         assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("avg_l", false), Terms.Order.term(true));
     }
 
-    public void testSingleValuedFieldOrderedByCountAscAndSingleValueSubAggregationAsc() throws Exception {
-        long[] expectedKeys = new long[] { 6, 7, 3, 4, 5, 1, 2 };
-        assertMultiSortResponse(expectedKeys, Terms.Order.count(true), Terms.Order.aggregation("avg_l", true));
-    }
-
     public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscSingleValueSubAggregationAsc() throws Exception {
         long[] expectedKeys = new long[] { 6, 7, 3, 5, 4, 1, 2 };
         assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("sum_d", true), Terms.Order.aggregation("avg_l", true));
@@ -1037,7 +1032,7 @@ public class LongTermsTests extends AbstractTermsTestCase {
 
     public void testSingleValuedFieldOrderedByThreeCriteria() throws Exception {
         long[] expectedKeys = new long[] { 2, 1, 4, 5, 3, 6, 7 };
-        assertMultiSortResponse(expectedKeys, Terms.Order.count(false), Terms.Order.aggregation("sum_d", false), Terms.Order.aggregation("avg_l", false));
+        assertMultiSortResponse(expectedKeys, Terms.Order.count(), Terms.Order.aggregation("sum_d", false), Terms.Order.aggregation("avg_l", false));
     }
 
     public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscAsCompound() throws Exception {
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java
index 662d4d2f30c..c24551ed588 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java
@@ -173,36 +173,20 @@ public class MinDocCountTests extends AbstractTermsTestCase {
         testMinDocCountOnTerms("s", Script.YES, Terms.Order.term(false));
     }
 
-    public void testStringCountAsc() throws Exception {
-        testMinDocCountOnTerms("s", Script.NO, Terms.Order.count(true));
-    }
-
-    public void testStringScriptCountAsc() throws Exception {
-        testMinDocCountOnTerms("s", Script.YES, Terms.Order.count(true));
-    }
-
     public void testStringCountDesc() throws Exception {
-        testMinDocCountOnTerms("s", Script.NO, Terms.Order.count(false));
+        testMinDocCountOnTerms("s", Script.NO, Terms.Order.count());
     }
 
     public void testStringScriptCountDesc() throws Exception {
-        testMinDocCountOnTerms("s", Script.YES, Terms.Order.count(false));
-    }
-
-    public void testStringCountAscWithInclude() throws Exception {
-        testMinDocCountOnTerms("s", Script.NO, Terms.Order.count(true), ".*a.*", true);
-    }
-
-    public void testStringScriptCountAscWithInclude() throws Exception {
-        testMinDocCountOnTerms("s", Script.YES, Terms.Order.count(true), ".*a.*", true);
+        testMinDocCountOnTerms("s", Script.YES, Terms.Order.count());
     }
 
     public void testStringCountDescWithInclude() throws Exception {
-        testMinDocCountOnTerms("s", Script.NO, Terms.Order.count(false), ".*a.*", true);
+        testMinDocCountOnTerms("s", Script.NO, Terms.Order.count(), ".*a.*", true);
     }
 
     public void testStringScriptCountDescWithInclude() throws Exception {
-        testMinDocCountOnTerms("s", Script.YES, Terms.Order.count(false), ".*a.*", true);
+        testMinDocCountOnTerms("s", Script.YES, Terms.Order.count(), ".*a.*", true);
     }
 
     public void testLongTermAsc() throws Exception {
@@ -221,20 +205,12 @@ public class MinDocCountTests extends AbstractTermsTestCase {
         testMinDocCountOnTerms("l", Script.YES, Terms.Order.term(false));
     }
 
-    public void testLongCountAsc() throws Exception {
-        testMinDocCountOnTerms("l", Script.NO, Terms.Order.count(true));
-    }
-
-    public void testLongScriptCountAsc() throws Exception {
-        testMinDocCountOnTerms("l", Script.YES, Terms.Order.count(true));
-    }
-
     public void testLongCountDesc() throws Exception {
-        testMinDocCountOnTerms("l", Script.NO, Terms.Order.count(false));
+        testMinDocCountOnTerms("l", Script.NO, Terms.Order.count());
     }
 
     public void testLongScriptCountDesc() throws Exception {
-        testMinDocCountOnTerms("l", Script.YES, Terms.Order.count(false));
+        testMinDocCountOnTerms("l", Script.YES, Terms.Order.count());
     }
 
     public void testDoubleTermAsc() throws Exception {
@@ -253,20 +229,12 @@ public class MinDocCountTests extends AbstractTermsTestCase {
         testMinDocCountOnTerms("d", Script.YES, Terms.Order.term(false));
     }
 
-    public void testDoubleCountAsc() throws Exception {
-        testMinDocCountOnTerms("d", Script.NO, Terms.Order.count(true));
-    }
-
-    public void testDoubleScriptCountAsc() throws Exception {
-        testMinDocCountOnTerms("d", Script.YES, Terms.Order.count(true));
-    }
-
     public void testDoubleCountDesc() throws Exception {
-        testMinDocCountOnTerms("d", Script.NO, Terms.Order.count(false));
+        testMinDocCountOnTerms("d", Script.NO, Terms.Order.count());
     }
 
     public void testDoubleScriptCountDesc() throws Exception {
-        testMinDocCountOnTerms("d", Script.YES, Terms.Order.count(false));
+        testMinDocCountOnTerms("d", Script.YES, Terms.Order.count());
     }
 
     private void testMinDocCountOnTerms(String field, Script script, Terms.Order order) throws Exception {
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java
index b098db154da..a5825d23172 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java
@@ -1375,11 +1375,6 @@ public class StringTermsTests extends AbstractTermsTestCase {
         assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("avg_l", false), Terms.Order.term(true));
     }
 
-    public void testSingleValuedFieldOrderedByCountAscAndSingleValueSubAggregationAsc() throws Exception {
-        String[] expectedKeys = new String[] { "val6", "val7", "val3", "val4", "val5", "val1", "val2" };
-        assertMultiSortResponse(expectedKeys, Terms.Order.count(true), Terms.Order.aggregation("avg_l", true));
-    }
-
     public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscSingleValueSubAggregationAsc() throws Exception {
         String[] expectedKeys = new String[] { "val6", "val7", "val3", "val5", "val4", "val1", "val2" };
         assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("sum_d", true), Terms.Order.aggregation("avg_l", true));
@@ -1387,7 +1382,7 @@ public class StringTermsTests extends AbstractTermsTestCase {
 
     public void testSingleValuedFieldOrderedByThreeCriteria() throws Exception {
         String[] expectedKeys = new String[] { "val2", "val1", "val4", "val5", "val3", "val6", "val7" };
-        assertMultiSortResponse(expectedKeys, Terms.Order.count(false), Terms.Order.aggregation("sum_d", false),
+        assertMultiSortResponse(expectedKeys, Terms.Order.count(), Terms.Order.aggregation("sum_d", false),
                 Terms.Order.aggregation("avg_l", false));
     }
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml
index c35e79e6cfe..c079e6b6f7e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml
@@ -350,4 +350,17 @@ setup:
   
   - match: { aggregations.date_terms.buckets.0.key_as_string: "2014-09-01T00:00:00.000Z" }
 
-  - match: { aggregations.date_terms.buckets.0.doc_count: 1 }  
+  - match: { aggregations.date_terms.buckets.0.doc_count: 1 }
+
+---
+"Terms count order test":
+  - do:
+      catch:  /Sort by ascending _count is not supported in/
+      search:
+        body: { "size" : 0, "aggs" : { "count_order" : { "terms" : { "field" : "f1", "order": {"_count": "asc"} } } } }
+
+  - do:
+      search:
+        body: { "size" : 0, "aggs" : { "count_order" : { "terms" : { "field" : "f1", "order": "_count" } } } }
+
+

From bce1abec48c521a422f6016479343b28f2538c7d Mon Sep 17 00:00:00 2001
From: Mateusz Rzepa <mrzepa89@gmail.com>
Date: Fri, 17 Jun 2016 16:19:04 +0200
Subject: [PATCH 0928/1311] Update integrations.asciidoc (#18915)

---
 docs/plugins/integrations.asciidoc | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/docs/plugins/integrations.asciidoc b/docs/plugins/integrations.asciidoc
index ee3ccef194c..d39d02efeac 100644
--- a/docs/plugins/integrations.asciidoc
+++ b/docs/plugins/integrations.asciidoc
@@ -187,3 +187,18 @@ releases 2.0 and later do not support rivers.
 * https://www.wireshark.org/[Wireshark]:
   Protocol dissection for Zen discovery, HTTP and the binary protocol
 
+* https://www.itemsapi.com/[ItemsAPI]:
+  Search backend for mobile and web
+
+
+These projects appear to have been abandoned:
+
+* http://www.github.com/neogenix/daikon[daikon]:
+  Daikon Elasticsearch CLI
+
+* https://github.com/fullscale/dangle[dangle]:
+  A set of AngularJS directives that provide common visualizations for elasticsearch based on
+  D3.
+* https://github.com/OlegKunitsyn/eslogd[eslogd]:
+  Linux daemon that replicates events to a central Elasticsearch server in realtime
+>>>>>>> 02602a3... Update integrations.asciidoc (#18915)

From 82fee9f7a714c756534fe32fcdedcae5c42cf8fd Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Fri, 17 Jun 2016 17:02:00 +0200
Subject: [PATCH 0929/1311] Revert change about registering Repository settings

Will create another issue to change that. Related to this discussion: https://github.com/elastic/elasticsearch/pull/17784/files/f4cd3bd3480bf6c0a51faf408d8ea9c8a8733a6b#r67291936
---
 .../plugin/repository/s3/S3RepositoryPlugin.java | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
index 7922174f40e..26ff53a6e62 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
@@ -129,6 +129,22 @@ public class S3RepositoryPlugin extends Plugin {
         settingsModule.registerSetting(S3Repository.Repositories.CANNED_ACL_SETTING);
         settingsModule.registerSetting(S3Repository.Repositories.BASE_PATH_SETTING);
         settingsModule.registerSetting(S3Repository.Repositories.USE_THROTTLE_RETRIES_SETTING);
+
+        // Register S3 single repository settings
+        settingsModule.registerSetting(S3Repository.Repository.KEY_SETTING);
+        settingsModule.registerSetting(S3Repository.Repository.SECRET_SETTING);
+        settingsModule.registerSetting(S3Repository.Repository.BUCKET_SETTING);
+        settingsModule.registerSetting(S3Repository.Repository.ENDPOINT_SETTING);
+        settingsModule.registerSetting(S3Repository.Repository.PROTOCOL_SETTING);
+        settingsModule.registerSetting(S3Repository.Repository.REGION_SETTING);
+        settingsModule.registerSetting(S3Repository.Repository.SERVER_SIDE_ENCRYPTION_SETTING);
+        settingsModule.registerSetting(S3Repository.Repository.BUFFER_SIZE_SETTING);
+        settingsModule.registerSetting(S3Repository.Repository.MAX_RETRIES_SETTING);
+        settingsModule.registerSetting(S3Repository.Repository.CHUNK_SIZE_SETTING);
+        settingsModule.registerSetting(S3Repository.Repository.COMPRESS_SETTING);
+        settingsModule.registerSetting(S3Repository.Repository.STORAGE_CLASS_SETTING);
+        settingsModule.registerSetting(S3Repository.Repository.CANNED_ACL_SETTING);
+        settingsModule.registerSetting(S3Repository.Repository.BASE_PATH_SETTING);
     }
 
     /**

From bdb6dcea3ae53e7dea8ce060c46100ed8f25385e Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Fri, 17 Jun 2016 17:07:19 +0200
Subject: [PATCH 0930/1311] Cleanup ClusterService dependencies and detached
 from Guice (#18941)

This change removes some unnecessary dependencies from ClusterService
and cleans up ClusterName creation. ClusterService is now not created
by guice anymore.
---
 .../allocation/AllocationBenchmark.java       |  3 +-
 .../resources/checkstyle_suppressions.xml     |  3 --
 .../health/TransportClusterHealthAction.java  |  9 ++--
 .../TransportNodesHotThreadsAction.java       |  6 +--
 .../node/info/TransportNodesInfoAction.java   |  7 ++-
 .../node/liveness/LivenessResponse.java       |  2 +-
 .../liveness/TransportLivenessAction.java     |  8 +---
 .../node/stats/TransportNodesStatsAction.java |  7 ++-
 .../cancel/TransportCancelTasksAction.java    |  5 +--
 .../tasks/list/TransportListTasksAction.java  |  5 +--
 .../TransportVerifyRepositoryAction.java      |  7 +--
 .../verify/VerifyRepositoryResponse.java      |  2 +-
 .../status/TransportNodesSnapshotsStatus.java |  9 ++--
 .../cluster/state/ClusterStateResponse.java   |  2 +-
 .../state/TransportClusterStateAction.java    |  7 +--
 .../stats/TransportClusterStatsAction.java    |  9 ++--
 .../action/main/MainResponse.java             |  2 +-
 .../support/nodes/BaseNodesResponse.java      |  2 +-
 .../support/nodes/TransportNodesAction.java   |  6 +--
 .../support/tasks/TransportTasksAction.java   |  5 +--
 .../client/transport/TransportClient.java     |  2 -
 .../TransportClientNodesService.java          |  4 +-
 .../elasticsearch/cluster/ClusterModule.java  |  7 +--
 .../elasticsearch/cluster/ClusterName.java    | 41 +++++------------
 .../cluster/ClusterNameModule.java            | 40 -----------------
 .../elasticsearch/cluster/ClusterState.java   |  6 +--
 .../cluster/health/ClusterStateHealth.java    | 13 ------
 .../cluster/routing/OperationRouting.java     | 18 ++++----
 .../routing/allocation/AllocationService.java |  9 +++-
 .../allocator/BalancedShardsAllocator.java    |  1 -
 .../decider/AwarenessAllocationDecider.java   |  9 ----
 .../cluster/service/ClusterService.java       | 19 +++++---
 .../common/network/NetworkModule.java         |  6 +--
 .../common/settings/ClusterSettings.java      |  2 +-
 .../discovery/local/LocalDiscovery.java       |  4 +-
 .../discovery/zen/ZenDiscovery.java           | 10 ++---
 .../zen/fd/MasterFaultDetection.java          |  6 +--
 .../discovery/zen/fd/NodesFaultDetection.java |  2 +-
 .../discovery/zen/ping/ZenPing.java           |  7 +--
 .../zen/ping/unicast/UnicastZenPing.java      | 34 ++++++--------
 .../org/elasticsearch/env/Environment.java    |  5 ++-
 .../org/elasticsearch/gateway/Gateway.java    |  2 +-
 .../TransportNodesListGatewayMetaState.java   |  6 +--
 ...ransportNodesListGatewayStartedShards.java |  6 +--
 .../indices/store/IndicesStore.java           |  4 +-
 .../TransportNodesListShardStoreMetaData.java |  6 +--
 .../java/org/elasticsearch/node/Node.java     |  6 +--
 .../transport/TransportService.java           | 13 ++----
 .../health/ClusterHealthResponsesTests.java   |  3 +-
 .../node/tasks/CancellableTasksTests.java     |  7 ++-
 .../node/tasks/TaskManagerTestCase.java       | 15 +++----
 .../cluster/node/tasks/TestTaskPlugin.java    | 10 ++---
 .../node/tasks/TransportTasksActionTests.java | 17 ++++---
 .../cluster/reroute/ClusterRerouteTests.java  |  3 +-
 .../shrink/TransportShrinkActionTests.java    |  3 +-
 .../bulk/TransportBulkActionTookTests.java    |  2 +-
 .../action/support/AutoCreateIndexTests.java  |  2 +-
 .../TransportBroadcastByNodeActionTests.java  |  2 +-
 .../TransportMasterNodeActionTests.java       |  2 +-
 .../nodes/TransportNodesActionTests.java      | 14 +++---
 .../BroadcastReplicationTests.java            |  2 +-
 .../TransportReplicationActionTests.java      |  2 +-
 ...ortInstanceSingleOperationActionTests.java |  2 +-
 .../AbstractTermVectorsTestCase.java          |  2 +-
 .../transport/FailAndRetryMockTransport.java  |  4 +-
 .../TransportClientHeadersTests.java          |  7 +--
 .../TransportClientNodesServiceTests.java     |  7 +--
 .../cluster/ClusterModuleTests.java           | 20 +++++----
 .../cluster/ClusterStateTests.java            | 15 ++++---
 .../cluster/NodeConnectionsServiceTests.java  |  2 +-
 ...rdFailedClusterStateTaskExecutorTests.java |  2 +-
 .../action/shard/ShardStateActionTests.java   |  2 +-
 .../health/ClusterStateHealthTests.java       |  7 ++-
 .../MetaDataCreateIndexServiceTests.java      |  6 ++-
 .../DelayedAllocationServiceTests.java        |  8 ++--
 .../routing/OperationRoutingTests.java        |  4 +-
 .../cluster/routing/PrimaryTermsTests.java    |  4 +-
 .../RoutingBackwardCompatibilityTests.java    | 13 ++++--
 .../cluster/routing/RoutingTableTests.java    |  2 +-
 .../cluster/routing/UnassignedInfoTests.java  | 22 +++++-----
 .../allocation/ActiveAllocationIdTests.java   |  4 +-
 .../allocation/AddIncrementallyTests.java     |  2 +-
 .../allocation/AllocationCommandsTests.java   |  6 +--
 .../allocation/AllocationPriorityTests.java   |  2 +-
 .../allocation/AwarenessAllocationTests.java  | 22 +++++-----
 .../allocation/BalanceConfigurationTests.java |  4 +-
 .../allocation/CatAllocationTestCase.java     |  2 +-
 .../ClusterRebalanceRoutingTests.java         | 16 +++----
 .../ConcurrentRebalanceRoutingTests.java      |  2 +-
 .../allocation/DeadNodesAllocationTests.java  |  6 +--
 ...ReplicaAsPrimaryDuringRelocationTests.java |  2 +-
 .../ExpectedShardSizeAllocationTests.java     |  4 +-
 .../allocation/FailedNodeRoutingTests.java    |  4 +-
 .../allocation/FailedShardsRoutingTests.java  | 16 +++----
 .../FilterAllocationDeciderTests.java         |  4 +-
 .../allocation/FilterRoutingTests.java        |  6 +--
 .../routing/allocation/IndexBalanceTests.java |  6 +--
 .../MaxRetryAllocationDeciderTests.java       |  3 +-
 .../NodeVersionAllocationDeciderTests.java    | 10 ++---
 ...alPrimariesToRelocatingPrimariesTests.java |  2 +-
 .../PreferPrimaryAllocationTests.java         |  2 +-
 .../PrimaryElectionRoutingTests.java          |  4 +-
 ...yNotRelocatedWhileBeingRecoveredTests.java |  2 +-
 .../RandomAllocationDeciderTests.java         |  2 +-
 .../allocation/RebalanceAfterActiveTests.java |  2 +-
 .../ReplicaAllocatedAfterPrimaryTests.java    |  2 +-
 .../RoutingNodesIntegrityTests.java           |  6 +--
 .../allocation/SameShardRoutingTests.java     |  3 +-
 .../allocation/ShardVersioningTests.java      |  2 +-
 .../ShardsLimitAllocationTests.java           |  6 +--
 .../SingleShardNoReplicasRoutingTests.java    |  8 ++--
 .../SingleShardOneReplicaRoutingTests.java    |  2 +-
 .../allocation/StartedShardsRoutingTests.java |  3 +-
 .../TenShardsOneReplicaRoutingTests.java      |  2 +-
 .../allocation/ThrottlingAllocationTests.java |  8 ++--
 .../UpdateNumberOfReplicasTests.java          |  2 +-
 .../decider/DiskThresholdDeciderTests.java    | 15 ++++---
 .../DiskThresholdDeciderUnitTests.java        |  8 ++--
 .../decider/EnableAllocationTests.java        | 10 ++---
 .../ClusterSerializationTests.java            |  4 +-
 .../ClusterStateToStringTests.java            |  6 ++-
 .../cluster/service/ClusterServiceTests.java  | 11 +++--
 .../structure/RoutingIteratorTests.java       | 15 ++++---
 .../discovery/ZenFaultDetectionTests.java     | 20 +++++----
 .../discovery/zen/ZenPingTests.java           |  4 +-
 .../zen/ping/unicast/UnicastZenPingIT.java    | 44 +++++++++++--------
 .../PendingClusterStatesQueueTests.java       |  3 +-
 .../PublishClusterStateActionTests.java       |  4 +-
 .../gateway/GatewayMetaStateTests.java        |  4 +-
 .../gateway/GatewayServiceTests.java          |  4 +-
 .../gateway/PrimaryShardAllocatorTests.java   | 10 ++---
 .../gateway/ReplicaShardAllocatorTests.java   |  4 +-
 .../elasticsearch/http/HttpServerTests.java   |  5 +--
 .../mapper/DynamicMappingDisabledTests.java   |  2 +-
 .../indices/cluster/ClusterStateChanges.java  |  2 +-
 ...ClusterStateServiceRandomUpdatesTests.java |  2 +-
 .../indices/store/IndicesStoreTests.java      |  2 +-
 .../org/elasticsearch/mget/SimpleMgetIT.java  |  4 +-
 .../InternalSettingsPreparerTests.java        |  7 ---
 .../bucket/ShardSizeTestCase.java             |  4 +-
 .../AbstractSimpleTransportTestCase.java      | 11 ++---
 .../NettySizeHeaderFrameDecoderTests.java     |  2 +-
 .../NettyTransportServiceHandshakeTests.java  | 35 ++++++---------
 .../transport/TransportModuleTests.java       |  4 +-
 .../local/SimpleLocalTransportTests.java      |  4 +-
 .../netty/NettyScheduledPingTests.java        |  6 +--
 .../netty/SimpleNettyTransportTests.java      |  4 +-
 .../messy/tests/IndicesRequestTests.java      |  4 +-
 .../reindex/TransportRethrottleAction.java    |  4 +-
 .../discovery/ec2/Ec2DiscoveryTests.java      |  2 +-
 .../discovery/gce/GceDiscoveryTests.java      |  2 +-
 .../MockInternalClusterInfoService.java       |  2 +-
 .../test/ClusterServiceUtils.java             |  4 +-
 .../elasticsearch/test/ESIntegTestCase.java   |  5 +--
 .../test/ESSingleNodeTestCase.java            |  2 -
 .../org/elasticsearch/test/ExternalNode.java  |  1 -
 .../test/InternalTestCluster.java             |  4 +-
 .../test/transport/MockTransportService.java  | 13 +++---
 158 files changed, 497 insertions(+), 592 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/cluster/ClusterNameModule.java

diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java
index d5bb0916eca..5e5f35f6040 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java
@@ -147,7 +147,8 @@ public class AllocationBenchmark {
         for (int i = 1; i <= numNodes; i++) {
             nb.put(Allocators.newNode("node" + i, Collections.singletonMap("tag", "tag_" + (i % numTags))));
         }
-        initialClusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).nodes
+        initialClusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+            .metaData(metaData).routingTable(routingTable).nodes
                 (nb).build();
     }
 
diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 0a4dbbf3ff9..94494a06542 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -754,7 +754,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterHealthIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterInfoServiceIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterStateDiffIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]ClusterStateTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]DiskUsageTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]MinimumMasterNodesIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]NoMasterNodeIT.java" checks="LineLength" />
@@ -784,13 +783,11 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]AllocationIdTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]DelayedAllocationIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]PrimaryAllocationIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]RoutingBackwardCompatibilityTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]RoutingServiceTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]RoutingTableTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]ShardRoutingHelper.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]ShardRoutingTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]UnassignedInfoTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]allocation[/\\]ActiveAllocationIdTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]allocation[/\\]AddIncrementallyTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]allocation[/\\]AllocationCommandsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]allocation[/\\]AllocationPriorityTests.java" checks="LineLength" />
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
index 204eab7aaf4..de616c61907 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
@@ -23,7 +23,6 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
@@ -47,16 +46,14 @@ import org.elasticsearch.transport.TransportService;
  */
 public class TransportClusterHealthAction extends TransportMasterNodeReadAction<ClusterHealthRequest, ClusterHealthResponse> {
 
-    private final ClusterName clusterName;
     private final GatewayAllocator gatewayAllocator;
 
     @Inject
     public TransportClusterHealthAction(Settings settings, TransportService transportService, ClusterService clusterService,
-                                        ThreadPool threadPool, ClusterName clusterName, ActionFilters actionFilters,
+                                        ThreadPool threadPool, ActionFilters actionFilters,
                                         IndexNameExpressionResolver indexNameExpressionResolver, GatewayAllocator gatewayAllocator) {
         super(settings, ClusterHealthAction.NAME, false, transportService, clusterService, threadPool, actionFilters,
             indexNameExpressionResolver, ClusterHealthRequest::new);
-        this.clusterName = clusterName;
         this.gatewayAllocator = gatewayAllocator;
     }
 
@@ -284,14 +281,14 @@ public class TransportClusterHealthAction extends TransportMasterNodeReadAction<
             concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request);
         } catch (IndexNotFoundException e) {
             // one of the specified indices is not there - treat it as RED.
-            ClusterHealthResponse response = new ClusterHealthResponse(clusterName.value(), Strings.EMPTY_ARRAY, clusterState,
+            ClusterHealthResponse response = new ClusterHealthResponse(clusterState.getClusterName().value(), Strings.EMPTY_ARRAY, clusterState,
                     numberOfPendingTasks, numberOfInFlightFetch, UnassignedInfo.getNumberOfDelayedUnassigned(clusterState),
                     pendingTaskTimeInQueue);
             response.setStatus(ClusterHealthStatus.RED);
             return response;
         }
 
-        return new ClusterHealthResponse(clusterName.value(), concreteIndices, clusterState, numberOfPendingTasks,
+        return new ClusterHealthResponse(clusterState.getClusterName().value(), concreteIndices, clusterState, numberOfPendingTasks,
                 numberOfInFlightFetch, UnassignedInfo.getNumberOfDelayedUnassigned(clusterState), pendingTaskTimeInQueue);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java
index 7198851fd28..73403f40318 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java
@@ -47,17 +47,17 @@ public class TransportNodesHotThreadsAction extends TransportNodesAction<NodesHo
                                                                          NodeHotThreads> {
 
     @Inject
-    public TransportNodesHotThreadsAction(Settings settings, ClusterName clusterName, ThreadPool threadPool,
+    public TransportNodesHotThreadsAction(Settings settings, ThreadPool threadPool,
                                           ClusterService clusterService, TransportService transportService,
                                           ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, NodesHotThreadsAction.NAME, clusterName, threadPool, clusterService, transportService, actionFilters,
+        super(settings, NodesHotThreadsAction.NAME, threadPool, clusterService, transportService, actionFilters,
               indexNameExpressionResolver, NodesHotThreadsRequest::new, NodeRequest::new, ThreadPool.Names.GENERIC, NodeHotThreads.class);
     }
 
     @Override
     protected NodesHotThreadsResponse newResponse(NodesHotThreadsRequest request,
                                                   List<NodeHotThreads> responses, List<FailedNodeException> failures) {
-        return new NodesHotThreadsResponse(clusterName, responses, failures);
+        return new NodesHotThreadsResponse(clusterService.getClusterName(), responses, failures);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
index f68e2d65903..8b6cc8210fa 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
@@ -23,7 +23,6 @@ import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.nodes.BaseNodeRequest;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
@@ -48,11 +47,11 @@ public class TransportNodesInfoAction extends TransportNodesAction<NodesInfoRequ
     private final NodeService nodeService;
 
     @Inject
-    public TransportNodesInfoAction(Settings settings, ClusterName clusterName, ThreadPool threadPool,
+    public TransportNodesInfoAction(Settings settings, ThreadPool threadPool,
                                     ClusterService clusterService, TransportService transportService,
                                     NodeService nodeService, ActionFilters actionFilters,
                                     IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, NodesInfoAction.NAME, clusterName, threadPool, clusterService, transportService, actionFilters,
+        super(settings, NodesInfoAction.NAME, threadPool, clusterService, transportService, actionFilters,
               indexNameExpressionResolver, NodesInfoRequest::new, NodeInfoRequest::new, ThreadPool.Names.MANAGEMENT, NodeInfo.class);
         this.nodeService = nodeService;
     }
@@ -60,7 +59,7 @@ public class TransportNodesInfoAction extends TransportNodesAction<NodesInfoRequ
     @Override
     protected NodesInfoResponse newResponse(NodesInfoRequest nodesInfoRequest,
                                             List<NodeInfo> responses, List<FailedNodeException> failures) {
-        return new NodesInfoResponse(clusterName, responses, failures);
+        return new NodesInfoResponse(clusterService.getClusterName(), responses, failures);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/liveness/LivenessResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/liveness/LivenessResponse.java
index b0a53190bf0..a9ba36ffeff 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/liveness/LivenessResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/liveness/LivenessResponse.java
@@ -47,7 +47,7 @@ public final class LivenessResponse extends ActionResponse {
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        clusterName = ClusterName.readClusterName(in);
+        clusterName = new ClusterName(in);
         node = in.readOptionalWriteable(DiscoveryNode::new);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/liveness/TransportLivenessAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/liveness/TransportLivenessAction.java
index a437fc5ee49..09c608ac842 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/liveness/TransportLivenessAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/liveness/TransportLivenessAction.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.action.admin.cluster.node.liveness;
 
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -30,20 +29,17 @@ import org.elasticsearch.transport.TransportService;
 public final class TransportLivenessAction implements TransportRequestHandler<LivenessRequest> {
 
     private final ClusterService clusterService;
-    private final ClusterName clusterName;
     public static final String NAME = "cluster:monitor/nodes/liveness";
 
     @Inject
-    public TransportLivenessAction(ClusterName clusterName,
-                                   ClusterService clusterService, TransportService transportService) {
+    public TransportLivenessAction(ClusterService clusterService, TransportService transportService) {
         this.clusterService = clusterService;
-        this.clusterName = clusterName;
         transportService.registerRequestHandler(NAME, LivenessRequest::new, ThreadPool.Names.SAME,
             false, false /*can not trip circuit breaker*/, this);
     }
 
     @Override
     public void messageReceived(LivenessRequest request, TransportChannel channel) throws Exception {
-        channel.sendResponse(new LivenessResponse(clusterName, clusterService.localNode()));
+        channel.sendResponse(new LivenessResponse(clusterService.getClusterName(), clusterService.localNode()));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
index d61e3f1acce..5863e54d08f 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
@@ -23,7 +23,6 @@ import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.nodes.BaseNodeRequest;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
@@ -48,18 +47,18 @@ public class TransportNodesStatsAction extends TransportNodesAction<NodesStatsRe
     private final NodeService nodeService;
 
     @Inject
-    public TransportNodesStatsAction(Settings settings, ClusterName clusterName, ThreadPool threadPool,
+    public TransportNodesStatsAction(Settings settings, ThreadPool threadPool,
                                      ClusterService clusterService, TransportService transportService,
                                      NodeService nodeService, ActionFilters actionFilters,
                                      IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, NodesStatsAction.NAME, clusterName, threadPool, clusterService, transportService, actionFilters,
+        super(settings, NodesStatsAction.NAME, threadPool, clusterService, transportService, actionFilters,
               indexNameExpressionResolver, NodesStatsRequest::new, NodeStatsRequest::new, ThreadPool.Names.MANAGEMENT, NodeStats.class);
         this.nodeService = nodeService;
     }
 
     @Override
     protected NodesStatsResponse newResponse(NodesStatsRequest request, List<NodeStats> responses, List<FailedNodeException> failures) {
-        return new NodesStatsResponse(clusterName, responses, failures);
+        return new NodesStatsResponse(clusterService.getClusterName(), responses, failures);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java
index bc69228055a..6d5936db67a 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/cancel/TransportCancelTasksAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -63,10 +62,10 @@ public class TransportCancelTasksAction extends TransportTasksAction<Cancellable
     public static final String BAN_PARENT_ACTION_NAME = "internal:admin/tasks/ban";
 
     @Inject
-    public TransportCancelTasksAction(Settings settings, ClusterName clusterName, ThreadPool threadPool, ClusterService clusterService,
+    public TransportCancelTasksAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
                                       TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver
                                           indexNameExpressionResolver) {
-        super(settings, CancelTasksAction.NAME, clusterName, threadPool, clusterService, transportService, actionFilters,
+        super(settings, CancelTasksAction.NAME, threadPool, clusterService, transportService, actionFilters,
             indexNameExpressionResolver, CancelTasksRequest::new, CancelTasksResponse::new, ThreadPool.Names.MANAGEMENT);
         transportService.registerRequestHandler(BAN_PARENT_ACTION_NAME, BanParentTaskRequest::new, ThreadPool.Names.SAME, new
             BanParentRequestHandler());
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
index 9acdcff81c9..26158800346 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/list/TransportListTasksAction.java
@@ -23,7 +23,6 @@ import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
@@ -55,9 +54,9 @@ public class TransportListTasksAction extends TransportTasksAction<Task, ListTas
     private static final TimeValue DEFAULT_WAIT_FOR_COMPLETION_TIMEOUT = timeValueSeconds(30);
 
     @Inject
-    public TransportListTasksAction(Settings settings, ClusterName clusterName, ThreadPool threadPool, ClusterService clusterService,
+    public TransportListTasksAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
             TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, ListTasksAction.NAME, clusterName, threadPool, clusterService, transportService, actionFilters,
+        super(settings, ListTasksAction.NAME, threadPool, clusterService, transportService, actionFilters,
                 indexNameExpressionResolver, ListTasksRequest::new, ListTasksResponse::new, ThreadPool.Names.MANAGEMENT);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java
index 2c75335dcaa..704cc511ccc 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java
@@ -22,7 +22,6 @@ package org.elasticsearch.action.admin.cluster.repositories.verify;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeAction;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
@@ -42,15 +41,13 @@ public class TransportVerifyRepositoryAction extends TransportMasterNodeAction<V
 
     private final RepositoriesService repositoriesService;
 
-    protected final ClusterName clusterName;
 
     @Inject
-    public TransportVerifyRepositoryAction(Settings settings, ClusterName clusterName, TransportService transportService, ClusterService clusterService,
+    public TransportVerifyRepositoryAction(Settings settings, TransportService transportService, ClusterService clusterService,
                                            RepositoriesService repositoriesService, ThreadPool threadPool, ActionFilters actionFilters,
                                            IndexNameExpressionResolver indexNameExpressionResolver) {
         super(settings, VerifyRepositoryAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, VerifyRepositoryRequest::new);
         this.repositoriesService = repositoriesService;
-        this.clusterName = clusterName;
     }
 
     @Override
@@ -76,7 +73,7 @@ public class TransportVerifyRepositoryAction extends TransportMasterNodeAction<V
                 if (verifyResponse.failed()) {
                     listener.onFailure(new RepositoryVerificationException(request.name(), verifyResponse.failureDescription()));
                 } else {
-                    listener.onResponse(new VerifyRepositoryResponse(clusterName, verifyResponse.nodes()));
+                    listener.onResponse(new VerifyRepositoryResponse(clusterService.getClusterName(), verifyResponse.nodes()));
                 }
             }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java
index 823487155f5..5de45fafa6d 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/VerifyRepositoryResponse.java
@@ -51,7 +51,7 @@ public class VerifyRepositoryResponse extends ActionResponse implements ToXConte
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        clusterName = ClusterName.readClusterName(in);
+        clusterName = new ClusterName(in);
         nodes = new DiscoveryNode[in.readVInt()];
         for (int i=0; i<nodes.length; i++){
             nodes[i] = new DiscoveryNode(in);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java
index 8e7361c1928..71a709f0b5b 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java
@@ -63,11 +63,11 @@ public class TransportNodesSnapshotsStatus extends TransportNodesAction<Transpor
     private final SnapshotShardsService snapshotShardsService;
 
     @Inject
-    public TransportNodesSnapshotsStatus(Settings settings, ClusterName clusterName, ThreadPool threadPool,
+    public TransportNodesSnapshotsStatus(Settings settings, ThreadPool threadPool,
                                          ClusterService clusterService, TransportService transportService,
                                          SnapshotShardsService snapshotShardsService, ActionFilters actionFilters,
                                          IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, ACTION_NAME, clusterName, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
+        super(settings, ACTION_NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
               Request::new, NodeRequest::new, ThreadPool.Names.GENERIC, NodeSnapshotStatus.class);
         this.snapshotShardsService = snapshotShardsService;
     }
@@ -89,7 +89,7 @@ public class TransportNodesSnapshotsStatus extends TransportNodesAction<Transpor
 
     @Override
     protected NodesSnapshotStatus newResponse(Request request, List<NodeSnapshotStatus> responses, List<FailedNodeException> failures) {
-        return new NodesSnapshotStatus(clusterName, responses, failures);
+        return new NodesSnapshotStatus(clusterService.getClusterName(), responses, failures);
     }
 
     @Override
@@ -158,9 +158,6 @@ public class TransportNodesSnapshotsStatus extends TransportNodesAction<Transpor
 
     public static class NodesSnapshotStatus extends BaseNodesResponse<NodeSnapshotStatus> {
 
-        NodesSnapshotStatus() {
-        }
-
         public NodesSnapshotStatus(ClusterName clusterName, List<NodeSnapshotStatus> nodes, List<FailedNodeException> failures) {
             super(clusterName, nodes, failures);
         }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java
index e9aa9b723fa..2a2f4707f69 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java
@@ -54,7 +54,7 @@ public class ClusterStateResponse extends ActionResponse {
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        clusterName = ClusterName.readClusterName(in);
+        clusterName = new ClusterName(in);
         clusterState = ClusterState.Builder.readFrom(in, null);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java
index 77a42108280..ea1c1507448 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java
@@ -23,7 +23,6 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -42,13 +41,11 @@ import org.elasticsearch.transport.TransportService;
  */
 public class TransportClusterStateAction extends TransportMasterNodeReadAction<ClusterStateRequest, ClusterStateResponse> {
 
-    private final ClusterName clusterName;
 
     @Inject
     public TransportClusterStateAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
-                                       ClusterName clusterName, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
+                                       ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
         super(settings, ClusterStateAction.NAME, false, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, ClusterStateRequest::new);
-        this.clusterName = clusterName;
     }
 
     @Override
@@ -127,7 +124,7 @@ public class TransportClusterStateAction extends TransportMasterNodeReadAction<C
         if (request.customs()) {
             builder.customs(currentState.customs());
         }
-        listener.onResponse(new ClusterStateResponse(clusterName, builder.build()));
+        listener.onResponse(new ClusterStateResponse(currentState.getClusterName(), builder.build()));
     }
 
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
index 3a0b1455209..744248a1ef8 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
@@ -28,7 +28,6 @@ import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.nodes.BaseNodeRequest;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.health.ClusterStateHealth;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -62,11 +61,11 @@ public class TransportClusterStatsAction extends TransportNodesAction<ClusterSta
 
 
     @Inject
-    public TransportClusterStatsAction(Settings settings, ClusterName clusterName, ThreadPool threadPool,
+    public TransportClusterStatsAction(Settings settings, ThreadPool threadPool,
                                        ClusterService clusterService, TransportService transportService,
                                        NodeService nodeService, IndicesService indicesService,
                                        ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, ClusterStatsAction.NAME, clusterName, threadPool, clusterService, transportService, actionFilters,
+        super(settings, ClusterStatsAction.NAME, threadPool, clusterService, transportService, actionFilters,
               indexNameExpressionResolver, ClusterStatsRequest::new, ClusterStatsNodeRequest::new, ThreadPool.Names.MANAGEMENT,
               ClusterStatsNodeResponse.class);
         this.nodeService = nodeService;
@@ -76,8 +75,8 @@ public class TransportClusterStatsAction extends TransportNodesAction<ClusterSta
     @Override
     protected ClusterStatsResponse newResponse(ClusterStatsRequest request,
                                                List<ClusterStatsNodeResponse> responses, List<FailedNodeException> failures) {
-        return new ClusterStatsResponse(System.currentTimeMillis(), clusterName, clusterService.state().metaData().clusterUUID(),
-                                        responses, failures);
+        return new ClusterStatsResponse(System.currentTimeMillis(), clusterService.getClusterName(),
+            clusterService.state().metaData().clusterUUID(), responses, failures);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/main/MainResponse.java b/core/src/main/java/org/elasticsearch/action/main/MainResponse.java
index b9d094c7e9e..2403c3ee49c 100644
--- a/core/src/main/java/org/elasticsearch/action/main/MainResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/main/MainResponse.java
@@ -84,7 +84,7 @@ public class MainResponse extends ActionResponse implements ToXContent {
         super.readFrom(in);
         nodeName = in.readString();
         version = Version.readVersion(in);
-        clusterName = ClusterName.readClusterName(in);
+        clusterName = new ClusterName(in);
         build = Build.readBuild(in);
         available = in.readBoolean();
     }
diff --git a/core/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesResponse.java b/core/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesResponse.java
index a49864154db..462f0b07bd6 100644
--- a/core/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesResponse.java
@@ -106,7 +106,7 @@ public abstract class BaseNodesResponse<TNodeResponse extends BaseNodeResponse>
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        clusterName = ClusterName.readClusterName(in);
+        clusterName = new ClusterName(in);
         nodes = readNodesFrom(in);
         failures = in.readList(FailedNodeException::new);
     }
diff --git a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
index 2b72ef8b31b..5d39d378b21 100644
--- a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
@@ -24,13 +24,11 @@ import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.NoSuchNodeException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -59,21 +57,19 @@ public abstract class TransportNodesAction<NodesRequest extends BaseNodesRequest
                                            NodeResponse extends BaseNodeResponse>
     extends HandledTransportAction<NodesRequest, NodesResponse> {
 
-    protected final ClusterName clusterName;
     protected final ClusterService clusterService;
     protected final TransportService transportService;
     protected final Class<NodeResponse> nodeResponseClass;
 
     final String transportNodeAction;
 
-    protected TransportNodesAction(Settings settings, String actionName, ClusterName clusterName, ThreadPool threadPool,
+    protected TransportNodesAction(Settings settings, String actionName, ThreadPool threadPool,
                                    ClusterService clusterService, TransportService transportService, ActionFilters actionFilters,
                                    IndexNameExpressionResolver indexNameExpressionResolver,
                                    Supplier<NodesRequest> request, Supplier<NodeRequest> nodeRequest,
                                    String nodeExecutor,
                                    Class<NodeResponse> nodeResponseClass) {
         super(settings, actionName, threadPool, transportService, actionFilters, indexNameExpressionResolver, request);
-        this.clusterName = Objects.requireNonNull(clusterName);
         this.clusterService = Objects.requireNonNull(clusterService);
         this.transportService = Objects.requireNonNull(transportService);
         this.nodeResponseClass = Objects.requireNonNull(nodeResponseClass);
diff --git a/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java b/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
index e3070832417..f28b70b88ed 100644
--- a/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
@@ -26,7 +26,6 @@ import org.elasticsearch.action.NoSuchNodeException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -67,7 +66,6 @@ public abstract class TransportTasksAction<
         TaskResponse extends Writeable
     > extends HandledTransportAction<TasksRequest, TasksResponse> {
 
-    protected final ClusterName clusterName;
     protected final ClusterService clusterService;
     protected final TransportService transportService;
     protected final Supplier<TasksRequest> requestSupplier;
@@ -75,13 +73,12 @@ public abstract class TransportTasksAction<
 
     protected final String transportNodeAction;
 
-    protected TransportTasksAction(Settings settings, String actionName, ClusterName clusterName, ThreadPool threadPool,
+    protected TransportTasksAction(Settings settings, String actionName, ThreadPool threadPool,
                                    ClusterService clusterService, TransportService transportService, ActionFilters actionFilters,
                                    IndexNameExpressionResolver indexNameExpressionResolver, Supplier<TasksRequest> requestSupplier,
                                    Supplier<TasksResponse> responseSupplier,
                                    String nodeExecutor) {
         super(settings, actionName, threadPool, transportService, actionFilters, indexNameExpressionResolver, requestSupplier);
-        this.clusterName = clusterName;
         this.clusterService = clusterService;
         this.transportService = transportService;
         this.transportNodeAction = actionName + "[n]";
diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
index 56f541631ab..089fb41a321 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
@@ -28,7 +28,6 @@ import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.client.support.AbstractClient;
 import org.elasticsearch.client.transport.support.TransportProxyClient;
-import org.elasticsearch.cluster.ClusterNameModule;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.inject.Injector;
@@ -137,7 +136,6 @@ public class TransportClient extends AbstractClient {
                 }
                 modules.add(new PluginsModule(pluginsService));
                 modules.add(new NetworkModule(networkService, settings, true, namedWriteableRegistry));
-                modules.add(new ClusterNameModule(settings));
                 modules.add(b -> b.bind(ThreadPool.class).toInstance(threadPool));
                 modules.add(new SearchModule(settings, namedWriteableRegistry) {
                     @Override
diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
index 71c5895669c..cfa8adc28a7 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
@@ -111,10 +111,10 @@ public class TransportClientNodesService extends AbstractComponent {
         Setting.boolSetting("client.transport.sniff", false, Property.NodeScope);
 
     @Inject
-    public TransportClientNodesService(Settings settings, ClusterName clusterName, TransportService transportService,
+    public TransportClientNodesService(Settings settings,TransportService transportService,
                                        ThreadPool threadPool, Version version) {
         super(settings);
-        this.clusterName = clusterName;
+        this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
         this.transportService = transportService;
         this.threadPool = threadPool;
         this.minCompatibilityVersion = version.minimumCompatibilityVersion();
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index c90dbcd3c54..cc11b58727d 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -100,17 +100,19 @@ public class ClusterModule extends AbstractModule {
     private final ExtensionPoint.SelectedType<ShardsAllocator> shardsAllocators = new ExtensionPoint.SelectedType<>("shards_allocator", ShardsAllocator.class);
     private final ExtensionPoint.ClassSet<AllocationDecider> allocationDeciders = new ExtensionPoint.ClassSet<>("allocation_decider", AllocationDecider.class, AllocationDeciders.class);
     private final ExtensionPoint.ClassSet<IndexTemplateFilter> indexTemplateFilters = new ExtensionPoint.ClassSet<>("index_template_filter", IndexTemplateFilter.class);
+    private final ClusterService clusterService;
 
     // pkg private so tests can mock
     Class<? extends ClusterInfoService> clusterInfoServiceImpl = InternalClusterInfoService.class;
 
-    public ClusterModule(Settings settings) {
+    public ClusterModule(Settings settings, ClusterService clusterService) {
         this.settings = settings;
         for (Class<? extends AllocationDecider> decider : ClusterModule.DEFAULT_ALLOCATION_DECIDERS) {
             registerAllocationDecider(decider);
         }
         registerShardsAllocator(ClusterModule.BALANCED_ALLOCATOR, BalancedShardsAllocator.class);
         registerShardsAllocator(ClusterModule.EVEN_SHARD_COUNT_ALLOCATOR, BalancedShardsAllocator.class);
+        this.clusterService = clusterService;
     }
 
     public void registerAllocationDecider(Class<? extends AllocationDecider> allocationDecider) {
@@ -140,9 +142,8 @@ public class ClusterModule extends AbstractModule {
         bind(GatewayAllocator.class).asEagerSingleton();
         bind(AllocationService.class).asEagerSingleton();
         bind(DiscoveryNodeService.class).asEagerSingleton();
-        bind(ClusterService.class).asEagerSingleton();
+        bind(ClusterService.class).toInstance(clusterService);
         bind(NodeConnectionsService.class).asEagerSingleton();
-        bind(OperationRouting.class).asEagerSingleton();
         bind(MetaDataCreateIndexService.class).asEagerSingleton();
         bind(MetaDataDeleteIndexService.class).asEagerSingleton();
         bind(MetaDataIndexStateService.class).asEagerSingleton();
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterName.java b/core/src/main/java/org/elasticsearch/cluster/ClusterName.java
index 09c64065dbd..36676300954 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterName.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterName.java
@@ -21,37 +21,29 @@ package org.elasticsearch.cluster;
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 
 import java.io.IOException;
+import java.util.Objects;
 
-/**
- *
- */
-public class ClusterName implements Streamable {
+public class ClusterName implements Writeable {
 
-    public static final Setting<String> CLUSTER_NAME_SETTING = new Setting<>("cluster.name", "elasticsearch", (s) -> {
+    public static final Setting<ClusterName> CLUSTER_NAME_SETTING = new Setting<>("cluster.name", "elasticsearch", (s) -> {
         if (s.isEmpty()) {
             throw new IllegalArgumentException("[cluster.name] must not be empty");
         }
-        return s;
-    }, Property.NodeScope);
+        return new ClusterName(s);
+    }, Setting.Property.NodeScope);
 
+    public static final ClusterName DEFAULT = CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY);
 
-    public static final ClusterName DEFAULT = new ClusterName(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY).intern());
+    private final String value;
 
-    private String value;
-
-    public static ClusterName clusterNameFromSettings(Settings settings) {
-        return new ClusterName(CLUSTER_NAME_SETTING.get(settings));
+    public ClusterName(StreamInput input) throws IOException {
+        this(input.readString());
     }
-
-    private ClusterName() {
-    }
-
     public ClusterName(String value) {
         this.value = value.intern();
     }
@@ -60,17 +52,6 @@ public class ClusterName implements Streamable {
         return this.value;
     }
 
-    public static ClusterName readClusterName(StreamInput in) throws IOException {
-        ClusterName clusterName = new ClusterName();
-        clusterName.readFrom(in);
-        return clusterName;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        value = in.readString().intern();
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(value);
@@ -90,7 +71,7 @@ public class ClusterName implements Streamable {
 
     @Override
     public int hashCode() {
-        return value != null ? value.hashCode() : 0;
+        return Objects.hash(value);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterNameModule.java b/core/src/main/java/org/elasticsearch/cluster/ClusterNameModule.java
deleted file mode 100644
index 55513c3c9be..00000000000
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterNameModule.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cluster;
-
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.settings.Settings;
-
-/**
- *
- */
-public class ClusterNameModule extends AbstractModule {
-
-    private final Settings settings;
-
-    public ClusterNameModule(Settings settings) {
-        this.settings = settings;
-    }
-
-    @Override
-    protected void configure() {
-        bind(ClusterName.class).toInstance(ClusterName.clusterNameFromSettings(settings));
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterState.java b/core/src/main/java/org/elasticsearch/cluster/ClusterState.java
index 17c0b646760..7674874c0ba 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterState.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterState.java
@@ -90,7 +90,7 @@ import java.util.Set;
  */
 public class ClusterState implements ToXContent, Diffable<ClusterState> {
 
-    public static final ClusterState PROTO = builder(ClusterName.DEFAULT).build();
+    public static final ClusterState PROTO = builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).build();
 
     public static enum ClusterStateStatus {
         UNKNOWN((byte) 0),
@@ -734,7 +734,7 @@ public class ClusterState implements ToXContent, Diffable<ClusterState> {
     }
 
     public ClusterState readFrom(StreamInput in, DiscoveryNode localNode) throws IOException {
-        ClusterName clusterName = ClusterName.readClusterName(in);
+        ClusterName clusterName = new ClusterName(in);
         Builder builder = new Builder(clusterName);
         builder.version = in.readLong();
         builder.uuid = in.readString();
@@ -805,7 +805,7 @@ public class ClusterState implements ToXContent, Diffable<ClusterState> {
         }
 
         public ClusterStateDiff(StreamInput in, ClusterState proto) throws IOException {
-            clusterName = ClusterName.readClusterName(in);
+            clusterName = new ClusterName(in);
             fromUuid = in.readString();
             toUuid = in.readString();
             toVersion = in.readLong();
diff --git a/core/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java b/core/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java
index 5d267fa142d..8aeb110c370 100644
--- a/core/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java
+++ b/core/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java
@@ -18,12 +18,9 @@
  */
 package org.elasticsearch.cluster.health;
 
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
-import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -50,16 +47,6 @@ public final class ClusterStateHealth implements Iterable<ClusterIndexHealth>, W
     private final ClusterHealthStatus status;
     private final Map<String, ClusterIndexHealth> indices = new HashMap<>();
 
-    /**
-     * Creates a new <code>ClusterStateHealth</code> instance based on cluster meta data and its routing table as a convenience.
-     *
-     * @param clusterMetaData Current cluster meta data. Must not be null.
-     * @param routingTables   Current routing table. Must not be null.
-     */
-    public ClusterStateHealth(final MetaData clusterMetaData, final RoutingTable routingTables) {
-        this(ClusterState.builder(ClusterName.DEFAULT).metaData(clusterMetaData).routingTable(routingTables).build());
-    }
-
     /**
      * Creates a new <code>ClusterStateHealth</code> instance considering the current cluster state and all indices in the cluster.
      *
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
index 129650de991..19189e0c286 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
@@ -26,7 +26,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationD
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.shard.ShardId;
@@ -42,12 +42,17 @@ import java.util.stream.Collectors;
 
 public class OperationRouting extends AbstractComponent {
 
-    private final AwarenessAllocationDecider awarenessAllocationDecider;
+    private String[] awarenessAttributes;
 
-    @Inject
-    public OperationRouting(Settings settings, AwarenessAllocationDecider awarenessAllocationDecider) {
+    public OperationRouting(Settings settings, ClusterSettings clusterSettings) {
         super(settings);
-        this.awarenessAllocationDecider = awarenessAllocationDecider;
+        this.awarenessAttributes = AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.get(settings);
+        clusterSettings.addSettingsUpdateConsumer(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING,
+            this::setAwarenessAttributes);
+    }
+
+    private void setAwarenessAttributes(String[] awarenessAttributes) {
+        this.awarenessAttributes = awarenessAttributes;
     }
 
     public ShardIterator indexShards(ClusterState clusterState, String index, String id, @Nullable String routing) {
@@ -111,7 +116,6 @@ public class OperationRouting extends AbstractComponent {
 
     private ShardIterator preferenceActiveShardIterator(IndexShardRoutingTable indexShard, String localNodeId, DiscoveryNodes nodes, @Nullable String preference) {
         if (preference == null || preference.isEmpty()) {
-            String[] awarenessAttributes = awarenessAllocationDecider.awarenessAttributes();
             if (awarenessAttributes.length == 0) {
                 return indexShard.activeInitializingShardsRandomIt();
             } else {
@@ -143,7 +147,6 @@ public class OperationRouting extends AbstractComponent {
                 }
                 // no more preference
                 if (index == -1 || index == preference.length() - 1) {
-                    String[] awarenessAttributes = awarenessAllocationDecider.awarenessAttributes();
                     if (awarenessAttributes.length == 0) {
                         return indexShard.activeInitializingShardsRandomIt();
                     } else {
@@ -186,7 +189,6 @@ public class OperationRouting extends AbstractComponent {
             }
         }
         // if not, then use it as the index
-        String[] awarenessAttributes = awarenessAllocationDecider.awarenessAttributes();
         if (awarenessAttributes.length == 0) {
             return indexShard.activeInitializingShardsIt(Murmur3HashFunction.hash(preference));
         } else {
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
index 55954f0f1e4..7a91c694399 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.cluster.routing.allocation;
 
 import org.elasticsearch.cluster.ClusterInfoService;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.health.ClusterStateHealth;
@@ -68,6 +69,7 @@ public class AllocationService extends AbstractComponent {
     private final GatewayAllocator gatewayAllocator;
     private final ShardsAllocator shardsAllocator;
     private final ClusterInfoService clusterInfoService;
+    private final ClusterName clusterName;
 
     @Inject
     public AllocationService(Settings settings, AllocationDeciders allocationDeciders, GatewayAllocator gatewayAllocator,
@@ -77,6 +79,7 @@ public class AllocationService extends AbstractComponent {
         this.gatewayAllocator = gatewayAllocator;
         this.shardsAllocator = shardsAllocator;
         this.clusterInfoService = clusterInfoService;
+        clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
     }
 
     /**
@@ -118,8 +121,10 @@ public class AllocationService extends AbstractComponent {
         MetaData newMetaData = updateMetaDataWithRoutingTable(oldMetaData, oldRoutingTable, newRoutingTable);
         assert newRoutingTable.validate(newMetaData); // validates the routing table is coherent with the cluster state metadata
         logClusterHealthStateChange(
-            new ClusterStateHealth(allocation.metaData(), allocation.routingTable()),
-            new ClusterStateHealth(newMetaData, newRoutingTable),
+            new ClusterStateHealth(ClusterState.builder(clusterName).
+                metaData(allocation.metaData()).routingTable(allocation.routingTable()).build()),
+            new ClusterStateHealth(ClusterState.builder(clusterName).
+                metaData(newMetaData).routingTable(newRoutingTable).build()),
             reason
         );
         return new RoutingAllocation.Result(true, newRoutingTable, newMetaData, explanations);
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
index 1f69b771fdb..046947d64eb 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
@@ -203,7 +203,6 @@ public class BalancedShardsAllocator extends AbstractComponent implements Shards
             final float weightIndex = node.numShards(index) + numAdditionalShards - balancer.avgShardsPerNode(index);
             return theta0 * weightShard + theta1 * weightIndex;
         }
-
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java
index ed7ab3bd68e..32eaa8ddec6 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java
@@ -126,15 +126,6 @@ public class AwarenessAllocationDecider extends AllocationDecider {
         this.forcedAwarenessAttributes = forcedAwarenessAttributes;
     }
 
-    /**
-     * Get the attributes defined by this instance
-     *
-     * @return attributes defined by this instance
-     */
-    public String[] awarenessAttributes() {
-        return this.awarenessAttributes;
-    }
-
     private void setAwarenessAttributes(String[] awarenessAttributes) {
         this.awarenessAttributes = awarenessAttributes;
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
index 13bbccc863a..7573f260a54 100644
--- a/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
@@ -44,7 +44,6 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.ClusterSettings;
@@ -97,6 +96,7 @@ public class ClusterService extends AbstractLifecycleComponent<ClusterService> {
 
     public static final String UPDATE_THREAD_NAME = "clusterService#updateTask";
     private final ThreadPool threadPool;
+    private final ClusterName clusterName;
 
     private BiConsumer<ClusterChangedEvent, Discovery.AckListener> clusterStatePublisher;
 
@@ -130,14 +130,13 @@ public class ClusterService extends AbstractLifecycleComponent<ClusterService> {
 
     private NodeConnectionsService nodeConnectionsService;
 
-    @Inject
-    public ClusterService(Settings settings, OperationRouting operationRouting,
-                          ClusterSettings clusterSettings, ThreadPool threadPool, ClusterName clusterName) {
+    public ClusterService(Settings settings,
+                          ClusterSettings clusterSettings, ThreadPool threadPool) {
         super(settings);
-        this.operationRouting = operationRouting;
+        this.operationRouting = new OperationRouting(settings, clusterSettings);
         this.threadPool = threadPool;
         this.clusterSettings = clusterSettings;
-
+        this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
         // will be replaced on doStart.
         this.clusterState = ClusterState.builder(clusterName).build();
 
@@ -490,6 +489,10 @@ public class ClusterService extends AbstractLifecycleComponent<ClusterService> {
         return true;
     }
 
+    public ClusterName getClusterName() {
+        return clusterName;
+    }
+
     static abstract class SourcePrioritizedRunnable extends PrioritizedRunnable {
         protected final String source;
 
@@ -1039,4 +1042,8 @@ public class ClusterService extends AbstractLifecycleComponent<ClusterService> {
     public ClusterSettings getClusterSettings() {
         return clusterSettings;
     }
+
+    public Settings getSettings() {
+        return settings;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
index 1484ad64018..9de4d557599 100644
--- a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
+++ b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
@@ -168,11 +168,11 @@ public class NetworkModule extends AbstractModule {
     public static final String LOCAL_TRANSPORT = "local";
     public static final String NETTY_TRANSPORT = "netty";
 
-    public static final Setting<String> HTTP_TYPE_SETTING = Setting.simpleString("http.type", Property.NodeScope);
+    public static final Setting<String> HTTP_TYPE_SETTING = Setting.simpleString(HTTP_TYPE_KEY, Property.NodeScope);
     public static final Setting<Boolean> HTTP_ENABLED = Setting.boolSetting("http.enabled", true, Property.NodeScope);
     public static final Setting<String> TRANSPORT_SERVICE_TYPE_SETTING =
-        Setting.simpleString("transport.service.type", Property.NodeScope);
-    public static final Setting<String> TRANSPORT_TYPE_SETTING = Setting.simpleString("transport.type", Property.NodeScope);
+        Setting.simpleString(TRANSPORT_SERVICE_TYPE_KEY, Property.NodeScope);
+    public static final Setting<String> TRANSPORT_TYPE_SETTING = Setting.simpleString(TRANSPORT_TYPE_KEY, Property.NodeScope);
 
 
 
diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index de2d3b75df6..5cb37d14858 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -24,11 +24,11 @@ import org.elasticsearch.action.support.AutoCreateIndex;
 import org.elasticsearch.action.support.DestructiveOperations;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
 import org.elasticsearch.bootstrap.BootstrapSettings;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.transport.TransportClientNodesService;
 import org.elasticsearch.cluster.ClusterModule;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.InternalClusterInfoService;
 import org.elasticsearch.cluster.NodeConnectionsService;
 import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
diff --git a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
index 03f27d093e8..696eb8812c9 100644
--- a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
@@ -75,9 +75,9 @@ public class LocalDiscovery extends AbstractLifecycleComponent<Discovery> implem
     private volatile ClusterState lastProcessedClusterState;
 
     @Inject
-    public LocalDiscovery(Settings settings, ClusterName clusterName, ClusterService clusterService, ClusterSettings clusterSettings) {
+    public LocalDiscovery(Settings settings, ClusterService clusterService, ClusterSettings clusterSettings) {
         super(settings);
-        this.clusterName = clusterName;
+        this.clusterName = clusterService.getClusterName();
         this.clusterService = clusterService;
         this.discoverySettings = new DiscoverySettings(settings, clusterSettings);
     }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
index 3c949b768d1..54ef9928585 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
@@ -150,12 +150,12 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
     private volatile NodeJoinController nodeJoinController;
 
     @Inject
-    public ZenDiscovery(Settings settings, ClusterName clusterName, ThreadPool threadPool,
+    public ZenDiscovery(Settings settings, ThreadPool threadPool,
                         TransportService transportService, final ClusterService clusterService, ClusterSettings clusterSettings,
                         ZenPingService pingService, ElectMasterService electMasterService) {
         super(settings);
-        this.clusterName = clusterName;
         this.clusterService = clusterService;
+        this.clusterName = clusterService.getClusterName();
         this.transportService = transportService;
         this.discoverySettings = new DiscoverySettings(settings, clusterSettings);
         this.pingService = pingService;
@@ -182,10 +182,10 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
             }
         });
 
-        this.masterFD = new MasterFaultDetection(settings, threadPool, transportService, clusterName, clusterService);
+        this.masterFD = new MasterFaultDetection(settings, threadPool, transportService, clusterService);
         this.masterFD.addListener(new MasterNodeFailureListener());
 
-        this.nodesFD = new NodesFaultDetection(settings, threadPool, transportService, clusterName);
+        this.nodesFD = new NodesFaultDetection(settings, threadPool, transportService, clusterService.getClusterName());
         this.nodesFD.addListener(new NodeFaultDetectionListener());
 
         this.publishClusterState =
@@ -195,7 +195,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
                         clusterService::state,
                         new NewPendingClusterStateListener(),
                         discoverySettings,
-                        clusterName);
+                        clusterService.getClusterName());
         this.pingService.setPingContextProvider(this);
         this.membership = new MembershipAction(settings, clusterService, transportService, this, new MembershipListener());
 
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
index d6d523508e1..807f8067490 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
@@ -76,8 +76,8 @@ public class MasterFaultDetection extends FaultDetection {
     private final AtomicBoolean notifiedMasterFailure = new AtomicBoolean();
 
     public MasterFaultDetection(Settings settings, ThreadPool threadPool, TransportService transportService,
-                                ClusterName clusterName, ClusterService clusterService) {
-        super(settings, threadPool, transportService, clusterName);
+                                ClusterService clusterService) {
+        super(settings, threadPool, transportService, clusterService.getClusterName());
         this.clusterService = clusterService;
 
         logger.debug("[master] uses ping_interval [{}], ping_timeout [{}], ping_retries [{}]", pingInterval, pingRetryTimeout, pingRetryCount);
@@ -418,7 +418,7 @@ public class MasterFaultDetection extends FaultDetection {
             super.readFrom(in);
             nodeId = in.readString();
             masterNodeId = in.readString();
-            clusterName = ClusterName.readClusterName(in);
+            clusterName = new ClusterName(in);
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java b/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java
index 74fdfe62118..6fb1c82b268 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java
@@ -314,7 +314,7 @@ public class NodesFaultDetection extends FaultDetection {
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
             nodeId = in.readString();
-            clusterName = ClusterName.readClusterName(in);
+            clusterName = new ClusterName(in);
             masterNode = new DiscoveryNode(in);
             clusterStateVersion = in.readLong();
         }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ping/ZenPing.java b/core/src/main/java/org/elasticsearch/discovery/zen/ping/ZenPing.java
index e28efa044d6..06db6ab3d5c 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ping/ZenPing.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ping/ZenPing.java
@@ -32,11 +32,6 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
 
-import static org.elasticsearch.cluster.ClusterName.readClusterName;
-
-/**
- *
- */
 public interface ZenPing extends LifecycleComponent<ZenPing> {
 
     void setPingContextProvider(PingContextProvider contextProvider);
@@ -118,7 +113,7 @@ public interface ZenPing extends LifecycleComponent<ZenPing> {
 
         @Override
         public void readFrom(StreamInput in) throws IOException {
-            clusterName = readClusterName(in);
+            clusterName = new ClusterName(in);
             node = new DiscoveryNode(in);
             if (in.readBoolean()) {
                 master = new DiscoveryNode(in);
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
index 1b4f1ba75b0..0d3c7f9d086 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
@@ -131,13 +131,13 @@ public class UnicastZenPing extends AbstractLifecycleComponent<ZenPing> implemen
     private volatile boolean closed = false;
 
     @Inject
-    public UnicastZenPing(Settings settings, ThreadPool threadPool, TransportService transportService, ClusterName clusterName,
+    public UnicastZenPing(Settings settings, ThreadPool threadPool, TransportService transportService,
                           Version version, ElectMasterService electMasterService,
                           @Nullable Set<UnicastHostsProvider> unicastHostsProviders) {
         super(settings);
         this.threadPool = threadPool;
         this.transportService = transportService;
-        this.clusterName = clusterName;
+        this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
         this.electMasterService = electMasterService;
 
         if (unicastHostsProviders != null) {
@@ -206,10 +206,6 @@ public class UnicastZenPing extends AbstractLifecycleComponent<ZenPing> implemen
         hostsProviders.add(provider);
     }
 
-    public void removeHostsProvider(UnicastHostsProvider provider) {
-        hostsProviders.remove(provider);
-    }
-
     @Override
     public void setPingContextProvider(PingContextProvider contextProvider) {
         this.contextProvider = contextProvider;
@@ -222,16 +218,13 @@ public class UnicastZenPing extends AbstractLifecycleComponent<ZenPing> implemen
         temporalResponses.clear();
     }
 
-    public PingResponse[] pingAndWait(TimeValue timeout) {
+    public PingResponse[] pingAndWait(TimeValue duration) {
         final AtomicReference<PingResponse[]> response = new AtomicReference<>();
         final CountDownLatch latch = new CountDownLatch(1);
-        ping(new PingListener() {
-            @Override
-            public void onPing(PingResponse[] pings) {
-                response.set(pings);
-                latch.countDown();
-            }
-        }, timeout);
+        ping(pings -> {
+            response.set(pings);
+            latch.countDown();
+        }, duration);
         try {
             latch.await();
             return response.get();
@@ -241,25 +234,26 @@ public class UnicastZenPing extends AbstractLifecycleComponent<ZenPing> implemen
     }
 
     @Override
-    public void ping(final PingListener listener, final TimeValue timeout) {
+    public void ping(final PingListener listener, final TimeValue duration) {
         final SendPingsHandler sendPingsHandler = new SendPingsHandler(pingHandlerIdGenerator.incrementAndGet());
         try {
             receivedResponses.put(sendPingsHandler.id(), sendPingsHandler);
             try {
-                sendPings(timeout, null, sendPingsHandler);
+                sendPings(duration, null, sendPingsHandler);
             } catch (RejectedExecutionException e) {
                 logger.debug("Ping execution rejected", e);
                 // The RejectedExecutionException can come from the fact unicastConnectExecutor is at its max down in sendPings
                 // But don't bail here, we can retry later on after the send ping has been scheduled.
             }
-            threadPool.schedule(TimeValue.timeValueMillis(timeout.millis() / 2), ThreadPool.Names.GENERIC, new AbstractRunnable() {
+
+            threadPool.schedule(TimeValue.timeValueMillis(duration.millis() / 2), ThreadPool.Names.GENERIC, new AbstractRunnable() {
                 @Override
                 protected void doRun() {
-                    sendPings(timeout, null, sendPingsHandler);
-                    threadPool.schedule(TimeValue.timeValueMillis(timeout.millis() / 2), ThreadPool.Names.GENERIC, new AbstractRunnable() {
+                    sendPings(duration, null, sendPingsHandler);
+                    threadPool.schedule(TimeValue.timeValueMillis(duration.millis() / 2), ThreadPool.Names.GENERIC, new AbstractRunnable() {
                         @Override
                         protected void doRun() throws Exception {
-                            sendPings(timeout, TimeValue.timeValueMillis(timeout.millis() / 2), sendPingsHandler);
+                            sendPings(duration, TimeValue.timeValueMillis(duration.millis() / 2), sendPingsHandler);
                             sendPingsHandler.close();
                             listener.onPing(sendPingsHandler.pingCollection().toArray());
                             for (DiscoveryNode node : sendPingsHandler.nodeToDisconnect) {
diff --git a/core/src/main/java/org/elasticsearch/env/Environment.java b/core/src/main/java/org/elasticsearch/env/Environment.java
index 6bec05fc82b..8c8b72bff63 100644
--- a/core/src/main/java/org/elasticsearch/env/Environment.java
+++ b/core/src/main/java/org/elasticsearch/env/Environment.java
@@ -130,16 +130,17 @@ public class Environment {
         pluginsFile = homeFile.resolve("plugins");
 
         List<String> dataPaths = PATH_DATA_SETTING.get(settings);
+        final ClusterName clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
         if (dataPaths.isEmpty() == false) {
             dataFiles = new Path[dataPaths.size()];
             dataWithClusterFiles = new Path[dataPaths.size()];
             for (int i = 0; i < dataPaths.size(); i++) {
                 dataFiles[i] = PathUtils.get(dataPaths.get(i));
-                dataWithClusterFiles[i] = dataFiles[i].resolve(ClusterName.clusterNameFromSettings(settings).value());
+                dataWithClusterFiles[i] = dataFiles[i].resolve(clusterName.value());
             }
         } else {
             dataFiles = new Path[]{homeFile.resolve("data")};
-            dataWithClusterFiles = new Path[]{homeFile.resolve("data").resolve(ClusterName.clusterNameFromSettings(settings).value())};
+            dataWithClusterFiles = new Path[]{homeFile.resolve("data").resolve(clusterName.value())};
         }
         if (PATH_SHARED_DATA_SETTING.exists(settings)) {
             sharedDataFile = PathUtils.get(cleanPath(PATH_SHARED_DATA_SETTING.get(settings)));
diff --git a/core/src/main/java/org/elasticsearch/gateway/Gateway.java b/core/src/main/java/org/elasticsearch/gateway/Gateway.java
index 0e6b1959279..af86633e235 100644
--- a/core/src/main/java/org/elasticsearch/gateway/Gateway.java
+++ b/core/src/main/java/org/elasticsearch/gateway/Gateway.java
@@ -155,7 +155,7 @@ public class Gateway extends AbstractComponent implements ClusterStateListener {
         final ClusterSettings clusterSettings = clusterService.getClusterSettings();
         metaDataBuilder.persistentSettings(clusterSettings.archiveUnknownOrBrokenSettings(metaDataBuilder.persistentSettings()));
         metaDataBuilder.transientSettings(clusterSettings.archiveUnknownOrBrokenSettings(metaDataBuilder.transientSettings()));
-        ClusterState.Builder builder = ClusterState.builder(clusterService.state().getClusterName());
+        ClusterState.Builder builder = ClusterState.builder(clusterService.getClusterName());
         builder.metaData(metaDataBuilder);
         listener.onSuccess(builder.build());
     }
diff --git a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java
index 2d0b894939d..24886dc72d3 100644
--- a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java
+++ b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java
@@ -58,10 +58,10 @@ public class TransportNodesListGatewayMetaState extends TransportNodesAction<Tra
     private GatewayMetaState metaState;
 
     @Inject
-    public TransportNodesListGatewayMetaState(Settings settings, ClusterName clusterName, ThreadPool threadPool,
+    public TransportNodesListGatewayMetaState(Settings settings, ThreadPool threadPool,
                                               ClusterService clusterService, TransportService transportService,
                                               ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, ACTION_NAME, clusterName, threadPool, clusterService, transportService, actionFilters,
+        super(settings, ACTION_NAME, threadPool, clusterService, transportService, actionFilters,
               indexNameExpressionResolver, Request::new, NodeRequest::new, ThreadPool.Names.GENERIC, NodeGatewayMetaState.class);
     }
 
@@ -91,7 +91,7 @@ public class TransportNodesListGatewayMetaState extends TransportNodesAction<Tra
 
     @Override
     protected NodesGatewayMetaState newResponse(Request request, List<NodeGatewayMetaState> responses, List<FailedNodeException> failures) {
-        return new NodesGatewayMetaState(clusterName, responses, failures);
+        return new NodesGatewayMetaState(clusterService.getClusterName(), responses, failures);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
index 675c0088082..5ae75a209eb 100644
--- a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
+++ b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
@@ -69,12 +69,12 @@ public class TransportNodesListGatewayStartedShards extends
 
 
     @Inject
-    public TransportNodesListGatewayStartedShards(Settings settings, ClusterName clusterName, ThreadPool threadPool,
+    public TransportNodesListGatewayStartedShards(Settings settings, ThreadPool threadPool,
                                                   ClusterService clusterService, TransportService transportService,
                                                   ActionFilters actionFilters,
                                                   IndexNameExpressionResolver indexNameExpressionResolver,
                                                   NodeEnvironment env) {
-        super(settings, ACTION_NAME, clusterName, threadPool, clusterService, transportService, actionFilters,
+        super(settings, ACTION_NAME, threadPool, clusterService, transportService, actionFilters,
               indexNameExpressionResolver, Request::new, NodeRequest::new, ThreadPool.Names.FETCH_SHARD_STARTED,
               NodeGatewayStartedShards.class);
         this.nodeEnv = env;
@@ -111,7 +111,7 @@ public class TransportNodesListGatewayStartedShards extends
     @Override
     protected NodesGatewayStartedShards newResponse(Request request,
                                                     List<NodeGatewayStartedShards> responses, List<FailedNodeException> failures) {
-        return new NodesGatewayStartedShards(clusterName, responses, failures);
+        return new NodesGatewayStartedShards(clusterService.getClusterName(), responses, failures);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java b/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
index 3402349743f..8766d23d3ee 100644
--- a/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
+++ b/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
@@ -351,7 +351,7 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe
         }
 
         private IndexShard getShard(ShardActiveRequest request) {
-            ClusterName thisClusterName = clusterService.state().getClusterName();
+            ClusterName thisClusterName = clusterService.getClusterName();
             if (!thisClusterName.equals(request.clusterName)) {
                 logger.trace("shard exists request meant for cluster[{}], but this is cluster[{}], ignoring request", request.clusterName, thisClusterName);
                 return null;
@@ -385,7 +385,7 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe
         @Override
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
-            clusterName = ClusterName.readClusterName(in);
+            clusterName = new ClusterName(in);
             indexUUID = in.readString();
             shardId = ShardId.readShardId(in);
             timeout = new TimeValue(in.readLong(), TimeUnit.MILLISECONDS);
diff --git a/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java b/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
index 92c9bd3b575..da5f9d16f51 100644
--- a/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
+++ b/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
@@ -76,11 +76,11 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
     private final NodeEnvironment nodeEnv;
 
     @Inject
-    public TransportNodesListShardStoreMetaData(Settings settings, ClusterName clusterName, ThreadPool threadPool,
+    public TransportNodesListShardStoreMetaData(Settings settings, ThreadPool threadPool,
                                                 ClusterService clusterService, TransportService transportService,
                                                 IndicesService indicesService, NodeEnvironment nodeEnv, ActionFilters actionFilters,
                                                 IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, ACTION_NAME, clusterName, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
+        super(settings, ACTION_NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
               Request::new, NodeRequest::new, ThreadPool.Names.FETCH_SHARD_STORE, NodeStoreFilesMetaData.class);
         this.indicesService = indicesService;
         this.nodeEnv = nodeEnv;
@@ -111,7 +111,7 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
     @Override
     protected NodesStoreFilesMetaData newResponse(Request request,
                                                   List<NodeStoreFilesMetaData> responses, List<FailedNodeException> failures) {
-        return new NodesStoreFilesMetaData(clusterName, responses, failures);
+        return new NodesStoreFilesMetaData(clusterService.getClusterName(), responses, failures);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index 87bfd99b900..f34e16db140 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -29,7 +29,6 @@ import org.elasticsearch.action.ActionModule;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.node.NodeClientModule;
 import org.elasticsearch.cluster.ClusterModule;
-import org.elasticsearch.cluster.ClusterNameModule;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.MasterNodeChangePredicate;
@@ -243,6 +242,8 @@ public class Node implements Closeable {
                 throw new IllegalStateException("Failed to created node environment", ex);
             }
             final NetworkService networkService = new NetworkService(settings);
+            final ClusterService clusterService = new ClusterService(settings, settingsModule.getClusterSettings(), threadPool);
+            resourcesToClose.add(clusterService);
             NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
             ModulesBuilder modules = new ModulesBuilder();
             modules.add(new Version.Module(version));
@@ -257,9 +258,8 @@ public class Node implements Closeable {
             modules.add(new NetworkModule(networkService, settings, false, namedWriteableRegistry));
             modules.add(scriptModule);
             modules.add(new NodeEnvironmentModule(nodeEnvironment));
-            modules.add(new ClusterNameModule(this.settings));
             modules.add(new DiscoveryModule(this.settings));
-            modules.add(new ClusterModule(this.settings));
+            modules.add(new ClusterModule(this.settings, clusterService));
             modules.add(new IndicesModule());
             modules.add(new SearchModule(settings, namedWriteableRegistry));
             modules.add(new ActionModule(DiscoveryNode.isIngestNode(settings), false));
diff --git a/core/src/main/java/org/elasticsearch/transport/TransportService.java b/core/src/main/java/org/elasticsearch/transport/TransportService.java
index c8f7be07b94..21a43560eba 100644
--- a/core/src/main/java/org/elasticsearch/transport/TransportService.java
+++ b/core/src/main/java/org/elasticsearch/transport/TransportService.java
@@ -77,7 +77,7 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
     private final CountDownLatch blockIncomingRequestsLatch = new CountDownLatch(1);
     protected final Transport transport;
     protected final ThreadPool threadPool;
-    private final ClusterName clusterName;
+    protected final ClusterName clusterName;
     protected final TaskManager taskManager;
 
     volatile Map<String, RequestHandlerRegistry> requestHandlers = Collections.emptyMap();
@@ -117,16 +117,12 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
     /** if set will call requests sent to this id to shortcut and executed locally */
     volatile DiscoveryNode localNode = null;
 
-    public TransportService(Transport transport, ThreadPool threadPool, ClusterName clusterName) {
-        this(EMPTY_SETTINGS, transport, threadPool, clusterName);
-    }
-
     @Inject
-    public TransportService(Settings settings, Transport transport, ThreadPool threadPool, ClusterName clusterName) {
+    public TransportService(Settings settings, Transport transport, ThreadPool threadPool) {
         super(settings);
         this.transport = transport;
         this.threadPool = threadPool;
-        this.clusterName = clusterName;
+        this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
         setTracerLogInclude(TRACE_LOG_INCLUDE_SETTING.get(settings));
         setTracerLogExclude(TRACE_LOG_EXCLUDE_SETTING.get(settings));
         tracerLog = Loggers.getLogger(logger, ".tracer");
@@ -397,7 +393,7 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
             discoveryNode = in.readOptionalWriteable(DiscoveryNode::new);
-            clusterName = ClusterName.readClusterName(in);
+            clusterName = new ClusterName(in);
             version = Version.readVersion(in);
         }
 
@@ -1054,5 +1050,4 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
         }
 
     }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java
index 99735a6ffec..704c1348b7e 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java
@@ -25,6 +25,7 @@ import org.elasticsearch.cluster.health.ClusterStateHealth;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
@@ -52,7 +53,7 @@ public class ClusterHealthResponsesTests extends ESTestCase {
     }
 
     public void testClusterHealth() throws IOException {
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).build();
         int pendingTasks = randomIntBetween(0, 200);
         int inFlight = randomIntBetween(0, 200);
         int delayedUnassigned = randomIntBetween(0, 200);
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
index b5e113dfcd3..594028f4e6f 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
@@ -28,7 +28,6 @@ import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.action.support.nodes.BaseNodeRequest;
 import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -140,10 +139,10 @@ public class CancellableTasksTests extends TaskManagerTestCase {
 
         final CountDownLatch actionStartedLatch;
 
-        CancellableTestNodesAction(Settings settings, String actionName, ClusterName clusterName, ThreadPool threadPool,
+        CancellableTestNodesAction(Settings settings, String actionName, ThreadPool threadPool,
                                    ClusterService clusterService, TransportService transportService, boolean shouldBlock, CountDownLatch
                                        actionStartedLatch) {
-            super(settings, actionName, clusterName, threadPool, clusterService, transportService, CancellableNodesRequest::new,
+            super(settings, actionName, threadPool, clusterService, transportService, CancellableNodesRequest::new,
                 CancellableNodeRequest::new);
             this.shouldBlock = shouldBlock;
             this.actionStartedLatch = actionStartedLatch;
@@ -202,7 +201,7 @@ public class CancellableTasksTests extends TaskManagerTestCase {
         for (int i = 0; i < testNodes.length; i++) {
             boolean shouldBlock = blockOnNodes.contains(testNodes[i]);
             logger.info("The action in the node [{}] should block: [{}]", testNodes[i].discoveryNode.getId(), shouldBlock);
-            actions[i] = new CancellableTestNodesAction(Settings.EMPTY, "testAction", CLUSTER_NAME, threadPool, testNodes[i]
+            actions[i] = new CancellableTestNodesAction(CLUSTER_SETTINGS, "testAction", threadPool, testNodes[i]
                 .clusterService, testNodes[i].transportService, shouldBlock, actionLatch);
         }
         Task task = actions[0].execute(request, listener);
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
index 95e8cde0a22..e317d4e6d7a 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
@@ -67,7 +67,7 @@ import static org.elasticsearch.test.ClusterServiceUtils.setState;
 public abstract class TaskManagerTestCase extends ESTestCase {
 
     protected static ThreadPool threadPool;
-    public static final ClusterName CLUSTER_NAME = new ClusterName("test-cluster");
+    public static final Settings CLUSTER_SETTINGS = Settings.builder().put("cluster.name", "test-cluster").build();
     protected TestNode[] testNodes;
     protected int nodesCount;
 
@@ -136,17 +136,17 @@ public abstract class TaskManagerTestCase extends ESTestCase {
     abstract class AbstractTestNodesAction<NodesRequest extends BaseNodesRequest<NodesRequest>, NodeRequest extends BaseNodeRequest>
             extends TransportNodesAction<NodesRequest, NodesResponse, NodeRequest, NodeResponse> {
 
-        AbstractTestNodesAction(Settings settings, String actionName, ClusterName clusterName, ThreadPool threadPool,
+        AbstractTestNodesAction(Settings settings, String actionName, ThreadPool threadPool,
                                 ClusterService clusterService, TransportService transportService, Supplier<NodesRequest> request,
                                 Supplier<NodeRequest> nodeRequest) {
-            super(settings, actionName, clusterName, threadPool, clusterService, transportService,
+            super(settings, actionName, threadPool, clusterService, transportService,
                     new ActionFilters(new HashSet<>()), new IndexNameExpressionResolver(Settings.EMPTY),
                     request, nodeRequest, ThreadPool.Names.GENERIC, NodeResponse.class);
         }
 
         @Override
         protected NodesResponse newResponse(NodesRequest request, List<NodeResponse> responses, List<FailedNodeException> failures) {
-            return new NodesResponse(clusterName, responses, failures);
+            return new NodesResponse(clusterService.getClusterName(), responses, failures);
         }
 
         @Override
@@ -166,10 +166,9 @@ public abstract class TaskManagerTestCase extends ESTestCase {
     public static class TestNode implements Releasable {
         public TestNode(String name, ThreadPool threadPool, Settings settings) {
             clusterService = createClusterService(threadPool);
-            ClusterName clusterName = clusterService.state().getClusterName();
             transportService = new TransportService(settings,
                     new LocalTransport(settings, threadPool, Version.CURRENT, new NamedWriteableRegistry(),
-                        new NoneCircuitBreakerService()), threadPool, clusterName) {
+                        new NoneCircuitBreakerService()), threadPool) {
                 @Override
                 protected TaskManager createTaskManager() {
                     if (MockTaskManager.USE_MOCK_TASK_MANAGER_SETTING.get(settings)) {
@@ -185,9 +184,9 @@ public abstract class TaskManagerTestCase extends ESTestCase {
                     emptyMap(), emptySet(), Version.CURRENT);
             IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(settings);
             ActionFilters actionFilters = new ActionFilters(emptySet());
-            transportListTasksAction = new TransportListTasksAction(settings, CLUSTER_NAME, threadPool, clusterService, transportService,
+            transportListTasksAction = new TransportListTasksAction(settings, threadPool, clusterService, transportService,
                     actionFilters, indexNameExpressionResolver);
-            transportCancelTasksAction = new TransportCancelTasksAction(settings, CLUSTER_NAME, threadPool, clusterService,
+            transportCancelTasksAction = new TransportCancelTasksAction(settings, threadPool, clusterService,
                     transportService, actionFilters, indexNameExpressionResolver);
             transportService.acceptIncomingRequests();
         }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
index 11b83d67a8c..8f8c10c8dda 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
@@ -246,9 +246,9 @@ public class TestTaskPlugin extends Plugin {
     public static class TransportTestTaskAction extends TransportNodesAction<NodesRequest, NodesResponse, NodeRequest, NodeResponse> {
 
         @Inject
-        public TransportTestTaskAction(Settings settings, ClusterName clusterName, ThreadPool threadPool,
+        public TransportTestTaskAction(Settings settings, ThreadPool threadPool,
                                        ClusterService clusterService, TransportService transportService) {
-            super(settings, TestTaskAction.NAME, clusterName, threadPool, clusterService, transportService,
+            super(settings, TestTaskAction.NAME, threadPool, clusterService, transportService,
                   new ActionFilters(new HashSet<>()), new IndexNameExpressionResolver(Settings.EMPTY),
                   NodesRequest::new, NodeRequest::new, ThreadPool.Names.GENERIC, NodeResponse.class);
         }
@@ -258,7 +258,7 @@ public class TestTaskPlugin extends Plugin {
             if (request.getShouldFail()) {
                 throw new IllegalStateException("Simulating operation failure");
             }
-            return new NodesResponse(clusterName, responses, failures);
+            return new NodesResponse(clusterService.getClusterName(), responses, failures);
         }
 
         @Override
@@ -425,10 +425,10 @@ public class TestTaskPlugin extends Plugin {
         UnblockTestTasksResponse, UnblockTestTaskResponse> {
 
         @Inject
-        public TransportUnblockTestTasksAction(Settings settings, ClusterName clusterName, ThreadPool threadPool, ClusterService
+        public TransportUnblockTestTasksAction(Settings settings,ThreadPool threadPool, ClusterService
             clusterService,
                                                TransportService transportService) {
-            super(settings, UnblockTestTasksAction.NAME, clusterName, threadPool, clusterService, transportService, new ActionFilters(new
+            super(settings, UnblockTestTasksAction.NAME, threadPool, clusterService, transportService, new ActionFilters(new
                 HashSet<>()), new IndexNameExpressionResolver(Settings.EMPTY),
                 UnblockTestTasksRequest::new, UnblockTestTasksResponse::new, ThreadPool.Names.MANAGEMENT);
         }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
index 326e909b3ff..c4d49d899b9 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
@@ -35,7 +35,6 @@ import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.action.support.tasks.BaseTasksRequest;
 import org.elasticsearch.action.support.tasks.BaseTasksResponse;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -170,9 +169,9 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
      */
     abstract class TestNodesAction extends AbstractTestNodesAction<NodesRequest, NodeRequest> {
 
-        TestNodesAction(Settings settings, String actionName, ClusterName clusterName, ThreadPool threadPool,
+        TestNodesAction(Settings settings, String actionName, ThreadPool threadPool,
                         ClusterService clusterService, TransportService transportService) {
-            super(settings, actionName, clusterName, threadPool, clusterService, transportService, NodesRequest::new, NodeRequest::new);
+            super(settings, actionName, threadPool, clusterService, transportService, NodesRequest::new, NodeRequest::new);
         }
 
         @Override
@@ -252,9 +251,9 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
      */
     static abstract class TestTasksAction extends TransportTasksAction<Task, TestTasksRequest, TestTasksResponse, TestTaskResponse> {
 
-        protected TestTasksAction(Settings settings, String actionName, ClusterName clusterName, ThreadPool threadPool,
+        protected TestTasksAction(Settings settings, String actionName, ThreadPool threadPool,
                 ClusterService clusterService, TransportService transportService) {
-            super(settings, actionName, clusterName, threadPool, clusterService, transportService, new ActionFilters(new HashSet<>()),
+            super(settings, actionName, threadPool, clusterService, transportService, new ActionFilters(new HashSet<>()),
                     new IndexNameExpressionResolver(Settings.EMPTY), TestTasksRequest::new, TestTasksResponse::new,
                     ThreadPool.Names.MANAGEMENT);
         }
@@ -298,7 +297,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
         TestNodesAction[] actions = new TestNodesAction[nodesCount];
         for (int i = 0; i < testNodes.length; i++) {
             final int node = i;
-            actions[i] = new TestNodesAction(Settings.EMPTY, "testAction", CLUSTER_NAME, threadPool, testNodes[i].clusterService,
+            actions[i] = new TestNodesAction(CLUSTER_SETTINGS, "testAction", threadPool, testNodes[i].clusterService,
                     testNodes[i].transportService) {
                 @Override
                 protected NodeResponse nodeOperation(NodeRequest request) {
@@ -583,7 +582,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
         RecordingTaskManagerListener[] listeners = setupListeners(testNodes, "testAction*");
         for (int i = 0; i < testNodes.length; i++) {
             final int node = i;
-            actions[i] = new TestNodesAction(Settings.EMPTY, "testAction", CLUSTER_NAME, threadPool, testNodes[i].clusterService,
+            actions[i] = new TestNodesAction(CLUSTER_SETTINGS, "testAction", threadPool, testNodes[i].clusterService,
                     testNodes[i].transportService) {
                 @Override
                 protected NodeResponse nodeOperation(NodeRequest request) {
@@ -623,7 +622,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
         for (int i = 0; i < testNodes.length; i++) {
             final int node = i;
             // Simulate task action that fails on one of the tasks on one of the nodes
-            tasksActions[i] = new TestTasksAction(Settings.EMPTY, "testTasksAction", CLUSTER_NAME, threadPool, testNodes[i].clusterService,
+            tasksActions[i] = new TestTasksAction(CLUSTER_SETTINGS, "testTasksAction", threadPool, testNodes[i].clusterService,
                     testNodes[i].transportService) {
                 @Override
                 protected TestTaskResponse taskOperation(TestTasksRequest request, Task task) {
@@ -681,7 +680,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
             final int node = i;
             // Simulate a task action that works on all nodes except nodes listed in filterNodes.
             // We are testing that it works.
-            tasksActions[i] = new TestTasksAction(Settings.EMPTY, "testTasksAction", CLUSTER_NAME, threadPool,
+            tasksActions[i] = new TestTasksAction(CLUSTER_SETTINGS, "testTasksAction", threadPool,
                 testNodes[i].clusterService, testNodes[i].transportService) {
 
                 @Override
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
index c11bf0b9ca6..00fcbf60a5a 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
@@ -163,7 +163,8 @@ public class ClusterRerouteTests extends ESAllocationTestCase {
         routingTableBuilder.addAsNew(metaData.index("idx"));
 
         RoutingTable routingTable = routingTableBuilder.build();
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING
+            .getDefault(Settings.EMPTY))
             .metaData(metaData).routingTable(routingTable).build();
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")))
             .build();
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java
index 01be8da8944..d78374d446f 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java
@@ -22,6 +22,7 @@ package org.elasticsearch.action.admin.indices.shrink;
 import org.apache.lucene.index.IndexWriter;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
 import org.elasticsearch.cluster.block.ClusterBlocks;
@@ -61,7 +62,7 @@ public class TransportShrinkActionTests extends ESTestCase {
         routingTableBuilder.addAsNew(metaData.index(name));
 
         RoutingTable routingTable = routingTableBuilder.build();
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metaData(metaData).routingTable(routingTable).blocks(ClusterBlocks.builder().addBlocks(indexMetaData)).build();
         return clusterState;
     }
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
index 908fb2ddd4d..6ae7559ba62 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
@@ -87,7 +87,7 @@ public class TransportBulkActionTookTests extends ESTestCase {
 
     private TransportBulkAction createAction(boolean controlled, AtomicLong expected) {
         CapturingTransport capturingTransport = new CapturingTransport();
-        TransportService transportService = new TransportService(capturingTransport, threadPool, clusterService.state().getClusterName());
+        TransportService transportService = new TransportService(clusterService.getSettings(), capturingTransport, threadPool);
         transportService.start();
         transportService.acceptIncomingRequests();
         IndexNameExpressionResolver resolver = new Resolver(Settings.EMPTY);
diff --git a/core/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java b/core/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java
index 40359e1c749..9572a2df652 100644
--- a/core/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/AutoCreateIndexTests.java
@@ -142,6 +142,6 @@ public class AutoCreateIndexTests extends ESTestCase {
         for (String index : indices) {
             metaData.put(IndexMetaData.builder(index).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1));
         }
-        return ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).build();
+        return ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).build();
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
index 07ddfa8e49f..1d65f277e3c 100644
--- a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
@@ -191,7 +191,7 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase {
         super.setUp();
         transport = new CapturingTransport();
         clusterService = createClusterService(THREAD_POOL);
-        final TransportService transportService = new TransportService(transport, THREAD_POOL, clusterService.state().getClusterName());
+        final TransportService transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL);
         transportService.start();
         transportService.acceptIncomingRequests();
         setClusterState(clusterService, TEST_INDEX);
diff --git a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
index d6edb972ef4..32fe6b1e408 100644
--- a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
@@ -86,7 +86,7 @@ public class TransportMasterNodeActionTests extends ESTestCase {
         super.setUp();
         transport = new CapturingTransport();
         clusterService = createClusterService(threadPool);
-        transportService = new TransportService(transport, threadPool, clusterService.state().getClusterName());
+        transportService = new TransportService(clusterService.getSettings(), transport, threadPool);
         transportService.start();
         transportService.acceptIncomingRequests();
         localNode = new DiscoveryNode("local_node", DummyTransportAddress.INSTANCE, Collections.emptyMap(),
diff --git a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
index 1b0a6f8d622..626850fd119 100644
--- a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
@@ -63,7 +63,6 @@ import static org.mockito.Mockito.mock;
 public class TransportNodesActionTests extends ESTestCase {
 
     private static ThreadPool THREAD_POOL;
-    private static ClusterName CLUSTER_NAME = new ClusterName("test-cluster");
 
     private ClusterService clusterService;
     private CapturingTransport transport;
@@ -178,7 +177,7 @@ public class TransportNodesActionTests extends ESTestCase {
         super.setUp();
         transport = new CapturingTransport();
         clusterService = createClusterService(THREAD_POOL);
-        transportService = new TransportService(transport, THREAD_POOL, clusterService.state().getClusterName());
+        transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL);
         transportService.start();
         transportService.acceptIncomingRequests();
         int numNodes = randomIntBetween(3, 10);
@@ -196,7 +195,7 @@ public class TransportNodesActionTests extends ESTestCase {
         }
         discoBuilder.localNodeId(randomFrom(discoveryNodes).getId());
         discoBuilder.masterNodeId(randomFrom(discoveryNodes).getId());
-        ClusterState.Builder stateBuilder = ClusterState.builder(CLUSTER_NAME);
+        ClusterState.Builder stateBuilder = ClusterState.builder(clusterService.getClusterName());
         stateBuilder.nodes(discoBuilder);
         ClusterState clusterState = stateBuilder.build();
         setState(clusterService, clusterState);
@@ -246,14 +245,14 @@ public class TransportNodesActionTests extends ESTestCase {
         TestTransportNodesAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService
                 transportService, ActionFilters actionFilters, Supplier<TestNodesRequest> request,
                                  Supplier<TestNodeRequest> nodeRequest, String nodeExecutor) {
-            super(settings, "indices:admin/test", CLUSTER_NAME, threadPool, clusterService, transportService, actionFilters,
+            super(settings, "indices:admin/test", threadPool, clusterService, transportService, actionFilters,
                     null, request, nodeRequest, nodeExecutor, TestNodeResponse.class);
         }
 
         @Override
         protected TestNodesResponse newResponse(TestNodesRequest request,
                                                 List<TestNodeResponse> responses, List<FailedNodeException> failures) {
-            return new TestNodesResponse(request, responses, failures);
+            return new TestNodesResponse(clusterService.getClusterName(), request, responses, failures);
         }
 
         @Override
@@ -302,8 +301,9 @@ public class TransportNodesActionTests extends ESTestCase {
 
         private final TestNodesRequest request;
 
-        TestNodesResponse(TestNodesRequest request, List<TestNodeResponse> nodeResponses, List<FailedNodeException> failures) {
-            super(CLUSTER_NAME, nodeResponses, failures);
+        TestNodesResponse(ClusterName clusterName, TestNodesRequest request, List<TestNodeResponse> nodeResponses,
+                          List<FailedNodeException> failures) {
+            super(clusterName, nodeResponses, failures);
             this.request = request;
         }
 
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
index ea4e55c02f4..abebb61c0b7 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
@@ -91,7 +91,7 @@ public class BroadcastReplicationTests extends ESTestCase {
         super.setUp();
         LocalTransport transport = new LocalTransport(Settings.EMPTY, threadPool, Version.CURRENT, new NamedWriteableRegistry(), circuitBreakerService);
         clusterService = createClusterService(threadPool);
-        transportService = new TransportService(transport, threadPool, clusterService.state().getClusterName());
+        transportService = new TransportService(clusterService.getSettings(), transport, threadPool);
         transportService.start();
         transportService.acceptIncomingRequests();
         broadcastReplicationAction = new TestBroadcastReplicationAction(Settings.EMPTY, threadPool, clusterService, transportService, new ActionFilters(new HashSet<ActionFilter>()), new IndexNameExpressionResolver(Settings.EMPTY), null);
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
index 8ee2499d040..229e3b6635e 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
@@ -119,7 +119,7 @@ public class TransportReplicationActionTests extends ESTestCase {
         super.setUp();
         transport = new CapturingTransport();
         clusterService = createClusterService(threadPool);
-        transportService = new TransportService(transport, threadPool, clusterService.state().getClusterName());
+        transportService = new TransportService(clusterService.getSettings(), transport, threadPool);
         transportService.start();
         transportService.acceptIncomingRequests();
         action = new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool);
diff --git a/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
index 6d7ac6128c9..c26d376b587 100644
--- a/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
@@ -142,7 +142,7 @@ public class TransportInstanceSingleOperationActionTests extends ESTestCase {
         super.setUp();
         transport = new CapturingTransport();
         clusterService = createClusterService(THREAD_POOL);
-        transportService = new TransportService(transport, THREAD_POOL, clusterService.state().getClusterName());
+        transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL);
         transportService.start();
         transportService.acceptIncomingRequests();
         action = new TestTransportInstanceSingleOperationAction(
diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java b/core/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java
index fb23f024b5d..208945a6179 100644
--- a/core/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java
+++ b/core/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java
@@ -238,7 +238,7 @@ public abstract class AbstractTermVectorsTestCase extends ESIntegTestCase {
                 contentArray[j] = fieldContentOptions[randomInt(fieldContentOptions.length - 1)];
                 docSource.put(fieldSettings[j].name, contentArray[j]);
             }
-            final String id = routingKeyForShard(index, "type", i);
+            final String id = routingKeyForShard(index, i);
             TestDoc doc = new TestDoc(id, fieldSettings, contentArray.clone());
             index(doc.index, doc.type, doc.id, docSource);
             testDocs[i] = doc;
diff --git a/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java b/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java
index 50fbbd4fcb8..c085c3164a0 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java
@@ -27,6 +27,7 @@ import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.component.LifecycleListener;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.transport.ConnectTransportException;
@@ -74,7 +75,8 @@ abstract class FailAndRetryMockTransport<Response extends TransportResponse> imp
         //we make sure that nodes get added to the connected ones when calling addTransportAddress, by returning proper nodes info
         if (connectMode) {
             TransportResponseHandler transportResponseHandler = transportServiceAdapter.onResponseReceived(requestId);
-            transportResponseHandler.handleResponse(new LivenessResponse(ClusterName.DEFAULT, node));
+            transportResponseHandler.handleResponse(new LivenessResponse(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY),
+                node));
             return;
         }
 
diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java
index e26ebfefcc9..eee74b66148 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientHeadersTests.java
@@ -67,6 +67,7 @@ public class TransportClientHeadersTests extends AbstractClientHeadersTestCase {
         TransportClient client = TransportClient.builder()
             .settings(Settings.builder()
                 .put("client.transport.sniff", false)
+                .put("cluster.name", "cluster1")
                 .put("node.name", "transport_client_" + this.getTestName())
                 .put(headersSettings)
                 .build())
@@ -115,8 +116,8 @@ public class TransportClientHeadersTests extends AbstractClientHeadersTestCase {
         CountDownLatch clusterStateLatch = new CountDownLatch(1);
 
         @Inject
-        public InternalTransportService(Settings settings, Transport transport, ThreadPool threadPool, ClusterName clusterName) {
-            super(settings, transport, threadPool, clusterName);
+        public InternalTransportService(Settings settings, Transport transport, ThreadPool threadPool) {
+            super(settings, transport, threadPool);
         }
 
         @Override @SuppressWarnings("unchecked")
@@ -124,7 +125,7 @@ public class TransportClientHeadersTests extends AbstractClientHeadersTestCase {
                                                               TransportRequestOptions options, TransportResponseHandler<T> handler) {
             if (TransportLivenessAction.NAME.equals(action)) {
                 assertHeaders(threadPool);
-                ((TransportResponseHandler<LivenessResponse>) handler).handleResponse(new LivenessResponse(ClusterName.DEFAULT, node));
+                ((TransportResponseHandler<LivenessResponse>) handler).handleResponse(new LivenessResponse(clusterName, node));
                 return;
             }
             if (ClusterStateAction.NAME.equals(action)) {
diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
index 0dddb301444..2a32ebd05af 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
@@ -63,7 +63,8 @@ public class TransportClientNodesServiceTests extends ESTestCase {
         private final int nodesCount;
 
         TestIteration() {
-            ClusterName clusterName = new ClusterName("test");
+            Settings settings = Settings.builder().put("cluster.name", "test").build();
+            ClusterName clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
             threadPool = new TestThreadPool("transport-client-nodes-service-tests");
             transport = new FailAndRetryMockTransport<TestResponse>(random(), clusterName) {
                 @Override
@@ -76,7 +77,7 @@ public class TransportClientNodesServiceTests extends ESTestCase {
                     return  new TestResponse();
                 }
             };
-            transportService = new TransportService(Settings.EMPTY, transport, threadPool, clusterName) {
+            transportService = new TransportService(settings, transport, threadPool) {
                 @Override
                 public <T extends TransportResponse> void sendRequest(DiscoveryNode node, String action,
                                                                       TransportRequest request, final TransportResponseHandler<T> handler) {
@@ -101,7 +102,7 @@ public class TransportClientNodesServiceTests extends ESTestCase {
             transportService.start();
             transportService.acceptIncomingRequests();
             transportClientNodesService =
-                    new TransportClientNodesService(Settings.EMPTY, clusterName, transportService, threadPool, Version.CURRENT);
+                    new TransportClientNodesService(settings, transportService, threadPool, Version.CURRENT);
             this.nodesCount = randomIntBetween(1, 10);
             for (int i = 0; i < nodesCount; i++) {
                 transportClientNodesService.addTransportAddresses(new LocalTransportAddress("node" + i));
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java b/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
index 98bb013f308..514968d1643 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java
@@ -29,6 +29,7 @@ import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllo
 import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.ModuleTestCase;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
@@ -40,7 +41,8 @@ import org.elasticsearch.common.settings.SettingsModule;
 import java.util.HashMap;
 import java.util.Map;
 public class ClusterModuleTests extends ModuleTestCase {
-
+    private ClusterService clusterService = new ClusterService(Settings.EMPTY,
+        new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null);
     public static class FakeAllocationDecider extends AllocationDecider {
         protected FakeAllocationDecider(Settings settings) {
             super(settings);
@@ -97,7 +99,7 @@ public class ClusterModuleTests extends ModuleTestCase {
     }
 
     public void testRegisterAllocationDeciderDuplicate() {
-        ClusterModule module = new ClusterModule(Settings.EMPTY);
+        ClusterModule module = new ClusterModule(Settings.EMPTY, clusterService);
         try {
             module.registerAllocationDecider(EnableAllocationDecider.class);
         } catch (IllegalArgumentException e) {
@@ -107,20 +109,20 @@ public class ClusterModuleTests extends ModuleTestCase {
     }
 
     public void testRegisterAllocationDecider() {
-        ClusterModule module = new ClusterModule(Settings.EMPTY);
+        ClusterModule module = new ClusterModule(Settings.EMPTY, clusterService);
         module.registerAllocationDecider(FakeAllocationDecider.class);
         assertSetMultiBinding(module, AllocationDecider.class, FakeAllocationDecider.class);
     }
 
     public void testRegisterShardsAllocator() {
         Settings settings = Settings.builder().put(ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING.getKey(), "custom").build();
-        ClusterModule module = new ClusterModule(settings);
+        ClusterModule module = new ClusterModule(settings, clusterService);
         module.registerShardsAllocator("custom", FakeShardsAllocator.class);
         assertBinding(module, ShardsAllocator.class, FakeShardsAllocator.class);
     }
 
     public void testRegisterShardsAllocatorAlreadyRegistered() {
-        ClusterModule module = new ClusterModule(Settings.EMPTY);
+        ClusterModule module = new ClusterModule(Settings.EMPTY, clusterService);
         try {
             module.registerShardsAllocator(ClusterModule.BALANCED_ALLOCATOR, FakeShardsAllocator.class);
         } catch (IllegalArgumentException e) {
@@ -130,19 +132,19 @@ public class ClusterModuleTests extends ModuleTestCase {
 
     public void testUnknownShardsAllocator() {
         Settings settings = Settings.builder().put(ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING.getKey(), "dne").build();
-        ClusterModule module = new ClusterModule(settings);
+        ClusterModule module = new ClusterModule(settings, clusterService);
         assertBindingFailure(module, "Unknown [shards_allocator]");
     }
 
     public void testEvenShardsAllocatorBackcompat() {
         Settings settings = Settings.builder()
             .put(ClusterModule.SHARDS_ALLOCATOR_TYPE_SETTING.getKey(), ClusterModule.EVEN_SHARD_COUNT_ALLOCATOR).build();
-        ClusterModule module = new ClusterModule(settings);
+        ClusterModule module = new ClusterModule(settings, clusterService);
         assertBinding(module, ShardsAllocator.class, BalancedShardsAllocator.class);
     }
 
     public void testRegisterIndexTemplateFilterDuplicate() {
-        ClusterModule module = new ClusterModule(Settings.EMPTY);
+        ClusterModule module = new ClusterModule(Settings.EMPTY, clusterService);
         try {
             module.registerIndexTemplateFilter(FakeIndexTemplateFilter.class);
             module.registerIndexTemplateFilter(FakeIndexTemplateFilter.class);
@@ -153,7 +155,7 @@ public class ClusterModuleTests extends ModuleTestCase {
     }
 
     public void testRegisterIndexTemplateFilter() {
-        ClusterModule module = new ClusterModule(Settings.EMPTY);
+        ClusterModule module = new ClusterModule(Settings.EMPTY, clusterService);
         module.registerIndexTemplateFilter(FakeIndexTemplateFilter.class);
         assertSetMultiBinding(module, IndexTemplateFilter.class, FakeIndexTemplateFilter.class);
     }
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/core/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java
index 92be2331a8c..5e272a27459 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.cluster;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.test.ESTestCase;
 
@@ -34,11 +35,15 @@ public class ClusterStateTests extends ESTestCase {
         final DiscoveryNode node1 = new DiscoveryNode("node1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
         final DiscoveryNode node2 = new DiscoveryNode("node2", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
         final DiscoveryNodes nodes = DiscoveryNodes.builder().put(node1).put(node2).build();
-        ClusterState noMaster1 = ClusterState.builder(ClusterName.DEFAULT).version(randomInt(5)).nodes(nodes).build();
-        ClusterState noMaster2 = ClusterState.builder(ClusterName.DEFAULT).version(randomInt(5)).nodes(nodes).build();
-        ClusterState withMaster1a = ClusterState.builder(ClusterName.DEFAULT).version(randomInt(5)).nodes(DiscoveryNodes.builder(nodes).masterNodeId(node1.getId())).build();
-        ClusterState withMaster1b = ClusterState.builder(ClusterName.DEFAULT).version(randomInt(5)).nodes(DiscoveryNodes.builder(nodes).masterNodeId(node1.getId())).build();
-        ClusterState withMaster2 = ClusterState.builder(ClusterName.DEFAULT).version(randomInt(5)).nodes(DiscoveryNodes.builder(nodes).masterNodeId(node2.getId())).build();
+        ClusterName name = ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY);
+        ClusterState noMaster1 = ClusterState.builder(name).version(randomInt(5)).nodes(nodes).build();
+        ClusterState noMaster2 = ClusterState.builder(name).version(randomInt(5)).nodes(nodes).build();
+        ClusterState withMaster1a = ClusterState.builder(name).version(randomInt(5)).nodes(DiscoveryNodes.builder(nodes)
+            .masterNodeId(node1.getId())).build();
+        ClusterState withMaster1b = ClusterState.builder(name).version(randomInt(5)).nodes(DiscoveryNodes.builder(nodes)
+            .masterNodeId(node1.getId())).build();
+        ClusterState withMaster2 = ClusterState.builder(name).version(randomInt(5)).nodes(DiscoveryNodes.builder(nodes)
+            .masterNodeId(node2.getId())).build();
 
         // states with no master should never supersede anything
         assertFalse(noMaster1.supersedes(noMaster2));
diff --git a/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
index a3bb73977e7..5eb5a34c44f 100644
--- a/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
@@ -149,7 +149,7 @@ public class NodeConnectionsServiceTests extends ESTestCase {
     public void setUp() throws Exception {
         super.setUp();
         this.transport = new MockTransport();
-        transportService = new TransportService(transport, THREAD_POOL, ClusterName.DEFAULT);
+        transportService = new TransportService(Settings.EMPTY, transport, THREAD_POOL);
         transportService.start();
         transportService.acceptIncomingRequests();
     }
diff --git a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java
index 426a574e2cd..1faac874114 100644
--- a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java
@@ -84,7 +84,7 @@ public class ShardFailedClusterStateTaskExecutorTests extends ESAllocationTestCa
         routingTable = RoutingTable.builder()
             .addAsNew(metaData.index(INDEX))
             .build();
-        clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
         executor = new ShardStateAction.ShardFailedClusterStateTaskExecutor(allocationService, null, logger);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
index bfc3929dd14..7f0ac1a6e45 100644
--- a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
@@ -107,7 +107,7 @@ public class ShardStateActionTests extends ESTestCase {
         super.setUp();
         this.transport = new CapturingTransport();
         clusterService = createClusterService(THREAD_POOL);
-        transportService = new TransportService(transport, THREAD_POOL, clusterService.state().getClusterName());
+        transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL);
         transportService.start();
         transportService.acceptIncomingRequests();
         shardStateAction = new TestShardStateAction(Settings.EMPTY, clusterService, transportService, null, null);
diff --git a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
index e99c862bfec..fd1e3e62466 100644
--- a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
@@ -81,7 +81,7 @@ public class ClusterStateHealthTests extends ESTestCase {
     public void setUp() throws Exception {
         super.setUp();
         clusterService = createClusterService(threadPool);
-        transportService = new TransportService(new CapturingTransport(), threadPool, clusterService.state().getClusterName());
+        transportService = new TransportService(clusterService.getSettings(), new CapturingTransport(), threadPool);
         transportService.start();
         transportService.acceptIncomingRequests();
     }
@@ -127,8 +127,7 @@ public class ClusterStateHealthTests extends ESTestCase {
         listenerCalled.await();
 
         TransportClusterHealthAction action = new TransportClusterHealthAction(Settings.EMPTY, transportService,
-            clusterService, threadPool, clusterService.state().getClusterName(), new ActionFilters(new HashSet<>()),
-            indexNameExpressionResolver, NoopGatewayAllocator.INSTANCE);
+            clusterService, threadPool, new ActionFilters(new HashSet<>()), indexNameExpressionResolver, NoopGatewayAllocator.INSTANCE);
         PlainActionFuture<ClusterHealthResponse> listener = new PlainActionFuture<>();
 
         action.execute(new ClusterHealthRequest(), listener);
@@ -158,7 +157,7 @@ public class ClusterStateHealthTests extends ESTestCase {
             metaData.put(indexMetaData, true);
             routingTable.add(indexRoutingTable);
         }
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable.build()).build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable.build()).build();
         String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, IndicesOptions.strictExpand(), (String[]) null);
         ClusterStateHealth clusterStateHealth = new ClusterStateHealth(clusterState, concreteIndices);
         logger.info("cluster status: {}, expected {}", clusterStateHealth.getStatus(), counter.status());
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
index 53dffc799b1..28a25d17337 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
@@ -61,7 +61,8 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
         routingTableBuilder.addAsNew(metaData.index(name));
 
         RoutingTable routingTable = routingTableBuilder.build();
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING
+            .getDefault(Settings.EMPTY))
             .metaData(metaData).routingTable(routingTable).blocks(ClusterBlocks.builder().addBlocks(indexMetaData)).build();
         return clusterState;
     }
@@ -200,7 +201,8 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
 
     private void validateIndexName(String indexName, String errorMessage) {
         InvalidIndexNameException e = expectThrows(InvalidIndexNameException.class,
-            () -> getCreateIndexService().validateIndexName(indexName, ClusterState.builder(ClusterName.DEFAULT).build()));
+            () -> getCreateIndexService().validateIndexName(indexName, ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING
+                .getDefault(Settings.EMPTY)).build()));
         assertThat(e.getMessage(), endsWith(errorMessage));
     }
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
index 2e1cdea56a4..14f4ccf5581 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationServiceTests.java
@@ -92,7 +92,7 @@ public class DelayedAllocationServiceTests extends ESAllocationTestCase {
                 .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "0"))
                 .numberOfShards(1).numberOfReplicas(1))
             .build();
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metaData(metaData)
             .routingTable(RoutingTable.builder().addAsNew(metaData.index("test")).build()).build();
         clusterState = ClusterState.builder(clusterState)
@@ -138,7 +138,7 @@ public class DelayedAllocationServiceTests extends ESAllocationTestCase {
                 .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), delaySetting))
                 .numberOfShards(1).numberOfReplicas(1))
             .build();
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metaData(metaData)
             .routingTable(RoutingTable.builder().addAsNew(metaData.index("test")).build()).build();
         clusterState = ClusterState.builder(clusterState)
@@ -236,7 +236,7 @@ public class DelayedAllocationServiceTests extends ESAllocationTestCase {
                 .settings(settings(Version.CURRENT).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), longDelaySetting))
                 .numberOfShards(1).numberOfReplicas(1))
             .build();
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData)
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData)
             .routingTable(RoutingTable.builder().addAsNew(metaData.index("short_delay")).addAsNew(metaData.index("long_delay")).build())
             .nodes(DiscoveryNodes.builder()
                 .put(newNode("node0", singleton(DiscoveryNode.Role.MASTER))).localNodeId("node0").masterNodeId("node0")
@@ -390,7 +390,7 @@ public class DelayedAllocationServiceTests extends ESAllocationTestCase {
                 .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), shorterDelaySetting))
                 .numberOfShards(1).numberOfReplicas(1))
             .build();
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metaData(metaData)
             .routingTable(RoutingTable.builder()
                 .addAsNew(metaData.index("foo"))
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java
index 0127702c5ea..370ac52956c 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java
@@ -24,6 +24,7 @@ import org.elasticsearch.action.support.replication.ClusterStateCreationUtils;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
@@ -210,7 +211,8 @@ public class OperationRoutingTests extends ESTestCase{
                 }
             }
             final ShardIterator it =
-                    new OperationRouting(Settings.EMPTY, new AwarenessAllocationDecider())
+                    new OperationRouting(Settings.EMPTY,  new ClusterSettings(Settings.EMPTY,
+                        ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
                             .getShards(clusterService.state(), indexName, 0, "_prefer_nodes:" + String.join(",", nodes));
             final List<ShardRouting> all = new ArrayList<>();
             ShardRouting shard;
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java
index c78b1f9c815..4dceb42cfbc 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java
@@ -79,8 +79,8 @@ public class PrimaryTermsTests extends ESAllocationTestCase {
                         .build())
                 .build();
 
-        this.clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData)
-                .routingTable(testRoutingTable).build();
+        this.clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+            .metaData(metaData).routingTable(testRoutingTable).build();
     }
 
     /**
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingBackwardCompatibilityTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingBackwardCompatibilityTests.java
index d8dedb1e8e8..c673a6fd226 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingBackwardCompatibilityTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingBackwardCompatibilityTests.java
@@ -25,6 +25,7 @@ import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
@@ -36,7 +37,8 @@ import java.util.Arrays;
 public class RoutingBackwardCompatibilityTests extends ESTestCase {
 
     public void testBackwardCompatibility() throws Exception {
-        try (BufferedReader reader = new BufferedReader(new InputStreamReader(RoutingBackwardCompatibilityTests.class.getResourceAsStream("/org/elasticsearch/cluster/routing/shard_routes.txt"), "UTF-8"))) {
+        try (BufferedReader reader = new BufferedReader(new InputStreamReader(RoutingBackwardCompatibilityTests.class
+            .getResourceAsStream("/org/elasticsearch/cluster/routing/shard_routes.txt"), "UTF-8"))) {
             for (String line = reader.readLine(); line != null; line = reader.readLine()) {
                 if (line.startsWith("#")) { // comment
                     continue;
@@ -51,17 +53,20 @@ public class RoutingBackwardCompatibilityTests extends ESTestCase {
                 final int pre20ExpectedShardId = Integer.parseInt(parts[5]); // not needed anymore - old hashing is gone
                 final int currentExpectedShard = Integer.parseInt(parts[6]);
 
-                OperationRouting operationRouting = new OperationRouting(Settings.EMPTY, null);
+                OperationRouting operationRouting = new OperationRouting(Settings.EMPTY, new ClusterSettings(Settings.EMPTY,
+                    ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
                 for (Version version : VersionUtils.allVersions()) {
                     if (version.onOrAfter(Version.V_2_0_0) == false) {
                         // unsupported version, no need to test
                         continue;
                     }
                     final Settings settings = settings(version).build();
-                    IndexMetaData indexMetaData = IndexMetaData.builder(index).settings(settings).numberOfShards(numberOfShards).numberOfReplicas(randomInt(3)).build();
+                    IndexMetaData indexMetaData = IndexMetaData.builder(index).settings(settings).numberOfShards(numberOfShards)
+                        .numberOfReplicas(randomInt(3)).build();
                     MetaData.Builder metaData = MetaData.builder().put(indexMetaData, false);
                     RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetaData).build();
-                    ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+                    ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+                        .metaData(metaData).routingTable(routingTable).build();
                     final int shardId = operationRouting.indexShards(clusterState, index, id, routing).shardId().getId();
                     assertEquals(currentExpectedShard, shardId);
                 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java
index 9e2b1df3696..2d1a467a001 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java
@@ -73,7 +73,7 @@ public class RoutingTableTests extends ESAllocationTestCase {
                 .add(new IndexRoutingTable.Builder(metaData.index(TEST_INDEX_1).getIndex()).initializeAsNew(metaData.index(TEST_INDEX_1)).build())
                 .add(new IndexRoutingTable.Builder(metaData.index(TEST_INDEX_2).getIndex()).initializeAsNew(metaData.index(TEST_INDEX_2)).build())
                 .build();
-        this.clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(testRoutingTable).build();
+        this.clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(testRoutingTable).build();
     }
 
     /**
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
index 82703c6f29e..75300a4beb8 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
@@ -94,7 +94,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(randomIntBetween(1, 3)).numberOfReplicas(randomIntBetween(0, 3)))
                 .build();
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metaData(metaData)
                 .routingTable(RoutingTable.builder().addAsNew(metaData.index("test")).build()).build();
         for (ShardRouting shard : clusterState.getRoutingNodes().shardsWithState(UNASSIGNED)) {
@@ -106,7 +106,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(randomIntBetween(1, 3)).numberOfReplicas(randomIntBetween(0, 3)))
                 .build();
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metaData(metaData)
                 .routingTable(RoutingTable.builder().addAsRecovery(metaData.index("test")).build()).build();
         for (ShardRouting shard : clusterState.getRoutingNodes().shardsWithState(UNASSIGNED)) {
@@ -118,7 +118,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(randomIntBetween(1, 3)).numberOfReplicas(randomIntBetween(0, 3)))
                 .build();
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metaData(metaData)
                 .routingTable(RoutingTable.builder().addAsFromCloseToOpen(metaData.index("test")).build()).build();
         for (ShardRouting shard : clusterState.getRoutingNodes().shardsWithState(UNASSIGNED)) {
@@ -130,7 +130,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(randomIntBetween(1, 3)).numberOfReplicas(randomIntBetween(0, 3)))
                 .build();
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metaData(metaData)
                 .routingTable(RoutingTable.builder().addAsNewRestore(metaData.index("test"), new RestoreSource(new Snapshot("rep1", new SnapshotId("snp1", UUIDs.randomBase64UUID())), Version.CURRENT, "test"), new IntHashSet()).build()).build();
         for (ShardRouting shard : clusterState.getRoutingNodes().shardsWithState(UNASSIGNED)) {
@@ -142,7 +142,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(randomIntBetween(1, 3)).numberOfReplicas(randomIntBetween(0, 3)))
                 .build();
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metaData(metaData)
                 .routingTable(RoutingTable.builder().addAsRestore(metaData.index("test"), new RestoreSource(new Snapshot("rep1", new SnapshotId("snp1", UUIDs.randomBase64UUID())), Version.CURRENT, "test")).build()).build();
         for (ShardRouting shard : clusterState.getRoutingNodes().shardsWithState(UNASSIGNED)) {
@@ -154,7 +154,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(randomIntBetween(1, 3)).numberOfReplicas(randomIntBetween(0, 3)))
                 .build();
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metaData(metaData)
                 .routingTable(RoutingTable.builder().addAsFromDangling(metaData.index("test")).build()).build();
         for (ShardRouting shard : clusterState.getRoutingNodes().shardsWithState(UNASSIGNED)) {
@@ -168,7 +168,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0))
                 .build();
         final Index index = metaData.index("test").getIndex();
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metaData(metaData)
                 .routingTable(RoutingTable.builder().addAsNew(metaData.index(index)).build()).build();
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build();
@@ -208,7 +208,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
                 .build();
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metaData(metaData)
                 .routingTable(RoutingTable.builder().addAsNew(metaData.index("test")).build()).build();
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
@@ -237,7 +237,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
                 .build();
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metaData(metaData)
                 .routingTable(RoutingTable.builder().addAsNew(metaData.index("test")).build()).build();
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
@@ -286,7 +286,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
                 .put(IndexMetaData.builder("test1").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
                 .put(IndexMetaData.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
                 .build();
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metaData(metaData)
                 .routingTable(RoutingTable.builder().addAsNew(metaData.index("test1")).addAsNew(metaData.index("test2")).build()).build();
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
@@ -314,7 +314,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
                 .put(IndexMetaData.builder("test1").settings(settings(Version.CURRENT).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), delayTest1)).numberOfShards(1).numberOfReplicas(1))
                 .put(IndexMetaData.builder("test2").settings(settings(Version.CURRENT).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), delayTest2)).numberOfShards(1).numberOfReplicas(1))
                 .build();
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metaData(metaData)
                 .routingTable(RoutingTable.builder().addAsNew(metaData.index("test1")).addAsNew(metaData.index("test2")).build()).build();
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ActiveAllocationIdTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ActiveAllocationIdTests.java
index 2e54512b95f..b3679208276 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ActiveAllocationIdTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ActiveAllocationIdTests.java
@@ -25,6 +25,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESAllocationTestCase;
 
 import java.util.Arrays;
@@ -50,7 +51,8 @@ public class ActiveAllocationIdTests extends ESAllocationTestCase {
         RoutingTable routingTable = RoutingTable.builder()
                 .addAsNew(metaData.index("test"))
                 .build();
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING
+            .getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("adding three nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java
index fe4937eea7e..cf90379a50f 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AddIncrementallyTests.java
@@ -306,7 +306,7 @@ public class AddIncrementallyTests extends ESAllocationTestCase {
         for (int i = 0; i < numberOfNodes; i++) {
             nodes.put(newNode("node" + i));
         }
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).nodes(nodes).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).nodes(nodes).metaData(metaData).routingTable(routingTable).build();
         routingTable = service.reroute(clusterState, "reroute").routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
index 0d637e451f0..28f27b8988c 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
@@ -76,7 +76,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
         RoutingTable routingTable = RoutingTable.builder()
                 .addAsNew(metaData.index("test"))
                 .build();
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
@@ -133,7 +133,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
         RoutingTable routingTable = RoutingTable.builder()
                 .addAsRecovery(metaData.index(index))
                 .build();
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
         final ShardId shardId = new ShardId(metaData.index(index).getIndex(), 0);
 
         logger.info("--> adding 3 nodes on same rack and do rerouting");
@@ -256,7 +256,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
         RoutingTable routingTable = RoutingTable.builder()
                 .addAsNew(metaData.index("test"))
                 .build();
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding 3 nodes");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationPriorityTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationPriorityTests.java
index f12ed123dc5..86f3bd5704b 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationPriorityTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationPriorityTests.java
@@ -65,7 +65,7 @@ public class AllocationPriorityTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("first"))
                 .addAsNew(metaData.index("second"))
                 .build();
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
         RoutingAllocation.Result rerouteResult = allocation.reroute(clusterState, "reroute");
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java
index 805ab0321ba..d78156b2217 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java
@@ -68,7 +68,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding two nodes on same rack and do rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -136,7 +136,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding two nodes on same rack and do rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -210,7 +210,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding two nodes on same rack and do rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -301,7 +301,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test2"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding two nodes on same rack and do rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -392,7 +392,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding two nodes on same rack and do rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -470,7 +470,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding two nodes on same rack and do rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -551,7 +551,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding two nodes on same rack and do rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -618,7 +618,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding two nodes on same rack and do rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -693,7 +693,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test2"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding two nodes on same rack and do rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -772,7 +772,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding two nodes in different zones and do rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -833,7 +833,7 @@ public class AwarenessAllocationTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding 5 nodes in different zones and do rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
index e606f007dee..9d0b27eac1a 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/BalanceConfigurationTests.java
@@ -131,7 +131,7 @@ public class BalanceConfigurationTests extends ESAllocationTestCase {
         for (int i = 0; i < numberOfNodes; i++) {
             nodes.put(newNode("node" + i));
         }
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).nodes(nodes).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).nodes(nodes).metaData(metaData).routingTable(routingTable).build();
         routingTable = strategy.reroute(clusterState, "reroute").routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
@@ -381,7 +381,7 @@ public class BalanceConfigurationTests extends ESAllocationTestCase {
             nodes.put(node);
         }
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).nodes(nodes).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).nodes(nodes).metaData(metaData).routingTable(routingTable).build();
         routingTable = strategy.reroute(clusterState, "reroute").routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java
index 2483832ef71..fe264e17378 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/CatAllocationTestCase.java
@@ -120,7 +120,7 @@ public abstract class CatAllocationTestCase extends ESAllocationTestCase {
         for (String node : nodes) {
             builderDiscoNodes.put(newNode(node));
         }
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).nodes(builderDiscoNodes.build()).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).nodes(builderDiscoNodes.build()).build();
         if (balanceFirst()) {
             clusterState = rebalance(clusterState);
         }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java
index 4e99d69ef89..95ce9d668ea 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ClusterRebalanceRoutingTests.java
@@ -60,7 +60,7 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test2"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("start two nodes");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
@@ -146,7 +146,7 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test2"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("start two nodes");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
@@ -250,7 +250,7 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test2"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("start two nodes");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
@@ -334,7 +334,7 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test2"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("start two nodes");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
@@ -457,7 +457,7 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test2"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("start two nodes");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
@@ -541,7 +541,7 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test2"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("start two nodes");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
@@ -661,7 +661,7 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test1"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("start two nodes");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build();
@@ -760,7 +760,7 @@ public class ClusterRebalanceRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test1"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("start two nodes");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build();
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ConcurrentRebalanceRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ConcurrentRebalanceRoutingTests.java
index 0fcffb8afe3..5d8cdcb838f 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ConcurrentRebalanceRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ConcurrentRebalanceRoutingTests.java
@@ -57,7 +57,7 @@ public class ConcurrentRebalanceRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         assertThat(routingTable.index("test").shards().size(), equalTo(5));
         for (int i = 0; i < routingTable.index("test").shards().size(); i++) {
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java
index b14aeca890e..856d8361779 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/DeadNodesAllocationTests.java
@@ -56,7 +56,7 @@ public class DeadNodesAllocationTests extends ESAllocationTestCase {
         RoutingTable routingTable = RoutingTable.builder()
                 .addAsNew(metaData.index("test"))
                 .build();
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding 2 nodes on same rack and do rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -107,7 +107,7 @@ public class DeadNodesAllocationTests extends ESAllocationTestCase {
         RoutingTable routingTable = RoutingTable.builder()
                 .addAsNew(metaData.index("test"))
                 .build();
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding 2 nodes on same rack and do rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -181,7 +181,7 @@ public class DeadNodesAllocationTests extends ESAllocationTestCase {
         RoutingTable routingTable = RoutingTable.builder()
                 .addAsNew(metaData.index("test"))
                 .build();
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding 2 nodes on same rack and do rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ElectReplicaAsPrimaryDuringRelocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ElectReplicaAsPrimaryDuringRelocationTests.java
index 0bf3ca93676..83a4c17a631 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ElectReplicaAsPrimaryDuringRelocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ElectReplicaAsPrimaryDuringRelocationTests.java
@@ -55,7 +55,7 @@ public class ElectReplicaAsPrimaryDuringRelocationTests extends ESAllocationTest
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("Adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java
index 19591eef425..7d2ffb07ddf 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ExpectedShardSizeAllocationTests.java
@@ -78,7 +78,7 @@ public class ExpectedShardSizeAllocationTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
         logger.info("Adding one node and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build();
         routingTable = strategy.reroute(clusterState, "reroute").routingTable();
@@ -130,7 +130,7 @@ public class ExpectedShardSizeAllocationTests extends ESAllocationTestCase {
         RoutingTable routingTable = RoutingTable.builder()
                 .addAsNew(metaData.index("test"))
                 .build();
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java
index ecf7ca741b4..3b5332eb237 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedNodeRoutingTests.java
@@ -54,7 +54,7 @@ public class FailedNodeRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test2"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("start 4 nodes");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")).put(newNode("node3")).put(newNode("node4"))).build();
@@ -99,4 +99,4 @@ public class FailedNodeRoutingTests extends ESAllocationTestCase {
             assertThat(routingNode.numberOfShardsWithState(INITIALIZING), equalTo(1));
         }
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java
index e859c5811c3..5f4d6b5a8ba 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FailedShardsRoutingTests.java
@@ -67,7 +67,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase {
         RoutingTable routingTable = RoutingTable.builder()
                 .addAsNew(metaData.index("test"))
                 .build();
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding 2 nodes on same rack and do rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -158,7 +158,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("Adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
@@ -239,7 +239,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("Adding single node and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build();
@@ -295,7 +295,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("Adding {} nodes and performing rerouting", numberOfReplicas + 1);
         DiscoveryNodes.Builder nodeBuilder = DiscoveryNodes.builder();
@@ -351,7 +351,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("Adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
@@ -411,7 +411,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("Adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
@@ -502,7 +502,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         // add 4 nodes
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")).put(newNode("node3")).put(newNode("node4"))).build();
@@ -548,7 +548,7 @@ public class FailedShardsRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         // add 4 nodes
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")).put(newNode("node3")).put(newNode("node4"))).build();
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterAllocationDeciderTests.java
index 44437237dc9..0255e18e529 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterAllocationDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterAllocationDeciderTests.java
@@ -114,8 +114,8 @@ public class FilterAllocationDeciderTests extends ESAllocationTestCase {
         routingTableBuilder.addAsNew(metaData.index("idx"));
 
         RoutingTable routingTable = routingTableBuilder.build();
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
-            .metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING
+            .getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")))
             .build();
         routingTable = service.reroute(clusterState, "reroute", false).routingTable();
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java
index 5a555621bec..ef0a589e50a 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java
@@ -62,7 +62,7 @@ public class FilterRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding four nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -110,7 +110,7 @@ public class FilterRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -180,7 +180,7 @@ public class FilterRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test2"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding two nodes and performing rerouting");
         DiscoveryNode node1 = newNode("node1", singletonMap("tag1", "value1"));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/IndexBalanceTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/IndexBalanceTests.java
index 874509d5cf1..79236ddd2b7 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/IndexBalanceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/IndexBalanceTests.java
@@ -58,7 +58,7 @@ public class IndexBalanceTests extends ESAllocationTestCase {
 
         RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).addAsNew(metaData.index("test1")).build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         assertThat(routingTable.index("test").shards().size(), equalTo(3));
         for (int i = 0; i < routingTable.index("test").shards().size(); i++) {
@@ -188,7 +188,7 @@ public class IndexBalanceTests extends ESAllocationTestCase {
 
         RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).addAsNew(metaData.index("test1")).build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         assertThat(routingTable.index("test").shards().size(), equalTo(3));
         for (int i = 0; i < routingTable.index("test").shards().size(); i++) {
@@ -349,7 +349,7 @@ public class IndexBalanceTests extends ESAllocationTestCase {
 
         RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         assertThat(routingTable.index("test").shards().size(), equalTo(3));
         for (int i = 0; i < routingTable.index("test").shards().size(); i++) {
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java
index f76851cfef9..a5acb11ce4b 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/MaxRetryAllocationDeciderTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.cluster.routing.allocation;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -63,7 +64,7 @@ public class MaxRetryAllocationDeciderTests extends ESAllocationTestCase {
         routingTableBuilder.addAsNew(metaData.index("idx"));
 
         RoutingTable routingTable = routingTableBuilder.build();
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metaData(metaData).routingTable(routingTable).build();
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")))
             .build();
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
index 54df74db1f5..6dd4df59060 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
@@ -91,7 +91,7 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         assertThat(routingTable.index("test").shards().size(), equalTo(5));
         for (int i = 0; i < routingTable.index("test").shards().size(); i++) {
@@ -209,7 +209,7 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase {
         }
         RoutingTable routingTable = rtBuilder.build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
         assertThat(routingTable.shardsWithState(UNASSIGNED).size(), equalTo(routingTable.allShards().size()));
         List<DiscoveryNode> nodes = new ArrayList<>();
         int nodeIdx = 0;
@@ -254,7 +254,7 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         assertThat(routingTable.index("test").shards().size(), equalTo(5));
         for (int i = 0; i < routingTable.index("test").shards().size(); i++) {
@@ -331,7 +331,7 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase {
                     .build())
             )
             .build();
-        ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+        ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metaData(metaData)
             .routingTable(routingTable)
             .nodes(DiscoveryNodes.builder().put(newNode).put(oldNode1).put(oldNode2)).build();
@@ -360,7 +360,7 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase {
                 (randomIntBetween(0, 3)))
             .build();
 
-        ClusterState state = ClusterState.builder(ClusterName.DEFAULT)
+        ClusterState state = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metaData(metaData)
             .routingTable(RoutingTable.builder().addAsRestore(metaData.index("test"),
                 new RestoreSource(new Snapshot("rep1", new SnapshotId("snp1", UUIDs.randomBase64UUID())),
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferLocalPrimariesToRelocatingPrimariesTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferLocalPrimariesToRelocatingPrimariesTests.java
index 79c59b33b8e..3812a550169 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferLocalPrimariesToRelocatingPrimariesTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferLocalPrimariesToRelocatingPrimariesTests.java
@@ -62,7 +62,7 @@ public class PreferLocalPrimariesToRelocatingPrimariesTests extends ESAllocation
                 .addAsNew(metaData.index("test2"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("adding two nodes and performing rerouting till all are allocated");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferPrimaryAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferPrimaryAllocationTests.java
index 7067009a19e..decf0ed0ea3 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferPrimaryAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PreferPrimaryAllocationTests.java
@@ -59,7 +59,7 @@ public class PreferPrimaryAllocationTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test2"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("adding two nodes and performing rerouting till all are allocated");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java
index 3c60a51f4fb..b5f1ab7235c 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryElectionRoutingTests.java
@@ -55,7 +55,7 @@ public class PrimaryElectionRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("Adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build();
@@ -108,7 +108,7 @@ public class PrimaryElectionRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("Adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryNotRelocatedWhileBeingRecoveredTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryNotRelocatedWhileBeingRecoveredTests.java
index a636334b5d3..264a009650e 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryNotRelocatedWhileBeingRecoveredTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/PrimaryNotRelocatedWhileBeingRecoveredTests.java
@@ -58,7 +58,7 @@ public class PrimaryNotRelocatedWhileBeingRecoveredTests extends ESAllocationTes
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("Adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build();
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
index 1f39706e4f4..7f62cced708 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RandomAllocationDeciderTests.java
@@ -78,7 +78,7 @@ public class RandomAllocationDeciderTests extends ESAllocationTestCase {
         }
 
         RoutingTable routingTable = routingTableBuilder.build();
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
         int numIters = scaledRandomIntBetween(5, 15);
         int nodeIdCounter = 0;
         int atMostNodes = scaledRandomIntBetween(Math.max(1, maxNumReplicas), 15);
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RebalanceAfterActiveTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RebalanceAfterActiveTests.java
index 035f6d3efff..0d743da39f3 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RebalanceAfterActiveTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RebalanceAfterActiveTests.java
@@ -87,7 +87,7 @@ public class RebalanceAfterActiveTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         assertThat(routingTable.index("test").shards().size(), equalTo(5));
         for (int i = 0; i < routingTable.index("test").shards().size(); i++) {
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ReplicaAllocatedAfterPrimaryTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ReplicaAllocatedAfterPrimaryTests.java
index e4b423c3f8f..aab3c8d6c2b 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ReplicaAllocatedAfterPrimaryTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ReplicaAllocatedAfterPrimaryTests.java
@@ -57,7 +57,7 @@ public class ReplicaAllocatedAfterPrimaryTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         assertThat(routingTable.index("test").shards().size(), equalTo(1));
         assertThat(routingTable.index("test").shard(0).size(), equalTo(2));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RoutingNodesIntegrityTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RoutingNodesIntegrityTests.java
index c23ab86b497..4283cb8475c 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RoutingNodesIntegrityTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/RoutingNodesIntegrityTests.java
@@ -58,7 +58,7 @@ public class RoutingNodesIntegrityTests extends ESAllocationTestCase {
 
         RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).addAsNew(metaData.index("test1")).build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
         RoutingNodes routingNodes = clusterState.getRoutingNodes();
 
         logger.info("Adding three node and performing rerouting");
@@ -130,7 +130,7 @@ public class RoutingNodesIntegrityTests extends ESAllocationTestCase {
 
         RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).addAsNew(metaData.index("test1")).build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("Adding one node and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build();
@@ -222,7 +222,7 @@ public class RoutingNodesIntegrityTests extends ESAllocationTestCase {
 
         RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("Adding three node and performing rerouting");
         clusterState = ClusterState.builder(clusterState)
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java
index 9984890e8a4..3d0475ed137 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.cluster.routing.allocation;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -56,7 +57,7 @@ public class SameShardRoutingTests extends ESAllocationTestCase {
         RoutingTable routingTable = RoutingTable.builder()
                 .addAsNew(metaData.index("test"))
                 .build();
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData)
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData)
                 .routingTable(routingTable).build();
 
         logger.info("--> adding two nodes with the same host");
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardVersioningTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardVersioningTests.java
index 1298bda30f9..29162aabb60 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardVersioningTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardVersioningTests.java
@@ -54,7 +54,7 @@ public class ShardVersioningTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test2"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("start two nodes");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java
index 26aa88a74a1..67fc93b600b 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ShardsLimitAllocationTests.java
@@ -61,7 +61,7 @@ public class ShardsLimitAllocationTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
         logger.info("Adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
         routingTable = strategy.reroute(clusterState, "reroute").routingTable();
@@ -105,7 +105,7 @@ public class ShardsLimitAllocationTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
         logger.info("Adding two nodes and performing rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2"))).build();
         routingTable = strategy.reroute(clusterState, "reroute").routingTable();
@@ -168,7 +168,7 @@ public class ShardsLimitAllocationTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
         logger.info("Adding one node and reroute");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build();
         routingTable = strategy.reroute(clusterState, "reroute").routingTable();
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardNoReplicasRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardNoReplicasRoutingTests.java
index 4aaf05cc99f..ad9c0b8eef5 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardNoReplicasRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardNoReplicasRoutingTests.java
@@ -71,7 +71,7 @@ public class SingleShardNoReplicasRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         assertThat(routingTable.index("test").shards().size(), equalTo(1));
         assertThat(routingTable.index("test").shard(0).size(), equalTo(1));
@@ -172,7 +172,7 @@ public class SingleShardNoReplicasRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         assertThat(routingTable.index("test").shards().size(), equalTo(1));
         assertThat(routingTable.index("test").shard(0).size(), equalTo(1));
@@ -229,7 +229,7 @@ public class SingleShardNoReplicasRoutingTests extends ESAllocationTestCase {
             routingTableBuilder.addAsNew(metaData.index("test" + i));
         }
         RoutingTable routingTable = routingTableBuilder.build();
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         assertThat(routingTable.indicesRouting().size(), equalTo(numberOfIndices));
         for (int i = 0; i < numberOfIndices; i++) {
@@ -342,7 +342,7 @@ public class SingleShardNoReplicasRoutingTests extends ESAllocationTestCase {
         }
         RoutingTable routingTable = routingTableBuilder.build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         assertThat(routingTable.indicesRouting().size(), equalTo(numberOfIndices));
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardOneReplicaRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardOneReplicaRoutingTests.java
index 64a7b2abcf0..5c791b954f9 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardOneReplicaRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SingleShardOneReplicaRoutingTests.java
@@ -56,7 +56,7 @@ public class SingleShardOneReplicaRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         assertThat(routingTable.index("test").shards().size(), equalTo(1));
         assertThat(routingTable.index("test").shard(0).size(), equalTo(2));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java
index 5d01caaa899..16a11a9c150 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/StartedShardsRoutingTests.java
@@ -31,6 +31,7 @@ import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESAllocationTestCase;
@@ -50,7 +51,7 @@ public class StartedShardsRoutingTests extends ESAllocationTestCase {
                 .numberOfShards(3).numberOfReplicas(0)
                 .build();
         final Index index = indexMetaData.getIndex();
-        ClusterState.Builder stateBuilder = ClusterState.builder(ClusterName.DEFAULT)
+        ClusterState.Builder stateBuilder = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")))
                 .metaData(MetaData.builder().put(indexMetaData, false));
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/TenShardsOneReplicaRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/TenShardsOneReplicaRoutingTests.java
index fa1e9c34576..becd61e6eb2 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/TenShardsOneReplicaRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/TenShardsOneReplicaRoutingTests.java
@@ -67,7 +67,7 @@ public class TenShardsOneReplicaRoutingTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         assertThat(routingTable.index("test").shards().size(), equalTo(10));
         for (int i = 0; i < routingTable.index("test").shards().size(); i++) {
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java
index ada5f4c19f8..0760367445d 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java
@@ -63,7 +63,7 @@ public class ThrottlingAllocationTests extends ESAllocationTestCase {
 
         RoutingTable routingTable = createRecoveryRoutingTable(metaData.index("test"));
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("start one node, do reroute, only 3 should initialize");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build();
@@ -122,7 +122,7 @@ public class ThrottlingAllocationTests extends ESAllocationTestCase {
 
         RoutingTable routingTable = createRecoveryRoutingTable(metaData.index("test"));
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("start one node, do reroute, only 3 should initialize");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build();
@@ -190,7 +190,7 @@ public class ThrottlingAllocationTests extends ESAllocationTestCase {
 
         RoutingTable routingTable = createRecoveryRoutingTable(metaData.index("test"));
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("start one node, do reroute, only 5 should initialize");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build();
@@ -253,7 +253,7 @@ public class ThrottlingAllocationTests extends ESAllocationTestCase {
 
         RoutingTable routingTable = createRecoveryRoutingTable(metaData.index("test"));
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("start one node, do reroute, only 1 should initialize");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1"))).build();
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/UpdateNumberOfReplicasTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/UpdateNumberOfReplicasTests.java
index 7d90e5ee2f2..47142062809 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/UpdateNumberOfReplicasTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/UpdateNumberOfReplicasTests.java
@@ -58,7 +58,7 @@ public class UpdateNumberOfReplicasTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         assertThat(routingTable.index("test").shards().size(), equalTo(1));
         assertThat(routingTable.index("test").shard(0).size(), equalTo(2));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
index 579e87150a7..87e5fd24a04 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java
@@ -22,6 +22,7 @@ package org.elasticsearch.cluster.routing.allocation.decider;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterInfo;
 import org.elasticsearch.cluster.ClusterInfoService;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.DiskUsage;
 import org.elasticsearch.cluster.MockInternalClusterInfoService.DevNullClusterInfo;
@@ -117,7 +118,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData)
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData)
                 .routingTable(routingTable).build();
 
         logger.info("--> adding two nodes");
@@ -318,7 +319,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData)
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData)
                 .routingTable(routingTable).build();
 
         logger.info("--> adding node1 and node2 node");
@@ -587,7 +588,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData)
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData)
                 .routingTable(routingTable).build();
         logger.info("--> adding node1");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -657,7 +658,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData)
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData)
                 .routingTable(routingTable).build();
         logger.info("--> adding node1");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -764,7 +765,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test2"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData)
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData)
                 .routingTable(routingTable).build();
 
         logger.info("--> adding two nodes");
@@ -853,7 +854,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                 MASTER_DATA_ROLES, Version.CURRENT);
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().put(discoveryNode1).put(discoveryNode2).build();
 
-        ClusterState baseClusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+        ClusterState baseClusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metaData(metaData)
                 .routingTable(routingTable)
                 .nodes(discoveryNodes)
@@ -969,7 +970,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase {
                 singleton(DiscoveryNode.Role.DATA), Version.CURRENT);
 
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().put(discoveryNode1).put(discoveryNode2).build();
-        ClusterState baseClusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+        ClusterState baseClusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metaData(metaData)
                 .routingTable(routingTable)
                 .nodes(discoveryNodes)
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
index 8966659e63c..be50c5f5331 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
@@ -119,7 +119,7 @@ public class DiskThresholdDeciderUnitTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
                         .put(node_0)
@@ -183,7 +183,7 @@ public class DiskThresholdDeciderUnitTests extends ESAllocationTestCase {
                 .addAsNew(indexMetaData)
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding two nodes");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -242,7 +242,7 @@ public class DiskThresholdDeciderUnitTests extends ESAllocationTestCase {
         RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
         routingTableBuilder.addAsNew(metaData.index("test"));
         routingTableBuilder.addAsNew(metaData.index("other"));
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metaData(metaData).routingTable(routingTableBuilder.build()).build();
         RoutingAllocation allocation = new RoutingAllocation(null, null, clusterState, info, 0, false);
 
@@ -314,7 +314,7 @@ public class DiskThresholdDeciderUnitTests extends ESAllocationTestCase {
         routingTableBuilder.addAsNew(metaData.index("test"));
         routingTableBuilder.addAsNew(metaData.index("target"));
         routingTableBuilder.addAsNew(metaData.index("target2"));
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metaData(metaData).routingTable(routingTableBuilder.build()).build();
 
         AllocationService allocationService = createAllocationService();
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationTests.java
index f2260aecfcb..53932eef390 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationTests.java
@@ -68,7 +68,7 @@ public class EnableAllocationTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding two nodes and do rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -96,7 +96,7 @@ public class EnableAllocationTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding two nodes do rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -130,7 +130,7 @@ public class EnableAllocationTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("enabled"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding two nodes and do rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -175,7 +175,7 @@ public class EnableAllocationTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("always_disabled"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding one nodes and do rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
@@ -275,7 +275,7 @@ public class EnableAllocationTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         logger.info("--> adding one nodes and do rerouting");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
diff --git a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
index e7ca4f8382b..7b81d3ece27 100644
--- a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
@@ -29,6 +29,7 @@ import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESAllocationTestCase;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -71,7 +72,8 @@ public class ClusterSerializationTests extends ESAllocationTestCase {
 
         DiscoveryNodes nodes = DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")).put(newNode("node3")).build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).nodes(nodes).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).nodes(nodes)
+            .metaData(metaData).routingTable(routingTable).build();
 
         AllocationService strategy = createAllocationService();
         RoutingTable source = strategy.reroute(clusterState, "reroute").routingTable();
diff --git a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java
index 3a30bd5c6aa..99cde60f086 100644
--- a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.cluster.serialization;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
@@ -27,6 +28,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.test.ESAllocationTestCase;
 
@@ -51,8 +53,8 @@ public class ClusterStateToStringTests extends ESAllocationTestCase {
         DiscoveryNodes nodes = DiscoveryNodes.builder().put(new DiscoveryNode("node_foo", DummyTransportAddress.INSTANCE,
                 emptyMap(), emptySet(), Version.CURRENT)).localNodeId("node_foo").masterNodeId("node_foo").build();
 
-        ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT).nodes(nodes).metaData(metaData)
-                .routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).nodes(nodes)
+            .metaData(metaData).routingTable(routingTable).build();
 
         AllocationService strategy = createAllocationService();
         clusterState = ClusterState.builder(clusterState).routingTable(strategy.reroute(clusterState, "reroute").routingTable()).build();
diff --git a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
index dd71fa6a4a7..306eecf81b5 100644
--- a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
@@ -108,9 +108,9 @@ public class ClusterServiceTests extends ESTestCase {
     }
 
     TimedClusterService createTimedClusterService(boolean makeMaster) throws InterruptedException {
-        TimedClusterService timedClusterService = new TimedClusterService(Settings.EMPTY, null,
-            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-            threadPool, new ClusterName("ClusterServiceTests"));
+        TimedClusterService timedClusterService = new TimedClusterService(Settings.builder().put("cluster.name",
+            "ClusterServiceTests").build(), new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+            threadPool);
         timedClusterService.setLocalNode(new DiscoveryNode("node1", DummyTransportAddress.INSTANCE, emptyMap(),
             emptySet(), Version.CURRENT));
         timedClusterService.setNodeConnectionsService(new NodeConnectionsService(Settings.EMPTY, null, null) {
@@ -941,9 +941,8 @@ public class ClusterServiceTests extends ESTestCase {
 
         public volatile Long currentTimeOverride = null;
 
-        public TimedClusterService(Settings settings, OperationRouting operationRouting, ClusterSettings clusterSettings,
-                                   ThreadPool threadPool, ClusterName clusterName) {
-            super(settings, operationRouting, clusterSettings, threadPool, clusterName);
+        public TimedClusterService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool) {
+            super(settings, clusterSettings, threadPool);
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java b/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java
index e9ec914d0dc..b8ebe9e6308 100644
--- a/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java
@@ -37,6 +37,7 @@ import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESAllocationTestCase;
@@ -237,7 +238,7 @@ public class RoutingIteratorTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         Map<String, String> node1Attributes = new HashMap<>();
         node1Attributes.put("rack_id", "rack_1");
@@ -291,7 +292,7 @@ public class RoutingIteratorTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
                         .put(newNode("fred", "node1", singletonMap("disk", "ebs")))
@@ -365,7 +366,7 @@ public class RoutingIteratorTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
                 .put(newNode("node1"))
@@ -381,7 +382,8 @@ public class RoutingIteratorTests extends ESAllocationTestCase {
         routingTable = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING)).routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
 
-        OperationRouting operationRouting = new OperationRouting(Settings.Builder.EMPTY_SETTINGS, new AwarenessAllocationDecider());
+        OperationRouting operationRouting = new OperationRouting(Settings.EMPTY, new ClusterSettings(Settings.EMPTY,
+            ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
 
         GroupShardsIterator shardIterators = operationRouting.searchShards(clusterState, new String[]{"test"}, null, "_shards:0");
         assertThat(shardIterators.size(), equalTo(1));
@@ -426,7 +428,8 @@ public class RoutingIteratorTests extends ESAllocationTestCase {
                 .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
                 .build());
 
-        OperationRouting operationRouting = new OperationRouting(Settings.Builder.EMPTY_SETTINGS, new AwarenessAllocationDecider());
+        OperationRouting operationRouting = new OperationRouting(Settings.EMPTY, new ClusterSettings(Settings.EMPTY,
+            ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
 
         MetaData metaData = MetaData.builder()
                 .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(2))
@@ -436,7 +439,7 @@ public class RoutingIteratorTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test"))
                 .build();
 
-        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metaData(metaData).routingTable(routingTable).build();
+        ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
 
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
                         .put(newNode("node1"))
diff --git a/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java b/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
index ea81e958342..90a3a2e8c1e 100644
--- a/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
@@ -142,8 +142,7 @@ public class ZenFaultDetectionTests extends ESTestCase {
                             .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), singleton(TransportLivenessAction.NAME))
                             .build(),
                         new LocalTransport(settings, threadPool, version, namedWriteableRegistry, circuitBreakerService),
-                        threadPool,
-                        ClusterName.DEFAULT);
+                        threadPool);
         transportService.start();
         transportService.acceptIncomingRequests();
         return transportService;
@@ -221,13 +220,15 @@ public class ZenFaultDetectionTests extends ESTestCase {
     public void testMasterFaultDetectionConnectOnDisconnect() throws InterruptedException {
         Settings.Builder settings = Settings.builder();
         boolean shouldRetry = randomBoolean();
+        ClusterName clusterName = new ClusterName(randomAsciiOfLengthBetween(3, 20));
+
         // make sure we don't ping
         settings.put(FaultDetection.CONNECT_ON_NETWORK_DISCONNECT_SETTING.getKey(), shouldRetry)
-                .put(FaultDetection.PING_INTERVAL_SETTING.getKey(), "5m");
-        ClusterName clusterName = new ClusterName(randomAsciiOfLengthBetween(3, 20));
+                .put(FaultDetection.PING_INTERVAL_SETTING.getKey(), "5m").put("cluster.name", clusterName.value());
+
         final ClusterState state = ClusterState.builder(clusterName).nodes(buildNodesForA(false)).build();
         setState(clusterServiceA, state);
-        MasterFaultDetection masterFD = new MasterFaultDetection(settings.build(), threadPool, serviceA, clusterName,
+        MasterFaultDetection masterFD = new MasterFaultDetection(settings.build(), threadPool, serviceA,
             clusterServiceA);
         masterFD.start(nodeB, "test");
 
@@ -258,10 +259,11 @@ public class ZenFaultDetectionTests extends ESTestCase {
     public void testMasterFaultDetectionNotSizeLimited() throws InterruptedException {
         Settings.Builder settings = Settings.builder();
         boolean shouldRetry = randomBoolean();
+        ClusterName clusterName = new ClusterName(randomAsciiOfLengthBetween(3, 20));
         settings
             .put(FaultDetection.CONNECT_ON_NETWORK_DISCONNECT_SETTING.getKey(), shouldRetry)
-            .put(FaultDetection.PING_INTERVAL_SETTING.getKey(), "1s");
-        ClusterName clusterName = new ClusterName(randomAsciiOfLengthBetween(3, 20));
+            .put(FaultDetection.PING_INTERVAL_SETTING.getKey(), "1s")
+        .put("cluster.name", clusterName.value());
         final ClusterState stateNodeA = ClusterState.builder(clusterName).nodes(buildNodesForA(false)).build();
         setState(clusterServiceA, stateNodeA);
 
@@ -273,14 +275,14 @@ public class ZenFaultDetectionTests extends ESTestCase {
         serviceA.addTracer(pingProbeA);
         serviceB.addTracer(pingProbeB);
 
-        MasterFaultDetection masterFDNodeA = new MasterFaultDetection(settings.build(), threadPool, serviceA, clusterName,
+        MasterFaultDetection masterFDNodeA = new MasterFaultDetection(settings.build(), threadPool, serviceA,
             clusterServiceA);
         masterFDNodeA.start(nodeB, "test");
 
         final ClusterState stateNodeB = ClusterState.builder(clusterName).nodes(buildNodesForB(true)).build();
         setState(clusterServiceB, stateNodeB);
 
-        MasterFaultDetection masterFDNodeB = new MasterFaultDetection(settings.build(), threadPool, serviceB, clusterName,
+        MasterFaultDetection masterFDNodeB = new MasterFaultDetection(settings.build(), threadPool, serviceB,
             clusterServiceB);
         masterFDNodeB.start(nodeB, "test");
 
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java
index 90b31ff3342..8aa5114c387 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java
@@ -22,6 +22,7 @@ package org.elasticsearch.discovery.zen;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.discovery.zen.ping.ZenPing;
 import org.elasticsearch.test.ESTestCase;
@@ -51,7 +52,8 @@ public class ZenPingTests extends ESTestCase {
                 masterNode = nodes[randomInt(nodes.length - 1)];
             }
             boolean hasJoinedOnce = randomBoolean();
-            ZenPing.PingResponse ping = new ZenPing.PingResponse(nodes[node], masterNode, ClusterName.DEFAULT, hasJoinedOnce);
+            ZenPing.PingResponse ping = new ZenPing.PingResponse(nodes[node], masterNode, ClusterName.CLUSTER_NAME_SETTING.
+                getDefault(Settings.EMPTY), hasJoinedOnce);
             if (rarely()) {
                 // ignore some pings
                 continue;
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java b/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java
index 2845e6a5bf0..11816512583 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java
@@ -56,33 +56,39 @@ import static org.hamcrest.Matchers.greaterThan;
 
 public class UnicastZenPingIT extends ESTestCase {
     public void testSimplePings() throws InterruptedException {
-        Settings settings = Settings.EMPTY;
         int startPort = 11000 + randomIntBetween(0, 1000);
         int endPort = startPort + 10;
-        settings = Settings.builder().put(settings).put(TransportSettings.PORT.getKey(), startPort + "-" + endPort).build();
+        Settings settings = Settings.builder()
+            .put("cluster.name", "test")
+            .put(TransportSettings.PORT.getKey(), startPort + "-" + endPort).build();
+
+        Settings settingsMismatch = Settings.builder().put(settings)
+            .put("cluster.name", "mismatch")
+            .put(TransportSettings.PORT.getKey(), startPort + "-" + endPort).build();
 
         ThreadPool threadPool = new TestThreadPool(getClass().getName());
-        ClusterName test = new ClusterName("test");
-        ClusterName mismatch = new ClusterName("mismatch");
         NetworkService networkService = new NetworkService(settings);
         ElectMasterService electMasterService = new ElectMasterService(settings, Version.CURRENT);
 
-        NetworkHandle handleA = startServices(settings, threadPool, networkService, "UZP_A", test, Version.CURRENT);
-        NetworkHandle handleB = startServices(settings, threadPool, networkService, "UZP_B", test, Version.CURRENT);
-        NetworkHandle handleC = startServices(settings, threadPool, networkService, "UZP_C", new ClusterName("mismatch"), Version.CURRENT);
+        NetworkHandle handleA = startServices(settings, threadPool, networkService, "UZP_A", Version.CURRENT);
+        NetworkHandle handleB = startServices(settings, threadPool, networkService, "UZP_B", Version.CURRENT);
+        NetworkHandle handleC = startServices(settingsMismatch, threadPool, networkService, "UZP_C", Version.CURRENT);
         // just fake that no versions are compatible with this node
         Version previousVersion = VersionUtils.getPreviousVersion(Version.CURRENT.minimumCompatibilityVersion());
         Version versionD = VersionUtils.randomVersionBetween(random(), previousVersion.minimumCompatibilityVersion(), previousVersion);
-        NetworkHandle handleD = startServices(settings, threadPool, networkService, "UZP_D", test, versionD);
+        NetworkHandle handleD = startServices(settingsMismatch, threadPool, networkService, "UZP_D", versionD);
 
-        Settings hostsSettings = Settings.builder().putArray("discovery.zen.ping.unicast.hosts",
+        Settings hostsSettings = Settings.builder()
+                .putArray("discovery.zen.ping.unicast.hosts",
                 NetworkAddress.format(new InetSocketAddress(handleA.address.address().getAddress(), handleA.address.address().getPort())),
                 NetworkAddress.format(new InetSocketAddress(handleB.address.address().getAddress(), handleB.address.address().getPort())),
                 NetworkAddress.format(new InetSocketAddress(handleC.address.address().getAddress(), handleC.address.address().getPort())),
                 NetworkAddress.format(new InetSocketAddress(handleD.address.address().getAddress(), handleD.address.address().getPort())))
+                .put("cluster.name", "test")
                 .build();
 
-        UnicastZenPing zenPingA = new UnicastZenPing(hostsSettings, threadPool, handleA.transportService, test, Version.CURRENT, electMasterService, null);
+        Settings hostsSettingsMismatch = Settings.builder().put(hostsSettings).put(settingsMismatch).build();
+        UnicastZenPing zenPingA = new UnicastZenPing(hostsSettings, threadPool, handleA.transportService, Version.CURRENT, electMasterService, null);
         zenPingA.setPingContextProvider(new PingContextProvider() {
             @Override
             public DiscoveryNodes nodes() {
@@ -96,7 +102,7 @@ public class UnicastZenPingIT extends ESTestCase {
         });
         zenPingA.start();
 
-        UnicastZenPing zenPingB = new UnicastZenPing(hostsSettings, threadPool, handleB.transportService, test, Version.CURRENT, electMasterService, null);
+        UnicastZenPing zenPingB = new UnicastZenPing(hostsSettings, threadPool, handleB.transportService, Version.CURRENT, electMasterService, null);
         zenPingB.setPingContextProvider(new PingContextProvider() {
             @Override
             public DiscoveryNodes nodes() {
@@ -110,7 +116,7 @@ public class UnicastZenPingIT extends ESTestCase {
         });
         zenPingB.start();
 
-        UnicastZenPing zenPingC = new UnicastZenPing(hostsSettings, threadPool, handleC.transportService, mismatch, versionD, electMasterService, null);
+        UnicastZenPing zenPingC = new UnicastZenPing(hostsSettingsMismatch, threadPool, handleC.transportService, versionD, electMasterService, null);
         zenPingC.setPingContextProvider(new PingContextProvider() {
             @Override
             public DiscoveryNodes nodes() {
@@ -124,7 +130,7 @@ public class UnicastZenPingIT extends ESTestCase {
         });
         zenPingC.start();
 
-        UnicastZenPing zenPingD = new UnicastZenPing(hostsSettings, threadPool, handleD.transportService, mismatch, Version.CURRENT, electMasterService, null);
+        UnicastZenPing zenPingD = new UnicastZenPing(hostsSettingsMismatch, threadPool, handleD.transportService, Version.CURRENT, electMasterService, null);
         zenPingD.setPingContextProvider(new PingContextProvider() {
             @Override
             public DiscoveryNodes nodes() {
@@ -140,7 +146,7 @@ public class UnicastZenPingIT extends ESTestCase {
 
         try {
             logger.info("ping from UZP_A");
-            ZenPing.PingResponse[] pingResponses = zenPingA.pingAndWait(TimeValue.timeValueSeconds(10));
+            ZenPing.PingResponse[] pingResponses = zenPingA.pingAndWait(TimeValue.timeValueSeconds(1));
             assertThat(pingResponses.length, equalTo(1));
             assertThat(pingResponses[0].node().getId(), equalTo("UZP_B"));
             assertTrue(pingResponses[0].hasJoinedOnce());
@@ -148,19 +154,19 @@ public class UnicastZenPingIT extends ESTestCase {
 
             // ping again, this time from B,
             logger.info("ping from UZP_B");
-            pingResponses = zenPingB.pingAndWait(TimeValue.timeValueSeconds(10));
+            pingResponses = zenPingB.pingAndWait(TimeValue.timeValueSeconds(1));
             assertThat(pingResponses.length, equalTo(1));
             assertThat(pingResponses[0].node().getId(), equalTo("UZP_A"));
             assertFalse(pingResponses[0].hasJoinedOnce());
             assertCounters(handleB, handleA, handleB, handleC, handleD);
 
             logger.info("ping from UZP_C");
-            pingResponses = zenPingC.pingAndWait(TimeValue.timeValueSeconds(10));
+            pingResponses = zenPingC.pingAndWait(TimeValue.timeValueSeconds(1));
             assertThat(pingResponses.length, equalTo(0));
             assertCounters(handleC, handleA, handleB, handleC, handleD);
 
             logger.info("ping from UZP_D");
-            pingResponses = zenPingD.pingAndWait(TimeValue.timeValueSeconds(10));
+            pingResponses = zenPingD.pingAndWait(TimeValue.timeValueSeconds(1));
             assertThat(pingResponses.length, equalTo(0));
             assertCounters(handleD, handleA, handleB, handleC, handleD);
         } finally {
@@ -185,9 +191,9 @@ public class UnicastZenPingIT extends ESTestCase {
         }
     }
 
-    private NetworkHandle startServices(Settings settings, ThreadPool threadPool, NetworkService networkService, String nodeId, ClusterName clusterName, Version version) {
+    private NetworkHandle startServices(Settings settings, ThreadPool threadPool, NetworkService networkService, String nodeId, Version version) {
         NettyTransport transport = new NettyTransport(settings, threadPool, networkService, BigArrays.NON_RECYCLING_INSTANCE, version, new NamedWriteableRegistry(), new NoneCircuitBreakerService());
-        final TransportService transportService = new TransportService(transport, threadPool, clusterName);
+        final TransportService transportService = new TransportService(settings, transport, threadPool);
         transportService.start();
         transportService.acceptIncomingRequests();
         ConcurrentMap<TransportAddress, AtomicInteger> counters = new ConcurrentHashMap<>();
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueueTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueueTests.java
index 9c90e8de900..ab9aed6ba44 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueueTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueueTests.java
@@ -24,6 +24,7 @@ import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.discovery.zen.publish.PendingClusterStatesQueue.ClusterStateContext;
 import org.elasticsearch.test.ESTestCase;
@@ -235,7 +236,7 @@ public class PendingClusterStatesQueueTests extends ESTestCase {
             int masterIndex = randomInt(masters.length - 1);
             ClusterState state = lastClusterStatePerMaster[masterIndex];
             if (state == null) {
-                state = ClusterState.builder(ClusterName.DEFAULT).nodes(DiscoveryNodes.builder()
+                state = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).nodes(DiscoveryNodes.builder()
                                 .put(new DiscoveryNode(masters[masterIndex], DummyTransportAddress.INSTANCE,
                                         emptyMap(), emptySet(),Version.CURRENT)).masterNodeId(masters[masterIndex]).build()
                 ).build();
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java
index 7cc319c8579..ed09efaaada 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java
@@ -85,7 +85,7 @@ import static org.hamcrest.Matchers.nullValue;
 @TestLogging("discovery.zen.publish:TRACE")
 public class PublishClusterStateActionTests extends ESTestCase {
 
-    private static final ClusterName CLUSTER_NAME = ClusterName.DEFAULT;
+    private static final ClusterName CLUSTER_NAME = ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY);
 
     protected ThreadPool threadPool;
     protected Map<String, MockNode> nodes = new HashMap<>();
@@ -233,7 +233,7 @@ public class PublishClusterStateActionTests extends ESTestCase {
     }
 
     protected MockTransportService buildTransportService(Settings settings, Version version) {
-        MockTransportService transportService = MockTransportService.local(Settings.EMPTY, version, threadPool, CLUSTER_NAME);
+        MockTransportService transportService = MockTransportService.local(Settings.EMPTY, version, threadPool);
         transportService.start();
         transportService.acceptIncomingRequests();
         return transportService;
diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
index b7d8a916315..f41b55a5ea5 100644
--- a/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java
@@ -72,7 +72,7 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
                 .build();
 
         // assign all shards
-        ClusterState init = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+        ClusterState init = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metaData(metaDataOldClusterState)
                 .routingTable(routingTableOldClusterState)
                 .nodes(generateDiscoveryNodes(masterEligible))
@@ -129,7 +129,7 @@ public class GatewayMetaStateTests extends ESAllocationTestCase {
                 .build();
 
         // assign all shards
-        ClusterState init = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+        ClusterState init = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metaData(metaDataIndexCreated)
                 .routingTable(routingTableIndexCreated)
                 .nodes(generateDiscoveryNodes(masterEligible))
diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java b/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
index 3a3f5f67e77..4f65c5fafdd 100644
--- a/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
@@ -33,9 +33,9 @@ import java.io.IOException;
 public class GatewayServiceTests extends ESTestCase {
 
     private GatewayService createService(Settings.Builder settings) {
-        ClusterService clusterService = new ClusterService(Settings.EMPTY, null,
+        ClusterService clusterService = new ClusterService(Settings.builder().put("cluster.name", "GatewayServiceTests").build(),
                 new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-                null, new ClusterName("ClusterServiceTests"));
+                null);
         return new GatewayService(Settings.builder()
                 .put("http.enabled", "false")
                 .put("discovery.type", "local")
diff --git a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
index 3e088f4f82e..96e360550af 100644
--- a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
@@ -344,7 +344,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
         RoutingTable routingTable = RoutingTable.builder()
             .addAsRestore(metaData.index(shardId.getIndex()), new RestoreSource(snapshot, version, shardId.getIndexName()))
             .build();
-        ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+        ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metaData(metaData)
             .routingTable(routingTable)
             .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build();
@@ -423,7 +423,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
         RoutingTable routingTable = RoutingTable.builder()
             .addAsRestore(metaData.index(shardId.getIndex()), new RestoreSource(new Snapshot("test", new SnapshotId("test", UUIDs.randomBase64UUID())), Version.CURRENT, shardId.getIndexName()))
             .build();
-        ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+        ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .metaData(metaData)
             .routingTable(routingTable)
             .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build();
@@ -441,7 +441,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
         RoutingTable routingTable = RoutingTable.builder()
                 .addAsRecovery(metaData.index(shardId.getIndex()))
                 .build();
-        ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+        ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metaData(metaData)
                 .routingTable(routingTable)
                 .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build();
@@ -482,7 +482,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
         RoutingTable routingTable = RoutingTable.builder()
                 .addAsRecovery(metaData.index(shardId.getIndex()))
                 .build();
-        ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+        ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metaData(metaData)
                 .routingTable(routingTable)
                 .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build();
@@ -523,7 +523,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
         } else {
             routingTableBuilder.addAsRecovery(metaData.index(shardId.getIndex()));
         }
-        ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+        ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metaData(metaData)
                 .routingTable(routingTableBuilder.build())
                 .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build();
diff --git a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
index 38dc2f9e632..99d6c56c88d 100644
--- a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
@@ -302,7 +302,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
                                         .build())
                 )
                 .build();
-        ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+        ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metaData(metaData)
                 .routingTable(routingTable)
                 .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build();
@@ -324,7 +324,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
                                         .build())
                 )
                 .build();
-        ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.DEFAULT)
+        ClusterState state = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
                 .metaData(metaData)
                 .routingTable(routingTable)
                 .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build();
diff --git a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
index 9f002afd2dc..32f9bbbd84c 100644
--- a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
+++ b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.http;
 
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.bytes.ByteBufferBytesReference;
@@ -73,8 +72,8 @@ public class HttpServerTests extends ESTestCase {
                 throw new IllegalArgumentException("test error");
             });
 
-        ClusterService clusterService = new ClusterService(Settings.EMPTY, null,
-            new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null, ClusterName.DEFAULT);
+        ClusterService clusterService = new ClusterService(Settings.EMPTY,
+            new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null);
         NodeService nodeService = new NodeService(Settings.EMPTY, null, null, null, null, null, null, null, null, null,
             clusterService, null);
         httpServer = new HttpServer(settings, httpServerTransport, restController, nodeService, circuitBreakerService);
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
index 8960f9dfe6e..cc8b05819c6 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
@@ -77,7 +77,7 @@ public class DynamicMappingDisabledTests extends ESSingleNodeTestCase {
         clusterService = createClusterService(THREAD_POOL);
         transport =
                 new LocalTransport(settings, THREAD_POOL, Version.CURRENT, new NamedWriteableRegistry(), new NoneCircuitBreakerService());
-        transportService = new TransportService(transport, THREAD_POOL, clusterService.state().getClusterName());
+        transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL);
         indicesService = getInstanceFromNode(IndicesService.class);
         shardStateAction = new ShardStateAction(settings, clusterService, transportService, null, null, THREAD_POOL);
         actionFilters = new ActionFilters(Collections.emptySet());
diff --git a/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
index 84e83db6d1d..86d19ae1ea4 100644
--- a/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
+++ b/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
@@ -143,7 +143,7 @@ public class ClusterStateChanges {
         }
 
         // services
-        TransportService transportService = new TransportService(settings, transport, threadPool, null);
+        TransportService transportService = new TransportService(settings, transport, threadPool);
         MetaDataIndexUpgradeService metaDataIndexUpgradeService = new MetaDataIndexUpgradeService(settings, null, null) {
             // metaData upgrader should do nothing
             @Override
diff --git a/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
index 8c63c001a1e..78ef13dde56 100644
--- a/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
@@ -267,7 +267,7 @@ public class IndicesClusterStateServiceRandomUpdatesTests extends AbstractIndice
         final Executor executor = mock(Executor.class);
         when(threadPool.generic()).thenReturn(executor);
         final MockIndicesService indicesService = new MockIndicesService();
-        final TransportService transportService = new TransportService(Settings.EMPTY, null, threadPool, null);
+        final TransportService transportService = new TransportService(Settings.EMPTY, null, threadPool);
         final ClusterService clusterService = mock(ClusterService.class);
         final RepositoriesService repositoriesService = new RepositoriesService(Settings.EMPTY, clusterService,
             transportService, null, null);
diff --git a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
index f197073033d..e558f0f2a12 100644
--- a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
@@ -88,7 +88,7 @@ public class IndicesStoreTests extends ESTestCase {
     public void before() {
         localNode = new DiscoveryNode("abc", new LocalTransportAddress("abc"), emptyMap(), emptySet(), Version.CURRENT);
         clusterService = createClusterService(threadPool);
-        indicesStore = new IndicesStore(Settings.EMPTY, null, clusterService, new TransportService(null, null, clusterService.state().getClusterName()), null);
+        indicesStore = new IndicesStore(Settings.EMPTY, null, clusterService, new TransportService(clusterService.getSettings(), null, null), null);
     }
 
     @After
diff --git a/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java b/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java
index e7101869692..fd332321618 100644
--- a/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java
+++ b/core/src/test/java/org/elasticsearch/mget/SimpleMgetIT.java
@@ -153,8 +153,8 @@ public class SimpleMgetIT extends ESIntegTestCase {
                         .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, between(2, DEFAULT_MAX_NUM_SHARDS))));
         ensureYellow();
 
-        final String id = routingKeyForShard("test", "test", 0);
-        final String routingOtherShard = routingKeyForShard("test", "test", 1);
+        final String id = routingKeyForShard("test", 0);
+        final String routingOtherShard = routingKeyForShard("test", 1);
 
         client().prepareIndex("test", "test", id).setRefreshPolicy(IMMEDIATE).setRouting(routingOtherShard)
                 .setSource(jsonBuilder().startObject().field("foo", "bar").endObject())
diff --git a/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java b/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java
index 87abc20a0de..226f6eae958 100644
--- a/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java
+++ b/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java
@@ -68,13 +68,6 @@ public class InternalSettingsPreparerTests extends ESTestCase {
         assertTrue(configDir, configDir.startsWith(home));
     }
 
-    public void testClusterNameDefault() {
-        Settings settings = InternalSettingsPreparer.prepareSettings(Settings.EMPTY);
-        assertEquals(ClusterName.DEFAULT.value(), settings.get(ClusterName.CLUSTER_NAME_SETTING.getKey()));
-        settings = InternalSettingsPreparer.prepareEnvironment(baseEnvSettings, null).settings();
-        assertEquals(ClusterName.DEFAULT.value(), settings.get(ClusterName.CLUSTER_NAME_SETTING.getKey()));
-    }
-
     public void testReplacePromptPlaceholders() {
         MockTerminal terminal = new MockTerminal();
         terminal.addTextInput("text");
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTestCase.java
index a3723428874..fbd69755f24 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTestCase.java
@@ -69,8 +69,8 @@ public abstract class ShardSizeTestCase extends ESIntegTestCase {
 
         List<IndexRequestBuilder> docs = new ArrayList<>();
 
-        routing1 = routingKeyForShard("idx", "type", 0);
-        routing2 = routingKeyForShard("idx", "type", 1);
+        routing1 = routingKeyForShard("idx", 0);
+        routing2 = routingKeyForShard("idx", 1);
 
         docs.addAll(indexDoc(routing1, "1", 5));
         docs.addAll(indexDoc(routing1, "2", 4));
diff --git a/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
index 68c4c76d7e8..4fa99b3b80c 100644
--- a/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
+++ b/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
@@ -66,7 +66,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
     protected DiscoveryNode nodeB;
     protected MockTransportService serviceB;
 
-    protected abstract MockTransportService build(Settings settings, Version version, ClusterName clusterName);
+    protected abstract MockTransportService build(Settings settings, Version version);
 
     @Override
     @Before
@@ -79,8 +79,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
                     .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "")
                     .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
                     .build(),
-                version0,
-                ClusterName.DEFAULT);
+                version0);
         serviceA.acceptIncomingRequests();
         nodeA = new DiscoveryNode("TS_A", serviceA.boundAddress().publishAddress(), emptyMap(), emptySet(), version0);
         serviceB = build(
@@ -89,8 +88,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
                     .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "")
                     .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
                     .build(),
-                version1,
-                ClusterName.DEFAULT);
+                version1);
         serviceB.acceptIncomingRequests();
         nodeB = new DiscoveryNode("TS_B", serviceB.boundAddress().publishAddress(), emptyMap(), emptySet(), version1);
 
@@ -1299,8 +1297,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
                     .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "")
                     .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
                     .build(),
-                version0,
-                ClusterName.DEFAULT);
+                version0);
         AtomicBoolean requestProcessed = new AtomicBoolean();
         service.registerRequestHandler("action", TestRequest::new, ThreadPool.Names.SAME,
                 (request, channel) -> {
diff --git a/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java b/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java
index 3c382c39509..95ec6dad9bd 100644
--- a/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java
@@ -69,7 +69,7 @@ public class NettySizeHeaderFrameDecoderTests extends ESTestCase {
         nettyTransport = new NettyTransport(settings, threadPool, networkService, bigArrays, Version.CURRENT, new NamedWriteableRegistry(),
             new NoneCircuitBreakerService());
         nettyTransport.start();
-        TransportService transportService = new TransportService(nettyTransport, threadPool, ClusterName.DEFAULT);
+        TransportService transportService = new TransportService(settings, nettyTransport, threadPool);
         nettyTransport.transportServiceAdapter(transportService.createAdapter());
 
         TransportAddress[] boundAddresses = nettyTransport.boundAddress().boundAddresses();
diff --git a/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java b/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java
index a111911e994..2213d7eb4cf 100644
--- a/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java
@@ -57,7 +57,7 @@ public class NettyTransportServiceHandshakeTests extends ESTestCase {
 
     private List<TransportService> transportServices = new ArrayList<>();
 
-    private NetworkHandle startServices(String nodeNameAndId, Settings settings, Version version, ClusterName clusterName) {
+    private NetworkHandle startServices(String nodeNameAndId, Settings settings, Version version) {
         NettyTransport transport =
                 new NettyTransport(
                         settings,
@@ -67,7 +67,7 @@ public class NettyTransportServiceHandshakeTests extends ESTestCase {
                         Version.CURRENT,
                         new NamedWriteableRegistry(),
                         new NoneCircuitBreakerService());
-        TransportService transportService = new MockTransportService(settings, transport, threadPool, clusterName);
+        TransportService transportService = new MockTransportService(settings, transport, threadPool);
         transportService.start();
         transportService.acceptIncomingRequests();
         DiscoveryNode node =
@@ -99,17 +99,14 @@ public class NettyTransportServiceHandshakeTests extends ESTestCase {
     }
 
     public void testConnectToNodeLight() {
-        Settings settings = Settings.EMPTY;
+        Settings settings = Settings.builder().put("cluster.name", "test").build();
 
-        ClusterName test = new ClusterName("test");
-
-        NetworkHandle handleA = startServices("TS_A", settings, Version.CURRENT, test);
+        NetworkHandle handleA = startServices("TS_A", settings, Version.CURRENT);
         NetworkHandle handleB =
                 startServices(
                         "TS_B",
                         settings,
-                        VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.CURRENT),
-                        test);
+                        VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.CURRENT));
         DiscoveryNode discoveryNode = new DiscoveryNode(
             "",
             handleB.discoveryNode.getAddress(),
@@ -127,10 +124,9 @@ public class NettyTransportServiceHandshakeTests extends ESTestCase {
     }
 
     public void testMismatchedClusterName() {
-        Settings settings = Settings.EMPTY;
 
-        NetworkHandle handleA = startServices("TS_A", settings, Version.CURRENT, new ClusterName("a"));
-        NetworkHandle handleB = startServices("TS_B", settings, Version.CURRENT, new ClusterName("b"));
+        NetworkHandle handleA = startServices("TS_A", Settings.builder().put("cluster.name", "a").build(), Version.CURRENT);
+        NetworkHandle handleB = startServices("TS_B", Settings.builder().put("cluster.name", "b").build(), Version.CURRENT);
         DiscoveryNode discoveryNode = new DiscoveryNode(
             "",
             handleB.discoveryNode.getAddress(),
@@ -144,12 +140,10 @@ public class NettyTransportServiceHandshakeTests extends ESTestCase {
 }
 
     public void testIncompatibleVersions() {
-        Settings settings = Settings.EMPTY;
-
-        ClusterName test = new ClusterName("test");
-        NetworkHandle handleA = startServices("TS_A", settings, Version.CURRENT, test);
+        Settings settings = Settings.builder().put("cluster.name", "test").build();
+        NetworkHandle handleA = startServices("TS_A", settings, Version.CURRENT);
         NetworkHandle handleB =
-                startServices("TS_B", settings, VersionUtils.getPreviousVersion(Version.CURRENT.minimumCompatibilityVersion()), test);
+                startServices("TS_B", settings, VersionUtils.getPreviousVersion(Version.CURRENT.minimumCompatibilityVersion()));
         DiscoveryNode discoveryNode = new DiscoveryNode(
             "",
             handleB.discoveryNode.getAddress(),
@@ -163,15 +157,14 @@ public class NettyTransportServiceHandshakeTests extends ESTestCase {
     }
 
     public void testIgnoreMismatchedClusterName() {
-        Settings settings = Settings.EMPTY;
+        Settings settings = Settings.builder().put("cluster.name", "a").build();
 
-        NetworkHandle handleA = startServices("TS_A", settings, Version.CURRENT, new ClusterName("a"));
+        NetworkHandle handleA = startServices("TS_A", settings, Version.CURRENT);
         NetworkHandle handleB =
                 startServices(
                         "TS_B",
-                        settings,
-                        VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.CURRENT),
-                        new ClusterName("b")
+                        Settings.builder().put("cluster.name", "b").build(),
+                        VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.CURRENT)
                 );
         DiscoveryNode discoveryNode = new DiscoveryNode(
             "",
diff --git a/core/src/test/java/org/elasticsearch/transport/TransportModuleTests.java b/core/src/test/java/org/elasticsearch/transport/TransportModuleTests.java
index 4eb14014cfc..20413888281 100644
--- a/core/src/test/java/org/elasticsearch/transport/TransportModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/TransportModuleTests.java
@@ -44,8 +44,8 @@ public class TransportModuleTests extends ModuleTestCase {
 
     static class FakeTransportService extends TransportService {
         @Inject
-        public FakeTransportService(Settings settings, Transport transport, ThreadPool threadPool, ClusterName clusterName) {
-            super(settings, transport, threadPool, clusterName);
+        public FakeTransportService(Settings settings, Transport transport, ThreadPool threadPool) {
+            super(settings, transport, threadPool);
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/transport/local/SimpleLocalTransportTests.java b/core/src/test/java/org/elasticsearch/transport/local/SimpleLocalTransportTests.java
index 2907a0a6ca8..c4458b7883d 100644
--- a/core/src/test/java/org/elasticsearch/transport/local/SimpleLocalTransportTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/local/SimpleLocalTransportTests.java
@@ -29,8 +29,8 @@ import org.elasticsearch.transport.AbstractSimpleTransportTestCase;
 public class SimpleLocalTransportTests extends AbstractSimpleTransportTestCase {
 
     @Override
-    protected MockTransportService build(Settings settings, Version version, ClusterName clusterName) {
-        MockTransportService transportService = MockTransportService.local(settings, version, threadPool, clusterName);
+    protected MockTransportService build(Settings settings, Version version) {
+        MockTransportService transportService = MockTransportService.local(settings, version, threadPool);
         transportService.start();
         return transportService;
     }
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java
index de156659082..a98a038f6fd 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java
@@ -58,6 +58,7 @@ public class NettyScheduledPingTests extends ESTestCase {
         Settings settings = Settings.builder()
             .put(NettyTransport.PING_SCHEDULE.getKey(), "5ms")
             .put(TransportSettings.PORT.getKey(), 0)
+            .put("cluster.name", "test")
             .build();
 
         CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService();
@@ -65,15 +66,14 @@ public class NettyScheduledPingTests extends ESTestCase {
         NamedWriteableRegistry registryA = new NamedWriteableRegistry();
         final NettyTransport nettyA = new NettyTransport(settings, threadPool, new NetworkService(settings),
             BigArrays.NON_RECYCLING_INSTANCE, Version.CURRENT, registryA, circuitBreakerService);
-        ClusterName test = new ClusterName("test");
-        MockTransportService serviceA = new MockTransportService(settings, nettyA, threadPool, test);
+        MockTransportService serviceA = new MockTransportService(settings, nettyA, threadPool);
         serviceA.start();
         serviceA.acceptIncomingRequests();
 
         NamedWriteableRegistry registryB = new NamedWriteableRegistry();
         final NettyTransport nettyB = new NettyTransport(settings, threadPool, new NetworkService(settings),
             BigArrays.NON_RECYCLING_INSTANCE, Version.CURRENT, registryB, circuitBreakerService);
-        MockTransportService serviceB = new MockTransportService(settings, nettyB, threadPool, test);
+        MockTransportService serviceB = new MockTransportService(settings, nettyB, threadPool);
 
         serviceB.start();
         serviceB.acceptIncomingRequests();
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/SimpleNettyTransportTests.java b/core/src/test/java/org/elasticsearch/transport/netty/SimpleNettyTransportTests.java
index dac416128e5..f3cfcc3bebe 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/SimpleNettyTransportTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/SimpleNettyTransportTests.java
@@ -40,9 +40,9 @@ import static org.hamcrest.Matchers.containsString;
 public class SimpleNettyTransportTests extends AbstractSimpleTransportTestCase {
 
     @Override
-    protected MockTransportService build(Settings settings, Version version, ClusterName clusterName) {
+    protected MockTransportService build(Settings settings, Version version) {
         settings = Settings.builder().put(settings).put(TransportSettings.PORT.getKey(), "0").build();
-        MockTransportService transportService = MockTransportService.nettyFromThreadPool(settings, version, threadPool, clusterName);
+        MockTransportService transportService = MockTransportService.nettyFromThreadPool(settings, version, threadPool);
         transportService.start();
         return transportService;
     }
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java
index 27bb7fdc23f..c0751fe99e2 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java
@@ -719,8 +719,8 @@ public class IndicesRequestTests extends ESIntegTestCase {
         private final Map<String, List<TransportRequest>> requests = new HashMap<>();
 
         @Inject
-        public InterceptingTransportService(Settings settings, Transport transport, ThreadPool threadPool, ClusterName clusterName) {
-            super(settings, transport, threadPool, clusterName);
+        public InterceptingTransportService(Settings settings, Transport transport, ThreadPool threadPool) {
+            super(settings, transport, threadPool);
         }
 
         synchronized List<TransportRequest> consumeRequests(String action) {
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java
index b716050c036..2990461c0a9 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java
@@ -39,9 +39,9 @@ import java.util.List;
 
 public class TransportRethrottleAction extends TransportTasksAction<BulkByScrollTask, RethrottleRequest, ListTasksResponse, TaskInfo> {
     @Inject
-    public TransportRethrottleAction(Settings settings, ClusterName clusterName, ThreadPool threadPool, ClusterService clusterService,
+    public TransportRethrottleAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
             TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, RethrottleAction.NAME, clusterName, threadPool, clusterService, transportService, actionFilters,
+        super(settings, RethrottleAction.NAME, threadPool, clusterService, transportService, actionFilters,
                 indexNameExpressionResolver, RethrottleRequest::new, ListTasksResponse::new, ThreadPool.Names.MANAGEMENT);
     }
 
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
index 956732113c1..319222caa7e 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
@@ -67,7 +67,7 @@ public class Ec2DiscoveryTests extends ESTestCase {
 
     @Before
     public void createTransportService() {
-        transportService = MockTransportService.local(Settings.EMPTY, Version.CURRENT, threadPool, ClusterName.DEFAULT);
+        transportService = MockTransportService.local(Settings.EMPTY, Version.CURRENT, threadPool);
     }
 
     protected List<DiscoveryNode> buildDynamicNodes(Settings nodeSettings, int nodes) {
diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java
index 78d96fd0163..e1041360b1f 100644
--- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java
+++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java
@@ -95,7 +95,7 @@ public class GceDiscoveryTests extends ESTestCase {
 
     @Before
     public void createTransportService() {
-        transportService = MockTransportService.local(Settings.EMPTY, Version.CURRENT, threadPool, ClusterName.DEFAULT);
+        transportService = MockTransportService.local(Settings.EMPTY, Version.CURRENT, threadPool);
     }
 
     @Before
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
index 3bb77d34f8d..128b0d0e315 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
@@ -81,7 +81,7 @@ public class MockInternalClusterInfoService extends InternalClusterInfoService {
                                           TransportIndicesStatsAction transportIndicesStatsAction,
                                           ClusterService clusterService, ThreadPool threadPool) {
         super(settings, clusterSettings, transportNodesStatsAction, transportIndicesStatsAction, clusterService, threadPool);
-        this.clusterName = ClusterName.clusterNameFromSettings(settings);
+        this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
         stats[0] = makeStats("node_t1", new DiskUsage("node_t1", "n1", "/dev/null", 100, 100));
         stats[1] = makeStats("node_t2", new DiskUsage("node_t2", "n2", "/dev/null", 100, 100));
         stats[2] = makeStats("node_t3", new DiskUsage("node_t3", "n3", "/dev/null", 100, 100));
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
index 3e542231001..fe7ba74a327 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
@@ -43,9 +43,9 @@ import static junit.framework.TestCase.fail;
 public class ClusterServiceUtils {
 
     public static ClusterService createClusterService(ThreadPool threadPool) {
-        ClusterService clusterService = new ClusterService(Settings.EMPTY, null,
+        ClusterService clusterService = new ClusterService(Settings.builder().put("cluster.name", "ClusterServiceTests").build(),
                 new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
-                threadPool, new ClusterName("ClusterServiceTests"));
+                threadPool);
         clusterService.setLocalNode(new DiscoveryNode("node", DummyTransportAddress.INSTANCE, Collections.emptyMap(),
                 new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())),Version.CURRENT));
         clusterService.setNodeConnectionsService(new NodeConnectionsService(Settings.EMPTY, null, null) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index d892187fa94..095b1339800 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -80,7 +80,6 @@ import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -1980,8 +1979,8 @@ public abstract class ESIntegTestCase extends ESTestCase {
      * Compute a routing key that will route documents to the <code>shard</code>-th shard
      * of the provided index.
      */
-    protected String routingKeyForShard(String index, String type, int shard) {
-        return internalCluster().routingKeyForShard(resolveIndex(index), type, shard, random());
+    protected String routingKeyForShard(String index, int shard) {
+        return internalCluster().routingKeyForShard(resolveIndex(index), shard, random());
     }
 
     /**
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
index f4d5b4dd482..4cf058a6a5d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
@@ -34,7 +34,6 @@ import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.env.Environment;
@@ -43,7 +42,6 @@ import org.elasticsearch.index.IndexService;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.node.MockNode;
 import org.elasticsearch.node.Node;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.internal.SearchContext;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java b/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java
index c08c2793eaa..8216f67cfed 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ExternalNode.java
@@ -33,7 +33,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.node.Node;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
 
 import java.io.Closeable;
 import java.io.IOException;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index 66f02c8a376..bb33ad17dde 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -1760,7 +1760,7 @@ public final class InternalTestCluster extends TestCluster {
         }
     }
 
-    synchronized String routingKeyForShard(Index index, String type, int shard, Random random) {
+    synchronized String routingKeyForShard(Index index, int shard, Random random) {
         assertThat(shard, greaterThanOrEqualTo(0));
         assertThat(shard, greaterThanOrEqualTo(0));
         for (NodeAndClient n : nodes.values()) {
@@ -1770,7 +1770,7 @@ public final class InternalTestCluster extends TestCluster {
             IndexService indexService = indicesService.indexService(index);
             if (indexService != null) {
                 assertThat(indexService.getIndexSettings().getSettings().getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS, -1), greaterThan(shard));
-                OperationRouting operationRouting = getInstanceFromNode(OperationRouting.class, node);
+                OperationRouting operationRouting = clusterService.operationRouting();
                 while (true) {
                     String routing = RandomStrings.randomAsciiOfLength(random, 10);
                     final int targetShard = operationRouting.indexShards(clusterService.state(), index.getName(), null, routing).shardId().getId();
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
index eaa6faeca3f..110850e9a0f 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
@@ -94,29 +94,28 @@ public class MockTransportService extends TransportService {
         }
     }
 
-    public static MockTransportService local(Settings settings, Version version, ThreadPool threadPool, ClusterName clusterName) {
+    public static MockTransportService local(Settings settings, Version version, ThreadPool threadPool) {
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
         Transport transport = new LocalTransport(settings, threadPool, version, namedWriteableRegistry, new NoneCircuitBreakerService());
-        return new MockTransportService(settings, transport, threadPool, clusterName);
+        return new MockTransportService(settings, transport, threadPool);
     }
 
     public static MockTransportService nettyFromThreadPool(
             Settings settings,
             Version version,
-            ThreadPool threadPool,
-            ClusterName clusterName) {
+            ThreadPool threadPool) {
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
         Transport transport = new NettyTransport(settings, threadPool, new NetworkService(settings), BigArrays.NON_RECYCLING_INSTANCE,
                 version, namedWriteableRegistry, new NoneCircuitBreakerService());
-        return new MockTransportService(Settings.EMPTY, transport, threadPool, clusterName);
+        return new MockTransportService(Settings.EMPTY, transport, threadPool);
     }
 
 
     private final Transport original;
 
     @Inject
-    public MockTransportService(Settings settings, Transport transport, ThreadPool threadPool, ClusterName clusterName) {
-        super(settings, new LookupTestTransport(transport), threadPool, clusterName);
+    public MockTransportService(Settings settings, Transport transport, ThreadPool threadPool) {
+        super(settings, new LookupTestTransport(transport), threadPool);
         this.original = transport;
     }
 

From fb2a48d0f0fbdb58196557df24f148bb3316eba1 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Fri, 17 Jun 2016 17:14:01 +0200
Subject: [PATCH 0931/1311] Revert "Remove support for sorting terms
 aggregation by ascending count"

This is delayed after alpha4 since Kibana relies on it.
---
 .../bucket/terms/InternalOrder.java           | 19 +++-
 .../bucket/terms/InternalTerms.java           |  2 +-
 .../aggregations/bucket/terms/Terms.java      |  6 +-
 .../bucket/terms/TermsAggregationBuilder.java |  2 +-
 .../bucket/terms/TermsParser.java             | 20 +---
 .../aggregations/bucket/ShardSizeTermsIT.java | 24 ++---
 .../bucket/TermsDocCountErrorIT.java          | 93 +++++++++++++++++++
 .../aggregations/bucket/TermsTests.java       |  2 +-
 .../bucket/terms-aggregation.asciidoc         | 13 +--
 .../migrate_5_0/aggregations.asciidoc         |  4 -
 .../messy/tests/DoubleTermsTests.java         |  7 +-
 .../messy/tests/LongTermsTests.java           |  7 +-
 .../messy/tests/MinDocCountTests.java         | 48 ++++++++--
 .../messy/tests/StringTermsTests.java         |  7 +-
 .../test/search.aggregation/20_terms.yaml     | 15 +--
 15 files changed, 194 insertions(+), 75 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalOrder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalOrder.java
index 5ebeb6ee7fb..f3f87c09dca 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalOrder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalOrder.java
@@ -46,8 +46,9 @@ import java.util.Objects;
 class InternalOrder extends Terms.Order {
 
     private static final byte COUNT_DESC_ID = 1;
-    private static final byte TERM_DESC_ID = 2;
-    private static final byte TERM_ASC_ID = 3;
+    private static final byte COUNT_ASC_ID = 2;
+    private static final byte TERM_DESC_ID = 3;
+    private static final byte TERM_ASC_ID = 4;
 
     /**
      * Order by the (higher) count of each term.
@@ -59,6 +60,17 @@ class InternalOrder extends Terms.Order {
         }
     });
 
+    /**
+     * Order by the (lower) count of each term.
+     */
+    public static final InternalOrder COUNT_ASC = new InternalOrder(COUNT_ASC_ID, "_count", true, new Comparator<Terms.Bucket>() {
+
+        @Override
+        public int compare(Terms.Bucket o1, Terms.Bucket o2) {
+            return Long.compare(o1.getDocCount(), o2.getDocCount());
+        }
+    });
+
     /**
      * Order by the terms.
      */
@@ -81,7 +93,7 @@ class InternalOrder extends Terms.Order {
         }
     });
 
-    public static boolean isCountOrder(Terms.Order order) {
+    public static boolean isCountDesc(Terms.Order order) {
         if (order == COUNT_DESC) {
             return true;
         } else if (order instanceof CompoundOrder) {
@@ -336,6 +348,7 @@ class InternalOrder extends Terms.Order {
             byte id = in.readByte();
             switch (id) {
                 case COUNT_DESC_ID: return absoluteOrder ? new CompoundOrder(Collections.singletonList((Terms.Order) InternalOrder.COUNT_DESC)) : InternalOrder.COUNT_DESC;
+                case COUNT_ASC_ID: return absoluteOrder ? new CompoundOrder(Collections.singletonList((Terms.Order) InternalOrder.COUNT_ASC)) : InternalOrder.COUNT_ASC;
                 case TERM_DESC_ID: return InternalOrder.TERM_DESC;
                 case TERM_ASC_ID: return InternalOrder.TERM_ASC;
                 case Aggregation.ID:
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
index 5784066f85c..fcbd66dced6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
@@ -192,7 +192,7 @@ public abstract class InternalTerms<A extends InternalTerms, B extends InternalT
             final long thisAggDocCountError;
             if (terms.buckets.size() < this.shardSize || InternalOrder.isTermOrder(order)) {
                 thisAggDocCountError = 0;
-            } else if (InternalOrder.isCountOrder(this.order)) {
+            } else if (InternalOrder.isCountDesc(this.order)) {
                 thisAggDocCountError = terms.buckets.get(terms.buckets.size() - 1).docCount;
             } else {
                 thisAggDocCountError = -1;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/Terms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/Terms.java
index 3dd186c80f6..16fb7f1d81e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/Terms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/Terms.java
@@ -100,10 +100,10 @@ public interface Terms extends MultiBucketsAggregation {
     static abstract class Order implements ToXContent {
 
         /**
-         * @return a bucket ordering strategy that sorts buckets by their document counts (descending)
+         * @return a bucket ordering strategy that sorts buckets by their document counts (ascending or descending)
          */
-        public static Order count() {
-            return InternalOrder.COUNT_DESC;
+        public static Order count(boolean asc) {
+            return asc ? InternalOrder.COUNT_ASC : InternalOrder.COUNT_DESC;
         }
 
         /**
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
index e64a89e3a0f..87567d64df4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
@@ -53,7 +53,7 @@ public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder<Valu
     public static final ParseField SHOW_TERM_DOC_COUNT_ERROR = new ParseField("show_term_doc_count_error");
     public static final ParseField ORDER_FIELD = new ParseField("order");
 
-    private Terms.Order order = Terms.Order.compound(Terms.Order.count(), Terms.Order.term(true));
+    private Terms.Order order = Terms.Order.compound(Terms.Order.count(false), Terms.Order.term(true));
     private IncludeExclude includeExclude = null;
     private String executionHint = null;
     private SubAggCollectionMode collectMode = null;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsParser.java
index 7b61abcdccd..2a67dbe2218 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsParser.java
@@ -77,19 +77,9 @@ public class TermsParser extends AbstractTermsParser {
     @Override
     public boolean parseSpecial(String aggregationName, XContentParser parser, ParseFieldMatcher parseFieldMatcher, Token token,
             String currentFieldName, Map<ParseField, Object> otherOptions) throws IOException {
-        if (token == Token.VALUE_STRING) {
+        if (token == XContentParser.Token.START_OBJECT) {
             if (parseFieldMatcher.match(currentFieldName, TermsAggregationBuilder.ORDER_FIELD)) {
-                if ("_count".equals(parser.text())) {
-                    otherOptions.put(TermsAggregationBuilder.ORDER_FIELD,
-                        Collections.singletonList(new OrderElement(parser.text(), false)));
-                    return true;
-                }
-            }
-            return false;
-        } else if (token == XContentParser.Token.START_OBJECT) {
-            if (parseFieldMatcher.match(currentFieldName, TermsAggregationBuilder.ORDER_FIELD)) {
-                otherOptions.put(TermsAggregationBuilder.ORDER_FIELD,
-                    Collections.singletonList(parseOrderParam(aggregationName, parser)));
+                otherOptions.put(TermsAggregationBuilder.ORDER_FIELD, Collections.singletonList(parseOrderParam(aggregationName, parser)));
                 return true;
             }
         } else if (token == XContentParser.Token.START_ARRAY) {
@@ -127,10 +117,6 @@ public class TermsParser extends AbstractTermsParser {
             } else if (token == XContentParser.Token.VALUE_STRING) {
                 String dir = parser.text();
                 if ("asc".equalsIgnoreCase(dir)) {
-                    if ("_count".equals(orderKey)) {
-                        throw new ParsingException(parser.getTokenLocation(),
-                            "Sort by ascending _count is not supported in [" + aggregationName + "].");
-                    }
                     orderAsc = true;
                 } else if ("desc".equalsIgnoreCase(dir)) {
                     orderAsc = false;
@@ -181,7 +167,7 @@ public class TermsParser extends AbstractTermsParser {
             return Order.term(asc);
         }
         if ("_count".equals(key)) {
-            return Order.count();
+            return Order.count(asc);
         }
         return Order.aggregation(key, asc);
     }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java
index 82b60289c75..ecc16b85f13 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ShardSizeTermsIT.java
@@ -39,7 +39,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count()))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count(false)))
                 .execute().actionGet();
 
         Terms  terms = response.getAggregations().get("keys");
@@ -62,7 +62,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3).shardSize(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count()))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count(false)))
                 .execute().actionGet();
 
         Terms  terms = response.getAggregations().get("keys");
@@ -86,7 +86,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count()))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count(false)))
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
@@ -110,7 +110,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type").setRouting(routing1)
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count()))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count(false)))
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
@@ -156,7 +156,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count()))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count(false)))
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
@@ -179,7 +179,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3).shardSize(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count()))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count(false)))
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
@@ -202,7 +202,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count()))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count(false)))
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
@@ -226,7 +226,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type").setRouting(routing1)
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count()))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count(false)))
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
@@ -272,7 +272,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count()))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count(false)))
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
@@ -295,7 +295,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3).shardSize(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count()))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.count(false)))
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
@@ -318,7 +318,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count()))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count(false)))
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
@@ -341,7 +341,7 @@ public class ShardSizeTermsIT extends ShardSizeTestCase {
         SearchResponse response = client().prepareSearch("idx").setTypes("type").setRouting(routing1)
                 .setQuery(matchAllQuery())
                 .addAggregation(terms("keys").field("key").size(3)
-                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count()))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())).shardSize(5).order(Terms.Order.count(false)))
                 .execute().actionGet();
 
         Terms terms = response.getAggregations().get("keys");
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java
index 00c73b42246..6bbe1dc22b2 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsDocCountErrorIT.java
@@ -271,6 +271,37 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
         assertNoDocCountErrorSingleResponse(size, testResponse);
     }
 
+    public void testStringValueFieldDocCountAsc() throws Exception {
+        int size = randomIntBetween(1, 20);
+        int shardSize = randomIntBetween(size, size * 2);
+        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+                .addAggregation(terms("terms")
+                        .executionHint(randomExecutionHint())
+                        .field(STRING_FIELD_NAME)
+                        .showTermDocCountError(true)
+                        .size(10000).shardSize(10000)
+                        .order(Order.count(true))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())))
+                .execute().actionGet();
+
+        assertSearchResponse(accurateResponse);
+
+        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+                .addAggregation(terms("terms")
+                        .executionHint(randomExecutionHint())
+                        .field(STRING_FIELD_NAME)
+                        .showTermDocCountError(true)
+                        .size(size)
+                        .shardSize(shardSize)
+                        .order(Order.count(true))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())))
+                .execute().actionGet();
+
+        assertSearchResponse(testResponse);
+
+        assertUnboundedDocCountError(size, accurateResponse, testResponse);
+    }
+
     public void testStringValueFieldTermSortAsc() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
@@ -476,6 +507,37 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
         assertNoDocCountErrorSingleResponse(size, testResponse);
     }
 
+    public void testLongValueFieldDocCountAsc() throws Exception {
+        int size = randomIntBetween(1, 20);
+        int shardSize = randomIntBetween(size, size * 2);
+        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+                .addAggregation(terms("terms")
+                        .executionHint(randomExecutionHint())
+                        .field(LONG_FIELD_NAME)
+                        .showTermDocCountError(true)
+                        .size(10000).shardSize(10000)
+                        .order(Order.count(true))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())))
+                .execute().actionGet();
+
+        assertSearchResponse(accurateResponse);
+
+        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+                .addAggregation(terms("terms")
+                        .executionHint(randomExecutionHint())
+                        .field(LONG_FIELD_NAME)
+                        .showTermDocCountError(true)
+                        .size(size)
+                        .shardSize(shardSize)
+                        .order(Order.count(true))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())))
+                .execute().actionGet();
+
+        assertSearchResponse(testResponse);
+
+        assertUnboundedDocCountError(size, accurateResponse, testResponse);
+    }
+
     public void testLongValueFieldTermSortAsc() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
@@ -681,6 +743,37 @@ public class TermsDocCountErrorIT extends ESIntegTestCase {
         assertNoDocCountErrorSingleResponse(size, testResponse);
     }
 
+    public void testDoubleValueFieldDocCountAsc() throws Exception {
+        int size = randomIntBetween(1, 20);
+        int shardSize = randomIntBetween(size, size * 2);
+        SearchResponse accurateResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+                .addAggregation(terms("terms")
+                        .executionHint(randomExecutionHint())
+                        .field(DOUBLE_FIELD_NAME)
+                        .showTermDocCountError(true)
+                        .size(10000).shardSize(10000)
+                        .order(Order.count(true))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())))
+                .execute().actionGet();
+
+        assertSearchResponse(accurateResponse);
+
+        SearchResponse testResponse = client().prepareSearch("idx_single_shard").setTypes("type")
+                .addAggregation(terms("terms")
+                        .executionHint(randomExecutionHint())
+                        .field(DOUBLE_FIELD_NAME)
+                        .showTermDocCountError(true)
+                        .size(size)
+                        .shardSize(shardSize)
+                        .order(Order.count(true))
+                        .collectMode(randomFrom(SubAggCollectionMode.values())))
+                .execute().actionGet();
+
+        assertSearchResponse(testResponse);
+
+        assertUnboundedDocCountError(size, accurateResponse, testResponse);
+    }
+
     public void testDoubleValueFieldTermSortAsc() throws Exception {
         int size = randomIntBetween(1, 20);
         int shardSize = randomIntBetween(size, size * 2);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsTests.java
index 055258bb4cc..c8a14e82bf4 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/TermsTests.java
@@ -180,7 +180,7 @@ public class TermsTests extends BaseAggregationTestCase<TermsAggregationBuilder>
             orders.add(Terms.Order.term(randomBoolean()));
             break;
         case 1:
-            orders.add(Terms.Order.count());
+            orders.add(Terms.Order.count(randomBoolean()));
             break;
         case 2:
             orders.add(Terms.Order.aggregation(randomAsciiOfLengthBetween(3, 20), randomBoolean()));
diff --git a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
index 30e3d649179..3c1f4ae860a 100644
--- a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
@@ -239,7 +239,9 @@ determined and is given a value of -1 to indicate this.
 ==== Order
 
 The order of the buckets can be customized by setting the `order` parameter. By default, the buckets are ordered by
-their `doc_count` descending:
+their `doc_count` descending. It is also possible to change this behaviour as follows:
+
+Ordering the buckets by their `doc_count` in an ascending manner:
 
 [source,js]
 --------------------------------------------------
@@ -248,15 +250,12 @@ their `doc_count` descending:
         "genders" : {
             "terms" : {
                 "field" : "gender",
-                "order" : "_count" <1>
+                "order" : { "_count" : "asc" }
             }
         }
     }
 }
 --------------------------------------------------
-<1> This is the default if the order is missing.
-
-It is also possible to change this behaviour as follows:
 
 Ordering the buckets alphabetically by their terms in an ascending manner:
 
@@ -313,9 +312,7 @@ Ordering the buckets by multi value metrics sub-aggregation (identified by the a
 }
 --------------------------------------------------
 
-Ordering the buckets by their `doc_count` in an ascending manner is not supported.
-
-WARNING: Sorting by sub aggregation is discouraged as it increases the
+WARNING: Sorting by ascending `_count` or by sub aggregation is discouraged as it increases the
 <<search-aggregations-bucket-terms-aggregation-approximate-counts,error>> on document counts.
 It is fine when a single shard is queried, or when the field that is being aggregated was used
 as a routing key at index time: in these cases results will be accurate since shards have disjoint
diff --git a/docs/reference/migration/migrate_5_0/aggregations.asciidoc b/docs/reference/migration/migrate_5_0/aggregations.asciidoc
index 57afd44719b..d9227e91385 100644
--- a/docs/reference/migration/migrate_5_0/aggregations.asciidoc
+++ b/docs/reference/migration/migrate_5_0/aggregations.asciidoc
@@ -26,7 +26,3 @@ for `from` and `to` anymore.
 `size: 0` is no longer valid for the terms, significant terms and geohash grid
 aggregations. Instead a size should be explicitly specified with a number greater
 than zero.
-
-==== `order: {"count": "asc"}` on Terms
-
-The ability to sort the terms aggregation by ascending count has been removed.
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java
index 5e1dec0402f..6ed77fffcc8 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java
@@ -1048,6 +1048,11 @@ public class DoubleTermsTests extends AbstractTermsTestCase {
         assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("avg_l", false), Terms.Order.term(true));
     }
 
+    public void testSingleValuedFieldOrderedByCountAscAndSingleValueSubAggregationAsc() throws Exception {
+        double[] expectedKeys = new double[] { 6, 7, 3, 4, 5, 1, 2 };
+        assertMultiSortResponse(expectedKeys, Terms.Order.count(true), Terms.Order.aggregation("avg_l", true));
+    }
+
     public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscSingleValueSubAggregationAsc() throws Exception {
         double[] expectedKeys = new double[] { 6, 7, 3, 5, 4, 1, 2 };
         assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("sum_d", true), Terms.Order.aggregation("avg_l", true));
@@ -1055,7 +1060,7 @@ public class DoubleTermsTests extends AbstractTermsTestCase {
 
     public void testSingleValuedFieldOrderedByThreeCriteria() throws Exception {
         double[] expectedKeys = new double[] { 2, 1, 4, 5, 3, 6, 7 };
-        assertMultiSortResponse(expectedKeys, Terms.Order.count(), Terms.Order.aggregation("sum_d", false), Terms.Order.aggregation("avg_l", false));
+        assertMultiSortResponse(expectedKeys, Terms.Order.count(false), Terms.Order.aggregation("sum_d", false), Terms.Order.aggregation("avg_l", false));
     }
 
     public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscAsCompound() throws Exception {
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java
index efc9833a33d..fa049429a3a 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java
@@ -1025,6 +1025,11 @@ public class LongTermsTests extends AbstractTermsTestCase {
         assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("avg_l", false), Terms.Order.term(true));
     }
 
+    public void testSingleValuedFieldOrderedByCountAscAndSingleValueSubAggregationAsc() throws Exception {
+        long[] expectedKeys = new long[] { 6, 7, 3, 4, 5, 1, 2 };
+        assertMultiSortResponse(expectedKeys, Terms.Order.count(true), Terms.Order.aggregation("avg_l", true));
+    }
+
     public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscSingleValueSubAggregationAsc() throws Exception {
         long[] expectedKeys = new long[] { 6, 7, 3, 5, 4, 1, 2 };
         assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("sum_d", true), Terms.Order.aggregation("avg_l", true));
@@ -1032,7 +1037,7 @@ public class LongTermsTests extends AbstractTermsTestCase {
 
     public void testSingleValuedFieldOrderedByThreeCriteria() throws Exception {
         long[] expectedKeys = new long[] { 2, 1, 4, 5, 3, 6, 7 };
-        assertMultiSortResponse(expectedKeys, Terms.Order.count(), Terms.Order.aggregation("sum_d", false), Terms.Order.aggregation("avg_l", false));
+        assertMultiSortResponse(expectedKeys, Terms.Order.count(false), Terms.Order.aggregation("sum_d", false), Terms.Order.aggregation("avg_l", false));
     }
 
     public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscAsCompound() throws Exception {
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java
index c24551ed588..662d4d2f30c 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java
@@ -173,20 +173,36 @@ public class MinDocCountTests extends AbstractTermsTestCase {
         testMinDocCountOnTerms("s", Script.YES, Terms.Order.term(false));
     }
 
+    public void testStringCountAsc() throws Exception {
+        testMinDocCountOnTerms("s", Script.NO, Terms.Order.count(true));
+    }
+
+    public void testStringScriptCountAsc() throws Exception {
+        testMinDocCountOnTerms("s", Script.YES, Terms.Order.count(true));
+    }
+
     public void testStringCountDesc() throws Exception {
-        testMinDocCountOnTerms("s", Script.NO, Terms.Order.count());
+        testMinDocCountOnTerms("s", Script.NO, Terms.Order.count(false));
     }
 
     public void testStringScriptCountDesc() throws Exception {
-        testMinDocCountOnTerms("s", Script.YES, Terms.Order.count());
+        testMinDocCountOnTerms("s", Script.YES, Terms.Order.count(false));
+    }
+
+    public void testStringCountAscWithInclude() throws Exception {
+        testMinDocCountOnTerms("s", Script.NO, Terms.Order.count(true), ".*a.*", true);
+    }
+
+    public void testStringScriptCountAscWithInclude() throws Exception {
+        testMinDocCountOnTerms("s", Script.YES, Terms.Order.count(true), ".*a.*", true);
     }
 
     public void testStringCountDescWithInclude() throws Exception {
-        testMinDocCountOnTerms("s", Script.NO, Terms.Order.count(), ".*a.*", true);
+        testMinDocCountOnTerms("s", Script.NO, Terms.Order.count(false), ".*a.*", true);
     }
 
     public void testStringScriptCountDescWithInclude() throws Exception {
-        testMinDocCountOnTerms("s", Script.YES, Terms.Order.count(), ".*a.*", true);
+        testMinDocCountOnTerms("s", Script.YES, Terms.Order.count(false), ".*a.*", true);
     }
 
     public void testLongTermAsc() throws Exception {
@@ -205,12 +221,20 @@ public class MinDocCountTests extends AbstractTermsTestCase {
         testMinDocCountOnTerms("l", Script.YES, Terms.Order.term(false));
     }
 
+    public void testLongCountAsc() throws Exception {
+        testMinDocCountOnTerms("l", Script.NO, Terms.Order.count(true));
+    }
+
+    public void testLongScriptCountAsc() throws Exception {
+        testMinDocCountOnTerms("l", Script.YES, Terms.Order.count(true));
+    }
+
     public void testLongCountDesc() throws Exception {
-        testMinDocCountOnTerms("l", Script.NO, Terms.Order.count());
+        testMinDocCountOnTerms("l", Script.NO, Terms.Order.count(false));
     }
 
     public void testLongScriptCountDesc() throws Exception {
-        testMinDocCountOnTerms("l", Script.YES, Terms.Order.count());
+        testMinDocCountOnTerms("l", Script.YES, Terms.Order.count(false));
     }
 
     public void testDoubleTermAsc() throws Exception {
@@ -229,12 +253,20 @@ public class MinDocCountTests extends AbstractTermsTestCase {
         testMinDocCountOnTerms("d", Script.YES, Terms.Order.term(false));
     }
 
+    public void testDoubleCountAsc() throws Exception {
+        testMinDocCountOnTerms("d", Script.NO, Terms.Order.count(true));
+    }
+
+    public void testDoubleScriptCountAsc() throws Exception {
+        testMinDocCountOnTerms("d", Script.YES, Terms.Order.count(true));
+    }
+
     public void testDoubleCountDesc() throws Exception {
-        testMinDocCountOnTerms("d", Script.NO, Terms.Order.count());
+        testMinDocCountOnTerms("d", Script.NO, Terms.Order.count(false));
     }
 
     public void testDoubleScriptCountDesc() throws Exception {
-        testMinDocCountOnTerms("d", Script.YES, Terms.Order.count());
+        testMinDocCountOnTerms("d", Script.YES, Terms.Order.count(false));
     }
 
     private void testMinDocCountOnTerms(String field, Script script, Terms.Order order) throws Exception {
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java
index a5825d23172..b098db154da 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java
@@ -1375,6 +1375,11 @@ public class StringTermsTests extends AbstractTermsTestCase {
         assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("avg_l", false), Terms.Order.term(true));
     }
 
+    public void testSingleValuedFieldOrderedByCountAscAndSingleValueSubAggregationAsc() throws Exception {
+        String[] expectedKeys = new String[] { "val6", "val7", "val3", "val4", "val5", "val1", "val2" };
+        assertMultiSortResponse(expectedKeys, Terms.Order.count(true), Terms.Order.aggregation("avg_l", true));
+    }
+
     public void testSingleValuedFieldOrderedBySingleValueSubAggregationAscSingleValueSubAggregationAsc() throws Exception {
         String[] expectedKeys = new String[] { "val6", "val7", "val3", "val5", "val4", "val1", "val2" };
         assertMultiSortResponse(expectedKeys, Terms.Order.aggregation("sum_d", true), Terms.Order.aggregation("avg_l", true));
@@ -1382,7 +1387,7 @@ public class StringTermsTests extends AbstractTermsTestCase {
 
     public void testSingleValuedFieldOrderedByThreeCriteria() throws Exception {
         String[] expectedKeys = new String[] { "val2", "val1", "val4", "val5", "val3", "val6", "val7" };
-        assertMultiSortResponse(expectedKeys, Terms.Order.count(), Terms.Order.aggregation("sum_d", false),
+        assertMultiSortResponse(expectedKeys, Terms.Order.count(false), Terms.Order.aggregation("sum_d", false),
                 Terms.Order.aggregation("avg_l", false));
     }
 
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml
index c079e6b6f7e..c35e79e6cfe 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml
@@ -350,17 +350,4 @@ setup:
   
   - match: { aggregations.date_terms.buckets.0.key_as_string: "2014-09-01T00:00:00.000Z" }
 
-  - match: { aggregations.date_terms.buckets.0.doc_count: 1 }
-
----
-"Terms count order test":
-  - do:
-      catch:  /Sort by ascending _count is not supported in/
-      search:
-        body: { "size" : 0, "aggs" : { "count_order" : { "terms" : { "field" : "f1", "order": {"_count": "asc"} } } } }
-
-  - do:
-      search:
-        body: { "size" : 0, "aggs" : { "count_order" : { "terms" : { "field" : "f1", "order": "_count" } } } }
-
-
+  - match: { aggregations.date_terms.buckets.0.doc_count: 1 }  

From 46b40f73b7697eeb84e542c2f1801a2f96501bdc Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Fri, 17 Jun 2016 17:32:38 +0200
Subject: [PATCH 0932/1311] Inline reroute with process of node join/master
 election (#18938)

In the past, we had the semantics where the very first cluster state a node processed after joining could not contain shard assignment to it. This was to make sure the node cleans up local / stale shard copies before receiving new ones that might confuse it. Since then a lot of work in this area, most notably the introduction of allocation ids and #17270 . This means we don't have to be careful and just reroute in the same cluster state change where we process the join, keeping things simple and following the same pattern we have in other places.
---
 .../cluster/routing/RoutingService.java       |  4 --
 .../elasticsearch/discovery/Discovery.java    |  4 +-
 .../discovery/local/LocalDiscovery.java       | 24 ++++++------
 .../discovery/zen/NodeJoinController.java     | 39 +++++++------------
 .../discovery/zen/ZenDiscovery.java           | 17 ++++----
 .../java/org/elasticsearch/node/Node.java     |  3 +-
 .../zen/NodeJoinControllerTests.java          | 30 ++++++--------
 .../org/elasticsearch/test/NoopDiscovery.java |  4 +-
 8 files changed, 49 insertions(+), 76 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
index 78e7e15d389..d6ec9048688 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
@@ -72,10 +72,6 @@ public class RoutingService extends AbstractLifecycleComponent<RoutingService> {
     protected void doClose() {
     }
 
-    public AllocationService getAllocationService() {
-        return this.allocationService;
-    }
-
     /**
      * Initiates a reroute.
      */
diff --git a/core/src/main/java/org/elasticsearch/discovery/Discovery.java b/core/src/main/java/org/elasticsearch/discovery/Discovery.java
index 778e2d15053..dabd45e36dd 100644
--- a/core/src/main/java/org/elasticsearch/discovery/Discovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/Discovery.java
@@ -22,7 +22,7 @@ package org.elasticsearch.discovery;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RoutingService;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -44,7 +44,7 @@ public interface Discovery extends LifecycleComponent<Discovery> {
      * Another hack to solve dep injection problem..., note, this will be called before
      * any start is called.
      */
-    void setRoutingService(RoutingService routingService);
+    void setAllocationService(AllocationService allocationService);
 
     /**
      * Publish all the changes to the cluster from the master (can be called just by the master). The publish
diff --git a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
index 696eb8812c9..ec9158c5932 100644
--- a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
@@ -28,7 +28,7 @@ import org.elasticsearch.cluster.IncompatibleClusterStateVersionException;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.RoutingService;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
@@ -61,7 +61,7 @@ public class LocalDiscovery extends AbstractLifecycleComponent<Discovery> implem
     private static final LocalDiscovery[] NO_MEMBERS = new LocalDiscovery[0];
 
     private final ClusterService clusterService;
-    private RoutingService routingService;
+    private AllocationService allocationService;
     private final ClusterName clusterName;
 
     private final DiscoverySettings discoverySettings;
@@ -83,8 +83,8 @@ public class LocalDiscovery extends AbstractLifecycleComponent<Discovery> implem
     }
 
     @Override
-    public void setRoutingService(RoutingService routingService) {
-        this.routingService = routingService;
+    public void setAllocationService(AllocationService allocationService) {
+        this.allocationService = allocationService;
     }
 
     @Override
@@ -156,7 +156,12 @@ public class LocalDiscovery extends AbstractLifecycleComponent<Discovery> implem
                             nodesBuilder.put(discovery.localNode());
                         }
                         nodesBuilder.localNodeId(master.localNode().getId()).masterNodeId(master.localNode().getId());
-                        return ClusterState.builder(currentState).nodes(nodesBuilder).build();
+                        currentState = ClusterState.builder(currentState).nodes(nodesBuilder).build();
+                        RoutingAllocation.Result result =  master.allocationService.reroute(currentState, "node_add");
+                        if (result.changed()) {
+                            currentState = ClusterState.builder(currentState).routingResult(result).build();
+                        }
+                        return currentState;
                     }
 
                     @Override
@@ -164,13 +169,6 @@ public class LocalDiscovery extends AbstractLifecycleComponent<Discovery> implem
                         logger.error("unexpected failure during [{}]", t, source);
                     }
 
-                    @Override
-                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                        // we reroute not in the same cluster state update since in certain areas we rely on
-                        // the node to be in the cluster state (sampled from ClusterService#state) to be there, also
-                        // shard transitions need to better be handled in such cases
-                        master.routingService.reroute("post_node_add");
-                    }
                 });
             }
         } // else, no master node, the next node that will start will fill things in...
@@ -226,7 +224,7 @@ public class LocalDiscovery extends AbstractLifecycleComponent<Discovery> implem
                         }
                         // reroute here, so we eagerly remove dead nodes from the routing
                         ClusterState updatedState = ClusterState.builder(currentState).nodes(newNodes).build();
-                        RoutingAllocation.Result routingResult = master.routingService.getAllocationService().reroute(
+                        RoutingAllocation.Result routingResult = master.allocationService.reroute(
                                 ClusterState.builder(updatedState).build(), "elected as master");
                         return ClusterState.builder(updatedState).routingResult(routingResult).build();
                     }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
index 9659434e03d..49815cdb703 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
@@ -29,7 +29,7 @@ import org.elasticsearch.cluster.NotMasterException;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.RoutingService;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
@@ -58,7 +58,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 public class NodeJoinController extends AbstractComponent {
 
     private final ClusterService clusterService;
-    private final RoutingService routingService;
+    private final AllocationService allocationService;
     private final ElectMasterService electMaster;
     private final DiscoverySettings discoverySettings;
     private final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor();
@@ -68,10 +68,11 @@ public class NodeJoinController extends AbstractComponent {
     private ElectionContext electionContext = null;
 
 
-    public NodeJoinController(ClusterService clusterService, RoutingService routingService, ElectMasterService electMaster, DiscoverySettings discoverySettings, Settings settings) {
+    public NodeJoinController(ClusterService clusterService, AllocationService allocationService, ElectMasterService electMaster,
+                              DiscoverySettings discoverySettings, Settings settings) {
         super(settings);
         this.clusterService = clusterService;
-        this.routingService = routingService;
+        this.allocationService = allocationService;
         this.electMaster = electMaster;
         this.discoverySettings = discoverySettings;
     }
@@ -406,21 +407,7 @@ public class NodeJoinController extends AbstractComponent {
                 ClusterBlocks clusterBlocks = ClusterBlocks.builder().blocks(currentState.blocks())
                     .removeGlobalBlock(discoverySettings.getNoMasterBlock()).build();
                 newState.blocks(clusterBlocks);
-                newState.nodes(nodesBuilder);
                 nodesChanged = true;
-
-                // reroute now to remove any dead nodes (master may have stepped down when they left and didn't update the routing table)
-                // Note: also do it now to avoid assigning shards to these nodes. We will have another reroute after the cluster
-                // state is published.
-                // TODO: this publishing of a cluster state with no nodes assigned to joining nodes shouldn't be needed anymore. remove.
-
-                final ClusterState tmpState = newState.build();
-                RoutingAllocation.Result result = routingService.getAllocationService().reroute(tmpState, "nodes joined");
-                newState = ClusterState.builder(tmpState);
-                if (result.changed()) {
-                    newState.routingResult(result);
-                }
-                nodesBuilder = DiscoveryNodes.builder(tmpState.nodes());
             }
 
             if (nodesBuilder.isLocalNodeElectedMaster() == false) {
@@ -439,7 +426,8 @@ public class NodeJoinController extends AbstractComponent {
                     for (DiscoveryNode existingNode : currentNodes) {
                         if (node.getAddress().equals(existingNode.getAddress())) {
                             nodesBuilder.remove(existingNode.getId());
-                            logger.warn("received join request from node [{}], but found existing node {} with same address, removing existing node", node, existingNode);
+                            logger.warn("received join request from node [{}], but found existing node {} with same address, " +
+                                "removing existing node", node, existingNode);
                         }
                     }
                 }
@@ -448,6 +436,12 @@ public class NodeJoinController extends AbstractComponent {
 
             if (nodesChanged) {
                 newState.nodes(nodesBuilder);
+                final ClusterState tmpState = newState.build();
+                RoutingAllocation.Result result = allocationService.reroute(tmpState, "node_join");
+                newState = ClusterState.builder(tmpState);
+                if (result.changed()) {
+                    newState.routingResult(result);
+                }
             }
 
             // we must return a new cluster state instance to force publishing. This is important
@@ -463,13 +457,6 @@ public class NodeJoinController extends AbstractComponent {
 
         @Override
         public void clusterStatePublished(ClusterChangedEvent event) {
-            if (event.nodesDelta().hasChanges()) {
-                // we reroute not in the same cluster state update since in certain areas we rely on
-                // the node to be in the cluster state (sampled from ClusterService#state) to be there, also
-                // shard transitions need to better be handled in such cases
-                routingService.reroute("post_node_add");
-            }
-
             NodeJoinController.this.electMaster.logMinimumMasterNodesWarningIfNecessary(event.previousState(), event.state());
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
index 54ef9928585..6a9a64ff411 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
@@ -32,7 +32,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.RoutingService;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
@@ -113,7 +113,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
 
     private final TransportService transportService;
     private final ClusterService clusterService;
-    private RoutingService routingService;
+    private AllocationService allocationService;
     private final ClusterName clusterName;
     private final DiscoverySettings discoverySettings;
     private final ZenPingService pingService;
@@ -146,7 +146,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
     /** counts the time this node has joined the cluster or have elected it self as master */
     private final AtomicLong clusterJoinsCounter = new AtomicLong();
 
-    // must initialized in doStart(), when we have the routingService set
+    // must initialized in doStart(), when we have the allocationService set
     private volatile NodeJoinController nodeJoinController;
 
     @Inject
@@ -206,8 +206,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
     }
 
     @Override
-    public void setRoutingService(RoutingService routingService) {
-        this.routingService = routingService;
+    public void setAllocationService(AllocationService allocationService) {
+        this.allocationService = allocationService;
     }
 
     @Override
@@ -215,7 +215,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
         nodesFD.setLocalNode(clusterService.localNode());
         joinThreadControl.start();
         pingService.start();
-        this.nodeJoinController = new NodeJoinController(clusterService, routingService, electMaster, discoverySettings, settings);
+        this.nodeJoinController = new NodeJoinController(clusterService, allocationService, electMaster, discoverySettings, settings);
     }
 
     @Override
@@ -516,8 +516,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
                         return rejoin(currentState, "not enough master nodes");
                     }
                     // eagerly run reroute to remove dead nodes from routing table
-                    RoutingAllocation.Result routingResult = routingService.getAllocationService().reroute(
-                            ClusterState.builder(currentState).build(),
+                    RoutingAllocation.Result routingResult = allocationService.reroute(ClusterState.builder(currentState).build(),
                             "[" + node + "] left");
                     return ClusterState.builder(currentState).routingResult(routingResult).build();
                 }
@@ -561,7 +560,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
                     return rejoin(currentState, "not enough master nodes");
                 }
                 // eagerly run reroute to remove dead nodes from routing table
-                RoutingAllocation.Result routingResult = routingService.getAllocationService().reroute(
+                RoutingAllocation.Result routingResult = allocationService.reroute(
                         ClusterState.builder(currentState).build(),
                         "[" + node + "] failed");
                 return ClusterState.builder(currentState).routingResult(routingResult).build();
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index f34e16db140..043526306e9 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -37,6 +37,7 @@ import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeService;
 import org.elasticsearch.cluster.routing.RoutingService;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.StopWatch;
 import org.elasticsearch.common.component.Lifecycle;
@@ -321,7 +322,7 @@ public class Node implements Closeable {
         ESLogger logger = Loggers.getLogger(Node.class, NODE_NAME_SETTING.get(settings));
         logger.info("starting ...");
         // hack around dependency injection problem (for now...)
-        injector.getInstance(Discovery.class).setRoutingService(injector.getInstance(RoutingService.class));
+        injector.getInstance(Discovery.class).setAllocationService(injector.getInstance(AllocationService.class));
         for (Class<? extends LifecycleComponent> plugin : pluginsService.nodeServices()) {
             injector.getInstance(plugin).start();
         }
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
index 27c38e66074..683bebc55a3 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
@@ -24,7 +24,6 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.NotMasterException;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
@@ -99,7 +98,7 @@ public class NodeJoinControllerTests extends ESTestCase {
         // make sure we have a master
         setState(clusterService, ClusterState.builder(clusterService.state()).nodes(
             DiscoveryNodes.builder(initialNodes).masterNodeId(localNode.getId())));
-        nodeJoinController = new NodeJoinController(clusterService, new NoopRoutingService(Settings.EMPTY),
+        nodeJoinController = new NodeJoinController(clusterService, new NoopAllocationService(Settings.EMPTY),
             new ElectMasterService(Settings.EMPTY, Version.CURRENT),
             new DiscoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)),
             Settings.EMPTY);
@@ -298,7 +297,8 @@ public class NodeJoinControllerTests extends ESTestCase {
         }
 
         logger.debug("--> asserting master election didn't finish yet");
-        assertThat("election finished after [" + initialJoins + "] master nodes but required joins is [" + requiredJoins + "]", electionFuture.isDone(), equalTo(false));
+        assertThat("election finished after [" + initialJoins + "] master nodes but required joins is [" + requiredJoins + "]",
+            electionFuture.isDone(), equalTo(false));
 
         final int finalJoins = requiredJoins - initialJoins + randomInt(5);
         nodesToJoin.clear();
@@ -374,7 +374,8 @@ public class NodeJoinControllerTests extends ESTestCase {
         nodeJoinController.waitToBeElectedAsMaster(requiredJoins, TimeValue.timeValueMillis(1), new NodeJoinController.ElectionCallback() {
             @Override
             public void onElectedAsMaster(ClusterState state) {
-                assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(), equalTo(true));
+                assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(),
+                    equalTo(true));
                 latch.countDown();
             }
 
@@ -492,7 +493,8 @@ public class NodeJoinControllerTests extends ESTestCase {
         nodeJoinController.waitToBeElectedAsMaster(requiredJoins, TimeValue.timeValueHours(30), new NodeJoinController.ElectionCallback() {
             @Override
             public void onElectedAsMaster(ClusterState state) {
-                assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(), equalTo(true));
+                assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(),
+                    equalTo(true));
                 latch.countDown();
             }
 
@@ -516,18 +518,6 @@ public class NodeJoinControllerTests extends ESTestCase {
     }
 
 
-    static class NoopRoutingService extends RoutingService {
-
-        public NoopRoutingService(Settings settings) {
-            super(settings, null, new NoopAllocationService(settings));
-        }
-
-        @Override
-        protected void performReroute(String reason) {
-
-        }
-    }
-
     static class NoopAllocationService extends AllocationService {
 
         public NoopAllocationService(Settings settings) {
@@ -535,12 +525,14 @@ public class NodeJoinControllerTests extends ESTestCase {
         }
 
         @Override
-        public RoutingAllocation.Result applyStartedShards(ClusterState clusterState, List<? extends ShardRouting> startedShards, boolean withReroute) {
+        public RoutingAllocation.Result applyStartedShards(ClusterState clusterState, List<? extends ShardRouting> startedShards,
+                                                           boolean withReroute) {
             return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData());
         }
 
         @Override
-        public RoutingAllocation.Result applyFailedShards(ClusterState clusterState, List<FailedRerouteAllocation.FailedShard> failedShards) {
+        public RoutingAllocation.Result applyFailedShards(ClusterState clusterState,
+                                                          List<FailedRerouteAllocation.FailedShard> failedShards) {
             return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData());
         }
 
diff --git a/core/src/test/java/org/elasticsearch/test/NoopDiscovery.java b/core/src/test/java/org/elasticsearch/test/NoopDiscovery.java
index 3193aaf458e..967837e0044 100644
--- a/core/src/test/java/org/elasticsearch/test/NoopDiscovery.java
+++ b/core/src/test/java/org/elasticsearch/test/NoopDiscovery.java
@@ -20,7 +20,7 @@ package org.elasticsearch.test;
 
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RoutingService;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.component.LifecycleListener;
 import org.elasticsearch.discovery.Discovery;
@@ -41,7 +41,7 @@ public class NoopDiscovery implements Discovery {
     }
 
     @Override
-    public void setRoutingService(RoutingService routingService) {
+    public void setAllocationService(AllocationService allocationService) {
 
     }
 

From a3bd2de430a2195ad64937d72f0cdd878420cea5 Mon Sep 17 00:00:00 2001
From: Areek Zillur <areek.zillur@elasticsearch.com>
Date: Fri, 17 Jun 2016 12:14:45 -0400
Subject: [PATCH 0933/1311] [DOCS] fix missing rollover-index link

---
 docs/reference/indices.asciidoc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/reference/indices.asciidoc b/docs/reference/indices.asciidoc
index cad1a1477bd..3c1eadfa07e 100644
--- a/docs/reference/indices.asciidoc
+++ b/docs/reference/indices.asciidoc
@@ -76,6 +76,8 @@ include::indices/open-close.asciidoc[]
 
 include::indices/shrink-index.asciidoc[]
 
+include::indices/rollover-index.asciidoc[]
+
 include::indices/put-mapping.asciidoc[]
 
 include::indices/get-mapping.asciidoc[]

From 245def80f04c7d33a4fe10f873903ca23c561947 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 17 Jun 2016 15:17:56 -0400
Subject: [PATCH 0934/1311] Add note that thread pool settings are node-level

This commit adds a note to the breaking changes docs that since commit
da74323141a69bd06fc19a70bc072143e9f0e986, thread pool settings are no
longer cluster-level settings and thus not dynamically updatable.
---
 docs/reference/migration/migrate_5_0/settings.asciidoc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/reference/migration/migrate_5_0/settings.asciidoc b/docs/reference/migration/migrate_5_0/settings.asciidoc
index ed405428ab6..4863588818f 100644
--- a/docs/reference/migration/migrate_5_0/settings.asciidoc
+++ b/docs/reference/migration/migrate_5_0/settings.asciidoc
@@ -92,6 +92,9 @@ The queue size setting for a fixed thread pool must be `queue_size`
 (all other variants that were previously supported are no longer
 supported).
 
+Thread pool settings are now node-level settings. As such, it is not
+possible to update thread pool settings via the cluster settings API.
+
 ==== Analysis settings
 
 The `index.analysis.analyzer.default_index` analyzer is not supported anymore.

From d09d89f8c57c35146387268fc536d86b77d82435 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 17 Jun 2016 15:27:46 -0400
Subject: [PATCH 0935/1311] Remove only node preference

This commit removes the search preference _only_node as the same
functionality can be obtained by using the search preference
_only_nodes. This commit also adds a test that ensures that _only_nodes
will continue to support specifying node IDs.

Relates #18875
---
 .../routing/IndexShardRoutingTable.java       | 10 ++-
 .../cluster/routing/OperationRouting.java     |  4 --
 .../cluster/routing/Preference.java           |  7 --
 .../routing/OperationRoutingTests.java        | 64 ++++++++++++++++++-
 .../recovery/RecoveryWhileUnderLoadIT.java    |  2 +-
 .../search/preference/SearchPreferenceIT.java |  6 +-
 .../migration/migrate_5_0/search.asciidoc     |  6 +-
 .../search/request/preference.asciidoc        |  4 --
 8 files changed, 80 insertions(+), 23 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
index ceea83dbcaa..91b1df9b453 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
@@ -37,6 +37,7 @@ import java.util.Collections;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 
@@ -369,8 +370,13 @@ public class IndexShardRoutingTable implements Iterable<ShardRouting> {
             }
         }
         if (ordered.isEmpty()) {
-            throw new IllegalArgumentException("no data nodes with critera(s) " +
-                Strings.arrayToCommaDelimitedString(nodeAttributes) + "] found for shard:" + shardId());
+            final String message = String.format(
+                    Locale.ROOT,
+                    "no data nodes with %s [%s] found for shard: %s",
+                    nodeAttributes.length == 1 ? "criteria" : "criterion",
+                    String.join(",", nodeAttributes),
+                    shardId());
+            throw new IllegalArgumentException(message);
         }
         return new PlainShardIterator(shardId, ordered);
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
index 19189e0c286..5b24ac7edff 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
@@ -177,10 +177,6 @@ public class OperationRouting extends AbstractComponent {
                     return indexShard.replicaFirstActiveInitializingShardsIt();
                 case ONLY_LOCAL:
                     return indexShard.onlyNodeActiveInitializingShardsIt(localNodeId);
-                case ONLY_NODE:
-                    String nodeId = preference.substring(Preference.ONLY_NODE.type().length() + 1);
-                    ensureNodeIdExists(nodes, nodeId);
-                    return indexShard.onlyNodeActiveInitializingShardsIt(nodeId);
                 case ONLY_NODES:
                     String nodeAttributes = preference.substring(Preference.ONLY_NODES.type().length() + 1);
                     return indexShard.onlyNodeSelectorActiveInitializingShardsIt(nodeAttributes.split(","), nodes);
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/Preference.java b/core/src/main/java/org/elasticsearch/cluster/routing/Preference.java
index cf0dd6cc54d..d4685d7aead 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/Preference.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/Preference.java
@@ -64,11 +64,6 @@ public enum  Preference {
      */
     ONLY_LOCAL("_only_local"),
 
-    /**
-     * Route to specific node only
-     */
-    ONLY_NODE("_only_node"),
-
     /**
      * Route to only node with attribute
      */
@@ -100,8 +95,6 @@ public enum  Preference {
                 return SHARDS;
             case "_prefer_nodes":
                 return PREFER_NODES;
-            case "_only_node":
-                return ONLY_NODE;
             case "_local":
                 return LOCAL;
             case "_primary":
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java
index 370ac52956c..3118692b341 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java
@@ -41,7 +41,9 @@ import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
 
+import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.object.HasToString.hasToString;
 import static org.hamcrest.Matchers.equalTo;
 
 public class OperationRoutingTests extends ESTestCase{
@@ -211,8 +213,7 @@ public class OperationRoutingTests extends ESTestCase{
                 }
             }
             final ShardIterator it =
-                    new OperationRouting(Settings.EMPTY,  new ClusterSettings(Settings.EMPTY,
-                        ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
+                    new OperationRouting(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
                             .getShards(clusterService.state(), indexName, 0, "_prefer_nodes:" + String.join(",", nodes));
             final List<ShardRouting> all = new ArrayList<>();
             ShardRouting shard;
@@ -231,4 +232,63 @@ public class OperationRoutingTests extends ESTestCase{
         }
     }
 
+    public void testThatOnlyNodesSupportNodeIds() throws InterruptedException, IOException {
+        TestThreadPool threadPool = null;
+        ClusterService clusterService = null;
+        try {
+            threadPool = new TestThreadPool("testThatOnlyNodesSupportNodeIds");
+            clusterService = ClusterServiceUtils.createClusterService(threadPool);
+            final String indexName = "test";
+            ClusterServiceUtils.setState(clusterService, ClusterStateCreationUtils.stateWithActivePrimary(indexName, true, randomInt(8)));
+            final Index index = clusterService.state().metaData().index(indexName).getIndex();
+            final List<ShardRouting> shards = clusterService.state().getRoutingNodes().assignedShards(new ShardId(index, 0));
+            final int count = randomIntBetween(1, shards.size());
+            int position = 0;
+            final List<String> nodes = new ArrayList<>();
+            final List<ShardRouting> expected = new ArrayList<>();
+            for (int i = 0; i < count; i++) {
+                if (randomBoolean() && !shards.get(position).initializing()) {
+                    nodes.add(shards.get(position).currentNodeId());
+                    expected.add(shards.get(position));
+                    position++;
+                } else {
+                    nodes.add("missing_" + i);
+                }
+            }
+            if (expected.size() > 0) {
+                final ShardIterator it =
+                    new OperationRouting(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
+                        .getShards(clusterService.state(), indexName, 0, "_only_nodes:" + String.join(",", nodes));
+                final List<ShardRouting> only = new ArrayList<>();
+                ShardRouting shard;
+                while ((shard = it.nextOrNull()) != null) {
+                    only.add(shard);
+                }
+                assertThat(new HashSet<>(only), equalTo(new HashSet<>(expected)));
+            } else {
+                final ClusterService cs = clusterService;
+                final IllegalArgumentException e = expectThrows(
+                    IllegalArgumentException.class,
+                    () -> new OperationRouting(
+                            Settings.EMPTY,
+                            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS))
+                        .getShards(cs.state(), indexName, 0, "_only_nodes:" + String.join(",", nodes)));
+                if (nodes.size() == 1) {
+                    assertThat(
+                        e,
+                        hasToString(containsString(
+                            "no data nodes with criteria [" + String.join(",", nodes) + "] found for shard: [test][0]")));
+                } else {
+                    assertThat(
+                        e,
+                        hasToString(containsString(
+                            "no data nodes with criterion [" + String.join(",", nodes) + "] found for shard: [test][0]")));
+                }
+            }
+        } finally {
+            IOUtils.close(clusterService);
+            terminate(threadPool);
+        }
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java b/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java
index 5f6636a1bd7..63b572f4d56 100644
--- a/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java
+++ b/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java
@@ -296,7 +296,7 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase {
                     if (docShard.id() == shard) {
                         for (ShardRouting shardRouting : state.routingTable().shardRoutingTable("test", shard)) {
                             GetResponse response = client().prepareGet("test", "type", Long.toString(id))
-                                    .setPreference("_only_node:" + shardRouting.currentNodeId()).get();
+                                    .setPreference("_only_nodes:" + shardRouting.currentNodeId()).get();
                             if (response.isExists()) {
                                 logger.info("missing id [{}] on shard {}", id, shardRouting);
                             }
diff --git a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java
index 49142988a46..49046bae009 100644
--- a/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java
+++ b/core/src/test/java/org/elasticsearch/search/preference/SearchPreferenceIT.java
@@ -38,7 +38,9 @@ import java.util.Set;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.greaterThan;
@@ -157,10 +159,10 @@ public class SearchPreferenceIT extends ESIntegTestCase {
         ensureGreen();
 
         try {
-            client().prepareSearch().setQuery(matchAllQuery()).setPreference("_only_node:DOES-NOT-EXIST").execute().actionGet();
+            client().prepareSearch().setQuery(matchAllQuery()).setPreference("_only_nodes:DOES-NOT-EXIST").execute().actionGet();
             fail("Expected IllegalArgumentException");
         } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), is("No data node with id[DOES-NOT-EXIST] found"));
+            assertThat(e, hasToString(containsString("no data nodes with criteria [DOES-NOT-EXIST] found for shard: [test][")));
         }
     }
 
diff --git a/docs/reference/migration/migrate_5_0/search.asciidoc b/docs/reference/migration/migrate_5_0/search.asciidoc
index c20df657504..7ebd79e66ca 100644
--- a/docs/reference/migration/migrate_5_0/search.asciidoc
+++ b/docs/reference/migration/migrate_5_0/search.asciidoc
@@ -186,7 +186,11 @@ In the response for profiling queries, the `query_type` has been renamed to `typ
 `description`. These changes have been made so the response format is more friendly to supporting other types of profiling
 in the future.
 
-==== Search prefernce
+==== Search preferences
+
+The <<search-request-preference,search preference>> `_only_node` has
+been removed. The same behavior can be achieved by using `_only_nodes`
+and specifying a single node ID.
 
 The <<search-request-preference,search preference>> `_prefer_node` has
 been superseded by `_prefer_nodes`. By specifying a single node,
diff --git a/docs/reference/search/request/preference.asciidoc b/docs/reference/search/request/preference.asciidoc
index 761fdeaa369..d5237812b98 100644
--- a/docs/reference/search/request/preference.asciidoc
+++ b/docs/reference/search/request/preference.asciidoc
@@ -27,10 +27,6 @@ The `preference` is a query string parameter which can be set to:
 	The operation will prefer to be executed on a local
 	allocated shard if possible.
 
-`_only_node:xyz`:: 
-	Restricts the search to execute only on a node with
-	the provided node id (`xyz` in this case).
-
 `_prefer_nodes:abc,xyz`::
 	Prefers execution on the nodes with the provided
 	node ids (`abc` or `xyz` in this case) if applicable.

From 1e16c22d0340146834634ede17e3b69bf982c947 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 16 Jun 2016 17:29:10 -0400
Subject: [PATCH 0936/1311] Painless: move semicolon hack into lexer

Perviously we used token level lookbehind in the parser. That worked,
but only if the parser didn't have any ambiguity *at all*. Since the
parser has ambiguity it didn't work everywhere. In particular it failed
when parsing blocks in lambdas like `a -> {int b = a + 2; b * b}`.

This moves the hack from the parser into the lexer. There we can use
token lookbehind (same trick) to *insert* semicolons into the token
stream. This works much better for antlr because antlr's prediction
code can work with real tokens.

Also, the lexer is simpler than the parser, so if there is a place
to introduce a hack, that is a better place.
---
 .../src/main/antlr/PainlessLexer.g4           |   4 +-
 .../src/main/antlr/PainlessParser.g4          |  11 -
 .../painless/antlr/EnhancedPainlessLexer.java |  96 +++
 .../painless/antlr/PainlessLexer.java         |   4 +-
 .../painless/antlr/PainlessParser.java        | 740 +++++++++---------
 .../painless/antlr/SlashStrategy.java         |   7 +-
 .../elasticsearch/painless/antlr/Walker.java  |   2 +-
 .../elasticsearch/painless/LambdaTests.java   |   6 +-
 .../elasticsearch/painless/RegexTests.java    |   2 +-
 .../painless/antlr/ParserTests.java           |   2 +-
 10 files changed, 471 insertions(+), 403 deletions(-)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java

diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.g4 b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
index 62a154db40c..ca469eed69a 100644
--- a/modules/lang-painless/src/main/antlr/PainlessLexer.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
@@ -56,7 +56,7 @@ THIS:      'this';
 BOOLNOT: '!';
 BWNOT:   '~';
 MUL:     '*';
-DIV:     '/' { false == SlashStrategy.slashIsRegex(_factory) }?;
+DIV:     '/' { false == SlashStrategy.slashIsRegex(this) }?;
 REM:     '%';
 ADD:     '+';
 SUB:     '-';
@@ -104,7 +104,7 @@ INTEGER: ( '0' | [1-9] [0-9]* ) [lLfFdD]?;
 DECIMAL: ( '0' | [1-9] [0-9]* ) (DOT [0-9]+)? ( [eE] [+\-]? [0-9]+ )? [fF]?;
 
 STRING: ( '"' ( '\\"' | '\\\\' | ~[\\"] )*? '"' ) | ( '\'' ( '\\\'' | '\\\\' | ~[\\"] )*? '\'' );
-REGEX: '/' ( ~('/' | '\n') | '\\' ~'\n' )+ '/' [cilmsUux]* { SlashStrategy.slashIsRegex(_factory) }?;
+REGEX: '/' ( ~('/' | '\n') | '\\' ~'\n' )+ '/' [cilmsUux]* { SlashStrategy.slashIsRegex(this) }?;
 
 TRUE:  'true';
 FALSE: 'false';
diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index 6dbaa50c73d..037d8fd7bbd 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -92,17 +92,6 @@ trap
 delimiter
     : SEMICOLON
     | EOF
-    // RBRACK is a delimiter but we don't consume it because it is only valid
-    // in places where RBRACK can follow the statement. It is simpler to not
-    // consume it here then it is to consume it here. Unfortunately, they
-    // obvious syntax to do this `| { _input.LA(1) == RBRACK }?` generates an
-    // amazingly intense `adaptivePredict` call that doesn't actually work
-    // and builds a serious DFA. Huge. So instead we use standard ANTLR syntax
-    // to consume the token and then undo the consumption. This looks hairy but
-    // it is better than the alternatives.
-    |   { int mark = _input.mark(); int index = _input.index(); }
-            RBRACK
-        { _input.seek(index); _input.release(mark); }
     ;
 
 // Note we return the boolean s.  This is returned as true
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java
new file mode 100644
index 00000000000..94a2c258974
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.antlr;
+
+import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.Lexer;
+import org.antlr.v4.runtime.LexerNoViableAltException;
+import org.antlr.v4.runtime.Token;
+import org.antlr.v4.runtime.TokenSource;
+import org.antlr.v4.runtime.misc.Interval;
+import org.antlr.v4.runtime.misc.Pair;
+import org.elasticsearch.painless.Location;
+
+/**
+ * A lexer that is customized for painless. It will:
+ * <ul>
+ * <li>will override the default error behavior to fail on the first error
+ * <li>store the last token in case we need to do lookbehind for semicolon insertion and regex vs division detection
+ * <li>insert semicolons where they'd improve the language's readability. Rather than hack this into the parser and create a ton of
+ * ambiguity we hack them here where we can use heuristics to do it quickly.
+ * </ul>
+ */
+final class EnhancedPainlessLexer extends PainlessLexer {
+    final String sourceName;
+    private Token stashedNext = null;
+    private Token previous = null;
+
+    EnhancedPainlessLexer(CharStream charStream, String sourceName) {
+        super(charStream);
+        this.sourceName = sourceName;
+    }
+
+    public Token getPreviousToken() {
+        return previous;
+    }
+
+    @Override
+    public Token nextToken() {
+        if (stashedNext != null) {
+            previous = stashedNext;
+            stashedNext = null;
+            return previous;
+        }
+        Token next = super.nextToken();
+        if (insertSemicolon(previous, next)) {
+            stashedNext = next;
+            previous = _factory.create(new Pair<TokenSource, CharStream>(this, _input), PainlessLexer.SEMICOLON, ";",
+                    Lexer.DEFAULT_TOKEN_CHANNEL, next.getStartIndex(), next.getStopIndex(), next.getLine(), next.getCharPositionInLine());
+            return previous;
+        } else {
+            previous = next;
+            return next;
+        }
+    }
+
+    @Override
+    public void recover(final LexerNoViableAltException lnvae) {
+        final CharStream charStream = lnvae.getInputStream();
+        final int startIndex = lnvae.getStartIndex();
+        final String text = charStream.getText(Interval.of(startIndex, charStream.index()));
+
+        Location location = new Location(sourceName, _tokenStartCharIndex);
+        throw location.createError(new IllegalArgumentException("unexpected character [" + getErrorDisplay(text) + "].", lnvae));
+    }
+
+    private static boolean insertSemicolon(Token previous, Token next) {
+        if (previous == null || next.getType() != PainlessLexer.RBRACK) {
+            return false;
+        }
+        switch (previous.getType()) {
+        case PainlessLexer.RBRACK:     // };} would be weird!
+        case PainlessLexer.SEMICOLON:  // already have a semicolon, no need to add one
+        case PainlessLexer.LBRACK:     // empty blocks don't need a semicolon
+            return false;
+        default:
+            return true;
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
index cf0ac8605f0..f512ad20d7e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
@@ -140,14 +140,14 @@ class PainlessLexer extends Lexer {
   private boolean DIV_sempred(RuleContext _localctx, int predIndex) {
     switch (predIndex) {
     case 0:
-      return  false == SlashStrategy.slashIsRegex(_factory) ;
+      return  false == SlashStrategy.slashIsRegex(this) ;
     }
     return true;
   }
   private boolean REGEX_sempred(RuleContext _localctx, int predIndex) {
     switch (predIndex) {
     case 1:
-      return  SlashStrategy.slashIsRegex(_factory) ;
+      return  SlashStrategy.slashIsRegex(this) ;
     }
     return true;
   }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
index 7b994e699a2..39d1ca3d188 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
@@ -1287,7 +1287,6 @@ class PainlessParser extends Parser {
   public static class DelimiterContext extends ParserRuleContext {
     public TerminalNode SEMICOLON() { return getToken(PainlessParser.SEMICOLON, 0); }
     public TerminalNode EOF() { return getToken(PainlessParser.EOF, 0); }
-    public TerminalNode RBRACK() { return getToken(PainlessParser.RBRACK, 0); }
     public DelimiterContext(ParserRuleContext parent, int invokingState) {
       super(parent, invokingState);
     }
@@ -1302,34 +1301,17 @@ class PainlessParser extends Parser {
   public final DelimiterContext delimiter() throws RecognitionException {
     DelimiterContext _localctx = new DelimiterContext(_ctx, getState());
     enterRule(_localctx, 26, RULE_delimiter);
+    int _la;
     try {
-      setState(230);
-      switch (_input.LA(1)) {
-      case SEMICOLON:
-        enterOuterAlt(_localctx, 1);
-        {
-        setState(225);
-        match(SEMICOLON);
-        }
-        break;
-      case EOF:
-        enterOuterAlt(_localctx, 2);
-        {
-        setState(226);
-        match(EOF);
-        }
-        break;
-      case RBRACK:
-        enterOuterAlt(_localctx, 3);
-        {
-         int mark = _input.mark(); int index = _input.index(); 
-        setState(228);
-        match(RBRACK);
-         _input.seek(index); _input.release(mark); 
-        }
-        break;
-      default:
-        throw new NoViableAltException(this);
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(225);
+      _la = _input.LA(1);
+      if ( !(_la==EOF || _la==SEMICOLON) ) {
+      _errHandler.recoverInline(this);
+      } else {
+        consume();
+      }
       }
     }
     catch (RecognitionException re) {
@@ -1494,24 +1476,24 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(241);
-      switch ( getInterpreter().adaptivePredict(_input,19,_ctx) ) {
+      setState(236);
+      switch ( getInterpreter().adaptivePredict(_input,18,_ctx) ) {
       case 1:
         {
         _localctx = new AssignmentContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(233);
+        setState(228);
         chain(true);
-        setState(234);
+        setState(229);
         _la = _input.LA(1);
         if ( !(((((_la - 56)) & ~0x3f) == 0 && ((1L << (_la - 56)) & ((1L << (ASSIGN - 56)) | (1L << (AADD - 56)) | (1L << (ASUB - 56)) | (1L << (AMUL - 56)) | (1L << (ADIV - 56)) | (1L << (AREM - 56)) | (1L << (AAND - 56)) | (1L << (AXOR - 56)) | (1L << (AOR - 56)) | (1L << (ALSH - 56)) | (1L << (ARSH - 56)) | (1L << (AUSH - 56)))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(235);
+        setState(230);
         expression(1);
          ((AssignmentContext)_localctx).s =  false; 
         }
@@ -1521,37 +1503,37 @@ class PainlessParser extends Parser {
         _localctx = new SingleContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(238);
+        setState(233);
         ((SingleContext)_localctx).u = unary(false);
          ((SingleContext)_localctx).s =  ((SingleContext)_localctx).u.s; 
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(307);
+      setState(302);
       _errHandler.sync(this);
-      _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
+      _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(305);
-          switch ( getInterpreter().adaptivePredict(_input,20,_ctx) ) {
+          setState(300);
+          switch ( getInterpreter().adaptivePredict(_input,19,_ctx) ) {
           case 1:
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(243);
+            setState(238);
             if (!(precpred(_ctx, 13))) throw new FailedPredicateException(this, "precpred(_ctx, 13)");
-            setState(244);
+            setState(239);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << MUL) | (1L << DIV) | (1L << REM))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(245);
+            setState(240);
             expression(14);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1560,16 +1542,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(248);
+            setState(243);
             if (!(precpred(_ctx, 12))) throw new FailedPredicateException(this, "precpred(_ctx, 12)");
-            setState(249);
+            setState(244);
             _la = _input.LA(1);
             if ( !(_la==ADD || _la==SUB) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(250);
+            setState(245);
             expression(13);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1578,16 +1560,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(253);
+            setState(248);
             if (!(precpred(_ctx, 11))) throw new FailedPredicateException(this, "precpred(_ctx, 11)");
-            setState(254);
+            setState(249);
             _la = _input.LA(1);
             if ( !(_la==FIND || _la==MATCH) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(255);
+            setState(250);
             expression(12);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1596,16 +1578,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(258);
+            setState(253);
             if (!(precpred(_ctx, 10))) throw new FailedPredicateException(this, "precpred(_ctx, 10)");
-            setState(259);
+            setState(254);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LSH) | (1L << RSH) | (1L << USH))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(260);
+            setState(255);
             expression(11);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1614,16 +1596,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(263);
+            setState(258);
             if (!(precpred(_ctx, 9))) throw new FailedPredicateException(this, "precpred(_ctx, 9)");
-            setState(264);
+            setState(259);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LT) | (1L << LTE) | (1L << GT) | (1L << GTE))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(265);
+            setState(260);
             expression(10);
              ((CompContext)_localctx).s =  false; 
             }
@@ -1632,16 +1614,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(268);
+            setState(263);
             if (!(precpred(_ctx, 8))) throw new FailedPredicateException(this, "precpred(_ctx, 8)");
-            setState(269);
+            setState(264);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << EQ) | (1L << EQR) | (1L << NE) | (1L << NER))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(270);
+            setState(265);
             expression(9);
              ((CompContext)_localctx).s =  false; 
             }
@@ -1650,11 +1632,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(273);
+            setState(268);
             if (!(precpred(_ctx, 7))) throw new FailedPredicateException(this, "precpred(_ctx, 7)");
-            setState(274);
+            setState(269);
             match(BWAND);
-            setState(275);
+            setState(270);
             expression(8);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1663,11 +1645,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(278);
+            setState(273);
             if (!(precpred(_ctx, 6))) throw new FailedPredicateException(this, "precpred(_ctx, 6)");
-            setState(279);
+            setState(274);
             match(XOR);
-            setState(280);
+            setState(275);
             expression(7);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1676,11 +1658,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(283);
+            setState(278);
             if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)");
-            setState(284);
+            setState(279);
             match(BWOR);
-            setState(285);
+            setState(280);
             expression(6);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1689,11 +1671,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(288);
+            setState(283);
             if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)");
-            setState(289);
+            setState(284);
             match(BOOLAND);
-            setState(290);
+            setState(285);
             expression(5);
              ((BoolContext)_localctx).s =  false; 
             }
@@ -1702,11 +1684,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(293);
+            setState(288);
             if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)");
-            setState(294);
+            setState(289);
             match(BOOLOR);
-            setState(295);
+            setState(290);
             expression(4);
              ((BoolContext)_localctx).s =  false; 
             }
@@ -1715,15 +1697,15 @@ class PainlessParser extends Parser {
             {
             _localctx = new ConditionalContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(298);
+            setState(293);
             if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
-            setState(299);
+            setState(294);
             match(COND);
-            setState(300);
+            setState(295);
             ((ConditionalContext)_localctx).e0 = expression(0);
-            setState(301);
+            setState(296);
             match(COLON);
-            setState(302);
+            setState(297);
             ((ConditionalContext)_localctx).e1 = expression(2);
              ((ConditionalContext)_localctx).s =  ((ConditionalContext)_localctx).e0.s && ((ConditionalContext)_localctx).e1.s; 
             }
@@ -1731,9 +1713,9 @@ class PainlessParser extends Parser {
           }
           } 
         }
-        setState(309);
+        setState(304);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,21,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
       }
       }
     }
@@ -1878,22 +1860,22 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 30, RULE_unary);
     int _la;
     try {
-      setState(339);
-      switch ( getInterpreter().adaptivePredict(_input,22,_ctx) ) {
+      setState(334);
+      switch ( getInterpreter().adaptivePredict(_input,21,_ctx) ) {
       case 1:
         _localctx = new PreContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(310);
+        setState(305);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(311);
+        setState(306);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(312);
+        setState(307);
         chain(true);
         }
         break;
@@ -1901,11 +1883,11 @@ class PainlessParser extends Parser {
         _localctx = new PostContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(313);
+        setState(308);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(314);
+        setState(309);
         chain(true);
-        setState(315);
+        setState(310);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
@@ -1918,9 +1900,9 @@ class PainlessParser extends Parser {
         _localctx = new ReadContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(317);
+        setState(312);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(318);
+        setState(313);
         chain(false);
         }
         break;
@@ -1928,9 +1910,9 @@ class PainlessParser extends Parser {
         _localctx = new NumericContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(319);
+        setState(314);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(320);
+        setState(315);
         _la = _input.LA(1);
         if ( !(((((_la - 68)) & ~0x3f) == 0 && ((1L << (_la - 68)) & ((1L << (OCTAL - 68)) | (1L << (HEX - 68)) | (1L << (INTEGER - 68)) | (1L << (DECIMAL - 68)))) != 0)) ) {
         _errHandler.recoverInline(this);
@@ -1944,9 +1926,9 @@ class PainlessParser extends Parser {
         _localctx = new TrueContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(322);
+        setState(317);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(323);
+        setState(318);
         match(TRUE);
          ((TrueContext)_localctx).s =  false; 
         }
@@ -1955,9 +1937,9 @@ class PainlessParser extends Parser {
         _localctx = new FalseContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(325);
+        setState(320);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(326);
+        setState(321);
         match(FALSE);
          ((FalseContext)_localctx).s =  false; 
         }
@@ -1966,9 +1948,9 @@ class PainlessParser extends Parser {
         _localctx = new NullContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(328);
+        setState(323);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(329);
+        setState(324);
         match(NULL);
          ((NullContext)_localctx).s =  false; 
         }
@@ -1977,16 +1959,16 @@ class PainlessParser extends Parser {
         _localctx = new OperatorContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(331);
+        setState(326);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(332);
+        setState(327);
         _la = _input.LA(1);
         if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(333);
+        setState(328);
         unary(false);
         }
         break;
@@ -1994,13 +1976,13 @@ class PainlessParser extends Parser {
         _localctx = new CastContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(334);
+        setState(329);
         match(LP);
-        setState(335);
+        setState(330);
         decltype();
-        setState(336);
+        setState(331);
         match(RP);
-        setState(337);
+        setState(332);
         unary(_localctx.c);
         }
         break;
@@ -2109,29 +2091,29 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 32, RULE_chain);
     try {
       int _alt;
-      setState(375);
-      switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) {
+      setState(370);
+      switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) {
       case 1:
         _localctx = new DynamicContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(341);
+        setState(336);
         ((DynamicContext)_localctx).p = primary(_localctx.c);
-        setState(345);
+        setState(340);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(342);
+            setState(337);
             secondary(((DynamicContext)_localctx).p.s);
             }
             } 
           }
-          setState(347);
+          setState(342);
           _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
+          _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
         }
         }
         break;
@@ -2139,25 +2121,25 @@ class PainlessParser extends Parser {
         _localctx = new StaticContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(348);
+        setState(343);
         decltype();
-        setState(349);
+        setState(344);
         dot();
-        setState(353);
+        setState(348);
         _errHandler.sync(this);
-        _alt = getInterpreter().adaptivePredict(_input,24,_ctx);
+        _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(350);
+            setState(345);
             secondary(true);
             }
             } 
           }
-          setState(355);
+          setState(350);
           _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,24,_ctx);
+          _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
         }
         }
         break;
@@ -2165,11 +2147,11 @@ class PainlessParser extends Parser {
         _localctx = new NewarrayContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(356);
+        setState(351);
         match(NEW);
-        setState(357);
+        setState(352);
         match(TYPE);
-        setState(362); 
+        setState(357); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -2177,11 +2159,11 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(358);
+            setState(353);
             match(LBRACE);
-            setState(359);
+            setState(354);
             expression(0);
-            setState(360);
+            setState(355);
             match(RBRACE);
             }
             }
@@ -2189,31 +2171,31 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(364); 
+          setState(359); 
           _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,25,_ctx);
+          _alt = getInterpreter().adaptivePredict(_input,24,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
-        setState(373);
-        switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) {
+        setState(368);
+        switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) {
         case 1:
           {
-          setState(366);
+          setState(361);
           dot();
-          setState(370);
+          setState(365);
           _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,26,_ctx);
+          _alt = getInterpreter().adaptivePredict(_input,25,_ctx);
           while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
             if ( _alt==1 ) {
               {
               {
-              setState(367);
+              setState(362);
               secondary(true);
               }
               } 
             }
-            setState(372);
+            setState(367);
             _errHandler.sync(this);
-            _alt = getInterpreter().adaptivePredict(_input,26,_ctx);
+            _alt = getInterpreter().adaptivePredict(_input,25,_ctx);
           }
           }
           break;
@@ -2334,19 +2316,19 @@ class PainlessParser extends Parser {
     PrimaryContext _localctx = new PrimaryContext(_ctx, getState(), c);
     enterRule(_localctx, 34, RULE_primary);
     try {
-      setState(396);
-      switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
+      setState(391);
+      switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) {
       case 1:
         _localctx = new ExprprecContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(377);
+        setState(372);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(378);
+        setState(373);
         match(LP);
-        setState(379);
+        setState(374);
         ((ExprprecContext)_localctx).e = expression(0);
-        setState(380);
+        setState(375);
         match(RP);
          ((ExprprecContext)_localctx).s =  ((ExprprecContext)_localctx).e.s; 
         }
@@ -2355,13 +2337,13 @@ class PainlessParser extends Parser {
         _localctx = new ChainprecContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(383);
+        setState(378);
         if (!( _localctx.c )) throw new FailedPredicateException(this, " $c ");
-        setState(384);
+        setState(379);
         match(LP);
-        setState(385);
+        setState(380);
         unary(true);
-        setState(386);
+        setState(381);
         match(RP);
         }
         break;
@@ -2369,7 +2351,7 @@ class PainlessParser extends Parser {
         _localctx = new StringContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(388);
+        setState(383);
         match(STRING);
         }
         break;
@@ -2377,7 +2359,7 @@ class PainlessParser extends Parser {
         _localctx = new RegexContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(389);
+        setState(384);
         match(REGEX);
         }
         break;
@@ -2385,7 +2367,7 @@ class PainlessParser extends Parser {
         _localctx = new VariableContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(390);
+        setState(385);
         match(ID);
         }
         break;
@@ -2393,9 +2375,9 @@ class PainlessParser extends Parser {
         _localctx = new CalllocalContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(391);
+        setState(386);
         match(ID);
-        setState(392);
+        setState(387);
         arguments();
         }
         break;
@@ -2403,11 +2385,11 @@ class PainlessParser extends Parser {
         _localctx = new NewobjectContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(393);
+        setState(388);
         match(NEW);
-        setState(394);
+        setState(389);
         match(TYPE);
-        setState(395);
+        setState(390);
         arguments();
         }
         break;
@@ -2449,23 +2431,23 @@ class PainlessParser extends Parser {
     SecondaryContext _localctx = new SecondaryContext(_ctx, getState(), s);
     enterRule(_localctx, 36, RULE_secondary);
     try {
-      setState(402);
-      switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
+      setState(397);
+      switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(398);
+        setState(393);
         if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
-        setState(399);
+        setState(394);
         dot();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(400);
+        setState(395);
         if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
-        setState(401);
+        setState(396);
         brace();
         }
         break;
@@ -2523,17 +2505,17 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 38, RULE_dot);
     int _la;
     try {
-      setState(409);
-      switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) {
+      setState(404);
+      switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
       case 1:
         _localctx = new CallinvokeContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(404);
+        setState(399);
         match(DOT);
-        setState(405);
+        setState(400);
         match(DOTID);
-        setState(406);
+        setState(401);
         arguments();
         }
         break;
@@ -2541,9 +2523,9 @@ class PainlessParser extends Parser {
         _localctx = new FieldaccessContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(407);
+        setState(402);
         match(DOT);
-        setState(408);
+        setState(403);
         _la = _input.LA(1);
         if ( !(_la==DOTINTEGER || _la==DOTID) ) {
         _errHandler.recoverInline(this);
@@ -2597,11 +2579,11 @@ class PainlessParser extends Parser {
       _localctx = new BraceaccessContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(411);
+      setState(406);
       match(LBRACE);
-      setState(412);
+      setState(407);
       expression(0);
-      setState(413);
+      setState(408);
       match(RBRACE);
       }
     }
@@ -2648,34 +2630,34 @@ class PainlessParser extends Parser {
       enterOuterAlt(_localctx, 1);
       {
       {
-      setState(415);
+      setState(410);
       match(LP);
-      setState(424);
-      switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) {
+      setState(419);
+      switch ( getInterpreter().adaptivePredict(_input,32,_ctx) ) {
       case 1:
         {
-        setState(416);
+        setState(411);
         argument();
-        setState(421);
+        setState(416);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(417);
+          setState(412);
           match(COMMA);
-          setState(418);
+          setState(413);
           argument();
           }
           }
-          setState(423);
+          setState(418);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
         break;
       }
-      setState(426);
+      setState(421);
       match(RP);
       }
       }
@@ -2716,26 +2698,26 @@ class PainlessParser extends Parser {
     ArgumentContext _localctx = new ArgumentContext(_ctx, getState());
     enterRule(_localctx, 44, RULE_argument);
     try {
-      setState(431);
-      switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) {
+      setState(426);
+      switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(428);
+        setState(423);
         expression(0);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(429);
+        setState(424);
         lambda();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(430);
+        setState(425);
         funcref();
         }
         break;
@@ -2790,64 +2772,64 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(446);
+      setState(441);
       switch (_input.LA(1)) {
       case TYPE:
       case ID:
         {
-        setState(433);
+        setState(428);
         lamtype();
         }
         break;
       case LP:
         {
-        setState(434);
+        setState(429);
         match(LP);
-        setState(443);
+        setState(438);
         _la = _input.LA(1);
         if (_la==TYPE || _la==ID) {
           {
-          setState(435);
+          setState(430);
           lamtype();
-          setState(440);
+          setState(435);
           _errHandler.sync(this);
           _la = _input.LA(1);
           while (_la==COMMA) {
             {
             {
-            setState(436);
+            setState(431);
             match(COMMA);
-            setState(437);
+            setState(432);
             lamtype();
             }
             }
-            setState(442);
+            setState(437);
             _errHandler.sync(this);
             _la = _input.LA(1);
           }
           }
         }
 
-        setState(445);
+        setState(440);
         match(RP);
         }
         break;
       default:
         throw new NoViableAltException(this);
       }
-      setState(448);
+      setState(443);
       match(ARROW);
-      setState(451);
-      switch ( getInterpreter().adaptivePredict(_input,38,_ctx) ) {
+      setState(446);
+      switch ( getInterpreter().adaptivePredict(_input,37,_ctx) ) {
       case 1:
         {
-        setState(449);
+        setState(444);
         block();
         }
         break;
       case 2:
         {
-        setState(450);
+        setState(445);
         expression(0);
         }
         break;
@@ -2888,16 +2870,16 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(454);
+      setState(449);
       _la = _input.LA(1);
       if (_la==TYPE) {
         {
-        setState(453);
+        setState(448);
         decltype();
         }
       }
 
-      setState(456);
+      setState(451);
       match(ID);
       }
     }
@@ -2940,33 +2922,33 @@ class PainlessParser extends Parser {
     FuncrefContext _localctx = new FuncrefContext(_ctx, getState());
     enterRule(_localctx, 50, RULE_funcref);
     try {
-      setState(462);
-      switch ( getInterpreter().adaptivePredict(_input,40,_ctx) ) {
+      setState(457);
+      switch ( getInterpreter().adaptivePredict(_input,39,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(458);
+        setState(453);
         classFuncref();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(459);
+        setState(454);
         constructorFuncref();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(460);
+        setState(455);
         capturingFuncref();
         }
         break;
       case 4:
         enterOuterAlt(_localctx, 4);
         {
-        setState(461);
+        setState(456);
         localFuncref();
         }
         break;
@@ -3004,11 +2986,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(464);
+      setState(459);
       match(TYPE);
-      setState(465);
+      setState(460);
       match(REF);
-      setState(466);
+      setState(461);
       match(ID);
       }
     }
@@ -3046,11 +3028,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(468);
+      setState(463);
       decltype();
-      setState(469);
+      setState(464);
       match(REF);
-      setState(470);
+      setState(465);
       match(NEW);
       }
     }
@@ -3088,11 +3070,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(472);
+      setState(467);
       match(ID);
-      setState(473);
+      setState(468);
       match(REF);
-      setState(474);
+      setState(469);
       match(ID);
       }
     }
@@ -3128,11 +3110,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(476);
+      setState(471);
       match(THIS);
-      setState(477);
+      setState(472);
       match(REF);
-      setState(478);
+      setState(473);
       match(ID);
       }
     }
@@ -3239,7 +3221,7 @@ class PainlessParser extends Parser {
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3R\u01e3\4\2\t\2\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3R\u01de\4\2\t\2\4"+
     "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
     "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
     "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+
@@ -3255,171 +3237,169 @@ class PainlessParser extends Parser {
     "\16\7\u00bb\13\7\3\7\3\7\3\b\3\b\3\t\3\t\5\t\u00c3\n\t\3\n\3\n\3\13\3"+
     "\13\3\13\3\13\7\13\u00cb\n\13\f\13\16\13\u00ce\13\13\3\f\3\f\3\f\7\f\u00d3"+
     "\n\f\f\f\16\f\u00d6\13\f\3\r\3\r\3\r\5\r\u00db\n\r\3\16\3\16\3\16\3\16"+
-    "\3\16\3\16\3\16\3\17\3\17\3\17\3\17\3\17\5\17\u00e9\n\17\3\20\3\20\3\20"+
-    "\3\20\3\20\3\20\3\20\3\20\3\20\5\20\u00f4\n\20\3\20\3\20\3\20\3\20\3\20"+
+    "\3\16\3\16\3\16\3\17\3\17\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
+    "\5\20\u00ef\n\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
-    "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
-    "\3\20\7\20\u0134\n\20\f\20\16\20\u0137\13\20\3\21\3\21\3\21\3\21\3\21"+
+    "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\7\20\u012f\n\20\f\20\16"+
+    "\20\u0132\13\20\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
     "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
-    "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\5\21\u0156\n\21\3\22"+
-    "\3\22\7\22\u015a\n\22\f\22\16\22\u015d\13\22\3\22\3\22\3\22\7\22\u0162"+
-    "\n\22\f\22\16\22\u0165\13\22\3\22\3\22\3\22\3\22\3\22\3\22\6\22\u016d"+
-    "\n\22\r\22\16\22\u016e\3\22\3\22\7\22\u0173\n\22\f\22\16\22\u0176\13\22"+
-    "\5\22\u0178\n\22\5\22\u017a\n\22\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3"+
-    "\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\5\23\u018f"+
-    "\n\23\3\24\3\24\3\24\3\24\5\24\u0195\n\24\3\25\3\25\3\25\3\25\3\25\5\25"+
-    "\u019c\n\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\27\7\27\u01a6\n\27\f"+
-    "\27\16\27\u01a9\13\27\5\27\u01ab\n\27\3\27\3\27\3\30\3\30\3\30\5\30\u01b2"+
-    "\n\30\3\31\3\31\3\31\3\31\3\31\7\31\u01b9\n\31\f\31\16\31\u01bc\13\31"+
-    "\5\31\u01be\n\31\3\31\5\31\u01c1\n\31\3\31\3\31\3\31\5\31\u01c6\n\31\3"+
-    "\32\5\32\u01c9\n\32\3\32\3\32\3\33\3\33\3\33\3\33\5\33\u01d1\n\33\3\34"+
-    "\3\34\3\34\3\34\3\35\3\35\3\35\3\35\3\36\3\36\3\36\3\36\3\37\3\37\3\37"+
-    "\3\37\3\37\2\3\36 \2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,.\60"+
-    "\62\64\668:<\2\r\3\2:E\3\2\35\37\3\2 !\3\2\66\67\3\2\"$\3\2%(\3\2),\3"+
-    "\289\3\2FI\4\2\33\34 !\3\2QR\u0212\2A\3\2\2\2\4L\3\2\2\2\6Q\3\2\2\2\b"+
-    "\u00af\3\2\2\2\n\u00b3\3\2\2\2\f\u00b5\3\2\2\2\16\u00be\3\2\2\2\20\u00c2"+
-    "\3\2\2\2\22\u00c4\3\2\2\2\24\u00c6\3\2\2\2\26\u00cf\3\2\2\2\30\u00d7\3"+
-    "\2\2\2\32\u00dc\3\2\2\2\34\u00e8\3\2\2\2\36\u00f3\3\2\2\2 \u0155\3\2\2"+
-    "\2\"\u0179\3\2\2\2$\u018e\3\2\2\2&\u0194\3\2\2\2(\u019b\3\2\2\2*\u019d"+
-    "\3\2\2\2,\u01a1\3\2\2\2.\u01b1\3\2\2\2\60\u01c0\3\2\2\2\62\u01c8\3\2\2"+
-    "\2\64\u01d0\3\2\2\2\66\u01d2\3\2\2\28\u01d6\3\2\2\2:\u01da\3\2\2\2<\u01de"+
-    "\3\2\2\2>@\5\4\3\2?>\3\2\2\2@C\3\2\2\2A?\3\2\2\2AB\3\2\2\2BG\3\2\2\2C"+
-    "A\3\2\2\2DF\5\b\5\2ED\3\2\2\2FI\3\2\2\2GE\3\2\2\2GH\3\2\2\2HJ\3\2\2\2"+
-    "IG\3\2\2\2JK\7\2\2\3K\3\3\2\2\2LM\5\26\f\2MN\7P\2\2NO\5\6\4\2OP\5\f\7"+
-    "\2P\5\3\2\2\2Q]\7\t\2\2RS\5\26\f\2SZ\7P\2\2TU\7\f\2\2UV\5\26\f\2VW\7P"+
-    "\2\2WY\3\2\2\2XT\3\2\2\2Y\\\3\2\2\2ZX\3\2\2\2Z[\3\2\2\2[^\3\2\2\2\\Z\3"+
-    "\2\2\2]R\3\2\2\2]^\3\2\2\2^_\3\2\2\2_`\7\n\2\2`\7\3\2\2\2ab\7\16\2\2b"+
-    "c\7\t\2\2cd\5\36\20\2de\7\n\2\2ei\5\n\6\2fg\7\17\2\2gj\5\n\6\2hj\6\5\2"+
-    "\2if\3\2\2\2ih\3\2\2\2j\u00b0\3\2\2\2kl\7\20\2\2lm\7\t\2\2mn\5\36\20\2"+
-    "nq\7\n\2\2or\5\n\6\2pr\5\16\b\2qo\3\2\2\2qp\3\2\2\2r\u00b0\3\2\2\2st\7"+
-    "\21\2\2tu\5\f\7\2uv\7\20\2\2vw\7\t\2\2wx\5\36\20\2xy\7\n\2\2yz\5\34\17"+
-    "\2z\u00b0\3\2\2\2{|\7\22\2\2|~\7\t\2\2}\177\5\20\t\2~}\3\2\2\2~\177\3"+
-    "\2\2\2\177\u0080\3\2\2\2\u0080\u0082\7\r\2\2\u0081\u0083\5\36\20\2\u0082"+
-    "\u0081\3\2\2\2\u0082\u0083\3\2\2\2\u0083\u0084\3\2\2\2\u0084\u0086\7\r"+
-    "\2\2\u0085\u0087\5\22\n\2\u0086\u0085\3\2\2\2\u0086\u0087\3\2\2\2\u0087"+
-    "\u0088\3\2\2\2\u0088\u008b\7\n\2\2\u0089\u008c\5\n\6\2\u008a\u008c\5\16"+
-    "\b\2\u008b\u0089\3\2\2\2\u008b\u008a\3\2\2\2\u008c\u00b0\3\2\2\2\u008d"+
-    "\u008e\7\22\2\2\u008e\u008f\7\t\2\2\u008f\u0090\5\26\f\2\u0090\u0091\7"+
-    "P\2\2\u0091\u0092\7\63\2\2\u0092\u0093\5\36\20\2\u0093\u0094\7\n\2\2\u0094"+
-    "\u0095\5\n\6\2\u0095\u00b0\3\2\2\2\u0096\u0097\5\24\13\2\u0097\u0098\5"+
-    "\34\17\2\u0098\u00b0\3\2\2\2\u0099\u009a\7\23\2\2\u009a\u00b0\5\34\17"+
-    "\2\u009b\u009c\7\24\2\2\u009c\u00b0\5\34\17\2\u009d\u009e\7\25\2\2\u009e"+
-    "\u009f\5\36\20\2\u009f\u00a0\5\34\17\2\u00a0\u00b0\3\2\2\2\u00a1\u00a2"+
-    "\7\27\2\2\u00a2\u00a4\5\f\7\2\u00a3\u00a5\5\32\16\2\u00a4\u00a3\3\2\2"+
-    "\2\u00a5\u00a6\3\2\2\2\u00a6\u00a4\3\2\2\2\u00a6\u00a7\3\2\2\2\u00a7\u00b0"+
-    "\3\2\2\2\u00a8\u00a9\7\31\2\2\u00a9\u00aa\5\36\20\2\u00aa\u00ab\5\34\17"+
-    "\2\u00ab\u00b0\3\2\2\2\u00ac\u00ad\5\36\20\2\u00ad\u00ae\5\34\17\2\u00ae"+
-    "\u00b0\3\2\2\2\u00afa\3\2\2\2\u00afk\3\2\2\2\u00afs\3\2\2\2\u00af{\3\2"+
-    "\2\2\u00af\u008d\3\2\2\2\u00af\u0096\3\2\2\2\u00af\u0099\3\2\2\2\u00af"+
-    "\u009b\3\2\2\2\u00af\u009d\3\2\2\2\u00af\u00a1\3\2\2\2\u00af\u00a8\3\2"+
-    "\2\2\u00af\u00ac\3\2\2\2\u00b0\t\3\2\2\2\u00b1\u00b4\5\f\7\2\u00b2\u00b4"+
-    "\5\b\5\2\u00b3\u00b1\3\2\2\2\u00b3\u00b2\3\2\2\2\u00b4\13\3\2\2\2\u00b5"+
-    "\u00b9\7\5\2\2\u00b6\u00b8\5\b\5\2\u00b7\u00b6\3\2\2\2\u00b8\u00bb\3\2"+
-    "\2\2\u00b9\u00b7\3\2\2\2\u00b9\u00ba\3\2\2\2\u00ba\u00bc\3\2\2\2\u00bb"+
-    "\u00b9\3\2\2\2\u00bc\u00bd\7\6\2\2\u00bd\r\3\2\2\2\u00be\u00bf\7\r\2\2"+
-    "\u00bf\17\3\2\2\2\u00c0\u00c3\5\24\13\2\u00c1\u00c3\5\36\20\2\u00c2\u00c0"+
-    "\3\2\2\2\u00c2\u00c1\3\2\2\2\u00c3\21\3\2\2\2\u00c4\u00c5\5\36\20\2\u00c5"+
-    "\23\3\2\2\2\u00c6\u00c7\5\26\f\2\u00c7\u00cc\5\30\r\2\u00c8\u00c9\7\f"+
-    "\2\2\u00c9\u00cb\5\30\r\2\u00ca\u00c8\3\2\2\2\u00cb\u00ce\3\2\2\2\u00cc"+
-    "\u00ca\3\2\2\2\u00cc\u00cd\3\2\2\2\u00cd\25\3\2\2\2\u00ce\u00cc\3\2\2"+
-    "\2\u00cf\u00d4\7O\2\2\u00d0\u00d1\7\7\2\2\u00d1\u00d3\7\b\2\2\u00d2\u00d0"+
-    "\3\2\2\2\u00d3\u00d6\3\2\2\2\u00d4\u00d2\3\2\2\2\u00d4\u00d5\3\2\2\2\u00d5"+
-    "\27\3\2\2\2\u00d6\u00d4\3\2\2\2\u00d7\u00da\7P\2\2\u00d8\u00d9\7:\2\2"+
-    "\u00d9\u00db\5\36\20\2\u00da\u00d8\3\2\2\2\u00da\u00db\3\2\2\2\u00db\31"+
-    "\3\2\2\2\u00dc\u00dd\7\30\2\2\u00dd\u00de\7\t\2\2\u00de\u00df\7O\2\2\u00df"+
-    "\u00e0\7P\2\2\u00e0\u00e1\7\n\2\2\u00e1\u00e2\5\f\7\2\u00e2\33\3\2\2\2"+
-    "\u00e3\u00e9\7\r\2\2\u00e4\u00e9\7\2\2\3\u00e5\u00e6\b\17\1\2\u00e6\u00e7"+
-    "\7\6\2\2\u00e7\u00e9\b\17\1\2\u00e8\u00e3\3\2\2\2\u00e8\u00e4\3\2\2\2"+
-    "\u00e8\u00e5\3\2\2\2\u00e9\35\3\2\2\2\u00ea\u00eb\b\20\1\2\u00eb\u00ec"+
-    "\5\"\22\2\u00ec\u00ed\t\2\2\2\u00ed\u00ee\5\36\20\3\u00ee\u00ef\b\20\1"+
-    "\2\u00ef\u00f4\3\2\2\2\u00f0\u00f1\5 \21\2\u00f1\u00f2\b\20\1\2\u00f2"+
-    "\u00f4\3\2\2\2\u00f3\u00ea\3\2\2\2\u00f3\u00f0\3\2\2\2\u00f4\u0135\3\2"+
-    "\2\2\u00f5\u00f6\f\17\2\2\u00f6\u00f7\t\3\2\2\u00f7\u00f8\5\36\20\20\u00f8"+
-    "\u00f9\b\20\1\2\u00f9\u0134\3\2\2\2\u00fa\u00fb\f\16\2\2\u00fb\u00fc\t"+
-    "\4\2\2\u00fc\u00fd\5\36\20\17\u00fd\u00fe\b\20\1\2\u00fe\u0134\3\2\2\2"+
-    "\u00ff\u0100\f\r\2\2\u0100\u0101\t\5\2\2\u0101\u0102\5\36\20\16\u0102"+
-    "\u0103\b\20\1\2\u0103\u0134\3\2\2\2\u0104\u0105\f\f\2\2\u0105\u0106\t"+
-    "\6\2\2\u0106\u0107\5\36\20\r\u0107\u0108\b\20\1\2\u0108\u0134\3\2\2\2"+
-    "\u0109\u010a\f\13\2\2\u010a\u010b\t\7\2\2\u010b\u010c\5\36\20\f\u010c"+
-    "\u010d\b\20\1\2\u010d\u0134\3\2\2\2\u010e\u010f\f\n\2\2\u010f\u0110\t"+
-    "\b\2\2\u0110\u0111\5\36\20\13\u0111\u0112\b\20\1\2\u0112\u0134\3\2\2\2"+
-    "\u0113\u0114\f\t\2\2\u0114\u0115\7-\2\2\u0115\u0116\5\36\20\n\u0116\u0117"+
-    "\b\20\1\2\u0117\u0134\3\2\2\2\u0118\u0119\f\b\2\2\u0119\u011a\7.\2\2\u011a"+
-    "\u011b\5\36\20\t\u011b\u011c\b\20\1\2\u011c\u0134\3\2\2\2\u011d\u011e"+
-    "\f\7\2\2\u011e\u011f\7/\2\2\u011f\u0120\5\36\20\b\u0120\u0121\b\20\1\2"+
-    "\u0121\u0134\3\2\2\2\u0122\u0123\f\6\2\2\u0123\u0124\7\60\2\2\u0124\u0125"+
-    "\5\36\20\7\u0125\u0126\b\20\1\2\u0126\u0134\3\2\2\2\u0127\u0128\f\5\2"+
-    "\2\u0128\u0129\7\61\2\2\u0129\u012a\5\36\20\6\u012a\u012b\b\20\1\2\u012b"+
-    "\u0134\3\2\2\2\u012c\u012d\f\4\2\2\u012d\u012e\7\62\2\2\u012e\u012f\5"+
-    "\36\20\2\u012f\u0130\7\63\2\2\u0130\u0131\5\36\20\4\u0131\u0132\b\20\1"+
-    "\2\u0132\u0134\3\2\2\2\u0133\u00f5\3\2\2\2\u0133\u00fa\3\2\2\2\u0133\u00ff"+
-    "\3\2\2\2\u0133\u0104\3\2\2\2\u0133\u0109\3\2\2\2\u0133\u010e\3\2\2\2\u0133"+
-    "\u0113\3\2\2\2\u0133\u0118\3\2\2\2\u0133\u011d\3\2\2\2\u0133\u0122\3\2"+
-    "\2\2\u0133\u0127\3\2\2\2\u0133\u012c\3\2\2\2\u0134\u0137\3\2\2\2\u0135"+
-    "\u0133\3\2\2\2\u0135\u0136\3\2\2\2\u0136\37\3\2\2\2\u0137\u0135\3\2\2"+
-    "\2\u0138\u0139\6\21\17\3\u0139\u013a\t\t\2\2\u013a\u0156\5\"\22\2\u013b"+
-    "\u013c\6\21\20\3\u013c\u013d\5\"\22\2\u013d\u013e\t\t\2\2\u013e\u0156"+
-    "\3\2\2\2\u013f\u0140\6\21\21\3\u0140\u0156\5\"\22\2\u0141\u0142\6\21\22"+
-    "\3\u0142\u0143\t\n\2\2\u0143\u0156\b\21\1\2\u0144\u0145\6\21\23\3\u0145"+
-    "\u0146\7L\2\2\u0146\u0156\b\21\1\2\u0147\u0148\6\21\24\3\u0148\u0149\7"+
-    "M\2\2\u0149\u0156\b\21\1\2\u014a\u014b\6\21\25\3\u014b\u014c\7N\2\2\u014c"+
-    "\u0156\b\21\1\2\u014d\u014e\6\21\26\3\u014e\u014f\t\13\2\2\u014f\u0156"+
-    "\5 \21\2\u0150\u0151\7\t\2\2\u0151\u0152\5\26\f\2\u0152\u0153\7\n\2\2"+
-    "\u0153\u0154\5 \21\2\u0154\u0156\3\2\2\2\u0155\u0138\3\2\2\2\u0155\u013b"+
-    "\3\2\2\2\u0155\u013f\3\2\2\2\u0155\u0141\3\2\2\2\u0155\u0144\3\2\2\2\u0155"+
-    "\u0147\3\2\2\2\u0155\u014a\3\2\2\2\u0155\u014d\3\2\2\2\u0155\u0150\3\2"+
-    "\2\2\u0156!\3\2\2\2\u0157\u015b\5$\23\2\u0158\u015a\5&\24\2\u0159\u0158"+
-    "\3\2\2\2\u015a\u015d\3\2\2\2\u015b\u0159\3\2\2\2\u015b\u015c\3\2\2\2\u015c"+
-    "\u017a\3\2\2\2\u015d\u015b\3\2\2\2\u015e\u015f\5\26\f\2\u015f\u0163\5"+
-    "(\25\2\u0160\u0162\5&\24\2\u0161\u0160\3\2\2\2\u0162\u0165\3\2\2\2\u0163"+
-    "\u0161\3\2\2\2\u0163\u0164\3\2\2\2\u0164\u017a\3\2\2\2\u0165\u0163\3\2"+
-    "\2\2\u0166\u0167\7\26\2\2\u0167\u016c\7O\2\2\u0168\u0169\7\7\2\2\u0169"+
-    "\u016a\5\36\20\2\u016a\u016b\7\b\2\2\u016b\u016d\3\2\2\2\u016c\u0168\3"+
-    "\2\2\2\u016d\u016e\3\2\2\2\u016e\u016c\3\2\2\2\u016e\u016f\3\2\2\2\u016f"+
-    "\u0177\3\2\2\2\u0170\u0174\5(\25\2\u0171\u0173\5&\24\2\u0172\u0171\3\2"+
-    "\2\2\u0173\u0176\3\2\2\2\u0174\u0172\3\2\2\2\u0174\u0175\3\2\2\2\u0175"+
-    "\u0178\3\2\2\2\u0176\u0174\3\2\2\2\u0177\u0170\3\2\2\2\u0177\u0178\3\2"+
-    "\2\2\u0178\u017a\3\2\2\2\u0179\u0157\3\2\2\2\u0179\u015e\3\2\2\2\u0179"+
-    "\u0166\3\2\2\2\u017a#\3\2\2\2\u017b\u017c\6\23\27\3\u017c\u017d\7\t\2"+
-    "\2\u017d\u017e\5\36\20\2\u017e\u017f\7\n\2\2\u017f\u0180\b\23\1\2\u0180"+
-    "\u018f\3\2\2\2\u0181\u0182\6\23\30\3\u0182\u0183\7\t\2\2\u0183\u0184\5"+
-    " \21\2\u0184\u0185\7\n\2\2\u0185\u018f\3\2\2\2\u0186\u018f\7J\2\2\u0187"+
-    "\u018f\7K\2\2\u0188\u018f\7P\2\2\u0189\u018a\7P\2\2\u018a\u018f\5,\27"+
-    "\2\u018b\u018c\7\26\2\2\u018c\u018d\7O\2\2\u018d\u018f\5,\27\2\u018e\u017b"+
-    "\3\2\2\2\u018e\u0181\3\2\2\2\u018e\u0186\3\2\2\2\u018e\u0187\3\2\2\2\u018e"+
-    "\u0188\3\2\2\2\u018e\u0189\3\2\2\2\u018e\u018b\3\2\2\2\u018f%\3\2\2\2"+
-    "\u0190\u0191\6\24\31\3\u0191\u0195\5(\25\2\u0192\u0193\6\24\32\3\u0193"+
-    "\u0195\5*\26\2\u0194\u0190\3\2\2\2\u0194\u0192\3\2\2\2\u0195\'\3\2\2\2"+
-    "\u0196\u0197\7\13\2\2\u0197\u0198\7R\2\2\u0198\u019c\5,\27\2\u0199\u019a"+
-    "\7\13\2\2\u019a\u019c\t\f\2\2\u019b\u0196\3\2\2\2\u019b\u0199\3\2\2\2"+
-    "\u019c)\3\2\2\2\u019d\u019e\7\7\2\2\u019e\u019f\5\36\20\2\u019f\u01a0"+
-    "\7\b\2\2\u01a0+\3\2\2\2\u01a1\u01aa\7\t\2\2\u01a2\u01a7\5.\30\2\u01a3"+
-    "\u01a4\7\f\2\2\u01a4\u01a6\5.\30\2\u01a5\u01a3\3\2\2\2\u01a6\u01a9\3\2"+
-    "\2\2\u01a7\u01a5\3\2\2\2\u01a7\u01a8\3\2\2\2\u01a8\u01ab\3\2\2\2\u01a9"+
-    "\u01a7\3\2\2\2\u01aa\u01a2\3\2\2\2\u01aa\u01ab\3\2\2\2\u01ab\u01ac\3\2"+
-    "\2\2\u01ac\u01ad\7\n\2\2\u01ad-\3\2\2\2\u01ae\u01b2\5\36\20\2\u01af\u01b2"+
-    "\5\60\31\2\u01b0\u01b2\5\64\33\2\u01b1\u01ae\3\2\2\2\u01b1\u01af\3\2\2"+
-    "\2\u01b1\u01b0\3\2\2\2\u01b2/\3\2\2\2\u01b3\u01c1\5\62\32\2\u01b4\u01bd"+
-    "\7\t\2\2\u01b5\u01ba\5\62\32\2\u01b6\u01b7\7\f\2\2\u01b7\u01b9\5\62\32"+
-    "\2\u01b8\u01b6\3\2\2\2\u01b9\u01bc\3\2\2\2\u01ba\u01b8\3\2\2\2\u01ba\u01bb"+
-    "\3\2\2\2\u01bb\u01be\3\2\2\2\u01bc\u01ba\3\2\2\2\u01bd\u01b5\3\2\2\2\u01bd"+
-    "\u01be\3\2\2\2\u01be\u01bf\3\2\2\2\u01bf\u01c1\7\n\2\2\u01c0\u01b3\3\2"+
-    "\2\2\u01c0\u01b4\3\2\2\2\u01c1\u01c2\3\2\2\2\u01c2\u01c5\7\65\2\2\u01c3"+
-    "\u01c6\5\f\7\2\u01c4\u01c6\5\36\20\2\u01c5\u01c3\3\2\2\2\u01c5\u01c4\3"+
-    "\2\2\2\u01c6\61\3\2\2\2\u01c7\u01c9\5\26\f\2\u01c8\u01c7\3\2\2\2\u01c8"+
-    "\u01c9\3\2\2\2\u01c9\u01ca\3\2\2\2\u01ca\u01cb\7P\2\2\u01cb\63\3\2\2\2"+
-    "\u01cc\u01d1\5\66\34\2\u01cd\u01d1\58\35\2\u01ce\u01d1\5:\36\2\u01cf\u01d1"+
-    "\5<\37\2\u01d0\u01cc\3\2\2\2\u01d0\u01cd\3\2\2\2\u01d0\u01ce\3\2\2\2\u01d0"+
-    "\u01cf\3\2\2\2\u01d1\65\3\2\2\2\u01d2\u01d3\7O\2\2\u01d3\u01d4\7\64\2"+
-    "\2\u01d4\u01d5\7P\2\2\u01d5\67\3\2\2\2\u01d6\u01d7\5\26\f\2\u01d7\u01d8"+
-    "\7\64\2\2\u01d8\u01d9\7\26\2\2\u01d99\3\2\2\2\u01da\u01db\7P\2\2\u01db"+
-    "\u01dc\7\64\2\2\u01dc\u01dd\7P\2\2\u01dd;\3\2\2\2\u01de\u01df\7\32\2\2"+
-    "\u01df\u01e0\7\64\2\2\u01e0\u01e1\7P\2\2\u01e1=\3\2\2\2+AGZ]iq~\u0082"+
-    "\u0086\u008b\u00a6\u00af\u00b3\u00b9\u00c2\u00cc\u00d4\u00da\u00e8\u00f3"+
-    "\u0133\u0135\u0155\u015b\u0163\u016e\u0174\u0177\u0179\u018e\u0194\u019b"+
-    "\u01a7\u01aa\u01b1\u01ba\u01bd\u01c0\u01c5\u01c8\u01d0";
+    "\3\21\3\21\3\21\3\21\5\21\u0151\n\21\3\22\3\22\7\22\u0155\n\22\f\22\16"+
+    "\22\u0158\13\22\3\22\3\22\3\22\7\22\u015d\n\22\f\22\16\22\u0160\13\22"+
+    "\3\22\3\22\3\22\3\22\3\22\3\22\6\22\u0168\n\22\r\22\16\22\u0169\3\22\3"+
+    "\22\7\22\u016e\n\22\f\22\16\22\u0171\13\22\5\22\u0173\n\22\5\22\u0175"+
+    "\n\22\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23"+
+    "\3\23\3\23\3\23\3\23\3\23\3\23\5\23\u018a\n\23\3\24\3\24\3\24\3\24\5\24"+
+    "\u0190\n\24\3\25\3\25\3\25\3\25\3\25\5\25\u0197\n\25\3\26\3\26\3\26\3"+
+    "\26\3\27\3\27\3\27\3\27\7\27\u01a1\n\27\f\27\16\27\u01a4\13\27\5\27\u01a6"+
+    "\n\27\3\27\3\27\3\30\3\30\3\30\5\30\u01ad\n\30\3\31\3\31\3\31\3\31\3\31"+
+    "\7\31\u01b4\n\31\f\31\16\31\u01b7\13\31\5\31\u01b9\n\31\3\31\5\31\u01bc"+
+    "\n\31\3\31\3\31\3\31\5\31\u01c1\n\31\3\32\5\32\u01c4\n\32\3\32\3\32\3"+
+    "\33\3\33\3\33\3\33\5\33\u01cc\n\33\3\34\3\34\3\34\3\34\3\35\3\35\3\35"+
+    "\3\35\3\36\3\36\3\36\3\36\3\37\3\37\3\37\3\37\3\37\2\3\36 \2\4\6\b\n\f"+
+    "\16\20\22\24\26\30\32\34\36 \"$&(*,.\60\62\64\668:<\2\16\3\3\r\r\3\2:"+
+    "E\3\2\35\37\3\2 !\3\2\66\67\3\2\"$\3\2%(\3\2),\3\289\3\2FI\4\2\33\34 "+
+    "!\3\2QR\u020b\2A\3\2\2\2\4L\3\2\2\2\6Q\3\2\2\2\b\u00af\3\2\2\2\n\u00b3"+
+    "\3\2\2\2\f\u00b5\3\2\2\2\16\u00be\3\2\2\2\20\u00c2\3\2\2\2\22\u00c4\3"+
+    "\2\2\2\24\u00c6\3\2\2\2\26\u00cf\3\2\2\2\30\u00d7\3\2\2\2\32\u00dc\3\2"+
+    "\2\2\34\u00e3\3\2\2\2\36\u00ee\3\2\2\2 \u0150\3\2\2\2\"\u0174\3\2\2\2"+
+    "$\u0189\3\2\2\2&\u018f\3\2\2\2(\u0196\3\2\2\2*\u0198\3\2\2\2,\u019c\3"+
+    "\2\2\2.\u01ac\3\2\2\2\60\u01bb\3\2\2\2\62\u01c3\3\2\2\2\64\u01cb\3\2\2"+
+    "\2\66\u01cd\3\2\2\28\u01d1\3\2\2\2:\u01d5\3\2\2\2<\u01d9\3\2\2\2>@\5\4"+
+    "\3\2?>\3\2\2\2@C\3\2\2\2A?\3\2\2\2AB\3\2\2\2BG\3\2\2\2CA\3\2\2\2DF\5\b"+
+    "\5\2ED\3\2\2\2FI\3\2\2\2GE\3\2\2\2GH\3\2\2\2HJ\3\2\2\2IG\3\2\2\2JK\7\2"+
+    "\2\3K\3\3\2\2\2LM\5\26\f\2MN\7P\2\2NO\5\6\4\2OP\5\f\7\2P\5\3\2\2\2Q]\7"+
+    "\t\2\2RS\5\26\f\2SZ\7P\2\2TU\7\f\2\2UV\5\26\f\2VW\7P\2\2WY\3\2\2\2XT\3"+
+    "\2\2\2Y\\\3\2\2\2ZX\3\2\2\2Z[\3\2\2\2[^\3\2\2\2\\Z\3\2\2\2]R\3\2\2\2]"+
+    "^\3\2\2\2^_\3\2\2\2_`\7\n\2\2`\7\3\2\2\2ab\7\16\2\2bc\7\t\2\2cd\5\36\20"+
+    "\2de\7\n\2\2ei\5\n\6\2fg\7\17\2\2gj\5\n\6\2hj\6\5\2\2if\3\2\2\2ih\3\2"+
+    "\2\2j\u00b0\3\2\2\2kl\7\20\2\2lm\7\t\2\2mn\5\36\20\2nq\7\n\2\2or\5\n\6"+
+    "\2pr\5\16\b\2qo\3\2\2\2qp\3\2\2\2r\u00b0\3\2\2\2st\7\21\2\2tu\5\f\7\2"+
+    "uv\7\20\2\2vw\7\t\2\2wx\5\36\20\2xy\7\n\2\2yz\5\34\17\2z\u00b0\3\2\2\2"+
+    "{|\7\22\2\2|~\7\t\2\2}\177\5\20\t\2~}\3\2\2\2~\177\3\2\2\2\177\u0080\3"+
+    "\2\2\2\u0080\u0082\7\r\2\2\u0081\u0083\5\36\20\2\u0082\u0081\3\2\2\2\u0082"+
+    "\u0083\3\2\2\2\u0083\u0084\3\2\2\2\u0084\u0086\7\r\2\2\u0085\u0087\5\22"+
+    "\n\2\u0086\u0085\3\2\2\2\u0086\u0087\3\2\2\2\u0087\u0088\3\2\2\2\u0088"+
+    "\u008b\7\n\2\2\u0089\u008c\5\n\6\2\u008a\u008c\5\16\b\2\u008b\u0089\3"+
+    "\2\2\2\u008b\u008a\3\2\2\2\u008c\u00b0\3\2\2\2\u008d\u008e\7\22\2\2\u008e"+
+    "\u008f\7\t\2\2\u008f\u0090\5\26\f\2\u0090\u0091\7P\2\2\u0091\u0092\7\63"+
+    "\2\2\u0092\u0093\5\36\20\2\u0093\u0094\7\n\2\2\u0094\u0095\5\n\6\2\u0095"+
+    "\u00b0\3\2\2\2\u0096\u0097\5\24\13\2\u0097\u0098\5\34\17\2\u0098\u00b0"+
+    "\3\2\2\2\u0099\u009a\7\23\2\2\u009a\u00b0\5\34\17\2\u009b\u009c\7\24\2"+
+    "\2\u009c\u00b0\5\34\17\2\u009d\u009e\7\25\2\2\u009e\u009f\5\36\20\2\u009f"+
+    "\u00a0\5\34\17\2\u00a0\u00b0\3\2\2\2\u00a1\u00a2\7\27\2\2\u00a2\u00a4"+
+    "\5\f\7\2\u00a3\u00a5\5\32\16\2\u00a4\u00a3\3\2\2\2\u00a5\u00a6\3\2\2\2"+
+    "\u00a6\u00a4\3\2\2\2\u00a6\u00a7\3\2\2\2\u00a7\u00b0\3\2\2\2\u00a8\u00a9"+
+    "\7\31\2\2\u00a9\u00aa\5\36\20\2\u00aa\u00ab\5\34\17\2\u00ab\u00b0\3\2"+
+    "\2\2\u00ac\u00ad\5\36\20\2\u00ad\u00ae\5\34\17\2\u00ae\u00b0\3\2\2\2\u00af"+
+    "a\3\2\2\2\u00afk\3\2\2\2\u00afs\3\2\2\2\u00af{\3\2\2\2\u00af\u008d\3\2"+
+    "\2\2\u00af\u0096\3\2\2\2\u00af\u0099\3\2\2\2\u00af\u009b\3\2\2\2\u00af"+
+    "\u009d\3\2\2\2\u00af\u00a1\3\2\2\2\u00af\u00a8\3\2\2\2\u00af\u00ac\3\2"+
+    "\2\2\u00b0\t\3\2\2\2\u00b1\u00b4\5\f\7\2\u00b2\u00b4\5\b\5\2\u00b3\u00b1"+
+    "\3\2\2\2\u00b3\u00b2\3\2\2\2\u00b4\13\3\2\2\2\u00b5\u00b9\7\5\2\2\u00b6"+
+    "\u00b8\5\b\5\2\u00b7\u00b6\3\2\2\2\u00b8\u00bb\3\2\2\2\u00b9\u00b7\3\2"+
+    "\2\2\u00b9\u00ba\3\2\2\2\u00ba\u00bc\3\2\2\2\u00bb\u00b9\3\2\2\2\u00bc"+
+    "\u00bd\7\6\2\2\u00bd\r\3\2\2\2\u00be\u00bf\7\r\2\2\u00bf\17\3\2\2\2\u00c0"+
+    "\u00c3\5\24\13\2\u00c1\u00c3\5\36\20\2\u00c2\u00c0\3\2\2\2\u00c2\u00c1"+
+    "\3\2\2\2\u00c3\21\3\2\2\2\u00c4\u00c5\5\36\20\2\u00c5\23\3\2\2\2\u00c6"+
+    "\u00c7\5\26\f\2\u00c7\u00cc\5\30\r\2\u00c8\u00c9\7\f\2\2\u00c9\u00cb\5"+
+    "\30\r\2\u00ca\u00c8\3\2\2\2\u00cb\u00ce\3\2\2\2\u00cc\u00ca\3\2\2\2\u00cc"+
+    "\u00cd\3\2\2\2\u00cd\25\3\2\2\2\u00ce\u00cc\3\2\2\2\u00cf\u00d4\7O\2\2"+
+    "\u00d0\u00d1\7\7\2\2\u00d1\u00d3\7\b\2\2\u00d2\u00d0\3\2\2\2\u00d3\u00d6"+
+    "\3\2\2\2\u00d4\u00d2\3\2\2\2\u00d4\u00d5\3\2\2\2\u00d5\27\3\2\2\2\u00d6"+
+    "\u00d4\3\2\2\2\u00d7\u00da\7P\2\2\u00d8\u00d9\7:\2\2\u00d9\u00db\5\36"+
+    "\20\2\u00da\u00d8\3\2\2\2\u00da\u00db\3\2\2\2\u00db\31\3\2\2\2\u00dc\u00dd"+
+    "\7\30\2\2\u00dd\u00de\7\t\2\2\u00de\u00df\7O\2\2\u00df\u00e0\7P\2\2\u00e0"+
+    "\u00e1\7\n\2\2\u00e1\u00e2\5\f\7\2\u00e2\33\3\2\2\2\u00e3\u00e4\t\2\2"+
+    "\2\u00e4\35\3\2\2\2\u00e5\u00e6\b\20\1\2\u00e6\u00e7\5\"\22\2\u00e7\u00e8"+
+    "\t\3\2\2\u00e8\u00e9\5\36\20\3\u00e9\u00ea\b\20\1\2\u00ea\u00ef\3\2\2"+
+    "\2\u00eb\u00ec\5 \21\2\u00ec\u00ed\b\20\1\2\u00ed\u00ef\3\2\2\2\u00ee"+
+    "\u00e5\3\2\2\2\u00ee\u00eb\3\2\2\2\u00ef\u0130\3\2\2\2\u00f0\u00f1\f\17"+
+    "\2\2\u00f1\u00f2\t\4\2\2\u00f2\u00f3\5\36\20\20\u00f3\u00f4\b\20\1\2\u00f4"+
+    "\u012f\3\2\2\2\u00f5\u00f6\f\16\2\2\u00f6\u00f7\t\5\2\2\u00f7\u00f8\5"+
+    "\36\20\17\u00f8\u00f9\b\20\1\2\u00f9\u012f\3\2\2\2\u00fa\u00fb\f\r\2\2"+
+    "\u00fb\u00fc\t\6\2\2\u00fc\u00fd\5\36\20\16\u00fd\u00fe\b\20\1\2\u00fe"+
+    "\u012f\3\2\2\2\u00ff\u0100\f\f\2\2\u0100\u0101\t\7\2\2\u0101\u0102\5\36"+
+    "\20\r\u0102\u0103\b\20\1\2\u0103\u012f\3\2\2\2\u0104\u0105\f\13\2\2\u0105"+
+    "\u0106\t\b\2\2\u0106\u0107\5\36\20\f\u0107\u0108\b\20\1\2\u0108\u012f"+
+    "\3\2\2\2\u0109\u010a\f\n\2\2\u010a\u010b\t\t\2\2\u010b\u010c\5\36\20\13"+
+    "\u010c\u010d\b\20\1\2\u010d\u012f\3\2\2\2\u010e\u010f\f\t\2\2\u010f\u0110"+
+    "\7-\2\2\u0110\u0111\5\36\20\n\u0111\u0112\b\20\1\2\u0112\u012f\3\2\2\2"+
+    "\u0113\u0114\f\b\2\2\u0114\u0115\7.\2\2\u0115\u0116\5\36\20\t\u0116\u0117"+
+    "\b\20\1\2\u0117\u012f\3\2\2\2\u0118\u0119\f\7\2\2\u0119\u011a\7/\2\2\u011a"+
+    "\u011b\5\36\20\b\u011b\u011c\b\20\1\2\u011c\u012f\3\2\2\2\u011d\u011e"+
+    "\f\6\2\2\u011e\u011f\7\60\2\2\u011f\u0120\5\36\20\7\u0120\u0121\b\20\1"+
+    "\2\u0121\u012f\3\2\2\2\u0122\u0123\f\5\2\2\u0123\u0124\7\61\2\2\u0124"+
+    "\u0125\5\36\20\6\u0125\u0126\b\20\1\2\u0126\u012f\3\2\2\2\u0127\u0128"+
+    "\f\4\2\2\u0128\u0129\7\62\2\2\u0129\u012a\5\36\20\2\u012a\u012b\7\63\2"+
+    "\2\u012b\u012c\5\36\20\4\u012c\u012d\b\20\1\2\u012d\u012f\3\2\2\2\u012e"+
+    "\u00f0\3\2\2\2\u012e\u00f5\3\2\2\2\u012e\u00fa\3\2\2\2\u012e\u00ff\3\2"+
+    "\2\2\u012e\u0104\3\2\2\2\u012e\u0109\3\2\2\2\u012e\u010e\3\2\2\2\u012e"+
+    "\u0113\3\2\2\2\u012e\u0118\3\2\2\2\u012e\u011d\3\2\2\2\u012e\u0122\3\2"+
+    "\2\2\u012e\u0127\3\2\2\2\u012f\u0132\3\2\2\2\u0130\u012e\3\2\2\2\u0130"+
+    "\u0131\3\2\2\2\u0131\37\3\2\2\2\u0132\u0130\3\2\2\2\u0133\u0134\6\21\17"+
+    "\3\u0134\u0135\t\n\2\2\u0135\u0151\5\"\22\2\u0136\u0137\6\21\20\3\u0137"+
+    "\u0138\5\"\22\2\u0138\u0139\t\n\2\2\u0139\u0151\3\2\2\2\u013a\u013b\6"+
+    "\21\21\3\u013b\u0151\5\"\22\2\u013c\u013d\6\21\22\3\u013d\u013e\t\13\2"+
+    "\2\u013e\u0151\b\21\1\2\u013f\u0140\6\21\23\3\u0140\u0141\7L\2\2\u0141"+
+    "\u0151\b\21\1\2\u0142\u0143\6\21\24\3\u0143\u0144\7M\2\2\u0144\u0151\b"+
+    "\21\1\2\u0145\u0146\6\21\25\3\u0146\u0147\7N\2\2\u0147\u0151\b\21\1\2"+
+    "\u0148\u0149\6\21\26\3\u0149\u014a\t\f\2\2\u014a\u0151\5 \21\2\u014b\u014c"+
+    "\7\t\2\2\u014c\u014d\5\26\f\2\u014d\u014e\7\n\2\2\u014e\u014f\5 \21\2"+
+    "\u014f\u0151\3\2\2\2\u0150\u0133\3\2\2\2\u0150\u0136\3\2\2\2\u0150\u013a"+
+    "\3\2\2\2\u0150\u013c\3\2\2\2\u0150\u013f\3\2\2\2\u0150\u0142\3\2\2\2\u0150"+
+    "\u0145\3\2\2\2\u0150\u0148\3\2\2\2\u0150\u014b\3\2\2\2\u0151!\3\2\2\2"+
+    "\u0152\u0156\5$\23\2\u0153\u0155\5&\24\2\u0154\u0153\3\2\2\2\u0155\u0158"+
+    "\3\2\2\2\u0156\u0154\3\2\2\2\u0156\u0157\3\2\2\2\u0157\u0175\3\2\2\2\u0158"+
+    "\u0156\3\2\2\2\u0159\u015a\5\26\f\2\u015a\u015e\5(\25\2\u015b\u015d\5"+
+    "&\24\2\u015c\u015b\3\2\2\2\u015d\u0160\3\2\2\2\u015e\u015c\3\2\2\2\u015e"+
+    "\u015f\3\2\2\2\u015f\u0175\3\2\2\2\u0160\u015e\3\2\2\2\u0161\u0162\7\26"+
+    "\2\2\u0162\u0167\7O\2\2\u0163\u0164\7\7\2\2\u0164\u0165\5\36\20\2\u0165"+
+    "\u0166\7\b\2\2\u0166\u0168\3\2\2\2\u0167\u0163\3\2\2\2\u0168\u0169\3\2"+
+    "\2\2\u0169\u0167\3\2\2\2\u0169\u016a\3\2\2\2\u016a\u0172\3\2\2\2\u016b"+
+    "\u016f\5(\25\2\u016c\u016e\5&\24\2\u016d\u016c\3\2\2\2\u016e\u0171\3\2"+
+    "\2\2\u016f\u016d\3\2\2\2\u016f\u0170\3\2\2\2\u0170\u0173\3\2\2\2\u0171"+
+    "\u016f\3\2\2\2\u0172\u016b\3\2\2\2\u0172\u0173\3\2\2\2\u0173\u0175\3\2"+
+    "\2\2\u0174\u0152\3\2\2\2\u0174\u0159\3\2\2\2\u0174\u0161\3\2\2\2\u0175"+
+    "#\3\2\2\2\u0176\u0177\6\23\27\3\u0177\u0178\7\t\2\2\u0178\u0179\5\36\20"+
+    "\2\u0179\u017a\7\n\2\2\u017a\u017b\b\23\1\2\u017b\u018a\3\2\2\2\u017c"+
+    "\u017d\6\23\30\3\u017d\u017e\7\t\2\2\u017e\u017f\5 \21\2\u017f\u0180\7"+
+    "\n\2\2\u0180\u018a\3\2\2\2\u0181\u018a\7J\2\2\u0182\u018a\7K\2\2\u0183"+
+    "\u018a\7P\2\2\u0184\u0185\7P\2\2\u0185\u018a\5,\27\2\u0186\u0187\7\26"+
+    "\2\2\u0187\u0188\7O\2\2\u0188\u018a\5,\27\2\u0189\u0176\3\2\2\2\u0189"+
+    "\u017c\3\2\2\2\u0189\u0181\3\2\2\2\u0189\u0182\3\2\2\2\u0189\u0183\3\2"+
+    "\2\2\u0189\u0184\3\2\2\2\u0189\u0186\3\2\2\2\u018a%\3\2\2\2\u018b\u018c"+
+    "\6\24\31\3\u018c\u0190\5(\25\2\u018d\u018e\6\24\32\3\u018e\u0190\5*\26"+
+    "\2\u018f\u018b\3\2\2\2\u018f\u018d\3\2\2\2\u0190\'\3\2\2\2\u0191\u0192"+
+    "\7\13\2\2\u0192\u0193\7R\2\2\u0193\u0197\5,\27\2\u0194\u0195\7\13\2\2"+
+    "\u0195\u0197\t\r\2\2\u0196\u0191\3\2\2\2\u0196\u0194\3\2\2\2\u0197)\3"+
+    "\2\2\2\u0198\u0199\7\7\2\2\u0199\u019a\5\36\20\2\u019a\u019b\7\b\2\2\u019b"+
+    "+\3\2\2\2\u019c\u01a5\7\t\2\2\u019d\u01a2\5.\30\2\u019e\u019f\7\f\2\2"+
+    "\u019f\u01a1\5.\30\2\u01a0\u019e\3\2\2\2\u01a1\u01a4\3\2\2\2\u01a2\u01a0"+
+    "\3\2\2\2\u01a2\u01a3\3\2\2\2\u01a3\u01a6\3\2\2\2\u01a4\u01a2\3\2\2\2\u01a5"+
+    "\u019d\3\2\2\2\u01a5\u01a6\3\2\2\2\u01a6\u01a7\3\2\2\2\u01a7\u01a8\7\n"+
+    "\2\2\u01a8-\3\2\2\2\u01a9\u01ad\5\36\20\2\u01aa\u01ad\5\60\31\2\u01ab"+
+    "\u01ad\5\64\33\2\u01ac\u01a9\3\2\2\2\u01ac\u01aa\3\2\2\2\u01ac\u01ab\3"+
+    "\2\2\2\u01ad/\3\2\2\2\u01ae\u01bc\5\62\32\2\u01af\u01b8\7\t\2\2\u01b0"+
+    "\u01b5\5\62\32\2\u01b1\u01b2\7\f\2\2\u01b2\u01b4\5\62\32\2\u01b3\u01b1"+
+    "\3\2\2\2\u01b4\u01b7\3\2\2\2\u01b5\u01b3\3\2\2\2\u01b5\u01b6\3\2\2\2\u01b6"+
+    "\u01b9\3\2\2\2\u01b7\u01b5\3\2\2\2\u01b8\u01b0\3\2\2\2\u01b8\u01b9\3\2"+
+    "\2\2\u01b9\u01ba\3\2\2\2\u01ba\u01bc\7\n\2\2\u01bb\u01ae\3\2\2\2\u01bb"+
+    "\u01af\3\2\2\2\u01bc\u01bd\3\2\2\2\u01bd\u01c0\7\65\2\2\u01be\u01c1\5"+
+    "\f\7\2\u01bf\u01c1\5\36\20\2\u01c0\u01be\3\2\2\2\u01c0\u01bf\3\2\2\2\u01c1"+
+    "\61\3\2\2\2\u01c2\u01c4\5\26\f\2\u01c3\u01c2\3\2\2\2\u01c3\u01c4\3\2\2"+
+    "\2\u01c4\u01c5\3\2\2\2\u01c5\u01c6\7P\2\2\u01c6\63\3\2\2\2\u01c7\u01cc"+
+    "\5\66\34\2\u01c8\u01cc\58\35\2\u01c9\u01cc\5:\36\2\u01ca\u01cc\5<\37\2"+
+    "\u01cb\u01c7\3\2\2\2\u01cb\u01c8\3\2\2\2\u01cb\u01c9\3\2\2\2\u01cb\u01ca"+
+    "\3\2\2\2\u01cc\65\3\2\2\2\u01cd\u01ce\7O\2\2\u01ce\u01cf\7\64\2\2\u01cf"+
+    "\u01d0\7P\2\2\u01d0\67\3\2\2\2\u01d1\u01d2\5\26\f\2\u01d2\u01d3\7\64\2"+
+    "\2\u01d3\u01d4\7\26\2\2\u01d49\3\2\2\2\u01d5\u01d6\7P\2\2\u01d6\u01d7"+
+    "\7\64\2\2\u01d7\u01d8\7P\2\2\u01d8;\3\2\2\2\u01d9\u01da\7\32\2\2\u01da"+
+    "\u01db\7\64\2\2\u01db\u01dc\7P\2\2\u01dc=\3\2\2\2*AGZ]iq~\u0082\u0086"+
+    "\u008b\u00a6\u00af\u00b3\u00b9\u00c2\u00cc\u00d4\u00da\u00ee\u012e\u0130"+
+    "\u0150\u0156\u015e\u0169\u016f\u0172\u0174\u0189\u018f\u0196\u01a2\u01a5"+
+    "\u01ac\u01b5\u01b8\u01bb\u01c0\u01c3\u01cb";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SlashStrategy.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SlashStrategy.java
index 720259d5316..698a9dfc364 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SlashStrategy.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/SlashStrategy.java
@@ -20,15 +20,14 @@
 package org.elasticsearch.painless.antlr;
 
 import org.antlr.v4.runtime.Token;
-import org.antlr.v4.runtime.TokenFactory;
 
 /**
  * Utility to figure out if a {@code /} is division or the start of a regex literal.
  */
 public class SlashStrategy {
-    public static boolean slashIsRegex(TokenFactory<?> factory) {
-        StashingTokenFactory<?> stashingFactory = (StashingTokenFactory<?>) factory;
-        Token lastToken = stashingFactory.getLastToken();
+    public static boolean slashIsRegex(PainlessLexer lexer) {
+        EnhancedPainlessLexer realLexer = (EnhancedPainlessLexer) lexer;
+        Token lastToken = realLexer.getPreviousToken();
         if (lastToken == null) {
             return true;
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index 9d2ed2368fe..f3751892b97 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -183,7 +183,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
 
     private SourceContext buildAntlrTree(String source) {
         ANTLRInputStream stream = new ANTLRInputStream(source);
-        PainlessLexer lexer = new ErrorHandlingLexer(stream, sourceName);
+        PainlessLexer lexer = new EnhancedPainlessLexer(stream, sourceName);
         PainlessParser parser = new PainlessParser(new CommonTokenStream(lexer));
         ParserErrorStrategy strategy = new ParserErrorStrategy(sourceName);
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
index 5001e28a954..dada9a9bc37 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
@@ -79,7 +79,11 @@ public class LambdaTests extends ScriptTestCase {
     }
 
     public void testMultipleStatements() {
-        assertEquals(2, exec("int applyOne(IntFunction arg) { arg.apply(1) } applyOne(x -> { x = x + 1; return x;})"));
+        assertEquals(2, exec("int applyOne(IntFunction arg) { arg.apply(1) } applyOne(x -> { x = x + 1; return x })"));
+    }
+
+    public void testUnneededCurlyStatements() {
+        assertEquals(2, exec("int applyOne(IntFunction arg) { arg.apply(1) } applyOne(x -> { x + 1 })"));
     }
 
     public void testTwoLambdas() {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java
index a36eedd7058..e255a776bed 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java
@@ -201,6 +201,6 @@ public class RegexTests extends ScriptTestCase {
         IllegalArgumentException e = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("/asdf/b", emptyMap(), emptyMap(), null); // Not picky so we get a non-assertion error
         });
-        assertEquals("invalid sequence of tokens near ['b'].", e.getMessage());
+        assertEquals("unexpected token ['b'] was expecting one of [{<EOF>, ';'}].", e.getMessage());
     }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/antlr/ParserTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/antlr/ParserTests.java
index d1852ada27b..e2bd880b646 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/antlr/ParserTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/antlr/ParserTests.java
@@ -40,7 +40,7 @@ public class ParserTests extends ScriptTestCase {
 
     private SourceContext buildAntlrTree(String source) {
         ANTLRInputStream stream = new ANTLRInputStream(source);
-        PainlessLexer lexer = new ErrorHandlingLexer(stream, "testing");
+        PainlessLexer lexer = new EnhancedPainlessLexer(stream, "testing");
         PainlessParser parser = new PainlessParser(new CommonTokenStream(lexer));
         ParserErrorStrategy strategy = new ParserErrorStrategy("testing");
 

From e9f2548ee04f75c60b7e6d663a4c5d686aa7dddd Mon Sep 17 00:00:00 2001
From: Jeff Evans <wayne.mokane+github@gmail.com>
Date: Fri, 17 Jun 2016 15:41:25 -0500
Subject: [PATCH 0937/1311] Include script field even if it value is null

Include script field even if it value is null.

Closes #16408.
---
 .../script/ScriptFieldsFetchSubPhase.java     |  4 +---
 .../messy/tests/SearchFieldsTests.java        | 19 +++++++++++++++++++
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/fetch/script/ScriptFieldsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/script/ScriptFieldsFetchSubPhase.java
index 0838467bb6c..19a7631ccba 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/script/ScriptFieldsFetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/script/ScriptFieldsFetchSubPhase.java
@@ -64,9 +64,7 @@ public final class ScriptFieldsFetchSubPhase implements FetchSubPhase {
             SearchHitField hitField = hitContext.hit().fields().get(scriptField.name());
             if (hitField == null) {
                 final List<Object> values;
-                if (value == null) {
-                    values = Collections.emptyList();
-                } else if (value instanceof Collection) {
+                if (value instanceof Collection) {
                     // TODO: use diamond operator once JI-9019884 is fixed
                     values = new ArrayList<>((Collection<?>) value);
                 } else {
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
index bc604987a70..ba258490119 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
@@ -69,6 +69,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFa
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -369,6 +370,24 @@ public class SearchFieldsTests extends ESIntegTestCase {
         assertThat(((Map<?, ?>) sObj2Arr3.get(0)).get("arr3_field1").toString(), equalTo("arr3_value1"));
     }
 
+    public void testScriptFieldsForNullReturn() throws Exception {
+        client().prepareIndex("test", "type1", "1")
+            .setSource("foo", "bar")
+            .setRefresh(true).get();
+
+        SearchResponse response = client().prepareSearch().setQuery(matchAllQuery())
+            .addScriptField("test_script_1", new Script("return null"))
+            .get();
+
+        assertNoFailures(response);
+
+        SearchHitField fieldObj = response.getHits().getAt(0).field("test_script_1");
+        assertThat(fieldObj, notNullValue());
+        List<?> fieldValues = fieldObj.values();
+        assertThat(fieldValues, hasSize(1));
+        assertThat(fieldValues.get(0), nullValue());
+    }
+
     public void testPartialFields() throws Exception {
         createIndex("test");
         client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForYellowStatus().execute().actionGet();

From 084b35c08b2245d01a7c075bbf8e8dec2ea3f110 Mon Sep 17 00:00:00 2001
From: debadair <deb.adair@elasticsearch.com>
Date: Fri, 17 Jun 2016 14:31:03 -0700
Subject: [PATCH 0938/1311] Docs: Fixed code callout error.

---
 .../reference/indices/rollover-index.asciidoc | 21 +++++++++----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc
index 6879470267f..194dda80cf8 100644
--- a/docs/reference/indices/rollover-index.asciidoc
+++ b/docs/reference/indices/rollover-index.asciidoc
@@ -71,11 +71,12 @@ POST my_alias/_rollover/my_new_index_name
 [float]
 === Defining the new index
 
-The settings, mappings, and aliases for the new index will be taken from any
-matching <<indices-templates,index templates>>. Additionally, the body of the
-request accepts `settings`, `mappings`, and `aliases` just like the
-<<indices-create-index,create index>> API, which will override any values
-set in matching index templates:
+The settings, mappings, and aliases for the new index are taken from any
+matching <<indices-templates,index templates>>. Additionally, you can specify
+`settings`, `mappings`, and `aliases` in the body of the request, just like the
+<<indices-create-index,create index>> API. Values specified in the request 
+override any values set in matching index templates. For example, the following
+`rollover` request overrides the `index.number_of_shards` setting:
 
 [source,js]
 --------------------------------------------------
@@ -92,17 +93,15 @@ POST logs_write/_rollover
     "max_age": "7d",
     "max_docs": 1000
   },
-  "settings": { <1>
-    "index.number_of_shards": 2 <1>
+  "settings": { 
+    "index.number_of_shards": 2 
   }
 }
 --------------------------------------------------
 // CONSOLE
-<1> Set settings to override matching index template, `mappings` and `aliases`
-    can also be provided.
 
- [float]
- === Dry run
+[float]
+=== Dry run
 
 The rollover API supports `dry_run` mode, where request conditions can be
 checked without performing the actual rollover:

From e8826708c170b39713729904ebe5494b5d1d3a76 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Fri, 17 Jun 2016 17:40:52 -0400
Subject: [PATCH 0939/1311] Refactor variables

---
 .../org/elasticsearch/painless/Compiler.java  |   2 +-
 .../org/elasticsearch/painless/Constant.java  |  48 ++
 .../org/elasticsearch/painless/Globals.java   |  72 +++
 .../elasticsearch/painless/LambdaLocals.java  |  62 +++
 .../org/elasticsearch/painless/Locals.java    | 521 ++++++++----------
 .../elasticsearch/painless/antlr/Walker.java  |  33 +-
 .../painless/node/AExpression.java            |   3 +-
 .../elasticsearch/painless/node/ALink.java    |   7 +-
 .../painless/node/AStatement.java             |   6 +-
 .../elasticsearch/painless/node/EBinary.java  |  21 +-
 .../elasticsearch/painless/node/EBool.java    |  19 +-
 .../elasticsearch/painless/node/EBoolean.java |   3 +-
 .../painless/node/ECapturingFunctionRef.java  |  21 +-
 .../elasticsearch/painless/node/ECast.java    |   5 +-
 .../elasticsearch/painless/node/EChain.java   |  23 +-
 .../elasticsearch/painless/node/EComp.java    |   7 +-
 .../painless/node/EConditional.java           |   9 +-
 .../painless/node/EConstant.java              |   3 +-
 .../elasticsearch/painless/node/EDecimal.java |   3 +-
 .../painless/node/EExplicit.java              |   3 +-
 .../painless/node/EFunctionRef.java           |  15 +-
 .../elasticsearch/painless/node/ELambda.java  |  55 +-
 .../elasticsearch/painless/node/ENull.java    |   3 +-
 .../elasticsearch/painless/node/ENumeric.java |   3 +-
 .../elasticsearch/painless/node/EUnary.java   |   9 +-
 .../elasticsearch/painless/node/ILambda.java  |  40 ++
 .../painless/node/LArrayLength.java           |   7 +-
 .../elasticsearch/painless/node/LBrace.java   |   9 +-
 .../painless/node/LCallInvoke.java            |   9 +-
 .../painless/node/LCallLocal.java             |   9 +-
 .../elasticsearch/painless/node/LCast.java    |   7 +-
 .../painless/node/LDefArray.java              |   9 +-
 .../elasticsearch/painless/node/LDefCall.java |  28 +-
 .../painless/node/LDefField.java              |   7 +-
 .../elasticsearch/painless/node/LField.java   |   7 +-
 .../painless/node/LListShortcut.java          |   9 +-
 .../painless/node/LMapShortcut.java           |   9 +-
 .../painless/node/LNewArray.java              |   9 +-
 .../elasticsearch/painless/node/LNewObj.java  |   9 +-
 .../elasticsearch/painless/node/LRegex.java   |  12 +-
 .../painless/node/LShortcut.java              |   7 +-
 .../elasticsearch/painless/node/LStatic.java  |   7 +-
 .../elasticsearch/painless/node/LString.java  |   7 +-
 .../painless/node/LVariable.java              |  16 +-
 .../elasticsearch/painless/node/SBlock.java   |   5 +-
 .../elasticsearch/painless/node/SBreak.java   |   3 +-
 .../elasticsearch/painless/node/SCatch.java   |   9 +-
 .../painless/node/SContinue.java              |   3 +-
 .../painless/node/SDeclBlock.java             |   5 +-
 .../painless/node/SDeclaration.java           |   9 +-
 .../org/elasticsearch/painless/node/SDo.java  |  17 +-
 .../elasticsearch/painless/node/SEach.java    |  59 +-
 .../painless/node/SExpression.java            |   5 +-
 .../org/elasticsearch/painless/node/SFor.java |  25 +-
 .../painless/node/SFunction.java              |  81 ++-
 .../org/elasticsearch/painless/node/SIf.java  |  11 +-
 .../elasticsearch/painless/node/SIfElse.java  |  17 +-
 .../elasticsearch/painless/node/SReturn.java  |   5 +-
 .../elasticsearch/painless/node/SSource.java  | 134 +++--
 .../elasticsearch/painless/node/SThrow.java   |   5 +-
 .../org/elasticsearch/painless/node/STry.java |  15 +-
 .../elasticsearch/painless/node/SWhile.java   |  19 +-
 .../painless/ReservedWordTests.java           |   8 +-
 63 files changed, 977 insertions(+), 631 deletions(-)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/Constant.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/Globals.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaLocals.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ILambda.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
index 664c567091c..b81a9dee24c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java
@@ -109,7 +109,7 @@ final class Compiler {
             java.lang.reflect.Constructor<? extends Executable> constructor =
                     clazz.getConstructor(String.class, String.class, BitSet.class);
 
-            return constructor.newInstance(name, source, root.getExpressions());
+            return constructor.newInstance(name, source, root.getStatements());
         } catch (Exception exception) { // Catch everything to let the user know this is something caused internally.
             throw new IllegalStateException("An internal error occurred attempting to define the script [" + name + "].", exception);
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Constant.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Constant.java
new file mode 100644
index 00000000000..d42c267fc3f
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Constant.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import java.util.function.Consumer;
+
+/**
+ * A constant initializer to be added to the class file.
+ */
+public class Constant {
+    public final Location location;
+    public final String name;
+    public final org.objectweb.asm.Type type;
+    public final Consumer<MethodWriter> initializer;
+    
+    /**
+     * Create a new constant.
+     *
+     * @param location the location in the script that is creating it
+     * @param type the type of the constant
+     * @param name the name of the constant
+     * @param initializer code to initialize the constant. It will be called when generating the clinit method and is expected to leave the
+     *        value of the constant on the stack. Generating the load instruction is managed by the caller.
+     */
+    public Constant(Location location, org.objectweb.asm.Type type, String name, Consumer<MethodWriter> initializer) {
+        this.location = location;
+        this.name = name;
+        this.type = type;
+        this.initializer = initializer;
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Globals.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Globals.java
new file mode 100644
index 00000000000..83eb74d827f
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Globals.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.elasticsearch.painless.node.SFunction;
+
+import java.util.BitSet;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Program-wide globals (initializers, synthetic methods, etc)
+ */
+public class Globals {
+    private final Map<String,SFunction> syntheticMethods = new HashMap<>();
+    private final Map<String,Constant> constantInitializers = new HashMap<>();
+    private final BitSet statements;
+    
+    /** Create a new Globals from the set of statement boundaries */
+    public Globals(BitSet statements) {
+        this.statements = statements;
+    }
+    
+    /** Adds a new synthetic method to be written. It must be analyzed! */
+    public void addSyntheticMethod(SFunction function) {
+        if (!function.synthetic) {
+            throw new IllegalStateException("method: " + function.name + " is not synthetic");
+        }
+        if (syntheticMethods.put(function.name, function) != null) {
+            throw new IllegalStateException("synthetic method: " + function.name + " already exists");
+        }
+    }
+    
+    /** Adds a new constant initializer to be written */
+    public void addConstantInitializer(Constant constant) {
+        if (constantInitializers.put(constant.name, constant) != null) {
+            throw new IllegalStateException("constant initializer: " + constant.name + " already exists");
+        }
+    }
+    
+    /** Returns the current synthetic methods */
+    public Map<String,SFunction> getSyntheticMethods() {
+        return syntheticMethods;
+    }
+    
+    /** Returns the current initializers */
+    public Map<String,Constant> getConstantInitializers() {
+        return constantInitializers;
+    }
+    
+    /** Returns the set of statement boundaries */
+    public BitSet getStatements() {
+        return statements;
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaLocals.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaLocals.java
new file mode 100644
index 00000000000..9c01bc4c628
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaLocals.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.elasticsearch.painless.Definition.Type;
+
+import java.util.List;
+import java.util.Objects;
+
+/** Extension of locals for lambdas */
+// Note: this isn't functional yet, it throws UOE
+// TODO: implement slot renumbering for captures.
+class LambdaLocals extends Locals {
+    private List<Variable> captures;
+
+    LambdaLocals(Locals parent, List<Parameter> parameters, List<Variable> captures) {
+        super(parent);
+        for (Parameter parameter : parameters) {
+            defineVariable(parameter.location, parameter.type, parameter.name, false);
+        }
+        this.captures = Objects.requireNonNull(captures);
+    }
+    
+    @Override
+    public Variable getVariable(Location location, String name) {
+        Variable variable = lookupVariable(location, name);
+        if (variable != null) {
+            return variable;
+        }
+        if (getParent() != null) {
+            variable = getParent().getVariable(location, name);
+            if (variable != null) {
+                assert captures != null; // unused right now
+                // make it read-only, and record that it was used.
+                throw new UnsupportedOperationException("lambda capture is not supported");
+            }
+        }
+        throw location.createError(new IllegalArgumentException("Variable [" + name + "] is not defined."));
+    }
+
+    @Override
+    public Type getReturnType() {
+        return Definition.DEF_TYPE;
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
index 30f15c42dde..8cb99ed2404 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
@@ -23,141 +23,273 @@ import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.MethodKey;
 import org.elasticsearch.painless.Definition.Type;
 
-import java.util.ArrayDeque;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Deque;
 import java.util.HashMap;
-import java.util.Iterator;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.function.Consumer;
+import java.util.Set;
 
 /**
  * Tracks user defined methods and variables across compilation phases.
  */
-public final class Locals {
+public class Locals {
+    
+    /** Reserved word: params map parameter */
+    public static final String PARAMS = "params";
+    /** Reserved word: Lucene scorer parameter */
+    public static final String SCORER = "#scorer";
+    /** Reserved word: _value variable for aggregations */
+    public static final String VALUE  = "_value";
+    /** Reserved word: _score variable for search scripts */
+    public static final String SCORE  = "_score";
+    /** Reserved word: ctx map for executable scripts */
+    public static final String CTX    = "ctx";
+    /** Reserved word: loop counter */
+    public static final String LOOP   = "#loop";
+    /** Reserved word: unused */
+    public static final String THIS   = "#this";
+    /** Reserved word: unused */
+    public static final String DOC    = "doc";
+    
+    /** Map of always reserved keywords */
+    public static final Set<String> KEYWORDS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
+            THIS,PARAMS,SCORER,DOC,VALUE,SCORE,CTX,LOOP
+    )));
+    
+    /** Creates a new local variable scope (e.g. loop) inside the current scope */
+    public static Locals newLocalScope(Locals currentScope) {
+        return new Locals(currentScope);
+    }
+    
+    /** Creates a new lambda scope inside the current scope */
+    public static Locals newLambdaScope(Locals currentScope, List<Parameter> parameters, List<Variable> captures) {
+        return new LambdaLocals(currentScope, parameters, captures);
+    }
+    
+    /** Creates a new function scope inside the current scope */
+    public static Locals newFunctionScope(Locals programScope, Type returnType, List<Parameter> parameters, int maxLoopCounter) {
+        Locals locals = new Locals(programScope, returnType);
+        for (Parameter parameter : parameters) {
+            locals.defineVariable(parameter.location, parameter.type, parameter.name, false);
+        }
+        // Loop counter to catch infinite loops.  Internal use only.
+        if (maxLoopCounter > 0) {
+            locals.defineVariable(null, Definition.INT_TYPE, LOOP, true);
+        }
+        return locals;
+    }
+    
+    /** Creates a new main method scope */
+    public static Locals newMainMethodScope(Locals programScope, boolean usesScore, boolean usesCtx, int maxLoopCounter) {
+        Locals locals = new Locals(programScope, Definition.OBJECT_TYPE);
+        // This reference.  Internal use only.
+        locals.defineVariable(null, Definition.getType("Object"), THIS, true);
+
+        // Input map of variables passed to the script.
+        locals.defineVariable(null, Definition.getType("Map"), PARAMS, true);
+
+        // Scorer parameter passed to the script.  Internal use only.
+        locals.defineVariable(null, Definition.DEF_TYPE, SCORER, true);
+
+        // Doc parameter passed to the script. TODO: Currently working as a Map, we can do better?
+        locals.defineVariable(null, Definition.getType("Map"), DOC, true);
+
+        // Aggregation _value parameter passed to the script.
+        locals.defineVariable(null, Definition.DEF_TYPE, VALUE, true);
+
+        // Shortcut variables.
+
+        // Document's score as a read-only double.
+        if (usesScore) {
+            locals.defineVariable(null, Definition.DOUBLE_TYPE, SCORE, true);
+        }
+
+        // The ctx map set by executable scripts as a read-only map.
+        if (usesCtx) {
+            locals.defineVariable(null, Definition.getType("Map"), CTX, true);
+        }
+
+        // Loop counter to catch infinite loops.  Internal use only.
+        if (maxLoopCounter > 0) {
+            locals.defineVariable(null, Definition.INT_TYPE, LOOP, true);
+        }
+        return locals;
+    }
+    
+    /** Creates a new program scope: the list of methods. It is the parent for all methods */
+    public static Locals newProgramScope(Collection<Method> methods) {
+        Locals locals = new Locals(null, null);
+        for (Method method : methods) {
+            locals.addMethod(method);
+        }
+        return locals;
+    }
+    
+    /** Checks if a variable exists or not, in this scope or any parents. */
+    public final boolean hasVariable(String name) {
+        Variable variable = lookupVariable(null, name);
+        if (variable != null) {
+            return true;
+        }
+        if (parent != null) {
+            return parent.hasVariable(name);
+        }
+        return false;
+    }
+    
+    /** Accesses a variable. This will throw IAE if the variable does not exist */
+    public Variable getVariable(Location location, String name) {
+        Variable variable = lookupVariable(location, name);
+        if (variable != null) {
+            return variable;
+        }
+        if (parent != null) {
+            return parent.getVariable(location, name);
+        }
+        throw location.createError(new IllegalArgumentException("Variable [" + name + "] is not defined."));
+    }
+    
+    /** Looks up a method. Returns null if the method does not exist. */
+    public final Method getMethod(MethodKey key) {
+        Method method = lookupMethod(key);
+        if (method != null) {
+            return method;
+        }
+        if (parent != null) {
+            return parent.getMethod(key);
+        }
+        return null;
+    }
+    
+    /** Creates a new variable. Throws IAE if the variable has already been defined (even in a parent) or reserved. */
+    public final Variable addVariable(Location location, Type type, String name, boolean readonly) {
+        if (hasVariable(name)) {
+            throw location.createError(new IllegalArgumentException("Variable [" + name + "] is already defined."));
+        }
+        if (KEYWORDS.contains(name)) {
+            throw location.createError(new IllegalArgumentException("Variable [" + name + "] is reserved."));
+        }
+        return defineVariable(location, type, name, readonly);
+    }
+    
+    /** Return type of this scope (e.g. int, if inside a function that returns int) */
+    public Type getReturnType() {
+        return returnType;
+    }
+    
+    /** Returns the top-level program scope. */
+    public Locals getProgramScope() {
+        Locals locals = this;
+        while (locals.getParent() != null) {
+            locals = locals.getParent();
+        }
+        return locals;
+    }
+    
+    ///// private impl
+
+    // parent scope
+    private final Locals parent;
+    // return type of this scope
+    private final Type returnType;
+    // next slot number to assign
+    int nextSlotNumber;
+    // variable name -> variable
+    Map<String,Variable> variables;
+    // method name+arity -> methods
+    Map<MethodKey,Method> methods;
 
     /**
-     * Tracks reserved variables.  Must be given to any source of input
-     * prior to beginning the analysis phase so that reserved variables
-     * are known ahead of time to assign appropriate slots without
-     * being wasteful.
+     * Create a new Locals
      */
-    public interface Reserved {
-        void markReserved(String name);
-        boolean isReserved(String name);
-
-        void setMaxLoopCounter(int max);
-        int getMaxLoopCounter();
+    Locals(Locals parent) {
+        this(parent, parent.getReturnType());
     }
-
-    public static final class ExecuteReserved implements Reserved {
-        public static final String THIS   = "#this";
-        public static final String PARAMS = "params";
-        public static final String SCORER = "#scorer";
-        public static final String DOC    = "doc";
-        public static final String VALUE  = "_value";
-        public static final String SCORE  = "_score";
-        public static final String CTX    = "ctx";
-        public static final String LOOP   = "#loop";
-
-        private boolean score = false;
-        private boolean ctx = false;
-        private int maxLoopCounter = 0;
-
-        @Override
-        public void markReserved(String name) {
-            if (SCORE.equals(name)) {
-                score = true;
-            } else if (CTX.equals(name)) {
-                ctx = true;
-            }
-        }
-
-        @Override
-        public boolean isReserved(String name) {
-            return name.equals(THIS) || name.equals(PARAMS) || name.equals(SCORER) || name.equals(DOC) ||
-                name.equals(VALUE) || name.equals(SCORE) || name.equals(CTX) || name.equals(LOOP);
-        }
-
-        public boolean usesScore() {
-            return score;
-        }
-
-        public boolean usesCtx() {
-            return ctx;
-        }
-
-        @Override
-        public void setMaxLoopCounter(int max) {
-            maxLoopCounter = max;
-        }
-
-        @Override
-        public int getMaxLoopCounter() {
-            return maxLoopCounter;
+    
+    /**
+     * Create a new Locals with specified return type
+     */
+    Locals(Locals parent, Type returnType) {
+        this.parent = parent;
+        this.returnType = returnType;
+        if (parent == null) {
+            this.nextSlotNumber = 0;
+        } else {
+            this.nextSlotNumber = parent.getNextSlot();
         }
     }
 
-    public static final class FunctionReserved implements Reserved {
-        public static final String THIS = "#this";
-        public static final String LOOP = "#loop";
+    /** Returns the parent scope */
+    Locals getParent() {
+        return parent;
+    }
 
-        private int maxLoopCounter = 0;
-
-        public void markReserved(String name) {
-            // Do nothing.
+    /** Looks up a variable at this scope only. Returns null if the variable does not exist. */
+    Variable lookupVariable(Location location, String name) {
+        if (variables == null) {
+            return null;
         }
+        return variables.get(name);
+    }
 
-        public boolean isReserved(String name) {
-            return name.equals(THIS) || name.equals(LOOP);
+    /** Looks up a method at this scope only. Returns null if the method does not exist. */
+    Method lookupMethod(MethodKey key) {
+        if (methods == null) {
+            return null;
         }
+        return methods.get(key);
+    }
 
-        @Override
-        public void setMaxLoopCounter(int max) {
-            maxLoopCounter = max;
+    
+    /** Defines a variable at this scope internally. */
+    Variable defineVariable(Location location, Type type, String name, boolean readonly) {
+        if (variables == null) {
+            variables = new HashMap<>();
         }
+        Variable variable = new Variable(location, name, type, readonly);
+        variable.slot = getNextSlot();
+        variables.put(name, variable); // TODO: check result
+        nextSlotNumber += type.type.getSize();
+        return variable;
+    }
+    
+    // TODO: make private, thats bogus
+    public void addMethod(Method method) {
+        if (methods == null) {
+            methods = new HashMap<>();
+        }
+        methods.put(new MethodKey(method.name, method.arguments.size()), method);
+        // TODO: check result
+    }
 
-        @Override
-        public int getMaxLoopCounter() {
-            return maxLoopCounter;
-        }
+
+    int getNextSlot() {
+        return nextSlotNumber;
     }
 
     public static final class Variable {
         public final Location location;
         public final String name;
         public final Type type;
-        public final int slot;
+        int slot = -1;
         public final boolean readonly;
-
-        public boolean read = false;
-
-        private Variable(Location location, String name, Type type, int slot, boolean readonly) {
+        
+        public Variable(Location location, String name, Type type, boolean readonly) {
             this.location = location;
             this.name = name;
             this.type = type;
-            this.slot = slot;
             this.readonly = readonly;
         }
-    }
-
-    public static final class Constant {
-        public final Location location;
-        public final String name;
-        public final org.objectweb.asm.Type type;
-        public final Consumer<MethodWriter> initializer;
-
-        private Constant(Location location, String name, org.objectweb.asm.Type type, Consumer<MethodWriter> initializer) {
-            this.location = location;
-            this.name = name;
-            this.type = type;
-            this.initializer = initializer;
+        
+        public int getSlot() {
+            return slot;
         }
     }
-
-    public static final class Parameter {
+    
+    public static class Parameter {
         public final Location location;
         public final String name;
         public final Type type;
@@ -168,197 +300,4 @@ public final class Locals {
             this.type = type;
         }
     }
-
-    private final Reserved reserved;
-    private final Map<MethodKey, Method> methods;
-    private final Map<String, Constant> constants;
-    private final Type rtnType;
-
-    // TODO: this datastructure runs in linear time for nearly all operations. use linkedhashset instead?
-    private final Deque<Integer> scopes = new ArrayDeque<>();
-    private final Deque<Variable> variables = new ArrayDeque<>();
-
-    public Locals(ExecuteReserved reserved, Map<MethodKey, Method> methods) {
-        this.reserved = reserved;
-        this.methods = Collections.unmodifiableMap(methods);
-        this.constants = new HashMap<>();
-        this.rtnType = Definition.OBJECT_TYPE;
-
-        incrementScope();
-
-        // Method variables.
-
-        // This reference.  Internal use only.
-        addVariable(null, Definition.getType("Object"), ExecuteReserved.THIS, true, true);
-
-        // Input map of variables passed to the script.
-        addVariable(null, Definition.getType("Map"), ExecuteReserved.PARAMS, true, true);
-
-        // Scorer parameter passed to the script.  Internal use only.
-        addVariable(null, Definition.DEF_TYPE, ExecuteReserved.SCORER, true, true);
-
-        // Doc parameter passed to the script. TODO: Currently working as a Map, we can do better?
-        addVariable(null, Definition.getType("Map"), ExecuteReserved.DOC, true, true);
-
-        // Aggregation _value parameter passed to the script.
-        addVariable(null, Definition.DEF_TYPE, ExecuteReserved.VALUE, true, true);
-
-        // Shortcut variables.
-
-        // Document's score as a read-only double.
-        if (reserved.usesScore()) {
-            addVariable(null, Definition.DOUBLE_TYPE, ExecuteReserved.SCORE, true, true);
-        }
-
-        // The ctx map set by executable scripts as a read-only map.
-        if (reserved.usesCtx()) {
-            addVariable(null, Definition.getType("Map"), ExecuteReserved.CTX, true, true);
-        }
-
-        // Loop counter to catch infinite loops.  Internal use only.
-        if (reserved.getMaxLoopCounter() > 0) {
-            addVariable(null, Definition.INT_TYPE, ExecuteReserved.LOOP, true, true);
-        }
-    }
-
-    public Locals(FunctionReserved reserved, Locals locals, Type rtnType, List<Parameter> parameters) {
-        this.reserved = reserved;
-        this.methods = locals.methods;
-        this.constants = locals.constants;
-        this.rtnType = rtnType;
-
-        incrementScope();
-
-        for (Parameter parameter : parameters) {
-            addVariable(parameter.location, parameter.type, parameter.name, false, false);
-        }
-
-        // Loop counter to catch infinite loops.  Internal use only.
-        if (reserved.getMaxLoopCounter() > 0) {
-            addVariable(null, Definition.INT_TYPE, ExecuteReserved.LOOP, true, true);
-        }
-    }
-
-    public int getMaxLoopCounter() {
-        return reserved.getMaxLoopCounter();
-    }
-
-    public Method getMethod(MethodKey key) {
-        return methods.get(key);
-    }
-
-    public Type getReturnType() {
-        return rtnType;
-    }
-
-    public void incrementScope() {
-        scopes.push(0);
-    }
-
-    public void decrementScope() {
-        int remove = scopes.pop();
-
-        while (remove > 0) {
-            Variable variable = variables.pop();
-
-            // This checks whether or not a variable is used when exiting a local scope.
-            if (variable.read) {
-                throw variable.location.createError(new IllegalArgumentException("Variable [" + variable.name + "] is never used."));
-            }
-
-            --remove;
-        }
-    }
-
-    public Variable getVariable(Location location, String name) {
-         Iterator<Variable> itr = variables.iterator();
-
-        while (itr.hasNext()) {
-             Variable variable = itr.next();
-
-            if (variable.name.equals(name)) {
-                return variable;
-            }
-        }
-
-        throw location.createError(new IllegalArgumentException("Variable [" + name + "] is not defined."));
-    }
-
-    public boolean isVariable(String name) {
-        Iterator<Variable> itr = variables.iterator();
-
-        while (itr.hasNext()) {
-            Variable variable = itr.next();
-
-            if (variable.name.equals(name)) {
-                return true;
-            }
-        }
-
-        return false;
-    }
-
-    public Variable addVariable(Location location, Type type, String name, boolean readonly, boolean reserved) {
-        if (!reserved && this.reserved.isReserved(name)) {
-            throw location.createError(new IllegalArgumentException("Variable [" + name + "] is reserved."));
-        }
-
-        if (isVariable(name)) {
-            throw location.createError(new IllegalArgumentException("Variable [" + name + "] is already defined."));
-        }
-
-        Variable previous = variables.peekFirst();
-        int slot = 0;
-
-        if (previous != null) {
-            slot = previous.slot + previous.type.type.getSize();
-        }
-
-        Variable variable = new Variable(location, name, type, slot, readonly);
-        variables.push(variable);
-
-        int update = scopes.pop() + 1;
-        scopes.push(update);
-
-        return variable;
-    }
-
-    /**
-     * Create a new constant.
-     *
-     * @param location the location in the script that is creating it
-     * @param type the type of the constant
-     * @param name the name of the constant
-     * @param initializer code to initialize the constant. It will be called when generating the clinit method and is expected to leave the
-     *        value of the constant on the stack. Generating the load instruction is managed by the caller.
-     * @return the constant
-     */
-    public Constant addConstant(Location location, org.objectweb.asm.Type type, String name, Consumer<MethodWriter> initializer) {
-        if (constants.containsKey(name)) {
-            throw location.createError(new IllegalArgumentException("Constant [" + name + "] is already defined."));
-        }
-
-        Constant constant = new Constant(location, name, type, initializer);
-        constants.put(name, constant);
-        return constant;
-    }
-
-    /**
-     * Create a new constant.
-     *
-     * @param location the location in the script that is creating it
-     * @param type the type of the constant
-     * @param name the name of the constant
-     * @param initializer code to initialize the constant. It will be called when generating the clinit method and is expected to leave the
-     *        value of the constant on the stack. Generating the load instruction is managed by the caller.
-     * @return the constant
-     */
-    public Constant addConstant(Location location, Type type, String name, Consumer<MethodWriter> initializer) {
-        return addConstant(location, type.type, name, initializer);
-    }
-
-    public Collection<Constant> getConstants() {
-        return constants.values();
-    }
 }
-
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index f3751892b97..b92bd6733c9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -29,11 +29,12 @@ import org.antlr.v4.runtime.Recognizer;
 import org.antlr.v4.runtime.atn.PredictionMode;
 import org.antlr.v4.runtime.tree.TerminalNode;
 import org.elasticsearch.painless.CompilerSettings;
-import org.elasticsearch.painless.Locals.ExecuteReserved;
-import org.elasticsearch.painless.Locals.FunctionReserved;
+import org.elasticsearch.painless.Globals;
+import org.elasticsearch.painless.node.SFunction.Reserved;
+import org.elasticsearch.painless.node.SSource.MainMethodReserved;
+import org.elasticsearch.painless.node.SFunction.FunctionReserved;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Operation;
-import org.elasticsearch.painless.Locals.Reserved;
 import org.elasticsearch.painless.antlr.PainlessParser.AfterthoughtContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ArgumentContext;
 import org.elasticsearch.painless.antlr.PainlessParser.ArgumentsContext;
@@ -114,6 +115,7 @@ import org.elasticsearch.painless.node.EConditional;
 import org.elasticsearch.painless.node.EDecimal;
 import org.elasticsearch.painless.node.EExplicit;
 import org.elasticsearch.painless.node.EFunctionRef;
+import org.elasticsearch.painless.node.ELambda;
 import org.elasticsearch.painless.node.ENull;
 import org.elasticsearch.painless.node.ENumeric;
 import org.elasticsearch.painless.node.EUnary;
@@ -151,6 +153,7 @@ import org.objectweb.asm.util.Printer;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.BitSet;
 import java.util.Deque;
 import java.util.List;
 
@@ -170,7 +173,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
     private final String sourceText;
 
     private final Deque<Reserved> reserved = new ArrayDeque<>();
-    private final List<SFunction> synthetic = new ArrayList<>();
+    private final Globals globals;
     private int syntheticCounter = 0;
 
     private Walker(String sourceName, String sourceText, CompilerSettings settings, Printer debugStream) {
@@ -178,6 +181,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         this.settings = settings;
         this.sourceName = Location.computeSourceName(sourceName, sourceText);
         this.sourceText = sourceText;
+        this.globals = new Globals(new BitSet(sourceText.length()));
         this.source = (SSource)visit(buildAntlrTree(sourceText));
     }
 
@@ -223,7 +227,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
 
     @Override
     public Object visitSource(SourceContext ctx) {
-        reserved.push(new ExecuteReserved());
+        reserved.push(new MainMethodReserved());
 
         List<SFunction> functions = new ArrayList<>();
 
@@ -236,11 +240,9 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         for (StatementContext statement : ctx.statement()) {
             statements.add((AStatement)visit(statement));
         }
-        
-        functions.addAll(synthetic);
 
-        return new SSource(sourceName, sourceText, debugStream, 
-                          (ExecuteReserved)reserved.pop(), location(ctx), functions, statements);
+        return new SSource(sourceName, sourceText, debugStream, (MainMethodReserved)reserved.pop(), 
+                           location(ctx), functions, globals, statements);
     }
 
     @Override
@@ -265,7 +267,8 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
             statements.add((AStatement)visit(statement));
         }
 
-        return new SFunction((FunctionReserved)reserved.pop(), location(ctx), rtnType, name, paramTypes, paramNames, statements, false);
+        return new SFunction((FunctionReserved)reserved.pop(), location(ctx), rtnType, name, 
+                             paramTypes, paramNames, statements, false);
     }
 
     @Override
@@ -977,10 +980,8 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         }
         
         String name = nextLambda();
-        synthetic.add(new SFunction((FunctionReserved)reserved.pop(), location(ctx), "def", name, 
-                      paramTypes, paramNames, statements, true));
-        return new EFunctionRef(location(ctx), "this", name);
-        // TODO: use a real node for captures and shit
+        return new ELambda(name, (FunctionReserved)reserved.pop(), location(ctx), 
+                           paramTypes, paramNames, statements);
     }
 
     @Override
@@ -1021,8 +1022,8 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
                            new EChain(location, 
                            new LVariable(location, "size"))))));
             String name = nextLambda();
-            synthetic.add(new SFunction(new FunctionReserved(), location, arrayType, name, 
-                          Arrays.asList("int"), Arrays.asList("size"), Arrays.asList(code), true));
+            globals.addSyntheticMethod(new SFunction(new FunctionReserved(), location, arrayType, name, 
+                                       Arrays.asList("int"), Arrays.asList("size"), Arrays.asList(code), true));
             return new EFunctionRef(location(ctx), "this", name);
         }
         return new EFunctionRef(location(ctx), ctx.decltype().getText(), ctx.NEW().getText());
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
index fbaaa83a6ad..eb5f73334db 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java
@@ -21,6 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.Locals;
@@ -111,7 +112,7 @@ public abstract class AExpression extends ANode {
     /**
      * Writes ASM based on the data collected during the analysis phase.
      */
-    abstract void write(MethodWriter writer);
+    abstract void write(MethodWriter writer, Globals globals);
 
     /**
      * Inserts {@link ECast} nodes into the tree for implicit casts.  Also replaces
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
index 6d52bfe0de1..f8081d0020b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ALink.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition.Type;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
@@ -91,17 +92,17 @@ public abstract class ALink extends ANode {
     /**
      * Write values before a load/store occurs such as an array index.
      */
-    abstract void write(MethodWriter writer);
+    abstract void write(MethodWriter writer, Globals globals);
 
     /**
      * Write a load for the specific link type.
      */
-    abstract void load(MethodWriter writer);
+    abstract void load(MethodWriter writer, Globals globals);
 
     /**
      * Write a store for the specific link type.
      */
-    abstract void store(MethodWriter writer);
+    abstract void store(MethodWriter writer, Globals globals);
 
     /**
      * Used to copy link data from one to another during analysis in the case of replacement.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
index e34d174ac43..23210472b70 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AStatement.java
@@ -19,7 +19,9 @@
 
 package org.elasticsearch.painless.node;
 
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Locals.Variable;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
@@ -88,7 +90,7 @@ public abstract class AStatement extends ANode {
      * Set to the loop counter variable slot as a shortcut if loop statements
      * are being counted.
      */
-    int loopCounterSlot = -1;
+    Variable loopCounter = null;
 
     /**
      * Set to the approximate number of statements in a loop block to prevent
@@ -120,5 +122,5 @@ public abstract class AStatement extends ANode {
     /**
      * Writes ASM based on the data collected during the analysis phase.
      */
-    abstract void write(MethodWriter writer);
+    abstract void write(MethodWriter writer, Globals globals);
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index d3b92d7eb08..11b77b0441f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -21,6 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Location;
@@ -604,7 +605,7 @@ public final class EBinary extends AExpression {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
         if (promote.sort == Sort.STRING && operation == Operation.ADD) {
@@ -612,13 +613,13 @@ public final class EBinary extends AExpression {
                 writer.writeNewStrings();
             }
 
-            left.write(writer);
+            left.write(writer, globals);
 
             if (!(left instanceof EBinary) || ((EBinary)left).operation != Operation.ADD || left.actual.sort != Sort.STRING) {
                 writer.writeAppendStrings(left.actual);
             }
 
-            right.write(writer);
+            right.write(writer, globals);
 
             if (!(right instanceof EBinary) || ((EBinary)right).operation != Operation.ADD || right.actual.sort != Sort.STRING) {
                 writer.writeAppendStrings(right.actual);
@@ -628,14 +629,14 @@ public final class EBinary extends AExpression {
                 writer.writeToStrings();
             }
         } else if (operation == Operation.FIND) {
-            writeBuildMatcher(writer);
+            writeBuildMatcher(writer, globals);
             writer.invokeVirtual(Definition.MATCHER_TYPE.type, WriterConstants.MATCHER_FIND);
         } else if (operation == Operation.MATCH) {
-            writeBuildMatcher(writer);
+            writeBuildMatcher(writer, globals);
             writer.invokeVirtual(Definition.MATCHER_TYPE.type, WriterConstants.MATCHER_MATCHES);
         } else {
-            left.write(writer);
-            right.write(writer);
+            left.write(writer, globals);
+            right.write(writer, globals);
 
             if (promote.sort == Sort.DEF || (shiftDistance != null && shiftDistance.sort == Sort.DEF)) {
                 writer.writeDynamicBinaryInstruction(location, actual, left.actual, right.actual, operation, false);
@@ -647,9 +648,9 @@ public final class EBinary extends AExpression {
         writer.writeBranch(tru, fals);
     }
 
-    private void writeBuildMatcher(MethodWriter writer) {
-        right.write(writer);
-        left.write(writer);
+    private void writeBuildMatcher(MethodWriter writer, Globals globals) {
+        right.write(writer, globals);
+        left.write(writer, globals);
         writer.invokeVirtual(Definition.PATTERN_TYPE.type, WriterConstants.PATTERN_MATCHER);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
index 5aa2daeeb34..9e39ca712f5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Locals;
@@ -67,7 +68,7 @@ public final class EBool extends AExpression {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         if (tru != null || fals != null) {
             if (operation == Operation.AND) {
                 Label localfals = fals == null ? new Label() : fals;
@@ -76,8 +77,8 @@ public final class EBool extends AExpression {
                 right.tru = tru;
                 right.fals = fals;
 
-                left.write(writer);
-                right.write(writer);
+                left.write(writer, globals);
+                right.write(writer, globals);
 
                 if (fals == null) {
                     writer.mark(localfals);
@@ -89,8 +90,8 @@ public final class EBool extends AExpression {
                 right.tru = tru;
                 right.fals = fals;
 
-                left.write(writer);
-                right.write(writer);
+                left.write(writer, globals);
+                right.write(writer, globals);
 
                 if (tru == null) {
                     writer.mark(localtru);
@@ -106,8 +107,8 @@ public final class EBool extends AExpression {
                 left.fals = localfals;
                 right.fals = localfals;
 
-                left.write(writer);
-                right.write(writer);
+                left.write(writer, globals);
+                right.write(writer, globals);
 
                 writer.push(true);
                 writer.goTo(end);
@@ -122,8 +123,8 @@ public final class EBool extends AExpression {
                 left.tru = localtru;
                 right.fals = localfals;
 
-                left.write(writer);
-                right.write(writer);
+                left.write(writer, globals);
+                right.write(writer, globals);
 
                 writer.mark(localtru);
                 writer.push(true);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
index 877e79549af..a152422fac4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
@@ -41,7 +42,7 @@ public final class EBoolean extends AExpression {
     }
 
     @Override
-    void write(MethodWriter adapter) {
+    void write(MethodWriter adapter, Globals globals) {
         throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
index af4e3779b65..333787db962 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
@@ -22,6 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.FunctionRef;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.Locals;
@@ -37,7 +38,7 @@ import java.lang.invoke.LambdaMetafactory;
 /**
  * Represents a capturing function reference.
  */
-public class ECapturingFunctionRef extends AExpression {
+public class ECapturingFunctionRef extends AExpression implements ILambda {
     public final String type;
     public final String call;
     
@@ -79,21 +80,21 @@ public class ECapturingFunctionRef extends AExpression {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
         if (defPointer != null) {
             // dynamic interface: push captured parameter on stack
             // TODO: don't do this: its just to cutover :)
             writer.push((String)null);
-            writer.visitVarInsn(captured.type.type.getOpcode(Opcodes.ILOAD), captured.slot);
+            writer.visitVarInsn(captured.type.type.getOpcode(Opcodes.ILOAD), captured.getSlot());
         } else if (ref == null) {
             // typed interface, dynamic implementation
-            writer.visitVarInsn(captured.type.type.getOpcode(Opcodes.ILOAD), captured.slot);
+            writer.visitVarInsn(captured.type.type.getOpcode(Opcodes.ILOAD), captured.getSlot());
             String descriptor = Type.getMethodType(expected.type, captured.type.type).getDescriptor();
             writer.invokeDynamic(call, descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.REFERENCE, expected.name);
         } else {
             // typed interface, typed implementation
-            writer.visitVarInsn(captured.type.type.getOpcode(Opcodes.ILOAD), captured.slot);
+            writer.visitVarInsn(captured.type.type.getOpcode(Opcodes.ILOAD), captured.getSlot());
             // convert MethodTypes to asm Type for the constant pool.
             String invokedType = ref.invokedType.toMethodDescriptorString();
             Type samMethodType = Type.getMethodType(ref.samMethodType.toMethodDescriptorString());
@@ -119,4 +120,14 @@ public class ECapturingFunctionRef extends AExpression {
             }
         }
     }
+    
+    @Override
+    public String getPointer() {
+        return defPointer;
+    }
+
+    @Override
+    public Type[] getCaptures() {
+        return new Type[] { captured.type.type };
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
index 7892b918ed1..1073125c637 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
@@ -21,6 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Location;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -50,8 +51,8 @@ final class ECast extends AExpression {
     }
 
     @Override
-    void write(MethodWriter writer) {
-        child.write(writer);
+    void write(MethodWriter writer, Globals globals) {
+        child.write(writer, globals);
         writer.writeDebugInfo(location);
         writer.writeCast(cast);
         writer.writeBranch(tru, fals);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index 3be7f068261..b249cf4511e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
@@ -286,7 +287,7 @@ public final class EChain extends AExpression {
      * 10. call link{[0]}.store(...) -- store the value on the stack into the 0th index of the array x [...]
      */
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
         // For the case where the chain represents a String concatenation
@@ -306,7 +307,7 @@ public final class EChain extends AExpression {
         // See individual links for more information on what each of the
         // write, load, and store methods do.
         for (ALink link : links) {
-            link.write(writer); // call the write method on the link to prepare for a load/store operation
+            link.write(writer, globals); // call the write method on the link to prepare for a load/store operation
 
             if (link == last && link.store) {
                 if (cat) {
@@ -314,10 +315,10 @@ public final class EChain extends AExpression {
                     // representing a String concatenation.
 
                     writer.writeDup(link.size, catElementStackSize);  // dup the top element and insert it before concat helper on stack
-                    link.load(writer);                     // read the current link's value
+                    link.load(writer, globals);                       // read the current link's value
                     writer.writeAppendStrings(link.after); // append the link's value using the StringBuilder
 
-                    expression.write(writer); // write the bytecode for the rhs expression
+                    expression.write(writer, globals); // write the bytecode for the rhs expression
 
                     if (!(expression instanceof EBinary) ||
                         ((EBinary)expression).operation != Operation.ADD || expression.actual.sort != Sort.STRING) {
@@ -331,13 +332,13 @@ public final class EChain extends AExpression {
                         writer.writeDup(link.after.sort.size, link.size); // if this link is also read from dup the value onto the stack
                     }
 
-                    link.store(writer); // store the link's value from the stack in its respective variable/field/array
+                    link.store(writer, globals); // store the link's value from the stack in its respective variable/field/array
                 } else if (operation != null) {
                     // Handle the case where we are doing a compound assignment that
                     // does not represent a String concatenation.
 
                     writer.writeDup(link.size, 0); // if necessary, dup the previous link's value to be both loaded from and stored to
-                    link.load(writer);             // load the current link's value
+                    link.load(writer, globals);             // load the current link's value
 
                     if (link.load && post) {
                         writer.writeDup(link.after.sort.size, link.size); // dup the value if the link is also
@@ -346,7 +347,7 @@ public final class EChain extends AExpression {
 
                     writer.writeCast(there);                                     // if necessary cast the current link's value
                                                                                  // to the promotion type between the lhs and rhs types
-                    expression.write(writer);                                    // write the bytecode for the rhs expression
+                    expression.write(writer, globals);                           // write the bytecode for the rhs expression
                     // XXX: fix these types, but first we need def compound assignment tests.
                     // its tricky here as there are possibly explicit casts, too.
                     // write the operation instruction for compound assignment
@@ -364,22 +365,22 @@ public final class EChain extends AExpression {
                                                                           // read from and is not a post increment
                     }
 
-                    link.store(writer); // store the link's value from the stack in its respective variable/field/array
+                    link.store(writer, globals); // store the link's value from the stack in its respective variable/field/array
                 } else {
                     // Handle the case for a simple write.
 
-                    expression.write(writer); // write the bytecode for the rhs expression
+                    expression.write(writer, globals); // write the bytecode for the rhs expression
 
                     if (link.load) {
                         writer.writeDup(link.after.sort.size, link.size); // dup the value if the link is also read from
                     }
 
-                    link.store(writer); // store the link's value from the stack in its respective variable/field/array
+                    link.store(writer, globals); // store the link's value from the stack in its respective variable/field/array
                 }
             } else {
                 // Handle the case for a simple read.
 
-                link.load(writer); // read the link's value onto the stack
+                link.load(writer, globals); // read the link's value onto the stack
             }
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index 03c6351e95c..53684a19005 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Location;
@@ -436,15 +437,15 @@ public final class EComp extends AExpression {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
         boolean branch = tru != null || fals != null;
 
-        left.write(writer);
+        left.write(writer, globals);
 
         if (!right.isNull) {
-            right.write(writer);
+            right.write(writer, globals);
         }
 
         Label jump = tru != null ? tru : fals != null ? fals : new Label();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
index 024e0c800fc..600626348dd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.AnalyzerCaster;
@@ -78,7 +79,7 @@ public final class EConditional extends AExpression {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
         Label localfals = new Label();
@@ -88,11 +89,11 @@ public final class EConditional extends AExpression {
         left.tru = right.tru = tru;
         left.fals = right.fals = fals;
 
-        condition.write(writer);
-        left.write(writer);
+        condition.write(writer, globals);
+        left.write(writer, globals);
         writer.goTo(end);
         writer.mark(localfals);
-        right.write(writer);
+        right.write(writer, globals);
         writer.mark(end);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
index 267d32983b9..b037d25c2d6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
@@ -63,7 +64,7 @@ final class EConstant extends AExpression {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         Sort sort = actual.sort;
 
         switch (sort) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
index 0fd7fe46b51..efa453605cb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
@@ -57,7 +58,7 @@ public final class EDecimal extends AExpression {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
index 007a5d59e59..1c186faaeeb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
@@ -54,7 +55,7 @@ public final class EExplicit extends AExpression {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         throw createError(new IllegalStateException("Illegal tree structure."));
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index 7ab7703d02e..856a876550c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -21,6 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.FunctionRef;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.Definition.Method;
@@ -35,7 +36,7 @@ import java.lang.invoke.LambdaMetafactory;
 /**
  * Represents a function reference.
  */
-public class EFunctionRef extends AExpression {
+public class EFunctionRef extends AExpression implements ILambda {
     public final String type;
     public final String call;
 
@@ -83,7 +84,7 @@ public class EFunctionRef extends AExpression {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         if (ref != null) {
             writer.writeDebugInfo(location);
             // convert MethodTypes to asm Type for the constant pool.
@@ -114,4 +115,14 @@ public class EFunctionRef extends AExpression {
             writer.push((String)null);
         }
     }
+
+    @Override
+    public String getPointer() {
+        return defPointer;
+    }
+
+    @Override
+    public Type[] getCaptures() {
+        return new Type[0]; // no captures
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
index b513e3813c4..d0b531c74d3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
@@ -20,23 +20,33 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Locals;
-import org.elasticsearch.painless.Locals.FunctionReserved;
+import org.elasticsearch.painless.Locals.Variable;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.node.SFunction.FunctionReserved;
+import org.elasticsearch.painless.Globals;
+import org.objectweb.asm.Type;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
-public class ELambda extends AExpression {
+public class ELambda extends AExpression implements ILambda {
+    final String name;
     final FunctionReserved reserved;
     final List<String> paramTypeStrs;
     final List<String> paramNameStrs;
     final List<AStatement> statements;
+    // desugared synthetic method (lambda body)
+    SFunction desugared;
+    // method ref (impl detail)
+    ILambda impl;
 
-    public ELambda(FunctionReserved reserved, Location location,
-                   List<String> paramTypes, List<String> paramNames, List<AStatement> statements) {
+    public ELambda(String name, FunctionReserved reserved, 
+                   Location location, List<String> paramTypes, List<String> paramNames, 
+                   List<AStatement> statements) {
         super(location);
-
+        this.name = name;
         this.reserved = reserved;
         this.paramTypeStrs = Collections.unmodifiableList(paramTypes);
         this.paramNameStrs = Collections.unmodifiableList(paramNames);
@@ -45,11 +55,40 @@ public class ELambda extends AExpression {
 
     @Override
     void analyze(Locals locals) {
-        throw createError(new UnsupportedOperationException("Lambda functions are not supported."));
+        // desugar lambda body into a synthetic method
+        desugared = new SFunction(reserved, location, "def", name, 
+                                            paramTypeStrs, paramNameStrs, statements, true);
+        desugared.generate();
+        List<Variable> captures = new ArrayList<>();
+        desugared.analyze(Locals.newLambdaScope(locals.getProgramScope(), desugared.parameters, captures));
+        
+        // setup reference
+        EFunctionRef ref = new EFunctionRef(location, "this", name);
+        ref.expected = expected;
+        // hack, create a new scope, with our method, so the ref can see it (impl detail)
+        locals = Locals.newLocalScope(locals);
+        locals.addMethod(desugared.method);
+        ref.analyze(locals);
+        actual = ref.actual;
+        impl = ref;
     }
 
     @Override
-    void write(MethodWriter writer) {
-        throw createError(new IllegalStateException("Illegal tree structure."));
+    void write(MethodWriter writer, Globals globals) {
+        AExpression expr = (AExpression) impl;
+        expr.write(writer, globals);
+        // add synthetic method to the queue to be written
+        globals.addSyntheticMethod(desugared);
     }
+
+    @Override
+    public String getPointer() {
+        return impl.getPointer();
+    }
+
+    @Override
+    public Type[] getCaptures() {
+        return impl.getCaptures();
+    }
+    
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
index eefc09c5946..0a1242e6507 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Opcodes;
@@ -50,7 +51,7 @@ public final class ENull extends AExpression {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         writer.visitInsn(Opcodes.ACONST_NULL);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
index 9abdc8f6a12..e2314880448 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Locals;
@@ -96,7 +97,7 @@ public final class ENumeric extends AExpression {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index f4f73e6101b..f74b91dd5bc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
@@ -177,7 +178,7 @@ public final class EUnary extends AExpression {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
         if (operation == Operation.NOT) {
@@ -186,7 +187,7 @@ public final class EUnary extends AExpression {
                 Label end = new Label();
 
                 child.fals = localfals;
-                child.write(writer);
+                child.write(writer, globals);
 
                 writer.push(false);
                 writer.goTo(end);
@@ -196,11 +197,11 @@ public final class EUnary extends AExpression {
             } else {
                 child.tru = fals;
                 child.fals = tru;
-                child.write(writer);
+                child.write(writer, globals);
             }
         } else {
             Sort sort = promote.sort;
-            child.write(writer);
+            child.write(writer, globals);
 
             if (operation == Operation.BWNOT) {
                 if (sort == Sort.DEF) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ILambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ILambda.java
new file mode 100644
index 00000000000..5a279b3a162
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ILambda.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+/** 
+ * Interface for lambda/method reference nodes. They need special handling by LDefCall.
+ * <p>
+ * This is because they know nothing about the target interface, and can only push
+ * all their captures onto the stack and defer everything until link-time.
+ */
+interface ILambda {
+
+    /** Returns reference to resolve at link-time */
+    String getPointer();
+
+    /** Returns the types of captured parameters. Can be empty */
+    org.objectweb.asm.Type[] getCaptures();
+
+    /** Returns the number of captured parameters */
+    default int getCaptureCount() {
+        return getCaptures().length;
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
index 38c1ae43907..1a240747d73 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
@@ -55,18 +56,18 @@ public final class LArrayLength extends ALink {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         // Do nothing.
     }
 
     @Override
-    void load(MethodWriter writer) {
+    void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
         writer.arrayLength();
     }
 
     @Override
-    void store(MethodWriter writer) {
+    void store(MethodWriter writer, Globals globals) {
         throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
index 91346b7a208..e8c23bf3857 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Locals;
@@ -69,18 +70,18 @@ public final class LBrace extends ALink {
     }
 
     @Override
-    void write(MethodWriter writer) {
-        index.write(writer);
+    void write(MethodWriter writer, Globals globals) {
+        index.write(writer, globals);
     }
 
     @Override
-    void load(MethodWriter writer) {
+    void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
         writer.arrayLoad(after.type);
     }
 
     @Override
-    void store(MethodWriter writer) {
+    void store(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
         writer.arrayStore(after.type);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallInvoke.java
index 1f6e899b1dd..34ad0343aa9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallInvoke.java
@@ -24,6 +24,7 @@ import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Struct;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -86,16 +87,16 @@ public final class LCallInvoke extends ALink {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         // Do nothing.
     }
 
     @Override
-    void load(MethodWriter writer) {
+    void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
         for (AExpression argument : arguments) {
-            argument.write(writer);
+            argument.write(writer, globals);
         }
 
         if (java.lang.reflect.Modifier.isStatic(method.modifiers)) {
@@ -108,7 +109,7 @@ public final class LCallInvoke extends ALink {
     }
 
     @Override
-    void store(MethodWriter writer) {
+    void store(MethodWriter writer, Globals globals) {
         throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallLocal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallLocal.java
index 31085171bf6..4503692b48d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallLocal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallLocal.java
@@ -21,6 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.MethodKey;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
@@ -77,23 +78,23 @@ public class LCallLocal extends ALink {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         // Do nothing.
     }
 
     @Override
-    void load(MethodWriter writer) {
+    void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
         for (AExpression argument : arguments) {
-            argument.write(writer);
+            argument.write(writer, globals);
         }
 
         writer.invokeStatic(CLASS_TYPE, method.method);
     }
 
     @Override
-    void store(MethodWriter writer) {
+    void store(MethodWriter writer, Globals globals) {
         throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
index d2b4f83a823..bf900cedafc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.AnalyzerCaster;
@@ -61,18 +62,18 @@ public final class LCast extends ALink {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
         writer.writeCast(cast);
     }
 
     @Override
-    void load(MethodWriter writer) {
+    void load(MethodWriter writer, Globals globals) {
         // Do nothing.
     }
 
     @Override
-    void store(MethodWriter writer) {
+    void store(MethodWriter writer, Globals globals) {
         throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index 5a5333aabe8..02fbb2000f2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Locals;
@@ -53,12 +54,12 @@ final class LDefArray extends ALink implements IDefLink {
     }
 
     @Override
-    void write(MethodWriter writer) {
-        index.write(writer);
+    void write(MethodWriter writer, Globals globals) {
+        index.write(writer, globals);
     }
 
     @Override
-    void load(MethodWriter writer) {
+    void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
         String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type, index.actual.type);
@@ -66,7 +67,7 @@ final class LDefArray extends ALink implements IDefLink {
     }
 
     @Override
-    void store(MethodWriter writer) {
+    void store(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
         String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, index.actual.type, after.type);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index a2c2f150840..716b26d6687 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -20,10 +20,12 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
+import org.objectweb.asm.Type;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -64,13 +66,11 @@ final class LDefCall extends ALink implements IDefLink {
             expression.internal = true;
             expression.analyze(locals);
 
-            if (expression instanceof EFunctionRef) {
-                pointers.add(((EFunctionRef)expression).defPointer);
+            if (expression instanceof ILambda) {
+                ILambda lambda = (ILambda) expression;
+                pointers.add(lambda.getPointer());
                 recipe |= (1L << (argument + totalCaptures)); // mark argument as deferred reference
-            } else if (expression instanceof ECapturingFunctionRef) {
-                pointers.add(((ECapturingFunctionRef)expression).defPointer);
-                recipe |= (1L << (argument + totalCaptures)); // mark argument as deferred reference
-                totalCaptures++;
+                totalCaptures += lambda.getCaptureCount();
             }
 
             expression.expected = expression.actual;
@@ -84,12 +84,12 @@ final class LDefCall extends ALink implements IDefLink {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         // Do nothing.
     }
 
     @Override
-    void load(MethodWriter writer) {
+    void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
         StringBuilder signature = new StringBuilder();
@@ -100,11 +100,13 @@ final class LDefCall extends ALink implements IDefLink {
 
         for (AExpression argument : arguments) {
             signature.append(argument.actual.type.getDescriptor());
-            if (argument instanceof ECapturingFunctionRef) {
-                ECapturingFunctionRef capturingRef = (ECapturingFunctionRef) argument;
-                signature.append(capturingRef.captured.type.type.getDescriptor());
+            if (argument instanceof ILambda) {
+                ILambda lambda = (ILambda) argument;
+                for (Type capture : lambda.getCaptures()) {
+                    signature.append(capture.getDescriptor());
+                }
             }
-            argument.write(writer);
+            argument.write(writer, globals);
         }
 
         signature.append(')');
@@ -119,7 +121,7 @@ final class LDefCall extends ALink implements IDefLink {
     }
 
     @Override
-    void store(MethodWriter writer) {
+    void store(MethodWriter writer, Globals globals) {
         throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index 09b48a94f86..125b2bee6fc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Locals;
@@ -50,12 +51,12 @@ final class LDefField extends ALink implements IDefLink {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         // Do nothing.
     }
 
     @Override
-    void load(MethodWriter writer) {
+    void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
         String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type);
@@ -63,7 +64,7 @@ final class LDefField extends ALink implements IDefLink {
     }
 
     @Override
-    void store(MethodWriter writer) {
+    void store(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
         String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, after.type);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
index 049dd2d8524..9fc48c009a2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Field;
 import org.elasticsearch.painless.Definition.Sort;
@@ -100,12 +101,12 @@ public final class LField extends ALink {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         // Do nothing.
     }
 
     @Override
-    void load(MethodWriter writer) {
+    void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
         if (java.lang.reflect.Modifier.isStatic(field.modifiers)) {
@@ -116,7 +117,7 @@ public final class LField extends ALink {
     }
 
     @Override
-    void store(MethodWriter writer) {
+    void store(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
         if (java.lang.reflect.Modifier.isStatic(field.modifiers)) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
index 7b0feb094e1..1cd5620cc17 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
@@ -74,12 +75,12 @@ final class LListShortcut extends ALink {
     }
 
     @Override
-    void write(MethodWriter writer) {
-        index.write(writer);
+    void write(MethodWriter writer, Globals globals) {
+        index.write(writer, globals);
     }
 
     @Override
-    void load(MethodWriter writer) {
+    void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
@@ -94,7 +95,7 @@ final class LListShortcut extends ALink {
     }
 
     @Override
-    void store(MethodWriter writer) {
+    void store(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
index 5d4c5a9e50a..05c2192b870 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
@@ -73,12 +74,12 @@ final class LMapShortcut extends ALink {
     }
 
     @Override
-    void write(MethodWriter writer) {
-        index.write(writer);
+    void write(MethodWriter writer, Globals globals) {
+        index.write(writer, globals);
     }
 
     @Override
-    void load(MethodWriter writer) {
+    void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
@@ -93,7 +94,7 @@ final class LMapShortcut extends ALink {
     }
 
     @Override
-    void store(MethodWriter writer) {
+    void store(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
index 15aa01c1d26..312cbf925e1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Locals;
@@ -74,16 +75,16 @@ public final class LNewArray extends ALink {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         // Do nothing.
     }
 
     @Override
-    void load(MethodWriter writer) {
+    void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
         for (AExpression argument : arguments) {
-            argument.write(writer);
+            argument.write(writer, globals);
         }
 
         if (arguments.size() > 1) {
@@ -94,7 +95,7 @@ public final class LNewArray extends ALink {
     }
 
     @Override
-    void store(MethodWriter writer) {
+    void store(MethodWriter writer, Globals globals) {
         throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
index aeb6f64f9db..0b28a308e19 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Struct;
@@ -93,12 +94,12 @@ public final class LNewObj extends ALink {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         // Do nothing.
     }
 
     @Override
-    void load(MethodWriter writer) {
+    void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
         writer.newInstance(after.type);
 
@@ -107,14 +108,14 @@ public final class LNewObj extends ALink {
         }
 
         for (AExpression argument : arguments) {
-            argument.write(writer);
+            argument.write(writer, globals);
         }
 
         writer.invokeConstructor(constructor.owner.type, constructor.method);
     }
 
     @Override
-    void store(MethodWriter writer) {
+    void store(MethodWriter writer, Globals globals) {
         throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LRegex.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LRegex.java
index 324b6d72cf1..8727f992242 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LRegex.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LRegex.java
@@ -19,9 +19,10 @@
 
 package org.elasticsearch.painless.node;
 
+import org.elasticsearch.painless.Constant;
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
-import org.elasticsearch.painless.Locals.Constant;
 
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
@@ -64,25 +65,26 @@ public final class LRegex extends ALink {
             throw createError(new IllegalArgumentException("Regex constant may only be read [" + pattern + "]."));
         }
 
-        constant = locals.addConstant(location, Definition.PATTERN_TYPE, "regexAt$" + location.getOffset(), this::initializeConstant);
+        constant = new Constant(location, Definition.PATTERN_TYPE.type, "regexAt$" + location.getOffset(), this::initializeConstant);
         after = Definition.PATTERN_TYPE;
 
         return this;
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         // Do nothing.
     }
 
     @Override
-    void load(MethodWriter writer) {
+    void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
         writer.getStatic(WriterConstants.CLASS_TYPE, constant.name, Definition.PATTERN_TYPE.type);
+        globals.addConstantInitializer(constant);
     }
 
     @Override
-    void store(MethodWriter writer) {
+    void store(MethodWriter writer, Globals globals) {
         throw createError(new IllegalStateException("Illegal tree structure."));
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
index 73cbc201db4..efaf02bb601 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
@@ -79,12 +80,12 @@ final class LShortcut extends ALink {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         // Do nothing.
     }
 
     @Override
-    void load(MethodWriter writer) {
+    void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
         if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
@@ -99,7 +100,7 @@ final class LShortcut extends ALink {
     }
 
     @Override
-    void store(MethodWriter writer) {
+    void store(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
         if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java
index 6f04e9a22c4..3a7c9a35fe5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.Locals;
@@ -54,17 +55,17 @@ public final class LStatic extends ALink {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         throw createError(new IllegalStateException("Illegal tree structure."));
     }
 
     @Override
-    void load(MethodWriter writer) {
+    void load(MethodWriter writer, Globals globals) {
         throw createError(new IllegalStateException("Illegal tree structure."));
     }
 
     @Override
-    void store(MethodWriter writer) {
+    void store(MethodWriter writer, Globals globals) {
         throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
index 41eb027e3ab..fac9d413ad7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
@@ -51,17 +52,17 @@ public final class LString extends ALink {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         // Do nothing.
     }
 
     @Override
-    void load(MethodWriter writer) {
+    void load(MethodWriter writer, Globals globals) {
         writer.push(string);
     }
 
     @Override
-    void store(MethodWriter writer) {
+    void store(MethodWriter writer, Globals globals) {
         throw createError(new IllegalStateException("Illegal tree structure."));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
index 64dc852117d..ad9ba9dfbf6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
@@ -21,6 +21,7 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Locals.Variable;
 import org.objectweb.asm.Opcodes;
@@ -32,7 +33,7 @@ public final class LVariable extends ALink {
 
     final String name;
 
-    int slot;
+    Variable variable;
 
     public LVariable(Location location, String name) {
         super(location, 0);
@@ -46,30 +47,29 @@ public final class LVariable extends ALink {
             throw createError(new IllegalArgumentException("Illegal variable [" + name + "] access with target already defined."));
         }
 
-        Variable variable = locals.getVariable(location, name);
+        variable = locals.getVariable(location, name);
 
         if (store && variable.readonly) {
             throw createError(new IllegalArgumentException("Variable [" + variable.name + "] is read-only."));
         }
 
-        slot = variable.slot;
         after = variable.type;
 
         return this;
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         // Do nothing.
     }
 
     @Override
-    void load(MethodWriter writer) {
-        writer.visitVarInsn(after.type.getOpcode(Opcodes.ILOAD), slot);
+    void load(MethodWriter writer, Globals globals) {
+        writer.visitVarInsn(after.type.getOpcode(Opcodes.ILOAD), variable.getSlot());
     }
 
     @Override
-    void store(MethodWriter writer) {
-        writer.visitVarInsn(after.type.getOpcode(Opcodes.ISTORE), slot);
+    void store(MethodWriter writer, Globals globals) {
+        writer.visitVarInsn(after.type.getOpcode(Opcodes.ISTORE), variable.getSlot());
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
index 5fec362ec17..7d459f27504 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.painless.node;
 
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
@@ -70,11 +71,11 @@ public final class SBlock extends AStatement {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         for (AStatement statement : statements) {
             statement.continu = continu;
             statement.brake = brake;
-            statement.write(writer);
+            statement.write(writer, globals);
         }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
index d4f8bfff4e4..c791faeadd6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.painless.node;
 
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
@@ -45,7 +46,7 @@ public final class SBreak extends AStatement {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         writer.goTo(brake);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
index 42de28e48f5..8632c0eea6d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Locals;
@@ -65,7 +66,7 @@ public final class SCatch extends AStatement {
             throw createError(new ClassCastException("Not an exception type [" + this.type + "]."));
         }
 
-        variable = locals.addVariable(location, type, name, true, false);
+        variable = locals.addVariable(location, type, name, true);
 
         if (block != null) {
             block.lastSource = lastSource;
@@ -84,18 +85,18 @@ public final class SCatch extends AStatement {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         writer.writeStatementOffset(location);
 
         Label jump = new Label();
 
         writer.mark(jump);
-        writer.visitVarInsn(variable.type.type.getOpcode(Opcodes.ISTORE), variable.slot);
+        writer.visitVarInsn(variable.type.type.getOpcode(Opcodes.ISTORE), variable.getSlot());
 
         if (block != null) {
             block.continu = continu;
             block.brake = brake;
-            block.write(writer);
+            block.write(writer, globals);
         }
 
         writer.visitTryCatchBlock(begin, end, jump, variable.type.type.getInternalName());
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
index 802cf90087b..e70f625bb3d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.painless.node;
 
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
@@ -48,7 +49,7 @@ public final class SContinue extends AStatement {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         writer.goTo(continu);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
index 5365e72888a..58ff09e45d8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.painless.node;
 
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
@@ -49,9 +50,9 @@ public final class SDeclBlock extends AStatement {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         for (AStatement declaration : declarations) {
-            declaration.write(writer);
+            declaration.write(writer, globals);
         }
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
index a62a287df9c..c19adfd699e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Locals;
@@ -62,11 +63,11 @@ public final class SDeclaration extends AStatement {
             expression = expression.cast(locals);
         }
 
-        variable = locals.addVariable(location, type, name, false, false);
+        variable = locals.addVariable(location, type, name, false);
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         writer.writeStatementOffset(location);
 
         if (expression == null) {
@@ -83,9 +84,9 @@ public final class SDeclaration extends AStatement {
                 default:     writer.visitInsn(Opcodes.ACONST_NULL);
             }
         } else {
-            expression.write(writer);
+            expression.write(writer, globals);
         }
 
-        writer.visitVarInsn(variable.type.type.getOpcode(Opcodes.ISTORE), variable.slot);
+        writer.visitVarInsn(variable.type.type.getOpcode(Opcodes.ISTORE), variable.getSlot());
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
index b25d903dc41..3b94f5799ea 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
@@ -42,7 +43,7 @@ public final class SDo extends AStatement {
 
     @Override
     void analyze(Locals locals) {
-        locals.incrementScope();
+        locals = Locals.newLocalScope(locals);
 
         if (block == null) {
             throw createError(new IllegalArgumentException("Extraneous do while loop."));
@@ -76,15 +77,13 @@ public final class SDo extends AStatement {
 
         statementCount = 1;
 
-        if (locals.getMaxLoopCounter() > 0) {
-            loopCounterSlot = locals.getVariable(location, "#loop").slot;
+        if (locals.hasVariable(Locals.LOOP)) {
+            loopCounter = locals.getVariable(location, Locals.LOOP);
         }
-
-        locals.decrementScope();
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         writer.writeStatementOffset(location);
 
         Label start = new Label();
@@ -95,14 +94,14 @@ public final class SDo extends AStatement {
 
         block.continu = begin;
         block.brake = end;
-        block.write(writer);
+        block.write(writer, globals);
 
         writer.mark(begin);
 
         condition.fals = end;
-        condition.write(writer);
+        condition.write(writer, globals);
 
-        writer.writeLoopCounter(loopCounterSlot, Math.max(1, block.statementCount), location);
+        writer.writeLoopCounter(loopCounter.getSlot(), Math.max(1, block.statementCount), location);
 
         writer.goTo(start);
         writer.mark(end);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index 48a3f6e3eae..403b377ea9c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -22,6 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Definition.Cast;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.MethodKey;
@@ -85,9 +86,9 @@ public class SEach extends AStatement {
             throw createError(new IllegalArgumentException("Not a type [" + this.type + "]."));
         }
 
-        locals.incrementScope();
+        locals = Locals.newLocalScope(locals);
 
-        variable = locals.addVariable(location, type, name, true, false);
+        variable = locals.addVariable(location, type, name, true);
 
         if (expression.actual.sort == Sort.ARRAY) {
             analyzeArray(locals, type);
@@ -112,18 +113,16 @@ public class SEach extends AStatement {
 
         statementCount = 1;
 
-        if (locals.getMaxLoopCounter() > 0) {
-            loopCounterSlot = locals.getVariable(location, "#loop").slot;
+        if (locals.hasVariable(Locals.LOOP)) {
+            loopCounter = locals.getVariable(location, Locals.LOOP);
         }
-
-        locals.decrementScope();
     }
 
     void analyzeArray(Locals variables, Type type) {
         // We must store the array and index as variables for securing slots on the stack, and
         // also add the location offset to make the names unique in case of nested for each loops.
-        array = variables.addVariable(location, expression.actual, "#array" + location.getOffset(), true, false);
-        index = variables.addVariable(location, Definition.INT_TYPE, "#index" + location.getOffset(), true, false);
+        array = variables.addVariable(location, expression.actual, "#array" + location.getOffset(), true);
+        index = variables.addVariable(location, Definition.INT_TYPE, "#index" + location.getOffset(), true);
         indexed = Definition.getType(expression.actual.struct, expression.actual.dimensions - 1);
         cast = AnalyzerCaster.getLegalCast(location, indexed, type, true, true);
     }
@@ -131,7 +130,7 @@ public class SEach extends AStatement {
     void analyzeIterable(Locals variables, Type type) {
         // We must store the iterator as a variable for securing a slot on the stack, and
         // also add the location offset to make the name unique in case of nested for each loops.
-        iterator = variables.addVariable(location, Definition.getType("Iterator"), "#itr" + location.getOffset(), true, false);
+        iterator = variables.addVariable(location, Definition.getType("Iterator"), "#itr" + location.getOffset(), true);
 
         if (expression.actual.sort == Sort.DEF) {
             method = null;
@@ -148,49 +147,49 @@ public class SEach extends AStatement {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         writer.writeStatementOffset(location);
 
         if (array != null) {
-            writeArray(writer);
+            writeArray(writer, globals);
         } else if (iterator != null) {
-            writeIterable(writer);
+            writeIterable(writer, globals);
         } else {
             throw createError(new IllegalStateException("Illegal tree structure."));
         }
     }
 
-    void writeArray(MethodWriter writer) {
-        expression.write(writer);
-        writer.visitVarInsn(array.type.type.getOpcode(Opcodes.ISTORE), array.slot);
+    void writeArray(MethodWriter writer, Globals globals) {
+        expression.write(writer, globals);
+        writer.visitVarInsn(array.type.type.getOpcode(Opcodes.ISTORE), array.getSlot());
         writer.push(-1);
-        writer.visitVarInsn(index.type.type.getOpcode(Opcodes.ISTORE), index.slot);
+        writer.visitVarInsn(index.type.type.getOpcode(Opcodes.ISTORE), index.getSlot());
 
         Label begin = new Label();
         Label end = new Label();
 
         writer.mark(begin);
 
-        writer.visitIincInsn(index.slot, 1);
-        writer.visitVarInsn(index.type.type.getOpcode(Opcodes.ILOAD), index.slot);
-        writer.visitVarInsn(array.type.type.getOpcode(Opcodes.ILOAD), array.slot);
+        writer.visitIincInsn(index.getSlot(), 1);
+        writer.visitVarInsn(index.type.type.getOpcode(Opcodes.ILOAD), index.getSlot());
+        writer.visitVarInsn(array.type.type.getOpcode(Opcodes.ILOAD), array.getSlot());
         writer.arrayLength();
         writer.ifICmp(MethodWriter.GE, end);
 
-        writer.visitVarInsn(array.type.type.getOpcode(Opcodes.ILOAD), array.slot);
-        writer.visitVarInsn(index.type.type.getOpcode(Opcodes.ILOAD), index.slot);
+        writer.visitVarInsn(array.type.type.getOpcode(Opcodes.ILOAD), array.getSlot());
+        writer.visitVarInsn(index.type.type.getOpcode(Opcodes.ILOAD), index.getSlot());
         writer.arrayLoad(indexed.type);
         writer.writeCast(cast);
-        writer.visitVarInsn(variable.type.type.getOpcode(Opcodes.ISTORE), variable.slot);
+        writer.visitVarInsn(variable.type.type.getOpcode(Opcodes.ISTORE), variable.getSlot());
 
-        block.write(writer);
+        block.write(writer, globals);
 
         writer.goTo(begin);
         writer.mark(end);
     }
 
-    void writeIterable(MethodWriter writer) {
-        expression.write(writer);
+    void writeIterable(MethodWriter writer, Globals globals) {
+        expression.write(writer, globals);
 
         if (method == null) {
             Type itr = Definition.getType("Iterator");
@@ -202,23 +201,23 @@ public class SEach extends AStatement {
             writer.invokeVirtual(method.owner.type, method.method);
         }
 
-        writer.visitVarInsn(iterator.type.type.getOpcode(Opcodes.ISTORE), iterator.slot);
+        writer.visitVarInsn(iterator.type.type.getOpcode(Opcodes.ISTORE), iterator.getSlot());
 
         Label begin = new Label();
         Label end = new Label();
 
         writer.mark(begin);
 
-        writer.visitVarInsn(iterator.type.type.getOpcode(Opcodes.ILOAD), iterator.slot);
+        writer.visitVarInsn(iterator.type.type.getOpcode(Opcodes.ILOAD), iterator.getSlot());
         writer.invokeInterface(ITERATOR_TYPE, ITERATOR_HASNEXT);
         writer.ifZCmp(MethodWriter.EQ, end);
 
-        writer.visitVarInsn(iterator.type.type.getOpcode(Opcodes.ILOAD), iterator.slot);
+        writer.visitVarInsn(iterator.type.type.getOpcode(Opcodes.ILOAD), iterator.getSlot());
         writer.invokeInterface(ITERATOR_TYPE, ITERATOR_NEXT);
         writer.writeCast(cast);
-        writer.visitVarInsn(variable.type.type.getOpcode(Opcodes.ISTORE), variable.slot);
+        writer.visitVarInsn(variable.type.type.getOpcode(Opcodes.ISTORE), variable.getSlot());
 
-        block.write(writer);
+        block.write(writer, globals);
 
         writer.goTo(begin);
         writer.mark(end);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index 8817db95be6..c6cf0a47c8f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -22,6 +22,7 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Sort;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -63,9 +64,9 @@ public final class SExpression extends AStatement {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         writer.writeStatementOffset(location);
-        expression.write(writer);
+        expression.write(writer, globals);
 
         if (methodEscape) {
             writer.returnValue();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index cc9dee20138..b2cf1da2594 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
@@ -46,7 +47,7 @@ public final class SFor extends AStatement {
 
     @Override
     void analyze(Locals locals) {
-        locals.incrementScope();
+        locals = Locals.newLocalScope(locals);
 
         boolean continuous = false;
 
@@ -116,15 +117,13 @@ public final class SFor extends AStatement {
 
         statementCount = 1;
 
-        if (locals.getMaxLoopCounter() > 0) {
-            loopCounterSlot = locals.getVariable(location, "#loop").slot;
+        if (locals.hasVariable(Locals.LOOP)) {
+            loopCounter = locals.getVariable(location, Locals.LOOP);
         }
-
-        locals.decrementScope();
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         writer.writeStatementOffset(location);
 
         Label start = new Label();
@@ -132,11 +131,11 @@ public final class SFor extends AStatement {
         Label end = new Label();
 
         if (initializer instanceof SDeclBlock) {
-            ((SDeclBlock)initializer).write(writer);
+            ((SDeclBlock)initializer).write(writer, globals);
         } else if (initializer instanceof AExpression) {
             AExpression initializer = (AExpression)this.initializer;
 
-            initializer.write(writer);
+            initializer.write(writer, globals);
             writer.writePop(initializer.expected.sort.size);
         }
 
@@ -144,7 +143,7 @@ public final class SFor extends AStatement {
 
         if (condition != null) {
             condition.fals = end;
-            condition.write(writer);
+            condition.write(writer, globals);
         }
 
         boolean allEscape = false;
@@ -158,15 +157,15 @@ public final class SFor extends AStatement {
                 ++statementCount;
             }
 
-            writer.writeLoopCounter(loopCounterSlot, statementCount, location);
-            block.write(writer);
+            writer.writeLoopCounter(loopCounter.getSlot(), statementCount, location);
+            block.write(writer, globals);
         } else {
-            writer.writeLoopCounter(loopCounterSlot, 1, location);
+            writer.writeLoopCounter(loopCounter.getSlot(), 1, location);
         }
 
         if (afterthought != null) {
             writer.mark(begin);
-            afterthought.write(writer);
+            afterthought.write(writer, globals);
         }
 
         if (afterthought != null || !allEscape) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index 60015844a96..8d3db7f811a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -19,27 +19,26 @@
 
 package org.elasticsearch.painless.node;
 
+import org.elasticsearch.painless.Constant;
 import org.elasticsearch.painless.Def;
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Locals.Parameter;
-import org.elasticsearch.painless.Locals.FunctionReserved;
 import org.elasticsearch.painless.Locals.Variable;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.WriterConstants;
 import org.objectweb.asm.ClassVisitor;
-import org.objectweb.asm.ClassWriter;
 import org.objectweb.asm.Handle;
 import org.objectweb.asm.Opcodes;
 
 import java.lang.invoke.MethodType;
 import java.lang.reflect.Modifier;
 import java.util.ArrayList;
-import java.util.BitSet;
 import java.util.Collections;
 import java.util.List;
 
@@ -51,17 +50,17 @@ import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE;
 public class SFunction extends AStatement {
     final FunctionReserved reserved;
     final String rtnTypeStr;
-    final String name;
+    public final String name;
     final List<String> paramTypeStrs;
     final List<String> paramNameStrs;
     final List<AStatement> statements;
-    final boolean synthetic;
+    public final boolean synthetic;
 
     Type rtnType = null;
     List<Parameter> parameters = new ArrayList<>();
     Method method = null;
 
-    Locals locals = null;
+    Variable loop = null;
 
     public SFunction(FunctionReserved reserved, Location location,
                      String rtnType, String name, List<String> paramTypes, 
@@ -115,10 +114,7 @@ public class SFunction extends AStatement {
             throw createError(new IllegalArgumentException("Cannot generate an empty function [" + name + "]."));
         }
 
-        this.locals = new Locals(reserved, locals, rtnType, parameters);
-        locals = this.locals;
-
-        locals.incrementScope();
+        locals = Locals.newLocalScope(locals);
 
         AStatement last = statements.get(statements.size() - 1);
 
@@ -141,37 +137,33 @@ public class SFunction extends AStatement {
             throw createError(new IllegalArgumentException("Not all paths provide a return value for method [" + name + "]."));
         }
 
-        locals.decrementScope();
-
-        String staticHandleFieldName = Def.getUserFunctionHandleFieldName(name, parameters.size());
-        locals.addConstant(location, WriterConstants.METHOD_HANDLE_TYPE, staticHandleFieldName, this::initializeConstant);
+        if (reserved.getMaxLoopCounter() > 0) {
+            loop = locals.getVariable(null, FunctionReserved.LOOP);
+        }
     }
     
     /** Writes the function to given ClassVisitor. */
-    void write (ClassVisitor writer, BitSet statements) {
+    void write (ClassVisitor writer, Globals globals) {
         int access = Opcodes.ACC_PRIVATE | Opcodes.ACC_STATIC;
         if (synthetic) {
             access |= Opcodes.ACC_SYNTHETIC;
         }
-        final MethodWriter function = new MethodWriter(access, method.method, writer, statements);
-        write(function);
+        final MethodWriter function = new MethodWriter(access, method.method, writer, globals.getStatements());
+        write(function, globals);
         function.endMethod();
     }
 
     @Override
-    void write(MethodWriter function) {
+    void write(MethodWriter function, Globals globals) {
         if (reserved.getMaxLoopCounter() > 0) {
             // if there is infinite loop protection, we do this once:
             // int #loop = settings.getMaxLoopCounter()
-
-            Variable loop = locals.getVariable(null, FunctionReserved.LOOP);
-
             function.push(reserved.getMaxLoopCounter());
-            function.visitVarInsn(Opcodes.ISTORE, loop.slot);
+            function.visitVarInsn(Opcodes.ISTORE, loop.getSlot());
         }
 
         for (AStatement statement : statements) {
-            statement.write(function);
+            statement.write(function, globals);
         }
 
         if (!methodEscape) {
@@ -181,6 +173,10 @@ public class SFunction extends AStatement {
                 throw createError(new IllegalStateException("Illegal tree structure."));
             }
         }
+
+        String staticHandleFieldName = Def.getUserFunctionHandleFieldName(name, parameters.size());
+        globals.addConstantInitializer(new Constant(location, WriterConstants.METHOD_HANDLE_TYPE, 
+                                                    staticHandleFieldName, this::initializeConstant));
     }
 
     private void initializeConstant(MethodWriter writer) {
@@ -191,4 +187,43 @@ public class SFunction extends AStatement {
                 false);
         writer.push(handle);
     }
+    
+    /**
+     * Tracks reserved variables.  Must be given to any source of input
+     * prior to beginning the analysis phase so that reserved variables
+     * are known ahead of time to assign appropriate slots without
+     * being wasteful.
+     */
+    public interface Reserved {
+        void markReserved(String name);
+        boolean isReserved(String name);
+
+        void setMaxLoopCounter(int max);
+        int getMaxLoopCounter();
+    }
+    
+    public static final class FunctionReserved implements Reserved {
+        public static final String THIS = "#this";
+        public static final String LOOP = "#loop";
+
+        private int maxLoopCounter = 0;
+
+        public void markReserved(String name) {
+            // Do nothing.
+        }
+
+        public boolean isReserved(String name) {
+            return name.equals(THIS) || name.equals(LOOP);
+        }
+
+        @Override
+        public void setMaxLoopCounter(int max) {
+            maxLoopCounter = max;
+        }
+
+        @Override
+        public int getMaxLoopCounter() {
+            return maxLoopCounter;
+        }
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
index a46075af9a1..1b0c0ffb348 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
@@ -58,9 +59,7 @@ public final class SIf extends AStatement {
         ifblock.inLoop = inLoop;
         ifblock.lastLoop = lastLoop;
 
-        locals.incrementScope();
-        ifblock.analyze(locals);
-        locals.decrementScope();
+        ifblock.analyze(Locals.newLocalScope(locals));
 
         anyContinue = ifblock.anyContinue;
         anyBreak = ifblock.anyBreak;
@@ -68,17 +67,17 @@ public final class SIf extends AStatement {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         writer.writeStatementOffset(location);
 
         Label fals = new Label();
 
         condition.fals = fals;
-        condition.write(writer);
+        condition.write(writer, globals);
 
         ifblock.continu = continu;
         ifblock.brake = brake;
-        ifblock.write(writer);
+        ifblock.write(writer, globals);
 
         writer.mark(fals);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
index 22cbfe25614..612969acfff 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
@@ -60,9 +61,7 @@ public final class SIfElse extends AStatement {
         ifblock.inLoop = inLoop;
         ifblock.lastLoop = lastLoop;
 
-        locals.incrementScope();
-        ifblock.analyze(locals);
-        locals.decrementScope();
+        ifblock.analyze(Locals.newLocalScope(locals));
 
         anyContinue = ifblock.anyContinue;
         anyBreak = ifblock.anyBreak;
@@ -76,9 +75,7 @@ public final class SIfElse extends AStatement {
         elseblock.inLoop = inLoop;
         elseblock.lastLoop = lastLoop;
 
-        locals.incrementScope();
-        elseblock.analyze(locals);
-        locals.decrementScope();
+        elseblock.analyze(Locals.newLocalScope(locals));
 
         methodEscape = ifblock.methodEscape && elseblock.methodEscape;
         loopEscape = ifblock.loopEscape && elseblock.loopEscape;
@@ -89,18 +86,18 @@ public final class SIfElse extends AStatement {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         writer.writeStatementOffset(location);
 
         Label end = new Label();
         Label fals = elseblock != null ? new Label() : end;
 
         condition.fals = fals;
-        condition.write(writer);
+        condition.write(writer, globals);
 
         ifblock.continu = continu;
         ifblock.brake = brake;
-        ifblock.write(writer);
+        ifblock.write(writer, globals);
 
         if (!ifblock.allEscape) {
             writer.goTo(end);
@@ -110,7 +107,7 @@ public final class SIfElse extends AStatement {
 
         elseblock.continu = continu;
         elseblock.brake = brake;
-        elseblock.write(writer);
+        elseblock.write(writer, globals);
 
         writer.mark(end);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
index e789f3372e3..2c46b3ecfaf 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Location;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -51,9 +52,9 @@ public final class SReturn extends AStatement {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         writer.writeStatementOffset(location);
-        expression.write(writer);
+        expression.write(writer, globals);
         writer.returnValue();
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index 1d3bb74bc31..5e932072ce1 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -19,13 +19,14 @@
 
 package org.elasticsearch.painless.node;
 
+import org.elasticsearch.painless.Constant;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.MethodKey;
 import org.elasticsearch.painless.Executable;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
-import org.elasticsearch.painless.Locals.Constant;
-import org.elasticsearch.painless.Locals.ExecuteReserved;
 import org.elasticsearch.painless.Locals.Variable;
+import org.elasticsearch.painless.node.SFunction.Reserved;
 import org.elasticsearch.painless.WriterConstants;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
@@ -35,7 +36,9 @@ import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.util.Printer;
 import org.objectweb.asm.util.TraceClassVisitor;
 
+import java.util.ArrayList;
 import java.util.BitSet;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -56,24 +59,28 @@ public final class SSource extends AStatement {
     final String name;
     final String source;
     final Printer debugStream;
-    final ExecuteReserved reserved;
+    final MainMethodReserved reserved;
     final List<SFunction> functions;
+    final Globals globals;
     final List<AStatement> statements;
 
-    private Locals locals;
-    private BitSet expressions;
+    private Locals mainMethod;
     private byte[] bytes;
 
-    public SSource(String name, String source, Printer debugStream, ExecuteReserved reserved, Location location, 
-                   List<SFunction> functions, List<AStatement> statements) {
+    public SSource(String name, String source, Printer debugStream, MainMethodReserved reserved, Location location, 
+                   List<SFunction> functions, Globals globals, List<AStatement> statements) {
         super(location);
 
         this.name = name;
         this.source = source;
         this.debugStream = debugStream;
         this.reserved = reserved;
+        // process any synthetic functions generated by walker (because right now, thats still easy)
+        functions.addAll(globals.getSyntheticMethods().values());
+        globals.getSyntheticMethods().clear();
         this.functions = Collections.unmodifiableList(functions);
         this.statements = Collections.unmodifiableList(statements);
+        this.globals = globals;
     }
 
     public void analyze() {
@@ -89,21 +96,22 @@ public final class SSource extends AStatement {
             }
         }
 
-        locals = new Locals(reserved, methods);
-        analyze(locals);
+        analyze(Locals.newProgramScope(methods.values()));
     }
 
     @Override
-    void analyze(Locals locals) {
+    void analyze(Locals program) {
         for (SFunction function : functions) {
-            function.analyze(locals);
+            Locals functionLocals = Locals.newFunctionScope(program, function.rtnType, function.parameters, 
+                                                            function.reserved.getMaxLoopCounter());
+            function.analyze(functionLocals);
         }
 
         if (statements == null || statements.isEmpty()) {
             throw createError(new IllegalArgumentException("Cannot generate an empty script."));
         }
 
-        locals.incrementScope();
+        mainMethod = Locals.newMainMethodScope(program, reserved.usesScore(), reserved.usesCtx(), reserved.getMaxLoopCounter());
 
         AStatement last = statements.get(statements.size() - 1);
 
@@ -116,13 +124,11 @@ public final class SSource extends AStatement {
 
             statement.lastSource = statement == last;
 
-            statement.analyze(locals);
+            statement.analyze(mainMethod);
 
             methodEscape = statement.methodEscape;
             allEscape = statement.allEscape;
         }
-
-        locals.decrementScope();
     }
 
     public void write() {
@@ -144,10 +150,8 @@ public final class SSource extends AStatement {
         visitor.visit(classVersion, classAccess, className, null, classBase, classInterfaces);
         visitor.visitSource(Location.computeSourceName(name, source), null);
 
-        expressions = new BitSet(source.length());
-
         // Write the constructor:
-        MethodWriter constructor = new MethodWriter(Opcodes.ACC_PUBLIC, CONSTRUCTOR, visitor, expressions);
+        MethodWriter constructor = new MethodWriter(Opcodes.ACC_PUBLIC, CONSTRUCTOR, visitor, globals.getStatements());
         constructor.visitCode();
         constructor.loadThis();
         constructor.loadArgs();
@@ -156,20 +160,31 @@ public final class SSource extends AStatement {
         constructor.endMethod();
 
         // Write the execute method:
-        MethodWriter execute = new MethodWriter(Opcodes.ACC_PUBLIC, EXECUTE, visitor, expressions);
+        MethodWriter execute = new MethodWriter(Opcodes.ACC_PUBLIC, EXECUTE, visitor, globals.getStatements());
         execute.visitCode();
-        write(execute);
+        write(execute, globals);
         execute.endMethod();
         
         // Write all functions:
         for (SFunction function : functions) {
-            function.write(visitor, expressions);
+            function.write(visitor, globals);
+        }
+        
+        // Write all synthetic functions. Note that this process may add more :)
+        while (!globals.getSyntheticMethods().isEmpty()) {
+            List<SFunction> current = new ArrayList<>(globals.getSyntheticMethods().values());
+            globals.getSyntheticMethods().clear();
+            for (SFunction function : current) {
+                function.write(visitor, globals);
+            }
         }
 
         // Write the constants
-        if (false == locals.getConstants().isEmpty()) {
+        if (false == globals.getConstantInitializers().isEmpty()) {
+            Collection<Constant> inits = globals.getConstantInitializers().values();
+
             // Fields
-            for (Constant constant : locals.getConstants()) {
+            for (Constant constant : inits) {
                 visitor.visitField(
                         Opcodes.ACC_FINAL | Opcodes.ACC_PRIVATE | Opcodes.ACC_STATIC,
                         constant.name,
@@ -180,8 +195,8 @@ public final class SSource extends AStatement {
 
             // Initialize the constants in a static initializer
             final MethodWriter clinit = new MethodWriter(Opcodes.ACC_STATIC, 
-                    WriterConstants.CLINIT, visitor, expressions);
-            for (Constant constant : locals.getConstants()) {
+                    WriterConstants.CLINIT, visitor, globals.getStatements());
+            for (Constant constant : inits) {
                 constant.initializer.accept(clinit);
                 clinit.putStatic(CLASS_TYPE, constant.name, constant.type);
             }
@@ -196,44 +211,44 @@ public final class SSource extends AStatement {
     }
     
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         if (reserved.usesScore()) {
             // if the _score value is used, we do this once:
             // final double _score = scorer.score();
-            Variable scorer = locals.getVariable(null, ExecuteReserved.SCORER);
-            Variable score = locals.getVariable(null, ExecuteReserved.SCORE);
+            Variable scorer = mainMethod.getVariable(null, Locals.SCORER);
+            Variable score = mainMethod.getVariable(null, Locals.SCORE);
 
-            writer.visitVarInsn(Opcodes.ALOAD, scorer.slot);
+            writer.visitVarInsn(Opcodes.ALOAD, scorer.getSlot());
             writer.invokeVirtual(WriterConstants.SCORER_TYPE, WriterConstants.SCORER_SCORE);
             writer.visitInsn(Opcodes.F2D);
-            writer.visitVarInsn(Opcodes.DSTORE, score.slot);
+            writer.visitVarInsn(Opcodes.DSTORE, score.getSlot());
         }
 
         if (reserved.usesCtx()) {
             // if the _ctx value is used, we do this once:
             // final Map<String,Object> ctx = input.get("ctx");
 
-            Variable input = locals.getVariable(null, ExecuteReserved.PARAMS);
-            Variable ctx = locals.getVariable(null, ExecuteReserved.CTX);
+            Variable input = mainMethod.getVariable(null, Locals.PARAMS);
+            Variable ctx = mainMethod.getVariable(null, Locals.CTX);
 
-            writer.visitVarInsn(Opcodes.ALOAD, input.slot);
-            writer.push(ExecuteReserved.CTX);
+            writer.visitVarInsn(Opcodes.ALOAD, input.getSlot());
+            writer.push(Locals.CTX);
             writer.invokeInterface(MAP_TYPE, MAP_GET);
-            writer.visitVarInsn(Opcodes.ASTORE, ctx.slot);
+            writer.visitVarInsn(Opcodes.ASTORE, ctx.getSlot());
         }
 
         if (reserved.getMaxLoopCounter() > 0) {
             // if there is infinite loop protection, we do this once:
             // int #loop = settings.getMaxLoopCounter()
 
-            Variable loop = locals.getVariable(null, ExecuteReserved.LOOP);
+            Variable loop = mainMethod.getVariable(null, Locals.LOOP);
 
             writer.push(reserved.getMaxLoopCounter());
-            writer.visitVarInsn(Opcodes.ISTORE, loop.slot);
+            writer.visitVarInsn(Opcodes.ISTORE, loop.getSlot());
         }
 
         for (AStatement statement : statements) {
-            statement.write(writer);
+            statement.write(writer, globals);
         }
 
         if (!methodEscape) {
@@ -242,11 +257,50 @@ public final class SSource extends AStatement {
         }
     }
 
-    public BitSet getExpressions() {
-        return expressions;
+    public BitSet getStatements() {
+        return globals.getStatements();
     }
 
     public byte[] getBytes() {
         return bytes;
     }
+    
+    
+    public static final class MainMethodReserved implements Reserved {
+        private boolean score = false;
+        private boolean ctx = false;
+        private int maxLoopCounter = 0;
+
+        @Override
+        public void markReserved(String name) {
+            if (Locals.SCORE.equals(name)) {
+                score = true;
+            } else if (Locals.CTX.equals(name)) {
+                ctx = true;
+            }
+        }
+
+        @Override
+        public boolean isReserved(String name) {
+            return Locals.KEYWORDS.contains(name);
+        }
+
+        public boolean usesScore() {
+            return score;
+        }
+
+        public boolean usesCtx() {
+            return ctx;
+        }
+
+        @Override
+        public void setMaxLoopCounter(int max) {
+            maxLoopCounter = max;
+        }
+
+        @Override
+        public int getMaxLoopCounter() {
+            return maxLoopCounter;
+        }
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
index db04d622839..ffa1b89dfe2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
@@ -50,9 +51,9 @@ public final class SThrow extends AStatement {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         writer.writeStatementOffset(location);
-        expression.write(writer);
+        expression.write(writer, globals);
         writer.throwException();
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
index 42fffc759ce..0295d32ae35 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.painless.node;
 
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.Location;
@@ -52,9 +53,7 @@ public final class STry extends AStatement {
         block.inLoop = inLoop;
         block.lastLoop = lastLoop;
 
-        locals.incrementScope();
-        block.analyze(locals);
-        locals.decrementScope();
+        block.analyze(Locals.newLocalScope(locals));
 
         methodEscape = block.methodEscape;
         loopEscape = block.loopEscape;
@@ -69,9 +68,7 @@ public final class STry extends AStatement {
             catc.inLoop = inLoop;
             catc.lastLoop = lastLoop;
 
-            locals.incrementScope();
-            catc.analyze(locals);
-            locals.decrementScope();
+            catc.analyze(Locals.newLocalScope(locals));
 
             methodEscape &= catc.methodEscape;
             loopEscape &= catc.loopEscape;
@@ -86,7 +83,7 @@ public final class STry extends AStatement {
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         writer.writeStatementOffset(location);
 
         Label begin = new Label();
@@ -97,7 +94,7 @@ public final class STry extends AStatement {
 
         block.continu = continu;
         block.brake = brake;
-        block.write(writer);
+        block.write(writer, globals);
 
         if (!block.allEscape) {
             writer.goTo(exception);
@@ -109,7 +106,7 @@ public final class STry extends AStatement {
             catc.begin = begin;
             catc.end = end;
             catc.exception = catches.size() > 1 ? exception : null;
-            catc.write(writer);
+            catc.write(writer, globals);
         }
 
         if (!block.allEscape || catches.size() > 1) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
index 20478a55aa0..39b48781b46 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
@@ -42,7 +43,7 @@ public final class SWhile extends AStatement {
 
     @Override
     void analyze(Locals locals) {
-        locals.incrementScope();
+        locals = Locals.newLocalScope(locals);
 
         condition.expected = Definition.BOOLEAN_TYPE;
         condition.analyze(locals);
@@ -82,15 +83,13 @@ public final class SWhile extends AStatement {
 
         statementCount = 1;
 
-        if (locals.getMaxLoopCounter() > 0) {
-            loopCounterSlot = locals.getVariable(location, "#loop").slot;
+        if (locals.hasVariable(Locals.LOOP)) {
+            loopCounter = locals.getVariable(location, Locals.LOOP);
         }
-
-        locals.decrementScope();
     }
 
     @Override
-    void write(MethodWriter writer) {
+    void write(MethodWriter writer, Globals globals) {
         writer.writeStatementOffset(location);
 
         Label begin = new Label();
@@ -99,16 +98,16 @@ public final class SWhile extends AStatement {
         writer.mark(begin);
 
         condition.fals = end;
-        condition.write(writer);
+        condition.write(writer, globals);
 
         if (block != null) {
-            writer.writeLoopCounter(loopCounterSlot, Math.max(1, block.statementCount), location);
+            writer.writeLoopCounter(loopCounter.getSlot(), Math.max(1, block.statementCount), location);
 
             block.continu = begin;
             block.brake = end;
-            block.write(writer);
+            block.write(writer, globals);
         } else {
-            writer.writeLoopCounter(loopCounterSlot, 1, location);
+            writer.writeLoopCounter(loopCounter.getSlot(), 1, location);
         }
 
         if (block == null || !block.allEscape) {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java
index e1dbe9db0f7..fdc4fba4313 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ReservedWordTests.java
@@ -30,7 +30,7 @@ public class ReservedWordTests extends ScriptTestCase {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("int _score = 5; return _score;");
         });
-        assertTrue(expected.getMessage().contains("Variable [_score] is reserved"));
+        assertTrue(expected.getMessage().contains("Variable [_score] is already defined"));
     }
 
     /** check that we can't write to _score, its read-only! */
@@ -46,7 +46,7 @@ public class ReservedWordTests extends ScriptTestCase {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("int doc = 5; return doc;");
         });
-        assertTrue(expected.getMessage().contains("Variable [doc] is reserved"));
+        assertTrue(expected.getMessage().contains("Variable [doc] is already defined"));
     }
 
     /** check that we can't write to doc, its read-only! */
@@ -62,7 +62,7 @@ public class ReservedWordTests extends ScriptTestCase {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("int ctx = 5; return ctx;");
         });
-        assertTrue(expected.getMessage().contains("Variable [ctx] is reserved"));
+        assertTrue(expected.getMessage().contains("Variable [ctx] is already defined"));
     }
 
     /** check that we can't write to ctx, its read-only! */
@@ -83,7 +83,7 @@ public class ReservedWordTests extends ScriptTestCase {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("int _value = 5; return _value;");
         });
-        assertTrue(expected.getMessage().contains("Variable [_value] is reserved"));
+        assertTrue(expected.getMessage().contains("Variable [_value] is already defined"));
     }
 
     /** check that we can't write to _value, its read-only! */

From 420dc721243f7c2ae1920349885a4fd6d7a808a7 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Sat, 18 Jun 2016 08:39:10 +0200
Subject: [PATCH 0940/1311] Add ClusterName#value as the default instead of
 it's toString method

---
 .../node/internal/InternalSettingsPreparer.java            | 2 +-
 .../node/internal/InternalSettingsPreparerTests.java       | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java b/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java
index 7a1c6321c43..3ee704c42aa 100644
--- a/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java
+++ b/core/src/main/java/org/elasticsearch/node/internal/InternalSettingsPreparer.java
@@ -163,7 +163,7 @@ public class InternalSettingsPreparer {
 
         // put the cluster name
         if (output.get(ClusterName.CLUSTER_NAME_SETTING.getKey()) == null) {
-            output.put(ClusterName.CLUSTER_NAME_SETTING.getKey(), ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY));
+            output.put(ClusterName.CLUSTER_NAME_SETTING.getKey(), ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY).value());
         }
 
         replacePromptPlaceholders(output, terminal);
diff --git a/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java b/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java
index 226f6eae958..5637e937c1f 100644
--- a/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java
+++ b/core/src/test/java/org/elasticsearch/node/internal/InternalSettingsPreparerTests.java
@@ -68,6 +68,13 @@ public class InternalSettingsPreparerTests extends ESTestCase {
         assertTrue(configDir, configDir.startsWith(home));
     }
 
+    public void testDefaultClusterName() {
+        Settings settings = InternalSettingsPreparer.prepareSettings(Settings.EMPTY);
+        assertEquals("elasticsearch", settings.get("cluster.name"));
+        settings = InternalSettingsPreparer.prepareSettings(Settings.builder().put("cluster.name", "foobar").build());
+        assertEquals("foobar", settings.get("cluster.name"));
+    }
+
     public void testReplacePromptPlaceholders() {
         MockTerminal terminal = new MockTerminal();
         terminal.addTextInput("text");

From 5c98818f480b2ab5f066def5e0f9366af90e68e0 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Sat, 18 Jun 2016 10:54:12 +0200
Subject: [PATCH 0941/1311] fix compilation

---
 .../java/org/elasticsearch/messy/tests/SearchFieldsTests.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
index ba258490119..d834f0fcf6d 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
@@ -22,6 +22,7 @@ package org.elasticsearch.messy.tests;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -373,7 +374,7 @@ public class SearchFieldsTests extends ESIntegTestCase {
     public void testScriptFieldsForNullReturn() throws Exception {
         client().prepareIndex("test", "type1", "1")
             .setSource("foo", "bar")
-            .setRefresh(true).get();
+            .setRefreshPolicy("true").get();
 
         SearchResponse response = client().prepareSearch().setQuery(matchAllQuery())
             .addScriptField("test_script_1", new Script("return null"))

From a14ba1e5b24280460b9c9163e105219514562d2a Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Sat, 18 Jun 2016 10:23:35 -0400
Subject: [PATCH 0942/1311] Painless: add lambda captures

---
 .../elasticsearch/painless/LambdaLocals.java  |  62 --------
 .../org/elasticsearch/painless/Locals.java    |  61 +++++---
 .../elasticsearch/painless/node/ANode.java    |  13 +-
 .../elasticsearch/painless/node/EBinary.java  |  16 +-
 .../elasticsearch/painless/node/EBool.java    |  16 +-
 .../elasticsearch/painless/node/EBoolean.java |   5 +
 .../painless/node/ECapturingFunctionRef.java  |  19 ++-
 .../elasticsearch/painless/node/ECast.java    |  14 +-
 .../elasticsearch/painless/node/EChain.java   |  14 +-
 .../elasticsearch/painless/node/EComp.java    |  16 +-
 .../painless/node/EConditional.java           |  17 ++-
 .../painless/node/EConstant.java              |   6 +
 .../elasticsearch/painless/node/EDecimal.java |  10 +-
 .../painless/node/EExplicit.java              |  12 +-
 .../painless/node/EFunctionRef.java           |   9 +-
 .../elasticsearch/painless/node/ELambda.java  | 141 +++++++++++++++---
 .../elasticsearch/painless/node/ENull.java    |   6 +
 .../elasticsearch/painless/node/ENumeric.java |   9 +-
 .../elasticsearch/painless/node/EUnary.java   |  13 +-
 .../painless/node/LArrayLength.java           |   8 +-
 .../elasticsearch/painless/node/LBrace.java   |   9 +-
 .../painless/node/LCallInvoke.java            |  15 +-
 .../painless/node/LCallLocal.java             |  13 +-
 .../elasticsearch/painless/node/LCast.java    |   6 +
 .../painless/node/LDefArray.java              |  11 +-
 .../elasticsearch/painless/node/LDefCall.java |  13 +-
 .../painless/node/LDefField.java              |   8 +-
 .../elasticsearch/painless/node/LField.java   |   7 +-
 .../painless/node/LListShortcut.java          |  11 +-
 .../painless/node/LMapShortcut.java           |  11 +-
 .../painless/node/LNewArray.java              |  13 +-
 .../elasticsearch/painless/node/LNewObj.java  |  13 +-
 .../elasticsearch/painless/node/LRegex.java   |   4 +
 .../painless/node/LShortcut.java              |   9 +-
 .../elasticsearch/painless/node/LStatic.java  |   9 +-
 .../elasticsearch/painless/node/LString.java  |   8 +-
 .../painless/node/LVariable.java              |  10 +-
 .../elasticsearch/painless/node/SBlock.java   |   8 +
 .../elasticsearch/painless/node/SBreak.java   |   5 +
 .../elasticsearch/painless/node/SCatch.java   |  16 +-
 .../painless/node/SContinue.java              |   5 +
 .../painless/node/SDeclBlock.java             |   8 +
 .../painless/node/SDeclaration.java           |  16 +-
 .../org/elasticsearch/painless/node/SDo.java  |  14 +-
 .../elasticsearch/painless/node/SEach.java    |  18 ++-
 .../painless/node/SExpression.java            |  11 +-
 .../org/elasticsearch/painless/node/SFor.java |  19 +++
 .../painless/node/SFunction.java              |  18 ++-
 .../org/elasticsearch/painless/node/SIf.java  |  14 +-
 .../elasticsearch/painless/node/SIfElse.java  |  17 ++-
 .../elasticsearch/painless/node/SReturn.java  |  10 +-
 .../elasticsearch/painless/node/SSource.java  |  18 ++-
 .../elasticsearch/painless/node/SThrow.java   |  10 +-
 .../org/elasticsearch/painless/node/STry.java |  11 ++
 .../elasticsearch/painless/node/SWhile.java   |  14 +-
 .../elasticsearch/painless/LambdaTests.java   |  30 ++++
 56 files changed, 718 insertions(+), 181 deletions(-)
 delete mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaLocals.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaLocals.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaLocals.java
deleted file mode 100644
index 9c01bc4c628..00000000000
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaLocals.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.painless;
-
-import org.elasticsearch.painless.Definition.Type;
-
-import java.util.List;
-import java.util.Objects;
-
-/** Extension of locals for lambdas */
-// Note: this isn't functional yet, it throws UOE
-// TODO: implement slot renumbering for captures.
-class LambdaLocals extends Locals {
-    private List<Variable> captures;
-
-    LambdaLocals(Locals parent, List<Parameter> parameters, List<Variable> captures) {
-        super(parent);
-        for (Parameter parameter : parameters) {
-            defineVariable(parameter.location, parameter.type, parameter.name, false);
-        }
-        this.captures = Objects.requireNonNull(captures);
-    }
-    
-    @Override
-    public Variable getVariable(Location location, String name) {
-        Variable variable = lookupVariable(location, name);
-        if (variable != null) {
-            return variable;
-        }
-        if (getParent() != null) {
-            variable = getParent().getVariable(location, name);
-            if (variable != null) {
-                assert captures != null; // unused right now
-                // make it read-only, and record that it was used.
-                throw new UnsupportedOperationException("lambda capture is not supported");
-            }
-        }
-        throw location.createError(new IllegalArgumentException("Variable [" + name + "] is not defined."));
-    }
-
-    @Override
-    public Type getReturnType() {
-        return Definition.DEF_TYPE;
-    }
-}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
index 8cb99ed2404..0893a51bc47 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
@@ -35,7 +35,7 @@ import java.util.Set;
 /**
  * Tracks user defined methods and variables across compilation phases.
  */
-public class Locals {
+public final class Locals {
     
     /** Reserved word: params map parameter */
     public static final String PARAMS = "params";
@@ -64,9 +64,23 @@ public class Locals {
         return new Locals(currentScope);
     }
     
-    /** Creates a new lambda scope inside the current scope */
-    public static Locals newLambdaScope(Locals currentScope, List<Parameter> parameters, List<Variable> captures) {
-        return new LambdaLocals(currentScope, parameters, captures);
+    /** 
+     * Creates a new lambda scope inside the current scope
+     * <p>
+     * This is just like {@link #newFunctionScope}, except the captured parameters are made read-only.
+     */
+    public static Locals newLambdaScope(Locals programScope, List<Parameter> parameters, int captureCount, int maxLoopCounter) {
+        Locals locals = new Locals(programScope, Definition.DEF_TYPE);
+        for (int i = 0; i < parameters.size(); i++) {
+            Parameter parameter = parameters.get(i);
+            boolean isCapture = i < captureCount;
+            locals.defineVariable(parameter.location, parameter.type, parameter.name, isCapture);
+        }
+        // Loop counter to catch infinite loops.  Internal use only.
+        if (maxLoopCounter > 0) {
+            locals.defineVariable(null, Definition.INT_TYPE, LOOP, true);
+        }
+        return locals;
     }
     
     /** Creates a new function scope inside the current scope */
@@ -129,7 +143,7 @@ public class Locals {
     }
     
     /** Checks if a variable exists or not, in this scope or any parents. */
-    public final boolean hasVariable(String name) {
+    public boolean hasVariable(String name) {
         Variable variable = lookupVariable(null, name);
         if (variable != null) {
             return true;
@@ -153,7 +167,7 @@ public class Locals {
     }
     
     /** Looks up a method. Returns null if the method does not exist. */
-    public final Method getMethod(MethodKey key) {
+    public Method getMethod(MethodKey key) {
         Method method = lookupMethod(key);
         if (method != null) {
             return method;
@@ -165,7 +179,7 @@ public class Locals {
     }
     
     /** Creates a new variable. Throws IAE if the variable has already been defined (even in a parent) or reserved. */
-    public final Variable addVariable(Location location, Type type, String name, boolean readonly) {
+    public Variable addVariable(Location location, Type type, String name, boolean readonly) {
         if (hasVariable(name)) {
             throw location.createError(new IllegalArgumentException("Variable [" + name + "] is already defined."));
         }
@@ -196,23 +210,23 @@ public class Locals {
     // return type of this scope
     private final Type returnType;
     // next slot number to assign
-    int nextSlotNumber;
+    private int nextSlotNumber;
     // variable name -> variable
-    Map<String,Variable> variables;
+    private Map<String,Variable> variables;
     // method name+arity -> methods
-    Map<MethodKey,Method> methods;
+    private Map<MethodKey,Method> methods;
 
     /**
      * Create a new Locals
      */
-    Locals(Locals parent) {
+    private Locals(Locals parent) {
         this(parent, parent.getReturnType());
     }
     
     /**
      * Create a new Locals with specified return type
      */
-    Locals(Locals parent, Type returnType) {
+    private Locals(Locals parent, Type returnType) {
         this.parent = parent;
         this.returnType = returnType;
         if (parent == null) {
@@ -223,12 +237,12 @@ public class Locals {
     }
 
     /** Returns the parent scope */
-    Locals getParent() {
+    private Locals getParent() {
         return parent;
     }
 
     /** Looks up a variable at this scope only. Returns null if the variable does not exist. */
-    Variable lookupVariable(Location location, String name) {
+    private Variable lookupVariable(Location location, String name) {
         if (variables == null) {
             return null;
         }
@@ -236,7 +250,7 @@ public class Locals {
     }
 
     /** Looks up a method at this scope only. Returns null if the method does not exist. */
-    Method lookupMethod(MethodKey key) {
+    private Method lookupMethod(MethodKey key) {
         if (methods == null) {
             return null;
         }
@@ -245,19 +259,17 @@ public class Locals {
 
     
     /** Defines a variable at this scope internally. */
-    Variable defineVariable(Location location, Type type, String name, boolean readonly) {
+    private Variable defineVariable(Location location, Type type, String name, boolean readonly) {
         if (variables == null) {
             variables = new HashMap<>();
         }
-        Variable variable = new Variable(location, name, type, readonly);
-        variable.slot = getNextSlot();
+        Variable variable = new Variable(location, name, type, getNextSlot(), readonly);
         variables.put(name, variable); // TODO: check result
         nextSlotNumber += type.type.getSize();
         return variable;
     }
     
-    // TODO: make private, thats bogus
-    public void addMethod(Method method) {
+    private void addMethod(Method method) {
         if (methods == null) {
             methods = new HashMap<>();
         }
@@ -266,7 +278,7 @@ public class Locals {
     }
 
 
-    int getNextSlot() {
+    private int getNextSlot() {
         return nextSlotNumber;
     }
 
@@ -274,13 +286,14 @@ public class Locals {
         public final Location location;
         public final String name;
         public final Type type;
-        int slot = -1;
         public final boolean readonly;
+        private final int slot;
         
-        public Variable(Location location, String name, Type type, boolean readonly) {
+        public Variable(Location location, String name, Type type, int slot, boolean readonly) {
             this.location = location;
             this.name = name;
             this.type = type;
+            this.slot = slot;
             this.readonly = readonly;
         }
         
@@ -289,7 +302,7 @@ public class Locals {
         }
     }
     
-    public static class Parameter {
+    public static final class Parameter {
         public final Location location;
         public final String name;
         public final Type type;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
index 08092a9e116..55d62108cba 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ANode.java
@@ -21,6 +21,9 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Location;
 
+import java.util.Objects;
+import java.util.Set;
+
 /**
  * The superclass for all other nodes.
  */
@@ -31,9 +34,17 @@ public abstract class ANode {
     final Location location;
 
     ANode(Location location) {
-        this.location = location;
+        this.location = Objects.requireNonNull(location);
     }
     
+    /**
+     * Adds all variable names referenced to the variable set.
+     * <p>
+     * This can be called at any time, e.g. to support lambda capture.
+     * @param variables set of variables referenced (any scope)
+     */
+    abstract void extractVariables(Set<String> variables);
+    
     public RuntimeException createError(RuntimeException exception) {
         return location.createError(exception);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 11b77b0441f..f0f79315c64 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -24,6 +24,10 @@ import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
+
+import java.util.Objects;
+import java.util.Set;
+
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 import org.elasticsearch.painless.Operation;
@@ -46,9 +50,15 @@ public final class EBinary extends AExpression {
     public EBinary(Location location, Operation operation, AExpression left, AExpression right) {
         super(location);
 
-        this.operation = operation;
-        this.left = left;
-        this.right = right;
+        this.operation = Objects.requireNonNull(operation);
+        this.left = Objects.requireNonNull(left);
+        this.right = Objects.requireNonNull(right);
+    }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        left.extractVariables(variables);
+        right.extractVariables(variables);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
index 9e39ca712f5..d707cc811f9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBool.java
@@ -25,6 +25,10 @@ import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
+
+import java.util.Objects;
+import java.util.Set;
+
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -39,9 +43,15 @@ public final class EBool extends AExpression {
     public EBool(Location location, Operation operation, AExpression left, AExpression right) {
         super(location);
 
-        this.operation = operation;
-        this.left = left;
-        this.right = right;
+        this.operation = Objects.requireNonNull(operation);
+        this.left = Objects.requireNonNull(left);
+        this.right = Objects.requireNonNull(right);
+    }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        left.extractVariables(variables);
+        right.extractVariables(variables);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
index a152422fac4..dc25bb0ed44 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBoolean.java
@@ -25,6 +25,8 @@ import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
+import java.util.Set;
+
 /**
  * Represents a boolean constant.
  */
@@ -35,6 +37,9 @@ public final class EBoolean extends AExpression {
 
         this.constant = constant;
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {}
 
     @Override
     void analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
index 333787db962..0445a8d20f5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
@@ -34,32 +34,39 @@ import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
 import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE;
 
 import java.lang.invoke.LambdaMetafactory;
+import java.util.Objects;
+import java.util.Set;
 
 /**
  * Represents a capturing function reference.
  */
 public class ECapturingFunctionRef extends AExpression implements ILambda {
-    public final String type;
+    public final String variable;
     public final String call;
     
     private FunctionRef ref;
     Variable captured;
     String defPointer;
 
-    public ECapturingFunctionRef(Location location, String type, String call) {
+    public ECapturingFunctionRef(Location location, String variable, String call) {
         super(location);
 
-        this.type = type;
-        this.call = call;
+        this.variable = Objects.requireNonNull(variable);
+        this.call = Objects.requireNonNull(call);
+    }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        variables.add(variable);
     }
 
     @Override
     void analyze(Locals variables) {
-        captured = variables.getVariable(location, type);
+        captured = variables.getVariable(location, variable);
         if (expected == null) {
             if (captured.type.sort == Definition.Sort.DEF) {
                 // dynamic implementation
-                defPointer = "D" + type + "." + call + ",1";
+                defPointer = "D" + variable + "." + call + ",1";
             } else {
                 // typed implementation
                 defPointer = "S" + captured.type.name + "." + call + ",1";
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
index 1073125c637..c7dda568ff5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java
@@ -20,6 +20,10 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition.Cast;
+
+import java.util.Objects;
+import java.util.Set;
+
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
@@ -40,9 +44,13 @@ final class ECast extends AExpression {
         super(location);
 
         this.type = null;
-        this.child = child;
-
-        this.cast = cast;
+        this.child = Objects.requireNonNull(child);
+        this.cast = Objects.requireNonNull(cast);
+    }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        child.extractVariables(variables);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index b249cf4511e..9e34da7ea64 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -32,6 +32,8 @@ import org.elasticsearch.painless.MethodWriter;
 
 import java.util.Arrays;
 import java.util.List;
+import java.util.Objects;
+import java.util.Set;
 
 /**
  * Represents the entirety of a variable/method chain for read/write operations.
@@ -59,12 +61,22 @@ public final class EChain extends AExpression {
                   boolean pre, boolean post, Operation operation, AExpression expression) {
         super(location);
 
-        this.links = links;
+        this.links = Objects.requireNonNull(links);
         this.pre = pre;
         this.post = post;
         this.operation = operation;
         this.expression = expression;
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        for (ALink link : links) {
+            link.extractVariables(variables);
+        }
+        if (expression != null) {
+            expression.extractVariables(variables);
+        }
+    }
 
     @Override
     void analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index 53684a19005..2fcda21c4ec 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -29,6 +29,10 @@ import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
+
+import java.util.Objects;
+import java.util.Set;
+
 import org.elasticsearch.painless.MethodWriter;
 
 import static org.elasticsearch.painless.WriterConstants.OBJECTS_TYPE;
@@ -48,9 +52,15 @@ public final class EComp extends AExpression {
     public EComp(Location location, Operation operation, AExpression left, AExpression right) {
         super(location);
 
-        this.operation = operation;
-        this.left = left;
-        this.right = right;
+        this.operation = Objects.requireNonNull(operation);
+        this.left = Objects.requireNonNull(left);
+        this.right = Objects.requireNonNull(right);
+    }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        left.extractVariables(variables);
+        right.extractVariables(variables);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
index 600626348dd..e05419e1c52 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConditional.java
@@ -26,6 +26,10 @@ import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
+
+import java.util.Objects;
+import java.util.Set;
+
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -40,9 +44,16 @@ public final class EConditional extends AExpression {
     public EConditional(Location location, AExpression condition, AExpression left, AExpression right) {
         super(location);
 
-        this.condition = condition;
-        this.left = left;
-        this.right = right;
+        this.condition = Objects.requireNonNull(condition);
+        this.left = Objects.requireNonNull(left);
+        this.right = Objects.requireNonNull(right);
+    }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        condition.extractVariables(variables);
+        left.extractVariables(variables);
+        right.extractVariables(variables);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
index b037d25c2d6..c5e10a340d6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EConstant.java
@@ -22,6 +22,9 @@ package org.elasticsearch.painless.node;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Definition.Sort;
+
+import java.util.Set;
+
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
@@ -37,6 +40,9 @@ final class EConstant extends AExpression {
 
         this.constant = constant;
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {}
 
     @Override
     void analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
index efa453605cb..e93b63c3bcd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EDecimal.java
@@ -25,8 +25,11 @@ import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
+import java.util.Objects;
+import java.util.Set;
+
 /**
- * Respresents a decimal constant.
+ * Represents a decimal constant.
  */
 public final class EDecimal extends AExpression {
 
@@ -35,8 +38,11 @@ public final class EDecimal extends AExpression {
     public EDecimal(Location location, String value) {
         super(location);
 
-        this.value = value;
+        this.value = Objects.requireNonNull(value);
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {}
 
     @Override
     void analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
index 1c186faaeeb..71ad952baff 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java
@@ -25,6 +25,9 @@ import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
+import java.util.Objects;
+import java.util.Set;
+
 /**
  * Represents an explicit cast.
  */
@@ -36,8 +39,13 @@ public final class EExplicit extends AExpression {
     public EExplicit(Location location, String type, AExpression child) {
         super(location);
 
-        this.type = type;
-        this.child = child;
+        this.type = Objects.requireNonNull(type);
+        this.child = Objects.requireNonNull(child);
+    }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        child.extractVariables(variables);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index 856a876550c..298b84ffe29 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -32,6 +32,8 @@ import org.objectweb.asm.Type;
 import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE;
 
 import java.lang.invoke.LambdaMetafactory;
+import java.util.Objects;
+import java.util.Set;
 
 /**
  * Represents a function reference.
@@ -46,9 +48,12 @@ public class EFunctionRef extends AExpression implements ILambda {
     public EFunctionRef(Location location, String type, String call) {
         super(location);
 
-        this.type = type;
-        this.call = call;
+        this.type = Objects.requireNonNull(type);
+        this.call = Objects.requireNonNull(call);
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {}
 
     @Override
     void analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
index d0b531c74d3..8adae6607a9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
@@ -23,13 +23,23 @@ import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Locals.Variable;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.node.SFunction.FunctionReserved;
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.FunctionRef;
 import org.elasticsearch.painless.Globals;
+import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.Type;
 
+import java.lang.invoke.LambdaMetafactory;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE;
 
 public class ELambda extends AExpression implements ILambda {
     final String name;
@@ -37,58 +47,147 @@ public class ELambda extends AExpression implements ILambda {
     final List<String> paramTypeStrs;
     final List<String> paramNameStrs;
     final List<AStatement> statements;
+
     // desugared synthetic method (lambda body)
     SFunction desugared;
-    // method ref (impl detail)
-    ILambda impl;
+    // captured variables
+    List<Variable> captures;
+    // static parent, static lambda
+    FunctionRef ref;
+    // dynamic parent, deferred until link time
+    String defPointer;
 
     public ELambda(String name, FunctionReserved reserved, 
                    Location location, List<String> paramTypes, List<String> paramNames, 
                    List<AStatement> statements) {
         super(location);
-        this.name = name;
-        this.reserved = reserved;
+        this.name = Objects.requireNonNull(name);
+        this.reserved = Objects.requireNonNull(reserved);
         this.paramTypeStrs = Collections.unmodifiableList(paramTypes);
         this.paramNameStrs = Collections.unmodifiableList(paramNames);
         this.statements = Collections.unmodifiableList(statements);
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        for (AStatement statement : statements) {
+            statement.extractVariables(variables);
+        }
+    }
 
     @Override
     void analyze(Locals locals) {
+        // gather any variables used by the lambda body first.
+        Set<String> variables = new HashSet<>();
+        for (AStatement statement : statements) {
+            statement.extractVariables(variables);
+        }
+        // any of those variables defined in our scope need to be captured
+        captures = new ArrayList<>();
+        for (String variable : variables) {
+            if (locals.hasVariable(variable)) {
+                captures.add(locals.getVariable(location, variable));
+            }
+        }
+        // prepend capture list to lambda's arguments
+        List<String> paramTypes = new ArrayList<>();
+        List<String> paramNames = new ArrayList<>();
+        for (Variable var : captures) {
+            paramTypes.add(var.type.name);
+            paramNames.add(var.name);
+        }
+        paramTypes.addAll(paramTypeStrs);
+        paramNames.addAll(paramNameStrs);
+        
         // desugar lambda body into a synthetic method
         desugared = new SFunction(reserved, location, "def", name, 
-                                            paramTypeStrs, paramNameStrs, statements, true);
+                                            paramTypes, paramNames, statements, true);
         desugared.generate();
-        List<Variable> captures = new ArrayList<>();
-        desugared.analyze(Locals.newLambdaScope(locals.getProgramScope(), desugared.parameters, captures));
+        desugared.analyze(Locals.newLambdaScope(locals.getProgramScope(), desugared.parameters, 
+                                                captures.size(), reserved.getMaxLoopCounter()));
         
-        // setup reference
-        EFunctionRef ref = new EFunctionRef(location, "this", name);
-        ref.expected = expected;
-        // hack, create a new scope, with our method, so the ref can see it (impl detail)
-        locals = Locals.newLocalScope(locals);
-        locals.addMethod(desugared.method);
-        ref.analyze(locals);
-        actual = ref.actual;
-        impl = ref;
+        // setup method reference to synthetic method
+        if (expected == null) {
+            ref = null;
+            actual = Definition.getType("String");
+            defPointer = "Sthis." + name + "," + captures.size();
+        } else {
+            defPointer = null;
+            try {
+                Method interfaceMethod = expected.struct.getFunctionalMethod();
+                if (interfaceMethod == null) {
+                    throw new IllegalArgumentException("Cannot pass lambda to [" + expected.name + 
+                            "], not a functional interface");
+                }
+                Class<?> captureClasses[] = new Class<?>[captures.size()];
+                for (int i = 0; i < captures.size(); i++) {
+                    captureClasses[i] = captures.get(i).type.clazz;
+                }
+                ref = new FunctionRef(expected, interfaceMethod, desugared.method, captureClasses);
+            } catch (IllegalArgumentException e) {
+                throw createError(e);
+            }
+            actual = expected;
+        }
     }
 
     @Override
     void write(MethodWriter writer, Globals globals) {
-        AExpression expr = (AExpression) impl;
-        expr.write(writer, globals);
+        writer.writeDebugInfo(location);
+
+        if (ref != null) {
+            writer.writeDebugInfo(location);
+            // load captures
+            for (Variable capture : captures) {
+                writer.visitVarInsn(capture.type.type.getOpcode(Opcodes.ILOAD), capture.getSlot());
+            }
+            // convert MethodTypes to asm Type for the constant pool.
+            String invokedType = ref.invokedType.toMethodDescriptorString();
+            Type samMethodType = Type.getMethodType(ref.samMethodType.toMethodDescriptorString());
+            Type interfaceType = Type.getMethodType(ref.interfaceMethodType.toMethodDescriptorString());
+            if (ref.needsBridges()) {
+                writer.invokeDynamic(ref.invokedName,
+                                     invokedType,
+                                     LAMBDA_BOOTSTRAP_HANDLE,
+                                     samMethodType,
+                                     ref.implMethodASM,
+                                     samMethodType,
+                                     LambdaMetafactory.FLAG_BRIDGES,
+                                     1,
+                                     interfaceType);
+            } else {
+                writer.invokeDynamic(ref.invokedName,
+                                     invokedType,
+                                     LAMBDA_BOOTSTRAP_HANDLE,
+                                     samMethodType,
+                                     ref.implMethodASM,
+                                     samMethodType,
+                                     0);
+            }
+        } else {
+            // placeholder
+            writer.push((String)null);
+            // load captures
+            for (Variable capture : captures) {
+                writer.visitVarInsn(capture.type.type.getOpcode(Opcodes.ILOAD), capture.getSlot());
+            }
+        }
+
         // add synthetic method to the queue to be written
         globals.addSyntheticMethod(desugared);
     }
 
     @Override
     public String getPointer() {
-        return impl.getPointer();
+        return defPointer;
     }
 
     @Override
     public Type[] getCaptures() {
-        return impl.getCaptures();
+        Type[] types = new Type[captures.size()];
+        for (int i = 0; i < types.length; i++) {
+            types[i] = captures.get(i).type.type;
+        }
+        return types;
     }
-    
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
index 0a1242e6507..ad6562c0e14 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java
@@ -24,6 +24,9 @@ import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Opcodes;
+
+import java.util.Set;
+
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -34,6 +37,9 @@ public final class ENull extends AExpression {
     public ENull(Location location) {
         super(location);
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {}
 
     @Override
     void analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
index e2314880448..e9a28a1e06a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENumeric.java
@@ -23,6 +23,10 @@ import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Sort;
+
+import java.util.Objects;
+import java.util.Set;
+
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -37,9 +41,12 @@ public final class ENumeric extends AExpression {
     public ENumeric(Location location, String value, int radix) {
         super(location);
 
-        this.value = value;
+        this.value = Objects.requireNonNull(value);
         this.radix = radix;
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {}
 
     @Override
     void analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index f74b91dd5bc..30b5310da4b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -29,6 +29,10 @@ import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
+
+import java.util.Objects;
+import java.util.Set;
+
 import org.elasticsearch.painless.MethodWriter;
 
 import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
@@ -45,8 +49,13 @@ public final class EUnary extends AExpression {
     public EUnary(Location location, Operation operation, AExpression child) {
         super(location);
 
-        this.operation = operation;
-        this.child = child;
+        this.operation = Objects.requireNonNull(operation);
+        this.child = Objects.requireNonNull(child);
+    }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        child.extractVariables(variables);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
index 1a240747d73..a2970ca2e53 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LArrayLength.java
@@ -25,6 +25,9 @@ import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
+import java.util.Objects;
+import java.util.Set;
+
 /**
  * Represents an array length field load.
  */
@@ -35,8 +38,11 @@ public final class LArrayLength extends ALink {
     LArrayLength(Location location, String value) {
         super(location, -1);
 
-        this.value = value;
+        this.value = Objects.requireNonNull(value);
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {}
 
     @Override
     ALink analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
index e8c23bf3857..16b9b8c5d44 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LBrace.java
@@ -28,6 +28,8 @@ import org.elasticsearch.painless.MethodWriter;
 
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
 
 /**
  * Represents an array load/store or defers to possible shortcuts.
@@ -39,7 +41,12 @@ public final class LBrace extends ALink {
     public LBrace(Location location, AExpression index) {
         super(location, 2);
 
-        this.index = index;
+        this.index = Objects.requireNonNull(index);
+    }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        index.extractVariables(variables);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallInvoke.java
index 34ad0343aa9..124755780cb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallInvoke.java
@@ -29,9 +29,11 @@ import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 import java.util.List;
+import java.util.Objects;
+import java.util.Set;
 
 /**
- * Represents a method call or deferes to a def call.
+ * Represents a method call or defers to a def call.
  */
 public final class LCallInvoke extends ALink {
 
@@ -43,8 +45,15 @@ public final class LCallInvoke extends ALink {
     public LCallInvoke(Location location, String name, List<AExpression> arguments) {
         super(location, -1);
 
-        this.name = name;
-        this.arguments = arguments;
+        this.name = Objects.requireNonNull(name);
+        this.arguments = Objects.requireNonNull(arguments);
+    }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        for (AExpression argument : arguments) {
+            argument.extractVariables(variables);
+        }
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallLocal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallLocal.java
index 4503692b48d..7aa46f004d6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallLocal.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallLocal.java
@@ -27,6 +27,8 @@ import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
 import java.util.List;
+import java.util.Objects;
+import java.util.Set;
 
 import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE;
 
@@ -43,8 +45,15 @@ public class LCallLocal extends ALink {
     public LCallLocal(Location location, String name, List<AExpression> arguments) {
         super(location, -1);
 
-        this.name = name;
-        this.arguments = arguments;
+        this.name = Objects.requireNonNull(name);
+        this.arguments = Objects.requireNonNull(arguments);
+    }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        for (AExpression argument : arguments) {
+            argument.extractVariables(variables);
+        }
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
index bf900cedafc..86dbdda24c4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCast.java
@@ -23,6 +23,9 @@ import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Cast;
+
+import java.util.Set;
+
 import org.elasticsearch.painless.AnalyzerCaster;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
@@ -41,6 +44,9 @@ public final class LCast extends ALink {
 
         this.type = type;
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {}
 
     @Override
     ALink analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index 02fbb2000f2..7c47103be37 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -25,6 +25,10 @@ import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Type;
+
+import java.util.Objects;
+import java.util.Set;
+
 import org.elasticsearch.painless.MethodWriter;
 
 import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
@@ -39,7 +43,12 @@ final class LDefArray extends ALink implements IDefLink {
     LDefArray(Location location, AExpression index) {
         super(location, 2);
 
-        this.index = index;
+        this.index = Objects.requireNonNull(index);
+    }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        index.extractVariables(variables);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index 716b26d6687..d216eacae19 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -29,6 +29,8 @@ import org.objectweb.asm.Type;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
+import java.util.Set;
 
 import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
 
@@ -45,8 +47,15 @@ final class LDefCall extends ALink implements IDefLink {
     LDefCall(Location location, String name, List<AExpression> arguments) {
         super(location, -1);
 
-        this.name = name;
-        this.arguments = arguments;
+        this.name = Objects.requireNonNull(name);
+        this.arguments = Objects.requireNonNull(arguments);
+    }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        for (AExpression argument : arguments) {
+            argument.extractVariables(variables);
+        }
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index 125b2bee6fc..6f9f7e24bd2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -25,6 +25,10 @@ import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Type;
+
+import java.util.Objects;
+import java.util.Set;
+
 import org.elasticsearch.painless.MethodWriter;
 
 import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
@@ -39,9 +43,11 @@ final class LDefField extends ALink implements IDefLink {
     LDefField(Location location, String value) {
         super(location, 1);
 
-        this.value = value;
+        this.value = Objects.requireNonNull(value);
     }
 
+    @Override
+    void extractVariables(Set<String> variables) {}
 
     @Override
     ALink analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
index 9fc48c009a2..30d0a033746 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LField.java
@@ -30,6 +30,8 @@ import org.elasticsearch.painless.MethodWriter;
 
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
 
 /**
  * Represents a field load/store or defers to a possible shortcuts.
@@ -43,8 +45,11 @@ public final class LField extends ALink {
     public LField(Location location, String value) {
         super(location, 1);
 
-        this.value = value;
+        this.value = Objects.requireNonNull(value);
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {}
 
     @Override
     ALink analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
index 1cd5620cc17..f2863ce3396 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
@@ -24,6 +24,10 @@ import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
+
+import java.util.Objects;
+import java.util.Set;
+
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -39,7 +43,12 @@ final class LListShortcut extends ALink {
     LListShortcut(Location location, AExpression index) {
         super(location, 2);
 
-        this.index = index;
+        this.index = Objects.requireNonNull(index);
+    }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        index.extractVariables(variables);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
index 05c2192b870..3bc9ab57a37 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
@@ -24,6 +24,10 @@ import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
+
+import java.util.Objects;
+import java.util.Set;
+
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -39,7 +43,12 @@ final class LMapShortcut extends ALink {
     LMapShortcut(Location location, AExpression index) {
         super(location, 2);
 
-        this.index = index;
+        this.index = Objects.requireNonNull(index);
+    }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        index.extractVariables(variables);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
index 312cbf925e1..cf83599210f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
@@ -27,6 +27,8 @@ import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 import java.util.List;
+import java.util.Objects;
+import java.util.Set;
 
 /**
  * Represents an array instantiation.
@@ -39,8 +41,15 @@ public final class LNewArray extends ALink {
     public LNewArray(Location location, String type, List<AExpression> arguments) {
         super(location, -1);
 
-        this.type = type;
-        this.arguments = arguments;
+        this.type = Objects.requireNonNull(type);
+        this.arguments = Objects.requireNonNull(arguments);
+    }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        for (AExpression argument : arguments) {
+            argument.extractVariables(variables);
+        }
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
index 0b28a308e19..0ac2762f336 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewObj.java
@@ -29,6 +29,8 @@ import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
 import java.util.List;
+import java.util.Objects;
+import java.util.Set;
 
 /**
  * Represents and object instantiation.
@@ -43,8 +45,15 @@ public final class LNewObj extends ALink {
     public LNewObj(Location location, String type, List<AExpression> arguments) {
         super(location, -1);
 
-        this.type = type;
-        this.arguments = arguments;
+        this.type = Objects.requireNonNull(type);
+        this.arguments = Objects.requireNonNull(arguments);
+    }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        for (AExpression argument : arguments) {
+            argument.extractVariables(variables);
+        }
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LRegex.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LRegex.java
index 8727f992242..c737c761cf2 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LRegex.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LRegex.java
@@ -24,6 +24,7 @@ import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 
+import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
 
@@ -54,6 +55,9 @@ public final class LRegex extends ALink {
             throw createError(e);
         }
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {}
 
     @Override
     ALink analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
index efaf02bb601..7f97b446382 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
@@ -25,6 +25,10 @@ import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Struct;
+
+import java.util.Objects;
+import java.util.Set;
+
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
@@ -41,8 +45,11 @@ final class LShortcut extends ALink {
     LShortcut(Location location, String value) {
         super(location, 1);
 
-        this.value = value;
+        this.value = Objects.requireNonNull(value);
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {}
 
     @Override
     ALink analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java
index 3a7c9a35fe5..d9077a4446f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LStatic.java
@@ -23,6 +23,10 @@ import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+
+import java.util.Objects;
+import java.util.Set;
+
 import org.elasticsearch.painless.Locals;
 
 /**
@@ -35,8 +39,11 @@ public final class LStatic extends ALink {
     public LStatic(Location location, String type) {
         super(location, 0);
 
-        this.type = type;
+        this.type = Objects.requireNonNull(type);
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {}
 
     @Override
     ALink analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
index fac9d413ad7..ab45c0f3a8e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LString.java
@@ -25,6 +25,9 @@ import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
+import java.util.Objects;
+import java.util.Set;
+
 /**
  * Represents a string constant.
  */
@@ -33,8 +36,11 @@ public final class LString extends ALink {
     public LString(Location location, String string) {
         super(location, -1);
 
-        this.string = string;
+        this.string = Objects.requireNonNull(string);
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {}
 
     @Override
     ALink analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
index ad9ba9dfbf6..4f016349c24 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LVariable.java
@@ -26,6 +26,9 @@ import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Locals.Variable;
 import org.objectweb.asm.Opcodes;
 
+import java.util.Objects;
+import java.util.Set;
+
 /**
  * Represents a variable load/store.
  */
@@ -38,7 +41,12 @@ public final class LVariable extends ALink {
     public LVariable(Location location, String name) {
         super(location, 0);
 
-        this.name = name;
+        this.name = Objects.requireNonNull(name);
+    }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        variables.add(name);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
index 7d459f27504..e688f0acec0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBlock.java
@@ -26,6 +26,7 @@ import org.elasticsearch.painless.MethodWriter;
 
 import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 
 /**
  * Represents a set of statements as a branch of control-flow.
@@ -39,6 +40,13 @@ public final class SBlock extends AStatement {
 
         this.statements = Collections.unmodifiableList(statements);
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        for (AStatement statement : statements) {
+            statement.extractVariables(variables);
+        }
+    }
 
     @Override
     void analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
index c791faeadd6..62560d1bd4e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SBreak.java
@@ -24,6 +24,8 @@ import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
+import java.util.Set;
+
 /**
  * Represents a break statement.
  */
@@ -32,6 +34,9 @@ public final class SBreak extends AStatement {
     public SBreak(Location location) {
         super(location);
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {}
 
     @Override
     void analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
index 8632c0eea6d..783af3d0022 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java
@@ -27,6 +27,10 @@ import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Locals.Variable;
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
+
+import java.util.Objects;
+import java.util.Set;
+
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -47,10 +51,18 @@ public final class SCatch extends AStatement {
     public SCatch(Location location, String type, String name, SBlock block) {
         super(location);
 
-        this.type = type;
-        this.name = name;
+        this.type = Objects.requireNonNull(type);
+        this.name = Objects.requireNonNull(name);
         this.block = block;
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        variables.add(name);
+        if (block != null) {
+            block.extractVariables(variables);
+        }
+    }
 
     @Override
     void analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
index e70f625bb3d..98ac804cc3a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SContinue.java
@@ -24,6 +24,8 @@ import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
+import java.util.Set;
+
 /**
  * Represents a continue statement.
  */
@@ -32,6 +34,9 @@ public final class SContinue extends AStatement {
     public SContinue(Location location) {
         super(location);
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {}
 
     @Override
     void analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
index 58ff09e45d8..d0dae5a68ac 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclBlock.java
@@ -26,6 +26,7 @@ import org.elasticsearch.painless.MethodWriter;
 
 import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 
 /**
  * Represents a series of declarations.
@@ -39,6 +40,13 @@ public final class SDeclBlock extends AStatement {
 
         this.declarations = Collections.unmodifiableList(declarations);
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        for (SDeclaration declaration : declarations) {
+            declaration.extractVariables(variables);
+        }
+    }
 
     @Override
     void analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
index c19adfd699e..de856722a0a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java
@@ -26,6 +26,10 @@ import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Locals.Variable;
 import org.objectweb.asm.Opcodes;
+
+import java.util.Objects;
+import java.util.Set;
+
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -42,10 +46,18 @@ public final class SDeclaration extends AStatement {
     public SDeclaration(Location location, String type, String name, AExpression expression) {
         super(location);
 
-        this.type = type;
-        this.name = name;
+        this.type = Objects.requireNonNull(type);
+        this.name = Objects.requireNonNull(name);
         this.expression = expression;
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        variables.add(name);
+        if (expression != null) {
+            expression.extractVariables(variables);
+        }
+    }
 
     @Override
     void analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
index 3b94f5799ea..e214703daf4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -24,6 +24,10 @@ import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
+
+import java.util.Objects;
+import java.util.Set;
+
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -37,9 +41,17 @@ public final class SDo extends AStatement {
     public SDo(Location location, SBlock block, AExpression condition) {
         super(location);
 
-        this.condition = condition;
+        this.condition = Objects.requireNonNull(condition);
         this.block = block;
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        condition.extractVariables(variables);
+        if (block != null) {
+            block.extractVariables(variables);
+        }
+    }
 
     @Override
     void analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index 403b377ea9c..3469f51639b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -35,6 +35,9 @@ import org.elasticsearch.painless.Locals.Variable;
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
 
+import java.util.Objects;
+import java.util.Set;
+
 import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
 import static org.elasticsearch.painless.WriterConstants.ITERATOR_HASNEXT;
 import static org.elasticsearch.painless.WriterConstants.ITERATOR_NEXT;
@@ -66,11 +69,20 @@ public class SEach extends AStatement {
     public SEach(Location location, String type, String name, AExpression expression, SBlock block) {
         super(location);
 
-        this.type = type;
-        this.name = name;
-        this.expression = expression;
+        this.type = Objects.requireNonNull(type);
+        this.name = Objects.requireNonNull(name);
+        this.expression = Objects.requireNonNull(expression);
         this.block = block;
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        variables.add(name);
+        expression.extractVariables(variables);
+        if (block != null) {
+            block.extractVariables(variables);
+        }
+    }
 
     @Override
     void analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
index c6cf0a47c8f..94eb180ea03 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SExpression.java
@@ -20,6 +20,10 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition.Type;
+
+import java.util.Objects;
+import java.util.Set;
+
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Globals;
@@ -36,7 +40,12 @@ public final class SExpression extends AStatement {
     public SExpression(Location location, AExpression expression) {
         super(location);
 
-        this.expression = expression;
+        this.expression = Objects.requireNonNull(expression);
+    }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        expression.extractVariables(variables);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index b2cf1da2594..06ad19d204d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -24,6 +24,9 @@ import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
+
+import java.util.Set;
+
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -44,6 +47,22 @@ public final class SFor extends AStatement {
         this.afterthought = afterthought;
         this.block = block;
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        if (initializer != null) {
+            initializer.extractVariables(variables);
+        }
+        if (condition != null) {
+            condition.extractVariables(variables);
+        }
+        if (afterthought != null) {
+            afterthought.extractVariables(variables);
+        }
+        if (block != null) {
+            block.extractVariables(variables);
+        }
+    }
 
     @Override
     void analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index 8d3db7f811a..629f76cdd52 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -41,6 +41,8 @@ import java.lang.reflect.Modifier;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
+import java.util.Set;
 
 import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE;
 
@@ -67,14 +69,24 @@ public class SFunction extends AStatement {
                      List<String> paramNames, List<AStatement> statements, boolean synthetic) {
         super(location);
 
-        this.reserved = reserved;
-        this.rtnTypeStr = rtnType;
-        this.name = name;
+        this.reserved = Objects.requireNonNull(reserved);
+        this.rtnTypeStr = Objects.requireNonNull(rtnType);
+        this.name = Objects.requireNonNull(name);
         this.paramTypeStrs = Collections.unmodifiableList(paramTypes);
         this.paramNameStrs = Collections.unmodifiableList(paramNames);
         this.statements = Collections.unmodifiableList(statements);
         this.synthetic = synthetic;
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        for (String paramName : paramNameStrs) {
+            variables.add(paramName);
+        }
+        for (AStatement statement : statements) {
+            statement.extractVariables(variables);
+        }
+    }
 
     void generate() {
         try {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
index 1b0c0ffb348..eb862b1177a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIf.java
@@ -24,6 +24,10 @@ import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
+
+import java.util.Objects;
+import java.util.Set;
+
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -37,9 +41,17 @@ public final class SIf extends AStatement {
     public SIf(Location location, AExpression condition, SBlock ifblock) {
         super(location);
 
-        this.condition = condition;
+        this.condition = Objects.requireNonNull(condition);
         this.ifblock = ifblock;
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        condition.extractVariables(variables);
+        if (ifblock != null) {
+            ifblock.extractVariables(variables);
+        }
+    }
 
     @Override
     void analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
index 612969acfff..8296ddcada0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SIfElse.java
@@ -24,6 +24,10 @@ import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
+
+import java.util.Objects;
+import java.util.Set;
+
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -38,10 +42,21 @@ public final class SIfElse extends AStatement {
     public SIfElse(Location location, AExpression condition, SBlock ifblock, SBlock elseblock) {
         super(location);
 
-        this.condition = condition;
+        this.condition = Objects.requireNonNull(condition);
         this.ifblock = ifblock;
         this.elseblock = elseblock;
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        condition.extractVariables(variables);
+        if (ifblock != null) {
+            ifblock.extractVariables(variables);
+        }
+        if (elseblock != null) {
+            elseblock.extractVariables(variables);
+        }
+    }
 
     @Override
     void analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
index 2c46b3ecfaf..36e629ca6d8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SReturn.java
@@ -24,6 +24,9 @@ import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
+import java.util.Objects;
+import java.util.Set;
+
 /**
  * Represents a return statement.
  */
@@ -34,7 +37,12 @@ public final class SReturn extends AStatement {
     public SReturn(Location location, AExpression expression) {
         super(location);
 
-        this.expression = expression;
+        this.expression = Objects.requireNonNull(expression);
+    }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        expression.extractVariables(variables);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index 5e932072ce1..1a47cfeba36 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -43,6 +43,8 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
 
 import static org.elasticsearch.painless.WriterConstants.BASE_CLASS_TYPE;
 import static org.elasticsearch.painless.WriterConstants.CLASS_TYPE;
@@ -71,10 +73,10 @@ public final class SSource extends AStatement {
                    List<SFunction> functions, Globals globals, List<AStatement> statements) {
         super(location);
 
-        this.name = name;
-        this.source = source;
+        this.name = Objects.requireNonNull(name);
+        this.source = Objects.requireNonNull(source);
         this.debugStream = debugStream;
-        this.reserved = reserved;
+        this.reserved = Objects.requireNonNull(reserved);
         // process any synthetic functions generated by walker (because right now, thats still easy)
         functions.addAll(globals.getSyntheticMethods().values());
         globals.getSyntheticMethods().clear();
@@ -82,6 +84,16 @@ public final class SSource extends AStatement {
         this.statements = Collections.unmodifiableList(statements);
         this.globals = globals;
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        for (AStatement statement : statements) {
+            statement.extractVariables(variables);
+        }
+        for (SFunction function : functions) {
+            function.extractVariables(variables);
+        }
+    }
 
     public void analyze() {
         Map<MethodKey, Method> methods = new HashMap<>();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
index ffa1b89dfe2..c78e0503f30 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SThrow.java
@@ -25,6 +25,9 @@ import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
+import java.util.Objects;
+import java.util.Set;
+
 /**
  * Represents a throw statement.
  */
@@ -35,7 +38,12 @@ public final class SThrow extends AStatement {
     public SThrow(Location location, AExpression expression) {
         super(location);
 
-        this.expression = expression;
+        this.expression = Objects.requireNonNull(expression);
+    }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        expression.extractVariables(variables);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
index 0295d32ae35..1986b5eeb66 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/STry.java
@@ -27,6 +27,7 @@ import org.elasticsearch.painless.MethodWriter;
 
 import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 
 /**
  * Represents the try block as part of a try-catch block.
@@ -42,6 +43,16 @@ public final class STry extends AStatement {
         this.block = block;
         this.catches = Collections.unmodifiableList(catches);
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        if (block != null) {
+            block.extractVariables(variables);
+        }
+        for (SCatch expr : catches) {
+            expr.extractVariables(variables);
+        }
+    }
 
     @Override
     void analyze(Locals locals) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
index 39b48781b46..dff8ae2592f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -24,6 +24,10 @@ import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
+
+import java.util.Objects;
+import java.util.Set;
+
 import org.elasticsearch.painless.MethodWriter;
 
 /**
@@ -37,9 +41,17 @@ public final class SWhile extends AStatement {
     public SWhile(Location location, AExpression condition, SBlock block) {
         super(location);
 
-        this.condition = condition;
+        this.condition = Objects.requireNonNull(condition);
         this.block = block;
     }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        condition.extractVariables(variables);
+        if (block != null) {
+            block.extractVariables(variables);
+        }
+    }
 
     @Override
     void analyze(Locals locals) {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
index dada9a9bc37..22abb8f5255 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
@@ -103,4 +103,34 @@ public class LambdaTests extends ScriptTestCase {
                                "}" +
                                "return sum;"));
     }
+    
+    public void testCapture() {
+        assertEquals(5, exec("int x = 5; return Optional.empty().orElseGet(() -> x);"));
+    }
+    
+    public void testTwoCaptures() {
+        assertEquals("1test", exec("int x = 1; String y = 'test'; return Optional.empty().orElseGet(() -> x + y);"));
+    }
+    
+    public void testCapturesAreReadOnly() {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("List l = new ArrayList(); l.add(1); l.add(1); "
+                    + "return l.stream().mapToInt(x -> { l = null; return x + 1 }).sum();");
+        });
+        assertTrue(expected.getMessage().contains("is read-only"));
+    }
+    
+    public void testOnlyCapturesAreReadOnly() {
+        assertEquals(4, exec("List l = new ArrayList(); l.add(1); l.add(1); "
+                           + "return l.stream().mapToInt(x -> { x += 1; return x }).sum();"));
+    }
+    
+    public void testCaptureDef() {
+        assertEquals(5, exec("int x = 5; def y = Optional.empty(); y.orElseGet(() -> x);"));
+    }
+    
+    public void testNestedCapture() {
+        assertEquals(1, exec("boolean x = false; int y = 1;" +
+                             "return Optional.empty().orElseGet(() -> x ? 5 : Optional.empty().orElseGet(() -> y));"));
+    }
 }

From 4cb2bb635b580eaa6377d53aa1e8c6f742698452 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Sat, 18 Jun 2016 17:06:18 +0200
Subject: [PATCH 0943/1311] Remove forked joda time BaseDateTime class (#18953)

This class was forked in 0.20 to remove a volatile keyword. While there
is no issue attached to the commit, no evidence of the criticality of the
change nor does it seem to be correct since we set this value internally as well
I think this class should be used as is from joda time even if we have to pay
the price of volatile reads. We can't do 3rd party optimization in our codebase that
way it just not maintainable.

This was added in 2280915d3cc3e9fe0fe5fab57ae57774e9044af7
---
 .../java/org/joda/time/base/BaseDateTime.java | 345 ------------------
 1 file changed, 345 deletions(-)
 delete mode 100644 core/src/main/java/org/joda/time/base/BaseDateTime.java

diff --git a/core/src/main/java/org/joda/time/base/BaseDateTime.java b/core/src/main/java/org/joda/time/base/BaseDateTime.java
deleted file mode 100644
index 29c34246249..00000000000
--- a/core/src/main/java/org/joda/time/base/BaseDateTime.java
+++ /dev/null
@@ -1,345 +0,0 @@
-/*
- *  Copyright 2001-2011 Stephen Colebourne
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.joda.time.base;
-
-import org.joda.time.Chronology;
-import org.joda.time.DateTimeUtils;
-import org.joda.time.DateTimeZone;
-import org.joda.time.ReadableDateTime;
-import org.joda.time.chrono.ISOChronology;
-import org.joda.time.convert.ConverterManager;
-import org.joda.time.convert.InstantConverter;
-
-/**
- * BaseDateTime is an abstract implementation of ReadableDateTime that stores
- * data in <code>long</code> and <code>Chronology</code> fields.
- * <p>
- * This class should generally not be used directly by API users.
- * The {@link ReadableDateTime} interface should be used when different
- * kinds of date/time objects are to be referenced.
- * <p>
- * BaseDateTime subclasses may be mutable and not thread-safe.
- *
- * @author Stephen Colebourne
- * @author Kandarp Shah
- * @author Brian S O'Neill
- * @since 1.0
- */
-public abstract class BaseDateTime
-        extends AbstractDateTime
-        implements ReadableDateTime {
-    /**
-     * The millis from 1970-01-01T00:00:00Z
-     */
-    // THIS IS THE ES CHANGE not to have it volatile...
-    private long iMillis;
-    /**
-     * The chronology to use
-     */
-    private volatile Chronology iChronology;
-
-    //-----------------------------------------------------------------------
-
-    /**
-     * Constructs an instance set to the current system millisecond time
-     * using <code>ISOChronology</code> in the default time zone.
-     */
-    public BaseDateTime() {
-        this(DateTimeUtils.currentTimeMillis(), ISOChronology.getInstance());
-    }
-
-    /**
-     * Constructs an instance set to the current system millisecond time
-     * using <code>ISOChronology</code> in the specified time zone.
-     * <p>
-     * If the specified time zone is null, the default zone is used.
-     *
-     * @param zone the time zone, null means default zone
-     */
-    public BaseDateTime(DateTimeZone zone) {
-        this(DateTimeUtils.currentTimeMillis(), ISOChronology.getInstance(zone));
-    }
-
-    /**
-     * Constructs an instance set to the current system millisecond time
-     * using the specified chronology.
-     * <p>
-     * If the chronology is null, <code>ISOChronology</code>
-     * in the default time zone is used.
-     *
-     * @param chronology the chronology, null means ISOChronology in default zone
-     */
-    public BaseDateTime(Chronology chronology) {
-        this(DateTimeUtils.currentTimeMillis(), chronology);
-    }
-
-    //-----------------------------------------------------------------------
-
-    /**
-     * Constructs an instance set to the milliseconds from 1970-01-01T00:00:00Z
-     * using <code>ISOChronology</code> in the default time zone.
-     *
-     * @param instant the milliseconds from 1970-01-01T00:00:00Z
-     */
-    public BaseDateTime(long instant) {
-        this(instant, ISOChronology.getInstance());
-    }
-
-    /**
-     * Constructs an instance set to the milliseconds from 1970-01-01T00:00:00Z
-     * using <code>ISOChronology</code> in the specified time zone.
-     * <p>
-     * If the specified time zone is null, the default zone is used.
-     *
-     * @param instant the milliseconds from 1970-01-01T00:00:00Z
-     * @param zone    the time zone, null means default zone
-     */
-    public BaseDateTime(long instant, DateTimeZone zone) {
-        this(instant, ISOChronology.getInstance(zone));
-    }
-
-    /**
-     * Constructs an instance set to the milliseconds from 1970-01-01T00:00:00Z
-     * using the specified chronology.
-     * <p>
-     * If the chronology is null, <code>ISOChronology</code>
-     * in the default time zone is used.
-     *
-     * @param instant    the milliseconds from 1970-01-01T00:00:00Z
-     * @param chronology the chronology, null means ISOChronology in default zone
-     */
-    public BaseDateTime(long instant, Chronology chronology) {
-        super();
-        iChronology = checkChronology(chronology);
-        iMillis = checkInstant(instant, iChronology);
-        // validate not over maximum
-        if (iChronology.year().isSupported()) {
-            iChronology.year().set(iMillis, iChronology.year().get(iMillis));
-        }
-    }
-
-    //-----------------------------------------------------------------------
-
-    /**
-     * Constructs an instance from an Object that represents a datetime,
-     * forcing the time zone to that specified.
-     * <p>
-     * If the object contains no chronology, <code>ISOChronology</code> is used.
-     * If the specified time zone is null, the default zone is used.
-     * <p>
-     * The recognised object types are defined in
-     * {@link org.joda.time.convert.ConverterManager ConverterManager} and
-     * include ReadableInstant, String, Calendar and Date.
-     *
-     * @param instant the datetime object
-     * @param zone    the time zone
-     * @throws IllegalArgumentException if the instant is invalid
-     */
-    public BaseDateTime(Object instant, DateTimeZone zone) {
-        super();
-        InstantConverter converter = ConverterManager.getInstance().getInstantConverter(instant);
-        Chronology chrono = checkChronology(converter.getChronology(instant, zone));
-        iChronology = chrono;
-        iMillis = checkInstant(converter.getInstantMillis(instant, chrono), chrono);
-    }
-
-    /**
-     * Constructs an instance from an Object that represents a datetime,
-     * using the specified chronology.
-     * <p>
-     * If the chronology is null, ISO in the default time zone is used.
-     * <p>
-     * The recognised object types are defined in
-     * {@link org.joda.time.convert.ConverterManager ConverterManager} and
-     * include ReadableInstant, String, Calendar and Date.
-     *
-     * @param instant    the datetime object
-     * @param chronology the chronology
-     * @throws IllegalArgumentException if the instant is invalid
-     */
-    public BaseDateTime(Object instant, Chronology chronology) {
-        super();
-        InstantConverter converter = ConverterManager.getInstance().getInstantConverter(instant);
-        iChronology = checkChronology(converter.getChronology(instant, chronology));
-        iMillis = checkInstant(converter.getInstantMillis(instant, chronology), iChronology);
-    }
-
-    //-----------------------------------------------------------------------
-
-    /**
-     * Constructs an instance from datetime field values
-     * using <code>ISOChronology</code> in the default time zone.
-     *
-     * @param year           the year
-     * @param monthOfYear    the month of the year
-     * @param dayOfMonth     the day of the month
-     * @param hourOfDay      the hour of the day
-     * @param minuteOfHour   the minute of the hour
-     * @param secondOfMinute the second of the minute
-     * @param millisOfSecond the millisecond of the second
-     */
-    public BaseDateTime(
-            int year,
-            int monthOfYear,
-            int dayOfMonth,
-            int hourOfDay,
-            int minuteOfHour,
-            int secondOfMinute,
-            int millisOfSecond) {
-        this(year, monthOfYear, dayOfMonth, hourOfDay,
-                minuteOfHour, secondOfMinute, millisOfSecond, ISOChronology.getInstance());
-    }
-
-    /**
-     * Constructs an instance from datetime field values
-     * using <code>ISOChronology</code> in the specified time zone.
-     * <p>
-     * If the specified time zone is null, the default zone is used.
-     *
-     * @param year           the year
-     * @param monthOfYear    the month of the year
-     * @param dayOfMonth     the day of the month
-     * @param hourOfDay      the hour of the day
-     * @param minuteOfHour   the minute of the hour
-     * @param secondOfMinute the second of the minute
-     * @param millisOfSecond the millisecond of the second
-     * @param zone           the time zone, null means default time zone
-     */
-    public BaseDateTime(
-            int year,
-            int monthOfYear,
-            int dayOfMonth,
-            int hourOfDay,
-            int minuteOfHour,
-            int secondOfMinute,
-            int millisOfSecond,
-            DateTimeZone zone) {
-        this(year, monthOfYear, dayOfMonth, hourOfDay,
-                minuteOfHour, secondOfMinute, millisOfSecond, ISOChronology.getInstance(zone));
-    }
-
-    /**
-     * Constructs an instance from datetime field values
-     * using the specified chronology.
-     * <p>
-     * If the chronology is null, <code>ISOChronology</code>
-     * in the default time zone is used.
-     *
-     * @param year           the year
-     * @param monthOfYear    the month of the year
-     * @param dayOfMonth     the day of the month
-     * @param hourOfDay      the hour of the day
-     * @param minuteOfHour   the minute of the hour
-     * @param secondOfMinute the second of the minute
-     * @param millisOfSecond the millisecond of the second
-     * @param chronology     the chronology, null means ISOChronology in default zone
-     */
-    public BaseDateTime(
-            int year,
-            int monthOfYear,
-            int dayOfMonth,
-            int hourOfDay,
-            int minuteOfHour,
-            int secondOfMinute,
-            int millisOfSecond,
-            Chronology chronology) {
-        super();
-        iChronology = checkChronology(chronology);
-        long instant = iChronology.getDateTimeMillis(year, monthOfYear, dayOfMonth,
-                hourOfDay, minuteOfHour, secondOfMinute, millisOfSecond);
-        iMillis = checkInstant(instant, iChronology);
-    }
-
-    //-----------------------------------------------------------------------
-
-    /**
-     * Checks the specified chronology before storing it, potentially altering it.
-     * This method must not access any instance variables.
-     * <p>
-     * This implementation converts nulls to ISOChronology in the default zone.
-     *
-     * @param chronology the chronology to use, may be null
-     * @return the chronology to store in this datetime, not null
-     */
-    protected Chronology checkChronology(Chronology chronology) {
-        return DateTimeUtils.getChronology(chronology);
-    }
-
-    /**
-     * Checks the specified instant before storing it, potentially altering it.
-     * This method must not access any instance variables.
-     * <p>
-     * This implementation simply returns the instant.
-     *
-     * @param instant    the milliseconds from 1970-01-01T00:00:00Z to round
-     * @param chronology the chronology to use, not null
-     * @return the instant to store in this datetime
-     */
-    protected long checkInstant(long instant, Chronology chronology) {
-        return instant;
-    }
-
-    //-----------------------------------------------------------------------
-
-    /**
-     * Gets the milliseconds of the datetime instant from the Java epoch
-     * of 1970-01-01T00:00:00Z.
-     *
-     * @return the number of milliseconds since 1970-01-01T00:00:00Z
-     */
-    @Override
-    public long getMillis() {
-        return iMillis;
-    }
-
-    /**
-     * Gets the chronology of the datetime.
-     *
-     * @return the Chronology that the datetime is using
-     */
-    @Override
-    public Chronology getChronology() {
-        return iChronology;
-    }
-
-    //-----------------------------------------------------------------------
-
-    /**
-     * Sets the milliseconds of the datetime.
-     * <p>
-     * All changes to the millisecond field occurs via this method.
-     * Override and block this method to make a subclass immutable.
-     *
-     * @param instant the milliseconds since 1970-01-01T00:00:00Z to set the datetime to
-     */
-    protected void setMillis(long instant) {
-        iMillis = checkInstant(instant, iChronology);
-    }
-
-    /**
-     * Sets the chronology of the datetime.
-     * <p>
-     * All changes to the chronology field occurs via this method.
-     * Override and block this method to make a subclass immutable.
-     *
-     * @param chronology the chronology to set
-     */
-    protected void setChronology(Chronology chronology) {
-        iChronology = checkChronology(chronology);
-    }
-
-}

From 535a245354c1dd57b46adee0d3c6ffdb8d8c2e63 Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Sat, 18 Jun 2016 18:44:54 +0200
Subject: [PATCH 0944/1311] testSingleBatchSubmission - random numbers don't
 have to be unique

---
 .../elasticsearch/cluster/service/ClusterServiceTests.java  | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
index 306eecf81b5..66f96f8cd3a 100644
--- a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
@@ -23,7 +23,6 @@ import org.apache.log4j.Logger;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateTaskConfig;
 import org.elasticsearch.cluster.ClusterStateTaskExecutor;
@@ -33,7 +32,6 @@ import org.elasticsearch.cluster.NodeConnectionsService;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.OperationRouting;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.lease.Releasable;
@@ -382,7 +380,7 @@ public class ClusterServiceTests extends ESTestCase {
         final int numOfTasks = randomInt(10);
         final CountDownLatch latch = new CountDownLatch(numOfTasks);
         for (int i = 0; i < numOfTasks; i++) {
-            tasks.put(randomInt(1024), new ClusterStateTaskListener() {
+            while (null != tasks.put(randomInt(1024), new ClusterStateTaskListener() {
                 @Override
                 public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                     latch.countDown();
@@ -392,7 +390,7 @@ public class ClusterServiceTests extends ESTestCase {
                 public void onFailure(String source, Throwable t) {
                     fail(ExceptionsHelper.detailedMessage(t));
                 }
-            });
+            })) ;
         }
 
         clusterService.submitStateUpdateTasks("test", tasks, ClusterStateTaskConfig.build(Priority.LANGUID),

From 14cd8a6794825d3903dd9b54a67261f4f26bf692 Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Sat, 18 Jun 2016 18:53:47 +0200
Subject: [PATCH 0945/1311] Introduce Replication unit tests using real shards
 (#18930)

This commit introduce unit testing infrastructure to test replication operations using real index shards. This is infra is complementary to the full integration tests and unit testing of ReplicationOperation we already have. The new ESIndexLevelReplicationTestCase base makes it easier to test and simulate failure mode that require real shards and but do not need the full blow stack of a complete node.

The commit also add a simple "nothing is wrong" test plus a test that checks we don't drop docs during the various stages of recovery.

For now, only single doc indexing is supported but this can be easily extended in the future.
---
 .../replication/ReplicationOperation.java     |  18 +-
 .../ESIndexLevelReplicationTestCase.java      | 474 ++++++++++++++++++
 .../IndexLevelReplicationTests.java           |  31 ++
 .../RecoveryDuringReplicationTests.java       | 120 +++++
 .../org/elasticsearch/test/ESTestCase.java    |   7 +-
 5 files changed, 640 insertions(+), 10 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
 create mode 100644 core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
 create mode 100644 core/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java

diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
index 8442e705257..0d99c4bfcdf 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
@@ -78,11 +78,11 @@ public class ReplicationOperation<
 
     private final List<ReplicationResponse.ShardInfo.Failure> shardReplicaFailures = Collections.synchronizedList(new ArrayList<>());
 
-    ReplicationOperation(Request request, Primary<Request, ReplicaRequest, PrimaryResultT> primary,
-                         ActionListener<PrimaryResultT> listener,
-                         boolean executeOnReplicas, boolean checkWriteConsistency,
-                         Replicas<ReplicaRequest> replicas,
-                         Supplier<ClusterState> clusterStateSupplier, ESLogger logger, String opType) {
+    public ReplicationOperation(Request request, Primary<Request, ReplicaRequest, PrimaryResultT> primary,
+                                ActionListener<PrimaryResultT> listener,
+                                boolean executeOnReplicas, boolean checkWriteConsistency,
+                                Replicas<ReplicaRequest> replicas,
+                                Supplier<ClusterState> clusterStateSupplier, ESLogger logger, String opType) {
         this.checkWriteConsistency = checkWriteConsistency;
         this.executeOnReplicas = executeOnReplicas;
         this.replicasProxy = replicas;
@@ -94,7 +94,7 @@ public class ReplicationOperation<
         this.opType = opType;
     }
 
-    void execute() throws Exception {
+    public void execute() throws Exception {
         final String writeConsistencyFailure = checkWriteConsistency ? checkWriteConsistency() : null;
         final ShardRouting primaryRouting = primary.routingEntry();
         final ShardId primaryId = primaryRouting.shardId();
@@ -294,7 +294,7 @@ public class ReplicationOperation<
     }
 
 
-    interface Primary<
+    public interface Primary<
                 Request extends ReplicationRequest<Request>,
                 ReplicaRequest extends ReplicationRequest<ReplicaRequest>,
                 PrimaryResultT extends PrimaryResult<ReplicaRequest>
@@ -322,7 +322,7 @@ public class ReplicationOperation<
 
     }
 
-    interface Replicas<ReplicaRequest extends ReplicationRequest<ReplicaRequest>> {
+    public interface Replicas<ReplicaRequest extends ReplicationRequest<ReplicaRequest>> {
 
         /**
          * performs the the given request on the specified replica
@@ -366,7 +366,7 @@ public class ReplicationOperation<
         }
     }
 
-    interface PrimaryResult<R extends ReplicationRequest<R>> {
+    public interface PrimaryResult<R extends ReplicationRequest<R>> {
 
         R replicaRequest();
 
diff --git a/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
new file mode 100644
index 00000000000..7ccf0a3e642
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
@@ -0,0 +1,474 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.replication;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.flush.FlushRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.action.index.TransportIndexAction;
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.action.support.replication.ReplicationOperation;
+import org.elasticsearch.action.support.replication.ReplicationResponse;
+import org.elasticsearch.action.support.replication.TransportWriteAction;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.ShardRoutingHelper;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.common.collect.Iterators;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.MapperTestUtils;
+import org.elasticsearch.index.cache.IndexCache;
+import org.elasticsearch.index.cache.query.DisabledQueryCache;
+import org.elasticsearch.index.engine.Engine;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.Uid;
+import org.elasticsearch.index.mapper.internal.UidFieldMapper;
+import org.elasticsearch.index.shard.IndexEventListener;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.shard.ShardPath;
+import org.elasticsearch.index.similarity.SimilarityService;
+import org.elasticsearch.index.store.DirectoryService;
+import org.elasticsearch.index.store.Store;
+import org.elasticsearch.indices.recovery.RecoveryFailedException;
+import org.elasticsearch.indices.recovery.RecoverySourceHandler;
+import org.elasticsearch.indices.recovery.RecoveryState;
+import org.elasticsearch.indices.recovery.RecoveryTarget;
+import org.elasticsearch.indices.recovery.RecoveryTargetService;
+import org.elasticsearch.indices.recovery.StartRecoveryRequest;
+import org.elasticsearch.test.DummyShardLock;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.junit.annotations.TestLogging;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportResponse;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.BiFunction;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.equalTo;
+
+public class ESIndexLevelReplicationTestCase extends ESTestCase {
+
+    private ThreadPool threadPool;
+    final private Index index = new Index("test", "uuid");
+    final private ShardId shardId = new ShardId(index, 0);
+    final private Map<String, String> indexMapping = Collections.singletonMap("type", "{ \"type\": {} }");
+    protected final static RecoveryTargetService.RecoveryListener recoveryListener = new RecoveryTargetService.RecoveryListener() {
+        @Override
+        public void onRecoveryDone(RecoveryState state) {
+
+        }
+
+        @Override
+        public void onRecoveryFailure(RecoveryState state, RecoveryFailedException e, boolean sendShardFailure) {
+            fail(ExceptionsHelper.detailedMessage(e));
+        }
+    };
+
+
+    @TestLogging("index.shard:TRACE,index.replication:TRACE,indices.recovery:TRACE")
+    public void testIndexingDuringFileRecovery() throws Exception {
+        try (ReplicationGroup shards = createGroup(randomInt(1))) {
+            shards.startAll();
+            int docs = shards.indexDocs(randomInt(50));
+            shards.flush();
+            IndexShard replica = shards.addReplica();
+            final CountDownLatch recoveryBlocked = new CountDownLatch(1);
+            final CountDownLatch releaseRecovery = new CountDownLatch(1);
+            final Future<Void> recoveryFuture = shards.asyncRecoverReplica(replica, new BiFunction<IndexShard, DiscoveryNode, RecoveryTarget>() {
+                @Override
+                public RecoveryTarget apply(IndexShard indexShard, DiscoveryNode node) {
+                    return new RecoveryTarget(indexShard, node, recoveryListener) {
+                        @Override
+                        public void renameAllTempFiles() throws IOException {
+                            super.renameAllTempFiles();
+                            recoveryBlocked.countDown();
+                            try {
+                                releaseRecovery.await();
+                            } catch (InterruptedException e) {
+                                throw new IOException("terminated by interrupt", e);
+                            }
+                        }
+                    };
+                }
+            });
+
+            recoveryBlocked.await();
+            docs += shards.indexDocs(randomInt(20));
+            releaseRecovery.countDown();
+            recoveryFuture.get();
+
+            shards.assertAllEqual(docs);
+        }
+    }
+
+
+    @Before
+    public void setup() {
+        threadPool = new TestThreadPool(getClass().getName());
+    }
+
+    @After
+    public void destroy() {
+        ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS);
+    }
+
+    private Store createStore(IndexSettings indexSettings, ShardPath shardPath) throws IOException {
+        final ShardId shardId = shardPath.getShardId();
+        final DirectoryService directoryService = new DirectoryService(shardId, indexSettings) {
+            @Override
+            public Directory newDirectory() throws IOException {
+                return newFSDirectory(shardPath.resolveIndex());
+            }
+
+            @Override
+            public long throttleTimeInNanos() {
+                return 0;
+            }
+        };
+        return new Store(shardId, indexSettings, directoryService, new DummyShardLock(shardId));
+    }
+
+    protected ReplicationGroup createGroup(int replicas) throws IOException {
+        final Path homePath = createTempDir();
+        Settings build = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, replicas)
+            .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+            .build();
+        IndexMetaData metaData = IndexMetaData.builder(index.getName()).settings(build).primaryTerm(0, 1).build();
+        return new ReplicationGroup(metaData, homePath);
+    }
+
+    private DiscoveryNode getDiscoveryNode(String id) {
+        return new DiscoveryNode(id, id, DummyTransportAddress.INSTANCE, Collections.emptyMap(),
+            Collections.singleton(DiscoveryNode.Role.DATA), Version.CURRENT);
+    }
+
+    private IndexShard newShard(boolean primary, DiscoveryNode node, IndexMetaData indexMetaData, Path homePath) throws IOException {
+        // add node name to settings for propper logging
+        final Settings nodeSettings = Settings.builder().put("node.name", node.getName()).build();
+        final IndexSettings indexSettings = new IndexSettings(indexMetaData, nodeSettings);
+        ShardRouting shardRouting = TestShardRouting.newShardRouting(shardId, node.getId(), primary, ShardRoutingState.INITIALIZING);
+        final Path path = Files.createDirectories(homePath.resolve(node.getId()));
+        final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(path);
+        ShardPath shardPath = new ShardPath(false, nodePath.resolve(shardId), nodePath.resolve(shardId), shardId);
+        Store store = createStore(indexSettings, shardPath);
+        IndexCache indexCache = new IndexCache(indexSettings, new DisabledQueryCache(indexSettings), null);
+        MapperService mapperService = MapperTestUtils.newMapperService(homePath, indexSettings.getSettings());
+        for (Map.Entry<String, String> type : indexMapping.entrySet()) {
+            mapperService.merge(type.getKey(), new CompressedXContent(type.getValue()), MapperService.MergeReason.MAPPING_RECOVERY, true);
+        }
+        SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
+        final IndexEventListener indexEventListener = new IndexEventListener() {
+        };
+        final Engine.Warmer warmer = searcher -> {
+        };
+        return new IndexShard(shardRouting, indexSettings, shardPath, store, indexCache, mapperService, similarityService, null, null,
+            indexEventListener, null, threadPool, BigArrays.NON_RECYCLING_INSTANCE, warmer, Collections.emptyList(),
+            Collections.emptyList());
+    }
+
+
+    class ReplicationGroup implements AutoCloseable, Iterable<IndexShard> {
+        private final IndexShard primary;
+        private final List<IndexShard> replicas;
+        private final IndexMetaData indexMetaData;
+        private final Path homePath;
+        private final AtomicInteger replicaId = new AtomicInteger();
+        private final AtomicInteger docId = new AtomicInteger();
+        boolean closed = false;
+
+        ReplicationGroup(final IndexMetaData indexMetaData, Path homePath) throws IOException {
+            primary = newShard(true, getDiscoveryNode("s0"), indexMetaData, homePath);
+            replicas = new ArrayList<>();
+            this.indexMetaData = indexMetaData;
+            this.homePath = homePath;
+            for (int i = 0; i < indexMetaData.getNumberOfReplicas(); i++) {
+                addReplica();
+            }
+
+        }
+
+        public int indexDocs(final int numOfDoc) throws Exception {
+            for (int doc = 0; doc < numOfDoc; doc++) {
+                final IndexRequest indexRequest = new IndexRequest(index.getName(), "type", Integer.toString(docId.incrementAndGet()))
+                    .source("{}");
+                final IndexResponse response = index(indexRequest);
+                assertThat(response.isCreated(), equalTo(true));
+            }
+            return numOfDoc;
+        }
+
+        public IndexResponse index(IndexRequest indexRequest) throws Exception {
+            PlainActionFuture<IndexingResult> listener = new PlainActionFuture<>();
+            IndexingOp op = new IndexingOp(indexRequest, listener, this);
+            op.execute();
+            return listener.get().finalResponse;
+        }
+
+        public synchronized void startAll() throws IOException {
+            final DiscoveryNode pNode = getDiscoveryNode(primary.routingEntry().currentNodeId());
+            primary.markAsRecovering("store", new RecoveryState(primary.shardId(), true, RecoveryState.Type.STORE, pNode, pNode));
+            primary.recoverFromStore();
+            primary.updateRoutingEntry(ShardRoutingHelper.moveToStarted(primary.routingEntry()));
+            for (IndexShard replicaShard : replicas) {
+                recoverReplica(replicaShard, (replica, sourceNode) -> new RecoveryTarget(replica, sourceNode, recoveryListener));
+            }
+        }
+
+        public synchronized IndexShard addReplica() throws IOException {
+            final IndexShard replica = newShard(false, getDiscoveryNode("s" + replicaId.incrementAndGet()), indexMetaData, homePath);
+            replicas.add(replica);
+            return replica;
+        }
+
+        public void recoverReplica(IndexShard replica, BiFunction<IndexShard, DiscoveryNode, RecoveryTarget> targetSupplier)
+            throws IOException {
+            final DiscoveryNode pNode;
+            synchronized (this) {
+                pNode = getDiscoveryNode(primary.routingEntry().currentNodeId());
+            }
+            final DiscoveryNode rNode = getDiscoveryNode(replica.routingEntry().currentNodeId());
+            replica.markAsRecovering("remote", new RecoveryState(replica.shardId(), false, RecoveryState.Type.REPLICA, pNode, rNode));
+            replica.prepareForIndexRecovery();
+            RecoveryTarget recoveryTarget = targetSupplier.apply(replica, pNode);
+            StartRecoveryRequest request = new StartRecoveryRequest(replica.shardId(), pNode, rNode,
+                replica.store().getMetadataOrEmpty(), RecoveryState.Type.REPLICA, 0);
+            RecoverySourceHandler recovery = new RecoverySourceHandler(primary, recoveryTarget, request, (int) ByteSizeUnit.MB.toKB(1),
+                logger);
+            recovery.recoverToTarget();
+            recoveryTarget.markAsDone();
+            replica.updateRoutingEntry(ShardRoutingHelper.moveToStarted(replica.routingEntry()));
+        }
+
+        public Future<Void> asyncRecoverReplica(IndexShard replica, BiFunction<IndexShard, DiscoveryNode, RecoveryTarget> targetSupplier)
+            throws IOException {
+            FutureTask<Void> task = new FutureTask<>(() -> {
+                recoverReplica(replica, targetSupplier);
+                return null;
+            });
+            threadPool.generic().execute(task);
+            return task;
+        }
+
+        public synchronized void assertAllEqual(int expectedCount) throws IOException {
+            Set<Uid> primaryIds = getShardDocUIDs(primary);
+            assertThat(primaryIds.size(), equalTo(expectedCount));
+            for (IndexShard replica : replicas) {
+                Set<Uid> replicaIds = getShardDocUIDs(replica);
+                Set<Uid> temp = new HashSet<>(primaryIds);
+                temp.removeAll(replicaIds);
+                assertThat(replica.routingEntry() + " is missing docs", temp, empty());
+                temp = new HashSet<>(replicaIds);
+                temp.removeAll(primaryIds);
+                assertThat(replica.routingEntry() + " has extra docs", temp, empty());
+            }
+        }
+
+        private Set<Uid> getShardDocUIDs(final IndexShard shard) throws IOException {
+            shard.refresh("get_uids");
+            try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
+                Set<Uid> ids = new HashSet<>();
+                for (LeafReaderContext leafContext : searcher.reader().leaves()) {
+                    LeafReader reader = leafContext.reader();
+                    Bits liveDocs = reader.getLiveDocs();
+                    for (int i = 0; i < reader.maxDoc(); i++) {
+                        if (liveDocs == null || liveDocs.get(i)) {
+                            Document uuid = reader.document(i, Collections.singleton(UidFieldMapper.NAME));
+                            ids.add(Uid.createUid(uuid.get(UidFieldMapper.NAME)));
+                        }
+                    }
+                }
+                return ids;
+            }
+        }
+
+        public synchronized void refresh(String source) {
+            for (IndexShard shard : this) {
+                shard.refresh(source);
+            }
+        }
+
+        public synchronized void flush() {
+            final FlushRequest request = new FlushRequest();
+            for (IndexShard shard : this) {
+                shard.flush(request);
+            }
+        }
+
+        public synchronized List<ShardRouting> shardRoutings() {
+            return StreamSupport.stream(this.spliterator(), false).map(IndexShard::routingEntry).collect(Collectors.toList());
+        }
+
+        @Override
+        public synchronized void close() throws Exception {
+            if (closed == false) {
+                closed = true;
+                for (IndexShard shard : this) {
+                    shard.close("eol", false);
+                    IOUtils.close(shard.store());
+                }
+            } else {
+                throw new AlreadyClosedException("too bad");
+            }
+        }
+
+        @Override
+        public Iterator<IndexShard> iterator() {
+            return Iterators.<IndexShard>concat(replicas.iterator(), Collections.singleton(primary).iterator());
+        }
+    }
+
+    class IndexingOp extends ReplicationOperation<IndexRequest, IndexRequest, IndexingResult> {
+
+        private final ReplicationGroup replicationGroup;
+
+        public IndexingOp(IndexRequest request, ActionListener<IndexingResult> listener, ReplicationGroup replicationGroup) {
+            super(request, new PrimaryRef(replicationGroup), listener, true, false, new ReplicasRef(replicationGroup),
+                () -> null, logger, "indexing");
+            this.replicationGroup = replicationGroup;
+            request.process(null, true, request.index());
+        }
+
+        @Override
+        protected List<ShardRouting> getShards(ShardId shardId, ClusterState state) {
+            return replicationGroup.shardRoutings();
+        }
+
+    }
+
+    private static class PrimaryRef implements ReplicationOperation.Primary<IndexRequest, IndexRequest, IndexingResult> {
+        final IndexShard primary;
+
+        private PrimaryRef(ReplicationGroup replicationGroup) {
+            this.primary = replicationGroup.primary;
+        }
+
+        @Override
+        public ShardRouting routingEntry() {
+            return primary.routingEntry();
+        }
+
+        @Override
+        public void failShard(String message, Throwable throwable) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public IndexingResult perform(IndexRequest request) throws Exception {
+            TransportWriteAction.WriteResult<IndexResponse> result = TransportIndexAction.executeIndexRequestOnPrimary(request, primary,
+                null);
+            request.primaryTerm(primary.getPrimaryTerm());
+            return new IndexingResult(request, result.getResponse());
+        }
+
+    }
+
+    private static class ReplicasRef implements ReplicationOperation.Replicas<IndexRequest> {
+        private final ReplicationGroup replicationGroup;
+
+        private ReplicasRef(ReplicationGroup replicationGroup) {
+            this.replicationGroup = replicationGroup;
+        }
+
+        @Override
+        public void performOn(ShardRouting replicaRouting, IndexRequest request, ActionListener<TransportResponse.Empty> listener) {
+            try {
+                IndexShard replica = replicationGroup.replicas.stream()
+                    .filter(s -> replicaRouting.isSameAllocation(s.routingEntry())).findFirst().get();
+                TransportIndexAction.executeIndexRequestOnReplica(request, replica);
+                listener.onResponse(TransportResponse.Empty.INSTANCE);
+            } catch (Throwable t) {
+                listener.onFailure(t);
+            }
+        }
+
+        @Override
+        public void failShard(ShardRouting replica, ShardRouting primary, String message, Throwable throwable, Runnable onSuccess, Consumer<Throwable> onPrimaryDemoted, Consumer<Throwable> onIgnoredFailure) {
+            throw new UnsupportedOperationException();
+        }
+    }
+
+
+    private static class IndexingResult implements ReplicationOperation.PrimaryResult<IndexRequest> {
+        final IndexRequest replicaRequest;
+        final IndexResponse finalResponse;
+
+        public IndexingResult(IndexRequest replicaRequest, IndexResponse finalResponse) {
+            this.replicaRequest = replicaRequest;
+            this.finalResponse = finalResponse;
+        }
+
+        @Override
+        public IndexRequest replicaRequest() {
+            return replicaRequest;
+        }
+
+        @Override
+        public void setShardInfo(ReplicationResponse.ShardInfo shardInfo) {
+            finalResponse.setShardInfo(shardInfo);
+        }
+
+        public void respond(ActionListener<IndexResponse> listener) {
+            listener.onResponse(finalResponse);
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java b/core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
new file mode 100644
index 00000000000..c6d7878406a
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.replication;
+
+public class IndexLevelReplicationTests extends ESIndexLevelReplicationTestCase {
+
+    public void testSimpleReplication() throws Exception {
+        try (ReplicationGroup shards = createGroup(randomInt(2))) {
+            shards.startAll();
+            final int docCount = randomInt(50);
+            shards.indexDocs(docCount);
+            shards.assertAllEqual(docCount);
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java b/core/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
new file mode 100644
index 00000000000..815884edf51
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.replication;
+
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.store.Store;
+import org.elasticsearch.index.translog.Translog;
+import org.elasticsearch.indices.recovery.RecoveryState;
+import org.elasticsearch.indices.recovery.RecoveryTarget;
+import org.elasticsearch.indices.recovery.RecoveryTargetService;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
+
+public class RecoveryDuringReplicationTests extends ESIndexLevelReplicationTestCase {
+
+    public void testIndexingDuringFileRecovery() throws Exception {
+        try (ReplicationGroup shards = createGroup(randomInt(1))) {
+            shards.startAll();
+            int docs = shards.indexDocs(randomInt(50));
+            shards.flush();
+            IndexShard replica = shards.addReplica();
+            final CountDownLatch recoveryBlocked = new CountDownLatch(1);
+            final CountDownLatch releaseRecovery = new CountDownLatch(1);
+            final RecoveryState.Stage blockOnStage = randomFrom(BlockingTarget.SUPPORTED_STAGES);
+            final Future<Void> recoveryFuture = shards.asyncRecoverReplica(replica, (indexShard, node) ->
+                new BlockingTarget(blockOnStage, recoveryBlocked, releaseRecovery, indexShard, node, recoveryListener, logger));
+
+            recoveryBlocked.await();
+            docs += shards.indexDocs(randomInt(20));
+            releaseRecovery.countDown();
+            recoveryFuture.get();
+
+            shards.assertAllEqual(docs);
+        }
+    }
+
+    private static class BlockingTarget extends RecoveryTarget {
+        private final CountDownLatch recoveryBlocked;
+        private final CountDownLatch releaseRecovery;
+        private final RecoveryState.Stage stageToBlock;
+        public static final EnumSet<RecoveryState.Stage> SUPPORTED_STAGES =
+            EnumSet.of(RecoveryState.Stage.INDEX, RecoveryState.Stage.TRANSLOG, RecoveryState.Stage.FINALIZE);
+        private final ESLogger logger;
+
+        BlockingTarget(RecoveryState.Stage stageToBlock, CountDownLatch recoveryBlocked, CountDownLatch releaseRecovery, IndexShard shard,
+                       DiscoveryNode sourceNode, RecoveryTargetService.RecoveryListener listener, ESLogger logger) {
+            super(shard, sourceNode, listener);
+            this.recoveryBlocked = recoveryBlocked;
+            this.releaseRecovery = releaseRecovery;
+            this.stageToBlock = stageToBlock;
+            this.logger = logger;
+            if (SUPPORTED_STAGES.contains(stageToBlock) == false) {
+                throw new UnsupportedOperationException(stageToBlock + " is not supported");
+            }
+        }
+
+        private boolean hasBlocked() {
+            return recoveryBlocked.getCount() == 0;
+        }
+
+        private void blockIfNeeded(RecoveryState.Stage currentStage) {
+            if (currentStage == stageToBlock) {
+                logger.info("--> blocking recovery on stage [{}]", currentStage);
+                recoveryBlocked.countDown();
+                try {
+                    releaseRecovery.await();
+                    logger.info("--> recovery continues from stage [{}]", currentStage);
+                } catch (InterruptedException e) {
+                    throw new RuntimeException("blockage released");
+                }
+            }
+        }
+
+        @Override
+        public void indexTranslogOperations(List<Translog.Operation> operations, int totalTranslogOps) {
+            if (hasBlocked() == false) {
+                blockIfNeeded(RecoveryState.Stage.TRANSLOG);
+            }
+            super.indexTranslogOperations(operations, totalTranslogOps);
+        }
+
+        @Override
+        public void cleanFiles(int totalTranslogOps, Store.MetadataSnapshot sourceMetaData) throws IOException {
+            blockIfNeeded(RecoveryState.Stage.INDEX);
+            super.cleanFiles(totalTranslogOps, sourceMetaData);
+        }
+
+        @Override
+        public void finalizeRecovery() {
+            if (hasBlocked() == false) {
+                // it maybe that not ops have been transferred, block now
+                blockIfNeeded(RecoveryState.Stage.TRANSLOG);
+            }
+            blockIfNeeded(RecoveryState.Stage.FINALIZE);
+            super.finalizeRecovery();
+        }
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index e0ae77d3f56..172c99592df 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -36,7 +36,6 @@ import org.apache.lucene.util.TestUtil;
 import org.apache.lucene.util.TimeUnits;
 import org.elasticsearch.Version;
 import org.elasticsearch.bootstrap.BootstrapForTesting;
-import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -47,6 +46,7 @@ import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -334,6 +334,11 @@ public abstract class ESTestCase extends LuceneTestCase {
         return RandomPicks.randomFrom(random(), list);
     }
 
+    /** Pick a random object from the given collection. */
+    public static <T> T randomFrom(Collection<T> collection) {
+        return RandomPicks.randomFrom(random(), collection);
+    }
+
     public static String randomAsciiOfLengthBetween(int minCodeUnits, int maxCodeUnits) {
         return RandomizedTest.randomAsciiOfLengthBetween(minCodeUnits, maxCodeUnits);
     }

From 61b7f49ed25b483a10dff5a56f7cbb992342e200 Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Sat, 18 Jun 2016 19:13:50 +0200
Subject: [PATCH 0946/1311] ESIndexLevelReplicationTestCase.java fix line
 lengths

---
 .../index/replication/ESIndexLevelReplicationTestCase.java  | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
index 7ccf0a3e642..924469ca6cb 100644
--- a/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
@@ -132,7 +132,8 @@ public class ESIndexLevelReplicationTestCase extends ESTestCase {
             IndexShard replica = shards.addReplica();
             final CountDownLatch recoveryBlocked = new CountDownLatch(1);
             final CountDownLatch releaseRecovery = new CountDownLatch(1);
-            final Future<Void> recoveryFuture = shards.asyncRecoverReplica(replica, new BiFunction<IndexShard, DiscoveryNode, RecoveryTarget>() {
+            final Future<Void> recoveryFuture = shards.asyncRecoverReplica(replica,
+                new BiFunction<IndexShard, DiscoveryNode, RecoveryTarget>() {
                 @Override
                 public RecoveryTarget apply(IndexShard indexShard, DiscoveryNode node) {
                     return new RecoveryTarget(indexShard, node, recoveryListener) {
@@ -441,7 +442,8 @@ public class ESIndexLevelReplicationTestCase extends ESTestCase {
         }
 
         @Override
-        public void failShard(ShardRouting replica, ShardRouting primary, String message, Throwable throwable, Runnable onSuccess, Consumer<Throwable> onPrimaryDemoted, Consumer<Throwable> onIgnoredFailure) {
+        public void failShard(ShardRouting replica, ShardRouting primary, String message, Throwable throwable, Runnable onSuccess,
+                              Consumer<Throwable> onPrimaryDemoted, Consumer<Throwable> onIgnoredFailure) {
             throw new UnsupportedOperationException();
         }
     }

From 9bbf3fc1d93d70facc9cee08b3f9de1dba7a18b0 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Sat, 18 Jun 2016 10:27:48 -0700
Subject: [PATCH 0947/1311] Build: Require exactly gradle 2.13

see #18935
---
 buildSrc/build.gradle | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
index a5a9b4209ba..47396096ab8 100644
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -23,6 +23,16 @@ apply plugin: 'groovy'
 
 group = 'org.elasticsearch.gradle'
 
+// TODO: remove this when upgrading to a version that supports ProgressLogger
+// gradle 2.14 made internal apis unavailable to plugins, and gradle considered
+// ProgressLogger to be an internal api. Until this is made available again,
+// we can't upgrade without losing our nice progress logging
+// NOTE that this check duplicates that in BuildPlugin, but we need to check
+// early here before trying to compile the broken classes in buildSrc
+if (GradleVersion.current() != GradleVersion.version('2.13')) {
+  throw new GradleException('Gradle 2.13 is required to build elasticsearch')
+}
+
 if (project == rootProject) {
   // change the build dir used during build init, so that doing a clean
   // won't wipe out the buildscript jar

From 8f96dd53aa5e220daad63c85ecfad7f28db274d8 Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Sat, 18 Jun 2016 19:39:37 +0200
Subject: [PATCH 0948/1311] mark ESIndexLevelReplicationTestCase as abstract so
 it won't fail naming conventions

---
 .../index/replication/ESIndexLevelReplicationTestCase.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
index 924469ca6cb..edf5a0d28f2 100644
--- a/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
@@ -104,7 +104,7 @@ import java.util.stream.StreamSupport;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 
-public class ESIndexLevelReplicationTestCase extends ESTestCase {
+public abstract class ESIndexLevelReplicationTestCase extends ESTestCase {
 
     private ThreadPool threadPool;
     final private Index index = new Index("test", "uuid");

From 8d9fa7e0b56635c084dc8e443f446d8f34d60f59 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Sun, 19 Jun 2016 03:19:45 -0400
Subject: [PATCH 0949/1311] Fix explicit casts and improve tests.

---
 .../painless/CompilerSettings.java            |  26 ++
 .../java/org/elasticsearch/painless/Def.java  |   3 +-
 .../elasticsearch/painless/DefBootstrap.java  |  72 +++--
 .../org/elasticsearch/painless/DefMath.java   | 121 +++----
 .../elasticsearch/painless/MethodWriter.java  |  49 +--
 .../painless/PainlessScriptEngineService.java |   6 +
 .../painless/WriterConstants.java             |   7 +-
 .../elasticsearch/painless/antlr/Walker.java  |   2 +-
 .../elasticsearch/painless/node/EBinary.java  |  11 +-
 .../painless/node/ECapturingFunctionRef.java  |   5 +-
 .../elasticsearch/painless/node/EChain.java   |   8 +-
 .../elasticsearch/painless/node/EComp.java    |  15 +-
 .../elasticsearch/painless/node/EUnary.java   |  16 +-
 .../painless/node/LDefArray.java              |  10 +-
 .../elasticsearch/painless/node/LDefCall.java |  20 +-
 .../painless/node/LDefField.java              |  10 +-
 .../elasticsearch/painless/node/SEach.java    |   5 +-
 .../painless/node/SFunction.java              |   5 +-
 .../elasticsearch/painless/node/SSource.java  |  17 +-
 .../org/elasticsearch/painless/CastTests.java | 298 ++++++++++++++++++
 .../painless/DefBootstrapTests.java           |  10 +
 .../painless/DefOptimizationTests.java        |  12 +-
 .../painless/ScriptTestCase.java              |   1 +
 23 files changed, 572 insertions(+), 157 deletions(-)
 create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/CastTests.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java
index 79b75b32b98..94d7a1305d6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java
@@ -33,6 +33,11 @@ public final class CompilerSettings {
      * Constant to be used for enabling additional internal compilation checks (slower).
      */
     public static final String PICKY = "picky";
+    
+    /**
+     * For testing: do not use.
+     */
+    public static final String INITIAL_CALL_SITE_DEPTH = "initialCallSiteDepth";
 
     /**
      * The maximum number of statements allowed to be run in a loop.
@@ -44,6 +49,11 @@ public final class CompilerSettings {
      * makes things slower too, it is only for debugging.
      */
     private boolean picky = false;
+    
+    /**
+     * For testing. Do not use.
+     */
+    private int initialCallSiteDepth = 0;
 
     /**
      * Returns the value for the cumulative total number of statements that can be made in all loops
@@ -78,4 +88,20 @@ public final class CompilerSettings {
     public void setPicky(boolean picky) {
       this.picky = picky;
     }
+    
+    /**
+     * Returns initial call site depth. This means we pretend we've already seen N different types,
+     * to better exercise fallback code in tests.
+     */
+    public int getInitialCallSiteDepth() {
+        return initialCallSiteDepth;
+    }
+    
+    /**
+     * For testing megamorphic fallbacks. Do not use.
+     * @see #getInitialCallSiteDepth()
+     */
+    public void setInitialCallSiteDepth(int depth) {
+        this.initialCallSiteDepth = depth;
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index a4033255b77..7c7ff5f7f5a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -279,7 +279,8 @@ public final class Def {
                      MethodType nestedType = MethodType.methodType(interfaceType.clazz, captures);
                      CallSite nested = DefBootstrap.bootstrap(lookup, 
                                                               call,
-                                                              nestedType, 
+                                                              nestedType,
+                                                              0,
                                                               DefBootstrap.REFERENCE,
                                                               interfaceType.name);
                      filter = nested.dynamicInvoker();
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index a60985eec67..fa533eff213 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -27,6 +27,7 @@ import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodHandles.Lookup;
 import java.lang.invoke.MethodType;
 import java.lang.invoke.MutableCallSite;
+import java.lang.invoke.WrongMethodTypeException;
 
 /**
  * Painless invokedynamic bootstrap for the call site.
@@ -80,9 +81,18 @@ public final class DefBootstrap {
     
     /**
      * static bootstrap parameter indicating the binary operator is part of compound assignment (e.g. +=).
-     * 
+     * <p>
+     * may require {@link MethodHandles#explicitCastArguments}, or a dynamic cast
+     * to cast back to the receiver's type, depending on types seen.
      */
     public static final int OPERATOR_COMPOUND_ASSIGNMENT = 1 << 1;
+    
+    /**
+     * static bootstrap parameter indicating an explicit cast to the return type.
+     * <p>
+     * may require {@link MethodHandles#explicitCastArguments}, depending on types seen.
+     */
+    public static final int OPERATOR_EXPLICIT_CAST = 1 << 2;
 
     /**
      * CallSite that implements the polymorphic inlining cache (PIC).
@@ -97,7 +107,7 @@ public final class DefBootstrap {
         private final Object[] args;
         int depth; // pkg-protected for testing
 
-        PIC(Lookup lookup, String name, MethodType type, int flavor, Object[] args) {
+        PIC(Lookup lookup, String name, MethodType type, int initialDepth, int flavor, Object[] args) {
             super(type);
             if (type.parameterType(0) != Object.class) {
                 throw new BootstrapMethodError("The receiver type (1st arg) of invokedynamic descriptor must be Object.");
@@ -106,6 +116,7 @@ public final class DefBootstrap {
             this.name = name;
             this.flavor = flavor;
             this.args = args;
+            this.depth = initialDepth;
 
             MethodHandle fallback = FALLBACK.bindTo(this)
               .asCollector(Object[].class, type.parameterCount())
@@ -226,11 +237,14 @@ public final class DefBootstrap {
         private final int flavor;
         private final int flags;
 
-        MIC(String name, MethodType type, int flavor, int flags) {
+        MIC(String name, MethodType type, int initialDepth, int flavor, int flags) {
             super(type);
             this.name = name;
             this.flavor = flavor;
             this.flags = flags;
+            if (initialDepth > 0) {
+                initialized = true;
+            }
             
             MethodHandle fallback = FALLBACK.bindTo(this)
               .asCollector(Object[].class, type.parameterCount())
@@ -248,7 +262,9 @@ public final class DefBootstrap {
                 case SHIFT_OPERATOR:
                     // shifts are treated as unary, as java allows long arguments without a cast (but bits are ignored)
                     MethodHandle unary = DefMath.lookupUnary(args[0].getClass(), name);
-                    if ((flags & OPERATOR_COMPOUND_ASSIGNMENT) != 0) {
+                    if ((flags & OPERATOR_EXPLICIT_CAST) != 0) {
+                        unary = DefMath.cast(type().returnType(), unary);
+                    } else if ((flags & OPERATOR_COMPOUND_ASSIGNMENT) != 0) {
                         unary = DefMath.cast(args[0].getClass(), unary);
                     }
                     return unary;
@@ -257,7 +273,9 @@ public final class DefBootstrap {
                         return lookupGeneric(); // can handle nulls, casts if supported
                     } else {
                         MethodHandle binary = DefMath.lookupBinary(args[0].getClass(), args[1].getClass(), name);
-                        if ((flags & OPERATOR_COMPOUND_ASSIGNMENT) != 0) {
+                        if ((flags & OPERATOR_EXPLICIT_CAST) != 0) {
+                            binary = DefMath.cast(type().returnType(), binary);
+                        } else if ((flags & OPERATOR_COMPOUND_ASSIGNMENT) != 0) {
                             binary = DefMath.cast(args[0].getClass(), binary);
                         }
                         return binary;
@@ -267,11 +285,15 @@ public final class DefBootstrap {
         }
         
         private MethodHandle lookupGeneric() {
-            if ((flags & OPERATOR_COMPOUND_ASSIGNMENT) != 0) {
-                return DefMath.lookupGenericWithCast(name);
-            } else {
-                return DefMath.lookupGeneric(name);
+            MethodHandle target = DefMath.lookupGeneric(name);
+            if ((flags & OPERATOR_EXPLICIT_CAST) != 0) {
+                // static cast to the return type
+                target = DefMath.dynamicCast(target, type().returnType());
+            } else if ((flags & OPERATOR_COMPOUND_ASSIGNMENT) != 0) {
+                // dynamic cast to the receiver's type
+                target = DefMath.dynamicCast(target);
             }
+            return target;
         }
         
         /**
@@ -288,7 +310,15 @@ public final class DefBootstrap {
             }
             
             final MethodType type = type();
-            final MethodHandle target = lookup(args).asType(type);
+            MethodHandle target = lookup(args);
+            // for math operators: WrongMethodType can be confusing. convert into a ClassCastException if they screw up.
+            try {
+                target = target.asType(type);
+            } catch (WrongMethodTypeException e) {
+                Exception exc = new ClassCastException("Cannot cast from: " + target.type().returnType() + " to " + type.returnType());
+                exc.initCause(e);
+                throw exc;
+            }
 
             final MethodHandle test;
             if (flavor == BINARY_OPERATOR || flavor == SHIFT_OPERATOR) {
@@ -384,12 +414,16 @@ public final class DefBootstrap {
     /**
      * invokeDynamic bootstrap method
      * <p>
-     * In addition to ordinary parameters, we also take a static parameter {@code flavor} which
-     * tells us what type of dynamic call it is (and which part of whitelist to look at).
+     * In addition to ordinary parameters, we also take some static parameters:
+     * <ul>
+     *   <li>{@code initialDepth}: initial call site depth. this is used to exercise megamorphic fallback.
+     *   <li>{@code flavor}: type of dynamic call it is (and which part of whitelist to look at).
+     *   <li>{@code args}: flavor-specific args.
+     * </ul>
      * <p>
      * see https://docs.oracle.com/javase/specs/jvms/se7/html/jvms-6.html#jvms-6.5.invokedynamic
      */
-    public static CallSite bootstrap(Lookup lookup, String name, MethodType type, int flavor, Object... args) {
+    public static CallSite bootstrap(Lookup lookup, String name, MethodType type, int initialDepth, int flavor, Object... args) {
         // validate arguments
         switch(flavor) {
             // "function-call" like things get a polymorphic cache
@@ -408,7 +442,7 @@ public final class DefBootstrap {
                 if (args.length != numLambdas + 1) {
                     throw new BootstrapMethodError("Illegal number of parameters: expected " + numLambdas + " references");
                 }
-                return new PIC(lookup, name, type, flavor, args);
+                return new PIC(lookup, name, type, initialDepth, flavor, args);
             case LOAD:
             case STORE:
             case ARRAY_LOAD:
@@ -417,7 +451,7 @@ public final class DefBootstrap {
                 if (args.length > 0) {
                     throw new BootstrapMethodError("Illegal static bootstrap parameters for flavor: " + flavor);
                 }
-                return new PIC(lookup, name, type, flavor, args);
+                return new PIC(lookup, name, type, initialDepth, flavor, args);
             case REFERENCE:
                 if (args.length != 1) {
                     throw new BootstrapMethodError("Invalid number of parameters for reference call");
@@ -425,7 +459,7 @@ public final class DefBootstrap {
                 if (args[0] instanceof String == false) {
                     throw new BootstrapMethodError("Illegal parameter for reference call: " + args[0]);
                 }
-                return new PIC(lookup, name, type, flavor, args);
+                return new PIC(lookup, name, type, initialDepth, flavor, args);
 
             // operators get monomorphic cache, with a generic impl for a fallback
             case UNARY_OPERATOR:
@@ -442,11 +476,11 @@ public final class DefBootstrap {
                     // we just don't need it anywhere else.
                     throw new BootstrapMethodError("This parameter is only supported for BINARY_OPERATORs");
                 }
-                if ((flags & OPERATOR_COMPOUND_ASSIGNMENT) != 0 && flavor != BINARY_OPERATOR) {
+                if ((flags & OPERATOR_COMPOUND_ASSIGNMENT) != 0 && flavor != BINARY_OPERATOR && flavor != SHIFT_OPERATOR) {
                     // we just don't need it anywhere else.
-                    throw new BootstrapMethodError("This parameter is only supported for BINARY_OPERATORs");
+                    throw new BootstrapMethodError("This parameter is only supported for BINARY/SHIFT_OPERATORs");
                 }
-                return new MIC(name, type, flavor, flags);
+                return new MIC(name, type, initialDepth, flavor, flags);
             default:
                 throw new BootstrapMethodError("Illegal static bootstrap parameter for flavor: " + flavor);
         }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java
index f56256f6cb2..65a26cd3da8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java
@@ -25,6 +25,7 @@ import java.lang.invoke.MethodType;
 import java.lang.invoke.MethodHandles.Lookup;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.IdentityHashMap;
 import java.util.Map;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -1027,25 +1028,7 @@ public class DefMath {
      * class if its not a boxed type.
      */
     private static Class<?> unbox(Class<?> clazz) {
-        if (clazz == Boolean.class) {
-            return boolean.class;
-        } else if (clazz == Byte.class) { 
-            return byte.class; 
-        } else if (clazz == Short.class) { 
-            return short.class; 
-        } else if (clazz == Character.class) {
-            return char.class;
-        } else if (clazz == Integer.class) {
-            return int.class;
-        } else if (clazz == Long.class) {
-            return long.class;
-        } else if (clazz == Float.class) {
-            return float.class;
-        } else if (clazz == Double.class) {
-            return double.class;
-        } else {
-            return clazz;
-        }
+        return MethodType.methodType(clazz).unwrap().returnType();
     }
     
     /** Unary promotion. All Objects are promoted to Object. */
@@ -1146,40 +1129,50 @@ public class DefMath {
     public static MethodHandle lookupGeneric(String name) {
         return TYPE_OP_MAPPING.get(Object.class).get(name);
     }
-    
-    
+
     /**
      * Slow dynamic cast: casts {@code returnValue} to the runtime type of {@code lhs}
      * based upon inspection. If {@code lhs} is null, no cast takes place.
      * This is used for the generic fallback case of compound assignment.
      */
-    static Object dynamicCast(Object returnValue, Object lhs) {
+    static Object dynamicReceiverCast(Object returnValue, Object lhs) {
         if (lhs != null) {
-            Class<?> c = lhs.getClass();
-            if (c == returnValue.getClass()) {
-                return returnValue;
-            }
-            if (c == Integer.class) {
-                return getNumber(returnValue).intValue();
-            } else if (c == Long.class) {
-                return getNumber(returnValue).longValue();
-            } else if (c == Double.class) {
-                return getNumber(returnValue).doubleValue();
-            } else if (c == Float.class) {
-                return getNumber(returnValue).floatValue();
-            } else if (c == Short.class) {
-                return getNumber(returnValue).shortValue();
-            } else if (c == Byte.class) {
-                return getNumber(returnValue).byteValue();
-            } else if (c == Character.class) {
-                return (char) getNumber(returnValue).intValue();
-            }
-            return lhs.getClass().cast(returnValue);
+            return dynamicCast(lhs.getClass(), returnValue);
         } else {
             return returnValue;
         }
     }
     
+    /**
+     * Slow dynamic cast: casts {@code value} to an instance of {@code clazz}
+     * based upon inspection. If {@code lhs} is null, no cast takes place.
+     */
+    static Object dynamicCast(Class<?> clazz, Object value) {
+        if (value != null) {
+            if (clazz == value.getClass()) {
+                return value;
+            }
+            if (clazz == Integer.class) {
+                return getNumber(value).intValue();
+            } else if (clazz == Long.class) {
+                return getNumber(value).longValue();
+            } else if (clazz == Double.class) {
+                return getNumber(value).doubleValue();
+            } else if (clazz == Float.class) {
+                return getNumber(value).floatValue();
+            } else if (clazz == Short.class) {
+                return getNumber(value).shortValue();
+            } else if (clazz == Byte.class) {
+                return getNumber(value).byteValue();
+            } else if (clazz == Character.class) {
+                return (char) getNumber(value).intValue();
+            }
+            return clazz.cast(value);
+        } else {
+            return value;
+        }
+    }
+    
     /** Slowly returns a Number for o. Just for supporting dynamicCast */
     static Number getNumber(Object o) {
         if (o instanceof Number) {
@@ -1192,28 +1185,40 @@ public class DefMath {
     }
     
     private static final MethodHandle DYNAMIC_CAST;
+    private static final MethodHandle DYNAMIC_RECEIVER_CAST;
     static {
         final Lookup lookup = MethodHandles.lookup();
         try {
             DYNAMIC_CAST = lookup.findStatic(lookup.lookupClass(), 
-                                             "dynamicCast", 
-                                             MethodType.methodType(Object.class, Object.class, Object.class));
+                                            "dynamicCast", 
+                                            MethodType.methodType(Object.class, Class.class, Object.class));
+            DYNAMIC_RECEIVER_CAST = lookup.findStatic(lookup.lookupClass(), 
+                                                     "dynamicReceiverCast", 
+                                                     MethodType.methodType(Object.class, Object.class, Object.class));
         } catch (ReflectiveOperationException e) {
             throw new AssertionError(e);
         }
     }
 
     /** Looks up generic method, with a dynamic cast to the receiver's type. (compound assignment) */
-    public static MethodHandle lookupGenericWithCast(String name) {
-        MethodHandle generic = lookupGeneric(name);
-        // adapt dynamic cast to the generic method
-        MethodHandle cast = DYNAMIC_CAST.asType(MethodType.methodType(generic.type().returnType(), 
-                                                                      generic.type().returnType(),
-                                                                      generic.type().parameterType(0)));
+    public static MethodHandle dynamicCast(MethodHandle target) {
+        // adapt dynamic receiver cast to the generic method
+        MethodHandle cast = DYNAMIC_RECEIVER_CAST.asType(MethodType.methodType(target.type().returnType(), 
+                                                                      target.type().returnType(),
+                                                                      target.type().parameterType(0)));
         // drop the RHS parameter
-        cast = MethodHandles.dropArguments(cast, 2, generic.type().parameterType(1));
+        cast = MethodHandles.dropArguments(cast, 2, target.type().parameterType(1));
         // combine: f(x,y) -> g(f(x,y), x, y);
-        return MethodHandles.foldArguments(cast, generic);
+        return MethodHandles.foldArguments(cast, target);
+    }
+    
+    /** Looks up generic method, with a dynamic cast to the specified type. (explicit assignment) */
+    public static MethodHandle dynamicCast(MethodHandle target, Class<?> desired) {
+        // adapt dynamic cast to the generic method
+        desired = MethodType.methodType(desired).wrap().returnType();
+        // bind to the boxed type
+        MethodHandle cast = DYNAMIC_CAST.bindTo(desired);
+        return MethodHandles.filterReturnValue(target, cast);
     }
     
     /** Forces a cast to class A for target (only if types differ) */
@@ -1221,12 +1226,18 @@ public class DefMath {
         MethodType newType = MethodType.methodType(classA).unwrap();
         MethodType targetType = MethodType.methodType(target.type().returnType()).unwrap();
         
+        // don't do a conversion if types are the same. explicitCastArguments has this opto,
+        // but we do it explicitly, to make the boolean check simpler
         if (newType.returnType() == targetType.returnType()) {
-            return target; // no conversion
+            return target;
         }
         
-        // this is safe for our uses of it here only, because we change just the return value,
-        // the original method itself does all the type checks correctly.
+        // we don't allow the to/from boolean conversions of explicitCastArguments
+        if (newType.returnType() == boolean.class || targetType.returnType() == boolean.class) {
+            throw new ClassCastException("Cannot cast " + targetType.returnType() + " to " + newType.returnType());
+        }
+        
+        // null return values are not possible for our arguments.
         return MethodHandles.explicitCastArguments(target, target.type().changeReturnType(newType.returnType()));
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index 6241b5b4207..97686655a97 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -30,6 +30,7 @@ import org.objectweb.asm.commons.Method;
 
 import java.util.ArrayDeque;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.BitSet;
 import java.util.Deque;
 import java.util.List;
@@ -78,15 +79,17 @@ import static org.elasticsearch.painless.WriterConstants.UTILITY_TYPE;
  */
 public final class MethodWriter extends GeneratorAdapter {
     private final BitSet statements;
+    private final CompilerSettings settings;
 
     private final Deque<List<org.objectweb.asm.Type>> stringConcatArgs =
         (INDY_STRING_CONCAT_BOOTSTRAP_HANDLE == null) ?  null : new ArrayDeque<>();
 
-    public MethodWriter(int access, Method method, ClassVisitor cw, BitSet statements) {
+    public MethodWriter(int access, Method method, ClassVisitor cw, BitSet statements, CompilerSettings settings) {
         super(Opcodes.ASM5, cw.visitMethod(access, method.getName(), method.getDescriptor(), null, null),
                 access, method.getName(), method.getDescriptor());
 
         this.statements = statements;
+        this.settings = settings;
     }
 
     /**
@@ -269,23 +272,18 @@ public final class MethodWriter extends GeneratorAdapter {
 
     /** Writes a dynamic binary instruction: returnType, lhs, and rhs can be different */
     public void writeDynamicBinaryInstruction(Location location, Type returnType, Type lhs, Type rhs, 
-                                              Operation operation, boolean compoundAssignment) {
+                                              Operation operation, int flags) {
         org.objectweb.asm.Type methodType = org.objectweb.asm.Type.getMethodType(returnType.type, lhs.type, rhs.type);
-        String descriptor = methodType.getDescriptor();
         
-        int flags = 0;
-        if (compoundAssignment) {
-            flags |= DefBootstrap.OPERATOR_COMPOUND_ASSIGNMENT;
-        }
         switch (operation) {
             case MUL:
-                invokeDynamic("mul", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, flags); 
+                invokeDefCall("mul", methodType, DefBootstrap.BINARY_OPERATOR, flags); 
                 break;
             case DIV:
-                invokeDynamic("div", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, flags); 
+                invokeDefCall("div", methodType, DefBootstrap.BINARY_OPERATOR, flags); 
                 break;
             case REM:
-                invokeDynamic("rem", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, flags); 
+                invokeDefCall("rem", methodType, DefBootstrap.BINARY_OPERATOR, flags); 
                 break;
             case ADD:
                 // if either side is primitive, then the + operator should always throw NPE on null,
@@ -295,28 +293,28 @@ public final class MethodWriter extends GeneratorAdapter {
                 if (!hasPrimitiveArg) {
                     flags |= DefBootstrap.OPERATOR_ALLOWS_NULL;
                 }
-                invokeDynamic("add", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, flags);
+                invokeDefCall("add", methodType, DefBootstrap.BINARY_OPERATOR, flags);
                 break;
             case SUB:
-                invokeDynamic("sub", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, flags); 
+                invokeDefCall("sub", methodType, DefBootstrap.BINARY_OPERATOR, flags); 
                 break;
             case LSH:
-                invokeDynamic("lsh", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR, flags);
+                invokeDefCall("lsh", methodType, DefBootstrap.SHIFT_OPERATOR, flags);
                 break;
             case USH:
-                invokeDynamic("ush", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR, flags); 
+                invokeDefCall("ush", methodType, DefBootstrap.SHIFT_OPERATOR, flags); 
                 break;
             case RSH:
-                invokeDynamic("rsh", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.SHIFT_OPERATOR, flags); 
+                invokeDefCall("rsh", methodType, DefBootstrap.SHIFT_OPERATOR, flags); 
                 break;
             case BWAND: 
-                invokeDynamic("and", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, flags);
+                invokeDefCall("and", methodType, DefBootstrap.BINARY_OPERATOR, flags);
                 break;
             case XOR:   
-                invokeDynamic("xor", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, flags);
+                invokeDefCall("xor", methodType, DefBootstrap.BINARY_OPERATOR, flags);
                 break;
             case BWOR:  
-                invokeDynamic("or", descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, flags);
+                invokeDefCall("or", methodType, DefBootstrap.BINARY_OPERATOR, flags);
                 break;
             default:
                 throw location.createError(new IllegalStateException("Illegal tree structure."));
@@ -391,4 +389,19 @@ public final class MethodWriter extends GeneratorAdapter {
     public void visitEnd() {
         throw new AssertionError("Should never call this method on MethodWriter, use endMethod() instead");
     }
+
+    /**
+     * Writes a dynamic call for a def method.
+     * @param name method name
+     * @param methodType callsite signature
+     * @param flavor type of call
+     * @param params flavor-specific parameters
+     */
+    public void invokeDefCall(String name, org.objectweb.asm.Type methodType, int flavor, Object... params) {
+        Object[] args = new Object[params.length + 2];
+        args[0] = settings.getInitialCallSiteDepth();
+        args[1] = flavor;
+        System.arraycopy(params, 0, args, 2, params.length);
+        invokeDynamic(name, methodType.getDescriptor(), DEF_BOOTSTRAP_HANDLE, args);
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
index bb94785d339..365060fe004 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
@@ -127,6 +127,12 @@ public final class PainlessScriptEngineService extends AbstractComponent impleme
             if (value != null) {
                 compilerSettings.setPicky(Boolean.parseBoolean(value));
             }
+            
+            value = copy.remove(CompilerSettings.INITIAL_CALL_SITE_DEPTH);
+            
+            if (value != null) {
+                compilerSettings.setInitialCallSiteDepth(Integer.parseInt(value));
+            }
 
             if (!copy.isEmpty()) {
                 throw new IllegalArgumentException("Unrecognized compile-time parameter(s): " + copy);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index 6960e906339..d581ba8518a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -85,9 +85,10 @@ public final class WriterConstants {
     public final static Method MATCHER_FIND = getAsmMethod(boolean.class, "find");
 
     /** dynamic callsite bootstrap signature */
-    public final static MethodType DEF_BOOTSTRAP_TYPE =
-        MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, int.class, Object[].class);
-    public final static Handle DEF_BOOTSTRAP_HANDLE =
+    final static MethodType DEF_BOOTSTRAP_TYPE =
+        MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, 
+                              int.class, int.class, Object[].class);
+    final static Handle DEF_BOOTSTRAP_HANDLE =
         new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(DefBootstrap.class),
             "bootstrap", DEF_BOOTSTRAP_TYPE.toMethodDescriptorString(), false);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index b92bd6733c9..a84247afe25 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -241,7 +241,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
             statements.add((AStatement)visit(statement));
         }
 
-        return new SSource(sourceName, sourceText, debugStream, (MainMethodReserved)reserved.pop(), 
+        return new SSource(settings, sourceName, sourceText, debugStream, (MainMethodReserved)reserved.pop(), 
                            location(ctx), functions, globals, statements);
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
index 11b77b0441f..87b12284789 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.AnalyzerCaster;
+import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Definition.Sort;
@@ -42,6 +43,7 @@ public final class EBinary extends AExpression {
     Type shiftDistance; // for shifts, the RHS is promoted independently
 
     boolean cat = false;
+    boolean originallyExplicit = false; // record whether there was originally an explicit cast
 
     public EBinary(Location location, Operation operation, AExpression left, AExpression right) {
         super(location);
@@ -53,6 +55,7 @@ public final class EBinary extends AExpression {
 
     @Override
     void analyze(Locals locals) {
+        originallyExplicit = explicit;
         if (operation == Operation.MUL) {
             analyzeMul(locals);
         } else if (operation == Operation.DIV) {
@@ -639,7 +642,13 @@ public final class EBinary extends AExpression {
             right.write(writer, globals);
 
             if (promote.sort == Sort.DEF || (shiftDistance != null && shiftDistance.sort == Sort.DEF)) {
-                writer.writeDynamicBinaryInstruction(location, actual, left.actual, right.actual, operation, false);
+                // def calls adopt the wanted return value. if there was a narrowing cast,
+                // we need to flag that so that its done at runtime.
+                int flags = 0;
+                if (originallyExplicit) {
+                    flags |= DefBootstrap.OPERATOR_EXPLICIT_CAST;
+                }
+                writer.writeDynamicBinaryInstruction(location, actual, left.actual, right.actual, operation, flags);
             } else {
                 writer.writeBinaryInstruction(location, actual, operation);
             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
index 333787db962..62ecf63b9b3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
@@ -30,7 +30,6 @@ import org.elasticsearch.painless.Locals.Variable;
 import org.objectweb.asm.Opcodes;
 import org.objectweb.asm.Type;
 
-import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
 import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE;
 
 import java.lang.invoke.LambdaMetafactory;
@@ -90,8 +89,8 @@ public class ECapturingFunctionRef extends AExpression implements ILambda {
         } else if (ref == null) {
             // typed interface, dynamic implementation
             writer.visitVarInsn(captured.type.type.getOpcode(Opcodes.ILOAD), captured.getSlot());
-            String descriptor = Type.getMethodType(expected.type, captured.type.type).getDescriptor();
-            writer.invokeDynamic(call, descriptor, DEF_BOOTSTRAP_HANDLE, DefBootstrap.REFERENCE, expected.name);
+            Type methodType = Type.getMethodType(expected.type, captured.type.type);
+            writer.invokeDefCall(call, methodType, DefBootstrap.REFERENCE, expected.name);
         } else {
             // typed interface, typed implementation
             writer.visitVarInsn(captured.type.type.getOpcode(Opcodes.ILOAD), captured.getSlot());
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
index b249cf4511e..aa5bbcaab9d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EChain.java
@@ -26,6 +26,7 @@ import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.AnalyzerCaster;
+import org.elasticsearch.painless.DefBootstrap;
 import org.elasticsearch.painless.Operation;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
@@ -214,7 +215,10 @@ public final class EChain extends AExpression {
 
             expression.expected = expression.actual;
         } else if (shift) {
-            if (shiftDistance.sort == Sort.LONG) {
+            if (promote.sort == Sort.DEF) {
+                // shifts are promoted independently, but for the def type, we need object.
+                expression.expected = promote;
+            } else if (shiftDistance.sort == Sort.LONG) {
                 expression.expected = Definition.INT_TYPE;
                 expression.explicit = true;   
             } else {
@@ -353,7 +357,7 @@ public final class EChain extends AExpression {
                     // write the operation instruction for compound assignment
                     if (promote.sort == Sort.DEF) {
                         writer.writeDynamicBinaryInstruction(location, promote, 
-                            Definition.DEF_TYPE, Definition.DEF_TYPE, operation, true);
+                            Definition.DEF_TYPE, Definition.DEF_TYPE, operation, DefBootstrap.OPERATOR_COMPOUND_ASSIGNMENT);
                     } else {
                         writer.writeBinaryInstruction(location, promote, operation);
                     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
index 53684a19005..31050f9f6ce 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java
@@ -33,7 +33,6 @@ import org.elasticsearch.painless.MethodWriter;
 
 import static org.elasticsearch.painless.WriterConstants.OBJECTS_TYPE;
 import static org.elasticsearch.painless.WriterConstants.EQUALS;
-import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
 
 /**
  * Represents a comparison expression.
@@ -498,8 +497,7 @@ public final class EComp extends AExpression {
                     if (right.isNull) {
                         writer.ifNull(jump);
                     } else if (!left.isNull && (operation == Operation.EQ || operation == Operation.NE)) {
-                        writer.invokeDynamic("eq", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR,
-                                                                                                     DefBootstrap.OPERATOR_ALLOWS_NULL);
+                        writer.invokeDefCall("eq", descriptor, DefBootstrap.BINARY_OPERATOR, DefBootstrap.OPERATOR_ALLOWS_NULL);
                         writejump = false;
                     } else {
                         writer.ifCmp(promotedType.type, MethodWriter.EQ, jump);
@@ -508,23 +506,22 @@ public final class EComp extends AExpression {
                     if (right.isNull) {
                         writer.ifNonNull(jump);
                     } else if (!left.isNull && (operation == Operation.EQ || operation == Operation.NE)) {
-                        writer.invokeDynamic("eq", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR,
-                                                                                                     DefBootstrap.OPERATOR_ALLOWS_NULL);
+                        writer.invokeDefCall("eq", descriptor, DefBootstrap.BINARY_OPERATOR, DefBootstrap.OPERATOR_ALLOWS_NULL);
                         writer.ifZCmp(MethodWriter.EQ, jump);
                     } else {
                         writer.ifCmp(promotedType.type, MethodWriter.NE, jump);
                     }
                 } else if (lt) {
-                    writer.invokeDynamic("lt", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, 0);
+                    writer.invokeDefCall("lt", descriptor, DefBootstrap.BINARY_OPERATOR, 0);
                     writejump = false;
                 } else if (lte) {
-                    writer.invokeDynamic("lte", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, 0);
+                    writer.invokeDefCall("lte", descriptor, DefBootstrap.BINARY_OPERATOR, 0);
                     writejump = false;
                 } else if (gt) {
-                    writer.invokeDynamic("gt", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, 0);
+                    writer.invokeDefCall("gt", descriptor, DefBootstrap.BINARY_OPERATOR, 0);
                     writejump = false;
                 } else if (gte) {
-                    writer.invokeDynamic("gte", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.BINARY_OPERATOR, 0);
+                    writer.invokeDefCall("gte", descriptor, DefBootstrap.BINARY_OPERATOR, 0);
                     writejump = false;
                 } else {
                     throw createError(new IllegalStateException("Illegal tree structure."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
index f74b91dd5bc..40776811b67 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java
@@ -31,8 +31,6 @@ import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Label;
 import org.elasticsearch.painless.MethodWriter;
 
-import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
-
 /**
  * Represents a unary math expression.
  */
@@ -41,6 +39,7 @@ public final class EUnary extends AExpression {
     final Operation operation;
     AExpression child;
     Type promote;
+    boolean originallyExplicit = false; // record whether there was originally an explicit cast
 
     public EUnary(Location location, Operation operation, AExpression child) {
         super(location);
@@ -51,6 +50,7 @@ public final class EUnary extends AExpression {
 
     @Override
     void analyze(Locals locals) {
+        originallyExplicit = explicit;
         if (operation == Operation.NOT) {
             analyzeNot(locals);
         } else if (operation == Operation.BWNOT) {
@@ -203,10 +203,16 @@ public final class EUnary extends AExpression {
             Sort sort = promote.sort;
             child.write(writer, globals);
 
+            // def calls adopt the wanted return value. if there was a narrowing cast,
+            // we need to flag that so that its done at runtime.
+            int defFlags = 0;
+            if (originallyExplicit) {
+                defFlags |= DefBootstrap.OPERATOR_EXPLICIT_CAST;
+            }
             if (operation == Operation.BWNOT) {
                 if (sort == Sort.DEF) {
                     org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(actual.type, child.actual.type);
-                    writer.invokeDynamic("not", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.UNARY_OPERATOR, 0);
+                    writer.invokeDefCall("not", descriptor, DefBootstrap.UNARY_OPERATOR, defFlags);
                 } else {
                     if (sort == Sort.INT) {
                         writer.push(-1);
@@ -221,14 +227,14 @@ public final class EUnary extends AExpression {
             } else if (operation == Operation.SUB) {
                 if (sort == Sort.DEF) {
                     org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(actual.type, child.actual.type);
-                    writer.invokeDynamic("neg", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.UNARY_OPERATOR, 0);
+                    writer.invokeDefCall("neg", descriptor, DefBootstrap.UNARY_OPERATOR, defFlags);
                 } else {
                     writer.math(MethodWriter.NEG, actual.type);
                 }
             } else if (operation == Operation.ADD) {
                 if (sort == Sort.DEF) {
                     org.objectweb.asm.Type descriptor = org.objectweb.asm.Type.getMethodType(actual.type, child.actual.type);
-                    writer.invokeDynamic("plus", descriptor.getDescriptor(), DEF_BOOTSTRAP_HANDLE, DefBootstrap.UNARY_OPERATOR, 0);
+                    writer.invokeDefCall("plus", descriptor, DefBootstrap.UNARY_OPERATOR, defFlags);
                 } 
             } else {
                 throw createError(new IllegalStateException("Illegal tree structure."));
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
index 02fbb2000f2..93fd21f00b0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefArray.java
@@ -27,8 +27,6 @@ import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Type;
 import org.elasticsearch.painless.MethodWriter;
 
-import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
-
 /**
  * Represents an array load/store or shortcut on a def type.  (Internal only.)
  */
@@ -62,15 +60,15 @@ final class LDefArray extends ALink implements IDefLink {
     void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
-        String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type, index.actual.type);
-        writer.invokeDynamic("arrayLoad", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_LOAD);
+        Type methodType = Type.getMethodType(after.type, Definition.DEF_TYPE.type, index.actual.type);
+        writer.invokeDefCall("arrayLoad", methodType, DefBootstrap.ARRAY_LOAD);
     }
 
     @Override
     void store(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
-        String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, index.actual.type, after.type);
-        writer.invokeDynamic("arrayStore", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ARRAY_STORE);
+        Type methodType = Type.getMethodType(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, index.actual.type, after.type);
+        writer.invokeDefCall("arrayStore", methodType, DefBootstrap.ARRAY_STORE);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index 716b26d6687..b104ce732ae 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -30,8 +30,6 @@ import org.objectweb.asm.Type;
 import java.util.ArrayList;
 import java.util.List;
 
-import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
-
 /**
  * Represents a method call made on a def type. (Internal only.)
  */
@@ -92,32 +90,30 @@ final class LDefCall extends ALink implements IDefLink {
     void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
-        StringBuilder signature = new StringBuilder();
+        List<Type> parameterTypes = new ArrayList<>();
 
-        signature.append('(');
         // first parameter is the receiver, we never know its type: always Object
-        signature.append(Definition.DEF_TYPE.type.getDescriptor());
+        parameterTypes.add(Definition.DEF_TYPE.type);
 
+        // append each argument
         for (AExpression argument : arguments) {
-            signature.append(argument.actual.type.getDescriptor());
+            parameterTypes.add(argument.actual.type);
             if (argument instanceof ILambda) {
                 ILambda lambda = (ILambda) argument;
                 for (Type capture : lambda.getCaptures()) {
-                    signature.append(capture.getDescriptor());
+                    parameterTypes.add(capture);
                 }
             }
             argument.write(writer, globals);
         }
 
-        signature.append(')');
-        // return value
-        signature.append(after.type.getDescriptor());
+        // create method type from return value and arguments
+        Type methodType = Type.getMethodType(after.type, parameterTypes.toArray(new Type[0]));
 
         List<Object> args = new ArrayList<>();
-        args.add(DefBootstrap.METHOD_CALL);
         args.add(recipe);
         args.addAll(pointers);
-        writer.invokeDynamic(name, signature.toString(), DEF_BOOTSTRAP_HANDLE, args.toArray());
+        writer.invokeDefCall(name, methodType, DefBootstrap.METHOD_CALL, args.toArray());
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
index 125b2bee6fc..ba474764cd0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefField.java
@@ -27,8 +27,6 @@ import org.elasticsearch.painless.Locals;
 import org.objectweb.asm.Type;
 import org.elasticsearch.painless.MethodWriter;
 
-import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
-
 /**
  * Represents a field load/store or shortcut on a def type.  (Internal only.)
  */
@@ -59,15 +57,15 @@ final class LDefField extends ALink implements IDefLink {
     void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
-        String desc = Type.getMethodDescriptor(after.type, Definition.DEF_TYPE.type);
-        writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.LOAD);
+        Type methodType = Type.getMethodType(after.type, Definition.DEF_TYPE.type);
+        writer.invokeDefCall(value, methodType, DefBootstrap.LOAD);
     }
 
     @Override
     void store(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
-        String desc = Type.getMethodDescriptor(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, after.type);
-        writer.invokeDynamic(value, desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.STORE);
+        Type methodType = Type.getMethodType(Definition.VOID_TYPE.type, Definition.DEF_TYPE.type, after.type);
+        writer.invokeDefCall(value, methodType, DefBootstrap.STORE);
     }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index 403b377ea9c..96258de0760 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -35,7 +35,6 @@ import org.elasticsearch.painless.Locals.Variable;
 import org.objectweb.asm.Label;
 import org.objectweb.asm.Opcodes;
 
-import static org.elasticsearch.painless.WriterConstants.DEF_BOOTSTRAP_HANDLE;
 import static org.elasticsearch.painless.WriterConstants.ITERATOR_HASNEXT;
 import static org.elasticsearch.painless.WriterConstants.ITERATOR_NEXT;
 import static org.elasticsearch.painless.WriterConstants.ITERATOR_TYPE;
@@ -193,8 +192,8 @@ public class SEach extends AStatement {
 
         if (method == null) {
             Type itr = Definition.getType("Iterator");
-            String desc = org.objectweb.asm.Type.getMethodDescriptor(itr.type, Definition.DEF_TYPE.type);
-            writer.invokeDynamic("iterator", desc, DEF_BOOTSTRAP_HANDLE, DefBootstrap.ITERATOR);
+            org.objectweb.asm.Type methodType = org.objectweb.asm.Type.getMethodType(itr.type, Definition.DEF_TYPE.type);
+            writer.invokeDefCall("iterator", methodType, DefBootstrap.ITERATOR);
         } else if (java.lang.reflect.Modifier.isInterface(method.owner.clazz.getModifiers())) {
             writer.invokeInterface(method.owner.type, method.method);
         } else {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index 8d3db7f811a..2426508d096 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.painless.node;
 
+import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Constant;
 import org.elasticsearch.painless.Def;
 import org.elasticsearch.painless.Definition;
@@ -143,12 +144,12 @@ public class SFunction extends AStatement {
     }
     
     /** Writes the function to given ClassVisitor. */
-    void write (ClassVisitor writer, Globals globals) {
+    void write (ClassVisitor writer, CompilerSettings settings, Globals globals) {
         int access = Opcodes.ACC_PRIVATE | Opcodes.ACC_STATIC;
         if (synthetic) {
             access |= Opcodes.ACC_SYNTHETIC;
         }
-        final MethodWriter function = new MethodWriter(access, method.method, writer, globals.getStatements());
+        final MethodWriter function = new MethodWriter(access, method.method, writer, globals.getStatements(), settings);
         write(function, globals);
         function.endMethod();
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index 5e932072ce1..6b870c7367d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.painless.node;
 
+import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Constant;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.MethodKey;
@@ -59,6 +60,7 @@ public final class SSource extends AStatement {
     final String name;
     final String source;
     final Printer debugStream;
+    final CompilerSettings settings;
     final MainMethodReserved reserved;
     final List<SFunction> functions;
     final Globals globals;
@@ -67,10 +69,11 @@ public final class SSource extends AStatement {
     private Locals mainMethod;
     private byte[] bytes;
 
-    public SSource(String name, String source, Printer debugStream, MainMethodReserved reserved, Location location, 
+    public SSource(CompilerSettings settings, String name, String source, Printer debugStream, 
+                   MainMethodReserved reserved, Location location, 
                    List<SFunction> functions, Globals globals, List<AStatement> statements) {
         super(location);
-
+        this.settings = settings;
         this.name = name;
         this.source = source;
         this.debugStream = debugStream;
@@ -151,7 +154,7 @@ public final class SSource extends AStatement {
         visitor.visitSource(Location.computeSourceName(name, source), null);
 
         // Write the constructor:
-        MethodWriter constructor = new MethodWriter(Opcodes.ACC_PUBLIC, CONSTRUCTOR, visitor, globals.getStatements());
+        MethodWriter constructor = new MethodWriter(Opcodes.ACC_PUBLIC, CONSTRUCTOR, visitor, globals.getStatements(), settings);
         constructor.visitCode();
         constructor.loadThis();
         constructor.loadArgs();
@@ -160,14 +163,14 @@ public final class SSource extends AStatement {
         constructor.endMethod();
 
         // Write the execute method:
-        MethodWriter execute = new MethodWriter(Opcodes.ACC_PUBLIC, EXECUTE, visitor, globals.getStatements());
+        MethodWriter execute = new MethodWriter(Opcodes.ACC_PUBLIC, EXECUTE, visitor, globals.getStatements(), settings);
         execute.visitCode();
         write(execute, globals);
         execute.endMethod();
         
         // Write all functions:
         for (SFunction function : functions) {
-            function.write(visitor, globals);
+            function.write(visitor, settings, globals);
         }
         
         // Write all synthetic functions. Note that this process may add more :)
@@ -175,7 +178,7 @@ public final class SSource extends AStatement {
             List<SFunction> current = new ArrayList<>(globals.getSyntheticMethods().values());
             globals.getSyntheticMethods().clear();
             for (SFunction function : current) {
-                function.write(visitor, globals);
+                function.write(visitor, settings, globals);
             }
         }
 
@@ -195,7 +198,7 @@ public final class SSource extends AStatement {
 
             // Initialize the constants in a static initializer
             final MethodWriter clinit = new MethodWriter(Opcodes.ACC_STATIC, 
-                    WriterConstants.CLINIT, visitor, globals.getStatements());
+                    WriterConstants.CLINIT, visitor, globals.getStatements(), settings);
             for (Constant constant : inits) {
                 constant.initializer.accept(clinit);
                 clinit.putStatic(CLASS_TYPE, constant.name, constant.type);
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/CastTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/CastTests.java
new file mode 100644
index 00000000000..fcee0ddeb3e
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/CastTests.java
@@ -0,0 +1,298 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+/** Tests for explicit casts */
+public class CastTests extends ScriptTestCase {
+    
+    /**
+     * Unary operator with explicit cast
+     */
+    public void testUnaryOperator() {
+        assertEquals((byte)5, exec("long x = 5L; return (byte) (+x);"));
+        assertEquals((short)5, exec("long x = 5L; return (short) (+x);"));
+        assertEquals((char)5, exec("long x = 5L; return (char) (+x);"));
+        assertEquals(5, exec("long x = 5L; return (int) (+x);"));
+        assertEquals(5F, exec("long x = 5L; return (float) (+x);"));
+        assertEquals(5L, exec("long x = 5L; return (long) (+x);"));
+        assertEquals(5D, exec("long x = 5L; return (double) (+x);"));
+    }
+    
+    /**
+     * Binary operators with explicit cast
+     */
+    public void testBinaryOperator() {
+        assertEquals((byte)6, exec("long x = 5L; return (byte) (x + 1);"));
+        assertEquals((short)6, exec("long x = 5L; return (short) (x + 1);"));
+        assertEquals((char)6, exec("long x = 5L; return (char) (x + 1);"));
+        assertEquals(6, exec("long x = 5L; return (int) (x + 1);"));
+        assertEquals(6F, exec("long x = 5L; return (float) (x + 1);"));
+        assertEquals(6L, exec("long x = 5L; return (long) (x + 1);"));
+        assertEquals(6D, exec("long x = 5L; return (double) (x + 1);"));
+    }
+
+    /**
+     * Binary compound assignment with explicit cast
+     */
+    public void testBinaryCompoundAssignment() {
+        assertEquals((byte)6, exec("long x = 5L; return (byte) (x += 1);"));
+        assertEquals((short)6, exec("long x = 5L; return (short) (x += 1);"));
+        assertEquals((char)6, exec("long x = 5L; return (char) (x += 1);"));
+        assertEquals(6, exec("long x = 5L; return (int) (x += 1);"));
+        assertEquals(6F, exec("long x = 5L; return (float) (x += 1);"));
+        assertEquals(6L, exec("long x = 5L; return (long) (x += 1);"));
+        assertEquals(6D, exec("long x = 5L; return (double) (x += 1);"));
+    }
+
+    /**
+     * Binary compound prefix with explicit cast
+     */
+    public void testBinaryPrefix() {
+        assertEquals((byte)6, exec("long x = 5L; return (byte) (++x);"));
+        assertEquals((short)6, exec("long x = 5L; return (short) (++x);"));
+        assertEquals((char)6, exec("long x = 5L; return (char) (++x);"));
+        assertEquals(6, exec("long x = 5L; return (int) (++x);"));
+        assertEquals(6F, exec("long x = 5L; return (float) (++x);"));
+        assertEquals(6L, exec("long x = 5L; return (long) (++x);"));
+        assertEquals(6D, exec("long x = 5L; return (double) (++x);"));
+    }
+    
+    /**
+     * Binary compound postifx with explicit cast
+     */
+    public void testBinaryPostfix() {
+        assertEquals((byte)5, exec("long x = 5L; return (byte) (x++);"));
+        assertEquals((short)5, exec("long x = 5L; return (short) (x++);"));
+        assertEquals((char)5, exec("long x = 5L; return (char) (x++);"));
+        assertEquals(5, exec("long x = 5L; return (int) (x++);"));
+        assertEquals(5F, exec("long x = 5L; return (float) (x++);"));
+        assertEquals(5L, exec("long x = 5L; return (long) (x++);"));
+        assertEquals(5D, exec("long x = 5L; return (double) (x++);"));
+    }
+    
+    /**
+     * Shift operators with explicit cast
+     */
+    public void testShiftOperator() {
+        assertEquals((byte)10, exec("long x = 5L; return (byte) (x << 1);"));
+        assertEquals((short)10, exec("long x = 5L; return (short) (x << 1);"));
+        assertEquals((char)10, exec("long x = 5L; return (char) (x << 1);"));
+        assertEquals(10, exec("long x = 5L; return (int) (x << 1);"));
+        assertEquals(10F, exec("long x = 5L; return (float) (x << 1);"));
+        assertEquals(10L, exec("long x = 5L; return (long) (x << 1);"));
+        assertEquals(10D, exec("long x = 5L; return (double) (x << 1);"));
+    }
+    
+    /**
+     * Shift compound assignment with explicit cast
+     */
+    public void testShiftCompoundAssignment() {
+        assertEquals((byte)10, exec("long x = 5L; return (byte) (x <<= 1);"));
+        assertEquals((short)10, exec("long x = 5L; return (short) (x <<= 1);"));
+        assertEquals((char)10, exec("long x = 5L; return (char) (x <<= 1);"));
+        assertEquals(10, exec("long x = 5L; return (int) (x <<= 1);"));
+        assertEquals(10F, exec("long x = 5L; return (float) (x <<= 1);"));
+        assertEquals(10L, exec("long x = 5L; return (long) (x <<= 1);"));
+        assertEquals(10D, exec("long x = 5L; return (double) (x <<= 1);"));
+    }
+    
+    /**
+     * Test that without a cast, we fail when conversions would narrow.
+     */
+    public void testIllegalConversions() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("long x = 5L; int y = +x; return y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("long x = 5L; int y = (x + x); return y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("boolean x = true; int y = +x; return y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("boolean x = true; int y = (x ^ false); return y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("long x = 5L; boolean y = +x; return y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("long x = 5L; boolean y = (x + x); return y");
+        });
+    }
+    
+    /**
+     * Test that even with a cast, some things aren't allowed.
+     */
+    public void testIllegalExplicitConversions() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("boolean x = true; int y = (int) +x; return y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("boolean x = true; int y = (int) (x ^ false); return y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("long x = 5L; boolean y = (boolean) +x; return y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("long x = 5L; boolean y = (boolean) (x + x); return y");
+        });
+    }
+
+    /**
+     * Currently these do not adopt the return value, we issue a separate cast!
+     */
+    public void testMethodCallDef() {
+        assertEquals(5, exec("def x = 5; return (int)x.longValue();"));
+    }
+    
+    /**
+     * Unary operators adopt the return value
+     */
+    public void testUnaryOperatorDef() {
+        assertEquals((byte)5, exec("def x = 5L; return (byte) (+x);"));
+        assertEquals((short)5, exec("def x = 5L; return (short) (+x);"));
+        assertEquals((char)5, exec("def x = 5L; return (char) (+x);"));
+        assertEquals(5, exec("def x = 5L; return (int) (+x);"));
+        assertEquals(5F, exec("def x = 5L; return (float) (+x);"));
+        assertEquals(5L, exec("def x = 5L; return (long) (+x);"));
+        assertEquals(5D, exec("def x = 5L; return (double) (+x);"));
+    }
+    
+    /**
+     * Binary operators adopt the return value
+     */
+    public void testBinaryOperatorDef() {
+        assertEquals((byte)6, exec("def x = 5L; return (byte) (x + 1);"));
+        assertEquals((short)6, exec("def x = 5L; return (short) (x + 1);"));
+        assertEquals((char)6, exec("def x = 5L; return (char) (x + 1);"));
+        assertEquals(6, exec("def x = 5L; return (int) (x + 1);"));
+        assertEquals(6F, exec("def x = 5L; return (float) (x + 1);"));
+        assertEquals(6L, exec("def x = 5L; return (long) (x + 1);"));
+        assertEquals(6D, exec("def x = 5L; return (double) (x + 1);"));
+    }
+    
+    /**
+     * Binary operators don't yet adopt the return value with compound assignment
+     */
+    public void testBinaryCompoundAssignmentDef() {
+        assertEquals((byte)6, exec("def x = 5L; return (byte) (x += 1);"));
+        assertEquals((short)6, exec("def x = 5L; return (short) (x += 1);"));
+        assertEquals((char)6, exec("def x = 5L; return (char) (x += 1);"));
+        assertEquals(6, exec("def x = 5L; return (int) (x += 1);"));
+        assertEquals(6F, exec("def x = 5L; return (float) (x += 1);"));
+        assertEquals(6L, exec("def x = 5L; return (long) (x += 1);"));
+        assertEquals(6D, exec("def x = 5L; return (double) (x += 1);"));
+    }
+    
+    /**
+     * Binary operators don't yet adopt the return value with compound assignment
+     */
+    public void testBinaryCompoundAssignmentPrefix() {
+        assertEquals((byte)6, exec("def x = 5L; return (byte) (++x);"));
+        assertEquals((short)6, exec("def x = 5L; return (short) (++x);"));
+        assertEquals((char)6, exec("def x = 5L; return (char) (++x);"));
+        assertEquals(6, exec("def x = 5L; return (int) (++x);"));
+        assertEquals(6F, exec("def x = 5L; return (float) (++x);"));
+        assertEquals(6L, exec("def x = 5L; return (long) (++x);"));
+        assertEquals(6D, exec("def x = 5L; return (double) (++x);"));
+    }
+    
+    /**
+     * Binary operators don't yet adopt the return value with compound assignment
+     */
+    public void testBinaryCompoundAssignmentPostfix() {
+        assertEquals((byte)5, exec("def x = 5L; return (byte) (x++);"));
+        assertEquals((short)5, exec("def x = 5L; return (short) (x++);"));
+        assertEquals((char)5, exec("def x = 5L; return (char) (x++);"));
+        assertEquals(5, exec("def x = 5L; return (int) (x++);"));
+        assertEquals(5F, exec("def x = 5L; return (float) (x++);"));
+        assertEquals(5L, exec("def x = 5L; return (long) (x++);"));
+        assertEquals(5D, exec("def x = 5L; return (double) (x++);"));
+    }
+    
+    /**
+     * Shift operators adopt the return value
+     */
+    public void testShiftOperatorDef() {
+        assertEquals((byte)10, exec("def x = 5L; return (byte) (x << 1);"));
+        assertEquals((short)10, exec("def x = 5L; return (short) (x << 1);"));
+        assertEquals((char)10, exec("def x = 5L; return (char) (x << 1);"));
+        assertEquals(10, exec("def x = 5L; return (int) (x << 1);"));
+        assertEquals(10F, exec("def x = 5L; return (float) (x << 1);"));
+        assertEquals(10L, exec("def x = 5L; return (long) (x << 1);"));
+        assertEquals(10D, exec("def x = 5L; return (double) (x << 1);"));
+    }
+    
+    /**
+     * Shift operators don't yet adopt the return value with compound assignment
+     */
+    public void testShiftCompoundAssignmentDef() {
+        assertEquals((byte)10, exec("def x = 5L; return (byte) (x <<= 1);"));
+        assertEquals((short)10, exec("def x = 5L; return (short) (x <<= 1);"));
+        assertEquals((char)10, exec("def x = 5L; return (char) (x <<= 1);"));
+        assertEquals(10, exec("def x = 5L; return (int) (x <<= 1);"));
+        assertEquals(10F, exec("def x = 5L; return (float) (x <<= 1);"));
+        assertEquals(10L, exec("def x = 5L; return (long) (x <<= 1);"));
+        assertEquals(10D, exec("def x = 5L; return (double) (x <<= 1);"));
+    }
+    
+    /**
+     * Test that without a cast, we fail when conversions would narrow.
+     */
+    public void testIllegalConversionsDef() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = 5L; int y = +x; return y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = 5L; int y = (x + x); return y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = true; int y = +x; return y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = true; int y = (x ^ false); return y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = 5L; boolean y = +x; return y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = 5L; boolean y = (x + x); return y");
+        });
+    }
+    
+    /**
+     * Test that even with a cast, some things aren't allowed.
+     * (stuff that methodhandles explicitCastArguments would otherwise allow)
+     */
+    public void testIllegalExplicitConversionsDef() {
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = true; int y = (int) +x; return y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = true; int y = (int) (x ^ false); return y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = 5L; boolean y = (boolean) +x; return y");
+        });
+        expectScriptThrows(ClassCastException.class, () -> {
+            exec("def x = 5L; boolean y = (boolean) (x + x); return y");
+        });
+    }
+}
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
index 6529bed3a48..6babe51334a 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
@@ -36,6 +36,7 @@ public class DefBootstrapTests extends ESTestCase {
         CallSite site = DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                   "toString", 
                                                   MethodType.methodType(String.class, Object.class), 
+                                                  0,
                                                   DefBootstrap.METHOD_CALL, 0L);
         MethodHandle handle = site.dynamicInvoker();
         assertDepthEquals(site, 0);
@@ -53,6 +54,7 @@ public class DefBootstrapTests extends ESTestCase {
         CallSite site = DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                   "toString", 
                                                   MethodType.methodType(String.class, Object.class), 
+                                                  0,
                                                   DefBootstrap.METHOD_CALL, 0L);
         MethodHandle handle = site.dynamicInvoker();
         assertDepthEquals(site, 0);
@@ -75,6 +77,7 @@ public class DefBootstrapTests extends ESTestCase {
         CallSite site = DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                   "toString", 
                                                   MethodType.methodType(String.class, Object.class), 
+                                                  0,
                                                   DefBootstrap.METHOD_CALL, 0L);
         MethodHandle handle = site.dynamicInvoker();
         assertDepthEquals(site, 0);
@@ -98,6 +101,7 @@ public class DefBootstrapTests extends ESTestCase {
         DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                                           "size", 
                                                                           MethodType.methodType(int.class, Object.class), 
+                                                                          0,
                                                                           DefBootstrap.METHOD_CALL, 0L);
         site.depth = DefBootstrap.PIC.MAX_DEPTH; // mark megamorphic
         MethodHandle handle = site.dynamicInvoker();
@@ -127,6 +131,7 @@ public class DefBootstrapTests extends ESTestCase {
         DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                                "add", 
                                                                MethodType.methodType(Object.class, Object.class, Object.class),
+                                                               0,
                                                                DefBootstrap.BINARY_OPERATOR, DefBootstrap.OPERATOR_ALLOWS_NULL);
         MethodHandle handle = site.dynamicInvoker();
         assertEquals("nulltest", (Object)handle.invokeExact((Object)null, (Object)"test"));
@@ -136,6 +141,7 @@ public class DefBootstrapTests extends ESTestCase {
         DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                                "add", 
                                                                MethodType.methodType(Object.class, Object.class, Object.class),
+                                                               0,
                                                                DefBootstrap.BINARY_OPERATOR, DefBootstrap.OPERATOR_ALLOWS_NULL);
         MethodHandle handle = site.dynamicInvoker();
         assertEquals(2, (Object)handle.invokeExact((Object)1, (Object)1));
@@ -146,6 +152,7 @@ public class DefBootstrapTests extends ESTestCase {
         DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                                "eq", 
                                                                MethodType.methodType(boolean.class, Object.class, Object.class),
+                                                               0,
                                                                DefBootstrap.BINARY_OPERATOR, DefBootstrap.OPERATOR_ALLOWS_NULL);
         MethodHandle handle = site.dynamicInvoker();
         assertFalse((boolean) handle.invokeExact((Object)null, (Object)"test"));
@@ -156,6 +163,7 @@ public class DefBootstrapTests extends ESTestCase {
         DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                                "eq", 
                                                                MethodType.methodType(boolean.class, Object.class, Object.class),
+                                                               0,
                                                                DefBootstrap.BINARY_OPERATOR, DefBootstrap.OPERATOR_ALLOWS_NULL);
         MethodHandle handle = site.dynamicInvoker();
         assertTrue((boolean) handle.invokeExact((Object)1, (Object)1));
@@ -171,6 +179,7 @@ public class DefBootstrapTests extends ESTestCase {
         DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                                "add", 
                                                                MethodType.methodType(Object.class, int.class, Object.class),
+                                                               0,
                                                                DefBootstrap.BINARY_OPERATOR, 0);
         MethodHandle handle = site.dynamicInvoker();
         expectThrows(NullPointerException.class, () -> {
@@ -182,6 +191,7 @@ public class DefBootstrapTests extends ESTestCase {
         DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                                "add", 
                                                                MethodType.methodType(Object.class, int.class, Object.class),
+                                                               0,
                                                                DefBootstrap.BINARY_OPERATOR, 0);
         MethodHandle handle = site.dynamicInvoker();
         assertEquals(2, (Object)handle.invokeExact(1, (Object)1));
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
index 6fb9151cb0d..fe0f7b6ed82 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
@@ -243,18 +243,22 @@ public class DefOptimizationTests extends ScriptTestCase {
     public void testAddOptNullGuards() {
         // needs null guard
         assertBytecodeHasPattern("def x = 1; def y = 2; return x + y", 
-                "(?s).*INVOKEDYNAMIC add.*arguments:\\s+" + DefBootstrap.BINARY_OPERATOR 
+                "(?s).*INVOKEDYNAMIC add.*arguments:\\s+" + "\\d+"
+                                                + ",\\s+" + DefBootstrap.BINARY_OPERATOR 
                                                 + ",\\s+" + DefBootstrap.OPERATOR_ALLOWS_NULL + ".*");
         // still needs null guard, NPE is the wrong thing!
         assertBytecodeHasPattern("def x = 1; def y = 2; double z = x + y", 
-                "(?s).*INVOKEDYNAMIC add.*arguments:\\s+" + DefBootstrap.BINARY_OPERATOR 
+                "(?s).*INVOKEDYNAMIC add.*arguments:\\s+" + "\\d+"
+                                                + ",\\s+" + DefBootstrap.BINARY_OPERATOR
                                                 + ",\\s+" + DefBootstrap.OPERATOR_ALLOWS_NULL + ".*");
         // a primitive argument is present: no null guard needed
         assertBytecodeHasPattern("def x = 1; int y = 2; return x + y", 
-                "(?s).*INVOKEDYNAMIC add.*arguments:\\s+" + DefBootstrap.BINARY_OPERATOR 
+                "(?s).*INVOKEDYNAMIC add.*arguments:\\s+" + "\\d+"
+                                                + ",\\s+" + DefBootstrap.BINARY_OPERATOR
                                                 + ",\\s+" + 0 + ".*");
         assertBytecodeHasPattern("int x = 1; def y = 2; return x + y", 
-                "(?s).*INVOKEDYNAMIC add.*arguments:\\s+" + DefBootstrap.BINARY_OPERATOR 
+                "(?s).*INVOKEDYNAMIC add.*arguments:\\s+" + "\\d+"
+                                                + ",\\s+" + DefBootstrap.BINARY_OPERATOR
                                                 + ",\\s+" + 0 + ".*");
     }
     
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
index 40f59e91120..0bec55b5428 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java
@@ -56,6 +56,7 @@ public abstract class ScriptTestCase extends ESTestCase {
     public Object exec(String script, Map<String, Object> vars) {
         Map<String,String> compilerSettings = new HashMap<>();
         compilerSettings.put(CompilerSettings.PICKY, "true");
+        compilerSettings.put(CompilerSettings.INITIAL_CALL_SITE_DEPTH, random().nextBoolean() ? "0" : "10");
         return exec(script, vars, compilerSettings, null);
     }
 

From 5475e18ad0c7958704d537483ab038ba459ce95a Mon Sep 17 00:00:00 2001
From: Uwe Schindler <uwe@thetaphi.de>
Date: Sun, 19 Jun 2016 20:16:53 +0200
Subject: [PATCH 0950/1311] Update forbiddenapis to 2.2 and fix painless tests

---
 buildSrc/build.gradle                                     | 2 +-
 .../src/main/resources/forbidden/es-all-signatures.txt    | 4 ++--
 .../test/java/org/elasticsearch/painless/ArrayTests.java  | 8 +++++++-
 .../org/elasticsearch/painless/DefBootstrapTests.java     | 2 +-
 4 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
index a5a9b4209ba..cbb97179a46 100644
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -84,7 +84,7 @@ dependencies {
   compile 'com.netflix.nebula:gradle-info-plugin:3.0.3'
   compile 'org.eclipse.jgit:org.eclipse.jgit:3.2.0.201312181205-r'
   compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
-  compile 'de.thetaphi:forbiddenapis:2.1'
+  compile 'de.thetaphi:forbiddenapis:2.2'
   compile 'com.bmuschko:gradle-nexus-plugin:2.3.1'
   compile 'org.apache.rat:apache-rat:0.11'
 }
diff --git a/buildSrc/src/main/resources/forbidden/es-all-signatures.txt b/buildSrc/src/main/resources/forbidden/es-all-signatures.txt
index 71eb1433521..37f03f4c91c 100644
--- a/buildSrc/src/main/resources/forbidden/es-all-signatures.txt
+++ b/buildSrc/src/main/resources/forbidden/es-all-signatures.txt
@@ -32,7 +32,7 @@ org.apache.lucene.index.IndexReader#getCombinedCoreAndDeletesKey()
 @defaultMessage Soon to be removed
 org.apache.lucene.document.FieldType#numericType()
 
-@defaultMessage Don't use MethodHandles in slow ways, dont be lenient in tests.
-# unfortunately, invoke() cannot be banned, because forbidden apis does not support signature polymorphic methods
+@defaultMessage Don't use MethodHandles in slow ways, don't be lenient in tests.
+java.lang.invoke.MethodHandle#invoke(java.lang.Object[])
 java.lang.invoke.MethodHandle#invokeWithArguments(java.lang.Object[])
 java.lang.invoke.MethodHandle#invokeWithArguments(java.util.List)
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
index e6a5537194d..acacc613ab3 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java
@@ -19,6 +19,9 @@
 
 package org.elasticsearch.painless;
 
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodType;
+
 /** Tests for or operator across all types */
 public class ArrayTests extends ScriptTestCase {
 
@@ -36,7 +39,10 @@ public class ArrayTests extends ScriptTestCase {
     }
 
     private void assertArrayLength(int length, Object array) throws Throwable {
-        assertEquals(length, (int) Def.arrayLengthGetter(array.getClass()).invoke(array));
+        final MethodHandle mh = Def.arrayLengthGetter(array.getClass());
+        assertSame(array.getClass(), mh.type().parameterType(0));
+        assertEquals(length, (int) mh.asType(MethodType.methodType(int.class, Object.class))
+                .invokeExact(array));
     }
 
     public void testArrayLoadStoreInt() {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
index 6529bed3a48..eb08825f387 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
@@ -139,7 +139,7 @@ public class DefBootstrapTests extends ESTestCase {
                                                                DefBootstrap.BINARY_OPERATOR, DefBootstrap.OPERATOR_ALLOWS_NULL);
         MethodHandle handle = site.dynamicInvoker();
         assertEquals(2, (Object)handle.invokeExact((Object)1, (Object)1));
-        assertEquals("nulltest", (Object)handle.invoke((Object)null, (Object)"test"));
+        assertEquals("nulltest", (Object)handle.invokeExact((Object)null, (Object)"test"));
     }
     
     public void testNullGuardEq() throws Throwable {

From 6b46bf13f04cd01cb51d3d11ccf259755b7f7253 Mon Sep 17 00:00:00 2001
From: Florian Hopf <mail@florian-hopf.de>
Date: Sun, 19 Jun 2016 23:25:18 +0200
Subject: [PATCH 0951/1311] Throw if the local node is not set

This commit adds an IllegalStateException if attempting to get the local
node from the cluster service when it is not set.

Relates #18963
---
 .../org/elasticsearch/cluster/service/ClusterService.java   | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
index 7573f260a54..50c91665485 100644
--- a/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
@@ -237,7 +237,11 @@ public class ClusterService extends AbstractLifecycleComponent<ClusterService> {
      * The local node.
      */
     public DiscoveryNode localNode() {
-        return clusterState.getNodes().getLocalNode();
+        DiscoveryNode localNode = clusterState.getNodes().getLocalNode();
+        if (localNode == null) {
+            throw new IllegalStateException("No local node found. Is the node started?");
+        }
+        return localNode;
     }
 
     public OperationRouting operationRouting() {

From 98951b1203a8bc34d39a190a00712b62598adacf Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Thu, 16 Jun 2016 10:39:14 +0200
Subject: [PATCH 0952/1311]  Compile each Groovy script in its own classloader

 closes #18572
---
 .../elasticsearch/script/CompiledScript.java  |   6 +-
 .../script/NativeScriptEngineService.java     |   5 -
 .../script/ScriptEngineService.java           |   8 -
 .../elasticsearch/script/ScriptService.java   |  12 +-
 .../script/ScriptModesTests.java              |   6 -
 .../script/ScriptServiceTests.java            |  11 --
 .../script/ScriptSettingsTests.java           |   6 -
 .../search/aggregations/metrics/AvgIT.java    |   8 -
 .../search/aggregations/metrics/SumIT.java    |   8 -
 .../aggregations/metrics/ValueCountIT.java    |   4 -
 .../org/elasticsearch/update/UpdateIT.java    |  17 --
 .../ExpressionScriptEngineService.java        |   5 -
 .../groovy/GroovyScriptEngineService.java     | 177 ++++++++----------
 .../plugin-metadata/plugin-security.policy    |   2 -
 .../script/groovy/GroovySecurityTests.java    |   4 +-
 .../test/lang_groovy/16_update2.yaml          |   2 +-
 .../mustache/MustacheScriptEngineService.java |   5 -
 .../painless/PainlessScriptEngineService.java |   9 -
 .../JavaScriptScriptEngineService.java        |   5 -
 .../python/PythonScriptEngineService.java     |   5 -
 .../script/MockScriptEngine.java              |   4 -
 21 files changed, 92 insertions(+), 217 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/script/CompiledScript.java b/core/src/main/java/org/elasticsearch/script/CompiledScript.java
index aa34678c041..ec2ad4192a2 100644
--- a/core/src/main/java/org/elasticsearch/script/CompiledScript.java
+++ b/core/src/main/java/org/elasticsearch/script/CompiledScript.java
@@ -39,9 +39,9 @@ public class CompiledScript {
     public CompiledScript(ScriptService.ScriptType type, String name, String lang, Object compiled) {
         this.type = type;
         this.name = name;
-            this.lang = lang;
-            this.compiled = compiled;
-        }
+        this.lang = lang;
+        this.compiled = compiled;
+    }
 
     /**
      * Method to get the type of language.
diff --git a/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java b/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java
index c9bfdfc1670..191c2b4bcf7 100644
--- a/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java
+++ b/core/src/main/java/org/elasticsearch/script/NativeScriptEngineService.java
@@ -90,11 +90,6 @@ public class NativeScriptEngineService extends AbstractComponent implements Scri
     public void close() {
     }
 
-    @Override
-    public void scriptRemoved(CompiledScript script) {
-        // Nothing to do here
-    }
-
     @Override
     public boolean isInlineScriptEnabled() {
         return true;
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java b/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java
index ab49c3c8011..55a931d8d57 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptEngineService.java
@@ -23,7 +23,6 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.Closeable;
-import java.util.List;
 import java.util.Map;
 
 /**
@@ -49,13 +48,6 @@ public interface ScriptEngineService extends Closeable {
 
     SearchScript search(CompiledScript compiledScript, SearchLookup lookup, @Nullable Map<String, Object> vars);
 
-    /**
-     * Handler method called when a script is removed from the Guava cache.
-     *
-     * The passed script may be null if it has already been garbage collected.
-     * */
-    void scriptRemoved(@Nullable CompiledScript script);
-
     /**
      * Returns <code>true</code> if this scripting engine can safely accept inline scripts by default. The default is <code>false</code>
      */
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java
index fe3eaa29142..8f939f265a4 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java
@@ -507,16 +507,10 @@ public class ScriptService extends AbstractComponent implements Closeable {
     private class ScriptCacheRemovalListener implements RemovalListener<CacheKey, CompiledScript> {
         @Override
         public void onRemoval(RemovalNotification<CacheKey, CompiledScript> notification) {
-            scriptMetrics.onCacheEviction();
-            for (ScriptEngineService service : scriptEngines) {
-                try {
-                    service.scriptRemoved(notification.getValue());
-                } catch (Exception e) {
-                    logger.warn("exception calling script removal listener for script service", e);
-                    // We don't rethrow because Guava would just catch the
-                    // exception and log it, which we have already done
-                }
+            if (logger.isDebugEnabled()) {
+                logger.debug("removed {} from cache, reason: {}", notification.getValue(), notification.getRemovalReason());
             }
+            scriptMetrics.onCacheEviction();
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java b/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
index 2927b4747dc..e4a1835ab63 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java
@@ -257,12 +257,6 @@ public class ScriptModesTests extends ESTestCase {
 
         @Override
         public void close() {
-
-        }
-
-        @Override
-        public void scriptRemoved(@Nullable CompiledScript script) {
-
         }
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
index db9e7d66731..597cb7e6032 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
@@ -518,11 +518,6 @@ public class ScriptServiceTests extends ESTestCase {
 
         }
 
-        @Override
-        public void scriptRemoved(CompiledScript script) {
-            // Nothing to do here
-        }
-
         @Override
         public boolean isInlineScriptEnabled() {
             return true;
@@ -562,12 +557,6 @@ public class ScriptServiceTests extends ESTestCase {
 
         @Override
         public void close() {
-
-        }
-
-        @Override
-        public void scriptRemoved(CompiledScript script) {
-            // Nothing to do here
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java b/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
index 0e2bb384152..053ccec652c 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptSettingsTests.java
@@ -95,12 +95,6 @@ public class ScriptSettingsTests extends ESTestCase {
 
         @Override
         public void close() {
-
-        }
-
-        @Override
-        public void scriptRemoved(@Nullable CompiledScript script) {
-
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
index ad2c0ddb9d5..ccf03df37fc 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java
@@ -456,10 +456,6 @@ public class AvgIT extends AbstractNumericTestCase {
                 }
             };
         }
-
-        @Override
-        public void scriptRemoved(CompiledScript script) {
-        }
     }
 
     /**
@@ -564,10 +560,6 @@ public class AvgIT extends AbstractNumericTestCase {
             };
         }
 
-        @Override
-        public void scriptRemoved(CompiledScript script) {
-        }
-
         @Override
         public boolean isInlineScriptEnabled() {
             return true;
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
index a142f43bd5a..186bc8fb27b 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java
@@ -453,10 +453,6 @@ public class SumIT extends AbstractNumericTestCase {
             };
         }
 
-        @Override
-        public void scriptRemoved(CompiledScript script) {
-        }
-
         @Override
         public boolean isInlineScriptEnabled() {
             return true;
@@ -573,10 +569,6 @@ public class SumIT extends AbstractNumericTestCase {
             };
         }
 
-        @Override
-        public void scriptRemoved(CompiledScript script) {
-        }
-
         @Override
         public boolean isInlineScriptEnabled() {
             return true;
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
index d79edcc41b6..7de37fdb6a8 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountIT.java
@@ -318,10 +318,6 @@ public class ValueCountIT extends ESIntegTestCase {
             };
         }
 
-        @Override
-        public void scriptRemoved(CompiledScript script) {
-        }
-
         @Override
         public boolean isInlineScriptEnabled() {
             return true;
diff --git a/core/src/test/java/org/elasticsearch/update/UpdateIT.java b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
index 0ae1b4e435a..4121543a7a0 100644
--- a/core/src/test/java/org/elasticsearch/update/UpdateIT.java
+++ b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
@@ -144,10 +144,6 @@ public class UpdateIT extends ESIntegTestCase {
             throw new UnsupportedOperationException();
         }
 
-        @Override
-        public void scriptRemoved(CompiledScript script) {
-        }
-
         @Override
         public boolean isInlineScriptEnabled() {
             return true;
@@ -214,15 +210,10 @@ public class UpdateIT extends ESIntegTestCase {
             throw new UnsupportedOperationException();
         }
 
-        @Override
-        public void scriptRemoved(CompiledScript script) {
-        }
-
         @Override
         public boolean isInlineScriptEnabled() {
             return true;
         }
-
     }
 
     public static class ScriptedUpsertScriptPlugin extends Plugin implements ScriptPlugin {
@@ -285,10 +276,6 @@ public class UpdateIT extends ESIntegTestCase {
             throw new UnsupportedOperationException();
         }
 
-        @Override
-        public void scriptRemoved(CompiledScript script) {
-        }
-
         @Override
         public boolean isInlineScriptEnabled() {
             return true;
@@ -357,10 +344,6 @@ public class UpdateIT extends ESIntegTestCase {
             throw new UnsupportedOperationException();
         }
 
-        @Override
-        public void scriptRemoved(CompiledScript script) {
-        }
-
         @Override
         public boolean isInlineScriptEnabled() {
             return true;
diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
index fdede1e77bc..72a5169593a 100644
--- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
+++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java
@@ -266,11 +266,6 @@ public class ExpressionScriptEngineService extends AbstractComponent implements
     @Override
     public void close() {}
 
-    @Override
-    public void scriptRemoved(CompiledScript script) {
-        // Nothing to do
-    }
-
     @Override
     public boolean isInlineScriptEnabled() {
         return true;
diff --git a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
index 017e51280c0..105d42c8c86 100644
--- a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
+++ b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
@@ -33,9 +33,11 @@ import org.codehaus.groovy.classgen.GeneratorContext;
 import org.codehaus.groovy.control.CompilationFailedException;
 import org.codehaus.groovy.control.CompilePhase;
 import org.codehaus.groovy.control.CompilerConfiguration;
+import org.codehaus.groovy.control.MultipleCompilationErrorsException;
 import org.codehaus.groovy.control.SourceUnit;
 import org.codehaus.groovy.control.customizers.CompilationCustomizer;
 import org.codehaus.groovy.control.customizers.ImportCustomizer;
+import org.codehaus.groovy.control.messages.Message;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.bootstrap.BootstrapInfo;
 import org.elasticsearch.common.Nullable;
@@ -49,20 +51,26 @@ import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.ScoreAccessor;
 import org.elasticsearch.script.ScriptEngineService;
-import org.elasticsearch.script.GeneralScriptException;
+import org.elasticsearch.script.ScriptException;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.lookup.LeafSearchLookup;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.math.BigDecimal;
 import java.nio.charset.StandardCharsets;
 import java.security.AccessControlContext;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
+import static java.util.Collections.emptyList;
+
 /**
  * Provides the infrastructure for Groovy as a scripting language for Elasticsearch
  */
@@ -78,81 +86,40 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
      */
     public static final String GROOVY_INDY_SETTING_NAME = "indy";
 
-    private final GroovyClassLoader loader;
+    /**
+     * Classloader used as a parent classloader for all Groovy scripts
+     */
+    private final ClassLoader loader;
 
     public GroovyScriptEngineService(Settings settings) {
         super(settings);
-
-        ImportCustomizer imports = new ImportCustomizer();
-        imports.addStarImports("org.joda.time");
-        imports.addStaticStars("java.lang.Math");
-
-        CompilerConfiguration config = new CompilerConfiguration();
-
-        config.addCompilationCustomizers(imports);
-        // Add BigDecimal -> Double transformer
-        config.addCompilationCustomizers(new GroovyBigDecimalTransformer(CompilePhase.CONVERSION));
-
-        // always enable invokeDynamic, not the crazy softreference-based stuff
-        config.getOptimizationOptions().put(GROOVY_INDY_SETTING_NAME, true);
-
-        // Groovy class loader to isolate Groovy-land code
-        // classloader created here
+        // Creates the classloader here in order to isolate Groovy-land code
         final SecurityManager sm = System.getSecurityManager();
         if (sm != null) {
             sm.checkPermission(new SpecialPermission());
         }
-        this.loader = AccessController.doPrivileged(new PrivilegedAction<GroovyClassLoader>() {
-            @Override
-            public GroovyClassLoader run() {
-                // snapshot our context (which has permissions for classes), since the script has none
-                final AccessControlContext engineContext = AccessController.getContext();
-                return new GroovyClassLoader(new ClassLoader(getClass().getClassLoader()) {
-                    @Override
-                    protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
-                        if (sm != null) {
-                            try {
-                                engineContext.checkPermission(new ClassPermission(name));
-                            } catch (SecurityException e) {
-                                throw new ClassNotFoundException(name, e);
-                            }
+        this.loader = AccessController.doPrivileged((PrivilegedAction<ClassLoader>) () -> {
+            // snapshot our context (which has permissions for classes), since the script has none
+            AccessControlContext context = AccessController.getContext();
+            return new ClassLoader(getClass().getClassLoader()) {
+                @Override
+                protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
+                    if (sm != null) {
+                        try {
+                            context.checkPermission(new ClassPermission(name));
+                        } catch (SecurityException e) {
+                            throw new ClassNotFoundException(name, e);
                         }
-                        return super.loadClass(name, resolve);
                     }
-                }, config);
-            }
-        });
-    }
-
-    @Override
-    public void close() {
-        loader.clearCache();
-        // close classloader here (why do we do this?)
-        SecurityManager sm = System.getSecurityManager();
-        if (sm != null) {
-            sm.checkPermission(new SpecialPermission());
-        }
-        AccessController.doPrivileged(new PrivilegedAction<Void>() {
-            @Override
-            public Void run() {
-                try {
-                    loader.close();
-                } catch (IOException e) {
-                    logger.warn("Unable to close Groovy loader", e);
+                    return super.loadClass(name, resolve);
                 }
-                return null;
-            }
+            };
         });
     }
 
     @Override
-    public void scriptRemoved(@Nullable CompiledScript script) {
-        // script could be null, meaning the script has already been garbage collected
-        if (script == null || NAME.equals(script.lang())) {
-            // Clear the cache, this removes old script versions from the
-            // cache to prevent running out of PermGen space
-            loader.clearCache();
-        }
+    public void close() throws IOException {
+        // Nothing to do here
     }
 
     @Override
@@ -167,30 +134,34 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
 
     @Override
     public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
-        try {
-            // we reuse classloader, so do a security check just in case.
-            SecurityManager sm = System.getSecurityManager();
-            if (sm != null) {
-                sm.checkPermission(new SpecialPermission());
-            }
-            String fake = MessageDigests.toHexString(MessageDigests.sha1().digest(scriptSource.getBytes(StandardCharsets.UTF_8)));
-            // same logic as GroovyClassLoader.parseClass() but with a different codesource string:
-            return AccessController.doPrivileged(new PrivilegedAction<Object>() {
-                @Override
-                public Class<?> run() {
-                    GroovyCodeSource gcs = new GroovyCodeSource(scriptSource, fake, BootstrapInfo.UNTRUSTED_CODEBASE);
-                    gcs.setCachable(false);
-                    // TODO: we could be more complicated and paranoid, and move this to separate block, to
-                    // sandbox the compilation process itself better.
-                    return loader.parseClass(gcs);
-                }
-            });
-        } catch (Throwable e) {
-            if (logger.isTraceEnabled()) {
-                logger.trace("exception compiling Groovy script:", e);
-            }
-            throw new GeneralScriptException("failed to compile groovy script", e);
+        // Create the script class name
+        String className = MessageDigests.toHexString(MessageDigests.sha1().digest(scriptSource.getBytes(StandardCharsets.UTF_8)));
+
+        final SecurityManager sm = System.getSecurityManager();
+        if (sm != null) {
+            sm.checkPermission(new SpecialPermission());
         }
+        return AccessController.doPrivileged((PrivilegedAction<Object>) () -> {
+            try {
+                GroovyCodeSource codeSource = new GroovyCodeSource(scriptSource, className, BootstrapInfo.UNTRUSTED_CODEBASE);
+                codeSource.setCachable(false);
+
+                CompilerConfiguration configuration = new CompilerConfiguration()
+                        .addCompilationCustomizers(new ImportCustomizer().addStarImports("org.joda.time").addStaticStars("java.lang.Math"))
+                        .addCompilationCustomizers(new GroovyBigDecimalTransformer(CompilePhase.CONVERSION));
+
+                // always enable invokeDynamic, not the crazy softreference-based stuff
+                configuration.getOptimizationOptions().put(GROOVY_INDY_SETTING_NAME, true);
+
+                GroovyClassLoader groovyClassLoader = new GroovyClassLoader(loader, configuration);
+                return groovyClassLoader.parseClass(codeSource);
+            } catch (Throwable e) {
+                if (logger.isTraceEnabled()) {
+                    logger.trace("Exception compiling Groovy script:", e);
+                }
+                throw convertToScriptException("Error compiling script " + className, scriptSource, e);
+            }
+        });
     }
 
     /**
@@ -215,7 +186,7 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
             }
             return new GroovyScript(compiledScript, createScript(compiledScript.compiled(), allVars), this.logger);
         } catch (ReflectiveOperationException e) {
-            throw new GeneralScriptException("failed to build executable " + compiledScript, e);
+            throw convertToScriptException("Failed to build executable script", compiledScript.name(), e);
         }
     }
 
@@ -235,7 +206,7 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
                 try {
                     scriptObject = createScript(compiledScript.compiled(), allVars);
                 } catch (ReflectiveOperationException e) {
-                    throw new GeneralScriptException("failed to build search " + compiledScript, e);
+                    throw convertToScriptException("Failed to build search script", compiledScript.name(), e);
                 }
                 return new GroovyScript(compiledScript, scriptObject, leafLookup, logger);
             }
@@ -248,6 +219,29 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
         };
     }
 
+    /**
+     * Converts a {@link Throwable} to a {@link ScriptException}
+     */
+    private ScriptException convertToScriptException(String message, String source, Throwable cause) {
+        List<String> stack = new ArrayList<>();
+        if (cause instanceof MultipleCompilationErrorsException) {
+            @SuppressWarnings({"unchecked"})
+            List<Message> errors = (List<Message>) ((MultipleCompilationErrorsException) cause).getErrorCollector().getErrors();
+            for (Message error : errors) {
+                try (StringWriter writer = new StringWriter()) {
+                    error.write(new PrintWriter(writer));
+                    stack.add(writer.toString());
+                } catch (IOException e1) {
+                    logger.error("failed to write compilation error message to the stack", e1);
+                }
+            }
+        } else if (cause instanceof CompilationFailedException) {
+            CompilationFailedException error = (CompilationFailedException) cause;
+            stack.add(error.getMessage());
+        }
+        throw new ScriptException(message, cause, stack, source, NAME);
+    }
+
     public static final class GroovyScript implements ExecutableScript, LeafSearchScript {
 
         private final CompiledScript compiledScript;
@@ -298,17 +292,12 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
             try {
                 // NOTE: we truncate the stack because IndyInterface has security issue (needs getClassLoader)
                 // we don't do a security check just as a tradeoff, it cannot really escalate to anything.
-                return AccessController.doPrivileged(new PrivilegedAction<Object>() {
-                    @Override
-                    public Object run() {
-                        return script.run();
-                    }
-                });
+                return AccessController.doPrivileged((PrivilegedAction<Object>) script::run);
             } catch (Throwable e) {
                 if (logger.isTraceEnabled()) {
                     logger.trace("failed to run {}", e, compiledScript);
                 }
-                throw new GeneralScriptException("failed to run " + compiledScript, e);
+                throw new ScriptException("Error evaluating " + compiledScript.name(), e, emptyList(), "", compiledScript.lang());
             }
         }
 
diff --git a/modules/lang-groovy/src/main/plugin-metadata/plugin-security.policy b/modules/lang-groovy/src/main/plugin-metadata/plugin-security.policy
index 82482448677..13d00d6d17c 100644
--- a/modules/lang-groovy/src/main/plugin-metadata/plugin-security.policy
+++ b/modules/lang-groovy/src/main/plugin-metadata/plugin-security.policy
@@ -26,8 +26,6 @@ grant {
   permission java.lang.RuntimePermission "accessDeclaredMembers";
   permission java.lang.RuntimePermission "accessClassInPackage.sun.reflect";
   permission java.lang.RuntimePermission "accessClassInPackage.jdk.internal.reflect";
-  // needed by GroovyScriptEngineService to close its classloader (why?)
-  permission java.lang.RuntimePermission "closeClassLoader";
   // Allow executing groovy scripts with codesource of /untrusted
   permission groovy.security.GroovyCodeSourcePermission "/untrusted";
 
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java
index 51aad9b2fba..96a5343913d 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovySecurityTests.java
@@ -24,7 +24,7 @@ import org.apache.lucene.util.Constants;
 import org.codehaus.groovy.control.MultipleCompilationErrorsException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.CompiledScript;
-import org.elasticsearch.script.GeneralScriptException;
+import org.elasticsearch.script.ScriptException;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
 
@@ -151,7 +151,7 @@ public class GroovySecurityTests extends ESTestCase {
         try {
             doTest(script);
             fail("did not get expected exception");
-        } catch (GeneralScriptException expected) {
+        } catch (ScriptException expected) {
             Throwable cause = expected.getCause();
             assertNotNull(cause);
             if (exceptionClass.isAssignableFrom(cause.getClass()) == false) {
diff --git a/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/16_update2.yaml b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/16_update2.yaml
index 5e1f99eeeff..5849874c02e 100644
--- a/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/16_update2.yaml
+++ b/modules/lang-groovy/src/test/resources/rest-api-spec/test/lang_groovy/16_update2.yaml
@@ -51,7 +51,7 @@
 
 
   - do:
-      catch: /Unable.to.parse.*/
+      catch: /script_exception,.+Error.compiling.script.*/
       put_script:
         id: "1"
         lang: "groovy"
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
index 283a72b02af..2a48567333b 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
@@ -133,11 +133,6 @@ public final class MustacheScriptEngineService extends AbstractComponent impleme
         // Nothing to do here
     }
 
-    @Override
-    public void scriptRemoved(CompiledScript script) {
-        // Nothing to do here
-    }
-
     // permission checked before doing crazy reflection
     static final SpecialPermission SPECIAL_PERMISSION = new SpecialPermission();
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
index bb94785d339..ba98c7ee62f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngineService.java
@@ -202,15 +202,6 @@ public final class PainlessScriptEngineService extends AbstractComponent impleme
         };
     }
 
-    /**
-     * Action taken when a script is removed from the cache.
-     * @param script The removed script.
-     */
-    @Override
-    public void scriptRemoved(final CompiledScript script) {
-        // Nothing to do.
-    }
-
     /**
      * Action taken when the engine is closed.
      */
diff --git a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
index 3ef030bdeb0..3a556ce4429 100644
--- a/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
+++ b/plugins/lang-javascript/src/main/java/org/elasticsearch/script/javascript/JavaScriptScriptEngineService.java
@@ -148,11 +148,6 @@ public class JavaScriptScriptEngineService extends AbstractComponent implements
 
     @Override
     public void close() {
-
-    }
-
-    @Override
-    public void scriptRemoved(@Nullable CompiledScript compiledScript) {
         // Nothing to do here
     }
 
diff --git a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
index 26bf1bed6fc..b642b7b0a73 100644
--- a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
+++ b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
@@ -142,11 +142,6 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri
         interp.cleanup();
     }
 
-    @Override
-    public void scriptRemoved(@Nullable CompiledScript compiledScript) {
-        // Nothing to do
-    }
-
     public class PythonExecutableScript implements ExecutableScript {
 
         private final PyCode code;
diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
index 64107f56fa7..61a0cf6c236 100644
--- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
+++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
@@ -110,10 +110,6 @@ public class MockScriptEngine implements ScriptEngineService {
         };
     }
 
-    @Override
-    public void scriptRemoved(@Nullable CompiledScript script) {
-    }
-
     @Override
     public void close() throws IOException {
     }

From 260f38fd76ec262bb0a84573e1b98af689663a68 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Fri, 17 Jun 2016 22:57:00 +0200
Subject: [PATCH 0953/1311] Remove VersionModule and use Version#current
 consistently.

We pretended to be able to ackt like a different version node for so long it's
time to be honest and remove this ability. It's just confusing and where needed
and tested we should build dedicated extension points.
---
 .../resources/checkstyle_suppressions.xml     |  1 -
 .../main/java/org/elasticsearch/Version.java  | 15 -------
 .../action/main/TransportMainAction.java      |  7 ++-
 .../client/transport/TransportClient.java     |  1 -
 .../TransportClientNodesService.java          |  4 +-
 .../metadata/MetaDataCreateIndexService.java  |  6 +--
 .../cluster/node/DiscoveryNodeService.java    |  6 +--
 .../zen/elect/ElectMasterService.java         |  4 +-
 .../zen/ping/unicast/UnicastZenPing.java      |  9 ++--
 .../BlobStoreIndexShardRepository.java        |  3 --
 .../java/org/elasticsearch/node/Node.java     |  7 ++-
 .../node/service/NodeService.java             |  9 ++--
 .../transport/local/LocalTransport.java       | 10 +++--
 .../transport/netty/NettyTransport.java       | 11 +++--
 .../elasticsearch/tribe/TribeClientNode.java  |  3 +-
 .../node/tasks/TaskManagerTestCase.java       |  2 +-
 .../MetaDataIndexTemplateServiceTests.java    |  2 -
 .../action/main/MainActionTests.java          |  2 +-
 .../BroadcastReplicationTests.java            |  2 +-
 .../TransportClientNodesServiceTests.java     |  2 +-
 .../MetaDataCreateIndexServiceTests.java      |  1 -
 .../node/DiscoveryNodeServiceTests.java       |  4 +-
 .../common/network/NetworkModuleTests.java    |  2 +-
 .../discovery/DiscoveryModuleTests.java       |  4 +-
 .../discovery/ZenFaultDetectionTests.java     |  7 ++-
 .../zen/ElectMasterServiceTests.java          |  5 +--
 .../zen/NodeJoinControllerTests.java          |  2 +-
 .../discovery/zen/ZenDiscoveryIT.java         |  2 +-
 .../zen/ping/unicast/UnicastZenPingIT.java    | 33 ++++++++++----
 .../PublishClusterStateActionTests.java       | 45 ++++++++-----------
 .../elasticsearch/http/HttpServerTests.java   |  2 +-
 .../mapper/DynamicMappingDisabledTests.java   |  2 +-
 .../indices/cluster/ClusterStateChanges.java  |  2 +-
 .../NettySizeHeaderFrameDecoderTests.java     |  2 +-
 .../NettyTransportServiceHandshakeTests.java  |  1 -
 .../transport/TransportModuleTests.java       |  4 +-
 .../netty/NettyScheduledPingTests.java        |  4 +-
 .../transport/netty/NettyTransportIT.java     |  4 +-
 .../netty/NettyTransportMultiPortTests.java   |  2 +-
 .../netty/SimpleNettyTransportTests.java      |  2 +-
 .../azure/AzureUnicastHostsProvider.java      |  7 +--
 .../ec2/AwsEc2UnicastHostsProvider.java       |  7 +--
 .../gce/GceUnicastHostsProvider.java          |  7 +--
 .../repositories/azure/AzureRepositoryF.java  |  3 +-
 .../java/org/elasticsearch/node/MockNode.java | 10 +----
 .../test/ESSingleNodeTestCase.java            |  7 +--
 .../test/InternalTestCluster.java             |  5 +--
 .../transport/AssertingLocalTransport.java    |  5 +--
 .../test/transport/MockTransportService.java  | 17 +++++--
 49 files changed, 137 insertions(+), 167 deletions(-)

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 94494a06542..ffad8d37c5b 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -859,7 +859,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]ZenUnicastDiscoveryIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]NodeJoinControllerTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]ZenDiscoveryUnitTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]ping[/\\]unicast[/\\]UnicastZenPingIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]publish[/\\]PublishClusterStateActionTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]document[/\\]DocumentActionsIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]env[/\\]EnvironmentTests.java" checks="LineLength" />
diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java
index e77b0f450dd..7e2ffd1dfa0 100644
--- a/core/src/main/java/org/elasticsearch/Version.java
+++ b/core/src/main/java/org/elasticsearch/Version.java
@@ -22,7 +22,6 @@ package org.elasticsearch;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
@@ -329,18 +328,4 @@ public class Version {
     public boolean isRC() {
         return build > 50 && build < 99;
     }
-
-    public static class Module extends AbstractModule {
-
-        private final Version version;
-
-        public Module(Version version) {
-            this.version = version;
-        }
-
-        @Override
-        protected void configure() {
-            bind(Version.class).toInstance(version);
-        }
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/main/TransportMainAction.java b/core/src/main/java/org/elasticsearch/action/main/TransportMainAction.java
index 6bf60835490..c37268a52de 100644
--- a/core/src/main/java/org/elasticsearch/action/main/TransportMainAction.java
+++ b/core/src/main/java/org/elasticsearch/action/main/TransportMainAction.java
@@ -37,15 +37,13 @@ import org.elasticsearch.transport.TransportService;
 public class TransportMainAction extends HandledTransportAction<MainRequest, MainResponse> {
 
     private final ClusterService clusterService;
-    private final Version version;
 
     @Inject
     public TransportMainAction(Settings settings, ThreadPool threadPool, TransportService transportService,
                                ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
-                               ClusterService clusterService, Version version) {
+                               ClusterService clusterService) {
         super(settings, MainAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, MainRequest::new);
         this.clusterService = clusterService;
-        this.version = version;
     }
 
     @Override
@@ -54,6 +52,7 @@ public class TransportMainAction extends HandledTransportAction<MainRequest, Mai
         assert Node.NODE_NAME_SETTING.exists(settings);
         final boolean available = clusterState.getBlocks().hasGlobalBlock(RestStatus.SERVICE_UNAVAILABLE) == false;
         listener.onResponse(
-            new MainResponse(Node.NODE_NAME_SETTING.get(settings), version, clusterState.getClusterName(), Build.CURRENT, available));
+            new MainResponse(Node.NODE_NAME_SETTING.get(settings), Version.CURRENT, clusterState.getClusterName(), Build.CURRENT,
+                available));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
index 089fb41a321..2c2a5ad0428 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
@@ -129,7 +129,6 @@ public class TransportClient extends AbstractClient {
             boolean success = false;
             try {
                 ModulesBuilder modules = new ModulesBuilder();
-                modules.add(new Version.Module(version));
                 // plugin modules must be added here, before others or we can get crazy injection errors...
                 for (Module pluginModule : pluginsService.nodeModules()) {
                     modules.add(pluginModule);
diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
index cfa8adc28a7..0781576524d 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
@@ -112,12 +112,12 @@ public class TransportClientNodesService extends AbstractComponent {
 
     @Inject
     public TransportClientNodesService(Settings settings,TransportService transportService,
-                                       ThreadPool threadPool, Version version) {
+                                       ThreadPool threadPool) {
         super(settings);
         this.clusterName = ClusterName.CLUSTER_NAME_SETTING.get(settings);
         this.transportService = transportService;
         this.threadPool = threadPool;
-        this.minCompatibilityVersion = version.minimumCompatibilityVersion();
+        this.minCompatibilityVersion = Version.CURRENT.minimumCompatibilityVersion();
 
         this.nodesSamplerInterval = CLIENT_TRANSPORT_NODES_SAMPLER_INTERVAL.get(this.settings);
         this.pingTimeout = CLIENT_TRANSPORT_PING_TIMEOUT.get(this.settings).millis();
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
index 2b482635879..456675abcf1 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
@@ -103,7 +103,6 @@ public class MetaDataCreateIndexService extends AbstractComponent {
     private final ClusterService clusterService;
     private final IndicesService indicesService;
     private final AllocationService allocationService;
-    private final Version version;
     private final AliasValidator aliasValidator;
     private final IndexTemplateFilter indexTemplateFilter;
     private final Environment env;
@@ -114,13 +113,12 @@ public class MetaDataCreateIndexService extends AbstractComponent {
     @Inject
     public MetaDataCreateIndexService(Settings settings, ClusterService clusterService,
                                       IndicesService indicesService, AllocationService allocationService,
-                                      Version version, AliasValidator aliasValidator,
+                                      AliasValidator aliasValidator,
                                       Set<IndexTemplateFilter> indexTemplateFilters, Environment env, NodeServicesProvider nodeServicesProvider, IndexScopedSettings indexScopedSettings) {
         super(settings);
         this.clusterService = clusterService;
         this.indicesService = indicesService;
         this.allocationService = allocationService;
-        this.version = version;
         this.aliasValidator = aliasValidator;
         this.env = env;
         this.nodeServicesProvider = nodeServicesProvider;
@@ -287,7 +285,7 @@ public class MetaDataCreateIndexService extends AbstractComponent {
 
                             if (indexSettingsBuilder.get(SETTING_VERSION_CREATED) == null) {
                                 DiscoveryNodes nodes = currentState.nodes();
-                                final Version createdVersion = Version.smallest(version, nodes.getSmallestNonClientNodeVersion());
+                                final Version createdVersion = Version.smallest(Version.CURRENT, nodes.getSmallestNonClientNodeVersion());
                                 indexSettingsBuilder.put(SETTING_VERSION_CREATED, createdVersion);
                             }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeService.java b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeService.java
index 177c67f2986..06d0c7dbf24 100644
--- a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeService.java
@@ -46,12 +46,10 @@ public class DiscoveryNodeService extends AbstractComponent {
             // don't use node.id.seed so it won't be seen as an attribute
             Setting.longSetting("node_id.seed", 0L, Long.MIN_VALUE, Property.NodeScope);
     private final List<CustomAttributesProvider> customAttributesProviders = new CopyOnWriteArrayList<>();
-    private final Version version;
 
     @Inject
-    public DiscoveryNodeService(Settings settings, Version version) {
+    public DiscoveryNodeService(Settings settings) {
         super(settings);
-        this.version = version;
     }
 
     public static String generateNodeId(Settings settings) {
@@ -93,7 +91,7 @@ public class DiscoveryNodeService extends AbstractComponent {
             }
         }
         return new DiscoveryNode(Node.NODE_NAME_SETTING.get(settings), nodeId, publishAddress, attributes,
-                roles, version);
+                roles, Version.CURRENT);
     }
 
     public interface CustomAttributesProvider {
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/elect/ElectMasterService.java b/core/src/main/java/org/elasticsearch/discovery/zen/elect/ElectMasterService.java
index 5ff79f112d2..76b7de6da08 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/elect/ElectMasterService.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/elect/ElectMasterService.java
@@ -53,9 +53,9 @@ public class ElectMasterService extends AbstractComponent {
     private volatile int minimumMasterNodes;
 
     @Inject
-    public ElectMasterService(Settings settings, Version version) {
+    public ElectMasterService(Settings settings) {
         super(settings);
-        this.minMasterVersion = version.minimumCompatibilityVersion();
+        this.minMasterVersion = Version.CURRENT.minimumCompatibilityVersion();
         this.minimumMasterNodes = DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.get(settings);
         logger.debug("using minimum_master_nodes [{}]", minimumMasterNodes);
     }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
index 0d3c7f9d086..06fc9eae13d 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
@@ -132,8 +132,7 @@ public class UnicastZenPing extends AbstractLifecycleComponent<ZenPing> implemen
 
     @Inject
     public UnicastZenPing(Settings settings, ThreadPool threadPool, TransportService transportService,
-                          Version version, ElectMasterService electMasterService,
-                          @Nullable Set<UnicastHostsProvider> unicastHostsProviders) {
+                          ElectMasterService electMasterService, @Nullable Set<UnicastHostsProvider> unicastHostsProviders) {
         super(settings);
         this.threadPool = threadPool;
         this.transportService = transportService;
@@ -166,7 +165,7 @@ public class UnicastZenPing extends AbstractLifecycleComponent<ZenPing> implemen
                 TransportAddress[] addresses = transportService.addressesFromString(host, limitPortCounts);
                 for (TransportAddress address : addresses) {
                     configuredTargetNodes.add(new DiscoveryNode(UNICAST_NODE_PREFIX + unicastNodeIdGenerator.incrementAndGet() + "#",
-                            address, emptyMap(), emptySet(), version.minimumCompatibilityVersion()));
+                            address, emptyMap(), emptySet(), getVersion().minimumCompatibilityVersion()));
                 }
             } catch (Exception e) {
                 throw new IllegalArgumentException("Failed to resolve address for [" + host + "]", e);
@@ -586,4 +585,8 @@ public class UnicastZenPing extends AbstractLifecycleComponent<ZenPing> implemen
             }
         }
     }
+
+    protected Version getVersion() {
+        return Version.CURRENT; // for tests
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java
index 94337ecdbc5..96230459934 100644
--- a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java
+++ b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java
@@ -105,8 +105,6 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements
 
     private RateLimiter restoreRateLimiter;
 
-    private RateLimiterListener rateLimiterListener;
-
     private RateLimitingInputStream.Listener snapshotThrottleListener;
 
     private RateLimitingInputStream.Listener restoreThrottleListener;
@@ -163,7 +161,6 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements
         this.chunkSize = chunkSize;
         this.snapshotRateLimiter = snapshotRateLimiter;
         this.restoreRateLimiter = restoreRateLimiter;
-        this.rateLimiterListener = rateLimiterListener;
         this.snapshotThrottleListener = nanos -> rateLimiterListener.onSnapshotPause(nanos);
         this.restoreThrottleListener = nanos -> rateLimiterListener.onRestorePause(nanos);
         this.compress = compress;
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index a2a09938ead..70e0c356ba2 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -174,17 +174,17 @@ public class Node implements Closeable {
      * @param preparedSettings Base settings to configure the node with
      */
     public Node(Settings preparedSettings) {
-        this(InternalSettingsPreparer.prepareEnvironment(preparedSettings, null), Version.CURRENT, Collections.<Class<? extends Plugin>>emptyList());
+        this(InternalSettingsPreparer.prepareEnvironment(preparedSettings, null), Collections.<Class<? extends Plugin>>emptyList());
     }
 
-    protected Node(Environment tmpEnv, Version version, Collection<Class<? extends Plugin>> classpathPlugins) {
+    protected Node(Environment tmpEnv, Collection<Class<? extends Plugin>> classpathPlugins) {
         Settings tmpSettings = Settings.builder().put(tmpEnv.settings())
                 .put(Client.CLIENT_TYPE_SETTING_S.getKey(), CLIENT_TYPE).build();
         final List<Closeable> resourcesToClose = new ArrayList<>(); // register everything we need to release in the case of an error
 
         tmpSettings = TribeService.processSettings(tmpSettings);
         ESLogger logger = Loggers.getLogger(Node.class, NODE_NAME_SETTING.get(tmpSettings));
-        final String displayVersion = version + (Build.CURRENT.isSnapshot() ? "-SNAPSHOT" : "");
+        final String displayVersion = Version.CURRENT + (Build.CURRENT.isSnapshot() ? "-SNAPSHOT" : "");
         final JvmInfo jvmInfo = JvmInfo.jvmInfo();
         logger.info(
             "version[{}], pid[{}], build[{}/{}], OS[{}/{}/{}], JVM[{}/{}/{}/{}]",
@@ -247,7 +247,6 @@ public class Node implements Closeable {
             resourcesToClose.add(clusterService);
             NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
             ModulesBuilder modules = new ModulesBuilder();
-            modules.add(new Version.Module(version));
             // plugin modules must be added here, before others or we can get crazy injection errors...
             for (Module pluginModule : pluginsService.nodeModules()) {
                 modules.add(pluginModule);
diff --git a/core/src/main/java/org/elasticsearch/node/service/NodeService.java b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
index 1cbdfe12d2c..6a07565b6cd 100644
--- a/core/src/main/java/org/elasticsearch/node/service/NodeService.java
+++ b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
@@ -70,14 +70,12 @@ public class NodeService extends AbstractComponent implements Closeable {
 
     private volatile Map<String, String> serviceAttributes = emptyMap();
 
-    private final Version version;
-
     private final Discovery discovery;
 
     @Inject
     public NodeService(Settings settings, ThreadPool threadPool, MonitorService monitorService,
                        Discovery discovery, TransportService transportService, IndicesService indicesService,
-                       PluginsService pluginService, CircuitBreakerService circuitBreakerService, Version version,
+                       PluginsService pluginService, CircuitBreakerService circuitBreakerService,
                        ProcessorsRegistry.Builder processorsRegistryBuilder, ClusterService clusterService, SettingsFilter settingsFilter) {
         super(settings);
         this.threadPool = threadPool;
@@ -85,7 +83,6 @@ public class NodeService extends AbstractComponent implements Closeable {
         this.transportService = transportService;
         this.indicesService = indicesService;
         this.discovery = discovery;
-        this.version = version;
         this.pluginService = pluginService;
         this.circuitBreakerService = circuitBreakerService;
         this.clusterService = clusterService;
@@ -126,7 +123,7 @@ public class NodeService extends AbstractComponent implements Closeable {
     }
 
     public NodeInfo info() {
-        return new NodeInfo(version, Build.CURRENT, discovery.localNode(), serviceAttributes,
+        return new NodeInfo(Version.CURRENT, Build.CURRENT, discovery.localNode(), serviceAttributes,
                 settings,
                 monitorService.osService().info(),
                 monitorService.processService().info(),
@@ -141,7 +138,7 @@ public class NodeService extends AbstractComponent implements Closeable {
 
     public NodeInfo info(boolean settings, boolean os, boolean process, boolean jvm, boolean threadPool,
                          boolean transport, boolean http, boolean plugin, boolean ingest) {
-        return new NodeInfo(version, Build.CURRENT, discovery.localNode(), serviceAttributes,
+        return new NodeInfo(Version.CURRENT, Build.CURRENT, discovery.localNode(), serviceAttributes,
                 settings ? settingsFilter.filter(this.settings) : null,
                 os ? monitorService.osService().info() : null,
                 process ? monitorService.processService().info() : null,
diff --git a/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java b/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
index d1ac749e4ac..4caf1a543c0 100644
--- a/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
@@ -77,7 +77,6 @@ public class LocalTransport extends AbstractLifecycleComponent<Transport> implem
     public static final String LOCAL_TRANSPORT_THREAD_NAME_PREFIX = "local_transport";
     final ThreadPool threadPool;
     private final ThreadPoolExecutor workers;
-    private final Version version;
     private volatile TransportServiceAdapter transportServiceAdapter;
     private volatile BoundTransportAddress boundAddress;
     private volatile LocalTransportAddress localAddress;
@@ -92,11 +91,10 @@ public class LocalTransport extends AbstractLifecycleComponent<Transport> implem
     public static final String TRANSPORT_LOCAL_QUEUE = "transport.local.queue";
 
     @Inject
-    public LocalTransport(Settings settings, ThreadPool threadPool, Version version,
+    public LocalTransport(Settings settings, ThreadPool threadPool,
                           NamedWriteableRegistry namedWriteableRegistry, CircuitBreakerService circuitBreakerService) {
         super(settings);
         this.threadPool = threadPool;
-        this.version = version;
         int workerCount = this.settings.getAsInt(TRANSPORT_LOCAL_WORKERS, EsExecutors.boundedNumberOfProcessors(settings));
         int queueSize = this.settings.getAsInt(TRANSPORT_LOCAL_QUEUE, -1);
         logger.debug("creating [{}] workers, queue_size [{}]", workerCount, queueSize);
@@ -207,7 +205,7 @@ public class LocalTransport extends AbstractLifecycleComponent<Transport> implem
     @Override
     public void sendRequest(final DiscoveryNode node, final long requestId, final String action, final TransportRequest request,
             TransportRequestOptions options) throws IOException, TransportException {
-        final Version version = Version.smallest(node.getVersion(), this.version);
+        final Version version = Version.smallest(node.getVersion(), getVersion());
 
         try (BytesStreamOutput stream = new BytesStreamOutput()) {
             stream.setVersion(version);
@@ -404,4 +402,8 @@ public class LocalTransport extends AbstractLifecycleComponent<Transport> implem
     public List<String> getLocalAddresses() {
         return Collections.singletonList("0.0.0.0");
     }
+
+    protected Version getVersion() { // for tests
+        return Version.CURRENT;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
index a5542f9b403..c9f02066836 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
@@ -210,7 +210,6 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
         intSetting("transport.netty.boss_count", 1, 1, Property.NodeScope);
 
     protected final NetworkService networkService;
-    protected final Version version;
 
     protected final boolean blockingClient;
     protected final TimeValue connectTimeout;
@@ -254,13 +253,12 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
     final ScheduledPing scheduledPing;
 
     @Inject
-    public NettyTransport(Settings settings, ThreadPool threadPool, NetworkService networkService, BigArrays bigArrays, Version version,
+    public NettyTransport(Settings settings, ThreadPool threadPool, NetworkService networkService, BigArrays bigArrays,
                           NamedWriteableRegistry namedWriteableRegistry, CircuitBreakerService circuitBreakerService) {
         super(settings);
         this.threadPool = threadPool;
         this.networkService = networkService;
         this.bigArrays = bigArrays;
-        this.version = version;
 
         this.workerCount = WORKER_COUNT.get(settings);
         this.blockingClient = TCP_BLOCKING_CLIENT.get(settings);
@@ -894,7 +892,7 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
             // we pick the smallest of the 2, to support both backward and forward compatibility
             // note, this is the only place we need to do this, since from here on, we use the serialized version
             // as the version to use also when the node receiving this request will send the response with
-            Version version = Version.smallest(this.version, node.getVersion());
+            Version version = Version.smallest(getCurrentVersion(), node.getVersion());
 
             stream.setVersion(version);
             threadPool.getThreadContext().writeTo(stream);
@@ -1401,4 +1399,9 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
             }
         }
     }
+
+    protected Version getCurrentVersion() {
+        // this is just for tests to mock stuff like the nodes version - tests can override this internally
+        return Version.CURRENT;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/tribe/TribeClientNode.java b/core/src/main/java/org/elasticsearch/tribe/TribeClientNode.java
index 688dfe5a92d..02957ab99a4 100644
--- a/core/src/main/java/org/elasticsearch/tribe/TribeClientNode.java
+++ b/core/src/main/java/org/elasticsearch/tribe/TribeClientNode.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.tribe;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.node.Node;
@@ -32,6 +31,6 @@ import java.util.Collections;
  */
 class TribeClientNode extends Node {
     TribeClientNode(Settings settings) {
-        super(new Environment(settings), Version.CURRENT, Collections.<Class<? extends Plugin>>emptyList());
+        super(new Environment(settings), Collections.<Class<? extends Plugin>>emptyList());
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
index e317d4e6d7a..eec285e7ffb 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskManagerTestCase.java
@@ -167,7 +167,7 @@ public abstract class TaskManagerTestCase extends ESTestCase {
         public TestNode(String name, ThreadPool threadPool, Settings settings) {
             clusterService = createClusterService(threadPool);
             transportService = new TransportService(settings,
-                    new LocalTransport(settings, threadPool, Version.CURRENT, new NamedWriteableRegistry(),
+                    new LocalTransport(settings, threadPool, new NamedWriteableRegistry(),
                         new NoneCircuitBreakerService()), threadPool) {
                 @Override
                 protected TaskManager createTaskManager() {
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
index 4c3f21f988d..d62fe30f6fa 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
@@ -158,7 +158,6 @@ public class MetaDataIndexTemplateServiceTests extends ESSingleNodeTestCase {
                 null,
                 null,
                 null,
-                Version.CURRENT,
                 null,
                 new HashSet<>(),
                 null,
@@ -189,7 +188,6 @@ public class MetaDataIndexTemplateServiceTests extends ESSingleNodeTestCase {
             clusterService,
             indicesService,
             null,
-            Version.CURRENT,
             null,
             new HashSet<>(),
             null,
diff --git a/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java b/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java
index ce3fa93d214..2bff71d3c40 100644
--- a/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java
@@ -112,7 +112,7 @@ public class MainActionTests extends ESTestCase {
         when(clusterService.state()).thenReturn(state);
 
         TransportMainAction action = new TransportMainAction(settings, mock(ThreadPool.class), mock(TransportService.class),
-            mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), clusterService, Version.CURRENT);
+            mock(ActionFilters.class), mock(IndexNameExpressionResolver.class), clusterService);
         AtomicReference<MainResponse> responseRef = new AtomicReference<>();
         action.doExecute(new MainRequest(), new ActionListener<MainResponse>() {
             @Override
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
index abebb61c0b7..e7971dd4c1d 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java
@@ -89,7 +89,7 @@ public class BroadcastReplicationTests extends ESTestCase {
     @Before
     public void setUp() throws Exception {
         super.setUp();
-        LocalTransport transport = new LocalTransport(Settings.EMPTY, threadPool, Version.CURRENT, new NamedWriteableRegistry(), circuitBreakerService);
+        LocalTransport transport = new LocalTransport(Settings.EMPTY, threadPool, new NamedWriteableRegistry(), circuitBreakerService);
         clusterService = createClusterService(threadPool);
         transportService = new TransportService(clusterService.getSettings(), transport, threadPool);
         transportService.start();
diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
index 2a32ebd05af..5c07f5e6f25 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
@@ -102,7 +102,7 @@ public class TransportClientNodesServiceTests extends ESTestCase {
             transportService.start();
             transportService.acceptIncomingRequests();
             transportClientNodesService =
-                    new TransportClientNodesService(settings, transportService, threadPool, Version.CURRENT);
+                    new TransportClientNodesService(settings, transportService, threadPool);
             this.nodesCount = randomIntBetween(1, 10);
             for (int i = 0; i < nodesCount; i++) {
                 transportClientNodesService.addTransportAddresses(new LocalTransportAddress("node" + i));
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
index 28a25d17337..04d27020273 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
@@ -212,7 +212,6 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
             null,
             null,
             null,
-            Version.CURRENT,
             null,
             new HashSet<>(),
             null,
diff --git a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeServiceTests.java
index c03a5ab06dc..fb38a428a76 100644
--- a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeServiceTests.java
@@ -54,7 +54,7 @@ public class DiscoveryNodeServiceTests extends ESTestCase {
                 }
             }
         }
-        DiscoveryNodeService discoveryNodeService = new DiscoveryNodeService(builder.build(), Version.CURRENT);
+        DiscoveryNodeService discoveryNodeService = new DiscoveryNodeService(builder.build());
         DiscoveryNode discoveryNode = discoveryNodeService.buildLocalNode(DummyTransportAddress.INSTANCE);
         assertThat(discoveryNode.getRoles(), equalTo(selectedRoles));
         assertThat(discoveryNode.getAttributes(), equalTo(expectedAttributes));
@@ -68,7 +68,7 @@ public class DiscoveryNodeServiceTests extends ESTestCase {
             builder.put("node.attr.attr" + i, "value" + i);
             expectedAttributes.put("attr" + i, "value" + i);
         }
-        DiscoveryNodeService discoveryNodeService = new DiscoveryNodeService(builder.build(), Version.CURRENT);
+        DiscoveryNodeService discoveryNodeService = new DiscoveryNodeService(builder.build());
         int numCustomAttributes = randomIntBetween(0, 5);
         Map<String, String> customAttributes = new HashMap<>();
         for (int i = 0; i < numCustomAttributes; i++) {
diff --git a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
index d5fc4630bcf..9868d92129f 100644
--- a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
@@ -58,7 +58,7 @@ public class NetworkModuleTests extends ModuleTestCase {
 
     static class FakeTransport extends AssertingLocalTransport {
         public FakeTransport() {
-            super(null, null, null, null, null);
+            super(null, null, null, null);
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/discovery/DiscoveryModuleTests.java b/core/src/test/java/org/elasticsearch/discovery/DiscoveryModuleTests.java
index 00ed8a2673c..86c3cd91bff 100644
--- a/core/src/test/java/org/elasticsearch/discovery/DiscoveryModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/DiscoveryModuleTests.java
@@ -33,8 +33,8 @@ public class DiscoveryModuleTests extends ModuleTestCase {
 
     public static class DummyMasterElectionService extends ElectMasterService {
 
-        public DummyMasterElectionService(Settings settings, Version version) {
-            super(settings, version);
+        public DummyMasterElectionService(Settings settings) {
+            super(settings);
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java b/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
index 90a3a2e8c1e..c6e55ac9181 100644
--- a/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/ZenFaultDetectionTests.java
@@ -141,7 +141,12 @@ public class ZenFaultDetectionTests extends ESTestCase {
                              // trace zenfd actions but keep the default otherwise
                             .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), singleton(TransportLivenessAction.NAME))
                             .build(),
-                        new LocalTransport(settings, threadPool, version, namedWriteableRegistry, circuitBreakerService),
+                        new LocalTransport(settings, threadPool, namedWriteableRegistry, circuitBreakerService) {
+                            @Override
+                            protected Version getVersion() {
+                                return version;
+                            }
+                        },
                         threadPool);
         transportService.start();
         transportService.acceptIncomingRequests();
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java
index 498aff28ee6..0f93e5d460c 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java
@@ -35,15 +35,12 @@ import java.util.Set;
 public class ElectMasterServiceTests extends ESTestCase {
 
     ElectMasterService electMasterService() {
-        return new ElectMasterService(Settings.EMPTY, Version.CURRENT);
+        return new ElectMasterService(Settings.EMPTY);
     }
 
     List<DiscoveryNode> generateRandomNodes() {
         int count = scaledRandomIntBetween(1, 100);
         ArrayList<DiscoveryNode> nodes = new ArrayList<>(count);
-
-
-
         for (int i = 0; i < count; i++) {
             Set<DiscoveryNode.Role> roles = new HashSet<>();
             if (randomBoolean()) {
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
index 683bebc55a3..e994c376c4d 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
@@ -99,7 +99,7 @@ public class NodeJoinControllerTests extends ESTestCase {
         setState(clusterService, ClusterState.builder(clusterService.state()).nodes(
             DiscoveryNodes.builder(initialNodes).masterNodeId(localNode.getId())));
         nodeJoinController = new NodeJoinController(clusterService, new NoopAllocationService(Settings.EMPTY),
-            new ElectMasterService(Settings.EMPTY, Version.CURRENT),
+            new ElectMasterService(Settings.EMPTY),
             new DiscoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)),
             Settings.EMPTY);
     }
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java
index d12a0ff55a3..c5f92e62de0 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java
@@ -320,7 +320,7 @@ public class ZenDiscoveryIT extends ESIntegTestCase {
     }
 
     public void testJoinElectedMaster_incompatibleMinVersion() {
-        ElectMasterService electMasterService = new ElectMasterService(Settings.EMPTY, Version.CURRENT);
+        ElectMasterService electMasterService = new ElectMasterService(Settings.EMPTY);
 
         DiscoveryNode node = new DiscoveryNode("_node_id", new LocalTransportAddress("_id"), emptyMap(),
                 Collections.singleton(DiscoveryNode.Role.MASTER), Version.CURRENT);
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java b/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java
index 11816512583..f072c5faf8a 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java
@@ -68,7 +68,7 @@ public class UnicastZenPingIT extends ESTestCase {
 
         ThreadPool threadPool = new TestThreadPool(getClass().getName());
         NetworkService networkService = new NetworkService(settings);
-        ElectMasterService electMasterService = new ElectMasterService(settings, Version.CURRENT);
+        ElectMasterService electMasterService = new ElectMasterService(settings);
 
         NetworkHandle handleA = startServices(settings, threadPool, networkService, "UZP_A", Version.CURRENT);
         NetworkHandle handleB = startServices(settings, threadPool, networkService, "UZP_B", Version.CURRENT);
@@ -88,7 +88,7 @@ public class UnicastZenPingIT extends ESTestCase {
                 .build();
 
         Settings hostsSettingsMismatch = Settings.builder().put(hostsSettings).put(settingsMismatch).build();
-        UnicastZenPing zenPingA = new UnicastZenPing(hostsSettings, threadPool, handleA.transportService, Version.CURRENT, electMasterService, null);
+        UnicastZenPing zenPingA = new UnicastZenPing(hostsSettings, threadPool, handleA.transportService, electMasterService, null);
         zenPingA.setPingContextProvider(new PingContextProvider() {
             @Override
             public DiscoveryNodes nodes() {
@@ -102,7 +102,7 @@ public class UnicastZenPingIT extends ESTestCase {
         });
         zenPingA.start();
 
-        UnicastZenPing zenPingB = new UnicastZenPing(hostsSettings, threadPool, handleB.transportService, Version.CURRENT, electMasterService, null);
+        UnicastZenPing zenPingB = new UnicastZenPing(hostsSettings, threadPool, handleB.transportService, electMasterService, null);
         zenPingB.setPingContextProvider(new PingContextProvider() {
             @Override
             public DiscoveryNodes nodes() {
@@ -116,7 +116,13 @@ public class UnicastZenPingIT extends ESTestCase {
         });
         zenPingB.start();
 
-        UnicastZenPing zenPingC = new UnicastZenPing(hostsSettingsMismatch, threadPool, handleC.transportService, versionD, electMasterService, null);
+        UnicastZenPing zenPingC = new UnicastZenPing(hostsSettingsMismatch, threadPool, handleC.transportService, electMasterService,
+            null) {
+            @Override
+            protected Version getVersion() {
+                return versionD;
+            }
+        };
         zenPingC.setPingContextProvider(new PingContextProvider() {
             @Override
             public DiscoveryNodes nodes() {
@@ -130,7 +136,7 @@ public class UnicastZenPingIT extends ESTestCase {
         });
         zenPingC.start();
 
-        UnicastZenPing zenPingD = new UnicastZenPing(hostsSettingsMismatch, threadPool, handleD.transportService, Version.CURRENT, electMasterService, null);
+        UnicastZenPing zenPingD = new UnicastZenPing(hostsSettingsMismatch, threadPool, handleD.transportService, electMasterService, null);
         zenPingD.setPingContextProvider(new PingContextProvider() {
             @Override
             public DiscoveryNodes nodes() {
@@ -191,8 +197,15 @@ public class UnicastZenPingIT extends ESTestCase {
         }
     }
 
-    private NetworkHandle startServices(Settings settings, ThreadPool threadPool, NetworkService networkService, String nodeId, Version version) {
-        NettyTransport transport = new NettyTransport(settings, threadPool, networkService, BigArrays.NON_RECYCLING_INSTANCE, version, new NamedWriteableRegistry(), new NoneCircuitBreakerService());
+    private NetworkHandle startServices(Settings settings, ThreadPool threadPool, NetworkService networkService, String nodeId,
+                                        Version version) {
+        NettyTransport transport = new NettyTransport(settings, threadPool, networkService, BigArrays.NON_RECYCLING_INSTANCE,
+            new NamedWriteableRegistry(), new NoneCircuitBreakerService()) {
+            @Override
+            protected Version getCurrentVersion() {
+                return version;
+            }
+        };
         final TransportService transportService = new TransportService(settings, transport, threadPool);
         transportService.start();
         transportService.acceptIncomingRequests();
@@ -208,7 +221,8 @@ public class UnicastZenPingIT extends ESTestCase {
             public void onNodeDisconnected(DiscoveryNode node) {
             }
         });
-        final DiscoveryNode node = new DiscoveryNode(nodeId, transportService.boundAddress().publishAddress(), emptyMap(), emptySet(), version);
+        final DiscoveryNode node = new DiscoveryNode(nodeId, transportService.boundAddress().publishAddress(), emptyMap(), emptySet(),
+            version);
         transportService.setLocalNode(node);
         return new NetworkHandle((InetSocketTransportAddress)transport.boundAddress().publishAddress(), transportService, node, counters);
     }
@@ -219,7 +233,8 @@ public class UnicastZenPingIT extends ESTestCase {
         public final DiscoveryNode node;
         public final ConcurrentMap<TransportAddress, AtomicInteger> counters;
 
-        public NetworkHandle(InetSocketTransportAddress address, TransportService transportService, DiscoveryNode discoveryNode, ConcurrentMap<TransportAddress, AtomicInteger> counters) {
+        public NetworkHandle(InetSocketTransportAddress address, TransportService transportService, DiscoveryNode discoveryNode,
+                             ConcurrentMap<TransportAddress, AtomicInteger> counters) {
             this.address = address;
             this.transportService = transportService;
             this.node = discoveryNode;
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java
index ed09efaaada..61374cc0d8f 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java
@@ -145,26 +145,22 @@ public class PublishClusterStateActionTests extends ESTestCase {
     }
 
     public MockNode createMockNode(final String name) throws Exception {
-        return createMockNode(name, Settings.EMPTY, Version.CURRENT);
+        return createMockNode(name, Settings.EMPTY);
     }
 
     public MockNode createMockNode(String name, Settings settings) throws Exception {
-        return createMockNode(name, settings, Version.CURRENT);
+        return createMockNode(name, settings, null);
     }
 
-    public MockNode createMockNode(final String name, Settings settings, Version version) throws Exception {
-        return createMockNode(name, settings, version, null);
-    }
-
-    public MockNode createMockNode(String name, Settings settings, Version version, @Nullable ClusterStateListener listener) throws Exception {
+    public MockNode createMockNode(String name, Settings settings, @Nullable ClusterStateListener listener) throws Exception {
         settings = Settings.builder()
                 .put("name", name)
                 .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "", TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
                 .put(settings)
                 .build();
 
-        MockTransportService service = buildTransportService(settings, version);
-        DiscoveryNodeService discoveryNodeService = new DiscoveryNodeService(settings, version);
+        MockTransportService service = buildTransportService(settings);
+        DiscoveryNodeService discoveryNodeService = new DiscoveryNodeService(settings);
         DiscoveryNode discoveryNode = discoveryNodeService.buildLocalNode(service.boundAddress().publishAddress());
         MockNode node = new MockNode(discoveryNode, service, listener, logger);
         node.action = buildPublishClusterStateAction(settings, service, () -> node.clusterState, node);
@@ -232,8 +228,8 @@ public class PublishClusterStateActionTests extends ESTestCase {
         terminate(threadPool);
     }
 
-    protected MockTransportService buildTransportService(Settings settings, Version version) {
-        MockTransportService transportService = MockTransportService.local(Settings.EMPTY, version, threadPool);
+    protected MockTransportService buildTransportService(Settings settings) {
+        MockTransportService transportService = MockTransportService.local(Settings.EMPTY, Version.CURRENT, threadPool);
         transportService.start();
         transportService.acceptIncomingRequests();
         return transportService;
@@ -257,8 +253,8 @@ public class PublishClusterStateActionTests extends ESTestCase {
     }
 
     public void testSimpleClusterStatePublishing() throws Exception {
-        MockNode nodeA = createMockNode("nodeA", Settings.EMPTY, Version.CURRENT).setAsMaster();
-        MockNode nodeB = createMockNode("nodeB", Settings.EMPTY, Version.CURRENT);
+        MockNode nodeA = createMockNode("nodeA", Settings.EMPTY).setAsMaster();
+        MockNode nodeB = createMockNode("nodeB", Settings.EMPTY);
 
         // Initial cluster state
         ClusterState clusterState = nodeA.clusterState;
@@ -286,7 +282,7 @@ public class PublishClusterStateActionTests extends ESTestCase {
 
         // Adding new node - this node should get full cluster state while nodeB should still be getting diffs
 
-        MockNode nodeC = createMockNode("nodeC", Settings.EMPTY, Version.CURRENT);
+        MockNode nodeC = createMockNode("nodeC", Settings.EMPTY);
 
         // cluster state update 3 - register node C
         previousClusterState = clusterState;
@@ -336,14 +332,11 @@ public class PublishClusterStateActionTests extends ESTestCase {
     }
 
     public void testUnexpectedDiffPublishing() throws Exception {
-        MockNode nodeA = createMockNode("nodeA", Settings.EMPTY, Version.CURRENT, new ClusterStateListener() {
-            @Override
-            public void clusterChanged(ClusterChangedEvent event) {
-                fail("Shouldn't send cluster state to myself");
-            }
+        MockNode nodeA = createMockNode("nodeA", Settings.EMPTY, event -> {
+            fail("Shouldn't send cluster state to myself");
         }).setAsMaster();
 
-        MockNode nodeB = createMockNode("nodeB", Settings.EMPTY, Version.CURRENT);
+        MockNode nodeB = createMockNode("nodeB", Settings.EMPTY);
 
         // Initial cluster state with both states - the second node still shouldn't get diff even though it's present in the previous cluster state
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder(nodeA.nodes()).put(nodeB.discoveryNode).build();
@@ -362,14 +355,14 @@ public class PublishClusterStateActionTests extends ESTestCase {
     public void testDisablingDiffPublishing() throws Exception {
         Settings noDiffPublishingSettings = Settings.builder().put(DiscoverySettings.PUBLISH_DIFF_ENABLE_SETTING.getKey(), false).build();
 
-        MockNode nodeA = createMockNode("nodeA", noDiffPublishingSettings, Version.CURRENT, new ClusterStateListener() {
+        MockNode nodeA = createMockNode("nodeA", noDiffPublishingSettings, new ClusterStateListener() {
             @Override
             public void clusterChanged(ClusterChangedEvent event) {
                 fail("Shouldn't send cluster state to myself");
             }
         });
 
-        MockNode nodeB = createMockNode("nodeB", noDiffPublishingSettings, Version.CURRENT, new ClusterStateListener() {
+        MockNode nodeB = createMockNode("nodeB", noDiffPublishingSettings, new ClusterStateListener() {
             @Override
             public void clusterChanged(ClusterChangedEvent event) {
                 assertFalse(event.state().wasReadFromDiff());
@@ -400,7 +393,7 @@ public class PublishClusterStateActionTests extends ESTestCase {
         int numberOfNodes = randomIntBetween(2, 10);
         int numberOfIterations = scaledRandomIntBetween(5, 50);
         Settings settings = Settings.builder().put(DiscoverySettings.PUBLISH_DIFF_ENABLE_SETTING.getKey(), randomBoolean()).build();
-        MockNode master = createMockNode("node0", settings, Version.CURRENT, new ClusterStateListener() {
+        MockNode master = createMockNode("node0", settings, new ClusterStateListener() {
             @Override
             public void clusterChanged(ClusterChangedEvent event) {
                 assertProperMetaDataForVersion(event.state().metaData(), event.state().version());
@@ -409,7 +402,7 @@ public class PublishClusterStateActionTests extends ESTestCase {
         DiscoveryNodes.Builder discoveryNodesBuilder = DiscoveryNodes.builder(master.nodes());
         for (int i = 1; i < numberOfNodes; i++) {
             final String name = "node" + i;
-            final MockNode node = createMockNode(name, settings, Version.CURRENT, new ClusterStateListener() {
+            final MockNode node = createMockNode(name, settings, new ClusterStateListener() {
                 @Override
                 public void clusterChanged(ClusterChangedEvent event) {
                     assertProperMetaDataForVersion(event.state().metaData(), event.state().version());
@@ -444,14 +437,14 @@ public class PublishClusterStateActionTests extends ESTestCase {
     }
 
     public void testSerializationFailureDuringDiffPublishing() throws Exception {
-        MockNode nodeA = createMockNode("nodeA", Settings.EMPTY, Version.CURRENT, new ClusterStateListener() {
+        MockNode nodeA = createMockNode("nodeA", Settings.EMPTY, new ClusterStateListener() {
             @Override
             public void clusterChanged(ClusterChangedEvent event) {
                 fail("Shouldn't send cluster state to myself");
             }
         }).setAsMaster();
 
-        MockNode nodeB = createMockNode("nodeB", Settings.EMPTY, Version.CURRENT);
+        MockNode nodeB = createMockNode("nodeB", Settings.EMPTY);
 
         // Initial cluster state with both states - the second node still shouldn't get diff even though it's present in the previous cluster state
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder(nodeA.nodes()).put(nodeB.discoveryNode).build();
diff --git a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
index 32f9bbbd84c..2ba7da84c14 100644
--- a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
+++ b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
@@ -74,7 +74,7 @@ public class HttpServerTests extends ESTestCase {
 
         ClusterService clusterService = new ClusterService(Settings.EMPTY,
             new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null);
-        NodeService nodeService = new NodeService(Settings.EMPTY, null, null, null, null, null, null, null, null, null,
+        NodeService nodeService = new NodeService(Settings.EMPTY, null, null, null, null, null, null, null, null,
             clusterService, null);
         httpServer = new HttpServer(settings, httpServerTransport, restController, nodeService, circuitBreakerService);
         httpServer.start();
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
index cc8b05819c6..f9fb5e77b70 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
@@ -76,7 +76,7 @@ public class DynamicMappingDisabledTests extends ESSingleNodeTestCase {
                 .build();
         clusterService = createClusterService(THREAD_POOL);
         transport =
-                new LocalTransport(settings, THREAD_POOL, Version.CURRENT, new NamedWriteableRegistry(), new NoneCircuitBreakerService());
+                new LocalTransport(settings, THREAD_POOL, new NamedWriteableRegistry(), new NoneCircuitBreakerService());
         transportService = new TransportService(clusterService.getSettings(), transport, THREAD_POOL);
         indicesService = getInstanceFromNode(IndicesService.class);
         shardStateAction = new ShardStateAction(settings, clusterService, transportService, null, null, THREAD_POOL);
diff --git a/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
index 86d19ae1ea4..fc5b68f87d1 100644
--- a/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
+++ b/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
@@ -158,7 +158,7 @@ public class ClusterStateChanges {
         MetaDataUpdateSettingsService metaDataUpdateSettingsService = new MetaDataUpdateSettingsService(settings, clusterService,
             allocationService, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, new IndexNameExpressionResolver(settings));
         MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(settings, clusterService, indicesService,
-            allocationService, Version.CURRENT, new AliasValidator(settings), Collections.emptySet(), environment,
+            allocationService, new AliasValidator(settings), Collections.emptySet(), environment,
             nodeServicesProvider, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
 
         transportCloseIndexAction = new TransportCloseIndexAction(settings, transportService, clusterService, threadPool,
diff --git a/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java b/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java
index 95ec6dad9bd..f436b719260 100644
--- a/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/NettySizeHeaderFrameDecoderTests.java
@@ -66,7 +66,7 @@ public class NettySizeHeaderFrameDecoderTests extends ESTestCase {
         threadPool = new ThreadPool(settings);
         NetworkService networkService = new NetworkService(settings);
         BigArrays bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
-        nettyTransport = new NettyTransport(settings, threadPool, networkService, bigArrays, Version.CURRENT, new NamedWriteableRegistry(),
+        nettyTransport = new NettyTransport(settings, threadPool, networkService, bigArrays, new NamedWriteableRegistry(),
             new NoneCircuitBreakerService());
         nettyTransport.start();
         TransportService transportService = new TransportService(settings, nettyTransport, threadPool);
diff --git a/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java b/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java
index 2213d7eb4cf..7bbd6e0fea7 100644
--- a/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/NettyTransportServiceHandshakeTests.java
@@ -64,7 +64,6 @@ public class NettyTransportServiceHandshakeTests extends ESTestCase {
                         threadPool,
                         new NetworkService(settings),
                         BigArrays.NON_RECYCLING_INSTANCE,
-                        Version.CURRENT,
                         new NamedWriteableRegistry(),
                         new NoneCircuitBreakerService());
         TransportService transportService = new MockTransportService(settings, transport, threadPool);
diff --git a/core/src/test/java/org/elasticsearch/transport/TransportModuleTests.java b/core/src/test/java/org/elasticsearch/transport/TransportModuleTests.java
index 20413888281..e5c734cbfb6 100644
--- a/core/src/test/java/org/elasticsearch/transport/TransportModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/TransportModuleTests.java
@@ -36,9 +36,9 @@ public class TransportModuleTests extends ModuleTestCase {
 
     static class FakeTransport extends AssertingLocalTransport {
         @Inject
-        public FakeTransport(Settings settings, CircuitBreakerService circuitBreakerService, ThreadPool threadPool, Version version,
+        public FakeTransport(Settings settings, CircuitBreakerService circuitBreakerService, ThreadPool threadPool,
                              NamedWriteableRegistry namedWriteableRegistry) {
-            super(settings, circuitBreakerService, threadPool, version, namedWriteableRegistry);
+            super(settings, circuitBreakerService, threadPool, namedWriteableRegistry);
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java
index a98a038f6fd..df7dcb0714b 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java
@@ -65,14 +65,14 @@ public class NettyScheduledPingTests extends ESTestCase {
 
         NamedWriteableRegistry registryA = new NamedWriteableRegistry();
         final NettyTransport nettyA = new NettyTransport(settings, threadPool, new NetworkService(settings),
-            BigArrays.NON_RECYCLING_INSTANCE, Version.CURRENT, registryA, circuitBreakerService);
+            BigArrays.NON_RECYCLING_INSTANCE, registryA, circuitBreakerService);
         MockTransportService serviceA = new MockTransportService(settings, nettyA, threadPool);
         serviceA.start();
         serviceA.acceptIncomingRequests();
 
         NamedWriteableRegistry registryB = new NamedWriteableRegistry();
         final NettyTransport nettyB = new NettyTransport(settings, threadPool, new NetworkService(settings),
-            BigArrays.NON_RECYCLING_INSTANCE, Version.CURRENT, registryB, circuitBreakerService);
+            BigArrays.NON_RECYCLING_INSTANCE, registryB, circuitBreakerService);
         MockTransportService serviceB = new MockTransportService(settings, nettyB, threadPool);
 
         serviceB.start();
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java
index f70c7b27492..2f89435c6df 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java
@@ -93,9 +93,9 @@ public class NettyTransportIT extends ESIntegTestCase {
 
         @Inject
         public ExceptionThrowingNettyTransport(Settings settings, ThreadPool threadPool, NetworkService networkService, BigArrays bigArrays,
-                                               Version version, NamedWriteableRegistry namedWriteableRegistry,
+                                               NamedWriteableRegistry namedWriteableRegistry,
                                                CircuitBreakerService circuitBreakerService) {
-            super(settings, threadPool, networkService, bigArrays, version, namedWriteableRegistry, circuitBreakerService);
+            super(settings, threadPool, networkService, bigArrays, namedWriteableRegistry, circuitBreakerService);
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java
index 0ed0cf3ec97..6fdc214d18d 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java
@@ -136,7 +136,7 @@ public class NettyTransportMultiPortTests extends ESTestCase {
     private NettyTransport startNettyTransport(Settings settings, ThreadPool threadPool) {
         BigArrays bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
 
-        NettyTransport nettyTransport = new NettyTransport(settings, threadPool, new NetworkService(settings), bigArrays, Version.CURRENT,
+        NettyTransport nettyTransport = new NettyTransport(settings, threadPool, new NetworkService(settings), bigArrays,
             new NamedWriteableRegistry(), new NoneCircuitBreakerService());
         nettyTransport.start();
 
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/SimpleNettyTransportTests.java b/core/src/test/java/org/elasticsearch/transport/netty/SimpleNettyTransportTests.java
index f3cfcc3bebe..96275d099db 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/SimpleNettyTransportTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/SimpleNettyTransportTests.java
@@ -42,7 +42,7 @@ public class SimpleNettyTransportTests extends AbstractSimpleTransportTestCase {
     @Override
     protected MockTransportService build(Settings settings, Version version) {
         settings = Settings.builder().put(settings).put(TransportSettings.PORT.getKey(), "0").build();
-        MockTransportService transportService = MockTransportService.nettyFromThreadPool(settings, version, threadPool);
+        MockTransportService transportService = MockTransportService.nettyFromThreadPool(settings, threadPool, version);
         transportService.start();
         return transportService;
     }
diff --git a/plugins/discovery-azure/src/main/java/org/elasticsearch/discovery/azure/AzureUnicastHostsProvider.java b/plugins/discovery-azure/src/main/java/org/elasticsearch/discovery/azure/AzureUnicastHostsProvider.java
index 869fda5f8e2..cb6c8238bf5 100644
--- a/plugins/discovery-azure/src/main/java/org/elasticsearch/discovery/azure/AzureUnicastHostsProvider.java
+++ b/plugins/discovery-azure/src/main/java/org/elasticsearch/discovery/azure/AzureUnicastHostsProvider.java
@@ -101,7 +101,6 @@ public class AzureUnicastHostsProvider extends AbstractComponent implements Unic
     private final AzureComputeService azureComputeService;
     private TransportService transportService;
     private NetworkService networkService;
-    private final Version version;
 
     private final TimeValue refreshInterval;
     private long lastRefresh;
@@ -114,13 +113,11 @@ public class AzureUnicastHostsProvider extends AbstractComponent implements Unic
     @Inject
     public AzureUnicastHostsProvider(Settings settings, AzureComputeService azureComputeService,
                                    TransportService transportService,
-                                   NetworkService networkService,
-                                   Version version) {
+                                   NetworkService networkService) {
         super(settings);
         this.azureComputeService = azureComputeService;
         this.transportService = transportService;
         this.networkService = networkService;
-        this.version = version;
 
         this.refreshInterval = Discovery.REFRESH_SETTING.get(settings);
 
@@ -253,7 +250,7 @@ public class AzureUnicastHostsProvider extends AbstractComponent implements Unic
                     for (TransportAddress address : addresses) {
                         logger.trace("adding {}, transport_address {}", networkAddress, address);
                         cachedDiscoNodes.add(new DiscoveryNode("#cloud-" + instance.getInstanceName(), address, emptyMap(),
-                                emptySet(), version.minimumCompatibilityVersion()));
+                                emptySet(), Version.CURRENT.minimumCompatibilityVersion()));
                     }
                 } catch (Exception e) {
                     logger.warn("can not convert [{}] to transport address. skipping. [{}]", networkAddress, e.getMessage());
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java
index 58be8645131..1f3043fe6dd 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/AwsEc2UnicastHostsProvider.java
@@ -59,8 +59,6 @@ public class AwsEc2UnicastHostsProvider extends AbstractComponent implements Uni
 
     private final AmazonEC2 client;
 
-    private final Version version;
-
     private final boolean bindAnyGroup;
 
     private final Set<String> groups;
@@ -74,11 +72,10 @@ public class AwsEc2UnicastHostsProvider extends AbstractComponent implements Uni
     private final DiscoNodesCache discoNodes;
 
     @Inject
-    public AwsEc2UnicastHostsProvider(Settings settings, TransportService transportService, AwsEc2Service awsEc2Service, Version version) {
+    public AwsEc2UnicastHostsProvider(Settings settings, TransportService transportService, AwsEc2Service awsEc2Service) {
         super(settings);
         this.transportService = transportService;
         this.client = awsEc2Service.client();
-        this.version = version;
 
         this.hostType = DISCOVERY_EC2.HOST_TYPE_SETTING.get(settings);
         this.discoNodes = new DiscoNodesCache(DISCOVERY_EC2.NODE_CACHE_TIME_SETTING.get(settings));
@@ -175,7 +172,7 @@ public class AwsEc2UnicastHostsProvider extends AbstractComponent implements Uni
                         for (int i = 0; i < addresses.length; i++) {
                             logger.trace("adding {}, address {}, transport_address {}", instance.getInstanceId(), address, addresses[i]);
                             discoNodes.add(new DiscoveryNode("#cloud-" + instance.getInstanceId() + "-" + i, addresses[i],
-                                    emptyMap(), emptySet(), version.minimumCompatibilityVersion()));
+                                    emptyMap(), emptySet(), Version.CURRENT.minimumCompatibilityVersion()));
                         }
                     } catch (Exception e) {
                         logger.warn("failed ot add {}, address {}", e, instance.getInstanceId(), address);
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java
index 29332f61678..c9dd2263245 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java
@@ -68,7 +68,6 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas
     private TransportService transportService;
     private NetworkService networkService;
 
-    private final Version version;
     private final String project;
     private final List<String> zones;
     private final List<String> tags;
@@ -80,13 +79,11 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas
     @Inject
     public GceUnicastHostsProvider(Settings settings, GceComputeService gceComputeService,
             TransportService transportService,
-            NetworkService networkService,
-            Version version) {
+            NetworkService networkService) {
         super(settings);
         this.gceComputeService = gceComputeService;
         this.transportService = transportService;
         this.networkService = networkService;
-        this.version = version;
 
         this.refreshInterval = GceComputeService.REFRESH_SETTING.get(settings);
         this.project = GceComputeService.PROJECT_SETTING.get(settings);
@@ -244,7 +241,7 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas
                             logger.trace("adding {}, type {}, address {}, transport_address {}, status {}", name, type,
                                     ip_private, transportAddress, status);
                             cachedDiscoNodes.add(new DiscoveryNode("#cloud-" + name + "-" + 0, transportAddress,
-                                    emptyMap(), emptySet(), version.minimumCompatibilityVersion()));
+                                    emptyMap(), emptySet(), Version.CURRENT.minimumCompatibilityVersion()));
                         }
                     }
                 } catch (Exception e) {
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositoryF.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositoryF.java
index 4150fe54979..bfa06219127 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositoryF.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositoryF.java
@@ -21,7 +21,6 @@ package org.elasticsearch.repositories.azure;
 
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.node.MockNode;
 import org.elasticsearch.node.Node;
@@ -111,7 +110,7 @@ public class AzureRepositoryF {
         // settings.put("cloud.azure.storage.my_account2.key", "account_key_secondary");
 
         final CountDownLatch latch = new CountDownLatch(1);
-        final Node node = new MockNode(settings.build(), Version.CURRENT, Collections.singletonList(AzureRepositoryPlugin.class));
+        final Node node = new MockNode(settings.build(), Collections.singletonList(AzureRepositoryPlugin.class));
         Runtime.getRuntime().addShutdownHook(new Thread() {
             @Override
             public void run() {
diff --git a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java
index 57dcc08f4fe..749f1f3279e 100644
--- a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java
+++ b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java
@@ -35,13 +35,10 @@ import java.util.Collection;
  */
 public class MockNode extends Node {
 
-    // these are kept here so a copy of this MockNode can be created, since Node does not store them
-    private Version version;
     private Collection<Class<? extends Plugin>> plugins;
 
-    public MockNode(Settings settings, Version version, Collection<Class<? extends Plugin>> classpathPlugins) {
-        super(InternalSettingsPreparer.prepareEnvironment(settings, null), version, classpathPlugins);
-        this.version = version;
+    public MockNode(Settings settings, Collection<Class<? extends Plugin>> classpathPlugins) {
+        super(InternalSettingsPreparer.prepareEnvironment(settings, null), classpathPlugins);
         this.plugins = classpathPlugins;
     }
 
@@ -49,7 +46,4 @@ public class MockNode extends Node {
         return plugins;
     }
 
-    public Version getVersion() {
-        return version;
-    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
index 4cf058a6a5d..8f06e75b038 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.test;
 
 import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
@@ -147,10 +146,6 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
         return false;
     }
 
-    /** The version of elasticsearch the node should act like. */
-    protected Version getVersion() {
-        return Version.CURRENT;
-    }
 
     /** The plugin classes that should be added to the node. */
     protected Collection<Class<? extends Plugin>> getPlugins() {
@@ -187,7 +182,7 @@ public abstract class ESSingleNodeTestCase extends ESTestCase {
             .put(Node.NODE_DATA_SETTING.getKey(), true)
             .put(nodeSettings()) // allow test cases to provide their own settings or override these
             .build();
-        Node build = new MockNode(settings, getVersion(), getPlugins());
+        Node build = new MockNode(settings, getPlugins());
         build.start();
         assertThat(DiscoveryNode.isLocalNode(build.settings()), is(true));
         return build;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index bb33ad17dde..950653cba5e 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -614,7 +614,7 @@ public final class InternalTestCluster extends TestCluster {
             .put("node.name", name)
             .put(DiscoveryNodeService.NODE_ID_SEED_SETTING.getKey(), seed)
             .build();
-        MockNode node = new MockNode(finalSettings, version, plugins);
+        MockNode node = new MockNode(finalSettings, plugins);
         return new NodeAndClient(name, node);
     }
 
@@ -888,8 +888,7 @@ public final class InternalTestCluster extends TestCluster {
             final long newIdSeed = DiscoveryNodeService.NODE_ID_SEED_SETTING.get(node.settings()) + 1; // use a new seed to make sure we have new node id
             Settings finalSettings = Settings.builder().put(node.settings()).put(newSettings).put(DiscoveryNodeService.NODE_ID_SEED_SETTING.getKey(), newIdSeed).build();
             Collection<Class<? extends Plugin>> plugins = node.getPlugins();
-            Version version = node.getVersion();
-            node = new MockNode(finalSettings, version, plugins);
+            node = new MockNode(finalSettings, plugins);
             node.start();
         }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/AssertingLocalTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/AssertingLocalTransport.java
index 0dd8edb75f8..0142acf8c17 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/AssertingLocalTransport.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/AssertingLocalTransport.java
@@ -27,7 +27,6 @@ import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -75,8 +74,8 @@ public class AssertingLocalTransport extends LocalTransport {
 
     @Inject
     public AssertingLocalTransport(Settings settings, CircuitBreakerService circuitBreakerService, ThreadPool threadPool,
-                                   Version version, NamedWriteableRegistry namedWriteableRegistry) {
-        super(settings, threadPool, version, namedWriteableRegistry, circuitBreakerService);
+                                   NamedWriteableRegistry namedWriteableRegistry) {
+        super(settings, threadPool, namedWriteableRegistry, circuitBreakerService);
         final long seed = ESIntegTestCase.INDEX_TEST_SEED_SETTING.get(settings);
         random = new Random(seed);
         minVersion = ASSERTING_TRANSPORT_MIN_VERSION_KEY.get(settings);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
index 110850e9a0f..7688e2842e6 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
@@ -96,17 +96,26 @@ public class MockTransportService extends TransportService {
 
     public static MockTransportService local(Settings settings, Version version, ThreadPool threadPool) {
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
-        Transport transport = new LocalTransport(settings, threadPool, version, namedWriteableRegistry, new NoneCircuitBreakerService());
+        Transport transport = new LocalTransport(settings, threadPool, namedWriteableRegistry, new NoneCircuitBreakerService()) {
+            @Override
+            protected Version getVersion() {
+                return version;
+            }
+        };
         return new MockTransportService(settings, transport, threadPool);
     }
 
     public static MockTransportService nettyFromThreadPool(
             Settings settings,
-            Version version,
-            ThreadPool threadPool) {
+            ThreadPool threadPool, final Version version) {
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
         Transport transport = new NettyTransport(settings, threadPool, new NetworkService(settings), BigArrays.NON_RECYCLING_INSTANCE,
-                version, namedWriteableRegistry, new NoneCircuitBreakerService());
+                namedWriteableRegistry, new NoneCircuitBreakerService()) {
+            @Override
+            protected Version getCurrentVersion() {
+                return version;
+            }
+        };
         return new MockTransportService(Settings.EMPTY, transport, threadPool);
     }
 

From b53d7356023b5b62ea47536e7df6f36d321dfe8c Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 20 Jun 2016 05:05:34 -0400
Subject: [PATCH 0954/1311] Function/Lambda parameters do not need to be
 lenient...

---
 .../src/main/java/org/elasticsearch/painless/Locals.java | 4 ++--
 .../java/org/elasticsearch/painless/LambdaTests.java     | 9 +++++++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
index 0893a51bc47..7ffa0a169b7 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
@@ -74,7 +74,7 @@ public final class Locals {
         for (int i = 0; i < parameters.size(); i++) {
             Parameter parameter = parameters.get(i);
             boolean isCapture = i < captureCount;
-            locals.defineVariable(parameter.location, parameter.type, parameter.name, isCapture);
+            locals.addVariable(parameter.location, parameter.type, parameter.name, isCapture);
         }
         // Loop counter to catch infinite loops.  Internal use only.
         if (maxLoopCounter > 0) {
@@ -87,7 +87,7 @@ public final class Locals {
     public static Locals newFunctionScope(Locals programScope, Type returnType, List<Parameter> parameters, int maxLoopCounter) {
         Locals locals = new Locals(programScope, returnType);
         for (Parameter parameter : parameters) {
-            locals.defineVariable(parameter.location, parameter.type, parameter.name, false);
+            locals.addVariable(parameter.location, parameter.type, parameter.name, false);
         }
         // Loop counter to catch infinite loops.  Internal use only.
         if (maxLoopCounter > 0) {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
index 22abb8f5255..3b008d5d6b6 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
@@ -125,6 +125,15 @@ public class LambdaTests extends ScriptTestCase {
                            + "return l.stream().mapToInt(x -> { x += 1; return x }).sum();"));
     }
     
+    /** Lambda parameters shouldn't be able to mask a variable already in scope */
+    public void testNoParamMasking() {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("int x = 0; List l = new ArrayList(); l.add(1); l.add(1); "
+                    + "return l.stream().mapToInt(x -> { x += 1; return x }).sum();");
+        });
+        assertTrue(expected.getMessage().contains("already defined"));
+    }
+
     public void testCaptureDef() {
         assertEquals(5, exec("int x = 5; def y = Optional.empty(); y.orElseGet(() -> x);"));
     }

From 7fea5bd8e7c45e5a5aaa05881d0fc2e8ba5b978f Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Mon, 20 Jun 2016 11:28:14 +0200
Subject: [PATCH 0955/1311] Remove obsolete Modules that can simply be inlined
 in node creation

---
 .../elasticsearch/env/EnvironmentModule.java  | 43 -----------------
 .../env/NodeEnvironmentModule.java            | 48 -------------------
 .../elasticsearch/gateway/GatewayModule.java  |  5 --
 .../java/org/elasticsearch/node/Node.java     | 24 ++++++----
 .../org/elasticsearch/tribe/TribeModule.java  | 32 -------------
 .../org/elasticsearch/tribe/TribeService.java |  2 -
 .../watcher/ResourceWatcherModule.java        | 31 ------------
 .../script/NativeScriptTests.java             |  8 ++--
 .../aggregations/AggregatorParsingTests.java  |  7 ++-
 .../aggregations/BaseAggregationTestCase.java |  6 ++-
 .../BasePipelineAggregationTestCase.java      |  6 ++-
 .../builder/SearchSourceBuilderTests.java     |  6 ++-
 .../messy/tests/TemplateQueryParserTests.java |  7 +--
 .../discovery/ec2/Ec2DiscoveryTests.java      | 13 ++---
 .../discovery/gce/GceDiscoveryTests.java      |  6 +--
 .../test/AbstractQueryTestCase.java           | 16 ++-----
 16 files changed, 51 insertions(+), 209 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/env/EnvironmentModule.java
 delete mode 100644 core/src/main/java/org/elasticsearch/env/NodeEnvironmentModule.java
 delete mode 100644 core/src/main/java/org/elasticsearch/tribe/TribeModule.java
 delete mode 100644 core/src/main/java/org/elasticsearch/watcher/ResourceWatcherModule.java

diff --git a/core/src/main/java/org/elasticsearch/env/EnvironmentModule.java b/core/src/main/java/org/elasticsearch/env/EnvironmentModule.java
deleted file mode 100644
index 6a893a73437..00000000000
--- a/core/src/main/java/org/elasticsearch/env/EnvironmentModule.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.env;
-
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.threadpool.ThreadPool;
-
-/**
- *
- */
-public class EnvironmentModule extends AbstractModule {
-
-    private final Environment environment;
-    private final ThreadPool threadPool;
-
-    public EnvironmentModule(Environment environment, ThreadPool threadPool) {
-        this.threadPool = threadPool;
-        this.environment = environment;
-    }
-
-    @Override
-    protected void configure() {
-        bind(ThreadPool.class).toInstance(threadPool);
-        bind(Environment.class).toInstance(environment);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/env/NodeEnvironmentModule.java b/core/src/main/java/org/elasticsearch/env/NodeEnvironmentModule.java
deleted file mode 100644
index 162c3108120..00000000000
--- a/core/src/main/java/org/elasticsearch/env/NodeEnvironmentModule.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.env;
-
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.inject.AbstractModule;
-
-/**
- *
- */
-public class NodeEnvironmentModule extends AbstractModule {
-
-    private final NodeEnvironment nodeEnvironment;
-
-    public NodeEnvironmentModule() {
-        this(null);
-    }
-
-    public NodeEnvironmentModule(@Nullable NodeEnvironment nodeEnvironment) {
-        this.nodeEnvironment = nodeEnvironment;
-    }
-
-    @Override
-    protected void configure() {
-        if (nodeEnvironment != null) {
-            bind(NodeEnvironment.class).toInstance(nodeEnvironment);
-        } else {
-            bind(NodeEnvironment.class).asEagerSingleton();
-        }
-    }
-}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayModule.java b/core/src/main/java/org/elasticsearch/gateway/GatewayModule.java
index 651123882a5..f736c070272 100644
--- a/core/src/main/java/org/elasticsearch/gateway/GatewayModule.java
+++ b/core/src/main/java/org/elasticsearch/gateway/GatewayModule.java
@@ -27,11 +27,6 @@ import org.elasticsearch.common.settings.Settings;
  */
 public class GatewayModule extends AbstractModule {
 
-    private final Settings settings;
-
-    public GatewayModule(Settings settings) {
-        this.settings = settings;
-    }
 
     @Override
     protected void configure() {
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index 70e0c356ba2..2069320104a 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -67,9 +67,7 @@ import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.EnvironmentModule;
 import org.elasticsearch.env.NodeEnvironment;
-import org.elasticsearch.env.NodeEnvironmentModule;
 import org.elasticsearch.gateway.GatewayAllocator;
 import org.elasticsearch.gateway.GatewayModule;
 import org.elasticsearch.gateway.GatewayService;
@@ -104,9 +102,7 @@ import org.elasticsearch.tasks.TaskPersistenceService;
 import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.tribe.TribeModule;
 import org.elasticsearch.tribe.TribeService;
-import org.elasticsearch.watcher.ResourceWatcherModule;
 import org.elasticsearch.watcher.ResourceWatcherService;
 
 import java.io.BufferedWriter;
@@ -242,9 +238,13 @@ public class Node implements Closeable {
             } catch (IOException ex) {
                 throw new IllegalStateException("Failed to created node environment", ex);
             }
+            final ResourceWatcherService resourceWatcherService = new ResourceWatcherService(settings, threadPool);
+            resourcesToClose.add(resourceWatcherService);
             final NetworkService networkService = new NetworkService(settings);
             final ClusterService clusterService = new ClusterService(settings, settingsModule.getClusterSettings(), threadPool);
             resourcesToClose.add(clusterService);
+            final TribeService tribeService = new TribeService(settings, clusterService);
+            resourcesToClose.add(tribeService);
             NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
             ModulesBuilder modules = new ModulesBuilder();
             // plugin modules must be added here, before others or we can get crazy injection errors...
@@ -253,28 +253,32 @@ public class Node implements Closeable {
             }
             final MonitorService monitorService = new MonitorService(settings, nodeEnvironment, threadPool);
             modules.add(new PluginsModule(pluginsService));
-            modules.add(new EnvironmentModule(environment, threadPool));
             modules.add(new NodeModule(this, monitorService));
             modules.add(new NetworkModule(networkService, settings, false, namedWriteableRegistry));
             modules.add(scriptModule);
-            modules.add(new NodeEnvironmentModule(nodeEnvironment));
             modules.add(new DiscoveryModule(this.settings));
             modules.add(new ClusterModule(this.settings, clusterService));
             modules.add(new IndicesModule(namedWriteableRegistry));
             modules.add(new SearchModule(settings, namedWriteableRegistry));
             modules.add(new ActionModule(DiscoveryNode.isIngestNode(settings), false));
-            modules.add(new GatewayModule(settings));
+            modules.add(new GatewayModule());
             modules.add(new NodeClientModule());
-            modules.add(new ResourceWatcherModule());
             modules.add(new RepositoriesModule());
-            modules.add(new TribeModule());
             modules.add(new AnalysisModule(environment));
             pluginsService.processModules(modules);
             CircuitBreakerService circuitBreakerService = createCircuitBreakerService(settingsModule.getSettings(),
                 settingsModule.getClusterSettings());
             resourcesToClose.add(circuitBreakerService);
             modules.add(settingsModule);
-            modules.add(b -> b.bind(CircuitBreakerService.class).toInstance(circuitBreakerService));
+            modules.add(b -> {
+                    b.bind(Environment.class).toInstance(new Environment(settings));
+                    b.bind(ThreadPool.class).toInstance(threadPool);
+                    b.bind(NodeEnvironment.class).toInstance(nodeEnvironment);
+                    b.bind(TribeService.class).toInstance(tribeService);
+                    b.bind(ResourceWatcherService.class).toInstance(resourceWatcherService);
+                    b.bind(CircuitBreakerService.class).toInstance(circuitBreakerService);
+                }
+            );
             injector = modules.createInjector();
             client = injector.getInstance(Client.class);
             success = true;
diff --git a/core/src/main/java/org/elasticsearch/tribe/TribeModule.java b/core/src/main/java/org/elasticsearch/tribe/TribeModule.java
deleted file mode 100644
index fb642d1b034..00000000000
--- a/core/src/main/java/org/elasticsearch/tribe/TribeModule.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.tribe;
-
-import org.elasticsearch.common.inject.AbstractModule;
-
-/**
- */
-public class TribeModule extends AbstractModule {
-
-    @Override
-    protected void configure() {
-        bind(TribeService.class).asEagerSingleton();
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/tribe/TribeService.java b/core/src/main/java/org/elasticsearch/tribe/TribeService.java
index 50be6e39b07..259a9d443a3 100644
--- a/core/src/main/java/org/elasticsearch/tribe/TribeService.java
+++ b/core/src/main/java/org/elasticsearch/tribe/TribeService.java
@@ -40,7 +40,6 @@ import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.regex.Regex;
@@ -175,7 +174,6 @@ public class TribeService extends AbstractLifecycleComponent<TribeService> {
 
     private final List<Node> nodes = new CopyOnWriteArrayList<>();
 
-    @Inject
     public TribeService(Settings settings, ClusterService clusterService) {
         super(settings);
         this.clusterService = clusterService;
diff --git a/core/src/main/java/org/elasticsearch/watcher/ResourceWatcherModule.java b/core/src/main/java/org/elasticsearch/watcher/ResourceWatcherModule.java
deleted file mode 100644
index a3ae84a954f..00000000000
--- a/core/src/main/java/org/elasticsearch/watcher/ResourceWatcherModule.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.watcher;
-
-import org.elasticsearch.common.inject.AbstractModule;
-
-/**
- *
- */
-public class ResourceWatcherModule extends AbstractModule {
-    @Override
-    protected void configure() {
-        bind(ResourceWatcherService.class).asEagerSingleton();
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
index 65c0a4fddef..d88813203b2 100644
--- a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
@@ -28,7 +28,6 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.EnvironmentModule;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -41,9 +40,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
-import static java.util.Collections.singleton;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
@@ -60,7 +57,10 @@ public class NativeScriptTests extends ESTestCase {
         SettingsModule settingsModule = new SettingsModule(settings, scriptSettings, Collections.emptyList());
         final ThreadPool threadPool = new ThreadPool(settings);
         Injector injector = new ModulesBuilder().add(
-                new EnvironmentModule(new Environment(settings), threadPool),
+                (b) -> {
+                    b.bind(Environment.class).toInstance(new Environment(settings));
+                    b.bind(ThreadPool.class).toInstance(threadPool);
+                },
                 new SettingsModule(settings),
                 scriptModule).createInjector();
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
index 5dd321bdf07..a78c405929a 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
@@ -38,7 +38,6 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.EnvironmentModule;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.index.query.QueryParseContext;
@@ -109,7 +108,11 @@ public class AggregatorParsingTests extends ESTestCase {
         List<Setting<?>> scriptSettings = scriptModule.getSettings();
         scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
         SettingsModule settingsModule = new SettingsModule(settings, scriptSettings, Collections.emptyList());
-        injector = new ModulesBuilder().add(new EnvironmentModule(new Environment(settings), threadPool), settingsModule
+        injector = new ModulesBuilder().add(
+            (b) -> {
+                b.bind(Environment.class).toInstance(new Environment(settings));
+                b.bind(ThreadPool.class).toInstance(threadPool);
+            }, settingsModule
             , scriptModule, new IndicesModule(namedWriteableRegistry) {
 
                     @Override
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
index c7d1084067a..b1065c90efa 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
@@ -42,7 +42,6 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.EnvironmentModule;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.index.query.QueryParseContext;
@@ -124,7 +123,10 @@ public abstract class BaseAggregationTestCase<AB extends AbstractAggregationBuil
         scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
         SettingsModule settingsModule = new SettingsModule(settings, scriptSettings, Collections.emptyList());
         injector = new ModulesBuilder().add(
-                new EnvironmentModule(new Environment(settings), threadPool),
+                (b) -> {
+                    b.bind(Environment.class).toInstance(new Environment(settings));
+                    b.bind(ThreadPool.class).toInstance(threadPool);
+                },
                 settingsModule,
                 scriptModule,
                 new IndicesModule(namedWriteableRegistry) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
index a907e75d5f7..0410c5d91d9 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
@@ -42,7 +42,6 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.EnvironmentModule;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.indices.IndicesModule;
@@ -124,7 +123,10 @@ public abstract class BasePipelineAggregationTestCase<AF extends AbstractPipelin
         scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
         SettingsModule settingsModule = new SettingsModule(settings, scriptSettings, Collections.emptyList());
         injector = new ModulesBuilder().add(
-                new EnvironmentModule(new Environment(settings),threadPool),
+                (b) -> {
+                    b.bind(Environment.class).toInstance(new Environment(settings));
+                    b.bind(ThreadPool.class).toInstance(threadPool);
+                },
                 settingsModule,
                 scriptModule,
                 new IndicesModule(namedWriteableRegistry) {
diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index 3fe1d2a0480..5822372bfa9 100644
--- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -46,7 +46,6 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.EnvironmentModule;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.QueryParseContext;
@@ -132,7 +131,10 @@ public class SearchSourceBuilderTests extends ESTestCase {
         scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
         SettingsModule settingsModule = new SettingsModule(settings, scriptSettings, Collections.emptyList());
         injector = new ModulesBuilder().add(
-                new EnvironmentModule(new Environment(settings), threadPool), settingsModule,
+                (b) -> {
+                    b.bind(Environment.class).toInstance(new Environment(settings));
+                    b.bind(ThreadPool.class).toInstance(threadPool);
+                }, settingsModule,
                 scriptModule, new IndicesModule(namedWriteableRegistry) {
                     @Override
                     protected void configure() {
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
index f0422314c74..60316818a79 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
@@ -38,7 +38,6 @@ import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.EnvironmentModule;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
@@ -59,7 +58,6 @@ import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.mapper.MapperRegistry;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.mustache.MustacheScriptEngineService;
@@ -110,7 +108,10 @@ public class TemplateQueryParserTests extends ESTestCase {
         SettingsModule settingsModule = new SettingsModule(settings, scriptSettings, Collections.emptyList());
         final ThreadPool threadPool = new ThreadPool(settings);
         injector = new ModulesBuilder().add(
-                new EnvironmentModule(new Environment(settings), threadPool),
+                (b) -> {
+                    b.bind(Environment.class).toInstance(new Environment(settings));
+                    b.bind(ThreadPool.class).toInstance(threadPool);
+                },
                 settingsModule,
                 new SearchModule(settings, new NamedWriteableRegistry()) {
                     @Override
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
index 319222caa7e..edb062d9e30 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
@@ -76,10 +76,7 @@ public class Ec2DiscoveryTests extends ESTestCase {
 
     protected List<DiscoveryNode> buildDynamicNodes(Settings nodeSettings, int nodes, List<List<Tag>> tagsList) {
         AwsEc2Service awsEc2Service = new AwsEc2ServiceMock(nodeSettings, nodes, tagsList);
-
-        AwsEc2UnicastHostsProvider provider = new AwsEc2UnicastHostsProvider(nodeSettings, transportService,
-                awsEc2Service, Version.CURRENT);
-
+        AwsEc2UnicastHostsProvider provider = new AwsEc2UnicastHostsProvider(nodeSettings, transportService, awsEc2Service);
         List<DiscoveryNode> discoveryNodes = provider.buildDynamicNodes();
         logger.debug("--> nodes found: {}", discoveryNodes);
         return discoveryNodes;
@@ -231,14 +228,14 @@ public class Ec2DiscoveryTests extends ESTestCase {
 
     abstract class DummyEc2HostProvider extends AwsEc2UnicastHostsProvider {
         public int fetchCount = 0;
-        public DummyEc2HostProvider(Settings settings, TransportService transportService, AwsEc2Service service, Version version) {
-            super(settings, transportService, service, version);
+        public DummyEc2HostProvider(Settings settings, TransportService transportService, AwsEc2Service service) {
+            super(settings, transportService, service);
         }
     }
 
     public void testGetNodeListEmptyCache() throws Exception {
         AwsEc2Service awsEc2Service = new AwsEc2ServiceMock(Settings.EMPTY, 1, null);
-        DummyEc2HostProvider provider = new DummyEc2HostProvider(Settings.EMPTY, transportService, awsEc2Service, Version.CURRENT) {
+        DummyEc2HostProvider provider = new DummyEc2HostProvider(Settings.EMPTY, transportService, awsEc2Service) {
             @Override
             protected List<DiscoveryNode> fetchDynamicNodes() {
                 fetchCount++;
@@ -255,7 +252,7 @@ public class Ec2DiscoveryTests extends ESTestCase {
         Settings.Builder builder = Settings.builder()
                 .put(DISCOVERY_EC2.NODE_CACHE_TIME_SETTING.getKey(), "500ms");
         AwsEc2Service awsEc2Service = new AwsEc2ServiceMock(Settings.EMPTY, 1, null);
-        DummyEc2HostProvider provider = new DummyEc2HostProvider(builder.build(), transportService, awsEc2Service, Version.CURRENT) {
+        DummyEc2HostProvider provider = new DummyEc2HostProvider(builder.build(), transportService, awsEc2Service) {
             @Override
             protected List<DiscoveryNode> fetchDynamicNodes() {
                 fetchCount++;
diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java
index e1041360b1f..07b2ef774b4 100644
--- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java
+++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java
@@ -78,7 +78,7 @@ public class GceDiscoveryTests extends ESTestCase {
 
     @AfterClass
     public static void stopThreadPool() {
-        if (threadPool !=null) {
+        if (threadPool != null) {
             threadPool.shutdownNow();
             threadPool = null;
         }
@@ -111,8 +111,8 @@ public class GceDiscoveryTests extends ESTestCase {
     }
 
     protected List<DiscoveryNode> buildDynamicNodes(GceComputeService gceComputeService, Settings nodeSettings) {
-        GceUnicastHostsProvider provider = new GceUnicastHostsProvider(nodeSettings, gceComputeService,
-                transportService, new NetworkService(Settings.EMPTY), Version.CURRENT);
+        GceUnicastHostsProvider provider = new GceUnicastHostsProvider(nodeSettings, gceComputeService, transportService,
+            new NetworkService(Settings.EMPTY));
 
         List<DiscoveryNode> discoveryNodes = provider.buildDynamicNodes();
         logger.info("--> nodes found: {}", discoveryNodes);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
index 62a843cee35..65c43fd38c6 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
@@ -53,7 +53,6 @@ import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.Injector;
 import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.inject.ModulesBuilder;
-import org.elasticsearch.common.inject.multibindings.Multibinder;
 import org.elasticsearch.common.inject.util.Providers;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
@@ -70,7 +69,6 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.env.EnvironmentModule;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
@@ -97,15 +95,9 @@ import org.elasticsearch.node.internal.InternalSettingsPreparer;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsModule;
 import org.elasticsearch.plugins.PluginsService;
-import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.Script.ScriptParseException;
-import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.ScriptContextRegistry;
-import org.elasticsearch.script.ScriptEngineRegistry;
-import org.elasticsearch.script.ScriptEngineService;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.script.ScriptSettings;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -120,14 +112,11 @@ import java.io.IOException;
 import java.lang.reflect.InvocationHandler;
 import java.lang.reflect.Method;
 import java.lang.reflect.Proxy;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.ExecutionException;
 
 import static org.hamcrest.Matchers.containsString;
@@ -902,7 +891,10 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             }
             modulesBuilder.add(new PluginsModule(pluginsService));
             modulesBuilder.add(
-                    new EnvironmentModule(new Environment(settings), threadPool),
+                    (b) -> {
+                        b.bind(Environment.class).toInstance(new Environment(settings));
+                        b.bind(ThreadPool.class).toInstance(threadPool);
+                    },
                     settingsModule, new IndicesModule(namedWriteableRegistry) {
                         @Override
                         public void configure() {

From 4d78be5b9effff0abc5190ff89562a8e25456eeb Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 20 Jun 2016 05:37:31 -0400
Subject: [PATCH 0956/1311] remove arity restriction (as def call incorporates
 all lambdas and all their captures)

---
 .../java/org/elasticsearch/painless/Def.java    | 17 ++++++++++++-----
 .../elasticsearch/painless/DefBootstrap.java    |  6 +++---
 .../elasticsearch/painless/node/LDefCall.java   | 17 ++++++-----------
 .../painless/DefBootstrapTests.java             |  8 ++++----
 4 files changed, 25 insertions(+), 23 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index a4033255b77..e575ed32183 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -28,6 +28,7 @@ import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
 import java.lang.invoke.MethodHandles.Lookup;
 import java.lang.invoke.MethodType;
+import java.util.BitSet;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
@@ -222,19 +223,25 @@ public final class Def {
      */
      static MethodHandle lookupMethod(Lookup lookup, MethodType callSiteType, 
              Class<?> receiverClass, String name, Object args[]) throws Throwable {
-         long recipe = (Long) args[0];
+         String recipeString = (String) args[0];
          int numArguments = callSiteType.parameterCount();
          // simple case: no lambdas
-         if (recipe == 0) {
+         if (recipeString.isEmpty()) {
              return lookupMethodInternal(receiverClass, name, numArguments - 1).handle;
          }
          
+         // convert recipe string to a bitset for convenience (the code below should be refactored...)
+         BitSet lambdaArgs = new BitSet();
+         for (int i = 0; i < recipeString.length(); i++) {
+             lambdaArgs.set(recipeString.charAt(i));
+         }
+
          // otherwise: first we have to compute the "real" arity. This is because we have extra arguments:
          // e.g. f(a, g(x), b, h(y), i()) looks like f(a, g, x, b, h, y, i). 
          int arity = callSiteType.parameterCount() - 1;
          int upTo = 1;
-         for (int i = 0; i < numArguments; i++) {
-             if ((recipe & (1L << (i - 1))) != 0) {
+         for (int i = 1; i < numArguments; i++) {
+             if (lambdaArgs.get(i - 1)) {
                  String signature = (String) args[upTo++];
                  int numCaptures = Integer.parseInt(signature.substring(signature.indexOf(',')+1));
                  arity -= numCaptures;
@@ -250,7 +257,7 @@ public final class Def {
          upTo = 1;
          for (int i = 1; i < numArguments; i++) {
              // its a functional reference, replace the argument with an impl
-             if ((recipe & (1L << (i - 1))) != 0) {
+             if (lambdaArgs.get(i - 1)) {
                  // decode signature of form 'type.call,2' 
                  String signature = (String) args[upTo++];
                  int separator = signature.indexOf('.');
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
index a60985eec67..9174911b79c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java
@@ -397,11 +397,11 @@ public final class DefBootstrap {
                 if (args.length == 0) {
                     throw new BootstrapMethodError("Invalid number of parameters for method call");
                 }
-                if (args[0] instanceof Long == false) {
+                if (args[0] instanceof String == false) {
                     throw new BootstrapMethodError("Illegal parameter for method call: " + args[0]);
                 }
-                long recipe = (Long) args[0];
-                int numLambdas = Long.bitCount(recipe);
+                String recipe = (String) args[0];
+                int numLambdas = recipe.length();
                 if (numLambdas > type.parameterCount()) {
                     throw new BootstrapMethodError("Illegal recipe for method call: too many bits");
                 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
index d216eacae19..4b04cfbf7a5 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LDefCall.java
@@ -41,7 +41,7 @@ final class LDefCall extends ALink implements IDefLink {
 
     final String name;
     final List<AExpression> arguments;
-    long recipe;
+    StringBuilder recipe;
     List<String> pointers = new ArrayList<>();
 
     LDefCall(Location location, String name, List<AExpression> arguments) {
@@ -60,14 +60,7 @@ final class LDefCall extends ALink implements IDefLink {
 
     @Override
     ALink analyze(Locals locals) {
-        if (arguments.size() > 63) {
-            // technically, the limitation is just methods with > 63 params, containing method references.
-            // this is because we are lazy and use a long as a bitset. we can always change to a "string" if need be.
-            // but NEED NOT BE. nothing with this many parameters is in the whitelist and we do not support varargs.
-            throw new UnsupportedOperationException("methods with > 63 arguments are currently not supported");
-        }
-
-        recipe = 0;
+        recipe = new StringBuilder();
         int totalCaptures = 0;
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
@@ -78,7 +71,9 @@ final class LDefCall extends ALink implements IDefLink {
             if (expression instanceof ILambda) {
                 ILambda lambda = (ILambda) expression;
                 pointers.add(lambda.getPointer());
-                recipe |= (1L << (argument + totalCaptures)); // mark argument as deferred reference
+                // encode this parameter as a deferred reference
+                char ch = (char) (argument + totalCaptures);
+                recipe.append(ch);
                 totalCaptures += lambda.getCaptureCount();
             }
 
@@ -124,7 +119,7 @@ final class LDefCall extends ALink implements IDefLink {
 
         List<Object> args = new ArrayList<>();
         args.add(DefBootstrap.METHOD_CALL);
-        args.add(recipe);
+        args.add(recipe.toString());
         args.addAll(pointers);
         writer.invokeDynamic(name, signature.toString(), DEF_BOOTSTRAP_HANDLE, args.toArray());
     }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
index 6529bed3a48..8cb065c2611 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java
@@ -36,7 +36,7 @@ public class DefBootstrapTests extends ESTestCase {
         CallSite site = DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                   "toString", 
                                                   MethodType.methodType(String.class, Object.class), 
-                                                  DefBootstrap.METHOD_CALL, 0L);
+                                                  DefBootstrap.METHOD_CALL, "");
         MethodHandle handle = site.dynamicInvoker();
         assertDepthEquals(site, 0);
 
@@ -53,7 +53,7 @@ public class DefBootstrapTests extends ESTestCase {
         CallSite site = DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                   "toString", 
                                                   MethodType.methodType(String.class, Object.class), 
-                                                  DefBootstrap.METHOD_CALL, 0L);
+                                                  DefBootstrap.METHOD_CALL, "");
         MethodHandle handle = site.dynamicInvoker();
         assertDepthEquals(site, 0);
 
@@ -75,7 +75,7 @@ public class DefBootstrapTests extends ESTestCase {
         CallSite site = DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                   "toString", 
                                                   MethodType.methodType(String.class, Object.class), 
-                                                  DefBootstrap.METHOD_CALL, 0L);
+                                                  DefBootstrap.METHOD_CALL, "");
         MethodHandle handle = site.dynamicInvoker();
         assertDepthEquals(site, 0);
 
@@ -98,7 +98,7 @@ public class DefBootstrapTests extends ESTestCase {
         DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(MethodHandles.publicLookup(), 
                                                                           "size", 
                                                                           MethodType.methodType(int.class, Object.class), 
-                                                                          DefBootstrap.METHOD_CALL, 0L);
+                                                                          DefBootstrap.METHOD_CALL, "");
         site.depth = DefBootstrap.PIC.MAX_DEPTH; // mark megamorphic
         MethodHandle handle = site.dynamicInvoker();
         assertEquals(2, (int)handle.invokeExact((Object) Arrays.asList("1", "2")));

From cfa468944515e9568a39cb3a4aa92538cc8fdc3c Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Mon, 20 Jun 2016 11:38:48 +0200
Subject: [PATCH 0957/1311] Share test injector creation

---
 .../aggregations/BaseAggregationTestCase.java | 104 +++++++++---------
 .../BasePipelineAggregationTestCase.java      |  78 +------------
 2 files changed, 57 insertions(+), 125 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
index b1065c90efa..7bce599f55a 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
@@ -102,56 +102,9 @@ public abstract class BaseAggregationTestCase<AB extends AbstractAggregationBuil
      */
     @BeforeClass
     public static void init() throws IOException {
-        // we have to prefer CURRENT since with the range of versions we support it's rather unlikely to get the current actually.
-        Version version = randomBoolean() ? Version.CURRENT
-                : VersionUtils.randomVersionBetween(random(), Version.V_2_0_0_beta1, Version.CURRENT);
-        Settings settings = Settings.builder()
-                .put("node.name", AbstractQueryTestCase.class.toString())
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-                .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
-                .build();
-
-        namedWriteableRegistry = new NamedWriteableRegistry();
         index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
-        Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        final ThreadPool threadPool = new ThreadPool(settings);
-        final ClusterService clusterService = createClusterService(threadPool);
-        setState(clusterService, new ClusterState.Builder(clusterService.state()).metaData(new MetaData.Builder()
-                .put(new IndexMetaData.Builder(index.getName()).settings(indexSettings).numberOfShards(1).numberOfReplicas(0))));
-        ScriptModule scriptModule = newTestScriptModule();
-        List<Setting<?>> scriptSettings = scriptModule.getSettings();
-        scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
-        SettingsModule settingsModule = new SettingsModule(settings, scriptSettings, Collections.emptyList());
-        injector = new ModulesBuilder().add(
-                (b) -> {
-                    b.bind(Environment.class).toInstance(new Environment(settings));
-                    b.bind(ThreadPool.class).toInstance(threadPool);
-                },
-                settingsModule,
-                scriptModule,
-                new IndicesModule(namedWriteableRegistry) {
-
-                    @Override
-                    protected void configure() {
-                        bindMapperExtension();
-                    }
-                }, new SearchModule(settings, namedWriteableRegistry) {
-                    @Override
-                    protected void configureSearch() {
-                        // Skip me
-                    }
-                },
-                new IndexSettingsModule(index, settings),
-
-                new AbstractModule() {
-                    @Override
-                    protected void configure() {
-                        bind(ClusterService.class).toProvider(Providers.of(clusterService));
-                        bind(CircuitBreakerService.class).to(NoneCircuitBreakerService.class);
-                        bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry);
-                    }
-                }
-        ).createInjector();
+        injector = buildInjector(index);
+        namedWriteableRegistry = injector.getInstance(NamedWriteableRegistry.class);
         aggParsers = injector.getInstance(AggregatorParsers.class);
         //create some random type with some default field, those types will stick around for all of the subclasses
         currentTypes = new String[randomIntBetween(0, 5)];
@@ -163,6 +116,59 @@ public abstract class BaseAggregationTestCase<AB extends AbstractAggregationBuil
         parseFieldMatcher = ParseFieldMatcher.STRICT;
     }
 
+    public static final Injector buildInjector(Index index) {
+        // we have to prefer CURRENT since with the range of versions we support it's rather unlikely to get the current actually.
+        Version version = randomBoolean() ? Version.CURRENT
+            : VersionUtils.randomVersionBetween(random(), Version.V_2_0_0_beta1, Version.CURRENT);
+        Settings settings = Settings.builder()
+            .put("node.name", AbstractQueryTestCase.class.toString())
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+            .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
+            .build();
+
+        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
+        Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
+        final ThreadPool threadPool = new ThreadPool(settings);
+        final ClusterService clusterService = createClusterService(threadPool);
+        setState(clusterService, new ClusterState.Builder(clusterService.state()).metaData(new MetaData.Builder()
+            .put(new IndexMetaData.Builder(index.getName()).settings(indexSettings).numberOfShards(1).numberOfReplicas(0))));
+        ScriptModule scriptModule = newTestScriptModule();
+        List<Setting<?>> scriptSettings = scriptModule.getSettings();
+        scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
+        SettingsModule settingsModule = new SettingsModule(settings, scriptSettings, Collections.emptyList());
+        return new ModulesBuilder().add(
+            (b) -> {
+                b.bind(Environment.class).toInstance(new Environment(settings));
+                b.bind(ThreadPool.class).toInstance(threadPool);
+            },
+            settingsModule,
+            scriptModule,
+            new IndicesModule(namedWriteableRegistry) {
+
+                @Override
+                protected void configure() {
+                    bindMapperExtension();
+                }
+            }, new SearchModule(settings, namedWriteableRegistry) {
+                @Override
+                protected void configureSearch() {
+                    // Skip me
+                }
+            },
+            new IndexSettingsModule(index, settings),
+
+            new AbstractModule() {
+                @Override
+                protected void configure() {
+                    bind(ClusterService.class).toProvider(Providers.of(clusterService));
+                    bind(CircuitBreakerService.class).to(NoneCircuitBreakerService.class);
+                    bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry);
+                }
+            }
+        ).createInjector();
+    }
+
+
     @AfterClass
     public static void afterClass() throws Exception {
         injector.getInstance(ClusterService.class).close();
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
index 0410c5d91d9..a3f9f11dc2e 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
@@ -19,54 +19,29 @@
 
 package org.elasticsearch.search.aggregations;
 
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.ModulesBuilder;
-import org.elasticsearch.common.inject.util.Providers;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.indices.IndicesModule;
-import org.elasticsearch.indices.breaker.CircuitBreakerService;
-import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
-import org.elasticsearch.script.ScriptModule;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.aggregations.pipeline.AbstractPipelineAggregationBuilder;
-import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.IndexSettingsModule;
-import org.elasticsearch.test.InternalSettingsPlugin;
-import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
 import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-
-import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
-import static org.elasticsearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.equalTo;
 
 public abstract class BasePipelineAggregationTestCase<AF extends AbstractPipelineAggregationBuilder<AF>> extends ESTestCase {
@@ -76,9 +51,6 @@ public abstract class BasePipelineAggregationTestCase<AF extends AbstractPipelin
     protected static final String DOUBLE_FIELD_NAME = "mapped_double";
     protected static final String BOOLEAN_FIELD_NAME = "mapped_boolean";
     protected static final String DATE_FIELD_NAME = "mapped_date";
-    protected static final String OBJECT_FIELD_NAME = "mapped_object";
-    protected static final String[] mappedFieldNames = new String[]{STRING_FIELD_NAME, INT_FIELD_NAME,
-            DOUBLE_FIELD_NAME, BOOLEAN_FIELD_NAME, DATE_FIELD_NAME, OBJECT_FIELD_NAME};
 
     private static Injector injector;
     private static Index index;
@@ -102,55 +74,9 @@ public abstract class BasePipelineAggregationTestCase<AF extends AbstractPipelin
      */
     @BeforeClass
     public static void init() throws IOException {
-        // we have to prefer CURRENT since with the range of versions we support it's rather unlikely to get the current actually.
-        Version version = randomBoolean() ? Version.CURRENT
-                : VersionUtils.randomVersionBetween(random(), Version.V_2_0_0_beta1, Version.CURRENT);
-        Settings settings = Settings.builder()
-                .put("node.name", AbstractQueryTestCase.class.toString())
-                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-                .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
-                .build();
-
-        namedWriteableRegistry = new NamedWriteableRegistry();
         index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
-        Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        final ThreadPool threadPool = new ThreadPool(settings);
-        final ClusterService clusterService = createClusterService(threadPool);
-        setState(clusterService, new ClusterState.Builder(clusterService.state()).metaData(new MetaData.Builder()
-                .put(new IndexMetaData.Builder(index.getName()).settings(indexSettings).numberOfShards(1).numberOfReplicas(0))));
-        ScriptModule scriptModule = newTestScriptModule();
-        List<Setting<?>> scriptSettings = scriptModule.getSettings();
-        scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
-        SettingsModule settingsModule = new SettingsModule(settings, scriptSettings, Collections.emptyList());
-        injector = new ModulesBuilder().add(
-                (b) -> {
-                    b.bind(Environment.class).toInstance(new Environment(settings));
-                    b.bind(ThreadPool.class).toInstance(threadPool);
-                },
-                settingsModule,
-                scriptModule,
-                new IndicesModule(namedWriteableRegistry) {
-
-                    @Override
-                    protected void configure() {
-                        bindMapperExtension();
-                    }
-                }, new SearchModule(settings, namedWriteableRegistry) {
-                    @Override
-                    protected void configureSearch() {
-                        // Skip me
-                    }
-                },
-                new IndexSettingsModule(index, settings),
-                new AbstractModule() {
-                    @Override
-                    protected void configure() {
-                        bind(ClusterService.class).toProvider(Providers.of(clusterService));
-                        bind(CircuitBreakerService.class).to(NoneCircuitBreakerService.class);
-                        bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry);
-                    }
-                }
-        ).createInjector();
+        injector = BaseAggregationTestCase.buildInjector(index);
+        namedWriteableRegistry = injector.getInstance(NamedWriteableRegistry.class);
         aggParsers = injector.getInstance(AggregatorParsers.class);
         //create some random type with some default field, those types will stick around for all of the subclasses
         currentTypes = new String[randomIntBetween(0, 5)];

From fb713774a140bd0dfe672694ca69159b8b009909 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Mon, 20 Jun 2016 11:45:52 +0200
Subject: [PATCH 0958/1311] Don't create enviroment more than once

---
 core/src/main/java/org/elasticsearch/node/Node.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index 2069320104a..9c849bbe14b 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -271,7 +271,7 @@ public class Node implements Closeable {
             resourcesToClose.add(circuitBreakerService);
             modules.add(settingsModule);
             modules.add(b -> {
-                    b.bind(Environment.class).toInstance(new Environment(settings));
+                    b.bind(Environment.class).toInstance(environment);
                     b.bind(ThreadPool.class).toInstance(threadPool);
                     b.bind(NodeEnvironment.class).toInstance(nodeEnvironment);
                     b.bind(TribeService.class).toInstance(tribeService);

From e50314bb6ede2fbde462fdcfab2d189cf6e5de63 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Mon, 20 Jun 2016 11:53:07 +0200
Subject: [PATCH 0959/1311] Remove NodeClientModule and PluginsModule

---
 .../client/node/NodeClientModule.java         | 34 ------------------
 .../client/transport/TransportClient.java     | 11 +++---
 .../java/org/elasticsearch/node/Node.java     |  7 ++--
 .../elasticsearch/plugins/PluginsModule.java  | 36 -------------------
 .../test/AbstractQueryTestCase.java           |  3 +-
 5 files changed, 8 insertions(+), 83 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/client/node/NodeClientModule.java
 delete mode 100644 core/src/main/java/org/elasticsearch/plugins/PluginsModule.java

diff --git a/core/src/main/java/org/elasticsearch/client/node/NodeClientModule.java b/core/src/main/java/org/elasticsearch/client/node/NodeClientModule.java
deleted file mode 100644
index de134887303..00000000000
--- a/core/src/main/java/org/elasticsearch/client/node/NodeClientModule.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.client.node;
-
-import org.elasticsearch.client.Client;
-import org.elasticsearch.common.inject.AbstractModule;
-
-/**
- *
- */
-public class NodeClientModule extends AbstractModule {
-
-    @Override
-    protected void configure() {
-        bind(Client.class).to(NodeClient.class).asEagerSingleton();
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
index 2c2a5ad0428..09e967b00c4 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.client.transport;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionModule;
@@ -46,7 +45,6 @@ import org.elasticsearch.monitor.MonitorService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.PluginsModule;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.threadpool.ExecutorBuilder;
@@ -120,9 +118,6 @@ public class TransportClient extends AbstractClient {
         public TransportClient build() {
             final PluginsService pluginsService = newPluginService(providedSettings);
             final Settings settings = pluginsService.updatedSettings();
-
-            Version version = Version.CURRENT;
-
             final ThreadPool threadPool = new ThreadPool(settings);
             final NetworkService networkService = new NetworkService(settings);
             NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
@@ -133,7 +128,6 @@ public class TransportClient extends AbstractClient {
                 for (Module pluginModule : pluginsService.nodeModules()) {
                     modules.add(pluginModule);
                 }
-                modules.add(new PluginsModule(pluginsService));
                 modules.add(new NetworkModule(networkService, settings, true, namedWriteableRegistry));
                 modules.add(b -> b.bind(ThreadPool.class).toInstance(threadPool));
                 modules.add(new SearchModule(settings, namedWriteableRegistry) {
@@ -156,7 +150,10 @@ public class TransportClient extends AbstractClient {
                 CircuitBreakerService circuitBreakerService = Node.createCircuitBreakerService(settingsModule.getSettings(),
                     settingsModule.getClusterSettings());
                 modules.add(settingsModule);
-                modules.add((b -> b.bind(CircuitBreakerService.class).toInstance(circuitBreakerService)));
+                modules.add((b -> {
+                    b.bind(PluginsService.class).toInstance(pluginsService);
+                    b.bind(CircuitBreakerService.class).toInstance(circuitBreakerService);
+                }));
 
                 Injector injector = modules.createInjector();
                 final TransportService transportService = injector.getInstance(TransportService.class);
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index 9c849bbe14b..68073261959 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -27,7 +27,7 @@ import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionModule;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.client.node.NodeClientModule;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateObserver;
@@ -87,7 +87,6 @@ import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
 import org.elasticsearch.node.service.NodeService;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.PluginsModule;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.repositories.RepositoriesModule;
@@ -252,7 +251,6 @@ public class Node implements Closeable {
                 modules.add(pluginModule);
             }
             final MonitorService monitorService = new MonitorService(settings, nodeEnvironment, threadPool);
-            modules.add(new PluginsModule(pluginsService));
             modules.add(new NodeModule(this, monitorService));
             modules.add(new NetworkModule(networkService, settings, false, namedWriteableRegistry));
             modules.add(scriptModule);
@@ -262,7 +260,6 @@ public class Node implements Closeable {
             modules.add(new SearchModule(settings, namedWriteableRegistry));
             modules.add(new ActionModule(DiscoveryNode.isIngestNode(settings), false));
             modules.add(new GatewayModule());
-            modules.add(new NodeClientModule());
             modules.add(new RepositoriesModule());
             modules.add(new AnalysisModule(environment));
             pluginsService.processModules(modules);
@@ -271,6 +268,8 @@ public class Node implements Closeable {
             resourcesToClose.add(circuitBreakerService);
             modules.add(settingsModule);
             modules.add(b -> {
+                    b.bind(PluginsService.class).toInstance(pluginsService);
+                    b.bind(Client.class).to(NodeClient.class).asEagerSingleton();
                     b.bind(Environment.class).toInstance(environment);
                     b.bind(ThreadPool.class).toInstance(threadPool);
                     b.bind(NodeEnvironment.class).toInstance(nodeEnvironment);
diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginsModule.java b/core/src/main/java/org/elasticsearch/plugins/PluginsModule.java
deleted file mode 100644
index 04e468cdd6c..00000000000
--- a/core/src/main/java/org/elasticsearch/plugins/PluginsModule.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.plugins;
-
-import org.elasticsearch.common.inject.AbstractModule;
-
-public class PluginsModule extends AbstractModule {
-
-    private final PluginsService pluginsService;
-
-    public PluginsModule(PluginsService pluginsService) {
-        this.pluginsService = pluginsService;
-    }
-
-    @Override
-    protected void configure() {
-        bind(PluginsService.class).toInstance(pluginsService);
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
index 65c43fd38c6..ea2154b355e 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
@@ -93,7 +93,6 @@ import org.elasticsearch.indices.mapper.MapperRegistry;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.PluginsModule;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.script.Script.ScriptParseException;
 import org.elasticsearch.script.ScriptModule;
@@ -889,9 +888,9 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             for (Module pluginModule : pluginsService.nodeModules()) {
                 modulesBuilder.add(pluginModule);
             }
-            modulesBuilder.add(new PluginsModule(pluginsService));
             modulesBuilder.add(
                     (b) -> {
+                        b.bind(PluginsService.class).toInstance(pluginsService);
                         b.bind(Environment.class).toInstance(new Environment(settings));
                         b.bind(ThreadPool.class).toInstance(threadPool);
                     },

From 0cb4e574a9c534847ceafcbf83ccebf76cd68123 Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Mon, 20 Jun 2016 12:16:22 +0200
Subject: [PATCH 0960/1311] index shard should be able to cancel check index on
 close. (#18839)

If someone sets `index.shard.check_on_startup`, indexing start up time can be slow (by design, it diligently goes and checks all data). If for some reason the shard is closed in that time, the store ref is kept around and prevents a new shard copy to be allocated to this node via the shard level locks. This is especially tricky if the shard is close due to a cancelled recovery which may re-restart soon.

This commit adds a cancellable threads instance to each IndexShard and perform index checking underneath it, so it can be cancelled on close.
---
 .../java/org/elasticsearch/index/shard/IndexShard.java    | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 26701703ce1..9ebbde77c6a 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -53,6 +53,7 @@ import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.Callback;
+import org.elasticsearch.common.util.CancellableThreads;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.SuspendableRefContainer;
 import org.elasticsearch.index.Index;
@@ -131,8 +132,8 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Consumer;
 import java.util.function.BiConsumer;
+import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
 public class IndexShard extends AbstractIndexShardComponent implements IndicesClusterStateService.Shard {
@@ -158,6 +159,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
     private final TranslogConfig translogConfig;
     private final IndexEventListener indexEventListener;
     private final QueryCachingPolicy cachingPolicy;
+    private final CancellableThreads cancellableThreads;
 
 
     /**
@@ -265,6 +267,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         primaryTerm = indexSettings.getIndexMetaData().primaryTerm(shardId.id());
         refreshListeners = buildRefreshListeners();
         persistMetadata(shardRouting, null);
+        cancellableThreads = new CancellableThreads();
     }
 
     public Store store() {
@@ -843,6 +846,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
                 } finally { // playing safe here and close the engine even if the above succeeds - close can be called multiple times
                     IOUtils.close(engine);
                 }
+                cancellableThreads.cancel(reason);
             }
         }
     }
@@ -1281,7 +1285,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
     private void checkIndex() throws IOException {
         if (store.tryIncRef()) {
             try {
-                doCheckIndex();
+                cancellableThreads.executeIO(this::doCheckIndex);
             } finally {
                 store.decRef();
             }

From 9506f605046c8aa05b2fd57cd430f207a4334a78 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Mon, 20 Jun 2016 13:10:35 +0200
Subject: [PATCH 0961/1311] Improve error message if a setting is not found
 (#18920)

Today we only emit that the setting wasn't found unless we have
some DYM suggestions. Yet, if a setting is not found at all and there
are no suggestions due to typos it's likely a removed setting or the plugin
that is supposed to be configured is not installed.
This commit adds some info text to the exception to help the user debugging
the problem before opening bugreports.

Instead of emitting:

`unknown setting [foo.bar]`

we now emit:

`unknown setting [foo.bar] please check the migration guide for removed settings and ensure that the plugin you are configuring is installed`

Relates to #18663
---
 .../resources/checkstyle_suppressions.xml     |  2 --
 .../settings/AbstractScopedSettings.java      |  3 +++
 .../admin/indices/create/CreateIndexIT.java   | 26 ++++++++++++-------
 .../common/settings/ScopedSettingsTests.java  |  6 +++--
 .../common/settings/SettingsModuleTests.java  |  6 +++--
 .../indices/state/SimpleIndexStateIT.java     | 12 ++++++---
 .../template/SimpleIndexTemplateIT.java       |  3 ++-
 7 files changed, 38 insertions(+), 20 deletions(-)

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index ffad8d37c5b..27b9efbf0a4 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -712,7 +712,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]cluster[/\\]stats[/\\]ClusterStatsIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]TransportAnalyzeActionTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]cache[/\\]clear[/\\]ClearIndicesCacheBlocksIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]create[/\\]CreateIndexIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]flush[/\\]SyncedFlushUnitTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]get[/\\]GetIndexIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]action[/\\]admin[/\\]indices[/\\]shards[/\\]IndicesShardStoreRequestIT.java" checks="LineLength" />
@@ -1022,7 +1021,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]state[/\\]CloseIndexDisableCloseAllIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]state[/\\]OpenCloseIndexIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]state[/\\]RareClusterStateIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]state[/\\]SimpleIndexStateIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]stats[/\\]IndexStatsIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]store[/\\]IndicesStoreIntegrationIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]store[/\\]IndicesStoreTests.java" checks="LineLength" />
diff --git a/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
index 0170c4218a2..817e109bf4d 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
@@ -263,6 +263,9 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
             List<String> keys = scoredKeys.stream().map((a) -> a.v2()).collect(Collectors.toList());
             if (keys.isEmpty() == false) {
                 msg += " did you mean " + (keys.size() == 1 ? "[" + keys.get(0) + "]": "any of " + keys.toString()) + "?";
+            } else {
+                msg += " please check that any required plugins are installed, or check the breaking changes documentation for removed " +
+                    "settings";
             }
             throw new IllegalArgumentException(msg);
         }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
index 8fbb489d9c2..3e7323dceeb 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
@@ -70,7 +70,8 @@ public class CreateIndexIT extends ESIntegTestCase {
             prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_CREATION_DATE, 4L)).get();
             fail();
         } catch (IllegalArgumentException ex) {
-            assertEquals("unknown setting [index.creation_date]", ex.getMessage());
+            assertEquals("unknown setting [index.creation_date] please check that any required plugins are installed, or check the " +
+                "breaking changes documentation for removed settings", ex.getMessage());
         }
     }
 
@@ -165,7 +166,8 @@ public class CreateIndexIT extends ESIntegTestCase {
                 .get();
             fail("should have thrown an exception about the shard count");
         } catch (IllegalArgumentException e) {
-            assertEquals("unknown setting [index.unknown.value]", e.getMessage());
+            assertEquals("unknown setting [index.unknown.value] please check that any required plugins are installed, or check the" +
+                " breaking changes documentation for removed settings", e.getMessage());
         }
     }
 
@@ -211,13 +213,16 @@ public class CreateIndexIT extends ESIntegTestCase {
                      @Override
                     public void run() {
                          try {
-                             client().prepareIndex("test", "test").setSource("index_version", indexVersion.get()).get(); // recreate that index
+                             // recreate that index
+                             client().prepareIndex("test", "test").setSource("index_version", indexVersion.get()).get();
                              synchronized (indexVersionLock) {
-                                 // we sync here since we have to ensure that all indexing operations below for a given ID are done before we increment the
-                                 // index version otherwise a doc that is in-flight could make it into an index that it was supposed to be deleted for and our assertion fail...
+                                 // we sync here since we have to ensure that all indexing operations below for a given ID are done before
+                                 // we increment the index version otherwise a doc that is in-flight could make it into an index that it
+                                 // was supposed to be deleted for and our assertion fail...
                                  indexVersion.incrementAndGet();
                              }
-                             assertAcked(client().admin().indices().prepareDelete("test").get()); // from here on all docs with index_version == 0|1 must be gone!!!! only 2 are ok;
+                             // from here on all docs with index_version == 0|1 must be gone!!!! only 2 are ok;
+                             assertAcked(client().admin().indices().prepareDelete("test").get());
                          } finally {
                              latch.countDown();
                          }
@@ -249,8 +254,10 @@ public class CreateIndexIT extends ESIntegTestCase {
         latch.await();
         refresh();
 
-        // we only really assert that we never reuse segments of old indices or anything like this here and that nothing fails with crazy exceptions
-        SearchResponse expected = client().prepareSearch("test").setIndicesOptions(IndicesOptions.lenientExpandOpen()).setQuery(new RangeQueryBuilder("index_version").from(indexVersion.get(), true)).get();
+        // we only really assert that we never reuse segments of old indices or anything like this here and that nothing fails with
+        // crazy exceptions
+        SearchResponse expected = client().prepareSearch("test").setIndicesOptions(IndicesOptions.lenientExpandOpen())
+            .setQuery(new RangeQueryBuilder("index_version").from(indexVersion.get(), true)).get();
         SearchResponse all = client().prepareSearch("test").setIndicesOptions(IndicesOptions.lenientExpandOpen()).get();
         assertEquals(expected + " vs. " + all, expected.getHits().getTotalHits(), all.getHits().getTotalHits());
         logger.info("total: {}", expected.getHits().getTotalHits());
@@ -283,7 +290,8 @@ public class CreateIndexIT extends ESIntegTestCase {
     }
 
     public void testRestartIndexCreationAfterFullClusterRestart() throws Exception {
-        client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put("cluster.routing.allocation.enable", "none")).get();
+        client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put("cluster.routing.allocation.enable",
+            "none")).get();
         client().admin().indices().prepareCreate("test").setSettings(indexSettings()).get();
         internalCluster().fullRestart();
         ensureGreen("test");
diff --git a/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java b/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
index 3afd60d86e4..664f8cb96ab 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
@@ -202,20 +202,22 @@ public class ScopedSettingsTests extends ESTestCase {
         IndexScopedSettings settings = new IndexScopedSettings(
             Settings.EMPTY,
             IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
+        String unknownMsgSuffix = " please check that any required plugins are installed, or check the breaking changes documentation for" +
+            " removed settings";
         settings.validate(Settings.builder().put("index.store.type", "boom"));
         settings.validate(Settings.builder().put("index.store.type", "boom").build());
         try {
             settings.validate(Settings.builder().put("index.store.type", "boom", "i.am.not.a.setting", true));
             fail();
         } catch (IllegalArgumentException e) {
-            assertEquals("unknown setting [i.am.not.a.setting]", e.getMessage());
+            assertEquals("unknown setting [i.am.not.a.setting]" + unknownMsgSuffix, e.getMessage());
         }
 
         try {
             settings.validate(Settings.builder().put("index.store.type", "boom", "i.am.not.a.setting", true).build());
             fail();
         } catch (IllegalArgumentException e) {
-            assertEquals("unknown setting [i.am.not.a.setting]", e.getMessage());
+            assertEquals("unknown setting [i.am.not.a.setting]" + unknownMsgSuffix, e.getMessage());
         }
 
         try {
diff --git a/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java b/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
index 353b7b61d6c..692134916ef 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/SettingsModuleTests.java
@@ -51,7 +51,8 @@ public class SettingsModuleTests extends ModuleTestCase {
                 () -> new SettingsModule(settings));
             assertEquals("Failed to parse value [[2.0]] for setting [cluster.routing.allocation.balance.shard]", ex.getMessage());
             assertEquals(1, ex.getSuppressed().length);
-            assertEquals("unknown setting [some.foo.bar]", ex.getSuppressed()[0].getMessage());
+            assertEquals("unknown setting [some.foo.bar] please check that any required plugins are installed, or check the breaking " +
+                "changes documentation for removed settings", ex.getSuppressed()[0].getMessage());
         }
 
         {
@@ -127,7 +128,8 @@ public class SettingsModuleTests extends ModuleTestCase {
                 new SettingsModule(settings);
                 fail();
             } catch (IllegalArgumentException ex) {
-                assertEquals("tribe.blocks validation failed: unknown setting [wtf]", ex.getMessage());
+                assertEquals("tribe.blocks validation failed: unknown setting [wtf] please check that any required plugins are" +
+                    " installed, or check the breaking changes documentation for removed settings", ex.getMessage());
             }
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/indices/state/SimpleIndexStateIT.java b/core/src/test/java/org/elasticsearch/indices/state/SimpleIndexStateIT.java
index f3898bc9f3c..ea3ebf5179b 100644
--- a/core/src/test/java/org/elasticsearch/indices/state/SimpleIndexStateIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/state/SimpleIndexStateIT.java
@@ -56,7 +56,8 @@ public class SimpleIndexStateIT extends ESIntegTestCase {
         ClusterStateResponse stateResponse = client().admin().cluster().prepareState().get();
         assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.OPEN));
         assertThat(stateResponse.getState().routingTable().index("test").shards().size(), equalTo(numShards.numPrimaries));
-        assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(), equalTo(numShards.totalNumShards));
+        assertEquals(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size()
+            , numShards.totalNumShards);
 
         logger.info("--> indexing a simple document");
         client().prepareIndex("test", "type1", "1").setSource("field1", "value1").get();
@@ -88,7 +89,8 @@ public class SimpleIndexStateIT extends ESIntegTestCase {
         assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.OPEN));
 
         assertThat(stateResponse.getState().routingTable().index("test").shards().size(), equalTo(numShards.numPrimaries));
-        assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(), equalTo(numShards.totalNumShards));
+        assertEquals(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(),
+            numShards.totalNumShards);
 
         logger.info("--> indexing a simple document");
         client().prepareIndex("test", "type1", "1").setSource("field1", "value1").get();
@@ -119,7 +121,8 @@ public class SimpleIndexStateIT extends ESIntegTestCase {
         ClusterStateResponse stateResponse = client().admin().cluster().prepareState().get();
         assertThat(stateResponse.getState().metaData().index("test").getState(), equalTo(IndexMetaData.State.OPEN));
         assertThat(stateResponse.getState().routingTable().index("test").shards().size(), equalTo(numShards.numPrimaries));
-        assertThat(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(), equalTo(numShards.totalNumShards));
+        assertEquals(stateResponse.getState().routingTable().index("test").shardsWithState(ShardRoutingState.STARTED).size(),
+            numShards.totalNumShards);
 
         logger.info("--> indexing a simple document");
         client().prepareIndex("test", "type1", "1").setSource("field1", "value1").get();
@@ -143,7 +146,8 @@ public class SimpleIndexStateIT extends ESIntegTestCase {
         }
 
         logger.info("--> creating test index with valid settings ");
-        CreateIndexResponse response = client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put("number_of_shards", 1)).get();
+        CreateIndexResponse response = client().admin().indices().prepareCreate("test")
+            .setSettings(Settings.builder().put("number_of_shards", 1)).get();
         assertThat(response.isAcknowledged(), equalTo(true));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
index 298cb9cd9e3..493f8b74e04 100644
--- a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
@@ -322,7 +322,8 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
                 .setTemplate("te*")
                 .setSettings(Settings.builder().put("does_not_exist", "test"))
                 .get());
-        assertEquals("unknown setting [index.does_not_exist]", e.getMessage());
+        assertEquals("unknown setting [index.does_not_exist] please check that any required plugins are" +
+            " installed, or check the breaking changes documentation for removed settings", e.getMessage());
 
         response = client().admin().indices().prepareGetTemplates().get();
         assertEquals(0, response.getIndexTemplates().size());

From 459665914b3aa3b107a3e3c9069ab812ebbf36b7 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Mon, 20 Jun 2016 13:18:19 +0200
Subject: [PATCH 0962/1311] Detach BigArrays from Guice (#18973)

BigArrays can be fully constructed without Guice, this change cleans up
it's creation and the mocking in MockNode.
---
 .../client/transport/TransportClient.java     | 39 ++++++++-----------
 .../TransportClientNodesService.java          |  3 +-
 .../elasticsearch/common/util/BigArrays.java  |  3 +-
 .../elasticsearch/env/NodeEnvironment.java    |  5 ---
 .../java/org/elasticsearch/node/Node.java     | 11 ++++++
 .../org/elasticsearch/node/NodeModule.java    |  9 -----
 .../common/util/MockBigArrays.java            |  1 -
 .../java/org/elasticsearch/node/MockNode.java | 14 ++++++-
 .../elasticsearch/node/NodeMocksPlugin.java   |  7 +---
 .../test/InternalTestCluster.java             |  9 +++--
 .../test/NodeConfigurationSource.java         |  8 ----
 11 files changed, 51 insertions(+), 58 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
index 09e967b00c4..f36fd199292 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.client.transport;
 
+import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionModule;
@@ -41,7 +42,6 @@ import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
-import org.elasticsearch.monitor.MonitorService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
 import org.elasticsearch.plugins.Plugin;
@@ -52,6 +52,7 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.netty.NettyTransport;
 
+import java.io.Closeable;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
@@ -118,10 +119,11 @@ public class TransportClient extends AbstractClient {
         public TransportClient build() {
             final PluginsService pluginsService = newPluginService(providedSettings);
             final Settings settings = pluginsService.updatedSettings();
+            final List<Closeable> resourcesToClose = new ArrayList<>();
             final ThreadPool threadPool = new ThreadPool(settings);
+            resourcesToClose.add(() -> ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS));
             final NetworkService networkService = new NetworkService(settings);
             NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
-            boolean success = false;
             try {
                 ModulesBuilder modules = new ModulesBuilder();
                 // plugin modules must be added here, before others or we can get crazy injection errors...
@@ -149,8 +151,12 @@ public class TransportClient extends AbstractClient {
                 SettingsModule settingsModule = new SettingsModule(settings, additionalSettings, additionalSettingsFilter);
                 CircuitBreakerService circuitBreakerService = Node.createCircuitBreakerService(settingsModule.getSettings(),
                     settingsModule.getClusterSettings());
+                resourcesToClose.add(circuitBreakerService);
+                BigArrays bigArrays = new BigArrays(settings, circuitBreakerService);
+                resourcesToClose.add(bigArrays);
                 modules.add(settingsModule);
                 modules.add((b -> {
+                    b.bind(BigArrays.class).toInstance(bigArrays);
                     b.bind(PluginsService.class).toInstance(pluginsService);
                     b.bind(CircuitBreakerService.class).toInstance(circuitBreakerService);
                 }));
@@ -159,14 +165,11 @@ public class TransportClient extends AbstractClient {
                 final TransportService transportService = injector.getInstance(TransportService.class);
                 transportService.start();
                 transportService.acceptIncomingRequests();
-
                 TransportClient transportClient = new TransportClient(injector);
-                success = true;
+                resourcesToClose.clear();
                 return transportClient;
             } finally {
-                if (!success) {
-                    ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
-                }
+                IOUtils.closeWhileHandlingException(resourcesToClose);
             }
         }
     }
@@ -261,24 +264,16 @@ public class TransportClient extends AbstractClient {
      */
     @Override
     public void close() {
-        injector.getInstance(TransportClientNodesService.class).close();
-        injector.getInstance(TransportService.class).close();
-        try {
-            injector.getInstance(MonitorService.class).close();
-        } catch (Exception e) {
-            // ignore, might not be bounded
-        }
+        List<Closeable> closeables = new ArrayList<>();
+        closeables.add(injector.getInstance(TransportClientNodesService.class));
+        closeables.add(injector.getInstance(TransportService.class));
 
         for (Class<? extends LifecycleComponent> plugin : injector.getInstance(PluginsService.class).nodeServices()) {
-            injector.getInstance(plugin).close();
+            closeables.add(injector.getInstance(plugin));
         }
-        try {
-            ThreadPool.terminate(injector.getInstance(ThreadPool.class), 10, TimeUnit.SECONDS);
-        } catch (Exception e) {
-            // ignore
-        }
-
-        injector.getInstance(BigArrays.class).close();
+        closeables.add(() -> ThreadPool.terminate(injector.getInstance(ThreadPool.class), 10, TimeUnit.SECONDS));
+        closeables.add(injector.getInstance(BigArrays.class));
+        IOUtils.closeWhileHandlingException(closeables);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
index 0781576524d..120a4711217 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
@@ -48,6 +48,7 @@ import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.Closeable;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -66,7 +67,7 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 /**
  *
  */
-public class TransportClientNodesService extends AbstractComponent {
+public class TransportClientNodesService extends AbstractComponent implements Closeable {
 
     private final TimeValue nodesSamplerInterval;
 
diff --git a/core/src/main/java/org/elasticsearch/common/util/BigArrays.java b/core/src/main/java/org/elasticsearch/common/util/BigArrays.java
index 017881a9218..c2c8e242acc 100644
--- a/core/src/main/java/org/elasticsearch/common/util/BigArrays.java
+++ b/core/src/main/java/org/elasticsearch/common/util/BigArrays.java
@@ -373,12 +373,11 @@ public class BigArrays implements Releasable {
     final boolean checkBreaker;
     private final BigArrays circuitBreakingInstance;
 
-    @Inject
     public BigArrays(Settings settings, @Nullable final CircuitBreakerService breakerService) {
         // Checking the breaker is disabled if not specified
         this(new PageCacheRecycler(settings), breakerService, false);
     }
-
+    // public for tests
     public BigArrays(PageCacheRecycler recycler, @Nullable final CircuitBreakerService breakerService, boolean checkBreaker) {
         this.checkBreaker = checkBreaker;
         this.recycler = recycler;
diff --git a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
index 2b2aab2ccd2..8815d18769d 100644
--- a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
+++ b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
@@ -32,12 +32,9 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.logging.DeprecationLogger;
-import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -66,7 +63,6 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -167,7 +163,6 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
     public static final String NODE_LOCK_FILENAME = "node.lock";
     public static final String UPGRADE_LOCK_FILENAME = "upgrade.lock";
 
-    @Inject
     public NodeEnvironment(Settings settings, Environment environment) throws IOException {
         super(settings);
 
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index 68073261959..de77f02ef51 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -266,6 +266,8 @@ public class Node implements Closeable {
             CircuitBreakerService circuitBreakerService = createCircuitBreakerService(settingsModule.getSettings(),
                 settingsModule.getClusterSettings());
             resourcesToClose.add(circuitBreakerService);
+            BigArrays bigArrays = createBigArrays(settings, circuitBreakerService);
+            resourcesToClose.add(bigArrays);
             modules.add(settingsModule);
             modules.add(b -> {
                     b.bind(PluginsService.class).toInstance(pluginsService);
@@ -276,6 +278,7 @@ public class Node implements Closeable {
                     b.bind(TribeService.class).toInstance(tribeService);
                     b.bind(ResourceWatcherService.class).toInstance(resourceWatcherService);
                     b.bind(CircuitBreakerService.class).toInstance(circuitBreakerService);
+                    b.bind(BigArrays.class).toInstance(bigArrays);
                 }
             );
             injector = modules.createInjector();
@@ -626,4 +629,12 @@ public class Node implements Closeable {
             throw new IllegalArgumentException("Unknown circuit breaker type [" + type + "]");
         }
     }
+
+    /**
+     * Creates a new {@link BigArrays} instance used for this node.
+     * This method can be overwritten by subclasses to change their {@link BigArrays} implementation for instance for testing
+     */
+    BigArrays createBigArrays(Settings settings, CircuitBreakerService circuitBreakerService) {
+        return new BigArrays(settings, circuitBreakerService);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/node/NodeModule.java b/core/src/main/java/org/elasticsearch/node/NodeModule.java
index c5d087c0ef2..85cfe5bd6b2 100644
--- a/core/src/main/java/org/elasticsearch/node/NodeModule.java
+++ b/core/src/main/java/org/elasticsearch/node/NodeModule.java
@@ -37,9 +37,6 @@ public class NodeModule extends AbstractModule {
     private final MonitorService monitorService;
     private final ProcessorsRegistry.Builder processorsRegistryBuilder;
 
-    // pkg private so tests can mock
-    Class<? extends BigArrays> bigArraysImpl = BigArrays.class;
-
     public NodeModule(Node node, MonitorService monitorService) {
         this.node = node;
         this.monitorService = monitorService;
@@ -48,12 +45,6 @@ public class NodeModule extends AbstractModule {
 
     @Override
     protected void configure() {
-        if (bigArraysImpl == BigArrays.class) {
-            bind(BigArrays.class).asEagerSingleton();
-        } else {
-            bind(BigArrays.class).to(bigArraysImpl).asEagerSingleton();
-        }
-
         bind(Node.class).toInstance(node);
         bind(MonitorService.class).toInstance(monitorService);
         bind(NodeService.class).asEagerSingleton();
diff --git a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
index 26e021bf031..613445c2271 100644
--- a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
+++ b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
@@ -74,7 +74,6 @@ public class MockBigArrays extends BigArrays {
     private final PageCacheRecycler recycler;
     private final CircuitBreakerService breakerService;
 
-    @Inject
     public MockBigArrays(Settings settings, CircuitBreakerService breakerService) {
         this(new MockPageCacheRecycler(settings), breakerService, false);
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java
index 749f1f3279e..33077d5ffbd 100644
--- a/test/framework/src/main/java/org/elasticsearch/node/MockNode.java
+++ b/test/framework/src/main/java/org/elasticsearch/node/MockNode.java
@@ -19,8 +19,10 @@
 
 package org.elasticsearch.node;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.MockBigArrays;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
 import org.elasticsearch.plugins.Plugin;
 
@@ -35,15 +37,25 @@ import java.util.Collection;
  */
 public class MockNode extends Node {
 
+    private final boolean mockBigArrays;
     private Collection<Class<? extends Plugin>> plugins;
 
     public MockNode(Settings settings, Collection<Class<? extends Plugin>> classpathPlugins) {
         super(InternalSettingsPreparer.prepareEnvironment(settings, null), classpathPlugins);
         this.plugins = classpathPlugins;
+        this.mockBigArrays = classpathPlugins.contains(NodeMocksPlugin.class); // if this plugin is present we mock bigarrays :)
     }
 
     public Collection<Class<? extends Plugin>> getPlugins() {
         return plugins;
     }
 
+    @Override
+    protected BigArrays createBigArrays(Settings settings, CircuitBreakerService circuitBreakerService) {
+        if (mockBigArrays) {
+            return new MockBigArrays(settings, circuitBreakerService);
+        } else {
+            return super.createBigArrays(settings, circuitBreakerService);
+        }
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/node/NodeMocksPlugin.java b/test/framework/src/main/java/org/elasticsearch/node/NodeMocksPlugin.java
index a121ad46bcc..2ca1ac7804d 100644
--- a/test/framework/src/main/java/org/elasticsearch/node/NodeMocksPlugin.java
+++ b/test/framework/src/main/java/org/elasticsearch/node/NodeMocksPlugin.java
@@ -18,12 +18,7 @@
  */
 package org.elasticsearch.node;
 
-import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.plugins.Plugin;
 
-public class NodeMocksPlugin extends Plugin {
-
-    public void onModule(NodeModule module) {
-        module.bigArraysImpl = MockBigArrays.class;
-    }
+public class NodeMocksPlugin extends Plugin { // just a marker plugin for MockNode to mock out BigArrays
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index 950653cba5e..025740dbe36 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -902,9 +902,12 @@ public final class InternalTestCluster extends TestCluster {
 
         @Override
         public void close() throws IOException {
-            resetClient();
-            closed.set(true);
-            closeNode();
+            try {
+                resetClient();
+            } finally {
+                closed.set(true);
+                closeNode();
+            }
         }
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/NodeConfigurationSource.java b/test/framework/src/main/java/org/elasticsearch/test/NodeConfigurationSource.java
index c54a00b174d..e04e840e525 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/NodeConfigurationSource.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/NodeConfigurationSource.java
@@ -19,18 +19,10 @@
 package org.elasticsearch.test;
 
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.MockEngineFactoryPlugin;
-import org.elasticsearch.node.NodeMocksPlugin;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.search.MockSearchService;
-import org.elasticsearch.test.store.MockFSIndexStore;
-import org.elasticsearch.test.transport.AssertingLocalTransport;
-import org.elasticsearch.test.transport.MockTransportService;
 
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.List;
 
 public abstract class NodeConfigurationSource {
 

From 93415d45061ca35042c21c45a1593cca39adb8d1 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Wed, 15 Jun 2016 09:07:18 +0200
Subject: [PATCH 0963/1311] Expose MMapDirectory.preLoad(). #18880

The MMapDirectory has a switch that allows the content of files to be loaded
into the filesystem cache upon opening. This commit exposes it with the new
`index.store.pre_load` setting.
---
 .../common/settings/IndexScopedSettings.java  |  1 +
 .../org/elasticsearch/index/IndexModule.java  |  9 +++
 .../index/store/FsDirectoryService.java       | 58 +++++++-------
 .../index/store/FsDirectoryServiceTests.java  | 44 ++++++++++-
 .../index/store/IndexStoreTests.java          | 14 +---
 docs/reference/index-modules/store.asciidoc   | 79 ++++++++++++++++---
 6 files changed, 148 insertions(+), 57 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
index fb60453d467..bbbbe57618b 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
@@ -134,6 +134,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING,
         BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING,
         IndexModule.INDEX_STORE_TYPE_SETTING,
+        IndexModule.INDEX_STORE_PRE_LOAD_SETTING,
         IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING,
         IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING,
         PrimaryShardAllocator.INDEX_RECOVERY_INITIAL_SHARDS_SETTING,
diff --git a/core/src/main/java/org/elasticsearch/index/IndexModule.java b/core/src/main/java/org/elasticsearch/index/IndexModule.java
index 6ceabd1146d..d233faf4f19 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexModule.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexModule.java
@@ -45,6 +45,7 @@ import org.elasticsearch.indices.mapper.MapperRegistry;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -74,6 +75,14 @@ public final class IndexModule {
 
     public static final Setting<String> INDEX_STORE_TYPE_SETTING =
         new Setting<>("index.store.type", "", Function.identity(), Property.IndexScope, Property.NodeScope);
+
+    /** On which extensions to load data into the file-system cache upon opening of files.
+     *  This only works with the mmap directory, and even in that case is still
+     *  best-effort only. */
+    public static final Setting<List<String>> INDEX_STORE_PRE_LOAD_SETTING =
+            Setting.listSetting("index.store.preload", Collections.emptyList(), Function.identity(),
+                    Property.IndexScope, Property.NodeScope);
+
     public static final String SIMILARITY_SETTINGS_PREFIX = "index.similarity";
 
     // whether to use the query cache
diff --git a/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java b/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java
index 5ad5c232f2e..69eedd7ef19 100644
--- a/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java
+++ b/core/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java
@@ -31,13 +31,11 @@ import org.apache.lucene.store.SimpleFSDirectory;
 import org.apache.lucene.store.SimpleFSLockFactory;
 import org.apache.lucene.store.SleepingLockWrapper;
 import org.apache.lucene.store.StoreRateLimiting;
-import org.apache.lucene.util.Constants;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.metrics.CounterMetric;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
-import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.ShardPath;
@@ -45,7 +43,7 @@ import org.elasticsearch.index.shard.ShardPath;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Collections;
+import java.util.HashSet;
 import java.util.Set;
 
 /**
@@ -87,8 +85,12 @@ public class FsDirectoryService extends DirectoryService implements StoreRateLim
     @Override
     public Directory newDirectory() throws IOException {
         final Path location = path.resolveIndex();
+        final LockFactory lockFactory = indexSettings.getValue(INDEX_LOCK_FACTOR_SETTING);
         Files.createDirectories(location);
-        Directory wrapped = newFSDirectory(location, indexSettings.getValue(INDEX_LOCK_FACTOR_SETTING));
+        Directory wrapped = newFSDirectory(location, lockFactory);
+        Set<String> preLoadExtensions = new HashSet<>(
+                indexSettings.getValue(IndexModule.INDEX_STORE_PRE_LOAD_SETTING));
+        wrapped = setPreload(wrapped, location, lockFactory, preLoadExtensions);
         if (IndexMetaData.isOnSharedFilesystem(indexSettings.getSettings())) {
             wrapped = new SleepingLockWrapper(wrapped, 5000);
         }
@@ -100,25 +102,11 @@ public class FsDirectoryService extends DirectoryService implements StoreRateLim
         rateLimitingTimeInNanos.inc(nanos);
     }
 
-    /*
-    * We are mmapping norms, docvalues as well as term dictionaries, all other files are served through NIOFS
-    * this provides good random access performance while not creating unnecessary mmaps for files like stored
-    * fields etc.
-    */
-    private static final Set<String> PRIMARY_EXTENSIONS = Collections.unmodifiableSet(Sets.newHashSet("nvd", "dvd", "tim"));
-
-
     protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
         final String storeType = indexSettings.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(),
             IndexModule.Type.FS.getSettingsKey());
-        if (IndexModule.Type.FS.match(storeType) || isDefault(storeType)) {
-            final FSDirectory open = FSDirectory.open(location, lockFactory); // use lucene defaults
-            if (open instanceof MMapDirectory
-                    && isDefault(storeType)
-                    && Constants.WINDOWS == false) {
-                return newDefaultDir(location, (MMapDirectory) open, lockFactory);
-            }
-            return open;
+        if (IndexModule.Type.FS.match(storeType) || IndexModule.Type.DEFAULT.match(storeType)) {
+            return FSDirectory.open(location, lockFactory); // use lucene defaults
         } else if (IndexModule.Type.SIMPLEFS.match(storeType)) {
             return new SimpleFSDirectory(location, lockFactory);
         } else if (IndexModule.Type.NIOFS.match(storeType)) {
@@ -129,17 +117,25 @@ public class FsDirectoryService extends DirectoryService implements StoreRateLim
         throw new IllegalArgumentException("No directory found for type [" + storeType + "]");
     }
 
-    private static boolean isDefault(String storeType) {
-        return IndexModule.Type.DEFAULT.match(storeType);
-    }
-
-    private Directory newDefaultDir(Path location, final MMapDirectory mmapDir, LockFactory lockFactory) throws IOException {
-        return new FileSwitchDirectory(PRIMARY_EXTENSIONS, mmapDir, new NIOFSDirectory(location, lockFactory), true) {
-            @Override
-            public String[] listAll() throws IOException {
-                // Avoid doing listAll twice:
-                return mmapDir.listAll();
+    private static Directory setPreload(Directory directory, Path location, LockFactory lockFactory,
+            Set<String> preLoadExtensions) throws IOException {
+        if (preLoadExtensions.isEmpty() == false
+                && directory instanceof MMapDirectory
+                && ((MMapDirectory) directory).getPreload() == false) {
+            if (preLoadExtensions.contains("*")) {
+                ((MMapDirectory) directory).setPreload(true);
+                return directory;
             }
-        };
+            MMapDirectory primary = new MMapDirectory(location, lockFactory);
+            primary.setPreload(true);
+            return new FileSwitchDirectory(preLoadExtensions, primary, directory, true) {
+                @Override
+                public String[] listAll() throws IOException {
+                    // avoid listing twice
+                    return primary.listAll();
+                }
+            };
+        }
+        return directory;
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/store/FsDirectoryServiceTests.java b/core/src/test/java/org/elasticsearch/index/store/FsDirectoryServiceTests.java
index f7d793f03ed..6042d471717 100644
--- a/core/src/test/java/org/elasticsearch/index/store/FsDirectoryServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/index/store/FsDirectoryServiceTests.java
@@ -19,13 +19,13 @@
 package org.elasticsearch.index.store;
 
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FilterDirectory;
+import org.apache.lucene.store.FileSwitchDirectory;
+import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.store.RateLimitedFSDirectory;
 import org.apache.lucene.store.SimpleFSDirectory;
 import org.apache.lucene.store.SleepingLockWrapper;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.ShardId;
@@ -36,6 +36,7 @@ import org.elasticsearch.test.IndexSettingsModule;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Arrays;
 
 public class FsDirectoryServiceTests extends ESTestCase {
 
@@ -73,4 +74,43 @@ public class FsDirectoryServiceTests extends ESTestCase {
         assertFalse(delegate instanceof SleepingLockWrapper);
         assertTrue(delegate instanceof SimpleFSDirectory);
     }
+
+    public void testPreload() throws IOException {
+        doTestPreload();
+        doTestPreload("nvd", "dvd", "tim");
+        doTestPreload("*");
+    }
+
+    private void doTestPreload(String...preload) throws IOException {
+        Settings build = Settings.builder()
+                .put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "mmapfs")
+                .putArray(IndexModule.INDEX_STORE_PRE_LOAD_SETTING.getKey(), preload)
+                .build();
+        IndexSettings settings = IndexSettingsModule.newIndexSettings("foo", build);
+        IndexStoreConfig config = new IndexStoreConfig(settings.getSettings());
+        IndexStore store = new IndexStore(settings, config);
+        Path tempDir = createTempDir().resolve(settings.getUUID()).resolve("0");
+        Files.createDirectories(tempDir);
+        ShardPath path = new ShardPath(false, tempDir, tempDir, new ShardId(settings.getIndex(), 0));
+        FsDirectoryService fsDirectoryService = new FsDirectoryService(settings, store, path);
+        Directory directory = fsDirectoryService.newDirectory();
+        assertTrue(directory instanceof RateLimitedFSDirectory);
+        RateLimitedFSDirectory rateLimitingDirectory = (RateLimitedFSDirectory) directory;
+        Directory delegate = rateLimitingDirectory.getDelegate();
+        assertFalse(delegate instanceof SleepingLockWrapper);
+        if (preload.length == 0) {
+            assertTrue(delegate.toString(), delegate instanceof MMapDirectory);
+            assertFalse(((MMapDirectory) delegate).getPreload());
+        } else if (Arrays.asList(preload).contains("*")) {
+            assertTrue(delegate.toString(), delegate instanceof MMapDirectory);
+            assertTrue(((MMapDirectory) delegate).getPreload());
+        } else {
+            assertTrue(delegate.toString(), delegate instanceof FileSwitchDirectory);
+            FileSwitchDirectory fsd = (FileSwitchDirectory) delegate;
+            assertTrue(fsd.getPrimaryDir() instanceof MMapDirectory);
+            assertTrue(((MMapDirectory) fsd.getPrimaryDir()).getPreload());
+            assertTrue(fsd.getSecondaryDir() instanceof MMapDirectory);
+            assertFalse(((MMapDirectory) fsd.getSecondaryDir()).getPreload());
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java b/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java
index 0e012f0d1a7..fe5410ccaf8 100644
--- a/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java
+++ b/core/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java
@@ -77,6 +77,7 @@ public class IndexStoreTests extends ESTestCase {
                     assertTrue(type + " " + directory.toString(), directory instanceof SimpleFSDirectory);
                     break;
                 case FS:
+                case DEFAULT:
                     if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
                         assertTrue(directory.toString(), directory instanceof MMapDirectory);
                     } else if (Constants.WINDOWS) {
@@ -85,19 +86,6 @@ public class IndexStoreTests extends ESTestCase {
                         assertTrue(directory.toString(), directory instanceof NIOFSDirectory);
                     }
                     break;
-                case DEFAULT:
-                   if (Constants.WINDOWS) {
-                        if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
-                            assertTrue(type + " " + directory.toString(), directory instanceof MMapDirectory);
-                        } else {
-                            assertTrue(type + " " + directory.toString(), directory instanceof SimpleFSDirectory);
-                        }
-                    }  else if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
-                        assertTrue(type + " " + directory.toString(), directory instanceof FileSwitchDirectory);
-                    } else  {
-                        assertTrue(type + " " + directory.toString(), directory instanceof NIOFSDirectory);
-                    }
-                    break;
                 default:
                     fail();
             }
diff --git a/docs/reference/index-modules/store.asciidoc b/docs/reference/index-modules/store.asciidoc
index 54080e365c2..ee7fd8766fd 100644
--- a/docs/reference/index-modules/store.asciidoc
+++ b/docs/reference/index-modules/store.asciidoc
@@ -7,9 +7,9 @@ The store module allows you to control how index data is stored and accessed on
 [[file-system]]
 === File system storage types
 
-There are different file system implementations or _storage types_. The best
-one for the operating environment will be automatically chosen: `simplefs` on
-Windows 32bit, `niofs` on other 32bit systems and `mmapfs` on 64bit systems.
+There are different file system implementations or _storage types_. By default,
+elasticsearch will pick the best implementation based on the operating
+environment.
 
 This can be overridden for all indices by adding this to the
 `config/elasticsearch.yml` file:
@@ -36,6 +36,12 @@ experimental[This is an expert-only setting and may be removed in the future]
 
 The following sections lists all the different storage types supported.
 
+`fs`::
+
+Default file system implementation. This will pick the best implementation
+depending on the operating environment: `simplefs` on Windows 32bit, `niofs`
+on other 32bit systems and `mmapfs` on 64bit systems.
+
 [[simplefs]]`simplefs`::
 
 The Simple FS type is a straightforward implementation of file system
@@ -60,13 +66,64 @@ process equal to the size of the file being mapped. Before using this
 class, be sure you have allowed plenty of
 <<vm-max-map-count,virtual address space>>.
 
-[[default_fs]]`default_fs` deprecated[5.0.0, The `default_fs` store type is deprecated - use `mmapfs` instead]::
+[[default_fs]]`default_fs` deprecated[5.0.0, The `default_fs` store type is deprecated - use `fs` instead]::
 
-The `default` type is a hybrid of NIO FS and MMapFS, which chooses the best
-file system for each type of file. Currently only the Lucene term dictionary,
-doc values and points files are memory mapped to reduce the impact on the
-operating system. All other files are opened using Lucene `NIOFSDirectory`.
-Address space settings (<<vm-max-map-count>>) might also apply if your term
-dictionary are large, if you index many fields that use points (numerics, dates
-and ip addresses) or if you have many fields with doc values.
+The `default` type is deprecated and is aliased to `fs` for backward
+compatibility.
 
+=== Pre-loading data into the file system cache
+
+experimental[This is an expert-only setting and may be removed in the future]
+
+By default, elasticsearch completely relies on the operating system file system
+cache for caching I/O operations. It is possible to set `index.store.preload`
+in order to tell the operating system to load the content of hot index
+files into memory upon opening. This setting accept a comma-separated list of
+files extensions: all files whose extenion is in the list will be pre-loaded
+upon opening. This can be useful to improve search performance of an index,
+especially when the host operating system is restarted, since this causes the
+file system cache to be trashed. However note that this may slow down the
+opening of indices, as they will only become available after data have been
+loaded into physical memory.
+
+This setting is best-effort only and may not work at all depending on the store
+type and host operating system.
+
+The `index.store.pre_load` is a static setting that can either be set in the
+`config/elasticsearch.yml`:
+
+[source,yaml]
+---------------------------------
+index.store.pre_load: ["nvd", "dvd"]
+---------------------------------
+
+or in the index settings at index creation time:
+
+[source,js]
+---------------------------------
+PUT /my_index
+{
+  "settings": {
+    "index.store.pre_load": ["nvd", "dvd"]
+  }
+}
+---------------------------------
+
+The default value is the empty array, which means that nothing will be loaded
+into the file-system cache eagerly. For indices that are actively searched,
+you might want to set it to `["nvd", "dvd"]`, which will cause norms and doc
+values to be loaded eagerly into physical memory. These are the two first
+extensions to look at since elasticsearch performs random access on them.
+
+A wildcard can be used in order to indicate that all files should be preloaded:
+`index.store.pre_load: ["*"]`. Note however that it is generally not useful to
+load all files into memory, in particular those for stored fields and term
+vectors, so a better option might be to set it to
+`["nvd", "dvd", "tim", "doc", "dim"]`, which will preload norms, doc values,
+terms dictionaries, postings lists and points, which are the most important
+parts of the index for search and aggregations.
+
+Note that this setting can be dangerous on indices that are larger than the size
+of the main memory of the host, as it would cause the filesystem cache to be
+trashed upon reopens after large merges, which would make indexing and searching
+_slower_.

From cd1a7b441cc5cef5cb3c66026a5f5ea6e74c8fa0 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 20 Jun 2016 07:57:00 -0400
Subject: [PATCH 0964/1311] Improve error messages for lambdas when the number
 of arguments is wrong

---
 .../main/java/org/elasticsearch/painless/Def.java  |  6 ++++++
 .../org/elasticsearch/painless/FunctionRef.java    |  7 ++++++-
 .../elasticsearch/painless/FunctionRefTests.java   | 14 ++++++++++++++
 .../org/elasticsearch/painless/LambdaTests.java    | 14 ++++++++++++++
 4 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index e575ed32183..8014a807b3e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -341,6 +341,12 @@ public final class Def {
                                                                  MethodHandle.class);
                  handle = (MethodHandle) accessor.invokeExact();
              } catch (NoSuchFieldException | IllegalAccessException e) {
+                 // is it a synthetic method? If we generated the method ourselves, be more helpful. It can only fail
+                 // because the arity does not match the expected interface type.
+                 if (call.contains("$")) {
+                     throw new IllegalArgumentException("Incorrect number of parameters for [" + interfaceMethod.name + 
+                                                        "] in [" + clazz.clazz + "]");
+                 }
                  throw new IllegalArgumentException("Unknown call [" + call + "] with [" + arity + "] arguments.");
              }
              ref = new FunctionRef(clazz, interfaceMethod, handle, captures);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
index 6a39d99f4e3..72676f1c04c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
@@ -176,7 +176,12 @@ public class FunctionRef {
      * If the interface expects a primitive type to be returned, we can't return Object,
      * But we can set SAM to the wrapper version, and a cast will take place 
      */
-    private static MethodType adapt(MethodType expected, MethodType actual) {
+    private MethodType adapt(MethodType expected, MethodType actual) {
+        // add some checks, now that we've set everything up, to deliver exceptions as early as possible.
+        if (expected.parameterCount() != actual.parameterCount()) {
+            throw new IllegalArgumentException("Incorrect number of parameters for [" + invokedName + 
+                                               "] in [" + invokedType.returnType() + "]");
+        }
         if (expected.returnType().isPrimitive() && actual.returnType() == Object.class) {
             actual = actual.changeReturnType(MethodType.methodType(expected.returnType()).wrap().returnType());
         }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
index 8ed2f3261cf..46d2b7c43fd 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
@@ -162,4 +162,18 @@ public class FunctionRefTests extends ScriptTestCase {
             exec("List l = new ArrayList(); l.add(2); l.add(1); l.sort(String::startsWith); return l.get(0);");
         });
     }
+    
+    public void testWrongArity() {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("Optional.empty().orElseGet(String::startsWith);");
+        });
+        assertTrue(expected.getMessage().contains("Unknown reference"));
+    }
+    
+    public void testWrongArityDef() {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("def y = Optional.empty(); return y.orElseGet(String::startsWith);");
+        });
+        assertTrue(expected.getMessage().contains("Unknown reference"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
index 3b008d5d6b6..01aeb440694 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
@@ -142,4 +142,18 @@ public class LambdaTests extends ScriptTestCase {
         assertEquals(1, exec("boolean x = false; int y = 1;" +
                              "return Optional.empty().orElseGet(() -> x ? 5 : Optional.empty().orElseGet(() -> y));"));
     }
+    
+    public void testWrongArity() {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("Optional.empty().orElseGet(x -> x);");
+        });
+        assertTrue(expected.getMessage().contains("Incorrect number of parameters"));
+    }
+    
+    public void testWrongArityDef() {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("def y = Optional.empty(); return y.orElseGet(x -> x);");
+        });
+        assertTrue(expected.getMessage(), expected.getMessage().contains("Incorrect number of parameters"));
+    }
 }

From 9111ed3e2cdfe6ac29bee1340b07400699c2db67 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 20 Jun 2016 08:24:41 -0400
Subject: [PATCH 0965/1311] add docs

---
 .../elasticsearch/painless/node/ELambda.java  | 23 +++++++++++++++++++
 .../elasticsearch/painless/LambdaTests.java   | 18 +++++++++++++++
 2 files changed, 41 insertions(+)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
index 8adae6607a9..1d1498a51c4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
@@ -41,6 +41,29 @@ import java.util.Set;
 
 import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE;
 
+/**
+ * Lambda expression node.
+ * <p>
+ * This can currently only be the direct argument of a call (method/constructor).
+ * When the argument is of a known type, it uses
+ * <a href="http://cr.openjdk.java.net/~briangoetz/lambda/lambda-translation.html">
+ * Java's lambda translation</a>. However, if its a def call, then we don't have 
+ * enough information, and have to defer this until link time. In that case a placeholder
+ * and all captures are pushed onto the stack and folded into the signature of the parent call.
+ * <p>
+ * For example:
+ * <br>
+ * {@code def list = new ArrayList(); int capture = 0; list.sort((x,y) -> x - y + capture)}
+ * <br>
+ * is converted into a call (pseudocode) such as:
+ * <br>
+ * {@code sort(list, lambda$0, capture)}
+ * <br>
+ * At link time, when we know the interface type, this is decomposed with MethodHandle
+ * combinators back into (pseudocode):
+ * <br>
+ * {@code sort(list, lambda$0(capture))}
+ */
 public class ELambda extends AExpression implements ILambda {
     final String name;
     final FunctionReserved reserved;
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
index 01aeb440694..c321b010ecf 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
@@ -85,6 +85,24 @@ public class LambdaTests extends ScriptTestCase {
     public void testUnneededCurlyStatements() {
         assertEquals(2, exec("int applyOne(IntFunction arg) { arg.apply(1) } applyOne(x -> { x + 1 })"));
     }
+    
+    /** interface ignores return value */
+    public void testVoidReturn() {
+        assertEquals(2, exec("List list = new ArrayList(); "
+                           + "list.add(2); "
+                           + "List list2 = new ArrayList(); "
+                           + "list.forEach(x -> list2.add(x));"
+                           + "return list[0]"));
+    }
+    
+    /** interface ignores return value */
+    public void testVoidReturnDef() {
+        assertEquals(2, exec("def list = new ArrayList(); "
+                           + "list.add(2); "
+                           + "List list2 = new ArrayList(); "
+                           + "list.forEach(x -> list2.add(x));"
+                           + "return list[0]"));
+    }
 
     public void testTwoLambdas() {
         assertEquals("testingcdefg", exec(

From 28b1b149abe54909336c41b22bc7a208c54bd763 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 20 Jun 2016 08:32:53 -0400
Subject: [PATCH 0966/1311] remove unused import

---
 .../src/main/java/org/elasticsearch/painless/DefMath.java        | 1 -
 1 file changed, 1 deletion(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java
index 65a26cd3da8..f89b1429f38 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java
@@ -25,7 +25,6 @@ import java.lang.invoke.MethodType;
 import java.lang.invoke.MethodHandles.Lookup;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.IdentityHashMap;
 import java.util.Map;
 import java.util.function.Function;
 import java.util.stream.Collectors;

From 5746d96f42b82797051b59a461fe4d5b2ef8b238 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Mon, 20 Jun 2016 14:37:13 +0200
Subject: [PATCH 0967/1311] Catch ClosedByInterruptException when interrupting
 check index

---
 .../main/java/org/elasticsearch/index/shard/IndexShard.java   | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 9ebbde77c6a..f20f2f5d372 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -1286,6 +1286,10 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         if (store.tryIncRef()) {
             try {
                 cancellableThreads.executeIO(this::doCheckIndex);
+            } catch (ClosedByInterruptException ex) {
+                assert cancellableThreads.isCancelled();
+                // that's fine we might run into this when we cancel the thread since Java NIO will close the channel on interrupt
+                // and on the next access we fail it.
             } finally {
                 store.decRef();
             }

From 0660de74720b70096334b86bf237666a80a442aa Mon Sep 17 00:00:00 2001
From: Eric Sherman <easherma@users.noreply.github.com>
Date: Mon, 20 Jun 2016 08:59:57 -0500
Subject: [PATCH 0968/1311] Update snapshots.asciidoc (#18923)

---
 docs/reference/modules/snapshots.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc
index d000805e5e9..5931dbc588f 100644
--- a/docs/reference/modules/snapshots.asciidoc
+++ b/docs/reference/modules/snapshots.asciidoc
@@ -8,7 +8,7 @@ backends are available via officially supported repository plugins.
 [float]
 === Repositories
 
-Before any snapshot or restore operation can be performed a snapshot repository should be registered in
+Before any snapshot or restore operation can be performed, a snapshot repository should be registered in
 Elasticsearch. The repository settings are repository-type specific. See below for details.
 
 [source,js]

From 9510a8f39e405b98be102b81d00fe754d326ac1e Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 20 Jun 2016 10:46:23 -0400
Subject: [PATCH 0969/1311] add a few more tests

---
 .../org/elasticsearch/painless/LambdaTests.java     | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
index c321b010ecf..7b2d5e6a935 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
@@ -161,6 +161,11 @@ public class LambdaTests extends ScriptTestCase {
                              "return Optional.empty().orElseGet(() -> x ? 5 : Optional.empty().orElseGet(() -> y));"));
     }
     
+    public void testNestedCaptureParams() {
+        assertEquals(2, exec("int foo(Function f) { return f.apply(1) }" +
+                             "return foo(x -> foo(y -> x + 1))"));
+    }
+    
     public void testWrongArity() {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("Optional.empty().orElseGet(x -> x);");
@@ -174,4 +179,12 @@ public class LambdaTests extends ScriptTestCase {
         });
         assertTrue(expected.getMessage(), expected.getMessage().contains("Incorrect number of parameters"));
     }
+    
+    public void testLambdaInFunction() {
+        assertEquals(5, exec("def foo() { Optional.empty().orElseGet(() -> 5) } return foo();"));
+    }
+    
+    public void testLambdaCaptureFunctionParam() {
+        assertEquals(5, exec("def foo(int x) { Optional.empty().orElseGet(() -> x) } return foo(5);"));
+    }
 }

From 9196ff255e3b4631ce94b143b83c2141d702166d Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Mon, 20 Jun 2016 16:46:03 +0200
Subject: [PATCH 0970/1311] Add extra ctor to FilterClient to support Guice
 proxies

just don't ask it's bad but some plugins are so involved they need this.

Closes #the_issue_that_never_existed
---
 .../java/org/elasticsearch/client/FilterClient.java  | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/client/FilterClient.java b/core/src/main/java/org/elasticsearch/client/FilterClient.java
index d2ea209a8c2..d0f52282c76 100644
--- a/core/src/main/java/org/elasticsearch/client/FilterClient.java
+++ b/core/src/main/java/org/elasticsearch/client/FilterClient.java
@@ -24,6 +24,8 @@ import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.client.support.AbstractClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.threadpool.ThreadPool;
 
 
 /**
@@ -42,7 +44,15 @@ public abstract class FilterClient extends AbstractClient {
      * @see #in()
      */
     public FilterClient(Client in) {
-        super(in.settings(), in.threadPool());
+        this(in.settings(), in.threadPool(), in);
+    }
+
+    /**
+     * A Constructor that allows to pass settings and threadpool separately. This is useful if the
+     * client is a proxy and not yet fully constructed ie. both dependencies are not available yet.
+     */
+    protected FilterClient(Settings settings, ThreadPool threadPool, Client in) {
+        super(settings, threadPool);
         this.in = in;
     }
 

From 6569d350944efbd91ec08959b23485caa2ae24b3 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 20 Jun 2016 11:49:17 -0400
Subject: [PATCH 0971/1311] Fail doc tests when any shard fails

ES only sends a non-200 response all shards fail but we should
fail the tests generated by docs if any of them fail.

Depending on the outcome of #18978 this might be a temporary
workaround.
---
 .../doc/RestTestsFromSnippetsTask.groovy      | 23 +++++++++++++++----
 .../modules/scripting/painless.asciidoc       |  2 +-
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
index c9f5668a326..ba7311fee6f 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
@@ -127,7 +127,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
                 current.println(setup)
             }
 
-            body(test)
+            body(test, false)
         }
 
         private void response(Snippet response) {
@@ -136,7 +136,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
         }
 
         void emitDo(String method, String pathAndQuery,
-                String body, String catchPart) {
+                String body, String catchPart, boolean inSetup) {
             def (String path, String query) = pathAndQuery.tokenize('?')
             current.println("  - do:")
             if (catchPart != null) {
@@ -160,6 +160,19 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
                 current.println("        body: |")
                 body.eachLine { current.println("          $it") }
             }
+            /* Catch any shard failures. These only cause a non-200 response if
+             * no shard succeeds. But we need to fail the tests on all of these
+             * because they mean invalid syntax or broken queries or something
+             * else that we don't want to teach people to do. The REST test
+             * framework doesn't allow us to has assertions in the setup
+             * section so we have to skip it there. We also have to skip _cat
+             * actions because they don't return json so we can't is_false
+             * them. That is ok because they don't have this
+             * partial-success-is-success thing.
+             */
+            if (false == inSetup && false == path.startsWith('_cat')) {
+                current.println("  - is_false: _shards.failures")
+            }
         }
 
         private void setup(Snippet setup) {
@@ -169,7 +182,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
             setupCurrent(setup)
             current.println('---')
             current.println("setup:")
-            body(setup)
+            body(setup, true)
             // always wait for yellow before anything is executed
             current.println(
                     "  - do:\n" +
@@ -179,7 +192,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
                     "        wait_for_status: \"yellow\"")
         }
 
-        private void body(Snippet snippet) {
+        private void body(Snippet snippet, boolean inSetup) {
             parse("$snippet", snippet.contents, SYNTAX) { matcher, last ->
                 if (matcher.group("comment") != null) {
                     // Comment
@@ -193,7 +206,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
                     // Leading '/'s break the generated paths
                     pathAndQuery = pathAndQuery.substring(1)
                 }
-                emitDo(method, pathAndQuery, body, catchPart)
+                emitDo(method, pathAndQuery, body, catchPart, inSetup)
             }
         }
 
diff --git a/docs/reference/modules/scripting/painless.asciidoc b/docs/reference/modules/scripting/painless.asciidoc
index 20b301bda31..2c753e09039 100644
--- a/docs/reference/modules/scripting/painless.asciidoc
+++ b/docs/reference/modules/scripting/painless.asciidoc
@@ -133,7 +133,7 @@ GET hockey/_search
       "order": "asc",
       "script": {
         "lang": "painless",
-        "inline": "doc['first'].value + ' ' + doc['last'].value"
+        "inline": "doc['first.keyword'].value + ' ' + doc['last.keyword'].value"
       }
     }
   }

From cb8073e990b91264ca218e7ded825debb8d861ce Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Mon, 20 Jun 2016 18:58:41 +0200
Subject: [PATCH 0972/1311] Wrong name for values field

We wrote that the document is:

```json
{
  "value" : ["foo", "bar", "baz"]
}
```

But the processor is using a `values` field:

```json
{
  "foreach" : {
    "field" : "values",
    "processors" : [
      // ...
    ]
  }
}
```

It should be `values`.
---
 docs/reference/ingest/ingest-node.asciidoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc
index c8a5a05002d..722b58302b5 100644
--- a/docs/reference/ingest/ingest-node.asciidoc
+++ b/docs/reference/ingest/ingest-node.asciidoc
@@ -882,7 +882,7 @@ Assume the following document:
 [source,js]
 --------------------------------------------------
 {
-  "value" : ["foo", "bar", "baz"]
+  "values" : ["foo", "bar", "baz"]
 }
 --------------------------------------------------
 
@@ -909,7 +909,7 @@ Then the document will look like this after preprocessing:
 [source,js]
 --------------------------------------------------
 {
-  "value" : ["FOO", "BAR", "BAZ"]
+  "values" : ["FOO", "BAR", "BAZ"]
 }
 --------------------------------------------------
 

From 26a73c39bf4ff74709b3f32bb0f96070364fe372 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 20 Jun 2016 13:01:59 -0400
Subject: [PATCH 0973/1311] throw error if the tree is totally malformed

---
 .../java/org/elasticsearch/painless/node/SFunction.java   | 8 ++------
 .../java/org/elasticsearch/painless/node/SSource.java     | 8 ++------
 2 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index 629f76cdd52..d15cd07345f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -80,12 +80,8 @@ public class SFunction extends AStatement {
     
     @Override
     void extractVariables(Set<String> variables) {
-        for (String paramName : paramNameStrs) {
-            variables.add(paramName);
-        }
-        for (AStatement statement : statements) {
-            statement.extractVariables(variables);
-        }
+        // we should never be extracting from a function, as functions are top-level!
+        throw new IllegalStateException("Illegal tree structure");
     }
 
     void generate() {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index 1a47cfeba36..22ad56c1be8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -87,12 +87,8 @@ public final class SSource extends AStatement {
     
     @Override
     void extractVariables(Set<String> variables) {
-        for (AStatement statement : statements) {
-            statement.extractVariables(variables);
-        }
-        for (SFunction function : functions) {
-            function.extractVariables(variables);
-        }
+        // we should never be extracting from a function, as functions are top-level!
+        throw new IllegalStateException("Illegal tree structure");
     }
 
     public void analyze() {

From 006829e89bf2bcf8e50f9e947fef7aa8626b6f01 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 20 Jun 2016 13:33:51 -0400
Subject: [PATCH 0974/1311] add simple arguments test

---
 .../test/java/org/elasticsearch/painless/CastTests.java  | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/CastTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/CastTests.java
index fcee0ddeb3e..0551965f067 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/CastTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/CastTests.java
@@ -162,6 +162,15 @@ public class CastTests extends ScriptTestCase {
         assertEquals(5, exec("def x = 5; return (int)x.longValue();"));
     }
     
+    /**
+     * Currently these do not adopt the argument value, we issue a separate cast!
+     */
+    public void testArgumentsDef() {
+        assertEquals(5, exec("def x = 5L; return (+(int)x);"));
+        assertEquals(6, exec("def x = 5; def y = 1L; return x + (int)y"));
+        assertEquals('b', exec("def x = 'abcdeg'; def y = 1L; x.charAt((int)y)"));
+    }
+    
     /**
      * Unary operators adopt the return value
      */

From ebc3c17c34c170a72cca528fdee6acedcab73a7e Mon Sep 17 00:00:00 2001
From: Mike McCandless <mikemccand@apache.org>
Date: Mon, 20 Jun 2016 14:20:23 -0400
Subject: [PATCH 0975/1311] add indices flag to nodes info request; use boolean
 to express 'null' indexing buffer value on the wire

---
 .../admin/cluster/node/info/NodeInfo.java     |  9 ++++++--
 .../cluster/node/info/NodesInfoRequest.java   | 21 +++++++++++++++++++
 .../node/info/NodesInfoRequestBuilder.java    |  8 +++++++
 .../cluster/node/info/NodesInfoResponse.java  |  5 +++--
 .../node/info/TransportNodesInfoAction.java   |  2 +-
 .../stats/TransportClusterStatsAction.java    |  2 +-
 .../node/service/NodeService.java             |  4 ++--
 .../node/info/RestNodesInfoAction.java        |  3 ++-
 .../nodesinfo/SimpleNodesInfoIT.java          | 11 ++++++++++
 9 files changed, 56 insertions(+), 9 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
index cf4f3e812d4..312b4a5a05c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
@@ -208,7 +208,11 @@ public class NodeInfo extends BaseNodeResponse {
         super.readFrom(in);
         version = Version.readVersion(in);
         build = Build.readBuild(in);
-        totalIndexingBuffer = new ByteSizeValue(in.readLong());
+        if (in.readBoolean()) {
+            totalIndexingBuffer = new ByteSizeValue(in.readLong());
+        } else {
+            totalIndexingBuffer = null;
+        }
         if (in.readBoolean()) {
             Map<String, String> builder = new HashMap<>();
             int size = in.readVInt();
@@ -253,8 +257,9 @@ public class NodeInfo extends BaseNodeResponse {
         out.writeVInt(version.id);
         Build.writeBuild(build, out);
         if (totalIndexingBuffer == null) {
-            out.writeLong(0);
+            out.writeBoolean(false);
         } else {
+            out.writeBoolean(true);
             out.writeLong(totalIndexingBuffer.bytes());
         }
         if (getServiceAttributes() == null) {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java
index 66c5cfd65d4..b547d1d7432 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java
@@ -39,6 +39,7 @@ public class NodesInfoRequest extends BaseNodesRequest<NodesInfoRequest> {
     private boolean http = true;
     private boolean plugins = true;
     private boolean ingest = true;
+    private boolean indices = true;
 
     public NodesInfoRequest() {
     }
@@ -64,6 +65,7 @@ public class NodesInfoRequest extends BaseNodesRequest<NodesInfoRequest> {
         http = false;
         plugins = false;
         ingest = false;
+        indices = false;
         return this;
     }
 
@@ -80,6 +82,7 @@ public class NodesInfoRequest extends BaseNodesRequest<NodesInfoRequest> {
         http = true;
         plugins = true;
         ingest = true;
+        indices = true;
         return this;
     }
 
@@ -221,6 +224,22 @@ public class NodesInfoRequest extends BaseNodesRequest<NodesInfoRequest> {
         return ingest;
     }
 
+    /**
+     * Should information about indices (currently just indexing buffers) be returned
+     * @param indices true if you want info
+     */
+    public NodesInfoRequest indices(boolean indices) {
+        this.indices = indices;
+        return this;
+    }
+
+    /**
+     * @return true if information about indices (currently just indexing buffers)
+     */
+    public boolean indices() {
+        return indices;
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
@@ -233,6 +252,7 @@ public class NodesInfoRequest extends BaseNodesRequest<NodesInfoRequest> {
         http = in.readBoolean();
         plugins = in.readBoolean();
         ingest = in.readBoolean();
+        indices = in.readBoolean();
     }
 
     @Override
@@ -247,5 +267,6 @@ public class NodesInfoRequest extends BaseNodesRequest<NodesInfoRequest> {
         out.writeBoolean(http);
         out.writeBoolean(plugins);
         out.writeBoolean(ingest);
+        out.writeBoolean(indices);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java
index fc484012379..16befb79aab 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java
@@ -118,4 +118,12 @@ public class NodesInfoRequestBuilder extends NodesOperationRequestBuilder<NodesI
         request().ingest(ingest);
         return this;
     }
+
+    /**
+     * Should the node indices info be returned.
+     */
+    public NodesInfoRequestBuilder setIndices(boolean indices) {
+        request().indices(indices);
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java
index c42deeac703..a9dc0ff2e45 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java
@@ -69,8 +69,9 @@ public class NodesInfoResponse extends BaseNodesResponse<NodeInfo> implements To
 
             builder.field("version", nodeInfo.getVersion());
             builder.field("build_hash", nodeInfo.getBuild().shortHash());
-            builder.field("total_indexing_buffer_in_bytes", nodeInfo.getTotalIndexingBuffer().bytes());
-            builder.field("total_indexing_buffer", nodeInfo.getTotalIndexingBuffer());
+            if (nodeInfo.getTotalIndexingBuffer() != null) {
+                builder.byteSizeField("total_indexing_buffer", "total_indexing_buffer_in_bytes", nodeInfo.getTotalIndexingBuffer());
+            }
 
             if (nodeInfo.getServiceAttributes() != null) {
                 for (Map.Entry<String, String> nodeAttribute : nodeInfo.getServiceAttributes().entrySet()) {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
index f68e2d65903..857ec494688 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java
@@ -77,7 +77,7 @@ public class TransportNodesInfoAction extends TransportNodesAction<NodesInfoRequ
     protected NodeInfo nodeOperation(NodeInfoRequest nodeRequest) {
         NodesInfoRequest request = nodeRequest.request;
         return nodeService.info(request.settings(), request.os(), request.process(), request.jvm(), request.threadPool(),
-                request.transport(), request.http(), request.plugins(), request.ingest());
+                request.transport(), request.http(), request.plugins(), request.ingest(), request.indices());
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
index 3a0b1455209..c4c6f3e6ecc 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
@@ -92,7 +92,7 @@ public class TransportClusterStatsAction extends TransportNodesAction<ClusterSta
 
     @Override
     protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeRequest) {
-        NodeInfo nodeInfo = nodeService.info(false, true, false, true, false, true, false, true, false);
+        NodeInfo nodeInfo = nodeService.info(false, true, false, true, false, true, false, true, false, true);
         NodeStats nodeStats = nodeService.stats(CommonStatsFlags.NONE, false, true, true, false, true, false, false, false, false, false, false);
         List<ShardStats> shardsStats = new ArrayList<>();
         for (IndexService indexService : indicesService) {
diff --git a/core/src/main/java/org/elasticsearch/node/service/NodeService.java b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
index 8280cb6d6ef..2f056ba1325 100644
--- a/core/src/main/java/org/elasticsearch/node/service/NodeService.java
+++ b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
@@ -141,7 +141,7 @@ public class NodeService extends AbstractComponent implements Closeable {
     }
 
     public NodeInfo info(boolean settings, boolean os, boolean process, boolean jvm, boolean threadPool,
-                         boolean transport, boolean http, boolean plugin, boolean ingest) {
+                         boolean transport, boolean http, boolean plugin, boolean ingest, boolean indices) {
         return new NodeInfo(version, Build.CURRENT, discovery.localNode(), serviceAttributes,
                 settings ? settingsFilter.filter(this.settings) : null,
                 os ? monitorService.osService().info() : null,
@@ -152,7 +152,7 @@ public class NodeService extends AbstractComponent implements Closeable {
                 http ? (httpServer == null ? null : httpServer.info()) : null,
                 plugin ? (pluginService == null ? null : pluginService.info()) : null,
                 ingest ? (ingestService == null ? null : ingestService.info()) : null,
-                indicesService == null ? null : indicesService.getTotalIndexingBufferBytes()
+                indices ? indicesService.getTotalIndexingBufferBytes() : null
         );
     }
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java
index 0f1e8691770..312fd0b0b7a 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java
@@ -43,7 +43,7 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 public class RestNodesInfoAction extends BaseRestHandler {
 
     private final SettingsFilter settingsFilter;
-    private final static Set<String> ALLOWED_METRICS = Sets.newHashSet("http", "jvm", "os", "plugins", "process", "settings", "thread_pool", "transport", "ingest");
+    private final static Set<String> ALLOWED_METRICS = Sets.newHashSet("http", "jvm", "os", "plugins", "process", "settings", "thread_pool", "transport", "ingest", "indices");
 
     @Inject
     public RestNodesInfoAction(Settings settings, RestController controller, Client client, SettingsFilter settingsFilter) {
@@ -97,6 +97,7 @@ public class RestNodesInfoAction extends BaseRestHandler {
             nodesInfoRequest.http(metrics.contains("http"));
             nodesInfoRequest.plugins(metrics.contains("plugins"));
             nodesInfoRequest.ingest(metrics.contains("ingest"));
+            nodesInfoRequest.indices(metrics.contains("indices"));
         }
 
         settingsFilter.addFilterSettingParams(request);
diff --git a/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java b/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java
index 73f0b99ecf4..0cc30f8d569 100644
--- a/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java
+++ b/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java
@@ -102,6 +102,17 @@ public class SimpleNodesInfoIT extends ESIntegTestCase {
         assertThat(response.getNodesMap().get(server2NodeId), notNullValue());
         assertNotNull(response.getNodesMap().get(server2NodeId).getTotalIndexingBuffer());
         assertThat(response.getNodesMap().get(server2NodeId).getTotalIndexingBuffer().bytes(), greaterThan(0L));
+
+        // again, using only the indices flag
+        response = client().admin().cluster().prepareNodesInfo().clear().setIndices(true).execute().actionGet();
+        assertThat(response.getNodes().size(), is(2));
+        assertThat(response.getNodesMap().get(server1NodeId), notNullValue());
+        assertNotNull(response.getNodesMap().get(server1NodeId).getTotalIndexingBuffer());
+        assertThat(response.getNodesMap().get(server1NodeId).getTotalIndexingBuffer().bytes(), greaterThan(0L));
+
+        assertThat(response.getNodesMap().get(server2NodeId), notNullValue());
+        assertNotNull(response.getNodesMap().get(server2NodeId).getTotalIndexingBuffer());
+        assertThat(response.getNodesMap().get(server2NodeId).getTotalIndexingBuffer().bytes(), greaterThan(0L));
     }
 
     public void testAllocatedProcessors() throws Exception {

From eecf094ac1b168962d797164ab4842b16a12a7e4 Mon Sep 17 00:00:00 2001
From: Mike McCandless <mikemccand@apache.org>
Date: Mon, 20 Jun 2016 14:23:32 -0400
Subject: [PATCH 0976/1311] add indices nodes info flag to docs

---
 docs/reference/cluster/nodes-info.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/cluster/nodes-info.asciidoc b/docs/reference/cluster/nodes-info.asciidoc
index 8b505218664..77f77da3c89 100644
--- a/docs/reference/cluster/nodes-info.asciidoc
+++ b/docs/reference/cluster/nodes-info.asciidoc
@@ -51,7 +51,7 @@ By default, it just returns all attributes and core settings for a node:
 	Elasticsearch version running on this node.
 
 It also allows to get only information on `settings`, `os`, `process`, `jvm`,
-`thread_pool`, `transport`, `http`, `plugins` and `ingest`:
+`thread_pool`, `transport`, `http`, `plugins`, `ingest` and `indices`:
 
 [source,js]
 --------------------------------------------------

From 1cc0264827e4766b01b2f781964d68819d5a93d8 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 20 Jun 2016 14:54:45 -0400
Subject: [PATCH 0977/1311] Infer lambda arguments/return type

---
 .../org/elasticsearch/painless/Locals.java    |  5 +-
 .../elasticsearch/painless/node/ELambda.java  | 49 ++++++++++++++++---
 .../painless/node/LCallInvoke.java            | 15 ++++++
 .../elasticsearch/painless/BasicAPITests.java |  7 ++-
 .../painless/DefOptimizationTests.java        | 10 ++++
 5 files changed, 75 insertions(+), 11 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
index 7ffa0a169b7..ae54b6e6271 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
@@ -69,8 +69,9 @@ public final class Locals {
      * <p>
      * This is just like {@link #newFunctionScope}, except the captured parameters are made read-only.
      */
-    public static Locals newLambdaScope(Locals programScope, List<Parameter> parameters, int captureCount, int maxLoopCounter) {
-        Locals locals = new Locals(programScope, Definition.DEF_TYPE);
+    public static Locals newLambdaScope(Locals programScope, Type returnType, List<Parameter> parameters, 
+                                        int captureCount, int maxLoopCounter) {
+        Locals locals = new Locals(programScope, returnType);
         for (int i = 0; i < parameters.size(); i++) {
             Parameter parameter = parameters.get(i);
             boolean isCapture = i < captureCount;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
index 1d1498a51c4..e0ea5e73aeb 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
@@ -100,6 +100,44 @@ public class ELambda extends AExpression implements ILambda {
 
     @Override
     void analyze(Locals locals) {
+        final Definition.Type returnType;
+        final List<String> actualParamTypeStrs;
+        Method interfaceMethod;
+        // inspect the target first, set interface method if we know it.
+        if (expected == null) {
+            interfaceMethod = null;
+            // we don't know anything: treat as def
+            returnType = Definition.DEF_TYPE;
+            // don't infer any types
+            actualParamTypeStrs = paramTypeStrs;
+        } else {
+            // we know the method statically, infer return type and any unknown/def types
+            interfaceMethod = expected.struct.getFunctionalMethod();
+            if (interfaceMethod == null) {
+                throw createError(new IllegalArgumentException("Cannot pass lambda to [" + expected.name + 
+                                                               "], not a functional interface"));
+            }
+            // check arity before we manipulate parameters
+            if (interfaceMethod.arguments.size() != paramTypeStrs.size())
+                throw new IllegalArgumentException("Incorrect number of parameters for [" + interfaceMethod.name + 
+                                                   "] in [" + expected.clazz + "]");
+            // for method invocation, its allowed to ignore the return value
+            if (interfaceMethod.rtn == Definition.VOID_TYPE) {
+                returnType = Definition.DEF_TYPE;
+            } else {
+                returnType = interfaceMethod.rtn;
+            }
+            // replace any def types with the actual type (which could still be def)
+            actualParamTypeStrs = new ArrayList<String>();
+            for (int i = 0; i < paramTypeStrs.size(); i++) {
+                String paramType = paramTypeStrs.get(i);
+                if (paramType.equals(Definition.DEF_TYPE.name)) {
+                    actualParamTypeStrs.add(interfaceMethod.arguments.get(i).name);
+                } else {
+                    actualParamTypeStrs.add(paramType);
+                }
+            }
+        }
         // gather any variables used by the lambda body first.
         Set<String> variables = new HashSet<>();
         for (AStatement statement : statements) {
@@ -119,14 +157,14 @@ public class ELambda extends AExpression implements ILambda {
             paramTypes.add(var.type.name);
             paramNames.add(var.name);
         }
-        paramTypes.addAll(paramTypeStrs);
+        paramTypes.addAll(actualParamTypeStrs);
         paramNames.addAll(paramNameStrs);
         
         // desugar lambda body into a synthetic method
-        desugared = new SFunction(reserved, location, "def", name, 
+        desugared = new SFunction(reserved, location, returnType.name, name, 
                                             paramTypes, paramNames, statements, true);
         desugared.generate();
-        desugared.analyze(Locals.newLambdaScope(locals.getProgramScope(), desugared.parameters, 
+        desugared.analyze(Locals.newLambdaScope(locals.getProgramScope(), returnType, desugared.parameters, 
                                                 captures.size(), reserved.getMaxLoopCounter()));
         
         // setup method reference to synthetic method
@@ -137,11 +175,6 @@ public class ELambda extends AExpression implements ILambda {
         } else {
             defPointer = null;
             try {
-                Method interfaceMethod = expected.struct.getFunctionalMethod();
-                if (interfaceMethod == null) {
-                    throw new IllegalArgumentException("Cannot pass lambda to [" + expected.name + 
-                            "], not a functional interface");
-                }
                 Class<?> captureClasses[] = new Class<?>[captures.size()];
                 for (int i = 0; i < captures.size(); i++) {
                     captureClasses[i] = captures.get(i).type.clazz;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallInvoke.java
index 124755780cb..21bfff65e8d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallInvoke.java
@@ -21,13 +21,16 @@ package org.elasticsearch.painless.node;
 
 import org.elasticsearch.painless.Definition.MethodKey;
 import org.elasticsearch.painless.Location;
+import org.elasticsearch.painless.Definition;
 import org.elasticsearch.painless.Definition.Method;
 import org.elasticsearch.painless.Definition.Sort;
 import org.elasticsearch.painless.Definition.Struct;
+import org.elasticsearch.painless.Definition.Type;
 import org.elasticsearch.painless.Globals;
 import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.MethodWriter;
 
+import java.lang.invoke.MethodType;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
@@ -41,6 +44,8 @@ public final class LCallInvoke extends ALink {
     final List<AExpression> arguments;
 
     Method method = null;
+    
+    boolean box = false; // true for primitive types
 
     public LCallInvoke(Location location, String name, List<AExpression> arguments) {
         super(location, -1);
@@ -68,6 +73,12 @@ public final class LCallInvoke extends ALink {
 
         MethodKey methodKey = new MethodKey(name, arguments.size());
         Struct struct = before.struct;
+        if (before.clazz.isPrimitive()) {
+            Class<?> wrapper = MethodType.methodType(before.clazz).wrap().returnType();
+            Type boxed = Definition.getType(wrapper.getSimpleName());
+            struct = boxed.struct;
+            box = true;
+        }
         method = statik ? struct.staticMethods.get(methodKey) : struct.methods.get(methodKey);
 
         if (method != null) {
@@ -103,6 +114,10 @@ public final class LCallInvoke extends ALink {
     @Override
     void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
+        
+        if (box) {
+            writer.box(before.type);
+        }
 
         for (AExpression argument : arguments) {
             argument.write(writer, globals);
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
index 885ce060aaa..ba6885eabf3 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.painless;
 
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -119,4 +118,10 @@ public class BasicAPITests extends ScriptTestCase {
         assertEquals("{}", exec("Map map = new HashMap(); return map.toString();"));
         assertEquals("{}", exec("def map = new HashMap(); return map.toString();"));
     }
+    
+    public void testPrimitivesHaveMethods() {
+        assertEquals(5, exec("int x = 5; return x.intValue();"));
+        assertEquals("5", exec("int x = 5; return x.toString();"));
+        assertEquals(0, exec("int x = 5; return x.compareTo(5);"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
index 6fb9151cb0d..da48275c829 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefOptimizationTests.java
@@ -452,4 +452,14 @@ public class DefOptimizationTests extends ScriptTestCase {
         assertBytecodeExists("def x = 1; double y = +x; return y", 
                              "INVOKEDYNAMIC plus(Ljava/lang/Object;)D");
     }
+    
+    public void testLambdaReturnType() {
+        assertBytecodeExists("List l = new ArrayList(); l.removeIf(x -> x < 10)",
+                             "synthetic lambda$0(Ljava/lang/Object;)Z");
+    }
+    
+    public void testLambdaArguments() {
+        assertBytecodeExists("List l = new ArrayList(); l.stream().mapToDouble(Double::valueOf).map(x -> x + 1)",
+                             "synthetic lambda$0(D)D");
+    }
 }

From a3e5aeea95d39c3f7e0056481cdd4222c5f9780f Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 20 Jun 2016 22:06:39 -0400
Subject: [PATCH 0978/1311] Add link to Discourse and IRC to

---
 README.textile | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/README.textile b/README.textile
index e48495a5524..7a136a91d1c 100644
--- a/README.textile
+++ b/README.textile
@@ -196,7 +196,14 @@ In order to play with the distributed nature of Elasticsearch, simply bring more
 
 h3. Where to go from here?
 
-We have just covered a very small portion of what Elasticsearch is all about. For more information, please refer to the "elastic.co":http://www.elastic.co/products/elasticsearch website.
+We have just covered a very small portion of what Elasticsearch is all
+about. For more information, please refer to the
+"elastic.co":http://www.elastic.co/products/elasticsearch
+website. General questions can be asked on the "Elastic Discourse
+forum":https://discuss.elastic.co or on IRC on Freenode at
+"#elasticsearch":https://webchat.freenode.net/#elasticsearch. The
+Elasticsearch GitHub repository is reserved for bug reports and feature
+requests only.
 
 h3. Building from Source
 

From b15bada2c8d8558b4aba73938c9d1f2f1c95009a Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 20 Jun 2016 22:14:53 -0400
Subject: [PATCH 0979/1311] Fix newlines in README

---
 README.textile | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/README.textile b/README.textile
index 7a136a91d1c..d9549e5eb3c 100644
--- a/README.textile
+++ b/README.textile
@@ -196,14 +196,7 @@ In order to play with the distributed nature of Elasticsearch, simply bring more
 
 h3. Where to go from here?
 
-We have just covered a very small portion of what Elasticsearch is all
-about. For more information, please refer to the
-"elastic.co":http://www.elastic.co/products/elasticsearch
-website. General questions can be asked on the "Elastic Discourse
-forum":https://discuss.elastic.co or on IRC on Freenode at
-"#elasticsearch":https://webchat.freenode.net/#elasticsearch. The
-Elasticsearch GitHub repository is reserved for bug reports and feature
-requests only.
+We have just covered a very small portion of what Elasticsearch is all about. For more information, please refer to the "elastic.co":http://www.elastic.co/products/elasticsearch website. General questions can be asked on the "Elastic Discourse forum":https://discuss.elastic.co or on IRC on Freenode at "#elasticsearch":https://webchat.freenode.net/#elasticsearch. The Elasticsearch GitHub repository is reserved for bug reports and feature requests only.
 
 h3. Building from Source
 

From 4401517b8518459e3103ea1bbcff1d03c6f939eb Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Tue, 21 Jun 2016 08:45:46 +0200
Subject: [PATCH 0980/1311] Revert #18839 as it causes file leaks

```
   > Throwable #1: java.lang.RuntimeException: file handle leaks: [SeekableByteChannel(/var/lib/jenkins/workspace/elastic+elasticsearch+master+g1gc/core/build/testrun/integTest/J0/temp/org.elasticsearch.search.suggest.CompletionSuggestSearch2xIT_518545A20D129C8C-001/tempDir-001/data/nodes/1/indices/4sTECv6WSJOJsw9L4CGamg/0/index/segments_1), SeekableByteChannel(/var/lib/jenkins/workspace/elastic+elasticsearch+master+g1gc/core/build/testrun/integTest/J0/temp/org.elasticsearch.search.suggest.CompletionSuggestSearch2xIT_518545A20D129C8C-001/tempDir-001/data/nodes/1/indices/4sTECv6WSJOJsw9L4CGamg/0/index/segments_1)]
   > 	at __randomizedtesting.SeedInfo.seed([518545A20D129C8C]:0)
   > 	at org.apache.lucene.mockfile.LeakFS.onClose(LeakFS.java:63)
   > 	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:77)
   > 	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:78)
   > 	at java.lang.Thread.run(Thread.java:745)
   > Caused by: java.lang.Exception
   > 	at org.apache.lucene.mockfile.LeakFS.onOpen(LeakFS.java:46)
   > 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
   > 	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:271)
   > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:212)
   > 	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:240)
   > 	at java.nio.file.Files.newByteChannel(Files.java:361)
   > 	at java.nio.file.Files.newByteChannel(Files.java:407)
   > 	at org.apache.lucene.store.SimpleFSDirectory.openInput(SimpleFSDirectory.java:77)
   > 	at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:94)
   > 	at org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2695)
   > 	at org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:737)
   > 	at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:94)
   > 	at org.elasticsearch.common.lucene.Lucene$1.doBody(Lucene.java:237)
   > 	at org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:685)
   > 	at org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:637)
   > 	at org.elasticsearch.common.lucene.Lucene.checkSegmentInfoIntegrity(Lucene.java:242)
   > 	at org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:847)
   > 	at org.elasticsearch.index.store.Store$MetadataSnapshot.<init>(Store.java:740)
   > 	at org.elasticsearch.index.store.Store.getMetadata(Store.java:260)
   > 	at org.elasticsearch.index.store.Store.getMetadata(Store.java:240)
   > 	at org.elasticsearch.index.shard.IndexShard.doCheckIndex(IndexShard.java:1310)
   > 	at org.elasticsearch.common.util.CancellableThreads.executeIO(CancellableThreads.java:102)
   > 	at org.elasticsearch.index.shard.IndexShard.checkIndex(IndexShard.java:1288)
   > 	at org.elasticsearch.index.shard.IndexShard.internalPerformTranslogRecovery(IndexShard.java:921)
   > 	at org.elasticsearch.index.shard.IndexShard.skipTranslogRecovery(IndexShard.java:964)
   > 	at org.elasticsearch.indices.recovery.RecoveryTarget.prepareForTranslogOperations(RecoveryTarget.java:297)
   > 	at
   ```
---
 .../java/org/elasticsearch/index/shard/IndexShard.java | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index f20f2f5d372..959b6f8f726 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -53,7 +53,6 @@ import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.Callback;
-import org.elasticsearch.common.util.CancellableThreads;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.SuspendableRefContainer;
 import org.elasticsearch.index.Index;
@@ -159,7 +158,6 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
     private final TranslogConfig translogConfig;
     private final IndexEventListener indexEventListener;
     private final QueryCachingPolicy cachingPolicy;
-    private final CancellableThreads cancellableThreads;
 
 
     /**
@@ -267,7 +265,6 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         primaryTerm = indexSettings.getIndexMetaData().primaryTerm(shardId.id());
         refreshListeners = buildRefreshListeners();
         persistMetadata(shardRouting, null);
-        cancellableThreads = new CancellableThreads();
     }
 
     public Store store() {
@@ -846,7 +843,6 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
                 } finally { // playing safe here and close the engine even if the above succeeds - close can be called multiple times
                     IOUtils.close(engine);
                 }
-                cancellableThreads.cancel(reason);
             }
         }
     }
@@ -1285,11 +1281,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
     private void checkIndex() throws IOException {
         if (store.tryIncRef()) {
             try {
-                cancellableThreads.executeIO(this::doCheckIndex);
-            } catch (ClosedByInterruptException ex) {
-                assert cancellableThreads.isCancelled();
-                // that's fine we might run into this when we cancel the thread since Java NIO will close the channel on interrupt
-                // and on the next access we fail it.
+                doCheckIndex();
             } finally {
                 store.decRef();
             }

From 82f7bfad98253e94305136df481cd1c7dc4e8ca8 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Mon, 13 Jun 2016 12:21:20 +0200
Subject: [PATCH 0981/1311] ingest: merged o.e.ingest.core with o.e.ingest and
 in ingest-common module added o.e.ingest.common package

and moved all code to that package.
---
 .../resources/checkstyle_suppressions.xml     | 13 ------
 .../admin/cluster/node/info/NodeInfo.java     |  2 +-
 .../ingest/PutPipelineTransportAction.java    |  2 +-
 .../ingest/SimulateDocumentBaseResult.java    |  2 +-
 .../ingest/SimulateExecutionService.java      |  6 +--
 .../ingest/SimulatePipelineRequest.java       |  8 ++--
 .../ingest/SimulateProcessorResult.java       |  4 +-
 .../ingest/TrackingResultProcessor.java       |  6 +--
 .../ingest/WriteableIngestDocument.java       |  2 +-
 .../ingest/{core => }/AbstractProcessor.java  |  3 +-
 .../{core => }/AbstractProcessorFactory.java  |  3 +-
 .../ingest/{core => }/CompoundProcessor.java  |  5 ++-
 .../ingest/{core => }/ConfigurationUtils.java |  3 +-
 .../ingest/{core => }/IngestDocument.java     | 44 ++++++++++++-------
 .../ingest/{core => }/IngestInfo.java         |  2 +-
 .../elasticsearch/ingest/IngestService.java   |  3 --
 .../ingest/InternalTemplateService.java       |  1 -
 .../ingest/{core => }/Pipeline.java           | 13 +++---
 .../ingest/PipelineExecutionService.java      |  7 +--
 .../elasticsearch/ingest/PipelineStore.java   | 16 +++----
 .../ingest/{core => }/Processor.java          |  2 +-
 .../ingest/{core => }/ProcessorInfo.java      |  2 +-
 .../ingest/ProcessorsRegistry.java            |  2 -
 .../ingest/{core => }/TemplateService.java    |  2 +-
 .../ingest/{core => }/ValueSource.java        |  2 +-
 .../org/elasticsearch/node/NodeModule.java    |  3 +-
 .../ingest/IngestActionFilterTests.java       |  7 ++-
 .../SimulateDocumentSimpleResultTests.java    |  4 +-
 .../ingest/SimulateExecutionServiceTests.java |  8 ++--
 .../SimulatePipelineRequestParsingTests.java  | 14 +++---
 .../ingest/SimulatePipelineResponseTests.java |  4 +-
 .../ingest/SimulateProcessorResultTests.java  |  4 +-
 .../ingest/TrackingResultProcessorTests.java  | 12 +++--
 .../ingest/WriteableIngestDocumentTests.java  |  4 +-
 .../{core => }/CompoundProcessorTests.java    |  3 +-
 .../{core => }/ConfigurationUtilsTests.java   |  4 +-
 .../elasticsearch/ingest/IngestClientIT.java  |  1 -
 .../elasticsearch/ingest/IngestCloseIT.java   |  2 -
 .../{core => }/IngestDocumentTests.java       |  4 +-
 ...gestProcessorNotInstalledOnAllNodesIT.java |  1 -
 .../ingest/PipelineExecutionServiceTests.java |  4 --
 .../{core => }/PipelineFactoryTests.java      |  7 +--
 .../ingest/PipelineStoreTests.java            |  5 ---
 .../ingest/{core => }/ValueSourceTests.java   |  4 +-
 .../nodesinfo/NodeInfoStreamingTests.java     |  2 +-
 modules/ingest-common/build.gradle            |  2 +-
 .../{ => common}/AbstractStringProcessor.java | 10 ++---
 .../ingest/{ => common}/AppendProcessor.java  | 14 +++---
 .../ingest/{ => common}/ConvertProcessor.java | 12 ++---
 .../ingest/{ => common}/DateFormat.java       |  2 +-
 .../{ => common}/DateIndexNameProcessor.java  | 10 ++---
 .../ingest/{ => common}/DateProcessor.java    | 10 ++---
 .../ingest/{ => common}/FailProcessor.java    | 12 ++---
 .../{ => common}/FailProcessorException.java  |  9 ++--
 .../ingest/{ => common}/ForEachProcessor.java | 17 +++----
 .../ingest/{ => common}/Grok.java             |  2 +-
 .../ingest/{ => common}/GrokMatchGroup.java   |  2 +-
 .../ingest/{ => common}/GrokProcessor.java    | 12 ++---
 .../ingest/{ => common}/GsubProcessor.java    | 12 ++---
 .../{ => common}/IngestCommonPlugin.java      |  2 +-
 .../ingest/{ => common}/JoinProcessor.java    | 10 ++---
 .../{ => common}/LowercaseProcessor.java      |  2 +-
 .../ingest/{ => common}/RemoveProcessor.java  | 12 ++---
 .../ingest/{ => common}/RenameProcessor.java  | 10 ++---
 .../ingest/{ => common}/ScriptProcessor.java  | 14 +++---
 .../ingest/{ => common}/SetProcessor.java     | 14 +++---
 .../ingest/{ => common}/SortProcessor.java    | 10 ++---
 .../ingest/{ => common}/SplitProcessor.java   | 10 ++---
 .../ingest/{ => common}/TrimProcessor.java    |  2 +-
 .../{ => common}/UppercaseProcessor.java      |  2 +-
 .../AbstractStringProcessorTestCase.java      |  7 +--
 .../AppendProcessorFactoryTests.java          |  5 ++-
 .../{ => common}/AppendProcessorTests.java    | 12 ++---
 .../ConvertProcessorFactoryTests.java         |  4 +-
 .../{ => common}/ConvertProcessorTests.java   |  9 ++--
 .../ingest/{ => common}/DateFormatTests.java  |  2 +-
 .../DateIndexNameFactoryTests.java            |  2 +-
 .../DateIndexNameProcessorTests.java          |  4 +-
 .../DateProcessorFactoryTests.java            |  4 +-
 .../{ => common}/DateProcessorTests.java      |  5 ++-
 .../FailProcessorFactoryTests.java            |  5 ++-
 .../{ => common}/FailProcessorTests.java      |  8 ++--
 .../ForEachProcessorFactoryTests.java         |  6 ++-
 .../{ => common}/ForEachProcessorTests.java   | 14 +++---
 .../GrokProcessorFactoryTests.java            |  4 +-
 .../{ => common}/GrokProcessorTests.java      |  5 +--
 .../ingest/{ => common}/GrokTests.java        |  2 +-
 .../GsubProcessorFactoryTests.java            |  4 +-
 .../{ => common}/GsubProcessorTests.java      |  7 +--
 .../{ => common}/IngestCommonRestIT.java      |  2 +-
 .../JoinProcessorFactoryTests.java            |  4 +-
 .../{ => common}/JoinProcessorTests.java      |  7 +--
 .../LowercaseProcessorFactoryTests.java       |  4 +-
 .../{ => common}/LowercaseProcessorTests.java |  2 +-
 .../RemoveProcessorFactoryTests.java          |  5 ++-
 .../{ => common}/RemoveProcessorTests.java    |  8 ++--
 .../RenameProcessorFactoryTests.java          |  4 +-
 .../{ => common}/RenameProcessorTests.java    |  7 +--
 .../ScriptProcessorFactoryTests.java          |  2 +-
 .../{ => common}/ScriptProcessorTests.java    |  5 ++-
 .../SetProcessorFactoryTests.java             |  5 ++-
 .../{ => common}/SetProcessorTests.java       | 12 ++---
 .../{ => common}/SortProcessorTests.java      |  9 ++--
 .../SplitProcessorFactoryTests.java           |  4 +-
 .../{ => common}/SplitProcessorTests.java     |  7 +--
 .../TrimProcessorFactoryTests.java            |  4 +-
 .../{ => common}/TrimProcessorTests.java      |  2 +-
 .../UppercaseProcessorFactoryTests.java       |  4 +-
 .../{ => common}/UppercaseProcessorTests.java |  2 +-
 .../attachment/AttachmentProcessor.java       | 14 +++---
 .../AttachmentProcessorFactoryTests.java      |  2 +-
 .../attachment/AttachmentProcessorTests.java  |  2 +-
 .../ingest/geoip/GeoIpProcessor.java          | 21 +++++----
 .../geoip/GeoIpProcessorFactoryTests.java     | 14 +++---
 .../ingest/geoip/GeoIpProcessorTests.java     | 14 +++---
 .../ingest/AbstractScriptTestCase.java        |  1 -
 .../ingest/IngestDocumentMustacheIT.java      |  3 --
 .../ingest/TemplateServiceIT.java             |  2 -
 .../ingest/ValueSourceMustacheIT.java         |  3 --
 .../ingest/RandomDocumentPicks.java           |  1 -
 .../elasticsearch/ingest/TestProcessor.java   |  4 --
 .../ingest/TestTemplateService.java           |  2 -
 122 files changed, 372 insertions(+), 385 deletions(-)
 rename core/src/main/java/org/elasticsearch/ingest/{core => }/AbstractProcessor.java (96%)
 rename core/src/main/java/org/elasticsearch/ingest/{core => }/AbstractProcessorFactory.java (97%)
 rename core/src/main/java/org/elasticsearch/ingest/{core => }/CompoundProcessor.java (97%)
 rename core/src/main/java/org/elasticsearch/ingest/{core => }/ConfigurationUtils.java (99%)
 rename core/src/main/java/org/elasticsearch/ingest/{core => }/IngestDocument.java (95%)
 rename core/src/main/java/org/elasticsearch/ingest/{core => }/IngestInfo.java (98%)
 rename core/src/main/java/org/elasticsearch/ingest/{core => }/Pipeline.java (90%)
 rename core/src/main/java/org/elasticsearch/ingest/{core => }/Processor.java (97%)
 rename core/src/main/java/org/elasticsearch/ingest/{core => }/ProcessorInfo.java (98%)
 rename core/src/main/java/org/elasticsearch/ingest/{core => }/TemplateService.java (97%)
 rename core/src/main/java/org/elasticsearch/ingest/{core => }/ValueSource.java (99%)
 rename core/src/test/java/org/elasticsearch/ingest/{core => }/CompoundProcessorTests.java (99%)
 rename core/src/test/java/org/elasticsearch/ingest/{core => }/ConfigurationUtilsTests.java (97%)
 rename core/src/test/java/org/elasticsearch/ingest/{core => }/IngestDocumentTests.java (99%)
 rename core/src/test/java/org/elasticsearch/ingest/{core => }/PipelineFactoryTests.java (96%)
 rename core/src/test/java/org/elasticsearch/ingest/{core => }/ValueSourceTests.java (95%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/AbstractStringProcessor.java (89%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/AppendProcessor.java (87%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/ConvertProcessor.java (94%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/DateFormat.java (98%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/DateIndexNameProcessor.java (95%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/DateProcessor.java (94%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/FailProcessor.java (86%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/FailProcessorException.java (79%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/ForEachProcessor.java (87%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/Grok.java (99%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/GrokMatchGroup.java (97%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/GrokProcessor.java (94%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/GsubProcessor.java (89%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/IngestCommonPlugin.java (99%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/JoinProcessor.java (90%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/LowercaseProcessor.java (97%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/RemoveProcessor.java (86%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/RenameProcessor.java (91%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/ScriptProcessor.java (91%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/SetProcessor.java (89%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/SortProcessor.java (93%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/SplitProcessor.java (90%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/TrimProcessor.java (97%)
 rename modules/ingest-common/src/main/java/org/elasticsearch/ingest/{ => common}/UppercaseProcessor.java (97%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/AbstractStringProcessorTestCase.java (95%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/AppendProcessorFactoryTests.java (95%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/AppendProcessorTests.java (96%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/ConvertProcessorFactoryTests.java (97%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/ConvertProcessorTests.java (98%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/DateFormatTests.java (98%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/DateIndexNameFactoryTests.java (99%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/DateIndexNameProcessorTests.java (97%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/DateProcessorFactoryTests.java (98%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/DateProcessorTests.java (98%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/FailProcessorFactoryTests.java (93%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/FailProcessorTests.java (86%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/ForEachProcessorFactoryTests.java (94%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/ForEachProcessorTests.java (95%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/GrokProcessorFactoryTests.java (98%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/GrokProcessorTests.java (98%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/GrokTests.java (99%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/GsubProcessorFactoryTests.java (97%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/GsubProcessorTests.java (95%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/IngestCommonRestIT.java (97%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/JoinProcessorFactoryTests.java (96%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/JoinProcessorTests.java (96%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/LowercaseProcessorFactoryTests.java (95%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/LowercaseProcessorTests.java (96%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/RemoveProcessorFactoryTests.java (93%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/RemoveProcessorTests.java (90%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/RenameProcessorFactoryTests.java (96%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/RenameProcessorTests.java (97%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/ScriptProcessorFactoryTests.java (98%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/ScriptProcessorTests.java (95%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/SetProcessorFactoryTests.java (96%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/SetProcessorTests.java (94%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/SortProcessorTests.java (98%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/SplitProcessorFactoryTests.java (96%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/SplitProcessorTests.java (96%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/TrimProcessorFactoryTests.java (95%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/TrimProcessorTests.java (97%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/UppercaseProcessorFactoryTests.java (95%)
 rename modules/ingest-common/src/test/java/org/elasticsearch/ingest/{ => common}/UppercaseProcessorTests.java (96%)

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 27b9efbf0a4..fbc5d149d99 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -495,11 +495,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]store[/\\]IndicesStore.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]store[/\\]TransportNodesListShardStoreMetaData.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]ttl[/\\]IndicesTTLService.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]PipelineExecutionService.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]PipelineStore.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]CompoundProcessor.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]IngestDocument.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]Pipeline.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]GcNames.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]HotThreads.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]JvmStats.java" checks="LineLength" />
@@ -1025,11 +1020,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]store[/\\]IndicesStoreIntegrationIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]store[/\\]IndicesStoreTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]template[/\\]SimpleIndexTemplateIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]PipelineExecutionServiceTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]PipelineStoreTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]CompoundProcessorTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]PipelineFactoryTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]core[/\\]ValueSourceTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]mget[/\\]SimpleMgetIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]jvm[/\\]JvmGcMonitorServiceSettingsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]monitor[/\\]os[/\\]OsProbeTests.java" checks="LineLength" />
@@ -1206,9 +1196,6 @@
   <suppress files="plugins[/\\]discovery-ec2[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]aws[/\\]AbstractAwsTestCase.java" checks="LineLength" />
   <suppress files="plugins[/\\]discovery-ec2[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]ec2[/\\]AmazonEC2Mock.java" checks="LineLength" />
   <suppress files="plugins[/\\]discovery-gce[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]gce[/\\]GceNetworkTests.java" checks="LineLength" />
-  <suppress files="plugins[/\\]ingest-geoip[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]geoip[/\\]GeoIpProcessor.java" checks="LineLength" />
-  <suppress files="plugins[/\\]ingest-geoip[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]geoip[/\\]GeoIpProcessorFactoryTests.java" checks="LineLength" />
-  <suppress files="plugins[/\\]ingest-geoip[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]ingest[/\\]geoip[/\\]GeoIpProcessorTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]lang-javascript[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]plugin[/\\]javascript[/\\]JavaScriptPlugin.java" checks="LineLength" />
   <suppress files="plugins[/\\]lang-javascript[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]javascript[/\\]JavaScriptScriptEngineService.java" checks="LineLength" />
   <suppress files="plugins[/\\]lang-javascript[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]javascript[/\\]JavaScriptScriptEngineTests.java" checks="LineLength" />
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
index 87ec2d052ab..068f974d893 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
@@ -28,7 +28,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.http.HttpInfo;
-import org.elasticsearch.ingest.core.IngestInfo;
+import org.elasticsearch.ingest.IngestInfo;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.monitor.os.OsInfo;
 import org.elasticsearch.monitor.process.ProcessInfo;
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
index bea5a2c8bc3..df92567b98d 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
@@ -35,7 +35,7 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.ingest.PipelineStore;
-import org.elasticsearch.ingest.core.IngestInfo;
+import org.elasticsearch.ingest.IngestInfo;
 import org.elasticsearch.node.service.NodeService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java
index d444922cfc3..8521484370c 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java
@@ -22,7 +22,7 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.IngestDocument;
 
 import java.io.IOException;
 
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java
index 2636dcb3ba8..db7397ba1f8 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java
@@ -21,9 +21,9 @@ package org.elasticsearch.action.ingest;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRunnable;
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.Pipeline;
-import org.elasticsearch.ingest.core.CompoundProcessor;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Pipeline;
+import org.elasticsearch.ingest.CompoundProcessor;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.ArrayList;
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
index 14092ad62cb..ce24969e8f2 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
@@ -24,9 +24,9 @@ import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.ingest.core.ConfigurationUtils;
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.Pipeline;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Pipeline;
 import org.elasticsearch.ingest.PipelineStore;
 
 import java.io.IOException;
@@ -35,7 +35,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.ingest.core.IngestDocument.MetaData;
+import static org.elasticsearch.ingest.IngestDocument.MetaData;
 
 public class SimulatePipelineRequest extends ActionRequest<SimulatePipelineRequest> {
 
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
index ba7ef195aad..6924550d79b 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
@@ -24,8 +24,8 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.IngestDocument;
 
 import java.io.IOException;
 
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/TrackingResultProcessor.java b/core/src/main/java/org/elasticsearch/action/ingest/TrackingResultProcessor.java
index 3693b5dcac7..b91c7b14fee 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/TrackingResultProcessor.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/TrackingResultProcessor.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.action.ingest;
 
-import org.elasticsearch.ingest.core.CompoundProcessor;
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.Processor;
+import org.elasticsearch.ingest.CompoundProcessor;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
 
 import java.util.ArrayList;
 import java.util.List;
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java b/core/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java
index a0416fe8338..137914701db 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java
@@ -24,7 +24,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.IngestDocument;
 
 import java.io.IOException;
 import java.util.Map;
diff --git a/core/src/main/java/org/elasticsearch/ingest/core/AbstractProcessor.java b/core/src/main/java/org/elasticsearch/ingest/AbstractProcessor.java
similarity index 96%
rename from core/src/main/java/org/elasticsearch/ingest/core/AbstractProcessor.java
rename to core/src/main/java/org/elasticsearch/ingest/AbstractProcessor.java
index e709ae395cb..c7204ee3e69 100644
--- a/core/src/main/java/org/elasticsearch/ingest/core/AbstractProcessor.java
+++ b/core/src/main/java/org/elasticsearch/ingest/AbstractProcessor.java
@@ -17,8 +17,7 @@
  * under the License.
  */
 
-
-package org.elasticsearch.ingest.core;
+package org.elasticsearch.ingest;
 
 /**
  * An Abstract Processor that holds a processorTag field to be used
diff --git a/core/src/main/java/org/elasticsearch/ingest/core/AbstractProcessorFactory.java b/core/src/main/java/org/elasticsearch/ingest/AbstractProcessorFactory.java
similarity index 97%
rename from core/src/main/java/org/elasticsearch/ingest/core/AbstractProcessorFactory.java
rename to core/src/main/java/org/elasticsearch/ingest/AbstractProcessorFactory.java
index 323344f8f41..0fb50228793 100644
--- a/core/src/main/java/org/elasticsearch/ingest/core/AbstractProcessorFactory.java
+++ b/core/src/main/java/org/elasticsearch/ingest/AbstractProcessorFactory.java
@@ -17,8 +17,7 @@
  * under the License.
  */
 
-
-package org.elasticsearch.ingest.core;
+package org.elasticsearch.ingest;
 
 import java.util.Map;
 
diff --git a/core/src/main/java/org/elasticsearch/ingest/core/CompoundProcessor.java b/core/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java
similarity index 97%
rename from core/src/main/java/org/elasticsearch/ingest/core/CompoundProcessor.java
rename to core/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java
index 652a2c184a2..4af2e8a811a 100644
--- a/core/src/main/java/org/elasticsearch/ingest/core/CompoundProcessor.java
+++ b/core/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java
@@ -18,7 +18,7 @@
  */
 //TODO(simonw): can all these classes go into org.elasticsearch.ingest?
 
-package org.elasticsearch.ingest.core;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchException;
 
@@ -103,7 +103,8 @@ public class CompoundProcessor implements Processor {
                     continue;
                 }
 
-                ElasticsearchException compoundProcessorException = newCompoundProcessorException(e, processor.getType(), processor.getTag());
+                ElasticsearchException compoundProcessorException =
+                        newCompoundProcessorException(e, processor.getType(), processor.getTag());
                 if (onFailureProcessors.isEmpty()) {
                     throw compoundProcessorException;
                 } else {
diff --git a/core/src/main/java/org/elasticsearch/ingest/core/ConfigurationUtils.java b/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
similarity index 99%
rename from core/src/main/java/org/elasticsearch/ingest/core/ConfigurationUtils.java
rename to core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
index 20370430bdc..d87a59fd625 100644
--- a/core/src/main/java/org/elasticsearch/ingest/core/ConfigurationUtils.java
+++ b/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
@@ -17,10 +17,9 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.core;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.ProcessorsRegistry;
 
 import java.util.ArrayList;
 import java.util.Arrays;
diff --git a/core/src/main/java/org/elasticsearch/ingest/core/IngestDocument.java b/core/src/main/java/org/elasticsearch/ingest/IngestDocument.java
similarity index 95%
rename from core/src/main/java/org/elasticsearch/ingest/core/IngestDocument.java
rename to core/src/main/java/org/elasticsearch/ingest/IngestDocument.java
index 8b1f02607d8..80bbd52bcf6 100644
--- a/core/src/main/java/org/elasticsearch/ingest/core/IngestDocument.java
+++ b/core/src/main/java/org/elasticsearch/ingest/IngestDocument.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.core;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.index.mapper.internal.IdFieldMapper;
@@ -56,7 +56,8 @@ public final class IngestDocument {
     private final Map<String, Object> sourceAndMetadata;
     private final Map<String, String> ingestMetadata;
 
-    public IngestDocument(String index, String type, String id, String routing, String parent, String timestamp, String ttl, Map<String, Object> source) {
+    public IngestDocument(String index, String type, String id, String routing, String parent, String timestamp,
+                          String ttl, Map<String, Object> source) {
         this.sourceAndMetadata = new HashMap<>();
         this.sourceAndMetadata.putAll(source);
         this.sourceAndMetadata.put(MetaData.INDEX.getFieldName(), index);
@@ -251,10 +252,12 @@ public final class IngestDocument {
             try {
                 index = Integer.parseInt(leafKey);
             } catch (NumberFormatException e) {
-                throw new IllegalArgumentException("[" + leafKey + "] is not an integer, cannot be used as an index as part of path [" + path + "]", e);
+                throw new IllegalArgumentException("[" + leafKey + "] is not an integer, cannot be used as an index as part of path [" +
+                        path + "]", e);
             }
             if (index < 0 || index >= list.size()) {
-                throw new IllegalArgumentException("[" + index + "] is out of bounds for array with length [" + list.size() + "] as part of path [" + path + "]");
+                throw new IllegalArgumentException("[" + index + "] is out of bounds for array with length [" + list.size() +
+                        "] as part of path [" + path + "]");
             }
             list.remove(index);
             return;
@@ -263,7 +266,8 @@ public final class IngestDocument {
         if (context == null) {
             throw new IllegalArgumentException("cannot remove [" + leafKey + "] from null as part of path [" + path + "]");
         }
-        throw new IllegalArgumentException("cannot remove [" + leafKey + "] from object of type [" + context.getClass().getName() + "] as part of path [" + path + "]");
+        throw new IllegalArgumentException("cannot remove [" + leafKey + "] from object of type [" + context.getClass().getName() +
+                "] as part of path [" + path + "]");
     }
 
     private static Object resolve(String pathElement, String fullPath, Object context) {
@@ -285,14 +289,17 @@ public final class IngestDocument {
             try {
                 index = Integer.parseInt(pathElement);
             } catch (NumberFormatException e) {
-                throw new IllegalArgumentException("[" + pathElement + "] is not an integer, cannot be used as an index as part of path [" + fullPath + "]", e);
+                throw new IllegalArgumentException("[" + pathElement + "] is not an integer, cannot be used as an index as part of path ["
+                        + fullPath + "]", e);
             }
             if (index < 0 || index >= list.size()) {
-                throw new IllegalArgumentException("[" + index + "] is out of bounds for array with length [" + list.size() + "] as part of path [" + fullPath + "]");
+                throw new IllegalArgumentException("[" + index + "] is out of bounds for array with length [" + list.size() +
+                        "] as part of path [" + fullPath + "]");
             }
             return list.get(index);
         }
-        throw new IllegalArgumentException("cannot resolve [" + pathElement + "] from object of type [" + context.getClass().getName() + "] as part of path [" + fullPath + "]");
+        throw new IllegalArgumentException("cannot resolve [" + pathElement + "] from object of type [" + context.getClass().getName() +
+                "] as part of path [" + fullPath + "]");
     }
 
     /**
@@ -381,14 +388,17 @@ public final class IngestDocument {
                 try {
                     index = Integer.parseInt(pathElement);
                 } catch (NumberFormatException e) {
-                    throw new IllegalArgumentException("[" + pathElement + "] is not an integer, cannot be used as an index as part of path [" + path + "]", e);
+                    throw new IllegalArgumentException("[" + pathElement +
+                            "] is not an integer, cannot be used as an index as part of path [" + path + "]", e);
                 }
                 if (index < 0 || index >= list.size()) {
-                    throw new IllegalArgumentException("[" + index + "] is out of bounds for array with length [" + list.size() + "] as part of path [" + path + "]");
+                    throw new IllegalArgumentException("[" + index + "] is out of bounds for array with length [" +
+                            list.size() + "] as part of path [" + path + "]");
                 }
                 context = list.get(index);
             } else {
-                throw new IllegalArgumentException("cannot resolve [" + pathElement + "] from object of type [" + context.getClass().getName() + "] as part of path [" + path + "]");
+                throw new IllegalArgumentException("cannot resolve [" + pathElement + "] from object of type [" +
+                        context.getClass().getName() + "] as part of path [" + path + "]");
             }
         }
 
@@ -421,10 +431,12 @@ public final class IngestDocument {
             try {
                 index = Integer.parseInt(leafKey);
             } catch (NumberFormatException e) {
-                throw new IllegalArgumentException("[" + leafKey + "] is not an integer, cannot be used as an index as part of path [" + path + "]", e);
+                throw new IllegalArgumentException("[" + leafKey + "] is not an integer, cannot be used as an index as part of path [" +
+                        path + "]", e);
             }
             if (index < 0 || index >= list.size()) {
-                throw new IllegalArgumentException("[" + index + "] is out of bounds for array with length [" + list.size() + "] as part of path [" + path + "]");
+                throw new IllegalArgumentException("[" + index + "] is out of bounds for array with length [" + list.size() +
+                        "] as part of path [" + path + "]");
             }
             if (append) {
                 Object object = list.get(index);
@@ -436,7 +448,8 @@ public final class IngestDocument {
             }
             list.set(index, value);
         } else {
-            throw new IllegalArgumentException("cannot set [" + leafKey + "] with parent object of type [" + context.getClass().getName() + "] as part of path [" + path + "]");
+            throw new IllegalArgumentException("cannot set [" + leafKey + "] with parent object of type [" +
+                    context.getClass().getName() + "] as part of path [" + path + "]");
         }
     }
 
@@ -471,7 +484,8 @@ public final class IngestDocument {
         if (clazz.isInstance(object)) {
             return clazz.cast(object);
         }
-        throw new IllegalArgumentException("field [" + path + "] of type [" + object.getClass().getName() + "] cannot be cast to [" + clazz.getName() + "]");
+        throw new IllegalArgumentException("field [" + path + "] of type [" + object.getClass().getName() + "] cannot be cast to [" +
+                clazz.getName() + "]");
     }
 
     public String renderTemplate(TemplateService.Template template) {
diff --git a/core/src/main/java/org/elasticsearch/ingest/core/IngestInfo.java b/core/src/main/java/org/elasticsearch/ingest/IngestInfo.java
similarity index 98%
rename from core/src/main/java/org/elasticsearch/ingest/core/IngestInfo.java
rename to core/src/main/java/org/elasticsearch/ingest/IngestInfo.java
index 1d72c6f86a1..7dbce888bba 100644
--- a/core/src/main/java/org/elasticsearch/ingest/core/IngestInfo.java
+++ b/core/src/main/java/org/elasticsearch/ingest/IngestInfo.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.core;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
diff --git a/core/src/main/java/org/elasticsearch/ingest/IngestService.java b/core/src/main/java/org/elasticsearch/ingest/IngestService.java
index 60fb3c26eea..228de5b37bc 100644
--- a/core/src/main/java/org/elasticsearch/ingest/IngestService.java
+++ b/core/src/main/java/org/elasticsearch/ingest/IngestService.java
@@ -21,9 +21,6 @@ package org.elasticsearch.ingest;
 
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.ingest.core.IngestInfo;
-import org.elasticsearch.ingest.core.Processor;
-import org.elasticsearch.ingest.core.ProcessorInfo;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.threadpool.ThreadPool;
 
diff --git a/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java b/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java
index 3a3fae9e997..2e1f1415b42 100644
--- a/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java
+++ b/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.ingest;
 
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.ingest.core.TemplateService;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
diff --git a/core/src/main/java/org/elasticsearch/ingest/core/Pipeline.java b/core/src/main/java/org/elasticsearch/ingest/Pipeline.java
similarity index 90%
rename from core/src/main/java/org/elasticsearch/ingest/core/Pipeline.java
rename to core/src/main/java/org/elasticsearch/ingest/Pipeline.java
index d0b1ca7f6cf..bea02f49358 100644
--- a/core/src/main/java/org/elasticsearch/ingest/core/Pipeline.java
+++ b/core/src/main/java/org/elasticsearch/ingest/Pipeline.java
@@ -17,17 +17,15 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.core;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.ProcessorsRegistry;
 
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
-
 /**
  * A pipeline is a list of {@link Processor} instances grouped under a unique id.
  */
@@ -104,12 +102,15 @@ public final class Pipeline {
             String description = ConfigurationUtils.readOptionalStringProperty(null, null, config, DESCRIPTION_KEY);
             List<Map<String, Map<String, Object>>> processorConfigs = ConfigurationUtils.readList(null, null, config, PROCESSORS_KEY);
             List<Processor> processors = ConfigurationUtils.readProcessorConfigs(processorConfigs, processorRegistry);
-            List<Map<String, Map<String, Object>>> onFailureProcessorConfigs = ConfigurationUtils.readOptionalList(null, null, config, ON_FAILURE_KEY);
+            List<Map<String, Map<String, Object>>> onFailureProcessorConfigs =
+                    ConfigurationUtils.readOptionalList(null, null, config, ON_FAILURE_KEY);
             List<Processor> onFailureProcessors = ConfigurationUtils.readProcessorConfigs(onFailureProcessorConfigs, processorRegistry);
             if (config.isEmpty() == false) {
-                throw new ElasticsearchParseException("pipeline [" + id + "] doesn't support one or more provided configuration parameters " + Arrays.toString(config.keySet().toArray()));
+                throw new ElasticsearchParseException("pipeline [" + id +
+                        "] doesn't support one or more provided configuration parameters " + Arrays.toString(config.keySet().toArray()));
             }
-            CompoundProcessor compoundProcessor = new CompoundProcessor(false, Collections.unmodifiableList(processors), Collections.unmodifiableList(onFailureProcessors));
+            CompoundProcessor compoundProcessor = new CompoundProcessor(false, Collections.unmodifiableList(processors),
+                    Collections.unmodifiableList(onFailureProcessors));
             return new Pipeline(id, description, compoundProcessor);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java b/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java
index 8674e805974..ef726e44843 100644
--- a/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java
+++ b/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java
@@ -19,18 +19,14 @@
 
 package org.elasticsearch.ingest;
 
-import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.metrics.CounterMetric;
 import org.elasticsearch.common.metrics.MeanMetric;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.Pipeline;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Collections;
@@ -90,7 +86,8 @@ public class PipelineExecutionService implements ClusterStateListener {
                         if (Strings.hasText(indexRequest.getPipeline())) {
                             try {
                                 innerExecute(indexRequest, getPipeline(indexRequest.getPipeline()));
-                                //this shouldn't be needed here but we do it for consistency with index api which requires it to prevent double execution
+                                //this shouldn't be needed here but we do it for consistency with index api
+                                // which requires it to prevent double execution
                                 indexRequest.setPipeline(null);
                             } catch (Throwable e) {
                                 itemFailureHandler.accept(indexRequest, e);
diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
index 54cd716f0e8..f96a7179656 100644
--- a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
+++ b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
@@ -37,10 +37,6 @@ import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.ingest.core.IngestInfo;
-import org.elasticsearch.ingest.core.Pipeline;
-import org.elasticsearch.ingest.core.Processor;
-import org.elasticsearch.ingest.core.TemplateService;
 import org.elasticsearch.script.ScriptService;
 
 import java.io.Closeable;
@@ -66,7 +62,8 @@ public class PipelineStore extends AbstractComponent implements Closeable, Clust
         super(settings);
     }
 
-    public void buildProcessorFactoryRegistry(ProcessorsRegistry.Builder processorsRegistryBuilder, ScriptService scriptService, ClusterService clusterService) {
+    public void buildProcessorFactoryRegistry(ProcessorsRegistry.Builder processorsRegistryBuilder, ScriptService scriptService,
+                                              ClusterService clusterService) {
         this.processorRegistry = processorsRegistryBuilder.build(scriptService, clusterService);
     }
 
@@ -105,7 +102,8 @@ public class PipelineStore extends AbstractComponent implements Closeable, Clust
      * Deletes the pipeline specified by id in the request.
      */
     public void delete(ClusterService clusterService, DeletePipelineRequest request, ActionListener<WritePipelineResponse> listener) {
-        clusterService.submitStateUpdateTask("delete-pipeline-" + request.getId(), new AckedClusterStateUpdateTask<WritePipelineResponse>(request, listener) {
+        clusterService.submitStateUpdateTask("delete-pipeline-" + request.getId(),
+                new AckedClusterStateUpdateTask<WritePipelineResponse>(request, listener) {
 
             @Override
             protected WritePipelineResponse newResponse(boolean acknowledged) {
@@ -141,10 +139,12 @@ public class PipelineStore extends AbstractComponent implements Closeable, Clust
     /**
      * Stores the specified pipeline definition in the request.
      */
-    public void put(ClusterService clusterService, Map<DiscoveryNode, IngestInfo> ingestInfos, PutPipelineRequest request, ActionListener<WritePipelineResponse> listener) throws Exception {
+    public void put(ClusterService clusterService, Map<DiscoveryNode, IngestInfo> ingestInfos, PutPipelineRequest request,
+                    ActionListener<WritePipelineResponse> listener) throws Exception {
         // validates the pipeline and processor configuration before submitting a cluster update task:
         validatePipeline(ingestInfos, request);
-        clusterService.submitStateUpdateTask("put-pipeline-" + request.getId(), new AckedClusterStateUpdateTask<WritePipelineResponse>(request, listener) {
+        clusterService.submitStateUpdateTask("put-pipeline-" + request.getId(),
+                new AckedClusterStateUpdateTask<WritePipelineResponse>(request, listener) {
 
             @Override
             protected WritePipelineResponse newResponse(boolean acknowledged) {
diff --git a/core/src/main/java/org/elasticsearch/ingest/core/Processor.java b/core/src/main/java/org/elasticsearch/ingest/Processor.java
similarity index 97%
rename from core/src/main/java/org/elasticsearch/ingest/core/Processor.java
rename to core/src/main/java/org/elasticsearch/ingest/Processor.java
index 8cdff8714c4..92c18464e94 100644
--- a/core/src/main/java/org/elasticsearch/ingest/core/Processor.java
+++ b/core/src/main/java/org/elasticsearch/ingest/Processor.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.core;
+package org.elasticsearch.ingest;
 
 import java.util.Map;
 
diff --git a/core/src/main/java/org/elasticsearch/ingest/core/ProcessorInfo.java b/core/src/main/java/org/elasticsearch/ingest/ProcessorInfo.java
similarity index 98%
rename from core/src/main/java/org/elasticsearch/ingest/core/ProcessorInfo.java
rename to core/src/main/java/org/elasticsearch/ingest/ProcessorInfo.java
index a98b446fce0..a7a2d122ed8 100644
--- a/core/src/main/java/org/elasticsearch/ingest/core/ProcessorInfo.java
+++ b/core/src/main/java/org/elasticsearch/ingest/ProcessorInfo.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.core;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
diff --git a/core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java b/core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java
index 88a021d2059..28abfc45313 100644
--- a/core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java
+++ b/core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java
@@ -21,8 +21,6 @@ package org.elasticsearch.ingest;
 
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.ingest.core.Processor;
-import org.elasticsearch.ingest.core.TemplateService;
 import org.elasticsearch.script.ScriptService;
 
 import java.io.Closeable;
diff --git a/core/src/main/java/org/elasticsearch/ingest/core/TemplateService.java b/core/src/main/java/org/elasticsearch/ingest/TemplateService.java
similarity index 97%
rename from core/src/main/java/org/elasticsearch/ingest/core/TemplateService.java
rename to core/src/main/java/org/elasticsearch/ingest/TemplateService.java
index 8988c924c35..2ece5a94304 100644
--- a/core/src/main/java/org/elasticsearch/ingest/core/TemplateService.java
+++ b/core/src/main/java/org/elasticsearch/ingest/TemplateService.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.ingest.core;
+package org.elasticsearch.ingest;
 
 import java.util.Map;
 
diff --git a/core/src/main/java/org/elasticsearch/ingest/core/ValueSource.java b/core/src/main/java/org/elasticsearch/ingest/ValueSource.java
similarity index 99%
rename from core/src/main/java/org/elasticsearch/ingest/core/ValueSource.java
rename to core/src/main/java/org/elasticsearch/ingest/ValueSource.java
index 1a38b967e62..fa483a5fbee 100644
--- a/core/src/main/java/org/elasticsearch/ingest/core/ValueSource.java
+++ b/core/src/main/java/org/elasticsearch/ingest/ValueSource.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.core;
+package org.elasticsearch.ingest;
 
 import java.util.ArrayList;
 import java.util.Arrays;
diff --git a/core/src/main/java/org/elasticsearch/node/NodeModule.java b/core/src/main/java/org/elasticsearch/node/NodeModule.java
index 85cfe5bd6b2..6d8be732dd2 100644
--- a/core/src/main/java/org/elasticsearch/node/NodeModule.java
+++ b/core/src/main/java/org/elasticsearch/node/NodeModule.java
@@ -22,7 +22,8 @@ package org.elasticsearch.node;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.ingest.ProcessorsRegistry;
-import org.elasticsearch.ingest.core.Processor;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.TemplateService;
 import org.elasticsearch.monitor.MonitorService;
 import org.elasticsearch.node.service.NodeService;
 
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/IngestActionFilterTests.java b/core/src/test/java/org/elasticsearch/action/ingest/IngestActionFilterTests.java
index d4bccafecbf..b04533fafc4 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/IngestActionFilterTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/IngestActionFilterTests.java
@@ -33,10 +33,9 @@ import org.elasticsearch.ingest.IngestService;
 import org.elasticsearch.ingest.PipelineExecutionService;
 import org.elasticsearch.ingest.PipelineStore;
 import org.elasticsearch.ingest.TestProcessor;
-import org.elasticsearch.ingest.core.CompoundProcessor;
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.Pipeline;
-import org.elasticsearch.ingest.core.Processor;
+import org.elasticsearch.ingest.CompoundProcessor;
+import org.elasticsearch.ingest.Pipeline;
+import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.node.service.NodeService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java
index 52c13b92701..323a8c0aaa6 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java
@@ -22,12 +22,12 @@ package org.elasticsearch.action.ingest;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.ingest.RandomDocumentPicks;
-import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 
-import static org.elasticsearch.ingest.core.IngestDocumentTests.assertIngestDocument;
+import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
 
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
index f7f2f0212dd..a7bc76341b8 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
@@ -23,9 +23,9 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.ingest.TestProcessor;
-import org.elasticsearch.ingest.core.CompoundProcessor;
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.Pipeline;
+import org.elasticsearch.ingest.CompoundProcessor;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Pipeline;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.junit.After;
@@ -34,7 +34,7 @@ import org.junit.Before;
 import java.util.Collections;
 import java.util.Map;
 
-import static org.elasticsearch.ingest.core.IngestDocumentTests.assertIngestDocument;
+import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.not;
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
index 4b50f69e4c8..fa4bdc6525d 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
@@ -24,10 +24,10 @@ import org.elasticsearch.ingest.PipelineStore;
 import org.elasticsearch.ingest.ProcessorsRegistry;
 import org.elasticsearch.ingest.TestProcessor;
 import org.elasticsearch.ingest.TestTemplateService;
-import org.elasticsearch.ingest.core.CompoundProcessor;
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.Pipeline;
-import org.elasticsearch.ingest.core.Processor;
+import org.elasticsearch.ingest.CompoundProcessor;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Pipeline;
+import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
@@ -42,9 +42,9 @@ import java.util.Map;
 
 import static org.elasticsearch.action.ingest.SimulatePipelineRequest.Fields;
 import static org.elasticsearch.action.ingest.SimulatePipelineRequest.SIMULATED_PIPELINE_ID;
-import static org.elasticsearch.ingest.core.IngestDocument.MetaData.ID;
-import static org.elasticsearch.ingest.core.IngestDocument.MetaData.INDEX;
-import static org.elasticsearch.ingest.core.IngestDocument.MetaData.TYPE;
+import static org.elasticsearch.ingest.IngestDocument.MetaData.ID;
+import static org.elasticsearch.ingest.IngestDocument.MetaData.INDEX;
+import static org.elasticsearch.ingest.IngestDocument.MetaData.TYPE;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.Mockito.mock;
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java
index c8f09a5ba88..1376ca4280e 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java
@@ -21,7 +21,7 @@ package org.elasticsearch.action.ingest;
 
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
 
@@ -30,7 +30,7 @@ import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
-import static org.elasticsearch.ingest.core.IngestDocumentTests.assertIngestDocument;
+import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.nullValue;
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java
index 0ee1a5699dc..f612f36c9d6 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java
@@ -22,12 +22,12 @@ package org.elasticsearch.action.ingest;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.ingest.RandomDocumentPicks;
-import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 
-import static org.elasticsearch.ingest.core.IngestDocumentTests.assertIngestDocument;
+import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/TrackingResultProcessorTests.java b/core/src/test/java/org/elasticsearch/action/ingest/TrackingResultProcessorTests.java
index ac7282051d5..30f17515f00 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/TrackingResultProcessorTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/TrackingResultProcessorTests.java
@@ -20,11 +20,9 @@
 package org.elasticsearch.action.ingest;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.ingest.SimulateProcessorResult;
-import org.elasticsearch.action.ingest.TrackingResultProcessor;
 import org.elasticsearch.ingest.TestProcessor;
-import org.elasticsearch.ingest.core.CompoundProcessor;
-import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.CompoundProcessor;
+import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
@@ -34,9 +32,9 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.ingest.core.CompoundProcessor.ON_FAILURE_MESSAGE_FIELD;
-import static org.elasticsearch.ingest.core.CompoundProcessor.ON_FAILURE_PROCESSOR_TAG_FIELD;
-import static org.elasticsearch.ingest.core.CompoundProcessor.ON_FAILURE_PROCESSOR_TYPE_FIELD;
+import static org.elasticsearch.ingest.CompoundProcessor.ON_FAILURE_MESSAGE_FIELD;
+import static org.elasticsearch.ingest.CompoundProcessor.ON_FAILURE_PROCESSOR_TAG_FIELD;
+import static org.elasticsearch.ingest.CompoundProcessor.ON_FAILURE_PROCESSOR_TYPE_FIELD;
 import static org.elasticsearch.action.ingest.TrackingResultProcessor.decorate;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.nullValue;
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java b/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java
index 810947c8e0a..a7ce842913d 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java
@@ -25,7 +25,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.ingest.RandomDocumentPicks;
-import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -34,7 +34,7 @@ import java.util.HashMap;
 import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.ToXContent.EMPTY_PARAMS;
-import static org.elasticsearch.ingest.core.IngestDocumentTests.assertIngestDocument;
+import static org.elasticsearch.ingest.IngestDocumentTests.assertIngestDocument;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
diff --git a/core/src/test/java/org/elasticsearch/ingest/core/CompoundProcessorTests.java b/core/src/test/java/org/elasticsearch/ingest/CompoundProcessorTests.java
similarity index 99%
rename from core/src/test/java/org/elasticsearch/ingest/core/CompoundProcessorTests.java
rename to core/src/test/java/org/elasticsearch/ingest/CompoundProcessorTests.java
index 0e110288597..6cb2c9dd53f 100644
--- a/core/src/test/java/org/elasticsearch/ingest/core/CompoundProcessorTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/CompoundProcessorTests.java
@@ -17,10 +17,9 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.core;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ingest.TestProcessor;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
diff --git a/core/src/test/java/org/elasticsearch/ingest/core/ConfigurationUtilsTests.java b/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java
similarity index 97%
rename from core/src/test/java/org/elasticsearch/ingest/core/ConfigurationUtilsTests.java
rename to core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java
index ff3670b22b2..fa78d5aa16c 100644
--- a/core/src/test/java/org/elasticsearch/ingest/core/ConfigurationUtilsTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java
@@ -17,14 +17,12 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.core;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.ingest.ProcessorsRegistry;
-import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.junit.Before;
diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java b/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java
index 72856c143e0..b94cc5ca319 100644
--- a/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java
+++ b/core/src/test/java/org/elasticsearch/ingest/IngestClientIT.java
@@ -37,7 +37,6 @@ import org.elasticsearch.action.ingest.SimulatePipelineResponse;
 import org.elasticsearch.action.ingest.WritePipelineResponse;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.ingest.core.IngestDocument;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 
diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestCloseIT.java b/core/src/test/java/org/elasticsearch/ingest/IngestCloseIT.java
index 007338cf918..ed0a302cf73 100644
--- a/core/src/test/java/org/elasticsearch/ingest/IngestCloseIT.java
+++ b/core/src/test/java/org/elasticsearch/ingest/IngestCloseIT.java
@@ -19,11 +19,9 @@
 
 package org.elasticsearch.ingest;
 
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
 import org.elasticsearch.node.NodeModule;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
-import org.junit.After;
 
 import java.io.Closeable;
 import java.io.IOException;
diff --git a/core/src/test/java/org/elasticsearch/ingest/core/IngestDocumentTests.java b/core/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java
similarity index 99%
rename from core/src/test/java/org/elasticsearch/ingest/core/IngestDocumentTests.java
rename to core/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java
index b706c85afac..ee4da02478d 100644
--- a/core/src/test/java/org/elasticsearch/ingest/core/IngestDocumentTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java
@@ -17,10 +17,8 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.core;
+package org.elasticsearch.ingest;
 
-import org.elasticsearch.action.ingest.WriteableIngestDocumentTests;
-import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java b/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java
index c6a096f238e..29cf9f97290 100644
--- a/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java
+++ b/core/src/test/java/org/elasticsearch/ingest/IngestProcessorNotInstalledOnAllNodesIT.java
@@ -22,7 +22,6 @@ package org.elasticsearch.ingest;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.ingest.WritePipelineResponse;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.ingest.core.Pipeline;
 import org.elasticsearch.node.service.NodeService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
index 1b96939eb38..d001a829443 100644
--- a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
@@ -28,10 +28,6 @@ import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.ingest.core.CompoundProcessor;
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.Pipeline;
-import org.elasticsearch.ingest.core.Processor;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.hamcrest.CustomTypeSafeMatcher;
diff --git a/core/src/test/java/org/elasticsearch/ingest/core/PipelineFactoryTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java
similarity index 96%
rename from core/src/test/java/org/elasticsearch/ingest/core/PipelineFactoryTests.java
rename to core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java
index dd12d020247..7c4f8236404 100644
--- a/core/src/test/java/org/elasticsearch/ingest/core/PipelineFactoryTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java
@@ -17,17 +17,12 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.core;
+package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.ingest.ProcessorsRegistry;
-import org.elasticsearch.ingest.TestProcessor;
-import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.threadpool.TestThreadPool;
 
 import java.util.Arrays;
 import java.util.Collections;
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
index 33992af0f67..55ea4360ece 100644
--- a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
@@ -32,11 +32,6 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.LocalTransportAddress;
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.IngestInfo;
-import org.elasticsearch.ingest.core.Pipeline;
-import org.elasticsearch.ingest.core.Processor;
-import org.elasticsearch.ingest.core.ProcessorInfo;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
diff --git a/core/src/test/java/org/elasticsearch/ingest/core/ValueSourceTests.java b/core/src/test/java/org/elasticsearch/ingest/ValueSourceTests.java
similarity index 95%
rename from core/src/test/java/org/elasticsearch/ingest/core/ValueSourceTests.java
rename to core/src/test/java/org/elasticsearch/ingest/ValueSourceTests.java
index 57a6eb7ee68..0ac353b4ae8 100644
--- a/core/src/test/java/org/elasticsearch/ingest/core/ValueSourceTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/ValueSourceTests.java
@@ -17,10 +17,8 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest.core;
+package org.elasticsearch.ingest;
 
-import org.elasticsearch.ingest.RandomDocumentPicks;
-import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
diff --git a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
index 92dac4bd784..b30d4c5af7f 100644
--- a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
+++ b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
@@ -33,7 +33,7 @@ import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.http.HttpInfo;
-import org.elasticsearch.ingest.core.IngestInfo;
+import org.elasticsearch.ingest.IngestInfo;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.monitor.os.DummyOsInfo;
 import org.elasticsearch.monitor.os.OsInfo;
diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle
index e7935cd2738..6d93f663116 100644
--- a/modules/ingest-common/build.gradle
+++ b/modules/ingest-common/build.gradle
@@ -19,7 +19,7 @@
 
 esplugin {
     description 'Module for ingest processors that do not require additional security permissions or have large dependencies and resources'
-    classname 'org.elasticsearch.ingest.IngestCommonPlugin'
+    classname 'org.elasticsearch.ingest.common.IngestCommonPlugin'
 }
 
 dependencies {
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/AbstractStringProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
similarity index 89%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/AbstractStringProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
index 2dc11d43af1..d9ffd34cac1 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/AbstractStringProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
@@ -17,12 +17,12 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.AbstractProcessor;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.ConfigurationUtils;
-import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestDocument;
 
 import java.util.Map;
 
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/AppendProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java
similarity index 87%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/AppendProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java
index 4019f8312ef..af163c3c187 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/AppendProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java
@@ -17,14 +17,14 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.AbstractProcessor;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.TemplateService;
-import org.elasticsearch.ingest.core.ValueSource;
-import org.elasticsearch.ingest.core.ConfigurationUtils;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.TemplateService;
+import org.elasticsearch.ingest.ValueSource;
 
 import java.util.Map;
 
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/ConvertProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
similarity index 94%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/ConvertProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
index d019ab01ef7..015c56c72c3 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/ConvertProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
@@ -17,19 +17,19 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.AbstractProcessor;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.ConfigurationUtils;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestDocument;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
-import static org.elasticsearch.ingest.core.ConfigurationUtils.newConfigurationException;
+import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
 
 /**
  * Processor that converts fields content to a different type. Supported types are: integer, float, boolean and string.
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/DateFormat.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java
similarity index 98%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/DateFormat.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java
index 2e27bd72cd2..4a97a7e7ab2 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/DateFormat.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateFormat.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/DateIndexNameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java
similarity index 95%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/DateIndexNameProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java
index cad66a3187b..a94d4d048a8 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/DateIndexNameProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java
@@ -17,13 +17,13 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.ingest.core.AbstractProcessor;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.ConfigurationUtils;
-import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestDocument;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.DateTimeFormat;
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/DateProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
similarity index 94%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/DateProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
index 273b42d2b8c..b82b9c8b76c 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/DateProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
@@ -17,13 +17,13 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.ingest.core.AbstractProcessor;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.ConfigurationUtils;
-import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestDocument;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.ISODateTimeFormat;
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/FailProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java
similarity index 86%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/FailProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java
index 2e0b79501e0..6c434d85d5a 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/FailProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java
@@ -17,13 +17,13 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.AbstractProcessor;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.ConfigurationUtils;
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.TemplateService;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.TemplateService;
 
 import java.util.Map;
 
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/FailProcessorException.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessorException.java
similarity index 79%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/FailProcessorException.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessorException.java
index bd010b1423c..9add5c5762f 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/FailProcessorException.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessorException.java
@@ -17,13 +17,16 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
+
+import org.elasticsearch.ingest.CompoundProcessor;
+import org.elasticsearch.ingest.IngestDocument;
 
 /**
  * Exception class thrown by {@link FailProcessor}.
  *
- * This exception is caught in the {@link org.elasticsearch.ingest.core.CompoundProcessor} and
- * then changes the state of {@link org.elasticsearch.ingest.core.IngestDocument}. This
+ * This exception is caught in the {@link CompoundProcessor} and
+ * then changes the state of {@link IngestDocument}. This
  * exception should get serialized.
  */
 public class FailProcessorException extends RuntimeException {
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/ForEachProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java
similarity index 87%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/ForEachProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java
index 2ca895779af..b6d14d1b8c5 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/ForEachProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java
@@ -17,13 +17,14 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.AbstractProcessor;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.ConfigurationUtils;
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.Processor;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.ProcessorsRegistry;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -31,8 +32,8 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.ingest.core.ConfigurationUtils.readList;
-import static org.elasticsearch.ingest.core.ConfigurationUtils.readStringProperty;
+import static org.elasticsearch.ingest.ConfigurationUtils.readList;
+import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
 
 /**
  * A processor that for each value in a list executes a one or more processors.
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/Grok.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Grok.java
similarity index 99%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/Grok.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Grok.java
index 7e5d1e6b516..b6e1877d3e0 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/Grok.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Grok.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.jcodings.specific.UTF8Encoding;
 import org.joni.Matcher;
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/GrokMatchGroup.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokMatchGroup.java
similarity index 97%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/GrokMatchGroup.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokMatchGroup.java
index 6fd979f179a..35f8fd312ff 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/GrokMatchGroup.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokMatchGroup.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 final class GrokMatchGroup {
     private static final String DEFAULT_TYPE = "string";
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/GrokProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
similarity index 94%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/GrokProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
index b709df6e7d4..aa9fbb905cc 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/GrokProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
@@ -17,19 +17,19 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.AbstractProcessor;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.ConfigurationUtils;
-import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestDocument;
 
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
-import static org.elasticsearch.ingest.core.ConfigurationUtils.newConfigurationException;
+import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
 
 public final class GrokProcessor extends AbstractProcessor {
 
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/GsubProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java
similarity index 89%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/GsubProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java
index 8b17766f845..72bc9e76710 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/GsubProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java
@@ -17,18 +17,18 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.AbstractProcessor;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.IngestDocument;
 
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import static org.elasticsearch.ingest.core.ConfigurationUtils.newConfigurationException;
-import static org.elasticsearch.ingest.core.ConfigurationUtils.readStringProperty;
+import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
+import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
 
 /**
  * Processor that allows to search for patterns in field content and replace them with corresponding string replacement.
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java
similarity index 99%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/IngestCommonPlugin.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java
index 92112a37b96..821e47b85bc 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/IngestCommonPlugin.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.node.NodeModule;
 import org.elasticsearch.plugins.Plugin;
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/JoinProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java
similarity index 90%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/JoinProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java
index 7db5e00b145..8114d20f28f 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/JoinProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java
@@ -17,12 +17,12 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.AbstractProcessor;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.ConfigurationUtils;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestDocument;
 
 import java.util.List;
 import java.util.Map;
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/LowercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java
similarity index 97%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/LowercaseProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java
index 8d5aebcf115..9f8ea7a5614 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/LowercaseProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import java.util.Locale;
 
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/RemoveProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java
similarity index 86%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/RemoveProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java
index 6670c462d6f..98c4e18a408 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/RemoveProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java
@@ -17,13 +17,13 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.AbstractProcessor;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.TemplateService;
-import org.elasticsearch.ingest.core.ConfigurationUtils;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.TemplateService;
 
 import java.util.Map;
 
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/RenameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java
similarity index 91%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/RenameProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java
index 4a6e8bf3ae5..9143321c4aa 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/RenameProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java
@@ -17,12 +17,12 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.AbstractProcessor;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.ConfigurationUtils;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestDocument;
 
 import java.util.Map;
 
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/ScriptProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
similarity index 91%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/ScriptProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
index 9051ffe408e..6ce85f8a6bf 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/ScriptProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
@@ -17,13 +17,13 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.ingest.core.AbstractProcessor;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
@@ -35,9 +35,9 @@ import java.util.Map;
 
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.common.Strings.hasLength;
-import static org.elasticsearch.ingest.core.ConfigurationUtils.newConfigurationException;
-import static org.elasticsearch.ingest.core.ConfigurationUtils.readOptionalStringProperty;
-import static org.elasticsearch.ingest.core.ConfigurationUtils.readStringProperty;
+import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
+import static org.elasticsearch.ingest.ConfigurationUtils.readOptionalStringProperty;
+import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
 import static org.elasticsearch.script.ScriptService.ScriptType.FILE;
 import static org.elasticsearch.script.ScriptService.ScriptType.INLINE;
 import static org.elasticsearch.script.ScriptService.ScriptType.STORED;
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/SetProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java
similarity index 89%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/SetProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java
index 6f576dcafc0..a78701645a9 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/SetProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java
@@ -17,14 +17,14 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.AbstractProcessor;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.TemplateService;
-import org.elasticsearch.ingest.core.ValueSource;
-import org.elasticsearch.ingest.core.ConfigurationUtils;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.TemplateService;
+import org.elasticsearch.ingest.ValueSource;
 
 import java.util.Map;
 
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/SortProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java
similarity index 93%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/SortProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java
index ee708d97d47..706a1cef9c5 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/SortProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java
@@ -17,12 +17,12 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.AbstractProcessor;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.ConfigurationUtils;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestDocument;
 
 import java.util.Collections;
 import java.util.List;
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/SplitProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java
similarity index 90%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/SplitProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java
index 2d43cae3f09..f7c5e8befc4 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/SplitProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java
@@ -17,12 +17,12 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.AbstractProcessor;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.ConfigurationUtils;
-import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestDocument;
 
 import java.util.ArrayList;
 import java.util.Collections;
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/TrimProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java
similarity index 97%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/TrimProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java
index 294cd59265f..a57a25125d6 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/TrimProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 /**
  * Processor that trims the content of string fields.
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/UppercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java
similarity index 97%
rename from modules/ingest-common/src/main/java/org/elasticsearch/ingest/UppercaseProcessor.java
rename to modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java
index cd51f0dbc71..a5c817352a1 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/UppercaseProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import java.util.Locale;
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/AbstractStringProcessorTestCase.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorTestCase.java
similarity index 95%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/AbstractStringProcessorTestCase.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorTestCase.java
index b1f517b3d88..e987510f25a 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/AbstractStringProcessorTestCase.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorTestCase.java
@@ -17,10 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.Processor;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Collections;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/AppendProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java
similarity index 95%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/AppendProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java
index 884bbb7903a..fbf77cc4285 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/AppendProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java
@@ -17,10 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/AppendProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorTests.java
similarity index 96%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/AppendProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorTests.java
index 63a1bb9cce2..16df08721d3 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/AppendProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorTests.java
@@ -17,12 +17,14 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.TemplateService;
-import org.elasticsearch.ingest.core.ValueSource;
-import org.elasticsearch.ingest.core.Processor;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.RandomDocumentPicks;
+import org.elasticsearch.ingest.TemplateService;
+import org.elasticsearch.ingest.TestTemplateService;
+import org.elasticsearch.ingest.ValueSource;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ConvertProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java
similarity index 97%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/ConvertProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java
index ce043f6aed4..1ec5362af14 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ConvertProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java
@@ -17,10 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ConvertProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
similarity index 98%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/ConvertProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
index f9ec7883190..02f859ecfa5 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ConvertProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java
@@ -17,10 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.Processor;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
@@ -30,7 +31,7 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
-import static org.elasticsearch.ingest.ConvertProcessor.Type;
+import static org.elasticsearch.ingest.common.ConvertProcessor.Type;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.sameInstance;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateFormatTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java
similarity index 98%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateFormatTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java
index 8fcff4789c7..0885371f867 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateFormatTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateFormatTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.test.ESTestCase;
 import org.joda.time.DateTime;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateIndexNameFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java
similarity index 99%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateIndexNameFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java
index d6b5f7a0ff7..42877236b88 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateIndexNameFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.test.ESTestCase;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateIndexNameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java
similarity index 97%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateIndexNameProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java
index 3baa3d3ac7c..c546f021823 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateIndexNameProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java
@@ -16,9 +16,9 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.test.ESTestCase;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java
similarity index 98%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java
index 9933f3fbb65..65dcdf6082c 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java
@@ -17,10 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 import org.joda.time.DateTimeZone;
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java
similarity index 98%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java
index 2d32282d332..768f402ee4b 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/DateProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorTests.java
@@ -17,9 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/FailProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java
similarity index 93%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/FailProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java
index 1f1e9c72b12..db16b78b316 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/FailProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java
@@ -17,10 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/FailProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorTests.java
similarity index 86%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/FailProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorTests.java
index 457aaf45c66..76685854d21 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/FailProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorTests.java
@@ -17,10 +17,12 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.Processor;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.RandomDocumentPicks;
+import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.test.ESTestCase;
 
 import static org.hamcrest.Matchers.equalTo;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ForEachProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java
similarity index 94%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/ForEachProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java
index 161b50c5e4c..d45e98ab06a 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ForEachProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java
@@ -17,10 +17,12 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.ingest.core.Processor;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.ProcessorsRegistry;
+import org.elasticsearch.ingest.TestProcessor;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ForEachProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java
similarity index 95%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/ForEachProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java
index 19f1e15f09e..05287935b49 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ForEachProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java
@@ -17,13 +17,15 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.CompoundProcessor;
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.Processor;
-import org.elasticsearch.ingest.core.TemplateService;
-import org.elasticsearch.ingest.core.ValueSource;
+import org.elasticsearch.ingest.CompoundProcessor;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.TemplateService;
+import org.elasticsearch.ingest.TestProcessor;
+import org.elasticsearch.ingest.TestTemplateService;
+import org.elasticsearch.ingest.ValueSource;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/GrokProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java
similarity index 98%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/GrokProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java
index 374a15f4f25..a7a133b4363 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/GrokProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java
@@ -17,10 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Collections;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/GrokProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java
similarity index 98%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/GrokProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java
index 919cd5ca548..f108e80e790 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/GrokProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java
@@ -17,10 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.RandomDocumentPicks;
-import org.elasticsearch.ingest.core.IngestDocument;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Arrays;
@@ -29,7 +29,6 @@ import java.util.HashMap;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.nullValue;
 
 
 public class GrokProcessorTests extends ESTestCase {
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/GrokTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokTests.java
similarity index 99%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/GrokTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokTests.java
index fd757c4962c..e8b88ea0d12 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/GrokTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/GsubProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java
similarity index 97%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/GsubProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java
index f840576e522..60cceb34024 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/GsubProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java
@@ -17,10 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/GsubProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorTests.java
similarity index 95%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/GsubProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorTests.java
index 066609f5e3a..20dc5b8c8de 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/GsubProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorTests.java
@@ -17,10 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.Processor;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Collections;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/IngestCommonRestIT.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonRestIT.java
similarity index 97%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/IngestCommonRestIT.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonRestIT.java
index 544e47b8e94..94418ac552c 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/IngestCommonRestIT.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonRestIT.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/JoinProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java
similarity index 96%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/JoinProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java
index 45e643579a4..970fd8b8b9a 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/JoinProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java
@@ -17,10 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/JoinProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorTests.java
similarity index 96%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/JoinProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorTests.java
index 39906fb4850..078d6365709 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/JoinProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorTests.java
@@ -17,10 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.Processor;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/LowercaseProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorFactoryTests.java
similarity index 95%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/LowercaseProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorFactoryTests.java
index b7a315b176b..4dec115458c 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/LowercaseProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorFactoryTests.java
@@ -17,10 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/LowercaseProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorTests.java
similarity index 96%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/LowercaseProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorTests.java
index 81e23ef4689..413b3f64650 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/LowercaseProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import java.util.Locale;
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/RemoveProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java
similarity index 93%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/RemoveProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java
index 672058b61e4..a5f88103e96 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/RemoveProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java
@@ -17,10 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/RemoveProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java
similarity index 90%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/RemoveProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java
index abc68aa60b6..fd569567d1a 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/RemoveProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java
@@ -17,10 +17,12 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.Processor;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.RandomDocumentPicks;
+import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/RenameProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java
similarity index 96%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/RenameProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java
index 005abb8c0d0..c078f09dd92 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/RenameProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java
@@ -17,10 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/RenameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java
similarity index 97%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/RenameProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java
index 85faca03e9c..dc4f7328791 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/RenameProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java
@@ -17,10 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.Processor;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ScriptProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
similarity index 98%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/ScriptProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
index e5cd4d3699e..59a5d7ceaa2 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ScriptProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.service.ClusterService;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ScriptProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java
similarity index 95%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/ScriptProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java
index 377b93ef454..6da127a798b 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/ScriptProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java
@@ -17,10 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/SetProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java
similarity index 96%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/SetProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java
index acb09b0c96f..b8c97a379cb 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/SetProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java
@@ -17,10 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/SetProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java
similarity index 94%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/SetProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java
index 1d28124d55a..9ef8fa44ac9 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/SetProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorTests.java
@@ -17,12 +17,14 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.TemplateService;
-import org.elasticsearch.ingest.core.ValueSource;
-import org.elasticsearch.ingest.core.Processor;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.RandomDocumentPicks;
+import org.elasticsearch.ingest.TemplateService;
+import org.elasticsearch.ingest.TestTemplateService;
+import org.elasticsearch.ingest.ValueSource;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/SortProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorTests.java
similarity index 98%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/SortProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorTests.java
index 40da784bf34..97352109e2c 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/SortProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorTests.java
@@ -17,11 +17,12 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.Processor;
-import org.elasticsearch.ingest.SortProcessor.SortOrder;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.RandomDocumentPicks;
+import org.elasticsearch.ingest.common.SortProcessor.SortOrder;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/SplitProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java
similarity index 96%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/SplitProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java
index 7d0c3c4d970..c747807b710 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/SplitProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java
@@ -17,10 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/SplitProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java
similarity index 96%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/SplitProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java
index 95bb4fe4109..13d45dc126b 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/SplitProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java
@@ -17,10 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.Processor;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Arrays;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/TrimProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorFactoryTests.java
similarity index 95%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/TrimProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorFactoryTests.java
index 0692d53c9e3..54904775478 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/TrimProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorFactoryTests.java
@@ -17,10 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/TrimProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorTests.java
similarity index 97%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/TrimProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorTests.java
index ce51dea3acb..4b776a2ee84 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/TrimProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 public class TrimProcessorTests extends AbstractStringProcessorTestCase {
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/UppercaseProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorFactoryTests.java
similarity index 95%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/UppercaseProcessorFactoryTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorFactoryTests.java
index 0b5eafd2173..cd4d1faf767 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/UppercaseProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorFactoryTests.java
@@ -17,10 +17,10 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/UppercaseProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorTests.java
similarity index 96%
rename from modules/ingest-common/src/test/java/org/elasticsearch/ingest/UppercaseProcessorTests.java
rename to modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorTests.java
index 6881d117946..ff8dc16b6e2 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/UppercaseProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.ingest;
+package org.elasticsearch.ingest.common;
 
 import java.util.Locale;
 
diff --git a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
index d47e03dcbfe..40ebe2592ab 100644
--- a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
+++ b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
@@ -24,9 +24,9 @@ import org.apache.tika.metadata.Metadata;
 import org.apache.tika.metadata.TikaCoreProperties;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.ingest.core.AbstractProcessor;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.IngestDocument;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -37,10 +37,10 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 
-import static org.elasticsearch.ingest.core.ConfigurationUtils.newConfigurationException;
-import static org.elasticsearch.ingest.core.ConfigurationUtils.readIntProperty;
-import static org.elasticsearch.ingest.core.ConfigurationUtils.readOptionalList;
-import static org.elasticsearch.ingest.core.ConfigurationUtils.readStringProperty;
+import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
+import static org.elasticsearch.ingest.ConfigurationUtils.readIntProperty;
+import static org.elasticsearch.ingest.ConfigurationUtils.readOptionalList;
+import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
 
 public final class AttachmentProcessor extends AbstractProcessor {
 
diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java
index 09fb4e3951d..6bd4e07702e 100644
--- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java
+++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.ingest.attachment;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java
index 56e72be351e..4e051d05724 100644
--- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java
+++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java
@@ -22,7 +22,7 @@ package org.elasticsearch.ingest.attachment;
 import org.apache.commons.io.IOUtils;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ingest.RandomDocumentPicks;
-import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
index 9d606419e57..c29bf4aa65d 100644
--- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
+++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
@@ -33,9 +33,9 @@ import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.network.NetworkAddress;
-import org.elasticsearch.ingest.core.AbstractProcessor;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.IngestDocument;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -51,9 +51,9 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 
-import static org.elasticsearch.ingest.core.ConfigurationUtils.newConfigurationException;
-import static org.elasticsearch.ingest.core.ConfigurationUtils.readOptionalList;
-import static org.elasticsearch.ingest.core.ConfigurationUtils.readStringProperty;
+import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
+import static org.elasticsearch.ingest.ConfigurationUtils.readOptionalList;
+import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
 
 public final class GeoIpProcessor extends AbstractProcessor {
 
@@ -96,7 +96,8 @@ public final class GeoIpProcessor extends AbstractProcessor {
                 }
                 break;
             default:
-                throw new ElasticsearchParseException("Unsupported database type [" + dbReader.getMetadata().getDatabaseType() + "]", new IllegalStateException());
+                throw new ElasticsearchParseException("Unsupported database type [" + dbReader.getMetadata().getDatabaseType()
+                        + "]", new IllegalStateException());
         }
         ingestDocument.setFieldValue(targetField, geoData);
     }
@@ -259,7 +260,8 @@ public final class GeoIpProcessor extends AbstractProcessor {
                 } else if (COUNTRY_DB_TYPE.equals(databaseType)) {
                     properties = DEFAULT_COUNTRY_PROPERTIES;
                 } else {
-                    throw newConfigurationException(TYPE, processorTag, "database_file", "Unsupported database type [" + databaseType + "]");
+                    throw newConfigurationException(TYPE, processorTag, "database_file", "Unsupported database type ["
+                            + databaseType + "]");
                 }
             }
 
@@ -312,7 +314,8 @@ public final class GeoIpProcessor extends AbstractProcessor {
                 }
                 return property;
             } catch (IllegalArgumentException e) {
-                throw new IllegalArgumentException("illegal property value [" + value + "]. valid values are " + Arrays.toString(validProperties.toArray()));
+                throw new IllegalArgumentException("illegal property value [" + value + "]. valid values are " +
+                        Arrays.toString(validProperties.toArray()));
             }
         }
     }
diff --git a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
index 25d8f1f6f5f..04729162729 100644
--- a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
+++ b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
@@ -23,7 +23,7 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import com.maxmind.geoip2.DatabaseReader;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Randomness;
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.StreamsUtils;
 import org.junit.AfterClass;
@@ -54,8 +54,10 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Path configDir = createTempDir();
         Path geoIpConfigDir = configDir.resolve("ingest-geoip");
         Files.createDirectories(geoIpConfigDir);
-        Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-City.mmdb.gz")), geoIpConfigDir.resolve("GeoLite2-City.mmdb.gz"));
-        Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-Country.mmdb.gz")), geoIpConfigDir.resolve("GeoLite2-Country.mmdb.gz"));
+        Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-City.mmdb.gz")),
+                geoIpConfigDir.resolve("GeoLite2-City.mmdb.gz"));
+        Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-Country.mmdb.gz")),
+                geoIpConfigDir.resolve("GeoLite2-Country.mmdb.gz"));
         databaseReaders = IngestGeoIpPlugin.loadDatabaseReaders(geoIpConfigDir);
     }
 
@@ -136,7 +138,8 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
             factory.create(config);
             fail("Exception expected");
         } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), equalTo("[properties] illegal property value [" + cityProperty + "]. valid values are [IP, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_NAME]"));
+            assertThat(e.getMessage(), equalTo("[properties] illegal property value [" + cityProperty +
+                    "]. valid values are [IP, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_NAME]"));
         }
     }
 
@@ -183,7 +186,8 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
             factory.create(config);
             fail("exception expected");
         } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [IP, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_NAME, REGION_NAME, CITY_NAME, TIMEZONE, LOCATION]"));
+            assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [IP, COUNTRY_ISO_CODE, " +
+                    "COUNTRY_NAME, CONTINENT_NAME, REGION_NAME, CITY_NAME, TIMEZONE, LOCATION]"));
         }
 
         config = new HashMap<>();
diff --git a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
index a64a42fa95e..f3141c735d2 100644
--- a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
+++ b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
@@ -21,7 +21,7 @@ package org.elasticsearch.ingest.geoip;
 
 import com.maxmind.geoip2.DatabaseReader;
 import org.elasticsearch.ingest.RandomDocumentPicks;
-import org.elasticsearch.ingest.core.IngestDocument;
+import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -38,7 +38,8 @@ public class GeoIpProcessorTests extends ESTestCase {
 
     public void testCity() throws Exception {
         InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
-        GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field", new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class));
+        GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field",
+                new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class));
 
         Map<String, Object> document = new HashMap<>();
         document.put("source_field", "8.8.8.8");
@@ -64,7 +65,8 @@ public class GeoIpProcessorTests extends ESTestCase {
 
     public void testCountry() throws Exception {
         InputStream database = getDatabaseFileInputStream("/GeoLite2-Country.mmdb.gz");
-        GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field", new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class));
+        GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field",
+                new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class));
 
         Map<String, Object> document = new HashMap<>();
         document.put("source_field", "82.170.213.79");
@@ -83,7 +85,8 @@ public class GeoIpProcessorTests extends ESTestCase {
 
     public void testAddressIsNotInTheDatabase() throws Exception {
         InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
-        GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field", new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class));
+        GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field",
+                new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class));
 
         Map<String, Object> document = new HashMap<>();
         document.put("source_field", "127.0.0.1");
@@ -97,7 +100,8 @@ public class GeoIpProcessorTests extends ESTestCase {
     /** Don't silently do DNS lookups or anything trappy on bogus data */
     public void testInvalid() throws Exception {
         InputStream database = getDatabaseFileInputStream("/GeoLite2-City.mmdb.gz");
-        GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field", new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class));
+        GeoIpProcessor processor = new GeoIpProcessor(randomAsciiOfLength(10), "source_field",
+                new DatabaseReader.Builder(database).build(), "target_field", EnumSet.allOf(GeoIpProcessor.Property.class));
 
         Map<String, Object> document = new HashMap<>();
         document.put("source_field", "www.google.com");
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java
index 77f09d4df37..1c60a32e883 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/AbstractScriptTestCase.java
@@ -21,7 +21,6 @@ package org.elasticsearch.ingest;
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.ingest.core.TemplateService;
 import org.elasticsearch.script.ScriptContextRegistry;
 import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptService;
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java
index 112f44b9f2c..345ccb0ddcd 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java
@@ -19,9 +19,6 @@
 
 package org.elasticsearch.ingest;
 
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.ValueSource;
-
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/TemplateServiceIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/TemplateServiceIT.java
index 98b215110b1..c13058311bb 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/TemplateServiceIT.java
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/TemplateServiceIT.java
@@ -19,8 +19,6 @@
 
 package org.elasticsearch.ingest;
 
-import org.elasticsearch.ingest.core.TemplateService;
-
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/ValueSourceMustacheIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/ValueSourceMustacheIT.java
index 854b615e9bb..884897885ca 100644
--- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/ValueSourceMustacheIT.java
+++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/ingest/ValueSourceMustacheIT.java
@@ -19,9 +19,6 @@
 
 package org.elasticsearch.ingest;
 
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.ValueSource;
-
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java b/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java
index 6dc920c6ff8..3be4cd3edf4 100644
--- a/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java
+++ b/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java
@@ -22,7 +22,6 @@ package org.elasticsearch.ingest;
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
-import org.elasticsearch.ingest.core.IngestDocument;
 
 import java.util.ArrayList;
 import java.util.HashMap;
diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java b/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java
index ae13174e7c1..e36d73a8d9f 100644
--- a/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java
+++ b/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java
@@ -19,10 +19,6 @@
 
 package org.elasticsearch.ingest;
 
-import org.elasticsearch.ingest.core.AbstractProcessorFactory;
-import org.elasticsearch.ingest.core.IngestDocument;
-import org.elasticsearch.ingest.core.Processor;
-
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/TestTemplateService.java b/test/framework/src/main/java/org/elasticsearch/ingest/TestTemplateService.java
index 9330db1bfcb..d44764fa8ac 100644
--- a/test/framework/src/main/java/org/elasticsearch/ingest/TestTemplateService.java
+++ b/test/framework/src/main/java/org/elasticsearch/ingest/TestTemplateService.java
@@ -19,8 +19,6 @@
 
 package org.elasticsearch.ingest;
 
-import org.elasticsearch.ingest.core.TemplateService;
-
 import java.util.Map;
 
 public class TestTemplateService implements TemplateService {

From 423291b6bcdafd736c749ff8eb14bc578d37b6d3 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Fri, 17 Jun 2016 19:08:40 +0200
Subject: [PATCH 0982/1311] Change default similarity to BM25

The default similarity was set to `classic` which refers to TFIDF and has not been moved after the upgrade to Lucene 6.

Though moving to BM25 could have some downside for queries that relies on coordination factor (match_query, multi_match_query) ?

relates #18944
---
 .../index/similarity/SimilarityService.java   |  6 ++--
 .../similarity/SimilarityServiceTests.java    | 16 +++++++--
 .../index-modules/similarity.asciidoc         | 34 +++++++++----------
 .../migration/migrate_5_0/search.asciidoc     |  4 +++
 4 files changed, 37 insertions(+), 23 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
index 865c0eb685b..4ba2e0d0a38 100644
--- a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
+++ b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
@@ -36,7 +36,7 @@ import java.util.function.BiFunction;
 
 public final class SimilarityService extends AbstractIndexComponent {
 
-    public final static String DEFAULT_SIMILARITY = "classic";
+    public final static String DEFAULT_SIMILARITY = "BM25";
     private final Similarity defaultSimilarity;
     private final Similarity baseSimilarity;
     private final Map<String, SimilarityProvider> similarities;
@@ -121,8 +121,8 @@ public final class SimilarityService extends AbstractIndexComponent {
         return similarities.get(name);
     }
 
-    public SimilarityProvider getDefaultSimilarity() {
-        return similarities.get("default");
+    Similarity getDefaultSimilarity() {
+        return defaultSimilarity;
     }
 
     static class PerFieldSimilarity extends PerFieldSimilarityWrapper {
diff --git a/core/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java b/core/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java
index edb337fd4e6..57d025128d8 100644
--- a/core/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/index/similarity/SimilarityServiceTests.java
@@ -18,6 +18,8 @@
  */
 package org.elasticsearch.index.similarity;
 
+import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
@@ -27,7 +29,15 @@ import org.elasticsearch.test.IndexSettingsModule;
 
 import java.util.Collections;
 
+import static org.hamcrest.Matchers.instanceOf;
+
 public class SimilarityServiceTests extends ESTestCase {
+    public void testDefaultSimilarity() {
+        Settings settings = Settings.builder().build();
+        IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", settings);
+        SimilarityService service = new SimilarityService(indexSettings, Collections.emptyMap());
+        assertThat(service.getDefaultSimilarity(), instanceOf(BM25Similarity.class));
+    }
 
     // Tests #16594
     public void testOverrideBuiltInSimilarity() {
@@ -53,10 +63,10 @@ public class SimilarityServiceTests extends ESTestCase {
     }
 
     // Tests #16594
-    public void testDefaultSimilarity() {
-        Settings settings = Settings.builder().put("index.similarity.default.type", "BM25").build();
+    public void testOverrideDefaultSimilarity() {
+        Settings settings = Settings.builder().put("index.similarity.default.type", "classic").build();
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", settings);
         SimilarityService service = new SimilarityService(indexSettings, Collections.emptyMap());
-        assertTrue(service.getDefaultSimilarity() instanceof BM25SimilarityProvider);
+        assertTrue(service.getDefaultSimilarity() instanceof ClassicSimilarity);
     }
 }
diff --git a/docs/reference/index-modules/similarity.asciidoc b/docs/reference/index-modules/similarity.asciidoc
index f1250ddf6c1..1833b45a9ba 100644
--- a/docs/reference/index-modules/similarity.asciidoc
+++ b/docs/reference/index-modules/similarity.asciidoc
@@ -47,25 +47,11 @@ Here we configure the DFRSimilarity so it can be referenced as
 [float]
 === Available similarities
 
-[float]
-[[classic-similarity]]
-==== Classic similarity
-
-The classic similarity that is based on the TF/IDF model. This
-similarity has the following option:
-
-`discount_overlaps`::
-    Determines whether overlap tokens (Tokens with
-    0 position increment) are ignored when computing norm. By default this
-    is true, meaning overlap tokens do not count when computing norms.
-
-Type name: `classic`
-
 [float]
 [[bm25]]
-==== BM25 similarity
+==== BM25 similarity (*default*)
 
-Another TF/IDF based similarity that has built-in tf normalization and
+TF/IDF based similarity that has built-in tf normalization and
 is supposed to work better for short fields (like names). See
 http://en.wikipedia.org/wiki/Okapi_BM25[Okapi_BM25] for more details.
 This similarity has the following options:
@@ -86,6 +72,20 @@ This similarity has the following options:
 
 Type name: `BM25`
 
+[float]
+[[classic-similarity]]
+==== Classic similarity
+
+The classic similarity that is based on the TF/IDF model. This
+similarity has the following option:
+
+`discount_overlaps`::
+    Determines whether overlap tokens (Tokens with
+    0 position increment) are ignored when computing norm. By default this
+    is true, meaning overlap tokens do not count when computing norms.
+
+Type name: `classic`
+
 [float]
 [[drf]]
 ==== DFR similarity
@@ -178,5 +178,5 @@ You can change the default similarity for all fields by putting the following se
 
 [source,js]
 --------------------------------------------------
-index.similarity.default.type: BM25
+index.similarity.default.type: classic
 --------------------------------------------------
diff --git a/docs/reference/migration/migrate_5_0/search.asciidoc b/docs/reference/migration/migrate_5_0/search.asciidoc
index 7ebd79e66ca..bb42bc20de5 100644
--- a/docs/reference/migration/migrate_5_0/search.asciidoc
+++ b/docs/reference/migration/migrate_5_0/search.asciidoc
@@ -196,3 +196,7 @@ The <<search-request-preference,search preference>> `_prefer_node` has
 been superseded by `_prefer_nodes`. By specifying a single node,
 `_prefer_nodes` provides the same functionality as `_prefer_node` but
 also supports specifying multiple nodes.
+
+==== Default similarity
+
+The default similarity has been changed to `BM25`.

From 9d685f6876abc0434ac46fcfdbdfe0cc6f3a3e0f Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Tue, 21 Jun 2016 12:05:44 +0200
Subject: [PATCH 0983/1311] Fix ut: remap default to classic similarity for
 indices created before 5.0.

---
 .../java/org/elasticsearch/index/mapper/core/TypeParsers.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java b/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java
index 8e6b538a2ae..3ada93acccb 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/TypeParsers.java
@@ -445,7 +445,7 @@ public class TypeParsers {
     private static SimilarityProvider resolveSimilarity(Mapper.TypeParser.ParserContext parserContext, String name, String value) {
         if (parserContext.indexVersionCreated().before(Version.V_5_0_0_alpha1) && "default".equals(value)) {
             // "default" similarity has been renamed into "classic" in 3.x.
-            value = SimilarityService.DEFAULT_SIMILARITY;
+            value = "classic";
         }
         SimilarityProvider similarityProvider = parserContext.getSimilarity(value);
         if (similarityProvider == null) {

From 8d5a5e500a70a68c31546d3d69a502dd46a51ddb Mon Sep 17 00:00:00 2001
From: Sakthipriyan Vairamani <thefourtheye@users.noreply.github.com>
Date: Tue, 21 Jun 2016 16:50:23 +0530
Subject: [PATCH 0984/1311] file is -> file name (#18994)

---
 docs/reference/index-modules/slowlog.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/index-modules/slowlog.asciidoc b/docs/reference/index-modules/slowlog.asciidoc
index 9ae78b36d28..217a55fb0b9 100644
--- a/docs/reference/index-modules/slowlog.asciidoc
+++ b/docs/reference/index-modules/slowlog.asciidoc
@@ -58,7 +58,7 @@ index_search_slow_log_file:
 === Index Slow log
 
 The indexing slow log, similar in functionality to the search slow
-log. The log file is ends with `_index_indexing_slowlog.log`. Log and
+log. The log file name ends with `_index_indexing_slowlog.log`. Log and
 the thresholds are configured in the elasticsearch.yml file in the same
 way as the search slowlog. Index slowlog sample:
 

From 3748abfd7a8864ff31a3fe8a90db44317ac04e9c Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 21 Jun 2016 07:38:31 -0400
Subject: [PATCH 0985/1311] Clarify building from sources docs in README

This commit clarifies a few issues with the building from sources docs
in the README:
 - modifies Gradle link to use https
 - version 2.13 of Gradle is required
 - the command "gradle assemble" is the preferred way to only create a
   distribution
 - fixes the specified location of the built distributions

Closes #18991
---
 README.textile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.textile b/README.textile
index d9549e5eb3c..69d3fd54767 100644
--- a/README.textile
+++ b/README.textile
@@ -200,11 +200,11 @@ We have just covered a very small portion of what Elasticsearch is all about. Fo
 
 h3. Building from Source
 
-Elasticsearch uses "Gradle":http://gradle.org for its build system. You'll need to have a modern version of Gradle installed - 2.8 should do.
+Elasticsearch uses "Gradle":https://gradle.org for its build system. You'll need to have a modern version of Gradle installed - 2.13 should do.
 
-In order to create a distribution, simply run the @gradle build@ command in the cloned directory.
+In order to create a distribution, simply run the @gradle assemble@ command in the cloned directory.
 
-The distribution for each project will be created under the @target/releases@ directory in that project.
+The distribution for each project will be created under the @build/distributions@ directory in that project.
 
 See the "TESTING":TESTING.asciidoc file for more information about
 running the Elasticsearch test suite.

From 5ad2fdaa8e13f95020a1fdfb2e007d185bf733ab Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Tue, 21 Jun 2016 13:59:11 +0200
Subject: [PATCH 0986/1311] inner_hits: Don't include `_id`, `_type` and
 `_index` keys in search response for inner hits

Closes #18091
---
 .../search/internal/InternalSearchHit.java    | 11 +--
 .../migration/migrate_5_0/search.asciidoc     |  3 +
 .../search/request/inner-hits.asciidoc        |  2 -
 .../test/search.inner_hits/10_basic.yaml      | 84 +++++++++++++++++++
 4 files changed, 93 insertions(+), 7 deletions(-)
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yaml

diff --git a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
index f67ed51b226..389b1ec89b6 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
@@ -439,13 +439,14 @@ public class InternalSearchHit implements SearchHit {
             builder.field("_shard", shard.shardId());
             builder.field("_node", shard.nodeIdText());
         }
-        if (shard != null) {
-            builder.field(Fields._INDEX, shard.indexText());
-        }
-        builder.field(Fields._TYPE, type);
-        builder.field(Fields._ID, id);
         if (nestedIdentity != null) {
             nestedIdentity.toXContent(builder, params);
+        } else {
+            if (shard != null) {
+                builder.field(Fields._INDEX, shard.indexText());
+            }
+            builder.field(Fields._TYPE, type);
+            builder.field(Fields._ID, id);
         }
         if (version != -1) {
             builder.field(Fields._VERSION, version);
diff --git a/docs/reference/migration/migrate_5_0/search.asciidoc b/docs/reference/migration/migrate_5_0/search.asciidoc
index bb42bc20de5..dd91089fcad 100644
--- a/docs/reference/migration/migrate_5_0/search.asciidoc
+++ b/docs/reference/migration/migrate_5_0/search.asciidoc
@@ -180,6 +180,9 @@ with inner hits defined inside the query dsl.
 * Source filtering for inner hits inside nested queries requires full field names instead of relative field names.
 This is now consistent for source filtering on other places in the search API.
 
+* Nested inner hits will now no longer include `_index`, `_type` and `_id` keys. For nested inner hits these values
+are always the same as the `_index`, `_type` and `_id` keys of the root search hit.
+
 ==== Query Profiler
 
 In the response for profiling queries, the `query_type` has been renamed to `type` and `lucene` has been renamed to
diff --git a/docs/reference/search/request/inner-hits.asciidoc b/docs/reference/search/request/inner-hits.asciidoc
index 7fe200292a9..345bc9abde2 100644
--- a/docs/reference/search/request/inner-hits.asciidoc
+++ b/docs/reference/search/request/inner-hits.asciidoc
@@ -118,8 +118,6 @@ An example of a response snippet that could be generated from the above search r
                  "total": ...,
                  "hits": [
                     {
-                       "_type": "question",
-                       "_id": "1",
                        "_nested": {
                           "field": "comments",
                           "offset": 2
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yaml
new file mode 100644
index 00000000000..66b36b58bb7
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yaml
@@ -0,0 +1,84 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: test
+        body:
+            mappings:
+              type_1: {
+                properties: {
+                  nested_field : {
+                    type: nested
+                  }
+                }
+              }
+              type_2: {}
+              type_3: {
+                _parent: {
+                  type: type_2
+                }
+              }
+
+---
+"Nested inner hits":
+    - do:
+        index:
+          index: test
+          type:  type_1
+          id:    1
+          body:  {
+            "nested_field" : [
+              {
+                "foo": "bar"
+              }
+            ]
+          }
+
+    - do:
+        indices.refresh: {}
+
+    - do:
+        search:
+          body: { "query" : { "nested" : { "path" : "nested_field", "query" : { "match_all" : {} }, "inner_hits" : {} } } }
+    - match: { hits.total: 1 }
+    - match: { hits.hits.0._index: "test" }
+    - match: { hits.hits.0._type: "type_1" }
+    - match: { hits.hits.0._id: "1" }
+    - is_false: hits.hits.0.inner_hits.nested_field.hits.hits.0._index
+    - is_false: hits.hits.0.inner_hits.nested_field.hits.hits.0._type
+    - is_false: hits.hits.0.inner_hits.nested_field.hits.hits.0._id
+    - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._nested.field: "nested_field" }
+    - match: { hits.hits.0.inner_hits.nested_field.hits.hits.0._nested.offset: 0 }
+    - is_false: hits.hits.0.inner_hits.nested_field.hits.hits.0._nested.child
+
+---
+"Parent/child inner hits":
+    - do:
+        index:
+          index: test
+          type:  type_2
+          id:    1
+          body:  {"foo": "bar"}
+
+    - do:
+        index:
+          index: test
+          type:  type_3
+          id:    1
+          parent: 1
+          body:  {"bar": "baz"}
+
+    - do:
+        indices.refresh: {}
+
+    - do:
+        search:
+          body: { "query" : { "has_child" : { "type" : "type_3", "query" : { "match_all" : {} }, "inner_hits" : {} } } }
+    - match: { hits.total: 1 }
+    - match: { hits.hits.0._index: "test" }
+    - match: { hits.hits.0._type: "type_2" }
+    - match: { hits.hits.0._id: "1" }
+    - match: { hits.hits.0.inner_hits.type_3.hits.hits.0._index: "test" }
+    - match: { hits.hits.0.inner_hits.type_3.hits.hits.0._type: "type_3" }
+    - match: { hits.hits.0.inner_hits.type_3.hits.hits.0._id: "1" }
+    - is_false: hits.hits.0.inner_hits.type_3.hits.hits.0._nested

From ba1d6907ab77ed05a38cea7198f65faa84e00c5a Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 20 Jun 2016 18:00:34 -0400
Subject: [PATCH 0987/1311] Quiet the logging of the docs tests

Significantly quiets the logging of the docs tests by:
1. Switching two log statements to debug level.
2. Only calling ESTestCase#afterIfFailed if the test failure wasn't
just assumptions being violated.
---
 .../java/org/elasticsearch/test/ESTestCase.java  | 16 ++++++++++++++--
 .../elasticsearch/test/rest/ESRestTestCase.java  |  4 ++--
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 172c99592df..e7e6eca5c88 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -73,6 +73,7 @@ import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Rule;
+import org.junit.internal.AssumptionViolatedException;
 import org.junit.rules.RuleChain;
 
 import java.io.IOException;
@@ -140,13 +141,24 @@ public abstract class ESTestCase extends LuceneTestCase {
         @Override
         protected void afterAlways(List<Throwable> errors) throws Throwable {
             if (errors != null && errors.isEmpty() == false) {
-                ESTestCase.this.afterIfFailed(errors);
+                boolean allAssumption = true;
+                for (Throwable error : errors) {
+                    if (false == error instanceof AssumptionViolatedException) {
+                        allAssumption = false;
+                        break;
+                    }
+                }
+                if (false == allAssumption) {
+                    ESTestCase.this.afterIfFailed(errors);
+                }
             }
             super.afterAlways(errors);
         }
     });
 
-    /** called when a test fails, supplying the errors it generated */
+    /**
+     * Called when a test fails, supplying the errors it generated. Not called when the test fails because assumptions are violated.
+     */
     protected void afterIfFailed(List<Throwable> errors) {
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index ab2c1d2afbe..a730e046335 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -386,11 +386,11 @@ public abstract class ESRestTestCase extends ESTestCase {
         }
 
         if (!testCandidate.getSetupSection().isEmpty()) {
-            logger.info("start setup test [{}]", testCandidate.getTestPath());
+            logger.debug("start setup test [{}]", testCandidate.getTestPath());
             for (DoSection doSection : testCandidate.getSetupSection().getDoSections()) {
                 doSection.execute(restTestExecutionContext);
             }
-            logger.info("end setup test [{}]", testCandidate.getTestPath());
+            logger.debug("end setup test [{}]", testCandidate.getTestPath());
         }
 
         restTestExecutionContext.clear();

From 80734c75b5f96fed862eb58c780bac861af74bee Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 21 Jun 2016 08:35:12 -0400
Subject: [PATCH 0988/1311] get things started

---
 .../elasticsearch/painless/Augmentation.java  | 126 ++++++++++++++++++
 .../elasticsearch/painless/Definition.java    | 126 ++++++++++++------
 .../painless/node/LCallInvoke.java            |  10 +-
 .../painless/node/LListShortcut.java          |  12 +-
 .../painless/node/LMapShortcut.java           |  12 +-
 .../painless/node/LShortcut.java              |  12 +-
 .../elasticsearch/painless/node/SEach.java    |   4 +-
 .../painless/node/SFunction.java              |   2 +-
 .../org/elasticsearch/painless/java.lang.txt  |   2 +
 .../painless/java.util.regex.txt              |   2 +-
 .../org/elasticsearch/painless/java.util.txt  |   3 +-
 .../painless/AugmentationTests.java           |  35 +++++
 12 files changed, 262 insertions(+), 84 deletions(-)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/Augmentation.java
 create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Augmentation.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Augmentation.java
new file mode 100644
index 00000000000..456642eb8c5
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Augmentation.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.function.ObjIntConsumer;
+import java.util.function.Predicate;
+import java.util.function.ToDoubleFunction;
+import java.util.regex.Matcher;
+
+public class Augmentation {
+    public static <T> int getLength(List<T> receiver) {
+        return receiver.size();
+    }
+    
+    public static String namedGroup(Matcher receiver, String name) {
+        return receiver.group(name);
+    }
+    
+    public static <T> boolean any(Iterable<T> receiver, Predicate<T> predicate) {
+        for (T t : receiver) {
+            if (predicate.test(t)) {
+                return true;
+            }
+        }
+        return false;
+    }
+    
+    public static <T> int count(Iterable<T> receiver, Predicate<T> predicate) {
+        int count = 0;
+        for (T t : receiver) {
+            if (predicate.test(t)) {
+                count++;
+            }
+        }
+        return count;
+    }
+    
+    public static <T> Iterable<T> each(Iterable<T> receiver, Consumer<T> consumer) {
+        receiver.forEach(consumer);
+        return receiver;
+    }
+    
+    public static <T> Iterable<T> eachWithIndex(Iterable<T> receiver, ObjIntConsumer<T> consumer) {
+        int count = 0;
+        for (T t : receiver) {
+            consumer.accept(t, count++);
+        }
+        return receiver;
+    }
+    
+    public static <T> boolean every(Iterable<T> receiver, Predicate<T> predicate) {
+        for (T t : receiver) {
+            if (predicate.test(t) == false) {
+                return false;
+            }
+        }
+        return true;
+    }
+    
+    public static <T,U> List<U> findResults(Iterable<T> receiver, Function<T,U> filter) {
+        List<U> list = new ArrayList<>();
+        for (T t: receiver) {
+           U result = filter.apply(t);
+           if (result != null) {
+               list.add(result);
+           }
+        }
+        return list;
+    }
+    
+    public static <T,U> Map<U,List<T>> groupBy(Iterable<T> receiver, Function<T,U> mapper) {
+        Map<U,List<T>> map = new LinkedHashMap<>();
+        for (T t : receiver) {
+            U mapped = mapper.apply(t);
+            List<T> results = map.get(mapped);
+            if (results == null) {
+                results = new ArrayList<>();
+                map.put(mapped, results);
+            }
+            results.add(t);
+        }
+        return map;
+    }
+    
+    public static <T> String join(Iterable<T> receiver, String separator) {
+        StringBuilder sb = new StringBuilder();
+        for (T t : receiver) {
+            if (sb.length() > 0) {
+                sb.append(separator);
+            }
+            sb.append(t);
+        }
+        return sb.toString();
+    }
+    
+    public static <T> double sum(Iterable<T> receiver, ToDoubleFunction<T> function) {
+        double sum = 0;
+        for (T t : receiver) {
+            sum += function.applyAsDouble(t);
+        }
+        return sum;
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 761a2afeeb1..376be44e28b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -186,15 +186,17 @@ public final class Definition {
     public static class Method {
         public final String name;
         public final Struct owner;
+        public final boolean augmentation;
         public final Type rtn;
         public final List<Type> arguments;
         public final org.objectweb.asm.commons.Method method;
         public final int modifiers;
         public final MethodHandle handle;
 
-        public Method(String name, Struct owner, Type rtn, List<Type> arguments,
+        public Method(String name, Struct owner, boolean augmentation, Type rtn, List<Type> arguments,
                       org.objectweb.asm.commons.Method method, int modifiers, MethodHandle handle) {
             this.name = name;
+            this.augmentation = augmentation;
             this.owner = owner;
             this.rtn = rtn;
             this.arguments = Collections.unmodifiableList(arguments);
@@ -217,7 +219,15 @@ public final class Definition {
             // otherwise compute it
             final Class<?> params[];
             final Class<?> returnValue;
-            if (Modifier.isStatic(modifiers)) {
+            if (augmentation) {
+                // virtual/interface method disguised as static
+                params = new Class<?>[1 + arguments.size()];
+                params[0] = Augmentation.class;
+                for (int i = 0; i < arguments.size(); i++) {
+                    params[i + 1] = arguments.get(i).clazz;
+                }
+                returnValue = rtn.clazz;
+            } else if (Modifier.isStatic(modifiers)) {
                 // static method: straightforward copy
                 params = new Class<?>[arguments.size()];
                 for (int i = 0; i < arguments.size(); i++) {
@@ -242,6 +252,24 @@ public final class Definition {
             }
             return MethodType.methodType(returnValue, params);
         }
+        
+        public void write(MethodWriter writer) {
+            final org.objectweb.asm.Type type;
+            if (augmentation) {
+                assert java.lang.reflect.Modifier.isStatic(modifiers);
+                type = org.objectweb.asm.Type.getType(Augmentation.class);
+            } else {
+                type = owner.type;
+            }
+
+            if (java.lang.reflect.Modifier.isStatic(modifiers)) {
+                writer.invokeStatic(type, method);
+            } else if (java.lang.reflect.Modifier.isInterface(owner.clazz.getModifiers())) {
+                writer.invokeInterface(type, method);
+            } else {
+                writer.invokeVirtual(type, method);
+            }
+        }
     }
 
     public static final class Field {
@@ -690,7 +718,7 @@ public final class Definition {
                 " with arguments " + Arrays.toString(classes) + ".");
         }
 
-        final Method constructor = new Method(name, owner, returnType, Arrays.asList(args), asm, reflect.getModifiers(), handle);
+        final Method constructor = new Method(name, owner, false, returnType, Arrays.asList(args), asm, reflect.getModifiers(), handle);
 
         owner.constructors.put(methodKey, constructor);
     }
@@ -734,24 +762,20 @@ public final class Definition {
                 }
                 addConstructorInternal(className, "<init>", args);
             } else {
-                if (methodName.indexOf('/') >= 0) {
-                    String nameAndAlias[] = methodName.split("/");
-                    if (nameAndAlias.length != 2) {
-                        throw new IllegalArgumentException("Currently only two aliases are allowed!");
-                    }
-                    addMethodInternal(className, nameAndAlias[0], nameAndAlias[1], rtn, args);
+                if (methodName.indexOf("*") >= 0) {
+                    addMethodInternal(className, methodName.substring(0, methodName.length() - 1), true, rtn, args);
                 } else {
-                    addMethodInternal(className, methodName, null, rtn, args);
+                    addMethodInternal(className, methodName, false, rtn, args);
                 }
             }
         } else {
             // field
-            addFieldInternal(className, elements[1], null, rtn);
+            addFieldInternal(className, elements[1], rtn);
         }
     }
 
-    private final void addMethodInternal(final String struct, final String name, final String alias,
-                                         final Type rtn, final Type[] args) {
+    private final void addMethodInternal(String struct, String name, boolean augmentation,
+                                         Type rtn, Type[] args) {
         final Struct owner = structsMap.get(struct);
 
         if (owner == null) {
@@ -777,20 +801,32 @@ public final class Definition {
                 "Duplicate method signature [" + methodKey + "] found within the struct [" + owner.name + "].");
         }
 
-        final Class<?>[] classes = new Class<?>[args.length];
-
-        for (int count = 0; count < classes.length; ++count) {
-            classes[count] = args[count].clazz;
+        final Class<?> implClass;
+        final Class<?>[] params;
+        
+        if (augmentation == false) {
+            implClass = owner.clazz;
+            params = new Class<?>[args.length];
+            for (int count = 0; count < args.length; ++count) {
+                params[count] = args[count].clazz;
+            }
+        } else {
+            implClass = Augmentation.class;
+            params = new Class<?>[args.length + 1];
+            params[0] = owner.clazz;
+            for (int count = 0; count < args.length; ++count) {
+                params[count+1] = args[count].clazz;
+            }
         }
-
+        
         final java.lang.reflect.Method reflect;
 
         try {
-            reflect = owner.clazz.getMethod(alias == null ? name : alias, classes);
-        } catch (final NoSuchMethodException exception) {
-            throw new IllegalArgumentException("Method [" + (alias == null ? name : alias) +
+            reflect = implClass.getMethod(name, params);
+        } catch (NoSuchMethodException exception) {
+            throw new IllegalArgumentException("Method [" + name +
                 "] not found for class [" + owner.clazz.getName() + "]" +
-                " with arguments " + Arrays.toString(classes) + ".");
+                " with arguments " + Arrays.toString(params) + ".");
         }
 
         if (!reflect.getReturnType().equals(rtn.clazz)) {
@@ -805,25 +841,24 @@ public final class Definition {
         MethodHandle handle;
 
         try {
-            handle = MethodHandles.publicLookup().in(owner.clazz).unreflect(reflect);
+            handle = MethodHandles.publicLookup().in(implClass).unreflect(reflect);
         } catch (final IllegalAccessException exception) {
-            throw new IllegalArgumentException("Method [" + (alias == null ? name : alias) + "]" +
-                " not found for class [" + owner.clazz.getName() + "]" +
-                " with arguments " + Arrays.toString(classes) + ".");
+            throw new IllegalArgumentException("Method [" + name + "]" +
+                " not found for class [" + implClass.getName() + "]" +
+                " with arguments " + Arrays.toString(params) + ".");
         }
 
         final int modifiers = reflect.getModifiers();
-        final Method method = new Method(name, owner, rtn, Arrays.asList(args), asm, modifiers, handle);
+        final Method method = new Method(name, owner, augmentation, rtn, Arrays.asList(args), asm, modifiers, handle);
 
-        if (java.lang.reflect.Modifier.isStatic(modifiers)) {
+        if (augmentation == false && java.lang.reflect.Modifier.isStatic(modifiers)) {
             owner.staticMethods.put(methodKey, method);
         } else {
             owner.methods.put(methodKey, method);
         }
     }
 
-    private final void addFieldInternal(final String struct, final String name, final String alias,
-                                        final Type type) {
+    private final void addFieldInternal(String struct, String name, Type type) {
         final Struct owner = structsMap.get(struct);
 
         if (owner == null) {
@@ -844,9 +879,9 @@ public final class Definition {
         java.lang.reflect.Field reflect;
 
         try {
-            reflect = owner.clazz.getField(alias == null ? name : alias);
+            reflect = owner.clazz.getField(name);
         } catch (final NoSuchFieldException exception) {
-            throw new IllegalArgumentException("Field [" + (alias == null ? name : alias) + "]" +
+            throw new IllegalArgumentException("Field [" + name + "]" +
                 " not found for class [" + owner.clazz.getName() + "].");
         }
 
@@ -862,7 +897,7 @@ public final class Definition {
                 setter = MethodHandles.publicLookup().unreflectSetter(reflect);
             }
         } catch (final IllegalAccessException exception) {
-            throw new IllegalArgumentException("Getter/Setter [" + (alias == null ? name : alias) + "]" +
+            throw new IllegalArgumentException("Getter/Setter [" + name + "]" +
                 " not found for class [" + owner.clazz.getName() + "].");
         }
 
@@ -875,9 +910,9 @@ public final class Definition {
                     " within the struct [" + owner.name + "] is not final.");
             }
 
-            owner.staticMembers.put(alias == null ? name : alias, field);
+            owner.staticMembers.put(name, field);
         } else {
-            owner.members.put(alias == null ? name : alias, field);
+            owner.members.put(name, field);
         }
     }
 
@@ -915,11 +950,24 @@ public final class Definition {
                         // https://bugs.openjdk.java.net/browse/JDK-8072746
                     } else {
                         try {
-                            Class<?> arguments[] = new Class<?>[method.arguments.size()];
-                            for (int i = 0; i < method.arguments.size(); i++) {
-                                arguments[i] = method.arguments.get(i).clazz;
+                            // TODO: we *have* to remove all these public members and use getter methods to encapsulate!
+                            final Class<?> impl;
+                            final Class<?> arguments[];
+                            if (method.augmentation) {
+                                impl = Augmentation.class;
+                                arguments = new Class<?>[method.arguments.size() + 1];
+                                arguments[0] = method.owner.clazz;
+                                for (int i = 0; i < method.arguments.size(); i++) {
+                                    arguments[i + 1] = method.arguments.get(i).clazz;
+                                }
+                            } else {
+                                impl = owner.clazz;
+                                arguments = new Class<?>[method.arguments.size()];
+                                for (int i = 0; i < method.arguments.size(); i++) {
+                                    arguments[i] = method.arguments.get(i).clazz;
+                                }
                             }
-                            java.lang.reflect.Method m = owner.clazz.getMethod(method.method.getName(), arguments);
+                            java.lang.reflect.Method m = impl.getMethod(method.method.getName(), arguments);
                             if (m.getReturnType() != method.rtn.clazz) {
                                 throw new IllegalStateException("missing covariant override for: " + m + " in " + owner.name);
                             }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallInvoke.java
index 21bfff65e8d..1056af2aaca 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallInvoke.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LCallInvoke.java
@@ -122,14 +122,8 @@ public final class LCallInvoke extends ALink {
         for (AExpression argument : arguments) {
             argument.write(writer, globals);
         }
-
-        if (java.lang.reflect.Modifier.isStatic(method.modifiers)) {
-            writer.invokeStatic(method.owner.type, method.method);
-        } else if (java.lang.reflect.Modifier.isInterface(method.owner.clazz.getModifiers())) {
-            writer.invokeInterface(method.owner.type, method.method);
-        } else {
-            writer.invokeVirtual(method.owner.type, method.method);
-        }
+        
+        method.write(writer);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
index f2863ce3396..6ef8aedb0bf 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LListShortcut.java
@@ -92,11 +92,7 @@ final class LListShortcut extends ALink {
     void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
-        if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
-            writer.invokeInterface(getter.owner.type, getter.method);
-        } else {
-            writer.invokeVirtual(getter.owner.type, getter.method);
-        }
+        getter.write(writer);
 
         if (!getter.rtn.clazz.equals(getter.handle.type().returnType())) {
             writer.checkCast(getter.rtn.type);
@@ -107,11 +103,7 @@ final class LListShortcut extends ALink {
     void store(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
-        if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
-            writer.invokeInterface(setter.owner.type, setter.method);
-        } else {
-            writer.invokeVirtual(setter.owner.type, setter.method);
-        }
+        setter.write(writer);
 
         writer.writePop(setter.rtn.sort.size);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
index 3bc9ab57a37..52d66b0fe75 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LMapShortcut.java
@@ -91,11 +91,7 @@ final class LMapShortcut extends ALink {
     void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
-        if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
-            writer.invokeInterface(getter.owner.type, getter.method);
-        } else {
-            writer.invokeVirtual(getter.owner.type, getter.method);
-        }
+        getter.write(writer);
 
         if (!getter.rtn.clazz.equals(getter.handle.type().returnType())) {
             writer.checkCast(getter.rtn.type);
@@ -106,11 +102,7 @@ final class LMapShortcut extends ALink {
     void store(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
-        if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
-            writer.invokeInterface(setter.owner.type, setter.method);
-        } else {
-            writer.invokeVirtual(setter.owner.type, setter.method);
-        }
+        setter.write(writer);
 
         writer.writePop(setter.rtn.sort.size);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
index 7f97b446382..6d669adc658 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LShortcut.java
@@ -95,11 +95,7 @@ final class LShortcut extends ALink {
     void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
-        if (java.lang.reflect.Modifier.isInterface(getter.owner.clazz.getModifiers())) {
-            writer.invokeInterface(getter.owner.type, getter.method);
-        } else {
-            writer.invokeVirtual(getter.owner.type, getter.method);
-        }
+        getter.write(writer);
 
         if (!getter.rtn.clazz.equals(getter.handle.type().returnType())) {
             writer.checkCast(getter.rtn.type);
@@ -110,11 +106,7 @@ final class LShortcut extends ALink {
     void store(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
-        if (java.lang.reflect.Modifier.isInterface(setter.owner.clazz.getModifiers())) {
-            writer.invokeInterface(setter.owner.type, setter.method);
-        } else {
-            writer.invokeVirtual(setter.owner.type, setter.method);
-        }
+        setter.write(writer);
 
         writer.writePop(setter.rtn.sort.size);
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
index a90baac3203..772c4af4c48 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java
@@ -206,10 +206,8 @@ public class SEach extends AStatement {
             Type itr = Definition.getType("Iterator");
             org.objectweb.asm.Type methodType = org.objectweb.asm.Type.getMethodType(itr.type, Definition.DEF_TYPE.type);
             writer.invokeDefCall("iterator", methodType, DefBootstrap.ITERATOR);
-        } else if (java.lang.reflect.Modifier.isInterface(method.owner.clazz.getModifiers())) {
-            writer.invokeInterface(method.owner.type, method.method);
         } else {
-            writer.invokeVirtual(method.owner.type, method.method);
+            method.write(writer);
         }
 
         writer.visitVarInsn(iterator.type.type.getOpcode(Opcodes.ISTORE), iterator.getSlot());
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index ba078d03dbd..46dc8af5ab9 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -114,7 +114,7 @@ public class SFunction extends AStatement {
 
         org.objectweb.asm.commons.Method method =
             new org.objectweb.asm.commons.Method(name, MethodType.methodType(rtnType.clazz, paramClasses).toMethodDescriptorString());
-        this.method = new Method(name, null, rtnType, paramTypes, method, Modifier.STATIC | Modifier.PRIVATE, null);
+        this.method = new Method(name, null, false, rtnType, paramTypes, method, Modifier.STATIC | Modifier.PRIVATE, null);
     }
 
     @Override
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
index ec1a46e633f..859a7b314dd 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
@@ -50,6 +50,8 @@ class Iterable -> java.lang.Iterable {
   void forEach(Consumer)
   Iterator iterator()
   Spliterator spliterator()
+  # some adaptations of groovy methods
+  boolean any*(Predicate)
 }
 
 # Readable: i/o
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt
index 6befc865731..aaea78a7a96 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.regex.txt
@@ -42,7 +42,7 @@ class Matcher -> java.util.regex.Matcher extends Object {
   boolean find(int)
   String group()
   String group(int)
-  String namedGroup/group(String)
+  String namedGroup*(String)
   int groupCount()
   boolean hasAnchoringBounds()
   boolean hasTransparentBounds()
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt
index cccbf60f1a3..2a41b2c2c13 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt
@@ -114,8 +114,7 @@ class List -> java.util.List extends Collection,Iterable {
   def remove(int)
   void replaceAll(UnaryOperator)
   def set(int,def)
-  # TODO: wtf?
-  int getLength/size()
+  int getLength*()
   void sort(Comparator)
   List subList(int,int)
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java
new file mode 100644
index 00000000000..ccc3620b1c9
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+public class AugmentationTests extends ScriptTestCase {
+    
+    @AwaitsFix(bugUrl = "rmuir is working on this")
+    public void testCapturingReference() {
+        assertEquals(1, exec("int foo(Supplier t) { return t.get() }" +
+                             "def l = new ArrayList(); l.add(1);" +
+                             "return foo(l::getLength);"));
+    }
+    
+    
+    public void testIterable_Any() {
+        assertEquals(true, exec("List l = new ArrayList(); l.add(1); l.any(x -> x == 1)"));
+    }
+}

From 0160d91c2c4d7a0df52e6ed2212d9d0bac6c5375 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Tue, 21 Jun 2016 15:18:59 +0200
Subject: [PATCH 0989/1311] Removed docs for precision_step - no longer used

---
 docs/reference/mapping/params.asciidoc        |  2 -
 .../mapping/params/precision-step.asciidoc    | 56 -------------------
 .../mapping/types/geo-point.asciidoc          |  6 --
 .../mapping/types/token-count.asciidoc        |  5 --
 4 files changed, 69 deletions(-)
 delete mode 100644 docs/reference/mapping/params/precision-step.asciidoc

diff --git a/docs/reference/mapping/params.asciidoc b/docs/reference/mapping/params.asciidoc
index d0591a7479f..5134ab6733a 100644
--- a/docs/reference/mapping/params.asciidoc
+++ b/docs/reference/mapping/params.asciidoc
@@ -80,8 +80,6 @@ include::params/null-value.asciidoc[]
 
 include::params/position-increment-gap.asciidoc[]
 
-include::params/precision-step.asciidoc[]
-
 include::params/properties.asciidoc[]
 
 include::params/search-analyzer.asciidoc[]
diff --git a/docs/reference/mapping/params/precision-step.asciidoc b/docs/reference/mapping/params/precision-step.asciidoc
deleted file mode 100644
index a7ea4d342e7..00000000000
--- a/docs/reference/mapping/params/precision-step.asciidoc
+++ /dev/null
@@ -1,56 +0,0 @@
-[[precision-step]]
-=== `precision_step`
-
-Most <<number,numeric>> datatypes index extra terms representing numeric
-ranges for each number to make <<query-dsl-range-query,`range` queries>>
-faster.  For instance, this `range` query:
-
-[source,js]
---------------------------------------------------
-  "range": {
-    "number": {
-      "gte": 0
-      "lte": 321
-    }
-  }
---------------------------------------------------
-
-might be executed internally as a <<query-dsl-terms-query,`terms` query>> that
-looks something like this:
-
-[source,js]
---------------------------------------------------
-  "terms": {
-    "number": [
-      "0-255",
-      "256-319"
-      "320",
-      "321"
-    ]
-  }
---------------------------------------------------
-
-These extra terms greatly reduce the number of terms that have to be examined,
-at the cost of increased disk space.
-
-The default value for `precision_step` depends on the `type` of the numeric field:
-
-[horizontal]
-`long`, `double`, `date`, `ip`::  `16` (3 extra terms)
-`integer`, `float`, `short`::     `8` (3 extra terms)
-`byte`::                          `2147483647` (0 extra terms)
-`token_count`::                   `32` (0 extra terms)
-
-The value of the `precision_step` setting indicates the number of bits that
-should be compressed into an extra term.  A `long` value consists of 64 bits,
-so a `precision_step` of 16 results in the following terms:
-
-[horizontal]
-Bits 0-15:: `value & 1111111111111111 0000000000000000 0000000000000000 0000000000000000`
-Bits 0-31:: `value & 1111111111111111 1111111111111111 0000000000000000 0000000000000000`
-Bits 0-47:: `value & 1111111111111111 1111111111111111 1111111111111111 0000000000000000`
-Bits 0-63:: `value`
-
-
-
-
diff --git a/docs/reference/mapping/types/geo-point.asciidoc b/docs/reference/mapping/types/geo-point.asciidoc
index 909d9444527..cf2ae13a99c 100644
--- a/docs/reference/mapping/types/geo-point.asciidoc
+++ b/docs/reference/mapping/types/geo-point.asciidoc
@@ -126,12 +126,6 @@ The following parameters are accepted by `geo_point` fields:
     Should the geo-point also be indexed as `.lat` and `.lon` sub-fields?
     Accepts `true` and `false` (default).
 
-<<precision-step,`precision_step`>>::
-
-    Controls the number of extra terms that are indexed for each lat/lon point.
-    Defaults to `16`.  Ignored if `lat_lon` is `false`.
-
-
 ==== Using geo-points in scripts
 
 When accessing the value of a geo-point in a script, the value is returned as
diff --git a/docs/reference/mapping/types/token-count.asciidoc b/docs/reference/mapping/types/token-count.asciidoc
index c472c7c7770..49e4e3b424f 100644
--- a/docs/reference/mapping/types/token-count.asciidoc
+++ b/docs/reference/mapping/types/token-count.asciidoc
@@ -96,11 +96,6 @@ The following parameters are accepted by `token_count` fields:
     substituted for any explicit `null` values.  Defaults to `null`, which
     means the field is treated as missing.
 
-<<precision-step,`precision_step`>>::
-
-    Controls the number of extra terms that are indexed to make
-    <<query-dsl-range-query,`range` queries>> faster. Defaults to `32`.
-
 <<mapping-store,`store`>>::
 
     Whether the field value should be stored and retrievable separately from

From cb4bfcb864576df8b8735a107783c97f346b9f53 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Tue, 21 Jun 2016 15:29:59 +0200
Subject: [PATCH 0990/1311] Take SniffOnFailureListener out of Sniffer and make
 FailureListener final on RestClient

---
 .../client/sniff/SniffOnFailureListener.java  | 65 +++++++++++++++++++
 .../elasticsearch/client/sniff/Sniffer.java   | 50 ++++++--------
 .../client/sniff/MockHostsSniffer.java        | 39 +++++++++++
 .../sniff/SniffOnFailureListenerTests.java    | 57 ++++++++++++++++
 .../client/sniff/SnifferBuilderTests.java     | 18 -----
 .../org/elasticsearch/client/RestClient.java  | 28 +++++---
 .../client/RestClientBuilderTests.java        |  7 ++
 .../client/RestClientMultipleHostsTests.java  |  6 +-
 .../client/RestClientSingleHostTests.java     |  4 +-
 9 files changed, 209 insertions(+), 65 deletions(-)
 create mode 100644 client-sniffer/src/main/java/org/elasticsearch/client/sniff/SniffOnFailureListener.java
 create mode 100644 client-sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java
 create mode 100644 client-sniffer/src/test/java/org/elasticsearch/client/sniff/SniffOnFailureListenerTests.java

diff --git a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/SniffOnFailureListener.java b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/SniffOnFailureListener.java
new file mode 100644
index 00000000000..76350057141
--- /dev/null
+++ b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/SniffOnFailureListener.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.sniff;
+
+import org.apache.http.HttpHost;
+import org.elasticsearch.client.RestClient;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * {@link org.elasticsearch.client.RestClient.FailureListener} implementation that allows to perform
+ * sniffing on failure. Gets notified whenever a failure happens and uses a {@link Sniffer} instance
+ * to manually reload hosts and sets them back to the {@link RestClient}. The {@link Sniffer} instance
+ * needs to be lazily set through {@link #setSniffer(Sniffer)}.
+ */
+public class SniffOnFailureListener extends RestClient.FailureListener {
+
+    private volatile Sniffer sniffer;
+    private final AtomicBoolean set;
+
+    public SniffOnFailureListener() {
+        this.set = new AtomicBoolean(false);
+    }
+
+    /**
+     * Sets the {@link Sniffer} instance used to perform sniffing
+     * @throws IllegalStateException if the sniffer was already set, as it can only be set once
+     */
+    public void setSniffer(Sniffer sniffer) {
+        Objects.requireNonNull(sniffer, "sniffer must not be null");
+        if (set.compareAndSet(false, true)) {
+            this.sniffer = sniffer;
+        } else {
+            throw new IllegalStateException("sniffer can only be set once");
+        }
+    }
+
+    @Override
+    public void onFailure(HttpHost host) throws IOException {
+        if (sniffer == null) {
+            throw new IllegalStateException("sniffer was not set, unable to sniff on failure");
+        }
+        //re-sniff immediately but take out the node that failed
+        sniffer.sniffOnFailure(host);
+    }
+}
diff --git a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
index c1aeb3efe36..74a28cdd222 100644
--- a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
+++ b/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
@@ -37,30 +37,26 @@ import java.util.concurrent.atomic.AtomicBoolean;
 /**
  * Class responsible for sniffing nodes from an elasticsearch cluster and setting them to a provided instance of {@link RestClient}.
  * Must be created via {@link Builder}, which allows to set all of the different options or rely on defaults.
- * A background task fetches the nodes from elasticsearch and updates them periodically.
- * Supports sniffing on failure, meaning that the client will notify the sniffer at each host failure, so that nodes can be updated
- * straightaway.
+ * A background task fetches the nodes through the {@link HostsSniffer} and sets them to the {@link RestClient} instance.
+ * It is possible to perform sniffing on failure by creating a {@link SniffOnFailureListener} and providing it as an argument to
+ * {@link org.elasticsearch.client.RestClient.Builder#setFailureListener(RestClient.FailureListener)}. The Sniffer implementation
+ * needs to be lazily set to the previously created SniffOnFailureListener through {@link SniffOnFailureListener#setSniffer(Sniffer)}.
  */
-public final class Sniffer extends RestClient.FailureListener implements Closeable {
+public final class Sniffer implements Closeable {
 
     private static final Log logger = LogFactory.getLog(Sniffer.class);
 
-    private final boolean sniffOnFailure;
     private final Task task;
 
-    private Sniffer(RestClient restClient, HostsSniffer hostsSniffer, long sniffInterval,
-                    boolean sniffOnFailure, long sniffAfterFailureDelay) {
+    private Sniffer(RestClient restClient, HostsSniffer hostsSniffer, long sniffInterval, long sniffAfterFailureDelay) {
         this.task = new Task(hostsSniffer, restClient, sniffInterval, sniffAfterFailureDelay);
-        this.sniffOnFailure = sniffOnFailure;
-        restClient.setFailureListener(this);
     }
 
-    @Override
-    public void onFailure(HttpHost host) throws IOException {
-        if (sniffOnFailure) {
-            //re-sniff immediately but take out the node that failed
-            task.sniffOnFailure(host);
-        }
+    /**
+     * Triggers a new sniffing round and explicitly takes out the failed host provided as argument
+     */
+    public void sniffOnFailure(HttpHost failedHost) {
+        this.task.sniffOnFailure(failedHost);
     }
 
     @Override
@@ -114,12 +110,16 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
         void sniff(HttpHost excludeHost, long nextSniffDelayMillis) {
             if (running.compareAndSet(false, true)) {
                 try {
-                    List<HttpHost> sniffedNodes = hostsSniffer.sniffHosts();
+                    List<HttpHost> sniffedHosts = hostsSniffer.sniffHosts();
+                    logger.debug("sniffed hosts: " + sniffedHosts);
                     if (excludeHost != null) {
-                        sniffedNodes.remove(excludeHost);
+                        sniffedHosts.remove(excludeHost);
+                    }
+                    if (sniffedHosts.isEmpty()) {
+                        logger.warn("no hosts to set, hosts will be updated at the next sniffing round");
+                    } else {
+                        this.restClient.setHosts(sniffedHosts.toArray(new HttpHost[sniffedHosts.size()]));
                     }
-                    logger.debug("sniffed nodes: " + sniffedNodes);
-                    this.restClient.setHosts(sniffedNodes.toArray(new HttpHost[sniffedNodes.size()]));
                 } catch (Exception e) {
                     logger.error("error while sniffing nodes", e);
                 } finally {
@@ -159,7 +159,6 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
         private final RestClient restClient;
         private final HostsSniffer hostsSniffer;
         private long sniffIntervalMillis = DEFAULT_SNIFF_INTERVAL;
-        private boolean sniffOnFailure = true;
         private long sniffAfterFailureDelayMillis = DEFAULT_SNIFF_AFTER_FAILURE_DELAY;
 
         /**
@@ -186,15 +185,6 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
             return this;
         }
 
-        /**
-         * Enables/disables sniffing on failure. If enabled, at each failure nodes will be reloaded, and a new sniff execution will
-         * be scheduled after a shorter time than usual (sniffAfterFailureDelayMillis).
-         */
-        public Builder setSniffOnFailure(boolean sniffOnFailure) {
-            this.sniffOnFailure = sniffOnFailure;
-            return this;
-        }
-
         /**
          * Sets the delay of a sniff execution scheduled after a failure (in milliseconds)
          */
@@ -210,7 +200,7 @@ public final class Sniffer extends RestClient.FailureListener implements Closeab
          * Creates the {@link Sniffer} based on the provided configuration.
          */
         public Sniffer build() {
-            return new Sniffer(restClient, hostsSniffer, sniffIntervalMillis, sniffOnFailure, sniffAfterFailureDelayMillis);
+            return new Sniffer(restClient, hostsSniffer, sniffIntervalMillis, sniffAfterFailureDelayMillis);
         }
     }
 }
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java
new file mode 100644
index 00000000000..bdc052d07c8
--- /dev/null
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.sniff;
+
+import org.apache.http.HttpHost;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+class MockHostsSniffer extends HostsSniffer {
+    MockHostsSniffer() {
+        super(null, -1, null);
+    }
+
+    @Override
+    public List<HttpHost> sniffHosts() throws IOException {
+        List<HttpHost> hosts = new ArrayList<>();
+        hosts.add(new HttpHost("localhost", 9200));
+        return hosts;
+    }
+}
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SniffOnFailureListenerTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SniffOnFailureListenerTests.java
new file mode 100644
index 00000000000..fe2555763ac
--- /dev/null
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SniffOnFailureListenerTests.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client.sniff;
+
+import org.apache.http.HttpHost;
+import org.apache.lucene.util.LuceneTestCase;
+import org.elasticsearch.client.RestClient;
+
+public class SniffOnFailureListenerTests extends LuceneTestCase {
+
+    public void testSetSniffer() throws Exception {
+        SniffOnFailureListener listener = new SniffOnFailureListener();
+
+        try {
+            listener.onFailure(null);
+            fail("should have failed");
+        } catch(IllegalStateException e) {
+            assertEquals("sniffer was not set, unable to sniff on failure", e.getMessage());
+        }
+
+        try {
+            listener.setSniffer(null);
+            fail("should have failed");
+        } catch(NullPointerException e) {
+            assertEquals("sniffer must not be null", e.getMessage());
+        }
+
+        RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200)).build();
+        try (Sniffer sniffer = Sniffer.builder(restClient, new MockHostsSniffer()).build()) {
+            listener.setSniffer(sniffer);
+            try {
+                listener.setSniffer(sniffer);
+                fail("should have failed");
+            } catch(IllegalStateException e) {
+                assertEquals("sniffer can only be set once", e.getMessage());
+            }
+            listener.onFailure(new HttpHost("localhost", 9200));
+        }
+    }
+}
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
index f09ef77f5cf..a3e8d8e80e7 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
@@ -24,10 +24,6 @@ import org.apache.http.HttpHost;
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.client.RestClient;
 
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-
 public class SnifferBuilderTests extends LuceneTestCase {
 
     public void testBuild() throws Exception {
@@ -80,23 +76,9 @@ public class SnifferBuilderTests extends LuceneTestCase {
             if (random().nextBoolean()) {
                 builder.setSniffAfterFailureDelayMillis(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
             }
-            if (random().nextBoolean()) {
-                builder.setSniffOnFailure(random().nextBoolean());
-            }
             try (Sniffer sniffer = builder.build()) {
                 assertNotNull(sniffer);
             }
         }
     }
-
-    private static class MockHostsSniffer extends HostsSniffer {
-        MockHostsSniffer() {
-            super(null, -1, null);
-        }
-
-        @Override
-        public List<HttpHost> sniffHosts() throws IOException {
-            return Collections.singletonList(new HttpHost("localhost", 9200));
-        }
-    }
 }
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 2d1c50af3cb..9baafb2b5b7 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -89,12 +89,14 @@ public final class RestClient implements Closeable {
     private final AtomicInteger lastHostIndex = new AtomicInteger(0);
     private volatile Set<HttpHost> hosts;
     private final ConcurrentMap<HttpHost, DeadHostState> blacklist = new ConcurrentHashMap<>();
-    private volatile FailureListener failureListener = new FailureListener();
+    private final FailureListener failureListener;
 
-    private RestClient(CloseableHttpClient client, long maxRetryTimeoutMillis, Header[] defaultHeaders, HttpHost[] hosts) {
+    private RestClient(CloseableHttpClient client, long maxRetryTimeoutMillis, Header[] defaultHeaders,
+                       HttpHost[] hosts, FailureListener failureListener) {
         this.client = client;
         this.maxRetryTimeoutMillis = maxRetryTimeoutMillis;
         this.defaultHeaders = defaultHeaders;
+        this.failureListener = failureListener;
         setHosts(hosts);
     }
 
@@ -278,13 +280,6 @@ public final class RestClient implements Closeable {
         failureListener.onFailure(host);
     }
 
-    /**
-     * Sets a {@link FailureListener} to be notified each and every time a host fails
-     */
-    public synchronized void setFailureListener(FailureListener failureListener) {
-        this.failureListener = failureListener;
-    }
-
     @Override
     public void close() throws IOException {
         client.close();
@@ -368,6 +363,7 @@ public final class RestClient implements Closeable {
         private CloseableHttpClient httpClient;
         private int maxRetryTimeout = DEFAULT_MAX_RETRY_TIMEOUT_MILLIS;
         private Header[] defaultHeaders = EMPTY_HEADERS;
+        private FailureListener failureListener;
 
         /**
          * Creates a new builder instance and sets the hosts that the client will send requests to.
@@ -418,6 +414,15 @@ public final class RestClient implements Closeable {
             return this;
         }
 
+        /**
+         * Sets the {@link FailureListener} to be notified for each request failure
+         */
+        public Builder setFailureListener(FailureListener failureListener) {
+            Objects.requireNonNull(failureListener, "failure listener must not be null");
+            this.failureListener = failureListener;
+            return this;
+        }
+
         /**
          * Creates a new {@link RestClient} based on the provided configuration.
          */
@@ -425,7 +430,10 @@ public final class RestClient implements Closeable {
             if (httpClient == null) {
                 httpClient = createDefaultHttpClient(null);
             }
-            return new RestClient(httpClient, maxRetryTimeout, defaultHeaders, hosts);
+            if (failureListener == null) {
+                failureListener = new FailureListener();
+            }
+            return new RestClient(httpClient, maxRetryTimeout, defaultHeaders, hosts, failureListener);
         }
 
         /**
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java b/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
index 88b1406d925..cb7de44cd5e 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
@@ -74,6 +74,13 @@ public class RestClientBuilderTests extends LuceneTestCase {
             assertEquals("default header must not be null", e.getMessage());
         }
 
+        try {
+            RestClient.builder(new HttpHost("localhost", 9200)).setFailureListener(null);
+            fail("should have failed");
+        } catch(NullPointerException e) {
+            assertEquals("failure listener must not be null", e.getMessage());
+        }
+
         int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
         HttpHost[] hosts = new HttpHost[numNodes];
         for (int i = 0; i < numNodes; i++) {
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java b/client/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
index 64792c2bb1b..beae3cfc9f1 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
@@ -87,14 +87,10 @@ public class RestClientMultipleHostsTests extends LuceneTestCase {
         for (int i = 0; i < numHosts; i++) {
             httpHosts[i] = new HttpHost("localhost", 9200 + i);
         }
-        restClient = RestClient.builder(httpHosts).setHttpClient(httpClient).build();
         failureListener = new TrackingFailureListener();
-        restClient.setFailureListener(failureListener);
+        restClient = RestClient.builder(httpHosts).setHttpClient(httpClient).setFailureListener(failureListener).build();
     }
 
-    /**
-     * Test that
-     */
     public void testRoundRobinOkStatusCodes() throws Exception {
         int numIters = RandomInts.randomIntBetween(random(), 1, 5);
         for (int i = 0; i < numIters; i++) {
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java b/client/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
index e2657722701..ef2a09fb570 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
@@ -122,9 +122,9 @@ public class RestClientSingleHostTests extends LuceneTestCase {
             defaultHeaders[i] = new BasicHeader(headerName, headerValue);
         }
         httpHost = new HttpHost("localhost", 9200);
-        restClient = RestClient.builder(httpHost).setHttpClient(httpClient).setDefaultHeaders(defaultHeaders).build();
         failureListener = new TrackingFailureListener();
-        restClient.setFailureListener(failureListener);
+        restClient = RestClient.builder(httpHost).setHttpClient(httpClient).setDefaultHeaders(defaultHeaders)
+                .setFailureListener(failureListener).build();
     }
 
     /**

From 881afcba6020e115e64a9bce2de0840912975bba Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Tue, 21 Jun 2016 15:42:42 +0200
Subject: [PATCH 0991/1311] Fixed tests that failed now that BM25 is the
 default similarity.

---
 .../functionscore/DecayFunctionScoreIT.java   | 15 +++++++++---
 .../functionscore/ExplainableScriptIT.java    |  2 +-
 .../search/nested/SimpleNestedIT.java         |  4 ++--
 .../search/query/MultiMatchQueryIT.java       |  4 ++--
 .../search/query/SearchQueryIT.java           | 24 +++++++++----------
 docs/plugins/mapper-attachments.asciidoc      |  4 ++--
 .../pattern-replace-charfilter.asciidoc       |  4 ++--
 .../tokenizers/edgengram-tokenizer.asciidoc   |  4 ++--
 docs/reference/search/uri-request.asciidoc    |  4 ++--
 .../test/percolate/16_existing_doc.yaml       |  2 +-
 10 files changed, 38 insertions(+), 29 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java
index e17f7bb3efd..1418f476e00 100644
--- a/core/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java
+++ b/core/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java
@@ -254,10 +254,19 @@ public class DecayFunctionScoreIT extends ESIntegTestCase {
     }
 
     public void testBoostModeSettingWorks() throws Exception {
-        assertAcked(prepareCreate("test").addMapping(
+        Settings settings = Settings.builder().put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1).build();
+        assertAcked(prepareCreate("test").setSettings(settings)
+            .addMapping(
                 "type1",
-                jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("test").field("type", "text")
-                        .endObject().startObject("loc").field("type", "geo_point").endObject().endObject().endObject().endObject()));
+                jsonBuilder()
+                    .startObject()
+                        .startObject("type1")
+                            .startObject("properties")
+                                .startObject("test").field("type", "text").endObject()
+                                .startObject("loc").field("type", "geo_point").endObject()
+                            .endObject()
+                        .endObject()
+                    .endObject()));
         ensureYellow();
 
         List<IndexRequestBuilder> indexBuilders = new ArrayList<>();
diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java
index eabae8be055..4e929017ebf 100644
--- a/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java
+++ b/core/src/test/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java
@@ -86,7 +86,7 @@ public class ExplainableScriptIT extends ESIntegTestCase {
             assertThat(hit.getId(), equalTo(Integer.toString(idCounter)));
             assertThat(hit.explanation().toString(),
                     containsString(Double.toString(idCounter) + " = This script returned " + Double.toString(idCounter)));
-            assertThat(hit.explanation().toString(), containsString("1.0 = tf(freq=1.0), with freq of"));
+            assertThat(hit.explanation().toString(), containsString("freq=1.0 = termFreq=1.0"));
             assertThat(hit.explanation().getDetails().length, equalTo(2));
             idCounter--;
         }
diff --git a/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java b/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
index e0aec941487..07865d6c2b4 100644
--- a/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
+++ b/core/src/test/java/org/elasticsearch/search/nested/SimpleNestedIT.java
@@ -317,8 +317,8 @@ public class SimpleNestedIT extends ESIntegTestCase {
         assertNoFailures(searchResponse);
         assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
         Explanation explanation = searchResponse.getHits().hits()[0].explanation();
-        assertThat(explanation.getValue(), equalTo(2f));
-        assertThat(explanation.toString(), startsWith("2.0 = sum of:\n  2.0 = Score based on 2 child docs in range from 0 to 1"));
+        assertThat(explanation.getValue(), equalTo(searchResponse.getHits().getHits()[0].score()));
+        assertThat(explanation.toString(), startsWith("0.36464313 = sum of:\n  0.36464313 = Score based on 2 child docs in range from 0 to 1"));
     }
 
     public void testSimpleNestedSorting() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java
index 33ef2b97216..875256a0f92 100644
--- a/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java
@@ -439,8 +439,8 @@ public class MultiMatchQueryIT extends ESIntegTestCase {
                 .setQuery(randomizeType(multiMatchQuery("marvel hero captain america", "full_name", "first_name", "last_name", "category")
                         .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS)
                         .operator(Operator.OR))).get();
-        assertFirstHit(searchResponse, hasId("theone"));
-        assertSecondHit(searchResponse, hasId("theother"));
+        assertFirstHit(searchResponse, hasId("theother"));
+        assertSecondHit(searchResponse, hasId("theone"));
         assertThat(searchResponse.getHits().hits()[0].getScore(), greaterThan(searchResponse.getHits().hits()[1].getScore()));
 
         searchResponse = client().prepareSearch("test")
diff --git a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
index 5e4a7d6f77f..8cb21dadb9b 100644
--- a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
@@ -283,8 +283,8 @@ public class SearchQueryIT extends ESIntegTestCase {
             searchResponse = client().prepareSearch("test").setQuery(
                     boolQuery().must(matchAllQuery()).must(constantScoreQuery(matchAllQuery()))).get();
             assertHitCount(searchResponse, 2L);
-            assertThat((double)searchResponse.getHits().getAt(0).score(), closeTo(Math.sqrt(2), 0.1));
-            assertThat((double)searchResponse.getHits().getAt(1).score(),closeTo(Math.sqrt(2), 0.1));
+            assertThat((double)searchResponse.getHits().getAt(0).score(), closeTo(2.0, 0.1));
+            assertThat((double)searchResponse.getHits().getAt(1).score(),closeTo(2.0, 0.1));
         }
     }
 
@@ -336,8 +336,8 @@ public class SearchQueryIT extends ESIntegTestCase {
 
         searchResponse = client().prepareSearch().setQuery(commonTermsQuery("field1", "the lazy fox brown").cutoffFrequency(1).highFreqMinimumShouldMatch("3")).get();
         assertHitCount(searchResponse, 2L);
-        assertFirstHit(searchResponse, hasId("1"));
-        assertSecondHit(searchResponse, hasId("2"));
+        assertFirstHit(searchResponse, hasId("2"));
+        assertSecondHit(searchResponse, hasId("1"));
 
         searchResponse = client().prepareSearch().setQuery(commonTermsQuery("field1", "the lazy fox brown").cutoffFrequency(1).highFreqMinimumShouldMatch("4")).get();
         assertHitCount(searchResponse, 1L);
@@ -377,9 +377,9 @@ public class SearchQueryIT extends ESIntegTestCase {
         // try the same with multi match query
         searchResponse = client().prepareSearch().setQuery(multiMatchQuery("the quick brown", "field1", "field2").cutoffFrequency(3).operator(Operator.AND)).get();
         assertHitCount(searchResponse, 3L);
-        assertFirstHit(searchResponse, hasId("1"));
-        assertSecondHit(searchResponse, hasId("2"));
-        assertThirdHit(searchResponse, hasId("3"));
+        assertFirstHit(searchResponse, hasId("3"));
+        assertSecondHit(searchResponse, hasId("1"));
+        assertThirdHit(searchResponse, hasId("2"));
     }
 
     public void testCommonTermsQueryStackedTokens() throws Exception {
@@ -423,8 +423,8 @@ public class SearchQueryIT extends ESIntegTestCase {
 
         searchResponse = client().prepareSearch().setQuery(commonTermsQuery("field1", "the fast lazy fox brown").cutoffFrequency(1).highFreqMinimumShouldMatch("5")).get();
         assertHitCount(searchResponse, 2L);
-        assertFirstHit(searchResponse, hasId("1"));
-        assertSecondHit(searchResponse, hasId("2"));
+        assertFirstHit(searchResponse, hasId("2"));
+        assertSecondHit(searchResponse, hasId("1"));
 
         searchResponse = client().prepareSearch().setQuery(commonTermsQuery("field1", "the fast lazy fox brown").cutoffFrequency(1).highFreqMinimumShouldMatch("6")).get();
         assertHitCount(searchResponse, 1L);
@@ -469,9 +469,9 @@ public class SearchQueryIT extends ESIntegTestCase {
         // try the same with multi match query
         searchResponse = client().prepareSearch().setQuery(multiMatchQuery("the fast brown", "field1", "field2").cutoffFrequency(3).operator(Operator.AND)).get();
         assertHitCount(searchResponse, 3L);
-        assertFirstHit(searchResponse, hasId("1"));
-        assertSecondHit(searchResponse, hasId("2"));
-        assertThirdHit(searchResponse, hasId("3"));
+        assertFirstHit(searchResponse, hasId("3"));
+        assertSecondHit(searchResponse, hasId("1"));
+        assertThirdHit(searchResponse, hasId("2"));
     }
 
     public void testQueryStringAnalyzedWildcard() throws Exception {
diff --git a/docs/plugins/mapper-attachments.asciidoc b/docs/plugins/mapper-attachments.asciidoc
index 86e80cb1a26..b287decd4cf 100644
--- a/docs/plugins/mapper-attachments.asciidoc
+++ b/docs/plugins/mapper-attachments.asciidoc
@@ -88,10 +88,10 @@ If you get a hit for your indexed document, the plugin should be installed and w
   "took": 53,
   "hits": {
     "total": 1,
-    "max_score": 0.3125,
+    "max_score": 0.25811607,
     "hits": [
       {
-        "_score": 0.3125,
+        "_score": 0.25811607,
         "_index": "trying-out-mapper-attachments",
         "_type": "person",
         "_id": "1",
diff --git a/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc b/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc
index 24cd203ae26..9299eb800b5 100644
--- a/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc
+++ b/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc
@@ -230,13 +230,13 @@ The output from the above is:
   },
   "hits": {
     "total": 1,
-    "max_score": 0.4375,
+    "max_score": 0.2824934,
     "hits": [
       {
         "_index": "my_index",
         "_type": "my_doc",
         "_id": "1",
-        "_score": 0.4375,
+        "_score": 0.2824934,
         "_source": {
           "text": "The fooBarBaz method"
         },
diff --git a/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc
index 2328354998e..0728767896a 100644
--- a/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc
+++ b/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc
@@ -302,13 +302,13 @@ GET my_index/_search
   },
   "hits": {
     "total": 1,
-    "max_score": 0.44194174,
+    "max_score": 0.51623213,
     "hits": [
       {
         "_index": "my_index",
         "_type": "doc",
         "_id": "1",
-        "_score": 0.44194174,
+        "_score": 0.51623213,
         "_source": {
           "title": "Quick Foxes"
         }
diff --git a/docs/reference/search/uri-request.asciidoc b/docs/reference/search/uri-request.asciidoc
index 9aa9c17727e..496f04ea4af 100644
--- a/docs/reference/search/uri-request.asciidoc
+++ b/docs/reference/search/uri-request.asciidoc
@@ -27,13 +27,13 @@ And here is a sample response:
     },
     "hits":{
         "total" : 1,
-        "max_score": 1.0,
+        "max_score": 0.2876821,
         "hits" : [
             {
                 "_index" : "twitter",
                 "_type" : "tweet",
                 "_id" : "0",
-                "_score": 1.0,
+                "_score": 0.2876821,
                 "_source" : {
                     "user" : "kimchy",
                     "date" : "2009-11-15T14:12:12",
diff --git a/modules/percolator/src/test/resources/rest-api-spec/test/percolate/16_existing_doc.yaml b/modules/percolator/src/test/resources/rest-api-spec/test/percolate/16_existing_doc.yaml
index 9da8b33b996..2d15e2cd222 100644
--- a/modules/percolator/src/test/resources/rest-api-spec/test/percolate/16_existing_doc.yaml
+++ b/modules/percolator/src/test/resources/rest-api-spec/test/percolate/16_existing_doc.yaml
@@ -114,5 +114,5 @@
             term:
               tag: tag1
 
-  - match: {'matches': [{_index: percolator_index, _id: test_percolator, _score: 1.0}]}
+  - match: {'matches': [{_index: percolator_index, _id: test_percolator, _score: 0.2876821}]}
 

From 0488372c20ea928f6a169c8d0df1edb0dbdf132c Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Tue, 21 Jun 2016 15:51:08 +0200
Subject: [PATCH 0992/1311] removed unused imports

---
 .../elasticsearch/percolator/MultiPercolateAction.java   | 2 --
 .../org/elasticsearch/percolator/PercolateAction.java    | 2 --
 .../elasticsearch/percolator/PercolatorFieldMapper.java  | 1 -
 .../org/elasticsearch/percolator/PercolatorPlugin.java   | 4 ----
 .../org/elasticsearch/percolator/MultiPercolatorIT.java  | 4 ----
 .../percolator/PercolateQueryBuilderTests.java           | 9 ++-------
 .../percolator/PercolatorAggregationsIT.java             | 3 ---
 .../PercolatorBackwardsCompatibilityTests.java           | 2 --
 .../PercolatorHighlightSubFetchPhaseTests.java           | 1 -
 .../java/org/elasticsearch/percolator/PercolatorIT.java  | 3 ---
 .../percolator/PercolatorQuerySearchIT.java              | 7 ++-----
 .../org/elasticsearch/percolator/PercolatorTestUtil.java | 1 -
 12 files changed, 4 insertions(+), 35 deletions(-)

diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateAction.java
index f80e1c45494..eefc6c996ba 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateAction.java
@@ -21,8 +21,6 @@ package org.elasticsearch.percolator;
 import org.elasticsearch.action.Action;
 import org.elasticsearch.client.ElasticsearchClient;
 
-/**
- */
 public class MultiPercolateAction extends Action<MultiPercolateRequest, MultiPercolateResponse, MultiPercolateRequestBuilder> {
 
     public static final MultiPercolateAction INSTANCE = new MultiPercolateAction();
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateAction.java
index 94f5a0f04df..64776f271ae 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateAction.java
@@ -22,8 +22,6 @@ package org.elasticsearch.percolator;
 import org.elasticsearch.action.Action;
 import org.elasticsearch.client.ElasticsearchClient;
 
-/**
- */
 public class PercolateAction extends Action<PercolateRequest, PercolateResponse, PercolateRequestBuilder> {
 
     public static final PercolateAction INSTANCE = new PercolateAction();
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
index a19a207eb76..e86b8e818d0 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
@@ -30,7 +30,6 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
index eee8ee02116..23251ca20d0 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
@@ -25,16 +25,12 @@ import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchModule;
-import org.elasticsearch.search.fetch.FetchSubPhase;
-import org.elasticsearch.search.highlight.HighlightPhase;
 
 import java.util.Arrays;
 import java.util.List;
-import java.util.Optional;
 
 public class PercolatorPlugin extends Plugin {
 
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java
index c45af9c089b..15d531467d4 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java
@@ -20,7 +20,6 @@ package org.elasticsearch.percolator;
 
 import org.apache.lucene.search.join.ScoreMode;
 import org.elasticsearch.client.Requests;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.query.Operator;
@@ -39,7 +38,6 @@ import static org.elasticsearch.common.xcontent.XContentFactory.yamlBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
-import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
 import static org.elasticsearch.percolator.PercolatorTestUtil.convertFromTextArray;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.percolator.PercolatorTestUtil.assertMatchCount;
@@ -53,8 +51,6 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
-/**
- */
 public class MultiPercolatorIT extends ESIntegTestCase {
 
     private final static String INDEX_NAME = "queries";
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
index 67024150fb7..f51e2b66af9 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java
@@ -22,13 +22,11 @@ package org.elasticsearch.percolator;
 import com.fasterxml.jackson.core.JsonParseException;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
-import org.apache.lucene.document.Document;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermQuery;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ResourceNotFoundException;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
@@ -38,22 +36,19 @@ import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.get.GetResult;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParsedDocument;
-import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.ingest.RandomDocumentPicks;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.test.AbstractQueryTestCase;
 import org.hamcrest.Matchers;
-import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.util.ArrayList;
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java
index c8f79d3b977..75d4f408774 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java
@@ -48,9 +48,6 @@ import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
-/**
- *
- */
 public class PercolatorAggregationsIT extends ESIntegTestCase {
 
     private final static String INDEX_NAME = "queries";
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java
index 1071ffe759b..701fd1b660f 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java
@@ -39,9 +39,7 @@ import java.util.Collections;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
-import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.percolator.PercolatorTestUtil.preparePercolate;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
index 1dfd941e8e0..ec4107fc2ed 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
@@ -34,7 +34,6 @@ import org.mockito.Mockito;
 
 import java.util.Collections;
 
-import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.sameInstance;
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
index 20c8c7f8b58..ff9bf6634cd 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
@@ -89,9 +89,6 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.nullValue;
 
-/**
- *
- */
 public class PercolatorIT extends ESIntegTestCase {
 
     private final static String INDEX_NAME = "queries";
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
index e7690e23df4..fc1a4206e82 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
@@ -20,13 +20,12 @@ package org.elasticsearch.percolator;
 
 import org.apache.lucene.search.join.ScoreMode;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
 import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
 import org.elasticsearch.index.query.MultiMatchQueryBuilder;
 import org.elasticsearch.index.query.Operator;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -49,11 +48,9 @@ import static org.elasticsearch.index.query.QueryBuilders.spanNearQuery;
 import static org.elasticsearch.index.query.QueryBuilders.spanNotQuery;
 import static org.elasticsearch.index.query.QueryBuilders.spanTermQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.hamcrest.Matchers.equalTo;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.startsWith;
 
 public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
 
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorTestUtil.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorTestUtil.java
index 32ea4c23ad4..456c2508d8e 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorTestUtil.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorTestUtil.java
@@ -23,7 +23,6 @@ import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.Strings;
 import org.junit.Assert;
 
-import static org.hamcrest.Matchers.greaterThan;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertVersionSerializable;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.formatShardStatus;
 

From 70482d1e39abc06c56b81ddb6ee6ebf031f0e0a2 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Tue, 21 Jun 2016 16:02:25 +0200
Subject: [PATCH 0993/1311] Update java.asciidoc

Fixed asciidoc
---
 docs/reference/migration/migrate_5_0/java.asciidoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/reference/migration/migrate_5_0/java.asciidoc b/docs/reference/migration/migrate_5_0/java.asciidoc
index 1ad34d11a7c..00b33897f7d 100644
--- a/docs/reference/migration/migrate_5_0/java.asciidoc
+++ b/docs/reference/migration/migrate_5_0/java.asciidoc
@@ -300,12 +300,12 @@ requests can now be validated at call time which results in much clearer errors.
 
 The `setQuery(BytesReference)` method have been removed in favor of using `setQuery(QueryBuilder)`
 
-=== ClusterStatsResponse
+==== ClusterStatsResponse
 
 Removed the `getMemoryAvailable` method from `OsStats`, which could be previously accessed calling
 `clusterStatsResponse.getNodesStats().getOs().getMemoryAvailable()`.
 
-=== setRefresh(boolean) has been removed
+==== setRefresh(boolean) has been removed
 
 `setRefresh(boolean)` has been removed in favor of `setRefreshPolicy(RefreshPolicy)` because there
 are now three options (NONE, IMMEDIATE, and WAIT_FOR). `setRefresh(IMMEDIATE)` has the same behavior

From f0b6abe439928eb1ec89eb9167deb5d60f5f2cc3 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Tue, 21 Jun 2016 17:10:18 +0200
Subject: [PATCH 0994/1311] rename onSuccess to onResponse

That makes it a bit clearer that it's about the response and whether we decide if it was a good one or a failure (based on status code)
---
 .../src/main/java/org/elasticsearch/client/RestClient.java  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/src/main/java/org/elasticsearch/client/RestClient.java
index 9baafb2b5b7..a8f46ee6b0e 100644
--- a/client/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/src/main/java/org/elasticsearch/client/RestClient.java
@@ -171,7 +171,7 @@ public final class RestClient implements Closeable {
             int statusCode = response.getStatusLine().getStatusCode();
             if (statusCode < 300 || (request.getMethod().equals(HttpHead.METHOD_NAME) && statusCode == 404) ) {
                 RequestLogger.logResponse(logger, request, host, httpResponse);
-                onSuccess(host);
+                onResponse(host);
                 return response;
             }
             RequestLogger.logResponse(logger, request, host, httpResponse);
@@ -195,7 +195,7 @@ public final class RestClient implements Closeable {
                     break;
                 default:
                     //mark host alive and don't retry, as the error should be a request problem
-                    onSuccess(host);
+                    onResponse(host);
                     throw lastSeenException;
             }
         }
@@ -254,7 +254,7 @@ public final class RestClient implements Closeable {
      * Called after each successful request call.
      * Receives as an argument the host that was used for the successful request.
      */
-    private void onSuccess(HttpHost host) {
+    private void onResponse(HttpHost host) {
         DeadHostState removedHost = this.blacklist.remove(host);
         if (logger.isDebugEnabled() && removedHost != null) {
             logger.debug("removed host [" + host + "] from blacklist");

From 7b68d44ddf2e6522d2b2aa1efead9ccc8b4ef224 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 21 Jun 2016 11:14:48 -0400
Subject: [PATCH 0995/1311] Read Elasticsearch manifest via URL

This commit modifies reading the Elasticsearch jar manifest via the URL
instead of converting the URL to an NIO path for increased portability.

Relates #18999
---
 .../main/java/org/elasticsearch/Build.java    | 21 +++++--------------
 .../java/org/elasticsearch/BuildTests.java    |  8 +++----
 2 files changed, 9 insertions(+), 20 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/Build.java b/core/src/main/java/org/elasticsearch/Build.java
index f844e3b4040..25da5f28166 100644
--- a/core/src/main/java/org/elasticsearch/Build.java
+++ b/core/src/main/java/org/elasticsearch/Build.java
@@ -19,16 +19,11 @@
 
 package org.elasticsearch;
 
-import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 
 import java.io.IOException;
-import java.net.URISyntaxException;
 import java.net.URL;
-import java.nio.file.Files;
-import java.nio.file.Path;
 import java.util.jar.JarInputStream;
 import java.util.jar.Manifest;
 
@@ -47,9 +42,9 @@ public class Build {
         final String date;
         final boolean isSnapshot;
 
-        Path path = getElasticsearchCodebase();
-        if (path.toString().endsWith(".jar")) {
-            try (JarInputStream jar = new JarInputStream(Files.newInputStream(path))) {
+        final URL url = getElasticsearchCodebase();
+        if (url.toString().endsWith(".jar")) {
+            try (JarInputStream jar = new JarInputStream(url.openStream())) {
                 Manifest manifest = jar.getManifest();
                 shortHash = manifest.getMainAttributes().getValue("Change");
                 date = manifest.getMainAttributes().getValue("Build-Date");
@@ -80,14 +75,8 @@ public class Build {
     /**
      * Returns path to elasticsearch codebase path
      */
-    @SuppressForbidden(reason = "looks up path of elasticsearch.jar directly")
-    static Path getElasticsearchCodebase() {
-        URL url = Build.class.getProtectionDomain().getCodeSource().getLocation();
-        try {
-            return PathUtils.get(url.toURI());
-        } catch (URISyntaxException bogus) {
-            throw new RuntimeException(bogus);
-        }
+    static URL getElasticsearchCodebase() {
+        return Build.class.getProtectionDomain().getCodeSource().getLocation();
     }
 
     private String shortHash;
diff --git a/core/src/test/java/org/elasticsearch/BuildTests.java b/core/src/test/java/org/elasticsearch/BuildTests.java
index f02d2f27e82..7bff83262b3 100644
--- a/core/src/test/java/org/elasticsearch/BuildTests.java
+++ b/core/src/test/java/org/elasticsearch/BuildTests.java
@@ -22,16 +22,16 @@ package org.elasticsearch;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
-import java.nio.file.AccessMode;
-import java.nio.file.Path;
+import java.io.InputStream;
+import java.net.URL;
 
 public class BuildTests extends ESTestCase {
 
     /** Asking for the jar metadata should not throw exception in tests, no matter how configured */
     public void testJarMetadata() throws IOException {
-        Path path = Build.getElasticsearchCodebase();
+        URL url = Build.getElasticsearchCodebase();
         // throws exception if does not exist, or we cannot access it
-        path.getFileSystem().provider().checkAccess(path, AccessMode.READ);
+        try (InputStream ignored = url.openStream()) {}
         // these should never be null
         assertNotNull(Build.CURRENT.date());
         assertNotNull(Build.CURRENT.shortHash());

From 42d60f9f28ba159a91e991c806831f375006fdb8 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 21 Jun 2016 11:25:43 -0400
Subject: [PATCH 0996/1311] maps n lists

---
 .../elasticsearch/painless/Augmentation.java  | 290 +++++++++++++++++-
 .../java/org/elasticsearch/painless/Def.java  |   4 +-
 .../elasticsearch/painless/Definition.java    |   4 +-
 .../elasticsearch/painless/FunctionRef.java   |  29 +-
 .../painless/WriterConstants.java             |   2 +
 .../painless/node/ECapturingFunctionRef.java  |   2 +-
 .../painless/node/EFunctionRef.java           |   4 +-
 .../elasticsearch/painless/node/ELambda.java  |   6 +-
 .../org/elasticsearch/painless/java.lang.txt  |   7 +
 .../org/elasticsearch/painless/java.util.txt  |  22 ++
 .../painless/AugmentationTests.java           | 149 ++++++++-
 11 files changed, 491 insertions(+), 28 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Augmentation.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Augmentation.java
index 456642eb8c5..4bca673b4dc 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Augmentation.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Augmentation.java
@@ -20,9 +20,14 @@
 package org.elasticsearch.painless;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.TreeMap;
+import java.util.function.BiConsumer;
+import java.util.function.BiFunction;
+import java.util.function.BiPredicate;
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.ObjIntConsumer;
@@ -30,15 +35,26 @@ import java.util.function.Predicate;
 import java.util.function.ToDoubleFunction;
 import java.util.regex.Matcher;
 
+/** Additional methods added to classes. These must be static methods with receiver as first argument */
 public class Augmentation {
+    
+    // static methods only!
+    private Augmentation() {}
+
+    /** Exposes List.size() as getLength(), so that .length shortcut works on lists */
     public static <T> int getLength(List<T> receiver) {
         return receiver.size();
     }
-    
+
+    /** Exposes Matcher.group(String) as namedGroup(String), so it doesn't conflict with group(int) */
     public static String namedGroup(Matcher receiver, String name) {
         return receiver.group(name);
     }
     
+    // some groovy methods on iterable
+    // see http://docs.groovy-lang.org/latest/html/groovy-jdk/java/lang/Iterable.html
+    
+    /** Iterates over the contents of an iterable, and checks whether a predicate is valid for at least one element. */
     public static <T> boolean any(Iterable<T> receiver, Predicate<T> predicate) {
         for (T t : receiver) {
             if (predicate.test(t)) {
@@ -48,6 +64,7 @@ public class Augmentation {
         return false;
     }
     
+    /** Counts the number of occurrences which satisfy the given predicate from inside this Iterable. */ 
     public static <T> int count(Iterable<T> receiver, Predicate<T> predicate) {
         int count = 0;
         for (T t : receiver) {
@@ -58,12 +75,20 @@ public class Augmentation {
         return count;
     }
     
-    public static <T> Iterable<T> each(Iterable<T> receiver, Consumer<T> consumer) {
+    // instead of covariant overrides for every possibility, we just return receiver as 'def' for now
+    // that way if someone chains the calls, everything works.
+
+    /** Iterates through an Iterable, passing each item to the given consumer. */
+    public static <T> Object each(Iterable<T> receiver, Consumer<T> consumer) {
         receiver.forEach(consumer);
         return receiver;
     }
     
-    public static <T> Iterable<T> eachWithIndex(Iterable<T> receiver, ObjIntConsumer<T> consumer) {
+    /** 
+     * Iterates through an iterable type, passing each item and the item's index 
+     * (a counter starting at zero) to the given consumer.
+     */
+    public static <T> Object eachWithIndex(Iterable<T> receiver, ObjIntConsumer<T> consumer) {
         int count = 0;
         for (T t : receiver) {
             consumer.accept(t, count++);
@@ -71,6 +96,9 @@ public class Augmentation {
         return receiver;
     }
     
+    /**
+     * Used to determine if the given predicate is valid (i.e. returns true for all items in this iterable).
+     */
     public static <T> boolean every(Iterable<T> receiver, Predicate<T> predicate) {
         for (T t : receiver) {
             if (predicate.test(t) == false) {
@@ -80,6 +108,10 @@ public class Augmentation {
         return true;
     }
     
+    /**
+     * Iterates through the Iterable transforming items using the supplied function and 
+     * collecting any non-null results. 
+     */
     public static <T,U> List<U> findResults(Iterable<T> receiver, Function<T,U> filter) {
         List<U> list = new ArrayList<>();
         for (T t: receiver) {
@@ -91,6 +123,9 @@ public class Augmentation {
         return list;
     }
     
+    /**
+     * Sorts all Iterable members into groups determined by the supplied mapping function. 
+     */
     public static <T,U> Map<U,List<T>> groupBy(Iterable<T> receiver, Function<T,U> mapper) {
         Map<U,List<T>> map = new LinkedHashMap<>();
         for (T t : receiver) {
@@ -105,6 +140,10 @@ public class Augmentation {
         return map;
     }
     
+    /**
+     * Concatenates the toString() representation of each item in this Iterable, 
+     * with the given String as a separator between each item. 
+     */
     public static <T> String join(Iterable<T> receiver, String separator) {
         StringBuilder sb = new StringBuilder();
         for (T t : receiver) {
@@ -116,6 +155,9 @@ public class Augmentation {
         return sb.toString();
     }
     
+    /**
+     * Sums the result of applying a function to each item of an Iterable. 
+     */
     public static <T> double sum(Iterable<T> receiver, ToDoubleFunction<T> function) {
         double sum = 0;
         for (T t : receiver) {
@@ -123,4 +165,246 @@ public class Augmentation {
         }
         return sum;
     }
+    
+    // some groovy methods on collection
+    // see http://docs.groovy-lang.org/latest/html/groovy-jdk/java/util/Collection.html
+    
+    /**
+     * Iterates through this collection transforming each entry into a new value using 
+     * the function, returning a list of transformed values. 
+     */
+    public static <T,U> List<U> collect(Collection<T> receiver, Function<T,U> function) {
+        List<U> list = new ArrayList<>();
+        for (T t : receiver) {
+            list.add(function.apply(t));
+        }
+        return list;
+    }
+    
+    /**
+     * Iterates through this collection transforming each entry into a new value using 
+     * the function, adding the values to the specified collection.
+     */
+    public static <T,U> Object collect(Collection<T> receiver, Collection<U> collection, Function<T,U> function) {
+        for (T t : receiver) {
+            collection.add(function.apply(t));
+        }
+        return collection;
+    }
+    
+    /**
+     * Finds the first value matching the predicate, or returns null.
+     */
+    public static <T> T find(Collection<T> receiver, Predicate<T> predicate) {
+        for (T t : receiver) {
+            if (predicate.test(t)) {
+                return t;
+            }
+        }
+        return null;
+    }
+    
+    /**
+     * Finds all values matching the predicate, returns as a list
+     */
+    public static <T> List<T> findAll(Collection<T> receiver, Predicate<T> predicate) {
+        List<T> list = new ArrayList<>();
+        for (T t : receiver) {
+            if (predicate.test(t)) {
+                list.add(t);
+            }
+        }
+        return list;
+    }
+    
+    /**
+     * Iterates through the collection calling the given function for each item 
+     * but stopping once the first non-null result is found and returning that result. 
+     * If all results are null, null is returned. 
+     */
+    public static <T,U> Object findResult(Collection<T> receiver, Function<T,U> function) {
+        return findResult(receiver, null, function);
+    }
+    
+    /**
+     * Iterates through the collection calling the given function for each item 
+     * but stopping once the first non-null result is found and returning that result. 
+     * If all results are null, defaultResult is returned.
+     */
+    public static <T,U> Object findResult(Collection<T> receiver, Object defaultResult, Function<T,U> function) {
+        for (T t : receiver) {
+            U value = function.apply(t);
+            if (value != null) {
+                return value;
+            }
+        }
+        return defaultResult;
+    }
+    
+    /**
+     * Splits all items into two collections based on the predicate. 
+     * The first list contains all items which match the closure expression. The second list all those that don't. 
+     */
+    public static <T> List<List<T>> split(Collection<T> receiver, Predicate<T> predicate) {
+        List<T> matched = new ArrayList<>();
+        List<T> unmatched = new ArrayList<>();
+        List<List<T>> result = new ArrayList<>(2);
+        result.add(matched);
+        result.add(unmatched);
+        for (T t : receiver) {
+            if (predicate.test(t)) {
+                matched.add(t);
+            } else {
+                unmatched.add(t);
+            }
+        }
+        return result;
+    }
+    
+    // some groovy methods on map
+    // see http://docs.groovy-lang.org/latest/html/groovy-jdk/java/util/Map.html
+    
+    /**
+     * Iterates through this map transforming each entry into a new value using 
+     * the function, returning a list of transformed values. 
+     */
+    public static <K,V,T> List<T> collect(Map<K,V> receiver, BiFunction<K,V,T> function) {
+        List<T> list = new ArrayList<>();
+        for (Map.Entry<K,V> kvPair : receiver.entrySet()) {
+            list.add(function.apply(kvPair.getKey(), kvPair.getValue()));
+        }
+        return list;
+    }
+    
+    /**
+     * Iterates through this map transforming each entry into a new value using 
+     * the function, adding the values to the specified collection.
+     */
+    public static <K,V,T> Object collect(Map<K,V> receiver, Collection<T> collection, BiFunction<K,V,T> function) {
+        for (Map.Entry<K,V> kvPair : receiver.entrySet()) {
+            collection.add(function.apply(kvPair.getKey(), kvPair.getValue()));
+        }
+        return collection;
+    }
+    
+    /** Counts the number of occurrences which satisfy the given predicate from inside this Map */ 
+    public static <K,V> int count(Map<K,V> receiver, BiPredicate<K,V> predicate) {
+        int count = 0;
+        for (Map.Entry<K,V> kvPair : receiver.entrySet()) {
+            if (predicate.test(kvPair.getKey(), kvPair.getValue())) {
+                count++;
+            }
+        }
+        return count;
+    }
+    
+    /** Iterates through a Map, passing each item to the given consumer. */
+    public static <K,V> Object each(Map<K,V> receiver, BiConsumer<K,V> consumer) {
+        receiver.forEach(consumer);
+        return receiver;
+    }
+    
+    /**
+     * Used to determine if the given predicate is valid (i.e. returns true for all items in this map).
+     */
+    public static <K,V> boolean every(Map<K,V> receiver, BiPredicate<K,V> predicate) {
+        for (Map.Entry<K,V> kvPair : receiver.entrySet()) {
+            if (predicate.test(kvPair.getKey(), kvPair.getValue()) == false) {
+                return false;
+            }
+        }
+        return true;
+    }
+    
+    /**
+     * Finds the first entry matching the predicate, or returns null.
+     */
+    public static <K,V> Map.Entry<K,V> find(Map<K,V> receiver, BiPredicate<K,V> predicate) {
+        for (Map.Entry<K,V> kvPair : receiver.entrySet()) {
+            if (predicate.test(kvPair.getKey(), kvPair.getValue())) {
+                return kvPair;
+            }
+        }
+        return null;
+    }
+    
+    /**
+     * Finds all values matching the predicate, returns as a map.
+     */
+    public static <K,V> Map<K,V> findAll(Map<K,V> receiver, BiPredicate<K,V> predicate) {
+        // try to preserve some properties of the receiver (see the groovy javadocs)
+        final Map<K,V> map;
+        if (receiver instanceof TreeMap) {
+            map = new TreeMap<>();
+        } else {
+            map = new LinkedHashMap<>();
+        }
+        for (Map.Entry<K,V> kvPair : receiver.entrySet()) {
+            if (predicate.test(kvPair.getKey(), kvPair.getValue())) {
+                map.put(kvPair.getKey(), kvPair.getValue());
+            }
+        }
+        return map;
+    }
+    
+    /**
+     * Iterates through the map calling the given function for each item 
+     * but stopping once the first non-null result is found and returning that result. 
+     * If all results are null, null is returned. 
+     */
+    public static <K,V,T> Object findResult(Map<K,V> receiver, BiFunction<K,V,T> function) {
+        return findResult(receiver, null, function);
+    }
+    
+    /**
+     * Iterates through the map calling the given function for each item 
+     * but stopping once the first non-null result is found and returning that result. 
+     * If all results are null, defaultResult is returned.
+     */
+    public static <K,V,T> Object findResult(Map<K,V> receiver, Object defaultResult, BiFunction<K,V,T> function) {
+        for (Map.Entry<K,V> kvPair : receiver.entrySet()) {
+            T value = function.apply(kvPair.getKey(), kvPair.getValue());
+            if (value != null) {
+                return value;
+            }
+        }
+        return defaultResult;
+    }
+    
+    /**
+     * Iterates through the map transforming items using the supplied function and 
+     * collecting any non-null results. 
+     */
+    public static <K,V,T> List<T> findResults(Map<K,V> receiver, BiFunction<K,V,T> filter) {
+        List<T> list = new ArrayList<>();
+        for (Map.Entry<K,V> kvPair : receiver.entrySet()) {
+           T result = filter.apply(kvPair.getKey(), kvPair.getValue());
+           if (result != null) {
+               list.add(result);
+           }
+        }
+        return list;
+    }
+    
+    /**
+     * Sorts all Map members into groups determined by the supplied mapping function. 
+     */
+    public static <K,V,T> Map<T,Map<K,V>> groupBy(Map<K,V> receiver, BiFunction<K,V,T> mapper) {
+        Map<T,Map<K,V>> map = new LinkedHashMap<>();
+        for (Map.Entry<K,V> kvPair : receiver.entrySet()) {
+            T mapped = mapper.apply(kvPair.getKey(), kvPair.getValue());
+            Map<K,V> results = map.get(mapped);
+            if (results == null) {
+                // try to preserve some properties of the receiver (see the groovy javadocs)
+                if (receiver instanceof TreeMap) {
+                    results = new TreeMap<>();
+                } else {
+                    results = new LinkedHashMap<>();
+                }
+                map.put(mapped, results);
+            }
+            results.put(kvPair.getKey(), kvPair.getValue());
+        }
+        return map;
+    }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
index 5461771bca6..cd761d0ad44 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -350,10 +350,10 @@ public final class Def {
                  }
                  throw new IllegalArgumentException("Unknown call [" + call + "] with [" + arity + "] arguments.");
              }
-             ref = new FunctionRef(clazz, interfaceMethod, handle, captures);
+             ref = new FunctionRef(clazz, interfaceMethod, handle, captures.length);
          } else {
              // whitelist lookup
-             ref = new FunctionRef(clazz, type, call, captures);
+             ref = new FunctionRef(clazz, type, call, captures.length);
          }
          final CallSite callSite;
          if (ref.needsBridges()) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 376be44e28b..8aa4da4015f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -257,7 +257,7 @@ public final class Definition {
             final org.objectweb.asm.Type type;
             if (augmentation) {
                 assert java.lang.reflect.Modifier.isStatic(modifiers);
-                type = org.objectweb.asm.Type.getType(Augmentation.class);
+                type = WriterConstants.AUGMENTATION_TYPE;
             } else {
                 type = owner.type;
             }
@@ -825,7 +825,7 @@ public final class Definition {
             reflect = implClass.getMethod(name, params);
         } catch (NoSuchMethodException exception) {
             throw new IllegalArgumentException("Method [" + name +
-                "] not found for class [" + owner.clazz.getName() + "]" +
+                "] not found for class [" + implClass.getName() + "]" +
                 " with arguments " + Arrays.toString(params) + ".");
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
index 72676f1c04c..d5e02e12058 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
@@ -53,10 +53,10 @@ public class FunctionRef {
      * @param expected interface type to implement.
      * @param type the left hand side of a method reference expression
      * @param call the right hand side of a method reference expression
-     * @param captures captured arguments
+     * @param numCaptures number of captured arguments
      */    
-    public FunctionRef(Definition.Type expected, String type, String call, Class<?>... captures) {
-        this(expected, expected.struct.getFunctionalMethod(), lookup(expected, type, call, captures.length > 0), captures);
+    public FunctionRef(Definition.Type expected, String type, String call, int numCaptures) {
+        this(expected, expected.struct.getFunctionalMethod(), lookup(expected, type, call, numCaptures > 0), numCaptures);
     }
 
     /**
@@ -64,13 +64,16 @@ public class FunctionRef {
      * @param expected interface type to implement
      * @param method functional interface method
      * @param impl implementation method
-     * @param captures captured arguments
+     * @param numCaptures number of captured arguments
      */   
-    public FunctionRef(Definition.Type expected, Definition.Method method, Definition.Method impl, Class<?>... captures) {
+    public FunctionRef(Definition.Type expected, Definition.Method method, Definition.Method impl, int numCaptures) {
         // e.g. compareTo
         invokedName = method.name;
         // e.g. (Object)Comparator
-        invokedType = MethodType.methodType(expected.clazz, captures);
+        MethodType implType = impl.getMethodType();
+        // only include captured parameters as arguments
+        invokedType = MethodType.methodType(expected.clazz, 
+                implType.dropParameterTypes(numCaptures, implType.parameterCount()));
         // e.g. (Object,Object)int
         interfaceMethodType = method.getMethodType().dropParameterTypes(0, 1);
 
@@ -90,6 +93,9 @@ public class FunctionRef {
             // owner == null: script class itself
             ownerIsInterface = false;
             owner = WriterConstants.CLASS_TYPE.getInternalName();
+        } else if (impl.augmentation) {
+            ownerIsInterface = false;
+            owner = WriterConstants.AUGMENTATION_TYPE.getInternalName();
         } else {
             ownerIsInterface = impl.owner.clazz.isInterface();
             owner = impl.owner.type.getInternalName();
@@ -98,7 +104,7 @@ public class FunctionRef {
         implMethod = impl.handle;
         
         // remove any prepended captured arguments for the 'natural' signature.
-        samMethodType = adapt(interfaceMethodType, impl.getMethodType().dropParameterTypes(0, captures.length));
+        samMethodType = adapt(interfaceMethodType, impl.getMethodType().dropParameterTypes(0, numCaptures));
     }
 
     /**
@@ -106,11 +112,14 @@ public class FunctionRef {
      * <p>
      * This will <b>not</b> set implMethodASM. It is for runtime use only.
      */
-    public FunctionRef(Definition.Type expected, Definition.Method method, MethodHandle impl, Class<?>... captures) {
+    public FunctionRef(Definition.Type expected, Definition.Method method, MethodHandle impl, int numCaptures) {
         // e.g. compareTo
         invokedName = method.name;
         // e.g. (Object)Comparator
-        invokedType = MethodType.methodType(expected.clazz, captures);
+        MethodType implType = impl.type();
+        // only include captured parameters as arguments
+        invokedType = MethodType.methodType(expected.clazz, 
+                implType.dropParameterTypes(numCaptures, implType.parameterCount()));
         // e.g. (Object,Object)int
         interfaceMethodType = method.getMethodType().dropParameterTypes(0, 1);
 
@@ -119,7 +128,7 @@ public class FunctionRef {
         implMethodASM = null;
         
         // remove any prepended captured arguments for the 'natural' signature.
-        samMethodType = adapt(interfaceMethodType, impl.type().dropParameterTypes(0, captures.length));
+        samMethodType = adapt(interfaceMethodType, impl.type().dropParameterTypes(0, numCaptures));
     }
 
     /** 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index d581ba8518a..e2bf804c181 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -72,6 +72,8 @@ public final class WriterConstants {
     public final static Method CHAR_TO_STRING = getAsmMethod(String.class, "charToString", char.class);
     
     public final static Type METHOD_HANDLE_TYPE = Type.getType(MethodHandle.class);
+    
+    public static final Type AUGMENTATION_TYPE = Type.getType(Augmentation.class);
 
     /**
      * A Method instance for {@linkplain Pattern#compile}. This isn't available from Definition because we intentionally don't add it there
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
index aa7f807aff1..7bf8e195d6d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
@@ -76,7 +76,7 @@ public class ECapturingFunctionRef extends AExpression implements ILambda {
             // static case
             if (captured.type.sort != Definition.Sort.DEF) {
                 try {
-                    ref = new FunctionRef(expected, captured.type.name, call, captured.type.clazz);
+                    ref = new FunctionRef(expected, captured.type.name, call, 1);
                 } catch (IllegalArgumentException e) {
                     throw createError(e);
                 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
index 298b84ffe29..380bfd6c43f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -76,10 +76,10 @@ public class EFunctionRef extends AExpression implements ILambda {
                         throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " +
                                                            "to [" + expected.name + "], function not found");
                     }
-                    ref = new FunctionRef(expected, interfaceMethod, implMethod);
+                    ref = new FunctionRef(expected, interfaceMethod, implMethod, 0);
                 } else {
                     // whitelist lookup
-                    ref = new FunctionRef(expected, type, call);
+                    ref = new FunctionRef(expected, type, call, 0);
                 }
             } catch (IllegalArgumentException e) {
                 throw createError(e);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
index e0ea5e73aeb..0cbb2ed1b33 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
@@ -175,11 +175,7 @@ public class ELambda extends AExpression implements ILambda {
         } else {
             defPointer = null;
             try {
-                Class<?> captureClasses[] = new Class<?>[captures.size()];
-                for (int i = 0; i < captures.size(); i++) {
-                    captureClasses[i] = captures.get(i).type.clazz;
-                }
-                ref = new FunctionRef(expected, interfaceMethod, desugared.method, captureClasses);
+                ref = new FunctionRef(expected, interfaceMethod, desugared.method, captures.size());
             } catch (IllegalArgumentException e) {
                 throw createError(e);
             }
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
index 859a7b314dd..035dc9ba0c8 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
@@ -52,6 +52,13 @@ class Iterable -> java.lang.Iterable {
   Spliterator spliterator()
   # some adaptations of groovy methods
   boolean any*(Predicate)
+  def each*(Consumer)
+  def eachWithIndex*(ObjIntConsumer)
+  boolean every*(Predicate)
+  List findResults*(Function)
+  Map groupBy*(Function)
+  String join*(String)
+  double sum*(ToDoubleFunction)
 }
 
 # Readable: i/o
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt
index 2a41b2c2c13..66f8f67d869 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt
@@ -39,6 +39,15 @@ class Collection -> java.util.Collection extends Iterable {
   Stream stream()
   def[] toArray()
   def[] toArray(def[])
+
+  # some adaptations of groovy methods
+  List collect*(Function)
+  def collect*(Collection,Function)
+  def find*(Predicate)
+  List findAll*(Predicate)
+  def findResult*(Function)
+  def findResult*(def,Function)
+  List split*(Predicate)
 }
 
 class Comparator -> java.util.Comparator {
@@ -152,6 +161,19 @@ class Map -> java.util.Map {
   void replaceAll(BiFunction)
   int size()
   Collection values()
+
+  # some adaptations of groovy methods
+  List collect*(BiFunction)
+  def collect*(Collection,BiFunction)
+  int count*(BiPredicate)
+  def each*(BiConsumer)
+  boolean every*(BiPredicate)
+  Map.Entry find*(BiPredicate)
+  Map findAll*(BiPredicate)
+  def findResult*(BiFunction)
+  def findResult*(def,BiFunction)
+  List findResults*(BiFunction)
+  Map groupBy*(BiFunction)
 }
 
 class Map.Entry -> java.util.Map$Entry {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java
index ccc3620b1c9..c0872dd1994 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java
@@ -19,17 +19,160 @@
 
 package org.elasticsearch.painless;
 
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
 public class AugmentationTests extends ScriptTestCase {
     
-    @AwaitsFix(bugUrl = "rmuir is working on this")
+    public void testStatic() {
+        assertEquals(1, exec("ArrayList l = new ArrayList(); l.add(1); return l.getLength();"));
+        assertEquals(1, exec("ArrayList l = new ArrayList(); l.add(1); return l.length;"));
+    }
+    
+    public void testSubclass() {
+        assertEquals(1, exec("List l = new ArrayList(); l.add(1); return l.getLength();"));
+        assertEquals(1, exec("List l = new ArrayList(); l.add(1); return l.length;"));
+    }
+    
+    public void testDef() {
+        assertEquals(1, exec("def l = new ArrayList(); l.add(1); return l.getLength();"));
+        assertEquals(1, exec("def l = new ArrayList(); l.add(1); return l.length;"));
+    }
+    
     public void testCapturingReference() {
+        assertEquals(1, exec("int foo(Supplier t) { return t.get() }" +
+                             "ArrayList l = new ArrayList(); l.add(1);" +
+                             "return foo(l::getLength);"));
+        assertEquals(1, exec("int foo(Supplier t) { return t.get() }" +
+                             "List l = new ArrayList(); l.add(1);" +
+                             "return foo(l::getLength);"));
         assertEquals(1, exec("int foo(Supplier t) { return t.get() }" +
                              "def l = new ArrayList(); l.add(1);" +
                              "return foo(l::getLength);"));
     }
     
-    
     public void testIterable_Any() {
-        assertEquals(true, exec("List l = new ArrayList(); l.add(1); l.any(x -> x == 1)"));
+        assertEquals(true, 
+                exec("List l = new ArrayList(); l.add(1); l.any(x -> x == 1)"));
+    }
+    
+    public void testIterable_Each() {
+        assertEquals(1, 
+                exec("List l = new ArrayList(); l.add(1); List l2 = new ArrayList(); l.each(l2::add); return l2.size()"));
+    }
+    
+    public void testIterable_EachWithIndex() {
+        assertEquals(0, 
+                exec("List l = new ArrayList(); l.add(2); Map m = new HashMap(); l.eachWithIndex(m::put); return m.get(2)"));
+    }
+    
+    public void testIterable_Every() {
+        assertEquals(false, exec("List l = new ArrayList(); l.add(1); l.add(2); l.every(x -> x == 1)"));
+    }
+    
+    public void testIterable_FindResults() {
+        assertEquals(1, 
+                exec("List l = new ArrayList(); l.add(1); l.add(2); l.findResults(x -> x == 1 ? x : null).size()"));
+    }
+    
+    public void testIterable_GroupBy() {
+        assertEquals(2, 
+                exec("List l = new ArrayList(); l.add(1); l.add(-1); l.groupBy(x -> x < 0 ? 'negative' : 'positive').size()"));
+    }
+    
+    public void testIterable_Join() {
+        assertEquals("test,ing", 
+                exec("List l = new ArrayList(); l.add('test'); l.add('ing'); l.join(',')"));
+    }
+    
+    public void testIterable_Sum() {
+        assertEquals(5.0D, 
+                exec("List l = new ArrayList(); l.add(1); l.add(2); l.sum(x -> x + 1)"));
+    }
+    
+    public void testCollection_Collect() {
+        assertEquals(Arrays.asList(2, 3), 
+                exec("List l = new ArrayList(); l.add(1); l.add(2); l.collect(x -> x + 1)"));
+        assertEquals(asSet(2, 3), 
+                exec("List l = new ArrayList(); l.add(1); l.add(2); l.collect(new HashSet(), x -> x + 1)"));
+    }
+    
+    public void testCollection_Find() {
+        assertEquals(2, 
+                exec("List l = new ArrayList(); l.add(1); l.add(2); return l.find(x -> x == 2)"));
+    }
+    
+    public void testCollection_FindAll() {
+        assertEquals(Arrays.asList(2), 
+                exec("List l = new ArrayList(); l.add(1); l.add(2); return l.findAll(x -> x == 2)"));
+    }
+    
+    public void testCollection_FindResult() {
+        assertEquals("found", 
+                exec("List l = new ArrayList(); l.add(1); l.add(2); return l.findResult(x -> x > 1 ? 'found' : null)"));
+        assertEquals("notfound", 
+                exec("List l = new ArrayList(); l.add(1); l.add(2); return l.findResult('notfound', x -> x > 10 ? 'found' : null)"));
+    }
+    
+    public void testCollection_Split() {
+        assertEquals(Arrays.asList(Arrays.asList(2), Arrays.asList(1)), 
+                exec("List l = new ArrayList(); l.add(1); l.add(2); return l.split(x -> x == 2)"));
+    }
+    
+    public void testMap_Collect() {
+        assertEquals(Arrays.asList("one1", "two2"), 
+                exec("Map m = new TreeMap(); m.one = 1; m.two = 2; m.collect((key,value) -> key + value)"));
+        assertEquals(asSet("one1", "two2"), 
+                exec("Map m = new TreeMap(); m.one = 1; m.two = 2; m.collect(new HashSet(), (key,value) -> key + value)"));
+    }
+    
+    public void testMap_Count() {
+        assertEquals(1, 
+                exec("Map m = new TreeMap(); m.one = 1; m.two = 2; m.count((key,value) -> value == 2)"));
+    }
+    
+    public void testMap_Each() {
+        assertEquals(2, 
+                exec("Map m = new TreeMap(); m.one = 1; m.two = 2; Map m2 = new TreeMap(); m.each(m2::put); return m2.size()"));
+    }
+    
+    public void testMap_Every() {
+        assertEquals(false, 
+                exec("Map m = new TreeMap(); m.one = 1; m.two = 2; m.every((key,value) -> value == 2)"));
+    }
+    
+    public void testMap_Find() {
+        assertEquals("two", 
+                exec("Map m = new TreeMap(); m.one = 1; m.two = 2; return m.find((key,value) -> value == 2).key"));
+    }
+    
+    public void testMap_FindAll() {
+        assertEquals(Collections.singletonMap("two", 2), 
+                exec("Map m = new TreeMap(); m.one = 1; m.two = 2; return m.findAll((key,value) -> value == 2)"));
+    }
+    
+    public void testMap_FindResult() {
+        assertEquals("found", 
+                exec("Map m = new TreeMap(); m.one = 1; m.two = 2; return m.findResult((key,value) -> value == 2 ? 'found' : null)"));
+        assertEquals("notfound", 
+                exec("Map m = new TreeMap(); m.one = 1; m.two = 2; " + 
+                     "return m.findResult('notfound', (key,value) -> value == 10 ? 'found' : null)"));
+    }
+    
+    public void testMap_FindResults() {
+        assertEquals(Arrays.asList("negative", "positive"),
+                exec("Map m = new TreeMap(); m.a = -1; m.b = 1; " + 
+                     "return m.findResults((key,value) -> value < 0 ? 'negative' : 'positive')"));
+    }
+    
+    public void testMap_GroupBy() {
+        Map<String,Map<String,Integer>> expected = new HashMap<>();
+        expected.put("negative", Collections.singletonMap("a", -1));
+        expected.put("positive", Collections.singletonMap("b", 1));
+        assertEquals(expected,
+                exec("Map m = new TreeMap(); m.a = -1; m.b = 1; " + 
+                     "return m.groupBy((key,value) -> value < 0 ? 'negative' : 'positive')"));
     }
 }

From 6177c0a90029a48638820f9910195a05636f89b1 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Tue, 21 Jun 2016 17:31:13 +0200
Subject: [PATCH 0997/1311] Upgrade `string` fields to `text`/`keyword` even if
 `include_in_all` is set. #19004

Closes #18974
---
 .../index/mapper/core/KeywordFieldMapper.java |  5 ++++
 .../index/mapper/core/StringFieldMapper.java  |  4 +--
 .../index/mapper/core/TextFieldMapper.java    |  5 ++++
 .../core/StringMappingUpgradeTests.java       | 26 +++++++++++++++++++
 4 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/KeywordFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/KeywordFieldMapper.java
index fe2b43aa512..a72e1e6836b 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/KeywordFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/KeywordFieldMapper.java
@@ -192,6 +192,11 @@ public final class KeywordFieldMapper extends FieldMapper implements AllFieldMap
         return (KeywordFieldMapper) super.clone();
     }
 
+    // pkg-private for testing
+    Boolean includeInAll() {
+        return includeInAll;
+    }
+
     @Override
     public KeywordFieldMapper includeInAll(Boolean includeInAll) {
         if (includeInAll != null) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java
index 197b34b67da..e8b91f9fbd6 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java
@@ -73,12 +73,12 @@ public class StringFieldMapper extends FieldMapper implements AllFieldMapper.Inc
             "type",
             // common keyword parameters, for which the upgrade is straightforward
             "index", "store", "doc_values", "omit_norms", "norms", "fields", "copy_to",
-            "fielddata", "ignore_above"));
+            "fielddata", "include_in_all", "ignore_above"));
     private static final Set<String> SUPPORTED_PARAMETERS_FOR_AUTO_UPGRADE_TO_TEXT = new HashSet<>(Arrays.asList(
             "type",
             // common text parameters, for which the upgrade is straightforward
             "index", "store", "doc_values", "omit_norms", "norms", "fields", "copy_to",
-            "fielddata", "analyzer", "search_analyzer", "search_quote_analyzer"));
+            "fielddata", "include_in_all", "analyzer", "search_analyzer", "search_quote_analyzer"));
 
     public static class Defaults {
         public static double FIELDDATA_MIN_FREQUENCY = 0;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/TextFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/TextFieldMapper.java
index 4696bb13185..61102895f8f 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/TextFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/TextFieldMapper.java
@@ -317,6 +317,11 @@ public class TextFieldMapper extends FieldMapper implements AllFieldMapper.Inclu
         return (TextFieldMapper) super.clone();
     }
 
+    // pkg-private for testing
+    Boolean includeInAll() {
+        return includeInAll;
+    }
+
     @Override
     public TextFieldMapper includeInAll(Boolean includeInAll) {
         if (includeInAll != null) {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/StringMappingUpgradeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/core/StringMappingUpgradeTests.java
index 41094fec58a..11c0011838e 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/core/StringMappingUpgradeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/core/StringMappingUpgradeTests.java
@@ -198,6 +198,32 @@ public class StringMappingUpgradeTests extends ESSingleNodeTestCase {
         assertEquals("keyword", field.fieldType().searchQuoteAnalyzer().name());
     }
 
+    public void testUpgradeTextIncludeInAll() throws IOException {
+        IndexService indexService = createIndex("test");
+        DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field").field("type", "string")
+                .field("include_in_all", false).endObject().endObject()
+                .endObject().endObject().string();
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        FieldMapper field = mapper.mappers().getMapper("field");
+        assertThat(field, instanceOf(TextFieldMapper.class));
+        assertFalse(((TextFieldMapper) field).includeInAll());
+    }
+
+    public void testUpgradeKeywordIncludeInAll() throws IOException {
+        IndexService indexService = createIndex("test");
+        DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
+        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties").startObject("field").field("type", "string")
+                .field("index", "not_analyzed").field("include_in_all", true).endObject().endObject()
+                .endObject().endObject().string();
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
+        FieldMapper field = mapper.mappers().getMapper("field");
+        assertThat(field, instanceOf(KeywordFieldMapper.class));
+        assertTrue(((KeywordFieldMapper) field).includeInAll());
+    }
+
     public void testUpgradeRandomMapping() throws IOException {
         final int iters = 20;
         for (int i = 0; i < iters; ++i) {

From 2f2ea0c280db993b79858ecffba4f076c5f94070 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Tue, 21 Jun 2016 18:02:55 +0200
Subject: [PATCH 0998/1311] Improved docs explaining the index upgrade process
 in breaking changes

---
 docs/reference/migration/migrate_5_0.asciidoc | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/docs/reference/migration/migrate_5_0.asciidoc b/docs/reference/migration/migrate_5_0.asciidoc
index 7dc79478c8a..da4ba2c521d 100644
--- a/docs/reference/migration/migrate_5_0.asciidoc
+++ b/docs/reference/migration/migrate_5_0.asciidoc
@@ -7,13 +7,9 @@ your application to Elasticsearch 5.0.
 [float]
 === Indices created before 5.0
 
-Elasticsearch 5.0 can read indices created in version 2.0 and above.  If any
-of your indices were created before 2.0 you will need to upgrade to the
-latest 2.x version of Elasticsearch first, in order to upgrade your indices or
-to delete the old indices. Elasticsearch will not start in the presence of old
-indices. To upgrade 2.x indices, first start a node which have access to all
-the data folders and let it upgrade all the indices before starting up rest of
-the cluster.
+Elasticsearch 5.0 can read indices created in version 2.0 or above.  An
+Elasticsearch 5.0 node will not start in the presence of indices created in a
+version of Elasticsearch before 2.0.
 
 [IMPORTANT]
 .Reindex indices from Elasticseach 1.x or before
@@ -26,6 +22,12 @@ way to do this is to upgrade to Elasticsearch 2.3 or later and to use the
 
 =========================================
 
+The first time Elasticsearch 5.0 starts, it will automatically rename index
+folders to use the index UUID instead of the index name. If you are using
+<<indices-shadow-replicas,shadow replicas>> with shared data folders, first
+start a single node with access to all data folders, and let it rename all
+index folders before starting other nodes in the cluster.
+
 [float]
 === Also see:
 

From 969e953645a4b1a28aaa834ca0c4826ba5ea19a2 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Mon, 20 Jun 2016 18:06:31 +0200
Subject: [PATCH 0999/1311] Remove `_timestamp` and `_ttl` on 5.x indices.
 #18980

This removes the ability to use `_timestamp` and `_ttl` on indices created on
or after 5.0.

Closes #18280
---
 .../index/mapper/internal/TTLFieldMapper.java |   7 +
 .../mapper/internal/TimestampFieldMapper.java |   7 +
 .../elasticsearch/aliases/IndexAliasesIT.java |   8 +-
 .../cluster/SpecificMasterNodesIT.java        |  11 +-
 .../org/elasticsearch/get/GetActionIT.java    |  39 +--
 .../mapper/all/SimpleAllMapperTests.java      |  20 --
 .../timestamp/TimestampMappingTests.java      | 104 ++++++--
 .../index/mapper/ttl/TTLMappingTests.java     |  54 +++-
 .../ParseMappingTypeLevelTests.java           |   4 +-
 .../update/UpdateMappingOnClusterIT.java      |  10 -
 .../mapper/update/UpdateMappingTests.java     |  50 ----
 .../aggregations/bucket/DateHistogramIT.java  |  11 +-
 .../search/sort/FieldSortIT.java              |  42 +---
 .../timestamp/SimpleTimestampIT.java          |  19 +-
 .../org/elasticsearch/ttl/SimpleTTLIT.java    |  15 +-
 .../update/TimestampTTLBWIT.java              | 237 ++++++++++++++++++
 .../org/elasticsearch/update/UpdateIT.java    |  59 +----
 .../messy/tests/SearchFieldsTests.java        |  12 +-
 .../rest-api-spec/test/README.asciidoc        |   4 +-
 .../test/create/70_timestamp.yaml             |  80 ------
 .../rest-api-spec/test/create/75_ttl.yaml     | 100 --------
 .../test/index/70_timestamp.yaml              |  70 ------
 .../rest-api-spec/test/index/75_ttl.yaml      |  85 -------
 .../test/update/70_timestamp.yaml             |  76 ------
 .../rest-api-spec/test/update/75_ttl.yaml     |  92 -------
 .../test/update/85_fields_meta.yaml           |  11 +-
 .../elasticsearch/test/ESIntegTestCase.java   |   8 +-
 27 files changed, 440 insertions(+), 795 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java
 delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/create/70_timestamp.yaml
 delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/create/75_ttl.yaml
 delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/index/70_timestamp.yaml
 delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/index/75_ttl.yaml
 delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/update/70_timestamp.yaml
 delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/update/75_ttl.yaml

diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java
index 33d194a7892..ec1024c07f5 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper.internal;
 
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -100,6 +101,9 @@ public class TTLFieldMapper extends MetadataFieldMapper {
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
         public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+            if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0_alpha4)) {
+                throw new IllegalArgumentException("[_ttl] is removed in 5.0. As a replacement, you should use time based indexes or cron a delete-by-query with a range query on a timestamp field.");
+            }
             Builder builder = new Builder();
             for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
                 Map.Entry<String, Object> entry = iterator.next();
@@ -165,6 +169,9 @@ public class TTLFieldMapper extends MetadataFieldMapper {
     private TTLFieldMapper(MappedFieldType fieldType, EnabledAttributeMapper enabled, long defaultTTL,
                              Settings indexSettings) {
         super(NAME, fieldType, Defaults.TTL_FIELD_TYPE, indexSettings);
+        if (enabled.enabled && Version.indexCreated(indexSettings).onOrAfter(Version.V_5_0_0_alpha4)) {
+            throw new IllegalArgumentException("[_ttl] is removed in 5.0. As a replacement, you should use time based indexes or cron a delete-by-query with a range query on a timestamp field.");
+        }
         this.enabledState = enabled;
         this.defaultTTL = defaultTTL;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
index 09f1144db28..24a86b11392 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
@@ -22,6 +22,7 @@ package org.elasticsearch.index.mapper.internal;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.index.IndexOptions;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.TimestampParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
@@ -126,6 +127,9 @@ public class TimestampFieldMapper extends MetadataFieldMapper {
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
         public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+            if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0_alpha4)) {
+                throw new IllegalArgumentException("[_timestamp] is removed in 5.0. As a replacement, you can use an ingest pipeline to add a field with the current timestamp to your documents.");
+            }
             Builder builder = new Builder(parserContext.mapperService().fullName(NAME), parserContext.mapperService().getIndexSettings().getSettings());
             boolean defaultSet = false;
             Boolean ignoreMissing = null;
@@ -200,6 +204,9 @@ public class TimestampFieldMapper extends MetadataFieldMapper {
     private TimestampFieldMapper(MappedFieldType fieldType, MappedFieldType defaultFieldType, EnabledAttributeMapper enabledState,
                                    String defaultTimestamp, Boolean ignoreMissing, Settings indexSettings) {
         super(NAME, fieldType, defaultFieldType, indexSettings);
+        if (enabledState.enabled && Version.indexCreated(indexSettings).onOrAfter(Version.V_5_0_0_alpha4)) {
+            throw new IllegalArgumentException("[_timestamp] is removed in 5.0. As a replacement, you can use an ingest pipeline to add a field with the current timestamp to your documents.");
+        }
         this.enabledState = enabledState;
         this.defaultTimestamp = defaultTimestamp;
         this.ignoreMissing = ignoreMissing;
diff --git a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
index 2a16625d037..5ac1bf40af6 100644
--- a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
+++ b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
@@ -1035,13 +1035,13 @@ public class IndexAliasesIT extends ESIntegTestCase {
     }
 
     public void testAliasFilterWithNowInRangeFilterAndQuery() throws Exception {
-        assertAcked(prepareCreate("my-index").addMapping("my-type", "_timestamp", "enabled=true"));
-        assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter1", rangeQuery("_timestamp").from("now-1d").to("now")));
-        assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter2", rangeQuery("_timestamp").from("now-1d").to("now")));
+        assertAcked(prepareCreate("my-index").addMapping("my-type", "timestamp", "type=date"));
+        assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter1", rangeQuery("timestamp").from("2016-12-01").to("2016-12-31")));
+        assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter2", rangeQuery("timestamp").from("2016-01-01").to("2016-12-31")));
 
         final int numDocs = scaledRandomIntBetween(5, 52);
         for (int i = 1; i <= numDocs; i++) {
-            client().prepareIndex("my-index", "my-type").setCreate(true).setSource("{}").get();
+            client().prepareIndex("my-index", "my-type").setCreate(true).setSource("timestamp", "2016-12-12").get();
             if (i % 2 == 0) {
                 refresh();
                 SearchResponse response = client().prepareSearch("filter1").get();
diff --git a/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java b/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
index 72db338ab80..4773aafbf3f 100644
--- a/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
@@ -30,6 +30,7 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 
 import java.io.IOException;
+import java.util.Map;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
@@ -111,16 +112,18 @@ public class SpecificMasterNodesIT extends ESIntegTestCase {
         internalCluster().startNode(settingsBuilder().put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false));
 
         createIndex("test");
-        assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("_timestamp", "enabled=true"));
+        assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("timestamp", "type=date"));
 
         MappingMetaData defaultMapping = client().admin().cluster().prepareState().get().getState().getMetaData().getIndices().get("test").getMappings().get("_default_");
-        assertThat(defaultMapping.getSourceAsMap().get("_timestamp"), notNullValue());
+        Map<?,?> properties = (Map<?, ?>) defaultMapping.getSourceAsMap().get("properties");
+        assertThat(properties.get("timestamp"), notNullValue());
 
-        assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("_timestamp", "enabled=true"));
+        assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("timestamp", "type=date"));
 
         assertAcked(client().admin().indices().preparePutMapping("test").setType("type1").setSource("foo", "enabled=true"));
         MappingMetaData type1Mapping = client().admin().cluster().prepareState().get().getState().getMetaData().getIndices().get("test").getMappings().get("type1");
-        assertThat(type1Mapping.getSourceAsMap().get("_timestamp"), notNullValue());
+        properties = (Map<?, ?>) type1Mapping.getSourceAsMap().get("properties");
+        assertThat(properties.get("timestamp"), notNullValue());
     }
 
     public void testAliasFilterValidation() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/get/GetActionIT.java b/core/src/test/java/org/elasticsearch/get/GetActionIT.java
index 8c4b699cbff..b73f563fbb3 100644
--- a/core/src/test/java/org/elasticsearch/get/GetActionIT.java
+++ b/core/src/test/java/org/elasticsearch/get/GetActionIT.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.get;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.flush.FlushResponse;
@@ -30,7 +29,6 @@ import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.get.MultiGetRequest;
 import org.elasticsearch.action.get.MultiGetRequestBuilder;
 import org.elasticsearch.action.get.MultiGetResponse;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -40,15 +38,11 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
-import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.InternalSettingsPlugin;
 
 import java.io.IOException;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
-import java.util.Map;
 import java.util.Set;
 
 import static java.util.Collections.singleton;
@@ -57,7 +51,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
@@ -536,7 +529,7 @@ public class GetActionIT extends ESIntegTestCase {
     public void testGetFieldsMetaData() throws Exception {
         assertAcked(prepareCreate("test")
                 .addMapping("parent")
-                .addMapping("my-type1", "_timestamp", "enabled=true", "_ttl", "enabled=true", "_parent", "type=parent")
+                .addMapping("my-type1", "_parent", "type=parent")
                 .addAlias(new Alias("alias"))
                 .setSettings(Settings.builder().put("index.refresh_interval", -1)));
 
@@ -557,12 +550,6 @@ public class GetActionIT extends ESIntegTestCase {
         assertThat(getResponse.getField("field1").getValue().toString(), equalTo("value"));
         assertThat(getResponse.getField("_routing").isMetadataField(), equalTo(true));
         assertThat(getResponse.getField("_routing").getValue().toString(), equalTo("1"));
-        assertThat(getResponse.getField("_timestamp").isMetadataField(), equalTo(true));
-        assertThat(getResponse.getField("_timestamp").getValue().toString(), equalTo("205097"));
-        assertThat(getResponse.getField("_ttl").isMetadataField(), equalTo(true));
-        // TODO: _ttl should return the original value, but it does not work today because
-        // it would use now() instead of the value of _timestamp to rebase
-        // assertThat(getResponse.getField("_ttl").getValue().toString(), equalTo("10000000205097"));
         assertThat(getResponse.getField("_parent").isMetadataField(), equalTo(true));
         assertThat(getResponse.getField("_parent").getValue().toString(), equalTo("parent_1"));
 
@@ -577,12 +564,6 @@ public class GetActionIT extends ESIntegTestCase {
         assertThat(getResponse.getField("field1").getValue().toString(), equalTo("value"));
         assertThat(getResponse.getField("_routing").isMetadataField(), equalTo(true));
         assertThat(getResponse.getField("_routing").getValue().toString(), equalTo("1"));
-        assertThat(getResponse.getField("_timestamp").isMetadataField(), equalTo(true));
-        assertThat(getResponse.getField("_timestamp").getValue().toString(), equalTo("205097"));
-        assertThat(getResponse.getField("_ttl").isMetadataField(), equalTo(true));
-        // TODO: _ttl should return the original value, but it does not work today because
-        // it would use now() instead of the value of _timestamp to rebase
-        //assertThat(getResponse.getField("_ttl").getValue().toString(), equalTo("10000000000000"));
         assertThat(getResponse.getField("_parent").isMetadataField(), equalTo(true));
         assertThat(getResponse.getField("_parent").getValue().toString(), equalTo("parent_1"));
     }
@@ -779,16 +760,10 @@ public class GetActionIT extends ESIntegTestCase {
                 "  },\n" +
                 "  \"mappings\": {\n" +
                 "    \"parentdoc\": {\n" +
-                "      \"_ttl\": {\n" +
-                "        \"enabled\": true\n" +
-                "      }\n" +
                 "    },\n" +
                 "    \"doc\": {\n" +
                 "      \"_parent\": {\n" +
                 "        \"type\": \"parentdoc\"\n" +
-                "      },\n" +
-                "      \"_ttl\": {\n" +
-                "        \"enabled\": true\n" +
                 "      }\n" +
                 "    }\n" +
                 "  }\n" +
@@ -798,7 +773,7 @@ public class GetActionIT extends ESIntegTestCase {
 
         client().prepareIndex("test", "doc").setId("1").setSource("{}").setParent("1").setTTL(TimeValue.timeValueHours(1).getMillis()).get();
 
-        String[] fieldsList = {"_ttl", "_parent"};
+        String[] fieldsList = {"_parent"};
         // before refresh - document is only in translog
         assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "1");
         refresh();
@@ -814,14 +789,6 @@ public class GetActionIT extends ESIntegTestCase {
             "  \"settings\": {\n" +
             "    \"index.translog.flush_threshold_size\": \"1pb\",\n" +
             "    \"refresh_interval\": \"-1\"\n" +
-            "  },\n" +
-            "  \"mappings\": {\n" +
-            "    \"parentdoc\": {},\n" +
-            "    \"doc\": {\n" +
-            "      \"_timestamp\": {\n" +
-            "        \"enabled\": true\n" +
-            "      }\n" +
-            "    }\n" +
             "  }\n" +
             "}";
 
@@ -831,7 +798,7 @@ public class GetActionIT extends ESIntegTestCase {
             "  \"text\": \"some text.\"\n" +
             "}\n";
         client().prepareIndex("test", "doc").setId("1").setSource(doc).setRouting("1").get();
-        String[] fieldsList = {"_timestamp", "_routing"};
+        String[] fieldsList = {"_routing"};
         // before refresh - document is only in translog
         assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "1");
         refresh();
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java
index 966146655b8..165b49d3145 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java
@@ -40,7 +40,6 @@ import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext.Document;
 import org.elasticsearch.index.mapper.internal.AllFieldMapper;
-import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -50,9 +49,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath;
@@ -430,23 +427,6 @@ public class SimpleAllMapperTests extends ESSingleNodeTestCase {
         parser.parse("test", new CompressedXContent(mapping));
     }
 
-    // issue https://github.com/elastic/elasticsearch/issues/5864
-    public void testMetadataMappersStillWorking() throws MapperParsingException, IOException {
-        String mapping = "{";
-        Map<String, String> rootTypes = new HashMap<>();
-        //just pick some example from DocumentMapperParser.rootTypeParsers
-        rootTypes.put(TimestampFieldMapper.NAME, "{\"enabled\" : true}");
-        rootTypes.put("include_in_all", "true");
-        rootTypes.put("dynamic_date_formats", "[\"yyyy-MM-dd\", \"dd-MM-yyyy\"]");
-        rootTypes.put("numeric_detection", "true");
-        rootTypes.put("dynamic_templates", "[]");
-        for (String key : rootTypes.keySet()) {
-            mapping += "\"" + key+ "\"" + ":" + rootTypes.get(key) + ",\n";
-        }
-        mapping += "\"properties\":{}}" ;
-        createIndex("test").mapperService().documentMapperParser().parse("test", new CompressedXContent(mapping));
-    }
-
     public void testDocValuesNotAllowed() throws IOException {
         String mapping = jsonBuilder().startObject().startObject("type")
             .startObject("_all")
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
index 14d86290db4..78da5abb746 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
@@ -22,6 +22,7 @@ package org.elasticsearch.index.mapper.timestamp;
 import org.apache.lucene.index.IndexOptions;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.TimestampParsingException;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
@@ -31,25 +32,31 @@ import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.joda.Joda;
+import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceToParse;
+import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
+import org.elasticsearch.test.VersionUtils;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.LinkedHashMap;
 
+import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
 import static org.elasticsearch.test.VersionUtils.randomVersion;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.containsString;
@@ -58,19 +65,35 @@ import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.startsWith;
 
 /**
  */
 public class TimestampMappingTests extends ESSingleNodeTestCase {
 
+    private static final Settings BW_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0).build();
+
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(InternalSettingsPlugin.class);
     }
 
+    public void testRejectedOn5x() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject()
+                .startObject("type")
+                    .startObject("_timestamp")
+                        .field("enabled", true)
+                    .endObject()
+                .endObject().endObject().string();
+        IndexService index = createIndex("test");
+        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class,
+                () -> index.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, false));
+        assertThat(expected.getMessage(), startsWith("[_timestamp] is removed"));
+    }
+
     public void testSimpleDisabled() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         BytesReference source = XContentFactory.jsonBuilder()
                 .startObject()
                 .field("field", "value")
@@ -85,7 +108,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_timestamp").field("enabled", "yes").endObject()
                 .endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         BytesReference source = XContentFactory.jsonBuilder()
                 .startObject()
                 .field("field", "value")
@@ -99,10 +122,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
     }
 
     public void testDefaultValues() throws Exception {
-        Version version;
-        do {
-            version = randomVersion(random());
-        } while (version.before(Version.V_2_0_0_beta1));
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0_beta1, Version.V_5_0_0_alpha3);
         for (String mapping : Arrays.asList(
                 XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(),
                 XContentFactory.jsonBuilder().startObject().startObject("type").startObject("_timestamp").endObject().endObject().endObject().string())) {
@@ -120,7 +140,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
         String enabledMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_timestamp").field("enabled", true).endObject()
                 .endObject().endObject().string();
-        MapperService mapperService = createIndex("test").mapperService();
+        MapperService mapperService = createIndex("test", BW_SETTINGS).mapperService();
         DocumentMapper enabledMapper = mapperService.merge("type", new CompressedXContent(enabledMapping), MapperService.MergeReason.MAPPING_UPDATE, false);
 
         String disabledMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
@@ -146,7 +166,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                     .field("foo", "bar")
                 .endObject();
 
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
+        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
         MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData();
 
         MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
@@ -172,7 +192,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                 .endObject();
 
         MetaData metaData = MetaData.builder().build();
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
+        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
 
         MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
 
@@ -195,7 +215,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                 .endObject()
                 .endObject().endObject();
         try {
-            createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
+            createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
             fail("we should reject the mapping with a TimestampParsingException: default timestamp can not be set to null");
         } catch (TimestampParsingException e) {
             assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set to null"));
@@ -212,7 +232,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                 .endObject().endObject();
 
         try {
-            createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
+            createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
             fail("we should reject the mapping with a TimestampParsingException: default timestamp can not be set to null");
         } catch (TimestampParsingException e) {
             assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set to null"));
@@ -230,7 +250,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                 .endObject().endObject();
 
         try {
-            createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
+            createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
             fail("we should reject the mapping with a TimestampParsingException: default timestamp can not be set with ignore_missing set to false");
         } catch (TimestampParsingException e) {
             assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set with ignore_missing set to false"));
@@ -250,7 +270,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                 .endObject();
 
         MetaData metaData = MetaData.builder().build();
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
+        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
 
         MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
 
@@ -323,7 +343,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                     .field("enabled", true)
                     .field("default", "1970-01-01")
                 .endObject().endObject().endObject().string();
-        DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
+        DocumentMapperParser parser = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser();
 
         DocumentMapper docMapper = parser.parse("type", new CompressedXContent(mapping));
         docMapper = parser.parse("type", docMapper.mappingSource());
@@ -360,7 +380,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("_timestamp").field("enabled", true).field("default", "1970").field("format", "YYYY").endObject()
             .endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
         try {
             docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
@@ -375,7 +395,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_timestamp").field("enabled", true).field("format", "yyyyMMddHH").endObject()
             .endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData();
 
         XContentBuilder doc = XContentFactory.jsonBuilder().startObject().endObject();
@@ -392,7 +412,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                 .endObject().endObject().string();
         BytesReference source = XContentFactory.jsonBuilder().startObject().field("field", "value").endObject().bytes();
         // test with 2.x
-        DocumentMapper currentMapper = createIndex("new-index").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper currentMapper = createIndex("new-index", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
         // this works with 2.x
         IndexRequest request = new IndexRequest("new-index", "type", "1").source(source).timestamp("1970-01-01");
@@ -407,4 +427,54 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
             assertThat(e.getMessage(), containsString("failed to parse timestamp [1234567890]"));
         }
     }
+
+    public void testSizeTimestampIndexParsing() throws IOException {
+        IndexService indexService = createIndex("test", BW_SETTINGS);
+        String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/default_mapping_with_disabled_root_types.json");
+        DocumentMapper documentMapper = indexService.mapperService().parse("type", new CompressedXContent(mapping), true);
+        assertThat(documentMapper.mappingSource().string(), equalTo(mapping));
+        documentMapper = indexService.mapperService().parse("type", new CompressedXContent(documentMapper.mappingSource().string()), true);
+        assertThat(documentMapper.mappingSource().string(), equalTo(mapping));
+    }
+
+    public void testDefaultApplied() throws IOException {
+        createIndex("test1", BW_SETTINGS);
+        createIndex("test2", BW_SETTINGS);
+        XContentBuilder defaultMapping = XContentFactory.jsonBuilder().startObject()
+                .startObject(MapperService.DEFAULT_MAPPING).startObject("_timestamp").field("enabled", true).endObject().endObject()
+                .endObject();
+        client().admin().indices().preparePutMapping().setType(MapperService.DEFAULT_MAPPING).setSource(defaultMapping).get();
+        XContentBuilder typeMapping = XContentFactory.jsonBuilder().startObject()
+                .startObject("type").startObject("_all").field("enabled", false).endObject().endObject()
+                .endObject();
+        client().admin().indices().preparePutMapping("test1").setType("type").setSource(typeMapping).get();
+        client().admin().indices().preparePutMapping("test1", "test2").setType("type").setSource(typeMapping).get();
+
+        GetMappingsResponse response = client().admin().indices().prepareGetMappings("test2").get();
+        assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_all"));
+        assertFalse((Boolean) ((LinkedHashMap) response.getMappings().get("test2").get("type").getSourceAsMap().get("_all")).get("enabled"));
+        assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_timestamp"));
+        assertTrue((Boolean)((LinkedHashMap)response.getMappings().get("test2").get("type").getSourceAsMap().get("_timestamp")).get("enabled"));
+    }
+
+    public void testTimestampParsing() throws IOException {
+        IndexService indexService = createIndex("test", BW_SETTINGS);
+        XContentBuilder indexMapping = XContentFactory.jsonBuilder();
+        boolean enabled = randomBoolean();
+        indexMapping.startObject()
+                .startObject("type")
+                .startObject("_timestamp")
+                .field("enabled", enabled)
+                .endObject()
+                .endObject()
+                .endObject();
+        DocumentMapper documentMapper = indexService.mapperService().parse("type", new CompressedXContent(indexMapping.string()), true);
+        assertThat(documentMapper.timestampFieldMapper().enabled(), equalTo(enabled));
+        assertTrue(documentMapper.timestampFieldMapper().fieldType().stored());
+        assertTrue(documentMapper.timestampFieldMapper().fieldType().hasDocValues());
+        documentMapper = indexService.mapperService().parse("type", new CompressedXContent(documentMapper.mappingSource().string()), true);
+        assertThat(documentMapper.timestampFieldMapper().enabled(), equalTo(enabled));
+        assertTrue(documentMapper.timestampFieldMapper().fieldType().hasDocValues());
+        assertTrue(documentMapper.timestampFieldMapper().fieldType().stored());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java
index fe057045882..ad9e03327b2 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java
@@ -20,7 +20,9 @@
 package org.elasticsearch.index.mapper.ttl;
 
 import org.apache.lucene.index.IndexOptions;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
@@ -30,22 +32,48 @@ import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.mapper.internal.TTLFieldMapper;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
 
 import java.io.IOException;
+import java.util.Collection;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.startsWith;
 
 public class TTLMappingTests extends ESSingleNodeTestCase {
+
+    private static final Settings BW_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0).build();
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(InternalSettingsPlugin.class);
+    }
+
+    public void testRejectedOn5x() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject()
+                .startObject("type")
+                    .startObject("_ttl")
+                        .field("enabled", true)
+                    .endObject()
+                .endObject().endObject().string();
+        IndexService index = createIndex("test");
+        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class,
+                () -> index.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, false));
+        assertThat(expected.getMessage(), startsWith("[_ttl] is removed"));
+    }
+
     public void testSimpleDisabled() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         BytesReference source = XContentFactory.jsonBuilder()
                 .startObject()
                 .field("field", "value")
@@ -60,7 +88,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_ttl").field("enabled", "yes").endObject()
                 .endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         BytesReference source = XContentFactory.jsonBuilder()
                 .startObject()
                 .field("field", "value")
@@ -75,7 +103,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
 
     public void testDefaultValues() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         assertThat(docMapper.TTLFieldMapper().enabled(), equalTo(TTLFieldMapper.Defaults.ENABLED_STATE.enabled));
         assertThat(docMapper.TTLFieldMapper().fieldType().stored(), equalTo(TTLFieldMapper.Defaults.TTL_FIELD_TYPE.stored()));
         assertThat(docMapper.TTLFieldMapper().fieldType().indexOptions(), equalTo(TTLFieldMapper.Defaults.TTL_FIELD_TYPE.indexOptions()));
@@ -93,7 +121,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
                 .startObject("properties").field("field").startObject().field("type", "text").endObject().endObject()
                 .endObject().endObject().string();
 
-        MapperService mapperService = createIndex("test").mapperService();
+        MapperService mapperService = createIndex("test", BW_SETTINGS).mapperService();
         DocumentMapper mapperWithoutTtl = mapperService.merge("type", new CompressedXContent(mappingWithoutTtl), MapperService.MergeReason.MAPPING_UPDATE, false);
         DocumentMapper mapperWithTtl = mapperService.merge("type", new CompressedXContent(mappingWithTtl), MapperService.MergeReason.MAPPING_UPDATE, false);
 
@@ -116,7 +144,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
                 .startObject("properties").field("field").startObject().field("type", "text").endObject().endObject()
                 .endObject().endObject().string();
 
-        MapperService mapperService = createIndex("test").mapperService();
+        MapperService mapperService = createIndex("test", BW_SETTINGS).mapperService();
         DocumentMapper initialMapper = mapperService.merge("type", new CompressedXContent(mappingWithTtl), MapperService.MergeReason.MAPPING_UPDATE, false);
         DocumentMapper updatedMapper = mapperService.merge("type", new CompressedXContent(updatedMapping), MapperService.MergeReason.MAPPING_UPDATE, false);
 
@@ -127,7 +155,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
     public void testThatDisablingTTLReportsConflict() throws Exception {
         String mappingWithTtl = getMappingWithTtlEnabled().string();
         String mappingWithTtlDisabled = getMappingWithTtlDisabled().string();
-        MapperService mapperService = createIndex("test").mapperService();
+        MapperService mapperService = createIndex("test", BW_SETTINGS).mapperService();
         DocumentMapper initialMapper = mapperService.merge("type", new CompressedXContent(mappingWithTtl), MapperService.MergeReason.MAPPING_UPDATE, false);
 
         try {
@@ -143,7 +171,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
     public void testThatDisablingTTLReportsConflictOnCluster() throws Exception {
         String mappingWithTtl = getMappingWithTtlEnabled().string();
         String mappingWithTtlDisabled = getMappingWithTtlDisabled().string();
-        assertAcked(client().admin().indices().prepareCreate("testindex").addMapping("type", mappingWithTtl));
+        assertAcked(client().admin().indices().prepareCreate("testindex").setSettings(BW_SETTINGS).addMapping("type", mappingWithTtl));
         GetMappingsResponse mappingsBeforeUpdateResponse = client().admin().indices().prepareGetMappings("testindex").addTypes("type").get();
         try {
             client().admin().indices().preparePutMapping("testindex").setSource(mappingWithTtlDisabled).setType("type").get();
@@ -158,7 +186,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
     public void testThatEnablingTTLAfterFirstDisablingWorks() throws Exception {
         String mappingWithTtl = getMappingWithTtlEnabled().string();
         String withTtlDisabled = getMappingWithTtlDisabled().string();
-        assertAcked(client().admin().indices().prepareCreate("testindex").addMapping("type", withTtlDisabled));
+        assertAcked(client().admin().indices().prepareCreate("testindex").setSettings(BW_SETTINGS).addMapping("type", withTtlDisabled));
         GetMappingsResponse mappingsAfterUpdateResponse = client().admin().indices().prepareGetMappings("testindex").addTypes("type").get();
         assertThat(mappingsAfterUpdateResponse.getMappings().get("testindex").get("type").sourceAsMap().get("_ttl").toString(), equalTo("{enabled=false}"));
         client().admin().indices().preparePutMapping("testindex").setSource(mappingWithTtl).setType("type").get();
@@ -167,20 +195,20 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
     }
 
     public void testNoConflictIfNothingSetAndDisabledLater() throws Exception {
-        IndexService indexService = createIndex("testindex", Settings.builder().build(), "type");
+        IndexService indexService = createIndex("testindex", BW_SETTINGS, "type");
         XContentBuilder mappingWithTtlDisabled = getMappingWithTtlDisabled("7d");
         indexService.mapperService().merge("type", new CompressedXContent(mappingWithTtlDisabled.string()), MapperService.MergeReason.MAPPING_UPDATE, false);
     }
 
     public void testNoConflictIfNothingSetAndEnabledLater() throws Exception {
-        IndexService indexService = createIndex("testindex", Settings.builder().build(), "type");
+        IndexService indexService = createIndex("testindex", BW_SETTINGS, "type");
         XContentBuilder mappingWithTtlEnabled = getMappingWithTtlEnabled("7d");
         indexService.mapperService().merge("type", new CompressedXContent(mappingWithTtlEnabled.string()), MapperService.MergeReason.MAPPING_UPDATE, false);
     }
 
     public void testMergeWithOnlyDefaultSet() throws Exception {
         XContentBuilder mappingWithTtlEnabled = getMappingWithTtlEnabled("7d");
-        IndexService indexService = createIndex("testindex", Settings.builder().build(), "type", mappingWithTtlEnabled);
+        IndexService indexService = createIndex("testindex", BW_SETTINGS, "type", mappingWithTtlEnabled);
         XContentBuilder mappingWithOnlyDefaultSet = getMappingWithOnlyTtlDefaultSet("6m");
         indexService.mapperService().merge("type", new CompressedXContent(mappingWithOnlyDefaultSet.string()), MapperService.MergeReason.MAPPING_UPDATE, false);
         CompressedXContent mappingAfterMerge = indexService.mapperService().documentMapper("type").mappingSource();
@@ -189,7 +217,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
 
     public void testMergeWithOnlyDefaultSetTtlDisabled() throws Exception {
         XContentBuilder mappingWithTtlEnabled = getMappingWithTtlDisabled("7d");
-        IndexService indexService = createIndex("testindex", Settings.builder().build(), "type", mappingWithTtlEnabled);
+        IndexService indexService = createIndex("testindex", BW_SETTINGS, "type", mappingWithTtlEnabled);
         CompressedXContent mappingAfterCreation = indexService.mapperService().documentMapper("type").mappingSource();
         assertThat(mappingAfterCreation, equalTo(new CompressedXContent("{\"type\":{\"_ttl\":{\"enabled\":false},\"properties\":{\"field\":{\"type\":\"text\"}}}}")));
         XContentBuilder mappingWithOnlyDefaultSet = getMappingWithOnlyTtlDefaultSet("6m");
@@ -202,7 +230,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("_ttl").field("enabled", true).endObject()
             .endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
         try {
             docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java b/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java
index 1d849d50932..3041cc8d13c 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java
@@ -31,12 +31,12 @@ import static org.hamcrest.Matchers.equalTo;
 public class ParseMappingTypeLevelTests extends ESSingleNodeTestCase {
     public void testTypeLevel() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("_timestamp").field("enabled", true).endObject()
+                .startObject("_all").field("enabled", false).endObject()
                 .endObject().endObject().string();
 
         DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
         assertThat(mapper.type(), equalTo("type"));
-        assertThat(mapper.timestampFieldMapper().enabled(), equalTo(true));
+        assertThat(mapper.allFieldMapper().enabled(), equalTo(false));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java
index ed20cdb7e35..afac48e2b89 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java
@@ -139,16 +139,6 @@ public class UpdateMappingOnClusterIT extends ESIntegTestCase {
         compareMappingOnNodes(mappingsBeforeUpdateResponse);
     }
 
-    // checks if the setting for timestamp and size are kept even if disabled
-    public void testDisabledSizeTimestampIndexDoNotLooseMappings() throws Exception {
-        String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/default_mapping_with_disabled_root_types.json");
-        prepareCreate(INDEX).addMapping(TYPE, mapping).get();
-        GetMappingsResponse mappingsBeforeGreen = client().admin().indices().prepareGetMappings(INDEX).addTypes(TYPE).get();
-        ensureGreen(INDEX);
-        // make sure all nodes have same cluster state
-        compareMappingOnNodes(mappingsBeforeGreen);
-    }
-
     protected void testConflict(String mapping, String mappingUpdate, String... errorMessages) throws InterruptedException {
         assertAcked(prepareCreate(INDEX).setSource(mapping).get());
         ensureGreen(INDEX);
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java
index 17cb8a84664..86624d938b6 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java
@@ -259,56 +259,6 @@ public class UpdateMappingTests extends ESSingleNodeTestCase {
         }
     }
 
-    public void testTimestampParsing() throws IOException {
-        IndexService indexService = createIndex("test");
-        XContentBuilder indexMapping = XContentFactory.jsonBuilder();
-        boolean enabled = randomBoolean();
-        indexMapping.startObject()
-                .startObject("type")
-                .startObject("_timestamp")
-                .field("enabled", enabled)
-                .endObject()
-                .endObject()
-                .endObject();
-        DocumentMapper documentMapper = indexService.mapperService().parse("type", new CompressedXContent(indexMapping.string()), true);
-        assertThat(documentMapper.timestampFieldMapper().enabled(), equalTo(enabled));
-        assertTrue(documentMapper.timestampFieldMapper().fieldType().stored());
-        assertTrue(documentMapper.timestampFieldMapper().fieldType().hasDocValues());
-        documentMapper = indexService.mapperService().parse("type", new CompressedXContent(documentMapper.mappingSource().string()), true);
-        assertThat(documentMapper.timestampFieldMapper().enabled(), equalTo(enabled));
-        assertTrue(documentMapper.timestampFieldMapper().fieldType().hasDocValues());
-        assertTrue(documentMapper.timestampFieldMapper().fieldType().stored());
-    }
-
-    public void testSizeTimestampIndexParsing() throws IOException {
-        IndexService indexService = createIndex("test", Settings.builder().build());
-        String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/default_mapping_with_disabled_root_types.json");
-        DocumentMapper documentMapper = indexService.mapperService().parse("type", new CompressedXContent(mapping), true);
-        assertThat(documentMapper.mappingSource().string(), equalTo(mapping));
-        documentMapper = indexService.mapperService().parse("type", new CompressedXContent(documentMapper.mappingSource().string()), true);
-        assertThat(documentMapper.mappingSource().string(), equalTo(mapping));
-    }
-
-    public void testDefaultApplied() throws IOException {
-        createIndex("test1", Settings.builder().build());
-        createIndex("test2", Settings.builder().build());
-        XContentBuilder defaultMapping = XContentFactory.jsonBuilder().startObject()
-                .startObject(MapperService.DEFAULT_MAPPING).startObject("_timestamp").field("enabled", true).endObject().endObject()
-                .endObject();
-        client().admin().indices().preparePutMapping().setType(MapperService.DEFAULT_MAPPING).setSource(defaultMapping).get();
-        XContentBuilder typeMapping = XContentFactory.jsonBuilder().startObject()
-                .startObject("type").startObject("_all").field("enabled", false).endObject().endObject()
-                .endObject();
-        client().admin().indices().preparePutMapping("test1").setType("type").setSource(typeMapping).get();
-        client().admin().indices().preparePutMapping("test1", "test2").setType("type").setSource(typeMapping).get();
-
-        GetMappingsResponse response = client().admin().indices().prepareGetMappings("test2").get();
-        assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_all"));
-        assertFalse((Boolean) ((LinkedHashMap) response.getMappings().get("test2").get("type").getSourceAsMap().get("_all")).get("enabled"));
-        assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_timestamp"));
-        assertTrue((Boolean)((LinkedHashMap)response.getMappings().get("test2").get("type").getSourceAsMap().get("_timestamp")).get("enabled"));
-    }
-
     public void testRejectFieldDefinedTwice() throws IOException {
         String mapping1 = XContentFactory.jsonBuilder().startObject()
                 .startObject("type1")
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
index cdb722ff9dd..4f7064a33bb 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
@@ -65,7 +65,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSear
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.core.IsNull.notNullValue;
 
@@ -107,8 +106,7 @@ public class DateHistogramIT extends ESIntegTestCase {
 
     @Override
     public void setupSuiteScopeCluster() throws Exception {
-        assertAcked(prepareCreate("idx").addMapping("type", "_timestamp", "enabled=true"));
-        createIndex("idx_unmapped");
+        createIndex("idx", "idx_unmapped");
         // TODO: would be nice to have more random data here
         assertAcked(prepareCreate("empty_bucket_idx").addMapping("type", "value", "type=integer"));
         List<IndexRequestBuilder> builders = new ArrayList<>();
@@ -1141,13 +1139,6 @@ public class DateHistogramIT extends ESIntegTestCase {
         }
     }
 
-    public void testTimestampField() { // see #11692
-        SearchResponse response = client().prepareSearch("idx").addAggregation(dateHistogram("histo").field("_timestamp").dateHistogramInterval(randomFrom(DateHistogramInterval.DAY, DateHistogramInterval.MONTH))).get();
-        assertSearchResponse(response);
-        Histogram histo = response.getAggregations().get("histo");
-        assertThat(histo.getBuckets().size(), greaterThan(0));
-    }
-
     /**
      * When DST ends, local time turns back one hour, so between 2am and 4am wall time we should have four buckets:
      * "2015-10-25T02:00:00.000+02:00",
diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
index 188ae22471a..99183b11c58 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
@@ -36,7 +36,6 @@ import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.hamcrest.Matchers;
@@ -70,10 +69,8 @@ import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThan;
-import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -1340,16 +1337,12 @@ public class FieldSortIT extends ESIntegTestCase {
     }
 
     public void testSortMetaField() throws Exception {
-        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("_timestamp").field("enabled", true).endObject()
-                .endObject().endObject();
-        assertAcked(prepareCreate("test")
-            .addMapping("type", mapping));
+        createIndex("test");
         ensureGreen();
         final int numDocs = randomIntBetween(10, 20);
         IndexRequestBuilder[] indexReqs = new IndexRequestBuilder[numDocs];
         for (int i = 0; i < numDocs; ++i) {
-            indexReqs[i] = client().prepareIndex("test", "type", Integer.toString(i)).setTimestamp(Integer.toString(randomInt(1000)))
+            indexReqs[i] = client().prepareIndex("test", "type", Integer.toString(i))
                     .setSource();
         }
         indexRandom(true, indexReqs);
@@ -1368,37 +1361,6 @@ public class FieldSortIT extends ESIntegTestCase {
             assertThat(previous, order == SortOrder.ASC ? lessThan(uid) : greaterThan(uid));
             previous = uid;
         }
-
-        /*
-        searchResponse = client().prepareSearch()
-                .setQuery(matchAllQuery())
-                .setSize(randomIntBetween(1, numDocs + 5))
-                .addSort("_id", order)
-                .execute().actionGet();
-        assertNoFailures(searchResponse);
-        hits = searchResponse.getHits().hits();
-        previous = order == SortOrder.ASC ? new BytesRef() : UnicodeUtil.BIG_TERM;
-        for (int i = 0; i < hits.length; ++i) {
-            final BytesRef id = new BytesRef(Uid.createUid(hits[i].type(), hits[i].id()));
-            assertThat(previous, order == SortOrder.ASC ? lessThan(id) : greaterThan(id));
-            previous = id;
-        }*/
-
-        searchResponse = client().prepareSearch()
-                .setQuery(matchAllQuery())
-                .setSize(randomIntBetween(1, numDocs + 5))
-                .addSort("_timestamp", order)
-                .addField("_timestamp")
-                .execute().actionGet();
-        assertNoFailures(searchResponse);
-        hits = searchResponse.getHits().hits();
-        Long previousTs = order == SortOrder.ASC ? 0 : Long.MAX_VALUE;
-        for (int i = 0; i < hits.length; ++i) {
-            SearchHitField timestampField = hits[i].getFields().get("_timestamp");
-            Long timestamp = timestampField.<Long>getValue();
-            assertThat(previousTs, order == SortOrder.ASC ? lessThanOrEqualTo(timestamp) : greaterThanOrEqualTo(timestamp));
-            previousTs = timestamp;
-        }
     }
 
     /**
diff --git a/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java b/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java
index 2fa94bc2357..bd24ab9f43b 100644
--- a/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java
+++ b/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java
@@ -19,14 +19,20 @@
 
 package org.elasticsearch.timestamp;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
 import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
 
+import java.util.Collection;
 import java.util.Locale;
 
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
@@ -41,8 +47,17 @@ import static org.hamcrest.Matchers.notNullValue;
 /**
  */
 public class SimpleTimestampIT extends ESIntegTestCase {
+
+    private static final Settings BW_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0).build();
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return pluginList(InternalSettingsPlugin.class);
+    }
+
     public void testSimpleTimestamp() throws Exception {
         client().admin().indices().prepareCreate("test")
+                .setSettings(BW_SETTINGS)
                 .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("_timestamp").field("enabled", true).endObject().endObject().endObject())
                 .execute().actionGet();
         client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet();
@@ -98,7 +113,7 @@ public class SimpleTimestampIT extends ESIntegTestCase {
         String type = "mytype";
 
         XContentBuilder builder = jsonBuilder().startObject().startObject("_timestamp").field("enabled", true).endObject().endObject();
-        assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
+        assertAcked(client().admin().indices().prepareCreate(index).setSettings(BW_SETTINGS).addMapping(type, builder));
 
         // check mapping again
         assertTimestampMappingEnabled(index, type, true);
@@ -117,7 +132,7 @@ public class SimpleTimestampIT extends ESIntegTestCase {
         String type = "mytype";
 
         XContentBuilder builder = jsonBuilder().startObject().startObject("_timestamp").field("enabled", true).endObject().endObject();
-        assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
+        assertAcked(client().admin().indices().prepareCreate(index).setSettings(BW_SETTINGS).addMapping(type, builder));
 
         // check mapping again
         assertTimestampMappingEnabled(index, type, true);
diff --git a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
index 003c08134bc..9e08ecde6fa 100644
--- a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
+++ b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.ttl;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
@@ -26,14 +27,19 @@ import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.update.UpdateRequestBuilder;
 import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 
 import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
@@ -60,6 +66,11 @@ public class SimpleTTLIT extends ESIntegTestCase {
         return 2;
     }
 
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singleton(InternalSettingsPlugin.class);
+    }
+
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
@@ -70,6 +81,7 @@ public class SimpleTTLIT extends ESIntegTestCase {
 
     public void testSimpleTTL() throws Exception {
         assertAcked(prepareCreate("test")
+                .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id)
                 .addMapping("type1", XContentFactory.jsonBuilder()
                         .startObject()
                         .startObject("type1")
@@ -209,7 +221,7 @@ public class SimpleTTLIT extends ESIntegTestCase {
         String type = "mytype";
 
         XContentBuilder builder = jsonBuilder().startObject().startObject("_ttl").field("enabled", true).endObject().endObject();
-        assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
+        assertAcked(client().admin().indices().prepareCreate(index).setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id).addMapping(type, builder));
 
         // check mapping again
         assertTTLMappingEnabled(index, type);
@@ -232,6 +244,7 @@ public class SimpleTTLIT extends ESIntegTestCase {
      */
     public void testNoopUpdate() throws IOException {
         assertAcked(prepareCreate("test")
+                .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id)
                 .addMapping("type1", XContentFactory.jsonBuilder()
                         .startObject()
                         .startObject("type1")
diff --git a/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java b/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java
new file mode 100644
index 00000000000..3573089fcaa
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.update;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.engine.DocumentMissingException;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHitField;
+import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+public class TimestampTTLBWIT extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(
+                UpdateIT.FieldIncrementScriptPlugin.class,
+                UpdateIT.ExtractContextInSourceScriptPlugin.class,
+                UpdateIT.PutFieldValuesScriptPlugin.class,
+                InternalSettingsPlugin.class
+        );
+    }
+
+    public void testSort() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("_timestamp").field("enabled", true).endObject()
+                .endObject().endObject();
+        assertAcked(prepareCreate("test")
+                .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id)
+                .addMapping("type", mapping));
+        ensureGreen();
+        final int numDocs = randomIntBetween(10, 20);
+        IndexRequestBuilder[] indexReqs = new IndexRequestBuilder[numDocs];
+        for (int i = 0; i < numDocs; ++i) {
+            indexReqs[i] = client().prepareIndex("test", "type", Integer.toString(i)).setTimestamp(Integer.toString(randomInt(1000)))
+                    .setSource();
+        }
+        indexRandom(true, indexReqs);
+
+        SortOrder order = randomFrom(SortOrder.values());
+
+        SearchResponse searchResponse = client().prepareSearch()
+                .setQuery(matchAllQuery())
+                .setSize(randomIntBetween(1, numDocs + 5))
+                .addSort("_timestamp", order)
+                .addField("_timestamp")
+                .execute().actionGet();
+        assertNoFailures(searchResponse);
+        SearchHit[] hits = searchResponse.getHits().hits();
+        Long previousTs = order == SortOrder.ASC ? 0 : Long.MAX_VALUE;
+        for (int i = 0; i < hits.length; ++i) {
+            SearchHitField timestampField = hits[i].getFields().get("_timestamp");
+            Long timestamp = timestampField.<Long>getValue();
+            assertThat(previousTs, order == SortOrder.ASC ? lessThanOrEqualTo(timestamp) : greaterThanOrEqualTo(timestamp));
+            previousTs = timestamp;
+        }
+    }
+
+    public void testUpdate() throws Exception {
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias"))
+                .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id)
+                .addMapping("type1", XContentFactory.jsonBuilder()
+                        .startObject()
+                        .startObject("type1")
+                        .startObject("_timestamp").field("enabled", true).endObject()
+                        .startObject("_ttl").field("enabled", true).endObject()
+                        .endObject()
+                        .endObject()));
+
+        ensureGreen();
+
+        try {
+            client().prepareUpdate(indexOrAlias(), "type1", "1")
+                    .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null)).execute().actionGet();
+            fail();
+        } catch (DocumentMissingException e) {
+            // all is well
+        }
+
+        // check TTL is kept after an update without TTL
+        client().prepareIndex("test", "type1", "2").setSource("field", 1).setTTL(86400000L).setRefreshPolicy(IMMEDIATE).get();
+        GetResponse getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
+        long ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue();
+        assertThat(ttl, greaterThan(0L));
+        client().prepareUpdate(indexOrAlias(), "type1", "2")
+                .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null)).execute().actionGet();
+        getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
+        ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue();
+        assertThat(ttl, greaterThan(0L));
+
+        // check TTL update
+        client().prepareUpdate(indexOrAlias(), "type1", "2")
+                .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values",
+                        Collections.singletonMap("_ctx", Collections.singletonMap("_ttl", 3600000)))).execute().actionGet();
+        getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
+        ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue();
+        assertThat(ttl, greaterThan(0L));
+        assertThat(ttl, lessThanOrEqualTo(3600000L));
+
+        // check timestamp update
+        client().prepareIndex("test", "type1", "3").setSource("field", 1).setRefreshPolicy(IMMEDIATE).get();
+        client().prepareUpdate(indexOrAlias(), "type1", "3")
+                .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values",
+                        Collections.singletonMap("_ctx", Collections.singletonMap("_timestamp", "2009-11-15T14:12:12")))).execute()
+                .actionGet();
+        getResponse = client().prepareGet("test", "type1", "3").setFields("_timestamp").execute().actionGet();
+        long timestamp = ((Number) getResponse.getField("_timestamp").getValue()).longValue();
+        assertThat(timestamp, equalTo(1258294332000L));
+    }
+
+    public void testContextVariables() throws Exception {
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias"))
+                        .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id)
+                        .addMapping("type1", XContentFactory.jsonBuilder()
+                                .startObject()
+                                .startObject("type1")
+                                .startObject("_timestamp").field("enabled", true).endObject()
+                                .startObject("_ttl").field("enabled", true).endObject()
+                                .endObject()
+                                .endObject())
+                        .addMapping("subtype1", XContentFactory.jsonBuilder()
+                                .startObject()
+                                .startObject("subtype1")
+                                .startObject("_parent").field("type", "type1").endObject()
+                                .startObject("_timestamp").field("enabled", true).endObject()
+                                .startObject("_ttl").field("enabled", true).endObject()
+                                .endObject()
+                                .endObject())
+        );
+        ensureGreen();
+
+        // Index some documents
+        long timestamp = System.currentTimeMillis();
+        client().prepareIndex()
+                .setIndex("test")
+                .setType("type1")
+                .setId("parentId1")
+                .setTimestamp(String.valueOf(timestamp-1))
+                .setSource("field1", 0, "content", "bar")
+                .execute().actionGet();
+
+        long ttl = 10000;
+        client().prepareIndex()
+                .setIndex("test")
+                .setType("subtype1")
+                .setId("id1")
+                .setParent("parentId1")
+                .setRouting("routing1")
+                .setTimestamp(String.valueOf(timestamp))
+                .setTTL(ttl)
+                .setSource("field1", 1, "content", "foo")
+                .execute().actionGet();
+
+        // Update the first object and note context variables values
+        UpdateResponse updateResponse = client().prepareUpdate("test", "subtype1", "id1")
+                .setRouting("routing1")
+                .setScript(new Script("", ScriptService.ScriptType.INLINE, "extract_ctx", null))
+                .execute().actionGet();
+
+        assertEquals(2, updateResponse.getVersion());
+
+        GetResponse getResponse = client().prepareGet("test", "subtype1", "id1").setRouting("routing1").execute().actionGet();
+        Map<String, Object> updateContext = (Map<String, Object>) getResponse.getSourceAsMap().get("update_context");
+        assertEquals("test", updateContext.get("_index"));
+        assertEquals("subtype1", updateContext.get("_type"));
+        assertEquals("id1", updateContext.get("_id"));
+        assertEquals(1, updateContext.get("_version"));
+        assertEquals("parentId1", updateContext.get("_parent"));
+        assertEquals("routing1", updateContext.get("_routing"));
+        assertThat(((Integer) updateContext.get("_ttl")).longValue(), allOf(greaterThanOrEqualTo(ttl-3000), lessThanOrEqualTo(ttl)));
+
+        // Idem with the second object
+        updateResponse = client().prepareUpdate("test", "type1", "parentId1")
+                .setScript(new Script("", ScriptService.ScriptType.INLINE, "extract_ctx", null))
+                .execute().actionGet();
+
+        assertEquals(2, updateResponse.getVersion());
+
+        getResponse = client().prepareGet("test", "type1", "parentId1").execute().actionGet();
+        updateContext = (Map<String, Object>) getResponse.getSourceAsMap().get("update_context");
+        assertEquals("test", updateContext.get("_index"));
+        assertEquals("type1", updateContext.get("_type"));
+        assertEquals("parentId1", updateContext.get("_id"));
+        assertEquals(1, updateContext.get("_version"));
+        assertNull(updateContext.get("_parent"));
+        assertNull(updateContext.get("_routing"));
+        assertNull(updateContext.get("_ttl"));
+    }
+
+    private static String indexOrAlias() {
+        return randomBoolean() ? "test" : "alias";
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/update/UpdateIT.java b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
index 4121543a7a0..55834c181b0 100644
--- a/core/src/test/java/org/elasticsearch/update/UpdateIT.java
+++ b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
@@ -62,14 +62,12 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 
-import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
@@ -357,21 +355,14 @@ public class UpdateIT extends ESIntegTestCase {
                 FieldIncrementScriptPlugin.class,
                 ScriptedUpsertScriptPlugin.class,
                 ExtractContextInSourceScriptPlugin.class,
-                InternalSettingsPlugin.class // uses index.merge.enabled
+                InternalSettingsPlugin.class
         );
     }
 
     private void createTestIndex() throws Exception {
         logger.info("--> creating index test");
 
-        assertAcked(prepareCreate("test").addAlias(new Alias("alias"))
-                .addMapping("type1", XContentFactory.jsonBuilder()
-                        .startObject()
-                        .startObject("type1")
-                        .startObject("_timestamp").field("enabled", true).endObject()
-                        .startObject("_ttl").field("enabled", true).endObject()
-                        .endObject()
-                        .endObject()));
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias")));
     }
 
     public void testUpsert() throws Exception {
@@ -638,34 +629,6 @@ public class UpdateIT extends ESIntegTestCase {
             assertThat(getResponse.isExists(), equalTo(false));
         }
 
-        // check TTL is kept after an update without TTL
-        client().prepareIndex("test", "type1", "2").setSource("field", 1).setTTL(86400000L).setRefreshPolicy(IMMEDIATE).get();
-        GetResponse getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
-        long ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue();
-        assertThat(ttl, greaterThan(0L));
-        client().prepareUpdate(indexOrAlias(), "type1", "2")
-                .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null)).execute().actionGet();
-        getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
-        ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue();
-        assertThat(ttl, greaterThan(0L));
-
-        // check TTL update
-        client().prepareUpdate(indexOrAlias(), "type1", "2")
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", Collections.singletonMap("_ctx", Collections.singletonMap("_ttl", 3600000)))).execute().actionGet();
-        getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
-        ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue();
-        assertThat(ttl, greaterThan(0L));
-        assertThat(ttl, lessThanOrEqualTo(3600000L));
-
-        // check timestamp update
-        client().prepareIndex("test", "type1", "3").setSource("field", 1).setRefreshPolicy(IMMEDIATE).get();
-        client().prepareUpdate(indexOrAlias(), "type1", "3")
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", Collections.singletonMap("_ctx", Collections.singletonMap("_timestamp", "2009-11-15T14:12:12")))).execute()
-                .actionGet();
-        getResponse = client().prepareGet("test", "type1", "3").setFields("_timestamp").execute().actionGet();
-        long timestamp = ((Number) getResponse.getField("_timestamp").getValue()).longValue();
-        assertThat(timestamp, equalTo(1258294332000L));
-
         // check fields parameter
         client().prepareIndex("test", "type1", "1").setSource("field", 1).execute().actionGet();
         updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1")
@@ -682,7 +645,7 @@ public class UpdateIT extends ESIntegTestCase {
         client().prepareIndex("test", "type1", "1").setSource("field", 1).execute().actionGet();
         updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(XContentFactory.jsonBuilder().startObject().field("field2", 2).endObject()).execute().actionGet();
         for (int i = 0; i < 5; i++) {
-            getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
+            GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
             assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("1"));
             assertThat(getResponse.getSourceAsMap().get("field2").toString(), equalTo("2"));
         }
@@ -690,7 +653,7 @@ public class UpdateIT extends ESIntegTestCase {
         // change existing field
         updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(XContentFactory.jsonBuilder().startObject().field("field", 3).endObject()).execute().actionGet();
         for (int i = 0; i < 5; i++) {
-            getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
+            GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
             assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("3"));
             assertThat(getResponse.getSourceAsMap().get("field2").toString(), equalTo("2"));
         }
@@ -708,7 +671,7 @@ public class UpdateIT extends ESIntegTestCase {
         client().prepareIndex("test", "type1", "1").setSource("map", testMap).execute().actionGet();
         updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(XContentFactory.jsonBuilder().startObject().field("map", testMap3).endObject()).execute().actionGet();
         for (int i = 0; i < 5; i++) {
-            getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
+            GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
             Map map1 = (Map) getResponse.getSourceAsMap().get("map");
             assertThat(map1.size(), equalTo(3));
             assertThat(map1.containsKey("map1"), equalTo(true));
@@ -760,16 +723,12 @@ public class UpdateIT extends ESIntegTestCase {
                         .addMapping("type1", XContentFactory.jsonBuilder()
                                 .startObject()
                                 .startObject("type1")
-                                .startObject("_timestamp").field("enabled", true).endObject()
-                                .startObject("_ttl").field("enabled", true).endObject()
                                 .endObject()
                                 .endObject())
                         .addMapping("subtype1", XContentFactory.jsonBuilder()
                                 .startObject()
                                 .startObject("subtype1")
                                 .startObject("_parent").field("type", "type1").endObject()
-                                .startObject("_timestamp").field("enabled", true).endObject()
-                                .startObject("_ttl").field("enabled", true).endObject()
                                 .endObject()
                                 .endObject())
         );
@@ -813,7 +772,6 @@ public class UpdateIT extends ESIntegTestCase {
         assertEquals(1, updateContext.get("_version"));
         assertEquals("parentId1", updateContext.get("_parent"));
         assertEquals("routing1", updateContext.get("_routing"));
-        assertThat(((Integer) updateContext.get("_ttl")).longValue(), allOf(greaterThanOrEqualTo(ttl-3000), lessThanOrEqualTo(ttl)));
 
         // Idem with the second object
         updateResponse = client().prepareUpdate("test", "type1", "parentId1")
@@ -904,13 +862,6 @@ public class UpdateIT extends ESIntegTestCase {
     public void testStressUpdateDeleteConcurrency() throws Exception {
         //We create an index with merging disabled so that deletes don't get merged away
         assertAcked(prepareCreate("test")
-                .addMapping("type1", XContentFactory.jsonBuilder()
-                        .startObject()
-                        .startObject("type1")
-                        .startObject("_timestamp").field("enabled", true).endObject()
-                        .startObject("_ttl").field("enabled", true).endObject()
-                        .endObject()
-                        .endObject())
                 .setSettings(Settings.builder().put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)));
         ensureGreen();
 
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
index d834f0fcf6d..28894c5c1cc 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
@@ -22,7 +22,6 @@ package org.elasticsearch.messy.tests;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -45,7 +44,6 @@ import org.elasticsearch.test.ESIntegTestCase;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 
-import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Base64;
@@ -88,8 +86,6 @@ public class SearchFieldsTests extends ESIntegTestCase {
         client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForYellowStatus().execute().actionGet();
 
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
-                // _timestamp is randomly enabled via templates but we don't want it here to test stored fields behaviour
-                .startObject("_timestamp").field("enabled", false).endObject()
                 .startObject("properties")
                 .startObject("field1").field("type", "text").field("store", true).endObject()
                 .startObject("field2").field("type", "text").field("store", false).endObject()
@@ -698,7 +694,7 @@ public class SearchFieldsTests extends ESIntegTestCase {
     public void testLoadMetadata() throws Exception {
         assertAcked(prepareCreate("test")
                 .addMapping("parent")
-                .addMapping("my-type1", "_timestamp", "enabled=true", "_ttl", "enabled=true", "_parent", "type=parent"));
+                .addMapping("my-type1", "_parent", "type=parent"));
 
         indexRandom(true,
                 client().prepareIndex("test", "my-type1", "1")
@@ -717,12 +713,6 @@ public class SearchFieldsTests extends ESIntegTestCase {
         assertThat(fields.get("field1"), nullValue());
         assertThat(fields.get("_routing").isMetadataField(), equalTo(true));
         assertThat(fields.get("_routing").getValue().toString(), equalTo("1"));
-        assertThat(fields.get("_timestamp").isMetadataField(), equalTo(true));
-        assertThat(fields.get("_timestamp").getValue().toString(), equalTo("205097"));
-        assertThat(fields.get("_ttl").isMetadataField(), equalTo(true));
-        // TODO: _ttl should return the original value, but it does not work today because
-        // it would use now() instead of the value of _timestamp to rebase
-        // assertThat(fields.get("_ttl").getValue().toString(), equalTo("10000000205097"));
         assertThat(fields.get("_parent").isMetadataField(), equalTo(true));
         assertThat(fields.get("_parent").getValue().toString(), equalTo("parent_1"));
     }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc b/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc
index 304d31978cf..688d8cbdc5b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc
@@ -255,7 +255,7 @@ Supports also regular expressions with flag X for more readability (accepts whit
 Compares two numeric values, eg:
 
 ....
-    - lt: { _ttl: 10000 }  # the `_ttl` value is less than 10,000
+    - lt: { foo: 10000 }  # the `foo` value is less than 10,000
 ....
 
 === `lte` and `gte`
@@ -263,7 +263,7 @@ Compares two numeric values, eg:
 Compares two numeric values, eg:
 
 ....
-    - lte: { _ttl: 10000 }  # the `_ttl` value is less than or equal to 10,000
+    - lte: { foo: 10000 }  # the `foo` value is less than or equal to 10,000
 ....
 
 === `length`
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_timestamp.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_timestamp.yaml
deleted file mode 100644
index 83173817b6e..00000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_timestamp.yaml
+++ /dev/null
@@ -1,80 +0,0 @@
----
-"Timestamp":
-
- - do:
-      indices.create:
-          index:    test_1
-          body:
-              mappings:
-                test:
-                  _timestamp:
-                    enabled:  1
- - do:
-      cluster.health:
-          wait_for_status: yellow
-
-# blank timestamp
- - do:
-      create:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _timestamp
-
- - is_true:    _timestamp
-
-# milliseconds since epoch
-
- - do:
-      delete:
-          index:   test_1
-          type:    test
-          id:      1
- - do:
-      create:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-          timestamp: 1372011280000
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _timestamp
-
- - match: { _timestamp: 1372011280000 }
-
-# date format
-
- - do:
-      delete:
-          index:   test_1
-          type:    test
-          id:      1
- - do:
-      create:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-          timestamp: 2013-06-23T18:14:40
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _timestamp
-
- - match: { _timestamp: 1372011280000 }
-
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/75_ttl.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/75_ttl.yaml
deleted file mode 100644
index 3f9b4763669..00000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/create/75_ttl.yaml
+++ /dev/null
@@ -1,100 +0,0 @@
----
-"TTL":
-
- - do:
-      indices.create:
-          index:    test_1
-          body:
-              mappings:
-                test:
-                  _ttl:
-                    enabled:  1
-                    default:  10s
- - do:
-      cluster.health:
-          wait_for_status: yellow
-
-# blank ttl
- - do:
-      create:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _ttl
-
- - lte:   { _ttl: 10000}
- - gt:    { _ttl: 0}
-
-# milliseconds
-
- - do:
-      delete:
-          index:   test_1
-          type:    test
-          id:      1
- - do:
-      create:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-          ttl:       100000ms
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _ttl
-
- - lte:   { _ttl: 100000}
- - gt:    { _ttl: 10000}
-
-# duration
-
- - do:
-      delete:
-          index:   test_1
-          type:    test
-          id:      1
- - do:
-      create:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-          ttl:       20s
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _ttl
-
- - lte:   { _ttl: 20000}
- - gt:    { _ttl: 10000}
-
-# with timestamp
-
- - do:
-      delete:
-          index:   test_1
-          type:    test
-          id:      1
- - do:
-      catch: /already_expired_exception/
-      create:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-          ttl:       20s
-          timestamp: 2013-06-23T18:14:40
-
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/70_timestamp.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/70_timestamp.yaml
deleted file mode 100644
index 938a358b103..00000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/70_timestamp.yaml
+++ /dev/null
@@ -1,70 +0,0 @@
----
-"Timestamp":
-
- - do:
-      indices.create:
-          index:    test_1
-          body:
-              mappings:
-                test:
-                  _timestamp:
-                    enabled:  1
- - do:
-      cluster.health:
-          wait_for_status: yellow
-
-# blank timestamp
- - do:
-      index:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _timestamp
-
- - is_true: _timestamp
-
-# milliseconds since epoch
-
- - do:
-      index:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-          timestamp: 1372011280000
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _timestamp
-
- - match: { _timestamp: 1372011280000 }
-
-# date format
-
- - do:
-      index:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-          timestamp: 2013-06-23T18:14:40
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _timestamp
-
- - match: { _timestamp: 1372011280000 }
-
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/75_ttl.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/75_ttl.yaml
deleted file mode 100644
index 2ec00eaf2f4..00000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/75_ttl.yaml
+++ /dev/null
@@ -1,85 +0,0 @@
----
-"TTL":
-
- - do:
-      indices.create:
-          index:    test_1
-          body:
-              mappings:
-                test:
-                  _ttl:
-                    enabled:  1
-                    default:  10s
- - do:
-      cluster.health:
-          wait_for_status: yellow
-
-# blank ttl
- - do:
-      index:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _ttl
-
- - lte:   { _ttl: 10000}
- - gt:    { _ttl: 0}
-
-# milliseconds
-
- - do:
-      index:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-          ttl:       100000ms
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _ttl
-
- - lte:   { _ttl: 100000}
- - gt:    { _ttl: 10000}
-
-# duration
-
- - do:
-      index:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-          ttl:       20s
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _ttl
-
- - lte:   { _ttl: 20000}
- - gt:    { _ttl: 10000}
-
-# with timestamp
-
- - do:
-      catch: /already_expired_exception/
-      index:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-          ttl:       20s
-          timestamp: 2013-06-23T18:14:40
-
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/70_timestamp.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/70_timestamp.yaml
deleted file mode 100644
index 864c219c448..00000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/70_timestamp.yaml
+++ /dev/null
@@ -1,76 +0,0 @@
----
-"Timestamp":
-
- - do:
-      indices.create:
-          index:    test_1
-          body:
-              mappings:
-                test:
-                  _timestamp:
-                    enabled:  1
- - do:
-      cluster.health:
-          wait_for_status: yellow
-
-# blank timestamp
- - do:
-      update:
-          index:     test_1
-          type:      test
-          id:        1
-          body:
-            doc:        { foo: baz }
-            upsert:     { foo: bar }
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _timestamp
-
- - is_true: _timestamp
-
-# milliseconds since epoch
-
- - do:
-      update:
-          index:     test_1
-          type:      test
-          id:        1
-          body:
-            doc:        { foo: baz }
-            upsert:     { foo: bar }
-          timestamp: 1372011280000
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _timestamp
-
- - match: { _timestamp: 1372011280000 }
-
-# date format
-
- - do:
-      update:
-          index:     test_1
-          type:      test
-          id:        1
-          body:
-            doc:        { foo: baz }
-            upsert:     { foo: bar }
-          timestamp: 2013-06-23T18:14:40
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _timestamp
-
- - match: { _timestamp: 1372011280000 }
-
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/75_ttl.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/75_ttl.yaml
deleted file mode 100644
index 09179ebd5ac..00000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/75_ttl.yaml
+++ /dev/null
@@ -1,92 +0,0 @@
----
-"TTL":
-
- - do:
-      indices.create:
-          index:    test_1
-          body:
-              mappings:
-                test:
-                  _ttl:
-                    enabled:  1
-                    default:  10s
- - do:
-      cluster.health:
-          wait_for_status: yellow
-
-# blank ttl
- - do:
-      update:
-          index:     test_1
-          type:      test
-          id:        1
-          body:
-            doc:        { foo: baz }
-            upsert:     { foo: bar }
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _ttl
-
- - lte:   { _ttl: 10000}
- - gt:    { _ttl: 0}
-
-# milliseconds
-
- - do:
-      update:
-          index:     test_1
-          type:      test
-          id:        1
-          body:
-            doc:        { foo: baz }
-            upsert:     { foo: bar }
-          ttl:       100000ms
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _ttl
-
- - lte:   { _ttl: 100000}
- - gt:    { _ttl: 10000}
-
-# seconds
-
- - do:
-      update:
-          index:     test_1
-          type:      test
-          id:        1
-          body:
-            doc:        { foo: baz }
-            upsert:     { foo: bar }
-            detect_noop:  false
-          ttl:       20s
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _ttl
-
- - lte:   { _ttl: 20000}
- - gt:    { _ttl: 10000}
-
-# with timestamp
-
- - do:
-      catch: /already_expired_exception/
-      index:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-          ttl:       20s
-          timestamp: 2013-06-23T18:14:40
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml
index 985048ad44d..0194453a99c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml
@@ -12,11 +12,6 @@
             mappings:
               test:
                 _parent: { type: "foo" }
-                _timestamp:
-                  enabled:  1
-                _ttl:
-                  enabled:  1
-                  default:  10s
 
   - do:
       cluster.health:
@@ -28,15 +23,13 @@
           type:   test
           id:     1
           parent: 5
-          fields: [ _parent, _routing, _timestamp, _ttl ]
+          fields: [ _parent, _routing ]
           body:
             doc:    { foo: baz }
             upsert: { foo: bar }
 
   - match:   { get._parent:  "5" }
   - match:   { get._routing: "5" }
-  - is_true:   get._timestamp
-  - is_true:   get._ttl
 
   - do:
       get:
@@ -44,6 +37,6 @@
           type:   test
           id:     1
           parent: 5
-          fields: [ _parent, _routing, _timestamp, _ttl ]
+          fields: [ _parent, _routing ]
 
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index 095b1339800..8df2fd911c4 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -383,13 +383,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
 
             XContentBuilder mappings = null;
             if (frequently() && randomDynamicTemplates()) {
-                mappings = XContentFactory.jsonBuilder().startObject().startObject("_default_");
-                if (randomBoolean()) {
-                    mappings.startObject(TimestampFieldMapper.NAME)
-                            .field("enabled", randomBoolean());
-                    mappings.endObject();
-                }
-                mappings.endObject().endObject();
+                mappings = XContentFactory.jsonBuilder().startObject().startObject("_default_").endObject().endObject();
             }
 
             for (String setting : randomSettingsBuilder.internalMap().keySet()) {

From f78ef232dcfabb06697ad73b70c463336a2bd66f Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 21 Jun 2016 12:05:10 -0400
Subject: [PATCH 1000/1311] fix bogus comment

---
 .../src/main/java/org/elasticsearch/painless/Definition.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index 8aa4da4015f..e57aad862aa 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -220,7 +220,7 @@ public final class Definition {
             final Class<?> params[];
             final Class<?> returnValue;
             if (augmentation) {
-                // virtual/interface method disguised as static
+                // static method disguised as virtual/interface method
                 params = new Class<?>[1 + arguments.size()];
                 params[0] = Augmentation.class;
                 for (int i = 0; i < arguments.size(); i++) {

From 1b9695a9aa854ecdddbd73af4d4133637c96022d Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Tue, 21 Jun 2016 12:15:59 -0400
Subject: [PATCH 1001/1311] beef up tests so we ensure you still get good
 errors in these cases

---
 .../elasticsearch/painless/FunctionRefTests.java | 14 ++++++++++++++
 .../org/elasticsearch/painless/LambdaTests.java  | 16 ++++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
index 46d2b7c43fd..7136807bd7e 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
@@ -170,10 +170,24 @@ public class FunctionRefTests extends ScriptTestCase {
         assertTrue(expected.getMessage().contains("Unknown reference"));
     }
     
+    public void testWrongArityNotEnough() {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("List l = new ArrayList(); l.add(2); l.add(1); l.sort(String::isEmpty);");
+        });
+        assertTrue(expected.getMessage().contains("Unknown reference"));
+    }
+    
     public void testWrongArityDef() {
         IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("def y = Optional.empty(); return y.orElseGet(String::startsWith);");
         });
         assertTrue(expected.getMessage().contains("Unknown reference"));
     }
+    
+    public void testWrongArityNotEnoughDef() {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("def l = new ArrayList(); l.add(2); l.add(1); l.sort(String::isEmpty);");
+        });
+        assertTrue(expected.getMessage().contains("Unknown reference"));
+    }
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
index 7b2d5e6a935..dbca5243ec2 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
@@ -180,6 +180,22 @@ public class LambdaTests extends ScriptTestCase {
         assertTrue(expected.getMessage(), expected.getMessage().contains("Incorrect number of parameters"));
     }
     
+    public void testWrongArityNotEnough() {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("List l = new ArrayList(); l.add(1); l.add(1); "
+               + "return l.stream().mapToInt(() -> 5).sum();");
+        });
+        assertTrue(expected.getMessage().contains("Incorrect number of parameters"));
+    }
+    
+    public void testWrongArityNotEnoughDef() {
+        IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> {
+            exec("def l = new ArrayList(); l.add(1); l.add(1); "
+               + "return l.stream().mapToInt(() -> 5).sum();");
+        });
+        assertTrue(expected.getMessage().contains("Incorrect number of parameters"));
+    }
+    
     public void testLambdaInFunction() {
         assertEquals(5, exec("def foo() { Optional.empty().orElseGet(() -> 5) } return foo();"));
     }

From b32d9a71e4ce65b8898bb10dfd622a1686e5fe0d Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Tue, 21 Jun 2016 18:23:40 +0200
Subject: [PATCH 1002/1311] inner_hits: Also never serialize `_index` key for
 parent/child inner hits as the _index is always the same of the parent search
 hit

---
 .../search/fetch/innerhits/InnerHitsFetchSubPhase.java          | 1 -
 .../rest-api-spec/test/search.inner_hits/10_basic.yaml          | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java
index 35a25b522c0..d9ebc77ec07 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsFetchSubPhase.java
@@ -68,7 +68,6 @@ public final class InnerHitsFetchSubPhase implements FetchSubPhase {
             for (int i = 0; i < internalHits.length; i++) {
                 ScoreDoc scoreDoc = topDocs.scoreDocs[i];
                 InternalSearchHit searchHitFields = internalHits[i];
-                searchHitFields.shard(innerHits.shardTarget());
                 searchHitFields.score(scoreDoc.score);
                 if (scoreDoc instanceof FieldDoc) {
                     FieldDoc fieldDoc = (FieldDoc) scoreDoc;
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yaml
index 66b36b58bb7..98e61dd9fa9 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yaml
@@ -78,7 +78,7 @@ setup:
     - match: { hits.hits.0._index: "test" }
     - match: { hits.hits.0._type: "type_2" }
     - match: { hits.hits.0._id: "1" }
-    - match: { hits.hits.0.inner_hits.type_3.hits.hits.0._index: "test" }
+    - is_false: hits.hits.0.inner_hits.type_3.hits.hits.0._index
     - match: { hits.hits.0.inner_hits.type_3.hits.hits.0._type: "type_3" }
     - match: { hits.hits.0.inner_hits.type_3.hits.hits.0._id: "1" }
     - is_false: hits.hits.0.inner_hits.type_3.hits.hits.0._nested

From 8078c205f9ba4443e8d5221df4347f13262e510a Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Tue, 21 Jun 2016 19:19:49 +0200
Subject: [PATCH 1003/1311] Revert "Remove `_timestamp` and `_ttl` on 5.x
 indices. #18980"

This reverts commit 969e953645a4b1a28aaa834ca0c4826ba5ea19a2.
Docs are failing because of the removed functionality. I will
fix the docs before pushing it again.
---
 .../index/mapper/internal/TTLFieldMapper.java |   7 -
 .../mapper/internal/TimestampFieldMapper.java |   7 -
 .../elasticsearch/aliases/IndexAliasesIT.java |   8 +-
 .../cluster/SpecificMasterNodesIT.java        |  11 +-
 .../org/elasticsearch/get/GetActionIT.java    |  39 ++-
 .../mapper/all/SimpleAllMapperTests.java      |  20 ++
 .../timestamp/TimestampMappingTests.java      | 104 ++------
 .../index/mapper/ttl/TTLMappingTests.java     |  54 +---
 .../ParseMappingTypeLevelTests.java           |   4 +-
 .../update/UpdateMappingOnClusterIT.java      |  10 +
 .../mapper/update/UpdateMappingTests.java     |  50 ++++
 .../aggregations/bucket/DateHistogramIT.java  |  11 +-
 .../search/sort/FieldSortIT.java              |  42 +++-
 .../timestamp/SimpleTimestampIT.java          |  19 +-
 .../org/elasticsearch/ttl/SimpleTTLIT.java    |  15 +-
 .../update/TimestampTTLBWIT.java              | 237 ------------------
 .../org/elasticsearch/update/UpdateIT.java    |  59 ++++-
 .../messy/tests/SearchFieldsTests.java        |  12 +-
 .../rest-api-spec/test/README.asciidoc        |   4 +-
 .../test/create/70_timestamp.yaml             |  80 ++++++
 .../rest-api-spec/test/create/75_ttl.yaml     | 100 ++++++++
 .../test/index/70_timestamp.yaml              |  70 ++++++
 .../rest-api-spec/test/index/75_ttl.yaml      |  85 +++++++
 .../test/update/70_timestamp.yaml             |  76 ++++++
 .../rest-api-spec/test/update/75_ttl.yaml     |  92 +++++++
 .../test/update/85_fields_meta.yaml           |  11 +-
 .../elasticsearch/test/ESIntegTestCase.java   |   8 +-
 27 files changed, 795 insertions(+), 440 deletions(-)
 delete mode 100644 core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/create/70_timestamp.yaml
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/create/75_ttl.yaml
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/index/70_timestamp.yaml
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/index/75_ttl.yaml
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/update/70_timestamp.yaml
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/update/75_ttl.yaml

diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java
index ec1024c07f5..33d194a7892 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java
@@ -21,7 +21,6 @@ package org.elasticsearch.index.mapper.internal;
 
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
-import org.elasticsearch.Version;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -101,9 +100,6 @@ public class TTLFieldMapper extends MetadataFieldMapper {
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
         public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
-            if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0_alpha4)) {
-                throw new IllegalArgumentException("[_ttl] is removed in 5.0. As a replacement, you should use time based indexes or cron a delete-by-query with a range query on a timestamp field.");
-            }
             Builder builder = new Builder();
             for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
                 Map.Entry<String, Object> entry = iterator.next();
@@ -169,9 +165,6 @@ public class TTLFieldMapper extends MetadataFieldMapper {
     private TTLFieldMapper(MappedFieldType fieldType, EnabledAttributeMapper enabled, long defaultTTL,
                              Settings indexSettings) {
         super(NAME, fieldType, Defaults.TTL_FIELD_TYPE, indexSettings);
-        if (enabled.enabled && Version.indexCreated(indexSettings).onOrAfter(Version.V_5_0_0_alpha4)) {
-            throw new IllegalArgumentException("[_ttl] is removed in 5.0. As a replacement, you should use time based indexes or cron a delete-by-query with a range query on a timestamp field.");
-        }
         this.enabledState = enabled;
         this.defaultTTL = defaultTTL;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
index 24a86b11392..09f1144db28 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
@@ -22,7 +22,6 @@ package org.elasticsearch.index.mapper.internal;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.index.IndexOptions;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.TimestampParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
@@ -127,9 +126,6 @@ public class TimestampFieldMapper extends MetadataFieldMapper {
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
         public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
-            if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0_alpha4)) {
-                throw new IllegalArgumentException("[_timestamp] is removed in 5.0. As a replacement, you can use an ingest pipeline to add a field with the current timestamp to your documents.");
-            }
             Builder builder = new Builder(parserContext.mapperService().fullName(NAME), parserContext.mapperService().getIndexSettings().getSettings());
             boolean defaultSet = false;
             Boolean ignoreMissing = null;
@@ -204,9 +200,6 @@ public class TimestampFieldMapper extends MetadataFieldMapper {
     private TimestampFieldMapper(MappedFieldType fieldType, MappedFieldType defaultFieldType, EnabledAttributeMapper enabledState,
                                    String defaultTimestamp, Boolean ignoreMissing, Settings indexSettings) {
         super(NAME, fieldType, defaultFieldType, indexSettings);
-        if (enabledState.enabled && Version.indexCreated(indexSettings).onOrAfter(Version.V_5_0_0_alpha4)) {
-            throw new IllegalArgumentException("[_timestamp] is removed in 5.0. As a replacement, you can use an ingest pipeline to add a field with the current timestamp to your documents.");
-        }
         this.enabledState = enabledState;
         this.defaultTimestamp = defaultTimestamp;
         this.ignoreMissing = ignoreMissing;
diff --git a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
index 5ac1bf40af6..2a16625d037 100644
--- a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
+++ b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
@@ -1035,13 +1035,13 @@ public class IndexAliasesIT extends ESIntegTestCase {
     }
 
     public void testAliasFilterWithNowInRangeFilterAndQuery() throws Exception {
-        assertAcked(prepareCreate("my-index").addMapping("my-type", "timestamp", "type=date"));
-        assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter1", rangeQuery("timestamp").from("2016-12-01").to("2016-12-31")));
-        assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter2", rangeQuery("timestamp").from("2016-01-01").to("2016-12-31")));
+        assertAcked(prepareCreate("my-index").addMapping("my-type", "_timestamp", "enabled=true"));
+        assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter1", rangeQuery("_timestamp").from("now-1d").to("now")));
+        assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter2", rangeQuery("_timestamp").from("now-1d").to("now")));
 
         final int numDocs = scaledRandomIntBetween(5, 52);
         for (int i = 1; i <= numDocs; i++) {
-            client().prepareIndex("my-index", "my-type").setCreate(true).setSource("timestamp", "2016-12-12").get();
+            client().prepareIndex("my-index", "my-type").setCreate(true).setSource("{}").get();
             if (i % 2 == 0) {
                 refresh();
                 SearchResponse response = client().prepareSearch("filter1").get();
diff --git a/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java b/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
index 4773aafbf3f..72db338ab80 100644
--- a/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
@@ -30,7 +30,6 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 
 import java.io.IOException;
-import java.util.Map;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
@@ -112,18 +111,16 @@ public class SpecificMasterNodesIT extends ESIntegTestCase {
         internalCluster().startNode(settingsBuilder().put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false));
 
         createIndex("test");
-        assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("timestamp", "type=date"));
+        assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("_timestamp", "enabled=true"));
 
         MappingMetaData defaultMapping = client().admin().cluster().prepareState().get().getState().getMetaData().getIndices().get("test").getMappings().get("_default_");
-        Map<?,?> properties = (Map<?, ?>) defaultMapping.getSourceAsMap().get("properties");
-        assertThat(properties.get("timestamp"), notNullValue());
+        assertThat(defaultMapping.getSourceAsMap().get("_timestamp"), notNullValue());
 
-        assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("timestamp", "type=date"));
+        assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("_timestamp", "enabled=true"));
 
         assertAcked(client().admin().indices().preparePutMapping("test").setType("type1").setSource("foo", "enabled=true"));
         MappingMetaData type1Mapping = client().admin().cluster().prepareState().get().getState().getMetaData().getIndices().get("test").getMappings().get("type1");
-        properties = (Map<?, ?>) type1Mapping.getSourceAsMap().get("properties");
-        assertThat(properties.get("timestamp"), notNullValue());
+        assertThat(type1Mapping.getSourceAsMap().get("_timestamp"), notNullValue());
     }
 
     public void testAliasFilterValidation() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/get/GetActionIT.java b/core/src/test/java/org/elasticsearch/get/GetActionIT.java
index b73f563fbb3..8c4b699cbff 100644
--- a/core/src/test/java/org/elasticsearch/get/GetActionIT.java
+++ b/core/src/test/java/org/elasticsearch/get/GetActionIT.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.get;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.flush.FlushResponse;
@@ -29,6 +30,7 @@ import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.get.MultiGetRequest;
 import org.elasticsearch.action.get.MultiGetRequestBuilder;
 import org.elasticsearch.action.get.MultiGetResponse;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -38,11 +40,15 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
 
 import static java.util.Collections.singleton;
@@ -51,6 +57,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
@@ -529,7 +536,7 @@ public class GetActionIT extends ESIntegTestCase {
     public void testGetFieldsMetaData() throws Exception {
         assertAcked(prepareCreate("test")
                 .addMapping("parent")
-                .addMapping("my-type1", "_parent", "type=parent")
+                .addMapping("my-type1", "_timestamp", "enabled=true", "_ttl", "enabled=true", "_parent", "type=parent")
                 .addAlias(new Alias("alias"))
                 .setSettings(Settings.builder().put("index.refresh_interval", -1)));
 
@@ -550,6 +557,12 @@ public class GetActionIT extends ESIntegTestCase {
         assertThat(getResponse.getField("field1").getValue().toString(), equalTo("value"));
         assertThat(getResponse.getField("_routing").isMetadataField(), equalTo(true));
         assertThat(getResponse.getField("_routing").getValue().toString(), equalTo("1"));
+        assertThat(getResponse.getField("_timestamp").isMetadataField(), equalTo(true));
+        assertThat(getResponse.getField("_timestamp").getValue().toString(), equalTo("205097"));
+        assertThat(getResponse.getField("_ttl").isMetadataField(), equalTo(true));
+        // TODO: _ttl should return the original value, but it does not work today because
+        // it would use now() instead of the value of _timestamp to rebase
+        // assertThat(getResponse.getField("_ttl").getValue().toString(), equalTo("10000000205097"));
         assertThat(getResponse.getField("_parent").isMetadataField(), equalTo(true));
         assertThat(getResponse.getField("_parent").getValue().toString(), equalTo("parent_1"));
 
@@ -564,6 +577,12 @@ public class GetActionIT extends ESIntegTestCase {
         assertThat(getResponse.getField("field1").getValue().toString(), equalTo("value"));
         assertThat(getResponse.getField("_routing").isMetadataField(), equalTo(true));
         assertThat(getResponse.getField("_routing").getValue().toString(), equalTo("1"));
+        assertThat(getResponse.getField("_timestamp").isMetadataField(), equalTo(true));
+        assertThat(getResponse.getField("_timestamp").getValue().toString(), equalTo("205097"));
+        assertThat(getResponse.getField("_ttl").isMetadataField(), equalTo(true));
+        // TODO: _ttl should return the original value, but it does not work today because
+        // it would use now() instead of the value of _timestamp to rebase
+        //assertThat(getResponse.getField("_ttl").getValue().toString(), equalTo("10000000000000"));
         assertThat(getResponse.getField("_parent").isMetadataField(), equalTo(true));
         assertThat(getResponse.getField("_parent").getValue().toString(), equalTo("parent_1"));
     }
@@ -760,10 +779,16 @@ public class GetActionIT extends ESIntegTestCase {
                 "  },\n" +
                 "  \"mappings\": {\n" +
                 "    \"parentdoc\": {\n" +
+                "      \"_ttl\": {\n" +
+                "        \"enabled\": true\n" +
+                "      }\n" +
                 "    },\n" +
                 "    \"doc\": {\n" +
                 "      \"_parent\": {\n" +
                 "        \"type\": \"parentdoc\"\n" +
+                "      },\n" +
+                "      \"_ttl\": {\n" +
+                "        \"enabled\": true\n" +
                 "      }\n" +
                 "    }\n" +
                 "  }\n" +
@@ -773,7 +798,7 @@ public class GetActionIT extends ESIntegTestCase {
 
         client().prepareIndex("test", "doc").setId("1").setSource("{}").setParent("1").setTTL(TimeValue.timeValueHours(1).getMillis()).get();
 
-        String[] fieldsList = {"_parent"};
+        String[] fieldsList = {"_ttl", "_parent"};
         // before refresh - document is only in translog
         assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "1");
         refresh();
@@ -789,6 +814,14 @@ public class GetActionIT extends ESIntegTestCase {
             "  \"settings\": {\n" +
             "    \"index.translog.flush_threshold_size\": \"1pb\",\n" +
             "    \"refresh_interval\": \"-1\"\n" +
+            "  },\n" +
+            "  \"mappings\": {\n" +
+            "    \"parentdoc\": {},\n" +
+            "    \"doc\": {\n" +
+            "      \"_timestamp\": {\n" +
+            "        \"enabled\": true\n" +
+            "      }\n" +
+            "    }\n" +
             "  }\n" +
             "}";
 
@@ -798,7 +831,7 @@ public class GetActionIT extends ESIntegTestCase {
             "  \"text\": \"some text.\"\n" +
             "}\n";
         client().prepareIndex("test", "doc").setId("1").setSource(doc).setRouting("1").get();
-        String[] fieldsList = {"_routing"};
+        String[] fieldsList = {"_timestamp", "_routing"};
         // before refresh - document is only in translog
         assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "1");
         refresh();
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java
index 165b49d3145..966146655b8 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java
@@ -40,6 +40,7 @@ import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext.Document;
 import org.elasticsearch.index.mapper.internal.AllFieldMapper;
+import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -49,7 +50,9 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath;
@@ -427,6 +430,23 @@ public class SimpleAllMapperTests extends ESSingleNodeTestCase {
         parser.parse("test", new CompressedXContent(mapping));
     }
 
+    // issue https://github.com/elastic/elasticsearch/issues/5864
+    public void testMetadataMappersStillWorking() throws MapperParsingException, IOException {
+        String mapping = "{";
+        Map<String, String> rootTypes = new HashMap<>();
+        //just pick some example from DocumentMapperParser.rootTypeParsers
+        rootTypes.put(TimestampFieldMapper.NAME, "{\"enabled\" : true}");
+        rootTypes.put("include_in_all", "true");
+        rootTypes.put("dynamic_date_formats", "[\"yyyy-MM-dd\", \"dd-MM-yyyy\"]");
+        rootTypes.put("numeric_detection", "true");
+        rootTypes.put("dynamic_templates", "[]");
+        for (String key : rootTypes.keySet()) {
+            mapping += "\"" + key+ "\"" + ":" + rootTypes.get(key) + ",\n";
+        }
+        mapping += "\"properties\":{}}" ;
+        createIndex("test").mapperService().documentMapperParser().parse("test", new CompressedXContent(mapping));
+    }
+
     public void testDocValuesNotAllowed() throws IOException {
         String mapping = jsonBuilder().startObject().startObject("type")
             .startObject("_all")
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
index 78da5abb746..14d86290db4 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
@@ -22,7 +22,6 @@ package org.elasticsearch.index.mapper.timestamp;
 import org.apache.lucene.index.IndexOptions;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.TimestampParsingException;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
@@ -32,31 +31,25 @@ import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.joda.Joda;
-import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceToParse;
-import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
-import org.elasticsearch.test.VersionUtils;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.LinkedHashMap;
 
-import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
 import static org.elasticsearch.test.VersionUtils.randomVersion;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.containsString;
@@ -65,35 +58,19 @@ import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.startsWith;
 
 /**
  */
 public class TimestampMappingTests extends ESSingleNodeTestCase {
 
-    private static final Settings BW_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0).build();
-
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(InternalSettingsPlugin.class);
     }
 
-    public void testRejectedOn5x() throws IOException {
-        String mapping = XContentFactory.jsonBuilder().startObject()
-                .startObject("type")
-                    .startObject("_timestamp")
-                        .field("enabled", true)
-                    .endObject()
-                .endObject().endObject().string();
-        IndexService index = createIndex("test");
-        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class,
-                () -> index.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, false));
-        assertThat(expected.getMessage(), startsWith("[_timestamp] is removed"));
-    }
-
     public void testSimpleDisabled() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         BytesReference source = XContentFactory.jsonBuilder()
                 .startObject()
                 .field("field", "value")
@@ -108,7 +85,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_timestamp").field("enabled", "yes").endObject()
                 .endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         BytesReference source = XContentFactory.jsonBuilder()
                 .startObject()
                 .field("field", "value")
@@ -122,7 +99,10 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
     }
 
     public void testDefaultValues() throws Exception {
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0_beta1, Version.V_5_0_0_alpha3);
+        Version version;
+        do {
+            version = randomVersion(random());
+        } while (version.before(Version.V_2_0_0_beta1));
         for (String mapping : Arrays.asList(
                 XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(),
                 XContentFactory.jsonBuilder().startObject().startObject("type").startObject("_timestamp").endObject().endObject().endObject().string())) {
@@ -140,7 +120,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
         String enabledMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_timestamp").field("enabled", true).endObject()
                 .endObject().endObject().string();
-        MapperService mapperService = createIndex("test", BW_SETTINGS).mapperService();
+        MapperService mapperService = createIndex("test").mapperService();
         DocumentMapper enabledMapper = mapperService.merge("type", new CompressedXContent(enabledMapping), MapperService.MergeReason.MAPPING_UPDATE, false);
 
         String disabledMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
@@ -166,7 +146,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                     .field("foo", "bar")
                 .endObject();
 
-        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
         MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData();
 
         MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
@@ -192,7 +172,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                 .endObject();
 
         MetaData metaData = MetaData.builder().build();
-        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
 
         MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
 
@@ -215,7 +195,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                 .endObject()
                 .endObject().endObject();
         try {
-            createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
+            createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
             fail("we should reject the mapping with a TimestampParsingException: default timestamp can not be set to null");
         } catch (TimestampParsingException e) {
             assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set to null"));
@@ -232,7 +212,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                 .endObject().endObject();
 
         try {
-            createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
+            createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
             fail("we should reject the mapping with a TimestampParsingException: default timestamp can not be set to null");
         } catch (TimestampParsingException e) {
             assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set to null"));
@@ -250,7 +230,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                 .endObject().endObject();
 
         try {
-            createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
+            createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
             fail("we should reject the mapping with a TimestampParsingException: default timestamp can not be set with ignore_missing set to false");
         } catch (TimestampParsingException e) {
             assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set with ignore_missing set to false"));
@@ -270,7 +250,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                 .endObject();
 
         MetaData metaData = MetaData.builder().build();
-        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
 
         MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
 
@@ -343,7 +323,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                     .field("enabled", true)
                     .field("default", "1970-01-01")
                 .endObject().endObject().endObject().string();
-        DocumentMapperParser parser = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser();
+        DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
 
         DocumentMapper docMapper = parser.parse("type", new CompressedXContent(mapping));
         docMapper = parser.parse("type", docMapper.mappingSource());
@@ -380,7 +360,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("_timestamp").field("enabled", true).field("default", "1970").field("format", "YYYY").endObject()
             .endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
         try {
             docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
@@ -395,7 +375,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_timestamp").field("enabled", true).field("format", "yyyyMMddHH").endObject()
             .endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData();
 
         XContentBuilder doc = XContentFactory.jsonBuilder().startObject().endObject();
@@ -412,7 +392,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                 .endObject().endObject().string();
         BytesReference source = XContentFactory.jsonBuilder().startObject().field("field", "value").endObject().bytes();
         // test with 2.x
-        DocumentMapper currentMapper = createIndex("new-index", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper currentMapper = createIndex("new-index").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
         // this works with 2.x
         IndexRequest request = new IndexRequest("new-index", "type", "1").source(source).timestamp("1970-01-01");
@@ -427,54 +407,4 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
             assertThat(e.getMessage(), containsString("failed to parse timestamp [1234567890]"));
         }
     }
-
-    public void testSizeTimestampIndexParsing() throws IOException {
-        IndexService indexService = createIndex("test", BW_SETTINGS);
-        String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/default_mapping_with_disabled_root_types.json");
-        DocumentMapper documentMapper = indexService.mapperService().parse("type", new CompressedXContent(mapping), true);
-        assertThat(documentMapper.mappingSource().string(), equalTo(mapping));
-        documentMapper = indexService.mapperService().parse("type", new CompressedXContent(documentMapper.mappingSource().string()), true);
-        assertThat(documentMapper.mappingSource().string(), equalTo(mapping));
-    }
-
-    public void testDefaultApplied() throws IOException {
-        createIndex("test1", BW_SETTINGS);
-        createIndex("test2", BW_SETTINGS);
-        XContentBuilder defaultMapping = XContentFactory.jsonBuilder().startObject()
-                .startObject(MapperService.DEFAULT_MAPPING).startObject("_timestamp").field("enabled", true).endObject().endObject()
-                .endObject();
-        client().admin().indices().preparePutMapping().setType(MapperService.DEFAULT_MAPPING).setSource(defaultMapping).get();
-        XContentBuilder typeMapping = XContentFactory.jsonBuilder().startObject()
-                .startObject("type").startObject("_all").field("enabled", false).endObject().endObject()
-                .endObject();
-        client().admin().indices().preparePutMapping("test1").setType("type").setSource(typeMapping).get();
-        client().admin().indices().preparePutMapping("test1", "test2").setType("type").setSource(typeMapping).get();
-
-        GetMappingsResponse response = client().admin().indices().prepareGetMappings("test2").get();
-        assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_all"));
-        assertFalse((Boolean) ((LinkedHashMap) response.getMappings().get("test2").get("type").getSourceAsMap().get("_all")).get("enabled"));
-        assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_timestamp"));
-        assertTrue((Boolean)((LinkedHashMap)response.getMappings().get("test2").get("type").getSourceAsMap().get("_timestamp")).get("enabled"));
-    }
-
-    public void testTimestampParsing() throws IOException {
-        IndexService indexService = createIndex("test", BW_SETTINGS);
-        XContentBuilder indexMapping = XContentFactory.jsonBuilder();
-        boolean enabled = randomBoolean();
-        indexMapping.startObject()
-                .startObject("type")
-                .startObject("_timestamp")
-                .field("enabled", enabled)
-                .endObject()
-                .endObject()
-                .endObject();
-        DocumentMapper documentMapper = indexService.mapperService().parse("type", new CompressedXContent(indexMapping.string()), true);
-        assertThat(documentMapper.timestampFieldMapper().enabled(), equalTo(enabled));
-        assertTrue(documentMapper.timestampFieldMapper().fieldType().stored());
-        assertTrue(documentMapper.timestampFieldMapper().fieldType().hasDocValues());
-        documentMapper = indexService.mapperService().parse("type", new CompressedXContent(documentMapper.mappingSource().string()), true);
-        assertThat(documentMapper.timestampFieldMapper().enabled(), equalTo(enabled));
-        assertTrue(documentMapper.timestampFieldMapper().fieldType().hasDocValues());
-        assertTrue(documentMapper.timestampFieldMapper().fieldType().stored());
-    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java
index ad9e03327b2..fe057045882 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java
@@ -20,9 +20,7 @@
 package org.elasticsearch.index.mapper.ttl;
 
 import org.apache.lucene.index.IndexOptions;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
@@ -32,48 +30,22 @@ import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.mapper.internal.TTLFieldMapper;
-import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
-import org.elasticsearch.test.InternalSettingsPlugin;
 
 import java.io.IOException;
-import java.util.Collection;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
-import static org.hamcrest.Matchers.startsWith;
 
 public class TTLMappingTests extends ESSingleNodeTestCase {
-
-    private static final Settings BW_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0).build();
-
-    @Override
-    protected Collection<Class<? extends Plugin>> getPlugins() {
-        return pluginList(InternalSettingsPlugin.class);
-    }
-
-    public void testRejectedOn5x() throws IOException {
-        String mapping = XContentFactory.jsonBuilder().startObject()
-                .startObject("type")
-                    .startObject("_ttl")
-                        .field("enabled", true)
-                    .endObject()
-                .endObject().endObject().string();
-        IndexService index = createIndex("test");
-        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class,
-                () -> index.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, false));
-        assertThat(expected.getMessage(), startsWith("[_ttl] is removed"));
-    }
-
     public void testSimpleDisabled() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         BytesReference source = XContentFactory.jsonBuilder()
                 .startObject()
                 .field("field", "value")
@@ -88,7 +60,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_ttl").field("enabled", "yes").endObject()
                 .endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         BytesReference source = XContentFactory.jsonBuilder()
                 .startObject()
                 .field("field", "value")
@@ -103,7 +75,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
 
     public void testDefaultValues() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         assertThat(docMapper.TTLFieldMapper().enabled(), equalTo(TTLFieldMapper.Defaults.ENABLED_STATE.enabled));
         assertThat(docMapper.TTLFieldMapper().fieldType().stored(), equalTo(TTLFieldMapper.Defaults.TTL_FIELD_TYPE.stored()));
         assertThat(docMapper.TTLFieldMapper().fieldType().indexOptions(), equalTo(TTLFieldMapper.Defaults.TTL_FIELD_TYPE.indexOptions()));
@@ -121,7 +93,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
                 .startObject("properties").field("field").startObject().field("type", "text").endObject().endObject()
                 .endObject().endObject().string();
 
-        MapperService mapperService = createIndex("test", BW_SETTINGS).mapperService();
+        MapperService mapperService = createIndex("test").mapperService();
         DocumentMapper mapperWithoutTtl = mapperService.merge("type", new CompressedXContent(mappingWithoutTtl), MapperService.MergeReason.MAPPING_UPDATE, false);
         DocumentMapper mapperWithTtl = mapperService.merge("type", new CompressedXContent(mappingWithTtl), MapperService.MergeReason.MAPPING_UPDATE, false);
 
@@ -144,7 +116,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
                 .startObject("properties").field("field").startObject().field("type", "text").endObject().endObject()
                 .endObject().endObject().string();
 
-        MapperService mapperService = createIndex("test", BW_SETTINGS).mapperService();
+        MapperService mapperService = createIndex("test").mapperService();
         DocumentMapper initialMapper = mapperService.merge("type", new CompressedXContent(mappingWithTtl), MapperService.MergeReason.MAPPING_UPDATE, false);
         DocumentMapper updatedMapper = mapperService.merge("type", new CompressedXContent(updatedMapping), MapperService.MergeReason.MAPPING_UPDATE, false);
 
@@ -155,7 +127,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
     public void testThatDisablingTTLReportsConflict() throws Exception {
         String mappingWithTtl = getMappingWithTtlEnabled().string();
         String mappingWithTtlDisabled = getMappingWithTtlDisabled().string();
-        MapperService mapperService = createIndex("test", BW_SETTINGS).mapperService();
+        MapperService mapperService = createIndex("test").mapperService();
         DocumentMapper initialMapper = mapperService.merge("type", new CompressedXContent(mappingWithTtl), MapperService.MergeReason.MAPPING_UPDATE, false);
 
         try {
@@ -171,7 +143,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
     public void testThatDisablingTTLReportsConflictOnCluster() throws Exception {
         String mappingWithTtl = getMappingWithTtlEnabled().string();
         String mappingWithTtlDisabled = getMappingWithTtlDisabled().string();
-        assertAcked(client().admin().indices().prepareCreate("testindex").setSettings(BW_SETTINGS).addMapping("type", mappingWithTtl));
+        assertAcked(client().admin().indices().prepareCreate("testindex").addMapping("type", mappingWithTtl));
         GetMappingsResponse mappingsBeforeUpdateResponse = client().admin().indices().prepareGetMappings("testindex").addTypes("type").get();
         try {
             client().admin().indices().preparePutMapping("testindex").setSource(mappingWithTtlDisabled).setType("type").get();
@@ -186,7 +158,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
     public void testThatEnablingTTLAfterFirstDisablingWorks() throws Exception {
         String mappingWithTtl = getMappingWithTtlEnabled().string();
         String withTtlDisabled = getMappingWithTtlDisabled().string();
-        assertAcked(client().admin().indices().prepareCreate("testindex").setSettings(BW_SETTINGS).addMapping("type", withTtlDisabled));
+        assertAcked(client().admin().indices().prepareCreate("testindex").addMapping("type", withTtlDisabled));
         GetMappingsResponse mappingsAfterUpdateResponse = client().admin().indices().prepareGetMappings("testindex").addTypes("type").get();
         assertThat(mappingsAfterUpdateResponse.getMappings().get("testindex").get("type").sourceAsMap().get("_ttl").toString(), equalTo("{enabled=false}"));
         client().admin().indices().preparePutMapping("testindex").setSource(mappingWithTtl).setType("type").get();
@@ -195,20 +167,20 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
     }
 
     public void testNoConflictIfNothingSetAndDisabledLater() throws Exception {
-        IndexService indexService = createIndex("testindex", BW_SETTINGS, "type");
+        IndexService indexService = createIndex("testindex", Settings.builder().build(), "type");
         XContentBuilder mappingWithTtlDisabled = getMappingWithTtlDisabled("7d");
         indexService.mapperService().merge("type", new CompressedXContent(mappingWithTtlDisabled.string()), MapperService.MergeReason.MAPPING_UPDATE, false);
     }
 
     public void testNoConflictIfNothingSetAndEnabledLater() throws Exception {
-        IndexService indexService = createIndex("testindex", BW_SETTINGS, "type");
+        IndexService indexService = createIndex("testindex", Settings.builder().build(), "type");
         XContentBuilder mappingWithTtlEnabled = getMappingWithTtlEnabled("7d");
         indexService.mapperService().merge("type", new CompressedXContent(mappingWithTtlEnabled.string()), MapperService.MergeReason.MAPPING_UPDATE, false);
     }
 
     public void testMergeWithOnlyDefaultSet() throws Exception {
         XContentBuilder mappingWithTtlEnabled = getMappingWithTtlEnabled("7d");
-        IndexService indexService = createIndex("testindex", BW_SETTINGS, "type", mappingWithTtlEnabled);
+        IndexService indexService = createIndex("testindex", Settings.builder().build(), "type", mappingWithTtlEnabled);
         XContentBuilder mappingWithOnlyDefaultSet = getMappingWithOnlyTtlDefaultSet("6m");
         indexService.mapperService().merge("type", new CompressedXContent(mappingWithOnlyDefaultSet.string()), MapperService.MergeReason.MAPPING_UPDATE, false);
         CompressedXContent mappingAfterMerge = indexService.mapperService().documentMapper("type").mappingSource();
@@ -217,7 +189,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
 
     public void testMergeWithOnlyDefaultSetTtlDisabled() throws Exception {
         XContentBuilder mappingWithTtlEnabled = getMappingWithTtlDisabled("7d");
-        IndexService indexService = createIndex("testindex", BW_SETTINGS, "type", mappingWithTtlEnabled);
+        IndexService indexService = createIndex("testindex", Settings.builder().build(), "type", mappingWithTtlEnabled);
         CompressedXContent mappingAfterCreation = indexService.mapperService().documentMapper("type").mappingSource();
         assertThat(mappingAfterCreation, equalTo(new CompressedXContent("{\"type\":{\"_ttl\":{\"enabled\":false},\"properties\":{\"field\":{\"type\":\"text\"}}}}")));
         XContentBuilder mappingWithOnlyDefaultSet = getMappingWithOnlyTtlDefaultSet("6m");
@@ -230,7 +202,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("_ttl").field("enabled", true).endObject()
             .endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
         try {
             docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java b/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java
index 3041cc8d13c..1d849d50932 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java
@@ -31,12 +31,12 @@ import static org.hamcrest.Matchers.equalTo;
 public class ParseMappingTypeLevelTests extends ESSingleNodeTestCase {
     public void testTypeLevel() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("_all").field("enabled", false).endObject()
+                .startObject("_timestamp").field("enabled", true).endObject()
                 .endObject().endObject().string();
 
         DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
         assertThat(mapper.type(), equalTo("type"));
-        assertThat(mapper.allFieldMapper().enabled(), equalTo(false));
+        assertThat(mapper.timestampFieldMapper().enabled(), equalTo(true));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java
index afac48e2b89..ed20cdb7e35 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java
@@ -139,6 +139,16 @@ public class UpdateMappingOnClusterIT extends ESIntegTestCase {
         compareMappingOnNodes(mappingsBeforeUpdateResponse);
     }
 
+    // checks if the setting for timestamp and size are kept even if disabled
+    public void testDisabledSizeTimestampIndexDoNotLooseMappings() throws Exception {
+        String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/default_mapping_with_disabled_root_types.json");
+        prepareCreate(INDEX).addMapping(TYPE, mapping).get();
+        GetMappingsResponse mappingsBeforeGreen = client().admin().indices().prepareGetMappings(INDEX).addTypes(TYPE).get();
+        ensureGreen(INDEX);
+        // make sure all nodes have same cluster state
+        compareMappingOnNodes(mappingsBeforeGreen);
+    }
+
     protected void testConflict(String mapping, String mappingUpdate, String... errorMessages) throws InterruptedException {
         assertAcked(prepareCreate(INDEX).setSource(mapping).get());
         ensureGreen(INDEX);
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java
index 86624d938b6..17cb8a84664 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java
@@ -259,6 +259,56 @@ public class UpdateMappingTests extends ESSingleNodeTestCase {
         }
     }
 
+    public void testTimestampParsing() throws IOException {
+        IndexService indexService = createIndex("test");
+        XContentBuilder indexMapping = XContentFactory.jsonBuilder();
+        boolean enabled = randomBoolean();
+        indexMapping.startObject()
+                .startObject("type")
+                .startObject("_timestamp")
+                .field("enabled", enabled)
+                .endObject()
+                .endObject()
+                .endObject();
+        DocumentMapper documentMapper = indexService.mapperService().parse("type", new CompressedXContent(indexMapping.string()), true);
+        assertThat(documentMapper.timestampFieldMapper().enabled(), equalTo(enabled));
+        assertTrue(documentMapper.timestampFieldMapper().fieldType().stored());
+        assertTrue(documentMapper.timestampFieldMapper().fieldType().hasDocValues());
+        documentMapper = indexService.mapperService().parse("type", new CompressedXContent(documentMapper.mappingSource().string()), true);
+        assertThat(documentMapper.timestampFieldMapper().enabled(), equalTo(enabled));
+        assertTrue(documentMapper.timestampFieldMapper().fieldType().hasDocValues());
+        assertTrue(documentMapper.timestampFieldMapper().fieldType().stored());
+    }
+
+    public void testSizeTimestampIndexParsing() throws IOException {
+        IndexService indexService = createIndex("test", Settings.builder().build());
+        String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/default_mapping_with_disabled_root_types.json");
+        DocumentMapper documentMapper = indexService.mapperService().parse("type", new CompressedXContent(mapping), true);
+        assertThat(documentMapper.mappingSource().string(), equalTo(mapping));
+        documentMapper = indexService.mapperService().parse("type", new CompressedXContent(documentMapper.mappingSource().string()), true);
+        assertThat(documentMapper.mappingSource().string(), equalTo(mapping));
+    }
+
+    public void testDefaultApplied() throws IOException {
+        createIndex("test1", Settings.builder().build());
+        createIndex("test2", Settings.builder().build());
+        XContentBuilder defaultMapping = XContentFactory.jsonBuilder().startObject()
+                .startObject(MapperService.DEFAULT_MAPPING).startObject("_timestamp").field("enabled", true).endObject().endObject()
+                .endObject();
+        client().admin().indices().preparePutMapping().setType(MapperService.DEFAULT_MAPPING).setSource(defaultMapping).get();
+        XContentBuilder typeMapping = XContentFactory.jsonBuilder().startObject()
+                .startObject("type").startObject("_all").field("enabled", false).endObject().endObject()
+                .endObject();
+        client().admin().indices().preparePutMapping("test1").setType("type").setSource(typeMapping).get();
+        client().admin().indices().preparePutMapping("test1", "test2").setType("type").setSource(typeMapping).get();
+
+        GetMappingsResponse response = client().admin().indices().prepareGetMappings("test2").get();
+        assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_all"));
+        assertFalse((Boolean) ((LinkedHashMap) response.getMappings().get("test2").get("type").getSourceAsMap().get("_all")).get("enabled"));
+        assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_timestamp"));
+        assertTrue((Boolean)((LinkedHashMap)response.getMappings().get("test2").get("type").getSourceAsMap().get("_timestamp")).get("enabled"));
+    }
+
     public void testRejectFieldDefinedTwice() throws IOException {
         String mapping1 = XContentFactory.jsonBuilder().startObject()
                 .startObject("type1")
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
index 4f7064a33bb..cdb722ff9dd 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
@@ -65,6 +65,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSear
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.core.IsNull.notNullValue;
 
@@ -106,7 +107,8 @@ public class DateHistogramIT extends ESIntegTestCase {
 
     @Override
     public void setupSuiteScopeCluster() throws Exception {
-        createIndex("idx", "idx_unmapped");
+        assertAcked(prepareCreate("idx").addMapping("type", "_timestamp", "enabled=true"));
+        createIndex("idx_unmapped");
         // TODO: would be nice to have more random data here
         assertAcked(prepareCreate("empty_bucket_idx").addMapping("type", "value", "type=integer"));
         List<IndexRequestBuilder> builders = new ArrayList<>();
@@ -1139,6 +1141,13 @@ public class DateHistogramIT extends ESIntegTestCase {
         }
     }
 
+    public void testTimestampField() { // see #11692
+        SearchResponse response = client().prepareSearch("idx").addAggregation(dateHistogram("histo").field("_timestamp").dateHistogramInterval(randomFrom(DateHistogramInterval.DAY, DateHistogramInterval.MONTH))).get();
+        assertSearchResponse(response);
+        Histogram histo = response.getAggregations().get("histo");
+        assertThat(histo.getBuckets().size(), greaterThan(0));
+    }
+
     /**
      * When DST ends, local time turns back one hour, so between 2am and 4am wall time we should have four buckets:
      * "2015-10-25T02:00:00.000+02:00",
diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
index 99183b11c58..188ae22471a 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
@@ -36,6 +36,7 @@ import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.hamcrest.Matchers;
@@ -69,8 +70,10 @@ import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -1337,12 +1340,16 @@ public class FieldSortIT extends ESIntegTestCase {
     }
 
     public void testSortMetaField() throws Exception {
-        createIndex("test");
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("_timestamp").field("enabled", true).endObject()
+                .endObject().endObject();
+        assertAcked(prepareCreate("test")
+            .addMapping("type", mapping));
         ensureGreen();
         final int numDocs = randomIntBetween(10, 20);
         IndexRequestBuilder[] indexReqs = new IndexRequestBuilder[numDocs];
         for (int i = 0; i < numDocs; ++i) {
-            indexReqs[i] = client().prepareIndex("test", "type", Integer.toString(i))
+            indexReqs[i] = client().prepareIndex("test", "type", Integer.toString(i)).setTimestamp(Integer.toString(randomInt(1000)))
                     .setSource();
         }
         indexRandom(true, indexReqs);
@@ -1361,6 +1368,37 @@ public class FieldSortIT extends ESIntegTestCase {
             assertThat(previous, order == SortOrder.ASC ? lessThan(uid) : greaterThan(uid));
             previous = uid;
         }
+
+        /*
+        searchResponse = client().prepareSearch()
+                .setQuery(matchAllQuery())
+                .setSize(randomIntBetween(1, numDocs + 5))
+                .addSort("_id", order)
+                .execute().actionGet();
+        assertNoFailures(searchResponse);
+        hits = searchResponse.getHits().hits();
+        previous = order == SortOrder.ASC ? new BytesRef() : UnicodeUtil.BIG_TERM;
+        for (int i = 0; i < hits.length; ++i) {
+            final BytesRef id = new BytesRef(Uid.createUid(hits[i].type(), hits[i].id()));
+            assertThat(previous, order == SortOrder.ASC ? lessThan(id) : greaterThan(id));
+            previous = id;
+        }*/
+
+        searchResponse = client().prepareSearch()
+                .setQuery(matchAllQuery())
+                .setSize(randomIntBetween(1, numDocs + 5))
+                .addSort("_timestamp", order)
+                .addField("_timestamp")
+                .execute().actionGet();
+        assertNoFailures(searchResponse);
+        hits = searchResponse.getHits().hits();
+        Long previousTs = order == SortOrder.ASC ? 0 : Long.MAX_VALUE;
+        for (int i = 0; i < hits.length; ++i) {
+            SearchHitField timestampField = hits[i].getFields().get("_timestamp");
+            Long timestamp = timestampField.<Long>getValue();
+            assertThat(previousTs, order == SortOrder.ASC ? lessThanOrEqualTo(timestamp) : greaterThanOrEqualTo(timestamp));
+            previousTs = timestamp;
+        }
     }
 
     /**
diff --git a/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java b/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java
index bd24ab9f43b..2fa94bc2357 100644
--- a/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java
+++ b/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java
@@ -19,20 +19,14 @@
 
 package org.elasticsearch.timestamp;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
 import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.InternalSettingsPlugin;
 
-import java.util.Collection;
 import java.util.Locale;
 
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
@@ -47,17 +41,8 @@ import static org.hamcrest.Matchers.notNullValue;
 /**
  */
 public class SimpleTimestampIT extends ESIntegTestCase {
-
-    private static final Settings BW_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0).build();
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return pluginList(InternalSettingsPlugin.class);
-    }
-
     public void testSimpleTimestamp() throws Exception {
         client().admin().indices().prepareCreate("test")
-                .setSettings(BW_SETTINGS)
                 .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("_timestamp").field("enabled", true).endObject().endObject().endObject())
                 .execute().actionGet();
         client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet();
@@ -113,7 +98,7 @@ public class SimpleTimestampIT extends ESIntegTestCase {
         String type = "mytype";
 
         XContentBuilder builder = jsonBuilder().startObject().startObject("_timestamp").field("enabled", true).endObject().endObject();
-        assertAcked(client().admin().indices().prepareCreate(index).setSettings(BW_SETTINGS).addMapping(type, builder));
+        assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
 
         // check mapping again
         assertTimestampMappingEnabled(index, type, true);
@@ -132,7 +117,7 @@ public class SimpleTimestampIT extends ESIntegTestCase {
         String type = "mytype";
 
         XContentBuilder builder = jsonBuilder().startObject().startObject("_timestamp").field("enabled", true).endObject().endObject();
-        assertAcked(client().admin().indices().prepareCreate(index).setSettings(BW_SETTINGS).addMapping(type, builder));
+        assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
 
         // check mapping again
         assertTimestampMappingEnabled(index, type, true);
diff --git a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
index 9e08ecde6fa..003c08134bc 100644
--- a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
+++ b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.ttl;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
@@ -27,19 +26,14 @@ import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.update.UpdateRequestBuilder;
 import org.elasticsearch.action.update.UpdateResponse;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 
 import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
 import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
@@ -66,11 +60,6 @@ public class SimpleTTLIT extends ESIntegTestCase {
         return 2;
     }
 
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(InternalSettingsPlugin.class);
-    }
-
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
@@ -81,7 +70,6 @@ public class SimpleTTLIT extends ESIntegTestCase {
 
     public void testSimpleTTL() throws Exception {
         assertAcked(prepareCreate("test")
-                .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id)
                 .addMapping("type1", XContentFactory.jsonBuilder()
                         .startObject()
                         .startObject("type1")
@@ -221,7 +209,7 @@ public class SimpleTTLIT extends ESIntegTestCase {
         String type = "mytype";
 
         XContentBuilder builder = jsonBuilder().startObject().startObject("_ttl").field("enabled", true).endObject().endObject();
-        assertAcked(client().admin().indices().prepareCreate(index).setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id).addMapping(type, builder));
+        assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
 
         // check mapping again
         assertTTLMappingEnabled(index, type);
@@ -244,7 +232,6 @@ public class SimpleTTLIT extends ESIntegTestCase {
      */
     public void testNoopUpdate() throws IOException {
         assertAcked(prepareCreate("test")
-                .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id)
                 .addMapping("type1", XContentFactory.jsonBuilder()
                         .startObject()
                         .startObject("type1")
diff --git a/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java b/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java
deleted file mode 100644
index 3573089fcaa..00000000000
--- a/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java
+++ /dev/null
@@ -1,237 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.update;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.action.admin.indices.alias.Alias;
-import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.update.UpdateResponse;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.index.engine.DocumentMissingException;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.script.Script;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.SearchHitField;
-import org.elasticsearch.search.sort.SortOrder;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.InternalSettingsPlugin;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-
-import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
-import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
-import static org.hamcrest.Matchers.allOf;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.lessThanOrEqualTo;
-
-public class TimestampTTLBWIT extends ESIntegTestCase {
-
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(
-                UpdateIT.FieldIncrementScriptPlugin.class,
-                UpdateIT.ExtractContextInSourceScriptPlugin.class,
-                UpdateIT.PutFieldValuesScriptPlugin.class,
-                InternalSettingsPlugin.class
-        );
-    }
-
-    public void testSort() throws Exception {
-        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("_timestamp").field("enabled", true).endObject()
-                .endObject().endObject();
-        assertAcked(prepareCreate("test")
-                .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id)
-                .addMapping("type", mapping));
-        ensureGreen();
-        final int numDocs = randomIntBetween(10, 20);
-        IndexRequestBuilder[] indexReqs = new IndexRequestBuilder[numDocs];
-        for (int i = 0; i < numDocs; ++i) {
-            indexReqs[i] = client().prepareIndex("test", "type", Integer.toString(i)).setTimestamp(Integer.toString(randomInt(1000)))
-                    .setSource();
-        }
-        indexRandom(true, indexReqs);
-
-        SortOrder order = randomFrom(SortOrder.values());
-
-        SearchResponse searchResponse = client().prepareSearch()
-                .setQuery(matchAllQuery())
-                .setSize(randomIntBetween(1, numDocs + 5))
-                .addSort("_timestamp", order)
-                .addField("_timestamp")
-                .execute().actionGet();
-        assertNoFailures(searchResponse);
-        SearchHit[] hits = searchResponse.getHits().hits();
-        Long previousTs = order == SortOrder.ASC ? 0 : Long.MAX_VALUE;
-        for (int i = 0; i < hits.length; ++i) {
-            SearchHitField timestampField = hits[i].getFields().get("_timestamp");
-            Long timestamp = timestampField.<Long>getValue();
-            assertThat(previousTs, order == SortOrder.ASC ? lessThanOrEqualTo(timestamp) : greaterThanOrEqualTo(timestamp));
-            previousTs = timestamp;
-        }
-    }
-
-    public void testUpdate() throws Exception {
-        assertAcked(prepareCreate("test").addAlias(new Alias("alias"))
-                .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id)
-                .addMapping("type1", XContentFactory.jsonBuilder()
-                        .startObject()
-                        .startObject("type1")
-                        .startObject("_timestamp").field("enabled", true).endObject()
-                        .startObject("_ttl").field("enabled", true).endObject()
-                        .endObject()
-                        .endObject()));
-
-        ensureGreen();
-
-        try {
-            client().prepareUpdate(indexOrAlias(), "type1", "1")
-                    .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null)).execute().actionGet();
-            fail();
-        } catch (DocumentMissingException e) {
-            // all is well
-        }
-
-        // check TTL is kept after an update without TTL
-        client().prepareIndex("test", "type1", "2").setSource("field", 1).setTTL(86400000L).setRefreshPolicy(IMMEDIATE).get();
-        GetResponse getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
-        long ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue();
-        assertThat(ttl, greaterThan(0L));
-        client().prepareUpdate(indexOrAlias(), "type1", "2")
-                .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null)).execute().actionGet();
-        getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
-        ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue();
-        assertThat(ttl, greaterThan(0L));
-
-        // check TTL update
-        client().prepareUpdate(indexOrAlias(), "type1", "2")
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values",
-                        Collections.singletonMap("_ctx", Collections.singletonMap("_ttl", 3600000)))).execute().actionGet();
-        getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
-        ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue();
-        assertThat(ttl, greaterThan(0L));
-        assertThat(ttl, lessThanOrEqualTo(3600000L));
-
-        // check timestamp update
-        client().prepareIndex("test", "type1", "3").setSource("field", 1).setRefreshPolicy(IMMEDIATE).get();
-        client().prepareUpdate(indexOrAlias(), "type1", "3")
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values",
-                        Collections.singletonMap("_ctx", Collections.singletonMap("_timestamp", "2009-11-15T14:12:12")))).execute()
-                .actionGet();
-        getResponse = client().prepareGet("test", "type1", "3").setFields("_timestamp").execute().actionGet();
-        long timestamp = ((Number) getResponse.getField("_timestamp").getValue()).longValue();
-        assertThat(timestamp, equalTo(1258294332000L));
-    }
-
-    public void testContextVariables() throws Exception {
-        assertAcked(prepareCreate("test").addAlias(new Alias("alias"))
-                        .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id)
-                        .addMapping("type1", XContentFactory.jsonBuilder()
-                                .startObject()
-                                .startObject("type1")
-                                .startObject("_timestamp").field("enabled", true).endObject()
-                                .startObject("_ttl").field("enabled", true).endObject()
-                                .endObject()
-                                .endObject())
-                        .addMapping("subtype1", XContentFactory.jsonBuilder()
-                                .startObject()
-                                .startObject("subtype1")
-                                .startObject("_parent").field("type", "type1").endObject()
-                                .startObject("_timestamp").field("enabled", true).endObject()
-                                .startObject("_ttl").field("enabled", true).endObject()
-                                .endObject()
-                                .endObject())
-        );
-        ensureGreen();
-
-        // Index some documents
-        long timestamp = System.currentTimeMillis();
-        client().prepareIndex()
-                .setIndex("test")
-                .setType("type1")
-                .setId("parentId1")
-                .setTimestamp(String.valueOf(timestamp-1))
-                .setSource("field1", 0, "content", "bar")
-                .execute().actionGet();
-
-        long ttl = 10000;
-        client().prepareIndex()
-                .setIndex("test")
-                .setType("subtype1")
-                .setId("id1")
-                .setParent("parentId1")
-                .setRouting("routing1")
-                .setTimestamp(String.valueOf(timestamp))
-                .setTTL(ttl)
-                .setSource("field1", 1, "content", "foo")
-                .execute().actionGet();
-
-        // Update the first object and note context variables values
-        UpdateResponse updateResponse = client().prepareUpdate("test", "subtype1", "id1")
-                .setRouting("routing1")
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "extract_ctx", null))
-                .execute().actionGet();
-
-        assertEquals(2, updateResponse.getVersion());
-
-        GetResponse getResponse = client().prepareGet("test", "subtype1", "id1").setRouting("routing1").execute().actionGet();
-        Map<String, Object> updateContext = (Map<String, Object>) getResponse.getSourceAsMap().get("update_context");
-        assertEquals("test", updateContext.get("_index"));
-        assertEquals("subtype1", updateContext.get("_type"));
-        assertEquals("id1", updateContext.get("_id"));
-        assertEquals(1, updateContext.get("_version"));
-        assertEquals("parentId1", updateContext.get("_parent"));
-        assertEquals("routing1", updateContext.get("_routing"));
-        assertThat(((Integer) updateContext.get("_ttl")).longValue(), allOf(greaterThanOrEqualTo(ttl-3000), lessThanOrEqualTo(ttl)));
-
-        // Idem with the second object
-        updateResponse = client().prepareUpdate("test", "type1", "parentId1")
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "extract_ctx", null))
-                .execute().actionGet();
-
-        assertEquals(2, updateResponse.getVersion());
-
-        getResponse = client().prepareGet("test", "type1", "parentId1").execute().actionGet();
-        updateContext = (Map<String, Object>) getResponse.getSourceAsMap().get("update_context");
-        assertEquals("test", updateContext.get("_index"));
-        assertEquals("type1", updateContext.get("_type"));
-        assertEquals("parentId1", updateContext.get("_id"));
-        assertEquals(1, updateContext.get("_version"));
-        assertNull(updateContext.get("_parent"));
-        assertNull(updateContext.get("_routing"));
-        assertNull(updateContext.get("_ttl"));
-    }
-
-    private static String indexOrAlias() {
-        return randomBoolean() ? "test" : "alias";
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/update/UpdateIT.java b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
index 55834c181b0..4121543a7a0 100644
--- a/core/src/test/java/org/elasticsearch/update/UpdateIT.java
+++ b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
@@ -62,12 +62,14 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
@@ -355,14 +357,21 @@ public class UpdateIT extends ESIntegTestCase {
                 FieldIncrementScriptPlugin.class,
                 ScriptedUpsertScriptPlugin.class,
                 ExtractContextInSourceScriptPlugin.class,
-                InternalSettingsPlugin.class
+                InternalSettingsPlugin.class // uses index.merge.enabled
         );
     }
 
     private void createTestIndex() throws Exception {
         logger.info("--> creating index test");
 
-        assertAcked(prepareCreate("test").addAlias(new Alias("alias")));
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias"))
+                .addMapping("type1", XContentFactory.jsonBuilder()
+                        .startObject()
+                        .startObject("type1")
+                        .startObject("_timestamp").field("enabled", true).endObject()
+                        .startObject("_ttl").field("enabled", true).endObject()
+                        .endObject()
+                        .endObject()));
     }
 
     public void testUpsert() throws Exception {
@@ -629,6 +638,34 @@ public class UpdateIT extends ESIntegTestCase {
             assertThat(getResponse.isExists(), equalTo(false));
         }
 
+        // check TTL is kept after an update without TTL
+        client().prepareIndex("test", "type1", "2").setSource("field", 1).setTTL(86400000L).setRefreshPolicy(IMMEDIATE).get();
+        GetResponse getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
+        long ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue();
+        assertThat(ttl, greaterThan(0L));
+        client().prepareUpdate(indexOrAlias(), "type1", "2")
+                .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null)).execute().actionGet();
+        getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
+        ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue();
+        assertThat(ttl, greaterThan(0L));
+
+        // check TTL update
+        client().prepareUpdate(indexOrAlias(), "type1", "2")
+                .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", Collections.singletonMap("_ctx", Collections.singletonMap("_ttl", 3600000)))).execute().actionGet();
+        getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
+        ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue();
+        assertThat(ttl, greaterThan(0L));
+        assertThat(ttl, lessThanOrEqualTo(3600000L));
+
+        // check timestamp update
+        client().prepareIndex("test", "type1", "3").setSource("field", 1).setRefreshPolicy(IMMEDIATE).get();
+        client().prepareUpdate(indexOrAlias(), "type1", "3")
+                .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", Collections.singletonMap("_ctx", Collections.singletonMap("_timestamp", "2009-11-15T14:12:12")))).execute()
+                .actionGet();
+        getResponse = client().prepareGet("test", "type1", "3").setFields("_timestamp").execute().actionGet();
+        long timestamp = ((Number) getResponse.getField("_timestamp").getValue()).longValue();
+        assertThat(timestamp, equalTo(1258294332000L));
+
         // check fields parameter
         client().prepareIndex("test", "type1", "1").setSource("field", 1).execute().actionGet();
         updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1")
@@ -645,7 +682,7 @@ public class UpdateIT extends ESIntegTestCase {
         client().prepareIndex("test", "type1", "1").setSource("field", 1).execute().actionGet();
         updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(XContentFactory.jsonBuilder().startObject().field("field2", 2).endObject()).execute().actionGet();
         for (int i = 0; i < 5; i++) {
-            GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
+            getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
             assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("1"));
             assertThat(getResponse.getSourceAsMap().get("field2").toString(), equalTo("2"));
         }
@@ -653,7 +690,7 @@ public class UpdateIT extends ESIntegTestCase {
         // change existing field
         updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(XContentFactory.jsonBuilder().startObject().field("field", 3).endObject()).execute().actionGet();
         for (int i = 0; i < 5; i++) {
-            GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
+            getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
             assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("3"));
             assertThat(getResponse.getSourceAsMap().get("field2").toString(), equalTo("2"));
         }
@@ -671,7 +708,7 @@ public class UpdateIT extends ESIntegTestCase {
         client().prepareIndex("test", "type1", "1").setSource("map", testMap).execute().actionGet();
         updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(XContentFactory.jsonBuilder().startObject().field("map", testMap3).endObject()).execute().actionGet();
         for (int i = 0; i < 5; i++) {
-            GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
+            getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
             Map map1 = (Map) getResponse.getSourceAsMap().get("map");
             assertThat(map1.size(), equalTo(3));
             assertThat(map1.containsKey("map1"), equalTo(true));
@@ -723,12 +760,16 @@ public class UpdateIT extends ESIntegTestCase {
                         .addMapping("type1", XContentFactory.jsonBuilder()
                                 .startObject()
                                 .startObject("type1")
+                                .startObject("_timestamp").field("enabled", true).endObject()
+                                .startObject("_ttl").field("enabled", true).endObject()
                                 .endObject()
                                 .endObject())
                         .addMapping("subtype1", XContentFactory.jsonBuilder()
                                 .startObject()
                                 .startObject("subtype1")
                                 .startObject("_parent").field("type", "type1").endObject()
+                                .startObject("_timestamp").field("enabled", true).endObject()
+                                .startObject("_ttl").field("enabled", true).endObject()
                                 .endObject()
                                 .endObject())
         );
@@ -772,6 +813,7 @@ public class UpdateIT extends ESIntegTestCase {
         assertEquals(1, updateContext.get("_version"));
         assertEquals("parentId1", updateContext.get("_parent"));
         assertEquals("routing1", updateContext.get("_routing"));
+        assertThat(((Integer) updateContext.get("_ttl")).longValue(), allOf(greaterThanOrEqualTo(ttl-3000), lessThanOrEqualTo(ttl)));
 
         // Idem with the second object
         updateResponse = client().prepareUpdate("test", "type1", "parentId1")
@@ -862,6 +904,13 @@ public class UpdateIT extends ESIntegTestCase {
     public void testStressUpdateDeleteConcurrency() throws Exception {
         //We create an index with merging disabled so that deletes don't get merged away
         assertAcked(prepareCreate("test")
+                .addMapping("type1", XContentFactory.jsonBuilder()
+                        .startObject()
+                        .startObject("type1")
+                        .startObject("_timestamp").field("enabled", true).endObject()
+                        .startObject("_ttl").field("enabled", true).endObject()
+                        .endObject()
+                        .endObject())
                 .setSettings(Settings.builder().put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)));
         ensureGreen();
 
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
index 28894c5c1cc..d834f0fcf6d 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
@@ -22,6 +22,7 @@ package org.elasticsearch.messy.tests;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -44,6 +45,7 @@ import org.elasticsearch.test.ESIntegTestCase;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Base64;
@@ -86,6 +88,8 @@ public class SearchFieldsTests extends ESIntegTestCase {
         client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForYellowStatus().execute().actionGet();
 
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
+                // _timestamp is randomly enabled via templates but we don't want it here to test stored fields behaviour
+                .startObject("_timestamp").field("enabled", false).endObject()
                 .startObject("properties")
                 .startObject("field1").field("type", "text").field("store", true).endObject()
                 .startObject("field2").field("type", "text").field("store", false).endObject()
@@ -694,7 +698,7 @@ public class SearchFieldsTests extends ESIntegTestCase {
     public void testLoadMetadata() throws Exception {
         assertAcked(prepareCreate("test")
                 .addMapping("parent")
-                .addMapping("my-type1", "_parent", "type=parent"));
+                .addMapping("my-type1", "_timestamp", "enabled=true", "_ttl", "enabled=true", "_parent", "type=parent"));
 
         indexRandom(true,
                 client().prepareIndex("test", "my-type1", "1")
@@ -713,6 +717,12 @@ public class SearchFieldsTests extends ESIntegTestCase {
         assertThat(fields.get("field1"), nullValue());
         assertThat(fields.get("_routing").isMetadataField(), equalTo(true));
         assertThat(fields.get("_routing").getValue().toString(), equalTo("1"));
+        assertThat(fields.get("_timestamp").isMetadataField(), equalTo(true));
+        assertThat(fields.get("_timestamp").getValue().toString(), equalTo("205097"));
+        assertThat(fields.get("_ttl").isMetadataField(), equalTo(true));
+        // TODO: _ttl should return the original value, but it does not work today because
+        // it would use now() instead of the value of _timestamp to rebase
+        // assertThat(fields.get("_ttl").getValue().toString(), equalTo("10000000205097"));
         assertThat(fields.get("_parent").isMetadataField(), equalTo(true));
         assertThat(fields.get("_parent").getValue().toString(), equalTo("parent_1"));
     }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc b/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc
index 688d8cbdc5b..304d31978cf 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc
@@ -255,7 +255,7 @@ Supports also regular expressions with flag X for more readability (accepts whit
 Compares two numeric values, eg:
 
 ....
-    - lt: { foo: 10000 }  # the `foo` value is less than 10,000
+    - lt: { _ttl: 10000 }  # the `_ttl` value is less than 10,000
 ....
 
 === `lte` and `gte`
@@ -263,7 +263,7 @@ Compares two numeric values, eg:
 Compares two numeric values, eg:
 
 ....
-    - lte: { foo: 10000 }  # the `foo` value is less than or equal to 10,000
+    - lte: { _ttl: 10000 }  # the `_ttl` value is less than or equal to 10,000
 ....
 
 === `length`
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_timestamp.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_timestamp.yaml
new file mode 100644
index 00000000000..83173817b6e
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_timestamp.yaml
@@ -0,0 +1,80 @@
+---
+"Timestamp":
+
+ - do:
+      indices.create:
+          index:    test_1
+          body:
+              mappings:
+                test:
+                  _timestamp:
+                    enabled:  1
+ - do:
+      cluster.health:
+          wait_for_status: yellow
+
+# blank timestamp
+ - do:
+      create:
+          index:     test_1
+          type:      test
+          id:        1
+          body:      { foo: bar }
+
+ - do:
+      get:
+          index:   test_1
+          type:    test
+          id:      1
+          fields:  _timestamp
+
+ - is_true:    _timestamp
+
+# milliseconds since epoch
+
+ - do:
+      delete:
+          index:   test_1
+          type:    test
+          id:      1
+ - do:
+      create:
+          index:     test_1
+          type:      test
+          id:        1
+          body:      { foo: bar }
+          timestamp: 1372011280000
+
+ - do:
+      get:
+          index:   test_1
+          type:    test
+          id:      1
+          fields:  _timestamp
+
+ - match: { _timestamp: 1372011280000 }
+
+# date format
+
+ - do:
+      delete:
+          index:   test_1
+          type:    test
+          id:      1
+ - do:
+      create:
+          index:     test_1
+          type:      test
+          id:        1
+          body:      { foo: bar }
+          timestamp: 2013-06-23T18:14:40
+
+ - do:
+      get:
+          index:   test_1
+          type:    test
+          id:      1
+          fields:  _timestamp
+
+ - match: { _timestamp: 1372011280000 }
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/75_ttl.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/75_ttl.yaml
new file mode 100644
index 00000000000..3f9b4763669
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/create/75_ttl.yaml
@@ -0,0 +1,100 @@
+---
+"TTL":
+
+ - do:
+      indices.create:
+          index:    test_1
+          body:
+              mappings:
+                test:
+                  _ttl:
+                    enabled:  1
+                    default:  10s
+ - do:
+      cluster.health:
+          wait_for_status: yellow
+
+# blank ttl
+ - do:
+      create:
+          index:     test_1
+          type:      test
+          id:        1
+          body:      { foo: bar }
+
+ - do:
+      get:
+          index:   test_1
+          type:    test
+          id:      1
+          fields:  _ttl
+
+ - lte:   { _ttl: 10000}
+ - gt:    { _ttl: 0}
+
+# milliseconds
+
+ - do:
+      delete:
+          index:   test_1
+          type:    test
+          id:      1
+ - do:
+      create:
+          index:     test_1
+          type:      test
+          id:        1
+          body:      { foo: bar }
+          ttl:       100000ms
+ - do:
+      get:
+          index:   test_1
+          type:    test
+          id:      1
+          fields:  _ttl
+
+ - lte:   { _ttl: 100000}
+ - gt:    { _ttl: 10000}
+
+# duration
+
+ - do:
+      delete:
+          index:   test_1
+          type:    test
+          id:      1
+ - do:
+      create:
+          index:     test_1
+          type:      test
+          id:        1
+          body:      { foo: bar }
+          ttl:       20s
+
+ - do:
+      get:
+          index:   test_1
+          type:    test
+          id:      1
+          fields:  _ttl
+
+ - lte:   { _ttl: 20000}
+ - gt:    { _ttl: 10000}
+
+# with timestamp
+
+ - do:
+      delete:
+          index:   test_1
+          type:    test
+          id:      1
+ - do:
+      catch: /already_expired_exception/
+      create:
+          index:     test_1
+          type:      test
+          id:        1
+          body:      { foo: bar }
+          ttl:       20s
+          timestamp: 2013-06-23T18:14:40
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/70_timestamp.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/70_timestamp.yaml
new file mode 100644
index 00000000000..938a358b103
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/70_timestamp.yaml
@@ -0,0 +1,70 @@
+---
+"Timestamp":
+
+ - do:
+      indices.create:
+          index:    test_1
+          body:
+              mappings:
+                test:
+                  _timestamp:
+                    enabled:  1
+ - do:
+      cluster.health:
+          wait_for_status: yellow
+
+# blank timestamp
+ - do:
+      index:
+          index:     test_1
+          type:      test
+          id:        1
+          body:      { foo: bar }
+
+ - do:
+      get:
+          index:   test_1
+          type:    test
+          id:      1
+          fields:  _timestamp
+
+ - is_true: _timestamp
+
+# milliseconds since epoch
+
+ - do:
+      index:
+          index:     test_1
+          type:      test
+          id:        1
+          body:      { foo: bar }
+          timestamp: 1372011280000
+
+ - do:
+      get:
+          index:   test_1
+          type:    test
+          id:      1
+          fields:  _timestamp
+
+ - match: { _timestamp: 1372011280000 }
+
+# date format
+
+ - do:
+      index:
+          index:     test_1
+          type:      test
+          id:        1
+          body:      { foo: bar }
+          timestamp: 2013-06-23T18:14:40
+
+ - do:
+      get:
+          index:   test_1
+          type:    test
+          id:      1
+          fields:  _timestamp
+
+ - match: { _timestamp: 1372011280000 }
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/75_ttl.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/75_ttl.yaml
new file mode 100644
index 00000000000..2ec00eaf2f4
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/75_ttl.yaml
@@ -0,0 +1,85 @@
+---
+"TTL":
+
+ - do:
+      indices.create:
+          index:    test_1
+          body:
+              mappings:
+                test:
+                  _ttl:
+                    enabled:  1
+                    default:  10s
+ - do:
+      cluster.health:
+          wait_for_status: yellow
+
+# blank ttl
+ - do:
+      index:
+          index:     test_1
+          type:      test
+          id:        1
+          body:      { foo: bar }
+
+ - do:
+      get:
+          index:   test_1
+          type:    test
+          id:      1
+          fields:  _ttl
+
+ - lte:   { _ttl: 10000}
+ - gt:    { _ttl: 0}
+
+# milliseconds
+
+ - do:
+      index:
+          index:     test_1
+          type:      test
+          id:        1
+          body:      { foo: bar }
+          ttl:       100000ms
+ - do:
+      get:
+          index:   test_1
+          type:    test
+          id:      1
+          fields:  _ttl
+
+ - lte:   { _ttl: 100000}
+ - gt:    { _ttl: 10000}
+
+# duration
+
+ - do:
+      index:
+          index:     test_1
+          type:      test
+          id:        1
+          body:      { foo: bar }
+          ttl:       20s
+
+ - do:
+      get:
+          index:   test_1
+          type:    test
+          id:      1
+          fields:  _ttl
+
+ - lte:   { _ttl: 20000}
+ - gt:    { _ttl: 10000}
+
+# with timestamp
+
+ - do:
+      catch: /already_expired_exception/
+      index:
+          index:     test_1
+          type:      test
+          id:        1
+          body:      { foo: bar }
+          ttl:       20s
+          timestamp: 2013-06-23T18:14:40
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/70_timestamp.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/70_timestamp.yaml
new file mode 100644
index 00000000000..864c219c448
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/70_timestamp.yaml
@@ -0,0 +1,76 @@
+---
+"Timestamp":
+
+ - do:
+      indices.create:
+          index:    test_1
+          body:
+              mappings:
+                test:
+                  _timestamp:
+                    enabled:  1
+ - do:
+      cluster.health:
+          wait_for_status: yellow
+
+# blank timestamp
+ - do:
+      update:
+          index:     test_1
+          type:      test
+          id:        1
+          body:
+            doc:        { foo: baz }
+            upsert:     { foo: bar }
+
+ - do:
+      get:
+          index:   test_1
+          type:    test
+          id:      1
+          fields:  _timestamp
+
+ - is_true: _timestamp
+
+# milliseconds since epoch
+
+ - do:
+      update:
+          index:     test_1
+          type:      test
+          id:        1
+          body:
+            doc:        { foo: baz }
+            upsert:     { foo: bar }
+          timestamp: 1372011280000
+
+ - do:
+      get:
+          index:   test_1
+          type:    test
+          id:      1
+          fields:  _timestamp
+
+ - match: { _timestamp: 1372011280000 }
+
+# date format
+
+ - do:
+      update:
+          index:     test_1
+          type:      test
+          id:        1
+          body:
+            doc:        { foo: baz }
+            upsert:     { foo: bar }
+          timestamp: 2013-06-23T18:14:40
+
+ - do:
+      get:
+          index:   test_1
+          type:    test
+          id:      1
+          fields:  _timestamp
+
+ - match: { _timestamp: 1372011280000 }
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/75_ttl.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/75_ttl.yaml
new file mode 100644
index 00000000000..09179ebd5ac
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/75_ttl.yaml
@@ -0,0 +1,92 @@
+---
+"TTL":
+
+ - do:
+      indices.create:
+          index:    test_1
+          body:
+              mappings:
+                test:
+                  _ttl:
+                    enabled:  1
+                    default:  10s
+ - do:
+      cluster.health:
+          wait_for_status: yellow
+
+# blank ttl
+ - do:
+      update:
+          index:     test_1
+          type:      test
+          id:        1
+          body:
+            doc:        { foo: baz }
+            upsert:     { foo: bar }
+
+ - do:
+      get:
+          index:   test_1
+          type:    test
+          id:      1
+          fields:  _ttl
+
+ - lte:   { _ttl: 10000}
+ - gt:    { _ttl: 0}
+
+# milliseconds
+
+ - do:
+      update:
+          index:     test_1
+          type:      test
+          id:        1
+          body:
+            doc:        { foo: baz }
+            upsert:     { foo: bar }
+          ttl:       100000ms
+
+ - do:
+      get:
+          index:   test_1
+          type:    test
+          id:      1
+          fields:  _ttl
+
+ - lte:   { _ttl: 100000}
+ - gt:    { _ttl: 10000}
+
+# seconds
+
+ - do:
+      update:
+          index:     test_1
+          type:      test
+          id:        1
+          body:
+            doc:        { foo: baz }
+            upsert:     { foo: bar }
+            detect_noop:  false
+          ttl:       20s
+
+ - do:
+      get:
+          index:   test_1
+          type:    test
+          id:      1
+          fields:  _ttl
+
+ - lte:   { _ttl: 20000}
+ - gt:    { _ttl: 10000}
+
+# with timestamp
+
+ - do:
+      catch: /already_expired_exception/
+      index:
+          index:     test_1
+          type:      test
+          id:        1
+          body:      { foo: bar }
+          ttl:       20s
+          timestamp: 2013-06-23T18:14:40
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml
index 0194453a99c..985048ad44d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml
@@ -12,6 +12,11 @@
             mappings:
               test:
                 _parent: { type: "foo" }
+                _timestamp:
+                  enabled:  1
+                _ttl:
+                  enabled:  1
+                  default:  10s
 
   - do:
       cluster.health:
@@ -23,13 +28,15 @@
           type:   test
           id:     1
           parent: 5
-          fields: [ _parent, _routing ]
+          fields: [ _parent, _routing, _timestamp, _ttl ]
           body:
             doc:    { foo: baz }
             upsert: { foo: bar }
 
   - match:   { get._parent:  "5" }
   - match:   { get._routing: "5" }
+  - is_true:   get._timestamp
+  - is_true:   get._ttl
 
   - do:
       get:
@@ -37,6 +44,6 @@
           type:   test
           id:     1
           parent: 5
-          fields: [ _parent, _routing ]
+          fields: [ _parent, _routing, _timestamp, _ttl ]
 
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index 8df2fd911c4..095b1339800 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -383,7 +383,13 @@ public abstract class ESIntegTestCase extends ESTestCase {
 
             XContentBuilder mappings = null;
             if (frequently() && randomDynamicTemplates()) {
-                mappings = XContentFactory.jsonBuilder().startObject().startObject("_default_").endObject().endObject();
+                mappings = XContentFactory.jsonBuilder().startObject().startObject("_default_");
+                if (randomBoolean()) {
+                    mappings.startObject(TimestampFieldMapper.NAME)
+                            .field("enabled", randomBoolean());
+                    mappings.endObject();
+                }
+                mappings.endObject().endObject();
             }
 
             for (String setting : randomSettingsBuilder.internalMap().keySet()) {

From 5f0292cb81aceecc81342aa0ea212f0ac945582d Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 15 Jun 2016 18:29:19 -0400
Subject: [PATCH 1004/1311] Fetch result when wait_for_completion

This makes this sequence:
```
curl -XDELETE localhost:9200/source,dest?pretty
for i in $( seq 1 100 ); do
  curl -XPOST localhost:9200/source/test -d'{"test": "test"}'; echo
done
curl localhost:9200/_refresh?pretty

curl -XPOST 'localhost:9200/_reindex?pretty&wait_for_completion=false' -d'{
  "source": {
    "index": "source"
  },
  "dest": {
    "index": "dest"
  }
}'

curl 'localhost:9200/_tasks/Jsyd6d9wSRW-O-NiiKbPcQ:237?wait_for_completion&pretty'
```

Return task *AND* the response to the user.

This also renames "result" to "response" in the persisted task info
to line it up with how we name the objects in Elasticsearch.
---
 .../tasks/get/TransportGetTaskAction.java     | 44 +++++++++--
 .../tasks/PersistedTaskInfo.java              | 78 ++++++++++---------
 .../tasks/task-index-mapping.json             |  2 +-
 .../admin/cluster/node/tasks/TasksIT.java     | 52 +++++++++----
 .../tasks/PersistedTaskInfoTests.java         |  8 +-
 .../test/delete_by_query/10_basic.yaml        | 11 +++
 .../rest-api-spec/test/reindex/10_basic.yaml  |  9 +++
 .../test/update_by_query/10_basic.yaml        | 12 +++
 8 files changed, 152 insertions(+), 64 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
index 0a869ff296a..477f646fe39 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
@@ -41,6 +41,7 @@ import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.tasks.PersistedTaskInfo;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.tasks.TaskInfo;
 import org.elasticsearch.tasks.TaskPersistenceService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.BaseTransportResponseHandler;
@@ -140,6 +141,7 @@ public class TransportGetTaskAction extends HandledTransportAction<GetTaskReques
     void getRunningTaskFromNode(Task thisTask, GetTaskRequest request, ActionListener<GetTaskResponse> listener) {
         Task runningTask = taskManager.getTask(request.getTaskId().getId());
         if (runningTask == null) {
+            // Task isn't running, go look in the task index
             getFinishedTaskFromIndex(thisTask, request, listener);
         } else {
             if (request.getWaitForCompletion()) {
@@ -148,9 +150,7 @@ public class TransportGetTaskAction extends HandledTransportAction<GetTaskReques
                     @Override
                     protected void doRun() throws Exception {
                         taskManager.waitForTaskCompletion(runningTask, waitForCompletionTimeout(request.getTimeout()));
-                        // TODO look up the task's result from the .tasks index now that it is done
-                        listener.onResponse(
-                                new GetTaskResponse(new PersistedTaskInfo(runningTask.taskInfo(clusterService.localNode(), true))));
+                        waitedForCompletion(thisTask, request, runningTask.taskInfo(clusterService.localNode(), true), listener);
                     }
 
                     @Override
@@ -159,15 +159,44 @@ public class TransportGetTaskAction extends HandledTransportAction<GetTaskReques
                     }
                 });
             } else {
-                listener.onResponse(new GetTaskResponse(new PersistedTaskInfo(runningTask.taskInfo(clusterService.localNode(), true))));
+                TaskInfo info = runningTask.taskInfo(clusterService.localNode(), true);
+                listener.onResponse(new GetTaskResponse(new PersistedTaskInfo(false, info)));
             }
         }
     }
 
     /**
-     * Send a {@link GetRequest} to the results index looking for the results of the task. It'll only be found only if the task's result was
-     * persisted. Called on the node that once had the task if that node is part of the cluster or on the coordinating node if the node
-     * wasn't part of the cluster.
+     * Called after waiting for the task to complete. Attempts to load the results of the task from the tasks index. If it isn't in the
+     * index then returns a snapshot of the task taken shortly after completion.
+     */
+    void waitedForCompletion(Task thisTask, GetTaskRequest request, TaskInfo snapshotOfRunningTask,
+            ActionListener<GetTaskResponse> listener) {
+        getFinishedTaskFromIndex(thisTask, request, new ActionListener<GetTaskResponse>() {
+            @Override
+            public void onResponse(GetTaskResponse response) {
+                // We were able to load the task from the task index. Let's send that back.
+                listener.onResponse(response);
+            }
+
+            @Override
+            public void onFailure(Throwable e) {
+                /*
+                 * We couldn't load the task from the task index. Instead of 404 we should use the snapshot we took after it finished. If
+                 * the error isn't a 404 then we'll just throw it back to the user.
+                 */
+                if (ExceptionsHelper.unwrap(e, ResourceNotFoundException.class) != null) {
+                    listener.onResponse(new GetTaskResponse(new PersistedTaskInfo(true, snapshotOfRunningTask)));
+                } else {
+                    listener.onFailure(e);
+                }
+            }
+        });
+    }
+
+    /**
+     * Send a {@link GetRequest} to the tasks index looking for a persisted copy of the task completed task. It'll only be found only if the
+     * task's result was persisted. Called on the node that once had the task if that node is still part of the cluster or on the
+     * coordinating node if the node is no longer part of the cluster.
      */
     void getFinishedTaskFromIndex(Task thisTask, GetTaskRequest request, ActionListener<GetTaskResponse> listener) {
         GetRequest get = new GetRequest(TaskPersistenceService.TASK_INDEX, TaskPersistenceService.TASK_TYPE,
@@ -202,6 +231,7 @@ public class TransportGetTaskAction extends HandledTransportAction<GetTaskReques
     void onGetFinishedTaskFromIndex(GetResponse response, ActionListener<GetTaskResponse> listener) throws IOException {
         if (false == response.isExists()) {
             listener.onFailure(new ResourceNotFoundException("task [{}] isn't running or persisted", response.getId()));
+            return;
         }
         if (response.isSourceEmpty()) {
             listener.onFailure(new ElasticsearchException("Stored task status for [{}] didn't contain any source!", response.getId()));
diff --git a/core/src/main/java/org/elasticsearch/tasks/PersistedTaskInfo.java b/core/src/main/java/org/elasticsearch/tasks/PersistedTaskInfo.java
index 7d551ce195c..06ead633641 100644
--- a/core/src/main/java/org/elasticsearch/tasks/PersistedTaskInfo.java
+++ b/core/src/main/java/org/elasticsearch/tasks/PersistedTaskInfo.java
@@ -38,6 +38,7 @@ import java.io.IOException;
 import java.util.Map;
 import java.util.Objects;
 
+import static java.util.Collections.emptyMap;
 import static java.util.Objects.requireNonNull;
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
@@ -45,56 +46,61 @@ import static org.elasticsearch.common.xcontent.XContentHelper.convertToMap;
 
 /**
  * Information about a persisted or running task. Running tasks just have a {@link #getTask()} while persisted tasks will have either a
- * {@link #getError()} or {@link #getResult()}.
+ * {@link #getError()} or {@link #getResponse()}.
  */
 public final class PersistedTaskInfo implements Writeable, ToXContent {
+    private final boolean completed;
     private final TaskInfo task;
     @Nullable
     private final BytesReference error;
     @Nullable
-    private final BytesReference result;
+    private final BytesReference response;
 
     /**
-     * Construct a {@linkplain PersistedTaskInfo} for a running task.
+     * Construct a {@linkplain PersistedTaskInfo} for a task for which we don't have a result or error. That usually means that the task
+     * is incomplete, but it could also mean that we waited for the task to complete but it didn't save any error information.
      */
-    public PersistedTaskInfo(TaskInfo task) {
-        this(task, null, null);
+    public PersistedTaskInfo(boolean completed, TaskInfo task) {
+        this(completed, task, null, null);
     }
 
     /**
      * Construct a {@linkplain PersistedTaskInfo} for a task that completed with an error.
      */
     public PersistedTaskInfo(TaskInfo task, Throwable error) throws IOException {
-        this(task, toXContent(error), null);
+        this(true, task, toXContent(error), null);
     }
 
     /**
      * Construct a {@linkplain PersistedTaskInfo} for a task that completed successfully.
      */
-    public PersistedTaskInfo(TaskInfo task, ToXContent result) throws IOException {
-        this(task, null, toXContent(result));
+    public PersistedTaskInfo(TaskInfo task, ToXContent response) throws IOException {
+        this(true, task, null, toXContent(response));
     }
 
-    private PersistedTaskInfo(TaskInfo task, @Nullable BytesReference error, @Nullable BytesReference result) {
+    private PersistedTaskInfo(boolean completed, TaskInfo task, @Nullable BytesReference error, @Nullable BytesReference result) {
+        this.completed = completed;
         this.task = requireNonNull(task, "task is required");
         this.error = error;
-        this.result = result;
+        this.response = result;
     }
 
     /**
      * Read from a stream.
      */
     public PersistedTaskInfo(StreamInput in) throws IOException {
+        completed = in.readBoolean();
         task = new TaskInfo(in);
         error = in.readOptionalBytesReference();
-        result = in.readOptionalBytesReference();
+        response = in.readOptionalBytesReference();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
+        out.writeBoolean(completed);
         task.writeTo(out);
         out.writeOptionalBytesReference(error);
-        out.writeOptionalBytesReference(result);
+        out.writeOptionalBytesReference(response);
     }
 
     /**
@@ -105,46 +111,45 @@ public final class PersistedTaskInfo implements Writeable, ToXContent {
     }
 
     /**
-     * Get the error that finished this task. Will return null if the task didn't finish with an error or it hasn't yet finished.
+     * Get the error that finished this task. Will return null if the task didn't finish with an error, it hasn't yet finished, or didn't
+     * persist its result.
      */
     public BytesReference getError() {
         return error;
     }
 
     /**
-     * Convert {@link #getError()} from XContent to a Map for easy processing. Will return null if the task didn't finish with an error or
-     * hasn't yet finished.
+     * Convert {@link #getError()} from XContent to a Map for easy processing. Will return an empty map if the task didn't finish with an
+     * error, hasn't yet finished, or didn't persist its result.
      */
     public Map<String, Object> getErrorAsMap() {
         if (error == null) {
-            return null;
+            return emptyMap();
         }
         return convertToMap(error, false).v2();
     }
 
     /**
-     * Get the result that this task finished with. Will return null if the task was finished by an error or it hasn't yet finished.
+     * Get the response that this task finished with. Will return null if the task was finished by an error, it hasn't yet finished, or
+     * didn't persist its result.
      */
-    public BytesReference getResult() {
-        return result;
+    public BytesReference getResponse() {
+        return response;
     }
 
     /**
-     * Convert {@link #getResult()} from XContent to a Map for easy processing. Will return null if the task was finished with an error or
-     * hasn't yet finished.
+     * Convert {@link #getResponse()} from XContent to a Map for easy processing. Will return an empty map if the task was finished with an
+     * error, hasn't yet finished, or didn't persist its result.
      */
-    public Map<String, Object> getResultAsMap() {
-        if (result == null) {
-            return null;
+    public Map<String, Object> getResponseAsMap() {
+        if (response == null) {
+            return emptyMap();
         }
-        return convertToMap(result, false).v2();
+        return convertToMap(response, false).v2();
     }
 
-    /**
-     * Was the task completed before returned?
-     */
     public boolean isCompleted() {
-        return error != null || result != null;
+        return completed;
     }
 
     @Override
@@ -159,18 +164,18 @@ public final class PersistedTaskInfo implements Writeable, ToXContent {
         if (error != null) {
             XContentHelper.writeRawField("error", error, builder, params);
         }
-        if (result != null) {
-            XContentHelper.writeRawField("result", result, builder, params);
+        if (response != null) {
+            XContentHelper.writeRawField("response", response, builder, params);
         }
         return builder;
     }
 
     public static final ConstructingObjectParser<PersistedTaskInfo, ParseFieldMatcherSupplier> PARSER = new ConstructingObjectParser<>(
-            "persisted_task_info", a -> new PersistedTaskInfo((TaskInfo) a[0], (BytesReference) a[1], (BytesReference) a[2]));
+            "persisted_task_info", a -> new PersistedTaskInfo(true, (TaskInfo) a[0], (BytesReference) a[1], (BytesReference) a[2]));
     static {
         PARSER.declareObject(constructorArg(), TaskInfo.PARSER, new ParseField("task"));
         PARSER.declareRawObject(optionalConstructorArg(), new ParseField("error"));
-        PARSER.declareRawObject(optionalConstructorArg(), new ParseField("result"));
+        PARSER.declareRawObject(optionalConstructorArg(), new ParseField("response"));
     }
 
     @Override
@@ -189,9 +194,10 @@ public final class PersistedTaskInfo implements Writeable, ToXContent {
          * Equality of error and result is done by converting them to a map first. Not efficient but ignores field order and spacing
          * differences so perfect for testing.
          */
-        return Objects.equals(task, other.task)
+        return Objects.equals(completed, other.completed)
+                && Objects.equals(task, other.task)
                 && Objects.equals(getErrorAsMap(), other.getErrorAsMap())
-                && Objects.equals(getResultAsMap(), other.getResultAsMap());
+                && Objects.equals(getResponseAsMap(), other.getResponseAsMap());
     }
 
     @Override
@@ -200,7 +206,7 @@ public final class PersistedTaskInfo implements Writeable, ToXContent {
          * Hashing of error and result is done by converting them to a map first. Not efficient but ignores field order and spacing
          * differences so perfect for testing.
          */
-        return Objects.hash(task, getErrorAsMap(), getResultAsMap());
+        return Objects.hash(completed, task, getErrorAsMap(), getResponseAsMap());
     }
 
     private static BytesReference toXContent(ToXContent result) throws IOException {
diff --git a/core/src/main/resources/org/elasticsearch/tasks/task-index-mapping.json b/core/src/main/resources/org/elasticsearch/tasks/task-index-mapping.json
index 30476b1fe3a..262519fd381 100644
--- a/core/src/main/resources/org/elasticsearch/tasks/task-index-mapping.json
+++ b/core/src/main/resources/org/elasticsearch/tasks/task-index-mapping.json
@@ -37,7 +37,7 @@
           }
         }
       },
-      "result" : {
+      "response" : {
         "type" : "object",
         "enabled" : false
       },
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
index 4c720536d0c..6f8e3fda156 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
@@ -70,7 +70,6 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -85,6 +84,7 @@ import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.emptyCollectionOf;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 
@@ -437,8 +437,8 @@ public class TasksIT extends ESIntegTestCase {
     }
 
     public void testListTasksWaitForCompletion() throws Exception {
-        waitForCompletionTestCase(id -> {
-            return client().admin().cluster().prepareListTasks().setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]")
+        waitForCompletionTestCase(randomBoolean(), id -> {
+            return client().admin().cluster().prepareListTasks().setActions(TestTaskPlugin.TestTaskAction.NAME)
                     .setWaitForCompletion(true).execute();
         }, response -> {
             assertThat(response.getNodeFailures(), empty());
@@ -446,25 +446,39 @@ public class TasksIT extends ESIntegTestCase {
         });
     }
 
-    public void testGetTaskWaitForCompletion() throws Exception {
-        waitForCompletionTestCase(id -> {
+    public void testGetTaskWaitForCompletionNoPersist() throws Exception {
+        waitForCompletionTestCase(false, id -> {
             return client().admin().cluster().prepareGetTask(id).setWaitForCompletion(true).execute();
         }, response -> {
-            // Really we're just happy we didn't get any exceptions
             assertNotNull(response.getTask().getTask());
+            assertTrue(response.getTask().isCompleted());
+            // We didn't persist the result so it won't come back when we wait
+            assertNull(response.getTask().getResponse());
+        });
+    }
+
+    public void testGetTaskWaitForCompletionWithPersist() throws Exception {
+        waitForCompletionTestCase(true, id -> {
+            return client().admin().cluster().prepareGetTask(id).setWaitForCompletion(true).execute();
+        }, response -> {
+            assertNotNull(response.getTask().getTask());
+            assertTrue(response.getTask().isCompleted());
+            // We persisted the task so we should get its results
+            assertEquals(0, response.getTask().getResponseAsMap().get("failure_count"));
         });
     }
 
     /**
      * Test wait for completion.
+     * @param persist should the task persist its results
      * @param wait start waiting for a task. Accepts that id of the task to wait for and returns a future waiting for it.
      * @param validator validate the response and return the task ids that were found
      */
-    private <T> void waitForCompletionTestCase(Function<TaskId, ListenableActionFuture<T>> wait, Consumer<T> validator)
+    private <T> void waitForCompletionTestCase(boolean persist, Function<TaskId, ListenableActionFuture<T>> wait, Consumer<T> validator)
             throws Exception {
         // Start blocking test task
         ListenableActionFuture<TestTaskPlugin.NodesResponse> future = TestTaskPlugin.TestTaskAction.INSTANCE.newRequestBuilder(client())
-                .execute();
+                .setShouldPersistResult(persist).execute();
 
         ListenableActionFuture<T> waitResponseFuture;
         TaskId taskId;
@@ -513,7 +527,7 @@ public class TasksIT extends ESIntegTestCase {
     public void testListTasksWaitForTimeout() throws Exception {
         waitForTimeoutTestCase(id -> {
             ListTasksResponse response = client().admin().cluster().prepareListTasks()
-                    .setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]").setWaitForCompletion(true).setTimeout(timeValueMillis(100))
+                    .setActions(TestTaskPlugin.TestTaskAction.NAME).setWaitForCompletion(true).setTimeout(timeValueMillis(100))
                     .get();
             assertThat(response.getNodeFailures(), not(empty()));
             return response.getNodeFailures();
@@ -539,6 +553,9 @@ public class TasksIT extends ESIntegTestCase {
         try {
             TaskId taskId = waitForTestTaskStartOnAllNodes();
 
+            // Wait for the task to start
+            assertBusy(() -> client().admin().cluster().prepareGetTask(taskId).get());
+
             // Spin up a request that should wait for those tasks to finish
             // It will timeout because we haven't unblocked the tasks
             Iterable<? extends Throwable> failures = wait.apply(taskId);
@@ -554,15 +571,18 @@ public class TasksIT extends ESIntegTestCase {
         future.get();
     }
 
+    /**
+     * Wait for the test task to be running on all nodes and return the TaskId of the primary task.
+     */
     private TaskId waitForTestTaskStartOnAllNodes() throws Exception {
-        AtomicReference<TaskId> result = new AtomicReference<>();
         assertBusy(() -> {
             List<TaskInfo> tasks = client().admin().cluster().prepareListTasks().setActions(TestTaskPlugin.TestTaskAction.NAME + "[n]")
                     .get().getTasks();
             assertEquals(internalCluster().size(), tasks.size());
-            result.set(tasks.get(0).getTaskId());
         });
-        return result.get();
+        List<TaskInfo> task = client().admin().cluster().prepareListTasks().setActions(TestTaskPlugin.TestTaskAction.NAME).get().getTasks();
+        assertThat(task, hasSize(1));
+        return task.get(0).getTaskId();
     }
 
     public void testTasksListWaitForNoTask() throws Exception {
@@ -626,7 +646,7 @@ public class TasksIT extends ESIntegTestCase {
         assertEquals(Long.toString(taskInfo.getId()), task.get("id").toString());
 
         @SuppressWarnings("unchecked")
-        Map<String, Object> result = (Map<String, Object>) source.get("result");
+        Map<String, Object> result = (Map<String, Object>) source.get("response");
         assertEquals("0", result.get("failure_count").toString());
 
         assertNull(source.get("failure"));
@@ -647,7 +667,7 @@ public class TasksIT extends ESIntegTestCase {
         assertEquals(1L, searchResponse.getHits().totalHits());
 
         GetTaskResponse getResponse = expectFinishedTask(taskId);
-        assertEquals(result, getResponse.getTask().getResultAsMap());
+        assertEquals(result, getResponse.getTask().getResponseAsMap());
         assertNull(getResponse.getTask().getError());
     }
 
@@ -688,7 +708,7 @@ public class TasksIT extends ESIntegTestCase {
         assertNull(source.get("result"));
 
         GetTaskResponse getResponse = expectFinishedTask(failedTaskId);
-        assertNull(getResponse.getTask().getResult());
+        assertNull(getResponse.getTask().getResponse());
         assertEquals(error, getResponse.getTask().getErrorAsMap());
     }
 
@@ -728,7 +748,7 @@ public class TasksIT extends ESIntegTestCase {
         GetTaskResponse response = expectFinishedTask(new TaskId("fake:1"));
         assertEquals("test", response.getTask().getTask().getAction());
         assertNotNull(response.getTask().getError());
-        assertNull(response.getTask().getResult());
+        assertNull(response.getTask().getResponse());
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/tasks/PersistedTaskInfoTests.java b/core/src/test/java/org/elasticsearch/tasks/PersistedTaskInfoTests.java
index d35d6ce82c2..bfbb2dff4c7 100644
--- a/core/src/test/java/org/elasticsearch/tasks/PersistedTaskInfoTests.java
+++ b/core/src/test/java/org/elasticsearch/tasks/PersistedTaskInfoTests.java
@@ -78,11 +78,11 @@ public class PersistedTaskInfoTests extends ESTestCase {
     private static PersistedTaskInfo randomTaskResult() throws IOException {
         switch (between(0, 2)) {
         case 0:
-            return new PersistedTaskInfo(randomTaskInfo());
+            return new PersistedTaskInfo(randomBoolean(), randomTaskInfo());
         case 1:
             return new PersistedTaskInfo(randomTaskInfo(), new RuntimeException("error"));
         case 2:
-            return new PersistedTaskInfo(randomTaskInfo(), randomTaskActionResult());
+            return new PersistedTaskInfo(randomTaskInfo(), randomTaskResponse());
         default:
             throw new UnsupportedOperationException("Unsupported random TaskResult constructor");
         }
@@ -117,7 +117,7 @@ public class PersistedTaskInfoTests extends ESTestCase {
         }
     }
 
-    private static ToXContent randomTaskActionResult() {
+    private static ToXContent randomTaskResponse() {
         Map<String, String> result = new TreeMap<>();
         int fields = between(0, 10);
         for (int f = 0; f < fields; f++) {
@@ -126,7 +126,7 @@ public class PersistedTaskInfoTests extends ESTestCase {
         return new ToXContent() {
             @Override
             public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-                // Results in Elasticsearch never output a leading startObject. There isn't really a good reason, they just don't.
+                // Responses in Elasticsearch never output a leading startObject. There isn't really a good reason, they just don't.
                 for (Map.Entry<String, String> entry : result.entrySet()) {
                     builder.field(entry.getKey(), entry.getValue());
                 }
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml
index 13a98921c05..85cd6143d69 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml
@@ -76,6 +76,17 @@
   # The task will be in the response even if it finished before we got here
   # because of task persistence.
   - is_true: task
+  - is_false: response.timed_out
+  - match: {response.deleted: 1}
+  - is_false: response.created
+  - is_false: response.updated
+  - match: {response.version_conflicts: 0}
+  - match: {response.batches: 1}
+  - match: {response.failures: []}
+  - match: {response.noops: 0}
+  - match: {response.throttled_millis: 0}
+  - gte: { response.took: 0 }
+  - is_false: response.task
 
 ---
 "Response for version conflict":
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml
index b54d8bd0e95..addbebd44a7 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml
@@ -100,6 +100,15 @@
   # The task will be in the response even if it finished before we got here
   # because of task persistence.
   - is_true: task
+  - match: {response.created: 1}
+  - match: {response.updated: 0}
+  - match: {response.version_conflicts: 0}
+  - match: {response.batches: 1}
+  - match: {response.failures: []}
+  - match: {response.throttled_millis: 0}
+  - gte: { response.took: 0 }
+  - is_false: response.task
+  - is_false: response.deleted
 
 ---
 "Response format for version conflict":
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yaml
index 8ba4013caa3..62c8677921d 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yaml
@@ -60,6 +60,18 @@
   # The task will be in the response even if it finished before we got here
   # because of task persistence.
   - is_true: task
+  - is_false: response.timed_out
+  - match: {response.updated: 1}
+  - match: {response.version_conflicts: 0}
+  - match: {response.batches: 1}
+  - match: {response.failures: []}
+  - match: {response.noops: 0}
+  - match: {response.throttled_millis: 0}
+  - gte: { response.took: 0 }
+  # Update by query can't create
+  - is_false: response.created
+  - is_false: response.task
+  - is_false: response.deleted
 
 ---
 "Response for version conflict":

From e9230dd889c70cdd6650a116ac648f29117638c9 Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Tue, 21 Jun 2016 21:08:43 +0200
Subject: [PATCH 1005/1311] RejectedExecutionException !=
 EdRejectedExecutionException

---
 .../elasticsearch/discovery/zen/fd/MasterFaultDetection.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
index 807f8067490..86c9ea86a80 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
@@ -32,6 +32,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.BaseTransportResponseHandler;
 import org.elasticsearch.transport.ConnectTransportException;
@@ -45,7 +46,6 @@ import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
 import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
@@ -203,7 +203,7 @@ public class MasterFaultDetection extends FaultDetection {
                         listener.onMasterFailure(masterNode, cause, reason);
                     }
                 });
-            } catch (RejectedExecutionException e) {
+            } catch (EsRejectedExecutionException e) {
                 logger.error("master failure notification was rejected, it's highly likely the node is shutting down", e);
             }
             stop("master failure, " + reason);

From c80e8376060320c4b5b1fd3de900f44ec61ca5ba Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Tue, 21 Jun 2016 21:25:01 +0200
Subject: [PATCH 1006/1311] Beef up Translog testing with random channel
 exceptions (#18997)

Today we only throw random exceptions on the translog writer. This commit
extends it to also throw exceptions during checkpoint writing etc to test
if the correct flags are provided to open method etc.
---
 .../index/translog/ChannelFactory.java        | 37 ++++++++++
 .../index/translog/Checkpoint.java            |  4 +-
 .../index/translog/Translog.java              |  6 +-
 .../index/translog/TranslogWriter.java        | 24 +++----
 .../index/translog/TranslogTests.java         | 70 ++++++++++++++-----
 5 files changed, 103 insertions(+), 38 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/index/translog/ChannelFactory.java

diff --git a/core/src/main/java/org/elasticsearch/index/translog/ChannelFactory.java b/core/src/main/java/org/elasticsearch/index/translog/ChannelFactory.java
new file mode 100644
index 00000000000..7e1dcec14df
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/translog/ChannelFactory.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.translog;
+
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.nio.file.OpenOption;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+
+/**
+ * only for testing until we have a disk-full FileSystem
+ */
+@FunctionalInterface
+interface ChannelFactory {
+    default FileChannel open(Path path) throws IOException {
+        return open(path, StandardOpenOption.WRITE, StandardOpenOption.READ, StandardOpenOption.CREATE_NEW);
+    }
+
+    FileChannel open(Path path, OpenOption... options) throws IOException;
+}
diff --git a/core/src/main/java/org/elasticsearch/index/translog/Checkpoint.java b/core/src/main/java/org/elasticsearch/index/translog/Checkpoint.java
index 54ba8638eb2..f630ba3faba 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/Checkpoint.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/Checkpoint.java
@@ -82,8 +82,8 @@ class Checkpoint {
         }
     }
 
-    public static void write(Path checkpointFile, Checkpoint checkpoint, OpenOption... options) throws IOException {
-        try (FileChannel channel = FileChannel.open(checkpointFile, options)) {
+    public static void write(ChannelFactory factory, Path checkpointFile, Checkpoint checkpoint, OpenOption... options) throws IOException {
+        try (FileChannel channel = factory.open(checkpointFile, options)) {
             checkpoint.write(channel);
             channel.force(false);
         }
diff --git a/core/src/main/java/org/elasticsearch/index/translog/Translog.java b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
index 57847972e42..788ee547f78 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/Translog.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
@@ -200,7 +200,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
                 Files.createDirectories(location);
                 final long generation = 1;
                 Checkpoint checkpoint = new Checkpoint(0, 0, generation);
-                Checkpoint.write(location.resolve(CHECKPOINT_FILE_NAME), checkpoint, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW);
+                Checkpoint.write(getChannelFactory(), location.resolve(CHECKPOINT_FILE_NAME), checkpoint, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW);
                 current = createWriter(generation);
                 this.lastCommittedTranslogFileGeneration = NOT_SET_GENERATION;
 
@@ -1313,8 +1313,8 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
         return outstandingViews.size();
     }
 
-    TranslogWriter.ChannelFactory getChannelFactory() {
-        return TranslogWriter.ChannelFactory.DEFAULT;
+    ChannelFactory getChannelFactory() {
+        return FileChannel::open;
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java b/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
index b2c0cc88cf9..59135684e02 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
@@ -49,6 +49,7 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
     public static final int VERSION = VERSION_CHECKPOINTS;
 
     private final ShardId shardId;
+    private final ChannelFactory channelFactory;
     /* the offset in bytes that was written when the file was last synced*/
     private volatile long lastSyncedOffset;
     /* the number of translog operations written to this file */
@@ -64,9 +65,10 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
     // lock order synchronized(syncLock) -> synchronized(this)
     private final Object syncLock = new Object();
 
-    public TranslogWriter(ShardId shardId, long generation, FileChannel channel, Path path, ByteSizeValue bufferSize) throws IOException {
+    public TranslogWriter(ChannelFactory channelFactory, ShardId shardId, long generation, FileChannel channel, Path path, ByteSizeValue bufferSize) throws IOException {
         super(generation, channel, path, channel.position());
         this.shardId = shardId;
+        this.channelFactory = channelFactory;
         this.outputStream = new BufferedChannelOutputStream(java.nio.channels.Channels.newOutputStream(channel), bufferSize.bytesAsInt());
         this.lastSyncedOffset = channel.position();
         totalOffset = lastSyncedOffset;
@@ -92,8 +94,8 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
             out.writeInt(ref.length);
             out.writeBytes(ref.bytes, ref.offset, ref.length);
             channel.force(true);
-            writeCheckpoint(headerLength, 0, file.getParent(), fileGeneration, StandardOpenOption.WRITE);
-            final TranslogWriter writer = new TranslogWriter(shardId, fileGeneration, channel, file, bufferSize);
+            writeCheckpoint(channelFactory, headerLength, 0, file.getParent(), fileGeneration);
+            final TranslogWriter writer = new TranslogWriter(channelFactory, shardId, fileGeneration, channel, file, bufferSize);
             return writer;
         } catch (Throwable throwable) {
             // if we fail to bake the file-generation into the checkpoint we stick with the file and once we recover and that
@@ -254,7 +256,7 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
                     // we can continue writing to the buffer etc.
                     try {
                         channel.force(false);
-                        writeCheckpoint(offsetToSync, opsCounter, path.getParent(), generation, StandardOpenOption.WRITE);
+                        writeCheckpoint(channelFactory, offsetToSync, opsCounter, path.getParent(), generation);
                     } catch (Throwable ex) {
                         closeWithTragicEvent(ex);
                         throw ex;
@@ -286,20 +288,10 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
         Channels.readFromFileChannelWithEofException(channel, position, targetBuffer);
     }
 
-    private static void writeCheckpoint(long syncPosition, int numOperations, Path translogFile, long generation, OpenOption... options) throws IOException {
+    private static void writeCheckpoint(ChannelFactory channelFactory, long syncPosition, int numOperations, Path translogFile, long generation) throws IOException {
         final Path checkpointFile = translogFile.resolve(Translog.CHECKPOINT_FILE_NAME);
         Checkpoint checkpoint = new Checkpoint(syncPosition, numOperations, generation);
-        Checkpoint.write(checkpointFile, checkpoint, options);
-    }
-
-    static class ChannelFactory {
-
-        static final ChannelFactory DEFAULT = new ChannelFactory();
-
-        // only for testing until we have a disk-full FileSystem
-        public FileChannel open(Path file) throws IOException {
-            return FileChannel.open(file, StandardOpenOption.WRITE, StandardOpenOption.READ, StandardOpenOption.CREATE_NEW);
-        }
+        Checkpoint.write(channelFactory::open, checkpointFile, checkpoint, StandardOpenOption.WRITE);
     }
 
     protected final void ensureOpen() {
diff --git a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
index 889897e21f6..0c6ee4117cb 100644
--- a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
+++ b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
@@ -60,7 +60,9 @@ import java.nio.charset.Charset;
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.Files;
 import java.nio.file.InvalidPathException;
+import java.nio.file.OpenOption;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -1125,13 +1127,13 @@ public class TranslogTests extends ESTestCase {
         Path ckp = config.getTranslogPath().resolve(Translog.CHECKPOINT_FILE_NAME);
         Checkpoint read = Checkpoint.read(ckp);
         Checkpoint corrupted = new Checkpoint(0, 0, 0);
-        Checkpoint.write(config.getTranslogPath().resolve(Translog.getCommitCheckpointFileName(read.generation)), corrupted, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW);
+        Checkpoint.write(FileChannel::open, config.getTranslogPath().resolve(Translog.getCommitCheckpointFileName(read.generation)), corrupted, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW);
         try (Translog translog = new Translog(config, translogGeneration)) {
             fail("corrupted");
         } catch (IllegalStateException ex) {
             assertEquals(ex.getMessage(), "Checkpoint file translog-2.ckp already exists but has corrupted content expected: Checkpoint{offset=2683, numOps=55, translogFileGeneration= 2} but got: Checkpoint{offset=0, numOps=0, translogFileGeneration= 0}");
         }
-        Checkpoint.write(config.getTranslogPath().resolve(Translog.getCommitCheckpointFileName(read.generation)), read, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING);
+        Checkpoint.write(FileChannel::open, config.getTranslogPath().resolve(Translog.getCommitCheckpointFileName(read.generation)), read, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING);
         try (Translog translog = new Translog(config, translogGeneration)) {
             assertNotNull(translogGeneration);
             assertEquals("lastCommitted must be 2 less than current - we never finished the commit", translogGeneration.translogFileGeneration + 2, translog.currentFileGeneration());
@@ -1564,22 +1566,20 @@ public class TranslogTests extends ESTestCase {
     private Translog getFailableTranslog(final FailSwitch fail, final TranslogConfig config, final boolean paritalWrites, final boolean throwUnknownException, Translog.TranslogGeneration generation) throws IOException {
         return new Translog(config, generation) {
             @Override
-            TranslogWriter.ChannelFactory getChannelFactory() {
-                final TranslogWriter.ChannelFactory factory = super.getChannelFactory();
+            ChannelFactory getChannelFactory() {
+                final ChannelFactory factory = super.getChannelFactory();
 
-                return new TranslogWriter.ChannelFactory() {
-                    @Override
-                    public FileChannel open(Path file) throws IOException {
-                        FileChannel channel = factory.open(file);
-                        boolean success = false;
-                        try {
-                            ThrowingFileChannel throwingFileChannel = new ThrowingFileChannel(fail, paritalWrites, throwUnknownException, channel);
-                            success = true;
-                            return throwingFileChannel;
-                        } finally {
-                            if (success == false) {
-                                IOUtils.closeWhileHandlingException(channel);
-                            }
+                return (file, openOption) -> {
+                    FileChannel channel = factory.open(file, openOption);
+                    boolean success = false;
+                    try {
+                        final boolean isCkpFile = file.getFileName().toString().endsWith(".ckp"); // don't do partial writes for checkpoints we rely on the fact that the 20bytes are written as an atomic operation
+                        ThrowingFileChannel throwingFileChannel = new ThrowingFileChannel(fail, isCkpFile ? false : paritalWrites, throwUnknownException, channel);
+                        success = true;
+                        return throwingFileChannel;
+                    } finally {
+                        if (success == false) {
+                            IOUtils.closeWhileHandlingException(channel);
                         }
                     }
                 };
@@ -1840,11 +1840,18 @@ public class TranslogTests extends ESTestCase {
                 } catch (IOException ex) {
                     assertEquals(ex.getMessage(), "__FAKE__ no space left on device");
                 } finally {
+                    Checkpoint checkpoint = failableTLog.readCheckpoint();
+                    if (checkpoint.numOps == unsynced.size() + syncedDocs.size()) {
+                        syncedDocs.addAll(unsynced); // failed in fsync but got fully written
+                        unsynced.clear();
+                    }
                     generation = failableTLog.getGeneration();
                     IOUtils.closeWhileHandlingException(failableTLog);
                 }
             } catch (TranslogException | MockDirectoryWrapper.FakeIOException ex) {
                 // failed - that's ok, we didn't even create it
+            } catch (IOException ex) {
+                assertEquals(ex.getMessage(), "__FAKE__ no space left on device");
             }
             // now randomly open this failing tlog again just to make sure we can also recover from failing during recovery
             if (randomBoolean()) {
@@ -1852,9 +1859,12 @@ public class TranslogTests extends ESTestCase {
                     IOUtils.close(getFailableTranslog(fail, config, randomBoolean(), false, generation));
                 } catch (TranslogException | MockDirectoryWrapper.FakeIOException ex) {
                     // failed - that's ok, we didn't even create it
+                } catch (IOException ex) {
+                    assertEquals(ex.getMessage(), "__FAKE__ no space left on device");
                 }
             }
 
+            fail.failNever(); // we don't wanna fail here but we might since we write a new checkpoint and create a new tlog file
             try (Translog translog = new Translog(config, generation)) {
                 Translog.Snapshot snapshot = translog.newSnapshot();
                 assertEquals(syncedDocs.size(), snapshot.totalOperations());
@@ -1866,4 +1876,30 @@ public class TranslogTests extends ESTestCase {
             }
         }
     }
+
+    public void testCheckpointOnDiskFull() throws IOException {
+        Checkpoint checkpoint = new Checkpoint(randomLong(), randomInt(), randomLong());
+        Path tempDir = createTempDir();
+        Checkpoint.write(FileChannel::open, tempDir.resolve("foo.cpk"), checkpoint, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW);
+        Checkpoint checkpoint2 = new Checkpoint(randomLong(), randomInt(), randomLong());
+        try {
+            Checkpoint.write((p, o) -> {
+                if (randomBoolean()) {
+                    throw new MockDirectoryWrapper.FakeIOException();
+                }
+                FileChannel open = FileChannel.open(p, o);
+                FailSwitch failSwitch = new FailSwitch();
+                failSwitch.failNever(); // don't fail in the ctor
+                ThrowingFileChannel channel = new ThrowingFileChannel(failSwitch, false, false, open);
+                failSwitch.failAlways();
+                return channel;
+
+            }, tempDir.resolve("foo.cpk"), checkpoint2, StandardOpenOption.WRITE);
+            fail("should have failed earlier");
+        } catch (MockDirectoryWrapper.FakeIOException ex) {
+            //fine
+        }
+        Checkpoint read = Checkpoint.read(tempDir.resolve("foo.cpk"));
+        assertEquals(read, checkpoint);
+    }
 }

From d2c823e4ccac8b44c794d13f28c9c1867a72830b Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Tue, 21 Jun 2016 12:32:10 -0700
Subject: [PATCH 1007/1311] Add initializers to Painless for arrays, lists, and
 maps.

---
 .../src/main/antlr/PainlessParser.g4          |   45 +-
 .../painless/antlr/PainlessParser.java        | 1512 +++++++++++------
 .../antlr/PainlessParserBaseVisitor.java      |   49 +
 .../painless/antlr/PainlessParserVisitor.java |   46 +
 .../elasticsearch/painless/antlr/Walker.java  |  128 +-
 .../painless/node/EListInit.java              |   97 ++
 .../elasticsearch/painless/node/EMapInit.java |  121 ++
 .../painless/node/LNewArray.java              |   39 +-
 .../painless/InitializerTests.java            |  142 ++
 9 files changed, 1577 insertions(+), 602 deletions(-)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
 create mode 100644 modules/lang-painless/src/test/java/org/elasticsearch/painless/InitializerTests.java

diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index 037d8fd7bbd..10471056acd 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -126,21 +126,23 @@ expression returns [boolean s = true]
 // processing a variable/method chain.  This prevents the chain
 // from being applied to rules where it wouldn't be allowed.
 unary[boolean c] returns [boolean s = true]
-    : { !$c }? ( INCR | DECR ) chain[true]                                  # pre
-    | { !$c }? chain[true] (INCR | DECR )                                   # post
-    | { !$c }? chain[false]                                                 # read
-    | { !$c }? ( OCTAL | HEX | INTEGER | DECIMAL )          { $s = false; } # numeric
-    | { !$c }? TRUE                                         { $s = false; } # true
-    | { !$c }? FALSE                                        { $s = false; } # false
-    | { !$c }? NULL                                         { $s = false; } # null
-    | { !$c }? ( BOOLNOT | BWNOT | ADD | SUB ) unary[false]                 # operator
-    |          LP decltype RP unary[$c]                                     # cast
+    : { !$c }? ( INCR | DECR ) chain[true]                                   # pre
+    | { !$c }? chain[true] (INCR | DECR )                                    # post
+    | { !$c }? chain[false]                                                  # read
+    | { !$c }? ( OCTAL | HEX | INTEGER | DECIMAL )           { $s = false; } # numeric
+    | { !$c }? TRUE                                          { $s = false; } # true
+    | { !$c }? FALSE                                         { $s = false; } # false
+    | { !$c }? NULL                                          { $s = false; } # null
+    | { !$c }? listinitializer                               { $s = false; } # listinit
+    | { !$c }? mapinitializer                                { $s = false; } # mapinit
+    | { !$c }? ( BOOLNOT | BWNOT | ADD | SUB ) unary[false]                  # operator
+    |          LP decltype RP unary[$c]                                      # cast
     ;
 
 chain[boolean c]
-    : p = primary[$c] secondary[$p.s]*                             # dynamic
-    | decltype dot secondary[true]*                                # static
-    | NEW TYPE (LBRACE expression RBRACE)+ (dot secondary[true]*)? # newarray
+    : p = primary[$c] secondary[$p.s]* # dynamic
+    | decltype dot secondary[true]*    # static
+    | arrayinitializer                 # newarray
     ;
 
 primary[boolean c] returns [boolean s = true]
@@ -213,3 +215,22 @@ capturingFuncref
 localFuncref
     : THIS REF ID
     ;
+
+arrayinitializer
+    : NEW TYPE (LBRACE expression RBRACE)+ (dot secondary[true]*)?                          # newstandardarray
+    | NEW TYPE LBRACE RBRACE LBRACK ( expression ( COMMA expression )* )? SEMICOLON? RBRACK # newinitializedarray
+    ;
+
+listinitializer
+    : LBRACE expression ( COMMA expression)* RBRACE
+    | LBRACE RBRACE
+    ;
+
+mapinitializer
+    : LBRACE maptoken ( COMMA maptoken )* RBRACE
+    | LBRACE COLON RBRACE
+    ;
+
+maptoken
+    : expression COLON expression
+    ;
\ No newline at end of file
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
index 39d1ca3d188..f6ace39be63 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
@@ -35,13 +35,15 @@ class PainlessParser extends Parser {
     RULE_chain = 16, RULE_primary = 17, RULE_secondary = 18, RULE_dot = 19, 
     RULE_brace = 20, RULE_arguments = 21, RULE_argument = 22, RULE_lambda = 23, 
     RULE_lamtype = 24, RULE_funcref = 25, RULE_classFuncref = 26, RULE_constructorFuncref = 27, 
-    RULE_capturingFuncref = 28, RULE_localFuncref = 29;
+    RULE_capturingFuncref = 28, RULE_localFuncref = 29, RULE_arrayinitializer = 30, 
+    RULE_listinitializer = 31, RULE_mapinitializer = 32, RULE_maptoken = 33;
   public static final String[] ruleNames = {
     "source", "function", "parameters", "statement", "trailer", "block", "empty", 
     "initializer", "afterthought", "declaration", "decltype", "declvar", "trap", 
     "delimiter", "expression", "unary", "chain", "primary", "secondary", "dot", 
     "brace", "arguments", "argument", "lambda", "lamtype", "funcref", "classFuncref", 
-    "constructorFuncref", "capturingFuncref", "localFuncref"
+    "constructorFuncref", "capturingFuncref", "localFuncref", "arrayinitializer", 
+    "listinitializer", "mapinitializer", "maptoken"
   };
 
   private static final String[] _LITERAL_NAMES = {
@@ -148,39 +150,39 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(63);
+      setState(71);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,0,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(60);
+          setState(68);
           function();
           }
           } 
         }
-        setState(65);
+        setState(73);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,0,_ctx);
       }
-      setState(69);
+      setState(77);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,1,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(66);
+          setState(74);
           statement();
           }
           } 
         }
-        setState(71);
+        setState(79);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,1,_ctx);
       }
-      setState(72);
+      setState(80);
       match(EOF);
       }
     }
@@ -223,13 +225,13 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(74);
+      setState(82);
       decltype();
-      setState(75);
+      setState(83);
       match(ID);
-      setState(76);
+      setState(84);
       parameters();
-      setState(77);
+      setState(85);
       block();
       }
     }
@@ -279,38 +281,38 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(79);
+      setState(87);
       match(LP);
-      setState(91);
+      setState(99);
       _la = _input.LA(1);
       if (_la==TYPE) {
         {
-        setState(80);
-        decltype();
-        setState(81);
-        match(ID);
         setState(88);
+        decltype();
+        setState(89);
+        match(ID);
+        setState(96);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(82);
+          setState(90);
           match(COMMA);
-          setState(83);
+          setState(91);
           decltype();
-          setState(84);
+          setState(92);
           match(ID);
           }
           }
-          setState(90);
+          setState(98);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
       }
 
-      setState(93);
+      setState(101);
       match(RP);
       }
     }
@@ -556,35 +558,35 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 6, RULE_statement);
     try {
       int _alt;
-      setState(173);
+      setState(181);
       switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) {
       case 1:
         _localctx = new IfContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(95);
-        match(IF);
-        setState(96);
-        match(LP);
-        setState(97);
-        expression(0);
-        setState(98);
-        match(RP);
-        setState(99);
-        trailer();
         setState(103);
+        match(IF);
+        setState(104);
+        match(LP);
+        setState(105);
+        expression(0);
+        setState(106);
+        match(RP);
+        setState(107);
+        trailer();
+        setState(111);
         switch ( getInterpreter().adaptivePredict(_input,4,_ctx) ) {
         case 1:
           {
-          setState(100);
+          setState(108);
           match(ELSE);
-          setState(101);
+          setState(109);
           trailer();
           }
           break;
         case 2:
           {
-          setState(102);
+          setState(110);
           if (!( _input.LA(1) != ELSE )) throw new FailedPredicateException(this, " _input.LA(1) != ELSE ");
           }
           break;
@@ -595,25 +597,25 @@ class PainlessParser extends Parser {
         _localctx = new WhileContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(105);
+        setState(113);
         match(WHILE);
-        setState(106);
+        setState(114);
         match(LP);
-        setState(107);
+        setState(115);
         expression(0);
-        setState(108);
+        setState(116);
         match(RP);
-        setState(111);
+        setState(119);
         switch ( getInterpreter().adaptivePredict(_input,5,_ctx) ) {
         case 1:
           {
-          setState(109);
+          setState(117);
           trailer();
           }
           break;
         case 2:
           {
-          setState(110);
+          setState(118);
           empty();
           }
           break;
@@ -624,19 +626,19 @@ class PainlessParser extends Parser {
         _localctx = new DoContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(113);
+        setState(121);
         match(DO);
-        setState(114);
+        setState(122);
         block();
-        setState(115);
+        setState(123);
         match(WHILE);
-        setState(116);
+        setState(124);
         match(LP);
-        setState(117);
+        setState(125);
         expression(0);
-        setState(118);
+        setState(126);
         match(RP);
-        setState(119);
+        setState(127);
         delimiter();
         }
         break;
@@ -644,54 +646,54 @@ class PainlessParser extends Parser {
         _localctx = new ForContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(121);
+        setState(129);
         match(FOR);
-        setState(122);
+        setState(130);
         match(LP);
-        setState(124);
+        setState(132);
         switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) {
         case 1:
           {
-          setState(123);
+          setState(131);
           initializer();
           }
           break;
         }
-        setState(126);
+        setState(134);
         match(SEMICOLON);
-        setState(128);
+        setState(136);
         switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) {
         case 1:
           {
-          setState(127);
+          setState(135);
           expression(0);
           }
           break;
         }
-        setState(130);
+        setState(138);
         match(SEMICOLON);
-        setState(132);
+        setState(140);
         switch ( getInterpreter().adaptivePredict(_input,8,_ctx) ) {
         case 1:
           {
-          setState(131);
+          setState(139);
           afterthought();
           }
           break;
         }
-        setState(134);
+        setState(142);
         match(RP);
-        setState(137);
+        setState(145);
         switch ( getInterpreter().adaptivePredict(_input,9,_ctx) ) {
         case 1:
           {
-          setState(135);
+          setState(143);
           trailer();
           }
           break;
         case 2:
           {
-          setState(136);
+          setState(144);
           empty();
           }
           break;
@@ -702,21 +704,21 @@ class PainlessParser extends Parser {
         _localctx = new EachContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(139);
+        setState(147);
         match(FOR);
-        setState(140);
+        setState(148);
         match(LP);
-        setState(141);
+        setState(149);
         decltype();
-        setState(142);
+        setState(150);
         match(ID);
-        setState(143);
+        setState(151);
         match(COLON);
-        setState(144);
+        setState(152);
         expression(0);
-        setState(145);
+        setState(153);
         match(RP);
-        setState(146);
+        setState(154);
         trailer();
         }
         break;
@@ -724,9 +726,9 @@ class PainlessParser extends Parser {
         _localctx = new DeclContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(148);
+        setState(156);
         declaration();
-        setState(149);
+        setState(157);
         delimiter();
         }
         break;
@@ -734,9 +736,9 @@ class PainlessParser extends Parser {
         _localctx = new ContinueContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(151);
+        setState(159);
         match(CONTINUE);
-        setState(152);
+        setState(160);
         delimiter();
         }
         break;
@@ -744,9 +746,9 @@ class PainlessParser extends Parser {
         _localctx = new BreakContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(153);
+        setState(161);
         match(BREAK);
-        setState(154);
+        setState(162);
         delimiter();
         }
         break;
@@ -754,11 +756,11 @@ class PainlessParser extends Parser {
         _localctx = new ReturnContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(155);
+        setState(163);
         match(RETURN);
-        setState(156);
+        setState(164);
         expression(0);
-        setState(157);
+        setState(165);
         delimiter();
         }
         break;
@@ -766,11 +768,11 @@ class PainlessParser extends Parser {
         _localctx = new TryContext(_localctx);
         enterOuterAlt(_localctx, 10);
         {
-        setState(159);
+        setState(167);
         match(TRY);
-        setState(160);
+        setState(168);
         block();
-        setState(162); 
+        setState(170); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -778,7 +780,7 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(161);
+            setState(169);
             trap();
             }
             }
@@ -786,7 +788,7 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(164); 
+          setState(172); 
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,10,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
@@ -796,11 +798,11 @@ class PainlessParser extends Parser {
         _localctx = new ThrowContext(_localctx);
         enterOuterAlt(_localctx, 11);
         {
-        setState(166);
+        setState(174);
         match(THROW);
-        setState(167);
+        setState(175);
         expression(0);
-        setState(168);
+        setState(176);
         delimiter();
         }
         break;
@@ -808,9 +810,9 @@ class PainlessParser extends Parser {
         _localctx = new ExprContext(_localctx);
         enterOuterAlt(_localctx, 12);
         {
-        setState(170);
+        setState(178);
         expression(0);
-        setState(171);
+        setState(179);
         delimiter();
         }
         break;
@@ -849,19 +851,19 @@ class PainlessParser extends Parser {
     TrailerContext _localctx = new TrailerContext(_ctx, getState());
     enterRule(_localctx, 8, RULE_trailer);
     try {
-      setState(177);
+      setState(185);
       switch ( getInterpreter().adaptivePredict(_input,12,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(175);
+        setState(183);
         block();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(176);
+        setState(184);
         statement();
         }
         break;
@@ -905,25 +907,25 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(179);
+      setState(187);
       match(LBRACK);
-      setState(183);
+      setState(191);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,13,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(180);
+          setState(188);
           statement();
           }
           } 
         }
-        setState(185);
+        setState(193);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,13,_ctx);
       }
-      setState(186);
+      setState(194);
       match(RBRACK);
       }
     }
@@ -957,7 +959,7 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(188);
+      setState(196);
       match(SEMICOLON);
       }
     }
@@ -994,19 +996,19 @@ class PainlessParser extends Parser {
     InitializerContext _localctx = new InitializerContext(_ctx, getState());
     enterRule(_localctx, 14, RULE_initializer);
     try {
-      setState(192);
+      setState(200);
       switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(190);
+        setState(198);
         declaration();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(191);
+        setState(199);
         expression(0);
         }
         break;
@@ -1044,7 +1046,7 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(194);
+      setState(202);
       expression(0);
       }
     }
@@ -1091,23 +1093,23 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(196);
+      setState(204);
       decltype();
-      setState(197);
+      setState(205);
       declvar();
-      setState(202);
+      setState(210);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==COMMA) {
         {
         {
-        setState(198);
+        setState(206);
         match(COMMA);
-        setState(199);
+        setState(207);
         declvar();
         }
         }
-        setState(204);
+        setState(212);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
@@ -1152,21 +1154,21 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(205);
+      setState(213);
       match(TYPE);
-      setState(210);
+      setState(218);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==LBRACE) {
         {
         {
-        setState(206);
+        setState(214);
         match(LBRACE);
-        setState(207);
+        setState(215);
         match(RBRACE);
         }
         }
-        setState(212);
+        setState(220);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
@@ -1207,15 +1209,15 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(213);
+      setState(221);
       match(ID);
-      setState(216);
+      setState(224);
       _la = _input.LA(1);
       if (_la==ASSIGN) {
         {
-        setState(214);
+        setState(222);
         match(ASSIGN);
-        setState(215);
+        setState(223);
         expression(0);
         }
       }
@@ -1259,17 +1261,17 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(218);
+      setState(226);
       match(CATCH);
-      setState(219);
+      setState(227);
       match(LP);
-      setState(220);
+      setState(228);
       match(TYPE);
-      setState(221);
+      setState(229);
       match(ID);
-      setState(222);
+      setState(230);
       match(RP);
-      setState(223);
+      setState(231);
       block();
       }
     }
@@ -1305,7 +1307,7 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(225);
+      setState(233);
       _la = _input.LA(1);
       if ( !(_la==EOF || _la==SEMICOLON) ) {
       _errHandler.recoverInline(this);
@@ -1476,7 +1478,7 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(236);
+      setState(244);
       switch ( getInterpreter().adaptivePredict(_input,18,_ctx) ) {
       case 1:
         {
@@ -1484,16 +1486,16 @@ class PainlessParser extends Parser {
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(228);
+        setState(236);
         chain(true);
-        setState(229);
+        setState(237);
         _la = _input.LA(1);
         if ( !(((((_la - 56)) & ~0x3f) == 0 && ((1L << (_la - 56)) & ((1L << (ASSIGN - 56)) | (1L << (AADD - 56)) | (1L << (ASUB - 56)) | (1L << (AMUL - 56)) | (1L << (ADIV - 56)) | (1L << (AREM - 56)) | (1L << (AAND - 56)) | (1L << (AXOR - 56)) | (1L << (AOR - 56)) | (1L << (ALSH - 56)) | (1L << (ARSH - 56)) | (1L << (AUSH - 56)))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(230);
+        setState(238);
         expression(1);
          ((AssignmentContext)_localctx).s =  false; 
         }
@@ -1503,14 +1505,14 @@ class PainlessParser extends Parser {
         _localctx = new SingleContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(233);
+        setState(241);
         ((SingleContext)_localctx).u = unary(false);
          ((SingleContext)_localctx).s =  ((SingleContext)_localctx).u.s; 
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(302);
+      setState(310);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -1518,22 +1520,22 @@ class PainlessParser extends Parser {
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(300);
+          setState(308);
           switch ( getInterpreter().adaptivePredict(_input,19,_ctx) ) {
           case 1:
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(238);
+            setState(246);
             if (!(precpred(_ctx, 13))) throw new FailedPredicateException(this, "precpred(_ctx, 13)");
-            setState(239);
+            setState(247);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << MUL) | (1L << DIV) | (1L << REM))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(240);
+            setState(248);
             expression(14);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1542,16 +1544,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(243);
+            setState(251);
             if (!(precpred(_ctx, 12))) throw new FailedPredicateException(this, "precpred(_ctx, 12)");
-            setState(244);
+            setState(252);
             _la = _input.LA(1);
             if ( !(_la==ADD || _la==SUB) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(245);
+            setState(253);
             expression(13);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1560,16 +1562,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(248);
+            setState(256);
             if (!(precpred(_ctx, 11))) throw new FailedPredicateException(this, "precpred(_ctx, 11)");
-            setState(249);
+            setState(257);
             _la = _input.LA(1);
             if ( !(_la==FIND || _la==MATCH) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(250);
+            setState(258);
             expression(12);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1578,16 +1580,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(253);
+            setState(261);
             if (!(precpred(_ctx, 10))) throw new FailedPredicateException(this, "precpred(_ctx, 10)");
-            setState(254);
+            setState(262);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LSH) | (1L << RSH) | (1L << USH))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(255);
+            setState(263);
             expression(11);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1596,16 +1598,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(258);
+            setState(266);
             if (!(precpred(_ctx, 9))) throw new FailedPredicateException(this, "precpred(_ctx, 9)");
-            setState(259);
+            setState(267);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LT) | (1L << LTE) | (1L << GT) | (1L << GTE))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(260);
+            setState(268);
             expression(10);
              ((CompContext)_localctx).s =  false; 
             }
@@ -1614,16 +1616,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(263);
+            setState(271);
             if (!(precpred(_ctx, 8))) throw new FailedPredicateException(this, "precpred(_ctx, 8)");
-            setState(264);
+            setState(272);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << EQ) | (1L << EQR) | (1L << NE) | (1L << NER))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(265);
+            setState(273);
             expression(9);
              ((CompContext)_localctx).s =  false; 
             }
@@ -1632,11 +1634,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(268);
+            setState(276);
             if (!(precpred(_ctx, 7))) throw new FailedPredicateException(this, "precpred(_ctx, 7)");
-            setState(269);
+            setState(277);
             match(BWAND);
-            setState(270);
+            setState(278);
             expression(8);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1645,11 +1647,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(273);
+            setState(281);
             if (!(precpred(_ctx, 6))) throw new FailedPredicateException(this, "precpred(_ctx, 6)");
-            setState(274);
+            setState(282);
             match(XOR);
-            setState(275);
+            setState(283);
             expression(7);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1658,11 +1660,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(278);
+            setState(286);
             if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)");
-            setState(279);
+            setState(287);
             match(BWOR);
-            setState(280);
+            setState(288);
             expression(6);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1671,11 +1673,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(283);
+            setState(291);
             if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)");
-            setState(284);
+            setState(292);
             match(BOOLAND);
-            setState(285);
+            setState(293);
             expression(5);
              ((BoolContext)_localctx).s =  false; 
             }
@@ -1684,11 +1686,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(288);
+            setState(296);
             if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)");
-            setState(289);
+            setState(297);
             match(BOOLOR);
-            setState(290);
+            setState(298);
             expression(4);
              ((BoolContext)_localctx).s =  false; 
             }
@@ -1697,15 +1699,15 @@ class PainlessParser extends Parser {
             {
             _localctx = new ConditionalContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(293);
+            setState(301);
             if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
-            setState(294);
+            setState(302);
             match(COND);
-            setState(295);
+            setState(303);
             ((ConditionalContext)_localctx).e0 = expression(0);
-            setState(296);
+            setState(304);
             match(COLON);
-            setState(297);
+            setState(305);
             ((ConditionalContext)_localctx).e1 = expression(2);
              ((ConditionalContext)_localctx).s =  ((ConditionalContext)_localctx).e0.s && ((ConditionalContext)_localctx).e1.s; 
             }
@@ -1713,7 +1715,7 @@ class PainlessParser extends Parser {
           }
           } 
         }
-        setState(304);
+        setState(312);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
       }
@@ -1747,6 +1749,17 @@ class PainlessParser extends Parser {
       this.s = ctx.s;
     }
   }
+  public static class ListinitContext extends UnaryContext {
+    public ListinitializerContext listinitializer() {
+      return getRuleContext(ListinitializerContext.class,0);
+    }
+    public ListinitContext(UnaryContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitListinit(this);
+      else return visitor.visitChildren(this);
+    }
+  }
   public static class CastContext extends UnaryContext {
     public TerminalNode LP() { return getToken(PainlessParser.LP, 0); }
     public DecltypeContext decltype() {
@@ -1809,6 +1822,17 @@ class PainlessParser extends Parser {
       else return visitor.visitChildren(this);
     }
   }
+  public static class MapinitContext extends UnaryContext {
+    public MapinitializerContext mapinitializer() {
+      return getRuleContext(MapinitializerContext.class,0);
+    }
+    public MapinitContext(UnaryContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitMapinit(this);
+      else return visitor.visitChildren(this);
+    }
+  }
   public static class TrueContext extends UnaryContext {
     public TerminalNode TRUE() { return getToken(PainlessParser.TRUE, 0); }
     public TrueContext(UnaryContext ctx) { copyFrom(ctx); }
@@ -1860,22 +1884,22 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 30, RULE_unary);
     int _la;
     try {
-      setState(334);
+      setState(350);
       switch ( getInterpreter().adaptivePredict(_input,21,_ctx) ) {
       case 1:
         _localctx = new PreContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(305);
+        setState(313);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(306);
+        setState(314);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(307);
+        setState(315);
         chain(true);
         }
         break;
@@ -1883,11 +1907,11 @@ class PainlessParser extends Parser {
         _localctx = new PostContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(308);
+        setState(316);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(309);
+        setState(317);
         chain(true);
-        setState(310);
+        setState(318);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
@@ -1900,9 +1924,9 @@ class PainlessParser extends Parser {
         _localctx = new ReadContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(312);
+        setState(320);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(313);
+        setState(321);
         chain(false);
         }
         break;
@@ -1910,9 +1934,9 @@ class PainlessParser extends Parser {
         _localctx = new NumericContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(314);
+        setState(322);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(315);
+        setState(323);
         _la = _input.LA(1);
         if ( !(((((_la - 68)) & ~0x3f) == 0 && ((1L << (_la - 68)) & ((1L << (OCTAL - 68)) | (1L << (HEX - 68)) | (1L << (INTEGER - 68)) | (1L << (DECIMAL - 68)))) != 0)) ) {
         _errHandler.recoverInline(this);
@@ -1926,9 +1950,9 @@ class PainlessParser extends Parser {
         _localctx = new TrueContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(317);
+        setState(325);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(318);
+        setState(326);
         match(TRUE);
          ((TrueContext)_localctx).s =  false; 
         }
@@ -1937,9 +1961,9 @@ class PainlessParser extends Parser {
         _localctx = new FalseContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(320);
+        setState(328);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(321);
+        setState(329);
         match(FALSE);
          ((FalseContext)_localctx).s =  false; 
         }
@@ -1948,41 +1972,63 @@ class PainlessParser extends Parser {
         _localctx = new NullContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(323);
+        setState(331);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(324);
+        setState(332);
         match(NULL);
          ((NullContext)_localctx).s =  false; 
         }
         break;
       case 8:
-        _localctx = new OperatorContext(_localctx);
+        _localctx = new ListinitContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(326);
+        setState(334);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(327);
+        setState(335);
+        listinitializer();
+         ((ListinitContext)_localctx).s =  false; 
+        }
+        break;
+      case 9:
+        _localctx = new MapinitContext(_localctx);
+        enterOuterAlt(_localctx, 9);
+        {
+        setState(338);
+        if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
+        setState(339);
+        mapinitializer();
+         ((MapinitContext)_localctx).s =  false; 
+        }
+        break;
+      case 10:
+        _localctx = new OperatorContext(_localctx);
+        enterOuterAlt(_localctx, 10);
+        {
+        setState(342);
+        if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
+        setState(343);
         _la = _input.LA(1);
         if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(328);
+        setState(344);
         unary(false);
         }
         break;
-      case 9:
+      case 11:
         _localctx = new CastContext(_localctx);
-        enterOuterAlt(_localctx, 9);
+        enterOuterAlt(_localctx, 11);
         {
-        setState(329);
+        setState(345);
         match(LP);
-        setState(330);
+        setState(346);
         decltype();
-        setState(331);
+        setState(347);
         match(RP);
-        setState(332);
+        setState(348);
         unary(_localctx.c);
         }
         break;
@@ -2053,30 +2099,8 @@ class PainlessParser extends Parser {
     }
   }
   public static class NewarrayContext extends ChainContext {
-    public TerminalNode NEW() { return getToken(PainlessParser.NEW, 0); }
-    public TerminalNode TYPE() { return getToken(PainlessParser.TYPE, 0); }
-    public List<TerminalNode> LBRACE() { return getTokens(PainlessParser.LBRACE); }
-    public TerminalNode LBRACE(int i) {
-      return getToken(PainlessParser.LBRACE, i);
-    }
-    public List<ExpressionContext> expression() {
-      return getRuleContexts(ExpressionContext.class);
-    }
-    public ExpressionContext expression(int i) {
-      return getRuleContext(ExpressionContext.class,i);
-    }
-    public List<TerminalNode> RBRACE() { return getTokens(PainlessParser.RBRACE); }
-    public TerminalNode RBRACE(int i) {
-      return getToken(PainlessParser.RBRACE, i);
-    }
-    public DotContext dot() {
-      return getRuleContext(DotContext.class,0);
-    }
-    public List<SecondaryContext> secondary() {
-      return getRuleContexts(SecondaryContext.class);
-    }
-    public SecondaryContext secondary(int i) {
-      return getRuleContext(SecondaryContext.class,i);
+    public ArrayinitializerContext arrayinitializer() {
+      return getRuleContext(ArrayinitializerContext.class,0);
     }
     public NewarrayContext(ChainContext ctx) { copyFrom(ctx); }
     @Override
@@ -2091,27 +2115,27 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 32, RULE_chain);
     try {
       int _alt;
-      setState(370);
-      switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) {
+      setState(368);
+      switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) {
       case 1:
         _localctx = new DynamicContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(336);
+        setState(352);
         ((DynamicContext)_localctx).p = primary(_localctx.c);
-        setState(340);
+        setState(356);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(337);
+            setState(353);
             secondary(((DynamicContext)_localctx).p.s);
             }
             } 
           }
-          setState(342);
+          setState(358);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
         }
@@ -2121,23 +2145,23 @@ class PainlessParser extends Parser {
         _localctx = new StaticContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(343);
+        setState(359);
         decltype();
-        setState(344);
+        setState(360);
         dot();
-        setState(348);
+        setState(364);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(345);
+            setState(361);
             secondary(true);
             }
             } 
           }
-          setState(350);
+          setState(366);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
         }
@@ -2147,59 +2171,8 @@ class PainlessParser extends Parser {
         _localctx = new NewarrayContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(351);
-        match(NEW);
-        setState(352);
-        match(TYPE);
-        setState(357); 
-        _errHandler.sync(this);
-        _alt = 1;
-        do {
-          switch (_alt) {
-          case 1:
-            {
-            {
-            setState(353);
-            match(LBRACE);
-            setState(354);
-            expression(0);
-            setState(355);
-            match(RBRACE);
-            }
-            }
-            break;
-          default:
-            throw new NoViableAltException(this);
-          }
-          setState(359); 
-          _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,24,_ctx);
-        } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
-        setState(368);
-        switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) {
-        case 1:
-          {
-          setState(361);
-          dot();
-          setState(365);
-          _errHandler.sync(this);
-          _alt = getInterpreter().adaptivePredict(_input,25,_ctx);
-          while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
-            if ( _alt==1 ) {
-              {
-              {
-              setState(362);
-              secondary(true);
-              }
-              } 
-            }
-            setState(367);
-            _errHandler.sync(this);
-            _alt = getInterpreter().adaptivePredict(_input,25,_ctx);
-          }
-          }
-          break;
-        }
+        setState(367);
+        arrayinitializer();
         }
         break;
       }
@@ -2316,19 +2289,19 @@ class PainlessParser extends Parser {
     PrimaryContext _localctx = new PrimaryContext(_ctx, getState(), c);
     enterRule(_localctx, 34, RULE_primary);
     try {
-      setState(391);
-      switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) {
+      setState(389);
+      switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) {
       case 1:
         _localctx = new ExprprecContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(372);
+        setState(370);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(373);
+        setState(371);
         match(LP);
-        setState(374);
+        setState(372);
         ((ExprprecContext)_localctx).e = expression(0);
-        setState(375);
+        setState(373);
         match(RP);
          ((ExprprecContext)_localctx).s =  ((ExprprecContext)_localctx).e.s; 
         }
@@ -2337,13 +2310,13 @@ class PainlessParser extends Parser {
         _localctx = new ChainprecContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(378);
+        setState(376);
         if (!( _localctx.c )) throw new FailedPredicateException(this, " $c ");
-        setState(379);
+        setState(377);
         match(LP);
-        setState(380);
+        setState(378);
         unary(true);
-        setState(381);
+        setState(379);
         match(RP);
         }
         break;
@@ -2351,7 +2324,7 @@ class PainlessParser extends Parser {
         _localctx = new StringContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(383);
+        setState(381);
         match(STRING);
         }
         break;
@@ -2359,7 +2332,7 @@ class PainlessParser extends Parser {
         _localctx = new RegexContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(384);
+        setState(382);
         match(REGEX);
         }
         break;
@@ -2367,7 +2340,7 @@ class PainlessParser extends Parser {
         _localctx = new VariableContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(385);
+        setState(383);
         match(ID);
         }
         break;
@@ -2375,9 +2348,9 @@ class PainlessParser extends Parser {
         _localctx = new CalllocalContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(386);
+        setState(384);
         match(ID);
-        setState(387);
+        setState(385);
         arguments();
         }
         break;
@@ -2385,11 +2358,11 @@ class PainlessParser extends Parser {
         _localctx = new NewobjectContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(388);
+        setState(386);
         match(NEW);
-        setState(389);
+        setState(387);
         match(TYPE);
-        setState(390);
+        setState(388);
         arguments();
         }
         break;
@@ -2431,23 +2404,23 @@ class PainlessParser extends Parser {
     SecondaryContext _localctx = new SecondaryContext(_ctx, getState(), s);
     enterRule(_localctx, 36, RULE_secondary);
     try {
-      setState(397);
-      switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
+      setState(395);
+      switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(393);
+        setState(391);
         if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
-        setState(394);
+        setState(392);
         dot();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(395);
+        setState(393);
         if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
-        setState(396);
+        setState(394);
         brace();
         }
         break;
@@ -2505,17 +2478,17 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 38, RULE_dot);
     int _la;
     try {
-      setState(404);
-      switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
+      setState(402);
+      switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) {
       case 1:
         _localctx = new CallinvokeContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(399);
+        setState(397);
         match(DOT);
-        setState(400);
+        setState(398);
         match(DOTID);
-        setState(401);
+        setState(399);
         arguments();
         }
         break;
@@ -2523,9 +2496,9 @@ class PainlessParser extends Parser {
         _localctx = new FieldaccessContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(402);
+        setState(400);
         match(DOT);
-        setState(403);
+        setState(401);
         _la = _input.LA(1);
         if ( !(_la==DOTINTEGER || _la==DOTID) ) {
         _errHandler.recoverInline(this);
@@ -2579,11 +2552,11 @@ class PainlessParser extends Parser {
       _localctx = new BraceaccessContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(406);
+      setState(404);
       match(LBRACE);
-      setState(407);
+      setState(405);
       expression(0);
-      setState(408);
+      setState(406);
       match(RBRACE);
       }
     }
@@ -2630,34 +2603,34 @@ class PainlessParser extends Parser {
       enterOuterAlt(_localctx, 1);
       {
       {
-      setState(410);
+      setState(408);
       match(LP);
-      setState(419);
-      switch ( getInterpreter().adaptivePredict(_input,32,_ctx) ) {
+      setState(417);
+      switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
       case 1:
         {
-        setState(411);
+        setState(409);
         argument();
-        setState(416);
+        setState(414);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(412);
+          setState(410);
           match(COMMA);
-          setState(413);
+          setState(411);
           argument();
           }
           }
-          setState(418);
+          setState(416);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
         break;
       }
-      setState(421);
+      setState(419);
       match(RP);
       }
       }
@@ -2698,26 +2671,26 @@ class PainlessParser extends Parser {
     ArgumentContext _localctx = new ArgumentContext(_ctx, getState());
     enterRule(_localctx, 44, RULE_argument);
     try {
-      setState(426);
-      switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) {
+      setState(424);
+      switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(423);
+        setState(421);
         expression(0);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(424);
+        setState(422);
         lambda();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(425);
+        setState(423);
         funcref();
         }
         break;
@@ -2772,64 +2745,64 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(441);
+      setState(439);
       switch (_input.LA(1)) {
       case TYPE:
       case ID:
         {
-        setState(428);
+        setState(426);
         lamtype();
         }
         break;
       case LP:
         {
-        setState(429);
+        setState(427);
         match(LP);
-        setState(438);
+        setState(436);
         _la = _input.LA(1);
         if (_la==TYPE || _la==ID) {
           {
-          setState(430);
+          setState(428);
           lamtype();
-          setState(435);
+          setState(433);
           _errHandler.sync(this);
           _la = _input.LA(1);
           while (_la==COMMA) {
             {
             {
-            setState(431);
+            setState(429);
             match(COMMA);
-            setState(432);
+            setState(430);
             lamtype();
             }
             }
-            setState(437);
+            setState(435);
             _errHandler.sync(this);
             _la = _input.LA(1);
           }
           }
         }
 
-        setState(440);
+        setState(438);
         match(RP);
         }
         break;
       default:
         throw new NoViableAltException(this);
       }
-      setState(443);
+      setState(441);
       match(ARROW);
-      setState(446);
-      switch ( getInterpreter().adaptivePredict(_input,37,_ctx) ) {
+      setState(444);
+      switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) {
       case 1:
         {
-        setState(444);
+        setState(442);
         block();
         }
         break;
       case 2:
         {
-        setState(445);
+        setState(443);
         expression(0);
         }
         break;
@@ -2870,16 +2843,16 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(449);
+      setState(447);
       _la = _input.LA(1);
       if (_la==TYPE) {
         {
-        setState(448);
+        setState(446);
         decltype();
         }
       }
 
-      setState(451);
+      setState(449);
       match(ID);
       }
     }
@@ -2922,33 +2895,33 @@ class PainlessParser extends Parser {
     FuncrefContext _localctx = new FuncrefContext(_ctx, getState());
     enterRule(_localctx, 50, RULE_funcref);
     try {
-      setState(457);
-      switch ( getInterpreter().adaptivePredict(_input,39,_ctx) ) {
+      setState(455);
+      switch ( getInterpreter().adaptivePredict(_input,36,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(453);
+        setState(451);
         classFuncref();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(454);
+        setState(452);
         constructorFuncref();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(455);
+        setState(453);
         capturingFuncref();
         }
         break;
       case 4:
         enterOuterAlt(_localctx, 4);
         {
-        setState(456);
+        setState(454);
         localFuncref();
         }
         break;
@@ -2986,11 +2959,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(459);
+      setState(457);
       match(TYPE);
-      setState(460);
+      setState(458);
       match(REF);
-      setState(461);
+      setState(459);
       match(ID);
       }
     }
@@ -3028,11 +3001,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(463);
+      setState(461);
       decltype();
-      setState(464);
+      setState(462);
       match(REF);
-      setState(465);
+      setState(463);
       match(NEW);
       }
     }
@@ -3070,11 +3043,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(467);
+      setState(465);
       match(ID);
-      setState(468);
+      setState(466);
       match(REF);
-      setState(469);
+      setState(467);
       match(ID);
       }
     }
@@ -3110,11 +3083,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(471);
+      setState(469);
       match(THIS);
-      setState(472);
+      setState(470);
       match(REF);
-      setState(473);
+      setState(471);
       match(ID);
       }
     }
@@ -3129,6 +3102,415 @@ class PainlessParser extends Parser {
     return _localctx;
   }
 
+  public static class ArrayinitializerContext extends ParserRuleContext {
+    public ArrayinitializerContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_arrayinitializer; }
+   
+    public ArrayinitializerContext() { }
+    public void copyFrom(ArrayinitializerContext ctx) {
+      super.copyFrom(ctx);
+    }
+  }
+  public static class NewstandardarrayContext extends ArrayinitializerContext {
+    public TerminalNode NEW() { return getToken(PainlessParser.NEW, 0); }
+    public TerminalNode TYPE() { return getToken(PainlessParser.TYPE, 0); }
+    public List<TerminalNode> LBRACE() { return getTokens(PainlessParser.LBRACE); }
+    public TerminalNode LBRACE(int i) {
+      return getToken(PainlessParser.LBRACE, i);
+    }
+    public List<ExpressionContext> expression() {
+      return getRuleContexts(ExpressionContext.class);
+    }
+    public ExpressionContext expression(int i) {
+      return getRuleContext(ExpressionContext.class,i);
+    }
+    public List<TerminalNode> RBRACE() { return getTokens(PainlessParser.RBRACE); }
+    public TerminalNode RBRACE(int i) {
+      return getToken(PainlessParser.RBRACE, i);
+    }
+    public DotContext dot() {
+      return getRuleContext(DotContext.class,0);
+    }
+    public List<SecondaryContext> secondary() {
+      return getRuleContexts(SecondaryContext.class);
+    }
+    public SecondaryContext secondary(int i) {
+      return getRuleContext(SecondaryContext.class,i);
+    }
+    public NewstandardarrayContext(ArrayinitializerContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitNewstandardarray(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+  public static class NewinitializedarrayContext extends ArrayinitializerContext {
+    public TerminalNode NEW() { return getToken(PainlessParser.NEW, 0); }
+    public TerminalNode TYPE() { return getToken(PainlessParser.TYPE, 0); }
+    public TerminalNode LBRACE() { return getToken(PainlessParser.LBRACE, 0); }
+    public TerminalNode RBRACE() { return getToken(PainlessParser.RBRACE, 0); }
+    public TerminalNode LBRACK() { return getToken(PainlessParser.LBRACK, 0); }
+    public TerminalNode RBRACK() { return getToken(PainlessParser.RBRACK, 0); }
+    public List<ExpressionContext> expression() {
+      return getRuleContexts(ExpressionContext.class);
+    }
+    public ExpressionContext expression(int i) {
+      return getRuleContext(ExpressionContext.class,i);
+    }
+    public TerminalNode SEMICOLON() { return getToken(PainlessParser.SEMICOLON, 0); }
+    public List<TerminalNode> COMMA() { return getTokens(PainlessParser.COMMA); }
+    public TerminalNode COMMA(int i) {
+      return getToken(PainlessParser.COMMA, i);
+    }
+    public NewinitializedarrayContext(ArrayinitializerContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitNewinitializedarray(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final ArrayinitializerContext arrayinitializer() throws RecognitionException {
+    ArrayinitializerContext _localctx = new ArrayinitializerContext(_ctx, getState());
+    enterRule(_localctx, 60, RULE_arrayinitializer);
+    int _la;
+    try {
+      int _alt;
+      setState(511);
+      switch ( getInterpreter().adaptivePredict(_input,43,_ctx) ) {
+      case 1:
+        _localctx = new NewstandardarrayContext(_localctx);
+        enterOuterAlt(_localctx, 1);
+        {
+        setState(473);
+        match(NEW);
+        setState(474);
+        match(TYPE);
+        setState(479); 
+        _errHandler.sync(this);
+        _alt = 1;
+        do {
+          switch (_alt) {
+          case 1:
+            {
+            {
+            setState(475);
+            match(LBRACE);
+            setState(476);
+            expression(0);
+            setState(477);
+            match(RBRACE);
+            }
+            }
+            break;
+          default:
+            throw new NoViableAltException(this);
+          }
+          setState(481); 
+          _errHandler.sync(this);
+          _alt = getInterpreter().adaptivePredict(_input,37,_ctx);
+        } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
+        setState(490);
+        switch ( getInterpreter().adaptivePredict(_input,39,_ctx) ) {
+        case 1:
+          {
+          setState(483);
+          dot();
+          setState(487);
+          _errHandler.sync(this);
+          _alt = getInterpreter().adaptivePredict(_input,38,_ctx);
+          while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
+            if ( _alt==1 ) {
+              {
+              {
+              setState(484);
+              secondary(true);
+              }
+              } 
+            }
+            setState(489);
+            _errHandler.sync(this);
+            _alt = getInterpreter().adaptivePredict(_input,38,_ctx);
+          }
+          }
+          break;
+        }
+        }
+        break;
+      case 2:
+        _localctx = new NewinitializedarrayContext(_localctx);
+        enterOuterAlt(_localctx, 2);
+        {
+        setState(492);
+        match(NEW);
+        setState(493);
+        match(TYPE);
+        setState(494);
+        match(LBRACE);
+        setState(495);
+        match(RBRACE);
+        setState(496);
+        match(LBRACK);
+        setState(505);
+        switch ( getInterpreter().adaptivePredict(_input,41,_ctx) ) {
+        case 1:
+          {
+          setState(497);
+          expression(0);
+          setState(502);
+          _errHandler.sync(this);
+          _la = _input.LA(1);
+          while (_la==COMMA) {
+            {
+            {
+            setState(498);
+            match(COMMA);
+            setState(499);
+            expression(0);
+            }
+            }
+            setState(504);
+            _errHandler.sync(this);
+            _la = _input.LA(1);
+          }
+          }
+          break;
+        }
+        setState(508);
+        _la = _input.LA(1);
+        if (_la==SEMICOLON) {
+          {
+          setState(507);
+          match(SEMICOLON);
+          }
+        }
+
+        setState(510);
+        match(RBRACK);
+        }
+        break;
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
+  public static class ListinitializerContext extends ParserRuleContext {
+    public TerminalNode LBRACE() { return getToken(PainlessParser.LBRACE, 0); }
+    public List<ExpressionContext> expression() {
+      return getRuleContexts(ExpressionContext.class);
+    }
+    public ExpressionContext expression(int i) {
+      return getRuleContext(ExpressionContext.class,i);
+    }
+    public TerminalNode RBRACE() { return getToken(PainlessParser.RBRACE, 0); }
+    public List<TerminalNode> COMMA() { return getTokens(PainlessParser.COMMA); }
+    public TerminalNode COMMA(int i) {
+      return getToken(PainlessParser.COMMA, i);
+    }
+    public ListinitializerContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_listinitializer; }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitListinitializer(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final ListinitializerContext listinitializer() throws RecognitionException {
+    ListinitializerContext _localctx = new ListinitializerContext(_ctx, getState());
+    enterRule(_localctx, 62, RULE_listinitializer);
+    int _la;
+    try {
+      setState(526);
+      switch ( getInterpreter().adaptivePredict(_input,45,_ctx) ) {
+      case 1:
+        enterOuterAlt(_localctx, 1);
+        {
+        setState(513);
+        match(LBRACE);
+        setState(514);
+        expression(0);
+        setState(519);
+        _errHandler.sync(this);
+        _la = _input.LA(1);
+        while (_la==COMMA) {
+          {
+          {
+          setState(515);
+          match(COMMA);
+          setState(516);
+          expression(0);
+          }
+          }
+          setState(521);
+          _errHandler.sync(this);
+          _la = _input.LA(1);
+        }
+        setState(522);
+        match(RBRACE);
+        }
+        break;
+      case 2:
+        enterOuterAlt(_localctx, 2);
+        {
+        setState(524);
+        match(LBRACE);
+        setState(525);
+        match(RBRACE);
+        }
+        break;
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
+  public static class MapinitializerContext extends ParserRuleContext {
+    public TerminalNode LBRACE() { return getToken(PainlessParser.LBRACE, 0); }
+    public List<MaptokenContext> maptoken() {
+      return getRuleContexts(MaptokenContext.class);
+    }
+    public MaptokenContext maptoken(int i) {
+      return getRuleContext(MaptokenContext.class,i);
+    }
+    public TerminalNode RBRACE() { return getToken(PainlessParser.RBRACE, 0); }
+    public List<TerminalNode> COMMA() { return getTokens(PainlessParser.COMMA); }
+    public TerminalNode COMMA(int i) {
+      return getToken(PainlessParser.COMMA, i);
+    }
+    public TerminalNode COLON() { return getToken(PainlessParser.COLON, 0); }
+    public MapinitializerContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_mapinitializer; }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitMapinitializer(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final MapinitializerContext mapinitializer() throws RecognitionException {
+    MapinitializerContext _localctx = new MapinitializerContext(_ctx, getState());
+    enterRule(_localctx, 64, RULE_mapinitializer);
+    int _la;
+    try {
+      setState(542);
+      switch ( getInterpreter().adaptivePredict(_input,47,_ctx) ) {
+      case 1:
+        enterOuterAlt(_localctx, 1);
+        {
+        setState(528);
+        match(LBRACE);
+        setState(529);
+        maptoken();
+        setState(534);
+        _errHandler.sync(this);
+        _la = _input.LA(1);
+        while (_la==COMMA) {
+          {
+          {
+          setState(530);
+          match(COMMA);
+          setState(531);
+          maptoken();
+          }
+          }
+          setState(536);
+          _errHandler.sync(this);
+          _la = _input.LA(1);
+        }
+        setState(537);
+        match(RBRACE);
+        }
+        break;
+      case 2:
+        enterOuterAlt(_localctx, 2);
+        {
+        setState(539);
+        match(LBRACE);
+        setState(540);
+        match(COLON);
+        setState(541);
+        match(RBRACE);
+        }
+        break;
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
+  public static class MaptokenContext extends ParserRuleContext {
+    public List<ExpressionContext> expression() {
+      return getRuleContexts(ExpressionContext.class);
+    }
+    public ExpressionContext expression(int i) {
+      return getRuleContext(ExpressionContext.class,i);
+    }
+    public TerminalNode COLON() { return getToken(PainlessParser.COLON, 0); }
+    public MaptokenContext(ParserRuleContext parent, int invokingState) {
+      super(parent, invokingState);
+    }
+    @Override public int getRuleIndex() { return RULE_maptoken; }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitMaptoken(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+
+  public final MaptokenContext maptoken() throws RecognitionException {
+    MaptokenContext _localctx = new MaptokenContext(_ctx, getState());
+    enterRule(_localctx, 66, RULE_maptoken);
+    try {
+      enterOuterAlt(_localctx, 1);
+      {
+      setState(544);
+      expression(0);
+      setState(545);
+      match(COLON);
+      setState(546);
+      expression(0);
+      }
+    }
+    catch (RecognitionException re) {
+      _localctx.exception = re;
+      _errHandler.reportError(this, re);
+      _errHandler.recover(this, re);
+    }
+    finally {
+      exitRule();
+    }
+    return _localctx;
+  }
+
   public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) {
     switch (ruleIndex) {
     case 3:
@@ -3198,208 +3580,240 @@ class PainlessParser extends Parser {
       return  !_localctx.c ;
     case 20:
       return  !_localctx.c ;
+    case 21:
+      return  !_localctx.c ;
+    case 22:
+      return  !_localctx.c ;
     }
     return true;
   }
   private boolean primary_sempred(PrimaryContext _localctx, int predIndex) {
     switch (predIndex) {
-    case 21:
+    case 23:
       return  !_localctx.c ;
-    case 22:
+    case 24:
       return  _localctx.c ;
     }
     return true;
   }
   private boolean secondary_sempred(SecondaryContext _localctx, int predIndex) {
     switch (predIndex) {
-    case 23:
+    case 25:
       return  _localctx.s ;
-    case 24:
+    case 26:
       return  _localctx.s ;
     }
     return true;
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3R\u01de\4\2\t\2\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3R\u0227\4\2\t\2\4"+
     "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
     "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
     "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+
-    "\4\32\t\32\4\33\t\33\4\34\t\34\4\35\t\35\4\36\t\36\4\37\t\37\3\2\7\2@"+
-    "\n\2\f\2\16\2C\13\2\3\2\7\2F\n\2\f\2\16\2I\13\2\3\2\3\2\3\3\3\3\3\3\3"+
-    "\3\3\3\3\4\3\4\3\4\3\4\3\4\3\4\3\4\7\4Y\n\4\f\4\16\4\\\13\4\5\4^\n\4\3"+
-    "\4\3\4\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\5\5j\n\5\3\5\3\5\3\5\3\5\3\5\3"+
-    "\5\5\5r\n\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\5\5\177\n\5\3"+
-    "\5\3\5\5\5\u0083\n\5\3\5\3\5\5\5\u0087\n\5\3\5\3\5\3\5\5\5\u008c\n\5\3"+
-    "\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5"+
-    "\3\5\3\5\3\5\3\5\3\5\6\5\u00a5\n\5\r\5\16\5\u00a6\3\5\3\5\3\5\3\5\3\5"+
-    "\3\5\3\5\5\5\u00b0\n\5\3\6\3\6\5\6\u00b4\n\6\3\7\3\7\7\7\u00b8\n\7\f\7"+
-    "\16\7\u00bb\13\7\3\7\3\7\3\b\3\b\3\t\3\t\5\t\u00c3\n\t\3\n\3\n\3\13\3"+
-    "\13\3\13\3\13\7\13\u00cb\n\13\f\13\16\13\u00ce\13\13\3\f\3\f\3\f\7\f\u00d3"+
-    "\n\f\f\f\16\f\u00d6\13\f\3\r\3\r\3\r\5\r\u00db\n\r\3\16\3\16\3\16\3\16"+
-    "\3\16\3\16\3\16\3\17\3\17\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
-    "\5\20\u00ef\n\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
+    "\4\32\t\32\4\33\t\33\4\34\t\34\4\35\t\35\4\36\t\36\4\37\t\37\4 \t \4!"+
+    "\t!\4\"\t\"\4#\t#\3\2\7\2H\n\2\f\2\16\2K\13\2\3\2\7\2N\n\2\f\2\16\2Q\13"+
+    "\2\3\2\3\2\3\3\3\3\3\3\3\3\3\3\3\4\3\4\3\4\3\4\3\4\3\4\3\4\7\4a\n\4\f"+
+    "\4\16\4d\13\4\5\4f\n\4\3\4\3\4\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\5\5r\n"+
+    "\5\3\5\3\5\3\5\3\5\3\5\3\5\5\5z\n\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3"+
+    "\5\3\5\3\5\5\5\u0087\n\5\3\5\3\5\5\5\u008b\n\5\3\5\3\5\5\5\u008f\n\5\3"+
+    "\5\3\5\3\5\5\5\u0094\n\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3"+
+    "\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\6\5\u00ad\n\5\r\5\16\5"+
+    "\u00ae\3\5\3\5\3\5\3\5\3\5\3\5\3\5\5\5\u00b8\n\5\3\6\3\6\5\6\u00bc\n\6"+
+    "\3\7\3\7\7\7\u00c0\n\7\f\7\16\7\u00c3\13\7\3\7\3\7\3\b\3\b\3\t\3\t\5\t"+
+    "\u00cb\n\t\3\n\3\n\3\13\3\13\3\13\3\13\7\13\u00d3\n\13\f\13\16\13\u00d6"+
+    "\13\13\3\f\3\f\3\f\7\f\u00db\n\f\f\f\16\f\u00de\13\f\3\r\3\r\3\r\5\r\u00e3"+
+    "\n\r\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\17\3\17\3\20\3\20\3\20\3\20"+
+    "\3\20\3\20\3\20\3\20\3\20\5\20\u00f7\n\20\3\20\3\20\3\20\3\20\3\20\3\20"+
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
-    "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\7\20\u012f\n\20\f\20\16"+
-    "\20\u0132\13\20\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
+    "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
+    "\7\20\u0137\n\20\f\20\16\20\u013a\13\20\3\21\3\21\3\21\3\21\3\21\3\21"+
     "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
-    "\3\21\3\21\3\21\3\21\5\21\u0151\n\21\3\22\3\22\7\22\u0155\n\22\f\22\16"+
-    "\22\u0158\13\22\3\22\3\22\3\22\7\22\u015d\n\22\f\22\16\22\u0160\13\22"+
-    "\3\22\3\22\3\22\3\22\3\22\3\22\6\22\u0168\n\22\r\22\16\22\u0169\3\22\3"+
-    "\22\7\22\u016e\n\22\f\22\16\22\u0171\13\22\5\22\u0173\n\22\5\22\u0175"+
-    "\n\22\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23"+
-    "\3\23\3\23\3\23\3\23\3\23\3\23\5\23\u018a\n\23\3\24\3\24\3\24\3\24\5\24"+
-    "\u0190\n\24\3\25\3\25\3\25\3\25\3\25\5\25\u0197\n\25\3\26\3\26\3\26\3"+
-    "\26\3\27\3\27\3\27\3\27\7\27\u01a1\n\27\f\27\16\27\u01a4\13\27\5\27\u01a6"+
-    "\n\27\3\27\3\27\3\30\3\30\3\30\5\30\u01ad\n\30\3\31\3\31\3\31\3\31\3\31"+
-    "\7\31\u01b4\n\31\f\31\16\31\u01b7\13\31\5\31\u01b9\n\31\3\31\5\31\u01bc"+
-    "\n\31\3\31\3\31\3\31\5\31\u01c1\n\31\3\32\5\32\u01c4\n\32\3\32\3\32\3"+
-    "\33\3\33\3\33\3\33\5\33\u01cc\n\33\3\34\3\34\3\34\3\34\3\35\3\35\3\35"+
-    "\3\35\3\36\3\36\3\36\3\36\3\37\3\37\3\37\3\37\3\37\2\3\36 \2\4\6\b\n\f"+
-    "\16\20\22\24\26\30\32\34\36 \"$&(*,.\60\62\64\668:<\2\16\3\3\r\r\3\2:"+
-    "E\3\2\35\37\3\2 !\3\2\66\67\3\2\"$\3\2%(\3\2),\3\289\3\2FI\4\2\33\34 "+
-    "!\3\2QR\u020b\2A\3\2\2\2\4L\3\2\2\2\6Q\3\2\2\2\b\u00af\3\2\2\2\n\u00b3"+
-    "\3\2\2\2\f\u00b5\3\2\2\2\16\u00be\3\2\2\2\20\u00c2\3\2\2\2\22\u00c4\3"+
-    "\2\2\2\24\u00c6\3\2\2\2\26\u00cf\3\2\2\2\30\u00d7\3\2\2\2\32\u00dc\3\2"+
-    "\2\2\34\u00e3\3\2\2\2\36\u00ee\3\2\2\2 \u0150\3\2\2\2\"\u0174\3\2\2\2"+
-    "$\u0189\3\2\2\2&\u018f\3\2\2\2(\u0196\3\2\2\2*\u0198\3\2\2\2,\u019c\3"+
-    "\2\2\2.\u01ac\3\2\2\2\60\u01bb\3\2\2\2\62\u01c3\3\2\2\2\64\u01cb\3\2\2"+
-    "\2\66\u01cd\3\2\2\28\u01d1\3\2\2\2:\u01d5\3\2\2\2<\u01d9\3\2\2\2>@\5\4"+
-    "\3\2?>\3\2\2\2@C\3\2\2\2A?\3\2\2\2AB\3\2\2\2BG\3\2\2\2CA\3\2\2\2DF\5\b"+
-    "\5\2ED\3\2\2\2FI\3\2\2\2GE\3\2\2\2GH\3\2\2\2HJ\3\2\2\2IG\3\2\2\2JK\7\2"+
-    "\2\3K\3\3\2\2\2LM\5\26\f\2MN\7P\2\2NO\5\6\4\2OP\5\f\7\2P\5\3\2\2\2Q]\7"+
-    "\t\2\2RS\5\26\f\2SZ\7P\2\2TU\7\f\2\2UV\5\26\f\2VW\7P\2\2WY\3\2\2\2XT\3"+
-    "\2\2\2Y\\\3\2\2\2ZX\3\2\2\2Z[\3\2\2\2[^\3\2\2\2\\Z\3\2\2\2]R\3\2\2\2]"+
-    "^\3\2\2\2^_\3\2\2\2_`\7\n\2\2`\7\3\2\2\2ab\7\16\2\2bc\7\t\2\2cd\5\36\20"+
-    "\2de\7\n\2\2ei\5\n\6\2fg\7\17\2\2gj\5\n\6\2hj\6\5\2\2if\3\2\2\2ih\3\2"+
-    "\2\2j\u00b0\3\2\2\2kl\7\20\2\2lm\7\t\2\2mn\5\36\20\2nq\7\n\2\2or\5\n\6"+
-    "\2pr\5\16\b\2qo\3\2\2\2qp\3\2\2\2r\u00b0\3\2\2\2st\7\21\2\2tu\5\f\7\2"+
-    "uv\7\20\2\2vw\7\t\2\2wx\5\36\20\2xy\7\n\2\2yz\5\34\17\2z\u00b0\3\2\2\2"+
-    "{|\7\22\2\2|~\7\t\2\2}\177\5\20\t\2~}\3\2\2\2~\177\3\2\2\2\177\u0080\3"+
-    "\2\2\2\u0080\u0082\7\r\2\2\u0081\u0083\5\36\20\2\u0082\u0081\3\2\2\2\u0082"+
-    "\u0083\3\2\2\2\u0083\u0084\3\2\2\2\u0084\u0086\7\r\2\2\u0085\u0087\5\22"+
-    "\n\2\u0086\u0085\3\2\2\2\u0086\u0087\3\2\2\2\u0087\u0088\3\2\2\2\u0088"+
-    "\u008b\7\n\2\2\u0089\u008c\5\n\6\2\u008a\u008c\5\16\b\2\u008b\u0089\3"+
-    "\2\2\2\u008b\u008a\3\2\2\2\u008c\u00b0\3\2\2\2\u008d\u008e\7\22\2\2\u008e"+
-    "\u008f\7\t\2\2\u008f\u0090\5\26\f\2\u0090\u0091\7P\2\2\u0091\u0092\7\63"+
-    "\2\2\u0092\u0093\5\36\20\2\u0093\u0094\7\n\2\2\u0094\u0095\5\n\6\2\u0095"+
-    "\u00b0\3\2\2\2\u0096\u0097\5\24\13\2\u0097\u0098\5\34\17\2\u0098\u00b0"+
-    "\3\2\2\2\u0099\u009a\7\23\2\2\u009a\u00b0\5\34\17\2\u009b\u009c\7\24\2"+
-    "\2\u009c\u00b0\5\34\17\2\u009d\u009e\7\25\2\2\u009e\u009f\5\36\20\2\u009f"+
-    "\u00a0\5\34\17\2\u00a0\u00b0\3\2\2\2\u00a1\u00a2\7\27\2\2\u00a2\u00a4"+
-    "\5\f\7\2\u00a3\u00a5\5\32\16\2\u00a4\u00a3\3\2\2\2\u00a5\u00a6\3\2\2\2"+
-    "\u00a6\u00a4\3\2\2\2\u00a6\u00a7\3\2\2\2\u00a7\u00b0\3\2\2\2\u00a8\u00a9"+
-    "\7\31\2\2\u00a9\u00aa\5\36\20\2\u00aa\u00ab\5\34\17\2\u00ab\u00b0\3\2"+
-    "\2\2\u00ac\u00ad\5\36\20\2\u00ad\u00ae\5\34\17\2\u00ae\u00b0\3\2\2\2\u00af"+
-    "a\3\2\2\2\u00afk\3\2\2\2\u00afs\3\2\2\2\u00af{\3\2\2\2\u00af\u008d\3\2"+
-    "\2\2\u00af\u0096\3\2\2\2\u00af\u0099\3\2\2\2\u00af\u009b\3\2\2\2\u00af"+
-    "\u009d\3\2\2\2\u00af\u00a1\3\2\2\2\u00af\u00a8\3\2\2\2\u00af\u00ac\3\2"+
-    "\2\2\u00b0\t\3\2\2\2\u00b1\u00b4\5\f\7\2\u00b2\u00b4\5\b\5\2\u00b3\u00b1"+
-    "\3\2\2\2\u00b3\u00b2\3\2\2\2\u00b4\13\3\2\2\2\u00b5\u00b9\7\5\2\2\u00b6"+
-    "\u00b8\5\b\5\2\u00b7\u00b6\3\2\2\2\u00b8\u00bb\3\2\2\2\u00b9\u00b7\3\2"+
-    "\2\2\u00b9\u00ba\3\2\2\2\u00ba\u00bc\3\2\2\2\u00bb\u00b9\3\2\2\2\u00bc"+
-    "\u00bd\7\6\2\2\u00bd\r\3\2\2\2\u00be\u00bf\7\r\2\2\u00bf\17\3\2\2\2\u00c0"+
-    "\u00c3\5\24\13\2\u00c1\u00c3\5\36\20\2\u00c2\u00c0\3\2\2\2\u00c2\u00c1"+
-    "\3\2\2\2\u00c3\21\3\2\2\2\u00c4\u00c5\5\36\20\2\u00c5\23\3\2\2\2\u00c6"+
-    "\u00c7\5\26\f\2\u00c7\u00cc\5\30\r\2\u00c8\u00c9\7\f\2\2\u00c9\u00cb\5"+
-    "\30\r\2\u00ca\u00c8\3\2\2\2\u00cb\u00ce\3\2\2\2\u00cc\u00ca\3\2\2\2\u00cc"+
-    "\u00cd\3\2\2\2\u00cd\25\3\2\2\2\u00ce\u00cc\3\2\2\2\u00cf\u00d4\7O\2\2"+
-    "\u00d0\u00d1\7\7\2\2\u00d1\u00d3\7\b\2\2\u00d2\u00d0\3\2\2\2\u00d3\u00d6"+
-    "\3\2\2\2\u00d4\u00d2\3\2\2\2\u00d4\u00d5\3\2\2\2\u00d5\27\3\2\2\2\u00d6"+
-    "\u00d4\3\2\2\2\u00d7\u00da\7P\2\2\u00d8\u00d9\7:\2\2\u00d9\u00db\5\36"+
-    "\20\2\u00da\u00d8\3\2\2\2\u00da\u00db\3\2\2\2\u00db\31\3\2\2\2\u00dc\u00dd"+
-    "\7\30\2\2\u00dd\u00de\7\t\2\2\u00de\u00df\7O\2\2\u00df\u00e0\7P\2\2\u00e0"+
-    "\u00e1\7\n\2\2\u00e1\u00e2\5\f\7\2\u00e2\33\3\2\2\2\u00e3\u00e4\t\2\2"+
-    "\2\u00e4\35\3\2\2\2\u00e5\u00e6\b\20\1\2\u00e6\u00e7\5\"\22\2\u00e7\u00e8"+
-    "\t\3\2\2\u00e8\u00e9\5\36\20\3\u00e9\u00ea\b\20\1\2\u00ea\u00ef\3\2\2"+
-    "\2\u00eb\u00ec\5 \21\2\u00ec\u00ed\b\20\1\2\u00ed\u00ef\3\2\2\2\u00ee"+
-    "\u00e5\3\2\2\2\u00ee\u00eb\3\2\2\2\u00ef\u0130\3\2\2\2\u00f0\u00f1\f\17"+
-    "\2\2\u00f1\u00f2\t\4\2\2\u00f2\u00f3\5\36\20\20\u00f3\u00f4\b\20\1\2\u00f4"+
-    "\u012f\3\2\2\2\u00f5\u00f6\f\16\2\2\u00f6\u00f7\t\5\2\2\u00f7\u00f8\5"+
-    "\36\20\17\u00f8\u00f9\b\20\1\2\u00f9\u012f\3\2\2\2\u00fa\u00fb\f\r\2\2"+
-    "\u00fb\u00fc\t\6\2\2\u00fc\u00fd\5\36\20\16\u00fd\u00fe\b\20\1\2\u00fe"+
-    "\u012f\3\2\2\2\u00ff\u0100\f\f\2\2\u0100\u0101\t\7\2\2\u0101\u0102\5\36"+
-    "\20\r\u0102\u0103\b\20\1\2\u0103\u012f\3\2\2\2\u0104\u0105\f\13\2\2\u0105"+
-    "\u0106\t\b\2\2\u0106\u0107\5\36\20\f\u0107\u0108\b\20\1\2\u0108\u012f"+
-    "\3\2\2\2\u0109\u010a\f\n\2\2\u010a\u010b\t\t\2\2\u010b\u010c\5\36\20\13"+
-    "\u010c\u010d\b\20\1\2\u010d\u012f\3\2\2\2\u010e\u010f\f\t\2\2\u010f\u0110"+
-    "\7-\2\2\u0110\u0111\5\36\20\n\u0111\u0112\b\20\1\2\u0112\u012f\3\2\2\2"+
-    "\u0113\u0114\f\b\2\2\u0114\u0115\7.\2\2\u0115\u0116\5\36\20\t\u0116\u0117"+
-    "\b\20\1\2\u0117\u012f\3\2\2\2\u0118\u0119\f\7\2\2\u0119\u011a\7/\2\2\u011a"+
-    "\u011b\5\36\20\b\u011b\u011c\b\20\1\2\u011c\u012f\3\2\2\2\u011d\u011e"+
-    "\f\6\2\2\u011e\u011f\7\60\2\2\u011f\u0120\5\36\20\7\u0120\u0121\b\20\1"+
-    "\2\u0121\u012f\3\2\2\2\u0122\u0123\f\5\2\2\u0123\u0124\7\61\2\2\u0124"+
-    "\u0125\5\36\20\6\u0125\u0126\b\20\1\2\u0126\u012f\3\2\2\2\u0127\u0128"+
-    "\f\4\2\2\u0128\u0129\7\62\2\2\u0129\u012a\5\36\20\2\u012a\u012b\7\63\2"+
-    "\2\u012b\u012c\5\36\20\4\u012c\u012d\b\20\1\2\u012d\u012f\3\2\2\2\u012e"+
-    "\u00f0\3\2\2\2\u012e\u00f5\3\2\2\2\u012e\u00fa\3\2\2\2\u012e\u00ff\3\2"+
-    "\2\2\u012e\u0104\3\2\2\2\u012e\u0109\3\2\2\2\u012e\u010e\3\2\2\2\u012e"+
-    "\u0113\3\2\2\2\u012e\u0118\3\2\2\2\u012e\u011d\3\2\2\2\u012e\u0122\3\2"+
-    "\2\2\u012e\u0127\3\2\2\2\u012f\u0132\3\2\2\2\u0130\u012e\3\2\2\2\u0130"+
-    "\u0131\3\2\2\2\u0131\37\3\2\2\2\u0132\u0130\3\2\2\2\u0133\u0134\6\21\17"+
-    "\3\u0134\u0135\t\n\2\2\u0135\u0151\5\"\22\2\u0136\u0137\6\21\20\3\u0137"+
-    "\u0138\5\"\22\2\u0138\u0139\t\n\2\2\u0139\u0151\3\2\2\2\u013a\u013b\6"+
-    "\21\21\3\u013b\u0151\5\"\22\2\u013c\u013d\6\21\22\3\u013d\u013e\t\13\2"+
-    "\2\u013e\u0151\b\21\1\2\u013f\u0140\6\21\23\3\u0140\u0141\7L\2\2\u0141"+
-    "\u0151\b\21\1\2\u0142\u0143\6\21\24\3\u0143\u0144\7M\2\2\u0144\u0151\b"+
-    "\21\1\2\u0145\u0146\6\21\25\3\u0146\u0147\7N\2\2\u0147\u0151\b\21\1\2"+
-    "\u0148\u0149\6\21\26\3\u0149\u014a\t\f\2\2\u014a\u0151\5 \21\2\u014b\u014c"+
-    "\7\t\2\2\u014c\u014d\5\26\f\2\u014d\u014e\7\n\2\2\u014e\u014f\5 \21\2"+
-    "\u014f\u0151\3\2\2\2\u0150\u0133\3\2\2\2\u0150\u0136\3\2\2\2\u0150\u013a"+
-    "\3\2\2\2\u0150\u013c\3\2\2\2\u0150\u013f\3\2\2\2\u0150\u0142\3\2\2\2\u0150"+
-    "\u0145\3\2\2\2\u0150\u0148\3\2\2\2\u0150\u014b\3\2\2\2\u0151!\3\2\2\2"+
-    "\u0152\u0156\5$\23\2\u0153\u0155\5&\24\2\u0154\u0153\3\2\2\2\u0155\u0158"+
-    "\3\2\2\2\u0156\u0154\3\2\2\2\u0156\u0157\3\2\2\2\u0157\u0175\3\2\2\2\u0158"+
-    "\u0156\3\2\2\2\u0159\u015a\5\26\f\2\u015a\u015e\5(\25\2\u015b\u015d\5"+
-    "&\24\2\u015c\u015b\3\2\2\2\u015d\u0160\3\2\2\2\u015e\u015c\3\2\2\2\u015e"+
-    "\u015f\3\2\2\2\u015f\u0175\3\2\2\2\u0160\u015e\3\2\2\2\u0161\u0162\7\26"+
-    "\2\2\u0162\u0167\7O\2\2\u0163\u0164\7\7\2\2\u0164\u0165\5\36\20\2\u0165"+
-    "\u0166\7\b\2\2\u0166\u0168\3\2\2\2\u0167\u0163\3\2\2\2\u0168\u0169\3\2"+
-    "\2\2\u0169\u0167\3\2\2\2\u0169\u016a\3\2\2\2\u016a\u0172\3\2\2\2\u016b"+
-    "\u016f\5(\25\2\u016c\u016e\5&\24\2\u016d\u016c\3\2\2\2\u016e\u0171\3\2"+
-    "\2\2\u016f\u016d\3\2\2\2\u016f\u0170\3\2\2\2\u0170\u0173\3\2\2\2\u0171"+
-    "\u016f\3\2\2\2\u0172\u016b\3\2\2\2\u0172\u0173\3\2\2\2\u0173\u0175\3\2"+
-    "\2\2\u0174\u0152\3\2\2\2\u0174\u0159\3\2\2\2\u0174\u0161\3\2\2\2\u0175"+
-    "#\3\2\2\2\u0176\u0177\6\23\27\3\u0177\u0178\7\t\2\2\u0178\u0179\5\36\20"+
-    "\2\u0179\u017a\7\n\2\2\u017a\u017b\b\23\1\2\u017b\u018a\3\2\2\2\u017c"+
-    "\u017d\6\23\30\3\u017d\u017e\7\t\2\2\u017e\u017f\5 \21\2\u017f\u0180\7"+
-    "\n\2\2\u0180\u018a\3\2\2\2\u0181\u018a\7J\2\2\u0182\u018a\7K\2\2\u0183"+
-    "\u018a\7P\2\2\u0184\u0185\7P\2\2\u0185\u018a\5,\27\2\u0186\u0187\7\26"+
-    "\2\2\u0187\u0188\7O\2\2\u0188\u018a\5,\27\2\u0189\u0176\3\2\2\2\u0189"+
-    "\u017c\3\2\2\2\u0189\u0181\3\2\2\2\u0189\u0182\3\2\2\2\u0189\u0183\3\2"+
-    "\2\2\u0189\u0184\3\2\2\2\u0189\u0186\3\2\2\2\u018a%\3\2\2\2\u018b\u018c"+
-    "\6\24\31\3\u018c\u0190\5(\25\2\u018d\u018e\6\24\32\3\u018e\u0190\5*\26"+
-    "\2\u018f\u018b\3\2\2\2\u018f\u018d\3\2\2\2\u0190\'\3\2\2\2\u0191\u0192"+
-    "\7\13\2\2\u0192\u0193\7R\2\2\u0193\u0197\5,\27\2\u0194\u0195\7\13\2\2"+
-    "\u0195\u0197\t\r\2\2\u0196\u0191\3\2\2\2\u0196\u0194\3\2\2\2\u0197)\3"+
-    "\2\2\2\u0198\u0199\7\7\2\2\u0199\u019a\5\36\20\2\u019a\u019b\7\b\2\2\u019b"+
-    "+\3\2\2\2\u019c\u01a5\7\t\2\2\u019d\u01a2\5.\30\2\u019e\u019f\7\f\2\2"+
-    "\u019f\u01a1\5.\30\2\u01a0\u019e\3\2\2\2\u01a1\u01a4\3\2\2\2\u01a2\u01a0"+
-    "\3\2\2\2\u01a2\u01a3\3\2\2\2\u01a3\u01a6\3\2\2\2\u01a4\u01a2\3\2\2\2\u01a5"+
-    "\u019d\3\2\2\2\u01a5\u01a6\3\2\2\2\u01a6\u01a7\3\2\2\2\u01a7\u01a8\7\n"+
-    "\2\2\u01a8-\3\2\2\2\u01a9\u01ad\5\36\20\2\u01aa\u01ad\5\60\31\2\u01ab"+
-    "\u01ad\5\64\33\2\u01ac\u01a9\3\2\2\2\u01ac\u01aa\3\2\2\2\u01ac\u01ab\3"+
-    "\2\2\2\u01ad/\3\2\2\2\u01ae\u01bc\5\62\32\2\u01af\u01b8\7\t\2\2\u01b0"+
-    "\u01b5\5\62\32\2\u01b1\u01b2\7\f\2\2\u01b2\u01b4\5\62\32\2\u01b3\u01b1"+
-    "\3\2\2\2\u01b4\u01b7\3\2\2\2\u01b5\u01b3\3\2\2\2\u01b5\u01b6\3\2\2\2\u01b6"+
-    "\u01b9\3\2\2\2\u01b7\u01b5\3\2\2\2\u01b8\u01b0\3\2\2\2\u01b8\u01b9\3\2"+
-    "\2\2\u01b9\u01ba\3\2\2\2\u01ba\u01bc\7\n\2\2\u01bb\u01ae\3\2\2\2\u01bb"+
-    "\u01af\3\2\2\2\u01bc\u01bd\3\2\2\2\u01bd\u01c0\7\65\2\2\u01be\u01c1\5"+
-    "\f\7\2\u01bf\u01c1\5\36\20\2\u01c0\u01be\3\2\2\2\u01c0\u01bf\3\2\2\2\u01c1"+
-    "\61\3\2\2\2\u01c2\u01c4\5\26\f\2\u01c3\u01c2\3\2\2\2\u01c3\u01c4\3\2\2"+
-    "\2\u01c4\u01c5\3\2\2\2\u01c5\u01c6\7P\2\2\u01c6\63\3\2\2\2\u01c7\u01cc"+
-    "\5\66\34\2\u01c8\u01cc\58\35\2\u01c9\u01cc\5:\36\2\u01ca\u01cc\5<\37\2"+
-    "\u01cb\u01c7\3\2\2\2\u01cb\u01c8\3\2\2\2\u01cb\u01c9\3\2\2\2\u01cb\u01ca"+
-    "\3\2\2\2\u01cc\65\3\2\2\2\u01cd\u01ce\7O\2\2\u01ce\u01cf\7\64\2\2\u01cf"+
-    "\u01d0\7P\2\2\u01d0\67\3\2\2\2\u01d1\u01d2\5\26\f\2\u01d2\u01d3\7\64\2"+
-    "\2\u01d3\u01d4\7\26\2\2\u01d49\3\2\2\2\u01d5\u01d6\7P\2\2\u01d6\u01d7"+
-    "\7\64\2\2\u01d7\u01d8\7P\2\2\u01d8;\3\2\2\2\u01d9\u01da\7\32\2\2\u01da"+
-    "\u01db\7\64\2\2\u01db\u01dc\7P\2\2\u01dc=\3\2\2\2*AGZ]iq~\u0082\u0086"+
-    "\u008b\u00a6\u00af\u00b3\u00b9\u00c2\u00cc\u00d4\u00da\u00ee\u012e\u0130"+
-    "\u0150\u0156\u015e\u0169\u016f\u0172\u0174\u0189\u018f\u0196\u01a2\u01a5"+
-    "\u01ac\u01b5\u01b8\u01bb\u01c0\u01c3\u01cb";
+    "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
+    "\3\21\3\21\3\21\5\21\u0161\n\21\3\22\3\22\7\22\u0165\n\22\f\22\16\22\u0168"+
+    "\13\22\3\22\3\22\3\22\7\22\u016d\n\22\f\22\16\22\u0170\13\22\3\22\5\22"+
+    "\u0173\n\22\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23"+
+    "\3\23\3\23\3\23\3\23\3\23\3\23\3\23\5\23\u0188\n\23\3\24\3\24\3\24\3\24"+
+    "\5\24\u018e\n\24\3\25\3\25\3\25\3\25\3\25\5\25\u0195\n\25\3\26\3\26\3"+
+    "\26\3\26\3\27\3\27\3\27\3\27\7\27\u019f\n\27\f\27\16\27\u01a2\13\27\5"+
+    "\27\u01a4\n\27\3\27\3\27\3\30\3\30\3\30\5\30\u01ab\n\30\3\31\3\31\3\31"+
+    "\3\31\3\31\7\31\u01b2\n\31\f\31\16\31\u01b5\13\31\5\31\u01b7\n\31\3\31"+
+    "\5\31\u01ba\n\31\3\31\3\31\3\31\5\31\u01bf\n\31\3\32\5\32\u01c2\n\32\3"+
+    "\32\3\32\3\33\3\33\3\33\3\33\5\33\u01ca\n\33\3\34\3\34\3\34\3\34\3\35"+
+    "\3\35\3\35\3\35\3\36\3\36\3\36\3\36\3\37\3\37\3\37\3\37\3 \3 \3 \3 \3"+
+    " \3 \6 \u01e2\n \r \16 \u01e3\3 \3 \7 \u01e8\n \f \16 \u01eb\13 \5 \u01ed"+
+    "\n \3 \3 \3 \3 \3 \3 \3 \3 \7 \u01f7\n \f \16 \u01fa\13 \5 \u01fc\n \3"+
+    " \5 \u01ff\n \3 \5 \u0202\n \3!\3!\3!\3!\7!\u0208\n!\f!\16!\u020b\13!"+
+    "\3!\3!\3!\3!\5!\u0211\n!\3\"\3\"\3\"\3\"\7\"\u0217\n\"\f\"\16\"\u021a"+
+    "\13\"\3\"\3\"\3\"\3\"\3\"\5\"\u0221\n\"\3#\3#\3#\3#\3#\2\3\36$\2\4\6\b"+
+    "\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,.\60\62\64\668:<>@BD\2\16\3\3\r"+
+    "\r\3\2:E\3\2\35\37\3\2 !\3\2\66\67\3\2\"$\3\2%(\3\2),\3\289\3\2FI\4\2"+
+    "\33\34 !\3\2QR\u025a\2I\3\2\2\2\4T\3\2\2\2\6Y\3\2\2\2\b\u00b7\3\2\2\2"+
+    "\n\u00bb\3\2\2\2\f\u00bd\3\2\2\2\16\u00c6\3\2\2\2\20\u00ca\3\2\2\2\22"+
+    "\u00cc\3\2\2\2\24\u00ce\3\2\2\2\26\u00d7\3\2\2\2\30\u00df\3\2\2\2\32\u00e4"+
+    "\3\2\2\2\34\u00eb\3\2\2\2\36\u00f6\3\2\2\2 \u0160\3\2\2\2\"\u0172\3\2"+
+    "\2\2$\u0187\3\2\2\2&\u018d\3\2\2\2(\u0194\3\2\2\2*\u0196\3\2\2\2,\u019a"+
+    "\3\2\2\2.\u01aa\3\2\2\2\60\u01b9\3\2\2\2\62\u01c1\3\2\2\2\64\u01c9\3\2"+
+    "\2\2\66\u01cb\3\2\2\28\u01cf\3\2\2\2:\u01d3\3\2\2\2<\u01d7\3\2\2\2>\u0201"+
+    "\3\2\2\2@\u0210\3\2\2\2B\u0220\3\2\2\2D\u0222\3\2\2\2FH\5\4\3\2GF\3\2"+
+    "\2\2HK\3\2\2\2IG\3\2\2\2IJ\3\2\2\2JO\3\2\2\2KI\3\2\2\2LN\5\b\5\2ML\3\2"+
+    "\2\2NQ\3\2\2\2OM\3\2\2\2OP\3\2\2\2PR\3\2\2\2QO\3\2\2\2RS\7\2\2\3S\3\3"+
+    "\2\2\2TU\5\26\f\2UV\7P\2\2VW\5\6\4\2WX\5\f\7\2X\5\3\2\2\2Ye\7\t\2\2Z["+
+    "\5\26\f\2[b\7P\2\2\\]\7\f\2\2]^\5\26\f\2^_\7P\2\2_a\3\2\2\2`\\\3\2\2\2"+
+    "ad\3\2\2\2b`\3\2\2\2bc\3\2\2\2cf\3\2\2\2db\3\2\2\2eZ\3\2\2\2ef\3\2\2\2"+
+    "fg\3\2\2\2gh\7\n\2\2h\7\3\2\2\2ij\7\16\2\2jk\7\t\2\2kl\5\36\20\2lm\7\n"+
+    "\2\2mq\5\n\6\2no\7\17\2\2or\5\n\6\2pr\6\5\2\2qn\3\2\2\2qp\3\2\2\2r\u00b8"+
+    "\3\2\2\2st\7\20\2\2tu\7\t\2\2uv\5\36\20\2vy\7\n\2\2wz\5\n\6\2xz\5\16\b"+
+    "\2yw\3\2\2\2yx\3\2\2\2z\u00b8\3\2\2\2{|\7\21\2\2|}\5\f\7\2}~\7\20\2\2"+
+    "~\177\7\t\2\2\177\u0080\5\36\20\2\u0080\u0081\7\n\2\2\u0081\u0082\5\34"+
+    "\17\2\u0082\u00b8\3\2\2\2\u0083\u0084\7\22\2\2\u0084\u0086\7\t\2\2\u0085"+
+    "\u0087\5\20\t\2\u0086\u0085\3\2\2\2\u0086\u0087\3\2\2\2\u0087\u0088\3"+
+    "\2\2\2\u0088\u008a\7\r\2\2\u0089\u008b\5\36\20\2\u008a\u0089\3\2\2\2\u008a"+
+    "\u008b\3\2\2\2\u008b\u008c\3\2\2\2\u008c\u008e\7\r\2\2\u008d\u008f\5\22"+
+    "\n\2\u008e\u008d\3\2\2\2\u008e\u008f\3\2\2\2\u008f\u0090\3\2\2\2\u0090"+
+    "\u0093\7\n\2\2\u0091\u0094\5\n\6\2\u0092\u0094\5\16\b\2\u0093\u0091\3"+
+    "\2\2\2\u0093\u0092\3\2\2\2\u0094\u00b8\3\2\2\2\u0095\u0096\7\22\2\2\u0096"+
+    "\u0097\7\t\2\2\u0097\u0098\5\26\f\2\u0098\u0099\7P\2\2\u0099\u009a\7\63"+
+    "\2\2\u009a\u009b\5\36\20\2\u009b\u009c\7\n\2\2\u009c\u009d\5\n\6\2\u009d"+
+    "\u00b8\3\2\2\2\u009e\u009f\5\24\13\2\u009f\u00a0\5\34\17\2\u00a0\u00b8"+
+    "\3\2\2\2\u00a1\u00a2\7\23\2\2\u00a2\u00b8\5\34\17\2\u00a3\u00a4\7\24\2"+
+    "\2\u00a4\u00b8\5\34\17\2\u00a5\u00a6\7\25\2\2\u00a6\u00a7\5\36\20\2\u00a7"+
+    "\u00a8\5\34\17\2\u00a8\u00b8\3\2\2\2\u00a9\u00aa\7\27\2\2\u00aa\u00ac"+
+    "\5\f\7\2\u00ab\u00ad\5\32\16\2\u00ac\u00ab\3\2\2\2\u00ad\u00ae\3\2\2\2"+
+    "\u00ae\u00ac\3\2\2\2\u00ae\u00af\3\2\2\2\u00af\u00b8\3\2\2\2\u00b0\u00b1"+
+    "\7\31\2\2\u00b1\u00b2\5\36\20\2\u00b2\u00b3\5\34\17\2\u00b3\u00b8\3\2"+
+    "\2\2\u00b4\u00b5\5\36\20\2\u00b5\u00b6\5\34\17\2\u00b6\u00b8\3\2\2\2\u00b7"+
+    "i\3\2\2\2\u00b7s\3\2\2\2\u00b7{\3\2\2\2\u00b7\u0083\3\2\2\2\u00b7\u0095"+
+    "\3\2\2\2\u00b7\u009e\3\2\2\2\u00b7\u00a1\3\2\2\2\u00b7\u00a3\3\2\2\2\u00b7"+
+    "\u00a5\3\2\2\2\u00b7\u00a9\3\2\2\2\u00b7\u00b0\3\2\2\2\u00b7\u00b4\3\2"+
+    "\2\2\u00b8\t\3\2\2\2\u00b9\u00bc\5\f\7\2\u00ba\u00bc\5\b\5\2\u00bb\u00b9"+
+    "\3\2\2\2\u00bb\u00ba\3\2\2\2\u00bc\13\3\2\2\2\u00bd\u00c1\7\5\2\2\u00be"+
+    "\u00c0\5\b\5\2\u00bf\u00be\3\2\2\2\u00c0\u00c3\3\2\2\2\u00c1\u00bf\3\2"+
+    "\2\2\u00c1\u00c2\3\2\2\2\u00c2\u00c4\3\2\2\2\u00c3\u00c1\3\2\2\2\u00c4"+
+    "\u00c5\7\6\2\2\u00c5\r\3\2\2\2\u00c6\u00c7\7\r\2\2\u00c7\17\3\2\2\2\u00c8"+
+    "\u00cb\5\24\13\2\u00c9\u00cb\5\36\20\2\u00ca\u00c8\3\2\2\2\u00ca\u00c9"+
+    "\3\2\2\2\u00cb\21\3\2\2\2\u00cc\u00cd\5\36\20\2\u00cd\23\3\2\2\2\u00ce"+
+    "\u00cf\5\26\f\2\u00cf\u00d4\5\30\r\2\u00d0\u00d1\7\f\2\2\u00d1\u00d3\5"+
+    "\30\r\2\u00d2\u00d0\3\2\2\2\u00d3\u00d6\3\2\2\2\u00d4\u00d2\3\2\2\2\u00d4"+
+    "\u00d5\3\2\2\2\u00d5\25\3\2\2\2\u00d6\u00d4\3\2\2\2\u00d7\u00dc\7O\2\2"+
+    "\u00d8\u00d9\7\7\2\2\u00d9\u00db\7\b\2\2\u00da\u00d8\3\2\2\2\u00db\u00de"+
+    "\3\2\2\2\u00dc\u00da\3\2\2\2\u00dc\u00dd\3\2\2\2\u00dd\27\3\2\2\2\u00de"+
+    "\u00dc\3\2\2\2\u00df\u00e2\7P\2\2\u00e0\u00e1\7:\2\2\u00e1\u00e3\5\36"+
+    "\20\2\u00e2\u00e0\3\2\2\2\u00e2\u00e3\3\2\2\2\u00e3\31\3\2\2\2\u00e4\u00e5"+
+    "\7\30\2\2\u00e5\u00e6\7\t\2\2\u00e6\u00e7\7O\2\2\u00e7\u00e8\7P\2\2\u00e8"+
+    "\u00e9\7\n\2\2\u00e9\u00ea\5\f\7\2\u00ea\33\3\2\2\2\u00eb\u00ec\t\2\2"+
+    "\2\u00ec\35\3\2\2\2\u00ed\u00ee\b\20\1\2\u00ee\u00ef\5\"\22\2\u00ef\u00f0"+
+    "\t\3\2\2\u00f0\u00f1\5\36\20\3\u00f1\u00f2\b\20\1\2\u00f2\u00f7\3\2\2"+
+    "\2\u00f3\u00f4\5 \21\2\u00f4\u00f5\b\20\1\2\u00f5\u00f7\3\2\2\2\u00f6"+
+    "\u00ed\3\2\2\2\u00f6\u00f3\3\2\2\2\u00f7\u0138\3\2\2\2\u00f8\u00f9\f\17"+
+    "\2\2\u00f9\u00fa\t\4\2\2\u00fa\u00fb\5\36\20\20\u00fb\u00fc\b\20\1\2\u00fc"+
+    "\u0137\3\2\2\2\u00fd\u00fe\f\16\2\2\u00fe\u00ff\t\5\2\2\u00ff\u0100\5"+
+    "\36\20\17\u0100\u0101\b\20\1\2\u0101\u0137\3\2\2\2\u0102\u0103\f\r\2\2"+
+    "\u0103\u0104\t\6\2\2\u0104\u0105\5\36\20\16\u0105\u0106\b\20\1\2\u0106"+
+    "\u0137\3\2\2\2\u0107\u0108\f\f\2\2\u0108\u0109\t\7\2\2\u0109\u010a\5\36"+
+    "\20\r\u010a\u010b\b\20\1\2\u010b\u0137\3\2\2\2\u010c\u010d\f\13\2\2\u010d"+
+    "\u010e\t\b\2\2\u010e\u010f\5\36\20\f\u010f\u0110\b\20\1\2\u0110\u0137"+
+    "\3\2\2\2\u0111\u0112\f\n\2\2\u0112\u0113\t\t\2\2\u0113\u0114\5\36\20\13"+
+    "\u0114\u0115\b\20\1\2\u0115\u0137\3\2\2\2\u0116\u0117\f\t\2\2\u0117\u0118"+
+    "\7-\2\2\u0118\u0119\5\36\20\n\u0119\u011a\b\20\1\2\u011a\u0137\3\2\2\2"+
+    "\u011b\u011c\f\b\2\2\u011c\u011d\7.\2\2\u011d\u011e\5\36\20\t\u011e\u011f"+
+    "\b\20\1\2\u011f\u0137\3\2\2\2\u0120\u0121\f\7\2\2\u0121\u0122\7/\2\2\u0122"+
+    "\u0123\5\36\20\b\u0123\u0124\b\20\1\2\u0124\u0137\3\2\2\2\u0125\u0126"+
+    "\f\6\2\2\u0126\u0127\7\60\2\2\u0127\u0128\5\36\20\7\u0128\u0129\b\20\1"+
+    "\2\u0129\u0137\3\2\2\2\u012a\u012b\f\5\2\2\u012b\u012c\7\61\2\2\u012c"+
+    "\u012d\5\36\20\6\u012d\u012e\b\20\1\2\u012e\u0137\3\2\2\2\u012f\u0130"+
+    "\f\4\2\2\u0130\u0131\7\62\2\2\u0131\u0132\5\36\20\2\u0132\u0133\7\63\2"+
+    "\2\u0133\u0134\5\36\20\4\u0134\u0135\b\20\1\2\u0135\u0137\3\2\2\2\u0136"+
+    "\u00f8\3\2\2\2\u0136\u00fd\3\2\2\2\u0136\u0102\3\2\2\2\u0136\u0107\3\2"+
+    "\2\2\u0136\u010c\3\2\2\2\u0136\u0111\3\2\2\2\u0136\u0116\3\2\2\2\u0136"+
+    "\u011b\3\2\2\2\u0136\u0120\3\2\2\2\u0136\u0125\3\2\2\2\u0136\u012a\3\2"+
+    "\2\2\u0136\u012f\3\2\2\2\u0137\u013a\3\2\2\2\u0138\u0136\3\2\2\2\u0138"+
+    "\u0139\3\2\2\2\u0139\37\3\2\2\2\u013a\u0138\3\2\2\2\u013b\u013c\6\21\17"+
+    "\3\u013c\u013d\t\n\2\2\u013d\u0161\5\"\22\2\u013e\u013f\6\21\20\3\u013f"+
+    "\u0140\5\"\22\2\u0140\u0141\t\n\2\2\u0141\u0161\3\2\2\2\u0142\u0143\6"+
+    "\21\21\3\u0143\u0161\5\"\22\2\u0144\u0145\6\21\22\3\u0145\u0146\t\13\2"+
+    "\2\u0146\u0161\b\21\1\2\u0147\u0148\6\21\23\3\u0148\u0149\7L\2\2\u0149"+
+    "\u0161\b\21\1\2\u014a\u014b\6\21\24\3\u014b\u014c\7M\2\2\u014c\u0161\b"+
+    "\21\1\2\u014d\u014e\6\21\25\3\u014e\u014f\7N\2\2\u014f\u0161\b\21\1\2"+
+    "\u0150\u0151\6\21\26\3\u0151\u0152\5@!\2\u0152\u0153\b\21\1\2\u0153\u0161"+
+    "\3\2\2\2\u0154\u0155\6\21\27\3\u0155\u0156\5B\"\2\u0156\u0157\b\21\1\2"+
+    "\u0157\u0161\3\2\2\2\u0158\u0159\6\21\30\3\u0159\u015a\t\f\2\2\u015a\u0161"+
+    "\5 \21\2\u015b\u015c\7\t\2\2\u015c\u015d\5\26\f\2\u015d\u015e\7\n\2\2"+
+    "\u015e\u015f\5 \21\2\u015f\u0161\3\2\2\2\u0160\u013b\3\2\2\2\u0160\u013e"+
+    "\3\2\2\2\u0160\u0142\3\2\2\2\u0160\u0144\3\2\2\2\u0160\u0147\3\2\2\2\u0160"+
+    "\u014a\3\2\2\2\u0160\u014d\3\2\2\2\u0160\u0150\3\2\2\2\u0160\u0154\3\2"+
+    "\2\2\u0160\u0158\3\2\2\2\u0160\u015b\3\2\2\2\u0161!\3\2\2\2\u0162\u0166"+
+    "\5$\23\2\u0163\u0165\5&\24\2\u0164\u0163\3\2\2\2\u0165\u0168\3\2\2\2\u0166"+
+    "\u0164\3\2\2\2\u0166\u0167\3\2\2\2\u0167\u0173\3\2\2\2\u0168\u0166\3\2"+
+    "\2\2\u0169\u016a\5\26\f\2\u016a\u016e\5(\25\2\u016b\u016d\5&\24\2\u016c"+
+    "\u016b\3\2\2\2\u016d\u0170\3\2\2\2\u016e\u016c\3\2\2\2\u016e\u016f\3\2"+
+    "\2\2\u016f\u0173\3\2\2\2\u0170\u016e\3\2\2\2\u0171\u0173\5> \2\u0172\u0162"+
+    "\3\2\2\2\u0172\u0169\3\2\2\2\u0172\u0171\3\2\2\2\u0173#\3\2\2\2\u0174"+
+    "\u0175\6\23\31\3\u0175\u0176\7\t\2\2\u0176\u0177\5\36\20\2\u0177\u0178"+
+    "\7\n\2\2\u0178\u0179\b\23\1\2\u0179\u0188\3\2\2\2\u017a\u017b\6\23\32"+
+    "\3\u017b\u017c\7\t\2\2\u017c\u017d\5 \21\2\u017d\u017e\7\n\2\2\u017e\u0188"+
+    "\3\2\2\2\u017f\u0188\7J\2\2\u0180\u0188\7K\2\2\u0181\u0188\7P\2\2\u0182"+
+    "\u0183\7P\2\2\u0183\u0188\5,\27\2\u0184\u0185\7\26\2\2\u0185\u0186\7O"+
+    "\2\2\u0186\u0188\5,\27\2\u0187\u0174\3\2\2\2\u0187\u017a\3\2\2\2\u0187"+
+    "\u017f\3\2\2\2\u0187\u0180\3\2\2\2\u0187\u0181\3\2\2\2\u0187\u0182\3\2"+
+    "\2\2\u0187\u0184\3\2\2\2\u0188%\3\2\2\2\u0189\u018a\6\24\33\3\u018a\u018e"+
+    "\5(\25\2\u018b\u018c\6\24\34\3\u018c\u018e\5*\26\2\u018d\u0189\3\2\2\2"+
+    "\u018d\u018b\3\2\2\2\u018e\'\3\2\2\2\u018f\u0190\7\13\2\2\u0190\u0191"+
+    "\7R\2\2\u0191\u0195\5,\27\2\u0192\u0193\7\13\2\2\u0193\u0195\t\r\2\2\u0194"+
+    "\u018f\3\2\2\2\u0194\u0192\3\2\2\2\u0195)\3\2\2\2\u0196\u0197\7\7\2\2"+
+    "\u0197\u0198\5\36\20\2\u0198\u0199\7\b\2\2\u0199+\3\2\2\2\u019a\u01a3"+
+    "\7\t\2\2\u019b\u01a0\5.\30\2\u019c\u019d\7\f\2\2\u019d\u019f\5.\30\2\u019e"+
+    "\u019c\3\2\2\2\u019f\u01a2\3\2\2\2\u01a0\u019e\3\2\2\2\u01a0\u01a1\3\2"+
+    "\2\2\u01a1\u01a4\3\2\2\2\u01a2\u01a0\3\2\2\2\u01a3\u019b\3\2\2\2\u01a3"+
+    "\u01a4\3\2\2\2\u01a4\u01a5\3\2\2\2\u01a5\u01a6\7\n\2\2\u01a6-\3\2\2\2"+
+    "\u01a7\u01ab\5\36\20\2\u01a8\u01ab\5\60\31\2\u01a9\u01ab\5\64\33\2\u01aa"+
+    "\u01a7\3\2\2\2\u01aa\u01a8\3\2\2\2\u01aa\u01a9\3\2\2\2\u01ab/\3\2\2\2"+
+    "\u01ac\u01ba\5\62\32\2\u01ad\u01b6\7\t\2\2\u01ae\u01b3\5\62\32\2\u01af"+
+    "\u01b0\7\f\2\2\u01b0\u01b2\5\62\32\2\u01b1\u01af\3\2\2\2\u01b2\u01b5\3"+
+    "\2\2\2\u01b3\u01b1\3\2\2\2\u01b3\u01b4\3\2\2\2\u01b4\u01b7\3\2\2\2\u01b5"+
+    "\u01b3\3\2\2\2\u01b6\u01ae\3\2\2\2\u01b6\u01b7\3\2\2\2\u01b7\u01b8\3\2"+
+    "\2\2\u01b8\u01ba\7\n\2\2\u01b9\u01ac\3\2\2\2\u01b9\u01ad\3\2\2\2\u01ba"+
+    "\u01bb\3\2\2\2\u01bb\u01be\7\65\2\2\u01bc\u01bf\5\f\7\2\u01bd\u01bf\5"+
+    "\36\20\2\u01be\u01bc\3\2\2\2\u01be\u01bd\3\2\2\2\u01bf\61\3\2\2\2\u01c0"+
+    "\u01c2\5\26\f\2\u01c1\u01c0\3\2\2\2\u01c1\u01c2\3\2\2\2\u01c2\u01c3\3"+
+    "\2\2\2\u01c3\u01c4\7P\2\2\u01c4\63\3\2\2\2\u01c5\u01ca\5\66\34\2\u01c6"+
+    "\u01ca\58\35\2\u01c7\u01ca\5:\36\2\u01c8\u01ca\5<\37\2\u01c9\u01c5\3\2"+
+    "\2\2\u01c9\u01c6\3\2\2\2\u01c9\u01c7\3\2\2\2\u01c9\u01c8\3\2\2\2\u01ca"+
+    "\65\3\2\2\2\u01cb\u01cc\7O\2\2\u01cc\u01cd\7\64\2\2\u01cd\u01ce\7P\2\2"+
+    "\u01ce\67\3\2\2\2\u01cf\u01d0\5\26\f\2\u01d0\u01d1\7\64\2\2\u01d1\u01d2"+
+    "\7\26\2\2\u01d29\3\2\2\2\u01d3\u01d4\7P\2\2\u01d4\u01d5\7\64\2\2\u01d5"+
+    "\u01d6\7P\2\2\u01d6;\3\2\2\2\u01d7\u01d8\7\32\2\2\u01d8\u01d9\7\64\2\2"+
+    "\u01d9\u01da\7P\2\2\u01da=\3\2\2\2\u01db\u01dc\7\26\2\2\u01dc\u01e1\7"+
+    "O\2\2\u01dd\u01de\7\7\2\2\u01de\u01df\5\36\20\2\u01df\u01e0\7\b\2\2\u01e0"+
+    "\u01e2\3\2\2\2\u01e1\u01dd\3\2\2\2\u01e2\u01e3\3\2\2\2\u01e3\u01e1\3\2"+
+    "\2\2\u01e3\u01e4\3\2\2\2\u01e4\u01ec\3\2\2\2\u01e5\u01e9\5(\25\2\u01e6"+
+    "\u01e8\5&\24\2\u01e7\u01e6\3\2\2\2\u01e8\u01eb\3\2\2\2\u01e9\u01e7\3\2"+
+    "\2\2\u01e9\u01ea\3\2\2\2\u01ea\u01ed\3\2\2\2\u01eb\u01e9\3\2\2\2\u01ec"+
+    "\u01e5\3\2\2\2\u01ec\u01ed\3\2\2\2\u01ed\u0202\3\2\2\2\u01ee\u01ef\7\26"+
+    "\2\2\u01ef\u01f0\7O\2\2\u01f0\u01f1\7\7\2\2\u01f1\u01f2\7\b\2\2\u01f2"+
+    "\u01fb\7\5\2\2\u01f3\u01f8\5\36\20\2\u01f4\u01f5\7\f\2\2\u01f5\u01f7\5"+
+    "\36\20\2\u01f6\u01f4\3\2\2\2\u01f7\u01fa\3\2\2\2\u01f8\u01f6\3\2\2\2\u01f8"+
+    "\u01f9\3\2\2\2\u01f9\u01fc\3\2\2\2\u01fa\u01f8\3\2\2\2\u01fb\u01f3\3\2"+
+    "\2\2\u01fb\u01fc\3\2\2\2\u01fc\u01fe\3\2\2\2\u01fd\u01ff\7\r\2\2\u01fe"+
+    "\u01fd\3\2\2\2\u01fe\u01ff\3\2\2\2\u01ff\u0200\3\2\2\2\u0200\u0202\7\6"+
+    "\2\2\u0201\u01db\3\2\2\2\u0201\u01ee\3\2\2\2\u0202?\3\2\2\2\u0203\u0204"+
+    "\7\7\2\2\u0204\u0209\5\36\20\2\u0205\u0206\7\f\2\2\u0206\u0208\5\36\20"+
+    "\2\u0207\u0205\3\2\2\2\u0208\u020b\3\2\2\2\u0209\u0207\3\2\2\2\u0209\u020a"+
+    "\3\2\2\2\u020a\u020c\3\2\2\2\u020b\u0209\3\2\2\2\u020c\u020d\7\b\2\2\u020d"+
+    "\u0211\3\2\2\2\u020e\u020f\7\7\2\2\u020f\u0211\7\b\2\2\u0210\u0203\3\2"+
+    "\2\2\u0210\u020e\3\2\2\2\u0211A\3\2\2\2\u0212\u0213\7\7\2\2\u0213\u0218"+
+    "\5D#\2\u0214\u0215\7\f\2\2\u0215\u0217\5D#\2\u0216\u0214\3\2\2\2\u0217"+
+    "\u021a\3\2\2\2\u0218\u0216\3\2\2\2\u0218\u0219\3\2\2\2\u0219\u021b\3\2"+
+    "\2\2\u021a\u0218\3\2\2\2\u021b\u021c\7\b\2\2\u021c\u0221\3\2\2\2\u021d"+
+    "\u021e\7\7\2\2\u021e\u021f\7\63\2\2\u021f\u0221\7\b\2\2\u0220\u0212\3"+
+    "\2\2\2\u0220\u021d\3\2\2\2\u0221C\3\2\2\2\u0222\u0223\5\36\20\2\u0223"+
+    "\u0224\7\63\2\2\u0224\u0225\5\36\20\2\u0225E\3\2\2\2\62IObeqy\u0086\u008a"+
+    "\u008e\u0093\u00ae\u00b7\u00bb\u00c1\u00ca\u00d4\u00dc\u00e2\u00f6\u0136"+
+    "\u0138\u0160\u0166\u016e\u0172\u0187\u018d\u0194\u01a0\u01a3\u01aa\u01b3"+
+    "\u01b6\u01b9\u01be\u01c1\u01c9\u01e3\u01e9\u01ec\u01f8\u01fb\u01fe\u0201"+
+    "\u0209\u0210\u0218\u0220";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
index 4b7daa213b3..498324441e6 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
@@ -277,6 +277,20 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitNull(PainlessParser.NullContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitListinit(PainlessParser.ListinitContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitMapinit(PainlessParser.MapinitContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
@@ -452,4 +466,39 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitLocalFuncref(PainlessParser.LocalFuncrefContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitNewstandardarray(PainlessParser.NewstandardarrayContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitNewinitializedarray(PainlessParser.NewinitializedarrayContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitListinitializer(PainlessParser.ListinitializerContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitMapinitializer(PainlessParser.MapinitializerContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitMaptoken(PainlessParser.MaptokenContext ctx) { return visitChildren(ctx); }
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
index f39bc84a7af..e57a274f8ef 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
@@ -263,6 +263,20 @@ interface PainlessParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitNull(PainlessParser.NullContext ctx);
+  /**
+   * Visit a parse tree produced by the {@code listinit}
+   * labeled alternative in {@link PainlessParser#unary}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitListinit(PainlessParser.ListinitContext ctx);
+  /**
+   * Visit a parse tree produced by the {@code mapinit}
+   * labeled alternative in {@link PainlessParser#unary}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitMapinit(PainlessParser.MapinitContext ctx);
   /**
    * Visit a parse tree produced by the {@code operator}
    * labeled alternative in {@link PainlessParser#unary}.
@@ -428,4 +442,36 @@ interface PainlessParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitLocalFuncref(PainlessParser.LocalFuncrefContext ctx);
+  /**
+   * Visit a parse tree produced by the {@code newstandardarray}
+   * labeled alternative in {@link PainlessParser#arrayinitializer}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitNewstandardarray(PainlessParser.NewstandardarrayContext ctx);
+  /**
+   * Visit a parse tree produced by the {@code newinitializedarray}
+   * labeled alternative in {@link PainlessParser#arrayinitializer}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitNewinitializedarray(PainlessParser.NewinitializedarrayContext ctx);
+  /**
+   * Visit a parse tree produced by {@link PainlessParser#listinitializer}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitListinitializer(PainlessParser.ListinitializerContext ctx);
+  /**
+   * Visit a parse tree produced by {@link PainlessParser#mapinitializer}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitMapinitializer(PainlessParser.MapinitializerContext ctx);
+  /**
+   * Visit a parse tree produced by {@link PainlessParser#maptoken}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitMaptoken(PainlessParser.MaptokenContext ctx);
 }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index a84247afe25..20384b57145 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -30,6 +30,15 @@ import org.antlr.v4.runtime.atn.PredictionMode;
 import org.antlr.v4.runtime.tree.TerminalNode;
 import org.elasticsearch.painless.CompilerSettings;
 import org.elasticsearch.painless.Globals;
+import org.elasticsearch.painless.antlr.PainlessParser.ListinitContext;
+import org.elasticsearch.painless.antlr.PainlessParser.ListinitializerContext;
+import org.elasticsearch.painless.antlr.PainlessParser.MapinitContext;
+import org.elasticsearch.painless.antlr.PainlessParser.MapinitializerContext;
+import org.elasticsearch.painless.antlr.PainlessParser.MaptokenContext;
+import org.elasticsearch.painless.antlr.PainlessParser.NewinitializedarrayContext;
+import org.elasticsearch.painless.antlr.PainlessParser.NewstandardarrayContext;
+import org.elasticsearch.painless.node.EListInit;
+import org.elasticsearch.painless.node.EMapInit;
 import org.elasticsearch.painless.node.SFunction.Reserved;
 import org.elasticsearch.painless.node.SSource.MainMethodReserved;
 import org.elasticsearch.painless.node.SFunction.FunctionReserved;
@@ -241,7 +250,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
             statements.add((AStatement)visit(statement));
         }
 
-        return new SSource(settings, sourceName, sourceText, debugStream, (MainMethodReserved)reserved.pop(), 
+        return new SSource(settings, sourceName, sourceText, debugStream, (MainMethodReserved)reserved.pop(),
                            location(ctx), functions, globals, statements);
     }
 
@@ -267,7 +276,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
             statements.add((AStatement)visit(statement));
         }
 
-        return new SFunction((FunctionReserved)reserved.pop(), location(ctx), rtnType, name, 
+        return new SFunction((FunctionReserved)reserved.pop(), location(ctx), rtnType, name,
                              paramTypes, paramNames, statements, false);
     }
 
@@ -720,6 +729,16 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         return new ENull(location(ctx));
     }
 
+    @Override
+    public Object visitListinit(ListinitContext ctx) {
+        return visit(ctx.listinitializer());
+    }
+
+    @Override
+    public Object visitMapinit(MapinitContext ctx) {
+        return visit(ctx.mapinitializer());
+    }
+
     @Override
     public Object visitOperator(OperatorContext ctx) {
         if (ctx.SUB() != null && ctx.unary() instanceof NumericContext) {
@@ -797,27 +816,7 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
 
     @Override
     public Object visitNewarray(NewarrayContext ctx) {
-        String type = ctx.TYPE().getText();
-        List<AExpression> expressions = new ArrayList<>();
-
-        for (ExpressionContext expression : ctx.expression()) {
-            expressions.add((AExpression)visitExpression(expression));
-        }
-
-        List<ALink> links = new ArrayList<>();
-        links.add(new LNewArray(location(ctx), type, expressions));
-
-        if (ctx.dot() != null) {
-            links.add((ALink)visit(ctx.dot()));
-
-            for (SecondaryContext secondary : ctx.secondary()) {
-                links.add((ALink)visit(secondary));
-            }
-        } else if (!ctx.secondary().isEmpty()) {
-            throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
-        }
-
-        return links;
+        return visit(ctx.arrayinitializer());
     }
 
     @Override
@@ -978,9 +977,9 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
                 statements.add((AStatement)visit(statement));
             }
         }
-        
+
         String name = nextLambda();
-        return new ELambda(name, (FunctionReserved)reserved.pop(), location(ctx), 
+        return new ELambda(name, (FunctionReserved)reserved.pop(), location(ctx),
                            paramTypes, paramNames, statements);
     }
 
@@ -1016,13 +1015,13 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
             // taking integer as argument and returning a new instance, and return a ref to that.
             Location location = location(ctx);
             String arrayType = ctx.decltype().getText();
-            SReturn code = new SReturn(location, 
+            SReturn code = new SReturn(location,
                            new EChain(location,
                            new LNewArray(location, arrayType, Arrays.asList(
-                           new EChain(location, 
-                           new LVariable(location, "size"))))));
+                           new EChain(location,
+                           new LVariable(location, "size"))), false)));
             String name = nextLambda();
-            globals.addSyntheticMethod(new SFunction(new FunctionReserved(), location, arrayType, name, 
+            globals.addSyntheticMethod(new SFunction(new FunctionReserved(), location, arrayType, name,
                                        Arrays.asList("int"), Arrays.asList("size"), Arrays.asList(code), true));
             return new EFunctionRef(location(ctx), "this", name);
         }
@@ -1038,7 +1037,76 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
     public Object visitLocalFuncref(LocalFuncrefContext ctx) {
         return new EFunctionRef(location(ctx), ctx.THIS().getText(), ctx.ID().getText());
     }
-    
+
+    @Override
+    public Object visitNewstandardarray(NewstandardarrayContext ctx) {
+        String type = ctx.TYPE().getText();
+        List<AExpression> expressions = new ArrayList<>();
+
+        for (ExpressionContext expression : ctx.expression()) {
+            expressions.add((AExpression)visitExpression(expression));
+        }
+
+        List<ALink> links = new ArrayList<>();
+        links.add(new LNewArray(location(ctx), type, expressions, false));
+
+        if (ctx.dot() != null) {
+            links.add((ALink)visit(ctx.dot()));
+
+            for (SecondaryContext secondary : ctx.secondary()) {
+                links.add((ALink)visit(secondary));
+            }
+        } else if (!ctx.secondary().isEmpty()) {
+            throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
+        }
+
+        return links;
+    }
+
+    @Override
+    public Object visitNewinitializedarray(NewinitializedarrayContext ctx) {
+        String type = ctx.TYPE().getText();
+        List<AExpression> expressions = new ArrayList<>();
+
+        for (ExpressionContext expression : ctx.expression()) {
+            expressions.add((AExpression)visitExpression(expression));
+        }
+
+        List<ALink> links = new ArrayList<>();
+        links.add(new LNewArray(location(ctx), type, expressions, true));
+
+        return links;
+    }
+
+    @Override
+    public Object visitListinitializer(ListinitializerContext ctx) {
+        List<AExpression> values = new ArrayList<>();
+
+        for (ExpressionContext expression : ctx.expression()) {
+            values.add((AExpression)visitExpression(expression));
+        }
+
+        return new EListInit(location(ctx), values);
+    }
+
+    @Override
+    public Object visitMapinitializer(MapinitializerContext ctx) {
+        List<AExpression> keys = new ArrayList<>();
+        List<AExpression> values = new ArrayList<>();
+
+        for (MaptokenContext maptoken : ctx.maptoken()) {
+            keys.add((AExpression)visitExpression(maptoken.expression(0)));
+            values.add((AExpression)visitExpression(maptoken.expression(1)));
+        }
+
+        return new EMapInit(location(ctx), keys, values);
+    }
+
+    @Override
+    public Object visitMaptoken(MaptokenContext ctx) {
+        throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
+    }
+
     /** Returns name of next lambda */
     private String nextLambda() {
         return "lambda$" + syntheticCounter++;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
new file mode 100644
index 00000000000..cb4d2d3a216
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Method;
+import org.elasticsearch.painless.Definition.MethodKey;
+import org.elasticsearch.painless.Globals;
+import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Location;
+import org.elasticsearch.painless.MethodWriter;
+
+import java.util.List;
+import java.util.Set;
+
+public class EListInit extends AExpression {
+    final List<AExpression> values;
+
+    Method constructor = null;
+    Method method = null;
+
+    public EListInit(Location location, List<AExpression> values) {
+        super(location);
+
+        this.values = values;
+    }
+
+    @Override
+    void extractVariables(Set<String> variables) {
+        for (AExpression value : values) {
+            value.extractVariables(variables);
+        }
+    }
+
+    @Override
+    void analyze(Locals locals) {
+        try {
+            actual = Definition.getType("ArrayList");
+        } catch (IllegalArgumentException exception) {
+            throw createError(new IllegalStateException("Illegal tree structure."));
+        }
+
+        constructor = actual.struct.constructors.get(new MethodKey("<init>", 0));
+
+        if (constructor == null) {
+            throw createError(new IllegalStateException("Illegal tree structure."));
+        }
+
+        method = actual.struct.methods.get(new MethodKey("add", 1));
+
+        if (method == null) {
+            throw createError(new IllegalStateException("Illegal tree structure."));
+        }
+
+        for (int index = 0; index < values.size(); ++index) {
+            AExpression expression = values.get(index);
+
+            expression.expected = Definition.DEF_TYPE;
+            expression.internal = true;
+            expression.analyze(locals);
+            values.set(index, expression.cast(locals));
+        }
+    }
+
+    @Override
+    void write(MethodWriter writer, Globals globals) {
+        writer.writeDebugInfo(location);
+
+        writer.newInstance(actual.type);
+        writer.dup();
+        writer.invokeConstructor(constructor.owner.type, constructor.method);
+
+        for (AExpression value : values) {
+            writer.dup();
+            value.write(writer, globals);
+            writer.invokeInterface(method.owner.type, method.method);
+            writer.pop();
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
new file mode 100644
index 00000000000..c583bc79e52
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Definition.Method;
+import org.elasticsearch.painless.Definition.MethodKey;
+import org.elasticsearch.painless.Globals;
+import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Location;
+import org.elasticsearch.painless.MethodWriter;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+public class EMapInit extends AExpression {
+    final List<AExpression> keys;
+    final List<AExpression> values;
+
+    Method constructor = null;
+    Method method = null;
+
+    public EMapInit(Location location, List<AExpression> keys, List<AExpression> values) {
+        super(location);
+
+        this.keys = keys;
+        this.values = values;
+    }
+
+    @Override
+    void extractVariables(Set<String> variables) {
+        for (AExpression key : keys) {
+            key.extractVariables(variables);
+        }
+
+        for (AExpression value : values) {
+            value.extractVariables(variables);
+        }
+    }
+
+    @Override
+    void analyze(Locals locals) {
+        try {
+            actual = Definition.getType("HashMap");
+        } catch (IllegalArgumentException exception) {
+            throw createError(new IllegalStateException("Illegal tree structure."));
+        }
+
+        constructor = actual.struct.constructors.get(new MethodKey("<init>", 0));
+
+        if (constructor == null) {
+            throw createError(new IllegalStateException("Illegal tree structure."));
+        }
+
+        method = actual.struct.methods.get(new MethodKey("put", 2));
+
+        if (method == null) {
+            throw createError(new IllegalStateException("Illegal tree structure."));
+        }
+
+        if (keys.size() != values.size()) {
+            throw createError(new IllegalStateException("Illegal tree structure."));
+        }
+
+        for (int index = 0; index < keys.size(); ++index) {
+            AExpression expression = keys.get(index);
+
+            expression.expected = Definition.DEF_TYPE;
+            expression.internal = true;
+            expression.analyze(locals);
+            keys.set(index, expression.cast(locals));
+        }
+
+        for (int index = 0; index < values.size(); ++index) {
+            AExpression expression = values.get(index);
+
+            expression.expected = Definition.DEF_TYPE;
+            expression.internal = true;
+            expression.analyze(locals);
+            values.set(index, expression.cast(locals));
+        }
+    }
+
+    @Override
+    void write(MethodWriter writer, Globals globals) {
+        writer.writeDebugInfo(location);
+
+        writer.newInstance(actual.type);
+        writer.dup();
+        writer.invokeConstructor(constructor.owner.type, constructor.method);
+
+        for (int index = 0; index < keys.size(); ++index) {
+            AExpression key = keys.get(index);
+            AExpression value = values.get(index);
+
+            writer.dup();
+            key.write(writer, globals);
+            value.write(writer, globals);
+            writer.invokeInterface(method.owner.type, method.method);
+            writer.pop();
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
index cf83599210f..f94f344c481 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/LNewArray.java
@@ -37,14 +37,16 @@ public final class LNewArray extends ALink {
 
     final String type;
     final List<AExpression> arguments;
+    final boolean initialize;
 
-    public LNewArray(Location location, String type, List<AExpression> arguments) {
+    public LNewArray(Location location, String type, List<AExpression> arguments, boolean initialize) {
         super(location, -1);
 
         this.type = Objects.requireNonNull(type);
         this.arguments = Objects.requireNonNull(arguments);
+        this.initialize = initialize;
     }
-    
+
     @Override
     void extractVariables(Set<String> variables) {
         for (AExpression argument : arguments) {
@@ -73,12 +75,13 @@ public final class LNewArray extends ALink {
         for (int argument = 0; argument < arguments.size(); ++argument) {
             AExpression expression = arguments.get(argument);
 
-            expression.expected = Definition.INT_TYPE;
+            expression.expected = initialize ? Definition.getType(type.struct, 0) : Definition.INT_TYPE;
+            expression.internal = true;
             expression.analyze(locals);
             arguments.set(argument, expression.cast(locals));
         }
 
-        after = Definition.getType(type.struct, arguments.size());
+        after = Definition.getType(type.struct, initialize ? 1 : arguments.size());
 
         return this;
     }
@@ -92,14 +95,28 @@ public final class LNewArray extends ALink {
     void load(MethodWriter writer, Globals globals) {
         writer.writeDebugInfo(location);
 
-        for (AExpression argument : arguments) {
-            argument.write(writer, globals);
-        }
-
-        if (arguments.size() > 1) {
-            writer.visitMultiANewArrayInsn(after.type.getDescriptor(), after.type.getDimensions());
-        } else {
+        if (initialize) {
+            writer.push(arguments.size());
             writer.newArray(Definition.getType(after.struct, 0).type);
+
+            for (int index = 0; index < arguments.size(); ++index) {
+                AExpression argument = arguments.get(index);
+
+                writer.dup();
+                writer.push(index);
+                argument.write(writer, globals);
+                writer.arrayStore(Definition.getType(after.struct, 0).type);
+            }
+        } else {
+            for (AExpression argument : arguments) {
+                argument.write(writer, globals);
+            }
+
+            if (arguments.size() > 1) {
+                writer.visitMultiANewArrayInsn(after.type.getDescriptor(), after.type.getDimensions());
+            } else {
+                writer.newArray(Definition.getType(after.struct, 0).type);
+            }
         }
     }
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/InitializerTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/InitializerTests.java
new file mode 100644
index 00000000000..282ad8a5f13
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/InitializerTests.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class InitializerTests extends ScriptTestCase {
+
+    public void testArrayInitializers() {
+        int[] ints = (int[])exec("new int[] {}");
+
+        assertEquals(0, ints.length);
+
+        ints = (int[])exec("new int[] {5, 7, -1, 14}");
+
+        assertEquals(4, ints.length);
+        assertEquals(5, ints[0]);
+        assertEquals(7, ints[1]);
+        assertEquals(-1, ints[2]);
+        assertEquals(14, ints[3]);
+
+        ints = (int[])exec("int y = 2; int z = 3; int[] x = new int[] {y*z, y + z, y - z, y, z}; return x;");
+
+        assertEquals(5, ints.length);
+        assertEquals(6, ints[0]);
+        assertEquals(5, ints[1]);
+        assertEquals(-1, ints[2]);
+        assertEquals(2, ints[3]);
+        assertEquals(3, ints[4]);
+
+        Object[] objects = (Object[])exec("int y = 2; List z = new ArrayList(); String s = 'aaa';" +
+            "Object[] x = new Object[] {y, z, 1 + s, s + 'aaa'}; return x;");
+
+        assertEquals(4, objects.length);
+        assertEquals(new Integer(2), objects[0]);
+        assertEquals(new ArrayList(), objects[1]);
+        assertEquals("1aaa", objects[2]);
+        assertEquals("aaaaaa", objects[3]);
+    }
+
+    public void testListInitializers() {
+        List list = (List)exec("[]");
+
+        assertEquals(0, list.size());
+
+        list = (List)exec("[5, 7, -1, 14]");
+
+        assertEquals(4, list.size());
+        assertEquals(5, list.get(0));
+        assertEquals(7, list.get(1));
+        assertEquals(-1, list.get(2));
+        assertEquals(14, list.get(3));
+
+        list = (List)exec("int y = 2; int z = 3; def x = [y*z, y + z, y - z, y, z]; return x;");
+
+        assertEquals(5, list.size());
+        assertEquals(6, list.get(0));
+        assertEquals(5, list.get(1));
+        assertEquals(-1, list.get(2));
+        assertEquals(2, list.get(3));
+        assertEquals(3, list.get(4));
+
+        list = (List)exec("int y = 2; List z = new ArrayList(); String s = 'aaa'; List x = [y, z, 1 + s, s + 'aaa']; return x;");
+
+        assertEquals(4, list.size());
+        assertEquals(new Integer(2), list.get(0));
+        assertEquals(new ArrayList(),  list.get(1));
+        assertEquals("1aaa",  list.get(2));
+        assertEquals("aaaaaa", list.get(3));
+    }
+
+    public void testMapInitializers() {
+        Map map = (Map)exec("[:]");
+
+        assertEquals(0, map.size());
+
+        map = (Map)exec("[5 : 7, -1 : 14]");
+
+        assertEquals(2, map.size());
+        assertEquals(new Integer(7), map.get(5));
+        assertEquals(new Integer(14), map.get(-1));
+
+        map = (Map)exec("int y = 2; int z = 3; Map x = [y*z : y + z, y - z : y, z : z]; return x;");
+
+        assertEquals(3, map.size());
+        assertEquals(new Integer(5), map.get(6));
+        assertEquals(new Integer(2), map.get(-1));
+        assertEquals(new Integer(3), map.get(3));
+
+        map = (Map)exec("int y = 2; List z = new ArrayList(); String s = 'aaa';" +
+            "def x = [y : z, 1 + s : s + 'aaa']; return x;");
+
+        assertEquals(2, map.size());
+        assertEquals(new ArrayList(), map.get(2));
+        assertEquals("aaaaaa", map.get("1aaa"));
+    }
+
+    public void testCrazyInitializer() {
+        Map map = (Map)exec("int y = 2; int z = 3; Map x = [y*z : y + z, 's' : [y, [y : [[z], [], [:]]]], z : [z, 9]]; return x;");
+
+        List<Object> list0 = new ArrayList<>();
+        list0.add(3);
+        List<Object> list1 = new ArrayList();
+        list1.add(list0);
+        list1.add(new ArrayList());
+        list1.add(new HashMap());
+        Map<Object, Object> map0 = new HashMap<>();
+        map0.put(2, list1);
+        List<Object> list2 = new ArrayList();
+        list2.add(2);
+        list2.add(map0);
+
+        List<Object> list3 = new ArrayList<>();
+        list3.add(3);
+        list3.add(9);
+
+        assertEquals(3, map.size());
+        assertEquals(new Integer(5), map.get(6));
+        assertEquals(list2, map.get("s"));
+        assertEquals(list3, map.get(3));
+    }
+}

From 346b1802ee61db362e6d43e65cb9be3d046da7b8 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Tue, 21 Jun 2016 12:53:04 -0700
Subject: [PATCH 1008/1311] Quick fixes for using built in method writes.

---
 .../elasticsearch/painless/node/EListInit.java   |  2 +-
 .../elasticsearch/painless/node/EMapInit.java    |  3 +--
 .../elasticsearch/painless/InitializerTests.java | 16 ++++++++++------
 3 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
index cb4d2d3a216..aa9288cc43a 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
@@ -90,7 +90,7 @@ public class EListInit extends AExpression {
         for (AExpression value : values) {
             writer.dup();
             value.write(writer, globals);
-            writer.invokeInterface(method.owner.type, method.method);
+            method.write(writer);
             writer.pop();
         }
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
index c583bc79e52..c031087a34d 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
@@ -27,7 +27,6 @@ import org.elasticsearch.painless.Locals;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
 
-import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 
@@ -114,7 +113,7 @@ public class EMapInit extends AExpression {
             writer.dup();
             key.write(writer, globals);
             value.write(writer, globals);
-            writer.invokeInterface(method.owner.type, method.method);
+            method.write(writer);
             writer.pop();
         }
     }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/InitializerTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/InitializerTests.java
index 282ad8a5f13..5d881632dee 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/InitializerTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/InitializerTests.java
@@ -26,6 +26,7 @@ import java.util.Map;
 
 public class InitializerTests extends ScriptTestCase {
 
+    @SuppressWarnings({"unchecked", "rawtypes"})
     public void testArrayInitializers() {
         int[] ints = (int[])exec("new int[] {}");
 
@@ -58,6 +59,7 @@ public class InitializerTests extends ScriptTestCase {
         assertEquals("aaaaaa", objects[3]);
     }
 
+    @SuppressWarnings({"unchecked", "rawtypes"})
     public void testListInitializers() {
         List list = (List)exec("[]");
 
@@ -84,11 +86,12 @@ public class InitializerTests extends ScriptTestCase {
 
         assertEquals(4, list.size());
         assertEquals(new Integer(2), list.get(0));
-        assertEquals(new ArrayList(),  list.get(1));
+        assertEquals(new ArrayList(), list.get(1));
         assertEquals("1aaa",  list.get(2));
         assertEquals("aaaaaa", list.get(3));
     }
 
+    @SuppressWarnings({"unchecked", "rawtypes"})
     public void testMapInitializers() {
         Map map = (Map)exec("[:]");
 
@@ -115,22 +118,23 @@ public class InitializerTests extends ScriptTestCase {
         assertEquals("aaaaaa", map.get("1aaa"));
     }
 
+    @SuppressWarnings({"unchecked", "rawtypes"})
     public void testCrazyInitializer() {
         Map map = (Map)exec("int y = 2; int z = 3; Map x = [y*z : y + z, 's' : [y, [y : [[z], [], [:]]]], z : [z, 9]]; return x;");
 
-        List<Object> list0 = new ArrayList<>();
+        List list0 = new ArrayList();
         list0.add(3);
-        List<Object> list1 = new ArrayList();
+        List list1 = new ArrayList();
         list1.add(list0);
         list1.add(new ArrayList());
         list1.add(new HashMap());
-        Map<Object, Object> map0 = new HashMap<>();
+        Map map0 = new HashMap();
         map0.put(2, list1);
-        List<Object> list2 = new ArrayList();
+        List list2 = new ArrayList();
         list2.add(2);
         list2.add(map0);
 
-        List<Object> list3 = new ArrayList<>();
+        List list3 = new ArrayList();
         list3.add(3);
         list3.add(9);
 

From 0b4fc853674cfe8f0bce4be7ee221d90b380bf76 Mon Sep 17 00:00:00 2001
From: Jack Conradson <osjdconrad@gmail.com>
Date: Tue, 21 Jun 2016 12:56:54 -0700
Subject: [PATCH 1009/1311] Added some brief docs.

---
 .../main/java/org/elasticsearch/painless/node/EListInit.java   | 3 +++
 .../main/java/org/elasticsearch/painless/node/EMapInit.java    | 3 +++
 .../java/org/elasticsearch/painless/node/package-info.java     | 2 ++
 3 files changed, 8 insertions(+)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
index aa9288cc43a..d6e4616c1d8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java
@@ -30,6 +30,9 @@ import org.elasticsearch.painless.MethodWriter;
 import java.util.List;
 import java.util.Set;
 
+/**
+ * Represents a list initialization shortcut.
+ */
 public class EListInit extends AExpression {
     final List<AExpression> values;
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
index c031087a34d..d36ac50c24e 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java
@@ -30,6 +30,9 @@ import org.elasticsearch.painless.MethodWriter;
 import java.util.List;
 import java.util.Set;
 
+/**
+ * Represents a map initialization shortcut.
+ */
 public class EMapInit extends AExpression {
     final List<AExpression> keys;
     final List<AExpression> values;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
index d98e5f68bcf..7111afa59f4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
@@ -44,6 +44,8 @@
  * {@link org.elasticsearch.painless.node.EDecimal} - Represents a decimal constant.
  * {@link org.elasticsearch.painless.node.EExplicit} - Represents an explicit cast.
  * {@link org.elasticsearch.painless.node.EFunctionRef} - Represents a function reference (non-capturing).
+ * {@link org.elasticsearch.painless.node.EListInit} - Represents a list initialization shortcut.
+ * {@link org.elasticsearch.painless.node.EMapInit} - Represents a map initializiation shortcut.
  * {@link org.elasticsearch.painless.node.ENull} - Represents a null constant.
  * {@link org.elasticsearch.painless.node.ENumeric} - Represents a non-decimal numeric constant.
  * {@link org.elasticsearch.painless.node.EUnary} - Represents a unary math expression.

From 28fd684eef75592346e72c92177e7dd86af82608 Mon Sep 17 00:00:00 2001
From: Tal Levy <JubBoy333@gmail.com>
Date: Tue, 21 Jun 2016 13:29:53 -0700
Subject: [PATCH 1010/1311] Fix ignore_failure behavior in _simulate?verbose
 (#18987)

- fix it so that processors with the `ignore_failure` option do not
record their exception in the response
- add more tests to make empty `on_failure`. This now throws an
  exception
---
 .../ingest/TrackingResultProcessor.java       | 24 ++++---
 .../ingest/ConfigurationUtils.java            |  8 ++-
 .../org/elasticsearch/ingest/Pipeline.java    |  3 +
 .../ingest/SimulateExecutionServiceTests.java | 16 +++++
 .../ingest/TrackingResultProcessorTests.java  | 20 +++++-
 .../ingest/PipelineFactoryTests.java          | 31 ++++++--
 .../test/ingest/50_on_failure.yaml            | 55 +++++++++++++++
 .../test/ingest/90_simulate.yaml              | 70 +++++++++++++++++++
 8 files changed, 209 insertions(+), 18 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/ingest/TrackingResultProcessor.java b/core/src/main/java/org/elasticsearch/action/ingest/TrackingResultProcessor.java
index b91c7b14fee..4f2383ef1b6 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/TrackingResultProcessor.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/TrackingResultProcessor.java
@@ -24,6 +24,7 @@ import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 /**
@@ -33,15 +34,12 @@ public final class TrackingResultProcessor implements Processor {
 
     private final Processor actualProcessor;
     private final List<SimulateProcessorResult> processorResultList;
+    private final boolean ignoreFailure;
 
-    public TrackingResultProcessor(Processor actualProcessor, List<SimulateProcessorResult> processorResultList) {
+    public TrackingResultProcessor(boolean ignoreFailure, Processor actualProcessor, List<SimulateProcessorResult> processorResultList) {
+        this.ignoreFailure = ignoreFailure;
         this.processorResultList = processorResultList;
-        if (actualProcessor instanceof CompoundProcessor) {
-            CompoundProcessor trackedCompoundProcessor = decorate((CompoundProcessor) actualProcessor, processorResultList);
-            this.actualProcessor = trackedCompoundProcessor;
-        } else {
-            this.actualProcessor = actualProcessor;
-        }
+        this.actualProcessor = actualProcessor;
     }
 
     @Override
@@ -50,7 +48,11 @@ public final class TrackingResultProcessor implements Processor {
             actualProcessor.execute(ingestDocument);
             processorResultList.add(new SimulateProcessorResult(actualProcessor.getTag(), new IngestDocument(ingestDocument)));
         } catch (Exception e) {
-            processorResultList.add(new SimulateProcessorResult(actualProcessor.getTag(), e));
+            if (ignoreFailure) {
+                processorResultList.add(new SimulateProcessorResult(actualProcessor.getTag(), new IngestDocument(ingestDocument)));
+            } else {
+                processorResultList.add(new SimulateProcessorResult(actualProcessor.getTag(), e));
+            }
             throw e;
         }
     }
@@ -71,7 +73,7 @@ public final class TrackingResultProcessor implements Processor {
             if (processor instanceof CompoundProcessor) {
                 processors.add(decorate((CompoundProcessor) processor, processorResultList));
             } else {
-                processors.add(new TrackingResultProcessor(processor, processorResultList));
+                processors.add(new TrackingResultProcessor(compoundProcessor.isIgnoreFailure(), processor, processorResultList));
             }
         }
         List<Processor> onFailureProcessors = new ArrayList<>(compoundProcessor.getProcessors().size());
@@ -79,10 +81,10 @@ public final class TrackingResultProcessor implements Processor {
             if (processor instanceof CompoundProcessor) {
                 onFailureProcessors.add(decorate((CompoundProcessor) processor, processorResultList));
             } else {
-                onFailureProcessors.add(new TrackingResultProcessor(processor, processorResultList));
+                onFailureProcessors.add(new TrackingResultProcessor(compoundProcessor.isIgnoreFailure(), processor, processorResultList));
             }
         }
-        return new CompoundProcessor(false, processors, onFailureProcessors);
+        return new CompoundProcessor(compoundProcessor.isIgnoreFailure(), processors, onFailureProcessors);
     }
 }
 
diff --git a/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java b/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
index d87a59fd625..623c4f315a4 100644
--- a/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
+++ b/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
@@ -243,6 +243,7 @@ public final class ConfigurationUtils {
                 }
             }
         }
+
         return processors;
     }
 
@@ -256,7 +257,12 @@ public final class ConfigurationUtils {
             List<Processor> onFailureProcessors = readProcessorConfigs(onFailureProcessorConfigs, processorRegistry);
             Processor processor;
             processor = factory.create(config);
-            if (!config.isEmpty()) {
+
+            if (onFailureProcessorConfigs != null && onFailureProcessors.isEmpty()) {
+                throw newConfigurationException(processor.getType(), processor.getTag(), Pipeline.ON_FAILURE_KEY,
+                    "processors list cannot be empty");
+            }
+            if (config.isEmpty() == false) {
                 throw new ElasticsearchParseException("processor [{}] doesn't support one or more provided configuration parameters {}",
                     type, Arrays.toString(config.keySet().toArray()));
             }
diff --git a/core/src/main/java/org/elasticsearch/ingest/Pipeline.java b/core/src/main/java/org/elasticsearch/ingest/Pipeline.java
index bea02f49358..d568f5521a4 100644
--- a/core/src/main/java/org/elasticsearch/ingest/Pipeline.java
+++ b/core/src/main/java/org/elasticsearch/ingest/Pipeline.java
@@ -109,6 +109,9 @@ public final class Pipeline {
                 throw new ElasticsearchParseException("pipeline [" + id +
                         "] doesn't support one or more provided configuration parameters " + Arrays.toString(config.keySet().toArray()));
             }
+            if (onFailureProcessorConfigs != null && onFailureProcessors.isEmpty()) {
+                throw new ElasticsearchParseException("pipeline [" + id + "] cannot have an empty on_failure option defined");
+            }
             CompoundProcessor compoundProcessor = new CompoundProcessor(false, Collections.unmodifiableList(processors),
                     Collections.unmodifiableList(onFailureProcessors));
             return new Pipeline(id, description, compoundProcessor);
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
index a7bc76341b8..5806d8c312b 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
@@ -159,6 +159,22 @@ public class SimulateExecutionServiceTests extends ESTestCase {
         assertThat(simulateDocumentVerboseResult.getProcessorResults().get(2).getFailure(), nullValue());
     }
 
+    public void testExecuteVerboseItemExceptionWithIgnoreFailure() throws Exception {
+        TestProcessor testProcessor = new TestProcessor("processor_0", "mock", ingestDocument -> { throw new RuntimeException("processor failed"); });
+        CompoundProcessor processor = new CompoundProcessor(true, Collections.singletonList(testProcessor), Collections.emptyList());
+        Pipeline pipeline = new Pipeline("_id", "_description", new CompoundProcessor(processor));
+        SimulateDocumentResult actualItemResponse = executionService.executeDocument(pipeline, ingestDocument, true);
+        assertThat(testProcessor.getInvokedCounter(), equalTo(1));
+        assertThat(actualItemResponse, instanceOf(SimulateDocumentVerboseResult.class));
+        SimulateDocumentVerboseResult simulateDocumentVerboseResult = (SimulateDocumentVerboseResult) actualItemResponse;
+        assertThat(simulateDocumentVerboseResult.getProcessorResults().size(), equalTo(1));
+        assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getProcessorTag(), equalTo("processor_0"));
+        assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getFailure(), nullValue());
+        assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getIngestDocument(), not(sameInstance(ingestDocument)));
+        assertIngestDocument(simulateDocumentVerboseResult.getProcessorResults().get(0).getIngestDocument(), ingestDocument);
+        assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getIngestDocument().getSourceAndMetadata(), not(sameInstance(ingestDocument.getSourceAndMetadata())));
+    }
+
     public void testExecuteItemWithFailure() throws Exception {
         TestProcessor processor = new TestProcessor(ingestDocument -> { throw new RuntimeException("processor failed"); });
         Pipeline pipeline = new Pipeline("_id", "_description", new CompoundProcessor(processor, processor));
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/TrackingResultProcessorTests.java b/core/src/test/java/org/elasticsearch/action/ingest/TrackingResultProcessorTests.java
index 30f17515f00..5b0a0599090 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/TrackingResultProcessorTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/TrackingResultProcessorTests.java
@@ -28,6 +28,7 @@ import org.junit.Before;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -52,7 +53,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
 
     public void testActualProcessor() throws Exception {
         TestProcessor actualProcessor = new TestProcessor(ingestDocument -> {});
-        TrackingResultProcessor trackingProcessor = new TrackingResultProcessor(actualProcessor, resultList);
+        TrackingResultProcessor trackingProcessor = new TrackingResultProcessor(false, actualProcessor, resultList);
         trackingProcessor.execute(ingestDocument);
 
         SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getTag(), ingestDocument);
@@ -127,4 +128,21 @@ public class TrackingResultProcessorTests extends ESTestCase {
         assertThat(resultList.get(3).getFailure(), nullValue());
         assertThat(resultList.get(3).getProcessorTag(), equalTo(expectedSuccessResult.getProcessorTag()));
     }
+
+    public void testActualCompoundProcessorWithIgnoreFailure() throws Exception {
+        RuntimeException exception = new RuntimeException("processor failed");
+        TestProcessor testProcessor = new TestProcessor(ingestDocument -> {  throw exception; });
+        CompoundProcessor actualProcessor = new CompoundProcessor(true, Collections.singletonList(testProcessor),
+            Collections.emptyList());
+        CompoundProcessor trackingProcessor = decorate(actualProcessor, resultList);
+
+        trackingProcessor.execute(ingestDocument);
+
+        SimulateProcessorResult expectedResult = new SimulateProcessorResult(testProcessor.getTag(), ingestDocument);
+        assertThat(testProcessor.getInvokedCounter(), equalTo(1));
+        assertThat(resultList.size(), equalTo(1));
+        assertThat(resultList.get(0).getIngestDocument(), equalTo(expectedResult.getIngestDocument()));
+        assertThat(resultList.get(0).getFailure(), nullValue());
+        assertThat(resultList.get(0).getProcessorTag(), equalTo(expectedResult.getProcessorTag()));
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java
index 7c4f8236404..cb7bd849a47 100644
--- a/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java
@@ -86,6 +86,30 @@ public class PipelineFactoryTests extends ESTestCase {
         assertThat(pipeline.getOnFailureProcessors().get(0).getType(), equalTo("test-processor"));
     }
 
+    public void testCreateWithPipelineEmptyOnFailure() throws Exception {
+        Map<String, Object> processorConfig = new HashMap<>();
+        Map<String, Object> pipelineConfig = new HashMap<>();
+        pipelineConfig.put(Pipeline.DESCRIPTION_KEY, "_description");
+        pipelineConfig.put(Pipeline.PROCESSORS_KEY, Collections.singletonList(Collections.singletonMap("test", processorConfig)));
+        pipelineConfig.put(Pipeline.ON_FAILURE_KEY, Collections.emptyList());
+        Pipeline.Factory factory = new Pipeline.Factory();
+        ProcessorsRegistry processorRegistry = createProcessorRegistry(Collections.singletonMap("test", new TestProcessor.Factory()));
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create("_id", pipelineConfig, processorRegistry));
+        assertThat(e.getMessage(), equalTo("pipeline [_id] cannot have an empty on_failure option defined"));
+    }
+
+    public void testCreateWithPipelineEmptyOnFailureInProcessor() throws Exception {
+        Map<String, Object> processorConfig = new HashMap<>();
+        processorConfig.put(Pipeline.ON_FAILURE_KEY, Collections.emptyList());
+        Map<String, Object> pipelineConfig = new HashMap<>();
+        pipelineConfig.put(Pipeline.DESCRIPTION_KEY, "_description");
+        pipelineConfig.put(Pipeline.PROCESSORS_KEY, Collections.singletonList(Collections.singletonMap("test", processorConfig)));
+        Pipeline.Factory factory = new Pipeline.Factory();
+        ProcessorsRegistry processorRegistry = createProcessorRegistry(Collections.singletonMap("test", new TestProcessor.Factory()));
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create("_id", pipelineConfig, processorRegistry));
+        assertThat(e.getMessage(), equalTo("[on_failure] processors list cannot be empty"));
+    }
+
     public void testCreateWithPipelineIgnoreFailure() throws Exception {
         Map<String, Object> processorConfig = new HashMap<>();
         processorConfig.put("ignore_failure", true);
@@ -116,11 +140,8 @@ public class PipelineFactoryTests extends ESTestCase {
         pipelineConfig.put(Pipeline.PROCESSORS_KEY, Collections.singletonList(Collections.singletonMap("test", processorConfig)));
         Pipeline.Factory factory = new Pipeline.Factory();
         ProcessorsRegistry processorRegistry = createProcessorRegistry(Collections.singletonMap("test", new TestProcessor.Factory()));
-        try {
-            factory.create("_id", pipelineConfig, processorRegistry);
-        } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), equalTo("processor [test] doesn't support one or more provided configuration parameters [unused]"));
-        }
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create("_id", pipelineConfig, processorRegistry));
+        assertThat(e.getMessage(), equalTo("processor [test] doesn't support one or more provided configuration parameters [unused]"));
     }
 
     public void testCreateProcessorsWithOnFailureProperties() throws Exception {
diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yaml
index bc0a765e2c0..464199f6d3b 100644
--- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yaml
+++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/50_on_failure.yaml
@@ -107,6 +107,61 @@
   - match: { _source.foofield: "exists" }
   - match: { _source.foofield2: "ran" }
 
+---
+"Test pipeline with empty on_failure in a processor":
+  - do:
+      catch: request
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "fail" : {
+                  "tag" : "emptyfail",
+                  "message" : "_message",
+                  "on_failure": []
+                }
+              }
+            ],
+            "on_failure": [
+              {
+                "set" : {
+                  "field": "on_failure_executed",
+                  "value": true
+                }
+              }
+            ]
+          }
+  - match: { error.root_cause.0.type: "parse_exception" }
+  - match: { error.root_cause.0.reason: "[on_failure] processors list cannot be empty" }
+  - match: { error.root_cause.0.header.processor_type: "fail" }
+  - match: { error.root_cause.0.header.processor_tag: "emptyfail" }
+  - match: { error.root_cause.0.header.property_name: "on_failure" }
+
+---
+"Test pipeline with empty on_failure in pipeline":
+  - do:
+      catch: request
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "set" : {
+                  "field" : "foo",
+                  "value" : "_message"
+                }
+              }
+            ],
+            "on_failure": []
+          }
+  - match: { error.root_cause.0.type: "parse_exception" }
+  - match: { error.root_cause.0.reason: "pipeline [my_pipeline] cannot have an empty on_failure option defined" }
+
 ---
 "Test pipeline with ignore_failure in a processor":
   - do:
diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/90_simulate.yaml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/90_simulate.yaml
index bc1b444ab54..4dfe8a31ee8 100644
--- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/90_simulate.yaml
+++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/90_simulate.yaml
@@ -479,3 +479,73 @@
   - match: { docs.0.processor_results.4.doc._source.foofield2: "ran" }
   - match: { docs.0.processor_results.4.doc._source.field1: "123.42 400 <foo>" }
   - match: { docs.0.processor_results.4.doc._source.status: 200 }
+
+---
+"Test verbose simulate with ignore_failure":
+  - do:
+      ingest.simulate:
+        verbose: true
+        body: >
+          {
+            "pipeline" : {
+              "description": "_description",
+              "processors": [
+                {
+                  "set" : {
+                    "tag" : "setstatus-1",
+                    "field" : "status",
+                    "value" : 200
+                  }
+                },
+                {
+                  "rename" : {
+                    "tag" : "rename-1",
+                    "field" : "foofield",
+                    "target_field" : "field1",
+                    "ignore_failure": true,
+                    "on_failure" : [
+                      {
+                        "set" : {
+                          "tag" : "set on_failure rename",
+                          "field" : "foofield",
+                          "value" : "exists"
+                        }
+                      },
+                      {
+                        "rename" : {
+                          "field" : "foofield2",
+                          "target_field" : "field1",
+                          "on_failure" : [
+                            {
+                              "set" : {
+                                "field" : "foofield2",
+                                "value" : "ran"
+                              }
+                            }
+                          ]
+                        }
+                      }
+                    ]
+                  }
+                }
+              ]
+            },
+            "docs": [
+              {
+                "_index": "index",
+                "_type": "type",
+                "_id": "id",
+                "_source": {
+                  "field1": "123.42 400 <foo>"
+                }
+              }
+            ]
+          }
+  - length: { docs: 1 }
+  - length: { docs.0.processor_results: 2 }
+  - match: { docs.0.processor_results.0.tag: "setstatus-1" }
+  - match: { docs.0.processor_results.0.doc._source.field1: "123.42 400 <foo>" }
+  - match: { docs.0.processor_results.0.doc._source.status: 200 }
+  - match: { docs.0.processor_results.1.tag: "rename-1" }
+  - match: { docs.0.processor_results.1.doc._source.field1: "123.42 400 <foo>" }
+  - match: { docs.0.processor_results.1.doc._source.status: 200 }

From 8925400f677815a5736c88fa622aa2c53d260a05 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 21 Jun 2016 11:53:19 -0400
Subject: [PATCH 1011/1311] Remove guice from ScriptService

Makes ScriptModule just a plain class that manages building the
ScriptSettings and ScriptService from plugins. When we *need*
to bind ScriptService with guice we bind it in a lambda.
---
 .../java/org/elasticsearch/node/Node.java     |  7 +-
 .../elasticsearch/script/ScriptModule.java    | 88 +++++++++----------
 .../elasticsearch/script/ScriptService.java   |  3 -
 .../elasticsearch/script/ScriptSettings.java  | 13 +--
 .../script/NativeScriptTests.java             | 29 ++----
 .../aggregations/AggregatorParsingTests.java  | 44 +++++-----
 .../aggregations/BaseAggregationTestCase.java | 20 ++---
 .../builder/SearchSourceBuilderTests.java     |  9 +-
 .../messy/tests/TemplateQueryParserTests.java | 32 +++----
 .../test/AbstractQueryTestCase.java           |  3 +-
 .../org/elasticsearch/test/ESTestCase.java    | 32 ++-----
 11 files changed, 123 insertions(+), 157 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index de77f02ef51..3c036ff3f29 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -225,7 +225,9 @@ public class Node implements Closeable {
             for (final ExecutorBuilder<?> builder : threadPool.builders()) {
                 additionalSettings.addAll(builder.getRegisteredSettings());
             }
-            final ScriptModule scriptModule = ScriptModule.create(settings, pluginsService.filterPlugins(ScriptPlugin.class));
+            final ResourceWatcherService resourceWatcherService = new ResourceWatcherService(settings, threadPool);
+            final ScriptModule scriptModule = ScriptModule.create(settings, environment, resourceWatcherService,
+                    pluginsService.filterPlugins(ScriptPlugin.class));
             additionalSettings.addAll(scriptModule.getSettings());
             // this is as early as we can validate settings at this point. we already pass them to ScriptModule as well as ThreadPool
             // so we might be late here already
@@ -237,7 +239,6 @@ public class Node implements Closeable {
             } catch (IOException ex) {
                 throw new IllegalStateException("Failed to created node environment", ex);
             }
-            final ResourceWatcherService resourceWatcherService = new ResourceWatcherService(settings, threadPool);
             resourcesToClose.add(resourceWatcherService);
             final NetworkService networkService = new NetworkService(settings);
             final ClusterService clusterService = new ClusterService(settings, settingsModule.getClusterSettings(), threadPool);
@@ -253,7 +254,6 @@ public class Node implements Closeable {
             final MonitorService monitorService = new MonitorService(settings, nodeEnvironment, threadPool);
             modules.add(new NodeModule(this, monitorService));
             modules.add(new NetworkModule(networkService, settings, false, namedWriteableRegistry));
-            modules.add(scriptModule);
             modules.add(new DiscoveryModule(this.settings));
             modules.add(new ClusterModule(this.settings, clusterService));
             modules.add(new IndicesModule(namedWriteableRegistry));
@@ -279,6 +279,7 @@ public class Node implements Closeable {
                     b.bind(ResourceWatcherService.class).toInstance(resourceWatcherService);
                     b.bind(CircuitBreakerService.class).toInstance(circuitBreakerService);
                     b.bind(BigArrays.class).toInstance(bigArrays);
+                    b.bind(ScriptService.class).toInstance(scriptModule.getScriptService());
                 }
             );
             injector = modules.createInjector();
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptModule.java b/core/src/main/java/org/elasticsearch/script/ScriptModule.java
index a66f642fa15..9d8c7ebda67 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptModule.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptModule.java
@@ -19,12 +19,13 @@
 
 package org.elasticsearch.script;
 
-import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.ScriptPlugin;
+import org.elasticsearch.watcher.ResourceWatcherService;
 
-import java.util.ArrayList;
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
@@ -34,57 +35,56 @@ import java.util.function.Function;
 import java.util.stream.Collectors;
 
 /**
- * An {@link org.elasticsearch.common.inject.Module} which manages {@link ScriptEngineService}s, as well
- * as named script
+ * Manages building {@link ScriptService} and {@link ScriptSettings} from a list of plugins.
  */
-public class ScriptModule extends AbstractModule {
-
-    protected final ScriptContextRegistry scriptContextRegistry;
-    protected final ScriptEngineRegistry scriptEngineRegistry;
-    protected final ScriptSettings scriptSettings;
-
-    public ScriptModule(ScriptEngineService... services) {
-        this(Arrays.asList(services), Collections.emptyList());
-    }
-
-    public ScriptModule(List<ScriptEngineService> scriptEngineServices,
-                        List<ScriptContext.Plugin> customScriptContexts) {
-        this.scriptContextRegistry = new ScriptContextRegistry(customScriptContexts);
-        this.scriptEngineRegistry = new ScriptEngineRegistry(scriptEngineServices);
-        this.scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
-    }
+public class ScriptModule {
+    private final ScriptSettings scriptSettings;
+    private final ScriptService scriptService;
 
     /**
-     * This method is called after all modules have been processed but before we actually validate all settings. This allows the
-     * script extensions to add all their settings.
+     * Build from {@linkplain ScriptPlugin}s. Convenient for normal use but not great for tests. See
+     * {@link ScriptModule#ScriptModule(Settings, Environment, ResourceWatcherService, List, List)} for easier use in tests.
      */
-    public List<Setting<?>> getSettings() {
-        ArrayList<Setting<?>> settings = new ArrayList<>();
-        scriptSettings.getScriptTypeSettings().forEach(settings::add);
-        scriptSettings.getScriptContextSettings().forEach(settings::add);
-        scriptSettings.getScriptLanguageSettings().forEach(settings::add);
-        settings.add(scriptSettings.getDefaultScriptLanguageSetting());
-        return settings;
-    }
-
-
-    @Override
-    protected void configure() {
-        ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
-        bind(ScriptContextRegistry.class).toInstance(scriptContextRegistry);
-        bind(ScriptEngineRegistry.class).toInstance(scriptEngineRegistry);
-        bind(ScriptSettings.class).toInstance(scriptSettings);
-        bind(ScriptService.class).asEagerSingleton();
-    }
-
-    public static ScriptModule create(Settings settings, List<ScriptPlugin> scriptPlugins) {
+    public static ScriptModule create(Settings settings, Environment environment, ResourceWatcherService resourceWatcherService,
+            List<ScriptPlugin> scriptPlugins) {
         Map<String, NativeScriptFactory> factoryMap = scriptPlugins.stream().flatMap(x -> x.getNativeScripts().stream())
             .collect(Collectors.toMap(NativeScriptFactory::getName, Function.identity()));
         NativeScriptEngineService nativeScriptEngineService = new NativeScriptEngineService(settings, factoryMap);
         List<ScriptEngineService> scriptEngineServices = scriptPlugins.stream().map(x -> x.getScriptEngineService(settings))
             .filter(Objects::nonNull).collect(Collectors.toList());
         scriptEngineServices.add(nativeScriptEngineService);
-        return new ScriptModule(scriptEngineServices, scriptPlugins.stream().map(x -> x.getCustomScriptContexts())
-            .filter(Objects::nonNull).collect(Collectors.toList()));
+        List<ScriptContext.Plugin> plugins = scriptPlugins.stream().map(x -> x.getCustomScriptContexts()).filter(Objects::nonNull)
+                .collect(Collectors.toList());
+        return new ScriptModule(settings, environment, resourceWatcherService, scriptEngineServices, plugins);
+    }
+
+    /**
+     * Build {@linkplain ScriptEngineService} and {@linkplain ScriptContext.Plugin}.
+     */
+    public ScriptModule(Settings settings, Environment environment, ResourceWatcherService resourceWatcherService,
+            List<ScriptEngineService> scriptEngineServices, List<ScriptContext.Plugin> customScriptContexts) {
+        ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(customScriptContexts);
+        ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(scriptEngineServices);
+        scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry);
+        try {
+            scriptService = new ScriptService(settings, environment, resourceWatcherService, scriptEngineRegistry, scriptContextRegistry,
+                    scriptSettings);
+        } catch (IOException e) {
+            throw new RuntimeException("Couldn't setup ScriptService", e);
+        }
+    }
+
+    /**
+     * Extra settings for scripts.
+     */
+    public List<Setting<?>> getSettings() {
+        return scriptSettings.getSettings();
+    }
+
+    /**
+     * Service responsible for managing scripts.
+     */
+    public ScriptService getScriptService() {
+        return scriptService;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java
index 8f939f265a4..57d46255e19 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java
@@ -41,7 +41,6 @@ import org.elasticsearch.common.cache.RemovalListener;
 import org.elasticsearch.common.cache.RemovalNotification;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -72,7 +71,6 @@ import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 
 import static java.util.Collections.unmodifiableMap;
@@ -132,7 +130,6 @@ public class ScriptService extends AbstractComponent implements Closeable {
     @Deprecated
     public static final ParseField SCRIPT_INLINE = new ParseField("script");
 
-    @Inject
     public ScriptService(Settings settings, Environment env,
                          ResourceWatcherService resourceWatcherService, ScriptEngineRegistry scriptEngineRegistry,
                          ScriptContextRegistry scriptContextRegistry, ScriptSettings scriptSettings) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java b/core/src/main/java/org/elasticsearch/script/ScriptSettings.java
index 71013f28a06..57fa8d58f2c 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptSettings.java
@@ -142,12 +142,13 @@ public class ScriptSettings {
         return scriptModeSettings;
     }
 
-    public Iterable<Setting<Boolean>> getScriptTypeSettings() {
-        return Collections.unmodifiableCollection(SCRIPT_TYPE_SETTING_MAP.values());
-    }
-
-    public Iterable<Setting<Boolean>> getScriptContextSettings() {
-        return Collections.unmodifiableCollection(scriptContextSettingMap.values());
+    public List<Setting<?>> getSettings() {
+        List<Setting<?>> settings = new ArrayList<>();
+        settings.addAll(SCRIPT_TYPE_SETTING_MAP.values());
+        settings.addAll(scriptContextSettingMap.values());
+        settings.addAll(scriptLanguageSettings);
+        settings.add(defaultScriptLanguageSetting);
+        return settings;
     }
 
     public Iterable<Setting<Boolean>> getScriptLanguageSettings() {
diff --git a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
index d88813203b2..70cc3588602 100644
--- a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
@@ -22,16 +22,12 @@ package org.elasticsearch.script;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.ModulesBuilder;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
-import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
 
 import java.io.IOException;
@@ -41,6 +37,9 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
@@ -49,28 +48,18 @@ public class NativeScriptTests extends ESTestCase {
         Settings settings = Settings.builder()
                 .put("node.name", "testNativeScript")
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+                .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
                 .build();
-        ScriptModule scriptModule = new ScriptModule(new NativeScriptEngineService(settings,
-            Collections.singletonMap("my", new MyNativeScriptFactory())));
+        ScriptModule scriptModule = new ScriptModule(settings, new Environment(settings), null,
+                singletonList(new NativeScriptEngineService(settings, singletonMap("my", new MyNativeScriptFactory()))), emptyList());
         List<Setting<?>> scriptSettings = scriptModule.getSettings();
         scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
-        SettingsModule settingsModule = new SettingsModule(settings, scriptSettings, Collections.emptyList());
-        final ThreadPool threadPool = new ThreadPool(settings);
-        Injector injector = new ModulesBuilder().add(
-                (b) -> {
-                    b.bind(Environment.class).toInstance(new Environment(settings));
-                    b.bind(ThreadPool.class).toInstance(threadPool);
-                },
-                new SettingsModule(settings),
-                scriptModule).createInjector();
-
-        ScriptService scriptService = injector.getInstance(ScriptService.class);
 
         ClusterState state = ClusterState.builder(new ClusterName("_name")).build();
-        ExecutableScript executable = scriptService.executable(new Script("my", ScriptType.INLINE, NativeScriptEngineService.NAME, null),
-                ScriptContext.Standard.SEARCH, Collections.emptyMap(), state);
+        ExecutableScript executable = scriptModule.getScriptService().executable(
+                new Script("my", ScriptType.INLINE, NativeScriptEngineService.NAME, null), ScriptContext.Standard.SEARCH,
+                Collections.emptyMap(), state);
         assertThat(executable.run().toString(), equalTo("test"));
-        terminate(injector.getInstance(ThreadPool.class));
     }
 
     public void testFineGrainedSettingsDontAffectNativeScripts() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
index a78c405929a..08109a437cc 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
@@ -29,7 +29,6 @@ import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.Injector;
 import org.elasticsearch.common.inject.ModulesBuilder;
-import org.elasticsearch.common.inject.util.Providers;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -112,28 +111,29 @@ public class AggregatorParsingTests extends ESTestCase {
             (b) -> {
                 b.bind(Environment.class).toInstance(new Environment(settings));
                 b.bind(ThreadPool.class).toInstance(threadPool);
-            }, settingsModule
-            , scriptModule, new IndicesModule(namedWriteableRegistry) {
+                b.bind(ScriptService.class).toInstance(scriptModule.getScriptService());
+            },
+            settingsModule,
+            new IndicesModule(namedWriteableRegistry) {
+                @Override
+                protected void configure() {
+                    bindMapperExtension();
+                }
+            }, new SearchModule(settings, namedWriteableRegistry) {
+                @Override
+                protected void configureSearch() {
+                    // Skip me
+                }
+            }, new IndexSettingsModule(index, settings),
 
-                    @Override
-                    protected void configure() {
-                        bindMapperExtension();
-                    }
-                }, new SearchModule(settings, namedWriteableRegistry) {
-                    @Override
-                    protected void configureSearch() {
-                        // Skip me
-                    }
-                }, new IndexSettingsModule(index, settings),
-
-                new AbstractModule() {
-                    @Override
-                    protected void configure() {
-                        bind(ClusterService.class).toProvider(Providers.of(clusterService));
-                        bind(CircuitBreakerService.class).to(NoneCircuitBreakerService.class);
-                        bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry);
-                    }
-                }).createInjector();
+            new AbstractModule() {
+                @Override
+                protected void configure() {
+                    bind(ClusterService.class).toInstance(clusterService);
+                    bind(CircuitBreakerService.class).toInstance(new NoneCircuitBreakerService());
+                    bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry);
+                }
+            }).createInjector();
         aggParsers = injector.getInstance(AggregatorParsers.class);
         // create some random type with some default field, those types will
         // stick around for all of the subclasses
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
index 7bce599f55a..d953eb02174 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
@@ -25,7 +25,6 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.Injector;
 import org.elasticsearch.common.inject.ModulesBuilder;
 import org.elasticsearch.common.inject.util.Providers;
@@ -140,31 +139,26 @@ public abstract class BaseAggregationTestCase<AB extends AbstractAggregationBuil
             (b) -> {
                 b.bind(Environment.class).toInstance(new Environment(settings));
                 b.bind(ThreadPool.class).toInstance(threadPool);
+                b.bind(ScriptService.class).toInstance(scriptModule.getScriptService());
+                b.bind(ClusterService.class).toProvider(Providers.of(clusterService));
+                b.bind(CircuitBreakerService.class).to(NoneCircuitBreakerService.class);
+                b.bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry);
             },
             settingsModule,
-            scriptModule,
             new IndicesModule(namedWriteableRegistry) {
 
                 @Override
                 protected void configure() {
                     bindMapperExtension();
                 }
-            }, new SearchModule(settings, namedWriteableRegistry) {
+            },
+            new SearchModule(settings, namedWriteableRegistry) {
                 @Override
                 protected void configureSearch() {
                     // Skip me
                 }
             },
-            new IndexSettingsModule(index, settings),
-
-            new AbstractModule() {
-                @Override
-                protected void configure() {
-                    bind(ClusterService.class).toProvider(Providers.of(clusterService));
-                    bind(CircuitBreakerService.class).to(NoneCircuitBreakerService.class);
-                    bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry);
-                }
-            }
+            new IndexSettingsModule(index, settings)
         ).createInjector();
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index 5822372bfa9..fbd9546ec2d 100644
--- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -134,13 +134,16 @@ public class SearchSourceBuilderTests extends ESTestCase {
                 (b) -> {
                     b.bind(Environment.class).toInstance(new Environment(settings));
                     b.bind(ThreadPool.class).toInstance(threadPool);
-                }, settingsModule,
-                scriptModule, new IndicesModule(namedWriteableRegistry) {
+                    b.bind(ScriptService.class).toInstance(scriptModule.getScriptService());
+                },
+                settingsModule,
+                new IndicesModule(namedWriteableRegistry) {
                     @Override
                     protected void configure() {
                         bindMapperExtension();
                     }
-                }, new SearchModule(settings, namedWriteableRegistry) {
+                },
+                new SearchModule(settings, namedWriteableRegistry) {
                     @Override
                     protected void configureSearch() {
                         // Skip me
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
index 60316818a79..0cca2c41938 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
@@ -75,6 +75,8 @@ import java.util.Collections;
 import java.util.List;
 import java.util.function.Supplier;
 
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
 import static org.hamcrest.Matchers.containsString;
 
 /**
@@ -93,7 +95,9 @@ public class TemplateQueryParserTests extends ESTestCase {
                 .put(Environment.PATH_CONF_SETTING.getKey(), this.getDataPath("config"))
                 .put("node.name", getClass().getName())
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
                 .build();
+        Environment environment = new Environment(settings);
         final Client proxy = (Client) Proxy.newProxyInstance(
                 Client.class.getClassLoader(),
                 new Class<?>[]{Client.class}, (proxy1, method, args) -> {
@@ -102,15 +106,18 @@ public class TemplateQueryParserTests extends ESTestCase {
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
         Index index = idxSettings.getIndex();
         // TODO: make this use a mock engine instead of mustache and it will no longer be messy!
-        ScriptModule scriptModule = new ScriptModule(new MustacheScriptEngineService(settings));
+        ScriptModule scriptModule = new ScriptModule(settings, environment, null, singletonList(new MustacheScriptEngineService(settings)),
+                emptyList());
         List<Setting<?>> scriptSettings = scriptModule.getSettings();
         scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
         SettingsModule settingsModule = new SettingsModule(settings, scriptSettings, Collections.emptyList());
-        final ThreadPool threadPool = new ThreadPool(settings);
         injector = new ModulesBuilder().add(
                 (b) -> {
-                    b.bind(Environment.class).toInstance(new Environment(settings));
-                    b.bind(ThreadPool.class).toInstance(threadPool);
+                    b.bind(ThreadPool.class).toInstance(new ThreadPool(settings));
+                    b.bind(Client.class).toInstance(proxy); // not needed here
+                    Multibinder.newSetBinder(b, ScoreFunctionParser.class);
+                    b.bind(ClusterService.class).toProvider(Providers.of((ClusterService) null));
+                    b.bind(CircuitBreakerService.class).to(NoneCircuitBreakerService.class);
                 },
                 settingsModule,
                 new SearchModule(settings, new NamedWriteableRegistry()) {
@@ -119,21 +126,10 @@ public class TemplateQueryParserTests extends ESTestCase {
                         // skip so we don't need transport
                     }
                 },
-                scriptModule,
-                new IndexSettingsModule(index, settings),
-                new AbstractModule() {
-                    @Override
-                    protected void configure() {
-                        bind(Client.class).toInstance(proxy); // not needed here
-                        Multibinder.newSetBinder(binder(), ScoreFunctionParser.class);
-                        bind(ClusterService.class).toProvider(Providers.of((ClusterService) null));
-                        bind(CircuitBreakerService.class).to(NoneCircuitBreakerService.class);
-                    }
-                }
+                new IndexSettingsModule(index, settings)
         ).createInjector();
 
-        AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
-        ScriptService scriptService = injector.getInstance(ScriptService.class);
+        AnalysisService analysisService = new AnalysisRegistry(null, environment).build(idxSettings);
         SimilarityService similarityService = new SimilarityService(idxSettings, Collections.emptyMap());
         MapperRegistry mapperRegistry = new IndicesModule(new NamedWriteableRegistry()).getMapperRegistry();
         MapperService mapperService = new MapperService(idxSettings, analysisService, similarityService, mapperRegistry, () ->
@@ -153,7 +149,7 @@ public class TemplateQueryParserTests extends ESTestCase {
         });
         IndicesQueriesRegistry indicesQueriesRegistry = injector.getInstance(IndicesQueriesRegistry.class);
         contextFactory =  () -> new QueryShardContext(idxSettings, bitsetFilterCache, indexFieldDataService, mapperService,
-                similarityService, scriptService, indicesQueriesRegistry, proxy, null, null);
+                similarityService, scriptModule.getScriptService(), indicesQueriesRegistry, proxy, null, null);
     }
 
     @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
index ea2154b355e..5eeaa635305 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
@@ -901,7 +901,6 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
                             bindMapperExtension();
                         }
                     },
-                    scriptModule,
                     new IndexSettingsModule(index, indexSettings),
                     searchModule,
                     new AbstractModule() {
@@ -919,7 +918,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             IndexScopedSettings indexScopedSettings = injector.getInstance(IndexScopedSettings.class);
             idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings, indexScopedSettings);
             AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
-            scriptService = injector.getInstance(ScriptService.class);
+            scriptService = scriptModule.getScriptService();
             similarityService = new SimilarityService(idxSettings, Collections.emptyMap());
             MapperRegistry mapperRegistry = injector.getInstance(MapperRegistry.class);
             mapperService = new MapperService(idxSettings, analysisService, similarityService, mapperRegistry,
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index e7e6eca5c88..76616fbdb44 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -58,11 +58,8 @@ import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.script.MockScriptEngine;
-import org.elasticsearch.script.ScriptContextRegistry;
-import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.script.ScriptSettings;
 import org.elasticsearch.search.MockSearchService;
 import org.elasticsearch.test.junit.listeners.LoggingListener;
 import org.elasticsearch.test.junit.listeners.ReproduceInfoPrinter;
@@ -97,6 +94,8 @@ import java.util.function.Consumer;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
 import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -796,25 +795,12 @@ public abstract class ESTestCase extends LuceneTestCase {
     }
 
     public static ScriptModule newTestScriptModule() {
-        return new ScriptModule(new MockScriptEngine()) {
-            @Override
-            protected void configure() {
-                Settings settings = Settings.builder()
-                    .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
-                    // no file watching, so we don't need a ResourceWatcherService
-                    .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
-                    .build();
-                bind(ScriptEngineRegistry.class).toInstance(scriptEngineRegistry);
-                bind(ScriptContextRegistry.class).toInstance(scriptContextRegistry);
-                bind(ScriptSettings.class).toInstance(scriptSettings);
-                try {
-                    ScriptService scriptService = new ScriptService(settings, new Environment(settings), null,
-                        scriptEngineRegistry, scriptContextRegistry, scriptSettings);
-                    bind(ScriptService.class).toInstance(scriptService);
-                } catch (IOException e) {
-                    throw new IllegalStateException("error while binding ScriptService", e);
-                }
-            }
-        };
+        Settings settings = Settings.builder()
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
+                // no file watching, so we don't need a ResourceWatcherService
+                .put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
+                .build();
+        Environment environment = new Environment(settings);
+        return new ScriptModule(settings, environment, null, singletonList(new MockScriptEngine()), emptyList());
     }
 }

From dcd394d83fed984df5341624271f506404dbe886 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 21 Jun 2016 15:39:55 -0400
Subject: [PATCH 1012/1311] Inline TaskManager#register

---
 .../org/elasticsearch/tasks/TaskManager.java  | 38 ++++++++++---------
 1 file changed, 21 insertions(+), 17 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskManager.java b/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
index 95adc850f82..7a7a71174a6 100644
--- a/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
+++ b/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
@@ -92,23 +92,7 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen
         }
 
         if (task instanceof CancellableTask) {
-            CancellableTask cancellableTask = (CancellableTask) task;
-            CancellableTaskHolder holder = new CancellableTaskHolder(cancellableTask);
-            CancellableTaskHolder oldHolder = cancellableTasks.put(task.getId(), holder);
-            assert oldHolder == null;
-            // Check if this task was banned before we start it
-            if (task.getParentTaskId().isSet() && banedParents.isEmpty() == false) {
-                String reason = banedParents.get(task.getParentTaskId());
-                if (reason != null) {
-                    try {
-                        holder.cancel(reason);
-                        throw new IllegalStateException("Task cancelled before it started: " + reason);
-                    } finally {
-                        // let's clean up the registration
-                        unregister(task);
-                    }
-                }
-            }
+            registerCancellableTask(task);
         } else {
             Task previousTask = tasks.put(task.getId(), task);
             assert previousTask == null;
@@ -116,6 +100,26 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen
         return task;
     }
 
+    private void registerCancellableTask(Task task) {
+        CancellableTask cancellableTask = (CancellableTask) task;
+        CancellableTaskHolder holder = new CancellableTaskHolder(cancellableTask);
+        CancellableTaskHolder oldHolder = cancellableTasks.put(task.getId(), holder);
+        assert oldHolder == null;
+        // Check if this task was banned before we start it
+        if (task.getParentTaskId().isSet() && banedParents.isEmpty() == false) {
+            String reason = banedParents.get(task.getParentTaskId());
+            if (reason != null) {
+                try {
+                    holder.cancel(reason);
+                    throw new IllegalStateException("Task cancelled before it started: " + reason);
+                } finally {
+                    // let's clean up the registration
+                    unregister(task);
+                }
+            }
+        }
+    }
+
     /**
      * Cancels a task
      * <p>

From 9d1ef6243185984705f570955b3eb3aaf69ab4f5 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 21 Jun 2016 15:43:10 -0400
Subject: [PATCH 1013/1311] Inline ReplicationOperation#execute

---
 .../support/replication/ReplicationOperation.java     | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
index 0d99c4bfcdf..830f092178f 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
@@ -112,6 +112,14 @@ public class ReplicationOperation<
         if (logger.isTraceEnabled()) {
             logger.trace("[{}] op [{}] completed on primary for request [{}]", primaryId, opType, request);
         }
+
+        performOnReplicas(primaryId, replicaRequest);
+
+        successfulShards.incrementAndGet();
+        decPendingAndFinishIfNeeded();
+    }
+
+    private void performOnReplicas(ShardId primaryId, ReplicaRequest replicaRequest) {
         // we have to get a new state after successfully indexing into the primary in order to honour recovery semantics.
         // we have to make sure that every operation indexed into the primary after recovery start will also be replicated
         // to the recovery target. If we use an old cluster state, we may miss a relocation that has started since then.
@@ -134,9 +142,6 @@ public class ReplicationOperation<
                 performOnReplica(shard.buildTargetRelocatingShard(), replicaRequest);
             }
         }
-
-        successfulShards.incrementAndGet();
-        decPendingAndFinishIfNeeded();
     }
 
     private void performOnReplica(final ShardRouting shard, final ReplicaRequest replicaRequest) {

From af7f98205a6ff0e4a32c04d8f161badc3d6ac055 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 21 Jun 2016 17:19:14 -0400
Subject: [PATCH 1014/1311] Inline StreamInput#readGenericValue

---
 .../common/io/stream/StreamInput.java         | 91 ++++++++++++-------
 1 file changed, 60 insertions(+), 31 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
index ec56103e7a0..30b5cd7f2cf 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
@@ -439,45 +439,21 @@ public abstract class StreamInput extends InputStream {
             case 5:
                 return readBoolean();
             case 6:
-                int bytesSize = readVInt();
-                byte[] value = new byte[bytesSize];
-                readBytes(value, 0, bytesSize);
-                return value;
+                return fastReadByteArray();
             case 7:
-                int size = readVInt();
-                List list = new ArrayList(size);
-                for (int i = 0; i < size; i++) {
-                    list.add(readGenericValue());
-                }
-                return list;
+                return readArrayList();
             case 8:
-                int size8 = readVInt();
-                Object[] list8 = new Object[size8];
-                for (int i = 0; i < size8; i++) {
-                    list8[i] = readGenericValue();
-                }
-                return list8;
+                return readArray();
             case 9:
-                int size9 = readVInt();
-                Map map9 = new LinkedHashMap(size9);
-                for (int i = 0; i < size9; i++) {
-                    map9.put(readString(), readGenericValue());
-                }
-                return map9;
+                return readLinkedHashMap();
             case 10:
-                int size10 = readVInt();
-                Map map10 = new HashMap(size10);
-                for (int i = 0; i < size10; i++) {
-                    map10.put(readString(), readGenericValue());
-                }
-                return map10;
+                return readHashMap();
             case 11:
                 return readByte();
             case 12:
-                return new Date(readLong());
+                return readDate();
             case 13:
-                final String timeZoneId = readString();
-                return new DateTime(readLong(), DateTimeZone.forID(timeZoneId));
+                return readDateTime();
             case 14:
                 return readBytesReference();
             case 15:
@@ -501,6 +477,59 @@ public abstract class StreamInput extends InputStream {
         }
     }
 
+    private byte[] fastReadByteArray() throws IOException {
+        int bytesSize = readVInt();
+        byte[] value = new byte[bytesSize];
+        readBytes(value, 0, bytesSize);
+        return value;
+    }
+
+    @SuppressWarnings("unchecked")
+    private List readArrayList() throws IOException {
+        int size = readVInt();
+        List list = new ArrayList(size);
+        for (int i = 0; i < size; i++) {
+            list.add(readGenericValue());
+        }
+        return list;
+    }
+
+    private DateTime readDateTime() throws IOException {
+        final String timeZoneId = readString();
+        return new DateTime(readLong(), DateTimeZone.forID(timeZoneId));
+    }
+
+    private Object[] readArray() throws IOException {
+        int size8 = readVInt();
+        Object[] list8 = new Object[size8];
+        for (int i = 0; i < size8; i++) {
+            list8[i] = readGenericValue();
+        }
+        return list8;
+    }
+
+    private Map readLinkedHashMap() throws IOException {
+        int size9 = readVInt();
+        Map map9 = new LinkedHashMap(size9);
+        for (int i = 0; i < size9; i++) {
+            map9.put(readString(), readGenericValue());
+        }
+        return map9;
+    }
+
+    private Map readHashMap() throws IOException {
+        int size10 = readVInt();
+        Map map10 = new HashMap(size10);
+        for (int i = 0; i < size10; i++) {
+            map10.put(readString(), readGenericValue());
+        }
+        return map10;
+    }
+
+    private Date readDate() throws IOException {
+        return new Date(readLong());
+    }
+
     /**
      * Reads a {@link GeoPoint} from this stream input
      */

From 93c3a899940b5a5abe4cce11f46658d03ee0ce54 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 21 Jun 2016 17:19:31 -0400
Subject: [PATCH 1015/1311] Inline StreamOutput#writeGenericValue

---
 .../common/io/stream/StreamOutput.java        | 222 +++++++++++-------
 1 file changed, 138 insertions(+), 84 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
index eb93c9af628..7d17029cd61 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
@@ -48,7 +48,9 @@ import java.nio.file.FileSystemException;
 import java.nio.file.FileSystemLoopException;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.NotDirectoryException;
+import java.util.Collections;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -401,100 +403,152 @@ public abstract class StreamOutput extends OutputStream {
         writeGenericValue(map);
     }
 
+    @FunctionalInterface
+    interface Writer {
+        void write(StreamOutput o, Object value) throws IOException;
+    }
+
+    private final static Map<Class<?>, Writer> WRITERS;
+
+    static {
+        Map<Class<?>, Writer> writers = new HashMap<>();
+        writers.put(String.class, (o, v) -> {
+            o.writeByte((byte) 0);
+            o.writeString((String) v);
+        });
+        writers.put(Integer.class, (o, v) -> {
+            o.writeByte((byte) 1);
+            o.writeInt((Integer) v);
+        });
+        writers.put(Long.class, (o, v) -> {
+            o.writeByte((byte) 2);
+            o.writeLong((Long) v);
+        });
+        writers.put(Float.class, (o, v) -> {
+            o.writeByte((byte) 3);
+            o.writeFloat((float) v);
+        });
+        writers.put(Double.class, (o, v) -> {
+            o.writeByte((byte) 4);
+            o.writeDouble((double) v);
+        });
+        writers.put(Boolean.class, (o, v) -> {
+            o.writeByte((byte) 5);
+            o.writeBoolean((boolean) v);
+        });
+        writers.put(byte[].class, (o, v) -> {
+            o.writeByte((byte) 6);
+            final byte[] bytes = (byte[]) v;
+            o.writeVInt(bytes.length);
+            o.writeBytes(bytes);
+        });
+        writers.put(List.class, (o, v) -> {
+            o.writeByte((byte) 7);
+            final List list = (List) v;
+            o.writeVInt(list.size());
+            for (Object item : list) {
+                o.writeGenericValue(item);
+            }
+        });
+        writers.put(Object[].class, (o, v) -> {
+            o.writeByte((byte) 8);
+            final Object[] list = (Object[]) v;
+            o.writeVInt(list.length);
+            for (Object item : list) {
+                o.writeGenericValue(item);
+            }
+        });
+        writers.put(Map.class, (o, v) -> {
+            if (v instanceof LinkedHashMap) {
+                o.writeByte((byte) 9);
+            } else {
+                o.writeByte((byte) 10);
+            }
+            @SuppressWarnings("unchecked")
+            final Map<String, Object> map = (Map<String, Object>) v;
+            o.writeVInt(map.size());
+            for (Map.Entry<String, Object> entry : map.entrySet()) {
+                o.writeString(entry.getKey());
+                o.writeGenericValue(entry.getValue());
+            }
+        });
+        writers.put(Byte.class, (o, v) -> {
+            o.writeByte((byte) 11);
+            o.writeByte((Byte) v);
+        });
+        writers.put(Date.class, (o, v) -> {
+            o.writeByte((byte) 12);
+            o.writeLong(((Date) v).getTime());
+        });
+        writers.put(ReadableInstant.class, (o, v) -> {
+            o.writeByte((byte) 13);
+            final ReadableInstant instant = (ReadableInstant) v;
+            o.writeString(instant.getZone().getID());
+            o.writeLong(instant.getMillis());
+        });
+        writers.put(BytesReference.class, (o, v) -> {
+            o.writeByte((byte) 14);
+            o.writeBytesReference((BytesReference) v);
+        });
+        writers.put(Text.class, (o, v) -> {
+            o.writeByte((byte) 15);
+            o.writeText((Text) v);
+        });
+        writers.put(Short.class, (o, v) -> {
+            o.writeByte((byte) 16);
+            o.writeShort((Short) v);
+        });
+        writers.put(int[].class, (o, v) -> {
+            o.writeByte((byte) 17);
+            o.writeIntArray((int[]) v);
+        });
+        writers.put(long[].class, (o, v) -> {
+            o.writeByte((byte) 18);
+            o.writeLongArray((long[]) v);
+        });
+        writers.put(float[].class, (o, v) -> {
+            o.writeByte((byte) 19);
+            o.writeFloatArray((float[]) v);
+        });
+        writers.put(double[].class, (o, v) -> {
+            o.writeByte((byte) 20);
+            o.writeDoubleArray((double[]) v);
+        });
+        writers.put(BytesRef.class, (o, v) -> {
+            o.writeByte((byte) 21);
+            o.writeBytesRef((BytesRef) v);
+        });
+        writers.put(GeoPoint.class, (o, v) -> {
+            o.writeByte((byte) 22);
+            o.writeGeoPoint((GeoPoint) v);
+        });
+        WRITERS = Collections.unmodifiableMap(writers);
+    }
+
     public void writeGenericValue(@Nullable Object value) throws IOException {
         if (value == null) {
             writeByte((byte) -1);
             return;
         }
-        Class type = value.getClass();
-        if (type == String.class) {
-            writeByte((byte) 0);
-            writeString((String) value);
-        } else if (type == Integer.class) {
-            writeByte((byte) 1);
-            writeInt((Integer) value);
-        } else if (type == Long.class) {
-            writeByte((byte) 2);
-            writeLong((Long) value);
-        } else if (type == Float.class) {
-            writeByte((byte) 3);
-            writeFloat((Float) value);
-        } else if (type == Double.class) {
-            writeByte((byte) 4);
-            writeDouble((Double) value);
-        } else if (type == Boolean.class) {
-            writeByte((byte) 5);
-            writeBoolean((Boolean) value);
-        } else if (type == byte[].class) {
-            writeByte((byte) 6);
-            writeVInt(((byte[]) value).length);
-            writeBytes(((byte[]) value));
-        } else if (value instanceof List) {
-            writeByte((byte) 7);
-            List list = (List) value;
-            writeVInt(list.size());
-            for (Object o : list) {
-                writeGenericValue(o);
-            }
+        final Class type;
+        if (value instanceof List) {
+            type = List.class;
         } else if (value instanceof Object[]) {
-            writeByte((byte) 8);
-            Object[] list = (Object[]) value;
-            writeVInt(list.length);
-            for (Object o : list) {
-                writeGenericValue(o);
-            }
+            type = Object[].class;
         } else if (value instanceof Map) {
-            if (value instanceof LinkedHashMap) {
-                writeByte((byte) 9);
-            } else {
-                writeByte((byte) 10);
-            }
-            @SuppressWarnings("unchecked")
-            Map<String, Object> map = (Map<String, Object>) value;
-            writeVInt(map.size());
-            for (Map.Entry<String, Object> entry : map.entrySet()) {
-                writeString(entry.getKey());
-                writeGenericValue(entry.getValue());
-            }
-        } else if (type == Byte.class) {
-            writeByte((byte) 11);
-            writeByte((Byte) value);
-        } else if (type == Date.class) {
-            writeByte((byte) 12);
-            writeLong(((Date) value).getTime());
+            type = Map.class;
         } else if (value instanceof ReadableInstant) {
-            writeByte((byte) 13);
-            writeString(((ReadableInstant) value).getZone().getID());
-            writeLong(((ReadableInstant) value).getMillis());
+            type = ReadableInstant.class;
         } else if (value instanceof BytesReference) {
-            writeByte((byte) 14);
-            writeBytesReference((BytesReference) value);
-        } else if (value instanceof Text) {
-            writeByte((byte) 15);
-            writeText((Text) value);
-        } else if (type == Short.class) {
-            writeByte((byte) 16);
-            writeShort((Short) value);
-        } else if (type == int[].class) {
-            writeByte((byte) 17);
-            writeIntArray((int[]) value);
-        } else if (type == long[].class) {
-            writeByte((byte) 18);
-            writeLongArray((long[]) value);
-        } else if (type == float[].class) {
-            writeByte((byte) 19);
-            writeFloatArray((float[]) value);
-        } else if (type == double[].class) {
-            writeByte((byte) 20);
-            writeDoubleArray((double[]) value);
-        } else if (value instanceof BytesRef) {
-            writeByte((byte) 21);
-            writeBytesRef((BytesRef) value);
-        } else if (type == GeoPoint.class) {
-            writeByte((byte) 22);
-            writeGeoPoint((GeoPoint) value);
+            type = BytesReference.class;
         } else {
-            throw new IOException("Can't write type [" + type + "]");
+            type = value.getClass();
+        }
+        final Writer writer = WRITERS.get(type);
+        if (writer != null) {
+            writer.write(this, value);
+        } else {
+            throw new IOException("can not write type [" + type + "]");
         }
     }
 

From 81ba43888f7fbf9783832a32717d066f4ddfef09 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 21 Jun 2016 17:33:33 -0400
Subject: [PATCH 1016/1311] Inline AbstractSearchAsyncAction#init

---
 .../action/search/AbstractSearchAsyncAction.java   | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
index 99dcd20d115..805d09a0830 100644
--- a/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
@@ -108,6 +108,15 @@ abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult>
 
         shardsIts = clusterService.operationRouting().searchShards(clusterState, concreteIndices, routingMap, request.preference());
         final int shardCount = shardsIts.size();
+        failIfOverShardCountLimit(clusterService, shardCount);
+        expectedSuccessfulOps = shardCount;
+        // we need to add 1 for non active partition, since we count it in the total!
+        expectedTotalOps = shardsIts.totalSizeWith1ForEmpty();
+
+        firstResults = new AtomicArray<>(shardsIts.size());
+    }
+
+    private void failIfOverShardCountLimit(ClusterService clusterService, int shardCount) {
         final long shardCountLimit = clusterService.getClusterSettings().get(TransportSearchAction.SHARD_COUNT_LIMIT_SETTING);
         if (shardCount > shardCountLimit) {
             throw new IllegalArgumentException("Trying to query " + shardCount + " shards, which is over the limit of "
@@ -116,11 +125,6 @@ abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult>
                     + "have a smaller number of larger shards. Update [" + TransportSearchAction.SHARD_COUNT_LIMIT_SETTING.getKey()
                     + "] to a greater value if you really want to query that many shards at the same time.");
         }
-        expectedSuccessfulOps = shardCount;
-        // we need to add 1 for non active partition, since we count it in the total!
-        expectedTotalOps = shardsIts.totalSizeWith1ForEmpty();
-
-        firstResults = new AtomicArray<>(shardsIts.size());
     }
 
     public void start() {

From abae58b5fb9a46ac407a9f587c2bbbbcf39067c1 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 21 Jun 2016 18:08:15 -0400
Subject: [PATCH 1017/1311] Inline TransportSearchAction#doExecute

---
 .../action/search/TransportSearchAction.java           | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
index 6e6583bae14..8a33bff8f0e 100644
--- a/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
@@ -42,9 +42,6 @@ import java.util.Set;
 import static org.elasticsearch.action.search.SearchType.QUERY_AND_FETCH;
 import static org.elasticsearch.action.search.SearchType.QUERY_THEN_FETCH;
 
-/**
- *
- */
 public class TransportSearchAction extends HandledTransportAction<SearchRequest, SearchResponse> {
 
     /** The maximum number of shards for a single search request. */
@@ -96,6 +93,10 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
             logger.debug("failed to optimize search type, continue as normal", e);
         }
 
+        searchAsyncAction(searchRequest, listener).start();
+    }
+
+    private AbstractSearchAsyncAction searchAsyncAction(SearchRequest searchRequest, ActionListener<SearchResponse> listener) {
         AbstractSearchAsyncAction searchAsyncAction;
         switch(searchRequest.searchType()) {
             case DFS_QUERY_THEN_FETCH:
@@ -117,6 +118,7 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
             default:
                 throw new IllegalStateException("Unknown search type: [" + searchRequest.searchType() + "]");
         }
-        searchAsyncAction.start();
+        return searchAsyncAction;
     }
+
 }

From 4f49a261a748b6fa75e0da46b11f4f4a8011ab47 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 21 Jun 2016 19:07:03 -0400
Subject: [PATCH 1018/1311] Refactor InternalEngine inner methods

This commit refactors InternalEngine#innerIndex and
InternalEngine#innerDelete to collapse some common logic into a single
method. This has the advantage that it shrinks the bytecode size of
InternalEngine#innerIndex so that it can be inlined.
---
 .../elasticsearch/index/engine/Engine.java    |   8 +
 .../index/engine/InternalEngine.java          | 204 ++++++++++--------
 .../index/engine/LiveVersionMap.java          |   9 +-
 3 files changed, 127 insertions(+), 94 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/engine/Engine.java b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
index 87ffb9331a6..5c1a20c16e3 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -840,6 +840,10 @@ public abstract class Engine implements Closeable {
         public long endTime() {
             return this.endTime;
         }
+
+        abstract String type();
+
+        abstract String id();
     }
 
     public static class Index extends Operation {
@@ -863,10 +867,12 @@ public abstract class Engine implements Closeable {
             return this.doc;
         }
 
+        @Override
         public String type() {
             return this.doc.type();
         }
 
+        @Override
         public String id() {
             return this.doc.id();
         }
@@ -929,10 +935,12 @@ public abstract class Engine implements Closeable {
             this(template.type(), template.id(), template.uid(), template.version(), versionType, template.origin(), template.startTime(), template.found());
         }
 
+        @Override
         public String type() {
             return this.type;
         }
 
+        @Override
         public String id() {
             return this.id;
         }
diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index 15667e79421..449c04ce4d5 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -314,7 +314,7 @@ public class InternalEngine extends Engine {
         try (ReleasableLock lock = readLock.acquire()) {
             ensureOpen();
             if (get.realtime()) {
-                VersionValue versionValue = versionMap.getUnderLock(get.uid().bytes());
+                VersionValue versionValue = versionMap.getUnderLock(get.uid());
                 if (versionValue != null) {
                     if (versionValue.delete()) {
                         return GetResult.NOT_EXISTS;
@@ -336,6 +336,59 @@ public class InternalEngine extends Engine {
         }
     }
 
+    private boolean checkVersionConflict(
+            final Operation op,
+            final long currentVersion,
+            final long expectedVersion,
+            final boolean deleted) {
+        if (op.versionType().isVersionConflictForWrites(currentVersion, expectedVersion, deleted)) {
+            if (op.origin().isRecovery()) {
+                // version conflict, but okay
+                return true;
+            } else {
+                // fatal version conflict
+                throw new VersionConflictEngineException(shardId, op.type(), op.id(),
+                        op.versionType().explainConflictForWrites(currentVersion, expectedVersion, deleted));
+            }
+        }
+        return false;
+    }
+
+    private long checkDeletedAndGCed(VersionValue versionValue) {
+        long currentVersion;
+        if (engineConfig.isEnableGcDeletes() && versionValue.delete() && (engineConfig.getThreadPool().estimatedTimeInMillis() - versionValue.time()) > getGcDeletesInMillis()) {
+            currentVersion = Versions.NOT_FOUND; // deleted, and GC
+        } else {
+            currentVersion = versionValue.version();
+        }
+        return currentVersion;
+    }
+
+    private static VersionValueSupplier NEW_VERSION_VALUE = (u, t, l) -> new VersionValue(u, l);
+
+    @FunctionalInterface
+    private interface VersionValueSupplier {
+        VersionValue apply(long updatedVersion, long time, Translog.Location location);
+    }
+
+    private <T extends Engine.Operation> void maybeAddToTranslog(
+            final T op,
+            final long updatedVersion,
+            final Function<T, Translog.Operation> toTranslogOp,
+            final VersionValueSupplier toVersionValue) throws IOException {
+        if (op.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) {
+            final Translog.Location translogLocation = translog.add(toTranslogOp.apply(op));
+            op.setTranslogLocation(translogLocation);
+            versionMap.putUnderLock(op.uid().bytes(), toVersionValue.apply(updatedVersion, engineConfig.getThreadPool().estimatedTimeInMillis(), op.getTranslogLocation()));
+        } else {
+            // we do not replay in to the translog, so there is no
+            // translog location; that is okay because real-time
+            // gets are not possible during recovery and we will
+            // flush when the recovery is complete
+            versionMap.putUnderLock(op.uid().bytes(), toVersionValue.apply(updatedVersion, engineConfig.getThreadPool().estimatedTimeInMillis(), null));
+        }
+    }
+
     @Override
     public boolean index(Index index) {
         final boolean created;
@@ -361,58 +414,56 @@ public class InternalEngine extends Engine {
             lastWriteNanos = index.startTime();
             final long currentVersion;
             final boolean deleted;
-            VersionValue versionValue = versionMap.getUnderLock(index.uid().bytes());
+            final VersionValue versionValue = versionMap.getUnderLock(index.uid());
             if (versionValue == null) {
                 currentVersion = loadCurrentVersionFromIndex(index.uid());
                 deleted = currentVersion == Versions.NOT_FOUND;
             } else {
+                currentVersion = checkDeletedAndGCed(versionValue);
                 deleted = versionValue.delete();
-                if (engineConfig.isEnableGcDeletes() && versionValue.delete() && (engineConfig.getThreadPool().estimatedTimeInMillis() - versionValue.time()) > getGcDeletesInMillis()) {
-                    currentVersion = Versions.NOT_FOUND; // deleted, and GC
-                } else {
-                    currentVersion = versionValue.version();
-                }
             }
 
-            long expectedVersion = index.version();
-            if (isVersionConflictForWrites(index, currentVersion, deleted, expectedVersion)) {
-                if (!index.origin().isRecovery()) {
-                    throw new VersionConflictEngineException(shardId, index.type(), index.id(),
-                        index.versionType().explainConflictForWrites(currentVersion, expectedVersion, deleted));
-                }
-                return false;
-            }
-            long updatedVersion = index.versionType().updateVersion(currentVersion, expectedVersion);
+            final long expectedVersion = index.version();
+            if (checkVersionConflict(index, currentVersion, expectedVersion, deleted)) return false;
 
-            final boolean created;
-            index.updateVersion(updatedVersion);
+            final long updatedVersion = updateVersion(index, currentVersion, expectedVersion);
 
-            if (currentVersion == Versions.NOT_FOUND) {
-                // document does not exists, we can optimize for create
-                created = true;
-                index(index, indexWriter);
-            } else {
-                created = update(index, versionValue, indexWriter);
-            }
+            final boolean created = indexOrUpdate(index, currentVersion, versionValue);
 
-            if (index.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) {
-                final Translog.Location translogLocation = translog.add(new Translog.Index(index));
-                index.setTranslogLocation(translogLocation);
-                versionMap.putUnderLock(index.uid().bytes(), new VersionValue(updatedVersion, index.getTranslogLocation()));
-            } else {
-                // we do not replay in to the translog, so there is no
-                // translog location; that is okay because real-time
-                // gets are not possible during recovery and we will
-                // flush when the recovery is complete
-                versionMap.putUnderLock(index.uid().bytes(), new VersionValue(updatedVersion, null));
-            }
+            maybeAddToTranslog(index, updatedVersion, Translog.Index::new, NEW_VERSION_VALUE);
 
             return created;
         }
     }
 
-    private static boolean update(Index index, VersionValue versionValue, IndexWriter indexWriter) throws IOException {
-        boolean created;
+    private long updateVersion(Engine.Operation op, long currentVersion, long expectedVersion) {
+        final long updatedVersion = op.versionType().updateVersion(currentVersion, expectedVersion);
+        op.updateVersion(updatedVersion);
+        return updatedVersion;
+    }
+
+    private boolean indexOrUpdate(final Index index, final long currentVersion, final VersionValue versionValue) throws IOException {
+        final boolean created;
+        if (currentVersion == Versions.NOT_FOUND) {
+            // document does not exists, we can optimize for create
+            created = true;
+            index(index, indexWriter);
+        } else {
+            created = update(index, versionValue, indexWriter);
+        }
+        return created;
+    }
+
+    private static void index(final Index index, final IndexWriter indexWriter) throws IOException {
+        if (index.docs().size() > 1) {
+            indexWriter.addDocuments(index.docs());
+        } else {
+            indexWriter.addDocument(index.docs().get(0));
+        }
+    }
+
+    private static boolean update(final Index index, final VersionValue versionValue, final IndexWriter indexWriter) throws IOException {
+        final boolean created;
         if (versionValue != null) {
             created = versionValue.delete(); // we have a delete which is not GC'ed...
         } else {
@@ -426,18 +477,6 @@ public class InternalEngine extends Engine {
         return created;
     }
 
-    private static void index(Index index, IndexWriter indexWriter) throws IOException {
-        if (index.docs().size() > 1) {
-            indexWriter.addDocuments(index.docs());
-        } else {
-            indexWriter.addDocument(index.docs().get(0));
-        }
-    }
-
-    private boolean isVersionConflictForWrites(Index index, long currentVersion, boolean deleted, long expectedVersion) {
-        return index.versionType().isVersionConflictForWrites(currentVersion, expectedVersion, deleted);
-    }
-
     @Override
     public void delete(Delete delete) throws EngineException {
         try (ReleasableLock lock = readLock.acquire()) {
@@ -465,59 +504,44 @@ public class InternalEngine extends Engine {
             lastWriteNanos = delete.startTime();
             final long currentVersion;
             final boolean deleted;
-            VersionValue versionValue = versionMap.getUnderLock(delete.uid().bytes());
+            final VersionValue versionValue = versionMap.getUnderLock(delete.uid());
             if (versionValue == null) {
                 currentVersion = loadCurrentVersionFromIndex(delete.uid());
                 deleted = currentVersion == Versions.NOT_FOUND;
             } else {
+                currentVersion = checkDeletedAndGCed(versionValue);
                 deleted = versionValue.delete();
-                if (engineConfig.isEnableGcDeletes() && versionValue.delete() && (engineConfig.getThreadPool().estimatedTimeInMillis() - versionValue.time()) > getGcDeletesInMillis()) {
-                    currentVersion = Versions.NOT_FOUND; // deleted, and GC
-                } else {
-                    currentVersion = versionValue.version();
-                }
             }
 
-            long updatedVersion;
-            long expectedVersion = delete.version();
-            if (delete.versionType().isVersionConflictForWrites(currentVersion, expectedVersion, deleted)) {
-                if (delete.origin().isRecovery()) {
-                    return;
-                } else {
-                    throw new VersionConflictEngineException(shardId, delete.type(), delete.id(),
-                            delete.versionType().explainConflictForWrites(currentVersion, expectedVersion, deleted));
-                }
-            }
-            updatedVersion = delete.versionType().updateVersion(currentVersion, expectedVersion);
-            final boolean found;
-            if (currentVersion == Versions.NOT_FOUND) {
-                // doc does not exist and no prior deletes
-                found = false;
-            } else if (versionValue != null && versionValue.delete()) {
-                // a "delete on delete", in this case, we still increment the version, log it, and return that version
-                found = false;
-            } else {
-                // we deleted a currently existing document
-                indexWriter.deleteDocuments(delete.uid());
-                found = true;
-            }
+            final long expectedVersion = delete.version();
+            if (checkVersionConflict(delete, currentVersion, expectedVersion, deleted)) return;
+
+            final long updatedVersion = updateVersion(delete, currentVersion, expectedVersion);
+
+            final boolean found = deleteIfFound(delete, currentVersion, deleted, versionValue);
 
             delete.updateVersion(updatedVersion, found);
 
-            if (delete.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) {
-                final Translog.Location translogLocation = translog.add(new Translog.Delete(delete));
-                delete.setTranslogLocation(translogLocation);
-                versionMap.putUnderLock(delete.uid().bytes(), new DeleteVersionValue(updatedVersion, engineConfig.getThreadPool().estimatedTimeInMillis(), delete.getTranslogLocation()));
-            } else {
-                // we do not replay in to the translog, so there is no
-                // translog location; that is okay because real-time
-                // gets are not possible during recovery and we will
-                // flush when the recovery is complete
-                versionMap.putUnderLock(delete.uid().bytes(), new DeleteVersionValue(updatedVersion, engineConfig.getThreadPool().estimatedTimeInMillis(), null));
-            }
+            maybeAddToTranslog(delete, updatedVersion, Translog.Delete::new, DeleteVersionValue::new);
         }
     }
 
+    private boolean deleteIfFound(Delete delete, long currentVersion, boolean deleted, VersionValue versionValue) throws IOException {
+        final boolean found;
+        if (currentVersion == Versions.NOT_FOUND) {
+            // doc does not exist and no prior deletes
+            found = false;
+        } else if (versionValue != null && deleted) {
+            // a "delete on delete", in this case, we still increment the version, log it, and return that version
+            found = false;
+        } else {
+            // we deleted a currently existing document
+            indexWriter.deleteDocuments(delete.uid());
+            found = true;
+        }
+        return found;
+    }
+
     @Override
     public void refresh(String source) throws EngineException {
         // we obtain a read lock here, since we don't want a flush to happen while we are refreshing
diff --git a/core/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java b/core/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java
index f962d31bf8b..3cf3c83749c 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/LiveVersionMap.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.engine;
 
+import org.apache.lucene.index.Term;
 import org.apache.lucene.search.ReferenceManager;
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.BytesRef;
@@ -126,21 +127,21 @@ class LiveVersionMap implements ReferenceManager.RefreshListener, Accountable {
     }
 
     /** Returns the live version (add or delete) for this uid. */
-    VersionValue getUnderLock(BytesRef uid) {
+    VersionValue getUnderLock(final Term uid) {
         Maps currentMaps = maps;
 
         // First try to get the "live" value:
-        VersionValue value = currentMaps.current.get(uid);
+        VersionValue value = currentMaps.current.get(uid.bytes());
         if (value != null) {
             return value;
         }
 
-        value = currentMaps.old.get(uid);
+        value = currentMaps.old.get(uid.bytes());
         if (value != null) {
             return value;
         }
 
-        return tombstones.get(uid);
+        return tombstones.get(uid.bytes());
     }
 
     /** Adds this uid/version to the pending adds map. */

From e817b5daa3c762e0adba87d1e524d53a99c5362f Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Tue, 21 Jun 2016 22:50:39 -0700
Subject: [PATCH 1019/1311] Plugins: Remove guice from Mapper plugins

This changes adds a MapperPlugin interface which allows pull style
retrieval of mappers and metadata mappers added by plugins. For now, I
have kept the MapperRegistry, but this should be removed in the future
as it is just a silly container for 2 maps which could themselves be
passed around.
---
 .../metadata/MetaDataIndexUpgradeService.java |   2 -
 .../elasticsearch/indices/IndicesModule.java  | 124 ++++++++----------
 .../java/org/elasticsearch/node/Node.java     |   3 +-
 .../elasticsearch/plugins/MapperPlugin.java   |  55 ++++++++
 .../elasticsearch/index/IndexModuleTests.java |   2 +-
 .../index/engine/InternalEngineTests.java     |   2 +-
 .../externalvalues/ExternalMapperPlugin.java  |  28 +++-
 .../internal/FieldNamesFieldMapperTests.java  |   9 +-
 .../aggregations/AggregatorParsingTests.java  |   3 +-
 .../aggregations/BaseAggregationTestCase.java |   3 +-
 .../builder/SearchSourceBuilderTests.java     |   2 +-
 .../messy/tests/TemplateQueryParserTests.java |  13 +-
 .../percolator/PercolatorPlugin.java          |  22 ++--
 .../attachments/MapperAttachmentsPlugin.java  |  19 +--
 .../attachments/AttachmentUnitTestCase.java   |  13 +-
 .../plugin/mapper/MapperMurmur3Plugin.java    |  14 +-
 .../plugin/mapper/MapperSizePlugin.java       |  13 +-
 .../index/mapper/size/SizeMappingTests.java   |  10 +-
 .../elasticsearch/index/MapperTestUtils.java  |   2 +-
 .../test/AbstractQueryTestCase.java           |   2 +-
 .../org/elasticsearch/test/ESTestCase.java    |  22 ++++
 21 files changed, 236 insertions(+), 127 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/plugins/MapperPlugin.java

diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
index 4536670872d..d1141aeb9f4 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java
@@ -34,8 +34,6 @@ import org.elasticsearch.indices.mapper.MapperRegistry;
 
 import java.util.Collections;
 
-import static org.elasticsearch.common.util.set.Sets.newHashSet;
-
 /**
  * This service is responsible for upgrading legacy index metadata to the current version
  * <p>
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
index 1817a71073c..e39e8b7e312 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
@@ -64,8 +64,11 @@ import org.elasticsearch.indices.recovery.RecoveryTargetService;
 import org.elasticsearch.indices.store.IndicesStore;
 import org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData;
 import org.elasticsearch.indices.ttl.IndicesTTLService;
+import org.elasticsearch.plugins.MapperPlugin;
 
+import java.util.Collections;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -73,17 +76,16 @@ import java.util.Map;
  */
 public class IndicesModule extends AbstractModule {
 
-    private final Map<String, Mapper.TypeParser> mapperParsers
-        = new LinkedHashMap<>();
-    // Use a LinkedHashMap for metadataMappers because iteration order matters
-    private final Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers
-        = new LinkedHashMap<>();
+    private final Map<String, Mapper.TypeParser> mapperParsers;
+    private final Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers;
+    private final MapperRegistry mapperRegistry;
     private final NamedWriteableRegistry namedWritableRegistry;
 
-    public IndicesModule(NamedWriteableRegistry namedWriteableRegistry) {
+    public IndicesModule(NamedWriteableRegistry namedWriteableRegistry, List<MapperPlugin> mapperPlugins) {
         this.namedWritableRegistry = namedWriteableRegistry;
-        registerBuiltInMappers();
-        registerBuiltInMetadataMappers();
+        this.mapperParsers = getMappers(mapperPlugins);
+        this.metadataMapperParsers = getMetadataMappers(mapperPlugins);
+        this.mapperRegistry = new MapperRegistry(mapperParsers, metadataMapperParsers);
         registerBuildInWritables();
     }
 
@@ -92,65 +94,63 @@ public class IndicesModule extends AbstractModule {
         namedWritableRegistry.register(Condition.class, MaxDocsCondition.NAME, MaxDocsCondition::new);
     }
 
-    private void registerBuiltInMappers() {
+    private Map<String, Mapper.TypeParser> getMappers(List<MapperPlugin> mapperPlugins) {
+        Map<String, Mapper.TypeParser> mappers = new LinkedHashMap<>();
+
+        // builtin mappers
         for (NumberFieldMapper.NumberType type : NumberFieldMapper.NumberType.values()) {
-            registerMapper(type.typeName(), new NumberFieldMapper.TypeParser(type));
+            mappers.put(type.typeName(), new NumberFieldMapper.TypeParser(type));
         }
-        registerMapper(BooleanFieldMapper.CONTENT_TYPE, new BooleanFieldMapper.TypeParser());
-        registerMapper(BinaryFieldMapper.CONTENT_TYPE, new BinaryFieldMapper.TypeParser());
-        registerMapper(DateFieldMapper.CONTENT_TYPE, new DateFieldMapper.TypeParser());
-        registerMapper(IpFieldMapper.CONTENT_TYPE, new IpFieldMapper.TypeParser());
-        registerMapper(StringFieldMapper.CONTENT_TYPE, new StringFieldMapper.TypeParser());
-        registerMapper(TextFieldMapper.CONTENT_TYPE, new TextFieldMapper.TypeParser());
-        registerMapper(KeywordFieldMapper.CONTENT_TYPE, new KeywordFieldMapper.TypeParser());
-        registerMapper(TokenCountFieldMapper.CONTENT_TYPE, new TokenCountFieldMapper.TypeParser());
-        registerMapper(ObjectMapper.CONTENT_TYPE, new ObjectMapper.TypeParser());
-        registerMapper(ObjectMapper.NESTED_CONTENT_TYPE, new ObjectMapper.TypeParser());
-        registerMapper(CompletionFieldMapper.CONTENT_TYPE, new CompletionFieldMapper.TypeParser());
-        registerMapper(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser());
-
+        mappers.put(BooleanFieldMapper.CONTENT_TYPE, new BooleanFieldMapper.TypeParser());
+        mappers.put(BinaryFieldMapper.CONTENT_TYPE, new BinaryFieldMapper.TypeParser());
+        mappers.put(DateFieldMapper.CONTENT_TYPE, new DateFieldMapper.TypeParser());
+        mappers.put(IpFieldMapper.CONTENT_TYPE, new IpFieldMapper.TypeParser());
+        mappers.put(StringFieldMapper.CONTENT_TYPE, new StringFieldMapper.TypeParser());
+        mappers.put(TextFieldMapper.CONTENT_TYPE, new TextFieldMapper.TypeParser());
+        mappers.put(KeywordFieldMapper.CONTENT_TYPE, new KeywordFieldMapper.TypeParser());
+        mappers.put(TokenCountFieldMapper.CONTENT_TYPE, new TokenCountFieldMapper.TypeParser());
+        mappers.put(ObjectMapper.CONTENT_TYPE, new ObjectMapper.TypeParser());
+        mappers.put(ObjectMapper.NESTED_CONTENT_TYPE, new ObjectMapper.TypeParser());
+        mappers.put(CompletionFieldMapper.CONTENT_TYPE, new CompletionFieldMapper.TypeParser());
+        mappers.put(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser());
         if (ShapesAvailability.JTS_AVAILABLE && ShapesAvailability.SPATIAL4J_AVAILABLE) {
-            registerMapper(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser());
+            mappers.put(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser());
         }
+
+        for (MapperPlugin mapperPlugin : mapperPlugins) {
+            mappers.putAll(mapperPlugin.getMappers());
+        }
+        return Collections.unmodifiableMap(mappers);
     }
 
-    private void registerBuiltInMetadataMappers() {
-        // NOTE: the order is important
+    private Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers(List<MapperPlugin> mapperPlugins) {
+        // Use a LinkedHashMap for metadataMappers because iteration order matters
+        Map<String, MetadataFieldMapper.TypeParser> metadataMappers = new LinkedHashMap<>();
 
+        // builtin metadata mappers
         // UID first so it will be the first stored field to load (so will benefit from "fields: []" early termination
-        registerMetadataMapper(UidFieldMapper.NAME, new UidFieldMapper.TypeParser());
-        registerMetadataMapper(IdFieldMapper.NAME, new IdFieldMapper.TypeParser());
-        registerMetadataMapper(RoutingFieldMapper.NAME, new RoutingFieldMapper.TypeParser());
-        registerMetadataMapper(IndexFieldMapper.NAME, new IndexFieldMapper.TypeParser());
-        registerMetadataMapper(SourceFieldMapper.NAME, new SourceFieldMapper.TypeParser());
-        registerMetadataMapper(TypeFieldMapper.NAME, new TypeFieldMapper.TypeParser());
-        registerMetadataMapper(AllFieldMapper.NAME, new AllFieldMapper.TypeParser());
-        registerMetadataMapper(TimestampFieldMapper.NAME, new TimestampFieldMapper.TypeParser());
-        registerMetadataMapper(TTLFieldMapper.NAME, new TTLFieldMapper.TypeParser());
-        registerMetadataMapper(VersionFieldMapper.NAME, new VersionFieldMapper.TypeParser());
-        registerMetadataMapper(ParentFieldMapper.NAME, new ParentFieldMapper.TypeParser());
-        // _field_names is not registered here, see #getMapperRegistry: we need to register it
-        // last so that it can see all other mappers, including those coming from plugins
-    }
+        metadataMappers.put(UidFieldMapper.NAME, new UidFieldMapper.TypeParser());
+        metadataMappers.put(IdFieldMapper.NAME, new IdFieldMapper.TypeParser());
+        metadataMappers.put(RoutingFieldMapper.NAME, new RoutingFieldMapper.TypeParser());
+        metadataMappers.put(IndexFieldMapper.NAME, new IndexFieldMapper.TypeParser());
+        metadataMappers.put(SourceFieldMapper.NAME, new SourceFieldMapper.TypeParser());
+        metadataMappers.put(TypeFieldMapper.NAME, new TypeFieldMapper.TypeParser());
+        metadataMappers.put(AllFieldMapper.NAME, new AllFieldMapper.TypeParser());
+        metadataMappers.put(TimestampFieldMapper.NAME, new TimestampFieldMapper.TypeParser());
+        metadataMappers.put(TTLFieldMapper.NAME, new TTLFieldMapper.TypeParser());
+        metadataMappers.put(VersionFieldMapper.NAME, new VersionFieldMapper.TypeParser());
+        metadataMappers.put(ParentFieldMapper.NAME, new ParentFieldMapper.TypeParser());
+        // _field_names is not registered here, see below
 
-    /**
-     * Register a mapper for the given type.
-     */
-    public synchronized void registerMapper(String type, Mapper.TypeParser parser) {
-        if (mapperParsers.containsKey(type)) {
-            throw new IllegalArgumentException("A mapper is already registered for type [" + type + "]");
+        for (MapperPlugin mapperPlugin : mapperPlugins) {
+            metadataMappers.putAll(mapperPlugin.getMetadataMappers());
         }
-        mapperParsers.put(type, parser);
-    }
-
-    /**
-     * Register a root mapper under the given name.
-     */
-    public synchronized void registerMetadataMapper(String name, MetadataFieldMapper.TypeParser parser) {
-        if (metadataMapperParsers.containsKey(name)) {
-            throw new IllegalArgumentException("A mapper is already registered for metadata mapper [" + name + "]");
+        if (metadataMappers.containsKey(FieldNamesFieldMapper.NAME)) {
+            throw new IllegalStateException("Plugin cannot contain metadata mapper [" + FieldNamesFieldMapper.NAME + "]");
         }
-        metadataMapperParsers.put(name, parser);
+        // we register _field_names here so that it has a chance to see all other mappers, including from plugins
+        metadataMappers.put(FieldNamesFieldMapper.NAME, new FieldNamesFieldMapper.TypeParser());
+        return Collections.unmodifiableMap(metadataMappers);
     }
 
     @Override
@@ -172,16 +172,8 @@ public class IndicesModule extends AbstractModule {
     }
 
     // public for testing
-    public synchronized MapperRegistry getMapperRegistry() {
-        // NOTE: we register _field_names here so that it has a chance to see all other
-        // mappers, including from plugins
-        if (metadataMapperParsers.containsKey(FieldNamesFieldMapper.NAME)) {
-            throw new IllegalStateException("Metadata mapper [" + FieldNamesFieldMapper.NAME + "] is already registered");
-        }
-        final Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers
-            = new LinkedHashMap<>(this.metadataMapperParsers);
-        metadataMapperParsers.put(FieldNamesFieldMapper.NAME, new FieldNamesFieldMapper.TypeParser());
-        return new MapperRegistry(mapperParsers, metadataMapperParsers);
+    public MapperRegistry getMapperRegistry() {
+        return mapperRegistry;
     }
 
     protected void bindMapperExtension() {
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index de77f02ef51..cfd6cfe4df0 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -86,6 +86,7 @@ import org.elasticsearch.monitor.MonitorService;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
 import org.elasticsearch.node.service.NodeService;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.plugins.ScriptPlugin;
@@ -256,7 +257,7 @@ public class Node implements Closeable {
             modules.add(scriptModule);
             modules.add(new DiscoveryModule(this.settings));
             modules.add(new ClusterModule(this.settings, clusterService));
-            modules.add(new IndicesModule(namedWriteableRegistry));
+            modules.add(new IndicesModule(namedWriteableRegistry, pluginsService.filterPlugins(MapperPlugin.class)));
             modules.add(new SearchModule(settings, namedWriteableRegistry));
             modules.add(new ActionModule(DiscoveryNode.isIngestNode(settings), false));
             modules.add(new GatewayModule());
diff --git a/core/src/main/java/org/elasticsearch/plugins/MapperPlugin.java b/core/src/main/java/org/elasticsearch/plugins/MapperPlugin.java
new file mode 100644
index 00000000000..5dfcdc6bda4
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/plugins/MapperPlugin.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import java.util.Collections;
+import java.util.Map;
+
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.MetadataFieldMapper;
+
+/**
+ * An extension point for {@link Plugin} implementations to add custom mappers
+ */
+public interface MapperPlugin {
+
+    /**
+     * Returns additional mapper implementations added by this plugin.
+     *
+     * The key of the returned {@link Map} is the unique name for the mapper which will be used
+     * as the mapping {@code type}, and the value is a {@link Mapper.TypeParser} to parse the
+     * mapper settings into a {@link Mapper}.
+     */
+    default Map<String, Mapper.TypeParser> getMappers() {
+        return Collections.emptyMap();
+    }
+
+    /**
+     * Returns additional metadata mapper implementations added by this plugin.
+     *
+     * The key of the returned {@link Map} is the unique name for the metadata mapper, which
+     * is used in the mapping json to configure the metadata mapper, and the value is a
+     * {@link MetadataFieldMapper.TypeParser} to parse the mapper settings into a
+     * {@link MetadataFieldMapper}.
+     */
+    default Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers() {
+        return Collections.emptyMap();
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
index 67b6bdadb91..f5ccd350348 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
@@ -134,7 +134,7 @@ public class IndexModuleTests extends ESTestCase {
         environment = new Environment(settings);
         nodeServicesProvider = newNodeServiceProvider(settings, environment, null);
         nodeEnvironment = new NodeEnvironment(settings, environment);
-        mapperRegistry = new IndicesModule(new NamedWriteableRegistry()).getMapperRegistry();
+        mapperRegistry = new IndicesModule(new NamedWriteableRegistry(), Collections.emptyList()).getMapperRegistry();
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 45c8fbdc39a..a0396b7abc6 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -2023,7 +2023,7 @@ public class InternalEngineTests extends ESTestCase {
             IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings);
             AnalysisService analysisService = new AnalysisService(indexSettings, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
             SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
-            MapperRegistry mapperRegistry = new IndicesModule(new NamedWriteableRegistry()).getMapperRegistry();
+            MapperRegistry mapperRegistry = new IndicesModule(new NamedWriteableRegistry(), Collections.emptyList()).getMapperRegistry();
             MapperService mapperService = new MapperService(indexSettings, analysisService, similarityService, mapperRegistry, () -> null);
             DocumentMapper.Builder b = new DocumentMapper.Builder(rootBuilder, mapperService);
             this.docMapper = b.build(mapperService);
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java
index 115faf2c6a1..81b7375ab2e 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java
@@ -19,21 +19,35 @@
 
 package org.elasticsearch.index.mapper.externalvalues;
 
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
 
-public class ExternalMapperPlugin extends Plugin {
+public class ExternalMapperPlugin extends Plugin implements MapperPlugin {
 
     public static final String EXTERNAL = "external";
     public static final String EXTERNAL_BIS = "external_bis";
     public static final String EXTERNAL_UPPER = "external_upper";
 
-    public void onModule(IndicesModule indicesModule) {
-        indicesModule.registerMetadataMapper(ExternalMetadataMapper.CONTENT_TYPE, new ExternalMetadataMapper.TypeParser());
-        indicesModule.registerMapper(EXTERNAL, new ExternalMapper.TypeParser(EXTERNAL, "foo"));
-        indicesModule.registerMapper(EXTERNAL_BIS, new ExternalMapper.TypeParser(EXTERNAL_BIS, "bar"));
-        indicesModule.registerMapper(EXTERNAL_UPPER, new ExternalMapper.TypeParser(EXTERNAL_UPPER, "FOO BAR"));
-        indicesModule.registerMapper(FakeStringFieldMapper.CONTENT_TYPE, new FakeStringFieldMapper.TypeParser());
+    @Override
+    public Map<String, Mapper.TypeParser> getMappers() {
+        Map<String, Mapper.TypeParser> mappers = new HashMap<>();
+        mappers.put(EXTERNAL, new ExternalMapper.TypeParser(EXTERNAL, "foo"));
+        mappers.put(EXTERNAL_BIS, new ExternalMapper.TypeParser(EXTERNAL_BIS, "bar"));
+        mappers.put(EXTERNAL_UPPER, new ExternalMapper.TypeParser(EXTERNAL_UPPER, "FOO BAR"));
+        mappers.put(FakeStringFieldMapper.CONTENT_TYPE, new FakeStringFieldMapper.TypeParser());
+        return Collections.unmodifiableMap(mappers);
+    }
+
+    @Override
+    public Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers() {
+        return Collections.singletonMap(ExternalMetadataMapper.CONTENT_TYPE, new ExternalMetadataMapper.TypeParser());
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java
index 2ec5aee9d15..11804ab1d2c 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java
@@ -31,6 +31,7 @@ import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
@@ -39,10 +40,12 @@ import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.TermBasedFieldType;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.mapper.MapperRegistry;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.SortedSet;
@@ -235,8 +238,10 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase {
 
     public void testSeesFieldsFromPlugins() throws IOException {
         IndexService indexService = createIndex("test");
-        IndicesModule indicesModule = new IndicesModule(new NamedWriteableRegistry());
-        indicesModule.registerMetadataMapper("_dummy", new DummyMetadataFieldMapper.TypeParser());
+        IndicesModule indicesModule = newTestIndicesModule(
+            Collections.emptyMap(),
+            Collections.singletonMap("_dummy", new DummyMetadataFieldMapper.TypeParser())
+        );
         final MapperRegistry mapperRegistry = indicesModule.getMapperRegistry();
         MapperService mapperService = new MapperService(indexService.getIndexSettings(), indexService.analysisService(), indexService.similarityService(), mapperRegistry, indexService::newQueryShardContext);
         DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), mapperService,
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
index a78c405929a..6f81d79693d 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
@@ -113,8 +113,7 @@ public class AggregatorParsingTests extends ESTestCase {
                 b.bind(Environment.class).toInstance(new Environment(settings));
                 b.bind(ThreadPool.class).toInstance(threadPool);
             }, settingsModule
-            , scriptModule, new IndicesModule(namedWriteableRegistry) {
-
+            , scriptModule, new IndicesModule(namedWriteableRegistry, Collections.emptyList()) {
                     @Override
                     protected void configure() {
                         bindMapperExtension();
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
index 7bce599f55a..43701aed9fd 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
@@ -143,8 +143,7 @@ public abstract class BaseAggregationTestCase<AB extends AbstractAggregationBuil
             },
             settingsModule,
             scriptModule,
-            new IndicesModule(namedWriteableRegistry) {
-
+            new IndicesModule(namedWriteableRegistry, Collections.emptyList()) {
                 @Override
                 protected void configure() {
                     bindMapperExtension();
diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index 5822372bfa9..617e7e9eb44 100644
--- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -135,7 +135,7 @@ public class SearchSourceBuilderTests extends ESTestCase {
                     b.bind(Environment.class).toInstance(new Environment(settings));
                     b.bind(ThreadPool.class).toInstance(threadPool);
                 }, settingsModule,
-                scriptModule, new IndicesModule(namedWriteableRegistry) {
+                scriptModule, new IndicesModule(namedWriteableRegistry, Collections.emptyList()) {
                     @Override
                     protected void configure() {
                         bindMapperExtension();
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
index 60316818a79..14c937b82ea 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
@@ -135,21 +135,16 @@ public class TemplateQueryParserTests extends ESTestCase {
         AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
         ScriptService scriptService = injector.getInstance(ScriptService.class);
         SimilarityService similarityService = new SimilarityService(idxSettings, Collections.emptyMap());
-        MapperRegistry mapperRegistry = new IndicesModule(new NamedWriteableRegistry()).getMapperRegistry();
+        MapperRegistry mapperRegistry = new IndicesModule(new NamedWriteableRegistry(), Collections.emptyList()).getMapperRegistry();
         MapperService mapperService = new MapperService(idxSettings, analysisService, similarityService, mapperRegistry, () ->
             contextFactory.get());
         IndicesFieldDataCache cache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() {});
-        IndexFieldDataService indexFieldDataService =new IndexFieldDataService(idxSettings, cache, injector.getInstance(CircuitBreakerService.class), mapperService);
+        IndexFieldDataService indexFieldDataService = new IndexFieldDataService(idxSettings, cache, injector.getInstance(CircuitBreakerService.class), mapperService);
         BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, new BitsetFilterCache.Listener() {
             @Override
-            public void onCache(ShardId shardId, Accountable accountable) {
-
-            }
-
+            public void onCache(ShardId shardId, Accountable accountable) {}
             @Override
-            public void onRemoval(ShardId shardId, Accountable accountable) {
-
-            }
+            public void onRemoval(ShardId shardId, Accountable accountable) {}
         });
         IndicesQueriesRegistry indicesQueriesRegistry = injector.getInstance(IndicesQueriesRegistry.class);
         contextFactory =  () -> new QueryShardContext(idxSettings, bitsetFilterCache, indexFieldDataService, mapperService,
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
index 23251ca20d0..7cbf07e1981 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
@@ -19,20 +19,23 @@
 
 package org.elasticsearch.percolator;
 
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
 import org.elasticsearch.action.ActionModule;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchModule;
 
-import java.util.Arrays;
-import java.util.List;
-
-public class PercolatorPlugin extends Plugin {
+public class PercolatorPlugin extends Plugin implements MapperPlugin {
 
     public static final String NAME = "percolator";
 
@@ -56,10 +59,6 @@ public class PercolatorPlugin extends Plugin {
         }
     }
 
-    public void onModule(IndicesModule module) {
-        module.registerMapper(PercolatorFieldMapper.CONTENT_TYPE, new PercolatorFieldMapper.TypeParser());
-    }
-
     public void onModule(SearchModule module) {
         module.registerQuery(PercolateQueryBuilder::new, PercolateQueryBuilder::fromXContent, PercolateQueryBuilder.QUERY_NAME_FIELD);
         module.registerFetchSubPhase(new PercolatorHighlightSubFetchPhase(settings, module.getHighlighters()));
@@ -70,6 +69,11 @@ public class PercolatorPlugin extends Plugin {
         return Arrays.asList(PercolatorFieldMapper.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING);
     }
 
+    @Override
+    public Map<String, Mapper.TypeParser> getMappers() {
+        return Collections.singletonMap(PercolatorFieldMapper.CONTENT_TYPE, new PercolatorFieldMapper.TypeParser());
+    }
+
     static boolean transportClientMode(Settings settings) {
         return TransportClient.CLIENT_TYPE.equals(settings.get(Client.CLIENT_TYPE_SETTING_S.getKey()));
     }
diff --git a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java
index d523e0f0184..6cf957f05c0 100644
--- a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java
+++ b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java
@@ -19,18 +19,20 @@
 
 package org.elasticsearch.mapper.attachments;
 
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.SettingsModule;
-import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
 
-import java.util.Arrays;
-import java.util.List;
-
-public class MapperAttachmentsPlugin extends Plugin {
+public class MapperAttachmentsPlugin extends Plugin implements MapperPlugin {
 
     private static ESLogger logger = ESLoggerFactory.getLogger("mapper.attachment");
     private static DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
@@ -44,7 +46,8 @@ public class MapperAttachmentsPlugin extends Plugin {
         AttachmentMapper.INDEX_ATTACHMENT_INDEXED_CHARS_SETTING);
     }
 
-    public void onModule(IndicesModule indicesModule) {
-        indicesModule.registerMapper("attachment", new AttachmentMapper.TypeParser());
+    @Override
+    public Map<String, Mapper.TypeParser> getMappers() {
+        return Collections.singletonMap("attachment", new AttachmentMapper.TypeParser());
     }
 }
diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/AttachmentUnitTestCase.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/AttachmentUnitTestCase.java
index e0d5d7a2ec6..29dfff66d96 100644
--- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/AttachmentUnitTestCase.java
+++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/AttachmentUnitTestCase.java
@@ -19,12 +19,18 @@
 
 package org.elasticsearch.mapper.attachments;
 
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
+
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
@@ -33,9 +39,10 @@ public abstract class AttachmentUnitTestCase extends ESTestCase {
     protected Settings testSettings;
 
     protected static IndicesModule getIndicesModuleWithRegisteredAttachmentMapper() {
-        IndicesModule indicesModule = new IndicesModule(new NamedWriteableRegistry());
-        indicesModule.registerMapper(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser());
-        return indicesModule;
+        return newTestIndicesModule(
+            Collections.singletonMap(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()),
+            Collections.emptyMap()
+        );
     }
 
     @Before
diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java
index 384fc4272f6..987a4cf9bc0 100644
--- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java
+++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java
@@ -19,14 +19,18 @@
 
 package org.elasticsearch.plugin.mapper;
 
+import java.util.Collections;
+import java.util.Map;
+
+import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.murmur3.Murmur3FieldMapper;
-import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
 
-public class MapperMurmur3Plugin extends Plugin {
+public class MapperMurmur3Plugin extends Plugin implements MapperPlugin {
 
-    public void onModule(IndicesModule indicesModule) {
-        indicesModule.registerMapper(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser());
+    @Override
+    public Map<String, Mapper.TypeParser> getMappers() {
+        return Collections.singletonMap(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser());
     }
-
 }
diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java b/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java
index 1aecfdf444d..1df48d5695a 100644
--- a/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java
+++ b/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java
@@ -19,13 +19,18 @@
 
 package org.elasticsearch.plugin.mapper;
 
+import java.util.Collections;
+import java.util.Map;
+
+import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.size.SizeFieldMapper;
-import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
 
-public class MapperSizePlugin extends Plugin {
+public class MapperSizePlugin extends Plugin implements MapperPlugin {
 
-    public void onModule(IndicesModule indicesModule) {
-        indicesModule.registerMetadataMapper(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser());
+    @Override
+    public Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers() {
+        return Collections.singletonMap(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser());
     }
 }
diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java
index ce44fef59a5..b0802a955df 100644
--- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java
+++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java
@@ -19,6 +19,9 @@
 
 package org.elasticsearch.index.mapper.size;
 
+import java.util.Collections;
+import java.util.Map;
+
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -26,10 +29,12 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperParser;
+import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.junit.Before;
 
@@ -48,8 +53,9 @@ public class SizeMappingTests extends ESSingleNodeTestCase {
     @Before
     public void before() {
         indexService = createIndex("test");
-        IndicesModule indices = new IndicesModule(new NamedWriteableRegistry());
-        indices.registerMetadataMapper(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser());
+        IndicesModule indices = newTestIndicesModule(Collections.emptyMap(),
+            Collections.singletonMap(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser())
+        );
         mapperService = new MapperService(indexService.getIndexSettings(), indexService.analysisService(), indexService.similarityService(), indices.getMapperRegistry(), indexService::newQueryShardContext);
         parser = mapperService.documentMapperParser();
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
index fe434246035..55659164d76 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
@@ -40,7 +40,7 @@ import java.util.Collections;
 public class MapperTestUtils {
 
     public static MapperService newMapperService(Path tempDir, Settings indexSettings) throws IOException {
-        IndicesModule indicesModule = new IndicesModule(new NamedWriteableRegistry());
+        IndicesModule indicesModule = new IndicesModule(new NamedWriteableRegistry(), Collections.emptyList());
         return newMapperService(tempDir, indexSettings, indicesModule);
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
index ea2154b355e..dc57352f26b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
@@ -894,7 +894,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
                         b.bind(Environment.class).toInstance(new Environment(settings));
                         b.bind(ThreadPool.class).toInstance(threadPool);
                     },
-                    settingsModule, new IndicesModule(namedWriteableRegistry) {
+                    settingsModule, new IndicesModule(namedWriteableRegistry, Collections.emptyList()) {
                         @Override
                         public void configure() {
                             // skip services
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index e7e6eca5c88..2055680bd4c 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -41,6 +41,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.io.PathUtilsForTesting;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
@@ -55,8 +56,12 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.analysis.AnalysisService;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.MetadataFieldMapper;
+import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.analysis.AnalysisModule;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.ScriptContextRegistry;
 import org.elasticsearch.script.ScriptEngineRegistry;
@@ -817,4 +822,21 @@ public abstract class ESTestCase extends LuceneTestCase {
             }
         };
     }
+
+    /** Creates an IndicesModule for testing with the given mappers and metadata mappers. */
+    public static IndicesModule newTestIndicesModule(Map<String, Mapper.TypeParser> extraMappers,
+                                                     Map<String, MetadataFieldMapper.TypeParser> extraMetadataMappers) {
+        return new IndicesModule(new NamedWriteableRegistry(), Collections.singletonList(
+            new MapperPlugin() {
+                @Override
+                public Map<String, Mapper.TypeParser> getMappers() {
+                    return extraMappers;
+                }
+                @Override
+                public Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers() {
+                    return extraMetadataMappers;
+                }
+            }
+        ));
+    }
 }

From 5dc88ffd2664869fa3ef84d37623865599cbc506 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Wed, 22 Jun 2016 08:29:50 +0200
Subject: [PATCH 1020/1311] docs: added note the inner hits migrate section

---
 docs/reference/migration/migrate_5_0/search.asciidoc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/reference/migration/migrate_5_0/search.asciidoc b/docs/reference/migration/migrate_5_0/search.asciidoc
index dd91089fcad..72b29e8a9e7 100644
--- a/docs/reference/migration/migrate_5_0/search.asciidoc
+++ b/docs/reference/migration/migrate_5_0/search.asciidoc
@@ -183,6 +183,9 @@ This is now consistent for source filtering on other places in the search API.
 * Nested inner hits will now no longer include `_index`, `_type` and `_id` keys. For nested inner hits these values
 are always the same as the `_index`, `_type` and `_id` keys of the root search hit.
 
+* Parent/child inner hits will now no longer include the `_index` key. For parent/child inner hits the `_index` key is
+always the same as the the parent search hit.
+
 ==== Query Profiler
 
 In the response for profiling queries, the `query_type` has been renamed to `type` and `lucene` has been renamed to

From db9af54ec0a7c8b3ffad036ab1acea819999ae9e Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Mon, 20 Jun 2016 18:06:31 +0200
Subject: [PATCH 1021/1311] Remove `_timestamp` and `_ttl` on 5.x indices.
 #18980

This removes the ability to use `_timestamp` and `_ttl` on indices created on
or after 5.0.

Closes #18280
---
 .../index/mapper/internal/TTLFieldMapper.java |   7 +
 .../mapper/internal/TimestampFieldMapper.java |   7 +
 .../elasticsearch/aliases/IndexAliasesIT.java |   8 +-
 .../cluster/SpecificMasterNodesIT.java        |  11 +-
 .../org/elasticsearch/get/GetActionIT.java    |  39 +--
 .../mapper/all/SimpleAllMapperTests.java      |  20 --
 .../timestamp/TimestampMappingTests.java      | 104 ++++++--
 .../index/mapper/ttl/TTLMappingTests.java     |  54 +++-
 .../ParseMappingTypeLevelTests.java           |   4 +-
 .../update/UpdateMappingOnClusterIT.java      |  10 -
 .../mapper/update/UpdateMappingTests.java     |  50 ----
 .../aggregations/bucket/DateHistogramIT.java  |  11 +-
 .../search/sort/FieldSortIT.java              |  42 +---
 .../timestamp/SimpleTimestampIT.java          |  19 +-
 .../org/elasticsearch/ttl/SimpleTTLIT.java    |  15 +-
 .../update/TimestampTTLBWIT.java              | 237 ++++++++++++++++++
 .../org/elasticsearch/update/UpdateIT.java    |  59 +----
 docs/reference/docs/bulk.asciidoc             |  20 --
 docs/reference/docs/index_.asciidoc           |  64 -----
 docs/reference/docs/reindex.asciidoc          |   2 -
 docs/reference/docs/update.asciidoc           |   2 +-
 docs/reference/index-modules.asciidoc         |   5 -
 docs/reference/ingest/ingest-node.asciidoc    |   3 +-
 docs/reference/mapping/fields.asciidoc        |   8 -
 .../mapping/fields/timestamp-field.asciidoc   |  97 -------
 .../mapping/fields/ttl-field.asciidoc         | 112 ---------
 .../migration/migrate_5_0/mapping.asciidoc    |   8 +
 docs/reference/modules/indices.asciidoc       |   4 -
 .../modules/indices/ttl_interval.asciidoc     |  16 --
 .../messy/tests/SearchFieldsTests.java        |  12 +-
 .../rest-api-spec/test/README.asciidoc        |   4 +-
 .../test/create/70_timestamp.yaml             |  80 ------
 .../rest-api-spec/test/create/75_ttl.yaml     | 100 --------
 .../test/index/70_timestamp.yaml              |  70 ------
 .../rest-api-spec/test/index/75_ttl.yaml      |  85 -------
 .../test/update/70_timestamp.yaml             |  76 ------
 .../rest-api-spec/test/update/75_ttl.yaml     |  92 -------
 .../test/update/85_fields_meta.yaml           |  11 +-
 .../elasticsearch/test/ESIntegTestCase.java   |   8 +-
 39 files changed, 450 insertions(+), 1126 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java
 delete mode 100644 docs/reference/mapping/fields/timestamp-field.asciidoc
 delete mode 100644 docs/reference/mapping/fields/ttl-field.asciidoc
 delete mode 100644 docs/reference/modules/indices/ttl_interval.asciidoc
 delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/create/70_timestamp.yaml
 delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/create/75_ttl.yaml
 delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/index/70_timestamp.yaml
 delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/index/75_ttl.yaml
 delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/update/70_timestamp.yaml
 delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/update/75_ttl.yaml

diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java
index 33d194a7892..ec1024c07f5 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper.internal;
 
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -100,6 +101,9 @@ public class TTLFieldMapper extends MetadataFieldMapper {
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
         public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+            if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0_alpha4)) {
+                throw new IllegalArgumentException("[_ttl] is removed in 5.0. As a replacement, you should use time based indexes or cron a delete-by-query with a range query on a timestamp field.");
+            }
             Builder builder = new Builder();
             for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
                 Map.Entry<String, Object> entry = iterator.next();
@@ -165,6 +169,9 @@ public class TTLFieldMapper extends MetadataFieldMapper {
     private TTLFieldMapper(MappedFieldType fieldType, EnabledAttributeMapper enabled, long defaultTTL,
                              Settings indexSettings) {
         super(NAME, fieldType, Defaults.TTL_FIELD_TYPE, indexSettings);
+        if (enabled.enabled && Version.indexCreated(indexSettings).onOrAfter(Version.V_5_0_0_alpha4)) {
+            throw new IllegalArgumentException("[_ttl] is removed in 5.0. As a replacement, you should use time based indexes or cron a delete-by-query with a range query on a timestamp field.");
+        }
         this.enabledState = enabled;
         this.defaultTTL = defaultTTL;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
index 09f1144db28..24a86b11392 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
@@ -22,6 +22,7 @@ package org.elasticsearch.index.mapper.internal;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.index.IndexOptions;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.TimestampParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
@@ -126,6 +127,9 @@ public class TimestampFieldMapper extends MetadataFieldMapper {
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
         public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+            if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0_alpha4)) {
+                throw new IllegalArgumentException("[_timestamp] is removed in 5.0. As a replacement, you can use an ingest pipeline to add a field with the current timestamp to your documents.");
+            }
             Builder builder = new Builder(parserContext.mapperService().fullName(NAME), parserContext.mapperService().getIndexSettings().getSettings());
             boolean defaultSet = false;
             Boolean ignoreMissing = null;
@@ -200,6 +204,9 @@ public class TimestampFieldMapper extends MetadataFieldMapper {
     private TimestampFieldMapper(MappedFieldType fieldType, MappedFieldType defaultFieldType, EnabledAttributeMapper enabledState,
                                    String defaultTimestamp, Boolean ignoreMissing, Settings indexSettings) {
         super(NAME, fieldType, defaultFieldType, indexSettings);
+        if (enabledState.enabled && Version.indexCreated(indexSettings).onOrAfter(Version.V_5_0_0_alpha4)) {
+            throw new IllegalArgumentException("[_timestamp] is removed in 5.0. As a replacement, you can use an ingest pipeline to add a field with the current timestamp to your documents.");
+        }
         this.enabledState = enabledState;
         this.defaultTimestamp = defaultTimestamp;
         this.ignoreMissing = ignoreMissing;
diff --git a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
index 2a16625d037..5ac1bf40af6 100644
--- a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
+++ b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
@@ -1035,13 +1035,13 @@ public class IndexAliasesIT extends ESIntegTestCase {
     }
 
     public void testAliasFilterWithNowInRangeFilterAndQuery() throws Exception {
-        assertAcked(prepareCreate("my-index").addMapping("my-type", "_timestamp", "enabled=true"));
-        assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter1", rangeQuery("_timestamp").from("now-1d").to("now")));
-        assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter2", rangeQuery("_timestamp").from("now-1d").to("now")));
+        assertAcked(prepareCreate("my-index").addMapping("my-type", "timestamp", "type=date"));
+        assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter1", rangeQuery("timestamp").from("2016-12-01").to("2016-12-31")));
+        assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter2", rangeQuery("timestamp").from("2016-01-01").to("2016-12-31")));
 
         final int numDocs = scaledRandomIntBetween(5, 52);
         for (int i = 1; i <= numDocs; i++) {
-            client().prepareIndex("my-index", "my-type").setCreate(true).setSource("{}").get();
+            client().prepareIndex("my-index", "my-type").setCreate(true).setSource("timestamp", "2016-12-12").get();
             if (i % 2 == 0) {
                 refresh();
                 SearchResponse response = client().prepareSearch("filter1").get();
diff --git a/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java b/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
index 72db338ab80..4773aafbf3f 100644
--- a/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
@@ -30,6 +30,7 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 
 import java.io.IOException;
+import java.util.Map;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
@@ -111,16 +112,18 @@ public class SpecificMasterNodesIT extends ESIntegTestCase {
         internalCluster().startNode(settingsBuilder().put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false));
 
         createIndex("test");
-        assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("_timestamp", "enabled=true"));
+        assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("timestamp", "type=date"));
 
         MappingMetaData defaultMapping = client().admin().cluster().prepareState().get().getState().getMetaData().getIndices().get("test").getMappings().get("_default_");
-        assertThat(defaultMapping.getSourceAsMap().get("_timestamp"), notNullValue());
+        Map<?,?> properties = (Map<?, ?>) defaultMapping.getSourceAsMap().get("properties");
+        assertThat(properties.get("timestamp"), notNullValue());
 
-        assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("_timestamp", "enabled=true"));
+        assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("timestamp", "type=date"));
 
         assertAcked(client().admin().indices().preparePutMapping("test").setType("type1").setSource("foo", "enabled=true"));
         MappingMetaData type1Mapping = client().admin().cluster().prepareState().get().getState().getMetaData().getIndices().get("test").getMappings().get("type1");
-        assertThat(type1Mapping.getSourceAsMap().get("_timestamp"), notNullValue());
+        properties = (Map<?, ?>) type1Mapping.getSourceAsMap().get("properties");
+        assertThat(properties.get("timestamp"), notNullValue());
     }
 
     public void testAliasFilterValidation() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/get/GetActionIT.java b/core/src/test/java/org/elasticsearch/get/GetActionIT.java
index 8c4b699cbff..b73f563fbb3 100644
--- a/core/src/test/java/org/elasticsearch/get/GetActionIT.java
+++ b/core/src/test/java/org/elasticsearch/get/GetActionIT.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.get;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.flush.FlushResponse;
@@ -30,7 +29,6 @@ import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.get.MultiGetRequest;
 import org.elasticsearch.action.get.MultiGetRequestBuilder;
 import org.elasticsearch.action.get.MultiGetResponse;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -40,15 +38,11 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
-import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.InternalSettingsPlugin;
 
 import java.io.IOException;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
-import java.util.Map;
 import java.util.Set;
 
 import static java.util.Collections.singleton;
@@ -57,7 +51,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
@@ -536,7 +529,7 @@ public class GetActionIT extends ESIntegTestCase {
     public void testGetFieldsMetaData() throws Exception {
         assertAcked(prepareCreate("test")
                 .addMapping("parent")
-                .addMapping("my-type1", "_timestamp", "enabled=true", "_ttl", "enabled=true", "_parent", "type=parent")
+                .addMapping("my-type1", "_parent", "type=parent")
                 .addAlias(new Alias("alias"))
                 .setSettings(Settings.builder().put("index.refresh_interval", -1)));
 
@@ -557,12 +550,6 @@ public class GetActionIT extends ESIntegTestCase {
         assertThat(getResponse.getField("field1").getValue().toString(), equalTo("value"));
         assertThat(getResponse.getField("_routing").isMetadataField(), equalTo(true));
         assertThat(getResponse.getField("_routing").getValue().toString(), equalTo("1"));
-        assertThat(getResponse.getField("_timestamp").isMetadataField(), equalTo(true));
-        assertThat(getResponse.getField("_timestamp").getValue().toString(), equalTo("205097"));
-        assertThat(getResponse.getField("_ttl").isMetadataField(), equalTo(true));
-        // TODO: _ttl should return the original value, but it does not work today because
-        // it would use now() instead of the value of _timestamp to rebase
-        // assertThat(getResponse.getField("_ttl").getValue().toString(), equalTo("10000000205097"));
         assertThat(getResponse.getField("_parent").isMetadataField(), equalTo(true));
         assertThat(getResponse.getField("_parent").getValue().toString(), equalTo("parent_1"));
 
@@ -577,12 +564,6 @@ public class GetActionIT extends ESIntegTestCase {
         assertThat(getResponse.getField("field1").getValue().toString(), equalTo("value"));
         assertThat(getResponse.getField("_routing").isMetadataField(), equalTo(true));
         assertThat(getResponse.getField("_routing").getValue().toString(), equalTo("1"));
-        assertThat(getResponse.getField("_timestamp").isMetadataField(), equalTo(true));
-        assertThat(getResponse.getField("_timestamp").getValue().toString(), equalTo("205097"));
-        assertThat(getResponse.getField("_ttl").isMetadataField(), equalTo(true));
-        // TODO: _ttl should return the original value, but it does not work today because
-        // it would use now() instead of the value of _timestamp to rebase
-        //assertThat(getResponse.getField("_ttl").getValue().toString(), equalTo("10000000000000"));
         assertThat(getResponse.getField("_parent").isMetadataField(), equalTo(true));
         assertThat(getResponse.getField("_parent").getValue().toString(), equalTo("parent_1"));
     }
@@ -779,16 +760,10 @@ public class GetActionIT extends ESIntegTestCase {
                 "  },\n" +
                 "  \"mappings\": {\n" +
                 "    \"parentdoc\": {\n" +
-                "      \"_ttl\": {\n" +
-                "        \"enabled\": true\n" +
-                "      }\n" +
                 "    },\n" +
                 "    \"doc\": {\n" +
                 "      \"_parent\": {\n" +
                 "        \"type\": \"parentdoc\"\n" +
-                "      },\n" +
-                "      \"_ttl\": {\n" +
-                "        \"enabled\": true\n" +
                 "      }\n" +
                 "    }\n" +
                 "  }\n" +
@@ -798,7 +773,7 @@ public class GetActionIT extends ESIntegTestCase {
 
         client().prepareIndex("test", "doc").setId("1").setSource("{}").setParent("1").setTTL(TimeValue.timeValueHours(1).getMillis()).get();
 
-        String[] fieldsList = {"_ttl", "_parent"};
+        String[] fieldsList = {"_parent"};
         // before refresh - document is only in translog
         assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "1");
         refresh();
@@ -814,14 +789,6 @@ public class GetActionIT extends ESIntegTestCase {
             "  \"settings\": {\n" +
             "    \"index.translog.flush_threshold_size\": \"1pb\",\n" +
             "    \"refresh_interval\": \"-1\"\n" +
-            "  },\n" +
-            "  \"mappings\": {\n" +
-            "    \"parentdoc\": {},\n" +
-            "    \"doc\": {\n" +
-            "      \"_timestamp\": {\n" +
-            "        \"enabled\": true\n" +
-            "      }\n" +
-            "    }\n" +
             "  }\n" +
             "}";
 
@@ -831,7 +798,7 @@ public class GetActionIT extends ESIntegTestCase {
             "  \"text\": \"some text.\"\n" +
             "}\n";
         client().prepareIndex("test", "doc").setId("1").setSource(doc).setRouting("1").get();
-        String[] fieldsList = {"_timestamp", "_routing"};
+        String[] fieldsList = {"_routing"};
         // before refresh - document is only in translog
         assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "1");
         refresh();
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java
index 966146655b8..165b49d3145 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java
@@ -40,7 +40,6 @@ import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext.Document;
 import org.elasticsearch.index.mapper.internal.AllFieldMapper;
-import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
@@ -50,9 +49,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath;
@@ -430,23 +427,6 @@ public class SimpleAllMapperTests extends ESSingleNodeTestCase {
         parser.parse("test", new CompressedXContent(mapping));
     }
 
-    // issue https://github.com/elastic/elasticsearch/issues/5864
-    public void testMetadataMappersStillWorking() throws MapperParsingException, IOException {
-        String mapping = "{";
-        Map<String, String> rootTypes = new HashMap<>();
-        //just pick some example from DocumentMapperParser.rootTypeParsers
-        rootTypes.put(TimestampFieldMapper.NAME, "{\"enabled\" : true}");
-        rootTypes.put("include_in_all", "true");
-        rootTypes.put("dynamic_date_formats", "[\"yyyy-MM-dd\", \"dd-MM-yyyy\"]");
-        rootTypes.put("numeric_detection", "true");
-        rootTypes.put("dynamic_templates", "[]");
-        for (String key : rootTypes.keySet()) {
-            mapping += "\"" + key+ "\"" + ":" + rootTypes.get(key) + ",\n";
-        }
-        mapping += "\"properties\":{}}" ;
-        createIndex("test").mapperService().documentMapperParser().parse("test", new CompressedXContent(mapping));
-    }
-
     public void testDocValuesNotAllowed() throws IOException {
         String mapping = jsonBuilder().startObject().startObject("type")
             .startObject("_all")
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
index 14d86290db4..78da5abb746 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
@@ -22,6 +22,7 @@ package org.elasticsearch.index.mapper.timestamp;
 import org.apache.lucene.index.IndexOptions;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.TimestampParsingException;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
@@ -31,25 +32,31 @@ import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.joda.Joda;
+import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperParser;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceToParse;
+import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
+import org.elasticsearch.test.VersionUtils;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.LinkedHashMap;
 
+import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
 import static org.elasticsearch.test.VersionUtils.randomVersion;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.containsString;
@@ -58,19 +65,35 @@ import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.startsWith;
 
 /**
  */
 public class TimestampMappingTests extends ESSingleNodeTestCase {
 
+    private static final Settings BW_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0).build();
+
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(InternalSettingsPlugin.class);
     }
 
+    public void testRejectedOn5x() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject()
+                .startObject("type")
+                    .startObject("_timestamp")
+                        .field("enabled", true)
+                    .endObject()
+                .endObject().endObject().string();
+        IndexService index = createIndex("test");
+        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class,
+                () -> index.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, false));
+        assertThat(expected.getMessage(), startsWith("[_timestamp] is removed"));
+    }
+
     public void testSimpleDisabled() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         BytesReference source = XContentFactory.jsonBuilder()
                 .startObject()
                 .field("field", "value")
@@ -85,7 +108,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_timestamp").field("enabled", "yes").endObject()
                 .endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         BytesReference source = XContentFactory.jsonBuilder()
                 .startObject()
                 .field("field", "value")
@@ -99,10 +122,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
     }
 
     public void testDefaultValues() throws Exception {
-        Version version;
-        do {
-            version = randomVersion(random());
-        } while (version.before(Version.V_2_0_0_beta1));
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0_beta1, Version.V_5_0_0_alpha3);
         for (String mapping : Arrays.asList(
                 XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(),
                 XContentFactory.jsonBuilder().startObject().startObject("type").startObject("_timestamp").endObject().endObject().endObject().string())) {
@@ -120,7 +140,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
         String enabledMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_timestamp").field("enabled", true).endObject()
                 .endObject().endObject().string();
-        MapperService mapperService = createIndex("test").mapperService();
+        MapperService mapperService = createIndex("test", BW_SETTINGS).mapperService();
         DocumentMapper enabledMapper = mapperService.merge("type", new CompressedXContent(enabledMapping), MapperService.MergeReason.MAPPING_UPDATE, false);
 
         String disabledMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
@@ -146,7 +166,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                     .field("foo", "bar")
                 .endObject();
 
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
+        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
         MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData();
 
         MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
@@ -172,7 +192,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                 .endObject();
 
         MetaData metaData = MetaData.builder().build();
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
+        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
 
         MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
 
@@ -195,7 +215,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                 .endObject()
                 .endObject().endObject();
         try {
-            createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
+            createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
             fail("we should reject the mapping with a TimestampParsingException: default timestamp can not be set to null");
         } catch (TimestampParsingException e) {
             assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set to null"));
@@ -212,7 +232,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                 .endObject().endObject();
 
         try {
-            createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
+            createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
             fail("we should reject the mapping with a TimestampParsingException: default timestamp can not be set to null");
         } catch (TimestampParsingException e) {
             assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set to null"));
@@ -230,7 +250,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                 .endObject().endObject();
 
         try {
-            createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
+            createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
             fail("we should reject the mapping with a TimestampParsingException: default timestamp can not be set with ignore_missing set to false");
         } catch (TimestampParsingException e) {
             assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set with ignore_missing set to false"));
@@ -250,7 +270,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                 .endObject();
 
         MetaData metaData = MetaData.builder().build();
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
+        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string()));
 
         MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
 
@@ -323,7 +343,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                     .field("enabled", true)
                     .field("default", "1970-01-01")
                 .endObject().endObject().endObject().string();
-        DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
+        DocumentMapperParser parser = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser();
 
         DocumentMapper docMapper = parser.parse("type", new CompressedXContent(mapping));
         docMapper = parser.parse("type", docMapper.mappingSource());
@@ -360,7 +380,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("_timestamp").field("enabled", true).field("default", "1970").field("format", "YYYY").endObject()
             .endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
         try {
             docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
@@ -375,7 +395,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_timestamp").field("enabled", true).field("format", "yyyyMMddHH").endObject()
             .endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData();
 
         XContentBuilder doc = XContentFactory.jsonBuilder().startObject().endObject();
@@ -392,7 +412,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
                 .endObject().endObject().string();
         BytesReference source = XContentFactory.jsonBuilder().startObject().field("field", "value").endObject().bytes();
         // test with 2.x
-        DocumentMapper currentMapper = createIndex("new-index").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper currentMapper = createIndex("new-index", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
         // this works with 2.x
         IndexRequest request = new IndexRequest("new-index", "type", "1").source(source).timestamp("1970-01-01");
@@ -407,4 +427,54 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
             assertThat(e.getMessage(), containsString("failed to parse timestamp [1234567890]"));
         }
     }
+
+    public void testSizeTimestampIndexParsing() throws IOException {
+        IndexService indexService = createIndex("test", BW_SETTINGS);
+        String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/default_mapping_with_disabled_root_types.json");
+        DocumentMapper documentMapper = indexService.mapperService().parse("type", new CompressedXContent(mapping), true);
+        assertThat(documentMapper.mappingSource().string(), equalTo(mapping));
+        documentMapper = indexService.mapperService().parse("type", new CompressedXContent(documentMapper.mappingSource().string()), true);
+        assertThat(documentMapper.mappingSource().string(), equalTo(mapping));
+    }
+
+    public void testDefaultApplied() throws IOException {
+        createIndex("test1", BW_SETTINGS);
+        createIndex("test2", BW_SETTINGS);
+        XContentBuilder defaultMapping = XContentFactory.jsonBuilder().startObject()
+                .startObject(MapperService.DEFAULT_MAPPING).startObject("_timestamp").field("enabled", true).endObject().endObject()
+                .endObject();
+        client().admin().indices().preparePutMapping().setType(MapperService.DEFAULT_MAPPING).setSource(defaultMapping).get();
+        XContentBuilder typeMapping = XContentFactory.jsonBuilder().startObject()
+                .startObject("type").startObject("_all").field("enabled", false).endObject().endObject()
+                .endObject();
+        client().admin().indices().preparePutMapping("test1").setType("type").setSource(typeMapping).get();
+        client().admin().indices().preparePutMapping("test1", "test2").setType("type").setSource(typeMapping).get();
+
+        GetMappingsResponse response = client().admin().indices().prepareGetMappings("test2").get();
+        assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_all"));
+        assertFalse((Boolean) ((LinkedHashMap) response.getMappings().get("test2").get("type").getSourceAsMap().get("_all")).get("enabled"));
+        assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_timestamp"));
+        assertTrue((Boolean)((LinkedHashMap)response.getMappings().get("test2").get("type").getSourceAsMap().get("_timestamp")).get("enabled"));
+    }
+
+    public void testTimestampParsing() throws IOException {
+        IndexService indexService = createIndex("test", BW_SETTINGS);
+        XContentBuilder indexMapping = XContentFactory.jsonBuilder();
+        boolean enabled = randomBoolean();
+        indexMapping.startObject()
+                .startObject("type")
+                .startObject("_timestamp")
+                .field("enabled", enabled)
+                .endObject()
+                .endObject()
+                .endObject();
+        DocumentMapper documentMapper = indexService.mapperService().parse("type", new CompressedXContent(indexMapping.string()), true);
+        assertThat(documentMapper.timestampFieldMapper().enabled(), equalTo(enabled));
+        assertTrue(documentMapper.timestampFieldMapper().fieldType().stored());
+        assertTrue(documentMapper.timestampFieldMapper().fieldType().hasDocValues());
+        documentMapper = indexService.mapperService().parse("type", new CompressedXContent(documentMapper.mappingSource().string()), true);
+        assertThat(documentMapper.timestampFieldMapper().enabled(), equalTo(enabled));
+        assertTrue(documentMapper.timestampFieldMapper().fieldType().hasDocValues());
+        assertTrue(documentMapper.timestampFieldMapper().fieldType().stored());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java
index fe057045882..ad9e03327b2 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java
@@ -20,7 +20,9 @@
 package org.elasticsearch.index.mapper.ttl;
 
 import org.apache.lucene.index.IndexOptions;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
@@ -30,22 +32,48 @@ import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.index.mapper.internal.TTLFieldMapper;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
 
 import java.io.IOException;
+import java.util.Collection;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.startsWith;
 
 public class TTLMappingTests extends ESSingleNodeTestCase {
+
+    private static final Settings BW_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0).build();
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(InternalSettingsPlugin.class);
+    }
+
+    public void testRejectedOn5x() throws IOException {
+        String mapping = XContentFactory.jsonBuilder().startObject()
+                .startObject("type")
+                    .startObject("_ttl")
+                        .field("enabled", true)
+                    .endObject()
+                .endObject().endObject().string();
+        IndexService index = createIndex("test");
+        IllegalArgumentException expected = expectThrows(IllegalArgumentException.class,
+                () -> index.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, false));
+        assertThat(expected.getMessage(), startsWith("[_ttl] is removed"));
+    }
+
     public void testSimpleDisabled() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         BytesReference source = XContentFactory.jsonBuilder()
                 .startObject()
                 .field("field", "value")
@@ -60,7 +88,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_ttl").field("enabled", "yes").endObject()
                 .endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         BytesReference source = XContentFactory.jsonBuilder()
                 .startObject()
                 .field("field", "value")
@@ -75,7 +103,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
 
     public void testDefaultValues() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
         assertThat(docMapper.TTLFieldMapper().enabled(), equalTo(TTLFieldMapper.Defaults.ENABLED_STATE.enabled));
         assertThat(docMapper.TTLFieldMapper().fieldType().stored(), equalTo(TTLFieldMapper.Defaults.TTL_FIELD_TYPE.stored()));
         assertThat(docMapper.TTLFieldMapper().fieldType().indexOptions(), equalTo(TTLFieldMapper.Defaults.TTL_FIELD_TYPE.indexOptions()));
@@ -93,7 +121,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
                 .startObject("properties").field("field").startObject().field("type", "text").endObject().endObject()
                 .endObject().endObject().string();
 
-        MapperService mapperService = createIndex("test").mapperService();
+        MapperService mapperService = createIndex("test", BW_SETTINGS).mapperService();
         DocumentMapper mapperWithoutTtl = mapperService.merge("type", new CompressedXContent(mappingWithoutTtl), MapperService.MergeReason.MAPPING_UPDATE, false);
         DocumentMapper mapperWithTtl = mapperService.merge("type", new CompressedXContent(mappingWithTtl), MapperService.MergeReason.MAPPING_UPDATE, false);
 
@@ -116,7 +144,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
                 .startObject("properties").field("field").startObject().field("type", "text").endObject().endObject()
                 .endObject().endObject().string();
 
-        MapperService mapperService = createIndex("test").mapperService();
+        MapperService mapperService = createIndex("test", BW_SETTINGS).mapperService();
         DocumentMapper initialMapper = mapperService.merge("type", new CompressedXContent(mappingWithTtl), MapperService.MergeReason.MAPPING_UPDATE, false);
         DocumentMapper updatedMapper = mapperService.merge("type", new CompressedXContent(updatedMapping), MapperService.MergeReason.MAPPING_UPDATE, false);
 
@@ -127,7 +155,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
     public void testThatDisablingTTLReportsConflict() throws Exception {
         String mappingWithTtl = getMappingWithTtlEnabled().string();
         String mappingWithTtlDisabled = getMappingWithTtlDisabled().string();
-        MapperService mapperService = createIndex("test").mapperService();
+        MapperService mapperService = createIndex("test", BW_SETTINGS).mapperService();
         DocumentMapper initialMapper = mapperService.merge("type", new CompressedXContent(mappingWithTtl), MapperService.MergeReason.MAPPING_UPDATE, false);
 
         try {
@@ -143,7 +171,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
     public void testThatDisablingTTLReportsConflictOnCluster() throws Exception {
         String mappingWithTtl = getMappingWithTtlEnabled().string();
         String mappingWithTtlDisabled = getMappingWithTtlDisabled().string();
-        assertAcked(client().admin().indices().prepareCreate("testindex").addMapping("type", mappingWithTtl));
+        assertAcked(client().admin().indices().prepareCreate("testindex").setSettings(BW_SETTINGS).addMapping("type", mappingWithTtl));
         GetMappingsResponse mappingsBeforeUpdateResponse = client().admin().indices().prepareGetMappings("testindex").addTypes("type").get();
         try {
             client().admin().indices().preparePutMapping("testindex").setSource(mappingWithTtlDisabled).setType("type").get();
@@ -158,7 +186,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
     public void testThatEnablingTTLAfterFirstDisablingWorks() throws Exception {
         String mappingWithTtl = getMappingWithTtlEnabled().string();
         String withTtlDisabled = getMappingWithTtlDisabled().string();
-        assertAcked(client().admin().indices().prepareCreate("testindex").addMapping("type", withTtlDisabled));
+        assertAcked(client().admin().indices().prepareCreate("testindex").setSettings(BW_SETTINGS).addMapping("type", withTtlDisabled));
         GetMappingsResponse mappingsAfterUpdateResponse = client().admin().indices().prepareGetMappings("testindex").addTypes("type").get();
         assertThat(mappingsAfterUpdateResponse.getMappings().get("testindex").get("type").sourceAsMap().get("_ttl").toString(), equalTo("{enabled=false}"));
         client().admin().indices().preparePutMapping("testindex").setSource(mappingWithTtl).setType("type").get();
@@ -167,20 +195,20 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
     }
 
     public void testNoConflictIfNothingSetAndDisabledLater() throws Exception {
-        IndexService indexService = createIndex("testindex", Settings.builder().build(), "type");
+        IndexService indexService = createIndex("testindex", BW_SETTINGS, "type");
         XContentBuilder mappingWithTtlDisabled = getMappingWithTtlDisabled("7d");
         indexService.mapperService().merge("type", new CompressedXContent(mappingWithTtlDisabled.string()), MapperService.MergeReason.MAPPING_UPDATE, false);
     }
 
     public void testNoConflictIfNothingSetAndEnabledLater() throws Exception {
-        IndexService indexService = createIndex("testindex", Settings.builder().build(), "type");
+        IndexService indexService = createIndex("testindex", BW_SETTINGS, "type");
         XContentBuilder mappingWithTtlEnabled = getMappingWithTtlEnabled("7d");
         indexService.mapperService().merge("type", new CompressedXContent(mappingWithTtlEnabled.string()), MapperService.MergeReason.MAPPING_UPDATE, false);
     }
 
     public void testMergeWithOnlyDefaultSet() throws Exception {
         XContentBuilder mappingWithTtlEnabled = getMappingWithTtlEnabled("7d");
-        IndexService indexService = createIndex("testindex", Settings.builder().build(), "type", mappingWithTtlEnabled);
+        IndexService indexService = createIndex("testindex", BW_SETTINGS, "type", mappingWithTtlEnabled);
         XContentBuilder mappingWithOnlyDefaultSet = getMappingWithOnlyTtlDefaultSet("6m");
         indexService.mapperService().merge("type", new CompressedXContent(mappingWithOnlyDefaultSet.string()), MapperService.MergeReason.MAPPING_UPDATE, false);
         CompressedXContent mappingAfterMerge = indexService.mapperService().documentMapper("type").mappingSource();
@@ -189,7 +217,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
 
     public void testMergeWithOnlyDefaultSetTtlDisabled() throws Exception {
         XContentBuilder mappingWithTtlEnabled = getMappingWithTtlDisabled("7d");
-        IndexService indexService = createIndex("testindex", Settings.builder().build(), "type", mappingWithTtlEnabled);
+        IndexService indexService = createIndex("testindex", BW_SETTINGS, "type", mappingWithTtlEnabled);
         CompressedXContent mappingAfterCreation = indexService.mapperService().documentMapper("type").mappingSource();
         assertThat(mappingAfterCreation, equalTo(new CompressedXContent("{\"type\":{\"_ttl\":{\"enabled\":false},\"properties\":{\"field\":{\"type\":\"text\"}}}}")));
         XContentBuilder mappingWithOnlyDefaultSet = getMappingWithOnlyTtlDefaultSet("6m");
@@ -202,7 +230,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
             .startObject("_ttl").field("enabled", true).endObject()
             .endObject().endObject().string();
-        DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
+        DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping));
 
         try {
             docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java b/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java
index 1d849d50932..3041cc8d13c 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java
@@ -31,12 +31,12 @@ import static org.hamcrest.Matchers.equalTo;
 public class ParseMappingTypeLevelTests extends ESSingleNodeTestCase {
     public void testTypeLevel() throws Exception {
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("_timestamp").field("enabled", true).endObject()
+                .startObject("_all").field("enabled", false).endObject()
                 .endObject().endObject().string();
 
         DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
         DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
         assertThat(mapper.type(), equalTo("type"));
-        assertThat(mapper.timestampFieldMapper().enabled(), equalTo(true));
+        assertThat(mapper.allFieldMapper().enabled(), equalTo(false));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java
index ed20cdb7e35..afac48e2b89 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java
@@ -139,16 +139,6 @@ public class UpdateMappingOnClusterIT extends ESIntegTestCase {
         compareMappingOnNodes(mappingsBeforeUpdateResponse);
     }
 
-    // checks if the setting for timestamp and size are kept even if disabled
-    public void testDisabledSizeTimestampIndexDoNotLooseMappings() throws Exception {
-        String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/default_mapping_with_disabled_root_types.json");
-        prepareCreate(INDEX).addMapping(TYPE, mapping).get();
-        GetMappingsResponse mappingsBeforeGreen = client().admin().indices().prepareGetMappings(INDEX).addTypes(TYPE).get();
-        ensureGreen(INDEX);
-        // make sure all nodes have same cluster state
-        compareMappingOnNodes(mappingsBeforeGreen);
-    }
-
     protected void testConflict(String mapping, String mappingUpdate, String... errorMessages) throws InterruptedException {
         assertAcked(prepareCreate(INDEX).setSource(mapping).get());
         ensureGreen(INDEX);
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java
index 17cb8a84664..86624d938b6 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java
@@ -259,56 +259,6 @@ public class UpdateMappingTests extends ESSingleNodeTestCase {
         }
     }
 
-    public void testTimestampParsing() throws IOException {
-        IndexService indexService = createIndex("test");
-        XContentBuilder indexMapping = XContentFactory.jsonBuilder();
-        boolean enabled = randomBoolean();
-        indexMapping.startObject()
-                .startObject("type")
-                .startObject("_timestamp")
-                .field("enabled", enabled)
-                .endObject()
-                .endObject()
-                .endObject();
-        DocumentMapper documentMapper = indexService.mapperService().parse("type", new CompressedXContent(indexMapping.string()), true);
-        assertThat(documentMapper.timestampFieldMapper().enabled(), equalTo(enabled));
-        assertTrue(documentMapper.timestampFieldMapper().fieldType().stored());
-        assertTrue(documentMapper.timestampFieldMapper().fieldType().hasDocValues());
-        documentMapper = indexService.mapperService().parse("type", new CompressedXContent(documentMapper.mappingSource().string()), true);
-        assertThat(documentMapper.timestampFieldMapper().enabled(), equalTo(enabled));
-        assertTrue(documentMapper.timestampFieldMapper().fieldType().hasDocValues());
-        assertTrue(documentMapper.timestampFieldMapper().fieldType().stored());
-    }
-
-    public void testSizeTimestampIndexParsing() throws IOException {
-        IndexService indexService = createIndex("test", Settings.builder().build());
-        String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/default_mapping_with_disabled_root_types.json");
-        DocumentMapper documentMapper = indexService.mapperService().parse("type", new CompressedXContent(mapping), true);
-        assertThat(documentMapper.mappingSource().string(), equalTo(mapping));
-        documentMapper = indexService.mapperService().parse("type", new CompressedXContent(documentMapper.mappingSource().string()), true);
-        assertThat(documentMapper.mappingSource().string(), equalTo(mapping));
-    }
-
-    public void testDefaultApplied() throws IOException {
-        createIndex("test1", Settings.builder().build());
-        createIndex("test2", Settings.builder().build());
-        XContentBuilder defaultMapping = XContentFactory.jsonBuilder().startObject()
-                .startObject(MapperService.DEFAULT_MAPPING).startObject("_timestamp").field("enabled", true).endObject().endObject()
-                .endObject();
-        client().admin().indices().preparePutMapping().setType(MapperService.DEFAULT_MAPPING).setSource(defaultMapping).get();
-        XContentBuilder typeMapping = XContentFactory.jsonBuilder().startObject()
-                .startObject("type").startObject("_all").field("enabled", false).endObject().endObject()
-                .endObject();
-        client().admin().indices().preparePutMapping("test1").setType("type").setSource(typeMapping).get();
-        client().admin().indices().preparePutMapping("test1", "test2").setType("type").setSource(typeMapping).get();
-
-        GetMappingsResponse response = client().admin().indices().prepareGetMappings("test2").get();
-        assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_all"));
-        assertFalse((Boolean) ((LinkedHashMap) response.getMappings().get("test2").get("type").getSourceAsMap().get("_all")).get("enabled"));
-        assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_timestamp"));
-        assertTrue((Boolean)((LinkedHashMap)response.getMappings().get("test2").get("type").getSourceAsMap().get("_timestamp")).get("enabled"));
-    }
-
     public void testRejectFieldDefinedTwice() throws IOException {
         String mapping1 = XContentFactory.jsonBuilder().startObject()
                 .startObject("type1")
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
index cdb722ff9dd..4f7064a33bb 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
@@ -65,7 +65,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSear
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.core.IsNull.notNullValue;
 
@@ -107,8 +106,7 @@ public class DateHistogramIT extends ESIntegTestCase {
 
     @Override
     public void setupSuiteScopeCluster() throws Exception {
-        assertAcked(prepareCreate("idx").addMapping("type", "_timestamp", "enabled=true"));
-        createIndex("idx_unmapped");
+        createIndex("idx", "idx_unmapped");
         // TODO: would be nice to have more random data here
         assertAcked(prepareCreate("empty_bucket_idx").addMapping("type", "value", "type=integer"));
         List<IndexRequestBuilder> builders = new ArrayList<>();
@@ -1141,13 +1139,6 @@ public class DateHistogramIT extends ESIntegTestCase {
         }
     }
 
-    public void testTimestampField() { // see #11692
-        SearchResponse response = client().prepareSearch("idx").addAggregation(dateHistogram("histo").field("_timestamp").dateHistogramInterval(randomFrom(DateHistogramInterval.DAY, DateHistogramInterval.MONTH))).get();
-        assertSearchResponse(response);
-        Histogram histo = response.getAggregations().get("histo");
-        assertThat(histo.getBuckets().size(), greaterThan(0));
-    }
-
     /**
      * When DST ends, local time turns back one hour, so between 2am and 4am wall time we should have four buckets:
      * "2015-10-25T02:00:00.000+02:00",
diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
index 188ae22471a..99183b11c58 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java
@@ -36,7 +36,6 @@ import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.hamcrest.Matchers;
@@ -70,10 +69,8 @@ import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThan;
-import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -1340,16 +1337,12 @@ public class FieldSortIT extends ESIntegTestCase {
     }
 
     public void testSortMetaField() throws Exception {
-        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("_timestamp").field("enabled", true).endObject()
-                .endObject().endObject();
-        assertAcked(prepareCreate("test")
-            .addMapping("type", mapping));
+        createIndex("test");
         ensureGreen();
         final int numDocs = randomIntBetween(10, 20);
         IndexRequestBuilder[] indexReqs = new IndexRequestBuilder[numDocs];
         for (int i = 0; i < numDocs; ++i) {
-            indexReqs[i] = client().prepareIndex("test", "type", Integer.toString(i)).setTimestamp(Integer.toString(randomInt(1000)))
+            indexReqs[i] = client().prepareIndex("test", "type", Integer.toString(i))
                     .setSource();
         }
         indexRandom(true, indexReqs);
@@ -1368,37 +1361,6 @@ public class FieldSortIT extends ESIntegTestCase {
             assertThat(previous, order == SortOrder.ASC ? lessThan(uid) : greaterThan(uid));
             previous = uid;
         }
-
-        /*
-        searchResponse = client().prepareSearch()
-                .setQuery(matchAllQuery())
-                .setSize(randomIntBetween(1, numDocs + 5))
-                .addSort("_id", order)
-                .execute().actionGet();
-        assertNoFailures(searchResponse);
-        hits = searchResponse.getHits().hits();
-        previous = order == SortOrder.ASC ? new BytesRef() : UnicodeUtil.BIG_TERM;
-        for (int i = 0; i < hits.length; ++i) {
-            final BytesRef id = new BytesRef(Uid.createUid(hits[i].type(), hits[i].id()));
-            assertThat(previous, order == SortOrder.ASC ? lessThan(id) : greaterThan(id));
-            previous = id;
-        }*/
-
-        searchResponse = client().prepareSearch()
-                .setQuery(matchAllQuery())
-                .setSize(randomIntBetween(1, numDocs + 5))
-                .addSort("_timestamp", order)
-                .addField("_timestamp")
-                .execute().actionGet();
-        assertNoFailures(searchResponse);
-        hits = searchResponse.getHits().hits();
-        Long previousTs = order == SortOrder.ASC ? 0 : Long.MAX_VALUE;
-        for (int i = 0; i < hits.length; ++i) {
-            SearchHitField timestampField = hits[i].getFields().get("_timestamp");
-            Long timestamp = timestampField.<Long>getValue();
-            assertThat(previousTs, order == SortOrder.ASC ? lessThanOrEqualTo(timestamp) : greaterThanOrEqualTo(timestamp));
-            previousTs = timestamp;
-        }
     }
 
     /**
diff --git a/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java b/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java
index 2fa94bc2357..bd24ab9f43b 100644
--- a/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java
+++ b/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java
@@ -19,14 +19,20 @@
 
 package org.elasticsearch.timestamp;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
 import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
 
+import java.util.Collection;
 import java.util.Locale;
 
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
@@ -41,8 +47,17 @@ import static org.hamcrest.Matchers.notNullValue;
 /**
  */
 public class SimpleTimestampIT extends ESIntegTestCase {
+
+    private static final Settings BW_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0).build();
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return pluginList(InternalSettingsPlugin.class);
+    }
+
     public void testSimpleTimestamp() throws Exception {
         client().admin().indices().prepareCreate("test")
+                .setSettings(BW_SETTINGS)
                 .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("_timestamp").field("enabled", true).endObject().endObject().endObject())
                 .execute().actionGet();
         client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet();
@@ -98,7 +113,7 @@ public class SimpleTimestampIT extends ESIntegTestCase {
         String type = "mytype";
 
         XContentBuilder builder = jsonBuilder().startObject().startObject("_timestamp").field("enabled", true).endObject().endObject();
-        assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
+        assertAcked(client().admin().indices().prepareCreate(index).setSettings(BW_SETTINGS).addMapping(type, builder));
 
         // check mapping again
         assertTimestampMappingEnabled(index, type, true);
@@ -117,7 +132,7 @@ public class SimpleTimestampIT extends ESIntegTestCase {
         String type = "mytype";
 
         XContentBuilder builder = jsonBuilder().startObject().startObject("_timestamp").field("enabled", true).endObject().endObject();
-        assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
+        assertAcked(client().admin().indices().prepareCreate(index).setSettings(BW_SETTINGS).addMapping(type, builder));
 
         // check mapping again
         assertTimestampMappingEnabled(index, type, true);
diff --git a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
index 003c08134bc..9e08ecde6fa 100644
--- a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
+++ b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.ttl;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
@@ -26,14 +27,19 @@ import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.index.IndexResponse;
 import org.elasticsearch.action.update.UpdateRequestBuilder;
 import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 
 import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
@@ -60,6 +66,11 @@ public class SimpleTTLIT extends ESIntegTestCase {
         return 2;
     }
 
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singleton(InternalSettingsPlugin.class);
+    }
+
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
         return Settings.builder()
@@ -70,6 +81,7 @@ public class SimpleTTLIT extends ESIntegTestCase {
 
     public void testSimpleTTL() throws Exception {
         assertAcked(prepareCreate("test")
+                .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id)
                 .addMapping("type1", XContentFactory.jsonBuilder()
                         .startObject()
                         .startObject("type1")
@@ -209,7 +221,7 @@ public class SimpleTTLIT extends ESIntegTestCase {
         String type = "mytype";
 
         XContentBuilder builder = jsonBuilder().startObject().startObject("_ttl").field("enabled", true).endObject().endObject();
-        assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
+        assertAcked(client().admin().indices().prepareCreate(index).setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id).addMapping(type, builder));
 
         // check mapping again
         assertTTLMappingEnabled(index, type);
@@ -232,6 +244,7 @@ public class SimpleTTLIT extends ESIntegTestCase {
      */
     public void testNoopUpdate() throws IOException {
         assertAcked(prepareCreate("test")
+                .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id)
                 .addMapping("type1", XContentFactory.jsonBuilder()
                         .startObject()
                         .startObject("type1")
diff --git a/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java b/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java
new file mode 100644
index 00000000000..3573089fcaa
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.update;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.alias.Alias;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.update.UpdateResponse;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.engine.DocumentMissingException;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHitField;
+import org.elasticsearch.search.sort.SortOrder;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.InternalSettingsPlugin;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
+
+public class TimestampTTLBWIT extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(
+                UpdateIT.FieldIncrementScriptPlugin.class,
+                UpdateIT.ExtractContextInSourceScriptPlugin.class,
+                UpdateIT.PutFieldValuesScriptPlugin.class,
+                InternalSettingsPlugin.class
+        );
+    }
+
+    public void testSort() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("_timestamp").field("enabled", true).endObject()
+                .endObject().endObject();
+        assertAcked(prepareCreate("test")
+                .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id)
+                .addMapping("type", mapping));
+        ensureGreen();
+        final int numDocs = randomIntBetween(10, 20);
+        IndexRequestBuilder[] indexReqs = new IndexRequestBuilder[numDocs];
+        for (int i = 0; i < numDocs; ++i) {
+            indexReqs[i] = client().prepareIndex("test", "type", Integer.toString(i)).setTimestamp(Integer.toString(randomInt(1000)))
+                    .setSource();
+        }
+        indexRandom(true, indexReqs);
+
+        SortOrder order = randomFrom(SortOrder.values());
+
+        SearchResponse searchResponse = client().prepareSearch()
+                .setQuery(matchAllQuery())
+                .setSize(randomIntBetween(1, numDocs + 5))
+                .addSort("_timestamp", order)
+                .addField("_timestamp")
+                .execute().actionGet();
+        assertNoFailures(searchResponse);
+        SearchHit[] hits = searchResponse.getHits().hits();
+        Long previousTs = order == SortOrder.ASC ? 0 : Long.MAX_VALUE;
+        for (int i = 0; i < hits.length; ++i) {
+            SearchHitField timestampField = hits[i].getFields().get("_timestamp");
+            Long timestamp = timestampField.<Long>getValue();
+            assertThat(previousTs, order == SortOrder.ASC ? lessThanOrEqualTo(timestamp) : greaterThanOrEqualTo(timestamp));
+            previousTs = timestamp;
+        }
+    }
+
+    public void testUpdate() throws Exception {
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias"))
+                .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id)
+                .addMapping("type1", XContentFactory.jsonBuilder()
+                        .startObject()
+                        .startObject("type1")
+                        .startObject("_timestamp").field("enabled", true).endObject()
+                        .startObject("_ttl").field("enabled", true).endObject()
+                        .endObject()
+                        .endObject()));
+
+        ensureGreen();
+
+        try {
+            client().prepareUpdate(indexOrAlias(), "type1", "1")
+                    .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null)).execute().actionGet();
+            fail();
+        } catch (DocumentMissingException e) {
+            // all is well
+        }
+
+        // check TTL is kept after an update without TTL
+        client().prepareIndex("test", "type1", "2").setSource("field", 1).setTTL(86400000L).setRefreshPolicy(IMMEDIATE).get();
+        GetResponse getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
+        long ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue();
+        assertThat(ttl, greaterThan(0L));
+        client().prepareUpdate(indexOrAlias(), "type1", "2")
+                .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null)).execute().actionGet();
+        getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
+        ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue();
+        assertThat(ttl, greaterThan(0L));
+
+        // check TTL update
+        client().prepareUpdate(indexOrAlias(), "type1", "2")
+                .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values",
+                        Collections.singletonMap("_ctx", Collections.singletonMap("_ttl", 3600000)))).execute().actionGet();
+        getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
+        ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue();
+        assertThat(ttl, greaterThan(0L));
+        assertThat(ttl, lessThanOrEqualTo(3600000L));
+
+        // check timestamp update
+        client().prepareIndex("test", "type1", "3").setSource("field", 1).setRefreshPolicy(IMMEDIATE).get();
+        client().prepareUpdate(indexOrAlias(), "type1", "3")
+                .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values",
+                        Collections.singletonMap("_ctx", Collections.singletonMap("_timestamp", "2009-11-15T14:12:12")))).execute()
+                .actionGet();
+        getResponse = client().prepareGet("test", "type1", "3").setFields("_timestamp").execute().actionGet();
+        long timestamp = ((Number) getResponse.getField("_timestamp").getValue()).longValue();
+        assertThat(timestamp, equalTo(1258294332000L));
+    }
+
+    public void testContextVariables() throws Exception {
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias"))
+                        .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id)
+                        .addMapping("type1", XContentFactory.jsonBuilder()
+                                .startObject()
+                                .startObject("type1")
+                                .startObject("_timestamp").field("enabled", true).endObject()
+                                .startObject("_ttl").field("enabled", true).endObject()
+                                .endObject()
+                                .endObject())
+                        .addMapping("subtype1", XContentFactory.jsonBuilder()
+                                .startObject()
+                                .startObject("subtype1")
+                                .startObject("_parent").field("type", "type1").endObject()
+                                .startObject("_timestamp").field("enabled", true).endObject()
+                                .startObject("_ttl").field("enabled", true).endObject()
+                                .endObject()
+                                .endObject())
+        );
+        ensureGreen();
+
+        // Index some documents
+        long timestamp = System.currentTimeMillis();
+        client().prepareIndex()
+                .setIndex("test")
+                .setType("type1")
+                .setId("parentId1")
+                .setTimestamp(String.valueOf(timestamp-1))
+                .setSource("field1", 0, "content", "bar")
+                .execute().actionGet();
+
+        long ttl = 10000;
+        client().prepareIndex()
+                .setIndex("test")
+                .setType("subtype1")
+                .setId("id1")
+                .setParent("parentId1")
+                .setRouting("routing1")
+                .setTimestamp(String.valueOf(timestamp))
+                .setTTL(ttl)
+                .setSource("field1", 1, "content", "foo")
+                .execute().actionGet();
+
+        // Update the first object and note context variables values
+        UpdateResponse updateResponse = client().prepareUpdate("test", "subtype1", "id1")
+                .setRouting("routing1")
+                .setScript(new Script("", ScriptService.ScriptType.INLINE, "extract_ctx", null))
+                .execute().actionGet();
+
+        assertEquals(2, updateResponse.getVersion());
+
+        GetResponse getResponse = client().prepareGet("test", "subtype1", "id1").setRouting("routing1").execute().actionGet();
+        Map<String, Object> updateContext = (Map<String, Object>) getResponse.getSourceAsMap().get("update_context");
+        assertEquals("test", updateContext.get("_index"));
+        assertEquals("subtype1", updateContext.get("_type"));
+        assertEquals("id1", updateContext.get("_id"));
+        assertEquals(1, updateContext.get("_version"));
+        assertEquals("parentId1", updateContext.get("_parent"));
+        assertEquals("routing1", updateContext.get("_routing"));
+        assertThat(((Integer) updateContext.get("_ttl")).longValue(), allOf(greaterThanOrEqualTo(ttl-3000), lessThanOrEqualTo(ttl)));
+
+        // Idem with the second object
+        updateResponse = client().prepareUpdate("test", "type1", "parentId1")
+                .setScript(new Script("", ScriptService.ScriptType.INLINE, "extract_ctx", null))
+                .execute().actionGet();
+
+        assertEquals(2, updateResponse.getVersion());
+
+        getResponse = client().prepareGet("test", "type1", "parentId1").execute().actionGet();
+        updateContext = (Map<String, Object>) getResponse.getSourceAsMap().get("update_context");
+        assertEquals("test", updateContext.get("_index"));
+        assertEquals("type1", updateContext.get("_type"));
+        assertEquals("parentId1", updateContext.get("_id"));
+        assertEquals(1, updateContext.get("_version"));
+        assertNull(updateContext.get("_parent"));
+        assertNull(updateContext.get("_routing"));
+        assertNull(updateContext.get("_ttl"));
+    }
+
+    private static String indexOrAlias() {
+        return randomBoolean() ? "test" : "alias";
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/update/UpdateIT.java b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
index 4121543a7a0..55834c181b0 100644
--- a/core/src/test/java/org/elasticsearch/update/UpdateIT.java
+++ b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
@@ -62,14 +62,12 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 
-import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
@@ -357,21 +355,14 @@ public class UpdateIT extends ESIntegTestCase {
                 FieldIncrementScriptPlugin.class,
                 ScriptedUpsertScriptPlugin.class,
                 ExtractContextInSourceScriptPlugin.class,
-                InternalSettingsPlugin.class // uses index.merge.enabled
+                InternalSettingsPlugin.class
         );
     }
 
     private void createTestIndex() throws Exception {
         logger.info("--> creating index test");
 
-        assertAcked(prepareCreate("test").addAlias(new Alias("alias"))
-                .addMapping("type1", XContentFactory.jsonBuilder()
-                        .startObject()
-                        .startObject("type1")
-                        .startObject("_timestamp").field("enabled", true).endObject()
-                        .startObject("_ttl").field("enabled", true).endObject()
-                        .endObject()
-                        .endObject()));
+        assertAcked(prepareCreate("test").addAlias(new Alias("alias")));
     }
 
     public void testUpsert() throws Exception {
@@ -638,34 +629,6 @@ public class UpdateIT extends ESIntegTestCase {
             assertThat(getResponse.isExists(), equalTo(false));
         }
 
-        // check TTL is kept after an update without TTL
-        client().prepareIndex("test", "type1", "2").setSource("field", 1).setTTL(86400000L).setRefreshPolicy(IMMEDIATE).get();
-        GetResponse getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
-        long ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue();
-        assertThat(ttl, greaterThan(0L));
-        client().prepareUpdate(indexOrAlias(), "type1", "2")
-                .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null)).execute().actionGet();
-        getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
-        ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue();
-        assertThat(ttl, greaterThan(0L));
-
-        // check TTL update
-        client().prepareUpdate(indexOrAlias(), "type1", "2")
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", Collections.singletonMap("_ctx", Collections.singletonMap("_ttl", 3600000)))).execute().actionGet();
-        getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet();
-        ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue();
-        assertThat(ttl, greaterThan(0L));
-        assertThat(ttl, lessThanOrEqualTo(3600000L));
-
-        // check timestamp update
-        client().prepareIndex("test", "type1", "3").setSource("field", 1).setRefreshPolicy(IMMEDIATE).get();
-        client().prepareUpdate(indexOrAlias(), "type1", "3")
-                .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", Collections.singletonMap("_ctx", Collections.singletonMap("_timestamp", "2009-11-15T14:12:12")))).execute()
-                .actionGet();
-        getResponse = client().prepareGet("test", "type1", "3").setFields("_timestamp").execute().actionGet();
-        long timestamp = ((Number) getResponse.getField("_timestamp").getValue()).longValue();
-        assertThat(timestamp, equalTo(1258294332000L));
-
         // check fields parameter
         client().prepareIndex("test", "type1", "1").setSource("field", 1).execute().actionGet();
         updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1")
@@ -682,7 +645,7 @@ public class UpdateIT extends ESIntegTestCase {
         client().prepareIndex("test", "type1", "1").setSource("field", 1).execute().actionGet();
         updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(XContentFactory.jsonBuilder().startObject().field("field2", 2).endObject()).execute().actionGet();
         for (int i = 0; i < 5; i++) {
-            getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
+            GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
             assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("1"));
             assertThat(getResponse.getSourceAsMap().get("field2").toString(), equalTo("2"));
         }
@@ -690,7 +653,7 @@ public class UpdateIT extends ESIntegTestCase {
         // change existing field
         updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(XContentFactory.jsonBuilder().startObject().field("field", 3).endObject()).execute().actionGet();
         for (int i = 0; i < 5; i++) {
-            getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
+            GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
             assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("3"));
             assertThat(getResponse.getSourceAsMap().get("field2").toString(), equalTo("2"));
         }
@@ -708,7 +671,7 @@ public class UpdateIT extends ESIntegTestCase {
         client().prepareIndex("test", "type1", "1").setSource("map", testMap).execute().actionGet();
         updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(XContentFactory.jsonBuilder().startObject().field("map", testMap3).endObject()).execute().actionGet();
         for (int i = 0; i < 5; i++) {
-            getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
+            GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet();
             Map map1 = (Map) getResponse.getSourceAsMap().get("map");
             assertThat(map1.size(), equalTo(3));
             assertThat(map1.containsKey("map1"), equalTo(true));
@@ -760,16 +723,12 @@ public class UpdateIT extends ESIntegTestCase {
                         .addMapping("type1", XContentFactory.jsonBuilder()
                                 .startObject()
                                 .startObject("type1")
-                                .startObject("_timestamp").field("enabled", true).endObject()
-                                .startObject("_ttl").field("enabled", true).endObject()
                                 .endObject()
                                 .endObject())
                         .addMapping("subtype1", XContentFactory.jsonBuilder()
                                 .startObject()
                                 .startObject("subtype1")
                                 .startObject("_parent").field("type", "type1").endObject()
-                                .startObject("_timestamp").field("enabled", true).endObject()
-                                .startObject("_ttl").field("enabled", true).endObject()
                                 .endObject()
                                 .endObject())
         );
@@ -813,7 +772,6 @@ public class UpdateIT extends ESIntegTestCase {
         assertEquals(1, updateContext.get("_version"));
         assertEquals("parentId1", updateContext.get("_parent"));
         assertEquals("routing1", updateContext.get("_routing"));
-        assertThat(((Integer) updateContext.get("_ttl")).longValue(), allOf(greaterThanOrEqualTo(ttl-3000), lessThanOrEqualTo(ttl)));
 
         // Idem with the second object
         updateResponse = client().prepareUpdate("test", "type1", "parentId1")
@@ -904,13 +862,6 @@ public class UpdateIT extends ESIntegTestCase {
     public void testStressUpdateDeleteConcurrency() throws Exception {
         //We create an index with merging disabled so that deletes don't get merged away
         assertAcked(prepareCreate("test")
-                .addMapping("type1", XContentFactory.jsonBuilder()
-                        .startObject()
-                        .startObject("type1")
-                        .startObject("_timestamp").field("enabled", true).endObject()
-                        .startObject("_ttl").field("enabled", true).endObject()
-                        .endObject()
-                        .endObject())
                 .setSettings(Settings.builder().put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)));
         ensureGreen();
 
diff --git a/docs/reference/docs/bulk.asciidoc b/docs/reference/docs/bulk.asciidoc
index 6b0a5b2e40d..1fbd0419e2e 100644
--- a/docs/reference/docs/bulk.asciidoc
+++ b/docs/reference/docs/bulk.asciidoc
@@ -127,26 +127,6 @@ Each bulk item can include the parent value using the `_parent`/`parent`
 field. It automatically follows the behavior of the index / delete
 operation based on the `_parent` / `_routing` mapping.
 
-[float]
-[[bulk-timestamp]]
-=== Timestamp
-
-deprecated[2.0.0,The `_timestamp` field is deprecated.  Instead, use a normal <<date,`date`>> field and set its value explicitly]
-
-Each bulk item can include the timestamp value using the
-`_timestamp`/`timestamp` field. It automatically follows the behavior of
-the index operation based on the `_timestamp` mapping.
-
-[float]
-[[bulk-ttl]]
-=== TTL
-
-deprecated[2.0.0,The current `_ttl` implementation is deprecated and will be replaced with a different implementation in a future version]
-
-Each bulk item can include the ttl value using the `_ttl`/`ttl` field.
-It automatically follows the behavior of the index operation based on
-the `_ttl` mapping.
-
 [float]
 [[bulk-consistency]]
 === Write Consistency
diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc
index aa62b65292e..e1c260ae48f 100644
--- a/docs/reference/docs/index_.asciidoc
+++ b/docs/reference/docs/index_.asciidoc
@@ -290,70 +290,6 @@ When indexing a child document, the routing value is automatically set
 to be the same as its parent, unless the routing value is explicitly
 specified using the `routing` parameter.
 
-[float]
-[[index-timestamp]]
-=== Timestamp
-
-deprecated[2.0.0,The `_timestamp` field is deprecated.  Instead, use a normal <<date,`date`>> field and set its value explicitly]
-
-A document can be indexed with a `timestamp` associated with it. The
-`timestamp` value of a document can be set using the `timestamp`
-parameter. For example:
-
-[source,js]
---------------------------------------------------
-PUT twitter/tweet/1?timestamp=2009-11-15T14:12:12
-{
-    "user" : "kimchy",
-    "message" : "trying out Elasticsearch"
-}
---------------------------------------------------
-// CONSOLE
-
-If the `timestamp` value is not provided externally or in the `_source`,
-the `timestamp` will be automatically set to the date the document was
-processed by the indexing chain. More information can be found on the
-<<mapping-timestamp-field,_timestamp mapping
-page>>.
-
-[float]
-[[index-ttl]]
-=== TTL
-
-deprecated[2.0.0,The current `_ttl` implementation is deprecated and will be replaced with a different implementation in a future version]
-
-
-A document can be indexed with a `ttl` (time to live) associated with
-it. Expired documents will be expunged automatically. The expiration
-date that will be set for a document with a provided `ttl` is relative
-to the `timestamp` of the document, meaning it can be based on the time
-of indexing or on any time provided. The provided `ttl` must be strictly
-positive and can be a number (in milliseconds) or any valid time value
-as shown in the following examples:
-
-[source,js]
---------------------------------------------------
-PUT twitter/tweet/1?ttl=86400000ms
-{
-    "user": "kimchy",
-    "message": "Trying out elasticsearch, so far so good?"
-}
---------------------------------------------------
-// CONSOLE
-
-[source,js]
---------------------------------------------------
-PUT twitter/tweet/1?ttl=1d
-{
-    "user": "kimchy",
-    "message": "Trying out elasticsearch, so far so good?"
-}
---------------------------------------------------
-// CONSOLE
-
-More information can be found on the
-<<mapping-ttl-field,_ttl mapping page>>.
-
 [float]
 [[index-distributed]]
 === Distributed
diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index bb2a06c0234..525552ae85a 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -272,8 +272,6 @@ change:
  * `_version`
  * `_routing`
  * `_parent`
- * `_timestamp`
- * `_ttl`
 
 Setting `_version` to `null` or clearing it from the `ctx` map is just like not
 sending the version in an indexing request. It will cause that document to be
diff --git a/docs/reference/docs/update.asciidoc b/docs/reference/docs/update.asciidoc
index 39261c5d21f..85e5bf2f67f 100644
--- a/docs/reference/docs/update.asciidoc
+++ b/docs/reference/docs/update.asciidoc
@@ -57,7 +57,7 @@ curl -XPOST 'localhost:9200/test/type1/1/_update' -d '{
 
 In addition to `_source`, the following variables are available through
 the `ctx` map: `_index`, `_type`, `_id`, `_version`, `_routing`,
-`_parent`, `_timestamp`, `_ttl`.
+`_parent`.
 
 We can also add a new field to the document:
 
diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc
index 3cc95e5af9a..ff1a3c62c7f 100644
--- a/docs/reference/index-modules.asciidoc
+++ b/docs/reference/index-modules.asciidoc
@@ -131,11 +131,6 @@ specific index module:
 
     Set to `true` to disable index metadata reads and writes.
 
-`index.ttl.disable_purge`::
-
-    experimental[] Disables the purge of <<mapping-ttl-field,expired docs>> on
-    the current index.
-
 `index.max_refresh_listeners`::
 
     Maximum number of refresh listeners available on each shard of the index.
diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc
index c8a5a05002d..2d906fd861a 100644
--- a/docs/reference/ingest/ingest-node.asciidoc
+++ b/docs/reference/ingest/ingest-node.asciidoc
@@ -439,8 +439,7 @@ The following example sets the `_id` metadata field of a document to `1`:
 }
 --------------------------------------------------
 
-The following metadata fields are accessible by a processor: `_index`, `_type`, `_id`, `_routing`, `_parent`,
-`_timestamp`, and `_ttl`.
+The following metadata fields are accessible by a processor: `_index`, `_type`, `_id`, `_routing`, `_parent`.
 
 [float]
 [[accessing-ingest-metadata]]
diff --git a/docs/reference/mapping/fields.asciidoc b/docs/reference/mapping/fields.asciidoc
index 90d75b97841..1cd886dde0e 100644
--- a/docs/reference/mapping/fields.asciidoc
+++ b/docs/reference/mapping/fields.asciidoc
@@ -48,14 +48,6 @@ can be customised when a mapping type is created.
 
     All fields in the document which contain non-null values.
 
-<<mapping-timestamp-field,`_timestamp`>>::
-
-    A timestamp associated with the document, either specified manually or auto-generated.
-
-<<mapping-ttl-field,`_ttl`>>::
-
-    How long a document should live before it is automatically deleted.
-
 [float]
 === Routing meta-fields
 
diff --git a/docs/reference/mapping/fields/timestamp-field.asciidoc b/docs/reference/mapping/fields/timestamp-field.asciidoc
deleted file mode 100644
index 104b07cfee1..00000000000
--- a/docs/reference/mapping/fields/timestamp-field.asciidoc
+++ /dev/null
@@ -1,97 +0,0 @@
-[[mapping-timestamp-field]]
-=== `_timestamp` field
-
-deprecated[2.0.0,The `_timestamp` field is deprecated.  Instead, use a normal <<date,`date`>> field and set its value explicitly]
-
-The `_timestamp` field, when enabled, allows a timestamp to be indexed and
-stored with a document. The timestamp may be specified manually, generated
-automatically, or set to a default value:
-
-[source,js]
-------------------------------------
-PUT my_index
-{
-  "mappings": {
-    "my_type": {
-      "_timestamp": { <1>
-        "enabled": true
-      }
-    }
-  }
-}
-
-PUT my_index/my_type/1?timestamp=2015-01-01 <2>
-{ "text": "Timestamp as a formatted date" }
-
-PUT my_index/my_type/2?timestamp=1420070400000 <3>
-{ "text": "Timestamp as milliseconds since the epoch" }
-
-PUT my_index/my_type/3 <4>
-{ "text": "Autogenerated timestamp set to now()" }
-
-------------------------------------
-// CONSOLE
-
-<1> Enable the `_timestamp` field with default settings.
-<2> Set the timestamp manually with a formatted date.
-<3> Set the timestamp with milliseconds since the epoch.
-<4> Auto-generates a timestamp with <<date-math,now()>>.
-
-The behaviour of the `_timestamp` field can be configured with the following parameters:
-
-`default`::
-
-    A default value to be used if none is provided.  Defaults to <<date-math,now()>>.
-
-`format`::
-
-    The <<mapping-date-format,date format>> (or formats) to use when parsing timestamps.  Defaults to `epoch_millis||strictDateOptionalTime`.
-
-`ignore_missing`::
-
-    If `true` (default), replace missing timestamps with the `default` value.  If `false`, throw an exception.
-
-
-The value of the `_timestamp` field is accessible in queries, aggregations, scripts,
-and when sorting:
-
-[source,js]
---------------------------
-GET my_index/_search
-{
-  "query": {
-    "range": {
-      "_timestamp": { <1>
-        "gte": "2015-01-01"
-      }
-    }
-  },
-  "aggs": {
-    "Timestamps": {
-      "terms": {
-        "field": "_timestamp", <2>
-        "size": 10
-      }
-    }
-  },
-  "sort": [
-    {
-      "_timestamp": { <3>
-        "order": "desc"
-      }
-    }
-  ],
-  "script_fields": {
-    "Timestamp": {
-      "script": "doc['_timestamp']" <4>
-    }
-  }
-}
---------------------------
-// CONSOLE
-// TEST[continued]
-
-<1> Querying on the `_timestamp` field
-<2> Aggregating on the `_timestamp` field
-<3> Sorting on the `_timestamp` field
-<4> Accessing the `_timestamp` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
diff --git a/docs/reference/mapping/fields/ttl-field.asciidoc b/docs/reference/mapping/fields/ttl-field.asciidoc
deleted file mode 100644
index 26760d5aca8..00000000000
--- a/docs/reference/mapping/fields/ttl-field.asciidoc
+++ /dev/null
@@ -1,112 +0,0 @@
-[[mapping-ttl-field]]
-=== `_ttl` field
-
-deprecated[2.0.0,The current `_ttl` implementation is deprecated and will be replaced with a different implementation in a future version]
-
-Some types of documents, such as session data or special offers, come with an
-expiration date. The `_ttl` field allows you to specify the minimum time a
-document should live, after which time the document is deleted automatically.
-
-[TIP]
-.Prefer index-per-timeframe to TTL
-======================================================
-
-With TTL , expired documents first have to be marked as deleted then later
-purged from the index when segments are merged.  For append-only time-based
-data such as log events, it is much more efficient to use an index-per-day /
-week / month instead of TTLs.  Old log data can be removed by simply deleting
-old indices.
-
-======================================================
-
-The `_ttl` field may be enabled as follows:
-
-[source,js]
--------------------------------
-PUT my_index
-{
-  "mappings": {
-    "my_type": {
-      "_ttl": {
-        "enabled": true
-      }
-    }
-  }
-}
-
-PUT my_index/my_type/1?ttl=10m <1>
-{
-  "text": "Will expire in 10 minutes"
-}
-
-PUT my_index/my_type/2 <2>
-{
-  "text": "Will not expire"
-}
--------------------------------
-// CONSOLE
-<1> This document will expire 10 minutes after being indexed.
-<2> This document has no TTL set and will not expire.
-
-The expiry time is calculated as the value of the
-<<mapping-timestamp-field,`_timestamp`>> field (or `now()` if the `_timestamp`
-is not enabled) plus the `ttl` specified in the indexing request.
-
-==== Default TTL
-
-You can provide a default `_ttl`, which will be applied to indexing requests where the `ttl` is not specified:
-
-[source,js]
--------------------------------
-PUT my_index
-{
-  "mappings": {
-    "my_type": {
-      "_ttl": {
-        "enabled": true,
-        "default": "5m"
-      }
-    }
-  }
-}
-
-PUT my_index/my_type/1?ttl=10m <1>
-{
-  "text": "Will expire in 10 minutes"
-}
-
-PUT my_index/my_type/2 <2>
-{
-  "text": "Will expire in 5 minutes"
-}
--------------------------------
-// CONSOLE
-<1> This document will expire 10 minutes after being indexed.
-<2> This document has no TTL set and so will expire after the default 5 minutes.
-
-The `default` value can use <<time-units,time units>> like `d` for days, and
-will use `ms` as the default unit if no time unit is provided.
-
-You can dynamically update the `default` value using the put mapping
-API. It won't change the `_ttl` of already indexed documents but will be
-used for future documents.
-
-==== Note on documents expiration
-
-Expired documents will be automatically deleted periodically. The following
-settings control the expiry process:
-
-`indices.ttl.interval`::
-
-How often the purge process should run. Defaults to `60s`. Expired documents
-may still be retrieved before they are purged.
-
-`indices.ttl.bulk_size`::
-
-How many deletions are handled by a single <<docs-bulk,`bulk`>> request. The
-default value is `10000`.
-
-==== Note on `detect_noop`
-If an update tries to update just the `_ttl` without changing the `_source` of
-the document it's expiration time won't be updated if `detect_noop` is `true`.
-In 2.1 `detect_noop` defaults to `true`.
diff --git a/docs/reference/migration/migrate_5_0/mapping.asciidoc b/docs/reference/migration/migrate_5_0/mapping.asciidoc
index 84dea729ec1..2d0ca4b9fb6 100644
--- a/docs/reference/migration/migrate_5_0/mapping.asciidoc
+++ b/docs/reference/migration/migrate_5_0/mapping.asciidoc
@@ -76,6 +76,14 @@ PUT my_index
 Also the `precision_step` parameter is now irrelevant and will be rejected on
 indices that are created on or after 5.0.
 
+==== `_timestamp` and `_ttl`
+
+The `_timestamp` and `_ttl` fields were deprecated and are now removed. As a
+replacement for `_timestamp`, you should populate a regular date field with the
+current timestamp on application side. For `_ttl`, you should either use
+time-based indices when applicable, or cron a delete-by-query with a range
+query on a timestamp field
+
 ==== `index` property
 
 On all field datatypes (except for the deprecated `string` field), the `index`
diff --git a/docs/reference/modules/indices.asciidoc b/docs/reference/modules/indices.asciidoc
index 309d4490123..34a1ccdf8fa 100644
--- a/docs/reference/modules/indices.asciidoc
+++ b/docs/reference/modules/indices.asciidoc
@@ -30,10 +30,6 @@ Available settings include:
 
     Control the resource limits on the shard recovery process.
 
-<<indices-ttl,TTL interval>>::
-
-    Control how expired documents are removed.
-
 include::indices/circuit_breaker.asciidoc[]
 
 include::indices/fielddata.asciidoc[]
diff --git a/docs/reference/modules/indices/ttl_interval.asciidoc b/docs/reference/modules/indices/ttl_interval.asciidoc
deleted file mode 100644
index 5e3069302d7..00000000000
--- a/docs/reference/modules/indices/ttl_interval.asciidoc
+++ /dev/null
@@ -1,16 +0,0 @@
-[[indices-ttl]]
-=== TTL interval
-
-Documents that have a <<mapping-ttl-field,`ttl`>> value set need to be deleted
-once they have expired.  How and how often they are deleted is controlled by
-the following dynamic cluster settings:
-
-`indices.ttl.interval`::
-
-    How often the deletion process runs.  Defaults to `60s`.
-
-`indices.ttl.bulk_size`::
-
-    The deletions are processed with a <<docs-bulk,bulk request>>.
-    The number of deletions processed can be configured with
-    this settings. Defaults to `10000`.
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
index d834f0fcf6d..28894c5c1cc 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
@@ -22,7 +22,6 @@ package org.elasticsearch.messy.tests;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -45,7 +44,6 @@ import org.elasticsearch.test.ESIntegTestCase;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 
-import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Base64;
@@ -88,8 +86,6 @@ public class SearchFieldsTests extends ESIntegTestCase {
         client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForYellowStatus().execute().actionGet();
 
         String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1")
-                // _timestamp is randomly enabled via templates but we don't want it here to test stored fields behaviour
-                .startObject("_timestamp").field("enabled", false).endObject()
                 .startObject("properties")
                 .startObject("field1").field("type", "text").field("store", true).endObject()
                 .startObject("field2").field("type", "text").field("store", false).endObject()
@@ -698,7 +694,7 @@ public class SearchFieldsTests extends ESIntegTestCase {
     public void testLoadMetadata() throws Exception {
         assertAcked(prepareCreate("test")
                 .addMapping("parent")
-                .addMapping("my-type1", "_timestamp", "enabled=true", "_ttl", "enabled=true", "_parent", "type=parent"));
+                .addMapping("my-type1", "_parent", "type=parent"));
 
         indexRandom(true,
                 client().prepareIndex("test", "my-type1", "1")
@@ -717,12 +713,6 @@ public class SearchFieldsTests extends ESIntegTestCase {
         assertThat(fields.get("field1"), nullValue());
         assertThat(fields.get("_routing").isMetadataField(), equalTo(true));
         assertThat(fields.get("_routing").getValue().toString(), equalTo("1"));
-        assertThat(fields.get("_timestamp").isMetadataField(), equalTo(true));
-        assertThat(fields.get("_timestamp").getValue().toString(), equalTo("205097"));
-        assertThat(fields.get("_ttl").isMetadataField(), equalTo(true));
-        // TODO: _ttl should return the original value, but it does not work today because
-        // it would use now() instead of the value of _timestamp to rebase
-        // assertThat(fields.get("_ttl").getValue().toString(), equalTo("10000000205097"));
         assertThat(fields.get("_parent").isMetadataField(), equalTo(true));
         assertThat(fields.get("_parent").getValue().toString(), equalTo("parent_1"));
     }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc b/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc
index 304d31978cf..688d8cbdc5b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc
@@ -255,7 +255,7 @@ Supports also regular expressions with flag X for more readability (accepts whit
 Compares two numeric values, eg:
 
 ....
-    - lt: { _ttl: 10000 }  # the `_ttl` value is less than 10,000
+    - lt: { foo: 10000 }  # the `foo` value is less than 10,000
 ....
 
 === `lte` and `gte`
@@ -263,7 +263,7 @@ Compares two numeric values, eg:
 Compares two numeric values, eg:
 
 ....
-    - lte: { _ttl: 10000 }  # the `_ttl` value is less than or equal to 10,000
+    - lte: { foo: 10000 }  # the `foo` value is less than or equal to 10,000
 ....
 
 === `length`
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_timestamp.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_timestamp.yaml
deleted file mode 100644
index 83173817b6e..00000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_timestamp.yaml
+++ /dev/null
@@ -1,80 +0,0 @@
----
-"Timestamp":
-
- - do:
-      indices.create:
-          index:    test_1
-          body:
-              mappings:
-                test:
-                  _timestamp:
-                    enabled:  1
- - do:
-      cluster.health:
-          wait_for_status: yellow
-
-# blank timestamp
- - do:
-      create:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _timestamp
-
- - is_true:    _timestamp
-
-# milliseconds since epoch
-
- - do:
-      delete:
-          index:   test_1
-          type:    test
-          id:      1
- - do:
-      create:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-          timestamp: 1372011280000
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _timestamp
-
- - match: { _timestamp: 1372011280000 }
-
-# date format
-
- - do:
-      delete:
-          index:   test_1
-          type:    test
-          id:      1
- - do:
-      create:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-          timestamp: 2013-06-23T18:14:40
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _timestamp
-
- - match: { _timestamp: 1372011280000 }
-
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/75_ttl.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/75_ttl.yaml
deleted file mode 100644
index 3f9b4763669..00000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/create/75_ttl.yaml
+++ /dev/null
@@ -1,100 +0,0 @@
----
-"TTL":
-
- - do:
-      indices.create:
-          index:    test_1
-          body:
-              mappings:
-                test:
-                  _ttl:
-                    enabled:  1
-                    default:  10s
- - do:
-      cluster.health:
-          wait_for_status: yellow
-
-# blank ttl
- - do:
-      create:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _ttl
-
- - lte:   { _ttl: 10000}
- - gt:    { _ttl: 0}
-
-# milliseconds
-
- - do:
-      delete:
-          index:   test_1
-          type:    test
-          id:      1
- - do:
-      create:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-          ttl:       100000ms
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _ttl
-
- - lte:   { _ttl: 100000}
- - gt:    { _ttl: 10000}
-
-# duration
-
- - do:
-      delete:
-          index:   test_1
-          type:    test
-          id:      1
- - do:
-      create:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-          ttl:       20s
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _ttl
-
- - lte:   { _ttl: 20000}
- - gt:    { _ttl: 10000}
-
-# with timestamp
-
- - do:
-      delete:
-          index:   test_1
-          type:    test
-          id:      1
- - do:
-      catch: /already_expired_exception/
-      create:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-          ttl:       20s
-          timestamp: 2013-06-23T18:14:40
-
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/70_timestamp.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/70_timestamp.yaml
deleted file mode 100644
index 938a358b103..00000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/70_timestamp.yaml
+++ /dev/null
@@ -1,70 +0,0 @@
----
-"Timestamp":
-
- - do:
-      indices.create:
-          index:    test_1
-          body:
-              mappings:
-                test:
-                  _timestamp:
-                    enabled:  1
- - do:
-      cluster.health:
-          wait_for_status: yellow
-
-# blank timestamp
- - do:
-      index:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _timestamp
-
- - is_true: _timestamp
-
-# milliseconds since epoch
-
- - do:
-      index:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-          timestamp: 1372011280000
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _timestamp
-
- - match: { _timestamp: 1372011280000 }
-
-# date format
-
- - do:
-      index:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-          timestamp: 2013-06-23T18:14:40
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _timestamp
-
- - match: { _timestamp: 1372011280000 }
-
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/75_ttl.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/75_ttl.yaml
deleted file mode 100644
index 2ec00eaf2f4..00000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/75_ttl.yaml
+++ /dev/null
@@ -1,85 +0,0 @@
----
-"TTL":
-
- - do:
-      indices.create:
-          index:    test_1
-          body:
-              mappings:
-                test:
-                  _ttl:
-                    enabled:  1
-                    default:  10s
- - do:
-      cluster.health:
-          wait_for_status: yellow
-
-# blank ttl
- - do:
-      index:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _ttl
-
- - lte:   { _ttl: 10000}
- - gt:    { _ttl: 0}
-
-# milliseconds
-
- - do:
-      index:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-          ttl:       100000ms
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _ttl
-
- - lte:   { _ttl: 100000}
- - gt:    { _ttl: 10000}
-
-# duration
-
- - do:
-      index:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-          ttl:       20s
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _ttl
-
- - lte:   { _ttl: 20000}
- - gt:    { _ttl: 10000}
-
-# with timestamp
-
- - do:
-      catch: /already_expired_exception/
-      index:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-          ttl:       20s
-          timestamp: 2013-06-23T18:14:40
-
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/70_timestamp.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/70_timestamp.yaml
deleted file mode 100644
index 864c219c448..00000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/70_timestamp.yaml
+++ /dev/null
@@ -1,76 +0,0 @@
----
-"Timestamp":
-
- - do:
-      indices.create:
-          index:    test_1
-          body:
-              mappings:
-                test:
-                  _timestamp:
-                    enabled:  1
- - do:
-      cluster.health:
-          wait_for_status: yellow
-
-# blank timestamp
- - do:
-      update:
-          index:     test_1
-          type:      test
-          id:        1
-          body:
-            doc:        { foo: baz }
-            upsert:     { foo: bar }
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _timestamp
-
- - is_true: _timestamp
-
-# milliseconds since epoch
-
- - do:
-      update:
-          index:     test_1
-          type:      test
-          id:        1
-          body:
-            doc:        { foo: baz }
-            upsert:     { foo: bar }
-          timestamp: 1372011280000
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _timestamp
-
- - match: { _timestamp: 1372011280000 }
-
-# date format
-
- - do:
-      update:
-          index:     test_1
-          type:      test
-          id:        1
-          body:
-            doc:        { foo: baz }
-            upsert:     { foo: bar }
-          timestamp: 2013-06-23T18:14:40
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _timestamp
-
- - match: { _timestamp: 1372011280000 }
-
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/75_ttl.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/75_ttl.yaml
deleted file mode 100644
index 09179ebd5ac..00000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/75_ttl.yaml
+++ /dev/null
@@ -1,92 +0,0 @@
----
-"TTL":
-
- - do:
-      indices.create:
-          index:    test_1
-          body:
-              mappings:
-                test:
-                  _ttl:
-                    enabled:  1
-                    default:  10s
- - do:
-      cluster.health:
-          wait_for_status: yellow
-
-# blank ttl
- - do:
-      update:
-          index:     test_1
-          type:      test
-          id:        1
-          body:
-            doc:        { foo: baz }
-            upsert:     { foo: bar }
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _ttl
-
- - lte:   { _ttl: 10000}
- - gt:    { _ttl: 0}
-
-# milliseconds
-
- - do:
-      update:
-          index:     test_1
-          type:      test
-          id:        1
-          body:
-            doc:        { foo: baz }
-            upsert:     { foo: bar }
-          ttl:       100000ms
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _ttl
-
- - lte:   { _ttl: 100000}
- - gt:    { _ttl: 10000}
-
-# seconds
-
- - do:
-      update:
-          index:     test_1
-          type:      test
-          id:        1
-          body:
-            doc:        { foo: baz }
-            upsert:     { foo: bar }
-            detect_noop:  false
-          ttl:       20s
-
- - do:
-      get:
-          index:   test_1
-          type:    test
-          id:      1
-          fields:  _ttl
-
- - lte:   { _ttl: 20000}
- - gt:    { _ttl: 10000}
-
-# with timestamp
-
- - do:
-      catch: /already_expired_exception/
-      index:
-          index:     test_1
-          type:      test
-          id:        1
-          body:      { foo: bar }
-          ttl:       20s
-          timestamp: 2013-06-23T18:14:40
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml
index 985048ad44d..0194453a99c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml
@@ -12,11 +12,6 @@
             mappings:
               test:
                 _parent: { type: "foo" }
-                _timestamp:
-                  enabled:  1
-                _ttl:
-                  enabled:  1
-                  default:  10s
 
   - do:
       cluster.health:
@@ -28,15 +23,13 @@
           type:   test
           id:     1
           parent: 5
-          fields: [ _parent, _routing, _timestamp, _ttl ]
+          fields: [ _parent, _routing ]
           body:
             doc:    { foo: baz }
             upsert: { foo: bar }
 
   - match:   { get._parent:  "5" }
   - match:   { get._routing: "5" }
-  - is_true:   get._timestamp
-  - is_true:   get._ttl
 
   - do:
       get:
@@ -44,6 +37,6 @@
           type:   test
           id:     1
           parent: 5
-          fields: [ _parent, _routing, _timestamp, _ttl ]
+          fields: [ _parent, _routing ]
 
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index 095b1339800..8df2fd911c4 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -383,13 +383,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
 
             XContentBuilder mappings = null;
             if (frequently() && randomDynamicTemplates()) {
-                mappings = XContentFactory.jsonBuilder().startObject().startObject("_default_");
-                if (randomBoolean()) {
-                    mappings.startObject(TimestampFieldMapper.NAME)
-                            .field("enabled", randomBoolean());
-                    mappings.endObject();
-                }
-                mappings.endObject().endObject();
+                mappings = XContentFactory.jsonBuilder().startObject().startObject("_default_").endObject().endObject();
             }
 
             for (String setting : randomSettingsBuilder.internalMap().keySet()) {

From 7d63f4b8db89eeae5d65cab21708a250b4d14958 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Wed, 22 Jun 2016 09:34:49 +0200
Subject: [PATCH 1022/1311] Fix doc build.

---
 docs/reference/mapping/fields.asciidoc  | 4 ----
 docs/reference/modules/indices.asciidoc | 2 --
 2 files changed, 6 deletions(-)

diff --git a/docs/reference/mapping/fields.asciidoc b/docs/reference/mapping/fields.asciidoc
index 1cd886dde0e..8bd86cd0085 100644
--- a/docs/reference/mapping/fields.asciidoc
+++ b/docs/reference/mapping/fields.asciidoc
@@ -84,10 +84,6 @@ include::fields/routing-field.asciidoc[]
 
 include::fields/source-field.asciidoc[]
 
-include::fields/timestamp-field.asciidoc[]
-
-include::fields/ttl-field.asciidoc[]
-
 include::fields/type-field.asciidoc[]
 
 include::fields/uid-field.asciidoc[]
diff --git a/docs/reference/modules/indices.asciidoc b/docs/reference/modules/indices.asciidoc
index 34a1ccdf8fa..5f7bb7b9aba 100644
--- a/docs/reference/modules/indices.asciidoc
+++ b/docs/reference/modules/indices.asciidoc
@@ -42,5 +42,3 @@ include::indices/request_cache.asciidoc[]
 
 include::indices/recovery.asciidoc[]
 
-include::indices/ttl_interval.asciidoc[]
-

From c7bd1a80aff32535772d30700e6d2db9439d34e1 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Wed, 22 Jun 2016 12:39:36 +0200
Subject: [PATCH 1023/1311] Changed path.script to path.scripts in docs

---
 docs/reference/modules/scripting/using.asciidoc | 2 +-
 docs/reference/setup/install/deb.asciidoc       | 2 +-
 docs/reference/setup/install/rpm.asciidoc       | 2 +-
 docs/reference/setup/install/windows.asciidoc   | 2 +-
 docs/reference/setup/install/zip-targz.asciidoc | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/reference/modules/scripting/using.asciidoc b/docs/reference/modules/scripting/using.asciidoc
index 90b8a12c3bf..e50c737fd95 100644
--- a/docs/reference/modules/scripting/using.asciidoc
+++ b/docs/reference/modules/scripting/using.asciidoc
@@ -114,7 +114,7 @@ files stored on every node in the cluster.  File scripts must be saved in the
 `scripts` directory whose default location depends on whether you use  the
 <<zip-targz-layout,`zip`/`tar.gz`>> (`$ES_HOME/config/scripts/`),
 <<rpm-layout,RPM>>, or <<deb-layout,Debian>> package.  The default may be
-changed with the `path.script` setting.
+changed with the `path.scripts` setting.
 
 The languages which are assumed to be safe by default are: `painless`,
 `expression`, and `mustache` (used for search and query templates).
diff --git a/docs/reference/setup/install/deb.asciidoc b/docs/reference/setup/install/deb.asciidoc
index e02d0f84a1d..a9bf4170791 100644
--- a/docs/reference/setup/install/deb.asciidoc
+++ b/docs/reference/setup/install/deb.asciidoc
@@ -183,7 +183,7 @@ locations for a Debian-based system:
 | script
   | Location of script files.
   | /etc/elasticsearch/scripts
-  | path.script
+  | path.scripts
 
 |=======================================================================
 
diff --git a/docs/reference/setup/install/rpm.asciidoc b/docs/reference/setup/install/rpm.asciidoc
index d7f216670dc..08dfc253f0d 100644
--- a/docs/reference/setup/install/rpm.asciidoc
+++ b/docs/reference/setup/install/rpm.asciidoc
@@ -169,7 +169,7 @@ locations for an RPM-based system:
 | script
   | Location of script files.
   | /etc/elasticsearch/scripts
-  | path.script
+  | path.scripts
 
 |=======================================================================
 
diff --git a/docs/reference/setup/install/windows.asciidoc b/docs/reference/setup/install/windows.asciidoc
index c18fc0669d3..e66c176470f 100644
--- a/docs/reference/setup/install/windows.asciidoc
+++ b/docs/reference/setup/install/windows.asciidoc
@@ -243,7 +243,7 @@ directory so that you do not delete important data later on.
 | script
   | Location of script files.
   | %ES_HOME%\scripts
-  | path.script
+  | path.scripts
 
 |=======================================================================
 
diff --git a/docs/reference/setup/install/zip-targz.asciidoc b/docs/reference/setup/install/zip-targz.asciidoc
index 16a79d000e7..5f85447bdcd 100644
--- a/docs/reference/setup/install/zip-targz.asciidoc
+++ b/docs/reference/setup/install/zip-targz.asciidoc
@@ -155,7 +155,7 @@ directory so that you do not delete important data later on.
 | script
   | Location of script files.
   | $ES_HOME/scripts
-  | path.script
+  | path.scripts
 
 |=======================================================================
 

From b0da4719aa2deb46fac6846366afd52a36a1789c Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 22 Jun 2016 07:37:58 -0400
Subject: [PATCH 1024/1311] Add missing field to PersistedTaskInfo

---
 .../org/elasticsearch/tasks/PersistedTaskInfo.java    | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/tasks/PersistedTaskInfo.java b/core/src/main/java/org/elasticsearch/tasks/PersistedTaskInfo.java
index 06ead633641..4e92cf2e874 100644
--- a/core/src/main/java/org/elasticsearch/tasks/PersistedTaskInfo.java
+++ b/core/src/main/java/org/elasticsearch/tasks/PersistedTaskInfo.java
@@ -160,6 +160,7 @@ public final class PersistedTaskInfo implements Writeable, ToXContent {
     }
 
     public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field("completed", completed);
         builder.field("task", task);
         if (error != null) {
             XContentHelper.writeRawField("error", error, builder, params);
@@ -171,8 +172,16 @@ public final class PersistedTaskInfo implements Writeable, ToXContent {
     }
 
     public static final ConstructingObjectParser<PersistedTaskInfo, ParseFieldMatcherSupplier> PARSER = new ConstructingObjectParser<>(
-            "persisted_task_info", a -> new PersistedTaskInfo(true, (TaskInfo) a[0], (BytesReference) a[1], (BytesReference) a[2]));
+            "persisted_task_info", a -> {
+                int i = 0;
+                boolean completed = (boolean) a[i++];
+                TaskInfo task = (TaskInfo) a[i++];
+                BytesReference error = (BytesReference) a[i++];
+                BytesReference response = (BytesReference) a[i++];
+                return new PersistedTaskInfo(completed, task, error, response);
+            });
     static {
+        PARSER.declareBoolean(constructorArg(), new ParseField("completed"));
         PARSER.declareObject(constructorArg(), TaskInfo.PARSER, new ParseField("task"));
         PARSER.declareRawObject(optionalConstructorArg(), new ParseField("error"));
         PARSER.declareRawObject(optionalConstructorArg(), new ParseField("response"));

From f8c55a5e7bc1e12f78a4a270f92a74ff60393885 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Wed, 22 Jun 2016 08:40:20 -0400
Subject: [PATCH 1025/1311] painless: fix disabled loop counter

---
 .../elasticsearch/painless/MethodWriter.java  | 19 ++++++++-------
 .../org/elasticsearch/painless/node/SDo.java  |  4 +++-
 .../org/elasticsearch/painless/node/SFor.java |  8 +++++--
 .../elasticsearch/painless/node/SWhile.java   |  8 +++++--
 .../painless/BasicStatementTests.java         | 23 +++++++++++++++++++
 5 files changed, 47 insertions(+), 15 deletions(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
index 97686655a97..43fd54c51a4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java
@@ -118,17 +118,16 @@ public final class MethodWriter extends GeneratorAdapter {
     }
 
     public void writeLoopCounter(int slot, int count, Location location) {
-        if (slot > -1) {
-            writeDebugInfo(location);
-            final Label end = new Label();
+        assert slot != -1;
+        writeDebugInfo(location);
+        final Label end = new Label();
 
-            iinc(slot, -count);
-            visitVarInsn(Opcodes.ILOAD, slot);
-            push(0);
-            ifICmp(GeneratorAdapter.GT, end);
-            throwException(PAINLESS_ERROR_TYPE, "The maximum number of statements that can be executed in a loop has been reached.");
-            mark(end);
-        }
+        iinc(slot, -count);
+        visitVarInsn(Opcodes.ILOAD, slot);
+        push(0);
+        ifICmp(GeneratorAdapter.GT, end);
+        throwException(PAINLESS_ERROR_TYPE, "The maximum number of statements that can be executed in a loop has been reached.");
+        mark(end);
     }
 
     public void writeCast(final Cast cast) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
index e214703daf4..62572e54610 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDo.java
@@ -113,7 +113,9 @@ public final class SDo extends AStatement {
         condition.fals = end;
         condition.write(writer, globals);
 
-        writer.writeLoopCounter(loopCounter.getSlot(), Math.max(1, block.statementCount), location);
+        if (loopCounter != null) {
+            writer.writeLoopCounter(loopCounter.getSlot(), Math.max(1, block.statementCount), location);
+        }
 
         writer.goTo(start);
         writer.mark(end);
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
index 06ad19d204d..c324682040b 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFor.java
@@ -176,10 +176,14 @@ public final class SFor extends AStatement {
                 ++statementCount;
             }
 
-            writer.writeLoopCounter(loopCounter.getSlot(), statementCount, location);
+            if (loopCounter != null) {
+                writer.writeLoopCounter(loopCounter.getSlot(), statementCount, location);
+            }
             block.write(writer, globals);
         } else {
-            writer.writeLoopCounter(loopCounter.getSlot(), 1, location);
+            if (loopCounter != null) {
+                writer.writeLoopCounter(loopCounter.getSlot(), 1, location);
+            }
         }
 
         if (afterthought != null) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
index dff8ae2592f..bd6740dd9b8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SWhile.java
@@ -113,13 +113,17 @@ public final class SWhile extends AStatement {
         condition.write(writer, globals);
 
         if (block != null) {
-            writer.writeLoopCounter(loopCounter.getSlot(), Math.max(1, block.statementCount), location);
+            if (loopCounter != null) {
+                writer.writeLoopCounter(loopCounter.getSlot(), Math.max(1, block.statementCount), location);
+            }
 
             block.continu = begin;
             block.brake = end;
             block.write(writer, globals);
         } else {
-            writer.writeLoopCounter(loopCounter.getSlot(), 1, location);
+            if (loopCounter != null) {
+                writer.writeLoopCounter(loopCounter.getSlot(), 1, location);
+            }
         }
 
         if (block == null || !block.allEscape) {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
index 98921f88c38..01f3ee42ae6 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
@@ -1,5 +1,7 @@
 package org.elasticsearch.painless;
 
+import java.util.Collections;
+
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -223,4 +225,25 @@ public class BasicStatementTests extends ScriptTestCase {
         assertEquals(10, exec("def i = 1; if (i == 1) {i = 2; return 10}"));
         assertEquals(10, exec("def i = 1; if (i == 1) {i = 2; return 10} else {return 12}"));
     }
+    
+    public void testArrayLoopWithoutCounter() {
+       assertEquals(6L, exec("long sum = 0; long[] array = new long[] { 1, 2, 3 };" + 
+                              "for (int i = 0; i < array.length; i++) { sum += array[i] } return sum", 
+                              Collections.emptyMap(),
+                              Collections.singletonMap(CompilerSettings.MAX_LOOP_COUNTER, "0"),
+                              null
+       ));
+       assertEquals(6L, exec("long sum = 0; long[] array = new long[] { 1, 2, 3 };" + 
+                             "int i = 0; while (i < array.length) { sum += array[i++] } return sum", 
+                             Collections.emptyMap(),
+                             Collections.singletonMap(CompilerSettings.MAX_LOOP_COUNTER, "0"),
+                             null
+       ));
+       assertEquals(6L, exec("long sum = 0; long[] array = new long[] { 1, 2, 3 };" + 
+                             "int i = 0; do { sum += array[i++] } while (i < array.length); return sum", 
+                             Collections.emptyMap(),
+                             Collections.singletonMap(CompilerSettings.MAX_LOOP_COUNTER, "0"),
+                             null
+       ));
+    }
 }

From 1bd0482393b5000a6d8f13703c8d8aae46302cf0 Mon Sep 17 00:00:00 2001
From: Mike McCandless <mikemccand@apache.org>
Date: Wed, 22 Jun 2016 09:52:54 -0400
Subject: [PATCH 1026/1311] don't include indexing buffer in cluster stats;
 randomize indexing buffer in NodeInfoStreamingTests; add @Nullable annotation

---
 .../action/admin/cluster/node/info/NodeInfo.java       |  2 +-
 .../cluster/stats/TransportClusterStatsAction.java     |  2 +-
 .../nodesinfo/NodeInfoStreamingTests.java              | 10 +++++++++-
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
index 312b4a5a05c..f1f3f20a492 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
@@ -88,7 +88,7 @@ public class NodeInfo extends BaseNodeResponse {
     public NodeInfo(Version version, Build build, DiscoveryNode node, @Nullable Map<String, String> serviceAttributes, @Nullable Settings settings,
                     @Nullable OsInfo os, @Nullable ProcessInfo process, @Nullable JvmInfo jvm, @Nullable ThreadPoolInfo threadPool,
                     @Nullable TransportInfo transport, @Nullable HttpInfo http, @Nullable PluginsAndModules plugins, @Nullable IngestInfo ingest,
-                    ByteSizeValue totalIndexingBuffer) {
+                    @Nullable ByteSizeValue totalIndexingBuffer) {
         super(node);
         this.version = version;
         this.build = build;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
index c4c6f3e6ecc..73a5e978fb7 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java
@@ -92,7 +92,7 @@ public class TransportClusterStatsAction extends TransportNodesAction<ClusterSta
 
     @Override
     protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeRequest) {
-        NodeInfo nodeInfo = nodeService.info(false, true, false, true, false, true, false, true, false, true);
+        NodeInfo nodeInfo = nodeService.info(false, true, false, true, false, true, false, true, false, false);
         NodeStats nodeStats = nodeService.stats(CommonStatsFlags.NONE, false, true, true, false, true, false, false, false, false, false, false);
         List<ShardStats> shardsStats = new ArrayList<>();
         for (IndexService indexService : indicesService) {
diff --git a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
index b84ffca106c..191eee160ff 100644
--- a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
+++ b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
@@ -30,6 +30,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.http.HttpInfo;
@@ -141,6 +142,13 @@ public class NodeInfoStreamingTests extends ESTestCase {
         plugins.addModule(DummyPluginInfo.INSTANCE);
         plugins.addPlugin(DummyPluginInfo.INSTANCE);
         IngestInfo ingestInfo = new IngestInfo(Collections.emptyList());
-        return new NodeInfo(VersionUtils.randomVersion(random()), build, node, serviceAttributes, settings, osInfo, process, jvm, threadPoolInfo, transport, htttpInfo, plugins, ingestInfo, null);
+        ByteSizeValue indexingBuffer;
+        if (random().nextBoolean()) {
+            indexingBuffer = null;
+        } else {
+            // pick a random long that sometimes exceeds an int:
+            indexingBuffer = new ByteSizeValue(random().nextLong() & ((1L<<40)-1));
+        }
+        return new NodeInfo(VersionUtils.randomVersion(random()), build, node, serviceAttributes, settings, osInfo, process, jvm, threadPoolInfo, transport, htttpInfo, plugins, ingestInfo, indexingBuffer);
     }
 }

From cbc7ff3f9cf4bd5fe7182fdbbdea5f1432f041dc Mon Sep 17 00:00:00 2001
From: Mike McCandless <mikemccand@apache.org>
Date: Wed, 22 Jun 2016 10:04:33 -0400
Subject: [PATCH 1027/1311] NodeService.indicesService is never null

---
 .../main/java/org/elasticsearch/node/service/NodeService.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/node/service/NodeService.java b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
index ce242fc66a1..ed03dc9375c 100644
--- a/core/src/main/java/org/elasticsearch/node/service/NodeService.java
+++ b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
@@ -133,7 +133,7 @@ public class NodeService extends AbstractComponent implements Closeable {
                 httpServer == null ? null : httpServer.info(),
                 pluginService == null ? null : pluginService.info(),
                 ingestService == null ? null : ingestService.info(),
-                indicesService == null ? null : indicesService.getTotalIndexingBufferBytes()
+                indicesService.getTotalIndexingBufferBytes()
         );
     }
 

From 2f46f53dc8feb78412e6d648751ffe97b1e35119 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Tue, 21 Jun 2016 11:27:27 +0200
Subject: [PATCH 1028/1311] Rename `fields` to `stored_fields` and add
 `docvalue_fields`

`stored_fields` parameter will no longer try to retrieve fields from the _source but will only return stored fields.
`fields` will throw an exception if the user uses it.
Add `docvalue_fields` as an adjunct to `fielddata_fields` which is deprecated. `docvalue_fields` will try to load the value from the docvalue and fallback to fielddata cache if docvalues are not enabled on that field.

Closes #18943
---
 .../action/search/SearchRequestBuilder.java   |  41 +++-
 .../index/query/InnerHitBuilder.java          | 228 ++++++++++++------
 .../rest/action/search/RestSearchAction.java  |  31 ++-
 .../elasticsearch/search/SearchService.java   |   8 +-
 .../tophits/TopHitsAggregationBuilder.java    |  12 +-
 .../search/builder/SearchSourceBuilder.java   | 174 ++++++-------
 .../index/IndexWithShadowReplicasIT.java      |   2 +-
 .../TokenCountFieldMapperIntegrationIT.java   |   4 +-
 .../mapper/geo/GeoPointFieldMapperTests.java  |   8 +-
 .../index/query/InnerHitBuilderTests.java     |  14 +-
 .../index/store/ExceptionRetryIT.java         |   2 +-
 .../template/SimpleIndexTemplateIT.java       |   4 +-
 .../elasticsearch/recovery/RelocationIT.java  |   2 +-
 .../metrics/AbstractGeoTestCase.java          |   2 +-
 .../builder/SearchSourceBuilderTests.java     |   4 +-
 .../search/child/ChildQuerySearchIT.java      |   8 +-
 .../elasticsearch/search/geo/GeoFilterIT.java |   4 +-
 .../search/innerhits/InnerHitsIT.java         |   4 +-
 .../search/source/SourceFetchingIT.java       |   4 +-
 .../update/TimestampTTLBWIT.java              |   2 +-
 docs/plugins/mapper-attachments.asciidoc      |   4 +-
 .../metrics/tophits-aggregation.asciidoc      |   2 +-
 docs/reference/mapping/params/store.asciidoc  |   2 +-
 .../migration/migrate_5_0/search.asciidoc     |  10 +-
 .../modules/scripting/painless.asciidoc       |   2 +-
 docs/reference/search/request-body.asciidoc   |   4 +-
 .../search/request/docvalue-fields.asciidoc   |  23 ++
 .../search/request/fielddata-fields.asciidoc  |  23 --
 .../search/request/highlighting.asciidoc      |   2 +-
 .../search/request/inner-hits.asciidoc        |   2 +-
 ...fields.asciidoc => stored-fields.asciidoc} |   6 +-
 docs/reference/search/uri-request.asciidoc    |   2 +-
 .../messy/tests/GeoDistanceTests.java         |  16 +-
 .../messy/tests/SearchFieldsTests.java        |  66 ++---
 .../reindex/AbstractBulkByScrollRequest.java  |   4 +-
 .../test/delete_by_query/20_validation.yaml   |   4 +-
 .../test/reindex/20_validation.yaml           |   4 +-
 .../test/update_by_query/20_validation.yaml   |   4 +-
 .../test/mapper_attachments/30_mapping.yaml   |   2 +-
 .../test/mapper_attachments/40_highlight.yaml |   2 +-
 .../50_files_supported.yaml                   |   4 +-
 .../size/SizeFieldMapperUpgradeTests.java     |   4 +-
 .../resources/rest-api-spec/api/search.json   |  10 +-
 .../test/search/10_source_filtering.yaml      |   9 +-
 .../rest-api-spec/test/search/issue4895.yaml  |   2 +-
 45 files changed, 453 insertions(+), 318 deletions(-)
 create mode 100644 docs/reference/search/request/docvalue-fields.asciidoc
 delete mode 100644 docs/reference/search/request/fielddata-fields.asciidoc
 rename docs/reference/search/request/{fields.asciidoc => stored-fields.asciidoc} (91%)

diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
index 47be65521d6..bfdea9b19a9 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
@@ -253,8 +253,8 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
     /**
      * Sets no fields to be loaded, resulting in only id and type to be returned per field.
      */
-    public SearchRequestBuilder setNoFields() {
-        sourceBuilder().noFields();
+    public SearchRequestBuilder setNoStoredFields() {
+        sourceBuilder().noStoredFields();
         return this;
     }
 
@@ -290,13 +290,23 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
         return this;
     }
 
+    /**
+     * Adds a docvalue based field to load and return. The field does not have to be stored,
+     * but its recommended to use non analyzed or numeric fields.
+     *
+     * @param name The field to get from the docvalue
+     */
+    public SearchRequestBuilder addDocValueField(String name) {
+        sourceBuilder().docValueField(name);
+        return this;
+    }
 
     /**
-     * Adds a field to load and return (note, it must be stored) as part of the search request.
+     * Adds a stored field to load and return (note, it must be stored) as part of the search request.
      * If none are specified, the source of the document will be return.
      */
-    public SearchRequestBuilder addField(String field) {
-        sourceBuilder().field(field);
+    public SearchRequestBuilder addStoredField(String field) {
+        sourceBuilder().storedField(field);
         return this;
     }
 
@@ -305,12 +315,15 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
      * but its recommended to use non analyzed or numeric fields.
      *
      * @param name The field to get from the field data cache
+     * @deprecated Use {@link SearchRequestBuilder#addDocValueField(String)} instead.
      */
+    @Deprecated
     public SearchRequestBuilder addFieldDataField(String name) {
-        sourceBuilder().fieldDataField(name);
+        sourceBuilder().docValueField(name);
         return this;
     }
 
+
     /**
      * Adds a script based field to load and return. The field does not have to be stored,
      * but its recommended to use non analyzed or numeric fields.
@@ -367,12 +380,24 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
         return this;
     }
 
+    /**
+     * Sets the stored fields to load and return as part of the search request. If none
+     * are specified, the source of the document will be returned.
+     *
+     * @deprecated Use {@link SearchRequestBuilder#storedFields(String...)} instead.
+     */
+    @Deprecated
+    public SearchRequestBuilder fields(String... fields) {
+        sourceBuilder().storedFields(Arrays.asList(fields));
+        return this;
+    }
+
     /**
      * Sets the fields to load and return as part of the search request. If none
      * are specified, the source of the document will be returned.
      */
-    public SearchRequestBuilder fields(String... fields) {
-        sourceBuilder().fields(Arrays.asList(fields));
+    public SearchRequestBuilder storedFields(String... fields) {
+        sourceBuilder().storedFields(Arrays.asList(fields));
         return this;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
index a017159b451..43e2f12055c 100644
--- a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
@@ -71,8 +71,14 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         PARSER.declareBoolean(InnerHitBuilder::setExplain, SearchSourceBuilder.EXPLAIN_FIELD);
         PARSER.declareBoolean(InnerHitBuilder::setVersion, SearchSourceBuilder.VERSION_FIELD);
         PARSER.declareBoolean(InnerHitBuilder::setTrackScores, SearchSourceBuilder.TRACK_SCORES_FIELD);
-        PARSER.declareStringArray(InnerHitBuilder::setFieldNames, SearchSourceBuilder.FIELDS_FIELD);
-        PARSER.declareStringArray(InnerHitBuilder::setFieldDataFields, SearchSourceBuilder.FIELDDATA_FIELDS_FIELD);
+        PARSER.declareStringArray(InnerHitBuilder::setStoredFieldNames, SearchSourceBuilder.STORED_FIELDS_FIELD);
+        PARSER.declareField((p, i, c) -> {
+            throw new ParsingException(p.getTokenLocation(), "The field [" +
+                SearchSourceBuilder.FIELDS_FIELD + "] is not longer supported, please use [" +
+                SearchSourceBuilder.STORED_FIELDS_FIELD + "] to retrieve stored fields or _source filtering " +
+                "if the field is not stored");
+        }, SearchSourceBuilder.FIELDS_FIELD, ObjectParser.ValueType.STRING_ARRAY);
+        PARSER.declareStringArray(InnerHitBuilder::setDocValueFields, SearchSourceBuilder.DOCVALUE_FIELDS_FIELD);
         PARSER.declareField((p, i, c) -> {
             try {
                 Set<ScriptField> scriptFields = new HashSet<>();
@@ -131,10 +137,10 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
     private boolean version;
     private boolean trackScores;
 
-    private List<String> fieldNames;
+    private List<String> storedFieldNames;
     private QueryBuilder query = DEFAULT_INNER_HIT_QUERY;
     private List<SortBuilder<?>> sorts;
-    private List<String> fieldDataFields;
+    private List<String> docValueFields;
     private Set<ScriptField> scriptFields;
     private HighlightBuilder highlightBuilder;
     private FetchSourceContext fetchSourceContext;
@@ -143,46 +149,6 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
     public InnerHitBuilder() {
     }
 
-    /**
-     * Read from a stream.
-     */
-    public InnerHitBuilder(StreamInput in) throws IOException {
-        name = in.readOptionalString();
-        nestedPath = in.readOptionalString();
-        parentChildType = in.readOptionalString();
-        from = in.readVInt();
-        size = in.readVInt();
-        explain = in.readBoolean();
-        version = in.readBoolean();
-        trackScores = in.readBoolean();
-        fieldNames = (List<String>) in.readGenericValue();
-        fieldDataFields = (List<String>) in.readGenericValue();
-        if (in.readBoolean()) {
-            int size = in.readVInt();
-            scriptFields = new HashSet<>(size);
-            for (int i = 0; i < size; i++) {
-                scriptFields.add(new ScriptField(in));
-            }
-        }
-        fetchSourceContext = in.readOptionalStreamable(FetchSourceContext::new);
-        if (in.readBoolean()) {
-            int size = in.readVInt();
-            sorts = new ArrayList<>(size);
-            for (int i = 0; i < size; i++) {
-                sorts.add(in.readNamedWriteable(SortBuilder.class));
-            }
-        }
-        highlightBuilder = in.readOptionalWriteable(HighlightBuilder::new);
-        query = in.readNamedWriteable(QueryBuilder.class);
-        if (in.readBoolean()) {
-            int size = in.readVInt();
-            childInnerHits = new HashMap<>(size);
-            for (int i = 0; i < size; i++) {
-                childInnerHits.put(in.readString(), new InnerHitBuilder(in));
-            }
-        }
-    }
-
     private InnerHitBuilder(InnerHitBuilder other) {
         name = other.name;
         from = other.from;
@@ -190,11 +156,11 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         explain = other.explain;
         version = other.version;
         trackScores = other.trackScores;
-        if (other.fieldNames != null) {
-            fieldNames = new ArrayList<>(other.fieldNames);
+        if (other.storedFieldNames != null) {
+            storedFieldNames = new ArrayList<>(other.storedFieldNames);
         }
-        if (other.fieldDataFields != null) {
-            fieldDataFields = new ArrayList<>(other.fieldDataFields);
+        if (other.docValueFields != null) {
+            docValueFields = new ArrayList<>(other.docValueFields);
         }
         if (other.scriptFields != null) {
             scriptFields = new HashSet<>(other.scriptFields);
@@ -232,6 +198,46 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         }
     }
 
+    /**
+     * Read from a stream.
+     */
+    public InnerHitBuilder(StreamInput in) throws IOException {
+        name = in.readOptionalString();
+        nestedPath = in.readOptionalString();
+        parentChildType = in.readOptionalString();
+        from = in.readVInt();
+        size = in.readVInt();
+        explain = in.readBoolean();
+        version = in.readBoolean();
+        trackScores = in.readBoolean();
+        storedFieldNames = (List<String>) in.readGenericValue();
+        docValueFields = (List<String>) in.readGenericValue();
+        if (in.readBoolean()) {
+            int size = in.readVInt();
+            scriptFields = new HashSet<>(size);
+            for (int i = 0; i < size; i++) {
+                scriptFields.add(new ScriptField(in));
+            }
+        }
+        fetchSourceContext = in.readOptionalStreamable(FetchSourceContext::new);
+        if (in.readBoolean()) {
+            int size = in.readVInt();
+            sorts = new ArrayList<>(size);
+            for (int i = 0; i < size; i++) {
+                sorts.add(in.readNamedWriteable(SortBuilder.class));
+            }
+        }
+        highlightBuilder = in.readOptionalWriteable(HighlightBuilder::new);
+        query = in.readNamedWriteable(QueryBuilder.class);
+        if (in.readBoolean()) {
+            int size = in.readVInt();
+            childInnerHits = new HashMap<>(size);
+            for (int i = 0; i < size; i++) {
+                childInnerHits.put(in.readString(), new InnerHitBuilder(in));
+            }
+        }
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(name);
@@ -242,8 +248,8 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         out.writeBoolean(explain);
         out.writeBoolean(version);
         out.writeBoolean(trackScores);
-        out.writeGenericValue(fieldNames);
-        out.writeGenericValue(fieldDataFields);
+        out.writeGenericValue(storedFieldNames);
+        out.writeGenericValue(docValueFields);
         boolean hasScriptFields = scriptFields != null;
         out.writeBoolean(hasScriptFields);
         if (hasScriptFields) {
@@ -334,29 +340,103 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         return this;
     }
 
+    /**
+     * Gets the stored fields to load and return.
+     *
+     * @deprecated Use {@link InnerHitBuilder#getStoredFieldNames()} instead.
+     */
+    @Deprecated
     public List<String> getFieldNames() {
-        return fieldNames;
+        return storedFieldNames;
     }
 
+    /**
+     * Sets the stored fields to load and return. If none
+     * are specified, the source of the document will be returned.
+     *
+     * @deprecated Use {@link InnerHitBuilder#setStoredFieldNames(List)} instead.
+     */
+    @Deprecated
     public InnerHitBuilder setFieldNames(List<String> fieldNames) {
-        this.fieldNames = fieldNames;
+        this.storedFieldNames = fieldNames;
         return this;
     }
 
+
+    /**
+     * Gets the stored fields to load and return.
+     */
+    public List<String> getStoredFieldNames() {
+        return storedFieldNames;
+    }
+
+    /**
+     * Sets the stored fields to load and return. If none
+     * are specified, the source of the document will be returned.
+     */
+    public InnerHitBuilder setStoredFieldNames(List<String> fieldNames) {
+        this.storedFieldNames = fieldNames;
+        return this;
+    }
+
+    /**
+     * Gets the docvalue fields.
+     *
+     * @deprecated Use {@link InnerHitBuilder#getDocValueFields()} instead.
+     */
+    @Deprecated
     public List<String> getFieldDataFields() {
-        return fieldDataFields;
+        return docValueFields;
     }
 
+    /**
+     * Sets the stored fields to load from the docvalue and return.
+     *
+     * @deprecated Use {@link InnerHitBuilder#setDocValueFields(List)} instead.
+     */
+    @Deprecated
     public InnerHitBuilder setFieldDataFields(List<String> fieldDataFields) {
-        this.fieldDataFields = fieldDataFields;
+        this.docValueFields = fieldDataFields;
         return this;
     }
 
+    /**
+     * Adds a field to load from the docvalue and return.
+     *
+     * @deprecated Use {@link InnerHitBuilder#addDocValueField(String)} instead.
+     */
+    @Deprecated
     public InnerHitBuilder addFieldDataField(String field) {
-        if (fieldDataFields == null) {
-            fieldDataFields = new ArrayList<>();
+        if (docValueFields == null) {
+            docValueFields = new ArrayList<>();
         }
-        fieldDataFields.add(field);
+        docValueFields.add(field);
+        return this;
+    }
+
+    /**
+     * Gets the docvalue fields.
+     */
+    public List<String> getDocValueFields() {
+        return docValueFields;
+    }
+
+    /**
+     * Sets the stored fields to load from the docvalue and return.
+     */
+    public InnerHitBuilder setDocValueFields(List<String> docValueFields) {
+        this.docValueFields = docValueFields;
+        return this;
+    }
+
+    /**
+     * Adds a field to load from the docvalue and return.
+     */
+    public InnerHitBuilder addDocValueField(String field) {
+        if (docValueFields == null) {
+            docValueFields = new ArrayList<>();
+        }
+        docValueFields.add(field);
         return this;
     }
 
@@ -484,19 +564,19 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         innerHitsContext.explain(explain);
         innerHitsContext.version(version);
         innerHitsContext.trackScores(trackScores);
-        if (fieldNames != null) {
-            if (fieldNames.isEmpty()) {
+        if (storedFieldNames != null) {
+            if (storedFieldNames.isEmpty()) {
                 innerHitsContext.emptyFieldNames();
             } else {
-                for (String fieldName : fieldNames) {
+                for (String fieldName : storedFieldNames) {
                     innerHitsContext.fieldNames().add(fieldName);
                 }
             }
         }
-        if (fieldDataFields != null) {
+        if (docValueFields != null) {
             FieldDataFieldsContext fieldDataFieldsContext = innerHitsContext
                     .getFetchSubPhaseContext(FieldDataFieldsFetchSubPhase.CONTEXT_FACTORY);
-            for (String field : fieldDataFields) {
+            for (String field : docValueFields) {
                 fieldDataFieldsContext.add(new FieldDataFieldsContext.FieldDataField(field));
             }
             fieldDataFieldsContext.setHitExecutionNeeded(true);
@@ -553,20 +633,20 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         if (fetchSourceContext != null) {
             builder.field(SearchSourceBuilder._SOURCE_FIELD.getPreferredName(), fetchSourceContext, params);
         }
-        if (fieldNames != null) {
-            if (fieldNames.size() == 1) {
-                builder.field(SearchSourceBuilder.FIELDS_FIELD.getPreferredName(), fieldNames.get(0));
+        if (storedFieldNames != null) {
+            if (storedFieldNames.size() == 1) {
+                builder.field(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), storedFieldNames.get(0));
             } else {
-                builder.startArray(SearchSourceBuilder.FIELDS_FIELD.getPreferredName());
-                for (String fieldName : fieldNames) {
+                builder.startArray(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName());
+                for (String fieldName : storedFieldNames) {
                     builder.value(fieldName);
                 }
                 builder.endArray();
             }
         }
-        if (fieldDataFields != null) {
-            builder.startArray(SearchSourceBuilder.FIELDDATA_FIELDS_FIELD.getPreferredName());
-            for (String fieldDataField : fieldDataFields) {
+        if (docValueFields != null) {
+            builder.startArray(SearchSourceBuilder.DOCVALUE_FIELDS_FIELD.getPreferredName());
+            for (String fieldDataField : docValueFields) {
                 builder.value(fieldDataField);
             }
             builder.endArray();
@@ -613,8 +693,8 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
                 Objects.equals(explain, that.explain) &&
                 Objects.equals(version, that.version) &&
                 Objects.equals(trackScores, that.trackScores) &&
-                Objects.equals(fieldNames, that.fieldNames) &&
-                Objects.equals(fieldDataFields, that.fieldDataFields) &&
+                Objects.equals(storedFieldNames, that.storedFieldNames) &&
+                Objects.equals(docValueFields, that.docValueFields) &&
                 Objects.equals(scriptFields, that.scriptFields) &&
                 Objects.equals(fetchSourceContext, that.fetchSourceContext) &&
                 Objects.equals(sorts, that.sorts) &&
@@ -625,8 +705,8 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
 
     @Override
     public int hashCode() {
-        return Objects.hash(name, nestedPath, parentChildType, from, size, explain, version, trackScores, fieldNames,
-                fieldDataFields, scriptFields, fetchSourceContext, sorts, highlightBuilder, query, childInnerHits);
+        return Objects.hash(name, nestedPath, parentChildType, from, size, explain, version, trackScores, storedFieldNames,
+            docValueFields, scriptFields, fetchSourceContext, sorts, highlightBuilder, query, childInnerHits);
     }
 
     public static InnerHitBuilder fromXContent(QueryParseContext context) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
index 8d6b003779b..713981fc0d5 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
@@ -24,6 +24,7 @@ import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
@@ -195,27 +196,35 @@ public class RestSearchAction extends BaseRestHandler {
             }
         }
 
-        String sField = request.param("fields");
+        if (request.param("fields") != null) {
+            throw new IllegalArgumentException("The parameter [" +
+                SearchSourceBuilder.FIELDS_FIELD + "] is not longer supported, please use [" +
+                SearchSourceBuilder.STORED_FIELDS_FIELD + "] to retrieve stored fields or _source filtering " +
+                "if the field is not stored");
+        }
+
+        String sField = request.param("stored_fields");
         if (sField != null) {
             if (!Strings.hasText(sField)) {
-                searchSourceBuilder.noFields();
+                searchSourceBuilder.noStoredFields();
             } else {
                 String[] sFields = Strings.splitStringByCommaToArray(sField);
                 if (sFields != null) {
                     for (String field : sFields) {
-                        searchSourceBuilder.field(field);
+                        searchSourceBuilder.storedField(field);
                     }
                 }
             }
         }
-        String sFieldDataFields = request.param("fielddata_fields");
-        if (sFieldDataFields != null) {
-            if (Strings.hasText(sFieldDataFields)) {
-                String[] sFields = Strings.splitStringByCommaToArray(sFieldDataFields);
-                if (sFields != null) {
-                    for (String field : sFields) {
-                        searchSourceBuilder.fieldDataField(field);
-                    }
+        String sDocValueFields = request.param("docvalue_fields");
+        if (sDocValueFields == null) {
+            sDocValueFields = request.param("fielddata_fields");
+        }
+        if (sDocValueFields != null) {
+            if (Strings.hasText(sDocValueFields)) {
+                String[] sFields = Strings.splitStringByCommaToArray(sDocValueFields);
+                for (String field : sFields) {
+                    searchSourceBuilder.docValueField(field);
                 }
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java
index dfbde3a6b19..362b3f0f416 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchService.java
@@ -733,8 +733,8 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
                 throw new SearchContextException(context, "failed to create RescoreSearchContext", e);
             }
         }
-        if (source.fields() != null) {
-            context.fieldNames().addAll(source.fields());
+        if (source.storedFields() != null) {
+            context.fieldNames().addAll(source.storedFields());
         }
         if (source.explain() != null) {
             context.explain(source.explain());
@@ -742,9 +742,9 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
         if (source.fetchSource() != null) {
             context.fetchSourceContext(source.fetchSource());
         }
-        if (source.fieldDataFields() != null) {
+        if (source.docValueFields() != null) {
             FieldDataFieldsContext fieldDataFieldsContext = context.getFetchSubPhaseContext(FieldDataFieldsFetchSubPhase.CONTEXT_FACTORY);
-            for (String field : source.fieldDataFields()) {
+            for (String field : source.docValueFields()) {
                 fieldDataFieldsContext.add(new FieldDataField(field));
             }
             fieldDataFieldsContext.setHitExecutionNeeded(true);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java
index 00d783c12e3..c9ed41c6ddc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java
@@ -567,9 +567,9 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
         }
         if (fieldNames != null) {
             if (fieldNames.size() == 1) {
-                builder.field(SearchSourceBuilder.FIELDS_FIELD.getPreferredName(), fieldNames.get(0));
+                builder.field(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), fieldNames.get(0));
             } else {
-                builder.startArray(SearchSourceBuilder.FIELDS_FIELD.getPreferredName());
+                builder.startArray(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName());
                 for (String fieldName : fieldNames) {
                     builder.value(fieldName);
                 }
@@ -577,7 +577,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
             }
         }
         if (fieldDataFields != null) {
-            builder.startArray(SearchSourceBuilder.FIELDDATA_FIELDS_FIELD.getPreferredName());
+            builder.startArray(SearchSourceBuilder.DOCVALUE_FIELDS_FIELD.getPreferredName());
             for (String fieldDataField : fieldDataFields) {
                 builder.value(fieldDataField);
             }
@@ -628,7 +628,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
                     factory.trackScores(parser.booleanValue());
                 } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder._SOURCE_FIELD)) {
                     factory.fetchSource(FetchSourceContext.parse(context));
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.FIELDS_FIELD)) {
+                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.STORED_FIELDS_FIELD)) {
                     List<String> fieldNames = new ArrayList<>();
                     fieldNames.add(parser.text());
                     factory.fields(fieldNames);
@@ -694,7 +694,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
 
-                if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.FIELDS_FIELD)) {
+                if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.STORED_FIELDS_FIELD)) {
                     List<String> fieldNames = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token == XContentParser.Token.VALUE_STRING) {
@@ -705,7 +705,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
                         }
                     }
                     factory.fields(fieldNames);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.FIELDDATA_FIELDS_FIELD)) {
+                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.DOCVALUE_FIELDS_FIELD)) {
                     List<String> fieldDataFields = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token == XContentParser.Token.VALUE_STRING) {
diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index 34b2b536259..e4b72ef3b28 100644
--- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -84,7 +84,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
     public static final ParseField EXPLAIN_FIELD = new ParseField("explain");
     public static final ParseField _SOURCE_FIELD = new ParseField("_source");
     public static final ParseField FIELDS_FIELD = new ParseField("fields");
-    public static final ParseField FIELDDATA_FIELDS_FIELD = new ParseField("fielddata_fields");
+    public static final ParseField STORED_FIELDS_FIELD = new ParseField("stored_fields");
+    public static final ParseField DOCVALUE_FIELDS_FIELD = new ParseField("docvalue_fields", "fielddata_fields");
     public static final ParseField SCRIPT_FIELDS_FIELD = new ParseField("script_fields");
     public static final ParseField SCRIPT_FIELD = new ParseField("script");
     public static final ParseField IGNORE_FAILURE_FIELD = new ParseField("ignore_failure");
@@ -147,8 +148,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
     private long timeoutInMillis = -1;
     private int terminateAfter = SearchContext.DEFAULT_TERMINATE_AFTER;
 
-    private List<String> fieldNames;
-    private List<String> fieldDataFields;
+    private List<String> storedFieldNames;
+    private List<String> docValueFields;
     private List<ScriptField> scriptFields;
     private FetchSourceContext fetchSourceContext;
 
@@ -182,22 +183,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         aggregations = in.readOptionalWriteable(AggregatorFactories.Builder::new);
         explain = in.readOptionalBoolean();
         fetchSourceContext = in.readOptionalStreamable(FetchSourceContext::new);
-        boolean hasFieldDataFields = in.readBoolean();
-        if (hasFieldDataFields) {
-            int size = in.readVInt();
-            fieldDataFields = new ArrayList<>(size);
-            for (int i = 0; i < size; i++) {
-                fieldDataFields.add(in.readString());
-            }
-        }
-        boolean hasFieldNames = in.readBoolean();
-        if (hasFieldNames) {
-            int size = in.readVInt();
-            fieldNames = new ArrayList<>(size);
-            for (int i = 0; i < size; i++) {
-                fieldNames.add(in.readString());
-            }
-        }
+        docValueFields = (List<String>) in.readGenericValue();
+        storedFieldNames = (List<String>) in.readGenericValue();
         from = in.readVInt();
         highlightBuilder = in.readOptionalWriteable(HighlightBuilder::new);
         boolean hasIndexBoost = in.readBoolean();
@@ -256,22 +243,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         out.writeOptionalWriteable(aggregations);
         out.writeOptionalBoolean(explain);
         out.writeOptionalStreamable(fetchSourceContext);
-        boolean hasFieldDataFields = fieldDataFields != null;
-        out.writeBoolean(hasFieldDataFields);
-        if (hasFieldDataFields) {
-            out.writeVInt(fieldDataFields.size());
-            for (String field : fieldDataFields) {
-                out.writeString(field);
-            }
-        }
-        boolean hasFieldNames = fieldNames != null;
-        out.writeBoolean(hasFieldNames);
-        if (hasFieldNames) {
-            out.writeVInt(fieldNames.size());
-            for (String field : fieldNames) {
-                out.writeString(field);
-            }
-        }
+        out.writeGenericValue(docValueFields);
+        out.writeGenericValue(storedFieldNames);
         out.writeVInt(from);
         out.writeOptionalWriteable(highlightBuilder);
         boolean hasIndexBoost = indexBoost != null;
@@ -733,62 +706,89 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
     }
 
     /**
-     * Adds a field to load and return (note, it must be stored) as part of the
+     * Adds a stored field to load and return as part of the
      * search request. If none are specified, the source of the document will be
      * return.
      */
-    public SearchSourceBuilder field(String name) {
-        if (fieldNames == null) {
-            fieldNames = new ArrayList<>();
+    public SearchSourceBuilder storedField(String name) {
+        if (storedFieldNames == null) {
+            storedFieldNames = new ArrayList<>();
         }
-        fieldNames.add(name);
+        storedFieldNames.add(name);
         return this;
     }
 
     /**
-     * Sets the fields to load and return as part of the search request. If none
+     * Sets the stored fields to load and return as part of the search request. If none
      * are specified, the source of the document will be returned.
      */
-    public SearchSourceBuilder fields(List<String> fields) {
-        this.fieldNames = fields;
+    public SearchSourceBuilder storedFields(List<String> fields) {
+        this.storedFieldNames = fields;
         return this;
     }
 
     /**
-     * Sets no fields to be loaded, resulting in only id and type to be returned
+     * Sets no stored fields to be loaded, resulting in only id and type to be returned
      * per field.
      */
-    public SearchSourceBuilder noFields() {
-        this.fieldNames = Collections.emptyList();
+    public SearchSourceBuilder noStoredFields() {
+        this.storedFieldNames = Collections.emptyList();
         return this;
     }
 
     /**
-     * Gets the fields to load and return as part of the search request.
+     * Gets the stored fields to load and return as part of the search request.
      */
-    public List<String> fields() {
-        return fieldNames;
+    public List<String> storedFields() {
+        return storedFieldNames;
+    }
+
+
+    /**
+     * Adds a field to load from the docvalue and return as part of the
+     * search request.
+     *
+     * @deprecated Use {@link SearchSourceBuilder#docValueField(String)} instead.
+     */
+    @Deprecated
+    public SearchSourceBuilder fieldDataField(String name) {
+        if (docValueFields == null) {
+            docValueFields = new ArrayList<>();
+        }
+        docValueFields.add(name);
+        return this;
     }
 
     /**
-     * Adds a field to load from the field data cache and return as part of the
+     * Gets the docvalue fields.
+     *
+     * @deprecated Use {@link SearchSourceBuilder#docValueFields()} instead.
+     */
+    @Deprecated
+    public List<String> fieldDataFields() {
+        return docValueFields;
+    }
+
+
+    /**
+     * Gets the docvalue fields.
+     */
+    public List<String> docValueFields() {
+        return docValueFields;
+    }
+
+    /**
+     * Adds a field to load from the docvalue and return as part of the
      * search request.
      */
-    public SearchSourceBuilder fieldDataField(String name) {
-        if (fieldDataFields == null) {
-            fieldDataFields = new ArrayList<>();
+    public SearchSourceBuilder docValueField(String name) {
+        if (docValueFields == null) {
+            docValueFields = new ArrayList<>();
         }
-        fieldDataFields.add(name);
+        docValueFields.add(name);
         return this;
     }
 
-    /**
-     * Gets the field-data fields.
-     */
-    public List<String> fieldDataFields() {
-        return fieldDataFields;
-    }
-
     /**
      * Adds a script field under the given name with the provided script.
      *
@@ -911,8 +911,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
             rewrittenBuilder.explain = explain;
             rewrittenBuilder.ext = ext;
             rewrittenBuilder.fetchSourceContext = fetchSourceContext;
-            rewrittenBuilder.fieldDataFields = fieldDataFields;
-            rewrittenBuilder.fieldNames = fieldNames;
+            rewrittenBuilder.docValueFields = docValueFields;
+            rewrittenBuilder.storedFieldNames = storedFieldNames;
             rewrittenBuilder.from = from;
             rewrittenBuilder.highlightBuilder = highlightBuilder;
             rewrittenBuilder.indexBoost = indexBoost;
@@ -972,12 +972,16 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                     trackScores = parser.booleanValue();
                 } else if (context.getParseFieldMatcher().match(currentFieldName, _SOURCE_FIELD)) {
                     fetchSourceContext = FetchSourceContext.parse(context);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) {
-                    field(parser.text());
+                } else if (context.getParseFieldMatcher().match(currentFieldName, STORED_FIELDS_FIELD)) {
+                    storedField(parser.text());
                 } else if (context.getParseFieldMatcher().match(currentFieldName, SORT_FIELD)) {
                     sort(parser.text());
                 } else if (context.getParseFieldMatcher().match(currentFieldName, PROFILE_FIELD)) {
                     profile = parser.booleanValue();
+                } else if (context.getParseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) {
+                    throw new ParsingException(parser.getTokenLocation(), "Deprecated field [" +
+                        SearchSourceBuilder.FIELDS_FIELD + "] used, expected [" +
+                        SearchSourceBuilder.STORED_FIELDS_FIELD + "] instead");
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].",
                             parser.getTokenLocation());
@@ -1027,22 +1031,21 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                             parser.getTokenLocation());
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-
-                if (context.getParseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) {
-                    fieldNames = new ArrayList<>();
+                if (context.getParseFieldMatcher().match(currentFieldName, STORED_FIELDS_FIELD)) {
+                    storedFieldNames = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token == XContentParser.Token.VALUE_STRING) {
-                            fieldNames.add(parser.text());
+                            storedFieldNames.add(parser.text());
                         } else {
                             throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.VALUE_STRING + "] in ["
                                     + currentFieldName + "] but found [" + token + "]", parser.getTokenLocation());
                         }
                     }
-                } else if (context.getParseFieldMatcher().match(currentFieldName, FIELDDATA_FIELDS_FIELD)) {
-                    fieldDataFields = new ArrayList<>();
+                } else if (context.getParseFieldMatcher().match(currentFieldName, DOCVALUE_FIELDS_FIELD)) {
+                    docValueFields = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token == XContentParser.Token.VALUE_STRING) {
-                            fieldDataFields.add(parser.text());
+                            docValueFields.add(parser.text());
                         } else {
                             throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.VALUE_STRING + "] in ["
                                     + currentFieldName + "] but found [" + token + "]", parser.getTokenLocation());
@@ -1069,6 +1072,11 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                     fetchSourceContext = FetchSourceContext.parse(context);
                 } else if (context.getParseFieldMatcher().match(currentFieldName, SEARCH_AFTER)) {
                     searchAfterBuilder = SearchAfterBuilder.fromXContent(parser, context.getParseFieldMatcher());
+                } else if (context.getParseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) {
+                    throw new ParsingException(parser.getTokenLocation(), "The field [" +
+                        SearchSourceBuilder.FIELDS_FIELD + "] is not longer supported, please use [" +
+                        SearchSourceBuilder.STORED_FIELDS_FIELD + "] to retrieve stored fields or _source filtering " +
+                        "if the field is not stored");
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].",
                             parser.getTokenLocation());
@@ -1132,21 +1140,21 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
             builder.field(_SOURCE_FIELD.getPreferredName(), fetchSourceContext);
         }
 
-        if (fieldNames != null) {
-            if (fieldNames.size() == 1) {
-                builder.field(FIELDS_FIELD.getPreferredName(), fieldNames.get(0));
+        if (storedFieldNames != null) {
+            if (storedFieldNames.size() == 1) {
+                builder.field(STORED_FIELDS_FIELD.getPreferredName(), storedFieldNames.get(0));
             } else {
-                builder.startArray(FIELDS_FIELD.getPreferredName());
-                for (String fieldName : fieldNames) {
+                builder.startArray(STORED_FIELDS_FIELD.getPreferredName());
+                for (String fieldName : storedFieldNames) {
                     builder.value(fieldName);
                 }
                 builder.endArray();
             }
         }
 
-        if (fieldDataFields != null) {
-            builder.startArray(FIELDDATA_FIELDS_FIELD.getPreferredName());
-            for (String fieldDataField : fieldDataFields) {
+        if (docValueFields != null) {
+            builder.startArray(DOCVALUE_FIELDS_FIELD.getPreferredName());
+            for (String fieldDataField : docValueFields) {
                 builder.value(fieldDataField);
             }
             builder.endArray();
@@ -1340,7 +1348,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
 
     @Override
     public int hashCode() {
-        return Objects.hash(aggregations, explain, fetchSourceContext, fieldDataFields, fieldNames, from,
+        return Objects.hash(aggregations, explain, fetchSourceContext, docValueFields, storedFieldNames, from,
                 highlightBuilder, indexBoost, minScore, postQueryBuilder, queryBuilder, rescoreBuilders, scriptFields,
                 size, sorts, searchAfterBuilder, sliceBuilder, stats, suggestBuilder, terminateAfter, timeoutInMillis, trackScores, version, profile);
     }
@@ -1357,8 +1365,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         return Objects.equals(aggregations, other.aggregations)
                 && Objects.equals(explain, other.explain)
                 && Objects.equals(fetchSourceContext, other.fetchSourceContext)
-                && Objects.equals(fieldDataFields, other.fieldDataFields)
-                && Objects.equals(fieldNames, other.fieldNames)
+                && Objects.equals(docValueFields, other.docValueFields)
+                && Objects.equals(storedFieldNames, other.storedFieldNames)
                 && Objects.equals(from, other.from)
                 && Objects.equals(highlightBuilder, other.highlightBuilder)
                 && Objects.equals(indexBoost, other.indexBoost)
diff --git a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
index f0e12abeac8..596714b1c5b 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
@@ -677,7 +677,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
         client().prepareIndex(IDX, "doc", "4").setSource("foo", "eggplant").get();
         flushAndRefresh(IDX);
 
-        SearchResponse resp = client().prepareSearch(IDX).setQuery(matchAllQuery()).addFieldDataField("foo").addSort("foo", SortOrder.ASC).get();
+        SearchResponse resp = client().prepareSearch(IDX).setQuery(matchAllQuery()).addDocValueField("foo").addSort("foo", SortOrder.ASC).get();
         assertHitCount(resp, 4);
         assertOrderedSearchHits(resp, "2", "3", "4", "1");
         SearchHit[] hits = resp.getHits().hits();
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java
index 31e4e2d0923..da0c3d081af 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java
@@ -166,9 +166,9 @@ public class TokenCountFieldMapperIntegrationIT extends ESIntegTestCase {
 
     private SearchRequestBuilder prepareSearch() {
         SearchRequestBuilder request = client().prepareSearch("test").setTypes("test");
-        request.addField("foo.token_count");
+        request.addStoredField("foo.token_count");
         if (loadCountedFields) {
-            request.addField("foo");
+            request.addStoredField("foo");
         }
         return request;
     }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java
index 202afd7a4b1..14dd370fbfd 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java
@@ -816,7 +816,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
                 .field("lon", -74.0059731).endObject().endObject()).setRefreshPolicy(IMMEDIATE).get();
 
         // match all search with geohash field
-        SearchResponse searchResponse = client().prepareSearch().addField("location.geohash").setQuery(matchAllQuery()).execute().actionGet();
+        SearchResponse searchResponse = client().prepareSearch().addStoredField("location.geohash").setQuery(matchAllQuery()).execute().actionGet();
         Map<String, SearchHitField> m = searchResponse.getHits().getAt(0).getFields();
 
         // ensure single geohash was indexed
@@ -841,7 +841,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
                 .field("lon", -74.0059731).endObject().endObject()).setRefreshPolicy(IMMEDIATE).get();
 
         // match all search with geohash field (includes prefixes)
-        SearchResponse searchResponse = client().prepareSearch().addField("location.geohash").setQuery(matchAllQuery()).execute().actionGet();
+        SearchResponse searchResponse = client().prepareSearch().addStoredField("location.geohash").setQuery(matchAllQuery()).execute().actionGet();
         Map<String, SearchHitField> m = searchResponse.getHits().getAt(0).getFields();
 
         List<Object> hashes = m.get("location.geohash").values();
@@ -872,11 +872,11 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
         }
 
         // query by geohash subfield
-        SearchResponse searchResponse = client().prepareSearch().addField("location.geohash").setQuery(matchAllQuery()).execute().actionGet();
+        SearchResponse searchResponse = client().prepareSearch().addStoredField("location.geohash").setQuery(matchAllQuery()).execute().actionGet();
         assertEquals(numDocs, searchResponse.getHits().totalHits());
 
         // query by latlon subfield
-        searchResponse = client().prepareSearch().addField("location.latlon").setQuery(matchAllQuery()).execute().actionGet();
+        searchResponse = client().prepareSearch().addStoredField("location.latlon").setQuery(matchAllQuery()).execute().actionGet();
         assertEquals(numDocs, searchResponse.getHits().totalHits());
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
index d4ba6ca9062..841f35bd975 100644
--- a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
@@ -218,8 +218,8 @@ public class InnerHitBuilderTests extends ESTestCase {
         innerHits.setExplain(randomBoolean());
         innerHits.setVersion(randomBoolean());
         innerHits.setTrackScores(randomBoolean());
-        innerHits.setFieldNames(randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16)));
-        innerHits.setFieldDataFields(randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16)));
+        innerHits.setStoredFieldNames(randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16)));
+        innerHits.setDocValueFields(randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16)));
         // Random script fields deduped on their field name.
         Map<String, SearchSourceBuilder.ScriptField> scriptFields = new HashMap<>();
         for (SearchSourceBuilder.ScriptField field: randomListStuff(16, InnerHitBuilderTests::randomScript)) {
@@ -294,11 +294,11 @@ public class InnerHitBuilderTests extends ESTestCase {
                 break;
             case 6:
                 if (randomBoolean()) {
-                    instance.setFieldDataFields(randomValueOtherThan(instance.getFieldDataFields(), () -> {
+                    instance.setDocValueFields(randomValueOtherThan(instance.getDocValueFields(), () -> {
                         return randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16));
                     }));
                 } else {
-                    instance.addFieldDataField(randomAsciiOfLengthBetween(1, 16));
+                    instance.addDocValueField(randomAsciiOfLengthBetween(1, 16));
                 }
                 break;
             case 7:
@@ -341,12 +341,12 @@ public class InnerHitBuilderTests extends ESTestCase {
                         HighlightBuilderTests::randomHighlighterBuilder));
                 break;
             case 11:
-                if (instance.getFieldNames() == null || randomBoolean()) {
-                    instance.setFieldNames(randomValueOtherThan(instance.getFieldNames(), () -> {
+                if (instance.getStoredFieldNames() == null || randomBoolean()) {
+                    instance.setStoredFieldNames(randomValueOtherThan(instance.getStoredFieldNames(), () -> {
                         return randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16));
                     }));
                 } else {
-                    instance.getFieldNames().add(randomAsciiOfLengthBetween(1, 16));
+                    instance.getStoredFieldNames().add(randomAsciiOfLengthBetween(1, 16));
                 }
                 break;
             default:
diff --git a/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java b/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java
index 163d72f4553..590c5c624f1 100644
--- a/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java
+++ b/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java
@@ -113,7 +113,7 @@ public class ExceptionRetryIT extends ESIntegTestCase {
         }
 
         refresh();
-        SearchResponse searchResponse = client().prepareSearch("index").setSize(numDocs * 2).addField("_id").get();
+        SearchResponse searchResponse = client().prepareSearch("index").setSize(numDocs * 2).addStoredField("_id").get();
 
         Set<String> uniqueIds = new HashSet();
         long dupCounter = 0;
diff --git a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
index 493f8b74e04..acdfdd12266 100644
--- a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
@@ -116,7 +116,7 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
         ensureGreen();
         SearchResponse searchResponse = client().prepareSearch("test_index")
                 .setQuery(termQuery("field1", "value1"))
-                .addField("field1").addField("field2")
+                .addStoredField("field1").addStoredField("field2")
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 1);
@@ -130,7 +130,7 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
         // now only match on one template (template_1)
         searchResponse = client().prepareSearch("text_index")
                 .setQuery(termQuery("field1", "value1"))
-                .addField("field1").addField("field2")
+                .addStoredField("field1").addStoredField("field2")
                 .execute().actionGet();
         if (searchResponse.getFailedShards() > 0) {
             logger.warn("failed search {}", Arrays.toString(searchResponse.getShardFailures()));
diff --git a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
index bc9909b21c1..1145e594b25 100644
--- a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
+++ b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
@@ -219,7 +219,7 @@ public class RelocationIT extends ESIntegTestCase {
             for (int i = 0; i < 10; i++) {
                 try {
                     logger.info("--> START search test round {}", i + 1);
-                    SearchHits hits = client().prepareSearch("test").setQuery(matchAllQuery()).setSize((int) indexer.totalIndexedDocs()).setNoFields().execute().actionGet().getHits();
+                    SearchHits hits = client().prepareSearch("test").setQuery(matchAllQuery()).setSize((int) indexer.totalIndexedDocs()).setNoStoredFields().execute().actionGet().getHits();
                     ranOnce = true;
                     if (hits.totalHits() != indexer.totalIndexedDocs()) {
                         int[] hitIds = new int[(int) indexer.totalIndexedDocs()];
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java
index 49ef9e1b6b5..5cc6ec58630 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java
@@ -181,7 +181,7 @@ public abstract class AbstractGeoTestCase extends ESIntegTestCase {
         // Added to debug a test failure where the terms aggregation seems to be reporting two documents with the same value for NUMBER_FIELD_NAME.  This will check that after
         // random indexing each document only has 1 value for NUMBER_FIELD_NAME and it is the correct value. Following this initial change its seems that this call was getting
         // more that 2000 hits (actual value was 2059) so now it will also check to ensure all hits have the correct index and type
-        SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME).addField(NUMBER_FIELD_NAME).addSort(SortBuilders.fieldSort(NUMBER_FIELD_NAME)
+        SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME).addStoredField(NUMBER_FIELD_NAME).addSort(SortBuilders.fieldSort(NUMBER_FIELD_NAME)
                 .order(SortOrder.ASC)).setSize(5000).get();
         assertSearchResponse(response);
         long totalHits = response.getHits().totalHits();
diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index fbd9546ec2d..ee5b65b5ebc 100644
--- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -219,12 +219,12 @@ public class SearchSourceBuilderTests extends ESTestCase {
             for (int i = 0; i < fieldsSize; i++) {
                 fields.add(randomAsciiOfLengthBetween(5, 50));
             }
-            builder.fields(fields);
+            builder.storedFields(fields);
         }
         if (randomBoolean()) {
             int fieldDataFieldsSize = randomInt(25);
             for (int i = 0; i < fieldDataFieldsSize; i++) {
-                builder.fieldDataField(randomAsciiOfLengthBetween(5, 50));
+                builder.docValueField(randomAsciiOfLengthBetween(5, 50));
             }
         }
         if (randomBoolean()) {
diff --git a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
index f8ca1e1aaf7..68679e89ae6 100644
--- a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
@@ -202,7 +202,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         refresh();
 
         // TEST FETCHING _parent from child
-        SearchResponse searchResponse = client().prepareSearch("test").setQuery(idsQuery("child").addIds("c1")).fields("_parent").execute()
+        SearchResponse searchResponse = client().prepareSearch("test").setQuery(idsQuery("child").addIds("c1")).storedFields("_parent").execute()
                 .actionGet();
         assertNoFailures(searchResponse);
         assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
@@ -210,7 +210,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         assertThat(searchResponse.getHits().getAt(0).field("_parent").value().toString(), equalTo("p1"));
 
         // TEST matching on parent
-        searchResponse = client().prepareSearch("test").setQuery(termQuery("_parent#parent", "p1")).fields("_parent").get();
+        searchResponse = client().prepareSearch("test").setQuery(termQuery("_parent#parent", "p1")).storedFields("_parent").get();
         assertNoFailures(searchResponse);
         assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
         assertThat(searchResponse.getHits().getAt(0).id(), anyOf(equalTo("c1"), equalTo("c2")));
@@ -218,7 +218,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         assertThat(searchResponse.getHits().getAt(1).id(), anyOf(equalTo("c1"), equalTo("c2")));
         assertThat(searchResponse.getHits().getAt(1).field("_parent").value().toString(), equalTo("p1"));
 
-        searchResponse = client().prepareSearch("test").setQuery(queryStringQuery("_parent#parent:p1")).fields("_parent").get();
+        searchResponse = client().prepareSearch("test").setQuery(queryStringQuery("_parent#parent:p1")).storedFields("_parent").get();
         assertNoFailures(searchResponse);
         assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
         assertThat(searchResponse.getHits().getAt(0).id(), anyOf(equalTo("c1"), equalTo("c2")));
@@ -1394,7 +1394,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
             SearchResponse scrollResponse = client().prepareSearch("test")
                     .setScroll(TimeValue.timeValueSeconds(30))
                     .setSize(1)
-                    .addField("_id")
+                    .addStoredField("_id")
                     .setQuery(query)
                     .execute()
                     .actionGet();
diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java
index 0debdb263af..985605c4e65 100644
--- a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java
+++ b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java
@@ -415,13 +415,13 @@ public class GeoFilterIT extends ESIntegTestCase {
             assertThat(hit.getId(), equalTo(key));
         }
 
-        SearchResponse world = client().prepareSearch().addField("pin").setQuery(
+        SearchResponse world = client().prepareSearch().addStoredField("pin").setQuery(
                 geoBoundingBoxQuery("pin").setCorners(90, -179.99999, -90, 179.99999)
         ).execute().actionGet();
 
         assertHitCount(world, 53);
 
-        SearchResponse distance = client().prepareSearch().addField("pin").setQuery(
+        SearchResponse distance = client().prepareSearch().addStoredField("pin").setQuery(
                 geoDistanceQuery("pin").distance("425km").point(51.11, 9.851)
                 ).execute().actionGet();
 
diff --git a/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java b/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
index 01f98564814..56b33c6007e 100644
--- a/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
@@ -156,7 +156,7 @@ public class InnerHitsIT extends ESIntegTestCase {
                 .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg).innerHit(
                         new InnerHitBuilder().setHighlightBuilder(new HighlightBuilder().field("comments.message"))
                                 .setExplain(true)
-                                .addFieldDataField("comments.message")
+                                .addDocValueField("comments.message")
                                 .addScriptField("script", new Script("5", ScriptService.ScriptType.INLINE, MockScriptEngine.NAME, Collections.emptyMap()))
                                 .setSize(1)
                 )).get();
@@ -287,7 +287,7 @@ public class InnerHitsIT extends ESIntegTestCase {
                 .setQuery(
                         hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit(
                                 new InnerHitBuilder()
-                                        .addFieldDataField("message")
+                                        .addDocValueField("message")
                                         .setHighlightBuilder(new HighlightBuilder().field("message"))
                                         .setExplain(true).setSize(1)
                                         .addScriptField("script", new Script("5", ScriptService.ScriptType.INLINE,
diff --git a/core/src/test/java/org/elasticsearch/search/source/SourceFetchingIT.java b/core/src/test/java/org/elasticsearch/search/source/SourceFetchingIT.java
index 33fcb55cada..df147ce106f 100644
--- a/core/src/test/java/org/elasticsearch/search/source/SourceFetchingIT.java
+++ b/core/src/test/java/org/elasticsearch/search/source/SourceFetchingIT.java
@@ -37,10 +37,10 @@ public class SourceFetchingIT extends ESIntegTestCase {
         SearchResponse response = client().prepareSearch("test").get();
         assertThat(response.getHits().getAt(0).getSourceAsString(), notNullValue());
 
-        response = client().prepareSearch("test").addField("bla").get();
+        response = client().prepareSearch("test").addStoredField("bla").get();
         assertThat(response.getHits().getAt(0).getSourceAsString(), nullValue());
 
-        response = client().prepareSearch("test").addField("_source").get();
+        response = client().prepareSearch("test").addStoredField("_source").get();
         assertThat(response.getHits().getAt(0).getSourceAsString(), notNullValue());
 
     }
diff --git a/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java b/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java
index 3573089fcaa..e81b4decb2d 100644
--- a/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java
+++ b/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java
@@ -87,7 +87,7 @@ public class TimestampTTLBWIT extends ESIntegTestCase {
                 .setQuery(matchAllQuery())
                 .setSize(randomIntBetween(1, numDocs + 5))
                 .addSort("_timestamp", order)
-                .addField("_timestamp")
+                .addStoredField("_timestamp")
                 .execute().actionGet();
         assertNoFailures(searchResponse);
         SearchHit[] hits = searchResponse.getHits().hits();
diff --git a/docs/plugins/mapper-attachments.asciidoc b/docs/plugins/mapper-attachments.asciidoc
index b287decd4cf..f2c034a317e 100644
--- a/docs/plugins/mapper-attachments.asciidoc
+++ b/docs/plugins/mapper-attachments.asciidoc
@@ -246,7 +246,7 @@ PUT /test/person/1?refresh=true
 }
 GET /test/person/_search
 {
-  "fields": [ "file.content_type" ],
+  "stored_fields": [ "file.content_type" ],
   "query": {
     "match": {
       "file.content_type": "text plain"
@@ -367,7 +367,7 @@ PUT /test/person/1?refresh=true
 }
 GET /test/person/_search
 {
-  "fields": [],
+  "stored_fields": [],
   "query": {
     "match": {
       "file.content": "king queen"
diff --git a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
index 1dd1524a58e..2bad60ba0ec 100644
--- a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
@@ -22,7 +22,7 @@ The top_hits aggregation returns regular search hits, because of this many per h
 * <<search-request-named-queries-and-filters,Named filters and queries>>
 * <<search-request-source-filtering,Source filtering>>
 * <<search-request-script-fields,Script fields>>
-* <<search-request-fielddata-fields,Fielddata fields>>
+* <<search-request-docvalue-fields,Doc value fields>>
 * <<search-request-version,Include versions>>
 
 ==== Example
diff --git a/docs/reference/mapping/params/store.asciidoc b/docs/reference/mapping/params/store.asciidoc
index d34d1016546..53cac7493ff 100644
--- a/docs/reference/mapping/params/store.asciidoc
+++ b/docs/reference/mapping/params/store.asciidoc
@@ -48,7 +48,7 @@ PUT my_index/my_type/1
 
 GET my_index/_search
 {
-  "fields": [ "title", "date" ] <2>
+  "stored_fields": [ "title", "date" ] <2>
 }
 --------------------------------------------------
 // CONSOLE
diff --git a/docs/reference/migration/migrate_5_0/search.asciidoc b/docs/reference/migration/migrate_5_0/search.asciidoc
index 72b29e8a9e7..09478ee3251 100644
--- a/docs/reference/migration/migrate_5_0/search.asciidoc
+++ b/docs/reference/migration/migrate_5_0/search.asciidoc
@@ -64,11 +64,15 @@ characteristics as the former `scan` search type.
 
 ==== `fields` parameter
 
-The `fields` parameter used to try to retrieve field values from stored
-fields, and fall back to extracting from the `_source` if a field is not
-marked as stored.  Now, the `fields` parameter will only return stored fields
+The `fields` parameter has been replaced by `stored_fields`.
+The `stored_fields` parameter will only return stored fields
 -- it will no longer extract values from the `_source`.
 
+==== `fielddata_fields` parameter
+
+The `fielddata_fields` has been deprecated, use parameter `docvalue_fields` instead.
+
+
 ==== search-exists API removed
 
 The search exists api has been removed in favour of using the search api with
diff --git a/docs/reference/modules/scripting/painless.asciidoc b/docs/reference/modules/scripting/painless.asciidoc
index 2c753e09039..b4af07035ee 100644
--- a/docs/reference/modules/scripting/painless.asciidoc
+++ b/docs/reference/modules/scripting/painless.asciidoc
@@ -152,7 +152,7 @@ First, let's look at the source data for a player by submitting the following re
 ----------------------------------------------------------------
 GET hockey/_search
 {
-  "fields": [
+  "stored_fields": [
     "_id",
     "_source"
   ],
diff --git a/docs/reference/search/request-body.asciidoc b/docs/reference/search/request-body.asciidoc
index 958320ea110..a9adc157bd3 100644
--- a/docs/reference/search/request-body.asciidoc
+++ b/docs/reference/search/request-body.asciidoc
@@ -143,11 +143,11 @@ include::request/sort.asciidoc[]
 
 include::request/source-filtering.asciidoc[]
 
-include::request/fields.asciidoc[]
+include::request/stored-fields.asciidoc[]
 
 include::request/script-fields.asciidoc[]
 
-include::request/fielddata-fields.asciidoc[]
+include::request/docvalue-fields.asciidoc[]
 
 include::request/post-filter.asciidoc[]
 
diff --git a/docs/reference/search/request/docvalue-fields.asciidoc b/docs/reference/search/request/docvalue-fields.asciidoc
new file mode 100644
index 00000000000..bd090a3bd8d
--- /dev/null
+++ b/docs/reference/search/request/docvalue-fields.asciidoc
@@ -0,0 +1,23 @@
+[[search-request-docvalue-fields]]
+=== Doc value Fields
+
+Allows to return the <<docvalue,doc value>> representation of a field for each hit, for
+example:
+
+[source,js]
+--------------------------------------------------
+GET /_search
+{
+    "query" : {
+        "match_all": {}
+    },
+    "docvalue_fields" : ["test1", "test2"]
+}
+--------------------------------------------------
+// CONSOLE
+
+Doc value fields can work on fields that are not stored.
+
+Note that if the fields parameter specifies fields without docvalues it will try to load the value from the fielddata cache
+causing the terms for that field to be loaded to memory (cached), which will result in more memory consumption.
+
diff --git a/docs/reference/search/request/fielddata-fields.asciidoc b/docs/reference/search/request/fielddata-fields.asciidoc
deleted file mode 100644
index f3a3508b144..00000000000
--- a/docs/reference/search/request/fielddata-fields.asciidoc
+++ /dev/null
@@ -1,23 +0,0 @@
-[[search-request-fielddata-fields]]
-=== Field Data Fields
-
-Allows to return the <<fielddata,field data>> representation of a field for each hit, for
-example:
-
-[source,js]
---------------------------------------------------
-GET /_search
-{
-    "query" : {
-        "match_all": {}
-    },
-    "fielddata_fields" : ["test1", "test2"]
-}
---------------------------------------------------
-// CONSOLE
-
-Field data fields can work on fields that are not stored.
-
-It's important to understand that using the `fielddata_fields` parameter will
-cause the terms for that field to be loaded to memory (cached), which will
-result in more memory consumption.
diff --git a/docs/reference/search/request/highlighting.asciidoc b/docs/reference/search/request/highlighting.asciidoc
index c235c37b338..efb7053c179 100644
--- a/docs/reference/search/request/highlighting.asciidoc
+++ b/docs/reference/search/request/highlighting.asciidoc
@@ -372,7 +372,7 @@ query and the rescore query in `highlight_query`.
 --------------------------------------------------
 GET /_search
 {
-    "fields": [ "_id" ],
+    "stored_fields": [ "_id" ],
     "query" : {
         "match": {
             "content": {
diff --git a/docs/reference/search/request/inner-hits.asciidoc b/docs/reference/search/request/inner-hits.asciidoc
index 345bc9abde2..3c8e0e9f00e 100644
--- a/docs/reference/search/request/inner-hits.asciidoc
+++ b/docs/reference/search/request/inner-hits.asciidoc
@@ -72,7 +72,7 @@ Inner hits also supports the following per document features:
 * <<search-request-explain,Explain>>
 * <<search-request-source-filtering,Source filtering>>
 * <<search-request-script-fields,Script fields>>
-* <<search-request-fielddata-fields,Fielddata fields>>
+* <<search-request-docvalue-fields,Doc value fields>>
 * <<search-request-version,Include versions>>
 
 [[nested-inner-hits]]
diff --git a/docs/reference/search/request/fields.asciidoc b/docs/reference/search/request/stored-fields.asciidoc
similarity index 91%
rename from docs/reference/search/request/fields.asciidoc
rename to docs/reference/search/request/stored-fields.asciidoc
index 3483d470ee2..3d5b8c01b47 100644
--- a/docs/reference/search/request/fields.asciidoc
+++ b/docs/reference/search/request/stored-fields.asciidoc
@@ -1,7 +1,7 @@
 [[search-request-fields]]
 === Fields
 
-WARNING: The `fields` parameter is about fields that are explicitly marked as
+WARNING: The `stored_fields` parameter is about fields that are explicitly marked as
 stored in the mapping, which is off by default and generally not recommended.
 Use <<search-request-source-filtering,source filtering>> instead to select
 subsets of the original source document to be returned.
@@ -13,7 +13,7 @@ by a search hit.
 --------------------------------------------------
 GET /_search
 {
-    "fields" : ["user", "postDate"],
+    "stored_fields" : ["user", "postDate"],
     "query" : {
         "term" : { "user" : "kimchy" }
     }
@@ -30,7 +30,7 @@ returned, for example:
 --------------------------------------------------
 GET /_search
 {
-    "fields" : [],
+    "stored_fields" : [],
     "query" : {
         "term" : { "user" : "kimchy" }
     }
diff --git a/docs/reference/search/uri-request.asciidoc b/docs/reference/search/uri-request.asciidoc
index 496f04ea4af..ba36992f6fb 100644
--- a/docs/reference/search/uri-request.asciidoc
+++ b/docs/reference/search/uri-request.asciidoc
@@ -83,7 +83,7 @@ hits was computed.
 part of the document by using `_source_include` & `_source_exclude` (see the <<search-request-source-filtering, request body>>
 documentation for more details)
 
-|`fields` |The selective stored fields of the document to return for each hit,
+|`stored_fields` |The selective stored fields of the document to return for each hit,
 comma delimited. Not specifying any value will cause no fields to return.
 
 |`sort` |Sorting to perform. Can either be in the form of `fieldName`, or
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java
index e9c4bf6e359..0d7dd4e12e2 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java
@@ -72,28 +72,28 @@ public class GeoDistanceTests extends ESIntegTestCase {
 
         refresh();
 
-        SearchResponse searchResponse1 = client().prepareSearch().addField("_source")
+        SearchResponse searchResponse1 = client().prepareSearch().addStoredField("_source")
                 .addScriptField("distance", new Script("doc['location'].arcDistance(" + target_lat + "," + target_long + ")")).execute()
                 .actionGet();
         Double resultDistance1 = searchResponse1.getHits().getHits()[0].getFields().get("distance").getValue();
         assertThat(resultDistance1,
                 closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.DEFAULT), 0.01d));
 
-        SearchResponse searchResponse2 = client().prepareSearch().addField("_source")
+        SearchResponse searchResponse2 = client().prepareSearch().addStoredField("_source")
                 .addScriptField("distance", new Script("doc['location'].distance(" + target_lat + "," + target_long + ")")).execute()
                 .actionGet();
         Double resultDistance2 = searchResponse2.getHits().getHits()[0].getFields().get("distance").getValue();
         assertThat(resultDistance2,
                 closeTo(GeoDistance.PLANE.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.DEFAULT), 0.01d));
 
-        SearchResponse searchResponse3 = client().prepareSearch().addField("_source")
+        SearchResponse searchResponse3 = client().prepareSearch().addStoredField("_source")
                 .addScriptField("distance", new Script("doc['location'].arcDistanceInKm(" + target_lat + "," + target_long + ")"))
                 .execute().actionGet();
         Double resultArcDistance3 = searchResponse3.getHits().getHits()[0].getFields().get("distance").getValue();
         assertThat(resultArcDistance3,
                 closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS), 0.01d));
 
-        SearchResponse searchResponse4 = client().prepareSearch().addField("_source")
+        SearchResponse searchResponse4 = client().prepareSearch().addStoredField("_source")
                 .addScriptField("distance", new Script("doc['location'].distanceInKm(" + target_lat + "," + target_long + ")")).execute()
                 .actionGet();
         Double resultDistance4 = searchResponse4.getHits().getHits()[0].getFields().get("distance").getValue();
@@ -102,7 +102,7 @@ public class GeoDistanceTests extends ESIntegTestCase {
 
         SearchResponse searchResponse5 = client()
                 .prepareSearch()
-                .addField("_source")
+                .addStoredField("_source")
                 .addScriptField("distance", new Script("doc['location'].arcDistanceInKm(" + (target_lat) + "," + (target_long + 360) + ")"))
                 .execute().actionGet();
         Double resultArcDistance5 = searchResponse5.getHits().getHits()[0].getFields().get("distance").getValue();
@@ -111,21 +111,21 @@ public class GeoDistanceTests extends ESIntegTestCase {
 
         SearchResponse searchResponse6 = client()
                 .prepareSearch()
-                .addField("_source")
+                .addStoredField("_source")
                 .addScriptField("distance", new Script("doc['location'].arcDistanceInKm(" + (target_lat + 360) + "," + (target_long) + ")"))
                 .execute().actionGet();
         Double resultArcDistance6 = searchResponse6.getHits().getHits()[0].getFields().get("distance").getValue();
         assertThat(resultArcDistance6,
                 closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS), 0.01d));
 
-        SearchResponse searchResponse7 = client().prepareSearch().addField("_source")
+        SearchResponse searchResponse7 = client().prepareSearch().addStoredField("_source")
                 .addScriptField("distance", new Script("doc['location'].arcDistanceInMiles(" + target_lat + "," + target_long + ")"))
                 .execute().actionGet();
         Double resultDistance7 = searchResponse7.getHits().getHits()[0].getFields().get("distance").getValue();
         assertThat(resultDistance7,
                 closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.MILES), 0.01d));
 
-        SearchResponse searchResponse8 = client().prepareSearch().addField("_source")
+        SearchResponse searchResponse8 = client().prepareSearch().addStoredField("_source")
                 .addScriptField("distance", new Script("doc['location'].distanceInMiles(" + target_lat + "," + target_long + ")"))
                 .execute().actionGet();
         Double resultDistance8 = searchResponse8.getHits().getHits()[0].getFields().get("distance").getValue();
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
index 28894c5c1cc..6a8be584085 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
@@ -102,33 +102,33 @@ public class SearchFieldsTests extends ESIntegTestCase {
 
         client().admin().indices().prepareRefresh().execute().actionGet();
 
-        SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("field1").execute().actionGet();
+        SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field1").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field1").value().toString(), equalTo("value1"));
 
         // field2 is not stored, check that it is not extracted from source.
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("field2").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field2").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(0));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field2"), nullValue());
 
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("field3").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field3").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3"));
 
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("*3").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("*3").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3"));
 
 
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("*3").addField("field1").addField("field2").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("*3").addStoredField("field1").addStoredField("field2").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(2));
@@ -136,20 +136,20 @@ public class SearchFieldsTests extends ESIntegTestCase {
         assertThat(searchResponse.getHits().getAt(0).fields().get("field1").value().toString(), equalTo("value1"));
 
 
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("field*").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field*").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(2));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3"));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field1").value().toString(), equalTo("value1"));
 
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("f*3").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("f*3").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3"));
 
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("*").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("*").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).source(), nullValue());
@@ -157,7 +157,7 @@ public class SearchFieldsTests extends ESIntegTestCase {
         assertThat(searchResponse.getHits().getAt(0).fields().get("field1").value().toString(), equalTo("value1"));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3"));
 
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("*").addField("_source").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("*").addStoredField("_source").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).source(), notNullValue());
@@ -437,15 +437,15 @@ public class SearchFieldsTests extends ESIntegTestCase {
         client().admin().indices().prepareRefresh().execute().actionGet();
 
         SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery())
-                .addField("byte_field")
-                .addField("short_field")
-                .addField("integer_field")
-                .addField("long_field")
-                .addField("float_field")
-                .addField("double_field")
-                .addField("date_field")
-                .addField("boolean_field")
-                .addField("binary_field")
+                .addStoredField("byte_field")
+                .addStoredField("short_field")
+                .addStoredField("integer_field")
+                .addStoredField("long_field")
+                .addStoredField("float_field")
+                .addStoredField("double_field")
+                .addStoredField("date_field")
+                .addStoredField("boolean_field")
+                .addStoredField("binary_field")
                 .execute().actionGet();
 
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
@@ -478,7 +478,7 @@ public class SearchFieldsTests extends ESIntegTestCase {
 
         SearchResponse searchResponse = client().prepareSearch("my-index")
                 .setTypes("my-type1")
-                .addField("field1").addField("_routing")
+                .addStoredField("field1").addStoredField("_routing")
                 .get();
 
         assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
@@ -493,7 +493,7 @@ public class SearchFieldsTests extends ESIntegTestCase {
                 .setRefreshPolicy(IMMEDIATE)
                 .get();
 
-        assertFailures(client().prepareSearch("my-index").setTypes("my-type1").addField("field1"),
+        assertFailures(client().prepareSearch("my-index").setTypes("my-type1").addStoredField("field1"),
                 RestStatus.BAD_REQUEST,
                 containsString("field [field1] isn't a leaf field"));
     }
@@ -557,14 +557,14 @@ public class SearchFieldsTests extends ESIntegTestCase {
 
 
         String field = "field1.field2.field3.field4";
-        SearchResponse searchResponse = client().prepareSearch("my-index").setTypes("my-type1").addField(field).get();
+        SearchResponse searchResponse = client().prepareSearch("my-index").setTypes("my-type1").addStoredField(field).get();
         assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().getAt(0).field(field).isMetadataField(), equalTo(false));
         assertThat(searchResponse.getHits().getAt(0).field(field).getValues().size(), equalTo(2));
         assertThat(searchResponse.getHits().getAt(0).field(field).getValues().get(0).toString(), equalTo("value1"));
         assertThat(searchResponse.getHits().getAt(0).field(field).getValues().get(1).toString(), equalTo("value2"));
 
-        searchResponse = client().prepareSearch("my-index").setTypes("my-type2").addField(field).get();
+        searchResponse = client().prepareSearch("my-index").setTypes("my-type2").addStoredField(field).get();
         assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().getAt(0).field(field).isMetadataField(), equalTo(false));
         assertThat(searchResponse.getHits().getAt(0).field(field).getValues().size(), equalTo(2));
@@ -621,16 +621,16 @@ public class SearchFieldsTests extends ESIntegTestCase {
         client().admin().indices().prepareRefresh().execute().actionGet();
 
         SearchRequestBuilder builder = client().prepareSearch().setQuery(matchAllQuery())
-                .addFieldDataField("text_field")
-                .addFieldDataField("keyword_field")
-                .addFieldDataField("byte_field")
-                .addFieldDataField("short_field")
-                .addFieldDataField("integer_field")
-                .addFieldDataField("long_field")
-                .addFieldDataField("float_field")
-                .addFieldDataField("double_field")
-                .addFieldDataField("date_field")
-                .addFieldDataField("boolean_field");
+                .addDocValueField("text_field")
+                .addDocValueField("keyword_field")
+                .addDocValueField("byte_field")
+                .addDocValueField("short_field")
+                .addDocValueField("integer_field")
+                .addDocValueField("long_field")
+                .addDocValueField("float_field")
+                .addDocValueField("double_field")
+                .addDocValueField("date_field")
+                .addDocValueField("boolean_field");
         SearchResponse searchResponse = builder.execute().actionGet();
 
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
@@ -704,7 +704,7 @@ public class SearchFieldsTests extends ESIntegTestCase {
                     .setParent("parent_1")
                     .setSource(jsonBuilder().startObject().field("field1", "value").endObject()));
 
-        SearchResponse response = client().prepareSearch("test").addField("field1").get();
+        SearchResponse response = client().prepareSearch("test").addStoredField("field1").get();
         assertSearchResponse(response);
         assertHitCount(response, 1);
 
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
index 80a6ff891da..7725ee7f519 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
@@ -123,8 +123,8 @@ public abstract class AbstractBulkByScrollRequest<Self extends AbstractBulkByScr
         if (searchRequest.source().from() != -1) {
             e = addValidationError("from is not supported in this context", e);
         }
-        if (searchRequest.source().fields() != null) {
-            e = addValidationError("fields is not supported in this context", e);
+        if (searchRequest.source().storedFields() != null) {
+            e = addValidationError("stored_fields is not supported in this context", e);
         }
         if (maxRetries < 0) {
             e = addValidationError("retries cannnot be negative", e);
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml
index 8648c9034ee..7f17befe909 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml
@@ -59,11 +59,11 @@
 ---
 "source fields may not be modified":
   - do:
-      catch: /fields is not supported in this context/
+      catch: /stored_fields is not supported in this context/
       delete_by_query:
         index: test
         body:
-          fields: [_id]
+          stored_fields: [_id]
 
 ---
 "requests_per_second cannot be an empty string":
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
index a7a5198e430..b3e35d2f49e 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
@@ -216,11 +216,11 @@
 ---
 "source fields may not be modified":
   - do:
-      catch: /fields is not supported in this context/
+      catch: /stored_fields is not supported in this context/
       reindex:
         body:
           source:
             index: test
-            fields: [_id]
+            stored_fields: [_id]
           dest:
             index: dest
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml
index ea487eb54e0..08ab3cbcc10 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml
@@ -67,11 +67,11 @@
 ---
 "source fields may not be modified":
   - do:
-      catch: /fields is not supported in this context/
+      catch: /stored_fields is not supported in this context/
       update_by_query:
         index: test
         body:
-          fields: [_id]
+          stored_fields: [_id]
 
 ---
 "requests_per_second cannot be an empty string":
diff --git a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/30_mapping.yaml b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/30_mapping.yaml
index 458990cc90c..f180f51dfc5 100644
--- a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/30_mapping.yaml
+++ b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/30_mapping.yaml
@@ -54,7 +54,7 @@
         search:
             index: test
             body:
-                fields: [file.content_type,file.name]
+                stored_fields: [file.content_type,file.name]
 
     - match: { hits.total: 1 }
     - match: { hits.hits.0.fields: { file.content_type: ["text/my-dummy-content-type"],  file.name: ["my-dummy-name-txt"] }}
diff --git a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/40_highlight.yaml b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/40_highlight.yaml
index dc6f800c078..c2d42be3b9a 100644
--- a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/40_highlight.yaml
+++ b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/40_highlight.yaml
@@ -57,7 +57,7 @@ setup:
                 query:
                     match:
                         file.content: "apache tika"
-                fields: []
+                stored_fields: []
                 highlight:
                   fields:
                     file.content: {}
diff --git a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/50_files_supported.yaml b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/50_files_supported.yaml
index 69991b9d0c0..99f427c1ab9 100644
--- a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/50_files_supported.yaml
+++ b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/50_files_supported.yaml
@@ -38,7 +38,7 @@ setup:
       search:
           index: test
           body:
-              fields: [file.content, file.author, file.date, file.content_length, file.content_type]
+              stored_fields: [file.content, file.author, file.date, file.content_length, file.content_type]
   - match: { hits.total: 1 }
   - match: { hits.hits.0.fields: {
       file.content: ["Test elasticsearch\n"],
@@ -65,7 +65,7 @@ setup:
       search:
           index: test
           body:
-              fields: [file.content, file.author, file.date, file.content_length, file.content_type]
+              stored_fields: [file.content, file.author, file.date, file.content_length, file.content_type]
   - match: { hits.total: 1 }
   - match: { hits.hits.0.fields: {
       file.content: ["Test elasticsearch\n"],
diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java
index 761fb5fd144..956645aebd2 100644
--- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java
+++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java
@@ -83,8 +83,8 @@ public class SizeFieldMapperUpgradeTests extends ESIntegTestCase {
         ElasticsearchAssertions.assertHitCount(countResponse, 3L);
 
         final SearchResponse sizeResponse = client().prepareSearch(indexName)
-                .addField("_source")
-                .addField("_size")
+                .addStoredField("_source")
+                .addStoredField("_size")
                 .get();
         ElasticsearchAssertions.assertHitCount(sizeResponse, 3L);
         for (SearchHit hit : sizeResponse.getHits().getHits()) {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
index d2b9b8cf9b4..21fda8dc805 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
@@ -38,13 +38,17 @@
           "type" : "boolean",
           "description" : "Specify whether to return detailed information about score computation as part of a hit"
         },
-        "fields": {
+        "stored_fields": {
           "type" : "list",
-          "description" : "A comma-separated list of fields to return as part of a hit"
+          "description" : "A comma-separated list of stored fields to return as part of a hit"
+        },
+        "docvalue_fields": {
+          "type" : "list",
+          "description" : "A comma-separated list of fields to return as the docvalue representation of a field for each hit"
         },
         "fielddata_fields": {
           "type" : "list",
-          "description" : "A comma-separated list of fields to return as the field data representation of a field for each hit"
+          "description" : "A comma-separated list of fields to return as the docvalue representation of a field for each hit"
         },
         "from": {
           "type" : "number",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml
index e0ac2aea2df..caf44d0c788 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml
@@ -77,20 +77,25 @@
   - do:
       search:
         body:
-          fields: [ include.field2 ]
+          stored_fields: [ include.field2 ]
           query: { match_all: {} }
   - is_false:  hits.hits.0._source
 
   - do:
         search:
           body:
-            fields: [ include.field2, _source ]
+            stored_fields: [ include.field2, _source ]
             query: { match_all: {} }
   - match: { hits.hits.0._source.include.field2: v2 }
   - is_true:  hits.hits.0._source
 
+  - do:
+      search:
+        docvalue_fields: [ "count" ]
+  - match: { hits.hits.0.fields.count: [1] }
 
   - do:
       search:
         fielddata_fields: [ "count" ]
   - match: { hits.hits.0.fields.count: [1] }
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yaml
index df7322f12c8..993cbed2647 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yaml
@@ -31,6 +31,6 @@ setup:
                   term:
                       data: some
                   preference: _local
-              fields: [user,amount]
+              stored_fields: [user,amount]
 
 

From 6d04c1e78e24481d4144e10137d577839718f13e Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 22 Jun 2016 11:56:04 -0400
Subject: [PATCH 1029/1311] Remove duplicated read byte array methods

This commit removes duplicated methods for reading byte arrays in
StreamInput. One method would read a byte array by repeatedly calling
StreamInput#readByte in a loop, and the other would just call
StreamInput#readBytes. In this commit, we remove the former.

Relates #19023
---
 .../common/io/stream/StreamInput.java         | 19 +++++--------------
 1 file changed, 5 insertions(+), 14 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
index 30b5cd7f2cf..ceb909fb49a 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
@@ -439,7 +439,7 @@ public abstract class StreamInput extends InputStream {
             case 5:
                 return readBoolean();
             case 6:
-                return fastReadByteArray();
+                return readByteArray();
             case 7:
                 return readArrayList();
             case 8:
@@ -477,13 +477,6 @@ public abstract class StreamInput extends InputStream {
         }
     }
 
-    private byte[] fastReadByteArray() throws IOException {
-        int bytesSize = readVInt();
-        byte[] value = new byte[bytesSize];
-        readBytes(value, 0, bytesSize);
-        return value;
-    }
-
     @SuppressWarnings("unchecked")
     private List readArrayList() throws IOException {
         int size = readVInt();
@@ -609,12 +602,10 @@ public abstract class StreamInput extends InputStream {
     }
 
     public byte[] readByteArray() throws IOException {
-        int length = readVInt();
-        byte[] values = new byte[length];
-        for (int i = 0; i < length; i++) {
-            values[i] = readByte();
-        }
-        return values;
+        final int length = readVInt();
+        final byte[] bytes = new byte[length];
+        readBytes(bytes, 0, bytes.length);
+        return bytes;
     }
 
     /**

From 6574243077a336c189edefa42be3ad2893d69b6b Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 22 Jun 2016 12:03:11 -0400
Subject: [PATCH 1030/1311] Fail to start if plugin tries broken onModule

If a plugin declares `onModule(SomethingThatIsntAModule)` then refuse
to start. Before this commit we just logged a warning that flies by in
the console and is easy to miss. You can't miss refusing to start!
---
 .../java/org/elasticsearch/plugins/PluginsService.java | 10 ++++++++--
 docs/reference/migration/migrate_5_0/plugins.asciidoc  |  5 +++++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
index 9b983fa2931..71081740d33 100644
--- a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
+++ b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
@@ -44,6 +44,7 @@ import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.script.NativeScriptFactory;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptEngineService;
+import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.threadpool.ExecutorBuilder;
 
 import java.io.IOException;
@@ -206,8 +207,13 @@ public class PluginsService extends AbstractComponent {
                 }
                 Class moduleClass = method.getParameterTypes()[0];
                 if (!Module.class.isAssignableFrom(moduleClass)) {
-                    logger.warn("Plugin: {} implementing onModule by the type is not of Module type {}", pluginEntry.v1().getName(), moduleClass);
-                    continue;
+                    if (moduleClass == ScriptModule.class) {
+                        // This is still part of the Plugin class to point the user to the new implementation
+                        continue;
+                    }
+                    throw new RuntimeException(
+                            "Plugin: [" + pluginEntry.v1().getName() + "] implements onModule taking a parameter that isn't a Module ["
+                                    + moduleClass.getSimpleName() + "]");
                 }
                 list.add(new OnModuleReference(moduleClass, method));
             }
diff --git a/docs/reference/migration/migrate_5_0/plugins.asciidoc b/docs/reference/migration/migrate_5_0/plugins.asciidoc
index 1578ae074df..a1c0dad9ca1 100644
--- a/docs/reference/migration/migrate_5_0/plugins.asciidoc
+++ b/docs/reference/migration/migrate_5_0/plugins.asciidoc
@@ -117,3 +117,8 @@ via ES_JAVA_OPTS.
 
 The ability to specify a custom plugins path via `path.plugins` has
 been removed.
+
+==== ScriptPlugin
+
+Plugins that register custom scripts should implement `ScriptPlugin` and remove
+their `onModule(ScriptModule)` implementation.

From 6671c0cf09ffe9dbc31936b25524df88d29612ed Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 22 Jun 2016 12:34:59 -0400
Subject: [PATCH 1031/1311] Tasks: Add completed to the mapping

---
 .../resources/org/elasticsearch/tasks/task-index-mapping.json  | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/core/src/main/resources/org/elasticsearch/tasks/task-index-mapping.json b/core/src/main/resources/org/elasticsearch/tasks/task-index-mapping.json
index 262519fd381..4863cdb539b 100644
--- a/core/src/main/resources/org/elasticsearch/tasks/task-index-mapping.json
+++ b/core/src/main/resources/org/elasticsearch/tasks/task-index-mapping.json
@@ -2,6 +2,9 @@
   "task" : {
     "dynamic" : "strict",
     "properties" : {
+      "completed": {
+        "type": "boolean"
+      },
       "task" : {
         "properties": {
           "action": {

From 5e960c0d0350b84a526907bea333fd1185bf3eac Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 22 Jun 2016 13:27:19 -0400
Subject: [PATCH 1032/1311] Move upgrade test to upgrade from version 2.3.3

This commit moves the upgrade test to test upgrading from version 2.3.3
instead of from version 2.0.0.

Relates #19029
---
 qa/vagrant/build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle
index 211f8ffa5d1..58e7e50e421 100644
--- a/qa/vagrant/build.gradle
+++ b/qa/vagrant/build.gradle
@@ -58,7 +58,7 @@ for (String box : getProperties().get('vagrant.boxes', 'sample').split(',')) {
 
 /* The version of elasticsearch that we upgrade *from* as part of testing
  * upgrades. */
-String upgradeFromVersion = '2.0.0'
+String upgradeFromVersion = '2.3.3'
 
 configurations {
   test

From c2839c1577241c30a8d718e13382cd053a162a87 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Wed, 22 Jun 2016 19:22:48 +0200
Subject: [PATCH 1033/1311] [TEST] Add client-test module and make client tests
 use randomized runner directly

The lucene-test dependency caused issues with IDEs as they would always load the lucene 5 jar although they shouldn't have, which caused jarhell in es core tests.

If we depend directly on randomized runner we don't have this problem. It is luckily still compatible with java 1.7. This requires though adding a thin module that includes the base test class which can be shared between client and client-sniffer.
---
 build.gradle                                  |  1 +
 client-sniffer/build.gradle                   |  6 +-
 .../sniff/HostsSnifferBuilderTests.java       | 20 ++++---
 .../client/sniff/HostsSnifferTests.java       | 41 +++++++------
 .../sniff/SniffOnFailureListenerTests.java    |  7 ++-
 .../client/sniff/SnifferBuilderTests.java     | 23 ++++---
 client-test/build.gradle                      | 60 +++++++++++++++++++
 .../client/RestClientTestCase.java            | 44 ++++++++++++++
 .../client/RestClientTestUtil.java            |  0
 client/build.gradle                           |  6 +-
 .../client/CloseableBasicHttpResponse.java    |  2 +-
 .../client/RequestLoggerTests.java            | 20 +++----
 .../client/RestClientBuilderTests.java        | 21 ++++---
 .../client/RestClientIntegTests.java          | 29 +++++----
 .../client/RestClientMultipleHostsTests.java  | 42 +++++++------
 .../client/RestClientSingleHostTests.java     | 54 +++++++++--------
 settings.gradle                               |  1 +
 17 files changed, 255 insertions(+), 122 deletions(-)
 create mode 100644 client-test/build.gradle
 create mode 100644 client-test/src/main/java/org/elasticsearch/client/RestClientTestCase.java
 rename {client/src/test => client-test/src/main}/java/org/elasticsearch/client/RestClientTestUtil.java (100%)

diff --git a/build.gradle b/build.gradle
index 436cc99d8f9..32fde425793 100644
--- a/build.gradle
+++ b/build.gradle
@@ -165,6 +165,7 @@ subprojects {
     "org.elasticsearch:elasticsearch:${version}": ':core',
     "org.elasticsearch:client:${version}": ':client',
     "org.elasticsearch:client-sniffer:${version}": ':client-sniffer',
+    "org.elasticsearch:client-test:${version}": ':client-test',
     "org.elasticsearch.test:framework:${version}": ':test:framework',
     "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:integ-test-zip',
     "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:zip',
diff --git a/client-sniffer/build.gradle b/client-sniffer/build.gradle
index 402bc2f61b4..ffd776ccae2 100644
--- a/client-sniffer/build.gradle
+++ b/client-sniffer/build.gradle
@@ -34,13 +34,10 @@ dependencies {
   compile "commons-logging:commons-logging:${versions.commonslogging}"
   compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
 
+  testCompile "org.elasticsearch:client-test:${version}"
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
-  //we use the last lucene-test version that is compatible with java 1.7
-  testCompile "org.apache.lucene:lucene-test-framework:5.5.1"
-  testCompile "org.apache.lucene:lucene-core:5.5.1"
-  testCompile "org.apache.lucene:lucene-codecs:5.5.1"
   testCompile "org.elasticsearch:securemock:${versions.securemock}"
   testCompile "org.codehaus.mojo:animal-sniffer-annotations:1.15"
   signature "org.codehaus.mojo.signature:java17:1.0@signature"
@@ -63,6 +60,7 @@ forbiddenApisTest {
 jarHell.enabled=false
 
 namingConventions {
+  testClass = 'org.elasticsearch.client.RestClientTestCase'
   //we don't have integration tests
   skipIntegTestInDisguise = true
 }
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferBuilderTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferBuilderTests.java
index ff9c150d954..c167a3a104b 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferBuilderTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferBuilderTests.java
@@ -22,10 +22,14 @@ package org.elasticsearch.client.sniff;
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import org.apache.http.HttpHost;
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientTestCase;
 
-public class HostsSnifferBuilderTests extends LuceneTestCase {
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+public class HostsSnifferBuilderTests extends RestClientTestCase {
 
     public void testBuild() throws Exception {
         try {
@@ -35,7 +39,7 @@ public class HostsSnifferBuilderTests extends LuceneTestCase {
             assertEquals(e.getMessage(), "restClient cannot be null");
         }
 
-        int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
+        int numNodes = RandomInts.randomIntBetween(getRandom(), 1, 5);
         HttpHost[] hosts = new HttpHost[numNodes];
         for (int i = 0; i < numNodes; i++) {
             hosts[i] = new HttpHost("localhost", 9200 + i);
@@ -50,18 +54,18 @@ public class HostsSnifferBuilderTests extends LuceneTestCase {
             }
 
             try {
-                HostsSniffer.builder(client).setSniffRequestTimeoutMillis(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+                HostsSniffer.builder(client).setSniffRequestTimeoutMillis(RandomInts.randomIntBetween(getRandom(), Integer.MIN_VALUE, 0));
                 fail("should have failed");
             } catch(IllegalArgumentException e) {
                 assertEquals(e.getMessage(), "sniffRequestTimeoutMillis must be greater than 0");
             }
 
             HostsSniffer.Builder builder = HostsSniffer.builder(client);
-            if (random().nextBoolean()) {
-                builder.setScheme(RandomPicks.randomFrom(random(), HostsSniffer.Scheme.values()));
+            if (getRandom().nextBoolean()) {
+                builder.setScheme(RandomPicks.randomFrom(getRandom(), HostsSniffer.Scheme.values()));
             }
-            if (random().nextBoolean()) {
-                builder.setSniffRequestTimeoutMillis(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+            if (getRandom().nextBoolean()) {
+                builder.setSniffRequestTimeoutMillis(RandomInts.randomIntBetween(getRandom(), 1, Integer.MAX_VALUE));
             }
             assertNotNull(builder.build());
         }
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
index c6daf9c6f40..1b38f034262 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
@@ -30,11 +30,11 @@ import com.sun.net.httpserver.HttpServer;
 import org.apache.http.Consts;
 import org.apache.http.HttpHost;
 import org.apache.http.client.methods.HttpGet;
-import org.apache.lucene.util.LuceneTestCase;
 import org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientTestCase;
 import org.junit.After;
 import org.junit.Before;
 
@@ -55,10 +55,13 @@ import java.util.Set;
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
 
 //animal-sniffer doesn't like our usage of com.sun.net.httpserver.* classes
 @IgnoreJRERequirement
-public class HostsSnifferTests extends LuceneTestCase {
+public class HostsSnifferTests extends RestClientTestCase {
 
     private int sniffRequestTimeout;
     private HostsSniffer.Scheme scheme;
@@ -67,8 +70,8 @@ public class HostsSnifferTests extends LuceneTestCase {
 
     @Before
     public void startHttpServer() throws IOException {
-        this.sniffRequestTimeout = RandomInts.randomIntBetween(random(), 1000, 10000);
-        this.scheme = RandomPicks.randomFrom(random(), HostsSniffer.Scheme.values());
+        this.sniffRequestTimeout = RandomInts.randomIntBetween(getRandom(), 1000, 10000);
+        this.scheme = RandomPicks.randomFrom(getRandom(), HostsSniffer.Scheme.values());
         if (rarely()) {
             this.sniffResponse = SniffResponse.buildFailure();
         } else {
@@ -148,28 +151,28 @@ public class HostsSnifferTests extends LuceneTestCase {
     }
 
     private static SniffResponse buildSniffResponse(HostsSniffer.Scheme scheme) throws IOException {
-        int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
+        int numNodes = RandomInts.randomIntBetween(getRandom(), 1, 5);
         List<HttpHost> hosts = new ArrayList<>(numNodes);
         JsonFactory jsonFactory = new JsonFactory();
         StringWriter writer = new StringWriter();
         JsonGenerator generator = jsonFactory.createGenerator(writer);
         generator.writeStartObject();
-        if (random().nextBoolean()) {
+        if (getRandom().nextBoolean()) {
             generator.writeStringField("cluster_name", "elasticsearch");
         }
-        if (random().nextBoolean()) {
+        if (getRandom().nextBoolean()) {
             generator.writeObjectFieldStart("bogus_object");
             generator.writeEndObject();
         }
         generator.writeObjectFieldStart("nodes");
         for (int i = 0; i < numNodes; i++) {
-            String nodeId = RandomStrings.randomAsciiOfLengthBetween(random(), 5, 10);
+            String nodeId = RandomStrings.randomAsciiOfLengthBetween(getRandom(), 5, 10);
             generator.writeObjectFieldStart(nodeId);
-            if (random().nextBoolean()) {
+            if (getRandom().nextBoolean()) {
                 generator.writeObjectFieldStart("bogus_object");
                 generator.writeEndObject();
             }
-            if (random().nextBoolean()) {
+            if (getRandom().nextBoolean()) {
                 generator.writeArrayFieldStart("bogus_array");
                 generator.writeStartObject();
                 generator.writeEndObject();
@@ -178,35 +181,35 @@ public class HostsSnifferTests extends LuceneTestCase {
             boolean isHttpEnabled = rarely() == false;
             if (isHttpEnabled) {
                 String host = "host" + i;
-                int port = RandomInts.randomIntBetween(random(), 9200, 9299);
+                int port = RandomInts.randomIntBetween(getRandom(), 9200, 9299);
                 HttpHost httpHost = new HttpHost(host, port, scheme.toString());
                 hosts.add(httpHost);
                 generator.writeObjectFieldStart("http");
-                if (random().nextBoolean()) {
+                if (getRandom().nextBoolean()) {
                     generator.writeArrayFieldStart("bound_address");
                     generator.writeString("[fe80::1]:" + port);
                     generator.writeString("[::1]:" + port);
                     generator.writeString("127.0.0.1:" + port);
                     generator.writeEndArray();
                 }
-                if (random().nextBoolean()) {
+                if (getRandom().nextBoolean()) {
                     generator.writeObjectFieldStart("bogus_object");
                     generator.writeEndObject();
                 }
                 generator.writeStringField("publish_address", httpHost.toHostString());
-                if (random().nextBoolean()) {
+                if (getRandom().nextBoolean()) {
                     generator.writeNumberField("max_content_length_in_bytes", 104857600);
                 }
                 generator.writeEndObject();
             }
-            if (random().nextBoolean()) {
+            if (getRandom().nextBoolean()) {
                 String[] roles = {"master", "data", "ingest"};
-                int numRoles = RandomInts.randomIntBetween(random(), 0, 3);
+                int numRoles = RandomInts.randomIntBetween(getRandom(), 0, 3);
                 Set<String> nodeRoles = new HashSet<>(numRoles);
                 for (int j = 0; j < numRoles; j++) {
                     String role;
                     do {
-                        role = RandomPicks.randomFrom(random(), roles);
+                        role = RandomPicks.randomFrom(getRandom(), roles);
                     } while(nodeRoles.add(role) == false);
                 }
                 generator.writeArrayFieldStart("roles");
@@ -215,7 +218,7 @@ public class HostsSnifferTests extends LuceneTestCase {
                 }
                 generator.writeEndArray();
             }
-            int numAttributes = RandomInts.randomIntBetween(random(), 0, 3);
+            int numAttributes = RandomInts.randomIntBetween(getRandom(), 0, 3);
             Map<String, String> attributes = new HashMap<>(numAttributes);
             for (int j = 0; j < numAttributes; j++) {
                 attributes.put("attr" + j, "value" + j);
@@ -264,6 +267,6 @@ public class HostsSnifferTests extends LuceneTestCase {
     }
 
     private static int randomErrorResponseCode() {
-        return RandomInts.randomIntBetween(random(), 400, 599);
+        return RandomInts.randomIntBetween(getRandom(), 400, 599);
     }
 }
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SniffOnFailureListenerTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SniffOnFailureListenerTests.java
index fe2555763ac..6a71d72f60e 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SniffOnFailureListenerTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SniffOnFailureListenerTests.java
@@ -20,10 +20,13 @@
 package org.elasticsearch.client.sniff;
 
 import org.apache.http.HttpHost;
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientTestCase;
 
-public class SniffOnFailureListenerTests extends LuceneTestCase {
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class SniffOnFailureListenerTests extends RestClientTestCase {
 
     public void testSetSniffer() throws Exception {
         SniffOnFailureListener listener = new SniffOnFailureListener();
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
index a3e8d8e80e7..defa83554a4 100644
--- a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
+++ b/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
@@ -21,13 +21,17 @@ package org.elasticsearch.client.sniff;
 
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
 import org.apache.http.HttpHost;
-import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientTestCase;
 
-public class SnifferBuilderTests extends LuceneTestCase {
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+public class SnifferBuilderTests extends RestClientTestCase {
 
     public void testBuild() throws Exception {
-        int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
+        int numNodes = RandomInts.randomIntBetween(getRandom(), 1, 5);
         HttpHost[] hosts = new HttpHost[numNodes];
         for (int i = 0; i < numNodes; i++) {
             hosts[i] = new HttpHost("localhost", 9200 + i);
@@ -51,7 +55,8 @@ public class SnifferBuilderTests extends LuceneTestCase {
             }
 
             try {
-                Sniffer.builder(client, hostsSniffer).setSniffIntervalMillis(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+                Sniffer.builder(client, hostsSniffer)
+                        .setSniffIntervalMillis(RandomInts.randomIntBetween(getRandom(), Integer.MIN_VALUE, 0));
                 fail("should have failed");
             } catch(IllegalArgumentException e) {
                 assertEquals("sniffIntervalMillis must be greater than 0", e.getMessage());
@@ -59,7 +64,7 @@ public class SnifferBuilderTests extends LuceneTestCase {
 
             try {
                 Sniffer.builder(client, hostsSniffer)
-                        .setSniffAfterFailureDelayMillis(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+                        .setSniffAfterFailureDelayMillis(RandomInts.randomIntBetween(getRandom(), Integer.MIN_VALUE, 0));
                 fail("should have failed");
             } catch(IllegalArgumentException e) {
                 assertEquals("sniffAfterFailureDelayMillis must be greater than 0", e.getMessage());
@@ -70,11 +75,11 @@ public class SnifferBuilderTests extends LuceneTestCase {
             }
 
             Sniffer.Builder builder = Sniffer.builder(client, hostsSniffer);
-            if (random().nextBoolean()) {
-                builder.setSniffIntervalMillis(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+            if (getRandom().nextBoolean()) {
+                builder.setSniffIntervalMillis(RandomInts.randomIntBetween(getRandom(), 1, Integer.MAX_VALUE));
             }
-            if (random().nextBoolean()) {
-                builder.setSniffAfterFailureDelayMillis(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+            if (getRandom().nextBoolean()) {
+                builder.setSniffAfterFailureDelayMillis(RandomInts.randomIntBetween(getRandom(), 1, Integer.MAX_VALUE));
             }
             try (Sniffer sniffer = builder.build()) {
                 assertNotNull(sniffer);
diff --git a/client-test/build.gradle b/client-test/build.gradle
new file mode 100644
index 00000000000..1296b9fa5b8
--- /dev/null
+++ b/client-test/build.gradle
@@ -0,0 +1,60 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.elasticsearch.gradle.precommit.PrecommitTasks
+import org.gradle.api.JavaVersion
+
+apply plugin: 'elasticsearch.build'
+apply plugin: 'ru.vyarus.animalsniffer'
+
+targetCompatibility = JavaVersion.VERSION_1_7
+sourceCompatibility = JavaVersion.VERSION_1_7
+
+dependencies {
+  compile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
+  compile "junit:junit:${versions.junit}"
+  compile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
+  compile "org.codehaus.mojo:animal-sniffer-annotations:1.15"
+  signature "org.codehaus.mojo.signature:java17:1.0@signature"
+}
+
+forbiddenApisMain {
+  //client does not depend on core, so only jdk signatures should be checked
+  signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
+}
+
+forbiddenApisTest {
+  //we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
+  bundledSignatures -= 'jdk-non-portable'
+  bundledSignatures += 'jdk-internal'
+  //client does not depend on core, so only jdk signatures should be checked
+  signaturesURLs = [PrecommitTasks.getResource('/forbidden/jdk-signatures.txt')]
+}
+
+//JarHell is part of es core, which we don't want to pull in
+jarHell.enabled=false
+
+// TODO: should we have licenses for our test deps?
+dependencyLicenses.enabled = false
+
+namingConventions.enabled = false
+
+//we aren't releasing this jar
+thirdPartyAudit.enabled = false
+test.enabled = false
\ No newline at end of file
diff --git a/client-test/src/main/java/org/elasticsearch/client/RestClientTestCase.java b/client-test/src/main/java/org/elasticsearch/client/RestClientTestCase.java
new file mode 100644
index 00000000000..30305478d77
--- /dev/null
+++ b/client-test/src/main/java/org/elasticsearch/client/RestClientTestCase.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import com.carrotsearch.randomizedtesting.JUnit3MethodProvider;
+import com.carrotsearch.randomizedtesting.MixWithSuiteName;
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+import com.carrotsearch.randomizedtesting.annotations.SeedDecorators;
+import com.carrotsearch.randomizedtesting.annotations.TestMethodProviders;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakAction;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakGroup;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies;
+import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
+
+@TestMethodProviders({
+        JUnit3MethodProvider.class
+})
+@SeedDecorators({MixWithSuiteName.class}) // See LUCENE-3995 for rationale.
+@ThreadLeakScope(ThreadLeakScope.Scope.SUITE)
+@ThreadLeakGroup(ThreadLeakGroup.Group.MAIN)
+@ThreadLeakAction({ThreadLeakAction.Action.WARN, ThreadLeakAction.Action.INTERRUPT})
+@ThreadLeakZombies(ThreadLeakZombies.Consequence.IGNORE_REMAINING_TESTS)
+@TimeoutSuite(millis = 2 * 60 * 60 * 1000)
+public abstract class RestClientTestCase extends RandomizedTest {
+
+}
\ No newline at end of file
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientTestUtil.java b/client-test/src/main/java/org/elasticsearch/client/RestClientTestUtil.java
similarity index 100%
rename from client/src/test/java/org/elasticsearch/client/RestClientTestUtil.java
rename to client-test/src/main/java/org/elasticsearch/client/RestClientTestUtil.java
diff --git a/client/build.gradle b/client/build.gradle
index a91c224c745..2a2e77f88a4 100644
--- a/client/build.gradle
+++ b/client/build.gradle
@@ -32,13 +32,10 @@ dependencies {
   compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
 
+  testCompile "org.elasticsearch:client-test:${version}"
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
-  //we use the last lucene-test version that is compatible with java 1.7
-  testCompile "org.apache.lucene:lucene-test-framework:5.5.1"
-  testCompile "org.apache.lucene:lucene-core:5.5.1"
-  testCompile "org.apache.lucene:lucene-codecs:5.5.1"
   testCompile "org.elasticsearch:securemock:${versions.securemock}"
   testCompile "org.codehaus.mojo:animal-sniffer-annotations:1.15"
   signature "org.codehaus.mojo.signature:java17:1.0@signature"
@@ -61,6 +58,7 @@ forbiddenApisTest {
 jarHell.enabled=false
 
 namingConventions {
+  testClass = 'org.elasticsearch.client.RestClientTestCase'
   //we don't have integration tests
   skipIntegTestInDisguise = true
 }
diff --git a/client/src/test/java/org/elasticsearch/client/CloseableBasicHttpResponse.java b/client/src/test/java/org/elasticsearch/client/CloseableBasicHttpResponse.java
index 904cbe7cfeb..dd866bac541 100644
--- a/client/src/test/java/org/elasticsearch/client/CloseableBasicHttpResponse.java
+++ b/client/src/test/java/org/elasticsearch/client/CloseableBasicHttpResponse.java
@@ -39,4 +39,4 @@ class CloseableBasicHttpResponse extends BasicHttpResponse implements CloseableH
     public void close() throws IOException {
         //nothing to close
     }
-}
+}
\ No newline at end of file
diff --git a/client/src/test/java/org/elasticsearch/client/RequestLoggerTests.java b/client/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
index c16d9e7876e..190774d46fc 100644
--- a/client/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
@@ -36,7 +36,6 @@ import org.apache.http.entity.StringEntity;
 import org.apache.http.message.BasicHttpResponse;
 import org.apache.http.message.BasicStatusLine;
 import org.apache.http.util.EntityUtils;
-import org.apache.lucene.util.LuceneTestCase;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
@@ -45,15 +44,16 @@ import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
 
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertThat;
 
-public class RequestLoggerTests extends LuceneTestCase {
+public class RequestLoggerTests extends RestClientTestCase {
 
     public void testTraceRequest() throws IOException, URISyntaxException {
-        HttpHost host = new HttpHost("localhost", 9200, random().nextBoolean() ? "http" : "https");
+        HttpHost host = new HttpHost("localhost", 9200, getRandom().nextBoolean() ? "http" : "https");
         URI uri = new URI("/index/type/_api");
 
         HttpRequestBase request;
-        int requestType = RandomInts.randomIntBetween(random(), 0, 7);
+        int requestType = RandomInts.randomIntBetween(getRandom(), 0, 7);
         switch(requestType) {
             case 0:
                 request = new HttpGetWithEntity(uri);
@@ -84,13 +84,13 @@ public class RequestLoggerTests extends LuceneTestCase {
         }
 
         String expected = "curl -iX " + request.getMethod() + " '" + host + uri + "'";
-        boolean hasBody = request instanceof HttpEntityEnclosingRequest && random().nextBoolean();
+        boolean hasBody = request instanceof HttpEntityEnclosingRequest && getRandom().nextBoolean();
         String requestBody = "{ \"field\": \"value\" }";
         if (hasBody) {
             expected += " -d '" + requestBody + "'";
             HttpEntityEnclosingRequest enclosingRequest = (HttpEntityEnclosingRequest) request;
             HttpEntity entity;
-            if (random().nextBoolean()) {
+            if (getRandom().nextBoolean()) {
                 entity = new StringEntity(requestBody, StandardCharsets.UTF_8);
             } else {
                 entity = new InputStreamEntity(new ByteArrayInputStream(requestBody.getBytes(StandardCharsets.UTF_8)));
@@ -109,25 +109,25 @@ public class RequestLoggerTests extends LuceneTestCase {
 
     public void testTraceResponse() throws IOException {
         ProtocolVersion protocolVersion = new ProtocolVersion("HTTP", 1, 1);
-        int statusCode = RandomInts.randomIntBetween(random(), 200, 599);
+        int statusCode = RandomInts.randomIntBetween(getRandom(), 200, 599);
         String reasonPhrase = "REASON";
         BasicStatusLine statusLine = new BasicStatusLine(protocolVersion, statusCode, reasonPhrase);
         String expected = "# " + statusLine.toString();
         BasicHttpResponse httpResponse = new BasicHttpResponse(statusLine);
-        int numHeaders = RandomInts.randomIntBetween(random(), 0, 3);
+        int numHeaders = RandomInts.randomIntBetween(getRandom(), 0, 3);
         for (int i = 0; i < numHeaders; i++) {
             httpResponse.setHeader("header" + i, "value");
             expected += "\n# header" + i + ": value";
         }
         expected += "\n#";
-        boolean hasBody = random().nextBoolean();
+        boolean hasBody = getRandom().nextBoolean();
         String responseBody = "{\n  \"field\": \"value\"\n}";
         if (hasBody) {
             expected += "\n# {";
             expected += "\n#   \"field\": \"value\"";
             expected += "\n# }";
             HttpEntity entity;
-            if (random().nextBoolean()) {
+            if (getRandom().nextBoolean()) {
                 entity = new StringEntity(responseBody, StandardCharsets.UTF_8);
             } else {
                 entity = new InputStreamEntity(new ByteArrayInputStream(responseBody.getBytes(StandardCharsets.UTF_8)));
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java b/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
index cb7de44cd5e..a16e961fd28 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
@@ -24,11 +24,14 @@ import org.apache.http.Header;
 import org.apache.http.HttpHost;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.message.BasicHeader;
-import org.apache.lucene.util.LuceneTestCase;
 
 import java.io.IOException;
 
-public class RestClientBuilderTests extends LuceneTestCase {
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+public class RestClientBuilderTests extends RestClientTestCase {
 
     public void testBuild() throws IOException {
         try {
@@ -54,7 +57,7 @@ public class RestClientBuilderTests extends LuceneTestCase {
 
         try {
             RestClient.builder(new HttpHost("localhost", 9200))
-                    .setMaxRetryTimeoutMillis(RandomInts.randomIntBetween(random(), Integer.MIN_VALUE, 0));
+                    .setMaxRetryTimeoutMillis(RandomInts.randomIntBetween(getRandom(), Integer.MIN_VALUE, 0));
             fail("should have failed");
         } catch(IllegalArgumentException e) {
             assertEquals("maxRetryTimeoutMillis must be greater than 0", e.getMessage());
@@ -81,25 +84,25 @@ public class RestClientBuilderTests extends LuceneTestCase {
             assertEquals("failure listener must not be null", e.getMessage());
         }
 
-        int numNodes = RandomInts.randomIntBetween(random(), 1, 5);
+        int numNodes = RandomInts.randomIntBetween(getRandom(), 1, 5);
         HttpHost[] hosts = new HttpHost[numNodes];
         for (int i = 0; i < numNodes; i++) {
             hosts[i] = new HttpHost("localhost", 9200 + i);
         }
         RestClient.Builder builder = RestClient.builder(hosts);
-        if (random().nextBoolean()) {
+        if (getRandom().nextBoolean()) {
             builder.setHttpClient(HttpClientBuilder.create().build());
         }
-        if (random().nextBoolean()) {
-            int numHeaders = RandomInts.randomIntBetween(random(), 1, 5);
+        if (getRandom().nextBoolean()) {
+            int numHeaders = RandomInts.randomIntBetween(getRandom(), 1, 5);
             Header[] headers = new Header[numHeaders];
             for (int i = 0; i < numHeaders; i++) {
                 headers[i] = new BasicHeader("header" + i, "value");
             }
             builder.setDefaultHeaders(headers);
         }
-        if (random().nextBoolean()) {
-            builder.setMaxRetryTimeoutMillis(RandomInts.randomIntBetween(random(), 1, Integer.MAX_VALUE));
+        if (getRandom().nextBoolean()) {
+            builder.setMaxRetryTimeoutMillis(RandomInts.randomIntBetween(getRandom(), 1, Integer.MAX_VALUE));
         }
         try (RestClient restClient = builder.build()) {
             assertNotNull(restClient);
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java b/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
index 7bc6dc76c91..4a14c174353 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
@@ -31,7 +31,6 @@ import org.apache.http.HttpHost;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.message.BasicHeader;
 import org.apache.http.util.EntityUtils;
-import org.apache.lucene.util.LuceneTestCase;
 import org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -53,6 +52,10 @@ import static org.elasticsearch.client.RestClientTestUtil.getAllStatusCodes;
 import static org.elasticsearch.client.RestClientTestUtil.getHttpMethods;
 import static org.elasticsearch.client.RestClientTestUtil.randomStatusCode;
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
 
 /**
  * Integration test to check interaction between {@link RestClient} and {@link org.apache.http.client.HttpClient}.
@@ -60,7 +63,7 @@ import static org.hamcrest.CoreMatchers.equalTo;
  */
 //animal-sniffer doesn't like our usage of com.sun.net.httpserver.* classes
 @IgnoreJRERequirement
-public class RestClientIntegTests extends LuceneTestCase {
+public class RestClientIntegTests extends RestClientTestCase {
 
     private static HttpServer httpServer;
     private static RestClient restClient;
@@ -74,11 +77,11 @@ public class RestClientIntegTests extends LuceneTestCase {
         for (int statusCode : getAllStatusCodes()) {
             createStatusCodeContext(httpServer, statusCode);
         }
-        int numHeaders = RandomInts.randomIntBetween(random(), 0, 3);
+        int numHeaders = RandomInts.randomIntBetween(getRandom(), 0, 3);
         defaultHeaders = new Header[numHeaders];
         for (int i = 0; i < numHeaders; i++) {
-            String headerName = "Header-default" + (random().nextBoolean() ? i : "");
-            String headerValue = RandomStrings.randomAsciiOfLengthBetween(random(), 3, 10);
+            String headerName = "Header-default" + (getRandom().nextBoolean() ? i : "");
+            String headerValue = RandomStrings.randomAsciiOfLengthBetween(getRandom(), 3, 10);
             defaultHeaders[i] = new BasicHeader(headerName, headerValue);
         }
         restClient = RestClient.builder(new HttpHost(httpServer.getAddress().getHostString(), httpServer.getAddress().getPort()))
@@ -144,20 +147,20 @@ public class RestClientIntegTests extends LuceneTestCase {
             if (method.equals("HEAD") == false) {
                 standardHeaders.add("Content-length");
             }
-            int numHeaders = RandomInts.randomIntBetween(random(), 1, 5);
+            int numHeaders = RandomInts.randomIntBetween(getRandom(), 1, 5);
             Map<String, String> expectedHeaders = new HashMap<>();
             for (Header defaultHeader : defaultHeaders) {
                 expectedHeaders.put(defaultHeader.getName(), defaultHeader.getValue());
             }
             Header[] headers = new Header[numHeaders];
             for (int i = 0; i < numHeaders; i++) {
-                String headerName = "Header" + (random().nextBoolean() ? i : "");
-                String headerValue = RandomStrings.randomAsciiOfLengthBetween(random(), 3, 10);
+                String headerName = "Header" + (getRandom().nextBoolean() ? i : "");
+                String headerValue = RandomStrings.randomAsciiOfLengthBetween(getRandom(), 3, 10);
                 headers[i] = new BasicHeader(headerName, headerValue);
                 expectedHeaders.put(headerName, headerValue);
             }
 
-            int statusCode = randomStatusCode(random());
+            int statusCode = randomStatusCode(getRandom());
             Response esResponse;
             try (Response response = restClient.performRequest(method, "/" + statusCode,
                     Collections.<String, String>emptyMap(), null, headers)) {
@@ -186,7 +189,7 @@ public class RestClientIntegTests extends LuceneTestCase {
      * Exercises the test http server ability to send back whatever body it received.
      */
     public void testDeleteWithBody() throws Exception {
-        testBody("DELETE");
+        bodyTest("DELETE");
     }
 
     /**
@@ -195,15 +198,15 @@ public class RestClientIntegTests extends LuceneTestCase {
      * Exercises the test http server ability to send back whatever body it received.
      */
     public void testGetWithBody() throws Exception {
-        testBody("GET");
+        bodyTest("GET");
     }
 
-    private void testBody(String method) throws Exception {
+    private void bodyTest(String method) throws Exception {
         String requestBody = "{ \"field\": \"value\" }";
         StringEntity entity = new StringEntity(requestBody);
         Response esResponse;
         String responseBody;
-        int statusCode = randomStatusCode(random());
+        int statusCode = randomStatusCode(getRandom());
         try (Response response = restClient.performRequest(method, "/" + statusCode,
                 Collections.<String, String>emptyMap(), entity)) {
             responseBody = EntityUtils.toString(response.getEntity());
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java b/client/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
index beae3cfc9f1..9d082466123 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.client;
 
+import com.carrotsearch.randomizedtesting.RandomizedTest;
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
 import org.apache.http.HttpHost;
 import org.apache.http.HttpRequest;
@@ -29,7 +30,6 @@ import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.http.conn.ConnectTimeoutException;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.message.BasicStatusLine;
-import org.apache.lucene.util.LuceneTestCase;
 import org.junit.Before;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
@@ -46,6 +46,10 @@ import static org.elasticsearch.client.RestClientTestUtil.randomHttpMethod;
 import static org.elasticsearch.client.RestClientTestUtil.randomOkStatusCode;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -54,7 +58,7 @@ import static org.mockito.Mockito.when;
  * Tests for {@link RestClient} behaviour against multiple hosts: fail-over, blacklisting etc.
  * Relies on a mock http client to intercept requests and return desired responses based on request path.
  */
-public class RestClientMultipleHostsTests extends LuceneTestCase {
+public class RestClientMultipleHostsTests extends RestClientTestCase {
 
     private RestClient restClient;
     private HttpHost[] httpHosts;
@@ -82,7 +86,7 @@ public class RestClientMultipleHostsTests extends LuceneTestCase {
             }
         });
 
-        int numHosts = RandomInts.randomIntBetween(random(), 2, 5);
+        int numHosts = RandomInts.randomIntBetween(getRandom(), 2, 5);
         httpHosts = new HttpHost[numHosts];
         for (int i = 0; i < numHosts; i++) {
             httpHosts[i] = new HttpHost("localhost", 9200 + i);
@@ -92,13 +96,13 @@ public class RestClientMultipleHostsTests extends LuceneTestCase {
     }
 
     public void testRoundRobinOkStatusCodes() throws Exception {
-        int numIters = RandomInts.randomIntBetween(random(), 1, 5);
+        int numIters = RandomInts.randomIntBetween(getRandom(), 1, 5);
         for (int i = 0; i < numIters; i++) {
             Set<HttpHost> hostsSet = new HashSet<>();
             Collections.addAll(hostsSet, httpHosts);
             for (int j = 0; j < httpHosts.length; j++) {
-                int statusCode = randomOkStatusCode(random());
-                try (Response response = restClient.performRequest(randomHttpMethod(random()), "/" + statusCode,
+                int statusCode = randomOkStatusCode(getRandom());
+                try (Response response = restClient.performRequest(randomHttpMethod(getRandom()), "/" + statusCode,
                         Collections.<String, String>emptyMap(), null)) {
                     assertThat(response.getStatusLine().getStatusCode(), equalTo(statusCode));
                     assertTrue("host not found: " + response.getHost(), hostsSet.remove(response.getHost()));
@@ -110,13 +114,13 @@ public class RestClientMultipleHostsTests extends LuceneTestCase {
     }
 
     public void testRoundRobinNoRetryErrors() throws Exception {
-        int numIters = RandomInts.randomIntBetween(random(), 1, 5);
+        int numIters = RandomInts.randomIntBetween(getRandom(), 1, 5);
         for (int i = 0; i < numIters; i++) {
             Set<HttpHost> hostsSet = new HashSet<>();
             Collections.addAll(hostsSet, httpHosts);
             for (int j = 0; j < httpHosts.length; j++) {
-                String method = randomHttpMethod(random());
-                int statusCode = randomErrorNoRetryStatusCode(random());
+                String method = randomHttpMethod(getRandom());
+                int statusCode = randomErrorNoRetryStatusCode(getRandom());
                 try (Response response = restClient.performRequest(method, "/" + statusCode,
                         Collections.<String, String>emptyMap(), null)) {
                     if (method.equals("HEAD") && statusCode == 404) {
@@ -145,7 +149,7 @@ public class RestClientMultipleHostsTests extends LuceneTestCase {
     public void testRoundRobinRetryErrors() throws Exception {
         String retryEndpoint = randomErrorRetryEndpoint();
         try  {
-            restClient.performRequest(randomHttpMethod(random()), retryEndpoint, Collections.<String, String>emptyMap(), null);
+            restClient.performRequest(randomHttpMethod(getRandom()), retryEndpoint, Collections.<String, String>emptyMap(), null);
             fail("request should have failed");
         } catch(ResponseException e) {
             Set<HttpHost> hostsSet = new HashSet<>();
@@ -187,7 +191,7 @@ public class RestClientMultipleHostsTests extends LuceneTestCase {
             assertEquals("every host should have been used but some weren't: " + hostsSet, 0, hostsSet.size());
         }
 
-        int numIters = RandomInts.randomIntBetween(random(), 2, 5);
+        int numIters = RandomInts.randomIntBetween(getRandom(), 2, 5);
         for (int i = 1; i <= numIters; i++) {
             //check that one different host is resurrected at each new attempt
             Set<HttpHost> hostsSet = new HashSet<>();
@@ -195,7 +199,7 @@ public class RestClientMultipleHostsTests extends LuceneTestCase {
             for (int j = 0; j < httpHosts.length; j++) {
                 retryEndpoint = randomErrorRetryEndpoint();
                 try  {
-                    restClient.performRequest(randomHttpMethod(random()), retryEndpoint, Collections.<String, String>emptyMap(), null);
+                    restClient.performRequest(randomHttpMethod(getRandom()), retryEndpoint, Collections.<String, String>emptyMap(), null);
                     fail("request should have failed");
                 } catch(ResponseException e) {
                     Response response = e.getResponse();
@@ -214,14 +218,14 @@ public class RestClientMultipleHostsTests extends LuceneTestCase {
                 }
             }
             assertEquals("every host should have been used but some weren't: " + hostsSet, 0, hostsSet.size());
-            if (random().nextBoolean()) {
+            if (getRandom().nextBoolean()) {
                 //mark one host back alive through a successful request and check that all requests after that are sent to it
                 HttpHost selectedHost = null;
-                int iters = RandomInts.randomIntBetween(random(), 2, 10);
+                int iters = RandomInts.randomIntBetween(getRandom(), 2, 10);
                 for (int y = 0; y < iters; y++) {
-                    int statusCode = randomErrorNoRetryStatusCode(random());
+                    int statusCode = randomErrorNoRetryStatusCode(getRandom());
                     Response response;
-                    try (Response esResponse = restClient.performRequest(randomHttpMethod(random()), "/" + statusCode,
+                    try (Response esResponse = restClient.performRequest(randomHttpMethod(getRandom()), "/" + statusCode,
                             Collections.<String, String>emptyMap(), null)) {
                         response = esResponse;
                     }
@@ -241,7 +245,7 @@ public class RestClientMultipleHostsTests extends LuceneTestCase {
                 for (int y = 0; y < i + 1; y++) {
                     retryEndpoint = randomErrorRetryEndpoint();
                     try {
-                        restClient.performRequest(randomHttpMethod(random()), retryEndpoint,
+                        restClient.performRequest(randomHttpMethod(getRandom()), retryEndpoint,
                                 Collections.<String, String>emptyMap(), null);
                         fail("request should have failed");
                     } catch(ResponseException e) {
@@ -260,9 +264,9 @@ public class RestClientMultipleHostsTests extends LuceneTestCase {
     }
 
     private static String randomErrorRetryEndpoint() {
-        switch(RandomInts.randomIntBetween(random(), 0, 3)) {
+        switch(RandomInts.randomIntBetween(getRandom(), 0, 3)) {
             case 0:
-                return "/" + randomErrorRetryStatusCode(random());
+                return "/" + randomErrorRetryStatusCode(getRandom());
             case 1:
                 return "/coe";
             case 2:
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java b/client/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
index ef2a09fb570..59e88b0ed63 100644
--- a/client/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
+++ b/client/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.client;
 
+import com.carrotsearch.randomizedtesting.RandomizedTest;
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 import org.apache.http.Header;
@@ -43,7 +44,6 @@ import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.message.BasicHeader;
 import org.apache.http.message.BasicStatusLine;
 import org.apache.http.util.EntityUtils;
-import org.apache.lucene.util.LuceneTestCase;
 import org.junit.Before;
 import org.mockito.ArgumentCaptor;
 import org.mockito.invocation.InvocationOnMock;
@@ -65,6 +65,12 @@ import static org.elasticsearch.client.RestClientTestUtil.randomHttpMethod;
 import static org.elasticsearch.client.RestClientTestUtil.randomStatusCode;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
@@ -76,7 +82,7 @@ import static org.mockito.Mockito.when;
  * body, different status codes and corresponding responses/exceptions.
  * Relies on a mock http client to intercept requests and return desired responses based on request path.
  */
-public class RestClientSingleHostTests extends LuceneTestCase {
+public class RestClientSingleHostTests extends RestClientTestCase {
 
     private RestClient restClient;
     private Header[] defaultHeaders;
@@ -114,11 +120,11 @@ public class RestClientSingleHostTests extends LuceneTestCase {
                 return httpResponse;
             }
         });
-        int numHeaders = RandomInts.randomIntBetween(random(), 0, 3);
+        int numHeaders = RandomInts.randomIntBetween(getRandom(), 0, 3);
         defaultHeaders = new Header[numHeaders];
         for (int i = 0; i < numHeaders; i++) {
-            String headerName = "Header-default" + (random().nextBoolean() ? i : "");
-            String headerValue = RandomStrings.randomAsciiOfLengthBetween(random(), 3, 10);
+            String headerName = "Header-default" + (getRandom().nextBoolean() ? i : "");
+            String headerValue = RandomStrings.randomAsciiOfLengthBetween(getRandom(), 3, 10);
             defaultHeaders[i] = new BasicHeader(headerName, headerValue);
         }
         httpHost = new HttpHost("localhost", 9200);
@@ -269,7 +275,7 @@ public class RestClientSingleHostTests extends LuceneTestCase {
         }
         for (String method : Arrays.asList("HEAD", "OPTIONS", "TRACE")) {
             try {
-                restClient.performRequest(method, "/" + randomStatusCode(random()),
+                restClient.performRequest(method, "/" + randomStatusCode(getRandom()),
                         Collections.<String, String>emptyMap(), entity);
                 fail("request should have failed");
             } catch(UnsupportedOperationException e) {
@@ -279,8 +285,8 @@ public class RestClientSingleHostTests extends LuceneTestCase {
     }
 
     public void testNullHeaders() throws Exception {
-        String method = randomHttpMethod(random());
-        int statusCode = randomStatusCode(random());
+        String method = randomHttpMethod(getRandom());
+        int statusCode = randomStatusCode(getRandom());
         try {
             restClient.performRequest(method, "/" + statusCode, Collections.<String, String>emptyMap(), null, (Header[])null);
             fail("request should have failed");
@@ -296,8 +302,8 @@ public class RestClientSingleHostTests extends LuceneTestCase {
     }
 
     public void testNullParams() throws Exception {
-        String method = randomHttpMethod(random());
-        int statusCode = randomStatusCode(random());
+        String method = randomHttpMethod(getRandom());
+        int statusCode = randomStatusCode(getRandom());
         try {
             restClient.performRequest(method, "/" + statusCode, null, null);
             fail("request should have failed");
@@ -316,16 +322,16 @@ public class RestClientSingleHostTests extends LuceneTestCase {
             for (Header defaultHeader : defaultHeaders) {
                 expectedHeaders.put(defaultHeader.getName(), defaultHeader.getValue());
             }
-            int numHeaders = RandomInts.randomIntBetween(random(), 1, 5);
+            int numHeaders = RandomInts.randomIntBetween(getRandom(), 1, 5);
             Header[] headers = new Header[numHeaders];
             for (int i = 0; i < numHeaders; i++) {
-                String headerName = "Header" + (random().nextBoolean() ? i : "");
-                String headerValue = RandomStrings.randomAsciiOfLengthBetween(random(), 3, 10);
+                String headerName = "Header" + (getRandom().nextBoolean() ? i : "");
+                String headerValue = RandomStrings.randomAsciiOfLengthBetween(getRandom(), 3, 10);
                 headers[i] = new BasicHeader(headerName, headerValue);
                 expectedHeaders.put(headerName, headerValue);
             }
 
-            int statusCode = randomStatusCode(random());
+            int statusCode = randomStatusCode(getRandom());
             Response esResponse;
             try (Response response = restClient.performRequest(method, "/" + statusCode,
                     Collections.<String, String>emptyMap(), null, headers)) {
@@ -343,15 +349,15 @@ public class RestClientSingleHostTests extends LuceneTestCase {
     }
 
     private HttpUriRequest performRandomRequest(String method) throws IOException, URISyntaxException {
-        String uriAsString = "/" + randomStatusCode(random());
+        String uriAsString = "/" + randomStatusCode(getRandom());
         URIBuilder uriBuilder = new URIBuilder(uriAsString);
         Map<String, String> params = Collections.emptyMap();
-        if (random().nextBoolean()) {
-            int numParams = RandomInts.randomIntBetween(random(), 1, 3);
+        if (getRandom().nextBoolean()) {
+            int numParams = RandomInts.randomIntBetween(getRandom(), 1, 3);
             params = new HashMap<>(numParams);
             for (int i = 0; i < numParams; i++) {
                 String paramKey = "param-" + i;
-                String paramValue = RandomStrings.randomAsciiOfLengthBetween(random(), 3, 10);
+                String paramValue = RandomStrings.randomAsciiOfLengthBetween(getRandom(), 3, 10);
                 params.put(paramKey, paramValue);
                 uriBuilder.addParameter(paramKey, paramValue);
             }
@@ -389,8 +395,8 @@ public class RestClientSingleHostTests extends LuceneTestCase {
         }
 
         HttpEntity entity = null;
-        if (request instanceof HttpEntityEnclosingRequest && random().nextBoolean()) {
-            entity = new StringEntity(RandomStrings.randomAsciiOfLengthBetween(random(), 10, 100));
+        if (request instanceof HttpEntityEnclosingRequest && getRandom().nextBoolean()) {
+            entity = new StringEntity(RandomStrings.randomAsciiOfLengthBetween(getRandom(), 10, 100));
             ((HttpEntityEnclosingRequest) request).setEntity(entity);
         }
 
@@ -399,12 +405,12 @@ public class RestClientSingleHostTests extends LuceneTestCase {
             //default headers are expected but not sent for each request
             request.setHeader(defaultHeader);
         }
-        if (random().nextBoolean()) {
-            int numHeaders = RandomInts.randomIntBetween(random(), 1, 5);
+        if (getRandom().nextBoolean()) {
+            int numHeaders = RandomInts.randomIntBetween(getRandom(), 1, 5);
             headers = new Header[numHeaders];
             for (int i = 0; i < numHeaders; i++) {
-                String headerName = "Header" + (random().nextBoolean() ? i : "");
-                String headerValue = RandomStrings.randomAsciiOfLengthBetween(random(), 3, 10);
+                String headerName = "Header" + (getRandom().nextBoolean() ? i : "");
+                String headerValue = RandomStrings.randomAsciiOfLengthBetween(getRandom(), 3, 10);
                 BasicHeader basicHeader = new BasicHeader(headerName, headerValue);
                 headers[i] = basicHeader;
                 request.setHeader(basicHeader);
diff --git a/settings.gradle b/settings.gradle
index 97195202e0f..a1585a7bdf2 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -7,6 +7,7 @@ List projects = [
   'docs',
   'client',
   'client-sniffer',
+  'client-test',
   'benchmarks',
   'distribution:integ-test-zip',
   'distribution:zip',

From 0bf447c697c18ec70bdec2b921b13c93be2f8eb0 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 22 Jun 2016 13:40:01 -0400
Subject: [PATCH 1034/1311] Group client projects under :client

:client ---------> :client:rest
:client-sniffer -> :client:sniffer
:client-test ----> :client:test

This lines the client up with how we do things like modules and
plugins.
---
 build.gradle                                  |  6 ++---
 client/{ => rest}/build.gradle                |  2 +-
 .../licenses/commons-codec-1.10.jar.sha1      |  0
 .../rest}/licenses/commons-codec-LICENSE.txt  |  0
 .../rest}/licenses/commons-codec-NOTICE.txt   |  0
 .../licenses/commons-logging-1.1.3.jar.sha1   |  0
 .../licenses/commons-logging-LICENSE.txt      |  0
 .../rest}/licenses/commons-logging-NOTICE.txt |  0
 .../rest}/licenses/httpclient-4.5.2.jar.sha1  |  0
 .../rest}/licenses/httpclient-LICENSE.txt     |  0
 .../rest}/licenses/httpclient-NOTICE.txt      |  0
 .../rest}/licenses/httpcore-4.4.4.jar.sha1    |  0
 .../rest}/licenses/httpcore-LICENSE.txt       |  0
 .../rest}/licenses/httpcore-NOTICE.txt        |  0
 .../elasticsearch/client/DeadHostState.java   |  0
 .../client/HttpDeleteWithEntity.java          |  0
 .../client/HttpGetWithEntity.java             |  0
 .../elasticsearch/client/RequestLogger.java   |  0
 .../org/elasticsearch/client/Response.java    |  0
 .../client/ResponseException.java             |  0
 .../org/elasticsearch/client/RestClient.java  |  0
 .../client/CloseableBasicHttpResponse.java    |  0
 .../client/RequestLoggerTests.java            |  0
 .../client/RestClientBuilderTests.java        |  0
 .../client/RestClientIntegTests.java          |  0
 .../client/RestClientMultipleHostsTests.java  |  0
 .../client/RestClientSingleHostTests.java     |  0
 .../client/TrackingFailureListener.java       |  0
 .../sniffer}/build.gradle                     |  4 ++--
 .../licenses/commons-codec-1.10.jar.sha1      |  0
 .../licenses/commons-codec-LICENSE.txt        |  0
 .../licenses/commons-codec-NOTICE.txt         |  0
 .../licenses/commons-logging-1.1.3.jar.sha1   |  0
 .../licenses/commons-logging-LICENSE.txt      |  0
 .../licenses/commons-logging-NOTICE.txt       |  0
 .../licenses/httpclient-4.5.2.jar.sha1        |  0
 .../licenses/httpclient-LICENSE.txt           |  0
 .../licenses/httpclient-NOTICE.txt            |  0
 .../licenses/httpcore-4.4.4.jar.sha1          |  0
 .../licenses/httpcore-LICENSE.txt             |  0
 .../licenses/httpcore-NOTICE.txt              |  0
 .../licenses/jackson-core-2.7.1.jar.sha1      |  0
 .../sniffer}/licenses/jackson-core-LICENSE    |  0
 .../sniffer}/licenses/jackson-core-NOTICE     |  0
 .../client/sniff/HostsSniffer.java            |  0
 .../client/sniff/SniffOnFailureListener.java  |  0
 .../elasticsearch/client/sniff/Sniffer.java   |  0
 .../sniff/HostsSnifferBuilderTests.java       |  0
 .../client/sniff/HostsSnifferTests.java       |  0
 .../client/sniff/MockHostsSniffer.java        |  0
 .../sniff/SniffOnFailureListenerTests.java    |  0
 .../client/sniff/SnifferBuilderTests.java     |  0
 {client-test => client/test}/build.gradle     |  0
 .../client/RestClientTestCase.java            |  0
 .../client/RestClientTestUtil.java            |  0
 settings.gradle                               |  6 ++---
 test/framework/build.gradle                   | 24 +++++++++----------
 57 files changed, 21 insertions(+), 21 deletions(-)
 rename client/{ => rest}/build.gradle (98%)
 rename {client-sniffer => client/rest}/licenses/commons-codec-1.10.jar.sha1 (100%)
 rename {client-sniffer => client/rest}/licenses/commons-codec-LICENSE.txt (100%)
 rename {client-sniffer => client/rest}/licenses/commons-codec-NOTICE.txt (100%)
 rename {client-sniffer => client/rest}/licenses/commons-logging-1.1.3.jar.sha1 (100%)
 rename {client-sniffer => client/rest}/licenses/commons-logging-LICENSE.txt (100%)
 rename {client-sniffer => client/rest}/licenses/commons-logging-NOTICE.txt (100%)
 rename {client-sniffer => client/rest}/licenses/httpclient-4.5.2.jar.sha1 (100%)
 rename {client-sniffer => client/rest}/licenses/httpclient-LICENSE.txt (100%)
 rename {client-sniffer => client/rest}/licenses/httpclient-NOTICE.txt (100%)
 rename {client-sniffer => client/rest}/licenses/httpcore-4.4.4.jar.sha1 (100%)
 rename {client-sniffer => client/rest}/licenses/httpcore-LICENSE.txt (100%)
 rename {client-sniffer => client/rest}/licenses/httpcore-NOTICE.txt (100%)
 rename client/{ => rest}/src/main/java/org/elasticsearch/client/DeadHostState.java (100%)
 rename client/{ => rest}/src/main/java/org/elasticsearch/client/HttpDeleteWithEntity.java (100%)
 rename client/{ => rest}/src/main/java/org/elasticsearch/client/HttpGetWithEntity.java (100%)
 rename client/{ => rest}/src/main/java/org/elasticsearch/client/RequestLogger.java (100%)
 rename client/{ => rest}/src/main/java/org/elasticsearch/client/Response.java (100%)
 rename client/{ => rest}/src/main/java/org/elasticsearch/client/ResponseException.java (100%)
 rename client/{ => rest}/src/main/java/org/elasticsearch/client/RestClient.java (100%)
 rename client/{ => rest}/src/test/java/org/elasticsearch/client/CloseableBasicHttpResponse.java (100%)
 rename client/{ => rest}/src/test/java/org/elasticsearch/client/RequestLoggerTests.java (100%)
 rename client/{ => rest}/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java (100%)
 rename client/{ => rest}/src/test/java/org/elasticsearch/client/RestClientIntegTests.java (100%)
 rename client/{ => rest}/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java (100%)
 rename client/{ => rest}/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java (100%)
 rename client/{ => rest}/src/test/java/org/elasticsearch/client/TrackingFailureListener.java (100%)
 rename {client-sniffer => client/sniffer}/build.gradle (96%)
 rename client/{ => sniffer}/licenses/commons-codec-1.10.jar.sha1 (100%)
 rename client/{ => sniffer}/licenses/commons-codec-LICENSE.txt (100%)
 rename client/{ => sniffer}/licenses/commons-codec-NOTICE.txt (100%)
 rename client/{ => sniffer}/licenses/commons-logging-1.1.3.jar.sha1 (100%)
 rename client/{ => sniffer}/licenses/commons-logging-LICENSE.txt (100%)
 rename client/{ => sniffer}/licenses/commons-logging-NOTICE.txt (100%)
 rename client/{ => sniffer}/licenses/httpclient-4.5.2.jar.sha1 (100%)
 rename client/{ => sniffer}/licenses/httpclient-LICENSE.txt (100%)
 rename client/{ => sniffer}/licenses/httpclient-NOTICE.txt (100%)
 rename client/{ => sniffer}/licenses/httpcore-4.4.4.jar.sha1 (100%)
 rename client/{ => sniffer}/licenses/httpcore-LICENSE.txt (100%)
 rename client/{ => sniffer}/licenses/httpcore-NOTICE.txt (100%)
 rename {client-sniffer => client/sniffer}/licenses/jackson-core-2.7.1.jar.sha1 (100%)
 rename {client-sniffer => client/sniffer}/licenses/jackson-core-LICENSE (100%)
 rename {client-sniffer => client/sniffer}/licenses/jackson-core-NOTICE (100%)
 rename {client-sniffer => client/sniffer}/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java (100%)
 rename {client-sniffer => client/sniffer}/src/main/java/org/elasticsearch/client/sniff/SniffOnFailureListener.java (100%)
 rename {client-sniffer => client/sniffer}/src/main/java/org/elasticsearch/client/sniff/Sniffer.java (100%)
 rename {client-sniffer => client/sniffer}/src/test/java/org/elasticsearch/client/sniff/HostsSnifferBuilderTests.java (100%)
 rename {client-sniffer => client/sniffer}/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java (100%)
 rename {client-sniffer => client/sniffer}/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java (100%)
 rename {client-sniffer => client/sniffer}/src/test/java/org/elasticsearch/client/sniff/SniffOnFailureListenerTests.java (100%)
 rename {client-sniffer => client/sniffer}/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java (100%)
 rename {client-test => client/test}/build.gradle (100%)
 rename {client-test => client/test}/src/main/java/org/elasticsearch/client/RestClientTestCase.java (100%)
 rename {client-test => client/test}/src/main/java/org/elasticsearch/client/RestClientTestUtil.java (100%)

diff --git a/build.gradle b/build.gradle
index 32fde425793..540f27503f0 100644
--- a/build.gradle
+++ b/build.gradle
@@ -163,9 +163,9 @@ subprojects {
     "org.elasticsearch.gradle:build-tools:${version}": ':build-tools',
     "org.elasticsearch:rest-api-spec:${version}": ':rest-api-spec',
     "org.elasticsearch:elasticsearch:${version}": ':core',
-    "org.elasticsearch:client:${version}": ':client',
-    "org.elasticsearch:client-sniffer:${version}": ':client-sniffer',
-    "org.elasticsearch:client-test:${version}": ':client-test',
+    "org.elasticsearch.client:rest:${version}": ':client:rest',
+    "org.elasticsearch.client:sniffer:${version}": ':client:sniffer',
+    "org.elasticsearch.client:test:${version}": ':client:test',
     "org.elasticsearch.test:framework:${version}": ':test:framework',
     "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:integ-test-zip',
     "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:zip',
diff --git a/client/build.gradle b/client/rest/build.gradle
similarity index 98%
rename from client/build.gradle
rename to client/rest/build.gradle
index 2a2e77f88a4..e578126b381 100644
--- a/client/build.gradle
+++ b/client/rest/build.gradle
@@ -32,7 +32,7 @@ dependencies {
   compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
 
-  testCompile "org.elasticsearch:client-test:${version}"
+  testCompile "org.elasticsearch.client:test:${version}"
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
diff --git a/client-sniffer/licenses/commons-codec-1.10.jar.sha1 b/client/rest/licenses/commons-codec-1.10.jar.sha1
similarity index 100%
rename from client-sniffer/licenses/commons-codec-1.10.jar.sha1
rename to client/rest/licenses/commons-codec-1.10.jar.sha1
diff --git a/client-sniffer/licenses/commons-codec-LICENSE.txt b/client/rest/licenses/commons-codec-LICENSE.txt
similarity index 100%
rename from client-sniffer/licenses/commons-codec-LICENSE.txt
rename to client/rest/licenses/commons-codec-LICENSE.txt
diff --git a/client-sniffer/licenses/commons-codec-NOTICE.txt b/client/rest/licenses/commons-codec-NOTICE.txt
similarity index 100%
rename from client-sniffer/licenses/commons-codec-NOTICE.txt
rename to client/rest/licenses/commons-codec-NOTICE.txt
diff --git a/client-sniffer/licenses/commons-logging-1.1.3.jar.sha1 b/client/rest/licenses/commons-logging-1.1.3.jar.sha1
similarity index 100%
rename from client-sniffer/licenses/commons-logging-1.1.3.jar.sha1
rename to client/rest/licenses/commons-logging-1.1.3.jar.sha1
diff --git a/client-sniffer/licenses/commons-logging-LICENSE.txt b/client/rest/licenses/commons-logging-LICENSE.txt
similarity index 100%
rename from client-sniffer/licenses/commons-logging-LICENSE.txt
rename to client/rest/licenses/commons-logging-LICENSE.txt
diff --git a/client-sniffer/licenses/commons-logging-NOTICE.txt b/client/rest/licenses/commons-logging-NOTICE.txt
similarity index 100%
rename from client-sniffer/licenses/commons-logging-NOTICE.txt
rename to client/rest/licenses/commons-logging-NOTICE.txt
diff --git a/client-sniffer/licenses/httpclient-4.5.2.jar.sha1 b/client/rest/licenses/httpclient-4.5.2.jar.sha1
similarity index 100%
rename from client-sniffer/licenses/httpclient-4.5.2.jar.sha1
rename to client/rest/licenses/httpclient-4.5.2.jar.sha1
diff --git a/client-sniffer/licenses/httpclient-LICENSE.txt b/client/rest/licenses/httpclient-LICENSE.txt
similarity index 100%
rename from client-sniffer/licenses/httpclient-LICENSE.txt
rename to client/rest/licenses/httpclient-LICENSE.txt
diff --git a/client-sniffer/licenses/httpclient-NOTICE.txt b/client/rest/licenses/httpclient-NOTICE.txt
similarity index 100%
rename from client-sniffer/licenses/httpclient-NOTICE.txt
rename to client/rest/licenses/httpclient-NOTICE.txt
diff --git a/client-sniffer/licenses/httpcore-4.4.4.jar.sha1 b/client/rest/licenses/httpcore-4.4.4.jar.sha1
similarity index 100%
rename from client-sniffer/licenses/httpcore-4.4.4.jar.sha1
rename to client/rest/licenses/httpcore-4.4.4.jar.sha1
diff --git a/client-sniffer/licenses/httpcore-LICENSE.txt b/client/rest/licenses/httpcore-LICENSE.txt
similarity index 100%
rename from client-sniffer/licenses/httpcore-LICENSE.txt
rename to client/rest/licenses/httpcore-LICENSE.txt
diff --git a/client-sniffer/licenses/httpcore-NOTICE.txt b/client/rest/licenses/httpcore-NOTICE.txt
similarity index 100%
rename from client-sniffer/licenses/httpcore-NOTICE.txt
rename to client/rest/licenses/httpcore-NOTICE.txt
diff --git a/client/src/main/java/org/elasticsearch/client/DeadHostState.java b/client/rest/src/main/java/org/elasticsearch/client/DeadHostState.java
similarity index 100%
rename from client/src/main/java/org/elasticsearch/client/DeadHostState.java
rename to client/rest/src/main/java/org/elasticsearch/client/DeadHostState.java
diff --git a/client/src/main/java/org/elasticsearch/client/HttpDeleteWithEntity.java b/client/rest/src/main/java/org/elasticsearch/client/HttpDeleteWithEntity.java
similarity index 100%
rename from client/src/main/java/org/elasticsearch/client/HttpDeleteWithEntity.java
rename to client/rest/src/main/java/org/elasticsearch/client/HttpDeleteWithEntity.java
diff --git a/client/src/main/java/org/elasticsearch/client/HttpGetWithEntity.java b/client/rest/src/main/java/org/elasticsearch/client/HttpGetWithEntity.java
similarity index 100%
rename from client/src/main/java/org/elasticsearch/client/HttpGetWithEntity.java
rename to client/rest/src/main/java/org/elasticsearch/client/HttpGetWithEntity.java
diff --git a/client/src/main/java/org/elasticsearch/client/RequestLogger.java b/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java
similarity index 100%
rename from client/src/main/java/org/elasticsearch/client/RequestLogger.java
rename to client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java
diff --git a/client/src/main/java/org/elasticsearch/client/Response.java b/client/rest/src/main/java/org/elasticsearch/client/Response.java
similarity index 100%
rename from client/src/main/java/org/elasticsearch/client/Response.java
rename to client/rest/src/main/java/org/elasticsearch/client/Response.java
diff --git a/client/src/main/java/org/elasticsearch/client/ResponseException.java b/client/rest/src/main/java/org/elasticsearch/client/ResponseException.java
similarity index 100%
rename from client/src/main/java/org/elasticsearch/client/ResponseException.java
rename to client/rest/src/main/java/org/elasticsearch/client/ResponseException.java
diff --git a/client/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java
similarity index 100%
rename from client/src/main/java/org/elasticsearch/client/RestClient.java
rename to client/rest/src/main/java/org/elasticsearch/client/RestClient.java
diff --git a/client/src/test/java/org/elasticsearch/client/CloseableBasicHttpResponse.java b/client/rest/src/test/java/org/elasticsearch/client/CloseableBasicHttpResponse.java
similarity index 100%
rename from client/src/test/java/org/elasticsearch/client/CloseableBasicHttpResponse.java
rename to client/rest/src/test/java/org/elasticsearch/client/CloseableBasicHttpResponse.java
diff --git a/client/src/test/java/org/elasticsearch/client/RequestLoggerTests.java b/client/rest/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
similarity index 100%
rename from client/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
rename to client/rest/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
similarity index 100%
rename from client/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
rename to client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
similarity index 100%
rename from client/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
rename to client/rest/src/test/java/org/elasticsearch/client/RestClientIntegTests.java
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
similarity index 100%
rename from client/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
rename to client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
diff --git a/client/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
similarity index 100%
rename from client/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
rename to client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
diff --git a/client/src/test/java/org/elasticsearch/client/TrackingFailureListener.java b/client/rest/src/test/java/org/elasticsearch/client/TrackingFailureListener.java
similarity index 100%
rename from client/src/test/java/org/elasticsearch/client/TrackingFailureListener.java
rename to client/rest/src/test/java/org/elasticsearch/client/TrackingFailureListener.java
diff --git a/client-sniffer/build.gradle b/client/sniffer/build.gradle
similarity index 96%
rename from client-sniffer/build.gradle
rename to client/sniffer/build.gradle
index ffd776ccae2..748c3e2261e 100644
--- a/client-sniffer/build.gradle
+++ b/client/sniffer/build.gradle
@@ -27,14 +27,14 @@ targetCompatibility = JavaVersion.VERSION_1_7
 sourceCompatibility = JavaVersion.VERSION_1_7
 
 dependencies {
-  compile "org.elasticsearch:client:${version}"
+  compile "org.elasticsearch.client:rest:${version}"
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   compile "commons-codec:commons-codec:${versions.commonscodec}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
   compile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
 
-  testCompile "org.elasticsearch:client-test:${version}"
+  testCompile "org.elasticsearch.client:test:${version}"
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
diff --git a/client/licenses/commons-codec-1.10.jar.sha1 b/client/sniffer/licenses/commons-codec-1.10.jar.sha1
similarity index 100%
rename from client/licenses/commons-codec-1.10.jar.sha1
rename to client/sniffer/licenses/commons-codec-1.10.jar.sha1
diff --git a/client/licenses/commons-codec-LICENSE.txt b/client/sniffer/licenses/commons-codec-LICENSE.txt
similarity index 100%
rename from client/licenses/commons-codec-LICENSE.txt
rename to client/sniffer/licenses/commons-codec-LICENSE.txt
diff --git a/client/licenses/commons-codec-NOTICE.txt b/client/sniffer/licenses/commons-codec-NOTICE.txt
similarity index 100%
rename from client/licenses/commons-codec-NOTICE.txt
rename to client/sniffer/licenses/commons-codec-NOTICE.txt
diff --git a/client/licenses/commons-logging-1.1.3.jar.sha1 b/client/sniffer/licenses/commons-logging-1.1.3.jar.sha1
similarity index 100%
rename from client/licenses/commons-logging-1.1.3.jar.sha1
rename to client/sniffer/licenses/commons-logging-1.1.3.jar.sha1
diff --git a/client/licenses/commons-logging-LICENSE.txt b/client/sniffer/licenses/commons-logging-LICENSE.txt
similarity index 100%
rename from client/licenses/commons-logging-LICENSE.txt
rename to client/sniffer/licenses/commons-logging-LICENSE.txt
diff --git a/client/licenses/commons-logging-NOTICE.txt b/client/sniffer/licenses/commons-logging-NOTICE.txt
similarity index 100%
rename from client/licenses/commons-logging-NOTICE.txt
rename to client/sniffer/licenses/commons-logging-NOTICE.txt
diff --git a/client/licenses/httpclient-4.5.2.jar.sha1 b/client/sniffer/licenses/httpclient-4.5.2.jar.sha1
similarity index 100%
rename from client/licenses/httpclient-4.5.2.jar.sha1
rename to client/sniffer/licenses/httpclient-4.5.2.jar.sha1
diff --git a/client/licenses/httpclient-LICENSE.txt b/client/sniffer/licenses/httpclient-LICENSE.txt
similarity index 100%
rename from client/licenses/httpclient-LICENSE.txt
rename to client/sniffer/licenses/httpclient-LICENSE.txt
diff --git a/client/licenses/httpclient-NOTICE.txt b/client/sniffer/licenses/httpclient-NOTICE.txt
similarity index 100%
rename from client/licenses/httpclient-NOTICE.txt
rename to client/sniffer/licenses/httpclient-NOTICE.txt
diff --git a/client/licenses/httpcore-4.4.4.jar.sha1 b/client/sniffer/licenses/httpcore-4.4.4.jar.sha1
similarity index 100%
rename from client/licenses/httpcore-4.4.4.jar.sha1
rename to client/sniffer/licenses/httpcore-4.4.4.jar.sha1
diff --git a/client/licenses/httpcore-LICENSE.txt b/client/sniffer/licenses/httpcore-LICENSE.txt
similarity index 100%
rename from client/licenses/httpcore-LICENSE.txt
rename to client/sniffer/licenses/httpcore-LICENSE.txt
diff --git a/client/licenses/httpcore-NOTICE.txt b/client/sniffer/licenses/httpcore-NOTICE.txt
similarity index 100%
rename from client/licenses/httpcore-NOTICE.txt
rename to client/sniffer/licenses/httpcore-NOTICE.txt
diff --git a/client-sniffer/licenses/jackson-core-2.7.1.jar.sha1 b/client/sniffer/licenses/jackson-core-2.7.1.jar.sha1
similarity index 100%
rename from client-sniffer/licenses/jackson-core-2.7.1.jar.sha1
rename to client/sniffer/licenses/jackson-core-2.7.1.jar.sha1
diff --git a/client-sniffer/licenses/jackson-core-LICENSE b/client/sniffer/licenses/jackson-core-LICENSE
similarity index 100%
rename from client-sniffer/licenses/jackson-core-LICENSE
rename to client/sniffer/licenses/jackson-core-LICENSE
diff --git a/client-sniffer/licenses/jackson-core-NOTICE b/client/sniffer/licenses/jackson-core-NOTICE
similarity index 100%
rename from client-sniffer/licenses/jackson-core-NOTICE
rename to client/sniffer/licenses/jackson-core-NOTICE
diff --git a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
similarity index 100%
rename from client-sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
rename to client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
diff --git a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/SniffOnFailureListener.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/SniffOnFailureListener.java
similarity index 100%
rename from client-sniffer/src/main/java/org/elasticsearch/client/sniff/SniffOnFailureListener.java
rename to client/sniffer/src/main/java/org/elasticsearch/client/sniff/SniffOnFailureListener.java
diff --git a/client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
similarity index 100%
rename from client-sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
rename to client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferBuilderTests.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferBuilderTests.java
similarity index 100%
rename from client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferBuilderTests.java
rename to client/sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferBuilderTests.java
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
similarity index 100%
rename from client-sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
rename to client/sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java
similarity index 100%
rename from client-sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java
rename to client/sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SniffOnFailureListenerTests.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/SniffOnFailureListenerTests.java
similarity index 100%
rename from client-sniffer/src/test/java/org/elasticsearch/client/sniff/SniffOnFailureListenerTests.java
rename to client/sniffer/src/test/java/org/elasticsearch/client/sniff/SniffOnFailureListenerTests.java
diff --git a/client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
similarity index 100%
rename from client-sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
rename to client/sniffer/src/test/java/org/elasticsearch/client/sniff/SnifferBuilderTests.java
diff --git a/client-test/build.gradle b/client/test/build.gradle
similarity index 100%
rename from client-test/build.gradle
rename to client/test/build.gradle
diff --git a/client-test/src/main/java/org/elasticsearch/client/RestClientTestCase.java b/client/test/src/main/java/org/elasticsearch/client/RestClientTestCase.java
similarity index 100%
rename from client-test/src/main/java/org/elasticsearch/client/RestClientTestCase.java
rename to client/test/src/main/java/org/elasticsearch/client/RestClientTestCase.java
diff --git a/client-test/src/main/java/org/elasticsearch/client/RestClientTestUtil.java b/client/test/src/main/java/org/elasticsearch/client/RestClientTestUtil.java
similarity index 100%
rename from client-test/src/main/java/org/elasticsearch/client/RestClientTestUtil.java
rename to client/test/src/main/java/org/elasticsearch/client/RestClientTestUtil.java
diff --git a/settings.gradle b/settings.gradle
index a1585a7bdf2..c0d7a72b2a5 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -5,9 +5,9 @@ List projects = [
   'rest-api-spec',
   'core',
   'docs',
-  'client',
-  'client-sniffer',
-  'client-test',
+  'client:rest',
+  'client:sniffer',
+  'client:test',
   'benchmarks',
   'distribution:integ-test-zip',
   'distribution:zip',
diff --git a/test/framework/build.gradle b/test/framework/build.gradle
index 6ddcb7598a1..8be06574df8 100644
--- a/test/framework/build.gradle
+++ b/test/framework/build.gradle
@@ -26,7 +26,7 @@ dependencies {
   compile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
   compile "org.apache.lucene:lucene-test-framework:${versions.lucene}"
   compile "org.apache.lucene:lucene-codecs:${versions.lucene}"
-  compile "org.elasticsearch:client:${version}"
+  compile "org.elasticsearch.client:rest:${version}"
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
@@ -48,17 +48,17 @@ dependencyLicenses.enabled = false
 
 thirdPartyAudit.excludes = [
   // classes are missing
-  'javax.servlet.ServletContextEvent', 
-  'javax.servlet.ServletContextListener', 
-  'org.apache.avalon.framework.logger.Logger', 
-  'org.apache.log.Hierarchy', 
-  'org.apache.log.Logger', 
+  'javax.servlet.ServletContextEvent',
+  'javax.servlet.ServletContextListener',
+  'org.apache.avalon.framework.logger.Logger',
+  'org.apache.log.Hierarchy',
+  'org.apache.log.Logger',
    // we intentionally exclude the ant tasks because people were depending on them from their tests!!!!!!!
-  'org.apache.tools.ant.BuildException', 
-  'org.apache.tools.ant.DirectoryScanner', 
-  'org.apache.tools.ant.Task', 
-  'org.apache.tools.ant.types.FileSet', 
-  'org.easymock.EasyMock', 
-  'org.easymock.IArgumentMatcher', 
+  'org.apache.tools.ant.BuildException',
+  'org.apache.tools.ant.DirectoryScanner',
+  'org.apache.tools.ant.Task',
+  'org.apache.tools.ant.types.FileSet',
+  'org.easymock.EasyMock',
+  'org.easymock.IArgumentMatcher',
   'org.jmock.core.Constraint',
 ]

From 02761f5fe00085a5af76a4c321b467fb7d7bcb54 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 21 Jun 2016 15:33:25 -0400
Subject: [PATCH 1035/1311] Docs: migration notes for _timestamp and _ttl

We aren't able to actually create an index with _timestamp enabled
to test the migration, or, at least, we won't be able to after #18980
is re-merged. But the docs are still ok.

Closes #19007
---
 .../migration/migrate_5_0/mapping.asciidoc    | 105 ++++++++++++++++++
 1 file changed, 105 insertions(+)

diff --git a/docs/reference/migration/migrate_5_0/mapping.asciidoc b/docs/reference/migration/migrate_5_0/mapping.asciidoc
index 2d0ca4b9fb6..fa5411cf426 100644
--- a/docs/reference/migration/migrate_5_0/mapping.asciidoc
+++ b/docs/reference/migration/migrate_5_0/mapping.asciidoc
@@ -157,3 +157,108 @@ Per-field boosts on the `_all` are now compressed into a single byte instead
 of the 4 bytes used previously. While this will make the index much more
 space-efficient, it also means that index time boosts will be less accurately
 encoded.
+
+==== `_ttl` and `_timestamp` cannot be created
+
+You can no longer create indexes with `_ttl` or `_timestamp` enabled. Indexes
+with them enabled created before 5.0 will continue to work.
+
+You should replace `_timestamp` in new indexes by adding a field to your source
+either in the application producing the data or with an ingest pipline like
+this one:
+
+[source,js]
+---------------
+PUT _ingest/pipeline/timestamp
+{
+  "description" : "Adds a timestamp field at the current time",
+  "processors" : [ {
+    "set" : {
+      "field": "timestamp",
+      "value": "{{_ingest.timestamp}}"
+    }
+  } ]
+}
+
+PUT newindex/type/1?pipeline=timestamp
+{
+  "example": "data"
+}
+
+GET newindex/type/1
+---------------
+// CONSOLE
+
+Which produces
+[source,js]
+---------------
+{
+  "_source": {
+    "example": "data",
+    "timestamp": "2016-06-21T18:48:55.560+0000"
+  },
+  ...
+}
+---------------
+// TESTRESPONSE[s/\.\.\./"found": true, "_id": "1", "_index": "newindex", "_type": "type", "_version": 1/]
+// TESTRESPONSE[s/"2016-06-21T18:48:55.560\+0000"/"$body._source.timestamp"/]
+
+If you have an old index created with 2.x that has `_timestamp` enabled then
+you can migrate it to a new index with the a `timestamp` field in the source
+with reindex:
+
+[source,js]
+---------------
+POST _reindex
+{
+  "source": {
+    "index": "oldindex"
+  },
+  "dest": {
+    "index": "newindex"
+  },
+  "script": {
+    "lang": "painless",
+    "inline": "ctx._source.timestamp = ctx._timestamp; ctx._timestamp = null"
+  }
+}
+---------------
+// CONSOLE
+// TEST[s/^/PUT oldindex\nGET _cluster\/health?wait_for_status=yellow\n/]
+
+You can replace `_ttl` with time based index names (preferred) or by adding a
+cron job which runs a delete-by-query on a timestamp field in the source
+document. If you had documents like this:
+
+[source,js]
+---------------
+POST index/type/_bulk
+{"index":{"_id":1}}
+{"example": "data", "timestamp": "2016-06-21T18:48:55.560+0000" }
+{"index":{"_id":2}}
+{"example": "data", "timestamp": "2016-04-21T18:48:55.560+0000" }
+---------------
+// CONSOLE
+
+Then you could delete all of the documents from before June 1st with:
+
+[source,js]
+---------------
+POST index/type/_delete_by_query
+{
+  "query": {
+    "range" : {
+      "timestamp" : {
+        "lt" : "2016-05-01"
+      }
+    }
+  }
+}
+---------------
+// CONSOLE
+// TEST[continued]
+
+IMPORTANT: Keep in mind that deleting documents from an index is very expensive
+compared to deleting whole indexes. That is why time based indexes are
+recommended over this sort of thing and why `_ttl` was deprecated in the first
+place.

From ee2a77143bbdb9ad40963d0e9d1fa5297cc228ce Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 21 Jun 2016 11:24:06 -0400
Subject: [PATCH 1036/1311] Docs: Convert aggs/misc to CONSOLE

They should be more readable and tested during the build.
---
 docs/reference/aggregations/misc.asciidoc | 36 ++++++++++++++---------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/docs/reference/aggregations/misc.asciidoc b/docs/reference/aggregations/misc.asciidoc
index 1be8373c1ee..7210daf1417 100644
--- a/docs/reference/aggregations/misc.asciidoc
+++ b/docs/reference/aggregations/misc.asciidoc
@@ -17,7 +17,8 @@ setting `size=0`. For example:
 
 [source,js]
 --------------------------------------------------
-$ curl -XGET 'http://localhost:9200/twitter/tweet/_search' -d '{
+GET /twitter/tweet/_search
+{
   "size": 0,
   "aggregations": {
     "my_agg": {
@@ -27,8 +28,9 @@ $ curl -XGET 'http://localhost:9200/twitter/tweet/_search' -d '{
     }
   }
 }
-'
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 Setting `size` to `0` avoids executing the fetch phase of the search making the request more efficient.
 
@@ -42,35 +44,41 @@ Consider this example where we want to associate the color blue with our `terms`
 
 [source,js]
 --------------------------------------------------
+GET /twitter/tweet/_search
 {
-    ...
-    "aggs": {
-        "titles": {
-            "terms": {
-                "field": "title"
-            },
-            "meta": {
-                "color": "blue"
-            },
-        }
+  "size": 0,
+  "aggs": {
+    "titles": {
+      "terms": {
+        "field": "title"
+      },
+      "meta": {
+        "color": "blue"
+      }
     }
+  }
 }
 --------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
 
 Then that piece of metadata will be returned in place for our `titles` terms aggregation
 
 [source,js]
 --------------------------------------------------
 {
-    ...
     "aggregations": {
         "titles": {
             "meta": {
                 "color" : "blue"
             },
+            "doc_count_error_upper_bound" : 0,
+            "sum_other_doc_count" : 0,
             "buckets": [
             ]
         }
-    }
+    },
+    ...
 }
 --------------------------------------------------
+// TESTRESPONSE[s/\.\.\./"took": "$body.took", "timed_out": false, "_shards": "$body._shards", "hits": "$body.hits"/]

From 6dd9cd72b9d4b32765449be3f8d9160763e5c932 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 22 Jun 2016 14:56:42 -0400
Subject: [PATCH 1037/1311] Build valid slices in SearchSourceBuilderTests

The test had a 1 in 500 chance of building and invalid slice.
---
 .../elasticsearch/search/builder/SearchSourceBuilderTests.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index ee5b65b5ebc..419b11dac60 100644
--- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -394,7 +394,7 @@ public class SearchSourceBuilderTests extends ESTestCase {
         }
         if (randomBoolean()) {
             String field = randomBoolean() ? null : randomAsciiOfLengthBetween(5, 20);
-            int max = randomInt(1000);
+            int max = between(2, 1000);
             int id = randomInt(max-1);
             if (field == null) {
                 builder.slice(new SliceBuilder(id, max));

From 4be94cdc959a39150fc029a3966674b56d94977a Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Thu, 23 Jun 2016 08:41:44 +0200
Subject: [PATCH 1038/1311] revert - Inline reroute with process of node
 join/master election (#18938)

There are secondary issues with async shard fetch going out to nodes before they have a cluster state published to them that need to be solved first. For example:
- async fetch uses transport node action that resolves nodes based on the cluster state (but it's not yet exposed by ClusterService since we inline the reroute)
- after disruption nodes will respond with an allocated shard (they didn't clean up their shards yet) which throws of decisions master side.
- nodes deed the index meta data in question but they may not have if they didn't recieve the latest CS
---
 .../cluster/routing/RoutingService.java       |  4 ++
 .../elasticsearch/discovery/Discovery.java    |  4 +-
 .../discovery/local/LocalDiscovery.java       | 24 ++++++------
 .../discovery/zen/NodeJoinController.java     | 39 ++++++++++++-------
 .../discovery/zen/ZenDiscovery.java           | 17 ++++----
 .../java/org/elasticsearch/node/Node.java     |  3 +-
 .../zen/NodeJoinControllerTests.java          | 30 ++++++++------
 .../org/elasticsearch/test/NoopDiscovery.java |  4 +-
 8 files changed, 76 insertions(+), 49 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
index d6ec9048688..78e7e15d389 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
@@ -72,6 +72,10 @@ public class RoutingService extends AbstractLifecycleComponent<RoutingService> {
     protected void doClose() {
     }
 
+    public AllocationService getAllocationService() {
+        return this.allocationService;
+    }
+
     /**
      * Initiates a reroute.
      */
diff --git a/core/src/main/java/org/elasticsearch/discovery/Discovery.java b/core/src/main/java/org/elasticsearch/discovery/Discovery.java
index dabd45e36dd..778e2d15053 100644
--- a/core/src/main/java/org/elasticsearch/discovery/Discovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/Discovery.java
@@ -22,7 +22,7 @@ package org.elasticsearch.discovery;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -44,7 +44,7 @@ public interface Discovery extends LifecycleComponent<Discovery> {
      * Another hack to solve dep injection problem..., note, this will be called before
      * any start is called.
      */
-    void setAllocationService(AllocationService allocationService);
+    void setRoutingService(RoutingService routingService);
 
     /**
      * Publish all the changes to the cluster from the master (can be called just by the master). The publish
diff --git a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
index ec9158c5932..696eb8812c9 100644
--- a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
@@ -28,7 +28,7 @@ import org.elasticsearch.cluster.IncompatibleClusterStateVersionException;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
@@ -61,7 +61,7 @@ public class LocalDiscovery extends AbstractLifecycleComponent<Discovery> implem
     private static final LocalDiscovery[] NO_MEMBERS = new LocalDiscovery[0];
 
     private final ClusterService clusterService;
-    private AllocationService allocationService;
+    private RoutingService routingService;
     private final ClusterName clusterName;
 
     private final DiscoverySettings discoverySettings;
@@ -83,8 +83,8 @@ public class LocalDiscovery extends AbstractLifecycleComponent<Discovery> implem
     }
 
     @Override
-    public void setAllocationService(AllocationService allocationService) {
-        this.allocationService = allocationService;
+    public void setRoutingService(RoutingService routingService) {
+        this.routingService = routingService;
     }
 
     @Override
@@ -156,12 +156,7 @@ public class LocalDiscovery extends AbstractLifecycleComponent<Discovery> implem
                             nodesBuilder.put(discovery.localNode());
                         }
                         nodesBuilder.localNodeId(master.localNode().getId()).masterNodeId(master.localNode().getId());
-                        currentState = ClusterState.builder(currentState).nodes(nodesBuilder).build();
-                        RoutingAllocation.Result result =  master.allocationService.reroute(currentState, "node_add");
-                        if (result.changed()) {
-                            currentState = ClusterState.builder(currentState).routingResult(result).build();
-                        }
-                        return currentState;
+                        return ClusterState.builder(currentState).nodes(nodesBuilder).build();
                     }
 
                     @Override
@@ -169,6 +164,13 @@ public class LocalDiscovery extends AbstractLifecycleComponent<Discovery> implem
                         logger.error("unexpected failure during [{}]", t, source);
                     }
 
+                    @Override
+                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
+                        // we reroute not in the same cluster state update since in certain areas we rely on
+                        // the node to be in the cluster state (sampled from ClusterService#state) to be there, also
+                        // shard transitions need to better be handled in such cases
+                        master.routingService.reroute("post_node_add");
+                    }
                 });
             }
         } // else, no master node, the next node that will start will fill things in...
@@ -224,7 +226,7 @@ public class LocalDiscovery extends AbstractLifecycleComponent<Discovery> implem
                         }
                         // reroute here, so we eagerly remove dead nodes from the routing
                         ClusterState updatedState = ClusterState.builder(currentState).nodes(newNodes).build();
-                        RoutingAllocation.Result routingResult = master.allocationService.reroute(
+                        RoutingAllocation.Result routingResult = master.routingService.getAllocationService().reroute(
                                 ClusterState.builder(updatedState).build(), "elected as master");
                         return ClusterState.builder(updatedState).routingResult(routingResult).build();
                     }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
index 49815cdb703..9659434e03d 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
@@ -29,7 +29,7 @@ import org.elasticsearch.cluster.NotMasterException;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
@@ -58,7 +58,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 public class NodeJoinController extends AbstractComponent {
 
     private final ClusterService clusterService;
-    private final AllocationService allocationService;
+    private final RoutingService routingService;
     private final ElectMasterService electMaster;
     private final DiscoverySettings discoverySettings;
     private final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor();
@@ -68,11 +68,10 @@ public class NodeJoinController extends AbstractComponent {
     private ElectionContext electionContext = null;
 
 
-    public NodeJoinController(ClusterService clusterService, AllocationService allocationService, ElectMasterService electMaster,
-                              DiscoverySettings discoverySettings, Settings settings) {
+    public NodeJoinController(ClusterService clusterService, RoutingService routingService, ElectMasterService electMaster, DiscoverySettings discoverySettings, Settings settings) {
         super(settings);
         this.clusterService = clusterService;
-        this.allocationService = allocationService;
+        this.routingService = routingService;
         this.electMaster = electMaster;
         this.discoverySettings = discoverySettings;
     }
@@ -407,7 +406,21 @@ public class NodeJoinController extends AbstractComponent {
                 ClusterBlocks clusterBlocks = ClusterBlocks.builder().blocks(currentState.blocks())
                     .removeGlobalBlock(discoverySettings.getNoMasterBlock()).build();
                 newState.blocks(clusterBlocks);
+                newState.nodes(nodesBuilder);
                 nodesChanged = true;
+
+                // reroute now to remove any dead nodes (master may have stepped down when they left and didn't update the routing table)
+                // Note: also do it now to avoid assigning shards to these nodes. We will have another reroute after the cluster
+                // state is published.
+                // TODO: this publishing of a cluster state with no nodes assigned to joining nodes shouldn't be needed anymore. remove.
+
+                final ClusterState tmpState = newState.build();
+                RoutingAllocation.Result result = routingService.getAllocationService().reroute(tmpState, "nodes joined");
+                newState = ClusterState.builder(tmpState);
+                if (result.changed()) {
+                    newState.routingResult(result);
+                }
+                nodesBuilder = DiscoveryNodes.builder(tmpState.nodes());
             }
 
             if (nodesBuilder.isLocalNodeElectedMaster() == false) {
@@ -426,8 +439,7 @@ public class NodeJoinController extends AbstractComponent {
                     for (DiscoveryNode existingNode : currentNodes) {
                         if (node.getAddress().equals(existingNode.getAddress())) {
                             nodesBuilder.remove(existingNode.getId());
-                            logger.warn("received join request from node [{}], but found existing node {} with same address, " +
-                                "removing existing node", node, existingNode);
+                            logger.warn("received join request from node [{}], but found existing node {} with same address, removing existing node", node, existingNode);
                         }
                     }
                 }
@@ -436,12 +448,6 @@ public class NodeJoinController extends AbstractComponent {
 
             if (nodesChanged) {
                 newState.nodes(nodesBuilder);
-                final ClusterState tmpState = newState.build();
-                RoutingAllocation.Result result = allocationService.reroute(tmpState, "node_join");
-                newState = ClusterState.builder(tmpState);
-                if (result.changed()) {
-                    newState.routingResult(result);
-                }
             }
 
             // we must return a new cluster state instance to force publishing. This is important
@@ -457,6 +463,13 @@ public class NodeJoinController extends AbstractComponent {
 
         @Override
         public void clusterStatePublished(ClusterChangedEvent event) {
+            if (event.nodesDelta().hasChanges()) {
+                // we reroute not in the same cluster state update since in certain areas we rely on
+                // the node to be in the cluster state (sampled from ClusterService#state) to be there, also
+                // shard transitions need to better be handled in such cases
+                routingService.reroute("post_node_add");
+            }
+
             NodeJoinController.this.electMaster.logMinimumMasterNodesWarningIfNecessary(event.previousState(), event.state());
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
index 6a9a64ff411..54ef9928585 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
@@ -32,7 +32,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
@@ -113,7 +113,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
 
     private final TransportService transportService;
     private final ClusterService clusterService;
-    private AllocationService allocationService;
+    private RoutingService routingService;
     private final ClusterName clusterName;
     private final DiscoverySettings discoverySettings;
     private final ZenPingService pingService;
@@ -146,7 +146,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
     /** counts the time this node has joined the cluster or have elected it self as master */
     private final AtomicLong clusterJoinsCounter = new AtomicLong();
 
-    // must initialized in doStart(), when we have the allocationService set
+    // must initialized in doStart(), when we have the routingService set
     private volatile NodeJoinController nodeJoinController;
 
     @Inject
@@ -206,8 +206,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
     }
 
     @Override
-    public void setAllocationService(AllocationService allocationService) {
-        this.allocationService = allocationService;
+    public void setRoutingService(RoutingService routingService) {
+        this.routingService = routingService;
     }
 
     @Override
@@ -215,7 +215,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
         nodesFD.setLocalNode(clusterService.localNode());
         joinThreadControl.start();
         pingService.start();
-        this.nodeJoinController = new NodeJoinController(clusterService, allocationService, electMaster, discoverySettings, settings);
+        this.nodeJoinController = new NodeJoinController(clusterService, routingService, electMaster, discoverySettings, settings);
     }
 
     @Override
@@ -516,7 +516,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
                         return rejoin(currentState, "not enough master nodes");
                     }
                     // eagerly run reroute to remove dead nodes from routing table
-                    RoutingAllocation.Result routingResult = allocationService.reroute(ClusterState.builder(currentState).build(),
+                    RoutingAllocation.Result routingResult = routingService.getAllocationService().reroute(
+                            ClusterState.builder(currentState).build(),
                             "[" + node + "] left");
                     return ClusterState.builder(currentState).routingResult(routingResult).build();
                 }
@@ -560,7 +561,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
                     return rejoin(currentState, "not enough master nodes");
                 }
                 // eagerly run reroute to remove dead nodes from routing table
-                RoutingAllocation.Result routingResult = allocationService.reroute(
+                RoutingAllocation.Result routingResult = routingService.getAllocationService().reroute(
                         ClusterState.builder(currentState).build(),
                         "[" + node + "] failed");
                 return ClusterState.builder(currentState).routingResult(routingResult).build();
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index 3c036ff3f29..fb38c530d45 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -37,7 +37,6 @@ import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeService;
 import org.elasticsearch.cluster.routing.RoutingService;
-import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.StopWatch;
 import org.elasticsearch.common.component.Lifecycle;
@@ -328,7 +327,7 @@ public class Node implements Closeable {
         ESLogger logger = Loggers.getLogger(Node.class, NODE_NAME_SETTING.get(settings));
         logger.info("starting ...");
         // hack around dependency injection problem (for now...)
-        injector.getInstance(Discovery.class).setAllocationService(injector.getInstance(AllocationService.class));
+        injector.getInstance(Discovery.class).setRoutingService(injector.getInstance(RoutingService.class));
         for (Class<? extends LifecycleComponent> plugin : pluginsService.nodeServices()) {
             injector.getInstance(plugin).start();
         }
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
index e994c376c4d..135352343b6 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
@@ -24,6 +24,7 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.NotMasterException;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
@@ -98,7 +99,7 @@ public class NodeJoinControllerTests extends ESTestCase {
         // make sure we have a master
         setState(clusterService, ClusterState.builder(clusterService.state()).nodes(
             DiscoveryNodes.builder(initialNodes).masterNodeId(localNode.getId())));
-        nodeJoinController = new NodeJoinController(clusterService, new NoopAllocationService(Settings.EMPTY),
+        nodeJoinController = new NodeJoinController(clusterService, new NoopRoutingService(Settings.EMPTY),
             new ElectMasterService(Settings.EMPTY),
             new DiscoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)),
             Settings.EMPTY);
@@ -297,8 +298,7 @@ public class NodeJoinControllerTests extends ESTestCase {
         }
 
         logger.debug("--> asserting master election didn't finish yet");
-        assertThat("election finished after [" + initialJoins + "] master nodes but required joins is [" + requiredJoins + "]",
-            electionFuture.isDone(), equalTo(false));
+        assertThat("election finished after [" + initialJoins + "] master nodes but required joins is [" + requiredJoins + "]", electionFuture.isDone(), equalTo(false));
 
         final int finalJoins = requiredJoins - initialJoins + randomInt(5);
         nodesToJoin.clear();
@@ -374,8 +374,7 @@ public class NodeJoinControllerTests extends ESTestCase {
         nodeJoinController.waitToBeElectedAsMaster(requiredJoins, TimeValue.timeValueMillis(1), new NodeJoinController.ElectionCallback() {
             @Override
             public void onElectedAsMaster(ClusterState state) {
-                assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(),
-                    equalTo(true));
+                assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(), equalTo(true));
                 latch.countDown();
             }
 
@@ -493,8 +492,7 @@ public class NodeJoinControllerTests extends ESTestCase {
         nodeJoinController.waitToBeElectedAsMaster(requiredJoins, TimeValue.timeValueHours(30), new NodeJoinController.ElectionCallback() {
             @Override
             public void onElectedAsMaster(ClusterState state) {
-                assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(),
-                    equalTo(true));
+                assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(), equalTo(true));
                 latch.countDown();
             }
 
@@ -518,6 +516,18 @@ public class NodeJoinControllerTests extends ESTestCase {
     }
 
 
+    static class NoopRoutingService extends RoutingService {
+
+        public NoopRoutingService(Settings settings) {
+            super(settings, null, new NoopAllocationService(settings));
+        }
+
+        @Override
+        protected void performReroute(String reason) {
+
+        }
+    }
+
     static class NoopAllocationService extends AllocationService {
 
         public NoopAllocationService(Settings settings) {
@@ -525,14 +535,12 @@ public class NodeJoinControllerTests extends ESTestCase {
         }
 
         @Override
-        public RoutingAllocation.Result applyStartedShards(ClusterState clusterState, List<? extends ShardRouting> startedShards,
-                                                           boolean withReroute) {
+        public RoutingAllocation.Result applyStartedShards(ClusterState clusterState, List<? extends ShardRouting> startedShards, boolean withReroute) {
             return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData());
         }
 
         @Override
-        public RoutingAllocation.Result applyFailedShards(ClusterState clusterState,
-                                                          List<FailedRerouteAllocation.FailedShard> failedShards) {
+        public RoutingAllocation.Result applyFailedShards(ClusterState clusterState, List<FailedRerouteAllocation.FailedShard> failedShards) {
             return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData());
         }
 
diff --git a/core/src/test/java/org/elasticsearch/test/NoopDiscovery.java b/core/src/test/java/org/elasticsearch/test/NoopDiscovery.java
index 967837e0044..3193aaf458e 100644
--- a/core/src/test/java/org/elasticsearch/test/NoopDiscovery.java
+++ b/core/src/test/java/org/elasticsearch/test/NoopDiscovery.java
@@ -20,7 +20,7 @@ package org.elasticsearch.test;
 
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.allocation.AllocationService;
+import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.component.LifecycleListener;
 import org.elasticsearch.discovery.Discovery;
@@ -41,7 +41,7 @@ public class NoopDiscovery implements Discovery {
     }
 
     @Override
-    public void setAllocationService(AllocationService allocationService) {
+    public void setRoutingService(RoutingService routingService) {
 
     }
 

From 04da1bda0ddb42a66c6b8481a544f24647bd0ce4 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Tue, 7 Jun 2016 17:45:05 +0200
Subject: [PATCH 1039/1311] Move templates out of the Search API, into
 lang-mustache module

This commit moves template support out of the Search API to its own dedicated Search Template API in the lang-mustache module. It provides a new SearchTemplateAction that can be used to render templates before it gets delegated to the usual Search API. The current REST endpoint are identical, but the Render Search Template endpoint now uses the same Search Template API with a new "simulate" option. When this option is enabled, the Search Template API only renders template and returns immediatly, without executing the search.

Closes #17906
---
 .../elasticsearch/action/ActionModule.java    |   3 -
 .../template/RenderSearchTemplateRequest.java |  69 -----
 .../TransportRenderSearchTemplateAction.java  |  75 ------
 .../action/search/SearchRequest.java          |  20 --
 .../action/search/SearchRequestBuilder.java   |   9 -
 .../client/ClusterAdminClient.java            |  25 --
 .../client/support/AbstractClient.java        |  19 --
 .../common/network/NetworkModule.java         |  14 +-
 .../common/xcontent/ObjectParser.java         |   1 +
 .../elasticsearch/indices/IndicesService.java |   4 -
 .../RestPutStoredScriptAction.java            |  16 --
 .../RestRenderSearchTemplateAction.java       | 109 --------
 .../action/search/RestMultiSearchAction.java  | 113 ++++----
 .../rest/action/search/RestSearchAction.java  |  26 +-
 .../elasticsearch/search/SearchService.java   |  22 +-
 .../internal/ShardSearchLocalRequest.java     |  10 -
 .../search/internal/ShardSearchRequest.java   |   3 -
 .../internal/ShardSearchTransportRequest.java |   5 -
 .../search/MultiSearchRequestTests.java       |  74 ++----
 .../common/network/NetworkModuleTests.java    |   8 +
 .../index/SearchSlowLogTests.java             |   6 -
 .../elasticsearch/script/StoredScriptsIT.java |   6 -
 .../action/search/simple-msearch5.json        |   6 -
 .../migration/migrate_5_0/java.asciidoc       |  13 +-
 .../migration/migrate_5_0/rest.asciidoc       |  12 +
 .../migration/migrate_5_0/scripting.asciidoc  |   2 +-
 .../reference/search/search-template.asciidoc |  49 ++++
 modules/lang-mustache/build.gradle            |   4 +
 .../template/MultiSearchTemplateAction.java   |  20 +-
 .../template/MultiSearchTemplateRequest.java  | 107 ++++++++
 .../MultiSearchTemplateRequestBuilder.java    |  61 +++++
 .../template/MultiSearchTemplateResponse.java | 181 +++++++++++++
 .../search/template/SearchTemplateAction.java |  34 +--
 .../template/SearchTemplateRequest.java       | 156 +++++++++++
 .../SearchTemplateRequestBuilder.java         |  64 +++++
 .../template/SearchTemplateResponse.java      |  66 +++--
 .../TransportMultiSearchTemplateAction.java   |  78 ++++++
 .../TransportSearchTemplateAction.java        | 122 +++++++++
 .../RestDeleteSearchTemplateAction.java       |   2 +-
 .../RestGetSearchTemplateAction.java          |   9 +-
 .../RestMultiSearchTemplateAction.java        |  92 +++++++
 .../RestPutSearchTemplateAction.java          |  25 +-
 .../RestRenderSearchTemplateAction.java       |  63 +++++
 .../template/RestSearchTemplateAction.java    | 128 +++++++++
 .../script/mustache/MustachePlugin.java       |  28 ++
 .../template/MultiSearchTemplateIT.java       | 179 +++++++++++++
 .../MultiSearchTemplateRequestTests.java      |  72 +++++
 .../template/SearchTemplateRequestTests.java  | 155 +++++++++++
 .../tests/RenderSearchTemplateTests.java      |  55 ++--
 .../messy/tests/TemplateQueryTests.java       | 251 +++++++++---------
 .../template/simple-msearch-template.json     |   6 +
 .../tests/config/scripts/template_1.mustache  |  11 +
 .../40_search_request_template.yaml           |  13 +-
 .../lang_mustache/50_messy_test_msearch.yaml  |  22 +-
 .../template.msearch/10_basic.yaml            | 126 +++++++++
 .../template.msearch/20_stored_template.yaml  |  66 +++++
 .../template.msearch/30_file_template.yaml    |  72 +++++
 .../rest-api-spec/api/template.msearch.json   |  32 +++
 .../rest-api-spec/api/template.search.json    |  56 ++++
 .../rest-api-spec/test/msearch/10_basic.yaml  |  64 +++++
 .../search/MockSearchService.java             |   7 +-
 61 files changed, 2337 insertions(+), 809 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/action/admin/cluster/validate/template/RenderSearchTemplateRequest.java
 delete mode 100644 core/src/main/java/org/elasticsearch/action/admin/cluster/validate/template/TransportRenderSearchTemplateAction.java
 delete mode 100644 core/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/template/RestRenderSearchTemplateAction.java
 delete mode 100644 core/src/test/resources/org/elasticsearch/action/search/simple-msearch5.json
 rename core/src/main/java/org/elasticsearch/action/admin/cluster/validate/template/RenderSearchTemplateAction.java => modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateAction.java (57%)
 create mode 100644 modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateRequest.java
 create mode 100644 modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateRequestBuilder.java
 create mode 100644 modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateResponse.java
 rename core/src/main/java/org/elasticsearch/action/admin/cluster/validate/template/RenderSearchTemplateRequestBuilder.java => modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateAction.java (53%)
 create mode 100644 modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateRequest.java
 create mode 100644 modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateRequestBuilder.java
 rename core/src/main/java/org/elasticsearch/action/admin/cluster/validate/template/RenderSearchTemplateResponse.java => modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateResponse.java (53%)
 create mode 100644 modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportMultiSearchTemplateAction.java
 create mode 100644 modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportSearchTemplateAction.java
 rename {core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts => modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template}/RestDeleteSearchTemplateAction.java (96%)
 rename {core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts => modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template}/RestGetSearchTemplateAction.java (95%)
 create mode 100644 modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestMultiSearchTemplateAction.java
 rename {core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts => modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template}/RestPutSearchTemplateAction.java (64%)
 create mode 100644 modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestRenderSearchTemplateAction.java
 create mode 100644 modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestSearchTemplateAction.java
 create mode 100644 modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateIT.java
 create mode 100644 modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateRequestTests.java
 create mode 100644 modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/SearchTemplateRequestTests.java
 create mode 100644 modules/lang-mustache/src/test/resources/org/elasticsearch/action/search/template/simple-msearch-template.json
 create mode 100644 modules/lang-mustache/src/test/resources/org/elasticsearch/messy/tests/config/scripts/template_1.mustache
 create mode 100644 modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/10_basic.yaml
 create mode 100644 modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/20_stored_template.yaml
 create mode 100644 modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/30_file_template.yaml
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/template.msearch.json
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/template.search.json
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yaml

diff --git a/core/src/main/java/org/elasticsearch/action/ActionModule.java b/core/src/main/java/org/elasticsearch/action/ActionModule.java
index f04e5499207..a3a1fbca153 100644
--- a/core/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/core/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -66,8 +66,6 @@ import org.elasticsearch.action.admin.cluster.stats.ClusterStatsAction;
 import org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction;
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksAction;
 import org.elasticsearch.action.admin.cluster.tasks.TransportPendingClusterTasksAction;
-import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateAction;
-import org.elasticsearch.action.admin.cluster.validate.template.TransportRenderSearchTemplateAction;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction;
 import org.elasticsearch.action.admin.indices.alias.TransportIndicesAliasesAction;
 import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistAction;
@@ -340,7 +338,6 @@ public class ActionModule extends AbstractModule {
         registerAction(ExplainAction.INSTANCE, TransportExplainAction.class);
         registerAction(ClearScrollAction.INSTANCE, TransportClearScrollAction.class);
         registerAction(RecoveryAction.INSTANCE, TransportRecoveryAction.class);
-        registerAction(RenderSearchTemplateAction.INSTANCE, TransportRenderSearchTemplateAction.class);
 
         //Indexed scripts
         registerAction(PutStoredScriptAction.INSTANCE, TransportPutStoredScriptAction.class);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/validate/template/RenderSearchTemplateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/validate/template/RenderSearchTemplateRequest.java
deleted file mode 100644
index 81ff9021815..00000000000
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/validate/template/RenderSearchTemplateRequest.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.admin.cluster.validate.template;
-
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.script.Template;
-
-import java.io.IOException;
-
-public class RenderSearchTemplateRequest extends ActionRequest<RenderSearchTemplateRequest> {
-
-    private Template template;
-    
-    public void template(Template template) {
-        this.template = template;
-    }
-    
-    public Template template() {
-        return template;
-    }
-    
-    @Override
-    public ActionRequestValidationException validate() {
-        ActionRequestValidationException exception = null;
-        if (template == null) {
-            exception = new ActionRequestValidationException();
-            exception.addValidationError("template must not be null");
-        }
-        return exception;
-    }
-    
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        super.writeTo(out);
-        boolean hasTemplate = template!= null;
-        out.writeBoolean(hasTemplate);
-        if (hasTemplate) {
-            template.writeTo(out);
-        }
-    }
-    
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        super.readFrom(in);
-        if (in.readBoolean()) {
-            template = new Template(in);
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/validate/template/TransportRenderSearchTemplateAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/validate/template/TransportRenderSearchTemplateAction.java
deleted file mode 100644
index ad573c4a074..00000000000
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/validate/template/TransportRenderSearchTemplateAction.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.admin.cluster.validate.template;
-
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.support.ActionFilters;
-import org.elasticsearch.action.support.HandledTransportAction;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.script.ExecutableScript;
-import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.TransportService;
-
-import java.util.Collections;
-
-public class TransportRenderSearchTemplateAction extends HandledTransportAction<RenderSearchTemplateRequest, RenderSearchTemplateResponse> {
-
-    private final ScriptService scriptService;
-    private final ClusterService clusterService;
-
-    @Inject
-    public TransportRenderSearchTemplateAction(ScriptService scriptService, Settings settings, ThreadPool threadPool,
-            TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
-            ClusterService clusterService) {
-        super(settings, RenderSearchTemplateAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, RenderSearchTemplateRequest::new);
-        this.scriptService = scriptService;
-        this.clusterService = clusterService;
-    }
-
-    @Override
-    protected void doExecute(final RenderSearchTemplateRequest request, final ActionListener<RenderSearchTemplateResponse> listener) {
-        threadPool.generic().execute(new AbstractRunnable() {
-
-            @Override
-            public void onFailure(Throwable t) {
-                listener.onFailure(t);
-            }
-
-            @Override
-            protected void doRun() throws Exception {
-                ExecutableScript executable = scriptService.executable(request.template(), ScriptContext.Standard.SEARCH,
-                        Collections.emptyMap(), clusterService.state());
-                BytesReference processedTemplate = (BytesReference) executable.run();
-                RenderSearchTemplateResponse response = new RenderSearchTemplateResponse();
-                response.source(processedTemplate);
-                listener.onResponse(response);
-            }
-        });
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java
index a0ea18c2965..1973e4c756d 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequest.java
@@ -29,7 +29,6 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.script.Template;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
@@ -71,8 +70,6 @@ public class SearchRequest extends ActionRequest<SearchRequest> implements Indic
 
     private IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS;
 
-    private Template template;
-
     public SearchRequest() {
     }
 
@@ -222,21 +219,6 @@ public class SearchRequest extends ActionRequest<SearchRequest> implements Indic
         return source;
     }
 
-
-    /**
-     * The stored template
-     */
-    public void template(Template template) {
-        this.template = template;
-    }
-
-    /**
-     * The stored template
-     */
-    public Template template() {
-        return template;
-    }
-
     /**
      * The tye of search to execute.
      */
@@ -326,7 +308,6 @@ public class SearchRequest extends ActionRequest<SearchRequest> implements Indic
         indicesOptions = IndicesOptions.readIndicesOptions(in);
 
         requestCache = in.readOptionalBoolean();
-        template = in.readOptionalWriteable(Template::new);
     }
 
     @Override
@@ -357,6 +338,5 @@ public class SearchRequest extends ActionRequest<SearchRequest> implements Indic
         out.writeStringArray(types);
         indicesOptions.writeIndicesOptions(out);
         out.writeOptionalBoolean(requestCache);
-        out.writeOptionalWriteable(template);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
index bfdea9b19a9..5c08acb99ea 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
@@ -26,7 +26,6 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.Template;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
@@ -496,14 +495,6 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
         return this;
     }
 
-    /**
-     * template stuff
-     */
-    public SearchRequestBuilder setTemplate(Template template) {
-        request.template(template);
-        return this;
-    }
-
     /**
      * Sets if this request should use the request cache or not, assuming that it can (for
      * example, if "now" is used, it will never be cached). By default (not set, or null,
diff --git a/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java b/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java
index 953ca578579..9e0d1a94119 100644
--- a/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java
+++ b/core/src/main/java/org/elasticsearch/client/ClusterAdminClient.java
@@ -99,9 +99,6 @@ import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRespo
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequestBuilder;
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
-import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateRequest;
-import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateRequestBuilder;
-import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateResponse;
 import org.elasticsearch.action.ingest.DeletePipelineRequest;
 import org.elasticsearch.action.ingest.DeletePipelineRequestBuilder;
 import org.elasticsearch.action.ingest.GetPipelineRequest;
@@ -536,28 +533,6 @@ public interface ClusterAdminClient extends ElasticsearchClient {
      */
     SnapshotsStatusRequestBuilder prepareSnapshotStatus();
 
-
-    /**
-     * Return the rendered search request for a given search template.
-     *
-     * @param request The request
-     * @return The result future
-     */
-    ActionFuture<RenderSearchTemplateResponse> renderSearchTemplate(RenderSearchTemplateRequest request);
-
-    /**
-     * Return the rendered search request for a given search template.
-     *
-     * @param request  The request
-     * @param listener A listener to be notified of the result
-     */
-    void renderSearchTemplate(RenderSearchTemplateRequest request, ActionListener<RenderSearchTemplateResponse> listener);
-
-    /**
-     * Return the rendered search request for a given search template.
-     */
-    RenderSearchTemplateRequestBuilder prepareRenderSearchTemplate();
-
     /**
      * Stores an ingest pipeline
      */
diff --git a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
index 38ac4cd355e..c3816d8d37f 100644
--- a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
+++ b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java
@@ -129,10 +129,6 @@ import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksAction;
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequestBuilder;
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
-import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateAction;
-import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateRequest;
-import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateRequestBuilder;
-import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateResponse;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
@@ -1073,21 +1069,6 @@ public abstract class AbstractClient extends AbstractComponent implements Client
             return new SnapshotsStatusRequestBuilder(this, SnapshotsStatusAction.INSTANCE);
         }
 
-        @Override
-        public ActionFuture<RenderSearchTemplateResponse> renderSearchTemplate(final RenderSearchTemplateRequest request) {
-            return execute(RenderSearchTemplateAction.INSTANCE, request);
-        }
-
-        @Override
-        public void renderSearchTemplate(final RenderSearchTemplateRequest request, final ActionListener<RenderSearchTemplateResponse> listener) {
-            execute(RenderSearchTemplateAction.INSTANCE, request, listener);
-        }
-
-        @Override
-        public RenderSearchTemplateRequestBuilder prepareRenderSearchTemplate() {
-            return new RenderSearchTemplateRequestBuilder(this, RenderSearchTemplateAction.INSTANCE);
-        }
-
         @Override
         public void putPipeline(PutPipelineRequest request, ActionListener<WritePipelineResponse> listener) {
             execute(PutPipelineAction.INSTANCE, request, listener);
diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
index df8f9316a68..13f92ba698d 100644
--- a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
+++ b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
@@ -66,11 +66,8 @@ import org.elasticsearch.rest.action.admin.cluster.snapshots.restore.RestRestore
 import org.elasticsearch.rest.action.admin.cluster.snapshots.status.RestSnapshotsStatusAction;
 import org.elasticsearch.rest.action.admin.cluster.state.RestClusterStateAction;
 import org.elasticsearch.rest.action.admin.cluster.stats.RestClusterStatsAction;
-import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestDeleteSearchTemplateAction;
 import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestDeleteStoredScriptAction;
-import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestGetSearchTemplateAction;
 import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestGetStoredScriptAction;
-import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestPutSearchTemplateAction;
 import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestPutStoredScriptAction;
 import org.elasticsearch.rest.action.admin.cluster.tasks.RestPendingClusterTasksAction;
 import org.elasticsearch.rest.action.admin.indices.RestRolloverIndexAction;
@@ -108,7 +105,6 @@ import org.elasticsearch.rest.action.admin.indices.template.head.RestHeadIndexTe
 import org.elasticsearch.rest.action.admin.indices.template.put.RestPutIndexTemplateAction;
 import org.elasticsearch.rest.action.admin.indices.upgrade.RestUpgradeAction;
 import org.elasticsearch.rest.action.admin.indices.validate.query.RestValidateQueryAction;
-import org.elasticsearch.rest.action.admin.indices.validate.template.RestRenderSearchTemplateAction;
 import org.elasticsearch.rest.action.bulk.RestBulkAction;
 import org.elasticsearch.rest.action.cat.AbstractCatAction;
 import org.elasticsearch.rest.action.cat.RestAliasAction;
@@ -256,7 +252,6 @@ public class NetworkModule extends AbstractModule {
         RestSearchScrollAction.class,
         RestClearScrollAction.class,
         RestMultiSearchAction.class,
-        RestRenderSearchTemplateAction.class,
 
         RestValidateQueryAction.class,
 
@@ -264,11 +259,6 @@ public class NetworkModule extends AbstractModule {
 
         RestRecoveryAction.class,
 
-        // Templates API
-        RestGetSearchTemplateAction.class,
-        RestPutSearchTemplateAction.class,
-        RestDeleteSearchTemplateAction.class,
-
         // Scripts API
         RestGetStoredScriptAction.class,
         RestPutStoredScriptAction.class,
@@ -359,6 +349,10 @@ public class NetworkModule extends AbstractModule {
         }
     }
 
+    public boolean isTransportClient() {
+        return transportClient;
+    }
+
     /** Adds a transport service implementation that can be selected by setting {@link #TRANSPORT_SERVICE_TYPE_KEY}. */
     public void registerTransportService(String name, Class<? extends TransportService> clazz) {
         transportServiceTypes.registerExtension(name, clazz);
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
index 36cc05c3515..41cb3bca82b 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
@@ -426,6 +426,7 @@ public final class ObjectParser<Value, Context extends ParseFieldMatcherSupplier
         OBJECT(START_OBJECT),
         OBJECT_ARRAY(START_OBJECT, START_ARRAY),
         OBJECT_OR_BOOLEAN(START_OBJECT, VALUE_BOOLEAN),
+        OBJECT_OR_STRING(START_OBJECT, VALUE_STRING),
         VALUE(VALUE_BOOLEAN, VALUE_NULL, VALUE_EMBEDDED_OBJECT, VALUE_NUMBER, VALUE_STRING);
 
         private final EnumSet<XContentParser.Token> tokens;
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index 89db99be923..edd5e5d8e4b 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -1048,10 +1048,6 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService>
      * Can the shard request be cached at all?
      */
     public boolean canCache(ShardSearchRequest request, SearchContext context) {
-        if (request.template() != null) {
-            return false;
-        }
-
         // for now, only enable it for requests with no hits
         if (context.size() != 0) {
             return false;
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestPutStoredScriptAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestPutStoredScriptAction.java
index 889ff459393..0ecb27b8d1e 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestPutStoredScriptAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestPutStoredScriptAction.java
@@ -43,21 +43,6 @@ public class RestPutStoredScriptAction extends BaseRestHandler {
         if (registerDefaultHandlers) {
             controller.registerHandler(POST, "/_scripts/{lang}/{id}", this);
             controller.registerHandler(PUT, "/_scripts/{lang}/{id}", this);
-
-            controller.registerHandler(PUT, "/_scripts/{lang}/{id}/_create", new CreateHandler(settings, controller, client));
-            controller.registerHandler(POST, "/_scripts/{lang}/{id}/_create", new CreateHandler(settings, controller, client));
-        }
-    }
-
-    final class CreateHandler extends BaseRestHandler {
-        protected CreateHandler(Settings settings, RestController controller, Client client) {
-            super(settings, client);
-        }
-
-        @Override
-        public void handleRequest(RestRequest request, RestChannel channel, final Client client) {
-            request.params().put("op_type", "create");
-            RestPutStoredScriptAction.this.handleRequest(request, channel, client);
         }
     }
 
@@ -71,5 +56,4 @@ public class RestPutStoredScriptAction extends BaseRestHandler {
         putRequest.script(request.content());
         client.admin().cluster().putStoredScript(putRequest, new AcknowledgedRestListener<>(channel));
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/template/RestRenderSearchTemplateAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/template/RestRenderSearchTemplateAction.java
deleted file mode 100644
index 0db4b5b179c..00000000000
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/template/RestRenderSearchTemplateAction.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.rest.action.admin.indices.validate.template;
-
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateRequest;
-import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.BytesRestResponse;
-import org.elasticsearch.rest.RestChannel;
-import org.elasticsearch.rest.RestController;
-import org.elasticsearch.rest.RestRequest;
-import org.elasticsearch.rest.RestResponse;
-import org.elasticsearch.rest.action.support.RestActions;
-import org.elasticsearch.rest.action.support.RestBuilderListener;
-import org.elasticsearch.script.Script.ScriptField;
-import org.elasticsearch.script.ScriptService.ScriptType;
-import org.elasticsearch.script.Template;
-
-import java.util.Map;
-
-import static org.elasticsearch.rest.RestRequest.Method.GET;
-import static org.elasticsearch.rest.RestRequest.Method.POST;
-import static org.elasticsearch.rest.RestStatus.OK;
-
-public class RestRenderSearchTemplateAction extends BaseRestHandler {
-
-    @Inject
-    public RestRenderSearchTemplateAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
-        controller.registerHandler(GET, "/_render/template", this);
-        controller.registerHandler(POST, "/_render/template", this);
-        controller.registerHandler(GET, "/_render/template/{id}", this);
-        controller.registerHandler(POST, "/_render/template/{id}", this);
-    }
-
-    @Override
-    protected void handleRequest(RestRequest request, RestChannel channel, Client client) throws Exception {
-        RenderSearchTemplateRequest renderSearchTemplateRequest;
-        BytesReference source = RestActions.getRestContent(request);
-        try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
-            String templateId = request.param("id");
-            final Template template;
-            if (templateId == null) {
-                template = Template.parse(parser, parseFieldMatcher);
-            } else {
-                Map<String, Object> params = null;
-                String currentFieldName = null;
-                XContentParser.Token token = parser.nextToken();
-                if (token != XContentParser.Token.START_OBJECT) {
-                    throw new ElasticsearchParseException("failed to parse request. request body must be an object but found [{}] instead",
-                            token);
-                }
-                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                    if (token == XContentParser.Token.FIELD_NAME) {
-                        currentFieldName = parser.currentName();
-                    } else if (parseFieldMatcher.match(currentFieldName, ScriptField.PARAMS)) {
-                        if (token == XContentParser.Token.START_OBJECT) {
-                            params = parser.map();
-                        } else {
-                            throw new ElasticsearchParseException(
-                                    "failed to parse request. field [{}] is expected to be an object, but found [{}] instead",
-                                    currentFieldName, token);
-                        }
-                    } else {
-                        throw new ElasticsearchParseException("failed to parse request. unknown field [{}] of type [{}]", currentFieldName,
-                                token);
-                    }
-                }
-                template = new Template(templateId, ScriptType.STORED, Template.DEFAULT_LANG, null, params);
-            }
-            renderSearchTemplateRequest = new RenderSearchTemplateRequest();
-            renderSearchTemplateRequest.template(template);
-        }
-        client.admin().cluster().renderSearchTemplate(renderSearchTemplateRequest, new RestBuilderListener<RenderSearchTemplateResponse>(channel) {
-
-            @Override
-            public RestResponse buildResponse(RenderSearchTemplateResponse response, XContentBuilder builder) throws Exception {
-                builder.prettyPrint();
-                response.toXContent(builder, ToXContent.EMPTY_PARAMS);
-                return new BytesRestResponse(OK, builder);
-            }});
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
index 2050360a9d5..2935c88dab2 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
@@ -19,19 +19,11 @@
 
 package org.elasticsearch.rest.action.search;
 
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringArrayValue;
-import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringValue;
-import static org.elasticsearch.rest.RestRequest.Method.GET;
-import static org.elasticsearch.rest.RestRequest.Method.POST;
-
-import java.util.Map;
-
+import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.search.MultiSearchRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -41,7 +33,6 @@ import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.index.query.TemplateQueryBuilder;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestChannel;
@@ -49,11 +40,20 @@ import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.support.RestActions;
 import org.elasticsearch.rest.action.support.RestToXContentListener;
-import org.elasticsearch.script.Template;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.suggest.Suggesters;
 
+import java.io.IOException;
+import java.util.Map;
+import java.util.function.BiConsumer;
+
+import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
+import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringArrayValue;
+import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeStringValue;
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+
 /**
  */
 public class RestMultiSearchAction extends BaseRestHandler {
@@ -77,48 +77,55 @@ public class RestMultiSearchAction extends BaseRestHandler {
         controller.registerHandler(GET, "/{index}/{type}/_msearch", this);
         controller.registerHandler(POST, "/{index}/{type}/_msearch", this);
 
-        controller.registerHandler(GET, "/_msearch/template", this);
-        controller.registerHandler(POST, "/_msearch/template", this);
-        controller.registerHandler(GET, "/{index}/_msearch/template", this);
-        controller.registerHandler(POST, "/{index}/_msearch/template", this);
-        controller.registerHandler(GET, "/{index}/{type}/_msearch/template", this);
-        controller.registerHandler(POST, "/{index}/{type}/_msearch/template", this);
-
         this.allowExplicitIndex = MULTI_ALLOW_EXPLICIT_INDEX.get(settings);
         this.indicesQueriesRegistry = indicesQueriesRegistry;
     }
 
     @Override
     public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws Exception {
-        MultiSearchRequest multiSearchRequest = new MultiSearchRequest();
-        if (request.hasParam("max_concurrent_searches")) {
-            multiSearchRequest.maxConcurrentSearchRequests(request.paramAsInt("max_concurrent_searches", 0));
-        }
-
-        String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
-        String[] types = Strings.splitStringByCommaToArray(request.param("type"));
-        String path = request.path();
-        boolean isTemplateRequest = isTemplateRequest(path);
-        IndicesOptions indicesOptions = IndicesOptions.fromRequest(request, multiSearchRequest.indicesOptions());
-        parseRequest(multiSearchRequest, RestActions.getRestContent(request), isTemplateRequest, indices, types,
-                request.param("search_type"), request.param("routing"), indicesOptions, allowExplicitIndex, indicesQueriesRegistry,
-                parseFieldMatcher, aggParsers, suggesters);
+        MultiSearchRequest multiSearchRequest = parseRequest(request, allowExplicitIndex, indicesQueriesRegistry, parseFieldMatcher,
+                aggParsers, suggesters);
         client.multiSearch(multiSearchRequest, new RestToXContentListener<>(channel));
     }
 
-    private boolean isTemplateRequest(String path) {
-        return (path != null && path.endsWith("/template"));
+    /**
+     * Parses a {@link RestRequest} body and returns a {@link MultiSearchRequest}
+     */
+    public static MultiSearchRequest parseRequest(RestRequest restRequest, boolean allowExplicitIndex,
+                                                  IndicesQueriesRegistry queriesRegistry, ParseFieldMatcher parseFieldMatcher,
+                                                  AggregatorParsers aggParsers, Suggesters suggesters) throws IOException {
+
+        MultiSearchRequest multiRequest = new MultiSearchRequest();
+        if (restRequest.hasParam("max_concurrent_searches")) {
+            multiRequest.maxConcurrentSearchRequests(restRequest.paramAsInt("max_concurrent_searches", 0));
+        }
+
+        parseMultiLineRequest(restRequest, multiRequest.indicesOptions(), allowExplicitIndex, (searchRequest, bytes) -> {
+            try (XContentParser requestParser = XContentFactory.xContent(bytes).createParser(bytes)) {
+                final QueryParseContext queryParseContext = new QueryParseContext(queriesRegistry, requestParser, parseFieldMatcher);
+                searchRequest.source(SearchSourceBuilder.fromXContent(queryParseContext, aggParsers, suggesters));
+                multiRequest.add(searchRequest);
+            } catch (IOException e) {
+                throw new ElasticsearchParseException("Exception when parsing search request", e);
+            }
+        });
+
+        return multiRequest;
     }
 
-    public static MultiSearchRequest parseRequest(MultiSearchRequest msr, BytesReference data, boolean isTemplateRequest,
-                                                   @Nullable String[] indices,
-                                                   @Nullable String[] types,
-                                                   @Nullable String searchType,
-                                                   @Nullable String routing,
-                                                   IndicesOptions indicesOptions,
-                                                   boolean allowExplicitIndex, IndicesQueriesRegistry indicesQueriesRegistry,
-                                                   ParseFieldMatcher parseFieldMatcher, AggregatorParsers aggParsers,
-                                                   Suggesters suggesters) throws Exception {
+    /**
+     * Parses a multi-line {@link RestRequest} body, instanciating a {@link SearchRequest} for each line and applying the given consumer.
+     */
+    public static void parseMultiLineRequest(RestRequest request, IndicesOptions indicesOptions, boolean allowExplicitIndex,
+                                             BiConsumer<SearchRequest, BytesReference> consumer) throws IOException {
+
+        String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
+        String[] types = Strings.splitStringByCommaToArray(request.param("type"));
+        String searchType = request.param("search_type");
+        String routing = request.param("routing");
+
+        final BytesReference data = RestActions.getRestContent(request);
+
         XContent xContent = XContentFactory.xContent(data);
         int from = 0;
         int length = data.length();
@@ -147,7 +154,9 @@ public class RestMultiSearchAction extends BaseRestHandler {
             if (routing != null) {
                 searchRequest.routing(routing);
             }
-            searchRequest.searchType(searchType);
+            if (searchType != null) {
+                searchRequest.searchType(searchType);
+            }
 
             IndicesOptions defaultOptions = IndicesOptions.strictExpandOpenAndForbidClosed();
 
@@ -187,26 +196,10 @@ public class RestMultiSearchAction extends BaseRestHandler {
             if (nextMarker == -1) {
                 break;
             }
-            final BytesReference slice = data.slice(from, nextMarker - from);
-            if (isTemplateRequest) {
-                try (XContentParser parser = XContentFactory.xContent(slice).createParser(slice)) {
-                    final QueryParseContext queryParseContext = new QueryParseContext(indicesQueriesRegistry, parser, parseFieldMatcher);
-                    Template template = TemplateQueryBuilder.parse(parser, queryParseContext.getParseFieldMatcher(), "params", "template");
-                    searchRequest.template(template);
-                }
-            } else {
-                try (XContentParser requestParser = XContentFactory.xContent(slice).createParser(slice)) {
-                    final QueryParseContext queryParseContext = new QueryParseContext(indicesQueriesRegistry, requestParser,
-                            parseFieldMatcher);
-                    searchRequest.source(SearchSourceBuilder.fromXContent(queryParseContext, aggParsers, suggesters));
-                }
-            }
+            consumer.accept(searchRequest, data.slice(from, nextMarker - from));
             // move pointers
             from = nextMarker + 1;
-
-            msr.add(searchRequest);
         }
-        return msr;
     }
 
     private static int findNextMarker(byte marker, int from, BytesReference data, int length) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
index 713981fc0d5..f46c41dcd17 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
@@ -33,7 +33,6 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.index.query.TemplateQueryBuilder;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestChannel;
@@ -41,7 +40,6 @@ import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.support.RestActions;
 import org.elasticsearch.rest.action.support.RestStatusToXContentListener;
-import org.elasticsearch.script.Template;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -82,18 +80,13 @@ public class RestSearchAction extends BaseRestHandler {
         controller.registerHandler(POST, "/{index}/_search", this);
         controller.registerHandler(GET, "/{index}/{type}/_search", this);
         controller.registerHandler(POST, "/{index}/{type}/_search", this);
-        controller.registerHandler(GET, "/_search/template", this);
-        controller.registerHandler(POST, "/_search/template", this);
-        controller.registerHandler(GET, "/{index}/_search/template", this);
-        controller.registerHandler(POST, "/{index}/_search/template", this);
-        controller.registerHandler(GET, "/{index}/{type}/_search/template", this);
-        controller.registerHandler(POST, "/{index}/{type}/_search/template", this);
     }
 
     @Override
     public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws IOException {
         SearchRequest searchRequest = new SearchRequest();
-        parseSearchRequest(searchRequest, queryRegistry, request, parseFieldMatcher, aggParsers, suggesters, null);
+        BytesReference restContent = RestActions.hasBodyContent(request) ? RestActions.getRestContent(request) : null;
+        parseSearchRequest(searchRequest, queryRegistry, request, parseFieldMatcher, aggParsers, suggesters, restContent);
         client.search(searchRequest, new RestStatusToXContentListener<>(channel));
     }
 
@@ -114,23 +107,10 @@ public class RestSearchAction extends BaseRestHandler {
             searchRequest.source(new SearchSourceBuilder());
         }
         searchRequest.indices(Strings.splitStringByCommaToArray(request.param("index")));
-        // get the content, and put it in the body
-        // add content/source as template if template flag is set
-        boolean isTemplateRequest = request.path().endsWith("/template");
-        if (restContent == null) {
-            if (RestActions.hasBodyContent(request)) {
-                restContent = RestActions.getRestContent(request);
-            }
-        }
         if (restContent != null) {
             try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) {
                 QueryParseContext context = new QueryParseContext(indicesQueriesRegistry, parser, parseFieldMatcher);
-                if (isTemplateRequest) {
-                    Template template = TemplateQueryBuilder.parse(parser, context.getParseFieldMatcher(), "params", "template");
-                    searchRequest.template(template);
-                } else {
-                    searchRequest.source().parseXContent(context, aggParsers, suggesters);
-                }
+                searchRequest.source().parseXContent(context, aggParsers, suggesters);
             }
         }
 
diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java
index 362b3f0f416..6ceb021d9f6 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchService.java
@@ -39,7 +39,6 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lucene.Lucene;
@@ -58,20 +57,17 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.InnerHitBuilder;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.SearchOperationListener;
 import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.aggregations.AggregationInitializationException;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
-import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.aggregations.SearchContextAggregations;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -103,7 +99,6 @@ import org.elasticsearch.search.rescore.RescoreBuilder;
 import org.elasticsearch.search.searchafter.SearchAfterBuilder;
 import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.sort.SortBuilder;
-import org.elasticsearch.search.suggest.Suggesters;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import com.carrotsearch.hppc.ObjectFloatHashMap;
@@ -153,16 +148,11 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
     private final Map<String, SearchParseElement> elementParsers;
 
     private final ParseFieldMatcher parseFieldMatcher;
-    private final AggregatorParsers aggParsers;
-    private final Suggesters suggesters;
 
     @Inject
     public SearchService(Settings settings, ClusterSettings clusterSettings, ClusterService clusterService, IndicesService indicesService,
-                         ThreadPool threadPool, ScriptService scriptService, BigArrays bigArrays,
-                          FetchPhase fetchPhase, AggregatorParsers aggParsers, Suggesters suggesters) {
+                         ThreadPool threadPool, ScriptService scriptService, BigArrays bigArrays, FetchPhase fetchPhase) {
         super(settings);
-        this.aggParsers = aggParsers;
-        this.suggesters = suggesters;
         this.parseFieldMatcher = new ParseFieldMatcher(settings);
         this.threadPool = threadPool;
         this.clusterService = clusterService;
@@ -556,16 +546,6 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
                 context.scrollContext(new ScrollContext());
                 context.scrollContext().scroll = request.scroll();
             }
-            if (request.template() != null) {
-                ExecutableScript executable = this.scriptService.executable(request.template(), ScriptContext.Standard.SEARCH,
-                        Collections.emptyMap(), context.getQueryShardContext().getClusterState());
-                BytesReference run = (BytesReference) executable.run();
-                try (XContentParser parser = XContentFactory.xContent(run).createParser(run)) {
-                    QueryParseContext queryParseContext = new QueryParseContext(indicesService.getIndicesQueryRegistry(), parser,
-                            parseFieldMatcher);
-                    parseSource(context, SearchSourceBuilder.fromXContent(queryParseContext, aggParsers, suggesters));
-                }
-            }
             parseSource(context, request.source());
 
             // if the from and size are still not set, default them
diff --git a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java
index 8c5a4686d3f..5b12c941185 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java
@@ -29,7 +29,6 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.script.Template;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
@@ -66,7 +65,6 @@ public class ShardSearchLocalRequest implements ShardSearchRequest {
     private String[] types = Strings.EMPTY_ARRAY;
     private String[] filteringAliases;
     private SearchSourceBuilder source;
-    private Template template;
     private Boolean requestCache;
     private long nowInMillis;
 
@@ -79,7 +77,6 @@ public class ShardSearchLocalRequest implements ShardSearchRequest {
                             String[] filteringAliases, long nowInMillis) {
         this(shardRouting.shardId(), numberOfShards, searchRequest.searchType(),
                 searchRequest.source(), searchRequest.types(), searchRequest.requestCache());
-        this.template = searchRequest.template();
         this.scroll = searchRequest.scroll();
         this.filteringAliases = filteringAliases;
         this.nowInMillis = nowInMillis;
@@ -145,10 +142,6 @@ public class ShardSearchLocalRequest implements ShardSearchRequest {
     public long nowInMillis() {
         return nowInMillis;
     }
-    @Override
-    public Template template() {
-        return template;
-    }
 
     @Override
     public Boolean requestCache() {
@@ -183,7 +176,6 @@ public class ShardSearchLocalRequest implements ShardSearchRequest {
         types = in.readStringArray();
         filteringAliases = in.readStringArray();
         nowInMillis = in.readVLong();
-        template = in.readOptionalWriteable(Template::new);
         requestCache = in.readOptionalBoolean();
     }
 
@@ -211,8 +203,6 @@ public class ShardSearchLocalRequest implements ShardSearchRequest {
         if (!asKey) {
             out.writeVLong(nowInMillis);
         }
-
-        out.writeOptionalWriteable(template);
         out.writeOptionalBoolean(requestCache);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
index aa148e215c8..6c237322f04 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java
@@ -23,7 +23,6 @@ import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.script.Template;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
@@ -52,8 +51,6 @@ public interface ShardSearchRequest {
 
     long nowInMillis();
 
-    Template template();
-
     Boolean requestCache();
 
     Scroll scroll();
diff --git a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java
index cd6460a686f..45124dc8b52 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java
@@ -30,7 +30,6 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.script.Template;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.transport.TransportRequest;
@@ -113,10 +112,6 @@ public class ShardSearchTransportRequest extends TransportRequest implements Sha
     public long nowInMillis() {
         return shardSearchLocalRequest.nowInMillis();
     }
-    @Override
-    public Template template() {
-        return shardSearchLocalRequest.template();
-    }
 
     @Override
     public Boolean requestCache() {
diff --git a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
index 1039df68645..f0f03347773 100644
--- a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.action.search;
 
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -28,21 +29,21 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryParser;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.search.RestMultiSearchAction;
-import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.StreamsUtils;
+import org.elasticsearch.test.rest.FakeRestRequest;
 
 import java.io.IOException;
 
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 
 public class MultiSearchRequestTests extends ESTestCase {
     public void testSimpleAdd() throws Exception {
-        byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/search/simple-msearch1.json");
-        MultiSearchRequest request = RestMultiSearchAction.parseRequest(new MultiSearchRequest(), new BytesArray(data), false, null, null,
-                null, null, IndicesOptions.strictExpandOpenAndForbidClosed(), true, registry(), ParseFieldMatcher.EMPTY, null, null);
+        MultiSearchRequest request = parseMultiSearchRequest("/org/elasticsearch/action/search/simple-msearch1.json");
         assertThat(request.requests().size(), equalTo(8));
         assertThat(request.requests().get(0).indices()[0], equalTo("test"));
         assertThat(request.requests().get(0).indicesOptions(), equalTo(IndicesOptions.fromOptions(true, true, true, true, IndicesOptions.strictExpandOpenAndForbidClosed())));
@@ -56,37 +57,33 @@ public class MultiSearchRequestTests extends ESTestCase {
         assertThat(request.requests().get(3).indicesOptions(), equalTo(IndicesOptions.fromOptions(true, true, true, true, IndicesOptions.strictExpandOpenAndForbidClosed())));
         assertThat(request.requests().get(4).indices()[0], equalTo("test"));
         assertThat(request.requests().get(4).indicesOptions(), equalTo(IndicesOptions.fromOptions(true, false, false, true, IndicesOptions.strictExpandOpenAndForbidClosed())));
-        assertThat(request.requests().get(5).indices(), nullValue());
+        assertThat(request.requests().get(5).indices(), is(Strings.EMPTY_ARRAY));
         assertThat(request.requests().get(5).types().length, equalTo(0));
-        assertThat(request.requests().get(6).indices(), nullValue());
+        assertThat(request.requests().get(6).indices(), is(Strings.EMPTY_ARRAY));
         assertThat(request.requests().get(6).types().length, equalTo(0));
         assertThat(request.requests().get(6).searchType(), equalTo(SearchType.DFS_QUERY_THEN_FETCH));
-        assertThat(request.requests().get(7).indices(), nullValue());
+        assertThat(request.requests().get(7).indices(), is(Strings.EMPTY_ARRAY));
         assertThat(request.requests().get(7).types().length, equalTo(0));
     }
 
     public void testSimpleAdd2() throws Exception {
-        byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/search/simple-msearch2.json");
-        MultiSearchRequest request = RestMultiSearchAction.parseRequest(new MultiSearchRequest(), new BytesArray(data), false, null, null,
-                null, null, IndicesOptions.strictExpandOpenAndForbidClosed(), true, registry(), ParseFieldMatcher.EMPTY, null, null);
+        MultiSearchRequest request = parseMultiSearchRequest("/org/elasticsearch/action/search/simple-msearch2.json");
         assertThat(request.requests().size(), equalTo(5));
         assertThat(request.requests().get(0).indices()[0], equalTo("test"));
         assertThat(request.requests().get(0).types().length, equalTo(0));
         assertThat(request.requests().get(1).indices()[0], equalTo("test"));
         assertThat(request.requests().get(1).types()[0], equalTo("type1"));
-        assertThat(request.requests().get(2).indices(), nullValue());
+        assertThat(request.requests().get(2).indices(), is(Strings.EMPTY_ARRAY));
         assertThat(request.requests().get(2).types().length, equalTo(0));
-        assertThat(request.requests().get(3).indices(), nullValue());
+        assertThat(request.requests().get(3).indices(), is(Strings.EMPTY_ARRAY));
         assertThat(request.requests().get(3).types().length, equalTo(0));
         assertThat(request.requests().get(3).searchType(), equalTo(SearchType.DFS_QUERY_THEN_FETCH));
-        assertThat(request.requests().get(4).indices(), nullValue());
+        assertThat(request.requests().get(4).indices(), is(Strings.EMPTY_ARRAY));
         assertThat(request.requests().get(4).types().length, equalTo(0));
     }
 
     public void testSimpleAdd3() throws Exception {
-        byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/search/simple-msearch3.json");
-        MultiSearchRequest request = RestMultiSearchAction.parseRequest(new MultiSearchRequest(), new BytesArray(data), false, null, null,
-                null, null, IndicesOptions.strictExpandOpenAndForbidClosed(), true, registry(), ParseFieldMatcher.EMPTY, null, null);
+        MultiSearchRequest request = parseMultiSearchRequest("/org/elasticsearch/action/search/simple-msearch3.json");
         assertThat(request.requests().size(), equalTo(4));
         assertThat(request.requests().get(0).indices()[0], equalTo("test0"));
         assertThat(request.requests().get(0).indices()[1], equalTo("test1"));
@@ -97,15 +94,13 @@ public class MultiSearchRequestTests extends ESTestCase {
         assertThat(request.requests().get(2).indices()[1], equalTo("test1"));
         assertThat(request.requests().get(2).types()[0], equalTo("type2"));
         assertThat(request.requests().get(2).types()[1], equalTo("type1"));
-        assertThat(request.requests().get(3).indices(), nullValue());
+        assertThat(request.requests().get(3).indices(), is(Strings.EMPTY_ARRAY));
         assertThat(request.requests().get(3).types().length, equalTo(0));
         assertThat(request.requests().get(3).searchType(), equalTo(SearchType.DFS_QUERY_THEN_FETCH));
     }
 
     public void testSimpleAdd4() throws Exception {
-        byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/search/simple-msearch4.json");
-        MultiSearchRequest request = RestMultiSearchAction.parseRequest(new MultiSearchRequest(), new BytesArray(data), false, null, null,
-                null, null, IndicesOptions.strictExpandOpenAndForbidClosed(), true, registry(), ParseFieldMatcher.EMPTY, null, null);
+        MultiSearchRequest request = parseMultiSearchRequest("/org/elasticsearch/action/search/simple-msearch4.json");
         assertThat(request.requests().size(), equalTo(3));
         assertThat(request.requests().get(0).indices()[0], equalTo("test0"));
         assertThat(request.requests().get(0).indices()[1], equalTo("test1"));
@@ -123,39 +118,6 @@ public class MultiSearchRequestTests extends ESTestCase {
         assertThat(request.requests().get(2).routing(), equalTo("123"));
     }
 
-    public void testSimpleAdd5() throws Exception {
-        byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/search/simple-msearch5.json");
-        MultiSearchRequest request = RestMultiSearchAction.parseRequest(new MultiSearchRequest(), new BytesArray(data), true, null, null,
-                null, null, IndicesOptions.strictExpandOpenAndForbidClosed(), true, registry(), ParseFieldMatcher.EMPTY, null, null);
-        assertThat(request.requests().size(), equalTo(3));
-        assertThat(request.requests().get(0).indices()[0], equalTo("test0"));
-        assertThat(request.requests().get(0).indices()[1], equalTo("test1"));
-        assertThat(request.requests().get(0).requestCache(), equalTo(true));
-        assertThat(request.requests().get(0).preference(), nullValue());
-        assertThat(request.requests().get(1).indices()[0], equalTo("test2"));
-        assertThat(request.requests().get(1).indices()[1], equalTo("test3"));
-        assertThat(request.requests().get(1).types()[0], equalTo("type1"));
-        assertThat(request.requests().get(1).requestCache(), nullValue());
-        assertThat(request.requests().get(1).preference(), equalTo("_local"));
-        assertThat(request.requests().get(2).indices()[0], equalTo("test4"));
-        assertThat(request.requests().get(2).indices()[1], equalTo("test1"));
-        assertThat(request.requests().get(2).types()[0], equalTo("type2"));
-        assertThat(request.requests().get(2).types()[1], equalTo("type1"));
-        assertThat(request.requests().get(2).routing(), equalTo("123"));
-        assertNotNull(request.requests().get(0).template());
-        assertNotNull(request.requests().get(1).template());
-        assertNotNull(request.requests().get(2).template());
-        assertEquals(ScriptService.ScriptType.INLINE, request.requests().get(0).template().getType());
-        assertEquals(ScriptService.ScriptType.INLINE, request.requests().get(1).template().getType());
-        assertEquals(ScriptService.ScriptType.INLINE, request.requests().get(2).template().getType());
-        assertEquals("{\"query\":{\"match_{{template}}\":{}}}", request.requests().get(0).template().getScript());
-        assertEquals("{\"query\":{\"match_{{template}}\":{}}}", request.requests().get(1).template().getScript());
-        assertEquals("{\"query\":{\"match_{{template}}\":{}}}", request.requests().get(2).template().getScript());
-        assertEquals(1, request.requests().get(0).template().getParams().size());
-        assertEquals(1, request.requests().get(1).template().getParams().size());
-        assertEquals(1, request.requests().get(2).template().getParams().size());
-    }
-
     public void testResponseErrorToXContent() throws IOException {
         MultiSearchResponse response = new MultiSearchResponse(new MultiSearchResponse.Item[]{new MultiSearchResponse.Item(null, new IllegalStateException("foobar")), new MultiSearchResponse.Item(null, new IllegalStateException("baaaaaazzzz"))});
         XContentBuilder builder = XContentFactory.jsonBuilder();
@@ -171,6 +133,12 @@ public class MultiSearchRequestTests extends ESTestCase {
                 request.maxConcurrentSearchRequests(randomIntBetween(Integer.MIN_VALUE, 0)));
     }
 
+    private MultiSearchRequest parseMultiSearchRequest(String sample) throws IOException {
+        byte[] data = StreamsUtils.copyToBytesFromClasspath(sample);
+        RestRequest restRequest = new FakeRestRequest.Builder().withContent(new BytesArray(data)).build();
+        return RestMultiSearchAction.parseRequest(restRequest, true, registry(), ParseFieldMatcher.EMPTY, null, null);
+    }
+
     private IndicesQueriesRegistry registry() {
         IndicesQueriesRegistry registry = new IndicesQueriesRegistry();
         QueryParser<MatchAllQueryBuilder> parser = MatchAllQueryBuilder::fromXContent;
diff --git a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
index 9868d92129f..d54edbcaa9d 100644
--- a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
@@ -115,11 +115,13 @@ public class NetworkModuleTests extends ModuleTestCase {
         NetworkModule module = new NetworkModule(new NetworkService(settings), settings, false, new NamedWriteableRegistry());
         module.registerTransportService("custom", FakeTransportService.class);
         assertBinding(module, TransportService.class, FakeTransportService.class);
+        assertFalse(module.isTransportClient());
 
         // check it works with transport only as well
         module = new NetworkModule(new NetworkService(settings), settings, true, new NamedWriteableRegistry());
         module.registerTransportService("custom", FakeTransportService.class);
         assertBinding(module, TransportService.class, FakeTransportService.class);
+        assertTrue(module.isTransportClient());
     }
 
     public void testRegisterTransport() {
@@ -127,11 +129,13 @@ public class NetworkModuleTests extends ModuleTestCase {
         NetworkModule module = new NetworkModule(new NetworkService(settings), settings, false, new NamedWriteableRegistry());
         module.registerTransport("custom", FakeTransport.class);
         assertBinding(module, Transport.class, FakeTransport.class);
+        assertFalse(module.isTransportClient());
 
         // check it works with transport only as well
         module = new NetworkModule(new NetworkService(settings), settings, true, new NamedWriteableRegistry());
         module.registerTransport("custom", FakeTransport.class);
         assertBinding(module, Transport.class, FakeTransport.class);
+        assertTrue(module.isTransportClient());
     }
 
     public void testRegisterHttpTransport() {
@@ -139,9 +143,11 @@ public class NetworkModuleTests extends ModuleTestCase {
         NetworkModule module = new NetworkModule(new NetworkService(settings), settings, false, new NamedWriteableRegistry());
         module.registerHttpTransport("custom", FakeHttpTransport.class);
         assertBinding(module, HttpServerTransport.class, FakeHttpTransport.class);
+        assertFalse(module.isTransportClient());
 
         // check registration not allowed for transport only
         module = new NetworkModule(new NetworkService(settings), settings, true, new NamedWriteableRegistry());
+        assertTrue(module.isTransportClient());
         try {
             module.registerHttpTransport("custom", FakeHttpTransport.class);
             fail();
@@ -154,6 +160,7 @@ public class NetworkModuleTests extends ModuleTestCase {
         settings = Settings.builder().put(NetworkModule.HTTP_ENABLED.getKey(), false).build();
         module = new NetworkModule(new NetworkService(settings), settings, false, new NamedWriteableRegistry());
         assertNotBound(module, HttpServerTransport.class);
+        assertFalse(module.isTransportClient());
     }
 
     public void testRegisterRestHandler() {
@@ -186,6 +193,7 @@ public class NetworkModuleTests extends ModuleTestCase {
         NamedWriteableRegistry registry = new NamedWriteableRegistry();
         Settings settings = Settings.EMPTY;
         NetworkModule module = new NetworkModule(new NetworkService(settings), settings, false, registry);
+        assertFalse(module.isTransportClient());
 
         // Builtin reader comes back
         assertNotNull(registry.getReader(Task.Status.class, ReplicationTask.Status.NAME));
diff --git a/core/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java b/core/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java
index 3c82d3063af..04291d957a8 100644
--- a/core/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java
+++ b/core/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java
@@ -29,7 +29,6 @@ import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.script.Template;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.internal.SearchContext;
@@ -93,11 +92,6 @@ public class SearchSlowLogTests extends ESSingleNodeTestCase {
                         return 0;
                     }
 
-                    @Override
-                    public Template template() {
-                        return null;
-                    }
-
                     @Override
                     public Boolean requestCache() {
                         return null;
diff --git a/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java b/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java
index a46ee0d6640..d8d6b0f5409 100644
--- a/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java
+++ b/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java
@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.script;
 
-import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateResponse;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
@@ -55,11 +54,6 @@ public class StoredScriptsIT extends ESIntegTestCase {
         assertNotNull(script);
         assertEquals("1", script);
 
-        RenderSearchTemplateResponse response = client().admin().cluster().prepareRenderSearchTemplate()
-                .template(new Template("/" + LANG + "/foobar", ScriptService.ScriptType.STORED, LANG, null, null))
-                .get();
-        assertEquals("1", response.source().toUtf8());
-
         assertAcked(client().admin().cluster().prepareDeleteStoredScript()
                 .setId("foobar")
                 .setScriptLang(LANG));
diff --git a/core/src/test/resources/org/elasticsearch/action/search/simple-msearch5.json b/core/src/test/resources/org/elasticsearch/action/search/simple-msearch5.json
deleted file mode 100644
index b337eae0654..00000000000
--- a/core/src/test/resources/org/elasticsearch/action/search/simple-msearch5.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{"index":["test0", "test1"], "request_cache": true}
-{"template": {"query" : {"match_{{template}}" :{}}}, "params": {"template": "all" } } }
-{"index" : "test2,test3", "type" : "type1", "preference": "_local"}
-{"template": {"query" : {"match_{{template}}" :{}}}, "params": {"template": "all" } } }
-{"index" : ["test4", "test1"], "type" :  [ "type2", "type1" ], "routing": "123"}
-{"template": {"query" : {"match_{{template}}" :{}}}, "params": {"template": "all" } } }
diff --git a/docs/reference/migration/migrate_5_0/java.asciidoc b/docs/reference/migration/migrate_5_0/java.asciidoc
index 00b33897f7d..7d3a21b3197 100644
--- a/docs/reference/migration/migrate_5_0/java.asciidoc
+++ b/docs/reference/migration/migrate_5_0/java.asciidoc
@@ -265,13 +265,14 @@ The `setTemplateSource(String)` and `setTemplateSource(BytesReference)` methods
 
 ==== SearchRequest
 
-All `template` methods have been removed in favor of a single `template(Template)` method.
-
 All `source` methods have been removed in favor of a single `source(SearchSourceBuilder)` method. This means that all search requests can now be validated
 at call time which results in much clearer errors.
 
 All `extraSource` methods have been removed.
 
+All `template` methods have been removed in favor of a new Search Template API. A new `SearchTemplateRequest` now accepts a template and
+a `SearchRequest` and must be executed using the new `SearchTemplateAction` action.
+
 ==== SearchResponse
 
 Sort values for `string` fields are now return as `java.lang.String` objects rather than `org.elasticsearch.common.text.Text`.
@@ -310,3 +311,11 @@ Removed the `getMemoryAvailable` method from `OsStats`, which could be previousl
 `setRefresh(boolean)` has been removed in favor of `setRefreshPolicy(RefreshPolicy)` because there
 are now three options (NONE, IMMEDIATE, and WAIT_FOR). `setRefresh(IMMEDIATE)` has the same behavior
 as `setRefresh(true)` used to have. See `setRefreshPolicy`'s javadoc for more.
+
+=== Render Search Template Java API has been removed
+
+The Render Search Template Java API including `RenderSearchTemplateAction`, `RenderSearchTemplateRequest` and
+`RenderSearchTemplateResponse` has been removed in favor of a new `simulate` option in the Search Template Java API.
+ This Search Template API is now included in the `lang-mustache` module and the `simulate` flag must be set on the
+ `SearchTemplateRequest` object.
+
diff --git a/docs/reference/migration/migrate_5_0/rest.asciidoc b/docs/reference/migration/migrate_5_0/rest.asciidoc
index 23928551f77..a9f70d2caae 100644
--- a/docs/reference/migration/migrate_5_0/rest.asciidoc
+++ b/docs/reference/migration/migrate_5_0/rest.asciidoc
@@ -62,3 +62,15 @@ renamed `filter`/`token_filter`/`char_filter`.
 
 The `DELETE /_query` endpoint provided by the Delete-By-Query plugin has been
 removed and replaced by the <<docs-delete-by-query,Delete By Query API>>.
+
+==== Create stored script endpoint removed
+
+The `PUT /_scripts/{lang}/{id}/_create` endpoint that previously allowed to create
+ indexed scripts has been removed. Indexed scripts have been replaced
+ by <<modules-scripting-stored-scripts,stored scripts>>.
+
+==== Create stored template endpoint removed
+
+The `PUT /_search/template/{id}/_create` endpoint that previously allowed to create
+ indexed template has been removed. Indexed templates have been replaced
+ by <<pre-registered-templates, Pre-registered templates>>.
diff --git a/docs/reference/migration/migrate_5_0/scripting.asciidoc b/docs/reference/migration/migrate_5_0/scripting.asciidoc
index 1d1a21034c7..f416984afa5 100644
--- a/docs/reference/migration/migrate_5_0/scripting.asciidoc
+++ b/docs/reference/migration/migrate_5_0/scripting.asciidoc
@@ -3,7 +3,7 @@
 
 ==== Indexed scripts and templates
 
-Indexed scrips and templates have been replaced by <<modules-scripting-stored-scripts,stored scripts>>
+Indexed scripts and templates have been replaced by <<modules-scripting-stored-scripts,stored scripts>>
 which stores the scripts and templates in the cluster state instead of a dedicate `.scripts` index.
 
 For the size of stored scripts there is a soft limit of 65535 bytes. If scripts exceed that size then
diff --git a/docs/reference/search/search-template.asciidoc b/docs/reference/search/search-template.asciidoc
index 25c4c0423f1..8533984428b 100644
--- a/docs/reference/search/search-template.asciidoc
+++ b/docs/reference/search/search-template.asciidoc
@@ -369,3 +369,52 @@ GET /_render/template/<template_name>
   }
 }
 ------------------------------------------
+
+[[multi-search-template]]
+== Multi Search Template
+
+The multi search template API allows to execute several search template
+requests within the same API using the `_msearch/template` endpoint.
+
+The format of the request is similar to the <<search-multi-search, Multi
+Search API>> format:
+
+[source,js]
+--------------------------------------------------
+header\n
+body\n
+header\n
+body\n
+--------------------------------------------------
+
+The header part supports the same `index`, `types`, `search_type`,
+`preference`, and `routing` options as the usual Multi Search API.
+
+The body includes a search template body request and supports inline,
+stored and file templates. Here is an example:
+
+[source,js]
+--------------------------------------------------
+$ cat requests
+{"index": "test"}
+{"inline": {"query": {"match":  {"user" : "{{username}}" }}}, "params": {"username": "john"}} <1>
+{"index": "_all", "types": "accounts"}
+{"inline": {"query": {"{{query_type}}": {"name": "{{name}}" }}}, "params": {"query_type": "match_phrase_prefix", "name": "Smith"}}
+{"index": "_all"}
+{"id": "template_1", "params": {"query_string": "search for these words" }} <2>
+{"types": "users"}
+{"file": "template_2", "params": {"field_name": "fullname", "field_value": "john smith" }} <3>
+
+$ curl -XGET localhost:9200/_msearch/template --data-binary "@requests"; echo
+--------------------------------------------------
+<1> Inline search template request
+
+<2> Search template request based on a stored template
+
+<3> Search template request based on a file template
+
+The response returns a `responses` array, which includes the search template
+response for each search template request matching its order in the original
+multi search template request. If there was a complete failure for that specific
+search template request, an object with `error` message will be returned in place
+of the actual search response.
diff --git a/modules/lang-mustache/build.gradle b/modules/lang-mustache/build.gradle
index 374fd94e7d9..eb5a48485b4 100644
--- a/modules/lang-mustache/build.gradle
+++ b/modules/lang-mustache/build.gradle
@@ -30,5 +30,9 @@ integTest {
   cluster {
     setting 'script.inline', 'true'
     setting 'script.stored', 'true'
+
+
+    File template = new File('src/test/resources/org/elasticsearch/messy/tests/config/scripts/template_1.mustache')
+    extraConfigFile 'scripts/template_1.mustache', template
   }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/validate/template/RenderSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateAction.java
similarity index 57%
rename from core/src/main/java/org/elasticsearch/action/admin/cluster/validate/template/RenderSearchTemplateAction.java
rename to modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateAction.java
index 427d0c4dc64..f5ab172f8ff 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/validate/template/RenderSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateAction.java
@@ -17,28 +17,28 @@
  * under the License.
  */
 
-package org.elasticsearch.action.admin.cluster.validate.template;
+package org.elasticsearch.action.search.template;
 
 import org.elasticsearch.action.Action;
 import org.elasticsearch.client.ElasticsearchClient;
 
-public class RenderSearchTemplateAction extends Action<RenderSearchTemplateRequest, RenderSearchTemplateResponse, RenderSearchTemplateRequestBuilder> {
+public class MultiSearchTemplateAction
+        extends Action<MultiSearchTemplateRequest, MultiSearchTemplateResponse, MultiSearchTemplateRequestBuilder> {
 
-    public static final RenderSearchTemplateAction INSTANCE = new RenderSearchTemplateAction();
-    public static final String NAME = "cluster:admin/render/template/search";
+    public static final MultiSearchTemplateAction INSTANCE = new MultiSearchTemplateAction();
+    public static final String NAME = "indices:data/read/msearch/template";
 
-    public RenderSearchTemplateAction() {
+    private MultiSearchTemplateAction() {
         super(NAME);
     }
 
     @Override
-    public RenderSearchTemplateRequestBuilder newRequestBuilder(ElasticsearchClient client) {
-        return new RenderSearchTemplateRequestBuilder(client, this);
+    public MultiSearchTemplateResponse newResponse() {
+        return new MultiSearchTemplateResponse();
     }
 
     @Override
-    public RenderSearchTemplateResponse newResponse() {
-        return new RenderSearchTemplateResponse();
+    public MultiSearchTemplateRequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new MultiSearchTemplateRequestBuilder(client, this);
     }
-
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateRequest.java b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateRequest.java
new file mode 100644
index 00000000000..6fce2b2d492
--- /dev/null
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateRequest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search.template;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.CompositeIndicesRequest;
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+public class MultiSearchTemplateRequest extends ActionRequest<MultiSearchTemplateRequest> implements CompositeIndicesRequest {
+
+    private List<SearchTemplateRequest> requests = new ArrayList<>();
+
+    private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed();
+
+    /**
+     * Add a search template request to execute. Note, the order is important, the search response will be returned in the
+     * same order as the search requests.
+     */
+    public MultiSearchTemplateRequest add(SearchTemplateRequestBuilder request) {
+        requests.add(request.request());
+        return this;
+    }
+
+    /**
+     * Add a search template request to execute. Note, the order is important, the search response will be returned in the
+     * same order as the search requests.
+     */
+    public MultiSearchTemplateRequest add(SearchTemplateRequest request) {
+        requests.add(request);
+        return this;
+    }
+
+    public List<SearchTemplateRequest> requests() {
+        return this.requests;
+    }
+
+    @Override
+    public List<? extends IndicesRequest> subRequests() {
+        return requests;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (requests.isEmpty()) {
+            validationException = addValidationError("no requests added", validationException);
+        }
+        for (SearchTemplateRequest request : requests) {
+            ActionRequestValidationException ex = request.validate();
+            if (ex != null) {
+                if (validationException == null) {
+                    validationException = new ActionRequestValidationException();
+                }
+                validationException.addValidationErrors(ex.validationErrors());
+            }
+        }
+        return validationException;
+    }
+
+    public IndicesOptions indicesOptions() {
+        return indicesOptions;
+    }
+
+    public MultiSearchTemplateRequest indicesOptions(IndicesOptions indicesOptions) {
+        this.indicesOptions = indicesOptions;
+        return this;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        requests = in.readStreamableList(SearchTemplateRequest::new);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeStreamableList(requests);
+    }
+}
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateRequestBuilder.java b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateRequestBuilder.java
new file mode 100644
index 00000000000..5b0f802ab83
--- /dev/null
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateRequestBuilder.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search.template;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.client.ElasticsearchClient;
+
+public class MultiSearchTemplateRequestBuilder
+        extends ActionRequestBuilder<MultiSearchTemplateRequest, MultiSearchTemplateResponse, MultiSearchTemplateRequestBuilder> {
+
+    protected MultiSearchTemplateRequestBuilder(ElasticsearchClient client, MultiSearchTemplateAction action) {
+        super(client, action, new MultiSearchTemplateRequest());
+    }
+
+    public MultiSearchTemplateRequestBuilder(ElasticsearchClient client) {
+        this(client, MultiSearchTemplateAction.INSTANCE);
+    }
+
+    public MultiSearchTemplateRequestBuilder add(SearchTemplateRequest request) {
+        if (request.getRequest().indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed()
+                && request().indicesOptions() != IndicesOptions.strictExpandOpenAndForbidClosed()) {
+            request.getRequest().indicesOptions(request().indicesOptions());
+        }
+
+        super.request.add(request);
+        return this;
+    }
+
+    public MultiSearchTemplateRequestBuilder add(SearchTemplateRequestBuilder request) {
+        if (request.request().getRequest().indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed()
+                && request().indicesOptions() != IndicesOptions.strictExpandOpenAndForbidClosed()) {
+            request.request().getRequest().indicesOptions(request().indicesOptions());
+        }
+
+        super.request.add(request);
+        return this;
+    }
+
+    public MultiSearchTemplateRequestBuilder setIndicesOptions(IndicesOptions indicesOptions) {
+        request().indicesOptions(indicesOptions);
+        return this;
+    }
+}
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateResponse.java
new file mode 100644
index 00000000000..c779757f61b
--- /dev/null
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateResponse.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search.template;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Iterator;
+
+public class MultiSearchTemplateResponse extends ActionResponse implements Iterable<MultiSearchTemplateResponse.Item>, ToXContent {
+
+    /**
+     * A search template response item, holding the actual search template response, or an error message if it failed.
+     */
+    public static class Item implements Streamable {
+        private SearchTemplateResponse response;
+        private Throwable throwable;
+
+        Item() {
+        }
+
+        public Item(SearchTemplateResponse response, Throwable throwable) {
+            this.response = response;
+            this.throwable = throwable;
+        }
+
+        /**
+         * Is it a failed search?
+         */
+        public boolean isFailure() {
+            return throwable != null;
+        }
+
+        /**
+         * The actual failure message, null if its not a failure.
+         */
+        @Nullable
+        public String getFailureMessage() {
+            return throwable == null ? null : throwable.getMessage();
+        }
+
+        /**
+         * The actual search response, null if its a failure.
+         */
+        @Nullable
+        public SearchTemplateResponse getResponse() {
+            return this.response;
+        }
+
+        public static Item readItem(StreamInput in) throws IOException {
+            Item item = new Item();
+            item.readFrom(in);
+            return item;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            if (in.readBoolean()) {
+                this.response = new SearchTemplateResponse();
+                response.readFrom(in);
+            } else {
+                throwable = in.readThrowable();
+            }
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            if (response != null) {
+                out.writeBoolean(true);
+                response.writeTo(out);
+            } else {
+                out.writeBoolean(false);
+                out.writeThrowable(throwable);
+            }
+        }
+
+        public Throwable getFailure() {
+            return throwable;
+        }
+    }
+
+    private Item[] items;
+
+    MultiSearchTemplateResponse() {
+    }
+
+    public MultiSearchTemplateResponse(Item[] items) {
+        this.items = items;
+    }
+
+    @Override
+    public Iterator<Item> iterator() {
+        return Arrays.stream(items).iterator();
+    }
+
+    /**
+     * The list of responses, the order is the same as the one provided in the request.
+     */
+    public Item[] getResponses() {
+        return this.items;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        items = new Item[in.readVInt()];
+        for (int i = 0; i < items.length; i++) {
+            items[i] = Item.readItem(in);
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeVInt(items.length);
+        for (Item item : items) {
+            item.writeTo(out);
+        }
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        builder.startArray(Fields.RESPONSES);
+        for (Item item : items) {
+            builder.startObject();
+            if (item.isFailure()) {
+                ElasticsearchException.renderThrowable(builder, params, item.getFailure());
+            } else {
+                item.getResponse().toXContent(builder, params);
+            }
+            builder.endObject();
+        }
+        builder.endArray();
+        return builder;
+    }
+
+    static final class Fields {
+        static final String RESPONSES = "responses";
+        static final String ERROR = "error";
+        static final String ROOT_CAUSE = "root_cause";
+    }
+
+    @Override
+    public String toString() {
+        try {
+            XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint();
+            builder.startObject();
+            toXContent(builder, EMPTY_PARAMS);
+            builder.endObject();
+            return builder.string();
+        } catch (IOException e) {
+            return "{ \"error\" : \"" + e.getMessage() + "\"}";
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/validate/template/RenderSearchTemplateRequestBuilder.java b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateAction.java
similarity index 53%
rename from core/src/main/java/org/elasticsearch/action/admin/cluster/validate/template/RenderSearchTemplateRequestBuilder.java
rename to modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateAction.java
index f7e3da111b2..a3501f51bb1 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/validate/template/RenderSearchTemplateRequestBuilder.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateAction.java
@@ -17,26 +17,28 @@
  * under the License.
  */
 
-package org.elasticsearch.action.admin.cluster.validate.template;
+package org.elasticsearch.action.search.template;
 
-import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.script.Template;
 
-public class RenderSearchTemplateRequestBuilder extends ActionRequestBuilder<RenderSearchTemplateRequest, RenderSearchTemplateResponse, RenderSearchTemplateRequestBuilder> {
+public class SearchTemplateAction extends Action<SearchTemplateRequest, SearchTemplateResponse, SearchTemplateRequestBuilder> {
 
-    public RenderSearchTemplateRequestBuilder(ElasticsearchClient client,
-            RenderSearchTemplateAction action) {
-        super(client, action, new RenderSearchTemplateRequest());
-    }
-    
-    public RenderSearchTemplateRequestBuilder template(Template template) {
-        request.template(template);
-        return this;
-    }
-    
-    public Template template() {
-        return request.template();
+    public static final SearchTemplateAction INSTANCE = new SearchTemplateAction();
+    public static final String NAME = "indices:data/read/search/template";
+
+    private SearchTemplateAction() {
+        super(NAME);
     }
 
+    @Override
+    public SearchTemplateRequestBuilder newRequestBuilder(ElasticsearchClient client) {
+        return new SearchTemplateRequestBuilder(client, this);
+    }
+
+    @Override
+    public SearchTemplateResponse newResponse() {
+        return new SearchTemplateResponse();
+    }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateRequest.java b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateRequest.java
new file mode 100644
index 00000000000..ff3d97401a3
--- /dev/null
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateRequest.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search.template;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.IndicesRequest;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.script.ScriptService;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.elasticsearch.action.ValidateActions.addValidationError;
+
+/**
+ * A request to execute a search based on a search template.
+ */
+public class SearchTemplateRequest extends ActionRequest<SearchTemplateRequest> implements IndicesRequest {
+
+    private SearchRequest request;
+    private boolean simulate = false;
+    private ScriptService.ScriptType scriptType;
+    private String script;
+    private Map<String, Object> scriptParams;
+
+    public SearchTemplateRequest() {
+    }
+
+    public SearchTemplateRequest(SearchRequest searchRequest) {
+        this.request = searchRequest;
+    }
+
+    public void setRequest(SearchRequest request) {
+        this.request = request;
+    }
+
+    public SearchRequest getRequest() {
+        return request;
+    }
+
+
+    public boolean isSimulate() {
+        return simulate;
+    }
+
+    public void setSimulate(boolean simulate) {
+        this.simulate = simulate;
+    }
+
+    public ScriptService.ScriptType getScriptType() {
+        return scriptType;
+    }
+
+    public void setScriptType(ScriptService.ScriptType scriptType) {
+        this.scriptType = scriptType;
+    }
+
+    public String getScript() {
+        return script;
+    }
+
+    public void setScript(String script) {
+        this.script = script;
+    }
+
+    public Map<String, Object> getScriptParams() {
+        return scriptParams;
+    }
+
+    public void setScriptParams(Map<String, Object> scriptParams) {
+        this.scriptParams = scriptParams;
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        ActionRequestValidationException validationException = null;
+        if (script == null || script.isEmpty()) {
+            validationException = addValidationError("template is missing", validationException);
+        }
+        if (scriptType == null) {
+            validationException = addValidationError("template's script type is missing", validationException);
+        }
+        if (simulate == false) {
+            if (request == null) {
+                validationException = addValidationError("search request is missing", validationException);
+            } else {
+                ActionRequestValidationException ex = request.validate();
+                if (ex != null) {
+                    if (validationException == null) {
+                        validationException = new ActionRequestValidationException();
+                    }
+                    validationException.addValidationErrors(ex.validationErrors());
+                }
+            }
+        }
+        return validationException;
+    }
+
+    @Override
+    public void readFrom(StreamInput in) throws IOException {
+        super.readFrom(in);
+        request = in.readOptionalStreamable(SearchRequest::new);
+        simulate = in.readBoolean();
+        scriptType = ScriptService.ScriptType.readFrom(in);
+        script = in.readOptionalString();
+        if (in.readBoolean()) {
+            scriptParams = in.readMap();
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeOptionalStreamable(request);
+        out.writeBoolean(simulate);
+        ScriptService.ScriptType.writeTo(scriptType, out);
+        out.writeOptionalString(script);
+        boolean hasParams = scriptParams != null;
+        out.writeBoolean(hasParams);
+        if (hasParams) {
+            out.writeMap(scriptParams);
+        }
+    }
+
+    @Override
+    public String[] indices() {
+        return request != null ? request.indices() : Strings.EMPTY_ARRAY;
+    }
+
+    @Override
+    public IndicesOptions indicesOptions() {
+        return request != null ? request.indicesOptions() : SearchRequest.DEFAULT_INDICES_OPTIONS;
+    }
+}
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateRequestBuilder.java b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateRequestBuilder.java
new file mode 100644
index 00000000000..3d44535d80b
--- /dev/null
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateRequestBuilder.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search.template;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.script.ScriptService;
+
+import java.util.Map;
+
+public class SearchTemplateRequestBuilder
+        extends ActionRequestBuilder<SearchTemplateRequest, SearchTemplateResponse, SearchTemplateRequestBuilder> {
+
+    SearchTemplateRequestBuilder(ElasticsearchClient client, SearchTemplateAction action) {
+        super(client, action, new SearchTemplateRequest());
+    }
+
+    public SearchTemplateRequestBuilder(ElasticsearchClient client) {
+        this(client, SearchTemplateAction.INSTANCE);
+    }
+
+    public SearchTemplateRequestBuilder setRequest(SearchRequest searchRequest) {
+        request.setRequest(searchRequest);
+        return this;
+    }
+
+    public SearchTemplateRequestBuilder setSimulate(boolean simulate) {
+        request.setSimulate(simulate);
+        return this;
+    }
+
+    public SearchTemplateRequestBuilder setScriptType(ScriptService.ScriptType scriptType) {
+        request.setScriptType(scriptType);
+        return this;
+    }
+
+    public SearchTemplateRequestBuilder setScript(String script) {
+        request.setScript(script);
+        return this;
+    }
+
+    public SearchTemplateRequestBuilder setScriptParams(Map<String, Object> scriptParams) {
+        request.setScriptParams(scriptParams);
+        return this;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/validate/template/RenderSearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateResponse.java
similarity index 53%
rename from core/src/main/java/org/elasticsearch/action/admin/cluster/validate/template/RenderSearchTemplateResponse.java
rename to modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateResponse.java
index d14a9a4f06a..86b12a0fe56 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/validate/template/RenderSearchTemplateResponse.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/SearchTemplateResponse.java
@@ -17,52 +17,80 @@
  * under the License.
  */
 
-package org.elasticsearch.action.admin.cluster.validate.template;
+package org.elasticsearch.action.search.template;
 
 import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.StatusToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
 
-public class RenderSearchTemplateResponse extends ActionResponse implements ToXContent {
+public class SearchTemplateResponse  extends ActionResponse implements StatusToXContent {
 
+    /** Contains the source of the rendered template **/
     private BytesReference source;
 
-    public BytesReference source() {
+    /** Contains the search response, if any **/
+    private SearchResponse response;
+
+    SearchTemplateResponse() {
+    }
+
+    public BytesReference getSource() {
         return source;
     }
-    
-    public void source(BytesReference source) {
+
+    public void setSource(BytesReference source) {
         this.source = source;
     }
-    
+
+    public SearchResponse getResponse() {
+        return response;
+    }
+
+    public void setResponse(SearchResponse searchResponse) {
+        this.response = searchResponse;
+    }
+
+    public boolean hasResponse() {
+        return response != null;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        boolean hasSource = source != null;
-        out.writeBoolean(hasSource);
-        if (hasSource) {
-            out.writeBytesReference(source);
-        }
+        out.writeOptionalBytesReference(source);
+        out.writeOptionalStreamable(response);
     }
-    
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
-        if (in.readBoolean()) {
-            source = in.readBytesReference();
-        }
+        source = in.readOptionalBytesReference();
+        response = in.readOptionalStreamable(SearchResponse::new);
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
-        builder.startObject();
-        builder.rawField("template_output", source);
-        builder.endObject();
+        if (hasResponse()) {
+            response.toXContent(builder, params);
+        } else {
+            builder.rawField("template_output", source);
+        }
         return builder;
     }
+
+    @Override
+    public RestStatus status() {
+        if (hasResponse()) {
+            return response.status();
+        } else {
+            return RestStatus.OK;
+        }
+    }
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportMultiSearchTemplateAction.java
new file mode 100644
index 00000000000..1ffb19b5fc4
--- /dev/null
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportMultiSearchTemplateAction.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search.template;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.AtomicArray;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class TransportMultiSearchTemplateAction extends HandledTransportAction<MultiSearchTemplateRequest, MultiSearchTemplateResponse> {
+
+    private final TransportSearchTemplateAction searchTemplateAction;
+
+    @Inject
+    public TransportMultiSearchTemplateAction(Settings settings, ThreadPool threadPool, TransportService transportService,
+                                              ActionFilters actionFilters, IndexNameExpressionResolver resolver,
+                                              TransportSearchTemplateAction searchTemplateAction) {
+        super(settings, MultiSearchTemplateAction.NAME, threadPool, transportService, actionFilters, resolver,
+                MultiSearchTemplateRequest::new);
+        this.searchTemplateAction = searchTemplateAction;
+    }
+
+    @Override
+    protected void doExecute(MultiSearchTemplateRequest request, ActionListener<MultiSearchTemplateResponse> listener) {
+        final AtomicArray<MultiSearchTemplateResponse.Item> responses = new AtomicArray<>(request.requests().size());
+        final AtomicInteger counter = new AtomicInteger(responses.length());
+
+        for (int i = 0; i < responses.length(); i++) {
+            final int index = i;
+            searchTemplateAction.execute(request.requests().get(i), new ActionListener<SearchTemplateResponse>() {
+                @Override
+                public void onResponse(SearchTemplateResponse searchTemplateResponse) {
+                    responses.set(index, new MultiSearchTemplateResponse.Item(searchTemplateResponse, null));
+                    if (counter.decrementAndGet() == 0) {
+                        finishHim();
+                    }
+                }
+
+                @Override
+                public void onFailure(Throwable e) {
+                    responses.set(index, new MultiSearchTemplateResponse.Item(null, e));
+                    if (counter.decrementAndGet() == 0) {
+                        finishHim();
+                    }
+                }
+
+                private void finishHim() {
+                    MultiSearchTemplateResponse.Item[] items = responses.toArray(new MultiSearchTemplateResponse.Item[responses.length()]);
+                    listener.onResponse(new MultiSearchTemplateResponse(items));
+                }
+            });
+        }
+    }
+}
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportSearchTemplateAction.java
new file mode 100644
index 00000000000..4c956bb209a
--- /dev/null
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportSearchTemplateAction.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search.template;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.TransportSearchAction;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.script.ExecutableScript;
+import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.mustache.MustacheScriptEngineService;
+import org.elasticsearch.search.aggregations.AggregatorParsers;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.suggest.Suggesters;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+
+import java.io.IOException;
+
+import static java.util.Collections.emptyMap;
+import static org.elasticsearch.script.ScriptContext.Standard.SEARCH;
+
+public class TransportSearchTemplateAction extends HandledTransportAction<SearchTemplateRequest, SearchTemplateResponse> {
+
+    private static final String TEMPLATE_LANG = MustacheScriptEngineService.NAME;
+
+    private final ClusterService clusterService;
+    private final ScriptService scriptService;
+    private final TransportSearchAction searchAction;
+    private final IndicesQueriesRegistry queryRegistry;
+    private final AggregatorParsers aggsParsers;
+    private final Suggesters suggesters;
+
+    @Inject
+    public TransportSearchTemplateAction(Settings settings, ThreadPool threadPool, TransportService transportService,
+                                         ActionFilters actionFilters, IndexNameExpressionResolver resolver,
+                                         ClusterService clusterService, ScriptService scriptService,
+                                         TransportSearchAction searchAction, IndicesQueriesRegistry indicesQueryRegistry,
+                                         AggregatorParsers aggregatorParsers, Suggesters suggesters) {
+        super(settings, SearchTemplateAction.NAME, threadPool, transportService, actionFilters, resolver, SearchTemplateRequest::new);
+        this.clusterService = clusterService;
+        this.scriptService = scriptService;
+        this.searchAction = searchAction;
+        this.queryRegistry = indicesQueryRegistry;
+        this.aggsParsers = aggregatorParsers;
+        this.suggesters = suggesters;
+    }
+
+    @Override
+    protected void doExecute(SearchTemplateRequest request, ActionListener<SearchTemplateResponse> listener) {
+        final SearchTemplateResponse response = new SearchTemplateResponse();
+        try {
+            Script script = new Script(request.getScript(), request.getScriptType(), TEMPLATE_LANG, request.getScriptParams());
+            ExecutableScript executable = scriptService.executable(script, SEARCH, emptyMap(), clusterService.state());
+
+            BytesReference source = (BytesReference) executable.run();
+            response.setSource(source);
+
+            if (request.isSimulate()) {
+                listener.onResponse(response);
+                return;
+            }
+
+            // Executes the search
+            SearchRequest searchRequest = request.getRequest();
+
+            try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
+                SearchSourceBuilder builder = SearchSourceBuilder.searchSource();
+                builder.parseXContent(new QueryParseContext(queryRegistry, parser, parseFieldMatcher), aggsParsers, suggesters);
+                searchRequest.source(builder);
+
+                searchAction.execute(searchRequest, new ActionListener<SearchResponse>() {
+                    @Override
+                    public void onResponse(SearchResponse searchResponse) {
+                        try {
+                            response.setResponse(searchResponse);
+                            listener.onResponse(response);
+                        } catch (Throwable t) {
+                            listener.onFailure(t);
+                        }
+                    }
+
+                    @Override
+                    public void onFailure(Throwable t) {
+                        listener.onFailure(t);
+                    }
+                });
+            }
+        } catch (Throwable t) {
+            listener.onFailure(t);
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestDeleteSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestDeleteSearchTemplateAction.java
similarity index 96%
rename from core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestDeleteSearchTemplateAction.java
rename to modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestDeleteSearchTemplateAction.java
index 4a745d87e71..46bad569933 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestDeleteSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestDeleteSearchTemplateAction.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.rest.action.admin.cluster.storedscripts;
+package org.elasticsearch.rest.action.search.template;
 
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.inject.Inject;
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestGetSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestGetSearchTemplateAction.java
similarity index 95%
rename from core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestGetSearchTemplateAction.java
rename to modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestGetSearchTemplateAction.java
index cb672d3567f..61fcaf1ccec 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestGetSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestGetSearchTemplateAction.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.rest.action.admin.cluster.storedscripts;
+package org.elasticsearch.rest.action.search.template;
 
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.inject.Inject;
@@ -28,11 +28,10 @@ import org.elasticsearch.script.Template;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 
-/**
- *
- */
 public class RestGetSearchTemplateAction extends RestGetStoredScriptAction {
 
+    private static final String TEMPLATE = "template";
+
     @Inject
     public RestGetSearchTemplateAction(Settings settings, RestController controller, Client client) {
         super(settings, controller, false, client);
@@ -48,6 +47,4 @@ public class RestGetSearchTemplateAction extends RestGetStoredScriptAction {
     protected String getScriptFieldName() {
         return TEMPLATE;
     }
-
-    private static final String TEMPLATE = "template";
 }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestMultiSearchTemplateAction.java
new file mode 100644
index 00000000000..48f69dda50f
--- /dev/null
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestMultiSearchTemplateAction.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.action.search.template;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.action.search.template.MultiSearchTemplateAction;
+import org.elasticsearch.action.search.template.MultiSearchTemplateRequest;
+import org.elasticsearch.action.search.template.SearchTemplateRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.search.RestMultiSearchAction;
+import org.elasticsearch.rest.action.support.RestActions;
+import org.elasticsearch.rest.action.support.RestToXContentListener;
+
+import java.io.IOException;
+
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+
+public class RestMultiSearchTemplateAction extends BaseRestHandler {
+
+    private final boolean allowExplicitIndex;
+
+    @Inject
+    public RestMultiSearchTemplateAction(Settings settings, RestController controller, Client client) {
+        super(settings, client);
+        this.allowExplicitIndex = MULTI_ALLOW_EXPLICIT_INDEX.get(settings);
+
+        controller.registerHandler(GET, "/_msearch/template", this);
+        controller.registerHandler(POST, "/_msearch/template", this);
+        controller.registerHandler(GET, "/{index}/_msearch/template", this);
+        controller.registerHandler(POST, "/{index}/_msearch/template", this);
+        controller.registerHandler(GET, "/{index}/{type}/_msearch/template", this);
+        controller.registerHandler(POST, "/{index}/{type}/_msearch/template", this);
+    }
+
+    @Override
+    protected void handleRequest(RestRequest request, RestChannel channel, Client client) throws Exception {
+        if (RestActions.hasBodyContent(request) == false) {
+            throw new ElasticsearchException("request body is required");
+        }
+
+        MultiSearchTemplateRequest multiRequest = parseRequest(request, allowExplicitIndex);
+        client.execute(MultiSearchTemplateAction.INSTANCE, multiRequest, new RestToXContentListener<>(channel));
+    }
+
+    /**
+     * Parses a {@link RestRequest} body and returns a {@link MultiSearchTemplateRequest}
+     */
+    public static MultiSearchTemplateRequest parseRequest(RestRequest restRequest, boolean allowExplicitIndex) throws IOException {
+
+        MultiSearchTemplateRequest multiRequest = new MultiSearchTemplateRequest();
+        RestMultiSearchAction.parseMultiLineRequest(restRequest, multiRequest.indicesOptions(), allowExplicitIndex,
+                (searchRequest, bytes) -> {
+                    try {
+                        SearchTemplateRequest searchTemplateRequest = RestSearchTemplateAction.parse(bytes);
+                        if (searchTemplateRequest.getScript() != null) {
+                            searchTemplateRequest.setRequest(searchRequest);
+                            multiRequest.add(searchTemplateRequest);
+                        } else {
+                            throw new IllegalArgumentException("Malformed search template");
+                        }
+                    } catch (IOException e) {
+                        throw new ElasticsearchParseException("Exception when parsing search template request", e);
+                    }
+                });
+        return multiRequest;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestPutSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestPutSearchTemplateAction.java
similarity index 64%
rename from core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestPutSearchTemplateAction.java
rename to modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestPutSearchTemplateAction.java
index e9284d51474..90854bee01c 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestPutSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestPutSearchTemplateAction.java
@@ -16,47 +16,26 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.rest.action.admin.cluster.storedscripts;
+package org.elasticsearch.rest.action.search.template;
 
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.rest.BaseRestHandler;
-import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestPutStoredScriptAction;
 import org.elasticsearch.script.Template;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
 
-/**
- *
- */
 public class RestPutSearchTemplateAction extends RestPutStoredScriptAction {
 
     @Inject
     public RestPutSearchTemplateAction(Settings settings, RestController controller, Client client) {
         super(settings, controller, false, client);
-
-        //controller.registerHandler(GET, "/template", this);
         controller.registerHandler(POST, "/_search/template/{id}", this);
         controller.registerHandler(PUT, "/_search/template/{id}", this);
-
-        controller.registerHandler(PUT, "/_search/template/{id}/_create", new CreateHandler(settings, controller, client));
-        controller.registerHandler(POST, "/_search/template/{id}/_create", new CreateHandler(settings, controller, client));
-    }
-
-    final class CreateHandler extends BaseRestHandler {
-        protected CreateHandler(Settings settings, RestController controller, Client client) {
-            super(settings, client);
-        }
-
-        @Override
-        public void handleRequest(RestRequest request, RestChannel channel, final Client client) {
-            request.params().put("op_type", "create");
-            RestPutSearchTemplateAction.this.handleRequest(request, channel, client);
-        }
     }
 
     @Override
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestRenderSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestRenderSearchTemplateAction.java
new file mode 100644
index 00000000000..388837962c5
--- /dev/null
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestRenderSearchTemplateAction.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.action.search.template;
+
+import org.elasticsearch.action.search.template.SearchTemplateAction;
+import org.elasticsearch.action.search.template.SearchTemplateRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.support.RestActions;
+import org.elasticsearch.rest.action.support.RestToXContentListener;
+import org.elasticsearch.script.ScriptService;
+
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+
+public class RestRenderSearchTemplateAction extends BaseRestHandler {
+
+    @Inject
+    public RestRenderSearchTemplateAction(Settings settings, RestController controller, Client client) {
+        super(settings, client);
+        controller.registerHandler(GET, "/_render/template", this);
+        controller.registerHandler(POST, "/_render/template", this);
+        controller.registerHandler(GET, "/_render/template/{id}", this);
+        controller.registerHandler(POST, "/_render/template/{id}", this);
+    }
+
+    @Override
+    protected void handleRequest(RestRequest request, RestChannel channel, Client client) throws Exception {
+        // Creates the render template request
+        SearchTemplateRequest renderRequest = RestSearchTemplateAction.parse(RestActions.getRestContent(request));
+        renderRequest.setSimulate(true);
+
+        String id = request.param("id");
+        if (id != null) {
+            renderRequest.setScriptType(ScriptService.ScriptType.STORED);
+            renderRequest.setScript(id);
+        }
+
+        client.execute(SearchTemplateAction.INSTANCE, renderRequest, new RestToXContentListener<>(channel));
+    }
+}
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestSearchTemplateAction.java
new file mode 100644
index 00000000000..b15efa74420
--- /dev/null
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestSearchTemplateAction.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.action.search.template;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.template.SearchTemplateAction;
+import org.elasticsearch.action.search.template.SearchTemplateRequest;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParseFieldMatcherSupplier;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.search.RestSearchAction;
+import org.elasticsearch.rest.action.support.RestActions;
+import org.elasticsearch.rest.action.support.RestStatusToXContentListener;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.search.aggregations.AggregatorParsers;
+import org.elasticsearch.search.suggest.Suggesters;
+
+import java.io.IOException;
+
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+import static org.elasticsearch.rest.RestRequest.Method.POST;
+
+public class RestSearchTemplateAction extends BaseRestHandler {
+
+    private static ObjectParser<SearchTemplateRequest, ParseFieldMatcherSupplier> PARSER;
+    static {
+        PARSER = new ObjectParser<>("search_template");
+        PARSER.declareField((parser, request, s) ->
+                        request.setScriptParams(parser.map())
+                , new ParseField("params"), ObjectParser.ValueType.OBJECT);
+        PARSER.declareString((request, s) -> {
+            request.setScriptType(ScriptService.ScriptType.FILE);
+            request.setScript(s);
+        }, new ParseField("file"));
+        PARSER.declareString((request, s) -> {
+            request.setScriptType(ScriptService.ScriptType.STORED);
+            request.setScript(s);
+        }, new ParseField("id"));
+        PARSER.declareField((parser, request, value) -> {
+            request.setScriptType(ScriptService.ScriptType.INLINE);
+            if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
+                try (XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType())) {
+                    request.setScript(builder.copyCurrentStructure(parser).bytes().toUtf8());
+                } catch (IOException e) {
+                    throw new ParsingException(parser.getTokenLocation(), "Could not parse inline template", e);
+                }
+            } else {
+                request.setScript(parser.text());
+            }
+        }, new ParseField("inline", "template"), ObjectParser.ValueType.OBJECT_OR_STRING);
+    }
+
+    private final IndicesQueriesRegistry queryRegistry;
+    private final AggregatorParsers aggParsers;
+    private final Suggesters suggesters;
+
+    @Inject
+    public RestSearchTemplateAction(Settings settings, RestController controller, Client client, IndicesQueriesRegistry queryRegistry,
+                                    AggregatorParsers aggregatorParsers, Suggesters suggesters) {
+        super(settings, client);
+        this.queryRegistry = queryRegistry;
+        this.aggParsers = aggregatorParsers;
+        this.suggesters = suggesters;
+
+        controller.registerHandler(GET, "/_search/template", this);
+        controller.registerHandler(POST, "/_search/template", this);
+        controller.registerHandler(GET, "/{index}/_search/template", this);
+        controller.registerHandler(POST, "/{index}/_search/template", this);
+        controller.registerHandler(GET, "/{index}/{type}/_search/template", this);
+        controller.registerHandler(POST, "/{index}/{type}/_search/template", this);
+    }
+
+    @Override
+    protected void handleRequest(RestRequest request, RestChannel channel, Client client) throws Exception {
+        if (RestActions.hasBodyContent(request) == false) {
+            throw new ElasticsearchException("request body is required");
+        }
+
+        // Creates the search request with all required params
+        SearchRequest searchRequest = new SearchRequest();
+        RestSearchAction.parseSearchRequest(searchRequest, queryRegistry, request, parseFieldMatcher, aggParsers, suggesters, null);
+
+        // Creates the search template request
+        SearchTemplateRequest searchTemplateRequest = parse(RestActions.getRestContent(request));
+        searchTemplateRequest.setRequest(searchRequest);
+
+        client.execute(SearchTemplateAction.INSTANCE, searchTemplateRequest, new RestStatusToXContentListener<>(channel));
+    }
+
+    public static SearchTemplateRequest parse(BytesReference bytes) throws IOException {
+        try (XContentParser parser = XContentHelper.createParser(bytes)) {
+            return PARSER.parse(parser, new SearchTemplateRequest(), () -> ParseFieldMatcher.STRICT);
+        }
+    }
+}
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
index c7b68072509..42f0da3c109 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
@@ -19,9 +19,21 @@
 
 package org.elasticsearch.script.mustache;
 
+import org.elasticsearch.action.ActionModule;
+import org.elasticsearch.action.search.template.MultiSearchTemplateAction;
+import org.elasticsearch.action.search.template.SearchTemplateAction;
+import org.elasticsearch.action.search.template.TransportMultiSearchTemplateAction;
+import org.elasticsearch.action.search.template.TransportSearchTemplateAction;
+import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ScriptPlugin;
+import org.elasticsearch.rest.action.search.template.RestDeleteSearchTemplateAction;
+import org.elasticsearch.rest.action.search.template.RestGetSearchTemplateAction;
+import org.elasticsearch.rest.action.search.template.RestMultiSearchTemplateAction;
+import org.elasticsearch.rest.action.search.template.RestPutSearchTemplateAction;
+import org.elasticsearch.rest.action.search.template.RestRenderSearchTemplateAction;
+import org.elasticsearch.rest.action.search.template.RestSearchTemplateAction;
 import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
 import org.elasticsearch.script.ScriptModule;
@@ -32,4 +44,20 @@ public class MustachePlugin extends Plugin implements ScriptPlugin {
     public ScriptEngineService getScriptEngineService(Settings settings) {
         return new MustacheScriptEngineService(settings);
     }
+
+    public void onModule(ActionModule module) {
+        module.registerAction(SearchTemplateAction.INSTANCE, TransportSearchTemplateAction.class);
+        module.registerAction(MultiSearchTemplateAction.INSTANCE, TransportMultiSearchTemplateAction.class);
+    }
+
+    public void onModule(NetworkModule module) {
+        if (module.isTransportClient() == false) {
+            module.registerRestHandler(RestSearchTemplateAction.class);
+            module.registerRestHandler(RestMultiSearchTemplateAction.class);
+            module.registerRestHandler(RestGetSearchTemplateAction.class);
+            module.registerRestHandler(RestPutSearchTemplateAction.class);
+            module.registerRestHandler(RestDeleteSearchTemplateAction.class);
+            module.registerRestHandler(RestRenderSearchTemplateAction.class);
+        }
+    }
 }
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateIT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateIT.java
new file mode 100644
index 00000000000..9c28225c081
--- /dev/null
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateIT.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search.template;
+
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.index.IndexNotFoundException;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.script.mustache.MustachePlugin;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.arrayWithSize;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.core.Is.is;
+
+public class MultiSearchTemplateIT extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Collections.singleton(MustachePlugin.class);
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
+        return nodePlugins();
+    }
+
+    public void testBasic() throws Exception {
+        createIndex("msearch");
+        final int numDocs = randomIntBetween(10, 100);
+        IndexRequestBuilder[] indexRequestBuilders = new IndexRequestBuilder[numDocs];
+        for (int i = 0; i < numDocs; i++) {
+            indexRequestBuilders[i] = client().prepareIndex("msearch", "test", String.valueOf(i))
+                    .setSource("odd", (i % 2 == 0), "group", (i % 3));
+        }
+        indexRandom(true, indexRequestBuilders);
+
+        final String template = jsonBuilder().startObject()
+                                                .startObject("query")
+                                                    .startObject("{{query_type}}")
+                                                        .field("{{field_name}}", "{{field_value}}")
+                                                    .endObject()
+                                                .endObject()
+                                            .endObject().string();
+
+        MultiSearchTemplateRequest multiRequest = new MultiSearchTemplateRequest();
+
+        // Search #1
+        SearchTemplateRequest search1 = new SearchTemplateRequest();
+        search1.setRequest(new SearchRequest("msearch"));
+        search1.setScriptType(ScriptService.ScriptType.INLINE);
+        search1.setScript(template);
+
+        Map<String, Object> params1 = new HashMap<>();
+        params1.put("query_type", "match");
+        params1.put("field_name", "odd");
+        params1.put("field_value", true);
+        search1.setScriptParams(params1);
+        multiRequest.add(search1);
+
+        // Search #2 (Simulate is true)
+        SearchTemplateRequest search2 = new SearchTemplateRequest();
+        search2.setRequest(new SearchRequest("msearch"));
+        search2.setScriptType(ScriptService.ScriptType.INLINE);
+        search2.setScript(template);
+        search2.setSimulate(true);
+
+        Map<String, Object> params2 = new HashMap<>();
+        params2.put("query_type", "match_phrase_prefix");
+        params2.put("field_name", "message");
+        params2.put("field_value", "quick brown f");
+        search2.setScriptParams(params2);
+        multiRequest.add(search2);
+
+        // Search #3
+        SearchTemplateRequest search3 = new SearchTemplateRequest();
+        search3.setRequest(new SearchRequest("msearch"));
+        search3.setScriptType(ScriptService.ScriptType.INLINE);
+        search3.setScript(template);
+        search3.setSimulate(false);
+
+        Map<String, Object> params3 = new HashMap<>();
+        params3.put("query_type", "term");
+        params3.put("field_name", "odd");
+        params3.put("field_value", "false");
+        search3.setScriptParams(params3);
+        multiRequest.add(search3);
+
+        // Search #4 (Fail because of unknown index)
+        SearchTemplateRequest search4 = new SearchTemplateRequest();
+        search4.setRequest(new SearchRequest("unknown"));
+        search4.setScriptType(ScriptService.ScriptType.INLINE);
+        search4.setScript(template);
+
+        Map<String, Object> params4 = new HashMap<>();
+        params4.put("query_type", "match");
+        params4.put("field_name", "group");
+        params4.put("field_value", "test");
+        search4.setScriptParams(params4);
+        multiRequest.add(search4);
+
+        // Search #5 (Simulate is true)
+        SearchTemplateRequest search5 = new SearchTemplateRequest();
+        search5.setRequest(new SearchRequest("msearch"));
+        search5.setScriptType(ScriptService.ScriptType.INLINE);
+        search5.setScript("{{! ignore me }}{\"query\":{\"terms\":{\"group\":[{{#groups}}{{.}},{{/groups}}]}}}");
+        search5.setSimulate(true);
+
+        Map<String, Object> params5 = new HashMap<>();
+        params5.put("groups", Arrays.asList(1, 2, 3));
+        search5.setScriptParams(params5);
+        multiRequest.add(search5);
+
+        MultiSearchTemplateResponse response = client().execute(MultiSearchTemplateAction.INSTANCE, multiRequest).get();
+        assertThat(response.getResponses(), arrayWithSize(5));
+
+        MultiSearchTemplateResponse.Item response1 = response.getResponses()[0];
+        assertThat(response1.isFailure(), is(false));
+        SearchTemplateResponse searchTemplateResponse1 = response1.getResponse();
+        assertThat(searchTemplateResponse1.hasResponse(), is(true));
+        assertHitCount(searchTemplateResponse1.getResponse(), (numDocs / 2) + (numDocs % 2));
+        assertThat(searchTemplateResponse1.getSource().toUtf8(),
+                equalTo("{\"query\":{\"match\":{\"odd\":\"true\"}}}"));
+
+        MultiSearchTemplateResponse.Item response2 = response.getResponses()[1];
+        assertThat(response2.isFailure(), is(false));
+        SearchTemplateResponse searchTemplateResponse2 = response2.getResponse();
+        assertThat(searchTemplateResponse2.hasResponse(), is(false));
+        assertThat(searchTemplateResponse2.getSource().toUtf8(),
+                equalTo("{\"query\":{\"match_phrase_prefix\":{\"message\":\"quick brown f\"}}}"));
+
+        MultiSearchTemplateResponse.Item response3 = response.getResponses()[2];
+        assertThat(response3.isFailure(), is(false));
+        SearchTemplateResponse searchTemplateResponse3 = response3.getResponse();
+        assertThat(searchTemplateResponse3.hasResponse(), is(true));
+        assertHitCount(searchTemplateResponse3.getResponse(), (numDocs / 2));
+        assertThat(searchTemplateResponse3.getSource().toUtf8(),
+                equalTo("{\"query\":{\"term\":{\"odd\":\"false\"}}}"));
+
+        MultiSearchTemplateResponse.Item response4 = response.getResponses()[3];
+        assertThat(response4.isFailure(), is(true));
+        assertThat(response4.getFailure(), instanceOf(IndexNotFoundException.class));
+        assertThat(response4.getFailure().getMessage(), equalTo("no such index"));
+
+        MultiSearchTemplateResponse.Item response5 = response.getResponses()[4];
+        assertThat(response5.isFailure(), is(false));
+        SearchTemplateResponse searchTemplateResponse5 = response5.getResponse();
+        assertThat(searchTemplateResponse5.hasResponse(), is(false));
+        assertThat(searchTemplateResponse5.getSource().toUtf8(),
+                equalTo("{\"query\":{\"terms\":{\"group\":[1,2,3,]}}}"));
+    }
+}
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateRequestTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateRequestTests.java
new file mode 100644
index 00000000000..4724a758107
--- /dev/null
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateRequestTests.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search.template;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.search.template.RestMultiSearchTemplateAction;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.StreamsUtils;
+import org.elasticsearch.test.rest.FakeRestRequest;
+
+import static org.hamcrest.Matchers.arrayContaining;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+
+public class MultiSearchTemplateRequestTests extends ESTestCase {
+
+    public void testParseRequest() throws Exception {
+        byte[] data = StreamsUtils.copyToBytesFromClasspath("/org/elasticsearch/action/search/template/simple-msearch-template.json");
+        RestRequest restRequest = new FakeRestRequest.Builder().withContent(new BytesArray(data)).build();
+
+        MultiSearchTemplateRequest request = RestMultiSearchTemplateAction.parseRequest(restRequest, true);
+
+        assertThat(request.requests().size(), equalTo(3));
+        assertThat(request.requests().get(0).getRequest().indices()[0], equalTo("test0"));
+        assertThat(request.requests().get(0).getRequest().indices()[1], equalTo("test1"));
+        assertThat(request.requests().get(0).indices(), arrayContaining("test0", "test1"));
+        assertThat(request.requests().get(0).getRequest().requestCache(), equalTo(true));
+        assertThat(request.requests().get(0).getRequest().preference(), nullValue());
+        assertThat(request.requests().get(1).indices()[0], equalTo("test2"));
+        assertThat(request.requests().get(1).indices()[1], equalTo("test3"));
+        assertThat(request.requests().get(1).getRequest().types()[0], equalTo("type1"));
+        assertThat(request.requests().get(1).getRequest().requestCache(), nullValue());
+        assertThat(request.requests().get(1).getRequest().preference(), equalTo("_local"));
+        assertThat(request.requests().get(2).indices()[0], equalTo("test4"));
+        assertThat(request.requests().get(2).indices()[1], equalTo("test1"));
+        assertThat(request.requests().get(2).getRequest().types()[0], equalTo("type2"));
+        assertThat(request.requests().get(2).getRequest().types()[1], equalTo("type1"));
+        assertThat(request.requests().get(2).getRequest().routing(), equalTo("123"));
+        assertNotNull(request.requests().get(0).getScript());
+        assertNotNull(request.requests().get(1).getScript());
+        assertNotNull(request.requests().get(2).getScript());
+
+        assertEquals(ScriptService.ScriptType.INLINE, request.requests().get(0).getScriptType());
+        assertEquals(ScriptService.ScriptType.INLINE, request.requests().get(1).getScriptType());
+        assertEquals(ScriptService.ScriptType.INLINE, request.requests().get(2).getScriptType());
+        assertEquals("{\"query\":{\"match_{{template}}\":{}}}", request.requests().get(0).getScript());
+        assertEquals("{\"query\":{\"match_{{template}}\":{}}}", request.requests().get(1).getScript());
+        assertEquals("{\"query\":{\"match_{{template}}\":{}}}", request.requests().get(2).getScript());
+        assertEquals(1, request.requests().get(0).getScriptParams().size());
+        assertEquals(1, request.requests().get(1).getScriptParams().size());
+        assertEquals(1, request.requests().get(2).getScriptParams().size());
+    }
+}
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/SearchTemplateRequestTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/SearchTemplateRequestTests.java
new file mode 100644
index 00000000000..bbac6dabcee
--- /dev/null
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/SearchTemplateRequestTests.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.search.template;
+
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.rest.action.search.template.RestSearchTemplateAction;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.List;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasItems;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.nullValue;
+
+public class SearchTemplateRequestTests extends ESTestCase {
+
+    public void testParseInlineTemplate() throws Exception {
+        String source = "{" +
+                "    'inline' : {\n" +
+                "    'query': {\n" +
+                "      'terms': {\n" +
+                "        'status': [\n" +
+                "          '{{#status}}',\n" +
+                "          '{{.}}',\n" +
+                "          '{{/status}}'\n" +
+                "        ]\n" +
+                "      }\n" +
+                "    }\n" +
+                "  }" +
+                "}";
+
+        SearchTemplateRequest request = RestSearchTemplateAction.parse(newBytesReference(source));
+        assertThat(request.getScript(), equalTo("{\"query\":{\"terms\":{\"status\":[\"{{#status}}\",\"{{.}}\",\"{{/status}}\"]}}}"));
+        assertThat(request.getScriptType(), equalTo(ScriptService.ScriptType.INLINE));
+        assertThat(request.getScriptParams(), nullValue());
+    }
+
+    public void testParseInlineTemplateWithParams() throws Exception {
+        String source = "{" +
+                "    'inline' : {" +
+                "      'query': { 'match' : { '{{my_field}}' : '{{my_value}}' } }," +
+                "      'size' : '{{my_size}}'" +
+                "    }," +
+                "    'params' : {" +
+                "        'my_field' : 'foo'," +
+                "        'my_value' : 'bar'," +
+                "        'my_size' : 5" +
+                "    }" +
+                "}";
+
+        SearchTemplateRequest request = RestSearchTemplateAction.parse(newBytesReference(source));
+        assertThat(request.getScript(), equalTo("{\"query\":{\"match\":{\"{{my_field}}\":\"{{my_value}}\"}},\"size\":\"{{my_size}}\"}"));
+        assertThat(request.getScriptType(), equalTo(ScriptService.ScriptType.INLINE));
+        assertThat(request.getScriptParams().size(), equalTo(3));
+        assertThat(request.getScriptParams(), hasEntry("my_field", "foo"));
+        assertThat(request.getScriptParams(), hasEntry("my_value", "bar"));
+        assertThat(request.getScriptParams(), hasEntry("my_size", 5));
+    }
+
+    public void testParseInlineTemplateAsString() throws Exception {
+        String source = "{'inline' : '{\\\"query\\\":{\\\"bool\\\":{\\\"must\\\":{\\\"match\\\":{\\\"foo\\\":\\\"{{text}}\\\"}}}}}'}";
+
+        SearchTemplateRequest request = RestSearchTemplateAction.parse(newBytesReference(source));
+        assertThat(request.getScript(), equalTo("{\"query\":{\"bool\":{\"must\":{\"match\":{\"foo\":\"{{text}}\"}}}}}"));
+        assertThat(request.getScriptType(), equalTo(ScriptService.ScriptType.INLINE));
+        assertThat(request.getScriptParams(), nullValue());
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testParseInlineTemplateAsStringWithParams() throws Exception {
+        String source = "{'inline' : '{\\\"query\\\":{\\\"match\\\":{\\\"{{field}}\\\":\\\"{{value}}\\\"}}}', " +
+                "'params': {'status': ['pending', 'published']}}";
+
+        SearchTemplateRequest request = RestSearchTemplateAction.parse(newBytesReference(source));
+        assertThat(request.getScript(), equalTo("{\"query\":{\"match\":{\"{{field}}\":\"{{value}}\"}}}"));
+        assertThat(request.getScriptType(), equalTo(ScriptService.ScriptType.INLINE));
+        assertThat(request.getScriptParams().size(), equalTo(1));
+        assertThat(request.getScriptParams(), hasKey("status"));
+        assertThat((List<String>) request.getScriptParams().get("status"), hasItems("pending", "published"));
+    }
+
+    public void testParseFileTemplate() throws Exception {
+        String source = "{'file' : 'fileTemplate'}";
+
+        SearchTemplateRequest request = RestSearchTemplateAction.parse(newBytesReference(source));
+        assertThat(request.getScript(), equalTo("fileTemplate"));
+        assertThat(request.getScriptType(), equalTo(ScriptService.ScriptType.FILE));
+        assertThat(request.getScriptParams(), nullValue());
+    }
+
+    public void testParseFileTemplateWithParams() throws Exception {
+        String source = "{'file' : 'template_foo', 'params' : {'foo': 'bar', 'size': 500}}";
+
+        SearchTemplateRequest request = RestSearchTemplateAction.parse(newBytesReference(source));
+        assertThat(request.getScript(), equalTo("template_foo"));
+        assertThat(request.getScriptType(), equalTo(ScriptService.ScriptType.FILE));
+        assertThat(request.getScriptParams().size(), equalTo(2));
+        assertThat(request.getScriptParams(), hasEntry("foo", "bar"));
+        assertThat(request.getScriptParams(), hasEntry("size", 500));
+    }
+
+    public void testParseStoredTemplate() throws Exception {
+        String source = "{'id' : 'storedTemplate'}";
+
+        SearchTemplateRequest request = RestSearchTemplateAction.parse(newBytesReference(source));
+        assertThat(request.getScript(), equalTo("storedTemplate"));
+        assertThat(request.getScriptType(), equalTo(ScriptService.ScriptType.STORED));
+        assertThat(request.getScriptParams(), nullValue());
+    }
+
+    public void testParseStoredTemplateWithParams() throws Exception {
+        String source = "{'id' : 'another_template', 'params' : {'bar': 'foo'}}";
+
+        SearchTemplateRequest request = RestSearchTemplateAction.parse(newBytesReference(source));
+        assertThat(request.getScript(), equalTo("another_template"));
+        assertThat(request.getScriptType(), equalTo(ScriptService.ScriptType.STORED));
+        assertThat(request.getScriptParams().size(), equalTo(1));
+        assertThat(request.getScriptParams(), hasEntry("bar", "foo"));
+    }
+
+    public void testParseWrongTemplate() {
+        // Unclosed template id
+        expectThrows(ParsingException.class, () -> RestSearchTemplateAction.parse(newBytesReference("{'id' : 'another_temp }")));
+    }
+
+    /**
+     * Creates a {@link BytesReference} with the given string while replacing single quote to double quotes.
+     */
+    private static BytesReference newBytesReference(String s) {
+        assertNotNull(s);
+        return new BytesArray(s.replace("'", "\""));
+    }
+}
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/RenderSearchTemplateTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/RenderSearchTemplateTests.java
index f2345c6583b..1e352e1ab4c 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/RenderSearchTemplateTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/RenderSearchTemplateTests.java
@@ -19,16 +19,15 @@
 
 package org.elasticsearch.messy.tests;
 
-import org.elasticsearch.action.admin.cluster.validate.template.RenderSearchTemplateResponse;
+import org.elasticsearch.action.search.template.SearchTemplateRequestBuilder;
+import org.elasticsearch.action.search.template.SearchTemplateResponse;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptService.ScriptType;
-import org.elasticsearch.script.Template;
 import org.elasticsearch.script.mustache.MustachePlugin;
 import org.elasticsearch.script.mustache.MustacheScriptEngineService;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -40,6 +39,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ExecutionException;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
@@ -53,6 +53,11 @@ public class RenderSearchTemplateTests extends ESIntegTestCase {
         return Collections.singleton(MustachePlugin.class);
     }
 
+    @Override
+    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
+        return nodePlugins();
+    }
+
     @Override
     protected void setupSuiteScopeCluster() throws Exception {
         ElasticsearchAssertions.assertAcked(client().admin().cluster().preparePutStoredScript()
@@ -75,14 +80,14 @@ public class RenderSearchTemplateTests extends ESIntegTestCase {
                 .put(Environment.PATH_CONF_SETTING.getKey(), configDir).build();
     }
 
-    public void testInlineTemplate() {
+    public void testInlineTemplate() throws ExecutionException, InterruptedException {
         Map<String, Object> params = new HashMap<>();
         params.put("value", "bar");
         params.put("size", 20);
-        Template template = new Template(TEMPLATE_CONTENTS, ScriptType.INLINE, MustacheScriptEngineService.NAME, XContentType.JSON, params);
-        RenderSearchTemplateResponse response = client().admin().cluster().prepareRenderSearchTemplate().template(template).get();
+        SearchTemplateResponse response = prepareRenderSearchTemplate(TEMPLATE_CONTENTS, ScriptType.INLINE, params).get();
         assertThat(response, notNullValue());
-        BytesReference source = response.source();
+        assertFalse(response.hasResponse());
+        BytesReference source = response.getSource();
         assertThat(source, notNullValue());
         Map<String, Object> sourceAsMap = XContentHelper.convertToMap(source, false).v2();
         assertThat(sourceAsMap, notNullValue());
@@ -93,10 +98,10 @@ public class RenderSearchTemplateTests extends ESIntegTestCase {
         params = new HashMap<>();
         params.put("value", "baz");
         params.put("size", 100);
-        template = new Template(TEMPLATE_CONTENTS, ScriptType.INLINE, MustacheScriptEngineService.NAME, XContentType.JSON, params);
-        response = client().admin().cluster().prepareRenderSearchTemplate().template(template).get();
+        response = prepareRenderSearchTemplate(TEMPLATE_CONTENTS, ScriptType.INLINE, params).get();
         assertThat(response, notNullValue());
-        source = response.source();
+        assertFalse(response.hasResponse());
+        source = response.getSource();
         assertThat(source, notNullValue());
         sourceAsMap = XContentHelper.convertToMap(source, false).v2();
         expected = TEMPLATE_CONTENTS.replace("{{value}}", "baz").replace("{{size}}", "100");
@@ -104,14 +109,14 @@ public class RenderSearchTemplateTests extends ESIntegTestCase {
         assertThat(sourceAsMap, equalTo(expectedMap));
     }
 
-    public void testIndexedTemplate() {
+    public void testIndexedTemplate() throws ExecutionException, InterruptedException {
         Map<String, Object> params = new HashMap<>();
         params.put("value", "bar");
         params.put("size", 20);
-        Template template = new Template("index_template_1", ScriptType.STORED, MustacheScriptEngineService.NAME, XContentType.JSON, params);
-        RenderSearchTemplateResponse response = client().admin().cluster().prepareRenderSearchTemplate().template(template).get();
+        SearchTemplateResponse response = prepareRenderSearchTemplate("index_template_1", ScriptType.STORED, params).get();
         assertThat(response, notNullValue());
-        BytesReference source = response.source();
+        assertFalse(response.hasResponse());
+        BytesReference source = response.getSource();
         assertThat(source, notNullValue());
         Map<String, Object> sourceAsMap = XContentHelper.convertToMap(source, false).v2();
         assertThat(sourceAsMap, notNullValue());
@@ -122,10 +127,9 @@ public class RenderSearchTemplateTests extends ESIntegTestCase {
         params = new HashMap<>();
         params.put("value", "baz");
         params.put("size", 100);
-        template = new Template("index_template_1", ScriptType.STORED, MustacheScriptEngineService.NAME, XContentType.JSON, params);
-        response = client().admin().cluster().prepareRenderSearchTemplate().template(template).get();
+        response = prepareRenderSearchTemplate("index_template_1", ScriptType.STORED, params).get();
         assertThat(response, notNullValue());
-        source = response.source();
+        source = response.getSource();
         assertThat(source, notNullValue());
         sourceAsMap = XContentHelper.convertToMap(source, false).v2();
         expected = TEMPLATE_CONTENTS.replace("{{value}}", "baz").replace("{{size}}", "100");
@@ -133,14 +137,14 @@ public class RenderSearchTemplateTests extends ESIntegTestCase {
         assertThat(sourceAsMap, equalTo(expectedMap));
     }
 
-    public void testFileTemplate() {
+    public void testFileTemplate() throws ExecutionException, InterruptedException {
         Map<String, Object> params = new HashMap<>();
         params.put("value", "bar");
         params.put("size", 20);
-        Template template = new Template("file_template_1", ScriptType.FILE, MustacheScriptEngineService.NAME, XContentType.JSON, params);
-        RenderSearchTemplateResponse response = client().admin().cluster().prepareRenderSearchTemplate().template(template).get();
+        SearchTemplateResponse response = prepareRenderSearchTemplate("file_template_1", ScriptType.FILE, params).get();
         assertThat(response, notNullValue());
-        BytesReference source = response.source();
+        assertFalse(response.hasResponse());
+        BytesReference source = response.getSource();
         assertThat(source, notNullValue());
         Map<String, Object> sourceAsMap = XContentHelper.convertToMap(source, false).v2();
         assertThat(sourceAsMap, notNullValue());
@@ -151,14 +155,17 @@ public class RenderSearchTemplateTests extends ESIntegTestCase {
         params = new HashMap<>();
         params.put("value", "baz");
         params.put("size", 100);
-        template = new Template("file_template_1", ScriptType.FILE, MustacheScriptEngineService.NAME, XContentType.JSON, params);
-        response = client().admin().cluster().prepareRenderSearchTemplate().template(template).get();
+        response = prepareRenderSearchTemplate("file_template_1", ScriptType.FILE, params).get();
         assertThat(response, notNullValue());
-        source = response.source();
+        source = response.getSource();
         assertThat(source, notNullValue());
         sourceAsMap = XContentHelper.convertToMap(source, false).v2();
         expected = TEMPLATE_CONTENTS.replace("{{value}}", "baz").replace("{{size}}", "100");
         expectedMap = XContentHelper.convertToMap(new BytesArray(expected), false).v2();
         assertThat(sourceAsMap, equalTo(expectedMap));
     }
+
+    private SearchTemplateRequestBuilder prepareRenderSearchTemplate(String script, ScriptType type, Map<String, Object> params) {
+        return new SearchTemplateRequestBuilder(client()).setScript(script).setScriptType(type).setScriptParams(params).setSimulate(true);
+    }
 }
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryTests.java
index 50e1a53207e..7ab58b7d1a7 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryTests.java
@@ -18,20 +18,16 @@
  */
 package org.elasticsearch.messy.tests;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse;
-import org.elasticsearch.action.search.SearchPhaseExecutionException;
+import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.template.SearchTemplateAction;
+import org.elasticsearch.action.search.template.SearchTemplateRequest;
+import org.elasticsearch.action.search.template.SearchTemplateRequestBuilder;
+import org.elasticsearch.action.search.template.SearchTemplateResponse;
 import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -40,6 +36,7 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.TemplateQueryBuilder;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.action.search.template.RestSearchTemplateAction;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptService.ScriptType;
 import org.elasticsearch.script.Template;
@@ -49,9 +46,16 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.junit.Before;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.hamcrest.Matchers.containsString;
@@ -69,6 +73,11 @@ public class TemplateQueryTests extends ESIntegTestCase {
         return Collections.singleton(MustachePlugin.class);
     }
 
+    @Override
+    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
+        return nodePlugins();
+    }
+
     @Before
     public void setup() throws IOException {
         createIndex("test");
@@ -142,11 +151,11 @@ public class TemplateQueryTests extends ESIntegTestCase {
         SearchRequest searchRequest = new SearchRequest();
         searchRequest.indices("_all");
 
-        String query = "{ \"template\" : { \"query\": {\"match_{{template}}\": {} } }, \"params\" : { \"template\":\"all\" } }";
-        searchRequest.template(parseTemplate(query));
-
-        SearchResponse searchResponse = client().search(searchRequest).get();
-        assertHitCount(searchResponse, 2);
+        String query = "{ \"inline\" : { \"query\": {\"match_{{template}}\": {} } }, \"params\" : { \"template\":\"all\" } }";
+        SearchTemplateRequest request =  RestSearchTemplateAction.parse(new BytesArray(query));
+        request.setRequest(searchRequest);
+        SearchTemplateResponse response = client().execute(SearchTemplateAction.INSTANCE, request).get();
+        assertHitCount(response.getResponse(), 2);
     }
 
     private Template parseTemplate(String template) throws IOException {
@@ -155,23 +164,28 @@ public class TemplateQueryTests extends ESIntegTestCase {
         }
     }
 
-    // Releates to #6318
+    // Relates to #6318
     public void testSearchRequestFail() throws Exception {
+        String query = "{ \"query\": {\"match_all\": {}}, \"size\" : \"{{my_size}}\"  }";
+
         SearchRequest searchRequest = new SearchRequest();
         searchRequest.indices("_all");
-        try {
-            String query = "{ \"template\" : { \"query\": {\"match_all\": {}}, \"size\" : \"{{my_size}}\"  } }";
-            searchRequest.template(parseTemplate(query));
-            client().search(searchRequest).get();
-            fail("expected exception");
-        } catch (Exception ex) {
-            // expected - no params
-        }
-        String query = "{ \"template\" : { \"query\": {\"match_all\": {}}, \"size\" : \"{{my_size}}\"  }, \"params\" : { \"my_size\": 1 } }";
-        searchRequest.template(parseTemplate(query));
 
-        SearchResponse searchResponse = client().search(searchRequest).get();
-        assertThat(searchResponse.getHits().hits().length, equalTo(1));
+        expectThrows(Exception.class, () -> new SearchTemplateRequestBuilder(client())
+                .setRequest(searchRequest)
+                .setScript(query)
+                .setScriptType(ScriptType.INLINE)
+                .setScriptParams(randomBoolean() ? null : Collections.emptyMap())
+                .get());
+
+        SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client())
+                .setRequest(searchRequest)
+                .setScript(query)
+                .setScriptType(ScriptType.INLINE)
+                .setScriptParams(Collections.singletonMap("my_size", 1))
+                .get();
+
+        assertThat(searchResponse.getResponse().getHits().hits().length, equalTo(1));
     }
 
     public void testThatParametersCanBeSet() throws Exception {
@@ -187,18 +201,24 @@ public class TemplateQueryTests extends ESIntegTestCase {
         templateParams.put("myField", "theField");
         templateParams.put("myValue", "foo");
 
-        SearchResponse searchResponse = client().prepareSearch("test").setTypes("type")
-                .setTemplate(new Template("full-query-template", ScriptType.FILE, MustacheScriptEngineService.NAME, null, templateParams))
+        SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client())
+                .setRequest(new SearchRequest("test").types("type"))
+                .setScript("full-query-template")
+                .setScriptType(ScriptType.FILE)
+                .setScriptParams(templateParams)
                 .get();
-        assertHitCount(searchResponse, 4);
+        assertHitCount(searchResponse.getResponse(), 4);
         // size kicks in here...
-        assertThat(searchResponse.getHits().getHits().length, is(2));
+        assertThat(searchResponse.getResponse().getHits().getHits().length, is(2));
 
         templateParams.put("myField", "otherField");
-        searchResponse = client().prepareSearch("test").setTypes("type")
-                .setTemplate(new Template("full-query-template", ScriptType.FILE, MustacheScriptEngineService.NAME, null, templateParams))
+        searchResponse = new SearchTemplateRequestBuilder(client())
+                .setRequest(new SearchRequest("test").types("type"))
+                .setScript("full-query-template")
+                .setScriptType(ScriptType.FILE)
+                .setScriptParams(templateParams)
                 .get();
-        assertHitCount(searchResponse, 1);
+        assertHitCount(searchResponse.getResponse(), 1);
     }
 
     public void testSearchTemplateQueryFromFile() throws Exception {
@@ -206,9 +226,10 @@ public class TemplateQueryTests extends ESIntegTestCase {
         searchRequest.indices("_all");
         String query = "{" + "  \"file\": \"full-query-template\"," + "  \"params\":{" + "    \"mySize\": 2,"
                 + "    \"myField\": \"text\"," + "    \"myValue\": \"value1\"" + "  }" + "}";
-        searchRequest.template(parseTemplate(query));
-        SearchResponse searchResponse = client().search(searchRequest).get();
-        assertThat(searchResponse.getHits().hits().length, equalTo(1));
+        SearchTemplateRequest request =  RestSearchTemplateAction.parse(new BytesArray(query));
+        request.setRequest(searchRequest);
+        SearchTemplateResponse searchResponse = client().execute(SearchTemplateAction.INSTANCE, request).get();
+        assertThat(searchResponse.getResponse().getHits().hits().length, equalTo(1));
     }
 
     /**
@@ -217,11 +238,12 @@ public class TemplateQueryTests extends ESIntegTestCase {
     public void testTemplateQueryAsEscapedString() throws Exception {
         SearchRequest searchRequest = new SearchRequest();
         searchRequest.indices("_all");
-        String query = "{" + "  \"template\" : \"{ \\\"size\\\": \\\"{{size}}\\\", \\\"query\\\":{\\\"match_all\\\":{}}}\","
+        String query = "{" + "  \"inline\" : \"{ \\\"size\\\": \\\"{{size}}\\\", \\\"query\\\":{\\\"match_all\\\":{}}}\","
                 + "  \"params\":{" + "    \"size\": 1" + "  }" + "}";
-        searchRequest.template(parseTemplate(query));
-        SearchResponse searchResponse = client().search(searchRequest).get();
-        assertThat(searchResponse.getHits().hits().length, equalTo(1));
+        SearchTemplateRequest request =  RestSearchTemplateAction.parse(new BytesArray(query));
+        request.setRequest(searchRequest);
+        SearchTemplateResponse searchResponse = client().execute(SearchTemplateAction.INSTANCE, request).get();
+        assertThat(searchResponse.getResponse().getHits().hits().length, equalTo(1));
     }
 
     /**
@@ -232,11 +254,12 @@ public class TemplateQueryTests extends ESIntegTestCase {
         SearchRequest searchRequest = new SearchRequest();
         searchRequest.indices("_all");
         String templateString = "{"
-                + "  \"template\" : \"{ {{#use_size}} \\\"size\\\": \\\"{{size}}\\\", {{/use_size}} \\\"query\\\":{\\\"match_all\\\":{}}}\","
+                + "  \"inline\" : \"{ {{#use_size}} \\\"size\\\": \\\"{{size}}\\\", {{/use_size}} \\\"query\\\":{\\\"match_all\\\":{}}}\","
                 + "  \"params\":{" + "    \"size\": 1," + "    \"use_size\": true" + "  }" + "}";
-        searchRequest.template(parseTemplate(templateString));
-        SearchResponse searchResponse = client().search(searchRequest).get();
-        assertThat(searchResponse.getHits().hits().length, equalTo(1));
+        SearchTemplateRequest request =  RestSearchTemplateAction.parse(new BytesArray(templateString));
+        request.setRequest(searchRequest);
+        SearchTemplateResponse searchResponse = client().execute(SearchTemplateAction.INSTANCE, request).get();
+        assertThat(searchResponse.getResponse().getHits().hits().length, equalTo(1));
     }
 
     /**
@@ -249,9 +272,10 @@ public class TemplateQueryTests extends ESIntegTestCase {
         String templateString = "{"
                 + "  \"inline\" : \"{ \\\"query\\\":{\\\"match_all\\\":{}} {{#use_size}}, \\\"size\\\": \\\"{{size}}\\\" {{/use_size}} }\","
                 + "  \"params\":{" + "    \"size\": 1," + "    \"use_size\": true" + "  }" + "}";
-        searchRequest.template(parseTemplate(templateString));
-        SearchResponse searchResponse = client().search(searchRequest).get();
-        assertThat(searchResponse.getHits().hits().length, equalTo(1));
+        SearchTemplateRequest request =  RestSearchTemplateAction.parse(new BytesArray(templateString));
+        request.setRequest(searchRequest);
+        SearchTemplateResponse searchResponse = client().execute(SearchTemplateAction.INSTANCE, request).get();
+        assertThat(searchResponse.getResponse().getHits().hits().length, equalTo(1));
     }
 
     public void testIndexedTemplateClient() throws Exception {
@@ -296,10 +320,11 @@ public class TemplateQueryTests extends ESIntegTestCase {
         Map<String, Object> templateParams = new HashMap<>();
         templateParams.put("fieldParam", "foo");
 
-        SearchResponse searchResponse = client().prepareSearch("test").setTypes("type")
-                .setTemplate(new Template("testTemplate", ScriptType.STORED, MustacheScriptEngineService.NAME, null, templateParams))
+        SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client())
+                .setRequest(new SearchRequest("test").types("type"))
+                .setScript("testTemplate").setScriptType(ScriptType.STORED).setScriptParams(templateParams)
                 .get();
-        assertHitCount(searchResponse, 4);
+        assertHitCount(searchResponse.getResponse(), 4);
 
         assertAcked(client().admin().cluster()
                 .prepareDeleteStoredScript(MustacheScriptEngineService.NAME, "testTemplate"));
@@ -308,16 +333,11 @@ public class TemplateQueryTests extends ESIntegTestCase {
                 .prepareGetStoredScript(MustacheScriptEngineService.NAME, "testTemplate").get();
         assertNull(getResponse.getStoredScript());
 
-        try {
-            client().prepareSearch("test")
-                    .setTypes("type")
-                    .setTemplate(
-                            new Template("/template_index/mustache/1000", ScriptType.STORED, MustacheScriptEngineService.NAME, null,
-                                    templateParams)).get();
-            fail("Expected SearchPhaseExecutionException");
-        } catch (SearchPhaseExecutionException e) {
-            assertThat(e.toString(), containsString("Illegal index script format"));
-        }
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new SearchTemplateRequestBuilder(client())
+                .setRequest(new SearchRequest("test").types("type"))
+                .setScript("/template_index/mustache/1000").setScriptType(ScriptType.STORED).setScriptParams(templateParams)
+                .get());
+        assertThat(e.getMessage(), containsString("Illegal index script format [/template_index/mustache/1000] should be /lang/id"));
     }
 
     public void testIndexedTemplate() throws Exception {
@@ -357,8 +377,6 @@ public class TemplateQueryTests extends ESIntegTestCase {
                         "}"))
         );
 
-
-
         List<IndexRequestBuilder> builders = new ArrayList<>();
         builders.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}"));
         builders.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}"));
@@ -370,50 +388,34 @@ public class TemplateQueryTests extends ESIntegTestCase {
         Map<String, Object> templateParams = new HashMap<>();
         templateParams.put("fieldParam", "foo");
 
-        SearchResponse searchResponse = client()
-                .prepareSearch("test")
-                .setTypes("type")
-                .setTemplate(
-                        new Template("/mustache/1a", ScriptService.ScriptType.STORED, MustacheScriptEngineService.NAME, null,
-                                templateParams)).get();
-        assertHitCount(searchResponse, 4);
-
-        try {
-            client().prepareSearch("test")
-                    .setTypes("type")
-                    .setTemplate(
-                            new Template("/template_index/mustache/1000", ScriptService.ScriptType.STORED,
-                                    MustacheScriptEngineService.NAME, null, templateParams)).get();
-            fail("shouldn't get here");
-        } catch (SearchPhaseExecutionException spee) {
-            //all good
-        }
-
-        try {
-            searchResponse = client()
-                    .prepareSearch("test")
-                    .setTypes("type")
-                    .setTemplate(
-                            new Template("/myindex/mustache/1", ScriptService.ScriptType.STORED, MustacheScriptEngineService.NAME, null,
-                                    templateParams)).get();
-            assertFailures(searchResponse);
-        } catch (SearchPhaseExecutionException spee) {
-            //all good
-        }
-
-        searchResponse = client().prepareSearch("test").setTypes("type")
-                .setTemplate(new Template("1a", ScriptService.ScriptType.STORED, MustacheScriptEngineService.NAME, null, templateParams))
+        SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client())
+                .setRequest(new SearchRequest().indices("test").types("type"))
+                .setScript("/mustache/1a")
+                .setScriptType(ScriptType.STORED)
+                .setScriptParams(templateParams)
                 .get();
-        assertHitCount(searchResponse, 4);
+        assertHitCount(searchResponse.getResponse(), 4);
+
+        expectThrows(IllegalArgumentException.class, () -> new SearchTemplateRequestBuilder(client())
+                .setRequest(new SearchRequest().indices("test").types("type"))
+                .setScript("/template_index/mustache/1000")
+                .setScriptType(ScriptType.STORED)
+                .setScriptParams(templateParams)
+                .get());
+
+        expectThrows(IllegalArgumentException.class, () ->  new SearchTemplateRequestBuilder(client())
+                    .setRequest(new SearchRequest().indices("test").types("type"))
+                    .setScript("/myindex/mustache/1")
+                    .setScriptType(ScriptType.STORED)
+                    .setScriptParams(templateParams)
+                    .get());
 
         templateParams.put("fieldParam", "bar");
-        searchResponse = client()
-                .prepareSearch("test")
-                .setTypes("type")
-                .setTemplate(
-                        new Template("/mustache/2", ScriptService.ScriptType.STORED, MustacheScriptEngineService.NAME, null,
-                                templateParams)).get();
-        assertHitCount(searchResponse, 1);
+        searchResponse = new SearchTemplateRequestBuilder(client())
+                .setRequest(new SearchRequest("test").types("type"))
+                .setScript("/mustache/2").setScriptType(ScriptService.ScriptType.STORED).setScriptParams(templateParams)
+                .get();
+        assertHitCount(searchResponse.getResponse(), 1);
 
         Map<String, Object> vars = new HashMap<>();
         vars.put("fieldParam", "bar");
@@ -459,16 +461,11 @@ public class TemplateQueryTests extends ESIntegTestCase {
             Map<String, Object> templateParams = new HashMap<>();
             templateParams.put("P_Keyword1", "dev");
 
-            try {
-                client().prepareSearch("testindex")
-                        .setTypes("test")
-                        .setTemplate(
-                                new Template("git01", ScriptService.ScriptType.STORED, MustacheScriptEngineService.NAME, null,
-                                        templateParams)).get();
-                fail("Broken test template is parsing w/o error.");
-            } catch (SearchPhaseExecutionException e) {
-                // the above is expected to fail
-            }
+            ParsingException e = expectThrows(ParsingException.class, () -> new SearchTemplateRequestBuilder(client())
+                    .setRequest(new SearchRequest("testindex").types("test"))
+                    .setScript("git01").setScriptType(ScriptService.ScriptType.STORED).setScriptParams(templateParams)
+                    .get());
+            assertThat(e.getMessage(), containsString("[match] query does not support type ooophrase_prefix"));
 
             assertAcked(client().admin().cluster().preparePutStoredScript()
                     .setScriptLang(MustacheScriptEngineService.NAME)
@@ -476,13 +473,11 @@ public class TemplateQueryTests extends ESIntegTestCase {
                     .setSource(new BytesArray("{\"query\": {\"match\": {\"searchtext\": {\"query\": \"{{P_Keyword1}}\"," +
                             "\"type\": \"phrase_prefix\"}}}}")));
 
-            SearchResponse searchResponse = client()
-                    .prepareSearch("testindex")
-                    .setTypes("test")
-                    .setTemplate(
-                            new Template("git01", ScriptService.ScriptType.STORED, MustacheScriptEngineService.NAME, null, templateParams))
+            SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client())
+                    .setRequest(new SearchRequest("testindex").types("test"))
+                    .setScript("git01").setScriptType(ScriptService.ScriptType.STORED).setScriptParams(templateParams)
                     .get();
-            assertHitCount(searchResponse, 1);
+            assertHitCount(searchResponse.getResponse(), 1);
         }
     }
 
@@ -508,13 +503,11 @@ public class TemplateQueryTests extends ESIntegTestCase {
       String[] fieldParams = {"foo","bar"};
       arrayTemplateParams.put("fieldParam", fieldParams);
 
-        SearchResponse searchResponse = client()
-                .prepareSearch("test")
-                .setTypes("type")
-                .setTemplate(
-                        new Template("/mustache/4", ScriptService.ScriptType.STORED, MustacheScriptEngineService.NAME, null,
-                                arrayTemplateParams)).get();
-        assertHitCount(searchResponse, 5);
+        SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client())
+                .setRequest(new SearchRequest("test").types("type"))
+                .setScript("/mustache/4").setScriptType(ScriptService.ScriptType.STORED).setScriptParams(arrayTemplateParams)
+                .get();
+        assertHitCount(searchResponse.getResponse(), 5);
     }
 
 }
diff --git a/modules/lang-mustache/src/test/resources/org/elasticsearch/action/search/template/simple-msearch-template.json b/modules/lang-mustache/src/test/resources/org/elasticsearch/action/search/template/simple-msearch-template.json
new file mode 100644
index 00000000000..ac99bc299b3
--- /dev/null
+++ b/modules/lang-mustache/src/test/resources/org/elasticsearch/action/search/template/simple-msearch-template.json
@@ -0,0 +1,6 @@
+{"index":["test0", "test1"], "request_cache": true}
+{"inline": {"query" : {"match_{{template}}" :{}}}, "params": {"template": "all" } }
+{"index" : "test2,test3", "type" : "type1", "preference": "_local"}
+{"inline": {"query" : {"match_{{template}}" :{}}}, "params": {"template": "all" } }
+{"index" : ["test4", "test1"], "type" :  [ "type2", "type1" ], "routing": "123"}
+{"inline": {"query" : {"match_{{template}}" :{}}}, "params": {"template": "all" } }
diff --git a/modules/lang-mustache/src/test/resources/org/elasticsearch/messy/tests/config/scripts/template_1.mustache b/modules/lang-mustache/src/test/resources/org/elasticsearch/messy/tests/config/scripts/template_1.mustache
new file mode 100644
index 00000000000..4b1c02f3b24
--- /dev/null
+++ b/modules/lang-mustache/src/test/resources/org/elasticsearch/messy/tests/config/scripts/template_1.mustache
@@ -0,0 +1,11 @@
+{
+    "query": {
+        "match": {
+            "{{field}}": {
+                "query" : "{{value}}",
+                "operator" : "{{operator}}{{^operator}}or{{/operator}}"
+            }
+        }
+    },
+    "size": {{size}}
+}
\ No newline at end of file
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_search_request_template.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_search_request_template.yaml
index a0a6695023a..8d3154cc8d4 100644
--- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_search_request_template.yaml
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_search_request_template.yaml
@@ -18,12 +18,21 @@
 
   - do:
       search_template:
-        body: { "template" : { "query": { "term": { "text": { "value": "{{template}}" } } } }, "params": { "template": "value1" } }
+        body: { "inline" : { "query": { "term": { "text": { "value": "{{template}}" } } } }, "params": { "template": "value1" } }
 
   - match: { hits.total: 1 }
 
   - do:
       search_template:
-        body: { "template" : { "query": { "match_{{template}}": {} } }, "params" : { "template" : "all" } }
+        body: { "inline" : { "query": { "match_{{template}}": {} } }, "params" : { "template" : "all" } }
 
   - match: { hits.total: 2 }
+
+---
+"Missing template search request":
+
+  - do:
+      catch: missing
+      search_template:
+        body: { "id" : "unknown", "params": { "template": "value1" } }
+
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_messy_test_msearch.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_messy_test_msearch.yaml
index da9c417ffeb..ef1cac040b0 100644
--- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_messy_test_msearch.yaml
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_messy_test_msearch.yaml
@@ -1,5 +1,5 @@
 ---
-"Basic multi-search":
+"Basic multi-search with template query":
   - do:
       index:
           index:  test_1
@@ -24,26 +24,6 @@
   - do:
       indices.refresh: {}
 
-  - do:
-      msearch:
-        body:
-          - index: test_1
-          - query:
-              match_all: {}
-          - index: test_2
-          - query:
-              match_all: {}
-          - search_type: query_then_fetch
-            index: test_1
-          - query:
-              match: {foo: bar}
-
-  - match:  { responses.0.hits.total:     3  }
-  - match:  { responses.1.error.root_cause.0.type: index_not_found_exception }
-  - match:  { responses.1.error.root_cause.0.reason: "/no.such.index/" }
-  - match:  { responses.1.error.root_cause.0.index: test_2 }
-  - match:  { responses.2.hits.total:     1  }
-
   - do:
       msearch:
         body: 
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/10_basic.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/10_basic.yaml
new file mode 100644
index 00000000000..2a32e75331e
--- /dev/null
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/10_basic.yaml
@@ -0,0 +1,126 @@
+---
+setup:
+
+  - do:
+      index:
+          index:  index_1
+          type:   test
+          id:     1
+          body:   { foo: bar }
+
+  - do:
+      index:
+          index:  index_1
+          type:   test
+          id:     2
+          body:   { foo: baz }
+
+  - do:
+      index:
+          index:  index_1
+          type:   test
+          id:     3
+          body:   { foo: foo }
+
+  - do:
+      index:
+          index:  index_2
+          type:   test
+          id:     1
+          body:   { foo: foo }
+
+  - do:
+      indices.refresh: {}
+
+---
+"Basic multi-search template":
+
+  - do:
+      template.msearch:
+        body:
+          - index: index_*
+          - inline: '{"query": {"match": {"foo": "{{value}}"} } }'
+            params:
+              value: "foo"
+          - index: index_*
+          - inline: '{"query": {"match": {"{{field}}": "{{value}}"} } }'
+            params:
+              field: "foo"
+              value: "bar"
+          - index: _all
+          - inline: '{"query": {"{{query_type}}": {{query_content}} } }'
+            params:
+              query_type: "match_all"
+              query_content: "{}"
+          - index: _all
+          - inline:
+              query:
+                match: {foo: "{{text}}"}
+              size: 0
+            params:
+              text: "baz"
+
+  - match:  { responses.0.hits.total:     2  }
+  - match:  { responses.1.hits.total:     1  }
+  - match:  { responses.2.hits.total:     4  }
+  - match:  { responses.3.hits.total:     1  }
+  - length: { responses.3.hits.hits:      0  }
+
+
+---
+"Multi-search template with errors":
+
+  - do:
+      template.msearch:
+        body:
+          # Search 0 is OK
+          - index: index_*
+          - inline: '{"query": {"match": {"foo": "{{value}}"} } }'
+            params:
+              value: "foo"
+          # Search 1 has an unclosed JSON template
+          - index: index_*
+          - inline: '{"query": {"match": {'
+            params:
+              field: "foo"
+              value: "bar"
+          # Search 2 is OK
+          - index: _all
+          - inline:
+              query:
+                match: {foo: "{{text}}"}
+            params:
+              text: "baz"
+          # Search 3 has an unknown query type
+          - index: index_*
+          - inline: '{"query": {"{{query_type}}": {} }' # Unknown query type
+            params:
+              query_type: "unknown"
+
+  - match:  { responses.0.hits.total:     2  }
+  - match:  { responses.1.error.root_cause.0.type: json_parse_exception }
+  - match:  { responses.1.error.root_cause.0.reason: "/Unexpected.end.of.input/" }
+  - match:  { responses.2.hits.total:     1  }
+  - match:  { responses.3.error.root_cause.0.type: parsing_exception }
+  - match:  { responses.3.error.root_cause.0.reason: "/no.\\[query\\].registered.for.\\[unknown\\]/" }
+
+---
+"Multi-search template with invalid request":
+
+  - do:
+      catch: /(.)*action_request_validation_exception(.)*template.is.missing(.)*/
+      template.msearch:
+        body:
+          # Search 0 is OK
+          - index: index_*
+          - inline: '{"query": {"match": {"foo": "{{value}}"} } }'
+            params:
+              value: "foo"
+          # Search 1 has not template
+          - index: _all
+          - inline: ""
+          # Search 2 is OK
+          - index: index_*
+          - inline: '{"query": {"match": {"foo": "{{value}}"} } }'
+            params:
+              value: "bar"
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/20_stored_template.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/20_stored_template.yaml
new file mode 100644
index 00000000000..5d04449e541
--- /dev/null
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/20_stored_template.yaml
@@ -0,0 +1,66 @@
+---
+setup:
+
+  - do:
+      index:
+          index:  index_1
+          type:   test
+          id:     1
+          body:   { foo: bar }
+
+  - do:
+      index:
+          index:  index_1
+          type:   test
+          id:     2
+          body:   { foo: baz }
+
+  - do:
+      index:
+          index:  index_1
+          type:   test
+          id:     3
+          body:   { foo: foo }
+
+  - do:
+      index:
+          index:  index_2
+          type:   test
+          id:     1
+          body:   { foo: foo }
+
+  - do:
+      indices.refresh: {}
+
+---
+"Basic multi-search using stored template":
+
+  - do:
+      put_template:
+        id: stored_template_1
+        body: {"template": {"query": {"match": {"{{field}}": "{{value}}" }}}}
+  - match: { acknowledged: true }
+
+  - do:
+      template.msearch:
+        body:
+          - index: index_*
+          - id: stored_template_1
+            params:
+              field: "foo"
+              value: "foo"
+          - index: _all
+          - id: stored_template_1
+            params:
+              field: "foo"
+              value: "bar"
+          - index: index_2
+          - id: stored_template_1
+            params:
+              field: "foo"
+              value: "foo"
+
+  - match:  { responses.0.hits.total:     2  }
+  - match:  { responses.1.hits.total:     1  }
+  - match:  { responses.2.hits.total:     1  }
+
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/30_file_template.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/30_file_template.yaml
new file mode 100644
index 00000000000..0c6ad8021e1
--- /dev/null
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/30_file_template.yaml
@@ -0,0 +1,72 @@
+---
+setup:
+
+  - do:
+      index:
+          index:  index_1
+          type:   test
+          id:     1
+          body:   { foo: bar }
+
+  - do:
+      index:
+          index:  index_1
+          type:   test
+          id:     2
+          body:   { foo: baz }
+
+  - do:
+      index:
+          index:  index_1
+          type:   test
+          id:     3
+          body:   { foo: foo }
+
+  - do:
+      index:
+          index:  index_2
+          type:   test
+          id:     1
+          body:   { foo: foo }
+
+  - do:
+      indices.refresh: {}
+
+---
+"Basic multi-search using file template":
+
+  - do:
+      render_search_template:
+        body: { "file": "template_1", "params": { "field": "foo", "value": "bar", "size": 20 } }
+
+  - match: { template_output.query.match.foo.query: "bar" }
+  - match: { template_output.query.match.foo.operator: "or" }
+  - match: { template_output.size: 20 }
+
+  - do:
+      template.msearch:
+        body:
+          - index: index_*
+          - file: template_1
+            params:
+              field: "foo"
+              value: "foo"
+              size: 10
+          - index: _all
+          - file: template_1
+            params:
+              field: "foo"
+              value: "bar"
+              operator: "and"
+              size: 50
+          - index: index_2
+          - file: template_1
+            params:
+              field: "foo"
+              value: "foo"
+              size: 0
+
+  - match:  { responses.0.hits.total:     2  }
+  - match:  { responses.1.hits.total:     1  }
+  - match:  { responses.2.hits.total:     1  }
+
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/template.msearch.json b/rest-api-spec/src/main/resources/rest-api-spec/api/template.msearch.json
new file mode 100644
index 00000000000..379b7bdf362
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/template.msearch.json
@@ -0,0 +1,32 @@
+{
+  "template.msearch": {
+    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/search-template.html",
+    "methods": ["GET", "POST"],
+    "url": {
+      "path": "/_msearch/template",
+      "paths": ["/_msearch/template", "/{index}/_msearch/template", "/{index}/{type}/_msearch/template"],
+      "parts": {
+        "index": {
+          "type" : "list",
+          "description" : "A comma-separated list of index names to use as default"
+        },
+        "type": {
+          "type" : "list",
+          "description" : "A comma-separated list of document types to use as default"
+        }
+      },
+      "params": {
+        "search_type": {
+          "type" : "enum",
+          "options" : ["query_then_fetch", "query_and_fetch", "dfs_query_then_fetch", "dfs_query_and_fetch"],
+          "description" : "Search operation type"
+        }
+      }
+    },
+    "body": {
+      "description": "The request definitions (metadata-search request definition pairs), separated by newlines",
+      "required" : true,
+      "serialize" : "bulk"
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/template.search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/template.search.json
new file mode 100644
index 00000000000..6a2a8c1d7e2
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/template.search.json
@@ -0,0 +1,56 @@
+{
+  "template.search": {
+    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/search-template.html",
+    "methods": ["GET", "POST"],
+    "url": {
+      "path": "/_search/template",
+      "paths": ["/_search/template", "/{index}/_search/template", "/{index}/{type}/_search/template"],
+      "parts": {
+        "index": {
+         "type" : "list",
+         "description" : "A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices"
+        },
+        "type": {
+          "type" : "list",
+          "description" : "A comma-separated list of document types to search; leave empty to perform the operation on all types"
+        }
+      },
+      "params" : {
+        "ignore_unavailable": {
+            "type" : "boolean",
+            "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
+        },
+        "allow_no_indices": {
+            "type" : "boolean",
+            "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
+        },
+        "expand_wildcards": {
+            "type" : "enum",
+            "options" : ["open","closed","none","all"],
+            "default" : "open",
+            "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both."
+        },
+        "preference": {
+          "type" : "string",
+          "description" : "Specify the node or shard the operation should be performed on (default: random)"
+        },
+        "routing": {
+          "type" : "list",
+          "description" : "A comma-separated list of specific routing values"
+        },
+        "scroll": {
+          "type" : "duration",
+          "description" : "Specify how long a consistent view of the index should be maintained for scrolled search"
+        },
+        "search_type": {
+          "type" : "enum",
+          "options" : ["query_then_fetch", "query_and_fetch", "dfs_query_then_fetch", "dfs_query_and_fetch"],
+          "description" : "Search operation type"
+        }
+      }
+    },
+    "body": {
+      "description": "The search definition template and its params"
+    }
+  }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yaml
new file mode 100644
index 00000000000..6e6b0cd3b4f
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yaml
@@ -0,0 +1,64 @@
+---
+setup:
+
+  - do:
+      index:
+          index:  index_1
+          type:   test
+          id:     1
+          body:   { foo: bar }
+
+  - do:
+      index:
+          index:  index_1
+          type:   test
+          id:     2
+          body:   { foo: baz }
+
+  - do:
+      index:
+          index:  index_1
+          type:   test
+          id:     3
+          body:   { foo: foo }
+
+  - do:
+      index:
+          index:  index_2
+          type:   test
+          id:     1
+          body:   { foo: foo }
+
+  - do:
+      indices.refresh: {}
+
+---
+"Basic multi-search":
+
+  - do:
+      msearch:
+        body:
+          - index: index_*
+          - query:
+              match: {foo: foo}
+          - index: index_2
+          - query:
+              match_all: {}
+          - index: index_1
+          - query:
+              match: {foo: foo}
+          - index: index_3
+          - query:
+              match_all: {}
+          - type: test
+          - query:
+              match_all: {}
+
+  - match:  { responses.0.hits.total:     2  }
+  - match:  { responses.1.hits.total:     1  }
+  - match:  { responses.2.hits.total:     1  }
+  - match:  { responses.3.error.root_cause.0.type: index_not_found_exception }
+  - match:  { responses.3.error.root_cause.0.reason: "/no.such.index/" }
+  - match:  { responses.3.error.root_cause.0.index: index_3 }
+  - match:  { responses.4.hits.total:     4  }
+
diff --git a/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java b/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
index cabb67a8771..9125fd47e30 100644
--- a/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
+++ b/test/framework/src/main/java/org/elasticsearch/search/MockSearchService.java
@@ -28,8 +28,10 @@ import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
+import org.elasticsearch.search.dfs.DfsPhase;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.query.QueryPhase;
 import org.elasticsearch.search.suggest.Suggesters;
 import org.elasticsearch.threadpool.ThreadPool;
 
@@ -74,9 +76,8 @@ public class MockSearchService extends SearchService {
     @Inject
     public MockSearchService(Settings settings, ClusterSettings clusterSettings, ClusterService clusterService,
             IndicesService indicesService, ThreadPool threadPool, ScriptService scriptService,
-            BigArrays bigArrays, FetchPhase fetchPhase, AggregatorParsers aggParsers, Suggesters suggesters) {
-        super(settings, clusterSettings, clusterService, indicesService, threadPool, scriptService, bigArrays,
-                fetchPhase, aggParsers, suggesters);
+            BigArrays bigArrays, FetchPhase fetchPhase) {
+        super(settings, clusterSettings, clusterService, indicesService, threadPool, scriptService, bigArrays, fetchPhase);
     }
 
     @Override

From c87ba0bfa8b141df2b5424f09a6f046602d0b05b Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Thu, 23 Jun 2016 09:44:33 +0200
Subject: [PATCH 1040/1311] Fix docs build.

---
 docs/reference/search/request/docvalue-fields.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/search/request/docvalue-fields.asciidoc b/docs/reference/search/request/docvalue-fields.asciidoc
index bd090a3bd8d..b4d2493d853 100644
--- a/docs/reference/search/request/docvalue-fields.asciidoc
+++ b/docs/reference/search/request/docvalue-fields.asciidoc
@@ -1,7 +1,7 @@
 [[search-request-docvalue-fields]]
 === Doc value Fields
 
-Allows to return the <<docvalue,doc value>> representation of a field for each hit, for
+Allows to return the <<doc-values,doc value>> representation of a field for each hit, for
 example:
 
 [source,js]

From 6c8744ecb5a53ba8e4cb06a3955db3b11e601894 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Thu, 23 Jun 2016 10:16:04 +0200
Subject: [PATCH 1041/1311] Attempt at fixing
 IndexStatsIT.testFilterCacheStats.

I suspect recent failures are due to the fact that the cache disables itself
when there is contention. This runs assertions in an assertBusy block since
they should eventually succeed.
---
 .../indices/stats/IndexStatsIT.java           | 54 ++++++++++---------
 1 file changed, 30 insertions(+), 24 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
index a4096fde9da..6bce95af184 100644
--- a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
@@ -30,7 +30,6 @@ import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -1015,22 +1014,27 @@ public class IndexStatsIT extends ESIntegTestCase {
         assertCumulativeQueryCacheStats(response);
         assertEquals(0, response.getTotal().queryCache.getCacheSize());
 
-        SearchResponse r;
-        assertSearchResponse(r = client().prepareSearch("index").setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("foo", "baz"))).get());
-        response = client().admin().indices().prepareStats("index").setQueryCache(true).get();
-        assertCumulativeQueryCacheStats(response);
-        assertThat(response.getTotal().queryCache.getHitCount(), equalTo(0L));
-        assertThat(response.getTotal().queryCache.getEvictions(), equalTo(0L));
-        assertThat(response.getTotal().queryCache.getMissCount(), greaterThan(0L));
-        assertThat(response.getTotal().queryCache.getCacheSize(), greaterThan(0L));
+        // the query cache has an optimization that disables it automatically if there is contention,
+        // so we run it in an assertBusy block which should eventually succeed
+        assertBusy(() -> {
+            assertSearchResponse(client().prepareSearch("index").setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("foo", "baz"))).get());
+            IndicesStatsResponse stats = client().admin().indices().prepareStats("index").setQueryCache(true).get();
+            assertCumulativeQueryCacheStats(stats);
+            assertThat(stats.getTotal().queryCache.getHitCount(), equalTo(0L));
+            assertThat(stats.getTotal().queryCache.getEvictions(), equalTo(0L));
+            assertThat(stats.getTotal().queryCache.getMissCount(), greaterThan(0L));
+            assertThat(stats.getTotal().queryCache.getCacheSize(), greaterThan(0L));
+        });
 
-        assertSearchResponse(client().prepareSearch("index").setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("foo", "baz"))).get());
-        response = client().admin().indices().prepareStats("index").setQueryCache(true).get();
-        assertCumulativeQueryCacheStats(response);
-        assertThat(response.getTotal().queryCache.getHitCount(), greaterThan(0L));
-        assertThat(response.getTotal().queryCache.getEvictions(), equalTo(0L));
-        assertThat(response.getTotal().queryCache.getMissCount(), greaterThan(0L));
-        assertThat(response.getTotal().queryCache.getCacheSize(), greaterThan(0L));
+        assertBusy(() -> {
+            assertSearchResponse(client().prepareSearch("index").setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("foo", "baz"))).get());
+            IndicesStatsResponse stats = client().admin().indices().prepareStats("index").setQueryCache(true).get();
+            assertCumulativeQueryCacheStats(stats);
+            assertThat(stats.getTotal().queryCache.getHitCount(), greaterThan(0L));
+            assertThat(stats.getTotal().queryCache.getEvictions(), equalTo(0L));
+            assertThat(stats.getTotal().queryCache.getMissCount(), greaterThan(0L));
+            assertThat(stats.getTotal().queryCache.getCacheSize(), greaterThan(0L));
+        });
 
         assertTrue(client().prepareDelete("index", "type", "1").get().isFound());
         assertTrue(client().prepareDelete("index", "type", "2").get().isFound());
@@ -1045,15 +1049,17 @@ public class IndexStatsIT extends ESIntegTestCase {
         indexRandom(true,
                 client().prepareIndex("index", "type", "1").setSource("foo", "bar"),
                 client().prepareIndex("index", "type", "2").setSource("foo", "baz"));
-        assertSearchResponse(client().prepareSearch("index").setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("foo", "baz"))).get());
 
-        response = client().admin().indices().prepareStats("index").setQueryCache(true).get();
-        assertCumulativeQueryCacheStats(response);
-        assertThat(response.getTotal().queryCache.getHitCount(), greaterThan(0L));
-        assertThat(response.getTotal().queryCache.getEvictions(), greaterThan(0L));
-        assertThat(response.getTotal().queryCache.getMissCount(), greaterThan(0L));
-        assertThat(response.getTotal().queryCache.getCacheSize(), greaterThan(0L));
-        assertThat(response.getTotal().queryCache.getMemorySizeInBytes(), greaterThan(0L));
+        assertBusy(() -> {
+            assertSearchResponse(client().prepareSearch("index").setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.matchQuery("foo", "baz"))).get());
+            IndicesStatsResponse stats = client().admin().indices().prepareStats("index").setQueryCache(true).get();
+            assertCumulativeQueryCacheStats(stats);
+            assertThat(stats.getTotal().queryCache.getHitCount(), greaterThan(0L));
+            assertThat(stats.getTotal().queryCache.getEvictions(), greaterThan(0L));
+            assertThat(stats.getTotal().queryCache.getMissCount(), greaterThan(0L));
+            assertThat(stats.getTotal().queryCache.getCacheSize(), greaterThan(0L));
+            assertThat(stats.getTotal().queryCache.getMemorySizeInBytes(), greaterThan(0L));
+        });
 
         assertAllSuccessful(client().admin().indices().prepareClearCache("index").setQueryCache(true).get());
         response = client().admin().indices().prepareStats("index").setQueryCache(true).get();

From 7ba5bceebe40779961c1c56700bac01b1b0538fb Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Wed, 22 Jun 2016 18:31:39 +0200
Subject: [PATCH 1042/1311] Add a MultiTermAwareComponent marker interface to
 analysis factories. #19028

This is the same as what Lucene does for its analysis factories, and we hawe
tests that make sure that the elasticsearch factories are in sync with
Lucene's. This is a first step to move forward on #9978 and #18064.
---
 .../ASCIIFoldingTokenFilterFactory.java       |   7 +-
 .../ArabicNormalizationFilterFactory.java     |   7 +-
 .../index/analysis/CJKWidthFilterFactory.java |   7 +-
 .../analysis/DecimalDigitFilterFactory.java   |   7 +-
 .../analysis/ElisionTokenFilterFactory.java   |   7 +-
 .../GermanNormalizationFilterFactory.java     |   6 +-
 .../HindiNormalizationFilterFactory.java      |   6 +-
 .../IndicNormalizationFilterFactory.java      |   6 +-
 .../analysis/LowerCaseTokenFilterFactory.java |   7 +-
 .../analysis/LowerCaseTokenizerFactory.java   |   7 +-
 .../analysis/MappingCharFilterFactory.java    |   7 +-
 .../analysis/MultiTermAwareComponent.java     |  30 ++
 .../PersianNormalizationFilterFactory.java    |   6 +-
 .../SerbianNormalizationFilterFactory.java    |   7 +-
 .../SoraniNormalizationFilterFactory.java     |   7 +-
 .../analysis/UpperCaseTokenFilterFactory.java |   7 +-
 .../index/analysis/AnalysisFactoryTests.java  | 190 +---------
 .../IcuFoldingTokenFilterFactory.java         |   7 +-
 .../IcuNormalizerCharFilterFactory.java       |   7 +-
 .../IcuNormalizerTokenFilterFactory.java      |   7 +-
 .../IcuTransformTokenFilterFactory.java       |   7 +-
 .../analysis/AnalysisICUFactoryTests.java     |  52 +++
 ...uromojiIterationMarkCharFilterFactory.java |   7 +-
 .../AnalysisKuromojiFactoryTests.java         |  54 +++
 .../AnalysisPhoneticFactoryTests.java         |  37 ++
 .../AnalysisSmartChineseFactoryTests.java     |  36 ++
 .../analysis/AnalysisPolishFactoryTests.java  |  37 ++
 .../AnalysisFactoryTestCase.java              | 325 ++++++++++++++++++
 28 files changed, 690 insertions(+), 207 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/index/analysis/MultiTermAwareComponent.java
 create mode 100644 plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/AnalysisICUFactoryTests.java
 create mode 100644 plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/AnalysisKuromojiFactoryTests.java
 create mode 100644 plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/AnalysisPhoneticFactoryTests.java
 create mode 100644 plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/AnalysisSmartChineseFactoryTests.java
 create mode 100644 plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishFactoryTests.java
 create mode 100644 test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java

diff --git a/core/src/main/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactory.java
index 2e6ba6377a9..b7417b26374 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/ASCIIFoldingTokenFilterFactory.java
@@ -29,7 +29,7 @@ import org.elasticsearch.index.IndexSettings;
 /**
  * Factory for ASCIIFoldingFilter.
  */
-public class ASCIIFoldingTokenFilterFactory extends AbstractTokenFilterFactory {
+public class ASCIIFoldingTokenFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
     public static ParseField PRESERVE_ORIGINAL = new ParseField("preserve_original");
     public static boolean DEFAULT_PRESERVE_ORIGINAL = false;
 
@@ -44,4 +44,9 @@ public class ASCIIFoldingTokenFilterFactory extends AbstractTokenFilterFactory {
     public TokenStream create(TokenStream tokenStream) {
         return new ASCIIFoldingFilter(tokenStream, preserveOriginal);
     }
+
+    @Override
+    public Object getMultiTermComponent() {
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/ArabicNormalizationFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/ArabicNormalizationFilterFactory.java
index 7c274a4ef8c..265e050efee 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/ArabicNormalizationFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/ArabicNormalizationFilterFactory.java
@@ -27,7 +27,7 @@ import org.elasticsearch.index.IndexSettings;
 /**
  *
  */
-public class ArabicNormalizationFilterFactory extends AbstractTokenFilterFactory {
+public class ArabicNormalizationFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
 
     public ArabicNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
@@ -37,4 +37,9 @@ public class ArabicNormalizationFilterFactory extends AbstractTokenFilterFactory
     public TokenStream create(TokenStream tokenStream) {
         return new ArabicNormalizationFilter(tokenStream);
     }
+
+    @Override
+    public Object getMultiTermComponent() {
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/CJKWidthFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/CJKWidthFilterFactory.java
index d7b6ab02511..44cd11c81ed 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/CJKWidthFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/CJKWidthFilterFactory.java
@@ -25,7 +25,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
 
-public final class CJKWidthFilterFactory extends AbstractTokenFilterFactory {
+public final class CJKWidthFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
 
     public CJKWidthFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
@@ -36,4 +36,9 @@ public final class CJKWidthFilterFactory extends AbstractTokenFilterFactory {
         return new CJKWidthFilter(tokenStream);
     }
 
+    @Override
+    public Object getMultiTermComponent() {
+        return this;
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/DecimalDigitFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/DecimalDigitFilterFactory.java
index a9ca009e373..b4806ab7073 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/DecimalDigitFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/DecimalDigitFilterFactory.java
@@ -28,7 +28,7 @@ import org.elasticsearch.index.IndexSettings;
 /**
  * Factory for {@link DecimalDigitFilter}
  */
-public final class DecimalDigitFilterFactory extends AbstractTokenFilterFactory {
+public final class DecimalDigitFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
 
     public DecimalDigitFilterFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
         super(indexSettings, name, settings);
@@ -38,4 +38,9 @@ public final class DecimalDigitFilterFactory extends AbstractTokenFilterFactory
     public TokenStream create(TokenStream tokenStream) {
         return new DecimalDigitFilter(tokenStream);
     }
+
+    @Override
+    public Object getMultiTermComponent() {
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/ElisionTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/ElisionTokenFilterFactory.java
index 4ca0e399f18..2291e199b3a 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/ElisionTokenFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/ElisionTokenFilterFactory.java
@@ -29,7 +29,7 @@ import org.elasticsearch.index.IndexSettings;
 /**
  *
  */
-public class ElisionTokenFilterFactory extends AbstractTokenFilterFactory {
+public class ElisionTokenFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
 
     private final CharArraySet articles;
 
@@ -42,4 +42,9 @@ public class ElisionTokenFilterFactory extends AbstractTokenFilterFactory {
     public TokenStream create(TokenStream tokenStream) {
         return new ElisionFilter(tokenStream, articles);
     }
+
+    @Override
+    public Object getMultiTermComponent() {
+        return this;
+    }
 }
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/GermanNormalizationFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/GermanNormalizationFilterFactory.java
index 840a70ba265..dcdcb4882e7 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/GermanNormalizationFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/GermanNormalizationFilterFactory.java
@@ -27,7 +27,7 @@ import org.elasticsearch.index.IndexSettings;
 /**
  * Factory for {@link GermanNormalizationFilter}
  */
-public class GermanNormalizationFilterFactory extends AbstractTokenFilterFactory {
+public class GermanNormalizationFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
 
     public GermanNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
@@ -38,4 +38,8 @@ public class GermanNormalizationFilterFactory extends AbstractTokenFilterFactory
         return new GermanNormalizationFilter(tokenStream);
     }
 
+    @Override
+    public Object getMultiTermComponent() {
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/HindiNormalizationFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/HindiNormalizationFilterFactory.java
index 80c772c5010..a957c3dd15c 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/HindiNormalizationFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/HindiNormalizationFilterFactory.java
@@ -27,7 +27,7 @@ import org.elasticsearch.index.IndexSettings;
 /**
  * Factory for {@link HindiNormalizationFilter}
  */
-public class HindiNormalizationFilterFactory extends AbstractTokenFilterFactory {
+public class HindiNormalizationFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
 
     public HindiNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
@@ -38,4 +38,8 @@ public class HindiNormalizationFilterFactory extends AbstractTokenFilterFactory
         return new HindiNormalizationFilter(tokenStream);
     }
 
+    @Override
+    public Object getMultiTermComponent() {
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/IndicNormalizationFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/IndicNormalizationFilterFactory.java
index f0c82500ecc..67b51f02dba 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/IndicNormalizationFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/IndicNormalizationFilterFactory.java
@@ -27,7 +27,7 @@ import org.elasticsearch.index.IndexSettings;
 /**
  * Factory for {@link IndicNormalizationFilter}
  */
-public class IndicNormalizationFilterFactory extends AbstractTokenFilterFactory {
+public class IndicNormalizationFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
 
     public IndicNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
@@ -38,4 +38,8 @@ public class IndicNormalizationFilterFactory extends AbstractTokenFilterFactory
         return new IndicNormalizationFilter(tokenStream);
     }
 
+    @Override
+    public Object getMultiTermComponent() {
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/LowerCaseTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/LowerCaseTokenFilterFactory.java
index e608e53c089..dcad6960ba4 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/LowerCaseTokenFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/LowerCaseTokenFilterFactory.java
@@ -37,7 +37,7 @@ import org.elasticsearch.index.IndexSettings;
  *   <li>turkish: {@link TurkishLowerCaseFilter}
  * </ul>
  */
-public class LowerCaseTokenFilterFactory extends AbstractTokenFilterFactory {
+public class LowerCaseTokenFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
 
     private final String lang;
 
@@ -60,6 +60,11 @@ public class LowerCaseTokenFilterFactory extends AbstractTokenFilterFactory {
             throw new IllegalArgumentException("language [" + lang + "] not support for lower case");
         }
     }
+
+    @Override
+    public Object getMultiTermComponent() {
+        return this;
+    }
 }
 
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/LowerCaseTokenizerFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/LowerCaseTokenizerFactory.java
index 3de1c893141..961307f7015 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/LowerCaseTokenizerFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/LowerCaseTokenizerFactory.java
@@ -28,7 +28,7 @@ import org.elasticsearch.index.IndexSettings;
 /**
  *
  */
-public class LowerCaseTokenizerFactory extends AbstractTokenizerFactory {
+public class LowerCaseTokenizerFactory extends AbstractTokenizerFactory implements MultiTermAwareComponent {
 
     public LowerCaseTokenizerFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
@@ -38,4 +38,9 @@ public class LowerCaseTokenizerFactory extends AbstractTokenizerFactory {
     public Tokenizer create() {
         return new LowerCaseTokenizer();
     }
+
+    @Override
+    public Object getMultiTermComponent() {
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/MappingCharFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/MappingCharFilterFactory.java
index 4efce1935e1..c5a4e4bbdcc 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/MappingCharFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/MappingCharFilterFactory.java
@@ -30,7 +30,7 @@ import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-public class MappingCharFilterFactory extends AbstractCharFilterFactory {
+public class MappingCharFilterFactory extends AbstractCharFilterFactory implements MultiTermAwareComponent {
 
     private final NormalizeCharMap normMap;
 
@@ -114,4 +114,9 @@ public class MappingCharFilterFactory extends AbstractCharFilterFactory {
         }
         return new String(out, 0, writePos);
     }
+
+    @Override
+    public Object getMultiTermComponent() {
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/MultiTermAwareComponent.java b/core/src/main/java/org/elasticsearch/index/analysis/MultiTermAwareComponent.java
new file mode 100644
index 00000000000..0662352e79c
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/analysis/MultiTermAwareComponent.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+/** Elasticsearch counterpart of {@link org.apache.lucene.analysis.util.MultiTermAwareComponent}. */
+public interface MultiTermAwareComponent {
+
+    /** Returns an analysis component to handle analysis if multi-term queries.
+     * The returned component must be a TokenizerFactory, TokenFilterFactory or CharFilterFactory.
+     */
+    public Object getMultiTermComponent();
+
+}
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/PersianNormalizationFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/PersianNormalizationFilterFactory.java
index 94538260000..4f5751d985d 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/PersianNormalizationFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/PersianNormalizationFilterFactory.java
@@ -27,7 +27,7 @@ import org.elasticsearch.index.IndexSettings;
 /**
  *
  */
-public class PersianNormalizationFilterFactory extends AbstractTokenFilterFactory {
+public class PersianNormalizationFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
 
     public PersianNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
@@ -38,4 +38,8 @@ public class PersianNormalizationFilterFactory extends AbstractTokenFilterFactor
         return new PersianNormalizationFilter(tokenStream);
     }
 
+    @Override
+    public Object getMultiTermComponent() {
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/SerbianNormalizationFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/SerbianNormalizationFilterFactory.java
index 41f571922dd..8fc6052247c 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/SerbianNormalizationFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/SerbianNormalizationFilterFactory.java
@@ -27,7 +27,7 @@ import org.elasticsearch.index.IndexSettings;
 /**
  *
  */
-public class SerbianNormalizationFilterFactory extends AbstractTokenFilterFactory {
+public class SerbianNormalizationFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
 
     public SerbianNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
@@ -37,4 +37,9 @@ public class SerbianNormalizationFilterFactory extends AbstractTokenFilterFactor
     public TokenStream create(TokenStream tokenStream) {
         return new SerbianNormalizationFilter(tokenStream);
     }
+
+    @Override
+    public Object getMultiTermComponent() {
+        return this;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/SoraniNormalizationFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/SoraniNormalizationFilterFactory.java
index e5a67f241ee..afe2f51ddd8 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/SoraniNormalizationFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/SoraniNormalizationFilterFactory.java
@@ -27,7 +27,7 @@ import org.elasticsearch.index.IndexSettings;
 /**
  * Factory for {@link SoraniNormalizationFilter}
  */
-public class SoraniNormalizationFilterFactory extends AbstractTokenFilterFactory {
+public class SoraniNormalizationFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
 
     public SoraniNormalizationFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
@@ -38,4 +38,9 @@ public class SoraniNormalizationFilterFactory extends AbstractTokenFilterFactory
         return new SoraniNormalizationFilter(tokenStream);
     }
 
+    @Override
+    public Object getMultiTermComponent() {
+        return this;
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/UpperCaseTokenFilterFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/UpperCaseTokenFilterFactory.java
index 0784c3b5db5..c2074cb9393 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/UpperCaseTokenFilterFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/UpperCaseTokenFilterFactory.java
@@ -28,7 +28,7 @@ import org.elasticsearch.index.IndexSettings;
 /**
  *
  */
-public class UpperCaseTokenFilterFactory extends AbstractTokenFilterFactory {
+public class UpperCaseTokenFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
 
     public UpperCaseTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
         super(indexSettings, name, settings);
@@ -38,6 +38,11 @@ public class UpperCaseTokenFilterFactory extends AbstractTokenFilterFactory {
     public TokenStream create(TokenStream tokenStream) {
         return new UpperCaseFilter(tokenStream);
     }
+
+    @Override
+    public Object getMultiTermComponent() {
+        return this;
+    }
 }
 
 
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisFactoryTests.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisFactoryTests.java
index afb34cda8a0..6893fda75b8 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisFactoryTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisFactoryTests.java
@@ -19,192 +19,8 @@
 
 package org.elasticsearch.index.analysis;
 
-import org.elasticsearch.index.analysis.compound.DictionaryCompoundWordTokenFilterFactory;
-import org.elasticsearch.index.analysis.compound.HyphenationCompoundWordTokenFilterFactory;
-import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.AnalysisFactoryTestCase;
 
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
-/** 
- * Alerts us if new analyzers are added to lucene, so we don't miss them.
- * <p>
- * If we don't want to expose one for a specific reason, just map it to Void
- */
-public class AnalysisFactoryTests extends ESTestCase {
-    
-    static final Map<String,Class<?>> KNOWN_TOKENIZERS = new HashMap<String,Class<?>>() {{
-        // deprecated ones, we dont care about these
-        put("arabicletter",  Deprecated.class);
-        put("chinese",       Deprecated.class);
-        put("cjk",           Deprecated.class);
-        put("russianletter", Deprecated.class);
-        
-        // exposed in ES
-        put("classic",       ClassicTokenizerFactory.class);
-        put("edgengram",     EdgeNGramTokenizerFactory.class);
-        put("keyword",       KeywordTokenizerFactory.class);
-        put("letter",        LetterTokenizerFactory.class);
-        put("lowercase",     LowerCaseTokenizerFactory.class);
-        put("ngram",         NGramTokenizerFactory.class);
-        put("pathhierarchy", PathHierarchyTokenizerFactory.class);
-        put("pattern",       PatternTokenizerFactory.class);
-        put("standard",      StandardTokenizerFactory.class);
-        put("thai",          ThaiTokenizerFactory.class);
-        put("uax29urlemail", UAX29URLEmailTokenizerFactory.class);
-        put("whitespace",    WhitespaceTokenizerFactory.class);
-        
-        // this one "seems to mess up offsets". probably shouldn't be a tokenizer...
-        put("wikipedia",     Void.class);
-    }};
-    
-    public void testTokenizers() {
-        Set<String> missing = new TreeSet<String>(org.apache.lucene.analysis.util.TokenizerFactory.availableTokenizers());
-        missing.removeAll(KNOWN_TOKENIZERS.keySet());
-        assertTrue("new tokenizers found, please update KNOWN_TOKENIZERS: " + missing.toString(), missing.isEmpty());
-    }
-    
-    static final Map<String,Class<?>> KNOWN_TOKENFILTERS = new HashMap<String,Class<?>>() {{
-        // deprecated ones, we dont care about these
-        put("chinese",                Deprecated.class);
-        put("collationkey",           Deprecated.class);
-        put("position",               Deprecated.class);
-        put("thaiword",               Deprecated.class);
-        
-        
-        // exposed in ES
-        put("apostrophe",                ApostropheFilterFactory.class);
-        put("arabicnormalization",       ArabicNormalizationFilterFactory.class);
-        put("arabicstem",                ArabicStemTokenFilterFactory.class);
-        put("asciifolding",              ASCIIFoldingTokenFilterFactory.class);
-        put("brazilianstem",             BrazilianStemTokenFilterFactory.class);
-        put("bulgarianstem",             StemmerTokenFilterFactory.class);
-        put("cjkbigram",                 CJKBigramFilterFactory.class);
-        put("cjkwidth",                  CJKWidthFilterFactory.class);
-        put("classic",                   ClassicFilterFactory.class);
-        put("commongrams",               CommonGramsTokenFilterFactory.class);
-        put("commongramsquery",          CommonGramsTokenFilterFactory.class);
-        put("czechstem",                 CzechStemTokenFilterFactory.class);
-        put("decimaldigit",              DecimalDigitFilterFactory.class);
-        put("delimitedpayload",          DelimitedPayloadTokenFilterFactory.class);
-        put("dictionarycompoundword",    DictionaryCompoundWordTokenFilterFactory.class);
-        put("edgengram",                 EdgeNGramTokenFilterFactory.class);
-        put("elision",                   ElisionTokenFilterFactory.class);
-        put("englishminimalstem",        StemmerTokenFilterFactory.class);
-        put("englishpossessive",         StemmerTokenFilterFactory.class);
-        put("finnishlightstem",          StemmerTokenFilterFactory.class);
-        put("frenchlightstem",           StemmerTokenFilterFactory.class);
-        put("frenchminimalstem",         StemmerTokenFilterFactory.class);
-        put("galicianminimalstem",       StemmerTokenFilterFactory.class);
-        put("galicianstem",              StemmerTokenFilterFactory.class);
-        put("germanstem",                GermanStemTokenFilterFactory.class);
-        put("germanlightstem",           StemmerTokenFilterFactory.class);
-        put("germanminimalstem",         StemmerTokenFilterFactory.class);
-        put("germannormalization",       GermanNormalizationFilterFactory.class);
-        put("greeklowercase",            LowerCaseTokenFilterFactory.class);
-        put("greekstem",                 StemmerTokenFilterFactory.class);
-        put("hindinormalization",        HindiNormalizationFilterFactory.class);
-        put("hindistem",                 StemmerTokenFilterFactory.class);
-        put("hungarianlightstem",        StemmerTokenFilterFactory.class);
-        put("hunspellstem",              HunspellTokenFilterFactory.class);
-        put("hyphenationcompoundword",   HyphenationCompoundWordTokenFilterFactory.class);
-        put("indicnormalization",        IndicNormalizationFilterFactory.class);
-        put("irishlowercase",            LowerCaseTokenFilterFactory.class);
-        put("indonesianstem",            StemmerTokenFilterFactory.class);
-        put("italianlightstem",          StemmerTokenFilterFactory.class);
-        put("keepword",                  KeepWordFilterFactory.class);
-        put("keywordmarker",             KeywordMarkerTokenFilterFactory.class);
-        put("kstem",                     KStemTokenFilterFactory.class);
-        put("latvianstem",               StemmerTokenFilterFactory.class);
-        put("length",                    LengthTokenFilterFactory.class);
-        put("limittokencount",           LimitTokenCountFilterFactory.class);
-        put("lowercase",                 LowerCaseTokenFilterFactory.class);
-        put("ngram",                     NGramTokenFilterFactory.class);
-        put("norwegianlightstem",        StemmerTokenFilterFactory.class);
-        put("norwegianminimalstem",      StemmerTokenFilterFactory.class);
-        put("patterncapturegroup",       PatternCaptureGroupTokenFilterFactory.class);
-        put("patternreplace",            PatternReplaceTokenFilterFactory.class);
-        put("persiannormalization",      PersianNormalizationFilterFactory.class);
-        put("porterstem",                PorterStemTokenFilterFactory.class);
-        put("portuguesestem",            StemmerTokenFilterFactory.class);
-        put("portugueselightstem",       StemmerTokenFilterFactory.class);
-        put("portugueseminimalstem",     StemmerTokenFilterFactory.class);
-        put("reversestring",             ReverseTokenFilterFactory.class);
-        put("russianlightstem",          StemmerTokenFilterFactory.class);
-        put("scandinavianfolding",       ScandinavianFoldingFilterFactory.class);
-        put("scandinaviannormalization", ScandinavianNormalizationFilterFactory.class);
-        put("serbiannormalization",      SerbianNormalizationFilterFactory.class);
-        put("shingle",                   ShingleTokenFilterFactory.class);
-        put("snowballporter",            SnowballTokenFilterFactory.class);
-        put("soraninormalization",       SoraniNormalizationFilterFactory.class);
-        put("soranistem",                StemmerTokenFilterFactory.class);
-        put("spanishlightstem",          StemmerTokenFilterFactory.class);
-        put("standard",                  StandardTokenFilterFactory.class);
-        put("stemmeroverride",           StemmerOverrideTokenFilterFactory.class);
-        put("stop",                      StopTokenFilterFactory.class);
-        put("swedishlightstem",          StemmerTokenFilterFactory.class);
-        put("synonym",                   SynonymTokenFilterFactory.class);
-        put("trim",                      TrimTokenFilterFactory.class);
-        put("truncate",                  TruncateTokenFilterFactory.class);
-        put("turkishlowercase",          LowerCaseTokenFilterFactory.class);
-        put("type",                      KeepTypesFilterFactory.class);
-        put("uppercase",                 UpperCaseTokenFilterFactory.class);
-        put("worddelimiter",             WordDelimiterTokenFilterFactory.class);
-                
-        // TODO: these tokenfilters are not yet exposed: useful?
-
-        // suggest stop
-        put("suggeststop",               Void.class);
-        // capitalizes tokens
-        put("capitalization",            Void.class);
-        // like length filter (but codepoints)
-        put("codepointcount",            Void.class);
-        // puts hyphenated words back together
-        put("hyphenatedwords",           Void.class);
-        // repeats anything marked as keyword
-        put("keywordrepeat",             Void.class);
-        // like limittokencount, but by offset
-        put("limittokenoffset",          Void.class);
-        // like limittokencount, but by position
-        put("limittokenposition",        Void.class);
-        // ???
-        put("numericpayload",            Void.class);
-        // removes duplicates at the same position (this should be used by the existing factory)
-        put("removeduplicates",          Void.class);
-        // ???
-        put("tokenoffsetpayload",        Void.class);
-        // puts the type into the payload
-        put("typeaspayload",             Void.class);
-        // fingerprint
-        put("fingerprint",               Void.class);
-        // for tee-sinks
-        put("daterecognizer",            Void.class);
-    }};
-    
-    public void testTokenFilters() {
-        Set<String> missing = new TreeSet<String>(org.apache.lucene.analysis.util.TokenFilterFactory.availableTokenFilters());
-        missing.removeAll(KNOWN_TOKENFILTERS.keySet());
-        assertTrue("new tokenfilters found, please update KNOWN_TOKENFILTERS: " + missing.toString(), missing.isEmpty());
-    }
-    
-    static final Map<String,Class<?>> KNOWN_CHARFILTERS = new HashMap<String,Class<?>>() {{        
-        // exposed in ES
-        put("htmlstrip",      HtmlStripCharFilterFactory.class);
-        put("mapping",        MappingCharFilterFactory.class);
-        put("patternreplace", PatternReplaceCharFilterFactory.class);
-                
-        // TODO: these charfilters are not yet exposed: useful?
-        // handling of zwnj for persian
-        put("persian",        Void.class);
-    }};
-    
-    public void testCharFilters() {
-        Set<String> missing = new TreeSet<String>(org.apache.lucene.analysis.util.CharFilterFactory.availableCharFilters());
-        missing.removeAll(KNOWN_CHARFILTERS.keySet());
-        assertTrue("new charfilters found, please update KNOWN_CHARFILTERS: " + missing.toString(), missing.isEmpty());
-    }
-   
-    
+public class AnalysisFactoryTests extends AnalysisFactoryTestCase {
+    // tests are inherited
 }
diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuFoldingTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuFoldingTokenFilterFactory.java
index b31502cdd7d..5fd3199e99a 100644
--- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuFoldingTokenFilterFactory.java
+++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuFoldingTokenFilterFactory.java
@@ -40,7 +40,7 @@ import org.elasticsearch.index.IndexSettings;
  *
  * @author kimchy (shay.banon)
  */
-public class IcuFoldingTokenFilterFactory extends AbstractTokenFilterFactory {
+public class IcuFoldingTokenFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
     private final String unicodeSetFilter;
 
     public IcuFoldingTokenFilterFactory(IndexSettings indexSettings, Environment environment, String name, Settings settings) {
@@ -67,4 +67,9 @@ public class IcuFoldingTokenFilterFactory extends AbstractTokenFilterFactory {
             return new ICUFoldingFilter(tokenStream);
         }
     }
+
+    @Override
+    public Object getMultiTermComponent() {
+        return this;
+    }
 }
diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerCharFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerCharFilterFactory.java
index 02f9b5a3371..72bc45a0232 100644
--- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerCharFilterFactory.java
+++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerCharFilterFactory.java
@@ -34,7 +34,7 @@ import java.io.Reader;
  * <p>The <tt>name</tt> can be used to provide the type of normalization to perform.</p>
  * <p>The <tt>mode</tt> can be used to provide 'compose' or 'decompose'. Default is compose.</p>
  */
-public class IcuNormalizerCharFilterFactory extends AbstractCharFilterFactory {
+public class IcuNormalizerCharFilterFactory extends AbstractCharFilterFactory implements MultiTermAwareComponent {
 
     private final String name;
 
@@ -55,4 +55,9 @@ public class IcuNormalizerCharFilterFactory extends AbstractCharFilterFactory {
     public Reader create(Reader reader) {
         return new ICUNormalizer2CharFilter(reader, normalizer);
     }
+
+    @Override
+    public Object getMultiTermComponent() {
+        return this;
+    }
 }
diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerTokenFilterFactory.java
index 4833e887153..2632958d203 100644
--- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerTokenFilterFactory.java
+++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuNormalizerTokenFilterFactory.java
@@ -32,7 +32,7 @@ import org.elasticsearch.index.IndexSettings;
  *
  *
  */
-public class IcuNormalizerTokenFilterFactory extends AbstractTokenFilterFactory {
+public class IcuNormalizerTokenFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
 
     private final String name;
 
@@ -45,4 +45,9 @@ public class IcuNormalizerTokenFilterFactory extends AbstractTokenFilterFactory
     public TokenStream create(TokenStream tokenStream) {
         return new org.apache.lucene.analysis.icu.ICUNormalizer2Filter(tokenStream, Normalizer2.getInstance(null, name, Normalizer2.Mode.COMPOSE));
     }
+
+    @Override
+    public Object getMultiTermComponent() {
+        return this;
+    }
 }
diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTransformTokenFilterFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTransformTokenFilterFactory.java
index f145ad4ae30..9cc42e726a5 100644
--- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTransformTokenFilterFactory.java
+++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTransformTokenFilterFactory.java
@@ -29,7 +29,7 @@ import org.elasticsearch.index.IndexSettings;
 
 /**
  */
-public class IcuTransformTokenFilterFactory extends AbstractTokenFilterFactory {
+public class IcuTransformTokenFilterFactory extends AbstractTokenFilterFactory implements MultiTermAwareComponent {
 
     private final String id;
     private final int dir;
@@ -47,4 +47,9 @@ public class IcuTransformTokenFilterFactory extends AbstractTokenFilterFactory {
     public TokenStream create(TokenStream tokenStream) {
         return new ICUTransformFilter(tokenStream, transliterator);
     }
+
+    @Override
+    public Object getMultiTermComponent() {
+        return this;
+    }
 }
diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/AnalysisICUFactoryTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/AnalysisICUFactoryTests.java
new file mode 100644
index 00000000000..704ca61985a
--- /dev/null
+++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/AnalysisICUFactoryTests.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.elasticsearch.AnalysisFactoryTestCase;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class AnalysisICUFactoryTests extends AnalysisFactoryTestCase {
+
+    @Override
+    protected Map<String, Class<?>> getTokenizers() {
+        Map<String, Class<?>> tokenizers = new HashMap<>(super.getTokenizers());
+        tokenizers.put("icu", IcuTokenizerFactory.class);
+        return tokenizers;
+    }
+
+    @Override
+    protected Map<String, Class<?>> getTokenFilters() {
+        Map<String, Class<?>> filters = new HashMap<>(super.getTokenFilters());
+        filters.put("icufolding", IcuFoldingTokenFilterFactory.class);
+        filters.put("icunormalizer2", IcuNormalizerTokenFilterFactory.class);
+        filters.put("icutransform", IcuTransformTokenFilterFactory.class);
+        return filters;
+    }
+
+    @Override
+    protected Map<String, Class<?>> getCharFilters() {
+        Map<String, Class<?>> filters = new HashMap<>(super.getCharFilters());
+        filters.put("icunormalizer2", IcuNormalizerCharFilterFactory.class);
+        return filters;
+    }
+
+}
diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiIterationMarkCharFilterFactory.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiIterationMarkCharFilterFactory.java
index a1220dba2be..836dbbdfae2 100644
--- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiIterationMarkCharFilterFactory.java
+++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/index/analysis/KuromojiIterationMarkCharFilterFactory.java
@@ -26,7 +26,7 @@ import org.elasticsearch.index.IndexSettings;
 
 import java.io.Reader;
 
-public class KuromojiIterationMarkCharFilterFactory extends AbstractCharFilterFactory {
+public class KuromojiIterationMarkCharFilterFactory extends AbstractCharFilterFactory implements MultiTermAwareComponent {
 
     private final boolean normalizeKanji;
     private final boolean normalizeKana;
@@ -41,4 +41,9 @@ public class KuromojiIterationMarkCharFilterFactory extends AbstractCharFilterFa
     public Reader create(Reader reader) {
         return new JapaneseIterationMarkCharFilter(reader, normalizeKanji, normalizeKana);
     }
+
+    @Override
+    public Object getMultiTermComponent() {
+        return this;
+    }
 }
diff --git a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/AnalysisKuromojiFactoryTests.java b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/AnalysisKuromojiFactoryTests.java
new file mode 100644
index 00000000000..9db7def101e
--- /dev/null
+++ b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/AnalysisKuromojiFactoryTests.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.apache.lucene.analysis.ja.JapaneseTokenizerFactory;
+import org.elasticsearch.AnalysisFactoryTestCase;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class AnalysisKuromojiFactoryTests extends AnalysisFactoryTestCase {
+
+    @Override
+    protected Map<String, Class<?>> getTokenizers() {
+        Map<String, Class<?>> tokenizers = new HashMap<>(super.getTokenizers());
+        tokenizers.put("japanese", JapaneseTokenizerFactory.class);
+        return tokenizers;
+    }
+
+    @Override
+    protected Map<String, Class<?>> getTokenFilters() {
+        Map<String, Class<?>> filters = new HashMap<>(super.getTokenFilters());
+        filters.put("japanesebaseform", KuromojiBaseFormFilterFactory.class);
+        filters.put("japanesepartofspeechstop", KuromojiPartOfSpeechFilterFactory.class);
+        filters.put("japanesereadingform", KuromojiReadingFormFilterFactory.class);
+        filters.put("japanesekatakanastem", KuromojiKatakanaStemmerFactory.class);
+        filters.put("japanesenumber", KuromojiNumberFilterFactory.class);
+        return filters;
+    }
+
+    @Override
+    protected Map<String, Class<?>> getCharFilters() {
+        Map<String, Class<?>> filters = new HashMap<>(super.getCharFilters());
+        filters.put("japaneseiterationmark", KuromojiIterationMarkCharFilterFactory.class);
+        return filters;
+    }
+}
diff --git a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/AnalysisPhoneticFactoryTests.java b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/AnalysisPhoneticFactoryTests.java
new file mode 100644
index 00000000000..0546fb468c9
--- /dev/null
+++ b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/AnalysisPhoneticFactoryTests.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.elasticsearch.AnalysisFactoryTestCase;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class AnalysisPhoneticFactoryTests extends AnalysisFactoryTestCase {
+
+    @Override
+    protected Map<String, Class<?>> getTokenFilters() {
+        Map<String, Class<?>> filters = new HashMap<>(super.getTokenFilters());
+        filters.put("beidermorse", PhoneticTokenFilterFactory.class);
+        filters.put("doublemetaphone", PhoneticTokenFilterFactory.class);
+        filters.put("phonetic", PhoneticTokenFilterFactory.class);
+        return filters;
+    }
+}
diff --git a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/AnalysisSmartChineseFactoryTests.java b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/AnalysisSmartChineseFactoryTests.java
new file mode 100644
index 00000000000..d8aad322dcb
--- /dev/null
+++ b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/AnalysisSmartChineseFactoryTests.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.elasticsearch.AnalysisFactoryTestCase;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class AnalysisSmartChineseFactoryTests extends AnalysisFactoryTestCase {
+
+    @Override
+    protected Map<String, Class<?>> getTokenizers() {
+        Map<String, Class<?>> tokenizers = new HashMap<>(super.getTokenizers());
+        tokenizers.put("hmmchinese", SmartChineseTokenizerTokenizerFactory.class);
+        return tokenizers;
+    }
+
+}
diff --git a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishFactoryTests.java b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishFactoryTests.java
new file mode 100644
index 00000000000..abf739d010a
--- /dev/null
+++ b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/AnalysisPolishFactoryTests.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.analysis;
+
+import org.elasticsearch.AnalysisFactoryTestCase;
+import org.elasticsearch.index.analysis.pl.PolishStemTokenFilterFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class AnalysisPolishFactoryTests extends AnalysisFactoryTestCase {
+
+    @Override
+    protected Map<String, Class<?>> getTokenFilters() {
+        Map<String, Class<?>> filters = new HashMap<>(super.getTokenFilters());
+        filters.put("stempelpolishstem", PolishStemTokenFilterFactory.class);
+        return filters;
+    }
+
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java
new file mode 100644
index 00000000000..35ef0868eba
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/AnalysisFactoryTestCase.java
@@ -0,0 +1,325 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch;
+
+import org.elasticsearch.common.collect.MapBuilder;
+import org.elasticsearch.index.analysis.ASCIIFoldingTokenFilterFactory;
+import org.elasticsearch.index.analysis.ApostropheFilterFactory;
+import org.elasticsearch.index.analysis.ArabicNormalizationFilterFactory;
+import org.elasticsearch.index.analysis.ArabicStemTokenFilterFactory;
+import org.elasticsearch.index.analysis.BrazilianStemTokenFilterFactory;
+import org.elasticsearch.index.analysis.CJKBigramFilterFactory;
+import org.elasticsearch.index.analysis.CJKWidthFilterFactory;
+import org.elasticsearch.index.analysis.ClassicFilterFactory;
+import org.elasticsearch.index.analysis.ClassicTokenizerFactory;
+import org.elasticsearch.index.analysis.CommonGramsTokenFilterFactory;
+import org.elasticsearch.index.analysis.CzechStemTokenFilterFactory;
+import org.elasticsearch.index.analysis.DecimalDigitFilterFactory;
+import org.elasticsearch.index.analysis.DelimitedPayloadTokenFilterFactory;
+import org.elasticsearch.index.analysis.EdgeNGramTokenFilterFactory;
+import org.elasticsearch.index.analysis.EdgeNGramTokenizerFactory;
+import org.elasticsearch.index.analysis.ElisionTokenFilterFactory;
+import org.elasticsearch.index.analysis.GermanNormalizationFilterFactory;
+import org.elasticsearch.index.analysis.GermanStemTokenFilterFactory;
+import org.elasticsearch.index.analysis.HindiNormalizationFilterFactory;
+import org.elasticsearch.index.analysis.HtmlStripCharFilterFactory;
+import org.elasticsearch.index.analysis.HunspellTokenFilterFactory;
+import org.elasticsearch.index.analysis.IndicNormalizationFilterFactory;
+import org.elasticsearch.index.analysis.KStemTokenFilterFactory;
+import org.elasticsearch.index.analysis.KeepTypesFilterFactory;
+import org.elasticsearch.index.analysis.KeepWordFilterFactory;
+import org.elasticsearch.index.analysis.KeywordMarkerTokenFilterFactory;
+import org.elasticsearch.index.analysis.KeywordTokenizerFactory;
+import org.elasticsearch.index.analysis.LengthTokenFilterFactory;
+import org.elasticsearch.index.analysis.LetterTokenizerFactory;
+import org.elasticsearch.index.analysis.LimitTokenCountFilterFactory;
+import org.elasticsearch.index.analysis.LowerCaseTokenFilterFactory;
+import org.elasticsearch.index.analysis.LowerCaseTokenizerFactory;
+import org.elasticsearch.index.analysis.MappingCharFilterFactory;
+import org.elasticsearch.index.analysis.MultiTermAwareComponent;
+import org.elasticsearch.index.analysis.NGramTokenFilterFactory;
+import org.elasticsearch.index.analysis.NGramTokenizerFactory;
+import org.elasticsearch.index.analysis.PathHierarchyTokenizerFactory;
+import org.elasticsearch.index.analysis.PatternCaptureGroupTokenFilterFactory;
+import org.elasticsearch.index.analysis.PatternReplaceCharFilterFactory;
+import org.elasticsearch.index.analysis.PatternReplaceTokenFilterFactory;
+import org.elasticsearch.index.analysis.PatternTokenizerFactory;
+import org.elasticsearch.index.analysis.PersianNormalizationFilterFactory;
+import org.elasticsearch.index.analysis.PorterStemTokenFilterFactory;
+import org.elasticsearch.index.analysis.ReverseTokenFilterFactory;
+import org.elasticsearch.index.analysis.ScandinavianFoldingFilterFactory;
+import org.elasticsearch.index.analysis.ScandinavianNormalizationFilterFactory;
+import org.elasticsearch.index.analysis.SerbianNormalizationFilterFactory;
+import org.elasticsearch.index.analysis.ShingleTokenFilterFactory;
+import org.elasticsearch.index.analysis.SnowballTokenFilterFactory;
+import org.elasticsearch.index.analysis.SoraniNormalizationFilterFactory;
+import org.elasticsearch.index.analysis.StandardTokenFilterFactory;
+import org.elasticsearch.index.analysis.StandardTokenizerFactory;
+import org.elasticsearch.index.analysis.StemmerOverrideTokenFilterFactory;
+import org.elasticsearch.index.analysis.StemmerTokenFilterFactory;
+import org.elasticsearch.index.analysis.StopTokenFilterFactory;
+import org.elasticsearch.index.analysis.SynonymTokenFilterFactory;
+import org.elasticsearch.index.analysis.ThaiTokenizerFactory;
+import org.elasticsearch.index.analysis.TrimTokenFilterFactory;
+import org.elasticsearch.index.analysis.TruncateTokenFilterFactory;
+import org.elasticsearch.index.analysis.UAX29URLEmailTokenizerFactory;
+import org.elasticsearch.index.analysis.UpperCaseTokenFilterFactory;
+import org.elasticsearch.index.analysis.WhitespaceTokenizerFactory;
+import org.elasticsearch.index.analysis.WordDelimiterTokenFilterFactory;
+import org.elasticsearch.index.analysis.compound.DictionaryCompoundWordTokenFilterFactory;
+import org.elasticsearch.index.analysis.compound.HyphenationCompoundWordTokenFilterFactory;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+/** 
+ * Alerts us if new analyzers are added to lucene, so we don't miss them.
+ * <p>
+ * If we don't want to expose one for a specific reason, just map it to Void.
+ * The deprecated ones can be mapped to Deprecated.class.
+ */
+public class AnalysisFactoryTestCase extends ESTestCase {
+
+    static final Map<String,Class<?>> KNOWN_TOKENIZERS = new MapBuilder<String,Class<?>>()
+        // exposed in ES
+        .put("classic",       ClassicTokenizerFactory.class)
+        .put("edgengram",     EdgeNGramTokenizerFactory.class)
+        .put("keyword",       KeywordTokenizerFactory.class)
+        .put("letter",        LetterTokenizerFactory.class)
+        .put("lowercase",     LowerCaseTokenizerFactory.class)
+        .put("ngram",         NGramTokenizerFactory.class)
+        .put("pathhierarchy", PathHierarchyTokenizerFactory.class)
+        .put("pattern",       PatternTokenizerFactory.class)
+        .put("standard",      StandardTokenizerFactory.class)
+        .put("thai",          ThaiTokenizerFactory.class)
+        .put("uax29urlemail", UAX29URLEmailTokenizerFactory.class)
+        .put("whitespace",    WhitespaceTokenizerFactory.class)
+        
+        // this one "seems to mess up offsets". probably shouldn't be a tokenizer...
+        .put("wikipedia",     Void.class)
+        .immutableMap();
+    
+    static final Map<String,Class<?>> KNOWN_TOKENFILTERS = new MapBuilder<String,Class<?>>()
+        // exposed in ES
+        .put("apostrophe",                ApostropheFilterFactory.class)
+        .put("arabicnormalization",       ArabicNormalizationFilterFactory.class)
+        .put("arabicstem",                ArabicStemTokenFilterFactory.class)
+        .put("asciifolding",              ASCIIFoldingTokenFilterFactory.class)
+        .put("brazilianstem",             BrazilianStemTokenFilterFactory.class)
+        .put("bulgarianstem",             StemmerTokenFilterFactory.class)
+        .put("cjkbigram",                 CJKBigramFilterFactory.class)
+        .put("cjkwidth",                  CJKWidthFilterFactory.class)
+        .put("classic",                   ClassicFilterFactory.class)
+        .put("commongrams",               CommonGramsTokenFilterFactory.class)
+        .put("commongramsquery",          CommonGramsTokenFilterFactory.class)
+        .put("czechstem",                 CzechStemTokenFilterFactory.class)
+        .put("decimaldigit",              DecimalDigitFilterFactory.class)
+        .put("delimitedpayload",          DelimitedPayloadTokenFilterFactory.class)
+        .put("dictionarycompoundword",    DictionaryCompoundWordTokenFilterFactory.class)
+        .put("edgengram",                 EdgeNGramTokenFilterFactory.class)
+        .put("elision",                   ElisionTokenFilterFactory.class)
+        .put("englishminimalstem",        StemmerTokenFilterFactory.class)
+        .put("englishpossessive",         StemmerTokenFilterFactory.class)
+        .put("finnishlightstem",          StemmerTokenFilterFactory.class)
+        .put("frenchlightstem",           StemmerTokenFilterFactory.class)
+        .put("frenchminimalstem",         StemmerTokenFilterFactory.class)
+        .put("galicianminimalstem",       StemmerTokenFilterFactory.class)
+        .put("galicianstem",              StemmerTokenFilterFactory.class)
+        .put("germanstem",                GermanStemTokenFilterFactory.class)
+        .put("germanlightstem",           StemmerTokenFilterFactory.class)
+        .put("germanminimalstem",         StemmerTokenFilterFactory.class)
+        .put("germannormalization",       GermanNormalizationFilterFactory.class)
+        .put("greeklowercase",            LowerCaseTokenFilterFactory.class)
+        .put("greekstem",                 StemmerTokenFilterFactory.class)
+        .put("hindinormalization",        HindiNormalizationFilterFactory.class)
+        .put("hindistem",                 StemmerTokenFilterFactory.class)
+        .put("hungarianlightstem",        StemmerTokenFilterFactory.class)
+        .put("hunspellstem",              HunspellTokenFilterFactory.class)
+        .put("hyphenationcompoundword",   HyphenationCompoundWordTokenFilterFactory.class)
+        .put("indicnormalization",        IndicNormalizationFilterFactory.class)
+        .put("irishlowercase",            LowerCaseTokenFilterFactory.class)
+        .put("indonesianstem",            StemmerTokenFilterFactory.class)
+        .put("italianlightstem",          StemmerTokenFilterFactory.class)
+        .put("keepword",                  KeepWordFilterFactory.class)
+        .put("keywordmarker",             KeywordMarkerTokenFilterFactory.class)
+        .put("kstem",                     KStemTokenFilterFactory.class)
+        .put("latvianstem",               StemmerTokenFilterFactory.class)
+        .put("length",                    LengthTokenFilterFactory.class)
+        .put("limittokencount",           LimitTokenCountFilterFactory.class)
+        .put("lowercase",                 LowerCaseTokenFilterFactory.class)
+        .put("ngram",                     NGramTokenFilterFactory.class)
+        .put("norwegianlightstem",        StemmerTokenFilterFactory.class)
+        .put("norwegianminimalstem",      StemmerTokenFilterFactory.class)
+        .put("patterncapturegroup",       PatternCaptureGroupTokenFilterFactory.class)
+        .put("patternreplace",            PatternReplaceTokenFilterFactory.class)
+        .put("persiannormalization",      PersianNormalizationFilterFactory.class)
+        .put("porterstem",                PorterStemTokenFilterFactory.class)
+        .put("portuguesestem",            StemmerTokenFilterFactory.class)
+        .put("portugueselightstem",       StemmerTokenFilterFactory.class)
+        .put("portugueseminimalstem",     StemmerTokenFilterFactory.class)
+        .put("reversestring",             ReverseTokenFilterFactory.class)
+        .put("russianlightstem",          StemmerTokenFilterFactory.class)
+        .put("scandinavianfolding",       ScandinavianFoldingFilterFactory.class)
+        .put("scandinaviannormalization", ScandinavianNormalizationFilterFactory.class)
+        .put("serbiannormalization",      SerbianNormalizationFilterFactory.class)
+        .put("shingle",                   ShingleTokenFilterFactory.class)
+        .put("snowballporter",            SnowballTokenFilterFactory.class)
+        .put("soraninormalization",       SoraniNormalizationFilterFactory.class)
+        .put("soranistem",                StemmerTokenFilterFactory.class)
+        .put("spanishlightstem",          StemmerTokenFilterFactory.class)
+        .put("standard",                  StandardTokenFilterFactory.class)
+        .put("stemmeroverride",           StemmerOverrideTokenFilterFactory.class)
+        .put("stop",                      StopTokenFilterFactory.class)
+        .put("swedishlightstem",          StemmerTokenFilterFactory.class)
+        .put("synonym",                   SynonymTokenFilterFactory.class)
+        .put("trim",                      TrimTokenFilterFactory.class)
+        .put("truncate",                  TruncateTokenFilterFactory.class)
+        .put("turkishlowercase",          LowerCaseTokenFilterFactory.class)
+        .put("type",                      KeepTypesFilterFactory.class)
+        .put("uppercase",                 UpperCaseTokenFilterFactory.class)
+        .put("worddelimiter",             WordDelimiterTokenFilterFactory.class)
+                
+        // TODO: these tokenfilters are not yet exposed: useful?
+
+        // suggest stop
+        .put("suggeststop",               Void.class)
+        // capitalizes tokens
+        .put("capitalization",            Void.class)
+        // like length filter (but codepoints)
+        .put("codepointcount",            Void.class)
+        // puts hyphenated words back together
+        .put("hyphenatedwords",           Void.class)
+        // repeats anything marked as keyword
+        .put("keywordrepeat",             Void.class)
+        // like limittokencount, but by offset
+        .put("limittokenoffset",          Void.class)
+        // like limittokencount, but by position
+        .put("limittokenposition",        Void.class)
+        // ???
+        .put("numericpayload",            Void.class)
+        // removes duplicates at the same position (this should be used by the existing factory)
+        .put("removeduplicates",          Void.class)
+        // ???
+        .put("tokenoffsetpayload",        Void.class)
+        // puts the type into the payload
+        .put("typeaspayload",             Void.class)
+        // fingerprint
+        .put("fingerprint",               Void.class)
+        // for tee-sinks
+        .put("daterecognizer",            Void.class)
+        .immutableMap();
+    
+    static final Map<String,Class<?>> KNOWN_CHARFILTERS = new MapBuilder<String,Class<?>>()
+        // exposed in ES
+        .put("htmlstrip",      HtmlStripCharFilterFactory.class)
+        .put("mapping",        MappingCharFilterFactory.class)
+        .put("patternreplace", PatternReplaceCharFilterFactory.class)
+                
+        // TODO: these charfilters are not yet exposed: useful?
+        // handling of zwnj for persian
+        .put("persian",        Void.class)
+        .immutableMap();
+
+    protected Map<String, Class<?>> getTokenizers() {
+        return KNOWN_TOKENIZERS;
+    }
+
+    protected Map<String, Class<?>> getTokenFilters() {
+        return KNOWN_TOKENFILTERS;
+    }
+
+    protected Map<String, Class<?>> getCharFilters() {
+        return KNOWN_CHARFILTERS;
+    }
+
+    public void testTokenizers() {
+        Set<String> missing = new TreeSet<String>(org.apache.lucene.analysis.util.TokenizerFactory.availableTokenizers());
+        missing.removeAll(getTokenizers().keySet());
+        assertTrue("new tokenizers found, please update KNOWN_TOKENIZERS: " + missing.toString(), missing.isEmpty());
+    }
+
+    public void testCharFilters() {
+        Set<String> missing = new TreeSet<String>(org.apache.lucene.analysis.util.CharFilterFactory.availableCharFilters());
+        missing.removeAll(getCharFilters().keySet());
+        assertTrue("new charfilters found, please update KNOWN_CHARFILTERS: " + missing.toString(), missing.isEmpty());
+    }
+
+    public void testTokenFilters() {
+        Set<String> missing = new TreeSet<String>(org.apache.lucene.analysis.util.TokenFilterFactory.availableTokenFilters());
+        missing.removeAll(getTokenFilters().keySet());
+        assertTrue("new tokenfilters found, please update KNOWN_TOKENFILTERS: " + missing.toString(), missing.isEmpty());
+    }
+
+    public void testMultiTermAware() {
+        Collection<Class<?>> expected = new HashSet<>();
+        for (Map.Entry<String, Class<?>> entry : getTokenizers().entrySet()) {
+            if (org.apache.lucene.analysis.util.MultiTermAwareComponent.class.isAssignableFrom(
+                    org.apache.lucene.analysis.util.TokenizerFactory.lookupClass(entry.getKey()))) {
+                expected.add(entry.getValue());
+            }
+        }
+        for (Map.Entry<String, Class<?>> entry : getTokenFilters().entrySet()) {
+            if (org.apache.lucene.analysis.util.MultiTermAwareComponent.class.isAssignableFrom(
+                    org.apache.lucene.analysis.util.TokenFilterFactory.lookupClass(entry.getKey()))) {
+                expected.add(entry.getValue());
+            }
+        }
+        for (Map.Entry<String, Class<?>> entry : getCharFilters().entrySet()) {
+            if (org.apache.lucene.analysis.util.MultiTermAwareComponent.class.isAssignableFrom(
+                    org.apache.lucene.analysis.util.CharFilterFactory.lookupClass(entry.getKey()))) {
+                expected.add(entry.getValue());
+            }
+        }
+        expected.remove(Void.class);
+        expected.remove(Deprecated.class);
+
+        Collection<Class<?>> actual = new HashSet<>();
+        for (Class<?> clazz : getTokenizers().values()) {
+            if (MultiTermAwareComponent.class.isAssignableFrom(clazz)) {
+                actual.add(clazz);
+            }
+        }
+        for (Class<?> clazz : getTokenFilters().values()) {
+            if (MultiTermAwareComponent.class.isAssignableFrom(clazz)) {
+                actual.add(clazz);
+            }
+        }
+        for (Class<?> clazz : getCharFilters().values()) {
+            if (MultiTermAwareComponent.class.isAssignableFrom(clazz)) {
+                actual.add(clazz);
+            }
+        }
+
+        Set<Class<?>> classesMissingMultiTermSupport = new HashSet<>(expected);
+        classesMissingMultiTermSupport.removeAll(actual);
+        assertTrue("Classes are missing multi-term support: " + classesMissingMultiTermSupport,
+                classesMissingMultiTermSupport.isEmpty());
+
+        Set<Class<?>> classesThatShouldNotHaveMultiTermSupport = new HashSet<>(actual);
+        classesThatShouldNotHaveMultiTermSupport.removeAll(expected);
+        assertTrue("Classes should not have multi-term support: " + classesThatShouldNotHaveMultiTermSupport,
+                classesThatShouldNotHaveMultiTermSupport.isEmpty());
+    }
+
+}

From 33313df0a201d4ffd8eb4888d9a5c25b8d8e7616 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Thu, 23 Jun 2016 10:18:05 +0200
Subject: [PATCH 1043/1311] Add ThreadLeakLingering option to Rest client tests

Some Rest tests use the Sun HTTP server which has lingering threads after shutdown. Similar to ESTestCase, this adds an
option to wait 5 seconds for these threads to terminate.
---
 .../java/org/elasticsearch/client/RestClientTestCase.java     | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/client/test/src/main/java/org/elasticsearch/client/RestClientTestCase.java b/client/test/src/main/java/org/elasticsearch/client/RestClientTestCase.java
index 30305478d77..8c506beb5ac 100644
--- a/client/test/src/main/java/org/elasticsearch/client/RestClientTestCase.java
+++ b/client/test/src/main/java/org/elasticsearch/client/RestClientTestCase.java
@@ -26,6 +26,7 @@ import com.carrotsearch.randomizedtesting.annotations.SeedDecorators;
 import com.carrotsearch.randomizedtesting.annotations.TestMethodProviders;
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakAction;
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakGroup;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies;
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
@@ -38,7 +39,8 @@ import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 @ThreadLeakGroup(ThreadLeakGroup.Group.MAIN)
 @ThreadLeakAction({ThreadLeakAction.Action.WARN, ThreadLeakAction.Action.INTERRUPT})
 @ThreadLeakZombies(ThreadLeakZombies.Consequence.IGNORE_REMAINING_TESTS)
+@ThreadLeakLingering(linger = 5000) // 5 sec lingering
 @TimeoutSuite(millis = 2 * 60 * 60 * 1000)
 public abstract class RestClientTestCase extends RandomizedTest {
 
-}
\ No newline at end of file
+}

From 5a08e36f9cec26c0b9cbc67c8b8f0a07a6c16b17 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Thu, 23 Jun 2016 13:10:50 +0200
Subject: [PATCH 1044/1311] Update migrate_5_0.asciidoc

Updated breaking changes to state that upgraded indices still need to be reindexed,
and to mention the migration plugin
---
 docs/reference/migration/migrate_5_0.asciidoc | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/docs/reference/migration/migrate_5_0.asciidoc b/docs/reference/migration/migrate_5_0.asciidoc
index da4ba2c521d..c35ea743fbe 100644
--- a/docs/reference/migration/migrate_5_0.asciidoc
+++ b/docs/reference/migration/migrate_5_0.asciidoc
@@ -4,6 +4,14 @@
 This section discusses the changes that you need to be aware of when migrating
 your application to Elasticsearch 5.0.
 
+[float]
+[[migration-plugin]]
+=== Migration Plugin
+
+The https://github.com/elastic/elasticsearch-migration/blob/2.x/README.asciidoc[`elasticsearch-migration` plugin]
+(compatible with Elasticsearch 2.3.0 and above) will help you to find issues
+that need to be addressed when upgrading to Elasticsearch 5.0.  
+
 [float]
 === Indices created before 5.0
 
@@ -16,9 +24,10 @@ version of Elasticsearch before 2.0.
 =========================================
 
 Indices created in Elasticsearch 1.x or before will need to be reindexed with
-Elasticsearch 2.x in order to be readable by Elasticsearch 5.x. The easiest
-way to do this is to upgrade to Elasticsearch 2.3 or later and to use the
-`reindex` API.
+Elasticsearch 2.x in order to be readable by Elasticsearch 5.x. It is not
+sufficient to use the <<indices-upgrade,`upgrade`>> API.  The easiest
+way to reindex old indices is to upgrade to Elasticsearch 2.3 or later and to use the
+`reindex` API, or the reindex UI provided by the <<migration-plugin,Migration Plugin>>.
 
 =========================================
 

From a5908a5da5135c19c187632627568a2e9457cd30 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Thu, 23 Jun 2016 14:05:56 +0200
Subject: [PATCH 1045/1311] [TEST] Increase timeouts for Rest test client
 (#19042)

Some Rest / Doc tests were running into the default socket timeout of 10 seconds.
---
 .../test/rest/client/RestTestClient.java      | 21 +++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java
index 6a9dd888504..6879e02747d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java
@@ -22,6 +22,7 @@ import com.carrotsearch.randomizedtesting.RandomizedTest;
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHost;
+import org.apache.http.client.config.RequestConfig;
 import org.apache.http.config.Registry;
 import org.apache.http.config.RegistryBuilder;
 import org.apache.http.conn.socket.ConnectionSocketFactory;
@@ -29,6 +30,8 @@ import org.apache.http.conn.socket.PlainConnectionSocketFactory;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.entity.StringEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
 import org.apache.http.message.BasicHeader;
 import org.apache.http.ssl.SSLContexts;
 import org.apache.lucene.util.IOUtils;
@@ -79,6 +82,10 @@ public class RestTestClient implements Closeable {
     public static final String TRUSTSTORE_PATH = "truststore.path";
     public static final String TRUSTSTORE_PASSWORD = "truststore.password";
 
+    public static final int CONNECT_TIMEOUT_MILLIS = 1000;
+    public static final int SOCKET_TIMEOUT_MILLIS = 30000;
+    public static final int MAX_RETRY_TIMEOUT_MILLIS = SOCKET_TIMEOUT_MILLIS;
+    public static final int CONNECTION_REQUEST_TIMEOUT_MILLIS = 500;
 
     private static final ESLogger logger = Loggers.getLogger(RestTestClient.class);
     //query_string params that don't need to be declared in the spec, thay are supported by default
@@ -296,7 +303,17 @@ public class RestTestClient implements Closeable {
                 .register("http", PlainConnectionSocketFactory.getSocketFactory())
                 .register("https", sslsf)
                 .build();
-        CloseableHttpClient httpClient = RestClient.Builder.createDefaultHttpClient(socketFactoryRegistry);
+        PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
+        //default settings may be too constraining
+        connectionManager.setDefaultMaxPerRoute(10);
+        connectionManager.setMaxTotal(30);
+
+        //default timeouts are all infinite
+        RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(CONNECT_TIMEOUT_MILLIS)
+            .setSocketTimeout(SOCKET_TIMEOUT_MILLIS)
+            .setConnectionRequestTimeout(CONNECTION_REQUEST_TIMEOUT_MILLIS).build();
+        CloseableHttpClient httpClient = HttpClientBuilder.create()
+            .setConnectionManager(connectionManager).setDefaultRequestConfig(requestConfig).build();
 
         String protocol = settings.get(PROTOCOL, "http");
         HttpHost[] hosts = new HttpHost[urls.length];
@@ -305,7 +322,7 @@ public class RestTestClient implements Closeable {
             hosts[i] = new HttpHost(url.getHost(), url.getPort(), protocol);
         }
 
-        RestClient.Builder builder = RestClient.builder(hosts).setHttpClient(httpClient);
+        RestClient.Builder builder = RestClient.builder(hosts).setHttpClient(httpClient).setMaxRetryTimeoutMillis(MAX_RETRY_TIMEOUT_MILLIS);
         try (ThreadContext threadContext = new ThreadContext(settings)) {
             Header[] defaultHeaders = new Header[threadContext.getHeaders().size()];
             int i = 0;

From 7f101743629451c0ce115aa81e0577cee947d6f4 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 23 Jun 2016 09:21:40 -0400
Subject: [PATCH 1046/1311] Upgrade JNA to 4.2.2 and remove optionality

This commit upgrades JNA from version 4.1.0 to 4.2.2. Additionally, this
dependency is now non-optional as JNA is dual-licensed with Apache
License 2.0 since JNA 4.0.0.

Relates #19045
---
 buildSrc/version.properties | 3 +--
 core/build.gradle           | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index 39662a309d1..7e23db81bc4 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -7,8 +7,7 @@ jts               = 1.13
 jackson           = 2.7.1
 log4j             = 1.2.17
 slf4j             = 1.6.2
-jna               = 4.1.0
-
+jna               = 4.2.2
 
 # test dependencies
 randomizedrunner  = 2.3.2
diff --git a/core/build.gradle b/core/build.gradle
index 9137ce68824..88c5cc2a16d 100644
--- a/core/build.gradle
+++ b/core/build.gradle
@@ -89,7 +89,7 @@ dependencies {
   compile "log4j:log4j:${versions.log4j}", optional
   compile "log4j:apache-log4j-extras:${versions.log4j}", optional
 
-  compile "net.java.dev.jna:jna:${versions.jna}", optional
+  compile "net.java.dev.jna:jna:${versions.jna}"
 
   if (isEclipse == false || project.path == ":core-tests") {
     testCompile("org.elasticsearch.test:framework:${version}") {

From 0cae9ad30eb8fed74c6396a73343e83bc8547f73 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Thu, 23 Jun 2016 15:32:26 +0200
Subject: [PATCH 1047/1311] docs: removed obsolete information, percolator
 queries are not longer loaded into jvm heap memory.

---
 docs/reference/mapping/types/percolator.asciidoc | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/docs/reference/mapping/types/percolator.asciidoc b/docs/reference/mapping/types/percolator.asciidoc
index d430f40f1d0..4c7ff113fa9 100644
--- a/docs/reference/mapping/types/percolator.asciidoc
+++ b/docs/reference/mapping/types/percolator.asciidoc
@@ -79,8 +79,4 @@ against child documents such as `has_child` and `has_parent`.
 There are a number of queries that fetch data via a get call during query parsing. For example the `terms` query when
 using terms lookup, `template` query when using indexed scripts and `geo_shape` when using pre-indexed shapes. When these
 queries are indexed by the `percolator` field type then the get call is executed once. So each time the `percolator`
-query evaluates these queries, the fetches terms, shapes etc. as the were upon index time will be used.
-
-The `wildcard` and `regexp` query natively use a lot of memory and because the percolator keeps the queries into memory
-this can easily take up the available memory in the heap space. If possible try to use a `prefix` query or ngramming to
-achieve the same result (with way less memory being used).
\ No newline at end of file
+query evaluates these queries, the fetches terms, shapes etc. as the were upon index time will be used.
\ No newline at end of file

From d6ddce2be8c5d73381063b8c1f4e511e98e0e02c Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 23 Jun 2016 11:42:25 -0400
Subject: [PATCH 1048/1311] Update SHA for JNA dependency

This commit updates the SHA for the JNA dependency. The JNA dependency
was upgraded from version 4.1.0 to version 4.2.2 in commit
7f101743629451c0ce115aa81e0577cee947d6f4 but the SHA was not updated.
---
 distribution/licenses/jna-4.1.0.jar.sha1 | 1 -
 distribution/licenses/jna-4.2.2.jar.sha1 | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)
 delete mode 100644 distribution/licenses/jna-4.1.0.jar.sha1
 create mode 100644 distribution/licenses/jna-4.2.2.jar.sha1

diff --git a/distribution/licenses/jna-4.1.0.jar.sha1 b/distribution/licenses/jna-4.1.0.jar.sha1
deleted file mode 100644
index 09ad9094922..00000000000
--- a/distribution/licenses/jna-4.1.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-1c12d070e602efd8021891cdd7fd18bc129372d4
diff --git a/distribution/licenses/jna-4.2.2.jar.sha1 b/distribution/licenses/jna-4.2.2.jar.sha1
new file mode 100644
index 00000000000..8b1acbbe5d7
--- /dev/null
+++ b/distribution/licenses/jna-4.2.2.jar.sha1
@@ -0,0 +1 @@
+5012450aee579c3118ff09461d5ce210e0cdc2a9
\ No newline at end of file

From ca6fa9ef199e89f8be3bd726b07fe4024b0971d9 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Thu, 23 Jun 2016 17:52:09 +0200
Subject: [PATCH 1049/1311] Fix block checks when no indices are specified
 (#19047)

Global cluster blocks were not checked if an empty set of indices were passed as argument to the block checking method. This would lead to issues where some operations are already executed on a cluster before it has recovered its cluster state.
---
 .../cluster/block/ClusterBlocks.java          | 13 +++---
 .../admin/indices/exists/IndicesExistsIT.java | 46 +++++++++++++++++++
 .../cluster/block/ClusterBlockTests.java      | 16 +++++++
 3 files changed, 69 insertions(+), 6 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java

diff --git a/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java b/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java
index aae2abc0f15..e6f04c8702c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java
+++ b/core/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java
@@ -150,8 +150,12 @@ public class ClusterBlocks extends AbstractDiffable<ClusterBlocks> {
         }
     }
 
+    private boolean globalBlocked(ClusterBlockLevel level) {
+        return global(level).isEmpty() == false;
+    }
+
     public ClusterBlockException globalBlockedException(ClusterBlockLevel level) {
-        if (global(level).isEmpty()) {
+        if (globalBlocked(level) == false) {
             return null;
         }
         return new ClusterBlockException(global(level));
@@ -175,10 +179,7 @@ public class ClusterBlocks extends AbstractDiffable<ClusterBlocks> {
     }
 
     public boolean indexBlocked(ClusterBlockLevel level, String index) {
-        if (!global(level).isEmpty()) {
-            return true;
-        }
-        return !blocksForIndex(level, index).isEmpty();
+        return globalBlocked(level) || blocksForIndex(level, index).isEmpty() == false;
     }
 
     public ClusterBlockException indicesBlockedException(ClusterBlockLevel level, String[] indices) {
@@ -188,7 +189,7 @@ public class ClusterBlocks extends AbstractDiffable<ClusterBlocks> {
                 indexIsBlocked = true;
             }
         }
-        if (!indexIsBlocked) {
+        if (globalBlocked(level) == false && indexIsBlocked == false) {
             return null;
         }
         Function<String, Stream<ClusterBlock>> blocksForIndexAtLevel = index -> blocksForIndex(level, index).stream();
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java
new file mode 100644
index 00000000000..e289d90c7a8
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/exists/IndicesExistsIT.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.admin.indices.exists;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.discovery.MasterNotDiscoveredException;
+import org.elasticsearch.gateway.GatewayService;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
+import org.elasticsearch.test.InternalTestCluster;
+
+import java.io.IOException;
+
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
+
+@ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
+public class IndicesExistsIT extends ESIntegTestCase {
+
+    public void testIndexExistsWithBlocksInPlace() throws IOException {
+        Settings settings = Settings.builder().put(GatewayService.RECOVER_AFTER_NODES_SETTING.getKey(), 99).build();
+        String node = internalCluster().startNode(settings);
+
+        assertThrows(client(node).admin().indices().prepareExists("test").setMasterNodeTimeout(TimeValue.timeValueSeconds(0)),
+            MasterNotDiscoveredException.class);
+
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node)); // shut down node so that test properly cleans up
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java b/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java
index daf715f385e..7f2d0828128 100644
--- a/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java
@@ -20,11 +20,13 @@
 package org.elasticsearch.cluster.block;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
 
+import java.util.Collections;
 import java.util.EnumSet;
 
 import static org.elasticsearch.test.VersionUtils.randomVersion;
@@ -76,4 +78,18 @@ public class ClusterBlockTests extends ESTestCase {
                 randomBoolean(), randomFrom(RestStatus.values()), levels);
         assertThat(clusterBlock.toString(), not(endsWith(",")));
     }
+
+    public void testGlobalBlocksCheckedIfNoIndicesSpecified() {
+        EnumSet<ClusterBlockLevel> levels = EnumSet.noneOf(ClusterBlockLevel.class);
+        int nbLevels = randomIntBetween(1, ClusterBlockLevel.values().length);
+        for (int j = 0; j < nbLevels; j++) {
+            levels.add(randomFrom(ClusterBlockLevel.values()));
+        }
+        ClusterBlock globalBlock = new ClusterBlock(randomInt(), "cluster block #" + randomInt(), randomBoolean(),
+            randomBoolean(), randomFrom(RestStatus.values()), levels);
+        ClusterBlocks clusterBlocks = new ClusterBlocks(Collections.singleton(globalBlock), ImmutableOpenMap.of());
+        ClusterBlockException exception = clusterBlocks.indicesBlockedException(randomFrom(globalBlock.levels()), new String[0]);
+        assertNotNull(exception);
+        assertEquals(exception.blocks(), Collections.singleton(globalBlock));
+    }
 }

From 9a3227108bd73220f8b86c5f58266d6bf75e3e07 Mon Sep 17 00:00:00 2001
From: Lee Hinman <lee@writequit.org>
Date: Thu, 23 Jun 2016 09:47:49 -0600
Subject: [PATCH 1050/1311] [TEST] Add ensureGreen for IpRangeIT

Resolves #18584
---
 .../elasticsearch/search/aggregations/bucket/IpRangeIT.java   | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java
index f34b6220050..d9f2ed7c365 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/IpRangeIT.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.search.aggregations.bucket;
 
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.containsString;
@@ -54,6 +55,7 @@ public class IpRangeIT extends ESIntegTestCase {
     public void setupSuiteScopeCluster() throws Exception {
         assertAcked(prepareCreate("idx")
                 .addMapping("type", "ip", "type=ip", "ips", "type=ip"));
+        waitForRelocation(ClusterHealthStatus.GREEN);
 
         indexRandom(true,
                 client().prepareIndex("idx", "type", "1").setSource(
@@ -67,6 +69,8 @@ public class IpRangeIT extends ESIntegTestCase {
                         "ips", Arrays.asList("2001:db8::ff00:42:8329", "2001:db8::ff00:42:8380")));
 
         assertAcked(prepareCreate("idx_unmapped"));
+        waitForRelocation(ClusterHealthStatus.GREEN);
+        refresh();
     }
 
     public void testSingleValuedField() {

From ae8f885ef9dde93d65b3316c47cf2e4180b9e8bf Mon Sep 17 00:00:00 2001
From: Karel Minarik <karmi@karmi.cz>
Date: Thu, 23 Jun 2016 18:13:52 +0200
Subject: [PATCH 1051/1311] Fixed incorrect indentation in the YAML integration
 test for the "Rollover" API

---
 .../rest-api-spec/test/indices.rollover/10_basic.yaml  | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml
index 647907614fa..3a4821193e6 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/10_basic.yaml
@@ -30,11 +30,11 @@
 
   # perform alias rollover
   - do:
-    indices.rollover:
-      alias: "logs_search"
-      body:
-        conditions:
-          max_docs: 1
+      indices.rollover:
+        alias: "logs_search"
+        body:
+          conditions:
+            max_docs: 1
 
   - match: { old_index: logs-1 }
   - match: { new_index: logs-2 }

From ca322a12fc39314beb5df127502b8f296aa8e7a9 Mon Sep 17 00:00:00 2001
From: Areek Zillur <areek.zillur@elasticsearch.com>
Date: Thu, 23 Jun 2016 12:37:04 -0400
Subject: [PATCH 1052/1311] remove extra brace from rollover rest api spec

---
 .../src/main/resources/rest-api-spec/api/indices.rollover.json  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
index d3660c4a679..218871a9765 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.rollover.json
@@ -3,7 +3,7 @@
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-rollover-index.html",
     "methods": ["POST"],
     "url": {
-      "path": "/{alias}/_rollover}",
+      "path": "/{alias}/_rollover",
       "paths": ["/{alias}/_rollover", "/{alias}/_rollover/{new_index}"],
       "parts": {
         "alias": {

From 148e64d654c694313e94afeb2f8948d44608cf58 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Thu, 23 Jun 2016 22:49:19 +0200
Subject: [PATCH 1053/1311] [TEST] Port testcase from #19035 to master

---
 .../index/translog/TranslogTests.java         | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
index 0c6ee4117cb..c72a6ecef27 100644
--- a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
+++ b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
@@ -1902,4 +1902,23 @@ public class TranslogTests extends ESTestCase {
         Checkpoint read = Checkpoint.read(tempDir.resolve("foo.cpk"));
         assertEquals(read, checkpoint);
     }
+
+    /**
+     * Tests that closing views after the translog is fine and we can reopen the translog
+     */
+    public void testPendingDelete() throws IOException {
+        translog.add(new Translog.Index("test", "1", new byte[]{1}));
+        translog.prepareCommit();
+        Translog.TranslogGeneration generation = translog.getGeneration();
+        TranslogConfig config = translog.getConfig();
+        translog.close();
+        translog = new Translog(config, generation);
+        translog.add(new Translog.Index("test", "2", new byte[]{2}));
+        translog.prepareCommit();
+        Translog.View view = translog.newView();
+        translog.add(new Translog.Index("test", "3", new byte[]{3}));
+        translog.close();
+        IOUtils.close(view);
+        translog = new Translog(config, generation);
+    }
 }

From d73194b9b7b79f643513a87c17ef89c472f55ee3 Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Fri, 24 Jun 2016 10:58:29 +0200
Subject: [PATCH 1054/1311] Remove settings filtering for service_account in
 GCS repository

Related to #18945 and to this https://github.com/elastic/elasticsearch/commit/35d3bdab84fa05c71e8ae019aaf661759c8b1622#commitcomment-17914150

In GCS Repository plugin we defined a `service_account` setting which is defined as `Property.Filtered`.
It's not needed as it's only a path to a file.

Closes #18946
---
 .../repositories/gcs/GoogleCloudStorageRepository.java          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
index 337fbcf8d72..3616e18e83d 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
@@ -62,7 +62,7 @@ public class GoogleCloudStorageRepository extends BlobStoreRepository {
     public static final Setting<String> APPLICATION_NAME =
             new Setting<>("application_name", GoogleCloudStoragePlugin.NAME, Function.identity(), Property.NodeScope, Property.Dynamic);
     public static final Setting<String> SERVICE_ACCOUNT =
-            simpleString("service_account", Property.NodeScope, Property.Dynamic, Property.Filtered);
+            simpleString("service_account", Property.NodeScope, Property.Dynamic);
     public static final Setting<TimeValue> HTTP_READ_TIMEOUT =
             timeSetting("http.read_timeout", NO_TIMEOUT, Property.NodeScope, Property.Dynamic);
     public static final Setting<TimeValue> HTTP_CONNECT_TIMEOUT =

From fbad3af3524ddb5a1e45f085d3723e7b247fc0d7 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Tue, 21 Jun 2016 16:31:44 +0200
Subject: [PATCH 1055/1311] Add a how-to section to the docs. #18998

This moves the "Performance Considerations for Elasticsearch Indexing" blog post
to the reference guide and adds similar recommendations for tuning disk usage
and search speed.
---
 docs/reference/how-to.asciidoc                |  22 ++
 docs/reference/how-to/disk-usage.asciidoc     | 159 ++++++++++++++
 docs/reference/how-to/indexing-speed.asciidoc | 106 ++++++++++
 docs/reference/how-to/search-speed.asciidoc   | 194 ++++++++++++++++++
 docs/reference/index.asciidoc                 |   2 +
 5 files changed, 483 insertions(+)
 create mode 100644 docs/reference/how-to.asciidoc
 create mode 100644 docs/reference/how-to/disk-usage.asciidoc
 create mode 100644 docs/reference/how-to/indexing-speed.asciidoc
 create mode 100644 docs/reference/how-to/search-speed.asciidoc

diff --git a/docs/reference/how-to.asciidoc b/docs/reference/how-to.asciidoc
new file mode 100644
index 00000000000..ee954553617
--- /dev/null
+++ b/docs/reference/how-to.asciidoc
@@ -0,0 +1,22 @@
+[[how-to]]
+= How To
+
+[partintro]
+--
+Elasticsearch ships with defaults which are intended to give a good out of
+the box experience. Full text search, highlighting, aggregations, indexing
+should all just work without the user having to change anything.
+
+Once you better understand how you want to use Elasticsearch, however,
+there are a number of optimizations you can make to improve performance
+for your use case.
+
+This section provides guidance about which changes should and shouldn't be
+made.
+--
+
+include::how-to/indexing-speed.asciidoc[]
+
+include::how-to/search-speed.asciidoc[]
+
+include::how-to/disk-usage.asciidoc[]
diff --git a/docs/reference/how-to/disk-usage.asciidoc b/docs/reference/how-to/disk-usage.asciidoc
new file mode 100644
index 00000000000..6465690ec96
--- /dev/null
+++ b/docs/reference/how-to/disk-usage.asciidoc
@@ -0,0 +1,159 @@
+[[tune-for-disk-usage]]
+== Tune for disk usage
+
+[float]
+=== Disable the features you do not need
+
+By default elasticsearch indexes and adds doc values to most fields so that they
+can be searched and aggregated out of the box. For instance if you have a numeric
+field called `foo` that you need to run histograms on but that you never need to
+filter on, you can safely disable indexing on this field in your
+<<mappings,mappings>>:
+
+[source,js]
+--------------------------------------------------
+PUT index
+{
+  "mappings": {
+    "type": {
+      "properties": {
+        "foo": {
+          "type": "integer",
+          "index": false
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+<<text,`text`>> fields store normalization factors in the index in order to be
+able to score documents. If you only need matching capabilities on a `text`
+field but do not care about the produced scores, you can configure elasticsearch
+to not write norms to the index:
+
+[source,js]
+--------------------------------------------------
+PUT index
+{
+  "mappings": {
+    "type": {
+      "properties": {
+        "foo": {
+          "type": "text",
+          "norms": false
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+<<text,`text`>> fields also store frequencies and positions in the index by
+default. Frequencies are used to compute scores and positions are used to run
+phrase queries. If you do not need to run phrase queries, you can tell
+elasticsearch to not index positions:
+
+[source,js]
+--------------------------------------------------
+PUT index
+{
+  "mappings": {
+    "type": {
+      "properties": {
+        "foo": {
+          "type": "text",
+          "index_options": "freqs"
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+Furthermore if you do not care about scoring either, you can configure
+elasticsearch to just index matching documents for every term. You will
+still be able to search on this field, but phrase queries will raise errors
+and scoring will assume that terms appear only once in every document.
+
+[source,js]
+--------------------------------------------------
+PUT index
+{
+  "mappings": {
+    "type": {
+      "properties": {
+        "foo": {
+          "type": "text",
+          "norms": false,
+          "index_options": "freqs"
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+[float]
+=== Don't use default dynamic string mappings
+
+The default <<dynamic-mapping,dynamic string mappings>> will index string fields
+both as <<text,`text`>> and <<keyword,`keyword`>>. This is wasteful if you only
+need one of them. Typically an `id` field will only need to be indexed as a
+`keyword` while a `body` field will only need to be indexed as a `text` field.
+
+This can be disabled by either configuring explicit mappings on string fields
+or setting up dynamic templates that will map string fields as either `text`
+or `keyword`.
+
+For instance, here is a template that can be used in order to only map string
+fields as `keyword`:
+
+[source,js]
+--------------------------------------------------
+PUT index
+{
+  "mappings": {
+    "type": {
+      "dynamic_templates": [
+        {
+          "strings": {
+            "match_mapping_type": "string",
+            "mapping": {
+              "type": "keyword"
+            }
+          }
+        }
+      ]
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+[float]
+=== Disable `_all`
+
+The <<mapping-all-field,`_all`>> field indexes the value of all fields of a
+document and can use significant space. If you never need to search against all
+fields at the same time, it can be disabled.
+
+[float]
+=== Use `best_compression`
+
+The `_source` and stored fields can easily take a non negligible amount of disk
+space. They can be compressed more aggressively by using the `best_compression`
+<<index-codec,codec>>.
+
+[float]
+=== Use the smallest numeric type that is sufficient
+
+When storing <<number,numeric data>>, using `float` over `double`, or `half_float`
+over `float` can help save storage. This is also true for integer types, but less
+since Elasticsearch will more easily compress them based on the number of bits
+that they actually need.
+
diff --git a/docs/reference/how-to/indexing-speed.asciidoc b/docs/reference/how-to/indexing-speed.asciidoc
new file mode 100644
index 00000000000..8175c21ee8a
--- /dev/null
+++ b/docs/reference/how-to/indexing-speed.asciidoc
@@ -0,0 +1,106 @@
+[[tune-for-indexing-speed]]
+== Tune for indexing speed
+
+[float]
+=== Use bulk requests
+
+Bulk requests will yield much better performance than single-document index
+requests. In order to know the optimal size of a bulk request, you shoud run
+a benchmark on a single node with a single shard. First try to index 100
+documents at once, then 200, then 400, etc. doubling the number of documents
+in a bulk request in every benchmark run. When the indexing speed starts to
+plateau then you know you reached the optimal size of a bulk request for your
+data. In case of tie, it is better to err in the direction of too few rather
+than too many documents. Beware that too large bulk requests might put the
+cluster under memory pressure when many of them are sent concurrently, so
+it is advisable to avoid going beyond a couple tens of megabytes per request
+even if larger requests seem to perform better.
+
+[float]
+=== Use multiple workers/threads to send data to elasticsearch
+
+A single thread sending bulk requests is unlikely to be able to max out the
+indexing capacity of an elasticsearch cluster. In order to use all resources
+of the cluster, you should send data from multiple threads or processes. In
+addition to making better use of the resources of the cluster, this should
+help reduce the cost of each fsync.
+
+Make sure to watch for `TOO_MANY_REQUESTS (429)` response codes
+(`EsRejectedExecutionException` with the Java client), which is the way that
+elasticsearch tells you that it cannot keep up with the current indexing rate.
+When it happens, you should pause ndexing a bit before trying again, ideally
+with randomized exponential backoff.
+
+Similarly to sizing bulk requests, only testing can tell what the optimal
+number of workers is. This can be tested by progressivily increasing the
+number of workers until either I/O or CPU is saturated on the cluster.
+
+[float]
+=== Increase the refresh interval
+
+The default <<dynamic-index-settings,`index.refresh_interval`>> is `1s`, which
+forces elasticsearch to create a new segment every second.
+Increasing this value (to say, `30s`) will allow larger segments to flush and
+decreases future merge pressure.
+
+[float]
+=== Disable refresh and replicas for initial loads
+
+If you need to load a large amount of data at once, you should disable refresh
+by setting `index.refresh_interval` to `-1` and set `index.number_of_replicas`
+to `0`. This will temporarily put your index at risk since the loss of any shard
+will cause data loss, but at the same time indexing will be faster since
+documents will be indexed only once. Once the initial loading is finished, you
+can set `index.refresh_interval` and `index.number_of_replicas` back to their
+original values.
+
+[float]
+=== Disable swapping
+
+You should make sure that the operating system is not swapping out the java
+process by <<setup-configuration-memory,disabling swappping>>.
+
+[float]
+=== Give memory to the filesystem cache
+
+The filesystem cache will be used in order to buffer I/O operations. You should
+make sure to give at least half the memory of the machine running elasticsearch
+to the filesystem cache.
+
+[float]
+=== Use faster hardware
+
+If indexing is I/O bound, you should investigate giving more memory to the
+filesystem cache (see above) or buying faster drives. In particular SSD drives
+are known to perform better than spinning disks. Always use local storage,
+remote filesystems such as `NFS` or `SMB` should be avoided. Also beware of
+virtualized storage such as Amazon's `Elastic Block Storage`. Virtualized
+storage works very well with Elasticsearch, and it is appealing since it is so
+fast and simple to set up, but it is also unfortunately inherently slower on an
+ongoing basis when compared to dedicated local storage. If you put an index on
+`EBS`, be sure to use provisioned IOPS otherwise operations could be quickly
+throttled.
+
+Stripe your index across multiple SSDs by configuring a RAID 0 array. Remember
+that it will increase the risk of failure since the failure of any one SSD
+destroys the index. However this is typically the right tradeoff to make:
+optimize single shards for maximum performance, and then add replicas across
+different nodes so there's redundancy for any node failures. You can also use
+<<modules-snapshots,snapshot and restore>> to backup the index for further
+insurance.
+
+[float]
+=== Indexing buffer size
+
+If your node is doing only heavy indexing, be sure
+<<indexing-buffer,`indices.memory.index_buffer_size`>> is large enough to give
+at most 512 MB indexing buffer per shard doing heavy indexing (beyond that
+indexing performance does not typically improve). Elasticsearch takes that
+setting (a percentage of the java heap or an absolute byte-size), and
+uses it as a shared buffer across all active shards. Very active shards will
+naturally use this buffer more than shards that are performing lightweight
+indexing.
+
+The default is `10%` which is often plenty: for example, if you give the JVM
+10GB of memory, it will give 1GB to the index buffer, which is enough to host
+two shards that are heavily indexing.
diff --git a/docs/reference/how-to/search-speed.asciidoc b/docs/reference/how-to/search-speed.asciidoc
new file mode 100644
index 00000000000..b6e9c4e927d
--- /dev/null
+++ b/docs/reference/how-to/search-speed.asciidoc
@@ -0,0 +1,194 @@
+[[tune-for-search-speed]]
+== Tune for search speed
+
+[float]
+=== Give memory to the filesystem cache
+
+Elasticsearch heavily relies on the filesystem cache in order to make search
+fast. In general, you should make sure that at least half the available memory
+goes to the filesystem cache so that elasticsearch can keep hot regions of the
+index in physical memory.
+
+[float]
+=== Use faster hardware
+
+If your search is I/O bound, you should investigate giving more memory to the
+filesystem cache (see above) or buying faster drives. In particular SSD drives
+are known to perform better than spinning disks. Always use local storage,
+remote filesystems such as `NFS` or `SMB` should be avoided. Also beware of
+virtualized storage such as Amazon's `Elastic Block Storage`. Virtualized
+storage works very well with Elasticsearch, and it is appealing since it is so
+fast and simple to set up, but it is also unfortunately inherently slower on an
+ongoing basis when compared to dedicated local storage. If you put an index on
+`EBS`, be sure to use provisioned IOPS otherwise operations could be quickly
+throttled.
+
+If your search is CPU-bound, you should investigate buying faster CPUs.
+
+[float]
+=== Document modeling
+
+Documents should be modeled so that search-time operations are as cheap as possible.
+
+In particular, joins should be avoided. <<nested,`nested`>> can make queries
+several times slower and <<mapping-parent-field,parent-child>> relations can make
+queries hundreds of times slower. So if the same questions can be answered without
+joins by denormalizing documents, significant speedups can be expected.
+
+[float]
+=== Pre-index data
+
+You should leverage patterns in your queries to optimize the way data is indexed.
+For instance, if all your documents have a `price` field and most queries run
+<<search-aggregations-bucket-range-aggregation,`range`>> aggregations on a fixed
+list of ranges, you could make this aggregation faster by pre-indexing the ranges
+into the index and using a <<search-aggregations-bucket-terms-aggregation,`terms`>>
+aggregations.
+
+For instance, if documents look like:
+
+[source,js]
+--------------------------------------------------
+PUT index/type/1
+{
+  "designation": "spoon",
+  "price": 13
+}
+--------------------------------------------------
+// CONSOLE
+
+and search requests look like:
+
+[source,js]
+--------------------------------------------------
+GET index/_search
+{
+  "aggs": {
+    "price_ranges": {
+      "range": {
+        "field": "price",
+        "ranges": [
+          { "to": 10 },
+          { "from": 10, "to": 100 },
+          { "from": 100 }
+        ]
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+Then documents could be enriched by a `price_range` field at index time, which
+should be mapped as a <<keyword,`keyword`>>:
+
+[source,js]
+--------------------------------------------------
+PUT index
+{
+  "mappings": {
+    "type": {
+      "properties": {
+        "price_range": {
+          "type": "keyword"
+        }
+      }
+    }
+  }
+}
+
+PUT index/type/1
+{
+  "designation": "spoon",
+  "price": 13,
+  "price_range": "10-100"
+}
+--------------------------------------------------
+// CONSOLE
+
+And then search requests could aggregate this new field rather than running a
+`range` aggregation on the `price` field.
+
+[source,js]
+--------------------------------------------------
+GET index/_search
+{
+  "aggs": {
+    "price_ranges": {
+      "terms": {
+        "field": "price_range"
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+[float]
+=== Mappings
+
+The fact that some data is numeric does not mean it should always be mapped as a
+<<number,numeric field>>. Typically, fields storing identifiers such as an `ISBN`
+or any number identifying a record from another database, might benefit from
+being mapped as <<keyword,`keyword`>> rather than `integer` or `long`.
+
+[float]
+=== Avoid scripts
+
+In general, scripts should be avoided. If they are absolutely needed, you
+should prefer the `painless` and `expressions` engines.
+
+[float]
+=== Force-merge read-only indices
+
+Indices that are read-only would benefit from being
+<<indices-forcemerge,merged down to a single segment>>. This is typically the
+case with time-based indices: only the index for the current time frame is
+getting new documents while older indices are read-only.
+
+IMPORTANT: Don't force-merge indices that are still being written to -- leave
+merging to the background merge process.
+
+[float]
+=== Warm up global ordinals
+
+Global ordinals are a data-structure that is used in order to run
+<<search-aggregations-bucket-terms-aggregation,`terms`>> aggregations on
+<<keyword,`keyword`>> fields. They are loaded lazily in memory because
+elasticsearch does not know which fields will be used in `terms` aggregations
+and which fields won't. You can tell elasticsearch to load global ordinals
+eagerly at refresh-time by configuring mappings as described below:
+
+[source,js]
+--------------------------------------------------
+PUT index
+{
+  "mappings": {
+    "type": {
+      "properties": {
+        "foo": {
+          "type": "keyword",
+          "eager_global_ordinals": true
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+[float]
+=== Warm up the filesystem cache
+
+If the machine running elasticsearch is restarted, the filesystem cache will be
+empty, so it will take some time before the operating system loads hot regions
+of the index into memory so that search operations are fast. You can explicitly
+tell the operating system which files should be loaded into memory eagerly
+depending on the file extension using the <<file-system,`index.store.preload`>>
+setting.
+
+WARNING: Loading data into the filesystem cache eagerly on too many indices or
+too many files will make searh _slower_ if the filesystem cache is not large
+enough to hold all the data. Use with caution.
diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc
index 132d763f714..3a625ac1a2c 100644
--- a/docs/reference/index.asciidoc
+++ b/docs/reference/index.asciidoc
@@ -43,6 +43,8 @@ include::index-modules.asciidoc[]
 
 include::ingest.asciidoc[]
 
+include::how-to.asciidoc[]
+
 include::testing.asciidoc[]
 
 include::glossary.asciidoc[]

From 6d5b4a78fefdf66e0a3fa7aa32681ca6b7cf99fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Thu, 23 Jun 2016 17:04:56 +0200
Subject: [PATCH 1056/1311] Make parsing of bool queries stricter

Currently we don't throw an error when there is more than one query clause
specified in a must/must_not/should/filter object of the bool query without
using array notation, e.g.:
 { "bool" : { "must" : { "match" : { ... }, "match": { ... }}}}

In these cases, only the first query will be parsed and further behaviour is
unspecified, possibly leading to silently ignoring the rest of the query.
Instead we should throw a ParsingException if we don't encounter an END_OBJECT
token after having parsed the query clause.
---
 .../index/query/BoolQueryBuilder.java         |  4 ++++
 .../index/query/BoolQueryBuilderTests.java    | 23 +++++++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
index 052ddd85663..c37da43d399 100644
--- a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java
@@ -340,6 +340,10 @@ public class BoolQueryBuilder extends AbstractQueryBuilder<BoolQueryBuilder> {
                 default:
                     throw new ParsingException(parser.getTokenLocation(), "[bool] query does not support [" + currentFieldName + "]");
                 }
+                if (parser.currentToken() != XContentParser.Token.END_OBJECT) {
+                    throw new ParsingException(parser.getTokenLocation(),
+                            "expected [END_OBJECT] but got [{}], possibly too many query clauses", parser.currentToken());
+                }
             } else if (token == XContentParser.Token.START_ARRAY) {
                 while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                     switch (currentFieldName) {
diff --git a/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
index 89da227df87..3839254bf1d 100644
--- a/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java
@@ -25,6 +25,7 @@ import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -370,6 +371,28 @@ public class BoolQueryBuilderTests extends AbstractQueryTestCase<BoolQueryBuilde
         assertThat(ex.getMessage(), startsWith("query malformed, empty clause found at"));
     }
 
+    /**
+     * test that unknown query names in the clauses throw an error
+     */
+    public void testUnknownQueryName() throws IOException {
+        String query = "{\"bool\" : {\"must\" : { \"unknown_query\" : { } } } }";
+
+        ParsingException ex = expectThrows(ParsingException.class, () -> parseQuery(query, ParseFieldMatcher.EMPTY));
+        assertEquals("no [query] registered for [unknown_query]", ex.getMessage());
+    }
+
+    /**
+     * test that two queries in object throws error
+     */
+    public void testTooManyQueriesInObject() throws IOException {
+        String clauseType = randomFrom(new String[] {"must", "should", "must_not", "filter"});
+        // should also throw error if invalid query is preceded by a valid one
+        String query = "{\"bool\" : {\"" + clauseType
+                + "\" : { \"match\" : { \"foo\" : \"bar\" } , \"match\" : { \"baz\" : \"buzz\" } } } }";
+        ParsingException ex = expectThrows(ParsingException.class, () -> parseQuery(query, ParseFieldMatcher.EMPTY));
+        assertEquals("expected [END_OBJECT] but got [FIELD_NAME], possibly too many query clauses", ex.getMessage());
+    }
+
     public void testRewrite() throws IOException {
         BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
         boolean mustRewrite = false;

From 599a5489981e3b8f58e77c0433c382096dcfd591 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Thu, 2 Jun 2016 12:53:05 +0200
Subject: [PATCH 1057/1311] percolator: Don't verify candidate matches with
 MemoryIndex that are verified matches

If we don't care about scoring then for certain candidate matches we can be certain, that if they are a candidate match,
then they will always match. So verifying these queries with the MemoryIndex can be skipped.
---
 .../common/lucene/LuceneTests.java            |  47 ++-
 .../percolator/ExtractQueryTermsService.java  | 348 ++++++++++++------
 .../percolator/PercolateQuery.java            |  54 ++-
 .../percolator/PercolateQueryBuilder.java     |   5 +-
 .../percolator/PercolatorFieldMapper.java     |  35 +-
 .../ExtractQueryTermsServiceTests.java        | 293 ++++++++++++---
 .../percolator/PercolateQueryTests.java       | 228 ++++++++++--
 .../PercolatorFieldMapperTests.java           |  23 +-
 .../percolator/PercolatorQuerySearchIT.java   |  13 +-
 9 files changed, 799 insertions(+), 247 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java b/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java
index 224a3cf6c88..e9f07acb9d1 100644
--- a/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java
+++ b/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.common.lucene;
 
 import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Field.Store;
@@ -27,6 +28,8 @@ import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NoDeletionPolicy;
 import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.RandomIndexWriter;
@@ -35,9 +38,11 @@ import org.apache.lucene.index.Term;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.Weight;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.util.Bits;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -49,9 +54,6 @@ import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-/**
- *
- */
 public class LuceneTests extends ESTestCase {
     public void testWaitForIndex() throws Exception {
         final MockDirectoryWrapper dir = newMockDirectory();
@@ -355,6 +357,45 @@ public class LuceneTests extends ESTestCase {
         dir.close();
     }
 
+    public void testAsSequentialAccessBits() throws Exception {
+        Directory dir = newDirectory();
+        IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new KeywordAnalyzer()));
+
+        Document doc = new Document();
+        doc.add(new StringField("foo", "bar", Store.NO));
+        w.addDocument(doc);
+
+        doc = new Document();
+        w.addDocument(doc);
+
+        doc = new Document();
+        doc.add(new StringField("foo", "bar", Store.NO));
+        w.addDocument(doc);
+
+
+        try (DirectoryReader reader = DirectoryReader.open(w)) {
+            IndexSearcher searcher = newSearcher(reader);
+            Weight termWeight = new TermQuery(new Term("foo", "bar")).createWeight(searcher, false);
+            assertEquals(1, reader.leaves().size());
+            LeafReaderContext leafReaderContext = reader.leaves().get(0);
+            Bits bits = Lucene.asSequentialAccessBits(leafReaderContext.reader().maxDoc(), termWeight.scorer(leafReaderContext));
+
+            expectThrows(IndexOutOfBoundsException.class, () -> bits.get(-1));
+            expectThrows(IndexOutOfBoundsException.class, () -> bits.get(leafReaderContext.reader().maxDoc()));
+            assertTrue(bits.get(0));
+            assertTrue(bits.get(0));
+            assertFalse(bits.get(1));
+            assertFalse(bits.get(1));
+            expectThrows(IllegalArgumentException.class, () -> bits.get(0));
+            assertTrue(bits.get(2));
+            assertTrue(bits.get(2));
+            expectThrows(IllegalArgumentException.class, () -> bits.get(1));
+        }
+
+        w.close();
+        dir.close();
+    }
+
     /**
      * Test that the "unmap hack" is detected as supported by lucene.
      * This works around the following bug: https://bugs.openjdk.java.net/browse/JDK-4724038
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java
index c451a245754..fbe55782e69 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java
@@ -53,10 +53,13 @@ import org.elasticsearch.index.mapper.ParseContext;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.function.Function;
 
 /**
  * Utility to extract query terms from queries and create queries from documents.
@@ -64,64 +67,179 @@ import java.util.Set;
 public final class ExtractQueryTermsService {
 
     private static final byte FIELD_VALUE_SEPARATOR = 0;  // nul code point
+    public static final String EXTRACTION_COMPLETE = "complete";
+    public static final String EXTRACTION_PARTIAL = "partial";
+    public static final String EXTRACTION_FAILED = "failed";
+
+    static final Map<Class<? extends Query>, Function<Query, Result>> queryProcessors;
+
+    static {
+        Map<Class<? extends Query>, Function<Query, Result>> map = new HashMap<>(16);
+        map.put(MatchNoDocsQuery.class, matchNoDocsQuery());
+        map.put(ConstantScoreQuery.class, constantScoreQuery());
+        map.put(BoostQuery.class, boostQuery());
+        map.put(TermQuery.class, termQuery());
+        map.put(TermsQuery.class, termsQuery());
+        map.put(CommonTermsQuery.class, commonTermsQuery());
+        map.put(BlendedTermQuery.class, blendedTermQuery());
+        map.put(PhraseQuery.class, phraseQuery());
+        map.put(SpanTermQuery.class, spanTermQuery());
+        map.put(SpanNearQuery.class, spanNearQuery());
+        map.put(SpanOrQuery.class, spanOrQuery());
+        map.put(SpanFirstQuery.class, spanFirstQuery());
+        map.put(SpanNotQuery.class, spanNotQuery());
+        map.put(BooleanQuery.class, booleanQuery());
+        map.put(DisjunctionMaxQuery.class, disjunctionMaxQuery());
+        queryProcessors = Collections.unmodifiableMap(map);
+    }
 
     private ExtractQueryTermsService() {
     }
 
     /**
      * Extracts all terms from the specified query and adds it to the specified document.
+     *
      * @param query                 The query to extract terms from
      * @param document              The document to add the extracted terms to
      * @param queryTermsFieldField  The field in the document holding the extracted terms
-     * @param unknownQueryField     The field used to mark a document that not all query terms could be extracted.
-     *                              For example the query contained an unsupported query (e.g. WildcardQuery).
-     * @param fieldType The field type for the query metadata field
+     * @param extractionResultField The field contains whether query term extraction was successful, partial or
+     *                              failed. (For example the query contained an unsupported query (e.g. WildcardQuery)
+     *                              then query extraction would fail)
+     * @param fieldType             The field type for the query metadata field
      */
     public static void extractQueryTerms(Query query, ParseContext.Document document, String queryTermsFieldField,
-                                         String unknownQueryField, FieldType fieldType) {
-        Set<Term> queryTerms;
+                                         String extractionResultField, FieldType fieldType) {
+        Result result;
         try {
-            queryTerms = extractQueryTerms(query);
+            result = extractQueryTerms(query);
         } catch (UnsupportedQueryException e) {
-            document.add(new Field(unknownQueryField, new BytesRef(), fieldType));
+            document.add(new Field(extractionResultField, EXTRACTION_FAILED, fieldType));
             return;
         }
-        for (Term term : queryTerms) {
+        for (Term term : result.terms) {
             BytesRefBuilder builder = new BytesRefBuilder();
             builder.append(new BytesRef(term.field()));
             builder.append(FIELD_VALUE_SEPARATOR);
             builder.append(term.bytes());
             document.add(new Field(queryTermsFieldField, builder.toBytesRef(), fieldType));
         }
+        if (result.verified) {
+            document.add(new Field(extractionResultField, EXTRACTION_COMPLETE, fieldType));
+        } else {
+            document.add(new Field(extractionResultField, EXTRACTION_PARTIAL, fieldType));
+        }
+    }
+
+    /**
+     * Creates a terms query containing all terms from all fields of the specified index reader.
+     */
+    public static Query createQueryTermsQuery(IndexReader indexReader, String queryMetadataField,
+                                              Term... optionalTerms) throws IOException {
+        Objects.requireNonNull(queryMetadataField);
+
+        List<Term> extractedTerms = new ArrayList<>();
+        Collections.addAll(extractedTerms, optionalTerms);
+
+        Fields fields = MultiFields.getFields(indexReader);
+        for (String field : fields) {
+            Terms terms = fields.terms(field);
+            if (terms == null) {
+                continue;
+            }
+
+            BytesRef fieldBr = new BytesRef(field);
+            TermsEnum tenum = terms.iterator();
+            for (BytesRef term = tenum.next(); term != null; term = tenum.next()) {
+                BytesRefBuilder builder = new BytesRefBuilder();
+                builder.append(fieldBr);
+                builder.append(FIELD_VALUE_SEPARATOR);
+                builder.append(term);
+                extractedTerms.add(new Term(queryMetadataField, builder.toBytesRef()));
+            }
+        }
+        return new TermsQuery(extractedTerms);
     }
 
     /**
      * Extracts all query terms from the provided query and adds it to specified list.
-     *
+     * <p>
      * From boolean query with no should clauses or phrase queries only the longest term are selected,
      * since that those terms are likely to be the rarest. Boolean query's must_not clauses are always ignored.
-     *
+     * <p>
      * If from part of the query, no query terms can be extracted then term extraction is stopped and
      * an UnsupportedQueryException is thrown.
      */
-    static Set<Term> extractQueryTerms(Query query) {
-        if (query instanceof MatchNoDocsQuery) {
-            // no terms to extract as this query matches no docs
-            return Collections.emptySet();
-        } else if (query instanceof TermQuery) {
-            return Collections.singleton(((TermQuery) query).getTerm());
-        } else if (query instanceof TermsQuery) {
-            Set<Term> terms = new HashSet<>();
+    static Result extractQueryTerms(Query query) {
+        Class queryClass = query.getClass();
+        if (queryClass.isAnonymousClass()) {
+            // Sometimes queries have anonymous classes in that case we need the direct super class.
+            // (for example blended term query)
+            queryClass = queryClass.getSuperclass();
+        }
+        Function<Query, Result> queryProcessor = queryProcessors.get(queryClass);
+        if (queryProcessor != null) {
+            return queryProcessor.apply(query);
+        } else {
+            throw new UnsupportedQueryException(query);
+        }
+    }
+
+    static Function<Query, Result> matchNoDocsQuery() {
+        return (query -> new Result(true, Collections.emptySet()));
+    }
+
+    static Function<Query, Result> constantScoreQuery() {
+        return query -> {
+            Query wrappedQuery = ((ConstantScoreQuery) query).getQuery();
+            return extractQueryTerms(wrappedQuery);
+        };
+    }
+
+    static Function<Query, Result> boostQuery() {
+        return query -> {
+            Query wrappedQuery = ((BoostQuery) query).getQuery();
+            return extractQueryTerms(wrappedQuery);
+        };
+    }
+
+    static Function<Query, Result> termQuery() {
+        return (query -> {
+            TermQuery termQuery = (TermQuery) query;
+            return new Result(true, Collections.singleton(termQuery.getTerm()));
+        });
+    }
+
+    static Function<Query, Result> termsQuery() {
+        return query -> {
             TermsQuery termsQuery = (TermsQuery) query;
+            Set<Term> terms = new HashSet<>();
             PrefixCodedTerms.TermIterator iterator = termsQuery.getTermData().iterator();
             for (BytesRef term = iterator.next(); term != null; term = iterator.next()) {
                 terms.add(new Term(iterator.field(), term));
             }
-            return terms;
-        } else if (query instanceof PhraseQuery) {
+            return new Result(true, terms);
+        };
+    }
+
+    static Function<Query, Result> commonTermsQuery() {
+        return query -> {
+            List<Term> terms = ((CommonTermsQuery) query).getTerms();
+            return new Result(false, new HashSet<>(terms));
+        };
+    }
+
+    static Function<Query, Result> blendedTermQuery() {
+        return query -> {
+            List<Term> terms = ((BlendedTermQuery) query).getTerms();
+            return new Result(true, new HashSet<>(terms));
+        };
+    }
+
+    static Function<Query, Result> phraseQuery() {
+        return query -> {
             Term[] terms = ((PhraseQuery) query).getTerms();
             if (terms.length == 0) {
-                return Collections.emptySet();
+                return new Result(true, Collections.emptySet());
             }
 
             // the longest term is likely to be the rarest,
@@ -132,19 +250,76 @@ public final class ExtractQueryTermsService {
                     longestTerm = term;
                 }
             }
-            return Collections.singleton(longestTerm);
-        } else if (query instanceof BooleanQuery) {
-            List<BooleanClause> clauses = ((BooleanQuery) query).clauses();
-            boolean hasRequiredClauses = false;
+            return new Result(false, Collections.singleton(longestTerm));
+        };
+    }
+
+    static Function<Query, Result> spanTermQuery() {
+        return query -> {
+            Term term = ((SpanTermQuery) query).getTerm();
+            return new Result(true, Collections.singleton(term));
+        };
+    }
+
+    static Function<Query, Result> spanNearQuery() {
+        return query -> {
+            Set<Term> bestClauses = null;
+            SpanNearQuery spanNearQuery = (SpanNearQuery) query;
+            for (SpanQuery clause : spanNearQuery.getClauses()) {
+                Result temp = extractQueryTerms(clause);
+                bestClauses = selectTermListWithTheLongestShortestTerm(temp.terms, bestClauses);
+            }
+            return new Result(false, bestClauses);
+        };
+    }
+
+    static Function<Query, Result> spanOrQuery() {
+        return query -> {
+            Set<Term> terms = new HashSet<>();
+            SpanOrQuery spanOrQuery = (SpanOrQuery) query;
+            for (SpanQuery clause : spanOrQuery.getClauses()) {
+                terms.addAll(extractQueryTerms(clause).terms);
+            }
+            return new Result(false, terms);
+        };
+    }
+
+    static Function<Query, Result> spanNotQuery() {
+        return query -> {
+            Result result = extractQueryTerms(((SpanNotQuery) query).getInclude());
+            return new Result(false, result.terms);
+        };
+    }
+
+    static Function<Query, Result> spanFirstQuery() {
+        return query -> {
+            Result result = extractQueryTerms(((SpanFirstQuery) query).getMatch());
+            return new Result(false, result.terms);
+        };
+    }
+
+    static Function<Query, Result> booleanQuery() {
+        return query -> {
+            BooleanQuery bq = (BooleanQuery) query;
+            List<BooleanClause> clauses = bq.clauses();
+            int minimumShouldMatch = bq.getMinimumNumberShouldMatch();
+            int numRequiredClauses = 0;
+            int numOptionalClauses = 0;
+            int numProhibitedClauses = 0;
             for (BooleanClause clause : clauses) {
                 if (clause.isRequired()) {
-                    hasRequiredClauses = true;
-                    break;
+                    numRequiredClauses++;
+                }
+                if (clause.isProhibited()) {
+                    numProhibitedClauses++;
+                }
+                if (clause.getOccur() == BooleanClause.Occur.SHOULD) {
+                    numOptionalClauses++;
                 }
             }
-            if (hasRequiredClauses) {
-                UnsupportedQueryException uqe = null;
+            if (numRequiredClauses > 0) {
                 Set<Term> bestClause = null;
+                UnsupportedQueryException uqe = null;
                 for (BooleanClause clause : clauses) {
                     if (clause.isRequired() == false) {
                         // skip must_not clauses, we don't need to remember the things that do *not* match...
@@ -153,77 +328,56 @@ public final class ExtractQueryTermsService {
                         continue;
                     }
 
-                    Set<Term> temp;
+                    Result temp;
                     try {
                         temp = extractQueryTerms(clause.getQuery());
                     } catch (UnsupportedQueryException e) {
                         uqe = e;
                         continue;
                     }
-                    bestClause = selectTermListWithTheLongestShortestTerm(temp, bestClause);
+                    bestClause = selectTermListWithTheLongestShortestTerm(temp.terms, bestClause);
                 }
                 if (bestClause != null) {
-                    return bestClause;
+                    return new Result(false, bestClause);
                 } else {
                     if (uqe != null) {
+                        // we're unable to select the best clause and an exception occurred, so we bail
                         throw uqe;
+                    } else {
+                        // We didn't find a clause and no exception occurred, so this bq only contained MatchNoDocsQueries,
+                        return new Result(true, Collections.emptySet());
                     }
-                    return Collections.emptySet();
                 }
             } else {
-                Set<Term> terms = new HashSet<>();
+                List<Query> disjunctions = new ArrayList<>(numOptionalClauses);
                 for (BooleanClause clause : clauses) {
-                    if (clause.isProhibited()) {
-                        // we don't need to remember the things that do *not* match...
-                        continue;
+                    if (clause.getOccur() == BooleanClause.Occur.SHOULD) {
+                        disjunctions.add(clause.getQuery());
                     }
-                    terms.addAll(extractQueryTerms(clause.getQuery()));
                 }
-                return terms;
+                return handleDisjunction(disjunctions, minimumShouldMatch, numProhibitedClauses > 0);
             }
-        } else if (query instanceof ConstantScoreQuery) {
-            Query wrappedQuery = ((ConstantScoreQuery) query).getQuery();
-            return extractQueryTerms(wrappedQuery);
-        } else if (query instanceof BoostQuery) {
-            Query wrappedQuery = ((BoostQuery) query).getQuery();
-            return extractQueryTerms(wrappedQuery);
-        } else if (query instanceof CommonTermsQuery) {
-            List<Term> terms = ((CommonTermsQuery) query).getTerms();
-            return new HashSet<>(terms);
-        } else if (query instanceof BlendedTermQuery) {
-            List<Term> terms = ((BlendedTermQuery) query).getTerms();
-            return new HashSet<>(terms);
-        } else if (query instanceof DisjunctionMaxQuery) {
+        };
+    }
+
+    static Function<Query, Result> disjunctionMaxQuery() {
+        return query -> {
             List<Query> disjuncts = ((DisjunctionMaxQuery) query).getDisjuncts();
-            Set<Term> terms = new HashSet<>();
-            for (Query disjunct : disjuncts) {
-                terms.addAll(extractQueryTerms(disjunct));
+            return handleDisjunction(disjuncts, 1, false);
+        };
+    }
+
+    static Result handleDisjunction(List<Query> disjunctions, int minimumShouldMatch, boolean otherClauses) {
+        boolean verified = minimumShouldMatch <= 1 && otherClauses == false;
+        Set<Term> terms = new HashSet<>();
+        for (Query disjunct : disjunctions) {
+            Result subResult = extractQueryTerms(disjunct);
+            if (subResult.verified == false) {
+                verified = false;
             }
-            return terms;
-        } else if (query instanceof SpanTermQuery) {
-            return Collections.singleton(((SpanTermQuery) query).getTerm());
-        } else if (query instanceof SpanNearQuery) {
-            Set<Term> bestClause = null;
-            SpanNearQuery spanNearQuery = (SpanNearQuery) query;
-            for (SpanQuery clause : spanNearQuery.getClauses()) {
-                Set<Term> temp = extractQueryTerms(clause);
-                bestClause = selectTermListWithTheLongestShortestTerm(temp, bestClause);
-            }
-            return bestClause;
-        } else if (query instanceof SpanOrQuery) {
-            Set<Term> terms = new HashSet<>();
-            SpanOrQuery spanOrQuery = (SpanOrQuery) query;
-            for (SpanQuery clause : spanOrQuery.getClauses()) {
-                terms.addAll(extractQueryTerms(clause));
-            }
-            return terms;
-        } else if (query instanceof SpanFirstQuery) {
-            return extractQueryTerms(((SpanFirstQuery)query).getMatch());
-        } else if (query instanceof SpanNotQuery) {
-            return extractQueryTerms(((SpanNotQuery) query).getInclude());
-        } else {
-            throw new UnsupportedQueryException(query);
+            terms.addAll(subResult.terms);
         }
+        return new Result(verified, terms);
     }
 
     static Set<Term> selectTermListWithTheLongestShortestTerm(Set<Term> terms1, Set<Term> terms2) {
@@ -243,7 +397,7 @@ public final class ExtractQueryTermsService {
         }
     }
 
-    private static int minTermLength(Set<Term> terms) {
+    static int minTermLength(Set<Term> terms) {
         int min = Integer.MAX_VALUE;
         for (Term term : terms) {
             min = Math.min(min, term.bytes().length);
@@ -251,40 +405,22 @@ public final class ExtractQueryTermsService {
         return min;
     }
 
-    /**
-     * Creates a boolean query with a should clause for each term on all fields of the specified index reader.
-     */
-    public static Query createQueryTermsQuery(IndexReader indexReader, String queryMetadataField,
-                                              String unknownQueryField) throws IOException {
-        Objects.requireNonNull(queryMetadataField);
-        Objects.requireNonNull(unknownQueryField);
+    static class Result {
 
-        List<Term> extractedTerms = new ArrayList<>();
-        extractedTerms.add(new Term(unknownQueryField));
-        Fields fields = MultiFields.getFields(indexReader);
-        for (String field : fields) {
-            Terms terms = fields.terms(field);
-            if (terms == null) {
-                continue;
-            }
+        final Set<Term> terms;
+        final boolean verified;
 
-            BytesRef fieldBr = new BytesRef(field);
-            TermsEnum tenum = terms.iterator();
-            for (BytesRef term = tenum.next(); term != null ; term = tenum.next()) {
-                BytesRefBuilder builder = new BytesRefBuilder();
-                builder.append(fieldBr);
-                builder.append(FIELD_VALUE_SEPARATOR);
-                builder.append(term);
-                extractedTerms.add(new Term(queryMetadataField, builder.toBytesRef()));
-            }
+        Result(boolean verified, Set<Term> terms) {
+            this.terms = terms;
+            this.verified = verified;
         }
-        return new TermsQuery(extractedTerms);
+
     }
 
     /**
      * Exception indicating that none or some query terms couldn't extracted from a percolator query.
      */
-    public static class UnsupportedQueryException extends RuntimeException {
+    static class UnsupportedQueryException extends RuntimeException {
 
         private final Query unsupportedQuery;
 
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
index fbf06468e75..311b612a18d 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
@@ -28,12 +28,15 @@ import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.Bits;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 
 import java.io.IOException;
 import java.util.Objects;
@@ -54,6 +57,7 @@ public final class PercolateQuery extends Query implements Accountable {
         private final IndexSearcher percolatorIndexSearcher;
 
         private Query queriesMetaDataQuery;
+        private Query verifiedQueriesQuery = new MatchNoDocsQuery("");
         private Query percolateTypeQuery;
 
         /**
@@ -64,21 +68,32 @@ public final class PercolateQuery extends Query implements Accountable {
          */
         public Builder(String docType, QueryStore queryStore, BytesReference documentSource, IndexSearcher percolatorIndexSearcher) {
             this.docType = Objects.requireNonNull(docType);
+            this.queryStore = Objects.requireNonNull(queryStore);
             this.documentSource = Objects.requireNonNull(documentSource);
             this.percolatorIndexSearcher = Objects.requireNonNull(percolatorIndexSearcher);
-            this.queryStore = Objects.requireNonNull(queryStore);
         }
 
         /**
          * Optionally sets a query that reduces the number of queries to percolate based on extracted terms from
          * the document to be percolated.
-         *
-         * @param extractedTermsFieldName The name of the field to get the extracted terms from
-         * @param unknownQueryFieldname The field used to mark documents whose queries couldn't all get extracted
+         * @param extractedTermsFieldName   The name of the field to get the extracted terms from
+         * @param extractionResultField     The field to indicate for a document whether query term extraction was complete,
+         *                                  partial or failed. If query extraction was complete, the MemoryIndex doesn't
          */
-        public void extractQueryTermsQuery(String extractedTermsFieldName, String unknownQueryFieldname) throws IOException {
+        public void extractQueryTermsQuery(String extractedTermsFieldName, String extractionResultField) throws IOException {
+            // We can only skip the MemoryIndex verification when percolating a single document.
+            // When the document being percolated contains a nested object field then the MemoryIndex contains multiple
+            // documents. In this case the term query that indicates whether memory index verification can be skipped
+            // can incorrectly indicate that non nested queries would match, while their nested variants would not.
+            if (percolatorIndexSearcher.getIndexReader().maxDoc() == 1) {
+                this.verifiedQueriesQuery = new TermQuery(new Term(extractionResultField, ExtractQueryTermsService.EXTRACTION_COMPLETE));
+            }
             this.queriesMetaDataQuery = ExtractQueryTermsService.createQueryTermsQuery(
-                    percolatorIndexSearcher.getIndexReader(), extractedTermsFieldName, unknownQueryFieldname
+                    percolatorIndexSearcher.getIndexReader(), extractedTermsFieldName,
+                    // include extractionResultField:failed, because docs with this term have no extractedTermsField
+                    // and otherwise we would fail to return these docs. Docs that failed query term extraction
+                    // always need to be verified by MemoryIndex:
+                    new Term(extractionResultField, ExtractQueryTermsService.EXTRACTION_FAILED)
             );
         }
 
@@ -94,14 +109,15 @@ public final class PercolateQuery extends Query implements Accountable {
                 throw new IllegalStateException("Either filter by deprecated percolator type or by query metadata");
             }
             // The query that selects which percolator queries will be evaluated by MemoryIndex:
-            BooleanQuery.Builder builder = new BooleanQuery.Builder();
+            BooleanQuery.Builder queriesQuery = new BooleanQuery.Builder();
             if (percolateTypeQuery != null) {
-                builder.add(percolateTypeQuery, FILTER);
+                queriesQuery.add(percolateTypeQuery, FILTER);
             }
             if (queriesMetaDataQuery != null) {
-                builder.add(queriesMetaDataQuery, FILTER);
+                queriesQuery.add(queriesMetaDataQuery, FILTER);
             }
-            return new PercolateQuery(docType, queryStore, documentSource, builder.build(), percolatorIndexSearcher);
+            return new PercolateQuery(docType, queryStore, documentSource, queriesQuery.build(), percolatorIndexSearcher,
+                    verifiedQueriesQuery);
         }
 
     }
@@ -110,22 +126,25 @@ public final class PercolateQuery extends Query implements Accountable {
     private final QueryStore queryStore;
     private final BytesReference documentSource;
     private final Query percolatorQueriesQuery;
+    private final Query verifiedQueriesQuery;
     private final IndexSearcher percolatorIndexSearcher;
 
     private PercolateQuery(String documentType, QueryStore queryStore, BytesReference documentSource,
-                           Query percolatorQueriesQuery, IndexSearcher percolatorIndexSearcher) {
+                           Query percolatorQueriesQuery, IndexSearcher percolatorIndexSearcher, Query verifiedQueriesQuery) {
         this.documentType = documentType;
         this.documentSource = documentSource;
         this.percolatorQueriesQuery = percolatorQueriesQuery;
         this.queryStore = queryStore;
         this.percolatorIndexSearcher = percolatorIndexSearcher;
+        this.verifiedQueriesQuery = verifiedQueriesQuery;
     }
 
     @Override
     public Query rewrite(IndexReader reader) throws IOException {
         Query rewritten = percolatorQueriesQuery.rewrite(reader);
         if (rewritten != percolatorQueriesQuery) {
-            return new PercolateQuery(documentType, queryStore, documentSource, rewritten, percolatorIndexSearcher);
+            return new PercolateQuery(documentType, queryStore, documentSource, rewritten, percolatorIndexSearcher,
+                    verifiedQueriesQuery);
         } else {
             return this;
         }
@@ -133,6 +152,7 @@ public final class PercolateQuery extends Query implements Accountable {
 
     @Override
     public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
+        final Weight verifiedQueriesQueryWeight = verifiedQueriesQuery.createWeight(searcher, false);
         final Weight innerWeight = percolatorQueriesQuery.createWeight(searcher, needsScores);
         return new Weight(this) {
             @Override
@@ -206,6 +226,8 @@ public final class PercolateQuery extends Query implements Accountable {
                         }
                     };
                 } else {
+                    Scorer verifiedDocsScorer = verifiedQueriesQueryWeight.scorer(leafReaderContext);
+                    Bits verifiedDocsBits = Lucene.asSequentialAccessBits(leafReaderContext.reader().maxDoc(), verifiedDocsScorer);
                     return new BaseScorer(this, approximation, queries, percolatorIndexSearcher) {
 
                         @Override
@@ -214,6 +236,14 @@ public final class PercolateQuery extends Query implements Accountable {
                         }
 
                         boolean matchDocId(int docId) throws IOException {
+                            // We use the verifiedDocsBits to skip the expensive MemoryIndex verification.
+                            // If docId also appears in the verifiedDocsBits then that means during indexing
+                            // we were able to extract all query terms and for this candidate match
+                            // and we determined based on the nature of the query that it is safe to skip
+                            // the MemoryIndex verification.
+                            if (verifiedDocsBits.get(docId)) {
+                                return true;
+                            }
                             Query query = percolatorQueries.getQuery(docId);
                             return query != null && Lucene.exists(percolatorIndexSearcher, query);
                         }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
index 9955375b183..1c47610ec15 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
@@ -420,10 +420,11 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
                         "] to be of type [percolator], but is of type [" + fieldType.typeName() + "]");
             }
             PercolatorFieldMapper.PercolatorFieldType pft = (PercolatorFieldMapper.PercolatorFieldType) fieldType;
+            PercolateQuery.QueryStore queryStore = createStore(pft, context, mapUnmappedFieldsAsString);
             PercolateQuery.Builder builder = new PercolateQuery.Builder(
-                    documentType, createStore(pft, context, mapUnmappedFieldsAsString), document, docSearcher
+                    documentType, queryStore, document, docSearcher
             );
-            builder.extractQueryTermsQuery(pft.getExtractedTermsField(), pft.getUnknownQueryFieldName());
+            builder.extractQueryTermsQuery(pft.getExtractedTermsField(), pft.getExtractionResultFieldName());
             return builder.build();
         } else {
             Query percolateTypeQuery = new TermQuery(new Term(TypeFieldMapper.NAME, MapperService.PERCOLATOR_LEGACY_TYPE_NAME));
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
index e86b8e818d0..53f74c97200 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
@@ -58,7 +58,7 @@ public class PercolatorFieldMapper extends FieldMapper {
     private static final PercolatorFieldType FIELD_TYPE = new PercolatorFieldType();
 
     public static final String EXTRACTED_TERMS_FIELD_NAME = "extracted_terms";
-    public static final String UNKNOWN_QUERY_FIELD_NAME = "unknown_query";
+    public static final String EXTRACTION_RESULT_FIELD_NAME = "extraction_result";
     public static final String QUERY_BUILDER_FIELD_NAME = "query_builder_field";
 
     public static class Builder extends FieldMapper.Builder<Builder, PercolatorFieldMapper> {
@@ -75,15 +75,15 @@ public class PercolatorFieldMapper extends FieldMapper {
             context.path().add(name());
             KeywordFieldMapper extractedTermsField = createExtractQueryFieldBuilder(EXTRACTED_TERMS_FIELD_NAME, context);
             ((PercolatorFieldType) fieldType).queryTermsField = extractedTermsField.fieldType();
-            KeywordFieldMapper unknownQueryField = createExtractQueryFieldBuilder(UNKNOWN_QUERY_FIELD_NAME, context);
-            ((PercolatorFieldType) fieldType).unknownQueryField = unknownQueryField.fieldType();
+            KeywordFieldMapper extractionResultField = createExtractQueryFieldBuilder(EXTRACTION_RESULT_FIELD_NAME, context);
+            ((PercolatorFieldType) fieldType).extractionResultField = extractionResultField.fieldType();
             BinaryFieldMapper queryBuilderField = createQueryBuilderFieldBuilder(context);
             ((PercolatorFieldType) fieldType).queryBuilderField = queryBuilderField.fieldType();
             context.path().remove();
             setupFieldType(context);
             return new PercolatorFieldMapper(name(), fieldType, defaultFieldType, context.indexSettings(),
                     multiFieldsBuilder.build(this, context), copyTo, queryShardContext, extractedTermsField,
-                    unknownQueryField, queryBuilderField);
+                    extractionResultField, queryBuilderField);
         }
 
         static KeywordFieldMapper createExtractQueryFieldBuilder(String name, BuilderContext context) {
@@ -102,6 +102,7 @@ public class PercolatorFieldMapper extends FieldMapper {
             builder.fieldType().setDocValuesType(DocValuesType.BINARY);
             return builder.build(context);
         }
+
     }
 
     public static class TypeParser implements FieldMapper.TypeParser {
@@ -115,7 +116,7 @@ public class PercolatorFieldMapper extends FieldMapper {
     public static class PercolatorFieldType extends MappedFieldType {
 
         private MappedFieldType queryTermsField;
-        private MappedFieldType unknownQueryField;
+        private MappedFieldType extractionResultField;
         private MappedFieldType queryBuilderField;
 
         public PercolatorFieldType() {
@@ -127,7 +128,7 @@ public class PercolatorFieldMapper extends FieldMapper {
         public PercolatorFieldType(PercolatorFieldType ref) {
             super(ref);
             queryTermsField = ref.queryTermsField;
-            unknownQueryField = ref.unknownQueryField;
+            extractionResultField = ref.extractionResultField;
             queryBuilderField = ref.queryBuilderField;
         }
 
@@ -135,8 +136,8 @@ public class PercolatorFieldMapper extends FieldMapper {
             return queryTermsField.name();
         }
 
-        public String getUnknownQueryFieldName() {
-            return unknownQueryField.name();
+        public String getExtractionResultFieldName() {
+            return extractionResultField.name();
         }
 
         public String getQueryBuilderFieldName() {
@@ -162,17 +163,17 @@ public class PercolatorFieldMapper extends FieldMapper {
     private final boolean mapUnmappedFieldAsString;
     private final QueryShardContext queryShardContext;
     private KeywordFieldMapper queryTermsField;
-    private KeywordFieldMapper unknownQueryField;
+    private KeywordFieldMapper extractionResultField;
     private BinaryFieldMapper queryBuilderField;
 
     public PercolatorFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
                                  Settings indexSettings, MultiFields multiFields, CopyTo copyTo, QueryShardContext queryShardContext,
-                                 KeywordFieldMapper queryTermsField, KeywordFieldMapper unknownQueryField,
+                                 KeywordFieldMapper queryTermsField, KeywordFieldMapper extractionResultField,
                                  BinaryFieldMapper queryBuilderField) {
         super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
         this.queryShardContext = queryShardContext;
         this.queryTermsField = queryTermsField;
-        this.unknownQueryField = unknownQueryField;
+        this.extractionResultField = extractionResultField;
         this.queryBuilderField = queryBuilderField;
         this.mapUnmappedFieldAsString = INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING.get(indexSettings);
     }
@@ -181,18 +182,18 @@ public class PercolatorFieldMapper extends FieldMapper {
     public FieldMapper updateFieldType(Map<String, MappedFieldType> fullNameToFieldType) {
         PercolatorFieldMapper updated = (PercolatorFieldMapper) super.updateFieldType(fullNameToFieldType);
         KeywordFieldMapper queryTermsUpdated = (KeywordFieldMapper) queryTermsField.updateFieldType(fullNameToFieldType);
-        KeywordFieldMapper unknownQueryUpdated = (KeywordFieldMapper) unknownQueryField.updateFieldType(fullNameToFieldType);
+        KeywordFieldMapper extractionResultUpdated = (KeywordFieldMapper) extractionResultField.updateFieldType(fullNameToFieldType);
         BinaryFieldMapper queryBuilderUpdated = (BinaryFieldMapper) queryBuilderField.updateFieldType(fullNameToFieldType);
 
-        if (updated == this || queryTermsUpdated == queryTermsField || unknownQueryUpdated == unknownQueryField
-                || queryBuilderUpdated == queryBuilderField) {
+        if (updated == this && queryTermsUpdated == queryTermsField && extractionResultUpdated == extractionResultField
+                && queryBuilderUpdated == queryBuilderField) {
             return this;
         }
         if (updated == this) {
             updated = (PercolatorFieldMapper) updated.clone();
         }
         updated.queryTermsField = queryTermsUpdated;
-        updated.unknownQueryField = unknownQueryUpdated;
+        updated.extractionResultField = extractionResultUpdated;
         updated.queryBuilderField = queryBuilderUpdated;
         return updated;
     }
@@ -220,7 +221,7 @@ public class PercolatorFieldMapper extends FieldMapper {
         }
 
         Query query = toQuery(queryShardContext, mapUnmappedFieldAsString, queryBuilder);
-        ExtractQueryTermsService.extractQueryTerms(query, context.doc(), queryTermsField.name(), unknownQueryField.name(),
+        ExtractQueryTermsService.extractQueryTerms(query, context.doc(), queryTermsField.name(), extractionResultField.name(),
                 queryTermsField.fieldType());
         return null;
     }
@@ -258,7 +259,7 @@ public class PercolatorFieldMapper extends FieldMapper {
 
     @Override
     public Iterator<Mapper> iterator() {
-        return Arrays.<Mapper>asList(queryTermsField, unknownQueryField, queryBuilderField).iterator();
+        return Arrays.<Mapper>asList(queryTermsField, extractionResultField, queryBuilderField).iterator();
     }
 
     @Override
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java
index b9430a32425..f876a0af9fc 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java
@@ -22,6 +22,7 @@ import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.PrefixCodedTerms;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.memory.MemoryIndex;
@@ -33,6 +34,7 @@ import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.DisjunctionMaxQuery;
+import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TermRangeQuery;
@@ -44,6 +46,7 @@ import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.percolator.ExtractQueryTermsService.Result;
 import org.elasticsearch.test.ESTestCase;
 
 
@@ -54,18 +57,22 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import static org.elasticsearch.percolator.ExtractQueryTermsService.EXTRACTION_COMPLETE;
+import static org.elasticsearch.percolator.ExtractQueryTermsService.EXTRACTION_FAILED;
+import static org.elasticsearch.percolator.ExtractQueryTermsService.EXTRACTION_PARTIAL;
 import static org.elasticsearch.percolator.ExtractQueryTermsService.UnsupportedQueryException;
 import static org.elasticsearch.percolator.ExtractQueryTermsService.extractQueryTerms;
 import static org.elasticsearch.percolator.ExtractQueryTermsService.createQueryTermsQuery;
 import static org.elasticsearch.percolator.ExtractQueryTermsService.selectTermListWithTheLongestShortestTerm;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.sameInstance;
 
 public class ExtractQueryTermsServiceTests extends ESTestCase {
 
     public final static String QUERY_TERMS_FIELD = "extracted_terms";
-    public final static String UNKNOWN_QUERY_FIELD = "unknown_query";
-    public static FieldType QUERY_TERMS_FIELD_TYPE = new FieldType();
+    public final static String EXTRACTION_RESULT_FIELD = "extraction_result";
+    public final static FieldType QUERY_TERMS_FIELD_TYPE = new FieldType();
 
     static {
         QUERY_TERMS_FIELD_TYPE.setTokenized(false);
@@ -81,33 +88,41 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         bq.add(termQuery2, BooleanClause.Occur.SHOULD);
 
         ParseContext.Document document = new ParseContext.Document();
-        extractQueryTerms(bq.build(), document, QUERY_TERMS_FIELD, UNKNOWN_QUERY_FIELD, QUERY_TERMS_FIELD_TYPE);
-        Collections.sort(document.getFields(), (field1, field2) -> field1.binaryValue().compareTo(field2.binaryValue()));
-        assertThat(document.getFields().size(), equalTo(2));
-        assertThat(document.getFields().get(0).name(), equalTo(QUERY_TERMS_FIELD));
-        assertThat(document.getFields().get(0).binaryValue().utf8ToString(), equalTo("field1\u0000term1"));
-        assertThat(document.getFields().get(1).name(), equalTo(QUERY_TERMS_FIELD));
-        assertThat(document.getFields().get(1).binaryValue().utf8ToString(), equalTo("field2\u0000term2"));
+        extractQueryTerms(bq.build(), document, QUERY_TERMS_FIELD, EXTRACTION_RESULT_FIELD, QUERY_TERMS_FIELD_TYPE);
+        assertThat(document.getField(EXTRACTION_RESULT_FIELD).stringValue(), equalTo(EXTRACTION_COMPLETE));
+        List<IndexableField> fields = new ArrayList<>(Arrays.asList(document.getFields(QUERY_TERMS_FIELD)));
+        Collections.sort(fields, (field1, field2) -> field1.binaryValue().compareTo(field2.binaryValue()));
+        assertThat(fields.size(), equalTo(2));
+        assertThat(fields.get(0).name(), equalTo(QUERY_TERMS_FIELD));
+        assertThat(fields.get(0).binaryValue().utf8ToString(), equalTo("field1\u0000term1"));
+        assertThat(fields.get(1).name(), equalTo(QUERY_TERMS_FIELD));
+        assertThat(fields.get(1).binaryValue().utf8ToString(), equalTo("field2\u0000term2"));
     }
 
     public void testExtractQueryMetadata_unsupported() {
-        BooleanQuery.Builder bq = new BooleanQuery.Builder();
-        TermQuery termQuery1 = new TermQuery(new Term("field1", "term1"));
-        bq.add(termQuery1, BooleanClause.Occur.SHOULD);
-        TermQuery termQuery2 = new TermQuery(new Term("field2", "term2"));
-        bq.add(termQuery2, BooleanClause.Occur.SHOULD);
-
         TermRangeQuery query = new TermRangeQuery("field1", new BytesRef("a"), new BytesRef("z"), true, true);
         ParseContext.Document document = new ParseContext.Document();
-        extractQueryTerms(query, document, QUERY_TERMS_FIELD, UNKNOWN_QUERY_FIELD, QUERY_TERMS_FIELD_TYPE);
+        extractQueryTerms(query, document, QUERY_TERMS_FIELD, EXTRACTION_RESULT_FIELD, QUERY_TERMS_FIELD_TYPE);
         assertThat(document.getFields().size(), equalTo(1));
-        assertThat(document.getFields().get(0).name(), equalTo(UNKNOWN_QUERY_FIELD));
-        assertThat(document.getFields().get(0).binaryValue().utf8ToString(), equalTo(""));
+        assertThat(document.getField(EXTRACTION_RESULT_FIELD).stringValue(), equalTo(EXTRACTION_FAILED));
+    }
+
+    public void testExtractQueryMetadata_notVerified() {
+        PhraseQuery phraseQuery = new PhraseQuery("field", "term");
+
+        ParseContext.Document document = new ParseContext.Document();
+        extractQueryTerms(phraseQuery, document, QUERY_TERMS_FIELD, EXTRACTION_RESULT_FIELD, QUERY_TERMS_FIELD_TYPE);
+        assertThat(document.getFields().size(), equalTo(2));
+        assertThat(document.getFields().get(0).name(), equalTo(QUERY_TERMS_FIELD));
+        assertThat(document.getFields().get(0).binaryValue().utf8ToString(), equalTo("field\u0000term"));
+        assertThat(document.getField(EXTRACTION_RESULT_FIELD).stringValue(), equalTo(EXTRACTION_PARTIAL));
     }
 
     public void testExtractQueryMetadata_termQuery() {
         TermQuery termQuery = new TermQuery(new Term("_field", "_term"));
-        List<Term> terms = new ArrayList<>(extractQueryTerms(termQuery));
+        Result result = extractQueryTerms(termQuery);
+        assertThat(result.verified, is(true));
+        List<Term> terms = new ArrayList<>(result.terms);
         assertThat(terms.size(), equalTo(1));
         assertThat(terms.get(0).field(), equalTo(termQuery.getTerm().field()));
         assertThat(terms.get(0).bytes(), equalTo(termQuery.getTerm().bytes()));
@@ -115,7 +130,9 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
 
     public void testExtractQueryMetadata_termsQuery() {
         TermsQuery termsQuery = new TermsQuery("_field", new BytesRef("_term1"), new BytesRef("_term2"));
-        List<Term> terms = new ArrayList<>(extractQueryTerms(termsQuery));
+        Result result = extractQueryTerms(termsQuery);
+        assertThat(result.verified, is(true));
+        List<Term> terms = new ArrayList<>(result.terms);
         Collections.sort(terms);
         assertThat(terms.size(), equalTo(2));
         assertThat(terms.get(0).field(), equalTo("_field"));
@@ -125,7 +142,9 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
 
         // test with different fields
         termsQuery = new TermsQuery(new Term("_field1", "_term1"), new Term("_field2", "_term2"));
-        terms = new ArrayList<>(extractQueryTerms(termsQuery));
+        result = extractQueryTerms(termsQuery);
+        assertThat(result.verified, is(true));
+        terms = new ArrayList<>(result.terms);
         Collections.sort(terms);
         assertThat(terms.size(), equalTo(2));
         assertThat(terms.get(0).field(), equalTo("_field1"));
@@ -136,7 +155,9 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
 
     public void testExtractQueryMetadata_phraseQuery() {
         PhraseQuery phraseQuery = new PhraseQuery("_field", "_term1", "term2");
-        List<Term> terms = new ArrayList<>(extractQueryTerms(phraseQuery));
+        Result result = extractQueryTerms(phraseQuery);
+        assertThat(result.verified, is(false));
+        List<Term> terms = new ArrayList<>(result.terms);
         assertThat(terms.size(), equalTo(1));
         assertThat(terms.get(0).field(), equalTo(phraseQuery.getTerms()[0].field()));
         assertThat(terms.get(0).bytes(), equalTo(phraseQuery.getTerms()[0].bytes()));
@@ -157,7 +178,9 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         builder.add(subBuilder.build(), BooleanClause.Occur.SHOULD);
 
         BooleanQuery booleanQuery = builder.build();
-        List<Term> terms = new ArrayList<>(extractQueryTerms(booleanQuery));
+        Result result = extractQueryTerms(booleanQuery);
+        assertThat("Should clause with phrase query isn't verified, so entire query can't be verified", result.verified, is(false));
+        List<Term> terms = new ArrayList<>(result.terms);
         Collections.sort(terms);
         assertThat(terms.size(), equalTo(3));
         assertThat(terms.get(0).field(), equalTo(termQuery1.getTerm().field()));
@@ -183,7 +206,9 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         builder.add(subBuilder.build(), BooleanClause.Occur.SHOULD);
 
         BooleanQuery booleanQuery = builder.build();
-        List<Term> terms = new ArrayList<>(extractQueryTerms(booleanQuery));
+        Result result = extractQueryTerms(booleanQuery);
+        assertThat(result.verified, is(true));
+        List<Term> terms = new ArrayList<>(result.terms);
         Collections.sort(terms);
         assertThat(terms.size(), equalTo(4));
         assertThat(terms.get(0).field(), equalTo(termQuery1.getTerm().field()));
@@ -204,16 +229,74 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         builder.add(phraseQuery, BooleanClause.Occur.SHOULD);
 
         BooleanQuery booleanQuery = builder.build();
-        List<Term> terms = new ArrayList<>(extractQueryTerms(booleanQuery));
+        Result result = extractQueryTerms(booleanQuery);
+        assertThat(result.verified, is(false));
+        List<Term> terms = new ArrayList<>(result.terms);
         assertThat(terms.size(), equalTo(1));
         assertThat(terms.get(0).field(), equalTo(phraseQuery.getTerms()[0].field()));
         assertThat(terms.get(0).bytes(), equalTo(phraseQuery.getTerms()[0].bytes()));
     }
 
+    public void testExactMatch_booleanQuery() {
+        BooleanQuery.Builder builder = new BooleanQuery.Builder();
+        TermQuery termQuery1 = new TermQuery(new Term("_field", "_term1"));
+        builder.add(termQuery1, BooleanClause.Occur.SHOULD);
+        TermQuery termQuery2 = new TermQuery(new Term("_field", "_term2"));
+        builder.add(termQuery2, BooleanClause.Occur.SHOULD);
+        Result result = extractQueryTerms(builder.build());
+        assertThat("All clauses are exact, so candidate matches are verified", result.verified, is(true));
+
+        builder = new BooleanQuery.Builder();
+        builder.add(termQuery1, BooleanClause.Occur.SHOULD);
+        PhraseQuery phraseQuery1 = new PhraseQuery("_field", "_term1", "_term2");
+        builder.add(phraseQuery1, BooleanClause.Occur.SHOULD);
+        result = extractQueryTerms(builder.build());
+        assertThat("Clause isn't exact, so candidate matches are not verified", result.verified, is(false));
+
+        builder = new BooleanQuery.Builder();
+        builder.add(phraseQuery1, BooleanClause.Occur.SHOULD);
+        PhraseQuery phraseQuery2 = new PhraseQuery("_field", "_term3", "_term4");
+        builder.add(phraseQuery2, BooleanClause.Occur.SHOULD);
+        result = extractQueryTerms(builder.build());
+        assertThat("No clause is exact, so candidate matches are not verified", result.verified, is(false));
+
+        builder = new BooleanQuery.Builder();
+        builder.add(termQuery1, BooleanClause.Occur.MUST_NOT);
+        builder.add(termQuery2, BooleanClause.Occur.SHOULD);
+        result = extractQueryTerms(builder.build());
+        assertThat("There is a must_not clause, so candidate matches are not verified", result.verified, is(false));
+
+        builder = new BooleanQuery.Builder();
+        builder.setMinimumNumberShouldMatch(randomIntBetween(2, 32));
+        builder.add(termQuery1, BooleanClause.Occur.SHOULD);
+        builder.add(termQuery2, BooleanClause.Occur.SHOULD);
+        result = extractQueryTerms(builder.build());
+        assertThat("Minimum match is >= 1, so candidate matches are not verified", result.verified, is(false));
+
+        builder = new BooleanQuery.Builder();
+        builder.add(termQuery1, randomBoolean() ? BooleanClause.Occur.MUST : BooleanClause.Occur.FILTER);
+        result = extractQueryTerms(builder.build());
+        assertThat("Single required clause, so candidate matches are verified", result.verified, is(false));
+
+        builder = new BooleanQuery.Builder();
+        builder.add(termQuery1, randomBoolean() ? BooleanClause.Occur.MUST : BooleanClause.Occur.FILTER);
+        builder.add(termQuery2, randomBoolean() ? BooleanClause.Occur.MUST : BooleanClause.Occur.FILTER);
+        result = extractQueryTerms(builder.build());
+        assertThat("Two or more required clauses, so candidate matches are not verified", result.verified, is(false));
+
+        builder = new BooleanQuery.Builder();
+        builder.add(termQuery1, randomBoolean() ? BooleanClause.Occur.MUST : BooleanClause.Occur.FILTER);
+        builder.add(termQuery2, BooleanClause.Occur.MUST_NOT);
+        result = extractQueryTerms(builder.build());
+        assertThat("Required and prohibited clauses, so candidate matches are not verified", result.verified, is(false));
+    }
+
     public void testExtractQueryMetadata_constantScoreQuery() {
         TermQuery termQuery1 = new TermQuery(new Term("_field", "_term"));
         ConstantScoreQuery constantScoreQuery = new ConstantScoreQuery(termQuery1);
-        List<Term> terms = new ArrayList<>(extractQueryTerms(constantScoreQuery));
+        Result result = extractQueryTerms(constantScoreQuery);
+        assertThat(result.verified, is(true));
+        List<Term> terms = new ArrayList<>(result.terms);
         assertThat(terms.size(), equalTo(1));
         assertThat(terms.get(0).field(), equalTo(termQuery1.getTerm().field()));
         assertThat(terms.get(0).bytes(), equalTo(termQuery1.getTerm().bytes()));
@@ -222,7 +305,9 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
     public void testExtractQueryMetadata_boostQuery() {
         TermQuery termQuery1 = new TermQuery(new Term("_field", "_term"));
         BoostQuery constantScoreQuery = new BoostQuery(termQuery1, 1f);
-        List<Term> terms = new ArrayList<>(extractQueryTerms(constantScoreQuery));
+        Result result = extractQueryTerms(constantScoreQuery);
+        assertThat(result.verified, is(true));
+        List<Term> terms = new ArrayList<>(result.terms);
         assertThat(terms.size(), equalTo(1));
         assertThat(terms.get(0).field(), equalTo(termQuery1.getTerm().field()));
         assertThat(terms.get(0).bytes(), equalTo(termQuery1.getTerm().bytes()));
@@ -232,7 +317,9 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         CommonTermsQuery commonTermsQuery = new CommonTermsQuery(BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, 100);
         commonTermsQuery.add(new Term("_field", "_term1"));
         commonTermsQuery.add(new Term("_field", "_term2"));
-        List<Term> terms = new ArrayList<>(extractQueryTerms(commonTermsQuery));
+        Result result = extractQueryTerms(commonTermsQuery);
+        assertThat(result.verified, is(false));
+        List<Term> terms = new ArrayList<>(result.terms);
         Collections.sort(terms);
         assertThat(terms.size(), equalTo(2));
         assertThat(terms.get(0).field(), equalTo("_field"));
@@ -242,15 +329,17 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
     }
 
     public void testExtractQueryMetadata_blendedTermQuery() {
-        Term[] terms = new Term[]{new Term("_field", "_term1"), new Term("_field", "_term2")};
-        BlendedTermQuery commonTermsQuery = BlendedTermQuery.booleanBlendedQuery(terms, false);
-        List<Term> result = new ArrayList<>(extractQueryTerms(commonTermsQuery));
-        Collections.sort(result);
-        assertThat(result.size(), equalTo(2));
-        assertThat(result.get(0).field(), equalTo("_field"));
-        assertThat(result.get(0).text(), equalTo("_term1"));
-        assertThat(result.get(1).field(), equalTo("_field"));
-        assertThat(result.get(1).text(), equalTo("_term2"));
+        Term[] termsArr = new Term[]{new Term("_field", "_term1"), new Term("_field", "_term2")};
+        BlendedTermQuery commonTermsQuery = BlendedTermQuery.booleanBlendedQuery(termsArr, false);
+        Result result = extractQueryTerms(commonTermsQuery);
+        assertThat(result.verified, is(true));
+        List<Term> terms = new ArrayList<>(result.terms);
+        Collections.sort(terms);
+        assertThat(terms.size(), equalTo(2));
+        assertThat(terms.get(0).field(), equalTo("_field"));
+        assertThat(terms.get(0).text(), equalTo("_term1"));
+        assertThat(terms.get(1).field(), equalTo("_field"));
+        assertThat(terms.get(1).text(), equalTo("_term2"));
     }
 
     public void testExtractQueryMetadata_spanTermQuery() {
@@ -266,8 +355,9 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         // 4) FieldMaskingSpanQuery is a tricky query so we shouldn't optimize this
 
         SpanTermQuery spanTermQuery1 = new SpanTermQuery(new Term("_field", "_short_term"));
-        Set<Term> terms = extractQueryTerms(spanTermQuery1);
-        assertTermsEqual(terms, spanTermQuery1.getTerm());
+        Result result = extractQueryTerms(spanTermQuery1);
+        assertThat(result.verified, is(true));
+        assertTermsEqual(result.terms, spanTermQuery1.getTerm());
     }
 
     public void testExtractQueryMetadata_spanNearQuery() {
@@ -275,48 +365,109 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         SpanTermQuery spanTermQuery2 = new SpanTermQuery(new Term("_field", "_very_long_term"));
         SpanNearQuery spanNearQuery = new SpanNearQuery.Builder("_field", true)
                 .addClause(spanTermQuery1).addClause(spanTermQuery2).build();
-        Set<Term> terms = extractQueryTerms(spanNearQuery);
-        assertTermsEqual(terms, spanTermQuery2.getTerm());
+
+        Result result = extractQueryTerms(spanNearQuery);
+        assertThat(result.verified, is(false));
+        assertTermsEqual(result.terms, spanTermQuery2.getTerm());
     }
 
     public void testExtractQueryMetadata_spanOrQuery() {
         SpanTermQuery spanTermQuery1 = new SpanTermQuery(new Term("_field", "_short_term"));
         SpanTermQuery spanTermQuery2 = new SpanTermQuery(new Term("_field", "_very_long_term"));
         SpanOrQuery spanOrQuery = new SpanOrQuery(spanTermQuery1, spanTermQuery2);
-        Set<Term> terms = extractQueryTerms(spanOrQuery);
-        assertTermsEqual(terms, spanTermQuery1.getTerm(), spanTermQuery2.getTerm());
+        Result result = extractQueryTerms(spanOrQuery);
+        assertThat(result.verified, is(false));
+        assertTermsEqual(result.terms, spanTermQuery1.getTerm(), spanTermQuery2.getTerm());
     }
 
     public void testExtractQueryMetadata_spanFirstQuery() {
         SpanTermQuery spanTermQuery1 = new SpanTermQuery(new Term("_field", "_short_term"));
         SpanFirstQuery spanFirstQuery = new SpanFirstQuery(spanTermQuery1, 20);
-        Set<Term> terms = extractQueryTerms(spanFirstQuery);
-        assertTermsEqual(terms, spanTermQuery1.getTerm());
+        Result result = extractQueryTerms(spanFirstQuery);
+        assertThat(result.verified, is(false));
+        assertTermsEqual(result.terms, spanTermQuery1.getTerm());
     }
 
     public void testExtractQueryMetadata_spanNotQuery() {
         SpanTermQuery spanTermQuery1 = new SpanTermQuery(new Term("_field", "_short_term"));
         SpanTermQuery spanTermQuery2 = new SpanTermQuery(new Term("_field", "_very_long_term"));
         SpanNotQuery spanNotQuery = new SpanNotQuery(spanTermQuery1, spanTermQuery2);
-        Set<Term> terms = extractQueryTerms(spanNotQuery);
-        assertTermsEqual(terms, spanTermQuery1.getTerm());
+        Result result = extractQueryTerms(spanNotQuery);
+        assertThat(result.verified, is(false));
+        assertTermsEqual(result.terms, spanTermQuery1.getTerm());
     }
 
     public void testExtractQueryMetadata_matchNoDocsQuery() {
-        Set<Term> terms = extractQueryTerms(new MatchNoDocsQuery("sometimes there is no reason at all"));
-        assertEquals(0, terms.size());
+        Result result = extractQueryTerms(new MatchNoDocsQuery("sometimes there is no reason at all"));
+        assertThat(result.verified, is(true));
+        assertEquals(0, result.terms.size());
 
         BooleanQuery.Builder bq = new BooleanQuery.Builder();
         bq.add(new TermQuery(new Term("field", "value")), BooleanClause.Occur.MUST);
         bq.add(new MatchNoDocsQuery("sometimes there is no reason at all"), BooleanClause.Occur.MUST);
-        terms = extractQueryTerms(bq.build());
-        assertEquals(0, terms.size());
+        result = extractQueryTerms(bq.build());
+        assertThat(result.verified, is(false));
+        assertEquals(0, result.terms.size());
 
         bq = new BooleanQuery.Builder();
         bq.add(new TermQuery(new Term("field", "value")), BooleanClause.Occur.SHOULD);
         bq.add(new MatchNoDocsQuery("sometimes there is no reason at all"), BooleanClause.Occur.SHOULD);
-        terms = extractQueryTerms(bq.build());
-        assertTermsEqual(terms, new Term("field", "value"));
+        result = extractQueryTerms(bq.build());
+        assertThat(result.verified, is(true));
+        assertTermsEqual(result.terms, new Term("field", "value"));
+
+        DisjunctionMaxQuery disjunctionMaxQuery = new DisjunctionMaxQuery(
+                Arrays.asList(new TermQuery(new Term("field", "value")), new MatchNoDocsQuery("sometimes there is no reason at all")),
+                1f
+        );
+        result = extractQueryTerms(disjunctionMaxQuery);
+        assertThat(result.verified, is(true));
+        assertTermsEqual(result.terms, new Term("field", "value"));
+    }
+
+    public void testExtractQueryMetadata_matchAllDocsQuery() {
+        expectThrows(UnsupportedQueryException.class, () -> extractQueryTerms(new MatchAllDocsQuery()));
+
+        BooleanQuery.Builder builder = new BooleanQuery.Builder();
+        builder.add(new TermQuery(new Term("field", "value")), BooleanClause.Occur.MUST);
+        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
+        Result result = extractQueryTerms(builder.build());
+        assertThat(result.verified, is(false));
+        assertTermsEqual(result.terms, new Term("field", "value"));
+
+        builder = new BooleanQuery.Builder();
+        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
+        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
+        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
+        BooleanQuery bq1 = builder.build();
+        expectThrows(UnsupportedQueryException.class, () -> extractQueryTerms(bq1));
+
+        builder = new BooleanQuery.Builder();
+        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST_NOT);
+        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
+        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
+        BooleanQuery bq2 = builder.build();
+        expectThrows(UnsupportedQueryException.class, () -> extractQueryTerms(bq2));
+
+        builder = new BooleanQuery.Builder();
+        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
+        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
+        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
+        BooleanQuery bq3 = builder.build();
+        expectThrows(UnsupportedQueryException.class, () -> extractQueryTerms(bq3));
+
+        builder = new BooleanQuery.Builder();
+        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST_NOT);
+        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
+        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
+        BooleanQuery bq4 = builder.build();
+        expectThrows(UnsupportedQueryException.class, () -> extractQueryTerms(bq4));
+
+        builder = new BooleanQuery.Builder();
+        builder.add(new TermQuery(new Term("field", "value")), BooleanClause.Occur.SHOULD);
+        builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
+        BooleanQuery bq5 = builder.build();
+        expectThrows(UnsupportedQueryException.class, () -> extractQueryTerms(bq5));
     }
 
     public void testExtractQueryMetadata_unsupportedQuery() {
@@ -343,8 +494,9 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         builder.add(unsupportedQuery, BooleanClause.Occur.MUST);
         BooleanQuery bq1 = builder.build();
 
-        Set<Term> terms = extractQueryTerms(bq1);
-        assertTermsEqual(terms, termQuery1.getTerm());
+        Result result = extractQueryTerms(bq1);
+        assertThat(result.verified, is(false));
+        assertTermsEqual(result.terms, termQuery1.getTerm());
 
         TermQuery termQuery2 = new TermQuery(new Term("_field", "_longer_term"));
         builder = new BooleanQuery.Builder();
@@ -352,8 +504,9 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         builder.add(termQuery2, BooleanClause.Occur.MUST);
         builder.add(unsupportedQuery, BooleanClause.Occur.MUST);
         bq1 = builder.build();
-        terms = extractQueryTerms(bq1);
-        assertTermsEqual(terms, termQuery2.getTerm());
+        result = extractQueryTerms(bq1);
+        assertThat(result.verified, is(false));
+        assertTermsEqual(result.terms, termQuery2.getTerm());
 
         builder = new BooleanQuery.Builder();
         builder.add(unsupportedQuery, BooleanClause.Occur.MUST);
@@ -372,7 +525,27 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
                 Arrays.asList(termQuery1, termQuery2, termQuery3, termQuery4), 0.1f
         );
 
-        List<Term> terms = new ArrayList<>(extractQueryTerms(disjunctionMaxQuery));
+        Result result = extractQueryTerms(disjunctionMaxQuery);
+        assertThat(result.verified, is(true));
+        List<Term> terms = new ArrayList<>(result.terms);
+        Collections.sort(terms);
+        assertThat(terms.size(), equalTo(4));
+        assertThat(terms.get(0).field(), equalTo(termQuery1.getTerm().field()));
+        assertThat(terms.get(0).bytes(), equalTo(termQuery1.getTerm().bytes()));
+        assertThat(terms.get(1).field(), equalTo(termQuery2.getTerm().field()));
+        assertThat(terms.get(1).bytes(), equalTo(termQuery2.getTerm().bytes()));
+        assertThat(terms.get(2).field(), equalTo(termQuery3.getTerm().field()));
+        assertThat(terms.get(2).bytes(), equalTo(termQuery3.getTerm().bytes()));
+        assertThat(terms.get(3).field(), equalTo(termQuery4.getTerm().field()));
+        assertThat(terms.get(3).bytes(), equalTo(termQuery4.getTerm().bytes()));
+
+        disjunctionMaxQuery = new DisjunctionMaxQuery(
+                Arrays.asList(termQuery1, termQuery2, termQuery3, new PhraseQuery("_field", "_term4")), 0.1f
+        );
+
+        result = extractQueryTerms(disjunctionMaxQuery);
+        assertThat(result.verified, is(false));
+        terms = new ArrayList<>(result.terms);
         Collections.sort(terms);
         assertThat(terms.size(), equalTo(4));
         assertThat(terms.get(0).field(), equalTo(termQuery1.getTerm().field()));
@@ -394,7 +567,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
 
         IndexReader indexReader = memoryIndex.createSearcher().getIndexReader();
         TermsQuery query = (TermsQuery)
-                createQueryTermsQuery(indexReader, QUERY_TERMS_FIELD, UNKNOWN_QUERY_FIELD);
+                createQueryTermsQuery(indexReader, QUERY_TERMS_FIELD, new Term(EXTRACTION_RESULT_FIELD, EXTRACTION_FAILED));
 
         PrefixCodedTerms terms = query.getTermData();
         assertThat(terms.size(), equalTo(15L));
@@ -413,7 +586,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         assertTermIterator(termIterator, "field2\u0000some", QUERY_TERMS_FIELD);
         assertTermIterator(termIterator, "field2\u0000text", QUERY_TERMS_FIELD);
         assertTermIterator(termIterator, "field4\u0000123", QUERY_TERMS_FIELD);
-        assertTermIterator(termIterator, "", UNKNOWN_QUERY_FIELD);
+        assertTermIterator(termIterator, EXTRACTION_FAILED, EXTRACTION_RESULT_FIELD);
     }
 
     public void testSelectTermsListWithHighestSumOfTermLength() {
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java
index 4879badc7d3..79904f422bb 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java
@@ -29,6 +29,7 @@ import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.memory.MemoryIndex;
@@ -37,14 +38,21 @@ import org.apache.lucene.queries.CommonTermsQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.ConstantScoreScorer;
+import org.apache.lucene.search.ConstantScoreWeight;
+import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.FilterScorer;
+import org.apache.lucene.search.FilteredDocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.Weight;
 import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.search.spans.SpanNearQuery;
 import org.apache.lucene.search.spans.SpanNotQuery;
@@ -52,6 +60,7 @@ import org.apache.lucene.search.spans.SpanOrQuery;
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.Uid;
@@ -61,8 +70,11 @@ import org.junit.After;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 
 import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.equalTo;
@@ -72,7 +84,7 @@ public class PercolateQueryTests extends ESTestCase {
 
     public final static String EXTRACTED_TERMS_FIELD_NAME = "extracted_terms";
     public final static String UNKNOWN_QUERY_FIELD_NAME = "unknown_query";
-    public static FieldType EXTRACTED_TERMS_FIELD_TYPE = new FieldType();
+    public final static FieldType EXTRACTED_TERMS_FIELD_TYPE = new FieldType();
 
     static {
         EXTRACTED_TERMS_FIELD_TYPE.setTokenized(false);
@@ -247,34 +259,91 @@ public class PercolateQueryTests extends ESTestCase {
     }
 
     public void testDuel() throws Exception {
-        int numQueries = scaledRandomIntBetween(32, 256);
-        for (int i = 0; i < numQueries; i++) {
-            String id = Integer.toString(i);
-            Query query;
+        List<Function<String, Query>> queries = new ArrayList<>();
+        queries.add((id) -> new PrefixQuery(new Term("field", id)));
+        queries.add((id) -> new WildcardQuery(new Term("field", id + "*")));
+        queries.add((id) -> new CustomQuery(new Term("field", id)));
+        queries.add((id) -> new SpanTermQuery(new Term("field", id)));
+        queries.add((id) -> new TermQuery(new Term("field", id)));
+        queries.add((id) -> {
+            BooleanQuery.Builder builder = new BooleanQuery.Builder();
+            return builder.build();
+        });
+        queries.add((id) -> {
+            BooleanQuery.Builder builder = new BooleanQuery.Builder();
+            builder.add(new TermQuery(new Term("field", id)), BooleanClause.Occur.MUST);
             if (randomBoolean()) {
-                query = new PrefixQuery(new Term("field", id));
-            } else if (randomBoolean()) {
-                query = new WildcardQuery(new Term("field", id + "*"));
-            } else if (randomBoolean()) {
-                query = new CustomQuery(new Term("field", id + "*"));
-            } else if (randomBoolean()) {
-                query = new SpanTermQuery(new Term("field", id));
-            } else {
-                query = new TermQuery(new Term("field", id));
+                builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT);
             }
-            addPercolatorQuery(id, query);
+            if (randomBoolean()) {
+                builder.add(new CustomQuery(new Term("field", id)), BooleanClause.Occur.MUST);
+            }
+            return builder.build();
+        });
+        queries.add((id) -> {
+            BooleanQuery.Builder builder = new BooleanQuery.Builder();
+            builder.add(new TermQuery(new Term("field", id)), BooleanClause.Occur.SHOULD);
+            if (randomBoolean()) {
+                builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT);
+            }
+            if (randomBoolean()) {
+                builder.add(new CustomQuery(new Term("field", id)), BooleanClause.Occur.SHOULD);
+            }
+            return builder.build();
+        });
+        queries.add((id) -> {
+            BooleanQuery.Builder builder = new BooleanQuery.Builder();
+            builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
+            builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
+            if (randomBoolean()) {
+                builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT);
+            }
+            return builder.build();
+        });
+        queries.add((id) -> {
+            BooleanQuery.Builder builder = new BooleanQuery.Builder();
+            builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
+            builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
+            if (randomBoolean()) {
+                builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT);
+            }
+            return builder.build();
+        });
+        queries.add((id) -> {
+            BooleanQuery.Builder builder = new BooleanQuery.Builder();
+            builder.setMinimumNumberShouldMatch(randomIntBetween(0, 4));
+            builder.add(new TermQuery(new Term("field", id)), BooleanClause.Occur.SHOULD);
+            builder.add(new CustomQuery(new Term("field", id)), BooleanClause.Occur.SHOULD);
+            return builder.build();
+        });
+        queries.add((id) -> new MatchAllDocsQuery());
+        queries.add((id) -> new MatchNoDocsQuery("no reason at all"));
+
+        int numDocs = randomIntBetween(queries.size(), queries.size() * 3);
+        for (int i = 0; i < numDocs; i++) {
+            String id = Integer.toString(i);
+            addPercolatorQuery(id, queries.get(i % queries.size()).apply(id));
         }
 
         indexWriter.close();
         directoryReader = DirectoryReader.open(directory);
         IndexSearcher shardSearcher = newSearcher(directoryReader);
+        // Disable query cache, because ControlQuery cannot be cached...
+        shardSearcher.setQueryCache(null);
 
-        for (int i = 0; i < numQueries; i++) {
-            MemoryIndex memoryIndex = new MemoryIndex();
+        for (int i = 0; i < numDocs; i++) {
             String id = Integer.toString(i);
+            MemoryIndex memoryIndex = new MemoryIndex();
             memoryIndex.addField("field", id, new WhitespaceAnalyzer());
             duelRun(memoryIndex, shardSearcher);
         }
+
+        MemoryIndex memoryIndex = new MemoryIndex();
+        memoryIndex.addField("field", "value", new WhitespaceAnalyzer());
+        duelRun(memoryIndex, shardSearcher);
+        // Empty percolator doc:
+        memoryIndex = new MemoryIndex();
+        duelRun(memoryIndex, shardSearcher);
     }
 
     public void testDuelSpecificQueries() throws Exception {
@@ -312,6 +381,8 @@ public class PercolateQueryTests extends ESTestCase {
         indexWriter.close();
         directoryReader = DirectoryReader.open(directory);
         IndexSearcher shardSearcher = newSearcher(directoryReader);
+        // Disable query cache, because ControlQuery cannot be cached...
+        shardSearcher.setQueryCache(null);
 
         MemoryIndex memoryIndex = new MemoryIndex();
         memoryIndex.addField("field", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer());
@@ -332,33 +403,33 @@ public class PercolateQueryTests extends ESTestCase {
     }
 
     private void duelRun(MemoryIndex memoryIndex, IndexSearcher shardSearcher) throws IOException {
+        boolean requireScore = randomBoolean();
         IndexSearcher percolateSearcher = memoryIndex.createSearcher();
-        PercolateQuery.Builder builder1 = new PercolateQuery.Builder(
+        PercolateQuery.Builder builder = new PercolateQuery.Builder(
                 "docType",
                 queryStore,
                 new BytesArray("{}"),
                 percolateSearcher
         );
         // enables the optimization that prevents queries from being evaluated that don't match
-        builder1.extractQueryTermsQuery(EXTRACTED_TERMS_FIELD_NAME, UNKNOWN_QUERY_FIELD_NAME);
-        TopDocs topDocs1 = shardSearcher.search(builder1.build(), 10);
+        builder.extractQueryTermsQuery(EXTRACTED_TERMS_FIELD_NAME, UNKNOWN_QUERY_FIELD_NAME);
+        Query query = requireScore ? builder.build() : new ConstantScoreQuery(builder.build());
+        TopDocs topDocs = shardSearcher.search(query, 10);
 
-        PercolateQuery.Builder builder2 = new PercolateQuery.Builder(
-                "docType",
-                queryStore,
-                new BytesArray("{}"),
-                percolateSearcher
-        );
-        builder2.setPercolateTypeQuery(new MatchAllDocsQuery());
-        TopDocs topDocs2 = shardSearcher.search(builder2.build(), 10);
-        assertThat(topDocs1.totalHits, equalTo(topDocs2.totalHits));
-        assertThat(topDocs1.scoreDocs.length, equalTo(topDocs2.scoreDocs.length));
-        for (int j = 0; j < topDocs1.scoreDocs.length; j++) {
-            assertThat(topDocs1.scoreDocs[j].doc, equalTo(topDocs2.scoreDocs[j].doc));
-            assertThat(topDocs1.scoreDocs[j].score, equalTo(topDocs2.scoreDocs[j].score));
-            Explanation explain1 = shardSearcher.explain(builder1.build(), topDocs1.scoreDocs[j].doc);
-            Explanation explain2 = shardSearcher.explain(builder2.build(), topDocs2.scoreDocs[j].doc);
-            assertThat(explain1.toHtml(), equalTo(explain2.toHtml()));
+        Query controlQuery = new ControlQuery(memoryIndex, queryStore);
+        controlQuery = requireScore ? controlQuery : new ConstantScoreQuery(controlQuery);
+        TopDocs controlTopDocs = shardSearcher.search(controlQuery, 10);
+        assertThat(topDocs.totalHits, equalTo(controlTopDocs.totalHits));
+        assertThat(topDocs.scoreDocs.length, equalTo(controlTopDocs.scoreDocs.length));
+        for (int j = 0; j < topDocs.scoreDocs.length; j++) {
+            assertThat(topDocs.scoreDocs[j].doc, equalTo(controlTopDocs.scoreDocs[j].doc));
+            assertThat(topDocs.scoreDocs[j].score, equalTo(controlTopDocs.scoreDocs[j].score));
+            if (requireScore) {
+                Explanation explain1 = shardSearcher.explain(query, topDocs.scoreDocs[j].doc);
+                Explanation explain2 = shardSearcher.explain(controlQuery, controlTopDocs.scoreDocs[j].doc);
+                assertThat(explain1.isMatch(), equalTo(explain2.isMatch()));
+                assertThat(explain1.getValue(), equalTo(explain2.getValue()));
+            }
         }
     }
 
@@ -391,4 +462,89 @@ public class PercolateQueryTests extends ESTestCase {
         }
     }
 
+    private final static class ControlQuery extends Query {
+
+        private final MemoryIndex memoryIndex;
+        private final PercolateQuery.QueryStore queryStore;
+
+        private ControlQuery(MemoryIndex memoryIndex, PercolateQuery.QueryStore queryStore) {
+            this.memoryIndex = memoryIndex;
+            this.queryStore = queryStore;
+        }
+
+        @Override
+        public Weight createWeight(IndexSearcher searcher, boolean needsScores) {
+            return new ConstantScoreWeight(this) {
+
+                float _score;
+
+                @Override
+                public Explanation explain(LeafReaderContext context, int doc) throws IOException {
+                    Scorer scorer = scorer(context);
+                    if (scorer != null) {
+                        int result = scorer.iterator().advance(doc);
+                        if (result == doc) {
+                            return Explanation.match(scorer.score(), "ControlQuery");
+                        }
+                    }
+                    return Explanation.noMatch("ControlQuery");
+                }
+
+                @Override
+                public String toString() {
+                    return "weight(" + ControlQuery.this + ")";
+                }
+
+                @Override
+                public Scorer scorer(LeafReaderContext context) throws IOException {
+                    DocIdSetIterator allDocs = DocIdSetIterator.all(context.reader().maxDoc());
+                    PercolateQuery.QueryStore.Leaf leaf = queryStore.getQueries(context);
+                    FilteredDocIdSetIterator memoryIndexIterator = new FilteredDocIdSetIterator(allDocs) {
+
+                        @Override
+                        protected boolean match(int doc) {
+                            try {
+                                Query query = leaf.getQuery(doc);
+                                float score = memoryIndex.search(query);
+                                if (score != 0f) {
+                                    if (needsScores) {
+                                        _score = score;
+                                    }
+                                    return true;
+                                } else {
+                                    return false;
+                                }
+                            } catch (IOException e) {
+                                throw new RuntimeException(e);
+                            }
+                        }
+                    };
+                    return new FilterScorer(new ConstantScoreScorer(this, score(), memoryIndexIterator)) {
+
+                        @Override
+                        public float score() throws IOException {
+                            return _score;
+                        }
+                    };
+                }
+            };
+        }
+
+        @Override
+        public String toString(String field) {
+            return "control{" + field + "}";
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            return sameClassAs(obj);
+        }
+
+        @Override
+        public int hashCode() {
+            return classHash();
+        }
+
+    }
+
 }
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
index 2175dabfefb..d6221184b6e 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
@@ -49,6 +49,8 @@ import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termsLookupQuery;
 import static org.elasticsearch.index.query.QueryBuilders.wildcardQuery;
+import static org.elasticsearch.percolator.ExtractQueryTermsService.EXTRACTION_COMPLETE;
+import static org.elasticsearch.percolator.ExtractQueryTermsService.EXTRACTION_FAILED;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -71,7 +73,9 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
         indexService = createIndex("test", Settings.EMPTY);
         mapperService = indexService.mapperService();
 
-        String mapper = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties")
+        String mapper = XContentFactory.jsonBuilder().startObject().startObject("type")
+            .startObject("_field_names").field("enabled", false).endObject() // makes testing easier
+            .startObject("properties")
                 .startObject("field").field("type", "text").endObject()
                 .startObject("number_field").field("type", "long").endObject()
                 .startObject("date_field").field("type", "date").endObject()
@@ -96,20 +100,21 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
             .field(fieldName, queryBuilder)
             .endObject().bytes());
 
-        assertThat(doc.rootDoc().getFields(fieldType.getUnknownQueryFieldName()).length, equalTo(0));
         assertThat(doc.rootDoc().getFields(fieldType.getExtractedTermsField()).length, equalTo(1));
         assertThat(doc.rootDoc().getFields(fieldType.getExtractedTermsField())[0].binaryValue().utf8ToString(), equalTo("field\0value"));
         assertThat(doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName()).length, equalTo(1));
+        assertThat(doc.rootDoc().getFields(fieldType.getExtractionResultFieldName()).length, equalTo(1));
+        assertThat(doc.rootDoc().getFields(fieldType.getExtractionResultFieldName())[0].stringValue(), equalTo(EXTRACTION_COMPLETE));
         BytesRef qbSource = doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName())[0].binaryValue();
         assertQueryBuilder(qbSource, queryBuilder);
 
         // add an query for which we don't extract terms from
-        queryBuilder = matchAllQuery();
+        queryBuilder = rangeQuery("field").from("a").to("z");
         doc = mapperService.documentMapper(typeName).parse("test", typeName, "1", XContentFactory.jsonBuilder().startObject()
                 .field(fieldName, queryBuilder)
                 .endObject().bytes());
-        assertThat(doc.rootDoc().getFields(fieldType.getUnknownQueryFieldName()).length, equalTo(1));
-        assertThat(doc.rootDoc().getFields(fieldType.getUnknownQueryFieldName())[0].binaryValue(), equalTo(new BytesRef()));
+        assertThat(doc.rootDoc().getFields(fieldType.getExtractionResultFieldName()).length, equalTo(1));
+        assertThat(doc.rootDoc().getFields(fieldType.getExtractionResultFieldName())[0].stringValue(), equalTo(EXTRACTION_FAILED));
         assertThat(doc.rootDoc().getFields(fieldType.getExtractedTermsField()).length, equalTo(0));
         assertThat(doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName()).length, equalTo(1));
         qbSource = doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName())[0].binaryValue();
@@ -195,6 +200,7 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
     public void testMultiplePercolatorFields() throws Exception {
         String typeName = "another_type";
         String percolatorMapper = XContentFactory.jsonBuilder().startObject().startObject(typeName)
+                .startObject("_field_names").field("enabled", false).endObject() // makes testing easier
                 .startObject("properties")
                     .startObject("query_field1").field("type", "percolator").endObject()
                     .startObject("query_field2").field("type", "percolator").endObject()
@@ -209,7 +215,7 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
                         .field("query_field2", queryBuilder)
                         .endObject().bytes()
         );
-        assertThat(doc.rootDoc().getFields().size(), equalTo(22)); // also includes all other meta fields
+        assertThat(doc.rootDoc().getFields().size(), equalTo(11)); // also includes _uid (1), type (2), source (1)
         BytesRef queryBuilderAsBytes = doc.rootDoc().getField("query_field1.query_builder_field").binaryValue();
         assertQueryBuilder(queryBuilderAsBytes, queryBuilder);
 
@@ -221,6 +227,7 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
     public void testNestedPercolatorField() throws Exception {
         String typeName = "another_type";
         String percolatorMapper = XContentFactory.jsonBuilder().startObject().startObject(typeName)
+                .startObject("_field_names").field("enabled", false).endObject() // makes testing easier
                 .startObject("properties")
                 .startObject("object_field")
                     .field("type", "object")
@@ -238,7 +245,7 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
                             .field("query_field", queryBuilder)
                         .endObject().endObject().bytes()
         );
-        assertThat(doc.rootDoc().getFields().size(), equalTo(18)); // also includes all other meta fields
+        assertThat(doc.rootDoc().getFields().size(), equalTo(8)); // also includes _uid (1), type (2), source (1)
         BytesRef queryBuilderAsBytes = doc.rootDoc().getField("object_field.query_field.query_builder_field").binaryValue();
         assertQueryBuilder(queryBuilderAsBytes, queryBuilder);
 
@@ -249,7 +256,7 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
                             .endArray()
                         .endObject().bytes()
         );
-        assertThat(doc.rootDoc().getFields().size(), equalTo(18)); // also includes all other meta fields
+        assertThat(doc.rootDoc().getFields().size(), equalTo(8)); // also includes _uid (1), type (2), source (1)
         queryBuilderAsBytes = doc.rootDoc().getField("object_field.query_field.query_builder_field").binaryValue();
         assertQueryBuilder(queryBuilderAsBytes, queryBuilder);
 
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
index fc1a4206e82..e677b6126ee 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
@@ -397,12 +397,16 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
                 .addMapping("employee", mapping)
                 .addMapping("queries", "query", "type=percolator")
         );
-        client().prepareIndex("test", "queries", "q").setSource(jsonBuilder().startObject()
+        client().prepareIndex("test", "queries", "q1").setSource(jsonBuilder().startObject()
                 .field("query", QueryBuilders.nestedQuery("employee",
                         QueryBuilders.matchQuery("employee.name", "virginia potts").operator(Operator.AND), ScoreMode.Avg)
                 ).endObject())
-                .setRefreshPolicy(IMMEDIATE)
                 .get();
+        // this query should never match as it doesn't use nested query:
+        client().prepareIndex("test", "queries", "q2").setSource(jsonBuilder().startObject()
+                .field("query", QueryBuilders.matchQuery("employee.name", "virginia")).endObject())
+                .get();
+        client().admin().indices().prepareRefresh().get();
 
         SearchResponse response = client().prepareSearch()
                 .setQuery(new PercolateQueryBuilder("query", "employee",
@@ -413,9 +417,10 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
                                     .startObject().field("name", "tony stark").endObject()
                                 .endArray()
                             .endObject().bytes()))
+                .addSort("_doc", SortOrder.ASC)
                 .get();
         assertHitCount(response, 1);
-        assertThat(response.getHits().getAt(0).getId(), equalTo("q"));
+        assertThat(response.getHits().getAt(0).getId(), equalTo("q1"));
 
         response = client().prepareSearch()
                 .setQuery(new PercolateQueryBuilder("query", "employee",
@@ -426,12 +431,14 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
                                     .startObject().field("name", "tony stark").endObject()
                                 .endArray()
                             .endObject().bytes()))
+                .addSort("_doc", SortOrder.ASC)
                 .get();
         assertHitCount(response, 0);
 
         response = client().prepareSearch()
                 .setQuery(new PercolateQueryBuilder("query", "employee",
                         XContentFactory.jsonBuilder().startObject().field("companyname", "notstark").endObject().bytes()))
+                .addSort("_doc", SortOrder.ASC)
                 .get();
         assertHitCount(response, 0);
     }

From 001a060c84f9d5fe590afcb58ad8be2815c5f6ee Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Fri, 24 Jun 2016 12:06:41 -0400
Subject: [PATCH 1058/1311] Bring painless docs closer to reality

---
 docs/reference/modules/scripting.asciidoc     |   2 +
 .../scripting/painless-syntax.asciidoc        | 164 ++++
 .../modules/scripting/painless.asciidoc       | 558 +-----------
 .../src/main/antlr/PainlessLexer.g4           |   1 +
 .../src/main/antlr/PainlessLexer.tokens       | 206 ++---
 .../src/main/antlr/PainlessParser.g4          |   1 +
 .../src/main/antlr/PainlessParser.tokens      | 206 ++---
 .../elasticsearch/painless/Augmentation.java  |  35 +
 .../painless/antlr/PainlessLexer.java         | 429 ++++-----
 .../painless/antlr/PainlessParser.java        | 848 +++++++++---------
 .../antlr/PainlessParserBaseVisitor.java      |   7 +
 .../painless/antlr/PainlessParserVisitor.java |   7 +
 .../elasticsearch/painless/antlr/Walker.java  |   9 +
 .../painless/node/EInstanceof.java            |  89 ++
 .../painless/node/package-info.java           |   7 +-
 .../org/elasticsearch/painless/java.lang.txt  |   3 +
 .../painless/AugmentationTests.java           |  13 +
 .../painless/ComparisonTests.java             |  18 +
 18 files changed, 1243 insertions(+), 1360 deletions(-)
 create mode 100644 docs/reference/modules/scripting/painless-syntax.asciidoc
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java

diff --git a/docs/reference/modules/scripting.asciidoc b/docs/reference/modules/scripting.asciidoc
index feb8113a6da..76065210569 100644
--- a/docs/reference/modules/scripting.asciidoc
+++ b/docs/reference/modules/scripting.asciidoc
@@ -96,6 +96,8 @@ include::scripting/groovy.asciidoc[]
 
 include::scripting/painless.asciidoc[]
 
+include::scripting/painless-syntax.asciidoc[]
+
 include::scripting/expression.asciidoc[]
 
 include::scripting/native.asciidoc[]
diff --git a/docs/reference/modules/scripting/painless-syntax.asciidoc b/docs/reference/modules/scripting/painless-syntax.asciidoc
new file mode 100644
index 00000000000..500a276c852
--- /dev/null
+++ b/docs/reference/modules/scripting/painless-syntax.asciidoc
@@ -0,0 +1,164 @@
+[[modules-scripting-painless-syntax]]
+=== Painless Syntax
+
+experimental[The Painless scripting language is new and is still marked as experimental. The syntax or API may be changed in the future in non-backwards compatible ways if required.]
+
+[float]
+[[painless-types]]
+=== Variable types
+
+Painless supports all of https://docs.oracle.com/javase/tutorial/java/nutsandbolts/variables.html[Java's types],
+including array types, but adds some additional built-in types.
+
+[float]
+[[painless-def]]
+==== Def
+
+The dynamic type `def` serves as a placeholder for any other type. It adopts the behavior
+of whatever runtime type it represents. 
+
+[float]
+[[painless-strings]]
+==== String
+
+String constants can be declared with single quotes, to avoid escaping horrors with JSON:
+
+[source,js]
+---------------------------------------------------------
+def mystring = 'foo';
+---------------------------------------------------------
+
+[float]
+[[painless-lists]]
+==== List
+
+Lists can be created explicitly (e.g. `new ArrayList()`) or initialized similar to Groovy:
+
+[source,js]
+---------------------------------------------------------
+def list = [1,2,3];
+---------------------------------------------------------
+
+Lists can also be accessed similar to arrays: they support subscript and `.length`:
+
+[source,js]
+---------------------------------------------------------
+def list = [1,2,3];
+return list[0]
+---------------------------------------------------------
+
+[float]
+[[painless-maps]]
+==== Map
+
+Maps can be created explicitly (e.g. `new HashMap()`) or initialized similar to Groovy:
+
+[source,js]
+---------------------------------------------------------
+def person = ['name': 'Joe', 'age': 63];
+---------------------------------------------------------
+
+Map keys can also be accessed as properties.
+
+[source,js]
+---------------------------------------------------------
+def person = ['name': 'Joe', 'age': 63];
+person.retired = true;
+return person.name
+---------------------------------------------------------
+
+Map keys can also be accessed via subscript (for keys containing special characters):
+
+[source,js]
+---------------------------------------------------------
+return map['something-absurd!']
+---------------------------------------------------------
+
+[float]
+[[painless-pattern]]
+==== Pattern
+
+Regular expression constants are directly supported:
+
+[source,js]
+---------------------------------------------------------
+Pattern p = /[aeiou]/
+---------------------------------------------------------
+
+Patterns can only be created via this mechanism. This ensures fast performance, regular expressions
+are always constants and compiled efficiently a single time. 
+
+[float]
+[[modules-scripting-painless-regex-flags]]
+==== Pattern flags
+
+You can define flags on patterns in Painless by adding characters after the
+trailing `/` like `/foo/i` or `/foo \w #comment/iUx`. Painless exposes all the
+flags from
+https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html[Java's Pattern class]
+using these characters:
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+| Character | Java Constant | Example
+|`c` | CANON_EQ                  | `'Ã¥' ==~ /aÌŠ/c` (open in hex editor to see)
+|`i` | CASE_INSENSITIVE          | `'A' ==~ /a/i`
+|`l` | LITERAL                   | `'[a]' ==~ /[a]/l`
+|`m` | MULTILINE                 | `'a\nb\nc' =~ /^b$/m`
+|`s` | DOTALL (aka single line)  | `'a\nb\nc' =~ /.b./s`
+|`U` | UNICODE_CHARACTER_CLASS   | `'�' ==~ /\\w/U`
+|`u` | UNICODE_CASE              | `'�' ==~ /ɛ/iu`
+|`x` | COMMENTS (aka extended)   | `'a' ==~ /a #comment/x`
+|=======================================================================
+
+[float]
+[[painless-operators]]
+=== Operators
+
+All of Java's https://docs.oracle.com/javase/tutorial/java/nutsandbolts/operators.html[operators] are
+supported with the same precedence, promotion, and semantics. 
+
+There are only a few minor differences and add-ons:
+
+* `==` behaves as Java's for numeric types, but for non-numeric types acts as https://docs.oracle.com/javase/8/docs/api/java/lang/Object.html#equals-java.lang.Object-[`Object.equals()`]
+* `===` and `!==` support exact reference comparison (e.g. `x === y`)
+* `=~` true if a portion of the text matches a pattern (e.g. `x =~ /b/`)
+* `==~` true if the entire text matches a pattern (e.g. `x ==~ /[Bb]ob/`)
+
+[float]
+[[painless-control-flow]]
+=== Control flow
+
+Java's https://docs.oracle.com/javase/tutorial/java/nutsandbolts/flow.html[control flow statements] are supported, with the exception
+of the `switch` statement.
+
+[float]
+[[painless-functions]]
+=== Functions
+
+Functions can be declared at the beginning of the script, for example:
+
+[source,js]
+---------------------------------------------------------
+boolean isNegative(def x) { x < 0 }
+...
+if (isNegative(someVar)) {
+  ...
+}
+---------------------------------------------------------
+
+[float]
+[[painless-lambda-expressions]]
+=== Lambda expressions
+Lambda expressions and method references work the same as https://docs.oracle.com/javase/tutorial/java/javaOO/lambdaexpressions.html[Java's].
+
+[source,js]
+---------------------------------------------------------
+list.removeIf(item -> item == 2);
+list.removeIf((int item) -> item == 2);
+list.removeIf((int item) -> { item == 2 });
+list.sort((x, y) -> x - y);
+list.sort(Integer::compare);
+---------------------------------------------------------
+
+Method references to functions within the script can be accomplished using `this`, e.g. `list.sort(this::mycompare)`.
diff --git a/docs/reference/modules/scripting/painless.asciidoc b/docs/reference/modules/scripting/painless.asciidoc
index 2c753e09039..db68db2c7a3 100644
--- a/docs/reference/modules/scripting/painless.asciidoc
+++ b/docs/reference/modules/scripting/painless.asciidoc
@@ -8,9 +8,7 @@ by default. It is designed specifically for use with Elasticsearch and can
 safely be used with `inline` and `stored` scripting, which is enabled by
 default.
 
-A Painless script is essentially a single function. Painless does not provide support
-for defining multiple functions within a script. The Painless syntax is similar to
-http://groovy-lang.org/index.html[Groovy].
+The Painless syntax is similar to http://groovy-lang.org/index.html[Groovy].
 
 You can use Painless anywhere a script can be used in Elasticsearch--simply set the `lang` parameter
 to `painless`.
@@ -19,22 +17,15 @@ to `painless`.
 [float]
 == Painless Features
 
-* Control flow: `for` loops, `while` loops, `do/while` loops, `if/else`
+* Fast performance: https://benchmarks.elastic.co/index.html#search_qps_scripts[several times faster] than the alternatives.
 
-* Fully Typed: all available types/methods described in <<painless-api, Painless API>>
+* Safety: Fine-grained <<painless-api, whitelist>> with method call/field granularity.
 
-* Arithmetic operators: multiplication `*`, division `/`, addition `+`, subtraction `-`, precedence `( )`
+* Optional typing: Variables and parameters can use explicit types or the dynamic `def` type.
 
-* Comparison operators: less than `<`, less than or equal to `<=`, greater than `>`, greater than or equal to `>=`, equal to `==`, and not equal to `!=`, reference equals `===`, reference not equals `!==`
-
-* Boolean operators: not `!`, and `&&`, or `||`
-
-* Bitwise operators: shift left `<<`, shift right `>>`, unsigned shift `>>>`, and `&`, or `|`, xor `^`, not `~`
-
-* Shortcuts for list, map access using the dot `.` operator
-
-* Native support for regular expressions with `/pattern/`, `=~`, and `==~`
+* Syntax: Extends Java's syntax with a subset of Groovy for ease of use. See the <<modules-scripting-painless-syntax, Syntax Overview>>.
 
+* Optimizations: Designed specifically for Elasticsearch scripting.
 
 [[painless-examples]]
 [float]
@@ -74,7 +65,7 @@ PUT hockey/player/_bulk?refresh
 [float]
 === Accessing Doc Values from Painless
 
-Document values can be accessed from a `Map<String,def>` named `doc`.
+Document values can be accessed from a `Map` named `doc`.
 
 For example, the following script calculates a player's total goals. This example uses a strongly typed `int` and a `for` loop.
 
@@ -268,528 +259,27 @@ Note: all of the `_update_by_query` examples above could really do with a
 because script queries aren't able to use the inverted index to limit the
 documents that they have to check.
 
-We intentionally don't allow scripts to call `Pattern.compile` to get a new
-pattern on the fly because building a `Pattern` is (comparatively) slow.
-Pattern literals (`/apattern/`) have fancy constant extraction so no matter
-where they show up in the painless script they are built only when the script
-is first used. It is fairly similar to how `String` literals work in Java.
-
 [float]
-[[modules-scripting-painless-regex-flags]]
-==== Regular expression flags
-
-You can define flags on patterns in Painless by adding characters after the
-trailing `/` like `/foo/i` or `/foo \w #comment/iUx`. Painless exposes all the
-flags from
-https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html[Java's Pattern class]
-using these characters:
-
-[cols="<,<,<",options="header",]
-|=======================================================================
-| Character | Java Constant | Example
-|`c` | CANON_EQ                  | `'Ã¥' ==~ /aÌŠ/c` (open in hex editor to see)
-|`i` | CASE_INSENSITIVE          | `'A' ==~ /a/i`
-|`l` | LITERAL                   | `'[a]' ==~ /[a]/l`
-|`m` | MULTILINE                 | `'a\nb\nc' =~ /^b$/m`
-|`s` | DOTALL (aka single line)  | `'a\nb\nc' =~ /.b./s`
-|`U` | UNICODE_CHARACTER_CLASS   | `'�' ==~ /\\w/U`
-|`u` | UNICODE_CASE              | `'�' ==~ /ɛ/iu`
-|`x` | COMMENTS (aka extended)   | `'a' ==~ /a #comment/x`
-|=======================================================================
-
-
 [[painless-api]]
-[float]
 == Painless API
 
-The following types are available for use in the Painless language. Most types and methods map directly to their Java equivalents--for more information, see the corresponding https://docs.oracle.com/javase/8/docs/api/java/lang/package-summary.html[Javadoc].
+The following Java packages are available for use in the Painless language:
 
+* https://docs.oracle.com/javase/8/docs/api/java/lang/package-summary.html[java.lang] 
+* https://docs.oracle.com/javase/8/docs/api/java/math/package-summary.html[java.math] 
+* https://docs.oracle.com/javase/8/docs/api/java/text/package-summary.html[java.text] 
+* https://docs.oracle.com/javase/8/docs/api/java/time/package-summary.html[java.time] 
+* https://docs.oracle.com/javase/8/docs/api/java/time/chrono/package-summary.html[java.time.chrono] 
+* https://docs.oracle.com/javase/8/docs/api/java/time/format/package-summary.html[java.time.format] 
+* https://docs.oracle.com/javase/8/docs/api/java/time/temporal/package-summary.html[java.time.temporal] 
+* https://docs.oracle.com/javase/8/docs/api/java/time/zone/package-summary.html[java.time.zone] 
+* https://docs.oracle.com/javase/8/docs/api/java/util/package-summary.html[java.util] 
+* https://docs.oracle.com/javase/8/docs/api/java/util/function/package-summary.html[java.util.function] 
+* https://docs.oracle.com/javase/8/docs/api/java/util/regex/package-summary.html[java.util.regex] 
+* https://docs.oracle.com/javase/8/docs/api/java/util/stream/package-summary.html[java.util.stream] 
 
-[float]
-=== Dynamic Types
+Note that unsafe classes and methods are not included, there is no support for:
 
-* `def` (This type can be used to represent any other type.)
-
-[float]
-=== Basic Types
-
-* `void`
-* `boolean`
-* `short`
-* `char`
-* `int`
-* `long`
-* `float`
-* `double`
-
-[float]
-=== Complex Types
-
-Non-static methods/members in superclasses are available to subclasses.
-Generic types with unspecified generic parameters are parameters of type `def`.
-
------
-ArithmeticException extends Exception
-   <init>()
------
-
------
-ArrayList extends List
-   <init>()
------
-
------
-ArrayList<Object> extends List<Object>
-   <init>()
------
-
------
-ArrayList<String> extends List<String>
-    <init>()
------
-
------
-Boolean extends Object
-   <init>(boolean)
-   static Boolean valueOf(boolean)
-   boolean booleanValue()
------
-
------
-Character extends Object
-    <init>(char)
-    static Character valueOf(char)
-    char charValue()
-    static char MIN_VALUE
-    static char MAX_VALUE
------
-
------
-CharSequence extends Object
-    char charAt(int)
-    int length()
------
-
------
-Collection extends Object
-    boolean add(def)
-    void clear()
-    boolean contains(def)
-    boolean isEmpty()
-    Iterator iterator()
-    boolean remove(def)
-    int size()
------
-
------
-Collection<Object> extends Object
-    boolean add(Object)
-    void clear()
-    boolean contains(Object)
-    boolean isEmpty()
-    Iterator iterator()
-    boolean remove(Object)
-    int size()
------
-
------
-Collection<String> extends Object
-    boolean add(String)
-    void clear()
-    boolean contains(String)
-    boolean isEmpty()
-    Iterator iterator()
-    boolean remove(String)
-    int size()
------
-
------
-Double extends Number
-    <init>(double)
-    static Double valueOf(double)
-    static double MIN_VALUE
-    static double MAX_VALUE
------
-
------
-Exception extends Object
-    String getMessage()
------
-
------
-Float extends Number
-    <init>(float)
-    static Float valueOf(float)
-    static float MIN_VALUE
-    static float MAX_VALUE
------
-
------
-HashMap extends Map
-    <init>()
------
-
------
-HashMap<Object,Object> extends Map<Object,Object>
-    <init>()
------
-
------
-HashMap<String,def> extends Map<String,def>
-    <init>()
------
-
------
-HashMap<String,Object> extends Map<String,Object>
-    <init>()
------
-
------
-IllegalArgument extends Exception
-    <init>()
------
-
------
-IllegalState extends Exception
-    <init>()
------
-
------
-Integer extends Number
-    <init>(int)
-    static Integer valueOf(int)
-    static int MIN_VALUE
-    static int MAX_VALUE
------
-
------
-Iterator extends Object
-    boolean hasNext()
-    def next()
-    void remove()
------
-
------
-Iterator<String> extends Object
-    boolean hasNext()
-    String next()
-    void remove()
------
-
------
-List extends Collection
-    def set(int, def)
-    def get(int)
-    def remove(int)
------
-
------
-List<Object> extends Collection
-    Object set(int, Object)
-    Object get(int)
-    Object remove(int)
------
-
------
-List<String> extends Collection
-    String set(int, String)
-    String get(int)
-    String remove(int)
------
-
------
-Long extends Number
-    <init>(long)
-    static Long valueOf(long)
-    static long MIN_VALUE
-    static long MAX_VALUE
------
-
------
-Map extends Object
-    def put (def, def)
-    def get (def)
-    def remove (def)
-    boolean isEmpty()
-    int size()
-    boolean containsKey(def)
-    boolean containsValue(def)
-    Set keySet()
-    Collection values()
------
-
------
-Map<Object,Object> extends Object
-    Object put (Object, Object)
-    Object get (Object)
-    Object remove (Object)
-    boolean isEmpty()
-    int size()
-    boolean containsKey(Object)
-    boolean containsValue(Object)
-    Set keySet()
-    Collection values()
------
-
------
-Map<String,def> extends Object
-    def put (String, def)
-    def get (String)
-    def remove (String)
-    boolean isEmpty()
-    int size()
-    boolean containsKey(String)
-    boolean containsValue(def)
-    Set<String> keySet()
-    Collection values()
------
-
------
-Map<String,Object> extends Object
-    Object put (String, Object)
-    Object get (String)
-    Object remove (String)
-    boolean isEmpty()
-    int size()
-    boolean containsKey(String)
-    boolean containsValue(Object)
-    Set<String> keySet()
-    Collection values()
------
-
------
-Number extends Object
-    short shortValue()
-    short shortValue()
-    int intValue()
-    long longValue()
-    float floatValue()
-    double doubleValue()
------
-
------
-Object
-    String toString()
-    boolean equals(Object)
-    int hashCode()
------
-
------
-Set extends Collection
------
-
------
-Set<Object> extends Collection<Object>
------
-
------
-Set<String> extends Collection<String>
------
-
------
-Short extends Number
-    <init>(short)
-    static Short valueOf(short)
-    static short MIN_VALUE
-    static short MAX_VALUE
------
-
------
-String extends CharSequence
-    <init>(String)
-    int codePointAt(int)
-    int compareTo(String)
-    String concat(String)
-    boolean endsWith(String)
-    int indexOf(String, int)
-    boolean isEmpty()
-    String replace(CharSequence, CharSequence)
-    boolean startsWith(String)
-    String substring(int, int)
-    char[] toCharArray()
-    String trim()
------
-
------
-NumberFormatException extends Exception
-    <init>()
------
-
------
-Void extends Object
------
-
-[float]
-==== Utility Classes
-
------
-Math
-   static double abs(double)
-   static float fabs(float)
-   static long labs(long)
-   static int iabs(int)
-   static double acos(double)
-   static double asin(double)
-   static double atan(double)
-   static double atan2(double)
-   static double cbrt(double)
-   static double ceil(double)
-   static double cos(double)
-   static double cosh(double)
-   static double exp(double)
-   static double expm1(double)
-   static double floor(double)
-   static double hypt(double, double)
-   static double abs(double)
-   static double log(double)
-   static double log10(double)
-   static double log1p(double)
-   static double max(double, double)
-   static float fmax(float, float)
-   static long lmax(long, long)
-   static int imax(int, int)
-   static double min(double, double)
-   static float fmin(float, float)
-   static long lmin(long, long)
-   static int imin(int, int)
-   static double pow(double, double)
-   static double random()
-   static double rint(double)
-   static long round(double)
-   static double sin(double)
-   static double sinh(double)
-   static double sqrt(double)
-   static double tan(double)
-   static double tanh(double)
-   static double toDegrees(double)
-   static double toRadians(double)
------
-
------
-Utility
-   static boolean NumberToboolean(Number)
-   static char NumberTochar(Number)
-   static Boolean NumberToBoolean(Number)
-   static Short NumberToShort(Number)
-   static Character NumberToCharacter(Number)
-   static Integer NumberToInteger(Number)
-   static Long NumberToLong(Number)
-   static Float NumberToFloat(Number)
-   static Double NumberToDouble(Number)
-   static byte booleanTobyte(boolean)
-   static short booleanToshort(boolean)
-   static char booleanTochar(boolean)
-   static int booleanToint(boolean)
-   static long booleanTolong(boolean)
-   static float booleanTofloat(boolean)
-   static double booleanTodouble(boolean)
-   static Integer booleanToInteger(boolean)
-   static byte BooleanTobyte(Boolean)
-   static short BooleanToshort(Boolean)
-   static char BooleanTochar(Boolean)
-   static int BooleanToint(Boolean)
-   static long BooleanTolong(Boolean)
-   static float BooleanTofloat(Boolean)
-   static double BooleanTodouble(Boolean)
-   static Byte BooleanToByte(Boolean)
-   static Short BooleanToShort(Boolean)
-   static Character BooleanToCharacter(Boolean)
-   static Integer BooleanToInteger(Boolean)
-   static Long BooleanToLong(Boolean)
-   static Float BooleanToFloat(Boolean)
-   static Double BooleanToDouble(Boolean)
-   static boolean byteToboolean(byte)
-   static Short byteToShort(byte)
-   static Character byteToCharacter(byte)
-   static Integer byteToInteger(byte)
-   static Long byteToLong(byte)
-   static Float byteToFloat(byte)
-   static Double byteToDouble(byte)
-   static boolean ByteToboolean(Byte)
-   static char ByteTochar(Byte)
-   static boolean shortToboolean(short)
-   static Byte shortToByte(short)
-   static Character shortToCharacter(short)
-   static Integer shortToInteger(short)
-   static Long shortToLong(short)
-   static Float shortToFloat(short)
-   static Double shortToDouble(short)
-   static boolean ShortToboolean(Short)
-   static char ShortTochar(Short)
-   static boolean charToboolean(char)
-   static Byte charToByte(char)
-   static Short charToShort(char)
-   static Integer charToInteger(char)
-   static Long charToLong(char)
-   static Float charToFloat(char)
-   static Double charToDouble(char)
-   static boolean CharacterToboolean(Character)
-   static byte CharacterTobyte(Character)
-   static short CharacterToshort(Character)
-   static int CharacterToint(Character)
-   static long CharacterTolong(Character)
-   static float CharacterTofloat(Character)
-   static double CharacterTodouble(Character)
-   static Boolean CharacterToBoolean(Character)
-   static Byte CharacterToByte(Character)
-   static Short CharacterToShort(Character)
-   static Integer CharacterToInteger(Character)
-   static Long CharacterToLong(Character)
-   static Float CharacterToFloat(Character)
-   static Double CharacterToDouble(Character)
-   static boolean intToboolean(int)
-   static Byte intToByte(int)
-   static Short intToShort(int)
-   static Character intToCharacter(int)
-   static Long intToLong(int)
-   static Float intToFloat(int)
-   static Double intToDouble(int)
-   static boolean IntegerToboolean(Integer)
-   static char IntegerTochar(Integer)
-   static boolean longToboolean(long)
-   static Byte longToByte(long)
-   static Short longToShort(long)
-   static Character longToCharacter(long)
-   static Integer longToInteger(long)
-   static Float longToFloat(long)
-   static Double longToDouble(long)
-   static boolean LongToboolean(Long)
-   static char LongTochar(Long)
-   static boolean floatToboolean(float)
-   static Byte floatToByte(float)
-   static Short floatToShort(float)
-   static Character floatToCharacter(float)
-   static Integer floatToInteger(float)
-   static Long floatToLong(float)
-   static Double floatToDouble(float)
-   static boolean FloatToboolean(Float)
-   static char FloatTochar(Float)
-   static boolean doubleToboolean(double)
-   static Byte doubleToByte(double)
-   static Short doubleToShort(double)
-   static Character doubleToCharacter(double)
-   static Integer doubleToInteger(double)
-   static Long doubleToLong(double)
-   static Float doubleToFloat(double)
-   static boolean DoubleToboolean(Double)
-   static char DoubleTochar(Double)
------
-
------
-Def
-    static boolean defToboolean(def)
-    static byte defTobyte(def)
-    static short defToshort(def)
-    static char defTochar(def)
-    static int defToint(def)
-    static long defTolong(def)
-    static float defTofloat(def)
-    static double defTodouble(def)
-    static Boolean defToBoolean(def)
-    static Byte defToByte(def)
-    static Character defToCharacter(def)
-    static Integer defToInteger(def)
-    static Long defToLong(def)
-    static Float defToFloat(def)
-    static Double defToDouble(def)
------
+* Manipulation of processes and threads
+* Input/Output
+* Reflection
diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.g4 b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
index ca469eed69a..027b1b370d1 100644
--- a/modules/lang-painless/src/main/antlr/PainlessLexer.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
@@ -52,6 +52,7 @@ TRY:       'try';
 CATCH:     'catch';
 THROW:     'throw';
 THIS:      'this';
+INSTANCEOF: 'instanceof';
 
 BOOLNOT: '!';
 BWNOT:   '~';
diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.tokens b/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
index 3968464f015..8bb8c7e4448 100644
--- a/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
@@ -22,62 +22,63 @@ TRY=21
 CATCH=22
 THROW=23
 THIS=24
-BOOLNOT=25
-BWNOT=26
-MUL=27
-DIV=28
-REM=29
-ADD=30
-SUB=31
-LSH=32
-RSH=33
-USH=34
-LT=35
-LTE=36
-GT=37
-GTE=38
-EQ=39
-EQR=40
-NE=41
-NER=42
-BWAND=43
-XOR=44
-BWOR=45
-BOOLAND=46
-BOOLOR=47
-COND=48
-COLON=49
-REF=50
-ARROW=51
-FIND=52
-MATCH=53
-INCR=54
-DECR=55
-ASSIGN=56
-AADD=57
-ASUB=58
-AMUL=59
-ADIV=60
-AREM=61
-AAND=62
-AXOR=63
-AOR=64
-ALSH=65
-ARSH=66
-AUSH=67
-OCTAL=68
-HEX=69
-INTEGER=70
-DECIMAL=71
-STRING=72
-REGEX=73
-TRUE=74
-FALSE=75
-NULL=76
-TYPE=77
-ID=78
-DOTINTEGER=79
-DOTID=80
+INSTANCEOF=25
+BOOLNOT=26
+BWNOT=27
+MUL=28
+DIV=29
+REM=30
+ADD=31
+SUB=32
+LSH=33
+RSH=34
+USH=35
+LT=36
+LTE=37
+GT=38
+GTE=39
+EQ=40
+EQR=41
+NE=42
+NER=43
+BWAND=44
+XOR=45
+BWOR=46
+BOOLAND=47
+BOOLOR=48
+COND=49
+COLON=50
+REF=51
+ARROW=52
+FIND=53
+MATCH=54
+INCR=55
+DECR=56
+ASSIGN=57
+AADD=58
+ASUB=59
+AMUL=60
+ADIV=61
+AREM=62
+AAND=63
+AXOR=64
+AOR=65
+ALSH=66
+ARSH=67
+AUSH=68
+OCTAL=69
+HEX=70
+INTEGER=71
+DECIMAL=72
+STRING=73
+REGEX=74
+TRUE=75
+FALSE=76
+NULL=77
+TYPE=78
+ID=79
+DOTINTEGER=80
+DOTID=81
 '{'=3
 '}'=4
 '['=5
@@ -100,49 +101,50 @@ DOTID=80
 'catch'=22
 'throw'=23
 'this'=24
-'!'=25
-'~'=26
-'*'=27
-'/'=28
-'%'=29
-'+'=30
-'-'=31
-'<<'=32
-'>>'=33
-'>>>'=34
-'<'=35
-'<='=36
-'>'=37
-'>='=38
-'=='=39
-'==='=40
-'!='=41
-'!=='=42
-'&'=43
-'^'=44
-'|'=45
-'&&'=46
-'||'=47
-'?'=48
-':'=49
-'::'=50
-'->'=51
-'=~'=52
-'==~'=53
-'++'=54
-'--'=55
-'='=56
-'+='=57
-'-='=58
-'*='=59
-'/='=60
-'%='=61
-'&='=62
-'^='=63
-'|='=64
-'<<='=65
-'>>='=66
-'>>>='=67
-'true'=74
-'false'=75
-'null'=76
+'instanceof'=25
+'!'=26
+'~'=27
+'*'=28
+'/'=29
+'%'=30
+'+'=31
+'-'=32
+'<<'=33
+'>>'=34
+'>>>'=35
+'<'=36
+'<='=37
+'>'=38
+'>='=39
+'=='=40
+'==='=41
+'!='=42
+'!=='=43
+'&'=44
+'^'=45
+'|'=46
+'&&'=47
+'||'=48
+'?'=49
+':'=50
+'::'=51
+'->'=52
+'=~'=53
+'==~'=54
+'++'=55
+'--'=56
+'='=57
+'+='=58
+'-='=59
+'*='=60
+'/='=61
+'%='=62
+'&='=63
+'^='=64
+'|='=65
+'<<='=66
+'>>='=67
+'>>>='=68
+'true'=75
+'false'=76
+'null'=77
diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index 10471056acd..7e28b0ad258 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -105,6 +105,7 @@ expression returns [boolean s = true]
     |               expression ( FIND | MATCH ) expression                 { $s = false; }          # binary
     |               expression ( LSH | RSH | USH ) expression              { $s = false; }          # binary
     |               expression ( LT | LTE | GT | GTE ) expression          { $s = false; }          # comp
+    |               expression INSTANCEOF decltype                         { $s = false; }          # instanceof
     |               expression ( EQ | EQR | NE | NER ) expression          { $s = false; }          # comp
     |               expression BWAND expression                            { $s = false; }          # binary
     |               expression XOR expression                              { $s = false; }          # binary
diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.tokens b/modules/lang-painless/src/main/antlr/PainlessParser.tokens
index 3968464f015..8bb8c7e4448 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.tokens
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.tokens
@@ -22,62 +22,63 @@ TRY=21
 CATCH=22
 THROW=23
 THIS=24
-BOOLNOT=25
-BWNOT=26
-MUL=27
-DIV=28
-REM=29
-ADD=30
-SUB=31
-LSH=32
-RSH=33
-USH=34
-LT=35
-LTE=36
-GT=37
-GTE=38
-EQ=39
-EQR=40
-NE=41
-NER=42
-BWAND=43
-XOR=44
-BWOR=45
-BOOLAND=46
-BOOLOR=47
-COND=48
-COLON=49
-REF=50
-ARROW=51
-FIND=52
-MATCH=53
-INCR=54
-DECR=55
-ASSIGN=56
-AADD=57
-ASUB=58
-AMUL=59
-ADIV=60
-AREM=61
-AAND=62
-AXOR=63
-AOR=64
-ALSH=65
-ARSH=66
-AUSH=67
-OCTAL=68
-HEX=69
-INTEGER=70
-DECIMAL=71
-STRING=72
-REGEX=73
-TRUE=74
-FALSE=75
-NULL=76
-TYPE=77
-ID=78
-DOTINTEGER=79
-DOTID=80
+INSTANCEOF=25
+BOOLNOT=26
+BWNOT=27
+MUL=28
+DIV=29
+REM=30
+ADD=31
+SUB=32
+LSH=33
+RSH=34
+USH=35
+LT=36
+LTE=37
+GT=38
+GTE=39
+EQ=40
+EQR=41
+NE=42
+NER=43
+BWAND=44
+XOR=45
+BWOR=46
+BOOLAND=47
+BOOLOR=48
+COND=49
+COLON=50
+REF=51
+ARROW=52
+FIND=53
+MATCH=54
+INCR=55
+DECR=56
+ASSIGN=57
+AADD=58
+ASUB=59
+AMUL=60
+ADIV=61
+AREM=62
+AAND=63
+AXOR=64
+AOR=65
+ALSH=66
+ARSH=67
+AUSH=68
+OCTAL=69
+HEX=70
+INTEGER=71
+DECIMAL=72
+STRING=73
+REGEX=74
+TRUE=75
+FALSE=76
+NULL=77
+TYPE=78
+ID=79
+DOTINTEGER=80
+DOTID=81
 '{'=3
 '}'=4
 '['=5
@@ -100,49 +101,50 @@ DOTID=80
 'catch'=22
 'throw'=23
 'this'=24
-'!'=25
-'~'=26
-'*'=27
-'/'=28
-'%'=29
-'+'=30
-'-'=31
-'<<'=32
-'>>'=33
-'>>>'=34
-'<'=35
-'<='=36
-'>'=37
-'>='=38
-'=='=39
-'==='=40
-'!='=41
-'!=='=42
-'&'=43
-'^'=44
-'|'=45
-'&&'=46
-'||'=47
-'?'=48
-':'=49
-'::'=50
-'->'=51
-'=~'=52
-'==~'=53
-'++'=54
-'--'=55
-'='=56
-'+='=57
-'-='=58
-'*='=59
-'/='=60
-'%='=61
-'&='=62
-'^='=63
-'|='=64
-'<<='=65
-'>>='=66
-'>>>='=67
-'true'=74
-'false'=75
-'null'=76
+'instanceof'=25
+'!'=26
+'~'=27
+'*'=28
+'/'=29
+'%'=30
+'+'=31
+'-'=32
+'<<'=33
+'>>'=34
+'>>>'=35
+'<'=36
+'<='=37
+'>'=38
+'>='=39
+'=='=40
+'==='=41
+'!='=42
+'!=='=43
+'&'=44
+'^'=45
+'|'=46
+'&&'=47
+'||'=48
+'?'=49
+':'=50
+'::'=51
+'->'=52
+'=~'=53
+'==~'=54
+'++'=55
+'--'=56
+'='=57
+'+='=58
+'-='=59
+'*='=60
+'/='=61
+'%='=62
+'&='=63
+'^='=64
+'|='=65
+'<<='=66
+'>>='=67
+'>>>='=68
+'true'=75
+'false'=76
+'null'=77
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Augmentation.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Augmentation.java
index 4bca673b4dc..edd0eeab67f 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Augmentation.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Augmentation.java
@@ -64,6 +64,30 @@ public class Augmentation {
         return false;
     }
     
+    /** Converts this Iterable to a Collection. Returns the original Iterable if it is already a Collection. */
+    public static <T> Collection<T> asCollection(Iterable<T> receiver) {
+        if (receiver instanceof Collection) {
+            return (Collection<T>)receiver;
+        }
+        List<T> list = new ArrayList<>();
+        for (T t : receiver) {
+            list.add(t);
+        }
+        return list;
+    }
+    
+    /** Converts this Iterable to a List. Returns the original Iterable if it is already a List. */
+    public static <T> List<T> asList(Iterable<T> receiver) {
+        if (receiver instanceof List) {
+            return (List<T>)receiver;
+        }
+        List<T> list = new ArrayList<>();
+        for (T t : receiver) {
+            list.add(t);
+        }
+        return list;
+    }
+    
     /** Counts the number of occurrences which satisfy the given predicate from inside this Iterable. */ 
     public static <T> int count(Iterable<T> receiver, Predicate<T> predicate) {
         int count = 0;
@@ -155,6 +179,17 @@ public class Augmentation {
         return sb.toString();
     }
     
+    /**
+     * Sums the result of an Iterable
+     */
+    public static <T extends Number> double sum(Iterable<T> receiver) {
+        double sum = 0;
+        for (T t : receiver) {
+            sum += t.doubleValue();
+        }
+        return sum;
+    }
+    
     /**
      * Sums the result of applying a function to each item of an Iterable. 
      */
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
index f512ad20d7e..6d791979243 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
@@ -22,14 +22,14 @@ class PainlessLexer extends Lexer {
   public static final int
     WS=1, COMMENT=2, LBRACK=3, RBRACK=4, LBRACE=5, RBRACE=6, LP=7, RP=8, DOT=9, 
     COMMA=10, SEMICOLON=11, IF=12, ELSE=13, WHILE=14, DO=15, FOR=16, CONTINUE=17, 
-    BREAK=18, RETURN=19, NEW=20, TRY=21, CATCH=22, THROW=23, THIS=24, BOOLNOT=25, 
-    BWNOT=26, MUL=27, DIV=28, REM=29, ADD=30, SUB=31, LSH=32, RSH=33, USH=34, 
-    LT=35, LTE=36, GT=37, GTE=38, EQ=39, EQR=40, NE=41, NER=42, BWAND=43, 
-    XOR=44, BWOR=45, BOOLAND=46, BOOLOR=47, COND=48, COLON=49, REF=50, ARROW=51, 
-    FIND=52, MATCH=53, INCR=54, DECR=55, ASSIGN=56, AADD=57, ASUB=58, AMUL=59, 
-    ADIV=60, AREM=61, AAND=62, AXOR=63, AOR=64, ALSH=65, ARSH=66, AUSH=67, 
-    OCTAL=68, HEX=69, INTEGER=70, DECIMAL=71, STRING=72, REGEX=73, TRUE=74, 
-    FALSE=75, NULL=76, TYPE=77, ID=78, DOTINTEGER=79, DOTID=80;
+    BREAK=18, RETURN=19, NEW=20, TRY=21, CATCH=22, THROW=23, THIS=24, INSTANCEOF=25, 
+    BOOLNOT=26, BWNOT=27, MUL=28, DIV=29, REM=30, ADD=31, SUB=32, LSH=33, 
+    RSH=34, USH=35, LT=36, LTE=37, GT=38, GTE=39, EQ=40, EQR=41, NE=42, NER=43, 
+    BWAND=44, XOR=45, BWOR=46, BOOLAND=47, BOOLOR=48, COND=49, COLON=50, REF=51, 
+    ARROW=52, FIND=53, MATCH=54, INCR=55, DECR=56, ASSIGN=57, AADD=58, ASUB=59, 
+    AMUL=60, ADIV=61, AREM=62, AAND=63, AXOR=64, AOR=65, ALSH=66, ARSH=67, 
+    AUSH=68, OCTAL=69, HEX=70, INTEGER=71, DECIMAL=72, STRING=73, REGEX=74, 
+    TRUE=75, FALSE=76, NULL=77, TYPE=78, ID=79, DOTINTEGER=80, DOTID=81;
   public static final int AFTER_DOT = 1;
   public static String[] modeNames = {
     "DEFAULT_MODE", "AFTER_DOT"
@@ -38,9 +38,9 @@ class PainlessLexer extends Lexer {
   public static final String[] ruleNames = {
     "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", "DOT", 
     "COMMA", "SEMICOLON", "IF", "ELSE", "WHILE", "DO", "FOR", "CONTINUE", 
-    "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", "BOOLNOT", 
-    "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", 
-    "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", 
+    "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", "INSTANCEOF", 
+    "BOOLNOT", "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", 
+    "LT", "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", 
     "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "FIND", "MATCH", 
     "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", "AAND", 
     "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", "DECIMAL", 
@@ -51,20 +51,20 @@ class PainlessLexer extends Lexer {
   private static final String[] _LITERAL_NAMES = {
     null, null, null, "'{'", "'}'", "'['", "']'", "'('", "')'", "'.'", "','", 
     "';'", "'if'", "'else'", "'while'", "'do'", "'for'", "'continue'", "'break'", 
-    "'return'", "'new'", "'try'", "'catch'", "'throw'", "'this'", "'!'", "'~'", 
-    "'*'", "'/'", "'%'", "'+'", "'-'", "'<<'", "'>>'", "'>>>'", "'<'", "'<='", 
-    "'>'", "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", "'^'", "'|'", 
-    "'&&'", "'||'", "'?'", "':'", "'::'", "'->'", "'=~'", "'==~'", "'++'", 
-    "'--'", "'='", "'+='", "'-='", "'*='", "'/='", "'%='", "'&='", "'^='", 
-    "'|='", "'<<='", "'>>='", "'>>>='", null, null, null, null, null, null, 
-    "'true'", "'false'", "'null'"
+    "'return'", "'new'", "'try'", "'catch'", "'throw'", "'this'", "'instanceof'", 
+    "'!'", "'~'", "'*'", "'/'", "'%'", "'+'", "'-'", "'<<'", "'>>'", "'>>>'", 
+    "'<'", "'<='", "'>'", "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", 
+    "'^'", "'|'", "'&&'", "'||'", "'?'", "':'", "'::'", "'->'", "'=~'", "'==~'", 
+    "'++'", "'--'", "'='", "'+='", "'-='", "'*='", "'/='", "'%='", "'&='", 
+    "'^='", "'|='", "'<<='", "'>>='", "'>>>='", null, null, null, null, null, 
+    null, "'true'", "'false'", "'null'"
   };
   private static final String[] _SYMBOLIC_NAMES = {
     null, "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", 
     "DOT", "COMMA", "SEMICOLON", "IF", "ELSE", "WHILE", "DO", "FOR", "CONTINUE", 
-    "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", "BOOLNOT", 
-    "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", 
-    "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", 
+    "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", "INSTANCEOF", 
+    "BOOLNOT", "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", 
+    "LT", "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", 
     "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "FIND", "MATCH", 
     "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", "AAND", 
     "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", "DECIMAL", 
@@ -128,11 +128,11 @@ class PainlessLexer extends Lexer {
   @Override
   public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) {
     switch (ruleIndex) {
-    case 27:
+    case 28:
       return DIV_sempred((RuleContext)_localctx, predIndex);
-    case 72:
+    case 73:
       return REGEX_sempred((RuleContext)_localctx, predIndex);
-    case 76:
+    case 77:
       return TYPE_sempred((RuleContext)_localctx, predIndex);
     }
     return true;
@@ -160,7 +160,7 @@ class PainlessLexer extends Lexer {
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2R\u0239\b\1\b\1\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2S\u0246\b\1\b\1\4"+
     "\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n"+
     "\4\13\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22"+
     "\t\22\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31"+
@@ -169,196 +169,201 @@ class PainlessLexer extends Lexer {
     "+\4,\t,\4-\t-\4.\t.\4/\t/\4\60\t\60\4\61\t\61\4\62\t\62\4\63\t\63\4\64"+
     "\t\64\4\65\t\65\4\66\t\66\4\67\t\67\48\t8\49\t9\4:\t:\4;\t;\4<\t<\4=\t"+
     "=\4>\t>\4?\t?\4@\t@\4A\tA\4B\tB\4C\tC\4D\tD\4E\tE\4F\tF\4G\tG\4H\tH\4"+
-    "I\tI\4J\tJ\4K\tK\4L\tL\4M\tM\4N\tN\4O\tO\4P\tP\4Q\tQ\3\2\6\2\u00a6\n\2"+
-    "\r\2\16\2\u00a7\3\2\3\2\3\3\3\3\3\3\3\3\7\3\u00b0\n\3\f\3\16\3\u00b3\13"+
-    "\3\3\3\3\3\3\3\3\3\3\3\7\3\u00ba\n\3\f\3\16\3\u00bd\13\3\3\3\3\3\5\3\u00c1"+
-    "\n\3\3\3\3\3\3\4\3\4\3\5\3\5\3\6\3\6\3\7\3\7\3\b\3\b\3\t\3\t\3\n\3\n\3"+
-    "\n\3\n\3\13\3\13\3\f\3\f\3\r\3\r\3\r\3\16\3\16\3\16\3\16\3\16\3\17\3\17"+
-    "\3\17\3\17\3\17\3\17\3\20\3\20\3\20\3\21\3\21\3\21\3\21\3\22\3\22\3\22"+
-    "\3\22\3\22\3\22\3\22\3\22\3\22\3\23\3\23\3\23\3\23\3\23\3\23\3\24\3\24"+
-    "\3\24\3\24\3\24\3\24\3\24\3\25\3\25\3\25\3\25\3\26\3\26\3\26\3\26\3\27"+
-    "\3\27\3\27\3\27\3\27\3\27\3\30\3\30\3\30\3\30\3\30\3\30\3\31\3\31\3\31"+
-    "\3\31\3\31\3\32\3\32\3\33\3\33\3\34\3\34\3\35\3\35\3\35\3\36\3\36\3\37"+
-    "\3\37\3 \3 \3!\3!\3!\3\"\3\"\3\"\3#\3#\3#\3#\3$\3$\3%\3%\3%\3&\3&\3\'"+
-    "\3\'\3\'\3(\3(\3(\3)\3)\3)\3)\3*\3*\3*\3+\3+\3+\3+\3,\3,\3-\3-\3.\3.\3"+
-    "/\3/\3/\3\60\3\60\3\60\3\61\3\61\3\62\3\62\3\63\3\63\3\63\3\64\3\64\3"+
-    "\64\3\65\3\65\3\65\3\66\3\66\3\66\3\66\3\67\3\67\3\67\38\38\38\39\39\3"+
-    ":\3:\3:\3;\3;\3;\3<\3<\3<\3=\3=\3=\3>\3>\3>\3?\3?\3?\3@\3@\3@\3A\3A\3"+
-    "A\3B\3B\3B\3B\3C\3C\3C\3C\3D\3D\3D\3D\3D\3E\3E\6E\u019a\nE\rE\16E\u019b"+
-    "\3E\5E\u019f\nE\3F\3F\3F\6F\u01a4\nF\rF\16F\u01a5\3F\5F\u01a9\nF\3G\3"+
-    "G\3G\7G\u01ae\nG\fG\16G\u01b1\13G\5G\u01b3\nG\3G\5G\u01b6\nG\3H\3H\3H"+
-    "\7H\u01bb\nH\fH\16H\u01be\13H\5H\u01c0\nH\3H\3H\6H\u01c4\nH\rH\16H\u01c5"+
-    "\5H\u01c8\nH\3H\3H\5H\u01cc\nH\3H\6H\u01cf\nH\rH\16H\u01d0\5H\u01d3\n"+
-    "H\3H\5H\u01d6\nH\3I\3I\3I\3I\3I\3I\7I\u01de\nI\fI\16I\u01e1\13I\3I\3I"+
-    "\3I\3I\3I\3I\3I\7I\u01ea\nI\fI\16I\u01ed\13I\3I\5I\u01f0\nI\3J\3J\3J\3"+
-    "J\6J\u01f6\nJ\rJ\16J\u01f7\3J\3J\7J\u01fc\nJ\fJ\16J\u01ff\13J\3J\3J\3"+
-    "K\3K\3K\3K\3K\3L\3L\3L\3L\3L\3L\3M\3M\3M\3M\3M\3N\3N\3N\3N\7N\u0217\n"+
-    "N\fN\16N\u021a\13N\3N\3N\3O\3O\7O\u0220\nO\fO\16O\u0223\13O\3P\3P\3P\7"+
-    "P\u0228\nP\fP\16P\u022b\13P\5P\u022d\nP\3P\3P\3Q\3Q\7Q\u0233\nQ\fQ\16"+
-    "Q\u0236\13Q\3Q\3Q\6\u00b1\u00bb\u01df\u01eb\2R\4\3\6\4\b\5\n\6\f\7\16"+
-    "\b\20\t\22\n\24\13\26\f\30\r\32\16\34\17\36\20 \21\"\22$\23&\24(\25*\26"+
-    ",\27.\30\60\31\62\32\64\33\66\348\35:\36<\37> @!B\"D#F$H%J&L\'N(P)R*T"+
-    "+V,X-Z.\\/^\60`\61b\62d\63f\64h\65j\66l\67n8p9r:t;v<x=z>|?~@\u0080A\u0082"+
-    "B\u0084C\u0086D\u0088E\u008aF\u008cG\u008eH\u0090I\u0092J\u0094K\u0096"+
-    "L\u0098M\u009aN\u009cO\u009eP\u00a0Q\u00a2R\4\2\3\24\5\2\13\f\17\17\""+
-    "\"\4\2\f\f\17\17\3\2\629\4\2NNnn\4\2ZZzz\5\2\62;CHch\3\2\63;\3\2\62;\b"+
-    "\2FFHHNNffhhnn\4\2GGgg\4\2--//\4\2HHhh\4\2$$^^\4\2\f\f\61\61\3\2\f\f\t"+
-    "\2WWeekknouuwwzz\5\2C\\aac|\6\2\62;C\\aac|\u0259\2\4\3\2\2\2\2\6\3\2\2"+
-    "\2\2\b\3\2\2\2\2\n\3\2\2\2\2\f\3\2\2\2\2\16\3\2\2\2\2\20\3\2\2\2\2\22"+
-    "\3\2\2\2\2\24\3\2\2\2\2\26\3\2\2\2\2\30\3\2\2\2\2\32\3\2\2\2\2\34\3\2"+
-    "\2\2\2\36\3\2\2\2\2 \3\2\2\2\2\"\3\2\2\2\2$\3\2\2\2\2&\3\2\2\2\2(\3\2"+
-    "\2\2\2*\3\2\2\2\2,\3\2\2\2\2.\3\2\2\2\2\60\3\2\2\2\2\62\3\2\2\2\2\64\3"+
-    "\2\2\2\2\66\3\2\2\2\28\3\2\2\2\2:\3\2\2\2\2<\3\2\2\2\2>\3\2\2\2\2@\3\2"+
-    "\2\2\2B\3\2\2\2\2D\3\2\2\2\2F\3\2\2\2\2H\3\2\2\2\2J\3\2\2\2\2L\3\2\2\2"+
-    "\2N\3\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2T\3\2\2\2\2V\3\2\2\2\2X\3\2\2\2\2Z"+
-    "\3\2\2\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3\2\2\2\2b\3\2\2\2\2d\3\2\2\2\2f\3"+
-    "\2\2\2\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2\2\2n\3\2\2\2\2p\3\2\2\2\2r\3\2\2"+
-    "\2\2t\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2z\3\2\2\2\2|\3\2\2\2\2~\3\2\2\2\2"+
-    "\u0080\3\2\2\2\2\u0082\3\2\2\2\2\u0084\3\2\2\2\2\u0086\3\2\2\2\2\u0088"+
-    "\3\2\2\2\2\u008a\3\2\2\2\2\u008c\3\2\2\2\2\u008e\3\2\2\2\2\u0090\3\2\2"+
-    "\2\2\u0092\3\2\2\2\2\u0094\3\2\2\2\2\u0096\3\2\2\2\2\u0098\3\2\2\2\2\u009a"+
-    "\3\2\2\2\2\u009c\3\2\2\2\2\u009e\3\2\2\2\3\u00a0\3\2\2\2\3\u00a2\3\2\2"+
-    "\2\4\u00a5\3\2\2\2\6\u00c0\3\2\2\2\b\u00c4\3\2\2\2\n\u00c6\3\2\2\2\f\u00c8"+
-    "\3\2\2\2\16\u00ca\3\2\2\2\20\u00cc\3\2\2\2\22\u00ce\3\2\2\2\24\u00d0\3"+
-    "\2\2\2\26\u00d4\3\2\2\2\30\u00d6\3\2\2\2\32\u00d8\3\2\2\2\34\u00db\3\2"+
-    "\2\2\36\u00e0\3\2\2\2 \u00e6\3\2\2\2\"\u00e9\3\2\2\2$\u00ed\3\2\2\2&\u00f6"+
-    "\3\2\2\2(\u00fc\3\2\2\2*\u0103\3\2\2\2,\u0107\3\2\2\2.\u010b\3\2\2\2\60"+
-    "\u0111\3\2\2\2\62\u0117\3\2\2\2\64\u011c\3\2\2\2\66\u011e\3\2\2\28\u0120"+
-    "\3\2\2\2:\u0122\3\2\2\2<\u0125\3\2\2\2>\u0127\3\2\2\2@\u0129\3\2\2\2B"+
-    "\u012b\3\2\2\2D\u012e\3\2\2\2F\u0131\3\2\2\2H\u0135\3\2\2\2J\u0137\3\2"+
-    "\2\2L\u013a\3\2\2\2N\u013c\3\2\2\2P\u013f\3\2\2\2R\u0142\3\2\2\2T\u0146"+
-    "\3\2\2\2V\u0149\3\2\2\2X\u014d\3\2\2\2Z\u014f\3\2\2\2\\\u0151\3\2\2\2"+
-    "^\u0153\3\2\2\2`\u0156\3\2\2\2b\u0159\3\2\2\2d\u015b\3\2\2\2f\u015d\3"+
-    "\2\2\2h\u0160\3\2\2\2j\u0163\3\2\2\2l\u0166\3\2\2\2n\u016a\3\2\2\2p\u016d"+
-    "\3\2\2\2r\u0170\3\2\2\2t\u0172\3\2\2\2v\u0175\3\2\2\2x\u0178\3\2\2\2z"+
-    "\u017b\3\2\2\2|\u017e\3\2\2\2~\u0181\3\2\2\2\u0080\u0184\3\2\2\2\u0082"+
-    "\u0187\3\2\2\2\u0084\u018a\3\2\2\2\u0086\u018e\3\2\2\2\u0088\u0192\3\2"+
-    "\2\2\u008a\u0197\3\2\2\2\u008c\u01a0\3\2\2\2\u008e\u01b2\3\2\2\2\u0090"+
-    "\u01bf\3\2\2\2\u0092\u01ef\3\2\2\2\u0094\u01f1\3\2\2\2\u0096\u0202\3\2"+
-    "\2\2\u0098\u0207\3\2\2\2\u009a\u020d\3\2\2\2\u009c\u0212\3\2\2\2\u009e"+
-    "\u021d\3\2\2\2\u00a0\u022c\3\2\2\2\u00a2\u0230\3\2\2\2\u00a4\u00a6\t\2"+
-    "\2\2\u00a5\u00a4\3\2\2\2\u00a6\u00a7\3\2\2\2\u00a7\u00a5\3\2\2\2\u00a7"+
-    "\u00a8\3\2\2\2\u00a8\u00a9\3\2\2\2\u00a9\u00aa\b\2\2\2\u00aa\5\3\2\2\2"+
-    "\u00ab\u00ac\7\61\2\2\u00ac\u00ad\7\61\2\2\u00ad\u00b1\3\2\2\2\u00ae\u00b0"+
-    "\13\2\2\2\u00af\u00ae\3\2\2\2\u00b0\u00b3\3\2\2\2\u00b1\u00b2\3\2\2\2"+
-    "\u00b1\u00af\3\2\2\2\u00b2\u00b4\3\2\2\2\u00b3\u00b1\3\2\2\2\u00b4\u00c1"+
-    "\t\3\2\2\u00b5\u00b6\7\61\2\2\u00b6\u00b7\7,\2\2\u00b7\u00bb\3\2\2\2\u00b8"+
-    "\u00ba\13\2\2\2\u00b9\u00b8\3\2\2\2\u00ba\u00bd\3\2\2\2\u00bb\u00bc\3"+
-    "\2\2\2\u00bb\u00b9\3\2\2\2\u00bc\u00be\3\2\2\2\u00bd\u00bb\3\2\2\2\u00be"+
-    "\u00bf\7,\2\2\u00bf\u00c1\7\61\2\2\u00c0\u00ab\3\2\2\2\u00c0\u00b5\3\2"+
-    "\2\2\u00c1\u00c2\3\2\2\2\u00c2\u00c3\b\3\2\2\u00c3\7\3\2\2\2\u00c4\u00c5"+
-    "\7}\2\2\u00c5\t\3\2\2\2\u00c6\u00c7\7\177\2\2\u00c7\13\3\2\2\2\u00c8\u00c9"+
-    "\7]\2\2\u00c9\r\3\2\2\2\u00ca\u00cb\7_\2\2\u00cb\17\3\2\2\2\u00cc\u00cd"+
-    "\7*\2\2\u00cd\21\3\2\2\2\u00ce\u00cf\7+\2\2\u00cf\23\3\2\2\2\u00d0\u00d1"+
-    "\7\60\2\2\u00d1\u00d2\3\2\2\2\u00d2\u00d3\b\n\3\2\u00d3\25\3\2\2\2\u00d4"+
-    "\u00d5\7.\2\2\u00d5\27\3\2\2\2\u00d6\u00d7\7=\2\2\u00d7\31\3\2\2\2\u00d8"+
-    "\u00d9\7k\2\2\u00d9\u00da\7h\2\2\u00da\33\3\2\2\2\u00db\u00dc\7g\2\2\u00dc"+
-    "\u00dd\7n\2\2\u00dd\u00de\7u\2\2\u00de\u00df\7g\2\2\u00df\35\3\2\2\2\u00e0"+
-    "\u00e1\7y\2\2\u00e1\u00e2\7j\2\2\u00e2\u00e3\7k\2\2\u00e3\u00e4\7n\2\2"+
-    "\u00e4\u00e5\7g\2\2\u00e5\37\3\2\2\2\u00e6\u00e7\7f\2\2\u00e7\u00e8\7"+
-    "q\2\2\u00e8!\3\2\2\2\u00e9\u00ea\7h\2\2\u00ea\u00eb\7q\2\2\u00eb\u00ec"+
-    "\7t\2\2\u00ec#\3\2\2\2\u00ed\u00ee\7e\2\2\u00ee\u00ef\7q\2\2\u00ef\u00f0"+
-    "\7p\2\2\u00f0\u00f1\7v\2\2\u00f1\u00f2\7k\2\2\u00f2\u00f3\7p\2\2\u00f3"+
-    "\u00f4\7w\2\2\u00f4\u00f5\7g\2\2\u00f5%\3\2\2\2\u00f6\u00f7\7d\2\2\u00f7"+
-    "\u00f8\7t\2\2\u00f8\u00f9\7g\2\2\u00f9\u00fa\7c\2\2\u00fa\u00fb\7m\2\2"+
-    "\u00fb\'\3\2\2\2\u00fc\u00fd\7t\2\2\u00fd\u00fe\7g\2\2\u00fe\u00ff\7v"+
-    "\2\2\u00ff\u0100\7w\2\2\u0100\u0101\7t\2\2\u0101\u0102\7p\2\2\u0102)\3"+
-    "\2\2\2\u0103\u0104\7p\2\2\u0104\u0105\7g\2\2\u0105\u0106\7y\2\2\u0106"+
-    "+\3\2\2\2\u0107\u0108\7v\2\2\u0108\u0109\7t\2\2\u0109\u010a\7{\2\2\u010a"+
-    "-\3\2\2\2\u010b\u010c\7e\2\2\u010c\u010d\7c\2\2\u010d\u010e\7v\2\2\u010e"+
-    "\u010f\7e\2\2\u010f\u0110\7j\2\2\u0110/\3\2\2\2\u0111\u0112\7v\2\2\u0112"+
-    "\u0113\7j\2\2\u0113\u0114\7t\2\2\u0114\u0115\7q\2\2\u0115\u0116\7y\2\2"+
-    "\u0116\61\3\2\2\2\u0117\u0118\7v\2\2\u0118\u0119\7j\2\2\u0119\u011a\7"+
-    "k\2\2\u011a\u011b\7u\2\2\u011b\63\3\2\2\2\u011c\u011d\7#\2\2\u011d\65"+
-    "\3\2\2\2\u011e\u011f\7\u0080\2\2\u011f\67\3\2\2\2\u0120\u0121\7,\2\2\u0121"+
-    "9\3\2\2\2\u0122\u0123\7\61\2\2\u0123\u0124\6\35\2\2\u0124;\3\2\2\2\u0125"+
-    "\u0126\7\'\2\2\u0126=\3\2\2\2\u0127\u0128\7-\2\2\u0128?\3\2\2\2\u0129"+
-    "\u012a\7/\2\2\u012aA\3\2\2\2\u012b\u012c\7>\2\2\u012c\u012d\7>\2\2\u012d"+
-    "C\3\2\2\2\u012e\u012f\7@\2\2\u012f\u0130\7@\2\2\u0130E\3\2\2\2\u0131\u0132"+
-    "\7@\2\2\u0132\u0133\7@\2\2\u0133\u0134\7@\2\2\u0134G\3\2\2\2\u0135\u0136"+
-    "\7>\2\2\u0136I\3\2\2\2\u0137\u0138\7>\2\2\u0138\u0139\7?\2\2\u0139K\3"+
-    "\2\2\2\u013a\u013b\7@\2\2\u013bM\3\2\2\2\u013c\u013d\7@\2\2\u013d\u013e"+
-    "\7?\2\2\u013eO\3\2\2\2\u013f\u0140\7?\2\2\u0140\u0141\7?\2\2\u0141Q\3"+
-    "\2\2\2\u0142\u0143\7?\2\2\u0143\u0144\7?\2\2\u0144\u0145\7?\2\2\u0145"+
-    "S\3\2\2\2\u0146\u0147\7#\2\2\u0147\u0148\7?\2\2\u0148U\3\2\2\2\u0149\u014a"+
-    "\7#\2\2\u014a\u014b\7?\2\2\u014b\u014c\7?\2\2\u014cW\3\2\2\2\u014d\u014e"+
-    "\7(\2\2\u014eY\3\2\2\2\u014f\u0150\7`\2\2\u0150[\3\2\2\2\u0151\u0152\7"+
-    "~\2\2\u0152]\3\2\2\2\u0153\u0154\7(\2\2\u0154\u0155\7(\2\2\u0155_\3\2"+
-    "\2\2\u0156\u0157\7~\2\2\u0157\u0158\7~\2\2\u0158a\3\2\2\2\u0159\u015a"+
-    "\7A\2\2\u015ac\3\2\2\2\u015b\u015c\7<\2\2\u015ce\3\2\2\2\u015d\u015e\7"+
-    "<\2\2\u015e\u015f\7<\2\2\u015fg\3\2\2\2\u0160\u0161\7/\2\2\u0161\u0162"+
-    "\7@\2\2\u0162i\3\2\2\2\u0163\u0164\7?\2\2\u0164\u0165\7\u0080\2\2\u0165"+
-    "k\3\2\2\2\u0166\u0167\7?\2\2\u0167\u0168\7?\2\2\u0168\u0169\7\u0080\2"+
-    "\2\u0169m\3\2\2\2\u016a\u016b\7-\2\2\u016b\u016c\7-\2\2\u016co\3\2\2\2"+
-    "\u016d\u016e\7/\2\2\u016e\u016f\7/\2\2\u016fq\3\2\2\2\u0170\u0171\7?\2"+
-    "\2\u0171s\3\2\2\2\u0172\u0173\7-\2\2\u0173\u0174\7?\2\2\u0174u\3\2\2\2"+
-    "\u0175\u0176\7/\2\2\u0176\u0177\7?\2\2\u0177w\3\2\2\2\u0178\u0179\7,\2"+
-    "\2\u0179\u017a\7?\2\2\u017ay\3\2\2\2\u017b\u017c\7\61\2\2\u017c\u017d"+
-    "\7?\2\2\u017d{\3\2\2\2\u017e\u017f\7\'\2\2\u017f\u0180\7?\2\2\u0180}\3"+
-    "\2\2\2\u0181\u0182\7(\2\2\u0182\u0183\7?\2\2\u0183\177\3\2\2\2\u0184\u0185"+
-    "\7`\2\2\u0185\u0186\7?\2\2\u0186\u0081\3\2\2\2\u0187\u0188\7~\2\2\u0188"+
-    "\u0189\7?\2\2\u0189\u0083\3\2\2\2\u018a\u018b\7>\2\2\u018b\u018c\7>\2"+
-    "\2\u018c\u018d\7?\2\2\u018d\u0085\3\2\2\2\u018e\u018f\7@\2\2\u018f\u0190"+
-    "\7@\2\2\u0190\u0191\7?\2\2\u0191\u0087\3\2\2\2\u0192\u0193\7@\2\2\u0193"+
-    "\u0194\7@\2\2\u0194\u0195\7@\2\2\u0195\u0196\7?\2\2\u0196\u0089\3\2\2"+
-    "\2\u0197\u0199\7\62\2\2\u0198\u019a\t\4\2\2\u0199\u0198\3\2\2\2\u019a"+
-    "\u019b\3\2\2\2\u019b\u0199\3\2\2\2\u019b\u019c\3\2\2\2\u019c\u019e\3\2"+
-    "\2\2\u019d\u019f\t\5\2\2\u019e\u019d\3\2\2\2\u019e\u019f\3\2\2\2\u019f"+
-    "\u008b\3\2\2\2\u01a0\u01a1\7\62\2\2\u01a1\u01a3\t\6\2\2\u01a2\u01a4\t"+
-    "\7\2\2\u01a3\u01a2\3\2\2\2\u01a4\u01a5\3\2\2\2\u01a5\u01a3\3\2\2\2\u01a5"+
-    "\u01a6\3\2\2\2\u01a6\u01a8\3\2\2\2\u01a7\u01a9\t\5\2\2\u01a8\u01a7\3\2"+
-    "\2\2\u01a8\u01a9\3\2\2\2\u01a9\u008d\3\2\2\2\u01aa\u01b3\7\62\2\2\u01ab"+
-    "\u01af\t\b\2\2\u01ac\u01ae\t\t\2\2\u01ad\u01ac\3\2\2\2\u01ae\u01b1\3\2"+
-    "\2\2\u01af\u01ad\3\2\2\2\u01af\u01b0\3\2\2\2\u01b0\u01b3\3\2\2\2\u01b1"+
-    "\u01af\3\2\2\2\u01b2\u01aa\3\2\2\2\u01b2\u01ab\3\2\2\2\u01b3\u01b5\3\2"+
-    "\2\2\u01b4\u01b6\t\n\2\2\u01b5\u01b4\3\2\2\2\u01b5\u01b6\3\2\2\2\u01b6"+
+    "I\tI\4J\tJ\4K\tK\4L\tL\4M\tM\4N\tN\4O\tO\4P\tP\4Q\tQ\4R\tR\3\2\6\2\u00a8"+
+    "\n\2\r\2\16\2\u00a9\3\2\3\2\3\3\3\3\3\3\3\3\7\3\u00b2\n\3\f\3\16\3\u00b5"+
+    "\13\3\3\3\3\3\3\3\3\3\3\3\7\3\u00bc\n\3\f\3\16\3\u00bf\13\3\3\3\3\3\5"+
+    "\3\u00c3\n\3\3\3\3\3\3\4\3\4\3\5\3\5\3\6\3\6\3\7\3\7\3\b\3\b\3\t\3\t\3"+
+    "\n\3\n\3\n\3\n\3\13\3\13\3\f\3\f\3\r\3\r\3\r\3\16\3\16\3\16\3\16\3\16"+
+    "\3\17\3\17\3\17\3\17\3\17\3\17\3\20\3\20\3\20\3\21\3\21\3\21\3\21\3\22"+
+    "\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\23\3\23\3\23\3\23\3\23\3\23"+
+    "\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\25\3\25\3\25\3\25\3\26\3\26\3\26"+
+    "\3\26\3\27\3\27\3\27\3\27\3\27\3\27\3\30\3\30\3\30\3\30\3\30\3\30\3\31"+
+    "\3\31\3\31\3\31\3\31\3\32\3\32\3\32\3\32\3\32\3\32\3\32\3\32\3\32\3\32"+
+    "\3\32\3\33\3\33\3\34\3\34\3\35\3\35\3\36\3\36\3\36\3\37\3\37\3 \3 \3!"+
+    "\3!\3\"\3\"\3\"\3#\3#\3#\3$\3$\3$\3$\3%\3%\3&\3&\3&\3\'\3\'\3(\3(\3(\3"+
+    ")\3)\3)\3*\3*\3*\3*\3+\3+\3+\3,\3,\3,\3,\3-\3-\3.\3.\3/\3/\3\60\3\60\3"+
+    "\60\3\61\3\61\3\61\3\62\3\62\3\63\3\63\3\64\3\64\3\64\3\65\3\65\3\65\3"+
+    "\66\3\66\3\66\3\67\3\67\3\67\3\67\38\38\38\39\39\39\3:\3:\3;\3;\3;\3<"+
+    "\3<\3<\3=\3=\3=\3>\3>\3>\3?\3?\3?\3@\3@\3@\3A\3A\3A\3B\3B\3B\3C\3C\3C"+
+    "\3C\3D\3D\3D\3D\3E\3E\3E\3E\3E\3F\3F\6F\u01a7\nF\rF\16F\u01a8\3F\5F\u01ac"+
+    "\nF\3G\3G\3G\6G\u01b1\nG\rG\16G\u01b2\3G\5G\u01b6\nG\3H\3H\3H\7H\u01bb"+
+    "\nH\fH\16H\u01be\13H\5H\u01c0\nH\3H\5H\u01c3\nH\3I\3I\3I\7I\u01c8\nI\f"+
+    "I\16I\u01cb\13I\5I\u01cd\nI\3I\3I\6I\u01d1\nI\rI\16I\u01d2\5I\u01d5\n"+
+    "I\3I\3I\5I\u01d9\nI\3I\6I\u01dc\nI\rI\16I\u01dd\5I\u01e0\nI\3I\5I\u01e3"+
+    "\nI\3J\3J\3J\3J\3J\3J\7J\u01eb\nJ\fJ\16J\u01ee\13J\3J\3J\3J\3J\3J\3J\3"+
+    "J\7J\u01f7\nJ\fJ\16J\u01fa\13J\3J\5J\u01fd\nJ\3K\3K\3K\3K\6K\u0203\nK"+
+    "\rK\16K\u0204\3K\3K\7K\u0209\nK\fK\16K\u020c\13K\3K\3K\3L\3L\3L\3L\3L"+
+    "\3M\3M\3M\3M\3M\3M\3N\3N\3N\3N\3N\3O\3O\3O\3O\7O\u0224\nO\fO\16O\u0227"+
+    "\13O\3O\3O\3P\3P\7P\u022d\nP\fP\16P\u0230\13P\3Q\3Q\3Q\7Q\u0235\nQ\fQ"+
+    "\16Q\u0238\13Q\5Q\u023a\nQ\3Q\3Q\3R\3R\7R\u0240\nR\fR\16R\u0243\13R\3"+
+    "R\3R\6\u00b3\u00bd\u01ec\u01f8\2S\4\3\6\4\b\5\n\6\f\7\16\b\20\t\22\n\24"+
+    "\13\26\f\30\r\32\16\34\17\36\20 \21\"\22$\23&\24(\25*\26,\27.\30\60\31"+
+    "\62\32\64\33\66\348\35:\36<\37> @!B\"D#F$H%J&L\'N(P)R*T+V,X-Z.\\/^\60"+
+    "`\61b\62d\63f\64h\65j\66l\67n8p9r:t;v<x=z>|?~@\u0080A\u0082B\u0084C\u0086"+
+    "D\u0088E\u008aF\u008cG\u008eH\u0090I\u0092J\u0094K\u0096L\u0098M\u009a"+
+    "N\u009cO\u009eP\u00a0Q\u00a2R\u00a4S\4\2\3\24\5\2\13\f\17\17\"\"\4\2\f"+
+    "\f\17\17\3\2\629\4\2NNnn\4\2ZZzz\5\2\62;CHch\3\2\63;\3\2\62;\b\2FFHHN"+
+    "Nffhhnn\4\2GGgg\4\2--//\4\2HHhh\4\2$$^^\4\2\f\f\61\61\3\2\f\f\t\2WWee"+
+    "kknouuwwzz\5\2C\\aac|\6\2\62;C\\aac|\u0266\2\4\3\2\2\2\2\6\3\2\2\2\2\b"+
+    "\3\2\2\2\2\n\3\2\2\2\2\f\3\2\2\2\2\16\3\2\2\2\2\20\3\2\2\2\2\22\3\2\2"+
+    "\2\2\24\3\2\2\2\2\26\3\2\2\2\2\30\3\2\2\2\2\32\3\2\2\2\2\34\3\2\2\2\2"+
+    "\36\3\2\2\2\2 \3\2\2\2\2\"\3\2\2\2\2$\3\2\2\2\2&\3\2\2\2\2(\3\2\2\2\2"+
+    "*\3\2\2\2\2,\3\2\2\2\2.\3\2\2\2\2\60\3\2\2\2\2\62\3\2\2\2\2\64\3\2\2\2"+
+    "\2\66\3\2\2\2\28\3\2\2\2\2:\3\2\2\2\2<\3\2\2\2\2>\3\2\2\2\2@\3\2\2\2\2"+
+    "B\3\2\2\2\2D\3\2\2\2\2F\3\2\2\2\2H\3\2\2\2\2J\3\2\2\2\2L\3\2\2\2\2N\3"+
+    "\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2T\3\2\2\2\2V\3\2\2\2\2X\3\2\2\2\2Z\3\2\2"+
+    "\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3\2\2\2\2b\3\2\2\2\2d\3\2\2\2\2f\3\2\2\2"+
+    "\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2\2\2n\3\2\2\2\2p\3\2\2\2\2r\3\2\2\2\2t"+
+    "\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2z\3\2\2\2\2|\3\2\2\2\2~\3\2\2\2\2\u0080"+
+    "\3\2\2\2\2\u0082\3\2\2\2\2\u0084\3\2\2\2\2\u0086\3\2\2\2\2\u0088\3\2\2"+
+    "\2\2\u008a\3\2\2\2\2\u008c\3\2\2\2\2\u008e\3\2\2\2\2\u0090\3\2\2\2\2\u0092"+
+    "\3\2\2\2\2\u0094\3\2\2\2\2\u0096\3\2\2\2\2\u0098\3\2\2\2\2\u009a\3\2\2"+
+    "\2\2\u009c\3\2\2\2\2\u009e\3\2\2\2\2\u00a0\3\2\2\2\3\u00a2\3\2\2\2\3\u00a4"+
+    "\3\2\2\2\4\u00a7\3\2\2\2\6\u00c2\3\2\2\2\b\u00c6\3\2\2\2\n\u00c8\3\2\2"+
+    "\2\f\u00ca\3\2\2\2\16\u00cc\3\2\2\2\20\u00ce\3\2\2\2\22\u00d0\3\2\2\2"+
+    "\24\u00d2\3\2\2\2\26\u00d6\3\2\2\2\30\u00d8\3\2\2\2\32\u00da\3\2\2\2\34"+
+    "\u00dd\3\2\2\2\36\u00e2\3\2\2\2 \u00e8\3\2\2\2\"\u00eb\3\2\2\2$\u00ef"+
+    "\3\2\2\2&\u00f8\3\2\2\2(\u00fe\3\2\2\2*\u0105\3\2\2\2,\u0109\3\2\2\2."+
+    "\u010d\3\2\2\2\60\u0113\3\2\2\2\62\u0119\3\2\2\2\64\u011e\3\2\2\2\66\u0129"+
+    "\3\2\2\28\u012b\3\2\2\2:\u012d\3\2\2\2<\u012f\3\2\2\2>\u0132\3\2\2\2@"+
+    "\u0134\3\2\2\2B\u0136\3\2\2\2D\u0138\3\2\2\2F\u013b\3\2\2\2H\u013e\3\2"+
+    "\2\2J\u0142\3\2\2\2L\u0144\3\2\2\2N\u0147\3\2\2\2P\u0149\3\2\2\2R\u014c"+
+    "\3\2\2\2T\u014f\3\2\2\2V\u0153\3\2\2\2X\u0156\3\2\2\2Z\u015a\3\2\2\2\\"+
+    "\u015c\3\2\2\2^\u015e\3\2\2\2`\u0160\3\2\2\2b\u0163\3\2\2\2d\u0166\3\2"+
+    "\2\2f\u0168\3\2\2\2h\u016a\3\2\2\2j\u016d\3\2\2\2l\u0170\3\2\2\2n\u0173"+
+    "\3\2\2\2p\u0177\3\2\2\2r\u017a\3\2\2\2t\u017d\3\2\2\2v\u017f\3\2\2\2x"+
+    "\u0182\3\2\2\2z\u0185\3\2\2\2|\u0188\3\2\2\2~\u018b\3\2\2\2\u0080\u018e"+
+    "\3\2\2\2\u0082\u0191\3\2\2\2\u0084\u0194\3\2\2\2\u0086\u0197\3\2\2\2\u0088"+
+    "\u019b\3\2\2\2\u008a\u019f\3\2\2\2\u008c\u01a4\3\2\2\2\u008e\u01ad\3\2"+
+    "\2\2\u0090\u01bf\3\2\2\2\u0092\u01cc\3\2\2\2\u0094\u01fc\3\2\2\2\u0096"+
+    "\u01fe\3\2\2\2\u0098\u020f\3\2\2\2\u009a\u0214\3\2\2\2\u009c\u021a\3\2"+
+    "\2\2\u009e\u021f\3\2\2\2\u00a0\u022a\3\2\2\2\u00a2\u0239\3\2\2\2\u00a4"+
+    "\u023d\3\2\2\2\u00a6\u00a8\t\2\2\2\u00a7\u00a6\3\2\2\2\u00a8\u00a9\3\2"+
+    "\2\2\u00a9\u00a7\3\2\2\2\u00a9\u00aa\3\2\2\2\u00aa\u00ab\3\2\2\2\u00ab"+
+    "\u00ac\b\2\2\2\u00ac\5\3\2\2\2\u00ad\u00ae\7\61\2\2\u00ae\u00af\7\61\2"+
+    "\2\u00af\u00b3\3\2\2\2\u00b0\u00b2\13\2\2\2\u00b1\u00b0\3\2\2\2\u00b2"+
+    "\u00b5\3\2\2\2\u00b3\u00b4\3\2\2\2\u00b3\u00b1\3\2\2\2\u00b4\u00b6\3\2"+
+    "\2\2\u00b5\u00b3\3\2\2\2\u00b6\u00c3\t\3\2\2\u00b7\u00b8\7\61\2\2\u00b8"+
+    "\u00b9\7,\2\2\u00b9\u00bd\3\2\2\2\u00ba\u00bc\13\2\2\2\u00bb\u00ba\3\2"+
+    "\2\2\u00bc\u00bf\3\2\2\2\u00bd\u00be\3\2\2\2\u00bd\u00bb\3\2\2\2\u00be"+
+    "\u00c0\3\2\2\2\u00bf\u00bd\3\2\2\2\u00c0\u00c1\7,\2\2\u00c1\u00c3\7\61"+
+    "\2\2\u00c2\u00ad\3\2\2\2\u00c2\u00b7\3\2\2\2\u00c3\u00c4\3\2\2\2\u00c4"+
+    "\u00c5\b\3\2\2\u00c5\7\3\2\2\2\u00c6\u00c7\7}\2\2\u00c7\t\3\2\2\2\u00c8"+
+    "\u00c9\7\177\2\2\u00c9\13\3\2\2\2\u00ca\u00cb\7]\2\2\u00cb\r\3\2\2\2\u00cc"+
+    "\u00cd\7_\2\2\u00cd\17\3\2\2\2\u00ce\u00cf\7*\2\2\u00cf\21\3\2\2\2\u00d0"+
+    "\u00d1\7+\2\2\u00d1\23\3\2\2\2\u00d2\u00d3\7\60\2\2\u00d3\u00d4\3\2\2"+
+    "\2\u00d4\u00d5\b\n\3\2\u00d5\25\3\2\2\2\u00d6\u00d7\7.\2\2\u00d7\27\3"+
+    "\2\2\2\u00d8\u00d9\7=\2\2\u00d9\31\3\2\2\2\u00da\u00db\7k\2\2\u00db\u00dc"+
+    "\7h\2\2\u00dc\33\3\2\2\2\u00dd\u00de\7g\2\2\u00de\u00df\7n\2\2\u00df\u00e0"+
+    "\7u\2\2\u00e0\u00e1\7g\2\2\u00e1\35\3\2\2\2\u00e2\u00e3\7y\2\2\u00e3\u00e4"+
+    "\7j\2\2\u00e4\u00e5\7k\2\2\u00e5\u00e6\7n\2\2\u00e6\u00e7\7g\2\2\u00e7"+
+    "\37\3\2\2\2\u00e8\u00e9\7f\2\2\u00e9\u00ea\7q\2\2\u00ea!\3\2\2\2\u00eb"+
+    "\u00ec\7h\2\2\u00ec\u00ed\7q\2\2\u00ed\u00ee\7t\2\2\u00ee#\3\2\2\2\u00ef"+
+    "\u00f0\7e\2\2\u00f0\u00f1\7q\2\2\u00f1\u00f2\7p\2\2\u00f2\u00f3\7v\2\2"+
+    "\u00f3\u00f4\7k\2\2\u00f4\u00f5\7p\2\2\u00f5\u00f6\7w\2\2\u00f6\u00f7"+
+    "\7g\2\2\u00f7%\3\2\2\2\u00f8\u00f9\7d\2\2\u00f9\u00fa\7t\2\2\u00fa\u00fb"+
+    "\7g\2\2\u00fb\u00fc\7c\2\2\u00fc\u00fd\7m\2\2\u00fd\'\3\2\2\2\u00fe\u00ff"+
+    "\7t\2\2\u00ff\u0100\7g\2\2\u0100\u0101\7v\2\2\u0101\u0102\7w\2\2\u0102"+
+    "\u0103\7t\2\2\u0103\u0104\7p\2\2\u0104)\3\2\2\2\u0105\u0106\7p\2\2\u0106"+
+    "\u0107\7g\2\2\u0107\u0108\7y\2\2\u0108+\3\2\2\2\u0109\u010a\7v\2\2\u010a"+
+    "\u010b\7t\2\2\u010b\u010c\7{\2\2\u010c-\3\2\2\2\u010d\u010e\7e\2\2\u010e"+
+    "\u010f\7c\2\2\u010f\u0110\7v\2\2\u0110\u0111\7e\2\2\u0111\u0112\7j\2\2"+
+    "\u0112/\3\2\2\2\u0113\u0114\7v\2\2\u0114\u0115\7j\2\2\u0115\u0116\7t\2"+
+    "\2\u0116\u0117\7q\2\2\u0117\u0118\7y\2\2\u0118\61\3\2\2\2\u0119\u011a"+
+    "\7v\2\2\u011a\u011b\7j\2\2\u011b\u011c\7k\2\2\u011c\u011d\7u\2\2\u011d"+
+    "\63\3\2\2\2\u011e\u011f\7k\2\2\u011f\u0120\7p\2\2\u0120\u0121\7u\2\2\u0121"+
+    "\u0122\7v\2\2\u0122\u0123\7c\2\2\u0123\u0124\7p\2\2\u0124\u0125\7e\2\2"+
+    "\u0125\u0126\7g\2\2\u0126\u0127\7q\2\2\u0127\u0128\7h\2\2\u0128\65\3\2"+
+    "\2\2\u0129\u012a\7#\2\2\u012a\67\3\2\2\2\u012b\u012c\7\u0080\2\2\u012c"+
+    "9\3\2\2\2\u012d\u012e\7,\2\2\u012e;\3\2\2\2\u012f\u0130\7\61\2\2\u0130"+
+    "\u0131\6\36\2\2\u0131=\3\2\2\2\u0132\u0133\7\'\2\2\u0133?\3\2\2\2\u0134"+
+    "\u0135\7-\2\2\u0135A\3\2\2\2\u0136\u0137\7/\2\2\u0137C\3\2\2\2\u0138\u0139"+
+    "\7>\2\2\u0139\u013a\7>\2\2\u013aE\3\2\2\2\u013b\u013c\7@\2\2\u013c\u013d"+
+    "\7@\2\2\u013dG\3\2\2\2\u013e\u013f\7@\2\2\u013f\u0140\7@\2\2\u0140\u0141"+
+    "\7@\2\2\u0141I\3\2\2\2\u0142\u0143\7>\2\2\u0143K\3\2\2\2\u0144\u0145\7"+
+    ">\2\2\u0145\u0146\7?\2\2\u0146M\3\2\2\2\u0147\u0148\7@\2\2\u0148O\3\2"+
+    "\2\2\u0149\u014a\7@\2\2\u014a\u014b\7?\2\2\u014bQ\3\2\2\2\u014c\u014d"+
+    "\7?\2\2\u014d\u014e\7?\2\2\u014eS\3\2\2\2\u014f\u0150\7?\2\2\u0150\u0151"+
+    "\7?\2\2\u0151\u0152\7?\2\2\u0152U\3\2\2\2\u0153\u0154\7#\2\2\u0154\u0155"+
+    "\7?\2\2\u0155W\3\2\2\2\u0156\u0157\7#\2\2\u0157\u0158\7?\2\2\u0158\u0159"+
+    "\7?\2\2\u0159Y\3\2\2\2\u015a\u015b\7(\2\2\u015b[\3\2\2\2\u015c\u015d\7"+
+    "`\2\2\u015d]\3\2\2\2\u015e\u015f\7~\2\2\u015f_\3\2\2\2\u0160\u0161\7("+
+    "\2\2\u0161\u0162\7(\2\2\u0162a\3\2\2\2\u0163\u0164\7~\2\2\u0164\u0165"+
+    "\7~\2\2\u0165c\3\2\2\2\u0166\u0167\7A\2\2\u0167e\3\2\2\2\u0168\u0169\7"+
+    "<\2\2\u0169g\3\2\2\2\u016a\u016b\7<\2\2\u016b\u016c\7<\2\2\u016ci\3\2"+
+    "\2\2\u016d\u016e\7/\2\2\u016e\u016f\7@\2\2\u016fk\3\2\2\2\u0170\u0171"+
+    "\7?\2\2\u0171\u0172\7\u0080\2\2\u0172m\3\2\2\2\u0173\u0174\7?\2\2\u0174"+
+    "\u0175\7?\2\2\u0175\u0176\7\u0080\2\2\u0176o\3\2\2\2\u0177\u0178\7-\2"+
+    "\2\u0178\u0179\7-\2\2\u0179q\3\2\2\2\u017a\u017b\7/\2\2\u017b\u017c\7"+
+    "/\2\2\u017cs\3\2\2\2\u017d\u017e\7?\2\2\u017eu\3\2\2\2\u017f\u0180\7-"+
+    "\2\2\u0180\u0181\7?\2\2\u0181w\3\2\2\2\u0182\u0183\7/\2\2\u0183\u0184"+
+    "\7?\2\2\u0184y\3\2\2\2\u0185\u0186\7,\2\2\u0186\u0187\7?\2\2\u0187{\3"+
+    "\2\2\2\u0188\u0189\7\61\2\2\u0189\u018a\7?\2\2\u018a}\3\2\2\2\u018b\u018c"+
+    "\7\'\2\2\u018c\u018d\7?\2\2\u018d\177\3\2\2\2\u018e\u018f\7(\2\2\u018f"+
+    "\u0190\7?\2\2\u0190\u0081\3\2\2\2\u0191\u0192\7`\2\2\u0192\u0193\7?\2"+
+    "\2\u0193\u0083\3\2\2\2\u0194\u0195\7~\2\2\u0195\u0196\7?\2\2\u0196\u0085"+
+    "\3\2\2\2\u0197\u0198\7>\2\2\u0198\u0199\7>\2\2\u0199\u019a\7?\2\2\u019a"+
+    "\u0087\3\2\2\2\u019b\u019c\7@\2\2\u019c\u019d\7@\2\2\u019d\u019e\7?\2"+
+    "\2\u019e\u0089\3\2\2\2\u019f\u01a0\7@\2\2\u01a0\u01a1\7@\2\2\u01a1\u01a2"+
+    "\7@\2\2\u01a2\u01a3\7?\2\2\u01a3\u008b\3\2\2\2\u01a4\u01a6\7\62\2\2\u01a5"+
+    "\u01a7\t\4\2\2\u01a6\u01a5\3\2\2\2\u01a7\u01a8\3\2\2\2\u01a8\u01a6\3\2"+
+    "\2\2\u01a8\u01a9\3\2\2\2\u01a9\u01ab\3\2\2\2\u01aa\u01ac\t\5\2\2\u01ab"+
+    "\u01aa\3\2\2\2\u01ab\u01ac\3\2\2\2\u01ac\u008d\3\2\2\2\u01ad\u01ae\7\62"+
+    "\2\2\u01ae\u01b0\t\6\2\2\u01af\u01b1\t\7\2\2\u01b0\u01af\3\2\2\2\u01b1"+
+    "\u01b2\3\2\2\2\u01b2\u01b0\3\2\2\2\u01b2\u01b3\3\2\2\2\u01b3\u01b5\3\2"+
+    "\2\2\u01b4\u01b6\t\5\2\2\u01b5\u01b4\3\2\2\2\u01b5\u01b6\3\2\2\2\u01b6"+
     "\u008f\3\2\2\2\u01b7\u01c0\7\62\2\2\u01b8\u01bc\t\b\2\2\u01b9\u01bb\t"+
     "\t\2\2\u01ba\u01b9\3\2\2\2\u01bb\u01be\3\2\2\2\u01bc\u01ba\3\2\2\2\u01bc"+
     "\u01bd\3\2\2\2\u01bd\u01c0\3\2\2\2\u01be\u01bc\3\2\2\2\u01bf\u01b7\3\2"+
-    "\2\2\u01bf\u01b8\3\2\2\2\u01c0\u01c7\3\2\2\2\u01c1\u01c3\5\24\n\2\u01c2"+
-    "\u01c4\t\t\2\2\u01c3\u01c2\3\2\2\2\u01c4\u01c5\3\2\2\2\u01c5\u01c3\3\2"+
-    "\2\2\u01c5\u01c6\3\2\2\2\u01c6\u01c8\3\2\2\2\u01c7\u01c1\3\2\2\2\u01c7"+
-    "\u01c8\3\2\2\2\u01c8\u01d2\3\2\2\2\u01c9\u01cb\t\13\2\2\u01ca\u01cc\t"+
-    "\f\2\2\u01cb\u01ca\3\2\2\2\u01cb\u01cc\3\2\2\2\u01cc\u01ce\3\2\2\2\u01cd"+
-    "\u01cf\t\t\2\2\u01ce\u01cd\3\2\2\2\u01cf\u01d0\3\2\2\2\u01d0\u01ce\3\2"+
-    "\2\2\u01d0\u01d1\3\2\2\2\u01d1\u01d3\3\2\2\2\u01d2\u01c9\3\2\2\2\u01d2"+
-    "\u01d3\3\2\2\2\u01d3\u01d5\3\2\2\2\u01d4\u01d6\t\r\2\2\u01d5\u01d4\3\2"+
-    "\2\2\u01d5\u01d6\3\2\2\2\u01d6\u0091\3\2\2\2\u01d7\u01df\7$\2\2\u01d8"+
-    "\u01d9\7^\2\2\u01d9\u01de\7$\2\2\u01da\u01db\7^\2\2\u01db\u01de\7^\2\2"+
-    "\u01dc\u01de\n\16\2\2\u01dd\u01d8\3\2\2\2\u01dd\u01da\3\2\2\2\u01dd\u01dc"+
-    "\3\2\2\2\u01de\u01e1\3\2\2\2\u01df\u01e0\3\2\2\2\u01df\u01dd\3\2\2\2\u01e0"+
-    "\u01e2\3\2\2\2\u01e1\u01df\3\2\2\2\u01e2\u01f0\7$\2\2\u01e3\u01eb\7)\2"+
-    "\2\u01e4\u01e5\7^\2\2\u01e5\u01ea\7)\2\2\u01e6\u01e7\7^\2\2\u01e7\u01ea"+
-    "\7^\2\2\u01e8\u01ea\n\16\2\2\u01e9\u01e4\3\2\2\2\u01e9\u01e6\3\2\2\2\u01e9"+
-    "\u01e8\3\2\2\2\u01ea\u01ed\3\2\2\2\u01eb\u01ec\3\2\2\2\u01eb\u01e9\3\2"+
-    "\2\2\u01ec\u01ee\3\2\2\2\u01ed\u01eb\3\2\2\2\u01ee\u01f0\7)\2\2\u01ef"+
-    "\u01d7\3\2\2\2\u01ef\u01e3\3\2\2\2\u01f0\u0093\3\2\2\2\u01f1\u01f5\7\61"+
-    "\2\2\u01f2\u01f6\n\17\2\2\u01f3\u01f4\7^\2\2\u01f4\u01f6\n\20\2\2\u01f5"+
-    "\u01f2\3\2\2\2\u01f5\u01f3\3\2\2\2\u01f6\u01f7\3\2\2\2\u01f7\u01f5\3\2"+
-    "\2\2\u01f7\u01f8\3\2\2\2\u01f8\u01f9\3\2\2\2\u01f9\u01fd\7\61\2\2\u01fa"+
-    "\u01fc\t\21\2\2\u01fb\u01fa\3\2\2\2\u01fc\u01ff\3\2\2\2\u01fd\u01fb\3"+
-    "\2\2\2\u01fd\u01fe\3\2\2\2\u01fe\u0200\3\2\2\2\u01ff\u01fd\3\2\2\2\u0200"+
-    "\u0201\6J\3\2\u0201\u0095\3\2\2\2\u0202\u0203\7v\2\2\u0203\u0204\7t\2"+
-    "\2\u0204\u0205\7w\2\2\u0205\u0206\7g\2\2\u0206\u0097\3\2\2\2\u0207\u0208"+
-    "\7h\2\2\u0208\u0209\7c\2\2\u0209\u020a\7n\2\2\u020a\u020b\7u\2\2\u020b"+
-    "\u020c\7g\2\2\u020c\u0099\3\2\2\2\u020d\u020e\7p\2\2\u020e\u020f\7w\2"+
-    "\2\u020f\u0210\7n\2\2\u0210\u0211\7n\2\2\u0211\u009b\3\2\2\2\u0212\u0218"+
-    "\5\u009eO\2\u0213\u0214\5\24\n\2\u0214\u0215\5\u009eO\2\u0215\u0217\3"+
-    "\2\2\2\u0216\u0213\3\2\2\2\u0217\u021a\3\2\2\2\u0218\u0216\3\2\2\2\u0218"+
-    "\u0219\3\2\2\2\u0219\u021b\3\2\2\2\u021a\u0218\3\2\2\2\u021b\u021c\6N"+
-    "\4\2\u021c\u009d\3\2\2\2\u021d\u0221\t\22\2\2\u021e\u0220\t\23\2\2\u021f"+
-    "\u021e\3\2\2\2\u0220\u0223\3\2\2\2\u0221\u021f\3\2\2\2\u0221\u0222\3\2"+
-    "\2\2\u0222\u009f\3\2\2\2\u0223\u0221\3\2\2\2\u0224\u022d\7\62\2\2\u0225"+
-    "\u0229\t\b\2\2\u0226\u0228\t\t\2\2\u0227\u0226\3\2\2\2\u0228\u022b\3\2"+
-    "\2\2\u0229\u0227\3\2\2\2\u0229\u022a\3\2\2\2\u022a\u022d\3\2\2\2\u022b"+
-    "\u0229\3\2\2\2\u022c\u0224\3\2\2\2\u022c\u0225\3\2\2\2\u022d\u022e\3\2"+
-    "\2\2\u022e\u022f\bP\4\2\u022f\u00a1\3\2\2\2\u0230\u0234\t\22\2\2\u0231"+
-    "\u0233\t\23\2\2\u0232\u0231\3\2\2\2\u0233\u0236\3\2\2\2\u0234\u0232\3"+
-    "\2\2\2\u0234\u0235\3\2\2\2\u0235\u0237\3\2\2\2\u0236\u0234\3\2\2\2\u0237"+
-    "\u0238\bQ\4\2\u0238\u00a3\3\2\2\2$\2\3\u00a7\u00b1\u00bb\u00c0\u019b\u019e"+
-    "\u01a5\u01a8\u01af\u01b2\u01b5\u01bc\u01bf\u01c5\u01c7\u01cb\u01d0\u01d2"+
-    "\u01d5\u01dd\u01df\u01e9\u01eb\u01ef\u01f5\u01f7\u01fd\u0218\u0221\u0229"+
-    "\u022c\u0234\5\b\2\2\4\3\2\4\2\2";
+    "\2\2\u01bf\u01b8\3\2\2\2\u01c0\u01c2\3\2\2\2\u01c1\u01c3\t\n\2\2\u01c2"+
+    "\u01c1\3\2\2\2\u01c2\u01c3\3\2\2\2\u01c3\u0091\3\2\2\2\u01c4\u01cd\7\62"+
+    "\2\2\u01c5\u01c9\t\b\2\2\u01c6\u01c8\t\t\2\2\u01c7\u01c6\3\2\2\2\u01c8"+
+    "\u01cb\3\2\2\2\u01c9\u01c7\3\2\2\2\u01c9\u01ca\3\2\2\2\u01ca\u01cd\3\2"+
+    "\2\2\u01cb\u01c9\3\2\2\2\u01cc\u01c4\3\2\2\2\u01cc\u01c5\3\2\2\2\u01cd"+
+    "\u01d4\3\2\2\2\u01ce\u01d0\5\24\n\2\u01cf\u01d1\t\t\2\2\u01d0\u01cf\3"+
+    "\2\2\2\u01d1\u01d2\3\2\2\2\u01d2\u01d0\3\2\2\2\u01d2\u01d3\3\2\2\2\u01d3"+
+    "\u01d5\3\2\2\2\u01d4\u01ce\3\2\2\2\u01d4\u01d5\3\2\2\2\u01d5\u01df\3\2"+
+    "\2\2\u01d6\u01d8\t\13\2\2\u01d7\u01d9\t\f\2\2\u01d8\u01d7\3\2\2\2\u01d8"+
+    "\u01d9\3\2\2\2\u01d9\u01db\3\2\2\2\u01da\u01dc\t\t\2\2\u01db\u01da\3\2"+
+    "\2\2\u01dc\u01dd\3\2\2\2\u01dd\u01db\3\2\2\2\u01dd\u01de\3\2\2\2\u01de"+
+    "\u01e0\3\2\2\2\u01df\u01d6\3\2\2\2\u01df\u01e0\3\2\2\2\u01e0\u01e2\3\2"+
+    "\2\2\u01e1\u01e3\t\r\2\2\u01e2\u01e1\3\2\2\2\u01e2\u01e3\3\2\2\2\u01e3"+
+    "\u0093\3\2\2\2\u01e4\u01ec\7$\2\2\u01e5\u01e6\7^\2\2\u01e6\u01eb\7$\2"+
+    "\2\u01e7\u01e8\7^\2\2\u01e8\u01eb\7^\2\2\u01e9\u01eb\n\16\2\2\u01ea\u01e5"+
+    "\3\2\2\2\u01ea\u01e7\3\2\2\2\u01ea\u01e9\3\2\2\2\u01eb\u01ee\3\2\2\2\u01ec"+
+    "\u01ed\3\2\2\2\u01ec\u01ea\3\2\2\2\u01ed\u01ef\3\2\2\2\u01ee\u01ec\3\2"+
+    "\2\2\u01ef\u01fd\7$\2\2\u01f0\u01f8\7)\2\2\u01f1\u01f2\7^\2\2\u01f2\u01f7"+
+    "\7)\2\2\u01f3\u01f4\7^\2\2\u01f4\u01f7\7^\2\2\u01f5\u01f7\n\16\2\2\u01f6"+
+    "\u01f1\3\2\2\2\u01f6\u01f3\3\2\2\2\u01f6\u01f5\3\2\2\2\u01f7\u01fa\3\2"+
+    "\2\2\u01f8\u01f9\3\2\2\2\u01f8\u01f6\3\2\2\2\u01f9\u01fb\3\2\2\2\u01fa"+
+    "\u01f8\3\2\2\2\u01fb\u01fd\7)\2\2\u01fc\u01e4\3\2\2\2\u01fc\u01f0\3\2"+
+    "\2\2\u01fd\u0095\3\2\2\2\u01fe\u0202\7\61\2\2\u01ff\u0203\n\17\2\2\u0200"+
+    "\u0201\7^\2\2\u0201\u0203\n\20\2\2\u0202\u01ff\3\2\2\2\u0202\u0200\3\2"+
+    "\2\2\u0203\u0204\3\2\2\2\u0204\u0202\3\2\2\2\u0204\u0205\3\2\2\2\u0205"+
+    "\u0206\3\2\2\2\u0206\u020a\7\61\2\2\u0207\u0209\t\21\2\2\u0208\u0207\3"+
+    "\2\2\2\u0209\u020c\3\2\2\2\u020a\u0208\3\2\2\2\u020a\u020b\3\2\2\2\u020b"+
+    "\u020d\3\2\2\2\u020c\u020a\3\2\2\2\u020d\u020e\6K\3\2\u020e\u0097\3\2"+
+    "\2\2\u020f\u0210\7v\2\2\u0210\u0211\7t\2\2\u0211\u0212\7w\2\2\u0212\u0213"+
+    "\7g\2\2\u0213\u0099\3\2\2\2\u0214\u0215\7h\2\2\u0215\u0216\7c\2\2\u0216"+
+    "\u0217\7n\2\2\u0217\u0218\7u\2\2\u0218\u0219\7g\2\2\u0219\u009b\3\2\2"+
+    "\2\u021a\u021b\7p\2\2\u021b\u021c\7w\2\2\u021c\u021d\7n\2\2\u021d\u021e"+
+    "\7n\2\2\u021e\u009d\3\2\2\2\u021f\u0225\5\u00a0P\2\u0220\u0221\5\24\n"+
+    "\2\u0221\u0222\5\u00a0P\2\u0222\u0224\3\2\2\2\u0223\u0220\3\2\2\2\u0224"+
+    "\u0227\3\2\2\2\u0225\u0223\3\2\2\2\u0225\u0226\3\2\2\2\u0226\u0228\3\2"+
+    "\2\2\u0227\u0225\3\2\2\2\u0228\u0229\6O\4\2\u0229\u009f\3\2\2\2\u022a"+
+    "\u022e\t\22\2\2\u022b\u022d\t\23\2\2\u022c\u022b\3\2\2\2\u022d\u0230\3"+
+    "\2\2\2\u022e\u022c\3\2\2\2\u022e\u022f\3\2\2\2\u022f\u00a1\3\2\2\2\u0230"+
+    "\u022e\3\2\2\2\u0231\u023a\7\62\2\2\u0232\u0236\t\b\2\2\u0233\u0235\t"+
+    "\t\2\2\u0234\u0233\3\2\2\2\u0235\u0238\3\2\2\2\u0236\u0234\3\2\2\2\u0236"+
+    "\u0237\3\2\2\2\u0237\u023a\3\2\2\2\u0238\u0236\3\2\2\2\u0239\u0231\3\2"+
+    "\2\2\u0239\u0232\3\2\2\2\u023a\u023b\3\2\2\2\u023b\u023c\bQ\4\2\u023c"+
+    "\u00a3\3\2\2\2\u023d\u0241\t\22\2\2\u023e\u0240\t\23\2\2\u023f\u023e\3"+
+    "\2\2\2\u0240\u0243\3\2\2\2\u0241\u023f\3\2\2\2\u0241\u0242\3\2\2\2\u0242"+
+    "\u0244\3\2\2\2\u0243\u0241\3\2\2\2\u0244\u0245\bR\4\2\u0245\u00a5\3\2"+
+    "\2\2$\2\3\u00a9\u00b3\u00bd\u00c2\u01a8\u01ab\u01b2\u01b5\u01bc\u01bf"+
+    "\u01c2\u01c9\u01cc\u01d2\u01d4\u01d8\u01dd\u01df\u01e2\u01ea\u01ec\u01f6"+
+    "\u01f8\u01fc\u0202\u0204\u020a\u0225\u022e\u0236\u0239\u0241\5\b\2\2\4"+
+    "\3\2\4\2\2";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
index f6ace39be63..c1b698a9530 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
@@ -19,14 +19,14 @@ class PainlessParser extends Parser {
   public static final int
     WS=1, COMMENT=2, LBRACK=3, RBRACK=4, LBRACE=5, RBRACE=6, LP=7, RP=8, DOT=9, 
     COMMA=10, SEMICOLON=11, IF=12, ELSE=13, WHILE=14, DO=15, FOR=16, CONTINUE=17, 
-    BREAK=18, RETURN=19, NEW=20, TRY=21, CATCH=22, THROW=23, THIS=24, BOOLNOT=25, 
-    BWNOT=26, MUL=27, DIV=28, REM=29, ADD=30, SUB=31, LSH=32, RSH=33, USH=34, 
-    LT=35, LTE=36, GT=37, GTE=38, EQ=39, EQR=40, NE=41, NER=42, BWAND=43, 
-    XOR=44, BWOR=45, BOOLAND=46, BOOLOR=47, COND=48, COLON=49, REF=50, ARROW=51, 
-    FIND=52, MATCH=53, INCR=54, DECR=55, ASSIGN=56, AADD=57, ASUB=58, AMUL=59, 
-    ADIV=60, AREM=61, AAND=62, AXOR=63, AOR=64, ALSH=65, ARSH=66, AUSH=67, 
-    OCTAL=68, HEX=69, INTEGER=70, DECIMAL=71, STRING=72, REGEX=73, TRUE=74, 
-    FALSE=75, NULL=76, TYPE=77, ID=78, DOTINTEGER=79, DOTID=80;
+    BREAK=18, RETURN=19, NEW=20, TRY=21, CATCH=22, THROW=23, THIS=24, INSTANCEOF=25, 
+    BOOLNOT=26, BWNOT=27, MUL=28, DIV=29, REM=30, ADD=31, SUB=32, LSH=33, 
+    RSH=34, USH=35, LT=36, LTE=37, GT=38, GTE=39, EQ=40, EQR=41, NE=42, NER=43, 
+    BWAND=44, XOR=45, BWOR=46, BOOLAND=47, BOOLOR=48, COND=49, COLON=50, REF=51, 
+    ARROW=52, FIND=53, MATCH=54, INCR=55, DECR=56, ASSIGN=57, AADD=58, ASUB=59, 
+    AMUL=60, ADIV=61, AREM=62, AAND=63, AXOR=64, AOR=65, ALSH=66, ARSH=67, 
+    AUSH=68, OCTAL=69, HEX=70, INTEGER=71, DECIMAL=72, STRING=73, REGEX=74, 
+    TRUE=75, FALSE=76, NULL=77, TYPE=78, ID=79, DOTINTEGER=80, DOTID=81;
   public static final int
     RULE_source = 0, RULE_function = 1, RULE_parameters = 2, RULE_statement = 3, 
     RULE_trailer = 4, RULE_block = 5, RULE_empty = 6, RULE_initializer = 7, 
@@ -49,20 +49,20 @@ class PainlessParser extends Parser {
   private static final String[] _LITERAL_NAMES = {
     null, null, null, "'{'", "'}'", "'['", "']'", "'('", "')'", "'.'", "','", 
     "';'", "'if'", "'else'", "'while'", "'do'", "'for'", "'continue'", "'break'", 
-    "'return'", "'new'", "'try'", "'catch'", "'throw'", "'this'", "'!'", "'~'", 
-    "'*'", "'/'", "'%'", "'+'", "'-'", "'<<'", "'>>'", "'>>>'", "'<'", "'<='", 
-    "'>'", "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", "'^'", "'|'", 
-    "'&&'", "'||'", "'?'", "':'", "'::'", "'->'", "'=~'", "'==~'", "'++'", 
-    "'--'", "'='", "'+='", "'-='", "'*='", "'/='", "'%='", "'&='", "'^='", 
-    "'|='", "'<<='", "'>>='", "'>>>='", null, null, null, null, null, null, 
-    "'true'", "'false'", "'null'"
+    "'return'", "'new'", "'try'", "'catch'", "'throw'", "'this'", "'instanceof'", 
+    "'!'", "'~'", "'*'", "'/'", "'%'", "'+'", "'-'", "'<<'", "'>>'", "'>>>'", 
+    "'<'", "'<='", "'>'", "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", 
+    "'^'", "'|'", "'&&'", "'||'", "'?'", "':'", "'::'", "'->'", "'=~'", "'==~'", 
+    "'++'", "'--'", "'='", "'+='", "'-='", "'*='", "'/='", "'%='", "'&='", 
+    "'^='", "'|='", "'<<='", "'>>='", "'>>>='", null, null, null, null, null, 
+    null, "'true'", "'false'", "'null'"
   };
   private static final String[] _SYMBOLIC_NAMES = {
     null, "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", 
     "DOT", "COMMA", "SEMICOLON", "IF", "ELSE", "WHILE", "DO", "FOR", "CONTINUE", 
-    "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", "BOOLNOT", 
-    "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", "LT", 
-    "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", 
+    "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", "INSTANCEOF", 
+    "BOOLNOT", "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", 
+    "LT", "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", 
     "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "FIND", "MATCH", 
     "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", "AAND", 
     "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", "DECIMAL", 
@@ -1150,27 +1150,29 @@ class PainlessParser extends Parser {
   public final DecltypeContext decltype() throws RecognitionException {
     DecltypeContext _localctx = new DecltypeContext(_ctx, getState());
     enterRule(_localctx, 20, RULE_decltype);
-    int _la;
     try {
+      int _alt;
       enterOuterAlt(_localctx, 1);
       {
       setState(213);
       match(TYPE);
       setState(218);
       _errHandler.sync(this);
-      _la = _input.LA(1);
-      while (_la==LBRACE) {
-        {
-        {
-        setState(214);
-        match(LBRACE);
-        setState(215);
-        match(RBRACE);
-        }
+      _alt = getInterpreter().adaptivePredict(_input,16,_ctx);
+      while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
+        if ( _alt==1 ) {
+          {
+          {
+          setState(214);
+          match(LBRACE);
+          setState(215);
+          match(RBRACE);
+          }
+          } 
         }
         setState(220);
         _errHandler.sync(this);
-        _la = _input.LA(1);
+        _alt = getInterpreter().adaptivePredict(_input,16,_ctx);
       }
       }
     }
@@ -1461,6 +1463,21 @@ class PainlessParser extends Parser {
       else return visitor.visitChildren(this);
     }
   }
+  public static class InstanceofContext extends ExpressionContext {
+    public ExpressionContext expression() {
+      return getRuleContext(ExpressionContext.class,0);
+    }
+    public TerminalNode INSTANCEOF() { return getToken(PainlessParser.INSTANCEOF, 0); }
+    public DecltypeContext decltype() {
+      return getRuleContext(DecltypeContext.class,0);
+    }
+    public InstanceofContext(ExpressionContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitInstanceof(this);
+      else return visitor.visitChildren(this);
+    }
+  }
 
   public final ExpressionContext expression() throws RecognitionException {
     return expression(0);
@@ -1490,7 +1507,7 @@ class PainlessParser extends Parser {
         chain(true);
         setState(237);
         _la = _input.LA(1);
-        if ( !(((((_la - 56)) & ~0x3f) == 0 && ((1L << (_la - 56)) & ((1L << (ASSIGN - 56)) | (1L << (AADD - 56)) | (1L << (ASUB - 56)) | (1L << (AMUL - 56)) | (1L << (ADIV - 56)) | (1L << (AREM - 56)) | (1L << (AAND - 56)) | (1L << (AXOR - 56)) | (1L << (AOR - 56)) | (1L << (ALSH - 56)) | (1L << (ARSH - 56)) | (1L << (AUSH - 56)))) != 0)) ) {
+        if ( !(((((_la - 57)) & ~0x3f) == 0 && ((1L << (_la - 57)) & ((1L << (ASSIGN - 57)) | (1L << (AADD - 57)) | (1L << (ASUB - 57)) | (1L << (AMUL - 57)) | (1L << (ADIV - 57)) | (1L << (AREM - 57)) | (1L << (AAND - 57)) | (1L << (AXOR - 57)) | (1L << (AOR - 57)) | (1L << (ALSH - 57)) | (1L << (ARSH - 57)) | (1L << (AUSH - 57)))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
@@ -1512,7 +1529,7 @@ class PainlessParser extends Parser {
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(310);
+      setState(315);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -1520,14 +1537,14 @@ class PainlessParser extends Parser {
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(308);
+          setState(313);
           switch ( getInterpreter().adaptivePredict(_input,19,_ctx) ) {
           case 1:
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(246);
-            if (!(precpred(_ctx, 13))) throw new FailedPredicateException(this, "precpred(_ctx, 13)");
+            if (!(precpred(_ctx, 14))) throw new FailedPredicateException(this, "precpred(_ctx, 14)");
             setState(247);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << MUL) | (1L << DIV) | (1L << REM))) != 0)) ) {
@@ -1536,7 +1553,7 @@ class PainlessParser extends Parser {
               consume();
             }
             setState(248);
-            expression(14);
+            expression(15);
              ((BinaryContext)_localctx).s =  false; 
             }
             break;
@@ -1545,7 +1562,7 @@ class PainlessParser extends Parser {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(251);
-            if (!(precpred(_ctx, 12))) throw new FailedPredicateException(this, "precpred(_ctx, 12)");
+            if (!(precpred(_ctx, 13))) throw new FailedPredicateException(this, "precpred(_ctx, 13)");
             setState(252);
             _la = _input.LA(1);
             if ( !(_la==ADD || _la==SUB) ) {
@@ -1554,7 +1571,7 @@ class PainlessParser extends Parser {
               consume();
             }
             setState(253);
-            expression(13);
+            expression(14);
              ((BinaryContext)_localctx).s =  false; 
             }
             break;
@@ -1563,7 +1580,7 @@ class PainlessParser extends Parser {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(256);
-            if (!(precpred(_ctx, 11))) throw new FailedPredicateException(this, "precpred(_ctx, 11)");
+            if (!(precpred(_ctx, 12))) throw new FailedPredicateException(this, "precpred(_ctx, 12)");
             setState(257);
             _la = _input.LA(1);
             if ( !(_la==FIND || _la==MATCH) ) {
@@ -1572,7 +1589,7 @@ class PainlessParser extends Parser {
               consume();
             }
             setState(258);
-            expression(12);
+            expression(13);
              ((BinaryContext)_localctx).s =  false; 
             }
             break;
@@ -1581,7 +1598,7 @@ class PainlessParser extends Parser {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(261);
-            if (!(precpred(_ctx, 10))) throw new FailedPredicateException(this, "precpred(_ctx, 10)");
+            if (!(precpred(_ctx, 11))) throw new FailedPredicateException(this, "precpred(_ctx, 11)");
             setState(262);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LSH) | (1L << RSH) | (1L << USH))) != 0)) ) {
@@ -1590,7 +1607,7 @@ class PainlessParser extends Parser {
               consume();
             }
             setState(263);
-            expression(11);
+            expression(12);
              ((BinaryContext)_localctx).s =  false; 
             }
             break;
@@ -1599,7 +1616,7 @@ class PainlessParser extends Parser {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
             setState(266);
-            if (!(precpred(_ctx, 9))) throw new FailedPredicateException(this, "precpred(_ctx, 9)");
+            if (!(precpred(_ctx, 10))) throw new FailedPredicateException(this, "precpred(_ctx, 10)");
             setState(267);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LT) | (1L << LTE) | (1L << GT) | (1L << GTE))) != 0)) ) {
@@ -1608,7 +1625,7 @@ class PainlessParser extends Parser {
               consume();
             }
             setState(268);
-            expression(10);
+            expression(11);
              ((CompContext)_localctx).s =  false; 
             }
             break;
@@ -1712,10 +1729,23 @@ class PainlessParser extends Parser {
              ((ConditionalContext)_localctx).s =  ((ConditionalContext)_localctx).e0.s && ((ConditionalContext)_localctx).e1.s; 
             }
             break;
+          case 13:
+            {
+            _localctx = new InstanceofContext(new ExpressionContext(_parentctx, _parentState));
+            pushNewRecursionContext(_localctx, _startState, RULE_expression);
+            setState(308);
+            if (!(precpred(_ctx, 9))) throw new FailedPredicateException(this, "precpred(_ctx, 9)");
+            setState(309);
+            match(INSTANCEOF);
+            setState(310);
+            decltype();
+             ((InstanceofContext)_localctx).s =  false; 
+            }
+            break;
           }
           } 
         }
-        setState(312);
+        setState(317);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
       }
@@ -1884,22 +1914,22 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 30, RULE_unary);
     int _la;
     try {
-      setState(350);
+      setState(355);
       switch ( getInterpreter().adaptivePredict(_input,21,_ctx) ) {
       case 1:
         _localctx = new PreContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(313);
+        setState(318);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(314);
+        setState(319);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(315);
+        setState(320);
         chain(true);
         }
         break;
@@ -1907,11 +1937,11 @@ class PainlessParser extends Parser {
         _localctx = new PostContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(316);
+        setState(321);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(317);
+        setState(322);
         chain(true);
-        setState(318);
+        setState(323);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
@@ -1924,9 +1954,9 @@ class PainlessParser extends Parser {
         _localctx = new ReadContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(320);
+        setState(325);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(321);
+        setState(326);
         chain(false);
         }
         break;
@@ -1934,11 +1964,11 @@ class PainlessParser extends Parser {
         _localctx = new NumericContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(322);
+        setState(327);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(323);
+        setState(328);
         _la = _input.LA(1);
-        if ( !(((((_la - 68)) & ~0x3f) == 0 && ((1L << (_la - 68)) & ((1L << (OCTAL - 68)) | (1L << (HEX - 68)) | (1L << (INTEGER - 68)) | (1L << (DECIMAL - 68)))) != 0)) ) {
+        if ( !(((((_la - 69)) & ~0x3f) == 0 && ((1L << (_la - 69)) & ((1L << (OCTAL - 69)) | (1L << (HEX - 69)) | (1L << (INTEGER - 69)) | (1L << (DECIMAL - 69)))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
@@ -1950,9 +1980,9 @@ class PainlessParser extends Parser {
         _localctx = new TrueContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(325);
+        setState(330);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(326);
+        setState(331);
         match(TRUE);
          ((TrueContext)_localctx).s =  false; 
         }
@@ -1961,9 +1991,9 @@ class PainlessParser extends Parser {
         _localctx = new FalseContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(328);
+        setState(333);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(329);
+        setState(334);
         match(FALSE);
          ((FalseContext)_localctx).s =  false; 
         }
@@ -1972,9 +2002,9 @@ class PainlessParser extends Parser {
         _localctx = new NullContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(331);
+        setState(336);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(332);
+        setState(337);
         match(NULL);
          ((NullContext)_localctx).s =  false; 
         }
@@ -1983,9 +2013,9 @@ class PainlessParser extends Parser {
         _localctx = new ListinitContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(334);
+        setState(339);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(335);
+        setState(340);
         listinitializer();
          ((ListinitContext)_localctx).s =  false; 
         }
@@ -1994,9 +2024,9 @@ class PainlessParser extends Parser {
         _localctx = new MapinitContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(338);
+        setState(343);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(339);
+        setState(344);
         mapinitializer();
          ((MapinitContext)_localctx).s =  false; 
         }
@@ -2005,16 +2035,16 @@ class PainlessParser extends Parser {
         _localctx = new OperatorContext(_localctx);
         enterOuterAlt(_localctx, 10);
         {
-        setState(342);
+        setState(347);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(343);
+        setState(348);
         _la = _input.LA(1);
         if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(344);
+        setState(349);
         unary(false);
         }
         break;
@@ -2022,13 +2052,13 @@ class PainlessParser extends Parser {
         _localctx = new CastContext(_localctx);
         enterOuterAlt(_localctx, 11);
         {
-        setState(345);
+        setState(350);
         match(LP);
-        setState(346);
+        setState(351);
         decltype();
-        setState(347);
+        setState(352);
         match(RP);
-        setState(348);
+        setState(353);
         unary(_localctx.c);
         }
         break;
@@ -2115,27 +2145,27 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 32, RULE_chain);
     try {
       int _alt;
-      setState(368);
+      setState(373);
       switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) {
       case 1:
         _localctx = new DynamicContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(352);
+        setState(357);
         ((DynamicContext)_localctx).p = primary(_localctx.c);
-        setState(356);
+        setState(361);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(353);
+            setState(358);
             secondary(((DynamicContext)_localctx).p.s);
             }
             } 
           }
-          setState(358);
+          setState(363);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
         }
@@ -2145,23 +2175,23 @@ class PainlessParser extends Parser {
         _localctx = new StaticContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(359);
-        decltype();
-        setState(360);
-        dot();
         setState(364);
+        decltype();
+        setState(365);
+        dot();
+        setState(369);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(361);
+            setState(366);
             secondary(true);
             }
             } 
           }
-          setState(366);
+          setState(371);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
         }
@@ -2171,7 +2201,7 @@ class PainlessParser extends Parser {
         _localctx = new NewarrayContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(367);
+        setState(372);
         arrayinitializer();
         }
         break;
@@ -2289,19 +2319,19 @@ class PainlessParser extends Parser {
     PrimaryContext _localctx = new PrimaryContext(_ctx, getState(), c);
     enterRule(_localctx, 34, RULE_primary);
     try {
-      setState(389);
+      setState(394);
       switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) {
       case 1:
         _localctx = new ExprprecContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(370);
+        setState(375);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(371);
+        setState(376);
         match(LP);
-        setState(372);
+        setState(377);
         ((ExprprecContext)_localctx).e = expression(0);
-        setState(373);
+        setState(378);
         match(RP);
          ((ExprprecContext)_localctx).s =  ((ExprprecContext)_localctx).e.s; 
         }
@@ -2310,13 +2340,13 @@ class PainlessParser extends Parser {
         _localctx = new ChainprecContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(376);
+        setState(381);
         if (!( _localctx.c )) throw new FailedPredicateException(this, " $c ");
-        setState(377);
+        setState(382);
         match(LP);
-        setState(378);
+        setState(383);
         unary(true);
-        setState(379);
+        setState(384);
         match(RP);
         }
         break;
@@ -2324,7 +2354,7 @@ class PainlessParser extends Parser {
         _localctx = new StringContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(381);
+        setState(386);
         match(STRING);
         }
         break;
@@ -2332,7 +2362,7 @@ class PainlessParser extends Parser {
         _localctx = new RegexContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(382);
+        setState(387);
         match(REGEX);
         }
         break;
@@ -2340,7 +2370,7 @@ class PainlessParser extends Parser {
         _localctx = new VariableContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(383);
+        setState(388);
         match(ID);
         }
         break;
@@ -2348,9 +2378,9 @@ class PainlessParser extends Parser {
         _localctx = new CalllocalContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(384);
+        setState(389);
         match(ID);
-        setState(385);
+        setState(390);
         arguments();
         }
         break;
@@ -2358,11 +2388,11 @@ class PainlessParser extends Parser {
         _localctx = new NewobjectContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(386);
+        setState(391);
         match(NEW);
-        setState(387);
+        setState(392);
         match(TYPE);
-        setState(388);
+        setState(393);
         arguments();
         }
         break;
@@ -2404,23 +2434,23 @@ class PainlessParser extends Parser {
     SecondaryContext _localctx = new SecondaryContext(_ctx, getState(), s);
     enterRule(_localctx, 36, RULE_secondary);
     try {
-      setState(395);
+      setState(400);
       switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(391);
+        setState(396);
         if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
-        setState(392);
+        setState(397);
         dot();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(393);
+        setState(398);
         if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
-        setState(394);
+        setState(399);
         brace();
         }
         break;
@@ -2478,17 +2508,17 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 38, RULE_dot);
     int _la;
     try {
-      setState(402);
+      setState(407);
       switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) {
       case 1:
         _localctx = new CallinvokeContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(397);
+        setState(402);
         match(DOT);
-        setState(398);
+        setState(403);
         match(DOTID);
-        setState(399);
+        setState(404);
         arguments();
         }
         break;
@@ -2496,9 +2526,9 @@ class PainlessParser extends Parser {
         _localctx = new FieldaccessContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(400);
+        setState(405);
         match(DOT);
-        setState(401);
+        setState(406);
         _la = _input.LA(1);
         if ( !(_la==DOTINTEGER || _la==DOTID) ) {
         _errHandler.recoverInline(this);
@@ -2552,11 +2582,11 @@ class PainlessParser extends Parser {
       _localctx = new BraceaccessContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(404);
+      setState(409);
       match(LBRACE);
-      setState(405);
+      setState(410);
       expression(0);
-      setState(406);
+      setState(411);
       match(RBRACE);
       }
     }
@@ -2603,34 +2633,34 @@ class PainlessParser extends Parser {
       enterOuterAlt(_localctx, 1);
       {
       {
-      setState(408);
+      setState(413);
       match(LP);
-      setState(417);
+      setState(422);
       switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
       case 1:
         {
-        setState(409);
-        argument();
         setState(414);
+        argument();
+        setState(419);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(410);
+          setState(415);
           match(COMMA);
-          setState(411);
+          setState(416);
           argument();
           }
           }
-          setState(416);
+          setState(421);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
         break;
       }
-      setState(419);
+      setState(424);
       match(RP);
       }
       }
@@ -2671,26 +2701,26 @@ class PainlessParser extends Parser {
     ArgumentContext _localctx = new ArgumentContext(_ctx, getState());
     enterRule(_localctx, 44, RULE_argument);
     try {
-      setState(424);
+      setState(429);
       switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(421);
+        setState(426);
         expression(0);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(422);
+        setState(427);
         lambda();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(423);
+        setState(428);
         funcref();
         }
         break;
@@ -2745,64 +2775,64 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(439);
+      setState(444);
       switch (_input.LA(1)) {
       case TYPE:
       case ID:
         {
-        setState(426);
+        setState(431);
         lamtype();
         }
         break;
       case LP:
         {
-        setState(427);
+        setState(432);
         match(LP);
-        setState(436);
+        setState(441);
         _la = _input.LA(1);
         if (_la==TYPE || _la==ID) {
           {
-          setState(428);
-          lamtype();
           setState(433);
+          lamtype();
+          setState(438);
           _errHandler.sync(this);
           _la = _input.LA(1);
           while (_la==COMMA) {
             {
             {
-            setState(429);
+            setState(434);
             match(COMMA);
-            setState(430);
+            setState(435);
             lamtype();
             }
             }
-            setState(435);
+            setState(440);
             _errHandler.sync(this);
             _la = _input.LA(1);
           }
           }
         }
 
-        setState(438);
+        setState(443);
         match(RP);
         }
         break;
       default:
         throw new NoViableAltException(this);
       }
-      setState(441);
+      setState(446);
       match(ARROW);
-      setState(444);
+      setState(449);
       switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) {
       case 1:
         {
-        setState(442);
+        setState(447);
         block();
         }
         break;
       case 2:
         {
-        setState(443);
+        setState(448);
         expression(0);
         }
         break;
@@ -2843,16 +2873,16 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(447);
+      setState(452);
       _la = _input.LA(1);
       if (_la==TYPE) {
         {
-        setState(446);
+        setState(451);
         decltype();
         }
       }
 
-      setState(449);
+      setState(454);
       match(ID);
       }
     }
@@ -2895,33 +2925,33 @@ class PainlessParser extends Parser {
     FuncrefContext _localctx = new FuncrefContext(_ctx, getState());
     enterRule(_localctx, 50, RULE_funcref);
     try {
-      setState(455);
+      setState(460);
       switch ( getInterpreter().adaptivePredict(_input,36,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(451);
+        setState(456);
         classFuncref();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(452);
+        setState(457);
         constructorFuncref();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(453);
+        setState(458);
         capturingFuncref();
         }
         break;
       case 4:
         enterOuterAlt(_localctx, 4);
         {
-        setState(454);
+        setState(459);
         localFuncref();
         }
         break;
@@ -2959,11 +2989,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(457);
+      setState(462);
       match(TYPE);
-      setState(458);
+      setState(463);
       match(REF);
-      setState(459);
+      setState(464);
       match(ID);
       }
     }
@@ -3001,11 +3031,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(461);
+      setState(466);
       decltype();
-      setState(462);
+      setState(467);
       match(REF);
-      setState(463);
+      setState(468);
       match(NEW);
       }
     }
@@ -3043,11 +3073,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(465);
+      setState(470);
       match(ID);
-      setState(466);
+      setState(471);
       match(REF);
-      setState(467);
+      setState(472);
       match(ID);
       }
     }
@@ -3083,11 +3113,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(469);
+      setState(474);
       match(THIS);
-      setState(470);
+      setState(475);
       match(REF);
-      setState(471);
+      setState(476);
       match(ID);
       }
     }
@@ -3178,17 +3208,17 @@ class PainlessParser extends Parser {
     int _la;
     try {
       int _alt;
-      setState(511);
+      setState(516);
       switch ( getInterpreter().adaptivePredict(_input,43,_ctx) ) {
       case 1:
         _localctx = new NewstandardarrayContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(473);
+        setState(478);
         match(NEW);
-        setState(474);
+        setState(479);
         match(TYPE);
-        setState(479); 
+        setState(484); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -3196,11 +3226,11 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(475);
+            setState(480);
             match(LBRACE);
-            setState(476);
+            setState(481);
             expression(0);
-            setState(477);
+            setState(482);
             match(RBRACE);
             }
             }
@@ -3208,29 +3238,29 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(481); 
+          setState(486); 
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,37,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
-        setState(490);
+        setState(495);
         switch ( getInterpreter().adaptivePredict(_input,39,_ctx) ) {
         case 1:
           {
-          setState(483);
+          setState(488);
           dot();
-          setState(487);
+          setState(492);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,38,_ctx);
           while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
             if ( _alt==1 ) {
               {
               {
-              setState(484);
+              setState(489);
               secondary(true);
               }
               } 
             }
-            setState(489);
+            setState(494);
             _errHandler.sync(this);
             _alt = getInterpreter().adaptivePredict(_input,38,_ctx);
           }
@@ -3243,51 +3273,51 @@ class PainlessParser extends Parser {
         _localctx = new NewinitializedarrayContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(492);
+        setState(497);
         match(NEW);
-        setState(493);
+        setState(498);
         match(TYPE);
-        setState(494);
+        setState(499);
         match(LBRACE);
-        setState(495);
+        setState(500);
         match(RBRACE);
-        setState(496);
+        setState(501);
         match(LBRACK);
-        setState(505);
+        setState(510);
         switch ( getInterpreter().adaptivePredict(_input,41,_ctx) ) {
         case 1:
           {
-          setState(497);
-          expression(0);
           setState(502);
+          expression(0);
+          setState(507);
           _errHandler.sync(this);
           _la = _input.LA(1);
           while (_la==COMMA) {
             {
             {
-            setState(498);
+            setState(503);
             match(COMMA);
-            setState(499);
+            setState(504);
             expression(0);
             }
             }
-            setState(504);
+            setState(509);
             _errHandler.sync(this);
             _la = _input.LA(1);
           }
           }
           break;
         }
-        setState(508);
+        setState(513);
         _la = _input.LA(1);
         if (_la==SEMICOLON) {
           {
-          setState(507);
+          setState(512);
           match(SEMICOLON);
           }
         }
 
-        setState(510);
+        setState(515);
         match(RBRACK);
         }
         break;
@@ -3333,41 +3363,41 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 62, RULE_listinitializer);
     int _la;
     try {
-      setState(526);
+      setState(531);
       switch ( getInterpreter().adaptivePredict(_input,45,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(513);
+        setState(518);
         match(LBRACE);
-        setState(514);
-        expression(0);
         setState(519);
+        expression(0);
+        setState(524);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(515);
+          setState(520);
           match(COMMA);
-          setState(516);
+          setState(521);
           expression(0);
           }
           }
-          setState(521);
+          setState(526);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(522);
+        setState(527);
         match(RBRACE);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(524);
+        setState(529);
         match(LBRACE);
-        setState(525);
+        setState(530);
         match(RBRACE);
         }
         break;
@@ -3414,43 +3444,43 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 64, RULE_mapinitializer);
     int _la;
     try {
-      setState(542);
+      setState(547);
       switch ( getInterpreter().adaptivePredict(_input,47,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(528);
+        setState(533);
         match(LBRACE);
-        setState(529);
-        maptoken();
         setState(534);
+        maptoken();
+        setState(539);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(530);
+          setState(535);
           match(COMMA);
-          setState(531);
+          setState(536);
           maptoken();
           }
           }
-          setState(536);
+          setState(541);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(537);
+        setState(542);
         match(RBRACE);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(539);
+        setState(544);
         match(LBRACE);
-        setState(540);
+        setState(545);
         match(COLON);
-        setState(541);
+        setState(546);
         match(RBRACE);
         }
         break;
@@ -3492,11 +3522,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(544);
+      setState(549);
       expression(0);
-      setState(545);
+      setState(550);
       match(COLON);
-      setState(546);
+      setState(551);
       expression(0);
       }
     }
@@ -3536,15 +3566,15 @@ class PainlessParser extends Parser {
   private boolean expression_sempred(ExpressionContext _localctx, int predIndex) {
     switch (predIndex) {
     case 1:
-      return precpred(_ctx, 13);
+      return precpred(_ctx, 14);
     case 2:
-      return precpred(_ctx, 12);
+      return precpred(_ctx, 13);
     case 3:
-      return precpred(_ctx, 11);
+      return precpred(_ctx, 12);
     case 4:
-      return precpred(_ctx, 10);
+      return precpred(_ctx, 11);
     case 5:
-      return precpred(_ctx, 9);
+      return precpred(_ctx, 10);
     case 6:
       return precpred(_ctx, 8);
     case 7:
@@ -3559,13 +3589,13 @@ class PainlessParser extends Parser {
       return precpred(_ctx, 3);
     case 12:
       return precpred(_ctx, 2);
+    case 13:
+      return precpred(_ctx, 9);
     }
     return true;
   }
   private boolean unary_sempred(UnaryContext _localctx, int predIndex) {
     switch (predIndex) {
-    case 13:
-      return  !_localctx.c ;
     case 14:
       return  !_localctx.c ;
     case 15:
@@ -3584,30 +3614,32 @@ class PainlessParser extends Parser {
       return  !_localctx.c ;
     case 22:
       return  !_localctx.c ;
+    case 23:
+      return  !_localctx.c ;
     }
     return true;
   }
   private boolean primary_sempred(PrimaryContext _localctx, int predIndex) {
     switch (predIndex) {
-    case 23:
-      return  !_localctx.c ;
     case 24:
+      return  !_localctx.c ;
+    case 25:
       return  _localctx.c ;
     }
     return true;
   }
   private boolean secondary_sempred(SecondaryContext _localctx, int predIndex) {
     switch (predIndex) {
-    case 25:
-      return  _localctx.s ;
     case 26:
       return  _localctx.s ;
+    case 27:
+      return  _localctx.s ;
     }
     return true;
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3R\u0227\4\2\t\2\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3S\u022c\4\2\t\2\4"+
     "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
     "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
     "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+
@@ -3629,191 +3661,193 @@ class PainlessParser extends Parser {
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
-    "\7\20\u0137\n\20\f\20\16\20\u013a\13\20\3\21\3\21\3\21\3\21\3\21\3\21"+
+    "\3\20\3\20\3\20\3\20\3\20\7\20\u013c\n\20\f\20\16\20\u013f\13\20\3\21"+
     "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
     "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
-    "\3\21\3\21\3\21\5\21\u0161\n\21\3\22\3\22\7\22\u0165\n\22\f\22\16\22\u0168"+
-    "\13\22\3\22\3\22\3\22\7\22\u016d\n\22\f\22\16\22\u0170\13\22\3\22\5\22"+
-    "\u0173\n\22\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23"+
-    "\3\23\3\23\3\23\3\23\3\23\3\23\3\23\5\23\u0188\n\23\3\24\3\24\3\24\3\24"+
-    "\5\24\u018e\n\24\3\25\3\25\3\25\3\25\3\25\5\25\u0195\n\25\3\26\3\26\3"+
-    "\26\3\26\3\27\3\27\3\27\3\27\7\27\u019f\n\27\f\27\16\27\u01a2\13\27\5"+
-    "\27\u01a4\n\27\3\27\3\27\3\30\3\30\3\30\5\30\u01ab\n\30\3\31\3\31\3\31"+
-    "\3\31\3\31\7\31\u01b2\n\31\f\31\16\31\u01b5\13\31\5\31\u01b7\n\31\3\31"+
-    "\5\31\u01ba\n\31\3\31\3\31\3\31\5\31\u01bf\n\31\3\32\5\32\u01c2\n\32\3"+
-    "\32\3\32\3\33\3\33\3\33\3\33\5\33\u01ca\n\33\3\34\3\34\3\34\3\34\3\35"+
-    "\3\35\3\35\3\35\3\36\3\36\3\36\3\36\3\37\3\37\3\37\3\37\3 \3 \3 \3 \3"+
-    " \3 \6 \u01e2\n \r \16 \u01e3\3 \3 \7 \u01e8\n \f \16 \u01eb\13 \5 \u01ed"+
-    "\n \3 \3 \3 \3 \3 \3 \3 \3 \7 \u01f7\n \f \16 \u01fa\13 \5 \u01fc\n \3"+
-    " \5 \u01ff\n \3 \5 \u0202\n \3!\3!\3!\3!\7!\u0208\n!\f!\16!\u020b\13!"+
-    "\3!\3!\3!\3!\5!\u0211\n!\3\"\3\"\3\"\3\"\7\"\u0217\n\"\f\"\16\"\u021a"+
-    "\13\"\3\"\3\"\3\"\3\"\3\"\5\"\u0221\n\"\3#\3#\3#\3#\3#\2\3\36$\2\4\6\b"+
-    "\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,.\60\62\64\668:<>@BD\2\16\3\3\r"+
-    "\r\3\2:E\3\2\35\37\3\2 !\3\2\66\67\3\2\"$\3\2%(\3\2),\3\289\3\2FI\4\2"+
-    "\33\34 !\3\2QR\u025a\2I\3\2\2\2\4T\3\2\2\2\6Y\3\2\2\2\b\u00b7\3\2\2\2"+
-    "\n\u00bb\3\2\2\2\f\u00bd\3\2\2\2\16\u00c6\3\2\2\2\20\u00ca\3\2\2\2\22"+
-    "\u00cc\3\2\2\2\24\u00ce\3\2\2\2\26\u00d7\3\2\2\2\30\u00df\3\2\2\2\32\u00e4"+
-    "\3\2\2\2\34\u00eb\3\2\2\2\36\u00f6\3\2\2\2 \u0160\3\2\2\2\"\u0172\3\2"+
-    "\2\2$\u0187\3\2\2\2&\u018d\3\2\2\2(\u0194\3\2\2\2*\u0196\3\2\2\2,\u019a"+
-    "\3\2\2\2.\u01aa\3\2\2\2\60\u01b9\3\2\2\2\62\u01c1\3\2\2\2\64\u01c9\3\2"+
-    "\2\2\66\u01cb\3\2\2\28\u01cf\3\2\2\2:\u01d3\3\2\2\2<\u01d7\3\2\2\2>\u0201"+
-    "\3\2\2\2@\u0210\3\2\2\2B\u0220\3\2\2\2D\u0222\3\2\2\2FH\5\4\3\2GF\3\2"+
-    "\2\2HK\3\2\2\2IG\3\2\2\2IJ\3\2\2\2JO\3\2\2\2KI\3\2\2\2LN\5\b\5\2ML\3\2"+
-    "\2\2NQ\3\2\2\2OM\3\2\2\2OP\3\2\2\2PR\3\2\2\2QO\3\2\2\2RS\7\2\2\3S\3\3"+
-    "\2\2\2TU\5\26\f\2UV\7P\2\2VW\5\6\4\2WX\5\f\7\2X\5\3\2\2\2Ye\7\t\2\2Z["+
-    "\5\26\f\2[b\7P\2\2\\]\7\f\2\2]^\5\26\f\2^_\7P\2\2_a\3\2\2\2`\\\3\2\2\2"+
-    "ad\3\2\2\2b`\3\2\2\2bc\3\2\2\2cf\3\2\2\2db\3\2\2\2eZ\3\2\2\2ef\3\2\2\2"+
-    "fg\3\2\2\2gh\7\n\2\2h\7\3\2\2\2ij\7\16\2\2jk\7\t\2\2kl\5\36\20\2lm\7\n"+
-    "\2\2mq\5\n\6\2no\7\17\2\2or\5\n\6\2pr\6\5\2\2qn\3\2\2\2qp\3\2\2\2r\u00b8"+
-    "\3\2\2\2st\7\20\2\2tu\7\t\2\2uv\5\36\20\2vy\7\n\2\2wz\5\n\6\2xz\5\16\b"+
-    "\2yw\3\2\2\2yx\3\2\2\2z\u00b8\3\2\2\2{|\7\21\2\2|}\5\f\7\2}~\7\20\2\2"+
-    "~\177\7\t\2\2\177\u0080\5\36\20\2\u0080\u0081\7\n\2\2\u0081\u0082\5\34"+
-    "\17\2\u0082\u00b8\3\2\2\2\u0083\u0084\7\22\2\2\u0084\u0086\7\t\2\2\u0085"+
-    "\u0087\5\20\t\2\u0086\u0085\3\2\2\2\u0086\u0087\3\2\2\2\u0087\u0088\3"+
-    "\2\2\2\u0088\u008a\7\r\2\2\u0089\u008b\5\36\20\2\u008a\u0089\3\2\2\2\u008a"+
-    "\u008b\3\2\2\2\u008b\u008c\3\2\2\2\u008c\u008e\7\r\2\2\u008d\u008f\5\22"+
-    "\n\2\u008e\u008d\3\2\2\2\u008e\u008f\3\2\2\2\u008f\u0090\3\2\2\2\u0090"+
-    "\u0093\7\n\2\2\u0091\u0094\5\n\6\2\u0092\u0094\5\16\b\2\u0093\u0091\3"+
-    "\2\2\2\u0093\u0092\3\2\2\2\u0094\u00b8\3\2\2\2\u0095\u0096\7\22\2\2\u0096"+
-    "\u0097\7\t\2\2\u0097\u0098\5\26\f\2\u0098\u0099\7P\2\2\u0099\u009a\7\63"+
-    "\2\2\u009a\u009b\5\36\20\2\u009b\u009c\7\n\2\2\u009c\u009d\5\n\6\2\u009d"+
-    "\u00b8\3\2\2\2\u009e\u009f\5\24\13\2\u009f\u00a0\5\34\17\2\u00a0\u00b8"+
-    "\3\2\2\2\u00a1\u00a2\7\23\2\2\u00a2\u00b8\5\34\17\2\u00a3\u00a4\7\24\2"+
-    "\2\u00a4\u00b8\5\34\17\2\u00a5\u00a6\7\25\2\2\u00a6\u00a7\5\36\20\2\u00a7"+
-    "\u00a8\5\34\17\2\u00a8\u00b8\3\2\2\2\u00a9\u00aa\7\27\2\2\u00aa\u00ac"+
-    "\5\f\7\2\u00ab\u00ad\5\32\16\2\u00ac\u00ab\3\2\2\2\u00ad\u00ae\3\2\2\2"+
-    "\u00ae\u00ac\3\2\2\2\u00ae\u00af\3\2\2\2\u00af\u00b8\3\2\2\2\u00b0\u00b1"+
-    "\7\31\2\2\u00b1\u00b2\5\36\20\2\u00b2\u00b3\5\34\17\2\u00b3\u00b8\3\2"+
-    "\2\2\u00b4\u00b5\5\36\20\2\u00b5\u00b6\5\34\17\2\u00b6\u00b8\3\2\2\2\u00b7"+
-    "i\3\2\2\2\u00b7s\3\2\2\2\u00b7{\3\2\2\2\u00b7\u0083\3\2\2\2\u00b7\u0095"+
-    "\3\2\2\2\u00b7\u009e\3\2\2\2\u00b7\u00a1\3\2\2\2\u00b7\u00a3\3\2\2\2\u00b7"+
-    "\u00a5\3\2\2\2\u00b7\u00a9\3\2\2\2\u00b7\u00b0\3\2\2\2\u00b7\u00b4\3\2"+
-    "\2\2\u00b8\t\3\2\2\2\u00b9\u00bc\5\f\7\2\u00ba\u00bc\5\b\5\2\u00bb\u00b9"+
-    "\3\2\2\2\u00bb\u00ba\3\2\2\2\u00bc\13\3\2\2\2\u00bd\u00c1\7\5\2\2\u00be"+
-    "\u00c0\5\b\5\2\u00bf\u00be\3\2\2\2\u00c0\u00c3\3\2\2\2\u00c1\u00bf\3\2"+
-    "\2\2\u00c1\u00c2\3\2\2\2\u00c2\u00c4\3\2\2\2\u00c3\u00c1\3\2\2\2\u00c4"+
-    "\u00c5\7\6\2\2\u00c5\r\3\2\2\2\u00c6\u00c7\7\r\2\2\u00c7\17\3\2\2\2\u00c8"+
-    "\u00cb\5\24\13\2\u00c9\u00cb\5\36\20\2\u00ca\u00c8\3\2\2\2\u00ca\u00c9"+
-    "\3\2\2\2\u00cb\21\3\2\2\2\u00cc\u00cd\5\36\20\2\u00cd\23\3\2\2\2\u00ce"+
-    "\u00cf\5\26\f\2\u00cf\u00d4\5\30\r\2\u00d0\u00d1\7\f\2\2\u00d1\u00d3\5"+
-    "\30\r\2\u00d2\u00d0\3\2\2\2\u00d3\u00d6\3\2\2\2\u00d4\u00d2\3\2\2\2\u00d4"+
-    "\u00d5\3\2\2\2\u00d5\25\3\2\2\2\u00d6\u00d4\3\2\2\2\u00d7\u00dc\7O\2\2"+
-    "\u00d8\u00d9\7\7\2\2\u00d9\u00db\7\b\2\2\u00da\u00d8\3\2\2\2\u00db\u00de"+
-    "\3\2\2\2\u00dc\u00da\3\2\2\2\u00dc\u00dd\3\2\2\2\u00dd\27\3\2\2\2\u00de"+
-    "\u00dc\3\2\2\2\u00df\u00e2\7P\2\2\u00e0\u00e1\7:\2\2\u00e1\u00e3\5\36"+
-    "\20\2\u00e2\u00e0\3\2\2\2\u00e2\u00e3\3\2\2\2\u00e3\31\3\2\2\2\u00e4\u00e5"+
-    "\7\30\2\2\u00e5\u00e6\7\t\2\2\u00e6\u00e7\7O\2\2\u00e7\u00e8\7P\2\2\u00e8"+
-    "\u00e9\7\n\2\2\u00e9\u00ea\5\f\7\2\u00ea\33\3\2\2\2\u00eb\u00ec\t\2\2"+
-    "\2\u00ec\35\3\2\2\2\u00ed\u00ee\b\20\1\2\u00ee\u00ef\5\"\22\2\u00ef\u00f0"+
-    "\t\3\2\2\u00f0\u00f1\5\36\20\3\u00f1\u00f2\b\20\1\2\u00f2\u00f7\3\2\2"+
-    "\2\u00f3\u00f4\5 \21\2\u00f4\u00f5\b\20\1\2\u00f5\u00f7\3\2\2\2\u00f6"+
-    "\u00ed\3\2\2\2\u00f6\u00f3\3\2\2\2\u00f7\u0138\3\2\2\2\u00f8\u00f9\f\17"+
-    "\2\2\u00f9\u00fa\t\4\2\2\u00fa\u00fb\5\36\20\20\u00fb\u00fc\b\20\1\2\u00fc"+
-    "\u0137\3\2\2\2\u00fd\u00fe\f\16\2\2\u00fe\u00ff\t\5\2\2\u00ff\u0100\5"+
-    "\36\20\17\u0100\u0101\b\20\1\2\u0101\u0137\3\2\2\2\u0102\u0103\f\r\2\2"+
-    "\u0103\u0104\t\6\2\2\u0104\u0105\5\36\20\16\u0105\u0106\b\20\1\2\u0106"+
-    "\u0137\3\2\2\2\u0107\u0108\f\f\2\2\u0108\u0109\t\7\2\2\u0109\u010a\5\36"+
-    "\20\r\u010a\u010b\b\20\1\2\u010b\u0137\3\2\2\2\u010c\u010d\f\13\2\2\u010d"+
-    "\u010e\t\b\2\2\u010e\u010f\5\36\20\f\u010f\u0110\b\20\1\2\u0110\u0137"+
-    "\3\2\2\2\u0111\u0112\f\n\2\2\u0112\u0113\t\t\2\2\u0113\u0114\5\36\20\13"+
-    "\u0114\u0115\b\20\1\2\u0115\u0137\3\2\2\2\u0116\u0117\f\t\2\2\u0117\u0118"+
-    "\7-\2\2\u0118\u0119\5\36\20\n\u0119\u011a\b\20\1\2\u011a\u0137\3\2\2\2"+
-    "\u011b\u011c\f\b\2\2\u011c\u011d\7.\2\2\u011d\u011e\5\36\20\t\u011e\u011f"+
-    "\b\20\1\2\u011f\u0137\3\2\2\2\u0120\u0121\f\7\2\2\u0121\u0122\7/\2\2\u0122"+
-    "\u0123\5\36\20\b\u0123\u0124\b\20\1\2\u0124\u0137\3\2\2\2\u0125\u0126"+
-    "\f\6\2\2\u0126\u0127\7\60\2\2\u0127\u0128\5\36\20\7\u0128\u0129\b\20\1"+
-    "\2\u0129\u0137\3\2\2\2\u012a\u012b\f\5\2\2\u012b\u012c\7\61\2\2\u012c"+
-    "\u012d\5\36\20\6\u012d\u012e\b\20\1\2\u012e\u0137\3\2\2\2\u012f\u0130"+
-    "\f\4\2\2\u0130\u0131\7\62\2\2\u0131\u0132\5\36\20\2\u0132\u0133\7\63\2"+
-    "\2\u0133\u0134\5\36\20\4\u0134\u0135\b\20\1\2\u0135\u0137\3\2\2\2\u0136"+
-    "\u00f8\3\2\2\2\u0136\u00fd\3\2\2\2\u0136\u0102\3\2\2\2\u0136\u0107\3\2"+
-    "\2\2\u0136\u010c\3\2\2\2\u0136\u0111\3\2\2\2\u0136\u0116\3\2\2\2\u0136"+
-    "\u011b\3\2\2\2\u0136\u0120\3\2\2\2\u0136\u0125\3\2\2\2\u0136\u012a\3\2"+
-    "\2\2\u0136\u012f\3\2\2\2\u0137\u013a\3\2\2\2\u0138\u0136\3\2\2\2\u0138"+
-    "\u0139\3\2\2\2\u0139\37\3\2\2\2\u013a\u0138\3\2\2\2\u013b\u013c\6\21\17"+
-    "\3\u013c\u013d\t\n\2\2\u013d\u0161\5\"\22\2\u013e\u013f\6\21\20\3\u013f"+
-    "\u0140\5\"\22\2\u0140\u0141\t\n\2\2\u0141\u0161\3\2\2\2\u0142\u0143\6"+
-    "\21\21\3\u0143\u0161\5\"\22\2\u0144\u0145\6\21\22\3\u0145\u0146\t\13\2"+
-    "\2\u0146\u0161\b\21\1\2\u0147\u0148\6\21\23\3\u0148\u0149\7L\2\2\u0149"+
-    "\u0161\b\21\1\2\u014a\u014b\6\21\24\3\u014b\u014c\7M\2\2\u014c\u0161\b"+
-    "\21\1\2\u014d\u014e\6\21\25\3\u014e\u014f\7N\2\2\u014f\u0161\b\21\1\2"+
-    "\u0150\u0151\6\21\26\3\u0151\u0152\5@!\2\u0152\u0153\b\21\1\2\u0153\u0161"+
-    "\3\2\2\2\u0154\u0155\6\21\27\3\u0155\u0156\5B\"\2\u0156\u0157\b\21\1\2"+
-    "\u0157\u0161\3\2\2\2\u0158\u0159\6\21\30\3\u0159\u015a\t\f\2\2\u015a\u0161"+
-    "\5 \21\2\u015b\u015c\7\t\2\2\u015c\u015d\5\26\f\2\u015d\u015e\7\n\2\2"+
-    "\u015e\u015f\5 \21\2\u015f\u0161\3\2\2\2\u0160\u013b\3\2\2\2\u0160\u013e"+
-    "\3\2\2\2\u0160\u0142\3\2\2\2\u0160\u0144\3\2\2\2\u0160\u0147\3\2\2\2\u0160"+
-    "\u014a\3\2\2\2\u0160\u014d\3\2\2\2\u0160\u0150\3\2\2\2\u0160\u0154\3\2"+
-    "\2\2\u0160\u0158\3\2\2\2\u0160\u015b\3\2\2\2\u0161!\3\2\2\2\u0162\u0166"+
-    "\5$\23\2\u0163\u0165\5&\24\2\u0164\u0163\3\2\2\2\u0165\u0168\3\2\2\2\u0166"+
-    "\u0164\3\2\2\2\u0166\u0167\3\2\2\2\u0167\u0173\3\2\2\2\u0168\u0166\3\2"+
-    "\2\2\u0169\u016a\5\26\f\2\u016a\u016e\5(\25\2\u016b\u016d\5&\24\2\u016c"+
-    "\u016b\3\2\2\2\u016d\u0170\3\2\2\2\u016e\u016c\3\2\2\2\u016e\u016f\3\2"+
-    "\2\2\u016f\u0173\3\2\2\2\u0170\u016e\3\2\2\2\u0171\u0173\5> \2\u0172\u0162"+
-    "\3\2\2\2\u0172\u0169\3\2\2\2\u0172\u0171\3\2\2\2\u0173#\3\2\2\2\u0174"+
-    "\u0175\6\23\31\3\u0175\u0176\7\t\2\2\u0176\u0177\5\36\20\2\u0177\u0178"+
-    "\7\n\2\2\u0178\u0179\b\23\1\2\u0179\u0188\3\2\2\2\u017a\u017b\6\23\32"+
-    "\3\u017b\u017c\7\t\2\2\u017c\u017d\5 \21\2\u017d\u017e\7\n\2\2\u017e\u0188"+
-    "\3\2\2\2\u017f\u0188\7J\2\2\u0180\u0188\7K\2\2\u0181\u0188\7P\2\2\u0182"+
-    "\u0183\7P\2\2\u0183\u0188\5,\27\2\u0184\u0185\7\26\2\2\u0185\u0186\7O"+
-    "\2\2\u0186\u0188\5,\27\2\u0187\u0174\3\2\2\2\u0187\u017a\3\2\2\2\u0187"+
-    "\u017f\3\2\2\2\u0187\u0180\3\2\2\2\u0187\u0181\3\2\2\2\u0187\u0182\3\2"+
-    "\2\2\u0187\u0184\3\2\2\2\u0188%\3\2\2\2\u0189\u018a\6\24\33\3\u018a\u018e"+
-    "\5(\25\2\u018b\u018c\6\24\34\3\u018c\u018e\5*\26\2\u018d\u0189\3\2\2\2"+
-    "\u018d\u018b\3\2\2\2\u018e\'\3\2\2\2\u018f\u0190\7\13\2\2\u0190\u0191"+
-    "\7R\2\2\u0191\u0195\5,\27\2\u0192\u0193\7\13\2\2\u0193\u0195\t\r\2\2\u0194"+
-    "\u018f\3\2\2\2\u0194\u0192\3\2\2\2\u0195)\3\2\2\2\u0196\u0197\7\7\2\2"+
-    "\u0197\u0198\5\36\20\2\u0198\u0199\7\b\2\2\u0199+\3\2\2\2\u019a\u01a3"+
-    "\7\t\2\2\u019b\u01a0\5.\30\2\u019c\u019d\7\f\2\2\u019d\u019f\5.\30\2\u019e"+
-    "\u019c\3\2\2\2\u019f\u01a2\3\2\2\2\u01a0\u019e\3\2\2\2\u01a0\u01a1\3\2"+
-    "\2\2\u01a1\u01a4\3\2\2\2\u01a2\u01a0\3\2\2\2\u01a3\u019b\3\2\2\2\u01a3"+
-    "\u01a4\3\2\2\2\u01a4\u01a5\3\2\2\2\u01a5\u01a6\7\n\2\2\u01a6-\3\2\2\2"+
-    "\u01a7\u01ab\5\36\20\2\u01a8\u01ab\5\60\31\2\u01a9\u01ab\5\64\33\2\u01aa"+
-    "\u01a7\3\2\2\2\u01aa\u01a8\3\2\2\2\u01aa\u01a9\3\2\2\2\u01ab/\3\2\2\2"+
-    "\u01ac\u01ba\5\62\32\2\u01ad\u01b6\7\t\2\2\u01ae\u01b3\5\62\32\2\u01af"+
-    "\u01b0\7\f\2\2\u01b0\u01b2\5\62\32\2\u01b1\u01af\3\2\2\2\u01b2\u01b5\3"+
-    "\2\2\2\u01b3\u01b1\3\2\2\2\u01b3\u01b4\3\2\2\2\u01b4\u01b7\3\2\2\2\u01b5"+
-    "\u01b3\3\2\2\2\u01b6\u01ae\3\2\2\2\u01b6\u01b7\3\2\2\2\u01b7\u01b8\3\2"+
-    "\2\2\u01b8\u01ba\7\n\2\2\u01b9\u01ac\3\2\2\2\u01b9\u01ad\3\2\2\2\u01ba"+
-    "\u01bb\3\2\2\2\u01bb\u01be\7\65\2\2\u01bc\u01bf\5\f\7\2\u01bd\u01bf\5"+
-    "\36\20\2\u01be\u01bc\3\2\2\2\u01be\u01bd\3\2\2\2\u01bf\61\3\2\2\2\u01c0"+
-    "\u01c2\5\26\f\2\u01c1\u01c0\3\2\2\2\u01c1\u01c2\3\2\2\2\u01c2\u01c3\3"+
-    "\2\2\2\u01c3\u01c4\7P\2\2\u01c4\63\3\2\2\2\u01c5\u01ca\5\66\34\2\u01c6"+
-    "\u01ca\58\35\2\u01c7\u01ca\5:\36\2\u01c8\u01ca\5<\37\2\u01c9\u01c5\3\2"+
-    "\2\2\u01c9\u01c6\3\2\2\2\u01c9\u01c7\3\2\2\2\u01c9\u01c8\3\2\2\2\u01ca"+
-    "\65\3\2\2\2\u01cb\u01cc\7O\2\2\u01cc\u01cd\7\64\2\2\u01cd\u01ce\7P\2\2"+
-    "\u01ce\67\3\2\2\2\u01cf\u01d0\5\26\f\2\u01d0\u01d1\7\64\2\2\u01d1\u01d2"+
-    "\7\26\2\2\u01d29\3\2\2\2\u01d3\u01d4\7P\2\2\u01d4\u01d5\7\64\2\2\u01d5"+
-    "\u01d6\7P\2\2\u01d6;\3\2\2\2\u01d7\u01d8\7\32\2\2\u01d8\u01d9\7\64\2\2"+
-    "\u01d9\u01da\7P\2\2\u01da=\3\2\2\2\u01db\u01dc\7\26\2\2\u01dc\u01e1\7"+
-    "O\2\2\u01dd\u01de\7\7\2\2\u01de\u01df\5\36\20\2\u01df\u01e0\7\b\2\2\u01e0"+
-    "\u01e2\3\2\2\2\u01e1\u01dd\3\2\2\2\u01e2\u01e3\3\2\2\2\u01e3\u01e1\3\2"+
-    "\2\2\u01e3\u01e4\3\2\2\2\u01e4\u01ec\3\2\2\2\u01e5\u01e9\5(\25\2\u01e6"+
-    "\u01e8\5&\24\2\u01e7\u01e6\3\2\2\2\u01e8\u01eb\3\2\2\2\u01e9\u01e7\3\2"+
-    "\2\2\u01e9\u01ea\3\2\2\2\u01ea\u01ed\3\2\2\2\u01eb\u01e9\3\2\2\2\u01ec"+
-    "\u01e5\3\2\2\2\u01ec\u01ed\3\2\2\2\u01ed\u0202\3\2\2\2\u01ee\u01ef\7\26"+
-    "\2\2\u01ef\u01f0\7O\2\2\u01f0\u01f1\7\7\2\2\u01f1\u01f2\7\b\2\2\u01f2"+
-    "\u01fb\7\5\2\2\u01f3\u01f8\5\36\20\2\u01f4\u01f5\7\f\2\2\u01f5\u01f7\5"+
-    "\36\20\2\u01f6\u01f4\3\2\2\2\u01f7\u01fa\3\2\2\2\u01f8\u01f6\3\2\2\2\u01f8"+
-    "\u01f9\3\2\2\2\u01f9\u01fc\3\2\2\2\u01fa\u01f8\3\2\2\2\u01fb\u01f3\3\2"+
-    "\2\2\u01fb\u01fc\3\2\2\2\u01fc\u01fe\3\2\2\2\u01fd\u01ff\7\r\2\2\u01fe"+
-    "\u01fd\3\2\2\2\u01fe\u01ff\3\2\2\2\u01ff\u0200\3\2\2\2\u0200\u0202\7\6"+
-    "\2\2\u0201\u01db\3\2\2\2\u0201\u01ee\3\2\2\2\u0202?\3\2\2\2\u0203\u0204"+
-    "\7\7\2\2\u0204\u0209\5\36\20\2\u0205\u0206\7\f\2\2\u0206\u0208\5\36\20"+
-    "\2\u0207\u0205\3\2\2\2\u0208\u020b\3\2\2\2\u0209\u0207\3\2\2\2\u0209\u020a"+
-    "\3\2\2\2\u020a\u020c\3\2\2\2\u020b\u0209\3\2\2\2\u020c\u020d\7\b\2\2\u020d"+
-    "\u0211\3\2\2\2\u020e\u020f\7\7\2\2\u020f\u0211\7\b\2\2\u0210\u0203\3\2"+
-    "\2\2\u0210\u020e\3\2\2\2\u0211A\3\2\2\2\u0212\u0213\7\7\2\2\u0213\u0218"+
-    "\5D#\2\u0214\u0215\7\f\2\2\u0215\u0217\5D#\2\u0216\u0214\3\2\2\2\u0217"+
-    "\u021a\3\2\2\2\u0218\u0216\3\2\2\2\u0218\u0219\3\2\2\2\u0219\u021b\3\2"+
-    "\2\2\u021a\u0218\3\2\2\2\u021b\u021c\7\b\2\2\u021c\u0221\3\2\2\2\u021d"+
-    "\u021e\7\7\2\2\u021e\u021f\7\63\2\2\u021f\u0221\7\b\2\2\u0220\u0212\3"+
-    "\2\2\2\u0220\u021d\3\2\2\2\u0221C\3\2\2\2\u0222\u0223\5\36\20\2\u0223"+
-    "\u0224\7\63\2\2\u0224\u0225\5\36\20\2\u0225E\3\2\2\2\62IObeqy\u0086\u008a"+
-    "\u008e\u0093\u00ae\u00b7\u00bb\u00c1\u00ca\u00d4\u00dc\u00e2\u00f6\u0136"+
-    "\u0138\u0160\u0166\u016e\u0172\u0187\u018d\u0194\u01a0\u01a3\u01aa\u01b3"+
-    "\u01b6\u01b9\u01be\u01c1\u01c9\u01e3\u01e9\u01ec\u01f8\u01fb\u01fe\u0201"+
-    "\u0209\u0210\u0218\u0220";
+    "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\5\21\u0166\n\21\3\22\3\22\7\22"+
+    "\u016a\n\22\f\22\16\22\u016d\13\22\3\22\3\22\3\22\7\22\u0172\n\22\f\22"+
+    "\16\22\u0175\13\22\3\22\5\22\u0178\n\22\3\23\3\23\3\23\3\23\3\23\3\23"+
+    "\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\5\23"+
+    "\u018d\n\23\3\24\3\24\3\24\3\24\5\24\u0193\n\24\3\25\3\25\3\25\3\25\3"+
+    "\25\5\25\u019a\n\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\27\7\27\u01a4"+
+    "\n\27\f\27\16\27\u01a7\13\27\5\27\u01a9\n\27\3\27\3\27\3\30\3\30\3\30"+
+    "\5\30\u01b0\n\30\3\31\3\31\3\31\3\31\3\31\7\31\u01b7\n\31\f\31\16\31\u01ba"+
+    "\13\31\5\31\u01bc\n\31\3\31\5\31\u01bf\n\31\3\31\3\31\3\31\5\31\u01c4"+
+    "\n\31\3\32\5\32\u01c7\n\32\3\32\3\32\3\33\3\33\3\33\3\33\5\33\u01cf\n"+
+    "\33\3\34\3\34\3\34\3\34\3\35\3\35\3\35\3\35\3\36\3\36\3\36\3\36\3\37\3"+
+    "\37\3\37\3\37\3 \3 \3 \3 \3 \3 \6 \u01e7\n \r \16 \u01e8\3 \3 \7 \u01ed"+
+    "\n \f \16 \u01f0\13 \5 \u01f2\n \3 \3 \3 \3 \3 \3 \3 \3 \7 \u01fc\n \f"+
+    " \16 \u01ff\13 \5 \u0201\n \3 \5 \u0204\n \3 \5 \u0207\n \3!\3!\3!\3!"+
+    "\7!\u020d\n!\f!\16!\u0210\13!\3!\3!\3!\3!\5!\u0216\n!\3\"\3\"\3\"\3\""+
+    "\7\"\u021c\n\"\f\"\16\"\u021f\13\"\3\"\3\"\3\"\3\"\3\"\5\"\u0226\n\"\3"+
+    "#\3#\3#\3#\3#\2\3\36$\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,."+
+    "\60\62\64\668:<>@BD\2\16\3\3\r\r\3\2;F\3\2\36 \3\2!\"\3\2\678\3\2#%\3"+
+    "\2&)\3\2*-\3\29:\3\2GJ\4\2\34\35!\"\3\2RS\u0260\2I\3\2\2\2\4T\3\2\2\2"+
+    "\6Y\3\2\2\2\b\u00b7\3\2\2\2\n\u00bb\3\2\2\2\f\u00bd\3\2\2\2\16\u00c6\3"+
+    "\2\2\2\20\u00ca\3\2\2\2\22\u00cc\3\2\2\2\24\u00ce\3\2\2\2\26\u00d7\3\2"+
+    "\2\2\30\u00df\3\2\2\2\32\u00e4\3\2\2\2\34\u00eb\3\2\2\2\36\u00f6\3\2\2"+
+    "\2 \u0165\3\2\2\2\"\u0177\3\2\2\2$\u018c\3\2\2\2&\u0192\3\2\2\2(\u0199"+
+    "\3\2\2\2*\u019b\3\2\2\2,\u019f\3\2\2\2.\u01af\3\2\2\2\60\u01be\3\2\2\2"+
+    "\62\u01c6\3\2\2\2\64\u01ce\3\2\2\2\66\u01d0\3\2\2\28\u01d4\3\2\2\2:\u01d8"+
+    "\3\2\2\2<\u01dc\3\2\2\2>\u0206\3\2\2\2@\u0215\3\2\2\2B\u0225\3\2\2\2D"+
+    "\u0227\3\2\2\2FH\5\4\3\2GF\3\2\2\2HK\3\2\2\2IG\3\2\2\2IJ\3\2\2\2JO\3\2"+
+    "\2\2KI\3\2\2\2LN\5\b\5\2ML\3\2\2\2NQ\3\2\2\2OM\3\2\2\2OP\3\2\2\2PR\3\2"+
+    "\2\2QO\3\2\2\2RS\7\2\2\3S\3\3\2\2\2TU\5\26\f\2UV\7Q\2\2VW\5\6\4\2WX\5"+
+    "\f\7\2X\5\3\2\2\2Ye\7\t\2\2Z[\5\26\f\2[b\7Q\2\2\\]\7\f\2\2]^\5\26\f\2"+
+    "^_\7Q\2\2_a\3\2\2\2`\\\3\2\2\2ad\3\2\2\2b`\3\2\2\2bc\3\2\2\2cf\3\2\2\2"+
+    "db\3\2\2\2eZ\3\2\2\2ef\3\2\2\2fg\3\2\2\2gh\7\n\2\2h\7\3\2\2\2ij\7\16\2"+
+    "\2jk\7\t\2\2kl\5\36\20\2lm\7\n\2\2mq\5\n\6\2no\7\17\2\2or\5\n\6\2pr\6"+
+    "\5\2\2qn\3\2\2\2qp\3\2\2\2r\u00b8\3\2\2\2st\7\20\2\2tu\7\t\2\2uv\5\36"+
+    "\20\2vy\7\n\2\2wz\5\n\6\2xz\5\16\b\2yw\3\2\2\2yx\3\2\2\2z\u00b8\3\2\2"+
+    "\2{|\7\21\2\2|}\5\f\7\2}~\7\20\2\2~\177\7\t\2\2\177\u0080\5\36\20\2\u0080"+
+    "\u0081\7\n\2\2\u0081\u0082\5\34\17\2\u0082\u00b8\3\2\2\2\u0083\u0084\7"+
+    "\22\2\2\u0084\u0086\7\t\2\2\u0085\u0087\5\20\t\2\u0086\u0085\3\2\2\2\u0086"+
+    "\u0087\3\2\2\2\u0087\u0088\3\2\2\2\u0088\u008a\7\r\2\2\u0089\u008b\5\36"+
+    "\20\2\u008a\u0089\3\2\2\2\u008a\u008b\3\2\2\2\u008b\u008c\3\2\2\2\u008c"+
+    "\u008e\7\r\2\2\u008d\u008f\5\22\n\2\u008e\u008d\3\2\2\2\u008e\u008f\3"+
+    "\2\2\2\u008f\u0090\3\2\2\2\u0090\u0093\7\n\2\2\u0091\u0094\5\n\6\2\u0092"+
+    "\u0094\5\16\b\2\u0093\u0091\3\2\2\2\u0093\u0092\3\2\2\2\u0094\u00b8\3"+
+    "\2\2\2\u0095\u0096\7\22\2\2\u0096\u0097\7\t\2\2\u0097\u0098\5\26\f\2\u0098"+
+    "\u0099\7Q\2\2\u0099\u009a\7\64\2\2\u009a\u009b\5\36\20\2\u009b\u009c\7"+
+    "\n\2\2\u009c\u009d\5\n\6\2\u009d\u00b8\3\2\2\2\u009e\u009f\5\24\13\2\u009f"+
+    "\u00a0\5\34\17\2\u00a0\u00b8\3\2\2\2\u00a1\u00a2\7\23\2\2\u00a2\u00b8"+
+    "\5\34\17\2\u00a3\u00a4\7\24\2\2\u00a4\u00b8\5\34\17\2\u00a5\u00a6\7\25"+
+    "\2\2\u00a6\u00a7\5\36\20\2\u00a7\u00a8\5\34\17\2\u00a8\u00b8\3\2\2\2\u00a9"+
+    "\u00aa\7\27\2\2\u00aa\u00ac\5\f\7\2\u00ab\u00ad\5\32\16\2\u00ac\u00ab"+
+    "\3\2\2\2\u00ad\u00ae\3\2\2\2\u00ae\u00ac\3\2\2\2\u00ae\u00af\3\2\2\2\u00af"+
+    "\u00b8\3\2\2\2\u00b0\u00b1\7\31\2\2\u00b1\u00b2\5\36\20\2\u00b2\u00b3"+
+    "\5\34\17\2\u00b3\u00b8\3\2\2\2\u00b4\u00b5\5\36\20\2\u00b5\u00b6\5\34"+
+    "\17\2\u00b6\u00b8\3\2\2\2\u00b7i\3\2\2\2\u00b7s\3\2\2\2\u00b7{\3\2\2\2"+
+    "\u00b7\u0083\3\2\2\2\u00b7\u0095\3\2\2\2\u00b7\u009e\3\2\2\2\u00b7\u00a1"+
+    "\3\2\2\2\u00b7\u00a3\3\2\2\2\u00b7\u00a5\3\2\2\2\u00b7\u00a9\3\2\2\2\u00b7"+
+    "\u00b0\3\2\2\2\u00b7\u00b4\3\2\2\2\u00b8\t\3\2\2\2\u00b9\u00bc\5\f\7\2"+
+    "\u00ba\u00bc\5\b\5\2\u00bb\u00b9\3\2\2\2\u00bb\u00ba\3\2\2\2\u00bc\13"+
+    "\3\2\2\2\u00bd\u00c1\7\5\2\2\u00be\u00c0\5\b\5\2\u00bf\u00be\3\2\2\2\u00c0"+
+    "\u00c3\3\2\2\2\u00c1\u00bf\3\2\2\2\u00c1\u00c2\3\2\2\2\u00c2\u00c4\3\2"+
+    "\2\2\u00c3\u00c1\3\2\2\2\u00c4\u00c5\7\6\2\2\u00c5\r\3\2\2\2\u00c6\u00c7"+
+    "\7\r\2\2\u00c7\17\3\2\2\2\u00c8\u00cb\5\24\13\2\u00c9\u00cb\5\36\20\2"+
+    "\u00ca\u00c8\3\2\2\2\u00ca\u00c9\3\2\2\2\u00cb\21\3\2\2\2\u00cc\u00cd"+
+    "\5\36\20\2\u00cd\23\3\2\2\2\u00ce\u00cf\5\26\f\2\u00cf\u00d4\5\30\r\2"+
+    "\u00d0\u00d1\7\f\2\2\u00d1\u00d3\5\30\r\2\u00d2\u00d0\3\2\2\2\u00d3\u00d6"+
+    "\3\2\2\2\u00d4\u00d2\3\2\2\2\u00d4\u00d5\3\2\2\2\u00d5\25\3\2\2\2\u00d6"+
+    "\u00d4\3\2\2\2\u00d7\u00dc\7P\2\2\u00d8\u00d9\7\7\2\2\u00d9\u00db\7\b"+
+    "\2\2\u00da\u00d8\3\2\2\2\u00db\u00de\3\2\2\2\u00dc\u00da\3\2\2\2\u00dc"+
+    "\u00dd\3\2\2\2\u00dd\27\3\2\2\2\u00de\u00dc\3\2\2\2\u00df\u00e2\7Q\2\2"+
+    "\u00e0\u00e1\7;\2\2\u00e1\u00e3\5\36\20\2\u00e2\u00e0\3\2\2\2\u00e2\u00e3"+
+    "\3\2\2\2\u00e3\31\3\2\2\2\u00e4\u00e5\7\30\2\2\u00e5\u00e6\7\t\2\2\u00e6"+
+    "\u00e7\7P\2\2\u00e7\u00e8\7Q\2\2\u00e8\u00e9\7\n\2\2\u00e9\u00ea\5\f\7"+
+    "\2\u00ea\33\3\2\2\2\u00eb\u00ec\t\2\2\2\u00ec\35\3\2\2\2\u00ed\u00ee\b"+
+    "\20\1\2\u00ee\u00ef\5\"\22\2\u00ef\u00f0\t\3\2\2\u00f0\u00f1\5\36\20\3"+
+    "\u00f1\u00f2\b\20\1\2\u00f2\u00f7\3\2\2\2\u00f3\u00f4\5 \21\2\u00f4\u00f5"+
+    "\b\20\1\2\u00f5\u00f7\3\2\2\2\u00f6\u00ed\3\2\2\2\u00f6\u00f3\3\2\2\2"+
+    "\u00f7\u013d\3\2\2\2\u00f8\u00f9\f\20\2\2\u00f9\u00fa\t\4\2\2\u00fa\u00fb"+
+    "\5\36\20\21\u00fb\u00fc\b\20\1\2\u00fc\u013c\3\2\2\2\u00fd\u00fe\f\17"+
+    "\2\2\u00fe\u00ff\t\5\2\2\u00ff\u0100\5\36\20\20\u0100\u0101\b\20\1\2\u0101"+
+    "\u013c\3\2\2\2\u0102\u0103\f\16\2\2\u0103\u0104\t\6\2\2\u0104\u0105\5"+
+    "\36\20\17\u0105\u0106\b\20\1\2\u0106\u013c\3\2\2\2\u0107\u0108\f\r\2\2"+
+    "\u0108\u0109\t\7\2\2\u0109\u010a\5\36\20\16\u010a\u010b\b\20\1\2\u010b"+
+    "\u013c\3\2\2\2\u010c\u010d\f\f\2\2\u010d\u010e\t\b\2\2\u010e\u010f\5\36"+
+    "\20\r\u010f\u0110\b\20\1\2\u0110\u013c\3\2\2\2\u0111\u0112\f\n\2\2\u0112"+
+    "\u0113\t\t\2\2\u0113\u0114\5\36\20\13\u0114\u0115\b\20\1\2\u0115\u013c"+
+    "\3\2\2\2\u0116\u0117\f\t\2\2\u0117\u0118\7.\2\2\u0118\u0119\5\36\20\n"+
+    "\u0119\u011a\b\20\1\2\u011a\u013c\3\2\2\2\u011b\u011c\f\b\2\2\u011c\u011d"+
+    "\7/\2\2\u011d\u011e\5\36\20\t\u011e\u011f\b\20\1\2\u011f\u013c\3\2\2\2"+
+    "\u0120\u0121\f\7\2\2\u0121\u0122\7\60\2\2\u0122\u0123\5\36\20\b\u0123"+
+    "\u0124\b\20\1\2\u0124\u013c\3\2\2\2\u0125\u0126\f\6\2\2\u0126\u0127\7"+
+    "\61\2\2\u0127\u0128\5\36\20\7\u0128\u0129\b\20\1\2\u0129\u013c\3\2\2\2"+
+    "\u012a\u012b\f\5\2\2\u012b\u012c\7\62\2\2\u012c\u012d\5\36\20\6\u012d"+
+    "\u012e\b\20\1\2\u012e\u013c\3\2\2\2\u012f\u0130\f\4\2\2\u0130\u0131\7"+
+    "\63\2\2\u0131\u0132\5\36\20\2\u0132\u0133\7\64\2\2\u0133\u0134\5\36\20"+
+    "\4\u0134\u0135\b\20\1\2\u0135\u013c\3\2\2\2\u0136\u0137\f\13\2\2\u0137"+
+    "\u0138\7\33\2\2\u0138\u0139\5\26\f\2\u0139\u013a\b\20\1\2\u013a\u013c"+
+    "\3\2\2\2\u013b\u00f8\3\2\2\2\u013b\u00fd\3\2\2\2\u013b\u0102\3\2\2\2\u013b"+
+    "\u0107\3\2\2\2\u013b\u010c\3\2\2\2\u013b\u0111\3\2\2\2\u013b\u0116\3\2"+
+    "\2\2\u013b\u011b\3\2\2\2\u013b\u0120\3\2\2\2\u013b\u0125\3\2\2\2\u013b"+
+    "\u012a\3\2\2\2\u013b\u012f\3\2\2\2\u013b\u0136\3\2\2\2\u013c\u013f\3\2"+
+    "\2\2\u013d\u013b\3\2\2\2\u013d\u013e\3\2\2\2\u013e\37\3\2\2\2\u013f\u013d"+
+    "\3\2\2\2\u0140\u0141\6\21\20\3\u0141\u0142\t\n\2\2\u0142\u0166\5\"\22"+
+    "\2\u0143\u0144\6\21\21\3\u0144\u0145\5\"\22\2\u0145\u0146\t\n\2\2\u0146"+
+    "\u0166\3\2\2\2\u0147\u0148\6\21\22\3\u0148\u0166\5\"\22\2\u0149\u014a"+
+    "\6\21\23\3\u014a\u014b\t\13\2\2\u014b\u0166\b\21\1\2\u014c\u014d\6\21"+
+    "\24\3\u014d\u014e\7M\2\2\u014e\u0166\b\21\1\2\u014f\u0150\6\21\25\3\u0150"+
+    "\u0151\7N\2\2\u0151\u0166\b\21\1\2\u0152\u0153\6\21\26\3\u0153\u0154\7"+
+    "O\2\2\u0154\u0166\b\21\1\2\u0155\u0156\6\21\27\3\u0156\u0157\5@!\2\u0157"+
+    "\u0158\b\21\1\2\u0158\u0166\3\2\2\2\u0159\u015a\6\21\30\3\u015a\u015b"+
+    "\5B\"\2\u015b\u015c\b\21\1\2\u015c\u0166\3\2\2\2\u015d\u015e\6\21\31\3"+
+    "\u015e\u015f\t\f\2\2\u015f\u0166\5 \21\2\u0160\u0161\7\t\2\2\u0161\u0162"+
+    "\5\26\f\2\u0162\u0163\7\n\2\2\u0163\u0164\5 \21\2\u0164\u0166\3\2\2\2"+
+    "\u0165\u0140\3\2\2\2\u0165\u0143\3\2\2\2\u0165\u0147\3\2\2\2\u0165\u0149"+
+    "\3\2\2\2\u0165\u014c\3\2\2\2\u0165\u014f\3\2\2\2\u0165\u0152\3\2\2\2\u0165"+
+    "\u0155\3\2\2\2\u0165\u0159\3\2\2\2\u0165\u015d\3\2\2\2\u0165\u0160\3\2"+
+    "\2\2\u0166!\3\2\2\2\u0167\u016b\5$\23\2\u0168\u016a\5&\24\2\u0169\u0168"+
+    "\3\2\2\2\u016a\u016d\3\2\2\2\u016b\u0169\3\2\2\2\u016b\u016c\3\2\2\2\u016c"+
+    "\u0178\3\2\2\2\u016d\u016b\3\2\2\2\u016e\u016f\5\26\f\2\u016f\u0173\5"+
+    "(\25\2\u0170\u0172\5&\24\2\u0171\u0170\3\2\2\2\u0172\u0175\3\2\2\2\u0173"+
+    "\u0171\3\2\2\2\u0173\u0174\3\2\2\2\u0174\u0178\3\2\2\2\u0175\u0173\3\2"+
+    "\2\2\u0176\u0178\5> \2\u0177\u0167\3\2\2\2\u0177\u016e\3\2\2\2\u0177\u0176"+
+    "\3\2\2\2\u0178#\3\2\2\2\u0179\u017a\6\23\32\3\u017a\u017b\7\t\2\2\u017b"+
+    "\u017c\5\36\20\2\u017c\u017d\7\n\2\2\u017d\u017e\b\23\1\2\u017e\u018d"+
+    "\3\2\2\2\u017f\u0180\6\23\33\3\u0180\u0181\7\t\2\2\u0181\u0182\5 \21\2"+
+    "\u0182\u0183\7\n\2\2\u0183\u018d\3\2\2\2\u0184\u018d\7K\2\2\u0185\u018d"+
+    "\7L\2\2\u0186\u018d\7Q\2\2\u0187\u0188\7Q\2\2\u0188\u018d\5,\27\2\u0189"+
+    "\u018a\7\26\2\2\u018a\u018b\7P\2\2\u018b\u018d\5,\27\2\u018c\u0179\3\2"+
+    "\2\2\u018c\u017f\3\2\2\2\u018c\u0184\3\2\2\2\u018c\u0185\3\2\2\2\u018c"+
+    "\u0186\3\2\2\2\u018c\u0187\3\2\2\2\u018c\u0189\3\2\2\2\u018d%\3\2\2\2"+
+    "\u018e\u018f\6\24\34\3\u018f\u0193\5(\25\2\u0190\u0191\6\24\35\3\u0191"+
+    "\u0193\5*\26\2\u0192\u018e\3\2\2\2\u0192\u0190\3\2\2\2\u0193\'\3\2\2\2"+
+    "\u0194\u0195\7\13\2\2\u0195\u0196\7S\2\2\u0196\u019a\5,\27\2\u0197\u0198"+
+    "\7\13\2\2\u0198\u019a\t\r\2\2\u0199\u0194\3\2\2\2\u0199\u0197\3\2\2\2"+
+    "\u019a)\3\2\2\2\u019b\u019c\7\7\2\2\u019c\u019d\5\36\20\2\u019d\u019e"+
+    "\7\b\2\2\u019e+\3\2\2\2\u019f\u01a8\7\t\2\2\u01a0\u01a5\5.\30\2\u01a1"+
+    "\u01a2\7\f\2\2\u01a2\u01a4\5.\30\2\u01a3\u01a1\3\2\2\2\u01a4\u01a7\3\2"+
+    "\2\2\u01a5\u01a3\3\2\2\2\u01a5\u01a6\3\2\2\2\u01a6\u01a9\3\2\2\2\u01a7"+
+    "\u01a5\3\2\2\2\u01a8\u01a0\3\2\2\2\u01a8\u01a9\3\2\2\2\u01a9\u01aa\3\2"+
+    "\2\2\u01aa\u01ab\7\n\2\2\u01ab-\3\2\2\2\u01ac\u01b0\5\36\20\2\u01ad\u01b0"+
+    "\5\60\31\2\u01ae\u01b0\5\64\33\2\u01af\u01ac\3\2\2\2\u01af\u01ad\3\2\2"+
+    "\2\u01af\u01ae\3\2\2\2\u01b0/\3\2\2\2\u01b1\u01bf\5\62\32\2\u01b2\u01bb"+
+    "\7\t\2\2\u01b3\u01b8\5\62\32\2\u01b4\u01b5\7\f\2\2\u01b5\u01b7\5\62\32"+
+    "\2\u01b6\u01b4\3\2\2\2\u01b7\u01ba\3\2\2\2\u01b8\u01b6\3\2\2\2\u01b8\u01b9"+
+    "\3\2\2\2\u01b9\u01bc\3\2\2\2\u01ba\u01b8\3\2\2\2\u01bb\u01b3\3\2\2\2\u01bb"+
+    "\u01bc\3\2\2\2\u01bc\u01bd\3\2\2\2\u01bd\u01bf\7\n\2\2\u01be\u01b1\3\2"+
+    "\2\2\u01be\u01b2\3\2\2\2\u01bf\u01c0\3\2\2\2\u01c0\u01c3\7\66\2\2\u01c1"+
+    "\u01c4\5\f\7\2\u01c2\u01c4\5\36\20\2\u01c3\u01c1\3\2\2\2\u01c3\u01c2\3"+
+    "\2\2\2\u01c4\61\3\2\2\2\u01c5\u01c7\5\26\f\2\u01c6\u01c5\3\2\2\2\u01c6"+
+    "\u01c7\3\2\2\2\u01c7\u01c8\3\2\2\2\u01c8\u01c9\7Q\2\2\u01c9\63\3\2\2\2"+
+    "\u01ca\u01cf\5\66\34\2\u01cb\u01cf\58\35\2\u01cc\u01cf\5:\36\2\u01cd\u01cf"+
+    "\5<\37\2\u01ce\u01ca\3\2\2\2\u01ce\u01cb\3\2\2\2\u01ce\u01cc\3\2\2\2\u01ce"+
+    "\u01cd\3\2\2\2\u01cf\65\3\2\2\2\u01d0\u01d1\7P\2\2\u01d1\u01d2\7\65\2"+
+    "\2\u01d2\u01d3\7Q\2\2\u01d3\67\3\2\2\2\u01d4\u01d5\5\26\f\2\u01d5\u01d6"+
+    "\7\65\2\2\u01d6\u01d7\7\26\2\2\u01d79\3\2\2\2\u01d8\u01d9\7Q\2\2\u01d9"+
+    "\u01da\7\65\2\2\u01da\u01db\7Q\2\2\u01db;\3\2\2\2\u01dc\u01dd\7\32\2\2"+
+    "\u01dd\u01de\7\65\2\2\u01de\u01df\7Q\2\2\u01df=\3\2\2\2\u01e0\u01e1\7"+
+    "\26\2\2\u01e1\u01e6\7P\2\2\u01e2\u01e3\7\7\2\2\u01e3\u01e4\5\36\20\2\u01e4"+
+    "\u01e5\7\b\2\2\u01e5\u01e7\3\2\2\2\u01e6\u01e2\3\2\2\2\u01e7\u01e8\3\2"+
+    "\2\2\u01e8\u01e6\3\2\2\2\u01e8\u01e9\3\2\2\2\u01e9\u01f1\3\2\2\2\u01ea"+
+    "\u01ee\5(\25\2\u01eb\u01ed\5&\24\2\u01ec\u01eb\3\2\2\2\u01ed\u01f0\3\2"+
+    "\2\2\u01ee\u01ec\3\2\2\2\u01ee\u01ef\3\2\2\2\u01ef\u01f2\3\2\2\2\u01f0"+
+    "\u01ee\3\2\2\2\u01f1\u01ea\3\2\2\2\u01f1\u01f2\3\2\2\2\u01f2\u0207\3\2"+
+    "\2\2\u01f3\u01f4\7\26\2\2\u01f4\u01f5\7P\2\2\u01f5\u01f6\7\7\2\2\u01f6"+
+    "\u01f7\7\b\2\2\u01f7\u0200\7\5\2\2\u01f8\u01fd\5\36\20\2\u01f9\u01fa\7"+
+    "\f\2\2\u01fa\u01fc\5\36\20\2\u01fb\u01f9\3\2\2\2\u01fc\u01ff\3\2\2\2\u01fd"+
+    "\u01fb\3\2\2\2\u01fd\u01fe\3\2\2\2\u01fe\u0201\3\2\2\2\u01ff\u01fd\3\2"+
+    "\2\2\u0200\u01f8\3\2\2\2\u0200\u0201\3\2\2\2\u0201\u0203\3\2\2\2\u0202"+
+    "\u0204\7\r\2\2\u0203\u0202\3\2\2\2\u0203\u0204\3\2\2\2\u0204\u0205\3\2"+
+    "\2\2\u0205\u0207\7\6\2\2\u0206\u01e0\3\2\2\2\u0206\u01f3\3\2\2\2\u0207"+
+    "?\3\2\2\2\u0208\u0209\7\7\2\2\u0209\u020e\5\36\20\2\u020a\u020b\7\f\2"+
+    "\2\u020b\u020d\5\36\20\2\u020c\u020a\3\2\2\2\u020d\u0210\3\2\2\2\u020e"+
+    "\u020c\3\2\2\2\u020e\u020f\3\2\2\2\u020f\u0211\3\2\2\2\u0210\u020e\3\2"+
+    "\2\2\u0211\u0212\7\b\2\2\u0212\u0216\3\2\2\2\u0213\u0214\7\7\2\2\u0214"+
+    "\u0216\7\b\2\2\u0215\u0208\3\2\2\2\u0215\u0213\3\2\2\2\u0216A\3\2\2\2"+
+    "\u0217\u0218\7\7\2\2\u0218\u021d\5D#\2\u0219\u021a\7\f\2\2\u021a\u021c"+
+    "\5D#\2\u021b\u0219\3\2\2\2\u021c\u021f\3\2\2\2\u021d\u021b\3\2\2\2\u021d"+
+    "\u021e\3\2\2\2\u021e\u0220\3\2\2\2\u021f\u021d\3\2\2\2\u0220\u0221\7\b"+
+    "\2\2\u0221\u0226\3\2\2\2\u0222\u0223\7\7\2\2\u0223\u0224\7\64\2\2\u0224"+
+    "\u0226\7\b\2\2\u0225\u0217\3\2\2\2\u0225\u0222\3\2\2\2\u0226C\3\2\2\2"+
+    "\u0227\u0228\5\36\20\2\u0228\u0229\7\64\2\2\u0229\u022a\5\36\20\2\u022a"+
+    "E\3\2\2\2\62IObeqy\u0086\u008a\u008e\u0093\u00ae\u00b7\u00bb\u00c1\u00ca"+
+    "\u00d4\u00dc\u00e2\u00f6\u013b\u013d\u0165\u016b\u0173\u0177\u018c\u0192"+
+    "\u0199\u01a5\u01a8\u01af\u01b8\u01bb\u01be\u01c3\u01c6\u01ce\u01e8\u01ee"+
+    "\u01f1\u01fd\u0200\u0203\u0206\u020e\u0215\u021d\u0225";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
index 498324441e6..a11a856edd4 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
@@ -228,6 +228,13 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitBinary(PainlessParser.BinaryContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitInstanceof(PainlessParser.InstanceofContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
index e57a274f8ef..bc9e8c4f994 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
@@ -214,6 +214,13 @@ interface PainlessParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitBinary(PainlessParser.BinaryContext ctx);
+  /**
+   * Visit a parse tree produced by the {@code instanceof}
+   * labeled alternative in {@link PainlessParser#expression}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitInstanceof(PainlessParser.InstanceofContext ctx);
   /**
    * Visit a parse tree produced by the {@code pre}
    * labeled alternative in {@link PainlessParser#unary}.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index 20384b57145..88774bb9984 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -82,6 +82,7 @@ import org.elasticsearch.painless.antlr.PainlessParser.FuncrefContext;
 import org.elasticsearch.painless.antlr.PainlessParser.FunctionContext;
 import org.elasticsearch.painless.antlr.PainlessParser.IfContext;
 import org.elasticsearch.painless.antlr.PainlessParser.InitializerContext;
+import org.elasticsearch.painless.antlr.PainlessParser.InstanceofContext;
 import org.elasticsearch.painless.antlr.PainlessParser.LambdaContext;
 import org.elasticsearch.painless.antlr.PainlessParser.LamtypeContext;
 import org.elasticsearch.painless.antlr.PainlessParser.LocalFuncrefContext;
@@ -124,6 +125,7 @@ import org.elasticsearch.painless.node.EConditional;
 import org.elasticsearch.painless.node.EDecimal;
 import org.elasticsearch.painless.node.EExplicit;
 import org.elasticsearch.painless.node.EFunctionRef;
+import org.elasticsearch.painless.node.EInstanceof;
 import org.elasticsearch.painless.node.ELambda;
 import org.elasticsearch.painless.node.ENull;
 import org.elasticsearch.painless.node.ENumeric;
@@ -1107,6 +1109,13 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
         throw location(ctx).createError(new IllegalStateException("Illegal tree structure."));
     }
 
+    @Override
+    public Object visitInstanceof(InstanceofContext ctx) {
+        AExpression expr = (AExpression)visitExpression(ctx.expression());
+        String type = ctx.decltype().getText();
+        return new EInstanceof(location(ctx), expr, type);
+    }
+
     /** Returns name of next lambda */
     private String nextLambda() {
         return "lambda$" + syntheticCounter++;
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
new file mode 100644
index 00000000000..312bb10efec
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless.node;
+
+import org.elasticsearch.painless.Definition;
+import org.elasticsearch.painless.Globals;
+import org.elasticsearch.painless.Locals;
+import org.elasticsearch.painless.Location;
+import org.elasticsearch.painless.MethodWriter;
+
+import java.lang.invoke.MethodType;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Represents instanceof operator.
+ * <p>
+ * Unlike java's, this works for primitive types too.
+ */
+public class EInstanceof extends AExpression {
+    AExpression expression;
+    final String type;
+    Class<?> resolvedType;
+    Class<?> expressionType;
+    boolean primitiveExpression;
+
+    public EInstanceof(Location location, AExpression expression, String type) {
+        super(location);
+        this.expression = Objects.requireNonNull(expression);
+        this.type = Objects.requireNonNull(type);
+    }
+    
+    @Override
+    void extractVariables(Set<String> variables) {
+        expression.extractVariables(variables);
+    }
+
+    @Override
+    void analyze(Locals locals) {
+        Definition.Type raw = Definition.getType(type);
+        // map to wrapped type for primitive types
+        resolvedType = MethodType.methodType(raw.clazz).wrap().returnType();
+        expression.analyze(locals);
+        actual = Definition.BOOLEAN_TYPE;
+        
+        Definition.Type expressionRaw = expression.actual;
+        if (expressionRaw == null) {
+            expressionRaw = Definition.DEF_TYPE;
+        }
+        // record if the expression returns a primitive
+        primitiveExpression = expressionRaw.clazz.isPrimitive();
+        // map to wrapped type for primitive types
+        expressionType = MethodType.methodType(expressionRaw.clazz).wrap().returnType();
+    }
+
+    @Override
+    void write(MethodWriter writer, Globals globals) {
+        // primitive types
+        if (primitiveExpression) {
+            // run the expression anyway (who knows what it does)
+            expression.write(writer, globals);
+            // discard its result
+            writer.writePop(expression.actual.type.getSize());
+            // push our result: its a primitive so it cannot be null.
+            writer.push(resolvedType == Object.class || resolvedType == expressionType);
+        } else {
+            // ordinary instanceof
+            expression.write(writer, globals);
+            writer.instanceOf(org.objectweb.asm.Type.getType(resolvedType));
+        }
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
index 7111afa59f4..2e3d07ebf92 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/package-info.java
@@ -22,7 +22,7 @@
  * <p>
  * The following are the types of nodes:
  * A* (abstract) - These are the abstract nodes that are the superclasses for the other types.
- * I* (interface) -- Thse are marker interfaces to denote a property of the node.
+ * I* (interface) -- These are marker interfaces to denote a property of the node.
  * S* (statement) - These are nodes that represent a statement in Painless.  These are the highest level nodes.
  * E* (expression) - These are nodes that represent an expression in Painless.  These are the middle level nodes.
  * L* (link) - These are nodes that represent a piece of a variable/method chain.  The are the lowest level nodes.
@@ -44,8 +44,9 @@
  * {@link org.elasticsearch.painless.node.EDecimal} - Represents a decimal constant.
  * {@link org.elasticsearch.painless.node.EExplicit} - Represents an explicit cast.
  * {@link org.elasticsearch.painless.node.EFunctionRef} - Represents a function reference (non-capturing).
+ * {@link org.elasticsearch.painless.node.EInstanceof} - Represents an instanceof check.
  * {@link org.elasticsearch.painless.node.EListInit} - Represents a list initialization shortcut.
- * {@link org.elasticsearch.painless.node.EMapInit} - Represents a map initializiation shortcut.
+ * {@link org.elasticsearch.painless.node.EMapInit} - Represents a map initialization shortcut.
  * {@link org.elasticsearch.painless.node.ENull} - Represents a null constant.
  * {@link org.elasticsearch.painless.node.ENumeric} - Represents a non-decimal numeric constant.
  * {@link org.elasticsearch.painless.node.EUnary} - Represents a unary math expression.
@@ -92,7 +93,7 @@
  * All Painless trees must start with an SSource node at the root.  Each node has a constructor that requires
  * all of its values and children be passed in at the time of instantiation.  This means that Painless trees
  * are build bottom-up; however, this helps enforce tree structure to be correct and fits naturally with a
- * standard recurvise-descent parser.
+ * standard recursive-descent parser.
  * <p>
  * Generally, statement nodes have member data that evaluate legal control-flow during the analysis phase.
  * The typical order for statement nodes is for each node to call analyze on it's children during the analysis phase
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
index 035dc9ba0c8..17016f1d318 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
@@ -52,12 +52,15 @@ class Iterable -> java.lang.Iterable {
   Spliterator spliterator()
   # some adaptations of groovy methods
   boolean any*(Predicate)
+  Collection asCollection*()
+  List asList*()
   def each*(Consumer)
   def eachWithIndex*(ObjIntConsumer)
   boolean every*(Predicate)
   List findResults*(Function)
   Map groupBy*(Function)
   String join*(String)
+  double sum*()
   double sum*(ToDoubleFunction)
 }
 
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java
index c0872dd1994..241de3c5db8 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AugmentationTests.java
@@ -58,6 +58,18 @@ public class AugmentationTests extends ScriptTestCase {
                 exec("List l = new ArrayList(); l.add(1); l.any(x -> x == 1)"));
     }
     
+    public void testIterable_AsCollection() {
+        assertEquals(true, 
+                exec("List l = new ArrayList(); return l.asCollection() === l"));
+    }
+    
+    public void testIterable_AsList() {
+        assertEquals(true, 
+                exec("List l = new ArrayList(); return l.asList() === l"));
+        assertEquals(5, 
+                exec("Set l = new HashSet(); l.add(5); return l.asList()[0]"));
+    }
+    
     public void testIterable_Each() {
         assertEquals(1, 
                 exec("List l = new ArrayList(); l.add(1); List l2 = new ArrayList(); l.each(l2::add); return l2.size()"));
@@ -88,6 +100,7 @@ public class AugmentationTests extends ScriptTestCase {
     }
     
     public void testIterable_Sum() {
+        assertEquals(3.0D, exec("def l = [1,2]; return l.sum()"));
         assertEquals(5.0D, 
                 exec("List l = new ArrayList(); l.add(1); l.add(2); l.sum(x -> x + 1)"));
     }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ComparisonTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ComparisonTests.java
index 355a1223273..73c0c6a3698 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ComparisonTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ComparisonTests.java
@@ -439,4 +439,22 @@ public class ComparisonTests extends ScriptTestCase {
         assertEquals(true, exec("def x = (float)6; double y = (double)2; return x >= y"));
         assertEquals(true, exec("def x = (double)7; double y = (double)1; return x >= y"));
     }
+    
+    public void testInstanceOf() {
+        assertEquals(true, exec("int x = 5; return x instanceof int"));
+        assertEquals(true, exec("def x = 5; return x instanceof int"));
+        assertEquals(true, exec("def x = 5; return x instanceof def"));
+        assertEquals(true, exec("def x = 5; return x instanceof Object"));
+        assertEquals(true, exec("def x = 5; return x instanceof Integer"));
+        assertEquals(true, exec("def x = 5; return x instanceof Number"));
+        assertEquals(false, exec("def x = 5; return x instanceof float"));
+        assertEquals(false, exec("def x = 5; return x instanceof Map"));
+        assertEquals(true, exec("List l = new ArrayList(); return l instanceof List"));
+        assertEquals(false, exec("List l = null; return l instanceof List"));
+        assertEquals(true, exec("List l = new ArrayList(); return l instanceof Collection"));
+        assertEquals(false, exec("List l = new ArrayList(); return l instanceof Map"));
+        assertEquals(true, exec("int[] x = new int[] { 5 }; return x instanceof int[]"));
+        assertEquals(false, exec("int[] x = new int[] { 5 }; return x instanceof float[]"));
+        assertEquals(false, exec("int[] x = new int[] { 5 }; return x instanceof int[][]"));
+    }
 }

From 2a196d4068e786d2983eedba3c466cbd089d597f Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Fri, 24 Jun 2016 18:18:02 +0200
Subject: [PATCH 1059/1311] docs: update example for finding percolator where
 query terms couldn't be extracted successfully

---
 docs/reference/query-dsl/percolate-query.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/query-dsl/percolate-query.asciidoc b/docs/reference/query-dsl/percolate-query.asciidoc
index 896fa7ec3ee..0d079f6072c 100644
--- a/docs/reference/query-dsl/percolate-query.asciidoc
+++ b/docs/reference/query-dsl/percolate-query.asciidoc
@@ -367,7 +367,7 @@ GET /_search
 {
   "query": {
     "term" : {
-      "query.unknown_query" : ""
+      "query.extraction_result" : "failed"
     }
   }
 }

From e6819648ccf09db6a47d794a6f09cea85b8df961 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Fri, 24 Jun 2016 12:44:57 -0400
Subject: [PATCH 1060/1311] fix the primitive case of instanceof

---
 .../java/org/elasticsearch/painless/node/EInstanceof.java     | 2 +-
 .../test/java/org/elasticsearch/painless/ComparisonTests.java | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
index 312bb10efec..5f08ff521d8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java
@@ -79,7 +79,7 @@ public class EInstanceof extends AExpression {
             // discard its result
             writer.writePop(expression.actual.type.getSize());
             // push our result: its a primitive so it cannot be null.
-            writer.push(resolvedType == Object.class || resolvedType == expressionType);
+            writer.push(resolvedType.isAssignableFrom(expressionType));
         } else {
             // ordinary instanceof
             expression.write(writer, globals);
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ComparisonTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ComparisonTests.java
index 73c0c6a3698..d9259cf2008 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ComparisonTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ComparisonTests.java
@@ -442,6 +442,10 @@ public class ComparisonTests extends ScriptTestCase {
     
     public void testInstanceOf() {
         assertEquals(true, exec("int x = 5; return x instanceof int"));
+        assertEquals(true, exec("int x = 5; return x instanceof Number"));
+        assertEquals(true, exec("int x = 5; return x instanceof Integer"));
+        assertEquals(true, exec("int x = 5; return x instanceof def"));
+        assertEquals(true, exec("int x = 5; return x instanceof Object"));
         assertEquals(true, exec("def x = 5; return x instanceof int"));
         assertEquals(true, exec("def x = 5; return x instanceof def"));
         assertEquals(true, exec("def x = 5; return x instanceof Object"));

From 7c87d39f0c62d5e9f84d9f199ebc7ba15191bead Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 24 Jun 2016 14:32:45 -0400
Subject: [PATCH 1061/1311] Revert slow tests heartbeat

This commit reverts the slow tests heartbeat added in
b6fbd18e091de7b22ba8020241d01ce5dfd8accc. The heartbeat has not served
its stated purpose of drawing attention to slow tests, and the heartbeat
can kill builds with SELinux enforcing enabled. While the heartbeats can
be disabled via the environment variable PULSE_SERVER, and SELinux
policy files can be changed, since the heartbeats are not accomplishing
their intended purpose they should be removed.

Relates #19071
---
 .../gradle/junit4/TestReportLogger.groovy     |  33 ------------------
 buildSrc/src/main/resources/beat.wav          | Bin 156718 -> 0 bytes
 2 files changed, 33 deletions(-)
 delete mode 100644 buildSrc/src/main/resources/beat.wav

diff --git a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestReportLogger.groovy b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestReportLogger.groovy
index 450d3645182..6ed6ecf8619 100644
--- a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestReportLogger.groovy
+++ b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestReportLogger.groovy
@@ -28,12 +28,6 @@ import org.gradle.api.logging.LogLevel
 import org.gradle.api.logging.Logger
 import org.junit.runner.Description
 
-import javax.sound.sampled.AudioSystem
-import javax.sound.sampled.Clip
-import javax.sound.sampled.Line
-import javax.sound.sampled.LineEvent
-import javax.sound.sampled.LineListener
-import java.util.concurrent.atomic.AtomicBoolean
 import java.util.concurrent.atomic.AtomicInteger
 
 import static com.carrotsearch.ant.tasks.junit4.FormattingUtils.formatDescription
@@ -123,36 +117,9 @@ class TestReportLogger extends TestsSummaryEventListener implements AggregatedEv
                 formatTime(e.getCurrentTime()) + ", stalled for " +
                 formatDurationInSeconds(e.getNoEventDuration()) + " at: " +
                 (e.getDescription() == null ? "<unknown>" : formatDescription(e.getDescription())))
-        try {
-            playBeat();
-        } catch (Exception nosound) { /* handling exceptions with style */ }
         slowTestsFound = true
     }
 
-    void playBeat() throws Exception {
-        Clip clip = (Clip)AudioSystem.getLine(new Line.Info(Clip.class));
-        final AtomicBoolean stop = new AtomicBoolean();
-        clip.addLineListener(new LineListener() {
-            @Override
-            public void update(LineEvent event) {
-                if (event.getType() == LineEvent.Type.STOP) {
-                    stop.set(true);
-                }
-            }
-        });
-        InputStream stream = getClass().getResourceAsStream("/beat.wav");
-        try {
-            clip.open(AudioSystem.getAudioInputStream(stream));
-            clip.start();
-            while (!stop.get()) {
-                Thread.sleep(20);
-            }
-            clip.close();
-        } finally {
-            stream.close();
-        }
-    }
-
     @Subscribe
     void onQuit(AggregatedQuitEvent e) throws IOException {
         if (config.showNumFailuresAtEnd > 0 && !failedTests.isEmpty()) {
diff --git a/buildSrc/src/main/resources/beat.wav b/buildSrc/src/main/resources/beat.wav
deleted file mode 100644
index 4083a4ce618139af5dc0a448d6a4d3569f2a180e..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 156718
zcmeFZb#xR<*Du^M-QyZ}S7O8n5}c3#!Gl9^5AN;<4svima1MkJBoGghkl+CV0TSZw
z$z<GmdS<%6p69%*=Xu}nu66HS>;6gC+Ff0{cI~R#HPgT9>Ah!iTuhAjVhET%Y}%-$
zNt;|n004jh1dMzL0D=q%paBAkvv|R#1pp=W>-c4XUl#affnOH*Wr1H7_+^1#7Wid>
zUl#affnOH*Wr1H7_+^1#7Wid>Ul#affnOH*Wr1H7_+^1#7Wid>Ul#affnOH*e{BKq
z|5zbNz0duh>3`Rl{IBDGC-=Xq|DS#R&$9Xb^8fAeKXv?P`Tthe|4RPf`u_j5|E~M@
zxc}2W|9Neeg{buJHs;blWwU0I{O|hzEe8THmCR-0Ukc4u`geckRzGF`Q^SAiGdKIs
zy8qthzw7!>IfDGB$N%ilJZ9Q|C5Rf=PhI9Qn$yoYncJAJ%{`m-{2ZhC`lnv<U;6*)
z<7Zp*Sbxfxa}$+}R5JHtE*q$1=9v$3ZswWm%zc@YnP-+U^UZyl^_p|D{Le9&>&%+X
zhq<kpZ$AFTpibsm^I_)ytovF2FG1?`kB)!j%(6e*{Jb`o&27#7n-84C|1}RY&n#m;
z%wzaDH?y4CuK)D-?=|^ZMuC6Q&-s}7X8W6S^YKsl9~;mB<X^fFDmN!H_dn%+_G2ES
zS;tSgpRzy4^3MqWv6EQ`1E5o*K>n*o;9s^jkKa53^L)&<H?N49_wz8%+dMwAPIG@h
z`!GwIXJl?`=9^`H+RWV6+)hU&^IDpZf9lD9mH+9*Ja+Tw|FN%G=I5H4`!~y&=WMS3
zIe&B6EcK7<zjL$he`L&dGJj_OS$ngDx%^Yl&$*j9=5d+Znd{8yXM6KV4OIW;xtUkj
zY%%k^%!xr+5dYVF06;=YNj~L4(?j-<_JAEwOV*M{Ouw1zjn2k!;~?W+<4I$d@tx7l
zIKcSGP+~|n6dTG6MnkE=WH?~RG0ZlcH_R|xGOVYrhZ&9=*c3;H#~Y0J3@V>tc#69i
zPU8lAFD}DR;Gkg>F2%>=U+}^B8+-^}g2&)hxFep8<NC|ERDTZF>i@=Ds60=<2EVOe
zfZx<F#Q)N-#1H5v<E!+;@HzVa_ze9(e6+qV?y0xL`FaG0^n{+JXX9$jps&F?_202J
zeJ!Tcw__k)hY|WNjE`$E4@!18?x>%F+vyXid=4J0--Hj;AI5$3n{j9TI;wstF48CA
zOe*ihM&V7^V7wk1fVX4Ask}d4iUr_5uu!}W8%EWSz$>vZyae;c-(#*6x!~Ct6Tgnh
z^{21~it6;ou?qbL?5Tbkc3-~~OVO{v?&+6fH}$izKlIbE-}IBQ{rU;mZv8-Ph29sN
zrgy-`=q<64dKe4Scj~P5<vLjZL5E?lbaL#j?g#dpE(cq!yNb=$9m7WG)?==^c#NS7
z$Fy2EtXFG=RcZ;{8*PuSK-)@D8%15ZG;N*kj<!mdOyzee{%viG?!2~A_q(=KcSu{J
zJ3!HTZGmo)HcK~0o32aHrs*bXAL&MF@9IWtAM2)RU+ETTzv)(Mn{=DC65T;9fL+l#
zVE475Se|wi_EI|o`=(uob!tyydTlZ$(7nLIb*<QZoj`w7*H3?6H=UZtK0Sz~=|d>H
z%*NRG9xMdEjm^M6V~24amPy&Z4v*LK4EyvW4e9!AhDQAZ0}pRBMB`S*_4owiUHo@r
z1)gt23}Rz{gOh2UVS?$dVXvvd@X%yqtT9bAvWVlxfkdHkIiWFLBtlFj#0u&LaLW`*
z)|r-)*2EQZ0#QO9rM^fnh!8+V%mX~h)4*J^7&uGfz)R8_6qDmXCtw>m6G#J(165!K
zpa)w4Z^#2&2+ab|LkGZT&;zg$`UXO<9`uF%plR?7=peigN`;f55Ab`a3vP!%8VbA6
z0^s4avG79L5_lJFH+-IU0lrU5gI~}-!sWCMxQj-@7|n)8M}lcK$S9gOGLsg8B+*79
z2WV4~OSC1(J=!MZ8SNnQnRXUwq}@d1v{VE}G7&qZ5b2A2LB=5U$ULMAS%WB$eTWgc
zgs{;}#0mX|^hFiONR*FGLqpIN=wx&cx)r^M-b6Fdcjy<i6YW9~I!O1SJJCneqv*@&
zQ|N!tH`4Ry=jqk-47!^Bo^HWtqen7mj2R3M#vaB9MhasQ<1=F~L(RCtaAxK)#xtuK
z`<Nz17SoN{&K$|KWi4lpWBtKAz<S1f&gy1rSdOf|?8&T^?B7^_v7fLi*gY&h$C(|=
znZ(}1+0V}AWU#wAO>8GFhclTQ#rci9l=Fgnk)z?h;{@<DoCQ30?q%KtZW(VMm&ecG
z#_?OZ2l$q}XZ#qRiocx~Ab7xAEU4vO6EOJS1>t;#Z~=di@DzWQ@EQNAu!H|jC=^JA
zQ34Cm96_XLk6^m!hG3KEso=EeyWoMSOYl}?6f}vrLPF#sbg~E$4zw67oMACexYc5o
z@S??XVUEQ%;SY=7g=&i%LcV3D(8uzvFxs+0ILWeIxX4m2+-Qjj_gNCc6PBRpq9r1_
zX-O9)TcV;=OGxz4(kOgvsS;*b_6RdAn}q3>mBM?LpM*Cp3xpRe9|=!d-V`3OJT2U7
zxlg#&a*Z&_a*lAJWvnp4GE5k2=`0**$q)uxDg>^U)dGQKp}=I3Ea<X0D)?@(Qt;Gb
ztl*}Fzu=GsQ?SgUlOJpGn(u3IgU_(o%I^|Q<QI#4`FBJFZ?CAFH$(J*7bx1x14NT}
zWkL^LicrJdCVb0{6Q1YV3KwzP1bw-W1O#WR;1g%4;39`ENaTFxdvnh5d)YJjnQSNi
z7Irgl0Q)x2#9G8FV0rNNu$sBitUtL1W&-yKlgr)Ae8UN4?%+rm!JNB{R(2xeJlm2n
zoc)e2Wo@BfV|meISl>_ub2s`Y(+`bhRv-$-LF5V}1R2U`pmoxZ)Bd3MqeanM;AZqB
zd<>0(`=YH-74iqP7a0ioB3<A&+68bYZ5ZfD>jgf+mw<KfaKHkVk~z>davBs%>Oc)~
z8N5wIf)j``z+hSfq?lO16w@7&Fb*SAjSa+P<8H!ea3oR;1*S=cc_t$cm{Re-jZ^Ru
zMnd0VxTimAn5OqPAo@?#oqILD1QX!4SdRXaZjOErb?5D?dw_LlCt=sMTHRFbbsbMT
zTvx2=(H_v8(+<=O)@sz<n#byMnw9Fo8Xt9+x=nRneM>b&Jzv$Ua#USaRV(9Emz0=t
zn(~g)S~*!+sUQ@W6{(6DipdIFg+X2=zb(HcA1j|GSII2p*JR&iBV>Qb#L}^{b5go&
zfb^BLMY2bFOfo<kEYV2Hdml@-^)8n<^twp$#Xovyi+}Ic_YCR1(4!Lf?MW7Y>YgHA
z(oOadUH5w~c1`IC>@s%0>AcfDr*mSrssryj-jUqp*)guGpk32Bz5UNlW!tFEKiZ@n
zfo)ehN?J#CtZr4db6an=r?telk8d%xDViU*oo`;y7S(Lg*3k66^>EX!)_|t4*78Pi
z%dW<h7VpLdE#DiQTJ|;6G>0@?Y;LKa+I+d*s(C_vbrZY(R@0}twN0n%hBeKqvuX0F
zQ#9&ozchZRO>ewbd!cb%?cT=GwMmU$weuPowNo3FHRBpvYhoL#YvLMz)QoQ|ubI(U
zQ?snGy=GS<R&%LQRQt3sw6>*jX052{aP63;+}fi};@XlX&pNB-1$B#?Z`b8EchzxQ
zLhF~e?5i(s`CcE`>eFzcbzcM6R@<<qt$$-f+qK3C?LgD3_T^2H9pz1rIz}}6cIGwT
z=?rf1>bl>O+U3_8)cv3}x4UoKu%4%FUwekN&li7g*NA6zob2uD@Re-qd?R6W&6i&8
z!lgdl=Vf`_f%2H1*Yb*<S&C(1xdIa(Q6B4cP}xczt5PJx)uGZK>U`-EO^i&VDU}`2
z&X99-&GPfQ<qCUDs<@7=S32pH%Io@dDmz@PI*Tt-GYnPg9fq-*F2i%p6r->9sqwtl
z#YAWinwIL?O<#1Qh%oFg;u3}t7&exis=q>}>Dx&V4+19Pn}I*^r$7g;2LcTd;6_6d
z_|$M4G#M(uVMYdYz}O!uGOmD3#*0vt=@qoeB!(WFM6lQt1$z_o;CaLW_#BY}zb4+p
za-tn}A!)S9q!;ZFIf9l=&ZV`JTWPkyIa)lBPCEpY(h31N%>-B=(O^GhFE|G&0{0^j
zbQg(%J|o8=4e||gLG9o%=t6iq`T)L<YTyQR2+fNAJ1w4GNjpOKLSE3fAUb*p62|aC
zH!$|2IgDo1#287BW2VusFl`t;%p;6w7RETwTFaDDs4|Ydg!P!+%ChDxX7A;6v3odc
zIn%i~=PmblZa<zAFO8SQ^Wl%--{H6NeFeJ&X#yKzxG+ojTsTQILZlRxh%Q)6x9D%#
zYEfso-g2K6Xysvj*6OvjyY*t5Ol!n;xXm5gQkxNWGi{sfnr(O5FS8RlblIgiEVb|N
z*lPdLG0|a$Q<X!T(*(!0&Tk#H&XG<#T~eG#7m@P;*ImxATZi)jw|E!AEz4!QyN#>D
zeVgktk4o2Ck5IR9p2yvudp5gyd4;<l_d4t@_9}ND<?ZZo&3mzj)ccOd5T6#06Fv@}
zRX!6uU48d?F7{3HyyaWzS?g=`6#6-Njq>a7wZ(6eSE}DKuPVPCUWos3uOR;`UX%Tk
zz1I6b^g7|6?v+f@GwQm8;??_K_3H6I<)!l9=Y{#N^V0bzdMW+Kcy;-Qde!<{Q!-l5
z=l<V4@B2USyySnt^N{~^&$a&ko-_PS9z*=UczF2V@Syvz^XT;(;qk@K(&M3DllxJ>
z`|gYVHo6b?8|ZH32fB-Wi`@!+Pr04;o$QwAYv~r~`@>c5bKdo>&lJ}`e1xvke7?9i
z`W$y@@{V!2<qfzj@qX&;@4e1h=H>02<5lal-Rq3gAg|F*CQs7wnP;x!X3sT_!Jb}@
zogOs~H$6@{O!XMzfO<&mbKI}lFLaNy=eld{a@}s(&2<}Rhq$S1(_F9EPIVn*i@Vg@
z+;rJvGt|Y!rqTI@^-kwRYfER7)dQ!iRzsX3tiCyxSk8A`U@37>S!{JUWC1u>TKr~z
zTg0&s5uLKj7jo@J3V*l#BtUE@3ijBP@zvH-`Ae<8^UAHp^P;WZasRR$z$GjmaAsQA
zaUP2fv1y`S)->TnR*E2lsp9jPgZV2Mhj@kb_gqUl%3Xqv;N&2?*$|S=8b|A9o`;>7
zwNO069okM`3f@BR0mVoUDWUn1JlbL+2)<?-2YoXx15x81pugcPnS>`3$@(l)4OVF6
zVV?{`bU*MUZKM8@rVD$amh0M8n3k<FYJ!!JCP9I!cgdNmzhx}tTPaJ?C1J@aFqVb%
zBGSoXNV2Ji5MS)ZdUCo{-Q}IVUCNG*PC-XgM{s+6`^2`owvDazt(RIFTk@M*np>MX
znpjQZ#?VGZ!`ud}{%Ac|pHs)KYpb=cwWxKi8Bx=xdSi7&)$OY2%JRyQ6-edSAN?w(
zl&|<Pr|f$9((k2Zo4+AtN4`dXzgoKOTgI21uOC0FOFK%!N@*n<zj%Mj`8@6;R<ilS
z;F4R#M?aOlul|U>5B@ml-LVgw-*y#eyct_8f1Un5{I&i2U9Wz7_u(b>*8b(%w@F1^
zZ(bKJd*f8t{d&iXwXbUnbgzaK9C&p<U-Zf@Kjr1A=YwA|pVt;0dA75N@yw;@)YJEc
z4o^20K78U?IO0joi>ADbFAnC-coC52@}e$RQgAl+Wx>SU>jhT1dkUI!mKJ2?Oe^?3
zXI#OuoVbFqIq?ONIWr1;b5<3&<s72u4t4#R;u#9Y<b=FfnzQo7?>P@%<m4z{bmt5$
z^vu0jxI9-=n3+4Lh?ZAUG$(KP%YwYZm)=i?zq<6~<14|_nXgYjmA<xlcKl89v%t4e
z&nw@4dw%5IhWr8V?F-cJKNMsa?|HHFL;u3KkNQHVPx(dKPlsNXmn6J;@;UJJtuL%M
zr%J_dc6}{-yY^e*yXD^>zh6}Lck#mViys#M_~YZUisPS>Do>PbsXG06fAz&LCu?q$
zUah_N^<G`xx4ioI-`_UWmX$Zk%3GR{3VE|*CD0mPC2X5e?a{uvCaU9j?YPbdbql&a
z)o<<=HyrI@HC-3`H9zi+X?ZPK+*&C;)FzQ7x5J7z9X86gPJb2LJy7l16RU|9Ptwll
zP0($YOvlbhr|BQd67W)af<dO3W)!Gqn)<5e5|cFZ$@SVrz-irL@G-U+D$y^3yYcxn
z#5fyqF(se_h<N%`as(p@h+-ZA{aH652X-FJ;*`>KoDQUwYeY+U0!9wsgLz#L!8#}$
z$zCp+%o%4fhZ|_Ql*hA5;)|^|3*KAr5#F--O|;AQoW*pzzbr%T)2uiS&#gNhifvvx
zR@h#1YPH+qthArv0yy?{rNF_>*;(Qi?DE-tsOv+IiEhU|7r3wVTJJH&d!MJD&nYj4
z?{)7E-+Ml<{IY$o`xp6b>hsAzE}){1Q($9&EU-K9WsoxHY_L9fQHUudv@Z|}g_5CP
zLk(d!!ZhJ4!+RsbBbp)&5oP^~B40)A@0Z?xNYu3fc+|mx>HSv>T0CG}wB^8{!LJ7L
zh9nIV53!1VH#9%`uVIS@ZyU}UGGWByA#Nil4ONVk4J{aTbl8y?m*MfFbB1$bM~!$B
zTRLKE+_aJ2W2#1e9W!;*=CL0~*^C<ylQS+QW_&z%bZh*E(HqDA7)^{1k3Br$k62`4
zTkO7xp>djt+vAo@dLH+65*8ORIb_WF$&1HGCZ8WOY|4u<r>3-wDVsu$5l*!lJ9?`3
z*v(V>j=eTDV(hc2;bY6E29E8V>O596l|gMT#AD!u;xSOdjWPJtHDkn6qsEj^HN@pk
z&5JudH7Rc4RL8i0sc&LsQ<ld*n1aO4pYrEu{*<86d6S>S%$z(pMmwos)bUACqZ}qZ
z8ks!NWu))K6C+Y4XokB?m_GdM`0Qc8c=oX6@pFfM8uws`?>Noi{bL6Xt{Ss1+HXwZ
zpe=Fuz@pgDfy~$y1L8+t?|(ApbJWLCrhdq%KK-IcPK(?)VqZkc@Vns^!%D(v!xW(*
zLoGvR4C&W*@8Fps$<aH4iw9i`>K^zkkU6k2z-NG}&xroqK8aDj{yX}O@H-nh*Y|$J
zcAui~3*J>>*<P~HAD;A3jfZnzYmbPKe(vLfr@Ji*+Tyw+@S@A<fIR1vKGjb7{zk`g
zKQ~9QZ;S)YceTB<&w0E4-UYT(y*g~xc?xY#cnr3_@1A7!-tD?&hii!i&DA9Ga0wO-
zcU~x5;B;BA$FY=u#R27K*+=s}*=^u9+1}x(Y%1A^4TEiI-H+vJwUp^+d5#fk@si$O
z)Qd(7t<YhDLCA3aeA-CfK{$qc2O7<J2aaZU0kJF=5X%f8V;J$oFnW?{AbQdmhCDR*
z(>~$OaF<>H(exzXgo(*;T^SLlePWuUxoTXa-fq~fnvS1PhUu><cvz~uQ=27wp(&7_
zQ@@ccR~7dTP<{||6z_T(WG}lPOY^&qND4aV_rB^FApX$K@A=Z!+*R88sPl8nj*d^w
zG3~`oqPDk<buF(N?lu?IZ)hs48{GJ!meo*DQ(0F~eY-ZlYHdw^<-qFv3bd;5$M=fY
z<u`s5m#r@U`aQa=>YLzu>(}<L($eRphA-#8u)eJMZ2vi~B%s9i(~wWhk5fO2i<f`+
z_I_XSlXq9&|NS=a-HA7qZ+E;lyjk_i^UeI1lU~m#+V^ToVfM>OFT_QY3cQOF@|P6O
zd7koO*)ws$wx{6*C!Xxfzmxav`Kw&-=WRLrpK)?pp7zTg{d9R&=951&-JVosoXfMx
zVCKzEKa_j>F`TP>bTntwBT>$cw7;`W4<oY|Jp7vV>A||J0S~OQ9^Eg>^u50>^WHtL
z%#eFc83m~~GbW`j$`IcT%=rDTAwB5sm-MQXyXi+#4yO-HS(&c8laQWwCnkN*ok8hw
zclxC}P^3&Al>Q}oWcs7z3F+sO6VrDmZ%j{0KAAp0`C<CB<S*%ylGPMBXCx$#&6uCO
zJ7aBfX2!wfo{THWUYS|Ri!#3@r(~*<#hLbZBC<x`IheKWPI=a&JAT=ncMfE`r_^UJ
zNEwiGE9FW~M+%k`bazJX*1J!0-`;i1vr0XXw<5JSuPAla6Z?Cgo@~1}<Z1Q2oTo$X
z2R=)^fA^W?1JCCN9wa|kKk&_8{_t^r!^5b88ELNzexyx$k?^SLMfIa~h4UXnMUuyt
ziuR-jy|l{s@G>)F*{i9UjMt{jwAT-_X1tl7O@Hf<lmE6RXUn^rx&7X+$TPn0_vC#s
z^XbJ8jZas6eEMwYr_0YBN_OPyOA-sJKgYhv{}NevyVSSn<X4xMJHFY!TKV1n^_()-
zH&e<3-p2nJ`Yyg=_WLQ7JBsI2{rO>e_3Mw@YGj{I*1CL7shjrYRsGr0=7zGbaFgwK
z-{ys7lUmZt_q4(lkK1Newzg+iIdodoOz+xKd#Ss=uBIoZ!L>KDaizq)IahkIg)Zl{
z%~YIdf2`znqU!Tq3pCC>FSHND&e%Z7etoI56JH`9ZzL4CrfVuUa)9O}P_M<o{n$#_
z7q6v#HN>MkOfTu4WF+$|@Q}3=a^rZ>E^)sg9R3#iUV$T1CVb7BXR(6w(UQ*#v(DjP
zvza9X>`2iP``eZu97bCQI(6Hec0O*WbO~^n;#T6A<-W>U<jHVd=XJ;Jv-dC$Ki@jf
zgMOR6tNppY0RgxCb_Mq9^D*c{fK|w>pm}|}gOfuy_w5WLLwzEC3ttk+ioDtH_kI;o
zjQ*?v2L?nB)DK)UC@K0{bj{#5gGUY(54kstJB&RdVED3;F(V2`%^qnvdiAKqvD;!Y
z<Mxd<j5!oLWZa>+J@I?TJRQGztbD?<ah{VB;-^lI9=~^r^Mt!ojT1g3{FvA|?fxWU
z`p(IMnK4tGW^tx^&i*{rXU@?C@3}*#In7f}<0amhjwQy-sGHw4<Jp2kGtVxxpS5UF
z>a4)UA+u$RGiKjk5;$kh(pz(oWum!vm+hQ8e0js%isb|5EnabIUeAiEc`H^rCN{2|
zo*2LC_r%Oqxru_+wTX*Y>k>0obLOj8+s_Y9a-F{@$z}e@B%Ar^NzC~llT?YdNk0->
zlO83uB<)SCOo~l>lf+0&Nh+MTKWXi}$w{{J9Fm^StzSK5?uFGIb7EF+m?K}Mn|)^0
z#@T&Vb<N6OIc?U6mCt6rTwyaaVnx!7zn2$GN0y`06PJyimbG+$0<k1(>hL8EQ}!%|
zr(`X1o7}jt|0LSNF%#VuBut2!KYM&^;;i^d^QMfOGB;*y{G8A+gJ;{s`OMOdX3YE=
z(>6V2)YEBOMjlNVHe%9L#&F9ipN5uAIx*zL#Mr?@CIHcr`0Rm~$1NH#cr34fb6jq}
zy|J?*?MEBKACI~kHe_UY=*QtDA(MtJ2reI@3Y;`}f53-9>^_4Ap7nds-`dwY>XP@q
zNE@$?@CzPe!-VdS`yO`{gdi?!g7!Fl48R;b`>eCy?bl^n?vrTK$GgJnfM>jAmHS&!
zu-jnaA(tooDyJ}BfMYsmpM4O!%=RJE$EFWsht+-bi$x#AO_T<25`;j-yez<x+n-#|
zE;PMm#u)7wrT7|jrv44BL+1c*)@}kBnlI#aRUaZudDK`U@4}bK;`B6Wn(lV5NIOEj
zUESW@to*%eoFcUIiL9>OQ+llJY;RO6B<^Y6)P19=r*mrK!VX?TL)*u?xvj@*8=EKA
zENQZ@mNax$Zm)k*!K^#^<3`Q0^8VE^WtEk_-?vrpzuEuLmll<Eep&gw`m_DFZzZLr
zpFbV@^7&)T=h6>oNkwt-$L9AZKPcWM6eI6k-#fmQzm0nH_RWmfH(&31wfohbm-Ani
z6~z>xMSTkg7rMOI{=%Z*X#pqSn9qJbDqryI%yWmQt<M6U40tv&@6yu+x%iXAIg6g8
zXP4zQXN}IY&w8CZGjn9_Ul}DilJr?QL(`?%mmeR^#vb`*&v{guRh+gfYd~6XR>s5b
z%%F$&GBY2n%N+C|I`hkYVdk>?y&25=pEI)VWoIn8cPqpB-o=c@)IT!rrk={!l6o#<
zZ0d~+@6^W`jMU-`#og|Vmb=2t+PlLj+LGCPH$78&SDp!__RDfkJ)AWxwK{86YFPHw
z)HB)NQZ?Cvdx<#-_rB!Zx;HXccdsaS^8Mj?Irl&1IX{^3<n)8yCya-Oo*sGV{*06M
z?%Bn(b<cesx#qukRFOaH@tFeS<4G@4(ya@pWi%DCGt-LRX6|`;C~NAg*zCa9q8#>{
zrX2B`oZQm4C-d^&t$K3%{kW%Ri~Bx1{K4V*u8)lT&7bu7>q{gBYd^QWSo@{1aD8cA
z(blhZFL!@ycy;7^^XpS(U2iUzE8gDvVS4wdf?J$l>HOhSRp`gs>ex?`n)xMU?e@>Y
z`U_vY8nQ~G8oz%X+ob%K*lba@p(UdHNbAfW*V=YhJZisP`MTp%RaK|DTGHiGOY0d|
z=OEruAJ%)fVT`1_alRC6-Yg4jIVPXkdR=k2?XfbWy+~Ex@lDO>YSM&v%eC`+NZkoB
zSD)8wkGD#^4Ln(haiF}PX}KbrIIkQ^7O92<J?degg?1=3P&XJ}fDNJ@)<+?!co<r2
z=tJ)?x-nqFis?Y2tPnuW8U{A7CqeHxbKz9(3ffWLMr0v>2ijk-m(CIHW7G@xG9QX|
zv9?=mWsk62&tY4w<bJhU#5-#}lRw!eULdfI7JjnrBRXVfZ86ZEw2;|%SpMbk(Q3To
zLu=gexXo>+rM7X-L+pB;t?W;_blLZDedh4q^>@bwZZn)PH!tT??lNaLk6f2bj~%W<
zJqNgzcxv3Hcs+El^qS`}-J9!K?w#v7&S#ESk&nqM*!QaURo`$Q(67X2iQfX>w|;70
zcmG3vyZnXz-~F%od-w6^v!&17KCk=u25<vX0uln;0<Q;L4r~tK1bGDR4q6l_4!RjQ
zBe**7Rj@F~FJyGk$&j5vl8}c%WBb+w-RX-28~X+Z$A?Y`z81PRxIOgu;67o=!E3{w
z2d9U94VHzq1_y+zgO`TGA%BMRL%xRFg^=N{A^s7bA(JA!LN-Tugj|ep4#|qJ2`P!-
zgtSJG!MX@pFe9=)*ebF(*flae*e~)zaB$?-;Lylv!F?mcgM%Uk!9I~aK~9mcgM^V+
zg22c%LDGmJL6s5Qp!|rc!0Qn=1Ghyi3Y-|>8R!+!9Ds#i3-}N|Ip9(_8n7rluTNn3
z(mu*Ci$2+5FZ?%#E%Of!<NJ4q=KB2=I@>QHl=P*C-tx`uJJNSy-!>ms-vd66L!5jj
zhvayxg2#BD39k1F3QqERA7u2L7j)E98ED~oFz}j(AkfX@O28d=mjG|~TYXa8-1~UA
z-Sofd>fmqZdcp6E3)heCa>#eLv))JHw8m$NQ?qx4<7Dp`$5&q24k2D{4%a+S*dv}=
zyCjeKc4h7#ZKK@7Y_Gdrw;^3=HVa%=S-*7oYUS(_X0_e<g5`H7orRy%OpD!)&qSpT
z4kA~F9m4hYRf0mh2m#mbG=GMzn0L!&6i;k@gBxzG<LtH?&v|Ego6WP-vZh+ZvaX1(
zG3$g9hO=-eW1ip~{RY1iZQu<+op>ja+1w`D6;2qfl6@GqWLH5GS^m&T<}UCf;|qW?
z+<+1E&E#J61Cfuo5_;NJQzY%1aU<Nvm;oIz$iZej7#xJJ0WRqul5$K;#9@7iTe_7d
zT>HQ{McZpg(*zk9nsxYMb(a2>N{6|r24VY^2XrlpkJ?cR3++AmJPludU%gePQ#DIR
zsp6&QmCq&ZiawHlia&dg$%Vc3vZLa@GD!TJbWcyC#Lyij+1`D&SJx#MZ|NE@)^tAZ
z+1zQ-qwCn#y`!VK3+RaNI@<oc)1tjk=dHFI9bs*vj`yuc+vl|c?O4l>whJwq*3gy>
zt>2pEE$f>%v{*E2oAa7>HP3BAo6)9oO__}zO$!=x8bys`8s9cFHSB2E*3h>>(9lzV
zzdoftrhajKTb)<^k-E;hK6MZ4%4;{)?X8Wj3#?_;HPqDBo~wCOJGSO<Exl%5?fdG1
zwa2RMYbR72YOJf9YMQEwYaUc()oiQ!yC$~kOpQa;p&EJRwwmI~H8r;?m)7j5oL@7y
za&FDw$~iUe6tOB3Ycv&$Yg#K-)s$CkswuA6Tl1pgL``1BwVKR|hcy`$MO0o{lUt#v
z$*<tnzNrYP{ai7=wz^_{ZAZnWT1`b!Exl4)>sV=97giZt7hk!(Ze``ey5B3C>(VN%
z>%UjV*W;B(>OHE8>L*m0>UUKQY<O6;v!S6Xzd=+DG{#krYCKqds_{j2O{1aOw`oAl
zj;5V8Uz%RjxHrSK+nUGJerY~e>)Fy+yQ?LzuA*gkU0Cb4y0fj`^{Upr^$Be?^-tQO
z8XViNH0*CTG<3DkYn<Hirm>(Sps7#i#ipyBP&2D*ee=$)=4MIP<dy~9uUjg*BU<Bo
zQd{5mIJ6BFpKg0DM%$x$_q1pC>e@pk8#^9L6dgg*^_`ETs?H$U)~<9J(G?~?(w!?8
z_6$_~+4D-#M?6OPTwJb<?_Hp3>XoTBOZKYSQi0}g>0g>CS*W&D_FB7KK2--Q+I80z
zTd*i4Ltn1EO1*ap#5w9hJXJl}FjCWIXwqyk?$I($PP*%+d|fax7kfhx*bMS7eJ?p2
z-wU+kBJddXP9?w?4t+Jgg*KX|!PZ0v{FK;2n?)iBPM$}u0PbiM_!#{G4x(>>is?Lf
z3gaGJ&lpKt%50$ZGPfe@S$tH(x`S?E_or*wpXr-9i3|lt&RE6W$86;aSTlJyS*5%{
z_6Ytnb}m1L<0tsexh9y)<p?{tJB4d_T|yOas%SI6P^97eTWk=VvycdIi>1Ou%X(pv
z<rI;V)jQD^t4NCytJ@YX);!B~)>|zLts5*kHY2TO*xa|eZNsw`+ibAzYg=x;**4rJ
z&-RRs&Q@X*X%}O=$?k@2x}DlqY!_$eX@A{rwtcVN8T$eD@9YoTWA<O{eH|<vraR1Z
zIN)&2;ekV~!*>UJN1elTM|;O}j)NROJI--LoOU@5aJuT4<do}p!|AKzC#OzFt&_pg
z#hL3g+S$=5$=T28jB~hCmUFaIxpRz@%6Xg<-({kczsn@2krd5!8Sk{oWvtWhF0oF3
zx{P#s>@w7;z-55b2bVCX?=JpMl`f7>H7;DIS{H+3l}o2%nafwlk1kIfi(LM6%yQZ1
znCvp&@q)`>$3rgmjvHLm4)a_}9b#Ql9s0T)aIkZk<p8+!b?9)WJA8Dmvw!5AZhy-8
zko{`sY4&l>KK35YIy<A&Yr8V1lXj_233j`j9PP$Cwb(j4rPxXxSK8(~2G|~Tl-Nvm
zOtZ0dTxC<^;A?Z$q0M@_!wqYZ!wl<B_I&FD_HV4B?6+D;?R>3n+f`Xkw>xafwhOf^
zvTd-~Wc#~?k8Pwytxb#Qgw07&v`v(#*Sbx3#rlkJq;<4VX4NCOY;{pE)M~h(!&1sW
zZh4s>Y&neo-J+Yf(c(1E!Xko~A*$g{5bfrQg|6J+gfBVn!ugy6fr&jsaFwm%_hX;r
zm$AJ0>sW<6K5GW=5mUj9XP)3n7(U#Sj8~jM#$3)9x}Lp`evWNH4`An^Z&<U@*(`v1
zPoIn&Va6c#%r06Q<0x$?!<Y7r{sCS@UjeI727DO33t6Khp?gRRI0)GfmeO3ng|s3-
z2QLIpz$oAjKOmn$6Uix%mgohq6Gy-igd5mnDge%zW&+VBE!k~6M_w}aB}W-c36)_j
zam&CXCK?`@2t3}DhW8q0<HwC`+|T$-Uu;;RUuLk?qlRMa4!#YG!TVs{`X=3R{RLfL
zeY~y`Lv;JF7g|4To3>mRs@<!TY65f*G&R~4nqyjDO@D2dx<_+QeND4kJx&v>Ce$ib
zraD)(OubX(tRAYWP|;O?sJ<#=Ro9fTYPs@-GE%uqi6{pt%c=MC$%>bXO^U;c7)7kY
zPQg-0<e%k5@-y;t)LZ(cawqu!d7F$ce<<saZIZo^MawS9=(6>)@6vIy8&Ypsk`$Ch
zOY5Zq>2qnj<f=4ZvRisyvPhaF86%CAgiF07E>c*+mbUilB(HkgB)5CNOAhrGNtX0J
zlEn62lLYl1lUVj{l^Dg#ByHlElF#CCk|*L}l4NnD<eWHAa!~9g*(7$6EEU^JW{Pbk
z@nRdvaIv){ilShOyeV>|cp|EfNy$KBM+wp6EHU)BO7I>}iu@=FmgsvTBv?<hMAtKd
zqA?V2G9@=tLK82PaKx)5*5WM^H}OG9p!l?;zxbwPl=!h^iuk2ufw)YvR@^DsE5;>%
zh<Vb##qQE9afI}pI8It6PLztpTcm{elvLDvSL)OIQX1V`Cr#+pNZ0i8Wk-8`Ww&~V
z%kq2Y$SQlc%9Oq5WE@GF%v(|{8!l;;&65zaoe~H6Wl5AgM>1XhU9v?kmt2tZq}lQS
z=~ww!sYJe7%2b??dMVPRBNShyixev9A%%_Xu40hvgJPkqS8+@xP^QZwl-06XN=Uv>
z86ZzpQks+p<wj+?+*{QkpQz$0cB-NkDXKM!@2VRLQdO!5QUl7l>QLoL^%7;i`jWC&
z{YmMl0aW8PVXA$aWvVRAbybJvyUI?>QpanDs}E>*sq?hCYKc~%_S6MwX6sgI&gxP%
z#kwvHsq@iBV#~B^u~hAStXtcG_0f6g*Xri$vvfD~I$a&*>t~PeqI}=-u}e6P{lFvj
z7KTmw35G2F?*^6rwILEW8Mfn*#(aFek;ESwhZ{PLzZpDCpD7O(OXF439Al*^*=SAl
z8mALsrmMtuQv*?CawAb<F*%O7OP(dV$x5OR;7BF`iR2^T7AXZf$RN-kNCMXa_rYwS
z6VwA9&|ol;@@cyYy#>o5I>d!zp;7QrXcy(jmIWhl4?F^Pr|pAh(F)+RG#oCb^`jBA
z&9pEio3;#5)2<;A$X8?wf}&55XcR*>qy5lF=w`~lEDLp`%h6f%K>9iQ3i=0n3Z104
z(8C!{jOC1JjBAWPC~va&43c4D1T({!^O?(;e=z@KzF?Ly#Y`s4mNkSminWEco|VSB
z!RlarWVy0+tl4Z&_C@w&b}4%wo6dR69>{5Aui<bxe{lwKzH*juK<+6{Uv4%hkz36<
z!6i6(Tz76WH-^jNt>A|7j&tYm(zu6tpSfwgUhWSbn`h+t@!TnYvKam%-U|L+-f{jF
zUK&4xSIRHuDftaNOM#jfAz<?p1Rng&f`R-Cg30`B!E%1NU^ibWILqe?@ACbG1^l7H
z@B9Q|Cx4|-&)*|t2+jyC1$Trlf+s>hL5VO-&?ForPzpy0P|<jSooJdMfbvO;7A+FQ
zi&hBch}H^Lh_+I`X}bk`L<a>&MZXEoh>i)aicSb_iB1aciT)5g7M&JkQ}l$oeopc8
zDVcoH0qS}u#oH*z5v>$tiWX3E36yVJoZyaVpx~w`KyXpyEI2OW3HFIh{0$;8f3c{F
zpCEe8A0<lThl?)q-9&r&JkfH#Q8<a;BOJ)D7JBiEg#v!A(7?Mb?Btyie&=l!=JVzY
z@9<)UCwL*kbvzs4RGvu?$!ihV^4<w>ZmOV``@0~YyGn4K8!y<w4G~P|S_s0p8a|I(
z$?xXm^51bT@NaWA@b_^h@aJ&?_yai%zAdMXr(qZJ%Gg(U8SG8GKiCs_tJwj(I5val
z!EWW6SkJkYtTWsU)?)54RzGeci@^<J)o^I62b>?wU7S12ahxqo2TmNbhwaFG#+EaV
zvEMLeurD#Z*((?-)&NE!i^Vw3s-`Ef?$TW;U$idfNcv-@h`yEChz@2xKvCvqw3IOv
zz06>v3mH{N0OKy@iLw=WMIVivqFW;~=xsDNdKRq<JwSVmPNHo_U1-s$6h_hKa4B*O
zzJyGN=OW&)JEDa;Xz!r=wDZtfS|Sum^M^FB8q9-VfLq|BU<5oF)IyHn6Q~{70X+f+
zL0bV6I2?Ema)BLS71<xWMPk4zG7ktNw*olP4|qjr$zwzo<*B-s@>umH>rE|`2kSLr
znQ1!F$7DswsBeC@@tSF;alUDk(c2_4ijB>N3}dEYyYZl5m~pCsW%Qzav$gm&LovR{
za2@wItiu(QhxJq30Y8At_2cljdTac;zCpiHe^)<NzfSM2AE4J`i2e)q4ZDrqz_wwl
zvGG`c%o{^7Qdg#{(fzH<(XG{;)(z3E*70?tbxm4#U6$6QJ*utNCTjDvQQGrb3+*PY
zT$7;vtclV-&^T+4X=vJI8i^)GQ=#$Fywt#&hni;fMNN@<ujYn&xn{R|qGq<bpQgXs
zMPs98Xf!INx=dB4&Q!fqpH`)-*Qu_mC#eprLsje4R;oE_gECg#q70|*%dYAiB}aWj
zsaG9QcB)n@D^-(~#i~f<Q<bCgfeKPySM?}Ps>&35R0WDPs=JDLs<Vm-s@;mAs^yAM
zRf57(HB4cx3Q{mt_6nmCRY;Xed9$)n{zF+J|Db#>FHqi>XDF}9Q<TT$*Oa^Dr<JSa
zN0f8qdz9nko0Ws*Ny-rU5~Y`XuF_sUO(~GaE9vqWrAZd8RLLThU9!H)dYQlSyUa`Z
zUgn}Kkl877WERRa8CRJiV<>OQ5amr7jUq((H&wnv@gK@8mDw^UWxmW;`BoOLERhXW
zmdWD(g<*}dWy)6BMrF5bk5VT4O{tNcq58V6MC5mse0df%h8Id7d9gB5UZ#wZH&9sD
zqg)}!lsn~g)hW5HDp~HQ%9jsRRmjJy6!JwXj$(((OL0~;MDb8HQ}JH4Nztl0qX5;9
z6fWv7ilJ($VxgL+JfaR#-dD#fKdV<O<?0hkq2_@ySW}{m*GQEsHA2-<O{D6sW{&Ee
z=76eGb6>^KepB_);;M;SH}z(1too{Ut-45iQQf9}p=Rp3)PXt+%_Q9r%~stS%}w2P
z%?I5ljY@}WoH1W*9G0Nni0#u}$L?w0QhwwT4A)xeJ#~Zh6LpIyFU;Tdk93*(MqRaD
zh(Y*p%Kv^NHVse4_EUcN4{=0aLE$HfFW37TuIMKjKIwNFO!~Wq5d4Q>F^(E9Q#kqs
zUur}R7md+|x5kYIl`)mV(mKOb6VG_qG{l%|T4(Gq{cW@&%8bJanrRIYX}U~(<6aS$
zO})eilMQJw4I=}IRpdhA0(prjBtKL53Xv8-Bsm0FNv;O|BrgNs$hTB<g#s7|ID#91
zSnw{e39JMD28G~fa1^Kqw}YP0BX9!L4DNuepkyc(Dus4J2Ivvw3pYXuun;}~4~Ns?
z&2Sx@3NvUma2Sn4OQc29j?<E8d9*)it+X#R4noqxkq~4SvVijXK84&to*|!*4n&R!
zPzyAgLf{qXEXvz^H~Jd=8<nANQ3rY#8bfE%*U|gXFVn};3n@hIpdY347%B7!#%uZ<
zMl<~Y6%3NXaHTML2&0oRm%(6eXZSPEGUA!{8S9uujC0HiMmAH*C}W~b1=EGeXZ2?W
zuqHFduvRjcvGy|$vMw@jvhFjVvhtbVSYMdktR^PFmN9ME1T&D$VGU>7u@czcti|jw
z)_V3()-Luq))DqJ)*1FZ)}QRftW@?&Ru+3L>jirw>jQf$tDL=q)x_S#>S6C?Y1ju?
z0Ot^k%{fAO{vBnxa*nfnIj2|woHMLo&UscS=Q1mTbA=T}<%2jES%WFw5YAaDKTXj|
zRy5}r#XrCr$k|EBY+{9RlBjk|SpJ+jEFaDkmM15U<-!@tvf)Irc$~g0nBz}{k+`xH
zY%5k5o6c%vW6W}P53`tE$9&HI!c1cqGXG+yGtaXBW*%gpV{Tv{W-efFV@_nRU=Cr=
zpynIHbYO=vnQUjKmc?Q=v$Tv7Rx=}q^@;H(E0=MQ^%r9W>lkAqYds^1HJ#zk8pPnS
z+!zKX%IIK9=wF$Y^ry^M^uL*p>BpGY=^L28(PuL^&_^(5)BPwM<1qv13Wg=UjDb@g
zla1&V#%pvpBL$t$_#GY2NJ2drlTerufwt0Z(KmD)Nuf6&ztLYI%jl`dX!>cyi@pt^
z(G!tYbQJO$^+S?TE^-)^)0UxSv@vKdEdaer<Dz?Ly~sk^M<kYtItZd2LPWGhh!!4(
zRKYIDQ`khi0@u)X!cS?l;VZOgD#F1TUO)rkI9eyvm-Yp6pgn;QS~4Vq&p=i1UZ@CO
z1Es);(C_dBXbn6VngRzv17JJI4W>aXSOKb_Mz9Y00=|b{fZ5cy<ZtLt@DJz|xC`0?
zu7K8p3D81t7?c18LUEuS6b+(KUr+^lflZ(-_yuHx`Jjo~)yja&U<+^<{6T&5J^)L=
z0$>K14vYiu0K>uSKtJ#t5I{wBxPgZOYj7990XG34xC+n$i+~<rE>I6l2fhJQfOkMV
z@EjNgWB`MKR3Hkt0fYe;fDqsm5DXjz0)WGSA8-Kh2KEB(z;3`9*akQN8vt8iHDE<)
z5&<&+9xxVQ1JP7`S`ZbJ<qBZbokBvznzc|7WL0D*`Gq>aB|nkR$d}|}@;P~n%q1_7
zndEUYjXX@=B@a-X1LQSwKY5PaOP(ZmQ{3&;c{90@Tu-hbSCfh4QZj*@M~)?@lS9ez
zWIr;73?ZXQKQf&3B>Rw#)LBG2kqBu^>Ie}jCYWRsVIrysIq{unB}#}&;vMmscuBk^
zo)a&KT;d6lL8S-8L*h1(N?fJRr->WH5#kcDi@MrC93_?!`-r*3Hexcdju=fWB?b|*
zi4bA};ZBSsL_~iAAOZ=g$&IKrSrG3{G$PBSGTk(_m`<3!nzow?Oe;+HOtVZEOyf=a
zO(RTeO#@7GO?|17k7=mM(G+eHn*2<N$;E^ltxOUl!_;gv7|V@P;~QhMG1FLSykRUc
z9yh);ZZ~Edmm8CfGmYnsV~q!l(Z==0VB<n6Hf@U0+BnL{Huf_@Mt`H;;AE5<1jZHv
z%~)p88(tZthDU}r!xcld;UE>mw%U+yNHE+tL>n#`JPmuOn6>2wIX=lyh4(kS#N7;O
zIMZ+ym*c-tv1yy|S9l`+5Fd+Q#{1!ia1VSf&ckO@@oKSnyFL>Cs`td7>qYn-J&6CI
zSLnCtTl5R{-}U44FZKQOkMth;D|&(cpx%hBq2kbH>&vMaub0>W{bQ_;{wC(6KaKJA
z2Qg5;6;o4tuP$sM)__gN%CWIn2{ssegN0xPm>ZUZSzu`xf+b^G-Je*8?jlyLJA;+z
zj#IH=hp}h6U09}WBX&=>3cI0Of}PeSVh40HvCX>4*h<|PY@u!_6*JZko2CoFChPpM
z@j7oTPUnS<(s^P-DH=pkKb<=jx8;U~=$x?tofGD#bHu!K4wxI2J5yx`if>ERTT$(V
zln%Zwkc#yR!7OxrF&kYd=BSIHWTP=}T`cv?I}HodEyAL8>!^NqVN-R#VTro)*b3b(
zY=bTx+odbSj_OLW3%W+^Z=DiL)1mrjI!FC`>KC3$-6$&dZH8X2Tcu}Fv2l*rX?-Ae
zOFtCL)=$CSQ?Ycl`dyepe+FaVcd59$7g!Xv7nq1ku;n;SzmJORyNZYFbEtT-ulO>(
zn2H-j4L9^|hUfZdL$!XU0n=|W*x?rp1MqCa9BQAi8^@@fvZL`CKFnB&FEOg{-;8`i
zs?pEz-Z;Y0Zk%m^OdAa@rsIZzrewo3Q=wrkwSPEjG8k@|9E=5~fmHn293y7hW3(au
zHufXlQP1VQ#%+YqbcKj8<rC9Qjl?z+OkOehkWWo<WQ}PhwWs)<bR_PR1Bs91ETV_p
zPB4K>gg1~!i~uUA*fce<0}zoHfe<nim_U{SNn{Ujl0?A=q&xVL91M1mvq6N4RdWZ<
zgTsKw;C!GM+y^v)HvtUH2ZT@+;14N*kq{f4OU18kh6aGgp$XtkXfc=xZ3ACJ$Ej!j
z8(<rhN!@+lQt@onARq1ponQj=hlNlS>;c8V5!AjU23ia!KpWx3(0+J5^as2fx(XkI
zlHm(b8hi`NfgeH7;pfmx>NkUTa2Zqr*F)do4yYQILXEHvYKK9%8|K0i*bbJ#Ua$fV
zhShKsjKf2z5aZD>oi-8X(q_Rz+G5y}wi>pfZG`P;TVY4qHrScUU1*!Bd?V~YTTj&|
z!ED-cs@+0Z1J9=PPJtWXv2YnY68-=Wh6~~Ta1I;+--r9cf58FpMc5lY4!gqpslK<u
zB6u~-gcndDxYMX{je#Z5V7LtmfomWS_$y=uzl9j^69|VMKw{`Gs17<0eT0rePoN!8
z3bYEk0L_LDL1Ul|&;TeA3V_BzE>ILCg1jLFvWD~^1L_5hU^6HOE5H`81S|)O!1rJ_
z_zX+|)4)sM-{3Lu9Jmua2quB+!Fg1g0FDEr!Tw+X=nXo6mLLlRK?5KKBtR|D41557
z08fCA6au{ft^!#U`rM^Z=NhmT_=7^41Hb}cBai?r0me}0!4z7B0ug{0m8_^lGw(ww
ztf{B4<^zQ_xzss@!kLTYBl0MjOkv4&3PUcC^T|`>H1Y^Jf!srmA-7N%vX&f1t|SML
zOUO_PWBe&}@g(C(7mDjZVT?5yO!7z{3RT=lBVkLb2tL_Mpky0C5OoAbloM(SS>zO^
zh$&?0pm3#y$R`?!r$jx4Ff~LrQ9)5Dkwbi-=p99GD9m|JVa_MwF7b`HL!nX%C3~M}
zC(?;tB9GEnNEnHa1Wf+;7lH{XB(o#cge%p*7l~4M$EVQFmKv8E8A**SnhYREQDcuM
z`;${BWSl|9QuCNdE~C(L9XX5KLCyI9RXR@1^gLyUo8&<Xb5D^4<W&llQ^^MM2^9(b
zfr^f<BY6~NyHL^6!2lN+0yqH^DeRw5q567if3P1&1kM8MfLqi`WCG`bS3n9-My*#n
zwPIRoCqbw7BDNp|_Mz5wFz5+R1;fEr;7Drqr-0YMC14h~1N;b{p*|_8paFaZT0*s8
zFr)$FAU?DR@`rXqqo~hcB6JVh1{FbPpi1a2B!>zhCj6cHw01&+U?UZIY5}i-{oy0<
zF!%;M6V8R#!KLtTa2HjAXxXq66=~X!)&x(cY2np09_=@p7wtCnG*L*KMysW*p=oJ{
zX?)~Qnh%mo8;pFRB_JKNB*bLi>mk<=UnB#GM&2Qlky>OKB13i}FnR*9MXw`)=tE>U
znvYCFOOVxQJ#qk*B3DoXNk;`#e5pHHi$+k-9ivf@J`=T|FGoG-+t4uj5p)Fo0y>3$
z2VFwXLATT2peN~N=->2aG@CA?_Nhkn8=XO~qgznVCeCy@-IuPThtLTsK9$ZGLKjj|
zL5_^kbYDgsJ(Mw)K8P`%K8i7kK9-R{pU9X&PhiZZ&t%M{o@Em0a~TULT11~s(R9Xq
z`eeo&`Z&fkI^{1*AHo<x@6YH@{Sp~O4`z7L`!F2oJ`4-GCxcC=ygum83<K)OP@}dC
zDJo)gqijYywU=&24fI-6O)p2K^iOCv{T13m&qHhIY3Nt_E%Yt@GWwK$0!^duL2uF5
zp_l0Msku!;570-VTj){fN_qe~kM4#}qEl~A=q$7!ok0BQD#VfAfpF<n2!Vb?Wau-b
z5xtLmMXw^S&|^p@x&^t7E=JCylaYhyFk~GXg3Li(kT`0u-VeoTK4?452K`1u(0rPb
ziWIJ=B8J}~hiK`@dfHWFF13cSv^7X5Z5rZCtu2G*i6~(fQU}Xu@2KeG$M6%{W%ved
zKYWC?0$xYmfoIdA;24@K97<zTpBpVir!_+=_%qZDKc(*I$xt@@2XqDALH#nk7@7yi
zLqjQK@Pr+x&m#aypf>O;R7{~t8WkaZ8r%kL0Ovqcsr`Bc=nGkc0!U9GQa#WNzNQfB
zK9ElB!!J_%@7>@MU=cVThyx>mK+qWwfOP8VSw@xt)zt4{MdWoLg**TpCzk{3$ao-u
z`W_CX{-eN^lu`GeuVfdIL4G36k!i$6@(eMB+(Lwsvk3t?l#mgg)Ln>4+#@8W!^Bt9
zLL$>Ngg9q%BG#L5(?nC1DcF=};+Za*WX5f#QsW#`x^cMaw9(hJ!N@aBH>!+-sokcJ
z@wriGylFHU4jOw5ON^C<7~@-mk1^fAFkUru8}=JM8J7Nk99;vL<60Yz(<V(d_Ux=}
z+qS)Tjk#;vw(Y%Z+qR9-jyvtBjFbO+{%4-kped$pPR@Dt$!Fz`kakc_9xsQ=?PVFT
zwsgrPKZlfq)6y4dqjW=>A?=X{OLL{x(jcjd1YUM2R#HS=%mn=Ht$1I&F76W#igU%)
zVt;WG;BdXf7_pwH3(;Z*j0ax`xx#7Tldw^^BTR>J-ym3PCCm{j38RD%p{JnmErcu>
zcfE&k+jZUs<33ba$$R*J`7FK{|DA8lKj+KxSNR}7@U*~2J~yy{{}f2%Zw3bQ2LoOC
z6@g~p>8}pX){=a~KnPzlK=E9F4+#F8K#u=s;Jg2I;JN=n;JW{M;H3Y2V2}SqV7-5T
zU@^E~r~20gM);Qndixgy+WBV%8u=#$YWl|nD)@&5O85r^!u@>%7Qh2(Xu7XgK=Abp
zczitqE@)ZMQlR~S{U3dO0<V4j0#AGc0(X6b0@r<m0~dWm0;gel9JUX^a<6Y_V25u+
zV2f{TV3Y4(XwzVQ7T`<s1N(eS0!MtS0w;W%;4ymw=X@sumwcB3H+^>lk9^MqZ+#yF
zzkGiJ*}h!hYWxA!NAgC02p{4v#V7b{@zwmT`NsY}d`JISet>@-Ki<EIpX)!yul3*J
z_xV5Y=lt3HeZRtg@Y{t9e_6rrZzLGO5gZd3EmR9E6j}y$3H<_>g?|HYgk^yY$VHHZ
zivgSXJWyUt3N#npfdQhCpC*>%H;B#nlj2bRF<@Fr;z6Dl@AJV@5?@0S`5sc3FjcB6
zY?Jy6SEWV5R~XlNrB6bLEDQDIGU5=qySPeTAYPVFiOKRe(V!qwZKa|#T<I%qR8~MP
z!WGG({FOpfv)n>$Bu`ey$%oXP@_Y4}tf*eOqL!fa(|Rf^wB^br?Yi<u%T{nbM6IN^
zQv2xB)W!NS^_>1q{RtcrhBVMBBNMfL$PsNR@>RQlQ2HOFE{r|L>NU~5`cU+(z7AFN
zTWA#|2gaR2*bbx__7?dU!_nPXBN$swMT@XgXcV4=w!_WX47@pZ6rYBD!cSlt{u8T2
z81epq8muIy;#Y{1cruZUQ>2BcPBte7k&}s4<RRiR`HA>VqNGOFBonC7WLs(vTy5S$
zk0_J(s0vgj)sHgL%c!dK8Mvx^q-N0`YA;+p?$Wg(#h@Rg7|f?R!x6fq;U(R`P(XJ#
z1OxZf(6HDr95M{n8m<_w8r~U_4cP|Ts2OS)!;J%hiCSjtZait6WPEL0Z_G2E1}^HU
zu{`s~*p?|Wj$jC8ArryuVyZ9~nP$ujraP0u3}rmb1cqd1Frn-MrVP8Bsl%>eTCwYy
z?(9ZpFuR!<#cpN(Ww$fa*&WP0ST2KoYuR<oW_AU$9q_Fk>>Oy*pe4fgNN59Ly*smu
zZ40dtvzx5}tsJb!F+15%xSwOTvN$|mFm7b?jceI7<4X36aS{8}IE%eu{Fglic+_5F
z4|c7w8Q@Zt*^z+5bp|}HmeIjP8b30c;TGUgN0_I;ksLS7V3r#OGb0VHm^Oy;Oj!fR
z7z}P>9{t_;n!aT`MehatX})n3J=EBOZefh2OBp4KG=8Tt4OglAhAq@i!z605p(EAN
zP>L#RkV)L|nM?(&=_b9H+)Vc;$I;cumNZSq(rJW1-6XzHn~Ag3cw#x#f*4Fi5Vfgd
zoTXmiDdYkC961A@OLoTFktOj^5@rv?YwQ}a7h6h<$GQ{su@Zy<u1i1A=lFl<PJA{x
z2JeK{!4psl7m=UXN8}WC44H$?KssP;kq9gjaiID7P4tnz3f->vgI{k&v<XDwgupeX
zKs${*(-tE8w4TT`tu)dJc$YHT3y2NcssDnskjrXaeT{0=|55*F4b*d5kh)kaQ2J=E
zl*-yMh13=+sp=5rrrKE9q{b*?R8?-GrpTe{J-JBP53|Pk@?m8Va59bMo=Uu2Ng-rP
zaY*U%d+EM>8Mv7p(tLTYG(;XMwUAp&W#p<7D@Ov_KubR%{pOLBBc7Iih+CzX;yme&
zI6^uPyv-3n8g_^!r8S~WS^!AHRIykbE2fD<#gAfN@qyS?yePI64~tF2Er2>K6RV3e
z#7g2gvAj4$EDcwd62SJviY?&^(-4-m#CWkPAQP3uGGaxstXKj5%8ONCtCm<wYz(YW
zTWH<kkpp1=aJXlJSVNpH))5zp^~JSt#7?n=cnprZD0UX_!t=ir2Z>+BG2&lws#qv4
zggl;gqE*@@CIBu{OS&qyl^%)xq>thRDMef;<->och^HizbYF^xt7RQ2SL!S&(kLky
zKAB4LCaIl#TpA_cl9tNvqyuney#-l9Uu2V9AeWSlkO5O#=^-~$66Jxwjn7by$(xi1
z@>%6KeENPlQ?V+DT2YBn+bRvz5lUZmkupQwr)*PiDVNnB%170w6aeOA(yFMHw2o>g
z!21)m73zBJgn9<>t5=#!&Cx6xsn>*=MqjOsK35x~@6*=kcVW(+1ao##bHL1l1P&kp
z>7_SArs*&z&=(<B^ux$^{Sjm<{RMOoL0baiI0k76$m1yV0<seQjGRQhFcJzz^I%qB
zf{{`cY&!4)+tG>GHJGD+MNeWy=p)RC{l-ebsJ}i&;@z<bd@NQGkkrO7@9v79!A9T@
zv6=XHY&D*b?Z$EZBpwMffNJ<Nm<xZy`{7ym1U!H*#EryOJd!wtS0En1Yyf7;kUi6r
z@Ix*QPD~{%#8M)f*i2L)_7U}o<3xMn9MPA!N{lA%0?+@HSWdhlwh~{7LqsxhhR7kV
z5pLohA;CQu`3kaY-V!YNjxfX0OuivF*k^!-k}n96cm~HlCY&(e$Rr-XjN>6Z&m-a;
zEMLI0KLJMLA-vLE;u1XTMR@je@GAciJK%N$+%6|J67z`F#AH|-MJy%;5Oay{@Q&LP
zV~G~T5TXImi>O9)BFYdg;Xi0d*oc}01^<VL#}fs32+Vs7@ZU=KOF$m(;OY2z{5yUK
zP=d|)J>Xd`;L~AiG`<C}fTehA*s29v33y*%VcO#u-T?PtmGMk04iF7H{tP4V>sSEh
zV0qXcECpMOeF5g?C7=TLu|e1stP9-J9NUZ4#@55!Z9X6a6R=>cA1Er?U;<hb%SB6I
z-%t*Ff(FowXexRDeFkX2S#&zg00*J7(U#~?v^?4twV+ke0L&Yc0hfLPnDi;+8_f0}
z0K$9<=KDKfuD=Ya1z2-DG8{1G?vM=D9P#SakzBnb@<$It-s=qV5K_LbK#JFY`Zt*6
z-`BV5Ct!xZNuRCH)5ijK+#6>6E%g?9b(kr~!i=7U2r@y-)3RXp{z-eJ-Pg{;{Clsq
z8xY@>+6-;FHXQ!C0P5R7YoV3Vs%t@7yoPFKz<L$+x9U`%!|f&Ym3lzE1z7J%bsmhY
z$Ez#Ve(Ef>HLTT82dO0h=e4O#R8*~^7AtXTiegn?DVTai@hb<ETxGTLOPQv;Rt77#
zmA1+mrJAx&i3GHlQ06K{@;ErB`zuf6cFGyKuCfKr`&n|3GFV0dn=6nj$iL(eKzC6X
zjTZr?mntoRUUrgvA9~mGQdjw))ExSf8uDVPv^+%$mxsd$y%%JVwUKzKo>U-Jkg_3r
zELGwF=T*fo&@a7{ev8k6CBH9S6t7BqppRN6?vo~p8>E5ae5t*dDAkADu!_(l$B2z2
zn^+k*$T&$9f+Ux~Kt>o28DXmUMDU9@g#z)skS!j8%&YCfZ*c`=fz1>?L;lxWaggv7
zvcK+#ZG}sq2RbQMflROx!fwd^+APw-8d2hxiUs^!P`ynNzw%=M7ac6#<$H>k_>SU#
zd@Jz)-$dNW*Aut$HN;JPC2<2^PF%;AfxNA<;tIZkxRkFVE&?odKHNW-uPM&rYl+kO
z+Tvt*Od{Vv90QLBOjsPuHy8WEvNzvU>;bJa?C$`}Hn42PHx^s+4PjXiw(E-RpmpHu
zi(TNpo_tGqTzhdC-%T9L!|a704zH02uQ@|p$uAN&@@qh?w-w&!e(?l<0yx|Y;w}EB
z_?UkrzT@ACKY-6o=N)1p9}qR3kSu~-Dj}4RYD2bK2ceZTT<9w;5GF`_g~iehVW;#(
zxG3cdZvfxThR+?5>xfZuSFyG{PV6Et6UPF+yA<%<eR8sRUDm{patX;Pw~}aOv{Xu2
z3u!+Wq<@sJ(qcuD4#5@qvC>w~Q6|bJb(dUIeJb}?-ST2JPC2P|hHJxoz;e$hrL;dv
zH;qx}YmL;?+9dS{C^2yThgwZHX@m8a+FE^qc2mEl<pLrV1~_d8P;xASS?e`mdYsVr
z$D;$0zUVGw1Nsel20XBcb^``$Gdc|N!?t5V*ek3zrenLYx_B}+8F0$e_$WLBKMVb}
z2m0u`&=1ZdUP0f9K}=y|vIRMZTt;3apOS?nL6xUkQzNKV)IRDp^@U;qJLya}rZ>}5
z=r8mMI>hjY?q@I?_Ct18nqjt~gz>y#xG}?U6vnw3#!kirW|46ubJ_SGlVNl+K};pK
zH8T-7*YoTp#>M6`<xMf{Xvpe1Vp`57nXa?JTsGU03o<R{nwoBK<4rDZt0~@m$JEoD
zVOnlBa@Wk&xeRk3&Ssg*HMQ*HCR*-r`z?RC*A@}9K;h<iYh7~(Yj5*(>lE`r>jv{n
z>nU@w^@%y$mSk=MYUW`!#<Ij#!gA17*K*s|!Sc;E*iv9iv=H{$mT>!WOF8={OKtmZ
zOH=z{OB?$MOK1BjOAly$?f+Q@+K*a>+V@*V*tc6o+t*vh+LuC`3+q#1-#E)K`%udO
zdoN2*ds|C8dwoj-dj(4cdzdA{j#-R$r<u2XGH2Uvnm^kPnD5$_nNQfpm^awknWx*z
znfuv{=6bd~F4*>lE3%&EURjrON3H$2dDe<tcdKG5X?<^UTMn7-TPB-US(=)<TUb++
z<r|x6-pigc4`s)jOR?q64knvB%N*tsnSop-CWLbuKblS%*O*2cTbdG#5}Rtc#O^hW
zVEY(CSc~B$^OBy)%%#gR73p8bBx<E`9aY`fi27qFAQu}BkfjWr$XB#LOrVbv9NnI{
zKzZ=a)IQurw#8SI9;_630DDBV!3Glfs2|^oZouoHrSRX#U2GoG6N^S%=zV=TIz*2_
zQT+yTLhFWf0QR9!U7>GPWA!>JAhk+I7~iI;WtFMw2bol7$h(y&xuWt&dLWO69+j2S
zq)Xx?sh5aAa^Dtlw~!##6Rrxs_}0Qq{x=`SPvmb02uKp#7r+8F0;m0N{N4S-{gO}i
z9rtbbb@f&DDc+afv)<v}{$AZfdk=bUc$#@edotZ&o>lI*?#k{Z?jObV+)Ikx#g&WC
z6#pq2RlKsOWO0q6%%Zfy6GiI_#}+j#tXSkKC@S1raJ{fY!Tdrgzg6Mcd~@O8{GSCu
z`G*VMy2cf(bk!_q<<j#>*T?+V&cpeeoip+WI6LK+c9zQb<{|k{^HN;f^X|FE<Q;T1
z%3J0N&KvK_cl30<a5Qlpaa43Ia74NWI9OK$hvJNK6gu_XbZ2(%cjx=ux6T{6&zwhc
zA2>JV-f=F;y$NoV>&}03uQ|u$-f)h9eZz7eJO6>>M&|x>j?c|;PRkXX3vxNvs@zhp
zZMltIhjaV7F62&eJ<Q$U`kH&j<;;EQA{;rccn6)|#8EzfkfU?{Jje9>eU1b94;-)Z
zvmL&Cb6$yphI!o!#^o(7*qwK);7wjpftFXku#R(7;biBL!t>75LYFhTsG@67QKIW$
z(Ro)!k=s?SxK94$;>G#5i{IrF?#P0!?g<6^-S-MyZcAZf&&a|po;!s(p5UTJ-YG@f
zyq}7k-WtWNeA|l;`}ktbH_$!Q|H%EsU)EDPu-UUFpm+)bW4%53AKu%1D_;rWvTuzL
z=XVPm{QX7E|5%(7s37GAwn;;IP5Q-;mwO4n<j+DIrHgn&c`KGv+e`b@7m`J5DKF8U
z$X>05GD3f<eA8R1ZIP$y9i*jJ1%0L+Mce8@*c*L0)*11`%x)~+4NbwnpgoAb*bCw>
z)__dJ&yy0KK&>S9P*yUC-brqvqX4lzOf3gwS2CQT6OGjj$;KOo4xm=K1<8bE*t#&Y
zzRD1$D(n=~SvJ{J-qeyiWje`~0+r}NZlXEF{KdS^T-SnF_E=_FBujy%pLK-wy7h<E
zZfj?oWV>y9XDe?nYu{~OVyA<C+ouOr3rY)G8PqQLPte8SD#0Nki-YHcd=5?yi3@2G
zIw9mx=(P|nR0<gs)-?2H*z!<w_^Z$n;U;igbPi)9HiQk0cpY{*0uRR`8-;g^oDse+
z@_cw^WJY-TsE~*WQLQ7+Moo*zjXD@nGWvDIplDCTj_B~nchQX_v6z1%YsV~$920Xa
za#zfw$QLomk=_`8WL&H%s$Fb+)a2NjQ9EK=N8OC=9rZJIWRy2{a+Eo4Zd6(DKs1S4
z71c9tUDT+!jZstMwnoj1+ZMGHmaAcVL)^-!t#I7VxWuSkaf4vlF=|U(y{NTuC8HL_
zaZ!`w{E<WAk|R6BJ&LRycO)_{Zeb)7*FVx1TQf2x)*SgHHZS78*ryRIV-G|OkDVUT
zFt%NUEjBVDGsY2qHRfvg(wO<-U1C~<hsDt0snKu4jzn(=8xh?(EFqc+`y2H<bbr)>
z(B4tCLb)hs$n(fUA&Vlrht!WmL%b1}gU?3%6FfA+9&C$v8uU1PdeD^cQbBRyU+r(g
zme{9;Rk4SLCE0F;F1Pg$t!R@&K3ETiOtv-+u~}1tuUh5>_qK!t^X3adyUcCCLy=<t
z!cDXP%NgwycffYYRMl3`^uYR=?Pwjx{;*)|Fv}h$&s>?AVt!~8xNgP;T)F|_rWlr)
z5Cd)6NUvj~Xe)b)+RoIb!k8!I9%DD~O{5X~42eXHLBJ2wEAUdZ4L?mC!YWhMuuJ3<
zv<BG=y-MUDHHn$XRq#vHz<1~uv66ZP?5cJgZKXw{-_-5MaMg?yDy#Kb3Z^5<Ty3rF
zR)gd*>V7FzDJ}I@E{GrGx?)@TiSS733aPXy{B;=3)exn?1z}B~f)E+_4}1_M_$K~?
zfzQ5}z;NGwzt<b-U*X;5i|_{dE_yb4+j>}UntQEhhMV=6+#B7;i-X*ai+30QEQ&9l
zR&=_^R#c<tV&R>_ZiUSY-36}-wik3Os8jGK|7ZT#{8{;eD?ER->$WSxHOO_=iMg6O
z&p5y3^>!xa;Z7>=YTgmY=)A^`$h<$fA00Dumpg)T8#*rK_;NevoX>S;kIr3_T_U$)
zc5==K$Q+!W)h8z=D>&zM=9lbQnVYlAW_HQ`ong&hm+?KTX~v-}fBJ;1i|GxsCZuy&
z<<iqLv;W@6Jo0y4=8(U`Go$|2&rC|QW*$h($rzFLAfsa1t_*+bl#IuzT{1SNR>|m}
zYRf2{%BOo%Qq!NOJWt=3axQ&l%J%eLDGSqUrTm*7oH8ukm)s})M{<|+JIQU+4<$EA
zUy)oVePVLe^uEdE(pw~#Os|q0mmZZIn{G^wPZyIC(p^cV)Bh%wPydxvIsJ1|&Gh$4
zjnm&Ib%Hh|{ZrCp*uOMAJ865mH|bP5mi#c?lAM$tn=GbROODEDmE0g>K=Oc$Dams)
z)+X=EIF|e%<6&}UMsl(-Q%SC#8J#j9vvJDG%pob)Gnb^~WS&R~&3d2GCW}v*m6ecs
zI;(4Ha@K-Wd-nO%4%um`OS6O1?q+vN3uLcItDf^PZDNj?b}py(--4W}f2-zR{yR0-
z^Y><M?Q|t~ZhA|{qxAI-D&vQvOGb&jof#AJGBWPuRnMfIOEY^qe`X$WR>|@>S7x<x
z{mt6rYMAYD?aJ<y@6A4y-z|qIIG^)RfhqT4!Gzq{!gsmz3oAR43pY6$7kL~9iu&d$
zMGx`@6_<71E8gmiajVWH?s2Xh_YYS`PpkZ^o@<bWSh`@jcTYi~*HYNux3ch=Pb;kE
zpILO!UsPlYj4qxRNG;9_^mX^)Ke?as9XwTqXP*5+Q!gXl@y-+L`0~XozQIxz{|D)`
zzo}d@a7jK8NKiufZOSGdQyF2Bx=Kh=NwKZAOuVWQQk=e0+MpX{SzjlQM@-5eWP{QX
zwW{~g?P@hFQag$r)k5$J`WpPQuHm(jX~b>BK{SH=!iVSwvIW+ZdVyV_+T$_wdwd<;
zoe=5o#8^W=^0y(0>|h*B-7)^9$}uD8?MybUGh+<n*<8a9c8sxw$zeQi8qb7sE@mM&
zfz9O{Y!~xL(@pbVQ=DZWx7_lJ%eC}0x3|7EpRqQxu(qp~L|Yl_TiYIMSvxq%?F($b
z>@Hiipg#6hL67XILFI#L1aAvk9;^j@2_6?57xFbYF{Dw*osgp;Oeh&TFm!b2+0YlE
zQfR5L)?rJ-Hio5zeGO|69vZ$aykEF8d{21uh+pBmBSIpIBf3Sji(C_NF!D)+CsK)M
z8C558Yt+QZw5Su2m7<d(XGdG3o<z5fVq@k;^^7?mwI?PuDm8|UPKd1=Jv?@J^ugHm
z(LZCaMO)&MqT9x4(F^0^Vy?$EiE+jaiiwS%6Vo+*Tg>A4^D!6VU&Q>0Pl>@x_+l!R
zU}Jlhh>M+9qGs&T5-nq2mFN-cD)CP&n=mo9e8Sw=)(OjEhb63wotv;Zc1yzc*b@o6
zVs9kuiG7~1H}+e??$`|2SCp_mR)XWOlCxv!l8Lcw$>Fi)l6_)rCELdamuwOnTC#d<
zNXdj)dr4cYsiYc1lyt;M37=yM6K=+&Cme|RlCU)9S;DB8YYA;*jwF<g*^ofR%u2|K
z9+L1Rx>dq~=yD0OqM3v)(asX3qF<E|qmGw&AGNT=p{PD3CPh^)(Ig5h!AAXz{}FjM
z{&3`?_;Hav<EurMiRU8%arYyh#4V3l6W1}KOPnpj9``-`XYBUygR$MhN5vY$E5tqx
z%a2(Ub}pt#*w`2?v{cNU(6s1jq5GrDh4zb13b93R2ze3JJY-gsKe$ZP>EQ2?gMybw
znu2Ra-U-T#7#p-DA~>i?#3OrNc%pq<c(}b$_%mB}*ks$LuvlCDu(#HX(An1Yp{1;~
zLqAzkLgrbPg_N<B3;ASz8$8#X7+lh93Vy?#4VuJt46<<r_UopN_Wq`-cAfoX`;VP$
z>%y9CKIW`-57Wllh)K1i8E06Q8d*ze<5BYqLw)mT!)J~#jN=Z|2-k$(VM?XSnif;H
z*aWHr`;1IwMv&7Olr%E?iG#+vL@nbt{H0+U-p^pi^XRLPao8JU=>U3=+J#o9>Yy*k
zA4q?4CX!EtAq$AxI!6rB58#Mi1wXDmz}jnFv3xZR-K0)NtEo8pN!f`^R>~tb<$-=x
z?xFXQb2Uj?pdFIzS}W<4>Jl5N+r>{x6LF;C5F~k<utjbpl$Ue)$I@25uhfipiG_hh
z;{HI8*e-BN@cNqw`~5%o*8YinF{se@0W;Xn*COEeX82Edm->5qOZaK;W8Xc`Fy9mp
z>MQFx>P>OC_U>@!dAhhadJOJbo@d3s-HVDBxf>Rjb_a^y7GEx!Sv(PDB9)516%`b&
zF1k|KxM)tHr?6GwNpSoPDNHR0DZEhdzF=X&3b<-EEJ!Hu<ooka=fBMVCx3T-BwRti
zx$5L^a*_ERT;E)z>xAoxbGB=-vy-d2v!si1O3rtAU!B|Y&N&C=t#p>k8}2M{G<Du^
z#5fl@)Vz+4)Vy%VqrB|gqj}eJSLUtE9iP`Pw`*R7+&Xz$PE6j{94ha0PLX4A&Tq$n
zoM(<YITsxvIR_m6><y0Z*^3-EvZp(CXHRg<%l^kPG`p{(MRq4g>FgE`I=h}DH>;}S
zWmXx->8u#X`mA8b^em%eNEYJgkR|5U&+_M%%kt)iXZdoCSwb$r>$ydlm?JxrcKppW
zInpwNphdxYc}GTO14m|N7e`j+2)KWiBR6x6!<D(u;m$no5HcS*u&l2RE-Tj&k)=6G
zWrgKc%c_~zFso}`yR30}eX^G3jn3MWH#_TM-kPkJdHb_6@~&j*d2h1foVi)eoK*G*
zXW8u4&UV=so#V29IoD=mu5;OyT_3aixjflRT_HIaTy=AjT>Wwk`LlEC=I_ZFm47#9
zYkpeJ{d_VvC%<a0xnN*!y@EBlBMWZlZZF8seN_<ePzriGY80+>j4gcPI9Mn;eiT;E
zvlflb>sWLoZ*|e1yyrzhPNcZAvr+MC=j`IA&a1_`v#_|atD<|pYpna8>p!;!*Su!=
zWjxFB$9SISpZ3rNE>DMoD&7qRQ@mdau6iR2ecn-pb$pi#=lJx(`@YUaxPNa^Cx33y
z4u7rUH2>n_a)EcnlLKMyn}HE-HE`M8ir3w1_%5E0`~gp>;Pea?>UvKJ%e`*lv$viY
z?^`HN@;wzF`HYgq-&Gpy-yxmzCrOgOgxn!8O5PVZE$0Uc<;HwXxK7ShGWaJ-9f48T
z3EkDd!fv&en5M24%W28t6s@XsS6e0#`X8yYUR~a$uaW=i8FB@rzA_Ejq})X^6auZK
zc1BmKJJ4ThDq2Y^jV;y2W52ZvSasclt<me?Y5D@ZCh{C#g;<Crq%Tno-A}AQ(}*Or
z1X&#$POiXCkUz0JvMgSinu$-Lp5a$1E8(RE5;f?P#3b5HoS_?$S@cRW#_*BsX$Yg%
z8U|6149BQ|A)l&XtW6IxE~Ymd-_p;GAqK@b*iZ|Y%YPY{;RI8|n8eI7+SvQXb}Y&)
zU|TVlAay>Iea(cJ3~XCdYj(D2F?-5%m;GT1uqao<RGyn?>du`u&EYalN4aS39oL)l
za~rrA^HZ*sS>(o<tC%;Khnp{&cbY$%-<S){s@Y_zYN==$2I<0^EaNQ?ENd(-%YT+I
z>r+c}Yno-eRkCcg23hY~D_AqFO|7i0r?t9mthJ|Yk#(AFCwKzSSuffigCFp-)nQAu
zV)i`n0(z{~AcwxSowW70n{DImVYYepSlfDgDcgQ~S=$ABIom^MA7K4Aw7>Q;wj6tD
zTY){^#@i!pcu=s-9AvSD2bpZ~K}K7-Aj(!b2(eWU60Eg?ilOCNYXqfQs|0<sRtS1!
zEfaLxnh<o_8XdID8WOb9Y6_ZW#e#-dd3!sn%U<1@Vvn$Xuw&K-_I%3)`*+Jh`vc1+
zz}gl9rj}@*X6a||Z)t9C0$5yvg|(BGd|S5pwe6|-r0tM-g>ANZq^+B|rLDX<-iDbK
zYYO+%dXu|i-NbFMj^xH!>vFBEEEi+_Yw}rcnBG{{nT}e9ndVt)nR;0arizwi7O`An
zznPb@|C#%+v(4q$_GXcZFu!J;+#co*H=0=o9>77IZmi6`H>#$C#&@R4#=WK%#_=Y*
zv7ss5z?m)?QrU%ub8HvGTsF?onk}L&>;w82vz|W044_9d<>^WcPkD^bsN2TP)M8^l
zs;x1eG8rA@XTue8hhYIZ&;ZP{!A`p9B;p!<jF?9!5>4rP1WFUcGwL&B)bGc8P^0k}
zsw$pG3fNWhIX0i%j<qC5VMekp_7V8$oy1RcFmVD+Af};tcuVvW9)!-qvyf)^b;N+L
zL_TAEko{O$WDF+gwE*G3u>JZcbdr7mZK;n%!$DzDplRR(d<$yeJxHQ925F(yMj}9!
zQLKK@U#ffcgX$Q4rdm_)q9S@3^}VJk`?ViRqIO<s4C)R}8w}iZUFE!LQ|72nxecJ(
zc6Fbe3eKl<N=JFFQcCWqC{mpAUGmG9q!022>6HAhv|R2g4V7zw`aM#z%ZTKXa>b{>
z*6x>Xh_j@FVlQchSX25}43heaUa`6O9T?$TVzhVws;(>-1!26HC3J(TE44t`9wuHE
zWZ|fgCTs`w`byy}cnY@(<Ak}wKw*T?33Tm^gqlJX0V0kB11On%JS${?cIgu@@Q?T+
z{u1xt5Ao^zMn0Ke!2bX|@dG~$v`oGEdwd7}8u$<|@wNDie0lx?9}6uQ8Ve2MFY`j+
z8ebT=#b*U>^GVRY!ulI%PXc$K7Rr78PT&!LFYpw$U%>v?pojX*e}}p#zXN%EazFrY
zBMEg(tU@8wH}MBb3o_I=K|ni?^IZj+A0n`PqG0A{3wC}P=-}52p`gPL=MRH6{tRf}
zuY&IVKIrIQfv*0W&=YE){KNZ&$vh)2;p2c0sU@D}+lh~$y1-X{s+a}dHa~w<Gyo42
z4ZKhdAxG>WXyR}oT$(G?mbMB#q;tY#>4mUfN*7K_ittnl6Vs#`q9Sz>!{vX0|JWe*
zfO;vD<*(vqIRN^g2<fX_AHLasQm8URs;g|4`YC6nIm$C>my#k~Qv%XQ#Vk9NGBT<*
zlf%_Pa%FWcXohylebsC7MD?S*Qq7YOtGIGkjR&`BGsO-3IjgNw%4%nnHrhL76lmL5
zY9{rhR#|<ibpxJ!no8(9)sp&6wdMcS6TG@w57SQT^|Y6uZBN(dYm&ZS3qtOIuIQK6
z9`S1<K!vdYxbvMzU;QF7Q-1^8c`kBR$I<72OlRn|Q4HyTmPUr6?U9+_(%FQrN6w<>
zfH!}SWTOR8B8fvIv5LU1wnMvMqtLO~5_Bo(+4p1j(CgSA^gS@InV`%Tu$tg&?*>kz
zM65Qx3TqFV^}+ak;9GuUD{wz(*=^vts{}l88$1OchI{e3ph@2j8uSamnZLjr5UF@4
z;L8Vt7jGg_l2`}|+Ks@GA0j3JbG;In@ZH1@;uLV>Hwhmw+Zyo;cyT8vYh|eG!jkc1
z7+HZ#0H=IqvME`YY)iHzJCj|=Ze)MZ){Y>1l8Iz5ayr?ETmXAkl7q-C<S_C8Ig&g@
zjs*>FB6*vfLOvpAkgtIM`9#hqf0K*IOmZoiPcA0|<SI}|tpf-CX7K-QCnKrdWGT>)
zRt24IJ?a42lsXKRV2*=s^c2~JI!pGT&Vz3EJlPjoFWBxzorL?2lFi}x`qU11+&Z!(
zwS)|zW{@N`7M{O9RG4W`Cc``bOqL>Fl2-CQsSsC57jcICLmUS7e;0Y5*hoT60P+|y
zm)uEAB-aqb$OS|XP;#{f4_Q63CsC1XMMRTzKrK*?;E71$FToP8AyMEmk&W*r-s1~F
zulx_O57Y@u@iL%EHGn2HA2|6>cqphEcx)g33tI#(gJJj~tU0~{OTfniC*K+P`0C)I
z492d30w0)pbOH7h*tOGWOYn}B!lr>wtRJW>nt%hdBxtT_bT68ZECL0_2=o@x3O$UJ
z22}<HKCnDw5b^<O0b1^|pyoCqGeAGz8~LTz0zTc2oB++~TJUR41TMV?xU%Yi^D07b
z2paToJry+SkF*ckVNkFy(iUn%w7yzn@I^&wtR|_spf-J`-cXN$?`fGjO&y{3R9mUF
z)lzDNN~oHWrDTG?>5XzsxvVTx_9&y2B}#i`oKjusri6iyNtNTkzr=#_RFqH34tcfw
zU7jF6mb(JiT|?d`2g~!p={-jN4ZfxOQX~18R90Rm+2m=UqZueUq~_9hshspwvPf4X
zQ9J@(rH#@zae?##>T2DFnp)>Uk8@P)1{$3X(mJsfsEV2b&)rZOBGv`(OiigVaNAYH
zvQoSlF9nO?P!r20>A-RO1VVBG$DJk!;&-UA^&WWc*MdWQ1vR%`3z_1Z|F;}icR_QD
zZ-oFHqllk?8~*~epS}vAP&X?cxbg~;1NhzmAekh*eh|EuIB}R%9_nz_5SPO1Zw0>n
z2-Nzz1peNK(j4&vyw6m)8Wl>HfdhUfhR8p}O0olLM#<t}*&@x9OG_K&hSEv72e4J+
zq@VI4saW1AG0J(Vr1C;)1RBXczyVKIg5<SIW%;<$T7IYuk$;1xz@u!FS@kk_gFnlS
zR3G$*;mUOI_H0#$Dp%E|%2)NI;#J=%L7H2ssRgUOw0i1HZ3uXLR;u^4OX^?k5Aee{
zxc19w-Sm#yT>W3|IH;4~>Nm9_Jx#MB2E7JSMemFBguZXKeiS*Vzd)Wrzvl#Pr4=cI
z)<oK(eUWkKJY*gGI?tgmkq>AgQiO(}cF<MUgnq6U@W(T-`{*vr1+}QcK#kG@L-Bt>
zjlC0Vi$B69;CbNDv4Ib!KK=(E4qlmccxmD~-U0fti3CY(A<7e1KsWOldb2`M{utrR
zElGYP8k6}%U(x`5O=;*)nv=(%w$yFtao&^L$-m@z(hKLlfl4EzsQ~m69Muez{@uZC
zJdEl@O`(QUOQ^ZjR%#n{lsZRU0oU;pDw+BUK7dRLp}mxiM(7f>iLOD1(5>h=x;N<i
zN7L2mS#*7R72TZPLAR$5(cOT#?n|Gh2h(Ti5%eW`6nz8QLs)-F52HWOLx9^JOn-&t
zS6F_g2SDpZe*)(F6Wtot8^N*;{gJLpzopC5FW?pL(~(fo%0?fiX?i^^Q!{8MXz_ni
zZRtm#96d>e(Ca9Pnn-0(U7#{nW$G?vqRxPyVJG#STueP6$3ef@9s1S=pvO<329idq
z3F#tBlJ7{GJWJ*gYsuG;gmRK-NiHK|$)Vt>Xh?h{LO{n<fG;4Pg7;!KeEO4c9<DC0
z@n~>nxIqp34C?~lRylked~;2)UsyPH9&@8hus3K=>=;@an}_<*9-t8_k8VYE<RA1a
zQVsnN!O&UY32K9!MS{V9lBxGWuIuGMA1~-5!A;sk->iq}gTRkdR)45@v`wJo844Pn
zYFck_W+rGK)nfIa`an%o*Qt%vzAC4dR#QQJb3u8fECdZsH)X0)3fz;b9HIOHRn0B=
zfxJ`RAWwt3ex2oba7a2NQMxR>krqgMr4G^rDFW2&4$%%Z`kdk_@HzGucY}^_qNoWi
z#E(L_cvvV9rU*}kmcl+E6x8c3zKie}JcGM=QkcSL@SXVwd`12MZ{ip7#ev~425Sf7
zvKsu+KrFv9!0}@tU!rZGAW$lh6wv)I1Hb)O0@wWe0vkbdKEXdO(8=F5P{m&}5af>t
zh(08c29=ne`+oS(`JVZA_%8bw`wsah`8N26Lf%9--!y+SU!uRJZ;Zd3Z=^rYH{2iU
z8|F9phWatzV87rU;4k*}^Siu#{JGxV{w!}_f4X;&|F3r#EJyk?yklYC1b?1)vcJ$f
z8;)P*Cw!ayLGT)-;I(S`p8H$*lKee<Zg@R9Fy0p(nBl7uSnO*W*x>658ufobmp(mk
z-?s+z?uS7Cej}jx-UmYb*@3El9rW{&{4jrAeh%o+clc-W*Ze#AZ~hy+&;OGThB0Z)
zK%~$s&=}ILh6x7)%Y^5F(?UVuGmJfDp*d)NCW0<#AJo@-#b1Pcp%k$!^s)V+R^bL=
zwDd^WA$f%tQfX0yk$WY1tvFP^CvK68#V2xt<dyqKWk4U(Pq`22TSd|XB|$D$`pOCF
z2DzvDLSCV2@=djl;!+cpIPf<1(tasxz}I+R>!iB1wcuxbtq#$R+Ah6=_C;T(nUHr{
zYs3uwTNiL~Y|_2R2fYT$B2&<I;89!->VqdpX((Aa7;6R^m__J2>@JGqJlX)SiA};M
zgRk%ccn1qGj;MgQgxYy?iKCzq&%pCw>=j3j0hiJlVjc8V&&f(8N{u5MQYXmiR62Qv
zilI^<`M^f+1eeWMYBp_#UZew^M6aM(!vk;(`RR#<iiX36e+-`uyA6o(v!RlaHTH+<
zb*qf?j5myzp^{yWF^Gv`nlN3Nam*5CJ9CM7z@#w_#>`sTrfeN{Dm#EZ#x7*Put!*p
zea@DFDnwmOtZAmHlIeh{lj)ghyvb==ZL)C3O*OekraoK}pbS3KevaiHaV5A6t}aKI
zyKyDW<GJSM<=hZpo#&aaal6bfxhv+M+*@-lmu}`cznL-{EJ5aION_a?rL?(~rJ}j7
zrJ8xHrLK9FrKx$PrM-E(rHA>bWq|pT<sb8X%NX-(OQQL!WwQB?Wtut7GTod3%PiQ=
zviu9{<6z%#bCRVW+~3Lk%hJsJ!&2M)#ZuY)4xay|CCdENVl_XqVCDxF4|m&=#a*@h
z;LcfIb0;kKxC538+z!h@ZjEIVH^;Jw8*Q1yb+ZiS>RCE)ah6&fWr^loW|aG6&NN*&
zKQrw%pD@iguQUxek2bY8w=q>Wmo`P12@_(@Vsp3`>^tr>dzo9$?&K!1bGV-DK(031
zhzn<<IFXS|znEX98_YG+HfEz~GBeiHk!fx!%|w}0qo4g|{J@?yo?w?5=d**1-9gh{
zhGmT+^Vjf}xo+6UtTuqk#?XYRVz4n9oooC+-!mSfHyI~`e!V4K(HKhm3{L8v;U2Zr
zupTt)1F2v`c`A(#kZ0(p<ZOBi*#@!^g6JyX$5M&Y)LUXQwTGxfjUrUCI`NED@O9*K
zyc@Xz4<`HIKfyD#k8oi_i5plcVhNUqcfu~<Vc?d{K<nZcQ5pL3M_3zl1!hG%V!shP
z=+l3r6Op5617refKx(1ibs5sR9)rheInq>bjc`z_^^f*SKd!CUCu%+PIxxmW!4vsj
zJ*4dgFVk4HBshw*RZKmheo#iLdz8}ZaK)iigr3|l&jbZ@6J@PT!x{WhE+g*-#q>Xr
z15_T|&BfAW={oq3=1Y-MTTnR$Ne@6fzDYa<+U|*BU9qOfiIR{hJQr>XTZGlZP@$ht
zS*R$;JR-c|fARZZ&Yj4w;v4ZpVXj@5{}ZtBXP|$cA9xk$7C0U#8CVnu_y-1F`RjsD
zC?t^R_xhXpzxadw7yWtA58wAq@Ne^V@K5lS_qTz(pOU^jAL@JOOM_&-yO8y>!@Jx!
z$ve^4$=k<Q(c8#p@|N@!cv#;@kKcR6^Vhr6^WM9_bJsh<bH>}=v&UQ2v&I|dnd8Mh
z<GmjDAa4d_{Csh@@;-Jq@?Lb;@*Z$k@vd{1_0D$3dq=p#y<Oa9Z$mfXE$@~*QSM?7
z=XQAX;=hpn^ULEX{tWp)Z#_SXUwYmbKL^zRIphGn@Z5r2otwq4Vfof`r}z`>`voo2
zbFaA2^AIw2o)u%>H^r>?b8(ROXK|c2rMQwetGI!;ptzk^Ebi?!xQBbg-IKj#-Alc-
z+&dw2=d8EA`?0r=`@46fJJ&nYEqYhG8Q)%aEadOh@;z~P^8Ip;^cA@0K_1;!U$Eyi
zuo6#vjXf#8J|59G*%Rzv3pqR|JnjAWJtO_!J&XJWp8bB@d&eK?{o$|S_4qq_O@UF~
zvVkSu7J)<FA%Xkexq)Qw&VcN_9*FXN3pDom0wa8({CZzK{)TT5pXOV}Q~q;&C16T=
z_)%f5zk+bm-$VH6p9?;xqeA__J7HWv6m|v5icbT5fMHoKhCv=rQ$7uJpRBZvuPZ&^
zhf7X=BY2qZNUek%$mg-iCxnLbZ()>d23(?rcwe3=7C^Pd7$rsQqJ&5bl+KWAw_N(8
z+>scyNNT7i0QS&JKBlgcf2wz6TFaN~Xz|KOt+%ob>bgGF9xHxLP|E35)dBh#b-jK_
zy{rFJow`+vK-y{TkcHYD<SMv2^0YtT5@gT;dP8)Fo``<Z55k%D32ltv*c7BDb^;jz
zXW|yjj6TF#q6OGYGy*?`wgLb9G@Qi_;f=AkkX91F4`T7ed#pX7U{j%%>t13I{+QSR
zbz2|cX3~w<Buf+h$bQ5kavgD!d_a6A-Gl-o!?ILQm{l(&=TaBI!TE=LOHouYRRf%`
zL#SHxTB<j=UuV-9;8n8H59p?JIz5rb4g0`_`V!o%ZhDd-%CODQ%5dE<&hXu^-rzNy
zGgyu9;98Y$Xbab?VaBq?S;nT|PwHhnY#eXAVq9W;V%%%|WV~ifF@7*Q;Lm5o7>tQw
zY)n0-1k;nL&P-yOGi#Y1%n4>B^B9u#l9<(upV`e=*i%e-_9oMuea`e{ze2s%3}!Ct
zX4bGO_>E}x1Z!b0vmxwlHi~`7mSCSi2H!KbH2ab*$-ZG@*!OH0`-!!)-{81kEXyXd
z6r08(Yz8YbIjj$~q{Wb<lg|WL2O~h`K0lkv6tjOISLX+l&VFQmvCo(f><#8A`yX?Y
z-N~F`S2G9M*`U%I&8!Cx)N-~VGlwn9B(lNGP?iKwmEYKk%`(;lzf(o_u`!yxXk^)g
zMuAyx%x306_Rx6aJ*KbmG}GL;lPL%FXwAlnjMvbI`E6*xJT^o#rwpobogvMbXn1Vw
zWH@51Xjo)q4TFqMx`FW}9c?^D>xKn%I(Uj78|u&pAZKTe!Atcue5UFfE>e+(4U|MD
zQora9)K$71wV9@=MDX#prC*S7^a;`r-~1bDFnN@!P0piCWN#{ys7l=>C~6y#OeTUq
zw-vF1j3N3%mQGdj1GwIf<0-@({07k(-$0ZGSB8$&A(Am8aT`m)w?hhJBDMx^35ZoB
zBo%ldrQt24Hv9(;k%i#1?TdE6s-Q6#ffk{EkSCCJvkhGZ*v4R_Cd?#BltF%j68REx
zTVDm}+8{`fsEV|MI^|{bpO9X40kDhJ`W@{beTUXipQ456-Lyij26*#>A^jzwF4TUi
z1GMXE9q`Tufp@kTytD5>&3{@A26aELj8K!5Ch8+4T>VceQr0WaVFs~Z>8s38>M327
zaHX=s%O>#67RYzuT69FdB(IgX!hB+yJV@>(x0dV3RpfB^6D3Se0v-N+(BB_|df>~X
zOVSu=A7rhplB!8lrBLwCYJg8T#Cm{Tlmib@gmg|cf-YZzd524!2~OHE;5+Isz5yT7
z6YwP6fvedyv9fpx98c%PQ1P6|il;?H{7>`?$3&NKSo|v-6n_c_0FT%Q_{1L2>hBWo
z2s_0~u>23U_X&H&t<cuO`U=>;T-YwI61IsOgdL#c-wXF2h55}{@tAN!JSjW|jsJV`
z8tD7)gVz5g;2~cG1h5T@lm{4xAMgznuni9CjYo)Mp<ekMsk*pMY5~Z4AMpa<Esv#@
zkYRR6ERe24PRScFUj8dKkbPoL*&s~-G=8~U3H(n@r5oT|crPbPS@KFrk`GB1<tB{X
zp^Cba4dZuRnyN&|>y?_2Thdv6s*C}|f2k}e2V|RiQ!Wp<ZgVwT9;hnvbTtHAYt{d+
zb$_6aQ@*L-aRzS`s$EecwAV^?EgclVBJ{m>bpfOf?$BC-18$J|Oq;GIX&Y3pc2Z^Z
zr;s<Aq1K1dNjE)S8>ctXmVpENpgvo>1;41z`ZZ7$f6;Knt;HZVy&h6k?}2oLJkpWy
zYn%ffsSWyVz&}1Am-KApiOwTGbQ;`xp(q8YTs%?-`s?;khkGzO1DT3$MOH&C?t_5b
zT}O+N_b7+vqUE5>eG5o~8irPfc|=?65IO{Kk=fV}bSqYfp2u+P1r~uN0YaFMwZ{}}
zIL3m5Jqq87Rl-k$kLoej88G01cri8}r}23(JKqHUsbk;<x`7V@Ja`8F1K)_};3r`A
zej9vLZy|#u6|YFR@p=RS9;6^ZQR0c-L`85T)gi_ZtpQQ#MNB7#gHvfDv4ofj?xn@V
zCP;7FLF^*-K=#5x;yie)u7lI*9(bOf5_iB|^^o`iUZ^B+OJxx+iG0`-0FM*`c?=A+
zAV6lKzzLf`ekRJ1Ux=#EYC)?{ekB@_--)K=Pog=rR<Pa{_P2v-;cdW;)g0Vd^#Plx
zPQD~6lTQGDy$$UWQGxuAC`}#&tYQ}tLT-UHxz*q<SO}QxRH6WURyp9aN(Z0SU!o(l
z)<i1Nl=w~5BR+xG>N!ysFz#q@YFXevL5c0)Qdmc1;!BAi_#F6jCK8YF5yTC=FL4g<
zNF2c%6FczA#47mI<^Y~M4pLwG<A3pX_*=X#pi^Z43kia>3b?vL#q>e=2S}T_1)i(p
z;J4ZesQp4L6&nNJU>EEX_^S3`(bzKhMia3D$O8L;HUlqJCCD3yz;*$~yA<`H6VYUJ
zAo>FB01m5$=x)GHmI7+_FJ!9p1<bcO`WvZ$zCeN?`Bp&=Lr%w9z<XyPHz3htKcsan
zL+T>q!B^E2VIbkdtCvQSbq;x{`}M1k1aeq^s;}2i>$4y!WVk*R(nWeg8cGd49@0mQ
zdV%H!{P%};6*9E;gL7)7wm_StjR4%G3wWj)Y9+LCnh6rJWJn)ys+sB!^#kBA_toR-
zd3BAtU!4NC1JxPel^PDo1YH3cZUWeFCDo%us+o|3^+OTC8I`ZxQ!<o`${*#h@>SWa
zyam+esWM)<4L$C8r41w%)KNAn<&*_Vlrl*%0g^1sgA|9{Tlpb(P+rK5lsj@|<-8oL
z9F}d$R+u3ylQnsk9FYH&i{w#qjyzaSgMRrJBpQ5?Tge~frt%xe#(FI`lwZjWU|Aow
z>p`mz*;x(b_i`gRt`QvHNd5~g2aa{f&7rk|^>%VDJdaE6BzpiolI3nPru2s7gdwt7
z87GG-)8TzAl*=nC<=V<dNG;tdcUBI{{gspQNadnD3Et~G<)yqD-un(ETRy7z<cpB$
zbq_q9Z<RRs45}(l$o*0PD>tdVl_<bx%K|=815#0&s_T@_>Rx5AdRm#N-c}Z>Z=tVC
zRStsR^McC2Cm*kV1Oz@^ZKwJmkBrgAtMS@=xXNu(J8H+(q1tVAn)X&*rKLdXYLR+T
z!?Y(_g!UbrZ+Ti9NTcbm*#Orqqp#2!=ts4#`eVpk{;SQ?HEp{dsbAJ>Lpnt_NRgSK
zv&ahgB^}TkBiHm^$Xk5^lBF+0RDCZJ224W@$cpKPWFV6e5nO&&@P?L0U%@Xm9T|uU
zkYHy+*F%2HambFm3;w__kRRy;JP^muq0x}nR2$fi&X^A!0Ykl6SUh+VYhnAbwvZ+{
z01{&+VxO_4SSGd$^Fp2x0en{ooMC0~4DcESAmiDD_X5AsKaj;V8E=FyhO=@r-Wwc|
zBf)(%9nQ{WkfpQ<e}(VGzd(<e4DQJ+=oMY~eaLTm3jN`0T*E)(1Y}ei0GnY52ef?L
zL=@r{=pXF_4-QNnnt=$0<e4DCPK3g3BrIbHEA*OXXe<#+&_oo05#fY_2SG}Wi73W#
zA`ch9uUUZqgxe3`+I$A@<1V;1F9J_@7(Wj0YZv%h*WmvGQ#S}7hj#$iW*u<7#zX3p
z0gnNM)d-nRKIl8rz?=C7JBVGy*1=~p3tIu#fJs;<_=IXhzD_Jw<^S1D`KSh;ZVq}A
zk_7jm*U`oQPv}{Mc0@;@)zLO+C|Ut>nat>4#EZOy5!D6cF|rdmgDeE5XbdtRzQd77
zeeh<+K}H}3Ie=L@0+@ata7CXWUFe4X4<z|DfgHaG=sWycE4U_>)Gui|r0V^FB)ywj
zcWoz(x@N(sYmnwwn`z(GQrb0@*0w-zF$wyLPU>Z~qPkr*sx#GGrH}edsiPiJLgBX_
zQ2Hr9l-kM-C0N-3=g3Uuo7_*i4(IxAxdb@SNo6P`l{AO*H9@`z{>bg}Uuhm>+l>I8
zsDr#hswR(>Lgh}9EY*<y0xIxA!lm<)OWYxSg$%X3;=jNR4V2c4?WCFD-yRAc$2KA>
zRTf1t2ryp-GT3s#$M{{iEj|-YiC19kbO?Nl>xB7`V>bmj>``J@p|99PXfIY38jA5k
zRnab#5(yy+a^37=F>e%|JSt}Ml8_3KBj0%+WZ8LzXMC}6pD%zs6Nhk)&k&CBe}%n#
zim;te61MQag^m0VVLi09{4Zf0pA6e+!d5<A*a^q(;j@H&ybE$r{E#UJBR=5WFTiWv
z<fFtVd@1oGcp_84zgPhIathxAvgL-0;lgyWjIdg)4gSm4!gaB?@LC)#q=>U&9Jdbg
z<&J<S@h0TUy%)Ml*}^E`;O7BPwN0uZo`*5q8}NkY!skQCTf{i|j96cO0^_!y;zYSf
zTm$2`<8mqBp_)rS<RMa_JYO;@`=rv!UGO8Of*%o;7l9h%0C*7}DC6Y}=nE;};47#}
za%YuLrl?hwU64rgP+b7{>ru#if2}mtJj!S-27K!6)vNy>j~!A;{f$~h52$_h(%KTe
zk9JmHrTqk7A_jBeDo}uM0F3%q0mps=Iilx4KNO4x^k!%YWIWmlvg2kU571+f2mcu{
z!%U(ERv+zyC87(lJ?L5NF`9(speCGyJh*z;P<#xw6yFUP?Q856E<w6yCGaW^!CQmJ
zd>Zi*KMLpZC-7nG#CW0-u)za>Azn>d$XmcC<&o3KC}5jALt@k-NUFU+wS%OnxfDg8
zqiWD8)G(R@_hKWM6OX5z^d8!7cnwKS5<S#V-mt>Z&u|uQzZfnXL_>-p-bfqU7^}kU
zw=d+zFE(C<q>dku9WNMFV+2!<X$UjjLCicz>o~?7X5PY#*TZBo;Vi~BhB@p=Kyufz
zL)a_qBK8k^h(%2g0LM*c+nWS-f+@tb-c;Rm7O>pckSXsl%{S5Dbu0rep;mzB4&i>9
zW^pdlCQb#{5XU_P_1F)tBA3tA=P+|yF2dZCt7aa;wF7*2uz4mo*}RNfY~I3cG9Ta$
znNM<;%@?_6<{OYdf1k?&%(vM5nUl=P9B#?sOqK#J*y84*EIuyY65vWft6=eS)c}vJ
zXDQ^GSn{|QfXlYFq;u^o$y_H;ly|p$<9b;>aD6OqxW1N`TyM*BuBYWG?0?90vE1g`
zTP|}gE&p+iz>`_avY9JyS;0kvN0YTAa)NmPmu+s%eKl9&9-Hm(-UQPjbD9a<nWnks
zi>5K=U8Y{<MW$xvF{Uc!uBJ$H9TR1aHWhK0=_i-R-s3*8N4Xp9T5caZg<Ht>=Z3K@
zxRz`Mt}JWiSk`MQWWJg{Fjq_$m`#w^JHeF5v@x|~5=;q<%F4!H>{sJu_Ka~gV7Wus
zfyUZwO`{30+)Ti7Z!^~nn*hrl4OnhNrZVK?YV=>@C&+L=MlUeVq&phh(h<gRI@gd(
zT{GMSEO#~4)zF`cGE}5;XqmiBzai(*yI@{EoV3#w$rQ>*oTTm(Q>fKM1F9E+QSrnp
zvH;&e-o<;8tMN#(7oG`N?m5B<XwOxwE-@byA^GqQZpG#TGSm=13|Q_c^fp!jorx6!
zigX3>fNtOlWLHf^1-&|Y4=OAy(a$1n^>KjVmPI~m+4^>1fBS30^f)a>&r*MDr`4U>
zziLmds%nF?_@jD5*$8KL7qyjQQCa1a@>$*rJ$FxK7`WRj$p7Q$ETGzGyDc2Ii37pi
zwODD3O9d^|m3pbWesy<ucUS5H)TrPtL4teSJ-O%ptLt5vOeO(JJ2U5;XYXC5SL%-G
zzk@%>5tzJX!35tLrf#j^|8Yyp1<#L@;NOu5E>Rlbta3H;fUa`XxNBO}?a-;cuAZZw
zp>_u)=Af!U^%~sSH>)P9VpL2OK~<@&Ql3;^Q>G|$lo(}*GGAd*Y*myfMk+2UwDLL7
zi}hCQluPAt@&Y+penS3BHd}sB<}Xi{>11?Sk*r>NMRrrVOqL^!mJO3KWm-v(^ta@r
z^o-<~bgpEsG*B`^suQ!M|HLB6MRA#AvG|!}r1*%0E?y+*6D3IUMNX2_BBOYYh$r?J
zeHZJ5_r#^bed0U9dEza?7;%QsPV6QGS-7BCR3Ru3Jrmp(9TV&mtrRR4We8G5fr3yG
zUEm;+@yQ}CUnTs>?-f4h*9$N5{|fi=zY14FPRVTkBVh_;mJH`#g$$DmLK^=x<eMB5
z_6_Y5)(mYGeji#Td_A;Kcy}mUco8yBjtxy0?j4#c+zxB&&=goxh5y06ZSeZep~djp
zD&e7_Ey5G<{&Pd8gja@c32zO(5<VLGCVVqgAuJeb6P6B%g^fdKQO}U2NIK*rLh++T
zO#TFsD}Sjdl)qb)$iFO_%zrIf#{VnY%Wo51=c`2@`Al&QKM*vU8Df@TtvEz*L7XM{
zEM6;U6Q32}ByR=&k_N#9iC(Zp;x4=?Nf-W>Y!-?ncZGJ+a^X0sUbso>C%Pw{C90Gj
z7a65_q97SxG+*W@z9^d@E|MJ*>%o6MNUjtwlm|(!$d^ef<&PvpMUNy-;U@i0F<1Id
zaZ@T(v`BrG4ze6&w(NoOrmRQV3R+KV`C?V3{DEq}yj%4_?yMe=PgA=nj;rS?^3)F%
zLqOEqX`+>xnq$i28m_Wj<E66HZdT3F7O9?U?bJ$8r6oXL_@s`h;p*mVT=j1>EA(j4
z=4WZqhFjVUgFt)P5Dluf<2nyxlWwWeTmRCyQ7<?C)JK|_hCQaahQFp<1I3IrjyF#;
zo-@BNa-rAYikfU%fqHEEgi@Pupb^bNUol@t51PBtQ7C`RDbzYl2kH|h6m7!nhn{XV
z`UKVy{79E!IJi%sh$dif<ECLdaZj<Xcs+JLJ{@-t91we9p6Ek}!ml74#=j)g;YGkU
z3?r^1tOcjPTp$)C#HGYwQa*7#33BPdKT1P-Pl_hvq4PS4{10+VXp|oEB#J*cL#?MY
zQNDl^6dIX;jD@Sr5u^|)Mhpmr8V*;K<#27eNiCyRQYkbvEtwWZJ4{<lE2dqhQRzjr
z@pL`yB#_5{>2v8O`WbpCqkx{n7@*%^uo+d1NCuWUpAo@4%*bKBWL#p_GJY|&3<1-L
zX~9ZhMzZEJ=L2hen01Z$lJ%Zh!zyQLSba=KwuzYtTyYLNn01Vu!g|S`&#Gc?V#(M?
zSZs^yzzXNGGA)W&D=pes$1D`ACl+}2PYX7?*20n9ZsElqv<P9VEuz?D%h7Bn%XD_I
z<z#k@Wi~s>atV8k<r?-_%Z==Duu@?=4fds3&VlVI>=es1SPAT8%Lq8m7mjyeCs;Ds
zah7;?tfihc%2LV-wd`a0Shlj9EUQ>-%R(03@*PWMahKI+0m&m4TUiAb^I4BAQdp-f
zf>|3aY*^DR%*<$uKBlupF%xa^jM>dT!u-u%1UV`3%-w7U<_xx$5z4LuP5%o<J8M7W
zn9N|DW(6@;u&|62Rs-FO^@NUMZKpRfC(vIo{pmZI1p0Vp8_kvZ4m=7Df>*&*@GS75
ztz_t_qZn0G9OEIilD>s{73P=o>1?Vmorg$hkCBhGwa5<IC?tV~LGZK^N(p$)pQ28q
zOrY9PEU7Kz2IMyRBC-%>lYV3hWRv`ax#WM8%|NV&lVoHO@eTPgaV>csaTwWwC?xR+
z&q;R)%Sbr{ACe!TpD4mVAilxRA#TE36G!4J2wL1B!e_`H*@TnB{P7W7v!>&pV#)Z$
z*snNX8gVTcN8CkBIrysY#FC+Z@(azw>_DFZ@5Rw@ol-#x!4vd+@IiM#QP9oieAE^5
zTGUjyKC#X1kYWI|KUA<|!Q}`KF0Fqc`QR{AAIZiRBguFk99lArN1y|d0*o5ju*XmW
zmBk6Y8tAjP;CHwZoL0Q`k9A!zSG)~QDzkMcosF(Y+n_z8JrCX3Ebx|MXvLZe@UA|o
zIi|_bOx4hV_pMU%)#ub7)sxiw)mG{WYOcyveM2QtEmVC|`KwN<l*(+?4`qnzxRR)v
zu55$skPk{Z%nu6{IWRwrQ!Y~2DI*k01zAz6;6YBvTlq=F5&3e(9Qjy9gxpKPkdqVx
zGNJsZtVVuK_DQ}G@<6g=d*nf~#d4Y~LoSd;$jhW|@)uI3{DjmfTO}QmO_VmsMoE9k
zT&1sNWa%xLL~=^jDA^4&zIC#vk_EC0l1Z|?knyor5+z$836)Ki_(K+myDUlKAd8W3
zWFsYPSs1K9SY8qfnUlm)#)16|iH!__V~7%08BXE_c~k+gLLs{&680y_SduiEwPZ3J
zKNs>umdk=9TV$go2W6ur=Vh6ayRw;*SF#+)BiSS=mmPv^kLwaCWLV+le<Tiat|VM8
zfJ_gJbeY^<x?es*dJE=?pXHmRwV+^<$=^#Y71h!aFiV`NAjx(pJY?4u39>xJ99fHE
zr%bQ7DYIApltn8CWOI~kP(#JYpFqx4mGZP)tNbYUQ1RrMDx6}g%3pC;HAPVZEQ?b0
zT;Z&4P^79+%8lwk<#lzovQT|iIi&ulw9uH8@tO$Ldd*7JW6fO^PtyP@ECg8R1oaN>
zCUvg%vARdwthUk7HEA#-+@aeJ*&S~+Ezl#U>Rq%6`YGB?`s3Os`k&fHy;e&!gz82a
zR_fLmp6DJ!f4j}#uD3ES(x(|8>i0q)`>WBzpn%DDATTL8rgp<ksEKM#Gr_U#k}=L)
zYus#RKpm8B>M$QOS)qzeX(*z3FDeOYpj~D@OhlZ~YV#!YaOf*<K;^;YL<n6%CyX!n
za4iLn_z5KT41kBBH^vj213lsgz|?hP3c(>t4UDiq?gn-)t`U0y`WgSAQ>w%5hk7Ru
zHwQ1nodMeSCuDQT@K^Az;E<a^KohnACwGUihEPO!PLL4>3E(+POa`)T3-JN*39*~l
z4BoQ{q;5o$E|XT0svsYWNG^pdj25ytyvX<ADpN<^O~z85kV7d|<hc|T`8fCxexyVL
zn=*$&1P{Sr<Q8Qn@(t!?EtD6C67n;caMcMwe5k3&6zXzhKlKRmn)(1~hFS+tYes@;
zT4W~8mb#A?Nqt0{NiCxNM;)YHpi=2Cse$xDY6iWDx`HmI9-`xEcj*qa0(vm5m7WB(
z%4C`~V=*m+v4xh(I8Ix{xJ}#5ct^Vib;nyqJ?$T(m)69P(*_wRx{iU+DNGLCmg!3O
zV*1fTn4$DY$c`Gr97&%5D;ri0a}<3IGlsr}8BgEMOrjrVj-{VurqItYQ|aemU0^2D
zuP{f`Z!+WP513J)oQ<Tvf|U!~Z~pffL4U~%f&G5;hfFu{ma(B<Vp8cRm?-)lrh>MS
z*-y)1HqmA>i)rJTpJ_?V$Fz~mOEiDxL7EeDBaOveOv5s#(BzCUv@YOSDj6=cZww0U
zHA4>i6dq((l~NBfzEIaQo=~$H7pTdMUDQy<GO7b(G8N5;g&Zp{>K{6l`j{?8j?=4<
zHIQdDfqo7Nqi;m4=+hAuEgGq(IV10AXyg>Fld_!lgEEeGo8n2^Nx{-)P`aoQl<!mv
z$~CHlyp~!@9!GsdcBbwlYmhnQY9x~U1hFFTM8u>CND;{sxkJ)WwvdV`8KkQeH^>lC
z5=T;shy==YVl8<U@fJCjxQxss4kvdKu;g4q6=^5oCipTgA=wj#kpy@pu>hY(JdWQ_
zoQ_W?y5p^gVq81SCLa?H<5m+U;=%~_I2FDJ`xXBZyBog+JQ(A#Y&;3vIjb=ba5pi_
za4Ru^xEKt$HG&^-19~g=CVCWh5n6)@M1RCU#}ZSB3dNj&x@a<Z{Mw?{pxe!X=tuCs
zZMFF+kp4?hCh%GO4Rz5m(<7KQE(e7`pt;>BhPvoAbTZbP;*BG~ZOLq`Hxxs+<BV~g
zVUjTcI$dn=<L%U6HoVeLHT(y)(f@a+nZQExpf-A<GwIgpe!%tTAWR8UbicJU-6?Ik
zcDnWixEQ8uC6KI;2T2NtfI^(8nXhrtcxpuIeszKRnfe&`<W5tEs6EvRm0a~v^;`9y
z>Y^%IwFr2_P*tJQtURPFQ>H0zDA~$2%4S89^1i}Kxk}Nm7@_#Az$lI@%AsDmEDu#I
zkP{U_a-Li-doM4NoseIYEtD^iMFXS4k?Vmq`6n%rU64MIEs*Y#g-U10aG)%0kkDmM
zB!ki&l0VW3l6z8b$zG{mJYQNVPLMtnyGeJ6anc#$en~i}PU+%Tl77)S$q&(H$qmtL
z$$z3m$#jvgBvQndI6*Ci7xBd+VY8SkEEWF}7Kon<bHx{hkHou$H^s|@7sQi=C&UTD
z1L9%A?P6!)dNEtLTuc!z7Gs4A#3sQ!u~sl2vKSYL`GUpbUcqv42h<|1g3aP4!FF-I
zV7IthuuohmI4G_V91&LuPKfISXT@B>6>+=Zp15D|N-P%S!Dkf1=QW9G!XdGZ&<L51
zOi6^$Ly`hnj<bd1C2NI?B>RQiC6|TgCC`K}B;SR9C3V6s$&k=2F$*1}mZA|-AJIf<
zv}l!dyy%E@p6I@GgQ!4yMARU?EfPvUi}12qk+Vz;o;5V_T$#UkuPjx3SGH38O?FD$
zEPEqXLuQt>TqYhRXG!MDLnH^}6D3dOn<N$TD-wnLtHe&xBuP+cBuf<5(xZw<=`+PV
zX{q9<RG|1EWhgtOVM;_cQ#o9AKsi^It2`=eRlb!GR4uX~6-GW)<tyI}_0Ut*MtP0u
zx?H3BEq73h<nd}-#bWTmIiX&oc&EOmXjT7J;59NupvF--MU$f3rP-`}s(Gla*VHSq
zT7oKA8==b9=BSQpuc|(1%T!%jgNmW^Q^)G2s#odus&D9Cs*7|@YPpV}anOfp#^@Jl
zR)f;{vi_gu8+ZZt>4#|<hK1Tl!)5IfLz(u10i*k27zydAt8});+q!Ya3f&f?0o2mo
zz|l<9V@<pD0jAgb$)+y-b`!^N*OY1~HtjITO)m`2W}YF{OgC;cCm3&-HyI1fPk^Us
zHnLHODH;`H$^mE8GpLuQJXD{l7qTQ-=uF6+*aJ+>Tk}<PulYCpFBd`&(F(H*6$j~d
zD=}@TE5OwhViJI>*?_jh-bY7cE6|I9?>vQdg|kK`rV+ab*qS#OD{vT(z%e0PG7h^3
zw;KBZcMJOuSB4#e*#QmjiVMZ3fqVHT+!p*@+(npbzQwC?RiLJk;FIxe!g_odC{`vD
z{^HjIt$m7M3DovT!au@nLLXrd0S_+oZp1P|EKxz2PP8Yk2hQds5adsYdx(X=ne&Lh
zz+JwV$Rv?S-oS*$lQN;hxRJDubPb$=zmx8hx<~~iEV+r~Mplwy$xP@o`hY8N9FU@u
z$+w~J_!+v6)#MB0A@U<Kj`E)DO!-NUqLh)RQCi3wC<5|H3X1ZWVg;Q?KT0blhN1zY
zl!MHpgdyuGncxw;99Wlw$TP|vq=51jsixEb8`F=-DS8BpFsK%YC)E@3EyIy4Y9g|T
znvQIyPDYMU=O9<%TK<H(75M<ZY(J@|k#g!4xTfDh`l*i)1@#4jqP;;7+9$+{_6>0c
z&eE4wj)c-`kSJOmGKSWGWYQX;PuGMjqO~AvXgpvqJCHrJZsZ`X4>?R5LXOZx$WfXM
zdU`VCI8B0_gnegeJ-}(UBiG>go3QTE+K|WazSp!)sD;{*JX$MK@V~`FzQObFaLf-H
z7gja&2#evEUq~)3A9+FhfIOwWg!eu~?$d4{H)!XP3$&x~HFm?--Gb}|7IZ6ZDY6Rs
zhzn>_k!iFvB!d<U-**@i2|dL?nl0i)qabt|2$`vU6fw1y(oOwIX`;TQR8lWf{y@f8
zKJcRNsS7DDsnaPBsTq_Tkn44p8cx{{nO+;9@3??!O_@j~Q<A7EayYe<>_x30+fWP0
z2=x`&gxn!ZkhA1=WG}f4Sxf$c%q2fY#*;50vCyLoAa6r#$tw^7c_yMFB_n;LFr<Ox
zjQk~0kar{%<u<8{a-38Gb>17w0@5W)I%zv4f;5-nOiG{-A>&I#L{n;so#ao%pX6)A
zd*toJedO81`Q&I~Jk)_MWIX{*ZYK1Sz7qbCt`nY+b`TDe<`5Q>5(!BpFM=D1LO_$m
z_)cOiWK@2_-zQ$hA0YmRhlB%sA~6E*LbS#kp;l}rwBkMzzT?gj?%`Gu4nm&S66kCs
z1ApX$>&6pt-|_v}tN0@92K*CjI^<Y-;+JC$xK!xc`#|>|i`|OjVWt9mGzxba=(nvH
zEN&*I3mb*`244SHu{!j6EEk=I{e*VIUWDuZ26Q!MB9Iot&_^L<W-(>}@-6>C^2`%Z
z=p93S`JZYp8I=PGHPL9TnT@VC4}!w)2kMadDr%v5Jt`g?6CKSikb9{zb4}&uk8odb
z-n7TO#xxV|+{4YjCOWv0_nCT(-%N$Z^QODT9Mc}-DAPP6Vu~|%KvvWzqY>^CS`AB$
zdB7rFh2*GB1|et}{s9krA8OW}hC!Hsm%!xvIZVS&z=V6b{tHZ}&jTB~LXQIqsYmCm
zFN6u$1E7x%L63F0?l`y*FVGFsMS^O|3i2}LkmFPhJkcxdWpF*-uFU~A<s|65x`Agh
z2AJ6njZBlTY0z8&kN!=X+nOnwgPI7<3XLt?Z|Kw`A*0C&vYCi5my&_fp;bL!{Z~C2
zSXyuB(o)n%K&!f0T@Ct&kE(R_b=7e7UeG_RP?6PB!G}Ll1^NK!;M%MHsfemCD!KBx
zszZ5ORSy39UzFf^uiUOWt6Z(x4W9d}m6KKTlqsrAWxOgz838HS0ica=1@C>1YEVH|
z)hP^+y(v`YD%zEI6*bCppjp_j_@dmX$W<;;JXU6dmSHO7bB<S>QjSp^Rz@ifKyvn8
zr5ETYoE6)Z_KGb^8^uPY6=*3e;E@T>X|SIRuM-seK}T^|X_OyR>g2~kA8|sdm7jv`
zGqCTx60NubdWoA#lHxvSE}kh_ig)l?1xi=NU!||2LK&jqLJnxRG7<C>849#&x`F}D
zRSv3E3SZS`#c-%S$Ec1&PQX>g67cxnqWY>hpek3KRdqpas!_aG(UqktZ)J}vPH9lh
zP+F)rK^pjJ<yiGo<$U!|<xX{z@`_rje5*!OrD_jVuR2;~hRg#y@azxQ99E6jJWwss
z6sq=Vx>dJ8=bNW-f!Sgd%n+x->~Mp67<jCvYu~GvX}Rh{;LG|*YY!PzqclS>E2Qhz
zX+l6}F-iASvqAS;b57Tx$<-M(W#CcTuZz=S^z*e2`hD7w`g__bpjX=fTD5a}sxDU_
z0?O;DI)Q$dj%s+M8)m4|O*E)=n+%TntA==efnh$(1NXr^@IJ6xf1vIi(62Po3@44D
zpzNMuC^zmkh`^(iVX83%o0Ntula+BDkl_~~YwDwEn~`g}Zd94R8ZFFC;8&`J9;%&b
zr8yEjOJ_np^cGVgFk0>Acc9CyHE|$oDg^Yn>2R066uLtPVODq_Iz_)wdFCz@4>Y+b
zln2@!H3mHf=)xtay+9e>gjBsR=pU#CNcNCJ`W^%0g!TnZM<Vc8voJeBqjnV%{NAIV
zVk*(UFoV#QGNZMSmt=+Yzzl;~;uyFWm=1l}rPzI#t=Q{8Iljl-g2a(`*glvo8Zj-9
z*Cc|xC^RIKSmM$_6*?cdE2v6vC$Q6Tcd$!wZ$TSU2-IXP_AXFJ?{EriA#hpMz!~sB
zD>8@+2a;+G4vU`(THG9*9o$QJf=Vj{sHa%`9b5|j4Q?{N5SI<))M9);B;TlUkYo+I
zS_}LcxX-u+b^LRDIQ|2ub_?*S_&=c1EeADhJt%ZrpuX?K=iqzrtMR?~Es(ys2i7rs
zKWOC!;GIMG$9N%VcV+lIyawKhB2<7vw-NMnouD8a!XpGZo&h-qR?r);13uUZRA(-5
z4+9281P_8I{0Sm>LyC?cAqq5U34{>BSV9=+>>@yMH<B=g5DD1_QJ{5+B4oq%Jm@Yg
zhNG7g!U=2PxGk`D0dag79#6ymi}1c{aO_P&9N{)05mG^uK&2NC+mVE;@Sd~q)s7H+
z2)p6yZ-(z@6@dZHvRJ}2Q2M12`XSky3yQxY(7%1eb3jW>A)E!x@ovZjT8ZaD4rwvy
zgWrKZ7@SE#zs$g|#cOc0@SVV<6#;wp8aT6yI0HD@cH`y)*_I6CTM)R7vw)3LU>D)I
z*s-{LtT+5d384M$$1VigBN6gXov}yYU%_H*3nmeCzpn5v1OxP5AGm6lVj?iFF;<v!
z7&Uqw<R(qP`~U**9@+{z{sv$+y1`Go2r{*vL!#D6paoY$9%Lr^D`@C$pxEeLC^ax&
zt*FtcLX;os1&ReMhsL}M)nQ(YDmJHqdwVeIhS>_p9ld##xdZgRe}VOS0a=O{Aa81?
zS!!BrZZeHG|1d?EADNuYr%XiiHprY=0@;d_O@-i*{TeiPSHUBDzlmvD2~OG5OcG;~
zsofY18V+Yuo{?^PVKf5wB`}^ewi<UCi-Gd^Xq;!fYn%xF*`pyVF~SJb5b)2oF{S{q
z6baWjKcK<vjRQbCwHZv1iYNokaKGV)p~djYP!7tEZ-&Q)T*F<s=3O^jHk=0<_bBvh
zcL9;R(Xi04!Y~<lurc7SJp#7;fSPgudWvBPH=qpx2D#qD(64tewCULfE-+fvdV~HS
za9ag>k^UpFVYzw<sEtMXNBSZCJz%?T=zBpK*sDJc+sEPYC}b)g)prBk+w;HYCt&|6
z;7-l}`Fj>V=d@k}wYmWkbIhPtMnMISGHizDn_&M2z4Cu+BXpHFfyQ|Y?B5E<ZP#Og
z4<j1(LYnm<J;QJun6Y!fj9t@v8Sd$afe-jdNW4x2PAuK<8@d7&`nf<FFEMoJ*8=ml
z%^=YqH0bna4R}yCvJFq+=luZc#_#acmcUP4XNWhn8`2Gfz|JWQ^B^Z6$3O!P)(#jl
z593<++umXbg7a&b@q{7Dc*!u@c+-$+yl<ER=i7zGcZQY5@1P4VH|zic`T+b^P5=#k
z$w&m1H^=za=mm=J2xBAge1pa*Mm=c0DL_x#nsyp}O-G=^an?8*lsuWH`#@K}ggctg
z#*L<5#(lt?od)*jnyJ(Hz$Ar^2o^f>){v0q56NgTkfb)=qy|MN1MZHz%zJ@7yJ$)?
zzc9@*7s7eH$+R1kpXbd+(-UA43(W53TF|EtfY-YLcts}a3~*?#%+bJpOhxs<otN2s
z3P`2rkZf9v%0%^{R)gdGX~->pgBl55)l<-F)CROI<kdx?-~Nw1+Yf20PavhN5<LQZ
z6=q^QfzO-_8O8^IuX>A-0%>KBrGv&b2D=_I$!-Ek_7}3qByje&2OcB^$kMI2OSp&7
zEv<%bsR1}vA2@$b!QY1SZ6iJxPXlFf66A+%h4a%Z!bU<5;UU2mRD9_~Ghq+p7Jnel
zAo3xz*p2j=IFr;zJWsM9m6BqiMq5T2Nxno{M*dB@LY9;Mkli4kIGyZBSx-);TqLij
zd?sI^G?5D^I&u#v`>04HWjHdOG8Z%khmfn3XGk9C585bl#6+>7+99#jFsQfEkfYRv
z$Sdkjq=tGGQByx6cC=d1r%6DkMx}11LHaN)f%=d(gIWMiLUpvgQ~~WQ6-~cQwWPnG
zdeXm8!|8vhN%TtU1bPECo8Am81&_KG*ov*-m%WePM?Fp-pkAO4Qg6Vz1M4n*2p)&1
zH(~o4Xy`7{8{zdL>RIsEK0$vD9@}@|z2~7Pdw@ERz5%$5xzus=RB8l$IMt2rN~J*$
zQcqJMLr|M>X}^&_ps#yPJB3`OZA1>xW<gDrfJ~uzK^;X!9B5*Sfm%u70hjTEdV=x*
zTni6V$AQkyoidJUhMK6EjHiAi_dvFBA=E*4kTmix#DhE=F_0pm=CMRxlLWwK{G!Z*
zKI90}1`3swN$DW^Q{EHtlw-s;m~DL|j{>(QHgN@+PZ&!sBsi0=5fr3#gd$Q3;ReZ#
zuz_U2k0&+a1HgBOOge?{B`(8%BaXpeBD&z$5RJGLVl&Q@_z8z4Uchw_*5dLBX}Bu{
zZ`?)#9+ySv#rhNeU`d2~*na$O>>vCr>>d0_$c?cDmcxh%#CKvS_+OYI+#O6gZYSmq
zZZ_r|E*`TE=ZzVUV`2huYBU1QR6^`GbOrV<`W5y7Oo^AGmt!-~$=EQo57rh<!WvQi
zm`>DR%wN<4%rn$J%xTnI%tokhra`VuB+S+9QFV}j^bXx-K8OBkUXOkVlj!5-AoN;L
z9!&<yDgy4FtRTZoX4;IZFik`~2NLU;2|=xZ{%5-BtJxp!#K@*;=3a13`DPTEE*L*U
z0?9tpC}S%0KrKuyh8E)!!z1HHpupncEJTB}gjW4u!wVq%x9cYxlJzbIwq5|riBFJ{
zvP-`jdZE!eJlt6p>)JqF@IsrR+XB7N1n7m5p>C<vj05EYPCH%mOXHx~rRjvq<PkXS
zELT5P2SH6DRoSaQsOnW)RL51LRB<YuQmM>SzEti4N+eF{tTZc{6h9Q_6nhlo6$uIi
zx}QbzKk`HJqw+*~mfR$>ljq6$W!q#QWRbFcGQD)X?1$7&c1YSU&5(YQT1bycn<R6j
zw<Mv`#S*-9n502s6hD!aiFZhDiYH1|iG3x}VzdMiH;Y?DAH*+2XT`fjtHoKObg`Sr
zSF92e#pS{Q(L-UWXqPZoG);I(6at>#OyMGtLNH#`A{Zt5A@CGE5Lk$g3UHz|0=004
zKp;#MbO-|k^+JxITxjP16Atlz2^;u%!r%ON!uR}V!iW5Oz>VA#9^hXSZU9zfA*^Yz
zJ)VC9e8X=Fqktg^hhv5TPZGp`3Xd;@A^f+(P~c2L;q$_PI|=7^3rFzf!bm<=6w9{|
zjpn<HQu(2xiTni7bl^X-`SV5d_!~qE`G<i0xGLJfe<9k<|0X)YuM}P8cZ%-wm7>>t
zviK|CN&J@|BCg|)5%=<EinaW8V!GhC*hBD894YuA&J=(Px<Db`Az(_*3xXss1mh(|
z;OpHZI4eO5-$@*WHIh+6sbs2<A>AMhl%5r4NOOg2q?N++QlapZlp<=DdW-bZM3KF0
zt|(fzTa+!kBibb^6y1{bioVN;;!c^5m>^FR`^(pgv*ee>8|B}{SLD6oJULm?E)SI8
z6j>5K#U@FX;+kZ$;+y2UqE+%;p^|heXi~J&L+Y%Il}0INN~b8dN>?c_NDnGsOK&Pm
zrC*eT(ncj&rc(OJELCZ;VATp)hU%DXx$24Ri0Y5*k*ZTxs4~eqRCaQMYJ}WcJy{;2
zUM){iACb>f-;r-sf03V3SIHl$`{em*qr6USsSv0G6*x_@!dA0b5un+xh|}CtOw#;R
ztk85Q4r<WKI~q4-zGk$tQL{uT2S$adeW4ttEmuy|3YD9*bk!AYu<DC;qN)*?6&3Vi
zIl6DE5xO4LOdU<VT^FgorCSQFl9$wjx<6`5y+WO+_t31>PuJYkpVE}-OEg+NMe7Hi
z=5q{}w3iH>+FC<^&ce7&H{DpGyJd9L4}fd_a8tGZfGN;W3;IQO^Ppjqd7QD_{K)8s
zBAJfEnSc+JSPDAI{0RNjjKkQXa$tH=j6tJ=u-nj=unIH=w-#tLF{EIv!_L5Ku;uuD
zxFk@OeFdkU;Y52#&b&v6Ao&u%klqoa$uXon@*mPDN*ehSrJ6hfnMU~x)4&MoJmd|v
z2ReZ{)LXQEss%lVc9`Br)6$pHS1<&i24BZWVX7HlnA@4YEEDq@Yd4F+MzeRY53ogS
zyv00={T4MAW{V`tt(I>s#g<N1i>!`VHCjom#&9NcUU5EfEUcN<8?BdG*I4IS2ij0<
zj@V4K>9M(G6KC52{<F@uBHP)vadua2kJ~lc*4bIv+1qE>EwJBf_sl-mPHfM!3wEH`
zuW%S{f7c<~zSd!%J=XEQeW2q{`-zUN_8S~!_U9aN4!MqWhcZV?haN|32aThh1LEZD
z;Naxy5a{IM5b0#+km$s480Um@NOe-%r#TJ4YOqgr`ei@X>4klq(<S@iPP^>=oEF+U
zIgPbvJ9*iI0LEV7*kjk>_{*-?@wVL?$8C0394FcBbo8^E>u9o#cdWK`alB`XcHCgw
z>5ydm)4{^_jzhQ24u^L((;W8NggRu{&>U=RdhNTd^Xy+)pSIs-y}*8qb%Z_58nLhC
zblP3#e6m~2IcXQdS!idnin1%Qvaq{g#kbA2`fclDb<<X9x!LxQWtQz_OF!FXmN?re
z%QhRD<!76Ai}N;bE!Nl^w#c-ZZ4qn}Y+-3bw9r|1vOBE5v;SG&X6J(6`8Dgs>;u+m
z?A6wx?3vc~>~Ypu_6X}CmalaU%icPVMYVpyGIFl4gq-87PR=eCm$Q~t%bCZj;!I$b
zbH=dBII*xs!5YCT;e@mPaza>zoFLYBP5|pG$A|Tv<H>r-abZ2;*t2eNtXY>hY}QE*
zjkS+MWNqe{m@7DP=0Z*%a~7wOIhj+;Oy_)Kj^Vs#MsuDq!#FpY-kdW`Th2ZvfwP$@
zwpz-pvzp5MWR<|YX64J=X2oDmvyw4}S=BNyR<9X#mPZ&5EVCKwEQ1+QmJmX1SxGOm
zxJJKXv53Cd!k->&A*QL>?`Z|>ZL|aI1X?<qNV8;@Qkz+)sW({@so5+J)sDqQ8kkp*
zlgz0|JktV^FsdlG7{@5dj5vyjE+wC)-zEppQ^|j6a?)JdHIkAVMLJ4tB-&Fq5MLm4
zVhnh>mr>#gGbpwAZgLKO1zC%ekxt_NBe~+t#JAW(#5gRKSc*AIn2xa`45F{#x1sIv
z2>KT88p;*tjRKDfvkNxHd;?Qtvc^m_okzDCS?C<&8Srvvpf(#$no$N4I8yI5ar9c4
z(yTLj>G+11+8jfKwp0H@GgF_gsnIp6Q*;Z|-?ai&7@U<JY0OF+&0*zHHC3roolz`T
zu@z0q3-VN@l{{Z@Sr($Ol|7bUmpaIurKe>NB^cQ-$rfpWm@i2a&z01RYJspEEAAD3
z6s;0^i8O+X!h-^wkSW+8xWez@yYMITpAP*V3K<F;`aF1VFm{kV_-kPAK=OdJzp{UB
z|K$F%K5k!R->kl;y&b(ay-Rx!_egux-P?PXcH?@QyH0jzbaA?Wbl&L-?+oaA(($#^
zsUx-Xe0ysL(!Q?aU>m6e-FCBmCoi;J*IL}RxpjV<q(#qL-g24O-#nbRq`9_ruxWkk
z$|iQJi2Jr>9d~kzs!`dzxAAH-yD_f$QA2-Ibi=u(>iYPmCG`rfvF-);V%-vMaGeLY
zqPC-PP3_G_%i7tEc{Q$$i)w}%=rtc2@~ihXtgfEW;8pF`&{3tTzgAUNKc(tHJ+o?O
zeOcwS`eT*D>(eW3>e-d%y4H$;x+fJibz3U(>&8_)sk5uNR3|AvQdd&Gv+im6#<~;b
zE9y3t=hV$BUs9J|zPc`|d|O=ryzWwdv5r%IuZ~*&t`1XPRHrWMtdo@K>x5;TdQn+O
zy|gT&URk!HUSD>s9#{Ufo>^W}?@-RK_b#V5gqDXk#FbBN$SU97kW+rQVQ2ZjhKuFm
zhS%jbjV0xYjeX^-8Zi}@8=WeCHIAwfH%_i_;jXKg!aY@Socpxm8~1kwpW9Jk-DIjr
zY_hLh-88cDM$@Fq;-(dq%BBOAF3mS8lbb(Qu5PZbJl8C!{MbyX;x+qK5n9r!LRwZ=
zWw)HGI^6QL>TOF;ReKAgn%NpzJ-T&q^?$8rs&iWls{31qs-1W?HB)%0HK%xcYD#$d
zHN-Y!&Dgf2+JkK;Ym3@iYnkoAb#vQ~*1d1<ucLM(*XMM+s{hkr+u+lAq~Uz0p#jsi
zxp7OEvQg6YA9qzZx=Gl5x@monTeG1jzxi12>=x%fbnB<Sm#vfg=kZki_HDNZI@&S@
zU$$e1c6EFkn$)?EAJ~;9pm*B|wcTQ&py!{cxA&pAyYHx^uYa>tJg{7b8d@Z`;LlU|
z3Fa%u2y;}6MXS}v#oINXCC9XU=`Edy{Ih<6qS5eBsW%E#?&fgKc+^fUWNqu7V(bh}
z*cC<!zQ7biV4~I$bI^}T-!N@t8=NJw37<->BkZPylitxUkOvr2iX$tPI)QzccF;mg
z|7<ylIbijg#j!>#5^NS)uCV=Lb;*um{lk8yO~1nvTZ)sy&c`{%A<1Q*<4o6IPOIEd
zE<4;KT@QM!b35kw!2PIKwZ}eht>+dWNAIP+F+NlLX86YVuk~{YIN)y%I3Lgycqi~>
z(DR@j!`=)V9sD+!6!I#hEaZOZiO>sSsbTxV(cvqHzX{Kbm^s{U1hjZZ+>9t0Iby{5
zQKchhMNS{-64f!XBWls8YtcQUGGgXO8e+JSPh(S}CdPe=QpCALpNT&h?VZpcotKal
zlaY8erY%trvpgv(R+4licJ1ieSpI0+xcOsd#np_t5*ItRI_|+(LOgz4MErtr3*x_y
zI~LDKei=VMxhy_6c`)9Xj7u1kVwZ3{#Xq4uWkdo!H6bBAH6`I->V$-^sWTFWQs*XE
zr_D=<Pn(^PlQt#ca9V1@gS5DW{IrmSsx-HR_B3Wff0`y<l-3-tO#2wGPdgQFOj{JM
zOB)q0O{2&6q;<vBrM-?TO#3hHO<G*s)ihk(wzQ(yiD^e;J=4a<%2SOo-&0@598aAS
zGbNQAW1IRqx+!I1^o0~zRC>ypD06a9)T`u@$mz+8B2DA8qwb75GAe!?dsNHV%Of|8
zwHrwsdvnB<F-{{q#$1bdF`5+-Jo>=!+$2?aVA7KCM~Mw#_K8Vh#}l4~s^aOP+41W_
zzQ<JuJH-VCZ;L%JtTu)l<QwA~v@?2pU`13(fPIv8z@o@`{x3#7_LGj}`-YDU^4&0E
zl}~QOb8r6eelPdoUS2c87kZuuyW{aKwAoz}!g6;D8Rs@Jc(?1`VP9Nc1<9S80)w2f
zfy<oy1MWFy`L{T1^0RQb>YHYt@3Y^o)jQ8t=OweT@$#{W@XWTJ>T#B{$^DPjIX9i<
zbJrlt-!6+RTAZ)2<xXWRsuPOk<rvOPa9GBeV}FOf!>*Zj-Ihc9Y%`HsYkdk4a7rme
z4vFG!l}wJegvp}CAL2GPj(CYRn(&Hw2>*vsjN{R%I3+C&i_i{XoT%T>!w~~2of3_j
zN8WB;NBU^mPm~xh68wx$@rw)vxLf*KELS&(0Y5-=s@58PNE3wmr5<fYtEZU~R5`}o
z$_<8m#csV?eq0wSzpPy)yQg^~eW~t}d{)^@eky@ZR~!^o$-fGlWnw|E)LkH!OyO(A
zM}}~aC_xu#2CRi)1CD}Kke2kM&wr@1H+aydH*_GQXL$dC?ufoGT_byiosm6Gosr#{
z9g$so+9NyPwvFoO<wdsJ@}k-@T4Q*-TjE>aG>>WNZc1ymYMRiL%ALyH*Ep;3Tf_VY
zef{$Kk@Z{ZHrE}l{ZM<gMqTr~I<n?#)z0dY%3oDY733;G`J_s0`P~YqGHH2iX;S(8
zk}GA0ip8a`i;_#5{@p7f{4*2}|2w^S>7UO<w|_eoaSM<AvnbU6o%n0*-wQu`|8RfI
z`s4DW_V=>ynZG}Nt1ZNRn^`!mp!?UW{FT3ueC5v-d3%3Wd}aL%`}*+5wJ$M0h+nwh
zw|(C7UG~Z0`>IdBz6n3B{<h(x^EbnXx`HDg&J<XG$SQdL-l|~qdtQFmyNCG)->uE}
zdl#Qy_trXpA6V!_ylu#nzWJ8-`puKPwQnxwg}>RKXUg4_SCzXW?`iJhyaT!O^A^Aw
zox3#8FL!gEW$uwYZ0_|u_3PX`+3Vsw>FYjtB<5>h56j2p&dRsSJ(};A`z3!&t}uUb
zu6M!V+(iY?a~~Ge<PH^>b3?v`zuEq6?VF-+Z{FB_SHD^NJ@#$U_akpTe>A*3{KNa5
z_{WZS**_cKRsJ0DKIzx3_g{aJKZF%-`H)*U@FDE??2n&+*M3a+GyYTQpR!NW|E7QL
z|6B2S>%U1~kfOFPcZ*hj9a(JnT2*`|Z+VGVKB@Fu{*%(_1u12!0)E+*Z>P$mzlT=z
zes8Tf`{Q_J+|TGL=}%$RgI_nRXBLjHVgJU}mi{iNJ@{u!UE<%+dh9=8{l9<D8%`Ci
zXq-|Uz;!GUal1-hH$5s{*Sx$ev?Zim*<vXF+*(qxjd!IovTaF~zHManw{~*P{tjME
zYUle}i>{M(tzAp%A9RmxSk>dvIJ_6jHTQOLOZp0%uJ%7{UOsTFC3<jeD{W{xuWe{_
z+iQMU`#ynZ$26gBXNZW=MHAz@`C>zNu|(1HNGj|-ARFpiBp>QeR`3S`l)^!lN;;%c
z>G-W`g0N7-5xvlQi7)AfOZMu=O4k}@%H|o@%O{$SDH6^1lwqhZDo=Ec+7=_wP_bkk
z3g@F&;8F}j_*KRp!g*5%@r${Y)QRGfshB!SB(?@whJ!99zK&K*Xr$|j&5U7W9&<6J
zoplrGV%JgoEfBiMGM=Hd+Q>9;9<$IkO>Dd^)sk!%YeltR$6+`;ux2|p+E_SI>@1w4
z?b$A?92l;*9BFQ4kh-CFrg(U};yfq1={&c&3%u@nw0Zybtn?Z1`tD2ddEpn}d&xiD
zZ)d=A{{?|Z0>%d25A+_EAA}FC8P*Zp6Z|Da9&#qs5V|l76&@0f8m<r55C1(}8gVqD
zb42=xqLHMLPe=V2c`$O_sM%5Wkpa=)BjwSvqjF;;QA=WXMq9^WVt&T$j9C~jjxohg
zkG+tPAM2H98}}h`bzEFhLEO(IVtmZ#@$v6QpN;n%QxSh*3_bxhc6h?lvGWqj#vVxs
z9QQKe__(TsfpOA=(aEgDTgkzRisa11F)1q(FQyzz>`8f?7@qnw@jz;0VomB$qEnhV
zacLSSDK{-BNs~4@DK>pt(t-4)N&nLSOCn|*OB$1LBWYL0%cPGPc}e1oze(QX%ai7e
zuT45XzB#FId}oqm{6LaJrZ6cvQ=GIuQ<`)wQ=U`+s~PqwGP{#lS=^-Htg@uEtnW$l
zv)(3c$hx1jC+l3&v8-K5=dyB=E@!1DUCRoFWdYmD#8X*yiF>kMC$7vom^dkGdg6#I
zzeKAnO@cV{d&2L`JqhPCqZ4Ll%Ho|e@5MKdPme!0UKTfY{KYtB#_+hC8C9_{8H-~Z
z(-kqR(s#v})37ne(+);6(ooSSQa4B8Qu`wROPL%wn4CXqLbBti+;Lk+lE+nzm_62g
z#IrH$B2=Tl438bH58szGBK&*ex-fadlTg=$){xYABxGG&MDWGf>|yU?b_W$lUkhxE
z$_?m@{Ovz9s_B2%N!bXck1#^#-8Ee2)fle#ED6(ld<|8*KMoPQoel1F-7>7jWoA&d
zb9CTeC;Naej#|HG4voIo>_7M%vpeU##de+70-G$)4C`SYqc}8oAFCl(TgyK#REvAg
zDE1yFIdhI<A0yVGh3;ryOVinwQ|oMsk>}Qhlmnas@?5JAq-e`mL~DyD1PSXtzJz%X
z_n2`Hdx-u3vy}E2okD$v@<VdXG|Fd_f>da1B~}{#5_tNzc!BO7&ZND7WowUM{4_h!
zW7Hc_3sfu2dzA}KcNH^@zvYt*gR(3=Lzbx<A)TO|Cz+%<CZ3}HD4MD27cNlR3s)*8
z3O31)^7qJo4V{*n2JcCt2fv7S3{;DL^bZN~{RH9oJ}<$w-ZcJT&)T8Lo{NJgyT1%{
zb+z|Lb&>locSiInI~Mm&?6}mE+g{wwYDaZ%ZX4Cr&fC<P!h75Cu2t6V(i+l!x@CPE
zy5${jQ?t5ts42E}Uel45CT?x>6s}`)P2-ZL%*K!0@&+<DqhW4iRsF|?DfNtowz}o@
z%j-(&w6(+Pj@DkSb*LrP=GN@2NdfSsxB9>8L)GT0!0HoKO;xT{C#v!*$5v%mf+<O5
zapk*;3zaJ>aw-EVVk%|jc9jL?rizp0{EGSIT@}O2+bT%qyo$E6_KGiMeHB;BBo*7r
zj1{v%02NbaU+GllSBWi)t{f`OtgI<rT=}(hd*%JobCst{pH%KD{Z_fAw7zmdsjzZd
zDX}V})V*qSX+l+W>Ab3l(!Eu|rFW|QO21XPm9|vbmujjkORcMs(&5$U(uviol9ko`
zl7rQqB{!-YN?uo&l>DsDE2*x2R?<;@r9@nPs6=1Av4l`Fzl2efUc#vvS>jmZS>jp4
zF7dB17l+gci$~P77RS_-6_2j@Ry@AuZSnM)d&LWCP8Bby*<QS=W?Aw2n(4)xYsMCD
zs|hXMQR7g&vjz{3!lM6bxJ6rQeiv=3$t_w_bE{}+&GDjnHUAaOs##Jry=GF;l$wO1
z$u(iHd|)}lbL*lRHLRk!HAvC28e-AL8hp{78dA}z8d}k<8q1>G8mFS)H9<wZnxrCK
z&8#Au+D%1KwWo{b)IKjdSX)u_x>jA(R_jyDs+(1uTz95;e_c^=VI8^zUmsVJQopa{
zbbU!lW4(2$XT#Fctqu956%AHpK8+j84mCEG^)*J7kK^7gf5oL%I5lmnxX?6EL2jN{
zxwZLErJy;vYIV!+Dt=3Bbx!M_YEf%e%~oD>4Z1C-_GFu~)}j4a-J5pz`mr4a^(`GU
z8rFB}8mL`28lQ9}aL08Ia0j~2H=XE-YYy!dH8=O(ZaLVO*&5PsYUTC6;vF5B+ctdA
zqP=UdwEfJ`zK%G4WT%KP>AWd;)HPl>s~aO?_Iwom?O7_`)oU#o*;go$_pOmW@3)uD
zANVV?8QdVR8FW^h9x7L4^0z5z0#8+mph|UExLutj^3tHhRhm5UHtlAKn=V9JsuM~#
z=%2`J3_0>|26x3`V~2ujx~_a<nxdL&W~#O3GW9joVNDb|LED1XX}4qE>Flv9^?6u#
z!%Q64pvIjxp2v?d1roHTKZF<NS;TCVj7UQtCVfXck=J70k=?M#lsarHWgl)85`fnu
z_4p&yT?8wdH}Nj5oajSeM|wrKB8M{a$R8Q=C?lCT<O}l&GK!T!{mfEON3w6yzOYBr
zBP|B$pDa!?hFgX*-&@u)!>l&4-dVBOA)MFj*PJv9KkF`wd)8Yl?QDovr)|zzp>1tB
zYi+M{nr&^Z<Lu5_-?78m=<Qb7WZBo*+_Mk271^J(4Rz?SUF#5N_t@b-yC#R9b_7Sf
zy|3d~`>~D(?B_WCvft=vv_I?^?r_0ziNh_&YYtBw%N^c2njG>R104$;XFHZT9(AmB
zeDB!c$a8FTBseuYhB|dP&T;B@JmMs9eB~r|taVa2YMpeBP9SeibVfNXb~Za6gU46S
zIHwwCs*}pu(#gif-YLq(#c7_4m(vlKK&M=n5l#&*NlrSKOeYuD*-mM$E1b5uZg+a@
zdeo`f^^%j`^%3NQeQ}!QR^fEYt=s8`o7PF?#&QmJ4{~1Pp6UGBeXX;^{k(IyN51oR
zk0Ixu9`-H-&kUCg&pj?zJm0(Ycn-J(dfB^f^_t-N$LoYE$GgNe#~bJN(L2_S?DL=7
ze4o#5c|KY<j&GFvI^SLHHNL;yhx^eyF8fXO(E8o;nB_0=_~9St=^b#~^F%<sr!2t5
zYg*uHul&HDUXDSQ-v0&VcsB-p@eUtG@i{karVoGEL!ZQ8q0i;uP+xxVCg13gkG{u3
z<h~6d!G12G%lvXeZ~8qAE%)Px8vFvo0{xeT&G5e&w#&aP?6$u??3=%Dc#HqUaJm16
zaALrTaO;5E;hq7n!-E69hK~sN9X>LkBs>h3cfcQb?He5PHeBg{C%o1FV0fPY;_!?9
z3E?aKt;1vd`@@L-xnU)Ko5GIzjR+g<rwTRtz6yQqJ3n*|#K<AOze3*otO%LrgM>)E
z9|s@v9v$rI-8}4z*Ro+rUh<$~&#gh3p1QyWj}3v7Jq7|w-4_Lnb?5qjc8m9Sc6;S_
z($&UK<+8(fhD)nYzH_9Hv-3so-A)3p7RN-d2*<0QXB?y+eESrSB>S80m+eGuL$<MQ
z(YD82PubMDv|Bs4gjg?h-phI7RBgp~bhQd{Tx+?;;hV*Cd!j|ZT?X63?mTOrZ4>jd
zjRUjTdJaR+xlIq?bkee|TxrKGv#I$O*AOAQiQ>$*rDU?Ek#{lAl3p^3i7j+2kwQ-(
zMA7!(*HC}ro*@L>03{XcLpg(4LFS_0l6=s5;!f0PVuSevVVJoIf5J2jFEk#-Wf}Xi
zpA2bO55p(SHN6LhsXvWAsxzYy-8R&9t=1f(U1EN(={JqhY%%q!HO4LK9Y&U#V*I4K
zVwk7$HPBS~`T}K^evPtM=b_xDYgSn5&MQ7@Cn&PDbOlLUCjY28CSRe+lzVF&<^5{0
z?6JB~wp@Kp7OdVV)2SxO{-`2k=Tr`|`6|3DR3(-YRISn$WtsG~GGBT?`9eBNc>~l4
zC#5vyZfU<_jr6->j`WHmQ@U9(8q|C-(l|wg)LjuI#VI_cgK}4CmE1}CL2f6#F1M8K
zkt5Q@a-4LW+$iypt0W}3OagjH$!}S=<c6$HvQbtj$&kGVHR3~wPI^{SEZrozCY>Q!
zC5@9rOWh?9#w4kbbcruZeu(EuZi~Gn+r)k13F3QVAMqS9T5Kb36qP{Q*%r|Tk((%5
zR3*fVb_z>{UczI-Qo$JEY5__}7km-C;I9=V^W6l!L$&<<L;LuyLxKEngWRDRgL{YM
z1A#-w2O0;R2lfoU@An!^=`SAW=vy|hr4O2KeNX!z^``d6^a}fGd(QT)>>1vN@8R~|
z=>D%ayxXC-s_R$Js;=cdv@TB1o6cX|(>vF8qdFbCpLdjXP3_p(h3yFIde_eHT-^S+
z)4F|8XK|Zb=dQM{j)=Ag9b(>+j_169jvSt%-Gle5y{Gkj`~B7h?b)rt?Y6Dvw$_%0
zwmU6v+h(>LYIALw+oo=gYb$KFX*=Dl;w^6G@nW0*@NAo3@KjBgdEBNWyf00=c-NXX
z^L93^<}GSk#>;40#2el;m*>=!%_BF>;mNskc<tPIylQR^?-zFk?+te~?+$kz?;LkC
z?=W{KZx?qTZ!`A@Z#DM}Z!z~OZ#wrjFN6D#H<tT~7sq|a8_E653+CqY{J8&k9^6`<
z1DD6M;tufGTp26_&yq`M<8Uq7?6_`i_S~R0M{ZP`3pb_BgFB<mkGs4rn7gfQB==NX
z0{4E~81DPFvD{y6$=vF;@!X!aX<U6<HkZ|&!wqO($4zP9!ClgRn0u)G4EJ99Wo};k
zEp9{mGp@4z3zyST!j0_c;4bXYb5D2JG=1v`Z{l~PH*q=_HN|)CYFgQOt?6p#*QVmm
z_9jCop*f^0xOqv}tma!?hnnlUUNsZDTbpCMnJxcyk8AnV{a=f$`$>y$PixDf9(L>P
zo^h>BJv&<Iy&qardqu6sd);_{dS~*meHVCR`l@)x`^ar2eQ|C0{w;0e`k%KQ>2GWM
z)6Z!q3`}fKA2{58V&HxIpMkD+)Sz9*=)q|nCk797{2hGXVICaph#g{gZWs#gyfrkt
zvu@}@rxpKaXF9*Jb0uHdd5#~@RmGp(MG)-h@)um|$`F)v?GyBO-4_VEUI^^Ed4kYx
zv@oQbEzImr5YFzNEzIs-E?m`pM7X>Aq3}@md*O-hpTfJ{Ey6F|3Sn_KQB>b;B^v7X
z6=8bfM4X;1k#Em@QC!ak(bS#;qBT92L??P)h@STR5*7C}izGcNkyS5S9Mu~tUeG&H
ze6n}F_-pS4@j&l;v0WcmJf#mKIoB5?sp^{{vFJZ4nbZGE@}?gxB@ZM?mku14mJKvZ
z!v}+9w+9c&m_zNd6GLO=Wd0lZLB5TGAlR=sBTy@xh3k|rg#F4$(L7bXs86+7yg|*8
zpfq`s^P2V22yKY0O)Hcg(7l$2>31k(`Yh!qgQse@5v@)$^{d^@)fyD)leQUkQ}+&i
zQhyn<*RUVE-nas{$TR^z+Z;uhf$||vN86C5VG!~xES|Cmhe1}tJ<vLWjJlO5pdBL#
z==aD1#yiRYvk>WJ)ls|H0$Qg9iP2%{&TO`dW7Tu!u&b^2Sd`d2u>4_LZuQkp$N6mU
zY4g@$lI=4{x4!4}&i;mTx5EV&3#a3*W1RQ7t#{ete#>>0N15AFPrdtmFF(&I-m|^N
z_?+|(_x<kU=_m8G_ILKj1f&ND19k?s1U?J;7gROu*)UD;>0qDG9U&9LazZzRr-fY_
zJ}kT-!fJT?2=#F7$c6}3<d+daQ4dFsjy^bQa?F&-4Y8q7yW)t^o8s%D_awZDIhMFD
z_Ha^C+~Lur_<du_;x|G6VBWaI#Bs@{#9=9UNvza0qot{iV`|dM$K<AO9eXgtVO-Yu
zPvbb56O&srg~`{m_NPppV3~?cyp>uo(KT(^q+4n1$qwmXCSOgTHHDF(nX)6}<kapA
zmuV^EUrf6{K5V*d{D<jrnc*`|Wxkowl<75-lXYokMwWTzo~(JZUSt)_s?M^SEzQcA
zO_}g=w(W$W*^Uz&vu!4%WHTnL%EnGOkS)qOpWTpkJ3BAyarV`$N7*a0u4IR1?aUTt
zPR+iR>7SjJX_~DZ|7Z5)@q1<mjSrulpV2)lG2`T{U+KeU#iak4`7&+NOsllU8Jkk4
z%&1H$pYE9wGksI?!)XQMu+xlVXHOkD_VtvtV+@lYj82?fpR{)pCaGXzP@-nSjD)ZW
z2jf>}y^On+SrPkh{9p_}10O@raF6y*9}yLmHZF2f>g-XgQr3?=kbGdorEzB>?u@-S
z{Nb3(;ZH|j4SSY!IrMSj`H%+*XM(TA9~yQ%Zd=fQu`2@?#AF8~N2mJxMuquNBVB!m
zMlyT~M`*pSM6`P@8~%SRodr}IdH45|u_&70?hY-bl%f@+wsm)R-MY)x+ikb*Teq!I
zFI0h+QrwEWLm+69@x1r{ocC}pGnr&&GQ)7~H}n1ca19E2<iZD@aQ@wYhttb`>m3gS
z%yEeF549KgS=$x)nAkq>Mr{^()mmA5CR*0H-?Mn(cGi4}>q;|gmto2ZCojcKM-$Un
zhYDlPF2N|<_Lbpkn~es3)+6PHR{dokEvh94&7X-Em|YUtC>Ihrrk?nD69G2IxE3)m
zdMf;6uwAf6K1Uxdi{UjAJMNaai&-qHr2U9osu6olo<~pV{y;Wqjf6`zzx3n#_VFRT
zVVrvp$(VINr}@rhREye^tm;r{i`xIwWU5y5rL^_yO=;D3|7dyFmDIeo^J`O>I=NBM
z@w4HpDz$!Z+t0e;t(mowmi(H`ro!s8jb&Af8rmy^>X{0p4zDPxF)M#r?N|19Rcz^^
z%9$k*73+(g%1;;JWzT*$mZTP@71tNMDH0dl`0bm2sBnDVhJx+63-TZ3Ovo$F9+E4{
z4$g_l3dr7?>7Ml>!!ENq-8#c9-8y~QFN<Ff)0AlqKP`TG{&Y^=km~>A!;jGK?Dw%L
zG2d4vpHDfF)S4XsZD?}k*E>myuhOJ(Up9X`^SSYB)u-8CgFjV#Is0+mm%hZ7&kGXw
zd@f3``#d(`+o!Y-OFxbNp!`(uKJDYo_q#tfzYF=e>m8G5_U=dG)3>J*<KE6nY=6@~
z@%$S}V#J%egx1&J68?StC}G;`3kk-r|4PVuwI$*2S8Ed{yjqc9_G)oL^~*&G&tEQ1
z*!yy6!o-)W5<FjSPQYFsN~n*&oRA#<U&8(PF9|2(3llcQs}g3$3lfLLTO{_24@`88
zADd_uzdTVHe<V>6|6ig!{(GW9d|je-y!4|(yx&Kc_z52a;&*=>8vpd;^!W0R8{*}k
z&csK5dLRGSr^@)RpU{`&C*PO-KhJu(?(?aauRf=|BtO$Hhkoh*>e!ccuZq6BeC77F
z=hddK174?o-SXP@+t=6Ize(N{f1C0qDCy3d3rRh1sHE7p3zDzC{gvGFHYjE6yPGL5
z-pRi!-|zgs^L@wnhWAT;jQvpk<Ku_9son|YsW%d4{4`4}`*|*LL7M5u_O#O<xBN2r
zBuqc`>F;#qXZwunpI>IUei@z_|D_~z*w>9&rC$}<Yrn;3BT1ujt|T?*3`jnfo15&H
z_h(9ep2hbq`QN_V6|DM^TA)l_Q}{L2{P(7xX}?|5wio56c^4o5Ra-nf{Xz+qKD_i*
zhNg5)=6_{=Sxd^hvK%X3W|vlM%(+k*k~_JI%QdY|&daVomVc~fQo)E?heB~(TVYyV
z{O=?6TZ>{FB8x?hM9I&_oRWi07fU0X=a-Sq-sNvv`pVa}ey#9sJ6>7aHm&N4%C&k@
zduO#l$H$tqj$^g^)ide_cly@%c5?O4x^fy;cHeIF?Ah4V)Dzu&uh+a~F7(hXHOZ~{
zn#*l}YuBho=!UfGbrv1($nK7{RF>MEe%x6_|J8MYnbG|R>(?XZguUOn-@SYIr+vZD
zy=&7?(f%v&(oGgpIth|SenzfRo6vQ%7Z%NwVOH!hyqygpzH+U^Y5t06zJ8>*zkn1Y
z!uyh3;W+6ngpzJR@5zQ^<K#-5k~iab4L%TK3{Qx(hI7TYjRGZOj0q`aTq3<=@>n*?
zbi2IUbcDebg}LD{Wusw-@|Dqfvwg-P=A%tY%`Hr~TQrzjTRvC3x7?<jYBj`6YlWL%
zu+B6OvN>mw17pq0Z5^x-yIQMjcK56Y+Ap)oviG!I=+I)Tad>EV)Nz@;os*}-OQ&Xs
zXy*rxCC-bT*0?x03tfwy&$*s<adR8t`rb|GHp2abTdDhM_vIdr9;C-_k0YMPJ<Yws
zz3zLpdj)yl^iJ`f>@&$n=3D3U-FJiUHb2(a%m1ifg}<f$>43-nL;DR1=<1geaI^o|
zet!g(_oD-s_rE)!r~inc?SZX9OyIGB+Xr|B>jwM`-WarSkUEGRv~u9tkj8-?gXaZ*
z99$GUHuR4{RiR%8tqAi8=?*&|vMpR6!iFy!d>|rgFcH}=^l;?gp%}D;c1Fd9bw%9_
zTR21$mOUgUJY?v(@LNMG!zIJqBGwIC6_GXUNrc<*riiV>Z6i~MkB?MD?~j}v{Veik
zbbe%2bZ?|ZjA7L1826}MG2u}UW5z^f#!QJ)$IOV5#ZHTIjU68q96Ky3D%L-0c&vHU
z&{!%mEVei@Fg8BYCH7#XVeI(G-Wc=9!kChX7cu7}4#bR#7$3ugTgE&LFO41@em=S*
zY)tgYFeKU|?A7p3p|gjN3DplP9{lgHIfFxn)r4dZT@W&VXw4w?keP#a49O2hhl~h5
z7WH|cBr0Iw;mBJ-e1u`p#)xeLn!>9CCxs6S{2X?#zk3+g|3v75eqDn<2237o8t@@x
zxxY=wXTR-(O#OZbFZB%we(!U9pv0#>Xoh!a&;zf(2XuSZ28MeE1s?G@(m&t5w4aT;
zcfUn$I|H7$X8LPfto@^0miQfYj`z)X()-vujrCdKc*Fa(L%SDmAL=#6{;20QyF!mv
zTPKeY+dth;+N8SGTidvWS+8(CZS}#W!BXiGYq`w%j>StSt-07~s`*041hWqgmSzSH
zTa`=g%M{=2hA3?89-3~lH8jn(*=pi#Q)_(0dbDw))mx)sRt`q@EKeFrELnp!7HbUh
z&8y{u%qPjOnWf1%WrS>j^0oA*!dKc)@j!Cj)JY;Xy(a$C#7taed`>jh*i7`*=pqql
zWI;SMyoTEvI^vfM9%Cj3f!GQ8N0gAqpnu775mYu0IU?;5%B6>dXCziaGs$zob@2#+
ztGG%3LUd3cE^^VQ6G{AZVji!;1^hnzK4*<j;9g@qI|h5oHlVZF-KZgJjV3b*$Yy2+
z;=|CwdiuKX1|1=sNmmIh=zW4h%1v;F%F<7uw(6CXv%ZAP<gb(4_=Th=?@5+%UAog;
zf-a6bs6)7kIv^+McC%V-IGdqmncL7K*sk5ljMGLi?ph%O1S`5&bBcbcnM(hyv7=XN
z8mSSQc*;|=lfpGIRBfLz^`)<Zyw>-G+}XF2oY^;q9NcF|n)j)7n%-}^BDe$geeW#Y
z)m|^%&R&gnMsJcfr1!X1**i(w)nl#A?y1%M*K=QUpl5|<T91#$v!|nv?0(w!qkBc)
zxo&4rIw<e;?LG_rfH+V>V0w0T#rK4FE$ZPr6+PcNle+hIuIUczwC*O=Kf7M5S9L8{
z8+SRX-*=XEOz%A1p;Hg-xT4m!ho~R5=XXqPU)ceo>K!jsm)obSeB1f9&#L=vf2c;c
zRkroEu4%j4O2hqRM_Q{}6s`MOZnZeK__m}pziXb~9NEk@{cO70G`49_Q$b^4<CMnr
zjU^58##s&b8>;HV8|KyL)z{W7sb5e>*Hzb^uA5csT31x_p>|x&=-OX!Z&`Hpnwqaw
zeD$EJlhw~EZK_==|Es!O5mIGTky?4Ud{U*ptfpdn*}97EQoeje=|APwB~Im2OWu@a
z7sr*w7MGSL6|E|bC}K(yf1fBB_}jW9zVK<WZ((%t<AT~Ew}L%IH}kEEtn**~zLGck
zw^d$q;f>rQh0eKNg%5MG3jA_56uizhD+tg2n4g-pAb&!ZIKMjcRo?o{d3k82Vcw;T
zuem-M8*-D=-EwE7m*({R`X}equecnaU;6B{v=7<K(*Derqy=O@``MK>`RD5_I(2K-
zt<><Wk*P$M_D6o^;~zIOXa886Y4Rg7GxfVk=Faae8A0EZGrCj$&3K)%BV$|2w2Y{f
z;0$SsRYq~LKK(&*OZtxF!t@ErDe1n+Z__2okJH<dZl~uaT}w|)x|04V>1z75q<_=@
zNxGMQEGa(yXi_pv%hFFJY0@tynP%Kg3e0$uG&v(9X=g@#(t`{p>34=Q8P5z%9-KKj
zd41;g<VTrzlIt>alI^q5l=)c^DfhE>rKqz$r3}v2ryR?U{a%y(&-bvLs_&O`0)FUn
z4*r;yTlM2}ZfL4U-qqAgd0eVEfAP;<`FTHk^M|FaDR`UKR^at(RpFgqU4{1P+kan6
zM~ZAS{w}(eVN>jx`Ly_T=8%%8tm2Zato5byvgKvk?8jwibK=VT=QfuY<{q!uo)=K*
zl3!Yxoqw=uS3&=3-@=;e>cVq17k`hb9aluuNsE)~eirYoKUfmpFuas*z{(OE)5~@>
zT__*gyrhC?38~CzkyV{(t*x5X_MzHSb*6?=t*y;yA6NG;)cNbwW(`9+=>|ntZDVU!
zX4CiX_szF@Znx~}J>ELCZ)013%{-NXc5HjQHlib4=dXT5I&>bQlwFHxS@$po?{Q=W
zy*$hHR&jLSSDw<`(bL)^0=;gX5T|A$GCCGDVg_Ie))BYl3<!IkCfxPSqJDxxahNb!
z5{tZ)PC{?X7GVF#*Wr5%b`dKL4~wQ6ofStL|0@|_@?7d-`c-DC$de0{wFW9>mtm0^
zZ<Gvo<UY5sGQDEyrZ`|VK)K3#nAsGYiRO{E^DNx$mRri~*IRYjZ?i6T*l+XE@u=-J
zr@!qsJD;`x!{w5LyX#d)z3Vln-)>i&AGu$2+2(Q9HQMvIo1xcU_d>5t9@o8BdoK1_
z?B(M-)w|6%(&vt!gYPtd-d7$_>Gv%l(SKdP^8pt9SM~eQe`x>dfo6faz{bF{1L6mG
z25k*W2nriGVjwrLXyC`-1;LvKwFG;GtQu4vq8fB~@RAVU(6W%s(9wgJhrJ(+ggb^_
z4nGv?9nl#2AtE|#c;v0Hj7TzUT-3<$tf;f$BZmAA|1!iZB4FsGi0ecDiNJ?tM64Qy
zL}m{g7#T2papd2_|A}lK{xLEvx*_s-v@|L&+B?cLCOT?D%*?1$F>9iJ#_W!wV~$3-
z$DWEB7kegZBiK2Ze-V2uDlK+jR8{PzsP5QBQE1#BQOdZWD5p5<D6cp!(l@R$(mO6O
z(k<?Mq(j`QNQ=0kk@7fYBp2Hdp^AMKQ53r;;!Es^h`X^S5r<++!WYNh2p<x=Alx$6
zIlLpLHtbu>^{{_pCWXz7A;LUk-i3BXFA04Utq9!`{dMrb=w*X@h8qliHvDDCvf-0M
zY=-v^`ZetIpiRTP2H6h#7MwhEa`5t@Ed%94cMg0uL_ToLkgGwhQEowpqMi+~j|v*_
zI`U&+bmZ{B?1*3er$qeGzbrhv-`w!A{c6I}17?Q}4JZhG?H?QJ;{RdrSwFAA0>7&v
zt9>OQMLt^wMflVN-}R0Q7JEM#xZ2BTV2<aXL4lqH1I~H$AJFZ7GH|SWQ~xJ!k^QA^
z=ld;pZ4LP05*py(a?F3LbFp8RldGSL(>mX+jtM>)4niM0hY{ZE?f>>lw#)T2wKMUY
zXFI{;sm&R8opr9;Fl$4%Q&uBgOD*@gxLLk+-eA${^x53rNn}3Rah%ynhx5ws_C*TH
z&QjrPH^+3Y?Ol_zHtohgt%HqOtKCNZtdb2ESc(kKS^Qz}-Tb^<V^$$^H*=LuRj!sE
zR=kqDGbP21CQ)LA$zf5XaTc-M$c#8+I0sKOxQErqTToo?gL=wVBV(jbg&QU9kQev~
zeu~!VyNH*(C86bp;E~Y9I?P@|GnpTVDbpjIPTLA^Qqh8Da;-j)yvpy<{p2#Vy{x6y
zo?W7eV_x;Gr}e!zsIk2n<n<m(r|Ndog?3NS9`D+vDervL=hOMScc&WdP45WovFcdR
zy|Vpu*Jst2PGgl?y{OGW{l0Z#hphET`@ELVs@Kh$HgU6G+x(^_t%;5QwwN~-H*ah(
zZqBbC+0?)OVB_h!q=wE~ef_xFsQUPteRYbOq}r|3NNr_RbWL>CL5O-ktE?*}RYxnv
zRcgyGRV*s6E6*tlC=V~&TlS<hx75DWx%5KGmJ)-K%;LkvuEm1l-9@{Kiht9;2mIdg
z`&=Pks4F;FIKMz%n4W*JAUNNp;7(qAzDeGY`~$gpdD@)Cd24dm+=lGSxwEr_bBnTy
zawcVM%_+#V&Y7I~IlCldLH4{1Np@@c>#S|*^Rp!B##z^XC1v{m+L)P~=8-ugttvzF
z^J>PqpEEOpe%fc0rq-qJPkor~o4Pi=<VSG&;U7r)pdSUlI=(;r_2~QVUn{;(`Q`V$
z|1V97(XTHlJ!vOWO44SfB&GSMJWaz>uBKHdpG^Cdd?4+5@~*VK$=lNwC2vg|1vUWY
z9g_E?86_V{6C|HY>rJ|y)|eEZR-E)TEj=ka?ORfHT0&A!+PfstulGp~zdj`m{`Ea+
z;;-zaRlh2dj{fRO`tO${x!{+3vLHP!IWT=?^3wFv$=B20C;v{bPd3dkO_`W6Jmq4>
zmXxZDXDObUO(~l*oxdk%F8FSmb?5t%torYXSsp)3v)BGunVt9}Ia``)n=?0cTh6=G
z(j23ok-4jWKFIy`(>Tv3ZCBo@w6?t7wAuMHf92+<{E90GNdH=JGd-lxB;!@#p$xy@
zY{t{yn=}22x-wrDt<DNAZq52sydpcgq%AwEWNps$(%zhg(w(^*%J4jW*}1$^<u3X5
z74P$(SBxnbQdw6}RJpZqZI$e|r25wHd)0%AhSua4Rn@F7-c@T@VqN#7<bB<!(kb<=
zrM>kh%FZ?fln-buEH7-_QL&}Tq0+MXN9D)nHC6LkEUWR>wCZQA+iNDb`P8y)Rkin2
zXX?hb$JI0KeEokNZyIK(S2T({?V8?o<}@wp+TU!{J+S3tcWcYio*S*kz2n<H^kS-c
zeF-W;v%dYg#-n4pwz>n+T~t5SjqRL5in@66Th~Ksd-pgxut!U`_uOC}_r|cx``TG|
z&3Ud)6V6}NHu4j6r}PqXkRXMuf%~=(3xjDNq=U{u?l7CssjLMi=Mu4GZYI8sr|<y%
z1)@nGD7q%d5seeB5hI9H@&<VzS&BwUEwCbK8jQ(r!%?{haasPG@H03d`eqOyo@7`q
zt~Wd^*=Q6fMU1PYXN-@_98H4d&rNFO;ikt8(oF{#PE}MG)+-JgtyOv(lgb?917;gd
zWKf@<HBU6Pu$Zm5YQZUNEw3xDTMjm}wJI{ZYPG@K!rH*%to0)cxy=yELpHxHIh*BH
zn`}v|X4}Ko)9e&BDRwt){O!GLFWJZ2^7cV?iyhwEr8x|?_jLSZf7mh7q2BSE!%(M4
z$D2+c962X{ry0(7onAUyITJ3You|2=E{|L`y69YLT;g0ux}JA^?ON?><L2(R&ux`k
zz1uUlX!j1cJMMn&diPcCGd&);zxHTwm%!cTvppwyJoP;6L3_UUi1ljrIOApJS>`pw
z)7g8O=L+wWp3l8scy@Z{dj>$k03*p>k9~r@8hs{uIry&jn&x}h>!j~(uO#0@uRh;A
zFL%F2uPJ^yul;^v@5g@D-r0UW-hF;y-d6r2y(9f6c`xvv>Al~7p7+203%nEk7kL+e
zslWsQ^Sw<2W_h~=O!n>{FxopZV5s+qfPvl<13bKE2AF#<48Xis26TIE2&nSf5|HV&
zH6Q`(f!F4MGhXWhc6hA{Sm3odV5HaFfB>&)0Sd3N0UFQffFjR90q;CL11@@61Z?%h
z1EzR(`UiRz`^!B)`qz8>>;J)HpZ{@>`TjFKBK^HQto^m_TE7(cJipWKPyJ@OANKQb
zpY7M<*3a*g8|ru1?YHkAZV!D;+_w1^y2koma5eRv?ponv?fTHC!exWcHJ31-X)d^r
zsY{l3mh*Y<W6m?ZW1a21QRix}uTD3;b~w%R3UYGvYIm&iyz6+?bC%;IPcuicXSTy<
zkAn{DJO(*9c{JJQxu3J&;~r)2<=$pj;C9Y#kK157FSiQYeAj)pJ6zptom?|)(p)y!
ztaCB9F>`ruo!~s(dWJJ)MLJ!#y5Ka#YJgLnWuD`H%k_>9md1{sEuK5fwixX|n>X5D
zG2df9$lSud(CoS0pJuUkhGrGEk70~`xKdzSq4>vUv%=oSK=IuAzG;+oglUddfyrE}
z<t8di&UmZkIb+1q+xVo#HzNy+$ws%$n_%pHtKmB{kzthCWrI|ukHH_xuku30B>7xL
zldRcvt8AUASf(+#BHdx)E9H&9N)8xLkPyZ-;$uc@#Bw7>blUK^2*|afD+bRA2ZJHR
zJ$WwfDW8u&k#%GKvV&N>)DjyYeTu%8M52L`T;!#AKGILzDSRQ?BlL&6Yn~A|1m1+7
z0H}(3H+-!A4pzn6VQcw|sGc)K|KW}!j+_X2&K?nlv4rp!b5Jmy5eVAo-TIw0!^`QN
z{56W@ys16hXHw5jAP=xDI-K3BJI<KvjF@NI^K^{XlCIWVqqb`tC{xWn@=2d38Qu3p
z2hX_bcJw~gn)iBXU-mrGjPCK&G<HAf+uI$`XVv|t_jOlD@7S&{J#C#aJx4mTyInh{
zcPFbGx)!L{cOh!F^Fhav&e0vFon7tM)#uth)dSn#z|+RVI(DdX+Z|N%+Ed%qs<mx<
zR2FT<s;{m0+LpJ5wwbmTw0>z>)4H}r)auxByQQ!>yybXvQS*@I4b7S+dGq6@hfQ;u
zqMH;=b&bCo_rs4<V`yViLvO=^hNlg}hGh+R>RlS5>#OTq>aN$HuA5mOT4!C~R9jwm
zzV>EabnW80z8asp`!#*FQ)|A|N@|YRey*NUyRq7_*1NjBrmpHk&9$n-HB+mm)+nmH
zYVs@j>hqPqtEW{ytG25=T-{i)p!!8cWcAJp`|1%DNVT#8LPL2$)${UCReQ?sRgEt{
zS>;l`sY+8euj)tH=&FllAyq5O+^U9@nO0eqah1KLEtOfNg_Vy=zgHe9eOtM*^ik!+
z((9GsrDrNVVEkNJ`d1}hdZ3ajIat{Z<KS&2rz;yuE>%{R+^sAvd0AOhk_=W<SzMw5
z6I7L!SX5P%1XR_QjHqfZSy0tkvb{=Ia<)oP`m9P?`m4&kw5iIq6ssOk>Qo(D8dW`~
zbYAtA(*4!vO7B;{EX}OWE7er@lseX!myN9nFWXbIsO)*o$+DW7w`HcaHD$52^737^
zq2;e@SC)6wUMcse`&GWEjw-)d=UY)xH?P8~{(Qx>`izQ8^+;t&eQ2e1!=}nv4KFIM
z!{|$8gIAS9<FcwnjSs3GH8xgtG`d#@Hm$7Q))Zg;rHQN-HAmHqZ$4aewK=P%q1n7P
zpk+bru9g?IzgoCj%ho^YR<z!zOKk0^leZ12pWAk>{$*QZJ+2zmFjIB3;f1Qa0dEgz
zoY8)`@p*e?Bia$zG^68aQ+!8dlSn<Vd7k>8=C|thW?ARRmW7><S`s@&t!7=zTL0|&
z(VE@m+~(cAr|n>Od0S=oAk~1L3#y|%y{gKdiR}Y=U$r0aRd&?%0(YUWqT_PkP<3D5
zJ@q(^xbul-b*D(1+qqafuq#P>rOQFbbnVj3>MqtLcK0JKdybNudMe4Bo<Pc{_b_$1
zx0Gt=^`VFM?WF(h%b=+~Yi5#W6$AIPfGmZaU89}NW@(?Y{yK&|r5nSw>#lPnNi}zu
z9Kti?S$+~#$3La|=_T|o{W3aL|D84w*fVnlTbL(;Y=#neva!Oy*t5cFwpuut^G42a
zTaXs+D>9TP(5w6(Xdiz89i^{CZ|L2z4*gmTh+Eic0f{vTLh(@H4*VZs0^TGf@xjO-
z;smmlC`S?rZ<HgpqLHE$beG5wdoP-fb&Kv|-r^2ykvIguD&CLhiPLeB#FU7TOeZ!;
zZV*pES+_>wDpE?9isGb?MBAk*(Gws|mrFN@WwKY|Fj<dywXC1yx@-+72tJljvKEP}
z+*3MMzCyZP{zUpfu9D`<ePnw13ZSGs07Z@lP+xSEKQWjk|7~zV&KqRPJq!_p@rEG=
z+YHwLG3K^GvSA^RVnl|xQK+Fi5WYtk-85WjRA6}0h%kC;6k?QNw8BVbbj3(!oNnZ6
zOdE|c_BLK)Jk$7;@iF7)#tFvR#;wL}#ug@oNsNh;$)6?>CO1uHnPi%5Goef_nz)$0
zG8t`}WwODv+2j&D^_^sDZQ5ZPWNM=rXBwwiWx7Fe*z~gEmgyJ8d(#F*7H|OSOnsC(
z(D;`rmMiTPhn0Sc`^s>|cjYKWy>hxjV763YZMIPnXtq}|!t8`%uGuBUX0yAB6K2m8
zx6R%wKA0sbvdq#Ib!NX6TC*yJ+`LKQV&0(`Y~HIFXU-@Vnd8dM=Elmy=GMw9=1$6I
z=AO#$=Kjhu^I&C{d6W{fh*O$dj8%GBOjU+i%u$ZDSfre9u~PY`#d_rdi!I83EOsex
zSp21YVsTvg-r}?}#p0YY)8djc&*G}G*y4t=3ar`Uy0Y8ivXZeluf#0RD5aLCl}47Q
zl%|#^l@^vKls1;fl#Z5%lx~*$l%AH`mEM+rD!nXM!TeHKwovJ8IR|XI(#di>Se(+q
za){E-G6e3<41o3BlxCJzN`<AN(%2GL$}D*WVX0LJEY<J~cY{J>QL0c|WGb31zA36K
z-Y5zz9x75Tt}5PIoK)Pi*rhmQu|l!iVv=IHMYv+3g^ME0LZWcD=rmPW{5C}_KAQHJ
z-!`o?KW>_5zS;DP`FztS=A%q6ng^NgF}E{aY)+WQnRlA_m=~KEn}0FUn%y<2FgtAW
z)og{yZL={Z2h0LY7Mq!v#F_ON`<Ue$8<@Q`?oysH&R4F7yH3X%UsU=TZ&k{Trz_iy
z1}U?Ql*;Euor+UNKNV|?{#A@J`cvU*6suqitrU5N?WPY6Kb!6`JZU<`aF(f$p_i%N
zfHcW5_-S(8;G)TLgC!<G20<o>0cMgX|80C*{=j&>e5-M!Jl0q)R~nbgYK@-AUKs6>
z?J^oK8)4)uQyBG1s|=H*j}1>rHyBQr1{=CaNdvVc!5~2bPkl=w48}-=1`5e{d5QS2
z{H8cozD$hE1H|8DI?+MdXHm3lzep$>F8V0NMH{6*2_NY(qERx6xFwMgvm~jwnPfkn
zBaXn2i)k!c{2rd^-h?HH0<k|uU8uL{Ia)`oL9Y|O=yakTp6`Bw6yU3ne{f%94BjOq
z@HfIw*bd<aEK2Bt3537VFM`A9cEKQYfS?Ug!!zHH_2ZC5dLd$~e<S?OFBAUFTMOg(
zYyraW7kuFc3U+eM`Vj7tUc<)fU$Pp09s7XyVW;!WjEuj_eBtIXe{!~rD_2IBu$Sml
z>=ZhhRnR<BKz(4&Q-3m3DIdm?YM{%>8}z^Abb1k~puNZ(s#AA@dJQDAtva5J(0w3r
z-3BsS>q}nNw&><-AL%@`D|DS2KivmSxAvgst#*QDyVgk)uI=d)XupH{<E6ginw5Ry
zG?9HujcH$TUsLauzQo?yeW!XI`WE!o_XhSp=tX;1_7?Q?>%G@Q_H6D+?uqL;-DBG`
ztEaEqy(hc7r~7gD*X~2z|8&pqUfdnlJ-FMv+qhfX)znqo^$~tfb=~S(*mbBYxNCWr
ztZQ^vO{ZVi`%dGoQ=OX5rJWU>5uGWWrk($Fwx~~cepYYlyr7=mxlSF~IZEx?>8_T8
zuC-R(*iobY*^#M!+>xmMyW_EX1B^sW?l`6n>DZyR?O3fwI_9Zc+b62?+T+v-?GftR
z?E}<D+kMn)+g;R?+pX2%?IvoEcCp&5o$o-~0hg}o>Zn$=cVw#CIzFmeI-aQ-I_|1!
zI<Bh9JI<*}I!=HcR8@5BR@HTERke2fsp{@nt)e=XsD$ddz#N*UG6roIOZ8Zly?Qig
zy^K(Ks$*0Ep!+fiY^XX;H3sxwrm6n`-IuYd)nMCT{*Zc%>KsgOsV9K`%XHOy^#WCj
zdX*|qy+zfa-lw8K55}nTips6?u`0CllWKftj%rzFooat)uj)o8)}Gj@Y%lI~0sWUi
z(0>^QT%d{V!@HKW&*<6?8ZdviAMLu`ey8hg`=_pq_M)yjc$$)J7k68AICY0~M0U^Y
znA^R(V}JLpjtAYz9hu$r9lhNI=)icXqkG1x*Yy0UzSwgKG++|cEj{IG<6c%hsMn@*
zR`1}>J-yR9Z}o2OOzyqV+1UHOQ`}eH>C-3Z8r$dAwXtth*X6#oU0?dnch&WM>=J3}
zyL>dV?g^T}?(LfC-M8TG)l|)+Zk6VDx3L!O3D)-OnWvr8b3%KpCqetBr$t-cqtwZJ
zqjX`tD|D-S&*-lAe$-|4w&<8%bJD9XhMd;7kv!CQlYG^ePFD5xkz$P_)lV~qnxWZ3
z9n{>UUTJ<%HJUcib1|aBwFBr?+8Ok9?EyMh`;^9YC3KjMVE)t%WFF}jG1a<rj0O3F
z8Ap=L5z>eKO3r0<<Ug!G^$qm2yV%Q=6PHO%<n+{jZUFs)TSk|2*J*;!p#yo8na2-e
zPV%dn1pYSD$d@w4dSf<RKZ0GR-@~5Qzh;y4Ev#B^&)Es4aiaxixb1>W?xBF-e}j%T
zCfvvm6h7vc3TydGLTmjG;Us;J@TA@WN!E`=wEEqMr{F0vQ&57O5FqF$fj8PAn26d6
zx1*zlH$e;L2l`mp0u(YCCPn<QFk~Vq@oxb|{%cq^@(mN9jaYwFiZ4R_@$=|J{0F)j
z?*o-zC+rJ84r{=7V^ZQN){iL1CKFN|SQPjbViEq4_!}<=MMoNx*Q`VmB1ANRm?oM{
zY!dwiRP+Cc|3taO50Q?j6WNJ)(NK}4co8s=4vR*LABz@<Gez6PouX4>Q}J!_K=Etw
z3~{P>x42AvQ`{x~2K0&=VDt$jArc44cuAyWsbr31hh&%Jl;oP^wj@#VR#Gha1(f<K
ziLtat;wL4fV}NI|1_(;0q><8RK&;P@u8=lM_ev4jIjOzuo-|bURytLdDqSxt0XDr#
zdI$8{J_2*TP-Z7n%e-YG`5>9Se7J0Ye1dF*e4cE!e3c9$4sb8_%g)PB$nMB50RQ5a
z?4$fY*$??^S&lqGRwDl-tCoM3HG;K*b%N<&nb06XCI_u*3xiiOCxdv%)LzN@8@!PX
z23_qT@LrrjBIr_oluZIn>*-)~V15BCUjpwhhxL|$2KQpnxt<NIpos=ILBIN<EX3f1
z%-3MI%-LX#%o6n23=KxeP=jC@2|C{@xtXj=j?41oebQuk9q3o*gLZYY^n(1A^sxN4
zbc_6N=`#6t=~Vd=X^eb=bbvfk>L~Y-8p^FeFPs3~@E%EltWxq-mMVd$3hbdvz>L@<
zSt45^86*2cGC&qAv60zJP#G?1ld8md(me48={xZa>2>h|=>hRF=?d`}=>%~gXt`NP
z?Zk`(5m!l?MBgMCqT7-eqQ4|(MT;dHMRAgeA}>jQkqj6UDseZFDb6FFiC+_E#OH}E
z;w{81;O|9=BMC>b8zB(Oi7HVq{z+7VUlJwa8$?&}QKD_QlV~!oB?9ntLW18U>aop4
zA~u>hgE@mvHHk-I*|;STxq8s8cqZsm-$R}89Vm^BNAoZb^dZJ0JFy~U9QFcn!Hyt0
zbRLq91|v6sPPra!5DrHZg=XkkVKcG{IDIj~<Df`81JMY)kaPhh{8x}B{8Mm27%Nx^
z+HL;AZapJN)u#!r=`RV^>z4u<W}v`Aj|tlNLj4!u_MPRo=$G>&^h0<Xy%paMij1H6
zU)<mPEp9fymGkAtf`+F%kc4^m4)7|ra8KFM+(Finn*)lHA#56J#$IK+nKf({GlYG_
zSO9Igo0$*P<VfZbW6%7>2${J|BMm%dTFE@8yXZ4?Hob{{N>8It(&6+v+LfM2%jsZR
zL)p+Z6b~wHHB=(?je1Dk0=m*(Y7aG^S^+B5lPGs8l9E&2R4-{xm4m|Dce0y&PF9nb
z$V_q{nLsWlACcqW)gbaH=|FBLCFDv{r<+MO=|+>qx=8Y;ZUC@WeaSmIH}bU3f!wXL
zB-iMS$oV>)oUCJXBX!-nA-XnQh^|Q&sH@lcfqCod!0L6rx&~c<t_2tx9bkRBemV}+
z<OtFi*7wm_z|@ZP*SV1WblzltU4JrA7YY_l2I<C<!MdqrsBR89M7Nk6p<6>v)a@YW
z=#G#pbQj32y1V2NT|9Y7_YG9)^5Oe6kw0_{S)wzh+I21zLJow+*a*s>oJS2Kw^38c
zbJR-mIkg9Lna`8e)Du!irI03c1?fZgk#V#EwU~CN_JSJTHF^&9j^0V-(^siZI*~G9
z$|zrkrN%J!^lB!GKE=!jZRWq|0_GvDW3p)*mZT$r(YlZw4?ODKOg#KG%h`O;WoDTE
zoEtlf8^<2vwy`gO$y>&yvKX%dCXoqH9z(dTpj3X9KL+aMFS$CtloRU(yuaR^pR6Ct
zZ`W_<uj}vgU-X5b!;I(=fge!0rs>BD_UX3@{?q>}DAIoypn_I`Kd^_V34%bmcDC@I
z;Gi&5@Sjj4$PqdSyMcDDK>h?4^cB$L{3_fBtk(O8G4d0MK$?M-E<&#$Ug&pZ9NLC#
z0e#LJXds%5PDks}y(oe`K%IcvJ`|YE3xLPG5A(yW1JV9HwhA;lk6~@t0}R8zV|I8Q
z5a$tmH132i1TExkpnrS{G>`A%XYlu+kemtn#`Sm=PT^YI7?gwE2rtkmjt08xJWvqc
zMJyvO6WfW`#NR|FC=WLhZwU@KNJ>!|;VWt*qD3?@OC$qAgRSVe$WL@j6e)Tm8YlW8
znkUK=tphUBZc!cRezuBE1BLIbh!UL>q2e<lnfSEG1k4P~63hzBMtn+SFFpw@rDGx&
zh*YlP12De_mhFJ|H;WYFbs|IYG7%x3E#gGuMcty|q6X1GQ3=qPvPBlKPm$;|M7%em
zW{7@e#J@n0_($}GI3RifQSo15spup`$el!-XeBXFG!qCDBM3v$Kte}25jBJX&?iXz
zIVe0|B!1z$iFkYosI-kCwgahRE^bW>2Nh^<P>D8%Gp5B};^jcK_=;@<y2T7or4GTD
zVCHx%)`xjuzk!PV9&1Lg15tM$kO~)KC(z+Q$Z*Ccf}ZL?v>LSn4mS&W{FU%m{enzH
zZ-S!sUc>;Mi>Q!DBn`1c?jsCb4W;1!-UHG1f=~`x{Vl=?!XLr_Ac`9Z{}QUe52g#E
zgik>A^=|<wST86LOcXo=D)~Wyv0#QkrS}*7fUEMRzC^!6|3W`ae^ei&U!+&)qx3zz
ztv-*Z`FOsVKh3}4*YamUopv1`!2iLU@V<N>C+CZSUj2zn<8E^IxdYsOZV5LB)Qm$p
zKhA<P;k2v<m_9{pBKv{;m%Yxyvjw0~yqb*zMP6@E=QRM5eJAMt<^!MpEvWWi0q*=>
zP$OQ<++{|Bvamn1hcN{Of0CZY)Y3zlEZUp-NSiVL(I|77?gJ(N7VxGO^kO=Po<b+l
zBk0%QRUgv*=znQv`ZBGg&(j2bnx?4ZpzwcyZl-pCYUU=okXldw0`B!UY89P8EvMg5
zi|AL>Jo+Uyhkj1YrJqpq=|`~a9<>D6*h}f#)H3=e*bQnWeGS&TO8p7bt*~r2{eb!l
z);UhUqt4P_sO$6(>Jgnqy{3OtUtnK9VV^m)nku3Ds5+XaI%pIqug0L@=|medL9`7M
zOS>~u=pbex9S!Q9Q<<Ig3g#H_yD!rxz&BlE-qTN*H2O1BO6M@`bRDCoHH-=9H@h%4
zY%pjfk79;{hWvEUTVBJSXZ~X2nX7Cj^9qOv>7a|;z``9XTqtYDO#yY$^`LG1H@lU4
z#$Etj<JX*;&Et%?F3z7b;zx4<{BqDMKFqD*A8;r5pWH*fl}q8Jd=2l-^ZZDl+pgjx
zKnZ(}{tdrJU(VmuqxvMhm%d&<R*wrd>fHqw^>M&lTPCOg4mGcTC~y*_3StBef|UXk
zbcbDpZv@f8Qo%AIFE}Q25k3{h3A2PNg&o3^LIdF4`y!dbQAnF`1+eZ9BVNcIWF+zl
zS%wsW{%|L96A`0{h%;J*grU92B-9v~@cuxEAA_ESxyR@sGzq<fmZ9I!UbG4r^DOFy
zSz!@aAT|}0iPvItf!?+W)QOK{53$?WJ1hZ91rJz^HDfIpgOQjV7vZ+JIqnPk=3#h$
zd<-6m&jCgFwfIzg555Sz;X2SU-UWK^M}Yix7SF}6;T8BD_+Nhn8tKo$FTTQA@Rb7M
z6OIx|pah=+D)1?wdHfAXHlJ`MWZ??%7be6LTuwZ|MZ_)m+!a7F{sZeB0nfSz*Wg=#
ze6tRe<(GlCoeR3_lR!5)4)zs>r{euUXWkuu2H)}~_?L6wt&iZ{*bclJTZLzX7W)@$
z6#g6w1=6WEeiF09cYtSKhIL|-urfFgKQT8rYB~7(9`piMjvl~L(KXmhbP7=JBCsQ<
z3$_lGU{gVjKLX7|ozOQZj$TBYLAC!EG86p|2}e&L4(J-tXCH?&B7xw4l*oUGR(KLA
z5w1ZJfv9&~*bmtwGyyU|w{WO1SLiH!CPaisg>{0(!W6*}h-$XNg91vhKu{!z6vPW`
z1jj*>eGx=J=;P`w1!wdc{c3%lK2HA(l&z2I1>lFO;fnjhTj{U!T7ElUz)$60@<IG5
zUcs;AdqH8lkh9_6ay{Gy?kBgA`<ENbZ3a#Cv77*Ym$5?bEoi5o1@-g|>;-lL=%fd-
z<5&eyLo}c|{u|WH-!WgAOUyN9J7|*6VkSb~=EpcP1mta1bS?c3h-3fIw}8jImyV<7
zLADk}>nTUNghIhrGy!|;7qyD|kNN}V{He9T!JI^Okb|jo(usORN~oh`C-9nolViw_
zq(AVRjmZ;aw{A08sGCQA(2W3oeLwP$&I<A{1ah!WU6`&CXwKO>1Kn30sePeq)ZPSU
z$Z6dt?E&3A?N;3h?P}eh+WESf+DW<?ZL}^>8>Dm6y6Q}|<~md>)%9xh+6IkATc}ZK
ze`@NqA2lV~cultUfhJXZUGr6YR`XGNRFk0Hr+Kg4s(A-2kvH0<npfI6nt1Ib%?s^F
z&2#M#un^5NZJ_3<)*s9dSR?^p{r<P?B`kjr_Ej6CNrTVj!RO1hqcx4%snC5|0Q+C1
zF$OM&lWxCe0I)fR>;BbD)WvHS0-s}}E?;v*R}VbNK25w1*M8TTYfE+RTD5MV79(ST
z(=kmOK(5jbC--Zok(acq$tS?-_^Q217HD6Qt=e=F(bbXmI+_gEDXCeye$-CgSn7su
z9hI#6hicKiqKwI6Dum>yd88|SnEZo|C$~a3?KTb1>e9iKj$TB$FsG>T%zJ7tQ$syv
zBy<TgkVe>rv=4iRp2Q~6yVxH30r=Jeb}}R64l)7UJ7yl&%ADft+0Wc$ww*i1+VEf4
zF?<)hn|J1(@sqep{vapSzvTw&o492_yStzt1>B9je3$+$ZzoUzlhQ%IQ!ojr;fM6)
z;2*`pE`5;DT`*5LTW}KmVj}Q0+JxN#Ys6VN5}7RAf*cXvK|TqykRG89@kCtE`N(+m
zGO`!VM_!;Z$P9*}hS(-F61xXv*aGx2hGQvM2-c3R#+1Okh`{r(<+vU@556o3SQaY$
zD0rUtcqq|`&m&BTgG4y-2r{+|;vCUIe1q&pP4t7zY=S5bGPL!Oe_a;66nz!t06VJ#
zaxkel5cpX$#4+N%;zi<n;{D<@=m4~fKZ=bcMdCn7mw1XqB-tvlmt2wzl)RJtA<364
zmZ+c?fJjeCtfaRj{iN}r)Ba5|Uz#J?B(0Vlk*Xz^q^#tjR3v>bHJ1LAT1bne_R<Ea
zo3u;nE2X9VrI>6W=(rD-DrI3(TUof&8Kz#c5MW>iNke4)q){?IX^hNQI!fjv9V_#e
zPLz2{C(B%+e_#)0DV-uS1(U-v0!#?+Gm^0~jbx;(Q!-rEDhZR-NP=X=5?@)S#7*`M
zYUei+L)jw<C%p>w^D#-Wbc-ZKI#2Rk8Y{UX^_Bc3HI=NDlHwWCDsimzhj@VWnb=W!
zL2Mx1BPJ!w#PyPK;!H`1_>II#d`%(~ACPE7D<xH;Ns?4igyfmXRdQY=mFxx<@KRB>
z_zzLMc!21<*bF)>8e+D%5Nhf-goF43!HKpKWuoasf@m;tS!7A<5Rv#SsIS9BAE1MC
z73%6;cpfnae@aB*M+gUeA)&{@i7L#T_=I)gSFj9x3-$<~g#85^;d!_b#Dfl0iD#f{
z$T8EfqriV!itfZ>(8-t!>Wc|c468&c&;;ZydI33wZa|iTe~(1{5OY+Hs1cPg4M~Tq
z@sV&3T$R(2rI2UFLbUc2whG0<pTbtbec>;`0pVk)T8;|_2-g6Ac)Xwn^2@gZh2Vss
zS3h5n3pMvMy-aXIU#DNLPt?Z%W5Wfg8$$5(HM}$UcaFc#m-0LL_xue0A|J|c=56^&
zynye|HvvZ@6Brt=xpeL-_ZTwFqd?7A1|B|!3*wwPYoKQ6A;)ZBU$MW~Q^3er&7NmR
zvpc}Y&u68OX|@0r>IY-O-evlj157bg=O39N%q{TpN5RW)05a4}$ghSm&mgxvM=O{e
zK$=`cx6|VwuZ*O#Xn*=EZBM_X4e9$dM_r{=)EV&qN9a_@{oX(xco*`*v(#*QA2pF)
zLye+mP{Zh0YA`*J@}u1#R#;Fb;D0b$OHx!l*+u1&t<)E?mWn4!sJmn?b%ji&PLp4#
zL*zSZCmBzzC!bPF$Vb#H@*XvbyhV*8uTf*ki!eP0ug`+A`*~_Ed6`;F{!6VQA5a^~
z7u0q#f!a%cr~ZQN9U&{I<76{zvj=Kqj=DyQ>06|dzDGLIk4S&|2^m7aB!|$6WGwxI
z97X4m<LF9q8r?!JqI<{<z%<$i-}e-)gc=&67#%>pr6Z^mdK6VaPo?VV1ymosio%$!
zlnHYPm`CR+Pv$-~5PWJ3lR-^oYN_SG_1(^x(kB^T`W6#MgH|Wh*?Va<bDfs33A8I)
zKo4Qt=-I3Q7#?QKMb?LT!43!2`?*ka?_j#wON@wn%{XwG%wVpbnZS|E3eJ$-19kX0
zE`ogwxoi?JJj%Gg*dFdGE9GCYj(iq7h;L)Z@nUW{@6PS#W4LSlV(tz97njZ7;ad14
z4$;?ewtAi$q=&0ZAI7f(M$+H<oxl&i&i~YZ=IbEa;2`TT7dQhQW;jq{mH;Q|FmRF{
z=<h)nDMio-oFp6=8g2pyU?fEdmk4GF4+*vjA3(?HC(uxv1Q|kH*bKQXh71=vA<Kjj
z$lvf^^G5h5RB5N6T6=<6L*_IXGN@^Y5Z!{<q34hg^aWIR>BuUy5%~)hqL)!S^cgw`
z^4aldF}fTwvID3HyN23h@6aGD7mdSOfsG`@R$vy`cFYH=_(<$3HW7P@EyF%zd$DZn
z3b2vhKs?9>21gTSfOD85Zh{ZM-SAjE6e7X|d<DK7YW%&BL7vAi;t%lq_&Z=AeaBPq
ze8@Jd@iNFk8}MG>8?lht5(Ec%D+1gi9B4&iAPvcYEM!EC2Aa_n!irb`Y@;=V3vfL=
zi37kgItE;#Q@|@a3p}C=5N)o4-GccCU{AoF5|J>Cf++NV_J7eRn0O3)qDRnIcmUhF
zO;`}u2t(o`?C%`G;{O0i=@g;DPeKGc0iEolL<xSF$jAQz^71~QL+*lndkgUqUr)Tn
zmlMzNxsYE^B5vSu#5st1M{sXq2h`>(fy^`ma_kt$t^4CSxDEaZ$AICa#?InJ*dEBA
zSKtqTMtKSwhHt|B@mY{V566VKAEv@Au_B;OrC^;vpeh9)nvDK~Jw<n8SI`yUQ>S5D
z;IFa>$W>!eFKjSsj=4kU#R65KQnUbS_|K>cc?gxuY4j_y6@84%M$aNqz&Ub47a%Aa
z3%nz5;2jwv@euJZAw|O7kWVi{Zb3G12$)B!g*M0}A&!I!yWq+y5z2&Lpfhw&SPfNF
zhG4z$J#gRe2!=v^?<L$KFcHp$IzL8GE%1Zt%R=y6AQW5@wCnc@ilP6R0#|iB)ciN}
zCW2#nEl^{s^fUB7q1unvd+IMk-M>e#=NAHHCKgzIKKdlyME@V;-e>s=@XsmyV*WpV
z9Dj}<%<tx1`K7!ObOcx~m{)PO(D_24YWzQ6T@0Cc3U`cq0onL<Amg0iCUZNuSZ*a3
z!p-Eoxlx=gH<&Yk99;l?f?ig{HM10C>MAyutzc7ub@ZA2#J*vlvQL0?bc;O!K7BiT
zgk1`LeG<Etjbax-|6mGqUq`S4Hk55=e1K_W&89Lk_8qW=?gPi@B2x<<{x>j&(wTM4
zH)av!`qP-_%oye&GYtHH2y`3#nX`;5(2Q)DL(pf~%SfRPK!H%i(d!uv^a(ql7H^=J
zFy-_@CZC=QUEG;WGChU)M2~0Q(qotx^avmm$1=C+80HGp?`I&69HXO{19UjE3%b7B
zpswFU2QcetA7&Np$t;0de*x{r%%NSGnJ}FOHi`CN#?oHQNQghtFdYJ~!(sVg+68h`
zXC{z#V*1gJVD`Y0vxV2zU{=h4|4nURnH{|6zy!c&17N*=u#Nt-A58nh>mbIT4h9<p
z^C7T&FsvI2?}dSH4rjdR2*v}<6U>K>gwI66`%%!B7|I00F$L43m_cx?A<&lyg=3GP
zXG8CF4l^9iMJzoZeEUMMrC`h9^(wH{Obkq;;k*rn<q`BEIF9*n9P?nib71?kV87ED
zZ+bH9dpy`5V54FGqu_gtWZc1A;hZ}Ee{+GU>;IOGV%*?!?r_dMU>#3-EKJA4vWf87
z$&4fXP5#eZ=t;1A0vyX&IOad#ILE><PlR(d6^>;l9K#&=PV?X#FM#7%1m}4PEME%W
zYbkt>MX;}Vu)UdJQ(=1(89UgPHJHW!Y#d_++q8r4>PW{y|0V`L7X$0W!S@*r$2As?
zaUyJQ3Y@Q*U~}QzErerV4EtXS-(&gzED+935Lhr+2v{h+k_m%z7D=y#^SYiHMQ?;_
zVk<M7-o-4X4>0SX544p&2iMaLh@tnHee^T9zTPkg=|s5JzA&fhWaa|>gSkehGq>nG
z<^j|+Pw8s7J{y@YbQ_aOcQKjJ<IaUU4GL+3Ev60FN?O4-(3WfmZO>|HH<qLQSd1CS
zN|`X$gc-`(Fe6wGW<2zlW&(wK89R~L#Li>(v#Xd>>}KW~FuNWDuj@T~n)$(AW%AgE
zOa=R%X=KwNc9$@E;EWk_8pfFuu|b>-aJBle6S!D*KJX0Jb1T^c+z#MqonW7E|FYk>
zS8P6)#@2IHY#-Oh5)i+wcpJ{2_v420k<efNgImbY0p{Uai0ON|v;0}^F7&5f^9fus
zWQO@bT&sspfR-2WGQAz|s2>1C(NTPyegQDEw)5-tXZR!f$Iz=v;@|5l_-s7|WFBka
zWku+ff_dNx_kc&dqF*U^tv@Enfu2<d#Cb!=VEqKFV6?y*Vtj~jw_vL9g5Xc#Gr@6T
zs^FHeM(|e12!07Ig;m0Vz|5K?ltO>r895~kMV<(!AgRK2NG;G(S;(Btkpv_F$wo#X
zb;v?QgX~5m=vBlXGVUO>1Q~-;K)<tuPJIY^4SM$P(5=vYxeSpb0mUKGIASz(XRNT9
zSTME~V*GjNWW9hMRyuUC8nEw}9;?M{pw=6N+vDT$2#D%)Agb@eFX3147x)`I9nXe1
z-U?A1Crp7l=t0Clyj@02B90Slfx&g0$cLz_gRZs>JhmSpLPWDgj-nkvQobyjBYG#=
zCMp!25%mC5*+~3V<RdN;#fsIU`C?4GQ*151Aodr>i-(IdfvemiUIU!2gJL_$RdKN7
zrFg6)O}t!EEj}pe6W^AIB~W2WDkc6BQZiIxBAq7jkgk?QN)JdTOD{@RNFPgfOA{q$
zr5Tb3(n`rks1I|cdPyVHhLqGnY6P_&kdmcgvXRmWKs;G4TO!>j+bq2#J0yK3yChAL
zJ&_j5zDQeSxl&$MBQ=w&rM_}jIz%p)O_1Bl=E{9#E9IfUSB{e%kWZAImd}=5lP{G$
zl&_V&k#7dZ@=jT<{D7=RehkKk&&oLYRhh)#u1sn06j;P>Wgam8>u-=E8({EL7Hp6X
z<Gh)`*vbM{Sf(t(AOpC|X|h4U<LYnlMdkzJ!|nzzU<~+y%+BBjj4z*)85taxi469^
znDb^C18ipv&{{g>Gl1tj0rojkRw^F?<HA9*pK=dbBJjx`$&F+e<S2}5>!h3H&A{v`
zkj{~Rk&c%?0;cgLX^{Mw)J?uyY9ZesmB<&v`0x~Ihb&rJB^w~kk-11eL+$$?)V$|p
zYTy}{Nao6tB$05ZqMhs?2_xMqDU;5WBuc|1*QHjHeNvrxvGliig!HXAKzd1RCEYF-
zOXomMJ51ap@fPPwEX406BB*I~qP>y^(Gp3PXpH2QsGsD5$V9S3q!!N;rHf-l55(T0
z17ewIu2@A35~mS{;s->d=n(N)w1~JWiXyf{Ejv@xjfaXdaC6ZkybJQOG-4Tki-^T{
z5H9#Mg2#f063m2njRET$OUGAX_wf<fe%u3_4NT)fz%(|+Q_&9WKGd;)p?9(Q=w2)W
z$l|u>0E|VXSOrppenQ>>L-HiL4OxUvhEB0RVu}(-D^e}|hI|lSK+XwQBWr}w$Oy=(
zoZ&y06=VvF1rMMzy<d1%Fh{sfFjzQBU<TcpE*O<d7u5eR^WP(Qq@M}-(jdVKU`EF2
zyY;U6Ea-ke(%16`fXp{be-FIQLEsoK=3Dt_p!vD-w|NY5q&lGgeS`eyCa}KtaoyYk
zE{_||#dB`lX^!C5K;AQ&OJ_Z~=Pb^hW$W3^Kzp1HBPD~`ZLAeL5$J|~Y&mrMKLd5~
z7Lx)M>@CP-b~Dk;48{$FltnNW&<f+qnRFrjf=-0-jyv=Y`WQW*UQdstr_uvpJj4OU
zLgciFCMgxwKxI<d)Kls`^*41BI+lm23Gh3Zno8MHQ4~sfQSGEDRZMbZ3fV%wgx=*%
zm^(qfCbyFJ$pt_^9{vB=It!>Oy6EjsUASO3b|EG<2DaGU-HP3c-G$iQo!AOkV1V7-
zU9XLS$b~yIXTIOP@B9B^t#7dow<gb=v*X#%-Y0oMasVrf<xL)zoGQ5oG3Hju9~?o+
z_Z$JqaSp%aBgBKZI=qsX5>uW5rWr!)c!1+6YmeP>baY&Bv~ol{8aW~zwH${Wl^q9&
zC+~L@ckFZc(egX?I&#u7(d>LSllD6jk`6e&B<*v&P1@~vk+j9}Bx#-FQPK*>!=$CW
zE_FQS{nMn?j^{}m950i$vfVDn2lo4%6y^AubjtA!obfyB0mmoZb^J<t!110q{wBQu
zUw!A$lfF1)$1jJMBgv7`VQ_9OIlm)Kav@^lMIFAxvr8pcCYD`?Sau6X^W-j$F3Eiz
z1Cob1#w1U0%t)T)SeU%Xu_k$qV<-0<nY`B#mmKN1oqWdeJoyH(>}QUI<S!13KFW|{
zOZH94kX$h(5Ap0`$?d@@`=-=S4k4yJ8-IUgN=WifFo2^etCNW!B|l0z2~Kb=*+_Yo
zoZk5@xu8=|4sfPQsmFT59h@bJLsd(e<ZOta-a2IyvF=08{wb%OV^Xd=XQw=Nt|HbH
zp7O(aD#di(A@=<-C9kfhl-1K=s}{fp3&753;GBaUx=tVIJgA2{&tpG4#%}mdd`&X0
zIx`xNoCS^d&a%cYe09Uw+_3B2jLgKC3g{D!(t4;-Q(s}U)VCVFiItDhV~u(G9b=RJ
zhV?ps8JBg5RqQgFzjZ%TBkq*dXkq#pearwD6t#^|vl;roGppY9Ck8dvIBU)^uA3{2
z$L3b!1F@+;X0#!RD@H2u#K<B(8+nDpC@vJUvPfe#5ZTQ3;I6$yY2sAXiM=<$F6d+~
z5ChEBVw|~6gqr)sD%M8cihZyT-#^NHE>4-B#6|PBxMijgv6QV>rYBf!YAfE%XeF9H
zmSN_#Y^=ALj<|hxQPC<WYFZ^lL)KAlX*Ce-toF?CzIa8WS%Y&rG5v*twar<FbBEYu
z9bld1C~??2DNcYP#9HUT60V4A)@@cge<W@bC%X&E_0al9Jh8rs=R{RsSvpvQV*O)z
zTHh=$q=^@vnB8)^o41u>rR8%bq9H!CoV2`_1H>%ZD$H-i*scVt)|Rw{RmPH~0N5;5
zEIX^Lr;_Se>5xttq{dbzsRgYat&5dW>c(qVaE;DHSvt}>x!-5xa~i3G<t?>h|HhUF
z`>Iksj@y9qHMV{b*ZV|!XEn86(_UH4X>Dko`P`FLbNg8ztU;_lI@tPd4YYppIl&6H
z5{cU+TD@pJxVJ9+){c91?c30LY}K)DTeYmKR!!?1Ee6l?sN4IjX&tnJSb?{mwa;o~
z?dAM?+^r$6b@{F~zf~tPRGHTb{9ex5ZI!mR(+?Zzk5%-|Vk^HDW@WQxS*fk5po<f%
zBr(d07Xz%1q8rbkHP50U&!o0>PgLRgl&6&<!d={YD2nh~Ve2igpG7h2hw$e)m9hRX
z7JiGW9HSQB*Rei`y4E||8{WU-I4?zY_6uZP)(RZA9LFh3D}(1;ir-4GZ4v7e?F+~J
zB1+RLSYJ8!cTvNA%y<#RXV-D-uze8w*0gSLzKfy~sAxIHR0->tD8%~Rxvj$@t91}>
ze4j{-NAAU|C!am72;LvZ_KiXNoE90aQ*3t<Hd7R~@F7u<Z40u3aRF;PpErp-)*6w+
zS|T!|t-P%<!Vvw%PtjStV^!m)qNTV;yCGVM%e3>fSU#Vjodu7L6K&CK?Zp+*LEPl^
z4&OZxjbR7WVBaz{Uq0s)S4C#Q)rbSaF1E2U@;YI%Ub$f|7M3}W&!M!re78^}VbA|I
zSBjtJD)F87U-<5W8HP<iM?5oUh)3pBanGE{ippa_7Dj{NjS#1q_h-QP;>_OSg4vyU
z-9?<I#qvJJ>?w|!eZ(Q!9^P*^yNZowN3q6iBbJ*j#UisQYx6b~(}^5TFl&lYW>x%(
zN@9>%o>p26G>a1h_Y=WpLD9=BAiC4Kn)yXHS`S)pKKC^XA!~|?!KOd^mKMXza=Zqx
ze+Blf#J+)SSB3qn@>(4XvO4=z=l5!?4_sZ0$FG@T))VuHwJ$e2uzq_lu5+;1Ylesj
z?)N13f1bX$38(0(xkdbAhO@HtVc|4muqZCmulI$Icr6NwudG0wB&rF8l}6KAoq6Vi
z8EaES0M9pw=iP?*Q$OpHn8<isgzVXD3315EV4blFFbB(8Z<&XGt(MIFo>q2ggjEDi
zQblQzRgaai+kx2kkz!dx_z}oaJnIN6(k3aZ6d@ItE=#ppS-Ue9(r8JQ=Sx2FW~r1M
zDK#Jx(35EIc-fK`$+_e$a%DM6Zo?|w!{pcUd^wr5g)=A@<ziT4L9FWCT}iD>RPrh-
zm2%1f{G&Lft8!l%rF>Arlq6+~;*F1#M}44_Q-3NARJ+=hsPss+3M+=UgX1)s2=y}c
zx_Vgspx#tX^|P8yQ?)W$9<8xfS?i;<Wi9ei+A?j4wg+T1mT3PoP|-yG(%Mpkgcc#D
zUf0&#*25MI5<1zo5FB(n(e)T&W{++6S=IeLE4U}x3VL|5f_hForjlTswODVxEm-Ct
z7)~=iu7Yg7@Hpi0%j2Af&GP|k8GrEb_e}Jt<*9kL^~~fs5MOGVXDQE>@SXN}HiLK6
z9S+iP&zGLliL5R4G(0yGU)$%I!+y-Os6EcJB8=O5_9ve0?C(7L+P`^**b}j-P0z)4
zyM2Q_vwfdEA5pjBtce_8e{Qc~|7ve&Pqw#(iQB`T)oXygkk?4!aTDy-y{6mid(E{s
z_gZXk$GXW~yf)f<dTp}@d+ov}+if368^Zgc?zWrPa6Y@*F8g4w?e;-lTi9-cy|>pI
z_FGPyZ|~(bgErRQ-D{}5Bj;-A6>M+d)yrPPtBbv&S6h22ug09erahNed3z?WqV`l?
zx$y@x*i-B_`wx4H=NtQX&%5?lo^kfuo=5N{x7rVRF12s=oC2R~kbN@uJis%^-rBRA
zy_%<=-4DK3CQrLv_Du3j@Oba}-s6tvJ&&`Vu^tCK_j_#cT;~z$8RjwEbG%1K&wlXB
zS`xtw^vLB|fal=t@g2PQp2rK@36Bf5P4F#dfddZ!18(Ng(^kQwzAe8;37%~x+i#HZ
z2exn8QQK2(jqN-baJbgbHeYLI8={rAwPd8YqN=%B&tBFJYj4$6TC6%=+Y0(V8zg?9
znn7#M>epq}J8CBNAgq*m>O;Kw<4R3D{Cw&pu-z{B9#xc^N={`zXuv#1{s841ESDW}
zDP<0<zW(xGxtaVxt{@+l^RSM+O&$bitFe4bDj^@1(%_T)V3ysI+QXNqA}y2hNg+fa
zT)eFs#MKH($I#vz(9|=mJJtaBTP>`GtYbge%7wm`tx{MW8Lg`#Q5+Nx#Y*C9lf+su
z+)1K|=ph0`U6EH5f#H)%{59jv*YI4gva<PMbCtQ;oM=updl74E#Cr8*%_>CN{7e;2
z*EhptW%aKhw2zEk#yMjNF}LxsV0(ejHa5nv?s{J%uhEX!TRkI5FK@il3mCVEzeVdZ
zI8g$)><4h!NBRU-pC6*1gde+KZ_bMI)nQ?j)K}<v^@aE}GjvTKqbGrneF3w33C4E^
z?Ck>f>k07517Moloh9^jV5H07P|b%EHp`hopX^Mdhd90UAx=A3s}1{B1!I-;mQK^z
z1m0Cc=U+Ht32-@n(tgn5opotJysyo7weh5a_&vy(#CyY8)2V>V*}(}@)6#&`r2)@O
z3$B+Q{4OK!GlL^$1@H6$zs;jJ1%GV?_Sy;DH5i<A7&z)Uu)!(diL<~07l4nhaE>Gn
zHje9`29Ir?e#E&FJbANz)wx%H=sc>wamI4bH}pTwr^F{e(iaJO0o~Hefo<1@VcbG5
z2o_h;7^(+?!!-bdYj14S`x^W75HPq|#s%#12l`Iqy&h#G=;saDxNoF4-oXP)Fv=OK
z*~rLb_A&~B!3CI0j7H`zaJgvg^1H@7ywMG&VH_~inz3d+^8wHOGjY}wv!m&azRM{l
zgUc;Me{Dpc9x{W`Q=`#Qv(Vow&`rBU4silI;3_u3Gtpao6XW2G&BJC{hi$zVJ1Isq
z!M^E&tuqiCXq;t$&85M1%k8$~4r2$N!cM$ub%%{O0=se+_U>BCi-?5}EXBgobE}f{
z(`txk-UVGWOe!i(mjam;jis$pC+UzhKsqH&z-yl`J(e~||G;znBb_E*cT-9)zm#&z
z@lr8#Qw2Gr93=b7Ex_)&$@SzRa$9+#94v>*qvfUY40%2CaF@JBj(|gXhIx5KJ}uuR
z*8L2&+CT7`e#`e{r~Fu!lvlE>eB$+&Y@yv0#gaV~o01B>mQKk8Kgx$_UoIt|l3(#t
z3d7_mO{)U4t*%1OnNmh+sZ=5kSOdLRN9m)~R|dj(8^SL`l~(Y2+w-}z(x28>>7WEF
zZP}(3ug&?~l-5}3jn?eW(K;!$`L%}9TB*tw6=`LZ7D{oB<fqhA3h+x_Vvf0$@=A8D
zAq&x_^h!a+i))jVjBvQTiStX!8`&X0BF1!!*#1TNEqDC{{e45;E62)P<w$uo`g@VQ
zQJyU?lPAzy!{n)QZ{Br~$H-0Pk?x+XDfgAD$X)5#ma@NGmtL+Y=ah@k<5^`dS(Q!X
z*-xI%Yw4VHljn0%S}X06LZzk1waHR%X^_;CCs~UpS{iwmA8R9{l!NE(E&T<v{cOcs
zFY!I@TKBB;M6DvN)7CcrF9nI2Zmnm`EMV+R#%~yokI)M{r?r(H&%wkV{|=k=1-9}H
z;Si_AN37F_Vk7p{LU9s#xL*tso3U$`A`@p}SB=At9VBd6#=o%_Ut-@~#U_k0Gm1^7
zg6;O(9A~~V2f#|`Y92FN!hWrX%&bPFD*zs>Khds&=4dm!IgBXRAXA1Hn`Cq~zZ>n$
zcSdtE2^xTT*D_BUmB}b5Wo|JFm@ADeaEt8bMC9ur<B!o98QaWwYt%8G7}bp1Mu2hI
z@JHU}Co-1VIBIwr2Mk@`4u5v7@sGaLcmZ$yfj$*k8)9752NMemHqOImkJWn`r)bA%
z5qb~!q1|a+dEeED)Vml_ydK53KhAGN>XFmIMvUIyILR@N>cfm9`Uv9yuLtn`5Axk%
zUL*A3w2|yTnt0n7_8DiK)h8L}^l5M{W^wK?Fn}erRb0zPu4$|BNZ)}j*k$}6c9256
zUolP^sf`PU5BFBccxIF_J{eWWst7U^vx$-3>;Pvq*eGU>LQYLLYMYC}5H`|>`;39~
z+gLdGv&?(OQuC#;$^2sMHUApNOq&^JW-)J~sh*h?%ui+=^DmJ{4I4Z&*&M!N0e1dY
zZ2u_a*G1&lef)}d_#}VK4Ls`uB0D3a1ino*@e&RBLv%rY4Haqd`Eyz;L{V!ue%VQ~
zMXriY*wlmZ2`5<&G2ik)N@c?zEsQT)(Yk;S`@m`oFE$u|cnm)JTq`{pD0$tH*Nd<n
zUsz40M60`$S{f-8l4eUam<3qejQ(@d6zPeym|3z-Qe-$Z@CpjZPoy$%3u?=Mq&Bk5
z49g^kFsG)=HRXA7J9z~*!*=H4QF*a^P2McOfp_^=jzJb(le54($ftb4=1!K&DIV~~
zGbxRb2JMjneUt!YxKc}*fSj0#44IF-S*A=<)+vjW9m*!6(Fc{Y%4y{`_RCA<HLUYr
zN;0ek8{FHBYF_^Q)#7R(OojStklI>pu69>DsYBG>>O^(08m5M*E7j@hW_7U|PHZDW
z-KCyU4`HhxQ=h}q{iI%16VwM-?yppD?K3g+1Tqy2I7yzwIx=dtvAddSMTmTq(R#sd
z>kG?afEEPDt*$mgt4}nf0TzBEZ3<gWA~G^TYY!WulQxbvLF>Txt;u$11|Om!pX<Z@
zt*;HI4b$rIS{>h@65o~4Ldb~tx0T?S#W_Y%j$4Fd7S$$e#n`_X$N9HS=DV@9;ru>G
z^V5R00$L9(m)23ssx{Zrb7Z?#QM0&*Kk)cJsXp2hH67U(9@<&%GYV$%KJ^PM<=5(3
z^(p;wAJ+11dh4n>N<B}XomTs*C)A#5q}oN@ueMjW)2nOMX6hof9z9tFpU0mq^3uy0
z>1_|SuIf|*@s3KWAK(%^1M9w{*wl+kiW05FD~FV~JjVyhdgYR`Scy?)C=p5soZbG)
zMs!LTI%EhLB+b!B<rx+E6cg|48_bpGauPb=lY9;xcMR<pE=S92VQYuUn~?U4<e~B;
zxfi_cc5+9#A-O44<m$-!a&kV#WEMHM?2WXyBu_XBDbjc8tMpEKip0MmU1xNkk)ov|
z{Oyo-GUiuHi{JpyK#PZ<w+Be0q;C9gCv}%vNNuG?QWGhNziLuNse)7(jgSS6uVMoy
zf?s{Y@_lNV)(x!R)98f5__W)tSU3z()*|qsS!jj{d>W2U2u3?}u*O+UtYKhI{m>0v
zEq`=F0T8Jyc)>O+(E1DB@`XA6TI2=G$%LM;6EhU#J^aP8`7YLo_hJEd%}j9{Cd5TC
z8jNEwzxE+NqAT`wdvr)kv0F3(`L7A)5rDm2Tr2?-nJ4nGR~|6|`)CB%$Uu=A=7fzs
z710Y!q$gNNH~jlfSmEtOyxC5?H`|IQ#6E9<5nMq3oC2#jZdOEYl_tli7`9hou^${_
z517O*GdKLE9LO~vvevS>+rRI9&{aNS1K({hvtqMl;j^n{19Qj$CX!36WZ&hqrL-kx
z9`KQT<S`Y1&su<JMLz7k++qsx*l8SRhP!14JITf8JnlAw-$VJlfVRZU%dvdLYOs;@
z*oj-svSK&ad;r`d!VH3?U0<9q8-s5&qqP>%U>`ByGvvi%NA^I!_TYYdA(Q)a??b_W
zLSX+*qCaLJpXZ8XbCD3(nI2+2tdngbiwLJb4~pD8gZ$!@C<I<r4D8Au467uVR2iOC
z1z0OJ#4GI4x9IkN=<#pluly0?z`dq|oy`UZTfp;P4sNm*8(}k;;7;t+ec*i&$oZ3^
znH4Kqf#<dZGwg)j+7p>S5L|E+HpOJ*zRP}H3Rb(GnDS2W$AfMi;Ex?!27I%Ebp_1w
zCV1+7M%z<H-7BjvdSR&b#R{>0VW~K*nT+B2V7SW}&l}MbyBXVutlTi-3*y}slcKG%
zQk+#;x{ls>Xw{NlT6Kv^HD>&`!3*pn{RXFTS_33a8jL;}E@faYWtJwv{FyH0lIBQ$
zQW!I5q1$_`%baV*+-nCbsH=2L>LbOVbIwR3q*!U3bWxhd91WH3vh^dZ>X*_gayQmW
z-=vLF0v2{MmYOc@CoUc#3F!oO^=ZjNJ}0G;uV7!_B5L+P$}T^b^2l$=_4q6mmVaVz
zCz6HffEntLs>n&Ol;9?j|4|1PQ%x+xs&Y}JM>)9x9&meFU%9+If)>L25V;Z-c_6Kd
zJXWqIk7JLq*pwl16TWN3YdhpodwCib=xk)sTp13k+ykVk4;dx>X#KI|`y;XXx!V$6
zm!T_{afam_X*qIig**kjc8a_K{ka8=x{V`kL(*+=OS`#9yxC~fnY0<S>3p6>o9vc^
zo4KOR9A_hK9app(i+F`R03V_c=k3k)^kdIKY(0$Y9M0B5X@h9}*rN|tb1!+6+#Sog
z6Ti0N*GBwWi(dn|ivaGbjNBZ_S|7Pu3+7i<FttkP{&HB-C6Ta2=+8Xz4=J<!32FO=
z=kgSP=e`sVUhq}ADt(kLU}vAfo;WVu!#ldolfTLHyvp-EhtxeTjpb<$!qVu$w{4{*
zSlzRcx8w1t1~X=QVw<!lOQR)B-X>BSsgC5uFPc<}+#Wy4VdVoC%q_hGJA6WX;TGff
zBAH~Th)+a;`|rnE-_3a6X+5;ITDO=XSD7i78SfVv`*FkzP7*^nfLz{6jA12Gcs`i^
z49gEWoeKmb3(<r0SW(^}8466{WH8Mi;N~B}&tHRaJ_e(|1+INT_%Kh?fscD*pLsBc
zWoxxaA!F^Qm@Ph#HS!!eeP47(Hnc`=H$-mN5QjuXu^l<FS`@|uC@AJ2qo*OGC&D})
ziyb(c43m-IA|sH|!;sB`MS|H^d__jRMMghEF5f2>bql$6!>l2$gZEwq|GkLJi$zAC
zME*sQJ$w*Zy$4yn6W?JIGIKRP^b+K6820E)GXt_UH8`|~2r(tFVI6NF#T?4(VAF&v
zXX0NNcn?m#bCTVnn`8NIf+@_&re#iJpBXUjW}2R~)U=FZHpiZeyk5w8meE#m&UMK0
zO<>yFxt87Jc<sZNIfzU?V%7yKZ-NYN$u+m-+B=AAW@qlH2U#_}xySz8>k#qT90mS7
z0o^o<epm>OzXE-=PDuDmHgsJYWN|k7GC#Vn7&de{P^Lg+aV_*@19wc}t>`2^g9rRV
z^QF+6k~Iq1I}W>g8t7D*$cFsQ4F|Unve%!OMFlIGxXL9Mi1(1cFGL5Pc^~U1eBER)
z1Buv<r!@z;yAUjUwUrq?pN$w%PR3GxS`lP$8LWXoZ1EuEaAWN8*4A}mR(FuS5AhM7
zBX>U#Df($GNA_+cZWa!{5MlXBXRK1vH7gLzyPovcY7T3%0~ty^8S{gXt7D`*%$g$1
zo|4iU817rJO2Uz=hrq*+Gk4BPZD3q>L52n+Lx*DXjF(=KXZaCX`a{|*Ik50$IT}>&
z0&?=E<SRdt{F%2ELHufBEj2_gwv;`P25FECS+TJ4AsdRx<C*c(k$+)WTg#An>yRy5
zkQlp=DhH8E$B{JWkvKQ`^`X2{evY;F28s3_>GTn4_LVJvV!!<b(a`0SvcR90lq<Bm
z;1|!Z_}<H2<lUtQ$H<PKP(V@f4$_k2m0bx0C#VgQ&<I4Kl~O_J$T!{aBKq*{KoE<e
zpb#UmAxHD;7^RIe793(c-o*s`i3wm6lfWycD1)#)hmzAbOqr*QP!@pkFT@gE#2$-y
zUCeij_<aGG$6Ss&3k!9cd%vErrMhyA4jijBTQ_0LAgtEPWCfN|dVp@UQSyL+_<(z4
zB8rgSecc7P_FTO80qMvJ0+Wqxa)80)r4{7AA4o_czA4E4<)!6j%Uo=m6VxU<s7N;a
zq%7b#85NCh6eW#ffsp93Cou~}Nu>XNf&YBRO8zV-(91vMm-O)?`2*O{EBQKB^JU(h
z!*Y(6<G_z%=>KDAj0ljW{oqQw&^(*aPOIcu@)ESeJT$^=G{H2Gs0nDm5V<qDpat*h
zqS2~=q`35>zg$BujQr0hmzA?ID$<f)=OO1H!!8rLAT>-D8@km(131wDNz!NDy(dQV
z0vr6k^a-i|mJ#~QtqUUfbV!QB3OL1>jF$HCZ8-M84zf13NKw){UYGEBrWA>XdO+&M
zx2?fw8emh@;+sH5czGf~{*3s-Qgd*cMpAC{gO5}d+oL?0#ig-DN}@0PB~Pgsh;?D+
zkT14-0ocg-CBw=uCE*YMvT{kEnR{>G;yogEeG{zDr9(~=-#$wG_z<z<{nkgU`8Tv@
zv`0jO@6+z_{XJ&;eO@2Af7=Bzv<-c;k+zmOzKZXcffOyqLzsu&nT^hwf;<>c2Fhsb
zgf)sYjN%NVkR2ne9ps~IM?UPZy7Ah<+KlE|gWOmS9=QbjZ3%YVQuNbOt2o|8A?&{V
zNRiy^?Sm%EZ1u6yp|8BK@D%veI<dDT(SjIqQ*>5iVs8!cE$Sl2YQh+*LhQH#`l~Fl
z=aSY<;?LKxKQAKxPNUBvh|%sQ<8dpv>1O2QCShUmI`Jl)=rmn~Awy^3lT1OLjur7}
zsBa)upTz)d+=0m3LFl?6#Ab(xyXd`pygoqYJ`$s_iN~S~Ct(jy$Cn8eU+`ys;h{J<
zu0RW_)_h^J=DC|{&E>qaINx-UjAnFTFDKJd+-)-7Pj=hTLd-y(&q3#g5r>?|G3E)4
zeLeWjWnUwyX>(~|_)~My(Q}Aj&qnsoCT1Bb((^f^+YX<F9+*uFC4N0mWU&_Vx)|NQ
zn7&v_%z6c{t9e~d|7;?ry_G)OPT%dIQD=)jD@iM3h10kDMP>Ri5W67IIxDKe(5hiw
zCdPi9XK`D!wC;<x_;?+723@SD@H(D}{^*$@JeT2c0EQEfA5L6;Bsymdw*Dl1!)f3=
zGtfUE(r`Z<BFst@3wgea;0L%~=kYm|?OeajWt;i<2aAZnEJIhVgpaTWUA3OEvf0YV
zC@o^`g14{_J#~a^`{QsLqVY>lfg7E{BaMUqa2c!THnG`<WaK}CL-7(^^EHU(d&cNz
z#_D%)rJszz-`G=$XhSC>(uFg5!cy}l(<(J_+_cgPxF8!~tnI``J80#Rj^k61v5k%^
zBi+TPcx2U<o>`5g*H$ZhijLAp{E08L?^ajlW;Za`o{~=1fMpFL{yUsG9Kw8_Af*DU
z%?MJJ9eI!!o6#39z9<r-w6sF1gjZifT8}i?iY>W^_lMAtk;sfFJoyN|J0vwkrZl6q
zBC6Gv-#UQ<_dq)J!v`6RY#IiJKNLMW7<+ZFbcfIPiGn@AN4bY*bBD9vk*4z93@qL`
z_$dppbC;oO*U&c6cEI1=3+`|PxpxxX8;3l+jD36q{d*Um<sm-H6MUAZ_$*I}Uq5sE
zE!pK)yuP8mgZKN6c)=SfzuT92hpzqy9PSIB<MC}0@NJTa&?QTyWhcIkAq9YJS40k1
z2A``WYoOkCJebs2@fmoZ8QVPzHhfmO4y`_)>+`*<)kS&-$=UfW5Ar=9T0S5CP#!+#
z;`<zIpPg49KKtPPWT)lg_k4H=1v#z@T`$P{{Op&PmYesvIA<<&#J{aPK2`;eQ;~fu
zaE$VtzZ~Z*!?{b#>8KNt245nzT!L1V=EvuPa(a%Kkt1fJW#)ZmzRN5ZqPY;BqP!O6
zwHU`L#xYB94u7uMpX>5xYkxG8tCi$)DXys$=Pt>%{_Nw=_9g$@zBs=X<+md4<E592
zb3OiCmp`9f$1cqFzI^uOnhJp`xtgzA?`7iX+4!8D{r^388O~ddUjw*<inPl7T9wZ=
z+^rCO=7%Qr<=GUZAM&wX9-6CfeAqrSEj_=b;WagV<jpmEaXp?~n<snPIhq%HdUM`X
zy#CuyE*<Q;=HgsqG4?OY{>9k7ICfky?771DX1?-QDIfk>PR37G`2%C_9lH9x<jwn3
ze3ypTG>oydWb?RUR2lG}UE?w>tW<CLK7QRT$pZ{nlVc@CJ|&rG>?C4ZznCrGY2WbR
zzoEV3nMFU)=ikxepO|^?(BUrs{W&x88FS+?nDRa1VmFvCmzg)`m_ujr|IRS0Pl35b
zqvMaF(GM{ncQZFPq3c&;*Dhi1&PSuqL6grwi%(|0j|E#8fsP-Deh)?_^gyR~LPoSg
zuQ$P7Za~&&17t`;^5yCyTWTR=DoGEJIXA2l(mCYM2_n~r@gMhDIlwzINZZi+>yTf|
z(EIb@q=bU$%|!1{LGF!5298D+4z;c!`!1rNW08q5M7ob5Cy!v`@3pF;x64|atiskh
z{D77C28-dk%&~sKefc1UBHMc+>s=U56ZCBmzC{&qodDvIW#CTv!>=j^mkOzj4qlHA
zUPP>DIyz_!e&rB!SFkWeH<18_^a+gbB|hXM{F}R?lDI~!=)5Q@qQRez-~)x@LvF!;
zTnEm#5}axo(e=eb1p_q9nN&=f1kY(KxYj6qt6}05e&q|ZpLh-i_yi8q12Dilv>W(g
zS7;aDE}a1nJc^IDAMR2(KId+H&Tx1{;btYUzw-EXWoc#jz6^d{SrG{z=_o#46u(E}
zmqxN}B=}(jaofXYZMLZmCRdx|)y5aB#c^umpVkILti^F^am*TgkB7{6k(}=sc-;x`
zx)`qG6h7i<uIn^-<!P?@6nNb!{KYfic(HCCmWz+on``ffPdrfE=Kdal<voSB<-+qm
zz~%a8P6N-ILw{gt&^M{+rwl~mvWcBwdHcajBZ$Eq13!%hM~wqdy$(0+AsFjx5e(;P
zAZ&&q*ek=aQ%1vkbNP`Ic}`QXm!^T+x@?Fr>;%^{Tn4tcnrFI^XS|c=eSqg4g>T7`
z;nxMh)og%W(2OzB2JT&Z^nXV;SMa9@#(o=y4j*UPtZ9s=Q1FO_;1Wy0j8~(-H^Ds&
zM{h@hTb#l^JkNN%isg3)OYkWu^E<HTZ;atTpwCW5q(b~BwX_voyc<1ym{@iUxhI#&
zjJ|7CB{MIG*mfhZy_Vo99nisD!S#AdDPVgRxhWpl+^MCB=-e6LF`?ix^Zvv1mZN9a
zOHD!3T?~=lXuDy|?MZGN?*X&-HOS6q<iRiWVKN$6lFv$BR0PU|@9ENSh0$qcs69{x
ze69{UtSSCP8)TTv2knOZ?}d!*ixwD&)EkB*9D{_LjOLhyv<yQ#EyPb*!qzLmvDSiG
zZI*lD-}J#R9YE}OgnR;|D~3Hzv&T7no~w90cko^x%KPM}M66!mA-yBU{FPYqUwkPU
zA2bb2nQZuBd5NGGbw{mTK4@Wgq=FUP@GAX@e3!uot$>$Ul}LDPS_6E-rg)F7h-r1e
zS9JM~gYYHC;&aYamf~fuA+oiVh{!(JX2+B$_-4^$UB)SQsigHtxuHBIOXnrAu6N2+
z<ufsp@3eT{$Mf4aqF`T?d(_){NVet++B@EV;=8Z>{#E%+OJa`{Vq{JwQ85&UB8ZkL
zYBDS!2T>VUOX7VJ5wt`Kf&F2d-)!}ht-mP=obeY&{Y_@D>ouPC1t#1FVmR-Jy}jeg
z-Yd^}eWtvjJ*Pe5|9#p$<q5A3_`6NJ?!KSv$_3>-_ZCA0?<oHxxZlHYkPc8)CtO*}
z|K;vD-!eSzFue2Wc-a&2#Y5=5Vf11@dbm4LqPBS0P3if1`1v*Q_XCL+mWRVo8gIKe
z-nJicBiD1vrg-9YJ7t>^FPl8kKd>dfz>|2*vwa9Uaf4BC5fAvZyoaa01wVKd$ixE1
z%nZiQc-RCZL12e4S_UxA`hePYW9+rY|E<G#43Hb6iECpoRKzYRf!~{t@#?}o?dVDq
zEAcm$<3H%dr|7}kSe;kEtk2=Gp29kfLTl~Aa^1q*Sc#1}k0{eLFz|6inMUKej=&}z
zf}RVO%DH{oR@kzQm}7PERBK`5Rz=rUl<fSf;M1DuuVn1#1hCI9_^z+PJ|ALh-=qf8
z6*vkPne(yO=f}bL_rrVM0?%?4GGQ?~ER0`gV&hK+`y7k@8VOQ52p!gkExUl0wr1<b
z_zFR6T@{Ztz?y`gFd9sB2)<@N;taj;C%RfrWL3OqjxSjk-z5;eRvO=?2v}%7e8_+C
z(4+WZhw;n8(Onzy+m<3X=io0+1`{1aTx2LZtS>sO2e@b#co1EXwOx?8ortHj2On*P
zEN&|HfZ>Is>voH(v<kGcVi)pxyYLg6V2`a9S<!K6@FZ2xRvrH8U-a8Ab2xFEf#9Wm
z#V7Q^2Xw^;;yfSG6JOA8-)Zsak$CjTcXY^i__{yYHUYhn2=183z6t1#U+9is=*a|*
z_lNe^90_N5G_jyDBE=lX`6hrxP9}ynjq}eGUff|iJmkz=j}M-0c65tN7w6<!^MF<Q
za^1za{!-jmIbuf@&@+{Z`BxR^z=SV~+Vn#Z{G~edNnP;Edic_f&^^uYom-11^ve?%
zAx}h4{Oker;VAs*DfDR=zVUMW-;L;_UBr|QfM-U+CqKq>iH0-48QebT6?D#Zuz;I9
zyIc66_wYfV5?6Xle!*wjFVP(zGZ?>d2)K8Mm64I+!-&a^Rg@nr!WSR1F#cmPbWbU8
z&Wenkn&_YgjHPDykZl-w9f>)02XF5O2VppT`-xy7q3EM!=%dZ#73{;#I)N^_z}USD
z=J|?R2VaRTC1SHVt=3@s-5BTn&^g1%gA4)F87CD)qm*FIxa`0x;GeaaNA+P?H6?b?
zny64mv`cra&tPWXU?M}Ku`4GNeVBtDT7=!Xl6koS{AmYu4iBJ<qQF9<nYU-LYcFE|
zx?)Gqq+D(v>!&-eQ5szoh#yrOTe|_iKoe$s3oL9`Or#gOXAqYCXgsKid^?@kPZ-}W
zMs}=1dTane+r}?@_~o!X2Yhsad<rRZ7Hsq!=;<Z2)OEOB_t@eoT#46g^<F*#w<`h+
zFAC{(oOX)u;`n?C4D}ka>=uaM1CYNLaw6X*qp=K7RSN{rrX<O!kbM~yA?E-u%nxqp
zM=L@r3YJ&|T~-(ru`rlqQ6&T1uk>iI49L-pXs|5swX)N4(el#r@m*e89)5Q*wsOJJ
z$n9?B`Mw;jEZdZIzvkid|Fm4Z_u>0YAfu@{hmC8n;C`fl=_bN7{Y7=acsL<n(XpSo
z_K$K3_wkoI{Y?-2Kwf|2{=R@Bf99S)(Od7qieJ(nkK{w>?7isj9Z2$x$naI*nTycz
zp?otPRC6MT<``^&;XH?deA^ps--W1JTaeA>SOE?AwkA@&GFC!ao>noUT?OEbQ9m8u
zs}_D#75u95*c~OXKm72i3&_8uJdB5&_(WOo*VBNP+rZ5YM%N$4&{t_Yw#`N`fVKED
zE0Oa{7<=>hWj4}(IzG%a#^qG(qVa4w0@)wT2<(VY(;RQ8E<Q~l=%znoJC7UN?*>NM
z691(Rm`7!>kJ9)rMZhNWfs6QHmu0|4^Jc!-z)oa*7!wKZU<UpME&LAN_z{Wy8j1ag
z8G8$C<`Vu&EdELib1@1$ai7(SdE1!zTL){fDi~q~W@Q;-8UAR3q9BmI%=uh!cr&wC
z8n>M92A0?X8?+U+Xk)ObI$%=OVDwfZucSOVC}pu%OXJ&=LWY#Yel20$!H>BC*YdLP
z#kMU#E=XQvP%euV@vKAOYzM*KTsb3$!08Ut4uRDjz>eOJu5jg(?BV?`bjCJx#uhNX
zjluwXw2+_*98MMA$R*<>YYX=GR=(fr-c|*J^FYtIa#B)rymTBl11%FR3(be`vU3hs
z%f{!d$i}SrT|UUhTwG&*FuFqcWQDo*qW|UNxb7){d#l2|)<ibeLEqI!J~jrAYe9ds
z17qt<-}E3J)Z5bNGcR!1wDe(Sq+U*N*#bO;Vo1NTa6c-dd8;5lt63Ysad%iX@EL3K
z^y(8QYHmf~Pafqtop3*|f!0}$avmH03Qy%G+V`$Cj^{cN9BwN9=uGtaT;fNI(C^Ff
zQ`dsOZHA}1)3SKVDsd!F<ZCKoN$J3{Gvo8-!1v7y=gSY<z64{c9C7=~@K|dw*6M(-
zHN?hmj?8U~FWs4O*d1G=H}TeB?ESv*T>Bz}`(cOt+j2<#iR%ww+z!M(83Oh;62E*L
zHp>){_L;O<?pBodMQE<q;(S+<R+iaO!7al>!R6-RJ1iz{wHzF7jno03zdI7TFSgQP
ze2J0Z0b`kM<Ct3$u%#wqQ%wRtm`rqeB5fS+Ly+&I!0JZw+bCjRW2E!Uw>Y+srJZG)
zGi-B)-(5JvX*9|yUZZIzXh*^LqPUYtcnFb1n<GFH4#P`00GhCu-UxSd6&i3|wYa`O
zW_USiE_!Gt_dAvQo`_x=Lm!L;lNf?u)E``;H?it2*lX?3T`lOt#$c5V=+k=GZFRBP
zYJ*8sXD$a&&!-3+i!5LdGTe(_Jk$4#l;=F>M~s|%Jp0?&X4l{jT?E-bL;Ub4F@b}O
zmvAuJ9oT4F7)$G|K&(45ZqUc`@eAjmr>9%lvD-2;uQOobxyGb7b+2s9X(76z-#ajd
z+cKV85)){`d~RsH#*TXm=i~u)!fkTSZh%W%1=DuTi8!$C(_r2w@M$BMHwT%=dzfR}
z@q0Hh=hic4*Dx1X5g%9qPQR4cz(P1KVd6Iy*-xyZ?_l(w!SMe9yLZ_uZ?OU25C?b-
zBk46UfmcMkUlQ$pNi5+7?K$l!{`F)0>__<9k3>IOUs@kx34O4Mf{7~xV>|Yt_5QE*
za=+el|Nieb_u20O`DG8?#~6&=1QNh;pS#aBnrj$?EjkWNWg@YNN%;1YiDgWsP3M|n
zm6IiwfF<{rdrHQVGsIf%b0acnn_Et~<V{+)oQXiL9D}17gS?AF_FcgizJ*-8j|_Q2
zpSp7O{y~1k)8C26Qv*L=<Ef@%#%Dxs<Y2bvXQmhC>H9Oo%keA&v6yNzyBjjQTQIxZ
zBQv@oFZyEH4`n=z1EZhL7?}@OeFgEU4dC)S!Q>BMV;x05oQBhL2{~{Z|KJI6uD4)b
zUo3ydU;uNzDzUI2#%B}ec3Z}AcUTqurOM2TAZATdw|_E^`LGJ@v;})=5A!a9nH$3_
zxB#|y3$6BqnerCw?<+Gd5g*9J%JKwH%!uyHO?`^O_(?9D*M$k!LaR2!Z*Rq?&Y;eH
zu$zYPZVYUbDQM_WbogSl_G)aU&Dcrd==mdf^3mx23wZH2v78=ZHNC<{_y-&4JGMgt
z|C6yG1Rfv7NU?CzV9jK~V#x*P+!wp21fF^THcVAK#vr`(hWs_>uN{Bgi0t;mhj)3K
zWAQqt;Cs#@&O1+;N_2RZvIM_$1u@|@yxYLLEy`+q_jSr1*oz17<Bw4N_$Z%F;?<tQ
zw~bXI@pzBn%b&mpK8Y_MjVF8x&-gSx`&s2N-tZH=;%9jE&)Mn)b-4a*uJ`{o*Z0q8
zPub=PM|;9ipD1T&r`i7$-gOM#b~JwWNpOPWoaZP<iDZw%%0<|+=ed?R7?x+0-E6g;
zEjIJ(dYG82`E~`@zm%(A$UTL^w4KJ?jaOz94WB~vdkj(Up`aH1@Va~8b$7t;Zi&a;
z2*0}yes^_xvLd!!DeOo;tiPOCdl|7LDcOOYCwLBjc^==f+CE^_zQAI;j~#avJN7Kx
z{U~hRy*$~iJli!yoEPJ5&%uJ8jHf+{aWIf9q#i^X+F_$L0-<LO9PspPVDMg`<c5?E
zE|)65!)|{7#(WvuD;nGD0PnUS)0YuxnTfqM8Z0W95!wM=(*#XW6S-UtWZ4&aoCTlS
zP6Wt+WAV!>fL)aZzuC?l(eYk>5)pd`vi1-;eAyb0-82|k+ttktS^>ru#+;mtA3Oqo
z$ffH$;{!J%u3F1_jxBTpIe7;8co=!G9h+noG9wI`HwjrYMBD_sJxkT9NVgvwfJIYG
zECy?wCDJ1SRidMbW-xwgXK>2q<Xr}VWxBA_5@4zY;4<P#nrYzHXyk6X_&Gnxi~7fG
z2gkgzdB?0_UNp;@(bTd!Z00m~o9U^2<!P=og)tva`V2GP7;nBeMw(BI!R8Gk*gOY^
z{Umt+hmB_DZlgZ+<ZGC#$pctol%*z4F>>Jx!gtSYPBOBaV~niSY|dy7Fw&TP$PMUD
z{!S<A9kwwHasyJRe)Si={BP=E#T!+PFI0{A$EZjZiSpzPlr~-&C8?%W3~qg4atI0;
z4~>Gf{A6$C`>*BYa~|@za`XCs+9UF}p0M3BqX7H+8n4;^jZu_imf+Z>I7eBy`W47e
zsKR+`kXICBIJg#*Yf`DrXE(bTsmU?QKz@IAbBK}O9Kn5s80E-q2&86T4QfyaanB9R
z?c_onFxs0ZjGpw%VDlz<NKcGu^j8?U63fj*xc!C^4zK?R{TfaGo~NI0keBqxtZKeB
z>yx$C%1kB)Lm~GzjTjC8X&PLsMewgSf_?9Yb9%~b1+S?a`1&9)i1FYFp>7^&8m!mc
z;2i#L|NjPj|L6GjU(i*5(MK||@U-aGT<Dk*=#?t??TxWVJE24RV|R{4ug|vfAo)w-
z0l*doyNLtGxr^8S4l5vmv8+ma$h|)W*MV$X<bQ4ABCW9TdNCJVHrsUM{bFXt7VMqF
z$oW_-n%l%O-{9B$A`&CZam0e|6F>Tg{91=pjk(wapQ$rm(jcOJ6Y;I)<7=%$itm8~
zbOJQ(3hy4{w|(MW5^}(Xzn2;AN<n0ES$xG>N=Y(<s)Oyh*xy}{w}U_cCm@@`kg02t
ztl>zm6G*hnNX<uJjxIbhQ85)w^&-nLi&{X<N8RlbYGo=G)m7`LEvZu26~6iawI7WC
z5$ZJf?DOHZuY$F{Rb35_eG8oSaQN+q;r^ebqTxCC@7L6i)H(XaCkOn0q5eaK!*6PO
zasaYx$#CZ_>gd_D($uRC&@yRNsJ~Z>%-;I&?VG~8Zw(*6gH}@OLUr#RY~jie=%WRa
zBT$tZh^{<=n(*^$!PBn|$G<i?0kz@u2f^pB<8G74A)p2-T>m=o`CTnY8$%0WyHQ#V
zZ6wDT#xVzL<+c7?MIZ74dQ!8$o90V)L4I<J^T7AdNoGMd_R2tYZEv<#HR{l){@P#F
zk2-;QsM454d#|R^UQo%&mFI9%{i|M7zpJOzcQE{)QnUOP*%9Z}73y)aCH7Iv+?Cb5
zgr1zGcB2kdYjqHr72V<ex2DJIk^LM<|ND~(omX`#8Ps^t<u~BS_dtoyqno18ZadL8
zt3Y?>qG=|8>JCD)c15c+LzmP*la%K9=SO#BQ0!=pL~5_U!SlbtXgPsTzYBkUCBFPD
zP}xx+vOPgnnt{Rwg2VW^ZHav7g0$eSCa6w47XEV>q*t*=juDmEj;+50n`JUq_h9^t
zj@af6u)`}L^L_DqTsWzM@0@^Se@CS54m0R95`8b&=W2Yh+4yQBnN!`d%bMcf1rm!X
zf<KrQ``3nlnFz-A9y`{>Yl#&d;N90@4wnXh%#DTXjn$e=Eb1fiss|>B9=ZHTWVny1
z;HT^4vwnk5`O@rw%&w0dsX*RVVRH{TTI)@3bD^mtcfJ{;%;!d5^E!3%V~i%`W>qKW
zs0_JBE}5K_TqG}Yl1wV}{3h4+BeL%~IhA*jkC%;H#u@S~qYTa1Yb5JijCg&u@s@gN
zkI09*LCusn@}-XJ1IVH3Y3!n|+D5&Nu~KhlEFy<<j$WHw&T7V3y`nK(FJla(o?&mj
zkkM7oZ?xBQQ)eZI(UjcJhI%HWo}QjsscERclFF#2dmDkemr;pU!QJfc*8si`;P>+X
zHP?IBcNJ(Ad0(0RtFZ1ub@Fm+>6tm6k5O08#j*1l^>ts?6ew&o*Qu~bo@r;j9P0}N
z8vUpXH<W7~!!=LU+Zv(djxExAa}R^Lmk?v8KAC%)O%B;2^3GNpH|c{X`abedkCF2g
zN6yy`Lq)HoL5KJlKhP`5=qQDHuBpw&^lb<9M6gkkKCVJvH=xhkQXj52`eB55%9v_i
zHx}}2Hke<@^>P{~cxIQ#Nq%S+H9t^Q@(<5c5$(<N=z+X&YW?BBRwWmr33|M%xCD;;
z3~V~yTncY>JKWx*;MbQ$ApS@*{Ec4V9U*X$=7O25ClBNxcte~!CfElbEd+T#2VZWD
zRR|A1kTKgFyT1=Ozy!wfV(k3waQ06S7r1U!As)~S8^0&`=_oQ6=eW6|E*?xrZ1-VU
z>9gE1xdhNs531s4hlf&J{w`IAH_`&WLoYDS(XbU}GwW7io9{--pLFMSZNsKJ3Ql(g
z%<;LLUiksGCo`WjGp}8-u|O<pmrU<LBxfX+<t(h`RoJ89*sU>GsyDHET{&F8iBGA>
z^2}77E2I`yE2shP{H^v#^1eu$F=TMfRwp2p=8%81jJ&NaWNYn5R-IH|sF$crcTY`0
zX4|!|$hRb9wW5_`Wtl2k4l=R|vD!i@tvAwe81mPZpEaA@ti@zVtwmCALz?bGDn}xD
zW3(CCMJn0dqC&%C)@6824cm{}W-VUZp(RkWAz3@9nOcOV+Ky>nwrHx@p4KwhVzn&f
z>HF9&QuXe#mdAEg%Wu1;`Py!1eyo8|)OJTJZo8+Ipp~@U*Zke>9<O)#{x<6(+|-J&
zePP=b_PNNu=Q&24mYv$KSvW^}&Xvk`MDw)mqwebts^@OhlC+iDFKr<!<;>CElU4Qt
zZSjD6xQ>=Nhvqq<b*45zYt~h0#L7A~sW?$l3niy)l2(vBwe0Amv}~cGFOt<dTD)3W
zdxwsAOs?Ava^7OeeTzajgsVT1{%_T#^zIyV!g%#4`Cz-y18d0i3Z>%v7`2=ltoo{L
z&;dc@ew8JnSU~-xWTJ|os$N4!$Dpkb;O}fy&f#Mm#H(1(cnC%QkAgYd6CbP@5xc4|
zUyC9Av%+Rol~0VQ2iOU5czFl;TaR}Tiu4~X4@Ua8Mf%qSn=OGKkPW0(VU)(hGkFB7
zG#1QfFBr}WqRUf2`TJwNwB~;`Z2cl2OPN8NEO4){#2OxekDVbF77nhm6nt$W@vdNe
z>lR?3Rq(qDfsLeRRynb=J|OdN5<@wP4YrBcN+@%5q_{-<KayC~7UDJwiRDZZb}Y37
z_y{klrE(b_M5LSNo?gs{A3p}Wu$TG5Y)*b|74sxAVz-$Un_M+#nZJ<nFOe(PkT1uL
zdge}}lDQoFbcT@=J2kc0!w^Ob;}^319k%X6Y~d@&^%(N#4;p`vY5(ZUjmP>N@>(a5
z_d1Nc`##9Oj^yDtHD(~kLy+ME^%B&w&X3&oam(u}RB$V;%Z8usWL2;PJ&XQTPtEER
z9{O|La6ZrzoHzAv&db!Ajb+`5llo(4q<)vWT-Tj@sClrHI-Xlu2WBI+0@hNaYb7;j
zm+3p4i>W)iP~SwI?G4ljT}Lf})y^<#fzGAIKqzev^~~o`A0d=Bmlj5w$9MDjZ2{XZ
zWFOc5OZ08d6&!Odbrm-0QO+HDG_?p~oe}y)uH_onbjNu~f5bJuppN%@>JogVzUUw4
zN3K0yx9NZNw7RKh(>;uQ=#yfsOA~+&sfq4rLSM9JWwc&;V`B(9V=OtwGxb6A)fi&~
z^+dz<h2#&fBVRb2+~s4&2kMC?vi5>T-)G?&_~GkR!1t+d<U&6b#Sf~44^bcgp&fox
zKl2E_)kW$SJTcazyZ4)xaoWse-a@ax!9PeKFWV;i;lEAB7NBxCT$h&cR{Bs=XdHIJ
z0^*08@c|>@?pz`c_zWL09$ZU;Pm+n81Xs<1%IJY6U`Rc{-$N`|tk@?T!TXMoOL!6F
z`6)W!2V+`u=MbC)-+g4&Brm27_)dT1|3r8ii;0zOC+2(%wDKx5<pt4+pYGV#L&-<}
zCb?oBHQ}MP!SC(|HZy_P$pY-JP1y7i{J)6*{sd0>H{`yHd!7NTr64w308yj{AOxM5
z;X{!5(_q*y10mS~-f#?C@De!2V`}(+#!hr9*;qNExSC6?j_ux3t*Z7>8zWOXB3t?+
zW5ywYLe*%j_Uq~n<a{J{YAkZ;7WV6NH8=UWF1xlGbx@ij&$=P&hhPs+!s4BW%v;U3
z+p)+GA^BsdP<jdZewX!jUTO<iU1z2C2RSOVotoXY9~m2kyp7QcB6~|9iz^_rYucV@
z4Oz#grR}5EfqLBCZHZc6TZ%T=W@@8sl5M=rMpaWU+bmleTNsth7uqtjE?G8emuIJH
zPcCYm=CN(C<+W|H<zsD}e73EuVz$**;J;Ro*Z*k++`n&fZ=aL>eAquTYw4tC6`nM<
z1vYQi%JASk3g;8rL|d{p#`Z@WZj09j+CFK$ZSSZ>`9f=Jd#p7@r_{AwXSIzBT7c~o
z`X&k;w2zh4wrQDIhr!#nSd(m_=#R<hw9)8}LE00%h@13N9C^u+_#Hc`RJ2+vq0QCu
z(qkF5f%rU~(GyM3t<|-cYH6yy=hrT%nY81os_j)1(Gl<QtL{-RHdY-*&6)1%2J(aF
zv1ZRWo@-y8Z)-KRT2nQY66$v)yZQpw$#vKz(dhm?SOIImq(bozLeLO>K%ZJF!FYo$
zLF%dzg)E9*$c$dF!1%tR7apP)V$lkF(F-dXjZ@JJ1JDbt(F;}43x!~1q`_)Q!dHER
zUbu=s5b5S#9L4h44w|_HyJr%ZW`B3CKsJzk35)0}c<2K-e`mo;_u>nzB4#!n%x4f9
zqAl^UYGAN_#BtJrxF+Lwzai#zmH8OKyxaf=JO`W2g->^)9$656fIq0P4;(ucH2Wu5
z&U5%`m*B4*h7-3QdvFfez({hax>3)hG1x&RY)4=7IMr;zks0g28s@WF&_rr73^tE~
z&ujymSqx4w*(l9A1No>`n1Q+}3hQY70Z;j0<N!1AMlYo3(X6wz-*~ETqW1ApY~NYN
z9(}B_9vR09gT{1Z-e|p%(O<7&bkPIcHgym-b_HztV%Xq$bj`@5>$)ACU{b63kMphm
z8TtCkd52ofm#N!)mU`pIkjDp+{ky0avsvHiT%&KM9`riaFkFc|UyOVYbIzi^%yjC^
zOwq@)c324YWrjOL^uf-N`T*xJYS9d44Tb@FcV|Cp;`E_DO>ey;^=LX$2fHKnxjWH1
z^S%q;b>a7J)T{2n_Pwb~-Pbup?@ztRfzHv?)o~qn5;e`Ia-NygqYmY~^YwY?kR{Gl
z=#=&7ldaUC4rjH?L;5l2adg*N{WAA*hxN;zQe*Nxx+C8Cnc6<Tsc+-dQ|YRajlL*A
zpZKFQE1^4s&>PLrrJc~J{its<8lP!8bte|-<LTSk#$I<{Z$xkGVHKyN=#BH}=DXCC
zcx_}f;=!$SFe-0qO=dS+8->Bn0>IMhP+z09xdGjO2pixW^&=j#4%%m97rx&yYTjHn
z^P5jtRpg7=fU(jAzj1__o3T@h9J*R?CfiVRuAg{_?)U<CQJ5R>BMy>37zcmkzGxvn
zqC1koj=ZS9mXkQDKe$m1{MT02C-T=^m{Df*M=>G=HOM<@O@8nI@Xjg35tqRm3MXE8
z8m#EPRh~FOeQc+W#Qp{`Z>BJRmV&A7AYOJHob?*fuow8<-@*Nb^bm~b3-NMODi79Q
z4@<2*bF&{vzbnsl9yzWX;65CbcQThxf(hSXZoj6|OM+~NpO78qRuSfXMf`{QScz@X
zCBb0wqreWPqfM58C~QV%90cb+jXt>v&+P?h&{rj$nvDHvQ_F)2)ddx5r<MT$stIn<
z3{BD*P0}At5`zCQLp@8agB#R<c&hFO_lZP~olzz2Dzy<FsJXOPY7wxK3e-%ftqHZ6
z=7qPA8O$XQK15-Bh|*Zzm9?c{GwZ4Cwi8?XAX4%;ws$Nx_*J&L3wrYuIr~<dg}#}`
znrDlVyDPBY*I>tQ&}^)~=xy7nrL*l}g`@-MpCicWDD3~^S|Qs>{Dv5G(P?D(S?Yep
zX=Tt&W$_Qn*)Fjv=4IsjrT^MRK40Ls^X~2Z@f(UE`wMf7Lii2^Y>`@C+hHvidd|mv
zF4sA?vbNj?e2UeYVq1nDT!3#AijOlLpJI~s4%F!d_wWGU=sNo0JP1<^_ZY#w?%|%d
za_?*CgGJz8v#5zNp1v8TrJ{di`pKa-#iyu=Pf?Cq6MouF@UwI1?Bn$BKI&_10a;rC
z$~Ko86BE@KbtpQc2e@2obva(&ED)s-u%&)rOC5NQ4SAk{_!Y%iNz|n~?CNb8igEB9
zBe5iQf(fo-EX>8H9FHa$$h*$UI7Ur>v`0ITr68CoE@n#}Jk!)l0t}OPAdz>`Bys#5
zA(phoJsRhsNyfT)e%%?T&9Q!JV*8XPPL&UieFpL>Wa^zH5Mg+SpK*^k!#N@g5k!Bt
zVM8q=#xRr2vr+KV`(R(SB^F*6ery1I*+S&7WI?ysz?uFMt^0_z_5=>@73{Q=L}B+4
z$6HUH%>wdnrdS1u`(-2!D3fXPix}Z+eE*wp5n_lZ?j=`gEx5p3;*evBRrVw1*$%FE
zJ+K2853VrLNFSn;ULaCBeD3eahL^+;Zh=3Yb?cr0Y9JJK*H@TrW)NerMF$W+=*$Wk
zO|eg_gI$!uKgn;N#b%A78gw|g#zx`=%gwjOTx8Q!^iha;+8BhL+lx9~9nAIM<xB8c
zW*gP0-&NimW)ufADS&>;Zni>4H3WOF!K!x^4JW?KA8@I7u>McrEpLqsVENwYEd{*F
z(66%s-equ^^Q@0_2AnFI`a#F=YmONA^#jIj<mz>OC-r-_g5_+Yw$FNM0<8hRT1m~H
z<;Ef8?>>F8+kW4rFEDoK^NsELJYySetGk8q8fI*vCXuUc=Ka5I9-rs)+XA*-$bO5#
z!j^EHrO5Q<#(rv-9i(Q^Va^qaUvQM#K_~FJPEk83R^Lr+vwi5<LtI-V*LWPC;S{<z
z)_A2~0B5?&z1#$MyHDM-Cx%0R$$h@(e!m!A#!q~>L?avhl9%-`3mYD0DSW+3)Iwvm
z19ZDDYrB;ozEU0^qZ$}gJ#z|uJ(oUTMr>juHHX5D(Pkvi<TSB~E5-`*KF{ivu^T+<
zi22uuW+jsgW@>y7A9RSX`Q9vz4_b}bL?d#v+nMR8L6id<#7|7cCRqSBwT{@tZtRpO
zFw1jr9q*u5UXx?}1B}WvVG@dc)VDZ6ZUNO`8GH4K)3?X&=}U}11TGHID{|tfoCaqm
zf}FQlxP`ai6ug8(_?_BBx|jg(AQUdl3UU^=f?FLVU*eS237+~uI1UrwJIsgoupZvZ
ze)y}Wu*t4l#o?(`hM!W87<Fr^ZuNk7G6a8q0u0!>_%SQ+?zb}Q4l?_qiC|oE=ap;(
z)7tOOMRA~=RCoT$XRs}YREUa(W#ExlBR{7hbG#k8sTUE(VKDh7qMt(H>Mf&6!bWtI
zE9M-Dt$7wC`Wj5Lhr}@7fO>s}m6k{}!y^0E3zl&vv{oK6c#9xw%D7|8^@)(SLJoCB
zfAs?g8^QkxXfYT2dm-`U)!4jSV1V!CT@;m#PP=8<e&r6Io`Rda1u^-8hD#u_n?l?}
zQlBfH>RWW&CnXCwdu}yB@kK5c$F44;DpWl3#Dhu$Dw7dhJ}aJ74p5rhNY#Ak!Gd^I
zendoyVx1SK8d6E@^3tICWkGbxsl&nBMskGF#81W{ZO4P}PsDzojAu3l{W<NwHl3r-
zpt)YBskLdfu=-uCCfn6;Z$F8vm`EgMJjWZ$aYu8Gk(_H7QG!97yT4ilKhKxz$WP6n
zTwI$EzF#KROihRM_vWZJ&MdRHp}6p*65MBD?l(X8os;{|j1EY{x<wx99T@9Z;j^D7
z8We*^8AYG%r|)(WW8cIttBI5^aqEUdu*rA9CEtV%u?oL*A@;;9cciHePoy!QYU8n1
z28$}iQ}e^_$c5&}1Yf|L=PHqL{a3b8hsa3<x?lLc|G>O_jt_i?NVbbX9!-SnAd&2C
zcqFTcf6aq|I2mm+0>r8hW2-$Qs}bD7>clL{z!&!gz4Br7rG`Z+@&7M*R9}f>zb1b1
z0R3_m9>p1O*+{rcyBX6Ph{Y}8-As5#V;S!Q;YoFckI{nYMjh<2O3aB8%!2$xIkMo-
zcr!CBcpZsw!oQGb`HC7Y_o=0Q1rEp=xM7j-Zo~0$Ho_ls<s*lpizb1A4JYc|54K1b
z;)gAWhScNNs_r=VbZo@&@KIc`?g98fJ*hL(3C>C@xGRm|0M>>t90(>+7Os$sBjihb
zBsX|jR(N*l$#d}nH&e)aF<?L^xp_b{;L}XRJ`E9X;o!W0lk*r3^IhVdH$+=-@aEzi
zIZdbWrA~@!=(kGbGL@s2gNr9!9KLj6YCQPDq0UF_JQtOad|=~bChn4+IC^Tg4xA@E
z(3C1@odoO1M2_j^4B|1<;A>AIzCIbOZW6WhCekL*#{cj3%l&==zfW|xKWz7h?GxGW
zubWRkony=t4vv{@&Y=cH7{^|~ITmxSrJQp)=UzqpXAL~`^<2|N?D8$}(znC?-bHL^
zFV}s58Xt$j{G*Vk$KkKX5H~s_%454%LcUfNSI9NGPCV%r{csQ7`y<g3Jg6Nq*Hs(4
zI~=9n^kF~ooBZk|d`~BRDPRvM<Q%zjoxO<{rz1}}6aDT3N75yi3&7JV0zb(gZc;gT
zj)8E%Yg*UfA>9#8$v<x6jwinZTlymUVt)*>9AY>eu`zJPCXzEe9ln1k+@yt!h2?O|
z))H&pVpV`CQ5}h0myy*3&SEP@SqExdbVIK9CNAOfDTXrEM#D!X`;DV~fm88=IK?0Q
z>}1BFfTyHM`{A=jGFGFBUBn^#FXMaMA}{FyWBHjoUR{YAL)F2ug53FhDb%cU<@jlg
zeJ^;5X_*I^&;&mC_PL4o6~Nms3<IPDv!)EVR|Wi*D&#BG0tu)`4Wh>I##`XWv}I0p
z#DnR|OzTNij9_qr{%}VI5hEFj78yb0dla_b7`TDsz|1D#^GqhMX&P-N^K%w)lu+7S
zcqj8{3*Z$lVg@e(QCo(!xDu~u753s9H&5y!cH<@D=~p<@RU+-zupqB<#_M=fu6E;p
zx9z-leZSqk%{8`l?Q;n!aFHuLPmAMNXK80>r)e>?Xxd5IaoREBphsy@aFQcAe<art
z$@N6Qusp(@A3|3ipcnSRJ`SfxcF{9CxTkIC(Je3<H*ufq>AAJs^D5XgE4cq<^yX6f
zVlmQZ5&g1&{+UNVh0$N3a3^QM*POvqm_}bthDA03_TyOi1R?l~BkAj5VDdxg_d#$e
z2f)ec2j3tVJ>Cn8xd--hH+Z03$Z+Wd7oh`^tSwe_E0Fu<*wsz&JsZL$t;ch%1Ba&;
zGOs$%x+=J4AY99eVEh5pMJdNv_@8<YCCCpi$`~n3PN*-r1g=^Sd9cfKQt!csag>$3
zwoGteGcc~wV`rpAW~OGGrGl5~4L{aSEeH?z);99tRCseTF=LCUfyv0$VdJJy&A|aH
z^Dm?Q4=n*{ngC<yx4T{9)zvQY-G%?!d0x-6?KxT;@6Yo4zwH;1yr0aTpS=FSGWr1m
z6;Bq$cRYpf%&dRgabAz}{lD!8um5j5!Tzpx5?ksd$ByP4G4A$<V<yu6(vmnwGR?{P
zbk1+U4mY_blWP++i))tPLdncSRl4ftnR{Z7dBGn}MQ%|VI3#JQ_?%w42dCr#yyA!S
z<zxEvDLUsl{ri%9q}TNITR7_<;8%UZ7y8D0{y`0~-{cx5GN+TR3fOgl%;W0d)3u3F
z)<b_a;`ue_Ikx6`w&%Ha=J|HRr|QX^4yJzC0LH*zP`hD_i%~rFvCQiUaJ!~3hGwvL
zD0|PxA6v}4UctOx%jnw3NZZD!+l^G%&t6B^>lkA@nsI)XF@K)DE;E|1F}iQD*Ink#
zL+0C4=HE+Z$2(@qM`q49n8H7qZHdh26lSDgYfZjK4&!5Z2rtNEd;^C55x(PBFp!^c
z9{+%+r;wK~qyXy8R)#ZEjr@X|*r0W=0~(M+))Z@?CD>(KVyzvqNxNYu^d$Qrn7on!
z*r$VGzYT{AH;U|pF=U>N$3vY2<~bGXbq0IPp}uVxY`KNl8%wcaSHPZI4V!Wudu?#z
z_95`>#<_D)c3_+AqO!|wtdwx<+`UA0_t6e;ltUn@hdI&_Y?ug+7D=2piX$F{fpLtZ
z9>=~pNjxqFUe9TceijrqmNUd*C!ZrOaQ^?>T)%DjudQcW*FLU&UB_6<u~uV2xsJP%
z<GaqY%ze%X?C-<mbR6P34q{cgsy*)G8uxO|;avYN?qfTVi7i-Ku6taI1-}v&(o(WR
z7h-jV(I>7NY}1KiOeT^&&RwIeK3fIRJJspi%Jf)yxJ#wzMOXfZFBoz@EVZ0OJY2OA
z($n+a<m0>i1wl;Q<vaYA@`7z;m*2u^eM-LkU3`nH<bb&NuF=?U5#;>sWxQ--?5x9v
zTZXKj$7q?393ICA8b<y{U&dlrq;P8_ZzK5cwZNk*G9vx4><Ti*vm^J?F@jwjaThQ5
z7xUyFV)@UBUEX1iT|_>`Fei^NcXu<-HxS2M22Xr0{FTW_n~}`+zQluE`Fkz+wJyH~
z@~c1iLILs+vZ9ew!D+GJe7o$o&tQTt;Vj-GSK|VF<rBm@_hUb72G?E=$7L=Yw8?O>
zM`DNc#Ww3qZNnDWV|BqC0>uVwvgO!fVc0fPMOqltDr}f!vp3v^j?}Vk&iXcW$sex*
zR#}GJ?IPrIllexyvDKzGoCt~9j!AG~;?0rpCi)X2?QT9W+rp`9jQ?88JPkH})GTRI
zUDDic<~G-xna$<+wewAl*r{Pmf|C~lMlsa<MEvaywZoqoo$!S@0{-$P>WQB<>cVfV
z$+~`3sMQ-le6_T>1)O5N;b*Rb<FOQfdm(&}FnEo#jI`!daN3E6C+h%d=4deX5ybq4
z!fzamuRai7VShM|eevD<fM@iEYuM}mYOc@!wm<xyXauuuKO>2K9L50nkpsbj2f2?W
zj3MA6L*XY5XQi!C;3Q)??*w9tQ>ZmO)5y&A_?U}`Gp^(sH*l@nx#qoae~u7yJV7kr
z44A-0;sV#X*L%bWp26jOOT6F<xWI4v#c9km6?1`^%3MJW;*Dlra~E;v!{Eu$a6~VF
z1Kg&cpRxkj2iE=iVU7nIm<=|tl-Pu;PUl{Di^sv5&$E8W9b($Azy`jPGwG^5=Y^%|
zgKh1Hof{x_V5dfqj~EAU<u<IRS0GUFSmHVr*}UPv`hW%aVPgj1Cj?o)@f#F;h>Z9X
z1<7SEN3KF0IPI=nhQ4T{acI>r@><rT;|{oembBzK<$<4Diu{xsaCV!c54vO54M+RC
z{Gg@C_ib2uk=QTiv2z|U2R~qmCAwwv8}hY&Gj~<8cC#?I{h03+krEA%rJax{gOD3j
zK%<u+cecYiI1V~^70&Vt_{;HlewLgCR<oZw53US($F9oet%z0i!k-)oCOrc`ei`27
zHuAV5h-;k#mAnU2@f|F<1lTRIYANZ7f#p@R;|2TSU6)fE;WKx@>+OqYJ_;7aba;x3
z)JY)Hp~O9x!?oDNYRh|I=tY51pJ7FjtMD=IsZJuHn)VMk^$#@%SgS9*M1OaE;&iYC
zv%yDn@c><%m*T`k%Tf>6#Tj(f7On$M+mPD9&EUVZ*5(uQTLN2gB^;P_#7H;6Qrt$g
zZzo&tAzr#4KFJ|ihey~m5<K`A9Fr5U7Ne<L978(=3hrt#?lwU?$<dGV`%!T32x5ze
zV4CcAA8P=|4CdH9I7esj@^)ayEkVMYfQZ)v8L!3lRD;D*iM`9Qmp=%4VUYCvpw2ni
zG9w%wFYZl-Rq0TRbKeED57aS!rlutpsc9GKlT+|EBf#-@Q^$D|YhSETFA#w|2{Lg2
zG<l0UkZ-!dVrc~~Q5PJd68zBO;3Ii>3h6;7RM3bd_=~Q%^fUO2H{h+DhQD|S4*eD)
z*Gpl~&je8j0k!D|zT6Rfxe<K$s$>uNgB;`~Ug4_lCc}JBVAQ<l?>_jz1*-K&p}BUT
zotA?v%>-K-1wXbI`luy3s1`^-Y3do|B`+eaoCt^UEisf^#4Mxn4R&K`ti&dtjx{m_
zyQU+Sw#!C#*}{1k`QB&@2R7CR^ub+hxHI_m`|z{YV2#ek@^jS@>%pgH_-s|(xo5%n
z?``qr>LT|8$YU$SoXiSO-4l7AOitZrWc^dH<?F<aPBEVkkbk@hzjP6K)l>0hhvDb;
zAlB3Z?s9EvYLr8s`+^x}rLLnVb$3&cA^&n3JHu&gij1jYo-oTGy9=6I;k&Fd?Z|Bx
zukn9Tbr#TRblu*^W*~uR-QC?&clW05?%t>F?(Xj1)Gc*)cW>H0bxVSb0^bkb_uh5C
z#X5l?!vHh;?ERO0eiO43z%U={oxInHU#E!Eam2LE;GN6RRm{}05Q9^JCx?2wfseK#
zhBnZD=~dC{mC;|J`+f+gb^{#wf*zrtr1$(m)?E(tao+;gx|a2qR|I>|zX#WS3EulC
zsCk?OKQ|8Cb|gBPVf2{~^pEcYKiVDKw<CCP8`h3(sjtwQ>Pxi7aC;5)xp1YkgL=qX
z`ZTQudYbC`BzVAyXb&ea#-Sw{3zzzzQHkqH+*e5-&zQjdf#*%)HIsv{t;sg(pi`{J
z+H?){`REoG@m|Z&U9RMP*P_AP$bM|4=lw3$xQ#={cm$1RJo|eAE#q}<3VrKmq2F8p
zzPy~zw;pVH8@TyC-N)J*R{ac^@@2l0TWIAU1$FkUc0#ZIpL%WPs%FeP9hhr+>k-U1
zX_@D;qwg<-?!65Ba#dD*Y=FkJCBH`(ewP9KKBM8Gr!kK#;&)ri9I_J}`3PF~^WG}n
z+ula3s@;LPqraDKob&;m?G5p*;#k<pym6d)<0|@=C-hhTgeEG`tJH)$JqjmPR*t8l
z<PDX<avFnu(4mDR>k)It2hfwhAUi%3Fb=SZ^!U&6;ggC1hp6qh!IWfRhA7OjTp2}s
zGkE##U=Jg}t!9z;tj1fu2hY<P>Y97Ox-AX5$2`=2rQt1WQUSK0rtC)DKa`4MGUwD{
zDuzu|u?NWW&ruQFA*X)>$McifLovAN<shOm3wTcQkO#&vchn`DY{U862Xt=?S?Fvs
z<ke)oyUCy9!2@q`d5PZrJM)K%kJ)bWv970s&J^iUyX6IKEDhFJ6Q(NAm9jg0)llNf
zWbnxaICs{8X6^#NJcc&nBE38Afc?HeFY$@k^aspWG36sxl`@Bex~4TZgyU?D#-baF
z$$><&(ab7SQRB@IYV>;1)qOB3$<cJTpNuwR4l~G7W{|bS$Zcpg_MxXa3h#N^tfPOo
zz<t^+_s!9kr^MFR<}9o&l#71x`Qh@4qJa;@-f|Y*9Eii!(S+0{Cf7#`-`Em?nlc*w
zc`CZyr-A<qtPGriNS~3}B``8sI{bg*KliueIc<pVt$BXnbuH*u-wYnHG23cL-}-uJ
zdTPTdR;Pb`CA5lVndeIK{)O3(yp|;9fM4*mpV2G6N2C7&&EiA!D7VZSN<}C0!4E$5
zM|%4Q_}vH00M}6zorUu~!u+}$w0Z-`^it-9*>H~&%q^IM>w`^KVO}T&9$kQ0A{!`n
z8s=*U2)Yqe-Ocat1Mcx%u<OceW`;Ys63#K_A4K1{8LW8;nZQ)$h+)hT-Iybq^J~{&
zjwrz#kpstk3LFn6j)z1Pj_>iU-80^&0=mHbaFCj4Gsn^rs+1|zZ$qgByHTgaQh8Kk
zb|{XfB|HAPDC0MS5$|~tSxy3|@O==Pvs4lLs36u++s*|w8H>WbA1FvWYKeN((B;59
z^5Y#(4~7;(EidE#{)+Sa8ClmgDuUy<L3e=Wu0ZcT10C52YJ{F>A6tR@)rPl;p`S+{
zv@U6=3j&^M4;?nXkoi3))4xVNZ~{zn7hKS4ILNu=gA?e3KA5$4x`CCp#KTmF{J%VN
zMPYgnXCwDdfe%K+52MmA_zyY%M{jp{$JX$U4Tvk%$o0#3_mb~#Cf{F)o_an$o2hV-
zqsjFLdz1Cv=p#DAbG7n5A-}(c_U<C{Mm#z3A@cfN%pIGEYpckc7wc)f^WY$-!>dgs
zpC3&wKMbw>KwTymcaod`)jJa#+w0%Z&wtTdFt;>=mu!q?qoMv5J^dTKE^|#C{U!19
zMR3&Ox~BeI53GSwo%?I>crBh+o9EXHzAo_kPkJodYX#5O7QV9sy1~x8UpMBkUepl%
zsV4@L_YWuMA05p1qsjNvknd*->iSor{oF{NzmvFp2)%ziI+07P%6nUH&S&m`R<x)0
zi$2KvM<3;t^vPa7dP5sEQM7lRH=}nuYmFRW&OPZZ=e@*QNq5kkKJ&JP-|c~pWH7q)
zap+NJ5v!N+TWm&)6h|NCcyH;TKIsWuaYC>UN-yU6q2&1!z^!M4RWB!IZw8l+L-QIB
zMt_Z5{}FNf9m?0AJVT-qFo7`#{I6-@t8?Ia4fFsgOU_@Do}*3S3)<t8>rHRuVd!fo
zlK0QS&$@ydbqm$#0hsSI9FMo~u|KD3{g+st1dinehYEqeNCgg;4UDcZJweI^aiSF9
zg4w`@icmL~2VbmBEfC8&&<U)vFPz{AaLuV;JPWAKSA*+pqb@i=ZGM`5bk{hC9?_Tf
zE!fRBa2ux~4K;o?>is}3!Qz~A<+)WIe4qh&S_|TSN6yAx=pYAy90ZgC6FFCBakef3
zD_O;vy8%pNJ7;km=)@7u=+pRDE`oAg#q)Rzx9WW|!e=;JUc(}Nq@%@GYKxzAB}&Ax
z>cOw7p?x;MCR&IRR(zC!uD=d6l@Zh((Qu2Y$T8EwOJyRSWMdY|Mc$c@=u!~1ePL>n
zV#JyfC}v8Lla?X!#K1lVIIVIh@yny93#`LbiJrxkiA$B3eJauuxgt@jB8rKMuwj*g
zqXW19Gb+GZRtVk>Jl2j8cy251YsvOnu-#a;A6PG{F?-XH_pZ-A)Ma04Guzc-uB*X5
zRwwdR#d}+sPf?LPxjg${7JjV^^I@rAcaYRL22${<NIpqKuv>B-K6ik7%gXnVk?$id
z-%ARU*BD_^=osRmGjuXtLw>`(en)-!8BNCra_)D`wy*h4U+~>NX7;@gQ+t~l?i#c4
zCGI&#ygfx-cPz-+1vG?bz>`h{f30o&UYq%EJ@fc#vh@|Pc}s}N3&E!5a%&cd(lq`~
z;`bgG<n~4qqlbd94ImTf#SzhkOFIy!7I1xy!IJ8O>D8vfti~}^ks~Muw5cSQBINt|
z$@+7mQ_RfYw8ZsjZaHw#S>XdsXiao*B?&J^lEHuvL*Zz3f<FE+yrWmmYw)FKR0)sx
zFHk4kf-Ahnaea~gqi4xqPJ&1tgJ(Dd?-1yvyaz5|JNIk^nOeg=%Q+_&Qd!Ie)tte-
zQ^5Bp(1T<&jKD}98OkGrKtlV2<M-y#9-LDF|5OL?)i#__t-z_8Q+GEdduj*{TbF0l
z0JEyXd03G-ryST-X`WGnXA}eBFG99eh#9FMXJ<Zgu-que{)5|Bp@yjl)>R(PGKP7o
z6qs*udLtC2zAwo6osa$#xxs~VfY)aOug?rtoRRuJJ(zu3FtOCkX(_?#Q-CW+fsaLk
z4}=pV9N^9&V9-{uXaQW>0zPd5t2TgH`-m<&xV%bCk-@k<;O{Q5ZzqUZ04x8?Z1@MS
z{BP#SUpOd#f};H(GJOM?_zJ`InfmGzRn<q3wGVJsA7DxoP%<RYeeXSZ#anXFS2)L>
zlbJq4<?saL?J@K0BVyV^6h8NecmZtV4i(vL)`PhP7j~UUcoqKjG7RelIy9WauX&bn
z2EObRSYAB5>`CV6<7iQjqLVlRM;qwRwI2j!A8~UJo@Ca7XKz{)M_a(h#j-a|VbvNF
zMH{kL_1UYs!TEhDobF<<mWB93=aWOvMb{eWLo$QDfz$mltaVT-7?;P>2WAX;_9!sJ
z5ok4r`|}aA177EVJ}E1ElbOBA#NK3NZ_?qMNQ+-06?+rS-bCVs4x{T+2swnv9+}D1
z4eX5%-Ma3piw>l=U%}5L!5h2qODFj%p=GZ?u2CK?IR-qkl>ZC-<iGTs_`n?U9!~i!
zyyYt}r02{lPv|N306z0Bn#SAAH8-hYufcO(hHE}g4~jGNyo!f&K1N-8h<-Br@ont!
zd0}2vY6}mTR5I1aU)b57^r!d=C;cysaRPJLTXejE^>m&w>pg~_en2gJ7u@qE9Q9TD
zYh5JYImf(r%GZHDU2UjRT7rW%MRVB@PM1!Y@YUO>Yc`?XUk_is%2ya{G#~X&4l3)+
zY&jiUPRR-ok#x9lFniiyZ!NI62J$5hY)xYB4R}a?qwV}go&AX(n{T{z$)Rfa9(t>Q
zqgKFMQU-laae7=9V$B-ZV{b0JCfVqZnHhg^de(QKb0zrdW;9;w!SdFCvo0s+TI@A}
zxB95lG;ltdRpDT>@g$53Vt+sJsC;9ExX(cxb`W`4e{zPttijpa`_9`FT(&#@@~&u!
zI+L$;^gi~s_dX=AxW`(IcfGBs2U~b=cw^D5HbcwY3=euU>i=f=8vpNT%6(0EtnvSj
zYyao=6>bL}zlKlZI^zbfy~TQIcks5~Cr5n9dp+hopYq->z=_|0UB2^nV?TSczkSd+
zfdaEG#ZT74`@`o-@=oS6&H$H~$LC%Gwz<-4W9=aa-%TXmi<Eq4>8Vw-_|AZ1TqF;=
z4n}<sul-~CjlKf6{(xWaJAUnCuxu6V!^CeD%F&R5?3dLi!M&%W^`6Jt9n10ZZ)9Dc
z-Q-3`sA<n|yj-Q?y$5pgg3S0M`O$BVKbZ{5%<&mcfA#cW?s+)Ai^I29^m{lrM9!I%
zu;$rd%!|U7SK<6@%()xr0pFj9G>#ZC7o2|$J>~ZhqvDBWH>q`B5)Z$_uc>Ht?eJ)s
zh`)jLQmTO&H7ADmgpD2rMmiUabUmEJe)x&=RJjkRaz8QGxWPyTV-Dt{5_Ai$&Wshy
zEY^*gZ5VmZH0Hk*V57U4FXNd*Z!z1xCU^PGTx?)Qk48C@oq4_lxkXJ=ACR%}RJaTA
z{cS~Ub3CZWIY__Ai)eKo;|=>{N{y~3AG)OqR8~#V19hkF8i{f#u#)l`)SvrMBAuf;
zyU%K4AIy8G=}u9j-eN`N*Z8o1Qj2Ql`gok%Qa|=({j<?jh_k^rS5i&x0B<}Bc6kX@
z@c~HZI}pcTppXh^rii9CB_6%3!V~nh?<_IGH%kp6+0q1GM_a)rbQhw90YV001nV(R
z6bcJ7h0^#iDhjKF8hAG93%i9_;gHZ)ILRu_=Y^iab)mm-PZ%aVVa4WG!Z_ibFiH4`
zTjPr`L->v}<QEQ(zrsS+ZeA?7g(ZTAAv08l#(g^1K3pSyVL8{!xo@eUaG%0s60dU#
zbA&`;y6{_=Ec{^mpM{aa2Vto2hP`|)^b#HkU4=VB2jQB~iv4NAe$^EY3e|+2LRn$G
zP?W9Z<}+mEl~KZAd?eik4L{3YOMT&!rIPRr&G1b$!h!GPAm7bazM~cBm1kO>@r_>N
zn>|V;zlBPF0bceoe9yh8)nhFdYVu@$hxg``mfPrhPf%}fW#znu=&Hx@YxM?ijis)x
z#4ni-H-0K=X#;4*4_45A0w#VA#rAG0=%pyO#`8CzjcY;&fO6!F*~uL3XzH9O>|T>E
zUm}OzO=i82Ol$<GO9#;V8f51M;Oe3{W);wy1lZaeWcmlmW>$dgOn~d{iCUl$=Rj#V
z<UsCVB`^5HeE)=*@)T#)CeE+`hSVSZN^@|f3d~VCnR6V>Cr+aJE6&$T%mKUUN4W&9
zYYcI`EBsb{VnInX_ZfUAiKpwRsV0GS_N3mb@7;)gb{aTYUof*~;AiE)(z2uNw}Y{{
z^oi7nz3^o>0jn&_`o!7v>F8gFfLC_V-{MERfydwwYZI^2!u462t_{_`YVEYgT5avJ
zR!TdfWz#m}(OaO&_#HmugLtI2)6T2)@ZXiidyx;%MS5+FYR6xptF2f=xVHLUjZq(|
z`PHjxCN*A-RO3{Ox>=Q!mFh2LftsL9SDz{4)Z5BG>P4l$dP3>0#wi`ttx8LEjnYJ2
ztkhL!DK*r|N@aDdQdS+Ilu!pNh1LE_0kw~kSM90fQM)O*)Xqu{wIid0l3i`jb-VwL
z_B_^s=X6r?GYT*Yt6h|0YImiS+FL27_G7z)mFntnr4HL~q)t(qtFx7Myl+=^wbEDJ
zq6|^@Dx=h+$|UuiGE2RwELI;YYt?tkcJ+sHh|h6WHLEw(2=%F&LH)qz{G*oRv)5Ha
zSYtU2K9QXGPKx6Psfxd_DgMOH+A(#Yc1<0xy})PqU0sJ?aGw^Ton<}ads-PSL94Gh
zwN9F#57W|u(-!2nsLF5B0_?R9nCk>Q9ZS&e?m)wQ3LN@A{Y5^*1*-aZFxBOHPB83p
z-YcvN`UWo2txv$iv7CB5F4&W$HS<-V{+`WzycKWgS$MT)`2YUWUo@cE&IyK64Lw+U
zG;qV1FXsCcj=pF(*@F0uYLPc}B*z-Xys(ITZV$7>CHUUgAp9;eLI>K6TyOyu$(dWA
zff_(I9q4(pmNVfn8iyOy!0)*wQ6+>KKhp=^XQ)8DZ%(z_pDK4MjMy5g+Cx;c*Qtlz
z;_FVLhO(MuV<xJz;+&Ut=-1GJ`fmv9;>-Z;T0?)1{jemLaP~bTzW?Cd_M%gbMA?}G
zUZoV>IBJ1WwnAsxC)kH$Iq2jr(6=*SbN4|dKY-3T`740wr3PEf1NIjKhgBC3S{q_c
zZ&1LIpo7z3gqFZdZG>;yhx_6r?u)CiNe_wkZ*V(*g_%mUT()SejBX*ehY7DOsflq}
zgdcbzlJHST`1-tf<IF;Je0z0;aG|k~LTHZPt~K$oJzl-ec=WpA(d#MX6#5eX`!NPE
z1_wuOVG!2?|9f9wp3{rxcE`8ZnfaofkXmTT+|dNjUOncK8u;}p3%aG8;Krx%$5I3z
zVLtqX*@YLD^mxIeh`Tnt<9=8(5AKn`{PfAP4$sI^%Y9g}E3EK&5-x2oF?ge;6Q4E~
z_hK!WxIj;lLa=CA_+}zmm(+v{BZ)Zt0sQl!If?J|J&uZd`0`HE^<+2rZ@~95m3ehA
zzeWf2xOHJ7O9gRF*<?3=rc3lgw6kaN;_YItUW&VB0=RS^e&yz1)s?{(@==#ZbJ4-H
zKT~br=h%qn7}-KrFc<E07<ocFYTjxbO9kMw0;_GRRH~m)K-?zd2x!mNP(@BABk6;x
zp$QdU47~)ikgr(L2mM7e^_)8EJayAfvYv%h6QelBJEJeGgI=LHbwfsa3kzr!f1{dw
z1_pVK>}v;E^Fku>D0%>QL7QIRXM~OZPCoY#d@J7T0!4hs`F6`Y1YEHV@wu9Jj+f3q
zU}c?&(e>!-Q_6b}e*Gl8|5mv9g<yeW@FeumIg9lp^f1~?pP@y1YJGw(z}P$KG4TQJ
z<^jDi&fzDH)3OjFqV##18Scle^<{nZHrflVu6A21tDVvEX$Q0n#O-ixxn|U6Yc70h
z-|(%yBYr<oyWpp8ji0WG7O&RP4icw!5X;xA#qiPP*XFA^v{`B<;#WFt5^K7TRa3Bb
zdn8`Ba6E4gyl<g+;6m`i*@$&khE40m2>joN`}+mgop<ouaBUcC(T`A5GC!olBbQm5
z%C=`SXDm>Q@Lr|xYnNyJ_o~b#b@5v_)y}GI@yK<@^F2U&t&Y^b;^lMV(e|k;wNPy<
z-s*!|PU2<>?WR^ud!{wj{?)qR862XS;Jc#r#mr-ywc^Cis`xdUz*%+13o(TCp{K#u
zt)SP&9_GNa;H(e8ML+4I&_^s_p4^VN@w8qVJwpR@3|;UrjPQcuf(LJ5J?oRy9rw8B
zlXoHb@h)=si|9#S<ArmRpGSZ@=cmr9N&VFYjB*@$;+6F9J3<Y3pL+2Zo`ew4#yr%m
zHK=zxqd6E)F0h7L{W$giL-c-$;0+FNiQM2AHNZMLgC34U{k0lq=s4WaL-OU{;DsUJ
zH@P_8tC5d%Brgf{s#`^#auf~J17hbd{A^aDW_He;ikv~MIg^HQS-_gcyHE;TMknx=
zTv($2MHHM)e)Rg)IXByJb`C=wKNlzTR{oxXyLf~e;5+?^4CGsB@T3*Ch_LzTh>rz{
zh?VJD(F`8HyJZ*g;xx|PyL7gAXW4`2>=b_ATf~}|_{zTF5tA&{h<{Cmh+xbbDC8qA
zEJc=D1<!E<VH-K*0ila<n#gxU7*1|EL3m5{_>EjNNmxs6x<xRHdjyAgSV$qp3+cu4
zLN@WLkVm{N6cq0Z#l*)#De<`wBfb>Mi?4-B;#;AL_)e%Uz87kU2|~@_c+d4a##`=t
z&3M7%&xESt6Q2J-C@0<#N{ctx))lsSUdYY+Wa0hNihG4fahqTh*9UWDH+l6>VXBaT
z6Z<Jq`6iM34EukO9DS=$iA=f}d!B==JeAOgytgf%y*}ST1-^@b$2$Z5x=_5)GIPim
zYJ?}`yBC>l;)txP`F5w{ZyOA6)t(xo4n0Lm;%Ccd33$&>nPq0MPpl+;7hdf+y-7Br
z;g|(uG=v$ijX6Wm!;s&c$nW|J{_YCh@%N($S;^mtrsd?i<3LQif`Hb+Gg27md<r_J
zC}6`0utC?!boWzjEk`LZmLsbR{>j>)h5^SwG;RSI4DdbZ-!)L|I550rC>BSfeCY&2
zSCdOYRFNrAOse2rAHl9}f<GMuPg)7bGl96;1C*%&$WaMUq4YS;jG#E*z)2pUBR@&M
zu}##hvxu-mPy@83wynWARhWz<E$D_Bb^2e{;C)5?d=>oQAXvd#?<4B#3*<Ei@R4oc
z9GwS0JjR=uYTOPgAfvtd0;f->H}7S<O^4xNHlvYR!Z|;Mo^Qj*hr5$Ywb0|>P&Vo1
z^kwvjn?-Isj+jgL3o>RS42lFt{u|!>Bk|;=7DN0kN=}v+A6QoUHKc>bkH)v<&^M5~
zt<?P5VolNJYA$Vt_E(#%{nW;jOOGZ-jnF=5!?btAsW;jndR+|EUNT<%cf8aFYp=+=
z-)KbW;B!Z7pS7{tH*KQ!TbrsSX)`sCHV=RFV%}pVF@L>ggRc+6E0%&-mmaP@JAIW3
z=r`$6^hm3qzlQh!4A1`;&#;PT*n;{iLLY%&doum+=CQ{7D!r1v4Q<0Ay^Vg3PkUP*
z4zDx?{%8?3!v?;uedxhX>zU!Y3VL6WUw+r?^1Zf#^XZAVYdD&&>2N(Os6TdqiyQ|7
zxlT>;9K7S3cN=l?1is<h-i+wC3RAmOVU3e!;85N19E<>~nvKq5Ex6WxDx8ZTRgb|a
zzJPAYtR!QnPiZFbxgzu?tp;xsiw~qHnBge!#W@^H>$!CRT=No<^a;r17v^yV6x5Es
zHxn3c5%Arr9Fxt^ba&_PaM0qJ;1H`hYWILXp8==73zzvG1Ur#@!3gdh&DuuULBor4
z%vS?DZwj6sK-UM8WlRK#Sjf4t0rh7bSi~95ms{ZbFUd;21sz!?_<~3>l}vCA1z;4)
z!8QcESk0*~yTD=$F!iL?976Uwj!b6;xy~XO+_m_Gc96v$#7%q}#{CNU&|M<)Q<Ix0
z?MKIDrFTJu*#&N)!^4}&ZbRu*6bbvD+FXR#T>>ppX*lC@JW|n|3w}N~5h0Jc4t}Tl
zME6EKvoXG_Cd7y)=3@WfXv`KG@mK@c=6Z0?f!EaJ_0_;dDznWB__$)wj+La2F2Z}}
zr*B41`t4)^KS{^_q`)T?W==AR)YpN1mD%S++_7Krj=dxDJjDrh8?O2ypYJ$IroHgh
zoA~rA$nxg!Jxt*H7=jBbz*)D1F{;gcRSr(4AbDRV7?=pEalm*}67|5xpt9*H%BJfu
zDe<U?_rQ^?3o<05$OL=wJGDd~Q4{>ABz1arG<T6G?tH8@^&6b&HS_09aHr#_EVjW~
zETLwf$dND*?x8)m>TxRuE+H4}LKM1wFa1ltqa}aJ+<cj1>mWG%dgkjn;N>H!dU^!$
z?gijv<H4&3P{+4NCsLnv@5+%c<|D49Mc*XiLvnMJe<rp);i$ht-gpEJ>sGu)OQ`E7
zgUJpCgY5+F+K8T;6~O^n@rN3JJGI&}@WUz8`9r)f!5nX*dpU(yb}zMlKvTH@etHsl
z%24|FfJDO4KG18yNyiZD3V64{=`QE|oeAGN8veJR{)1@t9*+14TBaNHYCA(unS;cG
zZF;!Anto*S>D4wBAM9v)2n^C+X+4Q`9ndYc(9fbrI-=D?uT%-0P-%2Zh0ze@(x=mp
zZGx6cA4UJPq4ZqqPh9Gu`KV(Ra#5GooEX@c{yz1Hfwk!IQ;j;elJ-_BPy8!O{EH#}
zm0^_rKi8##|G(tES3DkgUf}ufw2Hj8vi4D{PAsj>_Ug0U#@Zj=BboQ|XdQ`--8F;W
zM-%ly#Kqz4*ElUBx~Uv!p7QBSiQ8+zIkxc`_M?S5p*2I>)DHh_54^sEi0`9`mD7lo
z3w2RntEbR+;*aH<Bv&lU_f<=OOs~=R<cz<V9TYGXE85+ZXnC`Hck4yS<12atn%}4R
zWdDUHccF!~pdUy<G|WylP=dalHJGnjfG78&HW-cWc&=Ax=5nz9a0YPof@qd1qFZQ$
zhPe}U`C#gbDa6X994p&6W{!aZT?M^=3Rm!%irvksBqI9O)bJ8{z;R>ft5BETZ0+E<
z2l!*qx7Nce)sea;z~#(<_gUkA1AFtEsOiHQ6-gh$Y|Pgsg6mmlL8DZRioH5)Tyq>n
zJ&BbgiIua1zMfmm`7gn3zmWmRoJCgRWomNK+-TNI6EABLFI%Ew?}@T~1pMR-6!0s!
z?BF~-3Lkxiy6q7R=?9`^fOq!Malrw<nhteqZn`3u0K=_9cG3_oxGe~7FWjob=m$Iz
zZOdF@<w|157GmW-BIF6G#*0Ln+juXZ((5aM)rEeTLWq$m%wA%oNCXQfUZo~>WhNfx
zBDNI*6)Z`PTOQ3)HPknCs7V`<={5&7Y)y^Y0mQc}I@q3QbNZp!8B7)X4?3nmFQ9Q~
zIVYJ9(IMa%*;c%H9vDIZZ(M?UX*pdiSDJ5t72iVNbcYT!cX{PKaEAM2XAk~&1hk=n
zv7WJ(v4*jlv6APl;Q4`^ZYi%{!giLhy~XH;7SdsE0o$LCvS=>vHJb{1Cht3)_n#8f
zm>$FVdWimXaeR_pWba#1W3318SP81Vh+KX)PS+`X@^L7(Mg;T3O5h=7z($LjyAaLW
z@!hl_nl~nz*QE<yb-uR>^wccPJX4hKvH<ZtC-FVA`KKuj^?D>dHf`jV2C!6_dFn6d
z%U3Yfcl;*LKx6JwVFYxmXFz9;@yqSwx7&u}bv;<?a`Z`om5l>B)=40)qp3`WGB@^v
zx$g#B-yVLyIkCS%&<mRvzCN(BN4jABpAB3#9rJ4xTzsg>4gdZJ)xuYJfCTEDm#p6r
zSY7Eh$JZ5Zo#R&EhzqQ+v@h7n^8g(CZdmoL9FgnsX#_fZFGdwR7wl;UbH^l@hOx}@
z!&z&1FkPVg!9Vm2=D=;xU9`YQ+muTKF15K-XLhNKi>5rsd>Q(ImgEd5276JMY`Fj&
zdtT0pT+Bb&Vb?Qreq;zL`BI?nhy*uwpr{JrB62aqx*I?&5Rk&Q2XucDogQ4ce3HPH
z|8lnd;#~Uy2mcME`3ubZCv+nTu<!3!$?%QgIe(wP!#{?Zf5fc^{JqEDyXdoSbL$p}
z+YSC+=kGQCUV(qP#NYE|K4-agntAXP|Hbp)Nphl-{CAT7PU7K<XQn(E9LM<Y7`KiE
zFGu-%1Yh4_{vP7*Aub1lmjh^A_H&8*KVu)a17mM+#Bsfkn6j6#o6An>+Z|lCp%mCc
zO|hBFCg$XgFhv`{57&d;tRoJs1;ts5Rv>_}uSH1^7;6}-sX|sUR-hqRj;3-cT+?Fa
z^93l==5m?EWjbnuDX4KKz(tJ(2^xu>;2$!<A>@PuP#yGP-tU2KtqZDy4&;h$sk~c~
z5yVo5HzpD`fX}J}o?n9up$b}p@-SOvsMCw1=q$)8KzXQpvZEQui0Umhcz*;4sEta<
zNK_4AOvz{fe!~6#3+MlqTK_q<(F3^ro5bSF<P&Fb^BpC7je{ZBL7d(I*R}$-e?IFf
zOe5nM%YQ>*@%zBscczkR1uxzh2B9|jaAl%<X@5(&@`l9xnsDNk;PA@eu_*%gofq6T
zo8Jl7_XCbP0e;~*IMzKhu2<or&wytgVIJ87_q>_hWF>s_d>)y~Bcs8B2Qj<!AWvya
z_R@r0r55>m1sKH=WHI@D1AN)Y<I_@~MX(~F01K_*(@285{0fKp&RdkZEDyPVW^j^J
z@Wc+ZUKX%t&AS>LdXe{scP6~)1oE2Ua2WmJTe~qgwgqQt$~;-yyN7-eTk(sm12bDr
zUy%9K8`HtG$J76M1Ul}4=<#|{k8~pMY2|$Z?)`w;<c40&djYRaJpHu~Q?u;Va|QFD
zb!gm{;jx*A9&Z}`F2<wl9szbfm^!B~{a3q#y>&pZ+zOtg88uJ?dL7n6=Ur953l4Wf
zj{%b_p`WKdJdL*bB=yr#>cm6zxs5~fyh~51Z>Oio7W%nvfLB?o+sLN`uqHD#rAc4P
z>Agg!GDIuANcaACs9Y-ynfs;S^&+kp@_%3i?q5VMvN$+&uD!w62euQ~-g36PlA3Nc
z+Us?^=LX(;6Z^20ec8c2?GEl+R`R=?>}!5Ko_#;dXShTzbDg^HHu^!nF+OJ_`t~-b
zr+FLrgn%ZoCz`<l<ct5H*&avkI8_fps~tgK$kgOI8Nn)Z;04OZ_gjqbIVPxqyh+~q
z2tUvpy(KkwNAl5L{6<6gt;TpG;HA=&tL8*gS;V`^Tb4e<)xC$&=(CC@oO>5=wZ7Dh
z!{GZTkn7HdJ6}rPyB>~fH^<FUj-T`3s<+V_JVW>Yfi*mTqf3y{DOl)cPnTQx@L~A!
z$8&7X^i|?mt%u5@C7Ot?c!LIj*N#R<F`fEyF~|A_bQgPkiPV`2ESrh*!NGbXsr{ch
zR}%53Ypgk9V?~n`xEHdbB`HXzAA|m+1`K$hM`wGKHNDZd3?r|aKvggc9B&B@+jVGb
zc5*%+h95soJ$e;v`2kv=m*63v;K_fZ6_W9%8c`0~S%oGQ5hOFwB+wtUI8mko`BzOM
zP$QyID`HY-G*7*$ONIpN)`di`l|-|RM7JHpyg1_I5u#^2k@o^|`Z|B_66FJPz-wlN
zk5snbsB8Z+gSeSfbh0)RwUCW<#KKu$Dg`r9TIQw9ctCS-$;Txye+86cWw}&_hp$c*
zT!;Ct0r+|oYT@QoR;`%}+k?1wVs`Axr5E#LKa^;L(4!55BOO6kyiuqW$D&am&t(#o
z^Awzr)0k<ebD7Cy7MD4Ux#*7Pb87*&0vzi?X5~e}nR!05^8&^~#v)X3{~1fUULG6^
znaBTUEa3J6?hhbW^B8k*i_XULGK<&E49@b4nd28R7V`cJ7y(7yy#F0jc<0HCNsNh%
z32b3JTNuX{#_+0<JaagEKa^($w3q#PMqeK7&7-~W!}jDE-TB<z_~c#r4!ZEnPJ9y`
z8STj?+LBYW<=bh^_tc74w&dGt&a0a-nzDr^Fc^*5LKC*om@PD93k}Gk>#>EpV7ztM
zN^QRJT8x^((U#k7xvwpcwdFZ&d2WCMY0v1u=*Z|49QA_R32f^>qdwOS*nT74C-8pF
zc;Du{e=GK*4f_*V`=TTJ*@gY>&Zq6gXXwl43amvrgwHyh&pn#&U;^LA)ZlM9mv1ug
zD=p=lUkQu17KCIYdDb@a{#|5Waj<%a!Ap*Vn4E@Dzd#*u6;0|boNo8w*q@NWy(E)+
zhcf+Nkn5jR21z*IWMg>H{U(A`hN0$6fs#Ev*<m&??L43=g{Txtk}sA6S*b$)Serbu
z5&2|uFqU>`;k(lHxeuNE2a|h_q%+F|x&cooC!G)5w2T^JEgJeQ_^fuZ#>syCa>vlt
zp9Z_Q2yS}aWHjHUo_K7ECf5!0e#vS6YAS>eqBJ!|WwU0gO@7=29HuoIlFsDLy;159
z26Gt&mNF5BYX({L0<!6qR3007=1w@UgVY!&$;&TLUtFg%&3&HzoM*o$pZ`Kl@teHg
zMdhJUCm1bT%~s29FyH<7A`at=I6)8n)8-tO^JpI~gTvpT$Jrfo8OsB61^UKS#fMN6
zeM&uW`o`eJvG@~O(MPTwUWJbI-|vEEqMOBK?umz?w?#4cMZ?k`Ps2cZ%nh~}P^_46
z&Y3~+EqLK9=q&`;PLYmmR)&oc!v8kzxAC|QUQ~n+6maO7@y!|F0=(t{!R>m?z2FJD
z!3%c6FV_yeOiR2RvH0d1SrW{3EpO=i{G5KCkIXTayXIn+>-an_^0`i%Ggyw%+cVDW
zu<U?OT#vVIIh^8r^H;u~H`F<gsK;(lxt-%1KSI^A8+2<u)!h>6nHkh~qrtxh@LP1D
zhH1|4Q-|86JatPU^LtYke#<EH4U^e?ntJg7>&b2brCP=>J%h?6aD4Ovm1+gATpK(p
zhQ8tXIGWN?k%V$fHKsQGFq%y-sa$WO>pjWe9URfiInJjNS%z>{bmpvXM5R!Xx}ZRi
ziA)pB@13Y;KTtW}BjXO>=DUgAE67KupmQBaoNo(CS&N<m#e-TAFZksbdOF-ETRh3^
zv6<{}4jRjUsN>tCkF0_Au@D(xDlkzm`ou4+BX<vu>A3F?^V&;rfh#D)_R||{HTdc@
z=EOnFkL{Q%YoV<#4z|Q9AL!6ll7meIgXjgO)EHk?3@Ai4_{k7dcu7GYa|3k$G2~A<
z$(tha1?%9$-}I*B3+2cma_9%Z$JgTjnx!Yu@BS9}@=>tn4dBkRz*>gTzp_33JZphl
z7X`b?pxx1|T0Fg-cfda`)o#M+9jA}x7PS*RU?cd&O7Mn-(N1L6YQPzmQ2lClHBpUH
zKdMIciJGKbQ$H&4XfXDvSCozFF=eT`Q<;fYW2`zyp$n(dLmjBl!4oY<3v?ZIl^C_6
zQb;YX<W%!0>D7!%v>L61s38i=DJrTW$xh|3{7d;Je^C<T1m(5-Qh6djQSQn2l^gOc
z<&u0=IU`?Gj>~71L-I)_PClyamJcf1<v3-FyjR&I?^ZU*JC*hF4rQIZ{eMT`|G@p5
z<Xt>(kFrhPr|gjT^V&mf<5+N8$K|uiX|{bqzNTD}Z!0(D2g-f<neteEt-NGk64<9N
z?AsqXNpZ`H;*$-kRk5m3N`#t0Nu%acvZ_VVwNyY8Q(LK|Hb*PdS!sbbrn5R4EzL}H
zE6bH>XlWLyhm>{d1!cE-Upawx<%;@8d8B%k1T_r3N@nybh0*L(hL>%s=GD5XW#D&f
zX*1MT+A4aZ?xDZwS$btZpvUUJ^xl@$(^{Bz7cJ0xu(>4gHw$Yaq+tz&e0UWr(95nd
zd`MT;IvB2h;<!-oLPm0&<R*8nh-ak*xLLq=J{@o6dh*9(_)G4fRr;4)QP;=gtzY6T
z<lTvH<SgF4$J7Pi$@=}cN>ZU$$&c=-G9K_i&exm1OXFdL7PC&x4yuq-{C6MC)W57?
zqtLx5!vBFjP%1cVG#Ohy^0%r~QEky5523=EMRvB4Gwe9$*d6k-kDPNd8t;GxJSQ1g
zp!Z87TpZnDUq@3(E}|ygL9TV03i=_L)n_Vm6~AnlF@oM1S;?1*GapxHevf6Y>_#jY
zMwT%psN)Z4`!AqYcmO65@B{>Ucm?_ihntd&nek&40@o-H3RZ_1t_2>3uFP(O@%N5r
z9-B)PS_u-i72G7i-^GKNUnXYVWk!1jXYigG?K{24l0aKj<}H)?x+#=-D~ec`*8CF8
z<~{S)7ciV3ree%pWz0@f1!7_q=CA7b!fOU&qyX+?Ba()IhS|Y^9B3lLa4QF5XaplN
zIAXbO%KuHczY)&~Jio5lio!+!O*6t3_=vSC@8L0(L|<9N{5RM)GZ*il#r)Bf-uxD9
z?zzcfeq;*tVD>TtyTOhA5PiR}df7WPdC!Q+_t5nOw0URv^hd#s_R>3TbI_;Qmzlc@
z^LlGM(2e0zYVutL{2;~fe&j(Zk`b&Zif_?M)YkE8B=J3erLXI2u%QR!8CS?T0<M@i
z(8w+1CM(HQ=J7jC=9e1Ir7zh}M>42ba<JOuy%k^&iUpCJ0DcqBZ*3%Vc7xG;Bci_s
zhq=$OaEYAzD1Uc?j;sOIm`_$d2_#||oi}=tySD{t2=p4N21^?7CKLce$O?*(3eSN9
z^uPpi5U2?PJ%B#r9e7R8n}=`>*TD(SqWwRDzJE6r#3mF1D?khufDz1~-k886Bhlao
zj_3Y7+6%3HSD4oJ%o8oqXf*{rXn@i%u#QYMaNtTD|7CeaDQ1$QsQL?n59ebp$xS7b
zgYzRBimohdArt&;Mo@-~=;<>sGI2X_UuH&D9?#D6a^e8U4fc=^L?W=AqU=RUboymr
z<SI}jRptF^@xJw$xthTHw#1Fl9v?tg&bmG*AP0kO1o)GQ%y2W&-7MnMuI7B)0y44>
zFT-(omJ8HQxA`8P(`)1t{eJ%9yU@@_SixFS;`GSIoLLA~uRMHB9nRI3)Jol%Q-^ZC
zP68cTNM*Ev-(){n*SR1jmOwr97pBik736@=P9Lm!O5l#E&hH&d1{_cv|HEY(N5l#;
z+MS%iC+Pum3l->V96W!>QH`cQ##CUCdEuqXlbJRFtLzSPJqm}@T(ZzjoU2F4DX!BI
z_7%tUAG8)m^BsBzzBT3L94wC(wUIfB93>msOfk^hsvyBl!Ed{g7Y;?oI+c898EcAc
zBM&-8&UXzD>@#z9%NKJCv;e)xyGG-8olBpEb*#s<AD8NRIu$&mSMEnt1Ig%)&BA;%
zwQK0XyA%D_QTno9w8XL=NEdWz1L@m7mLBf21)pUlTC{EG&<>#^JBJ?QmQYf7hJN9r
zP)GQKQbR$xU=h0u;o<-^5F^lAOc3&kGlZhze4&iERH!JfMnADpsEanDvA9cUA?_2}
ziU)*FXePRg$AsSE386n4ib3LOVW@ak_(!}Tj1Vshqr^+X81b?&mNAYo{=Xw|dn}KQ
z4nFT3&p#y$32x)4&`UfhbU}yFR@@=PikpQ7;#%|@%Y+KzJaivZ(QAwma*9LHh4dDp
z#14X0j1{~n?VKp#zVW%<3i;8LWI?N(QaD9l`#AdOZxmdX#lm;X6yc5KAK|{Gd+>WY
zf=*={+T^9Ig))_1#lz4gceQ-BG~qk0WI2cKWe@t7739TJ$#Vyi)3zp$t%f$EfMq9I
zxMi#pFo}GA0GNJj*0QQj{$GgSISt6Xi7McisUkfjbC}PAqwnJQScDR1G*Pe%QN1p>
zcv0BUwA?aK;e0bjneM|aA7|EBPxUmF+NuvnXJcy4Qq+AJ;9ZPFrmvvIcfo)I$m|+W
zf=Qs8J@8D`=S(R^zmn9PI~r(X0@>sh5XD_Emh*7S3?VMG<h(D379%q~8BF+CzJmDO
zhIKjscCrjj=qUUn?U{3{_&($HxCSS=6VLr@`sDOwEs+N3+6sa1MDSmd-W2bCjQ2SG
z6F|RMv+oyp(qp`;r|7-A8H{Qc+~Q!g39ZNzD%0;EH{4+aYdc9=9k{(h=((b`!<t0j
z^f&5sv}prbhodz;-K(M>D?qRIR9a2dpcPhsscF>b<b{{f5+6WEybcZVY;~V9QeCC=
zRHu;(4MscMK`BWdm`g3Kq*8M#BKea<F7-|MMXvXfyz4r--7#{rZSqECnY>7uE>BiQ
z$U~Lha(AVz+*)ZMH&80eRh8m$X(hK@KuIrWQzGOvibW1n6v?9el{EQZ$tk~*e##G|
zf8`s}JNbh2Ts|p1ln+U_<vr3Bd7E@j-XNWnS4&6bWzqq8fwWJaBkhuBO55dW(pGtj
zv{{}kZDgzuj!FN$4%`=bEbyG^Ja?9~Q=TjBkrzt)<)zYLd6jfrUMHQAw@8=dUD6GC
zzjRMNCOwhQNw4H<(g*pT^j&@-CCMKoRsO}kdZaMLD5qBJa#kgkoL|W*msASKm6S4a
z9i@icOlc~2R65FimHzTbWsE#cnJq6-R>_-`U3@<$`JQg@eZ7=lDL>@ziZ07axMEYY
zDrw2D^Qtw;wOgU1?o0kX0qyh>a`K(ZeDySW`UB;#`kCBaRbHW${-fqnO<FlMwbq2+
zv!_~F8>hyy4ofe7=P~rgpRYYtH_;FOm?m&6WTtK?gC@E$I+tE(9Vcj|;GOHk-FAVK
z9-+UZce00m$l>}4j<37H{v7S#Wru@RE(G`8tyiFiX$B_Q$LrB2bBwO?=0z)175`5g
z);=4GMsqIulx;yhbPPVHMre$><BJ*tw!Mh{QajM6obmO?YdnMVVk5YDJpHL2!R`O>
zRRLdb!5rC-xo-+{?J8!`{hU{q(UHCc%m2eIGrspU==uVl!Yc3su|a;d7FlJxVDG|>
zFcSf9<W+c+7u1J8;d8v;b>VPASzwEbl7CdCqHG3M)1Au*khK76wuWf1mzn7d`OIDT
zt~V$n0)Ae_XbSpwQ-P9X3$AWgiCL;4HElcc$v(uG5%6z;b!C@;h5Sc<vWD2R4gE=g
zyNgGEa*6&kcZf?*!9m`cEaY$D<Z@}q8?!NY<tOhe&iqx59uU>w4eP?RHbGw!=y%_N
zI=maWM{nk|0YR>@9vyHRQI|KPdT)thtt|{{dtzY+*vyXLBc0$VyHLM(h5778wci6K
zv?rKJFHn%c=*`?17`=kmYq|YDV>Pz}&s)Z8mhjpD*SUag&kgF~TJk>4dB3K-ZzI^z
z`uKZlGv8K6Cs&DmD+h{Dns+MBD~s^Tg2Dd8fj-5pzz~{&<upY1QkTzI9Zs|oId7TZ
z=Qg8|@sd-^<krc2C%@rCzk(Ab1bg>rbY618N&hBG|4Ig%Kql~#9Q`S}m-}R~H^H2)
zz=EE~ofOXslZS)(?L_kXF+B1Qj|_kx?M3#`g>1hqIYlhBKm&fanq(FMk8~OG-J;|i
z`Kb_cf`Mfw?@b2+mI6O>IGJz=x)}@EumSC-j&fcyrp3z=(D;ReUxb)GgR6Zo8cc8C
zPG5kvJtaSWNM3W7uF^NimakBmTtH`YmRjW$O45_$J%PFBC<?*DAaw_M_I^;ieY|21
zYbWet?4YXIMvk<VEo{N@yajY`Gh5ok2z2FJ&sfJ;8yr`-4vcHucm2QP2G6<4b8nGv
z-Qjij*v3P)_ZU6mGqSK3Ft@LHzjqw-AIaZ8vmf8tpI@ke643#<*k745LFb(frkh5K
z=`PxW2gWedW9HB2<ZG{u=}hmaw>}!Po4y+Jn0``sB^pbZ+{Uu_x+_tK)i9Zvb8S=>
z4(hNd&X?5G7wP#mGV`nCpaRJQN>qTltSA^!NjU5n@WqOBXR3ylsW!h~19HEnRAnux
zS=yp@>V)2@8;D{r6j1%BX$F~Yp=Y?q8TgRj`zc#`!EFDAiY9?y{4-npPJQ=_t^VaG
zNM_4!YCV~wLL-;-Q5PA^>p0FffMIQ9Z06kD!r1!1L*RcgI5u;=iLsGdYdw#z<@u}0
zLsy~ET0!l#jCye?`rk!hVe_dU=Wxu<;Qgnt9~01@jX_&6oMU$o_-J2_-5wmfornx=
zIjWoUxf=2L>X6-5qhhPTo|R^Ai*Q8e<r8J&xJ}C|!%@459J9Ky8OLB9j=zc=dnNci
z^78v+G+k!ZzLOk}`-xgx`7KxR+s@@TpUAKH4|s8Jj;?mx+Xy7SDoA^AQ0bi1XKBF4
zZNx;)P{{b(kjePo;4t2&9=||cb_f=H3vo2C0`nA@lA%P|F0j2#@MKknT`5c*mKk4G
z7_nHV+WHA2`x++p7D!t>_0}#bla)mB8C-_L@ARMoYC#=SliH>fby80J#VNquOvL{r
zoRtZnZ4XeaUVtGz$Q-c=?dl@3rHS+;8Q?1hbDD`;Hhcjt`VxKcM&qG1;tlu<rtu!M
z<31SNdAQI0@SYpsJQu=uPC_d*gt@Al*NdL(CmPmQ^y0Y5x-BQ@o3w+TRLjBQrqVxT
zDE&gZkY6>`uQB(<v&KsteW^C1TUrh;I0tNQqW*<mQ!ljMXq-BtSB*u(RgeBoRnU%>
zMh{h({!)496O|1NCxhM(y;N5%l67W6^;op5_0dvQM^jZEZ$WYJo&wr0EeE=*%;>Mu
zqGgR{ehx>U6@p$%M2BzEo@fT`5jy=xtl#rk(^$(*RiEN>dd~kZdF+j520IebaD}iP
z2iuE8yOk2HSXvU)jOezqqT9;Fe&k12R!qyO$Dn7ef}XXmR-S#T0jAhc?@9hR1e|F+
zJ+$U%qxDs+KevOm5{}S+>!Nm$wGhs;e#1Ter}h@^^cQ)jre{FkUr<l)twev5rtrhv
z(D;vlr<@H(xn7^n`qQiNPQ`g&>KDC#^=I^a`9Z%&KboynzWQKK-O%}u@wW0UW$nzp
z-s!$8tatOydj!w)P4LwO_;3lmGAlHp!x~L)u*}znUL-^Dh|gucn{8-V&rm-<MZcN|
z4i#b;K`pzCzO#q@t*Q5iqFJ41hy+C_!qMA^O0_?H-5f(&<8J!4-Y_&F!|z9rh8gsN
z+6>lono9K<)j%?lGR$;=GwPkOy2)$o2nsTa`CtiZ%6;4ltWffS)k-vCPAXL4ML=-s
zQjK&)?>Uw^a5;1Fe)D!zMweK_@}1cO<{nNbq+G08SpgqZGrIHjVkMbz)H{pmtGB~)
z9hLt(TtaRfLt%o48q-F_m=6C@0hI6+af>!)?Xj-H5#b+n@iT<?!b-fSyM<6{xlCyF
zi;1tO8NX30dW4>$MI0eUiPObQ;!-iMxLGVA9uO;tXT+M~O|hZ)RBR!B5Ic%L#U7$t
z>?iugp`z6~QjD~Y6Vq8Ii&?GH#oX4JVnOR1v6yw9SjswIENfjLR<JG<D{)<!`>OC*
z6`oVsI+gn-@z^-AoOOg)#yUhSY3(N#we}DTSUZY2tu4h&)<$A#YYj2nT3!^aMMX`_
zCH@uDil0Qg_)PQ)H^o1~Y5W9x@f)neA23&pqh4Jn^buzZt*MV|irx5pjfIS2c|j2K
zP(w%IlhrI2@Ph3^_rH*Jc80U=O*>({rK-?__0j6HzGhLp5-BZ@EVB6+{LM=Eol%y}
za5q!oc6zX;Xe~I9+|-g5ysaNhGpHN8G7pqDkLDL`$y`x_l^Vi{!#|8;O;?Purj5n|
zrqSeqv9M}|nJEKD%?Cq!W4xibaWN{`J}8E&!9%16ol2sE`At?M-bh#Gf9S{*=+uyz
zsOO{$!gW}Mbtnx7qdcsR<}eM%_fK5w7wEdS9QM91k*X5hK{V&d7b?AY)THzExm0{@
zS#dod3J96={knEYTcJ%ym)V9GQV1WCQM-?4a=p4&?XON&%d0I_yIMeftr+lIK2^pl
zdzE_1WF@`Qnp&d>7`&*=m*2{r<>PX3dA=;jJ>}<8WooOm@&qYaY9`&4a!b1;%`;Vc
z;OQi7_mr2$deTVEJWfwO&wY>46X$v9p6fZ}?&F#3uIuUU&g-e{4)LUS|8ToqcioR%
z2i^N!%iQx^Bi;R6?cDWTmE8GVIozSH5cdz~U)OEtGgqARoNJMDr|Tc*B3E1ISXUKi
zPgh=NGgqXuvP(@a<oY)`qw9XM!*wdz>)e_A*SR$LgL6vq6X($68_urDXPvRhN1Qd1
z_c%)?Z*k^MUhPbqyx8eTp5-(qPj*U4W1WAJhC4qe4RO9r8sL1M)X(`isgLsk<6cr<
z=iQ|Kj6vKU#(krlPm;zvUnWg+zE7Ix{F=1f`8R2^Q%;I=nvzdB!+5{+$<Lj6lE3g?
zZfEsm!4;dF+SMaDpKD}tdDpz;Ca%rNJzd9>$GPq%FJr&sTzc|#S1RWRR}rV~s^?7S
z?&U1yp5}~oZ*dNBUvw^Ve{ddfn_Umwxm-!^I<6F+0j|=XrLMM~c-I8a2iG=Fi2IhO
zxI5X?!JR>x>8>FiaSxQ<yH`sN&sC|MCsFF-$t17#)RS*{M#;Kor(A^kxT|DUR!A|*
zZK*Fc={h9_&Sj8nRCmgy)C9S&nvS|LR(Yz<SM1sirJ80^M{AYTgK$LOw3BL9-L1A}
zE$79ogLa#kFL+&AWp7E=s~fH#XO2w9pHSFWj`i)wdv|fPeg?b9;jioK;$P_7NKdG@
zcuz7II{DigcKBD50lc6lOJf{mXlXoWSPsJb1g<=VX)$NgbK_F7zei+0(dK=orl9AG
z@sd9P-HEVVgJpH#OpgsZp6o(`rH+u7csE$Mj1S%@`f(rC6eozY#k1l)Q592I%UD}m
zhgsKK4_H51e^_(da@qRWI@|Wy*4qBqp4y6pgog|bsU31OWMYUb<U~lR&_5x=Lvx27
z4ebz`9J(~LsQpIhK)W||uf3T4i@k?EyJMBTo#U2$sYAElaTIb0VeK6i!{$1Mhs8Vg
zhJAG;gry8i8D1r<PI&*YvEeJi;=(V4y$k;q79J51UNNF{c;ATj;VU90hhL1?9{w}p
zMtI7|AK~R9?Gc?L^G8gNtP`;(vPZ<d$jK2&k?SH-L>-GL8FeorHtI{nkSK4&f~b^{
zyQA_&UX7|0`97*iq#V^XG9r3dWbWvxk>#V8MmCJz9N95CE^=V>$;k217bE9J-;7)p
z{UCB{^s~sg=vR@)qu)oKjs6gMDf(mN_2^HLH@Uve|F@&xaQiv;J>;>QJm*5>iRj~z
z2cq{z?ugzPxh{Hf<l^XQk<+7xM~;f_8QCYgMP!@k>XEghi$s=;&JdY7T8Okp|B3h;
z^(^9P)R~CWQJW$*Moo>F64fK3M^vqdT2Z+p@<a&{V$`SbPmveHFGOw#UmH0ld_-i6
z@FtOk!}CU3!~GHO!=6T*2-_VoKWuD7x3C5gF<}`aLc@NBzj2%jk8{inAMa=$-oTM9
zJcHwR*mwKMu;ccrVN>l5!W!FCg{8N@ar_&)#j!uMr(<Mjen;idUv_I~oc&(N0Q=IA
zg7(%SKSDEv><oQp>lV7nmL;^S?Nvxl+p>_))<z-Qt=5q4*7LR;)={<uv50NG_{G{z
zTxU%yHnTnzLaa-L^I{|L(@?O^tCkPKB+Ck+qNSdon0=Px=EIhuaQlVn%lpH0&%DDl
z+1v#lCNp*DYvWqeN@HW@1CuF&YVNjyb%qS-jnz<w`u!8==3Ue<5)P~cxWym3<Lsex
za=?+31D`?yotW3tVWk}^_VjSXFX<?^8iiR~E*aqV-e@<p4fvC~fYapA64fv2ZgrQ+
zI*EAxi>S{Pm$F2OR~jl4l~AP?7?MMoDL;{G%j;w-as0Ye0$j@=*CU39OKYXao<7oQ
zPmI*rW0UfFUV0MUdpsxI<2_T{O*{?UIX!9Js{6C+q5FtymwT#fqC3{r+MUx?%<XZ8
zxo<kZyVg5zxP}p@>p5q*vJ<1d$+cb2lMA?xBuBUwCA*vhlV3R-C0}$FCKg8~FLk<-
zCOTgbZ_g&Rc5Y6p<(!pN(m5z8hqF~u3TM?Mi?dLYCpkmXkL1v#_sMGF)8t=?cajql
zuO&ZCyqJ7D@l5ii#Q5Y>iN}+VCLT-P&-LEK6Un=|Z&%`3#>M14iPw|k67MA+N_?Jt
zA~7NPY~s)4tBIcEdx=KptHf~Um&8m?S7Je@C8?q_Wl|Go?xgO{n55Cp21yH?os+gZ
zM<ktf&P#gc+?tf=Jd+gadXkjW^*5=C%aPpPl|Olmt8VgYSD)l~*PP^+u6@ZWb5KTi
za&jei8fRB`dFKpwSLZ(W9OonV5vSsQ@66^2b=C8faSiu$b8Yf0a^3cvcS)W<uAEY4
zcT=g6d!jVS9VZ=fzmYz<?Q&{QIk}#vuROuCT0Y{rC4cv5au%te(o*WEERYr{*QCpe
zSCW)caz1sS+(F$XFH%3sm*LqZu)qS^Vx^0AM_HlS)H_-&)u>NX%jg%>{<>G)qT~C(
zn-S?ftkv@V)y9#RA7-7gZ`3xKeINB$Ul#8yUwiKv`ne{eA<gNJ^0oFi@XcpE`it}<
zaM3Fxuc0FGdo(%oQU5vCbVx*lo88#O(A>DzFwOYHa1_4o6Rb`O(?nx6(+T4c5Rgr9
zgZHRlHLBjc<}0QaAX<~av-W}gJVSlsM^m30wMs)egAOJ4U4h2s6sxblB{wq&e%$i~
zg_^iLd*b|@34gGY-0r$y5Wm2Q2x1d42f1ujaV`1kIdP2mUR*93tOvv_)*E6)>wB@C
z)gz9y+O12iS*-i5#jQ83)tLvHTUBd!YqaejYhK$_YbD!aYpiX(wYP1zb)4;}b%E`y
zb))UNb-(R_^{nlM^^PsU`qK8r`q}o=`q!3dm2FO|-{!IkA#Q6ZBRs@qjSg{IQ-vg2
zQ-}PrrVaUOO&9Wk>(|yaAy2F+L+)53LoQn#A@RhT16D)GHmhP=WlglrvwpEnw7#|t
zvEH|JwO+P0vmUoqweGYPwXU*dvd*-HT1VJCVmI3dv5D=DSl)I_%w^jkhTCR{9&3N`
zowccW#ade2ZOtIgxB7*_)=xsL^_o!9x=ToHoh3+cGS9_2!f`RLutF4t;o>JtbCGo=
z#SNBlah&Cc(8h92C}r6qL|G;Xzs;?Mo95!ec5|pO+5ExM&V0gB%Dli5Zth|EYN}v4
zZ;G<4GJP=*Hk~rpH7zt}Gj%h|#?t2d#t`#%<7?AcW1Oj(ae^r?G0$hnYI<x)GVU^5
zHjXnaGd4B!Fy=9oHhTS5<75AG!yf-m!zBMGLkoXBLjke^BVE^?`_9m{Yo33auf4w=
z-7kvKy~~O+`ki;Z@0fQ0tJPJc8VvW9r-F|LP5wbg=}RylYj6P#rPF3T{VM91mAK9O
zvLata(CR3v?0?CCPij@Q`C2BTrK*-@RdWY!_mApQ^`tsPU8puudl54$gOaCF|0;>f
z1LdZ2SlOy9Q^qMHlosT}MU?Wy(JW{~%*ti?yS!PxCr^@(k{hp;tIAX5>~eqEBDavg
zk{{oe3dje@k(Wt=JVtU#U8E0EUFm^TOu8VYmkvs1X^ZsRvrKyKnJHcLjFI9z1Ef`+
zuF`Z*OKGU50Wr9m)R<UY*;7O+>d7Nz_hgY$dD2NXPfE$_j*ybwcIms@CMCE<>4jU6
z9x`q-uDPw!C3lE)j>pgNywmO!(kXX3Dc+rh>s;Jdh{sAvm)#Ym>+agp9d{Gyk-MGr
z+}(q14VAvSCrN+Z^Ci{2P7*wOq$tlRDYNI6RKW8_it+rFYI=-vb5AO{yC<(a!c#$>
z?P(&f_w<wxdB)4vJj><Ro;W$#b4?DB66CCKA2CuYrKwa*=_fT*rb~U4jnWL|l(bcO
zDxFu7rMF6?tSLq0^lD?doH|5qr!JGnt0&}*>MLTIU;YAbWY_8{MYTanOO0PbJFRTd
zJ}EafyZTovrKZt4sg?CPYA^k$x>$dwp3-gFzj_HR%-c~b?VYQ2_Qq>-ykE3q-jsTR
zx4Q1|4b@|OTlDU}hx#I)kr-FZ``y>Yo7%tFTit)rJIL?!uJh;i-SoHidHi%fHJtV}
zHT?EXG-UC|8JhWD8>ahh#&~}z<1c?VV-~|AV>81!<8;F><0(T1({DpvQ#Ru$Qw!q`
z(=6i?(<!6T{Kr_BGo-D#g=waFw&{rZtSJE;z+uTjb=eA)-W<-Xvu662n7>$Z;MQ-2
zOMMQjji0qFqEpiux{Q3KPe}@KIu7}hu(02S#bO5Wlvt0dc#LSb?iNc}pNVa)f^{-c
zZHKk9^|p1M^{@4$HHGb8YiV1gt&OdMZJe#QZG&x@?V|0x?UU`hO$dn!$rDnRe79Rj
zuaJ2m(?gDhYz=u6axuglnh=sN)ECkyG-K$n(6XT`LtBKN2^}1o5IQ&1WZxE=!+s{T
zhW%-15Bu-X>2`~KyFG*bs=cWFlf9PR;An5p=ooA-?U-h7;#gtt<Jf7R>^Nax<+x_w
z?|5Rr?D%MZ=18>vAZOMcHb+=kv?EhkdPkwK9OTaV9Sy>YI@*Sna`X%<>lhYR$uTjk
znqy8_O~;b3I*v7A^&FeS8aQ?`_Aw5G)pZ==u_Iws9Y@2;GfFWEGjcl)hh=u`4@>3P
z6XtMi3o|)3gh}>QVL$DQ!`|6vg*~)S47+3>9(L5;H*CATW7u+g)39mw>S4p|CBr(~
zbA&apM~0QLYmRL8uZ~drBS&)RS;vde9gcIM3mn@*hdbtkws#B+t>$PJn#WNx)Zs`S
znrN3op4eZ89JikiS!G`vGTJ^aq>a5@NQ}K)NNRhU5ND{%b}#gyZFgv#ZEEN&TL<#a
zGNI*dk)bJV-<c;ahFr0(3R!0z6f(?OE2OS9V@OtOl1&n?+U|*~Y}>_twlQKwTVpZO
zmP7nt^$16-w}ffd^+I#&U?GRKx{xHMA~*eNSs<RYv=?Vv@{4UO9wEQwlAxOx2zSkG
zgbn7L!a(z1OC|G3OPG0z<&CL<Wv?l<Wei7SUDIB3TGI&g7h`4fQKQW~#rV+F)VSJ|
z)!5aPXvk+ePyAkJIBD!?m}F!HE@F4I@s9tMVXc3wp|5{{p`5>r!H#O?weOC9uWzM)
z9F<-}R+GuX>M@Dlv#cO8ixmQ5eSL7EQ~=eB@E)b}%0w!~`nW;TdOPEKC`GpwyFLje
zV<TLl8T19(cdaM!x)L7#)LKjJkD8m#O_I8aj!$FMb!rQBpjuR|26B-~{i=Ldt|;e}
zEkx?cN<XEuQeCO6WK^=E-7}#l{4DR5Z^+Z+eR3Chkz7e0E@zP2$+A>kenyNwDy1Yw
zd!>J*FH&>qzEo0*mr_aFB-yh_dg~cSZ0;@X^t6!X6OTuDib<V3*`+$36jE`IKwMNj
zcF!M=;{NFQ=6>OM?!NE2;l4)BeBQI$eZsTWebBSOy_fuXyJv)Zlc%qHou{*VwWpPP
zrKhobxu=eMsi&%YiKjf*rTM=&_ZN1r@f37#@Dy-wW9;_icOPIJ<G!<=BJQg^=Pu8E
z=BeO*@2T$o=Be*?dYZd^o{q%(-o*KT+<B$R?igu-yN<NZ-CEk??nmsOAl+~;W}etC
z{cxX>boV_e%=1ag;*q6do-nzFCx_hHQ&#TpX)I47mM<fg@9`{?FM9UMFFcot=dWc4
zJbPX^?CMfJrIS=s86$O7R!NhT<I)D@nREtz`xU&lLOz;SZ6TLchs$l$74jJMxV%Pv
zE}v2*`IVYhQPpxvI<37@R-34_(Y7d~v}?+0?YnY{c>Y=|r0QB@HIx32T1j86cGAzP
z6ZH@3MqL0;&aWk~UcJftkCw~3hP?HH*4z74o9zwP_j*g|R7iTFcZ#0Ew^J|WyQjDI
zx%JV$jNUcA>csdy-Z#EQUN6~0X8$X1WxwCs*`LEV$zR*I$=}a+0bJs}|A<fbzwo6s
z=)O{hY^*g~(?7(}%fHYt+rQ7S*MEz(2*1E18;#M1jK-3NGRBsMW?&ZmjjIhaK=w8n
zUm8vsWy53Gk>7X`1ygg_zoEvG;28B_SUQ_-8HdCE&4C%*2&;7z)bKjGxmREtzhHcP
z;KUK;I8#>hWm8e}3lxICO^r|oc0je#7j|?M-RY;JxL%C2bOT<}JvdWO&>iM7E1%qF
zy^~kwr>qk5#ry~Tx`cM!4~lH1%YK9<FTM3k(2Kt!YpT=+S8hTdzd-MpF7!C*!x}L|
zcyuJItBhw%W6WVJWGrW_M&GoN$G7vmJv{$_r7-Ke6k<)7f{c8OJakpe!Tp)(%ax9`
zV^Z?maLXn%Qftw*uRt%g$nx7f7d`xRG*lBTx4>4<n}?vQ>Sx)@?6=w6-m=2n+%nhP
zz%mhy)j#xh?2RV91C>BC^yszd?_bU=S_-1c&&(P)QS|k<@L4^kYpmCD0*&Qv^DWaF
z^C=VEu1w?L*anzqnL3$Anwpq<nyNDE7Bko6TP+1ooD2RgC3vkFZY|k(+mv98N87X0
zbk?}swAVP<#CmDW+U-pv;L5riOOwOWxf+hF0K8^e_)ZbT-$fq#(Qv@{z_8wU*1)=C
zhSA0~)bn!;EsSFf)r@@&MT~6>8H}~TdCQ;)%7a5R73u;r9-$=vI#y$yWw_-ZW;pKe
zWY|W|yVPIGFa-zrV7eD~LLt-;#bHIdX%s|dm=O(w1HZWLn@RVgVRXUmMAzH;{<Cy4
z+Rtj98(}*Zv8w7MR&E)Ld$2PdJeq(TRDlaChEp&H%Ar&s2_hN_nH5{U)7|t99bE2v
zBXGL==!x|QAJ`jQUbkVw&d|wa4_>i#bP$;j_ceiztAlWeb*5`p6C7hzyhXX>_GSfb
zj>b`MMyc+?!}o=L`Y++aZWG1N;bS|jKf<+t3rE~VeKDByEbf`0@5Rfuo&E<K;mg+O
ztg?c?Z;4(Fyt=GDo3+8Gp&yxqpKm;#x$*RV8H=W56#l&tc<zRwF&RW(fPVUSc(hOS
z3U~we^pu`b_u<rTX@UL-?ctr;)03;c9?v+&IK<e`WBVAp89R9HHmwuue0QcdRadmq
z-5I?YebCJJXAEKtp|`?t9v?+tmvQ=bZ4zAFbo#r@(F5y?$Fc6p0W{Tz;kk}!yLhkt
z^kq1zU(!zL*R_lK9qoqxP`j@`)1I-9Z`s$+aA3db8|I-JF?db*_(JhOM&sej0H2r(
z|6UO^GBJ4fs#3w!qrZPFYaDc-Z$U5m2M(pT|9E``9NA_#$9>){`bqBr{R(S-KV&_C
z_W?8r|E1<t^iZG8o6eV(epq?DWqoDv=GCDmR%?0-_V$kSjrPv;&GD|J7w|5)vv{=J
zw^`Hc4gH0GvzD!&J_k|$3}Cc{VI8W0K(&H9>+kF5pX?j&U*TKKtiFvN2PeU9Z~K3O
z-x}f2(x9y=h_<F0Yjn5rS2heJW={9_A}bgJr?voAV++0Aj-#!;4X>7f$Iguc$IjW2
z)i?xBZH}RyajT(|@r>ag<5R;N;~&FDn3ChDx$ofT{)iq_!SfVhnrX~s+F~rvS=tml
zxHqce2`IK#puX6LBI6P~+e?%ie__!rpl<15XNut%s*OviJzU#hI*?9BL$VgN%V9Jq
zH_)lP4R(~hVzyb{q3)D$@kG+)DewR5>ny;dM&GqRn#_~}3yZ_z?(Xi3`{G`txD|>+
zacGM}p;VwiDO#Y&QmnW;6j^+6x7Z{(zZrJF|M|Y_e1~2y$t0P~OlI;vcHj3yr;_^Q
z9lD|ZG?tk!3^l0DAUltdeYlHC)Mtl^W>k7q0t+~+q6^y^rN04auTCN3yqt{0HvUfm
z&6(;<Li6>EQ$PnRm++2yl<sUSq!qeTV;d!u66Oe?HiZ_#HlZ7Nia_C-5G*_w<_Z$|
z3b(jH$SEdL4?9eq?7Ywt&%THFR2VA07bc04ut2nl5u!g{er9p6_?viKEG%9U%ZRDu
zwx5W##kXQ3@tfFO^oVUlhpU5_#?_hJMmI6LtEZUP)tj7mU$MBWpIDkv!4)7@aSafw
zGitJ4$JJkK!1hM2K4N26FGdeWH%4c%p{t`<-_=H}>uS!ijl`<1x;($SSjtsFEaWOF
z<|b#F#g#)$=gJ@=(#7km!dLO5@J4(tJmNFl6we9g_)N*<V|IwKLaew<SSHRAW>W_o
zCk_z$i+>0m#HK<6u?ls>!a{B_qu@^-LlxdTUvM37pfh$9{OL9@r>ljz&RN1(Fs;3u
zU4&-TGAlbv33;7a$;+DHYF|44a$Iqqb|g7>I3lU_g^;Hm>>T81>1^#N=d6e>WG?We
zcE<z9EBjH$Mf(OvqJ1tlXs`p6lB1`+qyq&8M^So;`P*+0sqIB)VWs_mZ2~pFuJ#$=
z_XnVF(F}`S8gy1h5cHBYnjG(R+fi#D+Xk{-bFBGngTTQwM-QPatb<=c)<~AA)_ay7
z*o>Oi7;?WMpx6h1IBN#4zZ7hwUqDZPGds~?wz4IeyDA3c^=wdVfuJH<g3b0SvS$Iq
z?J;V>tjI%c(1NPX6Y!xYP`ZnQfiWA#?Et!TG=ULb%rHUKJ!gKufI`<ERH4@JHxuS|
zAR68sz&X^`OXx-P4E#x886J9GdQLEJZ(#-xN1HE*`MH;8CcRHapi$KweVS%)ovYDT
zt0?ns7Uo$8^Ws-+IeIeFz#I(GPHCOBedx<<(kh~il@G1M^eAWP>TvCi+8e#q_S#u^
zEeF)<;0#NnSDRm3q-NEol7Sha+O<Bas&-JnsEyUvYBlvSoS0O#fO<vEuAU(qa}3?u
zKfy2Vg@d$HeWfI*kI;a<qr`x1j6@f9wVI+VR}Z5Pdk|gQJ<1|=hq6H30yk+RBUYKC
zMxk{Zq0CfQGFB)d>N3U>_&H11z8G!UaE=e<d5d`dA~-vt%2LL1HB4E_zBL@Pj_1Vk
z+>PogC0<>lY**KTtBiru6Q?Ar@yan?^AxXrp7(GKzE7%pMtP)OR$iz#l@ICz<%jxQ
zG0?3Qw9iU<O;xgKb~T^okIrKzG-`iSYiNblhFTeTSXIH>)>r##E!Ck~XEjLct4>Gj
z5tN0x0xigBZ81EcNOcd_@rbrZJqJJN7F@2U;5OfDk5!rLZKpn$&SR?CJ$~RfGHazg
z`Ow@c1>UzRnmY}_ptQywcE$DuVDCn0lfZS(2FDSGcG@cJ-g<234!+}q=$M`0`@ZaX
zLcaI|c@;(bgiTiAxI6XVz<3t-WY;Tu3ShfR=`GQ1>_VT-08caewzbzMV#{X3D_RV`
zW34Bc-0}=P5qp;GS%yZ>I<x{df~(m9_h>&{^<!|=&%#x|g1*2VJv*7bLU51DqEBBP
z4YNjIc3Q)k>!SZ+^rs>+0xxDFwr(~aO}JhiE?om7hF)U{M45Zg&_7K7(KC7=Hf}U}
zP7|<m)4`R_GZcL>wMcJY_E@UvTa5g0(^1hks=`8V0Oq<CUQ}23=>cdmj6j=VA{q^|
z>4O?>xXcK&c;o0fumfHb9ow;omC%Z=4HmVrdEaPhJ~uj0z3GOIdtZ914n}Ko6gm*W
zV7q6))0jtfCmgKzYV?3&L414ldv~KCk<1aN(U`ajuf*Fs;wgLI!om1PUuPX3P@vy(
zI?HWlgeUY-eoHT>AM}H@z*J8IPbw=&m3%N5OTcZZ0tcfZT$i@KejVOk+)Gi{SO*e4
z9vn*|3LS@F#+*X+<1!faR8&Wvf>?jY+IRF#444@rh^`EHxw+tG6o!-T#lBVp^;I7p
zx|e_635G^*I2wU4s>i~yng;uLJ}ix8pa<814vYsgmI%N25ElFtzVa33zq{ZXpVFV@
zEnJ|l)-+(zvVuv=4_=})9d5k(GL6CXwFS@D4QoCCOxq}U9FwRL%mRZM3T|%&);$ue
z-UhZL_&7;R;192~oo4H0P#0eQ@B`Rc&)^Tg1MT-69+m+jL$qHeqPhh}^uCY(R0IxI
z8FX|iQFW*ZhO{AZL^Jrqt??2%fHCa~Z?!iuMgaKJK=2;J@f=3OB_5Bu+eEOclc_vU
z1-Ci_g|}I-t!5LE%>{uw4_(~_jD?I)Ft_21#f&A4rM}USWkVh}V0(R*^;y>AxVqqF
z>+qb~JijLV+3GM)s`|LglflIX^V(y14<qdb(F@80+BPT5<t)6%3}^<WB^GpnthIrv
z)roT?TQxpgdHZWyG5b^Eod@vGZh{-WU=!>q;0+IfXWUIKGJ)7<J$#1<u#n;4B4=^c
zg279UWa|J>jy>4g4rFpu5QufyTABE?G~Y%caGBXr2ujZu2k2LY==KBH&gWn~?}B5!
z#5bG*K6D>UuLK|N^Z>t$1hCSP{8GZ<<A=ax_jX(z1bVO=c*531$@O9BRz&}x2uSOk
zbYJpE9YKR3^d6m^CqzzOC56*)yY|DY*a~kq3Phn-VI~9>hVgI%1L;NM)r)Bh)1nbd
zF;!5`Du#mGZ`jlfXu&v9Ak@tC#O(syF5OI}*ULroHTTtHI8JxK8eWFEb&9&{Vfrd0
z!J6{&qGDlGMS#Isg1udcMtlgUoXK#w#?dQf1o!b^?q@F-tDBjPJ2^c{FfMe8t+1|C
zqlWp7=<I{9r$Z^`gyLB1LPT!)nIrO$U--?ujh69E#x=$jUk{U<=2`S%PBTt2Qs5IG
zGqRaS>A7~8k&G^ecX-#m;|TkXam;bw@ws?z9`ho;`xSbm+@M#=9iuF-=k4!V9d2@6
zaFUJC)M&-L)B#<MF4XRN&;zuu=?CvDJ)blaD9fDi#`42xD~``m7G6qaIBd0vJR5T@
zT7k*zgwA76u2=xqX&Bcli0d{5EZuC>bVBKivkI<Q3`ov+uJIoF?i^;$Is;$q8j<aN
z=B&S&vp&+dOEnwvowP!qr!#X`A9#3yd|zWw@R<xcbT-~l7!18t%w{p{NnkeH!~8{@
zjkd-Lehbkcr{bBd65;<HM5if*dFvwc)-CY*e_1kH|0Xx{-ckT>s~GtGGAKM%VlJ!6
zZ>s@Zvu6C-+OnlH*wvosIP~L}7zkh7+f`sZkEVcc^=eGaM_(bF*=+?pw6#2nwT6MQ
zUJ3?!CAjM~p#G!5KF5K{-oo#F2l(f`@Bj|t=_SKcJIZ}<g8A<>Xl!pk$ji)z*ZKdB
zj|1st3futKeT#eL4)dc|jV+Z*X)4MKULDI+xRR+X@9_9G%UdjOvi}CN=ry1A_G8ZV
zh;u#yvHg%&dC1-N7q9sM-1uGI!yVqmEwJM^n2oRV&aN^iU*=t2;Ju&Wjy%C<ISOj=
z5Oel^(7k)PPZRmP+rj>B0jU-TA2XV3u@<I71oQk#G$@vGox)&0EMWeh#nqaIQpH46
zD}uO-M{`|=gNO{|o(|y3_Qqy(<0^N=ezf6=x8UynovYsfi&7hP@alA7u7L7MNfc0u
z67dy4fh#ALClk!hOnh_MtsaJM$;jh0e48R2*ByMb2AE8l@BbSJV{iAicYO1&_+2~)
zulWf5tb5qE+x#l7Gh1AO({T>w;b~@#lgt~(U=JswaCHD|@?L&hyHK>+W{F_WN)VsR
z*|P+OaTvd^MeJF?p8045%w_K^_J%Oy%;bm}K4#-WRII#X0b?$|+gT`D%>?26Q`st*
zwefJjg7`m(zj2JQj4`Z@W^J@@IgZCctdD0ofi2$S|2=vPTSv2H6w8tSj1eq{bHq@N
z9l{9YOh3ms?EE-1YR34+&-HO&t;Z2vjAe}REdyBg_l>di${vfZjaS=xG~H0gvfaCF
zG>=EY<Q$Fa))+XmW8ia+hAlISS#=mJw!v^b2co<cz&zUzy@X!euRZ8O?CtsW2lH-w
z?%}rd{b~cYyA`b3=IALjCHvKwS-1i7aa|O-YQr0?1;@Jv+FX^%h?S=|WodeQ6(RD;
z&m5f#|0XLv3^I@<^QWhci@tw$@~$TARFygV2dweW=yZAgqrXw+dI;n84!S>AVcB|n
zX`Nse`x9>KZn(Hxn6F|%jz<tZEoB~CVAkY_@_1WCQS!<IXDACEm_OP=Rv5BB$a%ar
z>M(CsW?nAEJe(gNFAI5(G-TPV<`SPD_!bSL$7nm;gqL>)#jQV4Mc#!T*G4pw*T7c_
zLq~KbmFn^MnL|Ko_BQ^4D}RL^m?!j_=!up^OR12tmOM=;(b+Vx&0hXf06M*YkcDeT
zf1{f8J}M2TFE2eKens<KMAuitXMRt$<S{v+8}v**g*L=~aOnwXZAE!%=}SH3^*Qv1
z4+dL344)j8VRA?cddeT+{J%tR>LIzc8=!^Gf)_f1uI4^+Q9JPG<H%2~K~H%xIjcGN
z`cu&245IonlDyeqa$kMXw(kbtt0R4iTN58N^Mq)P(7CTme~cQQVZ;;zwep_c#1~z`
zuy@o7c-q1dYpLb%G$lrCgeG%6xP`UAlULKyc&eaHS_vLm1<m<?MtS1MvgnnTV_6aG
zc|}e1jmjKXP4oBE1Pfn_>V0i36X(k6X~MZ%fJ<+y<@NkQY|~RK?&*)F=wPt)!>RO-
z0iQk*e%f^UD$fO@9!8wAQtL;KFOYb3G<my;o@De@Pidj_QeK7TXe<?{1kW37H(32c
z#JH!(J6`c*C!<*ijp}mflGP^OY0fq6%(V?5Z#TwMRG)#iSvY-1qdhIj$92<>@V#6n
z#(C(O%XhT`-asr>u3dbONA)sjYt`oaZACoO1KsRlx&SURlMx2zEt=oNE;JR6)934k
z-h()02t0*gVwAbWCo75LH^PD2OJ{?V^c=fMmiHOja9`;*Wj8kykL>4ndKO>fu2CO5
z-xmAci{JEc_~Mh%!(E6jZUp|zCfMRhcrPcg*VpiM9>W0tfHn6JdH4}qWJB+yFunE4
z(<`|)eHoe&Q*`1!?#rX$sA5fIKA6MZzm!=a3e55rIQB^}ypOSV9)|Y~qKb$7|C%h{
zS9~`EE{L~xaz?ZkbMjXZB}T8VRTV5l9eh5onpGRDM`ySsJ+UeQ=s6AXDOUw!QKsWB
z&gK75RHBw)eIr<lBFfl+J@)oz+=1QN14n5;HY^!_(sAOBQ}B|`v*$9F>?#)R2E3%(
z*yg)f<$M1O@A`kneynT~wsH@)au*hM2exu65z7`V>qac=dTeDhmNk;|ug0pb;I)>*
z#tP@`3wg#otm$kl>2$n<N!ZHq*vV16%ONNl1Ykvb`E29_vY2Dw&y2tV2BKIQfCc<%
zA1e|~mEkHB=UNovisZw3=0wdh3tsWBs2-%la;9OAKt9yQ9s?$VihcZHNpJmvUHm|O
z;Vp{Uuke?jV-p_}zdhg^NJSOl25Vkrg!A~%r&&wE79Pb8Ci_$-5Aod{z&?8WCGEjR
z?!-oVHF37#S#8A{#-jtdnX%EQ3Aq`Ixe1H83EQ}dZ+{aJ<z_~_Z$z?;<hkqc)uK2<
z4Ch;qcNT}0-Ha_x;5E1Ns)^X>By9DcSmvWx=o7rVv#4lZ;=NyobD4@d+9Q57FHr{g
zU~yArNry*}8P6aOQ9)7o)n&OdRe4kwjkMqKAKG%Qy5Ljv<@ycfS_Yw)HJz*P?Gm*D
z4<?ph%{IQd1Nbv1eZPvscuD6_zP$-s?Wxsj`v@aXv*v)US%mJYmB=SIK)<3bQBO~p
zWJ8EYgJE9HgTt``W>ze0t)0Y3$tY!<CvHl$T_u`&ME8_;FuYWl050OjY;Z7(64O<M
z-O-eYvNP4Xfz<9M5G~GwDY*)D*v;gjlVE$C0LOd{<*7$>;{E{ZLnj}dhWc4HM_zim
zl%(=jm3(v~dSZ7VGu@k7-B9$N#>07^;aE&|I*JNg0vYQ4ROC_|hhc=Ab=)Uc{StP_
zSGsuVC=s~mdGo7NgmwQ5?E7r6@bkmQFX?Q;T5D$wdf(KCDbk#d5AEnG*2Ou|*^>@t
z{XtO=afUiak<lJc=ZDG882XQGgjtf{Tu6_IaJVH)o%`AQC!Hgboe|EX9Ce(IV<#A=
zoKa-Iqv4)Jp=Yp;WhDJ1yki~PBb_IG$9kW$isvo^s~twhd?6YL^T?rxI5#;b(Q7S;
zqes$rEs!Jn@m_k+r{WLiOf(TDp(!wi_Z`Ul@8hh%XDH6r{A|rZCyVsXD)f6O=``qh
z@r_*XYxDyiI;_qcj&EF_7mj2{D*b}aJ2tSkg1Y5g`kMtimOF+}`|Ls9zYYC$>d}|H
z0{Fv%bd}3Yl~bg$DcjxTixt$uUZWg#huZ5Y`zAE7mZMJ`LQFpzOnM)9t!?<$>+s!|
zM(H>QELH(Um#_Rv9;0t|g?z;!)F|T7x>`Y;J%eA^Fc^xRVe>UWslGIMk?gP^?Y1A(
zkRLNcUBFx4N4ycs46~3~aU9-uPyU*44|vsXv*T0Sh;rVN!@0>Fl+2wT$6X!D9T~)&
z-GlkN5wTWje6TFU#2%l2WhJNd7C-6+dWna~J;mV-g`w_0p7@r!IMHreqTC<Es(;ZN
z;uQTdw;K=0G@l{UoJgj54fZI6o;yR)0P2XX@@fJV11Fk^yqm??gq;f`>pUI}p}zWN
z^a+;emFUKpn@*A<%G;kj^{88wLbEap{BR5Y%R8!l-kv)t=%Q~Y%e)#6U5Lk@PTfDW
zuAZk_6Kr)QYHj(*O{PQhT%(u6E6^b~wYJ)E5F$I^#jVq_(wV_d&iRuXh;~6&^3A`K
zAFZmzs727X$_#hGMsJ)?uw5Rj4b*EO{Zhb4C#rt*M3c$+K3Ary*OX!EQKc7s&|0gJ
z=sGQ=XW1kr553AV(A&(WHdDST)sz=XapkU(Q@N<5Q4Yh!->JNpW0gnpa<C?Ilp}Jm
zvQr+WM9aOEC31UZrrcN=D_2(n<kCtf^b4BGnUxx{pHf;j!3%zuv&e5{Kl!n&OSk3E
z(j~BiC*=FmpXeDRq9YJ5ACjWbGFTyRk`|#iFbnO0Dd-mj$sy7Rd9pNE9xL^ehe|!<
zeo|+-yVPFpD7BVbO3mcPQbW0pR9CJh)sibn)#cJsW$+v2<bqNuaCyb#-=xBFE~${5
zLn<I=Wn_}_$r+{mat7AZO9kb0>`Tir{){xNr}3>9Wm^gMmEyQEJg1zTMXD%gld8!%
zrP`dcft*ijDi@Yo$|ZO&Wuz{0Io?}kslQxX8Y(xE#)37UD7TSj$X%p`axZDQ93Vx?
z!=yMlNZKw>m-eHva7<n)otHOoJ$6Y?<z(r-d=8z5R4I+}Qp&D;lZq%d^dK_GP0&#2
ztdx}pEA_bMZRG_@UwN%E2EDI2@)2bXnqJ%Fr)YeAN7u`(zLayRnp_4AvW99grL$U(
z@45?hfKl|6TR?4LgL08RbI;UNMN_{hS;)<oB`4oZt4A)rJ3aR%s4MA}v`b4=FKCz5
z7uq}3Mt(iFr>Is9U$+Aq8lyck@TFt4?eIL$<5#|<cZoyKPky{1nz#e#XS4vM>JCo~
zI<3d3yF8|nZZql=6AaXwP@f%)F5E)&Rd*V?ew}`B->K+iHAj+DT7_SC#F&LXYCQhk
z1@6vw-Y&bAS!OxQcB)2qsJR+u6!BIH5!g$jM}KRqrJVJg<qy1tN%#-Z*vb>c>94Sf
z{<fpmvbLAj4)n^Hz`PSlru7)?=V#b9k=ki-GNay(6r-?TE68^qK<Dos)=H=MO+Gr0
zH3CO62))7i^khq*PuV5MBgY4alb&kD>6zKu*^2H#!<pq5Igep6o;xo%-NGAcbZ()o
zP(&CfG!|A10U#JdgujFsIEBg7`%<ZQegmVOL7Xg>64#1N#DgFiQ^g754{-@d#?4|G
z*HN)080?|ouIIR1pc!+!_PZ*%uDF`JUbuR=OxFlk7WYh73HNeWUH1l8C-)xL5cf&f
zH1{o6xciwa#{J2)+pW7!xc%HW-C5nw-38$Jlm_ou&7Hxop*yc%D|bo1&h9FHz1{Wv
z2D+R3jc|AL3v&1Jo8%toH{CtlZ#I0O1@6gyq3-E^VeSyWaQ9rlCGPotOWg}uF5+=0
z+e7^pu|ALOA?%;Vag*Fr{l>W`_zicD@f+YC=GW65;MdXJ&98;KEw9_yue!U2Uukz4
zzr5}Oei_|a{Ggin{cy?d7hnVLz)v~tI`2+$9pE!<bT4%+;WJKkk8}mPyYm?vyXv{i
zx=OlpxH7x#F5UGW9C@mE*_9$Dfni_k3KwU(g21!)<k~eB%ehL3nW)i9VBznI_k};j
zW5POdqcBaJC-fErh5GO>3yS3hhnPwDTlnrgMQ^4!P^Tg24D}HvJL?ENojHZt^fCR-
zc>^1}6Wcn^nT)Q{8c?ef99f;6sFPQ4oTH+?!SMmS&jomto9yM`Ql@kCLszLD8cVs+
zj5KY*Xrr~UAG8&*uRy<H5*lRP$xaur^`@3u#1?B6Z8NMN$r)cFx3rlUVlJHX0eFXX
z!QSMu?!oU}g;qx}zU3d}W2%zp$VE=VB$|0bmiGjmw&TzO4nY~D54nI^W>r{c8R$Iy
z#<*{sHV&h)69sZ+3hKo@&?~MDS|&erAve9mKI!+-#7?1i@m6%OSL!iv#KZIf*svBb
z#mmDK&rYW!yIxNJK!1-rU^)(azQf6RM(y+_w(OK=kS7VPy*N1G%jsQ#+8z~ElNj_h
z_>-H=6-TjY+v#HzN!%O?*Lpg!?|4nshHIa+f!Ma*)K$B{({2m5y9vDRT5!56(4(?A
zedY4Q=*&S+(Dd37G+O?I>wN$;dlH&GJGJj>0wdlxK2s-rr*5SlxQkwwdl~z9ykGmw
z@jui*dCn1@ds;KqtBkv_LZ5KH*YL?d!vmMKVsOtZ!$Yq_EZ<bi?`fx%^YkG09|-?@
zjMjzs9Kd@Y3HN&<xSBcC443n{;;_|wJ&&jxe$j4FD}0I$_E)s8MZE_3iRRe-ZrK0f
zo{q%OgR%V+(Xw8o?_*9p>)ET{_nh)^zeRkAyt<Cxkilri{MeiPNf3RQL-hc9vIHB)
z^e~W2abQ;t@txgZu6#$FESRnES^63k&9TPsW{A<#TxyIm*Bdj@=3EAHY6Cg(-Q?$~
zsu2fgM|GeqtlMVj!}mx3Z4w$Wi_N;AvN~E0n1T36)A%JVqu#a&ow$SKu`ZCgdO&{q
zBe^$+brM<1`6wVnSn61JSXx?7Te?^u!fyC#8AseQpNd8_RkXcipHCC5+#$|=NtCFN
zLrr5_NA)DpTGEyRuI0M*cT^cVqlPiaW@A3iO3YCT|F9v^MmOqOqwyFQP|sQq&gB3q
z5tqQZJhbHkA6OO@!MgaDtx!to2B&-w6{;YrOtYwIErnAP#g<LzzwE#_-3PzsD0(hu
z=}~cwz7_ZF7wmuY|3~{<Vh@j9AolnbZ#Oqc!6G2~$~sEWYqApBW_2CS_-h3dy&bs0
z&W@h^^@Wu_2wwU~Vv-5`KOGg?c_`N|Mx8bS{j+Gg(rv`I-s%ViBe)FBuGMtrTnmmS
z*6|#a;2V7K4@53siCYv9HXd*`I!AdJ8lBh_I$(Yys(H`;SB`yXf9-TUA=bG^)N>QR
z{VMU#CC-1wF^y+T;5A0k|89_D9q(;9jL=0`s9BC0yqnV4sl2@3U+E0(a<p)Gz!-kA
z*Fc-Mg5xf_x0mdB9LLeTO(Oc*hJ}mcibUEUbN#MTkvmEC?+_8(Pp#WVT+yoRDMK{p
z)fvu3T`nzNzXvAMJKOK}$F}O|ij_eBF_-<gEv<bIb;vk$d{$6%n};gRBs6D+q1n-c
z>RSuG(;DcR6h-4C6Z$|l9(}eJvpq&9#jBn2r}YPUjVELkE|76g!takIbFhet#W=Dk
zJ;)3+Bq}atl`UD&-!O=+|0brtKop+n%U_QrC*7S~aU&G^%8;YYK~BeM@q?-LmAcIX
z>NuyE9e0o$T}|C*CK=N}>R26^C2OIvQJ9%BJt`ZD@fwWD4fH&Z5Cd#MS!0DUnpqPp
zw^7+>OCPbShCwa<sV<^R_5u9hEjqv*0yh{#@AlbxW$-3BK@VDCF2118=0!9h_A(zt
zd+vk$OYsDQ3+xA<v4v*=jP=Q$+#m$qsNhS?kAE{iUL(JE5HvWQk74j=^lf_$&g2$8
ze+q2JMD%Q<nA7HKH`Gz!AiB^$vkty~QEj}M5!{5Kwo>1~Z%<W=fFt=842cPD;|*Aj
z+h9FTpplh`R%<jpj~Aj@8;rjnsFcCa&#SgTtG1frgnObYX_b%45B$idXw%+6llGLn
z56!a8N~|2Agvnvb6ny@XGU~VT2&Ef7e|x!uPq(Zw{%LKr%PQgfmqhEUfLvC|CYMsu
z%Ej<s3oEKr5dKSE<*k%Uc_w95?n%EY*QNBzdC5;XDG5q4nqhk+tCApj@P&V%VfIO0
zC%t35{b%`}c;Ex$6XOfxJKI$`Ml#SOvng?sQ`s!Jl&$#9yQFlSD+5^VY{~&Czj7EJ
z;Yq2Ya!IPDz~$%ln<}5B4vIzYqok9ED>>y!N^yA+yzpqW$#yHP<rMVTt|$Z1#~mxb
zRA$KE`0REyQvOv<kPE5@<qGOKxt@AoZlivXd#hG@H)jN2QwaP_4YY4N(yMtCy0;6J
zm-yKpbjg0zZYm|U_evwp4L+y{{&fR-i}p|x(V4pd#^nXtw-!%|n!|HXt?2opw(_I_
zuTmI2l?LQ(`=Bv06|Cht?J4<OojzIF^zT{)Jv}`rihG9XO+549GscpmP3HG@N5Ad)
zj{aCC@R60^rFS&CfeV?ehr_JhLCkuQ9-^<%5pkLs$)}c}X4(iGM;{QkQ;qp3FhrS$
zj6LQ(u#d9wH}z7hr7X3ImLRo;nyoEiU>^6HGpSdsqt<+oit{bJk*`#wGk^##V~w=5
z0OL5=`T$;}YKbOhKS+-67K&nD$Xunh&9WA@MOtgx_E_84&cV2NXdP?&W}QPGU=>}N
z;>kxHL=WkbZ7>;x=_m%RrfY8^2(<II(p37rmH+mj82V!aM%&AR9%ulfpd)kGVCMa4
zcw5WChsT3?JVclFOU(6u+5H@EP&1e81wp};2CZF{{6>A|{1)UoI<loZxu?G1#RvHO
z{MO8?9mqa)X0Gqy7(u7ek<6(h$(oF2jvdboKb1Lmwqp&mZZva!3>F}YnRu-)LlwpL
zHI7vrv4kTRVI}61!J3Vd`AlraRL3yZ29nq6%?#d|v$o?ETJlPb*;<#^s=+&{LI>mu
zymlF|nI$=@C`T2bD}8P{b>>1PB?o)Gd73oDCRQ?NKgiX5z^1)mF8>QFcb)%Fq3gNN
zevT_~m}}$B+ib-mZXmje0A&%5rJMyqG>9uV0Ijl)_^FM+d{ly+T8t~37mP>_G<GwQ
z%Sj9F+|FDtgWZ1zlHv(kZMRVsyNI1WP6Xxcv9|>!$p{pg7LYld40dHWR=g)jik9Sh
zYJ&PN0oEladQJZHDKtQse}NhP0+h^M;<NMQDG!mQ^eTMCkh@$#4s#LJ*_mX($CHm9
zN^gaJcoE&nFn1)|*_^1ZE~tvi<U&i66D>kbHa9+VCbqiqJ`A`D-!0{>A1x)w6BV+)
zwB)fqwfx5KF_-l=NW{xnkkgi5tVf9%50ZOL!naAZnB;9Wvb!?*-0xVG&*b~xQx|=M
z*Zqo`=qqZWZ>W;KBg^!GI_O8%-(z`RvwY>-zL0w$-138CCG_<q^2Zw8g{;<1)CS_o
zVs9b8y9FO=3-2QVOmKpw0HdIFtEI4Yr=_@cANNkOrK0tirJD65KGqpaBbf8QTd#0G
z-L|x)-rtGO(T&g3o2XzQmE)1r{Da9C&)~DqCs(|b?D{(JC0oe4Cy_Bb%Jn*f6}`eW
zyvva<i79??loRBC2HSdTPFMhiZ2O6gPjEk8u{I$RX+xa+hwZzy7kT2LX!}e=eQX|T
zvn#>FY{Wl1fV$WPl)CPtBKC@$-Z#E?3yNTA(W}T|PqG!XA0wW==;L%$@!hX);dOkq
zb-`=si+b}w`>*sO$W5$W6vR(OI)&8-Nz@h<z8<KS4j}R#$**-1%;-7%hQsMjwu)bJ
zG@e^Lebe{YUvY<jLYqb=Lbo`c<HvnK8$-2cCu<3t5HBtpcgJtsCHaWk3v$;KCZk-O
zJE%1GdO7;tR%X;>G+;DkG;?(24(!8G15mXc$d>+iRlRxClPqa(?)?Dn(Llyf?$<!>
z;z8t12QVA-W4$+CU{5@<Zp;sVaBp{jSJIx*24Ac-+uE`0#OS~=tr*RiL7H)%#>_DF
zm<<}UtsTdAWP1zdga+)Z%WKu-mCEx9#h42U;D_eoy=CUpq~|^Rag>!KBzq;eCZ+N1
z3OXL3!*Bz??gDY=ab~=IT$vqQp?KIa8}ND8`h2{bWaZDGP<W85yd6(2jy`Rxx$_rr
z?@!?y83iV6AhT-^zO8olkG4j9gH^zk6=(L%gO8WdzK`sGJX&t6>07ao49ZM&<$_`M
zz>OxeyA>?eDt;}qVEc>(LluCp*8v7leXvtynCWx!D-&&pu>@Pl_pYP{6~Y}qimFsk
zez`62{Ay7*Dv7Pg&TrVk+B>)<x447g<Z*}IBhP!1yDpJ>e+(7;CDh)hQ)3@V1-=*Y
za7!YWs&vuJkKdQh@)&f^RWiTH;Gj06Lc4;_IdjMXkEc2wfI4-1^1$`Y&Q!_kkp(V+
zrOO6V$b}ZscjF<I?+g4Lqyu6CDr;+ug)mIU!7%9sTCxeeld@F8a~OrGfB!-a)1yDp
z--15A57+M;3bp&u`HZ8p!4jh9sd^oK2uzYLdRn>=eWg$4Jvb4^sk3e-UR(&1WF-BP
zI?+$5HtdnYo>BD7MOBpA^HZ%1+V;8d3H_)~)1ArJpZ7X3^Kp7mCTjJy7_E%9M9Zno
z)EwG4bW8_;J?^NTRO{1Uy{s0YdV8Oz2Xk!Ex{(WRran-s!YC}F9#k`_3Fz5IqR$@=
zUU#N?UI_xrI~W{sH}JkK$Pw07Co1Le5ewrZ=EO(LfNvwHCGZh*C?DX3KT{NX2)vTd
zD)8OtE3lnj1MBHSuu@qn&sS#26P0oDP<*J~_=z3KS2kCg$n}+4at)<|TnT@x4EfAr
zN>;g`;*UnGMb1U8Gb{dK2K+OB<-X)ruHw_3kSxl6NtU<c_idD3;uk(b+cp&r<8`!Y
zucGI1l|1TId6INRo*-S6$4Tep5z-lXFrzP{2ct8(xSgcq<cg2U9i*djd+9La5bNII
zUH5K3#r|{hALOJvp{LscZQFKe2ep&#vi~0Y9+1O*OwRd*+(-H-50E7C(7HT|_c)IC
z8-zbQUdpLV!QY%k&UO(#=W?mM5+&ggN)7QxTPjDTPReDekMfr^OnEH@E8nGg_{I_B
zb`z9*@<DRDXYrx$Db4A{&`H+uvHjr{W>-V-ITw-3T_M+0qvdA!pdH|h_f${GgVmdI
z5E@1^)wk$j$a1Xer|ePl(08GVa#L*s9y<{KbB>Zx+lWTSY5dQ>mA;ylUKqL1y{$@4
zq%AzeLFx}}hWd*qQZ4E^sMhn`QM-A*szIK#<adj}`>6*Xv4?gCEW{U2gqBX<uNBvC
z!b$w1_0coXN1>c&4H|d*zzW=;cgH7u(6q)aPjNVjO^i(BFe>OFMms&m7!5aZiGBwR
zz-Qx{p4R-X7dLZ&C$0^KxEK7yY48(c;3uXSkI}F*smtWHlrw8t+MC_cubXU%fKirg
z9!BH#zU8YaqpFz6S{%FC+|txK)Y8iuY6&7Yv6zh6HuAvdu#3+@W|`Q*oVGDA+`{41
zY_SH~j$3EiQmw0)mA0Y4d5l`<E&R?`cq=NNi@!Y#MCmq=v*&Cr?ayrib_1qeHX`Bj
zSfQqPRNcra45KUhbl8V0?M|3|86C%ntW)i!9G~sAnW0+~h4;qRjAG87$=Wh(#(I3g
zM102M#P8P}r%_M8Lq_0*!{Pkm$cQ?)w`y4oz74DmYJH9Q>p-2dCw~K-gQ!=IqAD3o
zHF7Fd#SmvaHN+$z2Pe{b$;ZRl>U=?^@;g;Zt8fJUyA#fg!X@;{Q_)d>;w*}8NEz7l
zRpIE=6Fxed3E!OU1l8G{rjP>!K^Q^4dOW&wQ-ySJb}|d0K8?Km<gN<|>wH>yC4@Mk
zw6IAiD{N*YFm^HavF(shl2Jl90N-a1kGJ!9vrw4h^PsVpO^6nL5!P`2rReX?<MpNq
z9yHm%p|}1Tz4gD~w%lNO!8t}ajaJ7=#tGI_&~QJ&z7*#W^w|gUnfjyY*Gt$74`v(h
zeG{7O>zz%7b?D%&<THh$%|6#z8Ey75!VEs|RA*jc0z8?K&a}cnbme<FJ=A=^JKLj^
z*TVVQ*??Sqb>~B__bs0meLm-5YO1@?r;l^`lh1cji}g4bIX|J-{uCX(+wf{Gp`~})
z(b{>$(ZIRKQQhg)yI%v(CfrfNIS0j-sg7*UAV+%VaEIs|jEYNuL&k!C!i#@}1%Je^
z;TB%}d34>6^XoW(SF#h1YCOJutRpLtLwe!{k*L8!rbmT4`prHX8$ZPH(%#SU*xm)b
zzE<!`>pQNarf`v8+X-y{0eGX^u>I@F>8v3RScZPzLZX2X^c5%BQ}Oq2f$qMJWxs;W
zKZ|ug$}e~iy2bIR4@ctztVDe{6wLN4GHSu(dxnwk=?w<EJ-_-U_H_0dFmubHL0pt9
zTz>mAw21GL^SOq{ciEN&Z!ZJ>fj@VPo284#PI5xtepVi$gdg~eAIRsuK%@8x?4d{0
zgdgHZ-iH%>2UXV_;0&%ZE;G)P|2u^jah#YUnOxEVvVeQQB5Y@Df=9fL+RZXNjQPYG
z)A2thg4_;bZ5&ZY5V^$hcpc-pgC~HhpNRJDR36PF9+^$;c{X0jY<$_dJZ~O)xAT1?
zob@GaTaG`wg5#I*jKw^AA(^Xrc))W&MuZTHPN(O>6k?i*V8h2VPmDu#cQk6r!$D9C
zrJg;M4A)Tfe20J}AB;NhKydy8$Z5jKWxW=jPz^F))j2~=yy)7n_-fPpsRn1Mf`?QN
zzo`Tsc0taPi!)@RKJL$6Cw`QLJvx56Zq38U0fr;9O(ur=MlAh--0DlKA6YavzPo6<
zW3}6ET6Hi%5_z;Q_*U=1@w^}_`<RI8E_H?L<kikoT|bWZb&yzK4+x(fU`OJKur}b0
zMG;{|P=#0u_hCN0-lmX!8%bQzo637@5Kgs;H%b$C<OQ?xD`*@SIbfYy{1<X@FF{B>
z0B3UzWXx$|pTopZNnlBLQlH;K%(MY4mA6j6l6-O)*vmPfC?{Kf$9t;<Ho82SyTat}
zve6C4jX$TDztJl?EetW8E+ik#BjyvZ);GXhp9BY)L<K({{CXq^^>Au~Gr&BKfi>0-
zHOY2hz3b2uy#!eFY~aKl<}qseiN;f7J-NLlV9Tb_g>)ERU3Zu+&5fx>4cITGj6OzQ
zGJTm)oph1y)2Z=)rU&G6^jL4`Zu&tgu!mpKCo+}#$XO~Q2Ve|uge|-TZrLR8ivz*I
zb<kgele<c<p+7yp!T~a<h`febb_-_NaWZ|osF6fdA6QJ!%xTm}hQpWbhF)$HFnSfx
z7R*QP?-x%v{miFm@8Ov}Apdt!Ye?;*jJAt9NDMi^FzOkTwZGLN<N>>CDb#Ftt2MP)
z{Jn6s5c>An>1*W=m&#C^svqH(JyDCP*VSz5N!6w9Q-6@3dPWWSicc?p5?rpq@WlR5
zXQQV-R;jM`qt8=2{JjQ>zgmgjRmBvU+W!agf`8M?;J)&SI`cE~f_L%xuF#+A6#b_T
z$Wip0T18K)CCXxX9`)#H<Oze6QQ!jl$pQE>-N_NQR~pE_E7jyW=-*T!KUhY|E*C`)
zC$Hj`bI{`;Gd^GjG``X+GWomDjMt23j3@Y4PskHK#wUD0j_)SRi#$Hf`Z4l`$>abJ
zk_$}2Puz)LwuP~gv7Ql2o-l^K1ksFj(l>bx$F1S`DE!Je@{RF4e+%bH;B~f3nUvk+
z43nhX_?(55gHj33T~;|x?(j6Z!wcjNuSt#3e`tw++Fp4ib>sc@Q(j0zc(0@IU4xYO
z(j?yd3`O$!c8l@tR^#W!@R<|v?e-}x>A}@qPE`WsH_8OrK%XuHx)BA{jdDeGuiQXQ
zk=v?Q<v!{|d5ro_4#B5eqT11?Os{OC7uNx`sFFf|i3@5&<u-k|o~Z%KXLY<{)fSRt
zj8=+jdzHFqSa#C>QUbN_3f(T$<ytXyyH-m*qqR^UYhCg60#!fHBsG^OR4wU=QLB0O
zsf|4s)efGg>OhaAPVxA|`_8Xz^;F02YfD{g5Viej<Q-ROne;@hq>jRxPSsAgQ_uPh
zep@9^0=2FadVkLY`nG-7BhVe%>nUJd^3*V%dpa1JX9#}aOrtcM>}L8l>i@@$S$e7w
ziywGY_ctHu#n7l|YW`*nG^-l3&9=sRGtf9<&Nfo1XMN@VO>22*6tQSVJxf-zo27yY
z@5>Ce(2EF7<Kvbj^O5DIsiQcM!<r7YtrC`Y)+Ux=*1ndx)@kG**ID+XS9H~S$MTAK
z%WBKSJX#5VuoIYo@zyc6)z-ySAhy{aSkIE5dI3vIu-CU0viG-9U9$DEZ?sLaqijqj
zY&SWKYarh~*=yR<I6BZpa2Q<$=VF!P?ESIE!Hy^P#SY!RnP~ke*8Mio{ySoQn{%!s
zyK|$XEFE8(P(|*EN^cMdwQvw!TbwOHlLgQTC)oK06oHO^mma0xJa{}Mg==_hf5ZA!
zovrZ9x}y3PC{z|E2~CAC9z~&2oFGgVlJM7#;cH!D?VfO6cr83c1N1WtYloOs%pw*P
zi@=+%C{`8giS@-6Vk_|vv8&iu93T!Ahk-g6BZi3M#ZYmIxPrAPaW0(eg(!!HqN@}R
z?|LcvaVy0uVx)Lej1}*RapGffgZMY=FT`k;(P+y>i;qAuJm3uX#4xZ7^Ta#iEb+ED
zmFEPD*Tu2oWpM=O86=(r>u?lx(gX0ecY=M`B-Y?HOT+apATAfPfq+OWE*9+!MF<za
z2n)q`LWuZ6m>@nB0>xY4CN2uC#1x^Pn1rVCCZUkH3LWM7f*=M9KZL=;3!#&cDm36a
zl*cd4&lSo5+uU$Y1D7#SxaDkvKUq!KLjRt{&R>LJc+~yiu(xtP#vi=qEDWzblXJJz
z;au;O9Lt<<9JA=xGsbz5y8j_~)mt4^oNFC<ob$nejG?yI1#iBt;~;t34UUXd>OA&|
z^iB$JT(h@v9OBop$)4A-lpO6eqJ^ROWSz;u)Fb~=oUCjnJT%kRA0At4dZ<;iCzC<j
zOg3#fyuJ{8q>;81{Dtj!5^JcE&ZAm74nA5hn^)idnYFy_29Zq)RzDF-9|iVz0oHvS
zY?!`S^tRxM>w_+?01h`lsN4)7k}V)_KY^$HiwxyC(5U;sqsGCFTnU18KKakd*z6Io
zLcE>mx`L)_LtdpJmbt30`&CVPhm|AaUkLtoW>^h&ItzdC^|O0tUZ;=Naq7&wsI*1H
zO9+EQHO1UOoozXMDsNA)Anfx%`hIn$M_zL>Ep@3cc)4RG;Em-q9~hb8iGlW^U-${w
z-pPh;?584^NH%5*y_2Kir!J@Ny%0phEaN3M{E0EixJORrrZEIf{65BIdQe^>Gjkps
z(Fypihd?>(F{+^uQo-12l;kW0=+B%TBt%AIo#9XSalu$dJueIl!~*&|&!&&~RO1Q!
z*}M7};?ohvd3`8#zCoym1TataGyc^3&>OlJeZ{+j`|At`w}TNyojyiyL!G`AeTteJ
z@p=n*xoyy$=xFSKo4bqScj|q4PCxh%eR*DQ=Dl9<BKjCB^#17d^rz0>2bI6>j4q7M
zjIM0!&c5#Gzjx=j9&mp9qGvIf8qNrwKZ@5I%j-?xwWf0JnMR^Mi}yC$IH1op4uPvW
zLcQmRzSKz8m-D_?G1jm>lChQ%!S)FDuLX-AX`Is6@mZtcZmg&0;b!BW4gyl&g970}
zYMqBsQaNsX(a*s1xNKNZ=oaA&`-9C-Z@l7KzC(586aBEi;y+17AwvbnZx|)uk(2<V
zP!vu{AtNm#9XfMqdF*c%V_ONsjsAvcR$x?Ty*~S!gYj=iznf0@ZJoh3bf-Sm%P0)S
zp(ypK;${zc$ermy+Mdyd-%V@wwKnoGax-$F!j+42ct;LKHZ(l5u>DuFBj@eT=*Jk$
z?`#D7f;j(FxXmG)ca~9xWjRI#Y))l!E?mO-C=V=zBNNV8%2+|2Y?aXfeUOH%H)2~u
z_SH8R!HJ#^X3aYm7&XmMmP?Fk98=j`OJDC;&K1w|6FL5X(F{&wTkKg^I5&OaG7N;v
zI20dsBs`q4aI43mIWmsg+c<pJapo%{$b4^%htD|C{9;T4%Q4%Oj0Nx>!b}655-as=
z2i!%0+@_1lwhIoU3!hk|<Cb?gS-0UWo5pIC6js64T){oJlw}y(7O;OlzVae74ct$E
z&XJCh5k>YaoGUxe%?_t1JM4*^<WKX$F)9udsUmlHP0+M;xyKuD$2T$0!a2H3HuWa#
zqI<9{AAvu54wv*L<28@pfU<p!Xa3q8z!=2ip=4!8F~&0{neVB{e=(<_05Kiz({wmc
zGvKez1gA2KF%MqULi!ghA{JNxS8BGInSEJN(ageenJgh@Mvlv1nagq^kHguw41CKf
zGlyjz3Y+VxUB<yBk3)4n&MaoxY?iWYLxDIEC8FJCH4C4HQH#gk-UxLqd(8UOHyc^@
zn~ljeHL)B=wdf3W&&%Ls_=M!1dcxNVuslTx?v*(Xp64VmGt;Ss&IXOO08U9bGvhM&
z>MO}lMZo`xhW{DQjJb=E1Wz{!&dFZJZq|3RZx6@pWqmK0<=telcfu3h0e&Hou@4;1
zpJ>S(f}e5--dQqC_~Y==&Js6Wfva{0=E`Gc>erwjJ~L#{&nA6G1Zzi&pS3gbS$EcZ
z!7A)Sy(@qk?NI7aV_;VVQ;V8Pc72v5H+i!H))jC_*ILTrD^+3CvPPlq6m4k&uClpx
zJ$Rqkf5rxu@zkWe7@*yj*3{2BTK8MJ<6rf)9-%IkVhN;n6==Og|A^a`5m=hB)@NWc
zUsz@^=D=)Rz%tbO#u9G*Y*_-sV>z{~H7sMP+ik)>^Nv_>MUjjMwntD$j<9+-#$$=H
zDwg#ezmc(l?dz!bt;VW(N2JxX#PE0nBhI(9z-w`UOLDQ#P1Gw=Ww23YG&zUKYgn+E
zHn!V|d7TWwcT6NP?m8GLQN+8^<Pg`Rakddm>1OJT@kGAfv4yb}F6UM-Pur+d>;Rvh
z$k+=;Wgnc4ebgnBtVM~hi^H8Q!M>8PXi9TjS)NmYbChS4<(y@F%Pl<KN=<VItdB(B
zsLt^<7_}I+*;bcj1NJu}l5fbU&w5?o*v2xR$D62v#!(;Lh^^nu*a}x<7i`tt)B+P3
zJHXLz<GBeOznSyKa{jgOQ&%xovc7_`ob_e+3CqZvF6I4%GUl>AllMHG&o_<tJcZ9O
ziO(1eN^vZ8$Kfa_42G{V0RB^dSUkPBet&SyTXIbsa*b>8omJtxuE2L*hVQ?G^)YqX
z`<CDE$Fo?k@teG0NoPHUS8~+iqz<Z+Klnzr;4OF06LJT)$s3%*D>+P-z>9v|4D&LI
zTF){VmvdlRPUOBAMaEz#_sT$!KfSomI?(^D36-FlKAzV{IN2|$3jM`=aD#c`Ec3|`
z=9)b|4)=Rz%4f`^cjygz5hhs*oXUeR#&?(<=@8nKIk6^s7Uh|#io-R_556)xd;IZX
zOy>V@%<pfp9Z#@Pcd$hleO&cl%p~kug5kk$`HbK4!dL(fbE<I-PTRSEl*<kn^+6d`
zCJ#}XtV9tw2KnGp=74dT3A|A{>P&8qw9`x5qjv)P*N%KdYq$q3@Kc)Nr!+KD;nH13
z59E|y5&xtlcD*2WJ~#b-avJ;LUGCO%;D_YE=g30OxeSbSjI_pjMigTWV+A8z_c!LF
zH!@SVqn%}fXY#;akWoeZj2_x6y}SOHT*rNQo2iUDtlwsN8+OBOy*(JH4)kX1#CkVy
zq<!><U`L<ngY}o3<1Kv8PkMiNpZz)C0JKd8qr^589QY`Bf#d1zIED8#lRibWd5`mX
zuZzgJgz-7T_2Mi`vn*pQWvpUEp??ymSH|b5g}>8?&)EWOWLx75KGS7<sN48!kI?^m
ztxq?;;s+V%pSX=ku4e+i$$oM?$MFZxaoukkzvDl)!ly(l&gg{ZNoVpzUGYPD<DU#h
zX>Kfj!wme0P_zbC<8Q3Tu5X1myVq!s4eySAK|k{<w)-JA`UABK3suEmu$_6p;g%#n
zRTV$DAvUoswy-BCv|*;jG7TFRN?vOf_Gbh3W*@)LtNb=!VYj|v^M05%i^NdD+;})%
zW2x}iJ2c;R?>;-j`}jY{c=vgD?A_*F*EmKq9sg}O+3#T3edB*V#=HN&4?Fjd9h|*`
zrFZ;%uJ@RqkN>mupn>2WI>Ydd|LpTV??1!)KE31LbN%}oHlAnWIPdVjrgwO^S^pXT
z?)!P1cc1CoXW{sN2Pp*Bz0dLP`*#@sZ2Ql4?=e4z$@1UxP;N4Rj-T6p?)%R+?{ofr
z^mEA*eE<0I`2K!ALac`4yyyS-@z1k*|3`Z6za9DW6}(6PyXJlF&wo}H|9*ylxBb7%
YpU3>)TfN)<_kaJ>v;OyI{Qvj<7cm`$mH+?%


From 3ca909dfea19263feecaf5c390b012280325820a Mon Sep 17 00:00:00 2001
From: Alex Benusovich <qwerty4030@gmail.com>
Date: Sat, 18 Jun 2016 20:43:33 -0700
Subject: [PATCH 1062/1311] Fix NPEs due to disabled source This commit fixes
 several NPEs caused by implicitly performing a get request for a document
 that exists with its _source disabled and then trying to access the source.
 Instead of causing an NPE the following queries will throw an exception with
 a "source disabled" message (similar behavior as if the document does not
 exist).: - GeoShape query for pre-indexed shape (throws
 IllegalArgumentException) - Percolate query for an existing document (throws
 IllegalArgumentException)

A Terms query with a lookup will ignore the document if the source does not exist (same as if the document does not exist).

GET and HEAD requests for the document _source will return a 404 if the source is disabled (even if the document exists).
---
 .../common/network/NetworkModule.java         |  3 +-
 .../index/query/GeoShapeQueryBuilder.java     |  3 ++
 .../index/query/TermsQueryBuilder.java        |  2 +-
 .../rest/action/get/RestGetSourceAction.java  |  2 +-
 .../rest/action/get/RestHeadAction.java       | 46 ++++++++++++++++---
 .../search/geo/GeoShapeQueryTests.java        | 27 +++++++++++
 .../search/query/SearchQueryIT.java           | 12 +++++
 docs/reference/docs/get.asciidoc              |  3 +-
 .../percolator/PercolateQueryBuilder.java     |  7 ++-
 .../percolator/PercolatorQuerySearchIT.java   | 26 ++++++++++-
 .../test/get_source/85_source_missing.yaml    | 40 ++++++++++++++++
 11 files changed, 159 insertions(+), 12 deletions(-)
 create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/get_source/85_source_missing.yaml

diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
index 13f92ba698d..b0287ad153a 100644
--- a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
+++ b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
@@ -238,7 +238,8 @@ public class NetworkModule extends AbstractModule {
         RestIndexAction.class,
         RestGetAction.class,
         RestGetSourceAction.class,
-        RestHeadAction.class,
+        RestHeadAction.Document.class,
+        RestHeadAction.Source.class,
         RestMultiGetAction.class,
         RestDeleteAction.class,
         org.elasticsearch.rest.action.count.RestCountAction.class,
diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
index 0b04629e2cc..e6276a76c84 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
@@ -378,6 +378,9 @@ public class GeoShapeQueryBuilder extends AbstractQueryBuilder<GeoShapeQueryBuil
         if (!response.isExists()) {
             throw new IllegalArgumentException("Shape with ID [" + getRequest.id() + "] in type [" + getRequest.type() + "] not found");
         }
+        if (response.isSourceEmpty()) {
+            throw new IllegalArgumentException("Shape with ID [" + getRequest.id() + "] in type [" + getRequest.type() + "] source disabled");
+        }
 
         String[] pathElements = path.split("\\.");
         int currentPathSlot = 0;
diff --git a/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
index 7d27a911f88..c0abe37e412 100644
--- a/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java
@@ -328,7 +328,7 @@ public class TermsQueryBuilder extends AbstractQueryBuilder<TermsQueryBuilder> {
         GetRequest getRequest = new GetRequest(termsLookup.index(), termsLookup.type(), termsLookup.id())
                 .preference("_local").routing(termsLookup.routing());
         final GetResponse getResponse = client.get(getRequest).actionGet();
-        if (getResponse.isExists()) {
+        if (getResponse.isSourceEmpty() == false) { // extract terms only if the doc source exists
             List<Object> extractedValues = XContentMapValues.extractRawValues(termsLookup.path(), getResponse.getSourceAsMap());
             terms.addAll(extractedValues);
         }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/get/RestGetSourceAction.java b/core/src/main/java/org/elasticsearch/rest/action/get/RestGetSourceAction.java
index af2faba01c9..7d8a39b02b9 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/get/RestGetSourceAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/get/RestGetSourceAction.java
@@ -78,7 +78,7 @@ public class RestGetSourceAction extends BaseRestHandler {
             @Override
             public RestResponse buildResponse(GetResponse response) throws Exception {
                 XContentBuilder builder = channel.newBuilder(response.getSourceInternal(), false);
-                if (!response.isExists()) {
+                if (response.isSourceEmpty()) { // check if doc source (or doc itself) is missing
                     return new BytesRestResponse(NOT_FOUND, builder);
                 } else {
                     builder.rawValue(response.getSourceInternal());
diff --git a/core/src/main/java/org/elasticsearch/rest/action/get/RestHeadAction.java b/core/src/main/java/org/elasticsearch/rest/action/get/RestHeadAction.java
index 747e62ea381..01f3be435e7 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/get/RestHeadAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/get/RestHeadAction.java
@@ -39,15 +39,47 @@ import static org.elasticsearch.rest.RestStatus.NOT_FOUND;
 import static org.elasticsearch.rest.RestStatus.OK;
 
 /**
- *
+ * Base class for {@code HEAD} request handlers for a single document.
  */
-public class RestHeadAction extends BaseRestHandler {
+public abstract class RestHeadAction extends BaseRestHandler {
 
-    @Inject
-    public RestHeadAction(Settings settings, RestController controller, Client client) {
+    /**
+     * Handler to check for document existence.
+     */
+    public static class Document extends RestHeadAction {
+
+        @Inject
+        public Document(Settings settings, RestController controller, Client client) {
+            super(settings, client, false);
+            controller.registerHandler(HEAD, "/{index}/{type}/{id}", this);
+        }
+    }
+
+    /**
+     * Handler to check for document source existence (may be disabled in the mapping).
+     */
+    public static class Source extends RestHeadAction {
+
+        @Inject
+        public Source(Settings settings, RestController controller, Client client) {
+            super(settings, client, true);
+            controller.registerHandler(HEAD, "/{index}/{type}/{id}/_source", this);
+        }
+    }
+
+    private final boolean source;
+
+    /**
+     * All subclasses must be registered in {@link org.elasticsearch.common.network.NetworkModule}.
+     *
+     * @param settings injected settings
+     * @param client   injected client
+     * @param source   {@code false} to check for {@link GetResponse#isExists()}.
+     *                 {@code true} to also check for {@link GetResponse#isSourceEmpty()}.
+     */
+    public RestHeadAction(Settings settings, Client client, boolean source) {
         super(settings, client);
-        controller.registerHandler(HEAD, "/{index}/{type}/{id}", this);
-        controller.registerHandler(HEAD, "/{index}/{type}/{id}/_source", this);
+        this.source = source;
     }
 
     @Override
@@ -68,6 +100,8 @@ public class RestHeadAction extends BaseRestHandler {
             public RestResponse buildResponse(GetResponse response) {
                 if (!response.isExists()) {
                     return new BytesRestResponse(NOT_FOUND, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
+                } else if (source && response.isSourceEmpty()) { // doc exists, but source might not (disabled in the mapping)
+                    return new BytesRestResponse(NOT_FOUND, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
                 } else {
                     return new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY);
                 }
diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java b/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
index 7da18342edf..067c2dcac05 100644
--- a/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
+++ b/core/src/test/java/org/elasticsearch/search/geo/GeoShapeQueryTests.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.search.geo;
 
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.common.settings.Settings;
 import org.locationtech.spatial4j.shape.Rectangle;
 import com.vividsolutions.jts.geom.Coordinate;
 
@@ -54,6 +56,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSear
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.nullValue;
 
 public class GeoShapeQueryTests extends ESSingleNodeTestCase {
@@ -197,6 +200,30 @@ public class GeoShapeQueryTests extends ESSingleNodeTestCase {
         assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1"));
     }
 
+    public void testIndexedShapeReferenceSourceDisabled() throws Exception {
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject()
+                .startObject("properties")
+                    .startObject("location")
+                        .field("type", "geo_shape")
+                        .field("tree", "quadtree")
+                    .endObject()
+                .endObject()
+            .endObject();
+        client().admin().indices().prepareCreate("test").addMapping("type1", mapping).get();
+        createIndex("shapes", Settings.EMPTY, "shape_type", "_source", "enabled=false");
+        ensureGreen();
+
+        ShapeBuilder shape = ShapeBuilders.newEnvelope(new Coordinate(-45, 45), new Coordinate(45, -45));
+
+        client().prepareIndex("shapes", "shape_type", "Big_Rectangle").setSource(jsonBuilder().startObject()
+            .field("shape", shape).endObject()).setRefreshPolicy(IMMEDIATE).get();
+
+        ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> client().prepareSearch("test").setTypes("type1")
+            .setQuery(geoIntersectionQuery("location", "Big_Rectangle", "shape_type")).get());
+        assertThat(e.getRootCause(), instanceOf(IllegalArgumentException.class));
+        assertThat(e.getRootCause().getMessage(), containsString("source disabled"));
+    }
+
     public void testReusableBuilder() throws IOException {
         ShapeBuilder polygon = ShapeBuilders.newPolygon(new CoordinatesBuilder()
                 .coordinate(170, -10).coordinate(190, -10).coordinate(190, 10).coordinate(170, 10).close())
diff --git a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
index 8cb21dadb9b..1d54b412d6c 100644
--- a/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java
@@ -1148,6 +1148,7 @@ public class SearchQueryIT extends ESIntegTestCase {
                 jsonBuilder().startObject().startObject("type").startObject("properties")
                         .startObject("arr").startObject("properties").startObject("term").field("type", "text")
                         .endObject().endObject().endObject().endObject().endObject().endObject()));
+        assertAcked(prepareCreate("lookup3").addMapping("type", "_source", "enabled=false", "terms","type=text"));
         assertAcked(prepareCreate("test").addMapping("type", "term", "type=text"));
 
         indexRandom(true,
@@ -1172,6 +1173,7 @@ public class SearchQueryIT extends ESIntegTestCase {
                         .startObject().field("term", "4").endObject()
                         .endArray()
                         .endObject()),
+                client().prepareIndex("lookup3", "type", "1").setSource("terms", new String[]{"1", "3"}),
                 client().prepareIndex("test", "type", "1").setSource("term", "1"),
                 client().prepareIndex("test", "type", "2").setSource("term", "2"),
                 client().prepareIndex("test", "type", "3").setSource("term", "3"),
@@ -1227,6 +1229,16 @@ public class SearchQueryIT extends ESIntegTestCase {
         searchResponse = client().prepareSearch("test")
                 .setQuery(termsLookupQuery("not_exists", new TermsLookup("lookup2", "type", "3", "arr.term"))).get();
         assertHitCount(searchResponse, 0L);
+
+        // index "lookup" type "type" id "missing" document does not exist: ignore the lookup terms
+        searchResponse = client().prepareSearch("test")
+            .setQuery(termsLookupQuery("term" , new TermsLookup("lookup", "type", "missing", "terms"))).get();
+        assertHitCount(searchResponse, 0L);
+
+        // index "lookup3" type "type" has the source disabled: ignore the lookup terms
+        searchResponse = client().prepareSearch("test")
+            .setQuery(termsLookupQuery("term" , new TermsLookup("lookup3", "type", "1", "terms"))).get();
+        assertHitCount(searchResponse, 0L);
     }
 
     public void testBasicQueryById() throws Exception {
diff --git a/docs/reference/docs/get.asciidoc b/docs/reference/docs/get.asciidoc
index d63ddb52f4b..1d6544cdd92 100644
--- a/docs/reference/docs/get.asciidoc
+++ b/docs/reference/docs/get.asciidoc
@@ -146,7 +146,8 @@ You can also use the same source filtering parameters to control which parts of
 curl -XGET 'http://localhost:9200/twitter/tweet/1/_source?_source_include=*.id&_source_exclude=entities'
 --------------------------------------------------
 
-Note, there is also a HEAD variant for the _source endpoint to efficiently test for document existence.
+Note, there is also a HEAD variant for the _source endpoint to efficiently test for document _source existence.
+An existing document will not have a _source if it is disabled in the <<mapping-source-field,mapping>>.
 Curl example:
 
 [source,js]
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
index 1c47610ec15..8ce12c1f7e6 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
@@ -359,6 +359,11 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
                     "indexed document [{}/{}/{}] couldn't be found", indexedDocumentIndex, indexedDocumentType, indexedDocumentId
             );
         }
+        if(getResponse.isSourceEmpty()) {
+            throw new IllegalArgumentException(
+                "indexed document [" + indexedDocumentIndex + "/" + indexedDocumentType + "/" + indexedDocumentId + "] source disabled"
+            );
+        }
         return new PercolateQueryBuilder(field, documentType, getResponse.getSourceAsBytesRef());
     }
 
@@ -369,7 +374,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
         }
 
         if (document == null) {
-            throw new IllegalStateException("nothing to percolator");
+            throw new IllegalStateException("no document to percolate");
         }
 
         MapperService mapperService = context.getMapperService();
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
index e677b6126ee..5125a7ea5cc 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.percolator;
 
 import org.apache.lucene.search.join.ScoreMode;
+import org.elasticsearch.action.search.SearchPhaseExecutionException;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -37,7 +38,6 @@ import org.elasticsearch.test.ESSingleNodeTestCase;
 import java.util.Collection;
 import java.util.Collections;
 
-import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
 import static org.elasticsearch.index.query.QueryBuilders.commonTermsQuery;
@@ -49,6 +49,7 @@ import static org.elasticsearch.index.query.QueryBuilders.spanNotQuery;
 import static org.elasticsearch.index.query.QueryBuilders.spanTermQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -158,6 +159,29 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
         assertThat(response.getHits().getAt(2).getId(), equalTo("3"));
     }
 
+    public void testPercolatorQueryExistingDocumentSourceDisabled() throws Exception {
+        createIndex("test", client().admin().indices().prepareCreate("test")
+            .addMapping("type", "_source", "enabled=false", "field1", "type=keyword")
+            .addMapping("queries", "query", "type=percolator")
+        );
+
+        client().prepareIndex("test", "queries", "1")
+            .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject())
+            .get();
+
+        client().prepareIndex("test", "type", "1").setSource("{}").get();
+        client().admin().indices().prepareRefresh().get();
+
+        logger.info("percolating empty doc with source disabled");
+        Throwable e = expectThrows(SearchPhaseExecutionException.class, () -> {
+            client().prepareSearch()
+                .setQuery(new PercolateQueryBuilder("query", "type", "test", "type", "1", null, null, null))
+                .get();
+        }).getRootCause();
+        assertThat(e, instanceOf(IllegalArgumentException.class));
+        assertThat(e.getMessage(), containsString("source disabled"));
+    }
+
     public void testPercolatorSpecificQueries()  throws Exception {
         createIndex("test", client().admin().indices().prepareCreate("test")
                 .addMapping("type", "field1", "type=text", "field2", "type=text")
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/85_source_missing.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/85_source_missing.yaml
new file mode 100644
index 00000000000..370f68d9504
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get_source/85_source_missing.yaml
@@ -0,0 +1,40 @@
+---
+setup:
+ - do:
+      indices.create:
+          index: test_1
+          body:
+            mappings:
+              test:
+                _source: { enabled: false }
+ - do:
+      cluster.health:
+          wait_for_status: yellow
+
+ - do:
+      index:
+          index:   test_1
+          type:    test
+          id:      1
+          body:    { foo: bar }
+
+
+---
+"Missing document source with catch":
+
+  - do:
+      catch:   missing
+      get_source:
+        index: test_1
+        type:  test
+        id:    1
+
+---
+"Missing document source with ignore":
+
+  - do:
+      get_source:
+        index:  test_1
+        type:   test
+        id:     1
+        ignore: 404

From 09c0285d9c1dc5d553939f8c85d6e5f98d760f2e Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Sat, 25 Jun 2016 21:23:41 +0200
Subject: [PATCH 1063/1311] [TEST] Add unittest for settings update validation

---
 .../index/IndexSettingsTests.java             | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
index 9c5040589ae..3909354c989 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
@@ -65,6 +65,31 @@ public class IndexSettingsTests extends ESTestCase {
         assertEquals(42, integer.get());
     }
 
+    public void testSettingsUpdateValidator() {
+        Version version = VersionUtils.getPreviousVersion();
+        Settings theSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version)
+            .put(IndexMetaData.SETTING_INDEX_UUID, "0xdeadbeef").build();
+        final AtomicInteger integer = new AtomicInteger(0);
+        Setting<Integer> integerSetting = Setting.intSetting("index.test.setting.int", -1,
+            Property.Dynamic, Property.IndexScope);
+        IndexMetaData metaData = newIndexMeta("index", theSettings);
+        IndexSettings settings = newIndexSettings(newIndexMeta("index", theSettings), Settings.EMPTY, integerSetting);
+        settings.getScopedSettings().addSettingsUpdateConsumer(integerSetting, integer::set,
+            (i) -> {if (i == 42) throw new AssertionError("boom");});
+
+        assertEquals(version, settings.getIndexVersionCreated());
+        assertEquals("0xdeadbeef", settings.getUUID());
+
+        assertFalse(settings.updateIndexMetaData(metaData));
+        assertEquals(metaData.getSettings().getAsMap(), settings.getSettings().getAsMap());
+        assertEquals(0, integer.get());
+        expectThrows(IllegalArgumentException.class, () -> settings.updateIndexMetaData(newIndexMeta("index",
+            Settings.builder().put(theSettings).put("index.test.setting.int", 42).build())));
+        assertTrue(settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(theSettings).put("index.test.setting.int", 41)
+            .build())));
+        assertEquals(41, integer.get());
+    }
+
     public void testMergedSettingsArePassed() {
         Version version = VersionUtils.getPreviousVersion();
         Settings theSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version)

From c79e27180ea14bd536d39e50d310a0345369d7c3 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Sat, 25 Jun 2016 16:18:25 -0400
Subject: [PATCH 1064/1311] Require timeout units when parsing query body

Today when parsing the timeout field in a query body, if time units are
supplied the parser throws a NumberFormatException. Addtionally, the
parsing allows the timeout field to not specify units (it assumes
milliseconds). This commit fixes this behavior by not only allowing time
units to be specified but requires time units to be specified. This is
consistent with the documented behavior and the behavior in 2.x.

Relates #19077
---
 .../search/builder/SearchSourceBuilder.java   |  7 +++---
 .../builder/SearchSourceBuilderTests.java     | 24 +++++++++++++++++++
 2 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index e4b72ef3b28..648fcaa859a 100644
--- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.builder;
 
 import com.carrotsearch.hppc.ObjectFloatHashMap;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
-
 import org.elasticsearch.action.support.ToXContentToBytes;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
@@ -42,7 +41,6 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
-import org.elasticsearch.search.slice.SliceBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
@@ -51,6 +49,7 @@ import org.elasticsearch.search.highlight.HighlightBuilder;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.rescore.RescoreBuilder;
 import org.elasticsearch.search.searchafter.SearchAfterBuilder;
+import org.elasticsearch.search.slice.SliceBuilder;
 import org.elasticsearch.search.sort.ScoreSortBuilder;
 import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.search.sort.SortBuilders;
@@ -959,7 +958,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                 } else if (context.getParseFieldMatcher().match(currentFieldName, SIZE_FIELD)) {
                     size = parser.intValue();
                 } else if (context.getParseFieldMatcher().match(currentFieldName, TIMEOUT_FIELD)) {
-                    timeoutInMillis = parser.longValue();
+                    timeoutInMillis = TimeValue.parseTimeValue(parser.text(), null, TIMEOUT_FIELD.getPreferredName()).millis();
                 } else if (context.getParseFieldMatcher().match(currentFieldName, TERMINATE_AFTER_FIELD)) {
                     terminateAfter = parser.intValue();
                 } else if (context.getParseFieldMatcher().match(currentFieldName, MIN_SCORE_FIELD)) {
@@ -1105,7 +1104,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         }
 
         if (timeoutInMillis != -1) {
-            builder.field(TIMEOUT_FIELD.getPreferredName(), timeoutInMillis);
+            builder.field(TIMEOUT_FIELD.getPreferredName(), TimeValue.timeValueMillis(timeoutInMillis).toString());
         }
 
         if (terminateAfter != SearchContext.DEFAULT_TERMINATE_AFTER) {
diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index 419b11dac60..575018a3097 100644
--- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.search.builder;
 
+import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -89,7 +90,9 @@ import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
 import static org.elasticsearch.test.ClusterServiceUtils.setState;
+import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasToString;
 
 public class SearchSourceBuilderTests extends ESTestCase {
     private static Injector injector;
@@ -593,6 +596,27 @@ public class SearchSourceBuilderTests extends ESTestCase {
         }
     }
 
+    public void testTimeoutWithUnits() throws IOException {
+        final String timeout = randomTimeValue();
+        final String query = "{ \"query\": { \"match_all\": {}}, \"timeout\": \"" + timeout + "\"}";
+        try (XContentParser parser = XContentFactory.xContent(query).createParser(query)) {
+            final SearchSourceBuilder builder = SearchSourceBuilder.fromXContent(createParseContext(parser), aggParsers, suggesters);
+            assertThat(builder.timeoutInMillis(), equalTo(TimeValue.parseTimeValue(timeout, null, "timeout").millis()));
+        }
+    }
+
+    public void testTimeoutWithoutUnits() throws IOException {
+        final int timeout = randomIntBetween(1, 1024);
+        final String query = "{ \"query\": { \"match_all\": {}}, \"timeout\": \"" + timeout + "\"}";
+        try (XContentParser parser = XContentFactory.xContent(query).createParser(query)) {
+            final ElasticsearchParseException e =
+                    expectThrows(
+                            ElasticsearchParseException.class,
+                            () -> SearchSourceBuilder.fromXContent(createParseContext(parser), aggParsers, suggesters));
+            assertThat(e, hasToString(containsString("unit is missing or unrecognized")));
+        }
+    }
+
     public void testEmptyPostFilter() throws IOException {
         SearchSourceBuilder builder = new SearchSourceBuilder();
         String query = "{ \"post_filter\": {} }";

From 71b95fb63c31ad40ad18e1730b277d3d5b007b95 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 24 Jun 2016 16:37:23 -0400
Subject: [PATCH 1065/1311] Switch analysis from push to pull

Instead of plugins calling `registerTokenizer` to extend the analyzer
they now instead have to implement `AnalysisPlugin` and override
`getTokenizer`. This lines up extending plugins in with extending
scripts. This allows `AnalysisModule` to construct the `AnalysisRegistry`
immediately as part of its constructor which makes testing anslysis
much simpler.

This also moves the default analysis configuration into `AnalysisModule`
which is how search is setup.

Like `ScriptModule`, `AnalysisModule` no longer extends `AbstractModule`.
Instead it is only responsible for building `AnslysisRegistry`. We still
bind `AnalysisRegistry` but we only do so in `Node`. This is means it
is available at module construction time so we slowly remove the need to
bind it in guice.
---
 .../resources/checkstyle_suppressions.xml     |   7 -
 .../index/analysis/AnalysisRegistry.java      | 203 ++------
 .../index/analysis/AnalysisService.java       |   6 +-
 .../PreBuiltAnalyzerProviderFactory.java      |   7 +-
 .../indices/analysis/AnalysisModule.java      | 455 +++++++++++++-----
 .../indices/analysis/HunspellService.java     |   8 +-
 .../java/org/elasticsearch/node/Node.java     |   5 +-
 .../elasticsearch/plugins/AnalysisPlugin.java |  84 ++++
 .../org/elasticsearch/plugins/Plugin.java     |   9 +
 .../elasticsearch/plugins/PluginsService.java |   5 +-
 .../elasticsearch/plugins/ScriptPlugin.java   |   3 +-
 .../indices/TransportAnalyzeActionTests.java  |   5 +-
 .../elasticsearch/index/IndexModuleTests.java |  50 +-
 .../index/analysis/AnalysisServiceTests.java  |  57 ++-
 .../index/analysis/AnalysisTestsHelper.java   |   9 +-
 .../index/analysis/CharFilterTests.java       |   8 +-
 .../index/analysis/CompoundAnalysisTests.java |  28 +-
 .../PatternCaptureTokenFilterTests.java       |   3 +-
 .../index/analysis/StopAnalyzerTests.java     |   4 +-
 .../synonyms/SynonymsAnalysisTests.java       |   6 +-
 .../elasticsearch/index/codec/CodecTests.java |   4 +-
 .../analysis/AnalysisModuleTests.java         |  59 ++-
 .../indices/analysis/DummyAnalysisPlugin.java |  35 +-
 ...rviceIT.java => HunspellServiceTests.java} |  68 +--
 .../migration/migrate_5_0/plugins.asciidoc    |   5 +
 .../messy/tests/TemplateQueryParserTests.java |   4 +-
 .../analysis/icu/AnalysisICUPlugin.java       |  40 +-
 .../analysis/IcuTokenizerFactoryTests.java    |   2 +-
 .../analysis/SimpleIcuAnalysisTests.java      |   3 +-
 .../SimpleIcuCollationTokenFilterTests.java   |  18 +-
 .../SimpleIcuNormalizerCharFilterTests.java   |   4 +-
 .../kuromoji/AnalysisKuromojiPlugin.java      |  52 +-
 .../index/analysis/KuromojiAnalysisTests.java |   2 +-
 .../analysis/AnalysisPhoneticPlugin.java      |  14 +-
 .../analysis/SimplePhoneticAnalysisTests.java |   3 +-
 .../smartcn/AnalysisSmartChinesePlugin.java   |  38 +-
 .../SimpleSmartChineseAnalysisTests.java      |   5 +-
 .../stempel/AnalysisStempelPlugin.java        |  22 +-
 .../index/analysis/PolishAnalysisTests.java   |   3 +-
 .../SimplePolishTokenFilterTests.java         |   5 +-
 .../elasticsearch/index/MapperTestUtils.java  |   4 +-
 .../test/AbstractQueryTestCase.java           |   6 +-
 .../org/elasticsearch/test/ESTestCase.java    |  39 +-
 43 files changed, 844 insertions(+), 553 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/plugins/AnalysisPlugin.java
 rename core/src/test/java/org/elasticsearch/{index => indices}/analysis/AnalysisModuleTests.java (87%)
 rename core/src/test/java/org/elasticsearch/indices/analyze/{HunspellServiceIT.java => HunspellServiceTests.java} (58%)

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 02aec49b1c7..745c8374063 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -477,8 +477,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]IndexingMemoryController.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]IndicesService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]IndicesWarmer.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]analysis[/\\]AnalysisModule.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]analysis[/\\]HunspellService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]analysis[/\\]PreBuiltCacheFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]analysis[/\\]PreBuiltTokenFilters.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]indices[/\\]breaker[/\\]HierarchyCircuitBreakerService.java" checks="LineLength" />
@@ -879,9 +877,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexingSlowLogTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]MergePolicySettingsTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]SearchSlowLogTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]AnalysisModuleTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]AnalysisServiceTests.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]CompoundAnalysisTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]NGramTokenizerFactoryTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]PatternCaptureTokenFilterTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]PreBuiltAnalyzerProviderFactoryTests.java" checks="LineLength" />
@@ -1186,8 +1181,6 @@
   <suppress files="plugins[/\\]analysis-kuromoji[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]JapaneseStopTokenFilterFactory.java" checks="LineLength" />
   <suppress files="plugins[/\\]analysis-kuromoji[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]KuromojiAnalysisTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]analysis-phonetic[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]PhoneticTokenFilterFactory.java" checks="LineLength" />
-  <suppress files="plugins[/\\]analysis-smartcn[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]SimpleSmartChineseAnalysisTests.java" checks="LineLength" />
-  <suppress files="plugins[/\\]analysis-stempel[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]PolishAnalysisTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]discovery-azure[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]AbstractAzureTestCase.java" checks="LineLength" />
   <suppress files="plugins[/\\]discovery-azure[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]azure[/\\]AzureMinimumMasterNodesTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]discovery-azure[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]azure[/\\]AzureSimpleTests.java" checks="LineLength" />
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
index 2d73df76f07..548bc91b0a5 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisRegistry.java
@@ -23,14 +23,11 @@ import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.compound.DictionaryCompoundWordTokenFilterFactory;
-import org.elasticsearch.index.analysis.compound.HyphenationCompoundWordTokenFilterFactory;
 import org.elasticsearch.indices.analysis.AnalysisModule;
-import org.elasticsearch.indices.analysis.HunspellService;
+import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
 import org.elasticsearch.indices.analysis.PreBuiltAnalyzers;
 import org.elasticsearch.indices.analysis.PreBuiltCharFilters;
 import org.elasticsearch.indices.analysis.PreBuiltTokenFilters;
@@ -45,48 +42,32 @@ import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
+import static java.util.Collections.unmodifiableMap;
+
 /**
  * An internal registry for tokenizer, token filter, char filter and analyzer.
  * This class exists per node and allows to create per-index {@link AnalysisService} via {@link #build(IndexSettings)}
  */
 public final class AnalysisRegistry implements Closeable {
-    private final Map<String, AnalysisModule.AnalysisProvider<CharFilterFactory>> charFilters;
-    private final Map<String, AnalysisModule.AnalysisProvider<TokenFilterFactory>> tokenFilters;
-    private final Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> tokenizers;
-    private final Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider>> analyzers;
+    private final PrebuiltAnalysis prebuiltAnalysis = new PrebuiltAnalysis();
     private final Map<String, Analyzer> cachedAnalyzer = new ConcurrentHashMap<>();
-    private final PrebuiltAnalysis prebuiltAnalysis;
-    private final HunspellService hunspellService;
+
     private final Environment environment;
+    private final Map<String, AnalysisProvider<CharFilterFactory>> charFilters;
+    private final Map<String, AnalysisProvider<TokenFilterFactory>> tokenFilters;
+    private final Map<String, AnalysisProvider<TokenizerFactory>> tokenizers;
+    private final Map<String, AnalysisProvider<AnalyzerProvider<?>>> analyzers;
 
-    public AnalysisRegistry(HunspellService hunspellService, Environment environment) {
-        this(hunspellService, environment, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
-    }
-
-    public AnalysisRegistry(HunspellService hunspellService, Environment environment,
-                            Map<String, AnalysisModule.AnalysisProvider<CharFilterFactory>> charFilters,
-                            Map<String, AnalysisModule.AnalysisProvider<TokenFilterFactory>> tokenFilters,
-                            Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> tokenizers,
-                            Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider>> analyzers) {
-        prebuiltAnalysis = new PrebuiltAnalysis();
-        this.hunspellService = hunspellService;
+    public AnalysisRegistry(Environment environment,
+                            Map<String, AnalysisProvider<CharFilterFactory>> charFilters,
+                            Map<String, AnalysisProvider<TokenFilterFactory>> tokenFilters,
+                            Map<String, AnalysisProvider<TokenizerFactory>> tokenizers,
+                            Map<String, AnalysisProvider<AnalyzerProvider<?>>> analyzers) {
         this.environment = environment;
-        final Map<String, AnalysisModule.AnalysisProvider<CharFilterFactory>> charFilterBuilder = new HashMap<>(charFilters);
-        final Map<String, AnalysisModule.AnalysisProvider<TokenFilterFactory>> tokenFilterBuilder = new HashMap<>(tokenFilters);
-        final Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> tokenizerBuilder = new HashMap<>(tokenizers);
-        final Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider>> analyzerBuilder= new HashMap<>(analyzers);
-        registerBuiltInAnalyzer(analyzerBuilder);
-        registerBuiltInCharFilter(charFilterBuilder);
-        registerBuiltInTokenizer(tokenizerBuilder);
-        registerBuiltInTokenFilters(tokenFilterBuilder);
-        this.tokenFilters = Collections.unmodifiableMap(tokenFilterBuilder);
-        this.tokenizers = Collections.unmodifiableMap(tokenizerBuilder);
-        this.charFilters = Collections.unmodifiableMap(charFilterBuilder);
-        this.analyzers = Collections.unmodifiableMap(analyzerBuilder);
-    }
-
-    public HunspellService getHunspellService() {
-        return hunspellService;
+        this.charFilters = unmodifiableMap(charFilters);
+        this.tokenFilters = unmodifiableMap(tokenFilters);
+        this.tokenizers = unmodifiableMap(tokenizers);
+        this.analyzers = unmodifiableMap(analyzers);
     }
 
     /**
@@ -114,9 +95,9 @@ public final class AnalysisRegistry implements Closeable {
      * Returns a registered {@link Analyzer} provider by name or <code>null</code> if the analyzer was not registered
      */
     public Analyzer getAnalyzer(String analyzer) throws IOException {
-        AnalysisModule.AnalysisProvider<AnalyzerProvider> analyzerProvider = this.prebuiltAnalysis.getAnalyzerProvider(analyzer);
+        AnalysisModule.AnalysisProvider<AnalyzerProvider<?>> analyzerProvider = this.prebuiltAnalysis.getAnalyzerProvider(analyzer);
         if (analyzerProvider == null) {
-            AnalysisModule.AnalysisProvider<AnalyzerProvider> provider = analyzers.get(analyzer);
+            AnalysisModule.AnalysisProvider<AnalyzerProvider<?>> provider = analyzers.get(analyzer);
             return provider == null ? null : cachedAnalyzer.computeIfAbsent(analyzer, (key) -> {
                         try {
                             return provider.get(environment, key).get();
@@ -157,7 +138,8 @@ public final class AnalysisRegistry implements Closeable {
          */
         tokenFilters.put("synonym", requriesAnalysisSettings((is, env, name, settings) -> new SynonymTokenFilterFactory(is, env, tokenizerFactories, name, settings)));
         final Map<String, TokenFilterFactory> tokenFilterFactories = buildMapping(false, "tokenfilter", indexSettings, tokenFiltersSettings, Collections.unmodifiableMap(tokenFilters), prebuiltAnalysis.tokenFilterFactories);
-        final Map<String, AnalyzerProvider> analyzierFactories = buildMapping(true, "analyzer", indexSettings, analyzersSettings, analyzers, prebuiltAnalysis.analyzerProviderFactories);
+        final Map<String, AnalyzerProvider<?>> analyzierFactories = buildMapping(true, "analyzer", indexSettings, analyzersSettings,
+                analyzers, prebuiltAnalysis.analyzerProviderFactories);
         return new AnalysisService(indexSettings, analyzierFactories, tokenizerFactories, charFilterFactories, tokenFilterFactories);
     }
 
@@ -175,140 +157,9 @@ public final class AnalysisRegistry implements Closeable {
         };
     }
 
-    private void registerBuiltInCharFilter(Map<String, AnalysisModule.AnalysisProvider<CharFilterFactory>> charFilters) {
-        charFilters.put("html_strip", HtmlStripCharFilterFactory::new);
-        charFilters.put("pattern_replace", requriesAnalysisSettings(PatternReplaceCharFilterFactory::new));
-        charFilters.put("mapping", requriesAnalysisSettings(MappingCharFilterFactory::new));
-    }
-
-    private void registerBuiltInTokenizer(Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> tokenizers) {
-        tokenizers.put("standard", StandardTokenizerFactory::new);
-        tokenizers.put("uax_url_email", UAX29URLEmailTokenizerFactory::new);
-        tokenizers.put("path_hierarchy", PathHierarchyTokenizerFactory::new);
-        tokenizers.put("PathHierarchy", PathHierarchyTokenizerFactory::new);
-        tokenizers.put("keyword", KeywordTokenizerFactory::new);
-        tokenizers.put("letter", LetterTokenizerFactory::new);
-        tokenizers.put("lowercase", LowerCaseTokenizerFactory::new);
-        tokenizers.put("whitespace", WhitespaceTokenizerFactory::new);
-        tokenizers.put("nGram", NGramTokenizerFactory::new);
-        tokenizers.put("ngram", NGramTokenizerFactory::new);
-        tokenizers.put("edgeNGram", EdgeNGramTokenizerFactory::new);
-        tokenizers.put("edge_ngram", EdgeNGramTokenizerFactory::new);
-        tokenizers.put("pattern", PatternTokenizerFactory::new);
-        tokenizers.put("classic", ClassicTokenizerFactory::new);
-        tokenizers.put("thai", ThaiTokenizerFactory::new);
-    }
-
-    private void registerBuiltInTokenFilters(Map<String, AnalysisModule.AnalysisProvider<TokenFilterFactory>> tokenFilters) {
-        tokenFilters.put("stop", StopTokenFilterFactory::new);
-        tokenFilters.put("reverse", ReverseTokenFilterFactory::new);
-        tokenFilters.put("asciifolding", ASCIIFoldingTokenFilterFactory::new);
-        tokenFilters.put("length", LengthTokenFilterFactory::new);
-        tokenFilters.put("lowercase", LowerCaseTokenFilterFactory::new);
-        tokenFilters.put("uppercase", UpperCaseTokenFilterFactory::new);
-        tokenFilters.put("porter_stem", PorterStemTokenFilterFactory::new);
-        tokenFilters.put("kstem", KStemTokenFilterFactory::new);
-        tokenFilters.put("standard", StandardTokenFilterFactory::new);
-        tokenFilters.put("nGram", NGramTokenFilterFactory::new);
-        tokenFilters.put("ngram", NGramTokenFilterFactory::new);
-        tokenFilters.put("edgeNGram", EdgeNGramTokenFilterFactory::new);
-        tokenFilters.put("edge_ngram", EdgeNGramTokenFilterFactory::new);
-        tokenFilters.put("shingle", ShingleTokenFilterFactory::new);
-        tokenFilters.put("unique", UniqueTokenFilterFactory::new);
-        tokenFilters.put("truncate", requriesAnalysisSettings(TruncateTokenFilterFactory::new));
-        tokenFilters.put("trim", TrimTokenFilterFactory::new);
-        tokenFilters.put("limit", LimitTokenCountFilterFactory::new);
-        tokenFilters.put("common_grams", requriesAnalysisSettings(CommonGramsTokenFilterFactory::new));
-        tokenFilters.put("snowball", SnowballTokenFilterFactory::new);
-        tokenFilters.put("stemmer", StemmerTokenFilterFactory::new);
-        tokenFilters.put("word_delimiter", WordDelimiterTokenFilterFactory::new);
-        tokenFilters.put("delimited_payload_filter", DelimitedPayloadTokenFilterFactory::new);
-        tokenFilters.put("elision", ElisionTokenFilterFactory::new);
-        tokenFilters.put("keep", requriesAnalysisSettings(KeepWordFilterFactory::new));
-        tokenFilters.put("keep_types", requriesAnalysisSettings(KeepTypesFilterFactory::new));
-        tokenFilters.put("pattern_capture", requriesAnalysisSettings(PatternCaptureGroupTokenFilterFactory::new));
-        tokenFilters.put("pattern_replace", requriesAnalysisSettings(PatternReplaceTokenFilterFactory::new));
-        tokenFilters.put("dictionary_decompounder", requriesAnalysisSettings(DictionaryCompoundWordTokenFilterFactory::new));
-        tokenFilters.put("hyphenation_decompounder", requriesAnalysisSettings(HyphenationCompoundWordTokenFilterFactory::new));
-        tokenFilters.put("arabic_stem", ArabicStemTokenFilterFactory::new);
-        tokenFilters.put("brazilian_stem", BrazilianStemTokenFilterFactory::new);
-        tokenFilters.put("czech_stem", CzechStemTokenFilterFactory::new);
-        tokenFilters.put("dutch_stem", DutchStemTokenFilterFactory::new);
-        tokenFilters.put("french_stem", FrenchStemTokenFilterFactory::new);
-        tokenFilters.put("german_stem", GermanStemTokenFilterFactory::new);
-        tokenFilters.put("russian_stem", RussianStemTokenFilterFactory::new);
-        tokenFilters.put("keyword_marker", requriesAnalysisSettings(KeywordMarkerTokenFilterFactory::new));
-        tokenFilters.put("stemmer_override", requriesAnalysisSettings(StemmerOverrideTokenFilterFactory::new));
-        tokenFilters.put("arabic_normalization", ArabicNormalizationFilterFactory::new);
-        tokenFilters.put("german_normalization", GermanNormalizationFilterFactory::new);
-        tokenFilters.put("hindi_normalization", HindiNormalizationFilterFactory::new);
-        tokenFilters.put("indic_normalization", IndicNormalizationFilterFactory::new);
-        tokenFilters.put("sorani_normalization", SoraniNormalizationFilterFactory::new);
-        tokenFilters.put("persian_normalization", PersianNormalizationFilterFactory::new);
-        tokenFilters.put("scandinavian_normalization", ScandinavianNormalizationFilterFactory::new);
-        tokenFilters.put("scandinavian_folding", ScandinavianFoldingFilterFactory::new);
-        tokenFilters.put("serbian_normalization", SerbianNormalizationFilterFactory::new);
-
-        if (hunspellService != null) {
-            tokenFilters.put("hunspell", requriesAnalysisSettings((indexSettings, env, name, settings) -> new HunspellTokenFilterFactory(indexSettings, name, settings, hunspellService)));
-        }
-        tokenFilters.put("cjk_bigram", CJKBigramFilterFactory::new);
-        tokenFilters.put("cjk_width", CJKWidthFilterFactory::new);
-
-        tokenFilters.put("apostrophe", ApostropheFilterFactory::new);
-        tokenFilters.put("classic", ClassicFilterFactory::new);
-        tokenFilters.put("decimal_digit", DecimalDigitFilterFactory::new);
-        tokenFilters.put("fingerprint", FingerprintTokenFilterFactory::new);
-    }
-
-    private void registerBuiltInAnalyzer(Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider>> analyzers) {
-        analyzers.put("default", StandardAnalyzerProvider::new);
-        analyzers.put("standard", StandardAnalyzerProvider::new);
-        analyzers.put("standard_html_strip", StandardHtmlStripAnalyzerProvider::new);
-        analyzers.put("simple", SimpleAnalyzerProvider::new);
-        analyzers.put("stop", StopAnalyzerProvider::new);
-        analyzers.put("whitespace", WhitespaceAnalyzerProvider::new);
-        analyzers.put("keyword", KeywordAnalyzerProvider::new);
-        analyzers.put("pattern", PatternAnalyzerProvider::new);
-        analyzers.put("snowball", SnowballAnalyzerProvider::new);
-        analyzers.put("arabic", ArabicAnalyzerProvider::new);
-        analyzers.put("armenian", ArmenianAnalyzerProvider::new);
-        analyzers.put("basque", BasqueAnalyzerProvider::new);
-        analyzers.put("brazilian", BrazilianAnalyzerProvider::new);
-        analyzers.put("bulgarian", BulgarianAnalyzerProvider::new);
-        analyzers.put("catalan", CatalanAnalyzerProvider::new);
-        analyzers.put("chinese", ChineseAnalyzerProvider::new);
-        analyzers.put("cjk", CjkAnalyzerProvider::new);
-        analyzers.put("czech", CzechAnalyzerProvider::new);
-        analyzers.put("danish", DanishAnalyzerProvider::new);
-        analyzers.put("dutch", DutchAnalyzerProvider::new);
-        analyzers.put("english", EnglishAnalyzerProvider::new);
-        analyzers.put("finnish", FinnishAnalyzerProvider::new);
-        analyzers.put("french", FrenchAnalyzerProvider::new);
-        analyzers.put("galician", GalicianAnalyzerProvider::new);
-        analyzers.put("german", GermanAnalyzerProvider::new);
-        analyzers.put("greek", GreekAnalyzerProvider::new);
-        analyzers.put("hindi", HindiAnalyzerProvider::new);
-        analyzers.put("hungarian", HungarianAnalyzerProvider::new);
-        analyzers.put("indonesian", IndonesianAnalyzerProvider::new);
-        analyzers.put("irish", IrishAnalyzerProvider::new);
-        analyzers.put("italian", ItalianAnalyzerProvider::new);
-        analyzers.put("latvian", LatvianAnalyzerProvider::new);
-        analyzers.put("lithuanian", LithuanianAnalyzerProvider::new);
-        analyzers.put("norwegian", NorwegianAnalyzerProvider::new);
-        analyzers.put("persian", PersianAnalyzerProvider::new);
-        analyzers.put("portuguese", PortugueseAnalyzerProvider::new);
-        analyzers.put("romanian", RomanianAnalyzerProvider::new);
-        analyzers.put("russian", RussianAnalyzerProvider::new);
-        analyzers.put("sorani", SoraniAnalyzerProvider::new);
-        analyzers.put("spanish", SpanishAnalyzerProvider::new);
-        analyzers.put("swedish", SwedishAnalyzerProvider::new);
-        analyzers.put("turkish", TurkishAnalyzerProvider::new);
-        analyzers.put("thai", ThaiAnalyzerProvider::new);
-        analyzers.put("fingerprint", FingerprintAnalyzerProvider::new);
-    }
-
-    private <T> Map<String, T> buildMapping(boolean analyzer, String toBuild, IndexSettings settings, Map<String, Settings> settingsMap, Map<String, AnalysisModule.AnalysisProvider<T>> providerMap, Map<String, AnalysisModule.AnalysisProvider<T>> defaultInstance) throws IOException {
+    private <T> Map<String, T> buildMapping(boolean analyzer, String toBuild, IndexSettings settings, Map<String, Settings> settingsMap,
+            Map<String, AnalysisModule.AnalysisProvider<T>> providerMap, Map<String, AnalysisModule.AnalysisProvider<T>> defaultInstance)
+            throws IOException {
         Settings defaultSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, settings.getIndexVersionCreated()).build();
         Map<String, T> factories = new HashMap<>();
         for (Map.Entry<String, Settings> entry : settingsMap.entrySet()) {
@@ -383,7 +234,7 @@ public final class AnalysisRegistry implements Closeable {
 
     private static class PrebuiltAnalysis implements Closeable {
 
-        final Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider>> analyzerProviderFactories;
+        final Map<String, AnalysisModule.AnalysisProvider<AnalyzerProvider<?>>> analyzerProviderFactories;
         final Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> tokenizerFactories;
         final Map<String, AnalysisModule.AnalysisProvider<TokenFilterFactory>> tokenFilterFactories;
         final Map<String, AnalysisModule.AnalysisProvider<CharFilterFactory>> charFilterFactories;
@@ -446,7 +297,7 @@ public final class AnalysisRegistry implements Closeable {
             return tokenizerFactories.get(name);
         }
 
-        public AnalysisModule.AnalysisProvider<AnalyzerProvider> getAnalyzerProvider(String name) {
+        public AnalysisModule.AnalysisProvider<AnalyzerProvider<?>> getAnalyzerProvider(String name) {
             return analyzerProviderFactories.get(name);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java
index b9146df8c96..0009fc95409 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/AnalysisService.java
@@ -48,7 +48,7 @@ public class AnalysisService extends AbstractIndexComponent implements Closeable
     private final NamedAnalyzer defaultSearchQuoteAnalyzer;
 
     public AnalysisService(IndexSettings indexSettings,
-                           Map<String, AnalyzerProvider> analyzerProviders,
+                           Map<String, AnalyzerProvider<?>> analyzerProviders,
                            Map<String, TokenizerFactory> tokenizerFactoryFactories,
                            Map<String, CharFilterFactory> charFilterFactoryFactories,
                            Map<String, TokenFilterFactory> tokenFilterFactoryFactories) {
@@ -69,8 +69,8 @@ public class AnalysisService extends AbstractIndexComponent implements Closeable
         }
 
         Map<String, NamedAnalyzer> analyzers = new HashMap<>();
-        for (Map.Entry<String, AnalyzerProvider> entry : analyzerProviders.entrySet()) {
-            AnalyzerProvider analyzerFactory = entry.getValue();
+        for (Map.Entry<String, AnalyzerProvider<?>> entry : analyzerProviders.entrySet()) {
+            AnalyzerProvider<?> analyzerFactory = entry.getValue();
             String name = entry.getKey();
             /*
              * Lucene defaults positionIncrementGap to 0 in all analyzers but
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java b/core/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java
index 00724dccfa1..786e24a0844 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java
@@ -32,7 +32,7 @@ import java.io.IOException;
 /**
  *
  */
-public class PreBuiltAnalyzerProviderFactory implements AnalysisModule.AnalysisProvider<AnalyzerProvider> {
+public class PreBuiltAnalyzerProviderFactory implements AnalysisModule.AnalysisProvider<AnalyzerProvider<?>> {
 
     private final PreBuiltAnalyzerProvider analyzerProvider;
 
@@ -40,7 +40,7 @@ public class PreBuiltAnalyzerProviderFactory implements AnalysisModule.AnalysisP
         analyzerProvider = new PreBuiltAnalyzerProvider(name, scope, analyzer);
     }
 
-    public AnalyzerProvider create(String name, Settings settings) {
+    public AnalyzerProvider<?> create(String name, Settings settings) {
         Version indexVersion = Version.indexCreated(settings);
         if (!Version.CURRENT.equals(indexVersion)) {
             PreBuiltAnalyzers preBuiltAnalyzers = PreBuiltAnalyzers.getOrDefault(name, null);
@@ -54,7 +54,8 @@ public class PreBuiltAnalyzerProviderFactory implements AnalysisModule.AnalysisP
     }
 
     @Override
-    public AnalyzerProvider get(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException {
+    public AnalyzerProvider<?> get(IndexSettings indexSettings, Environment environment, String name, Settings settings)
+            throws IOException {
         return create(name, settings);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
index b5b480fc909..12d0b8bbb6e 100644
--- a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
+++ b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
@@ -19,49 +19,141 @@
 
 package org.elasticsearch.indices.analysis;
 
-import org.apache.lucene.analysis.hunspell.Dictionary;
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
-import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.ASCIIFoldingTokenFilterFactory;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.index.analysis.AnalyzerProvider;
+import org.elasticsearch.index.analysis.ApostropheFilterFactory;
+import org.elasticsearch.index.analysis.ArabicAnalyzerProvider;
+import org.elasticsearch.index.analysis.ArabicNormalizationFilterFactory;
+import org.elasticsearch.index.analysis.ArabicStemTokenFilterFactory;
+import org.elasticsearch.index.analysis.ArmenianAnalyzerProvider;
+import org.elasticsearch.index.analysis.BasqueAnalyzerProvider;
+import org.elasticsearch.index.analysis.BrazilianAnalyzerProvider;
+import org.elasticsearch.index.analysis.BrazilianStemTokenFilterFactory;
+import org.elasticsearch.index.analysis.BulgarianAnalyzerProvider;
+import org.elasticsearch.index.analysis.CJKBigramFilterFactory;
+import org.elasticsearch.index.analysis.CJKWidthFilterFactory;
+import org.elasticsearch.index.analysis.CatalanAnalyzerProvider;
 import org.elasticsearch.index.analysis.CharFilterFactory;
+import org.elasticsearch.index.analysis.ChineseAnalyzerProvider;
+import org.elasticsearch.index.analysis.CjkAnalyzerProvider;
+import org.elasticsearch.index.analysis.ClassicFilterFactory;
+import org.elasticsearch.index.analysis.ClassicTokenizerFactory;
+import org.elasticsearch.index.analysis.CommonGramsTokenFilterFactory;
+import org.elasticsearch.index.analysis.CzechAnalyzerProvider;
+import org.elasticsearch.index.analysis.CzechStemTokenFilterFactory;
+import org.elasticsearch.index.analysis.DanishAnalyzerProvider;
+import org.elasticsearch.index.analysis.DecimalDigitFilterFactory;
+import org.elasticsearch.index.analysis.DelimitedPayloadTokenFilterFactory;
+import org.elasticsearch.index.analysis.DutchAnalyzerProvider;
+import org.elasticsearch.index.analysis.DutchStemTokenFilterFactory;
+import org.elasticsearch.index.analysis.EdgeNGramTokenFilterFactory;
+import org.elasticsearch.index.analysis.EdgeNGramTokenizerFactory;
+import org.elasticsearch.index.analysis.ElisionTokenFilterFactory;
+import org.elasticsearch.index.analysis.EnglishAnalyzerProvider;
+import org.elasticsearch.index.analysis.FingerprintAnalyzerProvider;
+import org.elasticsearch.index.analysis.FingerprintTokenFilterFactory;
+import org.elasticsearch.index.analysis.FinnishAnalyzerProvider;
+import org.elasticsearch.index.analysis.FrenchAnalyzerProvider;
+import org.elasticsearch.index.analysis.FrenchStemTokenFilterFactory;
+import org.elasticsearch.index.analysis.GalicianAnalyzerProvider;
+import org.elasticsearch.index.analysis.GermanAnalyzerProvider;
+import org.elasticsearch.index.analysis.GermanNormalizationFilterFactory;
+import org.elasticsearch.index.analysis.GermanStemTokenFilterFactory;
+import org.elasticsearch.index.analysis.GreekAnalyzerProvider;
+import org.elasticsearch.index.analysis.HindiAnalyzerProvider;
+import org.elasticsearch.index.analysis.HindiNormalizationFilterFactory;
+import org.elasticsearch.index.analysis.HtmlStripCharFilterFactory;
+import org.elasticsearch.index.analysis.HungarianAnalyzerProvider;
+import org.elasticsearch.index.analysis.HunspellTokenFilterFactory;
+import org.elasticsearch.index.analysis.IndicNormalizationFilterFactory;
+import org.elasticsearch.index.analysis.IndonesianAnalyzerProvider;
+import org.elasticsearch.index.analysis.IrishAnalyzerProvider;
+import org.elasticsearch.index.analysis.ItalianAnalyzerProvider;
+import org.elasticsearch.index.analysis.KStemTokenFilterFactory;
+import org.elasticsearch.index.analysis.KeepTypesFilterFactory;
+import org.elasticsearch.index.analysis.KeepWordFilterFactory;
+import org.elasticsearch.index.analysis.KeywordAnalyzerProvider;
+import org.elasticsearch.index.analysis.KeywordMarkerTokenFilterFactory;
+import org.elasticsearch.index.analysis.KeywordTokenizerFactory;
+import org.elasticsearch.index.analysis.LatvianAnalyzerProvider;
+import org.elasticsearch.index.analysis.LengthTokenFilterFactory;
+import org.elasticsearch.index.analysis.LetterTokenizerFactory;
+import org.elasticsearch.index.analysis.LimitTokenCountFilterFactory;
+import org.elasticsearch.index.analysis.LithuanianAnalyzerProvider;
+import org.elasticsearch.index.analysis.LowerCaseTokenFilterFactory;
+import org.elasticsearch.index.analysis.LowerCaseTokenizerFactory;
+import org.elasticsearch.index.analysis.MappingCharFilterFactory;
+import org.elasticsearch.index.analysis.NGramTokenFilterFactory;
+import org.elasticsearch.index.analysis.NGramTokenizerFactory;
+import org.elasticsearch.index.analysis.NorwegianAnalyzerProvider;
+import org.elasticsearch.index.analysis.PathHierarchyTokenizerFactory;
+import org.elasticsearch.index.analysis.PatternAnalyzerProvider;
+import org.elasticsearch.index.analysis.PatternCaptureGroupTokenFilterFactory;
+import org.elasticsearch.index.analysis.PatternReplaceCharFilterFactory;
+import org.elasticsearch.index.analysis.PatternReplaceTokenFilterFactory;
+import org.elasticsearch.index.analysis.PatternTokenizerFactory;
+import org.elasticsearch.index.analysis.PersianAnalyzerProvider;
+import org.elasticsearch.index.analysis.PersianNormalizationFilterFactory;
+import org.elasticsearch.index.analysis.PorterStemTokenFilterFactory;
+import org.elasticsearch.index.analysis.PortugueseAnalyzerProvider;
+import org.elasticsearch.index.analysis.ReverseTokenFilterFactory;
+import org.elasticsearch.index.analysis.RomanianAnalyzerProvider;
+import org.elasticsearch.index.analysis.RussianAnalyzerProvider;
+import org.elasticsearch.index.analysis.RussianStemTokenFilterFactory;
+import org.elasticsearch.index.analysis.ScandinavianFoldingFilterFactory;
+import org.elasticsearch.index.analysis.ScandinavianNormalizationFilterFactory;
+import org.elasticsearch.index.analysis.SerbianNormalizationFilterFactory;
+import org.elasticsearch.index.analysis.ShingleTokenFilterFactory;
+import org.elasticsearch.index.analysis.SimpleAnalyzerProvider;
+import org.elasticsearch.index.analysis.SnowballAnalyzerProvider;
+import org.elasticsearch.index.analysis.SnowballTokenFilterFactory;
+import org.elasticsearch.index.analysis.SoraniAnalyzerProvider;
+import org.elasticsearch.index.analysis.SoraniNormalizationFilterFactory;
+import org.elasticsearch.index.analysis.SpanishAnalyzerProvider;
+import org.elasticsearch.index.analysis.StandardAnalyzerProvider;
+import org.elasticsearch.index.analysis.StandardHtmlStripAnalyzerProvider;
+import org.elasticsearch.index.analysis.StandardTokenFilterFactory;
+import org.elasticsearch.index.analysis.StandardTokenizerFactory;
+import org.elasticsearch.index.analysis.StemmerOverrideTokenFilterFactory;
+import org.elasticsearch.index.analysis.StemmerTokenFilterFactory;
+import org.elasticsearch.index.analysis.StopAnalyzerProvider;
+import org.elasticsearch.index.analysis.StopTokenFilterFactory;
+import org.elasticsearch.index.analysis.SwedishAnalyzerProvider;
+import org.elasticsearch.index.analysis.ThaiAnalyzerProvider;
+import org.elasticsearch.index.analysis.ThaiTokenizerFactory;
 import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.index.analysis.TokenizerFactory;
+import org.elasticsearch.index.analysis.TrimTokenFilterFactory;
+import org.elasticsearch.index.analysis.TruncateTokenFilterFactory;
+import org.elasticsearch.index.analysis.TurkishAnalyzerProvider;
+import org.elasticsearch.index.analysis.UAX29URLEmailTokenizerFactory;
+import org.elasticsearch.index.analysis.UniqueTokenFilterFactory;
+import org.elasticsearch.index.analysis.UpperCaseTokenFilterFactory;
+import org.elasticsearch.index.analysis.WhitespaceAnalyzerProvider;
+import org.elasticsearch.index.analysis.WhitespaceTokenizerFactory;
+import org.elasticsearch.index.analysis.WordDelimiterTokenFilterFactory;
+import org.elasticsearch.index.analysis.compound.DictionaryCompoundWordTokenFilterFactory;
+import org.elasticsearch.index.analysis.compound.HyphenationCompoundWordTokenFilterFactory;
+import org.elasticsearch.plugins.AnalysisPlugin;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
+
+import static java.util.Objects.requireNonNull;
 
 /**
- * The AnalysisModule is the main extension point for node and index level analysis components. The lucene classes
- * {@link org.apache.lucene.analysis.Analyzer}, {@link org.apache.lucene.analysis.TokenFilter}, {@link org.apache.lucene.analysis.Tokenizer}
- * and {@link org.apache.lucene.analysis.CharFilter} can be extended in plugins and registered on node startup when the analysis module
- * gets loaded. Since elasticsearch needs to create multiple instances for different configurations dedicated factories need to be provided for
- * each of the components:
- * <ul>
- *     <li> {@link org.apache.lucene.analysis.Analyzer} can be exposed via {@link AnalyzerProvider} and registered on {@link #registerAnalyzer(String, AnalysisProvider)}</li>
- *     <li> {@link org.apache.lucene.analysis.TokenFilter} can be exposed via {@link TokenFilterFactory} and registered on {@link #registerTokenFilter(String, AnalysisProvider)}</li>
- *     <li> {@link org.apache.lucene.analysis.Tokenizer} can be exposed via {@link TokenizerFactory} and registered on {@link #registerTokenizer(String, AnalysisProvider)}</li>
- *     <li> {@link org.apache.lucene.analysis.CharFilter} can be exposed via {@link CharFilterFactory} and registered on {@link #registerCharFilter(String, AnalysisProvider)}</li>
- * </ul>
- *
- * The {@link org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider} is only a functional interface that allows to register factory constructors directly like the plugin example below:
- * <pre>
- *     public class MyAnalysisPlugin extends Plugin {
- *       public void onModule(AnalysisModule module) {
- *         module.registerAnalyzer("my-analyzer-name", MyAnalyzer::new);
- *       }
- *     }
- * </pre>
+ * Sets up {@link AnalysisRegistry}.
  */
-public final class AnalysisModule extends AbstractModule {
-
+public final class AnalysisModule {
     static {
         Settings build = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
@@ -71,108 +163,195 @@ public final class AnalysisModule extends AbstractModule {
         NA_INDEX_SETTINGS = new IndexSettings(metaData, Settings.EMPTY);
     }
     private static final IndexSettings NA_INDEX_SETTINGS;
-    private final Environment environment;
-    private final Map<String, AnalysisProvider<CharFilterFactory>> charFilters = new HashMap<>();
-    private final Map<String, AnalysisProvider<TokenFilterFactory>> tokenFilters = new HashMap<>();
-    private final Map<String, AnalysisProvider<TokenizerFactory>> tokenizers = new HashMap<>();
-    private final Map<String, AnalysisProvider<AnalyzerProvider>> analyzers = new HashMap<>();
-    private final Map<String, org.apache.lucene.analysis.hunspell.Dictionary> knownDictionaries = new HashMap<>();
 
-    /**
-     * Creates a new AnalysisModule
-     */
-    public AnalysisModule(Environment environment) {
-        this.environment = environment;
+    private final HunspellService hunspellService;
+    private final AnalysisRegistry analysisRegistry;
+
+    public AnalysisModule(Environment environment, List<AnalysisPlugin> plugins) throws IOException {
+        NamedRegistry<AnalysisProvider<CharFilterFactory>> charFilters = setupCharFilters(plugins);
+        NamedRegistry<org.apache.lucene.analysis.hunspell.Dictionary> hunspellDictionaries = setupHunspellDictionaries(plugins);
+        hunspellService = new HunspellService(environment.settings(), environment, hunspellDictionaries.registry);
+        NamedRegistry<AnalysisProvider<TokenFilterFactory>> tokenFilters = setupTokenFilters(plugins, hunspellService);
+        NamedRegistry<AnalysisProvider<TokenizerFactory>> tokenizers = setupTokenizers(plugins);
+        NamedRegistry<AnalysisProvider<AnalyzerProvider<?>>> analyzers = setupAnalyzers(plugins);
+        analysisRegistry = new AnalysisRegistry(environment, charFilters.registry, tokenFilters.registry,
+                tokenizers.registry, analyzers.registry);
+    }
+
+    HunspellService getHunspellService() {
+        return hunspellService;
+    }
+
+    public AnalysisRegistry getAnalysisRegistry() {
+        return analysisRegistry;
+    }
+
+    private NamedRegistry<AnalysisProvider<CharFilterFactory>> setupCharFilters(List<AnalysisPlugin> plugins) {
+        NamedRegistry<AnalysisProvider<CharFilterFactory>> charFilters = new NamedRegistry<>("char_filter");
+        charFilters.register("html_strip", HtmlStripCharFilterFactory::new);
+        charFilters.register("pattern_replace", requriesAnalysisSettings(PatternReplaceCharFilterFactory::new));
+        charFilters.register("mapping", requriesAnalysisSettings(MappingCharFilterFactory::new));
+        charFilters.registerPlugins(plugins, AnalysisPlugin::getCharFilters);
+        return charFilters;
+    }
+
+    public NamedRegistry<org.apache.lucene.analysis.hunspell.Dictionary> setupHunspellDictionaries(List<AnalysisPlugin> plugins) {
+        NamedRegistry<org.apache.lucene.analysis.hunspell.Dictionary> hunspellDictionaries = new NamedRegistry<>("dictionary");
+        hunspellDictionaries.registerPlugins(plugins, AnalysisPlugin::getHunspellDictionaries);
+        return hunspellDictionaries;
+    }
+
+    private NamedRegistry<AnalysisProvider<TokenFilterFactory>> setupTokenFilters(List<AnalysisPlugin> plugins,
+            HunspellService hunspellService) {
+        NamedRegistry<AnalysisProvider<TokenFilterFactory>> tokenFilters = new NamedRegistry<>("token_filter");
+        tokenFilters.register("stop", StopTokenFilterFactory::new);
+        tokenFilters.register("reverse", ReverseTokenFilterFactory::new);
+        tokenFilters.register("asciifolding", ASCIIFoldingTokenFilterFactory::new);
+        tokenFilters.register("length", LengthTokenFilterFactory::new);
+        tokenFilters.register("lowercase", LowerCaseTokenFilterFactory::new);
+        tokenFilters.register("uppercase", UpperCaseTokenFilterFactory::new);
+        tokenFilters.register("porter_stem", PorterStemTokenFilterFactory::new);
+        tokenFilters.register("kstem", KStemTokenFilterFactory::new);
+        tokenFilters.register("standard", StandardTokenFilterFactory::new);
+        tokenFilters.register("nGram", NGramTokenFilterFactory::new);
+        tokenFilters.register("ngram", NGramTokenFilterFactory::new);
+        tokenFilters.register("edgeNGram", EdgeNGramTokenFilterFactory::new);
+        tokenFilters.register("edge_ngram", EdgeNGramTokenFilterFactory::new);
+        tokenFilters.register("shingle", ShingleTokenFilterFactory::new);
+        tokenFilters.register("unique", UniqueTokenFilterFactory::new);
+        tokenFilters.register("truncate", requriesAnalysisSettings(TruncateTokenFilterFactory::new));
+        tokenFilters.register("trim", TrimTokenFilterFactory::new);
+        tokenFilters.register("limit", LimitTokenCountFilterFactory::new);
+        tokenFilters.register("common_grams", requriesAnalysisSettings(CommonGramsTokenFilterFactory::new));
+        tokenFilters.register("snowball", SnowballTokenFilterFactory::new);
+        tokenFilters.register("stemmer", StemmerTokenFilterFactory::new);
+        tokenFilters.register("word_delimiter", WordDelimiterTokenFilterFactory::new);
+        tokenFilters.register("delimited_payload_filter", DelimitedPayloadTokenFilterFactory::new);
+        tokenFilters.register("elision", ElisionTokenFilterFactory::new);
+        tokenFilters.register("keep", requriesAnalysisSettings(KeepWordFilterFactory::new));
+        tokenFilters.register("keep_types", requriesAnalysisSettings(KeepTypesFilterFactory::new));
+        tokenFilters.register("pattern_capture", requriesAnalysisSettings(PatternCaptureGroupTokenFilterFactory::new));
+        tokenFilters.register("pattern_replace", requriesAnalysisSettings(PatternReplaceTokenFilterFactory::new));
+        tokenFilters.register("dictionary_decompounder", requriesAnalysisSettings(DictionaryCompoundWordTokenFilterFactory::new));
+        tokenFilters.register("hyphenation_decompounder", requriesAnalysisSettings(HyphenationCompoundWordTokenFilterFactory::new));
+        tokenFilters.register("arabic_stem", ArabicStemTokenFilterFactory::new);
+        tokenFilters.register("brazilian_stem", BrazilianStemTokenFilterFactory::new);
+        tokenFilters.register("czech_stem", CzechStemTokenFilterFactory::new);
+        tokenFilters.register("dutch_stem", DutchStemTokenFilterFactory::new);
+        tokenFilters.register("french_stem", FrenchStemTokenFilterFactory::new);
+        tokenFilters.register("german_stem", GermanStemTokenFilterFactory::new);
+        tokenFilters.register("russian_stem", RussianStemTokenFilterFactory::new);
+        tokenFilters.register("keyword_marker", requriesAnalysisSettings(KeywordMarkerTokenFilterFactory::new));
+        tokenFilters.register("stemmer_override", requriesAnalysisSettings(StemmerOverrideTokenFilterFactory::new));
+        tokenFilters.register("arabic_normalization", ArabicNormalizationFilterFactory::new);
+        tokenFilters.register("german_normalization", GermanNormalizationFilterFactory::new);
+        tokenFilters.register("hindi_normalization", HindiNormalizationFilterFactory::new);
+        tokenFilters.register("indic_normalization", IndicNormalizationFilterFactory::new);
+        tokenFilters.register("sorani_normalization", SoraniNormalizationFilterFactory::new);
+        tokenFilters.register("persian_normalization", PersianNormalizationFilterFactory::new);
+        tokenFilters.register("scandinavian_normalization", ScandinavianNormalizationFilterFactory::new);
+        tokenFilters.register("scandinavian_folding", ScandinavianFoldingFilterFactory::new);
+        tokenFilters.register("serbian_normalization", SerbianNormalizationFilterFactory::new);
+
+        tokenFilters.register("hunspell", requriesAnalysisSettings(
+                (indexSettings, env, name, settings) -> new HunspellTokenFilterFactory(indexSettings, name, settings, hunspellService)));
+        tokenFilters.register("cjk_bigram", CJKBigramFilterFactory::new);
+        tokenFilters.register("cjk_width", CJKWidthFilterFactory::new);
+
+        tokenFilters.register("apostrophe", ApostropheFilterFactory::new);
+        tokenFilters.register("classic", ClassicFilterFactory::new);
+        tokenFilters.register("decimal_digit", DecimalDigitFilterFactory::new);
+        tokenFilters.register("fingerprint", FingerprintTokenFilterFactory::new);
+        tokenFilters.registerPlugins(plugins, AnalysisPlugin::getTokenFilters);
+        return tokenFilters;
+    }
+
+    private NamedRegistry<AnalysisProvider<TokenizerFactory>> setupTokenizers(List<AnalysisPlugin> plugins) {
+        NamedRegistry<AnalysisProvider<TokenizerFactory>> tokenizers = new NamedRegistry<>("tokenizer");
+        tokenizers.register("standard", StandardTokenizerFactory::new);
+        tokenizers.register("uax_url_email", UAX29URLEmailTokenizerFactory::new);
+        tokenizers.register("path_hierarchy", PathHierarchyTokenizerFactory::new);
+        tokenizers.register("PathHierarchy", PathHierarchyTokenizerFactory::new);
+        tokenizers.register("keyword", KeywordTokenizerFactory::new);
+        tokenizers.register("letter", LetterTokenizerFactory::new);
+        tokenizers.register("lowercase", LowerCaseTokenizerFactory::new);
+        tokenizers.register("whitespace", WhitespaceTokenizerFactory::new);
+        tokenizers.register("nGram", NGramTokenizerFactory::new);
+        tokenizers.register("ngram", NGramTokenizerFactory::new);
+        tokenizers.register("edgeNGram", EdgeNGramTokenizerFactory::new);
+        tokenizers.register("edge_ngram", EdgeNGramTokenizerFactory::new);
+        tokenizers.register("pattern", PatternTokenizerFactory::new);
+        tokenizers.register("classic", ClassicTokenizerFactory::new);
+        tokenizers.register("thai", ThaiTokenizerFactory::new);
+        tokenizers.registerPlugins(plugins, AnalysisPlugin::getTokenizers);
+        return tokenizers;
+    }
+
+    private NamedRegistry<AnalysisProvider<AnalyzerProvider<?>>> setupAnalyzers(List<AnalysisPlugin> plugins) {
+        NamedRegistry<AnalysisProvider<AnalyzerProvider<?>>> analyzers = new NamedRegistry<>("analyzer");
+        analyzers.register("default", StandardAnalyzerProvider::new);
+        analyzers.register("standard", StandardAnalyzerProvider::new);
+        analyzers.register("standard_html_strip", StandardHtmlStripAnalyzerProvider::new);
+        analyzers.register("simple", SimpleAnalyzerProvider::new);
+        analyzers.register("stop", StopAnalyzerProvider::new);
+        analyzers.register("whitespace", WhitespaceAnalyzerProvider::new);
+        analyzers.register("keyword", KeywordAnalyzerProvider::new);
+        analyzers.register("pattern", PatternAnalyzerProvider::new);
+        analyzers.register("snowball", SnowballAnalyzerProvider::new);
+        analyzers.register("arabic", ArabicAnalyzerProvider::new);
+        analyzers.register("armenian", ArmenianAnalyzerProvider::new);
+        analyzers.register("basque", BasqueAnalyzerProvider::new);
+        analyzers.register("brazilian", BrazilianAnalyzerProvider::new);
+        analyzers.register("bulgarian", BulgarianAnalyzerProvider::new);
+        analyzers.register("catalan", CatalanAnalyzerProvider::new);
+        analyzers.register("chinese", ChineseAnalyzerProvider::new);
+        analyzers.register("cjk", CjkAnalyzerProvider::new);
+        analyzers.register("czech", CzechAnalyzerProvider::new);
+        analyzers.register("danish", DanishAnalyzerProvider::new);
+        analyzers.register("dutch", DutchAnalyzerProvider::new);
+        analyzers.register("english", EnglishAnalyzerProvider::new);
+        analyzers.register("finnish", FinnishAnalyzerProvider::new);
+        analyzers.register("french", FrenchAnalyzerProvider::new);
+        analyzers.register("galician", GalicianAnalyzerProvider::new);
+        analyzers.register("german", GermanAnalyzerProvider::new);
+        analyzers.register("greek", GreekAnalyzerProvider::new);
+        analyzers.register("hindi", HindiAnalyzerProvider::new);
+        analyzers.register("hungarian", HungarianAnalyzerProvider::new);
+        analyzers.register("indonesian", IndonesianAnalyzerProvider::new);
+        analyzers.register("irish", IrishAnalyzerProvider::new);
+        analyzers.register("italian", ItalianAnalyzerProvider::new);
+        analyzers.register("latvian", LatvianAnalyzerProvider::new);
+        analyzers.register("lithuanian", LithuanianAnalyzerProvider::new);
+        analyzers.register("norwegian", NorwegianAnalyzerProvider::new);
+        analyzers.register("persian", PersianAnalyzerProvider::new);
+        analyzers.register("portuguese", PortugueseAnalyzerProvider::new);
+        analyzers.register("romanian", RomanianAnalyzerProvider::new);
+        analyzers.register("russian", RussianAnalyzerProvider::new);
+        analyzers.register("sorani", SoraniAnalyzerProvider::new);
+        analyzers.register("spanish", SpanishAnalyzerProvider::new);
+        analyzers.register("swedish", SwedishAnalyzerProvider::new);
+        analyzers.register("turkish", TurkishAnalyzerProvider::new);
+        analyzers.register("thai", ThaiAnalyzerProvider::new);
+        analyzers.register("fingerprint", FingerprintAnalyzerProvider::new);
+        analyzers.registerPlugins(plugins, AnalysisPlugin::getAnalyzers);
+        return analyzers;
+    }
+
+    private static <T> AnalysisModule.AnalysisProvider<T> requriesAnalysisSettings(AnalysisModule.AnalysisProvider<T> provider) {
+        return new AnalysisModule.AnalysisProvider<T>() {
+            @Override
+            public T get(IndexSettings indexSettings, Environment environment, String name, Settings settings) throws IOException {
+                return provider.get(indexSettings, environment, name, settings);
+            }
+            @Override
+            public boolean requiresAnalysisSettings() {
+                return true;
+            }
+        };
     }
 
     /**
-     * Registers a new {@link AnalysisProvider} to create
-     * {@link CharFilterFactory} instance per node as well as per index.
-     */
-    public void registerCharFilter(String name, AnalysisProvider<CharFilterFactory> charFilter) {
-        if (charFilter == null) {
-            throw new IllegalArgumentException("char_filter provider must not be null");
-        }
-        if (charFilters.putIfAbsent(name, charFilter) != null) {
-            throw new IllegalArgumentException("char_filter provider for name " + name + " already registered");
-        }
-    }
-
-    /**
-     * Registers a new {@link AnalysisProvider} to create
-     * {@link TokenFilterFactory} instance per node as well as per index.
-     */
-    public void registerTokenFilter(String name, AnalysisProvider<TokenFilterFactory> tokenFilter) {
-        if (tokenFilter == null) {
-            throw new IllegalArgumentException("token_filter provider must not be null");
-        }
-        if (tokenFilters.putIfAbsent(name, tokenFilter) != null) {
-            throw new IllegalArgumentException("token_filter provider for name " + name + " already registered");
-        }
-    }
-
-    /**
-     * Registers a new {@link AnalysisProvider} to create
-     * {@link TokenizerFactory} instance per node as well as per index.
-     */
-    public void registerTokenizer(String name, AnalysisProvider<TokenizerFactory> tokenizer) {
-        if (tokenizer == null) {
-            throw new IllegalArgumentException("tokenizer provider must not be null");
-        }
-        if (tokenizers.putIfAbsent(name, tokenizer) != null) {
-            throw new IllegalArgumentException("tokenizer provider for name " + name + " already registered");
-        }
-    }
-
-    /**
-     * Registers a new {@link AnalysisProvider} to create
-     * {@link AnalyzerProvider} instance per node as well as per index.
-     */
-    public void registerAnalyzer(String name, AnalysisProvider<AnalyzerProvider> analyzer) {
-        if (analyzer == null) {
-            throw new IllegalArgumentException("analyzer provider must not be null");
-        }
-        if (analyzers.putIfAbsent(name, analyzer) != null) {
-            throw new IllegalArgumentException("analyzer provider for name " + name + " already registered");
-        }
-    }
-
-    /**
-     * Registers a new hunspell {@link Dictionary} that can be referenced by the given name in
-     * hunspell analysis configuration.
-     */
-    public void registerHunspellDictionary(String name, Dictionary dictionary) {
-        if (knownDictionaries.putIfAbsent(name, dictionary) != null) {
-            throw new IllegalArgumentException("dictionary for [" + name + "] is already registered");
-        }
-    }
-
-    @Override
-    protected void configure() {
-        try {
-            AnalysisRegistry registry = buildRegistry();
-            bind(HunspellService.class).toInstance(registry.getHunspellService());
-            bind(AnalysisRegistry.class).toInstance(registry);
-        } catch (IOException e) {
-            throw new ElasticsearchException("failed to load hunspell service", e);
-        }
-    }
-
-    /**
-     * Builds an {@link AnalysisRegistry} from the current configuration.
-     */
-    public AnalysisRegistry buildRegistry() throws IOException {
-        return new AnalysisRegistry(new HunspellService(environment.settings(), environment, knownDictionaries), environment, charFilters, tokenFilters, tokenizers, analyzers);
-    }
-
-    /**
-     * AnalysisProvider is the basic factory interface for registering analysis components like:
-     * <ul>
-     *     <li>{@link TokenizerFactory} - see {@link AnalysisModule#registerTokenizer(String, AnalysisProvider)}</li>
-     *     <li>{@link CharFilterFactory} - see {@link AnalysisModule#registerCharFilter(String, AnalysisProvider)}</li>
-     *     <li>{@link AnalyzerProvider} - see {@link AnalysisModule#registerAnalyzer(String, AnalysisProvider)}</li>
-     *     <li>{@link TokenFilterFactory}- see {@link AnalysisModule#registerTokenFilter(String, AnalysisProvider)} )}</li>
-     * </ul>
+     * The basic factory interface for analysis components.
      */
     public interface AnalysisProvider<T> {
 
@@ -195,7 +374,8 @@ public final class AnalysisModule extends AbstractModule {
          * @param name the name of the analysis component
          * @return a new provider instance
          * @throws IOException if an {@link IOException} occurs
-         * @throws IllegalArgumentException if the provider requires analysis settings ie. if {@link #requiresAnalysisSettings()} returns <code>true</code>
+         * @throws IllegalArgumentException if the provider requires analysis settings ie. if {@link #requiresAnalysisSettings()} returns
+         *  <code>true</code>
          */
         default T get(Environment environment, String name) throws IOException {
             if (requiresAnalysisSettings()) {
@@ -212,4 +392,29 @@ public final class AnalysisModule extends AbstractModule {
             return false;
         }
     }
+
+    private static class NamedRegistry<T> {
+        private final Map<String, T> registry = new HashMap<>();
+        private final String targetName;
+
+        public NamedRegistry(String targetName) {
+            this.targetName = targetName;
+        }
+
+        private void register(String name, T t) {
+            requireNonNull(name, "name is required");
+            requireNonNull(t, targetName + " is required");
+            if (registry.putIfAbsent(name, t) != null) {
+                throw new IllegalArgumentException(targetName + " for name " + name + " already registered");
+            }
+        }
+
+        private <P> void registerPlugins(List<P> plugins, Function<P, Map<String, T>> lookup) {
+            for (P plugin : plugins) {
+                for (Map.Entry<String, T> entry : lookup.apply(plugin).entrySet()) {
+                    register(entry.getKey(), entry.getValue());
+                }
+            }
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java b/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java
index 27b7ab23c9f..97667fcdac6 100644
--- a/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java
+++ b/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java
@@ -52,7 +52,8 @@ import java.util.function.Function;
  * The following settings can be set for each dictionary:
  * <ul>
  * <li>{@code ignore_case} - If true, dictionary matching will be case insensitive (defaults to {@code false})</li>
- * <li>{@code strict_affix_parsing} - Determines whether errors while reading a affix rules file will cause exception or simple be ignored (defaults to {@code true})</li>
+ * <li>{@code strict_affix_parsing} - Determines whether errors while reading a affix rules file will cause exception or simple be ignored
+ *      (defaults to {@code true})</li>
  * </ul>
  * <p>
  * These settings can either be configured as node level configuration, such as:
@@ -86,7 +87,8 @@ public class HunspellService extends AbstractComponent {
     private final Path hunspellDir;
     private final Function<String, Dictionary> loadingFunction;
 
-    public HunspellService(final Settings settings, final Environment env, final Map<String, Dictionary> knownDictionaries) throws IOException {
+    public HunspellService(final Settings settings, final Environment env, final Map<String, Dictionary> knownDictionaries)
+            throws IOException {
         super(settings);
         this.knownDictionaries = Collections.unmodifiableMap(knownDictionaries);
         this.hunspellDir = resolveHunspellDirectory(env);
@@ -166,7 +168,7 @@ public class HunspellService extends AbstractComponent {
 
         // merging node settings with hunspell dictionary specific settings
         Settings dictSettings = HUNSPELL_DICTIONARY_OPTIONS.get(nodeSettings);
-        nodeSettings = loadDictionarySettings(dicDir, dictSettings.getByPrefix(locale));
+        nodeSettings = loadDictionarySettings(dicDir, dictSettings.getByPrefix(locale + "."));
 
         boolean ignoreCase = nodeSettings.getAsBoolean("ignore_case", defaultIgnoreCase);
 
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index fb38c530d45..afc0e5b4cd8 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -72,6 +72,7 @@ import org.elasticsearch.gateway.GatewayModule;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.http.HttpServer;
 import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.analysis.AnalysisModule;
@@ -85,6 +86,7 @@ import org.elasticsearch.monitor.MonitorService;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
 import org.elasticsearch.node.service.NodeService;
+import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.plugins.ScriptPlugin;
@@ -227,6 +229,7 @@ public class Node implements Closeable {
             final ResourceWatcherService resourceWatcherService = new ResourceWatcherService(settings, threadPool);
             final ScriptModule scriptModule = ScriptModule.create(settings, environment, resourceWatcherService,
                     pluginsService.filterPlugins(ScriptPlugin.class));
+            AnalysisModule analysisModule = new AnalysisModule(environment, pluginsService.filterPlugins(AnalysisPlugin.class));
             additionalSettings.addAll(scriptModule.getSettings());
             // this is as early as we can validate settings at this point. we already pass them to ScriptModule as well as ThreadPool
             // so we might be late here already
@@ -260,7 +263,6 @@ public class Node implements Closeable {
             modules.add(new ActionModule(DiscoveryNode.isIngestNode(settings), false));
             modules.add(new GatewayModule());
             modules.add(new RepositoriesModule());
-            modules.add(new AnalysisModule(environment));
             pluginsService.processModules(modules);
             CircuitBreakerService circuitBreakerService = createCircuitBreakerService(settingsModule.getSettings(),
                 settingsModule.getClusterSettings());
@@ -279,6 +281,7 @@ public class Node implements Closeable {
                     b.bind(CircuitBreakerService.class).toInstance(circuitBreakerService);
                     b.bind(BigArrays.class).toInstance(bigArrays);
                     b.bind(ScriptService.class).toInstance(scriptModule.getScriptService());
+                    b.bind(AnalysisRegistry.class).toInstance(analysisModule.getAnalysisRegistry());
                 }
             );
             injector = modules.createInjector();
diff --git a/core/src/main/java/org/elasticsearch/plugins/AnalysisPlugin.java b/core/src/main/java/org/elasticsearch/plugins/AnalysisPlugin.java
new file mode 100644
index 00000000000..ffd278a5653
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/plugins/AnalysisPlugin.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.CharFilter;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.Tokenizer;
+import org.elasticsearch.index.analysis.AnalyzerProvider;
+import org.elasticsearch.index.analysis.CharFilterFactory;
+import org.elasticsearch.index.analysis.TokenFilterFactory;
+import org.elasticsearch.index.analysis.TokenizerFactory;
+import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
+
+import java.util.Map;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * An additional extension point for {@link Plugin}s that extends Elasticsearch's analysis functionality. To add an additional
+ * {@link TokenFilter} just implement the interface and implement the {@link #getTokenFilters()} method:
+ *
+ * <pre>{@code
+ * public class AnalysisPhoneticPlugin extends Plugin implements AnalysisPlugin {
+ *     &#64;Override
+ *     public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
+ *         return singletonMap("phonetic", PhoneticTokenFilterFactory::new);
+ *     }
+ * }
+ * }</pre>
+ */
+public interface AnalysisPlugin {
+    /**
+     * Override to add additional {@link CharFilter}s.
+     */
+    default Map<String, AnalysisProvider<CharFilterFactory>> getCharFilters() {
+        return emptyMap();
+    }
+
+    /**
+     * Override to add additional {@link TokenFilter}s.
+     */
+    default Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
+        return emptyMap();
+    }
+
+    /**
+     * Override to add additional {@link Tokenizer}s.
+     */
+    default Map<String, AnalysisProvider<TokenizerFactory>> getTokenizers() {
+        return emptyMap();
+    }
+
+    /**
+     * Override to add additional {@link Analyzer}s.
+     */
+    default Map<String, AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> getAnalyzers() {
+        return emptyMap();
+    }
+
+    /**
+     * Override to add additional hunspell {@link org.apache.lucene.analysis.hunspell.Dictionary}s.
+     */
+    default Map<String, org.apache.lucene.analysis.hunspell.Dictionary> getHunspellDictionaries() {
+        return emptyMap();
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/plugins/Plugin.java b/core/src/main/java/org/elasticsearch/plugins/Plugin.java
index 0df9eaf467f..08a8ce7124c 100644
--- a/core/src/main/java/org/elasticsearch/plugins/Plugin.java
+++ b/core/src/main/java/org/elasticsearch/plugins/Plugin.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.index.IndexModule;
+import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.threadpool.ExecutorBuilder;
 
@@ -103,6 +104,14 @@ public abstract class Plugin {
     @Deprecated
     public final void onModule(ScriptModule module) {}
 
+    /**
+     * Old-style analysis extension point.
+     *
+     * @deprecated implement {@link AnalysisPlugin} instead
+     */
+    @Deprecated
+    public final void onModule(AnalysisModule module) {}
+
     /**
      * Provides the list of this plugin's custom thread pools, empty if
      * none.
diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
index 71081740d33..4769b5954ba 100644
--- a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
+++ b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
@@ -41,6 +41,7 @@ import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.index.IndexModule;
+import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.script.NativeScriptFactory;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptEngineService;
@@ -207,8 +208,8 @@ public class PluginsService extends AbstractComponent {
                 }
                 Class moduleClass = method.getParameterTypes()[0];
                 if (!Module.class.isAssignableFrom(moduleClass)) {
-                    if (moduleClass == ScriptModule.class) {
-                        // This is still part of the Plugin class to point the user to the new implementation
+                    if (method.getDeclaringClass() == Plugin.class) {
+                        // These are still part of the Plugin class to point the user to the new implementations
                         continue;
                     }
                     throw new RuntimeException(
diff --git a/core/src/main/java/org/elasticsearch/plugins/ScriptPlugin.java b/core/src/main/java/org/elasticsearch/plugins/ScriptPlugin.java
index 7f85b8182a5..c1e2a43c953 100644
--- a/core/src/main/java/org/elasticsearch/plugins/ScriptPlugin.java
+++ b/core/src/main/java/org/elasticsearch/plugins/ScriptPlugin.java
@@ -27,8 +27,7 @@ import java.util.Collections;
 import java.util.List;
 
 /**
- * An additional extension point to {@link Plugin}. Plugins extending the scripting functionality must implement this inteface
- * to provide access to script engines or script factories.
+ * An additional extension point for {@link Plugin}s that extends Elasticsearch's scripting functionality.
  */
 public interface ScriptPlugin {
 
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java
index ff7faab1f6a..5e2c503eba1 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/TransportAnalyzeActionTests.java
@@ -29,12 +29,15 @@ import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.mapper.internal.AllFieldMapper;
+import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 
 import java.io.IOException;
 import java.util.List;
 
+import static java.util.Collections.emptyList;
+
 public class TransportAnalyzeActionTests extends ESTestCase {
 
     private AnalysisService analysisService;
@@ -56,7 +59,7 @@ public class TransportAnalyzeActionTests extends ESTestCase {
                 .putArray("index.analysis.analyzer.custom_analyzer.filter", "lowercase", "wordDelimiter").build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
         environment = new Environment(settings);
-        registry = new AnalysisRegistry(null, environment);
+        registry = new AnalysisModule(environment, emptyList()).getAnalysisRegistry();
         analysisService = registry.build(idxSettings);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
index 67b6bdadb91..4467ea63082 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
@@ -36,7 +36,6 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -45,9 +44,9 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.env.ShardLock;
 import org.elasticsearch.index.analysis.AnalysisRegistry;
-import org.elasticsearch.index.cache.query.QueryCache;
-import org.elasticsearch.index.cache.query.IndexQueryCache;
 import org.elasticsearch.index.cache.query.DisabledQueryCache;
+import org.elasticsearch.index.cache.query.IndexQueryCache;
+import org.elasticsearch.index.cache.query.QueryCache;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.EngineException;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
@@ -61,9 +60,9 @@ import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.index.store.IndexStore;
 import org.elasticsearch.index.store.IndexStoreConfig;
 import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.indices.IndicesQueryCache;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
-import org.elasticsearch.indices.IndicesQueryCache;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.mapper.MapperRegistry;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
@@ -73,6 +72,7 @@ import org.elasticsearch.script.ScriptEngineService;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.ScriptSettings;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.TestSearchContext;
@@ -84,11 +84,12 @@ import org.elasticsearch.watcher.ResourceWatcherService;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
 
+import static java.util.Collections.emptyMap;
+
 public class IndexModuleTests extends ESTestCase {
     private Index index;
     private Settings settings;
@@ -147,7 +148,8 @@ public class IndexModuleTests extends ESTestCase {
     }
 
     public void testWrapperIsBound() throws IOException {
-        IndexModule module = new IndexModule(indexSettings, null, new AnalysisRegistry(null, environment));
+        IndexModule module = new IndexModule(indexSettings, null,
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         module.setSearcherWrapper((s) -> new Wrapper());
         module.engineFactory.set(new MockEngineFactory(AssertingDirectoryReader.class));
         IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry, new IndicesFieldDataCache(settings, listener));
@@ -165,7 +167,8 @@ public class IndexModuleTests extends ESTestCase {
             .put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), "foo_store")
             .build();
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings);
-        IndexModule module = new IndexModule(indexSettings, null, new AnalysisRegistry(null, environment));
+        IndexModule module = new IndexModule(indexSettings, null,
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         module.addIndexStore("foo_store", FooStore::new);
         try {
             module.addIndexStore("foo_store", FooStore::new);
@@ -188,7 +191,8 @@ public class IndexModuleTests extends ESTestCase {
             }
         };
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings);
-        IndexModule module = new IndexModule(indexSettings, null, new AnalysisRegistry(null, environment));
+        IndexModule module = new IndexModule(indexSettings, null,
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         Consumer<Settings> listener = (s) -> {};
         module.addIndexEventListener(eventListener);
         IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
@@ -204,7 +208,8 @@ public class IndexModuleTests extends ESTestCase {
 
     public void testListener() throws IOException {
         Setting<Boolean> booleanSetting = Setting.boolSetting("index.foo.bar", false, Property.Dynamic, Property.IndexScope);
-        IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings, booleanSetting), null, new AnalysisRegistry(null, environment));
+        IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings, booleanSetting), null,
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         Setting<Boolean> booleanSetting2 = Setting.boolSetting("index.foo.bar.baz", false, Property.Dynamic, Property.IndexScope);
         AtomicBoolean atomicBoolean = new AtomicBoolean(false);
         module.addSettingsUpdateConsumer(booleanSetting, atomicBoolean::set);
@@ -224,7 +229,8 @@ public class IndexModuleTests extends ESTestCase {
     }
 
     public void testAddIndexOperationListener() throws IOException {
-        IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), null, new AnalysisRegistry(null, environment));
+        IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), null,
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         AtomicBoolean executed = new AtomicBoolean(false);
         IndexingOperationListener listener = new IndexingOperationListener() {
             @Override
@@ -254,7 +260,8 @@ public class IndexModuleTests extends ESTestCase {
     }
 
     public void testAddSearchOperationListener() throws IOException {
-        IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), null, new AnalysisRegistry(null, environment));
+        IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), null,
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         AtomicBoolean executed = new AtomicBoolean(false);
         SearchOperationListener listener = new SearchOperationListener() {
 
@@ -289,7 +296,8 @@ public class IndexModuleTests extends ESTestCase {
                 .put("index.similarity.my_similarity.key", "there is a key")
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .build();
-        IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, new AnalysisRegistry(null, environment));
+        IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null,
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         module.addSimilarity("test_similarity", (string, settings) -> new SimilarityProvider() {
             @Override
             public String name() {
@@ -313,7 +321,8 @@ public class IndexModuleTests extends ESTestCase {
     }
 
     public void testFrozen() {
-        IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), null, new AnalysisRegistry(null, environment));
+        IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(index, settings), null,
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         module.freeze();
         String msg = "Can't modify IndexModule once the index service has been created";
         assertEquals(msg, expectThrows(IllegalStateException.class, () -> module.addSearchOperationListener(null)).getMessage());
@@ -331,7 +340,8 @@ public class IndexModuleTests extends ESTestCase {
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .build();
-        IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, new AnalysisRegistry(null, environment));
+        IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null,
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         try {
             module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
                 new IndicesFieldDataCache(settings, listener));
@@ -346,7 +356,8 @@ public class IndexModuleTests extends ESTestCase {
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                 .build();
-        IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, new AnalysisRegistry(null, environment));
+        IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null,
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         try {
             module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
                 new IndicesFieldDataCache(settings, listener));
@@ -359,7 +370,8 @@ public class IndexModuleTests extends ESTestCase {
         Settings indexSettings = Settings.builder()
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
-        IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, new AnalysisRegistry(null, environment));
+        IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null,
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         module.forceQueryCacheProvider((a, b) -> new CustomQueryCache());
         expectThrows(AlreadySetException.class, () -> module.forceQueryCacheProvider((a, b) -> new CustomQueryCache()));
         IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
@@ -372,7 +384,8 @@ public class IndexModuleTests extends ESTestCase {
         Settings indexSettings = Settings.builder()
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
-        IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, new AnalysisRegistry(null, environment));
+        IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null,
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
             new IndicesFieldDataCache(settings, listener));
         assertTrue(indexService.cache().query() instanceof IndexQueryCache);
@@ -384,7 +397,8 @@ public class IndexModuleTests extends ESTestCase {
             .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false)
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
-        IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null, new AnalysisRegistry(null, environment));
+        IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings("foo", indexSettings), null,
+                new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
         module.forceQueryCacheProvider((a, b) -> new CustomQueryCache());
         IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
             new IndicesFieldDataCache(settings, listener));
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java
index b72996bd1a1..52fcdd4bb2e 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisServiceTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.analysis;
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.en.EnglishAnalyzer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
@@ -29,6 +30,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.indices.analysis.PreBuiltAnalyzers;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
@@ -41,12 +43,15 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
 public class AnalysisServiceTests extends ESTestCase {
 
-    private static AnalyzerProvider analyzerProvider(final String name) {
+    private static AnalyzerProvider<?> analyzerProvider(final String name) {
         return new PreBuiltAnalyzerProvider(name, AnalyzerScope.INDEX, new EnglishAnalyzer());
     }
 
@@ -58,7 +63,8 @@ public class AnalysisServiceTests extends ESTestCase {
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
             .build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
-        AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
+        AnalysisService analysisService = new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(), emptyMap())
+                .build(idxSettings);
         assertThat(analysisService.defaultIndexAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
         assertThat(analysisService.defaultSearchAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
         assertThat(analysisService.defaultSearchQuoteAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
@@ -68,33 +74,28 @@ public class AnalysisServiceTests extends ESTestCase {
         Version version = VersionUtils.randomVersion(random());
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings),
-                Collections.singletonMap("default", analyzerProvider("default")),
-                Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
+                singletonMap("default", analyzerProvider("default")), emptyMap(), emptyMap(), emptyMap());
         assertThat(analysisService.defaultIndexAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
         assertThat(analysisService.defaultSearchAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
         assertThat(analysisService.defaultSearchQuoteAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
     }
 
-    public void testOverrideDefaultIndexAnalyzer() {
+    public void testOverrideDefaultIndexAnalyzerIsUnsupported() {
         Version version = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0_alpha1, Version.CURRENT);
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        try {
-            AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings),
-                    Collections.singletonMap("default_index", new PreBuiltAnalyzerProvider("default_index", AnalyzerScope.INDEX, new EnglishAnalyzer())),
-                    Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
-            fail("Expected ISE");
-        } catch (IllegalArgumentException e) {
-            // expected
-            assertTrue(e.getMessage().contains("[index.analysis.analyzer.default_index] is not supported"));
-        }
+        AnalyzerProvider<?> defaultIndex = new PreBuiltAnalyzerProvider("default_index", AnalyzerScope.INDEX, new EnglishAnalyzer()); 
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings),
+                        singletonMap("default_index", defaultIndex), emptyMap(), emptyMap(), emptyMap()));
+        assertTrue(e.getMessage().contains("[index.analysis.analyzer.default_index] is not supported"));
     }
 
     public void testBackCompatOverrideDefaultIndexAnalyzer() {
-        Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1));
+        Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(),
+                VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1));
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings),
-                Collections.singletonMap("default_index", analyzerProvider("default_index")),
-                Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
+                singletonMap("default_index", analyzerProvider("default_index")), emptyMap(), emptyMap(), emptyMap());
         assertThat(analysisService.defaultIndexAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
         assertThat(analysisService.defaultSearchAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
         assertThat(analysisService.defaultSearchQuoteAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
@@ -104,17 +105,17 @@ public class AnalysisServiceTests extends ESTestCase {
         Version version = VersionUtils.randomVersion(random());
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
         AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings),
-                Collections.singletonMap("default_search", analyzerProvider("default_search")),
-                Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
+                singletonMap("default_search", analyzerProvider("default_search")), emptyMap(), emptyMap(), emptyMap());
         assertThat(analysisService.defaultIndexAnalyzer().analyzer(), instanceOf(StandardAnalyzer.class));
         assertThat(analysisService.defaultSearchAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
         assertThat(analysisService.defaultSearchQuoteAnalyzer().analyzer(), instanceOf(EnglishAnalyzer.class));
     }
 
     public void testBackCompatOverrideDefaultIndexAndSearchAnalyzer() {
-        Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(), VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1));
+        Version version = VersionUtils.randomVersionBetween(random(), VersionUtils.getFirstVersion(),
+                VersionUtils.getPreviousVersion(Version.V_5_0_0_alpha1));
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        Map<String, AnalyzerProvider> analyzers = new HashMap<>();
+        Map<String, AnalyzerProvider<?>> analyzers = new HashMap<>();
         analyzers.put("default_index", analyzerProvider("default_index"));
         analyzers.put("default_search", analyzerProvider("default_search"));
         AnalysisService analysisService = new AnalysisService(IndexSettingsModule.newIndexSettings("index", settings),
@@ -125,7 +126,6 @@ public class AnalysisServiceTests extends ESTestCase {
     }
 
     public void testConfigureCamelCaseTokenFilter() throws IOException {
-        // tests a filter that
         Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
         Settings indexSettings = Settings.builder()
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
@@ -137,7 +137,9 @@ public class AnalysisServiceTests extends ESTestCase {
                 .putArray("index.analysis.analyzer.custom_analyzer_1.filter", "lowercase", "word_delimiter").build();
 
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
-        AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
+        
+        AnalysisService analysisService = new AnalysisModule(new Environment(settings), emptyList()).getAnalysisRegistry()
+                .build(idxSettings);
         try (NamedAnalyzer custom_analyser = analysisService.analyzer("custom_analyzer")) {
             assertNotNull(custom_analyser);
             TokenStream tokenStream = custom_analyser.tokenStream("foo", "J2SE j2ee");
@@ -176,8 +178,10 @@ public class AnalysisServiceTests extends ESTestCase {
         Settings indexSettings = Settings.builder()
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings);
-        AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
-        AnalysisService otherAnalysisSergice = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
+        AnalysisService analysisService = new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(), emptyMap())
+                .build(idxSettings);
+        AnalysisService otherAnalysisSergice = new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(),
+                emptyMap()).build(idxSettings);
         final int numIters = randomIntBetween(5, 20);
         for (int i = 0; i < numIters; i++) {
             PreBuiltAnalyzers preBuiltAnalyzers = RandomPicks.randomFrom(random(), PreBuiltAnalyzers.values());
@@ -196,7 +200,8 @@ public class AnalysisServiceTests extends ESTestCase {
             .build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
 
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new AnalysisRegistry(null, new Environment(settings)).build(idxSettings));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+                () -> new AnalysisRegistry(new Environment(settings), emptyMap(), emptyMap(), emptyMap(), emptyMap()).build(idxSettings));
         assertThat(e.getMessage(), equalTo("analyzer [test_analyzer] must specify either an analyzer type, or a tokenizer"));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java
index 3e33123f932..40ec2b412ff 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/AnalysisTestsHelper.java
@@ -23,14 +23,14 @@ import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.indices.analysis.HunspellService;
+import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.test.IndexSettingsModule;
 
 import java.io.IOException;
 import java.nio.file.Path;
-import java.util.Collections;
+
+import static java.util.Collections.emptyList;
 
 public class AnalysisTestsHelper {
 
@@ -49,7 +49,6 @@ public class AnalysisTestsHelper {
             settings = Settings.builder().put(settings).put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
         }
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
-        Environment environment = new Environment(settings);
-        return new AnalysisRegistry(new HunspellService(settings, environment, Collections.emptyMap()), environment).build(idxSettings);
+        return new AnalysisModule(new Environment(settings), emptyList()).getAnalysisRegistry().build(idxSettings);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java b/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java
index 2b2c9288f17..3f2b1461ef3 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/CharFilterTests.java
@@ -26,6 +26,8 @@ import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.test.ESTokenStreamTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 
+import static org.elasticsearch.test.ESTestCase.createAnalysisService;
+
 /**
  */
 public class CharFilterTests extends ESTokenStreamTestCase {
@@ -39,7 +41,7 @@ public class CharFilterTests extends ESTokenStreamTestCase {
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
-        AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
+        AnalysisService analysisService = createAnalysisService(idxSettings, settings);
         NamedAnalyzer analyzer1 = analysisService.analyzer("custom_with_char_filter");
 
         assertTokenStreamContents(analyzer1.tokenStream("test", "jeff quit phish"), new String[]{"jeff", "qit", "fish"});
@@ -56,7 +58,7 @@ public class CharFilterTests extends ESTokenStreamTestCase {
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
                 .build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
-        AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
+        AnalysisService analysisService = createAnalysisService(idxSettings, settings);
 
         NamedAnalyzer analyzer1 = analysisService.analyzer("custom_with_char_filter");
 
@@ -78,7 +80,7 @@ public class CharFilterTests extends ESTokenStreamTestCase {
             .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
             .build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
-        AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
+        AnalysisService analysisService = createAnalysisService(idxSettings, settings);
         NamedAnalyzer analyzer1 = analysisService.analyzer("custom_with_char_filter");
 
         assertTokenStreamContents(analyzer1.tokenStream("test", "faBBbBB aBbbbBf"), new String[]{"foo", "oof"});
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/CompoundAnalysisTests.java b/core/src/test/java/org/elasticsearch/index/analysis/CompoundAnalysisTests.java
index 0e3af58dc90..0c9010b2c9b 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/CompoundAnalysisTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/CompoundAnalysisTests.java
@@ -31,15 +31,20 @@ import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.compound.DictionaryCompoundWordTokenFilterFactory;
 import org.elasticsearch.index.analysis.filter1.MyFilterTokenFilterFactory;
+import org.elasticsearch.indices.analysis.AnalysisModule;
+import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
+import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.hamcrest.MatcherAssert;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItems;
 import static org.hamcrest.Matchers.instanceOf;
@@ -50,8 +55,13 @@ public class CompoundAnalysisTests extends ESTestCase {
     public void testDefaultsCompoundAnalysis() throws Exception {
         Settings settings = getJsonSettings();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
-        AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings),
-                Collections.emptyMap(),Collections.singletonMap("myfilter", MyFilterTokenFilterFactory::new),Collections.emptyMap(),Collections.emptyMap()).build(idxSettings);
+        AnalysisModule analysisModule = new AnalysisModule(new Environment(settings), singletonList(new AnalysisPlugin() {
+            @Override
+            public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
+                return singletonMap("myfilter", MyFilterTokenFilterFactory::new);
+            }
+        }));
+        AnalysisService analysisService = analysisModule.getAnalysisRegistry().build(idxSettings);
 
         TokenFilterFactory filterFactory = analysisService.tokenFilter("dict_dec");
         MatcherAssert.assertThat(filterFactory, instanceOf(DictionaryCompoundWordTokenFilterFactory.class));
@@ -62,14 +72,20 @@ public class CompoundAnalysisTests extends ESTestCase {
         for (Settings settings : settingsArr) {
             List<String> terms = analyze(settings, "decompoundingAnalyzer", "donaudampfschiff spargelcremesuppe");
             MatcherAssert.assertThat(terms.size(), equalTo(8));
-            MatcherAssert.assertThat(terms, hasItems("donau", "dampf", "schiff", "donaudampfschiff", "spargel", "creme", "suppe", "spargelcremesuppe"));
+            MatcherAssert.assertThat(terms,
+                    hasItems("donau", "dampf", "schiff", "donaudampfschiff", "spargel", "creme", "suppe", "spargelcremesuppe"));
         }
     }
 
     private List<String> analyze(Settings settings, String analyzerName, String text) throws IOException {
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("test", settings);
-        AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings),
-                Collections.emptyMap(), Collections.singletonMap("myfilter", MyFilterTokenFilterFactory::new),Collections.emptyMap(),Collections.emptyMap()).build(idxSettings);
+        AnalysisModule analysisModule = new AnalysisModule(new Environment(settings), singletonList(new AnalysisPlugin() {
+            @Override
+            public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
+                return singletonMap("myfilter", MyFilterTokenFilterFactory::new);
+            }
+        }));
+        AnalysisService analysisService = analysisModule.getAnalysisRegistry().build(idxSettings);
 
         Analyzer analyzer = analysisService.analyzer(analyzerName).analyzer();
 
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/PatternCaptureTokenFilterTests.java b/core/src/test/java/org/elasticsearch/index/analysis/PatternCaptureTokenFilterTests.java
index 3f5ad6fed4f..caefb1039c2 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/PatternCaptureTokenFilterTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/PatternCaptureTokenFilterTests.java
@@ -27,6 +27,7 @@ import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.test.ESTokenStreamTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 
+import static org.elasticsearch.test.ESTestCase.createAnalysisService;
 import static org.hamcrest.Matchers.containsString;
 
 public class PatternCaptureTokenFilterTests extends ESTokenStreamTestCase {
@@ -39,7 +40,7 @@ public class PatternCaptureTokenFilterTests extends ESTokenStreamTestCase {
                 .build();
 
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
-        AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
+        AnalysisService analysisService = createAnalysisService(idxSettings, settings);
 
         NamedAnalyzer analyzer1 = analysisService.analyzer("single");
 
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/StopAnalyzerTests.java b/core/src/test/java/org/elasticsearch/index/analysis/StopAnalyzerTests.java
index 39641c281aa..88c5fe692d6 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/StopAnalyzerTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/StopAnalyzerTests.java
@@ -27,6 +27,8 @@ import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.test.ESTokenStreamTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
 
+import static org.elasticsearch.test.ESTestCase.createAnalysisService;
+
 public class StopAnalyzerTests extends ESTokenStreamTestCase {
     public void testDefaultsCompoundAnalysis() throws Exception {
         String json = "/org/elasticsearch/index/analysis/stop.json";
@@ -36,7 +38,7 @@ public class StopAnalyzerTests extends ESTokenStreamTestCase {
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                 .build();
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
-        AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
+        AnalysisService analysisService = createAnalysisService(idxSettings, settings);
 
         NamedAnalyzer analyzer1 = analysisService.analyzer("analyzer1");
 
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/synonyms/SynonymsAnalysisTests.java b/core/src/test/java/org/elasticsearch/index/analysis/synonyms/SynonymsAnalysisTests.java
index 104bd17fb33..9e4d5b27ad7 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/synonyms/SynonymsAnalysisTests.java
+++ b/core/src/test/java/org/elasticsearch/index/analysis/synonyms/SynonymsAnalysisTests.java
@@ -26,13 +26,10 @@ import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.common.lucene.all.AllEntries;
 import org.elasticsearch.common.lucene.all.AllTokenStream;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.IndexSettingsModule;
@@ -67,8 +64,7 @@ public class SynonymsAnalysisTests extends ESTestCase {
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
 
         IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", settings);
-        analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
-
+        analysisService = createAnalysisService(idxSettings, settings);
 
         match("synonymAnalyzer", "kimchy is the dude abides", "shay is the elasticsearch man!");
         match("synonymAnalyzer_file", "kimchy is the dude abides", "shay is the elasticsearch man!");
diff --git a/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java
index fdf1411fdf8..80f1cbe46d0 100644
--- a/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java
+++ b/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java
@@ -36,9 +36,7 @@ import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.similarity.SimilarityService;
@@ -97,7 +95,7 @@ public class CodecTests extends ESTestCase {
                 .build();
         IndexSettings settings = IndexSettingsModule.newIndexSettings("_na", nodeSettings);
         SimilarityService similarityService = new SimilarityService(settings, Collections.emptyMap());
-        AnalysisService analysisService = new AnalysisRegistry(null, new Environment(nodeSettings)).build(settings);
+        AnalysisService analysisService = createAnalysisService(settings, nodeSettings);
         MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap());
         MapperService service = new MapperService(settings, analysisService, similarityService, mapperRegistry, () -> null);
         return new CodecService(service, ESLoggerFactory.getLogger("test"));
diff --git a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisModuleTests.java b/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java
similarity index 87%
rename from core/src/test/java/org/elasticsearch/index/analysis/AnalysisModuleTests.java
rename to core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java
index e20e7d1a7c9..869ac622b39 100644
--- a/core/src/test/java/org/elasticsearch/index/analysis/AnalysisModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.index.analysis;
+package org.elasticsearch.indices.analysis;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
@@ -36,9 +36,20 @@ import org.elasticsearch.common.inject.ModuleTestCase;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.analysis.Analysis;
+import org.elasticsearch.index.analysis.AnalysisRegistry;
+import org.elasticsearch.index.analysis.AnalysisService;
+import org.elasticsearch.index.analysis.AnalysisTestsHelper;
+import org.elasticsearch.index.analysis.CustomAnalyzer;
+import org.elasticsearch.index.analysis.MappingCharFilterFactory;
+import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.analysis.PatternReplaceCharFilterFactory;
+import org.elasticsearch.index.analysis.StandardTokenizerFactory;
+import org.elasticsearch.index.analysis.StopTokenFilterFactory;
+import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.index.analysis.filter1.MyFilterTokenFilterFactory;
-import org.elasticsearch.indices.analysis.AnalysisModule;
-import org.elasticsearch.indices.analysis.HunspellService;
+import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
+import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.hamcrest.MatcherAssert;
 
@@ -49,9 +60,11 @@ import java.io.StringReader;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Collections;
+import java.util.Map;
 import java.util.Set;
 
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -72,8 +85,16 @@ public class AnalysisModuleTests extends ModuleTestCase {
     }
 
     public AnalysisRegistry getNewRegistry(Settings settings) {
-       return new AnalysisRegistry(null, new Environment(settings),
-                Collections.emptyMap(), Collections.singletonMap("myfilter", MyFilterTokenFilterFactory::new), Collections.emptyMap(), Collections.emptyMap());
+        try {
+            return new AnalysisModule(new Environment(settings), singletonList(new AnalysisPlugin() {
+                @Override
+                public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
+                    return singletonMap("myfilter", MyFilterTokenFilterFactory::new);
+                }
+            })).getAnalysisRegistry();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
     }
     
     private Settings loadFromClasspath(String path) throws IOException {
@@ -125,7 +146,7 @@ public class AnalysisModuleTests extends ModuleTestCase {
         assertEquals(org.apache.lucene.util.Version.fromBits(3,6,0), analysisService2.analyzer("custom7").analyzer().getVersion());
     }
 
-    private void assertTokenFilter(String name, Class clazz) throws IOException {
+    private void assertTokenFilter(String name, Class<?> clazz) throws IOException {
         Settings settings = Settings.builder()
                                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
@@ -148,17 +169,9 @@ public class AnalysisModuleTests extends ModuleTestCase {
 
         StopTokenFilterFactory stop1 = (StopTokenFilterFactory) custom1.tokenFilters()[0];
         assertThat(stop1.stopWords().size(), equalTo(1));
-        //assertThat((Iterable<char[]>) stop1.stopWords(), hasItem("test-stop".toCharArray()));
 
         analyzer = analysisService.analyzer("custom2").analyzer();
         assertThat(analyzer, instanceOf(CustomAnalyzer.class));
-        CustomAnalyzer custom2 = (CustomAnalyzer) analyzer;
-
-//        HtmlStripCharFilterFactory html = (HtmlStripCharFilterFactory) custom2.charFilters()[0];
-//        assertThat(html.readAheadLimit(), equalTo(HTMLStripCharFilter.DEFAULT_READ_AHEAD));
-//
-//        html = (HtmlStripCharFilterFactory) custom2.charFilters()[1];
-//        assertThat(html.readAheadLimit(), equalTo(1024));
 
         // verify position increment gap
         analyzer = analysisService.analyzer("custom6").analyzer();
@@ -248,7 +261,8 @@ public class AnalysisModuleTests extends ModuleTestCase {
             getAnalysisService(settings);
             fail("This should fail with IllegalArgumentException because the analyzers name starts with _");
         } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), either(equalTo("analyzer name must not start with '_'. got \"_invalid_name\"")).or(equalTo("analyzer name must not start with '_'. got \"_invalidName\"")));
+            assertThat(e.getMessage(), either(equalTo("analyzer name must not start with '_'. got \"_invalid_name\""))
+                    .or(equalTo("analyzer name must not start with '_'. got \"_invalidName\"")));
         }
     }
 
@@ -289,13 +303,18 @@ public class AnalysisModuleTests extends ModuleTestCase {
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                 .build();
         Environment environment = new Environment(settings);
-        AnalysisModule module = new AnalysisModule(environment);
         InputStream aff = getClass().getResourceAsStream("/indices/analyze/conf_dir/hunspell/en_US/en_US.aff");
         InputStream dic = getClass().getResourceAsStream("/indices/analyze/conf_dir/hunspell/en_US/en_US.dic");
+        Dictionary dictionary;
         try (Directory tmp = new SimpleFSDirectory(environment.tmpFile())) {
-            Dictionary dictionary = new Dictionary(tmp, "hunspell", aff, dic);
-            module.registerHunspellDictionary("foo", dictionary);
-            assertInstanceBinding(module, HunspellService.class, (x) -> x.getDictionary("foo") == dictionary);
+            dictionary = new Dictionary(tmp, "hunspell", aff, dic);
         }
+        AnalysisModule module = new AnalysisModule(environment, singletonList(new AnalysisPlugin() {
+            @Override
+            public Map<String, Dictionary> getHunspellDictionaries() {
+                return singletonMap("foo", dictionary);
+            }
+        }));
+        assertSame(dictionary, module.getHunspellService().getDictionary("foo"));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/analysis/DummyAnalysisPlugin.java b/core/src/test/java/org/elasticsearch/indices/analysis/DummyAnalysisPlugin.java
index 86c54f2ece9..bbfeacfc590 100644
--- a/core/src/test/java/org/elasticsearch/indices/analysis/DummyAnalysisPlugin.java
+++ b/core/src/test/java/org/elasticsearch/indices/analysis/DummyAnalysisPlugin.java
@@ -19,15 +19,38 @@
 
 package org.elasticsearch.indices.analysis;
 
+import org.apache.lucene.analysis.Analyzer;
+import org.elasticsearch.index.analysis.AnalyzerProvider;
+import org.elasticsearch.index.analysis.CharFilterFactory;
+import org.elasticsearch.index.analysis.TokenFilterFactory;
+import org.elasticsearch.index.analysis.TokenizerFactory;
+import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
+import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.plugins.Plugin;
 
-public class DummyAnalysisPlugin extends Plugin {
+import java.util.Map;
 
-    public void onModule(AnalysisModule module) {
-        module.registerAnalyzer("dummy", (a, b, c, d) -> new DummyAnalyzerProvider());
-        module.registerTokenFilter("dummy_token_filter", (a, b, c, d) -> new DummyTokenFilterFactory());
-        module.registerTokenizer("dummy_tokenizer", (a, b, c, d) -> new DummyTokenizerFactory());
-        module.registerCharFilter("dummy_char_filter", (a, b, c, d) -> new DummyCharFilterFactory());
+import static java.util.Collections.singletonMap;
+
+public class DummyAnalysisPlugin extends Plugin implements AnalysisPlugin {
+    @Override
+    public Map<String, AnalysisProvider<CharFilterFactory>> getCharFilters() {
+        return singletonMap("dummy_char_filter", (a, b, c, d) -> new DummyCharFilterFactory());
+    }
+
+    @Override
+    public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
+        return singletonMap("dummy_token_filter", (a, b, c, d) -> new DummyTokenFilterFactory());
+    }
+
+    @Override
+    public Map<String, AnalysisProvider<TokenizerFactory>> getTokenizers() {
+        return singletonMap("dummy_tokenizer", (a, b, c, d) -> new DummyTokenizerFactory());
+    }
+
+    @Override
+    public Map<String, AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> getAnalyzers() {
+        return singletonMap("dummy", (a, b, c, d) -> new DummyAnalyzerProvider());
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/indices/analyze/HunspellServiceIT.java b/core/src/test/java/org/elasticsearch/indices/analyze/HunspellServiceTests.java
similarity index 58%
rename from core/src/test/java/org/elasticsearch/indices/analyze/HunspellServiceIT.java
rename to core/src/test/java/org/elasticsearch/indices/analyze/HunspellServiceTests.java
index 84ce2c5da50..ba4467a5630 100644
--- a/core/src/test/java/org/elasticsearch/indices/analyze/HunspellServiceIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/analyze/HunspellServiceTests.java
@@ -19,36 +19,30 @@
 package org.elasticsearch.indices.analyze;
 
 import org.apache.lucene.analysis.hunspell.Dictionary;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.indices.analysis.HunspellService;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
-import org.elasticsearch.test.ESIntegTestCase.Scope;
-import org.hamcrest.Matchers;
+import org.elasticsearch.test.ESTestCase;
 
+import static java.util.Collections.emptyMap;
 import static org.elasticsearch.indices.analysis.HunspellService.HUNSPELL_IGNORE_CASE;
 import static org.elasticsearch.indices.analysis.HunspellService.HUNSPELL_LAZY_LOAD;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.hasToString;
 import static org.hamcrest.Matchers.notNullValue;
 
-/**
- *
- */
-@ClusterScope(scope= Scope.TEST, numDataNodes=0)
-public class HunspellServiceIT extends ESIntegTestCase {
+public class HunspellServiceTests extends ESTestCase {
     public void testLocaleDirectoryWithNodeLevelConfig() throws Exception {
         Settings settings = Settings.builder()
                 .put(Environment.PATH_CONF_SETTING.getKey(), getDataPath("/indices/analyze/conf_dir"))
                 .put(HUNSPELL_LAZY_LOAD.getKey(), randomBoolean())
                 .put(HUNSPELL_IGNORE_CASE.getKey(), true)
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                 .build();
 
-        internalCluster().startNode(settings);
-        Dictionary dictionary = internalCluster().getInstance(HunspellService.class).getDictionary("en_US");
+        Dictionary dictionary = new HunspellService(settings, new Environment(settings), emptyMap()).getDictionary("en_US");
         assertThat(dictionary, notNullValue());
-        assertIgnoreCase(true, dictionary);
+        assertTrue(dictionary.getIgnoreCase());
     }
 
     public void testLocaleDirectoryWithLocaleSpecificConfig() throws Exception {
@@ -58,58 +52,42 @@ public class HunspellServiceIT extends ESIntegTestCase {
                 .put(HUNSPELL_IGNORE_CASE.getKey(), true)
                 .put("indices.analysis.hunspell.dictionary.en_US.strict_affix_parsing", false)
                 .put("indices.analysis.hunspell.dictionary.en_US.ignore_case", false)
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                 .build();
 
-        internalCluster().startNode(settings);
-        Dictionary dictionary = internalCluster().getInstance(HunspellService.class).getDictionary("en_US");
+        Dictionary dictionary = new HunspellService(settings, new Environment(settings), emptyMap()).getDictionary("en_US");
         assertThat(dictionary, notNullValue());
-        assertIgnoreCase(false, dictionary);
-
-
+        assertFalse(dictionary.getIgnoreCase());
 
         // testing that dictionary specific settings override node level settings
-        dictionary = internalCluster().getInstance(HunspellService.class).getDictionary("en_US_custom");
+        dictionary = new HunspellService(settings, new Environment(settings), emptyMap()).getDictionary("en_US_custom");
         assertThat(dictionary, notNullValue());
-        assertIgnoreCase(true, dictionary);
+        assertTrue(dictionary.getIgnoreCase());
     }
 
     public void testDicWithNoAff() throws Exception {
         Settings settings = Settings.builder()
                 .put(Environment.PATH_CONF_SETTING.getKey(), getDataPath("/indices/analyze/no_aff_conf_dir"))
                 .put(HUNSPELL_LAZY_LOAD.getKey(), randomBoolean())
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                 .build();
 
-        Dictionary dictionary = null;
-        try {
-            internalCluster().startNode(settings);
-            dictionary = internalCluster().getInstance(HunspellService.class).getDictionary("en_US");
-            fail("Missing affix file didn't throw an error");
-        }
-        catch (Throwable t) {
-            assertNull(dictionary);
-            assertThat(ExceptionsHelper.unwrap(t, ElasticsearchException.class).toString(), Matchers.containsString("Missing affix file"));
-        }
+        IllegalStateException e = expectThrows(IllegalStateException.class,
+                () -> new HunspellService(settings, new Environment(settings), emptyMap()).getDictionary("en_US"));
+        assertEquals("failed to load hunspell dictionary for locale: en_US", e.getMessage());
+        assertThat(e.getCause(), hasToString(containsString("Missing affix file")));
     }
 
     public void testDicWithTwoAffs() throws Exception {
         Settings settings = Settings.builder()
                 .put(Environment.PATH_CONF_SETTING.getKey(), getDataPath("/indices/analyze/two_aff_conf_dir"))
                 .put(HUNSPELL_LAZY_LOAD.getKey(), randomBoolean())
+                .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                 .build();
 
-        Dictionary dictionary = null;
-        try {
-            internalCluster().startNode(settings);
-            dictionary = internalCluster().getInstance(HunspellService.class).getDictionary("en_US");
-            fail("Multiple affix files didn't throw an error");
-        } catch (Throwable t) {
-            assertNull(dictionary);
-            assertThat(ExceptionsHelper.unwrap(t, ElasticsearchException.class).toString(), Matchers.containsString("Too many affix files"));
-        }
-    }
-
-    // TODO: on next upgrade of lucene, just use new getter
-    private void assertIgnoreCase(boolean expected, Dictionary dictionary) throws Exception {
-        // assertEquals(expected, dictionary.getIgnoreCase());
+        IllegalStateException e = expectThrows(IllegalStateException.class,
+                () -> new HunspellService(settings, new Environment(settings), emptyMap()).getDictionary("en_US"));
+        assertEquals("failed to load hunspell dictionary for locale: en_US", e.getMessage());
+        assertThat(e.getCause(), hasToString(containsString("Too many affix files")));
     }
 }
diff --git a/docs/reference/migration/migrate_5_0/plugins.asciidoc b/docs/reference/migration/migrate_5_0/plugins.asciidoc
index a1c0dad9ca1..9023c84b410 100644
--- a/docs/reference/migration/migrate_5_0/plugins.asciidoc
+++ b/docs/reference/migration/migrate_5_0/plugins.asciidoc
@@ -122,3 +122,8 @@ been removed.
 
 Plugins that register custom scripts should implement `ScriptPlugin` and remove
 their `onModule(ScriptModule)` implementation.
+
+==== AnalysisPlugin
+
+Plugins that register custom analysis components should implement
+`AnalysisPlugin` and remove their `onModule(AnalysisModule)` implementation.
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
index 0cca2c41938..8f0a75a282a 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
@@ -26,7 +26,6 @@ import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.Injector;
 import org.elasticsearch.common.inject.ModulesBuilder;
 import org.elasticsearch.common.inject.multibindings.Multibinder;
@@ -40,7 +39,6 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
@@ -129,7 +127,7 @@ public class TemplateQueryParserTests extends ESTestCase {
                 new IndexSettingsModule(index, settings)
         ).createInjector();
 
-        AnalysisService analysisService = new AnalysisRegistry(null, environment).build(idxSettings);
+        AnalysisService analysisService = createAnalysisService(idxSettings, settings);
         SimilarityService similarityService = new SimilarityService(idxSettings, Collections.emptyMap());
         MapperRegistry mapperRegistry = new IndicesModule(new NamedWriteableRegistry()).getMapperRegistry();
         MapperService mapperService = new MapperService(idxSettings, analysisService, similarityService, mapperRegistry, () ->
diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java
index e0e6c3c2e36..059dabb4f46 100644
--- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java
+++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/AnalysisICUPlugin.java
@@ -19,26 +19,42 @@
 
 package org.elasticsearch.plugin.analysis.icu;
 
+import org.elasticsearch.index.analysis.CharFilterFactory;
 import org.elasticsearch.index.analysis.IcuCollationTokenFilterFactory;
 import org.elasticsearch.index.analysis.IcuFoldingTokenFilterFactory;
 import org.elasticsearch.index.analysis.IcuNormalizerCharFilterFactory;
 import org.elasticsearch.index.analysis.IcuNormalizerTokenFilterFactory;
 import org.elasticsearch.index.analysis.IcuTokenizerFactory;
 import org.elasticsearch.index.analysis.IcuTransformTokenFilterFactory;
-import org.elasticsearch.indices.analysis.AnalysisModule;
+import org.elasticsearch.index.analysis.TokenFilterFactory;
+import org.elasticsearch.index.analysis.TokenizerFactory;
+import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
+import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.plugins.Plugin;
 
-public class AnalysisICUPlugin extends Plugin {
+import java.util.HashMap;
+import java.util.Map;
 
-    /**
-     * Automatically called with the analysis module.
-     */
-    public void onModule(AnalysisModule module) {
-        module.registerCharFilter("icu_normalizer", IcuNormalizerCharFilterFactory::new);
-        module.registerTokenizer("icu_tokenizer", IcuTokenizerFactory::new);
-        module.registerTokenFilter("icu_normalizer", IcuNormalizerTokenFilterFactory::new);
-        module.registerTokenFilter("icu_folding", IcuFoldingTokenFilterFactory::new);
-        module.registerTokenFilter("icu_collation", IcuCollationTokenFilterFactory::new);
-        module.registerTokenFilter("icu_transform", IcuTransformTokenFilterFactory::new);
+import static java.util.Collections.singletonMap;
+
+public class AnalysisICUPlugin extends Plugin implements AnalysisPlugin {
+    @Override
+    public Map<String, AnalysisProvider<CharFilterFactory>> getCharFilters() {
+        return singletonMap("icu_normalizer", IcuNormalizerCharFilterFactory::new);
+    }
+
+    @Override
+    public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
+        Map<String, AnalysisProvider<TokenFilterFactory>> extra = new HashMap<>();
+        extra.put("icu_normalizer", IcuNormalizerTokenFilterFactory::new);
+        extra.put("icu_folding", IcuFoldingTokenFilterFactory::new);
+        extra.put("icu_collation", IcuCollationTokenFilterFactory::new);
+        extra.put("icu_transform", IcuTransformTokenFilterFactory::new);
+        return extra;
+    }
+
+    @Override
+    public Map<String, AnalysisProvider<TokenizerFactory>> getTokenizers() {
+        return singletonMap("icu_tokenizer", IcuTokenizerFactory::new);
     }
 }
diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuTokenizerFactoryTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuTokenizerFactoryTests.java
index 1630d514ae3..180c4268612 100644
--- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuTokenizerFactoryTests.java
+++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuTokenizerFactoryTests.java
@@ -102,6 +102,6 @@ public class IcuTokenizerFactoryTests extends ESTestCase {
             .build();
         Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), home).build();
 
-        return createAnalysisService(new Index("test", "_na_"), nodeSettings, settings, new AnalysisICUPlugin()::onModule);
+        return createAnalysisService(new Index("test", "_na_"), nodeSettings, settings, new AnalysisICUPlugin());
     }
 }
diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuAnalysisTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuAnalysisTests.java
index b399dfd34f4..9255a250f16 100644
--- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuAnalysisTests.java
+++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuAnalysisTests.java
@@ -31,8 +31,7 @@ import static org.hamcrest.Matchers.instanceOf;
  */
 public class SimpleIcuAnalysisTests extends ESTestCase {
     public void testDefaultsIcuAnalysis() throws IOException {
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"),
-            Settings.EMPTY, new AnalysisICUPlugin()::onModule);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY, new AnalysisICUPlugin());
 
         TokenizerFactory tokenizerFactory = analysisService.tokenizer("icu_tokenizer");
         assertThat(tokenizerFactory, instanceOf(IcuTokenizerFactory.class));
diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java
index 71752196af0..62e9c9db145 100644
--- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java
+++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuCollationTokenFilterTests.java
@@ -50,7 +50,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
                 .put("index.analysis.filter.myCollator.language", "tr")
                 .put("index.analysis.filter.myCollator.strength", "primary")
                 .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
 
         TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
         assertCollatesToSame(filterFactory, "I WİLL USE TURKİSH CASING", "ı will use turkish casıng");
@@ -66,7 +66,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
                 .put("index.analysis.filter.myCollator.strength", "primary")
                 .put("index.analysis.filter.myCollator.decomposition", "canonical")
                 .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
 
         TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
         assertCollatesToSame(filterFactory, "I W\u0049\u0307LL USE TURKİSH CASING", "ı will use turkish casıng");
@@ -82,7 +82,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
                 .put("index.analysis.filter.myCollator.strength", "secondary")
                 .put("index.analysis.filter.myCollator.decomposition", "no")
                 .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
 
         TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
         assertCollatesToSame(filterFactory, "TESTING", "testing");
@@ -99,7 +99,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
                 .put("index.analysis.filter.myCollator.strength", "primary")
                 .put("index.analysis.filter.myCollator.alternate", "shifted")
                 .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
 
         TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
         assertCollatesToSame(filterFactory, "foo-bar", "foo bar");
@@ -117,7 +117,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
                 .put("index.analysis.filter.myCollator.alternate", "shifted")
                 .put("index.analysis.filter.myCollator.variableTop", " ")
                 .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
 
         TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
         assertCollatesToSame(filterFactory, "foo bar", "foobar");
@@ -135,7 +135,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
                 .put("index.analysis.filter.myCollator.language", "en")
                 .put("index.analysis.filter.myCollator.numeric", "true")
                 .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
 
         TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
         assertCollation(filterFactory, "foobar-9", "foobar-10", -1);
@@ -152,7 +152,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
                 .put("index.analysis.filter.myCollator.strength", "primary")
                 .put("index.analysis.filter.myCollator.caseLevel", "true")
                 .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
 
         TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
         assertCollatesToSame(filterFactory, "résumé", "resume");
@@ -172,7 +172,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
                 .put("index.analysis.filter.myCollator.strength", "tertiary")
                 .put("index.analysis.filter.myCollator.caseFirst", "upper")
                 .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
 
         TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
         assertCollation(filterFactory, "Resume", "resume", -1);
@@ -200,7 +200,7 @@ public class SimpleIcuCollationTokenFilterTests extends ESTestCase {
                 .put("index.analysis.filter.myCollator.rules", tailoredRules)
                 .put("index.analysis.filter.myCollator.strength", "primary")
                 .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
 
         TokenFilterFactory filterFactory = analysisService.tokenFilter("myCollator");
         assertCollatesToSame(filterFactory, "Töne", "Toene");
diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java
index 96defd0e076..b82accf0cf8 100644
--- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java
+++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/SimpleIcuNormalizerCharFilterTests.java
@@ -37,7 +37,7 @@ public class SimpleIcuNormalizerCharFilterTests extends ESTestCase {
         Settings settings = Settings.builder()
             .put("index.analysis.char_filter.myNormalizerChar.type", "icu_normalizer")
             .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
         CharFilterFactory charFilterFactory = analysisService.charFilter("myNormalizerChar");
 
         String input = "ʰ㌰゙5℃№㈱㌘，�ッファー�正�化�テスト．�㋑㋒㋓㋔ｶｷｸｹｺｻﾞｼﾞｽﾞｾﾞｿﾞg̈�/�நிเ�षिchkʷक�षि";
@@ -61,7 +61,7 @@ public class SimpleIcuNormalizerCharFilterTests extends ESTestCase {
             .put("index.analysis.char_filter.myNormalizerChar.name", "nfkc")
             .put("index.analysis.char_filter.myNormalizerChar.mode", "decompose")
             .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin()::onModule);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisICUPlugin());
         CharFilterFactory charFilterFactory = analysisService.charFilter("myNormalizerChar");
 
         String input = "ʰ㌰゙5℃№㈱㌘，�ッファー�正�化�テスト．�㋑㋒㋓㋔ｶｷｸｹｺｻﾞｼﾞｽﾞｾﾞｿﾞg̈�/�நிเ�षिchkʷक�षि";
diff --git a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java
index e92e782a96c..c4b4db53c4a 100644
--- a/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java
+++ b/plugins/analysis-kuromoji/src/main/java/org/elasticsearch/plugin/analysis/kuromoji/AnalysisKuromojiPlugin.java
@@ -19,6 +19,9 @@
 
 package org.elasticsearch.plugin.analysis.kuromoji;
 
+import org.apache.lucene.analysis.Analyzer;
+import org.elasticsearch.index.analysis.AnalyzerProvider;
+import org.elasticsearch.index.analysis.CharFilterFactory;
 import org.elasticsearch.index.analysis.JapaneseStopTokenFilterFactory;
 import org.elasticsearch.index.analysis.KuromojiAnalyzerProvider;
 import org.elasticsearch.index.analysis.KuromojiBaseFormFilterFactory;
@@ -28,23 +31,42 @@ import org.elasticsearch.index.analysis.KuromojiNumberFilterFactory;
 import org.elasticsearch.index.analysis.KuromojiPartOfSpeechFilterFactory;
 import org.elasticsearch.index.analysis.KuromojiReadingFormFilterFactory;
 import org.elasticsearch.index.analysis.KuromojiTokenizerFactory;
-import org.elasticsearch.indices.analysis.AnalysisModule;
+import org.elasticsearch.index.analysis.TokenFilterFactory;
+import org.elasticsearch.index.analysis.TokenizerFactory;
+import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
+import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.plugins.Plugin;
 
-/**
- *
- */
-public class AnalysisKuromojiPlugin extends Plugin {
+import java.util.HashMap;
+import java.util.Map;
 
-    public void onModule(AnalysisModule module) {
-        module.registerCharFilter("kuromoji_iteration_mark", KuromojiIterationMarkCharFilterFactory::new);
-        module.registerAnalyzer("kuromoji", KuromojiAnalyzerProvider::new);
-        module.registerTokenizer("kuromoji_tokenizer", KuromojiTokenizerFactory::new);
-        module.registerTokenFilter("kuromoji_baseform", KuromojiBaseFormFilterFactory::new);
-        module.registerTokenFilter("kuromoji_part_of_speech", KuromojiPartOfSpeechFilterFactory::new);
-        module.registerTokenFilter("kuromoji_readingform", KuromojiReadingFormFilterFactory::new);
-        module.registerTokenFilter("kuromoji_stemmer", KuromojiKatakanaStemmerFactory::new);
-        module.registerTokenFilter("ja_stop", JapaneseStopTokenFilterFactory::new);
-        module.registerTokenFilter("kuromoji_number", KuromojiNumberFilterFactory::new);
+import static java.util.Collections.singletonMap;
+
+public class AnalysisKuromojiPlugin extends Plugin implements AnalysisPlugin {
+    @Override
+    public Map<String, AnalysisProvider<CharFilterFactory>> getCharFilters() {
+        return singletonMap("kuromoji_iteration_mark", KuromojiIterationMarkCharFilterFactory::new);
+    }
+
+    @Override
+    public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
+        Map<String, AnalysisProvider<TokenFilterFactory>> extra = new HashMap<>();
+        extra.put("kuromoji_baseform", KuromojiBaseFormFilterFactory::new);
+        extra.put("kuromoji_part_of_speech", KuromojiPartOfSpeechFilterFactory::new);
+        extra.put("kuromoji_readingform", KuromojiReadingFormFilterFactory::new);
+        extra.put("kuromoji_stemmer", KuromojiKatakanaStemmerFactory::new);
+        extra.put("ja_stop", JapaneseStopTokenFilterFactory::new);
+        extra.put("kuromoji_number", KuromojiNumberFilterFactory::new);
+        return extra;
+    }
+
+    @Override
+    public Map<String, AnalysisProvider<TokenizerFactory>> getTokenizers() {
+        return singletonMap("kuromoji_tokenizer", KuromojiTokenizerFactory::new);
+    }
+
+    @Override
+    public Map<String, AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> getAnalyzers() {
+        return singletonMap("kuromoji", KuromojiAnalyzerProvider::new);
     }
 }
diff --git a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiAnalysisTests.java b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiAnalysisTests.java
index 540e11250d0..53196ac7462 100644
--- a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiAnalysisTests.java
+++ b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiAnalysisTests.java
@@ -198,7 +198,7 @@ public class KuromojiAnalysisTests extends ESTestCase {
             .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
             .build();
         Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), home).build();
-        return createAnalysisService(new Index("test", "_na_"), nodeSettings, settings, new AnalysisKuromojiPlugin()::onModule);
+        return createAnalysisService(new Index("test", "_na_"), nodeSettings, settings, new AnalysisKuromojiPlugin());
     }
 
     public static void assertSimpleTSOutput(TokenStream stream,
diff --git a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/AnalysisPhoneticPlugin.java b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/AnalysisPhoneticPlugin.java
index 3be97b444db..c7355e976ce 100644
--- a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/AnalysisPhoneticPlugin.java
+++ b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/plugin/analysis/AnalysisPhoneticPlugin.java
@@ -20,13 +20,19 @@
 package org.elasticsearch.plugin.analysis;
 
 import org.elasticsearch.index.analysis.PhoneticTokenFilterFactory;
-import org.elasticsearch.indices.analysis.AnalysisModule;
+import org.elasticsearch.index.analysis.TokenFilterFactory;
+import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
+import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.plugins.Plugin;
 
-public class AnalysisPhoneticPlugin extends Plugin {
+import java.util.Map;
 
-    public void onModule(AnalysisModule module) {
-        module.registerTokenFilter("phonetic", PhoneticTokenFilterFactory::new);
+import static java.util.Collections.singletonMap;
+
+public class AnalysisPhoneticPlugin extends Plugin implements AnalysisPlugin {
+    @Override
+    public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
+        return singletonMap("phonetic", PhoneticTokenFilterFactory::new);
     }
 }
 
diff --git a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java
index 18e49fa6e51..3dcfadce781 100644
--- a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java
+++ b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/SimplePhoneticAnalysisTests.java
@@ -39,8 +39,7 @@ public class SimplePhoneticAnalysisTests extends ESTestCase {
         Settings settings = Settings.builder().loadFromStream(yaml, getClass().getResourceAsStream(yaml))
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                 .build();
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings,
-            new AnalysisPhoneticPlugin()::onModule);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), settings, new AnalysisPhoneticPlugin());
         TokenFilterFactory filterFactory = analysisService.tokenFilter("phonetic");
         MatcherAssert.assertThat(filterFactory, instanceOf(PhoneticTokenFilterFactory.class));
     }
diff --git a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java
index 05c779bb5a9..b11a157c149 100644
--- a/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java
+++ b/plugins/analysis-smartcn/src/main/java/org/elasticsearch/plugin/analysis/smartcn/AnalysisSmartChinesePlugin.java
@@ -19,20 +19,40 @@
 
 package org.elasticsearch.plugin.analysis.smartcn;
 
+import org.apache.lucene.analysis.Analyzer;
+import org.elasticsearch.index.analysis.AnalyzerProvider;
 import org.elasticsearch.index.analysis.SmartChineseAnalyzerProvider;
 import org.elasticsearch.index.analysis.SmartChineseNoOpTokenFilterFactory;
 import org.elasticsearch.index.analysis.SmartChineseTokenizerTokenizerFactory;
-import org.elasticsearch.indices.analysis.AnalysisModule;
+import org.elasticsearch.index.analysis.TokenFilterFactory;
+import org.elasticsearch.index.analysis.TokenizerFactory;
+import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
+import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.plugins.Plugin;
 
-public class AnalysisSmartChinesePlugin extends Plugin {
+import java.util.HashMap;
+import java.util.Map;
 
-    public void onModule(AnalysisModule module) {
-            module.registerAnalyzer("smartcn", SmartChineseAnalyzerProvider::new);
-            module.registerTokenizer("smartcn_tokenizer", SmartChineseTokenizerTokenizerFactory::new);
-            // This is an alias to "smartcn_tokenizer"; it's here for backwards compat
-        module.registerTokenizer("smartcn_sentence", SmartChineseTokenizerTokenizerFactory::new);
-            // This is a noop token filter; it's here for backwards compat before we had "smartcn_tokenizer"
-        module.registerTokenFilter("smartcn_word", SmartChineseNoOpTokenFilterFactory::new);
+import static java.util.Collections.singletonMap;
+
+public class AnalysisSmartChinesePlugin extends Plugin implements AnalysisPlugin {
+    @Override
+    public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
+        // This is a noop token filter; it's here for backwards compat before we had "smartcn_tokenizer"
+        return singletonMap("smartcn_word", SmartChineseNoOpTokenFilterFactory::new);
+    }
+
+    @Override
+    public Map<String, AnalysisProvider<TokenizerFactory>> getTokenizers() {
+        Map<String, AnalysisProvider<TokenizerFactory>> extra = new HashMap<>();
+        extra.put("smartcn_tokenizer", SmartChineseTokenizerTokenizerFactory::new);
+        // This is an alias to "smartcn_tokenizer"; it's here for backwards compat
+        extra.put("smartcn_sentence", SmartChineseTokenizerTokenizerFactory::new);
+        return extra;
+    }
+
+    @Override
+    public Map<String, AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> getAnalyzers() {
+        return singletonMap("smartcn", SmartChineseAnalyzerProvider::new);
     }
 }
diff --git a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SimpleSmartChineseAnalysisTests.java b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SimpleSmartChineseAnalysisTests.java
index 0fcc42643d4..08aebdee2bb 100644
--- a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SimpleSmartChineseAnalysisTests.java
+++ b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SimpleSmartChineseAnalysisTests.java
@@ -29,11 +29,10 @@ import java.io.IOException;
 
 import static org.hamcrest.Matchers.instanceOf;
 
-/**
- */
 public class SimpleSmartChineseAnalysisTests extends ESTestCase {
     public void testDefaultsIcuAnalysis() throws IOException {
-        final AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY, new AnalysisSmartChinesePlugin()::onModule);
+        final AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY,
+                new AnalysisSmartChinesePlugin());
         TokenizerFactory tokenizerFactory = analysisService.tokenizer("smartcn_tokenizer");
         MatcherAssert.assertThat(tokenizerFactory, instanceOf(SmartChineseTokenizerTokenizerFactory.class));
     }
diff --git a/plugins/analysis-stempel/src/main/java/org/elasticsearch/plugin/analysis/stempel/AnalysisStempelPlugin.java b/plugins/analysis-stempel/src/main/java/org/elasticsearch/plugin/analysis/stempel/AnalysisStempelPlugin.java
index 8549795f4b6..98dd9634fb9 100644
--- a/plugins/analysis-stempel/src/main/java/org/elasticsearch/plugin/analysis/stempel/AnalysisStempelPlugin.java
+++ b/plugins/analysis-stempel/src/main/java/org/elasticsearch/plugin/analysis/stempel/AnalysisStempelPlugin.java
@@ -19,15 +19,27 @@
 
 package org.elasticsearch.plugin.analysis.stempel;
 
+import org.apache.lucene.analysis.Analyzer;
+import org.elasticsearch.index.analysis.AnalyzerProvider;
+import org.elasticsearch.index.analysis.TokenFilterFactory;
 import org.elasticsearch.index.analysis.pl.PolishAnalyzerProvider;
 import org.elasticsearch.index.analysis.pl.PolishStemTokenFilterFactory;
-import org.elasticsearch.indices.analysis.AnalysisModule;
+import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
+import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.plugins.Plugin;
 
-public class AnalysisStempelPlugin extends Plugin {
+import java.util.Map;
 
-    public void onModule(AnalysisModule module) {
-        module.registerAnalyzer("polish", PolishAnalyzerProvider::new);
-        module.registerTokenFilter("polish_stem", PolishStemTokenFilterFactory::new);
+import static java.util.Collections.singletonMap;
+
+public class AnalysisStempelPlugin extends Plugin implements AnalysisPlugin {
+    @Override
+    public Map<String, AnalysisProvider<TokenFilterFactory>> getTokenFilters() {
+        return singletonMap("polish_stem", PolishStemTokenFilterFactory::new);
+    }
+
+    @Override
+    public Map<String, AnalysisProvider<AnalyzerProvider<? extends Analyzer>>> getAnalyzers() {
+        return singletonMap("polish", PolishAnalyzerProvider::new);
     }
 }
diff --git a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java
index 9bfcc2c2f3f..4f7ee642ebd 100644
--- a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java
+++ b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/PolishAnalysisTests.java
@@ -36,7 +36,8 @@ import static org.hamcrest.Matchers.instanceOf;
  */
 public class PolishAnalysisTests extends ESTestCase {
     public void testDefaultsPolishAnalysis() throws IOException {
-        final AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY, new AnalysisStempelPlugin()::onModule);
+        final AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY,
+                new AnalysisStempelPlugin());
         TokenFilterFactory tokenizerFactory = analysisService.tokenFilter("polish_stem");
         MatcherAssert.assertThat(tokenizerFactory, instanceOf(PolishStemTokenFilterFactory.class));
 
diff --git a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java
index 9458b6920c4..3fc12ccdfed 100644
--- a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java
+++ b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/SimplePolishTokenFilterTests.java
@@ -49,7 +49,7 @@ public class SimplePolishTokenFilterTests extends ESTestCase {
         Settings settings = Settings.builder()
                 .put("index.analysis.filter.myStemmer.type", "polish_stem")
                 .build();
-        AnalysisService analysisService = createAnalysisService(index, settings, new AnalysisStempelPlugin()::onModule);
+        AnalysisService analysisService = createAnalysisService(index, settings, new AnalysisStempelPlugin());
 
         TokenFilterFactory filterFactory = analysisService.tokenFilter("myStemmer");
 
@@ -65,8 +65,7 @@ public class SimplePolishTokenFilterTests extends ESTestCase {
     }
 
     private void testAnalyzer(String source, String... expected_terms) throws IOException {
-        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY,
-            new AnalysisStempelPlugin()::onModule);
+        AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY, new AnalysisStempelPlugin());
 
         Analyzer analyzer = analysisService.analyzer("polish").analyzer();
 
diff --git a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
index fe434246035..1ac0b53cbab 100644
--- a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java
@@ -36,6 +36,8 @@ import java.io.IOException;
 import java.nio.file.Path;
 import java.util.Collections;
 
+import static org.elasticsearch.test.ESTestCase.createAnalysisService;
+
 
 public class MapperTestUtils {
 
@@ -54,7 +56,7 @@ public class MapperTestUtils {
         Settings finalSettings = settingsBuilder.build();
         MapperRegistry mapperRegistry = indicesModule.getMapperRegistry();
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("test", finalSettings);
-        AnalysisService analysisService = new AnalysisRegistry(null, new Environment(finalSettings)).build(indexSettings);
+        AnalysisService analysisService = createAnalysisService(indexSettings, finalSettings);
         SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
         return new MapperService(indexSettings,
             analysisService,
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
index 5eeaa635305..95fadc99e5c 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
@@ -71,7 +71,6 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.analysis.AnalysisRegistry;
 import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
@@ -86,6 +85,7 @@ import org.elasticsearch.index.query.support.QueryParsers;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.indices.analysis.AnalysisModule;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
@@ -118,6 +118,7 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 
+import static java.util.Collections.emptyList;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;
@@ -917,7 +918,8 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             injector = modulesBuilder.createInjector();
             IndexScopedSettings indexScopedSettings = injector.getInstance(IndexScopedSettings.class);
             idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings, indexScopedSettings);
-            AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings);
+            AnalysisModule analysisModule = new AnalysisModule(new Environment(settings), emptyList());
+            AnalysisService analysisService = analysisModule.getAnalysisRegistry().build(idxSettings);
             scriptService = scriptModule.getScriptService();
             similarityService = new SimilarityService(idxSettings, Collections.emptyMap());
             MapperRegistry mapperRegistry = injector.getInstance(MapperRegistry.class);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 76616fbdb44..50d6dc88aaf 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -28,6 +28,7 @@ import com.carrotsearch.randomizedtesting.generators.RandomInts;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter;
+
 import org.apache.lucene.uninverting.UninvertingReader;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
@@ -44,7 +45,6 @@ import org.elasticsearch.common.io.PathUtilsForTesting;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.util.MockBigArrays;
 import org.elasticsearch.common.util.MockPageCacheRecycler;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -54,9 +54,11 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.analysis.AnalysisModule;
+import org.elasticsearch.plugins.AnalysisPlugin;
 import org.elasticsearch.script.MockScriptEngine;
 import org.elasticsearch.script.ScriptModule;
 import org.elasticsearch.script.ScriptService;
@@ -768,29 +770,34 @@ public abstract class ESTestCase extends LuceneTestCase {
     }
 
     /**
-     * Creates an AnalysisService to test analysis factories and analyzers.
+     * Creates an AnalysisService with all the default analyzers configured.
      */
-    @SafeVarargs
-    public static AnalysisService createAnalysisService(Index index, Settings settings, Consumer<AnalysisModule>... moduleConsumers) throws IOException {
+    public static AnalysisService createAnalysisService(Index index, Settings settings, AnalysisPlugin... analysisPlugins)
+            throws IOException {
         Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build();
-        return createAnalysisService(index, nodeSettings, settings, moduleConsumers);
+        return createAnalysisService(index, nodeSettings, settings, analysisPlugins);
     }
 
     /**
-     * Creates an AnalysisService to test analysis factories and analyzers.
+     * Creates an AnalysisService with all the default analyzers configured.
      */
-    @SafeVarargs
-    public static AnalysisService createAnalysisService(Index index, Settings nodeSettings, Settings settings, Consumer<AnalysisModule>... moduleConsumers) throws IOException {
+    public static AnalysisService createAnalysisService(Index index, Settings nodeSettings, Settings settings,
+            AnalysisPlugin... analysisPlugins) throws IOException {
         Settings indexSettings = Settings.builder().put(settings)
-            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
-            .build();
+                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                .build();
+        return createAnalysisService(IndexSettingsModule.newIndexSettings(index, indexSettings), nodeSettings, analysisPlugins);
+    }
+
+    /**
+     * Creates an AnalysisService with all the default analyzers configured.
+     */
+    public static AnalysisService createAnalysisService(IndexSettings indexSettings, Settings nodeSettings,
+            AnalysisPlugin... analysisPlugins) throws IOException {
         Environment env = new Environment(nodeSettings);
-        AnalysisModule analysisModule = new AnalysisModule(env);
-        for (Consumer<AnalysisModule> consumer : moduleConsumers) {
-            consumer.accept(analysisModule);
-        }
-        SettingsModule settingsModule = new SettingsModule(nodeSettings, InternalSettingsPlugin.VERSION_CREATED);
-        final AnalysisService analysisService = analysisModule.buildRegistry().build(IndexSettingsModule.newIndexSettings(index, indexSettings));
+        AnalysisModule analysisModule = new AnalysisModule(env, Arrays.asList(analysisPlugins));
+        final AnalysisService analysisService = analysisModule.getAnalysisRegistry()
+                .build(indexSettings);
         return analysisService;
     }
 

From fec4a18835eabbb217faec465164b279e0f28592 Mon Sep 17 00:00:00 2001
From: Damien Alexandre <mobman02@gmail.com>
Date: Sun, 26 Jun 2016 17:41:34 +0200
Subject: [PATCH 1066/1311] Rename plainless into painless in migration doc

The scripting language was wrongly named.
---
 docs/reference/migration/migrate_5_0/settings.asciidoc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/reference/migration/migrate_5_0/settings.asciidoc b/docs/reference/migration/migrate_5_0/settings.asciidoc
index 4863588818f..be578b00c3b 100644
--- a/docs/reference/migration/migrate_5_0/settings.asciidoc
+++ b/docs/reference/migration/migrate_5_0/settings.asciidoc
@@ -250,11 +250,11 @@ Due to the fact that indexed script has been replaced by stored
 scripts the following settings have been replaced to:
 
 * `script.indexed` has been replaced by `script.stored`
-* `script.engine.*.indexed.aggs` has been replaced by `script.engine.*.stored.aggs` (where `*` represents the script language, like `groovy`, `mustache`, `plainless` etc.)
-* `script.engine.*.indexed.mapping` has been replaced by `script.engine.*.stored.mapping` (where `*` represents the script language, like `groovy`, `mustache`, `plainless` etc.)
-* `script.engine.*.indexed.search` has been replaced by `script.engine.*.stored.search` (where `*` represents the script language, like `groovy`, `mustache`, `plainless` etc.)
-* `script.engine.*.indexed.update` has been replaced by `script.engine.*.stored.update` (where `*` represents the script language, like `groovy`, `mustache`, `plainless` etc.)
-* `script.engine.*.indexed.plugin` has been replaced by `script.engine.*.stored.plugin` (where `*` represents the script language, like `groovy`, `mustache`, `plainless` etc.)
+* `script.engine.*.indexed.aggs` has been replaced by `script.engine.*.stored.aggs` (where `*` represents the script language, like `groovy`, `mustache`, `painless` etc.)
+* `script.engine.*.indexed.mapping` has been replaced by `script.engine.*.stored.mapping` (where `*` represents the script language, like `groovy`, `mustache`, `painless` etc.)
+* `script.engine.*.indexed.search` has been replaced by `script.engine.*.stored.search` (where `*` represents the script language, like `groovy`, `mustache`, `painless` etc.)
+* `script.engine.*.indexed.update` has been replaced by `script.engine.*.stored.update` (where `*` represents the script language, like `groovy`, `mustache`, `painless` etc.)
+* `script.engine.*.indexed.plugin` has been replaced by `script.engine.*.stored.plugin` (where `*` represents the script language, like `groovy`, `mustache`, `painless` etc.)
 
 ==== Script mode settings
 

From 9a0ce62550e82954cb8ce1bf1cca0c96eb377845 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Fri, 24 Jun 2016 18:12:44 +0200
Subject: [PATCH 1067/1311] percolator: Add support for the synonym query.

---
 .../percolator/ExtractQueryTermsService.java        |  9 +++++++++
 .../percolator/ExtractQueryTermsServiceTests.java   | 13 +++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java
index fbe55782e69..147eae6e4d1 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java
@@ -37,6 +37,7 @@ import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SynonymQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.spans.SpanFirstQuery;
 import org.apache.lucene.search.spans.SpanNearQuery;
@@ -90,6 +91,7 @@ public final class ExtractQueryTermsService {
         map.put(SpanNotQuery.class, spanNotQuery());
         map.put(BooleanQuery.class, booleanQuery());
         map.put(DisjunctionMaxQuery.class, disjunctionMaxQuery());
+        map.put(SynonymQuery.class, synonymQuery());
         queryProcessors = Collections.unmodifiableMap(map);
     }
 
@@ -221,6 +223,13 @@ public final class ExtractQueryTermsService {
         };
     }
 
+    static Function<Query, Result> synonymQuery() {
+        return query -> {
+            Set<Term> terms = new HashSet<>(((SynonymQuery) query).getTerms());
+            return new Result(true, terms);
+        };
+    }
+
     static Function<Query, Result> commonTermsQuery() {
         return query -> {
             List<Term> terms = ((CommonTermsQuery) query).getTerms();
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java
index f876a0af9fc..b9486a89f49 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java
@@ -36,6 +36,7 @@ import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.SynonymQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.search.spans.SpanFirstQuery;
@@ -558,6 +559,18 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         assertThat(terms.get(3).bytes(), equalTo(termQuery4.getTerm().bytes()));
     }
 
+    public void testSynonymQuery() {
+        SynonymQuery query = new SynonymQuery();
+        Result result = extractQueryTerms(query);
+        assertThat(result.verified, is(true));
+        assertThat(result.terms.isEmpty(), is(true));
+
+        query = new SynonymQuery(new Term("_field", "_value1"), new Term("_field", "_value2"));
+        result = extractQueryTerms(query);
+        assertThat(result.verified, is(true));
+        assertTermsEqual(result.terms, new Term("_field", "_value1"), new Term("_field", "_value2"));
+    }
+
     public void testCreateQueryMetadataQuery() throws Exception {
         MemoryIndex memoryIndex = new MemoryIndex(false);
         memoryIndex.addField("field1", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer());

From ba90508b919822d64c68f2c96ec8f44646fe6d78 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Mon, 27 Jun 2016 09:00:13 +0200
Subject: [PATCH 1068/1311] fix checkstyle issue

---
 .../org/elasticsearch/index/query/GeoShapeQueryBuilder.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
index e6276a76c84..63247019501 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoShapeQueryBuilder.java
@@ -379,7 +379,8 @@ public class GeoShapeQueryBuilder extends AbstractQueryBuilder<GeoShapeQueryBuil
             throw new IllegalArgumentException("Shape with ID [" + getRequest.id() + "] in type [" + getRequest.type() + "] not found");
         }
         if (response.isSourceEmpty()) {
-            throw new IllegalArgumentException("Shape with ID [" + getRequest.id() + "] in type [" + getRequest.type() + "] source disabled");
+            throw new IllegalArgumentException("Shape with ID [" + getRequest.id() + "] in type [" + getRequest.type() +
+                    "] source disabled");
         }
 
         String[] pathElements = path.split("\\.");

From 59762fe487185e6bee07f0ed1c09d506d1c9a28b Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Mon, 27 Jun 2016 12:04:59 +0200
Subject: [PATCH 1069/1311] Register group setting for repository-azure
 accounts

Since the Settings infrastructure has been improved, a group setting must be registered by the repository-azure plugin to allow settings like "cloud.azure.storage.my_account.account" to be coherent with Azure plugin documentation.
---
 plugins/repository-azure/build.gradle         |  8 ++++++
 .../azure/storage/AzureStorageService.java    |  4 +++
 .../azure/storage/AzureStorageSettings.java   |  3 +--
 .../azure/AzureRepositoryPlugin.java          | 15 +++++------
 .../test/repository_azure/20_repository.yaml  | 26 +++++++++++++++++++
 5 files changed, 46 insertions(+), 10 deletions(-)
 create mode 100644 plugins/repository-azure/src/test/resources/rest-api-spec/test/repository_azure/20_repository.yaml

diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
index d3ef723cb20..125b7bdd6c5 100644
--- a/plugins/repository-azure/build.gradle
+++ b/plugins/repository-azure/build.gradle
@@ -40,3 +40,11 @@ thirdPartyAudit.excludes = [
   'org.slf4j.Logger',
   'org.slf4j.LoggerFactory',
 ]
+
+integTest {
+  cluster {
+    setting 'cloud.azure.storage.my_account_test.account', 'cloudazureresource'
+    setting 'cloud.azure.storage.my_account_test.key', 'abcdefgh'
+    setting 'script.stored', 'true'
+  }
+}
\ No newline at end of file
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
index b6f5f12b266..13db36aeb5a 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
@@ -25,6 +25,7 @@ import com.microsoft.azure.storage.StorageException;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 
@@ -42,6 +43,9 @@ public interface AzureStorageService {
 
     final class Storage {
         public static final String PREFIX = "cloud.azure.storage.";
+
+        public static final Setting<Settings> STORAGE_ACCOUNTS = Setting.groupSetting(Storage.PREFIX, Setting.Property.NodeScope);
+
         public static final Setting<TimeValue> TIMEOUT_SETTING =
             Setting.timeSetting("cloud.azure.storage.timeout", TimeValue.timeValueMinutes(-1), Property.NodeScope);
         public static final Setting<String> ACCOUNT_SETTING =
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java
index 281ef79cb27..f64ffed4a03 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java
@@ -112,9 +112,8 @@ public final class AzureStorageSettings {
     }
 
     private static List<AzureStorageSettings> createStorageSettings(Settings settings) {
-        Setting<Settings> storageGroupSetting = Setting.groupSetting(Storage.PREFIX, Setting.Property.NodeScope);
         // ignore global timeout which has the same prefix but does not belong to any group
-        Settings groups = storageGroupSetting.get(settings.filter((k) -> k.equals(Storage.TIMEOUT_SETTING.getKey()) == false));
+        Settings groups = Storage.STORAGE_ACCOUNTS.get(settings.filter((k) -> k.equals(Storage.TIMEOUT_SETTING.getKey()) == false));
         List<AzureStorageSettings> storageSettings = new ArrayList<>();
         for (String groupName : groups.getAsGroups().keySet()) {
             storageSettings.add(
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java b/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
index ce122353f51..b04b613df21 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
@@ -26,7 +26,6 @@ import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.repositories.RepositoriesModule;
@@ -62,13 +61,13 @@ public class AzureRepositoryPlugin extends Plugin {
 
     @Override
     public List<Setting<?>> getSettings() {
-        return Arrays.asList(AzureStorageService.Storage.ACCOUNT_SETTING,
-        AzureStorageService.Storage.COMPRESS_SETTING,
-        AzureStorageService.Storage.CONTAINER_SETTING,
-        AzureStorageService.Storage.BASE_PATH_SETTING,
-        AzureStorageService.Storage.CHUNK_SIZE_SETTING,
-        AzureStorageService.Storage.LOCATION_MODE_SETTING);
-
+        return Arrays.asList(AzureStorageService.Storage.STORAGE_ACCOUNTS,
+                AzureStorageService.Storage.ACCOUNT_SETTING,
+                AzureStorageService.Storage.COMPRESS_SETTING,
+                AzureStorageService.Storage.CONTAINER_SETTING,
+                AzureStorageService.Storage.BASE_PATH_SETTING,
+                AzureStorageService.Storage.CHUNK_SIZE_SETTING,
+                AzureStorageService.Storage.LOCATION_MODE_SETTING);
     }
 
     @Override
diff --git a/plugins/repository-azure/src/test/resources/rest-api-spec/test/repository_azure/20_repository.yaml b/plugins/repository-azure/src/test/resources/rest-api-spec/test/repository_azure/20_repository.yaml
new file mode 100644
index 00000000000..e3979cd32c3
--- /dev/null
+++ b/plugins/repository-azure/src/test/resources/rest-api-spec/test/repository_azure/20_repository.yaml
@@ -0,0 +1,26 @@
+"Repository can be registered":
+    - do:
+        snapshot.create_repository:
+          repository: test_repo_azure
+          verify: false
+          body:
+            type: azure
+            settings:
+              account   : "my_test_account"
+              container : "backup-container"
+              base_path :  "backups"
+              chunk_size: "32m"
+              compress  : true
+
+    - is_true: acknowledged
+
+    - do:
+        snapshot.get_repository:
+          repository: test_repo_azure
+
+    - is_true : test_repo_azure
+    - match   : { test_repo_azure.settings.account   : "my_test_account" }
+    - match   : { test_repo_azure.settings.container : "backup-container" }
+    - match   : { test_repo_azure.settings.base_path : "backups" }
+    - match   : { test_repo_azure.settings.chunk_size: "32m" }
+    - match   : { test_repo_azure.settings.compress  : "true" }

From 1863ab95f862294db7d105717656bd6452eef9d9 Mon Sep 17 00:00:00 2001
From: Jerry Liu <jerryliu55@gmail.com>
Date: Mon, 27 Jun 2016 08:20:23 -0400
Subject: [PATCH 1070/1311] fixed typo 'if' -> 'is' (#19051)

---
 .../aggregations/metrics/scripted-metric-aggregation.asciidoc   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc
index 639fabb62fd..ba9899f9d68 100644
--- a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc
@@ -101,7 +101,7 @@ In the above example, the `init_script` creates an array `transactions` in the `
 map_script::        Executed once per document collected. This is the only required script. If no combine_script is specified, the resulting state 
                     needs to be stored in an object named `_agg`.
 +
-In the above example, the `map_script` checks the value of the type field. If the value if 'sale' the value of the amount field 
+In the above example, the `map_script` checks the value of the type field. If the value is 'sale' the value of the amount field 
 is added to the transactions array. If the value of the type field is not 'sale' the negated value of the amount field is added 
 to transactions.
 

From 453a4b9647753c8179b51a7787d604b2ae71720b Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Mon, 27 Jun 2016 14:49:37 +0200
Subject: [PATCH 1071/1311] Fix documentation typo in How-To docs

---
 docs/reference/how-to/indexing-speed.asciidoc | 2 +-
 docs/reference/how-to/search-speed.asciidoc   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/reference/how-to/indexing-speed.asciidoc b/docs/reference/how-to/indexing-speed.asciidoc
index 8175c21ee8a..bb5a367a04c 100644
--- a/docs/reference/how-to/indexing-speed.asciidoc
+++ b/docs/reference/how-to/indexing-speed.asciidoc
@@ -28,7 +28,7 @@ help reduce the cost of each fsync.
 Make sure to watch for `TOO_MANY_REQUESTS (429)` response codes
 (`EsRejectedExecutionException` with the Java client), which is the way that
 elasticsearch tells you that it cannot keep up with the current indexing rate.
-When it happens, you should pause ndexing a bit before trying again, ideally
+When it happens, you should pause indexing a bit before trying again, ideally
 with randomized exponential backoff.
 
 Similarly to sizing bulk requests, only testing can tell what the optimal
diff --git a/docs/reference/how-to/search-speed.asciidoc b/docs/reference/how-to/search-speed.asciidoc
index b6e9c4e927d..67848c9edca 100644
--- a/docs/reference/how-to/search-speed.asciidoc
+++ b/docs/reference/how-to/search-speed.asciidoc
@@ -190,5 +190,5 @@ depending on the file extension using the <<file-system,`index.store.preload`>>
 setting.
 
 WARNING: Loading data into the filesystem cache eagerly on too many indices or
-too many files will make searh _slower_ if the filesystem cache is not large
+too many files will make search _slower_ if the filesystem cache is not large
 enough to hold all the data. Use with caution.

From cb0824e957dcb7a20e8450f97283874eb61b9abe Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Mon, 27 Jun 2016 15:05:06 +0200
Subject: [PATCH 1072/1311] Make shard store fetch less dependent on the
 current cluster state, both on master and non data nodes (#19044)

#18938 has changed the timing in which we send out to nodes to fetch their shard stores. Instead of doing this after the cluster state resulting of the node's join was published, #18938 made it be sent concurrently to the publishing processes. This revealed a couple of points where the shard store fetching is dependent of the current state of affairs of the cluster state, both on the master and the data nodes. The problem discovered were already present without #18938 but required a failure/extreme situations to make them happen.This PR tries to remove as much as possible of these dependencies making shard store fetching simpler and make the way to re-introduce #18938 which was reverted.

These are the notable changes:
1) Allow TransportNodesAction (of which shard store fetching is derived) callers to supply concrete disco nodes, so it won't need the cluster state to resolve them. This was a problem because the cluster state containing the needed nodes was not yet made available through ClusterService. Note that long term we can expect the rest layer to resolve node ids to concrete nodes, making this mode the only one needed.
2) The data node relied on the cluster state to have the relevant index meta data so it can find data when custom paths are used. We now fall back to read the meta data from disk if needed.
3) The data node was relying on it's own IndexService state to indicate whether the data it has corresponds to an existing allocation. This is of course something it can not know until it got (and processed) the new cluster state from the master. This flag in the response is now removed. This is not a problem because we used that flag to protect against double assigning of a shard to the same node, but we are already protected from it by the allocation deciders.
4) I removed the redundant filterNodeIds method in TransportNodesAction - if people want to filter they can override resolveRequest.
---
 .../support/nodes/BaseNodesRequest.java       |  35 +++++-
 .../support/nodes/TransportNodesAction.java   |  41 +++----
 .../support/tasks/TransportTasksAction.java   |   2 +-
 .../cluster/node/DiscoveryNodes.java          |  24 ++--
 .../routing/IndexShardRoutingTable.java       |   3 +-
 .../common/io/stream/StreamInput.java         |  16 ++-
 .../common/io/stream/StreamInputReader.java   |  33 ------
 .../common/io/stream/StreamOutput.java        |  16 +++
 .../gateway/AsyncShardFetch.java              |  25 ++--
 .../gateway/ReplicaShardAllocator.java        |  29 +++--
 ...ransportNodesListGatewayStartedShards.java |  16 +--
 .../TransportNodesListShardStoreMetaData.java | 110 +++++++-----------
 .../bucket/range/AbstractRangeBuilder.java    |   4 +-
 .../cluster/node/tasks/TestTaskPlugin.java    |  12 --
 .../nodes/TransportNodesActionTests.java      |  20 ++--
 .../cluster/node/DiscoveryNodesTests.java     |   2 +-
 .../common/io/stream/StreamTests.java         |  62 +++++++++-
 .../DiscoveryWithServiceDisruptionsIT.java    |   1 +
 .../gateway/AsyncShardFetchTests.java         |   5 +-
 .../gateway/RecoveryFromGatewayIT.java        |   2 +-
 .../gateway/ReplicaShardAllocatorTests.java   |  59 +++++-----
 .../elasticsearch/index/store/StoreTests.java |   2 +-
 .../test/ESAllocationTestCase.java            |   7 +-
 23 files changed, 280 insertions(+), 246 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/common/io/stream/StreamInputReader.java

diff --git a/core/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java b/core/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java
index d5375ef9ef9..663537f25da 100644
--- a/core/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java
@@ -21,7 +21,8 @@ package org.elasticsearch.action.support.nodes;
 
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.common.Strings;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
@@ -33,10 +34,24 @@ import java.io.IOException;
  */
 public abstract class BaseNodesRequest<Request extends BaseNodesRequest<Request>> extends ActionRequest<Request> {
 
-    public static String[] ALL_NODES = Strings.EMPTY_ARRAY;
-
+    /**
+     * the list of nodesIds that will be used to resolve this request and {@link #concreteNodes}
+     * will be populated. Note that if {@link #concreteNodes} is not null, it will be used and nodeIds
+     * will be ignored.
+     *
+     * See {@link DiscoveryNodes#resolveNodes} for a full description of the options.
+     *
+     * TODO: once we stop using the transport client as a gateway to the cluster, we can get rid of this and resolve it to concrete nodes
+     * in the rest layer
+     **/
     private String[] nodesIds;
 
+    /**
+     * once {@link #nodesIds} are resolved this will contain the concrete nodes that are part of this request. If set, {@link #nodesIds}
+     * will be ignored and this will be used.
+     * */
+    private DiscoveryNode[] concreteNodes;
+
     private TimeValue timeout;
 
     protected BaseNodesRequest() {
@@ -47,6 +62,11 @@ public abstract class BaseNodesRequest<Request extends BaseNodesRequest<Request>
         this.nodesIds = nodesIds;
     }
 
+    protected BaseNodesRequest(DiscoveryNode... concreteNodes) {
+        this.nodesIds = null;
+        this.concreteNodes = concreteNodes;
+    }
+
     public final String[] nodesIds() {
         return nodesIds;
     }
@@ -72,6 +92,13 @@ public abstract class BaseNodesRequest<Request extends BaseNodesRequest<Request>
         this.timeout = TimeValue.parseTimeValue(timeout, null, getClass().getSimpleName() + ".timeout");
         return (Request) this;
     }
+    public DiscoveryNode[] concreteNodes() {
+        return concreteNodes;
+    }
+
+    public void setConcreteNodes(DiscoveryNode[] concreteNodes) {
+        this.concreteNodes = concreteNodes;
+    }
 
     @Override
     public ActionRequestValidationException validate() {
@@ -82,6 +109,7 @@ public abstract class BaseNodesRequest<Request extends BaseNodesRequest<Request>
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         nodesIds = in.readStringArray();
+        concreteNodes = in.readOptionalArray(DiscoveryNode::new, DiscoveryNode[]::new);
         timeout = in.readOptionalWriteable(TimeValue::new);
     }
 
@@ -89,6 +117,7 @@ public abstract class BaseNodesRequest<Request extends BaseNodesRequest<Request>
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeStringArrayNullable(nodesIds);
+        out.writeOptionalArray(concreteNodes);
         out.writeOptionalWriteable(timeout);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
index 5d39d378b21..1609a08bb14 100644
--- a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
@@ -27,7 +27,6 @@ import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.Task;
@@ -42,6 +41,7 @@ import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportService;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -148,20 +148,19 @@ public abstract class TransportNodesAction<NodesRequest extends BaseNodesRequest
 
     protected abstract boolean accumulateExceptions();
 
-    protected String[] filterNodeIds(DiscoveryNodes nodes, String[] nodesIds) {
-        return nodesIds;
-    }
-
-    protected String[] resolveNodes(NodesRequest request, ClusterState clusterState) {
-        return clusterState.nodes().resolveNodesIds(request.nodesIds());
+    /**
+     * resolve node ids to concrete nodes of the incoming request
+     **/
+    protected void resolveRequest(NodesRequest request, ClusterState clusterState) {
+        assert request.concreteNodes() == null : "request concreteNodes shouldn't be set";
+        String[] nodesIds = clusterState.nodes().resolveNodes(request.nodesIds());
+        request.setConcreteNodes(Arrays.stream(nodesIds).map(clusterState.nodes()::get).toArray(DiscoveryNode[]::new));
     }
 
 
     class AsyncAction {
 
         private final NodesRequest request;
-        private final String[] nodesIds;
-        private final DiscoveryNode[] nodes;
         private final ActionListener<NodesResponse> listener;
         private final AtomicReferenceArray<Object> responses;
         private final AtomicInteger counter = new AtomicInteger();
@@ -171,24 +170,18 @@ public abstract class TransportNodesAction<NodesRequest extends BaseNodesRequest
             this.task = task;
             this.request = request;
             this.listener = listener;
-            ClusterState clusterState = clusterService.state();
-            nodesIds = filterNodeIds(clusterState.nodes(), resolveNodes(request, clusterState));
-            this.nodes = new DiscoveryNode[nodesIds.length];
-            for (int i = 0; i < nodesIds.length; i++) {
-                this.nodes[i] = clusterState.nodes().get(nodesIds[i]);
+            if (request.concreteNodes() == null) {
+                resolveRequest(request, clusterService.state());
+                assert request.concreteNodes() != null;
             }
-            this.responses = new AtomicReferenceArray<>(this.nodesIds.length);
+            this.responses = new AtomicReferenceArray<>(request.concreteNodes().length);
         }
 
         void start() {
-            if (nodesIds.length == 0) {
+            final DiscoveryNode[] nodes = request.concreteNodes();
+            if (nodes.length == 0) {
                 // nothing to notify
-                threadPool.generic().execute(new Runnable() {
-                    @Override
-                    public void run() {
-                        listener.onResponse(newResponse(request, responses));
-                    }
-                });
+                threadPool.generic().execute(() -> listener.onResponse(newResponse(request, responses)));
                 return;
             }
             TransportRequestOptions.Builder builder = TransportRequestOptions.builder();
@@ -196,10 +189,10 @@ public abstract class TransportNodesAction<NodesRequest extends BaseNodesRequest
                 builder.withTimeout(request.timeout());
             }
             builder.withCompress(transportCompress());
-            for (int i = 0; i < nodesIds.length; i++) {
-                final String nodeId = nodesIds[i];
+            for (int i = 0; i < nodes.length; i++) {
                 final int idx = i;
                 final DiscoveryNode node = nodes[i];
+                final String nodeId = node.getId();
                 try {
                     if (node == null) {
                         onFailure(idx, nodeId, new NoSuchNodeException(nodeId));
diff --git a/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java b/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
index f28b70b88ed..0a45497e1fd 100644
--- a/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
@@ -124,7 +124,7 @@ public abstract class TransportTasksAction<
         if (request.getTaskId().isSet()) {
             return new String[]{request.getTaskId().getNodeId()};
         } else {
-            return clusterState.nodes().resolveNodesIds(request.getNodesIds());
+            return clusterState.nodes().resolveNodes(request.getNodesIds());
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
index 16ec31be0b1..43d71de726a 100644
--- a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
+++ b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
@@ -245,7 +245,7 @@ public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements
      * @throws IllegalArgumentException if more than one node matches the request or no nodes have been resolved
      */
     public DiscoveryNode resolveNode(String node) {
-        String[] resolvedNodeIds = resolveNodesIds(node);
+        String[] resolvedNodeIds = resolveNodes(node);
         if (resolvedNodeIds.length > 1) {
             throw new IllegalArgumentException("resolved [" + node + "] into [" + resolvedNodeIds.length + "] nodes, where expected to be resolved to a single node");
         }
@@ -255,17 +255,25 @@ public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements
         return nodes.get(resolvedNodeIds[0]);
     }
 
-    public String[] resolveNodesIds(String... nodesIds) {
-        if (isAllNodes(nodesIds)) {
+    /**
+     * resolves a set of node "descriptions" to concrete and existing node ids. "descriptions" can be (resolved in this order):
+     * - "_local" or "_master" for the relevant nodes
+     * - a node id
+     * - a wild card pattern that will be matched against node names
+     * - a "attr:value" pattern, where attr can be a node role (master, data, ingest etc.) in which case the value can be true of false
+     *   or a generic node attribute name in which case value will be treated as a wildcard and matched against the node attribute values.
+     */
+    public String[] resolveNodes(String... nodes) {
+        if (isAllNodes(nodes)) {
             int index = 0;
-            nodesIds = new String[nodes.size()];
+            nodes = new String[this.nodes.size()];
             for (DiscoveryNode node : this) {
-                nodesIds[index++] = node.getId();
+                nodes[index++] = node.getId();
             }
-            return nodesIds;
+            return nodes;
         } else {
-            ObjectHashSet<String> resolvedNodesIds = new ObjectHashSet<>(nodesIds.length);
-            for (String nodeId : nodesIds) {
+            ObjectHashSet<String> resolvedNodesIds = new ObjectHashSet<>(nodes.length);
+            for (String nodeId : nodes) {
                 if (nodeId.equals("_local")) {
                     String localNodeId = getLocalNodeId();
                     if (localNodeId != null) {
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
index 91b1df9b453..be09f17f866 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
@@ -22,7 +22,6 @@ package org.elasticsearch.cluster.routing;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Randomness;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -357,7 +356,7 @@ public class IndexShardRoutingTable implements Iterable<ShardRouting> {
      */
     public ShardIterator onlyNodeSelectorActiveInitializingShardsIt(String[] nodeAttributes, DiscoveryNodes discoveryNodes) {
         ArrayList<ShardRouting> ordered = new ArrayList<>(activeShards.size() + allInitializingShards.size());
-        Set<String> selectedNodes = Sets.newHashSet(discoveryNodes.resolveNodesIds(nodeAttributes));
+        Set<String> selectedNodes = Sets.newHashSet(discoveryNodes.resolveNodes(nodeAttributes));
         int seed = shuffler.nextSeed();
         for (ShardRouting shardRouting : shuffler.shuffle(activeShards, seed)) {
             if (selectedNodes.contains(shardRouting.currentNodeId())) {
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
index ceb909fb49a..ce0bf59e9e4 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
@@ -57,6 +57,7 @@ import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.IntFunction;
 import java.util.function.Supplier;
 
 import static org.elasticsearch.ElasticsearchException.readException;
@@ -608,6 +609,19 @@ public abstract class StreamInput extends InputStream {
         return bytes;
     }
 
+    public <T> T[] readArray(Writeable.Reader<T> reader, IntFunction<T[]> arraySupplier) throws IOException {
+        int length = readVInt();
+        T[] values = arraySupplier.apply(length);
+        for (int i = 0; i < length; i++) {
+            values[i] = reader.read(this);
+        }
+        return values;
+    }
+
+    public <T> T[] readOptionalArray(Writeable.Reader<T> reader, IntFunction<T[]> arraySupplier) throws IOException {
+        return readBoolean() ? readArray(reader, arraySupplier) : null;
+    }
+
     /**
      * Serializes a potential null value.
      */
@@ -782,7 +796,7 @@ public abstract class StreamInput extends InputStream {
     /**
      * Reads a list of objects
      */
-    public <T> List<T> readList(StreamInputReader<T> reader) throws IOException {
+    public <T> List<T> readList(Writeable.Reader<T> reader) throws IOException {
         int count = readVInt();
         List<T> builder = new ArrayList<>(count);
         for (int i=0; i<count; i++) {
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInputReader.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInputReader.java
deleted file mode 100644
index 6eb067f40ee..00000000000
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInputReader.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.io.stream;
-
-import java.io.IOException;
-
-/**
- * Defines a method for reading a list of objects from StreamInput.
- *
- * It can be used in {@link StreamInput#readList(StreamInputReader)} for reading
- * lists of immutable objects that implement StreamInput accepting constructors.
- */
-@FunctionalInterface
-public interface StreamInputReader<T> {
-    T read(StreamInput t) throws IOException;
-}
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
index 7d17029cd61..bb9af656ea8 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
@@ -594,6 +594,22 @@ public abstract class StreamOutput extends OutputStream {
         }
     }
 
+    public <T extends Writeable> void writeArray(T[] array) throws IOException {
+        writeVInt(array.length);
+        for (T value: array) {
+            value.writeTo(this);
+        }
+    }
+
+    public <T extends Writeable> void writeOptionalArray(@Nullable T[] array) throws IOException {
+        if (array == null) {
+            writeBoolean(false);
+        } else {
+            writeBoolean(true);
+            writeArray(array);
+        }
+    }
+
     /**
      * Serializes a potential null value.
      */
diff --git a/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java b/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java
index b74507a4acc..737ed7d591c 100644
--- a/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java
+++ b/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java
@@ -60,7 +60,7 @@ public abstract class AsyncShardFetch<T extends BaseNodeResponse> implements Rel
      * An action that lists the relevant shard data that needs to be fetched.
      */
     public interface Lister<NodesResponse extends BaseNodesResponse<NodeResponse>, NodeResponse extends BaseNodeResponse> {
-        void list(ShardId shardId, String[] nodesIds, ActionListener<NodesResponse> listener);
+        void list(ShardId shardId, DiscoveryNode[] nodes, ActionListener<NodesResponse> listener);
     }
 
     protected final ESLogger logger;
@@ -116,12 +116,9 @@ public abstract class AsyncShardFetch<T extends BaseNodeResponse> implements Rel
             for (NodeEntry<T> nodeEntry : nodesToFetch) {
                 nodeEntry.markAsFetching();
             }
-            String[] nodesIds = new String[nodesToFetch.size()];
-            int index = 0;
-            for (NodeEntry<T> nodeEntry : nodesToFetch) {
-                nodesIds[index++] = nodeEntry.getNodeId();
-            }
-            asyncFetch(shardId, nodesIds);
+            DiscoveryNode[] discoNodesToFetch = nodesToFetch.stream().map(NodeEntry::getNodeId).map(nodes::get)
+                .toArray(DiscoveryNode[]::new);
+            asyncFetch(shardId, discoNodesToFetch);
         }
 
         // if we are still fetching, return null to indicate it
@@ -187,7 +184,7 @@ public abstract class AsyncShardFetch<T extends BaseNodeResponse> implements Rel
                 if (nodeEntry.isFailed()) {
                     logger.trace("{} node {} has failed for [{}] (failure [{}])", shardId, nodeEntry.getNodeId(), type, nodeEntry.getFailure());
                 } else {
-                    logger.trace("{} marking {} as done for [{}]", shardId, nodeEntry.getNodeId(), type);
+                    logger.trace("{} marking {} as done for [{}], result is [{}]", shardId, nodeEntry.getNodeId(), type, response);
                     nodeEntry.doneFetching(response);
                 }
             }
@@ -268,9 +265,9 @@ public abstract class AsyncShardFetch<T extends BaseNodeResponse> implements Rel
      * Async fetches data for the provided shard with the set of nodes that need to be fetched from.
      */
     // visible for testing
-    void asyncFetch(final ShardId shardId, final String[] nodesIds) {
-        logger.trace("{} fetching [{}] from {}", shardId, type, nodesIds);
-        action.list(shardId, nodesIds, new ActionListener<BaseNodesResponse<T>>() {
+    void asyncFetch(final ShardId shardId, final DiscoveryNode[] nodes) {
+        logger.trace("{} fetching [{}] from {}", shardId, type, nodes);
+        action.list(shardId, nodes, new ActionListener<BaseNodesResponse<T>>() {
             @Override
             public void onResponse(BaseNodesResponse<T> response) {
                 processAsyncFetch(shardId, response.getNodes(), response.failures());
@@ -278,9 +275,9 @@ public abstract class AsyncShardFetch<T extends BaseNodeResponse> implements Rel
 
             @Override
             public void onFailure(Throwable e) {
-                List<FailedNodeException> failures = new ArrayList<>(nodesIds.length);
-                for (String nodeId : nodesIds) {
-                    failures.add(new FailedNodeException(nodeId, "total failure in fetching", e));
+                List<FailedNodeException> failures = new ArrayList<>(nodes.length);
+                for (final DiscoveryNode node: nodes) {
+                    failures.add(new FailedNodeException(node.getId(), "total failure in fetching", e));
                 }
                 processAsyncFetch(shardId, null, failures);
             }
diff --git a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
index 602a05c4321..e5f37c360c3 100644
--- a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
+++ b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java
@@ -38,13 +38,13 @@ import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.store.StoreFileMetaData;
 import org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData;
 
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 /**
  */
@@ -91,7 +91,7 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
                 ShardRouting primaryShard = allocation.routingNodes().activePrimary(shard.shardId());
                 assert primaryShard != null : "the replica shard can be allocated on at least one node, so there must be an active primary";
                 TransportNodesListShardStoreMetaData.StoreFilesMetaData primaryStore = findStore(primaryShard, allocation, shardStores);
-                if (primaryStore == null || primaryStore.allocated() == false) {
+                if (primaryStore == null) {
                     // if we can't find the primary data, it is probably because the primary shard is corrupted (and listing failed)
                     // just let the recovery find it out, no need to do anything about it for the initializing shard
                     logger.trace("{}: no primary shard store found or allocated, letting actual allocation figure it out", shard);
@@ -102,8 +102,15 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
                 if (matchingNodes.getNodeWithHighestMatch() != null) {
                     DiscoveryNode currentNode = allocation.nodes().get(shard.currentNodeId());
                     DiscoveryNode nodeWithHighestMatch = matchingNodes.getNodeWithHighestMatch();
+                    // current node will not be in matchingNodes as it is filtered away by SameShardAllocationDecider
+                    final String currentSyncId;
+                    if (shardStores.getData().containsKey(currentNode)) {
+                        currentSyncId = shardStores.getData().get(currentNode).storeFilesMetaData().syncId();
+                    } else {
+                        currentSyncId = null;
+                    }
                     if (currentNode.equals(nodeWithHighestMatch) == false
-                            && matchingNodes.isNodeMatchBySyncID(currentNode) == false
+                            && Objects.equals(currentSyncId, primaryStore.syncId()) == false
                             && matchingNodes.isNodeMatchBySyncID(nodeWithHighestMatch) == true) {
                         // we found a better match that has a full sync id match, the existing allocation is not fully synced
                         // so we found a better one, cancel this one
@@ -160,7 +167,7 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
             ShardRouting primaryShard = routingNodes.activePrimary(shard.shardId());
             assert primaryShard != null : "the replica shard can be allocated on at least one node, so there must be an active primary";
             TransportNodesListShardStoreMetaData.StoreFilesMetaData primaryStore = findStore(primaryShard, allocation, shardStores);
-            if (primaryStore == null || primaryStore.allocated() == false) {
+            if (primaryStore == null) {
                 // if we can't find the primary data, it is probably because the primary shard is corrupted (and listing failed)
                 // we want to let the replica be allocated in order to expose the actual problem with the primary that the replica
                 // will try and recover from
@@ -257,8 +264,8 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
         for (Map.Entry<DiscoveryNode, TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> nodeStoreEntry : data.getData().entrySet()) {
             DiscoveryNode discoNode = nodeStoreEntry.getKey();
             TransportNodesListShardStoreMetaData.StoreFilesMetaData storeFilesMetaData = nodeStoreEntry.getValue().storeFilesMetaData();
-            if (storeFilesMetaData == null) {
-                // already allocated on that node...
+            // we don't have any files at all, it is an empty index
+            if (storeFilesMetaData.isEmpty()) {
                 continue;
             }
 
@@ -275,16 +282,6 @@ public abstract class ReplicaShardAllocator extends AbstractComponent {
                 continue;
             }
 
-            // if it is already allocated, we can't assign to it... (and it might be primary as well)
-            if (storeFilesMetaData.allocated()) {
-                continue;
-            }
-
-            // we don't have any files at all, it is an empty index
-            if (storeFilesMetaData.iterator().hasNext() == false) {
-                continue;
-            }
-
             String primarySyncId = primaryStore.syncId();
             String replicaSyncId = storeFilesMetaData.syncId();
             // see if we have a sync id we can make use of
diff --git a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
index 5ae75a209eb..42fc8163706 100644
--- a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
+++ b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
@@ -29,7 +29,6 @@ import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -81,16 +80,9 @@ public class TransportNodesListGatewayStartedShards extends
     }
 
     @Override
-    public void list(ShardId shardId, String[] nodesIds,
+    public void list(ShardId shardId, DiscoveryNode[] nodes,
                      ActionListener<NodesGatewayStartedShards> listener) {
-        execute(new Request(shardId, nodesIds), listener);
-    }
-
-    @Override
-    protected String[] resolveNodes(Request request, ClusterState clusterState) {
-        // default implementation may filter out non existent nodes. it's important to keep exactly the ids
-        // we were given for accounting on the caller
-        return request.nodesIds();
+        execute(new Request(shardId, nodes), listener);
     }
 
     @Override
@@ -177,8 +169,8 @@ public class TransportNodesListGatewayStartedShards extends
         public Request() {
         }
 
-        public Request(ShardId shardId, String[] nodesIds) {
-            super(nodesIds);
+        public Request(ShardId shardId, DiscoveryNode[] nodes) {
+            super(nodes);
             this.shardId = shardId;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java b/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
index da5f9d16f51..a4517baf45b 100644
--- a/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
+++ b/core/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
@@ -29,7 +29,6 @@ import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.action.support.nodes.BaseNodesResponse;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -56,18 +55,17 @@ import org.elasticsearch.transport.TransportService;
 import java.io.IOException;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 /**
  *
  */
 public class TransportNodesListShardStoreMetaData extends TransportNodesAction<TransportNodesListShardStoreMetaData.Request,
-                                                                               TransportNodesListShardStoreMetaData.NodesStoreFilesMetaData,
-                                                                               TransportNodesListShardStoreMetaData.NodeRequest,
-                                                                               TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData>
+    TransportNodesListShardStoreMetaData.NodesStoreFilesMetaData,
+    TransportNodesListShardStoreMetaData.NodeRequest,
+    TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData>
     implements AsyncShardFetch.Lister<TransportNodesListShardStoreMetaData.NodesStoreFilesMetaData,
-                                      TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> {
+    TransportNodesListShardStoreMetaData.NodeStoreFilesMetaData> {
 
     public static final String ACTION_NAME = "internal:cluster/nodes/indices/shard/store";
 
@@ -81,21 +79,14 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
                                                 IndicesService indicesService, NodeEnvironment nodeEnv, ActionFilters actionFilters,
                                                 IndexNameExpressionResolver indexNameExpressionResolver) {
         super(settings, ACTION_NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
-              Request::new, NodeRequest::new, ThreadPool.Names.FETCH_SHARD_STORE, NodeStoreFilesMetaData.class);
+            Request::new, NodeRequest::new, ThreadPool.Names.FETCH_SHARD_STORE, NodeStoreFilesMetaData.class);
         this.indicesService = indicesService;
         this.nodeEnv = nodeEnv;
     }
 
     @Override
-    public void list(ShardId shardId, String[] nodesIds, ActionListener<NodesStoreFilesMetaData> listener) {
-        execute(new Request(shardId, false, nodesIds), listener);
-    }
-
-    @Override
-    protected String[] resolveNodes(Request request, ClusterState clusterState) {
-        // default implementation may filter out non existent nodes. it's important to keep exactly the ids
-        // we were given for accounting on the caller
-        return request.nodesIds();
+    public void list(ShardId shardId, DiscoveryNode[] nodes, ActionListener<NodesStoreFilesMetaData> listener) {
+        execute(new Request(shardId, nodes), listener);
     }
 
     @Override
@@ -116,19 +107,6 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
 
     @Override
     protected NodeStoreFilesMetaData nodeOperation(NodeRequest request) {
-        if (request.unallocated) {
-            IndexService indexService = indicesService.indexService(request.shardId.getIndex());
-            if (indexService == null) {
-                return new NodeStoreFilesMetaData(clusterService.localNode(), null);
-            }
-            if (!indexService.hasShard(request.shardId.id())) {
-                return new NodeStoreFilesMetaData(clusterService.localNode(), null);
-            }
-        }
-        IndexMetaData metaData = clusterService.state().metaData().index(request.shardId.getIndex());
-        if (metaData == null) {
-            return new NodeStoreFilesMetaData(clusterService.localNode(), null);
-        }
         try {
             return new NodeStoreFilesMetaData(clusterService.localNode(), listStoreMetaData(request.shardId));
         } catch (IOException e) {
@@ -149,7 +127,7 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
                     store.incRef();
                     try {
                         exists = true;
-                        return new StoreFilesMetaData(true, shardId, store.getMetadataOrEmpty());
+                        return new StoreFilesMetaData(shardId, store.getMetadataOrEmpty());
                     } finally {
                         store.decRef();
                     }
@@ -158,14 +136,21 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
             // try and see if we an list unallocated
             IndexMetaData metaData = clusterService.state().metaData().index(shardId.getIndex());
             if (metaData == null) {
-                return new StoreFilesMetaData(false, shardId, Store.MetadataSnapshot.EMPTY);
+                // we may send this requests while processing the cluster state that recovered the index
+                // sometimes the request comes in before the local node processed that cluster state
+                // in such cases we can load it from disk
+                metaData = IndexMetaData.FORMAT.loadLatestState(logger, nodeEnv.indexPaths(shardId.getIndex()));
+            }
+            if (metaData == null) {
+                logger.trace("{} node doesn't have meta data for the requests index, responding with empty", shardId);
+                return new StoreFilesMetaData(shardId, Store.MetadataSnapshot.EMPTY);
             }
             final IndexSettings indexSettings = indexService != null ? indexService.getIndexSettings() : new IndexSettings(metaData, settings);
             final ShardPath shardPath = ShardPath.loadShardPath(logger, nodeEnv, shardId, indexSettings);
             if (shardPath == null) {
-                return new StoreFilesMetaData(false, shardId, Store.MetadataSnapshot.EMPTY);
+                return new StoreFilesMetaData(shardId, Store.MetadataSnapshot.EMPTY);
             }
-            return new StoreFilesMetaData(false, shardId, Store.readMetadataSnapshot(shardPath.resolveIndex(), shardId, logger));
+            return new StoreFilesMetaData(shardId, Store.readMetadataSnapshot(shardPath.resolveIndex(), shardId, logger));
         } finally {
             TimeValue took = new TimeValue(System.nanoTime() - startTimeNS, TimeUnit.NANOSECONDS);
             if (exists) {
@@ -182,28 +167,25 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
     }
 
     public static class StoreFilesMetaData implements Iterable<StoreFileMetaData>, Streamable {
-        // here also trasmit sync id, else recovery will not use sync id because of stupid gateway allocator every now and then...
-        private boolean allocated;
         private ShardId shardId;
         Store.MetadataSnapshot metadataSnapshot;
 
         StoreFilesMetaData() {
         }
 
-        public StoreFilesMetaData(boolean allocated, ShardId shardId, Store.MetadataSnapshot metadataSnapshot) {
-            this.allocated = allocated;
+        public StoreFilesMetaData(ShardId shardId, Store.MetadataSnapshot metadataSnapshot) {
             this.shardId = shardId;
             this.metadataSnapshot = metadataSnapshot;
         }
 
-        public boolean allocated() {
-            return allocated;
-        }
-
         public ShardId shardId() {
             return this.shardId;
         }
 
+        public boolean isEmpty() {
+            return metadataSnapshot.size() == 0;
+        }
+
         @Override
         public Iterator<StoreFileMetaData> iterator() {
             return metadataSnapshot.iterator();
@@ -225,14 +207,12 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
 
         @Override
         public void readFrom(StreamInput in) throws IOException {
-            allocated = in.readBoolean();
             shardId = ShardId.readShardId(in);
             this.metadataSnapshot = new Store.MetadataSnapshot(in);
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            out.writeBoolean(allocated);
             shardId.writeTo(out);
             metadataSnapshot.writeTo(out);
         }
@@ -243,6 +223,14 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
         public String syncId() {
             return metadataSnapshot.getSyncId();
         }
+
+        @Override
+        public String toString() {
+            return "StoreFilesMetaData{" +
+                ", shardId=" + shardId +
+                ", metadataSnapshot{size=" + metadataSnapshot.size() + ", syncId=" + metadataSnapshot.getSyncId() + "}" +
+                '}';
+        }
     }
 
 
@@ -250,35 +238,24 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
 
         private ShardId shardId;
 
-        private boolean unallocated;
-
         public Request() {
         }
 
-        public Request(ShardId shardId, boolean unallocated, Set<String> nodesIds) {
-            super(nodesIds.toArray(new String[nodesIds.size()]));
+        public Request(ShardId shardId, DiscoveryNode[] nodes) {
+            super(nodes);
             this.shardId = shardId;
-            this.unallocated = unallocated;
-        }
-
-        public Request(ShardId shardId, boolean unallocated, String... nodesIds) {
-            super(nodesIds);
-            this.shardId = shardId;
-            this.unallocated = unallocated;
         }
 
         @Override
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
             shardId = ShardId.readShardId(in);
-            unallocated = in.readBoolean();
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             shardId.writeTo(out);
-            out.writeBoolean(unallocated);
         }
     }
 
@@ -307,29 +284,24 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
 
         private ShardId shardId;
 
-        private boolean unallocated;
-
         public NodeRequest() {
         }
 
         NodeRequest(String nodeId, TransportNodesListShardStoreMetaData.Request request) {
             super(nodeId);
             this.shardId = request.shardId;
-            this.unallocated = request.unallocated;
         }
 
         @Override
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
             shardId = ShardId.readShardId(in);
-            unallocated = in.readBoolean();
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             shardId.writeTo(out);
-            out.writeBoolean(unallocated);
         }
     }
 
@@ -358,20 +330,18 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesAction<T
         @Override
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
-            if (in.readBoolean()) {
-                storeFilesMetaData = StoreFilesMetaData.readStoreFilesMetaData(in);
-            }
+            storeFilesMetaData = StoreFilesMetaData.readStoreFilesMetaData(in);
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            if (storeFilesMetaData == null) {
-                out.writeBoolean(false);
-            } else {
-                out.writeBoolean(true);
-                storeFilesMetaData.writeTo(out);
-            }
+            storeFilesMetaData.writeTo(out);
+        }
+
+        @Override
+        public String toString() {
+            return "[[" + getNode() + "][" + storeFilesMetaData + "]]";
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java
index 0121da2fa90..13d10bd0a0c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/AbstractRangeBuilder.java
@@ -20,8 +20,8 @@
 package org.elasticsearch.search.aggregations.bucket.range;
 
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamInputReader;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregator.Range;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
@@ -47,7 +47,7 @@ public abstract class AbstractRangeBuilder<AB extends AbstractRangeBuilder<AB, R
     /**
      * Read from a stream.
      */
-    protected AbstractRangeBuilder(StreamInput in, InternalRange.Factory<?, ?> rangeFactory, StreamInputReader<R> rangeReader)
+    protected AbstractRangeBuilder(StreamInput in, InternalRange.Factory<?, ?> rangeFactory, Writeable.Reader<R> rangeReader)
             throws IOException {
         super(in, rangeFactory.type(), rangeFactory.getValueSourceType(), rangeFactory.getValueType());
         this.rangeFactory = rangeFactory;
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
index 8f8c10c8dda..812691f064b 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
@@ -38,7 +38,6 @@ import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -261,17 +260,6 @@ public class TestTaskPlugin extends Plugin {
             return new NodesResponse(clusterService.getClusterName(), responses, failures);
         }
 
-        @Override
-        protected String[] filterNodeIds(DiscoveryNodes nodes, String[] nodesIds) {
-            List<String> list = new ArrayList<>();
-            for (String node : nodesIds) {
-                if (nodes.nodeExists(node)) {
-                    list.add(node);
-                }
-            }
-            return list.toArray(new String[list.size()]);
-        }
-
         @Override
         protected NodeRequest newNodeRequest(String nodeId, NodesRequest request) {
             return new NodeRequest(request, nodeId, request.getShouldBlock());
diff --git a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
index 626850fd119..a15f89bced4 100644
--- a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
@@ -96,7 +96,7 @@ public class TransportNodesActionTests extends ESTestCase {
         TestNodesRequest request = new TestNodesRequest(finalNodesIds);
         action.new AsyncAction(null, request, new PlainActionFuture<>()).start();
         Map<String, List<CapturingTransport.CapturedRequest>> capturedRequests = transport.getCapturedRequestsByTargetNodeAndClear();
-        assertEquals(clusterService.state().nodes().resolveNodesIds(finalNodesIds).length, capturedRequests.size());
+        assertEquals(clusterService.state().nodes().resolveNodes(finalNodesIds).length, capturedRequests.size());
     }
 
     public void testNewResponseNullArray() {
@@ -129,9 +129,9 @@ public class TransportNodesActionTests extends ESTestCase {
         assertTrue(failures.containsAll(response.failures()));
     }
 
-    public void testFiltering() throws Exception {
-        TransportNodesAction action = getFilteringTestTransportNodesAction(transportService);
-        TestNodesRequest request = new TestNodesRequest();
+    public void testCustomResolving() throws Exception {
+        TransportNodesAction action = getDataNodesOnlyTransportNodesAction(transportService);
+        TestNodesRequest request = new TestNodesRequest(randomBoolean() ? null : generateRandomStringArray(10, 5, false, true));
         PlainActionFuture<TestNodesResponse> listener = new PlainActionFuture<>();
         action.new AsyncAction(null, request, listener).start();
         Map<String, List<CapturingTransport.CapturedRequest>> capturedRequests = transport.getCapturedRequestsByTargetNodeAndClear();
@@ -221,8 +221,8 @@ public class TransportNodesActionTests extends ESTestCase {
         );
     }
 
-    public FilteringTestTransportNodesAction getFilteringTestTransportNodesAction(TransportService transportService) {
-        return new FilteringTestTransportNodesAction(
+    public DataNodesOnlyTransportNodesAction getDataNodesOnlyTransportNodesAction(TransportService transportService) {
+        return new DataNodesOnlyTransportNodesAction(
             Settings.EMPTY,
             THREAD_POOL,
             clusterService,
@@ -276,18 +276,18 @@ public class TransportNodesActionTests extends ESTestCase {
         }
     }
 
-    private static class FilteringTestTransportNodesAction
+    private static class DataNodesOnlyTransportNodesAction
         extends TestTransportNodesAction {
 
-        FilteringTestTransportNodesAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService
+        DataNodesOnlyTransportNodesAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, TransportService
             transportService, ActionFilters actionFilters, Supplier<TestNodesRequest> request,
                                           Supplier<TestNodeRequest> nodeRequest, String nodeExecutor) {
             super(settings, threadPool, clusterService, transportService, actionFilters, request, nodeRequest, nodeExecutor);
         }
 
         @Override
-        protected String[] filterNodeIds(DiscoveryNodes nodes, String[] nodesIds) {
-            return nodes.getDataNodes().keys().toArray(String.class);
+        protected void resolveRequest(TestNodesRequest request, ClusterState clusterState) {
+            request.setConcreteNodes(clusterState.nodes().getDataNodes().values().toArray(DiscoveryNode.class));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java
index b0942ab401c..984cd31b7a0 100644
--- a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java
@@ -91,7 +91,7 @@ public class DiscoveryNodesTests extends ESTestCase {
             expectedNodeIdsSet.add(discoveryNode.getId());
         }
 
-        String[] resolvedNodesIds = discoveryNodes.resolveNodesIds(nodeSelectors.toArray(new String[nodeSelectors.size()]));
+        String[] resolvedNodesIds = discoveryNodes.resolveNodes(nodeSelectors.toArray(new String[nodeSelectors.size()]));
         Arrays.sort(resolvedNodesIds);
         String[] expectedNodesIds = expectedNodeIdsSet.toArray(new String[expectedNodeIdsSet.size()]);
         Arrays.sort(expectedNodesIds);
diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
index 72f933462e0..aa6016774b0 100644
--- a/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
+++ b/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
@@ -24,9 +24,7 @@ import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.ByteArrayInputStream;
-import java.io.FilterInputStream;
 import java.io.IOException;
-import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -35,6 +33,8 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
+import static org.hamcrest.Matchers.equalTo;
+
 public class StreamTests extends ESTestCase {
     public void testRandomVLongSerialization() throws IOException {
         for (int i = 0; i < 1024; i++) {
@@ -121,4 +121,62 @@ public class StreamTests extends ESTestCase {
         streamInput.readBytes(new byte[bytesToRead], 0, bytesToRead);
         assertEquals(streamInput.available(), length - bytesToRead);
     }
+
+    public void testWritableArrays() throws IOException {
+
+        final String[] strings = generateRandomStringArray(10, 10, false, true);
+        WriteableString[] sourceArray = Arrays.stream(strings).<WriteableString>map(WriteableString::new).toArray(WriteableString[]::new);
+        WriteableString[] targetArray;
+        BytesStreamOutput out = new BytesStreamOutput();
+
+        if (randomBoolean()) {
+            if (randomBoolean()) {
+                sourceArray = null;
+            }
+            out.writeOptionalArray(sourceArray);
+            targetArray = out.bytes().streamInput().readOptionalArray(WriteableString::new, WriteableString[]::new);
+        } else {
+            out.writeArray(sourceArray);
+            targetArray = out.bytes().streamInput().readArray(WriteableString::new, WriteableString[]::new);
+        }
+
+        assertThat(targetArray, equalTo(sourceArray));
+    }
+
+    final static class WriteableString implements Writeable {
+        final String string;
+
+        public WriteableString(String string) {
+            this.string = string;
+        }
+
+        public WriteableString(StreamInput in) throws IOException {
+            this(in.readString());
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+
+            WriteableString that = (WriteableString) o;
+
+            return string.equals(that.string);
+
+        }
+
+        @Override
+        public int hashCode() {
+            return string.hashCode();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(string);
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
index 109c59d845a..0187bb28f36 100644
--- a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
+++ b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
@@ -371,6 +371,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
      * This test isolates the master from rest of the cluster, waits for a new master to be elected, restores the partition
      * and verifies that all node agree on the new cluster state
      */
+    @TestLogging("_root:DEBUG,cluster.service:TRACE,gateway:TRACE,indices.store:TRACE")
     public void testIsolateMasterAndVerifyClusterStateConsensus() throws Exception {
         final List<String> nodes = startCluster(3);
 
diff --git a/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java b/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
index 9ce2aa44ab6..948f4820439 100644
--- a/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
@@ -270,8 +270,9 @@ public class AsyncShardFetchTests extends ESTestCase {
         }
 
         @Override
-        protected void asyncFetch(final ShardId shardId, String[] nodesIds) {
-            for (final String nodeId : nodesIds) {
+        protected void asyncFetch(final ShardId shardId, DiscoveryNode[] nodes) {
+            for (final DiscoveryNode node : nodes) {
+                final String nodeId = node.getId();
                 threadPool.generic().execute(new Runnable() {
                     @Override
                     public void run() {
diff --git a/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java b/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java
index aaa29ad1970..59f01f56ce1 100644
--- a/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java
+++ b/core/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java
@@ -565,7 +565,7 @@ public class RecoveryFromGatewayIT extends ESIntegTestCase {
 
         TransportNodesListGatewayStartedShards.NodesGatewayStartedShards response;
         response = internalCluster().getInstance(TransportNodesListGatewayStartedShards.class)
-            .execute(new TransportNodesListGatewayStartedShards.Request(shardId, new String[]{node.getId()}))
+            .execute(new TransportNodesListGatewayStartedShards.Request(shardId, new DiscoveryNode[]{node}))
             .get();
 
         assertThat(response.getNodes(), hasSize(1));
diff --git a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
index 99d6c56c88d..b417553a609 100644
--- a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java
@@ -36,11 +36,11 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
-import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
+import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.set.Sets;
@@ -118,8 +118,8 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
     public void testSimpleFullMatchAllocation() {
         RoutingAllocation allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders());
         DiscoveryNode nodeToMatch = randomBoolean() ? node2 : node3;
-        testAllocator.addData(node1, true, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"))
-                .addData(nodeToMatch, false, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"));
+        testAllocator.addData(node1, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"))
+                .addData(nodeToMatch, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"));
         testAllocator.allocateUnassigned(allocation);
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1));
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo(nodeToMatch.getId()));
@@ -131,8 +131,8 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
     public void testSyncIdMatch() {
         RoutingAllocation allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders());
         DiscoveryNode nodeToMatch = randomBoolean() ? node2 : node3;
-        testAllocator.addData(node1, true, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"))
-                .addData(nodeToMatch, false, "MATCH", new StoreFileMetaData("file1", 10, "NO_MATCH_CHECKSUM"));
+        testAllocator.addData(node1, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"))
+                .addData(nodeToMatch, "MATCH", new StoreFileMetaData("file1", 10, "NO_MATCH_CHECKSUM"));
         testAllocator.allocateUnassigned(allocation);
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1));
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo(nodeToMatch.getId()));
@@ -144,8 +144,8 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
     public void testFileChecksumMatch() {
         RoutingAllocation allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders());
         DiscoveryNode nodeToMatch = randomBoolean() ? node2 : node3;
-        testAllocator.addData(node1, true, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"))
-                .addData(nodeToMatch, false, "NO_MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"));
+        testAllocator.addData(node1, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"))
+                .addData(nodeToMatch, "NO_MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"));
         testAllocator.allocateUnassigned(allocation);
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1));
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo(nodeToMatch.getId()));
@@ -159,7 +159,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
      */
     public void testNoPrimaryData() {
         RoutingAllocation allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders());
-        testAllocator.addData(node2, true, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"));
+        testAllocator.addData(node2, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"));
         testAllocator.allocateUnassigned(allocation);
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(1));
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).get(0).shardId(), equalTo(shardId));
@@ -171,7 +171,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
      */
     public void testNoDataForReplicaOnAnyNode() {
         RoutingAllocation allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders());
-        testAllocator.addData(node1, true, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"));
+        testAllocator.addData(node1, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"));
         testAllocator.allocateUnassigned(allocation);
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(1));
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).get(0).shardId(), equalTo(shardId));
@@ -183,8 +183,8 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
      */
     public void testNoMatchingFilesForReplicaOnAnyNode() {
         RoutingAllocation allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders());
-        testAllocator.addData(node1, true, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"))
-                .addData(node2, false, "NO_MATCH", new StoreFileMetaData("file1", 10, "NO_MATCH_CHECKSUM"));
+        testAllocator.addData(node1, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"))
+                .addData(node2, "NO_MATCH", new StoreFileMetaData("file1", 10, "NO_MATCH_CHECKSUM"));
         testAllocator.allocateUnassigned(allocation);
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(1));
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).get(0).shardId(), equalTo(shardId));
@@ -196,8 +196,8 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
      */
     public void testNoOrThrottleDecidersRemainsInUnassigned() {
         RoutingAllocation allocation = onePrimaryOnNode1And1Replica(randomBoolean() ? noAllocationDeciders() : throttleAllocationDeciders());
-        testAllocator.addData(node1, true, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"))
-                .addData(node2, false, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"));
+        testAllocator.addData(node1, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"))
+                .addData(node2, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"));
         testAllocator.allocateUnassigned(allocation);
         assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1));
         assertThat(allocation.routingNodes().unassigned().ignored().get(0).shardId(), equalTo(shardId));
@@ -209,7 +209,8 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
      */
     public void testThrottleWhenAllocatingToMatchingNode() {
         RoutingAllocation allocation = onePrimaryOnNode1And1Replica(new AllocationDeciders(Settings.EMPTY,
-                new AllocationDecider[]{new TestAllocateDecision(Decision.YES), new AllocationDecider(Settings.EMPTY) {
+            new AllocationDecider[]{new TestAllocateDecision(Decision.YES), new SameShardAllocationDecider(Settings.EMPTY),
+                new AllocationDecider(Settings.EMPTY) {
                     @Override
                     public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
                         if (node.node().equals(node2)) {
@@ -218,8 +219,8 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
                         return Decision.YES;
                     }
                 }}));
-        testAllocator.addData(node1, true, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"))
-                .addData(node2, false, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"));
+        testAllocator.addData(node1, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"))
+                .addData(node2, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"));
         testAllocator.allocateUnassigned(allocation);
         assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1));
         assertThat(allocation.routingNodes().unassigned().ignored().get(0).shardId(), equalTo(shardId));
@@ -228,10 +229,10 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
     public void testDelayedAllocation() {
         RoutingAllocation allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders(),
                 Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), TimeValue.timeValueHours(1)).build(), UnassignedInfo.Reason.NODE_LEFT);
-        testAllocator.addData(node1, true, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"));
+        testAllocator.addData(node1, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"));
         if (randomBoolean()) {
             // we sometime return empty list of files, make sure we test this as well
-            testAllocator.addData(node2, false, null);
+            testAllocator.addData(node2, null);
         }
         boolean changed = testAllocator.allocateUnassigned(allocation);
         assertThat(changed, equalTo(false));
@@ -240,7 +241,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
 
         allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders(),
                 Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), TimeValue.timeValueHours(1)).build(), UnassignedInfo.Reason.NODE_LEFT);
-        testAllocator.addData(node2, false, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"));
+        testAllocator.addData(node2, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"));
         changed = testAllocator.allocateUnassigned(allocation);
         assertThat(changed, equalTo(true));
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1));
@@ -249,9 +250,9 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
 
     public void testCancelRecoveryBetterSyncId() {
         RoutingAllocation allocation = onePrimaryOnNode1And1ReplicaRecovering(yesAllocationDeciders());
-        testAllocator.addData(node1, true, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"))
-                .addData(node2, false, "NO_MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"))
-                .addData(node3, false, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"));
+        testAllocator.addData(node1, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"))
+                .addData(node2, "NO_MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"))
+                .addData(node3, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"));
         boolean changed = testAllocator.processExistingRecoveries(allocation);
         assertThat(changed, equalTo(true));
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(1));
@@ -260,9 +261,9 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
 
     public void testNotCancellingRecoveryIfSyncedOnExistingRecovery() {
         RoutingAllocation allocation = onePrimaryOnNode1And1ReplicaRecovering(yesAllocationDeciders());
-        testAllocator.addData(node1, true, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"))
-                .addData(node2, false, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"))
-                .addData(node3, false, randomBoolean() ? "MATCH" : "NO_MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"));
+        testAllocator.addData(node1, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"))
+                .addData(node2, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"))
+                .addData(node3, randomBoolean() ? "MATCH" : "NO_MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"));
         boolean changed = testAllocator.processExistingRecoveries(allocation);
         assertThat(changed, equalTo(false));
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(0));
@@ -270,8 +271,8 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
 
     public void testNotCancellingRecovery() {
         RoutingAllocation allocation = onePrimaryOnNode1And1ReplicaRecovering(yesAllocationDeciders());
-        testAllocator.addData(node1, true, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"))
-                .addData(node2, false, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"));
+        testAllocator.addData(node1, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"))
+                .addData(node2, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM"));
         boolean changed = testAllocator.processExistingRecoveries(allocation);
         assertThat(changed, equalTo(false));
         assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(0));
@@ -352,7 +353,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
             return fetchDataCalled.getAndSet(false);
         }
 
-        public TestAllocator addData(DiscoveryNode node, boolean allocated, String syncId, StoreFileMetaData... files) {
+        public TestAllocator addData(DiscoveryNode node, String syncId, StoreFileMetaData... files) {
             if (data == null) {
                 data = new HashMap<>();
             }
@@ -364,7 +365,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase {
             if (syncId != null) {
                 commitData.put(Engine.SYNC_COMMIT_ID, syncId);
             }
-            data.put(node, new TransportNodesListShardStoreMetaData.StoreFilesMetaData(allocated, shardId,
+            data.put(node, new TransportNodesListShardStoreMetaData.StoreFilesMetaData(shardId,
                     new Store.MetadataSnapshot(unmodifiableMap(filesAsMap), unmodifiableMap(commitData), randomInt())));
             return this;
         }
diff --git a/core/src/test/java/org/elasticsearch/index/store/StoreTests.java b/core/src/test/java/org/elasticsearch/index/store/StoreTests.java
index 4f3e68a9115..e005fa400ef 100644
--- a/core/src/test/java/org/elasticsearch/index/store/StoreTests.java
+++ b/core/src/test/java/org/elasticsearch/index/store/StoreTests.java
@@ -933,7 +933,7 @@ public class StoreTests extends ESTestCase {
 
     public void testStreamStoreFilesMetaData() throws Exception {
         Store.MetadataSnapshot metadataSnapshot = createMetaDataSnapshot();
-        TransportNodesListShardStoreMetaData.StoreFilesMetaData outStoreFileMetaData = new TransportNodesListShardStoreMetaData.StoreFilesMetaData(randomBoolean(), new ShardId("test", "_na_", 0),metadataSnapshot);
+        TransportNodesListShardStoreMetaData.StoreFilesMetaData outStoreFileMetaData = new TransportNodesListShardStoreMetaData.StoreFilesMetaData(new ShardId("test", "_na_", 0),metadataSnapshot);
         ByteArrayOutputStream outBuffer = new ByteArrayOutputStream();
         OutputStreamStreamOutput out = new OutputStreamStreamOutput(outBuffer);
         org.elasticsearch.Version targetNodeVersion = randomVersion(random());
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java
index 7cbb7e819c4..5704a178f48 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java
@@ -38,6 +38,7 @@ import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.Decision;
+import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -157,7 +158,8 @@ public abstract class ESAllocationTestCase extends ESTestCase {
     }
 
     protected static AllocationDeciders yesAllocationDeciders() {
-        return new AllocationDeciders(Settings.EMPTY, new AllocationDecider[] {new TestAllocateDecision(Decision.YES)});
+        return new AllocationDeciders(Settings.EMPTY, new AllocationDecider[] {new TestAllocateDecision(Decision.YES),
+            new SameShardAllocationDecider(Settings.EMPTY)});
     }
 
     protected static AllocationDeciders noAllocationDeciders() {
@@ -165,7 +167,8 @@ public abstract class ESAllocationTestCase extends ESTestCase {
     }
 
     protected static AllocationDeciders throttleAllocationDeciders() {
-        return new AllocationDeciders(Settings.EMPTY, new AllocationDecider[] {new TestAllocateDecision(Decision.THROTTLE)});
+        return new AllocationDeciders(Settings.EMPTY, new AllocationDecider[] {new TestAllocateDecision(Decision.THROTTLE),
+            new SameShardAllocationDecider(Settings.EMPTY)});
     }
 
     public static class TestAllocateDecision extends AllocationDecider {

From 108ba2307360463c21a932fdaca75f0559b2b75a Mon Sep 17 00:00:00 2001
From: Colin Goodheart-Smithe <colings86@users.noreply.github.com>
Date: Mon, 27 Jun 2016 10:46:33 +0100
Subject: [PATCH 1073/1311] Pass resolved extended bounds to unmapped histogram
 aggregator

Previous to this change the unresolved extended bounds was passed into the histogram aggregator which meant extendedbounds.min and extendedbounds.max was passed through as null. This had two effects on the histogram aggregator:

1. If the histogram aggregator was unmapped across all shards, the reduce phase would not add buckets for the extended bounds and the response would contain zero buckets
2. If the histogram aggregator was not unmapped in some shards, the reduce phase might sometimes chose to reduce based on the unmapped shard response and therefore the extended bounds would be ignored.

This change resolves the extended bounds in the unmapped case and solves the above two issues.

Closes #19009
---
 .../AbstractHistogramAggregatorFactory.java   |  9 ++-
 .../messy/tests/HistogramTests.java           | 78 +++++++++++++++++++
 2 files changed, 84 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramAggregatorFactory.java
index a9c3d24a429..d44f3133789 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AbstractHistogramAggregatorFactory.java
@@ -69,9 +69,7 @@ public abstract class AbstractHistogramAggregatorFactory<AF extends AbstractHist
     @Override
     protected Aggregator createUnmapped(Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
             throws IOException {
-        Rounding rounding = createRounding();
-        return new HistogramAggregator(name, factories, rounding, order, keyed, minDocCount, extendedBounds, null, config.format(),
-                histogramFactory, context, parent, pipelineAggregators, metaData);
+        return createAggregator(null, parent, pipelineAggregators, metaData);
     }
 
     protected Rounding createRounding() {
@@ -92,6 +90,11 @@ public abstract class AbstractHistogramAggregatorFactory<AF extends AbstractHist
         if (collectsFromSingleBucket == false) {
             return asMultiBucketAggregator(this, context, parent);
         }
+        return createAggregator(valuesSource, parent, pipelineAggregators, metaData);
+    }
+
+    private Aggregator createAggregator(ValuesSource.Numeric valuesSource, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
+            Map<String, Object> metaData) throws IOException {
         Rounding rounding = createRounding();
         // we need to round the bounds given by the user and we have to do it
         // for every aggregator we create
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java
index 5ada804954a..45274f44401 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.messy.tests;
 import com.carrotsearch.hppc.LongHashSet;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.groovy.GroovyPlugin;
@@ -825,6 +826,83 @@ public class HistogramTests extends ESIntegTestCase {
         }
     }
 
+    public void testEmptyWithExtendedBounds() throws Exception {
+        int lastDataBucketKey = (numValueBuckets - 1) * interval;
+
+        // randomizing the number of buckets on the min bound
+        // (can sometimes fall within the data range, but more frequently will fall before the data range)
+        int addedBucketsLeft = randomIntBetween(0, numValueBuckets);
+        long boundsMinKey = addedBucketsLeft * interval;
+        if (frequently()) {
+            boundsMinKey = -boundsMinKey;
+        } else {
+            addedBucketsLeft = 0;
+        }
+        long boundsMin = boundsMinKey + randomIntBetween(0, interval - 1);
+
+        // randomizing the number of buckets on the max bound
+        // (can sometimes fall within the data range, but more frequently will fall after the data range)
+        int addedBucketsRight = randomIntBetween(0, numValueBuckets);
+        long boundsMaxKeyDelta = addedBucketsRight * interval;
+        if (rarely()) {
+            addedBucketsRight = 0;
+            boundsMaxKeyDelta = -boundsMaxKeyDelta;
+        }
+        long boundsMaxKey = lastDataBucketKey + boundsMaxKeyDelta;
+        long boundsMax = boundsMaxKey + randomIntBetween(0, interval - 1);
+
+
+        // it could be that the random bounds.min we chose ended up greater than bounds.max - this should cause an
+        // error
+        boolean invalidBoundsError = boundsMin > boundsMax;
+
+        // constructing the newly expected bucket list
+        int bucketsCount = numValueBuckets + addedBucketsLeft + addedBucketsRight;
+        long[] extendedValueCounts = new long[bucketsCount];
+        System.arraycopy(valueCounts, 0, extendedValueCounts, addedBucketsLeft, valueCounts.length);
+
+        SearchResponse response = null;
+        try {
+            response = client().prepareSearch("idx")
+                    .setQuery(QueryBuilders.termQuery("foo", "bar"))
+                    .addAggregation(histogram("histo")
+                            .field(SINGLE_VALUED_FIELD_NAME)
+                            .interval(interval)
+                            .minDocCount(0)
+                            .extendedBounds(new ExtendedBounds(boundsMin, boundsMax)))
+                    .execute().actionGet();
+
+            if (invalidBoundsError) {
+                fail("Expected an exception to be thrown when bounds.min is greater than bounds.max");
+                return;
+            }
+
+        } catch (Exception e) {
+            if (invalidBoundsError) {
+                // expected
+                return;
+            } else {
+                throw e;
+            }
+        }
+        assertSearchResponse(response);
+
+        Histogram histo = response.getAggregations().get("histo");
+        assertThat(histo, notNullValue());
+        assertThat(histo.getName(), equalTo("histo"));
+        List<? extends Bucket> buckets = histo.getBuckets();
+        assertThat(buckets.size(), equalTo(bucketsCount));
+
+        long key = Math.min(boundsMinKey, 0);
+        for (int i = 0; i < bucketsCount; i++) {
+            Histogram.Bucket bucket = buckets.get(i);
+            assertThat(bucket, notNullValue());
+            assertThat(((Number) bucket.getKey()).longValue(), equalTo(key));
+            assertThat(bucket.getDocCount(), equalTo(0L));
+            key += interval;
+        }
+    }
+
     /**
      * see issue #9634, negative interval in histogram should raise exception
      */

From 6fc1a229770af82a1424eecca03faf57ccacd607 Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Mon, 27 Jun 2016 09:55:16 -0400
Subject: [PATCH 1074/1311] cutover some docs to painless

---
 docs/java-api/indexed-scripts.asciidoc        |    8 +-
 docs/java-api/query-dsl/script-query.asciidoc |   10 +-
 .../diversified-sampler-aggregation.asciidoc  |    9 +-
 .../bucket/range-aggregation.asciidoc         |   16 +-
 .../metrics/avg-aggregation.asciidoc          |   14 +-
 .../metrics/cardinality-aggregation.asciidoc  |    7 +-
 .../extendedstats-aggregation.asciidoc        |   14 +-
 .../metrics/max-aggregation.asciidoc          |   14 +-
 .../metrics/min-aggregation.asciidoc          |   14 +-
 .../metrics/percentile-aggregation.asciidoc   |    5 +-
 .../percentile-rank-aggregation.asciidoc      |    5 +-
 .../metrics/stats-aggregation.asciidoc        |   14 +-
 .../metrics/sum-aggregation.asciidoc          |   16 +-
 .../metrics/tophits-aggregation.asciidoc      |    5 +-
 .../metrics/valuecount-aggregation.asciidoc   |   11 +-
 docs/reference/docs/bulk.asciidoc             |    2 +-
 docs/reference/docs/reindex.asciidoc          |    3 +-
 docs/reference/docs/update-by-query.asciidoc  |    3 +-
 docs/reference/docs/update.asciidoc           |   12 +-
 .../mapping/fields/field-names-field.asciidoc |    7 +-
 .../mapping/fields/id-field.asciidoc          |    8 +-
 .../mapping/fields/index-field.asciidoc       |    7 +-
 .../mapping/fields/parent-field.asciidoc      |    7 +-
 .../mapping/fields/routing-field.asciidoc     |    7 +-
 .../mapping/fields/type-field.asciidoc        |    7 +-
 .../mapping/fields/uid-field.asciidoc         |    8 +-
 docs/reference/mapping/types/boolean.asciidoc |    6 +-
 .../scripting/painless-syntax.asciidoc        |    9 +
 .../query-dsl/function-score-query.asciidoc   |    9 +-
 .../reference/query-dsl/script-query.asciidoc |    8 +-
 .../reference/search/request/rescore.asciidoc |    5 +-
 .../search/request/script-fields.asciidoc     |    6 +-
 docs/reference/search/request/sort.asciidoc   |    3 +-
 .../src/main/antlr/PainlessLexer.g4           |    1 +
 .../src/main/antlr/PainlessLexer.tokens       |  258 ++--
 .../src/main/antlr/PainlessParser.g4          |    1 +
 .../src/main/antlr/PainlessParser.tokens      |  258 ++--
 .../org/elasticsearch/painless/Locals.java    |    6 +-
 .../painless/SimpleChecksAdapter.java         |   60 +
 .../painless/WriterConstants.java             |    2 +
 .../painless/antlr/PainlessLexer.java         |  452 +++---
 .../painless/antlr/PainlessParser.java        | 1217 +++++++++--------
 .../antlr/PainlessParserBaseVisitor.java      |    7 +
 .../painless/antlr/PainlessParserVisitor.java |    7 +
 .../elasticsearch/painless/antlr/Walker.java  |   12 +
 .../painless/node/SFunction.java              |    1 +
 .../elasticsearch/painless/node/SSource.java  |   10 +-
 .../painless/BasicStatementTests.java         |   26 +
 .../elasticsearch/painless/LambdaTests.java   |    3 +-
 49 files changed, 1451 insertions(+), 1149 deletions(-)
 create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/SimpleChecksAdapter.java

diff --git a/docs/java-api/indexed-scripts.asciidoc b/docs/java-api/indexed-scripts.asciidoc
index 45d19ae662d..a1259649a77 100644
--- a/docs/java-api/indexed-scripts.asciidoc
+++ b/docs/java-api/indexed-scripts.asciidoc
@@ -8,20 +8,20 @@ and delete indexed scripts and templates.
 [source,java]
 --------------------------------------------------
 PutIndexedScriptResponse = client.preparePutIndexedScript()
-			 .setScriptLang("groovy")		
+			 .setScriptLang("painless")
 			 .setId("script1") 
 			 .setSource("script", "_score * doc['my_numeric_field'].value")
 			 .execute()
 			 .actionGet();
 
 GetIndexedScriptResponse = client.prepareGetIndexedScript()
-			    .setScriptLang("groovy")
+			    .setScriptLang("painless")
 			    .setId("script1")
 			    .execute()
 			    .actionGet();
 
 DeleteIndexedScriptResponse = client.prepareDeleteIndexedScript()
-			    .setScriptLang("groovy")
+			    .setScriptLang("painless")
 			    .setId("script1")
 			    .execute()
 			    .actionGet();
@@ -33,4 +33,4 @@ To store templates simply use "mustache" for the scriptLang.
 
 The API allows one to set the language of the indexed script being 
 interacted with. If one is not provided the default scripting language
-will be used.
\ No newline at end of file
+will be used.
diff --git a/docs/java-api/query-dsl/script-query.asciidoc b/docs/java-api/query-dsl/script-query.asciidoc
index 534c803ab08..33786b693d2 100644
--- a/docs/java-api/query-dsl/script-query.asciidoc
+++ b/docs/java-api/query-dsl/script-query.asciidoc
@@ -12,11 +12,11 @@ QueryBuilder qb = scriptQuery(
 <1> inlined script
 
 
-If you have stored on each data node a script named `mygroovyscript.groovy` with:
+If you have stored on each data node a script named `myscript.painless` with:
 
-[source,groovy]
+[source,js]
 --------------------------------------------------
-doc['num1'].value > param1
+doc['num1'].value > params.param1
 --------------------------------------------------
 
 You can use it then with:
@@ -25,9 +25,9 @@ You can use it then with:
 --------------------------------------------------
 QueryBuilder qb = scriptQuery(
     new Script(
-        "mygroovyscript",                    <1>
+        "myscript",                          <1>
         ScriptType.FILE,       <2>
-        "groovy",                            <3>
+        "painless",                          <3>
         ImmutableMap.of("param1", 5))        <4>
 );
 --------------------------------------------------
diff --git a/docs/reference/aggregations/bucket/diversified-sampler-aggregation.asciidoc b/docs/reference/aggregations/bucket/diversified-sampler-aggregation.asciidoc
index 1a3180962b0..0307840d685 100644
--- a/docs/reference/aggregations/bucket/diversified-sampler-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/diversified-sampler-aggregation.asciidoc
@@ -115,8 +115,11 @@ Controlling diversity using a script:
 {
     "aggs" : {
         "sample" : {
-            "diverisfied_sampler" : {
-                "script" : "doc['author'].value + '/' + doc['genre'].value"
+            "diversified_sampler" : {
+                "script" : { 
+                    "lang" : "painless",
+                    "inline" : "doc['author'].value + '/' + doc['genre'].value"
+                }
             }
         }
     }
@@ -151,4 +154,4 @@ The de-duplication logic in the diversify settings applies only at a shard level
 
 ===== No specialized syntax for geo/date fields
 Currently the syntax for defining the diversifying values is defined by a choice of `field` or `script` - there is no added syntactical sugar for expressing geo or date units such as "1w" (1 week).
-This support may be added in a later release and users will currently have to create these sorts of values using a script.
\ No newline at end of file
+This support may be added in a later release and users will currently have to create these sorts of values using a script.
diff --git a/docs/reference/aggregations/bucket/range-aggregation.asciidoc b/docs/reference/aggregations/bucket/range-aggregation.asciidoc
index 44760fc728f..efcbd9715e1 100644
--- a/docs/reference/aggregations/bucket/range-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/range-aggregation.asciidoc
@@ -134,7 +134,10 @@ It is also possible to customize the key for each range:
     "aggs" : {
         "price_ranges" : {
             "range" : {
-                "script" : "doc['price'].value",
+                "script" : {
+                    "lang": "painless",
+                    "inline": "doc['price'].value"
+                },
                 "ranges" : [
                     { "to" : 50 },
                     { "from" : 50, "to" : 100 },
@@ -146,7 +149,7 @@ It is also possible to customize the key for each range:
 }
 --------------------------------------------------
 
-This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a file script use the following syntax:
+This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a file script use the following syntax:
 
 [source,js]
 --------------------------------------------------
@@ -184,9 +187,12 @@ Lets say the product prices are in USD but we would like to get the price ranges
         "price_ranges" : {
             "range" : {
                 "field" : "price",
-                "script" : "_value * conversion_rate",
-                "params" : {
-                    "conversion_rate" : 0.8
+                "script" : {
+                    "lang": "painless",
+                    "inline": "_value * params.conversion_rate",
+                    "params" : {
+                        "conversion_rate" : 0.8
+                    }
                 },
                 "ranges" : [
                     { "to" : 35 },
diff --git a/docs/reference/aggregations/metrics/avg-aggregation.asciidoc b/docs/reference/aggregations/metrics/avg-aggregation.asciidoc
index f81cd3eee33..9967f4c9d9e 100644
--- a/docs/reference/aggregations/metrics/avg-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/avg-aggregation.asciidoc
@@ -42,12 +42,19 @@ Computing the average grade based on a script:
     ...,
 
     "aggs" : {
-        "avg_grade" : { "avg" : { "script" : "doc['grade'].value" } }
+        "avg_grade" : { 
+            "avg" : { 
+                "script" : {
+                    "inline" : "doc['grade'].value",
+                    "lang" : "painless"
+                }
+            }
+        }
     }
 }
 --------------------------------------------------
 
-This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a file script use the following syntax:
+This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a file script use the following syntax:
 
 [source,js]
 --------------------------------------------------
@@ -86,7 +93,8 @@ It turned out that the exam was way above the level of the students and a grade
                 "avg" : {
                     "field" : "grade",
                     "script" : {
-                        "inline": "_value * correction",
+                        "lang": "painless",
+                        "inline": "_value * params.correction",
                         "params" : {
                             "correction" : 1.2
                         }
diff --git a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc
index 1eb0c08772f..9c4ee59cccf 100644
--- a/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/cardinality-aggregation.asciidoc
@@ -104,14 +104,17 @@ however since hashes need to be computed on the fly.
     "aggs" : {
         "author_count" : {
             "cardinality" : {
-                "script": "doc['author.first_name'].value + ' ' + doc['author.last_name'].value"
+                "script": {
+                    "lang": "painless",
+                    "inline": "doc['author.first_name'].value + ' ' + doc['author.last_name'].value"
+                }
             }
         }
     }
 }
 --------------------------------------------------
 
-This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a file script use the following syntax:
+This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a file script use the following syntax:
 
 [source,js]
 --------------------------------------------------
diff --git a/docs/reference/aggregations/metrics/extendedstats-aggregation.asciidoc b/docs/reference/aggregations/metrics/extendedstats-aggregation.asciidoc
index 30a5acf6809..0e324089dc7 100644
--- a/docs/reference/aggregations/metrics/extendedstats-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/extendedstats-aggregation.asciidoc
@@ -86,12 +86,19 @@ Computing the grades stats based on a script:
     ...,
 
     "aggs" : {
-        "grades_stats" : { "extended_stats" : { "script" : "doc['grade'].value" } }
+        "grades_stats" : { 
+            "extended_stats" : { 
+                "script" : { 
+                    "inline" : "doc['grade'].value",
+                    "lang" : "painless"
+                 }
+             }
+         }
     }
 }
 --------------------------------------------------
 
-This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a file script use the following syntax:
+This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a file script use the following syntax:
 
 [source,js]
 --------------------------------------------------
@@ -130,7 +137,8 @@ It turned out that the exam was way above the level of the students and a grade
                 "extended_stats" : {
                     "field" : "grade",
                     "script" : {
-                        "inline": "_value * correction",
+                        "lang" : "painless",
+                        "inline": "_value * params.correction",
                         "params" : {
                             "correction" : 1.2
                         }
diff --git a/docs/reference/aggregations/metrics/max-aggregation.asciidoc b/docs/reference/aggregations/metrics/max-aggregation.asciidoc
index 2a641fda5dc..8cfc0bd998e 100644
--- a/docs/reference/aggregations/metrics/max-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/max-aggregation.asciidoc
@@ -39,12 +39,19 @@ Computing the max price value across all document, this time using a script:
 --------------------------------------------------
 {
     "aggs" : {
-        "max_price" : { "max" : { "script" : "doc['price'].value" } }
+        "max_price" : { 
+            "max" : { 
+                "script" : {
+                    "inline" : "doc['price'].value",
+                    "lang" : "painless"
+                }
+            }
+        }
     }
 }
 --------------------------------------------------
 
-This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a file script use the following syntax:
+This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a file script use the following syntax:
 
 [source,js]
 --------------------------------------------------
@@ -78,7 +85,8 @@ Let's say that the prices of the documents in our index are in USD, but we would
             "max" : {
                 "field" : "price",
                 "script" : {
-                    "inline": "_value * conversion_rate",
+                    "lang": "painless",
+                    "inline": "_value * params.conversion_rate",
                     "params" : {
                         "conversion_rate" : 1.2
                     }
diff --git a/docs/reference/aggregations/metrics/min-aggregation.asciidoc b/docs/reference/aggregations/metrics/min-aggregation.asciidoc
index 7698a41202c..819d70343fb 100644
--- a/docs/reference/aggregations/metrics/min-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/min-aggregation.asciidoc
@@ -39,12 +39,19 @@ Computing the min price value across all document, this time using a script:
 --------------------------------------------------
 {
     "aggs" : {
-        "min_price" : { "min" : { "script" : "doc['price'].value" } }
+        "min_price" : { 
+            "min" : { 
+                "script" : {
+                    "inline" : "doc['price'].value",
+                    "lang" : "painless"
+                }
+            }
+        }
     }
 }
 --------------------------------------------------
 
-This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a file script use the following syntax:
+This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a file script use the following syntax:
 
 [source,js]
 --------------------------------------------------
@@ -78,7 +85,8 @@ Let's say that the prices of the documents in our index are in USD, but we would
             "min" : {
                 "field" : "price",
                 "script" : 
-                    "inline": "_value * conversion_rate",
+                    "lang" : "painless",
+                    "inline": "_value * params.conversion_rate",
                     "params" : {
                         "conversion_rate" : 1.2
                     }
diff --git a/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc b/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc
index 5357d00461e..dc8c8837344 100644
--- a/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/percentile-aggregation.asciidoc
@@ -101,7 +101,8 @@ a script to convert them on-the-fly:
         "load_time_outlier" : {
             "percentiles" : {
                 "script" : {
-                    "inline": "doc['load_time'].value / timeUnit", <1>
+                    "lang": "painless",
+                    "inline": "doc['load_time'].value / params.timeUnit", <1>
                     "params" : {
                         "timeUnit" : 1000   <2>
                     }
@@ -115,7 +116,7 @@ a script to convert them on-the-fly:
 script to generate values which percentiles are calculated on
 <2> Scripting supports parameterized input just like any other script
 
-This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a file script use the following syntax:
+This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a file script use the following syntax:
 
 [source,js]
 --------------------------------------------------
diff --git a/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc b/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc
index 759abb22be5..dcb953ae252 100644
--- a/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc
@@ -73,7 +73,8 @@ a script to convert them on-the-fly:
             "percentile_ranks" : {
                 "values" : [3, 5],
                 "script" : {
-                    "inline": "doc['load_time'].value / timeUnit", <1>
+                    "lang": "painless",
+                    "inline": "doc['load_time'].value / params.timeUnit", <1>
                     "params" : {
                         "timeUnit" : 1000   <2>
                     }
@@ -87,7 +88,7 @@ a script to convert them on-the-fly:
 script to generate values which percentile ranks are calculated on
 <2> Scripting supports parameterized input just like any other script
 
-This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a file script use the following syntax:
+This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a file script use the following syntax:
 
 [source,js]
 --------------------------------------------------
diff --git a/docs/reference/aggregations/metrics/stats-aggregation.asciidoc b/docs/reference/aggregations/metrics/stats-aggregation.asciidoc
index 852c1c3f7a9..a442fb12150 100644
--- a/docs/reference/aggregations/metrics/stats-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/stats-aggregation.asciidoc
@@ -48,12 +48,19 @@ Computing the grades stats based on a script:
     ...,
 
     "aggs" : {
-        "grades_stats" : { "stats" : { "script" : "doc['grade'].value" } }
+        "grades_stats" : { 
+             "stats" : {
+                 "script" : { 
+                     "lang": "painless",
+                     "inline": "doc['grade'].value" 
+                 }
+             } 
+         }
     }
 }
 --------------------------------------------------
 
-This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a file script use the following syntax:
+This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a file script use the following syntax:
 
 [source,js]
 --------------------------------------------------
@@ -92,7 +99,8 @@ It turned out that the exam was way above the level of the students and a grade
                 "stats" : {
                     "field" : "grade",
                     "script" : 
-                        "inline": "_value * correction",
+                        "lang": "painless",
+                        "inline": "_value * params.correction",
                         "params" : {
                             "correction" : 1.2
                         }
diff --git a/docs/reference/aggregations/metrics/sum-aggregation.asciidoc b/docs/reference/aggregations/metrics/sum-aggregation.asciidoc
index d55fcd01018..b9aee74b320 100644
--- a/docs/reference/aggregations/metrics/sum-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/sum-aggregation.asciidoc
@@ -49,12 +49,19 @@ Computing the intraday return based on a script:
     ...,
 
     "aggs" : {
-        "intraday_return" : { "sum" : { "script" : "doc['change'].value" } }
+        "intraday_return" : { 
+            "sum" : { 
+                "script" : {
+                   "lang": "painless",
+                   "inline": "doc['change'].value" 
+                } 
+            }
+        }
     }
 }
 --------------------------------------------------
 
-This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a file script use the following syntax:
+This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a file script use the following syntax:
 
 [source,js]
 --------------------------------------------------
@@ -92,7 +99,10 @@ Computing the sum of squares over all stock tick changes:
             "daytime_return" : {
                 "sum" : {
                     "field" : "change",
-                    "script" : "_value * _value"
+                    "script" : {
+                        "lang": "painless",
+                        "inline": "_value * _value"
+                    }
                 }
             }
         }
diff --git a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
index 2bad60ba0ec..1b955d2a898 100644
--- a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
@@ -182,7 +182,10 @@ relevancy order of the most relevant document in a bucket.
         },
         "top_hit" : {
           "max": {
-            "script": "_score"
+            "script": {
+              "lang": "painless",
+              "inline": "_score"
+            }
           }
         }
       }
diff --git a/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc b/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc
index fa2bfdbbb9d..925f5426187 100644
--- a/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/valuecount-aggregation.asciidoc
@@ -43,12 +43,19 @@ Counting the values generated by a script:
     ...,
 
     "aggs" : {
-        "grades_count" : { "value_count" : { "script" : "doc['grade'].value" } }
+        "grades_count" : { 
+            "value_count" : { 
+                "script" : {
+                    "inline" : "doc['grade'].value",
+                    "lang" : "painless"
+                }
+            }
+        }
     }
 }
 --------------------------------------------------
 
-This will interpret the `script` parameter as an `inline` script with the default script language and no script parameters. To use a file script use the following syntax:
+This will interpret the `script` parameter as an `inline` script with the `painless` script language and no script parameters. To use a file script use the following syntax:
 
 [source,js]
 --------------------------------------------------
diff --git a/docs/reference/docs/bulk.asciidoc b/docs/reference/docs/bulk.asciidoc
index 1fbd0419e2e..c9189b57c01 100644
--- a/docs/reference/docs/bulk.asciidoc
+++ b/docs/reference/docs/bulk.asciidoc
@@ -168,7 +168,7 @@ the options. Curl example with update actions:
 { "update" : {"_id" : "1", "_type" : "type1", "_index" : "index1", "_retry_on_conflict" : 3} }
 { "doc" : {"field" : "value"} }
 { "update" : { "_id" : "0", "_type" : "type1", "_index" : "index1", "_retry_on_conflict" : 3} }
-{ "script" : { "inline": "ctx._source.counter += param1", "lang" : "javascript", "params" : {"param1" : 1}}, "upsert" : {"counter" : 1}}
+{ "script" : { "inline": "ctx._source.counter += params.param1", "lang" : "painless", "params" : {"param1" : 1}}, "upsert" : {"counter" : 1}}
 { "update" : {"_id" : "2", "_type" : "type1", "_index" : "index1", "_retry_on_conflict" : 3} }
 { "doc" : {"field" : "value"}, "doc_as_upsert" : true }
 { "update" : {"_id" : "3", "_type" : "type1", "_index" : "index1", "fields" : ["_source"]} }
diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index 525552ae85a..aac722e96a3 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -238,7 +238,8 @@ POST _reindex
     "version_type": "external"
   },
   "script": {
-    "inline": "if (ctx._source.foo == 'bar') {ctx._version++; ctx._source.remove('foo')}"
+    "inline": "if (ctx._source.foo == 'bar') {ctx._version++; ctx._source.remove('foo')}",
+    "lang": "painless"
   }
 }
 --------------------------------------------------
diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc
index 6d6bfe64ecf..56ad1c7cd9a 100644
--- a/docs/reference/docs/update-by-query.asciidoc
+++ b/docs/reference/docs/update-by-query.asciidoc
@@ -104,7 +104,8 @@ will increment the `likes` field on all of kimchy's tweets:
 POST twitter/_update_by_query
 {
   "script": {
-    "inline": "ctx._source.likes++"
+    "inline": "ctx._source.likes++",
+    "lang": "painless"
   },
   "query": {
     "term": {
diff --git a/docs/reference/docs/update.asciidoc b/docs/reference/docs/update.asciidoc
index 85e5bf2f67f..2edb0a71a3f 100644
--- a/docs/reference/docs/update.asciidoc
+++ b/docs/reference/docs/update.asciidoc
@@ -32,7 +32,8 @@ Now, we can execute a script that would increment the counter:
 --------------------------------------------------
 curl -XPOST 'localhost:9200/test/type1/1/_update' -d '{
     "script" : {
-        "inline": "ctx._source.counter += count",
+        "inline": "ctx._source.counter += params.count",
+        "lang": "painless",
         "params" : {
             "count" : 4
         }
@@ -47,7 +48,8 @@ will still add it, since its a list):
 --------------------------------------------------
 curl -XPOST 'localhost:9200/test/type1/1/_update' -d '{
     "script" : {
-        "inline": "ctx._source.tags += tag",
+        "inline": "ctx._source.tags += params.tag",
+        "lang": "painless",
         "params" : {
             "tag" : "blue"
         }
@@ -85,7 +87,8 @@ the doc if the `tags` field contain `blue`, otherwise it does nothing
 --------------------------------------------------
 curl -XPOST 'localhost:9200/test/type1/1/_update' -d '{
     "script" : {
-        "inline": "ctx._source.tags.contains(tag) ? ctx.op = \"delete\" : ctx.op = \"none\"",
+        "inline": "ctx._source.tags.contains(params.tag) ? ctx.op = \"delete\" : ctx.op = \"none\"",
+        "lang": "painless",
         "params" : {
             "tag" : "blue"
         }
@@ -144,7 +147,8 @@ will be inserted as a new document.  If the document does exist, then the
 --------------------------------------------------
 curl -XPOST 'localhost:9200/test/type1/1/_update' -d '{
     "script" : {
-        "inline": "ctx._source.counter += count",
+        "inline": "ctx._source.counter += params.count",
+        "lang": "painless",
         "params" : {
             "count" : 4
         }
diff --git a/docs/reference/mapping/fields/field-names-field.asciidoc b/docs/reference/mapping/fields/field-names-field.asciidoc
index a56a9081e88..cf8c6398d2e 100644
--- a/docs/reference/mapping/fields/field-names-field.asciidoc
+++ b/docs/reference/mapping/fields/field-names-field.asciidoc
@@ -31,7 +31,10 @@ GET my_index/_search
   },
   "script_fields": {
     "Field names": {
-      "script": "doc['_field_names']" <2>
+      "script": {
+        "lang": "painless",
+        "inline": "doc['_field_names']" <2>
+      }
     }
   }
 }
@@ -40,4 +43,4 @@ GET my_index/_search
 // CONSOLE
 
 <1> Querying on the `_field_names` field (also see the <<query-dsl-exists-query,`exists`>> query)
-<2> Accessing the `_field_names` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
+<2> Accessing the `_field_names` field in scripts
diff --git a/docs/reference/mapping/fields/id-field.asciidoc b/docs/reference/mapping/fields/id-field.asciidoc
index a852f4f4770..f99f1ec9723 100644
--- a/docs/reference/mapping/fields/id-field.asciidoc
+++ b/docs/reference/mapping/fields/id-field.asciidoc
@@ -33,7 +33,10 @@ GET my_index/_search
   },
   "script_fields": {
     "UID": {
-      "script": "doc['_id']" <2>
+      "script": {
+         "lang": "painless",
+         "inline": "doc['_id']" <2>
+      }
     }
   }
 }
@@ -41,5 +44,4 @@ GET my_index/_search
 // CONSOLE
 
 <1> Querying on the `_id` field (also see the <<query-dsl-ids-query,`ids` query>>)
-<2> Accessing the `_id` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
-
+<2> Accessing the `_id` field in scripts
diff --git a/docs/reference/mapping/fields/index-field.asciidoc b/docs/reference/mapping/fields/index-field.asciidoc
index c31b6f1d0f7..599fedba62c 100644
--- a/docs/reference/mapping/fields/index-field.asciidoc
+++ b/docs/reference/mapping/fields/index-field.asciidoc
@@ -50,7 +50,10 @@ GET index_1,index_2/_search
   ],
   "script_fields": {
     "index_name": {
-      "script": "doc['_index']" <4>
+      "script": {
+        "lang": "painless",
+        "inline": "doc['_index']" <4>
+      }
     }
   }
 }
@@ -60,4 +63,4 @@ GET index_1,index_2/_search
 <1> Querying on the `_index` field
 <2> Aggregating on the `_index` field
 <3> Sorting on the `_index` field
-<4> Accessing the `_index` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
+<4> Accessing the `_index` field in scripts
diff --git a/docs/reference/mapping/fields/parent-field.asciidoc b/docs/reference/mapping/fields/parent-field.asciidoc
index 9dd8646172d..30b2e1a2086 100644
--- a/docs/reference/mapping/fields/parent-field.asciidoc
+++ b/docs/reference/mapping/fields/parent-field.asciidoc
@@ -81,7 +81,10 @@ GET my_index/_search
   },
   "script_fields": {
     "parent": {
-      "script": "doc['_parent']" <3>
+      "script": {
+         "lang": "painless",
+         "inline": "doc['_parent']" <3>
+      }
     }
   }
 }
@@ -91,7 +94,7 @@ GET my_index/_search
 
 <1> Querying on the `_parent` field (also see the <<query-dsl-has-parent-query,`has_parent` query>> and the <<query-dsl-has-child-query,`has_child` query>>)
 <2> Aggregating on the `_parent` field (also see the <<search-aggregations-bucket-children-aggregation,`children`>> aggregation)
-<3> Accessing the `_parent` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
+<3> Accessing the `_parent` field in scripts
 
 
 ==== Parent-child restrictions
diff --git a/docs/reference/mapping/fields/routing-field.asciidoc b/docs/reference/mapping/fields/routing-field.asciidoc
index 49f269511dc..c8a92de1368 100644
--- a/docs/reference/mapping/fields/routing-field.asciidoc
+++ b/docs/reference/mapping/fields/routing-field.asciidoc
@@ -42,7 +42,10 @@ GET my_index/_search
   },
   "script_fields": {
     "Routing value": {
-      "script": "doc['_routing']" <2>
+      "script": {
+        "lang": "painless",
+        "inline": "doc['_routing']" <2>
+      }
     }
   }
 }
@@ -50,7 +53,7 @@ GET my_index/_search
 // CONSOLE
 
 <1> Querying on the `_routing` field (also see the <<query-dsl-ids-query,`ids` query>>)
-<2> Accessing the `_routing` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
+<2> Accessing the `_routing` field in scripts
 
 
 ==== Searching with custom routing
diff --git a/docs/reference/mapping/fields/type-field.asciidoc b/docs/reference/mapping/fields/type-field.asciidoc
index 703ee9930d2..fecc0143e94 100644
--- a/docs/reference/mapping/fields/type-field.asciidoc
+++ b/docs/reference/mapping/fields/type-field.asciidoc
@@ -45,7 +45,10 @@ GET my_index/type_*/_search
   ],
   "script_fields": {
     "type": {
-      "script": "doc['_type']" <4>
+      "script": {
+        "lang": "painless",
+        "inline": "doc['_type']" <4>
+      }
     }
   }
 }
@@ -56,5 +59,5 @@ GET my_index/type_*/_search
 <1> Querying on the `_type` field
 <2> Aggregating on the `_type` field
 <3> Sorting on the `_type` field
-<4> Accessing the `_type` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
+<4> Accessing the `_type` field in scripts
 
diff --git a/docs/reference/mapping/fields/uid-field.asciidoc b/docs/reference/mapping/fields/uid-field.asciidoc
index 43d5751d426..82095160646 100644
--- a/docs/reference/mapping/fields/uid-field.asciidoc
+++ b/docs/reference/mapping/fields/uid-field.asciidoc
@@ -45,7 +45,10 @@ GET my_index/_search
   ],
   "script_fields": {
     "UID": {
-      "script": "doc['_uid']" <4>
+      "script": {
+         "lang": "painless",
+         "inline": "doc['_uid']" <4>
+      }
     }
   }
 }
@@ -55,5 +58,4 @@ GET my_index/_search
 <1> Querying on the `_uid` field (also see the <<query-dsl-ids-query,`ids` query>>)
 <2> Aggregating on the `_uid` field
 <3> Sorting on the `_uid` field
-<4> Accessing the `_uid` field in scripts (inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work)
-
+<4> Accessing the `_uid` field in scripts
diff --git a/docs/reference/mapping/types/boolean.asciidoc b/docs/reference/mapping/types/boolean.asciidoc
index 914646a7220..a5f559facc2 100644
--- a/docs/reference/mapping/types/boolean.asciidoc
+++ b/docs/reference/mapping/types/boolean.asciidoc
@@ -76,13 +76,15 @@ GET my_index/_search
   },
   "script_fields": {
     "is_published": {
-      "script": "doc['is_published'].value" <1>
+      "script": {
+        "lang": "painless",
+        "inline": "doc['is_published'].value"
+      }
     }
   }
 }
 --------------------------------------------------
 // CONSOLE
-<1> Inline scripts must be <<enable-dynamic-scripting,enabled>> for this example to work.
 
 [[boolean-params]]
 ==== Parameters for `boolean` fields
diff --git a/docs/reference/modules/scripting/painless-syntax.asciidoc b/docs/reference/modules/scripting/painless-syntax.asciidoc
index 500a276c852..88c2cdb11a9 100644
--- a/docs/reference/modules/scripting/painless-syntax.asciidoc
+++ b/docs/reference/modules/scripting/painless-syntax.asciidoc
@@ -132,6 +132,15 @@ There are only a few minor differences and add-ons:
 Java's https://docs.oracle.com/javase/tutorial/java/nutsandbolts/flow.html[control flow statements] are supported, with the exception
 of the `switch` statement.
 
+In addition to Java's `enhanced for` loop, the `for in` syntax from groovy can also be used:
+
+[source,js]
+---------------------------------------------------------
+for (item : list) {
+  ...
+}
+---------------------------------------------------------
+
 [float]
 [[painless-functions]]
 === Functions
diff --git a/docs/reference/query-dsl/function-score-query.asciidoc b/docs/reference/query-dsl/function-score-query.asciidoc
index e7e4b8f5877..b6e4dedbc88 100644
--- a/docs/reference/query-dsl/function-score-query.asciidoc
+++ b/docs/reference/query-dsl/function-score-query.asciidoc
@@ -124,7 +124,10 @@ simple sample:
 [source,js]
 --------------------------------------------------
 "script_score" : {
-    "script" : "_score * doc['my_numeric_field'].value"
+    "script" : {
+      "lang": "painless",
+      "inline": "_score * doc['my_numeric_field'].value"
+    }
 }
 --------------------------------------------------
 
@@ -140,12 +143,12 @@ script, and provide parameters to it:
 --------------------------------------------------
 "script_score": {
     "script": {
-        "lang": "lang",
+        "lang": "painless",
         "params": {
             "param1": value1,
             "param2": value2
         },
-        "inline": "_score * doc['my_numeric_field'].value / pow(param1, param2)"
+        "inline": "_score * doc['my_numeric_field'].value / Math.pow(params.param1, params.param2)"
     }
 }
 --------------------------------------------------
diff --git a/docs/reference/query-dsl/script-query.asciidoc b/docs/reference/query-dsl/script-query.asciidoc
index ee06a1b64bd..db82375abe0 100644
--- a/docs/reference/query-dsl/script-query.asciidoc
+++ b/docs/reference/query-dsl/script-query.asciidoc
@@ -13,7 +13,10 @@ GET /_search
         "bool" : {
             "must" : {
                 "script" : {
-                    "script" : "doc['num1'].value > 1"
+                    "script" : {
+                        "inline": "doc['num1'].value > 1",
+                        "lang": "painless"
+                     }
                 }
             }
         }
@@ -38,7 +41,8 @@ GET /_search
             "must" : {
                 "script" : {
                     "script" : {
-                        "inline" : "doc['num1'].value > param1",
+                        "inline" : "doc['num1'].value > params.param1",
+                        "lang"   : "painless",
                         "params" : {
                             "param1" : 5
                         }
diff --git a/docs/reference/search/request/rescore.asciidoc b/docs/reference/search/request/rescore.asciidoc
index 8bab1cf1f5c..488884ef8da 100644
--- a/docs/reference/search/request/rescore.asciidoc
+++ b/docs/reference/search/request/rescore.asciidoc
@@ -118,7 +118,10 @@ curl -s -XPOST 'localhost:9200/_search' -d '{
          "rescore_query" : {
             "function_score" : {
                "script_score": {
-                  "script": "log10(doc['numeric'].value + 2)"
+                  "script": {
+                    "lang": "painless",
+                    "inline": "Math.log10(doc['numeric'].value + 2)"
+                  }
                }
             }
          }
diff --git a/docs/reference/search/request/script-fields.asciidoc b/docs/reference/search/request/script-fields.asciidoc
index 6e054f02e1c..b544c79e4f2 100644
--- a/docs/reference/search/request/script-fields.asciidoc
+++ b/docs/reference/search/request/script-fields.asciidoc
@@ -13,10 +13,14 @@ GET /_search
     },
     "script_fields" : {
         "test1" : {
-            "script" : "doc['my_field_name'].value * 2"
+            "script" : {
+                "lang": "painless",
+                "inline": "doc['my_field_name'].value * 2"
+            }
         },
         "test2" : {
             "script" : {
+                "lang": "painless",
                 "inline": "doc['my_field_name'].value * factor",
                 "params" : {
                     "factor"  : 2.0
diff --git a/docs/reference/search/request/sort.asciidoc b/docs/reference/search/request/sort.asciidoc
index fd3dbffc1d6..d0a50f0ebfd 100644
--- a/docs/reference/search/request/sort.asciidoc
+++ b/docs/reference/search/request/sort.asciidoc
@@ -378,7 +378,8 @@ GET /_search
         "_script" : {
             "type" : "number",
             "script" : {
-                "inline": "doc['field_name'].value * factor",
+                "lang": "painless",
+                "inline": "doc['field_name'].value * params.factor",
                 "params" : {
                     "factor" : 1.1
                 }
diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.g4 b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
index 027b1b370d1..a2e7b921300 100644
--- a/modules/lang-painless/src/main/antlr/PainlessLexer.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.g4
@@ -40,6 +40,7 @@ DOT:       '.' -> mode(AFTER_DOT);
 COMMA:     ',';
 SEMICOLON: ';';
 IF:        'if';
+IN:        'in';
 ELSE:      'else';
 WHILE:     'while';
 DO:        'do';
diff --git a/modules/lang-painless/src/main/antlr/PainlessLexer.tokens b/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
index 8bb8c7e4448..dc62fe36b28 100644
--- a/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
+++ b/modules/lang-painless/src/main/antlr/PainlessLexer.tokens
@@ -10,75 +10,76 @@ DOT=9
 COMMA=10
 SEMICOLON=11
 IF=12
-ELSE=13
-WHILE=14
-DO=15
-FOR=16
-CONTINUE=17
-BREAK=18
-RETURN=19
-NEW=20
-TRY=21
-CATCH=22
-THROW=23
-THIS=24
-INSTANCEOF=25
-BOOLNOT=26
-BWNOT=27
-MUL=28
-DIV=29
-REM=30
-ADD=31
-SUB=32
-LSH=33
-RSH=34
-USH=35
-LT=36
-LTE=37
-GT=38
-GTE=39
-EQ=40
-EQR=41
-NE=42
-NER=43
-BWAND=44
-XOR=45
-BWOR=46
-BOOLAND=47
-BOOLOR=48
-COND=49
-COLON=50
-REF=51
-ARROW=52
-FIND=53
-MATCH=54
-INCR=55
-DECR=56
-ASSIGN=57
-AADD=58
-ASUB=59
-AMUL=60
-ADIV=61
-AREM=62
-AAND=63
-AXOR=64
-AOR=65
-ALSH=66
-ARSH=67
-AUSH=68
-OCTAL=69
-HEX=70
-INTEGER=71
-DECIMAL=72
-STRING=73
-REGEX=74
-TRUE=75
-FALSE=76
-NULL=77
-TYPE=78
-ID=79
-DOTINTEGER=80
-DOTID=81
+IN=13
+ELSE=14
+WHILE=15
+DO=16
+FOR=17
+CONTINUE=18
+BREAK=19
+RETURN=20
+NEW=21
+TRY=22
+CATCH=23
+THROW=24
+THIS=25
+INSTANCEOF=26
+BOOLNOT=27
+BWNOT=28
+MUL=29
+DIV=30
+REM=31
+ADD=32
+SUB=33
+LSH=34
+RSH=35
+USH=36
+LT=37
+LTE=38
+GT=39
+GTE=40
+EQ=41
+EQR=42
+NE=43
+NER=44
+BWAND=45
+XOR=46
+BWOR=47
+BOOLAND=48
+BOOLOR=49
+COND=50
+COLON=51
+REF=52
+ARROW=53
+FIND=54
+MATCH=55
+INCR=56
+DECR=57
+ASSIGN=58
+AADD=59
+ASUB=60
+AMUL=61
+ADIV=62
+AREM=63
+AAND=64
+AXOR=65
+AOR=66
+ALSH=67
+ARSH=68
+AUSH=69
+OCTAL=70
+HEX=71
+INTEGER=72
+DECIMAL=73
+STRING=74
+REGEX=75
+TRUE=76
+FALSE=77
+NULL=78
+TYPE=79
+ID=80
+DOTINTEGER=81
+DOTID=82
 '{'=3
 '}'=4
 '['=5
@@ -89,62 +90,63 @@ DOTID=81
 ','=10
 ';'=11
 'if'=12
-'else'=13
-'while'=14
-'do'=15
-'for'=16
-'continue'=17
-'break'=18
-'return'=19
-'new'=20
-'try'=21
-'catch'=22
-'throw'=23
-'this'=24
-'instanceof'=25
-'!'=26
-'~'=27
-'*'=28
-'/'=29
-'%'=30
-'+'=31
-'-'=32
-'<<'=33
-'>>'=34
-'>>>'=35
-'<'=36
-'<='=37
-'>'=38
-'>='=39
-'=='=40
-'==='=41
-'!='=42
-'!=='=43
-'&'=44
-'^'=45
-'|'=46
-'&&'=47
-'||'=48
-'?'=49
-':'=50
-'::'=51
-'->'=52
-'=~'=53
-'==~'=54
-'++'=55
-'--'=56
-'='=57
-'+='=58
-'-='=59
-'*='=60
-'/='=61
-'%='=62
-'&='=63
-'^='=64
-'|='=65
-'<<='=66
-'>>='=67
-'>>>='=68
-'true'=75
-'false'=76
-'null'=77
+'in'=13
+'else'=14
+'while'=15
+'do'=16
+'for'=17
+'continue'=18
+'break'=19
+'return'=20
+'new'=21
+'try'=22
+'catch'=23
+'throw'=24
+'this'=25
+'instanceof'=26
+'!'=27
+'~'=28
+'*'=29
+'/'=30
+'%'=31
+'+'=32
+'-'=33
+'<<'=34
+'>>'=35
+'>>>'=36
+'<'=37
+'<='=38
+'>'=39
+'>='=40
+'=='=41
+'==='=42
+'!='=43
+'!=='=44
+'&'=45
+'^'=46
+'|'=47
+'&&'=48
+'||'=49
+'?'=50
+':'=51
+'::'=52
+'->'=53
+'=~'=54
+'==~'=55
+'++'=56
+'--'=57
+'='=58
+'+='=59
+'-='=60
+'*='=61
+'/='=62
+'%='=63
+'&='=64
+'^='=65
+'|='=66
+'<<='=67
+'>>='=68
+'>>>='=69
+'true'=76
+'false'=77
+'null'=78
diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.g4 b/modules/lang-painless/src/main/antlr/PainlessParser.g4
index 7e28b0ad258..b102734a4f4 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.g4
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.g4
@@ -42,6 +42,7 @@ statement
     | DO block WHILE LP expression RP delimiter                                                # do
     | FOR LP initializer? SEMICOLON expression? SEMICOLON afterthought? RP ( trailer | empty ) # for
     | FOR LP decltype ID COLON expression RP trailer                                           # each
+    | FOR LP ID IN expression RP trailer                                                       # ineach
     | declaration delimiter                                                                    # decl
     | CONTINUE delimiter                                                                       # continue
     | BREAK delimiter                                                                          # break
diff --git a/modules/lang-painless/src/main/antlr/PainlessParser.tokens b/modules/lang-painless/src/main/antlr/PainlessParser.tokens
index 8bb8c7e4448..dc62fe36b28 100644
--- a/modules/lang-painless/src/main/antlr/PainlessParser.tokens
+++ b/modules/lang-painless/src/main/antlr/PainlessParser.tokens
@@ -10,75 +10,76 @@ DOT=9
 COMMA=10
 SEMICOLON=11
 IF=12
-ELSE=13
-WHILE=14
-DO=15
-FOR=16
-CONTINUE=17
-BREAK=18
-RETURN=19
-NEW=20
-TRY=21
-CATCH=22
-THROW=23
-THIS=24
-INSTANCEOF=25
-BOOLNOT=26
-BWNOT=27
-MUL=28
-DIV=29
-REM=30
-ADD=31
-SUB=32
-LSH=33
-RSH=34
-USH=35
-LT=36
-LTE=37
-GT=38
-GTE=39
-EQ=40
-EQR=41
-NE=42
-NER=43
-BWAND=44
-XOR=45
-BWOR=46
-BOOLAND=47
-BOOLOR=48
-COND=49
-COLON=50
-REF=51
-ARROW=52
-FIND=53
-MATCH=54
-INCR=55
-DECR=56
-ASSIGN=57
-AADD=58
-ASUB=59
-AMUL=60
-ADIV=61
-AREM=62
-AAND=63
-AXOR=64
-AOR=65
-ALSH=66
-ARSH=67
-AUSH=68
-OCTAL=69
-HEX=70
-INTEGER=71
-DECIMAL=72
-STRING=73
-REGEX=74
-TRUE=75
-FALSE=76
-NULL=77
-TYPE=78
-ID=79
-DOTINTEGER=80
-DOTID=81
+IN=13
+ELSE=14
+WHILE=15
+DO=16
+FOR=17
+CONTINUE=18
+BREAK=19
+RETURN=20
+NEW=21
+TRY=22
+CATCH=23
+THROW=24
+THIS=25
+INSTANCEOF=26
+BOOLNOT=27
+BWNOT=28
+MUL=29
+DIV=30
+REM=31
+ADD=32
+SUB=33
+LSH=34
+RSH=35
+USH=36
+LT=37
+LTE=38
+GT=39
+GTE=40
+EQ=41
+EQR=42
+NE=43
+NER=44
+BWAND=45
+XOR=46
+BWOR=47
+BOOLAND=48
+BOOLOR=49
+COND=50
+COLON=51
+REF=52
+ARROW=53
+FIND=54
+MATCH=55
+INCR=56
+DECR=57
+ASSIGN=58
+AADD=59
+ASUB=60
+AMUL=61
+ADIV=62
+AREM=63
+AAND=64
+AXOR=65
+AOR=66
+ALSH=67
+ARSH=68
+AUSH=69
+OCTAL=70
+HEX=71
+INTEGER=72
+DECIMAL=73
+STRING=74
+REGEX=75
+TRUE=76
+FALSE=77
+NULL=78
+TYPE=79
+ID=80
+DOTINTEGER=81
+DOTID=82
 '{'=3
 '}'=4
 '['=5
@@ -89,62 +90,63 @@ DOTID=81
 ','=10
 ';'=11
 'if'=12
-'else'=13
-'while'=14
-'do'=15
-'for'=16
-'continue'=17
-'break'=18
-'return'=19
-'new'=20
-'try'=21
-'catch'=22
-'throw'=23
-'this'=24
-'instanceof'=25
-'!'=26
-'~'=27
-'*'=28
-'/'=29
-'%'=30
-'+'=31
-'-'=32
-'<<'=33
-'>>'=34
-'>>>'=35
-'<'=36
-'<='=37
-'>'=38
-'>='=39
-'=='=40
-'==='=41
-'!='=42
-'!=='=43
-'&'=44
-'^'=45
-'|'=46
-'&&'=47
-'||'=48
-'?'=49
-':'=50
-'::'=51
-'->'=52
-'=~'=53
-'==~'=54
-'++'=55
-'--'=56
-'='=57
-'+='=58
-'-='=59
-'*='=60
-'/='=61
-'%='=62
-'&='=63
-'^='=64
-'|='=65
-'<<='=66
-'>>='=67
-'>>>='=68
-'true'=75
-'false'=76
-'null'=77
+'in'=13
+'else'=14
+'while'=15
+'do'=16
+'for'=17
+'continue'=18
+'break'=19
+'return'=20
+'new'=21
+'try'=22
+'catch'=23
+'throw'=24
+'this'=25
+'instanceof'=26
+'!'=27
+'~'=28
+'*'=29
+'/'=30
+'%'=31
+'+'=32
+'-'=33
+'<<'=34
+'>>'=35
+'>>>'=36
+'<'=37
+'<='=38
+'>'=39
+'>='=40
+'=='=41
+'==='=42
+'!='=43
+'!=='=44
+'&'=45
+'^'=46
+'|'=47
+'&&'=48
+'||'=49
+'?'=50
+':'=51
+'::'=52
+'->'=53
+'=~'=54
+'==~'=55
+'++'=56
+'--'=57
+'='=58
+'+='=59
+'-='=60
+'*='=61
+'/='=62
+'%='=63
+'&='=64
+'^='=65
+'|='=66
+'<<='=67
+'>>='=68
+'>>>='=69
+'true'=76
+'false'=77
+'null'=78
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
index ae54b6e6271..b02ea085904 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java
@@ -74,7 +74,11 @@ public final class Locals {
         Locals locals = new Locals(programScope, returnType);
         for (int i = 0; i < parameters.size(); i++) {
             Parameter parameter = parameters.get(i);
-            boolean isCapture = i < captureCount;
+            // TODO: allow non-captures to be r/w:
+            // boolean isCapture = i < captureCount;
+            // currently, this cannot be allowed, as we swap in real types,
+            // but that can prevent a store of a different type...
+            boolean isCapture = true;
             locals.addVariable(parameter.location, parameter.type, parameter.name, isCapture);
         }
         // Loop counter to catch infinite loops.  Internal use only.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/SimpleChecksAdapter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/SimpleChecksAdapter.java
new file mode 100644
index 00000000000..aa6d121945b
--- /dev/null
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/SimpleChecksAdapter.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.painless;
+
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.util.CheckClassAdapter;
+import org.objectweb.asm.util.CheckMethodAdapter;
+
+import java.util.HashMap;
+
+/** 
+ * A CheckClassAdapter that does not use setAccessible to try to access private fields of Label!
+ * <p>
+ * This means jump insns are not checked, but we still get all the other checking.
+ */
+// TODO: we should really try to get this fixed in ASM!
+public class SimpleChecksAdapter extends CheckClassAdapter {
+
+    public SimpleChecksAdapter(ClassVisitor cv) {
+        super(WriterConstants.ASM_VERSION, cv, false);
+    }
+
+    @Override
+    public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) {
+        MethodVisitor in = cv.visitMethod(access, name, desc, signature, exceptions);
+        CheckMethodAdapter checker = new CheckMethodAdapter(WriterConstants.ASM_VERSION, in, new HashMap<Label, Integer>()) {
+            @Override
+            public void visitJumpInsn(int opcode, Label label) {
+                mv.visitJumpInsn(opcode, label);
+            }
+
+            @Override
+            public void visitTryCatchBlock(Label start, Label end, Label handler, String type) {
+                mv.visitTryCatchBlock(start, end, handler, type);
+            }
+        };
+        checker.version = WriterConstants.CLASS_VERSION;
+        return checker;
+    }
+}
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index e2bf804c181..ca549522018 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -43,6 +43,8 @@ import java.util.regex.Pattern;
  */
 public final class WriterConstants {
 
+    public final static int CLASS_VERSION = Opcodes.V1_8;
+    public final static int ASM_VERSION = Opcodes.ASM5;
     public final static String BASE_CLASS_NAME = Executable.class.getName();
     public final static Type BASE_CLASS_TYPE   = Type.getType(Executable.class);
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
index 6d791979243..eae7150b9be 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessLexer.java
@@ -21,15 +21,16 @@ class PainlessLexer extends Lexer {
     new PredictionContextCache();
   public static final int
     WS=1, COMMENT=2, LBRACK=3, RBRACK=4, LBRACE=5, RBRACE=6, LP=7, RP=8, DOT=9, 
-    COMMA=10, SEMICOLON=11, IF=12, ELSE=13, WHILE=14, DO=15, FOR=16, CONTINUE=17, 
-    BREAK=18, RETURN=19, NEW=20, TRY=21, CATCH=22, THROW=23, THIS=24, INSTANCEOF=25, 
-    BOOLNOT=26, BWNOT=27, MUL=28, DIV=29, REM=30, ADD=31, SUB=32, LSH=33, 
-    RSH=34, USH=35, LT=36, LTE=37, GT=38, GTE=39, EQ=40, EQR=41, NE=42, NER=43, 
-    BWAND=44, XOR=45, BWOR=46, BOOLAND=47, BOOLOR=48, COND=49, COLON=50, REF=51, 
-    ARROW=52, FIND=53, MATCH=54, INCR=55, DECR=56, ASSIGN=57, AADD=58, ASUB=59, 
-    AMUL=60, ADIV=61, AREM=62, AAND=63, AXOR=64, AOR=65, ALSH=66, ARSH=67, 
-    AUSH=68, OCTAL=69, HEX=70, INTEGER=71, DECIMAL=72, STRING=73, REGEX=74, 
-    TRUE=75, FALSE=76, NULL=77, TYPE=78, ID=79, DOTINTEGER=80, DOTID=81;
+    COMMA=10, SEMICOLON=11, IF=12, IN=13, ELSE=14, WHILE=15, DO=16, FOR=17, 
+    CONTINUE=18, BREAK=19, RETURN=20, NEW=21, TRY=22, CATCH=23, THROW=24, 
+    THIS=25, INSTANCEOF=26, BOOLNOT=27, BWNOT=28, MUL=29, DIV=30, REM=31, 
+    ADD=32, SUB=33, LSH=34, RSH=35, USH=36, LT=37, LTE=38, GT=39, GTE=40, 
+    EQ=41, EQR=42, NE=43, NER=44, BWAND=45, XOR=46, BWOR=47, BOOLAND=48, BOOLOR=49, 
+    COND=50, COLON=51, REF=52, ARROW=53, FIND=54, MATCH=55, INCR=56, DECR=57, 
+    ASSIGN=58, AADD=59, ASUB=60, AMUL=61, ADIV=62, AREM=63, AAND=64, AXOR=65, 
+    AOR=66, ALSH=67, ARSH=68, AUSH=69, OCTAL=70, HEX=71, INTEGER=72, DECIMAL=73, 
+    STRING=74, REGEX=75, TRUE=76, FALSE=77, NULL=78, TYPE=79, ID=80, DOTINTEGER=81, 
+    DOTID=82;
   public static final int AFTER_DOT = 1;
   public static String[] modeNames = {
     "DEFAULT_MODE", "AFTER_DOT"
@@ -37,7 +38,7 @@ class PainlessLexer extends Lexer {
 
   public static final String[] ruleNames = {
     "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", "DOT", 
-    "COMMA", "SEMICOLON", "IF", "ELSE", "WHILE", "DO", "FOR", "CONTINUE", 
+    "COMMA", "SEMICOLON", "IF", "IN", "ELSE", "WHILE", "DO", "FOR", "CONTINUE", 
     "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", "INSTANCEOF", 
     "BOOLNOT", "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", 
     "LT", "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", 
@@ -50,25 +51,25 @@ class PainlessLexer extends Lexer {
 
   private static final String[] _LITERAL_NAMES = {
     null, null, null, "'{'", "'}'", "'['", "']'", "'('", "')'", "'.'", "','", 
-    "';'", "'if'", "'else'", "'while'", "'do'", "'for'", "'continue'", "'break'", 
-    "'return'", "'new'", "'try'", "'catch'", "'throw'", "'this'", "'instanceof'", 
-    "'!'", "'~'", "'*'", "'/'", "'%'", "'+'", "'-'", "'<<'", "'>>'", "'>>>'", 
-    "'<'", "'<='", "'>'", "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", 
-    "'^'", "'|'", "'&&'", "'||'", "'?'", "':'", "'::'", "'->'", "'=~'", "'==~'", 
-    "'++'", "'--'", "'='", "'+='", "'-='", "'*='", "'/='", "'%='", "'&='", 
-    "'^='", "'|='", "'<<='", "'>>='", "'>>>='", null, null, null, null, null, 
-    null, "'true'", "'false'", "'null'"
+    "';'", "'if'", "'in'", "'else'", "'while'", "'do'", "'for'", "'continue'", 
+    "'break'", "'return'", "'new'", "'try'", "'catch'", "'throw'", "'this'", 
+    "'instanceof'", "'!'", "'~'", "'*'", "'/'", "'%'", "'+'", "'-'", "'<<'", 
+    "'>>'", "'>>>'", "'<'", "'<='", "'>'", "'>='", "'=='", "'==='", "'!='", 
+    "'!=='", "'&'", "'^'", "'|'", "'&&'", "'||'", "'?'", "':'", "'::'", "'->'", 
+    "'=~'", "'==~'", "'++'", "'--'", "'='", "'+='", "'-='", "'*='", "'/='", 
+    "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", "'>>>='", null, null, 
+    null, null, null, null, "'true'", "'false'", "'null'"
   };
   private static final String[] _SYMBOLIC_NAMES = {
     null, "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", 
-    "DOT", "COMMA", "SEMICOLON", "IF", "ELSE", "WHILE", "DO", "FOR", "CONTINUE", 
-    "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", "INSTANCEOF", 
-    "BOOLNOT", "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", 
-    "LT", "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", 
-    "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "FIND", "MATCH", 
-    "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", "AAND", 
-    "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", "DECIMAL", 
-    "STRING", "REGEX", "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", 
+    "DOT", "COMMA", "SEMICOLON", "IF", "IN", "ELSE", "WHILE", "DO", "FOR", 
+    "CONTINUE", "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", 
+    "INSTANCEOF", "BOOLNOT", "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", 
+    "RSH", "USH", "LT", "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", 
+    "XOR", "BWOR", "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "FIND", 
+    "MATCH", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", 
+    "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", 
+    "DECIMAL", "STRING", "REGEX", "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", 
     "DOTID"
   };
   public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
@@ -128,11 +129,11 @@ class PainlessLexer extends Lexer {
   @Override
   public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) {
     switch (ruleIndex) {
-    case 28:
+    case 29:
       return DIV_sempred((RuleContext)_localctx, predIndex);
-    case 73:
+    case 74:
       return REGEX_sempred((RuleContext)_localctx, predIndex);
-    case 77:
+    case 78:
       return TYPE_sempred((RuleContext)_localctx, predIndex);
     }
     return true;
@@ -160,7 +161,7 @@ class PainlessLexer extends Lexer {
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2S\u0246\b\1\b\1\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2T\u024b\b\1\b\1\4"+
     "\2\t\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n"+
     "\4\13\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22"+
     "\t\22\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31"+
@@ -169,201 +170,202 @@ class PainlessLexer extends Lexer {
     "+\4,\t,\4-\t-\4.\t.\4/\t/\4\60\t\60\4\61\t\61\4\62\t\62\4\63\t\63\4\64"+
     "\t\64\4\65\t\65\4\66\t\66\4\67\t\67\48\t8\49\t9\4:\t:\4;\t;\4<\t<\4=\t"+
     "=\4>\t>\4?\t?\4@\t@\4A\tA\4B\tB\4C\tC\4D\tD\4E\tE\4F\tF\4G\tG\4H\tH\4"+
-    "I\tI\4J\tJ\4K\tK\4L\tL\4M\tM\4N\tN\4O\tO\4P\tP\4Q\tQ\4R\tR\3\2\6\2\u00a8"+
-    "\n\2\r\2\16\2\u00a9\3\2\3\2\3\3\3\3\3\3\3\3\7\3\u00b2\n\3\f\3\16\3\u00b5"+
-    "\13\3\3\3\3\3\3\3\3\3\3\3\7\3\u00bc\n\3\f\3\16\3\u00bf\13\3\3\3\3\3\5"+
-    "\3\u00c3\n\3\3\3\3\3\3\4\3\4\3\5\3\5\3\6\3\6\3\7\3\7\3\b\3\b\3\t\3\t\3"+
-    "\n\3\n\3\n\3\n\3\13\3\13\3\f\3\f\3\r\3\r\3\r\3\16\3\16\3\16\3\16\3\16"+
-    "\3\17\3\17\3\17\3\17\3\17\3\17\3\20\3\20\3\20\3\21\3\21\3\21\3\21\3\22"+
-    "\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\23\3\23\3\23\3\23\3\23\3\23"+
-    "\3\24\3\24\3\24\3\24\3\24\3\24\3\24\3\25\3\25\3\25\3\25\3\26\3\26\3\26"+
-    "\3\26\3\27\3\27\3\27\3\27\3\27\3\27\3\30\3\30\3\30\3\30\3\30\3\30\3\31"+
-    "\3\31\3\31\3\31\3\31\3\32\3\32\3\32\3\32\3\32\3\32\3\32\3\32\3\32\3\32"+
-    "\3\32\3\33\3\33\3\34\3\34\3\35\3\35\3\36\3\36\3\36\3\37\3\37\3 \3 \3!"+
-    "\3!\3\"\3\"\3\"\3#\3#\3#\3$\3$\3$\3$\3%\3%\3&\3&\3&\3\'\3\'\3(\3(\3(\3"+
-    ")\3)\3)\3*\3*\3*\3*\3+\3+\3+\3,\3,\3,\3,\3-\3-\3.\3.\3/\3/\3\60\3\60\3"+
-    "\60\3\61\3\61\3\61\3\62\3\62\3\63\3\63\3\64\3\64\3\64\3\65\3\65\3\65\3"+
-    "\66\3\66\3\66\3\67\3\67\3\67\3\67\38\38\38\39\39\39\3:\3:\3;\3;\3;\3<"+
-    "\3<\3<\3=\3=\3=\3>\3>\3>\3?\3?\3?\3@\3@\3@\3A\3A\3A\3B\3B\3B\3C\3C\3C"+
-    "\3C\3D\3D\3D\3D\3E\3E\3E\3E\3E\3F\3F\6F\u01a7\nF\rF\16F\u01a8\3F\5F\u01ac"+
-    "\nF\3G\3G\3G\6G\u01b1\nG\rG\16G\u01b2\3G\5G\u01b6\nG\3H\3H\3H\7H\u01bb"+
-    "\nH\fH\16H\u01be\13H\5H\u01c0\nH\3H\5H\u01c3\nH\3I\3I\3I\7I\u01c8\nI\f"+
-    "I\16I\u01cb\13I\5I\u01cd\nI\3I\3I\6I\u01d1\nI\rI\16I\u01d2\5I\u01d5\n"+
-    "I\3I\3I\5I\u01d9\nI\3I\6I\u01dc\nI\rI\16I\u01dd\5I\u01e0\nI\3I\5I\u01e3"+
-    "\nI\3J\3J\3J\3J\3J\3J\7J\u01eb\nJ\fJ\16J\u01ee\13J\3J\3J\3J\3J\3J\3J\3"+
-    "J\7J\u01f7\nJ\fJ\16J\u01fa\13J\3J\5J\u01fd\nJ\3K\3K\3K\3K\6K\u0203\nK"+
-    "\rK\16K\u0204\3K\3K\7K\u0209\nK\fK\16K\u020c\13K\3K\3K\3L\3L\3L\3L\3L"+
-    "\3M\3M\3M\3M\3M\3M\3N\3N\3N\3N\3N\3O\3O\3O\3O\7O\u0224\nO\fO\16O\u0227"+
-    "\13O\3O\3O\3P\3P\7P\u022d\nP\fP\16P\u0230\13P\3Q\3Q\3Q\7Q\u0235\nQ\fQ"+
-    "\16Q\u0238\13Q\5Q\u023a\nQ\3Q\3Q\3R\3R\7R\u0240\nR\fR\16R\u0243\13R\3"+
-    "R\3R\6\u00b3\u00bd\u01ec\u01f8\2S\4\3\6\4\b\5\n\6\f\7\16\b\20\t\22\n\24"+
-    "\13\26\f\30\r\32\16\34\17\36\20 \21\"\22$\23&\24(\25*\26,\27.\30\60\31"+
-    "\62\32\64\33\66\348\35:\36<\37> @!B\"D#F$H%J&L\'N(P)R*T+V,X-Z.\\/^\60"+
-    "`\61b\62d\63f\64h\65j\66l\67n8p9r:t;v<x=z>|?~@\u0080A\u0082B\u0084C\u0086"+
-    "D\u0088E\u008aF\u008cG\u008eH\u0090I\u0092J\u0094K\u0096L\u0098M\u009a"+
-    "N\u009cO\u009eP\u00a0Q\u00a2R\u00a4S\4\2\3\24\5\2\13\f\17\17\"\"\4\2\f"+
-    "\f\17\17\3\2\629\4\2NNnn\4\2ZZzz\5\2\62;CHch\3\2\63;\3\2\62;\b\2FFHHN"+
-    "Nffhhnn\4\2GGgg\4\2--//\4\2HHhh\4\2$$^^\4\2\f\f\61\61\3\2\f\f\t\2WWee"+
-    "kknouuwwzz\5\2C\\aac|\6\2\62;C\\aac|\u0266\2\4\3\2\2\2\2\6\3\2\2\2\2\b"+
-    "\3\2\2\2\2\n\3\2\2\2\2\f\3\2\2\2\2\16\3\2\2\2\2\20\3\2\2\2\2\22\3\2\2"+
-    "\2\2\24\3\2\2\2\2\26\3\2\2\2\2\30\3\2\2\2\2\32\3\2\2\2\2\34\3\2\2\2\2"+
-    "\36\3\2\2\2\2 \3\2\2\2\2\"\3\2\2\2\2$\3\2\2\2\2&\3\2\2\2\2(\3\2\2\2\2"+
-    "*\3\2\2\2\2,\3\2\2\2\2.\3\2\2\2\2\60\3\2\2\2\2\62\3\2\2\2\2\64\3\2\2\2"+
-    "\2\66\3\2\2\2\28\3\2\2\2\2:\3\2\2\2\2<\3\2\2\2\2>\3\2\2\2\2@\3\2\2\2\2"+
-    "B\3\2\2\2\2D\3\2\2\2\2F\3\2\2\2\2H\3\2\2\2\2J\3\2\2\2\2L\3\2\2\2\2N\3"+
-    "\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2T\3\2\2\2\2V\3\2\2\2\2X\3\2\2\2\2Z\3\2\2"+
-    "\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3\2\2\2\2b\3\2\2\2\2d\3\2\2\2\2f\3\2\2\2"+
-    "\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2\2\2n\3\2\2\2\2p\3\2\2\2\2r\3\2\2\2\2t"+
-    "\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2z\3\2\2\2\2|\3\2\2\2\2~\3\2\2\2\2\u0080"+
-    "\3\2\2\2\2\u0082\3\2\2\2\2\u0084\3\2\2\2\2\u0086\3\2\2\2\2\u0088\3\2\2"+
-    "\2\2\u008a\3\2\2\2\2\u008c\3\2\2\2\2\u008e\3\2\2\2\2\u0090\3\2\2\2\2\u0092"+
-    "\3\2\2\2\2\u0094\3\2\2\2\2\u0096\3\2\2\2\2\u0098\3\2\2\2\2\u009a\3\2\2"+
-    "\2\2\u009c\3\2\2\2\2\u009e\3\2\2\2\2\u00a0\3\2\2\2\3\u00a2\3\2\2\2\3\u00a4"+
-    "\3\2\2\2\4\u00a7\3\2\2\2\6\u00c2\3\2\2\2\b\u00c6\3\2\2\2\n\u00c8\3\2\2"+
-    "\2\f\u00ca\3\2\2\2\16\u00cc\3\2\2\2\20\u00ce\3\2\2\2\22\u00d0\3\2\2\2"+
-    "\24\u00d2\3\2\2\2\26\u00d6\3\2\2\2\30\u00d8\3\2\2\2\32\u00da\3\2\2\2\34"+
-    "\u00dd\3\2\2\2\36\u00e2\3\2\2\2 \u00e8\3\2\2\2\"\u00eb\3\2\2\2$\u00ef"+
-    "\3\2\2\2&\u00f8\3\2\2\2(\u00fe\3\2\2\2*\u0105\3\2\2\2,\u0109\3\2\2\2."+
-    "\u010d\3\2\2\2\60\u0113\3\2\2\2\62\u0119\3\2\2\2\64\u011e\3\2\2\2\66\u0129"+
-    "\3\2\2\28\u012b\3\2\2\2:\u012d\3\2\2\2<\u012f\3\2\2\2>\u0132\3\2\2\2@"+
-    "\u0134\3\2\2\2B\u0136\3\2\2\2D\u0138\3\2\2\2F\u013b\3\2\2\2H\u013e\3\2"+
-    "\2\2J\u0142\3\2\2\2L\u0144\3\2\2\2N\u0147\3\2\2\2P\u0149\3\2\2\2R\u014c"+
-    "\3\2\2\2T\u014f\3\2\2\2V\u0153\3\2\2\2X\u0156\3\2\2\2Z\u015a\3\2\2\2\\"+
-    "\u015c\3\2\2\2^\u015e\3\2\2\2`\u0160\3\2\2\2b\u0163\3\2\2\2d\u0166\3\2"+
-    "\2\2f\u0168\3\2\2\2h\u016a\3\2\2\2j\u016d\3\2\2\2l\u0170\3\2\2\2n\u0173"+
-    "\3\2\2\2p\u0177\3\2\2\2r\u017a\3\2\2\2t\u017d\3\2\2\2v\u017f\3\2\2\2x"+
-    "\u0182\3\2\2\2z\u0185\3\2\2\2|\u0188\3\2\2\2~\u018b\3\2\2\2\u0080\u018e"+
-    "\3\2\2\2\u0082\u0191\3\2\2\2\u0084\u0194\3\2\2\2\u0086\u0197\3\2\2\2\u0088"+
-    "\u019b\3\2\2\2\u008a\u019f\3\2\2\2\u008c\u01a4\3\2\2\2\u008e\u01ad\3\2"+
-    "\2\2\u0090\u01bf\3\2\2\2\u0092\u01cc\3\2\2\2\u0094\u01fc\3\2\2\2\u0096"+
-    "\u01fe\3\2\2\2\u0098\u020f\3\2\2\2\u009a\u0214\3\2\2\2\u009c\u021a\3\2"+
-    "\2\2\u009e\u021f\3\2\2\2\u00a0\u022a\3\2\2\2\u00a2\u0239\3\2\2\2\u00a4"+
-    "\u023d\3\2\2\2\u00a6\u00a8\t\2\2\2\u00a7\u00a6\3\2\2\2\u00a8\u00a9\3\2"+
-    "\2\2\u00a9\u00a7\3\2\2\2\u00a9\u00aa\3\2\2\2\u00aa\u00ab\3\2\2\2\u00ab"+
-    "\u00ac\b\2\2\2\u00ac\5\3\2\2\2\u00ad\u00ae\7\61\2\2\u00ae\u00af\7\61\2"+
-    "\2\u00af\u00b3\3\2\2\2\u00b0\u00b2\13\2\2\2\u00b1\u00b0\3\2\2\2\u00b2"+
-    "\u00b5\3\2\2\2\u00b3\u00b4\3\2\2\2\u00b3\u00b1\3\2\2\2\u00b4\u00b6\3\2"+
-    "\2\2\u00b5\u00b3\3\2\2\2\u00b6\u00c3\t\3\2\2\u00b7\u00b8\7\61\2\2\u00b8"+
-    "\u00b9\7,\2\2\u00b9\u00bd\3\2\2\2\u00ba\u00bc\13\2\2\2\u00bb\u00ba\3\2"+
-    "\2\2\u00bc\u00bf\3\2\2\2\u00bd\u00be\3\2\2\2\u00bd\u00bb\3\2\2\2\u00be"+
-    "\u00c0\3\2\2\2\u00bf\u00bd\3\2\2\2\u00c0\u00c1\7,\2\2\u00c1\u00c3\7\61"+
-    "\2\2\u00c2\u00ad\3\2\2\2\u00c2\u00b7\3\2\2\2\u00c3\u00c4\3\2\2\2\u00c4"+
-    "\u00c5\b\3\2\2\u00c5\7\3\2\2\2\u00c6\u00c7\7}\2\2\u00c7\t\3\2\2\2\u00c8"+
-    "\u00c9\7\177\2\2\u00c9\13\3\2\2\2\u00ca\u00cb\7]\2\2\u00cb\r\3\2\2\2\u00cc"+
-    "\u00cd\7_\2\2\u00cd\17\3\2\2\2\u00ce\u00cf\7*\2\2\u00cf\21\3\2\2\2\u00d0"+
-    "\u00d1\7+\2\2\u00d1\23\3\2\2\2\u00d2\u00d3\7\60\2\2\u00d3\u00d4\3\2\2"+
-    "\2\u00d4\u00d5\b\n\3\2\u00d5\25\3\2\2\2\u00d6\u00d7\7.\2\2\u00d7\27\3"+
-    "\2\2\2\u00d8\u00d9\7=\2\2\u00d9\31\3\2\2\2\u00da\u00db\7k\2\2\u00db\u00dc"+
-    "\7h\2\2\u00dc\33\3\2\2\2\u00dd\u00de\7g\2\2\u00de\u00df\7n\2\2\u00df\u00e0"+
-    "\7u\2\2\u00e0\u00e1\7g\2\2\u00e1\35\3\2\2\2\u00e2\u00e3\7y\2\2\u00e3\u00e4"+
-    "\7j\2\2\u00e4\u00e5\7k\2\2\u00e5\u00e6\7n\2\2\u00e6\u00e7\7g\2\2\u00e7"+
-    "\37\3\2\2\2\u00e8\u00e9\7f\2\2\u00e9\u00ea\7q\2\2\u00ea!\3\2\2\2\u00eb"+
-    "\u00ec\7h\2\2\u00ec\u00ed\7q\2\2\u00ed\u00ee\7t\2\2\u00ee#\3\2\2\2\u00ef"+
-    "\u00f0\7e\2\2\u00f0\u00f1\7q\2\2\u00f1\u00f2\7p\2\2\u00f2\u00f3\7v\2\2"+
-    "\u00f3\u00f4\7k\2\2\u00f4\u00f5\7p\2\2\u00f5\u00f6\7w\2\2\u00f6\u00f7"+
-    "\7g\2\2\u00f7%\3\2\2\2\u00f8\u00f9\7d\2\2\u00f9\u00fa\7t\2\2\u00fa\u00fb"+
-    "\7g\2\2\u00fb\u00fc\7c\2\2\u00fc\u00fd\7m\2\2\u00fd\'\3\2\2\2\u00fe\u00ff"+
-    "\7t\2\2\u00ff\u0100\7g\2\2\u0100\u0101\7v\2\2\u0101\u0102\7w\2\2\u0102"+
-    "\u0103\7t\2\2\u0103\u0104\7p\2\2\u0104)\3\2\2\2\u0105\u0106\7p\2\2\u0106"+
-    "\u0107\7g\2\2\u0107\u0108\7y\2\2\u0108+\3\2\2\2\u0109\u010a\7v\2\2\u010a"+
-    "\u010b\7t\2\2\u010b\u010c\7{\2\2\u010c-\3\2\2\2\u010d\u010e\7e\2\2\u010e"+
-    "\u010f\7c\2\2\u010f\u0110\7v\2\2\u0110\u0111\7e\2\2\u0111\u0112\7j\2\2"+
-    "\u0112/\3\2\2\2\u0113\u0114\7v\2\2\u0114\u0115\7j\2\2\u0115\u0116\7t\2"+
-    "\2\u0116\u0117\7q\2\2\u0117\u0118\7y\2\2\u0118\61\3\2\2\2\u0119\u011a"+
-    "\7v\2\2\u011a\u011b\7j\2\2\u011b\u011c\7k\2\2\u011c\u011d\7u\2\2\u011d"+
-    "\63\3\2\2\2\u011e\u011f\7k\2\2\u011f\u0120\7p\2\2\u0120\u0121\7u\2\2\u0121"+
-    "\u0122\7v\2\2\u0122\u0123\7c\2\2\u0123\u0124\7p\2\2\u0124\u0125\7e\2\2"+
-    "\u0125\u0126\7g\2\2\u0126\u0127\7q\2\2\u0127\u0128\7h\2\2\u0128\65\3\2"+
-    "\2\2\u0129\u012a\7#\2\2\u012a\67\3\2\2\2\u012b\u012c\7\u0080\2\2\u012c"+
-    "9\3\2\2\2\u012d\u012e\7,\2\2\u012e;\3\2\2\2\u012f\u0130\7\61\2\2\u0130"+
-    "\u0131\6\36\2\2\u0131=\3\2\2\2\u0132\u0133\7\'\2\2\u0133?\3\2\2\2\u0134"+
-    "\u0135\7-\2\2\u0135A\3\2\2\2\u0136\u0137\7/\2\2\u0137C\3\2\2\2\u0138\u0139"+
-    "\7>\2\2\u0139\u013a\7>\2\2\u013aE\3\2\2\2\u013b\u013c\7@\2\2\u013c\u013d"+
-    "\7@\2\2\u013dG\3\2\2\2\u013e\u013f\7@\2\2\u013f\u0140\7@\2\2\u0140\u0141"+
-    "\7@\2\2\u0141I\3\2\2\2\u0142\u0143\7>\2\2\u0143K\3\2\2\2\u0144\u0145\7"+
-    ">\2\2\u0145\u0146\7?\2\2\u0146M\3\2\2\2\u0147\u0148\7@\2\2\u0148O\3\2"+
-    "\2\2\u0149\u014a\7@\2\2\u014a\u014b\7?\2\2\u014bQ\3\2\2\2\u014c\u014d"+
-    "\7?\2\2\u014d\u014e\7?\2\2\u014eS\3\2\2\2\u014f\u0150\7?\2\2\u0150\u0151"+
-    "\7?\2\2\u0151\u0152\7?\2\2\u0152U\3\2\2\2\u0153\u0154\7#\2\2\u0154\u0155"+
-    "\7?\2\2\u0155W\3\2\2\2\u0156\u0157\7#\2\2\u0157\u0158\7?\2\2\u0158\u0159"+
-    "\7?\2\2\u0159Y\3\2\2\2\u015a\u015b\7(\2\2\u015b[\3\2\2\2\u015c\u015d\7"+
-    "`\2\2\u015d]\3\2\2\2\u015e\u015f\7~\2\2\u015f_\3\2\2\2\u0160\u0161\7("+
-    "\2\2\u0161\u0162\7(\2\2\u0162a\3\2\2\2\u0163\u0164\7~\2\2\u0164\u0165"+
-    "\7~\2\2\u0165c\3\2\2\2\u0166\u0167\7A\2\2\u0167e\3\2\2\2\u0168\u0169\7"+
-    "<\2\2\u0169g\3\2\2\2\u016a\u016b\7<\2\2\u016b\u016c\7<\2\2\u016ci\3\2"+
-    "\2\2\u016d\u016e\7/\2\2\u016e\u016f\7@\2\2\u016fk\3\2\2\2\u0170\u0171"+
-    "\7?\2\2\u0171\u0172\7\u0080\2\2\u0172m\3\2\2\2\u0173\u0174\7?\2\2\u0174"+
-    "\u0175\7?\2\2\u0175\u0176\7\u0080\2\2\u0176o\3\2\2\2\u0177\u0178\7-\2"+
-    "\2\u0178\u0179\7-\2\2\u0179q\3\2\2\2\u017a\u017b\7/\2\2\u017b\u017c\7"+
-    "/\2\2\u017cs\3\2\2\2\u017d\u017e\7?\2\2\u017eu\3\2\2\2\u017f\u0180\7-"+
-    "\2\2\u0180\u0181\7?\2\2\u0181w\3\2\2\2\u0182\u0183\7/\2\2\u0183\u0184"+
-    "\7?\2\2\u0184y\3\2\2\2\u0185\u0186\7,\2\2\u0186\u0187\7?\2\2\u0187{\3"+
-    "\2\2\2\u0188\u0189\7\61\2\2\u0189\u018a\7?\2\2\u018a}\3\2\2\2\u018b\u018c"+
-    "\7\'\2\2\u018c\u018d\7?\2\2\u018d\177\3\2\2\2\u018e\u018f\7(\2\2\u018f"+
-    "\u0190\7?\2\2\u0190\u0081\3\2\2\2\u0191\u0192\7`\2\2\u0192\u0193\7?\2"+
-    "\2\u0193\u0083\3\2\2\2\u0194\u0195\7~\2\2\u0195\u0196\7?\2\2\u0196\u0085"+
-    "\3\2\2\2\u0197\u0198\7>\2\2\u0198\u0199\7>\2\2\u0199\u019a\7?\2\2\u019a"+
-    "\u0087\3\2\2\2\u019b\u019c\7@\2\2\u019c\u019d\7@\2\2\u019d\u019e\7?\2"+
-    "\2\u019e\u0089\3\2\2\2\u019f\u01a0\7@\2\2\u01a0\u01a1\7@\2\2\u01a1\u01a2"+
-    "\7@\2\2\u01a2\u01a3\7?\2\2\u01a3\u008b\3\2\2\2\u01a4\u01a6\7\62\2\2\u01a5"+
-    "\u01a7\t\4\2\2\u01a6\u01a5\3\2\2\2\u01a7\u01a8\3\2\2\2\u01a8\u01a6\3\2"+
-    "\2\2\u01a8\u01a9\3\2\2\2\u01a9\u01ab\3\2\2\2\u01aa\u01ac\t\5\2\2\u01ab"+
-    "\u01aa\3\2\2\2\u01ab\u01ac\3\2\2\2\u01ac\u008d\3\2\2\2\u01ad\u01ae\7\62"+
-    "\2\2\u01ae\u01b0\t\6\2\2\u01af\u01b1\t\7\2\2\u01b0\u01af\3\2\2\2\u01b1"+
-    "\u01b2\3\2\2\2\u01b2\u01b0\3\2\2\2\u01b2\u01b3\3\2\2\2\u01b3\u01b5\3\2"+
-    "\2\2\u01b4\u01b6\t\5\2\2\u01b5\u01b4\3\2\2\2\u01b5\u01b6\3\2\2\2\u01b6"+
-    "\u008f\3\2\2\2\u01b7\u01c0\7\62\2\2\u01b8\u01bc\t\b\2\2\u01b9\u01bb\t"+
-    "\t\2\2\u01ba\u01b9\3\2\2\2\u01bb\u01be\3\2\2\2\u01bc\u01ba\3\2\2\2\u01bc"+
-    "\u01bd\3\2\2\2\u01bd\u01c0\3\2\2\2\u01be\u01bc\3\2\2\2\u01bf\u01b7\3\2"+
-    "\2\2\u01bf\u01b8\3\2\2\2\u01c0\u01c2\3\2\2\2\u01c1\u01c3\t\n\2\2\u01c2"+
-    "\u01c1\3\2\2\2\u01c2\u01c3\3\2\2\2\u01c3\u0091\3\2\2\2\u01c4\u01cd\7\62"+
-    "\2\2\u01c5\u01c9\t\b\2\2\u01c6\u01c8\t\t\2\2\u01c7\u01c6\3\2\2\2\u01c8"+
-    "\u01cb\3\2\2\2\u01c9\u01c7\3\2\2\2\u01c9\u01ca\3\2\2\2\u01ca\u01cd\3\2"+
-    "\2\2\u01cb\u01c9\3\2\2\2\u01cc\u01c4\3\2\2\2\u01cc\u01c5\3\2\2\2\u01cd"+
-    "\u01d4\3\2\2\2\u01ce\u01d0\5\24\n\2\u01cf\u01d1\t\t\2\2\u01d0\u01cf\3"+
-    "\2\2\2\u01d1\u01d2\3\2\2\2\u01d2\u01d0\3\2\2\2\u01d2\u01d3\3\2\2\2\u01d3"+
-    "\u01d5\3\2\2\2\u01d4\u01ce\3\2\2\2\u01d4\u01d5\3\2\2\2\u01d5\u01df\3\2"+
-    "\2\2\u01d6\u01d8\t\13\2\2\u01d7\u01d9\t\f\2\2\u01d8\u01d7\3\2\2\2\u01d8"+
-    "\u01d9\3\2\2\2\u01d9\u01db\3\2\2\2\u01da\u01dc\t\t\2\2\u01db\u01da\3\2"+
-    "\2\2\u01dc\u01dd\3\2\2\2\u01dd\u01db\3\2\2\2\u01dd\u01de\3\2\2\2\u01de"+
-    "\u01e0\3\2\2\2\u01df\u01d6\3\2\2\2\u01df\u01e0\3\2\2\2\u01e0\u01e2\3\2"+
-    "\2\2\u01e1\u01e3\t\r\2\2\u01e2\u01e1\3\2\2\2\u01e2\u01e3\3\2\2\2\u01e3"+
-    "\u0093\3\2\2\2\u01e4\u01ec\7$\2\2\u01e5\u01e6\7^\2\2\u01e6\u01eb\7$\2"+
-    "\2\u01e7\u01e8\7^\2\2\u01e8\u01eb\7^\2\2\u01e9\u01eb\n\16\2\2\u01ea\u01e5"+
-    "\3\2\2\2\u01ea\u01e7\3\2\2\2\u01ea\u01e9\3\2\2\2\u01eb\u01ee\3\2\2\2\u01ec"+
-    "\u01ed\3\2\2\2\u01ec\u01ea\3\2\2\2\u01ed\u01ef\3\2\2\2\u01ee\u01ec\3\2"+
-    "\2\2\u01ef\u01fd\7$\2\2\u01f0\u01f8\7)\2\2\u01f1\u01f2\7^\2\2\u01f2\u01f7"+
-    "\7)\2\2\u01f3\u01f4\7^\2\2\u01f4\u01f7\7^\2\2\u01f5\u01f7\n\16\2\2\u01f6"+
-    "\u01f1\3\2\2\2\u01f6\u01f3\3\2\2\2\u01f6\u01f5\3\2\2\2\u01f7\u01fa\3\2"+
-    "\2\2\u01f8\u01f9\3\2\2\2\u01f8\u01f6\3\2\2\2\u01f9\u01fb\3\2\2\2\u01fa"+
-    "\u01f8\3\2\2\2\u01fb\u01fd\7)\2\2\u01fc\u01e4\3\2\2\2\u01fc\u01f0\3\2"+
-    "\2\2\u01fd\u0095\3\2\2\2\u01fe\u0202\7\61\2\2\u01ff\u0203\n\17\2\2\u0200"+
-    "\u0201\7^\2\2\u0201\u0203\n\20\2\2\u0202\u01ff\3\2\2\2\u0202\u0200\3\2"+
-    "\2\2\u0203\u0204\3\2\2\2\u0204\u0202\3\2\2\2\u0204\u0205\3\2\2\2\u0205"+
-    "\u0206\3\2\2\2\u0206\u020a\7\61\2\2\u0207\u0209\t\21\2\2\u0208\u0207\3"+
-    "\2\2\2\u0209\u020c\3\2\2\2\u020a\u0208\3\2\2\2\u020a\u020b\3\2\2\2\u020b"+
-    "\u020d\3\2\2\2\u020c\u020a\3\2\2\2\u020d\u020e\6K\3\2\u020e\u0097\3\2"+
-    "\2\2\u020f\u0210\7v\2\2\u0210\u0211\7t\2\2\u0211\u0212\7w\2\2\u0212\u0213"+
-    "\7g\2\2\u0213\u0099\3\2\2\2\u0214\u0215\7h\2\2\u0215\u0216\7c\2\2\u0216"+
-    "\u0217\7n\2\2\u0217\u0218\7u\2\2\u0218\u0219\7g\2\2\u0219\u009b\3\2\2"+
-    "\2\u021a\u021b\7p\2\2\u021b\u021c\7w\2\2\u021c\u021d\7n\2\2\u021d\u021e"+
-    "\7n\2\2\u021e\u009d\3\2\2\2\u021f\u0225\5\u00a0P\2\u0220\u0221\5\24\n"+
-    "\2\u0221\u0222\5\u00a0P\2\u0222\u0224\3\2\2\2\u0223\u0220\3\2\2\2\u0224"+
-    "\u0227\3\2\2\2\u0225\u0223\3\2\2\2\u0225\u0226\3\2\2\2\u0226\u0228\3\2"+
-    "\2\2\u0227\u0225\3\2\2\2\u0228\u0229\6O\4\2\u0229\u009f\3\2\2\2\u022a"+
-    "\u022e\t\22\2\2\u022b\u022d\t\23\2\2\u022c\u022b\3\2\2\2\u022d\u0230\3"+
-    "\2\2\2\u022e\u022c\3\2\2\2\u022e\u022f\3\2\2\2\u022f\u00a1\3\2\2\2\u0230"+
-    "\u022e\3\2\2\2\u0231\u023a\7\62\2\2\u0232\u0236\t\b\2\2\u0233\u0235\t"+
-    "\t\2\2\u0234\u0233\3\2\2\2\u0235\u0238\3\2\2\2\u0236\u0234\3\2\2\2\u0236"+
-    "\u0237\3\2\2\2\u0237\u023a\3\2\2\2\u0238\u0236\3\2\2\2\u0239\u0231\3\2"+
-    "\2\2\u0239\u0232\3\2\2\2\u023a\u023b\3\2\2\2\u023b\u023c\bQ\4\2\u023c"+
-    "\u00a3\3\2\2\2\u023d\u0241\t\22\2\2\u023e\u0240\t\23\2\2\u023f\u023e\3"+
-    "\2\2\2\u0240\u0243\3\2\2\2\u0241\u023f\3\2\2\2\u0241\u0242\3\2\2\2\u0242"+
-    "\u0244\3\2\2\2\u0243\u0241\3\2\2\2\u0244\u0245\bR\4\2\u0245\u00a5\3\2"+
-    "\2\2$\2\3\u00a9\u00b3\u00bd\u00c2\u01a8\u01ab\u01b2\u01b5\u01bc\u01bf"+
-    "\u01c2\u01c9\u01cc\u01d2\u01d4\u01d8\u01dd\u01df\u01e2\u01ea\u01ec\u01f6"+
-    "\u01f8\u01fc\u0202\u0204\u020a\u0225\u022e\u0236\u0239\u0241\5\b\2\2\4"+
-    "\3\2\4\2\2";
+    "I\tI\4J\tJ\4K\tK\4L\tL\4M\tM\4N\tN\4O\tO\4P\tP\4Q\tQ\4R\tR\4S\tS\3\2\6"+
+    "\2\u00aa\n\2\r\2\16\2\u00ab\3\2\3\2\3\3\3\3\3\3\3\3\7\3\u00b4\n\3\f\3"+
+    "\16\3\u00b7\13\3\3\3\3\3\3\3\3\3\3\3\7\3\u00be\n\3\f\3\16\3\u00c1\13\3"+
+    "\3\3\3\3\5\3\u00c5\n\3\3\3\3\3\3\4\3\4\3\5\3\5\3\6\3\6\3\7\3\7\3\b\3\b"+
+    "\3\t\3\t\3\n\3\n\3\n\3\n\3\13\3\13\3\f\3\f\3\r\3\r\3\r\3\16\3\16\3\16"+
+    "\3\17\3\17\3\17\3\17\3\17\3\20\3\20\3\20\3\20\3\20\3\20\3\21\3\21\3\21"+
+    "\3\22\3\22\3\22\3\22\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\24"+
+    "\3\24\3\24\3\24\3\24\3\24\3\25\3\25\3\25\3\25\3\25\3\25\3\25\3\26\3\26"+
+    "\3\26\3\26\3\27\3\27\3\27\3\27\3\30\3\30\3\30\3\30\3\30\3\30\3\31\3\31"+
+    "\3\31\3\31\3\31\3\31\3\32\3\32\3\32\3\32\3\32\3\33\3\33\3\33\3\33\3\33"+
+    "\3\33\3\33\3\33\3\33\3\33\3\33\3\34\3\34\3\35\3\35\3\36\3\36\3\37\3\37"+
+    "\3\37\3 \3 \3!\3!\3\"\3\"\3#\3#\3#\3$\3$\3$\3%\3%\3%\3%\3&\3&\3\'\3\'"+
+    "\3\'\3(\3(\3)\3)\3)\3*\3*\3*\3+\3+\3+\3+\3,\3,\3,\3-\3-\3-\3-\3.\3.\3"+
+    "/\3/\3\60\3\60\3\61\3\61\3\61\3\62\3\62\3\62\3\63\3\63\3\64\3\64\3\65"+
+    "\3\65\3\65\3\66\3\66\3\66\3\67\3\67\3\67\38\38\38\38\39\39\39\3:\3:\3"+
+    ":\3;\3;\3<\3<\3<\3=\3=\3=\3>\3>\3>\3?\3?\3?\3@\3@\3@\3A\3A\3A\3B\3B\3"+
+    "B\3C\3C\3C\3D\3D\3D\3D\3E\3E\3E\3E\3F\3F\3F\3F\3F\3G\3G\6G\u01ac\nG\r"+
+    "G\16G\u01ad\3G\5G\u01b1\nG\3H\3H\3H\6H\u01b6\nH\rH\16H\u01b7\3H\5H\u01bb"+
+    "\nH\3I\3I\3I\7I\u01c0\nI\fI\16I\u01c3\13I\5I\u01c5\nI\3I\5I\u01c8\nI\3"+
+    "J\3J\3J\7J\u01cd\nJ\fJ\16J\u01d0\13J\5J\u01d2\nJ\3J\3J\6J\u01d6\nJ\rJ"+
+    "\16J\u01d7\5J\u01da\nJ\3J\3J\5J\u01de\nJ\3J\6J\u01e1\nJ\rJ\16J\u01e2\5"+
+    "J\u01e5\nJ\3J\5J\u01e8\nJ\3K\3K\3K\3K\3K\3K\7K\u01f0\nK\fK\16K\u01f3\13"+
+    "K\3K\3K\3K\3K\3K\3K\3K\7K\u01fc\nK\fK\16K\u01ff\13K\3K\5K\u0202\nK\3L"+
+    "\3L\3L\3L\6L\u0208\nL\rL\16L\u0209\3L\3L\7L\u020e\nL\fL\16L\u0211\13L"+
+    "\3L\3L\3M\3M\3M\3M\3M\3N\3N\3N\3N\3N\3N\3O\3O\3O\3O\3O\3P\3P\3P\3P\7P"+
+    "\u0229\nP\fP\16P\u022c\13P\3P\3P\3Q\3Q\7Q\u0232\nQ\fQ\16Q\u0235\13Q\3"+
+    "R\3R\3R\7R\u023a\nR\fR\16R\u023d\13R\5R\u023f\nR\3R\3R\3S\3S\7S\u0245"+
+    "\nS\fS\16S\u0248\13S\3S\3S\6\u00b5\u00bf\u01f1\u01fd\2T\4\3\6\4\b\5\n"+
+    "\6\f\7\16\b\20\t\22\n\24\13\26\f\30\r\32\16\34\17\36\20 \21\"\22$\23&"+
+    "\24(\25*\26,\27.\30\60\31\62\32\64\33\66\348\35:\36<\37> @!B\"D#F$H%J"+
+    "&L\'N(P)R*T+V,X-Z.\\/^\60`\61b\62d\63f\64h\65j\66l\67n8p9r:t;v<x=z>|?"+
+    "~@\u0080A\u0082B\u0084C\u0086D\u0088E\u008aF\u008cG\u008eH\u0090I\u0092"+
+    "J\u0094K\u0096L\u0098M\u009aN\u009cO\u009eP\u00a0Q\u00a2R\u00a4S\u00a6"+
+    "T\4\2\3\24\5\2\13\f\17\17\"\"\4\2\f\f\17\17\3\2\629\4\2NNnn\4\2ZZzz\5"+
+    "\2\62;CHch\3\2\63;\3\2\62;\b\2FFHHNNffhhnn\4\2GGgg\4\2--//\4\2HHhh\4\2"+
+    "$$^^\4\2\f\f\61\61\3\2\f\f\t\2WWeekknouuwwzz\5\2C\\aac|\6\2\62;C\\aac"+
+    "|\u026b\2\4\3\2\2\2\2\6\3\2\2\2\2\b\3\2\2\2\2\n\3\2\2\2\2\f\3\2\2\2\2"+
+    "\16\3\2\2\2\2\20\3\2\2\2\2\22\3\2\2\2\2\24\3\2\2\2\2\26\3\2\2\2\2\30\3"+
+    "\2\2\2\2\32\3\2\2\2\2\34\3\2\2\2\2\36\3\2\2\2\2 \3\2\2\2\2\"\3\2\2\2\2"+
+    "$\3\2\2\2\2&\3\2\2\2\2(\3\2\2\2\2*\3\2\2\2\2,\3\2\2\2\2.\3\2\2\2\2\60"+
+    "\3\2\2\2\2\62\3\2\2\2\2\64\3\2\2\2\2\66\3\2\2\2\28\3\2\2\2\2:\3\2\2\2"+
+    "\2<\3\2\2\2\2>\3\2\2\2\2@\3\2\2\2\2B\3\2\2\2\2D\3\2\2\2\2F\3\2\2\2\2H"+
+    "\3\2\2\2\2J\3\2\2\2\2L\3\2\2\2\2N\3\2\2\2\2P\3\2\2\2\2R\3\2\2\2\2T\3\2"+
+    "\2\2\2V\3\2\2\2\2X\3\2\2\2\2Z\3\2\2\2\2\\\3\2\2\2\2^\3\2\2\2\2`\3\2\2"+
+    "\2\2b\3\2\2\2\2d\3\2\2\2\2f\3\2\2\2\2h\3\2\2\2\2j\3\2\2\2\2l\3\2\2\2\2"+
+    "n\3\2\2\2\2p\3\2\2\2\2r\3\2\2\2\2t\3\2\2\2\2v\3\2\2\2\2x\3\2\2\2\2z\3"+
+    "\2\2\2\2|\3\2\2\2\2~\3\2\2\2\2\u0080\3\2\2\2\2\u0082\3\2\2\2\2\u0084\3"+
+    "\2\2\2\2\u0086\3\2\2\2\2\u0088\3\2\2\2\2\u008a\3\2\2\2\2\u008c\3\2\2\2"+
+    "\2\u008e\3\2\2\2\2\u0090\3\2\2\2\2\u0092\3\2\2\2\2\u0094\3\2\2\2\2\u0096"+
+    "\3\2\2\2\2\u0098\3\2\2\2\2\u009a\3\2\2\2\2\u009c\3\2\2\2\2\u009e\3\2\2"+
+    "\2\2\u00a0\3\2\2\2\2\u00a2\3\2\2\2\3\u00a4\3\2\2\2\3\u00a6\3\2\2\2\4\u00a9"+
+    "\3\2\2\2\6\u00c4\3\2\2\2\b\u00c8\3\2\2\2\n\u00ca\3\2\2\2\f\u00cc\3\2\2"+
+    "\2\16\u00ce\3\2\2\2\20\u00d0\3\2\2\2\22\u00d2\3\2\2\2\24\u00d4\3\2\2\2"+
+    "\26\u00d8\3\2\2\2\30\u00da\3\2\2\2\32\u00dc\3\2\2\2\34\u00df\3\2\2\2\36"+
+    "\u00e2\3\2\2\2 \u00e7\3\2\2\2\"\u00ed\3\2\2\2$\u00f0\3\2\2\2&\u00f4\3"+
+    "\2\2\2(\u00fd\3\2\2\2*\u0103\3\2\2\2,\u010a\3\2\2\2.\u010e\3\2\2\2\60"+
+    "\u0112\3\2\2\2\62\u0118\3\2\2\2\64\u011e\3\2\2\2\66\u0123\3\2\2\28\u012e"+
+    "\3\2\2\2:\u0130\3\2\2\2<\u0132\3\2\2\2>\u0134\3\2\2\2@\u0137\3\2\2\2B"+
+    "\u0139\3\2\2\2D\u013b\3\2\2\2F\u013d\3\2\2\2H\u0140\3\2\2\2J\u0143\3\2"+
+    "\2\2L\u0147\3\2\2\2N\u0149\3\2\2\2P\u014c\3\2\2\2R\u014e\3\2\2\2T\u0151"+
+    "\3\2\2\2V\u0154\3\2\2\2X\u0158\3\2\2\2Z\u015b\3\2\2\2\\\u015f\3\2\2\2"+
+    "^\u0161\3\2\2\2`\u0163\3\2\2\2b\u0165\3\2\2\2d\u0168\3\2\2\2f\u016b\3"+
+    "\2\2\2h\u016d\3\2\2\2j\u016f\3\2\2\2l\u0172\3\2\2\2n\u0175\3\2\2\2p\u0178"+
+    "\3\2\2\2r\u017c\3\2\2\2t\u017f\3\2\2\2v\u0182\3\2\2\2x\u0184\3\2\2\2z"+
+    "\u0187\3\2\2\2|\u018a\3\2\2\2~\u018d\3\2\2\2\u0080\u0190\3\2\2\2\u0082"+
+    "\u0193\3\2\2\2\u0084\u0196\3\2\2\2\u0086\u0199\3\2\2\2\u0088\u019c\3\2"+
+    "\2\2\u008a\u01a0\3\2\2\2\u008c\u01a4\3\2\2\2\u008e\u01a9\3\2\2\2\u0090"+
+    "\u01b2\3\2\2\2\u0092\u01c4\3\2\2\2\u0094\u01d1\3\2\2\2\u0096\u0201\3\2"+
+    "\2\2\u0098\u0203\3\2\2\2\u009a\u0214\3\2\2\2\u009c\u0219\3\2\2\2\u009e"+
+    "\u021f\3\2\2\2\u00a0\u0224\3\2\2\2\u00a2\u022f\3\2\2\2\u00a4\u023e\3\2"+
+    "\2\2\u00a6\u0242\3\2\2\2\u00a8\u00aa\t\2\2\2\u00a9\u00a8\3\2\2\2\u00aa"+
+    "\u00ab\3\2\2\2\u00ab\u00a9\3\2\2\2\u00ab\u00ac\3\2\2\2\u00ac\u00ad\3\2"+
+    "\2\2\u00ad\u00ae\b\2\2\2\u00ae\5\3\2\2\2\u00af\u00b0\7\61\2\2\u00b0\u00b1"+
+    "\7\61\2\2\u00b1\u00b5\3\2\2\2\u00b2\u00b4\13\2\2\2\u00b3\u00b2\3\2\2\2"+
+    "\u00b4\u00b7\3\2\2\2\u00b5\u00b6\3\2\2\2\u00b5\u00b3\3\2\2\2\u00b6\u00b8"+
+    "\3\2\2\2\u00b7\u00b5\3\2\2\2\u00b8\u00c5\t\3\2\2\u00b9\u00ba\7\61\2\2"+
+    "\u00ba\u00bb\7,\2\2\u00bb\u00bf\3\2\2\2\u00bc\u00be\13\2\2\2\u00bd\u00bc"+
+    "\3\2\2\2\u00be\u00c1\3\2\2\2\u00bf\u00c0\3\2\2\2\u00bf\u00bd\3\2\2\2\u00c0"+
+    "\u00c2\3\2\2\2\u00c1\u00bf\3\2\2\2\u00c2\u00c3\7,\2\2\u00c3\u00c5\7\61"+
+    "\2\2\u00c4\u00af\3\2\2\2\u00c4\u00b9\3\2\2\2\u00c5\u00c6\3\2\2\2\u00c6"+
+    "\u00c7\b\3\2\2\u00c7\7\3\2\2\2\u00c8\u00c9\7}\2\2\u00c9\t\3\2\2\2\u00ca"+
+    "\u00cb\7\177\2\2\u00cb\13\3\2\2\2\u00cc\u00cd\7]\2\2\u00cd\r\3\2\2\2\u00ce"+
+    "\u00cf\7_\2\2\u00cf\17\3\2\2\2\u00d0\u00d1\7*\2\2\u00d1\21\3\2\2\2\u00d2"+
+    "\u00d3\7+\2\2\u00d3\23\3\2\2\2\u00d4\u00d5\7\60\2\2\u00d5\u00d6\3\2\2"+
+    "\2\u00d6\u00d7\b\n\3\2\u00d7\25\3\2\2\2\u00d8\u00d9\7.\2\2\u00d9\27\3"+
+    "\2\2\2\u00da\u00db\7=\2\2\u00db\31\3\2\2\2\u00dc\u00dd\7k\2\2\u00dd\u00de"+
+    "\7h\2\2\u00de\33\3\2\2\2\u00df\u00e0\7k\2\2\u00e0\u00e1\7p\2\2\u00e1\35"+
+    "\3\2\2\2\u00e2\u00e3\7g\2\2\u00e3\u00e4\7n\2\2\u00e4\u00e5\7u\2\2\u00e5"+
+    "\u00e6\7g\2\2\u00e6\37\3\2\2\2\u00e7\u00e8\7y\2\2\u00e8\u00e9\7j\2\2\u00e9"+
+    "\u00ea\7k\2\2\u00ea\u00eb\7n\2\2\u00eb\u00ec\7g\2\2\u00ec!\3\2\2\2\u00ed"+
+    "\u00ee\7f\2\2\u00ee\u00ef\7q\2\2\u00ef#\3\2\2\2\u00f0\u00f1\7h\2\2\u00f1"+
+    "\u00f2\7q\2\2\u00f2\u00f3\7t\2\2\u00f3%\3\2\2\2\u00f4\u00f5\7e\2\2\u00f5"+
+    "\u00f6\7q\2\2\u00f6\u00f7\7p\2\2\u00f7\u00f8\7v\2\2\u00f8\u00f9\7k\2\2"+
+    "\u00f9\u00fa\7p\2\2\u00fa\u00fb\7w\2\2\u00fb\u00fc\7g\2\2\u00fc\'\3\2"+
+    "\2\2\u00fd\u00fe\7d\2\2\u00fe\u00ff\7t\2\2\u00ff\u0100\7g\2\2\u0100\u0101"+
+    "\7c\2\2\u0101\u0102\7m\2\2\u0102)\3\2\2\2\u0103\u0104\7t\2\2\u0104\u0105"+
+    "\7g\2\2\u0105\u0106\7v\2\2\u0106\u0107\7w\2\2\u0107\u0108\7t\2\2\u0108"+
+    "\u0109\7p\2\2\u0109+\3\2\2\2\u010a\u010b\7p\2\2\u010b\u010c\7g\2\2\u010c"+
+    "\u010d\7y\2\2\u010d-\3\2\2\2\u010e\u010f\7v\2\2\u010f\u0110\7t\2\2\u0110"+
+    "\u0111\7{\2\2\u0111/\3\2\2\2\u0112\u0113\7e\2\2\u0113\u0114\7c\2\2\u0114"+
+    "\u0115\7v\2\2\u0115\u0116\7e\2\2\u0116\u0117\7j\2\2\u0117\61\3\2\2\2\u0118"+
+    "\u0119\7v\2\2\u0119\u011a\7j\2\2\u011a\u011b\7t\2\2\u011b\u011c\7q\2\2"+
+    "\u011c\u011d\7y\2\2\u011d\63\3\2\2\2\u011e\u011f\7v\2\2\u011f\u0120\7"+
+    "j\2\2\u0120\u0121\7k\2\2\u0121\u0122\7u\2\2\u0122\65\3\2\2\2\u0123\u0124"+
+    "\7k\2\2\u0124\u0125\7p\2\2\u0125\u0126\7u\2\2\u0126\u0127\7v\2\2\u0127"+
+    "\u0128\7c\2\2\u0128\u0129\7p\2\2\u0129\u012a\7e\2\2\u012a\u012b\7g\2\2"+
+    "\u012b\u012c\7q\2\2\u012c\u012d\7h\2\2\u012d\67\3\2\2\2\u012e\u012f\7"+
+    "#\2\2\u012f9\3\2\2\2\u0130\u0131\7\u0080\2\2\u0131;\3\2\2\2\u0132\u0133"+
+    "\7,\2\2\u0133=\3\2\2\2\u0134\u0135\7\61\2\2\u0135\u0136\6\37\2\2\u0136"+
+    "?\3\2\2\2\u0137\u0138\7\'\2\2\u0138A\3\2\2\2\u0139\u013a\7-\2\2\u013a"+
+    "C\3\2\2\2\u013b\u013c\7/\2\2\u013cE\3\2\2\2\u013d\u013e\7>\2\2\u013e\u013f"+
+    "\7>\2\2\u013fG\3\2\2\2\u0140\u0141\7@\2\2\u0141\u0142\7@\2\2\u0142I\3"+
+    "\2\2\2\u0143\u0144\7@\2\2\u0144\u0145\7@\2\2\u0145\u0146\7@\2\2\u0146"+
+    "K\3\2\2\2\u0147\u0148\7>\2\2\u0148M\3\2\2\2\u0149\u014a\7>\2\2\u014a\u014b"+
+    "\7?\2\2\u014bO\3\2\2\2\u014c\u014d\7@\2\2\u014dQ\3\2\2\2\u014e\u014f\7"+
+    "@\2\2\u014f\u0150\7?\2\2\u0150S\3\2\2\2\u0151\u0152\7?\2\2\u0152\u0153"+
+    "\7?\2\2\u0153U\3\2\2\2\u0154\u0155\7?\2\2\u0155\u0156\7?\2\2\u0156\u0157"+
+    "\7?\2\2\u0157W\3\2\2\2\u0158\u0159\7#\2\2\u0159\u015a\7?\2\2\u015aY\3"+
+    "\2\2\2\u015b\u015c\7#\2\2\u015c\u015d\7?\2\2\u015d\u015e\7?\2\2\u015e"+
+    "[\3\2\2\2\u015f\u0160\7(\2\2\u0160]\3\2\2\2\u0161\u0162\7`\2\2\u0162_"+
+    "\3\2\2\2\u0163\u0164\7~\2\2\u0164a\3\2\2\2\u0165\u0166\7(\2\2\u0166\u0167"+
+    "\7(\2\2\u0167c\3\2\2\2\u0168\u0169\7~\2\2\u0169\u016a\7~\2\2\u016ae\3"+
+    "\2\2\2\u016b\u016c\7A\2\2\u016cg\3\2\2\2\u016d\u016e\7<\2\2\u016ei\3\2"+
+    "\2\2\u016f\u0170\7<\2\2\u0170\u0171\7<\2\2\u0171k\3\2\2\2\u0172\u0173"+
+    "\7/\2\2\u0173\u0174\7@\2\2\u0174m\3\2\2\2\u0175\u0176\7?\2\2\u0176\u0177"+
+    "\7\u0080\2\2\u0177o\3\2\2\2\u0178\u0179\7?\2\2\u0179\u017a\7?\2\2\u017a"+
+    "\u017b\7\u0080\2\2\u017bq\3\2\2\2\u017c\u017d\7-\2\2\u017d\u017e\7-\2"+
+    "\2\u017es\3\2\2\2\u017f\u0180\7/\2\2\u0180\u0181\7/\2\2\u0181u\3\2\2\2"+
+    "\u0182\u0183\7?\2\2\u0183w\3\2\2\2\u0184\u0185\7-\2\2\u0185\u0186\7?\2"+
+    "\2\u0186y\3\2\2\2\u0187\u0188\7/\2\2\u0188\u0189\7?\2\2\u0189{\3\2\2\2"+
+    "\u018a\u018b\7,\2\2\u018b\u018c\7?\2\2\u018c}\3\2\2\2\u018d\u018e\7\61"+
+    "\2\2\u018e\u018f\7?\2\2\u018f\177\3\2\2\2\u0190\u0191\7\'\2\2\u0191\u0192"+
+    "\7?\2\2\u0192\u0081\3\2\2\2\u0193\u0194\7(\2\2\u0194\u0195\7?\2\2\u0195"+
+    "\u0083\3\2\2\2\u0196\u0197\7`\2\2\u0197\u0198\7?\2\2\u0198\u0085\3\2\2"+
+    "\2\u0199\u019a\7~\2\2\u019a\u019b\7?\2\2\u019b\u0087\3\2\2\2\u019c\u019d"+
+    "\7>\2\2\u019d\u019e\7>\2\2\u019e\u019f\7?\2\2\u019f\u0089\3\2\2\2\u01a0"+
+    "\u01a1\7@\2\2\u01a1\u01a2\7@\2\2\u01a2\u01a3\7?\2\2\u01a3\u008b\3\2\2"+
+    "\2\u01a4\u01a5\7@\2\2\u01a5\u01a6\7@\2\2\u01a6\u01a7\7@\2\2\u01a7\u01a8"+
+    "\7?\2\2\u01a8\u008d\3\2\2\2\u01a9\u01ab\7\62\2\2\u01aa\u01ac\t\4\2\2\u01ab"+
+    "\u01aa\3\2\2\2\u01ac\u01ad\3\2\2\2\u01ad\u01ab\3\2\2\2\u01ad\u01ae\3\2"+
+    "\2\2\u01ae\u01b0\3\2\2\2\u01af\u01b1\t\5\2\2\u01b0\u01af\3\2\2\2\u01b0"+
+    "\u01b1\3\2\2\2\u01b1\u008f\3\2\2\2\u01b2\u01b3\7\62\2\2\u01b3\u01b5\t"+
+    "\6\2\2\u01b4\u01b6\t\7\2\2\u01b5\u01b4\3\2\2\2\u01b6\u01b7\3\2\2\2\u01b7"+
+    "\u01b5\3\2\2\2\u01b7\u01b8\3\2\2\2\u01b8\u01ba\3\2\2\2\u01b9\u01bb\t\5"+
+    "\2\2\u01ba\u01b9\3\2\2\2\u01ba\u01bb\3\2\2\2\u01bb\u0091\3\2\2\2\u01bc"+
+    "\u01c5\7\62\2\2\u01bd\u01c1\t\b\2\2\u01be\u01c0\t\t\2\2\u01bf\u01be\3"+
+    "\2\2\2\u01c0\u01c3\3\2\2\2\u01c1\u01bf\3\2\2\2\u01c1\u01c2\3\2\2\2\u01c2"+
+    "\u01c5\3\2\2\2\u01c3\u01c1\3\2\2\2\u01c4\u01bc\3\2\2\2\u01c4\u01bd\3\2"+
+    "\2\2\u01c5\u01c7\3\2\2\2\u01c6\u01c8\t\n\2\2\u01c7\u01c6\3\2\2\2\u01c7"+
+    "\u01c8\3\2\2\2\u01c8\u0093\3\2\2\2\u01c9\u01d2\7\62\2\2\u01ca\u01ce\t"+
+    "\b\2\2\u01cb\u01cd\t\t\2\2\u01cc\u01cb\3\2\2\2\u01cd\u01d0\3\2\2\2\u01ce"+
+    "\u01cc\3\2\2\2\u01ce\u01cf\3\2\2\2\u01cf\u01d2\3\2\2\2\u01d0\u01ce\3\2"+
+    "\2\2\u01d1\u01c9\3\2\2\2\u01d1\u01ca\3\2\2\2\u01d2\u01d9\3\2\2\2\u01d3"+
+    "\u01d5\5\24\n\2\u01d4\u01d6\t\t\2\2\u01d5\u01d4\3\2\2\2\u01d6\u01d7\3"+
+    "\2\2\2\u01d7\u01d5\3\2\2\2\u01d7\u01d8\3\2\2\2\u01d8\u01da\3\2\2\2\u01d9"+
+    "\u01d3\3\2\2\2\u01d9\u01da\3\2\2\2\u01da\u01e4\3\2\2\2\u01db\u01dd\t\13"+
+    "\2\2\u01dc\u01de\t\f\2\2\u01dd\u01dc\3\2\2\2\u01dd\u01de\3\2\2\2\u01de"+
+    "\u01e0\3\2\2\2\u01df\u01e1\t\t\2\2\u01e0\u01df\3\2\2\2\u01e1\u01e2\3\2"+
+    "\2\2\u01e2\u01e0\3\2\2\2\u01e2\u01e3\3\2\2\2\u01e3\u01e5\3\2\2\2\u01e4"+
+    "\u01db\3\2\2\2\u01e4\u01e5\3\2\2\2\u01e5\u01e7\3\2\2\2\u01e6\u01e8\t\r"+
+    "\2\2\u01e7\u01e6\3\2\2\2\u01e7\u01e8\3\2\2\2\u01e8\u0095\3\2\2\2\u01e9"+
+    "\u01f1\7$\2\2\u01ea\u01eb\7^\2\2\u01eb\u01f0\7$\2\2\u01ec\u01ed\7^\2\2"+
+    "\u01ed\u01f0\7^\2\2\u01ee\u01f0\n\16\2\2\u01ef\u01ea\3\2\2\2\u01ef\u01ec"+
+    "\3\2\2\2\u01ef\u01ee\3\2\2\2\u01f0\u01f3\3\2\2\2\u01f1\u01f2\3\2\2\2\u01f1"+
+    "\u01ef\3\2\2\2\u01f2\u01f4\3\2\2\2\u01f3\u01f1\3\2\2\2\u01f4\u0202\7$"+
+    "\2\2\u01f5\u01fd\7)\2\2\u01f6\u01f7\7^\2\2\u01f7\u01fc\7)\2\2\u01f8\u01f9"+
+    "\7^\2\2\u01f9\u01fc\7^\2\2\u01fa\u01fc\n\16\2\2\u01fb\u01f6\3\2\2\2\u01fb"+
+    "\u01f8\3\2\2\2\u01fb\u01fa\3\2\2\2\u01fc\u01ff\3\2\2\2\u01fd\u01fe\3\2"+
+    "\2\2\u01fd\u01fb\3\2\2\2\u01fe\u0200\3\2\2\2\u01ff\u01fd\3\2\2\2\u0200"+
+    "\u0202\7)\2\2\u0201\u01e9\3\2\2\2\u0201\u01f5\3\2\2\2\u0202\u0097\3\2"+
+    "\2\2\u0203\u0207\7\61\2\2\u0204\u0208\n\17\2\2\u0205\u0206\7^\2\2\u0206"+
+    "\u0208\n\20\2\2\u0207\u0204\3\2\2\2\u0207\u0205\3\2\2\2\u0208\u0209\3"+
+    "\2\2\2\u0209\u0207\3\2\2\2\u0209\u020a\3\2\2\2\u020a\u020b\3\2\2\2\u020b"+
+    "\u020f\7\61\2\2\u020c\u020e\t\21\2\2\u020d\u020c\3\2\2\2\u020e\u0211\3"+
+    "\2\2\2\u020f\u020d\3\2\2\2\u020f\u0210\3\2\2\2\u0210\u0212\3\2\2\2\u0211"+
+    "\u020f\3\2\2\2\u0212\u0213\6L\3\2\u0213\u0099\3\2\2\2\u0214\u0215\7v\2"+
+    "\2\u0215\u0216\7t\2\2\u0216\u0217\7w\2\2\u0217\u0218\7g\2\2\u0218\u009b"+
+    "\3\2\2\2\u0219\u021a\7h\2\2\u021a\u021b\7c\2\2\u021b\u021c\7n\2\2\u021c"+
+    "\u021d\7u\2\2\u021d\u021e\7g\2\2\u021e\u009d\3\2\2\2\u021f\u0220\7p\2"+
+    "\2\u0220\u0221\7w\2\2\u0221\u0222\7n\2\2\u0222\u0223\7n\2\2\u0223\u009f"+
+    "\3\2\2\2\u0224\u022a\5\u00a2Q\2\u0225\u0226\5\24\n\2\u0226\u0227\5\u00a2"+
+    "Q\2\u0227\u0229\3\2\2\2\u0228\u0225\3\2\2\2\u0229\u022c\3\2\2\2\u022a"+
+    "\u0228\3\2\2\2\u022a\u022b\3\2\2\2\u022b\u022d\3\2\2\2\u022c\u022a\3\2"+
+    "\2\2\u022d\u022e\6P\4\2\u022e\u00a1\3\2\2\2\u022f\u0233\t\22\2\2\u0230"+
+    "\u0232\t\23\2\2\u0231\u0230\3\2\2\2\u0232\u0235\3\2\2\2\u0233\u0231\3"+
+    "\2\2\2\u0233\u0234\3\2\2\2\u0234\u00a3\3\2\2\2\u0235\u0233\3\2\2\2\u0236"+
+    "\u023f\7\62\2\2\u0237\u023b\t\b\2\2\u0238\u023a\t\t\2\2\u0239\u0238\3"+
+    "\2\2\2\u023a\u023d\3\2\2\2\u023b\u0239\3\2\2\2\u023b\u023c\3\2\2\2\u023c"+
+    "\u023f\3\2\2\2\u023d\u023b\3\2\2\2\u023e\u0236\3\2\2\2\u023e\u0237\3\2"+
+    "\2\2\u023f\u0240\3\2\2\2\u0240\u0241\bR\4\2\u0241\u00a5\3\2\2\2\u0242"+
+    "\u0246\t\22\2\2\u0243\u0245\t\23\2\2\u0244\u0243\3\2\2\2\u0245\u0248\3"+
+    "\2\2\2\u0246\u0244\3\2\2\2\u0246\u0247\3\2\2\2\u0247\u0249\3\2\2\2\u0248"+
+    "\u0246\3\2\2\2\u0249\u024a\bS\4\2\u024a\u00a7\3\2\2\2$\2\3\u00ab\u00b5"+
+    "\u00bf\u00c4\u01ad\u01b0\u01b7\u01ba\u01c1\u01c4\u01c7\u01ce\u01d1\u01d7"+
+    "\u01d9\u01dd\u01e2\u01e4\u01e7\u01ef\u01f1\u01fb\u01fd\u0201\u0207\u0209"+
+    "\u020f\u022a\u0233\u023b\u023e\u0246\5\b\2\2\4\3\2\4\2\2";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
index c1b698a9530..933a5f35dcd 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParser.java
@@ -18,15 +18,16 @@ class PainlessParser extends Parser {
     new PredictionContextCache();
   public static final int
     WS=1, COMMENT=2, LBRACK=3, RBRACK=4, LBRACE=5, RBRACE=6, LP=7, RP=8, DOT=9, 
-    COMMA=10, SEMICOLON=11, IF=12, ELSE=13, WHILE=14, DO=15, FOR=16, CONTINUE=17, 
-    BREAK=18, RETURN=19, NEW=20, TRY=21, CATCH=22, THROW=23, THIS=24, INSTANCEOF=25, 
-    BOOLNOT=26, BWNOT=27, MUL=28, DIV=29, REM=30, ADD=31, SUB=32, LSH=33, 
-    RSH=34, USH=35, LT=36, LTE=37, GT=38, GTE=39, EQ=40, EQR=41, NE=42, NER=43, 
-    BWAND=44, XOR=45, BWOR=46, BOOLAND=47, BOOLOR=48, COND=49, COLON=50, REF=51, 
-    ARROW=52, FIND=53, MATCH=54, INCR=55, DECR=56, ASSIGN=57, AADD=58, ASUB=59, 
-    AMUL=60, ADIV=61, AREM=62, AAND=63, AXOR=64, AOR=65, ALSH=66, ARSH=67, 
-    AUSH=68, OCTAL=69, HEX=70, INTEGER=71, DECIMAL=72, STRING=73, REGEX=74, 
-    TRUE=75, FALSE=76, NULL=77, TYPE=78, ID=79, DOTINTEGER=80, DOTID=81;
+    COMMA=10, SEMICOLON=11, IF=12, IN=13, ELSE=14, WHILE=15, DO=16, FOR=17, 
+    CONTINUE=18, BREAK=19, RETURN=20, NEW=21, TRY=22, CATCH=23, THROW=24, 
+    THIS=25, INSTANCEOF=26, BOOLNOT=27, BWNOT=28, MUL=29, DIV=30, REM=31, 
+    ADD=32, SUB=33, LSH=34, RSH=35, USH=36, LT=37, LTE=38, GT=39, GTE=40, 
+    EQ=41, EQR=42, NE=43, NER=44, BWAND=45, XOR=46, BWOR=47, BOOLAND=48, BOOLOR=49, 
+    COND=50, COLON=51, REF=52, ARROW=53, FIND=54, MATCH=55, INCR=56, DECR=57, 
+    ASSIGN=58, AADD=59, ASUB=60, AMUL=61, ADIV=62, AREM=63, AAND=64, AXOR=65, 
+    AOR=66, ALSH=67, ARSH=68, AUSH=69, OCTAL=70, HEX=71, INTEGER=72, DECIMAL=73, 
+    STRING=74, REGEX=75, TRUE=76, FALSE=77, NULL=78, TYPE=79, ID=80, DOTINTEGER=81, 
+    DOTID=82;
   public static final int
     RULE_source = 0, RULE_function = 1, RULE_parameters = 2, RULE_statement = 3, 
     RULE_trailer = 4, RULE_block = 5, RULE_empty = 6, RULE_initializer = 7, 
@@ -48,25 +49,25 @@ class PainlessParser extends Parser {
 
   private static final String[] _LITERAL_NAMES = {
     null, null, null, "'{'", "'}'", "'['", "']'", "'('", "')'", "'.'", "','", 
-    "';'", "'if'", "'else'", "'while'", "'do'", "'for'", "'continue'", "'break'", 
-    "'return'", "'new'", "'try'", "'catch'", "'throw'", "'this'", "'instanceof'", 
-    "'!'", "'~'", "'*'", "'/'", "'%'", "'+'", "'-'", "'<<'", "'>>'", "'>>>'", 
-    "'<'", "'<='", "'>'", "'>='", "'=='", "'==='", "'!='", "'!=='", "'&'", 
-    "'^'", "'|'", "'&&'", "'||'", "'?'", "':'", "'::'", "'->'", "'=~'", "'==~'", 
-    "'++'", "'--'", "'='", "'+='", "'-='", "'*='", "'/='", "'%='", "'&='", 
-    "'^='", "'|='", "'<<='", "'>>='", "'>>>='", null, null, null, null, null, 
-    null, "'true'", "'false'", "'null'"
+    "';'", "'if'", "'in'", "'else'", "'while'", "'do'", "'for'", "'continue'", 
+    "'break'", "'return'", "'new'", "'try'", "'catch'", "'throw'", "'this'", 
+    "'instanceof'", "'!'", "'~'", "'*'", "'/'", "'%'", "'+'", "'-'", "'<<'", 
+    "'>>'", "'>>>'", "'<'", "'<='", "'>'", "'>='", "'=='", "'==='", "'!='", 
+    "'!=='", "'&'", "'^'", "'|'", "'&&'", "'||'", "'?'", "':'", "'::'", "'->'", 
+    "'=~'", "'==~'", "'++'", "'--'", "'='", "'+='", "'-='", "'*='", "'/='", 
+    "'%='", "'&='", "'^='", "'|='", "'<<='", "'>>='", "'>>>='", null, null, 
+    null, null, null, null, "'true'", "'false'", "'null'"
   };
   private static final String[] _SYMBOLIC_NAMES = {
     null, "WS", "COMMENT", "LBRACK", "RBRACK", "LBRACE", "RBRACE", "LP", "RP", 
-    "DOT", "COMMA", "SEMICOLON", "IF", "ELSE", "WHILE", "DO", "FOR", "CONTINUE", 
-    "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", "INSTANCEOF", 
-    "BOOLNOT", "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", "RSH", "USH", 
-    "LT", "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", "XOR", "BWOR", 
-    "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "FIND", "MATCH", 
-    "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", "AAND", 
-    "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", "DECIMAL", 
-    "STRING", "REGEX", "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", 
+    "DOT", "COMMA", "SEMICOLON", "IF", "IN", "ELSE", "WHILE", "DO", "FOR", 
+    "CONTINUE", "BREAK", "RETURN", "NEW", "TRY", "CATCH", "THROW", "THIS", 
+    "INSTANCEOF", "BOOLNOT", "BWNOT", "MUL", "DIV", "REM", "ADD", "SUB", "LSH", 
+    "RSH", "USH", "LT", "LTE", "GT", "GTE", "EQ", "EQR", "NE", "NER", "BWAND", 
+    "XOR", "BWOR", "BOOLAND", "BOOLOR", "COND", "COLON", "REF", "ARROW", "FIND", 
+    "MATCH", "INCR", "DECR", "ASSIGN", "AADD", "ASUB", "AMUL", "ADIV", "AREM", 
+    "AAND", "AXOR", "AOR", "ALSH", "ARSH", "AUSH", "OCTAL", "HEX", "INTEGER", 
+    "DECIMAL", "STRING", "REGEX", "TRUE", "FALSE", "NULL", "TYPE", "ID", "DOTINTEGER", 
     "DOTID"
   };
   public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES);
@@ -364,33 +365,6 @@ class PainlessParser extends Parser {
       else return visitor.visitChildren(this);
     }
   }
-  public static class ThrowContext extends StatementContext {
-    public TerminalNode THROW() { return getToken(PainlessParser.THROW, 0); }
-    public ExpressionContext expression() {
-      return getRuleContext(ExpressionContext.class,0);
-    }
-    public DelimiterContext delimiter() {
-      return getRuleContext(DelimiterContext.class,0);
-    }
-    public ThrowContext(StatementContext ctx) { copyFrom(ctx); }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitThrow(this);
-      else return visitor.visitChildren(this);
-    }
-  }
-  public static class ContinueContext extends StatementContext {
-    public TerminalNode CONTINUE() { return getToken(PainlessParser.CONTINUE, 0); }
-    public DelimiterContext delimiter() {
-      return getRuleContext(DelimiterContext.class,0);
-    }
-    public ContinueContext(StatementContext ctx) { copyFrom(ctx); }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitContinue(this);
-      else return visitor.visitChildren(this);
-    }
-  }
   public static class ForContext extends StatementContext {
     public TerminalNode FOR() { return getToken(PainlessParser.FOR, 0); }
     public TerminalNode LP() { return getToken(PainlessParser.LP, 0); }
@@ -421,38 +395,6 @@ class PainlessParser extends Parser {
       else return visitor.visitChildren(this);
     }
   }
-  public static class TryContext extends StatementContext {
-    public TerminalNode TRY() { return getToken(PainlessParser.TRY, 0); }
-    public BlockContext block() {
-      return getRuleContext(BlockContext.class,0);
-    }
-    public List<TrapContext> trap() {
-      return getRuleContexts(TrapContext.class);
-    }
-    public TrapContext trap(int i) {
-      return getRuleContext(TrapContext.class,i);
-    }
-    public TryContext(StatementContext ctx) { copyFrom(ctx); }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitTry(this);
-      else return visitor.visitChildren(this);
-    }
-  }
-  public static class ExprContext extends StatementContext {
-    public ExpressionContext expression() {
-      return getRuleContext(ExpressionContext.class,0);
-    }
-    public DelimiterContext delimiter() {
-      return getRuleContext(DelimiterContext.class,0);
-    }
-    public ExprContext(StatementContext ctx) { copyFrom(ctx); }
-    @Override
-    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitExpr(this);
-      else return visitor.visitChildren(this);
-    }
-  }
   public static class DoContext extends StatementContext {
     public TerminalNode DO() { return getToken(PainlessParser.DO, 0); }
     public BlockContext block() {
@@ -494,24 +436,22 @@ class PainlessParser extends Parser {
       else return visitor.visitChildren(this);
     }
   }
-  public static class IfContext extends StatementContext {
-    public TerminalNode IF() { return getToken(PainlessParser.IF, 0); }
+  public static class IneachContext extends StatementContext {
+    public TerminalNode FOR() { return getToken(PainlessParser.FOR, 0); }
     public TerminalNode LP() { return getToken(PainlessParser.LP, 0); }
+    public TerminalNode ID() { return getToken(PainlessParser.ID, 0); }
+    public TerminalNode IN() { return getToken(PainlessParser.IN, 0); }
     public ExpressionContext expression() {
       return getRuleContext(ExpressionContext.class,0);
     }
     public TerminalNode RP() { return getToken(PainlessParser.RP, 0); }
-    public List<TrailerContext> trailer() {
-      return getRuleContexts(TrailerContext.class);
+    public TrailerContext trailer() {
+      return getRuleContext(TrailerContext.class,0);
     }
-    public TrailerContext trailer(int i) {
-      return getRuleContext(TrailerContext.class,i);
-    }
-    public TerminalNode ELSE() { return getToken(PainlessParser.ELSE, 0); }
-    public IfContext(StatementContext ctx) { copyFrom(ctx); }
+    public IneachContext(StatementContext ctx) { copyFrom(ctx); }
     @Override
     public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
-      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitIf(this);
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitIneach(this);
       else return visitor.visitChildren(this);
     }
   }
@@ -537,6 +477,86 @@ class PainlessParser extends Parser {
       else return visitor.visitChildren(this);
     }
   }
+  public static class ThrowContext extends StatementContext {
+    public TerminalNode THROW() { return getToken(PainlessParser.THROW, 0); }
+    public ExpressionContext expression() {
+      return getRuleContext(ExpressionContext.class,0);
+    }
+    public DelimiterContext delimiter() {
+      return getRuleContext(DelimiterContext.class,0);
+    }
+    public ThrowContext(StatementContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitThrow(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+  public static class ContinueContext extends StatementContext {
+    public TerminalNode CONTINUE() { return getToken(PainlessParser.CONTINUE, 0); }
+    public DelimiterContext delimiter() {
+      return getRuleContext(DelimiterContext.class,0);
+    }
+    public ContinueContext(StatementContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitContinue(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+  public static class TryContext extends StatementContext {
+    public TerminalNode TRY() { return getToken(PainlessParser.TRY, 0); }
+    public BlockContext block() {
+      return getRuleContext(BlockContext.class,0);
+    }
+    public List<TrapContext> trap() {
+      return getRuleContexts(TrapContext.class);
+    }
+    public TrapContext trap(int i) {
+      return getRuleContext(TrapContext.class,i);
+    }
+    public TryContext(StatementContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitTry(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+  public static class ExprContext extends StatementContext {
+    public ExpressionContext expression() {
+      return getRuleContext(ExpressionContext.class,0);
+    }
+    public DelimiterContext delimiter() {
+      return getRuleContext(DelimiterContext.class,0);
+    }
+    public ExprContext(StatementContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitExpr(this);
+      else return visitor.visitChildren(this);
+    }
+  }
+  public static class IfContext extends StatementContext {
+    public TerminalNode IF() { return getToken(PainlessParser.IF, 0); }
+    public TerminalNode LP() { return getToken(PainlessParser.LP, 0); }
+    public ExpressionContext expression() {
+      return getRuleContext(ExpressionContext.class,0);
+    }
+    public TerminalNode RP() { return getToken(PainlessParser.RP, 0); }
+    public List<TrailerContext> trailer() {
+      return getRuleContexts(TrailerContext.class);
+    }
+    public TrailerContext trailer(int i) {
+      return getRuleContext(TrailerContext.class,i);
+    }
+    public TerminalNode ELSE() { return getToken(PainlessParser.ELSE, 0); }
+    public IfContext(StatementContext ctx) { copyFrom(ctx); }
+    @Override
+    public <T> T accept(ParseTreeVisitor<? extends T> visitor) {
+      if ( visitor instanceof PainlessParserVisitor ) return ((PainlessParserVisitor<? extends T>)visitor).visitIf(this);
+      else return visitor.visitChildren(this);
+    }
+  }
   public static class ReturnContext extends StatementContext {
     public TerminalNode RETURN() { return getToken(PainlessParser.RETURN, 0); }
     public ExpressionContext expression() {
@@ -558,7 +578,7 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 6, RULE_statement);
     try {
       int _alt;
-      setState(181);
+      setState(189);
       switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) {
       case 1:
         _localctx = new IfContext(_localctx);
@@ -723,56 +743,76 @@ class PainlessParser extends Parser {
         }
         break;
       case 6:
-        _localctx = new DeclContext(_localctx);
+        _localctx = new IneachContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
         setState(156);
-        declaration();
+        match(FOR);
         setState(157);
-        delimiter();
+        match(LP);
+        setState(158);
+        match(ID);
+        setState(159);
+        match(IN);
+        setState(160);
+        expression(0);
+        setState(161);
+        match(RP);
+        setState(162);
+        trailer();
         }
         break;
       case 7:
-        _localctx = new ContinueContext(_localctx);
+        _localctx = new DeclContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(159);
-        match(CONTINUE);
-        setState(160);
-        delimiter();
-        }
-        break;
-      case 8:
-        _localctx = new BreakContext(_localctx);
-        enterOuterAlt(_localctx, 8);
-        {
-        setState(161);
-        match(BREAK);
-        setState(162);
-        delimiter();
-        }
-        break;
-      case 9:
-        _localctx = new ReturnContext(_localctx);
-        enterOuterAlt(_localctx, 9);
-        {
-        setState(163);
-        match(RETURN);
         setState(164);
-        expression(0);
+        declaration();
         setState(165);
         delimiter();
         }
         break;
-      case 10:
-        _localctx = new TryContext(_localctx);
-        enterOuterAlt(_localctx, 10);
+      case 8:
+        _localctx = new ContinueContext(_localctx);
+        enterOuterAlt(_localctx, 8);
         {
         setState(167);
-        match(TRY);
+        match(CONTINUE);
         setState(168);
+        delimiter();
+        }
+        break;
+      case 9:
+        _localctx = new BreakContext(_localctx);
+        enterOuterAlt(_localctx, 9);
+        {
+        setState(169);
+        match(BREAK);
+        setState(170);
+        delimiter();
+        }
+        break;
+      case 10:
+        _localctx = new ReturnContext(_localctx);
+        enterOuterAlt(_localctx, 10);
+        {
+        setState(171);
+        match(RETURN);
+        setState(172);
+        expression(0);
+        setState(173);
+        delimiter();
+        }
+        break;
+      case 11:
+        _localctx = new TryContext(_localctx);
+        enterOuterAlt(_localctx, 11);
+        {
+        setState(175);
+        match(TRY);
+        setState(176);
         block();
-        setState(170); 
+        setState(178); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -780,7 +820,7 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(169);
+            setState(177);
             trap();
             }
             }
@@ -788,31 +828,31 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(172); 
+          setState(180); 
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,10,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
         }
         break;
-      case 11:
+      case 12:
         _localctx = new ThrowContext(_localctx);
-        enterOuterAlt(_localctx, 11);
+        enterOuterAlt(_localctx, 12);
         {
-        setState(174);
+        setState(182);
         match(THROW);
-        setState(175);
+        setState(183);
         expression(0);
-        setState(176);
+        setState(184);
         delimiter();
         }
         break;
-      case 12:
+      case 13:
         _localctx = new ExprContext(_localctx);
-        enterOuterAlt(_localctx, 12);
+        enterOuterAlt(_localctx, 13);
         {
-        setState(178);
+        setState(186);
         expression(0);
-        setState(179);
+        setState(187);
         delimiter();
         }
         break;
@@ -851,19 +891,19 @@ class PainlessParser extends Parser {
     TrailerContext _localctx = new TrailerContext(_ctx, getState());
     enterRule(_localctx, 8, RULE_trailer);
     try {
-      setState(185);
+      setState(193);
       switch ( getInterpreter().adaptivePredict(_input,12,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(183);
+        setState(191);
         block();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(184);
+        setState(192);
         statement();
         }
         break;
@@ -907,25 +947,25 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(187);
+      setState(195);
       match(LBRACK);
-      setState(191);
+      setState(199);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,13,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(188);
+          setState(196);
           statement();
           }
           } 
         }
-        setState(193);
+        setState(201);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,13,_ctx);
       }
-      setState(194);
+      setState(202);
       match(RBRACK);
       }
     }
@@ -959,7 +999,7 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(196);
+      setState(204);
       match(SEMICOLON);
       }
     }
@@ -996,19 +1036,19 @@ class PainlessParser extends Parser {
     InitializerContext _localctx = new InitializerContext(_ctx, getState());
     enterRule(_localctx, 14, RULE_initializer);
     try {
-      setState(200);
+      setState(208);
       switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(198);
+        setState(206);
         declaration();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(199);
+        setState(207);
         expression(0);
         }
         break;
@@ -1046,7 +1086,7 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(202);
+      setState(210);
       expression(0);
       }
     }
@@ -1093,23 +1133,23 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(204);
+      setState(212);
       decltype();
-      setState(205);
+      setState(213);
       declvar();
-      setState(210);
+      setState(218);
       _errHandler.sync(this);
       _la = _input.LA(1);
       while (_la==COMMA) {
         {
         {
-        setState(206);
+        setState(214);
         match(COMMA);
-        setState(207);
+        setState(215);
         declvar();
         }
         }
-        setState(212);
+        setState(220);
         _errHandler.sync(this);
         _la = _input.LA(1);
       }
@@ -1154,23 +1194,23 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(213);
+      setState(221);
       match(TYPE);
-      setState(218);
+      setState(226);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,16,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
         if ( _alt==1 ) {
           {
           {
-          setState(214);
+          setState(222);
           match(LBRACE);
-          setState(215);
+          setState(223);
           match(RBRACE);
           }
           } 
         }
-        setState(220);
+        setState(228);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,16,_ctx);
       }
@@ -1211,15 +1251,15 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(221);
+      setState(229);
       match(ID);
-      setState(224);
+      setState(232);
       _la = _input.LA(1);
       if (_la==ASSIGN) {
         {
-        setState(222);
+        setState(230);
         match(ASSIGN);
-        setState(223);
+        setState(231);
         expression(0);
         }
       }
@@ -1263,17 +1303,17 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(226);
+      setState(234);
       match(CATCH);
-      setState(227);
+      setState(235);
       match(LP);
-      setState(228);
+      setState(236);
       match(TYPE);
-      setState(229);
+      setState(237);
       match(ID);
-      setState(230);
+      setState(238);
       match(RP);
-      setState(231);
+      setState(239);
       block();
       }
     }
@@ -1309,7 +1349,7 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(233);
+      setState(241);
       _la = _input.LA(1);
       if ( !(_la==EOF || _la==SEMICOLON) ) {
       _errHandler.recoverInline(this);
@@ -1495,7 +1535,7 @@ class PainlessParser extends Parser {
       int _alt;
       enterOuterAlt(_localctx, 1);
       {
-      setState(244);
+      setState(252);
       switch ( getInterpreter().adaptivePredict(_input,18,_ctx) ) {
       case 1:
         {
@@ -1503,16 +1543,16 @@ class PainlessParser extends Parser {
         _ctx = _localctx;
         _prevctx = _localctx;
 
-        setState(236);
+        setState(244);
         chain(true);
-        setState(237);
+        setState(245);
         _la = _input.LA(1);
-        if ( !(((((_la - 57)) & ~0x3f) == 0 && ((1L << (_la - 57)) & ((1L << (ASSIGN - 57)) | (1L << (AADD - 57)) | (1L << (ASUB - 57)) | (1L << (AMUL - 57)) | (1L << (ADIV - 57)) | (1L << (AREM - 57)) | (1L << (AAND - 57)) | (1L << (AXOR - 57)) | (1L << (AOR - 57)) | (1L << (ALSH - 57)) | (1L << (ARSH - 57)) | (1L << (AUSH - 57)))) != 0)) ) {
+        if ( !(((((_la - 58)) & ~0x3f) == 0 && ((1L << (_la - 58)) & ((1L << (ASSIGN - 58)) | (1L << (AADD - 58)) | (1L << (ASUB - 58)) | (1L << (AMUL - 58)) | (1L << (ADIV - 58)) | (1L << (AREM - 58)) | (1L << (AAND - 58)) | (1L << (AXOR - 58)) | (1L << (AOR - 58)) | (1L << (ALSH - 58)) | (1L << (ARSH - 58)) | (1L << (AUSH - 58)))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(238);
+        setState(246);
         expression(1);
          ((AssignmentContext)_localctx).s =  false; 
         }
@@ -1522,14 +1562,14 @@ class PainlessParser extends Parser {
         _localctx = new SingleContext(_localctx);
         _ctx = _localctx;
         _prevctx = _localctx;
-        setState(241);
+        setState(249);
         ((SingleContext)_localctx).u = unary(false);
          ((SingleContext)_localctx).s =  ((SingleContext)_localctx).u.s; 
         }
         break;
       }
       _ctx.stop = _input.LT(-1);
-      setState(315);
+      setState(323);
       _errHandler.sync(this);
       _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
       while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
@@ -1537,22 +1577,22 @@ class PainlessParser extends Parser {
           if ( _parseListeners!=null ) triggerExitRuleEvent();
           _prevctx = _localctx;
           {
-          setState(313);
+          setState(321);
           switch ( getInterpreter().adaptivePredict(_input,19,_ctx) ) {
           case 1:
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(246);
+            setState(254);
             if (!(precpred(_ctx, 14))) throw new FailedPredicateException(this, "precpred(_ctx, 14)");
-            setState(247);
+            setState(255);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << MUL) | (1L << DIV) | (1L << REM))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(248);
+            setState(256);
             expression(15);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1561,16 +1601,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(251);
+            setState(259);
             if (!(precpred(_ctx, 13))) throw new FailedPredicateException(this, "precpred(_ctx, 13)");
-            setState(252);
+            setState(260);
             _la = _input.LA(1);
             if ( !(_la==ADD || _la==SUB) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(253);
+            setState(261);
             expression(14);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1579,16 +1619,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(256);
+            setState(264);
             if (!(precpred(_ctx, 12))) throw new FailedPredicateException(this, "precpred(_ctx, 12)");
-            setState(257);
+            setState(265);
             _la = _input.LA(1);
             if ( !(_la==FIND || _la==MATCH) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(258);
+            setState(266);
             expression(13);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1597,16 +1637,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(261);
+            setState(269);
             if (!(precpred(_ctx, 11))) throw new FailedPredicateException(this, "precpred(_ctx, 11)");
-            setState(262);
+            setState(270);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LSH) | (1L << RSH) | (1L << USH))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(263);
+            setState(271);
             expression(12);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1615,16 +1655,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(266);
+            setState(274);
             if (!(precpred(_ctx, 10))) throw new FailedPredicateException(this, "precpred(_ctx, 10)");
-            setState(267);
+            setState(275);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << LT) | (1L << LTE) | (1L << GT) | (1L << GTE))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(268);
+            setState(276);
             expression(11);
              ((CompContext)_localctx).s =  false; 
             }
@@ -1633,16 +1673,16 @@ class PainlessParser extends Parser {
             {
             _localctx = new CompContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(271);
+            setState(279);
             if (!(precpred(_ctx, 8))) throw new FailedPredicateException(this, "precpred(_ctx, 8)");
-            setState(272);
+            setState(280);
             _la = _input.LA(1);
             if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << EQ) | (1L << EQR) | (1L << NE) | (1L << NER))) != 0)) ) {
             _errHandler.recoverInline(this);
             } else {
               consume();
             }
-            setState(273);
+            setState(281);
             expression(9);
              ((CompContext)_localctx).s =  false; 
             }
@@ -1651,11 +1691,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(276);
+            setState(284);
             if (!(precpred(_ctx, 7))) throw new FailedPredicateException(this, "precpred(_ctx, 7)");
-            setState(277);
+            setState(285);
             match(BWAND);
-            setState(278);
+            setState(286);
             expression(8);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1664,11 +1704,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(281);
+            setState(289);
             if (!(precpred(_ctx, 6))) throw new FailedPredicateException(this, "precpred(_ctx, 6)");
-            setState(282);
+            setState(290);
             match(XOR);
-            setState(283);
+            setState(291);
             expression(7);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1677,11 +1717,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BinaryContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(286);
+            setState(294);
             if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)");
-            setState(287);
+            setState(295);
             match(BWOR);
-            setState(288);
+            setState(296);
             expression(6);
              ((BinaryContext)_localctx).s =  false; 
             }
@@ -1690,11 +1730,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(291);
+            setState(299);
             if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)");
-            setState(292);
+            setState(300);
             match(BOOLAND);
-            setState(293);
+            setState(301);
             expression(5);
              ((BoolContext)_localctx).s =  false; 
             }
@@ -1703,11 +1743,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new BoolContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(296);
+            setState(304);
             if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)");
-            setState(297);
+            setState(305);
             match(BOOLOR);
-            setState(298);
+            setState(306);
             expression(4);
              ((BoolContext)_localctx).s =  false; 
             }
@@ -1716,15 +1756,15 @@ class PainlessParser extends Parser {
             {
             _localctx = new ConditionalContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(301);
+            setState(309);
             if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)");
-            setState(302);
+            setState(310);
             match(COND);
-            setState(303);
+            setState(311);
             ((ConditionalContext)_localctx).e0 = expression(0);
-            setState(304);
+            setState(312);
             match(COLON);
-            setState(305);
+            setState(313);
             ((ConditionalContext)_localctx).e1 = expression(2);
              ((ConditionalContext)_localctx).s =  ((ConditionalContext)_localctx).e0.s && ((ConditionalContext)_localctx).e1.s; 
             }
@@ -1733,11 +1773,11 @@ class PainlessParser extends Parser {
             {
             _localctx = new InstanceofContext(new ExpressionContext(_parentctx, _parentState));
             pushNewRecursionContext(_localctx, _startState, RULE_expression);
-            setState(308);
+            setState(316);
             if (!(precpred(_ctx, 9))) throw new FailedPredicateException(this, "precpred(_ctx, 9)");
-            setState(309);
+            setState(317);
             match(INSTANCEOF);
-            setState(310);
+            setState(318);
             decltype();
              ((InstanceofContext)_localctx).s =  false; 
             }
@@ -1745,7 +1785,7 @@ class PainlessParser extends Parser {
           }
           } 
         }
-        setState(317);
+        setState(325);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,20,_ctx);
       }
@@ -1914,22 +1954,22 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 30, RULE_unary);
     int _la;
     try {
-      setState(355);
+      setState(363);
       switch ( getInterpreter().adaptivePredict(_input,21,_ctx) ) {
       case 1:
         _localctx = new PreContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(318);
+        setState(326);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(319);
+        setState(327);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(320);
+        setState(328);
         chain(true);
         }
         break;
@@ -1937,11 +1977,11 @@ class PainlessParser extends Parser {
         _localctx = new PostContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(321);
+        setState(329);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(322);
+        setState(330);
         chain(true);
-        setState(323);
+        setState(331);
         _la = _input.LA(1);
         if ( !(_la==INCR || _la==DECR) ) {
         _errHandler.recoverInline(this);
@@ -1954,9 +1994,9 @@ class PainlessParser extends Parser {
         _localctx = new ReadContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(325);
+        setState(333);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(326);
+        setState(334);
         chain(false);
         }
         break;
@@ -1964,11 +2004,11 @@ class PainlessParser extends Parser {
         _localctx = new NumericContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(327);
+        setState(335);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(328);
+        setState(336);
         _la = _input.LA(1);
-        if ( !(((((_la - 69)) & ~0x3f) == 0 && ((1L << (_la - 69)) & ((1L << (OCTAL - 69)) | (1L << (HEX - 69)) | (1L << (INTEGER - 69)) | (1L << (DECIMAL - 69)))) != 0)) ) {
+        if ( !(((((_la - 70)) & ~0x3f) == 0 && ((1L << (_la - 70)) & ((1L << (OCTAL - 70)) | (1L << (HEX - 70)) | (1L << (INTEGER - 70)) | (1L << (DECIMAL - 70)))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
@@ -1980,9 +2020,9 @@ class PainlessParser extends Parser {
         _localctx = new TrueContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(330);
+        setState(338);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(331);
+        setState(339);
         match(TRUE);
          ((TrueContext)_localctx).s =  false; 
         }
@@ -1991,9 +2031,9 @@ class PainlessParser extends Parser {
         _localctx = new FalseContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(333);
+        setState(341);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(334);
+        setState(342);
         match(FALSE);
          ((FalseContext)_localctx).s =  false; 
         }
@@ -2002,9 +2042,9 @@ class PainlessParser extends Parser {
         _localctx = new NullContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(336);
+        setState(344);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(337);
+        setState(345);
         match(NULL);
          ((NullContext)_localctx).s =  false; 
         }
@@ -2013,9 +2053,9 @@ class PainlessParser extends Parser {
         _localctx = new ListinitContext(_localctx);
         enterOuterAlt(_localctx, 8);
         {
-        setState(339);
+        setState(347);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(340);
+        setState(348);
         listinitializer();
          ((ListinitContext)_localctx).s =  false; 
         }
@@ -2024,9 +2064,9 @@ class PainlessParser extends Parser {
         _localctx = new MapinitContext(_localctx);
         enterOuterAlt(_localctx, 9);
         {
-        setState(343);
+        setState(351);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(344);
+        setState(352);
         mapinitializer();
          ((MapinitContext)_localctx).s =  false; 
         }
@@ -2035,16 +2075,16 @@ class PainlessParser extends Parser {
         _localctx = new OperatorContext(_localctx);
         enterOuterAlt(_localctx, 10);
         {
-        setState(347);
+        setState(355);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(348);
+        setState(356);
         _la = _input.LA(1);
         if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << BOOLNOT) | (1L << BWNOT) | (1L << ADD) | (1L << SUB))) != 0)) ) {
         _errHandler.recoverInline(this);
         } else {
           consume();
         }
-        setState(349);
+        setState(357);
         unary(false);
         }
         break;
@@ -2052,13 +2092,13 @@ class PainlessParser extends Parser {
         _localctx = new CastContext(_localctx);
         enterOuterAlt(_localctx, 11);
         {
-        setState(350);
+        setState(358);
         match(LP);
-        setState(351);
+        setState(359);
         decltype();
-        setState(352);
+        setState(360);
         match(RP);
-        setState(353);
+        setState(361);
         unary(_localctx.c);
         }
         break;
@@ -2145,27 +2185,27 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 32, RULE_chain);
     try {
       int _alt;
-      setState(373);
+      setState(381);
       switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) {
       case 1:
         _localctx = new DynamicContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(357);
+        setState(365);
         ((DynamicContext)_localctx).p = primary(_localctx.c);
-        setState(361);
+        setState(369);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(358);
+            setState(366);
             secondary(((DynamicContext)_localctx).p.s);
             }
             } 
           }
-          setState(363);
+          setState(371);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,22,_ctx);
         }
@@ -2175,23 +2215,23 @@ class PainlessParser extends Parser {
         _localctx = new StaticContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(364);
+        setState(372);
         decltype();
-        setState(365);
+        setState(373);
         dot();
-        setState(369);
+        setState(377);
         _errHandler.sync(this);
         _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
         while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
           if ( _alt==1 ) {
             {
             {
-            setState(366);
+            setState(374);
             secondary(true);
             }
             } 
           }
-          setState(371);
+          setState(379);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,23,_ctx);
         }
@@ -2201,7 +2241,7 @@ class PainlessParser extends Parser {
         _localctx = new NewarrayContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(372);
+        setState(380);
         arrayinitializer();
         }
         break;
@@ -2319,19 +2359,19 @@ class PainlessParser extends Parser {
     PrimaryContext _localctx = new PrimaryContext(_ctx, getState(), c);
     enterRule(_localctx, 34, RULE_primary);
     try {
-      setState(394);
+      setState(402);
       switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) {
       case 1:
         _localctx = new ExprprecContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(375);
+        setState(383);
         if (!( !_localctx.c )) throw new FailedPredicateException(this, " !$c ");
-        setState(376);
+        setState(384);
         match(LP);
-        setState(377);
+        setState(385);
         ((ExprprecContext)_localctx).e = expression(0);
-        setState(378);
+        setState(386);
         match(RP);
          ((ExprprecContext)_localctx).s =  ((ExprprecContext)_localctx).e.s; 
         }
@@ -2340,13 +2380,13 @@ class PainlessParser extends Parser {
         _localctx = new ChainprecContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(381);
+        setState(389);
         if (!( _localctx.c )) throw new FailedPredicateException(this, " $c ");
-        setState(382);
+        setState(390);
         match(LP);
-        setState(383);
+        setState(391);
         unary(true);
-        setState(384);
+        setState(392);
         match(RP);
         }
         break;
@@ -2354,7 +2394,7 @@ class PainlessParser extends Parser {
         _localctx = new StringContext(_localctx);
         enterOuterAlt(_localctx, 3);
         {
-        setState(386);
+        setState(394);
         match(STRING);
         }
         break;
@@ -2362,7 +2402,7 @@ class PainlessParser extends Parser {
         _localctx = new RegexContext(_localctx);
         enterOuterAlt(_localctx, 4);
         {
-        setState(387);
+        setState(395);
         match(REGEX);
         }
         break;
@@ -2370,7 +2410,7 @@ class PainlessParser extends Parser {
         _localctx = new VariableContext(_localctx);
         enterOuterAlt(_localctx, 5);
         {
-        setState(388);
+        setState(396);
         match(ID);
         }
         break;
@@ -2378,9 +2418,9 @@ class PainlessParser extends Parser {
         _localctx = new CalllocalContext(_localctx);
         enterOuterAlt(_localctx, 6);
         {
-        setState(389);
+        setState(397);
         match(ID);
-        setState(390);
+        setState(398);
         arguments();
         }
         break;
@@ -2388,11 +2428,11 @@ class PainlessParser extends Parser {
         _localctx = new NewobjectContext(_localctx);
         enterOuterAlt(_localctx, 7);
         {
-        setState(391);
+        setState(399);
         match(NEW);
-        setState(392);
+        setState(400);
         match(TYPE);
-        setState(393);
+        setState(401);
         arguments();
         }
         break;
@@ -2434,23 +2474,23 @@ class PainlessParser extends Parser {
     SecondaryContext _localctx = new SecondaryContext(_ctx, getState(), s);
     enterRule(_localctx, 36, RULE_secondary);
     try {
-      setState(400);
+      setState(408);
       switch ( getInterpreter().adaptivePredict(_input,26,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(396);
+        setState(404);
         if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
-        setState(397);
+        setState(405);
         dot();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(398);
+        setState(406);
         if (!( _localctx.s )) throw new FailedPredicateException(this, " $s ");
-        setState(399);
+        setState(407);
         brace();
         }
         break;
@@ -2508,17 +2548,17 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 38, RULE_dot);
     int _la;
     try {
-      setState(407);
+      setState(415);
       switch ( getInterpreter().adaptivePredict(_input,27,_ctx) ) {
       case 1:
         _localctx = new CallinvokeContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(402);
+        setState(410);
         match(DOT);
-        setState(403);
+        setState(411);
         match(DOTID);
-        setState(404);
+        setState(412);
         arguments();
         }
         break;
@@ -2526,9 +2566,9 @@ class PainlessParser extends Parser {
         _localctx = new FieldaccessContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(405);
+        setState(413);
         match(DOT);
-        setState(406);
+        setState(414);
         _la = _input.LA(1);
         if ( !(_la==DOTINTEGER || _la==DOTID) ) {
         _errHandler.recoverInline(this);
@@ -2582,11 +2622,11 @@ class PainlessParser extends Parser {
       _localctx = new BraceaccessContext(_localctx);
       enterOuterAlt(_localctx, 1);
       {
-      setState(409);
+      setState(417);
       match(LBRACE);
-      setState(410);
+      setState(418);
       expression(0);
-      setState(411);
+      setState(419);
       match(RBRACE);
       }
     }
@@ -2633,34 +2673,34 @@ class PainlessParser extends Parser {
       enterOuterAlt(_localctx, 1);
       {
       {
-      setState(413);
+      setState(421);
       match(LP);
-      setState(422);
+      setState(430);
       switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) {
       case 1:
         {
-        setState(414);
+        setState(422);
         argument();
-        setState(419);
+        setState(427);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(415);
+          setState(423);
           match(COMMA);
-          setState(416);
+          setState(424);
           argument();
           }
           }
-          setState(421);
+          setState(429);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
         }
         break;
       }
-      setState(424);
+      setState(432);
       match(RP);
       }
       }
@@ -2701,26 +2741,26 @@ class PainlessParser extends Parser {
     ArgumentContext _localctx = new ArgumentContext(_ctx, getState());
     enterRule(_localctx, 44, RULE_argument);
     try {
-      setState(429);
+      setState(437);
       switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(426);
+        setState(434);
         expression(0);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(427);
+        setState(435);
         lambda();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(428);
+        setState(436);
         funcref();
         }
         break;
@@ -2775,64 +2815,64 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(444);
+      setState(452);
       switch (_input.LA(1)) {
       case TYPE:
       case ID:
         {
-        setState(431);
+        setState(439);
         lamtype();
         }
         break;
       case LP:
         {
-        setState(432);
+        setState(440);
         match(LP);
-        setState(441);
+        setState(449);
         _la = _input.LA(1);
         if (_la==TYPE || _la==ID) {
           {
-          setState(433);
+          setState(441);
           lamtype();
-          setState(438);
+          setState(446);
           _errHandler.sync(this);
           _la = _input.LA(1);
           while (_la==COMMA) {
             {
             {
-            setState(434);
+            setState(442);
             match(COMMA);
-            setState(435);
+            setState(443);
             lamtype();
             }
             }
-            setState(440);
+            setState(448);
             _errHandler.sync(this);
             _la = _input.LA(1);
           }
           }
         }
 
-        setState(443);
+        setState(451);
         match(RP);
         }
         break;
       default:
         throw new NoViableAltException(this);
       }
-      setState(446);
+      setState(454);
       match(ARROW);
-      setState(449);
+      setState(457);
       switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) {
       case 1:
         {
-        setState(447);
+        setState(455);
         block();
         }
         break;
       case 2:
         {
-        setState(448);
+        setState(456);
         expression(0);
         }
         break;
@@ -2873,16 +2913,16 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(452);
+      setState(460);
       _la = _input.LA(1);
       if (_la==TYPE) {
         {
-        setState(451);
+        setState(459);
         decltype();
         }
       }
 
-      setState(454);
+      setState(462);
       match(ID);
       }
     }
@@ -2925,33 +2965,33 @@ class PainlessParser extends Parser {
     FuncrefContext _localctx = new FuncrefContext(_ctx, getState());
     enterRule(_localctx, 50, RULE_funcref);
     try {
-      setState(460);
+      setState(468);
       switch ( getInterpreter().adaptivePredict(_input,36,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(456);
+        setState(464);
         classFuncref();
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(457);
+        setState(465);
         constructorFuncref();
         }
         break;
       case 3:
         enterOuterAlt(_localctx, 3);
         {
-        setState(458);
+        setState(466);
         capturingFuncref();
         }
         break;
       case 4:
         enterOuterAlt(_localctx, 4);
         {
-        setState(459);
+        setState(467);
         localFuncref();
         }
         break;
@@ -2989,11 +3029,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(462);
+      setState(470);
       match(TYPE);
-      setState(463);
+      setState(471);
       match(REF);
-      setState(464);
+      setState(472);
       match(ID);
       }
     }
@@ -3031,11 +3071,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(466);
+      setState(474);
       decltype();
-      setState(467);
+      setState(475);
       match(REF);
-      setState(468);
+      setState(476);
       match(NEW);
       }
     }
@@ -3073,11 +3113,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(470);
+      setState(478);
       match(ID);
-      setState(471);
+      setState(479);
       match(REF);
-      setState(472);
+      setState(480);
       match(ID);
       }
     }
@@ -3113,11 +3153,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(474);
+      setState(482);
       match(THIS);
-      setState(475);
+      setState(483);
       match(REF);
-      setState(476);
+      setState(484);
       match(ID);
       }
     }
@@ -3208,17 +3248,17 @@ class PainlessParser extends Parser {
     int _la;
     try {
       int _alt;
-      setState(516);
+      setState(524);
       switch ( getInterpreter().adaptivePredict(_input,43,_ctx) ) {
       case 1:
         _localctx = new NewstandardarrayContext(_localctx);
         enterOuterAlt(_localctx, 1);
         {
-        setState(478);
+        setState(486);
         match(NEW);
-        setState(479);
+        setState(487);
         match(TYPE);
-        setState(484); 
+        setState(492); 
         _errHandler.sync(this);
         _alt = 1;
         do {
@@ -3226,11 +3266,11 @@ class PainlessParser extends Parser {
           case 1:
             {
             {
-            setState(480);
+            setState(488);
             match(LBRACE);
-            setState(481);
+            setState(489);
             expression(0);
-            setState(482);
+            setState(490);
             match(RBRACE);
             }
             }
@@ -3238,29 +3278,29 @@ class PainlessParser extends Parser {
           default:
             throw new NoViableAltException(this);
           }
-          setState(486); 
+          setState(494); 
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,37,_ctx);
         } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER );
-        setState(495);
+        setState(503);
         switch ( getInterpreter().adaptivePredict(_input,39,_ctx) ) {
         case 1:
           {
-          setState(488);
+          setState(496);
           dot();
-          setState(492);
+          setState(500);
           _errHandler.sync(this);
           _alt = getInterpreter().adaptivePredict(_input,38,_ctx);
           while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) {
             if ( _alt==1 ) {
               {
               {
-              setState(489);
+              setState(497);
               secondary(true);
               }
               } 
             }
-            setState(494);
+            setState(502);
             _errHandler.sync(this);
             _alt = getInterpreter().adaptivePredict(_input,38,_ctx);
           }
@@ -3273,51 +3313,51 @@ class PainlessParser extends Parser {
         _localctx = new NewinitializedarrayContext(_localctx);
         enterOuterAlt(_localctx, 2);
         {
-        setState(497);
+        setState(505);
         match(NEW);
-        setState(498);
+        setState(506);
         match(TYPE);
-        setState(499);
+        setState(507);
         match(LBRACE);
-        setState(500);
+        setState(508);
         match(RBRACE);
-        setState(501);
+        setState(509);
         match(LBRACK);
-        setState(510);
+        setState(518);
         switch ( getInterpreter().adaptivePredict(_input,41,_ctx) ) {
         case 1:
           {
-          setState(502);
+          setState(510);
           expression(0);
-          setState(507);
+          setState(515);
           _errHandler.sync(this);
           _la = _input.LA(1);
           while (_la==COMMA) {
             {
             {
-            setState(503);
+            setState(511);
             match(COMMA);
-            setState(504);
+            setState(512);
             expression(0);
             }
             }
-            setState(509);
+            setState(517);
             _errHandler.sync(this);
             _la = _input.LA(1);
           }
           }
           break;
         }
-        setState(513);
+        setState(521);
         _la = _input.LA(1);
         if (_la==SEMICOLON) {
           {
-          setState(512);
+          setState(520);
           match(SEMICOLON);
           }
         }
 
-        setState(515);
+        setState(523);
         match(RBRACK);
         }
         break;
@@ -3363,41 +3403,41 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 62, RULE_listinitializer);
     int _la;
     try {
-      setState(531);
+      setState(539);
       switch ( getInterpreter().adaptivePredict(_input,45,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(518);
+        setState(526);
         match(LBRACE);
-        setState(519);
+        setState(527);
         expression(0);
-        setState(524);
+        setState(532);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(520);
+          setState(528);
           match(COMMA);
-          setState(521);
+          setState(529);
           expression(0);
           }
           }
-          setState(526);
+          setState(534);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(527);
+        setState(535);
         match(RBRACE);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(529);
+        setState(537);
         match(LBRACE);
-        setState(530);
+        setState(538);
         match(RBRACE);
         }
         break;
@@ -3444,43 +3484,43 @@ class PainlessParser extends Parser {
     enterRule(_localctx, 64, RULE_mapinitializer);
     int _la;
     try {
-      setState(547);
+      setState(555);
       switch ( getInterpreter().adaptivePredict(_input,47,_ctx) ) {
       case 1:
         enterOuterAlt(_localctx, 1);
         {
-        setState(533);
+        setState(541);
         match(LBRACE);
-        setState(534);
+        setState(542);
         maptoken();
-        setState(539);
+        setState(547);
         _errHandler.sync(this);
         _la = _input.LA(1);
         while (_la==COMMA) {
           {
           {
-          setState(535);
+          setState(543);
           match(COMMA);
-          setState(536);
+          setState(544);
           maptoken();
           }
           }
-          setState(541);
+          setState(549);
           _errHandler.sync(this);
           _la = _input.LA(1);
         }
-        setState(542);
+        setState(550);
         match(RBRACE);
         }
         break;
       case 2:
         enterOuterAlt(_localctx, 2);
         {
-        setState(544);
+        setState(552);
         match(LBRACE);
-        setState(545);
+        setState(553);
         match(COLON);
-        setState(546);
+        setState(554);
         match(RBRACE);
         }
         break;
@@ -3522,11 +3562,11 @@ class PainlessParser extends Parser {
     try {
       enterOuterAlt(_localctx, 1);
       {
-      setState(549);
+      setState(557);
       expression(0);
-      setState(550);
+      setState(558);
       match(COLON);
-      setState(551);
+      setState(559);
       expression(0);
       }
     }
@@ -3639,7 +3679,7 @@ class PainlessParser extends Parser {
   }
 
   public static final String _serializedATN =
-    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3S\u022c\4\2\t\2\4"+
+    "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3T\u0234\4\2\t\2\4"+
     "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
     "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
     "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+
@@ -3650,204 +3690,207 @@ class PainlessParser extends Parser {
     "\5\3\5\3\5\3\5\3\5\3\5\3\5\5\5z\n\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3"+
     "\5\3\5\3\5\5\5\u0087\n\5\3\5\3\5\5\5\u008b\n\5\3\5\3\5\5\5\u008f\n\5\3"+
     "\5\3\5\3\5\5\5\u0094\n\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3"+
-    "\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\6\5\u00ad\n\5\r\5\16\5"+
-    "\u00ae\3\5\3\5\3\5\3\5\3\5\3\5\3\5\5\5\u00b8\n\5\3\6\3\6\5\6\u00bc\n\6"+
-    "\3\7\3\7\7\7\u00c0\n\7\f\7\16\7\u00c3\13\7\3\7\3\7\3\b\3\b\3\t\3\t\5\t"+
-    "\u00cb\n\t\3\n\3\n\3\13\3\13\3\13\3\13\7\13\u00d3\n\13\f\13\16\13\u00d6"+
-    "\13\13\3\f\3\f\3\f\7\f\u00db\n\f\f\f\16\f\u00de\13\f\3\r\3\r\3\r\5\r\u00e3"+
-    "\n\r\3\16\3\16\3\16\3\16\3\16\3\16\3\16\3\17\3\17\3\20\3\20\3\20\3\20"+
-    "\3\20\3\20\3\20\3\20\3\20\5\20\u00f7\n\20\3\20\3\20\3\20\3\20\3\20\3\20"+
+    "\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5\3\5"+
+    "\3\5\3\5\6\5\u00b5\n\5\r\5\16\5\u00b6\3\5\3\5\3\5\3\5\3\5\3\5\3\5\5\5"+
+    "\u00c0\n\5\3\6\3\6\5\6\u00c4\n\6\3\7\3\7\7\7\u00c8\n\7\f\7\16\7\u00cb"+
+    "\13\7\3\7\3\7\3\b\3\b\3\t\3\t\5\t\u00d3\n\t\3\n\3\n\3\13\3\13\3\13\3\13"+
+    "\7\13\u00db\n\13\f\13\16\13\u00de\13\13\3\f\3\f\3\f\7\f\u00e3\n\f\f\f"+
+    "\16\f\u00e6\13\f\3\r\3\r\3\r\5\r\u00eb\n\r\3\16\3\16\3\16\3\16\3\16\3"+
+    "\16\3\16\3\17\3\17\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\5\20\u00ff"+
+    "\n\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
     "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
-    "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20"+
-    "\3\20\3\20\3\20\3\20\3\20\7\20\u013c\n\20\f\20\16\20\u013f\13\20\3\21"+
-    "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
-    "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21"+
-    "\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\5\21\u0166\n\21\3\22\3\22\7\22"+
-    "\u016a\n\22\f\22\16\22\u016d\13\22\3\22\3\22\3\22\7\22\u0172\n\22\f\22"+
-    "\16\22\u0175\13\22\3\22\5\22\u0178\n\22\3\23\3\23\3\23\3\23\3\23\3\23"+
-    "\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\5\23"+
-    "\u018d\n\23\3\24\3\24\3\24\3\24\5\24\u0193\n\24\3\25\3\25\3\25\3\25\3"+
-    "\25\5\25\u019a\n\25\3\26\3\26\3\26\3\26\3\27\3\27\3\27\3\27\7\27\u01a4"+
-    "\n\27\f\27\16\27\u01a7\13\27\5\27\u01a9\n\27\3\27\3\27\3\30\3\30\3\30"+
-    "\5\30\u01b0\n\30\3\31\3\31\3\31\3\31\3\31\7\31\u01b7\n\31\f\31\16\31\u01ba"+
-    "\13\31\5\31\u01bc\n\31\3\31\5\31\u01bf\n\31\3\31\3\31\3\31\5\31\u01c4"+
-    "\n\31\3\32\5\32\u01c7\n\32\3\32\3\32\3\33\3\33\3\33\3\33\5\33\u01cf\n"+
-    "\33\3\34\3\34\3\34\3\34\3\35\3\35\3\35\3\35\3\36\3\36\3\36\3\36\3\37\3"+
-    "\37\3\37\3\37\3 \3 \3 \3 \3 \3 \6 \u01e7\n \r \16 \u01e8\3 \3 \7 \u01ed"+
-    "\n \f \16 \u01f0\13 \5 \u01f2\n \3 \3 \3 \3 \3 \3 \3 \3 \7 \u01fc\n \f"+
-    " \16 \u01ff\13 \5 \u0201\n \3 \5 \u0204\n \3 \5 \u0207\n \3!\3!\3!\3!"+
-    "\7!\u020d\n!\f!\16!\u0210\13!\3!\3!\3!\3!\5!\u0216\n!\3\"\3\"\3\"\3\""+
-    "\7\"\u021c\n\"\f\"\16\"\u021f\13\"\3\"\3\"\3\"\3\"\3\"\5\"\u0226\n\"\3"+
-    "#\3#\3#\3#\3#\2\3\36$\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,."+
-    "\60\62\64\668:<>@BD\2\16\3\3\r\r\3\2;F\3\2\36 \3\2!\"\3\2\678\3\2#%\3"+
-    "\2&)\3\2*-\3\29:\3\2GJ\4\2\34\35!\"\3\2RS\u0260\2I\3\2\2\2\4T\3\2\2\2"+
-    "\6Y\3\2\2\2\b\u00b7\3\2\2\2\n\u00bb\3\2\2\2\f\u00bd\3\2\2\2\16\u00c6\3"+
-    "\2\2\2\20\u00ca\3\2\2\2\22\u00cc\3\2\2\2\24\u00ce\3\2\2\2\26\u00d7\3\2"+
-    "\2\2\30\u00df\3\2\2\2\32\u00e4\3\2\2\2\34\u00eb\3\2\2\2\36\u00f6\3\2\2"+
-    "\2 \u0165\3\2\2\2\"\u0177\3\2\2\2$\u018c\3\2\2\2&\u0192\3\2\2\2(\u0199"+
-    "\3\2\2\2*\u019b\3\2\2\2,\u019f\3\2\2\2.\u01af\3\2\2\2\60\u01be\3\2\2\2"+
-    "\62\u01c6\3\2\2\2\64\u01ce\3\2\2\2\66\u01d0\3\2\2\28\u01d4\3\2\2\2:\u01d8"+
-    "\3\2\2\2<\u01dc\3\2\2\2>\u0206\3\2\2\2@\u0215\3\2\2\2B\u0225\3\2\2\2D"+
-    "\u0227\3\2\2\2FH\5\4\3\2GF\3\2\2\2HK\3\2\2\2IG\3\2\2\2IJ\3\2\2\2JO\3\2"+
-    "\2\2KI\3\2\2\2LN\5\b\5\2ML\3\2\2\2NQ\3\2\2\2OM\3\2\2\2OP\3\2\2\2PR\3\2"+
-    "\2\2QO\3\2\2\2RS\7\2\2\3S\3\3\2\2\2TU\5\26\f\2UV\7Q\2\2VW\5\6\4\2WX\5"+
-    "\f\7\2X\5\3\2\2\2Ye\7\t\2\2Z[\5\26\f\2[b\7Q\2\2\\]\7\f\2\2]^\5\26\f\2"+
-    "^_\7Q\2\2_a\3\2\2\2`\\\3\2\2\2ad\3\2\2\2b`\3\2\2\2bc\3\2\2\2cf\3\2\2\2"+
-    "db\3\2\2\2eZ\3\2\2\2ef\3\2\2\2fg\3\2\2\2gh\7\n\2\2h\7\3\2\2\2ij\7\16\2"+
-    "\2jk\7\t\2\2kl\5\36\20\2lm\7\n\2\2mq\5\n\6\2no\7\17\2\2or\5\n\6\2pr\6"+
-    "\5\2\2qn\3\2\2\2qp\3\2\2\2r\u00b8\3\2\2\2st\7\20\2\2tu\7\t\2\2uv\5\36"+
-    "\20\2vy\7\n\2\2wz\5\n\6\2xz\5\16\b\2yw\3\2\2\2yx\3\2\2\2z\u00b8\3\2\2"+
-    "\2{|\7\21\2\2|}\5\f\7\2}~\7\20\2\2~\177\7\t\2\2\177\u0080\5\36\20\2\u0080"+
-    "\u0081\7\n\2\2\u0081\u0082\5\34\17\2\u0082\u00b8\3\2\2\2\u0083\u0084\7"+
-    "\22\2\2\u0084\u0086\7\t\2\2\u0085\u0087\5\20\t\2\u0086\u0085\3\2\2\2\u0086"+
-    "\u0087\3\2\2\2\u0087\u0088\3\2\2\2\u0088\u008a\7\r\2\2\u0089\u008b\5\36"+
-    "\20\2\u008a\u0089\3\2\2\2\u008a\u008b\3\2\2\2\u008b\u008c\3\2\2\2\u008c"+
-    "\u008e\7\r\2\2\u008d\u008f\5\22\n\2\u008e\u008d\3\2\2\2\u008e\u008f\3"+
-    "\2\2\2\u008f\u0090\3\2\2\2\u0090\u0093\7\n\2\2\u0091\u0094\5\n\6\2\u0092"+
-    "\u0094\5\16\b\2\u0093\u0091\3\2\2\2\u0093\u0092\3\2\2\2\u0094\u00b8\3"+
-    "\2\2\2\u0095\u0096\7\22\2\2\u0096\u0097\7\t\2\2\u0097\u0098\5\26\f\2\u0098"+
-    "\u0099\7Q\2\2\u0099\u009a\7\64\2\2\u009a\u009b\5\36\20\2\u009b\u009c\7"+
-    "\n\2\2\u009c\u009d\5\n\6\2\u009d\u00b8\3\2\2\2\u009e\u009f\5\24\13\2\u009f"+
-    "\u00a0\5\34\17\2\u00a0\u00b8\3\2\2\2\u00a1\u00a2\7\23\2\2\u00a2\u00b8"+
-    "\5\34\17\2\u00a3\u00a4\7\24\2\2\u00a4\u00b8\5\34\17\2\u00a5\u00a6\7\25"+
-    "\2\2\u00a6\u00a7\5\36\20\2\u00a7\u00a8\5\34\17\2\u00a8\u00b8\3\2\2\2\u00a9"+
-    "\u00aa\7\27\2\2\u00aa\u00ac\5\f\7\2\u00ab\u00ad\5\32\16\2\u00ac\u00ab"+
-    "\3\2\2\2\u00ad\u00ae\3\2\2\2\u00ae\u00ac\3\2\2\2\u00ae\u00af\3\2\2\2\u00af"+
-    "\u00b8\3\2\2\2\u00b0\u00b1\7\31\2\2\u00b1\u00b2\5\36\20\2\u00b2\u00b3"+
-    "\5\34\17\2\u00b3\u00b8\3\2\2\2\u00b4\u00b5\5\36\20\2\u00b5\u00b6\5\34"+
-    "\17\2\u00b6\u00b8\3\2\2\2\u00b7i\3\2\2\2\u00b7s\3\2\2\2\u00b7{\3\2\2\2"+
-    "\u00b7\u0083\3\2\2\2\u00b7\u0095\3\2\2\2\u00b7\u009e\3\2\2\2\u00b7\u00a1"+
-    "\3\2\2\2\u00b7\u00a3\3\2\2\2\u00b7\u00a5\3\2\2\2\u00b7\u00a9\3\2\2\2\u00b7"+
-    "\u00b0\3\2\2\2\u00b7\u00b4\3\2\2\2\u00b8\t\3\2\2\2\u00b9\u00bc\5\f\7\2"+
-    "\u00ba\u00bc\5\b\5\2\u00bb\u00b9\3\2\2\2\u00bb\u00ba\3\2\2\2\u00bc\13"+
-    "\3\2\2\2\u00bd\u00c1\7\5\2\2\u00be\u00c0\5\b\5\2\u00bf\u00be\3\2\2\2\u00c0"+
-    "\u00c3\3\2\2\2\u00c1\u00bf\3\2\2\2\u00c1\u00c2\3\2\2\2\u00c2\u00c4\3\2"+
-    "\2\2\u00c3\u00c1\3\2\2\2\u00c4\u00c5\7\6\2\2\u00c5\r\3\2\2\2\u00c6\u00c7"+
-    "\7\r\2\2\u00c7\17\3\2\2\2\u00c8\u00cb\5\24\13\2\u00c9\u00cb\5\36\20\2"+
-    "\u00ca\u00c8\3\2\2\2\u00ca\u00c9\3\2\2\2\u00cb\21\3\2\2\2\u00cc\u00cd"+
-    "\5\36\20\2\u00cd\23\3\2\2\2\u00ce\u00cf\5\26\f\2\u00cf\u00d4\5\30\r\2"+
-    "\u00d0\u00d1\7\f\2\2\u00d1\u00d3\5\30\r\2\u00d2\u00d0\3\2\2\2\u00d3\u00d6"+
-    "\3\2\2\2\u00d4\u00d2\3\2\2\2\u00d4\u00d5\3\2\2\2\u00d5\25\3\2\2\2\u00d6"+
-    "\u00d4\3\2\2\2\u00d7\u00dc\7P\2\2\u00d8\u00d9\7\7\2\2\u00d9\u00db\7\b"+
-    "\2\2\u00da\u00d8\3\2\2\2\u00db\u00de\3\2\2\2\u00dc\u00da\3\2\2\2\u00dc"+
-    "\u00dd\3\2\2\2\u00dd\27\3\2\2\2\u00de\u00dc\3\2\2\2\u00df\u00e2\7Q\2\2"+
-    "\u00e0\u00e1\7;\2\2\u00e1\u00e3\5\36\20\2\u00e2\u00e0\3\2\2\2\u00e2\u00e3"+
-    "\3\2\2\2\u00e3\31\3\2\2\2\u00e4\u00e5\7\30\2\2\u00e5\u00e6\7\t\2\2\u00e6"+
-    "\u00e7\7P\2\2\u00e7\u00e8\7Q\2\2\u00e8\u00e9\7\n\2\2\u00e9\u00ea\5\f\7"+
-    "\2\u00ea\33\3\2\2\2\u00eb\u00ec\t\2\2\2\u00ec\35\3\2\2\2\u00ed\u00ee\b"+
-    "\20\1\2\u00ee\u00ef\5\"\22\2\u00ef\u00f0\t\3\2\2\u00f0\u00f1\5\36\20\3"+
-    "\u00f1\u00f2\b\20\1\2\u00f2\u00f7\3\2\2\2\u00f3\u00f4\5 \21\2\u00f4\u00f5"+
-    "\b\20\1\2\u00f5\u00f7\3\2\2\2\u00f6\u00ed\3\2\2\2\u00f6\u00f3\3\2\2\2"+
-    "\u00f7\u013d\3\2\2\2\u00f8\u00f9\f\20\2\2\u00f9\u00fa\t\4\2\2\u00fa\u00fb"+
-    "\5\36\20\21\u00fb\u00fc\b\20\1\2\u00fc\u013c\3\2\2\2\u00fd\u00fe\f\17"+
-    "\2\2\u00fe\u00ff\t\5\2\2\u00ff\u0100\5\36\20\20\u0100\u0101\b\20\1\2\u0101"+
-    "\u013c\3\2\2\2\u0102\u0103\f\16\2\2\u0103\u0104\t\6\2\2\u0104\u0105\5"+
-    "\36\20\17\u0105\u0106\b\20\1\2\u0106\u013c\3\2\2\2\u0107\u0108\f\r\2\2"+
-    "\u0108\u0109\t\7\2\2\u0109\u010a\5\36\20\16\u010a\u010b\b\20\1\2\u010b"+
-    "\u013c\3\2\2\2\u010c\u010d\f\f\2\2\u010d\u010e\t\b\2\2\u010e\u010f\5\36"+
-    "\20\r\u010f\u0110\b\20\1\2\u0110\u013c\3\2\2\2\u0111\u0112\f\n\2\2\u0112"+
-    "\u0113\t\t\2\2\u0113\u0114\5\36\20\13\u0114\u0115\b\20\1\2\u0115\u013c"+
-    "\3\2\2\2\u0116\u0117\f\t\2\2\u0117\u0118\7.\2\2\u0118\u0119\5\36\20\n"+
-    "\u0119\u011a\b\20\1\2\u011a\u013c\3\2\2\2\u011b\u011c\f\b\2\2\u011c\u011d"+
-    "\7/\2\2\u011d\u011e\5\36\20\t\u011e\u011f\b\20\1\2\u011f\u013c\3\2\2\2"+
-    "\u0120\u0121\f\7\2\2\u0121\u0122\7\60\2\2\u0122\u0123\5\36\20\b\u0123"+
-    "\u0124\b\20\1\2\u0124\u013c\3\2\2\2\u0125\u0126\f\6\2\2\u0126\u0127\7"+
-    "\61\2\2\u0127\u0128\5\36\20\7\u0128\u0129\b\20\1\2\u0129\u013c\3\2\2\2"+
-    "\u012a\u012b\f\5\2\2\u012b\u012c\7\62\2\2\u012c\u012d\5\36\20\6\u012d"+
-    "\u012e\b\20\1\2\u012e\u013c\3\2\2\2\u012f\u0130\f\4\2\2\u0130\u0131\7"+
-    "\63\2\2\u0131\u0132\5\36\20\2\u0132\u0133\7\64\2\2\u0133\u0134\5\36\20"+
-    "\4\u0134\u0135\b\20\1\2\u0135\u013c\3\2\2\2\u0136\u0137\f\13\2\2\u0137"+
-    "\u0138\7\33\2\2\u0138\u0139\5\26\f\2\u0139\u013a\b\20\1\2\u013a\u013c"+
-    "\3\2\2\2\u013b\u00f8\3\2\2\2\u013b\u00fd\3\2\2\2\u013b\u0102\3\2\2\2\u013b"+
-    "\u0107\3\2\2\2\u013b\u010c\3\2\2\2\u013b\u0111\3\2\2\2\u013b\u0116\3\2"+
-    "\2\2\u013b\u011b\3\2\2\2\u013b\u0120\3\2\2\2\u013b\u0125\3\2\2\2\u013b"+
-    "\u012a\3\2\2\2\u013b\u012f\3\2\2\2\u013b\u0136\3\2\2\2\u013c\u013f\3\2"+
-    "\2\2\u013d\u013b\3\2\2\2\u013d\u013e\3\2\2\2\u013e\37\3\2\2\2\u013f\u013d"+
-    "\3\2\2\2\u0140\u0141\6\21\20\3\u0141\u0142\t\n\2\2\u0142\u0166\5\"\22"+
-    "\2\u0143\u0144\6\21\21\3\u0144\u0145\5\"\22\2\u0145\u0146\t\n\2\2\u0146"+
-    "\u0166\3\2\2\2\u0147\u0148\6\21\22\3\u0148\u0166\5\"\22\2\u0149\u014a"+
-    "\6\21\23\3\u014a\u014b\t\13\2\2\u014b\u0166\b\21\1\2\u014c\u014d\6\21"+
-    "\24\3\u014d\u014e\7M\2\2\u014e\u0166\b\21\1\2\u014f\u0150\6\21\25\3\u0150"+
-    "\u0151\7N\2\2\u0151\u0166\b\21\1\2\u0152\u0153\6\21\26\3\u0153\u0154\7"+
-    "O\2\2\u0154\u0166\b\21\1\2\u0155\u0156\6\21\27\3\u0156\u0157\5@!\2\u0157"+
-    "\u0158\b\21\1\2\u0158\u0166\3\2\2\2\u0159\u015a\6\21\30\3\u015a\u015b"+
-    "\5B\"\2\u015b\u015c\b\21\1\2\u015c\u0166\3\2\2\2\u015d\u015e\6\21\31\3"+
-    "\u015e\u015f\t\f\2\2\u015f\u0166\5 \21\2\u0160\u0161\7\t\2\2\u0161\u0162"+
-    "\5\26\f\2\u0162\u0163\7\n\2\2\u0163\u0164\5 \21\2\u0164\u0166\3\2\2\2"+
-    "\u0165\u0140\3\2\2\2\u0165\u0143\3\2\2\2\u0165\u0147\3\2\2\2\u0165\u0149"+
-    "\3\2\2\2\u0165\u014c\3\2\2\2\u0165\u014f\3\2\2\2\u0165\u0152\3\2\2\2\u0165"+
-    "\u0155\3\2\2\2\u0165\u0159\3\2\2\2\u0165\u015d\3\2\2\2\u0165\u0160\3\2"+
-    "\2\2\u0166!\3\2\2\2\u0167\u016b\5$\23\2\u0168\u016a\5&\24\2\u0169\u0168"+
-    "\3\2\2\2\u016a\u016d\3\2\2\2\u016b\u0169\3\2\2\2\u016b\u016c\3\2\2\2\u016c"+
-    "\u0178\3\2\2\2\u016d\u016b\3\2\2\2\u016e\u016f\5\26\f\2\u016f\u0173\5"+
-    "(\25\2\u0170\u0172\5&\24\2\u0171\u0170\3\2\2\2\u0172\u0175\3\2\2\2\u0173"+
-    "\u0171\3\2\2\2\u0173\u0174\3\2\2\2\u0174\u0178\3\2\2\2\u0175\u0173\3\2"+
-    "\2\2\u0176\u0178\5> \2\u0177\u0167\3\2\2\2\u0177\u016e\3\2\2\2\u0177\u0176"+
-    "\3\2\2\2\u0178#\3\2\2\2\u0179\u017a\6\23\32\3\u017a\u017b\7\t\2\2\u017b"+
-    "\u017c\5\36\20\2\u017c\u017d\7\n\2\2\u017d\u017e\b\23\1\2\u017e\u018d"+
-    "\3\2\2\2\u017f\u0180\6\23\33\3\u0180\u0181\7\t\2\2\u0181\u0182\5 \21\2"+
-    "\u0182\u0183\7\n\2\2\u0183\u018d\3\2\2\2\u0184\u018d\7K\2\2\u0185\u018d"+
-    "\7L\2\2\u0186\u018d\7Q\2\2\u0187\u0188\7Q\2\2\u0188\u018d\5,\27\2\u0189"+
-    "\u018a\7\26\2\2\u018a\u018b\7P\2\2\u018b\u018d\5,\27\2\u018c\u0179\3\2"+
-    "\2\2\u018c\u017f\3\2\2\2\u018c\u0184\3\2\2\2\u018c\u0185\3\2\2\2\u018c"+
-    "\u0186\3\2\2\2\u018c\u0187\3\2\2\2\u018c\u0189\3\2\2\2\u018d%\3\2\2\2"+
-    "\u018e\u018f\6\24\34\3\u018f\u0193\5(\25\2\u0190\u0191\6\24\35\3\u0191"+
-    "\u0193\5*\26\2\u0192\u018e\3\2\2\2\u0192\u0190\3\2\2\2\u0193\'\3\2\2\2"+
-    "\u0194\u0195\7\13\2\2\u0195\u0196\7S\2\2\u0196\u019a\5,\27\2\u0197\u0198"+
-    "\7\13\2\2\u0198\u019a\t\r\2\2\u0199\u0194\3\2\2\2\u0199\u0197\3\2\2\2"+
-    "\u019a)\3\2\2\2\u019b\u019c\7\7\2\2\u019c\u019d\5\36\20\2\u019d\u019e"+
-    "\7\b\2\2\u019e+\3\2\2\2\u019f\u01a8\7\t\2\2\u01a0\u01a5\5.\30\2\u01a1"+
-    "\u01a2\7\f\2\2\u01a2\u01a4\5.\30\2\u01a3\u01a1\3\2\2\2\u01a4\u01a7\3\2"+
-    "\2\2\u01a5\u01a3\3\2\2\2\u01a5\u01a6\3\2\2\2\u01a6\u01a9\3\2\2\2\u01a7"+
-    "\u01a5\3\2\2\2\u01a8\u01a0\3\2\2\2\u01a8\u01a9\3\2\2\2\u01a9\u01aa\3\2"+
-    "\2\2\u01aa\u01ab\7\n\2\2\u01ab-\3\2\2\2\u01ac\u01b0\5\36\20\2\u01ad\u01b0"+
-    "\5\60\31\2\u01ae\u01b0\5\64\33\2\u01af\u01ac\3\2\2\2\u01af\u01ad\3\2\2"+
-    "\2\u01af\u01ae\3\2\2\2\u01b0/\3\2\2\2\u01b1\u01bf\5\62\32\2\u01b2\u01bb"+
-    "\7\t\2\2\u01b3\u01b8\5\62\32\2\u01b4\u01b5\7\f\2\2\u01b5\u01b7\5\62\32"+
-    "\2\u01b6\u01b4\3\2\2\2\u01b7\u01ba\3\2\2\2\u01b8\u01b6\3\2\2\2\u01b8\u01b9"+
-    "\3\2\2\2\u01b9\u01bc\3\2\2\2\u01ba\u01b8\3\2\2\2\u01bb\u01b3\3\2\2\2\u01bb"+
-    "\u01bc\3\2\2\2\u01bc\u01bd\3\2\2\2\u01bd\u01bf\7\n\2\2\u01be\u01b1\3\2"+
-    "\2\2\u01be\u01b2\3\2\2\2\u01bf\u01c0\3\2\2\2\u01c0\u01c3\7\66\2\2\u01c1"+
-    "\u01c4\5\f\7\2\u01c2\u01c4\5\36\20\2\u01c3\u01c1\3\2\2\2\u01c3\u01c2\3"+
-    "\2\2\2\u01c4\61\3\2\2\2\u01c5\u01c7\5\26\f\2\u01c6\u01c5\3\2\2\2\u01c6"+
-    "\u01c7\3\2\2\2\u01c7\u01c8\3\2\2\2\u01c8\u01c9\7Q\2\2\u01c9\63\3\2\2\2"+
-    "\u01ca\u01cf\5\66\34\2\u01cb\u01cf\58\35\2\u01cc\u01cf\5:\36\2\u01cd\u01cf"+
-    "\5<\37\2\u01ce\u01ca\3\2\2\2\u01ce\u01cb\3\2\2\2\u01ce\u01cc\3\2\2\2\u01ce"+
-    "\u01cd\3\2\2\2\u01cf\65\3\2\2\2\u01d0\u01d1\7P\2\2\u01d1\u01d2\7\65\2"+
-    "\2\u01d2\u01d3\7Q\2\2\u01d3\67\3\2\2\2\u01d4\u01d5\5\26\f\2\u01d5\u01d6"+
-    "\7\65\2\2\u01d6\u01d7\7\26\2\2\u01d79\3\2\2\2\u01d8\u01d9\7Q\2\2\u01d9"+
-    "\u01da\7\65\2\2\u01da\u01db\7Q\2\2\u01db;\3\2\2\2\u01dc\u01dd\7\32\2\2"+
-    "\u01dd\u01de\7\65\2\2\u01de\u01df\7Q\2\2\u01df=\3\2\2\2\u01e0\u01e1\7"+
-    "\26\2\2\u01e1\u01e6\7P\2\2\u01e2\u01e3\7\7\2\2\u01e3\u01e4\5\36\20\2\u01e4"+
-    "\u01e5\7\b\2\2\u01e5\u01e7\3\2\2\2\u01e6\u01e2\3\2\2\2\u01e7\u01e8\3\2"+
-    "\2\2\u01e8\u01e6\3\2\2\2\u01e8\u01e9\3\2\2\2\u01e9\u01f1\3\2\2\2\u01ea"+
-    "\u01ee\5(\25\2\u01eb\u01ed\5&\24\2\u01ec\u01eb\3\2\2\2\u01ed\u01f0\3\2"+
-    "\2\2\u01ee\u01ec\3\2\2\2\u01ee\u01ef\3\2\2\2\u01ef\u01f2\3\2\2\2\u01f0"+
-    "\u01ee\3\2\2\2\u01f1\u01ea\3\2\2\2\u01f1\u01f2\3\2\2\2\u01f2\u0207\3\2"+
-    "\2\2\u01f3\u01f4\7\26\2\2\u01f4\u01f5\7P\2\2\u01f5\u01f6\7\7\2\2\u01f6"+
-    "\u01f7\7\b\2\2\u01f7\u0200\7\5\2\2\u01f8\u01fd\5\36\20\2\u01f9\u01fa\7"+
-    "\f\2\2\u01fa\u01fc\5\36\20\2\u01fb\u01f9\3\2\2\2\u01fc\u01ff\3\2\2\2\u01fd"+
-    "\u01fb\3\2\2\2\u01fd\u01fe\3\2\2\2\u01fe\u0201\3\2\2\2\u01ff\u01fd\3\2"+
-    "\2\2\u0200\u01f8\3\2\2\2\u0200\u0201\3\2\2\2\u0201\u0203\3\2\2\2\u0202"+
-    "\u0204\7\r\2\2\u0203\u0202\3\2\2\2\u0203\u0204\3\2\2\2\u0204\u0205\3\2"+
-    "\2\2\u0205\u0207\7\6\2\2\u0206\u01e0\3\2\2\2\u0206\u01f3\3\2\2\2\u0207"+
-    "?\3\2\2\2\u0208\u0209\7\7\2\2\u0209\u020e\5\36\20\2\u020a\u020b\7\f\2"+
-    "\2\u020b\u020d\5\36\20\2\u020c\u020a\3\2\2\2\u020d\u0210\3\2\2\2\u020e"+
-    "\u020c\3\2\2\2\u020e\u020f\3\2\2\2\u020f\u0211\3\2\2\2\u0210\u020e\3\2"+
-    "\2\2\u0211\u0212\7\b\2\2\u0212\u0216\3\2\2\2\u0213\u0214\7\7\2\2\u0214"+
-    "\u0216\7\b\2\2\u0215\u0208\3\2\2\2\u0215\u0213\3\2\2\2\u0216A\3\2\2\2"+
-    "\u0217\u0218\7\7\2\2\u0218\u021d\5D#\2\u0219\u021a\7\f\2\2\u021a\u021c"+
-    "\5D#\2\u021b\u0219\3\2\2\2\u021c\u021f\3\2\2\2\u021d\u021b\3\2\2\2\u021d"+
-    "\u021e\3\2\2\2\u021e\u0220\3\2\2\2\u021f\u021d\3\2\2\2\u0220\u0221\7\b"+
-    "\2\2\u0221\u0226\3\2\2\2\u0222\u0223\7\7\2\2\u0223\u0224\7\64\2\2\u0224"+
-    "\u0226\7\b\2\2\u0225\u0217\3\2\2\2\u0225\u0222\3\2\2\2\u0226C\3\2\2\2"+
-    "\u0227\u0228\5\36\20\2\u0228\u0229\7\64\2\2\u0229\u022a\5\36\20\2\u022a"+
-    "E\3\2\2\2\62IObeqy\u0086\u008a\u008e\u0093\u00ae\u00b7\u00bb\u00c1\u00ca"+
-    "\u00d4\u00dc\u00e2\u00f6\u013b\u013d\u0165\u016b\u0173\u0177\u018c\u0192"+
-    "\u0199\u01a5\u01a8\u01af\u01b8\u01bb\u01be\u01c3\u01c6\u01ce\u01e8\u01ee"+
-    "\u01f1\u01fd\u0200\u0203\u0206\u020e\u0215\u021d\u0225";
+    "\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\7\20\u0144"+
+    "\n\20\f\20\16\20\u0147\13\20\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3"+
+    "\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3"+
+    "\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3\21\3"+
+    "\21\5\21\u016e\n\21\3\22\3\22\7\22\u0172\n\22\f\22\16\22\u0175\13\22\3"+
+    "\22\3\22\3\22\7\22\u017a\n\22\f\22\16\22\u017d\13\22\3\22\5\22\u0180\n"+
+    "\22\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3\23\3"+
+    "\23\3\23\3\23\3\23\3\23\3\23\5\23\u0195\n\23\3\24\3\24\3\24\3\24\5\24"+
+    "\u019b\n\24\3\25\3\25\3\25\3\25\3\25\5\25\u01a2\n\25\3\26\3\26\3\26\3"+
+    "\26\3\27\3\27\3\27\3\27\7\27\u01ac\n\27\f\27\16\27\u01af\13\27\5\27\u01b1"+
+    "\n\27\3\27\3\27\3\30\3\30\3\30\5\30\u01b8\n\30\3\31\3\31\3\31\3\31\3\31"+
+    "\7\31\u01bf\n\31\f\31\16\31\u01c2\13\31\5\31\u01c4\n\31\3\31\5\31\u01c7"+
+    "\n\31\3\31\3\31\3\31\5\31\u01cc\n\31\3\32\5\32\u01cf\n\32\3\32\3\32\3"+
+    "\33\3\33\3\33\3\33\5\33\u01d7\n\33\3\34\3\34\3\34\3\34\3\35\3\35\3\35"+
+    "\3\35\3\36\3\36\3\36\3\36\3\37\3\37\3\37\3\37\3 \3 \3 \3 \3 \3 \6 \u01ef"+
+    "\n \r \16 \u01f0\3 \3 \7 \u01f5\n \f \16 \u01f8\13 \5 \u01fa\n \3 \3 "+
+    "\3 \3 \3 \3 \3 \3 \7 \u0204\n \f \16 \u0207\13 \5 \u0209\n \3 \5 \u020c"+
+    "\n \3 \5 \u020f\n \3!\3!\3!\3!\7!\u0215\n!\f!\16!\u0218\13!\3!\3!\3!\3"+
+    "!\5!\u021e\n!\3\"\3\"\3\"\3\"\7\"\u0224\n\"\f\"\16\"\u0227\13\"\3\"\3"+
+    "\"\3\"\3\"\3\"\5\"\u022e\n\"\3#\3#\3#\3#\3#\2\3\36$\2\4\6\b\n\f\16\20"+
+    "\22\24\26\30\32\34\36 \"$&(*,.\60\62\64\668:<>@BD\2\16\3\3\r\r\3\2<G\3"+
+    "\2\37!\3\2\"#\3\289\3\2$&\3\2\'*\3\2+.\3\2:;\3\2HK\4\2\35\36\"#\3\2ST"+
+    "\u0269\2I\3\2\2\2\4T\3\2\2\2\6Y\3\2\2\2\b\u00bf\3\2\2\2\n\u00c3\3\2\2"+
+    "\2\f\u00c5\3\2\2\2\16\u00ce\3\2\2\2\20\u00d2\3\2\2\2\22\u00d4\3\2\2\2"+
+    "\24\u00d6\3\2\2\2\26\u00df\3\2\2\2\30\u00e7\3\2\2\2\32\u00ec\3\2\2\2\34"+
+    "\u00f3\3\2\2\2\36\u00fe\3\2\2\2 \u016d\3\2\2\2\"\u017f\3\2\2\2$\u0194"+
+    "\3\2\2\2&\u019a\3\2\2\2(\u01a1\3\2\2\2*\u01a3\3\2\2\2,\u01a7\3\2\2\2."+
+    "\u01b7\3\2\2\2\60\u01c6\3\2\2\2\62\u01ce\3\2\2\2\64\u01d6\3\2\2\2\66\u01d8"+
+    "\3\2\2\28\u01dc\3\2\2\2:\u01e0\3\2\2\2<\u01e4\3\2\2\2>\u020e\3\2\2\2@"+
+    "\u021d\3\2\2\2B\u022d\3\2\2\2D\u022f\3\2\2\2FH\5\4\3\2GF\3\2\2\2HK\3\2"+
+    "\2\2IG\3\2\2\2IJ\3\2\2\2JO\3\2\2\2KI\3\2\2\2LN\5\b\5\2ML\3\2\2\2NQ\3\2"+
+    "\2\2OM\3\2\2\2OP\3\2\2\2PR\3\2\2\2QO\3\2\2\2RS\7\2\2\3S\3\3\2\2\2TU\5"+
+    "\26\f\2UV\7R\2\2VW\5\6\4\2WX\5\f\7\2X\5\3\2\2\2Ye\7\t\2\2Z[\5\26\f\2["+
+    "b\7R\2\2\\]\7\f\2\2]^\5\26\f\2^_\7R\2\2_a\3\2\2\2`\\\3\2\2\2ad\3\2\2\2"+
+    "b`\3\2\2\2bc\3\2\2\2cf\3\2\2\2db\3\2\2\2eZ\3\2\2\2ef\3\2\2\2fg\3\2\2\2"+
+    "gh\7\n\2\2h\7\3\2\2\2ij\7\16\2\2jk\7\t\2\2kl\5\36\20\2lm\7\n\2\2mq\5\n"+
+    "\6\2no\7\20\2\2or\5\n\6\2pr\6\5\2\2qn\3\2\2\2qp\3\2\2\2r\u00c0\3\2\2\2"+
+    "st\7\21\2\2tu\7\t\2\2uv\5\36\20\2vy\7\n\2\2wz\5\n\6\2xz\5\16\b\2yw\3\2"+
+    "\2\2yx\3\2\2\2z\u00c0\3\2\2\2{|\7\22\2\2|}\5\f\7\2}~\7\21\2\2~\177\7\t"+
+    "\2\2\177\u0080\5\36\20\2\u0080\u0081\7\n\2\2\u0081\u0082\5\34\17\2\u0082"+
+    "\u00c0\3\2\2\2\u0083\u0084\7\23\2\2\u0084\u0086\7\t\2\2\u0085\u0087\5"+
+    "\20\t\2\u0086\u0085\3\2\2\2\u0086\u0087\3\2\2\2\u0087\u0088\3\2\2\2\u0088"+
+    "\u008a\7\r\2\2\u0089\u008b\5\36\20\2\u008a\u0089\3\2\2\2\u008a\u008b\3"+
+    "\2\2\2\u008b\u008c\3\2\2\2\u008c\u008e\7\r\2\2\u008d\u008f\5\22\n\2\u008e"+
+    "\u008d\3\2\2\2\u008e\u008f\3\2\2\2\u008f\u0090\3\2\2\2\u0090\u0093\7\n"+
+    "\2\2\u0091\u0094\5\n\6\2\u0092\u0094\5\16\b\2\u0093\u0091\3\2\2\2\u0093"+
+    "\u0092\3\2\2\2\u0094\u00c0\3\2\2\2\u0095\u0096\7\23\2\2\u0096\u0097\7"+
+    "\t\2\2\u0097\u0098\5\26\f\2\u0098\u0099\7R\2\2\u0099\u009a\7\65\2\2\u009a"+
+    "\u009b\5\36\20\2\u009b\u009c\7\n\2\2\u009c\u009d\5\n\6\2\u009d\u00c0\3"+
+    "\2\2\2\u009e\u009f\7\23\2\2\u009f\u00a0\7\t\2\2\u00a0\u00a1\7R\2\2\u00a1"+
+    "\u00a2\7\17\2\2\u00a2\u00a3\5\36\20\2\u00a3\u00a4\7\n\2\2\u00a4\u00a5"+
+    "\5\n\6\2\u00a5\u00c0\3\2\2\2\u00a6\u00a7\5\24\13\2\u00a7\u00a8\5\34\17"+
+    "\2\u00a8\u00c0\3\2\2\2\u00a9\u00aa\7\24\2\2\u00aa\u00c0\5\34\17\2\u00ab"+
+    "\u00ac\7\25\2\2\u00ac\u00c0\5\34\17\2\u00ad\u00ae\7\26\2\2\u00ae\u00af"+
+    "\5\36\20\2\u00af\u00b0\5\34\17\2\u00b0\u00c0\3\2\2\2\u00b1\u00b2\7\30"+
+    "\2\2\u00b2\u00b4\5\f\7\2\u00b3\u00b5\5\32\16\2\u00b4\u00b3\3\2\2\2\u00b5"+
+    "\u00b6\3\2\2\2\u00b6\u00b4\3\2\2\2\u00b6\u00b7\3\2\2\2\u00b7\u00c0\3\2"+
+    "\2\2\u00b8\u00b9\7\32\2\2\u00b9\u00ba\5\36\20\2\u00ba\u00bb\5\34\17\2"+
+    "\u00bb\u00c0\3\2\2\2\u00bc\u00bd\5\36\20\2\u00bd\u00be\5\34\17\2\u00be"+
+    "\u00c0\3\2\2\2\u00bfi\3\2\2\2\u00bfs\3\2\2\2\u00bf{\3\2\2\2\u00bf\u0083"+
+    "\3\2\2\2\u00bf\u0095\3\2\2\2\u00bf\u009e\3\2\2\2\u00bf\u00a6\3\2\2\2\u00bf"+
+    "\u00a9\3\2\2\2\u00bf\u00ab\3\2\2\2\u00bf\u00ad\3\2\2\2\u00bf\u00b1\3\2"+
+    "\2\2\u00bf\u00b8\3\2\2\2\u00bf\u00bc\3\2\2\2\u00c0\t\3\2\2\2\u00c1\u00c4"+
+    "\5\f\7\2\u00c2\u00c4\5\b\5\2\u00c3\u00c1\3\2\2\2\u00c3\u00c2\3\2\2\2\u00c4"+
+    "\13\3\2\2\2\u00c5\u00c9\7\5\2\2\u00c6\u00c8\5\b\5\2\u00c7\u00c6\3\2\2"+
+    "\2\u00c8\u00cb\3\2\2\2\u00c9\u00c7\3\2\2\2\u00c9\u00ca\3\2\2\2\u00ca\u00cc"+
+    "\3\2\2\2\u00cb\u00c9\3\2\2\2\u00cc\u00cd\7\6\2\2\u00cd\r\3\2\2\2\u00ce"+
+    "\u00cf\7\r\2\2\u00cf\17\3\2\2\2\u00d0\u00d3\5\24\13\2\u00d1\u00d3\5\36"+
+    "\20\2\u00d2\u00d0\3\2\2\2\u00d2\u00d1\3\2\2\2\u00d3\21\3\2\2\2\u00d4\u00d5"+
+    "\5\36\20\2\u00d5\23\3\2\2\2\u00d6\u00d7\5\26\f\2\u00d7\u00dc\5\30\r\2"+
+    "\u00d8\u00d9\7\f\2\2\u00d9\u00db\5\30\r\2\u00da\u00d8\3\2\2\2\u00db\u00de"+
+    "\3\2\2\2\u00dc\u00da\3\2\2\2\u00dc\u00dd\3\2\2\2\u00dd\25\3\2\2\2\u00de"+
+    "\u00dc\3\2\2\2\u00df\u00e4\7Q\2\2\u00e0\u00e1\7\7\2\2\u00e1\u00e3\7\b"+
+    "\2\2\u00e2\u00e0\3\2\2\2\u00e3\u00e6\3\2\2\2\u00e4\u00e2\3\2\2\2\u00e4"+
+    "\u00e5\3\2\2\2\u00e5\27\3\2\2\2\u00e6\u00e4\3\2\2\2\u00e7\u00ea\7R\2\2"+
+    "\u00e8\u00e9\7<\2\2\u00e9\u00eb\5\36\20\2\u00ea\u00e8\3\2\2\2\u00ea\u00eb"+
+    "\3\2\2\2\u00eb\31\3\2\2\2\u00ec\u00ed\7\31\2\2\u00ed\u00ee\7\t\2\2\u00ee"+
+    "\u00ef\7Q\2\2\u00ef\u00f0\7R\2\2\u00f0\u00f1\7\n\2\2\u00f1\u00f2\5\f\7"+
+    "\2\u00f2\33\3\2\2\2\u00f3\u00f4\t\2\2\2\u00f4\35\3\2\2\2\u00f5\u00f6\b"+
+    "\20\1\2\u00f6\u00f7\5\"\22\2\u00f7\u00f8\t\3\2\2\u00f8\u00f9\5\36\20\3"+
+    "\u00f9\u00fa\b\20\1\2\u00fa\u00ff\3\2\2\2\u00fb\u00fc\5 \21\2\u00fc\u00fd"+
+    "\b\20\1\2\u00fd\u00ff\3\2\2\2\u00fe\u00f5\3\2\2\2\u00fe\u00fb\3\2\2\2"+
+    "\u00ff\u0145\3\2\2\2\u0100\u0101\f\20\2\2\u0101\u0102\t\4\2\2\u0102\u0103"+
+    "\5\36\20\21\u0103\u0104\b\20\1\2\u0104\u0144\3\2\2\2\u0105\u0106\f\17"+
+    "\2\2\u0106\u0107\t\5\2\2\u0107\u0108\5\36\20\20\u0108\u0109\b\20\1\2\u0109"+
+    "\u0144\3\2\2\2\u010a\u010b\f\16\2\2\u010b\u010c\t\6\2\2\u010c\u010d\5"+
+    "\36\20\17\u010d\u010e\b\20\1\2\u010e\u0144\3\2\2\2\u010f\u0110\f\r\2\2"+
+    "\u0110\u0111\t\7\2\2\u0111\u0112\5\36\20\16\u0112\u0113\b\20\1\2\u0113"+
+    "\u0144\3\2\2\2\u0114\u0115\f\f\2\2\u0115\u0116\t\b\2\2\u0116\u0117\5\36"+
+    "\20\r\u0117\u0118\b\20\1\2\u0118\u0144\3\2\2\2\u0119\u011a\f\n\2\2\u011a"+
+    "\u011b\t\t\2\2\u011b\u011c\5\36\20\13\u011c\u011d\b\20\1\2\u011d\u0144"+
+    "\3\2\2\2\u011e\u011f\f\t\2\2\u011f\u0120\7/\2\2\u0120\u0121\5\36\20\n"+
+    "\u0121\u0122\b\20\1\2\u0122\u0144\3\2\2\2\u0123\u0124\f\b\2\2\u0124\u0125"+
+    "\7\60\2\2\u0125\u0126\5\36\20\t\u0126\u0127\b\20\1\2\u0127\u0144\3\2\2"+
+    "\2\u0128\u0129\f\7\2\2\u0129\u012a\7\61\2\2\u012a\u012b\5\36\20\b\u012b"+
+    "\u012c\b\20\1\2\u012c\u0144\3\2\2\2\u012d\u012e\f\6\2\2\u012e\u012f\7"+
+    "\62\2\2\u012f\u0130\5\36\20\7\u0130\u0131\b\20\1\2\u0131\u0144\3\2\2\2"+
+    "\u0132\u0133\f\5\2\2\u0133\u0134\7\63\2\2\u0134\u0135\5\36\20\6\u0135"+
+    "\u0136\b\20\1\2\u0136\u0144\3\2\2\2\u0137\u0138\f\4\2\2\u0138\u0139\7"+
+    "\64\2\2\u0139\u013a\5\36\20\2\u013a\u013b\7\65\2\2\u013b\u013c\5\36\20"+
+    "\4\u013c\u013d\b\20\1\2\u013d\u0144\3\2\2\2\u013e\u013f\f\13\2\2\u013f"+
+    "\u0140\7\34\2\2\u0140\u0141\5\26\f\2\u0141\u0142\b\20\1\2\u0142\u0144"+
+    "\3\2\2\2\u0143\u0100\3\2\2\2\u0143\u0105\3\2\2\2\u0143\u010a\3\2\2\2\u0143"+
+    "\u010f\3\2\2\2\u0143\u0114\3\2\2\2\u0143\u0119\3\2\2\2\u0143\u011e\3\2"+
+    "\2\2\u0143\u0123\3\2\2\2\u0143\u0128\3\2\2\2\u0143\u012d\3\2\2\2\u0143"+
+    "\u0132\3\2\2\2\u0143\u0137\3\2\2\2\u0143\u013e\3\2\2\2\u0144\u0147\3\2"+
+    "\2\2\u0145\u0143\3\2\2\2\u0145\u0146\3\2\2\2\u0146\37\3\2\2\2\u0147\u0145"+
+    "\3\2\2\2\u0148\u0149\6\21\20\3\u0149\u014a\t\n\2\2\u014a\u016e\5\"\22"+
+    "\2\u014b\u014c\6\21\21\3\u014c\u014d\5\"\22\2\u014d\u014e\t\n\2\2\u014e"+
+    "\u016e\3\2\2\2\u014f\u0150\6\21\22\3\u0150\u016e\5\"\22\2\u0151\u0152"+
+    "\6\21\23\3\u0152\u0153\t\13\2\2\u0153\u016e\b\21\1\2\u0154\u0155\6\21"+
+    "\24\3\u0155\u0156\7N\2\2\u0156\u016e\b\21\1\2\u0157\u0158\6\21\25\3\u0158"+
+    "\u0159\7O\2\2\u0159\u016e\b\21\1\2\u015a\u015b\6\21\26\3\u015b\u015c\7"+
+    "P\2\2\u015c\u016e\b\21\1\2\u015d\u015e\6\21\27\3\u015e\u015f\5@!\2\u015f"+
+    "\u0160\b\21\1\2\u0160\u016e\3\2\2\2\u0161\u0162\6\21\30\3\u0162\u0163"+
+    "\5B\"\2\u0163\u0164\b\21\1\2\u0164\u016e\3\2\2\2\u0165\u0166\6\21\31\3"+
+    "\u0166\u0167\t\f\2\2\u0167\u016e\5 \21\2\u0168\u0169\7\t\2\2\u0169\u016a"+
+    "\5\26\f\2\u016a\u016b\7\n\2\2\u016b\u016c\5 \21\2\u016c\u016e\3\2\2\2"+
+    "\u016d\u0148\3\2\2\2\u016d\u014b\3\2\2\2\u016d\u014f\3\2\2\2\u016d\u0151"+
+    "\3\2\2\2\u016d\u0154\3\2\2\2\u016d\u0157\3\2\2\2\u016d\u015a\3\2\2\2\u016d"+
+    "\u015d\3\2\2\2\u016d\u0161\3\2\2\2\u016d\u0165\3\2\2\2\u016d\u0168\3\2"+
+    "\2\2\u016e!\3\2\2\2\u016f\u0173\5$\23\2\u0170\u0172\5&\24\2\u0171\u0170"+
+    "\3\2\2\2\u0172\u0175\3\2\2\2\u0173\u0171\3\2\2\2\u0173\u0174\3\2\2\2\u0174"+
+    "\u0180\3\2\2\2\u0175\u0173\3\2\2\2\u0176\u0177\5\26\f\2\u0177\u017b\5"+
+    "(\25\2\u0178\u017a\5&\24\2\u0179\u0178\3\2\2\2\u017a\u017d\3\2\2\2\u017b"+
+    "\u0179\3\2\2\2\u017b\u017c\3\2\2\2\u017c\u0180\3\2\2\2\u017d\u017b\3\2"+
+    "\2\2\u017e\u0180\5> \2\u017f\u016f\3\2\2\2\u017f\u0176\3\2\2\2\u017f\u017e"+
+    "\3\2\2\2\u0180#\3\2\2\2\u0181\u0182\6\23\32\3\u0182\u0183\7\t\2\2\u0183"+
+    "\u0184\5\36\20\2\u0184\u0185\7\n\2\2\u0185\u0186\b\23\1\2\u0186\u0195"+
+    "\3\2\2\2\u0187\u0188\6\23\33\3\u0188\u0189\7\t\2\2\u0189\u018a\5 \21\2"+
+    "\u018a\u018b\7\n\2\2\u018b\u0195\3\2\2\2\u018c\u0195\7L\2\2\u018d\u0195"+
+    "\7M\2\2\u018e\u0195\7R\2\2\u018f\u0190\7R\2\2\u0190\u0195\5,\27\2\u0191"+
+    "\u0192\7\27\2\2\u0192\u0193\7Q\2\2\u0193\u0195\5,\27\2\u0194\u0181\3\2"+
+    "\2\2\u0194\u0187\3\2\2\2\u0194\u018c\3\2\2\2\u0194\u018d\3\2\2\2\u0194"+
+    "\u018e\3\2\2\2\u0194\u018f\3\2\2\2\u0194\u0191\3\2\2\2\u0195%\3\2\2\2"+
+    "\u0196\u0197\6\24\34\3\u0197\u019b\5(\25\2\u0198\u0199\6\24\35\3\u0199"+
+    "\u019b\5*\26\2\u019a\u0196\3\2\2\2\u019a\u0198\3\2\2\2\u019b\'\3\2\2\2"+
+    "\u019c\u019d\7\13\2\2\u019d\u019e\7T\2\2\u019e\u01a2\5,\27\2\u019f\u01a0"+
+    "\7\13\2\2\u01a0\u01a2\t\r\2\2\u01a1\u019c\3\2\2\2\u01a1\u019f\3\2\2\2"+
+    "\u01a2)\3\2\2\2\u01a3\u01a4\7\7\2\2\u01a4\u01a5\5\36\20\2\u01a5\u01a6"+
+    "\7\b\2\2\u01a6+\3\2\2\2\u01a7\u01b0\7\t\2\2\u01a8\u01ad\5.\30\2\u01a9"+
+    "\u01aa\7\f\2\2\u01aa\u01ac\5.\30\2\u01ab\u01a9\3\2\2\2\u01ac\u01af\3\2"+
+    "\2\2\u01ad\u01ab\3\2\2\2\u01ad\u01ae\3\2\2\2\u01ae\u01b1\3\2\2\2\u01af"+
+    "\u01ad\3\2\2\2\u01b0\u01a8\3\2\2\2\u01b0\u01b1\3\2\2\2\u01b1\u01b2\3\2"+
+    "\2\2\u01b2\u01b3\7\n\2\2\u01b3-\3\2\2\2\u01b4\u01b8\5\36\20\2\u01b5\u01b8"+
+    "\5\60\31\2\u01b6\u01b8\5\64\33\2\u01b7\u01b4\3\2\2\2\u01b7\u01b5\3\2\2"+
+    "\2\u01b7\u01b6\3\2\2\2\u01b8/\3\2\2\2\u01b9\u01c7\5\62\32\2\u01ba\u01c3"+
+    "\7\t\2\2\u01bb\u01c0\5\62\32\2\u01bc\u01bd\7\f\2\2\u01bd\u01bf\5\62\32"+
+    "\2\u01be\u01bc\3\2\2\2\u01bf\u01c2\3\2\2\2\u01c0\u01be\3\2\2\2\u01c0\u01c1"+
+    "\3\2\2\2\u01c1\u01c4\3\2\2\2\u01c2\u01c0\3\2\2\2\u01c3\u01bb\3\2\2\2\u01c3"+
+    "\u01c4\3\2\2\2\u01c4\u01c5\3\2\2\2\u01c5\u01c7\7\n\2\2\u01c6\u01b9\3\2"+
+    "\2\2\u01c6\u01ba\3\2\2\2\u01c7\u01c8\3\2\2\2\u01c8\u01cb\7\67\2\2\u01c9"+
+    "\u01cc\5\f\7\2\u01ca\u01cc\5\36\20\2\u01cb\u01c9\3\2\2\2\u01cb\u01ca\3"+
+    "\2\2\2\u01cc\61\3\2\2\2\u01cd\u01cf\5\26\f\2\u01ce\u01cd\3\2\2\2\u01ce"+
+    "\u01cf\3\2\2\2\u01cf\u01d0\3\2\2\2\u01d0\u01d1\7R\2\2\u01d1\63\3\2\2\2"+
+    "\u01d2\u01d7\5\66\34\2\u01d3\u01d7\58\35\2\u01d4\u01d7\5:\36\2\u01d5\u01d7"+
+    "\5<\37\2\u01d6\u01d2\3\2\2\2\u01d6\u01d3\3\2\2\2\u01d6\u01d4\3\2\2\2\u01d6"+
+    "\u01d5\3\2\2\2\u01d7\65\3\2\2\2\u01d8\u01d9\7Q\2\2\u01d9\u01da\7\66\2"+
+    "\2\u01da\u01db\7R\2\2\u01db\67\3\2\2\2\u01dc\u01dd\5\26\f\2\u01dd\u01de"+
+    "\7\66\2\2\u01de\u01df\7\27\2\2\u01df9\3\2\2\2\u01e0\u01e1\7R\2\2\u01e1"+
+    "\u01e2\7\66\2\2\u01e2\u01e3\7R\2\2\u01e3;\3\2\2\2\u01e4\u01e5\7\33\2\2"+
+    "\u01e5\u01e6\7\66\2\2\u01e6\u01e7\7R\2\2\u01e7=\3\2\2\2\u01e8\u01e9\7"+
+    "\27\2\2\u01e9\u01ee\7Q\2\2\u01ea\u01eb\7\7\2\2\u01eb\u01ec\5\36\20\2\u01ec"+
+    "\u01ed\7\b\2\2\u01ed\u01ef\3\2\2\2\u01ee\u01ea\3\2\2\2\u01ef\u01f0\3\2"+
+    "\2\2\u01f0\u01ee\3\2\2\2\u01f0\u01f1\3\2\2\2\u01f1\u01f9\3\2\2\2\u01f2"+
+    "\u01f6\5(\25\2\u01f3\u01f5\5&\24\2\u01f4\u01f3\3\2\2\2\u01f5\u01f8\3\2"+
+    "\2\2\u01f6\u01f4\3\2\2\2\u01f6\u01f7\3\2\2\2\u01f7\u01fa\3\2\2\2\u01f8"+
+    "\u01f6\3\2\2\2\u01f9\u01f2\3\2\2\2\u01f9\u01fa\3\2\2\2\u01fa\u020f\3\2"+
+    "\2\2\u01fb\u01fc\7\27\2\2\u01fc\u01fd\7Q\2\2\u01fd\u01fe\7\7\2\2\u01fe"+
+    "\u01ff\7\b\2\2\u01ff\u0208\7\5\2\2\u0200\u0205\5\36\20\2\u0201\u0202\7"+
+    "\f\2\2\u0202\u0204\5\36\20\2\u0203\u0201\3\2\2\2\u0204\u0207\3\2\2\2\u0205"+
+    "\u0203\3\2\2\2\u0205\u0206\3\2\2\2\u0206\u0209\3\2\2\2\u0207\u0205\3\2"+
+    "\2\2\u0208\u0200\3\2\2\2\u0208\u0209\3\2\2\2\u0209\u020b\3\2\2\2\u020a"+
+    "\u020c\7\r\2\2\u020b\u020a\3\2\2\2\u020b\u020c\3\2\2\2\u020c\u020d\3\2"+
+    "\2\2\u020d\u020f\7\6\2\2\u020e\u01e8\3\2\2\2\u020e\u01fb\3\2\2\2\u020f"+
+    "?\3\2\2\2\u0210\u0211\7\7\2\2\u0211\u0216\5\36\20\2\u0212\u0213\7\f\2"+
+    "\2\u0213\u0215\5\36\20\2\u0214\u0212\3\2\2\2\u0215\u0218\3\2\2\2\u0216"+
+    "\u0214\3\2\2\2\u0216\u0217\3\2\2\2\u0217\u0219\3\2\2\2\u0218\u0216\3\2"+
+    "\2\2\u0219\u021a\7\b\2\2\u021a\u021e\3\2\2\2\u021b\u021c\7\7\2\2\u021c"+
+    "\u021e\7\b\2\2\u021d\u0210\3\2\2\2\u021d\u021b\3\2\2\2\u021eA\3\2\2\2"+
+    "\u021f\u0220\7\7\2\2\u0220\u0225\5D#\2\u0221\u0222\7\f\2\2\u0222\u0224"+
+    "\5D#\2\u0223\u0221\3\2\2\2\u0224\u0227\3\2\2\2\u0225\u0223\3\2\2\2\u0225"+
+    "\u0226\3\2\2\2\u0226\u0228\3\2\2\2\u0227\u0225\3\2\2\2\u0228\u0229\7\b"+
+    "\2\2\u0229\u022e\3\2\2\2\u022a\u022b\7\7\2\2\u022b\u022c\7\65\2\2\u022c"+
+    "\u022e\7\b\2\2\u022d\u021f\3\2\2\2\u022d\u022a\3\2\2\2\u022eC\3\2\2\2"+
+    "\u022f\u0230\5\36\20\2\u0230\u0231\7\65\2\2\u0231\u0232\5\36\20\2\u0232"+
+    "E\3\2\2\2\62IObeqy\u0086\u008a\u008e\u0093\u00b6\u00bf\u00c3\u00c9\u00d2"+
+    "\u00dc\u00e4\u00ea\u00fe\u0143\u0145\u016d\u0173\u017b\u017f\u0194\u019a"+
+    "\u01a1\u01ad\u01b0\u01b7\u01c0\u01c3\u01c6\u01cb\u01ce\u01d6\u01f0\u01f6"+
+    "\u01f9\u0205\u0208\u020b\u020e\u0216\u021d\u0225\u022d";
   public static final ATN _ATN =
     new ATNDeserializer().deserialize(_serializedATN.toCharArray());
   static {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
index a11a856edd4..a1279d611e3 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserBaseVisitor.java
@@ -67,6 +67,13 @@ class PainlessParserBaseVisitor<T> extends AbstractParseTreeVisitor<T> implement
    * {@link #visitChildren} on {@code ctx}.</p>
    */
   @Override public T visitEach(PainlessParser.EachContext ctx) { return visitChildren(ctx); }
+  /**
+   * {@inheritDoc}
+   *
+   * <p>The default implementation returns the result of calling
+   * {@link #visitChildren} on {@code ctx}.</p>
+   */
+  @Override public T visitIneach(PainlessParser.IneachContext ctx) { return visitChildren(ctx); }
   /**
    * {@inheritDoc}
    *
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
index bc9e8c4f994..8a297651070 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/PainlessParserVisitor.java
@@ -63,6 +63,13 @@ interface PainlessParserVisitor<T> extends ParseTreeVisitor<T> {
    * @return the visitor result
    */
   T visitEach(PainlessParser.EachContext ctx);
+  /**
+   * Visit a parse tree produced by the {@code ineach}
+   * labeled alternative in {@link PainlessParser#statement}.
+   * @param ctx the parse tree
+   * @return the visitor result
+   */
+  T visitIneach(PainlessParser.IneachContext ctx);
   /**
    * Visit a parse tree produced by the {@code decl}
    * labeled alternative in {@link PainlessParser#statement}.
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
index 88774bb9984..55e3445bace 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java
@@ -81,6 +81,7 @@ import org.elasticsearch.painless.antlr.PainlessParser.ForContext;
 import org.elasticsearch.painless.antlr.PainlessParser.FuncrefContext;
 import org.elasticsearch.painless.antlr.PainlessParser.FunctionContext;
 import org.elasticsearch.painless.antlr.PainlessParser.IfContext;
+import org.elasticsearch.painless.antlr.PainlessParser.IneachContext;
 import org.elasticsearch.painless.antlr.PainlessParser.InitializerContext;
 import org.elasticsearch.painless.antlr.PainlessParser.InstanceofContext;
 import org.elasticsearch.painless.antlr.PainlessParser.LambdaContext;
@@ -358,6 +359,17 @@ public final class Walker extends PainlessParserBaseVisitor<Object> {
 
         return new SEach(location(ctx), type, name, expression, block);
     }
+    
+    @Override
+    public Object visitIneach(IneachContext ctx) {
+        reserved.peek().setMaxLoopCounter(settings.getMaxLoopCounter());
+
+        String name = ctx.ID().getText();
+        AExpression expression = (AExpression)visitExpression(ctx.expression());
+        SBlock block = (SBlock)visit(ctx.trailer());
+
+        return new SEach(location(ctx), "def", name, expression, block);
+    }
 
     @Override
     public Object visitDecl(DeclContext ctx) {
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
index 46dc8af5ab9..3eee2a7b2d8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java
@@ -158,6 +158,7 @@ public class SFunction extends AStatement {
             access |= Opcodes.ACC_SYNTHETIC;
         }
         final MethodWriter function = new MethodWriter(access, method.method, writer, globals.getStatements(), settings);
+        function.visitCode();
         write(function, globals);
         function.endMethod();
     }
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
index 950b021486f..e55ad91d492 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java
@@ -31,6 +31,7 @@ import org.elasticsearch.painless.node.SFunction.Reserved;
 import org.elasticsearch.painless.WriterConstants;
 import org.elasticsearch.painless.Location;
 import org.elasticsearch.painless.MethodWriter;
+import org.elasticsearch.painless.SimpleChecksAdapter;
 import org.objectweb.asm.ClassVisitor;
 import org.objectweb.asm.ClassWriter;
 import org.objectweb.asm.Opcodes;
@@ -146,7 +147,6 @@ public final class SSource extends AStatement {
         // Create the ClassWriter.
 
         int classFrames = ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS;
-        int classVersion = Opcodes.V1_8;
         int classAccess = Opcodes.ACC_PUBLIC | Opcodes.ACC_SUPER | Opcodes.ACC_FINAL;
         String classBase = BASE_CLASS_TYPE.getInternalName();
         String className = CLASS_TYPE.getInternalName();
@@ -155,10 +155,15 @@ public final class SSource extends AStatement {
         ClassWriter writer = new ClassWriter(classFrames);
         ClassVisitor visitor = writer;
         
+        // if picky is enabled, turn on some checks. instead of VerifyError at the end, you get a helpful stacktrace.
+        if (settings.isPicky()) {
+            visitor = new SimpleChecksAdapter(visitor);
+        }
+
         if (debugStream != null) {
             visitor = new TraceClassVisitor(visitor, debugStream, null);
         }
-        visitor.visit(classVersion, classAccess, className, null, classBase, classInterfaces);
+        visitor.visit(WriterConstants.CLASS_VERSION, classAccess, className, null, classBase, classInterfaces);
         visitor.visitSource(Location.computeSourceName(name, source), null);
 
         // Write the constructor:
@@ -207,6 +212,7 @@ public final class SSource extends AStatement {
             // Initialize the constants in a static initializer
             final MethodWriter clinit = new MethodWriter(Opcodes.ACC_STATIC, 
                     WriterConstants.CLINIT, visitor, globals.getStatements(), settings);
+            clinit.visitCode();
             for (Constant constant : inits) {
                 constant.initializer.accept(clinit);
                 clinit.putStatic(CLASS_TYPE, constant.name, constant.type);
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
index 01f3ee42ae6..e023ac364b3 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicStatementTests.java
@@ -130,39 +130,65 @@ public class BasicStatementTests extends ScriptTestCase {
     public void testIterableForEachStatement() {
         assertEquals(6, exec("List l = new ArrayList(); l.add(1); l.add(2); l.add(3); int total = 0;" +
             " for (int x : l) total += x; return total"));
+        assertEquals(6, exec("List l = new ArrayList(); l.add(1); l.add(2); l.add(3); int total = 0;" +
+            " for (x in l) total += x; return total"));
         assertEquals("123", exec("List l = new ArrayList(); l.add('1'); l.add('2'); l.add('3'); String cat = '';" +
             " for (String x : l) cat += x; return cat"));
+        assertEquals("123", exec("List l = new ArrayList(); l.add('1'); l.add('2'); l.add('3'); String cat = '';" +
+            " for (x in l) cat += x; return cat"));
         assertEquals("1236", exec("Map m = new HashMap(); m.put('1', 1); m.put('2', 2); m.put('3', 3);" +
             " String cat = ''; int total = 0;" +
             " for (Map.Entry e : m.entrySet()) { cat += e.getKey(); total += e.getValue(); } return cat + total"));
+        assertEquals("1236", exec("Map m = new HashMap(); m.put('1', 1); m.put('2', 2); m.put('3', 3);" +
+                " String cat = ''; int total = 0;" +
+                " for (e in m.entrySet()) { cat += e.getKey(); total += e.getValue(); } return cat + total"));
     }
 
     public void testIterableForEachStatementDef() {
         assertEquals(6, exec("def l = new ArrayList(); l.add(1); l.add(2); l.add(3); int total = 0;" +
             " for (int x : l) total += x; return total"));
+        assertEquals(6, exec("def l = new ArrayList(); l.add(1); l.add(2); l.add(3); int total = 0;" +
+            " for (x in l) total += x; return total"));
         assertEquals("123", exec("def l = new ArrayList(); l.add('1'); l.add('2'); l.add('3'); String cat = '';" +
             " for (String x : l) cat += x; return cat"));
+        assertEquals("123", exec("def l = new ArrayList(); l.add('1'); l.add('2'); l.add('3'); String cat = '';" +
+            " for (x in l) cat += x; return cat"));
         assertEquals("1236", exec("def m = new HashMap(); m.put('1', 1); m.put('2', 2); m.put('3', 3);" +
             " String cat = ''; int total = 0;" +
             " for (Map.Entry e : m.entrySet()) { cat += e.getKey(); total += e.getValue(); } return cat + total"));
+        assertEquals("1236", exec("def m = new HashMap(); m.put('1', 1); m.put('2', 2); m.put('3', 3);" +
+            " String cat = ''; int total = 0;" +
+            " for (e in m.entrySet()) { cat += e.getKey(); total += e.getValue(); } return cat + total"));
     }
 
     public void testArrayForEachStatement() {
         assertEquals(6, exec("int[] a = new int[3]; a[0] = 1; a[1] = 2; a[2] = 3; int total = 0;" +
             " for (int x : a) total += x; return total"));
+        assertEquals(6, exec("int[] a = new int[3]; a[0] = 1; a[1] = 2; a[2] = 3; int total = 0;" +
+            " for (x in a) total += x; return total"));
         assertEquals("123", exec("String[] a = new String[3]; a[0] = '1'; a[1] = '2'; a[2] = '3'; def total = '';" +
             " for (String x : a) total += x; return total"));
+        assertEquals("123", exec("String[] a = new String[3]; a[0] = '1'; a[1] = '2'; a[2] = '3'; def total = '';" +
+            " for (x in a) total += x; return total"));
         assertEquals(6, exec("int[][] i = new int[3][1]; i[0][0] = 1; i[1][0] = 2; i[2][0] = 3; int total = 0;" +
             " for (int[] j : i) total += j[0]; return total"));
+        assertEquals(6, exec("int[][] i = new int[3][1]; i[0][0] = 1; i[1][0] = 2; i[2][0] = 3; int total = 0;" +
+            " for (j in i) total += j[0]; return total"));
     }
 
     public void testArrayForEachStatementDef() {
         assertEquals(6, exec("def a = new int[3]; a[0] = 1; a[1] = 2; a[2] = 3; int total = 0;" +
             " for (int x : a) total += x; return total"));
+        assertEquals(6, exec("def a = new int[3]; a[0] = 1; a[1] = 2; a[2] = 3; int total = 0;" +
+            " for (x in a) total += x; return total"));
         assertEquals("123", exec("def a = new String[3]; a[0] = '1'; a[1] = '2'; a[2] = '3'; def total = '';" +
             " for (String x : a) total += x; return total"));
+        assertEquals("123", exec("def a = new String[3]; a[0] = '1'; a[1] = '2'; a[2] = '3'; def total = '';" +
+            " for (x in a) total += x; return total"));
         assertEquals(6, exec("def i = new int[3][1]; i[0][0] = 1; i[1][0] = 2; i[2][0] = 3; int total = 0;" +
             " for (int[] j : i) total += j[0]; return total"));
+        assertEquals(6, exec("def i = new int[3][1]; i[0][0] = 1; i[1][0] = 2; i[2][0] = 3; int total = 0;" +
+            " for (j in i) total += j[0]; return total"));
     }
 
     public void testDeclarationStatement() {
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
index dbca5243ec2..6bb800eb92c 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java
@@ -79,7 +79,7 @@ public class LambdaTests extends ScriptTestCase {
     }
 
     public void testMultipleStatements() {
-        assertEquals(2, exec("int applyOne(IntFunction arg) { arg.apply(1) } applyOne(x -> { x = x + 1; return x })"));
+        assertEquals(2, exec("int applyOne(IntFunction arg) { arg.apply(1) } applyOne(x -> { def y = x + 1; return y })"));
     }
 
     public void testUnneededCurlyStatements() {
@@ -138,6 +138,7 @@ public class LambdaTests extends ScriptTestCase {
         assertTrue(expected.getMessage().contains("is read-only"));
     }
     
+    @AwaitsFix(bugUrl = "def type tracking")
     public void testOnlyCapturesAreReadOnly() {
         assertEquals(4, exec("List l = new ArrayList(); l.add(1); l.add(1); "
                            + "return l.stream().mapToInt(x -> { x += 1; return x }).sum();"));

From 4fb1c4fe5ad7017d60b2b6aae08f55f407805386 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Mon, 27 Jun 2016 17:18:26 +0200
Subject: [PATCH 1075/1311] Validate settings against dynamic updaters on the
 master (#19088)

Today all settings are only validated against their validators
that are available when settings are registered. Yet, some settings updaters
have validators that are dynamic ie. their validation depends on other variables
that are only available at runtime. We do not run those validators when settings
are updated causing index updates to fail on the data nodes instead of on the master.

Relates to #19046
---
 .../cluster/settings/SettingsUpdater.java     |  2 +-
 .../MetaDataUpdateSettingsService.java        | 27 ++++++++----
 .../settings/AbstractScopedSettings.java      | 10 ++---
 .../org/elasticsearch/index/IndexModule.java  | 11 +++++
 .../elasticsearch/index/IndexSettings.java    |  3 +-
 .../elasticsearch/indices/IndicesService.java |  3 +-
 .../snapshots/RestoreService.java             |  3 +-
 .../common/settings/ScopedSettingsTests.java  |  4 +-
 .../indices/cluster/ClusterStateChanges.java  |  3 +-
 .../indices/settings/UpdateSettingsIT.java    | 42 +++++++++++++++++++
 10 files changed, 86 insertions(+), 22 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/SettingsUpdater.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/SettingsUpdater.java
index 652401194bb..575fbcd3b98 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/SettingsUpdater.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/SettingsUpdater.java
@@ -77,7 +77,7 @@ final class SettingsUpdater {
         Settings settings = build.metaData().settings();
         // now we try to apply things and if they are invalid we fail
         // this dryRun will validate & parse settings but won't actually apply them.
-        clusterSettings.dryRun(settings);
+        clusterSettings.validateUpdate(settings);
         return build;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
index 3daf6c4dd3a..54f6ad0705a 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.cluster.metadata;
 
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsClusterStateUpdateRequest;
@@ -43,7 +44,10 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.NodeServicesProvider;
+import org.elasticsearch.indices.IndicesService;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -61,17 +65,20 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
 
     private final AllocationService allocationService;
 
-    private final IndexNameExpressionResolver indexNameExpressionResolver;
     private final IndexScopedSettings indexScopedSettings;
+    private final IndicesService indicesService;
+    private final NodeServicesProvider nodeServiceProvider;
 
     @Inject
-    public MetaDataUpdateSettingsService(Settings settings, ClusterService clusterService, AllocationService allocationService, IndexScopedSettings indexScopedSettings, IndexNameExpressionResolver indexNameExpressionResolver) {
+    public MetaDataUpdateSettingsService(Settings settings, ClusterService clusterService, AllocationService allocationService,
+                                         IndexScopedSettings indexScopedSettings, IndicesService indicesService, NodeServicesProvider nodeServicesProvider) {
         super(settings);
         this.clusterService = clusterService;
-        this.indexNameExpressionResolver = indexNameExpressionResolver;
         this.clusterService.add(this);
         this.allocationService = allocationService;
         this.indexScopedSettings = indexScopedSettings;
+        this.indicesService = indicesService;
+        this.nodeServiceProvider = nodeServicesProvider;
     }
 
     @Override
@@ -266,11 +273,15 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
                 // now, reroute in case things change that require it (like number of replicas)
                 RoutingAllocation.Result routingResult = allocationService.reroute(updatedState, "settings update");
                 updatedState = ClusterState.builder(updatedState).routingResult(routingResult).build();
-                for (Index index : openIndices) {
-                    indexScopedSettings.dryRun(updatedState.metaData().getIndexSafe(index).getSettings());
-                }
-                for (Index index : closeIndices) {
-                    indexScopedSettings.dryRun(updatedState.metaData().getIndexSafe(index).getSettings());
+                try {
+                    for (Index index : openIndices) {
+                        indicesService.verifyIndexMetadata(nodeServiceProvider, updatedState.getMetaData().getIndexSafe(index));
+                    }
+                    for (Index index : closeIndices) {
+                        indicesService.verifyIndexMetadata(nodeServiceProvider, updatedState.getMetaData().getIndexSafe(index));
+                    }
+                } catch (IOException ex) {
+                    ExceptionsHelper.convertToElastic(ex);
                 }
                 return updatedState;
             }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
index 817e109bf4d..efa4c1316ac 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
@@ -115,18 +115,18 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
     }
 
     /**
-     * Applies the given settings to all listeners and rolls back the result after application. This
+     * Validates the given settings by running it through all update listeners without applying it. This
      * method will not change any settings but will fail if any of the settings can't be applied.
      */
-    public synchronized Settings dryRun(Settings settings) {
+    public synchronized Settings validateUpdate(Settings settings) {
         final Settings current = Settings.builder().put(this.settings).put(settings).build();
         final Settings previous = Settings.builder().put(this.settings).put(this.lastSettingsApplied).build();
         List<RuntimeException> exceptions = new ArrayList<>();
         for (SettingUpdater<?> settingUpdater : settingUpdaters) {
             try {
-                if (settingUpdater.hasChanged(current, previous)) {
-                    settingUpdater.getValue(current, previous);
-                }
+                // ensure running this through the updater / dynamic validator
+                // don't check if the value has changed we wanna test this anyways
+                settingUpdater.getValue(current, previous);
             } catch (RuntimeException ex) {
                 exceptions.add(ex);
                 logger.debug("failed to prepareCommit settings for [{}]", ex, settingUpdater);
diff --git a/core/src/main/java/org/elasticsearch/index/IndexModule.java b/core/src/main/java/org/elasticsearch/index/IndexModule.java
index d233faf4f19..f6227ca3276 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexModule.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexModule.java
@@ -126,6 +126,17 @@ public final class IndexModule {
         indexSettings.getScopedSettings().addSettingsUpdateConsumer(setting, consumer);
     }
 
+    /**
+     * Adds a Setting, it's consumer and validator for this index.
+     */
+    public <T> void addSettingsUpdateConsumer(Setting<T> setting, Consumer<T> consumer, Consumer<T> validator) {
+        ensureNotFrozen();
+        if (setting == null) {
+            throw new IllegalArgumentException("setting must not be null");
+        }
+        indexSettings.getScopedSettings().addSettingsUpdateConsumer(setting, consumer, validator);
+    }
+
     /**
      * Returns the index {@link Settings} for this index
      */
diff --git a/core/src/main/java/org/elasticsearch/index/IndexSettings.java b/core/src/main/java/org/elasticsearch/index/IndexSettings.java
index 2c20697d757..df348a5d6a1 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexSettings.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexSettings.java
@@ -275,6 +275,7 @@ public final class IndexSettings {
         scopedSettings.addSettingsUpdateConsumer(INDEX_REFRESH_INTERVAL_SETTING, this::setRefreshInterval);
         scopedSettings.addSettingsUpdateConsumer(MAX_REFRESH_LISTENERS_PER_SHARD, this::setMaxRefreshListeners);
         scopedSettings.addSettingsUpdateConsumer(MAX_SLICES_PER_SCROLL, this::setMaxSlicesPerScroll);
+
     }
 
     private void setTranslogFlushThresholdSize(ByteSizeValue byteSizeValue) {
@@ -545,5 +546,5 @@ public final class IndexSettings {
         this.maxSlicesPerScroll = value;
     }
 
-    IndexScopedSettings getScopedSettings() { return scopedSettings;}
+    public IndexScopedSettings getScopedSettings() { return scopedSettings;}
 }
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index edd5e5d8e4b..89cfad98c4f 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -425,12 +425,13 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService>
             // this will also fail if some plugin fails etc. which is nice since we can verify that early
             final IndexService service = createIndexService("metadata verification", nodeServicesProvider,
                 metaData, indicesQueryCache, indicesFieldDataCache, Collections.emptyList());
+            closeables.add(() -> service.close("metadata verification", false));
             for (ObjectCursor<MappingMetaData> typeMapping : metaData.getMappings().values()) {
                 // don't apply the default mapping, it has been applied when the mapping was created
                 service.mapperService().merge(typeMapping.value.type(), typeMapping.value.source(),
                     MapperService.MergeReason.MAPPING_RECOVERY, true);
             }
-            closeables.add(() -> service.close("metadata verification", false));
+            service.getIndexSettings().getScopedSettings().validateUpdate(metaData.getSettings());
         } finally {
             IOUtils.close(closeables);
         }
diff --git a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
index 44ca6fb972e..74434413ae9 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
@@ -39,7 +39,6 @@ import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
 import org.elasticsearch.cluster.metadata.MetaDataIndexUpgradeService;
 import org.elasticsearch.cluster.metadata.RepositoriesMetaData;
-import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.RestoreSource;
@@ -436,7 +435,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
                     if (request.includeGlobalState()) {
                         if (metaData.persistentSettings() != null) {
                             Settings settings = metaData.persistentSettings();
-                            clusterSettings.dryRun(settings);
+                            clusterSettings.validateUpdate(settings);
                             mdBuilder.persistentSettings(settings);
                         }
                         if (metaData.templates() != null) {
diff --git a/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java b/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
index 664f8cb96ab..dee20d6b32e 100644
--- a/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java
@@ -98,7 +98,7 @@ public class ScopedSettingsTests extends ESTestCase {
         assertEquals(0, aC.get());
         assertEquals(0, bC.get());
         try {
-            service.dryRun(Settings.builder().put("foo.bar", 2).put("foo.bar.baz", -15).build());
+            service.validateUpdate(Settings.builder().put("foo.bar", 2).put("foo.bar.baz", -15).build());
             fail("invalid value");
         } catch (IllegalArgumentException ex) {
             assertEquals("illegal value can't update [foo.bar.baz] from [1] to [-15]", ex.getMessage());
@@ -108,7 +108,7 @@ public class ScopedSettingsTests extends ESTestCase {
         assertEquals(0, consumer2.get());
         assertEquals(0, aC.get());
         assertEquals(0, bC.get());
-        service.dryRun(Settings.builder().put("foo.bar", 2).put("foo.bar.baz", 15).build());
+        service.validateUpdate(Settings.builder().put("foo.bar", 2).put("foo.bar.baz", 15).build());
         assertEquals(0, consumer.get());
         assertEquals(0, consumer2.get());
         assertEquals(0, aC.get());
diff --git a/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
index fc5b68f87d1..09441f70110 100644
--- a/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
+++ b/core/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.indices.cluster;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest;
 import org.elasticsearch.action.admin.cluster.reroute.TransportClusterRerouteAction;
@@ -156,7 +155,7 @@ public class ClusterStateChanges {
             metaDataIndexUpgradeService, nodeServicesProvider, indicesService);
         MetaDataDeleteIndexService deleteIndexService = new MetaDataDeleteIndexService(settings, clusterService, allocationService);
         MetaDataUpdateSettingsService metaDataUpdateSettingsService = new MetaDataUpdateSettingsService(settings, clusterService,
-            allocationService, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, new IndexNameExpressionResolver(settings));
+            allocationService, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, indicesService, nodeServicesProvider);
         MetaDataCreateIndexService createIndexService = new MetaDataCreateIndexService(settings, clusterService, indicesService,
             allocationService, new AliasValidator(settings), Collections.emptySet(), environment,
             nodeServicesProvider, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS);
diff --git a/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java b/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java
index e7e3cb32226..2a67742fc68 100644
--- a/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java
@@ -29,7 +29,9 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.MergePolicyConfig;
@@ -37,9 +39,13 @@ import org.elasticsearch.index.MergeSchedulerConfig;
 import org.elasticsearch.index.store.IndexStore;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_METADATA;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ;
@@ -53,6 +59,42 @@ import static org.hamcrest.Matchers.nullValue;
 
 public class UpdateSettingsIT extends ESIntegTestCase {
 
+
+    public void testInvalidDynamicUpdate() {
+        createIndex("test");
+        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () ->
+            client().admin().indices().prepareUpdateSettings("test")
+                .setSettings(Settings.builder()
+                    .put("index.dummy", "boom")
+                )
+                .execute().actionGet());
+        assertEquals(exception.getCause().getMessage(), "this setting goes boom");
+        IndexMetaData indexMetaData = client().admin().cluster().prepareState().execute().actionGet().getState().metaData().index("test");
+        assertNotEquals(indexMetaData.getSettings().get("index.dummy"), "invalid dynamic value");
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return pluginList(DummySettingPlugin.class);
+    }
+
+    public static class DummySettingPlugin extends Plugin {
+        public static final Setting<String> DUMMY_SETTING = Setting.simpleString("index.dummy",
+            Setting.Property.IndexScope, Setting.Property.Dynamic);
+        @Override
+        public void onIndexModule(IndexModule indexModule) {
+            indexModule.addSettingsUpdateConsumer(DUMMY_SETTING, (s) -> {}, (s) -> {
+                if (s.equals("boom"))
+                    throw new IllegalArgumentException("this setting goes boom");
+            });
+        }
+
+        @Override
+        public List<Setting<?>> getSettings() {
+            return Collections.singletonList(DUMMY_SETTING);
+        }
+    }
+
     public void testResetDefault() {
         createIndex("test");
 

From eb1e231a635a124f445364e4baabaf1c297399ff Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Mon, 27 Jun 2016 17:20:32 +0200
Subject: [PATCH 1076/1311] Revert "Rename `fields` to `stored_fields` and add
 `docvalue_fields`"

This reverts commit 2f46f53dc8feb78412e6d648751ffe97b1e35119.
---
 .../action/search/SearchRequestBuilder.java   |  41 +---
 .../index/query/InnerHitBuilder.java          | 228 ++++++------------
 .../rest/action/search/RestSearchAction.java  |  31 +--
 .../elasticsearch/search/SearchService.java   |   8 +-
 .../tophits/TopHitsAggregationBuilder.java    |  12 +-
 .../search/builder/SearchSourceBuilder.java   | 162 ++++++-------
 .../index/IndexWithShadowReplicasIT.java      |   2 +-
 .../TokenCountFieldMapperIntegrationIT.java   |   4 +-
 .../mapper/geo/GeoPointFieldMapperTests.java  |   8 +-
 .../index/query/InnerHitBuilderTests.java     |  14 +-
 .../index/store/ExceptionRetryIT.java         |   2 +-
 .../template/SimpleIndexTemplateIT.java       |   4 +-
 .../elasticsearch/recovery/RelocationIT.java  |   2 +-
 .../metrics/AbstractGeoTestCase.java          |   2 +-
 .../builder/SearchSourceBuilderTests.java     |   4 +-
 .../search/child/ChildQuerySearchIT.java      |   8 +-
 .../elasticsearch/search/geo/GeoFilterIT.java |   4 +-
 .../search/innerhits/InnerHitsIT.java         |   4 +-
 .../search/source/SourceFetchingIT.java       |   4 +-
 .../update/TimestampTTLBWIT.java              |   2 +-
 docs/plugins/mapper-attachments.asciidoc      |   4 +-
 .../metrics/tophits-aggregation.asciidoc      |   2 +-
 docs/reference/mapping/params/store.asciidoc  |   2 +-
 .../migration/migrate_5_0/search.asciidoc     |  10 +-
 .../modules/scripting/painless.asciidoc       |   2 +-
 docs/reference/search/request-body.asciidoc   |   4 +-
 .../search/request/docvalue-fields.asciidoc   |  23 --
 .../search/request/fielddata-fields.asciidoc  |  23 ++
 ...stored-fields.asciidoc => fields.asciidoc} |   6 +-
 .../search/request/highlighting.asciidoc      |   2 +-
 .../search/request/inner-hits.asciidoc        |   2 +-
 docs/reference/search/uri-request.asciidoc    |   2 +-
 .../messy/tests/GeoDistanceTests.java         |  16 +-
 .../messy/tests/SearchFieldsTests.java        |  66 ++---
 .../reindex/AbstractBulkByScrollRequest.java  |   4 +-
 .../test/delete_by_query/20_validation.yaml   |   4 +-
 .../test/reindex/20_validation.yaml           |   4 +-
 .../test/update_by_query/20_validation.yaml   |   4 +-
 .../test/mapper_attachments/30_mapping.yaml   |   2 +-
 .../test/mapper_attachments/40_highlight.yaml |   2 +-
 .../50_files_supported.yaml                   |   4 +-
 .../size/SizeFieldMapperUpgradeTests.java     |   4 +-
 .../resources/rest-api-spec/api/search.json   |  10 +-
 .../test/search/10_source_filtering.yaml      |   9 +-
 .../rest-api-spec/test/search/issue4895.yaml  |   2 +-
 45 files changed, 312 insertions(+), 447 deletions(-)
 delete mode 100644 docs/reference/search/request/docvalue-fields.asciidoc
 create mode 100644 docs/reference/search/request/fielddata-fields.asciidoc
 rename docs/reference/search/request/{stored-fields.asciidoc => fields.asciidoc} (91%)

diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
index 5c08acb99ea..2297c98b636 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
@@ -252,8 +252,8 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
     /**
      * Sets no fields to be loaded, resulting in only id and type to be returned per field.
      */
-    public SearchRequestBuilder setNoStoredFields() {
-        sourceBuilder().noStoredFields();
+    public SearchRequestBuilder setNoFields() {
+        sourceBuilder().noFields();
         return this;
     }
 
@@ -289,23 +289,13 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
         return this;
     }
 
-    /**
-     * Adds a docvalue based field to load and return. The field does not have to be stored,
-     * but its recommended to use non analyzed or numeric fields.
-     *
-     * @param name The field to get from the docvalue
-     */
-    public SearchRequestBuilder addDocValueField(String name) {
-        sourceBuilder().docValueField(name);
-        return this;
-    }
 
     /**
-     * Adds a stored field to load and return (note, it must be stored) as part of the search request.
+     * Adds a field to load and return (note, it must be stored) as part of the search request.
      * If none are specified, the source of the document will be return.
      */
-    public SearchRequestBuilder addStoredField(String field) {
-        sourceBuilder().storedField(field);
+    public SearchRequestBuilder addField(String field) {
+        sourceBuilder().field(field);
         return this;
     }
 
@@ -314,15 +304,12 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
      * but its recommended to use non analyzed or numeric fields.
      *
      * @param name The field to get from the field data cache
-     * @deprecated Use {@link SearchRequestBuilder#addDocValueField(String)} instead.
      */
-    @Deprecated
     public SearchRequestBuilder addFieldDataField(String name) {
-        sourceBuilder().docValueField(name);
+        sourceBuilder().fieldDataField(name);
         return this;
     }
 
-
     /**
      * Adds a script based field to load and return. The field does not have to be stored,
      * but its recommended to use non analyzed or numeric fields.
@@ -379,24 +366,12 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
         return this;
     }
 
-    /**
-     * Sets the stored fields to load and return as part of the search request. If none
-     * are specified, the source of the document will be returned.
-     *
-     * @deprecated Use {@link SearchRequestBuilder#storedFields(String...)} instead.
-     */
-    @Deprecated
-    public SearchRequestBuilder fields(String... fields) {
-        sourceBuilder().storedFields(Arrays.asList(fields));
-        return this;
-    }
-
     /**
      * Sets the fields to load and return as part of the search request. If none
      * are specified, the source of the document will be returned.
      */
-    public SearchRequestBuilder storedFields(String... fields) {
-        sourceBuilder().storedFields(Arrays.asList(fields));
+    public SearchRequestBuilder fields(String... fields) {
+        sourceBuilder().fields(Arrays.asList(fields));
         return this;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
index 43e2f12055c..a017159b451 100644
--- a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
@@ -71,14 +71,8 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         PARSER.declareBoolean(InnerHitBuilder::setExplain, SearchSourceBuilder.EXPLAIN_FIELD);
         PARSER.declareBoolean(InnerHitBuilder::setVersion, SearchSourceBuilder.VERSION_FIELD);
         PARSER.declareBoolean(InnerHitBuilder::setTrackScores, SearchSourceBuilder.TRACK_SCORES_FIELD);
-        PARSER.declareStringArray(InnerHitBuilder::setStoredFieldNames, SearchSourceBuilder.STORED_FIELDS_FIELD);
-        PARSER.declareField((p, i, c) -> {
-            throw new ParsingException(p.getTokenLocation(), "The field [" +
-                SearchSourceBuilder.FIELDS_FIELD + "] is not longer supported, please use [" +
-                SearchSourceBuilder.STORED_FIELDS_FIELD + "] to retrieve stored fields or _source filtering " +
-                "if the field is not stored");
-        }, SearchSourceBuilder.FIELDS_FIELD, ObjectParser.ValueType.STRING_ARRAY);
-        PARSER.declareStringArray(InnerHitBuilder::setDocValueFields, SearchSourceBuilder.DOCVALUE_FIELDS_FIELD);
+        PARSER.declareStringArray(InnerHitBuilder::setFieldNames, SearchSourceBuilder.FIELDS_FIELD);
+        PARSER.declareStringArray(InnerHitBuilder::setFieldDataFields, SearchSourceBuilder.FIELDDATA_FIELDS_FIELD);
         PARSER.declareField((p, i, c) -> {
             try {
                 Set<ScriptField> scriptFields = new HashSet<>();
@@ -137,10 +131,10 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
     private boolean version;
     private boolean trackScores;
 
-    private List<String> storedFieldNames;
+    private List<String> fieldNames;
     private QueryBuilder query = DEFAULT_INNER_HIT_QUERY;
     private List<SortBuilder<?>> sorts;
-    private List<String> docValueFields;
+    private List<String> fieldDataFields;
     private Set<ScriptField> scriptFields;
     private HighlightBuilder highlightBuilder;
     private FetchSourceContext fetchSourceContext;
@@ -149,6 +143,46 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
     public InnerHitBuilder() {
     }
 
+    /**
+     * Read from a stream.
+     */
+    public InnerHitBuilder(StreamInput in) throws IOException {
+        name = in.readOptionalString();
+        nestedPath = in.readOptionalString();
+        parentChildType = in.readOptionalString();
+        from = in.readVInt();
+        size = in.readVInt();
+        explain = in.readBoolean();
+        version = in.readBoolean();
+        trackScores = in.readBoolean();
+        fieldNames = (List<String>) in.readGenericValue();
+        fieldDataFields = (List<String>) in.readGenericValue();
+        if (in.readBoolean()) {
+            int size = in.readVInt();
+            scriptFields = new HashSet<>(size);
+            for (int i = 0; i < size; i++) {
+                scriptFields.add(new ScriptField(in));
+            }
+        }
+        fetchSourceContext = in.readOptionalStreamable(FetchSourceContext::new);
+        if (in.readBoolean()) {
+            int size = in.readVInt();
+            sorts = new ArrayList<>(size);
+            for (int i = 0; i < size; i++) {
+                sorts.add(in.readNamedWriteable(SortBuilder.class));
+            }
+        }
+        highlightBuilder = in.readOptionalWriteable(HighlightBuilder::new);
+        query = in.readNamedWriteable(QueryBuilder.class);
+        if (in.readBoolean()) {
+            int size = in.readVInt();
+            childInnerHits = new HashMap<>(size);
+            for (int i = 0; i < size; i++) {
+                childInnerHits.put(in.readString(), new InnerHitBuilder(in));
+            }
+        }
+    }
+
     private InnerHitBuilder(InnerHitBuilder other) {
         name = other.name;
         from = other.from;
@@ -156,11 +190,11 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         explain = other.explain;
         version = other.version;
         trackScores = other.trackScores;
-        if (other.storedFieldNames != null) {
-            storedFieldNames = new ArrayList<>(other.storedFieldNames);
+        if (other.fieldNames != null) {
+            fieldNames = new ArrayList<>(other.fieldNames);
         }
-        if (other.docValueFields != null) {
-            docValueFields = new ArrayList<>(other.docValueFields);
+        if (other.fieldDataFields != null) {
+            fieldDataFields = new ArrayList<>(other.fieldDataFields);
         }
         if (other.scriptFields != null) {
             scriptFields = new HashSet<>(other.scriptFields);
@@ -198,46 +232,6 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         }
     }
 
-    /**
-     * Read from a stream.
-     */
-    public InnerHitBuilder(StreamInput in) throws IOException {
-        name = in.readOptionalString();
-        nestedPath = in.readOptionalString();
-        parentChildType = in.readOptionalString();
-        from = in.readVInt();
-        size = in.readVInt();
-        explain = in.readBoolean();
-        version = in.readBoolean();
-        trackScores = in.readBoolean();
-        storedFieldNames = (List<String>) in.readGenericValue();
-        docValueFields = (List<String>) in.readGenericValue();
-        if (in.readBoolean()) {
-            int size = in.readVInt();
-            scriptFields = new HashSet<>(size);
-            for (int i = 0; i < size; i++) {
-                scriptFields.add(new ScriptField(in));
-            }
-        }
-        fetchSourceContext = in.readOptionalStreamable(FetchSourceContext::new);
-        if (in.readBoolean()) {
-            int size = in.readVInt();
-            sorts = new ArrayList<>(size);
-            for (int i = 0; i < size; i++) {
-                sorts.add(in.readNamedWriteable(SortBuilder.class));
-            }
-        }
-        highlightBuilder = in.readOptionalWriteable(HighlightBuilder::new);
-        query = in.readNamedWriteable(QueryBuilder.class);
-        if (in.readBoolean()) {
-            int size = in.readVInt();
-            childInnerHits = new HashMap<>(size);
-            for (int i = 0; i < size; i++) {
-                childInnerHits.put(in.readString(), new InnerHitBuilder(in));
-            }
-        }
-    }
-
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(name);
@@ -248,8 +242,8 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         out.writeBoolean(explain);
         out.writeBoolean(version);
         out.writeBoolean(trackScores);
-        out.writeGenericValue(storedFieldNames);
-        out.writeGenericValue(docValueFields);
+        out.writeGenericValue(fieldNames);
+        out.writeGenericValue(fieldDataFields);
         boolean hasScriptFields = scriptFields != null;
         out.writeBoolean(hasScriptFields);
         if (hasScriptFields) {
@@ -340,103 +334,29 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         return this;
     }
 
-    /**
-     * Gets the stored fields to load and return.
-     *
-     * @deprecated Use {@link InnerHitBuilder#getStoredFieldNames()} instead.
-     */
-    @Deprecated
     public List<String> getFieldNames() {
-        return storedFieldNames;
+        return fieldNames;
     }
 
-    /**
-     * Sets the stored fields to load and return. If none
-     * are specified, the source of the document will be returned.
-     *
-     * @deprecated Use {@link InnerHitBuilder#setStoredFieldNames(List)} instead.
-     */
-    @Deprecated
     public InnerHitBuilder setFieldNames(List<String> fieldNames) {
-        this.storedFieldNames = fieldNames;
+        this.fieldNames = fieldNames;
         return this;
     }
 
-
-    /**
-     * Gets the stored fields to load and return.
-     */
-    public List<String> getStoredFieldNames() {
-        return storedFieldNames;
-    }
-
-    /**
-     * Sets the stored fields to load and return. If none
-     * are specified, the source of the document will be returned.
-     */
-    public InnerHitBuilder setStoredFieldNames(List<String> fieldNames) {
-        this.storedFieldNames = fieldNames;
-        return this;
-    }
-
-    /**
-     * Gets the docvalue fields.
-     *
-     * @deprecated Use {@link InnerHitBuilder#getDocValueFields()} instead.
-     */
-    @Deprecated
     public List<String> getFieldDataFields() {
-        return docValueFields;
+        return fieldDataFields;
     }
 
-    /**
-     * Sets the stored fields to load from the docvalue and return.
-     *
-     * @deprecated Use {@link InnerHitBuilder#setDocValueFields(List)} instead.
-     */
-    @Deprecated
     public InnerHitBuilder setFieldDataFields(List<String> fieldDataFields) {
-        this.docValueFields = fieldDataFields;
+        this.fieldDataFields = fieldDataFields;
         return this;
     }
 
-    /**
-     * Adds a field to load from the docvalue and return.
-     *
-     * @deprecated Use {@link InnerHitBuilder#addDocValueField(String)} instead.
-     */
-    @Deprecated
     public InnerHitBuilder addFieldDataField(String field) {
-        if (docValueFields == null) {
-            docValueFields = new ArrayList<>();
+        if (fieldDataFields == null) {
+            fieldDataFields = new ArrayList<>();
         }
-        docValueFields.add(field);
-        return this;
-    }
-
-    /**
-     * Gets the docvalue fields.
-     */
-    public List<String> getDocValueFields() {
-        return docValueFields;
-    }
-
-    /**
-     * Sets the stored fields to load from the docvalue and return.
-     */
-    public InnerHitBuilder setDocValueFields(List<String> docValueFields) {
-        this.docValueFields = docValueFields;
-        return this;
-    }
-
-    /**
-     * Adds a field to load from the docvalue and return.
-     */
-    public InnerHitBuilder addDocValueField(String field) {
-        if (docValueFields == null) {
-            docValueFields = new ArrayList<>();
-        }
-        docValueFields.add(field);
+        fieldDataFields.add(field);
         return this;
     }
 
@@ -564,19 +484,19 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         innerHitsContext.explain(explain);
         innerHitsContext.version(version);
         innerHitsContext.trackScores(trackScores);
-        if (storedFieldNames != null) {
-            if (storedFieldNames.isEmpty()) {
+        if (fieldNames != null) {
+            if (fieldNames.isEmpty()) {
                 innerHitsContext.emptyFieldNames();
             } else {
-                for (String fieldName : storedFieldNames) {
+                for (String fieldName : fieldNames) {
                     innerHitsContext.fieldNames().add(fieldName);
                 }
             }
         }
-        if (docValueFields != null) {
+        if (fieldDataFields != null) {
             FieldDataFieldsContext fieldDataFieldsContext = innerHitsContext
                     .getFetchSubPhaseContext(FieldDataFieldsFetchSubPhase.CONTEXT_FACTORY);
-            for (String field : docValueFields) {
+            for (String field : fieldDataFields) {
                 fieldDataFieldsContext.add(new FieldDataFieldsContext.FieldDataField(field));
             }
             fieldDataFieldsContext.setHitExecutionNeeded(true);
@@ -633,20 +553,20 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         if (fetchSourceContext != null) {
             builder.field(SearchSourceBuilder._SOURCE_FIELD.getPreferredName(), fetchSourceContext, params);
         }
-        if (storedFieldNames != null) {
-            if (storedFieldNames.size() == 1) {
-                builder.field(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), storedFieldNames.get(0));
+        if (fieldNames != null) {
+            if (fieldNames.size() == 1) {
+                builder.field(SearchSourceBuilder.FIELDS_FIELD.getPreferredName(), fieldNames.get(0));
             } else {
-                builder.startArray(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName());
-                for (String fieldName : storedFieldNames) {
+                builder.startArray(SearchSourceBuilder.FIELDS_FIELD.getPreferredName());
+                for (String fieldName : fieldNames) {
                     builder.value(fieldName);
                 }
                 builder.endArray();
             }
         }
-        if (docValueFields != null) {
-            builder.startArray(SearchSourceBuilder.DOCVALUE_FIELDS_FIELD.getPreferredName());
-            for (String fieldDataField : docValueFields) {
+        if (fieldDataFields != null) {
+            builder.startArray(SearchSourceBuilder.FIELDDATA_FIELDS_FIELD.getPreferredName());
+            for (String fieldDataField : fieldDataFields) {
                 builder.value(fieldDataField);
             }
             builder.endArray();
@@ -693,8 +613,8 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
                 Objects.equals(explain, that.explain) &&
                 Objects.equals(version, that.version) &&
                 Objects.equals(trackScores, that.trackScores) &&
-                Objects.equals(storedFieldNames, that.storedFieldNames) &&
-                Objects.equals(docValueFields, that.docValueFields) &&
+                Objects.equals(fieldNames, that.fieldNames) &&
+                Objects.equals(fieldDataFields, that.fieldDataFields) &&
                 Objects.equals(scriptFields, that.scriptFields) &&
                 Objects.equals(fetchSourceContext, that.fetchSourceContext) &&
                 Objects.equals(sorts, that.sorts) &&
@@ -705,8 +625,8 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
 
     @Override
     public int hashCode() {
-        return Objects.hash(name, nestedPath, parentChildType, from, size, explain, version, trackScores, storedFieldNames,
-            docValueFields, scriptFields, fetchSourceContext, sorts, highlightBuilder, query, childInnerHits);
+        return Objects.hash(name, nestedPath, parentChildType, from, size, explain, version, trackScores, fieldNames,
+                fieldDataFields, scriptFields, fetchSourceContext, sorts, highlightBuilder, query, childInnerHits);
     }
 
     public static InnerHitBuilder fromXContent(QueryParseContext context) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
index f46c41dcd17..5f5fe84d573 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
@@ -176,35 +175,27 @@ public class RestSearchAction extends BaseRestHandler {
             }
         }
 
-        if (request.param("fields") != null) {
-            throw new IllegalArgumentException("The parameter [" +
-                SearchSourceBuilder.FIELDS_FIELD + "] is not longer supported, please use [" +
-                SearchSourceBuilder.STORED_FIELDS_FIELD + "] to retrieve stored fields or _source filtering " +
-                "if the field is not stored");
-        }
-
-        String sField = request.param("stored_fields");
+        String sField = request.param("fields");
         if (sField != null) {
             if (!Strings.hasText(sField)) {
-                searchSourceBuilder.noStoredFields();
+                searchSourceBuilder.noFields();
             } else {
                 String[] sFields = Strings.splitStringByCommaToArray(sField);
                 if (sFields != null) {
                     for (String field : sFields) {
-                        searchSourceBuilder.storedField(field);
+                        searchSourceBuilder.field(field);
                     }
                 }
             }
         }
-        String sDocValueFields = request.param("docvalue_fields");
-        if (sDocValueFields == null) {
-            sDocValueFields = request.param("fielddata_fields");
-        }
-        if (sDocValueFields != null) {
-            if (Strings.hasText(sDocValueFields)) {
-                String[] sFields = Strings.splitStringByCommaToArray(sDocValueFields);
-                for (String field : sFields) {
-                    searchSourceBuilder.docValueField(field);
+        String sFieldDataFields = request.param("fielddata_fields");
+        if (sFieldDataFields != null) {
+            if (Strings.hasText(sFieldDataFields)) {
+                String[] sFields = Strings.splitStringByCommaToArray(sFieldDataFields);
+                if (sFields != null) {
+                    for (String field : sFields) {
+                        searchSourceBuilder.fieldDataField(field);
+                    }
                 }
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java
index 6ceb021d9f6..50b59631e95 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchService.java
@@ -713,8 +713,8 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
                 throw new SearchContextException(context, "failed to create RescoreSearchContext", e);
             }
         }
-        if (source.storedFields() != null) {
-            context.fieldNames().addAll(source.storedFields());
+        if (source.fields() != null) {
+            context.fieldNames().addAll(source.fields());
         }
         if (source.explain() != null) {
             context.explain(source.explain());
@@ -722,9 +722,9 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
         if (source.fetchSource() != null) {
             context.fetchSourceContext(source.fetchSource());
         }
-        if (source.docValueFields() != null) {
+        if (source.fieldDataFields() != null) {
             FieldDataFieldsContext fieldDataFieldsContext = context.getFetchSubPhaseContext(FieldDataFieldsFetchSubPhase.CONTEXT_FACTORY);
-            for (String field : source.docValueFields()) {
+            for (String field : source.fieldDataFields()) {
                 fieldDataFieldsContext.add(new FieldDataField(field));
             }
             fieldDataFieldsContext.setHitExecutionNeeded(true);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java
index c9ed41c6ddc..00d783c12e3 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java
@@ -567,9 +567,9 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
         }
         if (fieldNames != null) {
             if (fieldNames.size() == 1) {
-                builder.field(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), fieldNames.get(0));
+                builder.field(SearchSourceBuilder.FIELDS_FIELD.getPreferredName(), fieldNames.get(0));
             } else {
-                builder.startArray(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName());
+                builder.startArray(SearchSourceBuilder.FIELDS_FIELD.getPreferredName());
                 for (String fieldName : fieldNames) {
                     builder.value(fieldName);
                 }
@@ -577,7 +577,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
             }
         }
         if (fieldDataFields != null) {
-            builder.startArray(SearchSourceBuilder.DOCVALUE_FIELDS_FIELD.getPreferredName());
+            builder.startArray(SearchSourceBuilder.FIELDDATA_FIELDS_FIELD.getPreferredName());
             for (String fieldDataField : fieldDataFields) {
                 builder.value(fieldDataField);
             }
@@ -628,7 +628,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
                     factory.trackScores(parser.booleanValue());
                 } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder._SOURCE_FIELD)) {
                     factory.fetchSource(FetchSourceContext.parse(context));
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.STORED_FIELDS_FIELD)) {
+                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.FIELDS_FIELD)) {
                     List<String> fieldNames = new ArrayList<>();
                     fieldNames.add(parser.text());
                     factory.fields(fieldNames);
@@ -694,7 +694,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
 
-                if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.STORED_FIELDS_FIELD)) {
+                if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.FIELDS_FIELD)) {
                     List<String> fieldNames = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token == XContentParser.Token.VALUE_STRING) {
@@ -705,7 +705,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
                         }
                     }
                     factory.fields(fieldNames);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.DOCVALUE_FIELDS_FIELD)) {
+                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.FIELDDATA_FIELDS_FIELD)) {
                     List<String> fieldDataFields = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token == XContentParser.Token.VALUE_STRING) {
diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index 648fcaa859a..24278bdf127 100644
--- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -83,8 +83,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
     public static final ParseField EXPLAIN_FIELD = new ParseField("explain");
     public static final ParseField _SOURCE_FIELD = new ParseField("_source");
     public static final ParseField FIELDS_FIELD = new ParseField("fields");
-    public static final ParseField STORED_FIELDS_FIELD = new ParseField("stored_fields");
-    public static final ParseField DOCVALUE_FIELDS_FIELD = new ParseField("docvalue_fields", "fielddata_fields");
+    public static final ParseField FIELDDATA_FIELDS_FIELD = new ParseField("fielddata_fields");
     public static final ParseField SCRIPT_FIELDS_FIELD = new ParseField("script_fields");
     public static final ParseField SCRIPT_FIELD = new ParseField("script");
     public static final ParseField IGNORE_FAILURE_FIELD = new ParseField("ignore_failure");
@@ -147,8 +146,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
     private long timeoutInMillis = -1;
     private int terminateAfter = SearchContext.DEFAULT_TERMINATE_AFTER;
 
-    private List<String> storedFieldNames;
-    private List<String> docValueFields;
+    private List<String> fieldNames;
+    private List<String> fieldDataFields;
     private List<ScriptField> scriptFields;
     private FetchSourceContext fetchSourceContext;
 
@@ -182,8 +181,22 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         aggregations = in.readOptionalWriteable(AggregatorFactories.Builder::new);
         explain = in.readOptionalBoolean();
         fetchSourceContext = in.readOptionalStreamable(FetchSourceContext::new);
-        docValueFields = (List<String>) in.readGenericValue();
-        storedFieldNames = (List<String>) in.readGenericValue();
+        boolean hasFieldDataFields = in.readBoolean();
+        if (hasFieldDataFields) {
+            int size = in.readVInt();
+            fieldDataFields = new ArrayList<>(size);
+            for (int i = 0; i < size; i++) {
+                fieldDataFields.add(in.readString());
+            }
+        }
+        boolean hasFieldNames = in.readBoolean();
+        if (hasFieldNames) {
+            int size = in.readVInt();
+            fieldNames = new ArrayList<>(size);
+            for (int i = 0; i < size; i++) {
+                fieldNames.add(in.readString());
+            }
+        }
         from = in.readVInt();
         highlightBuilder = in.readOptionalWriteable(HighlightBuilder::new);
         boolean hasIndexBoost = in.readBoolean();
@@ -242,8 +255,22 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         out.writeOptionalWriteable(aggregations);
         out.writeOptionalBoolean(explain);
         out.writeOptionalStreamable(fetchSourceContext);
-        out.writeGenericValue(docValueFields);
-        out.writeGenericValue(storedFieldNames);
+        boolean hasFieldDataFields = fieldDataFields != null;
+        out.writeBoolean(hasFieldDataFields);
+        if (hasFieldDataFields) {
+            out.writeVInt(fieldDataFields.size());
+            for (String field : fieldDataFields) {
+                out.writeString(field);
+            }
+        }
+        boolean hasFieldNames = fieldNames != null;
+        out.writeBoolean(hasFieldNames);
+        if (hasFieldNames) {
+            out.writeVInt(fieldNames.size());
+            for (String field : fieldNames) {
+                out.writeString(field);
+            }
+        }
         out.writeVInt(from);
         out.writeOptionalWriteable(highlightBuilder);
         boolean hasIndexBoost = indexBoost != null;
@@ -705,87 +732,60 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
     }
 
     /**
-     * Adds a stored field to load and return as part of the
+     * Adds a field to load and return (note, it must be stored) as part of the
      * search request. If none are specified, the source of the document will be
      * return.
      */
-    public SearchSourceBuilder storedField(String name) {
-        if (storedFieldNames == null) {
-            storedFieldNames = new ArrayList<>();
+    public SearchSourceBuilder field(String name) {
+        if (fieldNames == null) {
+            fieldNames = new ArrayList<>();
         }
-        storedFieldNames.add(name);
+        fieldNames.add(name);
         return this;
     }
 
     /**
-     * Sets the stored fields to load and return as part of the search request. If none
+     * Sets the fields to load and return as part of the search request. If none
      * are specified, the source of the document will be returned.
      */
-    public SearchSourceBuilder storedFields(List<String> fields) {
-        this.storedFieldNames = fields;
+    public SearchSourceBuilder fields(List<String> fields) {
+        this.fieldNames = fields;
         return this;
     }
 
     /**
-     * Sets no stored fields to be loaded, resulting in only id and type to be returned
+     * Sets no fields to be loaded, resulting in only id and type to be returned
      * per field.
      */
-    public SearchSourceBuilder noStoredFields() {
-        this.storedFieldNames = Collections.emptyList();
+    public SearchSourceBuilder noFields() {
+        this.fieldNames = Collections.emptyList();
         return this;
     }
 
     /**
-     * Gets the stored fields to load and return as part of the search request.
+     * Gets the fields to load and return as part of the search request.
      */
-    public List<String> storedFields() {
-        return storedFieldNames;
+    public List<String> fields() {
+        return fieldNames;
     }
 
-
     /**
-     * Adds a field to load from the docvalue and return as part of the
+     * Adds a field to load from the field data cache and return as part of the
      * search request.
-     *
-     * @deprecated Use {@link SearchSourceBuilder#docValueField(String)} instead.
      */
-    @Deprecated
     public SearchSourceBuilder fieldDataField(String name) {
-        if (docValueFields == null) {
-            docValueFields = new ArrayList<>();
+        if (fieldDataFields == null) {
+            fieldDataFields = new ArrayList<>();
         }
-        docValueFields.add(name);
+        fieldDataFields.add(name);
         return this;
     }
 
     /**
-     * Gets the docvalue fields.
-     *
-     * @deprecated Use {@link SearchSourceBuilder#docValueFields()} instead.
+     * Gets the field-data fields.
      */
-    @Deprecated
     public List<String> fieldDataFields() {
-        return docValueFields;
-    }
-
-
-    /**
-     * Gets the docvalue fields.
-     */
-    public List<String> docValueFields() {
-        return docValueFields;
-    }
-
-    /**
-     * Adds a field to load from the docvalue and return as part of the
-     * search request.
-     */
-    public SearchSourceBuilder docValueField(String name) {
-        if (docValueFields == null) {
-            docValueFields = new ArrayList<>();
-        }
-        docValueFields.add(name);
-        return this;
+        return fieldDataFields;
     }
 
     /**
@@ -910,8 +910,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
             rewrittenBuilder.explain = explain;
             rewrittenBuilder.ext = ext;
             rewrittenBuilder.fetchSourceContext = fetchSourceContext;
-            rewrittenBuilder.docValueFields = docValueFields;
-            rewrittenBuilder.storedFieldNames = storedFieldNames;
+            rewrittenBuilder.fieldDataFields = fieldDataFields;
+            rewrittenBuilder.fieldNames = fieldNames;
             rewrittenBuilder.from = from;
             rewrittenBuilder.highlightBuilder = highlightBuilder;
             rewrittenBuilder.indexBoost = indexBoost;
@@ -971,16 +971,12 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                     trackScores = parser.booleanValue();
                 } else if (context.getParseFieldMatcher().match(currentFieldName, _SOURCE_FIELD)) {
                     fetchSourceContext = FetchSourceContext.parse(context);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, STORED_FIELDS_FIELD)) {
-                    storedField(parser.text());
+                } else if (context.getParseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) {
+                    field(parser.text());
                 } else if (context.getParseFieldMatcher().match(currentFieldName, SORT_FIELD)) {
                     sort(parser.text());
                 } else if (context.getParseFieldMatcher().match(currentFieldName, PROFILE_FIELD)) {
                     profile = parser.booleanValue();
-                } else if (context.getParseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) {
-                    throw new ParsingException(parser.getTokenLocation(), "Deprecated field [" +
-                        SearchSourceBuilder.FIELDS_FIELD + "] used, expected [" +
-                        SearchSourceBuilder.STORED_FIELDS_FIELD + "] instead");
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].",
                             parser.getTokenLocation());
@@ -1030,21 +1026,22 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                             parser.getTokenLocation());
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-                if (context.getParseFieldMatcher().match(currentFieldName, STORED_FIELDS_FIELD)) {
-                    storedFieldNames = new ArrayList<>();
+
+                if (context.getParseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) {
+                    fieldNames = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token == XContentParser.Token.VALUE_STRING) {
-                            storedFieldNames.add(parser.text());
+                            fieldNames.add(parser.text());
                         } else {
                             throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.VALUE_STRING + "] in ["
                                     + currentFieldName + "] but found [" + token + "]", parser.getTokenLocation());
                         }
                     }
-                } else if (context.getParseFieldMatcher().match(currentFieldName, DOCVALUE_FIELDS_FIELD)) {
-                    docValueFields = new ArrayList<>();
+                } else if (context.getParseFieldMatcher().match(currentFieldName, FIELDDATA_FIELDS_FIELD)) {
+                    fieldDataFields = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token == XContentParser.Token.VALUE_STRING) {
-                            docValueFields.add(parser.text());
+                            fieldDataFields.add(parser.text());
                         } else {
                             throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.VALUE_STRING + "] in ["
                                     + currentFieldName + "] but found [" + token + "]", parser.getTokenLocation());
@@ -1071,11 +1068,6 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                     fetchSourceContext = FetchSourceContext.parse(context);
                 } else if (context.getParseFieldMatcher().match(currentFieldName, SEARCH_AFTER)) {
                     searchAfterBuilder = SearchAfterBuilder.fromXContent(parser, context.getParseFieldMatcher());
-                } else if (context.getParseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) {
-                    throw new ParsingException(parser.getTokenLocation(), "The field [" +
-                        SearchSourceBuilder.FIELDS_FIELD + "] is not longer supported, please use [" +
-                        SearchSourceBuilder.STORED_FIELDS_FIELD + "] to retrieve stored fields or _source filtering " +
-                        "if the field is not stored");
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].",
                             parser.getTokenLocation());
@@ -1139,21 +1131,21 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
             builder.field(_SOURCE_FIELD.getPreferredName(), fetchSourceContext);
         }
 
-        if (storedFieldNames != null) {
-            if (storedFieldNames.size() == 1) {
-                builder.field(STORED_FIELDS_FIELD.getPreferredName(), storedFieldNames.get(0));
+        if (fieldNames != null) {
+            if (fieldNames.size() == 1) {
+                builder.field(FIELDS_FIELD.getPreferredName(), fieldNames.get(0));
             } else {
-                builder.startArray(STORED_FIELDS_FIELD.getPreferredName());
-                for (String fieldName : storedFieldNames) {
+                builder.startArray(FIELDS_FIELD.getPreferredName());
+                for (String fieldName : fieldNames) {
                     builder.value(fieldName);
                 }
                 builder.endArray();
             }
         }
 
-        if (docValueFields != null) {
-            builder.startArray(DOCVALUE_FIELDS_FIELD.getPreferredName());
-            for (String fieldDataField : docValueFields) {
+        if (fieldDataFields != null) {
+            builder.startArray(FIELDDATA_FIELDS_FIELD.getPreferredName());
+            for (String fieldDataField : fieldDataFields) {
                 builder.value(fieldDataField);
             }
             builder.endArray();
@@ -1347,7 +1339,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
 
     @Override
     public int hashCode() {
-        return Objects.hash(aggregations, explain, fetchSourceContext, docValueFields, storedFieldNames, from,
+        return Objects.hash(aggregations, explain, fetchSourceContext, fieldDataFields, fieldNames, from,
                 highlightBuilder, indexBoost, minScore, postQueryBuilder, queryBuilder, rescoreBuilders, scriptFields,
                 size, sorts, searchAfterBuilder, sliceBuilder, stats, suggestBuilder, terminateAfter, timeoutInMillis, trackScores, version, profile);
     }
@@ -1364,8 +1356,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         return Objects.equals(aggregations, other.aggregations)
                 && Objects.equals(explain, other.explain)
                 && Objects.equals(fetchSourceContext, other.fetchSourceContext)
-                && Objects.equals(docValueFields, other.docValueFields)
-                && Objects.equals(storedFieldNames, other.storedFieldNames)
+                && Objects.equals(fieldDataFields, other.fieldDataFields)
+                && Objects.equals(fieldNames, other.fieldNames)
                 && Objects.equals(from, other.from)
                 && Objects.equals(highlightBuilder, other.highlightBuilder)
                 && Objects.equals(indexBoost, other.indexBoost)
diff --git a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
index 596714b1c5b..f0e12abeac8 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
@@ -677,7 +677,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
         client().prepareIndex(IDX, "doc", "4").setSource("foo", "eggplant").get();
         flushAndRefresh(IDX);
 
-        SearchResponse resp = client().prepareSearch(IDX).setQuery(matchAllQuery()).addDocValueField("foo").addSort("foo", SortOrder.ASC).get();
+        SearchResponse resp = client().prepareSearch(IDX).setQuery(matchAllQuery()).addFieldDataField("foo").addSort("foo", SortOrder.ASC).get();
         assertHitCount(resp, 4);
         assertOrderedSearchHits(resp, "2", "3", "4", "1");
         SearchHit[] hits = resp.getHits().hits();
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java
index da0c3d081af..31e4e2d0923 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java
@@ -166,9 +166,9 @@ public class TokenCountFieldMapperIntegrationIT extends ESIntegTestCase {
 
     private SearchRequestBuilder prepareSearch() {
         SearchRequestBuilder request = client().prepareSearch("test").setTypes("test");
-        request.addStoredField("foo.token_count");
+        request.addField("foo.token_count");
         if (loadCountedFields) {
-            request.addStoredField("foo");
+            request.addField("foo");
         }
         return request;
     }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java
index 14dd370fbfd..202afd7a4b1 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java
@@ -816,7 +816,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
                 .field("lon", -74.0059731).endObject().endObject()).setRefreshPolicy(IMMEDIATE).get();
 
         // match all search with geohash field
-        SearchResponse searchResponse = client().prepareSearch().addStoredField("location.geohash").setQuery(matchAllQuery()).execute().actionGet();
+        SearchResponse searchResponse = client().prepareSearch().addField("location.geohash").setQuery(matchAllQuery()).execute().actionGet();
         Map<String, SearchHitField> m = searchResponse.getHits().getAt(0).getFields();
 
         // ensure single geohash was indexed
@@ -841,7 +841,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
                 .field("lon", -74.0059731).endObject().endObject()).setRefreshPolicy(IMMEDIATE).get();
 
         // match all search with geohash field (includes prefixes)
-        SearchResponse searchResponse = client().prepareSearch().addStoredField("location.geohash").setQuery(matchAllQuery()).execute().actionGet();
+        SearchResponse searchResponse = client().prepareSearch().addField("location.geohash").setQuery(matchAllQuery()).execute().actionGet();
         Map<String, SearchHitField> m = searchResponse.getHits().getAt(0).getFields();
 
         List<Object> hashes = m.get("location.geohash").values();
@@ -872,11 +872,11 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
         }
 
         // query by geohash subfield
-        SearchResponse searchResponse = client().prepareSearch().addStoredField("location.geohash").setQuery(matchAllQuery()).execute().actionGet();
+        SearchResponse searchResponse = client().prepareSearch().addField("location.geohash").setQuery(matchAllQuery()).execute().actionGet();
         assertEquals(numDocs, searchResponse.getHits().totalHits());
 
         // query by latlon subfield
-        searchResponse = client().prepareSearch().addStoredField("location.latlon").setQuery(matchAllQuery()).execute().actionGet();
+        searchResponse = client().prepareSearch().addField("location.latlon").setQuery(matchAllQuery()).execute().actionGet();
         assertEquals(numDocs, searchResponse.getHits().totalHits());
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
index 841f35bd975..d4ba6ca9062 100644
--- a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
@@ -218,8 +218,8 @@ public class InnerHitBuilderTests extends ESTestCase {
         innerHits.setExplain(randomBoolean());
         innerHits.setVersion(randomBoolean());
         innerHits.setTrackScores(randomBoolean());
-        innerHits.setStoredFieldNames(randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16)));
-        innerHits.setDocValueFields(randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16)));
+        innerHits.setFieldNames(randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16)));
+        innerHits.setFieldDataFields(randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16)));
         // Random script fields deduped on their field name.
         Map<String, SearchSourceBuilder.ScriptField> scriptFields = new HashMap<>();
         for (SearchSourceBuilder.ScriptField field: randomListStuff(16, InnerHitBuilderTests::randomScript)) {
@@ -294,11 +294,11 @@ public class InnerHitBuilderTests extends ESTestCase {
                 break;
             case 6:
                 if (randomBoolean()) {
-                    instance.setDocValueFields(randomValueOtherThan(instance.getDocValueFields(), () -> {
+                    instance.setFieldDataFields(randomValueOtherThan(instance.getFieldDataFields(), () -> {
                         return randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16));
                     }));
                 } else {
-                    instance.addDocValueField(randomAsciiOfLengthBetween(1, 16));
+                    instance.addFieldDataField(randomAsciiOfLengthBetween(1, 16));
                 }
                 break;
             case 7:
@@ -341,12 +341,12 @@ public class InnerHitBuilderTests extends ESTestCase {
                         HighlightBuilderTests::randomHighlighterBuilder));
                 break;
             case 11:
-                if (instance.getStoredFieldNames() == null || randomBoolean()) {
-                    instance.setStoredFieldNames(randomValueOtherThan(instance.getStoredFieldNames(), () -> {
+                if (instance.getFieldNames() == null || randomBoolean()) {
+                    instance.setFieldNames(randomValueOtherThan(instance.getFieldNames(), () -> {
                         return randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16));
                     }));
                 } else {
-                    instance.getStoredFieldNames().add(randomAsciiOfLengthBetween(1, 16));
+                    instance.getFieldNames().add(randomAsciiOfLengthBetween(1, 16));
                 }
                 break;
             default:
diff --git a/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java b/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java
index 590c5c624f1..163d72f4553 100644
--- a/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java
+++ b/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java
@@ -113,7 +113,7 @@ public class ExceptionRetryIT extends ESIntegTestCase {
         }
 
         refresh();
-        SearchResponse searchResponse = client().prepareSearch("index").setSize(numDocs * 2).addStoredField("_id").get();
+        SearchResponse searchResponse = client().prepareSearch("index").setSize(numDocs * 2).addField("_id").get();
 
         Set<String> uniqueIds = new HashSet();
         long dupCounter = 0;
diff --git a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
index acdfdd12266..493f8b74e04 100644
--- a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
@@ -116,7 +116,7 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
         ensureGreen();
         SearchResponse searchResponse = client().prepareSearch("test_index")
                 .setQuery(termQuery("field1", "value1"))
-                .addStoredField("field1").addStoredField("field2")
+                .addField("field1").addField("field2")
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 1);
@@ -130,7 +130,7 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
         // now only match on one template (template_1)
         searchResponse = client().prepareSearch("text_index")
                 .setQuery(termQuery("field1", "value1"))
-                .addStoredField("field1").addStoredField("field2")
+                .addField("field1").addField("field2")
                 .execute().actionGet();
         if (searchResponse.getFailedShards() > 0) {
             logger.warn("failed search {}", Arrays.toString(searchResponse.getShardFailures()));
diff --git a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
index 1145e594b25..bc9909b21c1 100644
--- a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
+++ b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
@@ -219,7 +219,7 @@ public class RelocationIT extends ESIntegTestCase {
             for (int i = 0; i < 10; i++) {
                 try {
                     logger.info("--> START search test round {}", i + 1);
-                    SearchHits hits = client().prepareSearch("test").setQuery(matchAllQuery()).setSize((int) indexer.totalIndexedDocs()).setNoStoredFields().execute().actionGet().getHits();
+                    SearchHits hits = client().prepareSearch("test").setQuery(matchAllQuery()).setSize((int) indexer.totalIndexedDocs()).setNoFields().execute().actionGet().getHits();
                     ranOnce = true;
                     if (hits.totalHits() != indexer.totalIndexedDocs()) {
                         int[] hitIds = new int[(int) indexer.totalIndexedDocs()];
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java
index 5cc6ec58630..49ef9e1b6b5 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java
@@ -181,7 +181,7 @@ public abstract class AbstractGeoTestCase extends ESIntegTestCase {
         // Added to debug a test failure where the terms aggregation seems to be reporting two documents with the same value for NUMBER_FIELD_NAME.  This will check that after
         // random indexing each document only has 1 value for NUMBER_FIELD_NAME and it is the correct value. Following this initial change its seems that this call was getting
         // more that 2000 hits (actual value was 2059) so now it will also check to ensure all hits have the correct index and type
-        SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME).addStoredField(NUMBER_FIELD_NAME).addSort(SortBuilders.fieldSort(NUMBER_FIELD_NAME)
+        SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME).addField(NUMBER_FIELD_NAME).addSort(SortBuilders.fieldSort(NUMBER_FIELD_NAME)
                 .order(SortOrder.ASC)).setSize(5000).get();
         assertSearchResponse(response);
         long totalHits = response.getHits().totalHits();
diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index 575018a3097..6f2facb24a9 100644
--- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -222,12 +222,12 @@ public class SearchSourceBuilderTests extends ESTestCase {
             for (int i = 0; i < fieldsSize; i++) {
                 fields.add(randomAsciiOfLengthBetween(5, 50));
             }
-            builder.storedFields(fields);
+            builder.fields(fields);
         }
         if (randomBoolean()) {
             int fieldDataFieldsSize = randomInt(25);
             for (int i = 0; i < fieldDataFieldsSize; i++) {
-                builder.docValueField(randomAsciiOfLengthBetween(5, 50));
+                builder.fieldDataField(randomAsciiOfLengthBetween(5, 50));
             }
         }
         if (randomBoolean()) {
diff --git a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
index 68679e89ae6..f8ca1e1aaf7 100644
--- a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
@@ -202,7 +202,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         refresh();
 
         // TEST FETCHING _parent from child
-        SearchResponse searchResponse = client().prepareSearch("test").setQuery(idsQuery("child").addIds("c1")).storedFields("_parent").execute()
+        SearchResponse searchResponse = client().prepareSearch("test").setQuery(idsQuery("child").addIds("c1")).fields("_parent").execute()
                 .actionGet();
         assertNoFailures(searchResponse);
         assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
@@ -210,7 +210,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         assertThat(searchResponse.getHits().getAt(0).field("_parent").value().toString(), equalTo("p1"));
 
         // TEST matching on parent
-        searchResponse = client().prepareSearch("test").setQuery(termQuery("_parent#parent", "p1")).storedFields("_parent").get();
+        searchResponse = client().prepareSearch("test").setQuery(termQuery("_parent#parent", "p1")).fields("_parent").get();
         assertNoFailures(searchResponse);
         assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
         assertThat(searchResponse.getHits().getAt(0).id(), anyOf(equalTo("c1"), equalTo("c2")));
@@ -218,7 +218,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         assertThat(searchResponse.getHits().getAt(1).id(), anyOf(equalTo("c1"), equalTo("c2")));
         assertThat(searchResponse.getHits().getAt(1).field("_parent").value().toString(), equalTo("p1"));
 
-        searchResponse = client().prepareSearch("test").setQuery(queryStringQuery("_parent#parent:p1")).storedFields("_parent").get();
+        searchResponse = client().prepareSearch("test").setQuery(queryStringQuery("_parent#parent:p1")).fields("_parent").get();
         assertNoFailures(searchResponse);
         assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
         assertThat(searchResponse.getHits().getAt(0).id(), anyOf(equalTo("c1"), equalTo("c2")));
@@ -1394,7 +1394,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
             SearchResponse scrollResponse = client().prepareSearch("test")
                     .setScroll(TimeValue.timeValueSeconds(30))
                     .setSize(1)
-                    .addStoredField("_id")
+                    .addField("_id")
                     .setQuery(query)
                     .execute()
                     .actionGet();
diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java
index 985605c4e65..0debdb263af 100644
--- a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java
+++ b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java
@@ -415,13 +415,13 @@ public class GeoFilterIT extends ESIntegTestCase {
             assertThat(hit.getId(), equalTo(key));
         }
 
-        SearchResponse world = client().prepareSearch().addStoredField("pin").setQuery(
+        SearchResponse world = client().prepareSearch().addField("pin").setQuery(
                 geoBoundingBoxQuery("pin").setCorners(90, -179.99999, -90, 179.99999)
         ).execute().actionGet();
 
         assertHitCount(world, 53);
 
-        SearchResponse distance = client().prepareSearch().addStoredField("pin").setQuery(
+        SearchResponse distance = client().prepareSearch().addField("pin").setQuery(
                 geoDistanceQuery("pin").distance("425km").point(51.11, 9.851)
                 ).execute().actionGet();
 
diff --git a/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java b/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
index 56b33c6007e..01f98564814 100644
--- a/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
@@ -156,7 +156,7 @@ public class InnerHitsIT extends ESIntegTestCase {
                 .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg).innerHit(
                         new InnerHitBuilder().setHighlightBuilder(new HighlightBuilder().field("comments.message"))
                                 .setExplain(true)
-                                .addDocValueField("comments.message")
+                                .addFieldDataField("comments.message")
                                 .addScriptField("script", new Script("5", ScriptService.ScriptType.INLINE, MockScriptEngine.NAME, Collections.emptyMap()))
                                 .setSize(1)
                 )).get();
@@ -287,7 +287,7 @@ public class InnerHitsIT extends ESIntegTestCase {
                 .setQuery(
                         hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit(
                                 new InnerHitBuilder()
-                                        .addDocValueField("message")
+                                        .addFieldDataField("message")
                                         .setHighlightBuilder(new HighlightBuilder().field("message"))
                                         .setExplain(true).setSize(1)
                                         .addScriptField("script", new Script("5", ScriptService.ScriptType.INLINE,
diff --git a/core/src/test/java/org/elasticsearch/search/source/SourceFetchingIT.java b/core/src/test/java/org/elasticsearch/search/source/SourceFetchingIT.java
index df147ce106f..33fcb55cada 100644
--- a/core/src/test/java/org/elasticsearch/search/source/SourceFetchingIT.java
+++ b/core/src/test/java/org/elasticsearch/search/source/SourceFetchingIT.java
@@ -37,10 +37,10 @@ public class SourceFetchingIT extends ESIntegTestCase {
         SearchResponse response = client().prepareSearch("test").get();
         assertThat(response.getHits().getAt(0).getSourceAsString(), notNullValue());
 
-        response = client().prepareSearch("test").addStoredField("bla").get();
+        response = client().prepareSearch("test").addField("bla").get();
         assertThat(response.getHits().getAt(0).getSourceAsString(), nullValue());
 
-        response = client().prepareSearch("test").addStoredField("_source").get();
+        response = client().prepareSearch("test").addField("_source").get();
         assertThat(response.getHits().getAt(0).getSourceAsString(), notNullValue());
 
     }
diff --git a/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java b/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java
index e81b4decb2d..3573089fcaa 100644
--- a/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java
+++ b/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java
@@ -87,7 +87,7 @@ public class TimestampTTLBWIT extends ESIntegTestCase {
                 .setQuery(matchAllQuery())
                 .setSize(randomIntBetween(1, numDocs + 5))
                 .addSort("_timestamp", order)
-                .addStoredField("_timestamp")
+                .addField("_timestamp")
                 .execute().actionGet();
         assertNoFailures(searchResponse);
         SearchHit[] hits = searchResponse.getHits().hits();
diff --git a/docs/plugins/mapper-attachments.asciidoc b/docs/plugins/mapper-attachments.asciidoc
index f2c034a317e..b287decd4cf 100644
--- a/docs/plugins/mapper-attachments.asciidoc
+++ b/docs/plugins/mapper-attachments.asciidoc
@@ -246,7 +246,7 @@ PUT /test/person/1?refresh=true
 }
 GET /test/person/_search
 {
-  "stored_fields": [ "file.content_type" ],
+  "fields": [ "file.content_type" ],
   "query": {
     "match": {
       "file.content_type": "text plain"
@@ -367,7 +367,7 @@ PUT /test/person/1?refresh=true
 }
 GET /test/person/_search
 {
-  "stored_fields": [],
+  "fields": [],
   "query": {
     "match": {
       "file.content": "king queen"
diff --git a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
index 2bad60ba0ec..1dd1524a58e 100644
--- a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
@@ -22,7 +22,7 @@ The top_hits aggregation returns regular search hits, because of this many per h
 * <<search-request-named-queries-and-filters,Named filters and queries>>
 * <<search-request-source-filtering,Source filtering>>
 * <<search-request-script-fields,Script fields>>
-* <<search-request-docvalue-fields,Doc value fields>>
+* <<search-request-fielddata-fields,Fielddata fields>>
 * <<search-request-version,Include versions>>
 
 ==== Example
diff --git a/docs/reference/mapping/params/store.asciidoc b/docs/reference/mapping/params/store.asciidoc
index 53cac7493ff..d34d1016546 100644
--- a/docs/reference/mapping/params/store.asciidoc
+++ b/docs/reference/mapping/params/store.asciidoc
@@ -48,7 +48,7 @@ PUT my_index/my_type/1
 
 GET my_index/_search
 {
-  "stored_fields": [ "title", "date" ] <2>
+  "fields": [ "title", "date" ] <2>
 }
 --------------------------------------------------
 // CONSOLE
diff --git a/docs/reference/migration/migrate_5_0/search.asciidoc b/docs/reference/migration/migrate_5_0/search.asciidoc
index 09478ee3251..72b29e8a9e7 100644
--- a/docs/reference/migration/migrate_5_0/search.asciidoc
+++ b/docs/reference/migration/migrate_5_0/search.asciidoc
@@ -64,15 +64,11 @@ characteristics as the former `scan` search type.
 
 ==== `fields` parameter
 
-The `fields` parameter has been replaced by `stored_fields`.
-The `stored_fields` parameter will only return stored fields
+The `fields` parameter used to try to retrieve field values from stored
+fields, and fall back to extracting from the `_source` if a field is not
+marked as stored.  Now, the `fields` parameter will only return stored fields
 -- it will no longer extract values from the `_source`.
 
-==== `fielddata_fields` parameter
-
-The `fielddata_fields` has been deprecated, use parameter `docvalue_fields` instead.
-
-
 ==== search-exists API removed
 
 The search exists api has been removed in favour of using the search api with
diff --git a/docs/reference/modules/scripting/painless.asciidoc b/docs/reference/modules/scripting/painless.asciidoc
index 3633d2c3e71..db68db2c7a3 100644
--- a/docs/reference/modules/scripting/painless.asciidoc
+++ b/docs/reference/modules/scripting/painless.asciidoc
@@ -143,7 +143,7 @@ First, let's look at the source data for a player by submitting the following re
 ----------------------------------------------------------------
 GET hockey/_search
 {
-  "stored_fields": [
+  "fields": [
     "_id",
     "_source"
   ],
diff --git a/docs/reference/search/request-body.asciidoc b/docs/reference/search/request-body.asciidoc
index a9adc157bd3..958320ea110 100644
--- a/docs/reference/search/request-body.asciidoc
+++ b/docs/reference/search/request-body.asciidoc
@@ -143,11 +143,11 @@ include::request/sort.asciidoc[]
 
 include::request/source-filtering.asciidoc[]
 
-include::request/stored-fields.asciidoc[]
+include::request/fields.asciidoc[]
 
 include::request/script-fields.asciidoc[]
 
-include::request/docvalue-fields.asciidoc[]
+include::request/fielddata-fields.asciidoc[]
 
 include::request/post-filter.asciidoc[]
 
diff --git a/docs/reference/search/request/docvalue-fields.asciidoc b/docs/reference/search/request/docvalue-fields.asciidoc
deleted file mode 100644
index b4d2493d853..00000000000
--- a/docs/reference/search/request/docvalue-fields.asciidoc
+++ /dev/null
@@ -1,23 +0,0 @@
-[[search-request-docvalue-fields]]
-=== Doc value Fields
-
-Allows to return the <<doc-values,doc value>> representation of a field for each hit, for
-example:
-
-[source,js]
---------------------------------------------------
-GET /_search
-{
-    "query" : {
-        "match_all": {}
-    },
-    "docvalue_fields" : ["test1", "test2"]
-}
---------------------------------------------------
-// CONSOLE
-
-Doc value fields can work on fields that are not stored.
-
-Note that if the fields parameter specifies fields without docvalues it will try to load the value from the fielddata cache
-causing the terms for that field to be loaded to memory (cached), which will result in more memory consumption.
-
diff --git a/docs/reference/search/request/fielddata-fields.asciidoc b/docs/reference/search/request/fielddata-fields.asciidoc
new file mode 100644
index 00000000000..f3a3508b144
--- /dev/null
+++ b/docs/reference/search/request/fielddata-fields.asciidoc
@@ -0,0 +1,23 @@
+[[search-request-fielddata-fields]]
+=== Field Data Fields
+
+Allows to return the <<fielddata,field data>> representation of a field for each hit, for
+example:
+
+[source,js]
+--------------------------------------------------
+GET /_search
+{
+    "query" : {
+        "match_all": {}
+    },
+    "fielddata_fields" : ["test1", "test2"]
+}
+--------------------------------------------------
+// CONSOLE
+
+Field data fields can work on fields that are not stored.
+
+It's important to understand that using the `fielddata_fields` parameter will
+cause the terms for that field to be loaded to memory (cached), which will
+result in more memory consumption.
diff --git a/docs/reference/search/request/stored-fields.asciidoc b/docs/reference/search/request/fields.asciidoc
similarity index 91%
rename from docs/reference/search/request/stored-fields.asciidoc
rename to docs/reference/search/request/fields.asciidoc
index 3d5b8c01b47..3483d470ee2 100644
--- a/docs/reference/search/request/stored-fields.asciidoc
+++ b/docs/reference/search/request/fields.asciidoc
@@ -1,7 +1,7 @@
 [[search-request-fields]]
 === Fields
 
-WARNING: The `stored_fields` parameter is about fields that are explicitly marked as
+WARNING: The `fields` parameter is about fields that are explicitly marked as
 stored in the mapping, which is off by default and generally not recommended.
 Use <<search-request-source-filtering,source filtering>> instead to select
 subsets of the original source document to be returned.
@@ -13,7 +13,7 @@ by a search hit.
 --------------------------------------------------
 GET /_search
 {
-    "stored_fields" : ["user", "postDate"],
+    "fields" : ["user", "postDate"],
     "query" : {
         "term" : { "user" : "kimchy" }
     }
@@ -30,7 +30,7 @@ returned, for example:
 --------------------------------------------------
 GET /_search
 {
-    "stored_fields" : [],
+    "fields" : [],
     "query" : {
         "term" : { "user" : "kimchy" }
     }
diff --git a/docs/reference/search/request/highlighting.asciidoc b/docs/reference/search/request/highlighting.asciidoc
index efb7053c179..c235c37b338 100644
--- a/docs/reference/search/request/highlighting.asciidoc
+++ b/docs/reference/search/request/highlighting.asciidoc
@@ -372,7 +372,7 @@ query and the rescore query in `highlight_query`.
 --------------------------------------------------
 GET /_search
 {
-    "stored_fields": [ "_id" ],
+    "fields": [ "_id" ],
     "query" : {
         "match": {
             "content": {
diff --git a/docs/reference/search/request/inner-hits.asciidoc b/docs/reference/search/request/inner-hits.asciidoc
index 3c8e0e9f00e..345bc9abde2 100644
--- a/docs/reference/search/request/inner-hits.asciidoc
+++ b/docs/reference/search/request/inner-hits.asciidoc
@@ -72,7 +72,7 @@ Inner hits also supports the following per document features:
 * <<search-request-explain,Explain>>
 * <<search-request-source-filtering,Source filtering>>
 * <<search-request-script-fields,Script fields>>
-* <<search-request-docvalue-fields,Doc value fields>>
+* <<search-request-fielddata-fields,Fielddata fields>>
 * <<search-request-version,Include versions>>
 
 [[nested-inner-hits]]
diff --git a/docs/reference/search/uri-request.asciidoc b/docs/reference/search/uri-request.asciidoc
index ba36992f6fb..496f04ea4af 100644
--- a/docs/reference/search/uri-request.asciidoc
+++ b/docs/reference/search/uri-request.asciidoc
@@ -83,7 +83,7 @@ hits was computed.
 part of the document by using `_source_include` & `_source_exclude` (see the <<search-request-source-filtering, request body>>
 documentation for more details)
 
-|`stored_fields` |The selective stored fields of the document to return for each hit,
+|`fields` |The selective stored fields of the document to return for each hit,
 comma delimited. Not specifying any value will cause no fields to return.
 
 |`sort` |Sorting to perform. Can either be in the form of `fieldName`, or
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java
index 0d7dd4e12e2..e9c4bf6e359 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java
@@ -72,28 +72,28 @@ public class GeoDistanceTests extends ESIntegTestCase {
 
         refresh();
 
-        SearchResponse searchResponse1 = client().prepareSearch().addStoredField("_source")
+        SearchResponse searchResponse1 = client().prepareSearch().addField("_source")
                 .addScriptField("distance", new Script("doc['location'].arcDistance(" + target_lat + "," + target_long + ")")).execute()
                 .actionGet();
         Double resultDistance1 = searchResponse1.getHits().getHits()[0].getFields().get("distance").getValue();
         assertThat(resultDistance1,
                 closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.DEFAULT), 0.01d));
 
-        SearchResponse searchResponse2 = client().prepareSearch().addStoredField("_source")
+        SearchResponse searchResponse2 = client().prepareSearch().addField("_source")
                 .addScriptField("distance", new Script("doc['location'].distance(" + target_lat + "," + target_long + ")")).execute()
                 .actionGet();
         Double resultDistance2 = searchResponse2.getHits().getHits()[0].getFields().get("distance").getValue();
         assertThat(resultDistance2,
                 closeTo(GeoDistance.PLANE.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.DEFAULT), 0.01d));
 
-        SearchResponse searchResponse3 = client().prepareSearch().addStoredField("_source")
+        SearchResponse searchResponse3 = client().prepareSearch().addField("_source")
                 .addScriptField("distance", new Script("doc['location'].arcDistanceInKm(" + target_lat + "," + target_long + ")"))
                 .execute().actionGet();
         Double resultArcDistance3 = searchResponse3.getHits().getHits()[0].getFields().get("distance").getValue();
         assertThat(resultArcDistance3,
                 closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS), 0.01d));
 
-        SearchResponse searchResponse4 = client().prepareSearch().addStoredField("_source")
+        SearchResponse searchResponse4 = client().prepareSearch().addField("_source")
                 .addScriptField("distance", new Script("doc['location'].distanceInKm(" + target_lat + "," + target_long + ")")).execute()
                 .actionGet();
         Double resultDistance4 = searchResponse4.getHits().getHits()[0].getFields().get("distance").getValue();
@@ -102,7 +102,7 @@ public class GeoDistanceTests extends ESIntegTestCase {
 
         SearchResponse searchResponse5 = client()
                 .prepareSearch()
-                .addStoredField("_source")
+                .addField("_source")
                 .addScriptField("distance", new Script("doc['location'].arcDistanceInKm(" + (target_lat) + "," + (target_long + 360) + ")"))
                 .execute().actionGet();
         Double resultArcDistance5 = searchResponse5.getHits().getHits()[0].getFields().get("distance").getValue();
@@ -111,21 +111,21 @@ public class GeoDistanceTests extends ESIntegTestCase {
 
         SearchResponse searchResponse6 = client()
                 .prepareSearch()
-                .addStoredField("_source")
+                .addField("_source")
                 .addScriptField("distance", new Script("doc['location'].arcDistanceInKm(" + (target_lat + 360) + "," + (target_long) + ")"))
                 .execute().actionGet();
         Double resultArcDistance6 = searchResponse6.getHits().getHits()[0].getFields().get("distance").getValue();
         assertThat(resultArcDistance6,
                 closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS), 0.01d));
 
-        SearchResponse searchResponse7 = client().prepareSearch().addStoredField("_source")
+        SearchResponse searchResponse7 = client().prepareSearch().addField("_source")
                 .addScriptField("distance", new Script("doc['location'].arcDistanceInMiles(" + target_lat + "," + target_long + ")"))
                 .execute().actionGet();
         Double resultDistance7 = searchResponse7.getHits().getHits()[0].getFields().get("distance").getValue();
         assertThat(resultDistance7,
                 closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.MILES), 0.01d));
 
-        SearchResponse searchResponse8 = client().prepareSearch().addStoredField("_source")
+        SearchResponse searchResponse8 = client().prepareSearch().addField("_source")
                 .addScriptField("distance", new Script("doc['location'].distanceInMiles(" + target_lat + "," + target_long + ")"))
                 .execute().actionGet();
         Double resultDistance8 = searchResponse8.getHits().getHits()[0].getFields().get("distance").getValue();
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
index 6a8be584085..28894c5c1cc 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
@@ -102,33 +102,33 @@ public class SearchFieldsTests extends ESIntegTestCase {
 
         client().admin().indices().prepareRefresh().execute().actionGet();
 
-        SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field1").execute().actionGet();
+        SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("field1").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field1").value().toString(), equalTo("value1"));
 
         // field2 is not stored, check that it is not extracted from source.
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field2").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("field2").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(0));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field2"), nullValue());
 
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field3").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("field3").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3"));
 
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("*3").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("*3").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3"));
 
 
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("*3").addStoredField("field1").addStoredField("field2").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("*3").addField("field1").addField("field2").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(2));
@@ -136,20 +136,20 @@ public class SearchFieldsTests extends ESIntegTestCase {
         assertThat(searchResponse.getHits().getAt(0).fields().get("field1").value().toString(), equalTo("value1"));
 
 
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field*").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("field*").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(2));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3"));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field1").value().toString(), equalTo("value1"));
 
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("f*3").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("f*3").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3"));
 
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("*").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("*").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).source(), nullValue());
@@ -157,7 +157,7 @@ public class SearchFieldsTests extends ESIntegTestCase {
         assertThat(searchResponse.getHits().getAt(0).fields().get("field1").value().toString(), equalTo("value1"));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3"));
 
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("*").addStoredField("_source").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("*").addField("_source").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).source(), notNullValue());
@@ -437,15 +437,15 @@ public class SearchFieldsTests extends ESIntegTestCase {
         client().admin().indices().prepareRefresh().execute().actionGet();
 
         SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery())
-                .addStoredField("byte_field")
-                .addStoredField("short_field")
-                .addStoredField("integer_field")
-                .addStoredField("long_field")
-                .addStoredField("float_field")
-                .addStoredField("double_field")
-                .addStoredField("date_field")
-                .addStoredField("boolean_field")
-                .addStoredField("binary_field")
+                .addField("byte_field")
+                .addField("short_field")
+                .addField("integer_field")
+                .addField("long_field")
+                .addField("float_field")
+                .addField("double_field")
+                .addField("date_field")
+                .addField("boolean_field")
+                .addField("binary_field")
                 .execute().actionGet();
 
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
@@ -478,7 +478,7 @@ public class SearchFieldsTests extends ESIntegTestCase {
 
         SearchResponse searchResponse = client().prepareSearch("my-index")
                 .setTypes("my-type1")
-                .addStoredField("field1").addStoredField("_routing")
+                .addField("field1").addField("_routing")
                 .get();
 
         assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
@@ -493,7 +493,7 @@ public class SearchFieldsTests extends ESIntegTestCase {
                 .setRefreshPolicy(IMMEDIATE)
                 .get();
 
-        assertFailures(client().prepareSearch("my-index").setTypes("my-type1").addStoredField("field1"),
+        assertFailures(client().prepareSearch("my-index").setTypes("my-type1").addField("field1"),
                 RestStatus.BAD_REQUEST,
                 containsString("field [field1] isn't a leaf field"));
     }
@@ -557,14 +557,14 @@ public class SearchFieldsTests extends ESIntegTestCase {
 
 
         String field = "field1.field2.field3.field4";
-        SearchResponse searchResponse = client().prepareSearch("my-index").setTypes("my-type1").addStoredField(field).get();
+        SearchResponse searchResponse = client().prepareSearch("my-index").setTypes("my-type1").addField(field).get();
         assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().getAt(0).field(field).isMetadataField(), equalTo(false));
         assertThat(searchResponse.getHits().getAt(0).field(field).getValues().size(), equalTo(2));
         assertThat(searchResponse.getHits().getAt(0).field(field).getValues().get(0).toString(), equalTo("value1"));
         assertThat(searchResponse.getHits().getAt(0).field(field).getValues().get(1).toString(), equalTo("value2"));
 
-        searchResponse = client().prepareSearch("my-index").setTypes("my-type2").addStoredField(field).get();
+        searchResponse = client().prepareSearch("my-index").setTypes("my-type2").addField(field).get();
         assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().getAt(0).field(field).isMetadataField(), equalTo(false));
         assertThat(searchResponse.getHits().getAt(0).field(field).getValues().size(), equalTo(2));
@@ -621,16 +621,16 @@ public class SearchFieldsTests extends ESIntegTestCase {
         client().admin().indices().prepareRefresh().execute().actionGet();
 
         SearchRequestBuilder builder = client().prepareSearch().setQuery(matchAllQuery())
-                .addDocValueField("text_field")
-                .addDocValueField("keyword_field")
-                .addDocValueField("byte_field")
-                .addDocValueField("short_field")
-                .addDocValueField("integer_field")
-                .addDocValueField("long_field")
-                .addDocValueField("float_field")
-                .addDocValueField("double_field")
-                .addDocValueField("date_field")
-                .addDocValueField("boolean_field");
+                .addFieldDataField("text_field")
+                .addFieldDataField("keyword_field")
+                .addFieldDataField("byte_field")
+                .addFieldDataField("short_field")
+                .addFieldDataField("integer_field")
+                .addFieldDataField("long_field")
+                .addFieldDataField("float_field")
+                .addFieldDataField("double_field")
+                .addFieldDataField("date_field")
+                .addFieldDataField("boolean_field");
         SearchResponse searchResponse = builder.execute().actionGet();
 
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
@@ -704,7 +704,7 @@ public class SearchFieldsTests extends ESIntegTestCase {
                     .setParent("parent_1")
                     .setSource(jsonBuilder().startObject().field("field1", "value").endObject()));
 
-        SearchResponse response = client().prepareSearch("test").addStoredField("field1").get();
+        SearchResponse response = client().prepareSearch("test").addField("field1").get();
         assertSearchResponse(response);
         assertHitCount(response, 1);
 
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
index 7725ee7f519..80a6ff891da 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
@@ -123,8 +123,8 @@ public abstract class AbstractBulkByScrollRequest<Self extends AbstractBulkByScr
         if (searchRequest.source().from() != -1) {
             e = addValidationError("from is not supported in this context", e);
         }
-        if (searchRequest.source().storedFields() != null) {
-            e = addValidationError("stored_fields is not supported in this context", e);
+        if (searchRequest.source().fields() != null) {
+            e = addValidationError("fields is not supported in this context", e);
         }
         if (maxRetries < 0) {
             e = addValidationError("retries cannnot be negative", e);
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml
index 7f17befe909..8648c9034ee 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml
@@ -59,11 +59,11 @@
 ---
 "source fields may not be modified":
   - do:
-      catch: /stored_fields is not supported in this context/
+      catch: /fields is not supported in this context/
       delete_by_query:
         index: test
         body:
-          stored_fields: [_id]
+          fields: [_id]
 
 ---
 "requests_per_second cannot be an empty string":
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
index b3e35d2f49e..a7a5198e430 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
@@ -216,11 +216,11 @@
 ---
 "source fields may not be modified":
   - do:
-      catch: /stored_fields is not supported in this context/
+      catch: /fields is not supported in this context/
       reindex:
         body:
           source:
             index: test
-            stored_fields: [_id]
+            fields: [_id]
           dest:
             index: dest
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml
index 08ab3cbcc10..ea487eb54e0 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml
@@ -67,11 +67,11 @@
 ---
 "source fields may not be modified":
   - do:
-      catch: /stored_fields is not supported in this context/
+      catch: /fields is not supported in this context/
       update_by_query:
         index: test
         body:
-          stored_fields: [_id]
+          fields: [_id]
 
 ---
 "requests_per_second cannot be an empty string":
diff --git a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/30_mapping.yaml b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/30_mapping.yaml
index f180f51dfc5..458990cc90c 100644
--- a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/30_mapping.yaml
+++ b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/30_mapping.yaml
@@ -54,7 +54,7 @@
         search:
             index: test
             body:
-                stored_fields: [file.content_type,file.name]
+                fields: [file.content_type,file.name]
 
     - match: { hits.total: 1 }
     - match: { hits.hits.0.fields: { file.content_type: ["text/my-dummy-content-type"],  file.name: ["my-dummy-name-txt"] }}
diff --git a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/40_highlight.yaml b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/40_highlight.yaml
index c2d42be3b9a..dc6f800c078 100644
--- a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/40_highlight.yaml
+++ b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/40_highlight.yaml
@@ -57,7 +57,7 @@ setup:
                 query:
                     match:
                         file.content: "apache tika"
-                stored_fields: []
+                fields: []
                 highlight:
                   fields:
                     file.content: {}
diff --git a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/50_files_supported.yaml b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/50_files_supported.yaml
index 99f427c1ab9..69991b9d0c0 100644
--- a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/50_files_supported.yaml
+++ b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/50_files_supported.yaml
@@ -38,7 +38,7 @@ setup:
       search:
           index: test
           body:
-              stored_fields: [file.content, file.author, file.date, file.content_length, file.content_type]
+              fields: [file.content, file.author, file.date, file.content_length, file.content_type]
   - match: { hits.total: 1 }
   - match: { hits.hits.0.fields: {
       file.content: ["Test elasticsearch\n"],
@@ -65,7 +65,7 @@ setup:
       search:
           index: test
           body:
-              stored_fields: [file.content, file.author, file.date, file.content_length, file.content_type]
+              fields: [file.content, file.author, file.date, file.content_length, file.content_type]
   - match: { hits.total: 1 }
   - match: { hits.hits.0.fields: {
       file.content: ["Test elasticsearch\n"],
diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java
index 956645aebd2..761fb5fd144 100644
--- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java
+++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java
@@ -83,8 +83,8 @@ public class SizeFieldMapperUpgradeTests extends ESIntegTestCase {
         ElasticsearchAssertions.assertHitCount(countResponse, 3L);
 
         final SearchResponse sizeResponse = client().prepareSearch(indexName)
-                .addStoredField("_source")
-                .addStoredField("_size")
+                .addField("_source")
+                .addField("_size")
                 .get();
         ElasticsearchAssertions.assertHitCount(sizeResponse, 3L);
         for (SearchHit hit : sizeResponse.getHits().getHits()) {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
index 21fda8dc805..d2b9b8cf9b4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
@@ -38,17 +38,13 @@
           "type" : "boolean",
           "description" : "Specify whether to return detailed information about score computation as part of a hit"
         },
-        "stored_fields": {
+        "fields": {
           "type" : "list",
-          "description" : "A comma-separated list of stored fields to return as part of a hit"
-        },
-        "docvalue_fields": {
-          "type" : "list",
-          "description" : "A comma-separated list of fields to return as the docvalue representation of a field for each hit"
+          "description" : "A comma-separated list of fields to return as part of a hit"
         },
         "fielddata_fields": {
           "type" : "list",
-          "description" : "A comma-separated list of fields to return as the docvalue representation of a field for each hit"
+          "description" : "A comma-separated list of fields to return as the field data representation of a field for each hit"
         },
         "from": {
           "type" : "number",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml
index caf44d0c788..e0ac2aea2df 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml
@@ -77,25 +77,20 @@
   - do:
       search:
         body:
-          stored_fields: [ include.field2 ]
+          fields: [ include.field2 ]
           query: { match_all: {} }
   - is_false:  hits.hits.0._source
 
   - do:
         search:
           body:
-            stored_fields: [ include.field2, _source ]
+            fields: [ include.field2, _source ]
             query: { match_all: {} }
   - match: { hits.hits.0._source.include.field2: v2 }
   - is_true:  hits.hits.0._source
 
-  - do:
-      search:
-        docvalue_fields: [ "count" ]
-  - match: { hits.hits.0.fields.count: [1] }
 
   - do:
       search:
         fielddata_fields: [ "count" ]
   - match: { hits.hits.0.fields.count: [1] }
-
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yaml
index 993cbed2647..df7322f12c8 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yaml
@@ -31,6 +31,6 @@ setup:
                   term:
                       data: some
                   preference: _local
-              stored_fields: [user,amount]
+              fields: [user,amount]
 
 

From 3f5b9943fd39c99d9fcfa130e92cb3883253a527 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 27 Jun 2016 12:13:06 -0400
Subject: [PATCH 1077/1311] Randomize packaging upgrade test

This commit adds randomization for the packaging upgrade test. In
particular, we extract a list of the released version of Elasticsearch
from Maven Central and randomize the selection of the version to upgrade
from. The randomization is repeatable, and supports the tests.seed
property. Specific versions can be tested by setting the property
tests.packaging.upgrade.from.versions.

Relates #19033
---
 qa/vagrant/build.gradle | 66 +++++++++++++++++++++++++++++++++++++----
 qa/vagrant/versions     | 13 ++++++++
 2 files changed, 74 insertions(+), 5 deletions(-)
 create mode 100644 qa/vagrant/versions

diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle
index 58e7e50e421..905710a82be 100644
--- a/qa/vagrant/build.gradle
+++ b/qa/vagrant/build.gradle
@@ -40,8 +40,9 @@ List<String> availableBoxes = [
     'ubuntu-1504'
 ]
 
+String vagrantBoxes = getProperties().get('vagrant.boxes', 'sample')
 List<String> boxes = []
-for (String box : getProperties().get('vagrant.boxes', 'sample').split(',')) {
+for (String box : vagrantBoxes.split(',')) {
   if (box == 'sample') {
     boxes.add('centos-7')
     boxes.add('ubuntu-1404')
@@ -56,9 +57,28 @@ for (String box : getProperties().get('vagrant.boxes', 'sample').split(',')) {
   }
 }
 
-/* The version of elasticsearch that we upgrade *from* as part of testing
- * upgrades. */
-String upgradeFromVersion = '2.3.3'
+long seed
+String formattedSeed
+String[] upgradeFromVersions
+String upgradeFromVersion
+
+String maybeTestsSeed = System.getProperty("tests.seed", null);
+if (maybeTestsSeed != null) {
+  seed = new BigInteger(maybeTestsSeed, 16).longValue()
+  formattedSeed = maybeTestsSeed
+} else {
+  seed = new Random().nextLong()
+  formattedSeed = String.format("%016X", seed)
+}
+
+String maybeUpdradeFromVersions = System.getProperty("tests.packaging.upgrade.from.versions", null)
+if (maybeUpdradeFromVersions != null) {
+  upgradeFromVersions = maybeUpdradeFromVersions.split(",")
+} else {
+  upgradeFromVersions = new File(project.projectDir, 'versions')
+}
+
+upgradeFromVersion = upgradeFromVersions[new Random(seed).nextInt(upgradeFromVersions.length)]
 
 configurations {
   test
@@ -95,8 +115,37 @@ task stop {
   description 'Stop any tasks from tests that still may be running'
 }
 
+Set<String> getVersions() {
+  Node xml
+  new URL('http://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
+    xml = new XmlParser().parse(s)
+  }
+  return new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /2\.\d\.\d/ })
+}
+
+task updatePackagingTestUpgradeFromVersions {
+  doLast {
+    Set<String> versions = getVersions()
+    new File(project.projectDir, 'versions').text = versions.join('\n') + '\n'
+  }
+}
+
+task verifyPackagingTestUpgradeFromVersions {
+  doLast {
+    String maybeUpdateFromVersions = System.getProperty("tests.packaging.upgrade.from.versions", null)
+    if (maybeUpdateFromVersions == null) {
+      Set<String> versions = getVersions()
+      Set<String> actualVersions = new HashSet<>(Arrays.asList(upgradeFromVersions))
+      if (!versions.equals(actualVersions)) {
+        throw new GradleException("out-of-date versions [" + actualVersions + "], expected [" + versions + "]; run gradle updatePackagingTestUpgradeFromVersions")
+      }
+    }
+  }
+}
+
 File testRoot = new File("$buildDir/testroot")
 task createTestRoot {
+  dependsOn verifyPackagingTestUpgradeFromVersions
   outputs.dir testRoot
   doLast {
     testRoot.mkdirs()
@@ -147,6 +196,14 @@ task packagingTest {
               "    'sample' can be used to test a single yum and apt box. 'all' can be used to\n" +
               "    test all available boxes. The available boxes are: \n" +
               "    ${availableBoxes}"
+  gradle.addBuildListener new BuildAdapter() {
+    @Override
+    void buildFinished(BuildResult result) {
+      if (result.failure) {
+        println "Reproduce with: gradle packagingTest -Pvagrant.boxes=${vagrantBoxes} -Dtests.seed=${formattedSeed} -Dtests.packaging.upgrade.from.versions=${upgradeFromVersions.join(",")}"
+      }
+    }
+  }
 }
 
 // Each box gets it own set of tasks
@@ -199,4 +256,3 @@ for (String box : availableBoxes) {
   }
   packagingTest.dependsOn(packaging)
 }
-
diff --git a/qa/vagrant/versions b/qa/vagrant/versions
new file mode 100644
index 00000000000..5f6e0edf99c
--- /dev/null
+++ b/qa/vagrant/versions
@@ -0,0 +1,13 @@
+2.0.0
+2.0.1
+2.0.2
+2.1.0
+2.1.1
+2.1.2
+2.2.0
+2.2.1
+2.2.2
+2.3.0
+2.3.1
+2.3.2
+2.3.3

From f17fcce3edb2f0af6cb569f821c9353298886991 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Mon, 27 Jun 2016 11:17:58 -0700
Subject: [PATCH 1078/1311] Add duplicate mapper detection and tests

---
 .../elasticsearch/indices/IndicesModule.java  |  19 ++-
 .../indices/IndicesModuleTests.java           | 161 ++++++++++++++++++
 2 files changed, 175 insertions(+), 5 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java

diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
index e39e8b7e312..b7d4e321de8 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java
@@ -118,7 +118,11 @@ public class IndicesModule extends AbstractModule {
         }
 
         for (MapperPlugin mapperPlugin : mapperPlugins) {
-            mappers.putAll(mapperPlugin.getMappers());
+            for (Map.Entry<String, Mapper.TypeParser> entry : mapperPlugin.getMappers().entrySet()) {
+                if (mappers.put(entry.getKey(), entry.getValue()) != null) {
+                    throw new IllegalArgumentException("Mapper [" + entry.getKey() + "] is already registered");
+                }
+            }
         }
         return Collections.unmodifiableMap(mappers);
     }
@@ -143,11 +147,16 @@ public class IndicesModule extends AbstractModule {
         // _field_names is not registered here, see below
 
         for (MapperPlugin mapperPlugin : mapperPlugins) {
-            metadataMappers.putAll(mapperPlugin.getMetadataMappers());
-        }
-        if (metadataMappers.containsKey(FieldNamesFieldMapper.NAME)) {
-            throw new IllegalStateException("Plugin cannot contain metadata mapper [" + FieldNamesFieldMapper.NAME + "]");
+            for (Map.Entry<String, MetadataFieldMapper.TypeParser> entry : mapperPlugin.getMetadataMappers().entrySet()) {
+                if (entry.getKey().equals(FieldNamesFieldMapper.NAME)) {
+                    throw new IllegalArgumentException("Plugin cannot contain metadata mapper [" + FieldNamesFieldMapper.NAME + "]");
+                }
+                if (metadataMappers.put(entry.getKey(), entry.getValue()) != null) {
+                    throw new IllegalArgumentException("MetadataFieldMapper [" + entry.getKey() + "] is already registered");
+                }
+            }
         }
+
         // we register _field_names here so that it has a chance to see all other mappers, including from plugins
         metadataMappers.put(FieldNamesFieldMapper.NAME, new FieldNamesFieldMapper.TypeParser());
         return Collections.unmodifiableMap(metadataMappers);
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java
new file mode 100644
index 00000000000..73e78650128
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.indices;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.MetadataFieldMapper;
+import org.elasticsearch.index.mapper.core.TextFieldMapper;
+import org.elasticsearch.index.mapper.internal.FieldNamesFieldMapper;
+import org.elasticsearch.index.mapper.internal.IdFieldMapper;
+import org.elasticsearch.indices.mapper.MapperRegistry;
+import org.elasticsearch.plugins.MapperPlugin;
+import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matchers;
+
+public class IndicesModuleTests extends ESTestCase {
+
+    private static class FakeMapperParser implements Mapper.TypeParser {
+        @Override
+        public Mapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+            return null;
+        }
+    }
+
+    private static class FakeMetadataMapperParser implements MetadataFieldMapper.TypeParser {
+        @Override
+        public MetadataFieldMapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+            return null;
+        }
+        @Override
+        public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) {
+            return null;
+        }
+    }
+
+    List<MapperPlugin> fakePlugins = Arrays.asList(new MapperPlugin() {
+        @Override
+        public Map<String, Mapper.TypeParser> getMappers() {
+            return Collections.singletonMap("fake-mapper", new FakeMapperParser());
+        }
+        @Override
+        public Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers() {
+            return Collections.singletonMap("fake-metadata-mapper", new FakeMetadataMapperParser());
+        }
+    });
+
+    public void testBuiltinMappers() {
+        IndicesModule module = new IndicesModule(new NamedWriteableRegistry(), Collections.emptyList());
+        assertFalse(module.getMapperRegistry().getMapperParsers().isEmpty());
+        assertFalse(module.getMapperRegistry().getMetadataMapperParsers().isEmpty());
+    }
+
+    public void testBuiltinWithPlugins() {
+        IndicesModule module = new IndicesModule(new NamedWriteableRegistry(), fakePlugins);
+        MapperRegistry registry = module.getMapperRegistry();
+        assertThat(registry.getMapperParsers().size(), Matchers.greaterThan(1));
+        assertThat(registry.getMetadataMapperParsers().size(), Matchers.greaterThan(1));
+    }
+
+    public void testDuplicateBuiltinMapper() {
+        List<MapperPlugin> plugins = Arrays.asList(new MapperPlugin() {
+            @Override
+            public Map<String, Mapper.TypeParser> getMappers() {
+                return Collections.singletonMap(TextFieldMapper.CONTENT_TYPE, new FakeMapperParser());
+            }
+        });
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> new IndicesModule(new NamedWriteableRegistry(), plugins));
+        assertThat(e.getMessage(), Matchers.containsString("already registered"));
+    }
+
+    public void testDuplicateOtherPluginMapper() {
+        MapperPlugin plugin = new MapperPlugin() {
+            @Override
+            public Map<String, Mapper.TypeParser> getMappers() {
+                return Collections.singletonMap("foo", new FakeMapperParser());
+            }
+        };
+        List<MapperPlugin> plugins = Arrays.asList(plugin, plugin);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> new IndicesModule(new NamedWriteableRegistry(), plugins));
+        assertThat(e.getMessage(), Matchers.containsString("already registered"));
+    }
+
+    public void testDuplicateBuiltinMetadataMapper() {
+        List<MapperPlugin> plugins = Arrays.asList(new MapperPlugin() {
+            @Override
+            public Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers() {
+                return Collections.singletonMap(IdFieldMapper.NAME, new FakeMetadataMapperParser());
+            }
+        });
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> new IndicesModule(new NamedWriteableRegistry(), plugins));
+        assertThat(e.getMessage(), Matchers.containsString("already registered"));
+    }
+
+    public void testDuplicateOtherPluginMetadataMapper() {
+        MapperPlugin plugin = new MapperPlugin() {
+            @Override
+            public Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers() {
+                return Collections.singletonMap("foo", new FakeMetadataMapperParser());
+            }
+        };
+        List<MapperPlugin> plugins = Arrays.asList(plugin, plugin);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> new IndicesModule(new NamedWriteableRegistry(), plugins));
+        assertThat(e.getMessage(), Matchers.containsString("already registered"));
+    }
+
+    public void testDuplicateFieldNamesMapper() {
+        List<MapperPlugin> plugins = Arrays.asList(new MapperPlugin() {
+            @Override
+            public Map<String, MetadataFieldMapper.TypeParser> getMetadataMappers() {
+                return Collections.singletonMap(FieldNamesFieldMapper.NAME, new FakeMetadataMapperParser());
+            }
+        });
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
+            () -> new IndicesModule(new NamedWriteableRegistry(), plugins));
+        assertThat(e.getMessage(), Matchers.containsString("cannot contain metadata mapper [_field_names]"));
+    }
+
+    public void testFieldNamesIsLast() {
+        IndicesModule module = new IndicesModule(new NamedWriteableRegistry(), Collections.emptyList());
+        List<String> fieldNames = module.getMapperRegistry().getMetadataMapperParsers().keySet()
+            .stream().collect(Collectors.toList());
+        assertEquals(FieldNamesFieldMapper.NAME, fieldNames.get(fieldNames.size() - 1));
+    }
+
+    public void testFieldNamesIsLastWithPlugins() {
+        IndicesModule module = new IndicesModule(new NamedWriteableRegistry(), fakePlugins);
+        List<String> fieldNames = module.getMapperRegistry().getMetadataMapperParsers().keySet()
+            .stream().collect(Collectors.toList());
+        assertEquals(FieldNamesFieldMapper.NAME, fieldNames.get(fieldNames.size() - 1));
+    }
+}

From a07a3a9333f1dfcf8486144eea9b33b70ecd5904 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Mon, 27 Jun 2016 11:22:07 -0700
Subject: [PATCH 1079/1311] Add migration docs for MapperPlugin

---
 docs/reference/migration/migrate_5_0/plugins.asciidoc | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/docs/reference/migration/migrate_5_0/plugins.asciidoc b/docs/reference/migration/migrate_5_0/plugins.asciidoc
index 9023c84b410..8d35098deb9 100644
--- a/docs/reference/migration/migrate_5_0/plugins.asciidoc
+++ b/docs/reference/migration/migrate_5_0/plugins.asciidoc
@@ -127,3 +127,10 @@ their `onModule(ScriptModule)` implementation.
 
 Plugins that register custom analysis components should implement
 `AnalysisPlugin` and remove their `onModule(AnalysisModule)` implementation.
+
+==== MapperPlugin
+
+Plugins that register custom mappers should implement
+`MapperPlugin` and remove their `onModule(IndicesModule)` implementation.
+
+

From 3f2946ce6d13928e2bce04c1a1c1e0deef250fef Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Mon, 27 Jun 2016 11:33:22 -0700
Subject: [PATCH 1080/1311] Fix line length in new indices module tests.

---
 .../java/org/elasticsearch/indices/IndicesModuleTests.java  | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java
index 73e78650128..8587b3993cd 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java
@@ -43,14 +43,16 @@ public class IndicesModuleTests extends ESTestCase {
 
     private static class FakeMapperParser implements Mapper.TypeParser {
         @Override
-        public Mapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+        public Mapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext)
+            throws MapperParsingException {
             return null;
         }
     }
 
     private static class FakeMetadataMapperParser implements MetadataFieldMapper.TypeParser {
         @Override
-        public MetadataFieldMapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+        public MetadataFieldMapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext)
+            throws MapperParsingException {
             return null;
         }
         @Override

From eb67b3c6131d6027d9ca924a21815343e836a059 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 27 Jun 2016 12:34:10 -0400
Subject: [PATCH 1081/1311] Move the build vagrant build listener

That way it doesn't register until we actually try and set up
the vagrant test root. We don't need it all the time.
---
 qa/vagrant/build.gradle | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle
index 905710a82be..784655edaa3 100644
--- a/qa/vagrant/build.gradle
+++ b/qa/vagrant/build.gradle
@@ -170,6 +170,16 @@ task prepareTestRoot(type: Copy) {
   from configurations.test
 
   dependsOn createVersionFile, createUpgradeFromFile
+  doFirst {
+    gradle.addBuildListener new BuildAdapter() {
+      @Override
+      void buildFinished(BuildResult result) {
+        if (result.failure) {
+          println "Reproduce with: gradle packagingTest -Pvagrant.boxes=${vagrantBoxes} -Dtests.seed=${formattedSeed} -Dtests.packaging.upgrade.from.versions=${upgradeFromVersions.join(",")}"
+        }
+      }
+    }
+  }
 }
 
 task checkVagrantVersion(type: Exec) {
@@ -196,14 +206,6 @@ task packagingTest {
               "    'sample' can be used to test a single yum and apt box. 'all' can be used to\n" +
               "    test all available boxes. The available boxes are: \n" +
               "    ${availableBoxes}"
-  gradle.addBuildListener new BuildAdapter() {
-    @Override
-    void buildFinished(BuildResult result) {
-      if (result.failure) {
-        println "Reproduce with: gradle packagingTest -Pvagrant.boxes=${vagrantBoxes} -Dtests.seed=${formattedSeed} -Dtests.packaging.upgrade.from.versions=${upgradeFromVersions.join(",")}"
-      }
-    }
-  }
 }
 
 // Each box gets it own set of tasks
@@ -245,7 +247,7 @@ for (String box : availableBoxes) {
     commandLine 'vagrant', 'ssh', box, '--command',
       "set -o pipefail && ${smokeTestCommand} | sed -ue 's/^/    ${box}: /'"
   }
-  vagrantSmokeTest.dependsOn(smoke) 
+  vagrantSmokeTest.dependsOn(smoke)
 
   Task packaging = tasks.create("packagingTest${boxTask}", BatsOverVagrantTask) {
     dependsOn up

From 79fa778e33507120591a3782c017d950d2891d41 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 27 Jun 2016 15:34:36 -0400
Subject: [PATCH 1082/1311] Fix percolator tests

They need their plugin or they'll break!
---
 .../java/org/elasticsearch/test/AbstractQueryTestCase.java     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
index 6a5ea4f7afe..c3693905501 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
@@ -92,6 +92,7 @@ import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.mapper.MapperRegistry;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.script.Script.ScriptParseException;
@@ -895,7 +896,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
                         b.bind(Environment.class).toInstance(new Environment(settings));
                         b.bind(ThreadPool.class).toInstance(threadPool);
                     },
-                    settingsModule, new IndicesModule(namedWriteableRegistry, Collections.emptyList()) {
+                    settingsModule, new IndicesModule(namedWriteableRegistry, pluginsService.filterPlugins(MapperPlugin.class)) {
                         @Override
                         public void configure() {
                             // skip services

From aec033386ee2a5f43e8bfe3a486efe5e3889153d Mon Sep 17 00:00:00 2001
From: Lee Hinman <lee@writequit.org>
Date: Mon, 27 Jun 2016 13:49:10 -0600
Subject: [PATCH 1083/1311] Add the integ test node's configuration directory
 as a system property

---
 .../org/elasticsearch/gradle/test/RestIntegTestTask.groovy       | 1 +
 1 file changed, 1 insertion(+)

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
index 3bfe9d61018..fedcf6e87d3 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
@@ -62,6 +62,7 @@ public class RestIntegTestTask extends RandomizedTestingTask {
         project.gradle.projectsEvaluated {
             NodeInfo node = ClusterFormationTasks.setup(project, this, clusterConfig)
             systemProperty('tests.rest.cluster', "${-> node.httpUri()}")
+            systemProperty('tests.config.dir', "${-> node.confDir}")
             // TODO: our "client" qa tests currently use the rest-test plugin. instead they should have their own plugin
             // that sets up the test cluster and passes this transport uri instead of http uri. Until then, we pass
             // both as separate sysprops

From 2f638b5a23597967a98b1ced1deac91d64af5a44 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 27 Jun 2016 18:41:18 -0400
Subject: [PATCH 1084/1311] Keep input time unit when parsing TimeValues

This commit modifies TimeValue parsing to keep the input time unit. This
enables round-trip parsing from instances of String to instances of
TimeValue and vice-versa. With this, this commit removes support for the
unit "w" representing weeks, and also removes support for fractional
values of units (e.g., 0.5s).

Relates #19102
---
 .../resources/checkstyle_suppressions.xml     |   1 -
 .../common/io/stream/Streamable.java          |   5 +-
 .../common/io/stream/Writeable.java           |   3 +-
 .../elasticsearch/common/unit/TimeValue.java  | 123 ++++++++++++------
 .../elasticsearch/search/SearchService.java   |   2 +-
 .../search/builder/SearchSourceBuilder.java   |  24 ++--
 .../search/internal/DefaultSearchContext.java |  15 +--
 .../internal/FilteredSearchContext.java       |   9 +-
 .../search/internal/SearchContext.java        |   5 +-
 .../search/internal/SubSearchContext.java     |   3 +-
 .../search/query/QueryPhase.java              |   4 +-
 .../cluster/settings/ClusterSettingsIT.java   |   2 +-
 .../common/unit/TimeValueTests.java           |  43 +++---
 .../index/IndexingSlowLogTests.java           |   8 +-
 .../index/SearchSlowLogTests.java             |  16 +--
 .../index/mapper/ttl/TTLMappingTests.java     |   2 +-
 .../ingest/PipelineExecutionServiceTests.java |   2 +-
 .../builder/SearchSourceBuilderTests.java     |   7 +-
 .../functionscore/DecayFunctionScoreIT.java   |   4 +-
 .../versioning/SimpleVersioningIT.java        |   2 +-
 .../migration/migrate_5_0/settings.asciidoc   |   8 ++
 .../org/elasticsearch/test/ESTestCase.java    |   2 +-
 .../test/InternalTestCluster.java             |   8 +-
 .../elasticsearch/test/TestSearchContext.java |   7 +-
 24 files changed, 183 insertions(+), 122 deletions(-)

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 745c8374063..5a08a834c3c 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -323,7 +323,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]recycler[/\\]Recyclers.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]rounding[/\\]Rounding.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]unit[/\\]ByteSizeValue.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]unit[/\\]TimeValue.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]BigArrays.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]CancellableThreads.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]util[/\\]CollectionUtils.java" checks="LineLength" />
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/Streamable.java b/core/src/main/java/org/elasticsearch/common/io/stream/Streamable.java
index a37c6371482..99c054c4c78 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/Streamable.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/Streamable.java
@@ -24,10 +24,7 @@ import java.io.IOException;
 /**
  * Implementers can be written to a {@linkplain StreamOutput} and read from a {@linkplain StreamInput}. This allows them to be "thrown
  * across the wire" using Elasticsearch's internal protocol. If the implementer also implements equals and hashCode then a copy made by
- * serializing and deserializing must be equal and have the same hashCode. It isn't required that such a copy be entirely unchanged. For
- * example, {@link org.elasticsearch.common.unit.TimeValue} converts the time to nanoseconds for serialization.
- * {@linkplain org.elasticsearch.common.unit.TimeValue} actually implements {@linkplain Writeable} not {@linkplain Streamable} but it has
- * the same contract.
+ * serializing and deserializing must be equal and have the same hashCode. It isn't required that such a copy be entirely unchanged.
  *
  * Prefer implementing {@link Writeable} over implementing this interface where possible. Lots of code depends on this interface so this
  * isn't always possible.
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java b/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java
index cf127e5b968..16497533e29 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/Writeable.java
@@ -24,8 +24,7 @@ import java.io.IOException;
 /**
  * Implementers can be written to a {@linkplain StreamOutput} and read from a {@linkplain StreamInput}. This allows them to be "thrown
  * across the wire" using Elasticsearch's internal protocol. If the implementer also implements equals and hashCode then a copy made by
- * serializing and deserializing must be equal and have the same hashCode. It isn't required that such a copy be entirely unchanged. For
- * example, {@link org.elasticsearch.common.unit.TimeValue} converts the time to nanoseconds for serialization.
+ * serializing and deserializing must be equal and have the same hashCode. It isn't required that such a copy be entirely unchanged.
  *
  * Prefer implementing this interface over implementing {@link Streamable} where possible. Lots of code depends on {@linkplain Streamable}
  * so this isn't always possible.
diff --git a/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java b/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java
index 2058355d300..5a56603dad7 100644
--- a/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java
+++ b/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java
@@ -30,14 +30,50 @@ import org.joda.time.format.PeriodFormat;
 import org.joda.time.format.PeriodFormatter;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 public class TimeValue implements Writeable {
 
     /** How many nano-seconds in one milli-second */
-    public static final long NSEC_PER_MSEC = 1000000;
+    public static final long NSEC_PER_MSEC = TimeUnit.NANOSECONDS.convert(1, TimeUnit.MILLISECONDS);
+
+    private static Map<TimeUnit, Byte> TIME_UNIT_BYTE_MAP;
+    private static Map<Byte, TimeUnit> BYTE_TIME_UNIT_MAP;
+
+    static {
+        final Map<TimeUnit, Byte> timeUnitByteMap = new HashMap<>();
+        timeUnitByteMap.put(TimeUnit.NANOSECONDS, (byte)0);
+        timeUnitByteMap.put(TimeUnit.MICROSECONDS, (byte)1);
+        timeUnitByteMap.put(TimeUnit.MILLISECONDS, (byte)2);
+        timeUnitByteMap.put(TimeUnit.SECONDS, (byte)3);
+        timeUnitByteMap.put(TimeUnit.MINUTES, (byte)4);
+        timeUnitByteMap.put(TimeUnit.HOURS, (byte)5);
+        timeUnitByteMap.put(TimeUnit.DAYS, (byte)6);
+
+        final Set<Byte> bytes = new HashSet<>();
+        for (TimeUnit value : TimeUnit.values()) {
+            assert timeUnitByteMap.containsKey(value) : value;
+            assert bytes.add(timeUnitByteMap.get(value));
+        }
+
+        final Map<Byte, TimeUnit> byteTimeUnitMap = new HashMap<>();
+        for (Map.Entry<TimeUnit, Byte> entry : timeUnitByteMap.entrySet()) {
+            byteTimeUnitMap.put(entry.getValue(), entry.getKey());
+        }
+
+        TIME_UNIT_BYTE_MAP = Collections.unmodifiableMap(timeUnitByteMap);
+        BYTE_TIME_UNIT_MAP = Collections.unmodifiableMap(byteTimeUnitMap);
+    }
+
+    public static final TimeValue MINUS_ONE = timeValueMillis(-1);
+    public static final TimeValue ZERO = timeValueMillis(0);
 
     public static TimeValue timeValueNanos(long nanos) {
         return new TimeValue(nanos, TimeUnit.NANOSECONDS);
@@ -60,8 +96,19 @@ public class TimeValue implements Writeable {
     }
 
     private final long duration;
+
+    // visible for testing
+    long duration() {
+        return duration;
+    }
+
     private final TimeUnit timeUnit;
 
+    // visible for testing
+    TimeUnit timeUnit() {
+        return timeUnit;
+    }
+
     public TimeValue(long millis) {
         this(millis, TimeUnit.MILLISECONDS);
     }
@@ -76,12 +123,13 @@ public class TimeValue implements Writeable {
      */
     public TimeValue(StreamInput in) throws IOException {
         duration = in.readZLong();
-        timeUnit = TimeUnit.NANOSECONDS;
+        timeUnit = BYTE_TIME_UNIT_MAP.get(in.readByte());
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeZLong(nanos());
+        out.writeZLong(duration);
+        out.writeByte(TIME_UNIT_BYTE_MAP.get(timeUnit));
     }
 
     public long nanos() {
@@ -240,19 +288,19 @@ public class TimeValue implements Writeable {
         }
         switch (timeUnit) {
             case NANOSECONDS:
-                return Strings.format1Decimals(duration, "nanos");
+                return duration + "nanos";
             case MICROSECONDS:
-                return Strings.format1Decimals(duration, "micros");
+                return duration + "micros";
             case MILLISECONDS:
-                return Strings.format1Decimals(duration, "ms");
+                return duration + "ms";
             case SECONDS:
-                return Strings.format1Decimals(duration, "s");
+                return duration + "s";
             case MINUTES:
-                return Strings.format1Decimals(duration, "m");
+                return duration + "m";
             case HOURS:
-                return Strings.format1Decimals(duration, "h");
+                return duration + "h";
             case DAYS:
-                return Strings.format1Decimals(duration, "d");
+                return duration + "d";
             default:
                 throw new IllegalArgumentException("unknown time unit: " + timeUnit.name());
         }
@@ -270,47 +318,48 @@ public class TimeValue implements Writeable {
             return defaultValue;
         }
         try {
-            long millis;
             String lowerSValue = sValue.toLowerCase(Locale.ROOT).trim();
-            if (lowerSValue.endsWith("ms")) {
-                millis = parse(lowerSValue, 2, 1);
+            if (lowerSValue.endsWith("nanos")) {
+                return new TimeValue(parse(lowerSValue, 5), TimeUnit.NANOSECONDS);
+            } else if (lowerSValue.endsWith("micros")) {
+                return new TimeValue(parse(lowerSValue, 6), TimeUnit.MICROSECONDS);
+            } else if (lowerSValue.endsWith("ms")) {
+                return new TimeValue(parse(lowerSValue, 2), TimeUnit.MILLISECONDS);
             } else if (lowerSValue.endsWith("s")) {
-                millis = parse(lowerSValue, 1, 1000);
+                return new TimeValue(parse(lowerSValue, 1), TimeUnit.SECONDS);
             } else if (lowerSValue.endsWith("m")) {
-                millis = parse(lowerSValue, 1, 60 * 1000);
+                return new TimeValue(parse(lowerSValue, 1), TimeUnit.MINUTES);
             } else if (lowerSValue.endsWith("h")) {
-                millis = parse(lowerSValue, 1, 60 * 60 * 1000);
+                return new TimeValue(parse(lowerSValue, 1), TimeUnit.HOURS);
             } else if (lowerSValue.endsWith("d")) {
-                millis = parse(lowerSValue, 1, 24 * 60 * 60 * 1000);
-            } else if (lowerSValue.endsWith("w")) {
-                millis = parse(lowerSValue, 1, 7 * 24 * 60 * 60 * 1000);
-            } else if (lowerSValue.equals("-1")) {
-                // Allow this special value to be unit-less:
-                millis = -1;
-            } else if (lowerSValue.equals("0")) {
-                // Allow this special value to be unit-less:
-                millis = 0;
+                return new TimeValue(parse(lowerSValue, 1), TimeUnit.DAYS);
+            } else if (lowerSValue.matches("-0*1")) {
+                return TimeValue.MINUS_ONE;
+            } else if (lowerSValue.matches("0+")) {
+                return TimeValue.ZERO;
             } else {
                 // Missing units:
-                throw new ElasticsearchParseException("Failed to parse setting [{}] with value [{}] as a time value: unit is missing or unrecognized", settingName, sValue);
+                throw new ElasticsearchParseException(
+                        "failed to parse setting [{}] with value [{}] as a time value: unit is missing or unrecognized",
+                        settingName,
+                        sValue);
             }
-            return new TimeValue(millis, TimeUnit.MILLISECONDS);
         } catch (NumberFormatException e) {
-            throw new ElasticsearchParseException("Failed to parse [{}]", e, sValue);
+            throw new ElasticsearchParseException("failed to parse [{}]", e, sValue);
         }
     }
 
-    private static long parse(String s, int suffixLength, long scale) {
-        return (long) (Double.parseDouble(s.substring(0, s.length() - suffixLength)) * scale);
+    private static long parse(String s, int suffixLength) {
+        return Long.parseLong(s.substring(0, s.length() - suffixLength).trim());
     }
 
-    static final long C0 = 1L;
-    static final long C1 = C0 * 1000L;
-    static final long C2 = C1 * 1000L;
-    static final long C3 = C2 * 1000L;
-    static final long C4 = C3 * 60L;
-    static final long C5 = C4 * 60L;
-    static final long C6 = C5 * 24L;
+    private static final long C0 = 1L;
+    private static final long C1 = C0 * 1000L;
+    private static final long C2 = C1 * 1000L;
+    private static final long C3 = C2 * 1000L;
+    private static final long C4 = C3 * 60L;
+    private static final long C5 = C4 * 60L;
+    private static final long C6 = C5 * 24L;
 
     @Override
     public boolean equals(Object o) {
diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java
index 50b59631e95..96fee4b968f 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchService.java
@@ -685,7 +685,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
         if (source.profile()) {
             context.setProfilers(new Profilers(context.searcher()));
         }
-        context.timeoutInMillis(source.timeoutInMillis());
+        context.timeout(source.timeout());
         context.terminateAfter(source.terminateAfter());
         if (source.aggregations() != null) {
             try {
diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index 24278bdf127..ebf8880d4aa 100644
--- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -143,7 +143,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
 
     private Float minScore;
 
-    private long timeoutInMillis = -1;
+    private TimeValue timeout = null;
     private int terminateAfter = SearchContext.DEFAULT_TERMINATE_AFTER;
 
     private List<String> fieldNames;
@@ -241,7 +241,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         }
         suggestBuilder = in.readOptionalWriteable(SuggestBuilder::new);
         terminateAfter = in.readVInt();
-        timeoutInMillis = in.readLong();
+        timeout = in.readOptionalWriteable(TimeValue::new);
         trackScores = in.readBoolean();
         version = in.readOptionalBoolean();
         ext = in.readOptionalBytesReference();
@@ -320,7 +320,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         }
         out.writeOptionalWriteable(suggestBuilder);
         out.writeVInt(terminateAfter);
-        out.writeLong(timeoutInMillis);
+        out.writeOptionalWriteable(timeout);
         out.writeBoolean(trackScores);
         out.writeOptionalBoolean(version);
         out.writeOptionalBytesReference(ext);
@@ -446,15 +446,15 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
      * An optional timeout to control how long search is allowed to take.
      */
     public SearchSourceBuilder timeout(TimeValue timeout) {
-        this.timeoutInMillis = timeout.millis();
+        this.timeout = timeout;
         return this;
     }
 
     /**
      * Gets the timeout to control how long search is allowed to take.
      */
-    public long timeoutInMillis() {
-        return timeoutInMillis;
+    public TimeValue timeout() {
+        return timeout;
     }
 
     /**
@@ -928,7 +928,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
             rewrittenBuilder.stats = stats;
             rewrittenBuilder.suggestBuilder = suggestBuilder;
             rewrittenBuilder.terminateAfter = terminateAfter;
-            rewrittenBuilder.timeoutInMillis = timeoutInMillis;
+            rewrittenBuilder.timeout = timeout;
             rewrittenBuilder.trackScores = trackScores;
             rewrittenBuilder.version = version;
             return rewrittenBuilder;
@@ -958,7 +958,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                 } else if (context.getParseFieldMatcher().match(currentFieldName, SIZE_FIELD)) {
                     size = parser.intValue();
                 } else if (context.getParseFieldMatcher().match(currentFieldName, TIMEOUT_FIELD)) {
-                    timeoutInMillis = TimeValue.parseTimeValue(parser.text(), null, TIMEOUT_FIELD.getPreferredName()).millis();
+                    timeout = TimeValue.parseTimeValue(parser.text(), null, TIMEOUT_FIELD.getPreferredName());
                 } else if (context.getParseFieldMatcher().match(currentFieldName, TERMINATE_AFTER_FIELD)) {
                     terminateAfter = parser.intValue();
                 } else if (context.getParseFieldMatcher().match(currentFieldName, MIN_SCORE_FIELD)) {
@@ -1095,8 +1095,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
             builder.field(SIZE_FIELD.getPreferredName(), size);
         }
 
-        if (timeoutInMillis != -1) {
-            builder.field(TIMEOUT_FIELD.getPreferredName(), TimeValue.timeValueMillis(timeoutInMillis).toString());
+        if (timeout != null && !timeout.equals(TimeValue.MINUS_ONE)) {
+            builder.field(TIMEOUT_FIELD.getPreferredName(), timeout.getStringRep());
         }
 
         if (terminateAfter != SearchContext.DEFAULT_TERMINATE_AFTER) {
@@ -1341,7 +1341,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
     public int hashCode() {
         return Objects.hash(aggregations, explain, fetchSourceContext, fieldDataFields, fieldNames, from,
                 highlightBuilder, indexBoost, minScore, postQueryBuilder, queryBuilder, rescoreBuilders, scriptFields,
-                size, sorts, searchAfterBuilder, sliceBuilder, stats, suggestBuilder, terminateAfter, timeoutInMillis, trackScores, version, profile);
+                size, sorts, searchAfterBuilder, sliceBuilder, stats, suggestBuilder, terminateAfter, timeout, trackScores, version, profile);
     }
 
     @Override
@@ -1373,7 +1373,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                 && Objects.equals(stats, other.stats)
                 && Objects.equals(suggestBuilder, other.suggestBuilder)
                 && Objects.equals(terminateAfter, other.terminateAfter)
-                && Objects.equals(timeoutInMillis, other.timeoutInMillis)
+                && Objects.equals(timeout, other.timeout)
                 && Objects.equals(trackScores, other.trackScores)
                 && Objects.equals(version, other.version)
                 && Objects.equals(profile, other.profile);
diff --git a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
index 06df04db8a0..e10cafdaaf6 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/DefaultSearchContext.java
@@ -24,8 +24,8 @@ import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.Collector;
 import org.apache.lucene.search.ConstantScoreQuery;
-import org.apache.lucene.search.Query;
 import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Counter;
 import org.elasticsearch.action.search.SearchType;
@@ -99,8 +99,7 @@ public class DefaultSearchContext extends SearchContext {
     private final QuerySearchResult queryResult;
     private final FetchSearchResult fetchResult;
     private float queryBoost = 1.0f;
-    // timeout in millis
-    private long timeoutInMillis;
+    private TimeValue timeout;
     // terminate after count
     private int terminateAfter = DEFAULT_TERMINATE_AFTER;
     private List<String> groupStats;
@@ -174,7 +173,7 @@ public class DefaultSearchContext extends SearchContext {
         this.indexService = indexService;
         this.searcher = new ContextIndexSearcher(engineSearcher, indexService.cache().query(), indexShard.getQueryCachingPolicy());
         this.timeEstimateCounter = timeEstimateCounter;
-        this.timeoutInMillis = timeout.millis();
+        this.timeout = timeout;
         queryShardContext = indexService.newQueryShardContext(searcher.getIndexReader());
         queryShardContext.setTypes(request.types());
     }
@@ -512,13 +511,13 @@ public class DefaultSearchContext extends SearchContext {
     }
 
     @Override
-    public long timeoutInMillis() {
-        return timeoutInMillis;
+    public TimeValue timeout() {
+        return timeout;
     }
 
     @Override
-    public void timeoutInMillis(long timeoutInMillis) {
-        this.timeoutInMillis = timeoutInMillis;
+    public void timeout(TimeValue timeout) {
+        this.timeout = timeout;
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
index 822ac6f5422..6c646a62b6b 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
@@ -25,6 +25,7 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.util.Counter;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
@@ -264,13 +265,13 @@ public abstract class FilteredSearchContext extends SearchContext {
     }
 
     @Override
-    public long timeoutInMillis() {
-        return in.timeoutInMillis();
+    public TimeValue timeout() {
+        return in.timeout();
     }
 
     @Override
-    public void timeoutInMillis(long timeoutInMillis) {
-        in.timeoutInMillis(timeoutInMillis);
+    public void timeout(TimeValue timeout) {
+        in.timeout(timeout);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
index 5135f42ab56..7b29cabdf16 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
@@ -28,6 +28,7 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.iterable.Iterables;
 import org.elasticsearch.index.analysis.AnalysisService;
@@ -226,9 +227,9 @@ public abstract class SearchContext implements Releasable {
 
     public abstract IndexFieldDataService fieldData();
 
-    public abstract long timeoutInMillis();
+    public abstract TimeValue timeout();
 
-    public abstract void timeoutInMillis(long timeoutInMillis);
+    public abstract void timeout(TimeValue timeout);
 
     public abstract int terminateAfter();
 
diff --git a/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
index 37fb608fd0c..363b6ad076a 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
@@ -20,6 +20,7 @@ package org.elasticsearch.search.internal;
 
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.Counter;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.ParsedQuery;
 import org.elasticsearch.search.aggregations.SearchContextAggregations;
 import org.elasticsearch.search.fetch.FetchSearchResult;
@@ -155,7 +156,7 @@ public class SubSearchContext extends FilteredSearchContext {
     }
 
     @Override
-    public void timeoutInMillis(long timeoutInMillis) {
+    public void timeout(TimeValue timeout) {
         throw new UnsupportedOperationException("Not supported");
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index 4e3a642f694..22e6c40d338 100644
--- a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -349,12 +349,12 @@ public class QueryPhase implements SearchPhase {
                 }
             }
 
-            final boolean timeoutSet = searchContext.timeoutInMillis() != SearchService.NO_TIMEOUT.millis();
+            final boolean timeoutSet = searchContext.timeout() != null && !searchContext.timeout().equals(SearchService.NO_TIMEOUT);
             if (timeoutSet && collector != null) { // collector might be null if no collection is actually needed
                 final Collector child = collector;
                 // TODO: change to use our own counter that uses the scheduler in ThreadPool
                 // throws TimeLimitingCollector.TimeExceededException when timeout has reached
-                collector = Lucene.wrapTimeLimitingCollector(collector, searchContext.timeEstimateCounter(), searchContext.timeoutInMillis());
+                collector = Lucene.wrapTimeLimitingCollector(collector, searchContext.timeEstimateCounter(), searchContext.timeout().millis());
                 if (doProfile) {
                     collector = new InternalProfileCollector(collector, CollectorResult.REASON_SEARCH_TIMEOUT,
                             Collections.singletonList((InternalProfileCollector) child));
diff --git a/core/src/test/java/org/elasticsearch/cluster/settings/ClusterSettingsIT.java b/core/src/test/java/org/elasticsearch/cluster/settings/ClusterSettingsIT.java
index 1c25659d2cf..7f423d1bb9e 100644
--- a/core/src/test/java/org/elasticsearch/cluster/settings/ClusterSettingsIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/settings/ClusterSettingsIT.java
@@ -263,7 +263,7 @@ public class ClusterSettingsIT extends ESIntegTestCase {
                     .get();
             fail("bogus value");
         } catch (IllegalArgumentException ex) {
-            assertEquals(ex.getMessage(), "Failed to parse setting [discovery.zen.publish_timeout] with value [whatever] as a time value: unit is missing or unrecognized");
+            assertEquals(ex.getMessage(), "failed to parse setting [discovery.zen.publish_timeout] with value [whatever] as a time value: unit is missing or unrecognized");
         }
 
         assertThat(discoverySettings.getPublishTimeout().seconds(), equalTo(1L));
diff --git a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
index cc36625e68f..9b73f2f99af 100644
--- a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
+++ b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
@@ -85,9 +85,6 @@ public class TimeValueTests extends ESTestCase {
         assertEquals(new TimeValue(10, TimeUnit.SECONDS),
                      TimeValue.parseTimeValue("10S", null, "test"));
 
-        assertEquals(new TimeValue(100, TimeUnit.MILLISECONDS),
-                    TimeValue.parseTimeValue("0.1s", null, "test"));
-
         assertEquals(new TimeValue(10, TimeUnit.MINUTES),
                      TimeValue.parseTimeValue("10 m", null, "test"));
         assertEquals(new TimeValue(10, TimeUnit.MINUTES),
@@ -115,14 +112,17 @@ public class TimeValueTests extends ESTestCase {
         assertEquals(new TimeValue(10, TimeUnit.DAYS),
                      TimeValue.parseTimeValue("10D", null, "test"));
 
-        assertEquals(new TimeValue(70, TimeUnit.DAYS),
-                     TimeValue.parseTimeValue("10 w", null, "test"));
-        assertEquals(new TimeValue(70, TimeUnit.DAYS),
-                     TimeValue.parseTimeValue("10w", null, "test"));
-        assertEquals(new TimeValue(70, TimeUnit.DAYS),
-                     TimeValue.parseTimeValue("10 W", null, "test"));
-        assertEquals(new TimeValue(70, TimeUnit.DAYS),
-                     TimeValue.parseTimeValue("10W", null, "test"));
+        final int length = randomIntBetween(0, 8);
+        final String zeros = new String(new char[length]).replace('\0', '0');
+        assertTrue(TimeValue.parseTimeValue("-" + zeros + "1", null, "test") == TimeValue.MINUS_ONE);
+        assertTrue(TimeValue.parseTimeValue(zeros + "0", null, "test") == TimeValue.ZERO);
+    }
+
+    public void testRoundTrip() {
+        final String s = randomTimeValue();
+        assertThat(TimeValue.parseTimeValue(s, null, "test").getStringRep(), equalTo(s));
+        final TimeValue t = new TimeValue(randomIntBetween(1, 128), randomFrom(TimeUnit.values()));
+        assertThat(TimeValue.parseTimeValue(t.getStringRep(), null, "test"), equalTo(t));
     }
 
     private void assertEqualityAfterSerialize(TimeValue value, int expectedSize) throws IOException {
@@ -134,13 +134,20 @@ public class TimeValueTests extends ESTestCase {
         TimeValue inValue = new TimeValue(in);
 
         assertThat(inValue, equalTo(value));
+        assertThat(inValue.duration(), equalTo(value.duration()));
+        assertThat(inValue.timeUnit(), equalTo(value.timeUnit()));
     }
 
     public void testSerialize() throws Exception {
-        assertEqualityAfterSerialize(new TimeValue(100, TimeUnit.DAYS), 8);
-        assertEqualityAfterSerialize(timeValueNanos(-1), 1);
-        assertEqualityAfterSerialize(timeValueNanos(1), 1);
-        assertEqualityAfterSerialize(timeValueSeconds(30), 6);
+        assertEqualityAfterSerialize(new TimeValue(100, TimeUnit.DAYS), 3);
+        assertEqualityAfterSerialize(timeValueNanos(-1), 2);
+        assertEqualityAfterSerialize(timeValueNanos(1), 2);
+        assertEqualityAfterSerialize(timeValueSeconds(30), 2);
+
+        final TimeValue timeValue = new TimeValue(randomIntBetween(0, 1024), randomFrom(TimeUnit.values()));
+        BytesStreamOutput out = new BytesStreamOutput();
+        out.writeZLong(timeValue.duration());
+        assertEqualityAfterSerialize(timeValue, 1 + out.bytes().length());
     }
 
     public void testFailOnUnknownUnits() {
@@ -148,7 +155,7 @@ public class TimeValueTests extends ESTestCase {
             TimeValue.parseTimeValue("23tw", null, "test");
             fail("Expected ElasticsearchParseException");
         } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), containsString("Failed to parse"));
+            assertThat(e.getMessage(), containsString("failed to parse"));
         }
     }
 
@@ -157,7 +164,7 @@ public class TimeValueTests extends ESTestCase {
             TimeValue.parseTimeValue("42", null, "test");
             fail("Expected ElasticsearchParseException");
         } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), containsString("Failed to parse"));
+            assertThat(e.getMessage(), containsString("failed to parse"));
         }
     }
 
@@ -166,7 +173,7 @@ public class TimeValueTests extends ESTestCase {
             TimeValue.parseTimeValue("42ms.", null, "test");
             fail("Expected ElasticsearchParseException");
         } catch (ElasticsearchParseException e) {
-            assertThat(e.getMessage(), containsString("Failed to parse"));
+            assertThat(e.getMessage(), containsString("failed to parse"));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java b/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
index a076224eae5..8951123d0da 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java
@@ -176,28 +176,28 @@ public class IndexingSlowLogTests extends ESTestCase {
             settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_TRACE_SETTING.getKey(), "NOT A TIME VALUE").build()));
             fail();
         } catch (IllegalArgumentException ex) {
-            assertEquals(ex.getMessage(), "Failed to parse setting [index.indexing.slowlog.threshold.index.trace] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
+            assertEquals(ex.getMessage(), "failed to parse setting [index.indexing.slowlog.threshold.index.trace] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
         }
 
         try {
             settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_DEBUG_SETTING.getKey(), "NOT A TIME VALUE").build()));
             fail();
         } catch (IllegalArgumentException ex) {
-            assertEquals(ex.getMessage(), "Failed to parse setting [index.indexing.slowlog.threshold.index.debug] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
+            assertEquals(ex.getMessage(), "failed to parse setting [index.indexing.slowlog.threshold.index.debug] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
         }
 
         try {
             settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_INFO_SETTING.getKey(), "NOT A TIME VALUE").build()));
             fail();
         } catch (IllegalArgumentException ex) {
-            assertEquals(ex.getMessage(), "Failed to parse setting [index.indexing.slowlog.threshold.index.info] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
+            assertEquals(ex.getMessage(), "failed to parse setting [index.indexing.slowlog.threshold.index.info] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
         }
 
         try {
             settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_WARN_SETTING.getKey(), "NOT A TIME VALUE").build()));
             fail();
         } catch (IllegalArgumentException ex) {
-            assertEquals(ex.getMessage(), "Failed to parse setting [index.indexing.slowlog.threshold.index.warn] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
+            assertEquals(ex.getMessage(), "failed to parse setting [index.indexing.slowlog.threshold.index.warn] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java b/core/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java
index 04291d957a8..d2bffb0f749 100644
--- a/core/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java
+++ b/core/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java
@@ -249,28 +249,28 @@ public class SearchSlowLogTests extends ESSingleNodeTestCase {
             settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_TRACE_SETTING.getKey(), "NOT A TIME VALUE").build()));
             fail();
         } catch (IllegalArgumentException ex) {
-            assertEquals(ex.getMessage(), "Failed to parse setting [index.search.slowlog.threshold.query.trace] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
+            assertEquals(ex.getMessage(), "failed to parse setting [index.search.slowlog.threshold.query.trace] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
         }
 
         try {
             settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_DEBUG_SETTING.getKey(), "NOT A TIME VALUE").build()));
             fail();
         } catch (IllegalArgumentException ex) {
-            assertEquals(ex.getMessage(), "Failed to parse setting [index.search.slowlog.threshold.query.debug] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
+            assertEquals(ex.getMessage(), "failed to parse setting [index.search.slowlog.threshold.query.debug] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
         }
 
         try {
             settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_INFO_SETTING.getKey(), "NOT A TIME VALUE").build()));
             fail();
         } catch (IllegalArgumentException ex) {
-            assertEquals(ex.getMessage(), "Failed to parse setting [index.search.slowlog.threshold.query.info] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
+            assertEquals(ex.getMessage(), "failed to parse setting [index.search.slowlog.threshold.query.info] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
         }
 
         try {
             settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN_SETTING.getKey(), "NOT A TIME VALUE").build()));
             fail();
         } catch (IllegalArgumentException ex) {
-            assertEquals(ex.getMessage(), "Failed to parse setting [index.search.slowlog.threshold.query.warn] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
+            assertEquals(ex.getMessage(), "failed to parse setting [index.search.slowlog.threshold.query.warn] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
         }
     }
 
@@ -320,28 +320,28 @@ public class SearchSlowLogTests extends ESSingleNodeTestCase {
             settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_TRACE_SETTING.getKey(), "NOT A TIME VALUE").build()));
             fail();
         } catch (IllegalArgumentException ex) {
-            assertEquals(ex.getMessage(), "Failed to parse setting [index.search.slowlog.threshold.fetch.trace] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
+            assertEquals(ex.getMessage(), "failed to parse setting [index.search.slowlog.threshold.fetch.trace] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
         }
 
         try {
             settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING.getKey(), "NOT A TIME VALUE").build()));
             fail();
         } catch (IllegalArgumentException ex) {
-            assertEquals(ex.getMessage(), "Failed to parse setting [index.search.slowlog.threshold.fetch.debug] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
+            assertEquals(ex.getMessage(), "failed to parse setting [index.search.slowlog.threshold.fetch.debug] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
         }
 
         try {
             settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO_SETTING.getKey(), "NOT A TIME VALUE").build()));
             fail();
         } catch (IllegalArgumentException ex) {
-            assertEquals(ex.getMessage(), "Failed to parse setting [index.search.slowlog.threshold.fetch.info] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
+            assertEquals(ex.getMessage(), "failed to parse setting [index.search.slowlog.threshold.fetch.info] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
         }
 
         try {
             settings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN_SETTING.getKey(), "NOT A TIME VALUE").build()));
             fail();
         } catch (IllegalArgumentException ex) {
-            assertEquals(ex.getMessage(), "Failed to parse setting [index.search.slowlog.threshold.fetch.warn] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
+            assertEquals(ex.getMessage(), "failed to parse setting [index.search.slowlog.threshold.fetch.warn] with value [NOT A TIME VALUE] as a time value: unit is missing or unrecognized");
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java
index ad9e03327b2..620968ddbe8 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java
@@ -139,7 +139,7 @@ public class TTLMappingTests extends ESSingleNodeTestCase {
 
         String updatedMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
                 .startObject("_ttl")
-                .field("default", "1w")
+                .field("default", "7d")
                 .endObject()
                 .startObject("properties").field("field").startObject().field("type", "text").endObject().endObject()
                 .endObject().endObject().string();
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
index d001a829443..8bf6f77a026 100644
--- a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
@@ -140,7 +140,7 @@ public class PipelineExecutionServiceTests extends ESTestCase {
             IngestDocument ingestDocument = (IngestDocument) invocationOnMock.getArguments()[0];
             for (IngestDocument.MetaData metaData : IngestDocument.MetaData.values()) {
                 if (metaData == IngestDocument.MetaData.TTL) {
-                    ingestDocument.setFieldValue(IngestDocument.MetaData.TTL.getFieldName(), "5w");
+                    ingestDocument.setFieldValue(IngestDocument.MetaData.TTL.getFieldName(), "35d");
                 } else {
                     ingestDocument.setFieldValue(metaData.getFieldName(), "update" + metaData.getFieldName());
                 }
diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index 9d66669d07c..f3a78b65d78 100644
--- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -86,7 +86,6 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
 import static org.elasticsearch.test.ClusterServiceUtils.setState;
@@ -208,7 +207,7 @@ public class SearchSourceBuilderTests extends ESTestCase {
             builder.minScore(randomFloat() * 1000);
         }
         if (randomBoolean()) {
-            builder.timeout(new TimeValue(randomIntBetween(1, 100), randomFrom(TimeUnit.values())));
+            builder.timeout(TimeValue.parseTimeValue(randomTimeValue(), null, "timeout"));
         }
         if (randomBoolean()) {
             builder.terminateAfter(randomIntBetween(1, 100000));
@@ -456,7 +455,7 @@ public class SearchSourceBuilderTests extends ESTestCase {
 
     public void testEqualsAndHashcode() throws IOException {
         SearchSourceBuilder firstBuilder = createSearchSourceBuilder();
-        assertFalse("source builder is equal to null", firstBuilder.equals(null));
+        assertNotNull("source builder is equal to null", firstBuilder);
         assertFalse("source builder is equal to incompatible type", firstBuilder.equals(""));
         assertTrue("source builder is not equal to self", firstBuilder.equals(firstBuilder));
         assertThat("same source builder's hashcode returns different values if called multiple times", firstBuilder.hashCode(),
@@ -601,7 +600,7 @@ public class SearchSourceBuilderTests extends ESTestCase {
         final String query = "{ \"query\": { \"match_all\": {}}, \"timeout\": \"" + timeout + "\"}";
         try (XContentParser parser = XContentFactory.xContent(query).createParser(query)) {
             final SearchSourceBuilder builder = SearchSourceBuilder.fromXContent(createParseContext(parser), aggParsers, suggesters);
-            assertThat(builder.timeoutInMillis(), equalTo(TimeValue.parseTimeValue(timeout, null, "timeout").millis()));
+            assertThat(builder.timeout(), equalTo(TimeValue.parseTimeValue(timeout, null, "timeout")));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java
index 1418f476e00..155363f72a7 100644
--- a/core/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java
+++ b/core/src/test/java/org/elasticsearch/search/functionscore/DecayFunctionScoreIT.java
@@ -594,9 +594,9 @@ public class DecayFunctionScoreIT extends ESIntegTestCase {
                 searchRequest().searchType(SearchType.QUERY_THEN_FETCH).source(
                         searchSource().query(
                                 functionScoreQuery(QueryBuilders.matchAllQuery(), new FilterFunctionBuilder[]{
-                                        new FilterFunctionBuilder(linearDecayFunction("num1", null, "1000w")),
+                                        new FilterFunctionBuilder(linearDecayFunction("num1", null, "7000d")),
                                         new FilterFunctionBuilder(gaussDecayFunction("num1", null, "1d")),
-                                        new FilterFunctionBuilder(exponentialDecayFunction("num1", null, "1000w"))
+                                        new FilterFunctionBuilder(exponentialDecayFunction("num1", null, "7000d"))
                                 }).scoreMode(FiltersFunctionScoreQuery.ScoreMode.MULTIPLY))));
 
         SearchResponse sr = response.actionGet();
diff --git a/core/src/test/java/org/elasticsearch/versioning/SimpleVersioningIT.java b/core/src/test/java/org/elasticsearch/versioning/SimpleVersioningIT.java
index 2356395219f..74b910bf2b4 100644
--- a/core/src/test/java/org/elasticsearch/versioning/SimpleVersioningIT.java
+++ b/core/src/test/java/org/elasticsearch/versioning/SimpleVersioningIT.java
@@ -220,7 +220,7 @@ public class SimpleVersioningIT extends ESIntegTestCase {
             fail("did not hit expected exception");
         } catch (IllegalArgumentException iae) {
             // expected
-            assertTrue(iae.getMessage().contains("Failed to parse setting [index.gc_deletes] with value [42] as a time value: unit is missing or unrecognized"));
+            assertTrue(iae.getMessage().contains("failed to parse setting [index.gc_deletes] with value [42] as a time value: unit is missing or unrecognized"));
         }
     }
 
diff --git a/docs/reference/migration/migrate_5_0/settings.asciidoc b/docs/reference/migration/migrate_5_0/settings.asciidoc
index be578b00c3b..8a7c386acdb 100644
--- a/docs/reference/migration/migrate_5_0/settings.asciidoc
+++ b/docs/reference/migration/migrate_5_0/settings.asciidoc
@@ -286,3 +286,11 @@ The setting `bootstrap.mlockall` has been renamed to
 The default setting `include_global_state` for restoring snapshots has been
 changed from `true` to `false`. It has not been changed for taking snapshots and
 still defaults to `true` in that case.
+
+==== Time value parsing
+
+The unit 'w' representing weeks is no longer supported.
+
+Fractional time values (e.g., 0.5s) are no longer supported. For
+example, this means when setting timeouts "0.5s" will be rejected and
+should instead be input as "500ms".
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index 860f133a453..d04d12304de 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -413,7 +413,7 @@ public abstract class ESTestCase extends LuceneTestCase {
         return generateRandomStringArray(maxArraySize, maxStringSize, allowNull, true);
     }
 
-    private static String[] TIME_SUFFIXES = new String[]{"d", "H", "ms", "s", "S", "w"};
+    private static String[] TIME_SUFFIXES = new String[]{"d", "h", "ms", "s", "m"};
 
     private static String randomTimeValue(int lower, int upper) {
         return randomIntBetween(lower, upper) + randomFrom(TIME_SUFFIXES);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index 025740dbe36..9a00fbc4557 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -410,12 +410,12 @@ public final class InternalTestCluster extends TestCluster {
             builder.put("cache.recycler.page.type", RandomPicks.randomFrom(random, PageCacheRecycler.Type.values()));
         }
         if (random.nextInt(10) == 0) { // 10% of the nodes have a very frequent check interval
-            builder.put(SearchService.KEEPALIVE_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(10 + random.nextInt(2000)));
+            builder.put(SearchService.KEEPALIVE_INTERVAL_SETTING.getKey(), TimeValue.timeValueMillis(10 + random.nextInt(2000)).getStringRep());
         } else if (random.nextInt(10) != 0) { // 90% of the time - 10% of the time we don't set anything
-            builder.put(SearchService.KEEPALIVE_INTERVAL_SETTING.getKey(), TimeValue.timeValueSeconds(10 + random.nextInt(5 * 60)));
+            builder.put(SearchService.KEEPALIVE_INTERVAL_SETTING.getKey(), TimeValue.timeValueSeconds(10 + random.nextInt(5 * 60)).getStringRep());
         }
         if (random.nextBoolean()) { // sometimes set a
-            builder.put(SearchService.DEFAULT_KEEPALIVE_SETTING.getKey(), TimeValue.timeValueSeconds(100 + random.nextInt(5 * 60)));
+            builder.put(SearchService.DEFAULT_KEEPALIVE_SETTING.getKey(), TimeValue.timeValueSeconds(100 + random.nextInt(5 * 60)).getStringRep());
         }
 
         builder.put(EsExecutors.PROCESSORS_SETTING.getKey(), 1 + random.nextInt(3));
@@ -469,7 +469,7 @@ public final class InternalTestCluster extends TestCluster {
             builder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getKey(), RandomInts.randomIntBetween(random, 0, 2000));
         }
         if (random.nextBoolean()) {
-            builder.put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getKey(), TimeValue.timeValueMillis(RandomInts.randomIntBetween(random, 750, 10000000)));
+            builder.put(ScriptService.SCRIPT_CACHE_EXPIRE_SETTING.getKey(), TimeValue.timeValueMillis(RandomInts.randomIntBetween(random, 750, 10000000)).getStringRep());
         }
 
         return builder.build();
diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
index 8122abe483d..97240bd9e5e 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java
@@ -28,6 +28,7 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.util.Counter;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.analysis.AnalysisService;
@@ -311,12 +312,12 @@ public class TestSearchContext extends SearchContext {
     }
 
     @Override
-    public long timeoutInMillis() {
-        return 0;
+    public TimeValue timeout() {
+        return TimeValue.ZERO;
     }
 
     @Override
-    public void timeoutInMillis(long timeoutInMillis) {
+    public void timeout(TimeValue timeout) {
     }
 
     @Override

From 0ec07833a8f7fc9b8a2928444f2809514d8330ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Honza=20Kr=C3=A1l?= <Honza.Kral@gmail.com>
Date: Tue, 28 Jun 2016 01:54:49 +0200
Subject: [PATCH 1085/1311] [TEST] refactor search yaml tests (#19109)

---
 .../test/search/10_source_filtering.yaml      | 33 +++++++++++++++----
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml
index e0ac2aea2df..84bf44f7392 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml
@@ -1,6 +1,5 @@
 ---
-"Source filtering":
-
+setup:
   - do:
       index:
         index:  test_1
@@ -10,40 +9,54 @@
   - do:
       indices.refresh: {}
 
+---
+"_source: true":
 
   - do:
       search:
-      # stringified for boolean value
         body: { _source: true, query: { match_all: {} } }
 
   - length:   { hits.hits: 1  }
   - match: { hits.hits.0._source.count: 1 }
 
+---
+"_source: false":
   - do: { search: { body: { _source: false, query: { match_all: {} } } } }
   - length:   { hits.hits: 1  }
   - is_false: hits.hits.0._source
 
+---
+"no filtering":
   - do: { search: { body: { query: { match_all: {} } } } }
   - length:   { hits.hits: 1  }
   - match: { hits.hits.0._source.count: 1 }
 
+---
+"_source in body":
   - do: { search: { body: { _source: include.field1, query: { match_all: {} } } } }
   - match:  { hits.hits.0._source.include.field1: v1 }
   - is_false: hits.hits.0._source.include.field2
 
+---
+"_source include and _source in body":
   - do: { search: { _source_include: include.field1, body: { _source: include.field2, query: { match_all: {} } } } }
   - match:  { hits.hits.0._source.include.field1: v1 }
   - is_false: hits.hits.0._source.include.field2
 
+---
+"_source_include":
   - do: { search: { _source_include: include.field1, body: { query: { match_all: {} } } } }
   - match:  { hits.hits.0._source.include.field1: v1 }
   - is_false: hits.hits.0._source.include.field2
 
+---
+"_source_exclude":
   - do: { search: { _source_exclude: count, body: { query: { match_all: {} } } } }
   - match:  { hits.hits.0._source.include: { field1 : v1 , field2: v2 }}
   - is_false: hits.hits.0._source.count
 
-
+---
+"_source field1 field2":
   - do:
       search:
         body:
@@ -53,6 +66,8 @@
   - match:  { hits.hits.0._source.include.field2: v2 }
   - is_false: hits.hits.0._source.count
 
+---
+"_source.include field1 field2":
   - do:
       search:
         body:
@@ -63,6 +78,8 @@
   - match:  { hits.hits.0._source.include.field2: v2 }
   - is_false: hits.hits.0._source.count
 
+---
+"_source includes and excludes":
   - do:
       search:
         body:
@@ -73,7 +90,8 @@
   - match:  { hits.hits.0._source.include.field1: v1 }
   - is_false:  hits.hits.0._source.include.field2
 
-
+---
+"fields in body":
   - do:
       search:
         body:
@@ -81,6 +99,8 @@
           query: { match_all: {} }
   - is_false:  hits.hits.0._source
 
+---
+"fields in body with source":
   - do:
         search:
           body:
@@ -89,7 +109,8 @@
   - match: { hits.hits.0._source.include.field2: v2 }
   - is_true:  hits.hits.0._source
 
-
+---
+"fielddata_fields":
   - do:
       search:
         fielddata_fields: [ "count" ]

From f79851e23a32c31efbb9342a84ef3e50feb061bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Honza=20Kr=C3=A1l?= <Honza.Kral@gmail.com>
Date: Tue, 28 Jun 2016 02:07:46 +0200
Subject: [PATCH 1086/1311] [API] separate tasks.list and tasks.get APIs in the
 json definition (#19107)

---
 .../rest-api-spec/api/{task.get.json => tasks.get.json}         | 2 +-
 .../src/main/resources/rest-api-spec/api/tasks.list.json        | 2 +-
 .../rest-api-spec/test/{task.get => tasks.get}/10_basic.yaml    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)
 rename rest-api-spec/src/main/resources/rest-api-spec/api/{task.get.json => tasks.get.json} (97%)
 rename rest-api-spec/src/main/resources/rest-api-spec/test/{task.get => tasks.get}/10_basic.yaml (92%)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/task.get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.get.json
similarity index 97%
rename from rest-api-spec/src/main/resources/rest-api-spec/api/task.get.json
rename to rest-api-spec/src/main/resources/rest-api-spec/api/tasks.get.json
index 8024f015e96..f97206cd16f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/task.get.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.get.json
@@ -1,5 +1,5 @@
 {
-  "task.get": {
+  "tasks.get": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html",
     "methods": ["GET"],
     "url": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json
index a1913fbfc17..a966cb0e507 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json
@@ -4,7 +4,7 @@
     "methods": ["GET"],
     "url": {
       "path": "/_tasks",
-      "paths": ["/_tasks", "/_tasks/{task_id}"],
+      "paths": ["/_tasks"],
       "parts": {},
       "params": {
         "node_id": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/task.get/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/tasks.get/10_basic.yaml
similarity index 92%
rename from rest-api-spec/src/main/resources/rest-api-spec/test/task.get/10_basic.yaml
rename to rest-api-spec/src/main/resources/rest-api-spec/test/tasks.get/10_basic.yaml
index ba90e1541fe..48d9f46ac73 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/task.get/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/tasks.get/10_basic.yaml
@@ -6,5 +6,5 @@
 
   - do:
       catch: missing
-      task.get:
+      tasks.get:
         task_id: foo:1

From af989c0780e9b0c289fa7b96e55ffbc4b22a5524 Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Tue, 28 Jun 2016 08:50:50 +0200
Subject: [PATCH 1087/1311] Support new Asia Pacific (Mumbai) ap-south-1 AWS
 region

AWS [announced](http://www.allthingsdistributed.com/2016/06/introducing-aws-asia-pacific-mumbai-region.html) a new region: Asia Pacific (Mumbai)	`ap-south-1`.

We need to support it for:

* repository-s3: s3.ap-south-1.amazonaws.com or s3-ap-south-1.amazonaws.com
* discovery-ec2: ec2.ap-south-1.amazonaws.com

For reference: http://docs.aws.amazon.com/general/latest/gr/rande.html

Closes #19110.
---
 docs/plugins/discovery-ec2.asciidoc           | 25 +++++++++----------
 docs/plugins/repository-s3.asciidoc           | 25 +++++++++----------
 .../cloud/aws/AwsEc2ServiceImpl.java          |  2 ++
 .../cloud/aws/InternalAwsS3Service.java       |  2 ++
 4 files changed, 28 insertions(+), 26 deletions(-)

diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc
index 17659d496b8..224080c522c 100644
--- a/docs/plugins/discovery-ec2.asciidoc
+++ b/docs/plugins/discovery-ec2.asciidoc
@@ -105,19 +105,18 @@ cloud:
 The `cloud.aws.region` can be set to a region and will automatically use the relevant settings for both `ec2` and `s3`.
 The available values are:
 
-* `us-east` (`us-east-1`)
-* `us-west` (`us-west-1`)
-* `us-west-1`
-* `us-west-2`
-* `ap-southeast` (`ap-southeast-1`)
-* `ap-southeast-1`
-* `ap-southeast-2`
-* `ap-northeast` (`ap-northeast-1`)
-* `ap-northeast-2` (`ap-northeast-2`)
-* `eu-west` (`eu-west-1`)
-* `eu-central` (`eu-central-1`)
-* `sa-east` (`sa-east-1`)
-* `cn-north` (`cn-north-1`)
+* `us-east` (`us-east-1`) for US East (N. Virginia)
+* `us-west` (`us-west-1`) for US West (N. California)
+* `us-west-2` for US West (Oregon)
+* `ap-south-1` for Asia Pacific (Mumbai)
+* `ap-southeast` (`ap-southeast-1`) for Asia Pacific (Singapore)
+* `ap-southeast-2` for Asia Pacific (Sydney)
+* `ap-northeast` (`ap-northeast-1`) for Asia Pacific (Tokyo)
+* `ap-northeast-2` (`ap-northeast-2`) for Asia Pacific (Seoul)
+* `eu-west` (`eu-west-1`) for EU (Ireland)
+* `eu-central` (`eu-central-1`) for EU (Frankfurt)
+* `sa-east` (`sa-east-1`) for South America (São Paulo)
+* `cn-north` (`cn-north-1`) for China (Beijing)
 
 [[discovery-ec2-usage-signer]]
 ===== EC2 Signer API
diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc
index 63e3ad311b2..e1f07f6c8e2 100644
--- a/docs/plugins/repository-s3.asciidoc
+++ b/docs/plugins/repository-s3.asciidoc
@@ -110,19 +110,18 @@ The `cloud.aws.region` can be set to a region and will automatically use the rel
 You can specifically set it for s3 only using `cloud.aws.s3.region`.
 The available values are:
 
-* `us-east` (`us-east-1`)
-* `us-west` (`us-west-1`)
-* `us-west-1`
-* `us-west-2`
-* `ap-southeast` (`ap-southeast-1`)
-* `ap-southeast-1`
-* `ap-southeast-2`
-* `ap-northeast` (`ap-northeast-1`)
-* `ap-northeast-2` (`ap-northeast-2`)
-* `eu-west` (`eu-west-1`)
-* `eu-central` (`eu-central-1`)
-* `sa-east` (`sa-east-1`)
-* `cn-north` (`cn-north-1`)
+* `us-east` (`us-east-1`) for US East (N. Virginia)
+* `us-west` (`us-west-1`) for US West (N. California)
+* `us-west-2` for US West (Oregon)
+* `ap-south-1` for Asia Pacific (Mumbai)
+* `ap-southeast` (`ap-southeast-1`) for Asia Pacific (Singapore)
+* `ap-southeast-2` for Asia Pacific (Sydney)
+* `ap-northeast` (`ap-northeast-1`) for Asia Pacific (Tokyo)
+* `ap-northeast-2` (`ap-northeast-2`) for Asia Pacific (Seoul)
+* `eu-west` (`eu-west-1`) for EU (Ireland)
+* `eu-central` (`eu-central-1`) for EU (Frankfurt)
+* `sa-east` (`sa-east-1`) for South America (São Paulo)
+* `cn-north` (`cn-north-1`) for China (Beijing)
 
 [[repository-s3-usage-signer]]
 ===== S3 Signer API
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java
index 2aec30d6ddb..5ce110487c4 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java
@@ -148,6 +148,8 @@ public class AwsEc2ServiceImpl extends AbstractLifecycleComponent<AwsEc2Service>
                 endpoint = "ec2.ap-southeast-1.amazonaws.com";
             } else if (region.equals("us-gov-west") || region.equals("us-gov-west-1")) {
                 endpoint = "ec2.us-gov-west-1.amazonaws.com";
+            } else if (region.equals("ap-south-1")) {
+                endpoint = "ec2.ap-south-1.amazonaws.com";
             } else if (region.equals("ap-southeast-2")) {
                 endpoint = "ec2.ap-southeast-2.amazonaws.com";
             } else if (region.equals("ap-northeast") || region.equals("ap-northeast-1")) {
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
index 352aa196b06..287973a3a90 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
@@ -155,6 +155,8 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Servic
             return "s3-us-west-1.amazonaws.com";
         } else if ("us-west-2".equals(region)) {
             return "s3-us-west-2.amazonaws.com";
+        } else if (region.equals("ap-south-1")) {
+            return "s3-ap-south-1.amazonaws.com";
         } else if ("ap-southeast".equals(region) || "ap-southeast-1".equals(region)) {
             return "s3-ap-southeast-1.amazonaws.com";
         } else if ("ap-southeast-2".equals(region)) {

From ab8ff8909bc2401e7928ce851e74fa3e25bb7a48 Mon Sep 17 00:00:00 2001
From: Alexander Reelsen <alexander@reelsen.net>
Date: Tue, 28 Jun 2016 09:13:19 +0200
Subject: [PATCH 1088/1311] Tests: Rename task.get to tasks.get

The task.get action got renamed to tasks.get, some tests
did not change this.

Relates #19107
---
 .../rest-api-spec/test/delete_by_query/10_basic.yaml          | 2 +-
 .../rest-api-spec/test/delete_by_query/70_throttle.yaml       | 4 ++--
 .../test/resources/rest-api-spec/test/reindex/10_basic.yaml   | 2 +-
 .../resources/rest-api-spec/test/reindex/70_throttle.yaml     | 4 ++--
 .../rest-api-spec/test/update_by_query/10_basic.yaml          | 2 +-
 .../rest-api-spec/test/update_by_query/60_throttle.yaml       | 4 ++--
 6 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml
index 85cd6143d69..041aa127cd4 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/10_basic.yaml
@@ -69,7 +69,7 @@
   - is_false: deleted
 
   - do:
-      task.get:
+      tasks.get:
         wait_for_completion: true
         task_id: $task
   - is_false: node_failures
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yaml
index 65a22781550..96cfaa42b5a 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/70_throttle.yaml
@@ -134,7 +134,7 @@
         task_id: $task
 
   - do:
-      task.get:
+      tasks.get:
         wait_for_completion: true
         task_id: $task
 
@@ -197,6 +197,6 @@
         task_id: $task
 
   - do:
-      task.get:
+      tasks.get:
         wait_for_completion: true
         task_id: $task
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml
index addbebd44a7..a567ca67bfa 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/10_basic.yaml
@@ -93,7 +93,7 @@
   - is_false: deleted
 
   - do:
-      task.get:
+      tasks.get:
         wait_for_completion: true
         task_id: $task
   - is_false: node_failures
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yaml
index 73e1a3a3a94..05d7668ed2e 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/70_throttle.yaml
@@ -156,7 +156,7 @@
         task_id: $task
 
   - do:
-      task.get:
+      tasks.get:
         wait_for_completion: true
         task_id: $task
 
@@ -214,6 +214,6 @@
         task_id: $task
 
   - do:
-      task.get:
+      tasks.get:
         wait_for_completion: true
         task_id: $task
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yaml
index 62c8677921d..17b2dc77816 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/10_basic.yaml
@@ -53,7 +53,7 @@
   - is_false: deleted
 
   - do:
-      task.get:
+      tasks.get:
         wait_for_completion: true
         task_id: $task
   - is_false: node_failures
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yaml
index e13e29bc3f5..7ecf7000bfd 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/60_throttle.yaml
@@ -122,7 +122,7 @@
         task_id: $task
 
   - do:
-      task.get:
+      tasks.get:
         wait_for_completion: true
         task_id: $task
 
@@ -172,6 +172,6 @@
         task_id: $task
 
   - do:
-      task.get:
+      tasks.get:
         wait_for_completion: true
         task_id: $task

From c557663b90dc3ac3f0ae7391cb6eac543812a946 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Fri, 24 Jun 2016 16:51:49 +0200
Subject: [PATCH 1089/1311] Make discovery-azure work again

The discovery-plugin has been broken since 2.x because the code was not compliant with the security manager and because plugins have been refactored.

closes #18637, #15630
---
 docs/plugins/discovery-azure.asciidoc         |  4 +-
 .../management/AzureComputeServiceImpl.java   | 46 +++++++++++--------
 2 files changed, 31 insertions(+), 19 deletions(-)

diff --git a/docs/plugins/discovery-azure.asciidoc b/docs/plugins/discovery-azure.asciidoc
index e3bdc83e470..825c258133b 100644
--- a/docs/plugins/discovery-azure.asciidoc
+++ b/docs/plugins/discovery-azure.asciidoc
@@ -56,9 +56,11 @@ discovery:
 .Binding the network host
 ==============================================
 
+WARNING: The keystore file must be placed in a directory accessible by elasticsearch like the `config` directory.
+
 It's important to define `network.host` as by default it's bound to `localhost`.
 
-You can use {ref}/modules-network.html[core network host settings]. For example `_non_loopback_` or `_en0_`.
+You can use {ref}/modules-network.html[core network host settings]. For example `_en0_`.
 
 ==============================================
 
diff --git a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeServiceImpl.java b/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeServiceImpl.java
index 0764ec99c12..076ce52ff80 100644
--- a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeServiceImpl.java
+++ b/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeServiceImpl.java
@@ -20,19 +20,21 @@
 package org.elasticsearch.cloud.azure.management;
 
 import com.microsoft.windowsazure.Configuration;
+import com.microsoft.windowsazure.core.Builder;
+import com.microsoft.windowsazure.core.DefaultBuilder;
 import com.microsoft.windowsazure.core.utils.KeyStoreType;
 import com.microsoft.windowsazure.management.compute.ComputeManagementClient;
 import com.microsoft.windowsazure.management.compute.ComputeManagementService;
 import com.microsoft.windowsazure.management.compute.models.HostedServiceGetDetailedResponse;
 import com.microsoft.windowsazure.management.configuration.ManagementConfiguration;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.cloud.azure.AzureServiceDisableException;
 import org.elasticsearch.cloud.azure.AzureServiceRemoteException;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 
 import java.io.IOException;
+import java.util.ServiceLoader;
 
 /**
  *
@@ -40,7 +42,7 @@ import java.io.IOException;
 public class AzureComputeServiceImpl extends AbstractLifecycleComponent<AzureComputeServiceImpl>
     implements AzureComputeService {
 
-    private final ComputeManagementClient computeManagementClient;
+    private final ComputeManagementClient client;
     private final String serviceName;
 
     @Inject
@@ -54,28 +56,36 @@ public class AzureComputeServiceImpl extends AbstractLifecycleComponent<AzureCom
         KeyStoreType keystoreType = Management.KEYSTORE_TYPE_SETTING.get(settings);
 
         logger.trace("creating new Azure client for [{}], [{}]", subscriptionId, serviceName);
-        ComputeManagementClient result;
         try {
-            // Check that we have all needed properties
-            Configuration configuration = ManagementConfiguration.configure(Management.ENDPOINT_SETTING.get(settings),
-                subscriptionId, keystorePath, keystorePassword, keystoreType);
-            result = ComputeManagementService.create(configuration);
+            // Azure SDK configuration uses DefaultBuilder which uses java.util.ServiceLoader to load the
+            // various Azure services. By default, this will use the current thread's context classloader
+            // to load services. Since the current thread refers to the main application classloader it
+            // won't find any Azure service implementation.
+
+            // Here we basically create a new DefaultBuilder that uses the current class classloader to load services.
+            DefaultBuilder builder = new DefaultBuilder();
+            for (Builder.Exports exports : ServiceLoader.load(Builder.Exports.class, getClass().getClassLoader())) {
+                exports.register(builder);
+            }
+
+            // And create a new blank configuration based on the previous DefaultBuilder
+            Configuration configuration = new Configuration(builder);
+            configuration.setProperty(Configuration.PROPERTY_LOG_HTTP_REQUESTS, logger.isTraceEnabled());
+
+            Configuration managementConfig = ManagementConfiguration.configure(null, configuration,
+                    Management.ENDPOINT_SETTING.get(settings), subscriptionId, keystorePath, keystorePassword, keystoreType);
+
+            logger.debug("creating new Azure client for [{}], [{}]", subscriptionId, serviceName);
+            client = ComputeManagementService.create(managementConfig);
         } catch (IOException e) {
-            logger.error("can not start azure client: {}", e.getMessage());
-            result = null;
+            throw new ElasticsearchException("Unable to configure Azure compute service", e);
         }
-        this.computeManagementClient = result;
     }
 
     @Override
     public HostedServiceGetDetailedResponse getServiceDetails() {
-        if (computeManagementClient == null) {
-            // Azure plugin is disabled
-            throw new AzureServiceDisableException("azure plugin is disabled.");
-        }
-
         try {
-            return computeManagementClient.getHostedServicesOperations().getDetailed(serviceName);
+            return client.getHostedServicesOperations().getDetailed(serviceName);
         } catch (Exception e) {
             throw new AzureServiceRemoteException("can not get list of azure nodes", e);
         }
@@ -91,9 +101,9 @@ public class AzureComputeServiceImpl extends AbstractLifecycleComponent<AzureCom
 
     @Override
     protected void doClose() throws ElasticsearchException {
-        if (computeManagementClient != null) {
+        if (client != null) {
             try {
-                computeManagementClient.close();
+                client.close();
             } catch (IOException e) {
                 logger.error("error while closing Azure client", e);
             }

From b9e8ec1938147a467399f76fe4e978f746cecfc9 Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Tue, 28 Jun 2016 10:13:08 +0200
Subject: [PATCH 1090/1311] Update Java API doc for cluster health

In 995e4eda08be99f72ef56052b3f78ceef9100885 we changed the cluster health Java API.
We need to also change the documentation.

Backport of #19093 in master branch
---
 docs/java-api/admin/cluster/health.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/java-api/admin/cluster/health.asciidoc b/docs/java-api/admin/cluster/health.asciidoc
index 7d20fdde6a3..615a011cf72 100644
--- a/docs/java-api/admin/cluster/health.asciidoc
+++ b/docs/java-api/admin/cluster/health.asciidoc
@@ -14,7 +14,7 @@ String clusterName = healths.getClusterName();              <2>
 int numberOfDataNodes = healths.getNumberOfDataNodes();     <3>
 int numberOfNodes = healths.getNumberOfNodes();             <4>
 
-for (ClusterIndexHealth health : healths) {                 <5>
+for (ClusterIndexHealth health : healths.getIndices().values()) { <5>
     String index = health.getIndex();                       <6>
     int numberOfShards = health.getNumberOfShards();        <7>
     int numberOfReplicas = health.getNumberOfReplicas();    <8>

From 26e6a522c7fedac27ebf0c6a5d5e8007faf0bd1a Mon Sep 17 00:00:00 2001
From: Colin Goodheart-Smithe <colings86@users.noreply.github.com>
Date: Tue, 28 Jun 2016 09:37:34 +0100
Subject: [PATCH 1091/1311] [TEST] Fixed bounds calculation for extended bounds
 in histogram agg empty buckets test

---
 .../java/org/elasticsearch/messy/tests/HistogramTests.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java
index 45274f44401..a6349db0d07 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java
@@ -857,7 +857,7 @@ public class HistogramTests extends ESIntegTestCase {
         boolean invalidBoundsError = boundsMin > boundsMax;
 
         // constructing the newly expected bucket list
-        int bucketsCount = numValueBuckets + addedBucketsLeft + addedBucketsRight;
+        int bucketsCount = (int) ((boundsMaxKey - boundsMinKey) / interval) + 1;
         long[] extendedValueCounts = new long[bucketsCount];
         System.arraycopy(valueCounts, 0, extendedValueCounts, addedBucketsLeft, valueCounts.length);
 
@@ -893,7 +893,7 @@ public class HistogramTests extends ESIntegTestCase {
         List<? extends Bucket> buckets = histo.getBuckets();
         assertThat(buckets.size(), equalTo(bucketsCount));
 
-        long key = Math.min(boundsMinKey, 0);
+        long key = boundsMinKey;
         for (int i = 0; i < bucketsCount; i++) {
             Histogram.Bucket bucket = buckets.get(i);
             assertThat(bucket, notNullValue());

From fc9fa3afaf28f7e362eca2a9ab28b6f0dcd0e3cb Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Tue, 28 Jun 2016 12:26:03 +0200
Subject: [PATCH 1092/1311] Added release notes for 5.0.0-alpha4

---
 .../release-notes/5.0.0-alpha4.asciidoc       | 353 ++++++++++++++++++
 1 file changed, 353 insertions(+)
 create mode 100644 docs/reference/release-notes/5.0.0-alpha4.asciidoc

diff --git a/docs/reference/release-notes/5.0.0-alpha4.asciidoc b/docs/reference/release-notes/5.0.0-alpha4.asciidoc
new file mode 100644
index 00000000000..b85d4ae32a5
--- /dev/null
+++ b/docs/reference/release-notes/5.0.0-alpha4.asciidoc
@@ -0,0 +1,353 @@
+[[release-notes-5.0.0-alpha4]]
+== 5.0.0-alpha4 Release Notes
+
+Also see <<breaking-changes-5.0>>.
+
+IMPORTANT: This is an alpha release and is intended for _testing purposes only_. Indices created in this version will *not be compatible with Elasticsearch 5.0.0 GA*. Upgrading 5.0.0-alpha4 to any other version is not supported.
+
+[[breaking-5.0.0-alpha4]]
+[float]
+=== Breaking changes
+
+Aggregations::
+* Remove size 0 options in aggregations {pull}18854[#18854] (issue: {issue}18838[#18838])
+
+CRUD::
+* Wait for changes to be visible by search {pull}17986[#17986] (issue: {issue}1063[#1063])
+
+Core::
+* Register thread pool settings {pull}18674[#18674] (issues: {issue}18613[#18613], {issue}9216[#9216])
+* Remove cluster name from data path {pull}18554[#18554] (issue: {issue}17810[#17810])
+
+Highlighting::
+* Register Highlighter instances instead of classes {pull}18859[#18859]
+
+Inner Hits::
+* Also do not serialize `_index` key in search response for parent/child inner hits {pull}19011[#19011]
+* Don't include `_id`, `_type` and `_index` keys in search response for inner hits {pull}18995[#18995] (issue: {issue}18091[#18091])
+* Nested inner hits shouldn't use relative paths {pull}18567[#18567] (issue: {issue}16653[#16653])
+
+Internal::
+* Cleanup ClusterService dependencies and detached from Guice {pull}18941[#18941]
+* Simplify SubFetchPhase interface {pull}18881[#18881]
+* Simplify FetchSubPhase registration and detach it from Guice {pull}18862[#18862]
+
+Java API::
+* Remove setRefresh {pull}18752[#18752] (issue: {issue}1063[#1063])
+
+Mapping::
+* Remove `_timestamp` and `_ttl` on 5.x indices. {pull}18980[#18980] (issue: {issue}18280[#18280])
+
+Packaging::
+* Remove allow running as root {pull}18694[#18694] (issue: {issue}18688[#18688])
+
+Plugins::
+* Fail to start if plugin tries broken onModule {pull}19025[#19025]
+* Simplify ScriptModule and script registration {pull}18903[#18903]
+* Cut over settings registration to a pull model {pull}18890[#18890]
+* Plugins cleanup {pull}18594[#18594] (issue: {issue}18588[#18588])
+
+Scripting::
+* Move search template to lang-mustache module {pull}18765[#18765] (issue: {issue}17906[#17906])
+
+Search::
+* Remove only node preference {pull}18875[#18875] (issue: {issue}18822[#18822])
+* Add search preference to prefer multiple nodes {pull}18872[#18872] (issue: {issue}18822[#18822])
+
+Settings::
+* Rename boostrap.mlockall to bootstrap.memory_lock {pull}18669[#18669]
+
+Snapshot/Restore::
+* Change the default of `include_global_state` from true to false for snapshot restores {pull}18773[#18773] (issue: {issue}18569[#18569])
+
+
+
+[[feature-5.0.0-alpha4]]
+[float]
+=== New features
+
+Aggregations::
+* Adds aggregation profiling to the profile API {pull}18414[#18414] (issue: {issue}10538[#10538])
+* New Matrix Stats Aggregation module {pull}18300[#18300] (issue: {issue}16826[#16826])
+
+Index APIs::
+* Add rollover API to switch index aliases given some predicates {pull}18732[#18732] (issue: {issue}18647[#18647])
+
+Mapping::
+* Expose half-floats. {pull}18887[#18887]
+
+REST::
+* Low level Rest Client {pull}18735[#18735] (issue: {issue}7743[#7743])
+
+Scroll::
+* Add the ability to partition a scroll in multiple slices. {pull}18237[#18237] (issue: {issue}13494[#13494])
+
+Store::
+* Expose MMapDirectory.preLoad(). {pull}18880[#18880]
+* Add primitive to shrink an index into a single shard {pull}18270[#18270]
+
+
+
+[[enhancement-5.0.0-alpha4]]
+[float]
+=== Enhancements
+
+Aggregations::
+* Automatically set the collection mode to breadth_first in the terms aggregation when the cardinality of the field is unknown or smaller than the requested size. {pull}18779[#18779] (issue: {issue}9825[#9825])
+* Rename PipelineAggregatorBuilder to PipelineAggregationBuilder. {pull}18677[#18677] (issue: {issue}18377[#18377])
+* AggregatorBuilder and PipelineAggregatorBuilder do not need generics. {pull}18368[#18368] (issue: {issue}18133[#18133])
+
+Allocation::
+* Allow `_shrink` to N shards if source shards is a multiple of N {pull}18699[#18699]
+* Only filter intial recovery (post API) when shrinking an index {pull}18661[#18661]
+* Estimate shard size for shrinked indices {pull}18659[#18659]
+* Only fail relocation target shard if failing source shard is a primary {pull}18574[#18574] (issue: {issue}16144[#16144])
+* Simplify delayed shard allocation {pull}18351[#18351] (issue: {issue}18293[#18293])
+
+Analysis::
+* Add a MultiTermAwareComponent marker interface to analysis factories. {pull}19028[#19028] (issues: {issue}18064[#18064], {issue}9978[#9978])
+* Add Flags Parameter for Char Filter {pull}18363[#18363] (issue: {issue}18362[#18362])
+
+Cache::
+* Cache FieldStats in the request cache {pull}18768[#18768] (issue: {issue}18717[#18717])
+
+Cluster::
+* Index creation does not cause the cluster health to go RED {pull}18737[#18737] (issues: {issue}9106[#9106], {issue}9126[#9126])
+* Cluster Health class improvements {pull}18673[#18673]
+
+Core::
+* Read Elasticsearch manifest via URL {pull}18999[#18999] (issue: {issue}18996[#18996])
+* Throw if the local node is not set {pull}18963[#18963] (issue: {issue}18962[#18962])
+* Improve performance of applyDeletedShards {pull}18788[#18788] (issue: {issue}18776[#18776])
+* Bootstrap check for OnOutOfMemoryError and seccomp {pull}18756[#18756] (issue: {issue}18736[#18736])
+
+Dates::
+* Improve TimeZoneRoundingTests error messages {pull}18895[#18895]
+* Improve TimeUnitRounding for edge cases and DST transitions {pull}18589[#18589]
+
+Expressions::
+* improve date api for expressions/painless fields {pull}18658[#18658]
+
+Index APIs::
+* Add Shrink request source parser to parse create index request body {pull}18802[#18802]
+
+Index Templates::
+* Parse and validate mappings on index template creation {pull}8802[#8802] (issue: {issue}2415[#2415])
+
+Ingest::
+* Add `ignore_failure` option to all ingest processors {pull}18650[#18650] (issue: {issue}18493[#18493])
+* new ScriptProcessor for Ingest {pull}18193[#18193]
+
+Internal::
+* Hot methods redux {pull}19016[#19016] (issue: {issue}16725[#16725])
+* Remove forked joda time BaseDateTime class {pull}18953[#18953]
+* Support optional ctor args in ConstructingObjectParser {pull}18725[#18725]
+* Remove thread pool from page cache recycler {pull}18664[#18664] (issue: {issue}18613[#18613])
+
+Java API::
+* Switch QueryBuilders to new MatchPhraseQueryBuilder {pull}18753[#18753]
+
+Logging::
+* Throw IllegalStateException when handshake fails due to version or cluster mismatch {pull}18676[#18676]
+
+Mapping::
+* Upgrade `string` fields to `text`/`keyword` even if `include_in_all` is set. {pull}19004[#19004] (issue: {issue}18974[#18974])
+
+Network::
+* Exclude admin / diagnostic requests from HTTP request limiting {pull}18833[#18833] (issues: {issue}17951[#17951], {issue}18145[#18145])
+* Do not start scheduled pings until transport start {pull}18702[#18702]
+
+Packaging::
+* Remove explicit parallel new GC flag {pull}18767[#18767]
+* Use JAVA_HOME or java.exe in PATH like the Linux scripts do {pull}18685[#18685] (issue: {issue}4913[#4913])
+
+Percolator::
+* Add percolator query extraction support for dismax query {pull}18845[#18845]
+* Improve percolate query performance by not verifying certain candidate matches {pull}18696[#18696]
+* Improve percolator query term extraction {pull}18610[#18610]
+
+Plugin Lang Painless::
+* Painless Initializers {pull}19012[#19012]
+* Add augmentation {pull}19003[#19003]
+* Infer lambda arguments/return type {pull}18983[#18983]
+* Fix explicit casts and improve tests. {pull}18958[#18958]
+* Add lambda captures {pull}18954[#18954]
+* improve Debugger to print code even if it hits exception {pull}18932[#18932] (issue: {issue}1[#1])
+* Move semicolon hack into lexer {pull}18931[#18931]
+* Add flag support to regexes {pull}18927[#18927]
+* improve lambda syntax (allow single expression) {pull}18924[#18924]
+* Remove useless dropArguments in megamorphic cache {pull}18913[#18913]
+* non-capturing lambda support {pull}18911[#18911] (issue: {issue}18824[#18824])
+* fix bugs in operators and more improvements for the dynamic case {pull}18899[#18899]
+* improve unary operators and cleanup tests {pull}18867[#18867] (issue: {issue}18849[#18849])
+* Add support for the find operator (=~) and the match operator (==~) {pull}18858[#18858]
+* Remove casts and boxing for dynamic math {pull}18849[#18849] (issue: {issue}18847[#18847])
+* Refactor def math {pull}18847[#18847]
+* Add support for /regex/ {pull}18842[#18842]
+* Array constructor references {pull}18831[#18831]
+* Method references to user functions {pull}18828[#18828]
+* Add } as a delimiter.  {pull}18827[#18827] (issue: {issue}18821[#18821])
+* Add Lambda Stub Node {pull}18824[#18824]
+* Add capturing method references {pull}18818[#18818] (issue: {issue}18748[#18748])
+* Add Functions to Painless {pull}18810[#18810]
+* Add Method to Get New MethodWriters {pull}18771[#18771]
+* Static For Each {pull}18757[#18757]
+* Method reference support {pull}18748[#18748] (issue: {issue}18578[#18578])
+* Add support for the new Java 9 MethodHandles#arrayLength() factory {pull}18734[#18734]
+* Improve painless compile-time exceptions {pull}18711[#18711] (issue: {issue}18600[#18600])
+* add java.time packages to painless whitelist {pull}18621[#18621]
+* Add Function Reference Stub to Painless {pull}18578[#18578]
+
+Plugins::
+* Add did-you-mean for plugin cli {pull}18942[#18942] (issue: {issue}18896[#18896])
+* Plugins: Remove name() and description() from api {pull}18906[#18906]
+* Emit nicer error message when trying to install unknown plugin {pull}18876[#18876] (issue: {issue}17226[#17226])
+
+Query DSL::
+* Treat zero token in `common` terms query as MatchNoDocsQuery {pull}18656[#18656]
+* Handle empty query bodies at parse time and remove EmptyQueryBuilder {pull}17624[#17624] (issues: {issue}17540[#17540], {issue}17541[#17541])
+
+REST::
+* Adding status field in _msearch error request bodies {pull}18586[#18586] (issue: {issue}18013[#18013])
+
+Recovery::
+* index shard should be able to cancel check index on close. {pull}18839[#18839] (issue: {issue}12011[#12011])
+
+Reindex API::
+* Implement ctx.op = "delete" on _update_by_query and _reindex {pull}18614[#18614] (issue: {issue}18043[#18043])
+
+Scripting::
+*  Compile each Groovy script in its own classloader {pull}18918[#18918] (issue: {issue}18572[#18572])
+* Include script field even if it value is null {pull}18384[#18384] (issue: {issue}16408[#16408])
+
+Scroll::
+* Add an index setting to limit the maximum number of slices allowed in a scroll request. {pull}18782[#18782]
+
+Search::
+* Change default similarity to BM25 {pull}18948[#18948] (issue: {issue}18944[#18944])
+* Add a parameter to cap the number of searches the msearch api will concurrently execute {pull}18721[#18721]
+
+Sequence IDs::
+* Persist sequence number checkpoints {pull}18949[#18949] (issue: {issue}10708[#10708])
+* Add sequence numbers to cat shards API {pull}18772[#18772]
+
+Settings::
+* Improve error message if a setting is not found {pull}18920[#18920] (issue: {issue}18663[#18663])
+* Cleanup placeholder replacement {pull}17335[#17335]
+
+Snapshot/Restore::
+* Adds UUIDs to snapshots {pull}18228[#18228] (issue: {issue}18156[#18156])
+* Clarify the semantics of the BlobContainer interface {pull}18157[#18157] (issue: {issue}15580[#15580])
+
+Stats::
+* Add total_indexing_buffer/_in_bytes to nodes info API {pull}18914[#18914] (issue: {issue}18651[#18651])
+* Allow FieldStatsRequest to disable cache {pull}18900[#18900]
+* Remove index_writer_max_memory stat from segment stats {pull}18651[#18651] (issues: {issue}14121[#14121], {issue}7440[#7440])
+* Move DocStats under Engine to get more accurate numbers {pull}18587[#18587]
+
+Task Manager::
+* Fetch result when wait_for_completion {pull}18905[#18905]
+* Create get task API that falls back to the .tasks index {pull}18682[#18682]
+* Add ability to store results for long running tasks {pull}17928[#17928]
+
+Translog::
+* Beef up Translog testing with random channel exceptions {pull}18997[#18997]
+* Do not replay into translog on local recovery {pull}18547[#18547]
+
+
+
+[[bug-5.0.0-alpha4]]
+[float]
+=== Bug fixes
+
+Allocation::
+* Fix recovery throttling to properly handle relocating non-primary shards {pull}18701[#18701] (issue: {issue}18640[#18640])
+
+CAT API::
+* Fix merge stats rendering in RestIndicesAction {pull}18720[#18720]
+
+CRUD::
+* Squash a race condition in RefreshListeners {pull}18806[#18806]
+
+Circuit Breakers::
+* Never trip circuit breaker in liveness request {pull}18627[#18627] (issue: {issue}17951[#17951])
+
+Cluster::
+* Fix block checks when no indices are specified {pull}19047[#19047] (issue: {issue}8105[#8105])
+* Acknowledge index deletion requests based on standard cluster state acknowledgment {pull}18602[#18602] (issues: {issue}16442[#16442], {issue}18558[#18558])
+
+Core::
+* Throw exception if using a closed transport client {pull}18722[#18722] (issue: {issue}18708[#18708])
+
+Dates::
+* Fix invalid rounding value for TimeIntervalRounding close to DST transitions {pull}18800[#18800]
+* Fix problem with TimeIntervalRounding on DST end {pull}18780[#18780]
+
+Expressions::
+* replace ScriptException with a better one {pull}18600[#18600]
+
+Ingest::
+* Fix ignore_failure behavior in _simulate?verbose and more cleanup {pull}18987[#18987]
+
+Internal::
+* Fix filtering of node ids for TransportNodesAction {pull}18634[#18634] (issue: {issue}18618[#18618])
+
+Mapping::
+* Better error message when mapping configures null {pull}18809[#18809] (issue: {issue}18803[#18803])
+* Process dynamic templates in order. {pull}18638[#18638] (issues: {issue}18625[#18625], {issue}2401[#2401])
+
+Packaging::
+* Remove extra bin/ directory in bin folder {pull}18630[#18630]
+
+Plugin Lang Painless::
+* Fix compound assignment with string concats {pull}18933[#18933] (issue: {issue}18929[#18929])
+* Fix horrible capture {pull}18907[#18907] (issue: {issue}18899[#18899])
+* Fix Casting Bug {pull}18871[#18871]
+
+Query DSL::
+* Make parsing of bool queries stricter {pull}19052[#19052] (issue: {issue}19034[#19034])
+
+REST::
+* Get XContent params from request in Nodes rest actions {pull}18860[#18860] (issue: {issue}18794[#18794])
+
+Reindex API::
+* Fix a race condition in reindex's rethrottle {pull}18731[#18731] (issue: {issue}18744[#18744])
+
+Search::
+* Require timeout units when parsing query body {pull}19077[#19077] (issue: {issue}19075[#19075])
+* Close SearchContext if query rewrite failed {pull}18727[#18727]
+
+Settings::
+* Register "cloud.node.auto_attributes" setting in EC2 discovery plugin {pull}18678[#18678]
+
+Snapshot/Restore::
+* Better handling of an empty shard's segments_N file {pull}18784[#18784] (issue: {issue}18707[#18707])
+
+Stats::
+* Fix sync flush total shards statistics {pull}18766[#18766]
+
+Translog::
+* Fix translog replay multiple operations same doc {pull}18611[#18611] (issues: {issue}18547[#18547], {issue}18623[#18623])
+
+
+
+[[upgrade-5.0.0-alpha4]]
+[float]
+=== Upgrades
+
+Core::
+* Upgrade to Lucene 6.1.0. {pull}18926[#18926]
+* Upgrade to lucene-6.1.0-snapshot-3a57bea. {pull}18786[#18786]
+* Upgrade to Lucene 6.0.1. {pull}18648[#18648] (issues: {issue}17535[#17535], {issue}28[#28])
+
+Dates::
+* Upgrade joda-time to 2.9.4 {pull}18609[#18609] (issues: {issue}14524[#14524], {issue}18017[#18017])
+
+Packaging::
+* Upgrade JNA to 4.2.2 and remove optionality {pull}19045[#19045] (issue: {issue}13245[#13245])
+
+Plugin Discovery EC2::
+* Update aws sdk to 1.10.69 and add use_throttle_retries repository setting {pull}17784[#17784] (issues: {issue}538[#538], {issue}586[#586], {issue}589[#589])
+
+
+

From fa4844c3f47ebc56478ef1bb7913d73a962fe945 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 27 Jun 2016 17:33:01 -0400
Subject: [PATCH 1093/1311] Pull actions from plugins

Instead of implementing onModule(ActionModule) to register actions,
this has plugins implement ActionPlugin to declare actions. This is
yet another step in cleaning up the plugin infrastructure.

While I was in there I switched AutoCreateIndex and DestructiveOperations
to be eagerly constructed which makes them easier to use when
de-guice-ing the code base.
---
 .../elasticsearch/action/ActionModule.java    | 316 +++++++++---------
 .../action/support/AutoCreateIndex.java       |   1 -
 .../action/support/DestructiveOperations.java |   2 -
 .../client/transport/TransportClient.java     |  21 +-
 .../elasticsearch/cluster/ClusterModule.java  |   9 +-
 .../elasticsearch/common/NamedRegistry.java   |  59 ++++
 .../indices/analysis/AnalysisModule.java      |  47 +--
 .../java/org/elasticsearch/node/Node.java     |  10 +-
 .../elasticsearch/plugins/ActionPlugin.java   |  87 +++++
 .../org/elasticsearch/plugins/Plugin.java     |   9 +
 .../cluster/node/tasks/TestTaskPlugin.java    |  14 +-
 .../cluster/ClusterInfoServiceIT.java         |  13 +-
 .../ContextAndHeaderTransportIT.java          |  10 +-
 .../migration/migrate_5_0/plugins.asciidoc    |   3 +
 .../script/mustache/MustachePlugin.java       |  18 +-
 .../percolator/PercolatorPlugin.java          |  23 +-
 .../index/reindex/ReindexPlugin.java          |  20 +-
 17 files changed, 419 insertions(+), 243 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/common/NamedRegistry.java
 create mode 100644 core/src/main/java/org/elasticsearch/plugins/ActionPlugin.java

diff --git a/core/src/main/java/org/elasticsearch/action/ActionModule.java b/core/src/main/java/org/elasticsearch/action/ActionModule.java
index a3a1fbca153..f5c83e272ce 100644
--- a/core/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/core/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -64,6 +64,12 @@ import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
 import org.elasticsearch.action.admin.cluster.state.TransportClusterStateAction;
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsAction;
 import org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction;
+import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptAction;
+import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptAction;
+import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptAction;
+import org.elasticsearch.action.admin.cluster.storedscripts.TransportDeleteStoredScriptAction;
+import org.elasticsearch.action.admin.cluster.storedscripts.TransportGetStoredScriptAction;
+import org.elasticsearch.action.admin.cluster.storedscripts.TransportPutStoredScriptAction;
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksAction;
 import org.elasticsearch.action.admin.cluster.tasks.TransportPendingClusterTasksAction;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction;
@@ -151,18 +157,12 @@ import org.elasticsearch.action.get.TransportMultiGetAction;
 import org.elasticsearch.action.get.TransportShardMultiGetAction;
 import org.elasticsearch.action.index.IndexAction;
 import org.elasticsearch.action.index.TransportIndexAction;
-import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptAction;
-import org.elasticsearch.action.admin.cluster.storedscripts.TransportDeleteStoredScriptAction;
-import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptAction;
-import org.elasticsearch.action.admin.cluster.storedscripts.TransportGetStoredScriptAction;
-import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptAction;
-import org.elasticsearch.action.admin.cluster.storedscripts.TransportPutStoredScriptAction;
-import org.elasticsearch.action.ingest.IngestActionFilter;
-import org.elasticsearch.action.ingest.IngestProxyActionFilter;
 import org.elasticsearch.action.ingest.DeletePipelineAction;
 import org.elasticsearch.action.ingest.DeletePipelineTransportAction;
 import org.elasticsearch.action.ingest.GetPipelineAction;
 import org.elasticsearch.action.ingest.GetPipelineTransportAction;
+import org.elasticsearch.action.ingest.IngestActionFilter;
+import org.elasticsearch.action.ingest.IngestProxyActionFilter;
 import org.elasticsearch.action.ingest.PutPipelineAction;
 import org.elasticsearch.action.ingest.PutPipelineTransportAction;
 import org.elasticsearch.action.ingest.SimulatePipelineAction;
@@ -189,186 +189,204 @@ import org.elasticsearch.action.termvectors.TransportShardMultiTermsVectorAction
 import org.elasticsearch.action.termvectors.TransportTermVectorsAction;
 import org.elasticsearch.action.update.TransportUpdateAction;
 import org.elasticsearch.action.update.UpdateAction;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.NamedRegistry;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.multibindings.MapBinder;
 import org.elasticsearch.common.inject.multibindings.Multibinder;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.ActionPlugin.ActionHandler;
 
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static java.util.Collections.unmodifiableList;
+import static java.util.Collections.unmodifiableMap;
+
 /**
- *
+ * Builds and binds the generic action map, all {@link TransportAction}s, and {@link ActionFilters}.
  */
 public class ActionModule extends AbstractModule {
 
-    private final Map<String, ActionEntry> actions = new HashMap<>();
-    private final List<Class<? extends ActionFilter>> actionFilters = new ArrayList<>();
+    private final boolean transportClient;
+    private final Map<String, ActionHandler<?, ?>> actions;
+    private final List<Class<? extends ActionFilter>> actionFilters;
+    private final AutoCreateIndex autoCreateIndex;
+    private final DestructiveOperations destructiveOperations;
 
-    static class ActionEntry<Request extends ActionRequest<Request>, Response extends ActionResponse> {
-        public final GenericAction<Request, Response> action;
-        public final Class<? extends TransportAction<Request, Response>> transportAction;
-        public final Class[] supportTransportActions;
+    public ActionModule(boolean ingestEnabled, boolean transportClient, Settings settings, IndexNameExpressionResolver resolver,
+            ClusterSettings clusterSettings, List<ActionPlugin> actionPlugins) {
+        this.transportClient = transportClient;
+        actions = setupActions(actionPlugins);
+        actionFilters = setupActionFilters(actionPlugins, ingestEnabled);
+        autoCreateIndex = transportClient ? null : new AutoCreateIndex(settings, resolver);
+        destructiveOperations = new DestructiveOperations(settings, clusterSettings);
+    }
 
-        ActionEntry(GenericAction<Request, Response> action, Class<? extends TransportAction<Request, Response>> transportAction, Class... supportTransportActions) {
-            this.action = action;
-            this.transportAction = transportAction;
-            this.supportTransportActions = supportTransportActions;
+    private Map<String, ActionHandler<?, ?>> setupActions(List<ActionPlugin> actionPlugins) {
+        // Subclass NamedRegistry for easy registration
+        class ActionRegistry extends NamedRegistry<ActionHandler<?, ?>> {
+            public ActionRegistry() {
+                super("action");
+            }
+
+            public void register(ActionHandler<?, ?> handler) {
+                register(handler.getAction().name(), handler);
+            }
+
+            public <Request extends ActionRequest<Request>, Response extends ActionResponse> void register(
+                    GenericAction<Request, Response> action, Class<? extends TransportAction<Request, Response>> transportAction,
+                    Class<?>... supportTransportActions) {
+                register(new ActionHandler<>(action, transportAction, supportTransportActions));
+            }
         }
+        ActionRegistry actions = new ActionRegistry();
+
+        actions.register(MainAction.INSTANCE, TransportMainAction.class);
+        actions.register(NodesInfoAction.INSTANCE, TransportNodesInfoAction.class);
+        actions.register(NodesStatsAction.INSTANCE, TransportNodesStatsAction.class);
+        actions.register(NodesHotThreadsAction.INSTANCE, TransportNodesHotThreadsAction.class);
+        actions.register(ListTasksAction.INSTANCE, TransportListTasksAction.class);
+        actions.register(GetTaskAction.INSTANCE, TransportGetTaskAction.class);
+        actions.register(CancelTasksAction.INSTANCE, TransportCancelTasksAction.class);
+
+        actions.register(ClusterAllocationExplainAction.INSTANCE, TransportClusterAllocationExplainAction.class);
+        actions.register(ClusterStatsAction.INSTANCE, TransportClusterStatsAction.class);
+        actions.register(ClusterStateAction.INSTANCE, TransportClusterStateAction.class);
+        actions.register(ClusterHealthAction.INSTANCE, TransportClusterHealthAction.class);
+        actions.register(ClusterUpdateSettingsAction.INSTANCE, TransportClusterUpdateSettingsAction.class);
+        actions.register(ClusterRerouteAction.INSTANCE, TransportClusterRerouteAction.class);
+        actions.register(ClusterSearchShardsAction.INSTANCE, TransportClusterSearchShardsAction.class);
+        actions.register(PendingClusterTasksAction.INSTANCE, TransportPendingClusterTasksAction.class);
+        actions.register(PutRepositoryAction.INSTANCE, TransportPutRepositoryAction.class);
+        actions.register(GetRepositoriesAction.INSTANCE, TransportGetRepositoriesAction.class);
+        actions.register(DeleteRepositoryAction.INSTANCE, TransportDeleteRepositoryAction.class);
+        actions.register(VerifyRepositoryAction.INSTANCE, TransportVerifyRepositoryAction.class);
+        actions.register(GetSnapshotsAction.INSTANCE, TransportGetSnapshotsAction.class);
+        actions.register(DeleteSnapshotAction.INSTANCE, TransportDeleteSnapshotAction.class);
+        actions.register(CreateSnapshotAction.INSTANCE, TransportCreateSnapshotAction.class);
+        actions.register(RestoreSnapshotAction.INSTANCE, TransportRestoreSnapshotAction.class);
+        actions.register(SnapshotsStatusAction.INSTANCE, TransportSnapshotsStatusAction.class);
+
+        actions.register(IndicesStatsAction.INSTANCE, TransportIndicesStatsAction.class);
+        actions.register(IndicesSegmentsAction.INSTANCE, TransportIndicesSegmentsAction.class);
+        actions.register(IndicesShardStoresAction.INSTANCE, TransportIndicesShardStoresAction.class);
+        actions.register(CreateIndexAction.INSTANCE, TransportCreateIndexAction.class);
+        actions.register(ShrinkAction.INSTANCE, TransportShrinkAction.class);
+        actions.register(RolloverAction.INSTANCE, TransportRolloverAction.class);
+        actions.register(DeleteIndexAction.INSTANCE, TransportDeleteIndexAction.class);
+        actions.register(GetIndexAction.INSTANCE, TransportGetIndexAction.class);
+        actions.register(OpenIndexAction.INSTANCE, TransportOpenIndexAction.class);
+        actions.register(CloseIndexAction.INSTANCE, TransportCloseIndexAction.class);
+        actions.register(IndicesExistsAction.INSTANCE, TransportIndicesExistsAction.class);
+        actions.register(TypesExistsAction.INSTANCE, TransportTypesExistsAction.class);
+        actions.register(GetMappingsAction.INSTANCE, TransportGetMappingsAction.class);
+        actions.register(GetFieldMappingsAction.INSTANCE, TransportGetFieldMappingsAction.class,
+                TransportGetFieldMappingsIndexAction.class);
+        actions.register(PutMappingAction.INSTANCE, TransportPutMappingAction.class);
+        actions.register(IndicesAliasesAction.INSTANCE, TransportIndicesAliasesAction.class);
+        actions.register(UpdateSettingsAction.INSTANCE, TransportUpdateSettingsAction.class);
+        actions.register(AnalyzeAction.INSTANCE, TransportAnalyzeAction.class);
+        actions.register(PutIndexTemplateAction.INSTANCE, TransportPutIndexTemplateAction.class);
+        actions.register(GetIndexTemplatesAction.INSTANCE, TransportGetIndexTemplatesAction.class);
+        actions.register(DeleteIndexTemplateAction.INSTANCE, TransportDeleteIndexTemplateAction.class);
+        actions.register(ValidateQueryAction.INSTANCE, TransportValidateQueryAction.class);
+        actions.register(RefreshAction.INSTANCE, TransportRefreshAction.class);
+        actions.register(FlushAction.INSTANCE, TransportFlushAction.class);
+        actions.register(SyncedFlushAction.INSTANCE, TransportSyncedFlushAction.class);
+        actions.register(ForceMergeAction.INSTANCE, TransportForceMergeAction.class);
+        actions.register(UpgradeAction.INSTANCE, TransportUpgradeAction.class);
+        actions.register(UpgradeStatusAction.INSTANCE, TransportUpgradeStatusAction.class);
+        actions.register(UpgradeSettingsAction.INSTANCE, TransportUpgradeSettingsAction.class);
+        actions.register(ClearIndicesCacheAction.INSTANCE, TransportClearIndicesCacheAction.class);
+        actions.register(GetAliasesAction.INSTANCE, TransportGetAliasesAction.class);
+        actions.register(AliasesExistAction.INSTANCE, TransportAliasesExistAction.class);
+        actions.register(GetSettingsAction.INSTANCE, TransportGetSettingsAction.class);
+
+        actions.register(IndexAction.INSTANCE, TransportIndexAction.class);
+        actions.register(GetAction.INSTANCE, TransportGetAction.class);
+        actions.register(TermVectorsAction.INSTANCE, TransportTermVectorsAction.class);
+        actions.register(MultiTermVectorsAction.INSTANCE, TransportMultiTermVectorsAction.class,
+                TransportShardMultiTermsVectorAction.class);
+        actions.register(DeleteAction.INSTANCE, TransportDeleteAction.class);
+        actions.register(UpdateAction.INSTANCE, TransportUpdateAction.class);
+        actions.register(MultiGetAction.INSTANCE, TransportMultiGetAction.class,
+                TransportShardMultiGetAction.class);
+        actions.register(BulkAction.INSTANCE, TransportBulkAction.class,
+                TransportShardBulkAction.class);
+        actions.register(SearchAction.INSTANCE, TransportSearchAction.class);
+        actions.register(SearchScrollAction.INSTANCE, TransportSearchScrollAction.class);
+        actions.register(MultiSearchAction.INSTANCE, TransportMultiSearchAction.class);
+        actions.register(ExplainAction.INSTANCE, TransportExplainAction.class);
+        actions.register(ClearScrollAction.INSTANCE, TransportClearScrollAction.class);
+        actions.register(RecoveryAction.INSTANCE, TransportRecoveryAction.class);
+
+        //Indexed scripts
+        actions.register(PutStoredScriptAction.INSTANCE, TransportPutStoredScriptAction.class);
+        actions.register(GetStoredScriptAction.INSTANCE, TransportGetStoredScriptAction.class);
+        actions.register(DeleteStoredScriptAction.INSTANCE, TransportDeleteStoredScriptAction.class);
+
+        actions.register(FieldStatsAction.INSTANCE, TransportFieldStatsAction.class);
+
+        actions.register(PutPipelineAction.INSTANCE, PutPipelineTransportAction.class);
+        actions.register(GetPipelineAction.INSTANCE, GetPipelineTransportAction.class);
+        actions.register(DeletePipelineAction.INSTANCE, DeletePipelineTransportAction.class);
+        actions.register(SimulatePipelineAction.INSTANCE, SimulatePipelineTransportAction.class);
+
+        actionPlugins.stream().flatMap(p -> p.getActions().stream()).forEach(actions::register);
+
+        return unmodifiableMap(actions.getRegistry());
     }
 
-    private final boolean ingestEnabled;
-    private final boolean proxy;
+    private List<Class<? extends ActionFilter>> setupActionFilters(List<ActionPlugin> actionPlugins, boolean ingestEnabled) {
+        List<Class<? extends ActionFilter>> filters = new ArrayList<>();
+        if (transportClient == false) {
+            if (ingestEnabled) {
+                filters.add(IngestActionFilter.class);
+            } else {
+                filters.add(IngestProxyActionFilter.class);
+            }
+        }
 
-    public ActionModule(boolean ingestEnabled, boolean proxy) {
-        this.ingestEnabled = ingestEnabled;
-        this.proxy = proxy;
-    }
-
-    /**
-     * Registers an action.
-     *
-     * @param action                  The action type.
-     * @param transportAction         The transport action implementing the actual action.
-     * @param supportTransportActions Any support actions that are needed by the transport action.
-     * @param <Request>               The request type.
-     * @param <Response>              The response type.
-     */
-    public <Request extends ActionRequest<Request>, Response extends ActionResponse> void registerAction(GenericAction<Request, Response> action, Class<? extends TransportAction<Request, Response>> transportAction, Class... supportTransportActions) {
-        actions.put(action.name(), new ActionEntry<>(action, transportAction, supportTransportActions));
-    }
-
-    public ActionModule registerFilter(Class<? extends ActionFilter> actionFilter) {
-        actionFilters.add(actionFilter);
-        return this;
+        for (ActionPlugin plugin : actionPlugins) {
+            filters.addAll(plugin.getActionFilters());
+        }
+        return unmodifiableList(filters);
     }
 
     @Override
     protected void configure() {
-        if (proxy == false) {
-            if (ingestEnabled) {
-                registerFilter(IngestActionFilter.class);
-            } else {
-                registerFilter(IngestProxyActionFilter.class);
-            }
-        }
-
         Multibinder<ActionFilter> actionFilterMultibinder = Multibinder.newSetBinder(binder(), ActionFilter.class);
         for (Class<? extends ActionFilter> actionFilter : actionFilters) {
             actionFilterMultibinder.addBinding().to(actionFilter);
         }
         bind(ActionFilters.class).asEagerSingleton();
-        bind(AutoCreateIndex.class).asEagerSingleton();
-        bind(DestructiveOperations.class).asEagerSingleton();
-        registerAction(MainAction.INSTANCE, TransportMainAction.class);
-        registerAction(NodesInfoAction.INSTANCE, TransportNodesInfoAction.class);
-        registerAction(NodesStatsAction.INSTANCE, TransportNodesStatsAction.class);
-        registerAction(NodesHotThreadsAction.INSTANCE, TransportNodesHotThreadsAction.class);
-        registerAction(ListTasksAction.INSTANCE, TransportListTasksAction.class);
-        registerAction(GetTaskAction.INSTANCE, TransportGetTaskAction.class);
-        registerAction(CancelTasksAction.INSTANCE, TransportCancelTasksAction.class);
-
-        registerAction(ClusterAllocationExplainAction.INSTANCE, TransportClusterAllocationExplainAction.class);
-        registerAction(ClusterStatsAction.INSTANCE, TransportClusterStatsAction.class);
-        registerAction(ClusterStateAction.INSTANCE, TransportClusterStateAction.class);
-        registerAction(ClusterHealthAction.INSTANCE, TransportClusterHealthAction.class);
-        registerAction(ClusterUpdateSettingsAction.INSTANCE, TransportClusterUpdateSettingsAction.class);
-        registerAction(ClusterRerouteAction.INSTANCE, TransportClusterRerouteAction.class);
-        registerAction(ClusterSearchShardsAction.INSTANCE, TransportClusterSearchShardsAction.class);
-        registerAction(PendingClusterTasksAction.INSTANCE, TransportPendingClusterTasksAction.class);
-        registerAction(PutRepositoryAction.INSTANCE, TransportPutRepositoryAction.class);
-        registerAction(GetRepositoriesAction.INSTANCE, TransportGetRepositoriesAction.class);
-        registerAction(DeleteRepositoryAction.INSTANCE, TransportDeleteRepositoryAction.class);
-        registerAction(VerifyRepositoryAction.INSTANCE, TransportVerifyRepositoryAction.class);
-        registerAction(GetSnapshotsAction.INSTANCE, TransportGetSnapshotsAction.class);
-        registerAction(DeleteSnapshotAction.INSTANCE, TransportDeleteSnapshotAction.class);
-        registerAction(CreateSnapshotAction.INSTANCE, TransportCreateSnapshotAction.class);
-        registerAction(RestoreSnapshotAction.INSTANCE, TransportRestoreSnapshotAction.class);
-        registerAction(SnapshotsStatusAction.INSTANCE, TransportSnapshotsStatusAction.class);
-
-        registerAction(IndicesStatsAction.INSTANCE, TransportIndicesStatsAction.class);
-        registerAction(IndicesSegmentsAction.INSTANCE, TransportIndicesSegmentsAction.class);
-        registerAction(IndicesShardStoresAction.INSTANCE, TransportIndicesShardStoresAction.class);
-        registerAction(CreateIndexAction.INSTANCE, TransportCreateIndexAction.class);
-        registerAction(ShrinkAction.INSTANCE, TransportShrinkAction.class);
-        registerAction(RolloverAction.INSTANCE, TransportRolloverAction.class);
-        registerAction(DeleteIndexAction.INSTANCE, TransportDeleteIndexAction.class);
-        registerAction(GetIndexAction.INSTANCE, TransportGetIndexAction.class);
-        registerAction(OpenIndexAction.INSTANCE, TransportOpenIndexAction.class);
-        registerAction(CloseIndexAction.INSTANCE, TransportCloseIndexAction.class);
-        registerAction(IndicesExistsAction.INSTANCE, TransportIndicesExistsAction.class);
-        registerAction(TypesExistsAction.INSTANCE, TransportTypesExistsAction.class);
-        registerAction(GetMappingsAction.INSTANCE, TransportGetMappingsAction.class);
-        registerAction(GetFieldMappingsAction.INSTANCE, TransportGetFieldMappingsAction.class, TransportGetFieldMappingsIndexAction.class);
-        registerAction(PutMappingAction.INSTANCE, TransportPutMappingAction.class);
-        registerAction(IndicesAliasesAction.INSTANCE, TransportIndicesAliasesAction.class);
-        registerAction(UpdateSettingsAction.INSTANCE, TransportUpdateSettingsAction.class);
-        registerAction(AnalyzeAction.INSTANCE, TransportAnalyzeAction.class);
-        registerAction(PutIndexTemplateAction.INSTANCE, TransportPutIndexTemplateAction.class);
-        registerAction(GetIndexTemplatesAction.INSTANCE, TransportGetIndexTemplatesAction.class);
-        registerAction(DeleteIndexTemplateAction.INSTANCE, TransportDeleteIndexTemplateAction.class);
-        registerAction(ValidateQueryAction.INSTANCE, TransportValidateQueryAction.class);
-        registerAction(RefreshAction.INSTANCE, TransportRefreshAction.class);
-        registerAction(FlushAction.INSTANCE, TransportFlushAction.class);
-        registerAction(SyncedFlushAction.INSTANCE, TransportSyncedFlushAction.class);
-        registerAction(ForceMergeAction.INSTANCE, TransportForceMergeAction.class);
-        registerAction(UpgradeAction.INSTANCE, TransportUpgradeAction.class);
-        registerAction(UpgradeStatusAction.INSTANCE, TransportUpgradeStatusAction.class);
-        registerAction(UpgradeSettingsAction.INSTANCE, TransportUpgradeSettingsAction.class);
-        registerAction(ClearIndicesCacheAction.INSTANCE, TransportClearIndicesCacheAction.class);
-        registerAction(GetAliasesAction.INSTANCE, TransportGetAliasesAction.class);
-        registerAction(AliasesExistAction.INSTANCE, TransportAliasesExistAction.class);
-        registerAction(GetSettingsAction.INSTANCE, TransportGetSettingsAction.class);
-
-        registerAction(IndexAction.INSTANCE, TransportIndexAction.class);
-        registerAction(GetAction.INSTANCE, TransportGetAction.class);
-        registerAction(TermVectorsAction.INSTANCE, TransportTermVectorsAction.class);
-        registerAction(MultiTermVectorsAction.INSTANCE, TransportMultiTermVectorsAction.class,
-                TransportShardMultiTermsVectorAction.class);
-        registerAction(DeleteAction.INSTANCE, TransportDeleteAction.class);
-        registerAction(UpdateAction.INSTANCE, TransportUpdateAction.class);
-        registerAction(MultiGetAction.INSTANCE, TransportMultiGetAction.class,
-                TransportShardMultiGetAction.class);
-        registerAction(BulkAction.INSTANCE, TransportBulkAction.class,
-                TransportShardBulkAction.class);
-        registerAction(SearchAction.INSTANCE, TransportSearchAction.class);
-        registerAction(SearchScrollAction.INSTANCE, TransportSearchScrollAction.class);
-        registerAction(MultiSearchAction.INSTANCE, TransportMultiSearchAction.class);
-        registerAction(ExplainAction.INSTANCE, TransportExplainAction.class);
-        registerAction(ClearScrollAction.INSTANCE, TransportClearScrollAction.class);
-        registerAction(RecoveryAction.INSTANCE, TransportRecoveryAction.class);
-
-        //Indexed scripts
-        registerAction(PutStoredScriptAction.INSTANCE, TransportPutStoredScriptAction.class);
-        registerAction(GetStoredScriptAction.INSTANCE, TransportGetStoredScriptAction.class);
-        registerAction(DeleteStoredScriptAction.INSTANCE, TransportDeleteStoredScriptAction.class);
-
-        registerAction(FieldStatsAction.INSTANCE, TransportFieldStatsAction.class);
-
-        registerAction(PutPipelineAction.INSTANCE, PutPipelineTransportAction.class);
-        registerAction(GetPipelineAction.INSTANCE, GetPipelineTransportAction.class);
-        registerAction(DeletePipelineAction.INSTANCE, DeletePipelineTransportAction.class);
-        registerAction(SimulatePipelineAction.INSTANCE, SimulatePipelineTransportAction.class);
+        bind(DestructiveOperations.class).toInstance(destructiveOperations);
 
         // register Name -> GenericAction Map that can be injected to instances.
+        @SuppressWarnings("rawtypes")
         MapBinder<String, GenericAction> actionsBinder
                 = MapBinder.newMapBinder(binder(), String.class, GenericAction.class);
 
-        for (Map.Entry<String, ActionEntry> entry : actions.entrySet()) {
-            actionsBinder.addBinding(entry.getKey()).toInstance(entry.getValue().action);
+        for (Map.Entry<String, ActionHandler<?, ?>> entry : actions.entrySet()) {
+            actionsBinder.addBinding(entry.getKey()).toInstance(entry.getValue().getAction());
         }
         // register GenericAction -> transportAction Map that can be injected to instances.
         // also register any supporting classes
-        if (!proxy) {
+        if (false == transportClient) {
+            bind(AutoCreateIndex.class).toInstance(autoCreateIndex);
             bind(TransportLivenessAction.class).asEagerSingleton();
+            @SuppressWarnings("rawtypes")
             MapBinder<GenericAction, TransportAction> transportActionsBinder
                     = MapBinder.newMapBinder(binder(), GenericAction.class, TransportAction.class);
-            for (Map.Entry<String, ActionEntry> entry : actions.entrySet()) {
+            for (ActionHandler<?, ?> action : actions.values()) {
                 // bind the action as eager singleton, so the map binder one will reuse it
-                bind(entry.getValue().transportAction).asEagerSingleton();
-                transportActionsBinder.addBinding(entry.getValue().action).to(entry.getValue().transportAction).asEagerSingleton();
-                for (Class supportAction : entry.getValue().supportTransportActions) {
+                bind(action.getTransportAction()).asEagerSingleton();
+                transportActionsBinder.addBinding(action.getAction()).to(action.getTransportAction()).asEagerSingleton();
+                for (Class<?> supportAction : action.getSupportTransportActions()) {
                     bind(supportAction).asEagerSingleton();
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java b/core/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java
index 339abcb22bc..d4ddae78225 100644
--- a/core/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java
+++ b/core/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java
@@ -47,7 +47,6 @@ public final class AutoCreateIndex {
     private final IndexNameExpressionResolver resolver;
     private final AutoCreate autoCreate;
 
-    @Inject
     public AutoCreateIndex(Settings settings, IndexNameExpressionResolver resolver) {
         this.resolver = resolver;
         dynamicMappingDisabled = !MapperService.INDEX_MAPPER_DYNAMIC_SETTING.get(settings);
diff --git a/core/src/main/java/org/elasticsearch/action/support/DestructiveOperations.java b/core/src/main/java/org/elasticsearch/action/support/DestructiveOperations.java
index 31fc1d06175..56d5bf206f3 100644
--- a/core/src/main/java/org/elasticsearch/action/support/DestructiveOperations.java
+++ b/core/src/main/java/org/elasticsearch/action/support/DestructiveOperations.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.action.support;
 
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -38,7 +37,6 @@ public final class DestructiveOperations extends AbstractComponent {
         Setting.boolSetting("action.destructive_requires_name", false, Property.Dynamic, Property.NodeScope);
     private volatile boolean destructiveRequiresName;
 
-    @Inject
     public DestructiveOperations(Settings settings, ClusterSettings clusterSettings) {
         super(settings);
         destructiveRequiresName = REQUIRES_NAME_SETTING.get(settings);
diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
index f36fd199292..5786f6b1cfb 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
@@ -44,6 +44,7 @@ import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
+import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.search.SearchModule;
@@ -125,6 +126,15 @@ public class TransportClient extends AbstractClient {
             final NetworkService networkService = new NetworkService(settings);
             NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
             try {
+                final List<Setting<?>> additionalSettings = new ArrayList<>();
+                final List<String> additionalSettingsFilter = new ArrayList<>();
+                additionalSettings.addAll(pluginsService.getPluginSettings());
+                additionalSettingsFilter.addAll(pluginsService.getPluginSettingsFilter());
+                for (final ExecutorBuilder<?> builder : threadPool.builders()) {
+                    additionalSettings.addAll(builder.getRegisteredSettings());
+                }
+                SettingsModule settingsModule = new SettingsModule(settings, additionalSettings, additionalSettingsFilter);
+
                 ModulesBuilder modules = new ModulesBuilder();
                 // plugin modules must be added here, before others or we can get crazy injection errors...
                 for (Module pluginModule : pluginsService.nodeModules()) {
@@ -138,17 +148,10 @@ public class TransportClient extends AbstractClient {
                         // noop
                     }
                 });
-                modules.add(new ActionModule(false, true));
+                modules.add(new ActionModule(false, true, settings, null, settingsModule.getClusterSettings(),
+                        pluginsService.filterPlugins(ActionPlugin.class)));
 
                 pluginsService.processModules(modules);
-                final List<Setting<?>> additionalSettings = new ArrayList<>();
-                final List<String> additionalSettingsFilter = new ArrayList<>();
-                additionalSettings.addAll(pluginsService.getPluginSettings());
-                additionalSettingsFilter.addAll(pluginsService.getPluginSettingsFilter());
-                for (final ExecutorBuilder<?> builder : threadPool.builders()) {
-                    additionalSettings.addAll(builder.getRegisteredSettings());
-                }
-                SettingsModule settingsModule = new SettingsModule(settings, additionalSettings, additionalSettingsFilter);
                 CircuitBreakerService circuitBreakerService = Node.createCircuitBreakerService(settingsModule.getSettings(),
                     settingsModule.getClusterSettings());
                 resourcesToClose.add(circuitBreakerService);
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index cc11b58727d..47dcb6170bc 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -33,7 +33,6 @@ import org.elasticsearch.cluster.metadata.MetaDataMappingService;
 import org.elasticsearch.cluster.metadata.MetaDataUpdateSettingsService;
 import org.elasticsearch.cluster.node.DiscoveryNodeService;
 import org.elasticsearch.cluster.routing.DelayedAllocationService;
-import org.elasticsearch.cluster.routing.OperationRouting;
 import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
@@ -101,6 +100,7 @@ public class ClusterModule extends AbstractModule {
     private final ExtensionPoint.ClassSet<AllocationDecider> allocationDeciders = new ExtensionPoint.ClassSet<>("allocation_decider", AllocationDecider.class, AllocationDeciders.class);
     private final ExtensionPoint.ClassSet<IndexTemplateFilter> indexTemplateFilters = new ExtensionPoint.ClassSet<>("index_template_filter", IndexTemplateFilter.class);
     private final ClusterService clusterService;
+    private final IndexNameExpressionResolver indexNameExpressionResolver;
 
     // pkg private so tests can mock
     Class<? extends ClusterInfoService> clusterInfoServiceImpl = InternalClusterInfoService.class;
@@ -113,6 +113,7 @@ public class ClusterModule extends AbstractModule {
         registerShardsAllocator(ClusterModule.BALANCED_ALLOCATOR, BalancedShardsAllocator.class);
         registerShardsAllocator(ClusterModule.EVEN_SHARD_COUNT_ALLOCATOR, BalancedShardsAllocator.class);
         this.clusterService = clusterService;
+        indexNameExpressionResolver = new IndexNameExpressionResolver(settings);
     }
 
     public void registerAllocationDecider(Class<? extends AllocationDecider> allocationDecider) {
@@ -127,6 +128,10 @@ public class ClusterModule extends AbstractModule {
         indexTemplateFilters.registerExtension(indexTemplateFilter);
     }
 
+    public IndexNameExpressionResolver getIndexNameExpressionResolver() {
+        return indexNameExpressionResolver;
+    }
+
     @Override
     protected void configure() {
         // bind ShardsAllocator
@@ -151,7 +156,7 @@ public class ClusterModule extends AbstractModule {
         bind(MetaDataIndexAliasesService.class).asEagerSingleton();
         bind(MetaDataUpdateSettingsService.class).asEagerSingleton();
         bind(MetaDataIndexTemplateService.class).asEagerSingleton();
-        bind(IndexNameExpressionResolver.class).asEagerSingleton();
+        bind(IndexNameExpressionResolver.class).toInstance(indexNameExpressionResolver);
         bind(RoutingService.class).asEagerSingleton();
         bind(DelayedAllocationService.class).asEagerSingleton();
         bind(ShardStateAction.class).asEagerSingleton();
diff --git a/core/src/main/java/org/elasticsearch/common/NamedRegistry.java b/core/src/main/java/org/elasticsearch/common/NamedRegistry.java
new file mode 100644
index 00000000000..7573b5268fd
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/NamedRegistry.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * A registry from String to some class implementation. Used to ensure implementations are registered only once.
+ */
+public class NamedRegistry<T> {
+    private final Map<String, T> registry = new HashMap<>();
+    private final String targetName;
+
+    public NamedRegistry(String targetName) {
+        this.targetName = targetName;
+    }
+
+    public Map<String, T> getRegistry() {
+        return registry;
+    }
+
+    public void register(String name, T t) {
+        requireNonNull(name, "name is required");
+        requireNonNull(t, targetName + " is required");
+        if (registry.putIfAbsent(name, t) != null) {
+            throw new IllegalArgumentException(targetName + " for name " + name + " already registered");
+        }
+    }
+
+    public <P> void extractAndRegister(List<P> plugins, Function<P, Map<String, T>> lookup) {
+        for (P plugin : plugins) {
+            for (Map.Entry<String, T> entry : lookup.apply(plugin).entrySet()) {
+                register(entry.getKey(), entry.getValue());
+            }
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
index 12d0b8bbb6e..52647c8b8ff 100644
--- a/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
+++ b/core/src/main/java/org/elasticsearch/indices/analysis/AnalysisModule.java
@@ -21,6 +21,7 @@ package org.elasticsearch.indices.analysis;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.common.NamedRegistry;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexSettings;
@@ -143,12 +144,7 @@ import org.elasticsearch.index.analysis.compound.HyphenationCompoundWordTokenFil
 import org.elasticsearch.plugins.AnalysisPlugin;
 
 import java.io.IOException;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
-import java.util.function.Function;
-
-import static java.util.Objects.requireNonNull;
 
 /**
  * Sets up {@link AnalysisRegistry}.
@@ -170,12 +166,12 @@ public final class AnalysisModule {
     public AnalysisModule(Environment environment, List<AnalysisPlugin> plugins) throws IOException {
         NamedRegistry<AnalysisProvider<CharFilterFactory>> charFilters = setupCharFilters(plugins);
         NamedRegistry<org.apache.lucene.analysis.hunspell.Dictionary> hunspellDictionaries = setupHunspellDictionaries(plugins);
-        hunspellService = new HunspellService(environment.settings(), environment, hunspellDictionaries.registry);
+        hunspellService = new HunspellService(environment.settings(), environment, hunspellDictionaries.getRegistry());
         NamedRegistry<AnalysisProvider<TokenFilterFactory>> tokenFilters = setupTokenFilters(plugins, hunspellService);
         NamedRegistry<AnalysisProvider<TokenizerFactory>> tokenizers = setupTokenizers(plugins);
         NamedRegistry<AnalysisProvider<AnalyzerProvider<?>>> analyzers = setupAnalyzers(plugins);
-        analysisRegistry = new AnalysisRegistry(environment, charFilters.registry, tokenFilters.registry,
-                tokenizers.registry, analyzers.registry);
+        analysisRegistry = new AnalysisRegistry(environment, charFilters.getRegistry(), tokenFilters.getRegistry(),
+                tokenizers.getRegistry(), analyzers.getRegistry());
     }
 
     HunspellService getHunspellService() {
@@ -191,13 +187,13 @@ public final class AnalysisModule {
         charFilters.register("html_strip", HtmlStripCharFilterFactory::new);
         charFilters.register("pattern_replace", requriesAnalysisSettings(PatternReplaceCharFilterFactory::new));
         charFilters.register("mapping", requriesAnalysisSettings(MappingCharFilterFactory::new));
-        charFilters.registerPlugins(plugins, AnalysisPlugin::getCharFilters);
+        charFilters.extractAndRegister(plugins, AnalysisPlugin::getCharFilters);
         return charFilters;
     }
 
     public NamedRegistry<org.apache.lucene.analysis.hunspell.Dictionary> setupHunspellDictionaries(List<AnalysisPlugin> plugins) {
         NamedRegistry<org.apache.lucene.analysis.hunspell.Dictionary> hunspellDictionaries = new NamedRegistry<>("dictionary");
-        hunspellDictionaries.registerPlugins(plugins, AnalysisPlugin::getHunspellDictionaries);
+        hunspellDictionaries.extractAndRegister(plugins, AnalysisPlugin::getHunspellDictionaries);
         return hunspellDictionaries;
     }
 
@@ -262,7 +258,7 @@ public final class AnalysisModule {
         tokenFilters.register("classic", ClassicFilterFactory::new);
         tokenFilters.register("decimal_digit", DecimalDigitFilterFactory::new);
         tokenFilters.register("fingerprint", FingerprintTokenFilterFactory::new);
-        tokenFilters.registerPlugins(plugins, AnalysisPlugin::getTokenFilters);
+        tokenFilters.extractAndRegister(plugins, AnalysisPlugin::getTokenFilters);
         return tokenFilters;
     }
 
@@ -283,7 +279,7 @@ public final class AnalysisModule {
         tokenizers.register("pattern", PatternTokenizerFactory::new);
         tokenizers.register("classic", ClassicTokenizerFactory::new);
         tokenizers.register("thai", ThaiTokenizerFactory::new);
-        tokenizers.registerPlugins(plugins, AnalysisPlugin::getTokenizers);
+        tokenizers.extractAndRegister(plugins, AnalysisPlugin::getTokenizers);
         return tokenizers;
     }
 
@@ -333,7 +329,7 @@ public final class AnalysisModule {
         analyzers.register("turkish", TurkishAnalyzerProvider::new);
         analyzers.register("thai", ThaiAnalyzerProvider::new);
         analyzers.register("fingerprint", FingerprintAnalyzerProvider::new);
-        analyzers.registerPlugins(plugins, AnalysisPlugin::getAnalyzers);
+        analyzers.extractAndRegister(plugins, AnalysisPlugin::getAnalyzers);
         return analyzers;
     }
 
@@ -392,29 +388,4 @@ public final class AnalysisModule {
             return false;
         }
     }
-
-    private static class NamedRegistry<T> {
-        private final Map<String, T> registry = new HashMap<>();
-        private final String targetName;
-
-        public NamedRegistry(String targetName) {
-            this.targetName = targetName;
-        }
-
-        private void register(String name, T t) {
-            requireNonNull(name, "name is required");
-            requireNonNull(t, targetName + " is required");
-            if (registry.putIfAbsent(name, t) != null) {
-                throw new IllegalArgumentException(targetName + " for name " + name + " already registered");
-            }
-        }
-
-        private <P> void registerPlugins(List<P> plugins, Function<P, Map<String, T>> lookup) {
-            for (P plugin : plugins) {
-                for (Map.Entry<String, T> entry : lookup.apply(plugin).entrySet()) {
-                    register(entry.getKey(), entry.getValue());
-                }
-            }
-        }
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index f30fc9005b7..319cb14b110 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -86,8 +86,9 @@ import org.elasticsearch.monitor.MonitorService;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
 import org.elasticsearch.node.service.NodeService;
-import org.elasticsearch.plugins.MapperPlugin;
+import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.AnalysisPlugin;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.plugins.ScriptPlugin;
@@ -258,10 +259,13 @@ public class Node implements Closeable {
             modules.add(new NodeModule(this, monitorService));
             modules.add(new NetworkModule(networkService, settings, false, namedWriteableRegistry));
             modules.add(new DiscoveryModule(this.settings));
-            modules.add(new ClusterModule(this.settings, clusterService));
+            ClusterModule clusterModule = new ClusterModule(settings, clusterService);
+            modules.add(clusterModule);
             modules.add(new IndicesModule(namedWriteableRegistry, pluginsService.filterPlugins(MapperPlugin.class)));
             modules.add(new SearchModule(settings, namedWriteableRegistry));
-            modules.add(new ActionModule(DiscoveryNode.isIngestNode(settings), false));
+            modules.add(new ActionModule(DiscoveryNode.isIngestNode(settings), false, settings,
+                    clusterModule.getIndexNameExpressionResolver(), settingsModule.getClusterSettings(),
+                    pluginsService.filterPlugins(ActionPlugin.class)));
             modules.add(new GatewayModule());
             modules.add(new RepositoriesModule());
             pluginsService.processModules(modules);
diff --git a/core/src/main/java/org/elasticsearch/plugins/ActionPlugin.java b/core/src/main/java/org/elasticsearch/plugins/ActionPlugin.java
new file mode 100644
index 00000000000..0d4ee16ab89
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/plugins/ActionPlugin.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.GenericAction;
+import org.elasticsearch.action.support.ActionFilter;
+import org.elasticsearch.action.support.TransportAction;
+import org.elasticsearch.action.support.TransportActions;
+
+import java.util.List;
+
+import static java.util.Collections.emptyList;
+
+/**
+ * An additional extension point for {@link Plugin}s that extends Elasticsearch's scripting functionality. Implement it like this:
+ * <pre>{@code
+ *   {@literal @}Override
+ *   public List<ActionHandler<?, ?>> getActions() {
+ *       return Arrays.asList(new ActionHandler<>(ReindexAction.INSTANCE, TransportReindexAction.class),
+ *               new ActionHandler<>(UpdateByQueryAction.INSTANCE, TransportUpdateByQueryAction.class),
+ *               new ActionHandler<>(DeleteByQueryAction.INSTANCE, TransportDeleteByQueryAction.class),
+ *               new ActionHandler<>(RethrottleAction.INSTANCE, TransportRethrottleAction.class));
+ *   }
+ * }</pre>
+ */
+public interface ActionPlugin {
+    /**
+     * Actions added by this plugin.
+     */
+    default List<ActionHandler<? extends ActionRequest<?>, ? extends ActionResponse>> getActions() {
+        return emptyList();
+    }
+    /**
+     * Action filters added by this plugin.
+     */
+    default List<Class<? extends ActionFilter>> getActionFilters() {
+        return emptyList();
+    }
+
+    public static final class ActionHandler<Request extends ActionRequest<Request>, Response extends ActionResponse> {
+        private final GenericAction<Request, Response> action;
+        private final Class<? extends TransportAction<Request, Response>> transportAction;
+        private final Class<?>[] supportTransportActions;
+
+        /**
+         * Create a record of an action, the {@linkplain TransportAction} that handles it, and any supporting {@linkplain TransportActions}
+         * that are needed by that {@linkplain TransportAction}.
+         */
+        public ActionHandler(GenericAction<Request, Response> action, Class<? extends TransportAction<Request, Response>> transportAction,
+                Class<?>... supportTransportActions) {
+            this.action = action;
+            this.transportAction = transportAction;
+            this.supportTransportActions = supportTransportActions;
+        }
+
+        public GenericAction<Request, Response> getAction() {
+            return action;
+        }
+
+        public Class<? extends TransportAction<Request, Response>> getTransportAction() {
+            return transportAction;
+        }
+
+        public Class<?>[] getSupportTransportActions() {
+            return supportTransportActions;
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/plugins/Plugin.java b/core/src/main/java/org/elasticsearch/plugins/Plugin.java
index 08a8ce7124c..2b8f59d15f3 100644
--- a/core/src/main/java/org/elasticsearch/plugins/Plugin.java
+++ b/core/src/main/java/org/elasticsearch/plugins/Plugin.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.plugins;
 
+import org.elasticsearch.action.ActionModule;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.settings.Setting;
@@ -112,6 +113,14 @@ public abstract class Plugin {
     @Deprecated
     public final void onModule(AnalysisModule module) {}
 
+    /**
+     * Old-style action extension point.
+     *
+     * @deprecated implement {@link ActionPlugin} instead
+     */
+    @Deprecated
+    public final void onModule(ActionModule module) {}
+
     /**
      * Provides the list of this plugin's custom thread pools, empty if
      * none.
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
index 812691f064b..122ae910e7f 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java
@@ -20,8 +20,9 @@ package org.elasticsearch.action.admin.cluster.node.tasks;
 
 import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionModule;
+import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.support.ActionFilters;
@@ -46,6 +47,7 @@ import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
@@ -55,6 +57,7 @@ import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
@@ -64,11 +67,12 @@ import static org.elasticsearch.test.ESTestCase.awaitBusy;
 /**
  * A plugin that adds a cancellable blocking test task of integration testing of the task manager.
  */
-public class TestTaskPlugin extends Plugin {
+public class TestTaskPlugin extends Plugin implements ActionPlugin {
 
-    public void onModule(ActionModule module) {
-        module.registerAction(TestTaskAction.INSTANCE, TransportTestTaskAction.class);
-        module.registerAction(UnblockTestTasksAction.INSTANCE, TransportUnblockTestTasksAction.class);
+    @Override
+    public List<ActionHandler<? extends ActionRequest<?>, ? extends ActionResponse>> getActions() {
+        return Arrays.asList(new ActionHandler<>(TestTaskAction.INSTANCE, TransportTestTaskAction.class),
+                new ActionHandler<>(UnblockTestTasksAction.INSTANCE, TransportUnblockTestTasksAction.class));
     }
 
     static class TestTask extends CancellableTask {
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
index 27b9192006d..f0e45ff37bb 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterInfoServiceIT.java
@@ -22,7 +22,6 @@ package org.elasticsearch.cluster;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionModule;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsAction;
@@ -31,7 +30,6 @@ import org.elasticsearch.action.support.ActionFilter;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RoutingNodes;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -44,6 +42,7 @@ import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalTestCluster;
@@ -56,11 +55,13 @@ import org.hamcrest.Matchers;
 
 import java.io.IOException;
 import java.util.Collection;
+import java.util.List;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static java.util.Collections.emptySet;
+import static java.util.Collections.singletonList;
 import static java.util.Collections.unmodifiableSet;
 import static org.elasticsearch.common.util.set.Sets.newHashSet;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -74,10 +75,10 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class ClusterInfoServiceIT extends ESIntegTestCase {
 
-    public static class TestPlugin extends Plugin {
-
-        public void onModule(ActionModule module) {
-            module.registerFilter(BlockingActionFilter.class);
+    public static class TestPlugin extends Plugin implements ActionPlugin {
+        @Override
+        public List<Class<? extends ActionFilter>> getActionFilters() {
+            return singletonList(BlockingActionFilter.class);
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java b/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
index a5e2efb9f38..e9ec3787afc 100644
--- a/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
+++ b/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
@@ -21,7 +21,6 @@ package org.elasticsearch.transport;
 
 import org.apache.http.message.BasicHeader;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionModule;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
@@ -45,6 +44,7 @@ import org.elasticsearch.index.query.MoreLikeThisQueryBuilder.Item;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.TermsQueryBuilder;
 import org.elasticsearch.indices.TermsLookup;
+import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -61,6 +61,7 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.CopyOnWriteArrayList;
 
+import static java.util.Collections.singletonList;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.ESIntegTestCase.Scope.SUITE;
@@ -284,15 +285,16 @@ public class ContextAndHeaderTransportIT extends ESIntegTestCase {
         return internalCluster().transportClient().filterWithHeader(Collections.singletonMap(randomHeaderKey, randomHeaderValue));
     }
 
-    public static class ActionLoggingPlugin extends Plugin {
+    public static class ActionLoggingPlugin extends Plugin implements ActionPlugin {
 
         @Override
         public Collection<Module> nodeModules() {
             return Collections.<Module>singletonList(new ActionLoggingModule());
         }
 
-        public void onModule(ActionModule module) {
-            module.registerFilter(LoggingFilter.class);
+        @Override
+        public List<Class<? extends ActionFilter>> getActionFilters() {
+            return singletonList(LoggingFilter.class);
         }
     }
 
diff --git a/docs/reference/migration/migrate_5_0/plugins.asciidoc b/docs/reference/migration/migrate_5_0/plugins.asciidoc
index 8d35098deb9..e1e8e6c614b 100644
--- a/docs/reference/migration/migrate_5_0/plugins.asciidoc
+++ b/docs/reference/migration/migrate_5_0/plugins.asciidoc
@@ -133,4 +133,7 @@ Plugins that register custom analysis components should implement
 Plugins that register custom mappers should implement
 `MapperPlugin` and remove their `onModule(IndicesModule)` implementation.
 
+==== ActionPlugin
 
+Plugins that register custom actions should implement `ActionPlugin` and
+remove their `onModule(ActionModule)` implementation.
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
index 42f0da3c109..50bae57887c 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
@@ -19,13 +19,15 @@
 
 package org.elasticsearch.script.mustache;
 
-import org.elasticsearch.action.ActionModule;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.search.template.MultiSearchTemplateAction;
 import org.elasticsearch.action.search.template.SearchTemplateAction;
 import org.elasticsearch.action.search.template.TransportMultiSearchTemplateAction;
 import org.elasticsearch.action.search.template.TransportSearchTemplateAction;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.rest.action.search.template.RestDeleteSearchTemplateAction;
@@ -34,20 +36,22 @@ import org.elasticsearch.rest.action.search.template.RestMultiSearchTemplateActi
 import org.elasticsearch.rest.action.search.template.RestPutSearchTemplateAction;
 import org.elasticsearch.rest.action.search.template.RestRenderSearchTemplateAction;
 import org.elasticsearch.rest.action.search.template.RestSearchTemplateAction;
-import org.elasticsearch.script.ScriptEngineRegistry;
 import org.elasticsearch.script.ScriptEngineService;
-import org.elasticsearch.script.ScriptModule;
 
-public class MustachePlugin extends Plugin implements ScriptPlugin {
+import java.util.Arrays;
+import java.util.List;
+
+public class MustachePlugin extends Plugin implements ScriptPlugin, ActionPlugin {
 
     @Override
     public ScriptEngineService getScriptEngineService(Settings settings) {
         return new MustacheScriptEngineService(settings);
     }
 
-    public void onModule(ActionModule module) {
-        module.registerAction(SearchTemplateAction.INSTANCE, TransportSearchTemplateAction.class);
-        module.registerAction(MultiSearchTemplateAction.INSTANCE, TransportMultiSearchTemplateAction.class);
+    @Override
+    public List<ActionHandler<? extends ActionRequest<?>, ? extends ActionResponse>> getActions() {
+        return Arrays.asList(new ActionHandler<>(SearchTemplateAction.INSTANCE, TransportSearchTemplateAction.class),
+                new ActionHandler<>(MultiSearchTemplateAction.INSTANCE, TransportMultiSearchTemplateAction.class));
     }
 
     public void onModule(NetworkModule module) {
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
index 7cbf07e1981..87fc6726c56 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
@@ -19,23 +19,25 @@
 
 package org.elasticsearch.percolator;
 
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import org.elasticsearch.action.ActionModule;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.search.SearchModule;
 
-public class PercolatorPlugin extends Plugin implements MapperPlugin {
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class PercolatorPlugin extends Plugin implements MapperPlugin, ActionPlugin {
 
     public static final String NAME = "percolator";
 
@@ -47,9 +49,10 @@ public class PercolatorPlugin extends Plugin implements MapperPlugin {
         this.settings = settings;
     }
 
-    public void onModule(ActionModule module) {
-        module.registerAction(PercolateAction.INSTANCE, TransportPercolateAction.class);
-        module.registerAction(MultiPercolateAction.INSTANCE, TransportMultiPercolateAction.class);
+    @Override
+    public List<ActionHandler<? extends ActionRequest<?>, ? extends ActionResponse>> getActions() {
+        return Arrays.asList(new ActionHandler<>(PercolateAction.INSTANCE, TransportPercolateAction.class),
+                new ActionHandler<>(MultiPercolateAction.INSTANCE, TransportMultiPercolateAction.class));
     }
 
     public void onModule(NetworkModule module) {
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
index d4c81177571..89e6c1cc753 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
@@ -19,18 +19,24 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.action.ActionModule;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 
-public class ReindexPlugin extends Plugin {
+import java.util.Arrays;
+import java.util.List;
+
+public class ReindexPlugin extends Plugin implements ActionPlugin {
     public static final String NAME = "reindex";
 
-    public void onModule(ActionModule actionModule) {
-        actionModule.registerAction(ReindexAction.INSTANCE, TransportReindexAction.class);
-        actionModule.registerAction(UpdateByQueryAction.INSTANCE, TransportUpdateByQueryAction.class);
-        actionModule.registerAction(DeleteByQueryAction.INSTANCE, TransportDeleteByQueryAction.class);
-        actionModule.registerAction(RethrottleAction.INSTANCE, TransportRethrottleAction.class);
+    @Override
+    public List<ActionHandler<? extends ActionRequest<?>, ? extends ActionResponse>> getActions() {
+        return Arrays.asList(new ActionHandler<>(ReindexAction.INSTANCE, TransportReindexAction.class),
+                new ActionHandler<>(UpdateByQueryAction.INSTANCE, TransportUpdateByQueryAction.class),
+                new ActionHandler<>(DeleteByQueryAction.INSTANCE, TransportDeleteByQueryAction.class),
+                new ActionHandler<>(RethrottleAction.INSTANCE, TransportRethrottleAction.class));
     }
 
     public void onModule(NetworkModule networkModule) {

From ef0e3db0de6928ddca70d18e929223980f283fe1 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Tue, 28 Jun 2016 12:16:17 +0200
Subject: [PATCH 1094/1311] Validates new dynamic settings from the current
 state

Thanks to https://github.com/elastic/elasticsearch/pull/19088 the settings are now validated against dynamic updaters on the master.
Though only the new settings are applied to the IndexService created for the validation.
Because of this we cannot check the transition from one value to another in a dynamic updaters.
This change creates the IndexService from the current settings and validates that the new dynamic settings
can replace the current settings.
This change also removes the validation of dynamic settings when an index is opened.
The validation should have occurred when the settings have been updated.
---
 .../cluster/metadata/MetaDataIndexStateService.java  |  2 +-
 .../metadata/MetaDataUpdateSettingsService.java      | 10 +++++++---
 .../main/java/org/elasticsearch/gateway/Gateway.java |  2 +-
 .../org/elasticsearch/indices/IndicesService.java    | 12 ++++++++----
 4 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java
index e6e7084e4d9..53a0ede809a 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateService.java
@@ -172,7 +172,7 @@ public class MetaDataIndexStateService extends AbstractComponent {
                     // We need to check that this index can be upgraded to the current version
                     indexMetaData = metaDataIndexUpgradeService.upgradeIndexMetaData(indexMetaData);
                     try {
-                        indicesService.verifyIndexMetadata(nodeServiceProvider, indexMetaData);
+                        indicesService.verifyIndexMetadata(nodeServiceProvider, indexMetaData, indexMetaData);
                     } catch (Exception e) {
                         throw new ElasticsearchException("Failed to verify index " + indexMetaData.getIndex(), e);
                     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
index 54f6ad0705a..8bf7b5edd94 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
@@ -275,13 +275,17 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
                 updatedState = ClusterState.builder(updatedState).routingResult(routingResult).build();
                 try {
                     for (Index index : openIndices) {
-                        indicesService.verifyIndexMetadata(nodeServiceProvider, updatedState.getMetaData().getIndexSafe(index));
+                        final IndexMetaData currentMetaData = currentState.getMetaData().getIndexSafe(index);
+                        final IndexMetaData updatedMetaData = updatedState.metaData().getIndexSafe(index);
+                        indicesService.verifyIndexMetadata(nodeServiceProvider, currentMetaData, updatedMetaData);
                     }
                     for (Index index : closeIndices) {
-                        indicesService.verifyIndexMetadata(nodeServiceProvider, updatedState.getMetaData().getIndexSafe(index));
+                        final IndexMetaData currentMetaData = currentState.getMetaData().getIndexSafe(index);
+                        final IndexMetaData updatedMetaData = updatedState.metaData().getIndexSafe(index);
+                        indicesService.verifyIndexMetadata(nodeServiceProvider, currentMetaData, updatedMetaData);
                     }
                 } catch (IOException ex) {
-                    ExceptionsHelper.convertToElastic(ex);
+                    throw ExceptionsHelper.convertToElastic(ex);
                 }
                 return updatedState;
             }
diff --git a/core/src/main/java/org/elasticsearch/gateway/Gateway.java b/core/src/main/java/org/elasticsearch/gateway/Gateway.java
index af86633e235..d83a738aedf 100644
--- a/core/src/main/java/org/elasticsearch/gateway/Gateway.java
+++ b/core/src/main/java/org/elasticsearch/gateway/Gateway.java
@@ -141,7 +141,7 @@ public class Gateway extends AbstractComponent implements ClusterStateListener {
                     try {
                         if (electedIndexMetaData.getState() == IndexMetaData.State.OPEN) {
                             // verify that we can actually create this index - if not we recover it as closed with lots of warn logs
-                            indicesService.verifyIndexMetadata(nodeServicesProvider, electedIndexMetaData);
+                            indicesService.verifyIndexMetadata(nodeServicesProvider, electedIndexMetaData, electedIndexMetaData);
                         }
                     } catch (Exception e) {
                         logger.warn("recovering index {} failed - recovering as closed", e, electedIndexMetaData.getIndex());
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index 89cfad98c4f..b8b9b827e4b 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -412,10 +412,12 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService>
     }
 
     /**
-     * This method verifies that the given {@link IndexMetaData} holds sane values to create an {@link IndexService}. This method will throw an
-     * exception if the creation fails. The created {@link IndexService} will not be registered and will be closed immediately.
+     * This method verifies that the given {@code metaData} holds sane values to create an {@link IndexService}.
+     * This method tries to update the meta data of the created {@link IndexService} if the given {@code metaDataUpdate} is different from the given {@code metaData}.
+     * This method will throw an exception if the creation or the update fails.
+     * The created {@link IndexService} will not be registered and will be closed immediately.
      */
-    public synchronized void verifyIndexMetadata(final NodeServicesProvider nodeServicesProvider, IndexMetaData metaData) throws IOException {
+    public synchronized void verifyIndexMetadata(final NodeServicesProvider nodeServicesProvider, IndexMetaData metaData, IndexMetaData metaDataUpdate) throws IOException {
         final List<Closeable> closeables = new ArrayList<>();
         try {
             IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() {});
@@ -431,7 +433,9 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService>
                 service.mapperService().merge(typeMapping.value.type(), typeMapping.value.source(),
                     MapperService.MergeReason.MAPPING_RECOVERY, true);
             }
-            service.getIndexSettings().getScopedSettings().validateUpdate(metaData.getSettings());
+            if (metaData.equals(metaDataUpdate) == false) {
+                service.updateMetaData(metaDataUpdate);
+            }
         } finally {
             IOUtils.close(closeables);
         }

From 6d069078d34cbc854a24c913e340485dc652d347 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Tue, 28 Jun 2016 16:14:57 +0200
Subject: [PATCH 1095/1311] Fixed tests that assumed that broken settings can
 be updated

---
 .../elasticsearch/gateway/GatewayIndexStateIT.java | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
index d5dee3c1bdc..04810e6ebfd 100644
--- a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
+++ b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
@@ -446,13 +446,6 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
         assertNotNull(ex.getCause());
         assertEquals(IllegalArgumentException.class, ex.getCause().getClass());
         assertEquals(ex.getCause().getMessage(), "Unknown tokenfilter type [icu_collation] for [myCollator]");
-
-        client().admin().indices().prepareUpdateSettings()
-            .setSettings(Settings.builder().putNull("index.analysis.filter.myCollator.type")).get();
-        client().admin().indices().prepareOpen("test").get();
-        ensureYellow();
-        logger.info("--> verify 1 doc in the index");
-        assertHitCount(client().prepareSearch().setQuery(matchAllQuery()).get(), 1L);
     }
 
     /**
@@ -510,13 +503,6 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
         assertNotNull(ex.getCause());
         assertEquals(MapperParsingException.class, ex.getCause().getClass());
         assertEquals(ex.getCause().getMessage(), "analyzer [test] not found for field [field1]");
-
-        client().admin().indices().prepareUpdateSettings()
-            .setSettings(Settings.builder().put("index.analysis.analyzer.test.tokenizer", "keyword")).get();
-        client().admin().indices().prepareOpen("test").get();
-        ensureYellow();
-        logger.info("--> verify 1 doc in the index");
-        assertHitCount(client().prepareSearch().setQuery(matchQuery("field1", "value one")).get(), 1L);
     }
 
     public void testArchiveBrokenClusterSettings() throws Exception {

From 2512594d9ec6a979b525038ba60ffdeb8af0eae8 Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Tue, 28 Jun 2016 16:38:56 +0200
Subject: [PATCH 1096/1311] Testing infra - stablize data folder usage and
 clean up (#19111)

The plan for persistent node ids ( #17811 ) is to tie the node identity to a file stored in it's data folders. As such it becomes important that nodes in our testing infra have better affinity with their data folders and that their data folders are not cleaned underneath them. The first is important because we fix the random seed used for node id generation (for reproducibility) and allowing the same node to use two different data folders causes two separate nodes to have the same id, which prevents the cluster from forming. The second is important, for example, where a full cluster restart / single node restart need to maintain node identity and wiping the data folders at the wrong moment prevents this.

Concretely this commit does the following:
1) Remove previous attempts to have data folder per role using a prefix. This wasn't effective as it was using the data paths settings which are only used for part of the runs. An attempt to completely separate the paths via the home dir failed due to assumptions made by index custom path about node data folder ordinal uniqueness (see #19076)
2) Change full cluster restarts to start up nodes in the same order their were first created in, only randomly swapping nodes with the same roles.
3) Change test cluster reset methods to first shutdown the unneeded nodes and then re-start the shared nodes that were shut down, so they'll reclaim their data folders.
4) Improve data folder wiping logic and make sure it wipes only folders of "offline" nodes.
5) Add some very basic tests
---
 .../org/elasticsearch/gateway/Gateway.java    |  11 -
 .../java/org/elasticsearch/node/Node.java     |  10 +-
 .../discovery/zen/ZenDiscoveryIT.java         |   2 +-
 .../gateway/GatewayIndexStateIT.java          |   9 +-
 .../test/InternalTestCluster.java             | 274 ++++++++++--------
 .../test/test/InternalTestClusterTests.java   | 152 ++++++++++
 6 files changed, 314 insertions(+), 144 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/gateway/Gateway.java b/core/src/main/java/org/elasticsearch/gateway/Gateway.java
index d83a738aedf..919db87d386 100644
--- a/core/src/main/java/org/elasticsearch/gateway/Gateway.java
+++ b/core/src/main/java/org/elasticsearch/gateway/Gateway.java
@@ -21,7 +21,6 @@ package org.elasticsearch.gateway;
 
 import com.carrotsearch.hppc.ObjectFloatHashMap;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
-import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
@@ -38,7 +37,6 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.indices.IndicesService;
 
-import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.function.Supplier;
 
@@ -159,15 +157,6 @@ public class Gateway extends AbstractComponent implements ClusterStateListener {
         builder.metaData(metaDataBuilder);
         listener.onSuccess(builder.build());
     }
-    public void reset() throws Exception {
-        try {
-            Path[] dataPaths = nodeEnv.nodeDataPaths();
-            logger.trace("removing node data paths: [{}]", (Object)dataPaths);
-            IOUtils.rm(dataPaths);
-        } catch (Exception ex) {
-            logger.debug("failed to delete shard locations", ex);
-        }
-    }
 
     @Override
     public void clusterChanged(final ClusterChangedEvent event) {
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index 319cb14b110..675d29e9991 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -163,6 +163,7 @@ public class Node implements Closeable {
     private final Injector injector;
     private final Settings settings;
     private final Environment environment;
+    private final NodeEnvironment nodeEnvironment;
     private final PluginsService pluginsService;
     private final Client client;
 
@@ -236,7 +237,6 @@ public class Node implements Closeable {
             // this is as early as we can validate settings at this point. we already pass them to ScriptModule as well as ThreadPool
             // so we might be late here already
             final SettingsModule settingsModule = new SettingsModule(this.settings, additionalSettings, additionalSettingsFilter);
-            final NodeEnvironment nodeEnvironment;
             try {
                 nodeEnvironment = new NodeEnvironment(this.settings, this.environment);
                 resourcesToClose.add(nodeEnvironment);
@@ -324,6 +324,14 @@ public class Node implements Closeable {
         return environment;
     }
 
+    /**
+     * Returns the {@link NodeEnvironment} instance of this node
+     */
+    public NodeEnvironment getNodeEnvironment() {
+        return nodeEnvironment;
+    }
+
+
     /**
      * Start the node. If the node is already started, this method is no-op.
      */
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java
index c5f92e62de0..fd0b11eae01 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java
@@ -297,7 +297,7 @@ public class ZenDiscoveryIT extends ESIntegTestCase {
         Settings nodeSettings = Settings.builder()
                 .put("discovery.type", "zen") // <-- To override the local setting if set externally
                 .build();
-        String nodeName = internalCluster().startNode(nodeSettings, Version.CURRENT);
+        String nodeName = internalCluster().startNode(nodeSettings);
         ZenDiscovery zenDiscovery = (ZenDiscovery) internalCluster().getInstance(Discovery.class, nodeName);
         ClusterService clusterService = internalCluster().getInstance(ClusterService.class, nodeName);
         DiscoveryNode node = new DiscoveryNode("_node_id", new InetSocketTransportAddress(InetAddress.getByName("0.0.0.0"), 0),
diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
index 04810e6ebfd..049fd90cad6 100644
--- a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
+++ b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
@@ -298,15 +298,10 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
         assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(true));
 
         logger.info("--> restarting the nodes");
-        final Gateway gateway1 = internalCluster().getInstance(GatewayService.class, node_1).getGateway();
         internalCluster().fullRestart(new RestartCallback() {
             @Override
-            public Settings onNodeStopped(String nodeName) throws Exception {
-                if (node_1.equals(nodeName)) {
-                    logger.info("--> deleting the data for the first node");
-                    gateway1.reset();
-                }
-                return null;
+            public boolean clearData(String nodeName) {
+                return node_1.equals(nodeName);
             }
         });
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index 9a00fbc4557..d2014ebbd6e 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -28,6 +28,7 @@ import org.apache.lucene.store.StoreRateLimiting;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
@@ -38,6 +39,7 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNode.Role;
 import org.elasticsearch.cluster.node.DiscoveryNodeService;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.OperationRouting;
@@ -367,15 +369,6 @@ public final class InternalTestCluster extends TestCluster {
     private Settings getSettings(int nodeOrdinal, long nodeSeed, Settings others) {
         Builder builder = Settings.builder().put(defaultSettings)
             .put(getRandomNodeSettings(nodeSeed));
-        Settings interimSettings = builder.build();
-        final String dataSuffix = getRoleSuffix(interimSettings);
-        if (dataSuffix.isEmpty() == false) {
-            // to make sure that a master node will not pick up on the data folder of a data only node
-            // once restarted we append the role suffix to each path.
-            String[] dataPath = Environment.PATH_DATA_SETTING.get(interimSettings).stream()
-                .map(path -> path + dataSuffix).toArray(String[]::new);
-            builder.putArray(Environment.PATH_DATA_SETTING.getKey(), dataPath);
-        }
         Settings settings = nodeConfigurationSource.nodeSettings(nodeOrdinal);
         if (settings != null) {
             if (settings.get(ClusterName.CLUSTER_NAME_SETTING.getKey()) != null) {
@@ -498,7 +491,7 @@ public final class InternalTestCluster extends TestCluster {
             return randomNodeAndClient;
         }
         NodeAndClient buildNode = buildNode();
-        buildNode.node().start();
+        buildNode.startNode();
         publishNode(buildNode);
         return buildNode;
     }
@@ -569,7 +562,7 @@ public final class InternalTestCluster extends TestCluster {
             n == 0 ? nodes.values().stream() : nodes.values().stream().filter(new DataNodePredicate().and(new MasterNodePredicate(getMasterName()).negate()));
         final Iterator<NodeAndClient> values = collection.iterator();
 
-        logger.info("changing cluster size from {} to {}, {} data nodes", size(), n + numSharedCoordOnlyNodes, n);
+        logger.info("changing cluster size from {} data nodes to {}", size, n);
         Set<NodeAndClient> nodesToRemove = new HashSet<>();
         int numNodesAndClients = 0;
         while (values.hasNext() && numNodesAndClients++ < size - n) {
@@ -586,17 +579,17 @@ public final class InternalTestCluster extends TestCluster {
         }
     }
 
-    private NodeAndClient buildNode(Settings settings, Version version) {
+    private NodeAndClient buildNode(Settings settings) {
         int ord = nextNodeId.getAndIncrement();
-        return buildNode(ord, random.nextLong(), settings, version, false);
+        return buildNode(ord, random.nextLong(), settings, false);
     }
 
     private NodeAndClient buildNode() {
         int ord = nextNodeId.getAndIncrement();
-        return buildNode(ord, random.nextLong(), null, Version.CURRENT, false);
+        return buildNode(ord, random.nextLong(), null, false);
     }
 
-    private NodeAndClient buildNode(int nodeId, long seed, Settings settings, Version version, boolean reuseExisting) {
+    private NodeAndClient buildNode(int nodeId, long seed, Settings settings, boolean reuseExisting) {
         assert Thread.holdsLock(this);
         ensureOpen();
         settings = getSettings(nodeId, seed, settings);
@@ -615,7 +608,7 @@ public final class InternalTestCluster extends TestCluster {
             .put(DiscoveryNodeService.NODE_ID_SEED_SETTING.getKey(), seed)
             .build();
         MockNode node = new MockNode(finalSettings, plugins);
-        return new NodeAndClient(name, node);
+        return new NodeAndClient(name, node, nodeId);
     }
 
     private String buildNodeName(int id, Settings settings) {
@@ -630,10 +623,10 @@ public final class InternalTestCluster extends TestCluster {
     private String getRoleSuffix(Settings settings) {
         String suffix = "";
         if (Node.NODE_MASTER_SETTING.exists(settings) && Node.NODE_MASTER_SETTING.get(settings)) {
-            suffix = suffix + DiscoveryNode.Role.MASTER.getAbbreviation();
+            suffix = suffix + Role.MASTER.getAbbreviation();
         }
         if (Node.NODE_DATA_SETTING.exists(settings) && Node.NODE_DATA_SETTING.get(settings)) {
-            suffix = suffix + DiscoveryNode.Role.DATA.getAbbreviation();
+            suffix = suffix + Role.DATA.getAbbreviation();
         }
         if (Node.NODE_MASTER_SETTING.exists(settings) && Node.NODE_MASTER_SETTING.get(settings) == false &&
             Node.NODE_DATA_SETTING.exists(settings) && Node.NODE_DATA_SETTING.get(settings) == false
@@ -709,7 +702,7 @@ public final class InternalTestCluster extends TestCluster {
         return getRandomNodeAndClient(new NoDataNoMasterNodePredicate()).client(random);
     }
 
-    public synchronized Client startCoordinatingOnlyNode(Settings settings) {
+    public synchronized String startCoordinatingOnlyNode(Settings settings) {
         ensureOpen(); // currently unused
         Builder builder = Settings.builder().put(settings).put(Node.NODE_MASTER_SETTING.getKey(), false)
             .put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_INGEST_SETTING.getKey(), false);
@@ -717,8 +710,7 @@ public final class InternalTestCluster extends TestCluster {
             // if we are the first node - don't wait for a state
             builder.put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), 0);
         }
-        String name = startNode(builder);
-        return nodes.get(name).nodeClient();
+        return startNode(builder);
     }
 
     /**
@@ -771,7 +763,7 @@ public final class InternalTestCluster extends TestCluster {
     }
 
     @Override
-    public void close() {
+    public synchronized void close() {
         if (this.open.compareAndSet(true, false)) {
             if (activeDisruptionScheme != null) {
                 activeDisruptionScheme.testClusterClosed();
@@ -793,10 +785,13 @@ public final class InternalTestCluster extends TestCluster {
         private Client transportClient;
         private final AtomicBoolean closed = new AtomicBoolean(false);
         private final String name;
+        private final int nodeAndClientId;
 
-        NodeAndClient(String name, MockNode node) {
+        NodeAndClient(String name, MockNode node, int nodeAndClientId) {
             this.node = node;
             this.name = name;
+            this.nodeAndClientId = nodeAndClientId;
+            markNodeDataDirsAsNotEligableForWipe(node);
         }
 
         Node node() {
@@ -806,6 +801,10 @@ public final class InternalTestCluster extends TestCluster {
             return node;
         }
 
+        public int nodeAndClientId() {
+            return nodeAndClientId;
+        }
+
         Client client(Random random) {
             if (closed.get()) {
                 throw new RuntimeException("already closed");
@@ -860,12 +859,16 @@ public final class InternalTestCluster extends TestCluster {
             }
         }
 
+        void startNode() {
+            node.start();
+        }
+
         void closeNode() throws IOException {
-            registerDataPath();
+            markNodeDataDirsAsPendingForWipe(node);
             node.close();
         }
 
-        void restart(RestartCallback callback) throws Exception {
+        void restart(RestartCallback callback, boolean clearDataIfNeeded) throws Exception {
             assert callback != null;
             resetClient();
             if (!node.isClosed()) {
@@ -875,31 +878,32 @@ public final class InternalTestCluster extends TestCluster {
             if (newSettings == null) {
                 newSettings = Settings.EMPTY;
             }
-            if (callback.clearData(name)) {
-                NodeEnvironment nodeEnv = getInstanceFromNode(NodeEnvironment.class, node);
-                if (nodeEnv.hasNodeFile()) {
-                    IOUtils.rm(nodeEnv.nodeDataPaths());
-                }
+            if (clearDataIfNeeded) {
+                clearDataIfNeeded(callback);
             }
-            startNewNode(newSettings);
+            createNewNode(newSettings);
+            startNode();
         }
 
-        private void startNewNode(final Settings newSettings) {
+        private void clearDataIfNeeded(RestartCallback callback) throws IOException {
+            if (callback.clearData(name)) {
+                NodeEnvironment nodeEnv = node.getNodeEnvironment();
+                if (nodeEnv.hasNodeFile()) {
+                    final Path[] locations = nodeEnv.nodeDataPaths();
+                    logger.debug("removing node data paths: [{}]", (Object[]) locations);
+                    IOUtils.rm(locations);
+                }
+            }
+        }
+
+        private void createNewNode(final Settings newSettings) {
             final long newIdSeed = DiscoveryNodeService.NODE_ID_SEED_SETTING.get(node.settings()) + 1; // use a new seed to make sure we have new node id
             Settings finalSettings = Settings.builder().put(node.settings()).put(newSettings).put(DiscoveryNodeService.NODE_ID_SEED_SETTING.getKey(), newIdSeed).build();
             Collection<Class<? extends Plugin>> plugins = node.getPlugins();
             node = new MockNode(finalSettings, plugins);
-            node.start();
+            markNodeDataDirsAsNotEligableForWipe(node);
         }
 
-        void registerDataPath() {
-            NodeEnvironment nodeEnv = getInstanceFromNode(NodeEnvironment.class, node);
-            if (nodeEnv.hasNodeFile()) {
-                dataDirToClean.addAll(Arrays.asList(nodeEnv.nodeDataPaths()));
-            }
-        }
-
-
         @Override
         public void close() throws IOException {
             try {
@@ -972,25 +976,42 @@ public final class InternalTestCluster extends TestCluster {
             }
         }
         randomlyResetClients();
-        if (wipeData) {
-            wipeDataDirectories();
-        }
-        if (nextNodeId.get() == sharedNodesSeeds.length && nodes.size() == sharedNodesSeeds.length) {
-            logger.debug("Cluster hasn't changed - moving out - nodes: [{}] nextNodeId: [{}] numSharedNodes: [{}]", nodes.keySet(), nextNodeId.get(), sharedNodesSeeds.length);
+        final int newSize = sharedNodesSeeds.length;
+        if (nextNodeId.get() == newSize && nodes.size() == newSize) {
+            if (wipeData) {
+                wipePendingDataDirectories();
+            }
+            logger.debug("Cluster hasn't changed - moving out - nodes: [{}] nextNodeId: [{}] numSharedNodes: [{}]", nodes.keySet(), nextNodeId.get(), newSize);
             return;
         }
-        logger.debug("Cluster is NOT consistent - restarting shared nodes - nodes: [{}] nextNodeId: [{}] numSharedNodes: [{}]", nodes.keySet(), nextNodeId.get(), sharedNodesSeeds.length);
+        logger.debug("Cluster is NOT consistent - restarting shared nodes - nodes: [{}] nextNodeId: [{}] numSharedNodes: [{}]", nodes.keySet(), nextNodeId.get(), newSize);
+
+        // trash all nodes with id >= sharedNodesSeeds.length - they are non shared
 
 
-        Set<NodeAndClient> sharedNodes = new HashSet<>();
-        assert sharedNodesSeeds.length == numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes;
+        for (Iterator<NodeAndClient> iterator = nodes.values().iterator(); iterator.hasNext();) {
+            NodeAndClient nodeAndClient = iterator.next();
+            if (nodeAndClient.nodeAndClientId() >= sharedNodesSeeds.length) {
+                logger.debug("Close Node [{}] not shared", nodeAndClient.name);
+                nodeAndClient.close();
+                iterator.remove();
+            }
+        }
+
+        // clean up what the nodes left that is unused
+        if (wipeData) {
+            wipePendingDataDirectories();
+        }
+
+        // start any missing node
+        assert newSize == numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes;
         for (int i = 0; i < numSharedDedicatedMasterNodes; i++) {
             final Settings.Builder settings = Settings.builder();
             settings.put(Node.NODE_MASTER_SETTING.getKey(), true).build();
             settings.put(Node.NODE_DATA_SETTING.getKey(), false).build();
-            NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), Version.CURRENT, true);
-            nodeAndClient.node().start();
-            sharedNodes.add(nodeAndClient);
+            NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), true);
+            nodeAndClient.startNode();
+            publishNode(nodeAndClient);
         }
         for (int i = numSharedDedicatedMasterNodes; i < numSharedDedicatedMasterNodes + numSharedDataNodes; i++) {
             final Settings.Builder settings = Settings.builder();
@@ -999,44 +1020,35 @@ public final class InternalTestCluster extends TestCluster {
                 settings.put(Node.NODE_MASTER_SETTING.getKey(), false).build();
                 settings.put(Node.NODE_DATA_SETTING.getKey(), true).build();
             }
-            NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), Version.CURRENT, true);
-            nodeAndClient.node().start();
-            sharedNodes.add(nodeAndClient);
+            NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), true);
+            nodeAndClient.startNode();
+            publishNode(nodeAndClient);
         }
         for (int i = numSharedDedicatedMasterNodes + numSharedDataNodes;
              i < numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes; i++) {
             final Builder settings = Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false)
                 .put(Node.NODE_DATA_SETTING.getKey(), false).put(Node.NODE_INGEST_SETTING.getKey(), false);
-            NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), Version.CURRENT, true);
-            nodeAndClient.node().start();
-            sharedNodes.add(nodeAndClient);
-        }
-
-        for (NodeAndClient nodeAndClient : sharedNodes) {
-            nodes.remove(nodeAndClient.name);
-        }
-
-        // trash the remaining nodes
-        final Collection<NodeAndClient> toShutDown = nodes.values();
-        for (NodeAndClient nodeAndClient : toShutDown) {
-            logger.debug("Close Node [{}] not shared", nodeAndClient.name);
-            nodeAndClient.close();
-        }
-        nodes.clear();
-        for (NodeAndClient nodeAndClient : sharedNodes) {
+            NodeAndClient nodeAndClient = buildNode(i, sharedNodesSeeds[i], settings.build(), true);
+            nodeAndClient.startNode();
             publishNode(nodeAndClient);
         }
-        nextNodeId.set(sharedNodesSeeds.length);
-        assert size() == sharedNodesSeeds.length;
-        if (size() > 0) {
-            client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(sharedNodesSeeds.length)).get();
+
+        nextNodeId.set(newSize);
+        assert size() == newSize;
+        if (newSize > 0) {
+            ClusterHealthResponse response = client().admin().cluster().prepareHealth()
+                .setWaitForNodes(Integer.toString(newSize)).get();
+            if (response.isTimedOut()) {
+                logger.warn("failed to wait for a cluster of size [{}], got", newSize, response);
+                throw new IllegalStateException("cluster failed to reach the expected size of [" + newSize + "]");
+            }
         }
-        logger.debug("Cluster is consistent again - nodes: [{}] nextNodeId: [{}] numSharedNodes: [{}]", nodes.keySet(), nextNodeId.get(), sharedNodesSeeds.length);
+        logger.debug("Cluster is consistent again - nodes: [{}] nextNodeId: [{}] numSharedNodes: [{}]", nodes.keySet(), nextNodeId.get(), newSize);
     }
 
     @Override
     public synchronized void afterTest() throws IOException {
-        wipeDataDirectories();
+        wipePendingDataDirectories();
         randomlyResetClients(); /* reset all clients - each test gets its own client based on the Random instance created above. */
     }
 
@@ -1098,7 +1110,8 @@ public final class InternalTestCluster extends TestCluster {
         }
     }
 
-    private void wipeDataDirectories() {
+    private void wipePendingDataDirectories() {
+        assert Thread.holdsLock(this);
         if (!dataDirToClean.isEmpty()) {
             try {
                 for (Path path : dataDirToClean) {
@@ -1115,6 +1128,22 @@ public final class InternalTestCluster extends TestCluster {
         }
     }
 
+    private void markNodeDataDirsAsPendingForWipe(Node node) {
+        assert Thread.holdsLock(this);
+        NodeEnvironment nodeEnv = node.getNodeEnvironment();
+        if (nodeEnv.hasNodeFile()) {
+            dataDirToClean.addAll(Arrays.asList(nodeEnv.nodeDataPaths()));
+        }
+    }
+
+    private void markNodeDataDirsAsNotEligableForWipe(Node node) {
+        assert Thread.holdsLock(this);
+        NodeEnvironment nodeEnv = node.getNodeEnvironment();
+        if (nodeEnv.hasNodeFile()) {
+            dataDirToClean.removeAll(Arrays.asList(nodeEnv.nodeDataPaths()));
+        }
+    }
+
     /**
      * Returns a reference to a random node's {@link ClusterService}
      */
@@ -1254,7 +1283,7 @@ public final class InternalTestCluster extends TestCluster {
     /**
      * Stops any of the current nodes but not the master node.
      */
-    public void stopRandomNonMasterNode() throws IOException {
+    public synchronized void stopRandomNonMasterNode() throws IOException {
         NodeAndClient nodeAndClient = getRandomNodeAndClient(new MasterNodePredicate(getMasterName()).negate());
         if (nodeAndClient != null) {
             logger.info("Closing random non master node [{}] current master [{}] ", nodeAndClient.name, getMasterName());
@@ -1295,28 +1324,28 @@ public final class InternalTestCluster extends TestCluster {
     /**
      * Restarts a random node in the cluster and calls the callback during restart.
      */
-    private void restartRandomNode(Predicate<NodeAndClient> predicate, RestartCallback callback) throws Exception {
+    private synchronized void restartRandomNode(Predicate<NodeAndClient> predicate, RestartCallback callback) throws Exception {
         ensureOpen();
         NodeAndClient nodeAndClient = getRandomNodeAndClient(predicate);
         if (nodeAndClient != null) {
             logger.info("Restarting random node [{}] ", nodeAndClient.name);
-            nodeAndClient.restart(callback);
+            nodeAndClient.restart(callback, true);
         }
     }
 
     /**
      * Restarts a node and calls the callback during restart.
      */
-    public void restartNode(String nodeName, RestartCallback callback) throws Exception {
+    synchronized public void restartNode(String nodeName, RestartCallback callback) throws Exception {
         ensureOpen();
         NodeAndClient nodeAndClient = nodes.get(nodeName);
         if (nodeAndClient != null) {
             logger.info("Restarting node [{}] ", nodeAndClient.name);
-            nodeAndClient.restart(callback);
+            nodeAndClient.restart(callback, true);
         }
     }
 
-    private void restartAllNodes(boolean rollingRestart, RestartCallback callback) throws Exception {
+    synchronized private void restartAllNodes(boolean rollingRestart, RestartCallback callback) throws Exception {
         ensureOpen();
         List<NodeAndClient> toRemove = new ArrayList<>();
         try {
@@ -1344,13 +1373,15 @@ public final class InternalTestCluster extends TestCluster {
                 if (activeDisruptionScheme != null) {
                     activeDisruptionScheme.removeFromNode(nodeAndClient.name, this);
                 }
-                nodeAndClient.restart(callback);
+                nodeAndClient.restart(callback, true);
                 if (activeDisruptionScheme != null) {
                     activeDisruptionScheme.applyToNode(nodeAndClient.name, this);
                 }
             }
         } else {
             int numNodesRestarted = 0;
+            Set[] nodesRoleOrder = new Set[nextNodeId.get()];
+            Map<Set<Role>, List<NodeAndClient>> nodesByRoles = new HashMap<>();
             for (NodeAndClient nodeAndClient : nodes.values()) {
                 callback.doAfterNodes(numNodesRestarted++, nodeAndClient.nodeClient());
                 logger.info("Stopping node [{}] ", nodeAndClient.name);
@@ -1358,25 +1389,37 @@ public final class InternalTestCluster extends TestCluster {
                     activeDisruptionScheme.removeFromNode(nodeAndClient.name, this);
                 }
                 nodeAndClient.closeNode();
+                // delete data folders now, before we start other nodes that may claim it
+                nodeAndClient.clearDataIfNeeded(callback);
+
+
+                DiscoveryNode discoveryNode = getInstanceFromNode(ClusterService.class, nodeAndClient.node()).localNode();
+                nodesRoleOrder[nodeAndClient.nodeAndClientId()] = discoveryNode.getRoles();
+                nodesByRoles.computeIfAbsent(discoveryNode.getRoles(), k -> new ArrayList<>()).add(nodeAndClient);
             }
 
-            // starting master nodes first, for now so restart will be quick. If we'll start
-            // the data nodes first, they will wait for 30s for a master
-            List<DiscoveryNode> discoveryNodes = new ArrayList<>();
-            for (ClusterService clusterService : getInstances(ClusterService.class)) {
-                discoveryNodes.add(clusterService.localNode());
+            assert nodesByRoles.values().stream().collect(Collectors.summingInt(List::size)) == nodes.size();
+
+            // randomize start up order, but making sure that:
+            // 1) A data folder that was assigned to a data node will stay so
+            // 2) Data nodes will get the same node lock ordinal range, so custom index paths (where the ordinal is used)
+            //    will still belong to data nodes
+            for (List<NodeAndClient> sameRoleNodes : nodesByRoles.values()) {
+                Collections.shuffle(sameRoleNodes, random);
             }
 
-            discoveryNodes.sort((n1, n2) -> Boolean.compare(n1.isMasterNode() == false, n2.isMasterNode() == false));
-
-
-            for (DiscoveryNode node : discoveryNodes) {
-                NodeAndClient nodeAndClient = nodes.get(node.getName());
+            for (Set roles : nodesRoleOrder) {
+                if (roles == null) {
+                    // if some nodes were stopped, we want have a role for them
+                    continue;
+                }
+                NodeAndClient nodeAndClient = nodesByRoles.get(roles).remove(0);
                 logger.info("Starting node [{}] ", nodeAndClient.name);
                 if (activeDisruptionScheme != null) {
                     activeDisruptionScheme.removeFromNode(nodeAndClient.name, this);
                 }
-                nodeAndClient.restart(callback);
+                // we already cleared data folders, before starting nodes up
+                nodeAndClient.restart(callback, false);
                 if (activeDisruptionScheme != null) {
                     activeDisruptionScheme.applyToNode(nodeAndClient.name, this);
                 }
@@ -1487,36 +1530,22 @@ public final class InternalTestCluster extends TestCluster {
      * Starts a node with default settings and returns it's name.
      */
     public synchronized String startNode() {
-        return startNode(Settings.EMPTY, Version.CURRENT);
-    }
-
-    /**
-     * Starts a node with default settings ad the specified version and returns it's name.
-     */
-    public synchronized String startNode(Version version) {
-        return startNode(Settings.EMPTY, version);
+        return startNode(Settings.EMPTY);
     }
 
     /**
      * Starts a node with the given settings builder and returns it's name.
      */
     public synchronized String startNode(Settings.Builder settings) {
-        return startNode(settings.build(), Version.CURRENT);
+        return startNode(settings.build());
     }
 
     /**
      * Starts a node with the given settings and returns it's name.
      */
     public synchronized String startNode(Settings settings) {
-        return startNode(settings, Version.CURRENT);
-    }
-
-    /**
-     * Starts a node with the given settings and version and returns it's name.
-     */
-    public synchronized String startNode(Settings settings, Version version) {
-        NodeAndClient buildNode = buildNode(settings, version);
-        buildNode.node().start();
+        NodeAndClient buildNode = buildNode(settings);
+        buildNode.startNode();
         publishNode(buildNode);
         return buildNode.name;
     }
@@ -1550,7 +1579,7 @@ public final class InternalTestCluster extends TestCluster {
 
     public synchronized String startMasterOnlyNode(Settings settings) {
         Settings settings1 = Settings.builder().put(settings).put(Node.NODE_MASTER_SETTING.getKey(), true).put(Node.NODE_DATA_SETTING.getKey(), false).build();
-        return startNode(settings1, Version.CURRENT);
+        return startNode(settings1);
     }
 
     public synchronized Async<String> startDataOnlyNodeAsync() {
@@ -1564,7 +1593,7 @@ public final class InternalTestCluster extends TestCluster {
 
     public synchronized String startDataOnlyNode(Settings settings) {
         Settings settings1 = Settings.builder().put(settings).put(Node.NODE_MASTER_SETTING.getKey(), false).put(Node.NODE_DATA_SETTING.getKey(), true).build();
-        return startNode(settings1, Version.CURRENT);
+        return startNode(settings1);
     }
 
     /**
@@ -1585,9 +1614,9 @@ public final class InternalTestCluster extends TestCluster {
      * Starts a node in an async manner with the given settings and version and returns future with its name.
      */
     public synchronized Async<String> startNodeAsync(final Settings settings, final Version version) {
-        final NodeAndClient buildNode = buildNode(settings, version);
+        final NodeAndClient buildNode = buildNode(settings);
         final Future<String> submit = executor.submit(() -> {
-            buildNode.node().start();
+            buildNode.startNode();
             publishNode(buildNode);
             return buildNode.name;
         });
@@ -1646,10 +1675,6 @@ public final class InternalTestCluster extends TestCluster {
 
     private synchronized void publishNode(NodeAndClient nodeAndClient) {
         assert !nodeAndClient.node().isClosed();
-        NodeEnvironment nodeEnv = getInstanceFromNode(NodeEnvironment.class, nodeAndClient.node);
-        if (nodeEnv.hasNodeFile()) {
-            dataDirToClean.addAll(Arrays.asList(nodeEnv.nodeDataPaths()));
-        }
         nodes.put(nodeAndClient.name, nodeAndClient);
         applyDisruptionSchemeToNode(nodeAndClient);
     }
@@ -1927,7 +1952,8 @@ public final class InternalTestCluster extends TestCluster {
     public void assertAfterTest() throws IOException {
         super.assertAfterTest();
         assertRequestsFinished();
-        for (NodeEnvironment env : this.getInstances(NodeEnvironment.class)) {
+        for (NodeAndClient nodeAndClient : nodes.values()) {
+            NodeEnvironment env = nodeAndClient.node().getNodeEnvironment();
             Set<ShardId> shardIds = env.lockedShards();
             for (ShardId id : shardIds) {
                 try {
diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java
index ae774220ccf..63c5eb01fb0 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java
@@ -22,23 +22,39 @@ import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.discovery.DiscoverySettings;
+import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.NodeConfigurationSource;
 import org.elasticsearch.transport.TransportSettings;
 
+import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
+import static org.elasticsearch.cluster.node.DiscoveryNode.Role.DATA;
+import static org.elasticsearch.cluster.node.DiscoveryNode.Role.INGEST;
+import static org.elasticsearch.cluster.node.DiscoveryNode.Role.MASTER;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileNotExists;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.not;
 
 /**
  * Basic test that ensure that the internal cluster reproduces the same
@@ -153,4 +169,140 @@ public class InternalTestClusterTests extends ESTestCase {
             IOUtils.close(cluster0, cluster1);
         }
     }
+
+    public void testDataFolderAssignmentAndCleaning() throws IOException, InterruptedException {
+        long clusterSeed = randomLong();
+        boolean masterNodes = randomBoolean();
+        // we need one stable node
+        int minNumDataNodes = 2;
+        int maxNumDataNodes = 2;
+        final String clusterName1 = "shared1";
+        NodeConfigurationSource nodeConfigurationSource = NodeConfigurationSource.EMPTY;
+        int numClientNodes = 0;
+        boolean enableHttpPipelining = randomBoolean();
+        String nodePrefix = "test";
+        Path baseDir = createTempDir();
+        InternalTestCluster cluster = new InternalTestCluster("local", clusterSeed, baseDir, masterNodes,
+            minNumDataNodes, maxNumDataNodes, clusterName1, nodeConfigurationSource, numClientNodes,
+            enableHttpPipelining, nodePrefix, Collections.emptyList(), Function.identity());
+        try {
+            cluster.beforeTest(random(), 0.0);
+            final Map<String,Path[]> shardNodePaths = new HashMap<>();
+            for (String name: cluster.getNodeNames()) {
+                shardNodePaths.put(name, getNodePaths(cluster, name));
+            }
+            String poorNode = randomFrom(cluster.getNodeNames());
+            Path dataPath = getNodePaths(cluster, poorNode)[0];
+            final Path testMarker = dataPath.resolve("testMarker");
+            Files.createDirectories(testMarker);
+            cluster.stopRandomNode(InternalTestCluster.nameFilter(poorNode));
+            assertFileExists(testMarker); // stopping a node half way shouldn't clean data
+
+            final String stableNode = randomFrom(cluster.getNodeNames());
+            final Path stableDataPath = getNodePaths(cluster, stableNode)[0];
+            final Path stableTestMarker = stableDataPath.resolve("stableTestMarker");
+            assertThat(stableDataPath, not(dataPath));
+            Files.createDirectories(stableTestMarker);
+
+            final String newNode1 =  cluster.startNode();
+            assertThat(getNodePaths(cluster, newNode1)[0], equalTo(dataPath));
+            assertFileExists(testMarker); // starting a node should re-use data folders and not clean it
+
+            final String newNode2 =  cluster.startNode();
+            final Path newDataPath = getNodePaths(cluster, newNode2)[0];
+            final Path newTestMarker = newDataPath.resolve("newTestMarker");
+            assertThat(newDataPath, not(dataPath));
+            Files.createDirectories(newTestMarker);
+            cluster.beforeTest(random(), 0.0);
+            assertFileNotExists(newTestMarker); // the cluster should be reset for a new test, cleaning up the extra path we made
+            assertFileNotExists(testMarker); // a new unknown node used this path, it should be cleaned
+            assertFileExists(stableTestMarker); // but leaving the structure of existing, reused nodes
+            for (String name: cluster.getNodeNames()) {
+                assertThat("data paths for " + name + " changed", getNodePaths(cluster, name),
+                    equalTo(shardNodePaths.get(name)));
+            }
+
+            cluster.beforeTest(random(), 0.0);
+            assertFileExists(stableTestMarker); // but leaving the structure of existing, reused nodes
+            for (String name: cluster.getNodeNames()) {
+                assertThat("data paths for " + name + " changed", getNodePaths(cluster, name),
+                    equalTo(shardNodePaths.get(name)));
+            }
+
+        } finally {
+            cluster.close();
+        }
+    }
+
+    private Path[] getNodePaths(InternalTestCluster cluster, String name) {
+        final NodeEnvironment nodeEnvironment = cluster.getInstance(NodeEnvironment.class, name);
+        if (nodeEnvironment.hasNodeFile()) {
+            return nodeEnvironment.nodeDataPaths();
+        } else {
+            return new Path[0];
+        }
+    }
+
+    public void testDifferentRolesMaintainPathOnRestart() throws Exception {
+        final Path baseDir = createTempDir();
+        InternalTestCluster cluster = new InternalTestCluster("local", randomLong(), baseDir, true, 0, 0, "test",
+            new NodeConfigurationSource() {
+                @Override
+                public Settings nodeSettings(int nodeOrdinal) {
+                    return Settings.builder().put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), 0).build();
+                }
+
+                @Override
+                public Settings transportClientSettings() {
+                    return Settings.EMPTY;
+                }
+            }, 0, randomBoolean(), "", Collections.emptyList(), Function.identity());
+        cluster.beforeTest(random(), 0.0);
+        try {
+            Map<DiscoveryNode.Role, Set<String>> pathsPerRole = new HashMap<>();
+            for (int i = 0; i < 5; i++) {
+                final DiscoveryNode.Role role = randomFrom(MASTER, DiscoveryNode.Role.DATA, DiscoveryNode.Role.INGEST);
+                final String node;
+                switch (role) {
+                    case MASTER:
+                        node = cluster.startMasterOnlyNode(Settings.EMPTY);
+                        break;
+                    case DATA:
+                        node = cluster.startDataOnlyNode(Settings.EMPTY);
+                        break;
+                    case INGEST:
+                        node = cluster.startCoordinatingOnlyNode(Settings.EMPTY);
+                        break;
+                    default:
+                        throw new IllegalStateException("get your story straight");
+                }
+                Set<String> rolePaths = pathsPerRole.computeIfAbsent(role, k -> new HashSet<>());
+                for (Path path : getNodePaths(cluster, node)) {
+                    assertTrue(rolePaths.add(path.toString()));
+                }
+            }
+            cluster.fullRestart();
+
+            Map<DiscoveryNode.Role, Set<String>> result = new HashMap<>();
+            for (String name : cluster.getNodeNames()) {
+                DiscoveryNode node = cluster.getInstance(ClusterService.class, name).localNode();
+                List<String> paths = Arrays.stream(getNodePaths(cluster, name)).map(Path::toString).collect(Collectors.toList());
+                if (node.isMasterNode()) {
+                    result.computeIfAbsent(MASTER, k -> new HashSet<>()).addAll(paths);
+                } else if (node.isDataNode()) {
+                    result.computeIfAbsent(DATA, k -> new HashSet<>()).addAll(paths);
+                } else {
+                    result.computeIfAbsent(INGEST, k -> new HashSet<>()).addAll(paths);
+                }
+            }
+
+            assertThat(result.size(), equalTo(pathsPerRole.size()));
+            for (DiscoveryNode.Role role : result.keySet()) {
+                assertThat("path are not the same for " + role, result.get(role), equalTo(pathsPerRole.get(role)));
+            }
+        } finally {
+            cluster.close();
+        }
+
+    }
 }

From 98276111e1fdd63ce00821118fea9a1c4edef595 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Tue, 28 Jun 2016 15:39:59 +0200
Subject: [PATCH 1097/1311] Re-enable logger usage checks

It was inadvertently disabled after applying code review comments. This commit reenables the logger usage checker and makes it less naggy when encountering logging usages of the form  logger.info(someStringBuilder). Previously it would fail with the error message "First argument must be a string constant so that we can statically ensure proper place holder usage". Now it will only fail in case any arguments are provided as well, for example logger.info(someStringBuilder, 42).
---
 .../loggerusage/ESLoggerUsageChecker.java     | 33 +++++++++++--------
 .../test/loggerusage/ESLoggerUsageTests.java  |  8 +++--
 2 files changed, 25 insertions(+), 16 deletions(-)

diff --git a/test/logger-usage/src/main/java/org/elasticsearch/test/loggerusage/ESLoggerUsageChecker.java b/test/logger-usage/src/main/java/org/elasticsearch/test/loggerusage/ESLoggerUsageChecker.java
index 25d4052c162..041d21cc762 100644
--- a/test/logger-usage/src/main/java/org/elasticsearch/test/loggerusage/ESLoggerUsageChecker.java
+++ b/test/logger-usage/src/main/java/org/elasticsearch/test/loggerusage/ESLoggerUsageChecker.java
@@ -82,7 +82,7 @@ public class ESLoggerUsageChecker {
             Files.walkFileTree(root, new SimpleFileVisitor<Path>() {
                 @Override
                 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-                    if (Files.isRegularFile(file) && file.endsWith(".class")) {
+                    if (Files.isRegularFile(file) && file.getFileName().toString().endsWith(".class")) {
                         try (InputStream in = Files.newInputStream(file)) {
                             ESLoggerUsageChecker.check(wrongUsageCallback, in);
                         }
@@ -248,19 +248,6 @@ public class ESLoggerUsageChecker {
                     if (LOGGER_METHODS.contains(methodInsn.name) == false) {
                         continue;
                     }
-                    Type[] argumentTypes = Type.getArgumentTypes(methodInsn.desc);
-                    BasicValue logMessageLengthObject = getStackValue(stringFrames[i], argumentTypes.length - 1); // first argument
-                    if (logMessageLengthObject instanceof PlaceHolderStringBasicValue == false) {
-                        wrongUsageCallback.accept(new WrongLoggerUsage(className, methodNode.name, methodInsn.name, lineNumber,
-                            "First argument must be a string constant so that we can statically ensure proper place holder usage"));
-                        continue;
-                    }
-                    PlaceHolderStringBasicValue logMessageLength = (PlaceHolderStringBasicValue) logMessageLengthObject;
-                    if (logMessageLength.minValue != logMessageLength.maxValue) {
-                        wrongUsageCallback.accept(new WrongLoggerUsage(className, methodNode.name, methodInsn.name, lineNumber,
-                            "Multiple log messages with conflicting number of place holders"));
-                        continue;
-                    }
                     BasicValue varArgsSizeObject = getStackValue(arraySizeFrames[i], 0); // last argument
                     if (varArgsSizeObject instanceof ArraySizeBasicValue == false) {
                         wrongUsageCallback.accept(new WrongLoggerUsage(className, methodNode.name, methodInsn.name, lineNumber,
@@ -268,6 +255,24 @@ public class ESLoggerUsageChecker {
                         continue;
                     }
                     ArraySizeBasicValue varArgsSize = (ArraySizeBasicValue) varArgsSizeObject;
+                    Type[] argumentTypes = Type.getArgumentTypes(methodInsn.desc);
+                    BasicValue logMessageLengthObject = getStackValue(stringFrames[i], argumentTypes.length - 1); // first argument
+                    if (logMessageLengthObject instanceof PlaceHolderStringBasicValue == false) {
+                        if (varArgsSize.minValue > 0) {
+                            wrongUsageCallback.accept(new WrongLoggerUsage(className, methodNode.name, methodInsn.name, lineNumber,
+                                "First argument must be a string constant so that we can statically ensure proper place holder usage"));
+                            continue;
+                        } else {
+                            // don't check logger usage for logger.warn(someObject) as someObject will be fully logged
+                            continue;
+                        }
+                    }
+                    PlaceHolderStringBasicValue logMessageLength = (PlaceHolderStringBasicValue) logMessageLengthObject;
+                    if (logMessageLength.minValue != logMessageLength.maxValue) {
+                        wrongUsageCallback.accept(new WrongLoggerUsage(className, methodNode.name, methodInsn.name, lineNumber,
+                            "Multiple log messages with conflicting number of place holders"));
+                        continue;
+                    }
                     if (varArgsSize.minValue != varArgsSize.maxValue) {
                         wrongUsageCallback.accept(new WrongLoggerUsage(className, methodNode.name, methodInsn.name, lineNumber,
                             "Multiple parameter arrays with conflicting sizes"));
diff --git a/test/logger-usage/src/test/java/org/elasticsearch/test/loggerusage/ESLoggerUsageTests.java b/test/logger-usage/src/test/java/org/elasticsearch/test/loggerusage/ESLoggerUsageTests.java
index ab07ecbf45e..73449f4351c 100644
--- a/test/logger-usage/src/test/java/org/elasticsearch/test/loggerusage/ESLoggerUsageTests.java
+++ b/test/logger-usage/src/test/java/org/elasticsearch/test/loggerusage/ESLoggerUsageTests.java
@@ -129,8 +129,12 @@ public class ESLoggerUsageTests extends ESTestCase {
         logger.info("Hello {}, {}", "world", 42, new Exception());
     }
 
-    public void checkFailNonConstantMessage(boolean b) {
-        logger.info(Boolean.toString(b));
+    public void checkNonConstantMessageWithZeroArguments(boolean b) {
+        logger.info(Boolean.toString(b), new Exception());
+    }
+
+    public void checkFailNonConstantMessageWithArguments(boolean b) {
+        logger.info(Boolean.toString(b), new Exception(), 42);
     }
 
     public void checkComplexUsage(boolean b) {

From 051579184633030a07b1df08bedb16f6c7ade30c Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Tue, 28 Jun 2016 15:49:00 +0200
Subject: [PATCH 1098/1311] Fix logger usages

---
 .../search/AbstractSearchAsyncAction.java     | 23 +++++--------------
 .../SearchScrollQueryAndFetchAsyncAction.java |  4 ++--
 ...SearchScrollQueryThenFetchAsyncAction.java |  2 +-
 .../cluster/NodeConnectionsService.java       |  2 +-
 .../common/logging/ESLogger.java              |  2 ++
 .../zen/elect/ElectMasterService.java         |  9 ++++----
 .../elasticsearch/threadpool/ThreadPool.java  |  2 +-
 .../bootstrap/MaxMapCountCheckTests.java      |  2 ++
 .../cluster/routing/PrimaryTermsTests.java    |  4 ++--
 .../AbstractLifecycleRunnableTests.java       |  2 ++
 .../gateway/GatewayIndexStateIT.java          |  2 +-
 .../store/IndicesStoreIntegrationIT.java      |  2 +-
 .../elasticsearch/recovery/RelocationIT.java  |  2 +-
 .../AbstractAsyncBulkByScrollAction.java      |  2 +-
 14 files changed, 27 insertions(+), 33 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
index 805d09a0830..d843d8c0630 100644
--- a/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
@@ -190,7 +190,7 @@ abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult>
                 innerMoveToSecondPhase();
             } catch (Throwable e) {
                 if (logger.isDebugEnabled()) {
-                    logger.debug(shardIt.shardId() + ": Failed to execute [" + request + "] while moving to second phase", e);
+                    logger.debug("{}: Failed to execute [{}] while moving to second phase", e, shardIt.shardId(), request);
                 }
                 raiseEarlyFailure(new ReduceSearchPhaseException(firstPhaseName(), "", e, buildShardFailures()));
             }
@@ -210,11 +210,7 @@ abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult>
         if (totalOps.incrementAndGet() == expectedTotalOps) {
             if (logger.isDebugEnabled()) {
                 if (t != null && !TransportActions.isShardNotAvailableException(t)) {
-                    if (shard != null) {
-                        logger.debug(shard.shortSummary() + ": Failed to execute [" + request + "]", t);
-                    } else {
-                        logger.debug(shardIt.shardId() + ": Failed to execute [" + request + "]", t);
-                    }
+                    logger.debug("{}: Failed to execute [{}]", t, shard != null ? shard.shortSummary() : shardIt.shardId(), request);
                 } else if (logger.isTraceEnabled()) {
                     logger.trace("{}: Failed to execute [{}]", t, shard, request);
                 }
@@ -239,7 +235,8 @@ abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult>
             final boolean lastShard = nextShard == null;
             // trace log this exception
             if (logger.isTraceEnabled()) {
-                logger.trace(executionFailureMsg(shard, shardIt, request, lastShard), t);
+                logger.trace("{}: Failed to execute [{}] lastShard [{}]", t, shard != null ? shard.shortSummary() : shardIt.shardId(),
+                    request, lastShard);
             }
             if (!lastShard) {
                 try {
@@ -251,22 +248,14 @@ abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult>
                 // no more shards active, add a failure
                 if (logger.isDebugEnabled() && !logger.isTraceEnabled()) { // do not double log this exception
                     if (t != null && !TransportActions.isShardNotAvailableException(t)) {
-                        logger.debug(executionFailureMsg(shard, shardIt, request, lastShard), t);
+                        logger.debug("{}: Failed to execute [{}] lastShard [{}]", t,
+                            shard != null ? shard.shortSummary() : shardIt.shardId(), request, lastShard);
                     }
                 }
             }
         }
     }
 
-    private String executionFailureMsg(@Nullable ShardRouting shard, final ShardIterator shardIt, SearchRequest request,
-                                       boolean lastShard) {
-        if (shard != null) {
-            return shard.shortSummary() + ": Failed to execute [" + request + "] lastShard [" + lastShard + "]";
-        } else {
-            return shardIt.shardId() + ": Failed to execute [" + request + "] lastShard [" + lastShard + "]";
-        }
-    }
-
     protected final ShardSearchFailure[] buildShardFailures() {
         AtomicArray<ShardSearchFailure> shardFailures = this.shardFailures;
         if (shardFailures == null) {
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java
index 4e0ee3ff5e5..90bb0937327 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java
@@ -103,7 +103,7 @@ class SearchScrollQueryAndFetchAsyncAction extends AbstractAsyncAction {
                 executePhase(i, node, target.getScrollId());
             } else {
                 if (logger.isDebugEnabled()) {
-                    logger.debug("Node [" + target.getNode() + "] not available for scroll request [" + scrollId.getSource() + "]");
+                    logger.debug("Node [{}] not available for scroll request [{}]", target.getNode(), scrollId.getSource());
                 }
                 successfulOps.decrementAndGet();
                 if (counter.decrementAndGet() == 0) {
@@ -116,7 +116,7 @@ class SearchScrollQueryAndFetchAsyncAction extends AbstractAsyncAction {
             DiscoveryNode node = nodes.get(target.getNode());
             if (node == null) {
                 if (logger.isDebugEnabled()) {
-                    logger.debug("Node [" + target.getNode() + "] not available for scroll request [" + scrollId.getSource() + "]");
+                    logger.debug("Node [{}] not available for scroll request [{}]", target.getNode(), scrollId.getSource());
                 }
                 successfulOps.decrementAndGet();
                 if (counter.decrementAndGet() == 0) {
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
index 8e822302d2f..694ee16beac 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
@@ -107,7 +107,7 @@ class SearchScrollQueryThenFetchAsyncAction extends AbstractAsyncAction {
                 executeQueryPhase(i, counter, node, target.getScrollId());
             } else {
                 if (logger.isDebugEnabled()) {
-                    logger.debug("Node [" + target.getNode() + "] not available for scroll request [" + scrollId.getSource() + "]");
+                    logger.debug("Node [{}] not available for scroll request [{}]", target.getNode(), scrollId.getSource());
                 }
                 successfulOps.decrementAndGet();
                 if (counter.decrementAndGet() == 0) {
diff --git a/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java b/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
index 698f9d1090c..404ae57d5f7 100644
--- a/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
@@ -91,7 +91,7 @@ public class NodeConnectionsService extends AbstractLifecycleComponent<NodeConne
                 try {
                     transportService.disconnectFromNode(node);
                 } catch (Throwable e) {
-                    logger.warn("failed to disconnect to node [" + node + "]", e);
+                    logger.warn("failed to disconnect to node [{}]", e, node);
                 }
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/common/logging/ESLogger.java b/core/src/main/java/org/elasticsearch/common/logging/ESLogger.java
index 597619e61b4..b2a2aa333c7 100644
--- a/core/src/main/java/org/elasticsearch/common/logging/ESLogger.java
+++ b/core/src/main/java/org/elasticsearch/common/logging/ESLogger.java
@@ -21,12 +21,14 @@ package org.elasticsearch.common.logging;
 
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
+import org.elasticsearch.common.SuppressLoggerChecks;
 
 import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
 
 /**
  * Elasticsearch's logger wrapper.
  */
+@SuppressLoggerChecks(reason = "safely delegates to itself")
 public class ESLogger {
     private static final String FQCN = ESLogger.class.getName();
 
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/elect/ElectMasterService.java b/core/src/main/java/org/elasticsearch/discovery/zen/elect/ElectMasterService.java
index 76b7de6da08..3ef9138f933 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/elect/ElectMasterService.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/elect/ElectMasterService.java
@@ -94,11 +94,10 @@ public class ElectMasterService extends AbstractComponent {
     public void logMinimumMasterNodesWarningIfNecessary(ClusterState oldState, ClusterState newState) {
         // check if min_master_nodes setting is too low and log warning
         if (hasTooManyMasterNodes(oldState.nodes()) == false && hasTooManyMasterNodes(newState.nodes())) {
-            logger.warn("value for setting \""
-                    + ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey()
-                    + "\" is too low. This can result in data loss! Please set it to at least a quorum of master-eligible nodes "
-                    + "(current value: [{}], total number of master-eligible nodes used for publishing in this round: [{}])",
-                minimumMasterNodes(), newState.getNodes().getMasterNodes().size());
+            logger.warn("value for setting \"{}\" is too low. This can result in data loss! Please set it to at least a quorum of master-" +
+                    "eligible nodes (current value: [{}], total number of master-eligible nodes used for publishing in this round: [{}])",
+                ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minimumMasterNodes(),
+                newState.getNodes().getMasterNodes().size());
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
index 3f6a65c82c5..551eeea838d 100644
--- a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
+++ b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
@@ -196,7 +196,7 @@ public class ThreadPool extends AbstractComponent implements Closeable {
             if (executors.containsKey(executorHolder.info.getName())) {
                 throw new IllegalStateException("duplicate executors with name [" + executorHolder.info.getName() + "] registered");
             }
-            logger.debug("created thread pool: " + entry.getValue().formatInfo(executorHolder.info));
+            logger.debug("created thread pool: {}", entry.getValue().formatInfo(executorHolder.info));
             executors.put(entry.getKey(), executorHolder);
         }
 
diff --git a/core/src/test/java/org/elasticsearch/bootstrap/MaxMapCountCheckTests.java b/core/src/test/java/org/elasticsearch/bootstrap/MaxMapCountCheckTests.java
index 86965a679da..b48fcc78c6c 100644
--- a/core/src/test/java/org/elasticsearch/bootstrap/MaxMapCountCheckTests.java
+++ b/core/src/test/java/org/elasticsearch/bootstrap/MaxMapCountCheckTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.bootstrap;
 
 import org.apache.lucene.util.Constants;
+import org.elasticsearch.common.SuppressLoggerChecks;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.test.ESTestCase;
@@ -46,6 +47,7 @@ public class MaxMapCountCheckTests extends ESTestCase {
         }
     }
 
+    @SuppressLoggerChecks(reason = "mock usage")
     public void testGetMaxMapCount() throws IOException {
         final long procSysVmMaxMapCount = randomIntBetween(1, Integer.MAX_VALUE);
         final BufferedReader reader = mock(BufferedReader.class);
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java
index 4dceb42cfbc..2d3e44db68a 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java
@@ -65,7 +65,7 @@ public class PrimaryTermsTests extends ESAllocationTestCase {
                 .build());
         this.numberOfShards = randomIntBetween(1, 5);
         this.numberOfReplicas = randomIntBetween(1, 5);
-        logger.info("Setup test with " + this.numberOfShards + " shards and " + this.numberOfReplicas + " replicas.");
+        logger.info("Setup test with {} shards and {} replicas.", this.numberOfShards, this.numberOfReplicas);
         this.primaryTermsPerIndex.clear();
         MetaData metaData = MetaData.builder()
                 .put(createIndexMetaData(TEST_INDEX_1))
@@ -87,7 +87,7 @@ public class PrimaryTermsTests extends ESAllocationTestCase {
      * puts primary shard routings into initializing state
      */
     private void initPrimaries() {
-        logger.info("adding " + (this.numberOfReplicas + 1) + " nodes and performing rerouting");
+        logger.info("adding {} nodes and performing rerouting", this.numberOfReplicas + 1);
         Builder discoBuilder = DiscoveryNodes.builder();
         for (int i = 0; i < this.numberOfReplicas + 1; i++) {
             discoBuilder = discoBuilder.put(newNode("node" + i));
diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/AbstractLifecycleRunnableTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/AbstractLifecycleRunnableTests.java
index 8d4efab8fa1..4c2e4700943 100644
--- a/core/src/test/java/org/elasticsearch/common/util/concurrent/AbstractLifecycleRunnableTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/AbstractLifecycleRunnableTests.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.common.util.concurrent;
 
+import org.elasticsearch.common.SuppressLoggerChecks;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.test.ESTestCase;
@@ -67,6 +68,7 @@ public class AbstractLifecycleRunnableTests extends ESTestCase {
         inOrder.verifyNoMoreInteractions();
     }
 
+    @SuppressLoggerChecks(reason = "mock usage")
     public void testDoRunDoesNotRunWhenStoppedOrClosed() throws Exception {
         Callable<?> runCallable = mock(Callable.class);
 
diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
index 049fd90cad6..129495ea15e 100644
--- a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
+++ b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
@@ -343,7 +343,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
         } else {
             // test with a shadow replica index
             final Path dataPath = createTempDir();
-            logger.info("--> created temp data path for shadow replicas [" + dataPath + "]");
+            logger.info("--> created temp data path for shadow replicas [{}]", dataPath);
             logger.info("--> starting a cluster with " + numNodes + " nodes");
             final Settings nodeSettings = Settings.builder()
                                                   .put("node.add_id_to_custom_path", false)
diff --git a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java
index b500ffb7dd2..b4f66c2e17b 100644
--- a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java
@@ -263,7 +263,7 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
         assertThat(waitForShardDeletion(node_3, index, 0), equalTo(false));
 
         Path server2Shard = shardDirectory(node_2, index, 0);
-        logger.info("--> stopping node " + node_2);
+        logger.info("--> stopping node {}", node_2);
         internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node_2));
 
         logger.info("--> running cluster_health");
diff --git a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
index bc9909b21c1..620dfeb94c2 100644
--- a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
+++ b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
@@ -479,7 +479,7 @@ public class RelocationIT extends ESIntegTestCase {
         indexRandom(true, docs);
         numDocs *= 2;
 
-        logger.info(" --> waiting for relocation to complete", numDocs);
+        logger.info(" --> waiting for relocation to complete");
         ensureGreen("test");// move all shards to the new node (it waits on relocation)
         final int numIters = randomIntBetween(10, 20);
         for (int i = 0; i < numIters; i++) {
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
index ceb00b2f81f..3403a8077b9 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
@@ -404,7 +404,7 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
 
                 @Override
                 public void onFailure(Throwable e) {
-                    logger.warn("Failed to clear scroll [" + scrollId + ']', e);
+                    logger.warn("Failed to clear scroll [{}]", e, scrollId);
                 }
             });
         }

From 3cc2251e334188037b825e30bc74c9bad416c897 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Tue, 28 Jun 2016 17:38:56 +0200
Subject: [PATCH 1099/1311] Fix number of arguments provided to logger calls

---
 .../main/java/org/elasticsearch/test/InternalTestCluster.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index d2014ebbd6e..20af5cf50b2 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -890,7 +890,7 @@ public final class InternalTestCluster extends TestCluster {
                 NodeEnvironment nodeEnv = node.getNodeEnvironment();
                 if (nodeEnv.hasNodeFile()) {
                     final Path[] locations = nodeEnv.nodeDataPaths();
-                    logger.debug("removing node data paths: [{}]", (Object[]) locations);
+                    logger.debug("removing node data paths: [{}]", Arrays.toString(locations));
                     IOUtils.rm(locations);
                 }
             }
@@ -1039,7 +1039,7 @@ public final class InternalTestCluster extends TestCluster {
             ClusterHealthResponse response = client().admin().cluster().prepareHealth()
                 .setWaitForNodes(Integer.toString(newSize)).get();
             if (response.isTimedOut()) {
-                logger.warn("failed to wait for a cluster of size [{}], got", newSize, response);
+                logger.warn("failed to wait for a cluster of size [{}], got [{}]", newSize, response);
                 throw new IllegalStateException("cluster failed to reach the expected size of [" + newSize + "]");
             }
         }

From 9b9e17abf7ab34707e8a9264cbddc3afd196b7a3 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Tue, 28 Jun 2016 17:51:33 +0200
Subject: [PATCH 1100/1311] Cleanup Compressor interface (#19125)

Today we have several deprecated methods, leaking netty interfaces, support for
multiple compressors on the compressor interface. The netty interface can simply
be replaced by BytesReference which we already have an implementation for, all the
others are not used and are removed in this commit.
---
 .../common/compress/CompressedIndexInput.java | 215 ------------------
 .../compress/CompressedStreamInput.java       | 174 --------------
 .../common/compress/CompressedXContent.java   |   4 +-
 .../common/compress/Compressor.java           |  14 --
 .../common/compress/CompressorFactory.java    |  58 +----
 .../{deflate => }/DeflateCompressor.java      |  42 +---
 .../publish/PublishClusterStateAction.java    |   4 +-
 .../blobstore/ChecksumBlobStoreFormat.java    |   2 +-
 .../netty/MessageChannelHandler.java          |   3 +-
 .../transport/netty/NettyTransport.java       |   2 +-
 .../netty/NettyTransportChannel.java          |   2 +-
 ...estCase.java => DeflateCompressTests.java} |   8 +-
 ...va => DeflateCompressedXContentTests.java} |  52 ++---
 .../deflate/DeflateCompressedStreamTests.java |  30 ---
 .../deflate/DeflateXContentTests.java         |  30 ---
 .../mapper/binary/BinaryMappingTests.java     |   2 +-
 .../snapshots/BlobStoreFormatIT.java          |   2 +-
 17 files changed, 40 insertions(+), 604 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/common/compress/CompressedIndexInput.java
 delete mode 100644 core/src/main/java/org/elasticsearch/common/compress/CompressedStreamInput.java
 rename core/src/main/java/org/elasticsearch/common/compress/{deflate => }/DeflateCompressor.java (80%)
 rename core/src/test/java/org/elasticsearch/common/compress/{AbstractCompressedStreamTestCase.java => DeflateCompressTests.java} (98%)
 rename core/src/test/java/org/elasticsearch/common/compress/{AbstractCompressedXContentTestCase.java => DeflateCompressedXContentTests.java} (63%)
 delete mode 100644 core/src/test/java/org/elasticsearch/common/compress/deflate/DeflateCompressedStreamTests.java
 delete mode 100644 core/src/test/java/org/elasticsearch/common/compress/deflate/DeflateXContentTests.java

diff --git a/core/src/main/java/org/elasticsearch/common/compress/CompressedIndexInput.java b/core/src/main/java/org/elasticsearch/common/compress/CompressedIndexInput.java
deleted file mode 100644
index 599eaeae337..00000000000
--- a/core/src/main/java/org/elasticsearch/common/compress/CompressedIndexInput.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.compress;
-
-import org.apache.lucene.store.IndexInput;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.common.util.LongArray;
-
-import java.io.EOFException;
-import java.io.IOException;
-
-/**
- * @deprecated Used only for backward comp. to read old compressed files, since we now use codec based compression
- */
-@Deprecated
-public abstract class CompressedIndexInput extends IndexInput {
-
-    private IndexInput in;
-
-    private int version;
-    private long totalUncompressedLength;
-    private LongArray offsets;
-
-    private boolean closed;
-
-    protected byte[] uncompressed;
-    protected int uncompressedLength;
-    private int position = 0;
-    private int valid = 0;
-    private int currentOffsetIdx;
-    private long currentUncompressedChunkPointer;
-
-    public CompressedIndexInput(IndexInput in) throws IOException {
-        super("compressed(" + in.toString() + ")");
-        this.in = in;
-        readHeader(in);
-        this.version = in.readInt();
-        long metaDataPosition = in.readLong();
-        long headerLength = in.getFilePointer();
-        in.seek(metaDataPosition);
-        this.totalUncompressedLength = in.readVLong();
-        int size = in.readVInt();
-        offsets = BigArrays.NON_RECYCLING_INSTANCE.newLongArray(size);
-        for (int i = 0; i < size; i++) {
-            offsets.set(i, in.readVLong());
-        }
-        this.currentOffsetIdx = -1;
-        this.currentUncompressedChunkPointer = 0;
-        in.seek(headerLength);
-    }
-
-    /**
-     * Method is overridden to report number of bytes that can now be read
-     * from decoded data buffer, without reading bytes from the underlying
-     * stream.
-     * Never throws an exception; returns number of bytes available without
-     * further reads from underlying source; -1 if stream has been closed, or
-     * 0 if an actual read (and possible blocking) is needed to find out.
-     */
-    public int available() throws IOException {
-        // if closed, return -1;
-        if (closed) {
-            return -1;
-        }
-        int left = (valid - position);
-        return (left <= 0) ? 0 : left;
-    }
-
-    @Override
-    public byte readByte() throws IOException {
-        if (!readyBuffer()) {
-            throw new EOFException();
-        }
-        return uncompressed[position++];
-    }
-
-    public int read(byte[] buffer, int offset, int length, boolean fullRead) throws IOException {
-        if (length < 1) {
-            return 0;
-        }
-        if (!readyBuffer()) {
-            return -1;
-        }
-        // First let's read however much data we happen to have...
-        int chunkLength = Math.min(valid - position, length);
-        System.arraycopy(uncompressed, position, buffer, offset, chunkLength);
-        position += chunkLength;
-
-        if (chunkLength == length || !fullRead) {
-            return chunkLength;
-        }
-        // Need more data, then
-        int totalRead = chunkLength;
-        do {
-            offset += chunkLength;
-            if (!readyBuffer()) {
-                break;
-            }
-            chunkLength = Math.min(valid - position, (length - totalRead));
-            System.arraycopy(uncompressed, position, buffer, offset, chunkLength);
-            position += chunkLength;
-            totalRead += chunkLength;
-        } while (totalRead < length);
-
-        return totalRead;
-    }
-
-    @Override
-    public void readBytes(byte[] b, int offset, int len) throws IOException {
-        int result = read(b, offset, len, true /* we want to have full reads, that's the contract... */);
-        if (result < len) {
-            throw new EOFException();
-        }
-    }
-
-    @Override
-    public long getFilePointer() {
-        return currentUncompressedChunkPointer + position;
-    }
-
-    @Override
-    public void seek(long pos) throws IOException {
-        int idx = (int) (pos / uncompressedLength);
-        if (idx >= offsets.size()) {
-            // set the next "readyBuffer" to EOF
-            currentOffsetIdx = idx;
-            position = 0;
-            valid = 0;
-            return;
-        }
-
-        // TODO: optimize so we won't have to readyBuffer on seek, can keep the position around, and set it on readyBuffer in this case
-        if (idx != currentOffsetIdx) {
-            long pointer = offsets.get(idx);
-            in.seek(pointer);
-            position = 0;
-            valid = 0;
-            currentOffsetIdx = idx - 1; // we are going to increase it in readyBuffer...
-            readyBuffer();
-        }
-        position = (int) (pos % uncompressedLength);
-    }
-
-    @Override
-    public long length() {
-        return totalUncompressedLength;
-    }
-
-    @Override
-    public void close() throws IOException {
-        position = valid = 0;
-        if (!closed) {
-            closed = true;
-            doClose();
-            in.close();
-        }
-    }
-
-    protected abstract void doClose() throws IOException;
-
-    protected boolean readyBuffer() throws IOException {
-        if (position < valid) {
-            return true;
-        }
-        if (closed) {
-            return false;
-        }
-        // we reached the end...
-        if (currentOffsetIdx + 1 >= offsets.size()) {
-            return false;
-        }
-        valid = uncompress(in, uncompressed);
-        if (valid < 0) {
-            return false;
-        }
-        currentOffsetIdx++;
-        currentUncompressedChunkPointer = ((long) currentOffsetIdx) * uncompressedLength;
-        position = 0;
-        return (position < valid);
-    }
-
-    protected abstract void readHeader(IndexInput in) throws IOException;
-
-    /**
-     * Uncompress the data into the out array, returning the size uncompressed
-     */
-    protected abstract int uncompress(IndexInput in, byte[] out) throws IOException;
-
-    @Override
-    public IndexInput clone() {
-        // we clone and we need to make sure we keep the same positions!
-        CompressedIndexInput cloned = (CompressedIndexInput) super.clone();
-        cloned.uncompressed = new byte[uncompressedLength];
-        System.arraycopy(uncompressed, 0, cloned.uncompressed, 0, uncompressedLength);
-        cloned.in = (IndexInput) cloned.in.clone();
-        return cloned;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/common/compress/CompressedStreamInput.java b/core/src/main/java/org/elasticsearch/common/compress/CompressedStreamInput.java
deleted file mode 100644
index bdacbd4727b..00000000000
--- a/core/src/main/java/org/elasticsearch/common/compress/CompressedStreamInput.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.compress;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.common.io.stream.StreamInput;
-
-import java.io.EOFException;
-import java.io.IOException;
-
-/**
- */
-public abstract class CompressedStreamInput extends StreamInput {
-
-    private final StreamInput in;
-
-    private boolean closed;
-
-    protected byte[] uncompressed;
-    private int position = 0;
-    private int valid = 0;
-
-    public CompressedStreamInput(StreamInput in) throws IOException {
-        this.in = in;
-        super.setVersion(in.getVersion());
-        readHeader(in);
-    }
-
-    @Override
-    public void setVersion(Version version) {
-        in.setVersion(version);
-        super.setVersion(version);
-    }
-
-    /**
-     * Method is overridden to report number of bytes that can now be read
-     * from decoded data buffer, without reading bytes from the underlying
-     * stream.
-     * Never throws an exception; returns number of bytes available without
-     * further reads from underlying source; -1 if stream has been closed, or
-     * 0 if an actual read (and possible blocking) is needed to find out.
-     */
-    @Override
-    public int available() throws IOException {
-        // if closed, return -1;
-        if (closed) {
-            return -1;
-        }
-        int left = (valid - position);
-        return (left <= 0) ? 0 : left;
-    }
-
-    @Override
-    public int read() throws IOException {
-        if (!readyBuffer()) {
-            return -1;
-        }
-        return uncompressed[position++] & 255;
-    }
-
-    @Override
-    public byte readByte() throws IOException {
-        if (!readyBuffer()) {
-            throw new EOFException();
-        }
-        return uncompressed[position++];
-    }
-
-    @Override
-    public int read(byte[] buffer, int offset, int length) throws IOException {
-        return read(buffer, offset, length, false);
-    }
-
-    public int read(byte[] buffer, int offset, int length, boolean fullRead) throws IOException {
-        if (length < 1) {
-            return 0;
-        }
-        if (!readyBuffer()) {
-            return -1;
-        }
-        // First let's read however much data we happen to have...
-        int chunkLength = Math.min(valid - position, length);
-        System.arraycopy(uncompressed, position, buffer, offset, chunkLength);
-        position += chunkLength;
-
-        if (chunkLength == length || !fullRead) {
-            return chunkLength;
-        }
-        // Need more data, then
-        int totalRead = chunkLength;
-        do {
-            offset += chunkLength;
-            if (!readyBuffer()) {
-                break;
-            }
-            chunkLength = Math.min(valid - position, (length - totalRead));
-            System.arraycopy(uncompressed, position, buffer, offset, chunkLength);
-            position += chunkLength;
-            totalRead += chunkLength;
-        } while (totalRead < length);
-
-        return totalRead;
-    }
-
-    @Override
-    public void readBytes(byte[] b, int offset, int len) throws IOException {
-        int result = read(b, offset, len, true /* we want to have full reads, that's the contract... */);
-        if (result < len) {
-            throw new EOFException();
-        }
-    }
-
-    @Override
-    public void reset() throws IOException {
-        this.position = 0;
-        this.valid = 0;
-        in.reset();
-    }
-
-    @Override
-    public void close() throws IOException {
-        position = valid = 0;
-        if (!closed) {
-            closed = true;
-            doClose();
-            in.close();
-        }
-    }
-
-    protected abstract void doClose() throws IOException;
-
-    /**
-     * Fill the uncompressed bytes buffer by reading the underlying inputStream.
-     */
-    protected boolean readyBuffer() throws IOException {
-        if (position < valid) {
-            return true;
-        }
-        if (closed) {
-            return false;
-        }
-        valid = uncompress(in, uncompressed);
-        if (valid < 0) {
-            return false;
-        }
-        position = 0;
-        return (position < valid);
-    }
-
-    protected abstract void readHeader(StreamInput in) throws IOException;
-
-    /**
-     * Uncompress the data into the out array, returning the size uncompressed
-     */
-    protected abstract int uncompress(StreamInput in, byte[] out) throws IOException;
-
-}
diff --git a/core/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java b/core/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java
index 462b91aeef0..3864befcc04 100644
--- a/core/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java
@@ -80,7 +80,7 @@ public final class CompressedXContent {
      */
     public CompressedXContent(ToXContent xcontent, XContentType type, ToXContent.Params params) throws IOException {
         BytesStreamOutput bStream = new BytesStreamOutput();
-        OutputStream compressedStream = CompressorFactory.defaultCompressor().streamOutput(bStream);
+        OutputStream compressedStream = CompressorFactory.COMPRESSOR.streamOutput(bStream);
         CRC32 crc32 = new CRC32();
         OutputStream checkedStream = new CheckedOutputStream(compressedStream, crc32);
         try (XContentBuilder builder = XContentFactory.contentBuilder(type, checkedStream)) {
@@ -105,7 +105,7 @@ public final class CompressedXContent {
             this.crc32 = crc32(new BytesArray(uncompressed()));
         } else {
             BytesStreamOutput out = new BytesStreamOutput();
-            try (OutputStream compressedOutput = CompressorFactory.defaultCompressor().streamOutput(out)) {
+            try (OutputStream compressedOutput = CompressorFactory.COMPRESSOR.streamOutput(out)) {
                 data.writeTo(compressedOutput);
             }
             this.bytes = out.bytes().toBytes();
diff --git a/core/src/main/java/org/elasticsearch/common/compress/Compressor.java b/core/src/main/java/org/elasticsearch/common/compress/Compressor.java
index 252fad09807..62ed1cb6edb 100644
--- a/core/src/main/java/org/elasticsearch/common/compress/Compressor.java
+++ b/core/src/main/java/org/elasticsearch/common/compress/Compressor.java
@@ -33,21 +33,7 @@ public interface Compressor {
 
     boolean isCompressed(BytesReference bytes);
 
-    boolean isCompressed(ChannelBuffer buffer);
-
     StreamInput streamInput(StreamInput in) throws IOException;
 
     StreamOutput streamOutput(StreamOutput out) throws IOException;
-
-    /**
-     * @deprecated Used for backward comp. since we now use Lucene compressed codec.
-     */
-    @Deprecated
-    boolean isCompressed(IndexInput in) throws IOException;
-
-    /**
-     * @deprecated Used for backward comp. since we now use Lucene compressed codec.
-     */
-    @Deprecated
-    CompressedIndexInput indexInput(IndexInput in) throws IOException;
 }
diff --git a/core/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java b/core/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java
index e6c43a524ca..82e049704cc 100644
--- a/core/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java
+++ b/core/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java
@@ -19,16 +19,13 @@
 
 package org.elasticsearch.common.compress;
 
-import org.apache.lucene.store.IndexInput;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.compress.deflate.DeflateCompressor;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.jboss.netty.buffer.ChannelBuffer;
 
 import java.io.IOException;
 
@@ -36,47 +33,21 @@ import java.io.IOException;
  */
 public class CompressorFactory {
 
-    private static final Compressor[] compressors;
-    private static volatile Compressor defaultCompressor;
-
-    static {
-        compressors = new Compressor[] {
-                new DeflateCompressor()
-        };
-        defaultCompressor = new DeflateCompressor();
-    }
-
-    public static void setDefaultCompressor(Compressor defaultCompressor) {
-        CompressorFactory.defaultCompressor = defaultCompressor;
-    }
-
-    public static Compressor defaultCompressor() {
-        return defaultCompressor;
-    }
+    public static final Compressor COMPRESSOR = new DeflateCompressor();
 
     public static boolean isCompressed(BytesReference bytes) {
         return compressor(bytes) != null;
     }
 
-    /**
-     * @deprecated we don't compress lucene indexes anymore and rely on lucene codecs
-     */
-    @Deprecated
-    public static boolean isCompressed(IndexInput in) throws IOException {
-        return compressor(in) != null;
-    }
-
     @Nullable
     public static Compressor compressor(BytesReference bytes) {
-        for (Compressor compressor : compressors) {
-            if (compressor.isCompressed(bytes)) {
+            if (COMPRESSOR.isCompressed(bytes)) {
                 // bytes should be either detected as compressed or as xcontent,
                 // if we have bytes that can be either detected as compressed or
                 // as a xcontent, we have a problem
                 assert XContentFactory.xContentType(bytes) == null;
-                return compressor;
+                return COMPRESSOR;
             }
-        }
 
         XContentType contentType = XContentFactory.xContentType(bytes);
         if (contentType == null) {
@@ -97,29 +68,6 @@ public class CompressorFactory {
                (bytes.get(2) == 0 || bytes.get(2) == 1);
     }
 
-    public static Compressor compressor(ChannelBuffer buffer) {
-        for (Compressor compressor : compressors) {
-            if (compressor.isCompressed(buffer)) {
-                return compressor;
-            }
-        }
-        throw new NotCompressedException();
-    }
-
-    /**
-     * @deprecated we don't compress lucene indexes anymore and rely on lucene codecs
-     */
-    @Deprecated
-    @Nullable
-    public static Compressor compressor(IndexInput in) throws IOException {
-        for (Compressor compressor : compressors) {
-            if (compressor.isCompressed(in)) {
-                return compressor;
-            }
-        }
-        return null;
-    }
-
     /**
      * Uncompress the provided data, data can be detected as compressed using {@link #isCompressed(BytesReference)}.
      */
diff --git a/core/src/main/java/org/elasticsearch/common/compress/deflate/DeflateCompressor.java b/core/src/main/java/org/elasticsearch/common/compress/DeflateCompressor.java
similarity index 80%
rename from core/src/main/java/org/elasticsearch/common/compress/deflate/DeflateCompressor.java
rename to core/src/main/java/org/elasticsearch/common/compress/DeflateCompressor.java
index be396324788..42e2efa358c 100644
--- a/core/src/main/java/org/elasticsearch/common/compress/deflate/DeflateCompressor.java
+++ b/core/src/main/java/org/elasticsearch/common/compress/DeflateCompressor.java
@@ -17,17 +17,14 @@
  * under the License.
  */
 
-package org.elasticsearch.common.compress.deflate;
+package org.elasticsearch.common.compress;
 
-import org.apache.lucene.store.IndexInput;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.compress.CompressedIndexInput;
 import org.elasticsearch.common.compress.Compressor;
 import org.elasticsearch.common.io.stream.InputStreamStreamInput;
 import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.jboss.netty.buffer.ChannelBuffer;
 
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
@@ -35,6 +32,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.zip.Deflater;
 import java.util.zip.DeflaterOutputStream;
 import java.util.zip.Inflater;
@@ -69,20 +67,6 @@ public class DeflateCompressor implements Compressor {
         return true;
     }
 
-    @Override
-    public boolean isCompressed(ChannelBuffer buffer) {
-        if (buffer.readableBytes() < HEADER.length) {
-            return false;
-        }
-        final int offset = buffer.readerIndex();
-        for (int i = 0; i < HEADER.length; ++i) {
-            if (buffer.getByte(offset + i) != HEADER[i]) {
-                return false;
-            }
-        }
-        return true;
-    }
-
     @Override
     public StreamInput streamInput(StreamInput in) throws IOException {
         final byte[] headerBytes = new byte[HEADER.length];
@@ -103,16 +87,14 @@ public class DeflateCompressor implements Compressor {
         InputStream decompressedIn = new InflaterInputStream(in, inflater, BUFFER_SIZE);
         decompressedIn = new BufferedInputStream(decompressedIn, BUFFER_SIZE);
         return new InputStreamStreamInput(decompressedIn) {
-            private boolean closed = false;
-
+            final AtomicBoolean closed = new AtomicBoolean(false);
             public void close() throws IOException {
                 try {
                     super.close();
                 } finally {
-                    if (closed == false) {
+                    if (closed.compareAndSet(false, true)) {
                         // important to release native memory
                         inflater.end();
-                        closed = true;
                     }
                 }
             }
@@ -128,29 +110,17 @@ public class DeflateCompressor implements Compressor {
         OutputStream compressedOut = new DeflaterOutputStream(out, deflater, BUFFER_SIZE, syncFlush);
         compressedOut = new BufferedOutputStream(compressedOut, BUFFER_SIZE);
         return new OutputStreamStreamOutput(compressedOut) {
-            private boolean closed = false;
-
+            final AtomicBoolean closed = new AtomicBoolean(false);
             public void close() throws IOException {
                 try {
                     super.close();
                 } finally {
-                    if (closed == false) {
+                    if (closed.compareAndSet(false, true)) {
                         // important to release native memory
                         deflater.end();
-                        closed = true;
                     }
                 }
             }
         };
     }
-
-    @Override
-    public boolean isCompressed(IndexInput in) throws IOException {
-        return false;
-    }
-
-    @Override
-    public CompressedIndexInput indexInput(IndexInput in) throws IOException {
-        throw new UnsupportedOperationException();
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java b/core/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java
index c2333ff177d..48be9f64d5c 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java
@@ -335,7 +335,7 @@ public class PublishClusterStateAction extends AbstractComponent {
 
     public static BytesReference serializeFullClusterState(ClusterState clusterState, Version nodeVersion) throws IOException {
         BytesStreamOutput bStream = new BytesStreamOutput();
-        try (StreamOutput stream = CompressorFactory.defaultCompressor().streamOutput(bStream)) {
+        try (StreamOutput stream = CompressorFactory.COMPRESSOR.streamOutput(bStream)) {
             stream.setVersion(nodeVersion);
             stream.writeBoolean(true);
             clusterState.writeTo(stream);
@@ -345,7 +345,7 @@ public class PublishClusterStateAction extends AbstractComponent {
 
     public static BytesReference serializeDiffClusterState(Diff diff, Version nodeVersion) throws IOException {
         BytesStreamOutput bStream = new BytesStreamOutput();
-        try (StreamOutput stream = CompressorFactory.defaultCompressor().streamOutput(bStream)) {
+        try (StreamOutput stream = CompressorFactory.COMPRESSOR.streamOutput(bStream)) {
             stream.setVersion(nodeVersion);
             stream.writeBoolean(false);
             diff.writeTo(stream);
diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java
index 6cea34cf679..37df2ddfb90 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/ChecksumBlobStoreFormat.java
@@ -195,7 +195,7 @@ public class ChecksumBlobStoreFormat<T extends ToXContent> extends BlobStoreForm
     protected BytesReference write(T obj) throws IOException {
         try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()) {
             if (compress) {
-                try (StreamOutput compressedStreamOutput = CompressorFactory.defaultCompressor().streamOutput(bytesStreamOutput)) {
+                try (StreamOutput compressedStreamOutput = CompressorFactory.COMPRESSOR.streamOutput(bytesStreamOutput)) {
                     write(obj, compressedStreamOutput);
                 }
             } else {
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java b/core/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java
index e45635e3349..55793384b1b 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java
@@ -21,6 +21,7 @@ package org.elasticsearch.transport.netty;
 
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
+import org.elasticsearch.common.bytes.ChannelBufferBytesReference;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.compress.Compressor;
 import org.elasticsearch.common.compress.CompressorFactory;
@@ -110,7 +111,7 @@ public class MessageChannelHandler extends SimpleChannelUpstreamHandler {
             if (TransportStatus.isCompress(status) && hasMessageBytesToRead && buffer.readable()) {
                 Compressor compressor;
                 try {
-                    compressor = CompressorFactory.compressor(buffer);
+                    compressor = CompressorFactory.compressor(new ChannelBufferBytesReference(buffer));
                 } catch (NotCompressedException ex) {
                     int maxToRead = Math.min(buffer.readableBytes(), 10);
                     int offset = buffer.readerIndex();
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
index c9f02066836..53eb63c86c5 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
@@ -886,7 +886,7 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
             // the header part is compressed, and the "body" can't be extracted as compressed
             if (options.compress() && (!(request instanceof BytesTransportRequest))) {
                 status = TransportStatus.setCompress(status);
-                stream = CompressorFactory.defaultCompressor().streamOutput(stream);
+                stream = CompressorFactory.COMPRESSOR.streamOutput(stream);
             }
 
             // we pick the smallest of the 2, to support both backward and forward compatibility
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java
index 03856017c36..65ea00d75e5 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java
@@ -99,7 +99,7 @@ public class NettyTransportChannel implements TransportChannel {
             StreamOutput stream = bStream;
             if (options.compress()) {
                 status = TransportStatus.setCompress(status);
-                stream = CompressorFactory.defaultCompressor().streamOutput(stream);
+                stream = CompressorFactory.COMPRESSOR.streamOutput(stream);
             }
             stream.setVersion(version);
             response.writeTo(stream);
diff --git a/core/src/test/java/org/elasticsearch/common/compress/AbstractCompressedStreamTestCase.java b/core/src/test/java/org/elasticsearch/common/compress/DeflateCompressTests.java
similarity index 98%
rename from core/src/test/java/org/elasticsearch/common/compress/AbstractCompressedStreamTestCase.java
rename to core/src/test/java/org/elasticsearch/common/compress/DeflateCompressTests.java
index 0e94f6eaf80..33d11aa23d8 100644
--- a/core/src/test/java/org/elasticsearch/common/compress/AbstractCompressedStreamTestCase.java
+++ b/core/src/test/java/org/elasticsearch/common/compress/DeflateCompressTests.java
@@ -37,13 +37,9 @@ import java.util.concurrent.CountDownLatch;
 /**
  * Test streaming compression (e.g. used for recovery)
  */
-public abstract class AbstractCompressedStreamTestCase extends ESTestCase {
+public class DeflateCompressTests extends ESTestCase {
 
-    private final Compressor compressor;
-
-    protected AbstractCompressedStreamTestCase(Compressor compressor) {
-        this.compressor = compressor;
-    }
+    private final Compressor compressor = new DeflateCompressor();
 
     public void testRandom() throws IOException {
         Random r = random();
diff --git a/core/src/test/java/org/elasticsearch/common/compress/AbstractCompressedXContentTestCase.java b/core/src/test/java/org/elasticsearch/common/compress/DeflateCompressedXContentTests.java
similarity index 63%
rename from core/src/test/java/org/elasticsearch/common/compress/AbstractCompressedXContentTestCase.java
rename to core/src/test/java/org/elasticsearch/common/compress/DeflateCompressedXContentTests.java
index d1c862f8a69..72866d082ae 100644
--- a/core/src/test/java/org/elasticsearch/common/compress/AbstractCompressedXContentTestCase.java
+++ b/core/src/test/java/org/elasticsearch/common/compress/DeflateCompressedXContentTests.java
@@ -35,13 +35,9 @@ import static org.hamcrest.Matchers.not;
 /**
  *
  */
-public abstract class AbstractCompressedXContentTestCase extends ESTestCase {
+public class DeflateCompressedXContentTests extends ESTestCase {
 
-    private final Compressor compressor;
-
-    protected AbstractCompressedXContentTestCase(Compressor compressor) {
-        this.compressor = compressor;
-    }
+    private final Compressor compressor = new DeflateCompressor();
 
     private void assertEquals(CompressedXContent s1, CompressedXContent s2) {
         Assert.assertEquals(s1, s2);
@@ -50,38 +46,26 @@ public abstract class AbstractCompressedXContentTestCase extends ESTestCase {
     }
 
     public void simpleTests() throws IOException {
-        Compressor defaultCompressor = CompressorFactory.defaultCompressor();
-        try {
-            CompressorFactory.setDefaultCompressor(compressor);
-            String str = "---\nf:this is a simple string";
-            CompressedXContent cstr = new CompressedXContent(str);
-            assertThat(cstr.string(), equalTo(str));
-            assertThat(new CompressedXContent(str), equalTo(cstr));
+        String str = "---\nf:this is a simple string";
+        CompressedXContent cstr = new CompressedXContent(str);
+        assertThat(cstr.string(), equalTo(str));
+        assertThat(new CompressedXContent(str), equalTo(cstr));
 
-            String str2 = "---\nf:this is a simple string 2";
-            CompressedXContent cstr2 = new CompressedXContent(str2);
-            assertThat(cstr2.string(), not(equalTo(str)));
-            assertThat(new CompressedXContent(str2), not(equalTo(cstr)));
-            assertEquals(new CompressedXContent(str2), cstr2);
-        } finally {
-            CompressorFactory.setDefaultCompressor(defaultCompressor);
-        }
+        String str2 = "---\nf:this is a simple string 2";
+        CompressedXContent cstr2 = new CompressedXContent(str2);
+        assertThat(cstr2.string(), not(equalTo(str)));
+        assertThat(new CompressedXContent(str2), not(equalTo(cstr)));
+        assertEquals(new CompressedXContent(str2), cstr2);
     }
 
     public void testRandom() throws IOException {
-        Compressor defaultCompressor = CompressorFactory.defaultCompressor();
-        try {
-            CompressorFactory.setDefaultCompressor(compressor);
-            Random r = random();
-            for (int i = 0; i < 1000; i++) {
-                String string = TestUtil.randomUnicodeString(r, 10000);
-                // hack to make it detected as YAML
-                string = "---\n" + string;
-                CompressedXContent compressedXContent = new CompressedXContent(string);
-                assertThat(compressedXContent.string(), equalTo(string));
-            }
-        } finally {
-            CompressorFactory.setDefaultCompressor(defaultCompressor);
+        Random r = random();
+        for (int i = 0; i < 1000; i++) {
+            String string = TestUtil.randomUnicodeString(r, 10000);
+            // hack to make it detected as YAML
+            string = "---\n" + string;
+            CompressedXContent compressedXContent = new CompressedXContent(string);
+            assertThat(compressedXContent.string(), equalTo(string));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/common/compress/deflate/DeflateCompressedStreamTests.java b/core/src/test/java/org/elasticsearch/common/compress/deflate/DeflateCompressedStreamTests.java
deleted file mode 100644
index a6d33585dbc..00000000000
--- a/core/src/test/java/org/elasticsearch/common/compress/deflate/DeflateCompressedStreamTests.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.compress.deflate;
-
-import org.elasticsearch.common.compress.AbstractCompressedStreamTestCase;
-
-public class DeflateCompressedStreamTests extends AbstractCompressedStreamTestCase {
-
-    public DeflateCompressedStreamTests() {
-        super(new DeflateCompressor());
-    }
-
-}
diff --git a/core/src/test/java/org/elasticsearch/common/compress/deflate/DeflateXContentTests.java b/core/src/test/java/org/elasticsearch/common/compress/deflate/DeflateXContentTests.java
deleted file mode 100644
index 359a582e169..00000000000
--- a/core/src/test/java/org/elasticsearch/common/compress/deflate/DeflateXContentTests.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.compress.deflate;
-
-import org.elasticsearch.common.compress.AbstractCompressedXContentTestCase;
-
-public class DeflateXContentTests extends AbstractCompressedXContentTestCase {
-
-    public DeflateXContentTests() {
-        super(new DeflateCompressor());
-    }
-
-}
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/binary/BinaryMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/binary/BinaryMappingTests.java
index 7be0cc8031b..fc8e2ba1872 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/binary/BinaryMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/binary/BinaryMappingTests.java
@@ -76,7 +76,7 @@ public class BinaryMappingTests extends ESSingleNodeTestCase {
 
         // case 2: a value that looks compressed: this used to fail in 1.x
         BytesStreamOutput out = new BytesStreamOutput();
-        try (StreamOutput compressed = CompressorFactory.defaultCompressor().streamOutput(out)) {
+        try (StreamOutput compressed = CompressorFactory.COMPRESSOR.streamOutput(out)) {
             new BytesArray(binaryValue1).writeTo(compressed);
         }
         final byte[] binaryValue2 = out.bytes().toBytes();
diff --git a/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java b/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java
index b2b9e780205..e1589b4cd2f 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/BlobStoreFormatIT.java
@@ -138,7 +138,7 @@ public class BlobStoreFormatIT extends AbstractSnapshotIntegTestCase {
         private BytesReference write(T obj) throws IOException {
             try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()) {
                 if (compress) {
-                    try (StreamOutput compressedStreamOutput = CompressorFactory.defaultCompressor().streamOutput(bytesStreamOutput)) {
+                    try (StreamOutput compressedStreamOutput = CompressorFactory.COMPRESSOR.streamOutput(bytesStreamOutput)) {
                         write(obj, compressedStreamOutput);
                     }
                 } else {

From 258c3e86ab504321d9c40aeb456654575fe79f43 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Tue, 28 Jun 2016 10:52:07 -0700
Subject: [PATCH 1101/1311] Added IngestPlugin api, cutover common and geoip,
 changed ingest factory api to take ProcessorsRegistry

---
 .../ingest/AbstractProcessorFactory.java      |  6 +--
 .../ingest/ConfigurationUtils.java            |  3 +-
 .../org/elasticsearch/ingest/Processor.java   |  2 +-
 .../ingest/ProcessorsRegistry.java            | 47 +---------------
 .../elasticsearch/plugins/IngestPlugin.java   | 47 ++++++++++++++++
 .../elasticsearch/ingest/IngestCloseIT.java   |  2 +-
 .../common/AbstractStringProcessor.java       |  3 +-
 .../ingest/common/AppendProcessor.java        |  3 +-
 .../ingest/common/ConvertProcessor.java       |  3 +-
 .../ingest/common/DateIndexNameProcessor.java |  3 +-
 .../ingest/common/DateProcessor.java          |  3 +-
 .../ingest/common/FailProcessor.java          |  3 +-
 .../ingest/common/ForEachProcessor.java       |  7 ---
 .../ingest/common/GrokProcessor.java          |  3 +-
 .../ingest/common/GsubProcessor.java          |  3 +-
 .../ingest/common/IngestCommonPlugin.java     | 53 +++++++++++--------
 .../ingest/common/JoinProcessor.java          |  3 +-
 .../ingest/common/RemoveProcessor.java        |  3 +-
 .../ingest/common/RenameProcessor.java        |  3 +-
 .../ingest/common/ScriptProcessor.java        |  3 +-
 .../ingest/common/SetProcessor.java           |  3 +-
 .../ingest/common/SortProcessor.java          |  3 +-
 .../ingest/common/SplitProcessor.java         |  3 +-
 .../common/AppendProcessorFactoryTests.java   |  8 +--
 .../common/ConvertProcessorFactoryTests.java  | 10 ++--
 .../common/DateIndexNameFactoryTests.java     | 10 ++--
 .../common/DateProcessorFactoryTests.java     | 20 +++----
 .../common/FailProcessorFactoryTests.java     |  4 +-
 .../common/ForEachProcessorFactoryTests.java  | 14 ++---
 .../common/GrokProcessorFactoryTests.java     |  4 +-
 .../common/GsubProcessorFactoryTests.java     | 10 ++--
 .../common/JoinProcessorFactoryTests.java     |  6 +--
 .../LowercaseProcessorFactoryTests.java       |  4 +-
 .../common/RemoveProcessorFactoryTests.java   |  4 +-
 .../common/RenameProcessorFactoryTests.java   |  6 +--
 .../common/SetProcessorFactoryTests.java      | 10 ++--
 .../common/SplitProcessorFactoryTests.java    |  6 +--
 .../common/TrimProcessorFactoryTests.java     |  4 +-
 .../UppercaseProcessorFactoryTests.java       |  4 +-
 .../attachment/AttachmentProcessor.java       |  3 +-
 .../AttachmentProcessorFactoryTests.java      | 12 ++---
 .../ingest/geoip/GeoIpProcessor.java          |  3 +-
 .../ingest/geoip/IngestGeoIpPlugin.java       | 38 ++++++++++---
 .../geoip/GeoIpProcessorFactoryTests.java     | 18 +++----
 .../elasticsearch/ingest/TestProcessor.java   |  2 +-
 45 files changed, 229 insertions(+), 183 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/plugins/IngestPlugin.java

diff --git a/core/src/main/java/org/elasticsearch/ingest/AbstractProcessorFactory.java b/core/src/main/java/org/elasticsearch/ingest/AbstractProcessorFactory.java
index 0fb50228793..95c4a93c9bc 100644
--- a/core/src/main/java/org/elasticsearch/ingest/AbstractProcessorFactory.java
+++ b/core/src/main/java/org/elasticsearch/ingest/AbstractProcessorFactory.java
@@ -29,10 +29,10 @@ public abstract class AbstractProcessorFactory<P extends Processor> implements P
     public static final String TAG_KEY = "tag";
 
     @Override
-    public P create(Map<String, Object> config) throws Exception {
+    public P create(ProcessorsRegistry registry, Map<String, Object> config) throws Exception {
         String tag = ConfigurationUtils.readOptionalStringProperty(null, null, config, TAG_KEY);
-        return doCreate(tag, config);
+        return doCreate(registry, tag, config);
     }
 
-    protected abstract P doCreate(String tag, Map<String, Object> config) throws Exception;
+    protected abstract P doCreate(ProcessorsRegistry registry, String tag, Map<String, Object> config) throws Exception;
 }
diff --git a/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java b/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
index 623c4f315a4..22eea7d23b7 100644
--- a/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
+++ b/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
@@ -255,8 +255,7 @@ public final class ConfigurationUtils {
                 ConfigurationUtils.readOptionalList(null, null, config, Pipeline.ON_FAILURE_KEY);
 
             List<Processor> onFailureProcessors = readProcessorConfigs(onFailureProcessorConfigs, processorRegistry);
-            Processor processor;
-            processor = factory.create(config);
+            Processor processor = factory.create(processorRegistry, config);
 
             if (onFailureProcessorConfigs != null && onFailureProcessors.isEmpty()) {
                 throw newConfigurationException(processor.getType(), processor.getTag(), Pipeline.ON_FAILURE_KEY,
diff --git a/core/src/main/java/org/elasticsearch/ingest/Processor.java b/core/src/main/java/org/elasticsearch/ingest/Processor.java
index 92c18464e94..934cf52b91b 100644
--- a/core/src/main/java/org/elasticsearch/ingest/Processor.java
+++ b/core/src/main/java/org/elasticsearch/ingest/Processor.java
@@ -53,6 +53,6 @@ public interface Processor {
          * Implementations are responsible for removing the used keys, so that after creating a pipeline ingest can
          * verify if all configurations settings have been used.
          */
-        P create(Map<String, Object> config) throws Exception;
+        P create(ProcessorsRegistry registry, Map<String, Object> config) throws Exception;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java b/core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java
index 28abfc45313..bea00f00bf0 100644
--- a/core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java
+++ b/core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java
@@ -35,32 +35,9 @@ import java.util.function.Function;
 public final class ProcessorsRegistry implements Closeable {
 
     private final Map<String, Processor.Factory> processorFactories;
-    private final TemplateService templateService;
-    private final ScriptService scriptService;
-    private final ClusterService clusterService;
 
-    private ProcessorsRegistry(ScriptService scriptService, ClusterService clusterService,
-                               Map<String, Function<ProcessorsRegistry, Processor.Factory<?>>> providers) {
-        this.templateService = new InternalTemplateService(scriptService);
-        this.scriptService = scriptService;
-        this.clusterService = clusterService;
-        Map<String, Processor.Factory> processorFactories = new HashMap<>();
-        for (Map.Entry<String, Function<ProcessorsRegistry, Processor.Factory<?>>> entry : providers.entrySet()) {
-            processorFactories.put(entry.getKey(), entry.getValue().apply(this));
-        }
-        this.processorFactories = Collections.unmodifiableMap(processorFactories);
-    }
-
-    public TemplateService getTemplateService() {
-        return templateService;
-    }
-
-    public ScriptService getScriptService() {
-        return scriptService;
-    }
-
-    public ClusterService getClusterService() {
-        return clusterService;
+    public ProcessorsRegistry(Map<String, Processor.Factory> processors) {
+        this.processorFactories = Collections.unmodifiableMap(processors);
     }
 
     public Processor.Factory getProcessorFactory(String name) {
@@ -82,24 +59,4 @@ public final class ProcessorsRegistry implements Closeable {
     Map<String, Processor.Factory> getProcessorFactories() {
         return processorFactories;
     }
-
-    public static final class Builder {
-
-        private final Map<String, Function<ProcessorsRegistry, Processor.Factory<?>>> providers = new HashMap<>();
-
-        /**
-         * Adds a processor factory under a specific name.
-         */
-        public void registerProcessor(String name, Function<ProcessorsRegistry, Processor.Factory<?>> provider) {
-            Function<ProcessorsRegistry, Processor.Factory<?>> previous = this.providers.putIfAbsent(name, provider);
-            if (previous != null) {
-                throw new IllegalArgumentException("Processor factory already registered for name [" + name + "]");
-            }
-        }
-
-        public ProcessorsRegistry build(ScriptService scriptService, ClusterService clusterService) {
-            return new ProcessorsRegistry(scriptService, clusterService, providers);
-        }
-
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/plugins/IngestPlugin.java b/core/src/main/java/org/elasticsearch/plugins/IngestPlugin.java
new file mode 100644
index 00000000000..605ae49c32e
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/plugins/IngestPlugin.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import java.util.Collections;
+import java.util.Map;
+
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.TemplateService;
+import org.elasticsearch.script.ScriptService;
+
+/**
+ * An extension point for {@link Plugin} implementations to add custom ingest processors
+ */
+public interface IngestPlugin {
+
+    /**
+     * Returns additional ingest processor types added by this plugin.
+     *
+     * The key of the returned {@link Map} is the unique name for the processor which is specified
+     * in pipeline configurations, and the value is a {@link org.elasticsearch.ingest.Processor.Factory}
+     * to create the processor from a given pipeline configuration.
+     */
+    default Map<String, Processor.Factory> getProcessors(
+        Environment env, ClusterService clusterService, ScriptService scriptService, TemplateService templateService) {
+        return Collections.emptyMap();
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestCloseIT.java b/core/src/test/java/org/elasticsearch/ingest/IngestCloseIT.java
index ed0a302cf73..79e2cc4065f 100644
--- a/core/src/test/java/org/elasticsearch/ingest/IngestCloseIT.java
+++ b/core/src/test/java/org/elasticsearch/ingest/IngestCloseIT.java
@@ -58,7 +58,7 @@ public class IngestCloseIT extends ESSingleNodeTestCase {
 
     public static final class Factory extends AbstractProcessorFactory<TestProcessor> implements Closeable {
         @Override
-        protected TestProcessor doCreate(String tag, Map<String, Object> config) throws Exception {
+        protected TestProcessor doCreate(ProcessorsRegistry registry, String tag, Map<String, Object> config) throws Exception {
             return new TestProcessor("id", "test", ingestDocument -> {
                 throw new UnsupportedOperationException("this code is actually never called from the test");
             });
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
index d9ffd34cac1..45ef6835ff0 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
@@ -23,6 +23,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.ProcessorsRegistry;
 
 import java.util.Map;
 
@@ -61,7 +62,7 @@ abstract class AbstractStringProcessor extends AbstractProcessor {
         }
 
         @Override
-        public T doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public T doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(processorType, processorTag, config, "field");
             return newProcessor(processorTag, field);
         }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java
index af163c3c187..242e3411d3f 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java
@@ -23,6 +23,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.ProcessorsRegistry;
 import org.elasticsearch.ingest.TemplateService;
 import org.elasticsearch.ingest.ValueSource;
 
@@ -73,7 +74,7 @@ public final class AppendProcessor extends AbstractProcessor {
         }
 
         @Override
-        public AppendProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public AppendProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             Object value = ConfigurationUtils.readObject(TYPE, processorTag, config, "value");
             return new AppendProcessor(processorTag, templateService.compile(field), ValueSource.wrap(value, templateService));
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
index 015c56c72c3..4e35003aa88 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
@@ -23,6 +23,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.ProcessorsRegistry;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -162,7 +163,7 @@ public final class ConvertProcessor extends AbstractProcessor {
 
     public static final class Factory extends AbstractProcessorFactory<ConvertProcessor> {
         @Override
-        public ConvertProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public ConvertProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String typeProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "type");
             String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java
index a94d4d048a8..d89430c4bef 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java
@@ -24,6 +24,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.ProcessorsRegistry;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.DateTimeFormat;
@@ -123,7 +124,7 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
     public static final class Factory extends AbstractProcessorFactory<DateIndexNameProcessor> {
 
         @Override
-        protected DateIndexNameProcessor doCreate(String tag, Map<String, Object> config) throws Exception {
+        protected DateIndexNameProcessor doCreate(ProcessorsRegistry registry, String tag, Map<String, Object> config) throws Exception {
             String localeString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "locale");
             String timezoneString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "timezone");
             DateTimeZone timezone = timezoneString == null ? DateTimeZone.UTC : DateTimeZone.forID(timezoneString);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
index b82b9c8b76c..0b17123bcb0 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
@@ -24,6 +24,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.ProcessorsRegistry;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.ISODateTimeFormat;
@@ -111,7 +112,7 @@ public final class DateProcessor extends AbstractProcessor {
     public static final class Factory extends AbstractProcessorFactory<DateProcessor> {
 
         @SuppressWarnings("unchecked")
-        public DateProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public DateProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", DEFAULT_TARGET_FIELD);
             String timezoneString = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "timezone");
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java
index 6c434d85d5a..7c806c08cdf 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java
@@ -23,6 +23,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.ProcessorsRegistry;
 import org.elasticsearch.ingest.TemplateService;
 
 import java.util.Map;
@@ -65,7 +66,7 @@ public final class FailProcessor extends AbstractProcessor {
         }
 
         @Override
-        public FailProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public FailProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
             String message = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "message");
             return new FailProcessor(processorTag, templateService.compile(message));
         }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java
index b6d14d1b8c5..9ee244962ab 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java
@@ -84,13 +84,6 @@ public final class ForEachProcessor extends AbstractProcessor {
     }
 
     public static final class Factory extends AbstractProcessorFactory<ForEachProcessor> {
-
-        private final ProcessorsRegistry processorRegistry;
-
-        public Factory(ProcessorsRegistry processorRegistry) {
-            this.processorRegistry = processorRegistry;
-        }
-
         @Override
         protected ForEachProcessor doCreate(String tag, Map<String, Object> config) throws Exception {
             String field = readStringProperty(TYPE, tag, config, "field");
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
index aa9fbb905cc..1c11cd13487 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
@@ -23,6 +23,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.ProcessorsRegistry;
 
 import java.util.HashMap;
 import java.util.List;
@@ -123,7 +124,7 @@ public final class GrokProcessor extends AbstractProcessor {
         }
 
         @Override
-        public GrokProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public GrokProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
             String matchField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             List<String> matchPatterns = ConfigurationUtils.readList(TYPE, processorTag, config, "patterns");
             boolean traceMatch = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "trace_match", false);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java
index 72bc9e76710..124b1ded1ce 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java
@@ -22,6 +22,7 @@ package org.elasticsearch.ingest.common;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.ProcessorsRegistry;
 
 import java.util.Map;
 import java.util.regex.Matcher;
@@ -80,7 +81,7 @@ public final class GsubProcessor extends AbstractProcessor {
 
     public static final class Factory extends AbstractProcessorFactory<GsubProcessor> {
         @Override
-        public GsubProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public GsubProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
             String field = readStringProperty(TYPE, processorTag, config, "field");
             String pattern = readStringProperty(TYPE, processorTag, config, "pattern");
             String replacement = readStringProperty(TYPE, processorTag, config, "replacement");
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java
index 821e47b85bc..b3877fd5564 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java
@@ -19,8 +19,14 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.TemplateService;
 import org.elasticsearch.node.NodeModule;
+import org.elasticsearch.plugins.IngestPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.ScriptService;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -31,9 +37,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
-public class IngestCommonPlugin extends Plugin {
-
-    public static final String NAME = "ingest-common";
+public class IngestCommonPlugin extends Plugin implements IngestPlugin {
 
     private final Map<String, String> builtinPatterns;
 
@@ -41,26 +45,29 @@ public class IngestCommonPlugin extends Plugin {
         this.builtinPatterns = loadBuiltinPatterns();
     }
 
-    public void onModule(NodeModule nodeModule) {
-        nodeModule.registerProcessor(DateProcessor.TYPE, (registry) -> new DateProcessor.Factory());
-        nodeModule.registerProcessor(SetProcessor.TYPE, (registry) -> new SetProcessor.Factory(registry.getTemplateService()));
-        nodeModule.registerProcessor(AppendProcessor.TYPE, (registry) -> new AppendProcessor.Factory(registry.getTemplateService()));
-        nodeModule.registerProcessor(RenameProcessor.TYPE, (registry) -> new RenameProcessor.Factory());
-        nodeModule.registerProcessor(RemoveProcessor.TYPE, (registry) -> new RemoveProcessor.Factory(registry.getTemplateService()));
-        nodeModule.registerProcessor(SplitProcessor.TYPE, (registry) -> new SplitProcessor.Factory());
-        nodeModule.registerProcessor(JoinProcessor.TYPE, (registry) -> new JoinProcessor.Factory());
-        nodeModule.registerProcessor(UppercaseProcessor.TYPE, (registry) -> new UppercaseProcessor.Factory());
-        nodeModule.registerProcessor(LowercaseProcessor.TYPE, (registry) -> new LowercaseProcessor.Factory());
-        nodeModule.registerProcessor(TrimProcessor.TYPE, (registry) -> new TrimProcessor.Factory());
-        nodeModule.registerProcessor(ConvertProcessor.TYPE, (registry) -> new ConvertProcessor.Factory());
-        nodeModule.registerProcessor(GsubProcessor.TYPE, (registry) -> new GsubProcessor.Factory());
-        nodeModule.registerProcessor(FailProcessor.TYPE, (registry) -> new FailProcessor.Factory(registry.getTemplateService()));
-        nodeModule.registerProcessor(ForEachProcessor.TYPE, (registry) -> new ForEachProcessor.Factory(registry));
-        nodeModule.registerProcessor(DateIndexNameProcessor.TYPE, (registry) -> new DateIndexNameProcessor.Factory());
-        nodeModule.registerProcessor(SortProcessor.TYPE, (registry) -> new SortProcessor.Factory());
-        nodeModule.registerProcessor(GrokProcessor.TYPE, (registry) -> new GrokProcessor.Factory(builtinPatterns));
-        nodeModule.registerProcessor(ScriptProcessor.TYPE, (registry) ->
-            new ScriptProcessor.Factory(registry.getScriptService(), registry.getClusterService()));
+    @Override
+    public Map<String, Processor.Factory> getProcessors(
+        Environment env, ClusterService clusterService, ScriptService scriptService, TemplateService templateService) {
+        Map<String, Processor.Factory> processors = new HashMap<>();
+        processors.put(DateProcessor.TYPE, new DateProcessor.Factory());
+        processors.put(SetProcessor.TYPE, new SetProcessor.Factory(templateService));
+        processors.put(AppendProcessor.TYPE, new AppendProcessor.Factory(templateService));
+        processors.put(RenameProcessor.TYPE, new RenameProcessor.Factory());
+        processors.put(RemoveProcessor.TYPE, new RemoveProcessor.Factory(templateService));
+        processors.put(SplitProcessor.TYPE, new SplitProcessor.Factory());
+        processors.put(JoinProcessor.TYPE, new JoinProcessor.Factory());
+        processors.put(UppercaseProcessor.TYPE, new UppercaseProcessor.Factory());
+        processors.put(LowercaseProcessor.TYPE, new LowercaseProcessor.Factory());
+        processors.put(TrimProcessor.TYPE, new TrimProcessor.Factory());
+        processors.put(ConvertProcessor.TYPE, new ConvertProcessor.Factory());
+        processors.put(GsubProcessor.TYPE, new GsubProcessor.Factory());
+        processors.put(FailProcessor.TYPE, new FailProcessor.Factory(templateService));
+        processors.put(ForEachProcessor.TYPE, new ForEachProcessor.Factory());
+        processors.put(DateIndexNameProcessor.TYPE, new DateIndexNameProcessor.Factory());
+        processors.put(SortProcessor.TYPE, new SortProcessor.Factory());
+        processors.put(GrokProcessor.TYPE, new GrokProcessor.Factory(builtinPatterns));
+        processors.put(ScriptProcessor.TYPE, new ScriptProcessor.Factory(scriptService, clusterService));
+        return Collections.unmodifiableMap(processors);
     }
 
     // Code for loading built-in grok patterns packaged with the jar file:
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java
index 8114d20f28f..b0752d4fab9 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java
@@ -23,6 +23,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.ProcessorsRegistry;
 
 import java.util.List;
 import java.util.Map;
@@ -72,7 +73,7 @@ public final class JoinProcessor extends AbstractProcessor {
 
     public final static class Factory extends AbstractProcessorFactory<JoinProcessor> {
         @Override
-        public JoinProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public JoinProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String separator = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "separator");
             return new JoinProcessor(processorTag, field, separator);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java
index 98c4e18a408..cec88e65d95 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java
@@ -23,6 +23,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.ProcessorsRegistry;
 import org.elasticsearch.ingest.TemplateService;
 
 import java.util.Map;
@@ -64,7 +65,7 @@ public final class RemoveProcessor extends AbstractProcessor {
         }
 
         @Override
-        public RemoveProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public RemoveProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             return new RemoveProcessor(processorTag, templateService.compile(field));
         }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java
index 9143321c4aa..f4c242379e5 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java
@@ -23,6 +23,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.ProcessorsRegistry;
 
 import java.util.Map;
 
@@ -77,7 +78,7 @@ public final class RenameProcessor extends AbstractProcessor {
 
     public static final class Factory extends AbstractProcessorFactory<RenameProcessor> {
         @Override
-        public RenameProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public RenameProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field");
             return new RenameProcessor(processorTag, field, targetField);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
index 6ce85f8a6bf..f4fb302e46f 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.ProcessorsRegistry;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
@@ -91,7 +92,7 @@ public final class ScriptProcessor extends AbstractProcessor {
         }
 
         @Override
-        public ScriptProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public ScriptProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
             String field = readOptionalStringProperty(TYPE, processorTag, config, "field");
             String lang = readStringProperty(TYPE, processorTag, config, "lang");
             String inline = readOptionalStringProperty(TYPE, processorTag, config, "inline");
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java
index a78701645a9..61053fec7e4 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java
@@ -23,6 +23,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.ProcessorsRegistry;
 import org.elasticsearch.ingest.TemplateService;
 import org.elasticsearch.ingest.ValueSource;
 
@@ -84,7 +85,7 @@ public final class SetProcessor extends AbstractProcessor {
         }
 
         @Override
-        public SetProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public SetProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             Object value = ConfigurationUtils.readObject(TYPE, processorTag, config, "value");
             boolean overrideEnabled = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "override", true);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java
index 706a1cef9c5..152526a9f68 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java
@@ -23,6 +23,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.ProcessorsRegistry;
 
 import java.util.Collections;
 import java.util.List;
@@ -114,7 +115,7 @@ public final class SortProcessor extends AbstractProcessor {
     public final static class Factory extends AbstractProcessorFactory<SortProcessor> {
 
         @Override
-        public SortProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public SortProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, FIELD);
             try {
                 SortOrder direction = SortOrder.fromString(
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java
index f7c5e8befc4..92ce7a02341 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java
@@ -23,6 +23,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.ProcessorsRegistry;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -74,7 +75,7 @@ public final class SplitProcessor extends AbstractProcessor {
 
     public static class Factory extends AbstractProcessorFactory<SplitProcessor> {
         @Override
-        public SplitProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public SplitProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             return new SplitProcessor(processorTag, field, ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "separator"));
         }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java
index fbf77cc4285..12a4d50482f 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java
@@ -53,7 +53,7 @@ public class AppendProcessorFactoryTests extends ESTestCase {
         config.put("value", value);
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        AppendProcessor appendProcessor = factory.create(config);
+        AppendProcessor appendProcessor = factory.create(null, config);
         assertThat(appendProcessor.getTag(), equalTo(processorTag));
         assertThat(appendProcessor.getField().execute(Collections.emptyMap()), equalTo("field1"));
         assertThat(appendProcessor.getValue().copyAndResolve(Collections.emptyMap()), equalTo(value));
@@ -63,7 +63,7 @@ public class AppendProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("value", "value1");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@@ -74,7 +74,7 @@ public class AppendProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[value] required property is missing"));
@@ -86,7 +86,7 @@ public class AppendProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         config.put("value", null);
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[value] required property is missing"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java
index 1ec5362af14..c3afc2c5822 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java
@@ -40,7 +40,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
         config.put("type", type.toString());
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        ConvertProcessor convertProcessor = factory.create(config);
+        ConvertProcessor convertProcessor = factory.create(null, config);
         assertThat(convertProcessor.getTag(), equalTo(processorTag));
         assertThat(convertProcessor.getField(), equalTo("field1"));
         assertThat(convertProcessor.getTargetField(), equalTo("field1"));
@@ -54,7 +54,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         config.put("type", type);
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), Matchers.equalTo("[type] type [" + type + "] not supported, cannot convert field."));
@@ -70,7 +70,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
         String type = "type-" + randomAsciiOfLengthBetween(1, 10);
         config.put("type", type);
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), Matchers.equalTo("[field] required property is missing"));
@@ -82,7 +82,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), Matchers.equalTo("[type] required property is missing"));
@@ -98,7 +98,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
         config.put("type", type.toString());
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        ConvertProcessor convertProcessor = factory.create(config);
+        ConvertProcessor convertProcessor = factory.create(null, config);
         assertThat(convertProcessor.getTag(), equalTo(processorTag));
         assertThat(convertProcessor.getField(), equalTo("field1"));
         assertThat(convertProcessor.getTargetField(), equalTo("field2"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java
index 42877236b88..04e2dbf72a4 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java
@@ -36,7 +36,7 @@ public class DateIndexNameFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("date_rounding", "y");
 
-        DateIndexNameProcessor processor = factory.create(config);
+        DateIndexNameProcessor processor = factory.create(null, config);
         assertThat(processor.getDateFormats().size(), Matchers.equalTo(1));
         assertThat(processor.getField(), Matchers.equalTo("_field"));
         assertThat(processor.getIndexNamePrefix(), Matchers.equalTo(""));
@@ -53,7 +53,7 @@ public class DateIndexNameFactoryTests extends ESTestCase {
         config.put("date_rounding", "y");
         config.put("date_formats", Arrays.asList("UNIX", "UNIX_MS"));
 
-        DateIndexNameProcessor processor = factory.create(config);
+        DateIndexNameProcessor processor = factory.create(null, config);
         assertThat(processor.getDateFormats().size(), Matchers.equalTo(2));
 
         config = new HashMap<>();
@@ -62,7 +62,7 @@ public class DateIndexNameFactoryTests extends ESTestCase {
         config.put("date_rounding", "y");
         config.put("index_name_format", "yyyyMMdd");
 
-        processor = factory.create(config);
+        processor = factory.create(null, config);
         assertThat(processor.getIndexNameFormat(), Matchers.equalTo("yyyyMMdd"));
 
         config = new HashMap<>();
@@ -71,7 +71,7 @@ public class DateIndexNameFactoryTests extends ESTestCase {
         config.put("date_rounding", "y");
         config.put("timezone", "+02:00");
 
-        processor = factory.create(config);
+        processor = factory.create(null, config);
         assertThat(processor.getTimezone(), Matchers.equalTo(DateTimeZone.forOffsetHours(2)));
 
         config = new HashMap<>();
@@ -79,7 +79,7 @@ public class DateIndexNameFactoryTests extends ESTestCase {
         config.put("index_name_prefix", "_prefix");
         config.put("date_rounding", "y");
 
-        processor = factory.create(config);
+        processor = factory.create(null, config);
         assertThat(processor.getIndexNamePrefix(), Matchers.equalTo("_prefix"));
     }
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java
index 65dcdf6082c..9d95aa119fa 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java
@@ -43,7 +43,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("formats", Collections.singletonList("dd/MM/yyyyy"));
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        DateProcessor processor = factory.create(config);
+        DateProcessor processor = factory.create(null, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo(sourceField));
         assertThat(processor.getTargetField(), equalTo(DateProcessor.DEFAULT_TARGET_FIELD));
@@ -60,7 +60,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("formats", Collections.singletonList("dd/MM/yyyyy"));
 
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("processor creation should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), containsString("[field] required property is missing"));
@@ -76,7 +76,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("target_field", targetField);
 
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("processor creation should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), containsString("[formats] required property is missing"));
@@ -92,7 +92,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         Locale locale = randomLocale(random());
         config.put("locale", locale.toLanguageTag());
 
-        DateProcessor processor = factory.create(config);
+        DateProcessor processor = factory.create(null, config);
         assertThat(processor.getLocale().toLanguageTag(), equalTo(locale.toLanguageTag()));
     }
 
@@ -104,7 +104,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("formats", Collections.singletonList("dd/MM/yyyyy"));
         config.put("locale", "invalid_locale");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("should fail with invalid locale");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), equalTo("Invalid language tag specified: invalid_locale"));
@@ -120,7 +120,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
 
         DateTimeZone timezone = randomDateTimeZone();
         config.put("timezone", timezone.getID());
-        DateProcessor processor = factory.create(config);
+        DateProcessor processor = factory.create(null, config);
         assertThat(processor.getTimezone(), equalTo(timezone));
     }
 
@@ -132,7 +132,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("match_formats", Collections.singletonList("dd/MM/yyyyy"));
         config.put("timezone", "invalid_timezone");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("invalid timezone should fail");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), equalTo("The datetime zone id 'invalid_timezone' is not recognised"));
@@ -146,7 +146,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("field", sourceField);
         config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"));
 
-        DateProcessor processor = factory.create(config);
+        DateProcessor processor = factory.create(null, config);
         assertThat(processor.getFormats(), equalTo(Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy")));
     }
 
@@ -158,7 +158,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("formats", "dd/MM/yyyy");
 
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("processor creation should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), containsString("[formats] property isn't a list, but of type [java.lang.String]"));
@@ -174,7 +174,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("target_field", targetField);
         config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"));
 
-        DateProcessor processor = factory.create(config);
+        DateProcessor processor = factory.create(null, config);
         assertThat(processor.getTargetField(), equalTo(targetField));
     }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java
index db16b78b316..da2a96f368a 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java
@@ -45,7 +45,7 @@ public class FailProcessorFactoryTests extends ESTestCase {
         config.put("message", "error");
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        FailProcessor failProcessor = factory.create(config);
+        FailProcessor failProcessor = factory.create(null, config);
         assertThat(failProcessor.getTag(), equalTo(processorTag));
         assertThat(failProcessor.getMessage().execute(Collections.emptyMap()), equalTo("error"));
     }
@@ -53,7 +53,7 @@ public class FailProcessorFactoryTests extends ESTestCase {
     public void testCreateMissingMessageField() throws Exception {
         Map<String, Object> config = new HashMap<>();
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[message] required property is missing"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java
index d45e98ab06a..8f8aa85ae59 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java
@@ -36,16 +36,16 @@ import static org.mockito.Mockito.mock;
 public class ForEachProcessorFactoryTests extends ESTestCase {
 
     public void testCreate() throws Exception {
-        ProcessorsRegistry.Builder builder = new ProcessorsRegistry.Builder();
         Processor processor = new TestProcessor(ingestDocument -> {});
-        builder.registerProcessor("_name", (registry) -> config -> processor);
-        ProcessorsRegistry registry = builder.build(mock(ScriptService.class), mock(ClusterService.class));
-        ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(registry);
+        Map<String, Processor.Factory> processors = new HashMap<>();
+        processors.put("_name", (r, c) -> processor);
+        ProcessorsRegistry registry = new ProcessorsRegistry(processors);
+        ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory();
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("processors", Collections.singletonList(Collections.singletonMap("_name", Collections.emptyMap())));
-        ForEachProcessor forEachProcessor = forEachFactory.create(config);
+        ForEachProcessor forEachProcessor = forEachFactory.create(registry, config);
         assertThat(forEachProcessor, Matchers.notNullValue());
         assertThat(forEachProcessor.getField(), Matchers.equalTo("_field"));
         assertThat(forEachProcessor.getProcessors().size(), Matchers.equalTo(1));
@@ -54,7 +54,7 @@ public class ForEachProcessorFactoryTests extends ESTestCase {
         config = new HashMap<>();
         config.put("processors", Collections.singletonList(Collections.singletonMap("_name", Collections.emptyMap())));
         try {
-            forEachFactory.create(config);
+            forEachFactory.create(registry, config);
             fail("exception expected");
         } catch (Exception e) {
             assertThat(e.getMessage(), Matchers.equalTo("[field] required property is missing"));
@@ -63,7 +63,7 @@ public class ForEachProcessorFactoryTests extends ESTestCase {
         config = new HashMap<>();
         config.put("field", "_field");
         try {
-            forEachFactory.create(config);
+            forEachFactory.create(registry, config);
             fail("exception expected");
         } catch (Exception e) {
             assertThat(e.getMessage(), Matchers.equalTo("[processors] required property is missing"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java
index a7a133b4363..4b862f8f92d 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java
@@ -40,7 +40,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         config.put("patterns", Collections.singletonList("(?<foo>\\w+)"));
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        GrokProcessor processor = factory.create(config);
+        GrokProcessor processor = factory.create(null, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getMatchField(), equalTo("_field"));
         assertThat(processor.getGrok(), notNullValue());
@@ -78,7 +78,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("patterns", Collections.singletonList("%{MY_PATTERN:name}!"));
         config.put("pattern_definitions", Collections.singletonMap("MY_PATTERN", "foo"));
-        GrokProcessor processor = factory.create(config);
+        GrokProcessor processor = factory.create(null, config);
         assertThat(processor.getMatchField(), equalTo("_field"));
         assertThat(processor.getGrok(), notNullValue());
         assertThat(processor.getGrok().match("foo!"), equalTo(true));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java
index 60cceb34024..a579503165b 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java
@@ -39,7 +39,7 @@ public class GsubProcessorFactoryTests extends ESTestCase {
         config.put("replacement", "-");
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        GsubProcessor gsubProcessor = factory.create(config);
+        GsubProcessor gsubProcessor = factory.create(null, config);
         assertThat(gsubProcessor.getTag(), equalTo(processorTag));
         assertThat(gsubProcessor.getField(), equalTo("field1"));
         assertThat(gsubProcessor.getPattern().toString(), equalTo("\\."));
@@ -52,7 +52,7 @@ public class GsubProcessorFactoryTests extends ESTestCase {
         config.put("pattern", "\\.");
         config.put("replacement", "-");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@@ -65,7 +65,7 @@ public class GsubProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         config.put("replacement", "-");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[pattern] required property is missing"));
@@ -78,7 +78,7 @@ public class GsubProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         config.put("pattern", "\\.");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[replacement] required property is missing"));
@@ -92,7 +92,7 @@ public class GsubProcessorFactoryTests extends ESTestCase {
         config.put("pattern", "[");
         config.put("replacement", "-");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), containsString("[pattern] Invalid regex pattern. Unclosed character class"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java
index 970fd8b8b9a..43fd63ca228 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java
@@ -37,7 +37,7 @@ public class JoinProcessorFactoryTests extends ESTestCase {
         config.put("separator", "-");
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        JoinProcessor joinProcessor = factory.create(config);
+        JoinProcessor joinProcessor = factory.create(null, config);
         assertThat(joinProcessor.getTag(), equalTo(processorTag));
         assertThat(joinProcessor.getField(), equalTo("field1"));
         assertThat(joinProcessor.getSeparator(), equalTo("-"));
@@ -48,7 +48,7 @@ public class JoinProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("separator", "-");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@@ -60,7 +60,7 @@ public class JoinProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[separator] required property is missing"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorFactoryTests.java
index 4dec115458c..966a4a31161 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorFactoryTests.java
@@ -36,7 +36,7 @@ public class LowercaseProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        LowercaseProcessor uppercaseProcessor = factory.create(config);
+        LowercaseProcessor uppercaseProcessor = factory.create(null, config);
         assertThat(uppercaseProcessor.getTag(), equalTo(processorTag));
         assertThat(uppercaseProcessor.getField(), equalTo("field1"));
     }
@@ -45,7 +45,7 @@ public class LowercaseProcessorFactoryTests extends ESTestCase {
         LowercaseProcessor.Factory factory = new LowercaseProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java
index a5f88103e96..194380838db 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java
@@ -45,7 +45,7 @@ public class RemoveProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        RemoveProcessor removeProcessor = factory.create(config);
+        RemoveProcessor removeProcessor = factory.create(null, config);
         assertThat(removeProcessor.getTag(), equalTo(processorTag));
         assertThat(removeProcessor.getField().execute(Collections.emptyMap()), equalTo("field1"));
     }
@@ -53,7 +53,7 @@ public class RemoveProcessorFactoryTests extends ESTestCase {
     public void testCreateMissingField() throws Exception {
         Map<String, Object> config = new HashMap<>();
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java
index c078f09dd92..4c6f8cadba4 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java
@@ -37,7 +37,7 @@ public class RenameProcessorFactoryTests extends ESTestCase {
         config.put("target_field", "new_field");
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        RenameProcessor renameProcessor = factory.create(config);
+        RenameProcessor renameProcessor = factory.create(null, config);
         assertThat(renameProcessor.getTag(), equalTo(processorTag));
         assertThat(renameProcessor.getField(), equalTo("old_field"));
         assertThat(renameProcessor.getTargetField(), equalTo("new_field"));
@@ -48,7 +48,7 @@ public class RenameProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("target_field", "new_field");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@@ -60,7 +60,7 @@ public class RenameProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "old_field");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[target_field] required property is missing"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java
index b8c97a379cb..02b3f219980 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java
@@ -46,7 +46,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
         config.put("value", "value1");
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        SetProcessor setProcessor = factory.create(config);
+        SetProcessor setProcessor = factory.create(null, config);
         assertThat(setProcessor.getTag(), equalTo(processorTag));
         assertThat(setProcessor.getField().execute(Collections.emptyMap()), equalTo("field1"));
         assertThat(setProcessor.getValue().copyAndResolve(Collections.emptyMap()), equalTo("value1"));
@@ -61,7 +61,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
         config.put("override", overrideEnabled);
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        SetProcessor setProcessor = factory.create(config);
+        SetProcessor setProcessor = factory.create(null, config);
         assertThat(setProcessor.getTag(), equalTo(processorTag));
         assertThat(setProcessor.getField().execute(Collections.emptyMap()), equalTo("field1"));
         assertThat(setProcessor.getValue().copyAndResolve(Collections.emptyMap()), equalTo("value1"));
@@ -72,7 +72,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("value", "value1");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@@ -83,7 +83,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[value] required property is missing"));
@@ -95,7 +95,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         config.put("value", null);
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[value] required property is missing"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java
index c747807b710..0ed7c3d7191 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java
@@ -37,7 +37,7 @@ public class SplitProcessorFactoryTests extends ESTestCase {
         config.put("separator", "\\.");
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        SplitProcessor splitProcessor = factory.create(config);
+        SplitProcessor splitProcessor = factory.create(null, config);
         assertThat(splitProcessor.getTag(), equalTo(processorTag));
         assertThat(splitProcessor.getField(), equalTo("field1"));
         assertThat(splitProcessor.getSeparator(), equalTo("\\."));
@@ -48,7 +48,7 @@ public class SplitProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("separator", "\\.");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@@ -60,7 +60,7 @@ public class SplitProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[separator] required property is missing"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorFactoryTests.java
index 54904775478..145c7837b80 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorFactoryTests.java
@@ -36,7 +36,7 @@ public class TrimProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        TrimProcessor uppercaseProcessor = factory.create(config);
+        TrimProcessor uppercaseProcessor = factory.create(null, config);
         assertThat(uppercaseProcessor.getTag(), equalTo(processorTag));
         assertThat(uppercaseProcessor.getField(), equalTo("field1"));
     }
@@ -45,7 +45,7 @@ public class TrimProcessorFactoryTests extends ESTestCase {
         TrimProcessor.Factory factory = new TrimProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorFactoryTests.java
index cd4d1faf767..83d7a74277a 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorFactoryTests.java
@@ -36,7 +36,7 @@ public class UppercaseProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        UppercaseProcessor uppercaseProcessor = factory.create(config);
+        UppercaseProcessor uppercaseProcessor = factory.create(null, config);
         assertThat(uppercaseProcessor.getTag(), equalTo(processorTag));
         assertThat(uppercaseProcessor.getField(), equalTo("field1"));
     }
@@ -45,7 +45,7 @@ public class UppercaseProcessorFactoryTests extends ESTestCase {
         UppercaseProcessor.Factory factory = new UppercaseProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
diff --git a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
index 40ebe2592ab..689adfa1acb 100644
--- a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
+++ b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.ProcessorsRegistry;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -155,7 +156,7 @@ public final class AttachmentProcessor extends AbstractProcessor {
         static final Set<Property> DEFAULT_PROPERTIES = EnumSet.allOf(Property.class);
 
         @Override
-        public AttachmentProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public AttachmentProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
             String field = readStringProperty(TYPE, processorTag, config, "field");
             String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "attachment");
             List<String> properyNames = readOptionalList(TYPE, processorTag, config, "properties");
diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java
index 6bd4e07702e..dca2d2a2990 100644
--- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java
+++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java
@@ -48,7 +48,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
 
-        AttachmentProcessor processor = factory.create(config);
+        AttachmentProcessor processor = factory.create(null, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("attachment"));
@@ -63,7 +63,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
 
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        AttachmentProcessor processor = factory.create(config);
+        AttachmentProcessor processor = factory.create(null, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getIndexedChars(), is(indexedChars));
     }
@@ -72,7 +72,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("target_field", "_field");
-        AttachmentProcessor processor = factory.create(config);
+        AttachmentProcessor processor = factory.create(null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("_field"));
     }
@@ -89,7 +89,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("properties", fieldNames);
-        AttachmentProcessor processor = factory.create(config);
+        AttachmentProcessor processor = factory.create(null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getProperties(), equalTo(properties));
     }
@@ -99,7 +99,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("properties", Collections.singletonList("invalid"));
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("exception expected");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), containsString("[properties] illegal field option [invalid]"));
@@ -113,7 +113,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("properties", "invalid");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("exception expected");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]"));
diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
index c29bf4aa65d..083a0d07cdf 100644
--- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
+++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
@@ -36,6 +36,7 @@ import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.ProcessorsRegistry;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -231,7 +232,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
         }
 
         @Override
-        public GeoIpProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public GeoIpProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
             String ipField = readStringProperty(TYPE, processorTag, config, "field");
             String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "geoip");
             String databaseFile = readStringProperty(TYPE, processorTag, config, "database_file", "GeoLite2-City.mmdb.gz");
diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
index 8339ff1fc60..b5574616a4a 100644
--- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
+++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
@@ -20,9 +20,18 @@
 package org.elasticsearch.ingest.geoip;
 
 import com.maxmind.geoip2.DatabaseReader;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.TemplateService;
 import org.elasticsearch.node.NodeModule;
+import org.elasticsearch.plugins.IngestPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.ScriptService;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
@@ -36,15 +45,25 @@ import java.util.Map;
 import java.util.stream.Stream;
 import java.util.zip.GZIPInputStream;
 
-public class IngestGeoIpPlugin extends Plugin {
+public class IngestGeoIpPlugin extends Plugin implements IngestPlugin, Closeable {
 
-    public void onModule(NodeModule nodeModule) throws IOException {
-        Path geoIpConfigDirectory = nodeModule.getNode().getEnvironment().configFile().resolve("ingest-geoip");
-        Map<String, DatabaseReader> databaseReaders = loadDatabaseReaders(geoIpConfigDirectory);
-        nodeModule.registerProcessor(GeoIpProcessor.TYPE, (registry) -> new GeoIpProcessor.Factory(databaseReaders));
+    private Map<String, DatabaseReader> databaseReaders;
+
+    @Override
+    public synchronized Map<String, Processor.Factory> getProcessors(
+        Environment env, ClusterService clusterService, ScriptService scriptService, TemplateService templateService) {
+        if (databaseReaders == null) {
+            Path geoIpConfigDirectory = env.configFile().resolve("ingest-geoip");
+            try {
+                databaseReaders = loadDatabaseReaders(geoIpConfigDirectory);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+        return Collections.singletonMap(GeoIpProcessor.TYPE, new GeoIpProcessor.Factory(databaseReaders));
     }
 
-    public static Map<String, DatabaseReader> loadDatabaseReaders(Path geoIpConfigDirectory) throws IOException {
+    static Map<String, DatabaseReader> loadDatabaseReaders(Path geoIpConfigDirectory) throws IOException {
         if (Files.exists(geoIpConfigDirectory) == false && Files.isDirectory(geoIpConfigDirectory)) {
             throw new IllegalStateException("the geoip directory [" + geoIpConfigDirectory  + "] containing databases doesn't exist");
         }
@@ -65,4 +84,11 @@ public class IngestGeoIpPlugin extends Plugin {
         }
         return Collections.unmodifiableMap(databaseReaders);
     }
+
+    @Override
+    public synchronized void close() throws IOException {
+        if (databaseReaders != null) {
+            IOUtils.close(databaseReaders.values());
+        }
+    }
 }
diff --git a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
index 04729162729..0d9309a3948 100644
--- a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
+++ b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
@@ -78,7 +78,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
 
-        GeoIpProcessor processor = factory.create(config);
+        GeoIpProcessor processor = factory.create(null, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("geoip"));
@@ -96,7 +96,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
 
-        GeoIpProcessor processor = factory.create(config);
+        GeoIpProcessor processor = factory.create(null, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("geoip"));
@@ -109,7 +109,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("target_field", "_field");
-        GeoIpProcessor processor = factory.create(config);
+        GeoIpProcessor processor = factory.create(null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("_field"));
     }
@@ -119,7 +119,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("database_file", "GeoLite2-Country.mmdb.gz");
-        GeoIpProcessor processor = factory.create(config);
+        GeoIpProcessor processor = factory.create(null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("geoip"));
         assertThat(processor.getDbReader().getMetadata().getDatabaseType(), equalTo("GeoLite2-Country"));
@@ -135,7 +135,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         String cityProperty = RandomPicks.randomFrom(Randomness.get(), cityOnlyProperties).toString();
         config.put("properties", Collections.singletonList(cityProperty));
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("Exception expected");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[properties] illegal property value [" + cityProperty +
@@ -150,7 +150,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("database_file", "does-not-exist.mmdb.gz");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("Exception expected");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[database_file] database file [does-not-exist.mmdb.gz] doesn't exist"));
@@ -171,7 +171,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("properties", fieldNames);
-        GeoIpProcessor processor = factory.create(config);
+        GeoIpProcessor processor = factory.create(null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getProperties(), equalTo(properties));
     }
@@ -183,7 +183,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("properties", Collections.singletonList("invalid"));
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("exception expected");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [IP, COUNTRY_ISO_CODE, " +
@@ -194,7 +194,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("properties", "invalid");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("exception expected");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]"));
diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java b/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java
index e36d73a8d9f..48381b87142 100644
--- a/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java
+++ b/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java
@@ -66,7 +66,7 @@ public class TestProcessor implements Processor {
 
     public static final class Factory extends AbstractProcessorFactory<TestProcessor> {
         @Override
-        public TestProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public TestProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
             return new TestProcessor(processorTag, "test-processor", ingestDocument -> {});
         }
     }

From 44ee56c07367fe97585f4b933b97e464938fd168 Mon Sep 17 00:00:00 2001
From: Colin Goodheart-Smithe <colings86@users.noreply.github.com>
Date: Tue, 28 Jun 2016 19:05:52 +0100
Subject: [PATCH 1102/1311] Added documentation for aggregation profiling

---
 docs/reference/search/profile.asciidoc | 216 +++++++++++++++++++------
 1 file changed, 166 insertions(+), 50 deletions(-)

diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc
index 25820d04800..62e04e669a1 100644
--- a/docs/reference/search/profile.asciidoc
+++ b/docs/reference/search/profile.asciidoc
@@ -4,25 +4,12 @@
 experimental[]
 
 The Profile API provides detailed timing information about the execution of individual components
-in a query.  It gives the user insight into how queries are executed at a low level so that
-the user can understand why certain queries are slow, and take steps to improve their slow queries.
+in a search request.  It gives the user insight into how search requests are executed at a low level so that
+the user can understand why certain requests are slow, and take steps to improve them.
 
-The output from the Profile API is *very* verbose, especially for complicated queries executed across
+The output from the Profile API is *very* verbose, especially for complicated requests executed across
 many shards. Pretty-printing the response is recommended to help understand the output
 
-[NOTE]
-=======================================
-The details provided by the Profile API directly expose Lucene class names and concepts, which means
-that complete interpretation of the results require fairly advanced knowledge of Lucene.  This
-page attempts to give a crash-course in how Lucene executes queries so that you can use the Profile API to successfully
-diagnose and debug queries, but it is only an overview.  For complete understanding, please refer
-to Lucene's documentation and, in places, the code.
-
-With that said, a complete understanding is often not required to fix a slow query.  It is usually
-sufficient to see that a particular component of a query is slow, and not necessarily understand why
-the `advance` phase of that query is the cause, for example.
-=======================================
-
 [float]
 === Usage
 
@@ -35,7 +22,7 @@ curl -XGET 'localhost:9200/_search' -d '{
   "query" : {
     "match" : { "message" : "search test" }
   }
-}
+}'
 --------------------------------------------------
 <1> Setting the top-level `profile` parameter to `true` will enable profiling
 for the search
@@ -141,7 +128,8 @@ First, the overall structure of the profile response is as follows:
                     "rewrite_time": 185002,     <3>
                     "collector": [...]          <4>
                  }
-              ]
+              ],
+              "aggregations": [...]             <5>
            }
         ]
      }
@@ -152,6 +140,7 @@ by a unique ID
 <2> Each profile contains a section which holds details about the query execution
 <3> Each profile has a single time representing the cumulative rewrite time
 <4> Each profile also contains a section about the Lucene Collectors which run the search
+<5> Each profile contains a section which holds the details about the aggregation execution
 
 Because a search request may be executed against one or more shards in an index, and a search may cover
 one or more indices, the top level element in the profile response is an array of `shard` objects.
@@ -164,12 +153,26 @@ But occasionally multiple searches will be executed, such as including a global
 a secondary "match_all" query for the global context).
 
 Inside each `search` object there will be two arrays of profiled information:
-a `query` array and a `collector` array.  In the future, more sections may be added, such as `suggest`, `highlight`,
-`aggregations`, etc
+a `query` array and a `collector` array.  Alongside the `search` object is an `aggregations` object that contains the profile information for the aggregations. In the future, more sections may be added, such as `suggest`, `highlight`, etc
 
 There will also be a `rewrite` metric showing the total time spent rewriting the query (in nanoseconds).
 
-=== `query` Section
+=== Profiling Queries
+
+[NOTE]
+=======================================
+The details provided by the Profile API directly expose Lucene class names and concepts, which means
+that complete interpretation of the results require fairly advanced knowledge of Lucene.  This
+page attempts to give a crash-course in how Lucene executes queries so that you can use the Profile API to successfully
+diagnose and debug queries, but it is only an overview.  For complete understanding, please refer
+to Lucene's documentation and, in places, the code.
+
+With that said, a complete understanding is often not required to fix a slow query.  It is usually
+sufficient to see that a particular component of a query is slow, and not necessarily understand why
+the `advance` phase of that query is the cause, for example.
+=======================================
+
+==== `query` Section
 
 The `query` section contains detailed timing of the query tree executed by Lucene on a particular shard.
 The overall structure of this query tree will resemble your original Elasticsearch query, but may be slightly
@@ -217,9 +220,9 @@ that in a moment.  Finally, the `"children"` array lists any sub-queries that ma
 values ("search test"), our BooleanQuery holds two children TermQueries.  They have identical information (type, time,
 breakdown, etc).  Children are allowed to have their own children.
 
-==== Timing Breakdown
+===== Timing Breakdown
 
-The `breakdown` component lists detailed timing statistics about low-level Lucene execution:
+The `"breakdown"` component lists detailed timing statistics about low-level Lucene execution:
 
 [source,js]
 --------------------------------------------------
@@ -235,14 +238,14 @@ The `breakdown` component lists detailed timing statistics about low-level Lucen
 --------------------------------------------------
 
 Timings are listed in wall-clock nanoseconds and are not normalized at all.  All caveats about the overall
-`time` apply here.  The intention of the breakdown is to give you a feel for A) what machinery in Lucene is
+`"time"` apply here.  The intention of the breakdown is to give you a feel for A) what machinery in Lucene is
 actually eating time, and B) the magnitude of differences in times between the various components.  Like the overall time,
 the breakdown is inclusive of all children times.
 
 The meaning of the stats are as follows:
 
 [float]
-=== All parameters:
+==== All parameters:
 
 [horizontal]
 `create_weight`::
@@ -303,7 +306,7 @@ The meaning of the stats are as follows:
     This records the time taken to score a particular document via it's Scorer
 
 
-=== `collectors` Section
+==== `collectors` Section
 
 The Collectors portion of the response shows high-level execution details. Lucene works by defining a "Collector"
 which is responsible for coordinating the traversal, scoring and collection of matching documents.  Collectors
@@ -379,7 +382,7 @@ For reference, the various collector reason's are:
 
 
 
-=== `rewrite` Section
+==== `rewrite` Section
 
 All queries in Lucene undergo a "rewriting" process.  A query (and its sub-queries) may be rewritten one or
 more times, and the process continues until the query stops changing.  This process allows Lucene to perform
@@ -391,7 +394,7 @@ The rewriting process is complex and difficult to display, since queries can cha
 showing the intermediate results, the total rewrite time is simply displayed as a value (in nanoseconds).  This
 value is cumulative and contains the total time for all queries being rewritten.
 
-=== A more complex example
+==== A more complex example
 
 To demonstrate a slightly more complex query and the associated results, we can profile the following query:
 
@@ -563,28 +566,7 @@ The Collector tree is fairly straightforward, showing how a single MultiCollecto
 to execute the post_filter (and in turn wraps the normal scoring SimpleCollector), a BucketCollector to run
 all scoped aggregations.  In the MatchAll search, there is a single GlobalAggregator to run the global aggregation.
 
-=== Performance Notes
-
-Like any profiler, the Profile API introduce a non-negligible overhead to query execution.  The act of instrumenting
-low-level method calls such as `advance` and `next_doc` can be fairly expensive, since these methods are called
-in tight loops.  Therefore, profiling should not be enabled in production settings by default, and should not
-be compared against non-profiled query times.  Profiling is just a diagnostic tool.
-
-There are also cases where special Lucene optimizations are disabled, since they are not amenable to profiling.  This
-could cause some queries to report larger relative times than their non-profiled counterparts, but in general should
-not have a drastic effect compared to other components in the profiled query.
-
-=== Limitations
-
-- Profiling statistics are currently not available for suggestions, highlighting, `dfs_query_then_fetch`
-- Detailed breakdown for aggregations is not currently available past the high-level overview provided
-from the Collectors
-- The Profiler is still highly experimental. The Profiler is instrumenting parts of Lucene that were
-never designed to be exposed in this manner, and so all results should be viewed as a best effort to provide detailed
-diagnostics.  We hope to improve this over time. If you find obviously wrong numbers, strange query structures or
-other bugs, please report them!
-
-=== Understanding MultiTermQuery output
+==== Understanding MultiTermQuery output
 
 A special note needs to be made about the `MultiTermQuery` class of queries.  This includes wildcards, regex and fuzzy
 queries.  These queries emit very verbose responses, and are not overly structured.
@@ -602,3 +584,137 @@ just not the physical layout in the response, so it is sufficient to just analyz
 ignore it's children if you find the details too tricky to interpret.
 
 Hopefully this will be fixed in future iterations, but it is a tricky problem to solve and still in-progress :)
+
+=== Profiling Aggregations
+
+==== `aggregations` Section
+
+The `aggregations` section contains detailed timing of the aggregation tree executed by a particular shard.
+The overall structure of this aggregation tree will resemble your original Elasticsearch request.  Let's consider
+the following example aggregations request:
+
+[source,js]
+--------------------------------------------------
+curl -XGET "http://localhost:9200/house-prices/_search" -d'
+{
+  "profile": true,
+  "size": 0,
+  "aggs": {
+    "property_type": {
+      "terms": {
+        "field": "propertyType"
+      },
+      "aggs": {
+        "avg_price": {
+          "avg": {
+            "field": "price"
+          }
+        }
+      }
+    }
+  }
+}'
+--------------------------------------------------
+
+Which yields the following aggregation profile output
+
+[source,js]
+--------------------------------------------------
+"aggregations": [
+  {
+    "type": "org.elasticsearch.search.aggregations.bucket.terms.GlobalOrdinalsStringTermsAggregator",
+    "description": "property_type",
+    "time": "4280.456978ms",
+    "breakdown": {
+      "reduce": 0,
+      "build_aggregation": 49765,
+      "initialise": 52785,
+      "collect": 3155490036
+    },
+    "children": [
+      {
+        "type": "org.elasticsearch.search.aggregations.metrics.avg.AvgAggregator",
+        "description": "avg_price",
+        "time": "1124.864392ms",
+        "breakdown": {
+          "reduce": 0,
+          "build_aggregation": 1394,
+          "initialise": 2883,
+          "collect": 1124860115
+        }
+      }
+    ]
+  }
+]
+--------------------------------------------------
+
+From the profile structure we can see our `property_type` terms aggregation which is internally represented by the
+`GlobalOrdinalsStringTermsAggregator` class and the sub aggregator `avg_price` which is internally represented by the `AvgAggregator` class. The `type` field displays the class used internally to represent the aggregation. The `description` field displays the name of the aggregation.
+
+The `"time"` field shows that it took ~4 seconds for the entire aggregation to execute.  The recorded time is inclusive
+of all children.
+
+The `"breakdown"` field will give detailed stats about how the time was spent, we'll look at
+that in a moment.  Finally, the `"children"` array lists any sub-aggregations that may be present.  Because we have an `avg_price` aggregation as a sub-aggregation to the `property_type` aggregation we see it listed as a child of the `property_type` aggregation.  the two aggregation outputs have identical information (type, time,
+breakdown, etc).  Children are allowed to have their own children.
+
+===== Timing Breakdown
+
+The `"breakdown"` component lists detailed timing statistics about low-level Lucene execution:
+
+[source,js]
+--------------------------------------------------
+"breakdown": {
+  "reduce": 0,
+  "build_aggregation": 49765,
+  "initialise": 52785,
+  "collect": 3155490036
+}
+--------------------------------------------------
+
+Timings are listed in wall-clock nanoseconds and are not normalized at all.  All caveats about the overall
+`time` apply here.  The intention of the breakdown is to give you a feel for A) what machinery in Elasticsearch is
+actually eating time, and B) the magnitude of differences in times between the various components.  Like the overall time,
+the breakdown is inclusive of all children times.
+
+The meaning of the stats are as follows:
+
+[float]
+==== All parameters:
+
+[horizontal]
+`initialise`::
+
+    This times how long it takes to create and initialise the aggregation before starting to collect documents.
+
+`collect`::
+
+    This represents the cumulative time spent in the collect phase of the aggregation. This is where matching documents are passed to the aggregation and the state of the aggregator is updated based on the information contained in the documents.
+
+`build_aggregation`::
+
+    This represents the time spent creating the shard level results of the aggregation ready to pass back to the reducing node after the collection of documents is finished.
+
+`reduce`::
+
+    This is not currently used and will always report `0`. Currently aggregation profiling only times the shard level parts of the aggregation execution. Timing of the reduce phase will be added later.
+
+=== Performance Notes
+
+Like any profiler, the Profile API introduces a non-negligible overhead to search execution.  The act of instrumenting
+low-level method calls such as `collect`, `advance` and `next_doc` can be fairly expensive, since these methods are called
+in tight loops.  Therefore, profiling should not be enabled in production settings by default, and should not
+be compared against non-profiled query times.  Profiling is just a diagnostic tool.
+
+There are also cases where special Lucene optimizations are disabled, since they are not amenable to profiling.  This
+could cause some queries to report larger relative times than their non-profiled counterparts, but in general should
+not have a drastic effect compared to other components in the profiled query.
+
+=== Limitations
+
+- Profiling statistics are currently not available for suggestions, highlighting, `dfs_query_then_fetch`
+- Profiling of the reduce phase of aggregation is currently not available
+- The Profiler is still highly experimental. The Profiler is instrumenting parts of Lucene that were
+never designed to be exposed in this manner, and so all results should be viewed as a best effort to provide detailed
+diagnostics.  We hope to improve this over time. If you find obviously wrong numbers, strange query structures or
+other bugs, please report them!

From ecf6101798b053a910321ab90e4c45d0c5e87532 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Tue, 28 Jun 2016 13:20:00 -0700
Subject: [PATCH 1103/1311] Scripts: Remove ClusterState from compile api

Stored scripts are pulled from the cluster state, and the current api
requires passing the ClusterState on each call to compile. However, this
means every user of the ScriptService needs to depend on the
ClusterService. Instead, this change makes the ScriptService a
ClusterStateListener. It also simplifies tests a lot, as they no longer
need to create fake cluster states (except when testing stored scripts).
---
 .../action/update/UpdateHelper.java           |  3 +-
 .../index/query/InnerHitBuilder.java          |  2 +-
 .../index/query/ScriptQueryBuilder.java       |  6 +--
 .../index/query/TemplateQueryBuilder.java     |  2 +-
 .../ScriptScoreFunctionBuilder.java           |  2 +-
 .../ingest/InternalTemplateService.java       |  3 +-
 .../java/org/elasticsearch/node/Node.java     |  1 +
 .../elasticsearch/script/ScriptService.java   | 31 +++++++----
 .../elasticsearch/search/SearchService.java   |  2 +-
 .../heuristics/ScriptHeuristic.java           |  8 +--
 .../scripted/InternalScriptedMetric.java      |  2 +-
 .../scripted/ScriptedMetricAggregator.java    |  6 +--
 .../tophits/TopHitsAggregatorFactory.java     |  2 +-
 .../BucketScriptPipelineAggregator.java       |  2 +-
 .../BucketSelectorPipelineAggregator.java     |  2 +-
 .../ValuesSourceAggregationBuilder.java       |  3 +-
 .../search/sort/ScriptSortBuilder.java        |  2 +-
 .../phrase/PhraseSuggestionBuilder.java       |  2 +-
 .../elasticsearch/script/FileScriptTests.java |  4 +-
 .../script/NativeScriptTests.java             | 21 ++++----
 .../script/ScriptContextTests.java            | 14 ++---
 .../script/ScriptServiceTests.java            | 53 ++++++++-----------
 .../search/sort/AbstractSortTestCase.java     |  2 +-
 .../ingest/common/IngestCommonPlugin.java     |  2 +-
 .../ingest/common/ScriptProcessor.java        | 19 +++----
 .../common/ScriptProcessorFactoryTests.java   |  2 +-
 .../ingest/common/ScriptProcessorTests.java   | 12 ++---
 .../TransportSearchTemplateAction.java        |  6 +--
 .../AbstractAsyncBulkIndexByScrollAction.java |  6 +--
 .../index/reindex/TransportReindexAction.java |  6 +--
 .../reindex/TransportUpdateByQueryAction.java |  6 +--
 31 files changed, 108 insertions(+), 126 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java b/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
index 0c9c1c67978..e2a035d3b5c 100644
--- a/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
+++ b/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
@@ -251,8 +251,7 @@ public class UpdateHelper extends AbstractComponent {
     private Map<String, Object> executeScript(Script script, Map<String, Object> ctx) {
         try {
             if (scriptService != null) {
-                ClusterState state = clusterService.state();
-                ExecutableScript executableScript = scriptService.executable(script, ScriptContext.Standard.UPDATE, Collections.emptyMap(), state);
+                ExecutableScript executableScript = scriptService.executable(script, ScriptContext.Standard.UPDATE, Collections.emptyMap());
                 executableScript.setNextVar("ctx", ctx);
                 executableScript.run();
                 // we need to unwrap the ctx...
diff --git a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
index a017159b451..b54d7713cd3 100644
--- a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
@@ -504,7 +504,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         if (scriptFields != null) {
             for (ScriptField field : scriptFields) {
                 SearchScript searchScript = innerHitsContext.scriptService().search(innerHitsContext.lookup(), field.script(),
-                        ScriptContext.Standard.SEARCH, Collections.emptyMap(), context.getClusterState());
+                        ScriptContext.Standard.SEARCH, Collections.emptyMap());
                 innerHitsContext.scriptFields().add(new org.elasticsearch.search.fetch.script.ScriptFieldsContext.ScriptField(
                         field.fieldName(), searchScript, field.ignoreFailure()));
             }
diff --git a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
index 778501f0132..f0a50985b58 100644
--- a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryBuilder.java
@@ -157,7 +157,7 @@ public class ScriptQueryBuilder extends AbstractQueryBuilder<ScriptQueryBuilder>
 
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
-        return new ScriptQuery(script, context.getScriptService(), context.lookup(), context.getClusterState());
+        return new ScriptQuery(script, context.getScriptService(), context.lookup());
     }
 
     static class ScriptQuery extends Query {
@@ -166,9 +166,9 @@ public class ScriptQueryBuilder extends AbstractQueryBuilder<ScriptQueryBuilder>
 
         private final SearchScript searchScript;
 
-        public ScriptQuery(Script script, ScriptService scriptService, SearchLookup searchLookup, ClusterState state) {
+        public ScriptQuery(Script script, ScriptService scriptService, SearchLookup searchLookup) {
             this.script = script;
-            this.searchScript = scriptService.search(searchLookup, script, ScriptContext.Standard.SEARCH, Collections.emptyMap(), state);
+            this.searchScript = scriptService.search(searchLookup, script, ScriptContext.Standard.SEARCH, Collections.emptyMap());
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java
index 3b41abbbe82..3b47378d1e8 100644
--- a/core/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java
@@ -177,7 +177,7 @@ public class TemplateQueryBuilder extends AbstractQueryBuilder<TemplateQueryBuil
     @Override
     protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException {
         ExecutableScript executable = queryRewriteContext.getScriptService().executable(template,
-            ScriptContext.Standard.SEARCH, Collections.emptyMap(), queryRewriteContext.getClusterState());
+            ScriptContext.Standard.SEARCH, Collections.emptyMap());
         BytesReference querySource = (BytesReference) executable.run();
         try (XContentParser qSourceParser = XContentFactory.xContent(querySource).createParser(querySource)) {
             final QueryParseContext queryParseContext = queryRewriteContext.newParseContext(qSourceParser);
diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java
index 1684e58eb48..a33a9c14fb7 100644
--- a/core/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/ScriptScoreFunctionBuilder.java
@@ -103,7 +103,7 @@ public class ScriptScoreFunctionBuilder extends ScoreFunctionBuilder<ScriptScore
     protected ScoreFunction doToFunction(QueryShardContext context) {
         try {
             SearchScript searchScript = context.getScriptService().search(context.lookup(), script, ScriptContext.Standard.SEARCH,
-                    Collections.emptyMap(), context.getClusterState());
+                    Collections.emptyMap());
             return new ScriptScoreFunction(script, searchScript);
         } catch (Exception e) {
             throw new QueryShardException(context, "script_score: the script could not be loaded", e);
diff --git a/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java b/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java
index 2e1f1415b42..94b4c6fe6bf 100644
--- a/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java
+++ b/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java
@@ -46,8 +46,7 @@ public class InternalTemplateService implements TemplateService {
             CompiledScript compiledScript = scriptService.compile(
                 script,
                 ScriptContext.Standard.INGEST,
-                Collections.emptyMap(),
-                null); // null == OK, because ingest templates are only inline templates.
+                Collections.emptyMap());
             return new Template() {
                 @Override
                 public String execute(Map<String, Object> model) {
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index 675d29e9991..671164e5fff 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -246,6 +246,7 @@ public class Node implements Closeable {
             resourcesToClose.add(resourceWatcherService);
             final NetworkService networkService = new NetworkService(settings);
             final ClusterService clusterService = new ClusterService(settings, settingsModule.getClusterSettings(), threadPool);
+            clusterService.add(scriptModule.getScriptService());
             resourcesToClose.add(clusterService);
             final TribeService tribeService = new TribeService(settings, clusterService);
             resourcesToClose.add(tribeService);
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java
index 57d46255e19..94b4680e4d5 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java
@@ -28,7 +28,9 @@ import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptReque
 import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptResponse;
 import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
+import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParseField;
@@ -75,7 +77,7 @@ import java.util.concurrent.ConcurrentMap;
 
 import static java.util.Collections.unmodifiableMap;
 
-public class ScriptService extends AbstractComponent implements Closeable {
+public class ScriptService extends AbstractComponent implements Closeable, ClusterStateListener {
 
     static final String DISABLE_DYNAMIC_SCRIPTING_SETTING = "script.disable_dynamic";
 
@@ -105,6 +107,8 @@ public class ScriptService extends AbstractComponent implements Closeable {
     private final ParseFieldMatcher parseFieldMatcher;
     private final ScriptMetrics scriptMetrics = new ScriptMetrics();
 
+    private ClusterState clusterState;
+
     /**
      * @deprecated Use {@link org.elasticsearch.script.Script.ScriptField} instead. This should be removed in
      *             2.0
@@ -217,7 +221,7 @@ public class ScriptService extends AbstractComponent implements Closeable {
     /**
      * Checks if a script can be executed and compiles it if needed, or returns the previously compiled and cached script.
      */
-    public CompiledScript compile(Script script, ScriptContext scriptContext, Map<String, String> params, ClusterState state) {
+    public CompiledScript compile(Script script, ScriptContext scriptContext, Map<String, String> params) {
         if (script == null) {
             throw new IllegalArgumentException("The parameter script (Script) must not be null.");
         }
@@ -245,14 +249,14 @@ public class ScriptService extends AbstractComponent implements Closeable {
                     " operation [" + scriptContext.getKey() + "] and lang [" + lang + "] are not supported");
         }
 
-        return compileInternal(script, params, state);
+        return compileInternal(script, params);
     }
 
     /**
      * Compiles a script straight-away, or returns the previously compiled and cached script,
      * without checking if it can be executed based on settings.
      */
-    CompiledScript compileInternal(Script script, Map<String, String> params, ClusterState state) {
+    CompiledScript compileInternal(Script script, Map<String, String> params) {
         if (script == null) {
             throw new IllegalArgumentException("The parameter script (Script) must not be null.");
         }
@@ -289,7 +293,7 @@ public class ScriptService extends AbstractComponent implements Closeable {
             //the script has been updated in the index since the last look up.
             final IndexedScript indexedScript = new IndexedScript(lang, name);
             name = indexedScript.id;
-            code = getScriptFromClusterState(state, indexedScript.lang, indexedScript.id);
+            code = getScriptFromClusterState(indexedScript.lang, indexedScript.id);
         }
 
         CacheKey cacheKey = new CacheKey(scriptEngineService, type == ScriptType.INLINE ? null : name, code, params);
@@ -328,9 +332,9 @@ public class ScriptService extends AbstractComponent implements Closeable {
         return scriptLang;
     }
 
-    String getScriptFromClusterState(ClusterState state, String scriptLang, String id) {
+    String getScriptFromClusterState(String scriptLang, String id) {
         scriptLang = validateScriptLanguage(scriptLang);
-        ScriptMetaData scriptMetadata = state.metaData().custom(ScriptMetaData.TYPE);
+        ScriptMetaData scriptMetadata = clusterState.metaData().custom(ScriptMetaData.TYPE);
         if (scriptMetadata == null) {
             throw new ResourceNotFoundException("Unable to find script [" + scriptLang + "/" + id + "] in cluster state");
         }
@@ -443,8 +447,8 @@ public class ScriptService extends AbstractComponent implements Closeable {
     /**
      * Compiles (or retrieves from cache) and executes the provided script
      */
-    public ExecutableScript executable(Script script, ScriptContext scriptContext, Map<String, String> params, ClusterState state) {
-        return executable(compile(script, scriptContext, params, state), script.getParams());
+    public ExecutableScript executable(Script script, ScriptContext scriptContext, Map<String, String> params) {
+        return executable(compile(script, scriptContext, params), script.getParams());
     }
 
     /**
@@ -457,8 +461,8 @@ public class ScriptService extends AbstractComponent implements Closeable {
     /**
      * Compiles (or retrieves from cache) and executes the provided search script
      */
-    public SearchScript search(SearchLookup lookup, Script script, ScriptContext scriptContext, Map<String, String> params, ClusterState state) {
-        CompiledScript compiledScript = compile(script, scriptContext, params, state);
+    public SearchScript search(SearchLookup lookup, Script script, ScriptContext scriptContext, Map<String, String> params) {
+        CompiledScript compiledScript = compile(script, scriptContext, params);
         return getScriptEngineServiceForLang(compiledScript.lang()).search(compiledScript, lookup, script.getParams());
     }
 
@@ -496,6 +500,11 @@ public class ScriptService extends AbstractComponent implements Closeable {
         }
     }
 
+    @Override
+    public void clusterChanged(ClusterChangedEvent event) {
+        clusterState = event.state();
+    }
+
     /**
      * A small listener for the script cache that calls each
      * {@code ScriptEngineService}'s {@code scriptRemoved} method when the
diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java
index 96fee4b968f..3c17a506eb9 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchService.java
@@ -740,7 +740,7 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
         if (source.scriptFields() != null) {
             for (org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField field : source.scriptFields()) {
                 SearchScript searchScript = context.scriptService().search(context.lookup(), field.script(), ScriptContext.Standard.SEARCH,
-                        Collections.emptyMap(), context.getQueryShardContext().getClusterState());
+                        Collections.emptyMap());
                 context.scriptFields().add(new ScriptField(field.fieldName(), searchScript, field.ignoreFailure()));
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java
index 22fce0113e9..19673bf6e9d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/ScriptHeuristic.java
@@ -79,16 +79,16 @@ public class ScriptHeuristic extends SignificanceHeuristic {
 
     @Override
     public void initialize(InternalAggregation.ReduceContext context) {
-        initialize(context.scriptService(), context.clusterState());
+        initialize(context.scriptService());
     }
 
     @Override
     public void initialize(SearchContext context) {
-        initialize(context.scriptService(), context.getQueryShardContext().getClusterState());
+        initialize(context.scriptService());
     }
 
-    public void initialize(ScriptService scriptService, ClusterState state) {
-        searchScript = scriptService.executable(script, ScriptContext.Standard.AGGS, Collections.emptyMap(), state);
+    public void initialize(ScriptService scriptService) {
+        searchScript = scriptService.executable(script, ScriptContext.Standard.AGGS, Collections.emptyMap());
         searchScript.setNextVar("_subset_freq", subsetDfHolder);
         searchScript.setNextVar("_subset_size", subsetSizeHolder);
         searchScript.setNextVar("_superset_freq", supersetDfHolder);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java
index d6814871108..2c09d58afdb 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java
@@ -92,7 +92,7 @@ public class InternalScriptedMetric extends InternalMetricsAggregation implement
                 vars.putAll(firstAggregation.reduceScript.getParams());
             }
             CompiledScript compiledScript = reduceContext.scriptService().compile(firstAggregation.reduceScript,
-                    ScriptContext.Standard.AGGS, Collections.emptyMap(), reduceContext.clusterState());
+                    ScriptContext.Standard.AGGS, Collections.emptyMap());
             ExecutableScript script = reduceContext.scriptService().executable(compiledScript, vars);
             aggregation = script.run();
         } else {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java
index 053b0e82d69..44708eba409 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java
@@ -54,11 +54,11 @@ public class ScriptedMetricAggregator extends MetricsAggregator {
         ScriptService scriptService = context.searchContext().scriptService();
         ClusterState state = context.searchContext().getQueryShardContext().getClusterState();
         if (initScript != null) {
-            scriptService.executable(initScript, ScriptContext.Standard.AGGS, Collections.emptyMap(), state).run();
+            scriptService.executable(initScript, ScriptContext.Standard.AGGS, Collections.emptyMap()).run();
         }
-        this.mapScript = scriptService.search(context.searchContext().lookup(), mapScript, ScriptContext.Standard.AGGS, Collections.emptyMap(), state);
+        this.mapScript = scriptService.search(context.searchContext().lookup(), mapScript, ScriptContext.Standard.AGGS, Collections.emptyMap());
         if (combineScript != null) {
-            this.combineScript = scriptService.executable(combineScript, ScriptContext.Standard.AGGS, Collections.emptyMap(), state);
+            this.combineScript = scriptService.executable(combineScript, ScriptContext.Standard.AGGS, Collections.emptyMap());
         } else {
             this.combineScript = null;
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java
index ac001222301..65a8c24eb08 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregatorFactory.java
@@ -106,7 +106,7 @@ public class TopHitsAggregatorFactory extends AggregatorFactory<TopHitsAggregato
         if (scriptFields != null) {
             for (ScriptField field : scriptFields) {
                 SearchScript searchScript = subSearchContext.scriptService().search(subSearchContext.lookup(), field.script(),
-                        ScriptContext.Standard.SEARCH, Collections.emptyMap(), subSearchContext.getQueryShardContext().getClusterState());
+                        ScriptContext.Standard.SEARCH, Collections.emptyMap());
                 subSearchContext.scriptFields().add(new org.elasticsearch.search.fetch.script.ScriptFieldsContext.ScriptField(
                         field.fieldName(), searchScript, field.ignoreFailure()));
             }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java
index ea90bcd2f13..719f2c2ffe7 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java
@@ -93,7 +93,7 @@ public class BucketScriptPipelineAggregator extends PipelineAggregator {
         List<? extends Bucket> buckets = originalAgg.getBuckets();
 
         CompiledScript compiledScript = reduceContext.scriptService().compile(script, ScriptContext.Standard.AGGS,
-                Collections.emptyMap(), reduceContext.clusterState());
+                Collections.emptyMap());
         List newBuckets = new ArrayList<>();
         for (Bucket bucket : buckets) {
             Map<String, Object> vars = new HashMap<>();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java
index ed61b881d56..28fded6f559 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java
@@ -90,7 +90,7 @@ public class BucketSelectorPipelineAggregator extends PipelineAggregator {
         List<? extends Bucket> buckets = originalAgg.getBuckets();
 
         CompiledScript compiledScript = reduceContext.scriptService().compile(script, ScriptContext.Standard.AGGS,
-                Collections.emptyMap(), reduceContext.clusterState());
+                Collections.emptyMap());
         List newBuckets = new ArrayList<>();
         for (Bucket bucket : buckets) {
             Map<String, Object> vars = new HashMap<>();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
index cd03ceb9eca..a55b9035c03 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
@@ -377,8 +377,7 @@ public abstract class ValuesSourceAggregationBuilder<VS extends ValuesSource, AB
 
     private SearchScript createScript(Script script, SearchContext context) {
         return script == null ? null
-                : context.scriptService().search(context.lookup(), script, ScriptContext.Standard.AGGS, Collections.emptyMap(),
-                context.getQueryShardContext().getClusterState());
+                : context.scriptService().search(context.lookup(), script, ScriptContext.Standard.AGGS, Collections.emptyMap());
     }
 
     private static DocValueFormat resolveFormat(@Nullable String format, @Nullable ValueType valueType) {
diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
index f7baee509de..8354787df42 100644
--- a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java
@@ -304,7 +304,7 @@ public class ScriptSortBuilder extends SortBuilder<ScriptSortBuilder> {
     @Override
     public SortFieldAndFormat build(QueryShardContext context) throws IOException {
         final SearchScript searchScript = context.getScriptService().search(
-                context.lookup(), script, ScriptContext.Standard.SEARCH, Collections.emptyMap(), context.getClusterState());
+                context.lookup(), script, ScriptContext.Standard.SEARCH, Collections.emptyMap());
 
         MultiValueMode valueMode = null;
         if (sortMode != null) {
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
index 4ff74fb08df..d3ddbeb2f1a 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/PhraseSuggestionBuilder.java
@@ -631,7 +631,7 @@ public class PhraseSuggestionBuilder extends SuggestionBuilder<PhraseSuggestionB
 
         if (this.collateQuery != null) {
             CompiledScript compiledScript = context.getScriptService().compile(this.collateQuery, ScriptContext.Standard.SEARCH,
-                    Collections.emptyMap(), context.getClusterState());
+                    Collections.emptyMap());
             suggestionContext.setCollateQueryScript(compiledScript);
             if (this.collateParams != null) {
                 suggestionContext.setCollateScriptParams(this.collateParams);
diff --git a/core/src/test/java/org/elasticsearch/script/FileScriptTests.java b/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
index 1872ce8f050..9fbd4a81f19 100644
--- a/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
@@ -55,7 +55,7 @@ public class FileScriptTests extends ESTestCase {
             .put("script.engine." + MockScriptEngine.NAME + ".file.aggs", "false").build();
         ScriptService scriptService = makeScriptService(settings);
         Script script = new Script("script1", ScriptService.ScriptType.FILE, MockScriptEngine.NAME, null);
-        CompiledScript compiledScript = scriptService.compile(script, ScriptContext.Standard.SEARCH, Collections.emptyMap(), null);
+        CompiledScript compiledScript = scriptService.compile(script, ScriptContext.Standard.SEARCH, Collections.emptyMap());
         assertNotNull(compiledScript);
         MockCompiledScript executable = (MockCompiledScript) compiledScript.compiled();
         assertEquals("script1.mockscript", executable.name);
@@ -72,7 +72,7 @@ public class FileScriptTests extends ESTestCase {
         Script script = new Script("script1", ScriptService.ScriptType.FILE, MockScriptEngine.NAME, null);
         for (ScriptContext context : ScriptContext.Standard.values()) {
             try {
-                scriptService.compile(script, context, Collections.emptyMap(), null);
+                scriptService.compile(script, context, Collections.emptyMap());
                 fail(context.getKey() + " script should have been rejected");
             } catch(Exception e) {
                 assertTrue(e.getMessage(), e.getMessage().contains("scripts of type [file], operation [" + context.getKey() + "] and lang [" + MockScriptEngine.NAME + "] are disabled"));
diff --git a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
index 70cc3588602..1ee4c58455b 100644
--- a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java
@@ -19,8 +19,13 @@
 
 package org.elasticsearch.script;
 
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterState;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
@@ -30,13 +35,6 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.watcher.ResourceWatcherService;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
 import static java.util.Collections.singletonMap;
@@ -55,10 +53,9 @@ public class NativeScriptTests extends ESTestCase {
         List<Setting<?>> scriptSettings = scriptModule.getSettings();
         scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
 
-        ClusterState state = ClusterState.builder(new ClusterName("_name")).build();
         ExecutableScript executable = scriptModule.getScriptService().executable(
                 new Script("my", ScriptType.INLINE, NativeScriptEngineService.NAME, null), ScriptContext.Standard.SEARCH,
-                Collections.emptyMap(), state);
+                Collections.emptyMap());
         assertThat(executable.run().toString(), equalTo("test"));
     }
 
@@ -85,7 +82,7 @@ public class NativeScriptTests extends ESTestCase {
 
         for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) {
             assertThat(scriptService.compile(new Script("my", ScriptType.INLINE, NativeScriptEngineService.NAME, null), scriptContext,
-                    Collections.emptyMap(), null), notNullValue());
+                    Collections.emptyMap()), notNullValue());
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
index 57bdfff3f7f..c7ee421e7e0 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptContextTests.java
@@ -59,7 +59,7 @@ public class ScriptContextTests extends ESTestCase {
         for (ScriptService.ScriptType scriptType : ScriptService.ScriptType.values()) {
             try {
                 Script script = new Script("1", scriptType, MockScriptEngine.NAME, null);
-                scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_globally_disabled_op"), Collections.emptyMap(), null);
+                scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_globally_disabled_op"), Collections.emptyMap());
                 fail("script compilation should have been rejected");
             } catch (IllegalStateException e) {
                 assertThat(e.getMessage(), containsString("scripts of type [" + scriptType + "], operation [" + PLUGIN_NAME + "_custom_globally_disabled_op] and lang [" + MockScriptEngine.NAME + "] are disabled"));
@@ -71,16 +71,16 @@ public class ScriptContextTests extends ESTestCase {
         ScriptService scriptService = makeScriptService();
         Script script = new Script("1", ScriptService.ScriptType.INLINE, MockScriptEngine.NAME, null);
         try {
-            scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_exp_disabled_op"), Collections.emptyMap(), null);
+            scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_exp_disabled_op"), Collections.emptyMap());
             fail("script compilation should have been rejected");
         } catch (IllegalStateException e) {
             assertTrue(e.getMessage(), e.getMessage().contains("scripts of type [inline], operation [" + PLUGIN_NAME + "_custom_exp_disabled_op] and lang [" + MockScriptEngine.NAME + "] are disabled"));
         }
 
         // still works for other script contexts
-        assertNotNull(scriptService.compile(script, ScriptContext.Standard.AGGS, Collections.emptyMap(), null));
-        assertNotNull(scriptService.compile(script, ScriptContext.Standard.SEARCH, Collections.emptyMap(), null));
-        assertNotNull(scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_op"), Collections.emptyMap(), null));
+        assertNotNull(scriptService.compile(script, ScriptContext.Standard.AGGS, Collections.emptyMap()));
+        assertNotNull(scriptService.compile(script, ScriptContext.Standard.SEARCH, Collections.emptyMap()));
+        assertNotNull(scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "custom_op"), Collections.emptyMap()));
     }
 
     public void testUnknownPluginScriptContext() throws Exception {
@@ -88,7 +88,7 @@ public class ScriptContextTests extends ESTestCase {
         for (ScriptService.ScriptType scriptType : ScriptService.ScriptType.values()) {
             try {
                 Script script = new Script("1", scriptType, MockScriptEngine.NAME, null);
-                scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "unknown"), Collections.emptyMap(), null);
+                scriptService.compile(script, new ScriptContext.Plugin(PLUGIN_NAME, "unknown"), Collections.emptyMap());
                 fail("script compilation should have been rejected");
             } catch (IllegalArgumentException e) {
                 assertTrue(e.getMessage(), e.getMessage().contains("script context [" + PLUGIN_NAME + "_unknown] not supported"));
@@ -107,7 +107,7 @@ public class ScriptContextTests extends ESTestCase {
         for (ScriptService.ScriptType scriptType : ScriptService.ScriptType.values()) {
             try {
                 Script script = new Script("1", scriptType, MockScriptEngine.NAME, null);
-                scriptService.compile(script, context, Collections.emptyMap(), null);
+                scriptService.compile(script, context, Collections.emptyMap());
                 fail("script compilation should have been rejected");
             } catch (IllegalArgumentException e) {
                 assertTrue(e.getMessage(), e.getMessage().contains("script context [test] not supported"));
diff --git a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
index 597cb7e6032..43224488480 100644
--- a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java
@@ -114,9 +114,10 @@ public class ScriptServiceTests extends ESTestCase {
     private void buildScriptService(Settings additionalSettings) throws IOException {
         Settings finalSettings = Settings.builder().put(baseSettings).put(additionalSettings).build();
         Environment environment = new Environment(finalSettings);
+        // TODO:
         scriptService = new ScriptService(finalSettings, environment, resourceWatcherService, scriptEngineRegistry, scriptContextRegistry, scriptSettings) {
             @Override
-            String getScriptFromClusterState(ClusterState state, String scriptLang, String id) {
+            String getScriptFromClusterState(String scriptLang, String id) {
                 //mock the script that gets retrieved from an index
                 return "100";
             }
@@ -141,7 +142,7 @@ public class ScriptServiceTests extends ESTestCase {
         resourceWatcherService.notifyNow();
 
         CompiledScript compiledScript = scriptService.compile(new Script("test_script", ScriptType.FILE, "test", null),
-                ScriptContext.Standard.SEARCH, Collections.emptyMap(), emptyClusterState());
+                ScriptContext.Standard.SEARCH, Collections.emptyMap());
         assertThat(compiledScript.compiled(), equalTo((Object) "compiled_test_file"));
 
         Files.delete(testFileNoExt);
@@ -150,7 +151,7 @@ public class ScriptServiceTests extends ESTestCase {
 
         try {
             scriptService.compile(new Script("test_script", ScriptType.FILE, "test", null), ScriptContext.Standard.SEARCH,
-                    Collections.emptyMap(), emptyClusterState());
+                    Collections.emptyMap());
             fail("the script test_script should no longer exist");
         } catch (IllegalArgumentException ex) {
             assertThat(ex.getMessage(), containsString("Unable to find on disk file script [test_script] using lang [test]"));
@@ -168,7 +169,7 @@ public class ScriptServiceTests extends ESTestCase {
         resourceWatcherService.notifyNow();
 
         CompiledScript compiledScript = scriptService.compile(new Script("file_script", ScriptType.FILE, "test", null),
-                ScriptContext.Standard.SEARCH, Collections.emptyMap(), emptyClusterState());
+                ScriptContext.Standard.SEARCH, Collections.emptyMap());
         assertThat(compiledScript.compiled(), equalTo((Object) "compiled_test_file_script"));
 
         Files.delete(testHiddenFile);
@@ -179,9 +180,9 @@ public class ScriptServiceTests extends ESTestCase {
     public void testInlineScriptCompiledOnceCache() throws IOException {
         buildScriptService(Settings.EMPTY);
         CompiledScript compiledScript1 = scriptService.compile(new Script("1+1", ScriptType.INLINE, "test", null),
-                randomFrom(scriptContexts), Collections.emptyMap(), emptyClusterState());
+                randomFrom(scriptContexts), Collections.emptyMap());
         CompiledScript compiledScript2 = scriptService.compile(new Script("1+1", ScriptType.INLINE, "test", null),
-                randomFrom(scriptContexts), Collections.emptyMap(), emptyClusterState());
+                randomFrom(scriptContexts), Collections.emptyMap());
         assertThat(compiledScript1.compiled(), sameInstance(compiledScript2.compiled()));
     }
 
@@ -304,7 +305,7 @@ public class ScriptServiceTests extends ESTestCase {
         String type = scriptEngineService.getType();
         try {
             scriptService.compile(new Script("test", randomFrom(ScriptType.values()), type, null), new ScriptContext.Plugin(
-                            pluginName, unknownContext), Collections.emptyMap(), emptyClusterState());
+                            pluginName, unknownContext), Collections.emptyMap());
             fail("script compilation should have been rejected");
         } catch(IllegalArgumentException e) {
             assertThat(e.getMessage(), containsString("script context [" + pluginName + "_" + unknownContext + "] not supported"));
@@ -314,22 +315,20 @@ public class ScriptServiceTests extends ESTestCase {
     public void testCompileCountedInCompilationStats() throws IOException {
         buildScriptService(Settings.EMPTY);
         scriptService.compile(new Script("1+1", ScriptType.INLINE, "test", null), randomFrom(scriptContexts),
-                Collections.emptyMap(), emptyClusterState());
+                Collections.emptyMap());
         assertEquals(1L, scriptService.stats().getCompilations());
     }
 
     public void testExecutableCountedInCompilationStats() throws IOException {
         buildScriptService(Settings.EMPTY);
-        ClusterState state = ClusterState.builder(new ClusterName("_name")).build();
-        scriptService.executable(new Script("1+1", ScriptType.INLINE, "test", null), randomFrom(scriptContexts), Collections.emptyMap(), state);
+        scriptService.executable(new Script("1+1", ScriptType.INLINE, "test", null), randomFrom(scriptContexts), Collections.emptyMap());
         assertEquals(1L, scriptService.stats().getCompilations());
     }
 
     public void testSearchCountedInCompilationStats() throws IOException {
         buildScriptService(Settings.EMPTY);
-        ClusterState state = ClusterState.builder(new ClusterName("_name")).build();
         scriptService.search(null, new Script("1+1", ScriptType.INLINE, "test", null), randomFrom(scriptContexts),
-                Collections.emptyMap(), state);
+                Collections.emptyMap());
         assertEquals(1L, scriptService.stats().getCompilations());
     }
 
@@ -339,7 +338,7 @@ public class ScriptServiceTests extends ESTestCase {
         for (int i = 0; i < numberOfCompilations; i++) {
             scriptService
                     .compile(new Script(i + " + " + i, ScriptType.INLINE, "test", null), randomFrom(scriptContexts),
-                            Collections.emptyMap(), emptyClusterState());
+                            Collections.emptyMap());
         }
         assertEquals(numberOfCompilations, scriptService.stats().getCompilations());
     }
@@ -349,9 +348,8 @@ public class ScriptServiceTests extends ESTestCase {
         builder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getKey(), 1);
         builder.put("script.inline", "true");
         buildScriptService(builder.build());
-        ClusterState state = ClusterState.builder(new ClusterName("_name")).build();
-        scriptService.executable(new Script("1+1", ScriptType.INLINE, "test", null), randomFrom(scriptContexts), Collections.emptyMap(), state);
-        scriptService.executable(new Script("1+1", ScriptType.INLINE, "test", null), randomFrom(scriptContexts), Collections.emptyMap(), state);
+        scriptService.executable(new Script("1+1", ScriptType.INLINE, "test", null), randomFrom(scriptContexts), Collections.emptyMap());
+        scriptService.executable(new Script("1+1", ScriptType.INLINE, "test", null), randomFrom(scriptContexts), Collections.emptyMap());
         assertEquals(1L, scriptService.stats().getCompilations());
     }
 
@@ -359,14 +357,14 @@ public class ScriptServiceTests extends ESTestCase {
         buildScriptService(Settings.EMPTY);
         createFileScripts("test");
         scriptService.compile(new Script("file_script", ScriptType.FILE, "test", null), randomFrom(scriptContexts),
-                Collections.emptyMap(), emptyClusterState());
+                Collections.emptyMap());
         assertEquals(1L, scriptService.stats().getCompilations());
     }
 
     public void testIndexedScriptCountedInCompilationStats() throws IOException {
         buildScriptService(Settings.EMPTY);
         scriptService.compile(new Script("script", ScriptType.STORED, "test", null), randomFrom(scriptContexts),
-                Collections.emptyMap(), emptyClusterState());
+                Collections.emptyMap());
         assertEquals(1L, scriptService.stats().getCompilations());
     }
 
@@ -375,9 +373,8 @@ public class ScriptServiceTests extends ESTestCase {
         builder.put(ScriptService.SCRIPT_CACHE_SIZE_SETTING.getKey(), 1);
         builder.put("script.inline", "true");
         buildScriptService(builder.build());
-        ClusterState state = ClusterState.builder(new ClusterName("_name")).build();
-        scriptService.executable(new Script("1+1", ScriptType.INLINE, "test", null), randomFrom(scriptContexts), Collections.emptyMap(), state);
-        scriptService.executable(new Script("2+2", ScriptType.INLINE, "test", null), randomFrom(scriptContexts), Collections.emptyMap(), state);
+        scriptService.executable(new Script("1+1", ScriptType.INLINE, "test", null), randomFrom(scriptContexts), Collections.emptyMap());
+        scriptService.executable(new Script("2+2", ScriptType.INLINE, "test", null), randomFrom(scriptContexts), Collections.emptyMap());
         assertEquals(2L, scriptService.stats().getCompilations());
         assertEquals(1L, scriptService.stats().getCacheEvictions());
     }
@@ -388,7 +385,7 @@ public class ScriptServiceTests extends ESTestCase {
         builder.put("script.inline", "true");
         buildScriptService(builder.build());
         CompiledScript script = scriptService.compile(new Script("1 + 1", ScriptType.INLINE, null, null),
-                randomFrom(scriptContexts), Collections.emptyMap(), emptyClusterState());
+                randomFrom(scriptContexts), Collections.emptyMap());
         assertEquals(script.lang(), "test");
     }
 
@@ -469,7 +466,7 @@ public class ScriptServiceTests extends ESTestCase {
 
     private void assertCompileRejected(String lang, String script, ScriptType scriptType, ScriptContext scriptContext) {
         try {
-            scriptService.compile(new Script(script, scriptType, lang, null), scriptContext, Collections.emptyMap(), emptyClusterState());
+            scriptService.compile(new Script(script, scriptType, lang, null), scriptContext, Collections.emptyMap());
             fail("compile should have been rejected for lang [" + lang + "], script_type [" + scriptType + "], scripted_op [" + scriptContext + "]");
         } catch(IllegalStateException e) {
             //all good
@@ -477,9 +474,8 @@ public class ScriptServiceTests extends ESTestCase {
     }
 
     private void assertCompileAccepted(String lang, String script, ScriptType scriptType, ScriptContext scriptContext) {
-        ClusterState state = emptyClusterState();
         assertThat(
-                scriptService.compile(new Script(script, scriptType, lang, null), scriptContext, Collections.emptyMap(), state),
+                scriptService.compile(new Script(script, scriptType, lang, null), scriptContext, Collections.emptyMap()),
                 notNullValue()
         );
     }
@@ -528,8 +524,6 @@ public class ScriptServiceTests extends ESTestCase {
 
         public static final String NAME = "dtest";
 
-        public static final List<String> EXTENSIONS = Collections.unmodifiableList(Arrays.asList("dtest"));
-
         @Override
         public String getType() {
             return NAME;
@@ -559,9 +553,4 @@ public class ScriptServiceTests extends ESTestCase {
         public void close() {
         }
     }
-
-    private static ClusterState emptyClusterState() {
-        return ClusterState.builder(new ClusterName("_name")).build();
-    }
-
 }
diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
index 2ac7fd68e2e..bdd5c76534c 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
@@ -100,7 +100,7 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST
         scriptService = new ScriptService(baseSettings, environment,
                 new ResourceWatcherService(baseSettings, null), scriptEngineRegistry, scriptContextRegistry, scriptSettings) {
             @Override
-            public CompiledScript compile(Script script, ScriptContext scriptContext, Map<String, String> params, ClusterState state) {
+            public CompiledScript compile(Script script, ScriptContext scriptContext, Map<String, String> params) {
                 return new CompiledScript(ScriptType.INLINE, "mockName", "test", script);
             }
         };
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java
index 821e47b85bc..60bfdd37a9f 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java
@@ -60,7 +60,7 @@ public class IngestCommonPlugin extends Plugin {
         nodeModule.registerProcessor(SortProcessor.TYPE, (registry) -> new SortProcessor.Factory());
         nodeModule.registerProcessor(GrokProcessor.TYPE, (registry) -> new GrokProcessor.Factory(builtinPatterns));
         nodeModule.registerProcessor(ScriptProcessor.TYPE, (registry) ->
-            new ScriptProcessor.Factory(registry.getScriptService(), registry.getClusterService()));
+            new ScriptProcessor.Factory(registry.getScriptService()));
     }
 
     // Code for loading built-in grok patterns packaged with the jar file:
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
index 6ce85f8a6bf..e4881366165 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
@@ -19,7 +19,9 @@
 
 package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.cluster.service.ClusterService;
+import java.util.HashMap;
+import java.util.Map;
+
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
@@ -30,9 +32,6 @@ import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
 
-import java.util.HashMap;
-import java.util.Map;
-
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.common.Strings.hasLength;
 import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
@@ -52,14 +51,12 @@ public final class ScriptProcessor extends AbstractProcessor {
 
     private final Script script;
     private final ScriptService scriptService;
-    private final ClusterService clusterService;
     private final String field;
 
-    ScriptProcessor(String tag, Script script, ScriptService scriptService, ClusterService clusterService, String field)  {
+    ScriptProcessor(String tag, Script script, ScriptService scriptService, String field)  {
         super(tag);
         this.script = script;
         this.scriptService = scriptService;
-        this.clusterService = clusterService;
         this.field = field;
     }
 
@@ -67,7 +64,7 @@ public final class ScriptProcessor extends AbstractProcessor {
     public void execute(IngestDocument document) {
         Map<String, Object> vars = new HashMap<>();
         vars.put("ctx", document.getSourceAndMetadata());
-        CompiledScript compiledScript = scriptService.compile(script, ScriptContext.Standard.INGEST, emptyMap(), clusterService.state());
+        CompiledScript compiledScript = scriptService.compile(script, ScriptContext.Standard.INGEST, emptyMap());
         ExecutableScript executableScript = scriptService.executable(compiledScript, vars);
         Object value = executableScript.run();
         if (field != null) {
@@ -83,11 +80,9 @@ public final class ScriptProcessor extends AbstractProcessor {
     public static final class Factory extends AbstractProcessorFactory<ScriptProcessor> {
 
         private final ScriptService scriptService;
-        private final ClusterService clusterService;
 
-        public Factory(ScriptService scriptService, ClusterService clusterService) {
+        public Factory(ScriptService scriptService) {
             this.scriptService = scriptService;
-            this.clusterService = clusterService;
         }
 
         @Override
@@ -120,7 +115,7 @@ public final class ScriptProcessor extends AbstractProcessor {
                 throw newConfigurationException(TYPE, processorTag, null, "Could not initialize script");
             }
 
-            return new ScriptProcessor(processorTag, script, scriptService, clusterService, field);
+            return new ScriptProcessor(processorTag, script, scriptService, field);
         }
     }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
index 59a5d7ceaa2..ed47894d4d9 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
@@ -37,7 +37,7 @@ public class ScriptProcessorFactoryTests extends ESTestCase {
 
     @Before
     public void init() {
-        factory = new ScriptProcessor.Factory(mock(ScriptService.class), mock(ClusterService.class));
+        factory = new ScriptProcessor.Factory(mock(ScriptService.class));
     }
 
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java
index 6da127a798b..5fe7db77dd8 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorTests.java
@@ -19,7 +19,9 @@
 
 package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.cluster.service.ClusterService;
+import java.util.HashMap;
+import java.util.Map;
+
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.script.CompiledScript;
@@ -28,9 +30,6 @@ import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
 
-import java.util.HashMap;
-import java.util.Map;
-
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.core.Is.is;
 import static org.mockito.Mockito.any;
@@ -42,16 +41,15 @@ public class ScriptProcessorTests extends ESTestCase {
     public void testScripting() throws Exception {
         int randomInt = randomInt();
         ScriptService scriptService = mock(ScriptService.class);
-        ClusterService clusterService = mock(ClusterService.class);
         CompiledScript compiledScript = mock(CompiledScript.class);
         Script script = mock(Script.class);
-        when(scriptService.compile(any(), any(), any(), any())).thenReturn(compiledScript);
+        when(scriptService.compile(any(), any(), any())).thenReturn(compiledScript);
         ExecutableScript executableScript = mock(ExecutableScript.class);
         when(scriptService.executable(any(), any())).thenReturn(executableScript);
         when(executableScript.run()).thenReturn(randomInt);
 
         ScriptProcessor processor = new ScriptProcessor(randomAsciiOfLength(10), script,
-            scriptService, clusterService, "bytes_total");
+            scriptService, "bytes_total");
 
         Map<String, Object> document = new HashMap<>();
         document.put("bytes_in", 1234);
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportSearchTemplateAction.java
index 4c956bb209a..9e1dab3af00 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportSearchTemplateAction.java
@@ -53,7 +53,6 @@ public class TransportSearchTemplateAction extends HandledTransportAction<Search
 
     private static final String TEMPLATE_LANG = MustacheScriptEngineService.NAME;
 
-    private final ClusterService clusterService;
     private final ScriptService scriptService;
     private final TransportSearchAction searchAction;
     private final IndicesQueriesRegistry queryRegistry;
@@ -63,11 +62,10 @@ public class TransportSearchTemplateAction extends HandledTransportAction<Search
     @Inject
     public TransportSearchTemplateAction(Settings settings, ThreadPool threadPool, TransportService transportService,
                                          ActionFilters actionFilters, IndexNameExpressionResolver resolver,
-                                         ClusterService clusterService, ScriptService scriptService,
+                                         ScriptService scriptService,
                                          TransportSearchAction searchAction, IndicesQueriesRegistry indicesQueryRegistry,
                                          AggregatorParsers aggregatorParsers, Suggesters suggesters) {
         super(settings, SearchTemplateAction.NAME, threadPool, transportService, actionFilters, resolver, SearchTemplateRequest::new);
-        this.clusterService = clusterService;
         this.scriptService = scriptService;
         this.searchAction = searchAction;
         this.queryRegistry = indicesQueryRegistry;
@@ -80,7 +78,7 @@ public class TransportSearchTemplateAction extends HandledTransportAction<Search
         final SearchTemplateResponse response = new SearchTemplateResponse();
         try {
             Script script = new Script(request.getScript(), request.getScriptType(), TEMPLATE_LANG, request.getScriptParams());
-            ExecutableScript executable = scriptService.executable(script, SEARCH, emptyMap(), clusterService.state());
+            ExecutableScript executable = scriptService.executable(script, SEARCH, emptyMap());
 
             BytesReference source = (BytesReference) executable.run();
             response.setSource(source);
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java
index 6cb62207506..60ab088d76c 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java
@@ -439,19 +439,17 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
 
         private final BulkByScrollTask task;
         private final ScriptService scriptService;
-        private final ClusterState state;
         private final Script script;
         private final Map<String, Object> params;
 
         private ExecutableScript executable;
         private Map<String, Object> context;
 
-        public ScriptApplier(BulkByScrollTask task, ScriptService scriptService, Script script, ClusterState state,
+        public ScriptApplier(BulkByScrollTask task, ScriptService scriptService, Script script,
                              Map<String, Object> params) {
             this.task = task;
             this.scriptService = scriptService;
             this.script = script;
-            this.state = state;
             this.params = params;
         }
 
@@ -462,7 +460,7 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
                 return request;
             }
             if (executable == null) {
-                CompiledScript compiled = scriptService.compile(script, ScriptContext.Standard.UPDATE, emptyMap(), state);
+                CompiledScript compiled = scriptService.compile(script, ScriptContext.Standard.UPDATE, emptyMap());
                 executable = scriptService.executable(compiled, params);
             }
             if (context == null) {
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
index a49ba0a3b32..e98c45595c7 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
@@ -128,7 +128,7 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
         protected BiFunction<RequestWrapper<?>, SearchHit, RequestWrapper<?>> buildScriptApplier() {
             Script script = mainRequest.getScript();
             if (script != null) {
-                return new ReindexScriptApplier(task, scriptService, script, clusterState, script.getParams());
+                return new ReindexScriptApplier(task, scriptService, script, script.getParams());
             }
             return super.buildScriptApplier();
         }
@@ -205,9 +205,9 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
 
         class ReindexScriptApplier extends ScriptApplier {
 
-            ReindexScriptApplier(BulkByScrollTask task, ScriptService scriptService, Script script, ClusterState state,
+            ReindexScriptApplier(BulkByScrollTask task, ScriptService scriptService, Script script,
                                  Map<String, Object> params) {
-                super(task, scriptService, script, state, params);
+                super(task, scriptService, script, params);
             }
 
             /*
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
index 7459972ce64..e30f0d2b140 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
@@ -92,7 +92,7 @@ public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateB
         protected BiFunction<RequestWrapper<?>, SearchHit, RequestWrapper<?>> buildScriptApplier() {
             Script script = mainRequest.getScript();
             if (script != null) {
-                return new UpdateByQueryScriptApplier(task, scriptService, script, clusterState, script.getParams());
+                return new UpdateByQueryScriptApplier(task, scriptService, script, script.getParams());
             }
             return super.buildScriptApplier();
         }
@@ -112,9 +112,9 @@ public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateB
 
         class UpdateByQueryScriptApplier extends ScriptApplier {
 
-            UpdateByQueryScriptApplier(BulkByScrollTask task, ScriptService scriptService, Script script, ClusterState state,
+            UpdateByQueryScriptApplier(BulkByScrollTask task, ScriptService scriptService, Script script,
                                  Map<String, Object> params) {
-                super(task, scriptService, script, state, params);
+                super(task, scriptService, script, params);
             }
 
             @Override

From 67bfecc070392a222c53627438f290a3b3c78a3e Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 24 Jun 2016 13:26:46 -0400
Subject: [PATCH 1104/1311] Painless: add "".replaceAll and "".replaceFirst

These are useful methods in groovy that give you control over
the replacements used:
```
'the quick brown fox'.replaceAll(/[aeiou]/,
		m -> m.group().toUpperCase(Locale.ROOT))
```
---
 .../modules/scripting/painless.asciidoc       | 81 +++++++++++++++----
 .../elasticsearch/painless/Augmentation.java  | 44 ++++++++++
 .../org/elasticsearch/painless/java.lang.txt  |  2 +
 .../elasticsearch/painless/RegexTests.java    | 56 +++++++++++++
 4 files changed, 169 insertions(+), 14 deletions(-)

diff --git a/docs/reference/modules/scripting/painless.asciidoc b/docs/reference/modules/scripting/painless.asciidoc
index db68db2c7a3..93fb136913b 100644
--- a/docs/reference/modules/scripting/painless.asciidoc
+++ b/docs/reference/modules/scripting/painless.asciidoc
@@ -237,8 +237,8 @@ POST hockey/player/_update_by_query
 ----------------------------------------------------------------
 // CONSOLE
 
-Or you can use the `Pattern.matcher` directory to get a `Matcher` instance and
-remove all of the vowels in all of their names:
+You can use the `Pattern.matcher` directly to get a `Matcher` instance and
+remove all of the vowels in all of their last names:
 
 [source,js]
 ----------------------------------------------------------------
@@ -252,6 +252,59 @@ POST hockey/player/_update_by_query
 ----------------------------------------------------------------
 // CONSOLE
 
+`Matcher.replaceAll` is just a call to Java's `Matcher`'s
+http://docs.oracle.com/javase/8/docs/api/java/util/regex/Matcher.html#replaceAll-java.lang.String-[replaceAll]
+method so it supports `$1` and `\1` for replacements:
+
+[source,js]
+----------------------------------------------------------------
+POST hockey/player/_update_by_query
+{
+  "script": {
+    "lang": "painless",
+    "inline": "ctx._source.last = /n([aeiou])/.matcher(ctx._source.last).replaceAll('$1')"
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+
+If you need more control over replacements you can call `replaceAll` on a
+`CharSequence` with a `Function<Matcher, String>` that builds the replacement.
+This does not support `$1` or `\1` to access replacements because you already
+have a reference to the matcher and can get them with `m.group(1)`.
+
+IMPORTANT: Calling `Matcher.find` inside of the function that builds the
+replacement is rude and will likely break the replacement process.
+
+This will make all of the vowels in the hockey player's last names upper case:
+
+[source,js]
+----------------------------------------------------------------
+POST hockey/player/_update_by_query
+{
+  "script": {
+    "lang": "painless",
+    "inline": "ctx._source.last = ctx._source.last.replaceAll(/[aeiou]/, m -> m.group().toUpperCase(Locale.ROOT))"
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+
+Or you can use the `CharSequence.replaceFirst` to make the first vowel in their
+last names upper case:
+
+[source,js]
+----------------------------------------------------------------
+POST hockey/player/_update_by_query
+{
+  "script": {
+    "lang": "painless",
+    "inline": "ctx._source.last = ctx._source.last.replaceFirst(/[aeiou]/, m -> m.group().toUpperCase(Locale.ROOT))"
+  }
+}
+----------------------------------------------------------------
+// CONSOLE
+
 
 Note: all of the `_update_by_query` examples above could really do with a
 `query` to limit the data that they pull back. While you *could* use a
@@ -265,18 +318,18 @@ documents that they have to check.
 
 The following Java packages are available for use in the Painless language:
 
-* https://docs.oracle.com/javase/8/docs/api/java/lang/package-summary.html[java.lang] 
-* https://docs.oracle.com/javase/8/docs/api/java/math/package-summary.html[java.math] 
-* https://docs.oracle.com/javase/8/docs/api/java/text/package-summary.html[java.text] 
-* https://docs.oracle.com/javase/8/docs/api/java/time/package-summary.html[java.time] 
-* https://docs.oracle.com/javase/8/docs/api/java/time/chrono/package-summary.html[java.time.chrono] 
-* https://docs.oracle.com/javase/8/docs/api/java/time/format/package-summary.html[java.time.format] 
-* https://docs.oracle.com/javase/8/docs/api/java/time/temporal/package-summary.html[java.time.temporal] 
-* https://docs.oracle.com/javase/8/docs/api/java/time/zone/package-summary.html[java.time.zone] 
-* https://docs.oracle.com/javase/8/docs/api/java/util/package-summary.html[java.util] 
-* https://docs.oracle.com/javase/8/docs/api/java/util/function/package-summary.html[java.util.function] 
-* https://docs.oracle.com/javase/8/docs/api/java/util/regex/package-summary.html[java.util.regex] 
-* https://docs.oracle.com/javase/8/docs/api/java/util/stream/package-summary.html[java.util.stream] 
+* https://docs.oracle.com/javase/8/docs/api/java/lang/package-summary.html[java.lang]
+* https://docs.oracle.com/javase/8/docs/api/java/math/package-summary.html[java.math]
+* https://docs.oracle.com/javase/8/docs/api/java/text/package-summary.html[java.text]
+* https://docs.oracle.com/javase/8/docs/api/java/time/package-summary.html[java.time]
+* https://docs.oracle.com/javase/8/docs/api/java/time/chrono/package-summary.html[java.time.chrono]
+* https://docs.oracle.com/javase/8/docs/api/java/time/format/package-summary.html[java.time.format]
+* https://docs.oracle.com/javase/8/docs/api/java/time/temporal/package-summary.html[java.time.temporal]
+* https://docs.oracle.com/javase/8/docs/api/java/time/zone/package-summary.html[java.time.zone]
+* https://docs.oracle.com/javase/8/docs/api/java/util/package-summary.html[java.util]
+* https://docs.oracle.com/javase/8/docs/api/java/util/function/package-summary.html[java.util.function]
+* https://docs.oracle.com/javase/8/docs/api/java/util/regex/package-summary.html[java.util.regex]
+* https://docs.oracle.com/javase/8/docs/api/java/util/stream/package-summary.html[java.util.stream]
 
 Note that unsafe classes and methods are not included, there is no support for:
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Augmentation.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Augmentation.java
index edd0eeab67f..9302f3c899c 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Augmentation.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Augmentation.java
@@ -34,6 +34,7 @@ import java.util.function.ObjIntConsumer;
 import java.util.function.Predicate;
 import java.util.function.ToDoubleFunction;
 import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /** Additional methods added to classes. These must be static methods with receiver as first argument */
 public class Augmentation {
@@ -442,4 +443,47 @@ public class Augmentation {
         }
         return map;
     }
+
+    // CharSequence augmentation
+    /**
+     * Replace all matches. Similar to {@link Matcher#replaceAll(String)} but allows you to customize the replacement based on the match.
+     */
+    public static String replaceAll(CharSequence receiver, Pattern pattern, Function<Matcher, String> replacementBuilder) {
+        Matcher m = pattern.matcher(receiver);
+        if (false == m.find()) {
+            // CharSequqence's toString is *supposed* to always return the characters in the sequence as a String
+            return receiver.toString();
+        }
+        StringBuffer result = new StringBuffer(initialBufferForReplaceWith(receiver));
+        do {
+            m.appendReplacement(result, Matcher.quoteReplacement(replacementBuilder.apply(m)));
+        } while (m.find());
+        m.appendTail(result);
+        return result.toString();
+    }
+
+    /**
+     * Replace the first match. Similar to {@link Matcher#replaceFirst(String)} but allows you to customize the replacement based on the
+     * match.
+     */
+    public static String replaceFirst(CharSequence receiver, Pattern pattern, Function<Matcher, String> replacementBuilder) {
+        Matcher m = pattern.matcher(receiver);
+        if (false == m.find()) {
+            // CharSequqence's toString is *supposed* to always return the characters in the sequence as a String
+            return receiver.toString();
+        }
+        StringBuffer result = new StringBuffer(initialBufferForReplaceWith(receiver));
+        m.appendReplacement(result, Matcher.quoteReplacement(replacementBuilder.apply(m)));
+        m.appendTail(result);
+        return result.toString();
+    }
+
+    /**
+     * The initial size of the {@link StringBuilder} used for {@link #replaceFirst(CharSequence, Pattern, Function)} and
+     * {@link #replaceAll(CharSequence, Pattern, Function)} for a particular sequence. We ape
+     * {{@link StringBuilder#StringBuilder(CharSequence)} here and add 16 extra chars to the buffer to have a little room for growth.
+     */
+    private static int initialBufferForReplaceWith(CharSequence seq) {
+        return seq.length() + 16;
+    }
 }
diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
index 17016f1d318..13f28d3ebeb 100644
--- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
+++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.lang.txt
@@ -36,6 +36,8 @@ class CharSequence -> java.lang.CharSequence {
   IntStream chars()
   IntStream codePoints()
   int length()
+  String replaceAll*(Pattern,Function)
+  String replaceFirst*(Pattern,Function)
   CharSequence subSequence(int,int)
   String toString()
 }
diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java
index e255a776bed..615dec67dc4 100644
--- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java
+++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/RegexTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.painless;
 
+import java.nio.CharBuffer;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.regex.Pattern;
@@ -175,6 +176,61 @@ public class RegexTests extends ScriptTestCase {
         assertEquals(Pattern.CANON_EQ | Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE | Pattern.COMMENTS, exec("/./ciux.flags()"));
     }
 
+    public void testReplaceAllMatchesString() {
+        assertEquals("thE qUIck brOwn fOx", exec("'the quick brown fox'.replaceAll(/[aeiou]/, m -> m.group().toUpperCase(Locale.ROOT))"));
+    }
+
+    public void testReplaceAllMatchesCharSequence() {
+        CharSequence charSequence = CharBuffer.wrap("the quick brown fox");
+        assertEquals("thE qUIck brOwn fOx",
+                exec("params.a.replaceAll(/[aeiou]/, m -> m.group().toUpperCase(Locale.ROOT))", singletonMap("a", charSequence)));
+    }
+
+    public void testReplaceAllNoMatchString() {
+        assertEquals("i am cat", exec("'i am cat'.replaceAll(/dolphin/, m -> m.group().toUpperCase(Locale.ROOT))"));
+    }
+
+    public void testReplaceAllNoMatchCharSequence() {
+        CharSequence charSequence = CharBuffer.wrap("i am cat");
+        assertEquals("i am cat",
+                exec("params.a.replaceAll(/dolphin/, m -> m.group().toUpperCase(Locale.ROOT))", singletonMap("a", charSequence)));
+    }
+
+    public void testReplaceAllQuoteReplacement() {
+        assertEquals("th/E q/U/Ick br/Own f/Ox",
+                exec("'the quick brown fox'.replaceAll(/[aeiou]/, m -> '/' + m.group().toUpperCase(Locale.ROOT))"));
+        assertEquals("th$E q$U$Ick br$Own f$Ox",
+                exec("'the quick brown fox'.replaceAll(/[aeiou]/, m -> '$' + m.group().toUpperCase(Locale.ROOT))"));
+    }
+
+    public void testReplaceFirstMatchesString() {
+        assertEquals("thE quick brown fox",
+                exec("'the quick brown fox'.replaceFirst(/[aeiou]/, m -> m.group().toUpperCase(Locale.ROOT))"));
+    }
+
+    public void testReplaceFirstMatchesCharSequence() {
+        CharSequence charSequence = CharBuffer.wrap("the quick brown fox");
+        assertEquals("thE quick brown fox",
+                exec("params.a.replaceFirst(/[aeiou]/, m -> m.group().toUpperCase(Locale.ROOT))", singletonMap("a", charSequence)));
+    }
+
+    public void testReplaceFirstNoMatchString() {
+        assertEquals("i am cat", exec("'i am cat'.replaceFirst(/dolphin/, m -> m.group().toUpperCase(Locale.ROOT))"));
+    }
+
+    public void testReplaceFirstNoMatchCharSequence() {
+        CharSequence charSequence = CharBuffer.wrap("i am cat");
+        assertEquals("i am cat",
+                exec("params.a.replaceFirst(/dolphin/, m -> m.group().toUpperCase(Locale.ROOT))", singletonMap("a", charSequence)));
+    }
+
+    public void testReplaceFirstQuoteReplacement() {
+        assertEquals("th/E quick brown fox",
+                exec("'the quick brown fox'.replaceFirst(/[aeiou]/, m -> '/' + m.group().toUpperCase(Locale.ROOT))"));
+        assertEquals("th$E quick brown fox",
+                exec("'the quick brown fox'.replaceFirst(/[aeiou]/, m -> '$' + m.group().toUpperCase(Locale.ROOT))"));
+    }
+
     public void testCantUsePatternCompile() {
         IllegalArgumentException e = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("Pattern.compile('aa')");

From 6590e77c1a2c07c2627471fbbaff81381bb21d0e Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Tue, 28 Jun 2016 16:08:22 -0700
Subject: [PATCH 1105/1311] Plugins: Make plugins closeable

This change allows Plugin implementions to implement Closeable when they
have resources that should be released. As a first example of how this
can be used, I switched over ingest plugins, which just had the geoip
processor. The ingest framework had chains of closeable to support this,
which is now removed.
---
 .../elasticsearch/ingest/IngestService.java   |  8 +------
 .../elasticsearch/ingest/PipelineStore.java   |  9 +-------
 .../ingest/ProcessorsRegistry.java            | 22 ++++---------------
 .../java/org/elasticsearch/node/Node.java     |  1 +
 .../node/service/NodeService.java             |  1 -
 .../ingest/geoip/GeoIpProcessor.java          |  7 +-----
 .../ingest/geoip/IngestGeoIpPlugin.java       | 18 +++++++++++++--
 7 files changed, 24 insertions(+), 42 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/ingest/IngestService.java b/core/src/main/java/org/elasticsearch/ingest/IngestService.java
index 228de5b37bc..ce7d6dadee5 100644
--- a/core/src/main/java/org/elasticsearch/ingest/IngestService.java
+++ b/core/src/main/java/org/elasticsearch/ingest/IngestService.java
@@ -33,7 +33,7 @@ import java.util.Map;
 /**
  * Holder class for several ingest related services.
  */
-public class IngestService implements Closeable {
+public class IngestService {
 
     private final PipelineStore pipelineStore;
     private final PipelineExecutionService pipelineExecutionService;
@@ -65,10 +65,4 @@ public class IngestService implements Closeable {
         }
         return new IngestInfo(processorInfoList);
     }
-
-    @Override
-    public void close() throws IOException {
-        pipelineStore.close();
-    }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
index f96a7179656..3118d2b5076 100644
--- a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
+++ b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
@@ -47,7 +47,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-public class PipelineStore extends AbstractComponent implements Closeable, ClusterStateListener {
+public class PipelineStore extends AbstractComponent implements ClusterStateListener {
 
     private final Pipeline.Factory factory = new Pipeline.Factory();
     private ProcessorsRegistry processorRegistry;
@@ -67,13 +67,6 @@ public class PipelineStore extends AbstractComponent implements Closeable, Clust
         this.processorRegistry = processorsRegistryBuilder.build(scriptService, clusterService);
     }
 
-    @Override
-    public void close() throws IOException {
-        // TODO: When org.elasticsearch.node.Node can close Closable instances we should try to remove this code,
-        // since any wired closable should be able to close itself
-        processorRegistry.close();
-    }
-
     @Override
     public void clusterChanged(ClusterChangedEvent event) {
         innerUpdatePipelines(event.state());
diff --git a/core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java b/core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java
index 28abfc45313..dd16898aea2 100644
--- a/core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java
+++ b/core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java
@@ -19,20 +19,17 @@
 
 package org.elasticsearch.ingest;
 
-import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.script.ScriptService;
-
 import java.io.Closeable;
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
 
-public final class ProcessorsRegistry implements Closeable {
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.script.ScriptService;
+
+public final class ProcessorsRegistry {
 
     private final Map<String, Processor.Factory> processorFactories;
     private final TemplateService templateService;
@@ -67,17 +64,6 @@ public final class ProcessorsRegistry implements Closeable {
         return processorFactories.get(name);
     }
 
-    @Override
-    public void close() throws IOException {
-        List<Closeable> closeables = new ArrayList<>();
-        for (Processor.Factory factory : processorFactories.values()) {
-            if (factory instanceof Closeable) {
-                closeables.add((Closeable) factory);
-            }
-        }
-        IOUtils.close(closeables);
-    }
-
     // For testing:
     Map<String, Processor.Factory> getProcessorFactories() {
         return processorFactories;
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index 675d29e9991..211ff9ffe65 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -552,6 +552,7 @@ public class Node implements Closeable {
             toClose.add(() -> stopWatch.stop().start("plugin(" + plugin.getName() + ")"));
             toClose.add(injector.getInstance(plugin));
         }
+        toClose.addAll(pluginsService.filterPlugins(Closeable.class));
 
         toClose.add(() -> stopWatch.stop().start("script"));
         toClose.add(injector.getInstance(ScriptService.class));
diff --git a/core/src/main/java/org/elasticsearch/node/service/NodeService.java b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
index ed03dc9375c..0920a84bfd6 100644
--- a/core/src/main/java/org/elasticsearch/node/service/NodeService.java
+++ b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
@@ -199,7 +199,6 @@ public class NodeService extends AbstractComponent implements Closeable {
 
     @Override
     public void close() throws IOException {
-        ingestService.close();
         indicesService.close();
     }
 }
diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
index c29bf4aa65d..0b18cae25e7 100644
--- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
+++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
@@ -217,7 +217,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
         return geoData;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<GeoIpProcessor> implements Closeable {
+    public static final class Factory extends AbstractProcessorFactory<GeoIpProcessor> {
         static final Set<Property> DEFAULT_CITY_PROPERTIES = EnumSet.of(
             Property.CONTINENT_NAME, Property.COUNTRY_ISO_CODE, Property.REGION_NAME,
             Property.CITY_NAME, Property.LOCATION
@@ -267,11 +267,6 @@ public final class GeoIpProcessor extends AbstractProcessor {
 
             return new GeoIpProcessor(processorTag, ipField, databaseReader, targetField, properties);
         }
-
-        @Override
-        public void close() throws IOException {
-            IOUtils.close(databaseReaders.values());
-        }
     }
 
     // Geoip2's AddressNotFoundException is checked and due to the fact that we need run their code
diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
index 8339ff1fc60..d814ae46bea 100644
--- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
+++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
@@ -20,9 +20,11 @@
 package org.elasticsearch.ingest.geoip;
 
 import com.maxmind.geoip2.DatabaseReader;
+import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.node.NodeModule;
 import org.elasticsearch.plugins.Plugin;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
@@ -36,11 +38,16 @@ import java.util.Map;
 import java.util.stream.Stream;
 import java.util.zip.GZIPInputStream;
 
-public class IngestGeoIpPlugin extends Plugin {
+public class IngestGeoIpPlugin extends Plugin implements Closeable {
+
+    private Map<String, DatabaseReader> databaseReaders;
 
     public void onModule(NodeModule nodeModule) throws IOException {
+        if (databaseReaders != null) {
+            throw new IllegalStateException("called onModule twice for geoip plugin!!");
+        }
         Path geoIpConfigDirectory = nodeModule.getNode().getEnvironment().configFile().resolve("ingest-geoip");
-        Map<String, DatabaseReader> databaseReaders = loadDatabaseReaders(geoIpConfigDirectory);
+        databaseReaders = loadDatabaseReaders(geoIpConfigDirectory);
         nodeModule.registerProcessor(GeoIpProcessor.TYPE, (registry) -> new GeoIpProcessor.Factory(databaseReaders));
     }
 
@@ -65,4 +72,11 @@ public class IngestGeoIpPlugin extends Plugin {
         }
         return Collections.unmodifiableMap(databaseReaders);
     }
+
+    @Override
+    public void close() throws IOException {
+        if (databaseReaders != null) {
+            IOUtils.close(databaseReaders.values());
+        }
+    }
 }

From b97ea9954c1ff975e848cdfcafeb49cc73a5be26 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Tue, 28 Jun 2016 11:21:08 +0200
Subject: [PATCH 1106/1311] percolator: Use RamDirectory for percolating nested
 document instead of using multiple MemoryIndex instances with
 SlowCompositeReaderWrapper workaround

---
 .../percolator/PercolateQueryBuilder.java     | 29 +++++++------------
 1 file changed, 11 insertions(+), 18 deletions(-)

diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
index 8ce12c1f7e6..3ce3726aa5e 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
@@ -22,11 +22,11 @@ package org.elasticsearch.percolator;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;
 import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.MultiReader;
-import org.apache.lucene.index.SlowCompositeReaderWrapper;
 import org.apache.lucene.index.StoredFieldVisitor;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.memory.MemoryIndex;
@@ -36,6 +36,7 @@ import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.Weight;
+import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchException;
@@ -62,7 +63,6 @@ import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperForType;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
 import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
@@ -74,7 +74,6 @@ import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.QueryShardException;
 
 import java.io.IOException;
-import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -454,19 +453,13 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
     }
 
     static IndexSearcher createMultiDocumentSearcher(Analyzer analyzer, ParsedDocument doc) {
-        IndexReader[] memoryIndices = new IndexReader[doc.docs().size()];
-        List<ParseContext.Document> docs = doc.docs();
-        int rootDocIndex = docs.size() - 1;
-        assert rootDocIndex > 0;
-        for (int i = 0; i < docs.size(); i++) {
-            ParseContext.Document d = docs.get(i);
-            MemoryIndex memoryIndex = MemoryIndex.fromDocument(d, analyzer, true, false);
-            memoryIndices[i] = memoryIndex.createSearcher().getIndexReader();
-        }
-        try {
-            MultiReader mReader = new MultiReader(memoryIndices, true);
-            LeafReader slowReader = SlowCompositeReaderWrapper.wrap(mReader);
-            final IndexSearcher slowSearcher = new IndexSearcher(slowReader) {
+        RAMDirectory ramDirectory = new RAMDirectory();
+        try (IndexWriter indexWriter = new IndexWriter(ramDirectory, new IndexWriterConfig(analyzer))) {
+            indexWriter.addDocuments(doc.docs());
+            indexWriter.commit();
+            DirectoryReader directoryReader = DirectoryReader.open(ramDirectory);
+            assert directoryReader.leaves().size() == 1 : "Expected single leaf, but got [" + directoryReader.leaves().size() + "]";
+            final IndexSearcher slowSearcher = new IndexSearcher(directoryReader) {
 
                 @Override
                 public Weight createNormalizedWeight(Query query, boolean needsScores) throws IOException {

From 4820d491209fd848ddb9bb307b8b1a7c90c676e7 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Tue, 14 Jun 2016 15:14:54 +0200
Subject: [PATCH 1107/1311] Mustache: Add util functions to render JSON and
 join array values

This pull request adds two util functions to the Mustache templating engine:
- {{#toJson}}my_map{{/toJson}} to render a Map parameter as a JSON string
- {{#join}}my_iterable{{/join}} to render any iterable (including arrays) as a comma separated list of values like `1, 2, 3`. It's also possible de change the default delimiter (comma) to something else.

closes #18970
---
 .../reference/search/search-template.asciidoc | 123 ++++++++
 .../mustache/CustomMustacheFactory.java       | 279 ++++++++++++++++++
 .../mustache/JsonEscapingMustacheFactory.java |  41 ---
 .../mustache/MustacheScriptEngineService.java |  33 +--
 .../mustache/NoneEscapingMustacheFactory.java |  40 ---
 .../mustache/MustacheScriptEngineTests.java   |  11 +-
 .../script/mustache/MustacheTests.java        | 224 +++++++++++++-
 7 files changed, 635 insertions(+), 116 deletions(-)
 create mode 100644 modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java
 delete mode 100644 modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/JsonEscapingMustacheFactory.java
 delete mode 100644 modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/NoneEscapingMustacheFactory.java

diff --git a/docs/reference/search/search-template.asciidoc b/docs/reference/search/search-template.asciidoc
index 8533984428b..359b692f528 100644
--- a/docs/reference/search/search-template.asciidoc
+++ b/docs/reference/search/search-template.asciidoc
@@ -89,6 +89,89 @@ which is rendered as:
 }
 ------------------------------------------
 
+
+[float]
+===== Concatenating array of values
+
+The `{{#join}}array{{/join}}` function can be used to concatenate the
+values of an array as a comma delimited string:
+
+[source,js]
+------------------------------------------
+GET /_search/template
+{
+  "inline": {
+    "query": {
+      "match": {
+        "emails": "{{#join}}emails{{/join}}"
+      }
+    }
+  },
+  "params": {
+    "emails": [ "username@email.com", "lastname@email.com" ]
+  }
+}
+------------------------------------------
+
+which is rendered as:
+
+[source,js]
+------------------------------------------
+{
+    "query" : {
+        "match" : {
+            "emails" : "username@email.com,lastname@email.com"
+        }
+    }
+}
+------------------------------------------
+
+The function also accepts a custom delimiter:
+
+[source,js]
+------------------------------------------
+GET /_search/template
+{
+  "inline": {
+    "query": {
+      "range": {
+        "born": {
+            "gte"   : "{{date.min}}",
+            "lte"   : "{{date.max}}",
+            "format": "{{#join delimiter='||'}}date.formats{{/join delimiter='||'}}"
+	    }
+      }
+    }
+  },
+  "params": {
+    "date": {
+        "min": "2016",
+        "max": "31/12/2017",
+        "formats": ["dd/MM/yyyy", "yyyy"]
+    }
+  }
+}
+------------------------------------------
+
+which is rendered as:
+
+[source,js]
+------------------------------------------
+{
+    "query" : {
+      "range" : {
+        "born" : {
+          "gte" : "2016",
+          "lte" : "31/12/2017",
+          "format" : "dd/MM/yyyy||yyyy"
+        }
+      }
+    }
+}
+
+------------------------------------------
+
+
 [float]
 ===== Default values
 
@@ -140,6 +223,46 @@ for `end`:
 }
 ------------------------------------------
 
+[float]
+===== Converting parameters to JSON
+
+The `{{toJson}}parameter{{/toJson}}` function can be used to convert parameters
+like maps and array to their JSON representation:
+
+[source,js]
+------------------------------------------
+{
+    "inline": "{\"query\":{\"bool\":{\"must\": {{#toJson}}clauses{{/toJson}} }}}",
+    "params": {
+        "clauses": [
+            { "term": "foo" },
+            { "term": "bar" }
+        ]
+   }
+}
+------------------------------------------
+
+which is rendered as:
+
+[source,js]
+------------------------------------------
+{
+    "query" : {
+      "bool" : {
+        "must" : [
+          {
+            "term" : "foo"
+          },
+          {
+            "term" : "bar"
+          }
+        ]
+      }
+    }
+}
+------------------------------------------
+
+
 [float]
 ===== Conditional clauses
 
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java
new file mode 100644
index 00000000000..ceb187bcc63
--- /dev/null
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script.mustache;
+
+import com.fasterxml.jackson.core.io.JsonStringEncoder;
+import com.github.mustachejava.Code;
+import com.github.mustachejava.DefaultMustacheFactory;
+import com.github.mustachejava.DefaultMustacheVisitor;
+import com.github.mustachejava.Mustache;
+import com.github.mustachejava.MustacheException;
+import com.github.mustachejava.MustacheVisitor;
+import com.github.mustachejava.TemplateContext;
+import com.github.mustachejava.codes.IterableCode;
+import com.github.mustachejava.codes.WriteCode;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.StringJoiner;
+import java.util.function.BiConsumer;
+import java.util.function.Function;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class CustomMustacheFactory extends DefaultMustacheFactory {
+
+    private final BiConsumer<String, Writer> encoder;
+
+    public CustomMustacheFactory(boolean escaping) {
+        super();
+        setObjectHandler(new CustomReflectionObjectHandler());
+        if (escaping) {
+            this.encoder = new JsonEscapeEncoder();
+        } else {
+            this.encoder = new NoEscapeEncoder();
+        }
+    }
+
+    @Override
+    public void encode(String value, Writer writer) {
+        encoder.accept(value, writer);
+    }
+
+    @Override
+    public MustacheVisitor createMustacheVisitor() {
+        return new CustomMustacheVisitor(this);
+    }
+
+    class CustomMustacheVisitor extends DefaultMustacheVisitor {
+
+        public CustomMustacheVisitor(DefaultMustacheFactory df) {
+            super(df);
+        }
+
+        @Override
+        public void iterable(TemplateContext templateContext, String variable, Mustache mustache) {
+            if (ToJsonCode.match(variable)) {
+                list.add(new ToJsonCode(templateContext, df, mustache, variable));
+            } else if (JoinerCode.match(variable)) {
+                list.add(new JoinerCode(templateContext, df, mustache));
+            } else if (CustomJoinerCode.match(variable)) {
+                list.add(new CustomJoinerCode(templateContext, df, mustache, variable));
+            } else {
+                list.add(new IterableCode(templateContext, df, mustache, variable));
+            }
+        }
+    }
+
+    /**
+     * Base class for custom Mustache functions
+     */
+    static abstract class CustomCode extends IterableCode {
+
+        private final String code;
+
+        public CustomCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String code) {
+            super(tc, df, mustache, extractVariableName(code, mustache, tc));
+            this.code = Objects.requireNonNull(code);
+        }
+
+        @Override
+        public Writer execute(Writer writer, final List<Object> scopes) {
+            Object resolved = get(scopes);
+            writer = handle(writer, createFunction(resolved), scopes);
+            appendText(writer);
+            return writer;
+        }
+
+        @Override
+        protected void tag(Writer writer, String tag) throws IOException {
+            writer.write(tc.startChars());
+            writer.write(tag);
+            writer.write(code);
+            writer.write(tc.endChars());
+        }
+
+        protected abstract Function<String, String> createFunction(Object resolved);
+
+        /**
+         * At compile time, this function extracts the name of the variable:
+         * {{#toJson}}variable_name{{/toJson}}
+         */
+        protected static String extractVariableName(String fn, Mustache mustache, TemplateContext tc) {
+            Code[] codes = mustache.getCodes();
+            if (codes == null || codes.length != 1) {
+                throw new MustacheException("Mustache function [" + fn + "] must contain one and only one identifier");
+            }
+
+            try (StringWriter capture = new StringWriter()) {
+                // Variable name is in plain text and has type WriteCode
+                if (codes[0] instanceof WriteCode) {
+                    codes[0].execute(capture, Collections.emptyList());
+                    return capture.toString();
+                } else {
+                    codes[0].identity(capture);
+                    return capture.toString();
+                }
+            } catch (IOException e) {
+                throw new MustacheException("Exception while parsing mustache function [" + fn + "] at line " + tc.line(), e);
+            }
+        }
+    }
+
+    /**
+     * This function renders {@link Iterable} and {@link Map} as their JSON representation
+     */
+    static class ToJsonCode extends CustomCode {
+
+        private static final String CODE = "toJson";
+
+        public ToJsonCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String variable) {
+            super(tc, df, mustache, CODE);
+            if (CODE.equalsIgnoreCase(variable) == false) {
+                throw new MustacheException("Mismatch function code [" + CODE + "] cannot be applied to [" + variable + "]");
+            }
+        }
+
+        @Override
+        @SuppressWarnings("unchecked")
+        protected Function<String, String> createFunction(Object resolved) {
+            return s -> {
+                if (resolved == null) {
+                    return null;
+                }
+                try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
+                    if (resolved == null) {
+                        builder.nullValue();
+                    } else if (resolved instanceof Iterable) {
+                        builder.startArray();
+                        for (Object o : (Iterable) resolved) {
+                            builder.value(o);
+                        }
+                        builder.endArray();
+                    } else if (resolved instanceof Map) {
+                        builder.map((Map<String, ?>) resolved);
+                    } else {
+                        // Do not handle as JSON
+                        return oh.stringify(resolved);
+                    }
+                    return builder.string();
+                } catch (IOException e) {
+                    throw new MustacheException("Failed to convert object to JSON", e);
+                }
+            };
+        }
+
+        static boolean match(String variable) {
+            return CODE.equalsIgnoreCase(variable);
+        }
+    }
+
+    /**
+     * This function concatenates the values of an {@link Iterable} using a given delimiter
+     */
+    static class JoinerCode extends CustomCode {
+
+        protected static final String CODE = "join";
+        private static final String DEFAULT_DELIMITER = ",";
+
+        private final String delimiter;
+
+        public JoinerCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String delimiter) {
+            super(tc, df, mustache, CODE);
+            this.delimiter = delimiter;
+        }
+
+        public JoinerCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache) {
+            this(tc, df, mustache, DEFAULT_DELIMITER);
+        }
+
+        @Override
+        protected Function<String, String> createFunction(Object resolved) {
+            return s -> {
+                if (s == null) {
+                    return null;
+                } else if (resolved instanceof Iterable) {
+                    StringJoiner joiner = new StringJoiner(delimiter);
+                    for (Object o : (Iterable) resolved) {
+                        joiner.add(oh.stringify(o));
+                    }
+                    return joiner.toString();
+                }
+                return s;
+            };
+        }
+
+        static boolean match(String variable) {
+            return CODE.equalsIgnoreCase(variable);
+        }
+    }
+
+    static class CustomJoinerCode extends JoinerCode {
+
+        private static final Pattern PATTERN = Pattern.compile("^(?:" + CODE + " delimiter='(.*)')$");
+
+        public CustomJoinerCode(TemplateContext tc, DefaultMustacheFactory df, Mustache mustache, String variable) {
+            super(tc, df, mustache, extractDelimiter(variable));
+        }
+
+        private static String extractDelimiter(String variable) {
+            Matcher matcher = PATTERN.matcher(variable);
+            if (matcher.find()) {
+                return matcher.group(1);
+            }
+            throw new MustacheException("Failed to extract delimiter for join function");
+        }
+
+        static boolean match(String variable) {
+            return PATTERN.matcher(variable).matches();
+        }
+    }
+
+    class NoEscapeEncoder implements BiConsumer<String, Writer> {
+
+        @Override
+        public void accept(String s, Writer writer) {
+            try {
+                writer.write(s);
+            } catch (IOException e) {
+                throw new MustacheException("Failed to encode value: " + s);
+            }
+        }
+    }
+
+    class JsonEscapeEncoder implements BiConsumer<String, Writer> {
+
+        @Override
+        public void accept(String s, Writer writer) {
+            try {
+                writer.write(JsonStringEncoder.getInstance().quoteAsString(s));
+            } catch (IOException e) {
+                throw new MustacheException("Failed to escape and encode value: " + s);
+            }
+        }
+    }
+}
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/JsonEscapingMustacheFactory.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/JsonEscapingMustacheFactory.java
deleted file mode 100644
index 38d48b98f4e..00000000000
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/JsonEscapingMustacheFactory.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.script.mustache;
-
-import com.fasterxml.jackson.core.io.JsonStringEncoder;
-import com.github.mustachejava.DefaultMustacheFactory;
-import com.github.mustachejava.MustacheException;
-
-import java.io.IOException;
-import java.io.Writer;
-
-/**
- * A MustacheFactory that does simple JSON escaping.
- */
-final class JsonEscapingMustacheFactory extends DefaultMustacheFactory {
-
-    @Override
-    public void encode(String value, Writer writer) {
-        try {
-            writer.write(JsonStringEncoder.getInstance().quoteAsString(value));
-        } catch (IOException e) {
-            throw new MustacheException("Failed to encode value: " + value);
-        }
-    }
-}
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
index 2a48567333b..66ecf23fa02 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngineService.java
@@ -18,8 +18,8 @@
  */
 package org.elasticsearch.script.mustache;
 
-import com.github.mustachejava.DefaultMustacheFactory;
 import com.github.mustachejava.Mustache;
+import com.github.mustachejava.MustacheFactory;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
@@ -29,8 +29,8 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
-import org.elasticsearch.script.ScriptEngineService;
 import org.elasticsearch.script.GeneralScriptException;
+import org.elasticsearch.script.ScriptEngineService;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.lookup.SearchLookup;
 
@@ -89,21 +89,13 @@ public final class MustacheScriptEngineService extends AbstractComponent impleme
      * */
     @Override
     public Object compile(String templateName, String templateSource, Map<String, String> params) {
-        String contentType = params.getOrDefault(CONTENT_TYPE_PARAM, JSON_CONTENT_TYPE);
-        final DefaultMustacheFactory mustacheFactory;
-        switch (contentType){
-            case PLAIN_TEXT_CONTENT_TYPE:
-                mustacheFactory = new NoneEscapingMustacheFactory();
-                break;
-            case JSON_CONTENT_TYPE:
-            default:
-                // assume that the default is json encoding:
-                mustacheFactory = new JsonEscapingMustacheFactory();
-                break;
-        }
-        mustacheFactory.setObjectHandler(new CustomReflectionObjectHandler());
+        final MustacheFactory factory = new CustomMustacheFactory(isJsonEscapingEnabled(params));
         Reader reader = new FastStringReader(templateSource);
-        return mustacheFactory.compile(reader, "query-template");
+        return factory.compile(reader, "query-template");
+    }
+
+    private boolean isJsonEscapingEnabled(Map<String, String> params) {
+        return JSON_CONTENT_TYPE.equals(params.getOrDefault(CONTENT_TYPE_PARAM, JSON_CONTENT_TYPE));
     }
 
     @Override
@@ -168,12 +160,9 @@ public final class MustacheScriptEngineService extends AbstractComponent impleme
                 if (sm != null) {
                     sm.checkPermission(SPECIAL_PERMISSION);
                 }
-                AccessController.doPrivileged(new PrivilegedAction<Void>() {
-                    @Override
-                    public Void run() {
-                        ((Mustache) template.compiled()).execute(writer, vars);
-                        return null;
-                    }
+                AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
+                    ((Mustache) template.compiled()).execute(writer, vars);
+                    return null;
                 });
             } catch (Exception e) {
                 logger.error("Error running {}", e, template);
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/NoneEscapingMustacheFactory.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/NoneEscapingMustacheFactory.java
deleted file mode 100644
index 3539402df98..00000000000
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/NoneEscapingMustacheFactory.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.script.mustache;
-
-import com.github.mustachejava.DefaultMustacheFactory;
-import com.github.mustachejava.MustacheException;
-
-import java.io.IOException;
-import java.io.Writer;
-
-/**
- * A MustacheFactory that does no string escaping.
- */
-final class NoneEscapingMustacheFactory extends DefaultMustacheFactory {
-
-    @Override
-    public void encode(String value, Writer writer) {
-        try {
-            writer.write(value);
-        } catch (IOException e) {
-            throw new MustacheException("Failed to encode value: " + value);
-        }
-    }
-}
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java
index 254020066b5..054268ef681 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.script.mustache;
 
+import com.github.mustachejava.MustacheFactory;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.script.CompiledScript;
@@ -39,12 +40,12 @@ import static org.hamcrest.Matchers.equalTo;
  */
 public class MustacheScriptEngineTests extends ESTestCase {
     private MustacheScriptEngineService qe;
-    private JsonEscapingMustacheFactory escaper;
+    private MustacheFactory factory;
 
     @Before
     public void setup() {
         qe = new MustacheScriptEngineService(Settings.Builder.EMPTY_SETTINGS);
-        escaper = new JsonEscapingMustacheFactory();
+        factory = new CustomMustacheFactory(true);
     }
 
     public void testSimpleParameterReplace() {
@@ -75,12 +76,12 @@ public class MustacheScriptEngineTests extends ESTestCase {
     public void testEscapeJson() throws IOException {
         {
             StringWriter writer = new StringWriter();
-            escaper.encode("hello \n world", writer);
+            factory.encode("hello \n world", writer);
             assertThat(writer.toString(), equalTo("hello \\n world"));
         }
         {
             StringWriter writer = new StringWriter();
-            escaper.encode("\n", writer);
+            factory.encode("\n", writer);
             assertThat(writer.toString(), equalTo("\\n"));
         }
 
@@ -135,7 +136,7 @@ public class MustacheScriptEngineTests extends ESTestCase {
                 expect.append(escapedChars[charIndex]);
             }
             StringWriter target = new StringWriter();
-            escaper.encode(writer.toString(), target);
+            factory.encode(writer.toString(), target);
             assertThat(expect.toString(), equalTo(target.toString()));
         }
     }
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java
index f850f117cb6..8b6d0e69573 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java
@@ -19,13 +19,16 @@
 package org.elasticsearch.script.mustache;
 
 import com.github.mustachejava.Mustache;
+import com.github.mustachejava.MustacheException;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.ScriptEngineService;
-import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matcher;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -38,6 +41,8 @@ import java.util.Set;
 
 import static java.util.Collections.singleton;
 import static java.util.Collections.singletonMap;
+import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.script.ScriptService.ScriptType.INLINE;
 import static org.elasticsearch.script.mustache.MustacheScriptEngineService.CONTENT_TYPE_PARAM;
 import static org.elasticsearch.script.mustache.MustacheScriptEngineService.JSON_CONTENT_TYPE;
 import static org.elasticsearch.script.mustache.MustacheScriptEngineService.PLAIN_TEXT_CONTENT_TYPE;
@@ -45,6 +50,8 @@ import static org.hamcrest.Matchers.both;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.isEmptyOrNullString;
+import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 
 public class MustacheTests extends ESTestCase {
@@ -59,7 +66,7 @@ public class MustacheTests extends ESTestCase {
         Map<String, Object> params = Collections.singletonMap("boost_val", "0.2");
 
         Mustache mustache = (Mustache) engine.compile(null, template, Collections.emptyMap());
-        CompiledScript compiledScript = new CompiledScript(ScriptService.ScriptType.INLINE, "my-name", "mustache", mustache);
+        CompiledScript compiledScript = new CompiledScript(INLINE, "my-name", "mustache", mustache);
         ExecutableScript result = engine.executable(compiledScript, params);
         assertEquals(
                 "Mustache templating broken",
@@ -71,7 +78,7 @@ public class MustacheTests extends ESTestCase {
 
     public void testArrayAccess() throws Exception {
         String template = "{{data.0}} {{data.1}}";
-        CompiledScript mustache = new CompiledScript(ScriptService.ScriptType.INLINE, "inline", "mustache", engine.compile(null, template, Collections.emptyMap()));
+        CompiledScript mustache = new CompiledScript(INLINE, "inline", "mustache", engine.compile(null, template, Collections.emptyMap()));
         Map<String, Object> vars = new HashMap<>();
         Object data = randomFrom(
             new String[] { "foo", "bar" },
@@ -97,7 +104,7 @@ public class MustacheTests extends ESTestCase {
 
     public void testArrayInArrayAccess() throws Exception {
         String template = "{{data.0.0}} {{data.0.1}}";
-        CompiledScript mustache = new CompiledScript(ScriptService.ScriptType.INLINE, "inline", "mustache", engine.compile(null, template, Collections.emptyMap()));
+        CompiledScript mustache = new CompiledScript(INLINE, "inline", "mustache", engine.compile(null, template, Collections.emptyMap()));
         Map<String, Object> vars = new HashMap<>();
         Object data = randomFrom(
             new String[][] { new String[] { "foo", "bar" }},
@@ -114,7 +121,7 @@ public class MustacheTests extends ESTestCase {
 
     public void testMapInArrayAccess() throws Exception {
         String template = "{{data.0.key}} {{data.1.key}}";
-        CompiledScript mustache = new CompiledScript(ScriptService.ScriptType.INLINE, "inline", "mustache", engine.compile(null, template, Collections.emptyMap()));
+        CompiledScript mustache = new CompiledScript(INLINE, "inline", "mustache", engine.compile(null, template, Collections.emptyMap()));
         Map<String, Object> vars = new HashMap<>();
         Object data = randomFrom(
             new Object[] { singletonMap("key", "foo"), singletonMap("key", "bar") },
@@ -142,7 +149,7 @@ public class MustacheTests extends ESTestCase {
         // json string escaping enabled:
         Map<String, String> params = randomBoolean() ? Collections.emptyMap() : Collections.singletonMap(CONTENT_TYPE_PARAM, JSON_CONTENT_TYPE);
         Mustache mustache = (Mustache) engine.compile(null, "{ \"field1\": \"{{value}}\"}", Collections.emptyMap());
-        CompiledScript compiledScript = new CompiledScript(ScriptService.ScriptType.INLINE, "name", "mustache", mustache);
+        CompiledScript compiledScript = new CompiledScript(INLINE, "name", "mustache", mustache);
         ExecutableScript executableScript = engine.executable(compiledScript, Collections.singletonMap("value", "a \"value\""));
         BytesReference rawResult = (BytesReference) executableScript.run();
         String result = rawResult.toUtf8();
@@ -150,7 +157,7 @@ public class MustacheTests extends ESTestCase {
 
         // json string escaping disabled:
         mustache = (Mustache) engine.compile(null, "{ \"field1\": \"{{value}}\"}", Collections.singletonMap(CONTENT_TYPE_PARAM, PLAIN_TEXT_CONTENT_TYPE));
-        compiledScript = new CompiledScript(ScriptService.ScriptType.INLINE, "name", "mustache", mustache);
+        compiledScript = new CompiledScript(INLINE, "name", "mustache", mustache);
         executableScript = engine.executable(compiledScript, Collections.singletonMap("value", "a \"value\""));
         rawResult = (BytesReference) executableScript.run();
         result = rawResult.toUtf8();
@@ -162,7 +169,7 @@ public class MustacheTests extends ESTestCase {
         List<String> randomList = Arrays.asList(generateRandomStringArray(10, 20, false));
 
         String template = "{{data.array.size}} {{data.list.size}}";
-        CompiledScript mustache = new CompiledScript(ScriptService.ScriptType.INLINE, "inline", "mustache", engine.compile(null, template, Collections.emptyMap()));
+        CompiledScript mustache = new CompiledScript(INLINE, "inline", "mustache", engine.compile(null, template, Collections.emptyMap()));
         Map<String, Object> data = new HashMap<>();
         data.put("array", randomArrayValues);
         data.put("list", randomList);
@@ -177,4 +184,205 @@ public class MustacheTests extends ESTestCase {
         String expectedString = String.format(Locale.ROOT, "%s %s", randomArrayValues.length, randomList.size());
         assertThat(bytes.toUtf8(), equalTo(expectedString));
     }
+
+    public void testPrimitiveToJSON() throws Exception {
+        String template = "{{#toJson}}ctx{{/toJson}}";
+        assertScript(template, Collections.singletonMap("ctx", "value"), equalTo("value"));
+        assertScript(template, Collections.singletonMap("ctx", ""), equalTo(""));
+        assertScript(template, Collections.singletonMap("ctx", true), equalTo("true"));
+        assertScript(template, Collections.singletonMap("ctx", 42), equalTo("42"));
+        assertScript(template, Collections.singletonMap("ctx", 42L), equalTo("42"));
+        assertScript(template, Collections.singletonMap("ctx", 42.5f), equalTo("42.5"));
+        assertScript(template, Collections.singletonMap("ctx", null), equalTo(""));
+
+        template = "{{#toJson}}.{{/toJson}}";
+        assertScript(template, Collections.singletonMap("ctx", "value"), equalTo("{\"ctx\":\"value\"}"));
+        assertScript(template, Collections.singletonMap("ctx", ""), equalTo("{\"ctx\":\"\"}"));
+        assertScript(template, Collections.singletonMap("ctx", true), equalTo("{\"ctx\":true}"));
+        assertScript(template, Collections.singletonMap("ctx", 42), equalTo("{\"ctx\":42}"));
+        assertScript(template, Collections.singletonMap("ctx", 42L), equalTo("{\"ctx\":42}"));
+        assertScript(template, Collections.singletonMap("ctx", 42.5f), equalTo("{\"ctx\":42.5}"));
+        assertScript(template, Collections.singletonMap("ctx", null), equalTo("{\"ctx\":null}"));
+    }
+
+    public void testSimpleMapToJSON() throws Exception {
+        Map<String, Object> human0 = new HashMap<>();
+        human0.put("age", 42);
+        human0.put("name", "John Smith");
+        human0.put("height", 1.84);
+
+        Map<String, Object> ctx = Collections.singletonMap("ctx", human0);
+
+        assertScript("{{#toJson}}.{{/toJson}}", ctx, equalTo("{\"ctx\":{\"name\":\"John Smith\",\"age\":42,\"height\":1.84}}"));
+        assertScript("{{#toJson}}ctx{{/toJson}}", ctx, equalTo("{\"name\":\"John Smith\",\"age\":42,\"height\":1.84}"));
+        assertScript("{{#toJson}}ctx.name{{/toJson}}", ctx, equalTo("John Smith"));
+    }
+
+    public void testMultipleMapsToJSON() throws Exception {
+        Map<String, Object> human0 = new HashMap<>();
+        human0.put("age", 42);
+        human0.put("name", "John Smith");
+        human0.put("height", 1.84);
+
+        Map<String, Object> human1 = new HashMap<>();
+        human1.put("age", 27);
+        human1.put("name", "Dave Smith");
+        human1.put("height", 1.71);
+
+        Map<String, Object> humans = new HashMap<>();
+        humans.put("first", human0);
+        humans.put("second", human1);
+
+        Map<String, Object> ctx = Collections.singletonMap("ctx", humans);
+
+        assertScript("{{#toJson}}.{{/toJson}}", ctx,
+                equalTo("{\"ctx\":{\"first\":{\"name\":\"John Smith\",\"age\":42,\"height\":1.84},\"second\":{\"name\":\"Dave Smith\",\"age\":27,\"height\":1.71}}}"));
+
+        assertScript("{{#toJson}}ctx{{/toJson}}", ctx,
+                equalTo("{\"first\":{\"name\":\"John Smith\",\"age\":42,\"height\":1.84},\"second\":{\"name\":\"Dave Smith\",\"age\":27,\"height\":1.71}}"));
+
+        assertScript("{{#toJson}}ctx.first{{/toJson}}", ctx,
+                equalTo("{\"name\":\"John Smith\",\"age\":42,\"height\":1.84}"));
+
+        assertScript("{{#toJson}}ctx.second{{/toJson}}", ctx,
+                equalTo("{\"name\":\"Dave Smith\",\"age\":27,\"height\":1.71}"));
+    }
+
+    public void testSimpleArrayToJSON() throws Exception {
+        String[] array = new String[]{"one", "two", "three"};
+        Map<String, Object> ctx = Collections.singletonMap("array", array);
+
+        assertScript("{{#toJson}}.{{/toJson}}", ctx, equalTo("{\"array\":[\"one\",\"two\",\"three\"]}"));
+        assertScript("{{#toJson}}array{{/toJson}}", ctx, equalTo("[\"one\",\"two\",\"three\"]"));
+        assertScript("{{#toJson}}array.0{{/toJson}}", ctx, equalTo("one"));
+        assertScript("{{#toJson}}array.1{{/toJson}}", ctx, equalTo("two"));
+        assertScript("{{#toJson}}array.2{{/toJson}}", ctx, equalTo("three"));
+        assertScript("{{#toJson}}array.size{{/toJson}}", ctx, equalTo("3"));
+    }
+
+    public void testSimpleListToJSON() throws Exception {
+        List<String> list = Arrays.asList("one", "two", "three");
+        Map<String, Object> ctx = Collections.singletonMap("ctx", list);
+
+        assertScript("{{#toJson}}.{{/toJson}}", ctx, equalTo("{\"ctx\":[\"one\",\"two\",\"three\"]}"));
+        assertScript("{{#toJson}}ctx{{/toJson}}", ctx, equalTo("[\"one\",\"two\",\"three\"]"));
+        assertScript("{{#toJson}}ctx.0{{/toJson}}", ctx, equalTo("one"));
+        assertScript("{{#toJson}}ctx.1{{/toJson}}", ctx, equalTo("two"));
+        assertScript("{{#toJson}}ctx.2{{/toJson}}", ctx, equalTo("three"));
+        assertScript("{{#toJson}}ctx.size{{/toJson}}", ctx, equalTo("3"));
+    }
+
+    public void testsUnsupportedTagsToJson() {
+        MustacheException e = expectThrows(MustacheException.class, () -> compile("{{#toJson}}{{foo}}{{bar}}{{/toJson}}"));
+        assertThat(e.getMessage(), containsString("Mustache function [toJson] must contain one and only one identifier"));
+
+        e = expectThrows(MustacheException.class, () -> compile("{{#toJson}}{{/toJson}}"));
+        assertThat(e.getMessage(), containsString("Mustache function [toJson] must contain one and only one identifier"));
+    }
+
+    public void testEmbeddedToJSON() throws Exception {
+        XContentBuilder builder = jsonBuilder().startObject()
+                        .startArray("bulks")
+                            .startObject()
+                                .field("index", "index-1")
+                                .field("type", "type-1")
+                                .field("id", 1)
+                            .endObject()
+                            .startObject()
+                                .field("index", "index-2")
+                                .field("type", "type-2")
+                                .field("id", 2)
+                            .endObject()
+                        .endArray()
+                    .endObject();
+
+        Map<String, Object> ctx = Collections.singletonMap("ctx", XContentHelper.convertToMap(builder.bytes(), false).v2());
+
+        assertScript("{{#ctx.bulks}}{{#toJson}}.{{/toJson}}{{/ctx.bulks}}", ctx,
+                equalTo("{\"index\":\"index-1\",\"id\":1,\"type\":\"type-1\"}{\"index\":\"index-2\",\"id\":2,\"type\":\"type-2\"}"));
+
+        assertScript("{{#ctx.bulks}}<{{#toJson}}id{{/toJson}}>{{/ctx.bulks}}", ctx,
+                equalTo("<1><2>"));
+    }
+
+    public void testSimpleArrayJoin() throws Exception {
+        String template = "{{#join}}array{{/join}}";
+        assertScript(template, Collections.singletonMap("array", new String[]{"one", "two", "three"}), equalTo("one,two,three"));
+        assertScript(template, Collections.singletonMap("array", new int[]{1, 2, 3}), equalTo("1,2,3"));
+        assertScript(template, Collections.singletonMap("array", new long[]{1L, 2L, 3L}), equalTo("1,2,3"));
+        assertScript(template, Collections.singletonMap("array", new double[]{1.5, 2.5, 3.5}), equalTo("1.5,2.5,3.5"));
+        assertScript(template, Collections.singletonMap("array", new boolean[]{true, false, true}), equalTo("true,false,true"));
+        assertScript(template, Collections.singletonMap("array", new boolean[]{true, false, true}), equalTo("true,false,true"));
+    }
+
+    public void testEmbeddedArrayJoin() throws Exception {
+        XContentBuilder builder = jsonBuilder().startObject()
+                                                    .startArray("people")
+                                                        .startObject()
+                                                            .field("name", "John Smith")
+                                                            .startArray("emails")
+                                                                .value("john@smith.com")
+                                                                .value("john.smith@email.com")
+                                                                .value("jsmith@email.com")
+                                                            .endArray()
+                                                        .endObject()
+                                                        .startObject()
+                                                            .field("name", "John Doe")
+                                                            .startArray("emails")
+                                                                .value("john@doe.com")
+                                                                .value("john.doe@email.com")
+                                                                .value("jdoe@email.com")
+                                                            .endArray()
+                                                        .endObject()
+                                                    .endArray()
+                                                .endObject();
+
+        Map<String, Object> ctx = Collections.singletonMap("ctx", XContentHelper.convertToMap(builder.bytes(), false).v2());
+
+        assertScript("{{#join}}ctx.people.0.emails{{/join}}", ctx,
+                equalTo("john@smith.com,john.smith@email.com,jsmith@email.com"));
+
+        assertScript("{{#join}}ctx.people.1.emails{{/join}}", ctx,
+                equalTo("john@doe.com,john.doe@email.com,jdoe@email.com"));
+
+        assertScript("{{#ctx.people}}to: {{#join}}emails{{/join}};{{/ctx.people}}", ctx,
+                equalTo("to: john@smith.com,john.smith@email.com,jsmith@email.com;to: john@doe.com,john.doe@email.com,jdoe@email.com;"));
+    }
+
+    public void testJoinWithToJson() {
+        Map<String, Object> params = Collections.singletonMap("terms",
+                Arrays.asList(singletonMap("term", "foo"), singletonMap("term", "bar")));
+
+        assertScript("{{#join}}{{#toJson}}terms{{/toJson}}{{/join}}", params,
+                equalTo("[{\"term\":\"foo\"},{\"term\":\"bar\"}]"));
+    }
+
+    public void testsUnsupportedTagsJoin() {
+        MustacheException e = expectThrows(MustacheException.class, () -> compile("{{#join}}{{/join}}"));
+        assertThat(e.getMessage(), containsString("Mustache function [join] must contain one and only one identifier"));
+
+        e = expectThrows(MustacheException.class, () -> compile("{{#join delimiter='a'}}{{/join delimiter='b'}}"));
+        assertThat(e.getMessage(), containsString("Mismatched start/end tags"));
+    }
+
+    public void testJoinWithCustomDelimiter() {
+        Map<String, Object> params = Collections.singletonMap("params", Arrays.asList(1, 2, 3, 4));
+
+        assertScript("{{#join delimiter=''}}params{{/join delimiter=''}}", params, equalTo("1234"));
+        assertScript("{{#join delimiter=','}}params{{/join delimiter=','}}", params, equalTo("1,2,3,4"));
+        assertScript("{{#join delimiter='/'}}params{{/join delimiter='/'}}", params, equalTo("1/2/3/4"));
+        assertScript("{{#join delimiter=' and '}}params{{/join delimiter=' and '}}", params, equalTo("1 and 2 and 3 and 4"));
+    }
+
+    private void assertScript(String script, Map<String, Object> vars, Matcher<Object> matcher) {
+        Object result = engine.executable(new CompiledScript(INLINE, "inline", "mustache", compile(script)), vars).run();
+        assertThat(result, notNullValue());
+        assertThat(result, instanceOf(BytesReference.class));
+        assertThat(((BytesReference) result).toUtf8(), matcher);
+    }
+
+    private Object compile(String script) {
+        assertThat("cannot compile null or empty script", script, not(isEmptyOrNullString()));
+        return engine.compile(null, script, Collections.emptyMap());
+    }
 }

From 6b7acc0ca221f13031024872879e0e0f01f8c135 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Wed, 29 Jun 2016 10:24:43 +0200
Subject: [PATCH 1108/1311] Update index.asciidoc

In-flight requests circuit breaker is done
---
 docs/resiliency/index.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/resiliency/index.asciidoc b/docs/resiliency/index.asciidoc
index 24035e0772d..6f3ed169709 100644
--- a/docs/resiliency/index.asciidoc
+++ b/docs/resiliency/index.asciidoc
@@ -90,7 +90,7 @@ space.  The following issues have been identified:
 * Set a hard limit on `from`/`size` parameters {GIT}9311[#9311]. (STATUS: DONE, v2.1.0)
 * Prevent combinatorial explosion in aggregations from causing OOM {GIT}8081[#8081]. (STATUS: ONGOING)
 * Add the byte size of each hit to the request circuit breaker {GIT}9310[#9310]. (STATUS: ONGOING)
-* Limit the size of individual requests and also add a circuit breaker for the total memory used by in-flight request objects {GIT}16011[#16011]. (STATUS: ONGOING)
+* Limit the size of individual requests and also add a circuit breaker for the total memory used by in-flight request objects {GIT}16011[#16011]. (STATUS: DONE, v5.0.0)
 
 Other safeguards are tracked in the meta-issue {GIT}11511[#11511].
 

From 872cdffc2748afdb08073f230a226e02e061eba0 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Wed, 29 Jun 2016 10:45:05 +0200
Subject: [PATCH 1109/1311] Factor out ChannelBuffer from BytesReference
 (#19129)

The ChannelBuffer interface today leaks into the BytesReference abstraction
which causes a hard dependency on Netty across the board. This chance moves
this dependency and all BytesReference -> ChannelBuffer conversion into
NettyUtlis and removes the abstraction leak on BytesReference.
This change also removes unused methods on the BytesReference interface
and simplifies access to internal pages.
---
 .../common/bytes/BytesArray.java              |  14 --
 .../common/bytes/BytesReference.java          |  32 ++--
 .../common/bytes/PagedBytesReference.java     | 127 ++++-----------
 .../bytes/ReleasablePagedBytesReference.java  |   2 +-
 .../ChannelBufferBytesReference.java          |  16 +-
 .../common/netty/NettyUtils.java              |  40 +++++
 .../elasticsearch/common/util/ByteArray.java  |  12 +-
 .../http/netty/NettyHttpChannel.java          |   3 +-
 .../http/netty/NettyHttpRequest.java          |   4 +-
 .../netty/ChannelBufferStreamInput.java       |   8 +-
 .../netty/MessageChannelHandler.java          |   4 +-
 .../transport/netty/NettyTransport.java       |   6 +-
 .../netty/NettyTransportChannel.java          |   5 +-
 .../bytes/PagedBytesReferenceTests.java       | 153 ++++++++----------
 .../common/netty/NettyUtilsTests.java         |  95 +++++++++++
 .../bytes/ByteBufferBytesReference.java       |  14 --
 16 files changed, 278 insertions(+), 257 deletions(-)
 rename core/src/main/java/org/elasticsearch/common/{bytes => netty}/ChannelBufferBytesReference.java (88%)
 create mode 100644 core/src/test/java/org/elasticsearch/common/netty/NettyUtilsTests.java
 rename {core/src/test => test/framework/src/main}/java/org/elasticsearch/common/bytes/ByteBufferBytesReference.java (91%)

diff --git a/core/src/main/java/org/elasticsearch/common/bytes/BytesArray.java b/core/src/main/java/org/elasticsearch/common/bytes/BytesArray.java
index 5cb690b7375..5da3c3c6bf3 100644
--- a/core/src/main/java/org/elasticsearch/common/bytes/BytesArray.java
+++ b/core/src/main/java/org/elasticsearch/common/bytes/BytesArray.java
@@ -20,14 +20,10 @@
 package org.elasticsearch.common.bytes;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.io.Channels;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.jboss.netty.buffer.ChannelBuffer;
-import org.jboss.netty.buffer.ChannelBuffers;
 
 import java.io.IOException;
 import java.io.OutputStream;
-import java.nio.channels.GatheringByteChannel;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 
@@ -103,11 +99,6 @@ public class BytesArray implements BytesReference {
         os.write(bytes, offset, length);
     }
 
-    @Override
-    public void writeTo(GatheringByteChannel channel) throws IOException {
-        Channels.writeToChannel(bytes, offset, length(), channel);
-    }
-
     @Override
     public byte[] toBytes() {
         if (offset == 0 && bytes.length == length) {
@@ -126,11 +117,6 @@ public class BytesArray implements BytesReference {
         return new BytesArray(Arrays.copyOfRange(bytes, offset, offset + length));
     }
 
-    @Override
-    public ChannelBuffer toChannelBuffer() {
-        return ChannelBuffers.wrappedBuffer(bytes, offset, length);
-    }
-
     @Override
     public boolean hasArray() {
         return true;
diff --git a/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java b/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java
index a72346f9ee8..a07e7cda267 100644
--- a/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java
+++ b/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java
@@ -19,19 +19,18 @@
 package org.elasticsearch.common.bytes;
 
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefIterator;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.jboss.netty.buffer.ChannelBuffer;
 
 import java.io.IOException;
 import java.io.OutputStream;
-import java.nio.channels.GatheringByteChannel;
 
 /**
  * A reference to bytes.
  */
 public interface BytesReference {
 
-    public static class Helper {
+    class Helper {
 
         public static boolean bytesEqual(BytesReference a, BytesReference b) {
             if (a == b) {
@@ -108,10 +107,6 @@ public interface BytesReference {
      */
     void writeTo(OutputStream os) throws IOException;
 
-    /**
-     * Writes the bytes directly to the channel.
-     */
-    void writeTo(GatheringByteChannel channel) throws IOException;
 
     /**
      * Returns the bytes as a single byte array.
@@ -128,11 +123,6 @@ public interface BytesReference {
      */
     BytesArray copyBytesArray();
 
-    /**
-     * Returns the bytes as a channel buffer.
-     */
-    ChannelBuffer toChannelBuffer();
-
     /**
      * Is there an underlying byte array for this bytes reference.
      */
@@ -162,4 +152,22 @@ public interface BytesReference {
      * Converts to a copied Lucene BytesRef.
      */
     BytesRef copyBytesRef();
+
+    /**
+     * Returns a BytesRefIterator for this BytesReference. This method allows
+     * access to the internal pages of this reference without copying them. Use with care!
+     * @see BytesRefIterator
+     */
+    default BytesRefIterator iterator() {
+        return new BytesRefIterator() {
+            BytesRef ref = toBytesRef();
+            @Override
+            public BytesRef next() throws IOException {
+                BytesRef r = ref;
+                ref = null; // only return it once...
+                return r;
+            }
+        };
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java b/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java
index 16ce91dc38f..af4e1cb29fd 100644
--- a/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java
+++ b/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java
@@ -20,19 +20,15 @@
 package org.elasticsearch.common.bytes;
 
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefIterator;
 import org.apache.lucene.util.CharsRefBuilder;
-import org.elasticsearch.common.io.Channels;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.netty.NettyUtils;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.ByteArray;
-import org.jboss.netty.buffer.ChannelBuffer;
-import org.jboss.netty.buffer.ChannelBuffers;
 
 import java.io.EOFException;
 import java.io.IOException;
 import java.io.OutputStream;
-import java.nio.channels.GatheringByteChannel;
 import java.util.Arrays;
 
 /**
@@ -113,30 +109,6 @@ public class PagedBytesReference implements BytesReference {
         }
     }
 
-    @Override
-    public void writeTo(GatheringByteChannel channel) throws IOException {
-        // nothing to do
-        if (length == 0) {
-            return;
-        }
-
-        int currentLength = length;
-        int currentOffset = offset;
-        BytesRef ref = new BytesRef();
-
-        while (currentLength > 0) {
-            // try to align to the underlying pages while writing, so no new arrays will be created.
-            int fragmentSize = Math.min(currentLength, PAGE_SIZE - (currentOffset % PAGE_SIZE));
-            boolean newArray = bytearray.get(currentOffset, fragmentSize, ref);
-            assert !newArray : "PagedBytesReference failed to align with underlying bytearray. offset [" + currentOffset + "], size [" + fragmentSize + "]";
-            Channels.writeToChannel(ref.bytes, ref.offset, ref.length, channel);
-            currentLength -= ref.length;
-            currentOffset += ref.length;
-        }
-
-        assert currentLength == 0;
-    }
-
     @Override
     public byte[] toBytes() {
         if (length == 0) {
@@ -178,60 +150,6 @@ public class PagedBytesReference implements BytesReference {
         }
     }
 
-    @Override
-    public ChannelBuffer toChannelBuffer() {
-        // nothing to do
-        if (length == 0) {
-            return ChannelBuffers.EMPTY_BUFFER;
-        }
-
-        ChannelBuffer[] buffers;
-        ChannelBuffer currentBuffer = null;
-        BytesRef ref = new BytesRef();
-        int pos = 0;
-
-        // are we a slice?
-        if (offset != 0) {
-            // remaining size of page fragment at offset
-            int fragmentSize = Math.min(length, PAGE_SIZE - (offset % PAGE_SIZE));
-            bytearray.get(offset, fragmentSize, ref);
-            currentBuffer = ChannelBuffers.wrappedBuffer(ref.bytes, ref.offset, fragmentSize);
-            pos += fragmentSize;
-        }
-
-        // no need to create a composite buffer for a single page
-        if (pos == length && currentBuffer != null) {
-            return currentBuffer;
-        }
-
-        // a slice > pagesize will likely require extra buffers for initial/trailing fragments
-        int numBuffers = countRequiredBuffers((currentBuffer != null ? 1 : 0), length - pos);
-
-        buffers = new ChannelBuffer[numBuffers];
-        int bufferSlot = 0;
-
-        if (currentBuffer != null) {
-            buffers[bufferSlot] = currentBuffer;
-            bufferSlot++;
-        }
-
-        // handle remainder of pages + trailing fragment
-        while (pos < length) {
-            int remaining = length - pos;
-            int bulkSize = (remaining > PAGE_SIZE) ? PAGE_SIZE : remaining;
-            bytearray.get(offset + pos, bulkSize, ref);
-            currentBuffer = ChannelBuffers.wrappedBuffer(ref.bytes, ref.offset, bulkSize);
-            buffers[bufferSlot] = currentBuffer;
-            bufferSlot++;
-            pos += bulkSize;
-        }
-
-        // this would indicate that our numBuffer calculation is off by one.
-        assert (numBuffers == bufferSlot);
-
-        return ChannelBuffers.wrappedBuffer(NettyUtils.DEFAULT_GATHERING, buffers);
-    }
-
     @Override
     public boolean hasArray() {
         return (offset + length <= PAGE_SIZE);
@@ -338,17 +256,6 @@ public class PagedBytesReference implements BytesReference {
         return true;
     }
 
-    private int countRequiredBuffers(int initialCount, int numBytes) {
-        int numBuffers = initialCount;
-        // an "estimate" of how many pages remain - rounded down
-        int pages = numBytes / PAGE_SIZE;
-        // a remaining fragment < pagesize needs at least one buffer
-        numBuffers += (pages == 0) ? 1 : pages;
-        // a remainder that is not a multiple of pagesize also needs an extra buffer
-        numBuffers += (pages > 0 && numBytes % PAGE_SIZE > 0) ? 1 : 0;
-        return numBuffers;
-    }
-
     private static class PagedBytesReferenceStreamInput extends StreamInput {
 
         private final ByteArray bytearray;
@@ -451,4 +358,36 @@ public class PagedBytesReference implements BytesReference {
         }
 
     }
+
+    @Override
+    public final BytesRefIterator iterator() {
+        final int offset = this.offset;
+        final int length = this.length;
+        // this iteration is page aligned to ensure we do NOT materialize the pages from the ByteArray
+        // we calculate the initial fragment size here to ensure that if this reference is a slice we are still page aligned
+        // across the entire iteration. The first page is smaller if our offset != 0 then we start in the middle of the page
+        // otherwise we iterate full pages until we reach the last chunk which also might end within a page.
+        final int initialFragmentSize = offset != 0 ? PAGE_SIZE - (offset % PAGE_SIZE) : PAGE_SIZE;
+        return new BytesRefIterator() {
+            int position = 0;
+            int nextFragmentSize = Math.min(length, initialFragmentSize);
+            // this BytesRef is reused across the iteration on purpose - BytesRefIterator interface was designed for this
+            final BytesRef slice = new BytesRef();
+
+            @Override
+            public BytesRef next() throws IOException {
+                if (nextFragmentSize != 0) {
+                    final boolean materialized = bytearray.get(offset + position, nextFragmentSize, slice);
+                    assert materialized == false : "iteration should be page aligned but array got materialized";
+                    position += nextFragmentSize;
+                    final int remaining = length - position;
+                    nextFragmentSize = Math.min(remaining, PAGE_SIZE);
+                    return slice;
+                } else {
+                    assert nextFragmentSize == 0 : "fragmentSize expected [0] but was: [" + nextFragmentSize + "]";
+                    return null; // we are done with this iteration
+                }
+            }
+        };
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/bytes/ReleasablePagedBytesReference.java b/core/src/main/java/org/elasticsearch/common/bytes/ReleasablePagedBytesReference.java
index 603087a9213..2152aa226a8 100644
--- a/core/src/main/java/org/elasticsearch/common/bytes/ReleasablePagedBytesReference.java
+++ b/core/src/main/java/org/elasticsearch/common/bytes/ReleasablePagedBytesReference.java
@@ -28,7 +28,7 @@ import org.elasticsearch.common.util.ByteArray;
  * An extension to {@link PagedBytesReference} that requires releasing its content. This
  * class exists to make it explicit when a bytes reference needs to be released, and when not.
  */
-public class ReleasablePagedBytesReference extends PagedBytesReference implements Releasable {
+public final class ReleasablePagedBytesReference extends PagedBytesReference implements Releasable {
 
     public ReleasablePagedBytesReference(BigArrays bigarrays, ByteArray bytearray, int length) {
         super(bigarrays, bytearray, length);
diff --git a/core/src/main/java/org/elasticsearch/common/bytes/ChannelBufferBytesReference.java b/core/src/main/java/org/elasticsearch/common/netty/ChannelBufferBytesReference.java
similarity index 88%
rename from core/src/main/java/org/elasticsearch/common/bytes/ChannelBufferBytesReference.java
rename to core/src/main/java/org/elasticsearch/common/netty/ChannelBufferBytesReference.java
index 4d6c11214bb..51b3a57c5c1 100644
--- a/core/src/main/java/org/elasticsearch/common/bytes/ChannelBufferBytesReference.java
+++ b/core/src/main/java/org/elasticsearch/common/netty/ChannelBufferBytesReference.java
@@ -16,26 +16,26 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.common.bytes;
+package org.elasticsearch.common.netty;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.io.Channels;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.transport.netty.ChannelBufferStreamInputFactory;
 import org.jboss.netty.buffer.ChannelBuffer;
 
 import java.io.IOException;
 import java.io.OutputStream;
-import java.nio.channels.GatheringByteChannel;
 import java.nio.charset.StandardCharsets;
 
 /**
  */
-public class ChannelBufferBytesReference implements BytesReference {
+final class ChannelBufferBytesReference implements BytesReference {
 
     private final ChannelBuffer buffer;
 
-    public ChannelBufferBytesReference(ChannelBuffer buffer) {
+    ChannelBufferBytesReference(ChannelBuffer buffer) {
         this.buffer = buffer;
     }
 
@@ -64,11 +64,6 @@ public class ChannelBufferBytesReference implements BytesReference {
         buffer.getBytes(buffer.readerIndex(), os, length());
     }
 
-    @Override
-    public void writeTo(GatheringByteChannel channel) throws IOException {
-        Channels.writeToChannel(buffer, buffer.readerIndex(), length(), channel);
-    }
-
     @Override
     public byte[] toBytes() {
         return copyBytesArray().toBytes();
@@ -89,7 +84,6 @@ public class ChannelBufferBytesReference implements BytesReference {
         return new BytesArray(copy);
     }
 
-    @Override
     public ChannelBuffer toChannelBuffer() {
         return buffer.duplicate();
     }
diff --git a/core/src/main/java/org/elasticsearch/common/netty/NettyUtils.java b/core/src/main/java/org/elasticsearch/common/netty/NettyUtils.java
index 92b82fd2d8c..c37ca3ad6fb 100644
--- a/core/src/main/java/org/elasticsearch/common/netty/NettyUtils.java
+++ b/core/src/main/java/org/elasticsearch/common/netty/NettyUtils.java
@@ -18,12 +18,20 @@
  */
 package org.elasticsearch.common.netty;
 
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefIterator;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.transport.netty.NettyInternalESLoggerFactory;
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.buffer.ChannelBuffers;
 import org.jboss.netty.logging.InternalLogger;
 import org.jboss.netty.logging.InternalLoggerFactory;
 import org.jboss.netty.util.ThreadNameDeterminer;
 import org.jboss.netty.util.ThreadRenamingRunnable;
 
+import java.io.IOException;
+import java.util.ArrayList;
+
 /**
  */
 public class NettyUtils {
@@ -98,4 +106,36 @@ public class NettyUtils {
     public static void setup() {
 
     }
+
+    /**
+     * Turns the given BytesReference into a ChannelBuffer. Note: the returned ChannelBuffer will reference the internal
+     * pages of the BytesReference. Don't free the bytes of reference before the ChannelBuffer goes out of scope.
+     */
+    public static ChannelBuffer toChannelBuffer(BytesReference reference) {
+        if (reference.length() == 0) {
+            return ChannelBuffers.EMPTY_BUFFER;
+        }
+        if (reference instanceof ChannelBufferBytesReference) {
+            return ((ChannelBufferBytesReference) reference).toChannelBuffer();
+        } else {
+            final BytesRefIterator iterator = reference.iterator();
+            BytesRef slice;
+            final ArrayList<ChannelBuffer> buffers = new ArrayList<>();
+            try {
+                while ((slice = iterator.next()) != null) {
+                    buffers.add(ChannelBuffers.wrappedBuffer(slice.bytes, slice.offset, slice.length));
+                }
+                return ChannelBuffers.wrappedBuffer(DEFAULT_GATHERING, buffers.toArray(new ChannelBuffer[buffers.size()]));
+            } catch (IOException ex) {
+                throw new AssertionError("no IO happens here", ex);
+            }
+        }
+    }
+
+    /**
+     * Wraps the given ChannelBuffer with a BytesReference
+     */
+    public static BytesReference toBytesReference(ChannelBuffer channelBuffer) {
+        return new ChannelBufferBytesReference(channelBuffer);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/util/ByteArray.java b/core/src/main/java/org/elasticsearch/common/util/ByteArray.java
index 719701a0182..3c7408480d8 100644
--- a/core/src/main/java/org/elasticsearch/common/util/ByteArray.java
+++ b/core/src/main/java/org/elasticsearch/common/util/ByteArray.java
@@ -29,28 +29,28 @@ public interface ByteArray extends BigArray {
     /**
      * Get an element given its index.
      */
-    public abstract byte get(long index);
+    byte get(long index);
 
     /**
      * Set a value at the given index and return the previous value.
      */
-    public abstract byte set(long index, byte value);
+    byte set(long index, byte value);
 
     /**
      * Get a reference to a slice.
-     * 
+     *
      * @return <code>true</code> when a byte[] was materialized, <code>false</code> otherwise.
      */
-    public abstract boolean get(long index, int len, BytesRef ref);
+    boolean get(long index, int len, BytesRef ref);
 
     /**
      * Bulk set.
      */
-    public abstract void set(long index, byte[] buf, int offset, int len);
+    void set(long index, byte[] buf, int offset, int len);
 
     /**
      * Fill slots between <code>fromIndex</code> inclusive to <code>toIndex</code> exclusive with <code>value</code>.
      */
-    public abstract void fill(long fromIndex, long toIndex, byte value);
+    void fill(long fromIndex, long toIndex, byte value);
 
 }
diff --git a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpChannel.java b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpChannel.java
index 22d6743b186..3413a746963 100644
--- a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpChannel.java
+++ b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpChannel.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
 import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.netty.NettyUtils;
 import org.elasticsearch.common.netty.ReleaseChannelFutureListener;
 import org.elasticsearch.http.netty.cors.CorsHandler;
 import org.elasticsearch.http.netty.pipelining.OrderedDownstreamChannelEvent;
@@ -105,7 +106,7 @@ public final class NettyHttpChannel extends AbstractRestChannel {
         ChannelBuffer buffer;
         boolean addedReleaseListener = false;
         try {
-            buffer = content.toChannelBuffer();
+            buffer = NettyUtils.toChannelBuffer(content);
             resp.setContent(buffer);
 
             // If our response doesn't specify a content-type header, set one
diff --git a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpRequest.java b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpRequest.java
index 40ab2d184db..d26841ead97 100644
--- a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpRequest.java
+++ b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpRequest.java
@@ -21,7 +21,7 @@ package org.elasticsearch.http.netty;
 
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.bytes.ChannelBufferBytesReference;
+import org.elasticsearch.common.netty.NettyUtils;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.support.RestUtils;
 import org.jboss.netty.channel.Channel;
@@ -47,7 +47,7 @@ public class NettyHttpRequest extends RestRequest {
         this.channel = channel;
         this.params = new HashMap<>();
         if (request.getContent().readable()) {
-            this.content = new ChannelBufferBytesReference(request.getContent());
+            this.content = NettyUtils.toBytesReference(request.getContent());
         } else {
             this.content = BytesArray.EMPTY;
         }
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java b/core/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java
index ce0eea860ef..695be3db2fd 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java
@@ -21,8 +21,8 @@ package org.elasticsearch.transport.netty;
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.bytes.ChannelBufferBytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.netty.NettyUtils;
 import org.jboss.netty.buffer.ChannelBuffer;
 
 import java.io.EOFException;
@@ -37,10 +37,6 @@ public class ChannelBufferStreamInput extends StreamInput {
     private final int startIndex;
     private final int endIndex;
 
-    public ChannelBufferStreamInput(ChannelBuffer buffer) {
-        this(buffer, buffer.readableBytes());
-    }
-
     public ChannelBufferStreamInput(ChannelBuffer buffer, int length) {
         if (length > buffer.readableBytes()) {
             throw new IndexOutOfBoundsException();
@@ -53,7 +49,7 @@ public class ChannelBufferStreamInput extends StreamInput {
 
     @Override
     public BytesReference readBytesReference(int length) throws IOException {
-        ChannelBufferBytesReference ref = new ChannelBufferBytesReference(buffer.slice(buffer.readerIndex(), length));
+        BytesReference ref = NettyUtils.toBytesReference(buffer.slice(buffer.readerIndex(), length));
         buffer.skipBytes(length);
         return ref;
     }
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java b/core/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java
index 55793384b1b..3274aa7b975 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java
@@ -21,7 +21,6 @@ package org.elasticsearch.transport.netty;
 
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.Version;
-import org.elasticsearch.common.bytes.ChannelBufferBytesReference;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.compress.Compressor;
 import org.elasticsearch.common.compress.CompressorFactory;
@@ -29,6 +28,7 @@ import org.elasticsearch.common.compress.NotCompressedException;
 import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.netty.NettyUtils;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -111,7 +111,7 @@ public class MessageChannelHandler extends SimpleChannelUpstreamHandler {
             if (TransportStatus.isCompress(status) && hasMessageBytesToRead && buffer.readable()) {
                 Compressor compressor;
                 try {
-                    compressor = CompressorFactory.compressor(new ChannelBufferBytesReference(buffer));
+                    compressor = CompressorFactory.compressor(NettyUtils.toBytesReference(buffer));
                 } catch (NotCompressedException ex) {
                     int maxToRead = Math.min(buffer.readableBytes(), 10);
                     int offset = buffer.readerIndex();
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
index 53eb63c86c5..be1305244bc 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
@@ -909,14 +909,14 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
                 bRequest.writeThin(stream);
                 stream.close();
                 bytes = bStream.bytes();
-                ChannelBuffer headerBuffer = bytes.toChannelBuffer();
-                ChannelBuffer contentBuffer = bRequest.bytes().toChannelBuffer();
+                ChannelBuffer headerBuffer = NettyUtils.toChannelBuffer(bytes);
+                ChannelBuffer contentBuffer = NettyUtils.toChannelBuffer(bRequest.bytes());
                 buffer = ChannelBuffers.wrappedBuffer(NettyUtils.DEFAULT_GATHERING, headerBuffer, contentBuffer);
             } else {
                 request.writeTo(stream);
                 stream.close();
                 bytes = bStream.bytes();
-                buffer = bytes.toChannelBuffer();
+                buffer = NettyUtils.toChannelBuffer(bytes);
             }
             NettyHeader.writeHeader(buffer, requestId, status, version);
             ChannelFuture future = targetChannel.write(buffer);
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java
index 65ea00d75e5..0d5666408ea 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.common.netty.NettyUtils;
 import org.elasticsearch.common.netty.ReleaseChannelFutureListener;
 import org.elasticsearch.transport.RemoteTransportException;
 import org.elasticsearch.transport.TransportChannel;
@@ -106,7 +107,7 @@ public class NettyTransportChannel implements TransportChannel {
             stream.close();
 
             ReleasablePagedBytesReference bytes = bStream.bytes();
-            ChannelBuffer buffer = bytes.toChannelBuffer();
+            ChannelBuffer buffer = NettyUtils.toChannelBuffer(bytes);
             NettyHeader.writeHeader(buffer, requestId, status, version);
             ChannelFuture future = channel.write(buffer);
             ReleaseChannelFutureListener listener = new ReleaseChannelFutureListener(bytes);
@@ -136,7 +137,7 @@ public class NettyTransportChannel implements TransportChannel {
         status = TransportStatus.setError(status);
 
         BytesReference bytes = stream.bytes();
-        ChannelBuffer buffer = bytes.toChannelBuffer();
+        ChannelBuffer buffer = NettyUtils.toChannelBuffer(bytes);
         NettyHeader.writeHeader(buffer, requestId, status, version);
         ChannelFuture future = channel.write(buffer);
         ChannelFutureListener onResponseSentListener =
diff --git a/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java b/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java
index 9cb633e410e..ccdd50faff7 100644
--- a/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java
+++ b/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.common.bytes;
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.lucene.util.BytesRefIterator;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -29,38 +30,19 @@ import org.elasticsearch.common.util.ByteArray;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
-import org.jboss.netty.buffer.ChannelBuffer;
 import org.junit.After;
 import org.junit.Before;
 
 import java.io.EOFException;
 import java.io.IOException;
-import java.nio.channels.FileChannel;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardOpenOption;
 import java.util.Arrays;
 
 public class PagedBytesReferenceTests extends ESTestCase {
 
     private static final int PAGE_SIZE = BigArrays.BYTE_PAGE_SIZE;
+    private BigArrays bigarrays = new BigArrays(null, new NoneCircuitBreakerService(), false);
 
-    private BigArrays bigarrays;
-
-    @Override
-    @Before
-    public void setUp() throws Exception {
-        super.setUp();
-        bigarrays = new BigArrays(null, new NoneCircuitBreakerService(), false);
-    }
-
-    @Override
-    @After
-    public void tearDown() throws Exception {
-        super.tearDown();
-    }
-
-    public void testGet() {
+    public void testGet() throws IOException {
         int length = randomIntBetween(1, PAGE_SIZE * 3);
         BytesReference pbr = getRandomizedPagedBytesReference(length);
         int sliceOffset = randomIntBetween(0, length / 2);
@@ -70,7 +52,7 @@ public class PagedBytesReferenceTests extends ESTestCase {
         assertEquals(pbr.get(sliceOffset + sliceLength - 1), slice.get(sliceLength - 1));
     }
 
-    public void testLength() {
+    public void testLength() throws IOException {
         int[] sizes = {0, randomInt(PAGE_SIZE), PAGE_SIZE, randomInt(PAGE_SIZE * 3)};
 
         for (int i = 0; i < sizes.length; i++) {
@@ -79,7 +61,7 @@ public class PagedBytesReferenceTests extends ESTestCase {
         }
     }
 
-    public void testSlice() {
+    public void testSlice() throws IOException {
         int length = randomInt(PAGE_SIZE * 3);
         BytesReference pbr = getRandomizedPagedBytesReference(length);
         int sliceOffset = randomIntBetween(0, length / 2);
@@ -265,17 +247,6 @@ public class PagedBytesReferenceTests extends ESTestCase {
         out.close();
     }
 
-    public void testWriteToChannel() throws IOException {
-        int length = randomIntBetween(10, PAGE_SIZE * 4);
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
-        Path tFile = createTempFile();
-        try (FileChannel channel = FileChannel.open(tFile, StandardOpenOption.WRITE)) {
-            pbr.writeTo(channel);
-            assertEquals(pbr.length(), channel.position());
-        }
-        assertArrayEquals(pbr.toBytes(), Files.readAllBytes(tFile));
-    }
-
     public void testSliceWriteToOutputStream() throws IOException {
         int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 5));
         BytesReference pbr = getRandomizedPagedBytesReference(length);
@@ -289,21 +260,7 @@ public class PagedBytesReferenceTests extends ESTestCase {
         sliceOut.close();
     }
 
-    public void testSliceWriteToChannel() throws IOException {
-        int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 5));
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
-        int sliceOffset = randomIntBetween(1, length / 2);
-        int sliceLength = length - sliceOffset;
-        BytesReference slice = pbr.slice(sliceOffset, sliceLength);
-        Path tFile = createTempFile();
-        try (FileChannel channel = FileChannel.open(tFile, StandardOpenOption.WRITE)) {
-            slice.writeTo(channel);
-            assertEquals(slice.length(), channel.position());
-        }
-        assertArrayEquals(slice.toBytes(), Files.readAllBytes(tFile));
-    }
-
-    public void testToBytes() {
+    public void testToBytes() throws IOException {
         int[] sizes = {0, randomInt(PAGE_SIZE), PAGE_SIZE, randomIntBetween(2, PAGE_SIZE * randomIntBetween(2, 5))};
 
         for (int i = 0; i < sizes.length; i++) {
@@ -319,7 +276,7 @@ public class PagedBytesReferenceTests extends ESTestCase {
         }
     }
 
-    public void testToBytesArraySharedPage() {
+    public void testToBytesArraySharedPage() throws IOException {
         int length = randomIntBetween(10, PAGE_SIZE);
         BytesReference pbr = getRandomizedPagedBytesReference(length);
         BytesArray ba = pbr.toBytesArray();
@@ -332,7 +289,7 @@ public class PagedBytesReferenceTests extends ESTestCase {
         assertSame(ba.array(), ba2.array());
     }
 
-    public void testToBytesArrayMaterializedPages() {
+    public void testToBytesArrayMaterializedPages() throws IOException {
         // we need a length != (n * pagesize) to avoid page sharing at boundaries
         int length = 0;
         while ((length % PAGE_SIZE) == 0) {
@@ -349,7 +306,7 @@ public class PagedBytesReferenceTests extends ESTestCase {
         assertNotSame(ba.array(), ba2.array());
     }
 
-    public void testCopyBytesArray() {
+    public void testCopyBytesArray() throws IOException {
         // small PBR which would normally share the first page
         int length = randomIntBetween(10, PAGE_SIZE);
         BytesReference pbr = getRandomizedPagedBytesReference(length);
@@ -360,7 +317,7 @@ public class PagedBytesReferenceTests extends ESTestCase {
         assertNotSame(ba.array(), ba2.array());
     }
 
-    public void testSliceCopyBytesArray() {
+    public void testSliceCopyBytesArray() throws IOException {
         int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8));
         BytesReference pbr = getRandomizedPagedBytesReference(length);
         int sliceOffset = randomIntBetween(0, pbr.length());
@@ -377,45 +334,67 @@ public class PagedBytesReferenceTests extends ESTestCase {
         assertArrayEquals(ba1.array(), ba2.array());
     }
 
-    public void testToChannelBuffer() {
+    public void testEmptyToBytesRefIterator() throws IOException {
+        BytesReference pbr = getRandomizedPagedBytesReference(0);
+        assertNull(pbr.iterator().next());
+    }
+
+    public void testIterator() throws IOException {
         int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8));
         BytesReference pbr = getRandomizedPagedBytesReference(length);
-        ChannelBuffer cb = pbr.toChannelBuffer();
-        assertNotNull(cb);
-        byte[] bufferBytes = new byte[length];
-        cb.getBytes(0, bufferBytes);
-        assertArrayEquals(pbr.toBytes(), bufferBytes);
+        BytesRefIterator iterator = pbr.iterator();
+        BytesRef ref;
+        BytesRefBuilder builder = new BytesRefBuilder();
+        while((ref = iterator.next()) != null) {
+            builder.append(ref);
+        }
+        assertArrayEquals(pbr.toBytes(), BytesRef.deepCopyOf(builder.toBytesRef()).bytes);
     }
 
-    public void testEmptyToChannelBuffer() {
-        BytesReference pbr = getRandomizedPagedBytesReference(0);
-        ChannelBuffer cb = pbr.toChannelBuffer();
-        assertNotNull(cb);
-        assertEquals(0, pbr.length());
-        assertEquals(0, cb.capacity());
-    }
-
-    public void testSliceToChannelBuffer() {
+    public void testSliceIterator() throws IOException {
         int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8));
         BytesReference pbr = getRandomizedPagedBytesReference(length);
         int sliceOffset = randomIntBetween(0, pbr.length());
         int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset);
         BytesReference slice = pbr.slice(sliceOffset, sliceLength);
-        ChannelBuffer cbSlice = slice.toChannelBuffer();
-        assertNotNull(cbSlice);
-        byte[] sliceBufferBytes = new byte[sliceLength];
-        cbSlice.getBytes(0, sliceBufferBytes);
-        assertArrayEquals(slice.toBytes(), sliceBufferBytes);
+        BytesRefIterator iterator = slice.iterator();
+        BytesRef ref = null;
+        BytesRefBuilder builder = new BytesRefBuilder();
+        while((ref = iterator.next()) != null) {
+            builder.append(ref);
+        }
+        assertArrayEquals(slice.toBytes(), BytesRef.deepCopyOf(builder.toBytesRef()).bytes);
     }
 
-    public void testHasArray() {
+    public void testIteratorRandom() throws IOException {
+        int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8));
+        BytesReference pbr = getRandomizedPagedBytesReference(length);
+        if (randomBoolean()) {
+            int sliceOffset = randomIntBetween(0, pbr.length());
+            int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset);
+            pbr = pbr.slice(sliceOffset, sliceLength);
+        }
+
+        if (randomBoolean()) {
+            pbr = pbr.toBytesArray();
+        }
+        BytesRefIterator iterator = pbr.iterator();
+        BytesRef ref = null;
+        BytesRefBuilder builder = new BytesRefBuilder();
+        while((ref = iterator.next()) != null) {
+            builder.append(ref);
+        }
+        assertArrayEquals(pbr.toBytes(), BytesRef.deepCopyOf(builder.toBytesRef()).bytes);
+    }
+
+    public void testHasArray() throws IOException {
         int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(1, 3));
         BytesReference pbr = getRandomizedPagedBytesReference(length);
         // must return true for <= pagesize
         assertEquals(length <= PAGE_SIZE, pbr.hasArray());
     }
 
-    public void testArray() {
+    public void testArray() throws IOException {
         int[] sizes = {0, randomInt(PAGE_SIZE), PAGE_SIZE, randomIntBetween(2, PAGE_SIZE * randomIntBetween(2, 5))};
 
         for (int i = 0; i < sizes.length; i++) {
@@ -437,7 +416,7 @@ public class PagedBytesReferenceTests extends ESTestCase {
         }
     }
 
-    public void testArrayOffset() {
+    public void testArrayOffset() throws IOException {
         int length = randomInt(PAGE_SIZE * randomIntBetween(2, 5));
         BytesReference pbr = getRandomizedPagedBytesReference(length);
         if (pbr.hasArray()) {
@@ -452,7 +431,7 @@ public class PagedBytesReferenceTests extends ESTestCase {
         }
     }
 
-    public void testSliceArrayOffset() {
+    public void testSliceArrayOffset() throws IOException {
         int length = randomInt(PAGE_SIZE * randomIntBetween(2, 5));
         BytesReference pbr = getRandomizedPagedBytesReference(length);
         int sliceOffset = randomIntBetween(0, pbr.length());
@@ -477,7 +456,7 @@ public class PagedBytesReferenceTests extends ESTestCase {
         // TODO: good way to test?
     }
 
-    public void testToBytesRef() {
+    public void testToBytesRef() throws IOException {
         int length = randomIntBetween(0, PAGE_SIZE);
         BytesReference pbr = getRandomizedPagedBytesReference(length);
         BytesRef ref = pbr.toBytesRef();
@@ -486,7 +465,7 @@ public class PagedBytesReferenceTests extends ESTestCase {
         assertEquals(pbr.length(), ref.length);
     }
 
-    public void testSliceToBytesRef() {
+    public void testSliceToBytesRef() throws IOException {
         int length = randomIntBetween(0, PAGE_SIZE);
         BytesReference pbr = getRandomizedPagedBytesReference(length);
         // get a BytesRef from a slice
@@ -498,7 +477,7 @@ public class PagedBytesReferenceTests extends ESTestCase {
         assertEquals(sliceLength, sliceRef.length);
     }
 
-    public void testCopyBytesRef() {
+    public void testCopyBytesRef() throws IOException {
         int length = randomIntBetween(0, PAGE_SIZE * randomIntBetween(2, 5));
         BytesReference pbr = getRandomizedPagedBytesReference(length);
         BytesRef ref = pbr.copyBytesRef();
@@ -506,7 +485,7 @@ public class PagedBytesReferenceTests extends ESTestCase {
         assertEquals(pbr.length(), ref.length);
     }
 
-    public void testHashCode() {
+    public void testHashCode() throws IOException {
         // empty content must have hash 1 (JDK compat)
         BytesReference pbr = getRandomizedPagedBytesReference(0);
         assertEquals(Arrays.hashCode(BytesRef.EMPTY_BYTES), pbr.hashCode());
@@ -542,7 +521,7 @@ public class PagedBytesReferenceTests extends ESTestCase {
         assertEquals(pbr, pbr2);
     }
 
-    public void testEqualsPeerClass() {
+    public void testEqualsPeerClass() throws IOException {
         int length = randomIntBetween(100, PAGE_SIZE * randomIntBetween(2, 5));
         BytesReference pbr = getRandomizedPagedBytesReference(length);
         BytesReference ba = new BytesArray(pbr.toBytes());
@@ -569,15 +548,11 @@ public class PagedBytesReferenceTests extends ESTestCase {
         }
     }
 
-    private BytesReference getRandomizedPagedBytesReference(int length) {
+    private BytesReference getRandomizedPagedBytesReference(int length) throws IOException {
         // we know bytes stream output always creates a paged bytes reference, we use it to create randomized content
         ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(length, bigarrays);
-        try {
-            for (int i = 0; i < length; i++) {
-                out.writeByte((byte) random().nextInt(1 << 8));
-            }
-        } catch (IOException e) {
-            fail("should not happen " + e.getMessage());
+        for (int i = 0; i < length; i++) {
+            out.writeByte((byte) random().nextInt(1 << 8));
         }
         assertThat(out.size(), Matchers.equalTo(length));
         BytesReference ref = out.bytes();
diff --git a/core/src/test/java/org/elasticsearch/common/netty/NettyUtilsTests.java b/core/src/test/java/org/elasticsearch/common/netty/NettyUtilsTests.java
new file mode 100644
index 00000000000..89c282e7d81
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/netty/NettyUtilsTests.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.common.netty;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.test.ESTestCase;
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.buffer.ChannelBuffers;
+import org.jboss.netty.buffer.CompositeChannelBuffer;
+import org.junit.Before;
+
+import java.io.IOException;
+
+public class NettyUtilsTests extends ESTestCase {
+
+    private static final int PAGE_SIZE = BigArrays.BYTE_PAGE_SIZE;
+    private final BigArrays bigarrays = new BigArrays(null, new NoneCircuitBreakerService(), false);
+
+    public void testToChannelBufferWithEmptyRef() throws IOException {
+        ChannelBuffer channelBuffer = NettyUtils.toChannelBuffer(getRandomizedBytesReference(0));
+        assertSame(ChannelBuffers.EMPTY_BUFFER, channelBuffer);
+    }
+
+    public void testToChannelBufferWithSlice() throws IOException {
+        BytesReference ref = getRandomizedBytesReference(randomIntBetween(1, 3 * PAGE_SIZE));
+        int sliceOffset = randomIntBetween(0, ref.length());
+        int sliceLength = randomIntBetween(ref.length() - sliceOffset, ref.length() - sliceOffset);
+        BytesReference slice = ref.slice(sliceOffset, sliceLength);
+        ChannelBuffer channelBuffer = NettyUtils.toChannelBuffer(slice);
+        BytesReference bytesReference = NettyUtils.toBytesReference(channelBuffer);
+        assertArrayEquals(slice.toBytes(), bytesReference.toBytes());
+    }
+
+    public void testToChannelBufferWithSliceAfter() throws IOException {
+        BytesReference ref = getRandomizedBytesReference(randomIntBetween(1, 3 * PAGE_SIZE));
+        int sliceOffset = randomIntBetween(0, ref.length());
+        int sliceLength = randomIntBetween(ref.length() - sliceOffset, ref.length() - sliceOffset);
+        ChannelBuffer channelBuffer = NettyUtils.toChannelBuffer(ref);
+        BytesReference bytesReference = NettyUtils.toBytesReference(channelBuffer);
+        assertArrayEquals(ref.slice(sliceOffset, sliceLength).toBytes(), bytesReference.slice(sliceOffset, sliceLength).toBytes());
+    }
+
+    public void testToChannelBuffer() throws IOException {
+        BytesReference ref = getRandomizedBytesReference(randomIntBetween(1, 3 * PAGE_SIZE));
+        ChannelBuffer channelBuffer = NettyUtils.toChannelBuffer(ref);
+        BytesReference bytesReference = NettyUtils.toBytesReference(channelBuffer);
+        if (ref instanceof ChannelBufferBytesReference) {
+            assertEquals(channelBuffer, ((ChannelBufferBytesReference) ref).toChannelBuffer());
+        } else if (ref.hasArray() == false) { // we gather the buffers into a channel buffer
+            assertTrue(channelBuffer instanceof CompositeChannelBuffer);
+        }
+        assertArrayEquals(ref.toBytes(), bytesReference.toBytes());
+    }
+
+    private BytesReference getRandomizedBytesReference(int length) throws IOException {
+        // TODO we should factor out a BaseBytesReferenceTestCase
+        // we know bytes stream output always creates a paged bytes reference, we use it to create randomized content
+        ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(length, bigarrays);
+        for (int i = 0; i < length; i++) {
+            out.writeByte((byte) random().nextInt(1 << 8));
+        }
+        assertEquals(out.size(), length);
+        BytesReference ref = out.bytes();
+        assertEquals(ref.length(), length);
+        if (randomBoolean()) {
+            return ref.toBytesArray();
+        } else if (randomBoolean()) {
+            BytesArray bytesArray = ref.toBytesArray();
+            return NettyUtils.toBytesReference(ChannelBuffers.wrappedBuffer(bytesArray.array(), bytesArray.arrayOffset(),
+                bytesArray.length()));
+        } else {
+            return ref;
+        }
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/bytes/ByteBufferBytesReference.java b/test/framework/src/main/java/org/elasticsearch/common/bytes/ByteBufferBytesReference.java
similarity index 91%
rename from core/src/test/java/org/elasticsearch/common/bytes/ByteBufferBytesReference.java
rename to test/framework/src/main/java/org/elasticsearch/common/bytes/ByteBufferBytesReference.java
index a272b6627e4..a1c9da18470 100644
--- a/core/src/test/java/org/elasticsearch/common/bytes/ByteBufferBytesReference.java
+++ b/test/framework/src/main/java/org/elasticsearch/common/bytes/ByteBufferBytesReference.java
@@ -20,18 +20,14 @@
 package org.elasticsearch.common.bytes;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.io.Channels;
 import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.jboss.netty.buffer.ChannelBuffer;
-import org.jboss.netty.buffer.ChannelBuffers;
 import org.jboss.netty.util.CharsetUtil;
 
 import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.nio.CharBuffer;
-import java.nio.channels.GatheringByteChannel;
 import java.nio.charset.CharacterCodingException;
 import java.nio.charset.CharsetDecoder;
 import java.nio.charset.CoderResult;
@@ -85,11 +81,6 @@ public class ByteBufferBytesReference implements BytesReference {
         }
     }
 
-    @Override
-    public void writeTo(GatheringByteChannel channel) throws IOException {
-        Channels.writeToChannel(buffer, channel);
-    }
-
     @Override
     public byte[] toBytes() {
         if (!buffer.hasRemaining()) {
@@ -113,11 +104,6 @@ public class ByteBufferBytesReference implements BytesReference {
         return new BytesArray(toBytes());
     }
 
-    @Override
-    public ChannelBuffer toChannelBuffer() {
-        return ChannelBuffers.wrappedBuffer(buffer);
-    }
-
     @Override
     public boolean hasArray() {
         return buffer.hasArray();

From 39d38e513d2c185204624c95f62ccd2c79c98709 Mon Sep 17 00:00:00 2001
From: Britta Weber <britta.weber@elasticsearch.com>
Date: Wed, 29 Jun 2016 13:30:54 +0200
Subject: [PATCH 1110/1311] [TEST] add wait for yellow

---
 .../main/resources/rest-api-spec/test/msearch/11_status.yaml   | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/11_status.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/11_status.yaml
index 525ea7ac421..ef4d53167b6 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/11_status.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/11_status.yaml
@@ -3,6 +3,9 @@ setup:
   - do:
       indices.create:
         index: test_1
+  - do:
+      cluster.health:
+        wait_for_status: yellow
 
 ---
 "Check Status":

From 819fe40d616dd021ea7a80111c0c836caec8f7c3 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Wed, 29 Jun 2016 14:45:54 +0200
Subject: [PATCH 1111/1311] Extract AbstractBytesReferenceTestCase (#19141)

We have a ton of tests for PagedBytesReference but not really many for the other
implementation of BytesReference. This change factors out a basic AbstractBytesReferenceTestCase
that simplifies testing other implementations. It also caught a couple of bug here and there like
a missing mask when reading bytes as ints in PagedBytesReference.
---
 .../common/bytes/BytesArray.java              |   2 +-
 .../common/bytes/BytesReference.java          |   2 +-
 .../common/bytes/PagedBytesReference.java     |   2 +-
 .../netty/ChannelBufferBytesReference.java    |   2 +-
 .../netty/ChannelBufferStreamInput.java       |   2 +-
 .../bytes/AbstractBytesReferenceTestCase.java | 509 ++++++++++++++++++
 .../common/bytes/BytesArrayTests.java         |  41 ++
 .../bytes/PagedBytesReferenceTests.java       | 509 ++----------------
 .../ChannelBufferBytesReferenceTests.java     |  61 +++
 .../common/netty/NettyUtilsTests.java         |   1 -
 10 files changed, 646 insertions(+), 485 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
 create mode 100644 core/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java
 create mode 100644 core/src/test/java/org/elasticsearch/common/netty/ChannelBufferBytesReferenceTests.java

diff --git a/core/src/main/java/org/elasticsearch/common/bytes/BytesArray.java b/core/src/main/java/org/elasticsearch/common/bytes/BytesArray.java
index 5da3c3c6bf3..1e5e76f7994 100644
--- a/core/src/main/java/org/elasticsearch/common/bytes/BytesArray.java
+++ b/core/src/main/java/org/elasticsearch/common/bytes/BytesArray.java
@@ -27,7 +27,7 @@ import java.io.OutputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 
-public class BytesArray implements BytesReference {
+public final class BytesArray implements BytesReference {
 
     public static final BytesArray EMPTY = new BytesArray(BytesRef.EMPTY_BYTES, 0, 0);
 
diff --git a/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java b/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java
index a07e7cda267..97d4e22f6b2 100644
--- a/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java
+++ b/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java
@@ -160,7 +160,7 @@ public interface BytesReference {
      */
     default BytesRefIterator iterator() {
         return new BytesRefIterator() {
-            BytesRef ref = toBytesRef();
+            BytesRef ref = length() == 0 ? null : toBytesRef();
             @Override
             public BytesRef next() throws IOException {
                 BytesRef r = ref;
diff --git a/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java b/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java
index af4e1cb29fd..70617f903f1 100644
--- a/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java
+++ b/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java
@@ -297,7 +297,7 @@ public class PagedBytesReference implements BytesReference {
 
         @Override
         public int read() throws IOException {
-            return (pos < length) ? bytearray.get(offset + pos++) : -1;
+            return (pos < length) ? Byte.toUnsignedInt(bytearray.get(offset + pos++)) : -1;
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/common/netty/ChannelBufferBytesReference.java b/core/src/main/java/org/elasticsearch/common/netty/ChannelBufferBytesReference.java
index 51b3a57c5c1..42cfe3c611f 100644
--- a/core/src/main/java/org/elasticsearch/common/netty/ChannelBufferBytesReference.java
+++ b/core/src/main/java/org/elasticsearch/common/netty/ChannelBufferBytesReference.java
@@ -51,7 +51,7 @@ final class ChannelBufferBytesReference implements BytesReference {
 
     @Override
     public BytesReference slice(int from, int length) {
-        return new ChannelBufferBytesReference(buffer.slice(from, length));
+        return new ChannelBufferBytesReference(buffer.slice(buffer.readerIndex() + from, length));
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java b/core/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java
index 695be3db2fd..9e2b43d43db 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java
@@ -132,7 +132,7 @@ public class ChannelBufferStreamInput extends StreamInput {
     public void readBytes(byte[] b, int offset, int len) throws IOException {
         int read = read(b, offset, len);
         if (read < len) {
-            throw new EOFException();
+            throw new IndexOutOfBoundsException();
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java b/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
new file mode 100644
index 00000000000..dd9d93dd22a
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
@@ -0,0 +1,509 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.bytes;
+
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.lucene.util.BytesRefIterator;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.ByteArray;
+import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.test.ESTestCase;
+import org.hamcrest.Matchers;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.util.Arrays;
+
+public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
+
+    protected static final int PAGE_SIZE = BigArrays.BYTE_PAGE_SIZE;
+    protected final BigArrays bigarrays = new BigArrays(null, new NoneCircuitBreakerService(), false);
+
+    public void testGet() throws IOException {
+        int length = randomIntBetween(1, PAGE_SIZE * 3);
+        BytesReference pbr = newBytesReference(length);
+        int sliceOffset = randomIntBetween(0, length / 2);
+        int sliceLength = Math.max(1, length - sliceOffset - 1);
+        BytesReference slice = pbr.slice(sliceOffset, sliceLength);
+        assertEquals(pbr.get(sliceOffset), slice.get(0));
+        assertEquals(pbr.get(sliceOffset + sliceLength - 1), slice.get(sliceLength - 1));
+    }
+
+    public void testLength() throws IOException {
+        int[] sizes = {0, randomInt(PAGE_SIZE), PAGE_SIZE, randomInt(PAGE_SIZE * 3)};
+
+        for (int i = 0; i < sizes.length; i++) {
+            BytesReference pbr = newBytesReference(sizes[i]);
+            assertEquals(sizes[i], pbr.length());
+        }
+    }
+
+    public void testSlice() throws IOException {
+        int length = randomInt(PAGE_SIZE * 3);
+        BytesReference pbr = newBytesReference(length);
+        int sliceOffset = randomIntBetween(0, length / 2);
+        int sliceLength = Math.max(0, length - sliceOffset - 1);
+        BytesReference slice = pbr.slice(sliceOffset, sliceLength);
+        assertEquals(sliceLength, slice.length());
+
+        if (slice.hasArray()) {
+            assertEquals(sliceOffset, slice.arrayOffset());
+        } else {
+           expectThrows(IllegalStateException.class, () ->
+                slice.arrayOffset());
+        }
+    }
+
+    public void testStreamInput() throws IOException {
+        int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20));
+        BytesReference pbr = newBytesReference(length);
+        StreamInput si = pbr.streamInput();
+        assertNotNull(si);
+
+        // read single bytes one by one
+        assertEquals(pbr.get(0), si.readByte());
+        assertEquals(pbr.get(1), si.readByte());
+        assertEquals(pbr.get(2), si.readByte());
+
+        // reset the stream for bulk reading
+        si.reset();
+
+        // buffer for bulk reads
+        byte[] origBuf = new byte[length];
+        random().nextBytes(origBuf);
+        byte[] targetBuf = Arrays.copyOf(origBuf, origBuf.length);
+
+        // bulk-read 0 bytes: must not modify buffer
+        si.readBytes(targetBuf, 0, 0);
+        assertEquals(origBuf[0], targetBuf[0]);
+        si.reset();
+
+        // read a few few bytes as ints
+        int bytesToRead = randomIntBetween(1, length / 2);
+        for (int i = 0; i < bytesToRead; i++) {
+            int b = si.read();
+            assertEquals(pbr.get(i) & 0xff, b);
+        }
+        si.reset();
+
+        // bulk-read all
+        si.readFully(targetBuf);
+        assertArrayEquals(pbr.toBytes(), targetBuf);
+
+        // continuing to read should now fail with EOFException
+        try {
+            si.readByte();
+            fail("expected EOF");
+        } catch (EOFException | IndexOutOfBoundsException eof) {
+            // yay
+        }
+
+        // try to read more than the stream contains
+        si.reset();
+        expectThrows(IndexOutOfBoundsException.class, () ->
+            si.readBytes(targetBuf, 0, length * 2));
+    }
+
+    public void testStreamInputBulkReadWithOffset() throws IOException {
+        final int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20));
+        BytesReference pbr = newBytesReference(length);
+        StreamInput si = pbr.streamInput();
+        assertNotNull(si);
+
+        // read a bunch of single bytes one by one
+        int offset = randomIntBetween(1, length / 2);
+        for (int i = 0; i < offset; i++) {
+            assertEquals(si.available(), length - i);
+            assertEquals(pbr.get(i), si.readByte());
+        }
+
+        // now do NOT reset the stream - keep the stream's offset!
+
+        // buffer to compare remaining bytes against bulk read
+        byte[] pbrBytesWithOffset = Arrays.copyOfRange(pbr.toBytes(), offset, length);
+        // randomized target buffer to ensure no stale slots
+        byte[] targetBytes = new byte[pbrBytesWithOffset.length];
+        random().nextBytes(targetBytes);
+
+        // bulk-read all
+        si.readFully(targetBytes);
+        assertArrayEquals(pbrBytesWithOffset, targetBytes);
+        assertEquals(si.available(), 0);
+    }
+
+    public void testRandomReads() throws IOException {
+        int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20));
+        BytesReference pbr = newBytesReference(length);
+        StreamInput streamInput = pbr.streamInput();
+        BytesRefBuilder target = new BytesRefBuilder();
+        while (target.length() < pbr.length()) {
+            switch (randomIntBetween(0, 10)) {
+                case 6:
+                case 5:
+                    target.append(new BytesRef(new byte[]{streamInput.readByte()}));
+                    break;
+                case 4:
+                case 3:
+                    BytesRef bytesRef = streamInput.readBytesRef(scaledRandomIntBetween(1, pbr.length() - target.length()));
+                    target.append(bytesRef);
+                    break;
+                default:
+                    byte[] buffer = new byte[scaledRandomIntBetween(1, pbr.length() - target.length())];
+                    int offset = scaledRandomIntBetween(0, buffer.length - 1);
+                    int read = streamInput.read(buffer, offset, buffer.length - offset);
+                    target.append(new BytesRef(buffer, offset, read));
+                    break;
+            }
+        }
+        assertEquals(pbr.length(), target.length());
+        BytesRef targetBytes = target.get();
+        assertArrayEquals(pbr.toBytes(), Arrays.copyOfRange(targetBytes.bytes, targetBytes.offset, targetBytes.length));
+    }
+
+    public void testSliceStreamInput() throws IOException {
+        int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20));
+        BytesReference pbr = newBytesReference(length);
+
+        // test stream input over slice (upper half of original)
+        int sliceOffset = randomIntBetween(1, length / 2);
+        int sliceLength = length - sliceOffset;
+        BytesReference slice = pbr.slice(sliceOffset, sliceLength);
+        StreamInput sliceInput = slice.streamInput();
+        assertEquals(sliceInput.available(), sliceLength);
+
+        // single reads
+        assertEquals(slice.get(0), sliceInput.readByte());
+        assertEquals(slice.get(1), sliceInput.readByte());
+        assertEquals(slice.get(2), sliceInput.readByte());
+        assertEquals(sliceInput.available(), sliceLength - 3);
+
+        // reset the slice stream for bulk reading
+        sliceInput.reset();
+        assertEquals(sliceInput.available(), sliceLength);
+
+        // bulk read
+        byte[] sliceBytes = new byte[sliceLength];
+        sliceInput.readFully(sliceBytes);
+        assertEquals(sliceInput.available(), 0);
+
+        // compare slice content with upper half of original
+        byte[] pbrSliceBytes = Arrays.copyOfRange(pbr.toBytes(), sliceOffset, length);
+        assertArrayEquals(pbrSliceBytes, sliceBytes);
+
+        // compare slice bytes with bytes read from slice via streamInput :D
+        byte[] sliceToBytes = slice.toBytes();
+        assertEquals(sliceBytes.length, sliceToBytes.length);
+        assertArrayEquals(sliceBytes, sliceToBytes);
+
+        sliceInput.reset();
+        assertEquals(sliceInput.available(), sliceLength);
+        byte[] buffer = new byte[sliceLength + scaledRandomIntBetween(1, 100)];
+        int offset = scaledRandomIntBetween(0, Math.max(1, buffer.length - sliceLength - 1));
+        int read = sliceInput.read(buffer, offset, sliceLength / 2);
+        assertEquals(sliceInput.available(), sliceLength - read);
+        sliceInput.read(buffer, offset + read, sliceLength - read);
+        assertArrayEquals(sliceBytes, Arrays.copyOfRange(buffer, offset, offset + sliceLength));
+        assertEquals(sliceInput.available(), 0);
+    }
+
+    public void testWriteToOutputStream() throws IOException {
+        int length = randomIntBetween(10, PAGE_SIZE * 4);
+        BytesReference pbr = newBytesReference(length);
+        BytesStreamOutput out = new BytesStreamOutput();
+        pbr.writeTo(out);
+        assertEquals(pbr.length(), out.size());
+        assertArrayEquals(pbr.toBytes(), out.bytes().toBytes());
+        out.close();
+    }
+
+    public void testSliceWriteToOutputStream() throws IOException {
+        int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 5));
+        BytesReference pbr = newBytesReference(length);
+        int sliceOffset = randomIntBetween(1, length / 2);
+        int sliceLength = length - sliceOffset;
+        BytesReference slice = pbr.slice(sliceOffset, sliceLength);
+        BytesStreamOutput sliceOut = new BytesStreamOutput(sliceLength);
+        slice.writeTo(sliceOut);
+        assertEquals(slice.length(), sliceOut.size());
+        assertArrayEquals(slice.toBytes(), sliceOut.bytes().toBytes());
+        sliceOut.close();
+    }
+
+    public void testToBytes() throws IOException {
+        int[] sizes = {0, randomInt(PAGE_SIZE), PAGE_SIZE, randomIntBetween(2, PAGE_SIZE * randomIntBetween(2, 5))};
+        for (int i = 0; i < sizes.length; i++) {
+            BytesReference pbr = newBytesReference(sizes[i]);
+            byte[] bytes = pbr.toBytes();
+            assertEquals(sizes[i], bytes.length);
+        }
+    }
+
+    public void testToBytesArraySharedPage() throws IOException {
+        int length = randomIntBetween(10, PAGE_SIZE);
+        BytesReference pbr = newBytesReference(length);
+        BytesArray ba = pbr.toBytesArray();
+        BytesArray ba2 = pbr.toBytesArray();
+        assertNotNull(ba);
+        assertNotNull(ba2);
+        assertEquals(pbr.length(), ba.length());
+        assertEquals(ba.length(), ba2.length());
+        // single-page optimization
+        assertSame(ba.array(), ba2.array());
+    }
+
+    public void testToBytesArrayMaterializedPages() throws IOException {
+        // we need a length != (n * pagesize) to avoid page sharing at boundaries
+        int length = 0;
+        while ((length % PAGE_SIZE) == 0) {
+            length = randomIntBetween(PAGE_SIZE, PAGE_SIZE * randomIntBetween(2, 5));
+        }
+        BytesReference pbr = newBytesReference(length);
+        BytesArray ba = pbr.toBytesArray();
+        BytesArray ba2 = pbr.toBytesArray();
+        assertNotNull(ba);
+        assertNotNull(ba2);
+        assertEquals(pbr.length(), ba.length());
+        assertEquals(ba.length(), ba2.length());
+    }
+
+    public void testCopyBytesArray() throws IOException {
+        // small PBR which would normally share the first page
+        int length = randomIntBetween(10, PAGE_SIZE);
+        BytesReference pbr = newBytesReference(length);
+        BytesArray ba = pbr.copyBytesArray();
+        BytesArray ba2 = pbr.copyBytesArray();
+        assertNotNull(ba);
+        assertNotSame(ba, ba2);
+        assertNotSame(ba.array(), ba2.array());
+    }
+
+    public void testSliceCopyBytesArray() throws IOException {
+        int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8));
+        BytesReference pbr = newBytesReference(length);
+        int sliceOffset = randomIntBetween(0, pbr.length());
+        int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset);
+        BytesReference slice = pbr.slice(sliceOffset, sliceLength);
+
+        BytesArray ba1 = slice.copyBytesArray();
+        BytesArray ba2 = slice.copyBytesArray();
+        assertNotNull(ba1);
+        assertNotNull(ba2);
+        assertNotSame(ba1.array(), ba2.array());
+        assertArrayEquals(slice.toBytes(), ba1.array());
+        assertArrayEquals(slice.toBytes(), ba2.array());
+        assertArrayEquals(ba1.array(), ba2.array());
+    }
+
+    public void testEmptyToBytesRefIterator() throws IOException {
+        BytesReference pbr = newBytesReference(0);
+        assertNull(pbr.iterator().next());
+    }
+
+    public void testIterator() throws IOException {
+        int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8));
+        BytesReference pbr = newBytesReference(length);
+        BytesRefIterator iterator = pbr.iterator();
+        BytesRef ref;
+        BytesRefBuilder builder = new BytesRefBuilder();
+        while((ref = iterator.next()) != null) {
+            builder.append(ref);
+        }
+        assertArrayEquals(pbr.toBytes(), BytesRef.deepCopyOf(builder.toBytesRef()).bytes);
+    }
+
+    public void testSliceIterator() throws IOException {
+        int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8));
+        BytesReference pbr = newBytesReference(length);
+        int sliceOffset = randomIntBetween(0, pbr.length());
+        int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset);
+        BytesReference slice = pbr.slice(sliceOffset, sliceLength);
+        BytesRefIterator iterator = slice.iterator();
+        BytesRef ref = null;
+        BytesRefBuilder builder = new BytesRefBuilder();
+        while((ref = iterator.next()) != null) {
+            builder.append(ref);
+        }
+        assertArrayEquals(slice.toBytes(), BytesRef.deepCopyOf(builder.toBytesRef()).bytes);
+    }
+
+    public void testIteratorRandom() throws IOException {
+        int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8));
+        BytesReference pbr = newBytesReference(length);
+        if (randomBoolean()) {
+            int sliceOffset = randomIntBetween(0, pbr.length());
+            int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset);
+            pbr = pbr.slice(sliceOffset, sliceLength);
+        }
+
+        if (randomBoolean()) {
+            pbr = pbr.toBytesArray();
+        }
+        BytesRefIterator iterator = pbr.iterator();
+        BytesRef ref = null;
+        BytesRefBuilder builder = new BytesRefBuilder();
+        while((ref = iterator.next()) != null) {
+            builder.append(ref);
+        }
+        assertArrayEquals(pbr.toBytes(), BytesRef.deepCopyOf(builder.toBytesRef()).bytes);
+    }
+
+    public void testArray() throws IOException {
+        int[] sizes = {0, randomInt(PAGE_SIZE), PAGE_SIZE, randomIntBetween(2, PAGE_SIZE * randomIntBetween(2, 5))};
+
+        for (int i = 0; i < sizes.length; i++) {
+            BytesReference pbr = newBytesReference(sizes[i]);
+            byte[] array = pbr.array();
+            assertNotNull(array);
+            assertEquals(sizes[i], array.length);
+            assertSame(array, pbr.array());
+        }
+    }
+
+    public void testArrayOffset() throws IOException {
+        int length = randomInt(PAGE_SIZE * randomIntBetween(2, 5));
+        BytesReference pbr = newBytesReference(length);
+        if (pbr.hasArray()) {
+            assertEquals(0, pbr.arrayOffset());
+        } else {
+            expectThrows(IllegalStateException.class, () ->
+                pbr.arrayOffset());
+        }
+    }
+
+    public void testSliceArrayOffset() throws IOException {
+        int length = randomInt(PAGE_SIZE * randomIntBetween(2, 5));
+        BytesReference pbr = newBytesReference(length);
+        int sliceOffset = randomIntBetween(0, pbr.length());
+        int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset);
+        BytesReference slice = pbr.slice(sliceOffset, sliceLength);
+        if (slice.hasArray()) {
+            assertEquals(sliceOffset, slice.arrayOffset());
+        } else {
+            expectThrows(IllegalStateException.class, () ->
+                slice.arrayOffset());
+        }
+    }
+
+    public void testToUtf8() throws IOException {
+        // test empty
+        BytesReference pbr = newBytesReference(0);
+        assertEquals("", pbr.toUtf8());
+        // TODO: good way to test?
+    }
+
+    public void testToBytesRef() throws IOException {
+        int length = randomIntBetween(0, PAGE_SIZE);
+        BytesReference pbr = newBytesReference(length);
+        BytesRef ref = pbr.toBytesRef();
+        assertNotNull(ref);
+        assertEquals(pbr.arrayOffset(), ref.offset);
+        assertEquals(pbr.length(), ref.length);
+    }
+
+    public void testSliceToBytesRef() throws IOException {
+        int length = randomIntBetween(0, PAGE_SIZE);
+        BytesReference pbr = newBytesReference(length);
+        // get a BytesRef from a slice
+        int sliceOffset = randomIntBetween(0, pbr.length());
+        int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset);
+        BytesRef sliceRef = pbr.slice(sliceOffset, sliceLength).toBytesRef();
+        // note that these are only true if we have <= than a page, otherwise offset/length are shifted
+        assertEquals(sliceOffset, sliceRef.offset);
+        assertEquals(sliceLength, sliceRef.length);
+    }
+
+    public void testCopyBytesRef() throws IOException {
+        int length = randomIntBetween(0, PAGE_SIZE * randomIntBetween(2, 5));
+        BytesReference pbr = newBytesReference(length);
+        BytesRef ref = pbr.copyBytesRef();
+        assertNotNull(ref);
+        assertEquals(pbr.length(), ref.length);
+    }
+
+    public void testHashCode() throws IOException {
+        // empty content must have hash 1 (JDK compat)
+        BytesReference pbr = newBytesReference(0);
+        assertEquals(Arrays.hashCode(BytesRef.EMPTY_BYTES), pbr.hashCode());
+
+        // test with content
+        pbr = newBytesReference(randomIntBetween(0, PAGE_SIZE * randomIntBetween(2, 5)));
+        int jdkHash = Arrays.hashCode(pbr.toBytes());
+        int pbrHash = pbr.hashCode();
+        assertEquals(jdkHash, pbrHash);
+
+        // test hashes of slices
+        int sliceFrom = randomIntBetween(0, pbr.length());
+        int sliceLength = randomIntBetween(pbr.length() - sliceFrom, pbr.length() - sliceFrom);
+        BytesReference slice = pbr.slice(sliceFrom, sliceLength);
+        int sliceJdkHash = Arrays.hashCode(slice.toBytes());
+        int sliceHash = slice.hashCode();
+        assertEquals(sliceJdkHash, sliceHash);
+    }
+
+    public void testEquals() {
+        int length = randomIntBetween(100, PAGE_SIZE * randomIntBetween(2, 5));
+        ByteArray ba1 = bigarrays.newByteArray(length, false);
+        ByteArray ba2 = bigarrays.newByteArray(length, false);
+
+        // copy contents
+        for (long i = 0; i < length; i++) {
+            ba2.set(i, ba1.get(i));
+        }
+
+        // get refs & compare
+        BytesReference pbr = new PagedBytesReference(bigarrays, ba1, length);
+        BytesReference pbr2 = new PagedBytesReference(bigarrays, ba2, length);
+        assertEquals(pbr, pbr2);
+    }
+
+    public void testEqualsPeerClass() throws IOException {
+        int length = randomIntBetween(100, PAGE_SIZE * randomIntBetween(2, 5));
+        BytesReference pbr = newBytesReference(length);
+        BytesReference ba = new BytesArray(pbr.toBytes());
+        assertEquals(pbr, ba);
+    }
+
+    public void testSliceEquals() {
+        int length = randomIntBetween(100, PAGE_SIZE * randomIntBetween(2, 5));
+        ByteArray ba1 = bigarrays.newByteArray(length, false);
+        BytesReference pbr = new PagedBytesReference(bigarrays, ba1, length);
+
+        // test equality of slices
+        int sliceFrom = randomIntBetween(0, pbr.length());
+        int sliceLength = randomIntBetween(pbr.length() - sliceFrom, pbr.length() - sliceFrom);
+        BytesReference slice1 = pbr.slice(sliceFrom, sliceLength);
+        BytesReference slice2 = pbr.slice(sliceFrom, sliceLength);
+        assertArrayEquals(slice1.toBytes(), slice2.toBytes());
+
+        // test a slice with same offset but different length,
+        // unless randomized testing gave us a 0-length slice.
+        if (sliceLength > 0) {
+            BytesReference slice3 = pbr.slice(sliceFrom, sliceLength / 2);
+            assertFalse(Arrays.equals(slice1.toBytes(), slice3.toBytes()));
+        }
+    }
+
+    protected abstract BytesReference newBytesReference(int length) throws IOException;
+
+}
diff --git a/core/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java b/core/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java
new file mode 100644
index 00000000000..61d24ef44c3
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.common.bytes;
+
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+
+public class BytesArrayTests extends AbstractBytesReferenceTestCase {
+    @Override
+    protected BytesReference newBytesReference(int length) throws IOException {
+        // we know bytes stream output always creates a paged bytes reference, we use it to create randomized content
+        final BytesStreamOutput out = new BytesStreamOutput(length);
+        for (int i = 0; i < length; i++) {
+            out.writeByte((byte) random().nextInt(1 << 8));
+        }
+        assertEquals(length, out.size());
+        BytesArray ref = out.bytes().toBytesArray();
+        assertEquals(length, ref.length());
+        assertTrue(ref instanceof BytesArray);
+        assertThat(ref.length(), Matchers.equalTo(length));
+        return ref;
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java b/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java
index ccdd50faff7..5a299d82de8 100644
--- a/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java
+++ b/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java
@@ -30,263 +30,24 @@ import org.elasticsearch.common.util.ByteArray;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
-import org.junit.After;
-import org.junit.Before;
 
 import java.io.EOFException;
 import java.io.IOException;
 import java.util.Arrays;
 
-public class PagedBytesReferenceTests extends ESTestCase {
+public class PagedBytesReferenceTests extends AbstractBytesReferenceTestCase {
 
-    private static final int PAGE_SIZE = BigArrays.BYTE_PAGE_SIZE;
-    private BigArrays bigarrays = new BigArrays(null, new NoneCircuitBreakerService(), false);
-
-    public void testGet() throws IOException {
-        int length = randomIntBetween(1, PAGE_SIZE * 3);
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
-        int sliceOffset = randomIntBetween(0, length / 2);
-        int sliceLength = Math.max(1, length - sliceOffset - 1);
-        BytesReference slice = pbr.slice(sliceOffset, sliceLength);
-        assertEquals(pbr.get(sliceOffset), slice.get(0));
-        assertEquals(pbr.get(sliceOffset + sliceLength - 1), slice.get(sliceLength - 1));
-    }
-
-    public void testLength() throws IOException {
-        int[] sizes = {0, randomInt(PAGE_SIZE), PAGE_SIZE, randomInt(PAGE_SIZE * 3)};
-
-        for (int i = 0; i < sizes.length; i++) {
-            BytesReference pbr = getRandomizedPagedBytesReference(sizes[i]);
-            assertEquals(sizes[i], pbr.length());
+    protected BytesReference newBytesReference(int length) throws IOException {
+        // we know bytes stream output always creates a paged bytes reference, we use it to create randomized content
+        ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(length, bigarrays);
+        for (int i = 0; i < length; i++) {
+            out.writeByte((byte) random().nextInt(1 << 8));
         }
-    }
-
-    public void testSlice() throws IOException {
-        int length = randomInt(PAGE_SIZE * 3);
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
-        int sliceOffset = randomIntBetween(0, length / 2);
-        int sliceLength = Math.max(0, length - sliceOffset - 1);
-        BytesReference slice = pbr.slice(sliceOffset, sliceLength);
-        assertEquals(sliceLength, slice.length());
-
-        if (slice.hasArray()) {
-            assertEquals(sliceOffset, slice.arrayOffset());
-        } else {
-            try {
-                slice.arrayOffset();
-                fail("expected IllegalStateException");
-            } catch (IllegalStateException ise) {
-                // expected
-            }
-        }
-    }
-
-    public void testStreamInput() throws IOException {
-        int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20));
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
-        StreamInput si = pbr.streamInput();
-        assertNotNull(si);
-
-        // read single bytes one by one
-        assertEquals(pbr.get(0), si.readByte());
-        assertEquals(pbr.get(1), si.readByte());
-        assertEquals(pbr.get(2), si.readByte());
-
-        // reset the stream for bulk reading
-        si.reset();
-
-        // buffer for bulk reads
-        byte[] origBuf = new byte[length];
-        random().nextBytes(origBuf);
-        byte[] targetBuf = Arrays.copyOf(origBuf, origBuf.length);
-
-        // bulk-read 0 bytes: must not modify buffer
-        si.readBytes(targetBuf, 0, 0);
-        assertEquals(origBuf[0], targetBuf[0]);
-        si.reset();
-
-        // read a few few bytes as ints
-        int bytesToRead = randomIntBetween(1, length / 2);
-        for (int i = 0; i < bytesToRead; i++) {
-            int b = si.read();
-            assertEquals(pbr.get(i), b);
-        }
-        si.reset();
-
-        // bulk-read all
-        si.readFully(targetBuf);
-        assertArrayEquals(pbr.toBytes(), targetBuf);
-
-        // continuing to read should now fail with EOFException
-        try {
-            si.readByte();
-            fail("expected EOF");
-        } catch (EOFException eof) {
-            // yay
-        }
-
-        // try to read more than the stream contains
-        si.reset();
-        try {
-            si.readBytes(targetBuf, 0, length * 2);
-            fail("expected IndexOutOfBoundsException: le > stream.length");
-        } catch (IndexOutOfBoundsException ioob) {
-            // expected
-        }
-    }
-
-    public void testStreamInputBulkReadWithOffset() throws IOException {
-        final int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20));
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
-        StreamInput si = pbr.streamInput();
-        assertNotNull(si);
-
-        // read a bunch of single bytes one by one
-        int offset = randomIntBetween(1, length / 2);
-        for (int i = 0; i < offset; i++) {
-            assertEquals(si.available(), length - i);
-            assertEquals(pbr.get(i), si.readByte());
-        }
-
-        // now do NOT reset the stream - keep the stream's offset!
-
-        // buffer to compare remaining bytes against bulk read
-        byte[] pbrBytesWithOffset = Arrays.copyOfRange(pbr.toBytes(), offset, length);
-        // randomized target buffer to ensure no stale slots
-        byte[] targetBytes = new byte[pbrBytesWithOffset.length];
-        random().nextBytes(targetBytes);
-
-        // bulk-read all
-        si.readFully(targetBytes);
-        assertArrayEquals(pbrBytesWithOffset, targetBytes);
-        assertEquals(si.available(), 0);
-    }
-
-    public void testRandomReads() throws IOException {
-        int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20));
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
-        StreamInput streamInput = pbr.streamInput();
-        BytesRefBuilder target = new BytesRefBuilder();
-        while (target.length() < pbr.length()) {
-            switch (randomIntBetween(0, 10)) {
-                case 6:
-                case 5:
-                    target.append(new BytesRef(new byte[]{streamInput.readByte()}));
-                    break;
-                case 4:
-                case 3:
-                    BytesRef bytesRef = streamInput.readBytesRef(scaledRandomIntBetween(1, pbr.length() - target.length()));
-                    target.append(bytesRef);
-                    break;
-                default:
-                    byte[] buffer = new byte[scaledRandomIntBetween(1, pbr.length() - target.length())];
-                    int offset = scaledRandomIntBetween(0, buffer.length - 1);
-                    int read = streamInput.read(buffer, offset, buffer.length - offset);
-                    target.append(new BytesRef(buffer, offset, read));
-                    break;
-            }
-        }
-        assertEquals(pbr.length(), target.length());
-        BytesRef targetBytes = target.get();
-        assertArrayEquals(pbr.toBytes(), Arrays.copyOfRange(targetBytes.bytes, targetBytes.offset, targetBytes.length));
-    }
-
-    public void testSliceStreamInput() throws IOException {
-        int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20));
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
-
-        // test stream input over slice (upper half of original)
-        int sliceOffset = randomIntBetween(1, length / 2);
-        int sliceLength = length - sliceOffset;
-        BytesReference slice = pbr.slice(sliceOffset, sliceLength);
-        StreamInput sliceInput = slice.streamInput();
-        assertEquals(sliceInput.available(), sliceLength);
-
-        // single reads
-        assertEquals(slice.get(0), sliceInput.readByte());
-        assertEquals(slice.get(1), sliceInput.readByte());
-        assertEquals(slice.get(2), sliceInput.readByte());
-        assertEquals(sliceInput.available(), sliceLength - 3);
-
-        // reset the slice stream for bulk reading
-        sliceInput.reset();
-        assertEquals(sliceInput.available(), sliceLength);
-
-        // bulk read
-        byte[] sliceBytes = new byte[sliceLength];
-        sliceInput.readFully(sliceBytes);
-        assertEquals(sliceInput.available(), 0);
-
-        // compare slice content with upper half of original
-        byte[] pbrSliceBytes = Arrays.copyOfRange(pbr.toBytes(), sliceOffset, length);
-        assertArrayEquals(pbrSliceBytes, sliceBytes);
-
-        // compare slice bytes with bytes read from slice via streamInput :D
-        byte[] sliceToBytes = slice.toBytes();
-        assertEquals(sliceBytes.length, sliceToBytes.length);
-        assertArrayEquals(sliceBytes, sliceToBytes);
-
-        sliceInput.reset();
-        assertEquals(sliceInput.available(), sliceLength);
-        byte[] buffer = new byte[sliceLength + scaledRandomIntBetween(1, 100)];
-        int offset = scaledRandomIntBetween(0, Math.max(1, buffer.length - sliceLength - 1));
-        int read = sliceInput.read(buffer, offset, sliceLength / 2);
-        assertEquals(sliceInput.available(), sliceLength - read);
-        sliceInput.read(buffer, offset + read, sliceLength);
-        assertArrayEquals(sliceBytes, Arrays.copyOfRange(buffer, offset, offset + sliceLength));
-        assertEquals(sliceInput.available(), 0);
-    }
-
-    public void testWriteToOutputStream() throws IOException {
-        int length = randomIntBetween(10, PAGE_SIZE * 4);
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
-        BytesStreamOutput out = new BytesStreamOutput();
-        pbr.writeTo(out);
-        assertEquals(pbr.length(), out.size());
-        assertArrayEquals(pbr.toBytes(), out.bytes().toBytes());
-        out.close();
-    }
-
-    public void testSliceWriteToOutputStream() throws IOException {
-        int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 5));
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
-        int sliceOffset = randomIntBetween(1, length / 2);
-        int sliceLength = length - sliceOffset;
-        BytesReference slice = pbr.slice(sliceOffset, sliceLength);
-        BytesStreamOutput sliceOut = new BytesStreamOutput(sliceLength);
-        slice.writeTo(sliceOut);
-        assertEquals(slice.length(), sliceOut.size());
-        assertArrayEquals(slice.toBytes(), sliceOut.bytes().toBytes());
-        sliceOut.close();
-    }
-
-    public void testToBytes() throws IOException {
-        int[] sizes = {0, randomInt(PAGE_SIZE), PAGE_SIZE, randomIntBetween(2, PAGE_SIZE * randomIntBetween(2, 5))};
-
-        for (int i = 0; i < sizes.length; i++) {
-            BytesReference pbr = getRandomizedPagedBytesReference(sizes[i]);
-            byte[] bytes = pbr.toBytes();
-            assertEquals(sizes[i], bytes.length);
-            // verify that toBytes() is cheap for small payloads
-            if (sizes[i] <= PAGE_SIZE) {
-                assertSame(bytes, pbr.toBytes());
-            } else {
-                assertNotSame(bytes, pbr.toBytes());
-            }
-        }
-    }
-
-    public void testToBytesArraySharedPage() throws IOException {
-        int length = randomIntBetween(10, PAGE_SIZE);
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
-        BytesArray ba = pbr.toBytesArray();
-        BytesArray ba2 = pbr.toBytesArray();
-        assertNotNull(ba);
-        assertNotNull(ba2);
-        assertEquals(pbr.length(), ba.length());
-        assertEquals(ba.length(), ba2.length());
-        // single-page optimization
-        assertSame(ba.array(), ba2.array());
+        assertThat(out.size(), Matchers.equalTo(length));
+        BytesReference ref = out.bytes();
+        assertThat(ref.length(), Matchers.equalTo(length));
+        assertThat(ref, Matchers.instanceOf(PagedBytesReference.class));
+        return ref;
     }
 
     public void testToBytesArrayMaterializedPages() throws IOException {
@@ -295,7 +56,7 @@ public class PagedBytesReferenceTests extends ESTestCase {
         while ((length % PAGE_SIZE) == 0) {
             length = randomIntBetween(PAGE_SIZE, PAGE_SIZE * randomIntBetween(2, 5));
         }
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
+        BytesReference pbr = newBytesReference(length);
         BytesArray ba = pbr.toBytesArray();
         BytesArray ba2 = pbr.toBytesArray();
         assertNotNull(ba);
@@ -306,99 +67,11 @@ public class PagedBytesReferenceTests extends ESTestCase {
         assertNotSame(ba.array(), ba2.array());
     }
 
-    public void testCopyBytesArray() throws IOException {
-        // small PBR which would normally share the first page
-        int length = randomIntBetween(10, PAGE_SIZE);
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
-        BytesArray ba = pbr.copyBytesArray();
-        BytesArray ba2 = pbr.copyBytesArray();
-        assertNotNull(ba);
-        assertNotSame(ba, ba2);
-        assertNotSame(ba.array(), ba2.array());
-    }
-
-    public void testSliceCopyBytesArray() throws IOException {
-        int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8));
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
-        int sliceOffset = randomIntBetween(0, pbr.length());
-        int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset);
-        BytesReference slice = pbr.slice(sliceOffset, sliceLength);
-
-        BytesArray ba1 = slice.copyBytesArray();
-        BytesArray ba2 = slice.copyBytesArray();
-        assertNotNull(ba1);
-        assertNotNull(ba2);
-        assertNotSame(ba1.array(), ba2.array());
-        assertArrayEquals(slice.toBytes(), ba1.array());
-        assertArrayEquals(slice.toBytes(), ba2.array());
-        assertArrayEquals(ba1.array(), ba2.array());
-    }
-
-    public void testEmptyToBytesRefIterator() throws IOException {
-        BytesReference pbr = getRandomizedPagedBytesReference(0);
-        assertNull(pbr.iterator().next());
-    }
-
-    public void testIterator() throws IOException {
-        int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8));
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
-        BytesRefIterator iterator = pbr.iterator();
-        BytesRef ref;
-        BytesRefBuilder builder = new BytesRefBuilder();
-        while((ref = iterator.next()) != null) {
-            builder.append(ref);
-        }
-        assertArrayEquals(pbr.toBytes(), BytesRef.deepCopyOf(builder.toBytesRef()).bytes);
-    }
-
-    public void testSliceIterator() throws IOException {
-        int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8));
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
-        int sliceOffset = randomIntBetween(0, pbr.length());
-        int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset);
-        BytesReference slice = pbr.slice(sliceOffset, sliceLength);
-        BytesRefIterator iterator = slice.iterator();
-        BytesRef ref = null;
-        BytesRefBuilder builder = new BytesRefBuilder();
-        while((ref = iterator.next()) != null) {
-            builder.append(ref);
-        }
-        assertArrayEquals(slice.toBytes(), BytesRef.deepCopyOf(builder.toBytesRef()).bytes);
-    }
-
-    public void testIteratorRandom() throws IOException {
-        int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8));
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
-        if (randomBoolean()) {
-            int sliceOffset = randomIntBetween(0, pbr.length());
-            int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset);
-            pbr = pbr.slice(sliceOffset, sliceLength);
-        }
-
-        if (randomBoolean()) {
-            pbr = pbr.toBytesArray();
-        }
-        BytesRefIterator iterator = pbr.iterator();
-        BytesRef ref = null;
-        BytesRefBuilder builder = new BytesRefBuilder();
-        while((ref = iterator.next()) != null) {
-            builder.append(ref);
-        }
-        assertArrayEquals(pbr.toBytes(), BytesRef.deepCopyOf(builder.toBytesRef()).bytes);
-    }
-
-    public void testHasArray() throws IOException {
-        int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(1, 3));
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
-        // must return true for <= pagesize
-        assertEquals(length <= PAGE_SIZE, pbr.hasArray());
-    }
-
     public void testArray() throws IOException {
         int[] sizes = {0, randomInt(PAGE_SIZE), PAGE_SIZE, randomIntBetween(2, PAGE_SIZE * randomIntBetween(2, 5))};
 
         for (int i = 0; i < sizes.length; i++) {
-            BytesReference pbr = getRandomizedPagedBytesReference(sizes[i]);
+            BytesReference pbr = newBytesReference(sizes[i]);
             // verify that array() is cheap for small payloads
             if (sizes[i] <= PAGE_SIZE) {
                 byte[] array = pbr.array();
@@ -416,149 +89,27 @@ public class PagedBytesReferenceTests extends ESTestCase {
         }
     }
 
-    public void testArrayOffset() throws IOException {
-        int length = randomInt(PAGE_SIZE * randomIntBetween(2, 5));
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
-        if (pbr.hasArray()) {
-            assertEquals(0, pbr.arrayOffset());
-        } else {
-            try {
-                pbr.arrayOffset();
-                fail("expected IllegalStateException");
-            } catch (IllegalStateException ise) {
-                // expected
+    public void testToBytes() throws IOException {
+        int[] sizes = {0, randomInt(PAGE_SIZE), PAGE_SIZE, randomIntBetween(2, PAGE_SIZE * randomIntBetween(2, 5))};
+
+        for (int i = 0; i < sizes.length; i++) {
+            BytesReference pbr = newBytesReference(sizes[i]);
+            byte[] bytes = pbr.toBytes();
+            assertEquals(sizes[i], bytes.length);
+            // verify that toBytes() is cheap for small payloads
+            if (sizes[i] <= PAGE_SIZE) {
+                assertSame(bytes, pbr.toBytes());
+            } else {
+                assertNotSame(bytes, pbr.toBytes());
             }
         }
     }
 
-    public void testSliceArrayOffset() throws IOException {
-        int length = randomInt(PAGE_SIZE * randomIntBetween(2, 5));
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
-        int sliceOffset = randomIntBetween(0, pbr.length());
-        int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset);
-        BytesReference slice = pbr.slice(sliceOffset, sliceLength);
-        if (slice.hasArray()) {
-            assertEquals(sliceOffset, slice.arrayOffset());
-        } else {
-            try {
-                slice.arrayOffset();
-                fail("expected IllegalStateException");
-            } catch (IllegalStateException ise) {
-                // expected
-            }
-        }
-    }
-
-    public void testToUtf8() throws IOException {
-        // test empty
-        BytesReference pbr = getRandomizedPagedBytesReference(0);
-        assertEquals("", pbr.toUtf8());
-        // TODO: good way to test?
-    }
-
-    public void testToBytesRef() throws IOException {
-        int length = randomIntBetween(0, PAGE_SIZE);
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
-        BytesRef ref = pbr.toBytesRef();
-        assertNotNull(ref);
-        assertEquals(pbr.arrayOffset(), ref.offset);
-        assertEquals(pbr.length(), ref.length);
-    }
-
-    public void testSliceToBytesRef() throws IOException {
-        int length = randomIntBetween(0, PAGE_SIZE);
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
-        // get a BytesRef from a slice
-        int sliceOffset = randomIntBetween(0, pbr.length());
-        int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset);
-        BytesRef sliceRef = pbr.slice(sliceOffset, sliceLength).toBytesRef();
-        // note that these are only true if we have <= than a page, otherwise offset/length are shifted
-        assertEquals(sliceOffset, sliceRef.offset);
-        assertEquals(sliceLength, sliceRef.length);
-    }
-
-    public void testCopyBytesRef() throws IOException {
-        int length = randomIntBetween(0, PAGE_SIZE * randomIntBetween(2, 5));
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
-        BytesRef ref = pbr.copyBytesRef();
-        assertNotNull(ref);
-        assertEquals(pbr.length(), ref.length);
-    }
-
-    public void testHashCode() throws IOException {
-        // empty content must have hash 1 (JDK compat)
-        BytesReference pbr = getRandomizedPagedBytesReference(0);
-        assertEquals(Arrays.hashCode(BytesRef.EMPTY_BYTES), pbr.hashCode());
-
-        // test with content
-        pbr = getRandomizedPagedBytesReference(randomIntBetween(0, PAGE_SIZE * randomIntBetween(2, 5)));
-        int jdkHash = Arrays.hashCode(pbr.toBytes());
-        int pbrHash = pbr.hashCode();
-        assertEquals(jdkHash, pbrHash);
-
-        // test hashes of slices
-        int sliceFrom = randomIntBetween(0, pbr.length());
-        int sliceLength = randomIntBetween(pbr.length() - sliceFrom, pbr.length() - sliceFrom);
-        BytesReference slice = pbr.slice(sliceFrom, sliceLength);
-        int sliceJdkHash = Arrays.hashCode(slice.toBytes());
-        int sliceHash = slice.hashCode();
-        assertEquals(sliceJdkHash, sliceHash);
-    }
-
-    public void testEquals() {
-        int length = randomIntBetween(100, PAGE_SIZE * randomIntBetween(2, 5));
-        ByteArray ba1 = bigarrays.newByteArray(length, false);
-        ByteArray ba2 = bigarrays.newByteArray(length, false);
-
-        // copy contents
-        for (long i = 0; i < length; i++) {
-            ba2.set(i, ba1.get(i));
-        }
-
-        // get refs & compare
-        BytesReference pbr = new PagedBytesReference(bigarrays, ba1, length);
-        BytesReference pbr2 = new PagedBytesReference(bigarrays, ba2, length);
-        assertEquals(pbr, pbr2);
-    }
-
-    public void testEqualsPeerClass() throws IOException {
-        int length = randomIntBetween(100, PAGE_SIZE * randomIntBetween(2, 5));
-        BytesReference pbr = getRandomizedPagedBytesReference(length);
-        BytesReference ba = new BytesArray(pbr.toBytes());
-        assertEquals(pbr, ba);
-    }
-
-    public void testSliceEquals() {
-        int length = randomIntBetween(100, PAGE_SIZE * randomIntBetween(2, 5));
-        ByteArray ba1 = bigarrays.newByteArray(length, false);
-        BytesReference pbr = new PagedBytesReference(bigarrays, ba1, length);
-
-        // test equality of slices
-        int sliceFrom = randomIntBetween(0, pbr.length());
-        int sliceLength = randomIntBetween(pbr.length() - sliceFrom, pbr.length() - sliceFrom);
-        BytesReference slice1 = pbr.slice(sliceFrom, sliceLength);
-        BytesReference slice2 = pbr.slice(sliceFrom, sliceLength);
-        assertArrayEquals(slice1.toBytes(), slice2.toBytes());
-
-        // test a slice with same offset but different length,
-        // unless randomized testing gave us a 0-length slice.
-        if (sliceLength > 0) {
-            BytesReference slice3 = pbr.slice(sliceFrom, sliceLength / 2);
-            assertFalse(Arrays.equals(slice1.toBytes(), slice3.toBytes()));
-        }
-    }
-
-    private BytesReference getRandomizedPagedBytesReference(int length) throws IOException {
-        // we know bytes stream output always creates a paged bytes reference, we use it to create randomized content
-        ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(length, bigarrays);
-        for (int i = 0; i < length; i++) {
-            out.writeByte((byte) random().nextInt(1 << 8));
-        }
-        assertThat(out.size(), Matchers.equalTo(length));
-        BytesReference ref = out.bytes();
-        assertThat(ref.length(), Matchers.equalTo(length));
-        assertThat(ref, Matchers.instanceOf(PagedBytesReference.class));
-        return ref;
+    public void testHasArray() throws IOException {
+        int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(1, 3));
+        BytesReference pbr = newBytesReference(length);
+        // must return true for <= pagesize
+        assertEquals(length <= PAGE_SIZE, pbr.hasArray());
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/common/netty/ChannelBufferBytesReferenceTests.java b/core/src/test/java/org/elasticsearch/common/netty/ChannelBufferBytesReferenceTests.java
new file mode 100644
index 00000000000..76a8626fee5
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/netty/ChannelBufferBytesReferenceTests.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.common.netty;
+
+import org.elasticsearch.common.bytes.AbstractBytesReferenceTestCase;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.buffer.ChannelBuffers;
+
+import java.io.IOException;
+
+public class ChannelBufferBytesReferenceTests extends AbstractBytesReferenceTestCase {
+    @Override
+    protected BytesReference newBytesReference(int length) throws IOException {
+        ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(length, bigarrays);
+        for (int i = 0; i < length; i++) {
+            out.writeByte((byte) random().nextInt(1 << 8));
+        }
+        assertEquals(out.size(), length);
+        BytesReference ref = out.bytes();
+        assertEquals(ref.length(), length);
+        BytesArray bytesArray = ref.toBytesArray();
+        return NettyUtils.toBytesReference(ChannelBuffers.wrappedBuffer(bytesArray.array(), bytesArray.arrayOffset(),
+            bytesArray.length()));
+    }
+
+    public void testSliceOnAdvancedBuffer() throws IOException {
+        BytesReference bytesReference = newBytesReference(randomIntBetween(10, 3 * PAGE_SIZE));
+        BytesArray bytesArray = bytesReference.toBytesArray();
+
+        ChannelBuffer channelBuffer = ChannelBuffers.wrappedBuffer(bytesArray.array(), bytesArray.arrayOffset(),
+            bytesArray.length());
+        int numBytesToRead = randomIntBetween(1, 5);
+        for (int i = 0; i < numBytesToRead; i++) {
+            channelBuffer.readByte();
+        }
+        BytesReference other = NettyUtils.toBytesReference(channelBuffer);
+        BytesReference slice = bytesReference.slice(numBytesToRead, bytesReference.length() - numBytesToRead);
+        assertEquals(other, slice);
+
+        assertEquals(other.slice(3, 1), slice.slice(3, 1));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/common/netty/NettyUtilsTests.java b/core/src/test/java/org/elasticsearch/common/netty/NettyUtilsTests.java
index 89c282e7d81..2d981dc9eae 100644
--- a/core/src/test/java/org/elasticsearch/common/netty/NettyUtilsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/netty/NettyUtilsTests.java
@@ -73,7 +73,6 @@ public class NettyUtilsTests extends ESTestCase {
     }
 
     private BytesReference getRandomizedBytesReference(int length) throws IOException {
-        // TODO we should factor out a BaseBytesReferenceTestCase
         // we know bytes stream output always creates a paged bytes reference, we use it to create randomized content
         ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(length, bigarrays);
         for (int i = 0; i < length; i++) {

From 6d2df0dc183a3e7dc1b9aeb856fd6ab9b94d2b54 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Wed, 29 Jun 2016 15:25:51 +0200
Subject: [PATCH 1112/1311] Fix docs example for the _id field, the field is
 not accessible in scripts

---
 docs/reference/mapping/fields/id-field.asciidoc | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/docs/reference/mapping/fields/id-field.asciidoc b/docs/reference/mapping/fields/id-field.asciidoc
index f99f1ec9723..c640b561571 100644
--- a/docs/reference/mapping/fields/id-field.asciidoc
+++ b/docs/reference/mapping/fields/id-field.asciidoc
@@ -7,8 +7,8 @@ indexed as its value can be derived automatically from the
 <<mapping-uid-field,`_uid`>> field.
 
 The value of the `_id` field is accessible in certain queries (`term`,
-`terms`, `match`, `query_string`, `simple_query_string`) and scripts, but
-_not_ in aggregations or when sorting, where the <<mapping-uid-field,`_uid`>>
+`terms`, `match`, `query_string`, `simple_query_string`), but
+_not_ in aggregations, scripts or when sorting, where the <<mapping-uid-field,`_uid`>>
 field should be used instead:
 
 [source,js]
@@ -30,18 +30,9 @@ GET my_index/_search
     "terms": {
       "_id": [ "1", "2" ] <1>
     }
-  },
-  "script_fields": {
-    "UID": {
-      "script": {
-         "lang": "painless",
-         "inline": "doc['_id']" <2>
-      }
-    }
   }
 }
 --------------------------
 // CONSOLE
 
 <1> Querying on the `_id` field (also see the <<query-dsl-ids-query,`ids` query>>)
-<2> Accessing the `_id` field in scripts

From 6d5666553c34761fe4cf275974eac4dd1cf63889 Mon Sep 17 00:00:00 2001
From: Britta Weber <britta.weber@elasticsearch.com>
Date: Wed, 29 Jun 2016 15:53:57 +0200
Subject: [PATCH 1113/1311] [TEST] mute test because it fails about 1/100 runs

---
 .../test/java/org/elasticsearch/common/lucene/LuceneTests.java   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java b/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java
index e9f07acb9d1..f4260626160 100644
--- a/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java
+++ b/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java
@@ -357,6 +357,7 @@ public class LuceneTests extends ESTestCase {
         dir.close();
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/19151")
     public void testAsSequentialAccessBits() throws Exception {
         Directory dir = newDirectory();
         IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new KeywordAnalyzer()));

From 56fa751928c2d9f89063573de5fc1b97c471a669 Mon Sep 17 00:00:00 2001
From: Alexander Reelsen <alexander@reelsen.net>
Date: Wed, 29 Jun 2016 16:44:12 +0200
Subject: [PATCH 1114/1311] Plugins: Add status bar on download (#18695)

As some plugins are becoming big now, it is hard for the user to know, if the plugin
is being downloaded or just nothing happens.

This commit adds a progress bar during download, which can be disabled by using the `-q`
parameter.

In addition this updates to jimfs 1.1, which allows us to test the batch mode, as adding
security policies are now supported due to having jimfs:// protocol support in URL stream
handlers.
---
 .../java/org/elasticsearch/cli/Terminal.java  |  11 +-
 .../plugins/InstallPluginCommand.java         |  80 ++++++++----
 .../org/elasticsearch/plugins/PluginCli.java  |   2 -
 .../plugins/ProgressInputStream.java          |  83 +++++++++++++
 .../plugins/ProgressInputStreamTests.java     | 116 ++++++++++++++++++
 docs/plugins/plugin-script.asciidoc           |   6 +-
 qa/evil-tests/build.gradle                    |   2 +-
 .../plugins/InstallPluginCommandTests.java    |  46 ++++++-
 8 files changed, 315 insertions(+), 31 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/plugins/ProgressInputStream.java
 create mode 100644 core/src/test/java/org/elasticsearch/plugins/ProgressInputStreamTests.java

diff --git a/core/src/main/java/org/elasticsearch/cli/Terminal.java b/core/src/main/java/org/elasticsearch/cli/Terminal.java
index d2dc57263dc..58eb5012d07 100644
--- a/core/src/main/java/org/elasticsearch/cli/Terminal.java
+++ b/core/src/main/java/org/elasticsearch/cli/Terminal.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.cli;
 
+import org.elasticsearch.common.SuppressForbidden;
+
 import java.io.BufferedReader;
 import java.io.Console;
 import java.io.IOException;
@@ -26,8 +28,6 @@ import java.io.InputStreamReader;
 import java.io.PrintWriter;
 import java.nio.charset.Charset;
 
-import org.elasticsearch.common.SuppressForbidden;
-
 /**
  * A Terminal wraps access to reading input and writing output for a cli.
  *
@@ -81,8 +81,13 @@ public abstract class Terminal {
 
     /** Prints a line to the terminal at {@code verbosity} level. */
     public final void println(Verbosity verbosity, String msg) {
+        print(verbosity, msg + lineSeparator);
+    }
+
+    /** Prints message to the terminal at {@code verbosity} level, without a newline. */
+    public final void print(Verbosity verbosity, String msg) {
         if (this.verbosity.ordinal() >= verbosity.ordinal()) {
-            getWriter().print(msg + lineSeparator);
+            getWriter().print(msg);
             getWriter().flush();
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
index 51bb3b6b82f..e9ea2d11e37 100644
--- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
@@ -19,12 +19,31 @@
 
 package org.elasticsearch.plugins;
 
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import org.apache.lucene.search.spell.LevensteinDistance;
+import org.apache.lucene.util.CollectionUtil;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.Version;
+import org.elasticsearch.bootstrap.JarHell;
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.SettingCommand;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserError;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.hash.MessageDigests;
+import org.elasticsearch.common.io.FileSystemUtils;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.internal.InternalSettingsPreparer;
+
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.net.URL;
+import java.net.URLConnection;
 import java.net.URLDecoder;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.DirectoryStream;
@@ -49,24 +68,6 @@ import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
-import joptsimple.OptionSet;
-import joptsimple.OptionSpec;
-import org.apache.lucene.search.spell.LevensteinDistance;
-import org.apache.lucene.util.CollectionUtil;
-import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.Version;
-import org.elasticsearch.bootstrap.JarHell;
-import org.elasticsearch.cli.ExitCodes;
-import org.elasticsearch.cli.SettingCommand;
-import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.cli.UserError;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.hash.MessageDigests;
-import org.elasticsearch.common.io.FileSystemUtils;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.node.internal.InternalSettingsPreparer;
-
 import static org.elasticsearch.cli.Terminal.Verbosity.VERBOSE;
 
 /**
@@ -107,7 +108,7 @@ class InstallPluginCommand extends SettingCommand {
     static final Set<String> MODULES;
     static {
         try (InputStream stream = InstallPluginCommand.class.getResourceAsStream("/modules.txt");
-             BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
+            BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
             Set<String> modules = new HashSet<>();
             String line = reader.readLine();
             while (line != null) {
@@ -124,7 +125,7 @@ class InstallPluginCommand extends SettingCommand {
     static final Set<String> OFFICIAL_PLUGINS;
     static {
         try (InputStream stream = InstallPluginCommand.class.getResourceAsStream("/plugins.txt");
-             BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
+            BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
             Set<String> plugins = new TreeSet<>(); // use tree set to get sorting for help command
             String line = reader.readLine();
             while (line != null) {
@@ -141,6 +142,7 @@ class InstallPluginCommand extends SettingCommand {
     private final OptionSpec<Void> batchOption;
     private final OptionSpec<String> arguments;
 
+
     public static final Set<PosixFilePermission> DIR_AND_EXECUTABLE_PERMS;
     public static final Set<PosixFilePermission> FILE_PERMS;
 
@@ -273,13 +275,49 @@ class InstallPluginCommand extends SettingCommand {
         terminal.println(VERBOSE, "Retrieving zip from " + urlString);
         URL url = new URL(urlString);
         Path zip = Files.createTempFile(tmpDir, null, ".zip");
-        try (InputStream in = url.openStream()) {
+        URLConnection urlConnection = url.openConnection();
+        int contentLength = urlConnection.getContentLength();
+        try (InputStream in = new TerminalProgressInputStream(urlConnection.getInputStream(), contentLength, terminal)) {
             // must overwrite since creating the temp file above actually created the file
             Files.copy(in, zip, StandardCopyOption.REPLACE_EXISTING);
         }
         return zip;
     }
 
+    /**
+     * content length might be -1 for unknown and progress only makes sense if the content length is greater than 0
+     */
+    private class TerminalProgressInputStream extends ProgressInputStream {
+
+        private final Terminal terminal;
+        private int width = 50;
+        private final boolean enabled;
+
+        public TerminalProgressInputStream(InputStream is, int expectedTotalSize, Terminal terminal) {
+            super(is, expectedTotalSize);
+            this.terminal = terminal;
+            this.enabled = expectedTotalSize > 0;
+        }
+
+        @Override
+        public void onProgress(int percent) {
+            if (enabled) {
+                int currentPosition = percent * width / 100;
+                StringBuilder sb = new StringBuilder("\r[");
+                sb.append(String.join("=", Collections.nCopies(currentPosition, "")));
+                if (currentPosition > 0 && percent < 100) {
+                    sb.append(">");
+                }
+                sb.append(String.join(" ", Collections.nCopies(width - currentPosition, "")));
+                sb.append("] %s   ");
+                if (percent == 100) {
+                    sb.append("\n");
+                }
+                terminal.print(Terminal.Verbosity.NORMAL, String.format(Locale.ROOT, sb.toString(), percent + "%"));
+            }
+        }
+    }
+
     /** Downloads a zip from the url, as well as a SHA1 checksum, and checks the checksum. */
     private Path downloadZipAndChecksum(Terminal terminal, String urlString, Path tmpDir) throws Exception {
         Path zip = downloadZip(terminal, urlString, tmpDir);
diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginCli.java b/core/src/main/java/org/elasticsearch/plugins/PluginCli.java
index 3a88c4d0083..3ce60882cce 100644
--- a/core/src/main/java/org/elasticsearch/plugins/PluginCli.java
+++ b/core/src/main/java/org/elasticsearch/plugins/PluginCli.java
@@ -26,8 +26,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
 
-import java.util.Collections;
-
 /**
  * A cli tool for adding, removing and listing plugins for elasticsearch.
  */
diff --git a/core/src/main/java/org/elasticsearch/plugins/ProgressInputStream.java b/core/src/main/java/org/elasticsearch/plugins/ProgressInputStream.java
new file mode 100644
index 00000000000..16e1f203bb3
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/plugins/ProgressInputStream.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * An input stream that allows to add a listener to monitor progress
+ * The listener is triggered whenever a full percent is increased
+ * The listener is never triggered twice on the same percentage
+ * The listener will always return 99 percent, if the expectedTotalSize is exceeded, until it is finished
+ *
+ * Only used by the InstallPluginCommand, thus package private here
+ */
+abstract class ProgressInputStream extends FilterInputStream {
+
+    private final int expectedTotalSize;
+    private int currentPercent;
+    private int count = 0;
+
+    public ProgressInputStream(InputStream is, int expectedTotalSize) {
+        super(is);
+        this.expectedTotalSize = expectedTotalSize;
+        this.currentPercent = 0;
+    }
+
+    @Override
+    public int read() throws IOException {
+        int read = in.read();
+        checkProgress(read == -1 ? -1 : 1);
+        return read;
+    }
+
+    @Override
+    public int read(byte[] b, int off, int len) throws IOException {
+        int byteCount = super.read(b, off, len);
+        checkProgress(byteCount);
+        return byteCount;
+    }
+
+    @Override
+    public int read(byte b[]) throws IOException {
+        return read(b, 0, b.length);
+    }
+
+    void checkProgress(int byteCount) {
+        // are we done?
+        if (byteCount == -1) {
+            currentPercent = 100;
+            onProgress(currentPercent);
+        } else {
+            count += byteCount;
+            // rounding up to 100% would mean we say we are done, before we are...
+            // this also catches issues, when expectedTotalSize was guessed wrong
+            int percent = Math.min(99, (int) Math.floor(100.0*count/expectedTotalSize));
+            if (percent > currentPercent) {
+                currentPercent = percent;
+                onProgress(percent);
+            }
+        }
+    }
+
+    public void onProgress(int percent) {}
+}
diff --git a/core/src/test/java/org/elasticsearch/plugins/ProgressInputStreamTests.java b/core/src/test/java/org/elasticsearch/plugins/ProgressInputStreamTests.java
new file mode 100644
index 00000000000..81e937d26a9
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/plugins/ProgressInputStreamTests.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugins;
+
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasItems;
+import static org.hamcrest.Matchers.hasSize;
+
+public class ProgressInputStreamTests extends ESTestCase {
+
+    private List<Integer> progresses = new ArrayList<>();
+
+    public void testThatProgressListenerIsCalled() throws Exception {
+        ProgressInputStream is = newProgressInputStream(0);
+        is.checkProgress(-1);
+
+        assertThat(progresses, hasSize(1));
+        assertThat(progresses, hasItems(100));
+    }
+
+    public void testThatProgressListenerIsCalledOnUnexpectedCompletion() throws Exception {
+        ProgressInputStream is = newProgressInputStream(2);
+        is.checkProgress(-1);
+        assertThat(progresses, hasItems(100));
+    }
+
+    public void testThatProgressListenerReturnsMaxValueOnWrongExpectedSize() throws Exception {
+        ProgressInputStream is = newProgressInputStream(2);
+
+        is.checkProgress(1);
+        assertThat(progresses, hasItems(50));
+
+        is.checkProgress(3);
+        assertThat(progresses, hasItems(50, 99));
+
+        is.checkProgress(-1);
+        assertThat(progresses, hasItems(50, 99, 100));
+    }
+
+    public void testOneByte() throws Exception {
+        ProgressInputStream is = newProgressInputStream(1);
+        is.checkProgress(1);
+        is.checkProgress(-1);
+
+        assertThat(progresses, hasItems(99, 100));
+
+    }
+
+    public void testOddBytes() throws Exception {
+        int odd = (randomIntBetween(100, 200) / 2) + 1;
+        ProgressInputStream is = newProgressInputStream(odd);
+        for (int i = 0; i < odd; i++) {
+            is.checkProgress(1);
+        }
+        is.checkProgress(-1);
+
+        assertThat(progresses, hasSize(odd+1));
+        assertThat(progresses, hasItem(100));
+    }
+
+    public void testEvenBytes() throws Exception {
+        int even = (randomIntBetween(100, 200) / 2);
+        ProgressInputStream is = newProgressInputStream(even);
+
+        for (int i = 0; i < even; i++) {
+            is.checkProgress(1);
+        }
+        is.checkProgress(-1);
+
+        assertThat(progresses, hasSize(even+1));
+        assertThat(progresses, hasItem(100));
+    }
+
+    public void testOnProgressCannotBeCalledMoreThanOncePerPercent() throws Exception {
+        int count = randomIntBetween(150, 300);
+        ProgressInputStream is = newProgressInputStream(count);
+
+        for (int i = 0; i < count; i++) {
+            is.checkProgress(1);
+        }
+        is.checkProgress(-1);
+
+        assertThat(progresses, hasSize(100));
+    }
+
+    private ProgressInputStream newProgressInputStream(int expectedSize) {
+        return new ProgressInputStream(null, expectedSize) {
+            @Override
+            public void onProgress(int percent) {
+                progresses.add(percent);
+            }
+        };
+    }
+}
\ No newline at end of file
diff --git a/docs/plugins/plugin-script.asciidoc b/docs/plugins/plugin-script.asciidoc
index 1e21288e39c..987cc7c9758 100644
--- a/docs/plugins/plugin-script.asciidoc
+++ b/docs/plugins/plugin-script.asciidoc
@@ -51,7 +51,7 @@ sudo bin/elasticsearch-plugin install analysis-icu
 -----------------------------------
 
 This command will install the version of the plugin that matches your
-Elasticsearch version.
+Elasticsearch version and also show a progress bar while downloading.
 
 [float]
 === Custom URL or file system
@@ -117,8 +117,8 @@ The `plugin` scripts supports a number of other command line parameters:
 === Silent/Verbose mode
 
 The `--verbose` parameter outputs more debug information, while the `--silent`
-parameter turns off all output.  The script may return the following exit
-codes:
+parameter turns off all output including the progress bar. The script may
+return the following exit codes:
 
 [horizontal]
 `0`:: everything was OK
diff --git a/qa/evil-tests/build.gradle b/qa/evil-tests/build.gradle
index 53406f1aad9..cba9334fbca 100644
--- a/qa/evil-tests/build.gradle
+++ b/qa/evil-tests/build.gradle
@@ -26,7 +26,7 @@
 apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {
-  testCompile 'com.google.jimfs:jimfs:1.0'
+  testCompile 'com.google.jimfs:jimfs:1.1'
 }
 
 // TODO: give each evil test its own fresh JVM for more isolation.
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
index 2b9b6ec6ab9..e5117fa0aa0 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
@@ -25,6 +25,7 @@ import com.google.common.jimfs.Jimfs;
 import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.Version;
 import org.elasticsearch.cli.MockTerminal;
+import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cli.UserError;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.collect.Tuple;
@@ -70,6 +71,7 @@ import java.util.zip.ZipOutputStream;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.not;
 
 @LuceneTestCase.SuppressFileSystems("*")
 public class InstallPluginCommandTests extends ESTestCase {
@@ -179,6 +181,10 @@ public class InstallPluginCommandTests extends ESTestCase {
 
     /** creates a plugin .zip and returns the url for testing */
     static String createPlugin(String name, Path structure) throws IOException {
+        return createPlugin(name, structure, false);
+    }
+
+    static String createPlugin(String name, Path structure, boolean createSecurityPolicyFile) throws IOException {
         PluginTestUtil.writeProperties(structure,
             "description", "fake desc",
             "name", name,
@@ -186,6 +192,10 @@ public class InstallPluginCommandTests extends ESTestCase {
             "elasticsearch.version", Version.CURRENT.toString(),
             "java.version", System.getProperty("java.specification.version"),
             "classname", "FakePlugin");
+        if (createSecurityPolicyFile) {
+            String securityPolicyContent = "grant {\n  permission java.lang.RuntimePermission \"setFactory\";\n};\n";
+            Files.write(structure.resolve("plugin-security.policy"), securityPolicyContent.getBytes(StandardCharsets.UTF_8));
+        }
         writeJar(structure.resolve("plugin.jar"), "FakePlugin");
         return writeZip(structure, "elasticsearch");
     }
@@ -583,7 +593,41 @@ public class InstallPluginCommandTests extends ESTestCase {
         assertThat(e.getMessage(), containsString("Unknown plugin unknown_plugin"));
     }
 
-    // TODO: test batch flag?
+    public void testBatchFlag() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        installPlugin(terminal, true);
+        assertThat(terminal.getOutput(), containsString("WARNING: plugin requires additional permissions"));
+    }
+
+    public void testQuietFlagDisabled() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        terminal.setVerbosity(randomFrom(Terminal.Verbosity.NORMAL, Terminal.Verbosity.VERBOSE));
+        installPlugin(terminal, false);
+        assertThat(terminal.getOutput(), containsString("100%"));
+    }
+
+    public void testQuietFlagEnabled() throws Exception {
+        MockTerminal terminal = new MockTerminal();
+        terminal.setVerbosity(Terminal.Verbosity.SILENT);
+        installPlugin(terminal, false);
+        assertThat(terminal.getOutput(), not(containsString("100%")));
+    }
+
+    private void installPlugin(MockTerminal terminal, boolean isBatch) throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        // if batch is enabled, we also want to add a security policy
+        String pluginZip = createPlugin("fake", pluginDir, isBatch);
+
+        Map<String, String> settings = new HashMap<>();
+        settings.put("path.home", env.v1().toString());
+        new InstallPluginCommand() {
+            @Override
+            void jarHellCheck(Path candidate, Path pluginsDir) throws Exception {
+            }
+        }.execute(terminal, pluginZip, isBatch, settings);
+    }
+
     // TODO: test checksum (need maven/official below)
     // TODO: test maven, official, and staging install...need tests with fixtures...
 }

From 4f82b2de1a63793c775f6180f2234d39000c0949 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Wed, 29 Jun 2016 16:57:57 +0200
Subject: [PATCH 1115/1311] Fixed bad asciidoc in azure discovery

---
 docs/plugins/discovery-azure.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/plugins/discovery-azure.asciidoc b/docs/plugins/discovery-azure.asciidoc
index 825c258133b..f49f1568cab 100644
--- a/docs/plugins/discovery-azure.asciidoc
+++ b/docs/plugins/discovery-azure.asciidoc
@@ -56,7 +56,7 @@ discovery:
 .Binding the network host
 ==============================================
 
-WARNING: The keystore file must be placed in a directory accessible by elasticsearch like the `config` directory.
+The keystore file must be placed in a directory accessible by elasticsearch like the `config` directory.
 
 It's important to define `network.host` as by default it's bound to `localhost`.
 

From ef89e564f45d755c4b08fb030413b8dc28fff155 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 29 Jun 2016 11:04:54 -0400
Subject: [PATCH 1116/1311] Update Vagrant boxes before running packaging test

This commit adds an execution of a Vagrant box update task before
bringing a Vagrant box up for running packaging tests.

Relates #19155
---
 qa/vagrant/build.gradle | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle
index 784655edaa3..89668a06534 100644
--- a/qa/vagrant/build.gradle
+++ b/qa/vagrant/build.gradle
@@ -223,6 +223,12 @@ for (String box : availableBoxes) {
     continue;
   }
 
+  Task update = tasks.create("vagrant${boxTask}#update", VagrantCommandTask) {
+    boxName box
+    args 'box', 'update', box
+    dependsOn checkVagrantVersion
+  }
+
   Task up = tasks.create("vagrant${boxTask}#up", VagrantCommandTask) {
     boxName box
     /* Its important that we try to reprovision the box even if it already
@@ -238,7 +244,7 @@ for (String box : availableBoxes) {
     args 'up', box, '--provision', '--provider', 'virtualbox'
     /* It'd be possible to check if the box is already up here and output
       SKIPPED but that would require running vagrant status which is slow! */
-    dependsOn checkVagrantVersion
+    dependsOn update
   }
 
   Task smoke = tasks.create("vagrant${boxTask}#smoketest", Exec) {

From 83d7f199c76195bca371ce356e4171722e95bd5e Mon Sep 17 00:00:00 2001
From: Paul Echeverri <pecheverri@elastic.co>
Date: Wed, 22 Jun 2016 18:48:41 -0700
Subject: [PATCH 1117/1311] Partial draft for Java Update-by-Query

---
 docs/java-api/docs/update-by-query.asciidoc | 522 ++++++++++++++++++++
 1 file changed, 522 insertions(+)
 create mode 100644 docs/java-api/docs/update-by-query.asciidoc

diff --git a/docs/java-api/docs/update-by-query.asciidoc b/docs/java-api/docs/update-by-query.asciidoc
new file mode 100644
index 00000000000..eff4be0182d
--- /dev/null
+++ b/docs/java-api/docs/update-by-query.asciidoc
@@ -0,0 +1,522 @@
+[[docs-update-by-query]]
+== Update By Query API
+
+experimental[The update-by-query API is new and should still be considered experimental.  The API may change in ways that are not backwards compatible]
+
+The simplest usage of `updateByQuery` updates each
+document in an index without changing the source. This usage enables 
+<<picking-up-a-new-property,picking up a new property>> or another online
+mapping change.
+
+[source,java]
+--------------------------------------------------
+UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
+
+updateByQuery.source("cool_things")
+        .filter(termQuery("level", "awesome"))
+        .script(new Script("ctx._source.awesome = \"absolutely\""));
+
+BulkIndexByScrollResponse response = updateByQuery.get();
+--------------------------------------------------
+
+The `updatebyQuery` API returns a JSON object similar to the following example:
+
+[source,java]
+--------------------------------------------------
+{
+  "took" : 147,
+  "timed_out": false,
+  "updated": 120,
+  "deleted": 0,
+  "batches": 1,
+  "version_conflicts": 0,
+  "noops": 0,
+  "retries": {
+    "bulk": 0,
+    "search": 0
+  },
+  "throttled_millis": 0,
+  "requests_per_second": "unlimited",
+  "throttled_until_millis": 0,
+  "total": 120,
+  "failures" : [ ]
+}
+--------------------------------------------------
+// TESTRESPONSE[s/"took" : 147/"took" : "$body.took"/]
+
+Calls to the `updateByQuery` API start by getting a snapshot of the index, indexing
+any documents found using the `internal` versioning.
+
+NOTE: Version conflicts happen when a document changes between the time of the
+snapshot and the time the index request processes. 
+
+When the versions match, `updateByQuery` updates the document
+and increments the version number.
+
+All update and query failures cause `updateByQuery` to abort. These failures are
+listed in the `failures` section of the JSON response object. Any successful updates 
+remain and are not rolled back. While the first failure causes the abort, the JSON
+response object contains all of the failures generated by the failed bulk request.
+
+To prevent version conflicts from causing `updateByQuery` to abort, 
+set `conflicts=proceed` on the URL or `"conflicts": "proceed"`
+in the request body. The first example does this because it is trying to
+pick up an online mapping change and a version conflict means that the
+conflicting document was updated between the start of the `updateByQuery`
+and the time when it attempted to update the document. This is fine because
+that update will have picked up the online mapping update.
+
+Back to the API format, you can limit `updateByQuery` to a single type. This
+will only update `tweet` documents from the `twitter` index:
+
+// provide API Example
+
+[source,java]
+--------------------------------------------------
+
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+You can also limit `updateByQuery` using the
+<<query-dsl,Query DSL>>. This example updates all documents from the
+`twitter` index for the user `kimchy`:
+
+
+// provide API Example
+
+[source,java]
+--------------------------------------------------
+
+{
+  "query": { <1>
+    "term": {
+      "user": "kimchy"
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+<1> The query must be passed as a value to the `query` key, in the same
+way as the <<search-search,Search API>>. You can also use the `q`
+parameter in the same way as the search api.
+
+So far we've only been updating documents without changing their source. That
+is genuinely useful for things like
+<<picking-up-a-new-property,picking up new properties>> but it's only half the
+fun. `updateByQuery` supports a `script` object to update the document. This
+will increment the `likes` field on all of kimchy's tweets:
+
+// provide API Example
+
+[source,java]
+--------------------------------------------------
+
+{
+  "script": {
+    "inline": "ctx._source.likes++"
+  },
+  "query": {
+    "term": {
+      "user": "kimchy"
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+Just as in <<docs-update,Update API>> you can set `ctx.op` to change the
+operation that is executed:
+
+`noop`::
+
+Set `ctx.op = "noop"` if your script decides that it doesn't have to make any
+changes. That will cause `updateByQuery` to omit that document from its updates.
+ This no operation will be reported in the `noop` counter in the
+<<docs-update-by-query-response-body, response body>>.
+
+`delete`::
+
+Set `ctx.op = "delete"` if your script decides that the document must be
+ deleted. The deletion will be reported in the `deleted` counter in the
+<<docs-update-by-query-response-body, response body>>.
+
+Setting `ctx.op` to anything else is an error. Setting any
+other field in `ctx` is an error.
+
+Note that we stopped specifying `conflicts=proceed`. In this case we want a
+version conflict to abort the process so we can handle the failure.
+
+This API doesn't allow you to move the documents it touches, just modify their
+source. This is intentional! We've made no provisions for removing the document
+from its original location.
+
+It's also possible to do this whole thing on multiple indexes and multiple
+types at once, just like the search API:
+
+// provide API Example
+
+[source,java]
+--------------------------------------------------
+
+--------------------------------------------------
+// CONSOLE
+// TEST[s/^/PUT twitter\nPUT blog\nGET _cluster\/health?wait_for_status=yellow\n/]
+
+If you provide `routing` then the routing is copied to the scroll query,
+limiting the process to the shards that match that routing value:
+
+// provide API Example
+[source,java]
+--------------------------------------------------
+
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+By default `updateByQuery` uses scroll batches of 1000. You can change the
+batch size with the `scroll_size` URL parameter:
+
+// provide API Example
+[source,java]
+--------------------------------------------------
+
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+`updateByQuery` can also use the <<ingest>> feature by
+specifying a `pipeline` like this:
+
+// provide API Example
+[source,java]
+--------------------------------------------------
+
+{
+  "description" : "sets foo",
+  "processors" : [ {
+      "set" : {
+        "field": "foo",
+        "value": "bar"
+      }
+  } ]
+}
+POST twitter/_update_by_query?pipeline=set-foo
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:twitter]
+
+[float]
+=== URL Parameters
+
+In addition to the standard parameters like `pretty`, the Update By Query API
+also supports `refresh`, `wait_for_completion`, `consistency`, and `timeout`.
+
+Sending the `refresh` will update all shards in the index being updated when
+the request completes. This is different than the Index API's `refresh`
+parameter which causes just the shard that received the new data to be indexed.
+
+If the request contains `wait_for_completion=false` then Elasticsearch will
+perform some preflight checks, launch the request, and then return a `task`
+which can be used with <<docs-update-by-query-task-api,Tasks APIs>>
+to cancel or get the status of the task. Elasticsearch will also create a
+record of this task as a document at `.tasks/task/${taskId}`. This is yours
+to keep or remove as you see fit. When you are done with it, delete it so
+Elasticsearch can reclaim the space it uses.
+
+`consistency` controls how many copies of a shard must respond to each write
+request. `timeout` controls how long each write request waits for unavailable
+shards to become available. Both work exactly how they work in the
+<<docs-bulk,Bulk API>>.
+
+`requests_per_second` can be set to any decimal number (`1.4`, `6`, `1000`, etc)
+and throttles the number of requests per second that the update by query issues.
+The throttling is done waiting between bulk batches so that it can manipulate
+the scroll timeout. The wait time is the difference between the time it took the
+batch to complete and the time `requests_per_second * requests_in_the_batch`.
+Since the batch isn't broken into multiple bulk requests large batch sizes will
+cause Elasticsearch to create many requests and then wait for a while before
+starting the next set. This is "bursty" instead of "smooth". The default is
+`unlimited` which is also the only non-number value that it accepts.
+
+[float]
+[[docs-update-by-query-response-body]]
+=== Response body
+
+The JSON response looks like this:
+
+[source,java]
+--------------------------------------------------
+{
+  "took" : 639,
+  "updated": 0,
+  "batches": 1,
+  "version_conflicts": 2,
+  "retries": {
+    "bulk": 0,
+    "search": 0
+  }
+  "throttled_millis": 0,
+  "failures" : [ ]
+}
+--------------------------------------------------
+
+`took`::
+
+The number of milliseconds from start to end of the whole operation.
+
+`updated`::
+
+The number of documents that were successfully updated.
+
+`batches`::
+
+The number of scroll responses pulled back by the the update by query.
+
+`version_conflicts`::
+
+The number of version conflicts that the update by query hit.
+
+`retries`::
+
+The number of retries attempted by update-by-query. `bulk` is the number of bulk
+actions retried and `search` is the number of search actions retried.
+
+`throttled_millis`::
+
+Number of milliseconds the request slept to conform to `requests_per_second`.
+
+`failures`::
+
+Array of all indexing failures. If this is non-empty then the request aborted
+because of those failures. See `conflicts` for how to prevent version conflicts
+from aborting the operation.
+
+
+[float]
+[[docs-update-by-query-task-api]]
+=== Works with the Task API
+
+You can fetch the status of all running update-by-query requests with the
+<<tasks,Task API>>:
+
+// provide API Example
+
+[source,java]
+--------------------------------------------------
+
+--------------------------------------------------
+// CONSOLE
+
+The responses looks like:
+
+[source,java]
+--------------------------------------------------
+{
+  "nodes" : {
+    "r1A2WoRbTwKZ516z6NEs5A" : {
+      "name" : "Tyrannus",
+      "transport_address" : "127.0.0.1:9300",
+      "host" : "127.0.0.1",
+      "ip" : "127.0.0.1:9300",
+      "attributes" : {
+        "testattr" : "test",
+        "portsfile" : "true"
+      },
+      "tasks" : {
+        "r1A2WoRbTwKZ516z6NEs5A:36619" : {
+          "node" : "r1A2WoRbTwKZ516z6NEs5A",
+          "id" : 36619,
+          "type" : "transport",
+          "action" : "indices:data/write/update/byquery",
+          "status" : {    <1>
+            "total" : 6154,
+            "updated" : 3500,
+            "created" : 0,
+            "deleted" : 0,
+            "batches" : 4,
+            "version_conflicts" : 0,
+            "noops" : 0,
+            "retries": {
+              "bulk": 0,
+              "search": 0
+            }
+            "throttled_millis": 0
+          },
+          "description" : ""
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+
+<1> this object contains the actual status. It is just like the response json
+with the important addition of the `total` field. `total` is the total number
+of operations that the reindex expects to perform. You can estimate the
+progress by adding the `updated`, `created`, and `deleted` fields. The request
+will finish when their sum is equal to the `total` field.
+
+With the task id you can look up the task directly:
+
+// provide API Example
+[source,java]
+--------------------------------------------------
+
+--------------------------------------------------
+// CONSOLE
+// TEST[catch:missing]
+
+The advantage of this API is that it integrates with `wait_for_completion=false`
+to transparently return the status of completed tasks. If the task is completed
+and `wait_for_completion=false` was set on it them it'll come back with a
+`results` or an `error` field. The cost of this feature is the document that
+`wait_for_completion=false` creates at `.tasks/task/${taskId}`. It is up to
+you to delete that document.
+
+[float]
+[[docs-update-by-query-cancel-task-api]]
+=== Works with the Cancel Task API
+
+Any Update By Query can be canceled using the <<tasks,Task Cancel API>>:
+
+// provide API Example
+[source,java]
+--------------------------------------------------
+
+--------------------------------------------------
+// CONSOLE
+
+The `task_id` can be found using the tasks API above.
+
+Cancelation should happen quickly but might take a few seconds. The task status
+API above will continue to list the task until it is wakes to cancel itself.
+
+
+[float]
+[[docs-update-by-query-rethrottle]]
+=== Rethrottling
+
+The value of `requests_per_second` can be changed on a running update by query
+using the `_rethrottle` API:
+
+// provide API Example
+[source,java]
+--------------------------------------------------
+
+--------------------------------------------------
+// CONSOLE
+
+The `task_id` can be found using the tasks API above.
+
+Just like when setting it on the `updateByQuery` API `requests_per_second`
+can be either `unlimited` to disable throttling or any decimal number like `1.7`
+or `12` to throttle to that level. Rethrottling that speeds up the query takes
+effect immediately but rethrotting that slows down the query will take effect
+on after completing the current batch. This prevents scroll timeouts.
+
+
+[float]
+[[picking-up-a-new-property]]
+=== Pick up a new property
+
+Say you created an index without dynamic mapping, filled it with data, and then
+added a mapping value to pick up more fields from the data:
+
+[source,java]
+--------------------------------------------------
+PUT test
+{
+  "mappings": {
+    "test": {
+      "dynamic": false,   <1>
+      "properties": {
+        "text": {"type": "text"}
+      }
+    }
+  }
+}
+
+POST test/test?refresh
+{
+  "text": "words words",
+  "flag": "bar"
+}
+POST test/test?refresh
+{
+  "text": "words words",
+  "flag": "foo"
+}
+PUT test/_mapping/test   <2>
+{
+  "properties": {
+    "text": {"type": "text"},
+    "flag": {"type": "text", "analyzer": "keyword"}
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+<1> This means that new fields won't be indexed, just stored in `_source`.
+
+<2> This updates the mapping to add the new `flag` field. To pick up the new
+field you have to reindex all documents with it.
+
+Searching for the data won't find anything:
+
+// provide API Example
+[source,java]
+--------------------------------------------------
+
+{
+  "query": {
+    "match": {
+      "flag": "foo"
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+[source,java]
+--------------------------------------------------
+{
+  "hits" : {
+    "total" : 0
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE
+
+But you can issue an `updateByQuery` request to pick up the new mapping:
+
+// provide API Example
+[source,java]
+--------------------------------------------------
+
+{
+  "query": {
+    "match": {
+      "flag": "foo"
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+[source,java]
+--------------------------------------------------
+{
+  "hits" : {
+    "total" : 1
+  }
+}
+--------------------------------------------------
+// TESTRESPONSE
+
+You can do the exact same thing when adding a field to a multifield.

From ccab85835aae26fd0eef2030a6a5c39835ebf9a7 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 24 Jun 2016 11:12:39 -0400
Subject: [PATCH 1118/1311] Rework java update-by-query docs

---
 docs/java-api/docs/update-by-query.asciidoc | 464 ++++----------------
 1 file changed, 85 insertions(+), 379 deletions(-)

diff --git a/docs/java-api/docs/update-by-query.asciidoc b/docs/java-api/docs/update-by-query.asciidoc
index eff4be0182d..16382067501 100644
--- a/docs/java-api/docs/update-by-query.asciidoc
+++ b/docs/java-api/docs/update-by-query.asciidoc
@@ -4,7 +4,7 @@
 experimental[The update-by-query API is new and should still be considered experimental.  The API may change in ways that are not backwards compatible]
 
 The simplest usage of `updateByQuery` updates each
-document in an index without changing the source. This usage enables 
+document in an index without changing the source. This usage enables
 <<picking-up-a-new-property,picking up a new property>> or another online
 mapping change.
 
@@ -12,121 +12,93 @@ mapping change.
 --------------------------------------------------
 UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
 
-updateByQuery.source("cool_things")
-        .filter(termQuery("level", "awesome"))
-        .script(new Script("ctx._source.awesome = \"absolutely\""));
+updateByQuery.source("source_index").abortOnVersionConflict(false);
 
 BulkIndexByScrollResponse response = updateByQuery.get();
 --------------------------------------------------
 
-The `updatebyQuery` API returns a JSON object similar to the following example:
-
-[source,java]
---------------------------------------------------
-{
-  "took" : 147,
-  "timed_out": false,
-  "updated": 120,
-  "deleted": 0,
-  "batches": 1,
-  "version_conflicts": 0,
-  "noops": 0,
-  "retries": {
-    "bulk": 0,
-    "search": 0
-  },
-  "throttled_millis": 0,
-  "requests_per_second": "unlimited",
-  "throttled_until_millis": 0,
-  "total": 120,
-  "failures" : [ ]
-}
---------------------------------------------------
-// TESTRESPONSE[s/"took" : 147/"took" : "$body.took"/]
-
 Calls to the `updateByQuery` API start by getting a snapshot of the index, indexing
 any documents found using the `internal` versioning.
 
 NOTE: Version conflicts happen when a document changes between the time of the
-snapshot and the time the index request processes. 
+snapshot and the time the index request processes.
 
 When the versions match, `updateByQuery` updates the document
 and increments the version number.
 
 All update and query failures cause `updateByQuery` to abort. These failures are
-listed in the `failures` section of the JSON response object. Any successful updates 
-remain and are not rolled back. While the first failure causes the abort, the JSON
-response object contains all of the failures generated by the failed bulk request.
+available from the `BulkIndexByScrollResponse#getIndexingFailures` method. Any
+successful updates remain and are not rolled back. While the first failure
+causes the abort, the response contains all of the failures generated by the
+failed bulk request.
 
-To prevent version conflicts from causing `updateByQuery` to abort, 
-set `conflicts=proceed` on the URL or `"conflicts": "proceed"`
-in the request body. The first example does this because it is trying to
-pick up an online mapping change and a version conflict means that the
-conflicting document was updated between the start of the `updateByQuery`
+To prevent version conflicts from causing `updateByQuery` to abort, set
+`abortOnVersionConflict(false)`. The first example does this because it is
+trying to pick up an online mapping change and a version conflict means that
+the conflicting document was updated between the start of the `updateByQuery`
 and the time when it attempted to update the document. This is fine because
 that update will have picked up the online mapping update.
 
-Back to the API format, you can limit `updateByQuery` to a single type. This
-will only update `tweet` documents from the `twitter` index:
-
-// provide API Example
+Back to the API, `UpdateByQueryRequestBuilder` supports filtering the documents
+that are updated, limiting the total number updated, and updating documents
+with a script:
 
 [source,java]
 --------------------------------------------------
+UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
 
+updateByQuery.source("source_index")
+        .filter(termQuery("level", "awesome"))
+        .size(1000)
+        .script(new Script("ctx._source.awesome = 'absolutely'", ScriptType.INLINE, "painless", emptyMap()));
+
+BulkIndexByScrollResponse response = updateByQuery.get();
 --------------------------------------------------
-// CONSOLE
-// TEST[setup:twitter]
 
-You can also limit `updateByQuery` using the
-<<query-dsl,Query DSL>>. This example updates all documents from the
-`twitter` index for the user `kimchy`:
-
-
-// provide API Example
+`UpdateByQueryRequestBuilder` also allows you direct access to the query used
+to select the documents which you can use to change the default scroll size or
+otherwise modify the request for matching documents.
 
 [source,java]
 --------------------------------------------------
+UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
 
-{
-  "query": { <1>
-    "term": {
-      "user": "kimchy"
-    }
-  }
-}
+updateByQuery.source("source_index")
+        .source().setSize(500);
+
+BulkIndexByScrollResponse response = updateByQuery.get();
 --------------------------------------------------
-// CONSOLE
-// TEST[setup:twitter]
 
-<1> The query must be passed as a value to the `query` key, in the same
-way as the <<search-search,Search API>>. You can also use the `q`
-parameter in the same way as the search api.
-
-So far we've only been updating documents without changing their source. That
-is genuinely useful for things like
-<<picking-up-a-new-property,picking up new properties>> but it's only half the
-fun. `updateByQuery` supports a `script` object to update the document. This
-will increment the `likes` field on all of kimchy's tweets:
-
-// provide API Example
+You can also combine `size` with sorting to limit the documents updated:
 
 [source,java]
 --------------------------------------------------
+UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
 
-{
-  "script": {
-    "inline": "ctx._source.likes++"
-  },
-  "query": {
-    "term": {
-      "user": "kimchy"
-    }
-  }
-}
+updateByQuery.source("source_index").size(100)
+        .source().addSort("cat", SortOrder.DESC);
+
+BulkIndexByScrollResponse response = updateByQuery.get();
+--------------------------------------------------
+
+In addition to changing the `_source` of the document (see above) the script
+can change the update action similarly to the Update API:
+
+[source,java]
+--------------------------------------------------
+UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
+
+updateByQuery.source("source_index")
+        .script(new Script(
+            "if (ctx._source.awesome == 'absolutely) {"
+          + "  ctx.op='noop'
+          + "} else if (ctx._source.awesome == 'lame') {"
+          + "  ctx.op='delete'"
+          + "} else {"
+          + "ctx._source.awesome = 'absolutely'}", ScriptType.INLINE, "painless", emptyMap()));
+
+BulkIndexByScrollResponse response = updateByQuery.get();
 --------------------------------------------------
-// CONSOLE
-// TEST[setup:twitter]
 
 Just as in <<docs-update,Update API>> you can set `ctx.op` to change the
 operation that is executed:
@@ -141,15 +113,12 @@ changes. That will cause `updateByQuery` to omit that document from its updates.
 `delete`::
 
 Set `ctx.op = "delete"` if your script decides that the document must be
- deleted. The deletion will be reported in the `deleted` counter in the
+deleted. The deletion will be reported in the `deleted` counter in the
 <<docs-update-by-query-response-body, response body>>.
 
 Setting `ctx.op` to anything else is an error. Setting any
 other field in `ctx` is an error.
 
-Note that we stopped specifying `conflicts=proceed`. In this case we want a
-version conflict to abort the process so we can handle the failure.
-
 This API doesn't allow you to move the documents it touches, just modify their
 source. This is intentional! We've made no provisions for removing the document
 from its original location.
@@ -157,144 +126,38 @@ from its original location.
 It's also possible to do this whole thing on multiple indexes and multiple
 types at once, just like the search API:
 
-// provide API Example
-
 [source,java]
 --------------------------------------------------
+UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
 
+updateByQuery.source("foo", "bar").source().setTypes("a", "b");
+
+BulkIndexByScrollResponse response = updateByQuery.get();
 --------------------------------------------------
-// CONSOLE
-// TEST[s/^/PUT twitter\nPUT blog\nGET _cluster\/health?wait_for_status=yellow\n/]
 
 If you provide `routing` then the routing is copied to the scroll query,
 limiting the process to the shards that match that routing value:
 
-// provide API Example
 [source,java]
 --------------------------------------------------
+UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
 
+updateByQuery.source().setRouting("cat");
+
+BulkIndexByScrollResponse response = updateByQuery.get();
 --------------------------------------------------
-// CONSOLE
-// TEST[setup:twitter]
-
-By default `updateByQuery` uses scroll batches of 1000. You can change the
-batch size with the `scroll_size` URL parameter:
-
-// provide API Example
-[source,java]
---------------------------------------------------
-
---------------------------------------------------
-// CONSOLE
-// TEST[setup:twitter]
 
 `updateByQuery` can also use the <<ingest>> feature by
 specifying a `pipeline` like this:
 
-// provide API Example
 [source,java]
 --------------------------------------------------
+UpdateByQueryRequestBuilder updateByQuery = UpdateByQueryAction.INSTANCE.newRequestBuilder(client);
 
-{
-  "description" : "sets foo",
-  "processors" : [ {
-      "set" : {
-        "field": "foo",
-        "value": "bar"
-      }
-  } ]
-}
-POST twitter/_update_by_query?pipeline=set-foo
+updateByQuery.setPipeline("hurray");
+
+BulkIndexByScrollResponse response = updateByQuery.get();
 --------------------------------------------------
-// CONSOLE
-// TEST[setup:twitter]
-
-[float]
-=== URL Parameters
-
-In addition to the standard parameters like `pretty`, the Update By Query API
-also supports `refresh`, `wait_for_completion`, `consistency`, and `timeout`.
-
-Sending the `refresh` will update all shards in the index being updated when
-the request completes. This is different than the Index API's `refresh`
-parameter which causes just the shard that received the new data to be indexed.
-
-If the request contains `wait_for_completion=false` then Elasticsearch will
-perform some preflight checks, launch the request, and then return a `task`
-which can be used with <<docs-update-by-query-task-api,Tasks APIs>>
-to cancel or get the status of the task. Elasticsearch will also create a
-record of this task as a document at `.tasks/task/${taskId}`. This is yours
-to keep or remove as you see fit. When you are done with it, delete it so
-Elasticsearch can reclaim the space it uses.
-
-`consistency` controls how many copies of a shard must respond to each write
-request. `timeout` controls how long each write request waits for unavailable
-shards to become available. Both work exactly how they work in the
-<<docs-bulk,Bulk API>>.
-
-`requests_per_second` can be set to any decimal number (`1.4`, `6`, `1000`, etc)
-and throttles the number of requests per second that the update by query issues.
-The throttling is done waiting between bulk batches so that it can manipulate
-the scroll timeout. The wait time is the difference between the time it took the
-batch to complete and the time `requests_per_second * requests_in_the_batch`.
-Since the batch isn't broken into multiple bulk requests large batch sizes will
-cause Elasticsearch to create many requests and then wait for a while before
-starting the next set. This is "bursty" instead of "smooth". The default is
-`unlimited` which is also the only non-number value that it accepts.
-
-[float]
-[[docs-update-by-query-response-body]]
-=== Response body
-
-The JSON response looks like this:
-
-[source,java]
---------------------------------------------------
-{
-  "took" : 639,
-  "updated": 0,
-  "batches": 1,
-  "version_conflicts": 2,
-  "retries": {
-    "bulk": 0,
-    "search": 0
-  }
-  "throttled_millis": 0,
-  "failures" : [ ]
-}
---------------------------------------------------
-
-`took`::
-
-The number of milliseconds from start to end of the whole operation.
-
-`updated`::
-
-The number of documents that were successfully updated.
-
-`batches`::
-
-The number of scroll responses pulled back by the the update by query.
-
-`version_conflicts`::
-
-The number of version conflicts that the update by query hit.
-
-`retries`::
-
-The number of retries attempted by update-by-query. `bulk` is the number of bulk
-actions retried and `search` is the number of search actions retried.
-
-`throttled_millis`::
-
-Number of milliseconds the request slept to conform to `requests_per_second`.
-
-`failures`::
-
-Array of all indexing failures. If this is non-empty then the request aborted
-because of those failures. See `conflicts` for how to prevent version conflicts
-from aborting the operation.
-
 
 [float]
 [[docs-update-by-query-task-api]]
@@ -303,79 +166,26 @@ from aborting the operation.
 You can fetch the status of all running update-by-query requests with the
 <<tasks,Task API>>:
 
-// provide API Example
-
 [source,java]
 --------------------------------------------------
+ListTasksResponse tasksList = client.admin().cluster().prepareListTasks()
+        .setActions(UpdateByQueryAction.NAME).setDetailed(true).get();
 
---------------------------------------------------
-// CONSOLE
-
-The responses looks like:
-
-[source,java]
---------------------------------------------------
-{
-  "nodes" : {
-    "r1A2WoRbTwKZ516z6NEs5A" : {
-      "name" : "Tyrannus",
-      "transport_address" : "127.0.0.1:9300",
-      "host" : "127.0.0.1",
-      "ip" : "127.0.0.1:9300",
-      "attributes" : {
-        "testattr" : "test",
-        "portsfile" : "true"
-      },
-      "tasks" : {
-        "r1A2WoRbTwKZ516z6NEs5A:36619" : {
-          "node" : "r1A2WoRbTwKZ516z6NEs5A",
-          "id" : 36619,
-          "type" : "transport",
-          "action" : "indices:data/write/update/byquery",
-          "status" : {    <1>
-            "total" : 6154,
-            "updated" : 3500,
-            "created" : 0,
-            "deleted" : 0,
-            "batches" : 4,
-            "version_conflicts" : 0,
-            "noops" : 0,
-            "retries": {
-              "bulk": 0,
-              "search": 0
-            }
-            "throttled_millis": 0
-          },
-          "description" : ""
-        }
-      }
-    }
-  }
+for (TaskInfo info: tasksList.getTasks()) {
+    TaskId taskId = info.getTaskId();
+    BulkByScrollTask.Status status = (BulkByScrollTask.Status) info.getStatus();
+    // do stuff
 }
+
 --------------------------------------------------
 
-<1> this object contains the actual status. It is just like the response json
-with the important addition of the `total` field. `total` is the total number
-of operations that the reindex expects to perform. You can estimate the
-progress by adding the `updated`, `created`, and `deleted` fields. The request
-will finish when their sum is equal to the `total` field.
-
-With the task id you can look up the task directly:
+With the `TaskId` shown above you can look up the task directly:
 
 // provide API Example
 [source,java]
 --------------------------------------------------
-
+GetTaskResponse get = client.admin().cluster().prepareGetTask(taskId).get();
 --------------------------------------------------
-// CONSOLE
-// TEST[catch:missing]
-
-The advantage of this API is that it integrates with `wait_for_completion=false`
-to transparently return the status of completed tasks. If the task is completed
-and `wait_for_completion=false` was set on it them it'll come back with a
-`results` or an `error` field. The cost of this feature is the document that
-`wait_for_completion=false` creates at `.tasks/task/${taskId}`. It is up to
-you to delete that document.
 
 [float]
 [[docs-update-by-query-cancel-task-api]]
@@ -383,14 +193,15 @@ you to delete that document.
 
 Any Update By Query can be canceled using the <<tasks,Task Cancel API>>:
 
-// provide API Example
 [source,java]
 --------------------------------------------------
-
+// Cancel all update-by-query requests
+client.admin().cluster().prepareCancelTasks().setActions(UpdateByQueryAction.NAME).get().getTasks()
+// Cancel a specific update-by-query request
+client.admin().cluster().prepareCancelTasks().setTaskId(taskId).get().getTasks()
 --------------------------------------------------
-// CONSOLE
 
-The `task_id` can be found using the tasks API above.
+The `taskId` can be found using the list tasks API above.
 
 Cancelation should happen quickly but might take a few seconds. The task status
 API above will continue to list the task until it is wakes to cancel itself.
@@ -403,120 +214,15 @@ API above will continue to list the task until it is wakes to cancel itself.
 The value of `requests_per_second` can be changed on a running update by query
 using the `_rethrottle` API:
 
-// provide API Example
 [source,java]
 --------------------------------------------------
-
+RethrottleAction.INSTANCE.newRequestBuilder(client).setTaskId(taskId).setRequestsPerSecond(2.0f).get();
 --------------------------------------------------
-// CONSOLE
 
-The `task_id` can be found using the tasks API above.
+The `taskId` can be found using the tasks API above.
 
 Just like when setting it on the `updateByQuery` API `requests_per_second`
-can be either `unlimited` to disable throttling or any decimal number like `1.7`
-or `12` to throttle to that level. Rethrottling that speeds up the query takes
+can be either `Float.POSITIVE_INFINITY` to disable throttling or any positive
+float to throttle to that level. Rethrottling that speeds up the query takes
 effect immediately but rethrotting that slows down the query will take effect
 on after completing the current batch. This prevents scroll timeouts.
-
-
-[float]
-[[picking-up-a-new-property]]
-=== Pick up a new property
-
-Say you created an index without dynamic mapping, filled it with data, and then
-added a mapping value to pick up more fields from the data:
-
-[source,java]
---------------------------------------------------
-PUT test
-{
-  "mappings": {
-    "test": {
-      "dynamic": false,   <1>
-      "properties": {
-        "text": {"type": "text"}
-      }
-    }
-  }
-}
-
-POST test/test?refresh
-{
-  "text": "words words",
-  "flag": "bar"
-}
-POST test/test?refresh
-{
-  "text": "words words",
-  "flag": "foo"
-}
-PUT test/_mapping/test   <2>
-{
-  "properties": {
-    "text": {"type": "text"},
-    "flag": {"type": "text", "analyzer": "keyword"}
-  }
-}
---------------------------------------------------
-// CONSOLE
-
-<1> This means that new fields won't be indexed, just stored in `_source`.
-
-<2> This updates the mapping to add the new `flag` field. To pick up the new
-field you have to reindex all documents with it.
-
-Searching for the data won't find anything:
-
-// provide API Example
-[source,java]
---------------------------------------------------
-
-{
-  "query": {
-    "match": {
-      "flag": "foo"
-    }
-  }
-}
---------------------------------------------------
-// CONSOLE
-// TEST[continued]
-
-[source,java]
---------------------------------------------------
-{
-  "hits" : {
-    "total" : 0
-  }
-}
---------------------------------------------------
-// TESTRESPONSE
-
-But you can issue an `updateByQuery` request to pick up the new mapping:
-
-// provide API Example
-[source,java]
---------------------------------------------------
-
-{
-  "query": {
-    "match": {
-      "flag": "foo"
-    }
-  }
-}
---------------------------------------------------
-// CONSOLE
-// TEST[continued]
-
-[source,java]
---------------------------------------------------
-{
-  "hits" : {
-    "total" : 1
-  }
-}
---------------------------------------------------
-// TESTRESPONSE
-
-You can do the exact same thing when adding a field to a multifield.

From 57f413e85164600ca12c7f30c455b4c8bba7d63e Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 29 Jun 2016 11:08:54 -0400
Subject: [PATCH 1119/1311] More changes to java update-by-query api docs

---
 docs/java-api/docs/update-by-query.asciidoc | 53 ++++++++++-----------
 1 file changed, 25 insertions(+), 28 deletions(-)

diff --git a/docs/java-api/docs/update-by-query.asciidoc b/docs/java-api/docs/update-by-query.asciidoc
index 16382067501..a94899668ef 100644
--- a/docs/java-api/docs/update-by-query.asciidoc
+++ b/docs/java-api/docs/update-by-query.asciidoc
@@ -39,8 +39,8 @@ the conflicting document was updated between the start of the `updateByQuery`
 and the time when it attempted to update the document. This is fine because
 that update will have picked up the online mapping update.
 
-Back to the API, `UpdateByQueryRequestBuilder` supports filtering the documents
-that are updated, limiting the total number updated, and updating documents
+The `UpdateByQueryRequestBuilder` API supports filtering the updated documents,
+limiting the total number of documents to update, and updating documents
 with a script:
 
 [source,java]
@@ -55,8 +55,8 @@ updateByQuery.source("source_index")
 BulkIndexByScrollResponse response = updateByQuery.get();
 --------------------------------------------------
 
-`UpdateByQueryRequestBuilder` also allows you direct access to the query used
-to select the documents which you can use to change the default scroll size or
+`UpdateByQueryRequestBuilder` also enables direct access to the query used
+to select the documents. You can use this access to change the default scroll size or
 otherwise modify the request for matching documents.
 
 [source,java]
@@ -81,8 +81,8 @@ updateByQuery.source("source_index").size(100)
 BulkIndexByScrollResponse response = updateByQuery.get();
 --------------------------------------------------
 
-In addition to changing the `_source` of the document (see above) the script
-can change the update action similarly to the Update API:
+In addition to changing the `_source` field for the document, you can use a
+script to change the action, similar to the Update API:
 
 [source,java]
 --------------------------------------------------
@@ -100,14 +100,14 @@ updateByQuery.source("source_index")
 BulkIndexByScrollResponse response = updateByQuery.get();
 --------------------------------------------------
 
-Just as in <<docs-update,Update API>> you can set `ctx.op` to change the
-operation that is executed:
+As in the <<docs-update,Update API>>, you can set the value of `ctx.op` to change the
+operation that executes:
 
 `noop`::
 
-Set `ctx.op = "noop"` if your script decides that it doesn't have to make any
-changes. That will cause `updateByQuery` to omit that document from its updates.
- This no operation will be reported in the `noop` counter in the
+Set `ctx.op = "noop"` if your script doesn't make any
+changes. The `updateByQuery` operaton then omits that document from the updates.
+This behavior increments the `noop` counter in the
 <<docs-update-by-query-response-body, response body>>.
 
 `delete`::
@@ -116,15 +116,14 @@ Set `ctx.op = "delete"` if your script decides that the document must be
 deleted. The deletion will be reported in the `deleted` counter in the
 <<docs-update-by-query-response-body, response body>>.
 
-Setting `ctx.op` to anything else is an error. Setting any
-other field in `ctx` is an error.
+Setting `ctx.op` to any other value generates an error. Setting any
+other field in `ctx` generates an error.
 
 This API doesn't allow you to move the documents it touches, just modify their
 source. This is intentional! We've made no provisions for removing the document
 from its original location.
 
-It's also possible to do this whole thing on multiple indexes and multiple
-types at once, just like the search API:
+You can also perform these operations on multiple indices and types at once, similar to the search API:
 
 [source,java]
 --------------------------------------------------
@@ -135,7 +134,7 @@ updateByQuery.source("foo", "bar").source().setTypes("a", "b");
 BulkIndexByScrollResponse response = updateByQuery.get();
 --------------------------------------------------
 
-If you provide `routing` then the routing is copied to the scroll query,
+If you provide a `routing` value then the process copies the routing value to the scroll query,
 limiting the process to the shards that match that routing value:
 
 [source,java]
@@ -201,28 +200,26 @@ client.admin().cluster().prepareCancelTasks().setActions(UpdateByQueryAction.NAM
 client.admin().cluster().prepareCancelTasks().setTaskId(taskId).get().getTasks()
 --------------------------------------------------
 
-The `taskId` can be found using the list tasks API above.
-
-Cancelation should happen quickly but might take a few seconds. The task status
-API above will continue to list the task until it is wakes to cancel itself.
+Use the `list tasks` API to find the value of `taskId`.
 
+Cancelling a request is typically a very fast process but can take up to a few seconds.
+The task status API continues to list the task until the cancellation is complete.
 
 [float]
 [[docs-update-by-query-rethrottle]]
 === Rethrottling
 
-The value of `requests_per_second` can be changed on a running update by query
-using the `_rethrottle` API:
+Use the `_rethrottle` API to change the value of `requests_per_second` on a running update:
 
 [source,java]
 --------------------------------------------------
 RethrottleAction.INSTANCE.newRequestBuilder(client).setTaskId(taskId).setRequestsPerSecond(2.0f).get();
 --------------------------------------------------
 
-The `taskId` can be found using the tasks API above.
+Use the `list tasks` API to find the value of `taskId`.
 
-Just like when setting it on the `updateByQuery` API `requests_per_second`
-can be either `Float.POSITIVE_INFINITY` to disable throttling or any positive
-float to throttle to that level. Rethrottling that speeds up the query takes
-effect immediately but rethrotting that slows down the query will take effect
-on after completing the current batch. This prevents scroll timeouts.
+As with the `updateByQuery` API, the value of `requests_per_second`
+can be any positive float value to set the level of the throttle, or `Float.POSITIVE_INFINITY` to disable throttling.
+A value of `requests_per_second` that speeds up the process takes
+effect immediately. `requests_per_second` values that slow the query take effect
+after completing the current batch in order to prevent scroll timeouts.

From 0d81dee0137ebd1c80b5713b44b5581037a2c84b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Tue, 28 Jun 2016 19:32:56 +0200
Subject: [PATCH 1120/1311] Fix key_as_string for date histogram and
 epoch_millis/epoch_second format

When doing a `date_histogram` aggregation with `"format":"epoch_millis"` or
`"format" : "epoch_second"` and using a time zone other than UTC, the
`key_as_string` ouput in the response does not reflect the UTC timestamp that is
used as the key. This happens because when applying the `time_zone` in
DocValueFormat.DateTime to an epoch-based formatter, this adds the time zone
offset to the value being formated. Instead we should adjust the added display
offset to get back the utc instance in EpochTimePrinter.

Closes #19038
---
 .../org/elasticsearch/common/joda/Joda.java   | 18 +++++---
 .../elasticsearch/search/DocValueFormat.java  |  1 +
 .../aggregations/bucket/DateHistogramIT.java  | 41 +++++++++++++++++++
 3 files changed, 55 insertions(+), 5 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/joda/Joda.java b/core/src/main/java/org/elasticsearch/common/joda/Joda.java
index cffea836ac2..34c882d0d80 100644
--- a/core/src/main/java/org/elasticsearch/common/joda/Joda.java
+++ b/core/src/main/java/org/elasticsearch/common/joda/Joda.java
@@ -43,7 +43,6 @@ import org.joda.time.format.StrictISODateTimeFormat;
 import java.io.IOException;
 import java.io.Writer;
 import java.util.Locale;
-import java.util.regex.Pattern;
 
 /**
  *
@@ -375,21 +374,30 @@ public class Joda {
             return hasMilliSecondPrecision ? 19 : 16;
         }
 
+
+        /**
+         * We adjust the instant by displayOffset to adjust for the offset that might have been added in
+         * {@link DateTimeFormatter#printTo(Appendable, long, Chronology)} when using a time zone.
+         */
         @Override
         public void printTo(StringBuffer buf, long instant, Chronology chrono, int displayOffset, DateTimeZone displayZone, Locale locale) {
             if (hasMilliSecondPrecision) {
-                buf.append(instant);
+                buf.append(instant - displayOffset);
             } else {
-                buf.append(instant / 1000);
+                buf.append((instant  - displayOffset) / 1000);
             }
         }
 
+        /**
+         * We adjust the instant by displayOffset to adjust for the offset that might have been added in
+         * {@link DateTimeFormatter#printTo(Appendable, long, Chronology)} when using a time zone.
+         */
         @Override
         public void printTo(Writer out, long instant, Chronology chrono, int displayOffset, DateTimeZone displayZone, Locale locale) throws IOException {
             if (hasMilliSecondPrecision) {
-                out.write(String.valueOf(instant));
+                out.write(String.valueOf(instant - displayOffset));
             } else {
-                out.append(String.valueOf(instant / 1000));
+                out.append(String.valueOf((instant - displayOffset) / 1000));
             }
         }
 
diff --git a/core/src/main/java/org/elasticsearch/search/DocValueFormat.java b/core/src/main/java/org/elasticsearch/search/DocValueFormat.java
index 9fc36c38e76..4fc1967d7ec 100644
--- a/core/src/main/java/org/elasticsearch/search/DocValueFormat.java
+++ b/core/src/main/java/org/elasticsearch/search/DocValueFormat.java
@@ -115,6 +115,7 @@ public interface DocValueFormat extends NamedWriteable {
             return Double.parseDouble(value);
         }
 
+        @Override
         public BytesRef parseBytesRef(String value) {
             return new BytesRef(value);
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
index 4f7064a33bb..2c3534183e0 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java
@@ -47,6 +47,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Callable;
@@ -237,6 +238,46 @@ public class DateHistogramIT extends ESIntegTestCase {
         assertThat(bucket.getDocCount(), equalTo(1L));
     }
 
+    public void testSingleValued_timeZone_epoch() throws Exception {
+        String format = randomBoolean() ? "epoch_millis" : "epoch_second";
+        int millisDivider = format.equals("epoch_millis") ? 1 : 1000;
+        if (randomBoolean()) {
+            format = format + "||date_optional_time";
+        }
+        DateTimeZone tz = DateTimeZone.forID("+01:00");
+        SearchResponse response = client().prepareSearch("idx")
+                .addAggregation(dateHistogram("histo").field("date")
+                        .dateHistogramInterval(DateHistogramInterval.DAY).minDocCount(1)
+                        .timeZone(tz).format(format))
+                .execute()
+                .actionGet();
+        assertSearchResponse(response);
+
+        Histogram histo = response.getAggregations().get("histo");
+        assertThat(histo, notNullValue());
+        assertThat(histo.getName(), equalTo("histo"));
+        List<? extends Bucket> buckets = histo.getBuckets();
+        assertThat(buckets.size(), equalTo(6));
+
+        List<DateTime> expectedKeys = new ArrayList<>();
+        expectedKeys.add(new DateTime(2012, 1, 1, 23, 0, DateTimeZone.UTC));
+        expectedKeys.add(new DateTime(2012, 2, 1, 23, 0, DateTimeZone.UTC));
+        expectedKeys.add(new DateTime(2012, 2, 14, 23, 0, DateTimeZone.UTC));
+        expectedKeys.add(new DateTime(2012, 3, 1, 23, 0, DateTimeZone.UTC));
+        expectedKeys.add(new DateTime(2012, 3, 14, 23, 0, DateTimeZone.UTC));
+        expectedKeys.add(new DateTime(2012, 3, 22, 23, 0, DateTimeZone.UTC));
+
+
+        Iterator<DateTime> keyIterator = expectedKeys.iterator();
+        for (Histogram.Bucket bucket : buckets) {
+            assertThat(bucket, notNullValue());
+            DateTime expectedKey = keyIterator.next();
+            assertThat(bucket.getKeyAsString(), equalTo(Long.toString(expectedKey.getMillis() / millisDivider)));
+            assertThat(((DateTime) bucket.getKey()), equalTo(expectedKey));
+            assertThat(bucket.getDocCount(), equalTo(1L));
+        }
+    }
+
     public void testSingleValuedFieldOrderedByKeyAsc() throws Exception {
         SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(dateHistogram("histo")

From fc38e503e04194da01e719810c267be4d2799698 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 29 Jun 2016 13:36:11 -0400
Subject: [PATCH 1121/1311] Clearer error when handling fractional time values

In 2f638b5a23597967a98b1ced1deac91d64af5a44, support for fractional time
values was removed. While this change is documented, the error message
presented does not give an indication that fractional inputs are not
supported. This commit fixes this by detecting when the input is a time
value that would successfully parse as a double but will not parse as a
long and presenting a clear error message that fractional time values
are not supported.

Relates #19158
---
 .../elasticsearch/common/unit/TimeValue.java  | 68 ++++++++++---------
 .../common/unit/TimeValueTests.java           | 31 +++++++++
 2 files changed, 68 insertions(+), 31 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java b/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java
index 5a56603dad7..db8299cdc9a 100644
--- a/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java
+++ b/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java
@@ -317,40 +317,46 @@ public class TimeValue implements Writeable {
         if (sValue == null) {
             return defaultValue;
         }
-        try {
-            String lowerSValue = sValue.toLowerCase(Locale.ROOT).trim();
-            if (lowerSValue.endsWith("nanos")) {
-                return new TimeValue(parse(lowerSValue, 5), TimeUnit.NANOSECONDS);
-            } else if (lowerSValue.endsWith("micros")) {
-                return new TimeValue(parse(lowerSValue, 6), TimeUnit.MICROSECONDS);
-            } else if (lowerSValue.endsWith("ms")) {
-                return new TimeValue(parse(lowerSValue, 2), TimeUnit.MILLISECONDS);
-            } else if (lowerSValue.endsWith("s")) {
-                return new TimeValue(parse(lowerSValue, 1), TimeUnit.SECONDS);
-            } else if (lowerSValue.endsWith("m")) {
-                return new TimeValue(parse(lowerSValue, 1), TimeUnit.MINUTES);
-            } else if (lowerSValue.endsWith("h")) {
-                return new TimeValue(parse(lowerSValue, 1), TimeUnit.HOURS);
-            } else if (lowerSValue.endsWith("d")) {
-                return new TimeValue(parse(lowerSValue, 1), TimeUnit.DAYS);
-            } else if (lowerSValue.matches("-0*1")) {
-                return TimeValue.MINUS_ONE;
-            } else if (lowerSValue.matches("0+")) {
-                return TimeValue.ZERO;
-            } else {
-                // Missing units:
-                throw new ElasticsearchParseException(
-                        "failed to parse setting [{}] with value [{}] as a time value: unit is missing or unrecognized",
-                        settingName,
-                        sValue);
-            }
-        } catch (NumberFormatException e) {
-            throw new ElasticsearchParseException("failed to parse [{}]", e, sValue);
+        final String normalized = sValue.toLowerCase(Locale.ROOT).trim();
+        if (normalized.endsWith("nanos")) {
+            return new TimeValue(parse(sValue, normalized, 5), TimeUnit.NANOSECONDS);
+        } else if (normalized.endsWith("micros")) {
+            return new TimeValue(parse(sValue, normalized, 6), TimeUnit.MICROSECONDS);
+        } else if (normalized.endsWith("ms")) {
+            return new TimeValue(parse(sValue, normalized, 2), TimeUnit.MILLISECONDS);
+        } else if (normalized.endsWith("s")) {
+            return new TimeValue(parse(sValue, normalized, 1), TimeUnit.SECONDS);
+        } else if (normalized.endsWith("m")) {
+            return new TimeValue(parse(sValue, normalized, 1), TimeUnit.MINUTES);
+        } else if (normalized.endsWith("h")) {
+            return new TimeValue(parse(sValue, normalized, 1), TimeUnit.HOURS);
+        } else if (normalized.endsWith("d")) {
+            return new TimeValue(parse(sValue, normalized, 1), TimeUnit.DAYS);
+        } else if (normalized.matches("-0*1")) {
+            return TimeValue.MINUS_ONE;
+        } else if (normalized.matches("0+")) {
+            return TimeValue.ZERO;
+        } else {
+            // Missing units:
+            throw new ElasticsearchParseException(
+                "failed to parse setting [{}] with value [{}] as a time value: unit is missing or unrecognized",
+                settingName,
+                sValue);
         }
     }
 
-    private static long parse(String s, int suffixLength) {
-        return Long.parseLong(s.substring(0, s.length() - suffixLength).trim());
+    private static long parse(final String initialInput, final String normalized, final int suffixLength) {
+        final String s = normalized.substring(0, normalized.length() - suffixLength).trim();
+        try {
+            return Long.parseLong(s);
+        } catch (final NumberFormatException e) {
+            try {
+                @SuppressWarnings("unused") final double ignored = Double.parseDouble(s);
+                throw new ElasticsearchParseException("failed to parse [{}], fractional time values are not supported", e, initialInput);
+            } catch (final NumberFormatException ignored) {
+                throw new ElasticsearchParseException("failed to parse [{}]", e, initialInput);
+            }
+        }
     }
 
     private static final long C0 = 1L;
diff --git a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
index 9b73f2f99af..78afc9e514f 100644
--- a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
+++ b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
@@ -30,9 +30,12 @@ import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.not;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.lessThan;
+import static org.hamcrest.object.HasToString.hasToString;
 
 public class TimeValueTests extends ESTestCase {
 
@@ -125,6 +128,34 @@ public class TimeValueTests extends ESTestCase {
         assertThat(TimeValue.parseTimeValue(t.getStringRep(), null, "test"), equalTo(t));
     }
 
+    private static final String FRACTIONAL_TIME_VALUES_ARE_NOT_SUPPORTED = "fractional time values are not supported";
+
+    public void testNonFractionalTimeValues() {
+        final String s = randomAsciiOfLength(10) + randomTimeUnit();
+        final ElasticsearchParseException e =
+            expectThrows(ElasticsearchParseException.class, () -> TimeValue.parseTimeValue(s, null, "test"));
+        assertThat(e, hasToString(containsString("failed to parse [" + s + "]")));
+        assertThat(e, not(hasToString(containsString(FRACTIONAL_TIME_VALUES_ARE_NOT_SUPPORTED))));
+        assertThat(e.getCause(), instanceOf(NumberFormatException.class));
+    }
+
+    public void testFractionalTimeValues() {
+        double value;
+        do {
+            value = randomDouble();
+        } while (value == 0);
+        final String s = Double.toString(randomIntBetween(0, 128) + value) + randomTimeUnit();
+        final ElasticsearchParseException e =
+            expectThrows(ElasticsearchParseException.class, () -> TimeValue.parseTimeValue(s, null, "test"));
+        assertThat(e, hasToString(containsString("failed to parse [" + s + "]")));
+        assertThat(e, hasToString(containsString(FRACTIONAL_TIME_VALUES_ARE_NOT_SUPPORTED)));
+        assertThat(e.getCause(), instanceOf(NumberFormatException.class));
+    }
+
+    private String randomTimeUnit() {
+        return randomFrom("nanos", "micros", "ms", "s", "m", "h", "d");
+    }
+
     private void assertEqualityAfterSerialize(TimeValue value, int expectedSize) throws IOException {
         BytesStreamOutput out = new BytesStreamOutput();
         value.writeTo(out);

From 8b533b7ca9c315349edf329d2b8aa931548fe792 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Wed, 29 Jun 2016 11:09:35 -0700
Subject: [PATCH 1122/1311] Internal: Deprecate
 ExceptionsHelper.detailedMessage

This is a trappy "helper" and only hurts.
See #19069
---
 .../src/main/java/org/elasticsearch/ExceptionsHelper.java | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/core/src/main/java/org/elasticsearch/ExceptionsHelper.java b/core/src/main/java/org/elasticsearch/ExceptionsHelper.java
index 3842ab4e3bf..5996856cda0 100644
--- a/core/src/main/java/org/elasticsearch/ExceptionsHelper.java
+++ b/core/src/main/java/org/elasticsearch/ExceptionsHelper.java
@@ -89,10 +89,18 @@ public final class ExceptionsHelper {
         return result;
     }
 
+    /**
+     * @deprecated Don't swallow exceptions, allow them to propagate.
+     */
+    @Deprecated
     public static String detailedMessage(Throwable t) {
         return detailedMessage(t, false, 0);
     }
 
+    /**
+     * @deprecated Don't swallow exceptions, allow them to propagate.
+     */
+    @Deprecated
     public static String detailedMessage(Throwable t, boolean newLines, int initialCounter) {
         if (t == null) {
             return "Unknown";

From b2da5424b4048e375b263e06721a86ae31ec748a Mon Sep 17 00:00:00 2001
From: Britta Weber <brwe@users.noreply.github.com>
Date: Wed, 29 Jun 2016 20:18:25 +0200
Subject: [PATCH 1123/1311] [TEST] fix vagrant tests for seed with format
 ABC:DEF (#19157)

* [TEST] fix vagrant tests for seed with format ABC:DEF

Otherwise one gets an error message when passing
-Dtests.seed=ABC:DEF
to any test run.
---
 qa/vagrant/build.gradle | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle
index 89668a06534..878e9c5ef7f 100644
--- a/qa/vagrant/build.gradle
+++ b/qa/vagrant/build.gradle
@@ -58,15 +58,20 @@ for (String box : vagrantBoxes.split(',')) {
 }
 
 long seed
-String formattedSeed
+String formattedSeed = null
 String[] upgradeFromVersions
 String upgradeFromVersion
 
 String maybeTestsSeed = System.getProperty("tests.seed", null);
 if (maybeTestsSeed != null) {
-  seed = new BigInteger(maybeTestsSeed, 16).longValue()
-  formattedSeed = maybeTestsSeed
-} else {
+  List<String> seeds = maybeTestsSeed.tokenize(':')
+  if (seeds.size() != 0) {
+    String masterSeed = seeds.get(0)
+    seed = new BigInteger(masterSeed, 16).longValue()
+    formattedSeed = maybeTestsSeed
+  }
+}
+if (formattedSeed == null) {
   seed = new Random().nextLong()
   formattedSeed = String.format("%016X", seed)
 }

From b3daf7d683d28cc0943da20c748b75881d9f3196 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Wed, 29 Jun 2016 11:25:23 -0700
Subject: [PATCH 1124/1311] Remove unnecessary variant of detailedMessage

---
 .../org/elasticsearch/ExceptionsHelper.java   | 23 ++-----------------
 .../elasticsearch/index/store/StoreTests.java |  4 ++--
 2 files changed, 4 insertions(+), 23 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/ExceptionsHelper.java b/core/src/main/java/org/elasticsearch/ExceptionsHelper.java
index 5996856cda0..28f8d4391ec 100644
--- a/core/src/main/java/org/elasticsearch/ExceptionsHelper.java
+++ b/core/src/main/java/org/elasticsearch/ExceptionsHelper.java
@@ -94,18 +94,9 @@ public final class ExceptionsHelper {
      */
     @Deprecated
     public static String detailedMessage(Throwable t) {
-        return detailedMessage(t, false, 0);
-    }
-
-    /**
-     * @deprecated Don't swallow exceptions, allow them to propagate.
-     */
-    @Deprecated
-    public static String detailedMessage(Throwable t, boolean newLines, int initialCounter) {
         if (t == null) {
             return "Unknown";
         }
-        int counter = initialCounter + 1;
         if (t.getCause() != null) {
             StringBuilder sb = new StringBuilder();
             while (t != null) {
@@ -115,21 +106,11 @@ public final class ExceptionsHelper {
                     sb.append(t.getMessage());
                     sb.append("]");
                 }
-                if (!newLines) {
-                    sb.append("; ");
-                }
+                sb.append("; ");
                 t = t.getCause();
                 if (t != null) {
-                    if (newLines) {
-                        sb.append("\n");
-                        for (int i = 0; i < counter; i++) {
-                            sb.append("\t");
-                        }
-                    } else {
-                        sb.append("nested: ");
-                    }
+                    sb.append("nested: ");
                 }
-                counter++;
             }
             return sb.toString();
         } else {
diff --git a/core/src/test/java/org/elasticsearch/index/store/StoreTests.java b/core/src/test/java/org/elasticsearch/index/store/StoreTests.java
index e005fa400ef..e40f1c7f06f 100644
--- a/core/src/test/java/org/elasticsearch/index/store/StoreTests.java
+++ b/core/src/test/java/org/elasticsearch/index/store/StoreTests.java
@@ -1086,7 +1086,7 @@ public class StoreTests extends ESTestCase {
         String uuid = Store.CORRUPTED + UUIDs.randomBase64UUID();
         try (IndexOutput output = dir.createOutput(uuid, IOContext.DEFAULT)) {
             CodecUtil.writeHeader(output, Store.CODEC, Store.VERSION_STACK_TRACE);
-            output.writeString(ExceptionsHelper.detailedMessage(exception, true, 0));
+            output.writeString(ExceptionsHelper.detailedMessage(exception));
             output.writeString(ExceptionsHelper.stackTrace(exception));
             CodecUtil.writeFooter(output);
         }
@@ -1102,7 +1102,7 @@ public class StoreTests extends ESTestCase {
 
         try (IndexOutput output = dir.createOutput(uuid, IOContext.DEFAULT)) {
             CodecUtil.writeHeader(output, Store.CODEC, Store.VERSION_START);
-            output.writeString(ExceptionsHelper.detailedMessage(exception, true, 0));
+            output.writeString(ExceptionsHelper.detailedMessage(exception));
             CodecUtil.writeFooter(output);
         }
         try {

From 00356edd338613423ef497751b413f1dc06cce43 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 29 Jun 2016 17:02:15 -0400
Subject: [PATCH 1125/1311] Clarify time units usage in docs

This commit clarifies the distinction between supported time units for
durations and supported time units for durations in the docs.

Relates #19159
---
 .../bucket/datehistogram-aggregation.asciidoc |  9 +++--
 .../diversified-sampler-aggregation.asciidoc  |  6 ++--
 docs/reference/api-conventions.asciidoc       | 36 +++++++++++--------
 .../migrate_5_0/aggregations.asciidoc         |  5 +++
 .../migration/migrate_5_0/settings.asciidoc   |  5 ++-
 5 files changed, 36 insertions(+), 25 deletions(-)

diff --git a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
index 29ba5e49626..4d9147067ec 100644
--- a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc
@@ -26,8 +26,9 @@ Requesting bucket intervals of a month.
 
 Available expressions for interval: `year`, `quarter`, `month`, `week`, `day`, `hour`, `minute`, `second`
 
-
-Fractional values are allowed for seconds, minutes, hours, days and weeks. For example 1.5 hours:
+Time values can also be specified via abbreviations supported by <<time-units,time units>> parsing.
+Note that fractional time values are not supported, but you can address this by shifting to another
+time unit (e.g., `1.5h` could instead be specified as `90m`).
 
 [source,js]
 --------------------------------------------------
@@ -36,15 +37,13 @@ Fractional values are allowed for seconds, minutes, hours, days and weeks. For e
         "articles_over_time" : {
             "date_histogram" : {
                 "field" : "date",
-                "interval" : "1.5h"
+                "interval" : "90m"
             }
         }
     }
 }
 --------------------------------------------------
 
-See <<time-units>> for accepted abbreviations.
-
 ==== Keys
 
 Internally, a date is represented as a 64 bit number representing a timestamp
diff --git a/docs/reference/aggregations/bucket/diversified-sampler-aggregation.asciidoc b/docs/reference/aggregations/bucket/diversified-sampler-aggregation.asciidoc
index 0307840d685..70412d2680a 100644
--- a/docs/reference/aggregations/bucket/diversified-sampler-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/diversified-sampler-aggregation.asciidoc
@@ -153,5 +153,7 @@ In this situation an error will be thrown.
 The de-duplication logic in the diversify settings applies only at a shard level so will not apply across shards.
 
 ===== No specialized syntax for geo/date fields
-Currently the syntax for defining the diversifying values is defined by a choice of `field` or `script` - there is no added syntactical sugar for expressing geo or date units such as "1w" (1 week).
-This support may be added in a later release and users will currently have to create these sorts of values using a script.
+Currently the syntax for defining the diversifying values is defined by a choice of `field` or
+`script` - there is no added syntactical sugar for expressing geo or date units such as "7d" (7
+days). This support may be added in a later release and users will currently have to create these
+sorts of values using a script.
diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc
index 4e487325829..26c1a7476ed 100644
--- a/docs/reference/api-conventions.asciidoc
+++ b/docs/reference/api-conventions.asciidoc
@@ -171,8 +171,18 @@ one or more maths expressions:
 * `-1d` - subtract one day
 * `/d`  - round down to the nearest day
 
-The supported <<time-units,time units>> are: `y` (year), `M` (month), `w` (week),
-`d` (day), `h` (hour), `m` (minute), and `s` (second).
+The supported time units differ than those supported by <<time-units, time units>> for durations.
+The supported units are:
+
+[horizontal]
+`y`:: years
+`M`:: months
+`w`:: weeks
+`d`:: days
+`h`:: hours
+`H`:: hours
+`m`:: minutes
+`s`:: seconds
 
 Some examples are:
 
@@ -348,21 +358,17 @@ of supporting the native JSON number types.
 [float]
 === Time units
 
-Whenever durations need to be specified, eg for a `timeout` parameter, the
-duration must specify the unit, like `2d` for 2 days.  The supported units
-are:
+Whenever durations need to be specified, e.g. for a `timeout` parameter, the duration must specify
+the unit, like `2d` for 2 days.  The supported units are:
 
 [horizontal]
-`y`::   Year
-`M`::   Month
-`w`::   Week
-`d`::   Day
-`h`::   Hour
-`m`::   Minute
-`s`::   Second
-`ms`::  Milli-second
-`micros`::  Micro-second
-`nanos`::  Nano-second
+`d`::       days
+`h`::       hours
+`m`::       minutes
+`s`::       seconds
+`ms`::      milliseconds
+`micros`::  microseconds
+`nanos`::   nanoseconds
 
 [[byte-units]]
 [float]
diff --git a/docs/reference/migration/migrate_5_0/aggregations.asciidoc b/docs/reference/migration/migrate_5_0/aggregations.asciidoc
index d9227e91385..287da1efb99 100644
--- a/docs/reference/migration/migrate_5_0/aggregations.asciidoc
+++ b/docs/reference/migration/migrate_5_0/aggregations.asciidoc
@@ -26,3 +26,8 @@ for `from` and `to` anymore.
 `size: 0` is no longer valid for the terms, significant terms and geohash grid
 aggregations. Instead a size should be explicitly specified with a number greater
 than zero.
+
+==== Fractional time values
+
+Fractional time values (e.g., 0.5s) are no longer supported. For example, this means when setting
+date histogram intervals "1.5h" will be rejected and should instead be input as "90m".
diff --git a/docs/reference/migration/migrate_5_0/settings.asciidoc b/docs/reference/migration/migrate_5_0/settings.asciidoc
index 8a7c386acdb..ffe69aa3cfb 100644
--- a/docs/reference/migration/migrate_5_0/settings.asciidoc
+++ b/docs/reference/migration/migrate_5_0/settings.asciidoc
@@ -291,6 +291,5 @@ still defaults to `true` in that case.
 
 The unit 'w' representing weeks is no longer supported.
 
-Fractional time values (e.g., 0.5s) are no longer supported. For
-example, this means when setting timeouts "0.5s" will be rejected and
-should instead be input as "500ms".
+Fractional time values (e.g., 0.5s) are no longer supported. For example, this means when setting
+timeouts "0.5s" will be rejected and should instead be input as "500ms".

From 8db43c01079aaec4f5ca5c9adb89364e63eb9bbe Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 29 Jun 2016 14:55:31 -0400
Subject: [PATCH 1126/1311] Move RestHandler registration to ActionModule and
 ActionPlugin

`RestHandler`s are highly tied to actions so registering them in the
same place makes sense.

Removes the need to for plugins to check if they are in transport client
mode before registering a RestHandler - `getRestHandlers` isn't called
at all in transport client mode.

This caused guice to throw a massive fit about the circular dependency
between NodeClient and the allocation deciders. I broke the circular
dependency by registering the actions map with the node client after
instantiation.
---
 .../elasticsearch/action/ActionModule.java    | 279 +++++++++++++++++-
 .../elasticsearch/client/node/NodeClient.java |  16 +-
 .../metadata/IndexNameExpressionResolver.java |   2 -
 .../elasticsearch/common/NamedRegistry.java   |   2 +-
 .../common/network/NetworkModule.java         | 266 -----------------
 .../java/org/elasticsearch/node/Node.java     |  11 +-
 .../elasticsearch/plugins/ActionPlugin.java   |  34 +++
 .../elasticsearch/rest/RestController.java    |   2 -
 .../action/ActionModuleTests.java             | 129 ++++++++
 .../client/node/NodeClientHeadersTests.java   |   4 +-
 .../common/network/NetworkModuleTests.java    |  29 --
 .../TestResponseHeaderPlugin.java             |  13 +-
 .../migration/migrate_5_0/plugins.asciidoc    |   3 +
 .../script/mustache/MustachePlugin.java       |  15 +-
 .../percolator/PercolatorPlugin.java          |  12 +-
 .../index/reindex/ReindexPlugin.java          |  11 +-
 16 files changed, 488 insertions(+), 340 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/action/ActionModuleTests.java

diff --git a/core/src/main/java/org/elasticsearch/action/ActionModule.java b/core/src/main/java/org/elasticsearch/action/ActionModule.java
index f5c83e272ce..5ce7aaa4d95 100644
--- a/core/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/core/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -194,14 +194,120 @@ import org.elasticsearch.common.NamedRegistry;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.multibindings.MapBinder;
 import org.elasticsearch.common.inject.multibindings.Multibinder;
+import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.ActionPlugin.ActionHandler;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestHandler;
+import org.elasticsearch.rest.action.admin.cluster.allocation.RestClusterAllocationExplainAction;
+import org.elasticsearch.rest.action.admin.cluster.health.RestClusterHealthAction;
+import org.elasticsearch.rest.action.admin.cluster.node.hotthreads.RestNodesHotThreadsAction;
+import org.elasticsearch.rest.action.admin.cluster.node.info.RestNodesInfoAction;
+import org.elasticsearch.rest.action.admin.cluster.node.stats.RestNodesStatsAction;
+import org.elasticsearch.rest.action.admin.cluster.node.tasks.RestCancelTasksAction;
+import org.elasticsearch.rest.action.admin.cluster.node.tasks.RestGetTaskAction;
+import org.elasticsearch.rest.action.admin.cluster.node.tasks.RestListTasksAction;
+import org.elasticsearch.rest.action.admin.cluster.repositories.delete.RestDeleteRepositoryAction;
+import org.elasticsearch.rest.action.admin.cluster.repositories.get.RestGetRepositoriesAction;
+import org.elasticsearch.rest.action.admin.cluster.repositories.put.RestPutRepositoryAction;
+import org.elasticsearch.rest.action.admin.cluster.repositories.verify.RestVerifyRepositoryAction;
+import org.elasticsearch.rest.action.admin.cluster.reroute.RestClusterRerouteAction;
+import org.elasticsearch.rest.action.admin.cluster.settings.RestClusterGetSettingsAction;
+import org.elasticsearch.rest.action.admin.cluster.settings.RestClusterUpdateSettingsAction;
+import org.elasticsearch.rest.action.admin.cluster.shards.RestClusterSearchShardsAction;
+import org.elasticsearch.rest.action.admin.cluster.snapshots.create.RestCreateSnapshotAction;
+import org.elasticsearch.rest.action.admin.cluster.snapshots.delete.RestDeleteSnapshotAction;
+import org.elasticsearch.rest.action.admin.cluster.snapshots.get.RestGetSnapshotsAction;
+import org.elasticsearch.rest.action.admin.cluster.snapshots.restore.RestRestoreSnapshotAction;
+import org.elasticsearch.rest.action.admin.cluster.snapshots.status.RestSnapshotsStatusAction;
+import org.elasticsearch.rest.action.admin.cluster.state.RestClusterStateAction;
+import org.elasticsearch.rest.action.admin.cluster.stats.RestClusterStatsAction;
+import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestDeleteStoredScriptAction;
+import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestGetStoredScriptAction;
+import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestPutStoredScriptAction;
+import org.elasticsearch.rest.action.admin.cluster.tasks.RestPendingClusterTasksAction;
+import org.elasticsearch.rest.action.admin.indices.RestRolloverIndexAction;
+import org.elasticsearch.rest.action.admin.indices.RestShrinkIndexAction;
+import org.elasticsearch.rest.action.admin.indices.alias.RestIndicesAliasesAction;
+import org.elasticsearch.rest.action.admin.indices.alias.delete.RestIndexDeleteAliasesAction;
+import org.elasticsearch.rest.action.admin.indices.alias.get.RestGetAliasesAction;
+import org.elasticsearch.rest.action.admin.indices.alias.head.RestAliasesExistAction;
+import org.elasticsearch.rest.action.admin.indices.alias.put.RestIndexPutAliasAction;
+import org.elasticsearch.rest.action.admin.indices.analyze.RestAnalyzeAction;
+import org.elasticsearch.rest.action.admin.indices.cache.clear.RestClearIndicesCacheAction;
+import org.elasticsearch.rest.action.admin.indices.close.RestCloseIndexAction;
+import org.elasticsearch.rest.action.admin.indices.create.RestCreateIndexAction;
+import org.elasticsearch.rest.action.admin.indices.delete.RestDeleteIndexAction;
+import org.elasticsearch.rest.action.admin.indices.exists.indices.RestIndicesExistsAction;
+import org.elasticsearch.rest.action.admin.indices.exists.types.RestTypesExistsAction;
+import org.elasticsearch.rest.action.admin.indices.flush.RestFlushAction;
+import org.elasticsearch.rest.action.admin.indices.flush.RestSyncedFlushAction;
+import org.elasticsearch.rest.action.admin.indices.forcemerge.RestForceMergeAction;
+import org.elasticsearch.rest.action.admin.indices.get.RestGetIndicesAction;
+import org.elasticsearch.rest.action.admin.indices.mapping.get.RestGetFieldMappingAction;
+import org.elasticsearch.rest.action.admin.indices.mapping.get.RestGetMappingAction;
+import org.elasticsearch.rest.action.admin.indices.mapping.put.RestPutMappingAction;
+import org.elasticsearch.rest.action.admin.indices.open.RestOpenIndexAction;
+import org.elasticsearch.rest.action.admin.indices.recovery.RestRecoveryAction;
+import org.elasticsearch.rest.action.admin.indices.refresh.RestRefreshAction;
+import org.elasticsearch.rest.action.admin.indices.segments.RestIndicesSegmentsAction;
+import org.elasticsearch.rest.action.admin.indices.settings.RestGetSettingsAction;
+import org.elasticsearch.rest.action.admin.indices.settings.RestUpdateSettingsAction;
+import org.elasticsearch.rest.action.admin.indices.shards.RestIndicesShardStoresAction;
+import org.elasticsearch.rest.action.admin.indices.stats.RestIndicesStatsAction;
+import org.elasticsearch.rest.action.admin.indices.template.delete.RestDeleteIndexTemplateAction;
+import org.elasticsearch.rest.action.admin.indices.template.get.RestGetIndexTemplateAction;
+import org.elasticsearch.rest.action.admin.indices.template.head.RestHeadIndexTemplateAction;
+import org.elasticsearch.rest.action.admin.indices.template.put.RestPutIndexTemplateAction;
+import org.elasticsearch.rest.action.admin.indices.upgrade.RestUpgradeAction;
+import org.elasticsearch.rest.action.admin.indices.validate.query.RestValidateQueryAction;
+import org.elasticsearch.rest.action.bulk.RestBulkAction;
+import org.elasticsearch.rest.action.cat.AbstractCatAction;
+import org.elasticsearch.rest.action.cat.RestAliasAction;
+import org.elasticsearch.rest.action.cat.RestAllocationAction;
+import org.elasticsearch.rest.action.cat.RestCatAction;
+import org.elasticsearch.rest.action.cat.RestFielddataAction;
+import org.elasticsearch.rest.action.cat.RestHealthAction;
+import org.elasticsearch.rest.action.cat.RestIndicesAction;
+import org.elasticsearch.rest.action.cat.RestMasterAction;
+import org.elasticsearch.rest.action.cat.RestNodeAttrsAction;
+import org.elasticsearch.rest.action.cat.RestNodesAction;
+import org.elasticsearch.rest.action.cat.RestPluginsAction;
+import org.elasticsearch.rest.action.cat.RestRepositoriesAction;
+import org.elasticsearch.rest.action.cat.RestSegmentsAction;
+import org.elasticsearch.rest.action.cat.RestShardsAction;
+import org.elasticsearch.rest.action.cat.RestSnapshotAction;
+import org.elasticsearch.rest.action.cat.RestTasksAction;
+import org.elasticsearch.rest.action.cat.RestThreadPoolAction;
+import org.elasticsearch.rest.action.delete.RestDeleteAction;
+import org.elasticsearch.rest.action.explain.RestExplainAction;
+import org.elasticsearch.rest.action.fieldstats.RestFieldStatsAction;
+import org.elasticsearch.rest.action.get.RestGetAction;
+import org.elasticsearch.rest.action.get.RestGetSourceAction;
+import org.elasticsearch.rest.action.get.RestHeadAction;
+import org.elasticsearch.rest.action.get.RestMultiGetAction;
+import org.elasticsearch.rest.action.index.RestIndexAction;
+import org.elasticsearch.rest.action.ingest.RestDeletePipelineAction;
+import org.elasticsearch.rest.action.ingest.RestGetPipelineAction;
+import org.elasticsearch.rest.action.ingest.RestPutPipelineAction;
+import org.elasticsearch.rest.action.ingest.RestSimulatePipelineAction;
+import org.elasticsearch.rest.action.main.RestMainAction;
+import org.elasticsearch.rest.action.search.RestClearScrollAction;
+import org.elasticsearch.rest.action.search.RestMultiSearchAction;
+import org.elasticsearch.rest.action.search.RestSearchAction;
+import org.elasticsearch.rest.action.search.RestSearchScrollAction;
+import org.elasticsearch.rest.action.suggest.RestSuggestAction;
+import org.elasticsearch.rest.action.termvectors.RestMultiTermVectorsAction;
+import org.elasticsearch.rest.action.termvectors.RestTermVectorsAction;
+import org.elasticsearch.rest.action.update.RestUpdateAction;
 
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import static java.util.Collections.unmodifiableList;
 import static java.util.Collections.unmodifiableMap;
@@ -212,21 +318,27 @@ import static java.util.Collections.unmodifiableMap;
 public class ActionModule extends AbstractModule {
 
     private final boolean transportClient;
+    private final Settings settings;
+    private final List<ActionPlugin> actionPlugins;
     private final Map<String, ActionHandler<?, ?>> actions;
     private final List<Class<? extends ActionFilter>> actionFilters;
     private final AutoCreateIndex autoCreateIndex;
     private final DestructiveOperations destructiveOperations;
+    private final RestController restController;
 
     public ActionModule(boolean ingestEnabled, boolean transportClient, Settings settings, IndexNameExpressionResolver resolver,
             ClusterSettings clusterSettings, List<ActionPlugin> actionPlugins) {
         this.transportClient = transportClient;
+        this.settings = settings;
+        this.actionPlugins = actionPlugins;
         actions = setupActions(actionPlugins);
         actionFilters = setupActionFilters(actionPlugins, ingestEnabled);
         autoCreateIndex = transportClient ? null : new AutoCreateIndex(settings, resolver);
         destructiveOperations = new DestructiveOperations(settings, clusterSettings);
+        restController = new RestController(settings);
     }
 
-    private Map<String, ActionHandler<?, ?>> setupActions(List<ActionPlugin> actionPlugins) {
+    static Map<String, ActionHandler<?, ?>> setupActions(List<ActionPlugin> actionPlugins) {
         // Subclass NamedRegistry for easy registration
         class ActionRegistry extends NamedRegistry<ActionHandler<?, ?>> {
             public ActionRegistry() {
@@ -357,6 +469,149 @@ public class ActionModule extends AbstractModule {
         return unmodifiableList(filters);
     }
 
+    static Set<Class<? extends RestHandler>> setupRestHandlers(List<ActionPlugin> actionPlugins) {
+        Set<Class<? extends RestHandler>> handlers = new HashSet<>();
+        registerRestHandler(handlers, RestMainAction.class);
+        registerRestHandler(handlers, RestNodesInfoAction.class);
+        registerRestHandler(handlers, RestNodesStatsAction.class);
+        registerRestHandler(handlers, RestNodesHotThreadsAction.class);
+        registerRestHandler(handlers, RestClusterAllocationExplainAction.class);
+        registerRestHandler(handlers, RestClusterStatsAction.class);
+        registerRestHandler(handlers, RestClusterStateAction.class);
+        registerRestHandler(handlers, RestClusterHealthAction.class);
+        registerRestHandler(handlers, RestClusterUpdateSettingsAction.class);
+        registerRestHandler(handlers, RestClusterGetSettingsAction.class);
+        registerRestHandler(handlers, RestClusterRerouteAction.class);
+        registerRestHandler(handlers, RestClusterSearchShardsAction.class);
+        registerRestHandler(handlers, RestPendingClusterTasksAction.class);
+        registerRestHandler(handlers, RestPutRepositoryAction.class);
+        registerRestHandler(handlers, RestGetRepositoriesAction.class);
+        registerRestHandler(handlers, RestDeleteRepositoryAction.class);
+        registerRestHandler(handlers, RestVerifyRepositoryAction.class);
+        registerRestHandler(handlers, RestGetSnapshotsAction.class);
+        registerRestHandler(handlers, RestCreateSnapshotAction.class);
+        registerRestHandler(handlers, RestRestoreSnapshotAction.class);
+        registerRestHandler(handlers, RestDeleteSnapshotAction.class);
+        registerRestHandler(handlers, RestSnapshotsStatusAction.class);
+
+        registerRestHandler(handlers, RestIndicesExistsAction.class);
+        registerRestHandler(handlers, RestTypesExistsAction.class);
+        registerRestHandler(handlers, RestGetIndicesAction.class);
+        registerRestHandler(handlers, RestIndicesStatsAction.class);
+        registerRestHandler(handlers, RestIndicesSegmentsAction.class);
+        registerRestHandler(handlers, RestIndicesShardStoresAction.class);
+        registerRestHandler(handlers, RestGetAliasesAction.class);
+        registerRestHandler(handlers, RestAliasesExistAction.class);
+        registerRestHandler(handlers, RestIndexDeleteAliasesAction.class);
+        registerRestHandler(handlers, RestIndexPutAliasAction.class);
+        registerRestHandler(handlers, RestIndicesAliasesAction.class);
+        registerRestHandler(handlers, RestCreateIndexAction.class);
+        registerRestHandler(handlers, RestShrinkIndexAction.class);
+        registerRestHandler(handlers, RestRolloverIndexAction.class);
+        registerRestHandler(handlers, RestDeleteIndexAction.class);
+        registerRestHandler(handlers, RestCloseIndexAction.class);
+        registerRestHandler(handlers, RestOpenIndexAction.class);
+
+        registerRestHandler(handlers, RestUpdateSettingsAction.class);
+        registerRestHandler(handlers, RestGetSettingsAction.class);
+
+        registerRestHandler(handlers, RestAnalyzeAction.class);
+        registerRestHandler(handlers, RestGetIndexTemplateAction.class);
+        registerRestHandler(handlers, RestPutIndexTemplateAction.class);
+        registerRestHandler(handlers, RestDeleteIndexTemplateAction.class);
+        registerRestHandler(handlers, RestHeadIndexTemplateAction.class);
+
+        registerRestHandler(handlers, RestPutMappingAction.class);
+        registerRestHandler(handlers, RestGetMappingAction.class);
+        registerRestHandler(handlers, RestGetFieldMappingAction.class);
+
+        registerRestHandler(handlers, RestRefreshAction.class);
+        registerRestHandler(handlers, RestFlushAction.class);
+        registerRestHandler(handlers, RestSyncedFlushAction.class);
+        registerRestHandler(handlers, RestForceMergeAction.class);
+        registerRestHandler(handlers, RestUpgradeAction.class);
+        registerRestHandler(handlers, RestClearIndicesCacheAction.class);
+
+        registerRestHandler(handlers, RestIndexAction.class);
+        registerRestHandler(handlers, RestGetAction.class);
+        registerRestHandler(handlers, RestGetSourceAction.class);
+        registerRestHandler(handlers, RestHeadAction.Document.class);
+        registerRestHandler(handlers, RestHeadAction.Source.class);
+        registerRestHandler(handlers, RestMultiGetAction.class);
+        registerRestHandler(handlers, RestDeleteAction.class);
+        registerRestHandler(handlers, org.elasticsearch.rest.action.count.RestCountAction.class);
+        registerRestHandler(handlers, RestSuggestAction.class);
+        registerRestHandler(handlers, RestTermVectorsAction.class);
+        registerRestHandler(handlers, RestMultiTermVectorsAction.class);
+        registerRestHandler(handlers, RestBulkAction.class);
+        registerRestHandler(handlers, RestUpdateAction.class);
+
+        registerRestHandler(handlers, RestSearchAction.class);
+        registerRestHandler(handlers, RestSearchScrollAction.class);
+        registerRestHandler(handlers, RestClearScrollAction.class);
+        registerRestHandler(handlers, RestMultiSearchAction.class);
+
+        registerRestHandler(handlers, RestValidateQueryAction.class);
+
+        registerRestHandler(handlers, RestExplainAction.class);
+
+        registerRestHandler(handlers, RestRecoveryAction.class);
+
+        // Scripts API
+        registerRestHandler(handlers, RestGetStoredScriptAction.class);
+        registerRestHandler(handlers, RestPutStoredScriptAction.class);
+        registerRestHandler(handlers, RestDeleteStoredScriptAction.class);
+
+        registerRestHandler(handlers, RestFieldStatsAction.class);
+
+        // Tasks API
+        registerRestHandler(handlers, RestListTasksAction.class);
+        registerRestHandler(handlers, RestGetTaskAction.class);
+        registerRestHandler(handlers, RestCancelTasksAction.class);
+
+        // Ingest API
+        registerRestHandler(handlers, RestPutPipelineAction.class);
+        registerRestHandler(handlers, RestGetPipelineAction.class);
+        registerRestHandler(handlers, RestDeletePipelineAction.class);
+        registerRestHandler(handlers, RestSimulatePipelineAction.class);
+
+        // CAT API
+        registerRestHandler(handlers, RestCatAction.class);
+        registerRestHandler(handlers, RestAllocationAction.class);
+        registerRestHandler(handlers, RestShardsAction.class);
+        registerRestHandler(handlers, RestMasterAction.class);
+        registerRestHandler(handlers, RestNodesAction.class);
+        registerRestHandler(handlers, RestTasksAction.class);
+        registerRestHandler(handlers, RestIndicesAction.class);
+        registerRestHandler(handlers, RestSegmentsAction.class);
+        // Fully qualified to prevent interference with rest.action.count.RestCountAction
+        registerRestHandler(handlers, org.elasticsearch.rest.action.cat.RestCountAction.class);
+        // Fully qualified to prevent interference with rest.action.indices.RestRecoveryAction
+        registerRestHandler(handlers, org.elasticsearch.rest.action.cat.RestRecoveryAction.class);
+        registerRestHandler(handlers, RestHealthAction.class);
+        registerRestHandler(handlers, org.elasticsearch.rest.action.cat.RestPendingClusterTasksAction.class);
+        registerRestHandler(handlers, RestAliasAction.class);
+        registerRestHandler(handlers, RestThreadPoolAction.class);
+        registerRestHandler(handlers, RestPluginsAction.class);
+        registerRestHandler(handlers, RestFielddataAction.class);
+        registerRestHandler(handlers, RestNodeAttrsAction.class);
+        registerRestHandler(handlers, RestRepositoriesAction.class);
+        registerRestHandler(handlers, RestSnapshotAction.class);
+        for (ActionPlugin plugin : actionPlugins) {
+            for (Class<? extends RestHandler> handler : plugin.getRestHandlers()) {
+                registerRestHandler(handlers, handler);
+            }
+        }
+        return handlers;
+    }
+
+    private static void registerRestHandler(Set<Class<? extends RestHandler>> handlers, Class<? extends RestHandler> handler) {
+        if (handlers.contains(handler)) {
+            throw new IllegalArgumentException("can't register the same [rest_handler] more than once for [" + handler.getName() + "]");
+        }
+        handlers.add(handler);
+    }
+
     @Override
     protected void configure() {
         Multibinder<ActionFilter> actionFilterMultibinder = Multibinder.newSetBinder(binder(), ActionFilter.class);
@@ -374,11 +629,12 @@ public class ActionModule extends AbstractModule {
         for (Map.Entry<String, ActionHandler<?, ?>> entry : actions.entrySet()) {
             actionsBinder.addBinding(entry.getKey()).toInstance(entry.getValue().getAction());
         }
-        // register GenericAction -> transportAction Map that can be injected to instances.
-        // also register any supporting classes
         if (false == transportClient) {
+            // Supporting classes only used when not a transport client
             bind(AutoCreateIndex.class).toInstance(autoCreateIndex);
             bind(TransportLivenessAction.class).asEagerSingleton();
+
+            // register GenericAction -> transportAction Map used by NodeClient
             @SuppressWarnings("rawtypes")
             MapBinder<GenericAction, TransportAction> transportActionsBinder
                     = MapBinder.newMapBinder(binder(), GenericAction.class, TransportAction.class);
@@ -390,6 +646,23 @@ public class ActionModule extends AbstractModule {
                     bind(supportAction).asEagerSingleton();
                 }
             }
+
+            // Bind the RestController which is required (by Node) even if rest isn't enabled.
+            bind(RestController.class).toInstance(restController);
+
+            // Setup the RestHandlers
+            if (NetworkModule.HTTP_ENABLED.get(settings)) {
+                Multibinder<RestHandler> restHandlers = Multibinder.newSetBinder(binder(), RestHandler.class);
+                Multibinder<AbstractCatAction> catHandlers = Multibinder.newSetBinder(binder(), AbstractCatAction.class);
+                for (Class<? extends RestHandler> handler : setupRestHandlers(actionPlugins)) {
+                    bind(handler).asEagerSingleton();
+                    if (AbstractCatAction.class.isAssignableFrom(handler)) {
+                        catHandlers.addBinding().to(handler.asSubclass(AbstractCatAction.class));
+                    } else {
+                        restHandlers.addBinding().to(handler);
+                    }
+                }
+            }
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/client/node/NodeClient.java b/core/src/main/java/org/elasticsearch/client/node/NodeClient.java
index 3e9bed9e25d..59909df428b 100644
--- a/core/src/main/java/org/elasticsearch/client/node/NodeClient.java
+++ b/core/src/main/java/org/elasticsearch/client/node/NodeClient.java
@@ -27,25 +27,24 @@ import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.GenericAction;
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.client.support.AbstractClient;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Map;
 
-import static java.util.Collections.unmodifiableMap;
-
 /**
  *
  */
 public class NodeClient extends AbstractClient {
 
-    private final Map<GenericAction, TransportAction> actions;
+    private Map<GenericAction, TransportAction> actions;
 
-    @Inject
-    public NodeClient(Settings settings, ThreadPool threadPool, Map<GenericAction, TransportAction> actions) {
+    public NodeClient(Settings settings, ThreadPool threadPool) {
         super(settings, threadPool);
-        this.actions = unmodifiableMap(actions);
+    }
+
+    public void intialize(Map<GenericAction, TransportAction> actions) {
+        this.actions = actions;
     }
 
     @Override
@@ -57,6 +56,9 @@ public class NodeClient extends AbstractClient {
     @Override
     public <Request extends ActionRequest<Request>, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void doExecute(
             Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
+        if (actions == null) {
+            throw new IllegalStateException("NodeClient has not been initialized");
+        }
         TransportAction<Request, Response> transportAction = actions.get(action);
         if (transportAction == null) {
             throw new IllegalStateException("failed to find action [" + action + "] to execute");
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
index 2abbea04d51..4eefac0f17f 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
@@ -27,7 +27,6 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.joda.DateMathParser;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
 import org.elasticsearch.common.regex.Regex;
@@ -57,7 +56,6 @@ public class IndexNameExpressionResolver extends AbstractComponent {
     private final List<ExpressionResolver> expressionResolvers;
     private final DateMathExpressionResolver dateMathExpressionResolver;
 
-    @Inject
     public IndexNameExpressionResolver(Settings settings) {
         super(settings);
         expressionResolvers = Arrays.asList(
diff --git a/core/src/main/java/org/elasticsearch/common/NamedRegistry.java b/core/src/main/java/org/elasticsearch/common/NamedRegistry.java
index 7573b5268fd..c326da7a495 100644
--- a/core/src/main/java/org/elasticsearch/common/NamedRegistry.java
+++ b/core/src/main/java/org/elasticsearch/common/NamedRegistry.java
@@ -45,7 +45,7 @@ public class NamedRegistry<T> {
         requireNonNull(name, "name is required");
         requireNonNull(t, targetName + " is required");
         if (registry.putIfAbsent(name, t) != null) {
-            throw new IllegalArgumentException(targetName + " for name " + name + " already registered");
+            throw new IllegalArgumentException(targetName + " for name [" + name + "] already registered");
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
index b0287ad153a..2b94f224b9c 100644
--- a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
+++ b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
@@ -41,109 +41,6 @@ import org.elasticsearch.common.util.ExtensionPoint;
 import org.elasticsearch.http.HttpServer;
 import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.netty.NettyHttpServerTransport;
-import org.elasticsearch.rest.RestController;
-import org.elasticsearch.rest.RestHandler;
-import org.elasticsearch.rest.action.admin.cluster.allocation.RestClusterAllocationExplainAction;
-import org.elasticsearch.rest.action.admin.cluster.health.RestClusterHealthAction;
-import org.elasticsearch.rest.action.admin.cluster.node.hotthreads.RestNodesHotThreadsAction;
-import org.elasticsearch.rest.action.admin.cluster.node.info.RestNodesInfoAction;
-import org.elasticsearch.rest.action.admin.cluster.node.stats.RestNodesStatsAction;
-import org.elasticsearch.rest.action.admin.cluster.node.tasks.RestCancelTasksAction;
-import org.elasticsearch.rest.action.admin.cluster.node.tasks.RestGetTaskAction;
-import org.elasticsearch.rest.action.admin.cluster.node.tasks.RestListTasksAction;
-import org.elasticsearch.rest.action.admin.cluster.repositories.delete.RestDeleteRepositoryAction;
-import org.elasticsearch.rest.action.admin.cluster.repositories.get.RestGetRepositoriesAction;
-import org.elasticsearch.rest.action.admin.cluster.repositories.put.RestPutRepositoryAction;
-import org.elasticsearch.rest.action.admin.cluster.repositories.verify.RestVerifyRepositoryAction;
-import org.elasticsearch.rest.action.admin.cluster.reroute.RestClusterRerouteAction;
-import org.elasticsearch.rest.action.admin.cluster.settings.RestClusterGetSettingsAction;
-import org.elasticsearch.rest.action.admin.cluster.settings.RestClusterUpdateSettingsAction;
-import org.elasticsearch.rest.action.admin.cluster.shards.RestClusterSearchShardsAction;
-import org.elasticsearch.rest.action.admin.cluster.snapshots.create.RestCreateSnapshotAction;
-import org.elasticsearch.rest.action.admin.cluster.snapshots.delete.RestDeleteSnapshotAction;
-import org.elasticsearch.rest.action.admin.cluster.snapshots.get.RestGetSnapshotsAction;
-import org.elasticsearch.rest.action.admin.cluster.snapshots.restore.RestRestoreSnapshotAction;
-import org.elasticsearch.rest.action.admin.cluster.snapshots.status.RestSnapshotsStatusAction;
-import org.elasticsearch.rest.action.admin.cluster.state.RestClusterStateAction;
-import org.elasticsearch.rest.action.admin.cluster.stats.RestClusterStatsAction;
-import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestDeleteStoredScriptAction;
-import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestGetStoredScriptAction;
-import org.elasticsearch.rest.action.admin.cluster.storedscripts.RestPutStoredScriptAction;
-import org.elasticsearch.rest.action.admin.cluster.tasks.RestPendingClusterTasksAction;
-import org.elasticsearch.rest.action.admin.indices.RestRolloverIndexAction;
-import org.elasticsearch.rest.action.admin.indices.RestShrinkIndexAction;
-import org.elasticsearch.rest.action.admin.indices.alias.RestIndicesAliasesAction;
-import org.elasticsearch.rest.action.admin.indices.alias.delete.RestIndexDeleteAliasesAction;
-import org.elasticsearch.rest.action.admin.indices.alias.get.RestGetAliasesAction;
-import org.elasticsearch.rest.action.admin.indices.alias.head.RestAliasesExistAction;
-import org.elasticsearch.rest.action.admin.indices.alias.put.RestIndexPutAliasAction;
-import org.elasticsearch.rest.action.admin.indices.analyze.RestAnalyzeAction;
-import org.elasticsearch.rest.action.admin.indices.cache.clear.RestClearIndicesCacheAction;
-import org.elasticsearch.rest.action.admin.indices.close.RestCloseIndexAction;
-import org.elasticsearch.rest.action.admin.indices.create.RestCreateIndexAction;
-import org.elasticsearch.rest.action.admin.indices.delete.RestDeleteIndexAction;
-import org.elasticsearch.rest.action.admin.indices.exists.indices.RestIndicesExistsAction;
-import org.elasticsearch.rest.action.admin.indices.exists.types.RestTypesExistsAction;
-import org.elasticsearch.rest.action.admin.indices.flush.RestFlushAction;
-import org.elasticsearch.rest.action.admin.indices.flush.RestSyncedFlushAction;
-import org.elasticsearch.rest.action.admin.indices.forcemerge.RestForceMergeAction;
-import org.elasticsearch.rest.action.admin.indices.get.RestGetIndicesAction;
-import org.elasticsearch.rest.action.admin.indices.mapping.get.RestGetFieldMappingAction;
-import org.elasticsearch.rest.action.admin.indices.mapping.get.RestGetMappingAction;
-import org.elasticsearch.rest.action.admin.indices.mapping.put.RestPutMappingAction;
-import org.elasticsearch.rest.action.admin.indices.open.RestOpenIndexAction;
-import org.elasticsearch.rest.action.admin.indices.recovery.RestRecoveryAction;
-import org.elasticsearch.rest.action.admin.indices.refresh.RestRefreshAction;
-import org.elasticsearch.rest.action.admin.indices.segments.RestIndicesSegmentsAction;
-import org.elasticsearch.rest.action.admin.indices.settings.RestGetSettingsAction;
-import org.elasticsearch.rest.action.admin.indices.settings.RestUpdateSettingsAction;
-import org.elasticsearch.rest.action.admin.indices.shards.RestIndicesShardStoresAction;
-import org.elasticsearch.rest.action.admin.indices.stats.RestIndicesStatsAction;
-import org.elasticsearch.rest.action.admin.indices.template.delete.RestDeleteIndexTemplateAction;
-import org.elasticsearch.rest.action.admin.indices.template.get.RestGetIndexTemplateAction;
-import org.elasticsearch.rest.action.admin.indices.template.head.RestHeadIndexTemplateAction;
-import org.elasticsearch.rest.action.admin.indices.template.put.RestPutIndexTemplateAction;
-import org.elasticsearch.rest.action.admin.indices.upgrade.RestUpgradeAction;
-import org.elasticsearch.rest.action.admin.indices.validate.query.RestValidateQueryAction;
-import org.elasticsearch.rest.action.bulk.RestBulkAction;
-import org.elasticsearch.rest.action.cat.AbstractCatAction;
-import org.elasticsearch.rest.action.cat.RestAliasAction;
-import org.elasticsearch.rest.action.cat.RestAllocationAction;
-import org.elasticsearch.rest.action.cat.RestCatAction;
-import org.elasticsearch.rest.action.cat.RestFielddataAction;
-import org.elasticsearch.rest.action.cat.RestHealthAction;
-import org.elasticsearch.rest.action.cat.RestIndicesAction;
-import org.elasticsearch.rest.action.cat.RestMasterAction;
-import org.elasticsearch.rest.action.cat.RestNodeAttrsAction;
-import org.elasticsearch.rest.action.cat.RestNodesAction;
-import org.elasticsearch.rest.action.cat.RestPluginsAction;
-import org.elasticsearch.rest.action.cat.RestRepositoriesAction;
-import org.elasticsearch.rest.action.cat.RestSegmentsAction;
-import org.elasticsearch.rest.action.cat.RestShardsAction;
-import org.elasticsearch.rest.action.cat.RestSnapshotAction;
-import org.elasticsearch.rest.action.cat.RestTasksAction;
-import org.elasticsearch.rest.action.cat.RestThreadPoolAction;
-import org.elasticsearch.rest.action.delete.RestDeleteAction;
-import org.elasticsearch.rest.action.explain.RestExplainAction;
-import org.elasticsearch.rest.action.fieldstats.RestFieldStatsAction;
-import org.elasticsearch.rest.action.get.RestGetAction;
-import org.elasticsearch.rest.action.get.RestGetSourceAction;
-import org.elasticsearch.rest.action.get.RestHeadAction;
-import org.elasticsearch.rest.action.get.RestMultiGetAction;
-import org.elasticsearch.rest.action.index.RestIndexAction;
-import org.elasticsearch.rest.action.ingest.RestDeletePipelineAction;
-import org.elasticsearch.rest.action.ingest.RestGetPipelineAction;
-import org.elasticsearch.rest.action.ingest.RestPutPipelineAction;
-import org.elasticsearch.rest.action.ingest.RestSimulatePipelineAction;
-import org.elasticsearch.rest.action.main.RestMainAction;
-import org.elasticsearch.rest.action.search.RestClearScrollAction;
-import org.elasticsearch.rest.action.search.RestMultiSearchAction;
-import org.elasticsearch.rest.action.search.RestSearchAction;
-import org.elasticsearch.rest.action.search.RestSearchScrollAction;
-import org.elasticsearch.rest.action.suggest.RestSuggestAction;
-import org.elasticsearch.rest.action.termvectors.RestMultiTermVectorsAction;
-import org.elasticsearch.rest.action.termvectors.RestTermVectorsAction;
-import org.elasticsearch.rest.action.update.RestUpdateAction;
 import org.elasticsearch.tasks.RawTaskStatus;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.Transport;
@@ -151,9 +48,6 @@ import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.local.LocalTransport;
 import org.elasticsearch.transport.netty.NettyTransport;
 
-import java.util.Arrays;
-import java.util.List;
-
 /**
  * A module to handle registering and binding all network related classes.
  */
@@ -171,140 +65,6 @@ public class NetworkModule extends AbstractModule {
         Setting.simpleString(TRANSPORT_SERVICE_TYPE_KEY, Property.NodeScope);
     public static final Setting<String> TRANSPORT_TYPE_SETTING = Setting.simpleString(TRANSPORT_TYPE_KEY, Property.NodeScope);
 
-
-
-    private static final List<Class<? extends RestHandler>> builtinRestHandlers = Arrays.asList(
-        RestMainAction.class,
-        RestNodesInfoAction.class,
-        RestNodesStatsAction.class,
-        RestNodesHotThreadsAction.class,
-        RestClusterAllocationExplainAction.class,
-        RestClusterStatsAction.class,
-        RestClusterStateAction.class,
-        RestClusterHealthAction.class,
-        RestClusterUpdateSettingsAction.class,
-        RestClusterGetSettingsAction.class,
-        RestClusterRerouteAction.class,
-        RestClusterSearchShardsAction.class,
-        RestPendingClusterTasksAction.class,
-        RestPutRepositoryAction.class,
-        RestGetRepositoriesAction.class,
-        RestDeleteRepositoryAction.class,
-        RestVerifyRepositoryAction.class,
-        RestGetSnapshotsAction.class,
-        RestCreateSnapshotAction.class,
-        RestRestoreSnapshotAction.class,
-        RestDeleteSnapshotAction.class,
-        RestSnapshotsStatusAction.class,
-
-        RestIndicesExistsAction.class,
-        RestTypesExistsAction.class,
-        RestGetIndicesAction.class,
-        RestIndicesStatsAction.class,
-        RestIndicesSegmentsAction.class,
-        RestIndicesShardStoresAction.class,
-        RestGetAliasesAction.class,
-        RestAliasesExistAction.class,
-        RestIndexDeleteAliasesAction.class,
-        RestIndexPutAliasAction.class,
-        RestIndicesAliasesAction.class,
-        RestCreateIndexAction.class,
-        RestShrinkIndexAction.class,
-        RestRolloverIndexAction.class,
-        RestDeleteIndexAction.class,
-        RestCloseIndexAction.class,
-        RestOpenIndexAction.class,
-
-        RestUpdateSettingsAction.class,
-        RestGetSettingsAction.class,
-
-        RestAnalyzeAction.class,
-        RestGetIndexTemplateAction.class,
-        RestPutIndexTemplateAction.class,
-        RestDeleteIndexTemplateAction.class,
-        RestHeadIndexTemplateAction.class,
-
-        RestPutMappingAction.class,
-        RestGetMappingAction.class,
-        RestGetFieldMappingAction.class,
-
-        RestRefreshAction.class,
-        RestFlushAction.class,
-        RestSyncedFlushAction.class,
-        RestForceMergeAction.class,
-        RestUpgradeAction.class,
-        RestClearIndicesCacheAction.class,
-
-        RestIndexAction.class,
-        RestGetAction.class,
-        RestGetSourceAction.class,
-        RestHeadAction.Document.class,
-        RestHeadAction.Source.class,
-        RestMultiGetAction.class,
-        RestDeleteAction.class,
-        org.elasticsearch.rest.action.count.RestCountAction.class,
-        RestSuggestAction.class,
-        RestTermVectorsAction.class,
-        RestMultiTermVectorsAction.class,
-        RestBulkAction.class,
-        RestUpdateAction.class,
-
-        RestSearchAction.class,
-        RestSearchScrollAction.class,
-        RestClearScrollAction.class,
-        RestMultiSearchAction.class,
-
-        RestValidateQueryAction.class,
-
-        RestExplainAction.class,
-
-        RestRecoveryAction.class,
-
-        // Scripts API
-        RestGetStoredScriptAction.class,
-        RestPutStoredScriptAction.class,
-        RestDeleteStoredScriptAction.class,
-
-        RestFieldStatsAction.class,
-
-        // no abstract cat action
-        RestCatAction.class,
-
-        // Tasks API
-        RestListTasksAction.class,
-        RestGetTaskAction.class,
-        RestCancelTasksAction.class,
-
-        // Ingest API
-        RestPutPipelineAction.class,
-        RestGetPipelineAction.class,
-        RestDeletePipelineAction.class,
-        RestSimulatePipelineAction.class
-    );
-
-    private static final List<Class<? extends AbstractCatAction>> builtinCatHandlers = Arrays.asList(
-        RestAllocationAction.class,
-        RestShardsAction.class,
-        RestMasterAction.class,
-        RestNodesAction.class,
-        RestTasksAction.class,
-        RestIndicesAction.class,
-        RestSegmentsAction.class,
-        // Fully qualified to prevent interference with rest.action.count.RestCountAction
-        org.elasticsearch.rest.action.cat.RestCountAction.class,
-        // Fully qualified to prevent interference with rest.action.indices.RestRecoveryAction
-        org.elasticsearch.rest.action.cat.RestRecoveryAction.class,
-        RestHealthAction.class,
-        org.elasticsearch.rest.action.cat.RestPendingClusterTasksAction.class,
-        RestAliasAction.class,
-        RestThreadPoolAction.class,
-        RestPluginsAction.class,
-        RestFielddataAction.class,
-        RestNodeAttrsAction.class,
-        RestRepositoriesAction.class,
-        RestSnapshotAction.class
-    );
-
     private final NetworkService networkService;
     private final Settings settings;
     private final boolean transportClient;
@@ -313,9 +73,6 @@ public class NetworkModule extends AbstractModule {
     private final ExtensionPoint.SelectedType<TransportService> transportServiceTypes = new ExtensionPoint.SelectedType<>("transport_service", TransportService.class);
     private final ExtensionPoint.SelectedType<Transport> transportTypes = new ExtensionPoint.SelectedType<>("transport", Transport.class);
     private final ExtensionPoint.SelectedType<HttpServerTransport> httpTransportTypes = new ExtensionPoint.SelectedType<>("http_transport", HttpServerTransport.class);
-    private final ExtensionPoint.ClassSet<RestHandler> restHandlers = new ExtensionPoint.ClassSet<>("rest_handler", RestHandler.class);
-    // we must separate the cat rest handlers so RestCatAction can collect them...
-    private final ExtensionPoint.ClassSet<AbstractCatAction> catHandlers = new ExtensionPoint.ClassSet<>("cat_handler", AbstractCatAction.class);
     private final NamedWriteableRegistry namedWriteableRegistry;
 
     /**
@@ -340,13 +97,6 @@ public class NetworkModule extends AbstractModule {
 
         if (transportClient == false) {
             registerHttpTransport(NETTY_TRANSPORT, NettyHttpServerTransport.class);
-
-            for (Class<? extends AbstractCatAction> catAction : builtinCatHandlers) {
-                catHandlers.registerExtension(catAction);
-            }
-            for (Class<? extends RestHandler> restAction : builtinRestHandlers) {
-                restHandlers.registerExtension(restAction);
-            }
         }
     }
 
@@ -373,19 +123,6 @@ public class NetworkModule extends AbstractModule {
         httpTransportTypes.registerExtension(name, clazz);
     }
 
-    /** Adds an additional rest action. */
-    // TODO: change this further to eliminate the middle man, ie RestController, and just register method and path here
-    public void registerRestHandler(Class<? extends RestHandler> clazz) {
-        if (transportClient) {
-            throw new IllegalArgumentException("Cannot register rest handler " + clazz.getName() + " for transport client");
-        }
-        if (AbstractCatAction.class.isAssignableFrom(clazz)) {
-            catHandlers.registerExtension(clazz.asSubclass(AbstractCatAction.class));
-        } else {
-            restHandlers.registerExtension(clazz);
-        }
-    }
-
     public void registerTaskStatus(String name, Writeable.Reader<? extends Task.Status> reader) {
         namedWriteableRegistry.register(Task.Status.class, name, reader);
     }
@@ -430,9 +167,6 @@ public class NetworkModule extends AbstractModule {
                 bind(HttpServer.class).asEagerSingleton();
                 httpTransportTypes.bindType(binder(), settings, HTTP_TYPE_SETTING.getKey(), NETTY_TRANSPORT);
             }
-            bind(RestController.class).asEagerSingleton();
-            catHandlers.bind(binder());
-            restHandlers.bind(binder());
             // Bind the AllocationCommandRegistry so RestClusterRerouteAction can get it.
             bind(AllocationCommandRegistry.class).toInstance(allocationCommandRegistry);
         }
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index 211ff9ffe65..1d18f696163 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -26,6 +26,8 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionModule;
+import org.elasticsearch.action.GenericAction;
+import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterModule;
@@ -42,6 +44,7 @@ import org.elasticsearch.common.StopWatch;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.inject.Injector;
+import org.elasticsearch.common.inject.Key;
 import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.inject.ModulesBuilder;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
@@ -122,6 +125,7 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
@@ -165,7 +169,7 @@ public class Node implements Closeable {
     private final Environment environment;
     private final NodeEnvironment nodeEnvironment;
     private final PluginsService pluginsService;
-    private final Client client;
+    private final NodeClient client;
 
     /**
      * Constructs a node with the given settings.
@@ -275,9 +279,10 @@ public class Node implements Closeable {
             BigArrays bigArrays = createBigArrays(settings, circuitBreakerService);
             resourcesToClose.add(bigArrays);
             modules.add(settingsModule);
+            client = new NodeClient(settings, threadPool);
             modules.add(b -> {
                     b.bind(PluginsService.class).toInstance(pluginsService);
-                    b.bind(Client.class).to(NodeClient.class).asEagerSingleton();
+                    b.bind(Client.class).toInstance(client);
                     b.bind(Environment.class).toInstance(environment);
                     b.bind(ThreadPool.class).toInstance(threadPool);
                     b.bind(NodeEnvironment.class).toInstance(nodeEnvironment);
@@ -290,7 +295,7 @@ public class Node implements Closeable {
                 }
             );
             injector = modules.createInjector();
-            client = injector.getInstance(Client.class);
+            client.intialize(injector.getInstance(new Key<Map<GenericAction, TransportAction>>() {}));
             success = true;
         } catch (IOException ex) {
             throw new ElasticsearchException("failed to bind service", ex);
diff --git a/core/src/main/java/org/elasticsearch/plugins/ActionPlugin.java b/core/src/main/java/org/elasticsearch/plugins/ActionPlugin.java
index 0d4ee16ab89..f05c6654393 100644
--- a/core/src/main/java/org/elasticsearch/plugins/ActionPlugin.java
+++ b/core/src/main/java/org/elasticsearch/plugins/ActionPlugin.java
@@ -25,8 +25,11 @@ import org.elasticsearch.action.GenericAction;
 import org.elasticsearch.action.support.ActionFilter;
 import org.elasticsearch.action.support.TransportAction;
 import org.elasticsearch.action.support.TransportActions;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.rest.RestHandler;
 
 import java.util.List;
+import java.util.Objects;
 
 import static java.util.Collections.emptyList;
 
@@ -55,6 +58,12 @@ public interface ActionPlugin {
     default List<Class<? extends ActionFilter>> getActionFilters() {
         return emptyList();
     }
+    /**
+     * Rest handlers added by this plugin.
+     */
+    default List<Class<? extends RestHandler>> getRestHandlers() {
+        return emptyList();
+    }
 
     public static final class ActionHandler<Request extends ActionRequest<Request>, Response extends ActionResponse> {
         private final GenericAction<Request, Response> action;
@@ -83,5 +92,30 @@ public interface ActionPlugin {
         public Class<?>[] getSupportTransportActions() {
             return supportTransportActions;
         }
+
+        @Override
+        public String toString() {
+            StringBuilder b = new StringBuilder().append(action.name()).append(" is handled by ").append(transportAction.getName());
+            if (supportTransportActions.length > 0) {
+                b.append('[').append(Strings.arrayToCommaDelimitedString(supportTransportActions)).append(']');
+            }
+            return b.toString();
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (obj == null || obj.getClass() != ActionHandler.class) {
+                return false;
+            }
+            ActionHandler<?, ?> other = (ActionHandler<?, ?>) obj;
+            return Objects.equals(action, other.action)
+                    && Objects.equals(transportAction, other.transportAction)
+                    && Objects.deepEquals(supportTransportActions, other.supportTransportActions);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(action, transportAction, supportTransportActions);
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/RestController.java b/core/src/main/java/org/elasticsearch/rest/RestController.java
index be72ed19c40..e8212ff09c6 100644
--- a/core/src/main/java/org/elasticsearch/rest/RestController.java
+++ b/core/src/main/java/org/elasticsearch/rest/RestController.java
@@ -22,7 +22,6 @@ package org.elasticsearch.rest;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.path.PathTrie;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -59,7 +58,6 @@ public class RestController extends AbstractLifecycleComponent<RestController> {
     // non volatile since the assumption is that pre processors are registered on startup
     private RestFilter[] filters = new RestFilter[0];
 
-    @Inject
     public RestController(Settings settings) {
         super(settings);
     }
diff --git a/core/src/test/java/org/elasticsearch/action/ActionModuleTests.java b/core/src/test/java/org/elasticsearch/action/ActionModuleTests.java
new file mode 100644
index 00000000000..4b9a833e8c4
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/ActionModuleTests.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action;
+
+import org.elasticsearch.action.main.MainAction;
+import org.elasticsearch.action.main.TransportMainAction;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.TransportAction;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.ActionPlugin.ActionHandler;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.main.RestMainAction;
+import org.elasticsearch.tasks.TaskManager;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.util.List;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasItem;
+
+public class ActionModuleTests extends ESTestCase {
+    public void testSetupActionsContainsKnownBuiltin() {
+        assertThat(ActionModule.setupActions(emptyList()),
+                hasEntry(MainAction.INSTANCE.name(), new ActionHandler<>(MainAction.INSTANCE, TransportMainAction.class)));
+    }
+
+    public void testPluginCantOverwriteBuiltinAction() {
+        ActionPlugin dupsMainAction = new ActionPlugin() {
+            @Override
+            public List<ActionHandler<? extends ActionRequest<?>, ? extends ActionResponse>> getActions() {
+                return singletonList(new ActionHandler<>(MainAction.INSTANCE, TransportMainAction.class));
+            }
+        };
+        Exception e = expectThrows(IllegalArgumentException.class, () -> ActionModule.setupActions(singletonList(dupsMainAction)));
+        assertEquals("action for name [" + MainAction.NAME + "] already registered", e.getMessage());
+    }
+
+    public void testPluginCanRegisterAction() {
+        class FakeRequest extends ActionRequest<FakeRequest> {
+            @Override
+            public ActionRequestValidationException validate() {
+                return null;
+            }
+        }
+        class FakeTransportAction extends TransportAction<FakeRequest, ActionResponse> {
+            protected FakeTransportAction(Settings settings, String actionName, ThreadPool threadPool, ActionFilters actionFilters,
+                    IndexNameExpressionResolver indexNameExpressionResolver, TaskManager taskManager) {
+                super(settings, actionName, threadPool, actionFilters, indexNameExpressionResolver, taskManager);
+            }
+
+            @Override
+            protected void doExecute(FakeRequest request, ActionListener<ActionResponse> listener) {
+            }
+        }
+        class FakeAction extends GenericAction<FakeRequest, ActionResponse> {
+            protected FakeAction() {
+                super("fake");
+            }
+
+            @Override
+            public ActionResponse newResponse() {
+                return null;
+            }
+        }
+        FakeAction action = new FakeAction();
+        ActionPlugin registersFakeAction = new ActionPlugin() {
+            @Override
+            public List<ActionHandler<? extends ActionRequest<?>, ? extends ActionResponse>> getActions() {
+                return singletonList(new ActionHandler<>(action, FakeTransportAction.class));
+            }
+        };
+        assertThat(ActionModule.setupActions(singletonList(registersFakeAction)),
+                hasEntry("fake", new ActionHandler<>(action, FakeTransportAction.class)));
+    }
+
+    public void testSetupRestHandlerContainsKnownBuiltin() {
+        assertThat(ActionModule.setupRestHandlers(emptyList()), hasItem(RestMainAction.class));
+    }
+
+    public void testPluginCantOverwriteBuiltinRestHandler() {
+        ActionPlugin dupsMainAction = new ActionPlugin() {
+            @Override
+            public List<Class<? extends RestHandler>> getRestHandlers() {
+                return singletonList(RestMainAction.class);
+            }
+        };
+        Exception e = expectThrows(IllegalArgumentException.class, () -> ActionModule.setupRestHandlers(singletonList(dupsMainAction)));
+        assertEquals("can't register the same [rest_handler] more than once for [" + RestMainAction.class.getName() + "]", e.getMessage());
+    }
+
+    public void testPluginCanRegisterRestHandler() {
+        class FakeHandler implements RestHandler {
+            @Override
+            public void handleRequest(RestRequest request, RestChannel channel) throws Exception {
+            }
+        }
+        ActionPlugin registersFakeHandler = new ActionPlugin() {
+            @Override
+            public List<Class<? extends RestHandler>> getRestHandlers() {
+                return singletonList(FakeHandler.class);
+            }
+        };
+        assertThat(ActionModule.setupRestHandlers(singletonList(registersFakeHandler)), hasItem(FakeHandler.class));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/client/node/NodeClientHeadersTests.java b/core/src/test/java/org/elasticsearch/client/node/NodeClientHeadersTests.java
index f69c8f2da0b..04f7b73b1f2 100644
--- a/core/src/test/java/org/elasticsearch/client/node/NodeClientHeadersTests.java
+++ b/core/src/test/java/org/elasticsearch/client/node/NodeClientHeadersTests.java
@@ -46,7 +46,9 @@ public class NodeClientHeadersTests extends AbstractClientHeadersTestCase {
     protected Client buildClient(Settings headersSettings, GenericAction[] testedActions) {
         Settings settings = HEADER_SETTINGS;
         Actions actions = new Actions(settings, threadPool, testedActions);
-        return new NodeClient(settings, threadPool, actions);
+        NodeClient client = new NodeClient(settings, threadPool);
+        client.intialize(actions);
+        return client;
     }
 
     private static class Actions extends HashMap<GenericAction, TransportAction> {
diff --git a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
index d54edbcaa9d..749ffa3c9d9 100644
--- a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
@@ -36,11 +36,8 @@ import org.elasticsearch.http.HttpServerTransport;
 import org.elasticsearch.http.HttpStats;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestChannel;
-import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.cat.AbstractCatAction;
-import org.elasticsearch.rest.action.cat.RestNodesAction;
-import org.elasticsearch.rest.action.main.RestMainAction;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.transport.AssertingLocalTransport;
 import org.elasticsearch.transport.Transport;
@@ -163,32 +160,6 @@ public class NetworkModuleTests extends ModuleTestCase {
         assertFalse(module.isTransportClient());
     }
 
-    public void testRegisterRestHandler() {
-        Settings settings = Settings.EMPTY;
-        NetworkModule module = new NetworkModule(new NetworkService(settings), settings, false, new NamedWriteableRegistry());
-        module.registerRestHandler(FakeRestHandler.class);
-        // also check a builtin is bound
-        assertSetMultiBinding(module, RestHandler.class, FakeRestHandler.class, RestMainAction.class);
-
-        // check registration not allowed for transport only
-        module = new NetworkModule(new NetworkService(settings), settings, true, new NamedWriteableRegistry());
-        try {
-            module.registerRestHandler(FakeRestHandler.class);
-            fail();
-        } catch (IllegalArgumentException e) {
-            assertTrue(e.getMessage().contains("Cannot register rest handler"));
-            assertTrue(e.getMessage().contains("for transport client"));
-        }
-    }
-
-    public void testRegisterCatRestHandler() {
-        Settings settings = Settings.EMPTY;
-        NetworkModule module = new NetworkModule(new NetworkService(settings), settings, false, new NamedWriteableRegistry());
-        module.registerRestHandler(FakeCatRestHandler.class);
-        // also check a builtin is bound
-        assertSetMultiBinding(module, AbstractCatAction.class, FakeCatRestHandler.class, RestNodesAction.class);
-    }
-
     public void testRegisterTaskStatus() {
         NamedWriteableRegistry registry = new NamedWriteableRegistry();
         Settings settings = Settings.EMPTY;
diff --git a/core/src/test/java/org/elasticsearch/plugins/responseheader/TestResponseHeaderPlugin.java b/core/src/test/java/org/elasticsearch/plugins/responseheader/TestResponseHeaderPlugin.java
index 701d1154587..9dfd5b6a93a 100644
--- a/core/src/test/java/org/elasticsearch/plugins/responseheader/TestResponseHeaderPlugin.java
+++ b/core/src/test/java/org/elasticsearch/plugins/responseheader/TestResponseHeaderPlugin.java
@@ -19,12 +19,17 @@
 
 package org.elasticsearch.plugins.responseheader;
 
-import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestHandler;
 
-public class TestResponseHeaderPlugin extends Plugin {
+import java.util.List;
 
-    public void onModule(NetworkModule module) {
-        module.registerRestHandler(TestResponseHeaderRestAction.class);
+import static java.util.Collections.singletonList;
+
+public class TestResponseHeaderPlugin extends Plugin implements ActionPlugin {
+    @Override
+    public List<Class<? extends RestHandler>> getRestHandlers() {
+        return singletonList(TestResponseHeaderRestAction.class);
     }
 }
diff --git a/docs/reference/migration/migrate_5_0/plugins.asciidoc b/docs/reference/migration/migrate_5_0/plugins.asciidoc
index e1e8e6c614b..2826c822d15 100644
--- a/docs/reference/migration/migrate_5_0/plugins.asciidoc
+++ b/docs/reference/migration/migrate_5_0/plugins.asciidoc
@@ -137,3 +137,6 @@ Plugins that register custom mappers should implement
 
 Plugins that register custom actions should implement `ActionPlugin` and
 remove their `onModule(ActionModule)` implementation.
+
+Plugins that register custom `RestHandler`s should implement `ActionPlugin` and
+remove their `onModule(NetworkModule)` implemnetation.
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
index 50bae57887c..d3ffa13cd54 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustachePlugin.java
@@ -25,11 +25,11 @@ import org.elasticsearch.action.search.template.MultiSearchTemplateAction;
 import org.elasticsearch.action.search.template.SearchTemplateAction;
 import org.elasticsearch.action.search.template.TransportMultiSearchTemplateAction;
 import org.elasticsearch.action.search.template.TransportSearchTemplateAction;
-import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ScriptPlugin;
+import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.rest.action.search.template.RestDeleteSearchTemplateAction;
 import org.elasticsearch.rest.action.search.template.RestGetSearchTemplateAction;
 import org.elasticsearch.rest.action.search.template.RestMultiSearchTemplateAction;
@@ -54,14 +54,9 @@ public class MustachePlugin extends Plugin implements ScriptPlugin, ActionPlugin
                 new ActionHandler<>(MultiSearchTemplateAction.INSTANCE, TransportMultiSearchTemplateAction.class));
     }
 
-    public void onModule(NetworkModule module) {
-        if (module.isTransportClient() == false) {
-            module.registerRestHandler(RestSearchTemplateAction.class);
-            module.registerRestHandler(RestMultiSearchTemplateAction.class);
-            module.registerRestHandler(RestGetSearchTemplateAction.class);
-            module.registerRestHandler(RestPutSearchTemplateAction.class);
-            module.registerRestHandler(RestDeleteSearchTemplateAction.class);
-            module.registerRestHandler(RestRenderSearchTemplateAction.class);
-        }
+    @Override
+    public List<Class<? extends RestHandler>> getRestHandlers() {
+        return Arrays.asList(RestSearchTemplateAction.class, RestMultiSearchTemplateAction.class, RestGetSearchTemplateAction.class,
+                RestPutSearchTemplateAction.class, RestDeleteSearchTemplateAction.class, RestRenderSearchTemplateAction.class);
     }
 }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
index 87fc6726c56..a15acf67c02 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
@@ -23,13 +23,13 @@ import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.search.SearchModule;
 
 import java.util.Arrays;
@@ -41,11 +41,9 @@ public class PercolatorPlugin extends Plugin implements MapperPlugin, ActionPlug
 
     public static final String NAME = "percolator";
 
-    private final boolean transportClientMode;
     private final Settings settings;
 
     public PercolatorPlugin(Settings settings) {
-        this.transportClientMode = transportClientMode(settings);
         this.settings = settings;
     }
 
@@ -55,11 +53,9 @@ public class PercolatorPlugin extends Plugin implements MapperPlugin, ActionPlug
                 new ActionHandler<>(MultiPercolateAction.INSTANCE, TransportMultiPercolateAction.class));
     }
 
-    public void onModule(NetworkModule module) {
-        if (transportClientMode == false) {
-            module.registerRestHandler(RestPercolateAction.class);
-            module.registerRestHandler(RestMultiPercolateAction.class);
-        }
+    @Override
+    public List<Class<? extends RestHandler>> getRestHandlers() {
+        return Arrays.asList(RestPercolateAction.class, RestMultiPercolateAction.class);
     }
 
     public void onModule(SearchModule module) {
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
index 89e6c1cc753..5a7b81a7124 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
@@ -24,6 +24,7 @@ import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestHandler;
 
 import java.util.Arrays;
 import java.util.List;
@@ -39,11 +40,13 @@ public class ReindexPlugin extends Plugin implements ActionPlugin {
                 new ActionHandler<>(RethrottleAction.INSTANCE, TransportRethrottleAction.class));
     }
 
+    @Override
+    public List<Class<? extends RestHandler>> getRestHandlers() {
+        return Arrays.asList(RestReindexAction.class, RestUpdateByQueryAction.class, RestDeleteByQueryAction.class,
+                RestRethrottleAction.class);
+    }
+
     public void onModule(NetworkModule networkModule) {
-        networkModule.registerRestHandler(RestReindexAction.class);
-        networkModule.registerRestHandler(RestUpdateByQueryAction.class);
-        networkModule.registerRestHandler(RestDeleteByQueryAction.class);
-        networkModule.registerRestHandler(RestRethrottleAction.class);
         networkModule.registerTaskStatus(BulkByScrollTask.Status.NAME, BulkByScrollTask.Status::new);
     }
 }

From 172ced3e2df5c960c591dee04ab2bdfee8f24f1d Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Wed, 29 Jun 2016 15:56:36 -0700
Subject: [PATCH 1127/1311] Fix test bug in plugin cli progress tests

---
 .../elasticsearch/plugins/ProgressInputStreamTests.java   | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/plugins/ProgressInputStreamTests.java b/core/src/test/java/org/elasticsearch/plugins/ProgressInputStreamTests.java
index 81e937d26a9..813921963c0 100644
--- a/core/src/test/java/org/elasticsearch/plugins/ProgressInputStreamTests.java
+++ b/core/src/test/java/org/elasticsearch/plugins/ProgressInputStreamTests.java
@@ -69,19 +69,19 @@ public class ProgressInputStreamTests extends ESTestCase {
     }
 
     public void testOddBytes() throws Exception {
-        int odd = (randomIntBetween(100, 200) / 2) + 1;
+        int odd = randomIntBetween(10, 100) * 2 + 1;
         ProgressInputStream is = newProgressInputStream(odd);
         for (int i = 0; i < odd; i++) {
             is.checkProgress(1);
         }
         is.checkProgress(-1);
 
-        assertThat(progresses, hasSize(odd+1));
+        assertThat(progresses, hasSize(Math.min(odd + 1, 100)));
         assertThat(progresses, hasItem(100));
     }
 
     public void testEvenBytes() throws Exception {
-        int even = (randomIntBetween(100, 200) / 2);
+        int even = randomIntBetween(10, 100) * 2;
         ProgressInputStream is = newProgressInputStream(even);
 
         for (int i = 0; i < even; i++) {
@@ -89,7 +89,7 @@ public class ProgressInputStreamTests extends ESTestCase {
         }
         is.checkProgress(-1);
 
-        assertThat(progresses, hasSize(even+1));
+        assertThat(progresses, hasSize(Math.min(even + 1, 100)));
         assertThat(progresses, hasItem(100));
     }
 

From 7c50de182e9c5cc0a63f4d4e5138dec8d619d3ba Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Wed, 29 Jun 2016 16:23:16 -0700
Subject: [PATCH 1128/1311] Remove test for closing ingest processors, this is
 now handled at the plugin level

---
 .../elasticsearch/ingest/IngestCloseIT.java   | 73 -------------------
 1 file changed, 73 deletions(-)
 delete mode 100644 core/src/test/java/org/elasticsearch/ingest/IngestCloseIT.java

diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestCloseIT.java b/core/src/test/java/org/elasticsearch/ingest/IngestCloseIT.java
deleted file mode 100644
index ed0a302cf73..00000000000
--- a/core/src/test/java/org/elasticsearch/ingest/IngestCloseIT.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.ingest;
-
-import org.elasticsearch.node.NodeModule;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.test.ESSingleNodeTestCase;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import static org.hamcrest.Matchers.is;
-
-public class IngestCloseIT extends ESSingleNodeTestCase {
-
-    @Override
-    protected Collection<Class<? extends Plugin>> getPlugins() {
-        return pluginList(IngestPlugin.class);
-    }
-
-    private static AtomicBoolean called = new AtomicBoolean(false);
-
-    public void testCloseNode() throws Exception {
-        // We manually stop the node and check we called
-        stopNode();
-
-        assertThat(called.get(), is(true));
-
-        // We need to restart the node for the next tests (and because tearDown() expects a Node)
-        startNode();
-    }
-
-    public static class IngestPlugin extends Plugin {
-        public void onModule(NodeModule nodeModule) {
-            nodeModule.registerProcessor("test", (registry) -> new Factory());
-        }
-    }
-
-    public static final class Factory extends AbstractProcessorFactory<TestProcessor> implements Closeable {
-        @Override
-        protected TestProcessor doCreate(String tag, Map<String, Object> config) throws Exception {
-            return new TestProcessor("id", "test", ingestDocument -> {
-                throw new UnsupportedOperationException("this code is actually never called from the test");
-            });
-        }
-
-        @Override
-        public void close() throws IOException {
-            called.set(true);
-        }
-    }
-
-}

From 865b951b7d5ba0acfa51bae2c12416044642338d Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Wed, 29 Jun 2016 15:35:55 -0700
Subject: [PATCH 1129/1311] Internal: Changed rest handler interface to take
 NodeClient

Previously all rest handlers would take Client in their injected ctor.
However, it was only to hold the client around for runtime. Instead,
this can be done just once in the HttpService which handles rest
requests, and passed along through the handleRequest method. It also
should always be a NodeClient, and other types of Clients (eg a
TransportClient) would not work anyways (and some handlers can be
simplified in follow ups like reindex by taking NodeClient).
---
 .../org/elasticsearch/http/HttpServer.java    | 18 +++++---
 .../java/org/elasticsearch/node/Node.java     |  1 +
 .../elasticsearch/rest/BaseRestHandler.java   | 12 +----
 .../elasticsearch/rest/RestController.java    | 21 ++++-----
 .../org/elasticsearch/rest/RestFilter.java    |  4 +-
 .../elasticsearch/rest/RestFilterChain.java   |  4 +-
 .../org/elasticsearch/rest/RestHandler.java   | 11 ++++-
 .../RestClusterAllocationExplainAction.java   | 13 +++---
 .../health/RestClusterHealthAction.java       |  8 ++--
 .../hotthreads/RestNodesHotThreadsAction.java |  8 ++--
 .../node/info/RestNodesInfoAction.java        |  8 ++--
 .../node/stats/RestNodesStatsAction.java      |  8 ++--
 .../node/tasks/RestCancelTasksAction.java     |  8 ++--
 .../cluster/node/tasks/RestGetTaskAction.java |  8 ++--
 .../node/tasks/RestListTasksAction.java       |  8 ++--
 .../delete/RestDeleteRepositoryAction.java    |  8 ++--
 .../get/RestGetRepositoriesAction.java        |  8 ++--
 .../put/RestPutRepositoryAction.java          |  8 ++--
 .../verify/RestVerifyRepositoryAction.java    |  8 ++--
 .../reroute/RestClusterRerouteAction.java     |  8 ++--
 .../RestClusterGetSettingsAction.java         |  8 ++--
 .../RestClusterUpdateSettingsAction.java      |  8 ++--
 .../shards/RestClusterSearchShardsAction.java |  8 ++--
 .../create/RestCreateSnapshotAction.java      |  8 ++--
 .../delete/RestDeleteSnapshotAction.java      |  8 ++--
 .../snapshots/get/RestGetSnapshotsAction.java |  8 ++--
 .../restore/RestRestoreSnapshotAction.java    |  8 ++--
 .../status/RestSnapshotsStatusAction.java     |  8 ++--
 .../cluster/state/RestClusterStateAction.java |  8 ++--
 .../cluster/stats/RestClusterStatsAction.java |  8 ++--
 .../RestDeleteStoredScriptAction.java         | 12 ++---
 .../RestGetStoredScriptAction.java            | 12 ++---
 .../RestPutStoredScriptAction.java            | 12 ++---
 .../tasks/RestPendingClusterTasksAction.java  |  8 ++--
 .../indices/RestRolloverIndexAction.java      |  8 ++--
 .../admin/indices/RestShrinkIndexAction.java  |  8 ++--
 .../alias/RestIndicesAliasesAction.java       |  8 ++--
 .../delete/RestIndexDeleteAliasesAction.java  |  8 ++--
 .../alias/get/RestGetAliasesAction.java       |  8 ++--
 .../alias/head/RestAliasesExistAction.java    |  8 ++--
 .../alias/put/RestIndexPutAliasAction.java    |  8 ++--
 .../indices/analyze/RestAnalyzeAction.java    |  8 ++--
 .../clear/RestClearIndicesCacheAction.java    |  8 ++--
 .../indices/close/RestCloseIndexAction.java   |  8 ++--
 .../indices/create/RestCreateIndexAction.java |  8 ++--
 .../indices/delete/RestDeleteIndexAction.java |  8 ++--
 .../indices/RestIndicesExistsAction.java      |  8 ++--
 .../exists/types/RestTypesExistsAction.java   |  8 ++--
 .../admin/indices/flush/RestFlushAction.java  |  8 ++--
 .../indices/flush/RestSyncedFlushAction.java  |  8 ++--
 .../forcemerge/RestForceMergeAction.java      |  8 ++--
 .../indices/get/RestGetIndicesAction.java     |  8 ++--
 .../get/RestGetFieldMappingAction.java        |  8 ++--
 .../mapping/get/RestGetMappingAction.java     |  8 ++--
 .../mapping/put/RestPutMappingAction.java     |  8 ++--
 .../indices/open/RestOpenIndexAction.java     |  8 ++--
 .../indices/recovery/RestRecoveryAction.java  |  8 ++--
 .../indices/refresh/RestRefreshAction.java    |  8 ++--
 .../segments/RestIndicesSegmentsAction.java   |  8 ++--
 .../settings/RestGetSettingsAction.java       |  8 ++--
 .../settings/RestUpdateSettingsAction.java    |  8 ++--
 .../shards/RestIndicesShardStoresAction.java  |  8 ++--
 .../indices/stats/RestIndicesStatsAction.java |  8 ++--
 .../delete/RestDeleteIndexTemplateAction.java |  8 ++--
 .../get/RestGetIndexTemplateAction.java       |  8 ++--
 .../head/RestHeadIndexTemplateAction.java     |  8 ++--
 .../put/RestPutIndexTemplateAction.java       |  8 ++--
 .../indices/upgrade/RestUpgradeAction.java    | 13 +++---
 .../query/RestValidateQueryAction.java        |  8 ++--
 .../rest/action/bulk/RestBulkAction.java      |  8 ++--
 .../rest/action/cat/AbstractCatAction.java    | 10 ++---
 .../rest/action/cat/RestAliasAction.java      |  8 ++--
 .../rest/action/cat/RestAllocationAction.java |  8 ++--
 .../rest/action/cat/RestCatAction.java        |  8 ++--
 .../rest/action/cat/RestCountAction.java      |  8 ++--
 .../rest/action/cat/RestFielddataAction.java  |  8 ++--
 .../rest/action/cat/RestHealthAction.java     |  8 ++--
 .../rest/action/cat/RestIndicesAction.java    |  8 ++--
 .../rest/action/cat/RestMasterAction.java     |  8 ++--
 .../rest/action/cat/RestNodeAttrsAction.java  |  8 ++--
 .../rest/action/cat/RestNodesAction.java      |  8 ++--
 .../cat/RestPendingClusterTasksAction.java    |  8 ++--
 .../rest/action/cat/RestPluginsAction.java    |  8 ++--
 .../rest/action/cat/RestRecoveryAction.java   |  8 ++--
 .../action/cat/RestRepositoriesAction.java    |  8 ++--
 .../rest/action/cat/RestSegmentsAction.java   |  8 ++--
 .../rest/action/cat/RestShardsAction.java     |  8 ++--
 .../rest/action/cat/RestSnapshotAction.java   |  8 ++--
 .../rest/action/cat/RestTasksAction.java      |  8 ++--
 .../rest/action/cat/RestThreadPoolAction.java |  8 ++--
 .../rest/action/count/RestCountAction.java    |  8 ++--
 .../rest/action/delete/RestDeleteAction.java  |  8 ++--
 .../action/explain/RestExplainAction.java     |  8 ++--
 .../fieldstats/RestFieldStatsAction.java      |  8 ++--
 .../rest/action/get/RestGetAction.java        |  8 ++--
 .../rest/action/get/RestGetSourceAction.java  |  8 ++--
 .../rest/action/get/RestHeadAction.java       | 17 ++++---
 .../rest/action/get/RestMultiGetAction.java   |  8 ++--
 .../rest/action/index/RestIndexAction.java    | 16 +++----
 .../ingest/RestDeletePipelineAction.java      |  8 ++--
 .../action/ingest/RestGetPipelineAction.java  |  8 ++--
 .../action/ingest/RestPutPipelineAction.java  |  8 ++--
 .../ingest/RestSimulatePipelineAction.java    |  8 ++--
 .../rest/action/main/RestMainAction.java      |  8 ++--
 .../action/search/RestClearScrollAction.java  |  8 ++--
 .../action/search/RestMultiSearchAction.java  |  8 ++--
 .../rest/action/search/RestSearchAction.java  |  8 ++--
 .../action/search/RestSearchScrollAction.java |  8 ++--
 .../action/suggest/RestSuggestAction.java     |  8 ++--
 .../RestMultiTermVectorsAction.java           |  8 ++--
 .../termvectors/RestTermVectorsAction.java    |  8 ++--
 .../rest/action/update/RestUpdateAction.java  |  8 ++--
 .../action/ActionModuleTests.java             |  3 +-
 .../common/network/NetworkModuleTests.java    | 10 ++---
 .../elasticsearch/http/HttpServerTests.java   |  6 +--
 .../TestResponseHeaderRestAction.java         |  8 ++--
 .../rest/RestControllerTests.java             |  7 +--
 .../rest/RestFilterChainTests.java            | 35 +++++++--------
 .../action/cat/RestRecoveryActionTests.java   |  2 +-
 .../DedicatedClusterSnapshotRestoreIT.java    |  6 ++-
 .../RestDeleteSearchTemplateAction.java       |  6 +--
 .../template/RestGetSearchTemplateAction.java |  6 +--
 .../RestMultiSearchTemplateAction.java        |  8 ++--
 .../template/RestPutSearchTemplateAction.java |  6 +--
 .../RestRenderSearchTemplateAction.java       |  8 ++--
 .../template/RestSearchTemplateAction.java    |  8 ++--
 .../percolator/RestMultiPercolateAction.java  |  8 ++--
 .../percolator/RestPercolateAction.java       | 44 +++++++++----------
 .../AbstractBaseReindexRestHandler.java       | 10 ++---
 .../AbstractBulkByQueryRestHandler.java       |  4 +-
 .../reindex/RestDeleteByQueryAction.java      |  8 ++--
 .../index/reindex/RestReindexAction.java      |  8 ++--
 .../index/reindex/RestRethrottleAction.java   |  8 ++--
 .../reindex/RestUpdateByQueryAction.java      |  8 ++--
 .../plugin/example/ExampleCatAction.java      |  8 ++--
 135 files changed, 592 insertions(+), 585 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/http/HttpServer.java b/core/src/main/java/org/elasticsearch/http/HttpServer.java
index 31bbd235009..f44b839b2a3 100644
--- a/core/src/main/java/org/elasticsearch/http/HttpServer.java
+++ b/core/src/main/java/org/elasticsearch/http/HttpServer.java
@@ -19,6 +19,12 @@
 
 package org.elasticsearch.http;
 
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -39,11 +45,6 @@ import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.concurrent.atomic.AtomicBoolean;
-
 import static org.elasticsearch.rest.RestStatus.FORBIDDEN;
 import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR;
 
@@ -57,15 +58,18 @@ public class HttpServer extends AbstractLifecycleComponent<HttpServer> implement
 
     private final NodeService nodeService;
 
+    private final NodeClient client;
+
     private final CircuitBreakerService circuitBreakerService;
 
     @Inject
     public HttpServer(Settings settings, HttpServerTransport transport, RestController restController, NodeService nodeService,
-                      CircuitBreakerService circuitBreakerService) {
+                      NodeClient client, CircuitBreakerService circuitBreakerService) {
         super(settings);
         this.transport = transport;
         this.restController = restController;
         this.nodeService = nodeService;
+        this.client = client;
         this.circuitBreakerService = circuitBreakerService;
         nodeService.setHttpServer(this);
         transport.httpServerAdapter(this);
@@ -115,7 +119,7 @@ public class HttpServer extends AbstractLifecycleComponent<HttpServer> implement
             }
             // iff we could reserve bytes for the request we need to send the response also over this channel
             responseChannel = new ResourceHandlingHttpChannel(channel, circuitBreakerService, contentLength);
-            restController.dispatchRequest(request, responseChannel, threadContext);
+            restController.dispatchRequest(request, responseChannel, client, threadContext);
         } catch (Throwable t) {
             restController.sendErrorResponse(request, responseChannel, t);
         }
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index 1d18f696163..e12a5f552ff 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -283,6 +283,7 @@ public class Node implements Closeable {
             modules.add(b -> {
                     b.bind(PluginsService.class).toInstance(pluginsService);
                     b.bind(Client.class).toInstance(client);
+                    b.bind(NodeClient.class).toInstance(client);
                     b.bind(Environment.class).toInstance(environment);
                     b.bind(ThreadPool.class).toInstance(threadPool);
                     b.bind(NodeEnvironment.class).toInstance(nodeEnvironment);
diff --git a/core/src/main/java/org/elasticsearch/rest/BaseRestHandler.java b/core/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
index b406dfca545..cfa46eab514 100644
--- a/core/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
+++ b/core/src/main/java/org/elasticsearch/rest/BaseRestHandler.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.rest;
 
-import org.elasticsearch.client.Client;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.Setting;
@@ -37,19 +36,10 @@ import org.elasticsearch.common.settings.Settings;
 public abstract class BaseRestHandler extends AbstractComponent implements RestHandler {
     public static final Setting<Boolean> MULTI_ALLOW_EXPLICIT_INDEX =
         Setting.boolSetting("rest.action.multi.allow_explicit_index", true, Property.NodeScope);
-    private final Client client;
     protected final ParseFieldMatcher parseFieldMatcher;
 
-    protected BaseRestHandler(Settings settings, Client client) {
+    protected BaseRestHandler(Settings settings) {
         super(settings);
-        this.client = client;
         this.parseFieldMatcher = new ParseFieldMatcher(settings);
     }
-
-    @Override
-    public final void handleRequest(RestRequest request, RestChannel channel) throws Exception {
-        handleRequest(request, channel, client);
-    }
-
-    protected abstract void handleRequest(RestRequest request, RestChannel channel, Client client) throws Exception;
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/RestController.java b/core/src/main/java/org/elasticsearch/rest/RestController.java
index e8212ff09c6..8059dfd7ea5 100644
--- a/core/src/main/java/org/elasticsearch/rest/RestController.java
+++ b/core/src/main/java/org/elasticsearch/rest/RestController.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.rest;
 
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
@@ -150,7 +151,7 @@ public class RestController extends AbstractLifecycleComponent<RestController> {
         return (handler != null) ? handler.canTripCircuitBreaker() : true;
     }
 
-    public void dispatchRequest(final RestRequest request, final RestChannel channel, ThreadContext threadContext) throws Exception {
+    public void dispatchRequest(final RestRequest request, final RestChannel channel, final NodeClient client, ThreadContext threadContext) throws Exception {
         if (!checkRequestParameters(request, channel)) {
             return;
         }
@@ -162,10 +163,10 @@ public class RestController extends AbstractLifecycleComponent<RestController> {
                 }
             }
             if (filters.length == 0) {
-                executeHandler(request, channel);
+                executeHandler(request, channel, client);
             } else {
                 ControllerFilterChain filterChain = new ControllerFilterChain(handlerFilter);
-                filterChain.continueProcessing(request, channel);
+                filterChain.continueProcessing(request, channel, client);
             }
         }
     }
@@ -200,10 +201,10 @@ public class RestController extends AbstractLifecycleComponent<RestController> {
         return true;
     }
 
-    void executeHandler(RestRequest request, RestChannel channel) throws Exception {
+    void executeHandler(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
         final RestHandler handler = getHandler(request);
         if (handler != null) {
-            handler.handleRequest(request, channel);
+            handler.handleRequest(request, channel, client);
         } else {
             if (request.method() == RestRequest.Method.OPTIONS) {
                 // when we have OPTIONS request, simply send OK by default (with the Access Control Origin header which gets automatically added)
@@ -261,16 +262,16 @@ public class RestController extends AbstractLifecycleComponent<RestController> {
         }
 
         @Override
-        public void continueProcessing(RestRequest request, RestChannel channel) {
+        public void continueProcessing(RestRequest request, RestChannel channel, NodeClient client) {
             try {
                 int loc = index.getAndIncrement();
                 if (loc > filters.length) {
                     throw new IllegalStateException("filter continueProcessing was called more than expected");
                 } else if (loc == filters.length) {
-                    executionFilter.process(request, channel, this);
+                    executionFilter.process(request, channel, client, this);
                 } else {
                     RestFilter preProcessor = filters[loc];
-                    preProcessor.process(request, channel, this);
+                    preProcessor.process(request, channel, client, this);
                 }
             } catch (Exception e) {
                 try {
@@ -285,8 +286,8 @@ public class RestController extends AbstractLifecycleComponent<RestController> {
     class RestHandlerFilter extends RestFilter {
 
         @Override
-        public void process(RestRequest request, RestChannel channel, RestFilterChain filterChain) throws Exception {
-            executeHandler(request, channel);
+        public void process(RestRequest request, RestChannel channel, NodeClient client, RestFilterChain filterChain) throws Exception {
+            executeHandler(request, channel, client);
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/RestFilter.java b/core/src/main/java/org/elasticsearch/rest/RestFilter.java
index e3ff44ff1fc..276e99fc7e5 100644
--- a/core/src/main/java/org/elasticsearch/rest/RestFilter.java
+++ b/core/src/main/java/org/elasticsearch/rest/RestFilter.java
@@ -21,6 +21,8 @@ package org.elasticsearch.rest;
 
 import java.io.Closeable;
 
+import org.elasticsearch.client.node.NodeClient;
+
 /**
  * A filter allowing to filter rest operations.
  */
@@ -43,5 +45,5 @@ public abstract class RestFilter implements Closeable {
      * Process the rest request. Using the channel to send a response, or the filter chain to continue
      * processing the request.
      */
-    public abstract void process(RestRequest request, RestChannel channel, RestFilterChain filterChain) throws Exception;
+    public abstract void process(RestRequest request, RestChannel channel, NodeClient client, RestFilterChain filterChain) throws Exception;
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/RestFilterChain.java b/core/src/main/java/org/elasticsearch/rest/RestFilterChain.java
index be14d252b74..239a6c6b1bb 100644
--- a/core/src/main/java/org/elasticsearch/rest/RestFilterChain.java
+++ b/core/src/main/java/org/elasticsearch/rest/RestFilterChain.java
@@ -19,6 +19,8 @@
 
 package org.elasticsearch.rest;
 
+import org.elasticsearch.client.node.NodeClient;
+
 /**
  * A filter chain allowing to continue and process the rest request.
  */
@@ -28,5 +30,5 @@ public interface RestFilterChain {
      * Continue processing the request. Should only be called if a response has not been sent
      * through the channel.
      */
-    void continueProcessing(RestRequest request, RestChannel channel);
+    void continueProcessing(RestRequest request, RestChannel channel, NodeClient client);
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/RestHandler.java b/core/src/main/java/org/elasticsearch/rest/RestHandler.java
index 31970441493..393e425baf9 100644
--- a/core/src/main/java/org/elasticsearch/rest/RestHandler.java
+++ b/core/src/main/java/org/elasticsearch/rest/RestHandler.java
@@ -19,12 +19,21 @@
 
 package org.elasticsearch.rest;
 
+import org.elasticsearch.client.node.NodeClient;
+
 /**
  * Handler for REST requests
  */
 public interface RestHandler {
 
-    void handleRequest(RestRequest request, RestChannel channel) throws Exception;
+    /**
+     * Handles a rest request.
+     *
+     * @param request The request to handle
+     * @param channel The channel to write the request response to
+     * @param client A client to use to make internal requests on behalf of the original request
+     */
+    void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception;
 
     default boolean canTripCircuitBreaker() {
         return true;
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/allocation/RestClusterAllocationExplainAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/allocation/RestClusterAllocationExplainAction.java
index 0785151e8d7..e802e9e2b7f 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/allocation/RestClusterAllocationExplainAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/allocation/RestClusterAllocationExplainAction.java
@@ -19,11 +19,12 @@
 
 package org.elasticsearch.rest.action.admin.cluster.allocation;
 
+import java.io.IOException;
+
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainRequest;
 import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.Requests;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
@@ -42,22 +43,20 @@ import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.rest.action.support.RestActions;
 import org.elasticsearch.rest.action.support.RestBuilderListener;
 
-import java.io.IOException;
-
 /**
  * Class handling cluster allocation explanation at the REST level
  */
 public class RestClusterAllocationExplainAction extends BaseRestHandler {
 
     @Inject
-    public RestClusterAllocationExplainAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestClusterAllocationExplainAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/allocation/explain", this);
         controller.registerHandler(RestRequest.Method.POST, "/_cluster/allocation/explain", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         ClusterAllocationExplainRequest req;
         if (RestActions.hasBodyContent(request) == false) {
             // Empty request signals "explain the first unassigned shard you find"
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/health/RestClusterHealthAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/health/RestClusterHealthAction.java
index a2c6ffeaf14..d43d75ea5e7 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/health/RestClusterHealthAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/health/RestClusterHealthAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.admin.cluster.health;
 
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
@@ -42,15 +42,15 @@ import static org.elasticsearch.client.Requests.clusterHealthRequest;
 public class RestClusterHealthAction extends BaseRestHandler {
 
     @Inject
-    public RestClusterHealthAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestClusterHealthAction(Settings settings, RestController controller) {
+        super(settings);
 
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/health", this);
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/health/{index}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         ClusterHealthRequest clusterHealthRequest = clusterHealthRequest(Strings.splitStringByCommaToArray(request.param("index")));
         clusterHealthRequest.local(request.paramAsBoolean("local", clusterHealthRequest.local()));
         clusterHealthRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterHealthRequest.masterNodeTimeout()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/hotthreads/RestNodesHotThreadsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/hotthreads/RestNodesHotThreadsAction.java
index 2030d5e7d92..1d0daa3e7d6 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/hotthreads/RestNodesHotThreadsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/hotthreads/RestNodesHotThreadsAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.admin.cluster.node.hotthreads;
 import org.elasticsearch.action.admin.cluster.node.hotthreads.NodeHotThreads;
 import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequest;
 import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -42,8 +42,8 @@ import org.elasticsearch.rest.action.support.RestResponseListener;
 public class RestNodesHotThreadsAction extends BaseRestHandler {
 
     @Inject
-    public RestNodesHotThreadsAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestNodesHotThreadsAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/nodes/hotthreads", this);
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/nodes/hot_threads", this);
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/nodes/{nodeId}/hotthreads", this);
@@ -56,7 +56,7 @@ public class RestNodesHotThreadsAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId"));
         NodesHotThreadsRequest nodesHotThreadsRequest = new NodesHotThreadsRequest(nodesIds);
         nodesHotThreadsRequest.threads(request.paramAsInt("threads", nodesHotThreadsRequest.threads()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java
index 312fd0b0b7a..3df457efb9d 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.rest.action.admin.cluster.node.info;
 
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -46,8 +46,8 @@ public class RestNodesInfoAction extends BaseRestHandler {
     private final static Set<String> ALLOWED_METRICS = Sets.newHashSet("http", "jvm", "os", "plugins", "process", "settings", "thread_pool", "transport", "ingest", "indices");
 
     @Inject
-    public RestNodesInfoAction(Settings settings, RestController controller, Client client, SettingsFilter settingsFilter) {
-        super(settings, client);
+    public RestNodesInfoAction(Settings settings, RestController controller, SettingsFilter settingsFilter) {
+        super(settings);
         controller.registerHandler(GET, "/_nodes", this);
         // this endpoint is used for metrics, not for nodeIds, like /_nodes/fs
         controller.registerHandler(GET, "/_nodes/{nodeId}", this);
@@ -59,7 +59,7 @@ public class RestNodesInfoAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         String[] nodeIds;
         Set<String> metrics;
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java
index 8019de0b4e5..1b0e434ef5c 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/stats/RestNodesStatsAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.admin.cluster.node.stats;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -43,8 +43,8 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 public class RestNodesStatsAction extends BaseRestHandler {
 
     @Inject
-    public RestNodesStatsAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestNodesStatsAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(GET, "/_nodes/stats", this);
         controller.registerHandler(GET, "/_nodes/{nodeId}/stats", this);
 
@@ -57,7 +57,7 @@ public class RestNodesStatsAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId"));
         Set<String> metrics = Strings.splitStringByCommaToSet(request.param("metric", "_all"));
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestCancelTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestCancelTasksAction.java
index 0602abe651f..b9eec1ba98a 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestCancelTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestCancelTasksAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.admin.cluster.node.tasks;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
@@ -42,15 +42,15 @@ public class RestCancelTasksAction extends BaseRestHandler {
     private final ClusterService clusterService;
 
     @Inject
-    public RestCancelTasksAction(Settings settings, RestController controller, Client client, ClusterService clusterService) {
-        super(settings, client);
+    public RestCancelTasksAction(Settings settings, RestController controller, ClusterService clusterService) {
+        super(settings);
         this.clusterService = clusterService;
         controller.registerHandler(POST, "/_tasks/_cancel", this);
         controller.registerHandler(POST, "/_tasks/{taskId}/_cancel", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId"));
         TaskId taskId = new TaskId(request.param("taskId"));
         String[] actions = Strings.splitStringByCommaToArray(request.param("actions"));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestGetTaskAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestGetTaskAction.java
index e5617711014..501e96857fd 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestGetTaskAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestGetTaskAction.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.rest.action.admin.cluster.node.tasks;
 
 import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -35,13 +35,13 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestGetTaskAction extends BaseRestHandler {
     @Inject
-    public RestGetTaskAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestGetTaskAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(GET, "/_tasks/{taskId}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         TaskId taskId = new TaskId(request.param("taskId"));
         boolean waitForCompletion = request.paramAsBoolean("wait_for_completion", false);
         TimeValue timeout = request.paramAsTime("timeout", null);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java
index 8fa13e808ac..5d1c617e8e3 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.admin.cluster.node.tasks;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
@@ -42,8 +42,8 @@ public class RestListTasksAction extends BaseRestHandler {
     private final ClusterService clusterService;
 
     @Inject
-    public RestListTasksAction(Settings settings, RestController controller, Client client, ClusterService clusterService) {
-        super(settings, client);
+    public RestListTasksAction(Settings settings, RestController controller, ClusterService clusterService) {
+        super(settings);
         this.clusterService = clusterService;
         controller.registerHandler(GET, "/_tasks", this);
     }
@@ -67,7 +67,7 @@ public class RestListTasksAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         ActionListener<ListTasksResponse> listener = nodeSettingListener(clusterService, new RestToXContentListener<>(channel));
         client.admin().cluster().listTasks(generateListTasksRequest(request), listener);
     }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/delete/RestDeleteRepositoryAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/delete/RestDeleteRepositoryAction.java
index 136c1cfae3f..c2dce1cae36 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/delete/RestDeleteRepositoryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/delete/RestDeleteRepositoryAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.admin.cluster.repositories.delete;
 
 import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest;
 import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -39,13 +39,13 @@ import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 public class RestDeleteRepositoryAction extends BaseRestHandler {
 
     @Inject
-    public RestDeleteRepositoryAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestDeleteRepositoryAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(DELETE, "/_snapshot/{repository}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         DeleteRepositoryRequest deleteRepositoryRequest = deleteRepositoryRequest(request.param("repository"));
         deleteRepositoryRequest.masterNodeTimeout(request.paramAsTime("master_timeout", deleteRepositoryRequest.masterNodeTimeout()));
         deleteRepositoryRequest.timeout(request.paramAsTime("timeout", deleteRepositoryRequest.timeout()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/get/RestGetRepositoriesAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/get/RestGetRepositoriesAction.java
index 09422481cf3..34004156d5b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/get/RestGetRepositoriesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/get/RestGetRepositoriesAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.admin.cluster.repositories.get;
 
 import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
 import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.RepositoriesMetaData;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.Strings;
@@ -49,15 +49,15 @@ public class RestGetRepositoriesAction extends BaseRestHandler {
     private final SettingsFilter settingsFilter;
 
     @Inject
-    public RestGetRepositoriesAction(Settings settings, RestController controller, Client client, SettingsFilter settingsFilter) {
-        super(settings, client);
+    public RestGetRepositoriesAction(Settings settings, RestController controller, SettingsFilter settingsFilter) {
+        super(settings);
         controller.registerHandler(GET, "/_snapshot", this);
         controller.registerHandler(GET, "/_snapshot/{repository}", this);
         this.settingsFilter = settingsFilter;
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final String[] repositories = request.paramAsStringArray("repository", Strings.EMPTY_ARRAY);
         GetRepositoriesRequest getRepositoriesRequest = getRepositoryRequest(repositories);
         getRepositoriesRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getRepositoriesRequest.masterNodeTimeout()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/put/RestPutRepositoryAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/put/RestPutRepositoryAction.java
index 878eb2915bc..745dd7e0b9f 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/put/RestPutRepositoryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/put/RestPutRepositoryAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.admin.cluster.repositories.put;
 
 import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
 import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -40,15 +40,15 @@ import static org.elasticsearch.rest.RestRequest.Method.PUT;
 public class RestPutRepositoryAction extends BaseRestHandler {
 
     @Inject
-    public RestPutRepositoryAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestPutRepositoryAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(PUT, "/_snapshot/{repository}", this);
         controller.registerHandler(POST, "/_snapshot/{repository}", this);
     }
 
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         PutRepositoryRequest putRepositoryRequest = putRepositoryRequest(request.param("repository"));
         putRepositoryRequest.source(request.content().toUtf8());
         putRepositoryRequest.verify(request.paramAsBoolean("verify", true));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/verify/RestVerifyRepositoryAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/verify/RestVerifyRepositoryAction.java
index 306dcbb21b9..b3f6fe776fe 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/verify/RestVerifyRepositoryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/verify/RestVerifyRepositoryAction.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.rest.action.admin.cluster.repositories.verify;
 
 import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -35,13 +35,13 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 public class RestVerifyRepositoryAction extends BaseRestHandler {
 
     @Inject
-    public RestVerifyRepositoryAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestVerifyRepositoryAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(POST, "/_snapshot/{repository}/_verify", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         VerifyRepositoryRequest verifyRepositoryRequest = verifyRepositoryRequest(request.param("repository"));
         verifyRepositoryRequest.masterNodeTimeout(request.paramAsTime("master_timeout", verifyRepositoryRequest.masterNodeTimeout()));
         verifyRepositoryRequest.timeout(request.paramAsTime("timeout", verifyRepositoryRequest.timeout()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/reroute/RestClusterRerouteAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/reroute/RestClusterRerouteAction.java
index a06466137a5..ad4abaf10c4 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/reroute/RestClusterRerouteAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/reroute/RestClusterRerouteAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.admin.cluster.reroute;
 
 import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteRequest;
 import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.allocation.command.AllocationCommandRegistry;
@@ -65,16 +65,16 @@ public class RestClusterRerouteAction extends BaseRestHandler {
     private final AllocationCommandRegistry registry;
 
     @Inject
-    public RestClusterRerouteAction(Settings settings, RestController controller, Client client, SettingsFilter settingsFilter,
+    public RestClusterRerouteAction(Settings settings, RestController controller, SettingsFilter settingsFilter,
             AllocationCommandRegistry registry) {
-        super(settings, client);
+        super(settings);
         this.settingsFilter = settingsFilter;
         this.registry = registry;
         controller.registerHandler(RestRequest.Method.POST, "/_cluster/reroute", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws Exception {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
         ClusterRerouteRequest clusterRerouteRequest = createRequest(request, registry, parseFieldMatcher);
         client.admin().cluster().reroute(clusterRerouteRequest, new AcknowledgedRestListener<ClusterRerouteResponse>(channel) {
             @Override
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterGetSettingsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterGetSettingsAction.java
index 44a7f2f714b..a1e85c5d298 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterGetSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterGetSettingsAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.admin.cluster.settings;
 
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.inject.Inject;
@@ -50,15 +50,15 @@ public class RestClusterGetSettingsAction extends BaseRestHandler {
     private final SettingsFilter settingsFilter;
 
     @Inject
-    public RestClusterGetSettingsAction(Settings settings, RestController controller, Client client, ClusterSettings clusterSettings, SettingsFilter settingsFilter) {
-        super(settings, client);
+    public RestClusterGetSettingsAction(Settings settings, RestController controller, ClusterSettings clusterSettings, SettingsFilter settingsFilter) {
+        super(settings);
         this.clusterSettings = clusterSettings;
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/settings", this);
         this.settingsFilter = settingsFilter;
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         ClusterStateRequest clusterStateRequest = Requests.clusterStateRequest()
                 .routingTable(false)
                 .nodes(false);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterUpdateSettingsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterUpdateSettingsAction.java
index b25866d4520..055bcc4df60 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterUpdateSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/settings/RestClusterUpdateSettingsAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.admin.cluster.settings;
 
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -42,13 +42,13 @@ import java.util.Map;
 public class RestClusterUpdateSettingsAction extends BaseRestHandler {
 
     @Inject
-    public RestClusterUpdateSettingsAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestClusterUpdateSettingsAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(RestRequest.Method.PUT, "/_cluster/settings", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws Exception {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
         final ClusterUpdateSettingsRequest clusterUpdateSettingsRequest = Requests.clusterUpdateSettingsRequest();
         clusterUpdateSettingsRequest.timeout(request.paramAsTime("timeout", clusterUpdateSettingsRequest.timeout()));
         clusterUpdateSettingsRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterUpdateSettingsRequest.masterNodeTimeout()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/shards/RestClusterSearchShardsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/shards/RestClusterSearchShardsAction.java
index 860e110b2d6..82edb74c7e6 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/shards/RestClusterSearchShardsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/shards/RestClusterSearchShardsAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.admin.cluster.shards;
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest;
 import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
@@ -41,8 +41,8 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 public class RestClusterSearchShardsAction extends BaseRestHandler {
 
     @Inject
-    public RestClusterSearchShardsAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestClusterSearchShardsAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(GET, "/_search_shards", this);
         controller.registerHandler(POST, "/_search_shards", this);
         controller.registerHandler(GET, "/{index}/_search_shards", this);
@@ -52,7 +52,7 @@ public class RestClusterSearchShardsAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         final ClusterSearchShardsRequest clusterSearchShardsRequest = Requests.clusterSearchShardsRequest(indices);
         clusterSearchShardsRequest.local(request.paramAsBoolean("local", clusterSearchShardsRequest.local()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/create/RestCreateSnapshotAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/create/RestCreateSnapshotAction.java
index 9d6be664d48..3be27d286e2 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/create/RestCreateSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/create/RestCreateSnapshotAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.admin.cluster.snapshots.create;
 
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -40,14 +40,14 @@ import static org.elasticsearch.rest.RestRequest.Method.PUT;
 public class RestCreateSnapshotAction extends BaseRestHandler {
 
     @Inject
-    public RestCreateSnapshotAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestCreateSnapshotAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(PUT, "/_snapshot/{repository}/{snapshot}", this);
         controller.registerHandler(POST, "/_snapshot/{repository}/{snapshot}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         CreateSnapshotRequest createSnapshotRequest = createSnapshotRequest(request.param("repository"), request.param("snapshot"));
         createSnapshotRequest.source(request.content().toUtf8());
         createSnapshotRequest.masterNodeTimeout(request.paramAsTime("master_timeout", createSnapshotRequest.masterNodeTimeout()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/delete/RestDeleteSnapshotAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/delete/RestDeleteSnapshotAction.java
index 38c78bd5d88..5b99cb3a5ee 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/delete/RestDeleteSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/delete/RestDeleteSnapshotAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.admin.cluster.snapshots.delete;
 
 import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -39,13 +39,13 @@ import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 public class RestDeleteSnapshotAction extends BaseRestHandler {
 
     @Inject
-    public RestDeleteSnapshotAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestDeleteSnapshotAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(DELETE, "/_snapshot/{repository}/{snapshot}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         DeleteSnapshotRequest deleteSnapshotRequest = deleteSnapshotRequest(request.param("repository"), request.param("snapshot"));
         deleteSnapshotRequest.masterNodeTimeout(request.paramAsTime("master_timeout", deleteSnapshotRequest.masterNodeTimeout()));
         client.admin().cluster().deleteSnapshot(deleteSnapshotRequest, new AcknowledgedRestListener<DeleteSnapshotResponse>(channel));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/get/RestGetSnapshotsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/get/RestGetSnapshotsAction.java
index 1151fed8f23..835c23ac191 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/get/RestGetSnapshotsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/get/RestGetSnapshotsAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.admin.cluster.snapshots.get;
 
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -40,14 +40,14 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 public class RestGetSnapshotsAction extends BaseRestHandler {
 
     @Inject
-    public RestGetSnapshotsAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestGetSnapshotsAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(GET, "/_snapshot/{repository}/{snapshot}", this);
     }
 
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         String repository = request.param("repository");
         String[] snapshots = request.paramAsStringArray("snapshot", Strings.EMPTY_ARRAY);
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/restore/RestRestoreSnapshotAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/restore/RestRestoreSnapshotAction.java
index e2a16bd4b46..6a1eb0eaa50 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/restore/RestRestoreSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/restore/RestRestoreSnapshotAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.admin.cluster.snapshots.restore;
 
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -39,13 +39,13 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 public class RestRestoreSnapshotAction extends BaseRestHandler {
 
     @Inject
-    public RestRestoreSnapshotAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestRestoreSnapshotAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(POST, "/_snapshot/{repository}/{snapshot}/_restore", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         RestoreSnapshotRequest restoreSnapshotRequest = restoreSnapshotRequest(request.param("repository"), request.param("snapshot"));
         restoreSnapshotRequest.masterNodeTimeout(request.paramAsTime("master_timeout", restoreSnapshotRequest.masterNodeTimeout()));
         restoreSnapshotRequest.waitForCompletion(request.paramAsBoolean("wait_for_completion", false));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/status/RestSnapshotsStatusAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/status/RestSnapshotsStatusAction.java
index 2e8810e2ba7..484ee4e5120 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/status/RestSnapshotsStatusAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/status/RestSnapshotsStatusAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.admin.cluster.snapshots.status;
 
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -40,15 +40,15 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 public class RestSnapshotsStatusAction extends BaseRestHandler {
 
     @Inject
-    public RestSnapshotsStatusAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestSnapshotsStatusAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(GET, "/_snapshot/{repository}/{snapshot}/_status", this);
         controller.registerHandler(GET, "/_snapshot/{repository}/_status", this);
         controller.registerHandler(GET, "/_snapshot/_status", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         String repository = request.param("repository", "_all");
         String[] snapshots = request.paramAsStringArray("snapshot", Strings.EMPTY_ARRAY);
         if (snapshots.length == 1 && "_all".equalsIgnoreCase(snapshots[0])) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java
index c756796446e..b3f5a407d91 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/state/RestClusterStateAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.admin.cluster.state;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.Strings;
@@ -50,8 +50,8 @@ public class RestClusterStateAction extends BaseRestHandler {
     private final SettingsFilter settingsFilter;
 
     @Inject
-    public RestClusterStateAction(Settings settings, RestController controller, Client client, SettingsFilter settingsFilter) {
-        super(settings, client);
+    public RestClusterStateAction(Settings settings, RestController controller, SettingsFilter settingsFilter) {
+        super(settings);
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/state", this);
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/state/{metric}", this);
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/state/{metric}/{indices}", this);
@@ -60,7 +60,7 @@ public class RestClusterStateAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final ClusterStateRequest clusterStateRequest = Requests.clusterStateRequest();
         clusterStateRequest.indicesOptions(IndicesOptions.fromRequest(request, clusterStateRequest.indicesOptions()));
         clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/stats/RestClusterStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/stats/RestClusterStatsAction.java
index c83a5b7ae1b..a43b428ae9c 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/stats/RestClusterStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/stats/RestClusterStatsAction.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.rest.action.admin.cluster.stats;
 
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -35,14 +35,14 @@ import org.elasticsearch.rest.action.support.RestActions.NodesResponseRestListen
 public class RestClusterStatsAction extends BaseRestHandler {
 
     @Inject
-    public RestClusterStatsAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestClusterStatsAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/stats", this);
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/stats/nodes/{nodeId}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest().nodesIds(request.paramAsStringArray("nodeId", null));
         clusterStatsRequest.timeout(request.param("timeout"));
         client.admin().cluster().clusterStats(clusterStatsRequest, new NodesResponseRestListener<>(channel));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestDeleteStoredScriptAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestDeleteStoredScriptAction.java
index a8190d30278..6416a693550 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestDeleteStoredScriptAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestDeleteStoredScriptAction.java
@@ -19,7 +19,7 @@
 package org.elasticsearch.rest.action.admin.cluster.storedscripts;
 
 import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -33,12 +33,12 @@ import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 public class RestDeleteStoredScriptAction extends BaseRestHandler {
 
     @Inject
-    public RestDeleteStoredScriptAction(Settings settings, RestController controller, Client client) {
-        this(settings, controller, true, client);
+    public RestDeleteStoredScriptAction(Settings settings, RestController controller) {
+        this(settings, controller, true);
     }
 
-    protected RestDeleteStoredScriptAction(Settings settings, RestController controller, boolean registerDefaultHandlers, Client client) {
-        super(settings, client);
+    protected RestDeleteStoredScriptAction(Settings settings, RestController controller, boolean registerDefaultHandlers) {
+        super(settings);
         if (registerDefaultHandlers) {
             controller.registerHandler(DELETE, "/_scripts/{lang}/{id}", this);
         }
@@ -49,7 +49,7 @@ public class RestDeleteStoredScriptAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, NodeClient client) {
         DeleteStoredScriptRequest deleteStoredScriptRequest = new DeleteStoredScriptRequest(getScriptLang(request), request.param("id"));
         client.admin().cluster().deleteStoredScript(deleteStoredScriptRequest, new AcknowledgedRestListener<>(channel));
     }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestGetStoredScriptAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestGetStoredScriptAction.java
index bf286f623fb..fb6fd0ca7c9 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestGetStoredScriptAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestGetStoredScriptAction.java
@@ -20,7 +20,7 @@ package org.elasticsearch.rest.action.admin.cluster.storedscripts;
 
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -38,12 +38,12 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 public class RestGetStoredScriptAction extends BaseRestHandler {
 
     @Inject
-    public RestGetStoredScriptAction(Settings settings, RestController controller, Client client) {
-        this(settings, controller, true, client);
+    public RestGetStoredScriptAction(Settings settings, RestController controller) {
+        this(settings, controller, true);
     }
 
-    protected RestGetStoredScriptAction(Settings settings, RestController controller, boolean registerDefaultHandlers, Client client) {
-        super(settings, client);
+    protected RestGetStoredScriptAction(Settings settings, RestController controller, boolean registerDefaultHandlers) {
+        super(settings);
         if (registerDefaultHandlers) {
             controller.registerHandler(GET, "/_scripts/{lang}/{id}", this);
         }
@@ -58,7 +58,7 @@ public class RestGetStoredScriptAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, NodeClient client) {
         final GetStoredScriptRequest getRequest = new GetStoredScriptRequest(getScriptLang(request), request.param("id"));
         client.admin().cluster().getStoredScript(getRequest, new RestBuilderListener<GetStoredScriptResponse>(channel) {
             @Override
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestPutStoredScriptAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestPutStoredScriptAction.java
index 0ecb27b8d1e..bb62528fc17 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestPutStoredScriptAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/storedscripts/RestPutStoredScriptAction.java
@@ -19,7 +19,7 @@
 package org.elasticsearch.rest.action.admin.cluster.storedscripts;
 
 import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -34,12 +34,12 @@ import static org.elasticsearch.rest.RestRequest.Method.PUT;
 public class RestPutStoredScriptAction extends BaseRestHandler {
 
     @Inject
-    public RestPutStoredScriptAction(Settings settings, RestController controller, Client client) {
-        this(settings, controller, true, client);
+    public RestPutStoredScriptAction(Settings settings, RestController controller) {
+        this(settings, controller, true);
     }
 
-    protected RestPutStoredScriptAction(Settings settings, RestController controller, boolean registerDefaultHandlers, Client client) {
-        super(settings, client);
+    protected RestPutStoredScriptAction(Settings settings, RestController controller, boolean registerDefaultHandlers) {
+        super(settings);
         if (registerDefaultHandlers) {
             controller.registerHandler(POST, "/_scripts/{lang}/{id}", this);
             controller.registerHandler(PUT, "/_scripts/{lang}/{id}", this);
@@ -51,7 +51,7 @@ public class RestPutStoredScriptAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, NodeClient client) {
         PutStoredScriptRequest putRequest = new PutStoredScriptRequest(getScriptLang(request), request.param("id"));
         putRequest.script(request.content());
         client.admin().cluster().putStoredScript(putRequest, new AcknowledgedRestListener<>(channel));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/tasks/RestPendingClusterTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/tasks/RestPendingClusterTasksAction.java
index 333b6d64491..e106dd1f980 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/tasks/RestPendingClusterTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/tasks/RestPendingClusterTasksAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.admin.cluster.tasks;
 
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -35,13 +35,13 @@ import org.elasticsearch.rest.action.support.RestToXContentListener;
 public class RestPendingClusterTasksAction extends BaseRestHandler {
 
     @Inject
-    public RestPendingClusterTasksAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestPendingClusterTasksAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(RestRequest.Method.GET, "/_cluster/pending_tasks", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         PendingClusterTasksRequest pendingClusterTasksRequest = new PendingClusterTasksRequest();
         pendingClusterTasksRequest.masterNodeTimeout(request.paramAsTime("master_timeout", pendingClusterTasksRequest.masterNodeTimeout()));
         pendingClusterTasksRequest.local(request.paramAsBoolean("local", pendingClusterTasksRequest.local()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
index 7eecaaa738c..a922edff484 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.rest.action.admin.indices;
 
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -35,15 +35,15 @@ import org.elasticsearch.rest.action.support.RestToXContentListener;
 public class RestRolloverIndexAction extends BaseRestHandler {
 
     @Inject
-    public RestRolloverIndexAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestRolloverIndexAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(RestRequest.Method.POST, "/{index}/_rollover", this);
         controller.registerHandler(RestRequest.Method.POST, "/{index}/_rollover/{new_index}", this);
     }
 
     @SuppressWarnings({"unchecked"})
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         RolloverRequest rolloverIndexRequest = new RolloverRequest(request.param("index"), request.param("new_index"));
         if (request.hasContent()) {
             rolloverIndexRequest.source(request.content());
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java
index f145bcbf02a..96b4abbe063 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.rest.action.admin.indices;
 
 import org.elasticsearch.action.admin.indices.shrink.ShrinkRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -35,15 +35,15 @@ import org.elasticsearch.rest.action.support.AcknowledgedRestListener;
 public class RestShrinkIndexAction extends BaseRestHandler {
 
     @Inject
-    public RestShrinkIndexAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestShrinkIndexAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(RestRequest.Method.PUT, "/{index}/_shrink/{target}", this);
         controller.registerHandler(RestRequest.Method.POST, "/{index}/_shrink/{target}", this);
     }
 
     @SuppressWarnings({"unchecked"})
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         if (request.param("target") == null) {
             throw new IllegalArgumentException("no target index");
         }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/RestIndicesAliasesAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/RestIndicesAliasesAction.java
index 74982133f71..dff4c492f17 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/RestIndicesAliasesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/RestIndicesAliasesAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.admin.indices.alias;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.AliasAction;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -46,13 +46,13 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 public class RestIndicesAliasesAction extends BaseRestHandler {
 
     @Inject
-    public RestIndicesAliasesAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestIndicesAliasesAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(POST, "/_aliases", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws Exception {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
         IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest();
         indicesAliasesRequest.masterNodeTimeout(request.paramAsTime("master_timeout", indicesAliasesRequest.masterNodeTimeout()));
         try (XContentParser parser = XContentFactory.xContent(request.content()).createParser(request.content())) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/delete/RestIndexDeleteAliasesAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/delete/RestIndexDeleteAliasesAction.java
index 7fcaadc3d8b..eedac4d8eea 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/delete/RestIndexDeleteAliasesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/delete/RestIndexDeleteAliasesAction.java
@@ -20,7 +20,7 @@ package org.elasticsearch.rest.action.admin.indices.alias.delete;
 
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -37,14 +37,14 @@ import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 public class RestIndexDeleteAliasesAction extends BaseRestHandler {
 
     @Inject
-    public RestIndexDeleteAliasesAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestIndexDeleteAliasesAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(DELETE, "/{index}/_alias/{name}", this);
         controller.registerHandler(DELETE, "/{index}/_aliases/{name}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         final String[] aliases = Strings.splitStringByCommaToArray(request.param("name"));
         IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest();
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/get/RestGetAliasesAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/get/RestGetAliasesAction.java
index cbebf08516d..8ecbc64b27f 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/get/RestGetAliasesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/get/RestGetAliasesAction.java
@@ -23,7 +23,7 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
@@ -50,14 +50,14 @@ import static org.elasticsearch.rest.RestStatus.OK;
 public class RestGetAliasesAction extends BaseRestHandler {
 
     @Inject
-    public RestGetAliasesAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestGetAliasesAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(GET, "/_alias/{name}", this);
         controller.registerHandler(GET, "/{index}/_alias/{name}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final String[] aliases = request.paramAsStringArrayOrEmptyIfAll("name");
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         final GetAliasesRequest getAliasesRequest = new GetAliasesRequest(aliases);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/head/RestAliasesExistAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/head/RestAliasesExistAction.java
index 65f8fd363ea..7c539e1be48 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/head/RestAliasesExistAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/head/RestAliasesExistAction.java
@@ -24,7 +24,7 @@ import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.exists.AliasesExistResponse;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.inject.Inject;
@@ -44,15 +44,15 @@ import static org.elasticsearch.rest.RestStatus.OK;
 public class RestAliasesExistAction extends BaseRestHandler {
 
     @Inject
-    public RestAliasesExistAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestAliasesExistAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(HEAD, "/_alias/{name}", this);
         controller.registerHandler(HEAD, "/{index}/_alias/{name}", this);
         controller.registerHandler(HEAD, "/{index}/_alias", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         String[] aliases = request.paramAsStringArray("name", Strings.EMPTY_ARRAY);
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         GetAliasesRequest getAliasesRequest = new GetAliasesRequest(aliases);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/put/RestIndexPutAliasAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/put/RestIndexPutAliasAction.java
index 7a0c2ad466f..e9f5e294db4 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/put/RestIndexPutAliasAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/put/RestIndexPutAliasAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.admin.indices.alias.put;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.AliasAction;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
@@ -44,8 +44,8 @@ import static org.elasticsearch.rest.RestRequest.Method.PUT;
 public class RestIndexPutAliasAction extends BaseRestHandler {
 
     @Inject
-    public RestIndexPutAliasAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestIndexPutAliasAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(PUT, "/{index}/_alias/{name}", this);
         controller.registerHandler(PUT, "/_alias/{name}", this);
         controller.registerHandler(PUT, "/{index}/_aliases/{name}", this);
@@ -62,7 +62,7 @@ public class RestIndexPutAliasAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws Exception {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
         String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         String alias = request.param("name");
         Map<String, Object> filter = null;
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/analyze/RestAnalyzeAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/analyze/RestAnalyzeAction.java
index 75caea19856..588771ae1c8 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/analyze/RestAnalyzeAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/analyze/RestAnalyzeAction.java
@@ -20,7 +20,7 @@ package org.elasticsearch.rest.action.admin.indices.analyze;
 
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -60,8 +60,8 @@ public class RestAnalyzeAction extends BaseRestHandler {
     }
 
     @Inject
-    public RestAnalyzeAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestAnalyzeAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(GET, "/_analyze", this);
         controller.registerHandler(GET, "/{index}/_analyze", this);
         controller.registerHandler(POST, "/_analyze", this);
@@ -69,7 +69,7 @@ public class RestAnalyzeAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
 
         String[] texts = request.paramAsStringArrayOrEmptyIfAll("text");
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/cache/clear/RestClearIndicesCacheAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/cache/clear/RestClearIndicesCacheAction.java
index 97e424445d8..454e18cb2d7 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/cache/clear/RestClearIndicesCacheAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/cache/clear/RestClearIndicesCacheAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.admin.indices.cache.clear;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
@@ -50,8 +50,8 @@ import static org.elasticsearch.rest.action.support.RestActions.buildBroadcastSh
 public class RestClearIndicesCacheAction extends BaseRestHandler {
 
     @Inject
-    public RestClearIndicesCacheAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestClearIndicesCacheAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(POST, "/_cache/clear", this);
         controller.registerHandler(POST, "/{index}/_cache/clear", this);
 
@@ -60,7 +60,7 @@ public class RestClearIndicesCacheAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         ClearIndicesCacheRequest clearIndicesCacheRequest = new ClearIndicesCacheRequest(Strings.splitStringByCommaToArray(request.param("index")));
         clearIndicesCacheRequest.indicesOptions(IndicesOptions.fromRequest(request, clearIndicesCacheRequest.indicesOptions()));
         fromRequest(request, clearIndicesCacheRequest, parseFieldMatcher);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/close/RestCloseIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/close/RestCloseIndexAction.java
index 5f211b88d11..1022310a350 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/close/RestCloseIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/close/RestCloseIndexAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.admin.indices.close;
 import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
 import org.elasticsearch.action.admin.indices.close.CloseIndexResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -38,14 +38,14 @@ import org.elasticsearch.rest.action.support.AcknowledgedRestListener;
 public class RestCloseIndexAction extends BaseRestHandler {
 
     @Inject
-    public RestCloseIndexAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestCloseIndexAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(RestRequest.Method.POST, "/_close", this);
         controller.registerHandler(RestRequest.Method.POST, "/{index}/_close", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         CloseIndexRequest closeIndexRequest = new CloseIndexRequest(Strings.splitStringByCommaToArray(request.param("index")));
         closeIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", closeIndexRequest.masterNodeTimeout()));
         closeIndexRequest.timeout(request.paramAsTime("timeout", closeIndexRequest.timeout()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/create/RestCreateIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/create/RestCreateIndexAction.java
index 46bc9388972..46ee596286e 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/create/RestCreateIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/create/RestCreateIndexAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.admin.indices.create;
 
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -36,15 +36,15 @@ import org.elasticsearch.rest.action.support.AcknowledgedRestListener;
 public class RestCreateIndexAction extends BaseRestHandler {
 
     @Inject
-    public RestCreateIndexAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestCreateIndexAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(RestRequest.Method.PUT, "/{index}", this);
         controller.registerHandler(RestRequest.Method.POST, "/{index}", this);
     }
 
     @SuppressWarnings({"unchecked"})
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         CreateIndexRequest createIndexRequest = new CreateIndexRequest(request.param("index"));
         if (request.hasContent()) {
             createIndexRequest.source(request.content());
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/delete/RestDeleteIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/delete/RestDeleteIndexAction.java
index 4953842c54a..75564021a3b 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/delete/RestDeleteIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/delete/RestDeleteIndexAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.admin.indices.delete;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -38,14 +38,14 @@ import org.elasticsearch.rest.action.support.AcknowledgedRestListener;
 public class RestDeleteIndexAction extends BaseRestHandler {
 
     @Inject
-    public RestDeleteIndexAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestDeleteIndexAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(RestRequest.Method.DELETE, "/", this);
         controller.registerHandler(RestRequest.Method.DELETE, "/{index}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(Strings.splitStringByCommaToArray(request.param("index")));
         deleteIndexRequest.timeout(request.paramAsTime("timeout", deleteIndexRequest.timeout()));
         deleteIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", deleteIndexRequest.masterNodeTimeout()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/exists/indices/RestIndicesExistsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/exists/indices/RestIndicesExistsAction.java
index 0e240352a73..f45f607356e 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/exists/indices/RestIndicesExistsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/exists/indices/RestIndicesExistsAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.admin.indices.exists.indices;
 import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
 import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.inject.Inject;
@@ -45,13 +45,13 @@ import static org.elasticsearch.rest.RestStatus.OK;
 public class RestIndicesExistsAction extends BaseRestHandler {
 
     @Inject
-    public RestIndicesExistsAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestIndicesExistsAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(HEAD, "/{index}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(Strings.splitStringByCommaToArray(request.param("index")));
         indicesExistsRequest.indicesOptions(IndicesOptions.fromRequest(request, indicesExistsRequest.indicesOptions()));
         indicesExistsRequest.local(request.paramAsBoolean("local", indicesExistsRequest.local()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/exists/types/RestTypesExistsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/exists/types/RestTypesExistsAction.java
index 7a55be7e3af..2ac0d1354f1 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/exists/types/RestTypesExistsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/exists/types/RestTypesExistsAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.admin.indices.exists.types;
 import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequest;
 import org.elasticsearch.action.admin.indices.exists.types.TypesExistsResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.inject.Inject;
@@ -44,13 +44,13 @@ import static org.elasticsearch.rest.RestStatus.OK;
 public class RestTypesExistsAction extends BaseRestHandler {
 
     @Inject
-    public RestTypesExistsAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestTypesExistsAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(HEAD, "/{index}/{type}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         TypesExistsRequest typesExistsRequest = new TypesExistsRequest(
                 Strings.splitStringByCommaToArray(request.param("index")), Strings.splitStringByCommaToArray(request.param("type"))
         );
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/flush/RestFlushAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/flush/RestFlushAction.java
index f3b3304bcf9..5156a95c0cc 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/flush/RestFlushAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/flush/RestFlushAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.admin.indices.flush;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.flush.FlushResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -46,8 +46,8 @@ import static org.elasticsearch.rest.action.support.RestActions.buildBroadcastSh
 public class RestFlushAction extends BaseRestHandler {
 
     @Inject
-    public RestFlushAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestFlushAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(POST, "/_flush", this);
         controller.registerHandler(POST, "/{index}/_flush", this);
 
@@ -56,7 +56,7 @@ public class RestFlushAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         FlushRequest flushRequest = new FlushRequest(Strings.splitStringByCommaToArray(request.param("index")));
         flushRequest.indicesOptions(IndicesOptions.fromRequest(request, flushRequest.indicesOptions()));
         flushRequest.force(request.paramAsBoolean("force", flushRequest.force()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/flush/RestSyncedFlushAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/flush/RestSyncedFlushAction.java
index 9bb36f03d65..70d311c6e04 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/flush/RestSyncedFlushAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/flush/RestSyncedFlushAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.admin.indices.flush;
 import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequest;
 import org.elasticsearch.action.admin.indices.flush.SyncedFlushResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -44,8 +44,8 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 public class RestSyncedFlushAction extends BaseRestHandler {
 
     @Inject
-    public RestSyncedFlushAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestSyncedFlushAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(POST, "/_flush/synced", this);
         controller.registerHandler(POST, "/{index}/_flush/synced", this);
 
@@ -54,7 +54,7 @@ public class RestSyncedFlushAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         IndicesOptions indicesOptions = IndicesOptions.fromRequest(request, IndicesOptions.lenientExpandOpen());
         SyncedFlushRequest syncedFlushRequest = new SyncedFlushRequest(Strings.splitStringByCommaToArray(request.param("index")));
         syncedFlushRequest.indicesOptions(indicesOptions);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/forcemerge/RestForceMergeAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/forcemerge/RestForceMergeAction.java
index 8aa2683be5e..ecdbdeae14c 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/forcemerge/RestForceMergeAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/forcemerge/RestForceMergeAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.admin.indices.forcemerge;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -45,14 +45,14 @@ import static org.elasticsearch.rest.action.support.RestActions.buildBroadcastSh
 public class RestForceMergeAction extends BaseRestHandler {
 
     @Inject
-    public RestForceMergeAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestForceMergeAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(POST, "/_forcemerge", this);
         controller.registerHandler(POST, "/{index}/_forcemerge", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         ForceMergeRequest mergeRequest = new ForceMergeRequest(Strings.splitStringByCommaToArray(request.param("index")));
         mergeRequest.indicesOptions(IndicesOptions.fromRequest(request, mergeRequest.indicesOptions()));
         mergeRequest.maxNumSegments(request.paramAsInt("max_num_segments", mergeRequest.maxNumSegments()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/get/RestGetIndicesAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/get/RestGetIndicesAction.java
index f31c17d8b6e..578cd2e4a62 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/get/RestGetIndicesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/get/RestGetIndicesAction.java
@@ -23,7 +23,7 @@ import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest.Feature;
 import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.common.Strings;
@@ -57,8 +57,8 @@ public class RestGetIndicesAction extends BaseRestHandler {
     private final SettingsFilter settingsFilter;
 
     @Inject
-    public RestGetIndicesAction(Settings settings, RestController controller, Client client, IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter) {
-        super(settings, client);
+    public RestGetIndicesAction(Settings settings, RestController controller, IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter) {
+        super(settings);
         this.indexScopedSettings = indexScopedSettings;
         controller.registerHandler(GET, "/{index}", this);
         controller.registerHandler(GET, "/{index}/{type}", this);
@@ -66,7 +66,7 @@ public class RestGetIndicesAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         String[] featureParams = request.paramAsStringArray("type", null);
         // Work out if the indices is a list of features
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/mapping/get/RestGetFieldMappingAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/mapping/get/RestGetFieldMappingAction.java
index 0db931d0a7a..d2f77e95f52 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/mapping/get/RestGetFieldMappingAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/mapping/get/RestGetFieldMappingAction.java
@@ -23,7 +23,7 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsReques
 import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
 import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -50,8 +50,8 @@ import static org.elasticsearch.rest.RestStatus.OK;
 public class RestGetFieldMappingAction extends BaseRestHandler {
 
     @Inject
-    public RestGetFieldMappingAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestGetFieldMappingAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(GET, "/_mapping/field/{fields}", this);
         controller.registerHandler(GET, "/_mapping/{type}/field/{fields}", this);
         controller.registerHandler(GET, "/{index}/_mapping/field/{fields}", this);
@@ -60,7 +60,7 @@ public class RestGetFieldMappingAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         final String[] types = request.paramAsStringArrayOrEmptyIfAll("type");
         final String[] fields = Strings.splitStringByCommaToArray(request.param("fields"));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/mapping/get/RestGetMappingAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/mapping/get/RestGetMappingAction.java
index 0da54bc4e75..c4f9dbf2333 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/mapping/get/RestGetMappingAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/mapping/get/RestGetMappingAction.java
@@ -23,7 +23,7 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -50,8 +50,8 @@ import static org.elasticsearch.rest.RestStatus.OK;
 public class RestGetMappingAction extends BaseRestHandler {
 
     @Inject
-    public RestGetMappingAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestGetMappingAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(GET, "/{index}/{type}/_mapping", this);
         controller.registerHandler(GET, "/{index}/_mappings/{type}", this);
         controller.registerHandler(GET, "/{index}/_mapping/{type}", this);
@@ -59,7 +59,7 @@ public class RestGetMappingAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         final String[] types = request.paramAsStringArrayOrEmptyIfAll("type");
         GetMappingsRequest getMappingsRequest = new GetMappingsRequest();
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/mapping/put/RestPutMappingAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/mapping/put/RestPutMappingAction.java
index fdb16d2fb8f..cb2493f4e33 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/mapping/put/RestPutMappingAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/mapping/put/RestPutMappingAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.admin.indices.mapping.put;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -43,8 +43,8 @@ public class RestPutMappingAction extends BaseRestHandler {
 
 
     @Inject
-    public RestPutMappingAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestPutMappingAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(PUT, "/{index}/_mapping/", this);
         controller.registerHandler(PUT, "/{index}/{type}/_mapping", this);
         controller.registerHandler(PUT, "/{index}/_mapping/{type}", this);
@@ -68,7 +68,7 @@ public class RestPutMappingAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         PutMappingRequest putMappingRequest = putMappingRequest(Strings.splitStringByCommaToArray(request.param("index")));
         putMappingRequest.type(request.param("type"));
         putMappingRequest.source(request.content().toUtf8());
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/open/RestOpenIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/open/RestOpenIndexAction.java
index 58bda9d3a3d..9f27d8e3666 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/open/RestOpenIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/open/RestOpenIndexAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.admin.indices.open;
 import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
 import org.elasticsearch.action.admin.indices.open.OpenIndexResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -38,14 +38,14 @@ import org.elasticsearch.rest.action.support.AcknowledgedRestListener;
 public class RestOpenIndexAction extends BaseRestHandler {
 
     @Inject
-    public RestOpenIndexAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestOpenIndexAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(RestRequest.Method.POST, "/_open", this);
         controller.registerHandler(RestRequest.Method.POST, "/{index}/_open", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         OpenIndexRequest openIndexRequest = new OpenIndexRequest(Strings.splitStringByCommaToArray(request.param("index")));
         openIndexRequest.timeout(request.paramAsTime("timeout", openIndexRequest.timeout()));
         openIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", openIndexRequest.masterNodeTimeout()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/recovery/RestRecoveryAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/recovery/RestRecoveryAction.java
index 88bc9fb8c9f..6178d2559b5 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/recovery/RestRecoveryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/recovery/RestRecoveryAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.admin.indices.recovery;
 import org.elasticsearch.action.admin.indices.recovery.RecoveryRequest;
 import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -44,14 +44,14 @@ import static org.elasticsearch.rest.RestStatus.OK;
 public class RestRecoveryAction extends BaseRestHandler {
 
     @Inject
-    public RestRecoveryAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestRecoveryAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(GET, "/_recovery", this);
         controller.registerHandler(GET, "/{index}/_recovery", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
 
         final RecoveryRequest recoveryRequest = new RecoveryRequest(Strings.splitStringByCommaToArray(request.param("index")));
         recoveryRequest.detailed(request.paramAsBoolean("detailed", false));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/refresh/RestRefreshAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/refresh/RestRefreshAction.java
index fcc6d240b34..6e2fcec3f51 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/refresh/RestRefreshAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/refresh/RestRefreshAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.admin.indices.refresh;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -46,8 +46,8 @@ import static org.elasticsearch.rest.action.support.RestActions.buildBroadcastSh
 public class RestRefreshAction extends BaseRestHandler {
 
     @Inject
-    public RestRefreshAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestRefreshAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(POST, "/_refresh", this);
         controller.registerHandler(POST, "/{index}/_refresh", this);
 
@@ -56,7 +56,7 @@ public class RestRefreshAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         RefreshRequest refreshRequest = new RefreshRequest(Strings.splitStringByCommaToArray(request.param("index")));
         refreshRequest.indicesOptions(IndicesOptions.fromRequest(request, refreshRequest.indicesOptions()));
         client.admin().indices().refresh(refreshRequest, new RestBuilderListener<RefreshResponse>(channel) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/segments/RestIndicesSegmentsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/segments/RestIndicesSegmentsAction.java
index da76a769ce4..a1d5560e674 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/segments/RestIndicesSegmentsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/segments/RestIndicesSegmentsAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.admin.indices.segments;
 import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse;
 import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -44,14 +44,14 @@ import static org.elasticsearch.rest.action.support.RestActions.buildBroadcastSh
 public class RestIndicesSegmentsAction extends BaseRestHandler {
 
     @Inject
-    public RestIndicesSegmentsAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestIndicesSegmentsAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(GET, "/_segments", this);
         controller.registerHandler(GET, "/{index}/_segments", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         IndicesSegmentsRequest indicesSegmentsRequest = new IndicesSegmentsRequest(Strings.splitStringByCommaToArray(request.param("index")));
         indicesSegmentsRequest.verbose(request.paramAsBoolean("verbose", false));
         indicesSegmentsRequest.indicesOptions(IndicesOptions.fromRequest(request, indicesSegmentsRequest.indicesOptions()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/settings/RestGetSettingsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/settings/RestGetSettingsAction.java
index 8c24e2c68e3..46a101161b6 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/settings/RestGetSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/settings/RestGetSettingsAction.java
@@ -23,7 +23,7 @@ import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.IndexScopedSettings;
@@ -47,8 +47,8 @@ public class RestGetSettingsAction extends BaseRestHandler {
     private final SettingsFilter settingsFilter;
 
     @Inject
-    public RestGetSettingsAction(Settings settings, RestController controller, Client client, IndexScopedSettings indexScopedSettings, final SettingsFilter settingsFilter) {
-        super(settings, client);
+    public RestGetSettingsAction(Settings settings, RestController controller, IndexScopedSettings indexScopedSettings, final SettingsFilter settingsFilter) {
+        super(settings);
         this.indexScopedSettings = indexScopedSettings;
         controller.registerHandler(GET, "/{index}/_settings/{name}", this);
         controller.registerHandler(GET, "/_settings/{name}", this);
@@ -57,7 +57,7 @@ public class RestGetSettingsAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final String[] names = request.paramAsStringArrayOrEmptyIfAll("name");
         final boolean renderDefaults = request.paramAsBoolean("include_defaults", false);
         GetSettingsRequest getSettingsRequest = new GetSettingsRequest()
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/settings/RestUpdateSettingsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/settings/RestUpdateSettingsAction.java
index 5bd102594d7..70608930917 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/settings/RestUpdateSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/settings/RestUpdateSettingsAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.admin.indices.settings;
 
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -53,14 +53,14 @@ public class RestUpdateSettingsAction extends BaseRestHandler {
             "allow_no_indices"));
 
     @Inject
-    public RestUpdateSettingsAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestUpdateSettingsAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(RestRequest.Method.PUT, "/{index}/_settings", this);
         controller.registerHandler(RestRequest.Method.PUT, "/_settings", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         UpdateSettingsRequest updateSettingsRequest = updateSettingsRequest(Strings.splitStringByCommaToArray(request.param("index")));
         updateSettingsRequest.timeout(request.paramAsTime("timeout", updateSettingsRequest.timeout()));
         updateSettingsRequest.setPreserveExisting(request.paramAsBoolean("preserve_existing", updateSettingsRequest.isPreserveExisting()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/shards/RestIndicesShardStoresAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/shards/RestIndicesShardStoresAction.java
index 586599c1a1e..8a0d61c933c 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/shards/RestIndicesShardStoresAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/shards/RestIndicesShardStoresAction.java
@@ -23,7 +23,7 @@ import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresAction;
 import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresRequest;
 import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -45,14 +45,14 @@ import static org.elasticsearch.rest.RestStatus.OK;
 public class RestIndicesShardStoresAction extends BaseRestHandler {
 
     @Inject
-    public RestIndicesShardStoresAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestIndicesShardStoresAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(GET, "/_shard_stores", this);
         controller.registerHandler(GET, "/{index}/_shard_stores", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         IndicesShardStoresRequest indicesShardStoresRequest = new IndicesShardStoresRequest(Strings.splitStringByCommaToArray(request.param("index")));
         if (request.hasParam("status")) {
             indicesShardStoresRequest.shardStatuses(Strings.splitStringByCommaToArray(request.param("status")));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java
index 623e60cb4b9..93a1dea9b5e 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/stats/RestIndicesStatsAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.admin.indices.stats;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -46,8 +46,8 @@ import static org.elasticsearch.rest.action.support.RestActions.buildBroadcastSh
 public class RestIndicesStatsAction extends BaseRestHandler {
 
     @Inject
-    public RestIndicesStatsAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestIndicesStatsAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(GET, "/_stats", this);
         controller.registerHandler(GET, "/_stats/{metric}", this);
         controller.registerHandler(GET, "/_stats/{metric}/{indexMetric}", this);
@@ -56,7 +56,7 @@ public class RestIndicesStatsAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest();
         indicesStatsRequest.indicesOptions(IndicesOptions.fromRequest(request, indicesStatsRequest.indicesOptions()));
         indicesStatsRequest.indices(Strings.splitStringByCommaToArray(request.param("index")));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/template/delete/RestDeleteIndexTemplateAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/template/delete/RestDeleteIndexTemplateAction.java
index a59ab9ac704..00a029c34b9 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/template/delete/RestDeleteIndexTemplateAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/template/delete/RestDeleteIndexTemplateAction.java
@@ -20,7 +20,7 @@ package org.elasticsearch.rest.action.admin.indices.template.delete;
 
 import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest;
 import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -35,13 +35,13 @@ import org.elasticsearch.rest.action.support.AcknowledgedRestListener;
 public class RestDeleteIndexTemplateAction extends BaseRestHandler {
 
     @Inject
-    public RestDeleteIndexTemplateAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestDeleteIndexTemplateAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(RestRequest.Method.DELETE, "/_template/{name}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         DeleteIndexTemplateRequest deleteIndexTemplateRequest = new DeleteIndexTemplateRequest(request.param("name"));
         deleteIndexTemplateRequest.masterNodeTimeout(request.paramAsTime("master_timeout", deleteIndexTemplateRequest.masterNodeTimeout()));
         client.admin().indices().deleteTemplate(deleteIndexTemplateRequest, new AcknowledgedRestListener<DeleteIndexTemplateResponse>(channel));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/template/get/RestGetIndexTemplateAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/template/get/RestGetIndexTemplateAction.java
index d62d97400c5..de4a0f00442 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/template/get/RestGetIndexTemplateAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/template/get/RestGetIndexTemplateAction.java
@@ -20,7 +20,7 @@ package org.elasticsearch.rest.action.admin.indices.template.get;
 
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
@@ -49,15 +49,15 @@ import static org.elasticsearch.rest.RestStatus.OK;
 public class RestGetIndexTemplateAction extends BaseRestHandler {
 
     @Inject
-    public RestGetIndexTemplateAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestGetIndexTemplateAction(Settings settings, RestController controller) {
+        super(settings);
 
         controller.registerHandler(GET, "/_template", this);
         controller.registerHandler(GET, "/_template/{name}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final String[] names = Strings.splitStringByCommaToArray(request.param("name"));
 
         GetIndexTemplatesRequest getIndexTemplatesRequest = new GetIndexTemplatesRequest(names);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/template/head/RestHeadIndexTemplateAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/template/head/RestHeadIndexTemplateAction.java
index 2a40de984b9..b790782a01a 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/template/head/RestHeadIndexTemplateAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/template/head/RestHeadIndexTemplateAction.java
@@ -20,7 +20,7 @@ package org.elasticsearch.rest.action.admin.indices.template.head;
 
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -42,14 +42,14 @@ import static org.elasticsearch.rest.RestStatus.OK;
 public class RestHeadIndexTemplateAction extends BaseRestHandler {
 
     @Inject
-    public RestHeadIndexTemplateAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestHeadIndexTemplateAction(Settings settings, RestController controller) {
+        super(settings);
 
         controller.registerHandler(HEAD, "/_template/{name}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         GetIndexTemplatesRequest getIndexTemplatesRequest = new GetIndexTemplatesRequest(request.param("name"));
         getIndexTemplatesRequest.local(request.paramAsBoolean("local", getIndexTemplatesRequest.local()));
         getIndexTemplatesRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getIndexTemplatesRequest.masterNodeTimeout()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/template/put/RestPutIndexTemplateAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/template/put/RestPutIndexTemplateAction.java
index 0b08b64e89b..77f91cebdf7 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/template/put/RestPutIndexTemplateAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/template/put/RestPutIndexTemplateAction.java
@@ -20,7 +20,7 @@ package org.elasticsearch.rest.action.admin.indices.template.put;
 
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -35,15 +35,15 @@ import org.elasticsearch.rest.action.support.AcknowledgedRestListener;
 public class RestPutIndexTemplateAction extends BaseRestHandler {
 
     @Inject
-    public RestPutIndexTemplateAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestPutIndexTemplateAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(RestRequest.Method.PUT, "/_template/{name}", this);
         controller.registerHandler(RestRequest.Method.POST, "/_template/{name}", this);
     }
 
     @SuppressWarnings({"unchecked"})
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         PutIndexTemplateRequest putRequest = new PutIndexTemplateRequest(request.param("name"));
         putRequest.template(request.param("template", putRequest.template()));
         putRequest.order(request.paramAsInt("order", putRequest.order()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/upgrade/RestUpgradeAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/upgrade/RestUpgradeAction.java
index 7c4c0f51b8e..ee2aa156adc 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/upgrade/RestUpgradeAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/upgrade/RestUpgradeAction.java
@@ -23,7 +23,8 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.upgrade.get.UpgradeStatusResponse;
 import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequest;
 import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
@@ -48,8 +49,8 @@ import static org.elasticsearch.rest.action.support.RestActions.buildBroadcastSh
 public class RestUpgradeAction extends BaseRestHandler {
 
     @Inject
-    public RestUpgradeAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestUpgradeAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(POST, "/_upgrade", this);
         controller.registerHandler(POST, "/{index}/_upgrade", this);
 
@@ -58,7 +59,7 @@ public class RestUpgradeAction extends BaseRestHandler {
     }
 
     @Override
-    protected void handleRequest(RestRequest request, RestChannel channel, Client client) throws Exception {
+    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
         if (request.method().equals(RestRequest.Method.GET)) {
             handleGet(request, channel, client);
         } else if (request.method().equals(RestRequest.Method.POST)) {
@@ -66,7 +67,7 @@ public class RestUpgradeAction extends BaseRestHandler {
         }
     }
 
-    void handleGet(final RestRequest request, RestChannel channel, Client client) {
+    void handleGet(final RestRequest request, RestChannel channel, NodeClient client) {
         client.admin().indices().prepareUpgradeStatus(Strings.splitStringByCommaToArray(request.param("index")))
                 .execute(new RestBuilderListener<UpgradeStatusResponse>(channel) {
                     @Override
@@ -79,7 +80,7 @@ public class RestUpgradeAction extends BaseRestHandler {
                 });
     }
 
-    void handlePost(final RestRequest request, RestChannel channel, Client client) {
+    void handlePost(final RestRequest request, RestChannel channel, NodeClient client) {
         UpgradeRequest upgradeReq = new UpgradeRequest(Strings.splitStringByCommaToArray(request.param("index")));
         upgradeReq.upgradeOnlyAncientSegments(request.paramAsBoolean("only_ancient_segments", false));
         client.admin().indices().upgrade(upgradeReq, new RestBuilderListener<UpgradeResponse>(channel) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/query/RestValidateQueryAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/query/RestValidateQueryAction.java
index 6ac71708e32..2000078e011 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/query/RestValidateQueryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/validate/query/RestValidateQueryAction.java
@@ -23,7 +23,7 @@ import org.elasticsearch.action.admin.indices.validate.query.QueryExplanation;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
@@ -55,8 +55,8 @@ public class RestValidateQueryAction extends BaseRestHandler {
     private final IndicesQueriesRegistry indicesQueriesRegistry;
 
     @Inject
-    public RestValidateQueryAction(Settings settings, RestController controller, Client client, IndicesQueriesRegistry indicesQueriesRegistry) {
-        super(settings, client);
+    public RestValidateQueryAction(Settings settings, RestController controller, IndicesQueriesRegistry indicesQueriesRegistry) {
+        super(settings);
         controller.registerHandler(GET, "/_validate/query", this);
         controller.registerHandler(POST, "/_validate/query", this);
         controller.registerHandler(GET, "/{index}/_validate/query", this);
@@ -67,7 +67,7 @@ public class RestValidateQueryAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws Exception {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
         ValidateQueryRequest validateQueryRequest = new ValidateQueryRequest(Strings.splitStringByCommaToArray(request.param("index")));
         validateQueryRequest.indicesOptions(IndicesOptions.fromRequest(request, validateQueryRequest.indicesOptions()));
         validateQueryRequest.explain(request.paramAsBoolean("explain", false));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/bulk/RestBulkAction.java b/core/src/main/java/org/elasticsearch/rest/action/bulk/RestBulkAction.java
index d9dbb21e804..623af6d2f47 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/bulk/RestBulkAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/bulk/RestBulkAction.java
@@ -24,7 +24,7 @@ import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.bulk.BulkShardRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
@@ -56,8 +56,8 @@ public class RestBulkAction extends BaseRestHandler {
     private final boolean allowExplicitIndex;
 
     @Inject
-    public RestBulkAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestBulkAction(Settings settings, RestController controller) {
+        super(settings);
 
         controller.registerHandler(POST, "/_bulk", this);
         controller.registerHandler(PUT, "/_bulk", this);
@@ -70,7 +70,7 @@ public class RestBulkAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws Exception {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
         BulkRequest bulkRequest = Requests.bulkRequest();
         String defaultIndex = request.param("index");
         String defaultType = request.param("type");
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/AbstractCatAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/AbstractCatAction.java
index 12393f58007..a40463e3e27 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/AbstractCatAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/AbstractCatAction.java
@@ -18,7 +18,7 @@
  */
 package org.elasticsearch.rest.action.cat;
 
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.io.UTF8StreamWriter;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -38,18 +38,18 @@ import static org.elasticsearch.rest.action.support.RestTable.pad;
  */
 public abstract class AbstractCatAction extends BaseRestHandler {
 
-    public AbstractCatAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public AbstractCatAction(Settings settings, RestController controller) {
+        super(settings);
     }
 
-    protected abstract void doRequest(final RestRequest request, final RestChannel channel, final Client client);
+    protected abstract void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client);
 
     protected abstract void documentation(StringBuilder sb);
 
     protected abstract Table getTableWithHeader(final RestRequest request);
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws Exception {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
         boolean helpWanted = request.paramAsBoolean("help", false);
         if (helpWanted) {
             Table table = getTableWithHeader(request);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
index b322fef9f30..9c68d0abbc7 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.cat;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
@@ -44,15 +44,15 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 public class RestAliasAction extends AbstractCatAction {
 
     @Inject
-    public RestAliasAction(Settings settings, RestController controller, Client client) {
-        super(settings, controller, client);
+    public RestAliasAction(Settings settings, RestController controller) {
+        super(settings, controller);
         controller.registerHandler(GET, "/_cat/aliases", this);
         controller.registerHandler(GET, "/_cat/aliases/{alias}", this);
     }
 
 
     @Override
-    protected void doRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    protected void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final GetAliasesRequest getAliasesRequest = request.hasParam("alias") ?
                 new GetAliasesRequest(request.param("alias")) :
                 new GetAliasesRequest();
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestAllocationAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestAllocationAction.java
index 4a64ef409db..146b52a63a0 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestAllocationAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestAllocationAction.java
@@ -26,7 +26,7 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Strings;
@@ -48,8 +48,8 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 public class RestAllocationAction extends AbstractCatAction {
 
     @Inject
-    public RestAllocationAction(Settings settings, RestController controller, Client client) {
-        super(settings, controller, client);
+    public RestAllocationAction(Settings settings, RestController controller) {
+        super(settings, controller);
         controller.registerHandler(GET, "/_cat/allocation", this);
         controller.registerHandler(GET, "/_cat/allocation/{nodes}", this);
     }
@@ -60,7 +60,7 @@ public class RestAllocationAction extends AbstractCatAction {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final String[] nodes = Strings.splitStringByCommaToArray(request.param("nodes", "data:true"));
         final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
         clusterStateRequest.clear().routingTable(true);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestCatAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestCatAction.java
index 23229540b96..b9cc5011a81 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestCatAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestCatAction.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.rest.action.cat;
 
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -40,8 +40,8 @@ public class RestCatAction extends BaseRestHandler {
     private final String HELP;
 
     @Inject
-    public RestCatAction(Settings settings, RestController controller, Set<AbstractCatAction> catActions, Client client) {
-        super(settings, client);
+    public RestCatAction(Settings settings, RestController controller, Set<AbstractCatAction> catActions) {
+        super(settings);
         controller.registerHandler(GET, "/_cat", this);
         StringBuilder sb = new StringBuilder();
         sb.append(CAT_NL);
@@ -52,7 +52,7 @@ public class RestCatAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         channel.sendResponse(new BytesRestResponse(RestStatus.OK, HELP));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java
index 46e8fadd05a..032fd5c3aba 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.cat;
 
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -45,8 +45,8 @@ public class RestCountAction extends AbstractCatAction {
     private final IndicesQueriesRegistry indicesQueriesRegistry;
 
     @Inject
-    public RestCountAction(Settings settings, RestController restController, RestController controller, Client client, IndicesQueriesRegistry indicesQueriesRegistry) {
-        super(settings, controller, client);
+    public RestCountAction(Settings settings, RestController restController, RestController controller, IndicesQueriesRegistry indicesQueriesRegistry) {
+        super(settings, controller);
         restController.registerHandler(GET, "/_cat/count", this);
         restController.registerHandler(GET, "/_cat/count/{index}", this);
         this.indicesQueriesRegistry = indicesQueriesRegistry;
@@ -59,7 +59,7 @@ public class RestCountAction extends AbstractCatAction {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         SearchRequest countRequest = new SearchRequest(indices);
         String source = request.param("source");
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestFielddataAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestFielddataAction.java
index 8febf04e892..d3458b40d77 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestFielddataAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestFielddataAction.java
@@ -23,7 +23,7 @@ import com.carrotsearch.hppc.cursors.ObjectLongCursor;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -43,14 +43,14 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 public class RestFielddataAction extends AbstractCatAction {
 
     @Inject
-    public RestFielddataAction(Settings settings, RestController controller, Client client) {
-        super(settings, controller, client);
+    public RestFielddataAction(Settings settings, RestController controller) {
+        super(settings, controller);
         controller.registerHandler(GET, "/_cat/fielddata", this);
         controller.registerHandler(GET, "/_cat/fielddata/{fields}", this);
     }
 
     @Override
-    protected void doRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    protected void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final NodesStatsRequest nodesStatsRequest = new NodesStatsRequest("data:true");
         nodesStatsRequest.clear();
         nodesStatsRequest.indices(true);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestHealthAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestHealthAction.java
index 7dc7954a3a4..07f36cc2f6d 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestHealthAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestHealthAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.cat;
 
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -39,8 +39,8 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 public class RestHealthAction extends AbstractCatAction {
 
     @Inject
-    public RestHealthAction(Settings settings, RestController controller, Client client) {
-        super(settings, controller, client);
+    public RestHealthAction(Settings settings, RestController controller) {
+        super(settings, controller);
         controller.registerHandler(GET, "/_cat/health", this);
     }
 
@@ -50,7 +50,7 @@ public class RestHealthAction extends AbstractCatAction {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         ClusterHealthRequest clusterHealthRequest = new ClusterHealthRequest();
 
         client.admin().cluster().health(clusterHealthRequest, new RestResponseListener<ClusterHealthResponse>(channel) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
index 38cbba6fd5a..fcdfe009e18 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
@@ -27,7 +27,7 @@ import org.elasticsearch.action.admin.indices.stats.IndexStats;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.client.Requests;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterIndexHealth;
@@ -57,8 +57,8 @@ public class RestIndicesAction extends AbstractCatAction {
     private final IndexNameExpressionResolver indexNameExpressionResolver;
 
     @Inject
-    public RestIndicesAction(Settings settings, RestController controller, Client client, IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, controller, client);
+    public RestIndicesAction(Settings settings, RestController controller, IndexNameExpressionResolver indexNameExpressionResolver) {
+        super(settings, controller);
         this.indexNameExpressionResolver = indexNameExpressionResolver;
         controller.registerHandler(GET, "/_cat/indices", this);
         controller.registerHandler(GET, "/_cat/indices/{index}", this);
@@ -71,7 +71,7 @@ public class RestIndicesAction extends AbstractCatAction {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
         clusterStateRequest.clear().indices(indices).metaData(true);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestMasterAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestMasterAction.java
index e7a7d7afc96..164c2f79dac 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestMasterAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestMasterAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.cat;
 
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Table;
@@ -39,8 +39,8 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 public class RestMasterAction extends AbstractCatAction {
 
     @Inject
-    public RestMasterAction(Settings settings, RestController controller, Client client) {
-        super(settings, controller, client);
+    public RestMasterAction(Settings settings, RestController controller) {
+        super(settings, controller);
         controller.registerHandler(GET, "/_cat/master", this);
     }
 
@@ -50,7 +50,7 @@ public class RestMasterAction extends AbstractCatAction {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
         clusterStateRequest.clear().nodes(true);
         clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java
index 2190e1e2993..815895a2679 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java
@@ -24,7 +24,7 @@ import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
@@ -47,8 +47,8 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 public class RestNodeAttrsAction extends AbstractCatAction {
 
     @Inject
-    public RestNodeAttrsAction(Settings settings, RestController controller, Client client) {
-        super(settings, controller, client);
+    public RestNodeAttrsAction(Settings settings, RestController controller) {
+        super(settings, controller);
         controller.registerHandler(GET, "/_cat/nodeattrs", this);
     }
 
@@ -58,7 +58,7 @@ public class RestNodeAttrsAction extends AbstractCatAction {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
         clusterStateRequest.clear().nodes(true);
         clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
index 1069a879d94..cb7bd59be3c 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
@@ -27,7 +27,7 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
@@ -72,8 +72,8 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 public class RestNodesAction extends AbstractCatAction {
 
     @Inject
-    public RestNodesAction(Settings settings, RestController controller, Client client) {
-        super(settings, controller, client);
+    public RestNodesAction(Settings settings, RestController controller) {
+        super(settings, controller);
         controller.registerHandler(GET, "/_cat/nodes", this);
     }
 
@@ -83,7 +83,7 @@ public class RestNodesAction extends AbstractCatAction {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
         clusterStateRequest.clear().nodes(true);
         clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java
index b563450f8f6..b85906e25b4 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.cat;
 
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
 import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.service.PendingClusterTask;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.inject.Inject;
@@ -37,8 +37,8 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestPendingClusterTasksAction extends AbstractCatAction {
     @Inject
-    public RestPendingClusterTasksAction(Settings settings, RestController controller, Client client) {
-        super(settings, controller, client);
+    public RestPendingClusterTasksAction(Settings settings, RestController controller) {
+        super(settings, controller);
         controller.registerHandler(GET, "/_cat/pending_tasks", this);
     }
 
@@ -48,7 +48,7 @@ public class RestPendingClusterTasksAction extends AbstractCatAction {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         PendingClusterTasksRequest pendingClusterTasksRequest = new PendingClusterTasksRequest();
         pendingClusterTasksRequest.masterNodeTimeout(request.paramAsTime("master_timeout", pendingClusterTasksRequest.masterNodeTimeout()));
         pendingClusterTasksRequest.local(request.paramAsBoolean("local", pendingClusterTasksRequest.local()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java
index 189273a7722..13e1f2176d7 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java
@@ -24,7 +24,7 @@ import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Table;
@@ -44,8 +44,8 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 public class RestPluginsAction extends AbstractCatAction {
 
     @Inject
-    public RestPluginsAction(Settings settings, RestController controller, Client client) {
-        super(settings, controller, client);
+    public RestPluginsAction(Settings settings, RestController controller) {
+        super(settings, controller);
         controller.registerHandler(GET, "/_cat/plugins", this);
     }
 
@@ -55,7 +55,7 @@ public class RestPluginsAction extends AbstractCatAction {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
         clusterStateRequest.clear().nodes(true);
         clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java
index 07c2611f2ce..6308ce7e689 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java
@@ -23,7 +23,7 @@ import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.action.admin.indices.recovery.RecoveryRequest;
 import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.inject.Inject;
@@ -51,8 +51,8 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 public class RestRecoveryAction extends AbstractCatAction {
 
     @Inject
-    public RestRecoveryAction(Settings settings, RestController restController, RestController controller, Client client) {
-        super(settings, controller, client);
+    public RestRecoveryAction(Settings settings, RestController restController, RestController controller) {
+        super(settings, controller);
         restController.registerHandler(GET, "/_cat/recovery", this);
         restController.registerHandler(GET, "/_cat/recovery/{index}", this);
     }
@@ -64,7 +64,7 @@ public class RestRecoveryAction extends AbstractCatAction {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final RecoveryRequest recoveryRequest = new RecoveryRequest(Strings.splitStringByCommaToArray(request.param("index")));
         recoveryRequest.detailed(request.paramAsBoolean("detailed", false));
         recoveryRequest.activeOnly(request.paramAsBoolean("active_only", false));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRepositoriesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRepositoriesAction.java
index d19e2b19ca0..ee53e1241c7 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRepositoriesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRepositoriesAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.cat;
 
 import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
 import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.inject.Inject;
@@ -40,13 +40,13 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
  */
 public class RestRepositoriesAction extends AbstractCatAction {
     @Inject
-    public RestRepositoriesAction(Settings settings, RestController controller, Client client) {
-        super(settings, controller, client);
+    public RestRepositoriesAction(Settings settings, RestController controller) {
+        super(settings, controller);
         controller.registerHandler(GET, "/_cat/repositories", this);
     }
 
     @Override
-    protected void doRequest(RestRequest request, RestChannel channel, Client client) {
+    protected void doRequest(RestRequest request, RestChannel channel, NodeClient client) {
         GetRepositoriesRequest getRepositoriesRequest = new GetRepositoriesRequest();
         getRepositoriesRequest.local(request.paramAsBoolean("local", getRepositoriesRequest.local()));
         getRepositoriesRequest.masterNodeTimeout(request.paramAsTime("master_timeout", getRepositoriesRequest.masterNodeTimeout()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestSegmentsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestSegmentsAction.java
index e5a1b4b49a3..2540e194d84 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestSegmentsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestSegmentsAction.java
@@ -26,7 +26,7 @@ import org.elasticsearch.action.admin.indices.segments.IndexShardSegments;
 import org.elasticsearch.action.admin.indices.segments.IndicesSegmentResponse;
 import org.elasticsearch.action.admin.indices.segments.IndicesSegmentsRequest;
 import org.elasticsearch.action.admin.indices.segments.ShardSegments;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
@@ -49,14 +49,14 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 public class RestSegmentsAction extends AbstractCatAction {
 
     @Inject
-    public RestSegmentsAction(Settings settings, RestController controller, Client client) {
-        super(settings, controller, client);
+    public RestSegmentsAction(Settings settings, RestController controller) {
+        super(settings, controller);
         controller.registerHandler(GET, "/_cat/segments", this);
         controller.registerHandler(GET, "/_cat/segments/{index}", this);
     }
 
     @Override
-    protected void doRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    protected void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
 
         final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
index 0f6bec367d5..214979aef66 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
@@ -25,7 +25,7 @@ import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
@@ -49,8 +49,8 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 public class RestShardsAction extends AbstractCatAction {
 
     @Inject
-    public RestShardsAction(Settings settings, RestController controller, Client client) {
-        super(settings, controller, client);
+    public RestShardsAction(Settings settings, RestController controller) {
+        super(settings, controller);
         controller.registerHandler(GET, "/_cat/shards", this);
         controller.registerHandler(GET, "/_cat/shards/{index}", this);
     }
@@ -62,7 +62,7 @@ public class RestShardsAction extends AbstractCatAction {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
         clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java
index 94d178e4db9..e503118fbab 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.cat;
 
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -47,13 +47,13 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
  */
 public class RestSnapshotAction extends AbstractCatAction {
     @Inject
-    public RestSnapshotAction(Settings settings, RestController controller, Client client) {
-        super(settings, controller, client);
+    public RestSnapshotAction(Settings settings, RestController controller) {
+        super(settings, controller);
         controller.registerHandler(GET, "/_cat/snapshots/{repository}", this);
     }
 
     @Override
-    protected void doRequest(final RestRequest request, RestChannel channel, Client client) {
+    protected void doRequest(final RestRequest request, RestChannel channel, NodeClient client) {
         GetSnapshotsRequest getSnapshotsRequest = new GetSnapshotsRequest()
                 .repository(request.param("repository"))
                 .snapshots(new String[]{GetSnapshotsRequest.ALL_SNAPSHOTS});
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java
index 9bb01a7b166..7486acfbb80 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.cat;
 
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskGroup;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -51,8 +51,8 @@ public class RestTasksAction extends AbstractCatAction {
     private final ClusterService clusterService;
 
     @Inject
-    public RestTasksAction(Settings settings, RestController controller, Client client, ClusterService clusterService) {
-        super(settings, controller, client);
+    public RestTasksAction(Settings settings, RestController controller, ClusterService clusterService) {
+        super(settings, controller);
         controller.registerHandler(GET, "/_cat/tasks", this);
         this.clusterService = clusterService;
     }
@@ -63,7 +63,7 @@ public class RestTasksAction extends AbstractCatAction {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         client.admin().cluster().listTasks(generateListTasksRequest(request), new RestResponseListener<ListTasksResponse>(channel) {
             @Override
             public RestResponse buildResponse(ListTasksResponse listTasksResponse) throws Exception {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java
index 2ade4a1db39..caf28a9b018 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java
@@ -27,7 +27,7 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequest;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Strings;
@@ -108,8 +108,8 @@ public class RestThreadPoolAction extends AbstractCatAction {
     }
 
     @Inject
-    public RestThreadPoolAction(Settings settings, RestController controller, Client client) {
-        super(settings, controller, client);
+    public RestThreadPoolAction(Settings settings, RestController controller) {
+        super(settings, controller);
         controller.registerHandler(GET, "/_cat/thread_pool", this);
     }
 
@@ -119,7 +119,7 @@ public class RestThreadPoolAction extends AbstractCatAction {
     }
 
     @Override
-    public void doRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
         clusterStateRequest.clear().nodes(true);
         clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/count/RestCountAction.java b/core/src/main/java/org/elasticsearch/rest/action/count/RestCountAction.java
index 71e5832071c..d75bb7a0bac 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/count/RestCountAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/count/RestCountAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.count;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
@@ -53,8 +53,8 @@ public class RestCountAction extends BaseRestHandler {
     private final IndicesQueriesRegistry indicesQueriesRegistry;
 
     @Inject
-    public RestCountAction(Settings settings, RestController controller, Client client, IndicesQueriesRegistry indicesQueriesRegistry) {
-        super(settings, client);
+    public RestCountAction(Settings settings, RestController controller, IndicesQueriesRegistry indicesQueriesRegistry) {
+        super(settings);
         controller.registerHandler(POST, "/_count", this);
         controller.registerHandler(GET, "/_count", this);
         controller.registerHandler(POST, "/{index}/_count", this);
@@ -65,7 +65,7 @@ public class RestCountAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         SearchRequest countRequest = new SearchRequest(Strings.splitStringByCommaToArray(request.param("index")));
         countRequest.indicesOptions(IndicesOptions.fromRequest(request, countRequest.indicesOptions()));
         SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(0);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/delete/RestDeleteAction.java b/core/src/main/java/org/elasticsearch/rest/action/delete/RestDeleteAction.java
index 29316893504..2f9b10096cc 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/delete/RestDeleteAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/delete/RestDeleteAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.delete;
 
 import org.elasticsearch.action.WriteConsistencyLevel;
 import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.VersionType;
@@ -40,13 +40,13 @@ import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 public class RestDeleteAction extends BaseRestHandler {
 
     @Inject
-    public RestDeleteAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestDeleteAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(DELETE, "/{index}/{type}/{id}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         DeleteRequest deleteRequest = new DeleteRequest(request.param("index"), request.param("type"), request.param("id"));
         deleteRequest.routing(request.param("routing"));
         deleteRequest.parent(request.param("parent")); // order is important, set it after routing, so it will set the routing
diff --git a/core/src/main/java/org/elasticsearch/rest/action/explain/RestExplainAction.java b/core/src/main/java/org/elasticsearch/rest/action/explain/RestExplainAction.java
index 47f5e7da19b..19f55f4cd46 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/explain/RestExplainAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/explain/RestExplainAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.explain;
 import org.apache.lucene.search.Explanation;
 import org.elasticsearch.action.explain.ExplainRequest;
 import org.elasticsearch.action.explain.ExplainResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
@@ -56,15 +56,15 @@ public class RestExplainAction extends BaseRestHandler {
     private final IndicesQueriesRegistry indicesQueriesRegistry;
 
     @Inject
-    public RestExplainAction(Settings settings, RestController controller, Client client, IndicesQueriesRegistry indicesQueriesRegistry) {
-        super(settings, client);
+    public RestExplainAction(Settings settings, RestController controller, IndicesQueriesRegistry indicesQueriesRegistry) {
+        super(settings);
         this.indicesQueriesRegistry = indicesQueriesRegistry;
         controller.registerHandler(GET, "/{index}/{type}/{id}/_explain", this);
         controller.registerHandler(POST, "/{index}/{type}/{id}/_explain", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final ExplainRequest explainRequest = new ExplainRequest(request.param("index"), request.param("type"), request.param("id"));
         explainRequest.parent(request.param("parent"));
         explainRequest.routing(request.param("routing"));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/fieldstats/RestFieldStatsAction.java b/core/src/main/java/org/elasticsearch/rest/action/fieldstats/RestFieldStatsAction.java
index 744a76e7352..9f62024ab80 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/fieldstats/RestFieldStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/fieldstats/RestFieldStatsAction.java
@@ -23,7 +23,7 @@ import org.elasticsearch.action.fieldstats.FieldStats;
 import org.elasticsearch.action.fieldstats.FieldStatsRequest;
 import org.elasticsearch.action.fieldstats.FieldStatsResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -49,8 +49,8 @@ import static org.elasticsearch.rest.action.support.RestActions.buildBroadcastSh
 public class RestFieldStatsAction extends BaseRestHandler {
 
     @Inject
-    public RestFieldStatsAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestFieldStatsAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(GET, "/_field_stats", this);
         controller.registerHandler(POST, "/_field_stats", this);
         controller.registerHandler(GET, "/{index}/_field_stats", this);
@@ -59,7 +59,7 @@ public class RestFieldStatsAction extends BaseRestHandler {
 
     @Override
     public void handleRequest(final RestRequest request,
-                              final RestChannel channel, final Client client) throws Exception {
+                              final RestChannel channel, final NodeClient client) throws Exception {
         if (RestActions.hasBodyContent(request) && request.hasParam("fields")) {
             throw new IllegalArgumentException("can't specify a request body and [fields] request parameter, " +
                 "either specify a request body or the [fields] request parameter");
diff --git a/core/src/main/java/org/elasticsearch/rest/action/get/RestGetAction.java b/core/src/main/java/org/elasticsearch/rest/action/get/RestGetAction.java
index 193fdd2e747..3cc7d8fd1ae 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/get/RestGetAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/get/RestGetAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.get;
 
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -47,13 +47,13 @@ import static org.elasticsearch.rest.RestStatus.OK;
 public class RestGetAction extends BaseRestHandler {
 
     @Inject
-    public RestGetAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestGetAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(GET, "/{index}/{type}/{id}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final GetRequest getRequest = new GetRequest(request.param("index"), request.param("type"), request.param("id"));
         getRequest.operationThreaded(true);
         getRequest.refresh(request.paramAsBoolean("refresh", getRequest.refresh()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/get/RestGetSourceAction.java b/core/src/main/java/org/elasticsearch/rest/action/get/RestGetSourceAction.java
index 7d8a39b02b9..9e9b54b5b0c 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/get/RestGetSourceAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/get/RestGetSourceAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.get;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -47,13 +47,13 @@ import static org.elasticsearch.rest.RestStatus.OK;
 public class RestGetSourceAction extends BaseRestHandler {
 
     @Inject
-    public RestGetSourceAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestGetSourceAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(GET, "/{index}/{type}/{id}/_source", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final GetRequest getRequest = new GetRequest(request.param("index"), request.param("type"), request.param("id"));
         getRequest.operationThreaded(true);
         getRequest.refresh(request.paramAsBoolean("refresh", getRequest.refresh()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/get/RestHeadAction.java b/core/src/main/java/org/elasticsearch/rest/action/get/RestHeadAction.java
index 01f3be435e7..ca8900ab77f 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/get/RestHeadAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/get/RestHeadAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.get;
 
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.inject.Inject;
@@ -49,8 +49,8 @@ public abstract class RestHeadAction extends BaseRestHandler {
     public static class Document extends RestHeadAction {
 
         @Inject
-        public Document(Settings settings, RestController controller, Client client) {
-            super(settings, client, false);
+        public Document(Settings settings, RestController controller) {
+            super(settings, false);
             controller.registerHandler(HEAD, "/{index}/{type}/{id}", this);
         }
     }
@@ -61,8 +61,8 @@ public abstract class RestHeadAction extends BaseRestHandler {
     public static class Source extends RestHeadAction {
 
         @Inject
-        public Source(Settings settings, RestController controller, Client client) {
-            super(settings, client, true);
+        public Source(Settings settings, RestController controller) {
+            super(settings, true);
             controller.registerHandler(HEAD, "/{index}/{type}/{id}/_source", this);
         }
     }
@@ -73,17 +73,16 @@ public abstract class RestHeadAction extends BaseRestHandler {
      * All subclasses must be registered in {@link org.elasticsearch.common.network.NetworkModule}.
      *
      * @param settings injected settings
-     * @param client   injected client
      * @param source   {@code false} to check for {@link GetResponse#isExists()}.
      *                 {@code true} to also check for {@link GetResponse#isSourceEmpty()}.
      */
-    public RestHeadAction(Settings settings, Client client, boolean source) {
-        super(settings, client);
+    public RestHeadAction(Settings settings, boolean source) {
+        super(settings);
         this.source = source;
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         final GetRequest getRequest = new GetRequest(request.param("index"), request.param("type"), request.param("id"));
         getRequest.operationThreaded(true);
         getRequest.refresh(request.paramAsBoolean("refresh", getRequest.refresh()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/get/RestMultiGetAction.java b/core/src/main/java/org/elasticsearch/rest/action/get/RestMultiGetAction.java
index 1722daa4445..de4f433dcbf 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/get/RestMultiGetAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/get/RestMultiGetAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.get;
 
 import org.elasticsearch.action.get.MultiGetRequest;
 import org.elasticsearch.action.get.MultiGetResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -41,8 +41,8 @@ public class RestMultiGetAction extends BaseRestHandler {
     private final boolean allowExplicitIndex;
 
     @Inject
-    public RestMultiGetAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestMultiGetAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(GET, "/_mget", this);
         controller.registerHandler(POST, "/_mget", this);
         controller.registerHandler(GET, "/{index}/_mget", this);
@@ -54,7 +54,7 @@ public class RestMultiGetAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws Exception {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
         MultiGetRequest multiGetRequest = new MultiGetRequest();
         multiGetRequest.refresh(request.paramAsBoolean("refresh", multiGetRequest.refresh()));
         multiGetRequest.preference(request.param("preference"));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java
index f807e68088a..884adcf0d81 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.index;
 
 import org.elasticsearch.action.WriteConsistencyLevel;
 import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -46,30 +46,30 @@ import static org.elasticsearch.rest.RestStatus.BAD_REQUEST;
 public class RestIndexAction extends BaseRestHandler {
 
     @Inject
-    public RestIndexAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestIndexAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(POST, "/{index}/{type}", this); // auto id creation
         controller.registerHandler(PUT, "/{index}/{type}/{id}", this);
         controller.registerHandler(POST, "/{index}/{type}/{id}", this);
-        CreateHandler createHandler = new CreateHandler(settings, controller, client);
+        CreateHandler createHandler = new CreateHandler(settings, controller);
         controller.registerHandler(PUT, "/{index}/{type}/{id}/_create", createHandler);
         controller.registerHandler(POST, "/{index}/{type}/{id}/_create", createHandler);
     }
 
     final class CreateHandler extends BaseRestHandler {
-        protected CreateHandler(Settings settings, RestController controller, Client client) {
-            super(settings, client);
+        protected CreateHandler(Settings settings, RestController controller) {
+            super(settings);
         }
 
         @Override
-        public void handleRequest(RestRequest request, RestChannel channel, final Client client) {
+        public void handleRequest(RestRequest request, RestChannel channel, final NodeClient client) {
             request.params().put("op_type", "create");
             RestIndexAction.this.handleRequest(request, channel, client);
         }
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         IndexRequest indexRequest = new IndexRequest(request.param("index"), request.param("type"), request.param("id"));
         indexRequest.routing(request.param("routing"));
         indexRequest.parent(request.param("parent")); // order is important, set it after routing, so it will set the routing
diff --git a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestDeletePipelineAction.java b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestDeletePipelineAction.java
index 6559e32817f..7c6200ee97c 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestDeletePipelineAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestDeletePipelineAction.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.rest.action.ingest;
 
 import org.elasticsearch.action.ingest.DeletePipelineRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -32,13 +32,13 @@ import org.elasticsearch.rest.action.support.AcknowledgedRestListener;
 public class RestDeletePipelineAction extends BaseRestHandler {
 
     @Inject
-    public RestDeletePipelineAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestDeletePipelineAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(RestRequest.Method.DELETE, "/_ingest/pipeline/{id}", this);
     }
 
     @Override
-    protected void handleRequest(RestRequest restRequest, RestChannel channel, Client client) throws Exception {
+    public void handleRequest(RestRequest restRequest, RestChannel channel, NodeClient client) throws Exception {
         DeletePipelineRequest request = new DeletePipelineRequest(restRequest.param("id"));
         request.masterNodeTimeout(restRequest.paramAsTime("master_timeout", request.masterNodeTimeout()));
         request.timeout(restRequest.paramAsTime("timeout", request.timeout()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestGetPipelineAction.java b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestGetPipelineAction.java
index c11290329b3..55f14cfaa84 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestGetPipelineAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestGetPipelineAction.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.rest.action.ingest;
 
 import org.elasticsearch.action.ingest.GetPipelineRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -33,13 +33,13 @@ import org.elasticsearch.rest.action.support.RestStatusToXContentListener;
 public class RestGetPipelineAction extends BaseRestHandler {
 
     @Inject
-    public RestGetPipelineAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestGetPipelineAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(RestRequest.Method.GET, "/_ingest/pipeline/{id}", this);
     }
 
     @Override
-    protected void handleRequest(RestRequest restRequest, RestChannel channel, Client client) throws Exception {
+    public void handleRequest(RestRequest restRequest, RestChannel channel, NodeClient client) throws Exception {
         GetPipelineRequest request = new GetPipelineRequest(Strings.splitStringByCommaToArray(restRequest.param("id")));
         request.masterNodeTimeout(restRequest.paramAsTime("master_timeout", request.masterNodeTimeout()));
         client.admin().cluster().getPipeline(request, new RestStatusToXContentListener<>(channel));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java
index a96ed3d6424..7d7080e8775 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestPutPipelineAction.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.rest.action.ingest;
 
 import org.elasticsearch.action.ingest.PutPipelineRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -34,13 +34,13 @@ import org.elasticsearch.rest.action.support.RestActions;
 public class RestPutPipelineAction extends BaseRestHandler {
 
     @Inject
-    public RestPutPipelineAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestPutPipelineAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(RestRequest.Method.PUT, "/_ingest/pipeline/{id}", this);
     }
 
     @Override
-    protected void handleRequest(RestRequest restRequest, RestChannel channel, Client client) throws Exception {
+    public void handleRequest(RestRequest restRequest, RestChannel channel, NodeClient client) throws Exception {
         PutPipelineRequest request = new PutPipelineRequest(restRequest.param("id"), RestActions.getRestContent(restRequest));
         request.masterNodeTimeout(restRequest.paramAsTime("master_timeout", request.masterNodeTimeout()));
         request.timeout(restRequest.paramAsTime("timeout", request.timeout()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java
index fc2e834ea75..35ba1367e63 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulatePipelineAction.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.rest.action.ingest;
 
 import org.elasticsearch.action.ingest.SimulatePipelineRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -34,8 +34,8 @@ import org.elasticsearch.rest.action.support.RestToXContentListener;
 public class RestSimulatePipelineAction extends BaseRestHandler {
 
     @Inject
-    public RestSimulatePipelineAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestSimulatePipelineAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(RestRequest.Method.POST, "/_ingest/pipeline/{id}/_simulate", this);
         controller.registerHandler(RestRequest.Method.GET, "/_ingest/pipeline/{id}/_simulate", this);
         controller.registerHandler(RestRequest.Method.POST, "/_ingest/pipeline/_simulate", this);
@@ -43,7 +43,7 @@ public class RestSimulatePipelineAction extends BaseRestHandler {
     }
 
     @Override
-    protected void handleRequest(RestRequest restRequest, RestChannel channel, Client client) throws Exception {
+    public void handleRequest(RestRequest restRequest, RestChannel channel, NodeClient client) throws Exception {
         SimulatePipelineRequest request = new SimulatePipelineRequest(RestActions.getRestContent(restRequest));
         request.setId(restRequest.param("id"));
         request.setVerbose(restRequest.paramAsBoolean("verbose", false));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/main/RestMainAction.java b/core/src/main/java/org/elasticsearch/rest/action/main/RestMainAction.java
index e57bb790e5e..24f4e66e7db 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/main/RestMainAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/main/RestMainAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.main;
 import org.elasticsearch.action.main.MainAction;
 import org.elasticsearch.action.main.MainRequest;
 import org.elasticsearch.action.main.MainResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -46,14 +46,14 @@ import static org.elasticsearch.rest.RestRequest.Method.HEAD;
 public class RestMainAction extends BaseRestHandler {
 
     @Inject
-    public RestMainAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestMainAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(GET, "/", this);
         controller.registerHandler(HEAD, "/", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws Exception {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
         client.execute(MainAction.INSTANCE, new MainRequest(), new RestBuilderListener<MainResponse>(channel) {
             @Override
             public RestResponse buildResponse(MainResponse mainResponse, XContentBuilder builder) throws Exception {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java
index 0dce23bf3b1..a189a58a3ae 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.search;
 
 import org.elasticsearch.action.search.ClearScrollRequest;
 import org.elasticsearch.action.search.ClearScrollResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
@@ -46,15 +46,15 @@ import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 public class RestClearScrollAction extends BaseRestHandler {
 
     @Inject
-    public RestClearScrollAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestClearScrollAction(Settings settings, RestController controller) {
+        super(settings);
 
         controller.registerHandler(DELETE, "/_search/scroll", this);
         controller.registerHandler(DELETE, "/_search/scroll/{scroll_id}", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         String scrollIds = request.param("scroll_id");
         ClearScrollRequest clearRequest = new ClearScrollRequest();
         clearRequest.setScrollIds(Arrays.asList(splitScrollIds(scrollIds)));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
index 2935c88dab2..52965193a2e 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
@@ -23,7 +23,7 @@ import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.search.MultiSearchRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -64,9 +64,9 @@ public class RestMultiSearchAction extends BaseRestHandler {
     private final Suggesters suggesters;
 
     @Inject
-    public RestMultiSearchAction(Settings settings, RestController controller, Client client, IndicesQueriesRegistry indicesQueriesRegistry,
+    public RestMultiSearchAction(Settings settings, RestController controller, IndicesQueriesRegistry indicesQueriesRegistry,
                                  AggregatorParsers aggParsers, Suggesters suggesters) {
-        super(settings, client);
+        super(settings);
         this.aggParsers = aggParsers;
         this.suggesters = suggesters;
 
@@ -82,7 +82,7 @@ public class RestMultiSearchAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws Exception {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
         MultiSearchRequest multiSearchRequest = parseRequest(request, allowExplicitIndex, indicesQueriesRegistry, parseFieldMatcher,
                 aggParsers, suggesters);
         client.multiSearch(multiSearchRequest, new RestToXContentListener<>(channel));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
index 5f5fe84d573..870d4a9eb20 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.search;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -67,9 +67,9 @@ public class RestSearchAction extends BaseRestHandler {
     private final Suggesters suggesters;
 
     @Inject
-    public RestSearchAction(Settings settings, RestController controller, Client client, IndicesQueriesRegistry queryRegistry,
+    public RestSearchAction(Settings settings, RestController controller, IndicesQueriesRegistry queryRegistry,
             AggregatorParsers aggParsers, Suggesters suggesters) {
-        super(settings, client);
+        super(settings);
         this.queryRegistry = queryRegistry;
         this.aggParsers = aggParsers;
         this.suggesters = suggesters;
@@ -82,7 +82,7 @@ public class RestSearchAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws IOException {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws IOException {
         SearchRequest searchRequest = new SearchRequest();
         BytesReference restContent = RestActions.hasBodyContent(request) ? RestActions.getRestContent(request) : null;
         parseSearchRequest(searchRequest, queryRegistry, request, parseFieldMatcher, aggParsers, suggesters, restContent);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java
index 3a10db38ee1..6b9b09545ad 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.search;
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchScrollRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -50,8 +50,8 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 public class RestSearchScrollAction extends BaseRestHandler {
 
     @Inject
-    public RestSearchScrollAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestSearchScrollAction(Settings settings, RestController controller) {
+        super(settings);
 
         controller.registerHandler(GET, "/_search/scroll", this);
         controller.registerHandler(POST, "/_search/scroll", this);
@@ -60,7 +60,7 @@ public class RestSearchScrollAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         String scrollId = request.param("scroll_id");
         SearchScrollRequest searchScrollRequest = new SearchScrollRequest();
         searchScrollRequest.scrollId(scrollId);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/suggest/RestSuggestAction.java b/core/src/main/java/org/elasticsearch/rest/action/suggest/RestSuggestAction.java
index c5c6174b02b..f6acfc6daf5 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/suggest/RestSuggestAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/suggest/RestSuggestAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.suggest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
@@ -61,9 +61,9 @@ public class RestSuggestAction extends BaseRestHandler {
     private final Suggesters suggesters;
 
     @Inject
-    public RestSuggestAction(Settings settings, RestController controller, Client client,
+    public RestSuggestAction(Settings settings, RestController controller,
                              IndicesQueriesRegistry queryRegistry, Suggesters suggesters) {
-        super(settings, client);
+        super(settings);
         this.queryRegistry = queryRegistry;
         this.suggesters = suggesters;
         controller.registerHandler(POST, "/_suggest", this);
@@ -73,7 +73,7 @@ public class RestSuggestAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws IOException {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws IOException {
         final SearchRequest searchRequest = new SearchRequest(Strings.splitStringByCommaToArray(request.param("index")), new SearchSourceBuilder());
         searchRequest.indicesOptions(IndicesOptions.fromRequest(request, searchRequest.indicesOptions()));
         if (RestActions.hasBodyContent(request)) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/termvectors/RestMultiTermVectorsAction.java b/core/src/main/java/org/elasticsearch/rest/action/termvectors/RestMultiTermVectorsAction.java
index dfcbeef171c..888d854f40c 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/termvectors/RestMultiTermVectorsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/termvectors/RestMultiTermVectorsAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.termvectors;
 import org.elasticsearch.action.termvectors.MultiTermVectorsRequest;
 import org.elasticsearch.action.termvectors.MultiTermVectorsResponse;
 import org.elasticsearch.action.termvectors.TermVectorsRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -39,8 +39,8 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 public class RestMultiTermVectorsAction extends BaseRestHandler {
 
     @Inject
-    public RestMultiTermVectorsAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestMultiTermVectorsAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(GET, "/_mtermvectors", this);
         controller.registerHandler(POST, "/_mtermvectors", this);
         controller.registerHandler(GET, "/{index}/_mtermvectors", this);
@@ -50,7 +50,7 @@ public class RestMultiTermVectorsAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws Exception {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
         MultiTermVectorsRequest multiTermVectorsRequest = new MultiTermVectorsRequest();
         TermVectorsRequest template = new TermVectorsRequest();
         template.index(request.param("index"));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/termvectors/RestTermVectorsAction.java b/core/src/main/java/org/elasticsearch/rest/action/termvectors/RestTermVectorsAction.java
index d22afa615d8..6aa2d1bdb14 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/termvectors/RestTermVectorsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/termvectors/RestTermVectorsAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.termvectors;
 
 import org.elasticsearch.action.termvectors.TermVectorsRequest;
 import org.elasticsearch.action.termvectors.TermVectorsResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -48,8 +48,8 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 public class RestTermVectorsAction extends BaseRestHandler {
 
     @Inject
-    public RestTermVectorsAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestTermVectorsAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(GET, "/{index}/{type}/_termvectors", this);
         controller.registerHandler(POST, "/{index}/{type}/_termvectors", this);
         controller.registerHandler(GET, "/{index}/{type}/{id}/_termvectors", this);
@@ -63,7 +63,7 @@ public class RestTermVectorsAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws Exception {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
         TermVectorsRequest termVectorsRequest = new TermVectorsRequest(request.param("index"), request.param("type"), request.param("id"));
         if (RestActions.hasBodyContent(request)) {
             try (XContentParser parser = XContentFactory.xContent(RestActions.guessBodyContentType(request)).createParser(RestActions.getRestContent(request))){
diff --git a/core/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java b/core/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java
index bdea4e33e6d..7d739983bf4 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.rest.action.update;
 import org.elasticsearch.action.WriteConsistencyLevel;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -47,13 +47,13 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 public class RestUpdateAction extends BaseRestHandler {
 
     @Inject
-    public RestUpdateAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestUpdateAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(POST, "/{index}/{type}/{id}/_update", this);
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) throws Exception {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws Exception {
         UpdateRequest updateRequest = new UpdateRequest(request.param("index"), request.param("type"), request.param("id"));
         updateRequest.routing(request.param("routing"));
         updateRequest.parent(request.param("parent"));
diff --git a/core/src/test/java/org/elasticsearch/action/ActionModuleTests.java b/core/src/test/java/org/elasticsearch/action/ActionModuleTests.java
index 4b9a833e8c4..87c1a227328 100644
--- a/core/src/test/java/org/elasticsearch/action/ActionModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ActionModuleTests.java
@@ -23,6 +23,7 @@ import org.elasticsearch.action.main.MainAction;
 import org.elasticsearch.action.main.TransportMainAction;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.TransportAction;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.ActionPlugin;
@@ -115,7 +116,7 @@ public class ActionModuleTests extends ESTestCase {
     public void testPluginCanRegisterRestHandler() {
         class FakeHandler implements RestHandler {
             @Override
-            public void handleRequest(RestRequest request, RestChannel channel) throws Exception {
+            public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
             }
         }
         ActionPlugin registersFakeHandler = new ActionPlugin() {
diff --git a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
index 749ffa3c9d9..9ae0beadf59 100644
--- a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.common.network;
 
 import org.elasticsearch.action.support.replication.ReplicationTask;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.ModuleTestCase;
@@ -87,18 +87,18 @@ public class NetworkModuleTests extends ModuleTestCase {
 
     static class FakeRestHandler extends BaseRestHandler {
         public FakeRestHandler() {
-            super(null, null);
+            super(null);
         }
         @Override
-        protected void handleRequest(RestRequest request, RestChannel channel, Client client) throws Exception {}
+        public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {}
     }
 
     static class FakeCatRestHandler extends AbstractCatAction {
         public FakeCatRestHandler() {
-            super(null, null, null);
+            super(null, null);
         }
         @Override
-        protected void doRequest(RestRequest request, RestChannel channel, Client client) {}
+        protected void doRequest(RestRequest request, RestChannel channel, NodeClient client) {}
         @Override
         protected void documentation(StringBuilder sb) {}
         @Override
diff --git a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
index 2ba7da84c14..ce0c78c1c92 100644
--- a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
+++ b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
@@ -66,9 +66,9 @@ public class HttpServerTests extends ESTestCase {
         HttpServerTransport httpServerTransport = new TestHttpServerTransport();
         RestController restController = new RestController(settings);
         restController.registerHandler(RestRequest.Method.GET, "/",
-            (request, channel) -> channel.sendResponse(
+            (request, channel, client) -> channel.sendResponse(
                 new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)));
-        restController.registerHandler(RestRequest.Method.GET, "/error", (request, channel) -> {
+        restController.registerHandler(RestRequest.Method.GET, "/error", (request, channel, client) -> {
                 throw new IllegalArgumentException("test error");
             });
 
@@ -76,7 +76,7 @@ public class HttpServerTests extends ESTestCase {
             new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null);
         NodeService nodeService = new NodeService(Settings.EMPTY, null, null, null, null, null, null, null, null,
             clusterService, null);
-        httpServer = new HttpServer(settings, httpServerTransport, restController, nodeService, circuitBreakerService);
+        httpServer = new HttpServer(settings, httpServerTransport, restController, nodeService, null, circuitBreakerService);
         httpServer.start();
     }
 
diff --git a/core/src/test/java/org/elasticsearch/plugins/responseheader/TestResponseHeaderRestAction.java b/core/src/test/java/org/elasticsearch/plugins/responseheader/TestResponseHeaderRestAction.java
index 39432bd01ea..499b6fadc93 100644
--- a/core/src/test/java/org/elasticsearch/plugins/responseheader/TestResponseHeaderRestAction.java
+++ b/core/src/test/java/org/elasticsearch/plugins/responseheader/TestResponseHeaderRestAction.java
@@ -18,7 +18,7 @@
  */
 package org.elasticsearch.plugins.responseheader;
 
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -32,13 +32,13 @@ import org.elasticsearch.rest.RestStatus;
 public class TestResponseHeaderRestAction extends BaseRestHandler {
 
     @Inject
-    public TestResponseHeaderRestAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public TestResponseHeaderRestAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(RestRequest.Method.GET, "/_protected", this);
     }
 
     @Override
-    public void handleRequest(RestRequest request, RestChannel channel, Client client) {
+    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) {
         if ("password".equals(request.header("Secret"))) {
             RestResponse response = new BytesRestResponse(RestStatus.OK, "Access granted");
             response.addHeader("Secret", "granted");
diff --git a/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java
index 9cade7aa513..834afe5d5cd 100644
--- a/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.rest;
 
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.ESTestCase;
@@ -78,7 +79,7 @@ public class RestControllerTests extends ESTestCase {
             }
 
             @Override
-            void executeHandler(RestRequest request, RestChannel channel) throws Exception {
+            void executeHandler(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
                 assertEquals("true", threadContext.getHeader("header.1"));
                 assertEquals("true", threadContext.getHeader("header.2"));
                 assertNull(threadContext.getHeader("header.3"));
@@ -91,7 +92,7 @@ public class RestControllerTests extends ESTestCase {
         restHeaders.put("header.1", "true");
         restHeaders.put("header.2", "true");
         restHeaders.put("header.3", "false");
-        restController.dispatchRequest(new FakeRestRequest.Builder().withHeaders(restHeaders).build(), null, threadContext);
+        restController.dispatchRequest(new FakeRestRequest.Builder().withHeaders(restHeaders).build(), null, null, threadContext);
         assertNull(threadContext.getHeader("header.1"));
         assertNull(threadContext.getHeader("header.2"));
         assertEquals("true", threadContext.getHeader("header.3"));
@@ -117,7 +118,7 @@ public class RestControllerTests extends ESTestCase {
         }
 
         @Override
-        public void handleRequest(RestRequest request, RestChannel channel) throws Exception {
+        public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
             //no op
         }
 
diff --git a/core/src/test/java/org/elasticsearch/rest/RestFilterChainTests.java b/core/src/test/java/org/elasticsearch/rest/RestFilterChainTests.java
index 51f36d1e25f..dd6d1dac47b 100644
--- a/core/src/test/java/org/elasticsearch/rest/RestFilterChainTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/RestFilterChainTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.rest;
 
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -71,16 +72,13 @@ public class RestFilterChainTests extends ESTestCase {
             }
         }
 
-        restController.registerHandler(RestRequest.Method.GET, "/", new RestHandler() {
-            @Override
-            public void handleRequest(RestRequest request, RestChannel channel) throws Exception {
-                channel.sendResponse(new TestResponse());
-            }
+        restController.registerHandler(RestRequest.Method.GET, "/", (request, channel, client) -> {
+            channel.sendResponse(new TestResponse());
         });
 
         FakeRestRequest fakeRestRequest = new FakeRestRequest();
         FakeRestChannel fakeRestChannel = new FakeRestChannel(fakeRestRequest, randomBoolean(), 1);
-        restController.dispatchRequest(fakeRestRequest, fakeRestChannel, new ThreadContext(Settings.EMPTY));
+        restController.dispatchRequest(fakeRestRequest, fakeRestChannel, null, new ThreadContext(Settings.EMPTY));
         assertThat(fakeRestChannel.await(), equalTo(true));
 
 
@@ -117,12 +115,9 @@ public class RestFilterChainTests extends ESTestCase {
 
         final int additionalContinueCount = randomInt(10);
 
-        TestFilter testFilter = new TestFilter(randomInt(), new Callback() {
-            @Override
-            public void execute(final RestRequest request, final RestChannel channel, final RestFilterChain filterChain) throws Exception {
-                for (int i = 0; i <= additionalContinueCount; i++) {
-                    filterChain.continueProcessing(request, channel);
-                }
+        TestFilter testFilter = new TestFilter(randomInt(), (request, channel, client, filterChain) -> {
+            for (int i = 0; i <= additionalContinueCount; i++) {
+                filterChain.continueProcessing(request, channel, null);
             }
         });
 
@@ -131,14 +126,14 @@ public class RestFilterChainTests extends ESTestCase {
 
         restController.registerHandler(RestRequest.Method.GET, "/", new RestHandler() {
             @Override
-            public void handleRequest(RestRequest request, RestChannel channel) throws Exception {
+            public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
                 channel.sendResponse(new TestResponse());
             }
         });
 
         FakeRestRequest fakeRestRequest = new FakeRestRequest();
         FakeRestChannel fakeRestChannel = new FakeRestChannel(fakeRestRequest, randomBoolean(), additionalContinueCount + 1);
-        restController.dispatchRequest(fakeRestRequest, fakeRestChannel, new ThreadContext(Settings.EMPTY));
+        restController.dispatchRequest(fakeRestRequest, fakeRestChannel, null, new ThreadContext(Settings.EMPTY));
         fakeRestChannel.await();
 
         assertThat(testFilter.runs.get(), equalTo(1));
@@ -150,20 +145,20 @@ public class RestFilterChainTests extends ESTestCase {
     private static enum Operation implements Callback {
         CONTINUE_PROCESSING {
             @Override
-            public void execute(RestRequest request, RestChannel channel, RestFilterChain filterChain) throws Exception {
-                filterChain.continueProcessing(request, channel);
+            public void execute(RestRequest request, RestChannel channel, NodeClient client, RestFilterChain filterChain) throws Exception {
+                filterChain.continueProcessing(request, channel, client);
             }
         },
         CHANNEL_RESPONSE {
             @Override
-            public void execute(RestRequest request, RestChannel channel, RestFilterChain filterChain) throws Exception {
+            public void execute(RestRequest request, RestChannel channel, NodeClient client, RestFilterChain filterChain) throws Exception {
                 channel.sendResponse(new TestResponse());
             }
         }
     }
 
     private static interface Callback {
-        void execute(RestRequest request, RestChannel channel, RestFilterChain filterChain) throws Exception;
+        void execute(RestRequest request, RestChannel channel, NodeClient client, RestFilterChain filterChain) throws Exception;
     }
 
     private final AtomicInteger counter = new AtomicInteger();
@@ -180,10 +175,10 @@ public class RestFilterChainTests extends ESTestCase {
         }
 
         @Override
-        public void process(RestRequest request, RestChannel channel, RestFilterChain filterChain) throws Exception {
+        public void process(RestRequest request, RestChannel channel, NodeClient client, RestFilterChain filterChain) throws Exception {
             this.runs.incrementAndGet();
             this.executionToken = counter.incrementAndGet();
-            this.callback.execute(request, channel, filterChain);
+            this.callback.execute(request, channel, client, filterChain);
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java
index b603ded8697..34275d78d7b 100644
--- a/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/action/cat/RestRecoveryActionTests.java
@@ -51,7 +51,7 @@ public class RestRecoveryActionTests extends ESTestCase {
     public void testRestRecoveryAction() {
         final Settings settings = Settings.EMPTY;
         final RestController restController = new RestController(settings);
-        final RestRecoveryAction action = new RestRecoveryAction(settings, restController, restController, null);
+        final RestRecoveryAction action = new RestRecoveryAction(settings, restController, restController);
         final int totalShards = randomIntBetween(1, 32);
         final int successfulShards = Math.max(0, totalShards - randomIntBetween(1, 2));
         final int failedShards = totalShards - successfulShards;
diff --git a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
index 3ba6c875b68..19c508e2bb1 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
@@ -31,6 +31,7 @@ import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -635,6 +636,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
                         .put(MockRepository.Plugin.PASSWORD_SETTING.getKey(), "verysecretpassword")
         ).get();
 
+        NodeClient nodeClient = internalCluster().getInstance(NodeClient.class);
         RestGetRepositoriesAction getRepoAction = internalCluster().getInstance(RestGetRepositoriesAction.class);
         RestRequest getRepoRequest = new FakeRestRequest();
         getRepoRequest.params().put("repository", "test-repo");
@@ -651,7 +653,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
                 }
                 getRepoLatch.countDown();
             }
-        });
+        }, nodeClient);
         assertTrue(getRepoLatch.await(1, TimeUnit.SECONDS));
         if (getRepoError.get() != null) {
             throw getRepoError.get();
@@ -672,7 +674,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
                 }
                 clusterStateLatch.countDown();
             }
-        });
+        }, nodeClient);
         assertTrue(clusterStateLatch.await(1, TimeUnit.SECONDS));
         if (clusterStateError.get() != null) {
             throw clusterStateError.get();
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestDeleteSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestDeleteSearchTemplateAction.java
index 46bad569933..22adfe13ac4 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestDeleteSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestDeleteSearchTemplateAction.java
@@ -18,7 +18,7 @@
  */
 package org.elasticsearch.rest.action.search.template;
 
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.RestController;
@@ -31,8 +31,8 @@ import static org.elasticsearch.rest.RestRequest.Method.DELETE;
 public class RestDeleteSearchTemplateAction extends RestDeleteStoredScriptAction {
 
     @Inject
-    public RestDeleteSearchTemplateAction(Settings settings, RestController controller, Client client) {
-        super(settings, controller, false, client);
+    public RestDeleteSearchTemplateAction(Settings settings, RestController controller) {
+        super(settings, controller, false);
         controller.registerHandler(DELETE, "/_search/template/{id}", this);
     }
 
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestGetSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestGetSearchTemplateAction.java
index 61fcaf1ccec..f92cdc02a67 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestGetSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestGetSearchTemplateAction.java
@@ -18,7 +18,7 @@
  */
 package org.elasticsearch.rest.action.search.template;
 
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.RestController;
@@ -33,8 +33,8 @@ public class RestGetSearchTemplateAction extends RestGetStoredScriptAction {
     private static final String TEMPLATE = "template";
 
     @Inject
-    public RestGetSearchTemplateAction(Settings settings, RestController controller, Client client) {
-        super(settings, controller, false, client);
+    public RestGetSearchTemplateAction(Settings settings, RestController controller) {
+        super(settings, controller, false);
         controller.registerHandler(GET, "/_search/template/{id}", this);
     }
 
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestMultiSearchTemplateAction.java
index 48f69dda50f..add4719ad93 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestMultiSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestMultiSearchTemplateAction.java
@@ -24,7 +24,7 @@ import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.action.search.template.MultiSearchTemplateAction;
 import org.elasticsearch.action.search.template.MultiSearchTemplateRequest;
 import org.elasticsearch.action.search.template.SearchTemplateRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -45,8 +45,8 @@ public class RestMultiSearchTemplateAction extends BaseRestHandler {
     private final boolean allowExplicitIndex;
 
     @Inject
-    public RestMultiSearchTemplateAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestMultiSearchTemplateAction(Settings settings, RestController controller) {
+        super(settings);
         this.allowExplicitIndex = MULTI_ALLOW_EXPLICIT_INDEX.get(settings);
 
         controller.registerHandler(GET, "/_msearch/template", this);
@@ -58,7 +58,7 @@ public class RestMultiSearchTemplateAction extends BaseRestHandler {
     }
 
     @Override
-    protected void handleRequest(RestRequest request, RestChannel channel, Client client) throws Exception {
+    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
         if (RestActions.hasBodyContent(request) == false) {
             throw new ElasticsearchException("request body is required");
         }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestPutSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestPutSearchTemplateAction.java
index 90854bee01c..48e98bc9237 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestPutSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestPutSearchTemplateAction.java
@@ -18,7 +18,7 @@
  */
 package org.elasticsearch.rest.action.search.template;
 
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.RestController;
@@ -32,8 +32,8 @@ import static org.elasticsearch.rest.RestRequest.Method.PUT;
 public class RestPutSearchTemplateAction extends RestPutStoredScriptAction {
 
     @Inject
-    public RestPutSearchTemplateAction(Settings settings, RestController controller, Client client) {
-        super(settings, controller, false, client);
+    public RestPutSearchTemplateAction(Settings settings, RestController controller) {
+        super(settings, controller, false);
         controller.registerHandler(POST, "/_search/template/{id}", this);
         controller.registerHandler(PUT, "/_search/template/{id}", this);
     }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestRenderSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestRenderSearchTemplateAction.java
index 388837962c5..5e7e75e9c81 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestRenderSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestRenderSearchTemplateAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.rest.action.search.template;
 
 import org.elasticsearch.action.search.template.SearchTemplateAction;
 import org.elasticsearch.action.search.template.SearchTemplateRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -38,8 +38,8 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 public class RestRenderSearchTemplateAction extends BaseRestHandler {
 
     @Inject
-    public RestRenderSearchTemplateAction(Settings settings, RestController controller, Client client) {
-        super(settings, client);
+    public RestRenderSearchTemplateAction(Settings settings, RestController controller) {
+        super(settings);
         controller.registerHandler(GET, "/_render/template", this);
         controller.registerHandler(POST, "/_render/template", this);
         controller.registerHandler(GET, "/_render/template/{id}", this);
@@ -47,7 +47,7 @@ public class RestRenderSearchTemplateAction extends BaseRestHandler {
     }
 
     @Override
-    protected void handleRequest(RestRequest request, RestChannel channel, Client client) throws Exception {
+    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
         // Creates the render template request
         SearchTemplateRequest renderRequest = RestSearchTemplateAction.parse(RestActions.getRestContent(request));
         renderRequest.setSimulate(true);
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestSearchTemplateAction.java
index b15efa74420..131443da887 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestSearchTemplateAction.java
@@ -23,7 +23,7 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.template.SearchTemplateAction;
 import org.elasticsearch.action.search.template.SearchTemplateRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParseFieldMatcherSupplier;
@@ -88,9 +88,9 @@ public class RestSearchTemplateAction extends BaseRestHandler {
     private final Suggesters suggesters;
 
     @Inject
-    public RestSearchTemplateAction(Settings settings, RestController controller, Client client, IndicesQueriesRegistry queryRegistry,
+    public RestSearchTemplateAction(Settings settings, RestController controller, IndicesQueriesRegistry queryRegistry,
                                     AggregatorParsers aggregatorParsers, Suggesters suggesters) {
-        super(settings, client);
+        super(settings);
         this.queryRegistry = queryRegistry;
         this.aggParsers = aggregatorParsers;
         this.suggesters = suggesters;
@@ -104,7 +104,7 @@ public class RestSearchTemplateAction extends BaseRestHandler {
     }
 
     @Override
-    protected void handleRequest(RestRequest request, RestChannel channel, Client client) throws Exception {
+    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
         if (RestActions.hasBodyContent(request) == false) {
             throw new ElasticsearchException("request body is required");
         }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java
index a2902a9a7c2..6461ab3028d 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java
@@ -19,7 +19,7 @@
 package org.elasticsearch.percolator;
 
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -39,9 +39,9 @@ public class RestMultiPercolateAction extends BaseRestHandler {
     private final TransportMultiPercolateAction action;
 
     @Inject
-    public RestMultiPercolateAction(Settings settings, RestController controller, Client client,
+    public RestMultiPercolateAction(Settings settings, RestController controller,
                                     TransportMultiPercolateAction action) {
-        super(settings, client);
+        super(settings);
         this.action = action;
         controller.registerHandler(POST, "/_mpercolate", this);
         controller.registerHandler(POST, "/{index}/_mpercolate", this);
@@ -55,7 +55,7 @@ public class RestMultiPercolateAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest restRequest, final RestChannel restChannel, final Client client) throws Exception {
+    public void handleRequest(final RestRequest restRequest, final RestChannel restChannel, final NodeClient client) throws Exception {
         MultiPercolateRequest multiPercolateRequest = new MultiPercolateRequest();
         multiPercolateRequest.indicesOptions(IndicesOptions.fromRequest(restRequest, multiPercolateRequest.indicesOptions()));
         multiPercolateRequest.indices(Strings.splitStringByCommaToArray(restRequest.param("index")));
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java
index b752cc55f6c..a09107eedda 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java
@@ -20,7 +20,7 @@ package org.elasticsearch.percolator;
 
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -40,26 +40,26 @@ public class RestPercolateAction extends BaseRestHandler {
     private final TransportPercolateAction action;
 
     @Inject
-    public RestPercolateAction(Settings settings, RestController controller, Client client, TransportPercolateAction action) {
-        super(settings, client);
+    public RestPercolateAction(Settings settings, RestController controller, TransportPercolateAction action) {
+        super(settings);
         this.action = action;
         controller.registerHandler(GET, "/{index}/{type}/_percolate", this);
         controller.registerHandler(POST, "/{index}/{type}/_percolate", this);
 
-        RestPercolateExistingDocHandler existingDocHandler = new RestPercolateExistingDocHandler(settings, controller, client);
+        RestPercolateExistingDocHandler existingDocHandler = new RestPercolateExistingDocHandler(settings, controller);
         controller.registerHandler(GET, "/{index}/{type}/{id}/_percolate", existingDocHandler);
         controller.registerHandler(POST, "/{index}/{type}/{id}/_percolate", existingDocHandler);
 
-        RestCountPercolateDocHandler countHandler = new RestCountPercolateDocHandler(settings, controller, client);
+        RestCountPercolateDocHandler countHandler = new RestCountPercolateDocHandler(settings, controller);
         controller.registerHandler(GET, "/{index}/{type}/_percolate/count", countHandler);
         controller.registerHandler(POST, "/{index}/{type}/_percolate/count", countHandler);
 
-        RestCountPercolateExistingDocHandler countExistingDocHandler = new RestCountPercolateExistingDocHandler(settings, controller, client);
+        RestCountPercolateExistingDocHandler countExistingDocHandler = new RestCountPercolateExistingDocHandler(settings, controller);
         controller.registerHandler(GET, "/{index}/{type}/{id}/_percolate/count", countExistingDocHandler);
         controller.registerHandler(POST, "/{index}/{type}/{id}/_percolate/count", countExistingDocHandler);
     }
 
-    void parseDocPercolate(PercolateRequest percolateRequest, RestRequest restRequest, RestChannel restChannel, final Client client) {
+    void parseDocPercolate(PercolateRequest percolateRequest, RestRequest restRequest, RestChannel restChannel) {
         percolateRequest.indices(Strings.splitStringByCommaToArray(restRequest.param("index")));
         percolateRequest.documentType(restRequest.param("type"));
         percolateRequest.routing(restRequest.param("routing"));
@@ -70,7 +70,7 @@ public class RestPercolateAction extends BaseRestHandler {
         executePercolate(percolateRequest, restChannel);
     }
 
-    void parseExistingDocPercolate(PercolateRequest percolateRequest, RestRequest restRequest, RestChannel restChannel, final Client client) {
+    void parseExistingDocPercolate(PercolateRequest percolateRequest, RestRequest restRequest, RestChannel restChannel) {
         String index = restRequest.param("index");
         String type = restRequest.param("type");
         percolateRequest.indices(Strings.splitStringByCommaToArray(restRequest.param("percolate_index", index)));
@@ -99,49 +99,49 @@ public class RestPercolateAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(RestRequest restRequest, RestChannel restChannel, final Client client) {
+    public void handleRequest(RestRequest restRequest, RestChannel restChannel, final NodeClient client) {
         PercolateRequest percolateRequest = new PercolateRequest();
-        parseDocPercolate(percolateRequest, restRequest, restChannel, client);
+        parseDocPercolate(percolateRequest, restRequest, restChannel);
     }
 
     final class RestCountPercolateDocHandler extends BaseRestHandler {
 
-        private RestCountPercolateDocHandler(Settings settings, final RestController controller, Client client) {
-            super(settings, client);
+        private RestCountPercolateDocHandler(Settings settings, final RestController controller) {
+            super(settings);
         }
 
         @Override
-        public void handleRequest(RestRequest restRequest, RestChannel restChannel, final Client client) {
+        public void handleRequest(RestRequest restRequest, RestChannel restChannel, final NodeClient client) {
             PercolateRequest percolateRequest = new PercolateRequest();
             percolateRequest.onlyCount(true);
-            parseDocPercolate(percolateRequest, restRequest, restChannel, client);
+            parseDocPercolate(percolateRequest, restRequest, restChannel);
         }
     }
 
     final class RestPercolateExistingDocHandler extends BaseRestHandler {
 
-        protected RestPercolateExistingDocHandler(Settings settings, final RestController controller, Client client) {
-            super(settings, client);
+        protected RestPercolateExistingDocHandler(Settings settings, final RestController controller) {
+            super(settings);
         }
 
         @Override
-        public void handleRequest(RestRequest restRequest, RestChannel restChannel, final Client client) {
+        public void handleRequest(RestRequest restRequest, RestChannel restChannel, final NodeClient client) {
             PercolateRequest percolateRequest = new PercolateRequest();
-            parseExistingDocPercolate(percolateRequest, restRequest, restChannel, client);
+            parseExistingDocPercolate(percolateRequest, restRequest, restChannel);
         }
     }
 
     final class RestCountPercolateExistingDocHandler extends BaseRestHandler {
 
-        protected RestCountPercolateExistingDocHandler(Settings settings, final RestController controller, Client client) {
-            super(settings, client);
+        protected RestCountPercolateExistingDocHandler(Settings settings, final RestController controller) {
+            super(settings);
         }
 
         @Override
-        public void handleRequest(RestRequest restRequest, RestChannel restChannel, final Client client) {
+        public void handleRequest(RestRequest restRequest, RestChannel restChannel, final NodeClient client) {
             PercolateRequest percolateRequest = new PercolateRequest();
             percolateRequest.onlyCount(true);
-            parseExistingDocPercolate(percolateRequest, restRequest, restChannel, client);
+            parseExistingDocPercolate(percolateRequest, restRequest, restChannel);
         }
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
index 284e51e054f..048e4208fea 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
@@ -52,10 +52,10 @@ public abstract class AbstractBaseReindexRestHandler<
     private final ClusterService clusterService;
     private final TA action;
 
-    protected AbstractBaseReindexRestHandler(Settings settings, Client client,
-            IndicesQueriesRegistry indicesQueriesRegistry, AggregatorParsers aggParsers, Suggesters suggesters,
-            ClusterService clusterService, TA action) {
-        super(settings, client);
+    protected AbstractBaseReindexRestHandler(Settings settings, IndicesQueriesRegistry indicesQueriesRegistry,
+                                             AggregatorParsers aggParsers, Suggesters suggesters,
+                                             ClusterService clusterService, TA action) {
+        super(settings);
         this.indicesQueriesRegistry = indicesQueriesRegistry;
         this.aggParsers = aggParsers;
         this.suggesters = suggesters;
@@ -63,7 +63,7 @@ public abstract class AbstractBaseReindexRestHandler<
         this.action = action;
     }
 
-    protected void handleRequest(RestRequest request, RestChannel channel,
+    public void handleRequest(RestRequest request, RestChannel channel,
                                  boolean includeCreated, boolean includeUpdated) throws IOException {
 
         // Build the internal request
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java
index 926da3befdd..9e4d8fc6d4e 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java
@@ -50,10 +50,10 @@ public abstract class AbstractBulkByQueryRestHandler<
         Request extends AbstractBulkByScrollRequest<Request>,
         TA extends TransportAction<Request, BulkIndexByScrollResponse>> extends AbstractBaseReindexRestHandler<Request, TA> {
 
-    protected AbstractBulkByQueryRestHandler(Settings settings, Client client, IndicesQueriesRegistry indicesQueriesRegistry,
+    protected AbstractBulkByQueryRestHandler(Settings settings, IndicesQueriesRegistry indicesQueriesRegistry,
                                              AggregatorParsers aggParsers, Suggesters suggesters, ClusterService clusterService,
                                              TA action) {
-        super(settings, client, indicesQueriesRegistry, aggParsers, suggesters, clusterService, action);
+        super(settings, indicesQueriesRegistry, aggParsers, suggesters, clusterService, action);
     }
 
     protected void parseInternalRequest(Request internal, RestRequest restRequest,
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java
index bb894584c8b..7a1f466c3c0 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -42,16 +42,16 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 public class RestDeleteByQueryAction extends AbstractBulkByQueryRestHandler<DeleteByQueryRequest, TransportDeleteByQueryAction> {
 
     @Inject
-    public RestDeleteByQueryAction(Settings settings, RestController controller, Client client,
+    public RestDeleteByQueryAction(Settings settings, RestController controller,
                                    IndicesQueriesRegistry indicesQueriesRegistry, AggregatorParsers aggParsers, Suggesters suggesters,
                                    ClusterService clusterService, TransportDeleteByQueryAction action) {
-        super(settings, client, indicesQueriesRegistry, aggParsers, suggesters, clusterService, action);
+        super(settings, indicesQueriesRegistry, aggParsers, suggesters, clusterService, action);
         controller.registerHandler(POST, "/{index}/_delete_by_query", this);
         controller.registerHandler(POST, "/{index}/{type}/_delete_by_query", this);
     }
 
     @Override
-    protected void handleRequest(RestRequest request, RestChannel channel, Client client) throws Exception {
+    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
         if (false == request.hasContent()) {
             throw new ElasticsearchException("_delete_by_query requires a request body");
         }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
index 22fcd390430..af0bd168ff0 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
@@ -22,7 +22,7 @@ package org.elasticsearch.index.reindex;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParseFieldMatcher;
@@ -103,15 +103,15 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexReq
     }
 
     @Inject
-    public RestReindexAction(Settings settings, RestController controller, Client client,
+    public RestReindexAction(Settings settings, RestController controller,
             IndicesQueriesRegistry indicesQueriesRegistry, AggregatorParsers aggParsers, Suggesters suggesters,
             ClusterService clusterService, TransportReindexAction action) {
-        super(settings, client, indicesQueriesRegistry, aggParsers, suggesters, clusterService, action);
+        super(settings, indicesQueriesRegistry, aggParsers, suggesters, clusterService, action);
         controller.registerHandler(POST, "/_reindex", this);
     }
 
     @Override
-    public void handleRequest(RestRequest request, RestChannel channel, Client client) throws IOException {
+    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws IOException {
         if (false == request.hasContent()) {
             throw new ElasticsearchException("_reindex requires a request body");
         }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java
index 9841794ca2a..a4edc3dfd0f 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java
@@ -21,7 +21,7 @@ package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -40,9 +40,9 @@ public class RestRethrottleAction extends BaseRestHandler {
     private final ClusterService clusterService;
 
     @Inject
-    public RestRethrottleAction(Settings settings, RestController controller, Client client, TransportRethrottleAction action,
+    public RestRethrottleAction(Settings settings, RestController controller, TransportRethrottleAction action,
             ClusterService clusterService) {
-        super(settings, client);
+        super(settings);
         this.action = action;
         this.clusterService = clusterService;
         controller.registerHandler(POST, "/_update_by_query/{taskId}/_rethrottle", this);
@@ -51,7 +51,7 @@ public class RestRethrottleAction extends BaseRestHandler {
     }
 
     @Override
-    public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         RethrottleRequest internalRequest = new RethrottleRequest();
         internalRequest.setTaskId(new TaskId(request.param("taskId")));
         Float requestsPerSecond = AbstractBaseReindexRestHandler.parseRequestsPerSecond(request);
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
index 8eb0e30fe38..81bc0b5d27f 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -42,16 +42,16 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 public class RestUpdateByQueryAction extends AbstractBulkByQueryRestHandler<UpdateByQueryRequest, TransportUpdateByQueryAction> {
 
     @Inject
-    public RestUpdateByQueryAction(Settings settings, RestController controller, Client client,
+    public RestUpdateByQueryAction(Settings settings, RestController controller,
             IndicesQueriesRegistry indicesQueriesRegistry, AggregatorParsers aggParsers, Suggesters suggesters,
             ClusterService clusterService, TransportUpdateByQueryAction action) {
-        super(settings, client, indicesQueriesRegistry, aggParsers, suggesters, clusterService, action);
+        super(settings, indicesQueriesRegistry, aggParsers, suggesters, clusterService, action);
         controller.registerHandler(POST, "/{index}/_update_by_query", this);
         controller.registerHandler(POST, "/{index}/{type}/_update_by_query", this);
     }
 
     @Override
-    protected void handleRequest(RestRequest request, RestChannel channel, Client client) throws Exception {
+    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
         handleRequest(request, channel, false, true);
     }
 
diff --git a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java
index d5e0a62ecb5..75ae79d5f7d 100644
--- a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java
+++ b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java
@@ -19,6 +19,7 @@
 package org.elasticsearch.plugin.example;
 
 import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -38,15 +39,14 @@ public class ExampleCatAction extends AbstractCatAction {
     private final ExamplePluginConfiguration config;
 
     @Inject
-    public ExampleCatAction(Settings settings, RestController controller,
-            Client client, ExamplePluginConfiguration config) {
-        super(settings, controller, client);
+    public ExampleCatAction(Settings settings, RestController controller, ExamplePluginConfiguration config) {
+        super(settings, controller);
         this.config = config;
         controller.registerHandler(GET, "/_cat/configured_example", this);
     }
 
     @Override
-    protected void doRequest(final RestRequest request, final RestChannel channel, final Client client) {
+    protected void doRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         Table table = getTableWithHeader(request);
         table.startRow();
         table.addCell(config.getTestConfig());

From 8b43480b9464c3420ee7aee94e834024fd408293 Mon Sep 17 00:00:00 2001
From: Alexander Reelsen <alexander@reelsen.net>
Date: Thu, 30 Jun 2016 09:08:31 +0200
Subject: [PATCH 1130/1311] Tests: Fix vagrant tests to ignore progress bar in
 assertions

---
 .../scripts/module_and_plugin_test_cases.bash       | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
index c81f84cb778..336fa3ee305 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
@@ -418,17 +418,18 @@ fi
 @test "[$GROUP] install jvm-example with different logging modes and check output" {
     local relativePath=${1:-$(readlink -m jvm-example-*.zip)}
     sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/elasticsearch-plugin" install "file://$relativePath" > /tmp/plugin-cli-output
-    local loglines=$(cat /tmp/plugin-cli-output | wc -l)
+    # exclude progress line
+    local loglines=$(cat /tmp/plugin-cli-output | grep -v "^[[:cntrl:]]" | wc -l)
     if [ "$GROUP" == "TAR PLUGINS" ]; then
     # tar extraction does not create the plugins directory so the plugin tool will print an additional line that the directory will be created
         [ "$loglines" -eq "3" ] || {
-            echo "Expected 3 lines but the output was:"
+            echo "Expected 3 lines excluding progress bar but the output had $loglines lines and was:"
             cat /tmp/plugin-cli-output
             false
         }
     else
         [ "$loglines" -eq "2" ] || {
-            echo "Expected 2 lines but the output was:"
+            echo "Expected 2 lines excluding progress bar but the output had $loglines lines and was:"
             cat /tmp/plugin-cli-output
             false
         }
@@ -437,16 +438,16 @@ fi
 
     local relativePath=${1:-$(readlink -m jvm-example-*.zip)}
     sudo -E -u $ESPLUGIN_COMMAND_USER ES_JAVA_OPTS="-Des.logger.level=DEBUG" "$ESHOME/bin/elasticsearch-plugin" install "file://$relativePath" > /tmp/plugin-cli-output
-    local loglines=$(cat /tmp/plugin-cli-output | wc -l)
+    local loglines=$(cat /tmp/plugin-cli-output | grep -v "^[[:cntrl:]]" | wc -l)
     if [ "$GROUP" == "TAR PLUGINS" ]; then
         [ "$loglines" -gt "3" ] || {
-            echo "Expected more than 3 lines but the output was:"
+            echo "Expected more than 3 lines excluding progress bar but the output had $loglines lines and was:"
             cat /tmp/plugin-cli-output
             false
         }
     else
         [ "$loglines" -gt "2" ] || {
-            echo "Expected more than 2 lines but the output was:"
+            echo "Expected more than 2 lines excluding progress bar but the output had $loglines lines and was:"
             cat /tmp/plugin-cli-output
             false
         }

From 08b3b6264e8618deb125ab5a4825e50a7c04f9ab Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Thu, 30 Jun 2016 01:49:22 -0700
Subject: [PATCH 1131/1311] Tests pass, started removing generics from
 processor factory

---
 .../ingest/SimulatePipelineRequest.java       |  2 +-
 .../ingest/AbstractProcessorFactory.java      |  7 +--
 .../ingest/ConfigurationUtils.java            | 12 ++---
 .../elasticsearch/ingest/IngestService.java   | 39 ++++++++------
 .../org/elasticsearch/ingest/Pipeline.java    |  6 +--
 .../elasticsearch/ingest/PipelineStore.java   | 33 +++++-------
 .../org/elasticsearch/ingest/Processor.java   | 11 ++--
 .../ingest/ProcessorsRegistry.java            | 48 -----------------
 .../java/org/elasticsearch/node/Node.java     |  6 +++
 .../org/elasticsearch/node/NodeModule.java    | 26 ----------
 .../node/service/NodeService.java             | 17 +++----
 .../SimulatePipelineRequestParsingTests.java  | 10 ++--
 .../elasticsearch/http/HttpServerTests.java   |  7 ++-
 .../ingest/ConfigurationUtilsTests.java       | 20 +++-----
 .../ingest/PipelineFactoryTests.java          | 24 +++------
 .../ingest/PipelineStoreTests.java            |  9 ++--
 .../ingest/ProcessorsRegistryTests.java       | 51 -------------------
 .../common/AbstractStringProcessor.java       |  8 +--
 .../ingest/common/AppendProcessor.java        |  7 +--
 .../ingest/common/ConvertProcessor.java       |  7 +--
 .../ingest/common/DateIndexNameProcessor.java | 27 +++++-----
 .../ingest/common/DateProcessor.java          |  7 +--
 .../ingest/common/FailProcessor.java          |  7 +--
 .../ingest/common/ForEachProcessor.java       |  9 ++--
 .../ingest/common/GrokProcessor.java          |  7 +--
 .../ingest/common/GsubProcessor.java          |  7 +--
 .../ingest/common/JoinProcessor.java          |  7 +--
 .../ingest/common/LowercaseProcessor.java     |  2 +-
 .../ingest/common/RemoveProcessor.java        |  7 +--
 .../ingest/common/RenameProcessor.java        |  7 +--
 .../ingest/common/ScriptProcessor.java        |  7 +--
 .../ingest/common/SetProcessor.java           |  7 +--
 .../ingest/common/SortProcessor.java          |  7 +--
 .../ingest/common/SplitProcessor.java         |  7 +--
 .../ingest/common/TrimProcessor.java          |  2 +-
 .../ingest/common/UppercaseProcessor.java     |  2 +-
 .../common/DateIndexNameFactoryTests.java     |  4 +-
 .../common/FailProcessorFactoryTests.java     |  2 +-
 .../common/ForEachProcessorFactoryTests.java  |  7 ++-
 .../common/GrokProcessorFactoryTests.java     | 10 ++--
 .../common/ScriptProcessorFactoryTests.java   |  5 +-
 .../common/SplitProcessorFactoryTests.java    |  2 +-
 .../ingest/common/SplitProcessorTests.java    |  2 +-
 .../attachment/AttachmentProcessor.java       |  7 +--
 .../attachment/IngestAttachmentPlugin.java    | 20 +++++---
 .../ingest/geoip/GeoIpProcessor.java          | 46 ++++++++---------
 .../ingest/geoip/IngestGeoIpPlugin.java       |  2 +-
 .../geoip/GeoIpProcessorFactoryTests.java     | 10 ++--
 .../ingest/IngestTestPlugin.java              | 29 ++++++-----
 .../elasticsearch/ingest/TestProcessor.java   |  4 +-
 50 files changed, 259 insertions(+), 360 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java
 delete mode 100644 core/src/test/java/org/elasticsearch/ingest/ProcessorsRegistryTests.java

diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
index ce24969e8f2..18b1002ed6c 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
@@ -141,7 +141,7 @@ public class SimulatePipelineRequest extends ActionRequest<SimulatePipelineReque
 
     static Parsed parse(Map<String, Object> config, boolean verbose, PipelineStore pipelineStore) throws Exception {
         Map<String, Object> pipelineConfig = ConfigurationUtils.readMap(null, null, config, Fields.PIPELINE);
-        Pipeline pipeline = PIPELINE_FACTORY.create(SIMULATED_PIPELINE_ID, pipelineConfig, pipelineStore.getProcessorRegistry());
+        Pipeline pipeline = PIPELINE_FACTORY.create(SIMULATED_PIPELINE_ID, pipelineConfig, pipelineStore.getProcessorFactories());
         List<IngestDocument> ingestDocumentList = parseDocs(config);
         return new Parsed(pipeline, ingestDocumentList, verbose);
     }
diff --git a/core/src/main/java/org/elasticsearch/ingest/AbstractProcessorFactory.java b/core/src/main/java/org/elasticsearch/ingest/AbstractProcessorFactory.java
index 95c4a93c9bc..81f79c0fd4f 100644
--- a/core/src/main/java/org/elasticsearch/ingest/AbstractProcessorFactory.java
+++ b/core/src/main/java/org/elasticsearch/ingest/AbstractProcessorFactory.java
@@ -25,14 +25,15 @@ import java.util.Map;
  * A processor implementation may modify the data belonging to a document.
  * Whether changes are made and what exactly is modified is up to the implementation.
  */
-public abstract class AbstractProcessorFactory<P extends Processor> implements Processor.Factory<P> {
+public abstract class AbstractProcessorFactory implements Processor.Factory {
     public static final String TAG_KEY = "tag";
 
     @Override
-    public P create(ProcessorsRegistry registry, Map<String, Object> config) throws Exception {
+    public Processor create(Map<String, Processor.Factory> registry, Map<String, Object> config) throws Exception {
         String tag = ConfigurationUtils.readOptionalStringProperty(null, null, config, TAG_KEY);
         return doCreate(registry, tag, config);
     }
 
-    protected abstract P doCreate(ProcessorsRegistry registry, String tag, Map<String, Object> config) throws Exception;
+    protected abstract Processor doCreate(Map<String, Processor.Factory> registry, String tag,
+                                          Map<String, Object> config) throws Exception;
 }
diff --git a/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java b/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
index 22eea7d23b7..ff0f55a559b 100644
--- a/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
+++ b/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
@@ -234,12 +234,12 @@ public final class ConfigurationUtils {
     }
 
     public static List<Processor> readProcessorConfigs(List<Map<String, Map<String, Object>>> processorConfigs,
-                                                       ProcessorsRegistry processorRegistry) throws Exception {
+                                                       Map<String, Processor.Factory> processorFactories) throws Exception {
         List<Processor> processors = new ArrayList<>();
         if (processorConfigs != null) {
             for (Map<String, Map<String, Object>> processorConfigWithKey : processorConfigs) {
                 for (Map.Entry<String, Map<String, Object>> entry : processorConfigWithKey.entrySet()) {
-                    processors.add(readProcessor(processorRegistry, entry.getKey(), entry.getValue()));
+                    processors.add(readProcessor(processorFactories, entry.getKey(), entry.getValue()));
                 }
             }
         }
@@ -247,15 +247,15 @@ public final class ConfigurationUtils {
         return processors;
     }
 
-    private static Processor readProcessor(ProcessorsRegistry processorRegistry, String type, Map<String, Object> config) throws Exception {
-        Processor.Factory factory = processorRegistry.getProcessorFactory(type);
+    private static Processor readProcessor(Map<String, Processor.Factory> processorFactories, String type, Map<String, Object> config) throws Exception {
+        Processor.Factory factory = processorFactories.get(type);
         if (factory != null) {
             boolean ignoreFailure = ConfigurationUtils.readBooleanProperty(null, null, config, "ignore_failure", false);
             List<Map<String, Map<String, Object>>> onFailureProcessorConfigs =
                 ConfigurationUtils.readOptionalList(null, null, config, Pipeline.ON_FAILURE_KEY);
 
-            List<Processor> onFailureProcessors = readProcessorConfigs(onFailureProcessorConfigs, processorRegistry);
-            Processor processor = factory.create(processorRegistry, config);
+            List<Processor> onFailureProcessors = readProcessorConfigs(onFailureProcessorConfigs, processorFactories);
+            Processor processor = factory.create(processorFactories, config);
 
             if (onFailureProcessorConfigs != null && onFailureProcessors.isEmpty()) {
                 throw newConfigurationException(processor.getType(), processor.getTag(), Pipeline.ON_FAILURE_KEY,
diff --git a/core/src/main/java/org/elasticsearch/ingest/IngestService.java b/core/src/main/java/org/elasticsearch/ingest/IngestService.java
index ce7d6dadee5..c540d25334c 100644
--- a/core/src/main/java/org/elasticsearch/ingest/IngestService.java
+++ b/core/src/main/java/org/elasticsearch/ingest/IngestService.java
@@ -19,16 +19,18 @@
 
 package org.elasticsearch.ingest;
 
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.threadpool.ThreadPool;
-
-import java.io.Closeable;
-import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.plugins.IngestPlugin;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.threadpool.ThreadPool;
 
 /**
  * Holder class for several ingest related services.
@@ -37,11 +39,20 @@ public class IngestService {
 
     private final PipelineStore pipelineStore;
     private final PipelineExecutionService pipelineExecutionService;
-    private final ProcessorsRegistry.Builder processorsRegistryBuilder;
 
-    public IngestService(Settings settings, ThreadPool threadPool, ProcessorsRegistry.Builder processorsRegistryBuilder) {
-        this.processorsRegistryBuilder = processorsRegistryBuilder;
-        this.pipelineStore = new PipelineStore(settings);
+    public IngestService(Settings settings, ThreadPool threadPool,
+                         Environment env, ScriptService scriptService, List<IngestPlugin> ingestPlugins) {
+        final TemplateService templateService = new InternalTemplateService(scriptService);
+        Map<String, Processor.Factory> processorFactories = new HashMap<>();
+        for (IngestPlugin ingestPlugin : ingestPlugins) {
+            Map<String, Processor.Factory> newProcessors = ingestPlugin.getProcessors(env, scriptService, templateService);
+            for (Map.Entry<String, Processor.Factory> entry : newProcessors.entrySet()) {
+                if (processorFactories.put(entry.getKey(), entry.getValue()) != null) {
+                    throw new IllegalArgumentException("Ingest processor [" + entry.getKey() + "] is already registered");
+                }
+            }
+        }
+        this.pipelineStore = new PipelineStore(settings, Collections.unmodifiableMap(processorFactories));
         this.pipelineExecutionService = new PipelineExecutionService(pipelineStore, threadPool);
     }
 
@@ -53,12 +64,8 @@ public class IngestService {
         return pipelineExecutionService;
     }
 
-    public void buildProcessorsFactoryRegistry(ScriptService scriptService, ClusterService clusterService) {
-        pipelineStore.buildProcessorFactoryRegistry(processorsRegistryBuilder, scriptService, clusterService);
-    }
-
     public IngestInfo info() {
-        Map<String, Processor.Factory> processorFactories = pipelineStore.getProcessorRegistry().getProcessorFactories();
+        Map<String, Processor.Factory> processorFactories = pipelineStore.getProcessorFactories();
         List<ProcessorInfo> processorInfoList = new ArrayList<>(processorFactories.size());
         for (Map.Entry<String, Processor.Factory> entry : processorFactories.entrySet()) {
             processorInfoList.add(new ProcessorInfo(entry.getKey()));
diff --git a/core/src/main/java/org/elasticsearch/ingest/Pipeline.java b/core/src/main/java/org/elasticsearch/ingest/Pipeline.java
index d568f5521a4..3f4e05229b1 100644
--- a/core/src/main/java/org/elasticsearch/ingest/Pipeline.java
+++ b/core/src/main/java/org/elasticsearch/ingest/Pipeline.java
@@ -98,13 +98,13 @@ public final class Pipeline {
 
     public final static class Factory {
 
-        public Pipeline create(String id, Map<String, Object> config, ProcessorsRegistry processorRegistry) throws Exception {
+        public Pipeline create(String id, Map<String, Object> config, Map<String, Processor.Factory> processorFactories) throws Exception {
             String description = ConfigurationUtils.readOptionalStringProperty(null, null, config, DESCRIPTION_KEY);
             List<Map<String, Map<String, Object>>> processorConfigs = ConfigurationUtils.readList(null, null, config, PROCESSORS_KEY);
-            List<Processor> processors = ConfigurationUtils.readProcessorConfigs(processorConfigs, processorRegistry);
+            List<Processor> processors = ConfigurationUtils.readProcessorConfigs(processorConfigs, processorFactories);
             List<Map<String, Map<String, Object>>> onFailureProcessorConfigs =
                     ConfigurationUtils.readOptionalList(null, null, config, ON_FAILURE_KEY);
-            List<Processor> onFailureProcessors = ConfigurationUtils.readProcessorConfigs(onFailureProcessorConfigs, processorRegistry);
+            List<Processor> onFailureProcessors = ConfigurationUtils.readProcessorConfigs(onFailureProcessorConfigs, processorFactories);
             if (config.isEmpty() == false) {
                 throw new ElasticsearchParseException("pipeline [" + id +
                         "] doesn't support one or more provided configuration parameters " + Arrays.toString(config.keySet().toArray()));
diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
index 3118d2b5076..50ae19a2ca1 100644
--- a/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
+++ b/core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
@@ -19,6 +19,12 @@
 
 package org.elasticsearch.ingest;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.ResourceNotFoundException;
@@ -37,20 +43,11 @@ import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.script.ScriptService;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 
 public class PipelineStore extends AbstractComponent implements ClusterStateListener {
 
     private final Pipeline.Factory factory = new Pipeline.Factory();
-    private ProcessorsRegistry processorRegistry;
+    private final Map<String, Processor.Factory> processorFactories;
 
     // Ideally this should be in IngestMetadata class, but we don't have the processor factories around there.
     // We know of all the processor factories when a node with all its plugin have been initialized. Also some
@@ -58,13 +55,9 @@ public class PipelineStore extends AbstractComponent implements ClusterStateList
     // are loaded, so in the cluster state we just save the pipeline config and here we keep the actual pipelines around.
     volatile Map<String, Pipeline> pipelines = new HashMap<>();
 
-    public PipelineStore(Settings settings) {
+    public PipelineStore(Settings settings, Map<String, Processor.Factory> processorFactories) {
         super(settings);
-    }
-
-    public void buildProcessorFactoryRegistry(ProcessorsRegistry.Builder processorsRegistryBuilder, ScriptService scriptService,
-                                              ClusterService clusterService) {
-        this.processorRegistry = processorsRegistryBuilder.build(scriptService, clusterService);
+        this.processorFactories = processorFactories;
     }
 
     @Override
@@ -81,7 +74,7 @@ public class PipelineStore extends AbstractComponent implements ClusterStateList
         Map<String, Pipeline> pipelines = new HashMap<>();
         for (PipelineConfiguration pipeline : ingestMetadata.getPipelines().values()) {
             try {
-                pipelines.put(pipeline.getId(), factory.create(pipeline.getId(), pipeline.getConfigAsMap(), processorRegistry));
+                pipelines.put(pipeline.getId(), factory.create(pipeline.getId(), pipeline.getConfigAsMap(), processorFactories));
             } catch (ElasticsearchParseException e) {
                 throw e;
             } catch (Exception e) {
@@ -157,7 +150,7 @@ public class PipelineStore extends AbstractComponent implements ClusterStateList
         }
 
         Map<String, Object> pipelineConfig = XContentHelper.convertToMap(request.getSource(), false).v2();
-        Pipeline pipeline = factory.create(request.getId(), pipelineConfig, processorRegistry);
+        Pipeline pipeline = factory.create(request.getId(), pipelineConfig, processorFactories);
         List<IllegalArgumentException> exceptions = new ArrayList<>();
         for (Processor processor : pipeline.flattenAllProcessors()) {
             for (Map.Entry<DiscoveryNode, IngestInfo> entry : ingestInfos.entrySet()) {
@@ -194,8 +187,8 @@ public class PipelineStore extends AbstractComponent implements ClusterStateList
         return pipelines.get(id);
     }
 
-    public ProcessorsRegistry getProcessorRegistry() {
-        return processorRegistry;
+    public Map<String, Processor.Factory> getProcessorFactories() {
+        return processorFactories;
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/ingest/Processor.java b/core/src/main/java/org/elasticsearch/ingest/Processor.java
index 934cf52b91b..3a67bed6ea8 100644
--- a/core/src/main/java/org/elasticsearch/ingest/Processor.java
+++ b/core/src/main/java/org/elasticsearch/ingest/Processor.java
@@ -45,14 +45,17 @@ public interface Processor {
     /**
      * A factory that knows how to construct a processor based on a map of maps.
      */
-    interface Factory<P extends Processor> {
+    interface Factory {
 
         /**
          * Creates a processor based on the specified map of maps config.
          *
-         * Implementations are responsible for removing the used keys, so that after creating a pipeline ingest can
-         * verify if all configurations settings have been used.
+         * @param processorFactories Other processors which may be created inside this processor
+         * @param config The configuration for the processor
+         *
+         * <b>Note:</b> Implementations are responsible for removing the used configuration keys, so that after
+         * creating a pipeline ingest can verify if all configurations settings have been used.
          */
-        P create(ProcessorsRegistry registry, Map<String, Object> config) throws Exception;
+        Processor create(Map<String, Processor.Factory> processorFactories, Map<String, Object> config) throws Exception;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java b/core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java
deleted file mode 100644
index 1b968f0ffc7..00000000000
--- a/core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.ingest;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.function.Function;
-
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.script.ScriptService;
-
-public final class ProcessorsRegistry {
-
-    private final Map<String, Processor.Factory> processorFactories;
-
-    public ProcessorsRegistry(Map<String, Processor.Factory> processors) {
-        this.processorFactories = Collections.unmodifiableMap(processors);
-    }
-
-    public Processor.Factory getProcessorFactory(String name) {
-        return processorFactories.get(name);
-    }
-
-    // For testing:
-    Map<String, Processor.Factory> getProcessorFactories() {
-        return processorFactories;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index ff095f749c8..ec61f0feb91 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -85,12 +85,14 @@ import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.store.IndicesStore;
 import org.elasticsearch.indices.ttl.IndicesTTLService;
+import org.elasticsearch.ingest.IngestService;
 import org.elasticsearch.monitor.MonitorService;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.node.internal.InternalSettingsPreparer;
 import org.elasticsearch.node.service.NodeService;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.AnalysisPlugin;
+import org.elasticsearch.plugins.IngestPlugin;
 import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsService;
@@ -255,6 +257,9 @@ public class Node implements Closeable {
             final TribeService tribeService = new TribeService(settings, clusterService);
             resourcesToClose.add(tribeService);
             NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
+            final IngestService ingestService = new IngestService(settings, threadPool, environment,
+                scriptModule.getScriptService(), pluginsService.filterPlugins(IngestPlugin.class));
+
             ModulesBuilder modules = new ModulesBuilder();
             // plugin modules must be added here, before others or we can get crazy injection errors...
             for (Module pluginModule : pluginsService.nodeModules()) {
@@ -293,6 +298,7 @@ public class Node implements Closeable {
                     b.bind(BigArrays.class).toInstance(bigArrays);
                     b.bind(ScriptService.class).toInstance(scriptModule.getScriptService());
                     b.bind(AnalysisRegistry.class).toInstance(analysisModule.getAnalysisRegistry());
+                    b.bind(IngestService.class).toInstance(ingestService);
                 }
             );
             injector = modules.createInjector();
diff --git a/core/src/main/java/org/elasticsearch/node/NodeModule.java b/core/src/main/java/org/elasticsearch/node/NodeModule.java
index 6d8be732dd2..8299834435b 100644
--- a/core/src/main/java/org/elasticsearch/node/NodeModule.java
+++ b/core/src/main/java/org/elasticsearch/node/NodeModule.java
@@ -20,28 +20,17 @@
 package org.elasticsearch.node;
 
 import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.ingest.ProcessorsRegistry;
-import org.elasticsearch.ingest.Processor;
-import org.elasticsearch.ingest.TemplateService;
 import org.elasticsearch.monitor.MonitorService;
 import org.elasticsearch.node.service.NodeService;
 
-import java.util.function.Function;
-
-/**
- *
- */
 public class NodeModule extends AbstractModule {
 
     private final Node node;
     private final MonitorService monitorService;
-    private final ProcessorsRegistry.Builder processorsRegistryBuilder;
 
     public NodeModule(Node node, MonitorService monitorService) {
         this.node = node;
         this.monitorService = monitorService;
-        this.processorsRegistryBuilder = new ProcessorsRegistry.Builder();
     }
 
     @Override
@@ -49,20 +38,5 @@ public class NodeModule extends AbstractModule {
         bind(Node.class).toInstance(node);
         bind(MonitorService.class).toInstance(monitorService);
         bind(NodeService.class).asEagerSingleton();
-        bind(ProcessorsRegistry.Builder.class).toInstance(processorsRegistryBuilder);
-    }
-
-    /**
-     * Returns the node
-     */
-    public Node getNode() {
-        return node;
-    }
-
-    /**
-     * Adds a processor factory under a specific type name.
-     */
-    public void registerProcessor(String type, Function<ProcessorsRegistry, Processor.Factory<?>> provider) {
-        processorsRegistryBuilder.registerProcessor(type, provider);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/node/service/NodeService.java b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
index 0920a84bfd6..79fbb24a177 100644
--- a/core/src/main/java/org/elasticsearch/node/service/NodeService.java
+++ b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
@@ -19,6 +19,11 @@
 
 package org.elasticsearch.node.service;
 
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
 import org.elasticsearch.Build;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
@@ -35,18 +40,12 @@ import org.elasticsearch.http.HttpServer;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.ingest.IngestService;
-import org.elasticsearch.ingest.ProcessorsRegistry;
 import org.elasticsearch.monitor.MonitorService;
 import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.unmodifiableMap;
 
@@ -76,7 +75,7 @@ public class NodeService extends AbstractComponent implements Closeable {
     public NodeService(Settings settings, ThreadPool threadPool, MonitorService monitorService,
                        Discovery discovery, TransportService transportService, IndicesService indicesService,
                        PluginsService pluginService, CircuitBreakerService circuitBreakerService,
-                       ProcessorsRegistry.Builder processorsRegistryBuilder, ClusterService clusterService, SettingsFilter settingsFilter) {
+                       IngestService ingestService, ClusterService clusterService, SettingsFilter settingsFilter) {
         super(settings);
         this.threadPool = threadPool;
         this.monitorService = monitorService;
@@ -86,17 +85,17 @@ public class NodeService extends AbstractComponent implements Closeable {
         this.pluginService = pluginService;
         this.circuitBreakerService = circuitBreakerService;
         this.clusterService = clusterService;
-        this.ingestService = new IngestService(settings, threadPool, processorsRegistryBuilder);
+        this.ingestService = ingestService;
         this.settingsFilter = settingsFilter;
         clusterService.add(ingestService.getPipelineStore());
         clusterService.add(ingestService.getPipelineExecutionService());
     }
 
     // can not use constructor injection or there will be a circular dependency
+    // nocommit: try removing this...
     @Inject(optional = true)
     public void setScriptService(ScriptService scriptService) {
         this.scriptService = scriptService;
-        this.ingestService.buildProcessorsFactoryRegistry(scriptService, clusterService);
     }
 
     public void setHttpServer(@Nullable HttpServer httpServer) {
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
index fa4bdc6525d..6f335aef634 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
@@ -21,9 +21,8 @@ package org.elasticsearch.action.ingest;
 
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.ingest.PipelineStore;
-import org.elasticsearch.ingest.ProcessorsRegistry;
+import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.TestProcessor;
-import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.ingest.CompoundProcessor;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Pipeline;
@@ -59,12 +58,11 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
         TestProcessor processor = new TestProcessor(ingestDocument -> {});
         CompoundProcessor pipelineCompoundProcessor = new CompoundProcessor(processor);
         Pipeline pipeline = new Pipeline(SIMULATED_PIPELINE_ID, null, pipelineCompoundProcessor);
-        ProcessorsRegistry.Builder processorRegistryBuilder = new ProcessorsRegistry.Builder();
-        processorRegistryBuilder.registerProcessor("mock_processor", ((registry) -> mock(Processor.Factory.class)));
-        ProcessorsRegistry processorRegistry = processorRegistryBuilder.build(mock(ScriptService.class), mock(ClusterService.class));
+        Map<String, Processor.Factory> registry =
+            Collections.singletonMap("mock_processor", (factories, config) -> processor);
         store = mock(PipelineStore.class);
         when(store.get(SIMULATED_PIPELINE_ID)).thenReturn(pipeline);
-        when(store.getProcessorRegistry()).thenReturn(processorRegistry);
+        when(store.getProcessorFactories()).thenReturn(registry);
     }
 
     public void testParseUsingPipelineStore() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
index 2ba7da84c14..f78269d87d2 100644
--- a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
+++ b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
@@ -33,6 +33,7 @@ import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
+import org.elasticsearch.ingest.IngestService;
 import org.elasticsearch.node.service.NodeService;
 import org.elasticsearch.rest.AbstractRestChannel;
 import org.elasticsearch.rest.BytesRestResponse;
@@ -45,6 +46,7 @@ import org.junit.Before;
 
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
+import java.util.Collections;
 import java.util.Map;
 
 public class HttpServerTests extends ESTestCase {
@@ -74,8 +76,9 @@ public class HttpServerTests extends ESTestCase {
 
         ClusterService clusterService = new ClusterService(Settings.EMPTY,
             new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null);
-        NodeService nodeService = new NodeService(Settings.EMPTY, null, null, null, null, null, null, null, null,
-            clusterService, null);
+        IngestService ingestService = new IngestService(settings, null, null, null, Collections.emptyList());
+        NodeService nodeService = new NodeService(Settings.EMPTY, null, null, null, null, null, null, null,
+            ingestService, clusterService, null);
         httpServer = new HttpServer(settings, httpServerTransport, restController, nodeService, circuitBreakerService);
         httpServer.start();
     }
diff --git a/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java b/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java
index fa78d5aa16c..eab5793d3f6 100644
--- a/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java
@@ -19,14 +19,6 @@
 
 package org.elasticsearch.ingest;
 
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.ingest.ProcessorsRegistry;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.threadpool.TestThreadPool;
-import org.junit.Before;
-
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -34,6 +26,12 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.sameInstance;
 import static org.mockito.Mockito.mock;
@@ -97,10 +95,8 @@ public class ConfigurationUtilsTests extends ESTestCase {
 
     public void testReadProcessors() throws Exception {
         Processor processor = mock(Processor.class);
-        ProcessorsRegistry.Builder builder = new ProcessorsRegistry.Builder();
-        builder.registerProcessor("test_processor", (registry) -> config -> processor);
-        ProcessorsRegistry registry = builder.build(mock(ScriptService.class), mock(ClusterService.class));
-
+        Map<String, Processor.Factory> registry =
+            Collections.singletonMap("test_processor", (factories, config) -> processor);
 
         List<Map<String, Map<String, Object>>> config = new ArrayList<>();
         Map<String, Object> emptyConfig = Collections.emptyMap();
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java
index cb7bd849a47..8510c41b3d3 100644
--- a/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java
@@ -46,7 +46,7 @@ public class PipelineFactoryTests extends ESTestCase {
         pipelineConfig.put(Pipeline.PROCESSORS_KEY,
                 Arrays.asList(Collections.singletonMap("test", processorConfig0), Collections.singletonMap("test", processorConfig1)));
         Pipeline.Factory factory = new Pipeline.Factory();
-        ProcessorsRegistry processorRegistry = createProcessorRegistry(Collections.singletonMap("test", new TestProcessor.Factory()));
+        Map<String, Processor.Factory> processorRegistry = Collections.singletonMap("test", new TestProcessor.Factory());
         Pipeline pipeline = factory.create("_id", pipelineConfig, processorRegistry);
         assertThat(pipeline.getId(), equalTo("_id"));
         assertThat(pipeline.getDescription(), equalTo("_description"));
@@ -62,7 +62,7 @@ public class PipelineFactoryTests extends ESTestCase {
         pipelineConfig.put(Pipeline.DESCRIPTION_KEY, "_description");
         Pipeline.Factory factory = new Pipeline.Factory();
         try {
-            factory.create("_id", pipelineConfig, createProcessorRegistry(Collections.emptyMap()));
+            factory.create("_id", pipelineConfig, Collections.emptyMap());
             fail("should fail, missing required [processors] field");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[processors] required property is missing"));
@@ -76,7 +76,7 @@ public class PipelineFactoryTests extends ESTestCase {
         pipelineConfig.put(Pipeline.PROCESSORS_KEY, Collections.singletonList(Collections.singletonMap("test", processorConfig)));
         pipelineConfig.put(Pipeline.ON_FAILURE_KEY, Collections.singletonList(Collections.singletonMap("test", processorConfig)));
         Pipeline.Factory factory = new Pipeline.Factory();
-        ProcessorsRegistry processorRegistry = createProcessorRegistry(Collections.singletonMap("test", new TestProcessor.Factory()));
+        Map<String, Processor.Factory> processorRegistry = Collections.singletonMap("test", new TestProcessor.Factory());
         Pipeline pipeline = factory.create("_id", pipelineConfig, processorRegistry);
         assertThat(pipeline.getId(), equalTo("_id"));
         assertThat(pipeline.getDescription(), equalTo("_description"));
@@ -93,7 +93,7 @@ public class PipelineFactoryTests extends ESTestCase {
         pipelineConfig.put(Pipeline.PROCESSORS_KEY, Collections.singletonList(Collections.singletonMap("test", processorConfig)));
         pipelineConfig.put(Pipeline.ON_FAILURE_KEY, Collections.emptyList());
         Pipeline.Factory factory = new Pipeline.Factory();
-        ProcessorsRegistry processorRegistry = createProcessorRegistry(Collections.singletonMap("test", new TestProcessor.Factory()));
+        Map<String, Processor.Factory> processorRegistry = Collections.singletonMap("test", new TestProcessor.Factory());
         Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create("_id", pipelineConfig, processorRegistry));
         assertThat(e.getMessage(), equalTo("pipeline [_id] cannot have an empty on_failure option defined"));
     }
@@ -105,7 +105,7 @@ public class PipelineFactoryTests extends ESTestCase {
         pipelineConfig.put(Pipeline.DESCRIPTION_KEY, "_description");
         pipelineConfig.put(Pipeline.PROCESSORS_KEY, Collections.singletonList(Collections.singletonMap("test", processorConfig)));
         Pipeline.Factory factory = new Pipeline.Factory();
-        ProcessorsRegistry processorRegistry = createProcessorRegistry(Collections.singletonMap("test", new TestProcessor.Factory()));
+        Map<String, Processor.Factory> processorRegistry = Collections.singletonMap("test", new TestProcessor.Factory());
         Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create("_id", pipelineConfig, processorRegistry));
         assertThat(e.getMessage(), equalTo("[on_failure] processors list cannot be empty"));
     }
@@ -114,7 +114,7 @@ public class PipelineFactoryTests extends ESTestCase {
         Map<String, Object> processorConfig = new HashMap<>();
         processorConfig.put("ignore_failure", true);
 
-        ProcessorsRegistry processorRegistry = createProcessorRegistry(Collections.singletonMap("test", new TestProcessor.Factory()));
+        Map<String, Processor.Factory> processorRegistry = Collections.singletonMap("test", new TestProcessor.Factory());
         Pipeline.Factory factory = new Pipeline.Factory();
         Map<String, Object> pipelineConfig = new HashMap<>();
         pipelineConfig.put(Pipeline.DESCRIPTION_KEY, "_description");
@@ -139,7 +139,7 @@ public class PipelineFactoryTests extends ESTestCase {
         pipelineConfig.put(Pipeline.DESCRIPTION_KEY, "_description");
         pipelineConfig.put(Pipeline.PROCESSORS_KEY, Collections.singletonList(Collections.singletonMap("test", processorConfig)));
         Pipeline.Factory factory = new Pipeline.Factory();
-        ProcessorsRegistry processorRegistry = createProcessorRegistry(Collections.singletonMap("test", new TestProcessor.Factory()));
+        Map<String, Processor.Factory> processorRegistry = Collections.singletonMap("test", new TestProcessor.Factory());
         Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create("_id", pipelineConfig, processorRegistry));
         assertThat(e.getMessage(), equalTo("processor [test] doesn't support one or more provided configuration parameters [unused]"));
     }
@@ -152,7 +152,7 @@ public class PipelineFactoryTests extends ESTestCase {
         pipelineConfig.put(Pipeline.DESCRIPTION_KEY, "_description");
         pipelineConfig.put(Pipeline.PROCESSORS_KEY, Collections.singletonList(Collections.singletonMap("test", processorConfig)));
         Pipeline.Factory factory = new Pipeline.Factory();
-        ProcessorsRegistry processorRegistry = createProcessorRegistry(Collections.singletonMap("test", new TestProcessor.Factory()));
+        Map<String, Processor.Factory> processorRegistry = Collections.singletonMap("test", new TestProcessor.Factory());
         Pipeline pipeline = factory.create("_id", pipelineConfig, processorRegistry);
         assertThat(pipeline.getId(), equalTo("_id"));
         assertThat(pipeline.getDescription(), equalTo("_description"));
@@ -169,12 +169,4 @@ public class PipelineFactoryTests extends ESTestCase {
         List<Processor> flattened = pipeline.flattenAllProcessors();
         assertThat(flattened.size(), equalTo(4));
     }
-
-    private ProcessorsRegistry createProcessorRegistry(Map<String, Processor.Factory> processorRegistry) {
-        ProcessorsRegistry.Builder builder = new ProcessorsRegistry.Builder();
-        for (Map.Entry<String, Processor.Factory> entry : processorRegistry.entrySet()) {
-            builder.registerProcessor(entry.getKey(), ((registry) -> entry.getValue()));
-        }
-        return builder.build(mock(ScriptService.class), mock(ClusterService.class));
-    }
 }
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
index 55ea4360ece..fd4c9038e0d 100644
--- a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
@@ -56,9 +56,8 @@ public class PipelineStoreTests extends ESTestCase {
 
     @Before
     public void init() throws Exception {
-        store = new PipelineStore(Settings.EMPTY);
-        ProcessorsRegistry.Builder registryBuilder = new ProcessorsRegistry.Builder();
-        registryBuilder.registerProcessor("set", (registry) -> config -> {
+        Map<String, Processor.Factory> processorFactories = new HashMap<>();
+        processorFactories.put("set", (factories, config) -> {
             String field = (String) config.remove("field");
             String value = (String) config.remove("value");
             return new Processor() {
@@ -78,7 +77,7 @@ public class PipelineStoreTests extends ESTestCase {
                 }
             };
         });
-        registryBuilder.registerProcessor("remove", (registry) -> config -> {
+        processorFactories.put("remove", (factories, config) -> {
             String field = (String) config.remove("field");
             return new Processor() {
                 @Override
@@ -97,7 +96,7 @@ public class PipelineStoreTests extends ESTestCase {
                 }
             };
         });
-        store.buildProcessorFactoryRegistry(registryBuilder, mock(ScriptService.class), mock(ClusterService.class));
+        store = new PipelineStore(Settings.EMPTY, processorFactories);
     }
 
     public void testUpdatePipelines() {
diff --git a/core/src/test/java/org/elasticsearch/ingest/ProcessorsRegistryTests.java b/core/src/test/java/org/elasticsearch/ingest/ProcessorsRegistryTests.java
deleted file mode 100644
index 41701102584..00000000000
--- a/core/src/test/java/org/elasticsearch/ingest/ProcessorsRegistryTests.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.ingest;
-
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.test.ESTestCase;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.sameInstance;
-import static org.mockito.Mockito.mock;
-
-public class ProcessorsRegistryTests extends ESTestCase {
-
-    public void testBuildProcessorRegistry() {
-        ProcessorsRegistry.Builder builder = new ProcessorsRegistry.Builder();
-        TestProcessor.Factory factory1 = new TestProcessor.Factory();
-        builder.registerProcessor("1", (registry) -> factory1);
-        TestProcessor.Factory factory2 = new TestProcessor.Factory();
-        builder.registerProcessor("2", (registry) -> factory2);
-        TestProcessor.Factory factory3 = new TestProcessor.Factory();
-        try {
-            builder.registerProcessor("1", (registry) -> factory3);
-            fail("addProcessor should have failed");
-        } catch(IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("Processor factory already registered for name [1]"));
-        }
-
-        ProcessorsRegistry registry = builder.build(mock(ScriptService.class), mock(ClusterService.class));
-        assertThat(registry.getProcessorFactories().size(), equalTo(2));
-        assertThat(registry.getProcessorFactory("1"), sameInstance(factory1));
-        assertThat(registry.getProcessorFactory("2"), sameInstance(factory2));
-    }
-}
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
index 45ef6835ff0..d491da729f1 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
@@ -23,7 +23,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.ProcessorsRegistry;
+import org.elasticsearch.ingest.Processor;
 
 import java.util.Map;
 
@@ -54,7 +54,7 @@ abstract class AbstractStringProcessor extends AbstractProcessor {
 
     protected abstract String process(String value);
 
-    static abstract class Factory<T extends AbstractStringProcessor> extends AbstractProcessorFactory<T> {
+    static abstract class Factory extends AbstractProcessorFactory {
         protected final String processorType;
 
         protected Factory(String processorType) {
@@ -62,11 +62,11 @@ abstract class AbstractStringProcessor extends AbstractProcessor {
         }
 
         @Override
-        public T doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
+        public AbstractStringProcessor doCreate(Map<String, Processor.Factory> registry, String processorTag, Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(processorType, processorTag, config, "field");
             return newProcessor(processorTag, field);
         }
 
-        protected abstract T newProcessor(String processorTag, String field);
+        protected abstract AbstractStringProcessor newProcessor(String processorTag, String field);
     }
 }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java
index 242e3411d3f..3b810b64d69 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java
@@ -23,7 +23,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.ProcessorsRegistry;
+import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.TemplateService;
 import org.elasticsearch.ingest.ValueSource;
 
@@ -65,7 +65,7 @@ public final class AppendProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<AppendProcessor> {
+    public static final class Factory extends AbstractProcessorFactory {
 
         private final TemplateService templateService;
 
@@ -74,7 +74,8 @@ public final class AppendProcessor extends AbstractProcessor {
         }
 
         @Override
-        public AppendProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
+        public AppendProcessor doCreate(Map<String, Processor.Factory> registry, String processorTag,
+                                        Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             Object value = ConfigurationUtils.readObject(TYPE, processorTag, config, "value");
             return new AppendProcessor(processorTag, templateService.compile(field), ValueSource.wrap(value, templateService));
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
index 4e35003aa88..c8c0bdcf73f 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
@@ -23,7 +23,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.ProcessorsRegistry;
+import org.elasticsearch.ingest.Processor;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -161,9 +161,10 @@ public final class ConvertProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<ConvertProcessor> {
+    public static final class Factory extends AbstractProcessorFactory {
         @Override
-        public ConvertProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
+        public ConvertProcessor doCreate(Map<String, Processor.Factory> registry, String processorTag,
+                                         Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String typeProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "type");
             String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java
index d89430c4bef..9e38f58d407 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java
@@ -19,17 +19,6 @@
 
 package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
-import org.elasticsearch.ingest.ConfigurationUtils;
-import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.ProcessorsRegistry;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.IllformedLocaleException;
@@ -38,6 +27,17 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.function.Function;
 
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.ConfigurationUtils;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+
 public final class DateIndexNameProcessor extends AbstractProcessor {
 
     public static final String TYPE = "date_index_name";
@@ -121,10 +121,11 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
         return dateFormats;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<DateIndexNameProcessor> {
+    public static final class Factory extends AbstractProcessorFactory {
 
         @Override
-        protected DateIndexNameProcessor doCreate(ProcessorsRegistry registry, String tag, Map<String, Object> config) throws Exception {
+        protected DateIndexNameProcessor doCreate(Map<String, Processor.Factory> registry, String tag,
+                                                  Map<String, Object> config) throws Exception {
             String localeString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "locale");
             String timezoneString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "timezone");
             DateTimeZone timezone = timezoneString == null ? DateTimeZone.UTC : DateTimeZone.forID(timezoneString);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
index 0b17123bcb0..c780401974c 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
@@ -24,7 +24,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.ProcessorsRegistry;
+import org.elasticsearch.ingest.Processor;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.ISODateTimeFormat;
@@ -109,10 +109,11 @@ public final class DateProcessor extends AbstractProcessor {
         return formats;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<DateProcessor> {
+    public static final class Factory extends AbstractProcessorFactory {
 
         @SuppressWarnings("unchecked")
-        public DateProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
+        public DateProcessor doCreate(Map<String, Processor.Factory> registry, String processorTag,
+                                      Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", DEFAULT_TARGET_FIELD);
             String timezoneString = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "timezone");
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java
index 7c806c08cdf..52e882afe16 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java
@@ -23,7 +23,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.ProcessorsRegistry;
+import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.TemplateService;
 
 import java.util.Map;
@@ -57,7 +57,7 @@ public final class FailProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<FailProcessor> {
+    public static final class Factory extends AbstractProcessorFactory {
 
         private final TemplateService templateService;
 
@@ -66,7 +66,8 @@ public final class FailProcessor extends AbstractProcessor {
         }
 
         @Override
-        public FailProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
+        public FailProcessor doCreate(Map<String, Processor.Factory> registry, String processorTag,
+                                      Map<String, Object> config) throws Exception {
             String message = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "message");
             return new FailProcessor(processorTag, templateService.compile(message));
         }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java
index 9ee244962ab..d5795a4aed0 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java
@@ -24,7 +24,7 @@ import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
-import org.elasticsearch.ingest.ProcessorsRegistry;
+import org.elasticsearch.ingest.Processor;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -83,12 +83,13 @@ public final class ForEachProcessor extends AbstractProcessor {
         return processors;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<ForEachProcessor> {
+    public static final class Factory extends AbstractProcessorFactory {
         @Override
-        protected ForEachProcessor doCreate(String tag, Map<String, Object> config) throws Exception {
+        protected ForEachProcessor doCreate(Map<String, Processor.Factory> factories, String tag,
+                                            Map<String, Object> config) throws Exception {
             String field = readStringProperty(TYPE, tag, config, "field");
             List<Map<String, Map<String, Object>>> processorConfigs = readList(TYPE, tag, config, "processors");
-            List<Processor> processors = ConfigurationUtils.readProcessorConfigs(processorConfigs, processorRegistry);
+            List<Processor> processors = ConfigurationUtils.readProcessorConfigs(processorConfigs, factories);
             return new ForEachProcessor(tag, field, Collections.unmodifiableList(processors));
         }
     }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
index 1c11cd13487..95c0b2ae179 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
@@ -23,7 +23,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.ProcessorsRegistry;
+import org.elasticsearch.ingest.Processor;
 
 import java.util.HashMap;
 import java.util.List;
@@ -115,7 +115,7 @@ public final class GrokProcessor extends AbstractProcessor {
         return combinedPattern;
     }
 
-    public final static class Factory extends AbstractProcessorFactory<GrokProcessor> {
+    public final static class Factory extends AbstractProcessorFactory {
 
         private final Map<String, String> builtinPatterns;
 
@@ -124,7 +124,8 @@ public final class GrokProcessor extends AbstractProcessor {
         }
 
         @Override
-        public GrokProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
+        public GrokProcessor doCreate(Map<String, Processor.Factory> registry, String processorTag,
+                                      Map<String, Object> config) throws Exception {
             String matchField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             List<String> matchPatterns = ConfigurationUtils.readList(TYPE, processorTag, config, "patterns");
             boolean traceMatch = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "trace_match", false);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java
index 124b1ded1ce..d5399f1edc0 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java
@@ -22,7 +22,7 @@ package org.elasticsearch.ingest.common;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.ProcessorsRegistry;
+import org.elasticsearch.ingest.Processor;
 
 import java.util.Map;
 import java.util.regex.Matcher;
@@ -79,9 +79,10 @@ public final class GsubProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<GsubProcessor> {
+    public static final class Factory extends AbstractProcessorFactory {
         @Override
-        public GsubProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
+        public GsubProcessor doCreate(Map<String, Processor.Factory> registry, String processorTag,
+                                      Map<String, Object> config) throws Exception {
             String field = readStringProperty(TYPE, processorTag, config, "field");
             String pattern = readStringProperty(TYPE, processorTag, config, "pattern");
             String replacement = readStringProperty(TYPE, processorTag, config, "replacement");
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java
index b0752d4fab9..e4963af2f9a 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java
@@ -23,7 +23,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.ProcessorsRegistry;
+import org.elasticsearch.ingest.Processor;
 
 import java.util.List;
 import java.util.Map;
@@ -71,9 +71,10 @@ public final class JoinProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public final static class Factory extends AbstractProcessorFactory<JoinProcessor> {
+    public final static class Factory extends AbstractProcessorFactory {
         @Override
-        public JoinProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
+        public JoinProcessor doCreate(Map<String, Processor.Factory> registry, String processorTag,
+                                      Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String separator = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "separator");
             return new JoinProcessor(processorTag, field, separator);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java
index 9f8ea7a5614..e7a8f3f3e6a 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java
@@ -44,7 +44,7 @@ public final class LowercaseProcessor extends AbstractStringProcessor {
         return TYPE;
     }
 
-    public final static class Factory extends AbstractStringProcessor.Factory<LowercaseProcessor> {
+    public final static class Factory extends AbstractStringProcessor.Factory {
 
         public Factory() {
             super(TYPE);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java
index cec88e65d95..cf28d7a68e3 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java
@@ -23,7 +23,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.ProcessorsRegistry;
+import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.TemplateService;
 
 import java.util.Map;
@@ -56,7 +56,7 @@ public final class RemoveProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<RemoveProcessor> {
+    public static final class Factory extends AbstractProcessorFactory {
 
         private final TemplateService templateService;
 
@@ -65,7 +65,8 @@ public final class RemoveProcessor extends AbstractProcessor {
         }
 
         @Override
-        public RemoveProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
+        public RemoveProcessor doCreate(Map<String, Processor.Factory> registry, String processorTag,
+                                        Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             return new RemoveProcessor(processorTag, templateService.compile(field));
         }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java
index f4c242379e5..97d273b6e95 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java
@@ -23,7 +23,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.ProcessorsRegistry;
+import org.elasticsearch.ingest.Processor;
 
 import java.util.Map;
 
@@ -76,9 +76,10 @@ public final class RenameProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<RenameProcessor> {
+    public static final class Factory extends AbstractProcessorFactory {
         @Override
-        public RenameProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
+        public RenameProcessor doCreate(Map<String, Processor.Factory> registry, String processorTag,
+                                        Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field");
             return new RenameProcessor(processorTag, field, targetField);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
index cf147e174aa..9a4b6347592 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
@@ -26,7 +26,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.ProcessorsRegistry;
+import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
@@ -78,7 +78,7 @@ public final class ScriptProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<ScriptProcessor> {
+    public static final class Factory extends AbstractProcessorFactory {
 
         private final ScriptService scriptService;
 
@@ -87,7 +87,8 @@ public final class ScriptProcessor extends AbstractProcessor {
         }
 
         @Override
-        public ScriptProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
+        public ScriptProcessor doCreate(Map<String, Processor.Factory> registry, String processorTag,
+                                        Map<String, Object> config) throws Exception {
             String field = readOptionalStringProperty(TYPE, processorTag, config, "field");
             String lang = readStringProperty(TYPE, processorTag, config, "lang");
             String inline = readOptionalStringProperty(TYPE, processorTag, config, "inline");
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java
index 61053fec7e4..6f574eabfe7 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java
@@ -23,7 +23,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.ProcessorsRegistry;
+import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.TemplateService;
 import org.elasticsearch.ingest.ValueSource;
 
@@ -76,7 +76,7 @@ public final class SetProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<SetProcessor> {
+    public static final class Factory extends AbstractProcessorFactory {
 
         private final TemplateService templateService;
 
@@ -85,7 +85,8 @@ public final class SetProcessor extends AbstractProcessor {
         }
 
         @Override
-        public SetProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
+        public SetProcessor doCreate(Map<String, Processor.Factory> registry, String processorTag,
+                                     Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             Object value = ConfigurationUtils.readObject(TYPE, processorTag, config, "value");
             boolean overrideEnabled = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "override", true);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java
index 152526a9f68..5533e5f5b12 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java
@@ -23,7 +23,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.ProcessorsRegistry;
+import org.elasticsearch.ingest.Processor;
 
 import java.util.Collections;
 import java.util.List;
@@ -112,10 +112,11 @@ public final class SortProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public final static class Factory extends AbstractProcessorFactory<SortProcessor> {
+    public final static class Factory extends AbstractProcessorFactory {
 
         @Override
-        public SortProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
+        public SortProcessor doCreate(Map<String, Processor.Factory> registry, String processorTag,
+                                      Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, FIELD);
             try {
                 SortOrder direction = SortOrder.fromString(
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java
index 92ce7a02341..3dc67365edc 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java
@@ -23,7 +23,7 @@ import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.ProcessorsRegistry;
+import org.elasticsearch.ingest.Processor;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -73,9 +73,10 @@ public final class SplitProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public static class Factory extends AbstractProcessorFactory<SplitProcessor> {
+    public static class Factory extends AbstractProcessorFactory {
         @Override
-        public SplitProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
+        public SplitProcessor doCreate(Map<String, Processor.Factory> registry, String processorTag,
+                                       Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             return new SplitProcessor(processorTag, field, ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "separator"));
         }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java
index a57a25125d6..e852f887da0 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java
@@ -41,7 +41,7 @@ public final class TrimProcessor extends AbstractStringProcessor {
         return TYPE;
     }
 
-    public static final class Factory extends AbstractStringProcessor.Factory<TrimProcessor> {
+    public static final class Factory extends AbstractStringProcessor.Factory {
 
         public Factory() {
             super(TYPE);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java
index a5c817352a1..5585a130eaf 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java
@@ -43,7 +43,7 @@ public final class UppercaseProcessor extends AbstractStringProcessor {
         return TYPE;
     }
 
-    public static final class Factory extends AbstractStringProcessor.Factory<UppercaseProcessor> {
+    public static final class Factory extends AbstractStringProcessor.Factory {
 
         public Factory() {
             super(TYPE);
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java
index 04e2dbf72a4..cc272d0b120 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java
@@ -87,12 +87,12 @@ public class DateIndexNameFactoryTests extends ESTestCase {
         DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
         config.put("date_rounding", "y");
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config));
         assertThat(e.getMessage(), Matchers.equalTo("[field] required property is missing"));
 
         config.clear();
         config.put("field", "_field");
-        e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config));
         assertThat(e.getMessage(), Matchers.equalTo("[date_rounding] required property is missing"));
     }
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java
index da2a96f368a..ae1058480b4 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java
@@ -45,7 +45,7 @@ public class FailProcessorFactoryTests extends ESTestCase {
         config.put("message", "error");
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        FailProcessor failProcessor = factory.create(null, config);
+        FailProcessor failProcessor = (FailProcessor)factory.create(null, config);
         assertThat(failProcessor.getTag(), equalTo(processorTag));
         assertThat(failProcessor.getMessage().execute(Collections.emptyMap()), equalTo("error"));
     }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java
index 8f8aa85ae59..898e178422f 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java
@@ -21,7 +21,7 @@ package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.ingest.Processor;
-import org.elasticsearch.ingest.ProcessorsRegistry;
+import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.TestProcessor;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
@@ -37,9 +37,8 @@ public class ForEachProcessorFactoryTests extends ESTestCase {
 
     public void testCreate() throws Exception {
         Processor processor = new TestProcessor(ingestDocument -> {});
-        Map<String, Processor.Factory> processors = new HashMap<>();
-        processors.put("_name", (r, c) -> processor);
-        ProcessorsRegistry registry = new ProcessorsRegistry(processors);
+        Map<String, Processor.Factory> registry = new HashMap<>();
+        registry.put("_name", (r, c) -> processor);
         ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory();
 
         Map<String, Object> config = new HashMap<>();
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java
index 4b862f8f92d..2858583c492 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java
@@ -50,7 +50,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap());
         Map<String, Object> config = new HashMap<>();
         config.put("patterns", Collections.singletonList("(?<foo>\\w+)"));
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config));
         assertThat(e.getMessage(), equalTo("[field] required property is missing"));
     }
 
@@ -58,7 +58,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap());
         Map<String, Object> config = new HashMap<>();
         config.put("field", "foo");
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config));
         assertThat(e.getMessage(), equalTo("[patterns] required property is missing"));
     }
 
@@ -67,7 +67,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "foo");
         config.put("patterns", Collections.emptyList());
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config));
         assertThat(e.getMessage(), equalTo("[patterns] List of patterns must not be empty"));
     }
 
@@ -89,7 +89,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("patterns", Collections.singletonList("["));
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config));
         assertThat(e.getMessage(), equalTo("[patterns] Invalid regex pattern found in: [[]. premature end of char-class"));
     }
 
@@ -99,7 +99,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("patterns", Collections.singletonList("%{MY_PATTERN:name}!"));
         config.put("pattern_definitions", Collections.singletonMap("MY_PATTERN", "["));
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config));
         assertThat(e.getMessage(),
             equalTo("[patterns] Invalid regex pattern found in: [%{MY_PATTERN:name}!]. premature end of char-class"));
     }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
index ed47894d4d9..121dbc3a565 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
@@ -25,6 +25,7 @@ import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -55,7 +56,7 @@ public class ScriptProcessorFactoryTests extends ESTestCase {
         configMap.put("lang", "mockscript");
 
         ElasticsearchException exception = expectThrows(ElasticsearchException.class,
-            () -> factory.doCreate(randomAsciiOfLength(10), configMap));
+            () -> factory.doCreate(null, randomAsciiOfLength(10), configMap));
 
         assertThat(exception.getMessage(), is("[null] Only one of [file], [id], or [inline] may be configured"));
     }
@@ -66,7 +67,7 @@ public class ScriptProcessorFactoryTests extends ESTestCase {
         configMap.put("lang", "mockscript");
 
         ElasticsearchException exception = expectThrows(ElasticsearchException.class,
-            () -> factory.doCreate(randomAsciiOfLength(10), configMap));
+            () -> factory.doCreate(null, randomAsciiOfLength(10), configMap));
 
         assertThat(exception.getMessage(), is("[null] Need [file], [id], or [inline] parameter to refer to scripts"));
     }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java
index 0ed7c3d7191..ccf93708e86 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java
@@ -37,7 +37,7 @@ public class SplitProcessorFactoryTests extends ESTestCase {
         config.put("separator", "\\.");
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        SplitProcessor splitProcessor = factory.create(null, config);
+        SplitProcessor splitProcessor = (SplitProcessor)factory.create(null, config);
         assertThat(splitProcessor.getTag(), equalTo(processorTag));
         assertThat(splitProcessor.getField(), equalTo("field1"));
         assertThat(splitProcessor.getSeparator(), equalTo("\\."));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java
index 13d45dc126b..c2bb21d6422 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java
@@ -84,7 +84,7 @@ public class SplitProcessorTests extends ESTestCase {
         Map<String, Object> splitConfig = new HashMap<>();
         splitConfig.put("field", "flags");
         splitConfig.put("separator", "\\|");
-        Processor splitProcessor = (new SplitProcessor.Factory()).create(splitConfig);
+        Processor splitProcessor = (new SplitProcessor.Factory()).create(null, splitConfig);
         Map<String, Object> source = new HashMap<>();
         source.put("flags", "new|hot|super|fun|interesting");
         IngestDocument ingestDocument = new IngestDocument(source, new HashMap<>());
diff --git a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
index 689adfa1acb..73bed16ec0f 100644
--- a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
+++ b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
@@ -27,7 +27,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.ProcessorsRegistry;
+import org.elasticsearch.ingest.Processor;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -151,12 +151,13 @@ public final class AttachmentProcessor extends AbstractProcessor {
         return indexedChars;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<AttachmentProcessor> {
+    public static final class Factory extends AbstractProcessorFactory {
 
         static final Set<Property> DEFAULT_PROPERTIES = EnumSet.allOf(Property.class);
 
         @Override
-        public AttachmentProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
+        public AttachmentProcessor doCreate(Map<String, Processor.Factory> registry, String processorTag,
+                                            Map<String, Object> config) throws Exception {
             String field = readStringProperty(TYPE, processorTag, config, "field");
             String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "attachment");
             List<String> properyNames = readOptionalList(TYPE, processorTag, config, "properties");
diff --git a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/IngestAttachmentPlugin.java b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/IngestAttachmentPlugin.java
index 3156fe381fd..eaba639255b 100644
--- a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/IngestAttachmentPlugin.java
+++ b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/IngestAttachmentPlugin.java
@@ -19,15 +19,21 @@
 
 package org.elasticsearch.ingest.attachment;
 
-import org.elasticsearch.node.NodeModule;
+import java.util.Collections;
+import java.util.Map;
+
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.TemplateService;
+import org.elasticsearch.plugins.IngestPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.ScriptService;
 
-import java.io.IOException;
+public class IngestAttachmentPlugin extends Plugin implements IngestPlugin {
 
-public class IngestAttachmentPlugin extends Plugin {
-
-    public void onModule(NodeModule nodeModule) throws IOException {
-        nodeModule.registerProcessor(AttachmentProcessor.TYPE,
-            (registry) -> new AttachmentProcessor.Factory());
+    @Override
+    public Map<String, Processor.Factory> getProcessors(
+        Environment env, ScriptService scriptService, TemplateService templateService) {
+        return Collections.singletonMap(AttachmentProcessor.TYPE, new AttachmentProcessor.Factory());
     }
 }
diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
index 9d417c3534c..4f1fcac40f5 100644
--- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
+++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
@@ -19,26 +19,6 @@
 
 package org.elasticsearch.ingest.geoip;
 
-import com.maxmind.geoip2.DatabaseReader;
-import com.maxmind.geoip2.exception.AddressNotFoundException;
-import com.maxmind.geoip2.model.CityResponse;
-import com.maxmind.geoip2.model.CountryResponse;
-import com.maxmind.geoip2.record.City;
-import com.maxmind.geoip2.record.Continent;
-import com.maxmind.geoip2.record.Country;
-import com.maxmind.geoip2.record.Location;
-import com.maxmind.geoip2.record.Subdivision;
-import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.SpecialPermission;
-import org.elasticsearch.common.network.InetAddresses;
-import org.elasticsearch.common.network.NetworkAddress;
-import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
-import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.ProcessorsRegistry;
-
-import java.io.Closeable;
 import java.io.IOException;
 import java.net.InetAddress;
 import java.security.AccessController;
@@ -52,6 +32,24 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 
+import com.maxmind.geoip2.DatabaseReader;
+import com.maxmind.geoip2.exception.AddressNotFoundException;
+import com.maxmind.geoip2.model.CityResponse;
+import com.maxmind.geoip2.model.CountryResponse;
+import com.maxmind.geoip2.record.City;
+import com.maxmind.geoip2.record.Continent;
+import com.maxmind.geoip2.record.Country;
+import com.maxmind.geoip2.record.Location;
+import com.maxmind.geoip2.record.Subdivision;
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.SpecialPermission;
+import org.elasticsearch.common.network.InetAddresses;
+import org.elasticsearch.common.network.NetworkAddress;
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+
 import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
 import static org.elasticsearch.ingest.ConfigurationUtils.readOptionalList;
 import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
@@ -218,7 +216,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
         return geoData;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<GeoIpProcessor> {
+    public static final class Factory extends AbstractProcessorFactory {
         static final Set<Property> DEFAULT_CITY_PROPERTIES = EnumSet.of(
             Property.CONTINENT_NAME, Property.COUNTRY_ISO_CODE, Property.REGION_NAME,
             Property.CITY_NAME, Property.LOCATION
@@ -232,7 +230,8 @@ public final class GeoIpProcessor extends AbstractProcessor {
         }
 
         @Override
-        public GeoIpProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
+        public GeoIpProcessor doCreate(Map<String, Processor.Factory> registry, String processorTag,
+                                       Map<String, Object> config) throws Exception {
             String ipField = readStringProperty(TYPE, processorTag, config, "field");
             String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "geoip");
             String databaseFile = readStringProperty(TYPE, processorTag, config, "database_file", "GeoLite2-City.mmdb.gz");
@@ -240,7 +239,8 @@ public final class GeoIpProcessor extends AbstractProcessor {
 
             DatabaseReader databaseReader = databaseReaders.get(databaseFile);
             if (databaseReader == null) {
-                throw newConfigurationException(TYPE, processorTag, "database_file", "database file [" + databaseFile + "] doesn't exist");
+                throw newConfigurationException(TYPE, processorTag,
+                    "database_file", "database file [" + databaseFile + "] doesn't exist");
             }
 
             String databaseType = databaseReader.getMetadata().getDatabaseType();
diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
index 1b1753439d7..fb83f595e4c 100644
--- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
+++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
@@ -49,7 +49,7 @@ public class IngestGeoIpPlugin extends Plugin implements IngestPlugin, Closeable
 
     @Override
     public synchronized Map<String, Processor.Factory> getProcessors(
-        Environment env, ClusterService clusterService, ScriptService scriptService, TemplateService templateService) {
+        Environment env, ScriptService scriptService, TemplateService templateService) {
         if (databaseReaders != null) {
             throw new IllegalStateException("called onModule twice for geoip plugin!!");
         }
diff --git a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
index 0d9309a3948..d2de3b821ca 100644
--- a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
+++ b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
@@ -78,7 +78,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
 
-        GeoIpProcessor processor = factory.create(null, config);
+        GeoIpProcessor processor = (GeoIpProcessor)factory.create(null, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("geoip"));
@@ -96,7 +96,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         String processorTag = randomAsciiOfLength(10);
         config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
 
-        GeoIpProcessor processor = factory.create(null, config);
+        GeoIpProcessor processor = (GeoIpProcessor)factory.create(null, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("geoip"));
@@ -109,7 +109,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("target_field", "_field");
-        GeoIpProcessor processor = factory.create(null, config);
+        GeoIpProcessor processor = (GeoIpProcessor)factory.create(null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("_field"));
     }
@@ -119,7 +119,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("database_file", "GeoLite2-Country.mmdb.gz");
-        GeoIpProcessor processor = factory.create(null, config);
+        GeoIpProcessor processor = (GeoIpProcessor)factory.create(null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("geoip"));
         assertThat(processor.getDbReader().getMetadata().getDatabaseType(), equalTo("GeoLite2-Country"));
@@ -171,7 +171,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("properties", fieldNames);
-        GeoIpProcessor processor = factory.create(null, config);
+        GeoIpProcessor processor = (GeoIpProcessor)factory.create(null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getProperties(), equalTo(properties));
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java b/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java
index b32a2eab991..9661fc53dea 100644
--- a/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java
+++ b/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java
@@ -19,22 +19,27 @@
 
 package org.elasticsearch.ingest;
 
-import org.elasticsearch.node.NodeModule;
+import java.util.Collections;
+import java.util.Map;
+
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.plugins.IngestPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.ScriptService;
 
 /**
  * Adds an ingest processor to be used in tests.
  */
-public class IngestTestPlugin extends Plugin {
-
-    public void onModule(NodeModule nodeModule) {
-        nodeModule.registerProcessor("test", (registry) -> config ->
-                new TestProcessor("id", "test", doc -> {
-                    doc.setFieldValue("processed", true);
-                    if (doc.hasField("fail") && doc.getFieldValue("fail", Boolean.class)) {
-                        throw new IllegalArgumentException("test processor failed");
-                    }
-                })
-        );
+public class IngestTestPlugin extends Plugin implements IngestPlugin {
+    @Override
+    public Map<String, Processor.Factory> getProcessors(
+        Environment env, ScriptService scriptService, TemplateService templateService) {
+        return Collections.singletonMap("test", (factories, config) ->
+            new TestProcessor("id", "test", doc -> {
+                doc.setFieldValue("processed", true);
+                if (doc.hasField("fail") && doc.getFieldValue("fail", Boolean.class)) {
+                    throw new IllegalArgumentException("test processor failed");
+                }
+            }));
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java b/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java
index 48381b87142..51138985410 100644
--- a/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java
+++ b/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java
@@ -64,9 +64,9 @@ public class TestProcessor implements Processor {
         return invokedCounter.get();
     }
 
-    public static final class Factory extends AbstractProcessorFactory<TestProcessor> {
+    public static final class Factory extends AbstractProcessorFactory {
         @Override
-        public TestProcessor doCreate(ProcessorsRegistry registry, String processorTag, Map<String, Object> config) throws Exception {
+        public TestProcessor doCreate(Map<String, Processor.Factory> registry, String processorTag, Map<String, Object> config) throws Exception {
             return new TestProcessor(processorTag, "test-processor", ingestDocument -> {});
         }
     }

From 299c6fcc63d6172c6d50684840943abeda51c06b Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Thu, 30 Jun 2016 11:05:54 +0200
Subject: [PATCH 1132/1311] test: use the reader from the searcher
 (newSearcher(...) method may change the reader) instead of the reader we
 create in the test

Closes #19151
---
 .../test/java/org/elasticsearch/common/lucene/LuceneTests.java | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java b/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java
index f4260626160..33a0b855a7a 100644
--- a/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java
+++ b/core/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java
@@ -357,7 +357,6 @@ public class LuceneTests extends ESTestCase {
         dir.close();
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/19151")
     public void testAsSequentialAccessBits() throws Exception {
         Directory dir = newDirectory();
         IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new KeywordAnalyzer()));
@@ -378,7 +377,7 @@ public class LuceneTests extends ESTestCase {
             IndexSearcher searcher = newSearcher(reader);
             Weight termWeight = new TermQuery(new Term("foo", "bar")).createWeight(searcher, false);
             assertEquals(1, reader.leaves().size());
-            LeafReaderContext leafReaderContext = reader.leaves().get(0);
+            LeafReaderContext leafReaderContext = searcher.getIndexReader().leaves().get(0);
             Bits bits = Lucene.asSequentialAccessBits(leafReaderContext.reader().maxDoc(), termWeight.scorer(leafReaderContext));
 
             expectThrows(IndexOutOfBoundsException.class, () -> bits.get(-1));

From 66e3b15d21e183d652cce79c38301feedf7eb0a3 Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Thu, 30 Jun 2016 11:12:20 +0200
Subject: [PATCH 1133/1311] Fix NPE when GCE region is empty

When GCE region is empty we get back from the API something like:

```
{
  "id": "dummy"
}
```

instead of:

```
{
  "id": "dummy",
  "items":[ ]
}
```

This generates a NPE when we aggregate all the lists into a single one.

Closes #16967.
---
 .../cloud/gce/GceComputeServiceImpl.java      |  2 +-
 .../discovery/gce/GceDiscoveryTests.java      | 13 +++++++
 .../zones/europe-west1-b/instances            | 36 +++++++++++++++++++
 .../zones/us-central1-a/instances             |  3 ++
 4 files changed, 53 insertions(+), 1 deletion(-)
 create mode 100644 plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/emptyregion16967/zones/europe-west1-b/instances
 create mode 100644 plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/emptyregion16967/zones/us-central1-a/instances

diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java
index 85e0910736f..b27c9e409fe 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java
@@ -97,7 +97,7 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent<GceCompute
                     }
                 });
                 // assist type inference
-                return instanceList.isEmpty() ? Collections.<Instance>emptyList() : instanceList.getItems();
+                return instanceList.isEmpty()  || instanceList.getItems() == null ? Collections.<Instance>emptyList() : instanceList.getItems();
             } catch (PrivilegedActionException e) {
                 logger.warn("Problem fetching instance list for zone {}", e, zoneId);
                 logger.debug("Full exception:", e);
diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java
index 07b2ef774b4..a256b660d5e 100644
--- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java
+++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java
@@ -254,4 +254,17 @@ public class GceDiscoveryTests extends ESTestCase {
             assertThat(expected.getMessage(), containsString("one or more gce discovery settings are missing."));
         }
     }
+
+    /**
+     * For issue https://github.com/elastic/elasticsearch/issues/16967
+     */
+    public void testEmptyRegion16967() {
+        Settings nodeSettings = Settings.builder()
+            .put(GceComputeService.PROJECT_SETTING.getKey(), projectName)
+            .putArray(GceComputeService.ZONE_SETTING.getKey(), "europe-west1-b", "us-central1-a")
+            .build();
+        mock = new GceComputeServiceMock(nodeSettings, networkService);
+        List<DiscoveryNode> discoveryNodes = buildDynamicNodes(mock, nodeSettings);
+        assertThat(discoveryNodes, hasSize(1));
+    }
 }
diff --git a/plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/emptyregion16967/zones/europe-west1-b/instances b/plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/emptyregion16967/zones/europe-west1-b/instances
new file mode 100644
index 00000000000..049e0e1e1b1
--- /dev/null
+++ b/plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/emptyregion16967/zones/europe-west1-b/instances
@@ -0,0 +1,36 @@
+{
+  "id": "dummy",
+  "items":[
+    {
+      "description": "ES Node 1",
+      "id": "9309873766428965105",
+      "kind": "compute#instance",
+      "machineType": "n1-standard-1",
+      "name": "test1",
+      "networkInterfaces": [
+        {
+          "accessConfigs": [
+            {
+              "kind": "compute#accessConfig",
+              "name": "External NAT",
+              "natIP": "104.155.13.147",
+              "type": "ONE_TO_ONE_NAT"
+            }
+          ],
+          "name": "nic0",
+          "network": "default",
+          "networkIP": "10.240.79.59"
+        }
+      ],
+      "status": "RUNNING",
+      "tags": {
+        "fingerprint": "xA6QJb-rGtg=",
+        "items": [
+          "elasticsearch",
+          "dev"
+        ]
+      },
+      "zone": "europe-west1-b"
+    }
+  ]
+}
diff --git a/plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/emptyregion16967/zones/us-central1-a/instances b/plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/emptyregion16967/zones/us-central1-a/instances
new file mode 100644
index 00000000000..989b7507fe8
--- /dev/null
+++ b/plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/emptyregion16967/zones/us-central1-a/instances
@@ -0,0 +1,3 @@
+{
+  "id": "dummy"
+}

From f9d22b3598d61ddf123e185cc384c5540cd8a213 Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Thu, 30 Jun 2016 11:32:39 +0200
Subject: [PATCH 1134/1311] Add more javadoc and rename test

---
 .../org/elasticsearch/discovery/gce/GceDiscoveryTests.java  | 6 ++++--
 .../zones/europe-west1-b/instances                          | 0
 .../zones/us-central1-a/instances                           | 0
 3 files changed, 4 insertions(+), 2 deletions(-)
 rename plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/{emptyregion16967 => noregionreturnsemptylist}/zones/europe-west1-b/instances (100%)
 rename plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/{emptyregion16967 => noregionreturnsemptylist}/zones/us-central1-a/instances (100%)

diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java
index a256b660d5e..92eb12a99b2 100644
--- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java
+++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/GceDiscoveryTests.java
@@ -256,9 +256,11 @@ public class GceDiscoveryTests extends ESTestCase {
     }
 
     /**
-     * For issue https://github.com/elastic/elasticsearch/issues/16967
+     * For issue https://github.com/elastic/elasticsearch/issues/16967:
+     * When using multiple regions and one of them has no instance at all, this
+     * was producing a NPE as a result.
      */
-    public void testEmptyRegion16967() {
+    public void testNoRegionReturnsEmptyList() {
         Settings nodeSettings = Settings.builder()
             .put(GceComputeService.PROJECT_SETTING.getKey(), projectName)
             .putArray(GceComputeService.ZONE_SETTING.getKey(), "europe-west1-b", "us-central1-a")
diff --git a/plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/emptyregion16967/zones/europe-west1-b/instances b/plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/noregionreturnsemptylist/zones/europe-west1-b/instances
similarity index 100%
rename from plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/emptyregion16967/zones/europe-west1-b/instances
rename to plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/noregionreturnsemptylist/zones/europe-west1-b/instances
diff --git a/plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/emptyregion16967/zones/us-central1-a/instances b/plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/noregionreturnsemptylist/zones/us-central1-a/instances
similarity index 100%
rename from plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/emptyregion16967/zones/us-central1-a/instances
rename to plugins/discovery-gce/src/test/resources/org/elasticsearch/discovery/gce/compute/v1/projects/noregionreturnsemptylist/zones/us-central1-a/instances

From e4f265eb3a9cc3a1f160ef06670c4f3bc63a4bc2 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Thu, 30 Jun 2016 02:28:26 -0700
Subject: [PATCH 1135/1311] Ingest: Remove generics from Processor.Factory

The factory for ingest processor is generic, but that is only for the
return type of the create mehtod. However, the actual consumer of the
factories only cares about Processor, so generics are not needed.

This change removes the generic type from the factory. It also removes
AbstractProcessorFactory which only existed in order pull the optional
tag from config. This functionality is moved to the caller of the
factories in ConfigurationUtil, and the create method now takes the tag.
This allows the covariant return of the implementation to work with
tests not needing casts.
---
 .../ingest/SimulateProcessorResult.java       |  4 +-
 .../ingest/AbstractProcessorFactory.java      | 38 -------------------
 .../ingest/ConfigurationUtils.java            |  6 ++-
 .../org/elasticsearch/ingest/Processor.java   |  7 +++-
 .../ingest/ProcessorsRegistry.java            | 10 ++---
 .../org/elasticsearch/node/NodeModule.java    |  2 +-
 .../ingest/ConfigurationUtilsTests.java       |  2 +-
 .../ingest/PipelineFactoryTests.java          |  2 +-
 .../ingest/PipelineStoreTests.java            |  4 +-
 .../common/AbstractStringProcessor.java       |  8 ++--
 .../ingest/common/AppendProcessor.java        |  7 ++--
 .../ingest/common/ConvertProcessor.java       |  6 +--
 .../ingest/common/DateIndexNameProcessor.java |  6 +--
 .../ingest/common/DateProcessor.java          |  6 +--
 .../ingest/common/FailProcessor.java          |  6 +--
 .../ingest/common/ForEachProcessor.java       |  5 +--
 .../ingest/common/GrokProcessor.java          |  6 +--
 .../ingest/common/GsubProcessor.java          |  6 +--
 .../ingest/common/JoinProcessor.java          |  6 +--
 .../ingest/common/LowercaseProcessor.java     |  2 +-
 .../ingest/common/RemoveProcessor.java        |  6 +--
 .../ingest/common/RenameProcessor.java        |  6 +--
 .../ingest/common/ScriptProcessor.java        |  6 +--
 .../ingest/common/SetProcessor.java           |  6 +--
 .../ingest/common/SortProcessor.java          |  6 +--
 .../ingest/common/SplitProcessor.java         |  6 +--
 .../ingest/common/TrimProcessor.java          |  2 +-
 .../ingest/common/UppercaseProcessor.java     |  2 +-
 .../common/AppendProcessorFactoryTests.java   | 10 ++---
 .../common/ConvertProcessorFactoryTests.java  | 13 +++----
 .../common/DateIndexNameFactoryTests.java     | 14 +++----
 .../common/DateProcessorFactoryTests.java     | 22 +++++------
 .../common/FailProcessorFactoryTests.java     |  6 +--
 .../common/ForEachProcessorFactoryTests.java  |  8 ++--
 .../common/GrokProcessorFactoryTests.java     | 16 ++++----
 .../common/GsubProcessorFactoryTests.java     | 12 +++---
 .../common/JoinProcessorFactoryTests.java     |  8 ++--
 .../LowercaseProcessorFactoryTests.java       |  6 +--
 .../common/RemoveProcessorFactoryTests.java   |  6 +--
 .../common/RenameProcessorFactoryTests.java   |  8 ++--
 .../common/ScriptProcessorFactoryTests.java   |  4 +-
 .../common/SetProcessorFactoryTests.java      | 13 +++----
 .../common/SplitProcessorFactoryTests.java    |  8 ++--
 .../ingest/common/SplitProcessorTests.java    |  2 +-
 .../common/TrimProcessorFactoryTests.java     |  6 +--
 .../UppercaseProcessorFactoryTests.java       |  6 +--
 .../attachment/AttachmentProcessor.java       |  6 +--
 .../AttachmentProcessorFactoryTests.java      | 16 ++++----
 .../ingest/geoip/GeoIpProcessor.java          |  6 +--
 .../geoip/GeoIpProcessorFactoryTests.java     | 23 +++++------
 .../ingest/IngestTestPlugin.java              |  2 +-
 .../elasticsearch/ingest/TestProcessor.java   |  4 +-
 52 files changed, 167 insertions(+), 237 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/ingest/AbstractProcessorFactory.java

diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
index 6924550d79b..ce98767bcc8 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
@@ -24,7 +24,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
 
 import java.io.IOException;
@@ -91,7 +91,7 @@ public class SimulateProcessorResult implements Writeable, ToXContent {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         if (processorTag != null) {
-            builder.field(AbstractProcessorFactory.TAG_KEY, processorTag);
+            builder.field(ConfigurationUtils.TAG_KEY, processorTag);
         }
         if (failure == null) {
             ingestDocument.toXContent(builder, params);
diff --git a/core/src/main/java/org/elasticsearch/ingest/AbstractProcessorFactory.java b/core/src/main/java/org/elasticsearch/ingest/AbstractProcessorFactory.java
deleted file mode 100644
index 0fb50228793..00000000000
--- a/core/src/main/java/org/elasticsearch/ingest/AbstractProcessorFactory.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.ingest;
-
-import java.util.Map;
-
-/**
- * A processor implementation may modify the data belonging to a document.
- * Whether changes are made and what exactly is modified is up to the implementation.
- */
-public abstract class AbstractProcessorFactory<P extends Processor> implements Processor.Factory<P> {
-    public static final String TAG_KEY = "tag";
-
-    @Override
-    public P create(Map<String, Object> config) throws Exception {
-        String tag = ConfigurationUtils.readOptionalStringProperty(null, null, config, TAG_KEY);
-        return doCreate(tag, config);
-    }
-
-    protected abstract P doCreate(String tag, Map<String, Object> config) throws Exception;
-}
diff --git a/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java b/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
index 623c4f315a4..54fbc4a0237 100644
--- a/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
+++ b/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
@@ -29,6 +29,8 @@ import java.util.Map;
 
 public final class ConfigurationUtils {
 
+    public static final String TAG_KEY = "tag";
+
     private ConfigurationUtils() {
     }
 
@@ -255,8 +257,8 @@ public final class ConfigurationUtils {
                 ConfigurationUtils.readOptionalList(null, null, config, Pipeline.ON_FAILURE_KEY);
 
             List<Processor> onFailureProcessors = readProcessorConfigs(onFailureProcessorConfigs, processorRegistry);
-            Processor processor;
-            processor = factory.create(config);
+            String tag = ConfigurationUtils.readOptionalStringProperty(null, null, config, TAG_KEY);
+            Processor processor = factory.create(tag, config);
 
             if (onFailureProcessorConfigs != null && onFailureProcessors.isEmpty()) {
                 throw newConfigurationException(processor.getType(), processor.getTag(), Pipeline.ON_FAILURE_KEY,
diff --git a/core/src/main/java/org/elasticsearch/ingest/Processor.java b/core/src/main/java/org/elasticsearch/ingest/Processor.java
index 92c18464e94..d4a5462c1c9 100644
--- a/core/src/main/java/org/elasticsearch/ingest/Processor.java
+++ b/core/src/main/java/org/elasticsearch/ingest/Processor.java
@@ -45,14 +45,17 @@ public interface Processor {
     /**
      * A factory that knows how to construct a processor based on a map of maps.
      */
-    interface Factory<P extends Processor> {
+    interface Factory {
 
         /**
          * Creates a processor based on the specified map of maps config.
          *
+         * @param tag The tag for the processor
+         * @param config Configuration for the processor to create
+         *
          * Implementations are responsible for removing the used keys, so that after creating a pipeline ingest can
          * verify if all configurations settings have been used.
          */
-        P create(Map<String, Object> config) throws Exception;
+        Processor create(String tag, Map<String, Object> config) throws Exception;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java b/core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java
index dd16898aea2..461247b6d09 100644
--- a/core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java
+++ b/core/src/main/java/org/elasticsearch/ingest/ProcessorsRegistry.java
@@ -37,12 +37,12 @@ public final class ProcessorsRegistry {
     private final ClusterService clusterService;
 
     private ProcessorsRegistry(ScriptService scriptService, ClusterService clusterService,
-                               Map<String, Function<ProcessorsRegistry, Processor.Factory<?>>> providers) {
+                               Map<String, Function<ProcessorsRegistry, Processor.Factory>> providers) {
         this.templateService = new InternalTemplateService(scriptService);
         this.scriptService = scriptService;
         this.clusterService = clusterService;
         Map<String, Processor.Factory> processorFactories = new HashMap<>();
-        for (Map.Entry<String, Function<ProcessorsRegistry, Processor.Factory<?>>> entry : providers.entrySet()) {
+        for (Map.Entry<String, Function<ProcessorsRegistry, Processor.Factory>> entry : providers.entrySet()) {
             processorFactories.put(entry.getKey(), entry.getValue().apply(this));
         }
         this.processorFactories = Collections.unmodifiableMap(processorFactories);
@@ -71,13 +71,13 @@ public final class ProcessorsRegistry {
 
     public static final class Builder {
 
-        private final Map<String, Function<ProcessorsRegistry, Processor.Factory<?>>> providers = new HashMap<>();
+        private final Map<String, Function<ProcessorsRegistry, Processor.Factory>> providers = new HashMap<>();
 
         /**
          * Adds a processor factory under a specific name.
          */
-        public void registerProcessor(String name, Function<ProcessorsRegistry, Processor.Factory<?>> provider) {
-            Function<ProcessorsRegistry, Processor.Factory<?>> previous = this.providers.putIfAbsent(name, provider);
+        public void registerProcessor(String name, Function<ProcessorsRegistry, Processor.Factory> provider) {
+            Function<ProcessorsRegistry, Processor.Factory> previous = this.providers.putIfAbsent(name, provider);
             if (previous != null) {
                 throw new IllegalArgumentException("Processor factory already registered for name [" + name + "]");
             }
diff --git a/core/src/main/java/org/elasticsearch/node/NodeModule.java b/core/src/main/java/org/elasticsearch/node/NodeModule.java
index 6d8be732dd2..49de57a87b6 100644
--- a/core/src/main/java/org/elasticsearch/node/NodeModule.java
+++ b/core/src/main/java/org/elasticsearch/node/NodeModule.java
@@ -62,7 +62,7 @@ public class NodeModule extends AbstractModule {
     /**
      * Adds a processor factory under a specific type name.
      */
-    public void registerProcessor(String type, Function<ProcessorsRegistry, Processor.Factory<?>> provider) {
+    public void registerProcessor(String type, Function<ProcessorsRegistry, Processor.Factory> provider) {
         processorsRegistryBuilder.registerProcessor(type, provider);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java b/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java
index fa78d5aa16c..82886628c85 100644
--- a/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java
@@ -98,7 +98,7 @@ public class ConfigurationUtilsTests extends ESTestCase {
     public void testReadProcessors() throws Exception {
         Processor processor = mock(Processor.class);
         ProcessorsRegistry.Builder builder = new ProcessorsRegistry.Builder();
-        builder.registerProcessor("test_processor", (registry) -> config -> processor);
+        builder.registerProcessor("test_processor", (registry) -> (tag, config) -> processor);
         ProcessorsRegistry registry = builder.build(mock(ScriptService.class), mock(ClusterService.class));
 
 
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java
index cb7bd849a47..8fe7ddd84cb 100644
--- a/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java
@@ -40,7 +40,7 @@ public class PipelineFactoryTests extends ESTestCase {
     public void testCreate() throws Exception {
         Map<String, Object> processorConfig0 = new HashMap<>();
         Map<String, Object> processorConfig1 = new HashMap<>();
-        processorConfig0.put(AbstractProcessorFactory.TAG_KEY, "first-processor");
+        processorConfig0.put(ConfigurationUtils.TAG_KEY, "first-processor");
         Map<String, Object> pipelineConfig = new HashMap<>();
         pipelineConfig.put(Pipeline.DESCRIPTION_KEY, "_description");
         pipelineConfig.put(Pipeline.PROCESSORS_KEY,
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
index 55ea4360ece..ec41eda47b0 100644
--- a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
@@ -58,7 +58,7 @@ public class PipelineStoreTests extends ESTestCase {
     public void init() throws Exception {
         store = new PipelineStore(Settings.EMPTY);
         ProcessorsRegistry.Builder registryBuilder = new ProcessorsRegistry.Builder();
-        registryBuilder.registerProcessor("set", (registry) -> config -> {
+        registryBuilder.registerProcessor("set", (registry) -> (tag, config) -> {
             String field = (String) config.remove("field");
             String value = (String) config.remove("value");
             return new Processor() {
@@ -78,7 +78,7 @@ public class PipelineStoreTests extends ESTestCase {
                 }
             };
         });
-        registryBuilder.registerProcessor("remove", (registry) -> config -> {
+        registryBuilder.registerProcessor("remove", (registry) -> (tag, config) -> {
             String field = (String) config.remove("field");
             return new Processor() {
                 @Override
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
index d9ffd34cac1..f35f3fec534 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
@@ -20,9 +20,9 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
 
 import java.util.Map;
 
@@ -53,7 +53,7 @@ abstract class AbstractStringProcessor extends AbstractProcessor {
 
     protected abstract String process(String value);
 
-    static abstract class Factory<T extends AbstractStringProcessor> extends AbstractProcessorFactory<T> {
+    static abstract class Factory implements Processor.Factory {
         protected final String processorType;
 
         protected Factory(String processorType) {
@@ -61,11 +61,11 @@ abstract class AbstractStringProcessor extends AbstractProcessor {
         }
 
         @Override
-        public T doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public AbstractStringProcessor create(String processorTag, Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(processorType, processorTag, config, "field");
             return newProcessor(processorTag, field);
         }
 
-        protected abstract T newProcessor(String processorTag, String field);
+        protected abstract AbstractStringProcessor newProcessor(String processorTag, String field);
     }
 }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java
index af163c3c187..26b0e66a63c 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java
@@ -20,9 +20,10 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.TemplateService;
 import org.elasticsearch.ingest.ValueSource;
 
@@ -64,7 +65,7 @@ public final class AppendProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<AppendProcessor> {
+    public static final class Factory implements Processor.Factory {
 
         private final TemplateService templateService;
 
@@ -73,7 +74,7 @@ public final class AppendProcessor extends AbstractProcessor {
         }
 
         @Override
-        public AppendProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public AppendProcessor create(String processorTag, Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             Object value = ConfigurationUtils.readObject(TYPE, processorTag, config, "value");
             return new AppendProcessor(processorTag, templateService.compile(field), ValueSource.wrap(value, templateService));
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
index 015c56c72c3..558bdde96a2 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
@@ -20,9 +20,9 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -160,9 +160,9 @@ public final class ConvertProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<ConvertProcessor> {
+    public static final class Factory implements Processor.Factory {
         @Override
-        public ConvertProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public ConvertProcessor create(String processorTag, Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String typeProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "type");
             String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java
index a94d4d048a8..c750c84c576 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java
@@ -21,9 +21,9 @@ package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.DateTimeFormat;
@@ -120,10 +120,10 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
         return dateFormats;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<DateIndexNameProcessor> {
+    public static final class Factory implements Processor.Factory {
 
         @Override
-        protected DateIndexNameProcessor doCreate(String tag, Map<String, Object> config) throws Exception {
+        public DateIndexNameProcessor create(String tag, Map<String, Object> config) throws Exception {
             String localeString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "locale");
             String timezoneString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "timezone");
             DateTimeZone timezone = timezoneString == null ? DateTimeZone.UTC : DateTimeZone.forID(timezoneString);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
index b82b9c8b76c..e61ed513114 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
@@ -21,9 +21,9 @@ package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.ISODateTimeFormat;
@@ -108,10 +108,10 @@ public final class DateProcessor extends AbstractProcessor {
         return formats;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<DateProcessor> {
+    public static final class Factory implements Processor.Factory {
 
         @SuppressWarnings("unchecked")
-        public DateProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public DateProcessor create(String processorTag, Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", DEFAULT_TARGET_FIELD);
             String timezoneString = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "timezone");
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java
index 6c434d85d5a..a24322f556f 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java
@@ -20,9 +20,9 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.TemplateService;
 
 import java.util.Map;
@@ -56,7 +56,7 @@ public final class FailProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<FailProcessor> {
+    public static final class Factory implements Processor.Factory {
 
         private final TemplateService templateService;
 
@@ -65,7 +65,7 @@ public final class FailProcessor extends AbstractProcessor {
         }
 
         @Override
-        public FailProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public FailProcessor create(String processorTag, Map<String, Object> config) throws Exception {
             String message = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "message");
             return new FailProcessor(processorTag, templateService.compile(message));
         }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java
index b6d14d1b8c5..05be47633ba 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
@@ -83,7 +82,7 @@ public final class ForEachProcessor extends AbstractProcessor {
         return processors;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<ForEachProcessor> {
+    public static final class Factory implements Processor.Factory {
 
         private final ProcessorsRegistry processorRegistry;
 
@@ -92,7 +91,7 @@ public final class ForEachProcessor extends AbstractProcessor {
         }
 
         @Override
-        protected ForEachProcessor doCreate(String tag, Map<String, Object> config) throws Exception {
+        public ForEachProcessor create(String tag, Map<String, Object> config) throws Exception {
             String field = readStringProperty(TYPE, tag, config, "field");
             List<Map<String, Map<String, Object>>> processorConfigs = readList(TYPE, tag, config, "processors");
             List<Processor> processors = ConfigurationUtils.readProcessorConfigs(processorConfigs, processorRegistry);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
index aa9fbb905cc..32edc665d58 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
@@ -20,9 +20,9 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
 
 import java.util.HashMap;
 import java.util.List;
@@ -114,7 +114,7 @@ public final class GrokProcessor extends AbstractProcessor {
         return combinedPattern;
     }
 
-    public final static class Factory extends AbstractProcessorFactory<GrokProcessor> {
+    public final static class Factory implements Processor.Factory {
 
         private final Map<String, String> builtinPatterns;
 
@@ -123,7 +123,7 @@ public final class GrokProcessor extends AbstractProcessor {
         }
 
         @Override
-        public GrokProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public GrokProcessor create(String processorTag, Map<String, Object> config) throws Exception {
             String matchField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             List<String> matchPatterns = ConfigurationUtils.readList(TYPE, processorTag, config, "patterns");
             boolean traceMatch = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "trace_match", false);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java
index 72bc9e76710..1dfc566670f 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GsubProcessor.java
@@ -20,8 +20,8 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
 
 import java.util.Map;
 import java.util.regex.Matcher;
@@ -78,9 +78,9 @@ public final class GsubProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<GsubProcessor> {
+    public static final class Factory implements Processor.Factory {
         @Override
-        public GsubProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public GsubProcessor create(String processorTag, Map<String, Object> config) throws Exception {
             String field = readStringProperty(TYPE, processorTag, config, "field");
             String pattern = readStringProperty(TYPE, processorTag, config, "pattern");
             String replacement = readStringProperty(TYPE, processorTag, config, "replacement");
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java
index 8114d20f28f..f8bc2afffdd 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java
@@ -20,9 +20,9 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
 
 import java.util.List;
 import java.util.Map;
@@ -70,9 +70,9 @@ public final class JoinProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public final static class Factory extends AbstractProcessorFactory<JoinProcessor> {
+    public final static class Factory implements Processor.Factory {
         @Override
-        public JoinProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public JoinProcessor create(String processorTag, Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String separator = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "separator");
             return new JoinProcessor(processorTag, field, separator);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java
index 9f8ea7a5614..e7a8f3f3e6a 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java
@@ -44,7 +44,7 @@ public final class LowercaseProcessor extends AbstractStringProcessor {
         return TYPE;
     }
 
-    public final static class Factory extends AbstractStringProcessor.Factory<LowercaseProcessor> {
+    public final static class Factory extends AbstractStringProcessor.Factory {
 
         public Factory() {
             super(TYPE);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java
index 98c4e18a408..a7de33a7e2c 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java
@@ -20,9 +20,9 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.TemplateService;
 
 import java.util.Map;
@@ -55,7 +55,7 @@ public final class RemoveProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<RemoveProcessor> {
+    public static final class Factory implements Processor.Factory {
 
         private final TemplateService templateService;
 
@@ -64,7 +64,7 @@ public final class RemoveProcessor extends AbstractProcessor {
         }
 
         @Override
-        public RemoveProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public RemoveProcessor create(String processorTag, Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             return new RemoveProcessor(processorTag, templateService.compile(field));
         }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java
index 9143321c4aa..ae81291b644 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java
@@ -20,9 +20,9 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
 
 import java.util.Map;
 
@@ -75,9 +75,9 @@ public final class RenameProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<RenameProcessor> {
+    public static final class Factory implements Processor.Factory {
         @Override
-        public RenameProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public RenameProcessor create(String processorTag, Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field");
             return new RenameProcessor(processorTag, field, targetField);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
index e4881366165..4733c1f5866 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
@@ -24,8 +24,8 @@ import java.util.Map;
 
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
@@ -77,7 +77,7 @@ public final class ScriptProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<ScriptProcessor> {
+    public static final class Factory implements Processor.Factory {
 
         private final ScriptService scriptService;
 
@@ -86,7 +86,7 @@ public final class ScriptProcessor extends AbstractProcessor {
         }
 
         @Override
-        public ScriptProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public ScriptProcessor create(String processorTag, Map<String, Object> config) throws Exception {
             String field = readOptionalStringProperty(TYPE, processorTag, config, "field");
             String lang = readStringProperty(TYPE, processorTag, config, "lang");
             String inline = readOptionalStringProperty(TYPE, processorTag, config, "inline");
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java
index a78701645a9..ce328e34cb7 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java
@@ -20,9 +20,9 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.TemplateService;
 import org.elasticsearch.ingest.ValueSource;
 
@@ -75,7 +75,7 @@ public final class SetProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<SetProcessor> {
+    public static final class Factory implements Processor.Factory {
 
         private final TemplateService templateService;
 
@@ -84,7 +84,7 @@ public final class SetProcessor extends AbstractProcessor {
         }
 
         @Override
-        public SetProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public SetProcessor create(String processorTag, Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             Object value = ConfigurationUtils.readObject(TYPE, processorTag, config, "value");
             boolean overrideEnabled = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "override", true);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java
index 706a1cef9c5..5ab4016921d 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java
@@ -20,9 +20,9 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
 
 import java.util.Collections;
 import java.util.List;
@@ -111,10 +111,10 @@ public final class SortProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public final static class Factory extends AbstractProcessorFactory<SortProcessor> {
+    public final static class Factory implements Processor.Factory {
 
         @Override
-        public SortProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public SortProcessor create(String processorTag, Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, FIELD);
             try {
                 SortOrder direction = SortOrder.fromString(
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java
index f7c5e8befc4..87526f62297 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java
@@ -20,9 +20,9 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -72,9 +72,9 @@ public final class SplitProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public static class Factory extends AbstractProcessorFactory<SplitProcessor> {
+    public static class Factory implements Processor.Factory {
         @Override
-        public SplitProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public SplitProcessor create(String processorTag, Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             return new SplitProcessor(processorTag, field, ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "separator"));
         }
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java
index a57a25125d6..e852f887da0 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TrimProcessor.java
@@ -41,7 +41,7 @@ public final class TrimProcessor extends AbstractStringProcessor {
         return TYPE;
     }
 
-    public static final class Factory extends AbstractStringProcessor.Factory<TrimProcessor> {
+    public static final class Factory extends AbstractStringProcessor.Factory {
 
         public Factory() {
             super(TYPE);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java
index a5c817352a1..5585a130eaf 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UppercaseProcessor.java
@@ -43,7 +43,7 @@ public final class UppercaseProcessor extends AbstractStringProcessor {
         return TYPE;
     }
 
-    public static final class Factory extends AbstractStringProcessor.Factory<UppercaseProcessor> {
+    public static final class Factory extends AbstractStringProcessor.Factory {
 
         public Factory() {
             super(TYPE);
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java
index fbf77cc4285..b49a44cc04c 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
@@ -52,8 +51,7 @@ public class AppendProcessorFactoryTests extends ESTestCase {
         }
         config.put("value", value);
         String processorTag = randomAsciiOfLength(10);
-        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        AppendProcessor appendProcessor = factory.create(config);
+        AppendProcessor appendProcessor = factory.create(processorTag, config);
         assertThat(appendProcessor.getTag(), equalTo(processorTag));
         assertThat(appendProcessor.getField().execute(Collections.emptyMap()), equalTo("field1"));
         assertThat(appendProcessor.getValue().copyAndResolve(Collections.emptyMap()), equalTo(value));
@@ -63,7 +61,7 @@ public class AppendProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("value", "value1");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@@ -74,7 +72,7 @@ public class AppendProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[value] required property is missing"));
@@ -86,7 +84,7 @@ public class AppendProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         config.put("value", null);
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[value] required property is missing"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java
index 1ec5362af14..7dd8bbf97e2 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
 
@@ -39,8 +38,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         config.put("type", type.toString());
         String processorTag = randomAsciiOfLength(10);
-        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        ConvertProcessor convertProcessor = factory.create(config);
+        ConvertProcessor convertProcessor = factory.create(processorTag, config);
         assertThat(convertProcessor.getTag(), equalTo(processorTag));
         assertThat(convertProcessor.getField(), equalTo("field1"));
         assertThat(convertProcessor.getTargetField(), equalTo("field1"));
@@ -54,7 +52,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         config.put("type", type);
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), Matchers.equalTo("[type] type [" + type + "] not supported, cannot convert field."));
@@ -70,7 +68,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
         String type = "type-" + randomAsciiOfLengthBetween(1, 10);
         config.put("type", type);
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), Matchers.equalTo("[field] required property is missing"));
@@ -82,7 +80,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), Matchers.equalTo("[type] required property is missing"));
@@ -97,8 +95,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
         config.put("target_field", "field2");
         config.put("type", type.toString());
         String processorTag = randomAsciiOfLength(10);
-        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        ConvertProcessor convertProcessor = factory.create(config);
+        ConvertProcessor convertProcessor = factory.create(processorTag, config);
         assertThat(convertProcessor.getTag(), equalTo(processorTag));
         assertThat(convertProcessor.getField(), equalTo("field1"));
         assertThat(convertProcessor.getTargetField(), equalTo("field2"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java
index 42877236b88..cc272d0b120 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java
@@ -36,7 +36,7 @@ public class DateIndexNameFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("date_rounding", "y");
 
-        DateIndexNameProcessor processor = factory.create(config);
+        DateIndexNameProcessor processor = factory.create(null, config);
         assertThat(processor.getDateFormats().size(), Matchers.equalTo(1));
         assertThat(processor.getField(), Matchers.equalTo("_field"));
         assertThat(processor.getIndexNamePrefix(), Matchers.equalTo(""));
@@ -53,7 +53,7 @@ public class DateIndexNameFactoryTests extends ESTestCase {
         config.put("date_rounding", "y");
         config.put("date_formats", Arrays.asList("UNIX", "UNIX_MS"));
 
-        DateIndexNameProcessor processor = factory.create(config);
+        DateIndexNameProcessor processor = factory.create(null, config);
         assertThat(processor.getDateFormats().size(), Matchers.equalTo(2));
 
         config = new HashMap<>();
@@ -62,7 +62,7 @@ public class DateIndexNameFactoryTests extends ESTestCase {
         config.put("date_rounding", "y");
         config.put("index_name_format", "yyyyMMdd");
 
-        processor = factory.create(config);
+        processor = factory.create(null, config);
         assertThat(processor.getIndexNameFormat(), Matchers.equalTo("yyyyMMdd"));
 
         config = new HashMap<>();
@@ -71,7 +71,7 @@ public class DateIndexNameFactoryTests extends ESTestCase {
         config.put("date_rounding", "y");
         config.put("timezone", "+02:00");
 
-        processor = factory.create(config);
+        processor = factory.create(null, config);
         assertThat(processor.getTimezone(), Matchers.equalTo(DateTimeZone.forOffsetHours(2)));
 
         config = new HashMap<>();
@@ -79,7 +79,7 @@ public class DateIndexNameFactoryTests extends ESTestCase {
         config.put("index_name_prefix", "_prefix");
         config.put("date_rounding", "y");
 
-        processor = factory.create(config);
+        processor = factory.create(null, config);
         assertThat(processor.getIndexNamePrefix(), Matchers.equalTo("_prefix"));
     }
 
@@ -87,12 +87,12 @@ public class DateIndexNameFactoryTests extends ESTestCase {
         DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
         config.put("date_rounding", "y");
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config));
         assertThat(e.getMessage(), Matchers.equalTo("[field] required property is missing"));
 
         config.clear();
         config.put("field", "_field");
-        e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config));
         assertThat(e.getMessage(), Matchers.equalTo("[date_rounding] required property is missing"));
     }
 
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java
index 65dcdf6082c..9c5352c7ee4 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 import org.joda.time.DateTimeZone;
 
@@ -42,8 +41,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("field", sourceField);
         config.put("formats", Collections.singletonList("dd/MM/yyyyy"));
         String processorTag = randomAsciiOfLength(10);
-        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        DateProcessor processor = factory.create(config);
+        DateProcessor processor = factory.create(processorTag, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo(sourceField));
         assertThat(processor.getTargetField(), equalTo(DateProcessor.DEFAULT_TARGET_FIELD));
@@ -60,7 +58,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("formats", Collections.singletonList("dd/MM/yyyyy"));
 
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("processor creation should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), containsString("[field] required property is missing"));
@@ -76,7 +74,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("target_field", targetField);
 
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("processor creation should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), containsString("[formats] required property is missing"));
@@ -92,7 +90,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         Locale locale = randomLocale(random());
         config.put("locale", locale.toLanguageTag());
 
-        DateProcessor processor = factory.create(config);
+        DateProcessor processor = factory.create(null, config);
         assertThat(processor.getLocale().toLanguageTag(), equalTo(locale.toLanguageTag()));
     }
 
@@ -104,7 +102,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("formats", Collections.singletonList("dd/MM/yyyyy"));
         config.put("locale", "invalid_locale");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("should fail with invalid locale");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), equalTo("Invalid language tag specified: invalid_locale"));
@@ -120,7 +118,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
 
         DateTimeZone timezone = randomDateTimeZone();
         config.put("timezone", timezone.getID());
-        DateProcessor processor = factory.create(config);
+        DateProcessor processor = factory.create(null, config);
         assertThat(processor.getTimezone(), equalTo(timezone));
     }
 
@@ -132,7 +130,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("match_formats", Collections.singletonList("dd/MM/yyyyy"));
         config.put("timezone", "invalid_timezone");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("invalid timezone should fail");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), equalTo("The datetime zone id 'invalid_timezone' is not recognised"));
@@ -146,7 +144,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("field", sourceField);
         config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"));
 
-        DateProcessor processor = factory.create(config);
+        DateProcessor processor = factory.create(null, config);
         assertThat(processor.getFormats(), equalTo(Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy")));
     }
 
@@ -158,7 +156,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("formats", "dd/MM/yyyy");
 
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("processor creation should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), containsString("[formats] property isn't a list, but of type [java.lang.String]"));
@@ -174,7 +172,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("target_field", targetField);
         config.put("formats", Arrays.asList("dd/MM/yyyy", "dd-MM-yyyy"));
 
-        DateProcessor processor = factory.create(config);
+        DateProcessor processor = factory.create(null, config);
         assertThat(processor.getTargetField(), equalTo(targetField));
     }
 }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java
index db16b78b316..a385a58ef50 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
@@ -44,8 +43,7 @@ public class FailProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("message", "error");
         String processorTag = randomAsciiOfLength(10);
-        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        FailProcessor failProcessor = factory.create(config);
+        FailProcessor failProcessor = factory.create(processorTag, config);
         assertThat(failProcessor.getTag(), equalTo(processorTag));
         assertThat(failProcessor.getMessage().execute(Collections.emptyMap()), equalTo("error"));
     }
@@ -53,7 +51,7 @@ public class FailProcessorFactoryTests extends ESTestCase {
     public void testCreateMissingMessageField() throws Exception {
         Map<String, Object> config = new HashMap<>();
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[message] required property is missing"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java
index d45e98ab06a..0bbad532fb3 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java
@@ -38,14 +38,14 @@ public class ForEachProcessorFactoryTests extends ESTestCase {
     public void testCreate() throws Exception {
         ProcessorsRegistry.Builder builder = new ProcessorsRegistry.Builder();
         Processor processor = new TestProcessor(ingestDocument -> {});
-        builder.registerProcessor("_name", (registry) -> config -> processor);
+        builder.registerProcessor("_name", (registry) -> (tag, config) -> processor);
         ProcessorsRegistry registry = builder.build(mock(ScriptService.class), mock(ClusterService.class));
         ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(registry);
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("processors", Collections.singletonList(Collections.singletonMap("_name", Collections.emptyMap())));
-        ForEachProcessor forEachProcessor = forEachFactory.create(config);
+        ForEachProcessor forEachProcessor = forEachFactory.create(null, config);
         assertThat(forEachProcessor, Matchers.notNullValue());
         assertThat(forEachProcessor.getField(), Matchers.equalTo("_field"));
         assertThat(forEachProcessor.getProcessors().size(), Matchers.equalTo(1));
@@ -54,7 +54,7 @@ public class ForEachProcessorFactoryTests extends ESTestCase {
         config = new HashMap<>();
         config.put("processors", Collections.singletonList(Collections.singletonMap("_name", Collections.emptyMap())));
         try {
-            forEachFactory.create(config);
+            forEachFactory.create(null, config);
             fail("exception expected");
         } catch (Exception e) {
             assertThat(e.getMessage(), Matchers.equalTo("[field] required property is missing"));
@@ -63,7 +63,7 @@ public class ForEachProcessorFactoryTests extends ESTestCase {
         config = new HashMap<>();
         config.put("field", "_field");
         try {
-            forEachFactory.create(config);
+            forEachFactory.create(null, config);
             fail("exception expected");
         } catch (Exception e) {
             assertThat(e.getMessage(), Matchers.equalTo("[processors] required property is missing"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java
index a7a133b4363..1287d066420 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.Collections;
@@ -39,8 +38,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("patterns", Collections.singletonList("(?<foo>\\w+)"));
         String processorTag = randomAsciiOfLength(10);
-        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        GrokProcessor processor = factory.create(config);
+        GrokProcessor processor = factory.create(processorTag, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getMatchField(), equalTo("_field"));
         assertThat(processor.getGrok(), notNullValue());
@@ -50,7 +48,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap());
         Map<String, Object> config = new HashMap<>();
         config.put("patterns", Collections.singletonList("(?<foo>\\w+)"));
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create("tag", config));
         assertThat(e.getMessage(), equalTo("[field] required property is missing"));
     }
 
@@ -58,7 +56,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         GrokProcessor.Factory factory = new GrokProcessor.Factory(Collections.emptyMap());
         Map<String, Object> config = new HashMap<>();
         config.put("field", "foo");
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create("tag", config));
         assertThat(e.getMessage(), equalTo("[patterns] required property is missing"));
     }
 
@@ -67,7 +65,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "foo");
         config.put("patterns", Collections.emptyList());
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create("tag", config));
         assertThat(e.getMessage(), equalTo("[patterns] List of patterns must not be empty"));
     }
 
@@ -78,7 +76,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("patterns", Collections.singletonList("%{MY_PATTERN:name}!"));
         config.put("pattern_definitions", Collections.singletonMap("MY_PATTERN", "foo"));
-        GrokProcessor processor = factory.create(config);
+        GrokProcessor processor = factory.create(null, config);
         assertThat(processor.getMatchField(), equalTo("_field"));
         assertThat(processor.getGrok(), notNullValue());
         assertThat(processor.getGrok().match("foo!"), equalTo(true));
@@ -89,7 +87,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("patterns", Collections.singletonList("["));
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create("tag", config));
         assertThat(e.getMessage(), equalTo("[patterns] Invalid regex pattern found in: [[]. premature end of char-class"));
     }
 
@@ -99,7 +97,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("patterns", Collections.singletonList("%{MY_PATTERN:name}!"));
         config.put("pattern_definitions", Collections.singletonMap("MY_PATTERN", "["));
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create("tag", config));
         assertThat(e.getMessage(),
             equalTo("[patterns] Invalid regex pattern found in: [%{MY_PATTERN:name}!]. premature end of char-class"));
     }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java
index 60cceb34024..0086457f857 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
@@ -38,8 +37,7 @@ public class GsubProcessorFactoryTests extends ESTestCase {
         config.put("pattern", "\\.");
         config.put("replacement", "-");
         String processorTag = randomAsciiOfLength(10);
-        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        GsubProcessor gsubProcessor = factory.create(config);
+        GsubProcessor gsubProcessor = factory.create(processorTag, config);
         assertThat(gsubProcessor.getTag(), equalTo(processorTag));
         assertThat(gsubProcessor.getField(), equalTo("field1"));
         assertThat(gsubProcessor.getPattern().toString(), equalTo("\\."));
@@ -52,7 +50,7 @@ public class GsubProcessorFactoryTests extends ESTestCase {
         config.put("pattern", "\\.");
         config.put("replacement", "-");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@@ -65,7 +63,7 @@ public class GsubProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         config.put("replacement", "-");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[pattern] required property is missing"));
@@ -78,7 +76,7 @@ public class GsubProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         config.put("pattern", "\\.");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[replacement] required property is missing"));
@@ -92,7 +90,7 @@ public class GsubProcessorFactoryTests extends ESTestCase {
         config.put("pattern", "[");
         config.put("replacement", "-");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), containsString("[pattern] Invalid regex pattern. Unclosed character class"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java
index 970fd8b8b9a..b2386c17a2b 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
@@ -36,8 +35,7 @@ public class JoinProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         config.put("separator", "-");
         String processorTag = randomAsciiOfLength(10);
-        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        JoinProcessor joinProcessor = factory.create(config);
+        JoinProcessor joinProcessor = factory.create(processorTag, config);
         assertThat(joinProcessor.getTag(), equalTo(processorTag));
         assertThat(joinProcessor.getField(), equalTo("field1"));
         assertThat(joinProcessor.getSeparator(), equalTo("-"));
@@ -48,7 +46,7 @@ public class JoinProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("separator", "-");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@@ -60,7 +58,7 @@ public class JoinProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[separator] required property is missing"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorFactoryTests.java
index 4dec115458c..2e0682beb6e 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/LowercaseProcessorFactoryTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
@@ -35,8 +34,7 @@ public class LowercaseProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         String processorTag = randomAsciiOfLength(10);
-        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        LowercaseProcessor uppercaseProcessor = factory.create(config);
+        LowercaseProcessor uppercaseProcessor = (LowercaseProcessor)factory.create(processorTag, config);
         assertThat(uppercaseProcessor.getTag(), equalTo(processorTag));
         assertThat(uppercaseProcessor.getField(), equalTo("field1"));
     }
@@ -45,7 +43,7 @@ public class LowercaseProcessorFactoryTests extends ESTestCase {
         LowercaseProcessor.Factory factory = new LowercaseProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java
index a5f88103e96..133ddbeebfa 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
@@ -44,8 +43,7 @@ public class RemoveProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         String processorTag = randomAsciiOfLength(10);
-        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        RemoveProcessor removeProcessor = factory.create(config);
+        RemoveProcessor removeProcessor = factory.create(processorTag, config);
         assertThat(removeProcessor.getTag(), equalTo(processorTag));
         assertThat(removeProcessor.getField().execute(Collections.emptyMap()), equalTo("field1"));
     }
@@ -53,7 +51,7 @@ public class RemoveProcessorFactoryTests extends ESTestCase {
     public void testCreateMissingField() throws Exception {
         Map<String, Object> config = new HashMap<>();
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java
index c078f09dd92..b969a3b6247 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
@@ -36,8 +35,7 @@ public class RenameProcessorFactoryTests extends ESTestCase {
         config.put("field", "old_field");
         config.put("target_field", "new_field");
         String processorTag = randomAsciiOfLength(10);
-        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        RenameProcessor renameProcessor = factory.create(config);
+        RenameProcessor renameProcessor = factory.create(processorTag, config);
         assertThat(renameProcessor.getTag(), equalTo(processorTag));
         assertThat(renameProcessor.getField(), equalTo("old_field"));
         assertThat(renameProcessor.getTargetField(), equalTo("new_field"));
@@ -48,7 +46,7 @@ public class RenameProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("target_field", "new_field");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@@ -60,7 +58,7 @@ public class RenameProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "old_field");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[target_field] required property is missing"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
index ed47894d4d9..79048cd7769 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
@@ -55,7 +55,7 @@ public class ScriptProcessorFactoryTests extends ESTestCase {
         configMap.put("lang", "mockscript");
 
         ElasticsearchException exception = expectThrows(ElasticsearchException.class,
-            () -> factory.doCreate(randomAsciiOfLength(10), configMap));
+            () -> factory.create(randomAsciiOfLength(10), configMap));
 
         assertThat(exception.getMessage(), is("[null] Only one of [file], [id], or [inline] may be configured"));
     }
@@ -66,7 +66,7 @@ public class ScriptProcessorFactoryTests extends ESTestCase {
         configMap.put("lang", "mockscript");
 
         ElasticsearchException exception = expectThrows(ElasticsearchException.class,
-            () -> factory.doCreate(randomAsciiOfLength(10), configMap));
+            () -> factory.create(randomAsciiOfLength(10), configMap));
 
         assertThat(exception.getMessage(), is("[null] Need [file], [id], or [inline] parameter to refer to scripts"));
     }
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java
index b8c97a379cb..cbbf3f40902 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.TestTemplateService;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
@@ -45,8 +44,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         config.put("value", "value1");
         String processorTag = randomAsciiOfLength(10);
-        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        SetProcessor setProcessor = factory.create(config);
+        SetProcessor setProcessor = factory.create(processorTag, config);
         assertThat(setProcessor.getTag(), equalTo(processorTag));
         assertThat(setProcessor.getField().execute(Collections.emptyMap()), equalTo("field1"));
         assertThat(setProcessor.getValue().copyAndResolve(Collections.emptyMap()), equalTo("value1"));
@@ -60,8 +58,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
         config.put("value", "value1");
         config.put("override", overrideEnabled);
         String processorTag = randomAsciiOfLength(10);
-        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        SetProcessor setProcessor = factory.create(config);
+        SetProcessor setProcessor = factory.create(processorTag, config);
         assertThat(setProcessor.getTag(), equalTo(processorTag));
         assertThat(setProcessor.getField().execute(Collections.emptyMap()), equalTo("field1"));
         assertThat(setProcessor.getValue().copyAndResolve(Collections.emptyMap()), equalTo("value1"));
@@ -72,7 +69,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("value", "value1");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@@ -83,7 +80,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[value] required property is missing"));
@@ -95,7 +92,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         config.put("value", null);
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[value] required property is missing"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java
index c747807b710..4f85d61e629 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
@@ -36,8 +35,7 @@ public class SplitProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         config.put("separator", "\\.");
         String processorTag = randomAsciiOfLength(10);
-        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        SplitProcessor splitProcessor = factory.create(config);
+        SplitProcessor splitProcessor = factory.create(processorTag, config);
         assertThat(splitProcessor.getTag(), equalTo(processorTag));
         assertThat(splitProcessor.getField(), equalTo("field1"));
         assertThat(splitProcessor.getSeparator(), equalTo("\\."));
@@ -48,7 +46,7 @@ public class SplitProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("separator", "\\.");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@@ -60,7 +58,7 @@ public class SplitProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[separator] required property is missing"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java
index 13d45dc126b..a7e1313a099 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java
@@ -84,7 +84,7 @@ public class SplitProcessorTests extends ESTestCase {
         Map<String, Object> splitConfig = new HashMap<>();
         splitConfig.put("field", "flags");
         splitConfig.put("separator", "\\|");
-        Processor splitProcessor = (new SplitProcessor.Factory()).create(splitConfig);
+        Processor splitProcessor = (new SplitProcessor.Factory()).create("tag", splitConfig);
         Map<String, Object> source = new HashMap<>();
         source.put("flags", "new|hot|super|fun|interesting");
         IngestDocument ingestDocument = new IngestDocument(source, new HashMap<>());
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorFactoryTests.java
index 54904775478..fa9a33f41de 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/TrimProcessorFactoryTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
@@ -35,8 +34,7 @@ public class TrimProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         String processorTag = randomAsciiOfLength(10);
-        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        TrimProcessor uppercaseProcessor = factory.create(config);
+        TrimProcessor uppercaseProcessor = (TrimProcessor)factory.create(processorTag, config);
         assertThat(uppercaseProcessor.getTag(), equalTo(processorTag));
         assertThat(uppercaseProcessor.getField(), equalTo("field1"));
     }
@@ -45,7 +43,7 @@ public class TrimProcessorFactoryTests extends ESTestCase {
         TrimProcessor.Factory factory = new TrimProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorFactoryTests.java
index cd4d1faf767..91698d4fcc4 100644
--- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UppercaseProcessorFactoryTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
@@ -35,8 +34,7 @@ public class UppercaseProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         String processorTag = randomAsciiOfLength(10);
-        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        UppercaseProcessor uppercaseProcessor = factory.create(config);
+        UppercaseProcessor uppercaseProcessor = (UppercaseProcessor)factory.create(processorTag, config);
         assertThat(uppercaseProcessor.getTag(), equalTo(processorTag));
         assertThat(uppercaseProcessor.getField(), equalTo("field1"));
     }
@@ -45,7 +43,7 @@ public class UppercaseProcessorFactoryTests extends ESTestCase {
         UppercaseProcessor.Factory factory = new UppercaseProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("factory create should have failed");
         } catch(ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
diff --git a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
index 40ebe2592ab..fb4dc37d641 100644
--- a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
+++ b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
@@ -25,8 +25,8 @@ import org.apache.tika.metadata.TikaCoreProperties;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -150,12 +150,12 @@ public final class AttachmentProcessor extends AbstractProcessor {
         return indexedChars;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<AttachmentProcessor> {
+    public static final class Factory implements Processor.Factory {
 
         static final Set<Property> DEFAULT_PROPERTIES = EnumSet.allOf(Property.class);
 
         @Override
-        public AttachmentProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public AttachmentProcessor create(String processorTag, Map<String, Object> config) throws Exception {
             String field = readStringProperty(TYPE, processorTag, config, "field");
             String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "attachment");
             List<String> properyNames = readOptionalList(TYPE, processorTag, config, "properties");
diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java
index 6bd4e07702e..8d011056854 100644
--- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java
+++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.ingest.attachment;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
@@ -46,9 +46,8 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
 
         String processorTag = randomAsciiOfLength(10);
-        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
 
-        AttachmentProcessor processor = factory.create(config);
+        AttachmentProcessor processor = factory.create(processorTag, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("attachment"));
@@ -62,8 +61,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
         config.put("indexed_chars", indexedChars);
 
         String processorTag = randomAsciiOfLength(10);
-        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
-        AttachmentProcessor processor = factory.create(config);
+        AttachmentProcessor processor = factory.create(processorTag, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getIndexedChars(), is(indexedChars));
     }
@@ -72,7 +70,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("target_field", "_field");
-        AttachmentProcessor processor = factory.create(config);
+        AttachmentProcessor processor = factory.create(null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("_field"));
     }
@@ -89,7 +87,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("properties", fieldNames);
-        AttachmentProcessor processor = factory.create(config);
+        AttachmentProcessor processor = factory.create(null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getProperties(), equalTo(properties));
     }
@@ -99,7 +97,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("properties", Collections.singletonList("invalid"));
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("exception expected");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), containsString("[properties] illegal field option [invalid]"));
@@ -113,7 +111,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("properties", "invalid");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("exception expected");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]"));
diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
index 0b18cae25e7..92c4785dbaf 100644
--- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
+++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
@@ -34,8 +34,8 @@ import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -217,7 +217,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
         return geoData;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<GeoIpProcessor> {
+    public static final class Factory implements Processor.Factory {
         static final Set<Property> DEFAULT_CITY_PROPERTIES = EnumSet.of(
             Property.CONTINENT_NAME, Property.COUNTRY_ISO_CODE, Property.REGION_NAME,
             Property.CITY_NAME, Property.LOCATION
@@ -231,7 +231,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
         }
 
         @Override
-        public GeoIpProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public GeoIpProcessor create(String processorTag, Map<String, Object> config) throws Exception {
             String ipField = readStringProperty(TYPE, processorTag, config, "field");
             String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "geoip");
             String databaseFile = readStringProperty(TYPE, processorTag, config, "database_file", "GeoLite2-City.mmdb.gz");
diff --git a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
index 04729162729..28043171fcd 100644
--- a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
+++ b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
@@ -23,7 +23,6 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import com.maxmind.geoip2.DatabaseReader;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.Randomness;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.StreamsUtils;
 import org.junit.AfterClass;
@@ -74,11 +73,9 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-
         String processorTag = randomAsciiOfLength(10);
-        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
 
-        GeoIpProcessor processor = factory.create(config);
+        GeoIpProcessor processor = factory.create(processorTag, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("geoip"));
@@ -92,11 +89,9 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("database_file", "GeoLite2-Country.mmdb.gz");
-
         String processorTag = randomAsciiOfLength(10);
-        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
 
-        GeoIpProcessor processor = factory.create(config);
+        GeoIpProcessor processor = factory.create(processorTag, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("geoip"));
@@ -109,7 +104,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("target_field", "_field");
-        GeoIpProcessor processor = factory.create(config);
+        GeoIpProcessor processor = factory.create(null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("_field"));
     }
@@ -119,7 +114,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("database_file", "GeoLite2-Country.mmdb.gz");
-        GeoIpProcessor processor = factory.create(config);
+        GeoIpProcessor processor = factory.create(null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("geoip"));
         assertThat(processor.getDbReader().getMetadata().getDatabaseType(), equalTo("GeoLite2-Country"));
@@ -135,7 +130,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         String cityProperty = RandomPicks.randomFrom(Randomness.get(), cityOnlyProperties).toString();
         config.put("properties", Collections.singletonList(cityProperty));
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("Exception expected");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[properties] illegal property value [" + cityProperty +
@@ -150,7 +145,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("database_file", "does-not-exist.mmdb.gz");
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("Exception expected");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[database_file] database file [does-not-exist.mmdb.gz] doesn't exist"));
@@ -171,7 +166,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("properties", fieldNames);
-        GeoIpProcessor processor = factory.create(config);
+        GeoIpProcessor processor = factory.create(null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getProperties(), equalTo(properties));
     }
@@ -183,7 +178,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("properties", Collections.singletonList("invalid"));
         try {
-            factory.create(config);
+            factory.create(null, config);
             fail("exception expected");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [IP, COUNTRY_ISO_CODE, " +
@@ -194,7 +189,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("properties", "invalid");
         try {
-            factory.create(config);
+            factory.create("tag", config);
             fail("exception expected");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]"));
diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java b/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java
index b32a2eab991..a9585aa8d41 100644
--- a/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java
+++ b/test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java
@@ -28,7 +28,7 @@ import org.elasticsearch.plugins.Plugin;
 public class IngestTestPlugin extends Plugin {
 
     public void onModule(NodeModule nodeModule) {
-        nodeModule.registerProcessor("test", (registry) -> config ->
+        nodeModule.registerProcessor("test", (registry) -> (tag, config) ->
                 new TestProcessor("id", "test", doc -> {
                     doc.setFieldValue("processed", true);
                     if (doc.hasField("fail") && doc.getFieldValue("fail", Boolean.class)) {
diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java b/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java
index e36d73a8d9f..b309d94fe08 100644
--- a/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java
+++ b/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java
@@ -64,9 +64,9 @@ public class TestProcessor implements Processor {
         return invokedCounter.get();
     }
 
-    public static final class Factory extends AbstractProcessorFactory<TestProcessor> {
+    public static final class Factory implements Processor.Factory {
         @Override
-        public TestProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public TestProcessor create(String processorTag, Map<String, Object> config) throws Exception {
             return new TestProcessor(processorTag, "test-processor", ingestDocument -> {});
         }
     }

From 1ad3d2251fe16e1608a7ed26c4dc3fbe80d035ba Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Thu, 30 Jun 2016 11:43:07 +0200
Subject: [PATCH 1136/1311] Fix line width

---
 .../org/elasticsearch/cloud/gce/GceComputeServiceImpl.java     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java
index b27c9e409fe..63a19919a7b 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java
@@ -97,7 +97,8 @@ public class GceComputeServiceImpl extends AbstractLifecycleComponent<GceCompute
                     }
                 });
                 // assist type inference
-                return instanceList.isEmpty()  || instanceList.getItems() == null ? Collections.<Instance>emptyList() : instanceList.getItems();
+                return instanceList.isEmpty()  || instanceList.getItems() == null ?
+                    Collections.<Instance>emptyList() : instanceList.getItems();
             } catch (PrivilegedActionException e) {
                 logger.warn("Problem fetching instance list for zone {}", e, zoneId);
                 logger.debug("Full exception:", e);

From 57a734e641d9bfdff7e8dbae535e577db20e7db3 Mon Sep 17 00:00:00 2001
From: Britta Weber <brwe@users.noreply.github.com>
Date: Thu, 30 Jun 2016 11:52:53 +0200
Subject: [PATCH 1137/1311] [doc] explain avg in function_score better (#19154)

* [doc] explain avg in function_score better
---
 docs/reference/query-dsl/function-score-query.asciidoc | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/docs/reference/query-dsl/function-score-query.asciidoc b/docs/reference/query-dsl/function-score-query.asciidoc
index b6e4dedbc88..c6477b78d8b 100644
--- a/docs/reference/query-dsl/function-score-query.asciidoc
+++ b/docs/reference/query-dsl/function-score-query.asciidoc
@@ -83,9 +83,16 @@ First, each document is scored by the defined functions. The parameter
 `max`::         maximum score is used
 `min`::         minimum score is used
 
-Because scores can be on different scales (for example, between 0 and 1 for decay functions but arbitrary for `field_value_factor`) and also because sometimes a different impact of functions on the score is desirable, the score of each function can be adjusted with a user defined `weight` (). The `weight` can be defined per function in the `functions` array (example above) and is multiplied with the score computed by the respective function.
+Because scores can be on different scales (for example, between 0 and 1 for decay functions but arbitrary for `field_value_factor`) and also
+because sometimes a different impact of functions on the score is desirable, the score of each function can be adjusted with a user defined
+`weight`. The `weight` can be defined per function in the `functions` array (example above) and is multiplied with the score computed by
+the respective function.
 If weight is given without any other function declaration, `weight` acts as a function that simply returns the `weight`.
 
+In case `score_mode` is set to `avg` the individual scores will be combined by a **weighted** average.
+For example, if two functions return score 1 and 2 and their respective weights are 3 and 4, then their scores will be combined as
+`(1*3+2*4)/(3+4)` and **not** `(1*3+2*4)/2`.
+
 The new score can be restricted to not exceed a certain limit by setting
 the `max_boost` parameter. The default for `max_boost` is FLT_MAX.
 

From 0d7c11ea1d6ea3e33d6f493e406dbcd2144b69ad Mon Sep 17 00:00:00 2001
From: Colin Goodheart-Smithe <colings86@users.noreply.github.com>
Date: Thu, 30 Jun 2016 12:28:30 +0100
Subject: [PATCH 1138/1311] [DOCS] put profiling performance and limitations
 section on same page

---
 docs/reference/search/profile.asciidoc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc
index 62e04e669a1..150b1b93a36 100644
--- a/docs/reference/search/profile.asciidoc
+++ b/docs/reference/search/profile.asciidoc
@@ -699,7 +699,9 @@ The meaning of the stats are as follows:
 
     This is not currently used and will always report `0`. Currently aggregation profiling only times the shard level parts of the aggregation execution. Timing of the reduce phase will be added later.
 
-=== Performance Notes
+=== Profiling Considerations
+
+==== Performance Notes
 
 Like any profiler, the Profile API introduces a non-negligible overhead to search execution.  The act of instrumenting
 low-level method calls such as `collect`, `advance` and `next_doc` can be fairly expensive, since these methods are called
@@ -710,7 +712,7 @@ There are also cases where special Lucene optimizations are disabled, since they
 could cause some queries to report larger relative times than their non-profiled counterparts, but in general should
 not have a drastic effect compared to other components in the profiled query.
 
-=== Limitations
+==== Limitations
 
 - Profiling statistics are currently not available for suggestions, highlighting, `dfs_query_then_fetch`
 - Profiling of the reduce phase of aggregation is currently not available

From 40ec639c895aee0f027f1e641de06acb4f6055e6 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Thu, 30 Jun 2016 13:41:53 +0200
Subject: [PATCH 1139/1311] Factor out abstract TCPTransport* classes to reduce
 the netty footprint (#19096)

Today we have a ton of logic inside the NettyTransport* codebase. The footprint
of the code that has a direct netty dependency is large and alternative implementations
are pretty hard today since they need to know all about our proticol etc.
This change moves most of the code into TCPTransport* baseclasses and moves all
the protocol send code together. The base classes now contain the majority of the logic
while NettyTransport* classes remain to implement the glue code, configuration and optimization.
---
 .../elasticsearch/ElasticsearchException.java |    7 +-
 .../client/transport/TransportClient.java     |    4 +-
 .../org/elasticsearch/common/io/Channels.java |   20 -
 .../common/netty/KeepFrameDecoder.java        |   37 -
 .../netty/ReleaseChannelFutureListener.java   |   42 -
 .../common/settings/ClusterSettings.java      |   29 +-
 .../http/netty/ESHttpResponseEncoder.java     |    2 +-
 .../http/netty/NettyHttpChannel.java          |    5 +-
 .../http/netty/NettyHttpRequest.java          |    2 +-
 .../http/netty/NettyHttpServerTransport.java  |    4 +-
 .../elasticsearch/transport/TcpHeader.java    |   49 +
 .../elasticsearch/transport/TcpTransport.java | 1347 +++++++++++++++++
 .../transport/TcpTransportChannel.java        |  103 ++
 .../elasticsearch/transport/Transport.java    |    7 +
 .../elasticsearch/transport/Transports.java   |    9 +-
 .../netty/ChannelBufferBytesReference.java    |   23 +-
 .../netty/ChannelBufferStreamInput.java       |    4 +-
 .../ChannelBufferStreamInputFactory.java      |   36 -
 .../netty/MessageChannelHandler.java          |  437 ------
 .../transport/netty/NettyHeader.java          |   76 -
 .../netty/NettyInternalESLogger.java          |    4 +-
 .../netty/NettyInternalESLoggerFactory.java   |   35 -
 .../netty/NettyMessageChannelHandler.java     |   86 ++
 .../transport/netty/NettyTransport.java       | 1264 ++++------------
 .../netty/NettyTransportChannel.java          |  176 ---
 .../netty/NettyUtils.java                     |   18 +-
 .../netty/OpenChannelsHandler.java            |    7 +-
 .../netty/SizeHeaderFrameDecoder.java         |   98 +-
 .../ExceptionSerializationTests.java          |    3 +-
 .../elasticsearch/common/ChannelsTests.java   |   14 -
 .../util/concurrent}/KeyedLockTests.java      |    2 +-
 .../AbstractSimpleTransportTestCase.java      |   17 +-
 ...shPortTests.java => PublishPortTests.java} |    8 +-
 ...sportTests.java => TCPTransportTests.java} |   34 +-
 .../ChannelBufferBytesReferenceTests.java     |    3 +-
 .../netty/NettyScheduledPingTests.java        |   25 +-
 .../transport/netty/NettyTransportIT.java     |   54 +-
 .../netty/NettyTransportMultiPortTests.java   |   23 +-
 .../netty/NettyUtilsTests.java                |    3 +-
 .../test/InternalTestCluster.java             |    9 +-
 40 files changed, 2024 insertions(+), 2102 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/common/netty/KeepFrameDecoder.java
 delete mode 100644 core/src/main/java/org/elasticsearch/common/netty/ReleaseChannelFutureListener.java
 create mode 100644 core/src/main/java/org/elasticsearch/transport/TcpHeader.java
 create mode 100644 core/src/main/java/org/elasticsearch/transport/TcpTransport.java
 create mode 100644 core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java
 rename core/src/main/java/org/elasticsearch/{common => transport}/netty/ChannelBufferBytesReference.java (85%)
 delete mode 100644 core/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInputFactory.java
 delete mode 100644 core/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java
 delete mode 100644 core/src/main/java/org/elasticsearch/transport/netty/NettyHeader.java
 delete mode 100644 core/src/main/java/org/elasticsearch/transport/netty/NettyInternalESLoggerFactory.java
 create mode 100644 core/src/main/java/org/elasticsearch/transport/netty/NettyMessageChannelHandler.java
 delete mode 100644 core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java
 rename core/src/main/java/org/elasticsearch/{common => transport}/netty/NettyUtils.java (90%)
 rename core/src/main/java/org/elasticsearch/{common => transport}/netty/OpenChannelsHandler.java (95%)
 rename core/src/test/java/org/elasticsearch/{transport/netty => common/util/concurrent}/KeyedLockTests.java (99%)
 rename core/src/test/java/org/elasticsearch/transport/{netty/NettyPublishPortTests.java => PublishPortTests.java} (94%)
 rename core/src/test/java/org/elasticsearch/transport/{netty/NettyTransportTests.java => TCPTransportTests.java} (78%)
 rename core/src/test/java/org/elasticsearch/{common => transport}/netty/ChannelBufferBytesReferenceTests.java (96%)
 rename core/src/test/java/org/elasticsearch/{common => transport}/netty/NettyUtilsTests.java (98%)

diff --git a/core/src/main/java/org/elasticsearch/ElasticsearchException.java b/core/src/main/java/org/elasticsearch/ElasticsearchException.java
index ad5a2e79cd1..aa92fc176cf 100644
--- a/core/src/main/java/org/elasticsearch/ElasticsearchException.java
+++ b/core/src/main/java/org/elasticsearch/ElasticsearchException.java
@@ -30,6 +30,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.transport.TcpTransport;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -496,7 +497,7 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
                 org.elasticsearch.index.shard.IndexShardStartedException::new, 23),
         SEARCH_CONTEXT_MISSING_EXCEPTION(org.elasticsearch.search.SearchContextMissingException.class,
                 org.elasticsearch.search.SearchContextMissingException::new, 24),
-        GENERAL_SCRIPT_EXCEPTION(org.elasticsearch.script.GeneralScriptException.class, 
+        GENERAL_SCRIPT_EXCEPTION(org.elasticsearch.script.GeneralScriptException.class,
                 org.elasticsearch.script.GeneralScriptException::new, 25),
         BATCH_OPERATION_EXCEPTION(org.elasticsearch.index.shard.TranslogRecoveryPerformer.BatchOperationException.class,
                 org.elasticsearch.index.shard.TranslogRecoveryPerformer.BatchOperationException::new, 26),
@@ -676,8 +677,8 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
                 org.elasticsearch.indices.IndexAlreadyExistsException::new, 123),
         SCRIPT_PARSE_EXCEPTION(org.elasticsearch.script.Script.ScriptParseException.class,
                 org.elasticsearch.script.Script.ScriptParseException::new, 124),
-        HTTP_ON_TRANSPORT_EXCEPTION(org.elasticsearch.transport.netty.SizeHeaderFrameDecoder.HttpOnTransportException.class,
-                org.elasticsearch.transport.netty.SizeHeaderFrameDecoder.HttpOnTransportException::new, 125),
+        HTTP_ON_TRANSPORT_EXCEPTION(TcpTransport.HttpOnTransportException.class,
+                TcpTransport.HttpOnTransportException::new, 125),
         MAPPER_PARSING_EXCEPTION(org.elasticsearch.index.mapper.MapperParsingException.class,
                 org.elasticsearch.index.mapper.MapperParsingException::new, 126),
         SEARCH_CONTEXT_EXCEPTION(org.elasticsearch.search.SearchContextException.class,
diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
index 5786f6b1cfb..c9313fc08c4 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
@@ -50,8 +50,8 @@ import org.elasticsearch.plugins.PluginsService;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.TransportService;
-import org.elasticsearch.transport.netty.NettyTransport;
 
 import java.io.Closeable;
 import java.util.ArrayList;
@@ -107,7 +107,7 @@ public class TransportClient extends AbstractClient {
 
         private PluginsService newPluginService(final Settings settings) {
             final Settings.Builder settingsBuilder = Settings.builder()
-                    .put(NettyTransport.PING_SCHEDULE.getKey(), "5s") // enable by default the transport schedule ping interval
+                    .put(TcpTransport.PING_SCHEDULE.getKey(), "5s") // enable by default the transport schedule ping interval
                     .put(InternalSettingsPreparer.prepareSettings(settings))
                     .put(NetworkService.NETWORK_SERVER.getKey(), false)
                     .put(CLIENT_TYPE_SETTING_S.getKey(), CLIENT_TYPE);
diff --git a/core/src/main/java/org/elasticsearch/common/io/Channels.java b/core/src/main/java/org/elasticsearch/common/io/Channels.java
index 2fa7ca1cdec..71a2d66dee3 100644
--- a/core/src/main/java/org/elasticsearch/common/io/Channels.java
+++ b/core/src/main/java/org/elasticsearch/common/io/Channels.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.common.io;
 
 import org.elasticsearch.common.SuppressForbidden;
-import org.jboss.netty.buffer.ChannelBuffer;
 
 import java.io.EOFException;
 import java.io.IOException;
@@ -159,25 +158,6 @@ public final class Channels {
         return bytesRead;
     }
 
-
-    /**
-     * Copies bytes from source {@link org.jboss.netty.buffer.ChannelBuffer} to a {@link java.nio.channels.GatheringByteChannel}
-     *
-     * @param source      ChannelBuffer to copy from
-     * @param sourceIndex index in <i>source</i> to start copying from
-     * @param length      how many bytes to copy
-     * @param channel     target GatheringByteChannel
-     */
-    public static void writeToChannel(ChannelBuffer source, int sourceIndex, int length, GatheringByteChannel channel) throws IOException {
-        while (length > 0) {
-            int written = source.getBytes(sourceIndex, channel, length);
-            sourceIndex += written;
-            length -= written;
-        }
-        assert length == 0;
-    }
-
-
     /**
      * Writes part of a byte array to a {@link java.nio.channels.WritableByteChannel}
      *
diff --git a/core/src/main/java/org/elasticsearch/common/netty/KeepFrameDecoder.java b/core/src/main/java/org/elasticsearch/common/netty/KeepFrameDecoder.java
deleted file mode 100644
index d2b02b7c51e..00000000000
--- a/core/src/main/java/org/elasticsearch/common/netty/KeepFrameDecoder.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.common.netty;
-
-import org.jboss.netty.buffer.ChannelBuffer;
-import org.jboss.netty.channel.Channel;
-import org.jboss.netty.channel.ChannelHandlerContext;
-import org.jboss.netty.handler.codec.frame.FrameDecoder;
-
-/**
- * A marker to not remove frame decoder from the resulting jar so plugins can use it.
- */
-public class KeepFrameDecoder extends FrameDecoder {
-
-    public static final KeepFrameDecoder decoder = new KeepFrameDecoder();
-
-    @Override
-    protected Object decode(ChannelHandlerContext ctx, Channel channel, ChannelBuffer buffer) throws Exception {
-        return null;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/common/netty/ReleaseChannelFutureListener.java b/core/src/main/java/org/elasticsearch/common/netty/ReleaseChannelFutureListener.java
deleted file mode 100644
index 6f2979d13ca..00000000000
--- a/core/src/main/java/org/elasticsearch/common/netty/ReleaseChannelFutureListener.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.netty;
-
-import org.elasticsearch.common.lease.Releasable;
-import org.jboss.netty.channel.ChannelFuture;
-import org.jboss.netty.channel.ChannelFutureListener;
-
-/**
- * A channel listener that releases a {@link org.elasticsearch.common.lease.Releasable} when
- * the operation is complete.
- */
-public class ReleaseChannelFutureListener implements ChannelFutureListener {
-
-    private final Releasable releasable;
-
-    public ReleaseChannelFutureListener(Releasable releasable) {
-        this.releasable = releasable;
-    }
-
-    @Override
-    public void operationComplete(ChannelFuture future) throws Exception {
-        releasable.close();
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index 5cb37d14858..9469ac8b509 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -89,6 +89,7 @@ import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.TransportSettings;
@@ -279,14 +280,14 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     TransportSettings.PUBLISH_PORT,
                     TransportSettings.PORT,
                     NettyTransport.WORKER_COUNT,
-                    NettyTransport.CONNECTIONS_PER_NODE_RECOVERY,
-                    NettyTransport.CONNECTIONS_PER_NODE_BULK,
-                    NettyTransport.CONNECTIONS_PER_NODE_REG,
-                    NettyTransport.CONNECTIONS_PER_NODE_STATE,
-                    NettyTransport.CONNECTIONS_PER_NODE_PING,
-                    NettyTransport.PING_SCHEDULE,
-                    NettyTransport.TCP_BLOCKING_CLIENT,
-                    NettyTransport.TCP_CONNECT_TIMEOUT,
+                    TcpTransport.CONNECTIONS_PER_NODE_RECOVERY,
+                    TcpTransport.CONNECTIONS_PER_NODE_BULK,
+                    TcpTransport.CONNECTIONS_PER_NODE_REG,
+                    TcpTransport.CONNECTIONS_PER_NODE_STATE,
+                    TcpTransport.CONNECTIONS_PER_NODE_PING,
+                    TcpTransport.PING_SCHEDULE,
+                    TcpTransport.TCP_BLOCKING_CLIENT,
+                    TcpTransport.TCP_CONNECT_TIMEOUT,
                     NettyTransport.NETTY_MAX_CUMULATION_BUFFER_CAPACITY,
                     NettyTransport.NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS,
                     NettyTransport.NETTY_RECEIVE_PREDICTOR_SIZE,
@@ -294,12 +295,12 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     NettyTransport.NETTY_RECEIVE_PREDICTOR_MAX,
                     NetworkService.NETWORK_SERVER,
                     NettyTransport.NETTY_BOSS_COUNT,
-                    NettyTransport.TCP_NO_DELAY,
-                    NettyTransport.TCP_KEEP_ALIVE,
-                    NettyTransport.TCP_REUSE_ADDRESS,
-                    NettyTransport.TCP_SEND_BUFFER_SIZE,
-                    NettyTransport.TCP_RECEIVE_BUFFER_SIZE,
-                    NettyTransport.TCP_BLOCKING_SERVER,
+                    TcpTransport.TCP_NO_DELAY,
+                    TcpTransport.TCP_KEEP_ALIVE,
+                    TcpTransport.TCP_REUSE_ADDRESS,
+                    TcpTransport.TCP_SEND_BUFFER_SIZE,
+                    TcpTransport.TCP_RECEIVE_BUFFER_SIZE,
+                    TcpTransport.TCP_BLOCKING_SERVER,
                     NetworkService.GLOBAL_NETWORK_HOST_SETTING,
                     NetworkService.GLOBAL_NETWORK_BINDHOST_SETTING,
                     NetworkService.GLOBAL_NETWORK_PUBLISHHOST_SETTING,
diff --git a/core/src/main/java/org/elasticsearch/http/netty/ESHttpResponseEncoder.java b/core/src/main/java/org/elasticsearch/http/netty/ESHttpResponseEncoder.java
index 36f2c3a138b..afa69a2fe02 100644
--- a/core/src/main/java/org/elasticsearch/http/netty/ESHttpResponseEncoder.java
+++ b/core/src/main/java/org/elasticsearch/http/netty/ESHttpResponseEncoder.java
@@ -19,7 +19,7 @@
 
 package org.elasticsearch.http.netty;
 
-import org.elasticsearch.common.netty.NettyUtils;
+import org.elasticsearch.transport.netty.NettyUtils;
 import org.jboss.netty.buffer.ChannelBuffer;
 import org.jboss.netty.buffer.ChannelBuffers;
 import org.jboss.netty.buffer.CompositeChannelBuffer;
diff --git a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpChannel.java b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpChannel.java
index 3413a746963..c4253df2860 100644
--- a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpChannel.java
+++ b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpChannel.java
@@ -24,8 +24,7 @@ import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
 import org.elasticsearch.common.lease.Releasable;
-import org.elasticsearch.common.netty.NettyUtils;
-import org.elasticsearch.common.netty.ReleaseChannelFutureListener;
+import org.elasticsearch.transport.netty.NettyUtils;
 import org.elasticsearch.http.netty.cors.CorsHandler;
 import org.elasticsearch.http.netty.pipelining.OrderedDownstreamChannelEvent;
 import org.elasticsearch.http.netty.pipelining.OrderedUpstreamMessageEvent;
@@ -128,7 +127,7 @@ public final class NettyHttpChannel extends AbstractRestChannel {
             }
 
             if (content instanceof Releasable) {
-                future.addListener(new ReleaseChannelFutureListener((Releasable) content));
+                future.addListener((x) -> ((Releasable)content).close());
                 addedReleaseListener = true;
             }
 
diff --git a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpRequest.java b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpRequest.java
index d26841ead97..d62252bc0ce 100644
--- a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpRequest.java
+++ b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpRequest.java
@@ -21,7 +21,7 @@ package org.elasticsearch.http.netty;
 
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.netty.NettyUtils;
+import org.elasticsearch.transport.netty.NettyUtils;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.support.RestUtils;
 import org.jboss.netty.channel.Channel;
diff --git a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java
index 22852b7c0ff..77c140ce7ce 100644
--- a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java
+++ b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java
@@ -24,8 +24,8 @@ import com.carrotsearch.hppc.IntSet;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.netty.NettyUtils;
-import org.elasticsearch.common.netty.OpenChannelsHandler;
+import org.elasticsearch.transport.netty.NettyUtils;
+import org.elasticsearch.transport.netty.OpenChannelsHandler;
 import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting;
diff --git a/core/src/main/java/org/elasticsearch/transport/TcpHeader.java b/core/src/main/java/org/elasticsearch/transport/TcpHeader.java
new file mode 100644
index 00000000000..bbc54c0eb62
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/transport/TcpHeader.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.transport;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.StreamOutput;
+
+import java.io.IOException;
+
+public class TcpHeader {
+    public static final int MARKER_BYTES_SIZE = 2 * 1;
+
+    public static final int MESSAGE_LENGTH_SIZE = 4;
+
+    public static final int REQUEST_ID_SIZE = 8;
+
+    public static final int STATUS_SIZE = 1;
+
+    public static final int VERSION_ID_SIZE = 4;
+
+    public static final int HEADER_SIZE = MARKER_BYTES_SIZE + MESSAGE_LENGTH_SIZE + REQUEST_ID_SIZE + STATUS_SIZE + VERSION_ID_SIZE;
+
+    public static void writeHeader(StreamOutput output, long requestId, byte status, Version version, int messageSize) throws IOException {
+        output.writeByte((byte)'E');
+        output.writeByte((byte)'S');
+        // write the size, the size indicates the remaining message size, not including the size int
+        output.writeInt(messageSize - TcpHeader.MARKER_BYTES_SIZE - TcpHeader.MESSAGE_LENGTH_SIZE);
+        output.writeLong(requestId);
+        output.writeByte(status);
+        output.writeInt(version.id);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
new file mode 100644
index 00000000000..f9b5eceb4b8
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
@@ -0,0 +1,1347 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.transport;
+
+import com.carrotsearch.hppc.IntHashSet;
+import com.carrotsearch.hppc.IntSet;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.common.component.Lifecycle;
+import org.elasticsearch.common.compress.Compressor;
+import org.elasticsearch.common.compress.CompressorFactory;
+import org.elasticsearch.common.compress.NotCompressedException;
+import org.elasticsearch.common.io.ReleasableBytesStream;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.lease.Releasables;
+import org.elasticsearch.common.metrics.CounterMetric;
+import org.elasticsearch.common.network.NetworkAddress;
+import org.elasticsearch.common.network.NetworkService;
+import org.elasticsearch.common.network.NetworkUtils;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.BoundTransportAddress;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.transport.PortsRange;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.concurrent.AbstractLifecycleRunnable;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.KeyedLock;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.monitor.jvm.JvmInfo;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.support.TransportStatus;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.StreamCorruptedException;
+import java.net.BindException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.nio.channels.CancelledKeyException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static java.util.Collections.unmodifiableMap;
+import static org.elasticsearch.common.settings.Setting.boolSetting;
+import static org.elasticsearch.common.settings.Setting.intSetting;
+import static org.elasticsearch.common.settings.Setting.timeSetting;
+import static org.elasticsearch.common.transport.NetworkExceptionHelper.isCloseConnectionException;
+import static org.elasticsearch.common.transport.NetworkExceptionHelper.isConnectException;
+import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap;
+
+/**
+ */
+public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent<Transport> implements Transport {
+
+    public static final String HTTP_SERVER_WORKER_THREAD_NAME_PREFIX = "http_server_worker";
+    public static final String HTTP_SERVER_BOSS_THREAD_NAME_PREFIX = "http_server_boss";
+    public static final String TRANSPORT_CLIENT_WORKER_THREAD_NAME_PREFIX = "transport_client_worker";
+    public static final String TRANSPORT_CLIENT_BOSS_THREAD_NAME_PREFIX = "transport_client_boss";
+
+    // the scheduled internal ping interval setting, defaults to disabled (-1)
+    public static final Setting<TimeValue> PING_SCHEDULE =
+        timeSetting("transport.ping_schedule", TimeValue.timeValueSeconds(-1), Setting.Property.NodeScope);
+    public static final Setting<Integer> CONNECTIONS_PER_NODE_RECOVERY =
+        intSetting("transport.connections_per_node.recovery", 2, 1, Setting.Property.NodeScope);
+    public static final Setting<Integer> CONNECTIONS_PER_NODE_BULK =
+        intSetting("transport.connections_per_node.bulk", 3, 1, Setting.Property.NodeScope);
+    public static final Setting<Integer> CONNECTIONS_PER_NODE_REG =
+        intSetting("transport.connections_per_node.reg", 6, 1, Setting.Property.NodeScope);
+    public static final Setting<Integer> CONNECTIONS_PER_NODE_STATE =
+        intSetting("transport.connections_per_node.state", 1, 1, Setting.Property.NodeScope);
+    public static final Setting<Integer> CONNECTIONS_PER_NODE_PING =
+        intSetting("transport.connections_per_node.ping", 1, 1, Setting.Property.NodeScope);
+    public static final Setting<TimeValue> TCP_CONNECT_TIMEOUT =
+        timeSetting("transport.tcp.connect_timeout", NetworkService.TcpSettings.TCP_CONNECT_TIMEOUT, Setting.Property.NodeScope);
+    public static final Setting<Boolean> TCP_NO_DELAY =
+        boolSetting("transport.tcp_no_delay", NetworkService.TcpSettings.TCP_NO_DELAY, Setting.Property.NodeScope);
+    public static final Setting<Boolean> TCP_KEEP_ALIVE =
+        boolSetting("transport.tcp.keep_alive", NetworkService.TcpSettings.TCP_KEEP_ALIVE, Setting.Property.NodeScope);
+    public static final Setting<Boolean> TCP_REUSE_ADDRESS =
+        boolSetting("transport.tcp.reuse_address", NetworkService.TcpSettings.TCP_REUSE_ADDRESS, Setting.Property.NodeScope);
+    public static final Setting<Boolean> TCP_BLOCKING_CLIENT =
+        boolSetting("transport.tcp.blocking_client", NetworkService.TcpSettings.TCP_BLOCKING_CLIENT, Setting.Property.NodeScope);
+    public static final Setting<Boolean> TCP_BLOCKING_SERVER =
+        boolSetting("transport.tcp.blocking_server", NetworkService.TcpSettings.TCP_BLOCKING_SERVER, Setting.Property.NodeScope);
+    public static final Setting<ByteSizeValue> TCP_SEND_BUFFER_SIZE =
+        Setting.byteSizeSetting("transport.tcp.send_buffer_size", NetworkService.TcpSettings.TCP_SEND_BUFFER_SIZE,
+            Setting.Property.NodeScope);
+    public static final Setting<ByteSizeValue> TCP_RECEIVE_BUFFER_SIZE =
+        Setting.byteSizeSetting("transport.tcp.receive_buffer_size", NetworkService.TcpSettings.TCP_RECEIVE_BUFFER_SIZE,
+            Setting.Property.NodeScope);
+
+    private static final long NINETY_PER_HEAP_SIZE = (long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.9);
+    private static final int PING_DATA_SIZE = -1;
+
+    protected final int connectionsPerNodeRecovery;
+    protected final int connectionsPerNodeBulk;
+    protected final int connectionsPerNodeReg;
+    protected final int connectionsPerNodeState;
+    protected final int connectionsPerNodePing;
+    protected final TimeValue connectTimeout;
+    protected final boolean blockingClient;
+    private final CircuitBreakerService circuitBreakerService;
+    // package visibility for tests
+    protected final ScheduledPing scheduledPing;
+    private final TimeValue pingSchedule;
+    protected final ThreadPool threadPool;
+    private final BigArrays bigArrays;
+    protected final NetworkService networkService;
+
+    protected volatile TransportServiceAdapter transportServiceAdapter;
+    // node id to actual channel
+    protected final ConcurrentMap<DiscoveryNode, NodeChannels> connectedNodes = newConcurrentMap();
+    protected final Map<String, List<Channel>> serverChannels = newConcurrentMap();
+    protected final ConcurrentMap<String, BoundTransportAddress> profileBoundAddresses = newConcurrentMap();
+
+    protected final KeyedLock<String> connectionLock = new KeyedLock<>();
+    private final NamedWriteableRegistry namedWriteableRegistry;
+
+    // this lock is here to make sure we close this transport and disconnect all the client nodes
+    // connections while no connect operations is going on... (this might help with 100% CPU when stopping the transport?)
+    protected final ReadWriteLock globalLock = new ReentrantReadWriteLock();
+    protected final boolean compress;
+    protected volatile BoundTransportAddress boundAddress;
+    private final String transportName;
+
+    public TcpTransport(String transportName, Settings settings, ThreadPool threadPool, BigArrays bigArrays,
+                        CircuitBreakerService circuitBreakerService, NamedWriteableRegistry namedWriteableRegistry,
+                        NetworkService networkService) {
+        super(settings);
+        this.threadPool = threadPool;
+        this.bigArrays = bigArrays;
+        this.circuitBreakerService = circuitBreakerService;
+        this.scheduledPing = new ScheduledPing();
+        this.pingSchedule = PING_SCHEDULE.get(settings);
+        this.namedWriteableRegistry = namedWriteableRegistry;
+        this.compress = Transport.TRANSPORT_TCP_COMPRESS.get(settings);
+        this.networkService = networkService;
+        this.transportName = transportName;
+
+        this.connectionsPerNodeRecovery = CONNECTIONS_PER_NODE_RECOVERY.get(settings);
+        this.connectionsPerNodeBulk = CONNECTIONS_PER_NODE_BULK.get(settings);
+        this.connectionsPerNodeReg = CONNECTIONS_PER_NODE_REG.get(settings);
+        this.connectionsPerNodeState = CONNECTIONS_PER_NODE_STATE.get(settings);
+        this.connectionsPerNodePing = CONNECTIONS_PER_NODE_PING.get(settings);
+        this.connectTimeout = TCP_CONNECT_TIMEOUT.get(settings);
+        this.blockingClient = TCP_BLOCKING_CLIENT.get(settings);
+    }
+
+    @Override
+    protected void doStart() {
+        if (pingSchedule.millis() > 0) {
+            threadPool.schedule(pingSchedule, ThreadPool.Names.GENERIC, scheduledPing);
+        }
+    }
+
+    @Override
+    public CircuitBreaker getInFlightRequestBreaker() {
+        // We always obtain a fresh breaker to reflect changes to the breaker configuration.
+        return circuitBreakerService.getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS);
+    }
+
+    @Override
+    public void transportServiceAdapter(TransportServiceAdapter service) {
+        this.transportServiceAdapter = service;
+    }
+
+    public Settings settings() {
+        return this.settings;
+    }
+
+    public boolean isCompressed() {
+        return compress;
+    }
+
+    public class ScheduledPing extends AbstractLifecycleRunnable {
+
+        /**
+         * The magic number (must be lower than 0) for a ping message. This is handled
+         * specifically in {@link TcpTransport#validateMessageHeader}.
+         */
+        private final BytesReference pingHeader;
+        final CounterMetric successfulPings = new CounterMetric();
+        final CounterMetric failedPings = new CounterMetric();
+
+        public ScheduledPing() {
+            super(lifecycle, logger);
+            try (BytesStreamOutput out = new BytesStreamOutput()) {
+                out.writeByte((byte) 'E');
+                out.writeByte((byte) 'S');
+                out.writeInt(PING_DATA_SIZE);
+                pingHeader = out.bytes();
+            } catch (IOException e) {
+                throw new IllegalStateException(e.getMessage(), e); // won't happen
+            }
+        }
+
+        @Override
+        protected void doRunInLifecycle() throws Exception {
+            for (Map.Entry<DiscoveryNode, NodeChannels> entry : connectedNodes.entrySet()) {
+                DiscoveryNode node = entry.getKey();
+                NodeChannels channels = entry.getValue();
+                for (Channel channel : channels.allChannels) {
+                    try {
+                        sendMessage(channel, pingHeader, successfulPings::inc, false);
+                    } catch (Throwable t) {
+                        if (isOpen(channel)) {
+                            logger.debug("[{}] failed to send ping transport message", t, node);
+                            failedPings.inc();
+                        } else {
+                            logger.trace("[{}] failed to send ping transport message (channel closed)", t, node);
+                        }
+                    }
+                }
+            }
+        }
+
+        public long getSuccessfulPings() {
+            return successfulPings.count();
+        }
+
+        public long getFailedPings() {
+            return failedPings.count();
+        }
+
+        @Override
+        protected void onAfterInLifecycle() {
+            threadPool.schedule(pingSchedule, ThreadPool.Names.GENERIC, this);
+        }
+
+        @Override
+        public void onFailure(Throwable t) {
+            if (lifecycle.stoppedOrClosed()) {
+                logger.trace("failed to send ping transport message", t);
+            } else {
+                logger.warn("failed to send ping transport message", t);
+            }
+        }
+    }
+
+    public class NodeChannels implements Closeable {
+
+        public List<Channel> allChannels = Collections.emptyList();
+        public Channel[] recovery;
+        public final AtomicInteger recoveryCounter = new AtomicInteger();
+        public Channel[] bulk;
+        public final AtomicInteger bulkCounter = new AtomicInteger();
+        public Channel[] reg;
+        public final AtomicInteger regCounter = new AtomicInteger();
+        public Channel[] state;
+        public final AtomicInteger stateCounter = new AtomicInteger();
+        public Channel[] ping;
+        public final AtomicInteger pingCounter = new AtomicInteger();
+
+        public NodeChannels(Channel[] recovery, Channel[] bulk, Channel[] reg, Channel[] state, Channel[] ping) {
+            this.recovery = recovery;
+            this.bulk = bulk;
+            this.reg = reg;
+            this.state = state;
+            this.ping = ping;
+        }
+
+        public void start() {
+            List<Channel> newAllChannels = new ArrayList<>();
+            newAllChannels.addAll(Arrays.asList(recovery));
+            newAllChannels.addAll(Arrays.asList(bulk));
+            newAllChannels.addAll(Arrays.asList(reg));
+            newAllChannels.addAll(Arrays.asList(state));
+            newAllChannels.addAll(Arrays.asList(ping));
+            this.allChannels = Collections.unmodifiableList(newAllChannels);
+        }
+
+        public boolean hasChannel(Channel channel) {
+            for (Channel channel1 : allChannels) {
+                if (channel.equals(channel1)) {
+                    return true;
+                }
+            }
+            return false;
+        }
+
+        public Channel channel(TransportRequestOptions.Type type) {
+            if (type == TransportRequestOptions.Type.REG) {
+                return reg[Math.floorMod(regCounter.incrementAndGet(), reg.length)];
+            } else if (type == TransportRequestOptions.Type.STATE) {
+                return state[Math.floorMod(stateCounter.incrementAndGet(), state.length)];
+            } else if (type == TransportRequestOptions.Type.PING) {
+                return ping[Math.floorMod(pingCounter.incrementAndGet(), ping.length)];
+            } else if (type == TransportRequestOptions.Type.BULK) {
+                return bulk[Math.floorMod(bulkCounter.incrementAndGet(), bulk.length)];
+            } else if (type == TransportRequestOptions.Type.RECOVERY) {
+                return recovery[Math.floorMod(recoveryCounter.incrementAndGet(), recovery.length)];
+            } else {
+                throw new IllegalArgumentException("no type channel for [" + type + "]");
+            }
+        }
+
+        public synchronized void close() {
+            closeChannels(allChannels);
+        }
+    }
+
+    @Override
+    public boolean nodeConnected(DiscoveryNode node) {
+        return connectedNodes.containsKey(node);
+    }
+
+    @Override
+    public void connectToNodeLight(DiscoveryNode node) throws ConnectTransportException {
+        connectToNode(node, true);
+    }
+
+    @Override
+    public void connectToNode(DiscoveryNode node) {
+        connectToNode(node, false);
+    }
+
+    public void connectToNode(DiscoveryNode node, boolean light) {
+        if (!lifecycle.started()) {
+            throw new IllegalStateException("can't add nodes to a stopped transport");
+        }
+        if (node == null) {
+            throw new ConnectTransportException(null, "can't connect to a null node");
+        }
+        globalLock.readLock().lock();
+        try {
+
+            try (Releasable ignored = connectionLock.acquire(node.getId())) {
+                if (!lifecycle.started()) {
+                    throw new IllegalStateException("can't add nodes to a stopped transport");
+                }
+                NodeChannels nodeChannels = connectedNodes.get(node);
+                if (nodeChannels != null) {
+                    return;
+                }
+                try {
+                    if (light) {
+                        nodeChannels = connectToChannelsLight(node);
+                    } else {
+                        try {
+                            nodeChannels = connectToChannels(node);
+                        } catch (Throwable e) {
+                            logger.trace("failed to connect to [{}], cleaning dangling connections", e, node);
+                            throw e;
+                        }
+                    }
+                    // we acquire a connection lock, so no way there is an existing connection
+                    nodeChannels.start();
+                    connectedNodes.put(node, nodeChannels);
+                    if (logger.isDebugEnabled()) {
+                        logger.debug("connected to node [{}]", node);
+                    }
+                    transportServiceAdapter.raiseNodeConnected(node);
+                } catch (ConnectTransportException e) {
+                    throw e;
+                } catch (Exception e) {
+                    throw new ConnectTransportException(node, "general node connection failure", e);
+                }
+            }
+        } finally {
+            globalLock.readLock().unlock();
+        }
+    }
+    /**
+     * Disconnects from a node, only if the relevant channel is found to be part of the node channels.
+     */
+    protected boolean disconnectFromNode(DiscoveryNode node, Channel channel, String reason) {
+        // this might be called multiple times from all the node channels, so do a lightweight
+        // check outside of the lock
+        NodeChannels nodeChannels = connectedNodes.get(node);
+        if (nodeChannels != null && nodeChannels.hasChannel(channel)) {
+            try (Releasable ignored = connectionLock.acquire(node.getId())) {
+                nodeChannels = connectedNodes.get(node);
+                // check again within the connection lock, if its still applicable to remove it
+                if (nodeChannels != null && nodeChannels.hasChannel(channel)) {
+                    connectedNodes.remove(node);
+                    try {
+                        logger.debug("disconnecting from [{}], {}", node, reason);
+                        nodeChannels.close();
+                    } finally {
+                        logger.trace("disconnected from [{}], {}", node, reason);
+                        transportServiceAdapter.raiseNodeDisconnected(node);
+                    }
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Disconnects from a node if a channel is found as part of that nodes channels.
+     */
+    protected final void disconnectFromNodeChannel(final Channel channel, final Throwable failure) {
+        threadPool.generic().execute(() -> {
+            try {
+                closeChannels(Collections.singletonList(channel));
+            } finally {
+                for (DiscoveryNode node : connectedNodes.keySet()) {
+                    if (disconnectFromNode(node, channel, ExceptionsHelper.detailedMessage(failure))) {
+                        // if we managed to find this channel and disconnect from it, then break, no need to check on
+                        // the rest of the nodes
+                        break;
+                    }
+                }
+            }
+        });
+    }
+
+    protected Channel nodeChannel(DiscoveryNode node, TransportRequestOptions options) throws ConnectTransportException {
+        NodeChannels nodeChannels = connectedNodes.get(node);
+        if (nodeChannels == null) {
+            throw new NodeNotConnectedException(node, "Node not connected");
+        }
+        return nodeChannels.channel(options.type());
+    }
+
+    @Override
+    public void disconnectFromNode(DiscoveryNode node) {
+        try (Releasable ignored = connectionLock.acquire(node.getId())) {
+            NodeChannels nodeChannels = connectedNodes.remove(node);
+            if (nodeChannels != null) {
+                try {
+                    logger.debug("disconnecting from [{}] due to explicit disconnect call", node);
+                    nodeChannels.close();
+                } finally {
+                    logger.trace("disconnected from [{}] due to explicit disconnect call", node);
+                    transportServiceAdapter.raiseNodeDisconnected(node);
+                }
+            }
+        }
+    }
+
+    protected Version getCurrentVersion() {
+        // this is just for tests to mock stuff like the nodes version - tests can override this internally
+        return Version.CURRENT;
+    }
+
+    @Override
+    public boolean addressSupported(Class<? extends TransportAddress> address) {
+        return InetSocketTransportAddress.class.equals(address);
+    }
+
+    @Override
+    public BoundTransportAddress boundAddress() {
+        return this.boundAddress;
+    }
+
+    @Override
+    public Map<String, BoundTransportAddress> profileBoundAddresses() {
+        return unmodifiableMap(new HashMap<>(profileBoundAddresses));
+    }
+
+    protected Map<String, Settings> buildProfileSettings() {
+        // extract default profile first and create standard bootstrap
+        Map<String, Settings> profiles = TransportSettings.TRANSPORT_PROFILES_SETTING.get(settings()).getAsGroups(true);
+        if (!profiles.containsKey(TransportSettings.DEFAULT_PROFILE)) {
+            profiles = new HashMap<>(profiles);
+            profiles.put(TransportSettings.DEFAULT_PROFILE, Settings.EMPTY);
+        }
+        Settings defaultSettings = profiles.get(TransportSettings.DEFAULT_PROFILE);
+        Map<String, Settings> result = new HashMap<>();
+        // loop through all profiles and start them up, special handling for default one
+        for (Map.Entry<String, Settings> entry : profiles.entrySet()) {
+            Settings profileSettings = entry.getValue();
+            String name = entry.getKey();
+
+            if (!Strings.hasLength(name)) {
+                logger.info("transport profile configured without a name. skipping profile with settings [{}]",
+                    profileSettings.toDelimitedString(','));
+                continue;
+            } else if (TransportSettings.DEFAULT_PROFILE.equals(name)) {
+                profileSettings = Settings.builder()
+                    .put(profileSettings)
+                    .put("port", profileSettings.get("port", TransportSettings.PORT.get(this.settings)))
+                    .build();
+            } else if (profileSettings.get("port") == null) {
+                // if profile does not have a port, skip it
+                logger.info("No port configured for profile [{}], not binding", name);
+                continue;
+            }
+            Settings mergedSettings = Settings.builder()
+                .put(defaultSettings)
+                .put(profileSettings)
+                .build();
+            result.put(name, mergedSettings);
+        }
+        return result;
+    }
+
+    @Override
+    public List<String> getLocalAddresses() {
+        List<String> local = new ArrayList<>();
+        local.add("127.0.0.1");
+        // check if v6 is supported, if so, v4 will also work via mapped addresses.
+        if (NetworkUtils.SUPPORTS_V6) {
+            local.add("[::1]"); // may get ports appended!
+        }
+        return local;
+    }
+
+    protected void bindServer(final String name, final Settings settings) {
+        // Bind and start to accept incoming connections.
+        InetAddress hostAddresses[];
+        String bindHosts[] = settings.getAsArray("bind_host", null);
+        try {
+            hostAddresses = networkService.resolveBindHostAddresses(bindHosts);
+        } catch (IOException e) {
+            throw new BindTransportException("Failed to resolve host " + Arrays.toString(bindHosts) + "", e);
+        }
+        if (logger.isDebugEnabled()) {
+            String[] addresses = new String[hostAddresses.length];
+            for (int i = 0; i < hostAddresses.length; i++) {
+                addresses[i] = NetworkAddress.format(hostAddresses[i]);
+            }
+            logger.debug("binding server bootstrap to: {}", (Object)addresses);
+        }
+
+        assert hostAddresses.length > 0;
+
+        List<InetSocketAddress> boundAddresses = new ArrayList<>();
+        for (InetAddress hostAddress : hostAddresses) {
+            boundAddresses.add(bindToPort(name, hostAddress, settings.get("port")));
+        }
+
+        final BoundTransportAddress boundTransportAddress = createBoundTransportAddress(name, settings, boundAddresses);
+
+        if (TransportSettings.DEFAULT_PROFILE.equals(name)) {
+            this.boundAddress = boundTransportAddress;
+        } else {
+            profileBoundAddresses.put(name, boundTransportAddress);
+        }
+    }
+
+    protected InetSocketAddress bindToPort(final String name, final InetAddress hostAddress, String port) {
+        PortsRange portsRange = new PortsRange(port);
+        final AtomicReference<Exception> lastException = new AtomicReference<>();
+        final AtomicReference<InetSocketAddress> boundSocket = new AtomicReference<>();
+        boolean success = portsRange.iterate(portNumber -> {
+            try {
+                Channel channel = bind(name, new InetSocketAddress(hostAddress, portNumber));
+                synchronized (serverChannels) {
+                    List<Channel> list = serverChannels.get(name);
+                    if (list == null) {
+                        list = new ArrayList<>();
+                        serverChannels.put(name, list);
+                    }
+                    list.add(channel);
+                    boundSocket.set(getLocalAddress(channel));
+                }
+            } catch (Exception e) {
+                lastException.set(e);
+                return false;
+            }
+            return true;
+        });
+        if (!success) {
+            throw new BindTransportException("Failed to bind to [" + port + "]", lastException.get());
+        }
+
+        if (logger.isDebugEnabled()) {
+            logger.debug("Bound profile [{}] to address {{}}", name, NetworkAddress.format(boundSocket.get()));
+        }
+
+        return boundSocket.get();
+    }
+
+    private BoundTransportAddress createBoundTransportAddress(String name, Settings profileSettings,
+                                                              List<InetSocketAddress> boundAddresses) {
+        String[] boundAddressesHostStrings = new String[boundAddresses.size()];
+        TransportAddress[] transportBoundAddresses = new TransportAddress[boundAddresses.size()];
+        for (int i = 0; i < boundAddresses.size(); i++) {
+            InetSocketAddress boundAddress = boundAddresses.get(i);
+            boundAddressesHostStrings[i] = boundAddress.getHostString();
+            transportBoundAddresses[i] = new InetSocketTransportAddress(boundAddress);
+        }
+
+        final String[] publishHosts;
+        if (TransportSettings.DEFAULT_PROFILE.equals(name)) {
+            publishHosts = TransportSettings.PUBLISH_HOST.get(settings).toArray(Strings.EMPTY_ARRAY);
+        } else {
+            publishHosts = profileSettings.getAsArray("publish_host", boundAddressesHostStrings);
+        }
+
+        final InetAddress publishInetAddress;
+        try {
+            publishInetAddress = networkService.resolvePublishHostAddresses(publishHosts);
+        } catch (Exception e) {
+            throw new BindTransportException("Failed to resolve publish address", e);
+        }
+
+        final int publishPort = resolvePublishPort(name, settings, profileSettings, boundAddresses, publishInetAddress);
+        final TransportAddress publishAddress = new InetSocketTransportAddress(new InetSocketAddress(publishInetAddress, publishPort));
+        return new BoundTransportAddress(transportBoundAddresses, publishAddress);
+    }
+
+    // package private for tests
+    public static int resolvePublishPort(String profileName, Settings settings, Settings profileSettings,
+                                         List<InetSocketAddress> boundAddresses, InetAddress publishInetAddress) {
+        int publishPort;
+        if (TransportSettings.DEFAULT_PROFILE.equals(profileName)) {
+            publishPort = TransportSettings.PUBLISH_PORT.get(settings);
+        } else {
+            publishPort = profileSettings.getAsInt("publish_port", -1);
+        }
+
+        // if port not explicitly provided, search for port of address in boundAddresses that matches publishInetAddress
+        if (publishPort < 0) {
+            for (InetSocketAddress boundAddress : boundAddresses) {
+                InetAddress boundInetAddress = boundAddress.getAddress();
+                if (boundInetAddress.isAnyLocalAddress() || boundInetAddress.equals(publishInetAddress)) {
+                    publishPort = boundAddress.getPort();
+                    break;
+                }
+            }
+        }
+
+        // if no matching boundAddress found, check if there is a unique port for all bound addresses
+        if (publishPort < 0) {
+            final IntSet ports = new IntHashSet();
+            for (InetSocketAddress boundAddress : boundAddresses) {
+                ports.add(boundAddress.getPort());
+            }
+            if (ports.size() == 1) {
+                publishPort = ports.iterator().next().value;
+            }
+        }
+
+        if (publishPort < 0) {
+            String profileExplanation = TransportSettings.DEFAULT_PROFILE.equals(profileName) ? "" : " for profile " + profileName;
+            throw new BindTransportException("Failed to auto-resolve publish port" + profileExplanation + ", multiple bound addresses " +
+                boundAddresses + " with distinct ports and none of them matched the publish address (" + publishInetAddress + "). " +
+                "Please specify a unique port by setting " + TransportSettings.PORT.getKey() + " or " +
+                TransportSettings.PUBLISH_PORT.getKey());
+        }
+        return publishPort;
+    }
+
+    @Override
+    public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws Exception {
+        return parse(address, settings.get("transport.profiles.default.port", TransportSettings.PORT.get(settings)), perAddressLimit);
+    }
+
+    // this code is a take on guava's HostAndPort, like a HostAndPortRange
+
+    // pattern for validating ipv6 bracket addresses.
+    // not perfect, but PortsRange should take care of any port range validation, not a regex
+    private static final Pattern BRACKET_PATTERN = Pattern.compile("^\\[(.*:.*)\\](?::([\\d\\-]*))?$");
+
+    /** parse a hostname+port range spec into its equivalent addresses */
+    static TransportAddress[] parse(String hostPortString, String defaultPortRange, int perAddressLimit) throws UnknownHostException {
+        Objects.requireNonNull(hostPortString);
+        String host;
+        String portString = null;
+
+        if (hostPortString.startsWith("[")) {
+            // Parse a bracketed host, typically an IPv6 literal.
+            Matcher matcher = BRACKET_PATTERN.matcher(hostPortString);
+            if (!matcher.matches()) {
+                throw new IllegalArgumentException("Invalid bracketed host/port range: " + hostPortString);
+            }
+            host = matcher.group(1);
+            portString = matcher.group(2);  // could be null
+        } else {
+            int colonPos = hostPortString.indexOf(':');
+            if (colonPos >= 0 && hostPortString.indexOf(':', colonPos + 1) == -1) {
+                // Exactly 1 colon.  Split into host:port.
+                host = hostPortString.substring(0, colonPos);
+                portString = hostPortString.substring(colonPos + 1);
+            } else {
+                // 0 or 2+ colons.  Bare hostname or IPv6 literal.
+                host = hostPortString;
+                // 2+ colons and not bracketed: exception
+                if (colonPos >= 0) {
+                    throw new IllegalArgumentException("IPv6 addresses must be bracketed: " + hostPortString);
+                }
+            }
+        }
+
+        // if port isn't specified, fill with the default
+        if (portString == null || portString.isEmpty()) {
+            portString = defaultPortRange;
+        }
+
+        // generate address for each port in the range
+        Set<InetAddress> addresses = new HashSet<>(Arrays.asList(InetAddress.getAllByName(host)));
+        List<TransportAddress> transportAddresses = new ArrayList<>();
+        int[] ports = new PortsRange(portString).ports();
+        int limit = Math.min(ports.length, perAddressLimit);
+        for (int i = 0; i < limit; i++) {
+            for (InetAddress address : addresses) {
+                transportAddresses.add(new InetSocketTransportAddress(address, ports[i]));
+            }
+        }
+        return transportAddresses.toArray(new TransportAddress[transportAddresses.size()]);
+    }
+
+    @Override
+    protected final void doClose() {
+    }
+
+    @Override
+    protected final void doStop() {
+        final CountDownLatch latch = new CountDownLatch(1);
+        // make sure we run it on another thread than a possible IO handler thread
+        threadPool.generic().execute(() -> {
+            globalLock.writeLock().lock();
+            try {
+                for (Iterator<NodeChannels> it = connectedNodes.values().iterator(); it.hasNext(); ) {
+                    NodeChannels nodeChannels = it.next();
+                    it.remove();
+                    nodeChannels.close();
+                }
+
+                for (Map.Entry<String, List<Channel>> entry : serverChannels.entrySet()) {
+                    try {
+                        closeChannels(entry.getValue());
+                    } catch (Throwable t) {
+                        logger.debug("Error closing serverChannel for profile [{}]", t, entry.getKey());
+                    }
+                }
+                try {
+                    stopInternal();
+                } finally {
+                    for (Iterator<NodeChannels> it = connectedNodes.values().iterator(); it.hasNext(); ) {
+                        NodeChannels nodeChannels = it.next();
+                        it.remove();
+                        nodeChannels.close();
+                    }
+                }
+
+            } finally {
+                globalLock.writeLock().unlock();
+                latch.countDown();
+            }
+        });
+
+        try {
+            latch.await(30, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+            Thread.interrupted();
+            // ignore
+        }
+    }
+
+    protected void onException(Channel channel, Throwable e) {
+        if (!lifecycle.started()) {
+            // ignore
+            return;
+        }
+        if (isCloseConnectionException(e)) {
+            logger.trace("close connection exception caught on transport layer [{}], disconnecting from relevant node", e,
+                channel);
+            // close the channel, which will cause a node to be disconnected if relevant
+            disconnectFromNodeChannel(channel, e);
+        } else if (isConnectException(e)) {
+            logger.trace("connect exception caught on transport layer [{}]", e, channel);
+            // close the channel as safe measure, which will cause a node to be disconnected if relevant
+            disconnectFromNodeChannel(channel, e);
+        } else if (e instanceof BindException) {
+            logger.trace("bind exception caught on transport layer [{}]", e, channel);
+            // close the channel as safe measure, which will cause a node to be disconnected if relevant
+            disconnectFromNodeChannel(channel, e);
+        } else if (e instanceof CancelledKeyException) {
+            logger.trace("cancelled key exception caught on transport layer [{}], disconnecting from relevant node", e,
+                channel);
+            // close the channel as safe measure, which will cause a node to be disconnected if relevant
+            disconnectFromNodeChannel(channel, e);
+        } else if (e instanceof TcpTransport.HttpOnTransportException) {
+            // in case we are able to return data, serialize the exception content and sent it back to the client
+            if (isOpen(channel)) {
+                sendMessage(channel, new BytesArray(e.getMessage().getBytes(StandardCharsets.UTF_8)), () -> {}, true);
+            }
+        } else {
+            logger.warn("exception caught on transport layer [{}], closing connection", e, channel);
+            // close the channel, which will cause a node to be disconnected if relevant
+            disconnectFromNodeChannel(channel, e);
+        }
+    }
+
+    /**
+     * Returns the channels local address
+     */
+    protected abstract InetSocketAddress getLocalAddress(Channel channel);
+
+    /**
+     * Binds to the given {@link InetSocketAddress}
+     * @param name the profile name
+     * @param address the address to bind to
+     */
+    protected abstract Channel bind(String name, InetSocketAddress address);
+
+    /**
+     * Closes all channels in this list
+     */
+    protected abstract void closeChannels(List<Channel> channel);
+
+    /**
+     * Connects to the given node in a light way. This means we are not creating multiple connections like we do
+     * for production connections. This connection is for pings or handshakes
+     */
+    protected abstract NodeChannels connectToChannelsLight(DiscoveryNode node);
+
+
+    protected abstract void sendMessage(Channel channel, BytesReference reference, Runnable sendListener, boolean close);
+
+    /**
+     * Connects to the node in a <tt>heavy</tt> way.
+     *
+     * @see #connectToChannelsLight(DiscoveryNode)
+     */
+    protected abstract NodeChannels connectToChannels(DiscoveryNode node);
+
+    /**
+     * Called to tear down internal resources
+     */
+    protected void stopInternal() {}
+
+    public boolean canCompress(TransportRequest request) {
+        return compress;
+    }
+
+    @Override
+    public void sendRequest(final DiscoveryNode node, final long requestId, final String action, final TransportRequest request,
+                            TransportRequestOptions options) throws IOException, TransportException {
+
+        Channel targetChannel = nodeChannel(node, options);
+
+        if (compress) {
+            options = TransportRequestOptions.builder(options).withCompress(true).build();
+        }
+        byte status = 0;
+        status = TransportStatus.setRequest(status);
+        ReleasableBytesStreamOutput bStream = new ReleasableBytesStreamOutput(bigArrays);
+        boolean addedReleaseListener = false;
+        try {
+            bStream.skip(TcpHeader.HEADER_SIZE);
+            StreamOutput stream = bStream;
+            // only compress if asked, and, the request is not bytes, since then only
+            // the header part is compressed, and the "body" can't be extracted as compressed
+            if (options.compress() && canCompress(request)) {
+                status = TransportStatus.setCompress(status);
+                stream = CompressorFactory.COMPRESSOR.streamOutput(stream);
+            }
+
+            // we pick the smallest of the 2, to support both backward and forward compatibility
+            // note, this is the only place we need to do this, since from here on, we use the serialized version
+            // as the version to use also when the node receiving this request will send the response with
+            Version version = Version.smallest(getCurrentVersion(), node.getVersion());
+
+            stream.setVersion(version);
+            threadPool.getThreadContext().writeTo(stream);
+            stream.writeString(action);
+
+            Message<Channel> writeable = prepareSend(node.getVersion(), request, stream, bStream);
+            try (StreamOutput headerOutput = writeable.getHeaderOutput()) {
+                TcpHeader.writeHeader(headerOutput, requestId, status, version,
+                    writeable.size());
+            }
+            final TransportRequestOptions finalOptions = options;
+            Runnable onRequestSent = () -> {
+                try {
+                    Releasables.close(bStream.bytes());
+                } finally {
+                    transportServiceAdapter.onRequestSent(node, requestId, action, request, finalOptions);
+                }
+            };
+            writeable.send(targetChannel, onRequestSent);
+            addedReleaseListener = true;
+
+        } finally {
+            if (!addedReleaseListener) {
+                Releasables.close(bStream.bytes());
+            }
+        }
+    }
+
+    /**
+     * Sends back an error response to the caller via the given channel
+     * @param nodeVersion the caller node version
+     * @param channel the channel to send the response to
+     * @param error the error to return
+     * @param requestId the request ID this response replies to
+     * @param action the action this response replies to
+     */
+    public void sendErrorResponse(Version nodeVersion, Channel channel, final Throwable error, final long requestId,
+                                  final String action) throws IOException {
+        BytesStreamOutput stream = new BytesStreamOutput();
+        stream.setVersion(nodeVersion);
+        stream.skip(TcpHeader.HEADER_SIZE);
+        RemoteTransportException tx = new RemoteTransportException(
+            nodeName(), new InetSocketTransportAddress(getLocalAddress(channel)), action, error);
+        stream.writeThrowable(tx);
+        byte status = 0;
+        status = TransportStatus.setResponse(status);
+        status = TransportStatus.setError(status);
+
+        final BytesReference bytes = stream.bytes();
+        Message<Channel> writeable = prepareSend(nodeVersion, bytes);
+        try (StreamOutput headerOutput = writeable.getHeaderOutput()) {
+            TcpHeader.writeHeader(headerOutput, requestId, status, nodeVersion,
+                writeable.size());
+        }
+        Runnable onRequestSent = () -> {
+            transportServiceAdapter.onResponseSent(requestId, action, error);
+        };
+        writeable.send(channel, onRequestSent);
+    }
+
+    /**
+     * Sends the response to the given channel. This method should be used to send {@link TransportResponse} objects back to the caller.
+     *
+     * @see #sendErrorResponse(Version, Object, Throwable, long, String) for sending back errors to the caller
+     */
+    public void sendResponse(Version nodeVersion, Channel channel, final TransportResponse response, final long requestId,
+                             final String action, TransportResponseOptions options) throws IOException {
+        if (compress) {
+            options = TransportResponseOptions.builder(options).withCompress(true).build();
+        }
+
+        byte status = 0;
+        status = TransportStatus.setResponse(status); // TODO share some code with sendRequest
+        ReleasableBytesStreamOutput bStream = new ReleasableBytesStreamOutput(bigArrays);
+        boolean addedReleaseListener = false;
+        try {
+            bStream.skip(TcpHeader.HEADER_SIZE);
+            StreamOutput stream = bStream;
+            if (options.compress()) {
+                status = TransportStatus.setCompress(status);
+                stream = CompressorFactory.COMPRESSOR.streamOutput(stream);
+            }
+            stream.setVersion(nodeVersion);
+            Message<Channel> writeable = prepareSend(nodeVersion, response, stream, bStream);
+            try (StreamOutput headerOutput = writeable.getHeaderOutput()) {
+                TcpHeader.writeHeader(headerOutput, requestId, status, nodeVersion,
+                    writeable.size());
+            }
+            final TransportResponseOptions finalOptions = options;
+            Runnable onRequestSent = () -> {
+                try {
+                    Releasables.close(bStream.bytes());
+                } finally {
+                    transportServiceAdapter.onResponseSent(requestId, action, response, finalOptions);
+                }
+            };
+            writeable.send(channel, onRequestSent);
+            addedReleaseListener = true;
+
+        } finally {
+            if (!addedReleaseListener) {
+                Releasables.close(bStream.bytes());
+            }
+        }
+    }
+
+    /**
+     * Serializes the given message into a bytes representation and forwards to {@link #prepareSend(Version, TransportMessage,
+     * StreamOutput, ReleasableBytesStream)}
+     */
+    protected Message<Channel> prepareSend(Version nodeVersion, TransportMessage message, StreamOutput stream,
+                                                    ReleasableBytesStream writtenBytes) throws IOException {
+        message.writeTo(stream);
+        stream.close();
+        return prepareSend(nodeVersion, writtenBytes.bytes());
+    }
+
+    /**
+     * prepares a implementation specific message to send across the network
+     */
+    protected abstract Message<Channel> prepareSend(Version nodeVersion, BytesReference bytesReference) throws IOException;
+
+    /**
+     * Allows implementations to transform TransportMessages into implementation specific messages
+     */
+    protected interface Message<Channel> {
+        /**
+         * Creates an output to write the message header to.
+         */
+        StreamOutput getHeaderOutput();
+
+        /**
+         * Returns the size of the message in bytes
+         */
+        int size();
+
+        /**
+         * sends the message to the channel
+         * @param channel the channe to send the message to
+         * @param onRequestSent a callback executed once the message has been fully send
+         */
+        void send(Channel channel, Runnable onRequestSent);
+    }
+
+
+    /**
+     * Validates the first N bytes of the message header and returns <code>true</code> if the message is
+     * a ping message and has no payload ie. isn't a real user level message.
+     *
+     * @throws IllegalStateException if the message is too short, less than the header or less that the header plus the message size
+     * @throws HttpOnTransportException if the message has no valid header and appears to be a HTTP message
+     * @throws IllegalArgumentException if the message is greater that the maximum allowed frame size. This is dependent on the available
+     * memory.
+     */
+    public static boolean validateMessageHeader(BytesReference buffer) throws IOException {
+        final int sizeHeaderLength = TcpHeader.MARKER_BYTES_SIZE + TcpHeader.MESSAGE_LENGTH_SIZE;
+        if (buffer.length() < sizeHeaderLength) {
+            throw new IllegalStateException("message size must be >= to the header size");
+        }
+        int offset = 0;
+        if (buffer.get(offset) != 'E' || buffer.get(offset + 1) != 'S') {
+            // special handling for what is probably HTTP
+            if (bufferStartsWith(buffer, offset, "GET ") ||
+                bufferStartsWith(buffer, offset, "POST ") ||
+                bufferStartsWith(buffer, offset, "PUT ") ||
+                bufferStartsWith(buffer, offset, "HEAD ") ||
+                bufferStartsWith(buffer, offset, "DELETE ") ||
+                bufferStartsWith(buffer, offset, "OPTIONS ") ||
+                bufferStartsWith(buffer, offset, "PATCH ") ||
+                bufferStartsWith(buffer, offset, "TRACE ")) {
+
+                throw new HttpOnTransportException("This is not a HTTP port");
+            }
+
+            // we have 6 readable bytes, show 4 (should be enough)
+            throw new StreamCorruptedException("invalid internal transport message format, got ("
+                + Integer.toHexString(buffer.get(offset) & 0xFF) + ","
+                + Integer.toHexString(buffer.get(offset + 1) & 0xFF) + ","
+                + Integer.toHexString(buffer.get(offset + 2) & 0xFF) + ","
+                + Integer.toHexString(buffer.get(offset + 3) & 0xFF) + ")");
+        }
+
+        final int dataLen;
+        try (StreamInput input = buffer.streamInput()) {
+            input.skip(TcpHeader.MARKER_BYTES_SIZE);
+            dataLen = input.readInt();
+            if (dataLen == PING_DATA_SIZE) {
+                // discard the messages we read and continue, this is achieved by skipping the bytes
+                // and returning null
+                return false;
+            }
+        }
+        if (dataLen <= 0) {
+            throw new StreamCorruptedException("invalid data length: " + dataLen);
+        }
+        // safety against too large frames being sent
+        if (dataLen > NINETY_PER_HEAP_SIZE) {
+            throw new IllegalArgumentException("transport content length received [" + new ByteSizeValue(dataLen) + "] exceeded ["
+                + new ByteSizeValue(NINETY_PER_HEAP_SIZE) + "]");
+        }
+
+        if (buffer.length() < dataLen + sizeHeaderLength) {
+            throw new IllegalStateException("buffer must be >= to the message size but wasn't");
+        }
+        return true;
+    }
+
+    private static boolean bufferStartsWith(BytesReference buffer, int offset, String method) {
+        char[] chars = method.toCharArray();
+        for (int i = 0; i < chars.length; i++) {
+            if (buffer.get(offset+ i) != chars[i]) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    /**
+     * A helper exception to mark an incoming connection as potentially being HTTP
+     * so an appropriate error code can be returned
+     */
+    public static class HttpOnTransportException extends ElasticsearchException {
+
+        public HttpOnTransportException(String msg) {
+            super(msg);
+        }
+
+        @Override
+        public RestStatus status() {
+            return RestStatus.BAD_REQUEST;
+        }
+
+        public HttpOnTransportException(StreamInput in) throws IOException{
+            super(in);
+        }
+    }
+
+    protected abstract boolean isOpen(Channel channel);
+
+    /**
+     * This method handles the message receive part for both request and responses
+     */
+    public final void messageReceived(BytesReference reference, Channel channel, String profileName,
+                                      InetSocketAddress remoteAddress, int messageLengthBytes) throws IOException {
+        final int totalMessageSize = messageLengthBytes + TcpHeader.MARKER_BYTES_SIZE + TcpHeader.MESSAGE_LENGTH_SIZE;
+        transportServiceAdapter.received(totalMessageSize);
+        // we have additional bytes to read, outside of the header
+        boolean hasMessageBytesToRead = (totalMessageSize - TcpHeader.HEADER_SIZE) > 0;
+        StreamInput streamIn = reference.streamInput();
+        boolean success = false;
+        try (ThreadContext.StoredContext tCtx = threadPool.getThreadContext().stashContext()) {
+            long requestId = streamIn.readLong();
+            byte status = streamIn.readByte();
+            Version version = Version.fromId(streamIn.readInt());
+            if (TransportStatus.isCompress(status) && hasMessageBytesToRead && streamIn.available() > 0) {
+                Compressor compressor;
+                try {
+                    final int bytesConsumed = TcpHeader.REQUEST_ID_SIZE + TcpHeader.STATUS_SIZE + TcpHeader.VERSION_ID_SIZE;
+                    compressor = CompressorFactory.compressor(reference.slice(bytesConsumed, reference.length() - bytesConsumed));
+                } catch (NotCompressedException ex) {
+                    int maxToRead = Math.min(reference.length(), 10);
+                    StringBuilder sb = new StringBuilder("stream marked as compressed, but no compressor found, first [").append(maxToRead)
+                        .append("] content bytes out of [").append(reference.length())
+                        .append("] readable bytes with message size [").append(messageLengthBytes).append("] ").append("] are [");
+                    for (int i = 0; i < maxToRead; i++) {
+                        sb.append(reference.get(i)).append(",");
+                    }
+                    sb.append("]");
+                    throw new IllegalStateException(sb.toString());
+                }
+                streamIn = compressor.streamInput(streamIn);
+            }
+            if (version.onOrAfter(Version.CURRENT.minimumCompatibilityVersion()) == false || version.major != Version.CURRENT.major) {
+                throw new IllegalStateException("Received message from unsupported version: [" + version
+                    + "] minimal compatible version is: [" +Version.CURRENT.minimumCompatibilityVersion() + "]");
+            }
+            streamIn = new NamedWriteableAwareStreamInput(streamIn, namedWriteableRegistry);
+            streamIn.setVersion(version);
+            if (TransportStatus.isRequest(status)) {
+                threadPool.getThreadContext().readHeaders(streamIn);
+                handleRequest(channel, profileName, streamIn, requestId, messageLengthBytes, version, remoteAddress);
+            } else {
+                final TransportResponseHandler<?> handler = transportServiceAdapter.onResponseReceived(requestId);
+                // ignore if its null, the adapter logs it
+                if (handler != null) {
+                    if (TransportStatus.isError(status)) {
+                        handlerResponseError(streamIn, handler);
+                    } else {
+                        handleResponse(remoteAddress, streamIn, handler);
+                    }
+                    // Check the entire message has been read
+                    final int nextByte = streamIn.read();
+                    // calling read() is useful to make sure the message is fully read, even if there is an EOS marker
+                    if (nextByte != -1) {
+                        throw new IllegalStateException("Message not fully read (response) for requestId [" + requestId + "], handler ["
+                            + handler + "], error [" + TransportStatus.isError(status) + "]; resetting");
+                    }
+                }
+            }
+            success = true;
+        } finally {
+            if (success) {
+                IOUtils.close(streamIn);
+            } else {
+                IOUtils.closeWhileHandlingException(streamIn);
+            }
+        }
+    }
+
+    private void handleResponse(InetSocketAddress remoteAddress, final StreamInput stream, final TransportResponseHandler handler) {
+        final TransportResponse response = handler.newInstance();
+        response.remoteAddress(new InetSocketTransportAddress(remoteAddress));
+        try {
+            response.readFrom(stream);
+        } catch (Throwable e) {
+            handleException(handler, new TransportSerializationException(
+                "Failed to deserialize response of type [" + response.getClass().getName() + "]", e));
+            return;
+        }
+        threadPool.executor(handler.executor()).execute(new AbstractRunnable() {
+            @Override
+            public void onFailure(Throwable t) {
+                handleException(handler, new ResponseHandlerFailureTransportException(t));
+            }
+
+            @Override
+            protected void doRun() throws Exception {
+                handler.handleResponse(response);
+            }});
+
+    }
+
+    /**
+     * Executed for a received response error
+     */
+    private void handlerResponseError(StreamInput stream, final TransportResponseHandler handler) {
+        Throwable error;
+        try {
+            error = stream.readThrowable();
+        } catch (Throwable e) {
+            error = new TransportSerializationException("Failed to deserialize exception response from stream", e);
+        }
+        handleException(handler, error);
+    }
+
+    private void handleException(final TransportResponseHandler handler, Throwable error) {
+        if (!(error instanceof RemoteTransportException)) {
+            error = new RemoteTransportException(error.getMessage(), error);
+        }
+        final RemoteTransportException rtx = (RemoteTransportException) error;
+        threadPool.executor(handler.executor()).execute(() -> {
+            try {
+                handler.handleException(rtx);
+            } catch (Throwable e) {
+                logger.error("failed to handle exception response [{}]", e, handler);
+            }
+        });
+    }
+
+    protected String handleRequest(Channel channel, String profileName, final StreamInput stream, long requestId,
+                                   int messageLengthBytes, Version version, InetSocketAddress remoteAddress) throws IOException {
+        final String action = stream.readString();
+        transportServiceAdapter.onRequestReceived(requestId, action);
+        TransportChannel transportChannel = null;
+        try {
+            final RequestHandlerRegistry reg = transportServiceAdapter.getRequestHandler(action);
+            if (reg == null) {
+                throw new ActionNotFoundTransportException(action);
+            }
+            if (reg.canTripCircuitBreaker()) {
+                getInFlightRequestBreaker().addEstimateBytesAndMaybeBreak(messageLengthBytes, "<transport_request>");
+            } else {
+                getInFlightRequestBreaker().addWithoutBreaking(messageLengthBytes);
+            }
+            transportChannel = new TcpTransportChannel<>(this, channel, transportName, action, requestId, version, profileName,
+                messageLengthBytes);
+            final TransportRequest request = reg.newRequest();
+            request.remoteAddress(new InetSocketTransportAddress(remoteAddress));
+            request.readFrom(stream);
+            // in case we throw an exception, i.e. when the limit is hit, we don't want to verify
+            validateRequest(stream, requestId, action);
+            threadPool.executor(reg.getExecutor()).execute(new RequestHandler(reg, request, transportChannel));
+        } catch (Throwable e) {
+            // the circuit breaker tripped
+            if (transportChannel == null) {
+                transportChannel = new TcpTransportChannel<>(this, channel, transportName, action, requestId, version, profileName, 0);
+            }
+            try {
+                transportChannel.sendResponse(e);
+            } catch (IOException e1) {
+                logger.warn("Failed to send error message back to client for action [{}]", e, action);
+                logger.warn("Actual Exception", e1);
+            }
+        }
+        return action;
+    }
+
+    // This template method is needed to inject custom error checking logic in tests.
+    protected void validateRequest(StreamInput stream, long requestId, String action) throws IOException {
+        final int nextByte = stream.read();
+        // calling read() is useful to make sure the message is fully read, even if there some kind of EOS marker
+        if (nextByte != -1) {
+            throw new IllegalStateException("Message not fully read (request) for requestId [" + requestId + "], action [" + action
+                + "], available [" + stream.available() + "]; resetting");
+        }
+    }
+
+    class RequestHandler extends AbstractRunnable {
+        private final RequestHandlerRegistry reg;
+        private final TransportRequest request;
+        private final TransportChannel transportChannel;
+
+        public RequestHandler(RequestHandlerRegistry reg, TransportRequest request, TransportChannel transportChannel) {
+            this.reg = reg;
+            this.request = request;
+            this.transportChannel = transportChannel;
+        }
+
+        @SuppressWarnings({"unchecked"})
+        @Override
+        protected void doRun() throws Exception {
+            reg.processMessageReceived(request, transportChannel);
+        }
+
+        @Override
+        public boolean isForceExecution() {
+            return reg.isForceExecution();
+        }
+
+        @Override
+        public void onFailure(Throwable e) {
+            if (lifecycleState() == Lifecycle.State.STARTED) {
+                // we can only send a response transport is started....
+                try {
+                    transportChannel.sendResponse(e);
+                } catch (Throwable e1) {
+                    logger.warn("Failed to send error message back to client for action [{}]", e1, reg.getAction());
+                    logger.warn("Actual Exception", e);
+                }
+            }
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java b/core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java
new file mode 100644
index 00000000000..74bdad48746
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.transport;
+
+import org.elasticsearch.Version;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ *
+ */
+public final class TcpTransportChannel<Channel> implements TransportChannel {
+    private final TcpTransport<Channel> transport;
+    protected final Version version;
+    protected final String action;
+    protected final long requestId;
+    private final String profileName;
+    private final long reservedBytes;
+    private final AtomicBoolean released = new AtomicBoolean();
+    private final String channelType;
+    private final Channel channel;
+
+    public TcpTransportChannel(TcpTransport<Channel> transport, Channel channel, String channelType, String action,
+                               long requestId, Version version, String profileName, long reservedBytes) {
+        this.version = version;
+        this.channel = channel;
+        this.transport = transport;
+        this.action = action;
+        this.requestId = requestId;
+        this.profileName = profileName;
+        this.reservedBytes = reservedBytes;
+        this.channelType = channelType;
+    }
+
+    @Override
+    public final String getProfileName() {
+        return profileName;
+    }
+
+    @Override
+    public final String action() {
+        return this.action;
+    }
+
+    @Override
+    public final void sendResponse(TransportResponse response) throws IOException {
+        sendResponse(response, TransportResponseOptions.EMPTY);
+    }
+
+    @Override
+    public final void sendResponse(TransportResponse response, TransportResponseOptions options) throws IOException {
+        release();
+        transport.sendResponse(version, channel, response, requestId, action, options);
+
+    }
+
+    @Override
+    public void sendResponse(Throwable error) throws IOException {
+        release();
+       transport.sendErrorResponse(version, channel, error, requestId, action);
+    }
+
+    private void release() {
+        // attempt to release once atomically
+        if (released.compareAndSet(false, true) == false) {
+            throw new IllegalStateException("reserved bytes are already released");
+        }
+        transport.getInFlightRequestBreaker().addWithoutBreaking(-reservedBytes);
+    }
+
+    @Override
+    public final long getRequestId() {
+        return requestId;
+    }
+
+    @Override
+    public final String getChannelType() {
+        return channelType;
+    }
+
+    public Channel getChannel() {
+        return channel;
+    }
+
+}
+
diff --git a/core/src/main/java/org/elasticsearch/transport/Transport.java b/core/src/main/java/org/elasticsearch/transport/Transport.java
index 76793d5598f..754d2105e83 100644
--- a/core/src/main/java/org/elasticsearch/transport/Transport.java
+++ b/core/src/main/java/org/elasticsearch/transport/Transport.java
@@ -20,6 +20,8 @@
 package org.elasticsearch.transport;
 
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.breaker.NoopCircuitBreaker;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -94,4 +96,9 @@ public interface Transport extends LifecycleComponent<Transport> {
     long serverOpen();
 
     List<String> getLocalAddresses();
+
+    default CircuitBreaker getInFlightRequestBreaker() {
+        return new NoopCircuitBreaker("in-flight-noop");
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/transport/Transports.java b/core/src/main/java/org/elasticsearch/transport/Transports.java
index 68d828fc72f..1186c821309 100644
--- a/core/src/main/java/org/elasticsearch/transport/Transports.java
+++ b/core/src/main/java/org/elasticsearch/transport/Transports.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.transport;
 
 import org.elasticsearch.transport.local.LocalTransport;
-import org.elasticsearch.transport.netty.NettyTransport;
 
 import java.util.Arrays;
 
@@ -39,10 +38,10 @@ public enum Transports {
         final String threadName = t.getName();
         for (String s : Arrays.asList(
                 LocalTransport.LOCAL_TRANSPORT_THREAD_NAME_PREFIX,
-                NettyTransport.HTTP_SERVER_BOSS_THREAD_NAME_PREFIX,
-                NettyTransport.HTTP_SERVER_WORKER_THREAD_NAME_PREFIX,
-                NettyTransport.TRANSPORT_CLIENT_WORKER_THREAD_NAME_PREFIX,
-                NettyTransport.TRANSPORT_CLIENT_BOSS_THREAD_NAME_PREFIX,
+                TcpTransport.HTTP_SERVER_BOSS_THREAD_NAME_PREFIX,
+                TcpTransport.HTTP_SERVER_WORKER_THREAD_NAME_PREFIX,
+                TcpTransport.TRANSPORT_CLIENT_WORKER_THREAD_NAME_PREFIX,
+                TcpTransport.TRANSPORT_CLIENT_BOSS_THREAD_NAME_PREFIX,
                 TEST_MOCK_TRANSPORT_THREAD_PREFIX)) {
             if (threadName.contains(s)) {
                 return true;
diff --git a/core/src/main/java/org/elasticsearch/common/netty/ChannelBufferBytesReference.java b/core/src/main/java/org/elasticsearch/transport/netty/ChannelBufferBytesReference.java
similarity index 85%
rename from core/src/main/java/org/elasticsearch/common/netty/ChannelBufferBytesReference.java
rename to core/src/main/java/org/elasticsearch/transport/netty/ChannelBufferBytesReference.java
index 42cfe3c611f..60dbdd21bd1 100644
--- a/core/src/main/java/org/elasticsearch/common/netty/ChannelBufferBytesReference.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/ChannelBufferBytesReference.java
@@ -16,13 +16,12 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.common.netty;
+package org.elasticsearch.transport.netty;
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.transport.netty.ChannelBufferStreamInputFactory;
 import org.jboss.netty.buffer.ChannelBuffer;
 
 import java.io.IOException;
@@ -34,9 +33,12 @@ import java.nio.charset.StandardCharsets;
 final class ChannelBufferBytesReference implements BytesReference {
 
     private final ChannelBuffer buffer;
+    private final int size;
 
-    ChannelBufferBytesReference(ChannelBuffer buffer) {
+    ChannelBufferBytesReference(ChannelBuffer buffer, int size) {
         this.buffer = buffer;
+        this.size = size;
+        assert size <= buffer.readableBytes() : "size[" + size +"] > " + buffer.readableBytes();
     }
 
     @Override
@@ -46,25 +48,24 @@ final class ChannelBufferBytesReference implements BytesReference {
 
     @Override
     public int length() {
-        return buffer.readableBytes();
+        return size;
     }
 
     @Override
     public BytesReference slice(int from, int length) {
-        return new ChannelBufferBytesReference(buffer.slice(buffer.readerIndex() + from, length));
+        return new ChannelBufferBytesReference(buffer.slice(buffer.readerIndex() + from, length), length);
     }
 
     @Override
     public StreamInput streamInput() {
-        return ChannelBufferStreamInputFactory.create(buffer.duplicate());
+        return new ChannelBufferStreamInput(buffer.duplicate(), size);
     }
 
     @Override
     public void writeTo(OutputStream os) throws IOException {
-        buffer.getBytes(buffer.readerIndex(), os, length());
+        buffer.getBytes(buffer.readerIndex(), os, size);
     }
 
-    @Override
     public byte[] toBytes() {
         return copyBytesArray().toBytes();
     }
@@ -72,7 +73,7 @@ final class ChannelBufferBytesReference implements BytesReference {
     @Override
     public BytesArray toBytesArray() {
         if (buffer.hasArray()) {
-            return new BytesArray(buffer.array(), buffer.arrayOffset() + buffer.readerIndex(), buffer.readableBytes());
+            return new BytesArray(buffer.array(), buffer.arrayOffset() + buffer.readerIndex(), size);
         }
         return copyBytesArray();
     }
@@ -111,7 +112,7 @@ final class ChannelBufferBytesReference implements BytesReference {
     @Override
     public BytesRef toBytesRef() {
         if (buffer.hasArray()) {
-            return new BytesRef(buffer.array(), buffer.arrayOffset() + buffer.readerIndex(), buffer.readableBytes());
+            return new BytesRef(buffer.array(), buffer.arrayOffset() + buffer.readerIndex(), size);
         }
         byte[] copy = new byte[buffer.readableBytes()];
         buffer.getBytes(buffer.readerIndex(), copy);
@@ -120,7 +121,7 @@ final class ChannelBufferBytesReference implements BytesReference {
 
     @Override
     public BytesRef copyBytesRef() {
-        byte[] copy = new byte[buffer.readableBytes()];
+        byte[] copy = new byte[size];
         buffer.getBytes(buffer.readerIndex(), copy);
         return new BytesRef(copy);
     }
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java b/core/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java
index 9e2b43d43db..3b95ddd74c7 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInput.java
@@ -22,16 +22,14 @@ package org.elasticsearch.transport.netty;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.netty.NettyUtils;
 import org.jboss.netty.buffer.ChannelBuffer;
 
-import java.io.EOFException;
 import java.io.IOException;
 
 /**
  * A Netty {@link org.jboss.netty.buffer.ChannelBuffer} based {@link org.elasticsearch.common.io.stream.StreamInput}.
  */
-public class ChannelBufferStreamInput extends StreamInput {
+class ChannelBufferStreamInput extends StreamInput {
 
     private final ChannelBuffer buffer;
     private final int startIndex;
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInputFactory.java b/core/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInputFactory.java
deleted file mode 100644
index 554f710ec4d..00000000000
--- a/core/src/main/java/org/elasticsearch/transport/netty/ChannelBufferStreamInputFactory.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.transport.netty;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.jboss.netty.buffer.ChannelBuffer;
-
-/**
- */
-public class ChannelBufferStreamInputFactory {
-
-    public static StreamInput create(ChannelBuffer buffer) {
-        return new ChannelBufferStreamInput(buffer, buffer.readableBytes());
-    }
-
-    public static StreamInput create(ChannelBuffer buffer, int size) {
-        return new ChannelBufferStreamInput(buffer, size);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java b/core/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java
deleted file mode 100644
index 3274aa7b975..00000000000
--- a/core/src/main/java/org/elasticsearch/transport/netty/MessageChannelHandler.java
+++ /dev/null
@@ -1,437 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.transport.netty;
-
-import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.Version;
-import org.elasticsearch.common.component.Lifecycle;
-import org.elasticsearch.common.compress.Compressor;
-import org.elasticsearch.common.compress.CompressorFactory;
-import org.elasticsearch.common.compress.NotCompressedException;
-import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.netty.NettyUtils;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
-import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.common.util.concurrent.ThreadContext;
-import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.ActionNotFoundTransportException;
-import org.elasticsearch.transport.RemoteTransportException;
-import org.elasticsearch.transport.RequestHandlerRegistry;
-import org.elasticsearch.transport.ResponseHandlerFailureTransportException;
-import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.transport.TransportResponse;
-import org.elasticsearch.transport.TransportResponseHandler;
-import org.elasticsearch.transport.TransportSerializationException;
-import org.elasticsearch.transport.TransportServiceAdapter;
-import org.elasticsearch.transport.Transports;
-import org.elasticsearch.transport.support.TransportStatus;
-import org.jboss.netty.buffer.ChannelBuffer;
-import org.jboss.netty.channel.Channel;
-import org.jboss.netty.channel.ChannelHandlerContext;
-import org.jboss.netty.channel.ExceptionEvent;
-import org.jboss.netty.channel.MessageEvent;
-import org.jboss.netty.channel.SimpleChannelUpstreamHandler;
-import org.jboss.netty.channel.WriteCompletionEvent;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-/**
- * A handler (must be the last one!) that does size based frame decoding and forwards the actual message
- * to the relevant action.
- */
-public class MessageChannelHandler extends SimpleChannelUpstreamHandler {
-
-    protected final ESLogger logger;
-    protected final ThreadPool threadPool;
-    protected final TransportServiceAdapter transportServiceAdapter;
-    protected final NettyTransport transport;
-    protected final String profileName;
-    private final ThreadContext threadContext;
-
-    public MessageChannelHandler(NettyTransport transport, ESLogger logger, String profileName) {
-        this.threadPool = transport.threadPool();
-        this.threadContext = threadPool.getThreadContext();
-        this.transportServiceAdapter = transport.transportServiceAdapter();
-        this.transport = transport;
-        this.logger = logger;
-        this.profileName = profileName;
-    }
-
-    @Override
-    public void writeComplete(ChannelHandlerContext ctx, WriteCompletionEvent e) throws Exception {
-        transportServiceAdapter.sent(e.getWrittenAmount());
-        super.writeComplete(ctx, e);
-    }
-
-    @Override
-    public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
-        Transports.assertTransportThread();
-        Object m = e.getMessage();
-        if (!(m instanceof ChannelBuffer)) {
-            ctx.sendUpstream(e);
-            return;
-        }
-        ChannelBuffer buffer = (ChannelBuffer) m;
-        Marker marker = new Marker(buffer);
-        int size = marker.messageSizeWithRemainingHeaders();
-        transportServiceAdapter.received(marker.messageSizeWithAllHeaders());
-
-        // we have additional bytes to read, outside of the header
-        boolean hasMessageBytesToRead = marker.messageSize() != 0;
-
-        // netty always copies a buffer, either in NioWorker in its read handler, where it copies to a fresh
-        // buffer, or in the cumulation buffer, which is cleaned each time
-        StreamInput streamIn = ChannelBufferStreamInputFactory.create(buffer, size);
-        boolean success = false;
-        try (ThreadContext.StoredContext tCtx = threadContext.stashContext()) {
-            long requestId = streamIn.readLong();
-            byte status = streamIn.readByte();
-            Version version = Version.fromId(streamIn.readInt());
-
-            if (TransportStatus.isCompress(status) && hasMessageBytesToRead && buffer.readable()) {
-                Compressor compressor;
-                try {
-                    compressor = CompressorFactory.compressor(NettyUtils.toBytesReference(buffer));
-                } catch (NotCompressedException ex) {
-                    int maxToRead = Math.min(buffer.readableBytes(), 10);
-                    int offset = buffer.readerIndex();
-                    StringBuilder sb = new StringBuilder("stream marked as compressed, but no compressor found, first [").append(maxToRead)
-                            .append("] content bytes out of [").append(buffer.readableBytes())
-                            .append("] readable bytes with message size [").append(size).append("] ").append("] are [");
-                    for (int i = 0; i < maxToRead; i++) {
-                        sb.append(buffer.getByte(offset + i)).append(",");
-                    }
-                    sb.append("]");
-                    throw new IllegalStateException(sb.toString());
-                }
-                streamIn = compressor.streamInput(streamIn);
-            }
-            if (version.onOrAfter(Version.CURRENT.minimumCompatibilityVersion()) == false || version.major != Version.CURRENT.major) {
-                throw new IllegalStateException("Received message from unsupported version: [" + version
-                    + "] minimal compatible version is: [" +Version.CURRENT.minimumCompatibilityVersion() + "]");
-            }
-            streamIn.setVersion(version);
-            if (TransportStatus.isRequest(status)) {
-                threadContext.readHeaders(streamIn);
-                handleRequest(ctx.getChannel(), marker, streamIn, requestId, size, version);
-            } else {
-                TransportResponseHandler<?> handler = transportServiceAdapter.onResponseReceived(requestId);
-                // ignore if its null, the adapter logs it
-                if (handler != null) {
-                    if (TransportStatus.isError(status)) {
-                        handlerResponseError(streamIn, handler);
-                    } else {
-                        handleResponse(ctx.getChannel(), streamIn, handler);
-                    }
-                    marker.validateResponse(streamIn, requestId, handler, TransportStatus.isError(status));
-                }
-            }
-            success = true;
-        } finally {
-            try {
-                if (success) {
-                    IOUtils.close(streamIn);
-                } else {
-                    IOUtils.closeWhileHandlingException(streamIn);
-                }
-            } finally {
-                // Set the expected position of the buffer, no matter what happened
-                buffer.readerIndex(marker.expectedReaderIndex());
-            }
-        }
-    }
-
-    protected void handleResponse(Channel channel, StreamInput buffer, final TransportResponseHandler handler) {
-        buffer = new NamedWriteableAwareStreamInput(buffer, transport.namedWriteableRegistry);
-        final TransportResponse response = handler.newInstance();
-        response.remoteAddress(new InetSocketTransportAddress((InetSocketAddress) channel.getRemoteAddress()));
-        response.remoteAddress();
-        try {
-            response.readFrom(buffer);
-        } catch (Throwable e) {
-            handleException(handler, new TransportSerializationException(
-                    "Failed to deserialize response of type [" + response.getClass().getName() + "]", e));
-            return;
-        }
-        try {
-            if (ThreadPool.Names.SAME.equals(handler.executor())) {
-                //noinspection unchecked
-                handler.handleResponse(response);
-            } else {
-                threadPool.executor(handler.executor()).execute(new ResponseHandler(handler, response));
-            }
-        } catch (Throwable e) {
-            handleException(handler, new ResponseHandlerFailureTransportException(e));
-        }
-    }
-
-    private void handlerResponseError(StreamInput buffer, final TransportResponseHandler handler) {
-        Throwable error;
-        try {
-            error = buffer.readThrowable();
-        } catch (Throwable e) {
-            error = new TransportSerializationException("Failed to deserialize exception response from stream", e);
-        }
-        handleException(handler, error);
-    }
-
-    private void handleException(final TransportResponseHandler handler, Throwable error) {
-        if (!(error instanceof RemoteTransportException)) {
-            error = new RemoteTransportException(error.getMessage(), error);
-        }
-        final RemoteTransportException rtx = (RemoteTransportException) error;
-        if (ThreadPool.Names.SAME.equals(handler.executor())) {
-            try {
-                handler.handleException(rtx);
-            } catch (Throwable e) {
-                logger.error("failed to handle exception response [{}]", e, handler);
-            }
-        } else {
-            threadPool.executor(handler.executor()).execute(new Runnable() {
-                @Override
-                public void run() {
-                    try {
-                        handler.handleException(rtx);
-                    } catch (Throwable e) {
-                        logger.error("failed to handle exception response [{}]", e, handler);
-                    }
-                }
-            });
-        }
-    }
-
-    protected String handleRequest(Channel channel, Marker marker, StreamInput buffer, long requestId, int messageLengthBytes,
-                                   Version version) throws IOException {
-        buffer = new NamedWriteableAwareStreamInput(buffer, transport.namedWriteableRegistry);
-        final String action = buffer.readString();
-        transportServiceAdapter.onRequestReceived(requestId, action);
-        NettyTransportChannel transportChannel = null;
-        try {
-            final RequestHandlerRegistry reg = transportServiceAdapter.getRequestHandler(action);
-            if (reg == null) {
-                throw new ActionNotFoundTransportException(action);
-            }
-            if (reg.canTripCircuitBreaker()) {
-                transport.inFlightRequestsBreaker().addEstimateBytesAndMaybeBreak(messageLengthBytes, "<transport_request>");
-            } else {
-                transport.inFlightRequestsBreaker().addWithoutBreaking(messageLengthBytes);
-            }
-            transportChannel = new NettyTransportChannel(transport, transportServiceAdapter, action, channel,
-                requestId, version, profileName, messageLengthBytes);
-            final TransportRequest request = reg.newRequest();
-            request.remoteAddress(new InetSocketTransportAddress((InetSocketAddress) channel.getRemoteAddress()));
-            request.readFrom(buffer);
-            // in case we throw an exception, i.e. when the limit is hit, we don't want to verify
-            validateRequest(marker, buffer, requestId, action);
-            if (ThreadPool.Names.SAME.equals(reg.getExecutor())) {
-                //noinspection unchecked
-                reg.processMessageReceived(request, transportChannel);
-            } else {
-                threadPool.executor(reg.getExecutor()).execute(new RequestHandler(reg, request, transportChannel));
-            }
-        } catch (Throwable e) {
-            // the circuit breaker tripped
-            if (transportChannel == null) {
-                transportChannel = new NettyTransportChannel(transport, transportServiceAdapter, action, channel,
-                    requestId, version, profileName, 0);
-            }
-            try {
-                transportChannel.sendResponse(e);
-            } catch (IOException e1) {
-                logger.warn("Failed to send error message back to client for action [{}]", e, action);
-                logger.warn("Actual Exception", e1);
-            }
-        }
-        return action;
-    }
-
-    // This template method is needed to inject custom error checking logic in tests.
-    protected void validateRequest(Marker marker, StreamInput buffer, long requestId, String action) throws IOException {
-        marker.validateRequest(buffer, requestId, action);
-    }
-
-
-    @Override
-    public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
-        transport.exceptionCaught(ctx, e);
-    }
-
-    class ResponseHandler implements Runnable {
-
-        private final TransportResponseHandler handler;
-        private final TransportResponse response;
-
-        public ResponseHandler(TransportResponseHandler handler, TransportResponse response) {
-            this.handler = handler;
-            this.response = response;
-        }
-
-        @SuppressWarnings({"unchecked"})
-        @Override
-        public void run() {
-            try {
-                handler.handleResponse(response);
-            } catch (Throwable e) {
-                handleException(handler, new ResponseHandlerFailureTransportException(e));
-            }
-        }
-    }
-
-    class RequestHandler extends AbstractRunnable {
-        private final RequestHandlerRegistry reg;
-        private final TransportRequest request;
-        private final NettyTransportChannel transportChannel;
-
-        public RequestHandler(RequestHandlerRegistry reg, TransportRequest request, NettyTransportChannel transportChannel) {
-            this.reg = reg;
-            this.request = request;
-            this.transportChannel = transportChannel;
-        }
-
-        @SuppressWarnings({"unchecked"})
-        @Override
-        protected void doRun() throws Exception {
-            reg.processMessageReceived(request, transportChannel);
-        }
-
-        @Override
-        public boolean isForceExecution() {
-            return reg.isForceExecution();
-        }
-
-        @Override
-        public void onFailure(Throwable e) {
-            if (transport.lifecycleState() == Lifecycle.State.STARTED) {
-                // we can only send a response transport is started....
-                try {
-                    transportChannel.sendResponse(e);
-                } catch (Throwable e1) {
-                    logger.warn("Failed to send error message back to client for action [{}]", e1, reg.getAction());
-                    logger.warn("Actual Exception", e);
-                }
-            }
-        }
-    }
-
-    /**
-     * Internal helper class to store characteristic offsets of a buffer during processing
-     */
-    protected static final class Marker {
-        private final ChannelBuffer buffer;
-        private final int remainingMessageSize;
-        private final int expectedReaderIndex;
-
-        public Marker(ChannelBuffer buffer) {
-            this.buffer = buffer;
-            // when this constructor is called, we have read already two parts of the message header: the marker bytes and the message
-            // message length (see SizeHeaderFrameDecoder). Hence we have to rewind the index for MESSAGE_LENGTH_SIZE bytes to read the
-            // remaining message length again.
-            this.remainingMessageSize = buffer.getInt(buffer.readerIndex() - NettyHeader.MESSAGE_LENGTH_SIZE);
-            this.expectedReaderIndex = buffer.readerIndex() + remainingMessageSize;
-        }
-
-        /**
-         * @return the number of bytes that have yet to be read from the buffer
-         */
-        public int messageSizeWithRemainingHeaders() {
-            return remainingMessageSize;
-        }
-
-        /**
-         * @return the number in bytes for the message including all headers (even the ones that have been read from the buffer already)
-         */
-        public int messageSizeWithAllHeaders() {
-            return remainingMessageSize + NettyHeader.MARKER_BYTES_SIZE + NettyHeader.MESSAGE_LENGTH_SIZE;
-        }
-
-        /**
-         * @return the number of bytes for the message itself (excluding all headers).
-         */
-        public int messageSize() {
-            return messageSizeWithAllHeaders() - NettyHeader.HEADER_SIZE;
-        }
-
-        /**
-         * @return the expected index of the buffer's reader after the message has been consumed entirely.
-         */
-        public int expectedReaderIndex() {
-            return expectedReaderIndex;
-        }
-
-        /**
-         * Validates that a request has been fully read (not too few bytes but also not too many bytes).
-         *
-         * @param stream    A stream that is associated with the buffer that is tracked by this marker.
-         * @param requestId The current request id.
-         * @param action    The currently executed action.
-         * @throws IOException           Iff the stream could not be read.
-         * @throws IllegalStateException Iff the request has not been fully read.
-         */
-        public void validateRequest(StreamInput stream, long requestId, String action) throws IOException {
-            final int nextByte = stream.read();
-            // calling read() is useful to make sure the message is fully read, even if there some kind of EOS marker
-            if (nextByte != -1) {
-                throw new IllegalStateException("Message not fully read (request) for requestId [" + requestId + "], action [" + action
-                    + "], readerIndex [" + buffer.readerIndex() + "] vs expected [" + expectedReaderIndex + "]; resetting");
-            }
-            if (buffer.readerIndex() < expectedReaderIndex) {
-                throw new IllegalStateException("Message is fully read (request), yet there are "
-                    + (expectedReaderIndex - buffer.readerIndex()) + " remaining bytes; resetting");
-            }
-            if (buffer.readerIndex() > expectedReaderIndex) {
-                throw new IllegalStateException(
-                    "Message read past expected size (request) for requestId [" + requestId + "], action [" + action
-                        + "], readerIndex [" + buffer.readerIndex() + "] vs expected [" + expectedReaderIndex + "]; resetting");
-            }
-        }
-
-        /**
-         * Validates that a response has been fully read (not too few bytes but also not too many bytes).
-         *
-         * @param stream    A stream that is associated with the buffer that is tracked by this marker.
-         * @param requestId The corresponding request id for this response.
-         * @param handler   The current response handler.
-         * @param error     Whether validate an error response.
-         * @throws IOException           Iff the stream could not be read.
-         * @throws IllegalStateException Iff the request has not been fully read.
-         */
-        public void validateResponse(StreamInput stream, long requestId,
-                                     TransportResponseHandler<?> handler, boolean error) throws IOException {
-            // Check the entire message has been read
-            final int nextByte = stream.read();
-            // calling read() is useful to make sure the message is fully read, even if there is an EOS marker
-            if (nextByte != -1) {
-                throw new IllegalStateException("Message not fully read (response) for requestId [" + requestId + "], handler ["
-                    + handler + "], error [" + error + "]; resetting");
-            }
-            if (buffer.readerIndex() < expectedReaderIndex) {
-                throw new IllegalStateException("Message is fully read (response), yet there are "
-                    + (expectedReaderIndex - buffer.readerIndex()) + " remaining bytes; resetting");
-            }
-            if (buffer.readerIndex() > expectedReaderIndex) {
-                throw new IllegalStateException("Message read past expected size (response) for requestId [" + requestId
-                    + "], handler [" + handler + "], error [" + error + "]; resetting");
-            }
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/NettyHeader.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyHeader.java
deleted file mode 100644
index 8e4423fb447..00000000000
--- a/core/src/main/java/org/elasticsearch/transport/netty/NettyHeader.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.transport.netty;
-
-import org.elasticsearch.Version;
-import org.jboss.netty.buffer.ChannelBuffer;
-import org.jboss.netty.buffer.ChannelBuffers;
-
-/**
- */
-public class NettyHeader {
-    public static final int MARKER_BYTES_SIZE = 2 * 1;
-
-    public static final int MESSAGE_LENGTH_SIZE = 4;
-
-    public static final int REQUEST_ID_SIZE = 8;
-
-    public static final int STATUS_SIZE = 1;
-
-    public static final int VERSION_ID_SIZE = 4;
-
-    public static final int HEADER_SIZE = MARKER_BYTES_SIZE + MESSAGE_LENGTH_SIZE + REQUEST_ID_SIZE + STATUS_SIZE + VERSION_ID_SIZE;
-
-    /**
-     * The magic number (must be lower than 0) for a ping message. This is handled
-     * specifically in {@link org.elasticsearch.transport.netty.SizeHeaderFrameDecoder}.
-     */
-    public static final int PING_DATA_SIZE = -1;
-    private final static ChannelBuffer pingHeader;
-    static {
-        pingHeader = ChannelBuffers.buffer(6);
-        pingHeader.writeByte('E');
-        pingHeader.writeByte('S');
-        pingHeader.writeInt(PING_DATA_SIZE);
-    }
-
-    /**
-     * A ping header is same as regular header, just with -1 for the size of the message.
-     */
-    public static ChannelBuffer pingHeader() {
-        return pingHeader.duplicate();
-    }
-
-    public static void writeHeader(ChannelBuffer buffer, long requestId, byte status, Version version) {
-        int index = buffer.readerIndex();
-        buffer.setByte(index, 'E');
-        index += 1;
-        buffer.setByte(index, 'S');
-        index += 1;
-        // write the size, the size indicates the remaining message size, not including the size int
-        buffer.setInt(index, buffer.readableBytes() - MARKER_BYTES_SIZE - MESSAGE_LENGTH_SIZE);
-        index += MESSAGE_LENGTH_SIZE;
-        buffer.setLong(index, requestId);
-        index += REQUEST_ID_SIZE;
-        buffer.setByte(index, status);
-        index += STATUS_SIZE;
-        buffer.setInt(index, version.id);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/NettyInternalESLogger.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyInternalESLogger.java
index 2a1fc3226a4..d8307f32244 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/NettyInternalESLogger.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/NettyInternalESLogger.java
@@ -27,11 +27,11 @@ import org.jboss.netty.logging.AbstractInternalLogger;
  *
  */
 @SuppressLoggerChecks(reason = "safely delegates to logger")
-public class NettyInternalESLogger extends AbstractInternalLogger {
+final class NettyInternalESLogger extends AbstractInternalLogger {
 
     private final ESLogger logger;
 
-    public NettyInternalESLogger(ESLogger logger) {
+    NettyInternalESLogger(ESLogger logger) {
         this.logger = logger;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/NettyInternalESLoggerFactory.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyInternalESLoggerFactory.java
deleted file mode 100644
index 70c5e651c2c..00000000000
--- a/core/src/main/java/org/elasticsearch/transport/netty/NettyInternalESLoggerFactory.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.transport.netty;
-
-import org.elasticsearch.common.logging.Loggers;
-import org.jboss.netty.logging.InternalLogger;
-import org.jboss.netty.logging.InternalLoggerFactory;
-
-/**
- *
- */
-public class NettyInternalESLoggerFactory extends InternalLoggerFactory {
-
-    @Override
-    public InternalLogger newInstance(String name) {
-        return new NettyInternalESLogger(Loggers.getLogger(name));
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/NettyMessageChannelHandler.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyMessageChannelHandler.java
new file mode 100644
index 00000000000..0f2805459c9
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/transport/netty/NettyMessageChannelHandler.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.transport.netty;
+
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.transport.TcpHeader;
+import org.elasticsearch.transport.TcpTransport;
+import org.elasticsearch.transport.TcpTransportChannel;
+import org.elasticsearch.transport.TransportServiceAdapter;
+import org.elasticsearch.transport.Transports;
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.ExceptionEvent;
+import org.jboss.netty.channel.MessageEvent;
+import org.jboss.netty.channel.SimpleChannelUpstreamHandler;
+import org.jboss.netty.channel.WriteCompletionEvent;
+
+import java.net.InetSocketAddress;
+
+/**
+ * A handler (must be the last one!) that does size based frame decoding and forwards the actual message
+ * to the relevant action.
+ */
+class NettyMessageChannelHandler extends SimpleChannelUpstreamHandler {
+
+    protected final TransportServiceAdapter transportServiceAdapter;
+    protected final NettyTransport transport;
+    protected final String profileName;
+
+    NettyMessageChannelHandler(NettyTransport transport, String profileName) {
+        this.transportServiceAdapter = transport.transportServiceAdapter();
+        this.transport = transport;
+        this.profileName = profileName;
+    }
+
+    @Override
+    public void writeComplete(ChannelHandlerContext ctx, WriteCompletionEvent e) throws Exception {
+        transportServiceAdapter.sent(e.getWrittenAmount());
+        super.writeComplete(ctx, e);
+    }
+
+    @Override
+    public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
+        Transports.assertTransportThread();
+        Object m = e.getMessage();
+        if (!(m instanceof ChannelBuffer)) {
+            ctx.sendUpstream(e);
+            return;
+        }
+        final ChannelBuffer buffer = (ChannelBuffer) m;
+        final int remainingMessageSize = buffer.getInt(buffer.readerIndex() - TcpHeader.MESSAGE_LENGTH_SIZE);
+        final int expectedReaderIndex = buffer.readerIndex() + remainingMessageSize;
+        InetSocketAddress remoteAddress = (InetSocketAddress) ctx.getChannel().getRemoteAddress();
+        try {
+            // netty always copies a buffer, either in NioWorker in its read handler, where it copies to a fresh
+            // buffer, or in the cumulation buffer, which is cleaned each time so it could be bigger than the actual size
+            BytesReference reference = NettyUtils.toBytesReference(buffer, remainingMessageSize);
+            transport.messageReceived(reference, ctx.getChannel(), profileName, remoteAddress, remainingMessageSize);
+        } finally {
+            // Set the expected position of the buffer, no matter what happened
+            buffer.readerIndex(expectedReaderIndex);
+        }
+    }
+
+    @Override
+    public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
+        transport.exceptionCaught(ctx, e);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
index be1305244bc..3c75f68eeb0 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
@@ -19,58 +19,36 @@
 
 package org.elasticsearch.transport.netty;
 
-import com.carrotsearch.hppc.IntHashSet;
-import com.carrotsearch.hppc.IntSet;
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Booleans;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.bytes.ReleasablePagedBytesReference;
-import org.elasticsearch.common.component.AbstractLifecycleComponent;
-import org.elasticsearch.common.compress.CompressorFactory;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.ReleasableBytesStream;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lease.Releasables;
-import org.elasticsearch.common.metrics.CounterMetric;
-import org.elasticsearch.common.netty.NettyUtils;
-import org.elasticsearch.common.netty.OpenChannelsHandler;
-import org.elasticsearch.common.netty.ReleaseChannelFutureListener;
-import org.elasticsearch.common.network.NetworkAddress;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.network.NetworkService.TcpSettings;
 import org.elasticsearch.common.network.NetworkUtils;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
-import org.elasticsearch.common.transport.PortsRange;
-import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
-import org.elasticsearch.common.util.concurrent.AbstractLifecycleRunnable;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.common.util.concurrent.KeyedLock;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.BindTransportException;
 import org.elasticsearch.transport.BytesTransportRequest;
 import org.elasticsearch.transport.ConnectTransportException;
-import org.elasticsearch.transport.NodeNotConnectedException;
-import org.elasticsearch.transport.Transport;
-import org.elasticsearch.transport.TransportException;
+import org.elasticsearch.transport.TcpTransport;
+import org.elasticsearch.transport.TransportMessage;
 import org.elasticsearch.transport.TransportRequest;
-import org.elasticsearch.transport.TransportRequestOptions;
 import org.elasticsearch.transport.TransportServiceAdapter;
 import org.elasticsearch.transport.TransportSettings;
-import org.elasticsearch.transport.support.TransportStatus;
 import org.jboss.netty.bootstrap.ClientBootstrap;
 import org.jboss.netty.bootstrap.ServerBootstrap;
 import org.jboss.netty.buffer.ChannelBuffer;
@@ -94,42 +72,17 @@ import org.jboss.netty.channel.socket.oio.OioServerSocketChannelFactory;
 import org.jboss.netty.util.HashedWheelTimer;
 
 import java.io.IOException;
-import java.net.BindException;
-import java.net.InetAddress;
 import java.net.InetSocketAddress;
-import java.net.SocketAddress;
-import java.net.UnknownHostException;
-import java.nio.channels.CancelledKeyException;
-import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
-import static java.util.Collections.unmodifiableMap;
-import static org.elasticsearch.common.settings.Setting.boolSetting;
 import static org.elasticsearch.common.settings.Setting.byteSizeSetting;
 import static org.elasticsearch.common.settings.Setting.intSetting;
-import static org.elasticsearch.common.settings.Setting.timeSetting;
-import static org.elasticsearch.common.transport.NetworkExceptionHelper.isCloseConnectionException;
-import static org.elasticsearch.common.transport.NetworkExceptionHelper.isConnectException;
 import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap;
 import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory;
 
@@ -139,51 +92,16 @@ import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadF
  * longer. Med is for the typical search / single doc index. And High for things like cluster state. Ping is reserved for
  * sending out ping requests to other nodes.
  */
-public class NettyTransport extends AbstractLifecycleComponent<Transport> implements Transport {
+public class NettyTransport extends TcpTransport<Channel> {
 
     static {
         NettyUtils.setup();
     }
 
-    public static final String HTTP_SERVER_WORKER_THREAD_NAME_PREFIX = "http_server_worker";
-    public static final String HTTP_SERVER_BOSS_THREAD_NAME_PREFIX = "http_server_boss";
-    public static final String TRANSPORT_CLIENT_WORKER_THREAD_NAME_PREFIX = "transport_client_worker";
-    public static final String TRANSPORT_CLIENT_BOSS_THREAD_NAME_PREFIX = "transport_client_boss";
-
     public static final Setting<Integer> WORKER_COUNT =
         new Setting<>("transport.netty.worker_count",
             (s) -> Integer.toString(EsExecutors.boundedNumberOfProcessors(s) * 2),
             (s) -> Setting.parseInt(s, 1, "transport.netty.worker_count"), Property.NodeScope);
-    public static final Setting<Integer> CONNECTIONS_PER_NODE_RECOVERY =
-        intSetting("transport.connections_per_node.recovery", 2, 1, Property.NodeScope);
-    public static final Setting<Integer> CONNECTIONS_PER_NODE_BULK =
-        intSetting("transport.connections_per_node.bulk", 3, 1, Property.NodeScope);
-    public static final Setting<Integer> CONNECTIONS_PER_NODE_REG =
-        intSetting("transport.connections_per_node.reg", 6, 1, Property.NodeScope);
-    public static final Setting<Integer> CONNECTIONS_PER_NODE_STATE =
-        intSetting("transport.connections_per_node.state", 1, 1, Property.NodeScope);
-    public static final Setting<Integer> CONNECTIONS_PER_NODE_PING =
-        intSetting("transport.connections_per_node.ping", 1, 1, Property.NodeScope);
-    // the scheduled internal ping interval setting, defaults to disabled (-1)
-    public static final Setting<TimeValue> PING_SCHEDULE =
-        timeSetting("transport.ping_schedule", TimeValue.timeValueSeconds(-1), Property.NodeScope);
-    public static final Setting<Boolean> TCP_BLOCKING_CLIENT =
-        boolSetting("transport.tcp.blocking_client", TcpSettings.TCP_BLOCKING_CLIENT, Property.NodeScope);
-    public static final Setting<TimeValue> TCP_CONNECT_TIMEOUT =
-        timeSetting("transport.tcp.connect_timeout", TcpSettings.TCP_CONNECT_TIMEOUT, Property.NodeScope);
-    public static final Setting<Boolean> TCP_NO_DELAY =
-        boolSetting("transport.tcp_no_delay", TcpSettings.TCP_NO_DELAY, Property.NodeScope);
-    public static final Setting<Boolean> TCP_KEEP_ALIVE =
-        boolSetting("transport.tcp.keep_alive", TcpSettings.TCP_KEEP_ALIVE, Property.NodeScope);
-    public static final Setting<Boolean> TCP_BLOCKING_SERVER =
-        boolSetting("transport.tcp.blocking_server", TcpSettings.TCP_BLOCKING_SERVER, Property.NodeScope);
-    public static final Setting<Boolean> TCP_REUSE_ADDRESS =
-        boolSetting("transport.tcp.reuse_address", TcpSettings.TCP_REUSE_ADDRESS, Property.NodeScope);
-
-    public static final Setting<ByteSizeValue> TCP_SEND_BUFFER_SIZE =
-        Setting.byteSizeSetting("transport.tcp.send_buffer_size", TcpSettings.TCP_SEND_BUFFER_SIZE, Property.NodeScope);
-    public static final Setting<ByteSizeValue> TCP_RECEIVE_BUFFER_SIZE =
-        Setting.byteSizeSetting("transport.tcp.receive_buffer_size", TcpSettings.TCP_RECEIVE_BUFFER_SIZE, Property.NodeScope);
 
     public static final Setting<ByteSizeValue> NETTY_MAX_CUMULATION_BUFFER_CAPACITY =
         Setting.byteSizeSetting("transport.netty.max_cumulation_buffer_capacity", new ByteSizeValue(-1), Property.NodeScope);
@@ -192,16 +110,16 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
 
     // See AdaptiveReceiveBufferSizePredictor#DEFAULT_XXX for default values in netty..., we can use higher ones for us, even fixed one
     public static final Setting<ByteSizeValue> NETTY_RECEIVE_PREDICTOR_SIZE = Setting.byteSizeSetting(
-            "transport.netty.receive_predictor_size",
-            settings -> {
-                long defaultReceiverPredictor = 512 * 1024;
-                if (JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes() > 0) {
-                    // we can guess a better default...
-                    long l = (long) ((0.3 * JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes()) / WORKER_COUNT.get(settings));
-                    defaultReceiverPredictor = Math.min(defaultReceiverPredictor, Math.max(l, 64 * 1024));
-                }
-                return new ByteSizeValue(defaultReceiverPredictor).toString();
-            }, Property.NodeScope);
+        "transport.netty.receive_predictor_size",
+        settings -> {
+            long defaultReceiverPredictor = 512 * 1024;
+            if (JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes() > 0) {
+                // we can guess a better default...
+                long l = (long) ((0.3 * JvmInfo.jvmInfo().getMem().getDirectMemoryMax().bytes()) / WORKER_COUNT.get(settings));
+                defaultReceiverPredictor = Math.min(defaultReceiverPredictor, Math.max(l, 64 * 1024));
+            }
+            return new ByteSizeValue(defaultReceiverPredictor).toString();
+        }, Property.NodeScope);
     public static final Setting<ByteSizeValue> NETTY_RECEIVE_PREDICTOR_MIN =
         byteSizeSetting("transport.netty.receive_predictor_min", NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope);
     public static final Setting<ByteSizeValue> NETTY_RECEIVE_PREDICTOR_MAX =
@@ -209,69 +127,25 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
     public static final Setting<Integer> NETTY_BOSS_COUNT =
         intSetting("transport.netty.boss_count", 1, 1, Property.NodeScope);
 
-    protected final NetworkService networkService;
 
-    protected final boolean blockingClient;
-    protected final TimeValue connectTimeout;
     protected final ByteSizeValue maxCumulationBufferCapacity;
     protected final int maxCompositeBufferComponents;
-    protected final boolean compress;
     protected final ReceiveBufferSizePredictorFactory receiveBufferSizePredictorFactory;
     protected final int workerCount;
     protected final ByteSizeValue receivePredictorMin;
     protected final ByteSizeValue receivePredictorMax;
-
-    protected final int connectionsPerNodeRecovery;
-    protected final int connectionsPerNodeBulk;
-    protected final int connectionsPerNodeReg;
-    protected final int connectionsPerNodeState;
-    protected final int connectionsPerNodePing;
-
-    private final TimeValue pingSchedule;
-
-    protected final BigArrays bigArrays;
-    protected final ThreadPool threadPool;
     // package private for testing
     volatile OpenChannelsHandler serverOpenChannels;
     protected volatile ClientBootstrap clientBootstrap;
-    // node id to actual channel
-    protected final ConcurrentMap<DiscoveryNode, NodeChannels> connectedNodes = newConcurrentMap();
     protected final Map<String, ServerBootstrap> serverBootstraps = newConcurrentMap();
-    protected final Map<String, List<Channel>> serverChannels = newConcurrentMap();
-    protected final ConcurrentMap<String, BoundTransportAddress> profileBoundAddresses = newConcurrentMap();
-    protected volatile TransportServiceAdapter transportServiceAdapter;
-    protected volatile BoundTransportAddress boundAddress;
-    protected final KeyedLock<String> connectionLock = new KeyedLock<>();
-    protected final NamedWriteableRegistry namedWriteableRegistry;
-    private final CircuitBreakerService circuitBreakerService;
-
-    // this lock is here to make sure we close this transport and disconnect all the client nodes
-    // connections while no connect operations is going on... (this might help with 100% CPU when stopping the transport?)
-    private final ReadWriteLock globalLock = new ReentrantReadWriteLock();
-
-    // package visibility for tests
-    final ScheduledPing scheduledPing;
 
     @Inject
     public NettyTransport(Settings settings, ThreadPool threadPool, NetworkService networkService, BigArrays bigArrays,
                           NamedWriteableRegistry namedWriteableRegistry, CircuitBreakerService circuitBreakerService) {
-        super(settings);
-        this.threadPool = threadPool;
-        this.networkService = networkService;
-        this.bigArrays = bigArrays;
-
+        super("netty", settings, threadPool, bigArrays, circuitBreakerService, namedWriteableRegistry, networkService);
         this.workerCount = WORKER_COUNT.get(settings);
-        this.blockingClient = TCP_BLOCKING_CLIENT.get(settings);
-        this.connectTimeout = TCP_CONNECT_TIMEOUT.get(settings);
         this.maxCumulationBufferCapacity = NETTY_MAX_CUMULATION_BUFFER_CAPACITY.get(settings);
         this.maxCompositeBufferComponents = NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS.get(settings);
-        this.compress = Transport.TRANSPORT_TCP_COMPRESS.get(settings);
-
-        this.connectionsPerNodeRecovery = CONNECTIONS_PER_NODE_RECOVERY.get(settings);
-        this.connectionsPerNodeBulk = CONNECTIONS_PER_NODE_BULK.get(settings);
-        this.connectionsPerNodeReg = CONNECTIONS_PER_NODE_REG.get(settings);
-        this.connectionsPerNodeState = CONNECTIONS_PER_NODE_STATE.get(settings);
-        this.connectionsPerNodePing = CONNECTIONS_PER_NODE_PING.get(settings);
 
         // See AdaptiveReceiveBufferSizePredictor#DEFAULT_XXX for default values in netty..., we can use higher ones for us, even fixed one
         this.receivePredictorMin = NETTY_RECEIVE_PREDICTOR_MIN.get(settings);
@@ -280,37 +154,14 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
             receiveBufferSizePredictorFactory = new FixedReceiveBufferSizePredictorFactory((int) receivePredictorMax.bytes());
         } else {
             receiveBufferSizePredictorFactory = new AdaptiveReceiveBufferSizePredictorFactory((int) receivePredictorMin.bytes(),
-                    (int) receivePredictorMin.bytes(), (int) receivePredictorMax.bytes());
+                (int) receivePredictorMin.bytes(), (int) receivePredictorMax.bytes());
         }
-
-        this.scheduledPing = new ScheduledPing();
-        this.pingSchedule = PING_SCHEDULE.get(settings);
-        this.namedWriteableRegistry = namedWriteableRegistry;
-        this.circuitBreakerService = circuitBreakerService;
-    }
-
-    public Settings settings() {
-        return this.settings;
-    }
-
-    @Override
-    public void transportServiceAdapter(TransportServiceAdapter service) {
-        this.transportServiceAdapter = service;
     }
 
     TransportServiceAdapter transportServiceAdapter() {
         return transportServiceAdapter;
     }
 
-    ThreadPool threadPool() {
-        return threadPool;
-    }
-
-    CircuitBreaker inFlightRequestsBreaker() {
-        // We always obtain a fresh breaker to reflect changes to the breaker configuration.
-        return circuitBreakerService.getBreaker(CircuitBreaker.IN_FLIGHT_REQUESTS);
-    }
-
     @Override
     protected void doStart() {
         boolean success = false;
@@ -319,51 +170,17 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
             if (NetworkService.NETWORK_SERVER.get(settings)) {
                 final OpenChannelsHandler openChannels = new OpenChannelsHandler(logger);
                 this.serverOpenChannels = openChannels;
-
-                // extract default profile first and create standard bootstrap
-                Map<String, Settings> profiles = TransportSettings.TRANSPORT_PROFILES_SETTING.get(settings()).getAsGroups(true);
-                if (!profiles.containsKey(TransportSettings.DEFAULT_PROFILE)) {
-                    profiles = new HashMap<>(profiles);
-                    profiles.put(TransportSettings.DEFAULT_PROFILE, Settings.EMPTY);
-                }
-
-                Settings fallbackSettings = createFallbackSettings();
-                Settings defaultSettings = profiles.get(TransportSettings.DEFAULT_PROFILE);
-
                 // loop through all profiles and start them up, special handling for default one
-                for (Map.Entry<String, Settings> entry : profiles.entrySet()) {
-                    Settings profileSettings = entry.getValue();
-                    String name = entry.getKey();
-
-                    if (!Strings.hasLength(name)) {
-                        logger.info("transport profile configured without a name. skipping profile with settings [{}]",
-                                profileSettings.toDelimitedString(','));
-                        continue;
-                    } else if (TransportSettings.DEFAULT_PROFILE.equals(name)) {
-                        profileSettings = Settings.builder()
-                                .put(profileSettings)
-                                .put("port", profileSettings.get("port", TransportSettings.PORT.get(this.settings)))
-                                .build();
-                    } else if (profileSettings.get("port") == null) {
-                        // if profile does not have a port, skip it
-                        logger.info("No port configured for profile [{}], not binding", name);
-                        continue;
-                    }
-
+                for (Map.Entry<String, Settings> entry : buildProfileSettings().entrySet()) {
                     // merge fallback settings with default settings with profile settings so we have complete settings with default values
-                    Settings mergedSettings = Settings.builder()
-                            .put(fallbackSettings)
-                            .put(defaultSettings)
-                            .put(profileSettings)
-                            .build();
-
-                    createServerBootstrap(name, mergedSettings);
-                    bindServerBootstrap(name, mergedSettings);
+                    final Settings settings = Settings.builder()
+                        .put(createFallbackSettings())
+                        .put(entry.getValue()).build();
+                    createServerBootstrap(entry.getKey(), settings);
+                    bindServer(entry.getKey(), settings);
                 }
             }
-            if (pingSchedule.millis() > 0) {
-                threadPool.schedule(pingSchedule, ThreadPool.Names.GENERIC, scheduledPing);
-            }
+            super.doStart();
             success = true;
         } finally {
             if (success == false) {
@@ -372,24 +189,18 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
         }
     }
 
-    @Override
-    public Map<String, BoundTransportAddress> profileBoundAddresses() {
-        return unmodifiableMap(new HashMap<>(profileBoundAddresses));
-    }
-
     private ClientBootstrap createClientBootstrap() {
-
         if (blockingClient) {
             clientBootstrap = new ClientBootstrap(new OioClientSocketChannelFactory(
-                    Executors.newCachedThreadPool(daemonThreadFactory(settings, TRANSPORT_CLIENT_WORKER_THREAD_NAME_PREFIX))));
+                Executors.newCachedThreadPool(daemonThreadFactory(settings, TRANSPORT_CLIENT_WORKER_THREAD_NAME_PREFIX))));
         } else {
             int bossCount = NETTY_BOSS_COUNT.get(settings);
             clientBootstrap = new ClientBootstrap(
-                    new NioClientSocketChannelFactory(
-                            Executors.newCachedThreadPool(daemonThreadFactory(settings, TRANSPORT_CLIENT_BOSS_THREAD_NAME_PREFIX)),
-                            bossCount,
-                            new NioWorkerPool(Executors.newCachedThreadPool(
-                                    daemonThreadFactory(settings, TRANSPORT_CLIENT_WORKER_THREAD_NAME_PREFIX)), workerCount),
+                new NioClientSocketChannelFactory(
+                    Executors.newCachedThreadPool(daemonThreadFactory(settings, TRANSPORT_CLIENT_BOSS_THREAD_NAME_PREFIX)),
+                    bossCount,
+                    new NioWorkerPool(Executors.newCachedThreadPool(
+                        daemonThreadFactory(settings, TRANSPORT_CLIENT_WORKER_THREAD_NAME_PREFIX)), workerCount),
                     new HashedWheelTimer(daemonThreadFactory(settings, "transport_client_timer"))));
         }
         clientBootstrap.setPipelineFactory(configureClientChannelPipelineFactory());
@@ -436,19 +247,19 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
         fallbackSettingsBuilder.put("tcp_no_delay", fallbackTcpNoDelay);
 
         boolean fallbackTcpKeepAlive = settings.getAsBoolean("transport.netty.tcp_keep_alive", TcpSettings.TCP_KEEP_ALIVE.get(settings));
-            fallbackSettingsBuilder.put("tcp_keep_alive", fallbackTcpKeepAlive);
+        fallbackSettingsBuilder.put("tcp_keep_alive", fallbackTcpKeepAlive);
 
         boolean fallbackReuseAddress = settings.getAsBoolean("transport.netty.reuse_address", TcpSettings.TCP_REUSE_ADDRESS.get(settings));
         fallbackSettingsBuilder.put("reuse_address", fallbackReuseAddress);
 
         ByteSizeValue fallbackTcpSendBufferSize = settings.getAsBytesSize("transport.netty.tcp_send_buffer_size",
-                TCP_SEND_BUFFER_SIZE.get(settings));
+            TCP_SEND_BUFFER_SIZE.get(settings));
         if (fallbackTcpSendBufferSize.bytes() >= 0) {
             fallbackSettingsBuilder.put("tcp_send_buffer_size", fallbackTcpSendBufferSize);
         }
 
         ByteSizeValue fallbackTcpBufferSize = settings.getAsBytesSize("transport.netty.tcp_receive_buffer_size",
-                TCP_RECEIVE_BUFFER_SIZE.get(settings));
+            TCP_RECEIVE_BUFFER_SIZE.get(settings));
         if (fallbackTcpBufferSize.bytes() >= 0) {
             fallbackSettingsBuilder.put("tcp_receive_buffer_size", fallbackTcpBufferSize);
         }
@@ -456,146 +267,6 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
         return fallbackSettingsBuilder.build();
     }
 
-    private void bindServerBootstrap(final String name, final Settings settings) {
-        // Bind and start to accept incoming connections.
-        InetAddress hostAddresses[];
-        String bindHosts[] = settings.getAsArray("bind_host", null);
-        try {
-            hostAddresses = networkService.resolveBindHostAddresses(bindHosts);
-        } catch (IOException e) {
-            throw new BindTransportException("Failed to resolve host " + Arrays.toString(bindHosts) + "", e);
-        }
-        if (logger.isDebugEnabled()) {
-            String[] addresses = new String[hostAddresses.length];
-            for (int i = 0; i < hostAddresses.length; i++) {
-                addresses[i] = NetworkAddress.format(hostAddresses[i]);
-            }
-            logger.debug("binding server bootstrap to: {}", (Object)addresses);
-        }
-
-        assert hostAddresses.length > 0;
-
-        List<InetSocketAddress> boundAddresses = new ArrayList<>();
-        for (InetAddress hostAddress : hostAddresses) {
-            boundAddresses.add(bindToPort(name, hostAddress, settings.get("port")));
-        }
-
-        final BoundTransportAddress boundTransportAddress = createBoundTransportAddress(name, settings, boundAddresses);
-
-        if (TransportSettings.DEFAULT_PROFILE.equals(name)) {
-            this.boundAddress = boundTransportAddress;
-        } else {
-            profileBoundAddresses.put(name, boundTransportAddress);
-        }
-    }
-
-    private InetSocketAddress bindToPort(final String name, final InetAddress hostAddress, String port) {
-        PortsRange portsRange = new PortsRange(port);
-        final AtomicReference<Exception> lastException = new AtomicReference<>();
-        final AtomicReference<InetSocketAddress> boundSocket = new AtomicReference<>();
-        boolean success = portsRange.iterate(new PortsRange.PortCallback() {
-            @Override
-            public boolean onPortNumber(int portNumber) {
-                try {
-                    Channel channel = serverBootstraps.get(name).bind(new InetSocketAddress(hostAddress, portNumber));
-                    synchronized (serverChannels) {
-                        List<Channel> list = serverChannels.get(name);
-                        if (list == null) {
-                            list = new ArrayList<>();
-                            serverChannels.put(name, list);
-                        }
-                        list.add(channel);
-                        boundSocket.set((InetSocketAddress) channel.getLocalAddress());
-                    }
-                } catch (Exception e) {
-                    lastException.set(e);
-                    return false;
-                }
-                return true;
-            }
-        });
-        if (!success) {
-            throw new BindTransportException("Failed to bind to [" + port + "]", lastException.get());
-        }
-
-        if (logger.isDebugEnabled()) {
-            logger.debug("Bound profile [{}] to address {{}}", name, NetworkAddress.format(boundSocket.get()));
-        }
-
-        return boundSocket.get();
-    }
-
-    private BoundTransportAddress createBoundTransportAddress(String name, Settings profileSettings,
-            List<InetSocketAddress> boundAddresses) {
-        String[] boundAddressesHostStrings = new String[boundAddresses.size()];
-        TransportAddress[] transportBoundAddresses = new TransportAddress[boundAddresses.size()];
-        for (int i = 0; i < boundAddresses.size(); i++) {
-            InetSocketAddress boundAddress = boundAddresses.get(i);
-            boundAddressesHostStrings[i] = boundAddress.getHostString();
-            transportBoundAddresses[i] = new InetSocketTransportAddress(boundAddress);
-        }
-
-        final String[] publishHosts;
-        if (TransportSettings.DEFAULT_PROFILE.equals(name)) {
-            publishHosts = TransportSettings.PUBLISH_HOST.get(settings).toArray(Strings.EMPTY_ARRAY);
-        } else {
-            publishHosts = profileSettings.getAsArray("publish_host", boundAddressesHostStrings);
-        }
-
-        final InetAddress publishInetAddress;
-        try {
-            publishInetAddress = networkService.resolvePublishHostAddresses(publishHosts);
-        } catch (Exception e) {
-            throw new BindTransportException("Failed to resolve publish address", e);
-        }
-
-        final int publishPort = resolvePublishPort(name, settings, profileSettings, boundAddresses, publishInetAddress);
-        final TransportAddress publishAddress = new InetSocketTransportAddress(new InetSocketAddress(publishInetAddress, publishPort));
-        return new BoundTransportAddress(transportBoundAddresses, publishAddress);
-    }
-
-    // package private for tests
-    static int resolvePublishPort(String profileName, Settings settings, Settings profileSettings, List<InetSocketAddress> boundAddresses,
-                               InetAddress publishInetAddress) {
-        int publishPort;
-        if (TransportSettings.DEFAULT_PROFILE.equals(profileName)) {
-            publishPort = TransportSettings.PUBLISH_PORT.get(settings);
-        } else {
-            publishPort = profileSettings.getAsInt("publish_port", -1);
-        }
-
-        // if port not explicitly provided, search for port of address in boundAddresses that matches publishInetAddress
-        if (publishPort < 0) {
-            for (InetSocketAddress boundAddress : boundAddresses) {
-                InetAddress boundInetAddress = boundAddress.getAddress();
-                if (boundInetAddress.isAnyLocalAddress() || boundInetAddress.equals(publishInetAddress)) {
-                    publishPort = boundAddress.getPort();
-                    break;
-                }
-            }
-        }
-
-        // if no matching boundAddress found, check if there is a unique port for all bound addresses
-        if (publishPort < 0) {
-            final IntSet ports = new IntHashSet();
-            for (InetSocketAddress boundAddress : boundAddresses) {
-                ports.add(boundAddress.getPort());
-            }
-            if (ports.size() == 1) {
-                publishPort = ports.iterator().next().value;
-            }
-        }
-
-        if (publishPort < 0) {
-            String profileExplanation = TransportSettings.DEFAULT_PROFILE.equals(profileName) ? "" : " for profile " + profileName;
-            throw new BindTransportException("Failed to auto-resolve publish port" + profileExplanation + ", multiple bound addresses " +
-                boundAddresses + " with distinct ports and none of them matched the publish address (" + publishInetAddress + "). " +
-                "Please specify a unique port by setting " + TransportSettings.PORT.getKey() + " or " +
-                TransportSettings.PUBLISH_PORT.getKey());
-        }
-        return publishPort;
-    }
-
     private void createServerBootstrap(String name, Settings settings) {
         boolean blockingServer = TCP_BLOCKING_SERVER.get(settings);
         String port = settings.get("port");
@@ -609,10 +280,10 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
 
         if (logger.isDebugEnabled()) {
             logger.debug("using profile[{}], worker_count[{}], port[{}], bind_host[{}], publish_host[{}], compress[{}], "
-                            + "connect_timeout[{}], connections_per_node[{}/{}/{}/{}/{}], receive_predictor[{}->{}]",
-                    name, workerCount, port, bindHost, publishHost, compress, connectTimeout, connectionsPerNodeRecovery,
-                    connectionsPerNodeBulk, connectionsPerNodeReg, connectionsPerNodeState, connectionsPerNodePing, receivePredictorMin,
-                    receivePredictorMax);
+                    + "connect_timeout[{}], connections_per_node[{}/{}/{}/{}/{}], receive_predictor[{}->{}]",
+                name, workerCount, port, bindHost, publishHost, compress, connectTimeout, connectionsPerNodeRecovery,
+                connectionsPerNodeBulk, connectionsPerNodeReg, connectionsPerNodeState, connectionsPerNodePing, receivePredictorMin,
+                receivePredictorMax);
         }
 
         final ThreadFactory bossFactory = daemonThreadFactory(this.settings, HTTP_SERVER_BOSS_THREAD_NAME_PREFIX, name);
@@ -620,14 +291,14 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
         ServerBootstrap serverBootstrap;
         if (blockingServer) {
             serverBootstrap = new ServerBootstrap(new OioServerSocketChannelFactory(
-                    Executors.newCachedThreadPool(bossFactory),
-                    Executors.newCachedThreadPool(workerFactory)
+                Executors.newCachedThreadPool(bossFactory),
+                Executors.newCachedThreadPool(workerFactory)
             ));
         } else {
             serverBootstrap = new ServerBootstrap(new NioServerSocketChannelFactory(
-                    Executors.newCachedThreadPool(bossFactory),
-                    Executors.newCachedThreadPool(workerFactory),
-                    workerCount));
+                Executors.newCachedThreadPool(bossFactory),
+                Executors.newCachedThreadPool(workerFactory),
+                workerCount));
         }
         serverBootstrap.setPipelineFactory(configureServerChannelPipelineFactory(name, settings));
         if (!"default".equals(tcpNoDelay)) {
@@ -649,202 +320,8 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
         serverBootstraps.put(name, serverBootstrap);
     }
 
-    @Override
-    protected void doStop() {
-        final CountDownLatch latch = new CountDownLatch(1);
-        // make sure we run it on another thread than a possible IO handler thread
-        threadPool.generic().execute(new Runnable() {
-            @Override
-            public void run() {
-                globalLock.writeLock().lock();
-                try {
-                    for (Iterator<NodeChannels> it = connectedNodes.values().iterator(); it.hasNext(); ) {
-                        NodeChannels nodeChannels = it.next();
-                        it.remove();
-                        nodeChannels.close();
-                    }
-
-                    Iterator<Map.Entry<String, List<Channel>>> serverChannelIterator = serverChannels.entrySet().iterator();
-                    while (serverChannelIterator.hasNext()) {
-                        Map.Entry<String, List<Channel>> serverChannelEntry = serverChannelIterator.next();
-                        String name = serverChannelEntry.getKey();
-                        List<Channel> serverChannels = serverChannelEntry.getValue();
-                        for (Channel serverChannel : serverChannels) {
-                            try {
-                                serverChannel.close().awaitUninterruptibly();
-                            } catch (Throwable t) {
-                                logger.debug("Error closing serverChannel for profile [{}]", t, name);
-                            }
-                        }
-                        serverChannelIterator.remove();
-                    }
-
-                    if (serverOpenChannels != null) {
-                        serverOpenChannels.close();
-                        serverOpenChannels = null;
-                    }
-
-                    Iterator<Map.Entry<String, ServerBootstrap>> serverBootstrapIterator = serverBootstraps.entrySet().iterator();
-                    while (serverBootstrapIterator.hasNext()) {
-                        Map.Entry<String, ServerBootstrap> serverBootstrapEntry = serverBootstrapIterator.next();
-                        String name = serverBootstrapEntry.getKey();
-                        ServerBootstrap serverBootstrap = serverBootstrapEntry.getValue();
-
-                        try {
-                            serverBootstrap.releaseExternalResources();
-                        } catch (Throwable t) {
-                            logger.debug("Error closing serverBootstrap for profile [{}]", t, name);
-                        }
-
-                        serverBootstrapIterator.remove();
-                    }
-
-                    for (Iterator<NodeChannels> it = connectedNodes.values().iterator(); it.hasNext(); ) {
-                        NodeChannels nodeChannels = it.next();
-                        it.remove();
-                        nodeChannels.close();
-                    }
-
-                    if (clientBootstrap != null) {
-                        clientBootstrap.releaseExternalResources();
-                        clientBootstrap = null;
-                    }
-                } finally {
-                    globalLock.writeLock().unlock();
-                    latch.countDown();
-                }
-            }
-        });
-
-        try {
-            latch.await(30, TimeUnit.SECONDS);
-        } catch (InterruptedException e) {
-            // ignore
-        }
-    }
-
-    @Override
-    protected void doClose() {
-    }
-
-    @Override
-    public TransportAddress[] addressesFromString(String address, int perAddressLimit) throws Exception {
-        return parse(address, settings.get("transport.profiles.default.port", TransportSettings.PORT.get(settings)), perAddressLimit);
-    }
-
-    // this code is a take on guava's HostAndPort, like a HostAndPortRange
-
-    // pattern for validating ipv6 bracket addresses.
-    // not perfect, but PortsRange should take care of any port range validation, not a regex
-    private static final Pattern BRACKET_PATTERN = Pattern.compile("^\\[(.*:.*)\\](?::([\\d\\-]*))?$");
-
-    /** parse a hostname+port range spec into its equivalent addresses */
-    static TransportAddress[] parse(String hostPortString, String defaultPortRange, int perAddressLimit) throws UnknownHostException {
-        Objects.requireNonNull(hostPortString);
-        String host;
-        String portString = null;
-
-        if (hostPortString.startsWith("[")) {
-          // Parse a bracketed host, typically an IPv6 literal.
-          Matcher matcher = BRACKET_PATTERN.matcher(hostPortString);
-          if (!matcher.matches()) {
-              throw new IllegalArgumentException("Invalid bracketed host/port range: " + hostPortString);
-          }
-          host = matcher.group(1);
-          portString = matcher.group(2);  // could be null
-        } else {
-          int colonPos = hostPortString.indexOf(':');
-          if (colonPos >= 0 && hostPortString.indexOf(':', colonPos + 1) == -1) {
-            // Exactly 1 colon.  Split into host:port.
-            host = hostPortString.substring(0, colonPos);
-            portString = hostPortString.substring(colonPos + 1);
-          } else {
-            // 0 or 2+ colons.  Bare hostname or IPv6 literal.
-            host = hostPortString;
-            // 2+ colons and not bracketed: exception
-            if (colonPos >= 0) {
-                throw new IllegalArgumentException("IPv6 addresses must be bracketed: " + hostPortString);
-            }
-          }
-        }
-
-        // if port isn't specified, fill with the default
-        if (portString == null || portString.isEmpty()) {
-            portString = defaultPortRange;
-        }
-
-        // generate address for each port in the range
-        Set<InetAddress> addresses = new HashSet<>(Arrays.asList(InetAddress.getAllByName(host)));
-        List<TransportAddress> transportAddresses = new ArrayList<>();
-        int[] ports = new PortsRange(portString).ports();
-        int limit = Math.min(ports.length, perAddressLimit);
-        for (int i = 0; i < limit; i++) {
-            for (InetAddress address : addresses) {
-                transportAddresses.add(new InetSocketTransportAddress(address, ports[i]));
-            }
-        }
-        return transportAddresses.toArray(new TransportAddress[transportAddresses.size()]);
-    }
-
-    @Override
-    public boolean addressSupported(Class<? extends TransportAddress> address) {
-        return InetSocketTransportAddress.class.equals(address);
-    }
-
-    @Override
-    public BoundTransportAddress boundAddress() {
-        return this.boundAddress;
-    }
-
-    protected void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
-        if (!lifecycle.started()) {
-            // ignore
-            return;
-        }
-        if (isCloseConnectionException(e.getCause())) {
-            logger.trace("close connection exception caught on transport layer [{}], disconnecting from relevant node", e.getCause(),
-                    ctx.getChannel());
-            // close the channel, which will cause a node to be disconnected if relevant
-            ctx.getChannel().close();
-            disconnectFromNodeChannel(ctx.getChannel(), e.getCause());
-        } else if (isConnectException(e.getCause())) {
-            logger.trace("connect exception caught on transport layer [{}]", e.getCause(), ctx.getChannel());
-            // close the channel as safe measure, which will cause a node to be disconnected if relevant
-            ctx.getChannel().close();
-            disconnectFromNodeChannel(ctx.getChannel(), e.getCause());
-        } else if (e.getCause() instanceof BindException) {
-            logger.trace("bind exception caught on transport layer [{}]", e.getCause(), ctx.getChannel());
-            // close the channel as safe measure, which will cause a node to be disconnected if relevant
-            ctx.getChannel().close();
-            disconnectFromNodeChannel(ctx.getChannel(), e.getCause());
-        } else if (e.getCause() instanceof CancelledKeyException) {
-            logger.trace("cancelled key exception caught on transport layer [{}], disconnecting from relevant node", e.getCause(),
-                    ctx.getChannel());
-            // close the channel as safe measure, which will cause a node to be disconnected if relevant
-            ctx.getChannel().close();
-            disconnectFromNodeChannel(ctx.getChannel(), e.getCause());
-        } else if (e.getCause() instanceof SizeHeaderFrameDecoder.HttpOnTransportException) {
-            // in case we are able to return data, serialize the exception content and sent it back to the client
-            if (ctx.getChannel().isOpen()) {
-                ChannelBuffer buffer = ChannelBuffers.wrappedBuffer(e.getCause().getMessage().getBytes(StandardCharsets.UTF_8));
-                ChannelFuture channelFuture = ctx.getChannel().write(buffer);
-                channelFuture.addListener(new ChannelFutureListener() {
-                    @Override
-                    public void operationComplete(ChannelFuture future) throws Exception {
-                        future.getChannel().close();
-                    }
-                });
-            }
-        } else {
-            logger.warn("exception caught on transport layer [{}], closing connection", e.getCause(), ctx.getChannel());
-            // close the channel, which will cause a node to be disconnected if relevant
-            ctx.getChannel().close();
-            disconnectFromNodeChannel(ctx.getChannel(), e.getCause());
-        }
-    }
-
-    TransportAddress wrapAddress(SocketAddress socketAddress) {
-        return new InetSocketTransportAddress((InetSocketAddress) socketAddress);
+    protected final void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
+       onException(ctx.getChannel(), e.getCause());
     }
 
     @Override
@@ -853,153 +330,6 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
         return channels == null ? 0 : channels.numberOfOpenChannels();
     }
 
-    @Override
-    public List<String> getLocalAddresses() {
-        List<String> local = new ArrayList<>();
-        local.add("127.0.0.1");
-        // check if v6 is supported, if so, v4 will also work via mapped addresses.
-        if (NetworkUtils.SUPPORTS_V6) {
-            local.add("[::1]"); // may get ports appended!
-        }
-        return local;
-    }
-
-    @Override
-    public void sendRequest(final DiscoveryNode node, final long requestId, final String action, final TransportRequest request,
-            TransportRequestOptions options) throws IOException, TransportException {
-
-        Channel targetChannel = nodeChannel(node, options);
-
-        if (compress) {
-            options = TransportRequestOptions.builder(options).withCompress(true).build();
-        }
-
-        byte status = 0;
-        status = TransportStatus.setRequest(status);
-
-        ReleasableBytesStreamOutput bStream = new ReleasableBytesStreamOutput(bigArrays);
-        boolean addedReleaseListener = false;
-        try {
-            bStream.skip(NettyHeader.HEADER_SIZE);
-            StreamOutput stream = bStream;
-            // only compress if asked, and, the request is not bytes, since then only
-            // the header part is compressed, and the "body" can't be extracted as compressed
-            if (options.compress() && (!(request instanceof BytesTransportRequest))) {
-                status = TransportStatus.setCompress(status);
-                stream = CompressorFactory.COMPRESSOR.streamOutput(stream);
-            }
-
-            // we pick the smallest of the 2, to support both backward and forward compatibility
-            // note, this is the only place we need to do this, since from here on, we use the serialized version
-            // as the version to use also when the node receiving this request will send the response with
-            Version version = Version.smallest(getCurrentVersion(), node.getVersion());
-
-            stream.setVersion(version);
-            threadPool.getThreadContext().writeTo(stream);
-            stream.writeString(action);
-
-            ReleasablePagedBytesReference bytes;
-            ChannelBuffer buffer;
-            // it might be nice to somehow generalize this optimization, maybe a smart "paged" bytes output
-            // that create paged channel buffers, but its tricky to know when to do it (where this option is
-            // more explicit).
-            if (request instanceof BytesTransportRequest) {
-                BytesTransportRequest bRequest = (BytesTransportRequest) request;
-                assert node.getVersion().equals(bRequest.version());
-                bRequest.writeThin(stream);
-                stream.close();
-                bytes = bStream.bytes();
-                ChannelBuffer headerBuffer = NettyUtils.toChannelBuffer(bytes);
-                ChannelBuffer contentBuffer = NettyUtils.toChannelBuffer(bRequest.bytes());
-                buffer = ChannelBuffers.wrappedBuffer(NettyUtils.DEFAULT_GATHERING, headerBuffer, contentBuffer);
-            } else {
-                request.writeTo(stream);
-                stream.close();
-                bytes = bStream.bytes();
-                buffer = NettyUtils.toChannelBuffer(bytes);
-            }
-            NettyHeader.writeHeader(buffer, requestId, status, version);
-            ChannelFuture future = targetChannel.write(buffer);
-            ReleaseChannelFutureListener listener = new ReleaseChannelFutureListener(bytes);
-            future.addListener(listener);
-            addedReleaseListener = true;
-            final TransportRequestOptions finalOptions = options;
-            ChannelFutureListener channelFutureListener =
-                f -> transportServiceAdapter.onRequestSent(node, requestId, action, request, finalOptions);
-            future.addListener(channelFutureListener);
-        } finally {
-            if (!addedReleaseListener) {
-                Releasables.close(bStream.bytes());
-            }
-        }
-    }
-
-    @Override
-    public boolean nodeConnected(DiscoveryNode node) {
-        return connectedNodes.containsKey(node);
-    }
-
-    @Override
-    public void connectToNodeLight(DiscoveryNode node) throws ConnectTransportException {
-        connectToNode(node, true);
-    }
-
-    @Override
-    public void connectToNode(DiscoveryNode node) {
-        connectToNode(node, false);
-    }
-
-    public void connectToNode(DiscoveryNode node, boolean light) {
-        if (!lifecycle.started()) {
-            throw new IllegalStateException("can't add nodes to a stopped transport");
-        }
-        if (node == null) {
-            throw new ConnectTransportException(null, "can't connect to a null node");
-        }
-        globalLock.readLock().lock();
-        try {
-
-            try (Releasable ignored = connectionLock.acquire(node.getId())) {
-                if (!lifecycle.started()) {
-                    throw new IllegalStateException("can't add nodes to a stopped transport");
-                }
-                NodeChannels nodeChannels = connectedNodes.get(node);
-                if (nodeChannels != null) {
-                    return;
-                }
-                try {
-                    if (light) {
-                        nodeChannels = connectToChannelsLight(node);
-                    } else {
-                        nodeChannels = new NodeChannels(new Channel[connectionsPerNodeRecovery], new Channel[connectionsPerNodeBulk],
-                                new Channel[connectionsPerNodeReg], new Channel[connectionsPerNodeState],
-                                new Channel[connectionsPerNodePing]);
-                        try {
-                            connectToChannels(nodeChannels, node);
-                        } catch (Throwable e) {
-                            logger.trace("failed to connect to [{}], cleaning dangling connections", e, node);
-                            nodeChannels.close();
-                            throw e;
-                        }
-                    }
-                    // we acquire a connection lock, so no way there is an existing connection
-                    nodeChannels.start();
-                    connectedNodes.put(node, nodeChannels);
-                    if (logger.isDebugEnabled()) {
-                        logger.debug("connected to node [{}]", node);
-                    }
-                    transportServiceAdapter.raiseNodeConnected(node);
-                } catch (ConnectTransportException e) {
-                    throw e;
-                } catch (Exception e) {
-                    throw new ConnectTransportException(node, "general node connection failure", e);
-                }
-            }
-        } finally {
-            globalLock.readLock().unlock();
-        }
-    }
-
     protected NodeChannels connectToChannelsLight(DiscoveryNode node) {
         InetSocketAddress address = ((InetSocketTransportAddress) node.getAddress()).address();
         ChannelFuture connect = clientBootstrap.connect(address);
@@ -1013,176 +343,117 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
         return new NodeChannels(channels, channels, channels, channels, channels);
     }
 
-    protected void connectToChannels(NodeChannels nodeChannels, DiscoveryNode node) {
-        ChannelFuture[] connectRecovery = new ChannelFuture[nodeChannels.recovery.length];
-        ChannelFuture[] connectBulk = new ChannelFuture[nodeChannels.bulk.length];
-        ChannelFuture[] connectReg = new ChannelFuture[nodeChannels.reg.length];
-        ChannelFuture[] connectState = new ChannelFuture[nodeChannels.state.length];
-        ChannelFuture[] connectPing = new ChannelFuture[nodeChannels.ping.length];
-        InetSocketAddress address = ((InetSocketTransportAddress) node.getAddress()).address();
-        for (int i = 0; i < connectRecovery.length; i++) {
-            connectRecovery[i] = clientBootstrap.connect(address);
-        }
-        for (int i = 0; i < connectBulk.length; i++) {
-            connectBulk[i] = clientBootstrap.connect(address);
-        }
-        for (int i = 0; i < connectReg.length; i++) {
-            connectReg[i] = clientBootstrap.connect(address);
-        }
-        for (int i = 0; i < connectState.length; i++) {
-            connectState[i] = clientBootstrap.connect(address);
-        }
-        for (int i = 0; i < connectPing.length; i++) {
-            connectPing[i] = clientBootstrap.connect(address);
-        }
-
+    protected NodeChannels connectToChannels(DiscoveryNode node) {
+        final NodeChannels nodeChannels = new NodeChannels(new Channel[connectionsPerNodeRecovery], new Channel[connectionsPerNodeBulk],
+            new Channel[connectionsPerNodeReg], new Channel[connectionsPerNodeState],
+            new Channel[connectionsPerNodePing]);
+        boolean success = false;
         try {
+            ChannelFuture[] connectRecovery = new ChannelFuture[nodeChannels.recovery.length];
+            ChannelFuture[] connectBulk = new ChannelFuture[nodeChannels.bulk.length];
+            ChannelFuture[] connectReg = new ChannelFuture[nodeChannels.reg.length];
+            ChannelFuture[] connectState = new ChannelFuture[nodeChannels.state.length];
+            ChannelFuture[] connectPing = new ChannelFuture[nodeChannels.ping.length];
+            InetSocketAddress address = ((InetSocketTransportAddress) node.getAddress()).address();
             for (int i = 0; i < connectRecovery.length; i++) {
-                connectRecovery[i].awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
-                if (!connectRecovery[i].isSuccess()) {
-                    throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connectRecovery[i].getCause());
-                }
-                nodeChannels.recovery[i] = connectRecovery[i].getChannel();
-                nodeChannels.recovery[i].getCloseFuture().addListener(new ChannelCloseListener(node));
+                connectRecovery[i] = clientBootstrap.connect(address);
             }
-
             for (int i = 0; i < connectBulk.length; i++) {
-                connectBulk[i].awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
-                if (!connectBulk[i].isSuccess()) {
-                    throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connectBulk[i].getCause());
-                }
-                nodeChannels.bulk[i] = connectBulk[i].getChannel();
-                nodeChannels.bulk[i].getCloseFuture().addListener(new ChannelCloseListener(node));
+                connectBulk[i] = clientBootstrap.connect(address);
             }
-
             for (int i = 0; i < connectReg.length; i++) {
-                connectReg[i].awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
-                if (!connectReg[i].isSuccess()) {
-                    throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connectReg[i].getCause());
-                }
-                nodeChannels.reg[i] = connectReg[i].getChannel();
-                nodeChannels.reg[i].getCloseFuture().addListener(new ChannelCloseListener(node));
+                connectReg[i] = clientBootstrap.connect(address);
             }
-
             for (int i = 0; i < connectState.length; i++) {
-                connectState[i].awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
-                if (!connectState[i].isSuccess()) {
-                    throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connectState[i].getCause());
-                }
-                nodeChannels.state[i] = connectState[i].getChannel();
-                nodeChannels.state[i].getCloseFuture().addListener(new ChannelCloseListener(node));
+                connectState[i] = clientBootstrap.connect(address);
             }
-
             for (int i = 0; i < connectPing.length; i++) {
-                connectPing[i].awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
-                if (!connectPing[i].isSuccess()) {
-                    throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connectPing[i].getCause());
-                }
-                nodeChannels.ping[i] = connectPing[i].getChannel();
-                nodeChannels.ping[i].getCloseFuture().addListener(new ChannelCloseListener(node));
+                connectPing[i] = clientBootstrap.connect(address);
             }
 
-            if (nodeChannels.recovery.length == 0) {
-                if (nodeChannels.bulk.length > 0) {
-                    nodeChannels.recovery = nodeChannels.bulk;
-                } else {
-                    nodeChannels.recovery = nodeChannels.reg;
+            try {
+                for (int i = 0; i < connectRecovery.length; i++) {
+                    connectRecovery[i].awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
+                    if (!connectRecovery[i].isSuccess()) {
+                        throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connectRecovery[i].getCause());
+                    }
+                    nodeChannels.recovery[i] = connectRecovery[i].getChannel();
+                    nodeChannels.recovery[i].getCloseFuture().addListener(new ChannelCloseListener(node));
                 }
-            }
-            if (nodeChannels.bulk.length == 0) {
-                nodeChannels.bulk = nodeChannels.reg;
-            }
-        } catch (RuntimeException e) {
-            // clean the futures
-            List<ChannelFuture> futures = new ArrayList<>();
-            futures.addAll(Arrays.asList(connectRecovery));
-            futures.addAll(Arrays.asList(connectBulk));
-            futures.addAll(Arrays.asList(connectReg));
-            futures.addAll(Arrays.asList(connectState));
-            futures.addAll(Arrays.asList(connectPing));
-            for (ChannelFuture future : Collections.unmodifiableList(futures)) {
-                future.cancel();
-                if (future.getChannel() != null && future.getChannel().isOpen()) {
-                    try {
-                        future.getChannel().close();
-                    } catch (Exception e1) {
-                        // ignore
+
+                for (int i = 0; i < connectBulk.length; i++) {
+                    connectBulk[i].awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
+                    if (!connectBulk[i].isSuccess()) {
+                        throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connectBulk[i].getCause());
+                    }
+                    nodeChannels.bulk[i] = connectBulk[i].getChannel();
+                    nodeChannels.bulk[i].getCloseFuture().addListener(new ChannelCloseListener(node));
+                }
+
+                for (int i = 0; i < connectReg.length; i++) {
+                    connectReg[i].awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
+                    if (!connectReg[i].isSuccess()) {
+                        throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connectReg[i].getCause());
+                    }
+                    nodeChannels.reg[i] = connectReg[i].getChannel();
+                    nodeChannels.reg[i].getCloseFuture().addListener(new ChannelCloseListener(node));
+                }
+
+                for (int i = 0; i < connectState.length; i++) {
+                    connectState[i].awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
+                    if (!connectState[i].isSuccess()) {
+                        throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connectState[i].getCause());
+                    }
+                    nodeChannels.state[i] = connectState[i].getChannel();
+                    nodeChannels.state[i].getCloseFuture().addListener(new ChannelCloseListener(node));
+                }
+
+                for (int i = 0; i < connectPing.length; i++) {
+                    connectPing[i].awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
+                    if (!connectPing[i].isSuccess()) {
+                        throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connectPing[i].getCause());
+                    }
+                    nodeChannels.ping[i] = connectPing[i].getChannel();
+                    nodeChannels.ping[i].getCloseFuture().addListener(new ChannelCloseListener(node));
+                }
+
+                if (nodeChannels.recovery.length == 0) {
+                    if (nodeChannels.bulk.length > 0) {
+                        nodeChannels.recovery = nodeChannels.bulk;
+                    } else {
+                        nodeChannels.recovery = nodeChannels.reg;
                     }
                 }
-            }
-            throw e;
-        }
-    }
-
-    @Override
-    public void disconnectFromNode(DiscoveryNode node) {
-
-        try (Releasable ignored = connectionLock.acquire(node.getId())) {
-            NodeChannels nodeChannels = connectedNodes.remove(node);
-            if (nodeChannels != null) {
-                try {
-                    logger.debug("disconnecting from [{}] due to explicit disconnect call", node);
-                    nodeChannels.close();
-                } finally {
-                    logger.trace("disconnected from [{}] due to explicit disconnect call", node);
-                    transportServiceAdapter.raiseNodeDisconnected(node);
+                if (nodeChannels.bulk.length == 0) {
+                    nodeChannels.bulk = nodeChannels.reg;
                 }
-            }
-        }
-    }
-
-    /**
-     * Disconnects from a node, only if the relevant channel is found to be part of the node channels.
-     */
-    protected boolean disconnectFromNode(DiscoveryNode node, Channel channel, String reason) {
-        // this might be called multiple times from all the node channels, so do a lightweight
-        // check outside of the lock
-        NodeChannels nodeChannels = connectedNodes.get(node);
-        if (nodeChannels != null && nodeChannels.hasChannel(channel)) {
-            try (Releasable ignored = connectionLock.acquire(node.getId())) {
-                nodeChannels = connectedNodes.get(node);
-                // check again within the connection lock, if its still applicable to remove it
-                if (nodeChannels != null && nodeChannels.hasChannel(channel)) {
-                    connectedNodes.remove(node);
-                    try {
-                        logger.debug("disconnecting from [{}], {}", node, reason);
-                        nodeChannels.close();
-                    } finally {
-                        logger.trace("disconnected from [{}], {}", node, reason);
-                        transportServiceAdapter.raiseNodeDisconnected(node);
-                    }
-                    return true;
-                }
-            }
-        }
-        return false;
-    }
-
-    /**
-     * Disconnects from a node if a channel is found as part of that nodes channels.
-     */
-    protected void disconnectFromNodeChannel(final Channel channel, final Throwable failure) {
-        threadPool().generic().execute(new Runnable() {
-
-            @Override
-            public void run() {
-                for (DiscoveryNode node : connectedNodes.keySet()) {
-                    if (disconnectFromNode(node, channel, ExceptionsHelper.detailedMessage(failure))) {
-                        // if we managed to find this channel and disconnect from it, then break, no need to check on
-                        // the rest of the nodes
-                        break;
+            } catch (RuntimeException e) {
+                // clean the futures
+                List<ChannelFuture> futures = new ArrayList<>();
+                futures.addAll(Arrays.asList(connectRecovery));
+                futures.addAll(Arrays.asList(connectBulk));
+                futures.addAll(Arrays.asList(connectReg));
+                futures.addAll(Arrays.asList(connectState));
+                futures.addAll(Arrays.asList(connectPing));
+                for (ChannelFuture future : Collections.unmodifiableList(futures)) {
+                    future.cancel();
+                    if (future.getChannel() != null && future.getChannel().isOpen()) {
+                        try {
+                            future.getChannel().close();
+                        } catch (Exception e1) {
+                            // ignore
+                        }
                     }
                 }
+                throw e;
+            }
+            success = true;
+        } finally {
+            if (success == false) {
+                nodeChannels.close();
             }
-        });
-    }
-
-    protected Channel nodeChannel(DiscoveryNode node, TransportRequestOptions options) throws ConnectTransportException {
-        NodeChannels nodeChannels = connectedNodes.get(node);
-        if (nodeChannels == null) {
-            throw new NodeNotConnectedException(node, "Node not connected");
         }
-        return nodeChannels.channel(options.type());
+        return nodeChannels;
     }
 
     public ChannelPipelineFactory configureClientChannelPipelineFactory() {
@@ -1212,7 +483,7 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
             }
             channelPipeline.addLast("size", sizeHeader);
             // using a dot as a prefix means, this cannot come from any settings parsed
-            channelPipeline.addLast("dispatcher", new MessageChannelHandler(nettyTransport, nettyTransport.logger, ".client"));
+            channelPipeline.addLast("dispatcher", new NettyMessageChannelHandler(nettyTransport, ".client"));
             return channelPipeline;
         }
     }
@@ -1249,7 +520,7 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
                 sizeHeader.setMaxCumulationBufferComponents(nettyTransport.maxCompositeBufferComponents);
             }
             channelPipeline.addLast("size", sizeHeader);
-            channelPipeline.addLast("dispatcher", new MessageChannelHandler(nettyTransport, nettyTransport.logger, name));
+            channelPipeline.addLast("dispatcher", new NettyMessageChannelHandler(nettyTransport, name));
             return channelPipeline;
         }
     }
@@ -1266,142 +537,169 @@ public class NettyTransport extends AbstractLifecycleComponent<Transport> implem
         public void operationComplete(final ChannelFuture future) throws Exception {
             NodeChannels nodeChannels = connectedNodes.get(node);
             if (nodeChannels != null && nodeChannels.hasChannel(future.getChannel())) {
-                threadPool().generic().execute(new Runnable() {
-                    @Override
-                    public void run() {
-                        disconnectFromNode(node, future.getChannel(), "channel closed event");
-                    }
-                });
+                threadPool.generic().execute(() -> disconnectFromNode(node, future.getChannel(), "channel closed event"));
             }
         }
     }
 
-    public static class NodeChannels {
-
-        List<Channel> allChannels = Collections.emptyList();
-        private Channel[] recovery;
-        private final AtomicInteger recoveryCounter = new AtomicInteger();
-        private Channel[] bulk;
-        private final AtomicInteger bulkCounter = new AtomicInteger();
-        private Channel[] reg;
-        private final AtomicInteger regCounter = new AtomicInteger();
-        private Channel[] state;
-        private final AtomicInteger stateCounter = new AtomicInteger();
-        private Channel[] ping;
-        private final AtomicInteger pingCounter = new AtomicInteger();
-
-        public NodeChannels(Channel[] recovery, Channel[] bulk, Channel[] reg, Channel[] state, Channel[] ping) {
-            this.recovery = recovery;
-            this.bulk = bulk;
-            this.reg = reg;
-            this.state = state;
-            this.ping = ping;
-        }
-
-        public void start() {
-            List<Channel> newAllChannels = new ArrayList<>();
-            newAllChannels.addAll(Arrays.asList(recovery));
-            newAllChannels.addAll(Arrays.asList(bulk));
-            newAllChannels.addAll(Arrays.asList(reg));
-            newAllChannels.addAll(Arrays.asList(state));
-            newAllChannels.addAll(Arrays.asList(ping));
-            this.allChannels = Collections.unmodifiableList(newAllChannels);
-        }
-
-        public boolean hasChannel(Channel channel) {
-            for (Channel channel1 : allChannels) {
-                if (channel.equals(channel1)) {
-                    return true;
-                }
-            }
-            return false;
-        }
-
-        public Channel channel(TransportRequestOptions.Type type) {
-            if (type == TransportRequestOptions.Type.REG) {
-                return reg[Math.floorMod(regCounter.incrementAndGet(), reg.length)];
-            } else if (type == TransportRequestOptions.Type.STATE) {
-                return state[Math.floorMod(stateCounter.incrementAndGet(), state.length)];
-            } else if (type == TransportRequestOptions.Type.PING) {
-                return ping[Math.floorMod(pingCounter.incrementAndGet(), ping.length)];
-            } else if (type == TransportRequestOptions.Type.BULK) {
-                return bulk[Math.floorMod(bulkCounter.incrementAndGet(), bulk.length)];
-            } else if (type == TransportRequestOptions.Type.RECOVERY) {
-                return recovery[Math.floorMod(recoveryCounter.incrementAndGet(), recovery.length)];
-            } else {
-                throw new IllegalArgumentException("no type channel for [" + type + "]");
-            }
-        }
-
-        public synchronized void close() {
-            List<ChannelFuture> futures = new ArrayList<>();
-            for (Channel channel : allChannels) {
+    protected void sendMessage(Channel channel, BytesReference reference, Runnable sendListener, boolean close) {
+        final ChannelFuture future = channel.write(NettyUtils.toChannelBuffer(reference));
+        if (close) {
+            future.addListener(f -> {
                 try {
-                    if (channel != null && channel.isOpen()) {
-                        futures.add(channel.close());
-                    }
-                } catch (Exception e) {
-                    //ignore
+                    sendListener.run();
+                } finally {
+                    f.getChannel().close();
                 }
-            }
-            for (ChannelFuture future : futures) {
-                future.awaitUninterruptibly();
-            }
+            });
+        } else {
+            future.addListener(future1 -> sendListener.run());
         }
     }
 
-    class ScheduledPing extends AbstractLifecycleRunnable {
+    @Override
+    protected void closeChannels(List<Channel> channels) {
+        List<ChannelFuture> futures = new ArrayList<>();
 
-        final CounterMetric successfulPings = new CounterMetric();
-        final CounterMetric failedPings = new CounterMetric();
-
-        public ScheduledPing() {
-            super(lifecycle, logger);
-        }
-
-        @Override
-        protected void doRunInLifecycle() throws Exception {
-            for (Map.Entry<DiscoveryNode, NodeChannels> entry : connectedNodes.entrySet()) {
-                DiscoveryNode node = entry.getKey();
-                NodeChannels channels = entry.getValue();
-                for (Channel channel : channels.allChannels) {
-                    try {
-                        ChannelFuture future = channel.write(NettyHeader.pingHeader());
-                        future.addListener(new ChannelFutureListener() {
-                            @Override
-                            public void operationComplete(ChannelFuture future) throws Exception {
-                                successfulPings.inc();
-                            }
-                        });
-                    } catch (Throwable t) {
-                        if (channel.isOpen()) {
-                            logger.debug("[{}] failed to send ping transport message", t, node);
-                            failedPings.inc();
-                        } else {
-                            logger.trace("[{}] failed to send ping transport message (channel closed)", t, node);
-                        }
-                    }
+        for (Channel channel : channels) {
+            try {
+                if (channel != null && channel.isOpen()) {
+                    futures.add(channel.close());
                 }
+            } catch (Exception e) {
+                logger.trace("failed to close channel", e);
             }
         }
-
-        @Override
-        protected void onAfterInLifecycle() {
-            threadPool.schedule(pingSchedule, ThreadPool.Names.GENERIC, this);
+        for (ChannelFuture future : futures) {
+            future.awaitUninterruptibly();
         }
+    }
 
-        @Override
-        public void onFailure(Throwable t) {
-            if (lifecycle.stoppedOrClosed()) {
-                logger.trace("failed to send ping transport message", t);
+    @Override
+    protected InetSocketAddress getLocalAddress(Channel channel) {
+        return (InetSocketAddress) channel.getLocalAddress();
+    }
+
+    @Override
+    protected Channel bind(String name, InetSocketAddress address) {
+        return serverBootstraps.get(name).bind(address);
+    }
+
+    ScheduledPing getPing() {
+        return scheduledPing;
+    }
+
+    @Override
+    protected boolean isOpen(Channel channel) {
+        return channel.isOpen();
+    }
+
+    @Override
+    protected void stopInternal() {
+        Releasables.close(serverOpenChannels, () ->{
+            for (Map.Entry<String, ServerBootstrap> entry : serverBootstraps.entrySet()) {
+                String name = entry.getKey();
+                ServerBootstrap serverBootstrap = entry.getValue();
+                try {
+                    serverBootstrap.releaseExternalResources();
+                } catch (Throwable t) {
+                    logger.debug("Error closing serverBootstrap for profile [{}]", t, name);
+                }
+            }
+            serverBootstraps.clear();
+            if (clientBootstrap != null) {
+                clientBootstrap.releaseExternalResources();
+                clientBootstrap = null;
+            }
+        });
+    }
+
+    @Override
+    public Message<Channel> prepareSend(Version nodeVersion, TransportMessage message, StreamOutput stream,
+                                        ReleasableBytesStream writtenBytes) throws IOException {
+            // it might be nice to somehow generalize this optimization, maybe a smart "paged" bytes output
+            // that create paged channel buffers, but its tricky to know when to do it (where this option is
+            // more explicit).
+            if (message instanceof BytesTransportRequest) {
+                BytesTransportRequest bRequest = (BytesTransportRequest) message;
+                assert nodeVersion.equals(bRequest.version());
+                bRequest.writeThin(stream);
+                stream.close();
+                ReleasablePagedBytesReference bytes = writtenBytes.bytes();
+                ChannelBuffer headerBuffer = NettyUtils.toChannelBuffer(bytes);
+                ChannelBuffer contentBuffer = NettyUtils.toChannelBuffer(bRequest.bytes());
+                ChannelBuffer buffer = ChannelBuffers.wrappedBuffer(NettyUtils.DEFAULT_GATHERING, headerBuffer, contentBuffer);
+                return new NettyMessage(buffer);
             } else {
-                logger.warn("failed to send ping transport message", t);
+                return super.prepareSend(nodeVersion, message, stream, writtenBytes);
             }
+    }
+
+    @Override
+    public Message<Channel> prepareSend(Version nodeVersion, BytesReference bytesReference) {
+        return new NettyMessage(NettyUtils.toChannelBuffer(bytesReference));
+    }
+
+    @Override
+    public boolean canCompress(TransportRequest request) {
+        return super.canCompress(request) && (!(request instanceof BytesTransportRequest));
+    }
+
+    private class NettyMessage implements Message<Channel> {
+        private final ChannelBuffer buffer;
+
+        public NettyMessage(ChannelBuffer buffer) {
+            this.buffer = buffer;
+        }
+
+        public StreamOutput getHeaderOutput() {
+            return new ChannelBufferStreamOutput(buffer);
+        }
+
+        public int size() {
+            return buffer.readableBytes();
+        }
+
+        @Override
+        public void send(Channel channel, Runnable onRequestSent) {
+            ChannelFuture future = channel.write(buffer);
+            ChannelFutureListener channelFutureListener = f -> onRequestSent.run();
+            future.addListener(channelFutureListener);
         }
     }
 
-    protected Version getCurrentVersion() {
-        // this is just for tests to mock stuff like the nodes version - tests can override this internally
-        return Version.CURRENT;
+    private final static class ChannelBufferStreamOutput extends StreamOutput {
+
+        private final ChannelBuffer buffer;
+        private int offset;
+
+        public ChannelBufferStreamOutput(ChannelBuffer buffer) {
+            this.buffer = buffer;
+            this.offset = buffer.readerIndex();
+        }
+
+        @Override
+        public void writeByte(byte b) throws IOException {
+            buffer.setByte(offset++, b);
+        }
+
+        @Override
+        public void writeBytes(byte[] b, int offset, int length) throws IOException {
+            buffer.setBytes(this.offset, b, offset, length);
+            this.offset += length;
+        }
+
+        @Override
+        public void flush() throws IOException {
+        }
+
+        @Override
+        public void close() throws IOException {
+        }
+
+        @Override
+        public void reset() throws IOException {
+            throw new UnsupportedOperationException();
+        }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java
deleted file mode 100644
index 0d5666408ea..00000000000
--- a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransportChannel.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.transport.netty;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.bytes.ReleasablePagedBytesReference;
-import org.elasticsearch.common.compress.CompressorFactory;
-import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.lease.Releasables;
-import org.elasticsearch.common.netty.NettyUtils;
-import org.elasticsearch.common.netty.ReleaseChannelFutureListener;
-import org.elasticsearch.transport.RemoteTransportException;
-import org.elasticsearch.transport.TransportChannel;
-import org.elasticsearch.transport.TransportResponse;
-import org.elasticsearch.transport.TransportResponseOptions;
-import org.elasticsearch.transport.TransportServiceAdapter;
-import org.elasticsearch.transport.support.TransportStatus;
-import org.jboss.netty.buffer.ChannelBuffer;
-import org.jboss.netty.channel.Channel;
-import org.jboss.netty.channel.ChannelFuture;
-import org.jboss.netty.channel.ChannelFutureListener;
-
-import java.io.IOException;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-public class NettyTransportChannel implements TransportChannel {
-
-    private final NettyTransport transport;
-    private final TransportServiceAdapter transportServiceAdapter;
-    private final Version version;
-    private final String action;
-    private final Channel channel;
-    private final long requestId;
-    private final String profileName;
-    private final long reservedBytes;
-    private final AtomicBoolean released = new AtomicBoolean();
-
-    public NettyTransportChannel(NettyTransport transport, TransportServiceAdapter transportServiceAdapter, String action, Channel channel,
-                                 long requestId, Version version, String profileName, long reservedBytes) {
-        this.transportServiceAdapter = transportServiceAdapter;
-        this.version = version;
-        this.transport = transport;
-        this.action = action;
-        this.channel = channel;
-        this.requestId = requestId;
-        this.profileName = profileName;
-        this.reservedBytes = reservedBytes;
-    }
-
-    @Override
-    public String getProfileName() {
-        return profileName;
-    }
-
-    @Override
-    public String action() {
-        return this.action;
-    }
-
-    @Override
-    public void sendResponse(TransportResponse response) throws IOException {
-        sendResponse(response, TransportResponseOptions.EMPTY);
-    }
-
-    @Override
-    public void sendResponse(TransportResponse response, TransportResponseOptions options) throws IOException {
-        release();
-        if (transport.compress) {
-            options = TransportResponseOptions.builder(options).withCompress(transport.compress).build();
-        }
-
-        byte status = 0;
-        status = TransportStatus.setResponse(status);
-
-        ReleasableBytesStreamOutput bStream = null;
-        boolean addedReleaseListener = false;
-        try {
-            bStream = new ReleasableBytesStreamOutput(transport.bigArrays);
-            bStream.skip(NettyHeader.HEADER_SIZE);
-            StreamOutput stream = bStream;
-            if (options.compress()) {
-                status = TransportStatus.setCompress(status);
-                stream = CompressorFactory.COMPRESSOR.streamOutput(stream);
-            }
-            stream.setVersion(version);
-            response.writeTo(stream);
-            stream.close();
-
-            ReleasablePagedBytesReference bytes = bStream.bytes();
-            ChannelBuffer buffer = NettyUtils.toChannelBuffer(bytes);
-            NettyHeader.writeHeader(buffer, requestId, status, version);
-            ChannelFuture future = channel.write(buffer);
-            ReleaseChannelFutureListener listener = new ReleaseChannelFutureListener(bytes);
-            future.addListener(listener);
-            addedReleaseListener = true;
-            final TransportResponseOptions finalOptions = options;
-            ChannelFutureListener onResponseSentListener =
-                f -> transportServiceAdapter.onResponseSent(requestId, action, response, finalOptions);
-            future.addListener(onResponseSentListener);
-        } finally {
-            if (!addedReleaseListener && bStream != null) {
-                Releasables.close(bStream.bytes());
-            }
-        }
-    }
-
-    @Override
-    public void sendResponse(Throwable error) throws IOException {
-        release();
-        BytesStreamOutput stream = new BytesStreamOutput();
-        stream.skip(NettyHeader.HEADER_SIZE);
-        RemoteTransportException tx = new RemoteTransportException(
-            transport.nodeName(), transport.wrapAddress(channel.getLocalAddress()), action, error);
-        stream.writeThrowable(tx);
-        byte status = 0;
-        status = TransportStatus.setResponse(status);
-        status = TransportStatus.setError(status);
-
-        BytesReference bytes = stream.bytes();
-        ChannelBuffer buffer = NettyUtils.toChannelBuffer(bytes);
-        NettyHeader.writeHeader(buffer, requestId, status, version);
-        ChannelFuture future = channel.write(buffer);
-        ChannelFutureListener onResponseSentListener =
-            f -> transportServiceAdapter.onResponseSent(requestId, action, error);
-        future.addListener(onResponseSentListener);
-    }
-
-    private void release() {
-        // attempt to release once atomically
-        if (released.compareAndSet(false, true) == false) {
-            throw new IllegalStateException("reserved bytes are already released");
-        }
-        transport.inFlightRequestsBreaker().addWithoutBreaking(-reservedBytes);
-    }
-
-    @Override
-    public long getRequestId() {
-        return requestId;
-    }
-
-    @Override
-    public String getChannelType() {
-        return "netty";
-    }
-
-    /**
-     * Returns the underlying netty channel. This method is intended be used for access to netty to get additional
-     * details when processing the request and may be used by plugins. Responses should be sent using the methods
-     * defined in this class and not directly on the channel.
-     * @return underlying netty channel
-     */
-    public Channel getChannel() {
-        return channel;
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/common/netty/NettyUtils.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyUtils.java
similarity index 90%
rename from core/src/main/java/org/elasticsearch/common/netty/NettyUtils.java
rename to core/src/main/java/org/elasticsearch/transport/netty/NettyUtils.java
index c37ca3ad6fb..f3fdde5e91c 100644
--- a/core/src/main/java/org/elasticsearch/common/netty/NettyUtils.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/NettyUtils.java
@@ -16,12 +16,12 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.common.netty;
+package org.elasticsearch.transport.netty;
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefIterator;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.transport.netty.NettyInternalESLoggerFactory;
+import org.elasticsearch.common.logging.Loggers;
 import org.jboss.netty.buffer.ChannelBuffer;
 import org.jboss.netty.buffer.ChannelBuffers;
 import org.jboss.netty.logging.InternalLogger;
@@ -93,10 +93,11 @@ public class NettyUtils {
     }
 
     static {
-        InternalLoggerFactory.setDefaultFactory(new NettyInternalESLoggerFactory() {
+        InternalLoggerFactory.setDefaultFactory(new InternalLoggerFactory() {
             @Override
             public InternalLogger newInstance(String name) {
-                return super.newInstance(name.replace("org.jboss.netty.", "netty.").replace("org.jboss.netty.", "netty."));
+                name = name.replace("org.jboss.netty.", "netty.").replace("org.jboss.netty.", "netty.");
+                return new NettyInternalESLogger(Loggers.getLogger(name));
             }
         });
 
@@ -136,6 +137,13 @@ public class NettyUtils {
      * Wraps the given ChannelBuffer with a BytesReference
      */
     public static BytesReference toBytesReference(ChannelBuffer channelBuffer) {
-        return new ChannelBufferBytesReference(channelBuffer);
+        return toBytesReference(channelBuffer, channelBuffer.readableBytes());
+    }
+
+    /**
+     * Wraps the given ChannelBuffer with a BytesReference of a given size
+     */
+    public static BytesReference toBytesReference(ChannelBuffer channelBuffer, int size) {
+        return new ChannelBufferBytesReference(channelBuffer, size);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/netty/OpenChannelsHandler.java b/core/src/main/java/org/elasticsearch/transport/netty/OpenChannelsHandler.java
similarity index 95%
rename from core/src/main/java/org/elasticsearch/common/netty/OpenChannelsHandler.java
rename to core/src/main/java/org/elasticsearch/transport/netty/OpenChannelsHandler.java
index 324db75dc90..df7cd73a42b 100644
--- a/core/src/main/java/org/elasticsearch/common/netty/OpenChannelsHandler.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/OpenChannelsHandler.java
@@ -17,8 +17,9 @@
  * under the License.
  */
 
-package org.elasticsearch.common.netty;
+package org.elasticsearch.transport.netty;
 
+import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.metrics.CounterMetric;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
@@ -32,13 +33,14 @@ import org.jboss.netty.channel.ChannelState;
 import org.jboss.netty.channel.ChannelStateEvent;
 import org.jboss.netty.channel.ChannelUpstreamHandler;
 
+import java.io.Closeable;
 import java.util.Set;
 
 /**
  *
  */
 @ChannelHandler.Sharable
-public class OpenChannelsHandler implements ChannelUpstreamHandler {
+public class OpenChannelsHandler implements ChannelUpstreamHandler, Releasable {
 
     final Set<Channel> openChannels = ConcurrentCollections.newConcurrentSet();
     final CounterMetric openChannelsMetric = new CounterMetric();
@@ -91,6 +93,7 @@ public class OpenChannelsHandler implements ChannelUpstreamHandler {
         return totalChannelsMetric.count();
     }
 
+    @Override
     public void close() {
         for (Channel channel : openChannels) {
             channel.close().awaitUninterruptibly();
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/SizeHeaderFrameDecoder.java b/core/src/main/java/org/elasticsearch/transport/netty/SizeHeaderFrameDecoder.java
index 9c410e4b912..d098fae6c78 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/SizeHeaderFrameDecoder.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/SizeHeaderFrameDecoder.java
@@ -19,107 +19,29 @@
 
 package org.elasticsearch.transport.netty;
 
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.monitor.jvm.JvmInfo;
-import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.transport.TcpHeader;
+import org.elasticsearch.transport.TcpTransport;
 import org.jboss.netty.buffer.ChannelBuffer;
 import org.jboss.netty.channel.Channel;
 import org.jboss.netty.channel.ChannelHandlerContext;
 import org.jboss.netty.handler.codec.frame.FrameDecoder;
 import org.jboss.netty.handler.codec.frame.TooLongFrameException;
 
-import java.io.IOException;
-import java.io.StreamCorruptedException;
-
 /**
  */
-public class SizeHeaderFrameDecoder extends FrameDecoder {
-
-    private static final long NINETY_PER_HEAP_SIZE = (long) (JvmInfo.jvmInfo().getMem().getHeapMax().bytes() * 0.9);
+final class SizeHeaderFrameDecoder extends FrameDecoder {
 
     @Override
     protected Object decode(ChannelHandlerContext ctx, Channel channel, ChannelBuffer buffer) throws Exception {
-        final int sizeHeaderLength = NettyHeader.MARKER_BYTES_SIZE + NettyHeader.MESSAGE_LENGTH_SIZE;
-        if (buffer.readableBytes() < sizeHeaderLength) {
+        try {
+            boolean continueProcessing = TcpTransport.validateMessageHeader(NettyUtils.toBytesReference(buffer));
+            buffer.skipBytes(TcpHeader.MARKER_BYTES_SIZE + TcpHeader.MESSAGE_LENGTH_SIZE);
+            return continueProcessing ? buffer : null;
+        } catch (IllegalArgumentException ex) {
+            throw new TooLongFrameException(ex.getMessage(), ex);
+        } catch (IllegalStateException ex) {
             return null;
         }
-
-        int readerIndex = buffer.readerIndex();
-        if (buffer.getByte(readerIndex) != 'E' || buffer.getByte(readerIndex + 1) != 'S') {
-            // special handling for what is probably HTTP
-            if (bufferStartsWith(buffer, readerIndex, "GET ") ||
-                bufferStartsWith(buffer, readerIndex, "POST ") ||
-                bufferStartsWith(buffer, readerIndex, "PUT ") ||
-                bufferStartsWith(buffer, readerIndex, "HEAD ") ||
-                bufferStartsWith(buffer, readerIndex, "DELETE ") ||
-                bufferStartsWith(buffer, readerIndex, "OPTIONS ") ||
-                bufferStartsWith(buffer, readerIndex, "PATCH ") ||
-                bufferStartsWith(buffer, readerIndex, "TRACE ")) {
-
-                throw new HttpOnTransportException("This is not a HTTP port");
-            }
-
-            // we have 6 readable bytes, show 4 (should be enough)
-            throw new StreamCorruptedException("invalid internal transport message format, got ("
-                    + Integer.toHexString(buffer.getByte(readerIndex) & 0xFF) + ","
-                    + Integer.toHexString(buffer.getByte(readerIndex + 1) & 0xFF) + ","
-                    + Integer.toHexString(buffer.getByte(readerIndex + 2) & 0xFF) + ","
-                    + Integer.toHexString(buffer.getByte(readerIndex + 3) & 0xFF) + ")");
-        }
-
-        int dataLen = buffer.getInt(buffer.readerIndex() + NettyHeader.MARKER_BYTES_SIZE);
-        if (dataLen == NettyHeader.PING_DATA_SIZE) {
-            // discard the messages we read and continue, this is achieved by skipping the bytes
-            // and returning null
-            buffer.skipBytes(sizeHeaderLength);
-            return null;
-        }
-        if (dataLen <= 0) {
-            throw new StreamCorruptedException("invalid data length: " + dataLen);
-        }
-        // safety against too large frames being sent
-        if (dataLen > NINETY_PER_HEAP_SIZE) {
-            throw new TooLongFrameException("transport content length received [" + new ByteSizeValue(dataLen) + "] exceeded ["
-                    + new ByteSizeValue(NINETY_PER_HEAP_SIZE) + "]");
-        }
-
-        if (buffer.readableBytes() < dataLen + sizeHeaderLength) {
-            return null;
-        }
-        buffer.skipBytes(sizeHeaderLength);
-        return buffer;
     }
 
-    private boolean bufferStartsWith(ChannelBuffer buffer, int readerIndex, String method) {
-        char[] chars = method.toCharArray();
-        for (int i = 0; i < chars.length; i++) {
-            if (buffer.getByte(readerIndex + i) != chars[i]) {
-                return false;
-            }
-        }
-
-        return true;
-    }
-
-    /**
-     * A helper exception to mark an incoming connection as potentially being HTTP
-     * so an appropriate error code can be returned
-     */
-    public static class HttpOnTransportException extends ElasticsearchException {
-
-        public HttpOnTransportException(String msg) {
-            super(msg);
-        }
-
-        @Override
-        public RestStatus status() {
-            return RestStatus.BAD_REQUEST;
-        }
-
-        public HttpOnTransportException(StreamInput in) throws IOException{
-            super(in);
-        }
-    }
 }
diff --git a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
index d10fc890848..c0b5f2e9cf8 100644
--- a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
+++ b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
@@ -80,6 +80,7 @@ import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
 import org.elasticsearch.transport.ActionNotFoundTransportException;
 import org.elasticsearch.transport.ActionTransportException;
 import org.elasticsearch.transport.ConnectTransportException;
+import org.elasticsearch.transport.TcpTransport;
 
 import java.io.IOException;
 import java.net.URISyntaxException;
@@ -763,7 +764,7 @@ public class ExceptionSerializationTests extends ESTestCase {
         ids.put(122, null);
         ids.put(123, org.elasticsearch.indices.IndexAlreadyExistsException.class);
         ids.put(124, org.elasticsearch.script.Script.ScriptParseException.class);
-        ids.put(125, org.elasticsearch.transport.netty.SizeHeaderFrameDecoder.HttpOnTransportException.class);
+        ids.put(125, TcpTransport.HttpOnTransportException.class);
         ids.put(126, org.elasticsearch.index.mapper.MapperParsingException.class);
         ids.put(127, org.elasticsearch.search.SearchContextException.class);
         ids.put(128, org.elasticsearch.search.builder.SearchSourceBuilderException.class);
diff --git a/core/src/test/java/org/elasticsearch/common/ChannelsTests.java b/core/src/test/java/org/elasticsearch/common/ChannelsTests.java
index 5bb9c614b84..4f2bad36d4a 100644
--- a/core/src/test/java/org/elasticsearch/common/ChannelsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/ChannelsTests.java
@@ -162,20 +162,6 @@ public class ChannelsTests extends ESTestCase {
         assertTrue("read bytes didn't match written bytes", sourceRef.equals(copyRef));
     }
 
-
-    public void testWriteFromChannel() throws IOException {
-        int length = randomIntBetween(1, randomBytes.length / 2);
-        int offset = randomIntBetween(0, randomBytes.length - length);
-        ByteBuffer byteBuffer = ByteBuffer.wrap(randomBytes);
-        ChannelBuffer source = new ByteBufferBackedChannelBuffer(byteBuffer);
-        Channels.writeToChannel(source, offset, length, fileChannel);
-
-        BytesReference copyRef = new BytesArray(Channels.readFromFileChannel(fileChannel, 0, length));
-        BytesReference sourceRef = new BytesArray(randomBytes, offset, length);
-
-        assertTrue("read bytes didn't match written bytes", sourceRef.equals(copyRef));
-    }
-
     class MockFileChannel extends FileChannel {
 
         FileChannel delegate;
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/KeyedLockTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/KeyedLockTests.java
similarity index 99%
rename from core/src/test/java/org/elasticsearch/transport/netty/KeyedLockTests.java
rename to core/src/test/java/org/elasticsearch/common/util/concurrent/KeyedLockTests.java
index f9451375590..36335ee78d0 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/KeyedLockTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/KeyedLockTests.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.transport.netty;
+package org.elasticsearch.common.util.concurrent;
 
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.util.concurrent.KeyedLock;
diff --git a/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
index 4fa99b3b80c..9ae029a4aa4 100644
--- a/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
+++ b/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
@@ -324,16 +324,13 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
     public void testVoidMessageCompressed() {
         serviceA.registerRequestHandler("sayHello", TransportRequest.Empty::new, ThreadPool.Names.GENERIC,
-            new TransportRequestHandler<TransportRequest.Empty>() {
-                @Override
-                public void messageReceived(TransportRequest.Empty request, TransportChannel channel) {
-                    try {
-                        TransportResponseOptions responseOptions = TransportResponseOptions.builder().withCompress(true).build();
-                        channel.sendResponse(TransportResponse.Empty.INSTANCE, responseOptions);
-                    } catch (IOException e) {
-                        logger.error("Unexpected failure", e);
-                        fail(e.getMessage());
-                    }
+            (request, channel) -> {
+                try {
+                    TransportResponseOptions responseOptions = TransportResponseOptions.builder().withCompress(true).build();
+                    channel.sendResponse(TransportResponse.Empty.INSTANCE, responseOptions);
+                } catch (IOException e) {
+                    logger.error("Unexpected failure", e);
+                    fail(e.getMessage());
                 }
             });
 
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyPublishPortTests.java b/core/src/test/java/org/elasticsearch/transport/PublishPortTests.java
similarity index 94%
rename from core/src/test/java/org/elasticsearch/transport/netty/NettyPublishPortTests.java
rename to core/src/test/java/org/elasticsearch/transport/PublishPortTests.java
index 6f602dafc99..ffe7a2d7ce2 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/NettyPublishPortTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/PublishPortTests.java
@@ -17,13 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.transport.netty;
+package org.elasticsearch.transport;
 
 import org.elasticsearch.common.network.NetworkUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.transport.BindTransportException;
-import org.elasticsearch.transport.TransportSettings;
 
 import java.net.InetSocketAddress;
 import java.net.UnknownHostException;
@@ -32,11 +30,11 @@ import java.util.List;
 
 import static java.net.InetAddress.getByName;
 import static java.util.Arrays.asList;
-import static org.elasticsearch.transport.netty.NettyTransport.resolvePublishPort;
+import static org.elasticsearch.transport.TcpTransport.resolvePublishPort;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
-public class NettyPublishPortTests extends ESTestCase {
+public class PublishPortTests extends ESTestCase {
 
     public void testPublishPort() throws Exception {
         int boundPort = randomIntBetween(9000, 9100);
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportTests.java b/core/src/test/java/org/elasticsearch/transport/TCPTransportTests.java
similarity index 78%
rename from core/src/test/java/org/elasticsearch/transport/netty/NettyTransportTests.java
rename to core/src/test/java/org/elasticsearch/transport/TCPTransportTests.java
index a5bd6612cdf..da1dcf43e5d 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/TCPTransportTests.java
@@ -17,17 +17,17 @@
  * under the License.
  */
 
-package org.elasticsearch.transport.netty;
+package org.elasticsearch.transport;
 
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.test.ESTestCase;
 
-/** Unit tests for NettyTransport */
-public class NettyTransportTests extends ESTestCase {
-    
+/** Unit tests for TCPTransport */
+public class TCPTransportTests extends ESTestCase {
+
     /** Test ipv4 host with a default port works */
     public void testParseV4DefaultPort() throws Exception {
-        TransportAddress[] addresses = NettyTransport.parse("127.0.0.1", "1234", Integer.MAX_VALUE);
+        TransportAddress[] addresses = TcpTransport.parse("127.0.0.1", "1234", Integer.MAX_VALUE);
         assertEquals(1, addresses.length);
 
         assertEquals("127.0.0.1", addresses[0].getAddress());
@@ -36,19 +36,19 @@ public class NettyTransportTests extends ESTestCase {
 
     /** Test ipv4 host with a default port range works */
     public void testParseV4DefaultRange() throws Exception {
-        TransportAddress[] addresses = NettyTransport.parse("127.0.0.1", "1234-1235", Integer.MAX_VALUE);
+        TransportAddress[] addresses = TcpTransport.parse("127.0.0.1", "1234-1235", Integer.MAX_VALUE);
         assertEquals(2, addresses.length);
 
         assertEquals("127.0.0.1", addresses[0].getAddress());
         assertEquals(1234, addresses[0].getPort());
-        
+
         assertEquals("127.0.0.1", addresses[1].getAddress());
         assertEquals(1235, addresses[1].getPort());
     }
 
     /** Test ipv4 host with port works */
     public void testParseV4WithPort() throws Exception {
-        TransportAddress[] addresses = NettyTransport.parse("127.0.0.1:2345", "1234", Integer.MAX_VALUE);
+        TransportAddress[] addresses = TcpTransport.parse("127.0.0.1:2345", "1234", Integer.MAX_VALUE);
         assertEquals(1, addresses.length);
 
         assertEquals("127.0.0.1", addresses[0].getAddress());
@@ -57,7 +57,7 @@ public class NettyTransportTests extends ESTestCase {
 
     /** Test ipv4 host with port range works */
     public void testParseV4WithPortRange() throws Exception {
-        TransportAddress[] addresses = NettyTransport.parse("127.0.0.1:2345-2346", "1234", Integer.MAX_VALUE);
+        TransportAddress[] addresses = TcpTransport.parse("127.0.0.1:2345-2346", "1234", Integer.MAX_VALUE);
         assertEquals(2, addresses.length);
 
         assertEquals("127.0.0.1", addresses[0].getAddress());
@@ -70,7 +70,7 @@ public class NettyTransportTests extends ESTestCase {
     /** Test unbracketed ipv6 hosts in configuration fail. Leave no ambiguity */
     public void testParseV6UnBracketed() throws Exception {
         try {
-            NettyTransport.parse("::1", "1234", Integer.MAX_VALUE);
+            TcpTransport.parse("::1", "1234", Integer.MAX_VALUE);
             fail("should have gotten exception");
         } catch (IllegalArgumentException expected) {
             assertTrue(expected.getMessage().contains("must be bracketed"));
@@ -79,7 +79,7 @@ public class NettyTransportTests extends ESTestCase {
 
     /** Test ipv6 host with a default port works */
     public void testParseV6DefaultPort() throws Exception {
-        TransportAddress[] addresses = NettyTransport.parse("[::1]", "1234", Integer.MAX_VALUE);
+        TransportAddress[] addresses = TcpTransport.parse("[::1]", "1234", Integer.MAX_VALUE);
         assertEquals(1, addresses.length);
 
         assertEquals("::1", addresses[0].getAddress());
@@ -88,19 +88,19 @@ public class NettyTransportTests extends ESTestCase {
 
     /** Test ipv6 host with a default port range works */
     public void testParseV6DefaultRange() throws Exception {
-        TransportAddress[] addresses = NettyTransport.parse("[::1]", "1234-1235", Integer.MAX_VALUE);
+        TransportAddress[] addresses = TcpTransport.parse("[::1]", "1234-1235", Integer.MAX_VALUE);
         assertEquals(2, addresses.length);
 
         assertEquals("::1", addresses[0].getAddress());
         assertEquals(1234, addresses[0].getPort());
-        
+
         assertEquals("::1", addresses[1].getAddress());
         assertEquals(1235, addresses[1].getPort());
     }
 
     /** Test ipv6 host with port works */
     public void testParseV6WithPort() throws Exception {
-        TransportAddress[] addresses = NettyTransport.parse("[::1]:2345", "1234", Integer.MAX_VALUE);
+        TransportAddress[] addresses = TcpTransport.parse("[::1]:2345", "1234", Integer.MAX_VALUE);
         assertEquals(1, addresses.length);
 
         assertEquals("::1", addresses[0].getAddress());
@@ -109,7 +109,7 @@ public class NettyTransportTests extends ESTestCase {
 
     /** Test ipv6 host with port range works */
     public void testParseV6WithPortRange() throws Exception {
-        TransportAddress[] addresses = NettyTransport.parse("[::1]:2345-2346", "1234", Integer.MAX_VALUE);
+        TransportAddress[] addresses = TcpTransport.parse("[::1]:2345-2346", "1234", Integer.MAX_VALUE);
         assertEquals(2, addresses.length);
 
         assertEquals("::1", addresses[0].getAddress());
@@ -118,10 +118,10 @@ public class NettyTransportTests extends ESTestCase {
         assertEquals("::1", addresses[1].getAddress());
         assertEquals(2346, addresses[1].getPort());
     }
-    
+
     /** Test per-address limit */
     public void testAddressLimit() throws Exception {
-        TransportAddress[] addresses = NettyTransport.parse("[::1]:100-200", "1000", 3);
+        TransportAddress[] addresses = TcpTransport.parse("[::1]:100-200", "1000", 3);
         assertEquals(3, addresses.length);
         assertEquals(100, addresses[0].getPort());
         assertEquals(101, addresses[1].getPort());
diff --git a/core/src/test/java/org/elasticsearch/common/netty/ChannelBufferBytesReferenceTests.java b/core/src/test/java/org/elasticsearch/transport/netty/ChannelBufferBytesReferenceTests.java
similarity index 96%
rename from core/src/test/java/org/elasticsearch/common/netty/ChannelBufferBytesReferenceTests.java
rename to core/src/test/java/org/elasticsearch/transport/netty/ChannelBufferBytesReferenceTests.java
index 76a8626fee5..a284f6ea911 100644
--- a/core/src/test/java/org/elasticsearch/common/netty/ChannelBufferBytesReferenceTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/ChannelBufferBytesReferenceTests.java
@@ -16,12 +16,13 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.common.netty;
+package org.elasticsearch.transport.netty;
 
 import org.elasticsearch.common.bytes.AbstractBytesReferenceTestCase;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
+import org.elasticsearch.transport.netty.NettyUtils;
 import org.jboss.netty.buffer.ChannelBuffer;
 import org.jboss.netty.buffer.ChannelBuffers;
 
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java
index df7dcb0714b..c69f56c2cbd 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.transport.netty;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.lease.Releasables;
@@ -33,6 +32,7 @@ import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.BaseTransportResponseHandler;
+import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
@@ -56,7 +56,7 @@ public class NettyScheduledPingTests extends ESTestCase {
         ThreadPool threadPool = new TestThreadPool(getClass().getName());
 
         Settings settings = Settings.builder()
-            .put(NettyTransport.PING_SCHEDULE.getKey(), "5ms")
+            .put(TcpTransport.PING_SCHEDULE.getKey(), "5ms")
             .put(TransportSettings.PORT.getKey(), 0)
             .put("cluster.name", "test")
             .build();
@@ -89,12 +89,12 @@ public class NettyScheduledPingTests extends ESTestCase {
         assertBusy(new Runnable() {
             @Override
             public void run() {
-                assertThat(nettyA.scheduledPing.successfulPings.count(), greaterThan(100L));
-                assertThat(nettyB.scheduledPing.successfulPings.count(), greaterThan(100L));
+                assertThat(nettyA.getPing().getSuccessfulPings(), greaterThan(100L));
+                assertThat(nettyB.getPing().getSuccessfulPings(), greaterThan(100L));
             }
         });
-        assertThat(nettyA.scheduledPing.failedPings.count(), equalTo(0L));
-        assertThat(nettyB.scheduledPing.failedPings.count(), equalTo(0L));
+        assertThat(nettyA.getPing().getFailedPings(), equalTo(0L));
+        assertThat(nettyB.getPing().getFailedPings(), equalTo(0L));
 
         serviceA.registerRequestHandler("sayHello", TransportRequest.Empty::new, ThreadPool.Names.GENERIC,
             new TransportRequestHandler<TransportRequest.Empty>() {
@@ -137,15 +137,12 @@ public class NettyScheduledPingTests extends ESTestCase {
                     }).txGet();
         }
 
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                assertThat(nettyA.scheduledPing.successfulPings.count(), greaterThan(200L));
-                assertThat(nettyB.scheduledPing.successfulPings.count(), greaterThan(200L));
-            }
+        assertBusy(() -> {
+            assertThat(nettyA.getPing().getSuccessfulPings(), greaterThan(200L));
+            assertThat(nettyB.getPing().getSuccessfulPings(), greaterThan(200L));
         });
-        assertThat(nettyA.scheduledPing.failedPings.count(), equalTo(0L));
-        assertThat(nettyB.scheduledPing.failedPings.count(), equalTo(0L));
+        assertThat(nettyA.getPing().getFailedPings(), equalTo(0L));
+        assertThat(nettyB.getPing().getFailedPings(), equalTo(0L));
 
         Releasables.close(serviceA, serviceB);
         terminate(threadPool);
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java
index 2f89435c6df..310f804ef7c 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportIT.java
@@ -44,6 +44,7 @@ import org.jboss.netty.channel.ChannelPipeline;
 import org.jboss.netty.channel.ChannelPipelineFactory;
 
 import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.util.Collection;
 import java.util.Collections;
 
@@ -98,45 +99,24 @@ public class NettyTransportIT extends ESIntegTestCase {
             super(settings, threadPool, networkService, bigArrays, namedWriteableRegistry, circuitBreakerService);
         }
 
+        protected String handleRequest(Channel channel, String profileName,
+                                       StreamInput stream, long requestId, int messageLengthBytes, Version version,
+                                       InetSocketAddress remoteAddress) throws IOException {
+            String action = super.handleRequest(channel, profileName, stream, requestId, messageLengthBytes, version,
+                remoteAddress);
+            channelProfileName = TransportSettings.DEFAULT_PROFILE;
+            return action;
+        }
+
         @Override
-        public ChannelPipelineFactory configureServerChannelPipelineFactory(String name, Settings groupSettings) {
-            return new ErrorPipelineFactory(this, name, groupSettings);
-        }
-
-        private static class ErrorPipelineFactory extends ServerChannelPipelineFactory {
-
-            private final ESLogger logger;
-
-            public ErrorPipelineFactory(ExceptionThrowingNettyTransport nettyTransport, String name, Settings groupSettings) {
-                super(nettyTransport, name, groupSettings);
-                this.logger = nettyTransport.logger;
-            }
-
-            @Override
-            public ChannelPipeline getPipeline() throws Exception {
-                ChannelPipeline pipeline = super.getPipeline();
-                pipeline.replace("dispatcher", "dispatcher",
-                    new MessageChannelHandler(nettyTransport, logger, TransportSettings.DEFAULT_PROFILE) {
-
-                    @Override
-                    protected String handleRequest(Channel channel, Marker marker, StreamInput buffer, long requestId,
-                                                   int messageLengthBytes, Version version) throws IOException {
-                        String action = super.handleRequest(channel, marker, buffer, requestId, messageLengthBytes, version);
-                        channelProfileName = this.profileName;
-                        return action;
-                    }
-
-                    @Override
-                    protected void validateRequest(Marker marker, StreamInput buffer, long requestId, String action) throws IOException {
-                        super.validateRequest(marker, buffer, requestId, action);
-                        String error = threadPool.getThreadContext().getHeader("ERROR");
-                        if (error != null) {
-                            throw new ElasticsearchException(error);
-                        }
-                    }
-                });
-                return pipeline;
+        protected void validateRequest(StreamInput buffer, long requestId, String action)
+            throws IOException {
+            super.validateRequest(buffer, requestId, action);
+            String error = threadPool.getThreadContext().getHeader("ERROR");
+            if (error != null) {
+                throw new ElasticsearchException(error);
             }
         }
+
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java
index 6fdc214d18d..352c90d2317 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyTransportMultiPortTests.java
@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.transport.netty;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.network.NetworkService;
@@ -30,6 +29,7 @@ import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.TransportSettings;
 import org.junit.Before;
@@ -58,7 +58,7 @@ public class NettyTransportMultiPortTests extends ESTestCase {
                 .build();
 
         ThreadPool threadPool = new TestThreadPool("tst");
-        try (NettyTransport transport = startNettyTransport(settings, threadPool)) {
+        try (TcpTransport<?> transport = startTransport(settings, threadPool)) {
             assertEquals(1, transport.profileBoundAddresses().size());
             assertEquals(1, transport.boundAddress().boundAddresses().length);
         } finally {
@@ -74,7 +74,7 @@ public class NettyTransportMultiPortTests extends ESTestCase {
                 .build();
 
         ThreadPool threadPool = new TestThreadPool("tst");
-        try (NettyTransport transport = startNettyTransport(settings, threadPool)) {
+        try (TcpTransport<?> transport = startTransport(settings, threadPool)) {
             assertEquals(1, transport.profileBoundAddresses().size());
             assertEquals(1, transport.boundAddress().boundAddresses().length);
         } finally {
@@ -91,7 +91,7 @@ public class NettyTransportMultiPortTests extends ESTestCase {
                 .build();
 
         ThreadPool threadPool = new TestThreadPool("tst");
-        try (NettyTransport transport = startNettyTransport(settings, threadPool)) {
+        try (TcpTransport<?> transport = startTransport(settings, threadPool)) {
             assertEquals(0, transport.profileBoundAddresses().size());
             assertEquals(1, transport.boundAddress().boundAddresses().length);
         } finally {
@@ -107,7 +107,7 @@ public class NettyTransportMultiPortTests extends ESTestCase {
                 .build();
 
         ThreadPool threadPool = new TestThreadPool("tst");
-        try (NettyTransport transport = startNettyTransport(settings, threadPool)) {
+        try (TcpTransport<?> transport = startTransport(settings, threadPool)) {
             assertEquals(0, transport.profileBoundAddresses().size());
             assertEquals(1, transport.boundAddress().boundAddresses().length);
         } finally {
@@ -125,7 +125,7 @@ public class NettyTransportMultiPortTests extends ESTestCase {
                 .build();
 
         ThreadPool threadPool = new TestThreadPool("tst");
-        try (NettyTransport transport = startNettyTransport(settings, threadPool)) {
+        try (TcpTransport<?> transport = startTransport(settings, threadPool)) {
             assertEquals(0, transport.profileBoundAddresses().size());
             assertEquals(1, transport.boundAddress().boundAddresses().length);
         } finally {
@@ -133,14 +133,13 @@ public class NettyTransportMultiPortTests extends ESTestCase {
         }
     }
 
-    private NettyTransport startNettyTransport(Settings settings, ThreadPool threadPool) {
+    private TcpTransport<?> startTransport(Settings settings, ThreadPool threadPool) {
         BigArrays bigArrays = new MockBigArrays(Settings.EMPTY, new NoneCircuitBreakerService());
-
-        NettyTransport nettyTransport = new NettyTransport(settings, threadPool, new NetworkService(settings), bigArrays,
+        TcpTransport<?> transport = new NettyTransport(settings, threadPool, new NetworkService(settings), bigArrays,
             new NamedWriteableRegistry(), new NoneCircuitBreakerService());
-        nettyTransport.start();
+        transport.start();
 
-        assertThat(nettyTransport.lifecycleState(), is(Lifecycle.State.STARTED));
-        return nettyTransport;
+        assertThat(transport.lifecycleState(), is(Lifecycle.State.STARTED));
+        return transport;
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/netty/NettyUtilsTests.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyUtilsTests.java
similarity index 98%
rename from core/src/test/java/org/elasticsearch/common/netty/NettyUtilsTests.java
rename to core/src/test/java/org/elasticsearch/transport/netty/NettyUtilsTests.java
index 2d981dc9eae..fa8f30249bb 100644
--- a/core/src/test/java/org/elasticsearch/common/netty/NettyUtilsTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyUtilsTests.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.common.netty;
+package org.elasticsearch.transport.netty;
 
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -27,7 +27,6 @@ import org.elasticsearch.test.ESTestCase;
 import org.jboss.netty.buffer.ChannelBuffer;
 import org.jboss.netty.buffer.ChannelBuffers;
 import org.jboss.netty.buffer.CompositeChannelBuffer;
-import org.junit.Before;
 
 import java.io.IOException;
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index 20af5cf50b2..b6b75b1ec64 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -89,6 +89,7 @@ import org.elasticsearch.search.SearchService;
 import org.elasticsearch.test.disruption.ServiceDisruptionScheme;
 import org.elasticsearch.test.transport.AssertingLocalTransport;
 import org.elasticsearch.test.transport.MockTransportService;
+import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.TransportSettings;
@@ -421,9 +422,9 @@ public final class InternalTestCluster extends TestCluster {
         // randomize netty settings
         if (random.nextBoolean()) {
             builder.put(NettyTransport.WORKER_COUNT.getKey(), random.nextInt(3) + 1);
-            builder.put(NettyTransport.CONNECTIONS_PER_NODE_RECOVERY.getKey(), random.nextInt(2) + 1);
-            builder.put(NettyTransport.CONNECTIONS_PER_NODE_BULK.getKey(), random.nextInt(3) + 1);
-            builder.put(NettyTransport.CONNECTIONS_PER_NODE_REG.getKey(), random.nextInt(6) + 1);
+            builder.put(TcpTransport.CONNECTIONS_PER_NODE_RECOVERY.getKey(), random.nextInt(2) + 1);
+            builder.put(TcpTransport.CONNECTIONS_PER_NODE_BULK.getKey(), random.nextInt(3) + 1);
+            builder.put(TcpTransport.CONNECTIONS_PER_NODE_REG.getKey(), random.nextInt(6) + 1);
         }
 
         if (random.nextBoolean()) {
@@ -455,7 +456,7 @@ public final class InternalTestCluster extends TestCluster {
         }
 
         if (random.nextBoolean()) {
-            builder.put(NettyTransport.PING_SCHEDULE.getKey(), RandomInts.randomIntBetween(random, 100, 2000) + "ms");
+            builder.put(TcpTransport.PING_SCHEDULE.getKey(), RandomInts.randomIntBetween(random, 100, 2000) + "ms");
         }
 
         if (random.nextBoolean()) {

From 8c6c00ff1526b6ca21935fb491174e99c1558efc Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Thu, 30 Jun 2016 13:57:43 +0200
Subject: [PATCH 1140/1311] Update documentation for cat/plugins API

Cat API for plugins doesn't display anymore url or jvm/site flag
---
 docs/reference/cat/plugins.asciidoc | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/docs/reference/cat/plugins.asciidoc b/docs/reference/cat/plugins.asciidoc
index 81df5cfb127..0af1faa5c9c 100644
--- a/docs/reference/cat/plugins.asciidoc
+++ b/docs/reference/cat/plugins.asciidoc
@@ -6,14 +6,9 @@ The `plugins` command provides a view per node of running plugins. This informat
 [source,sh]
 ------------------------------------------------------------------------------
 % curl 'localhost:9200/_cat/plugins?v'
-name    component       version        type isolation url
-Abraxas discovery-azure 2.1.0-SNAPSHOT j    x
-Abraxas lang-javascript 2.0.0-SNAPSHOT j    x
-Abraxas marvel          NA             j/s  x         /_plugin/marvel/
-Abraxas lang-python     2.0.0-SNAPSHOT j    x
-Abraxas inquisitor      NA             s              /_plugin/inquisitor/
-Abraxas kopf            0.5.2          s              /_plugin/kopf/
-Abraxas segmentspy      NA             s              /_plugin/segmentspy/
+name    component       version        description
+Abraxas discovery-gce   5.0.0          The Google Compute Engine (GCE) Discovery plugin allows to use GCE API for the unicast discovery mechanism. 
+Abraxas lang-javascript 5.0.0          The JavaScript language plugin allows to have javascript as the language of scripts to execute.
 -------------------------------------------------------------------------------
 
 We can tell quickly how many plugins per node we have and which versions.

From 2dee980a1a864bd3346099e6e00289709e7e45d5 Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Thu, 30 Jun 2016 14:05:29 +0200
Subject: [PATCH 1141/1311] LICENSE.txt is not needed in plugin root dir

We have licenses in licenses dir and the global license for the whole project is in the root dir so this file is not needed here.
---
 plugins/discovery-azure/LICENSE.txt | 202 ----------------------------
 1 file changed, 202 deletions(-)
 delete mode 100644 plugins/discovery-azure/LICENSE.txt

diff --git a/plugins/discovery-azure/LICENSE.txt b/plugins/discovery-azure/LICENSE.txt
deleted file mode 100644
index d6456956733..00000000000
--- a/plugins/discovery-azure/LICENSE.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

From 74d5fb3197a968e1dde17e6f529ad85602bc3ca2 Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Thu, 30 Jun 2016 14:06:31 +0200
Subject: [PATCH 1142/1311] LICENSE.txt is not needed in plugin root dir

We have licenses in licenses dir and the global license for the whole project is in the root dir so this file is not needed here.
---
 plugins/store-smb/LICENSE.txt | 202 ----------------------------------
 1 file changed, 202 deletions(-)
 delete mode 100644 plugins/store-smb/LICENSE.txt

diff --git a/plugins/store-smb/LICENSE.txt b/plugins/store-smb/LICENSE.txt
deleted file mode 100644
index d6456956733..00000000000
--- a/plugins/store-smb/LICENSE.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

From cd6535ea9b7803e3210b20cef4d7cb0357f2bbbe Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Thu, 30 Jun 2016 14:07:01 +0200
Subject: [PATCH 1143/1311] LICENSE.txt is not needed in plugin root dir

We have licenses in licenses dir and the global license for the whole project is in the root dir so this file is not needed here.
---
 plugins/repository-azure/LICENSE.txt | 202 ---------------------------
 1 file changed, 202 deletions(-)
 delete mode 100644 plugins/repository-azure/LICENSE.txt

diff --git a/plugins/repository-azure/LICENSE.txt b/plugins/repository-azure/LICENSE.txt
deleted file mode 100644
index d6456956733..00000000000
--- a/plugins/repository-azure/LICENSE.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

From 527a9c7f48d571d06e5e300e64681ad5d952f398 Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Thu, 30 Jun 2016 14:39:23 +0200
Subject: [PATCH 1144/1311] Deprecate discovery-azure and rename it to
 discovery-azure-classic

As discussed at https://github.com/elastic/elasticsearch-cloud-azure/issues/91#issuecomment-229113595, we know that the current `discovery-azure` plugin only works with Azure Classic VMs / Services (which is somehow Legacy now).

The proposal here is to rename `discovery-azure` to `discovery-azure-classic` in case some users are using it.
And deprecate it for 5.0.

Closes #19144.
---
 .../resources/checkstyle_suppressions.xml     |  8 ++---
 dev-tools/smoke_test_rc.py                    |  2 +-
 docs/plugins/discovery-azure.asciidoc         | 32 +++++++++----------
 docs/plugins/discovery.asciidoc               |  6 ++--
 docs/plugins/redirects.asciidoc               |  2 +-
 .../migration/migrate_5_0/plugins.asciidoc    |  2 +-
 .../modules/discovery/azure.asciidoc          |  8 ++---
 .../LICENSE.txt                               |  0
 .../build.gradle                              |  2 +-
 .../licenses/azure-LICENSE.txt                |  0
 .../licenses/azure-NOTICE.txt                 |  0
 .../licenses/azure-core-0.9.3.jar.sha1        |  0
 .../azure-svc-mgmt-compute-0.9.3.jar.sha1     |  0
 .../licenses/commons-codec-1.10.jar.sha1      |  0
 .../licenses/commons-codec-LICENSE.txt        |  0
 .../licenses/commons-codec-NOTICE.txt         |  0
 .../licenses/commons-io-2.4.jar.sha1          |  0
 .../licenses/commons-io-LICENSE.txt           |  0
 .../licenses/commons-io-NOTICE.txt            |  0
 .../licenses/commons-lang-2.6.jar.sha1        |  0
 .../licenses/commons-lang-LICENSE.txt         |  0
 .../licenses/commons-lang-NOTICE.txt          |  0
 .../licenses/commons-logging-1.1.3.jar.sha1   |  0
 .../licenses/commons-logging-LICENSE.txt      |  0
 .../licenses/commons-logging-NOTICE.txt       |  0
 .../licenses/httpclient-4.5.2.jar.sha1        |  0
 .../licenses/httpclient-LICENSE.txt           |  0
 .../licenses/httpclient-NOTICE.txt            |  0
 .../licenses/httpcore-4.4.4.jar.sha1          |  0
 .../licenses/httpcore-LICENSE.txt             |  0
 .../licenses/httpcore-NOTICE.txt              |  0
 .../licenses/jackson-LICENSE                  |  0
 .../licenses/jackson-NOTICE                   |  0
 .../licenses/jackson-core-asl-1.9.2.jar.sha1  |  0
 .../licenses/jackson-jaxrs-1.9.2.jar.sha1     |  0
 .../jackson-mapper-asl-1.9.2.jar.sha1         |  0
 .../licenses/jackson-xc-1.9.2.jar.sha1        |  0
 .../licenses/javax.inject-1.jar.sha1          |  0
 .../licenses/javax.inject-LICENSE.txt         |  0
 .../licenses/javax.inject-NOTICE.txt          |  0
 .../licenses/jaxb-LICENSE.txt                 |  0
 .../licenses/jaxb-NOTICE.txt                  |  0
 .../licenses/jaxb-api-2.2.2.jar.sha1          |  0
 .../licenses/jaxb-impl-2.2.3-1.jar.sha1       |  0
 .../licenses/jersey-LICENSE.txt               |  0
 .../licenses/jersey-NOTICE.txt                |  0
 .../licenses/jersey-client-1.13.jar.sha1      |  0
 .../licenses/jersey-core-1.13.jar.sha1        |  0
 .../licenses/jersey-json-1.13.jar.sha1        |  0
 .../licenses/jettison-1.1.jar.sha1            |  0
 .../licenses/jettison-LICENSE.txt             |  0
 .../licenses/jettison-NOTICE.txt              |  0
 .../licenses/mail-1.4.5.jar.sha1              |  0
 .../licenses/mail-LICENSE.txt                 |  0
 .../licenses/mail-NOTICE.txt                  |  0
 .../azure/classic}/AzureDiscoveryModule.java  | 27 ++++++++--------
 .../AzureServiceDisableException.java         |  2 +-
 .../classic}/AzureServiceRemoteException.java |  2 +-
 .../management/AzureComputeService.java       |  6 ++--
 .../management/AzureComputeServiceImpl.java   |  4 +--
 .../classic}/AzureUnicastHostsProvider.java   | 10 +++---
 .../azure/classic}/AzureDiscoveryPlugin.java  | 14 ++++----
 .../AbstractAzureComputeServiceTestCase.java  |  8 ++---
 .../AzureComputeServiceSimpleMock.java        |  4 +--
 .../AzureComputeServiceTwoNodesMock.java      |  4 +--
 .../AzureComputeServiceAbstractMock.java      |  2 +-
 .../AzureDiscoveryClusterFormationTests.java  |  6 ++--
 .../azure/classic}/AzureDiscoveryRestIT.java  |  2 +-
 .../AzureMinimumMasterNodesTests.java         |  6 ++--
 .../azure/classic}/AzureSimpleTests.java      | 10 +++---
 .../classic}/AzureTwoStartedNodesTests.java   | 10 +++---
 .../discovery_azure_classic/10_basic.yaml     | 13 ++++++++
 .../test/discovery_azure/10_basic.yaml        | 13 --------
 .../scripts/module_and_plugin_test_cases.bash |  6 ++--
 settings.gradle                               |  2 +-
 75 files changed, 107 insertions(+), 106 deletions(-)
 rename plugins/{discovery-azure => discovery-azure-classic}/LICENSE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/build.gradle (99%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/azure-LICENSE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/azure-NOTICE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/azure-core-0.9.3.jar.sha1 (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/azure-svc-mgmt-compute-0.9.3.jar.sha1 (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/commons-codec-1.10.jar.sha1 (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/commons-codec-LICENSE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/commons-codec-NOTICE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/commons-io-2.4.jar.sha1 (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/commons-io-LICENSE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/commons-io-NOTICE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/commons-lang-2.6.jar.sha1 (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/commons-lang-LICENSE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/commons-lang-NOTICE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/commons-logging-1.1.3.jar.sha1 (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/commons-logging-LICENSE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/commons-logging-NOTICE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/httpclient-4.5.2.jar.sha1 (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/httpclient-LICENSE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/httpclient-NOTICE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/httpcore-4.4.4.jar.sha1 (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/httpcore-LICENSE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/httpcore-NOTICE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/jackson-LICENSE (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/jackson-NOTICE (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/jackson-core-asl-1.9.2.jar.sha1 (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/jackson-jaxrs-1.9.2.jar.sha1 (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/jackson-mapper-asl-1.9.2.jar.sha1 (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/jackson-xc-1.9.2.jar.sha1 (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/javax.inject-1.jar.sha1 (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/javax.inject-LICENSE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/javax.inject-NOTICE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/jaxb-LICENSE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/jaxb-NOTICE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/jaxb-api-2.2.2.jar.sha1 (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/jaxb-impl-2.2.3-1.jar.sha1 (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/jersey-LICENSE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/jersey-NOTICE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/jersey-client-1.13.jar.sha1 (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/jersey-core-1.13.jar.sha1 (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/jersey-json-1.13.jar.sha1 (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/jettison-1.1.jar.sha1 (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/jettison-LICENSE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/jettison-NOTICE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/mail-1.4.5.jar.sha1 (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/mail-LICENSE.txt (100%)
 rename plugins/{discovery-azure => discovery-azure-classic}/licenses/mail-NOTICE.txt (100%)
 rename plugins/{discovery-azure/src/main/java/org/elasticsearch/cloud/azure => discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic}/AzureDiscoveryModule.java (76%)
 rename plugins/{discovery-azure/src/main/java/org/elasticsearch/cloud/azure => discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic}/AzureServiceDisableException.java (95%)
 rename plugins/{discovery-azure/src/main/java/org/elasticsearch/cloud/azure => discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic}/AzureServiceRemoteException.java (95%)
 rename plugins/{discovery-azure/src/main/java/org/elasticsearch/cloud/azure => discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic}/management/AzureComputeService.java (94%)
 rename plugins/{discovery-azure/src/main/java/org/elasticsearch/cloud/azure => discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic}/management/AzureComputeServiceImpl.java (97%)
 rename plugins/{discovery-azure/src/main/java/org/elasticsearch/discovery/azure => discovery-azure-classic/src/main/java/org/elasticsearch/discovery/azure/classic}/AzureUnicastHostsProvider.java (96%)
 rename plugins/{discovery-azure/src/main/java/org/elasticsearch/plugin/discovery/azure => discovery-azure-classic/src/main/java/org/elasticsearch/plugin/discovery/azure/classic}/AzureDiscoveryPlugin.java (82%)
 rename plugins/{discovery-azure/src/test/java/org/elasticsearch/cloud/azure => discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic}/AbstractAzureComputeServiceTestCase.java (89%)
 rename plugins/{discovery-azure/src/test/java/org/elasticsearch/cloud/azure => discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic}/AzureComputeServiceSimpleMock.java (95%)
 rename plugins/{discovery-azure/src/test/java/org/elasticsearch/cloud/azure => discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic}/AzureComputeServiceTwoNodesMock.java (96%)
 rename plugins/{discovery-azure/src/test/java/org/elasticsearch/cloud/azure => discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic}/management/AzureComputeServiceAbstractMock.java (96%)
 rename plugins/{discovery-azure/src/test/java/org/elasticsearch/discovery/azure => discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic}/AzureDiscoveryClusterFormationTests.java (98%)
 rename plugins/{discovery-azure/src/test/java/org/elasticsearch/discovery/azure => discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic}/AzureDiscoveryRestIT.java (96%)
 rename plugins/{discovery-azure/src/test/java/org/elasticsearch/discovery/azure => discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic}/AzureMinimumMasterNodesTests.java (94%)
 rename plugins/{discovery-azure/src/test/java/org/elasticsearch/discovery/azure => discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic}/AzureSimpleTests.java (89%)
 rename plugins/{discovery-azure/src/test/java/org/elasticsearch/discovery/azure => discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic}/AzureTwoStartedNodesTests.java (89%)
 create mode 100644 plugins/discovery-azure-classic/src/test/resources/rest-api-spec/test/discovery_azure_classic/10_basic.yaml
 delete mode 100644 plugins/discovery-azure/src/test/resources/rest-api-spec/test/discovery_azure/10_basic.yaml

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 5a08a834c3c..94f962be19d 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -1180,10 +1180,10 @@
   <suppress files="plugins[/\\]analysis-kuromoji[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]JapaneseStopTokenFilterFactory.java" checks="LineLength" />
   <suppress files="plugins[/\\]analysis-kuromoji[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]KuromojiAnalysisTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]analysis-phonetic[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]analysis[/\\]PhoneticTokenFilterFactory.java" checks="LineLength" />
-  <suppress files="plugins[/\\]discovery-azure[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]AbstractAzureTestCase.java" checks="LineLength" />
-  <suppress files="plugins[/\\]discovery-azure[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]azure[/\\]AzureMinimumMasterNodesTests.java" checks="LineLength" />
-  <suppress files="plugins[/\\]discovery-azure[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]azure[/\\]AzureSimpleTests.java" checks="LineLength" />
-  <suppress files="plugins[/\\]discovery-azure[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]azure[/\\]AzureTwoStartedNodesTests.java" checks="LineLength" />
+  <suppress files="plugins[/\\]discovery-azure-classic[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]AbstractAzureTestCase.java" checks="LineLength" />
+  <suppress files="plugins[/\\]discovery-azure-classic[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]azure[/\\]AzureMinimumMasterNodesTests.java" checks="LineLength" />
+  <suppress files="plugins[/\\]discovery-azure-classic[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]azure[/\\]AzureSimpleTests.java" checks="LineLength" />
+  <suppress files="plugins[/\\]discovery-azure-classic[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]azure[/\\]AzureTwoStartedNodesTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]discovery-ec2[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]aws[/\\]AbstractAwsTestCase.java" checks="LineLength" />
   <suppress files="plugins[/\\]discovery-ec2[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]ec2[/\\]AmazonEC2Mock.java" checks="LineLength" />
   <suppress files="plugins[/\\]discovery-gce[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]gce[/\\]GceNetworkTests.java" checks="LineLength" />
diff --git a/dev-tools/smoke_test_rc.py b/dev-tools/smoke_test_rc.py
index 259081ddc10..33abbf96345 100644
--- a/dev-tools/smoke_test_rc.py
+++ b/dev-tools/smoke_test_rc.py
@@ -63,7 +63,7 @@ DEFAULT_PLUGINS = ["analysis-icu",
                    "analysis-phonetic",
                    "analysis-smartcn",
                    "analysis-stempel",
-                   "discovery-azure",
+                   "discovery-azure-classic",
                    "discovery-ec2",
                    "discovery-gce",
                    "ingest-attachment",
diff --git a/docs/plugins/discovery-azure.asciidoc b/docs/plugins/discovery-azure.asciidoc
index f49f1568cab..022c453e148 100644
--- a/docs/plugins/discovery-azure.asciidoc
+++ b/docs/plugins/discovery-azure.asciidoc
@@ -1,9 +1,9 @@
-[[discovery-azure]]
+[[discovery-azure-classic]]
 === Azure Discovery Plugin
 
-The Azure Discovery plugin uses the Azure API for unicast discovery.
+The Azure Classic Discovery plugin uses the Azure Classic API for unicast discovery.
 
-[[discovery-azure-install]]
+[[discovery-azure-classic-install]]
 [float]
 ==== Installation
 
@@ -11,13 +11,13 @@ This plugin can be installed using the plugin manager:
 
 [source,sh]
 ----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install discovery-azure
+sudo bin/elasticsearch-plugin install discovery-azure-classic
 ----------------------------------------------------------------
 
 The plugin must be installed on every node in the cluster, and each node must
 be restarted after installation.
 
-[[discovery-azure-remove]]
+[[discovery-azure-classic-remove]]
 [float]
 ==== Removal
 
@@ -25,12 +25,12 @@ The plugin can be removed with the following command:
 
 [source,sh]
 ----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove discovery-azure
+sudo bin/elasticsearch-plugin remove discovery-azure-classic
 ----------------------------------------------------------------
 
 The node must be stopped before removing the plugin.
 
-[[discovery-azure-usage]]
+[[discovery-azure-classic-usage]]
 ==== Azure Virtual Machine Discovery
 
 Azure VM discovery allows to use the azure APIs to perform automatic discovery (similar to multicast in non hostile
@@ -64,7 +64,7 @@ You can use {ref}/modules-network.html[core network host settings]. For example
 
 ==============================================
 
-[[discovery-azure-short]]
+[[discovery-azure-classic-short]]
 ===== How to start (short story)
 
 * Create Azure instances
@@ -73,7 +73,7 @@ You can use {ref}/modules-network.html[core network host settings]. For example
 * Modify `elasticsearch.yml` file
 * Start Elasticsearch
 
-[[discovery-azure-settings]]
+[[discovery-azure-classic-settings]]
 ===== Azure credential API settings
 
 The following are a list of settings that can further control the credential API:
@@ -100,7 +100,7 @@ The following are a list of settings that can further control the credential API
     your_azure_cloud_service_name
 
 
-[[discovery-azure-settings-advanced]]
+[[discovery-azure-classic-settings-advanced]]
 ===== Advanced settings
 
 The following are a list of settings that can further control the discovery:
@@ -143,7 +143,7 @@ discovery:
             slot: production
 ----
 
-[[discovery-azure-long]]
+[[discovery-azure-classic-long]]
 ==== Setup process for Azure Discovery
 
 We will expose here one strategy which is to hide our Elasticsearch cluster from outside.
@@ -153,7 +153,7 @@ other.  That means that with this mode, you can use elasticsearch unicast
 discovery to build a cluster, using the Azure API to retrieve information
 about your nodes.
 
-[[discovery-azure-long-prerequisites]]
+[[discovery-azure-classic-long-prerequisites]]
 ===== Prerequisites
 
 Before starting, you need to have:
@@ -243,7 +243,7 @@ azure account download
 azure account import /tmp/azure.publishsettings
 ----
 
-[[discovery-azure-long-instance]]
+[[discovery-azure-classic-long-instance]]
 ===== Creating your first instance
 
 You need to have a storage account available. Check http://www.windowsazure.com/en-us/develop/net/how-to-guides/blob-storage/#create-account[Azure Blob Storage documentation]
@@ -396,7 +396,7 @@ This command should give you a JSON result:
 }
 ----
 
-[[discovery-azure-long-plugin]]
+[[discovery-azure-classic-long-plugin]]
 ===== Install elasticsearch cloud azure plugin
 
 [source,sh]
@@ -405,7 +405,7 @@ This command should give you a JSON result:
 sudo service elasticsearch stop
 
 # Install the plugin
-sudo /usr/share/elasticsearch/bin/elasticsearch-plugin install discovery-azure
+sudo /usr/share/elasticsearch/bin/elasticsearch-plugin install discovery-azure-classic
 
 # Configure it
 sudo vi /etc/elasticsearch/elasticsearch.yml
@@ -441,7 +441,7 @@ sudo service elasticsearch start
 
 If anything goes wrong, check your logs in `/var/log/elasticsearch`.
 
-[[discovery-azure-scale]]
+[[discovery-azure-classic-scale]]
 ==== Scaling Out!
 
 You need first to create an image of your previous machine.
diff --git a/docs/plugins/discovery.asciidoc b/docs/plugins/discovery.asciidoc
index 62c5b4551ac..ed9cfc0ed3d 100644
--- a/docs/plugins/discovery.asciidoc
+++ b/docs/plugins/discovery.asciidoc
@@ -13,9 +13,9 @@ The core discovery plugins are:
 
 The EC2 discovery plugin uses the https://github.com/aws/aws-sdk-java[AWS API] for unicast discovery.
 
-<<discovery-azure,Azure discovery>>::
+<<discovery-azure-classic,Azure discovery>>::
 
-The Azure discovery plugin uses the Azure API for unicast discovery.
+The Azure Classic discovery plugin uses the Azure Classic API for unicast discovery.
 
 <<discovery-gce,GCE discovery>>::
 
@@ -33,7 +33,7 @@ A number of discovery plugins have been contributed by our community:
 
 include::discovery-ec2.asciidoc[]
 
-include::discovery-azure.asciidoc[]
+include::discovery-azure-classic.asciidoc[]
 
 include::discovery-gce.asciidoc[]
 
diff --git a/docs/plugins/redirects.asciidoc b/docs/plugins/redirects.asciidoc
index c8cf10c6319..0f9c0b40f2c 100644
--- a/docs/plugins/redirects.asciidoc
+++ b/docs/plugins/redirects.asciidoc
@@ -24,7 +24,7 @@ The `cloud-aws` plugin has been split into two separate plugins:
 
 The `cloud-azure` plugin has been split into two separate plugins:
 
-*  <<discovery-azure>> (`discovery-azure`)
+*  <<discovery-azure-classic>> (`discovery-azure-classic`)
 *  <<repository-azure>> (`repository-azure`)
 
 
diff --git a/docs/reference/migration/migrate_5_0/plugins.asciidoc b/docs/reference/migration/migrate_5_0/plugins.asciidoc
index 2826c822d15..e1ff497a8f3 100644
--- a/docs/reference/migration/migrate_5_0/plugins.asciidoc
+++ b/docs/reference/migration/migrate_5_0/plugins.asciidoc
@@ -63,7 +63,7 @@ Proxy settings for both plugins have been renamed:
 
 Cloud Azure plugin has been split in three plugins:
 
-* {plugins}/discovery-azure.html[Discovery Azure plugin]
+* {plugins}/discovery-azure-classic.html[Discovery Azure plugin]
 * {plugins}/repository-azure.html[Repository Azure plugin]
 * {plugins}/store-smb.html[Store SMB plugin]
 
diff --git a/docs/reference/modules/discovery/azure.asciidoc b/docs/reference/modules/discovery/azure.asciidoc
index 87d072564b3..1343819b02a 100644
--- a/docs/reference/modules/discovery/azure.asciidoc
+++ b/docs/reference/modules/discovery/azure.asciidoc
@@ -1,5 +1,5 @@
-[[modules-discovery-azure]]
-=== Azure Discovery
+[[modules-discovery-azure-classic]]
+=== Azure Classic Discovery
 
-Azure discovery allows to use the Azure APIs to perform automatic discovery (similar to multicast).
-It is available as a plugin. See {plugins}/discovery-azure.html[discovery-azure] for more information.
+Azure classic discovery allows to use the Azure Classic APIs to perform automatic discovery (similar to multicast).
+It is available as a plugin. See {plugins}/discovery-azure-classic.html[discovery-azure-classic] for more information.
diff --git a/plugins/discovery-azure/LICENSE.txt b/plugins/discovery-azure-classic/LICENSE.txt
similarity index 100%
rename from plugins/discovery-azure/LICENSE.txt
rename to plugins/discovery-azure-classic/LICENSE.txt
diff --git a/plugins/discovery-azure/build.gradle b/plugins/discovery-azure-classic/build.gradle
similarity index 99%
rename from plugins/discovery-azure/build.gradle
rename to plugins/discovery-azure-classic/build.gradle
index 8f0c641e150..42f6fd29106 100644
--- a/plugins/discovery-azure/build.gradle
+++ b/plugins/discovery-azure-classic/build.gradle
@@ -21,7 +21,7 @@ import org.elasticsearch.gradle.LoggedExec
 
 esplugin {
   description 'The Azure Discovery plugin allows to use Azure API for the unicast discovery mechanism'
-  classname 'org.elasticsearch.plugin.discovery.azure.AzureDiscoveryPlugin'
+  classname 'AzureDiscoveryPlugin'
 }
 
 versions << [
diff --git a/plugins/discovery-azure/licenses/azure-LICENSE.txt b/plugins/discovery-azure-classic/licenses/azure-LICENSE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/azure-LICENSE.txt
rename to plugins/discovery-azure-classic/licenses/azure-LICENSE.txt
diff --git a/plugins/discovery-azure/licenses/azure-NOTICE.txt b/plugins/discovery-azure-classic/licenses/azure-NOTICE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/azure-NOTICE.txt
rename to plugins/discovery-azure-classic/licenses/azure-NOTICE.txt
diff --git a/plugins/discovery-azure/licenses/azure-core-0.9.3.jar.sha1 b/plugins/discovery-azure-classic/licenses/azure-core-0.9.3.jar.sha1
similarity index 100%
rename from plugins/discovery-azure/licenses/azure-core-0.9.3.jar.sha1
rename to plugins/discovery-azure-classic/licenses/azure-core-0.9.3.jar.sha1
diff --git a/plugins/discovery-azure/licenses/azure-svc-mgmt-compute-0.9.3.jar.sha1 b/plugins/discovery-azure-classic/licenses/azure-svc-mgmt-compute-0.9.3.jar.sha1
similarity index 100%
rename from plugins/discovery-azure/licenses/azure-svc-mgmt-compute-0.9.3.jar.sha1
rename to plugins/discovery-azure-classic/licenses/azure-svc-mgmt-compute-0.9.3.jar.sha1
diff --git a/plugins/discovery-azure/licenses/commons-codec-1.10.jar.sha1 b/plugins/discovery-azure-classic/licenses/commons-codec-1.10.jar.sha1
similarity index 100%
rename from plugins/discovery-azure/licenses/commons-codec-1.10.jar.sha1
rename to plugins/discovery-azure-classic/licenses/commons-codec-1.10.jar.sha1
diff --git a/plugins/discovery-azure/licenses/commons-codec-LICENSE.txt b/plugins/discovery-azure-classic/licenses/commons-codec-LICENSE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/commons-codec-LICENSE.txt
rename to plugins/discovery-azure-classic/licenses/commons-codec-LICENSE.txt
diff --git a/plugins/discovery-azure/licenses/commons-codec-NOTICE.txt b/plugins/discovery-azure-classic/licenses/commons-codec-NOTICE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/commons-codec-NOTICE.txt
rename to plugins/discovery-azure-classic/licenses/commons-codec-NOTICE.txt
diff --git a/plugins/discovery-azure/licenses/commons-io-2.4.jar.sha1 b/plugins/discovery-azure-classic/licenses/commons-io-2.4.jar.sha1
similarity index 100%
rename from plugins/discovery-azure/licenses/commons-io-2.4.jar.sha1
rename to plugins/discovery-azure-classic/licenses/commons-io-2.4.jar.sha1
diff --git a/plugins/discovery-azure/licenses/commons-io-LICENSE.txt b/plugins/discovery-azure-classic/licenses/commons-io-LICENSE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/commons-io-LICENSE.txt
rename to plugins/discovery-azure-classic/licenses/commons-io-LICENSE.txt
diff --git a/plugins/discovery-azure/licenses/commons-io-NOTICE.txt b/plugins/discovery-azure-classic/licenses/commons-io-NOTICE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/commons-io-NOTICE.txt
rename to plugins/discovery-azure-classic/licenses/commons-io-NOTICE.txt
diff --git a/plugins/discovery-azure/licenses/commons-lang-2.6.jar.sha1 b/plugins/discovery-azure-classic/licenses/commons-lang-2.6.jar.sha1
similarity index 100%
rename from plugins/discovery-azure/licenses/commons-lang-2.6.jar.sha1
rename to plugins/discovery-azure-classic/licenses/commons-lang-2.6.jar.sha1
diff --git a/plugins/discovery-azure/licenses/commons-lang-LICENSE.txt b/plugins/discovery-azure-classic/licenses/commons-lang-LICENSE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/commons-lang-LICENSE.txt
rename to plugins/discovery-azure-classic/licenses/commons-lang-LICENSE.txt
diff --git a/plugins/discovery-azure/licenses/commons-lang-NOTICE.txt b/plugins/discovery-azure-classic/licenses/commons-lang-NOTICE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/commons-lang-NOTICE.txt
rename to plugins/discovery-azure-classic/licenses/commons-lang-NOTICE.txt
diff --git a/plugins/discovery-azure/licenses/commons-logging-1.1.3.jar.sha1 b/plugins/discovery-azure-classic/licenses/commons-logging-1.1.3.jar.sha1
similarity index 100%
rename from plugins/discovery-azure/licenses/commons-logging-1.1.3.jar.sha1
rename to plugins/discovery-azure-classic/licenses/commons-logging-1.1.3.jar.sha1
diff --git a/plugins/discovery-azure/licenses/commons-logging-LICENSE.txt b/plugins/discovery-azure-classic/licenses/commons-logging-LICENSE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/commons-logging-LICENSE.txt
rename to plugins/discovery-azure-classic/licenses/commons-logging-LICENSE.txt
diff --git a/plugins/discovery-azure/licenses/commons-logging-NOTICE.txt b/plugins/discovery-azure-classic/licenses/commons-logging-NOTICE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/commons-logging-NOTICE.txt
rename to plugins/discovery-azure-classic/licenses/commons-logging-NOTICE.txt
diff --git a/plugins/discovery-azure/licenses/httpclient-4.5.2.jar.sha1 b/plugins/discovery-azure-classic/licenses/httpclient-4.5.2.jar.sha1
similarity index 100%
rename from plugins/discovery-azure/licenses/httpclient-4.5.2.jar.sha1
rename to plugins/discovery-azure-classic/licenses/httpclient-4.5.2.jar.sha1
diff --git a/plugins/discovery-azure/licenses/httpclient-LICENSE.txt b/plugins/discovery-azure-classic/licenses/httpclient-LICENSE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/httpclient-LICENSE.txt
rename to plugins/discovery-azure-classic/licenses/httpclient-LICENSE.txt
diff --git a/plugins/discovery-azure/licenses/httpclient-NOTICE.txt b/plugins/discovery-azure-classic/licenses/httpclient-NOTICE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/httpclient-NOTICE.txt
rename to plugins/discovery-azure-classic/licenses/httpclient-NOTICE.txt
diff --git a/plugins/discovery-azure/licenses/httpcore-4.4.4.jar.sha1 b/plugins/discovery-azure-classic/licenses/httpcore-4.4.4.jar.sha1
similarity index 100%
rename from plugins/discovery-azure/licenses/httpcore-4.4.4.jar.sha1
rename to plugins/discovery-azure-classic/licenses/httpcore-4.4.4.jar.sha1
diff --git a/plugins/discovery-azure/licenses/httpcore-LICENSE.txt b/plugins/discovery-azure-classic/licenses/httpcore-LICENSE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/httpcore-LICENSE.txt
rename to plugins/discovery-azure-classic/licenses/httpcore-LICENSE.txt
diff --git a/plugins/discovery-azure/licenses/httpcore-NOTICE.txt b/plugins/discovery-azure-classic/licenses/httpcore-NOTICE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/httpcore-NOTICE.txt
rename to plugins/discovery-azure-classic/licenses/httpcore-NOTICE.txt
diff --git a/plugins/discovery-azure/licenses/jackson-LICENSE b/plugins/discovery-azure-classic/licenses/jackson-LICENSE
similarity index 100%
rename from plugins/discovery-azure/licenses/jackson-LICENSE
rename to plugins/discovery-azure-classic/licenses/jackson-LICENSE
diff --git a/plugins/discovery-azure/licenses/jackson-NOTICE b/plugins/discovery-azure-classic/licenses/jackson-NOTICE
similarity index 100%
rename from plugins/discovery-azure/licenses/jackson-NOTICE
rename to plugins/discovery-azure-classic/licenses/jackson-NOTICE
diff --git a/plugins/discovery-azure/licenses/jackson-core-asl-1.9.2.jar.sha1 b/plugins/discovery-azure-classic/licenses/jackson-core-asl-1.9.2.jar.sha1
similarity index 100%
rename from plugins/discovery-azure/licenses/jackson-core-asl-1.9.2.jar.sha1
rename to plugins/discovery-azure-classic/licenses/jackson-core-asl-1.9.2.jar.sha1
diff --git a/plugins/discovery-azure/licenses/jackson-jaxrs-1.9.2.jar.sha1 b/plugins/discovery-azure-classic/licenses/jackson-jaxrs-1.9.2.jar.sha1
similarity index 100%
rename from plugins/discovery-azure/licenses/jackson-jaxrs-1.9.2.jar.sha1
rename to plugins/discovery-azure-classic/licenses/jackson-jaxrs-1.9.2.jar.sha1
diff --git a/plugins/discovery-azure/licenses/jackson-mapper-asl-1.9.2.jar.sha1 b/plugins/discovery-azure-classic/licenses/jackson-mapper-asl-1.9.2.jar.sha1
similarity index 100%
rename from plugins/discovery-azure/licenses/jackson-mapper-asl-1.9.2.jar.sha1
rename to plugins/discovery-azure-classic/licenses/jackson-mapper-asl-1.9.2.jar.sha1
diff --git a/plugins/discovery-azure/licenses/jackson-xc-1.9.2.jar.sha1 b/plugins/discovery-azure-classic/licenses/jackson-xc-1.9.2.jar.sha1
similarity index 100%
rename from plugins/discovery-azure/licenses/jackson-xc-1.9.2.jar.sha1
rename to plugins/discovery-azure-classic/licenses/jackson-xc-1.9.2.jar.sha1
diff --git a/plugins/discovery-azure/licenses/javax.inject-1.jar.sha1 b/plugins/discovery-azure-classic/licenses/javax.inject-1.jar.sha1
similarity index 100%
rename from plugins/discovery-azure/licenses/javax.inject-1.jar.sha1
rename to plugins/discovery-azure-classic/licenses/javax.inject-1.jar.sha1
diff --git a/plugins/discovery-azure/licenses/javax.inject-LICENSE.txt b/plugins/discovery-azure-classic/licenses/javax.inject-LICENSE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/javax.inject-LICENSE.txt
rename to plugins/discovery-azure-classic/licenses/javax.inject-LICENSE.txt
diff --git a/plugins/discovery-azure/licenses/javax.inject-NOTICE.txt b/plugins/discovery-azure-classic/licenses/javax.inject-NOTICE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/javax.inject-NOTICE.txt
rename to plugins/discovery-azure-classic/licenses/javax.inject-NOTICE.txt
diff --git a/plugins/discovery-azure/licenses/jaxb-LICENSE.txt b/plugins/discovery-azure-classic/licenses/jaxb-LICENSE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/jaxb-LICENSE.txt
rename to plugins/discovery-azure-classic/licenses/jaxb-LICENSE.txt
diff --git a/plugins/discovery-azure/licenses/jaxb-NOTICE.txt b/plugins/discovery-azure-classic/licenses/jaxb-NOTICE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/jaxb-NOTICE.txt
rename to plugins/discovery-azure-classic/licenses/jaxb-NOTICE.txt
diff --git a/plugins/discovery-azure/licenses/jaxb-api-2.2.2.jar.sha1 b/plugins/discovery-azure-classic/licenses/jaxb-api-2.2.2.jar.sha1
similarity index 100%
rename from plugins/discovery-azure/licenses/jaxb-api-2.2.2.jar.sha1
rename to plugins/discovery-azure-classic/licenses/jaxb-api-2.2.2.jar.sha1
diff --git a/plugins/discovery-azure/licenses/jaxb-impl-2.2.3-1.jar.sha1 b/plugins/discovery-azure-classic/licenses/jaxb-impl-2.2.3-1.jar.sha1
similarity index 100%
rename from plugins/discovery-azure/licenses/jaxb-impl-2.2.3-1.jar.sha1
rename to plugins/discovery-azure-classic/licenses/jaxb-impl-2.2.3-1.jar.sha1
diff --git a/plugins/discovery-azure/licenses/jersey-LICENSE.txt b/plugins/discovery-azure-classic/licenses/jersey-LICENSE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/jersey-LICENSE.txt
rename to plugins/discovery-azure-classic/licenses/jersey-LICENSE.txt
diff --git a/plugins/discovery-azure/licenses/jersey-NOTICE.txt b/plugins/discovery-azure-classic/licenses/jersey-NOTICE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/jersey-NOTICE.txt
rename to plugins/discovery-azure-classic/licenses/jersey-NOTICE.txt
diff --git a/plugins/discovery-azure/licenses/jersey-client-1.13.jar.sha1 b/plugins/discovery-azure-classic/licenses/jersey-client-1.13.jar.sha1
similarity index 100%
rename from plugins/discovery-azure/licenses/jersey-client-1.13.jar.sha1
rename to plugins/discovery-azure-classic/licenses/jersey-client-1.13.jar.sha1
diff --git a/plugins/discovery-azure/licenses/jersey-core-1.13.jar.sha1 b/plugins/discovery-azure-classic/licenses/jersey-core-1.13.jar.sha1
similarity index 100%
rename from plugins/discovery-azure/licenses/jersey-core-1.13.jar.sha1
rename to plugins/discovery-azure-classic/licenses/jersey-core-1.13.jar.sha1
diff --git a/plugins/discovery-azure/licenses/jersey-json-1.13.jar.sha1 b/plugins/discovery-azure-classic/licenses/jersey-json-1.13.jar.sha1
similarity index 100%
rename from plugins/discovery-azure/licenses/jersey-json-1.13.jar.sha1
rename to plugins/discovery-azure-classic/licenses/jersey-json-1.13.jar.sha1
diff --git a/plugins/discovery-azure/licenses/jettison-1.1.jar.sha1 b/plugins/discovery-azure-classic/licenses/jettison-1.1.jar.sha1
similarity index 100%
rename from plugins/discovery-azure/licenses/jettison-1.1.jar.sha1
rename to plugins/discovery-azure-classic/licenses/jettison-1.1.jar.sha1
diff --git a/plugins/discovery-azure/licenses/jettison-LICENSE.txt b/plugins/discovery-azure-classic/licenses/jettison-LICENSE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/jettison-LICENSE.txt
rename to plugins/discovery-azure-classic/licenses/jettison-LICENSE.txt
diff --git a/plugins/discovery-azure/licenses/jettison-NOTICE.txt b/plugins/discovery-azure-classic/licenses/jettison-NOTICE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/jettison-NOTICE.txt
rename to plugins/discovery-azure-classic/licenses/jettison-NOTICE.txt
diff --git a/plugins/discovery-azure/licenses/mail-1.4.5.jar.sha1 b/plugins/discovery-azure-classic/licenses/mail-1.4.5.jar.sha1
similarity index 100%
rename from plugins/discovery-azure/licenses/mail-1.4.5.jar.sha1
rename to plugins/discovery-azure-classic/licenses/mail-1.4.5.jar.sha1
diff --git a/plugins/discovery-azure/licenses/mail-LICENSE.txt b/plugins/discovery-azure-classic/licenses/mail-LICENSE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/mail-LICENSE.txt
rename to plugins/discovery-azure-classic/licenses/mail-LICENSE.txt
diff --git a/plugins/discovery-azure/licenses/mail-NOTICE.txt b/plugins/discovery-azure-classic/licenses/mail-NOTICE.txt
similarity index 100%
rename from plugins/discovery-azure/licenses/mail-NOTICE.txt
rename to plugins/discovery-azure-classic/licenses/mail-NOTICE.txt
diff --git a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/AzureDiscoveryModule.java b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/AzureDiscoveryModule.java
similarity index 76%
rename from plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/AzureDiscoveryModule.java
rename to plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/AzureDiscoveryModule.java
index 2c9c6e0a486..da684fd824d 100644
--- a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/AzureDiscoveryModule.java
+++ b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/AzureDiscoveryModule.java
@@ -17,12 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.cloud.azure;
+package org.elasticsearch.cloud.azure.classic;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.cloud.azure.management.AzureComputeService;
-import org.elasticsearch.cloud.azure.management.AzureComputeService.Management;
-import org.elasticsearch.cloud.azure.management.AzureComputeServiceImpl;
+import org.elasticsearch.cloud.azure.classic.management.AzureComputeService;
+import org.elasticsearch.cloud.azure.classic.management.AzureComputeServiceImpl;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.inject.Inject;
@@ -31,7 +30,7 @@ import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.DiscoveryModule;
-import org.elasticsearch.plugin.discovery.azure.AzureDiscoveryPlugin;
+import org.elasticsearch.plugin.discovery.azure.classic.AzureDiscoveryPlugin;
 
 /**
  * Azure Module
@@ -41,7 +40,7 @@ import org.elasticsearch.plugin.discovery.azure.AzureDiscoveryPlugin;
  * to AzureComputeServiceImpl.</li>
  * </ul>
  *
- * @see org.elasticsearch.cloud.azure.management.AzureComputeServiceImpl
+ * @see AzureComputeServiceImpl
  */
 public class AzureDiscoveryModule extends AbstractModule {
     protected final ESLogger logger;
@@ -77,19 +76,19 @@ public class AzureDiscoveryModule extends AbstractModule {
             return false;
         }
 
-        if (isDefined(settings, Management.SUBSCRIPTION_ID_SETTING) &&
-            isDefined(settings, Management.SERVICE_NAME_SETTING) &&
-            isDefined(settings, Management.KEYSTORE_PATH_SETTING) &&
-            isDefined(settings, Management.KEYSTORE_PASSWORD_SETTING)) {
+        if (isDefined(settings, AzureComputeService.Management.SUBSCRIPTION_ID_SETTING) &&
+            isDefined(settings, AzureComputeService.Management.SERVICE_NAME_SETTING) &&
+            isDefined(settings, AzureComputeService.Management.KEYSTORE_PATH_SETTING) &&
+            isDefined(settings, AzureComputeService.Management.KEYSTORE_PASSWORD_SETTING)) {
             logger.trace("All required properties for Azure discovery are set!");
             return true;
         } else {
             logger.debug("One or more Azure discovery settings are missing. " +
                             "Check elasticsearch.yml file. Should have [{}], [{}], [{}] and [{}].",
-                    Management.SUBSCRIPTION_ID_SETTING.getKey(),
-                    Management.SERVICE_NAME_SETTING.getKey(),
-                    Management.KEYSTORE_PATH_SETTING.getKey(),
-                    Management.KEYSTORE_PASSWORD_SETTING.getKey());
+                    AzureComputeService.Management.SUBSCRIPTION_ID_SETTING.getKey(),
+                    AzureComputeService.Management.SERVICE_NAME_SETTING.getKey(),
+                    AzureComputeService.Management.KEYSTORE_PATH_SETTING.getKey(),
+                    AzureComputeService.Management.KEYSTORE_PASSWORD_SETTING.getKey());
             return false;
         }
     }
diff --git a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/AzureServiceDisableException.java b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/AzureServiceDisableException.java
similarity index 95%
rename from plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/AzureServiceDisableException.java
rename to plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/AzureServiceDisableException.java
index 487997d71b6..66488f90c31 100644
--- a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/AzureServiceDisableException.java
+++ b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/AzureServiceDisableException.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.cloud.azure;
+package org.elasticsearch.cloud.azure.classic;
 
 public class AzureServiceDisableException extends IllegalStateException {
     public AzureServiceDisableException(String msg) {
diff --git a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/AzureServiceRemoteException.java b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/AzureServiceRemoteException.java
similarity index 95%
rename from plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/AzureServiceRemoteException.java
rename to plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/AzureServiceRemoteException.java
index 4bd4f1d67f1..c961c03ba71 100644
--- a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/AzureServiceRemoteException.java
+++ b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/AzureServiceRemoteException.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.cloud.azure;
+package org.elasticsearch.cloud.azure.classic;
 
 public class AzureServiceRemoteException extends IllegalStateException {
     public AzureServiceRemoteException(String msg) {
diff --git a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeService.java b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeService.java
similarity index 94%
rename from plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeService.java
rename to plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeService.java
index 526f98025b7..49e609aad80 100644
--- a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeService.java
+++ b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeService.java
@@ -17,15 +17,15 @@
  * under the License.
  */
 
-package org.elasticsearch.cloud.azure.management;
+package org.elasticsearch.cloud.azure.classic.management;
 
 import com.microsoft.windowsazure.core.utils.KeyStoreType;
 import com.microsoft.windowsazure.management.compute.models.HostedServiceGetDetailedResponse;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.discovery.azure.AzureUnicastHostsProvider;
-import org.elasticsearch.discovery.azure.AzureUnicastHostsProvider.Deployment;
+import org.elasticsearch.discovery.azure.classic.AzureUnicastHostsProvider;
+import org.elasticsearch.discovery.azure.classic.AzureUnicastHostsProvider.Deployment;
 
 import java.net.URI;
 import java.net.URISyntaxException;
diff --git a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeServiceImpl.java b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java
similarity index 97%
rename from plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeServiceImpl.java
rename to plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java
index 076ce52ff80..7d6f81ad0b6 100644
--- a/plugins/discovery-azure/src/main/java/org/elasticsearch/cloud/azure/management/AzureComputeServiceImpl.java
+++ b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.cloud.azure.management;
+package org.elasticsearch.cloud.azure.classic.management;
 
 import com.microsoft.windowsazure.Configuration;
 import com.microsoft.windowsazure.core.Builder;
@@ -28,7 +28,7 @@ import com.microsoft.windowsazure.management.compute.ComputeManagementService;
 import com.microsoft.windowsazure.management.compute.models.HostedServiceGetDetailedResponse;
 import com.microsoft.windowsazure.management.configuration.ManagementConfiguration;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.cloud.azure.AzureServiceRemoteException;
+import org.elasticsearch.cloud.azure.classic.AzureServiceRemoteException;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
diff --git a/plugins/discovery-azure/src/main/java/org/elasticsearch/discovery/azure/AzureUnicastHostsProvider.java b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/discovery/azure/classic/AzureUnicastHostsProvider.java
similarity index 96%
rename from plugins/discovery-azure/src/main/java/org/elasticsearch/discovery/azure/AzureUnicastHostsProvider.java
rename to plugins/discovery-azure-classic/src/main/java/org/elasticsearch/discovery/azure/classic/AzureUnicastHostsProvider.java
index cb6c8238bf5..ed327a3a727 100644
--- a/plugins/discovery-azure/src/main/java/org/elasticsearch/discovery/azure/AzureUnicastHostsProvider.java
+++ b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/discovery/azure/classic/AzureUnicastHostsProvider.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.discovery.azure;
+package org.elasticsearch.discovery.azure.classic;
 
 import com.microsoft.windowsazure.management.compute.models.DeploymentSlot;
 import com.microsoft.windowsazure.management.compute.models.DeploymentStatus;
@@ -25,10 +25,10 @@ import com.microsoft.windowsazure.management.compute.models.HostedServiceGetDeta
 import com.microsoft.windowsazure.management.compute.models.InstanceEndpoint;
 import com.microsoft.windowsazure.management.compute.models.RoleInstance;
 import org.elasticsearch.Version;
-import org.elasticsearch.cloud.azure.AzureServiceDisableException;
-import org.elasticsearch.cloud.azure.AzureServiceRemoteException;
-import org.elasticsearch.cloud.azure.management.AzureComputeService;
-import org.elasticsearch.cloud.azure.management.AzureComputeService.Discovery;
+import org.elasticsearch.cloud.azure.classic.AzureServiceDisableException;
+import org.elasticsearch.cloud.azure.classic.AzureServiceRemoteException;
+import org.elasticsearch.cloud.azure.classic.management.AzureComputeService;
+import org.elasticsearch.cloud.azure.classic.management.AzureComputeService.Discovery;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractComponent;
diff --git a/plugins/discovery-azure/src/main/java/org/elasticsearch/plugin/discovery/azure/AzureDiscoveryPlugin.java b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/plugin/discovery/azure/classic/AzureDiscoveryPlugin.java
similarity index 82%
rename from plugins/discovery-azure/src/main/java/org/elasticsearch/plugin/discovery/azure/AzureDiscoveryPlugin.java
rename to plugins/discovery-azure-classic/src/main/java/org/elasticsearch/plugin/discovery/azure/classic/AzureDiscoveryPlugin.java
index 9c1df5b4c8d..a7e1816fff0 100644
--- a/plugins/discovery-azure/src/main/java/org/elasticsearch/plugin/discovery/azure/AzureDiscoveryPlugin.java
+++ b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/plugin/discovery/azure/classic/AzureDiscoveryPlugin.java
@@ -17,18 +17,18 @@
  * under the License.
  */
 
-package org.elasticsearch.plugin.discovery.azure;
+package org.elasticsearch.plugin.discovery.azure.classic;
 
-import org.elasticsearch.cloud.azure.AzureDiscoveryModule;
-import org.elasticsearch.cloud.azure.management.AzureComputeService;
+import org.elasticsearch.cloud.azure.classic.AzureDiscoveryModule;
+import org.elasticsearch.cloud.azure.classic.management.AzureComputeService;
 import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.discovery.DiscoveryModule;
-import org.elasticsearch.discovery.azure.AzureUnicastHostsProvider;
+import org.elasticsearch.discovery.azure.classic.AzureUnicastHostsProvider;
 import org.elasticsearch.discovery.zen.ZenDiscovery;
 import org.elasticsearch.plugins.Plugin;
 
@@ -45,7 +45,9 @@ public class AzureDiscoveryPlugin extends Plugin {
 
     public AzureDiscoveryPlugin(Settings settings) {
         this.settings = settings;
-        logger.trace("starting azure discovery plugin...");
+        DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
+        deprecationLogger.deprecated("azure classic discovery plugin is deprecated. Use azure arm discovery plugin instead");
+        logger.trace("starting azure classic discovery plugin...");
     }
 
     @Override
diff --git a/plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureComputeServiceTestCase.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/AbstractAzureComputeServiceTestCase.java
similarity index 89%
rename from plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureComputeServiceTestCase.java
rename to plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/AbstractAzureComputeServiceTestCase.java
index 0c57ec3f16e..e9d16408537 100644
--- a/plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureComputeServiceTestCase.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/AbstractAzureComputeServiceTestCase.java
@@ -17,14 +17,14 @@
  * under the License.
  */
 
-package org.elasticsearch.cloud.azure;
+package org.elasticsearch.cloud.azure.classic;
 
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
-import org.elasticsearch.cloud.azure.management.AzureComputeService.Discovery;
-import org.elasticsearch.cloud.azure.management.AzureComputeService.Management;
+import org.elasticsearch.cloud.azure.classic.management.AzureComputeService.Discovery;
+import org.elasticsearch.cloud.azure.classic.management.AzureComputeService.Management;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.node.Node;
-import org.elasticsearch.plugin.discovery.azure.AzureDiscoveryPlugin;
+import org.elasticsearch.plugin.discovery.azure.classic.AzureDiscoveryPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 
diff --git a/plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AzureComputeServiceSimpleMock.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/AzureComputeServiceSimpleMock.java
similarity index 95%
rename from plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AzureComputeServiceSimpleMock.java
rename to plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/AzureComputeServiceSimpleMock.java
index 26843eba532..66e853b5953 100644
--- a/plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AzureComputeServiceSimpleMock.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/AzureComputeServiceSimpleMock.java
@@ -17,14 +17,14 @@
  * under the License.
  */
 
-package org.elasticsearch.cloud.azure;
+package org.elasticsearch.cloud.azure.classic;
 
 import com.microsoft.windowsazure.management.compute.models.DeploymentSlot;
 import com.microsoft.windowsazure.management.compute.models.DeploymentStatus;
 import com.microsoft.windowsazure.management.compute.models.HostedServiceGetDetailedResponse;
 import com.microsoft.windowsazure.management.compute.models.InstanceEndpoint;
 import com.microsoft.windowsazure.management.compute.models.RoleInstance;
-import org.elasticsearch.cloud.azure.management.AzureComputeServiceAbstractMock;
+import org.elasticsearch.cloud.azure.classic.management.AzureComputeServiceAbstractMock;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
diff --git a/plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AzureComputeServiceTwoNodesMock.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/AzureComputeServiceTwoNodesMock.java
similarity index 96%
rename from plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AzureComputeServiceTwoNodesMock.java
rename to plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/AzureComputeServiceTwoNodesMock.java
index bf7589c8836..d75ce22d55c 100644
--- a/plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/AzureComputeServiceTwoNodesMock.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/AzureComputeServiceTwoNodesMock.java
@@ -17,14 +17,14 @@
  * under the License.
  */
 
-package org.elasticsearch.cloud.azure;
+package org.elasticsearch.cloud.azure.classic;
 
 import com.microsoft.windowsazure.management.compute.models.DeploymentSlot;
 import com.microsoft.windowsazure.management.compute.models.DeploymentStatus;
 import com.microsoft.windowsazure.management.compute.models.HostedServiceGetDetailedResponse;
 import com.microsoft.windowsazure.management.compute.models.InstanceEndpoint;
 import com.microsoft.windowsazure.management.compute.models.RoleInstance;
-import org.elasticsearch.cloud.azure.management.AzureComputeServiceAbstractMock;
+import org.elasticsearch.cloud.azure.classic.management.AzureComputeServiceAbstractMock;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
diff --git a/plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/management/AzureComputeServiceAbstractMock.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceAbstractMock.java
similarity index 96%
rename from plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/management/AzureComputeServiceAbstractMock.java
rename to plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceAbstractMock.java
index c11060a84a9..33f40a9159a 100644
--- a/plugins/discovery-azure/src/test/java/org/elasticsearch/cloud/azure/management/AzureComputeServiceAbstractMock.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceAbstractMock.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.cloud.azure.management;
+package org.elasticsearch.cloud.azure.classic.management;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
diff --git a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryClusterFormationTests.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java
similarity index 98%
rename from plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryClusterFormationTests.java
rename to plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java
index 0d1de07ed64..505f2d8b0db 100644
--- a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryClusterFormationTests.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryClusterFormationTests.java
@@ -17,14 +17,14 @@
  * under the License.
  */
 
-package org.elasticsearch.discovery.azure;
+package org.elasticsearch.discovery.azure.classic;
 
 import com.microsoft.windowsazure.management.compute.models.DeploymentSlot;
 import com.microsoft.windowsazure.management.compute.models.DeploymentStatus;
 import com.sun.net.httpserver.Headers;
 import com.sun.net.httpserver.HttpsConfigurator;
 import com.sun.net.httpserver.HttpsServer;
-import org.elasticsearch.cloud.azure.management.AzureComputeService;
+import org.elasticsearch.cloud.azure.classic.management.AzureComputeService;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.logging.Loggers;
@@ -33,7 +33,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.node.Node;
-import org.elasticsearch.plugin.discovery.azure.AzureDiscoveryPlugin;
+import org.elasticsearch.plugin.discovery.azure.classic.AzureDiscoveryPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.transport.TransportSettings;
diff --git a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryRestIT.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryRestIT.java
similarity index 96%
rename from plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryRestIT.java
rename to plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryRestIT.java
index 131f73d1ca9..cb04842cb4a 100644
--- a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureDiscoveryRestIT.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureDiscoveryRestIT.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.discovery.azure;
+package org.elasticsearch.discovery.azure.classic;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
diff --git a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureMinimumMasterNodesTests.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureMinimumMasterNodesTests.java
similarity index 94%
rename from plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureMinimumMasterNodesTests.java
rename to plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureMinimumMasterNodesTests.java
index 46c3f8af7c2..3acec43f7da 100644
--- a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureMinimumMasterNodesTests.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureMinimumMasterNodesTests.java
@@ -17,11 +17,11 @@
  * under the License.
  */
 
-package org.elasticsearch.discovery.azure;
+package org.elasticsearch.discovery.azure.classic;
 
 import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
-import org.elasticsearch.cloud.azure.AbstractAzureComputeServiceTestCase;
-import org.elasticsearch.cloud.azure.AzureComputeServiceTwoNodesMock;
+import org.elasticsearch.cloud.azure.classic.AbstractAzureComputeServiceTestCase;
+import org.elasticsearch.cloud.azure.classic.AzureComputeServiceTwoNodesMock;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
 import org.elasticsearch.discovery.zen.ZenDiscovery;
diff --git a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureSimpleTests.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureSimpleTests.java
similarity index 89%
rename from plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureSimpleTests.java
rename to plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureSimpleTests.java
index 7a85909a13e..05bed1cefb1 100644
--- a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureSimpleTests.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureSimpleTests.java
@@ -17,12 +17,12 @@
  * under the License.
  */
 
-package org.elasticsearch.discovery.azure;
+package org.elasticsearch.discovery.azure.classic;
 
-import org.elasticsearch.cloud.azure.AbstractAzureComputeServiceTestCase;
-import org.elasticsearch.cloud.azure.AzureComputeServiceSimpleMock;
-import org.elasticsearch.cloud.azure.management.AzureComputeService.Discovery;
-import org.elasticsearch.cloud.azure.management.AzureComputeService.Management;
+import org.elasticsearch.cloud.azure.classic.AbstractAzureComputeServiceTestCase;
+import org.elasticsearch.cloud.azure.classic.AzureComputeServiceSimpleMock;
+import org.elasticsearch.cloud.azure.classic.management.AzureComputeService.Discovery;
+import org.elasticsearch.cloud.azure.classic.management.AzureComputeService.Management;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase;
 
diff --git a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureTwoStartedNodesTests.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureTwoStartedNodesTests.java
similarity index 89%
rename from plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureTwoStartedNodesTests.java
rename to plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureTwoStartedNodesTests.java
index 6431696c2eb..89c918769dc 100644
--- a/plugins/discovery-azure/src/test/java/org/elasticsearch/discovery/azure/AzureTwoStartedNodesTests.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureTwoStartedNodesTests.java
@@ -17,12 +17,12 @@
  * under the License.
  */
 
-package org.elasticsearch.discovery.azure;
+package org.elasticsearch.discovery.azure.classic;
 
-import org.elasticsearch.cloud.azure.AbstractAzureComputeServiceTestCase;
-import org.elasticsearch.cloud.azure.AzureComputeServiceTwoNodesMock;
-import org.elasticsearch.cloud.azure.management.AzureComputeService.Discovery;
-import org.elasticsearch.cloud.azure.management.AzureComputeService.Management;
+import org.elasticsearch.cloud.azure.classic.AbstractAzureComputeServiceTestCase;
+import org.elasticsearch.cloud.azure.classic.AzureComputeServiceTwoNodesMock;
+import org.elasticsearch.cloud.azure.classic.management.AzureComputeService.Discovery;
+import org.elasticsearch.cloud.azure.classic.management.AzureComputeService.Management;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase;
 
diff --git a/plugins/discovery-azure-classic/src/test/resources/rest-api-spec/test/discovery_azure_classic/10_basic.yaml b/plugins/discovery-azure-classic/src/test/resources/rest-api-spec/test/discovery_azure_classic/10_basic.yaml
new file mode 100644
index 00000000000..ea042d8a52d
--- /dev/null
+++ b/plugins/discovery-azure-classic/src/test/resources/rest-api-spec/test/discovery_azure_classic/10_basic.yaml
@@ -0,0 +1,13 @@
+# Integration tests for Azure Classic Discovery component
+#
+"Discovery Azure Classic loaded":
+    - do:
+        cluster.state: {}
+
+    # Get master node id
+    - set: { master_node: master }
+
+    - do:
+        nodes.info: {}
+
+    - match:  { nodes.$master.plugins.0.name: discovery-azure-classic  }
diff --git a/plugins/discovery-azure/src/test/resources/rest-api-spec/test/discovery_azure/10_basic.yaml b/plugins/discovery-azure/src/test/resources/rest-api-spec/test/discovery_azure/10_basic.yaml
deleted file mode 100644
index 7a5acd1f001..00000000000
--- a/plugins/discovery-azure/src/test/resources/rest-api-spec/test/discovery_azure/10_basic.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-# Integration tests for Azure Discovery component
-#
-"Discovery Azure loaded":
-    - do:
-        cluster.state: {}
-
-    # Get master node id
-    - set: { master_node: master }
-
-    - do:
-        nodes.info: {}
-
-    - match:  { nodes.$master.plugins.0.name: discovery-azure  }
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
index 336fa3ee305..c17df96937a 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
@@ -209,7 +209,7 @@ fi
     install_and_check_plugin discovery gce google-api-client-*.jar
 }
 
-@test "[$GROUP] install discovery-azure plugin" {
+@test "[$GROUP] install discovery-azure-classic plugin" {
     install_and_check_plugin discovery azure azure-core-*.jar
 }
 
@@ -341,8 +341,8 @@ fi
     remove_plugin discovery-gce
 }
 
-@test "[$GROUP] remove discovery-azure plugin" {
-    remove_plugin discovery-azure
+@test "[$GROUP] remove discovery-azure-classic plugin" {
+    remove_plugin discovery-azure-classic
 }
 
 @test "[$GROUP] remove discovery-ec2 plugin" {
diff --git a/settings.gradle b/settings.gradle
index c0d7a72b2a5..6588b605a9d 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -31,7 +31,7 @@ List projects = [
   'plugins:analysis-phonetic',
   'plugins:analysis-smartcn',
   'plugins:analysis-stempel',
-  'plugins:discovery-azure',
+  'plugins:discovery-azure-classic',
   'plugins:discovery-ec2',
   'plugins:discovery-gce',
   'plugins:ingest-geoip',

From 4c2d6cf538da466ca469d8e68f25b7e5311616db Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Thu, 30 Jun 2016 14:43:28 +0200
Subject: [PATCH 1145/1311] percolator: removed unused code

---
 .../org/elasticsearch/percolator/PercolatorPlugin.java   | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
index a15acf67c02..4359568b3f6 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorPlugin.java
@@ -21,8 +21,6 @@ package org.elasticsearch.percolator;
 
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.Mapper;
@@ -39,8 +37,6 @@ import java.util.Map;
 
 public class PercolatorPlugin extends Plugin implements MapperPlugin, ActionPlugin {
 
-    public static final String NAME = "percolator";
-
     private final Settings settings;
 
     public PercolatorPlugin(Settings settings) {
@@ -65,7 +61,7 @@ public class PercolatorPlugin extends Plugin implements MapperPlugin, ActionPlug
 
     @Override
     public List<Setting<?>> getSettings() {
-        return Arrays.asList(PercolatorFieldMapper.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING);
+        return Collections.singletonList(PercolatorFieldMapper.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING);
     }
 
     @Override
@@ -73,7 +69,4 @@ public class PercolatorPlugin extends Plugin implements MapperPlugin, ActionPlug
         return Collections.singletonMap(PercolatorFieldMapper.CONTENT_TYPE, new PercolatorFieldMapper.TypeParser());
     }
 
-    static boolean transportClientMode(Settings settings) {
-        return TransportClient.CLIENT_TYPE.equals(settings.get(Client.CLIENT_TYPE_SETTING_S.getKey()));
-    }
 }

From a029c147a389daf445ae92bd46ddc009a136bfc4 Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Thu, 30 Jun 2016 15:16:51 +0200
Subject: [PATCH 1146/1311] Update plugin description

---
 plugins/discovery-azure-classic/build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/discovery-azure-classic/build.gradle b/plugins/discovery-azure-classic/build.gradle
index 42f6fd29106..3f68c3fcba5 100644
--- a/plugins/discovery-azure-classic/build.gradle
+++ b/plugins/discovery-azure-classic/build.gradle
@@ -20,7 +20,7 @@ import org.elasticsearch.gradle.LoggedExec
  */
 
 esplugin {
-  description 'The Azure Discovery plugin allows to use Azure API for the unicast discovery mechanism'
+  description 'The Azure Classic Discovery plugin allows to use Azure Classic API for the unicast discovery mechanism'
   classname 'AzureDiscoveryPlugin'
 }
 

From b5bb27cf902745b43a79b7f46fa67c12e1a09d96 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Thu, 30 Jun 2016 15:20:59 +0200
Subject: [PATCH 1147/1311] Bumped version to 5.0.0-alpha4

---
 docs/reference/index.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc
index 3a625ac1a2c..636475dfb47 100644
--- a/docs/reference/index.asciidoc
+++ b/docs/reference/index.asciidoc
@@ -1,7 +1,7 @@
 [[elasticsearch-reference]]
 = Elasticsearch Reference
 
-:version:       5.0.0-alpha3
+:version:       5.0.0-alpha4
 :major-version: 5.x
 :branch:        master
 :jdk:           1.8.0_73

From 7c7abc349c52ac1f5011abe47995f2b9c16f8a19 Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Thu, 30 Jun 2016 15:21:30 +0200
Subject: [PATCH 1148/1311] Fix checkstyle issues

---
 .../azure/classic/AzureMinimumMasterNodesTests.java  | 12 ++++++++----
 .../discovery/azure/classic/AzureSimpleTests.java    |  6 ++++--
 .../azure/classic/AzureTwoStartedNodesTests.java     | 12 ++++++++----
 3 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureMinimumMasterNodesTests.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureMinimumMasterNodesTests.java
index 3acec43f7da..72e1f2da791 100644
--- a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureMinimumMasterNodesTests.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureMinimumMasterNodesTests.java
@@ -63,20 +63,23 @@ public class AzureMinimumMasterNodesTests extends AbstractAzureComputeServiceTes
         logger.info("--> start data node / non master node");
         internalCluster().startNode();
         try {
-            assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("100ms").execute().actionGet().getState().nodes().getMasterNodeId(), nullValue());
+            assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("100ms").get().getState().nodes().getMasterNodeId(),
+                nullValue());
             fail("should not be able to find master");
         } catch (MasterNotDiscoveredException e) {
             // all is well, no master elected
         }
         logger.info("--> start another node");
         internalCluster().startNode();
-        assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").execute().actionGet().getState().nodes().getMasterNodeId(), notNullValue());
+        assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").get().getState().nodes().getMasterNodeId(),
+            notNullValue());
 
         logger.info("--> stop master node");
         internalCluster().stopCurrentMasterNode();
 
         try {
-            assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").execute().actionGet().getState().nodes().getMasterNodeId(), nullValue());
+            assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").get().getState().nodes().getMasterNodeId(),
+                nullValue());
             fail("should not be able to find master");
         } catch (MasterNotDiscoveredException e) {
             // all is well, no master elected
@@ -84,6 +87,7 @@ public class AzureMinimumMasterNodesTests extends AbstractAzureComputeServiceTes
 
         logger.info("--> start another node");
         internalCluster().startNode();
-        assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").execute().actionGet().getState().nodes().getMasterNodeId(), notNullValue());
+        assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").get().getState().nodes().getMasterNodeId(),
+            notNullValue());
     }
 }
diff --git a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureSimpleTests.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureSimpleTests.java
index 05bed1cefb1..f507b1ea501 100644
--- a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureSimpleTests.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureSimpleTests.java
@@ -45,7 +45,8 @@ public class AzureSimpleTests extends AbstractAzureComputeServiceTestCase {
 
         logger.info("--> start one node");
         internalCluster().startNode(settings);
-        assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").execute().actionGet().getState().nodes().getMasterNodeId(), notNullValue());
+        assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").get().getState().nodes().getMasterNodeId(),
+            notNullValue());
 
         // We expect having 1 node as part of the cluster, let's test that
         checkNumberOfNodes(1);
@@ -58,7 +59,8 @@ public class AzureSimpleTests extends AbstractAzureComputeServiceTestCase {
 
         logger.info("--> start one node");
         internalCluster().startNode(settings);
-        assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").execute().actionGet().getState().nodes().getMasterNodeId(), notNullValue());
+        assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").get().getState().nodes().getMasterNodeId(),
+            notNullValue());
 
         // We expect having 1 node as part of the cluster, let's test that
         checkNumberOfNodes(1);
diff --git a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureTwoStartedNodesTests.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureTwoStartedNodesTests.java
index 89c918769dc..35844c9b383 100644
--- a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureTwoStartedNodesTests.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureTwoStartedNodesTests.java
@@ -46,11 +46,13 @@ public class AzureTwoStartedNodesTests extends AbstractAzureComputeServiceTestCa
 
         logger.info("--> start first node");
         internalCluster().startNode(settings);
-        assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").execute().actionGet().getState().nodes().getMasterNodeId(), notNullValue());
+        assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").get().getState().nodes().getMasterNodeId(),
+            notNullValue());
 
         logger.info("--> start another node");
         internalCluster().startNode(settings);
-        assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").execute().actionGet().getState().nodes().getMasterNodeId(), notNullValue());
+        assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").get().getState().nodes().getMasterNodeId(),
+            notNullValue());
 
         // We expect having 2 nodes as part of the cluster, let's test that
         checkNumberOfNodes(2);
@@ -64,11 +66,13 @@ public class AzureTwoStartedNodesTests extends AbstractAzureComputeServiceTestCa
 
         logger.info("--> start first node");
         internalCluster().startNode(settings);
-        assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").execute().actionGet().getState().nodes().getMasterNodeId(), notNullValue());
+        assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").get().getState().nodes().getMasterNodeId(),
+            notNullValue());
 
         logger.info("--> start another node");
         internalCluster().startNode(settings);
-        assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").execute().actionGet().getState().nodes().getMasterNodeId(), notNullValue());
+        assertThat(client().admin().cluster().prepareState().setMasterNodeTimeout("1s").get().getState().nodes().getMasterNodeId(),
+            notNullValue());
 
         // We expect having 2 nodes as part of the cluster, let's test that
         checkNumberOfNodes(2);

From f3ddccad17c0b6281cab273009444151912a33fc Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Thu, 30 Jun 2016 15:26:54 +0200
Subject: [PATCH 1149/1311] Fix documentation filenames

---
 ...iscovery-azure.asciidoc => discovery-azure-classic.asciidoc} | 2 +-
 docs/plugins/discovery.asciidoc                                 | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename docs/plugins/{discovery-azure.asciidoc => discovery-azure-classic.asciidoc} (99%)

diff --git a/docs/plugins/discovery-azure.asciidoc b/docs/plugins/discovery-azure-classic.asciidoc
similarity index 99%
rename from docs/plugins/discovery-azure.asciidoc
rename to docs/plugins/discovery-azure-classic.asciidoc
index 022c453e148..54dc2533807 100644
--- a/docs/plugins/discovery-azure.asciidoc
+++ b/docs/plugins/discovery-azure-classic.asciidoc
@@ -1,5 +1,5 @@
 [[discovery-azure-classic]]
-=== Azure Discovery Plugin
+=== Azure Classic Discovery Plugin
 
 The Azure Classic Discovery plugin uses the Azure Classic API for unicast discovery.
 
diff --git a/docs/plugins/discovery.asciidoc b/docs/plugins/discovery.asciidoc
index ed9cfc0ed3d..999bf9c0e1f 100644
--- a/docs/plugins/discovery.asciidoc
+++ b/docs/plugins/discovery.asciidoc
@@ -13,7 +13,7 @@ The core discovery plugins are:
 
 The EC2 discovery plugin uses the https://github.com/aws/aws-sdk-java[AWS API] for unicast discovery.
 
-<<discovery-azure-classic,Azure discovery>>::
+<<discovery-azure-classic,Azure Classic discovery>>::
 
 The Azure Classic discovery plugin uses the Azure Classic API for unicast discovery.
 

From 72c220b1dff256b7e3b66695541a575ac955ed73 Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Thu, 30 Jun 2016 15:29:29 +0200
Subject: [PATCH 1150/1311] Add deprecation notice

---
 docs/plugins/discovery-azure-classic.asciidoc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/plugins/discovery-azure-classic.asciidoc b/docs/plugins/discovery-azure-classic.asciidoc
index 54dc2533807..d794adbea7e 100644
--- a/docs/plugins/discovery-azure-classic.asciidoc
+++ b/docs/plugins/discovery-azure-classic.asciidoc
@@ -3,6 +3,10 @@
 
 The Azure Classic Discovery plugin uses the Azure Classic API for unicast discovery.
 
+// TODO: Link to ARM plugin when ready
+// See issue https://github.com/elastic/elasticsearch/issues/19146
+deprecated[5.0.0, Use coming Azure ARM Discovery plugin instead]
+
 [[discovery-azure-classic-install]]
 [float]
 ==== Installation

From d78afc26eadc89d9ac04dd2a1e12bb32e615776b Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Thu, 30 Jun 2016 15:30:49 +0200
Subject: [PATCH 1151/1311] Fix classname

Package was removed by mistake
---
 plugins/discovery-azure-classic/build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/discovery-azure-classic/build.gradle b/plugins/discovery-azure-classic/build.gradle
index 3f68c3fcba5..88874968b21 100644
--- a/plugins/discovery-azure-classic/build.gradle
+++ b/plugins/discovery-azure-classic/build.gradle
@@ -21,7 +21,7 @@ import org.elasticsearch.gradle.LoggedExec
 
 esplugin {
   description 'The Azure Classic Discovery plugin allows to use Azure Classic API for the unicast discovery mechanism'
-  classname 'AzureDiscoveryPlugin'
+  classname 'org.elasticsearch.plugin.discovery.azure.classic.AzureDiscoveryPlugin'
 }
 
 versions << [

From 648b7b82b48be2325b0336b7e556ae4628375d4d Mon Sep 17 00:00:00 2001
From: David Pilato <david@pilato.fr>
Date: Thu, 30 Jun 2016 15:32:52 +0200
Subject: [PATCH 1152/1311] Fix method name typo

---
 .../elasticsearch/discovery/azure/classic/AzureSimpleTests.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureSimpleTests.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureSimpleTests.java
index f507b1ea501..d2234632122 100644
--- a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureSimpleTests.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/AzureSimpleTests.java
@@ -38,7 +38,7 @@ public class AzureSimpleTests extends AbstractAzureComputeServiceTestCase {
         super(AzureComputeServiceSimpleMock.TestPlugin.class);
     }
 
-    public void testOneNodeDhouldRunUsingPrivateIp() {
+    public void testOneNodeShouldRunUsingPrivateIp() {
         Settings.Builder settings = Settings.builder()
                 .put(Management.SERVICE_NAME_SETTING.getKey(), "dummy")
                 .put(Discovery.HOST_TYPE_SETTING.getKey(), "private_ip");

From e359be7632e5d65c2e08934bb2e5062db47d513f Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 29 Jun 2016 18:41:31 -0400
Subject: [PATCH 1153/1311] Don't inject TransportPercolateAction into
 RestPercolateAction

Instead use the client. This will help us build the actions more
easily in the future.
---
 .../resources/checkstyle_suppressions.xml     |  1 -
 .../percolator/RestMultiPercolateAction.java  |  8 +++-----
 .../percolator/RestPercolateAction.java       | 20 +++++++++----------
 3 files changed, 12 insertions(+), 17 deletions(-)

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 5a08a834c3c..71f2cf85ab4 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -1168,7 +1168,6 @@
   <suppress files="modules[/\\]percolator[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]TransportMultiPercolateAction.java" checks="LineLength" />
   <suppress files="modules[/\\]percolator[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]TransportPercolateAction.java" checks="LineLength" />
   <suppress files="modules[/\\]percolator[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]TransportShardMultiPercolateAction.java" checks="LineLength" />
-  <suppress files="modules[/\\]percolator[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]RestPercolateAction.java" checks="LineLength" />
   <suppress files="modules[/\\]percolator[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]MultiPercolatorIT.java" checks="LineLength" />
   <suppress files="modules[/\\]percolator[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]PercolatorIT.java" checks="LineLength" />
   <suppress files="modules[/\\]percolator[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]percolator[/\\]MultiPercolatorRequestTests.java" checks="LineLength" />
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java
index a2902a9a7c2..41de2de42d1 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java
@@ -36,13 +36,10 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 public class RestMultiPercolateAction extends BaseRestHandler {
 
     private final boolean allowExplicitIndex;
-    private final TransportMultiPercolateAction action;
 
     @Inject
-    public RestMultiPercolateAction(Settings settings, RestController controller, Client client,
-                                    TransportMultiPercolateAction action) {
+    public RestMultiPercolateAction(Settings settings, RestController controller, Client client) {
         super(settings, client);
-        this.action = action;
         controller.registerHandler(POST, "/_mpercolate", this);
         controller.registerHandler(POST, "/{index}/_mpercolate", this);
         controller.registerHandler(POST, "/{index}/{type}/_mpercolate", this);
@@ -61,7 +58,8 @@ public class RestMultiPercolateAction extends BaseRestHandler {
         multiPercolateRequest.indices(Strings.splitStringByCommaToArray(restRequest.param("index")));
         multiPercolateRequest.documentType(restRequest.param("type"));
         multiPercolateRequest.add(RestActions.getRestContent(restRequest), allowExplicitIndex);
-        action.execute(multiPercolateRequest, new RestToXContentListener<MultiPercolateResponse>(restChannel));
+        client.execute(MultiPercolateAction.INSTANCE, multiPercolateRequest,
+                new RestToXContentListener<MultiPercolateResponse>(restChannel));
     }
 
 }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java
index b752cc55f6c..6dffd5518c8 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java
@@ -36,13 +36,9 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 public class RestPercolateAction extends BaseRestHandler {
-
-    private final TransportPercolateAction action;
-
     @Inject
-    public RestPercolateAction(Settings settings, RestController controller, Client client, TransportPercolateAction action) {
+    public RestPercolateAction(Settings settings, RestController controller, Client client) {
         super(settings, client);
-        this.action = action;
         controller.registerHandler(GET, "/{index}/{type}/_percolate", this);
         controller.registerHandler(POST, "/{index}/{type}/_percolate", this);
 
@@ -54,7 +50,8 @@ public class RestPercolateAction extends BaseRestHandler {
         controller.registerHandler(GET, "/{index}/{type}/_percolate/count", countHandler);
         controller.registerHandler(POST, "/{index}/{type}/_percolate/count", countHandler);
 
-        RestCountPercolateExistingDocHandler countExistingDocHandler = new RestCountPercolateExistingDocHandler(settings, controller, client);
+        RestCountPercolateExistingDocHandler countExistingDocHandler = new RestCountPercolateExistingDocHandler(settings, controller,
+                client);
         controller.registerHandler(GET, "/{index}/{type}/{id}/_percolate/count", countExistingDocHandler);
         controller.registerHandler(POST, "/{index}/{type}/{id}/_percolate/count", countExistingDocHandler);
     }
@@ -67,10 +64,11 @@ public class RestPercolateAction extends BaseRestHandler {
         percolateRequest.source(RestActions.getRestContent(restRequest));
 
         percolateRequest.indicesOptions(IndicesOptions.fromRequest(restRequest, percolateRequest.indicesOptions()));
-        executePercolate(percolateRequest, restChannel);
+        executePercolate(client, percolateRequest, restChannel);
     }
 
-    void parseExistingDocPercolate(PercolateRequest percolateRequest, RestRequest restRequest, RestChannel restChannel, final Client client) {
+    void parseExistingDocPercolate(PercolateRequest percolateRequest, RestRequest restRequest, RestChannel restChannel,
+            final Client client) {
         String index = restRequest.param("index");
         String type = restRequest.param("type");
         percolateRequest.indices(Strings.splitStringByCommaToArray(restRequest.param("percolate_index", index)));
@@ -91,11 +89,11 @@ public class RestPercolateAction extends BaseRestHandler {
         percolateRequest.source(RestActions.getRestContent(restRequest));
 
         percolateRequest.indicesOptions(IndicesOptions.fromRequest(restRequest, percolateRequest.indicesOptions()));
-        executePercolate(percolateRequest, restChannel);
+        executePercolate(client, percolateRequest, restChannel);
     }
 
-    void executePercolate(final PercolateRequest percolateRequest, final RestChannel restChannel) {
-        action.execute(percolateRequest, new RestToXContentListener<>(restChannel));
+    void executePercolate(final Client client, final PercolateRequest percolateRequest, final RestChannel restChannel) {
+        client.execute(PercolateAction.INSTANCE, percolateRequest, new RestToXContentListener<>(restChannel));
     }
 
     @Override

From d57b780bb465dad3f692e44834df484c697d51ca Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 29 Jun 2016 18:46:12 -0400
Subject: [PATCH 1154/1311] Remote TransportRethrottleAction from
 RestRethrottleAction

Just use the client to call it.
---
 .../elasticsearch/index/reindex/RestRethrottleAction.java  | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java
index 9841794ca2a..cb53509a461 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestRethrottleAction.java
@@ -36,14 +36,11 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 import static org.elasticsearch.rest.action.admin.cluster.node.tasks.RestListTasksAction.nodeSettingListener;
 
 public class RestRethrottleAction extends BaseRestHandler {
-    private final TransportRethrottleAction action;
     private final ClusterService clusterService;
 
     @Inject
-    public RestRethrottleAction(Settings settings, RestController controller, Client client, TransportRethrottleAction action,
-            ClusterService clusterService) {
+    public RestRethrottleAction(Settings settings, RestController controller, Client client, ClusterService clusterService) {
         super(settings, client);
-        this.action = action;
         this.clusterService = clusterService;
         controller.registerHandler(POST, "/_update_by_query/{taskId}/_rethrottle", this);
         controller.registerHandler(POST, "/_delete_by_query/{taskId}/_rethrottle", this);
@@ -60,6 +57,6 @@ public class RestRethrottleAction extends BaseRestHandler {
         }
         internalRequest.setRequestsPerSecond(requestsPerSecond);
         ActionListener<ListTasksResponse> listener = nodeSettingListener(clusterService, new RestToXContentListener<>(channel));
-        action.execute(internalRequest, listener);
+        client.execute(RethrottleAction.INSTANCE, internalRequest, listener);
     }
 }

From dc53ce929d8c54e50376d04f421bf9012d5ff137 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Thu, 30 Jun 2016 12:41:31 +0200
Subject: [PATCH 1155/1311] Document Update/Delete-By-Query with version number
 zero

Update-By-Query and Delete-By-Query use internal versioning to update/delete documents. But documents can have a version number equal to zero using the external versioning... making the UBQ/DBQ request fail because zero is not a valid version number and they only support internal versioning for now. Sequence numbers might help to solve this issue in the future.
---
 docs/reference/docs/delete-by-query.asciidoc  |  4 +++
 docs/reference/docs/index_.asciidoc           |  8 +++++
 docs/reference/docs/update-by-query.asciidoc  |  4 +++
 .../test/delete_by_query/40_versioning.yaml   | 29 +++++++++++++++++++
 .../test/update_by_query/40_versioning.yaml   | 27 +++++++++++++++++
 5 files changed, 72 insertions(+)
 create mode 100644 modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/40_versioning.yaml

diff --git a/docs/reference/docs/delete-by-query.asciidoc b/docs/reference/docs/delete-by-query.asciidoc
index 8b37e0a1220..b7f31ce4789 100644
--- a/docs/reference/docs/delete-by-query.asciidoc
+++ b/docs/reference/docs/delete-by-query.asciidoc
@@ -54,6 +54,10 @@ conflict if the document changes between the time when the snapshot was taken
 and when the delete request is processed. When the versions match the document
 is deleted.
 
+NOTE: Since `internal` versioning does not support the value 0 as a valid
+version number, documents with version equal to zero cannot be deleted using
+`_delete_by_query` and will fail the request.
+
 During the `_delete_by_query` execution, multiple search requests are sequentially
 executed in order to find all the matching documents to delete. Every time a batch
 of documents is found, a corresponding bulk request is executed to delete all
diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc
index e1c260ae48f..dda75dd5aa5 100644
--- a/docs/reference/docs/index_.asciidoc
+++ b/docs/reference/docs/index_.asciidoc
@@ -119,6 +119,14 @@ indexed and the new version number used. If the value provided is less
 than or equal to the stored document's version number, a version
 conflict will occur and the index operation will fail.
 
+WARNING: External versioning supports the value 0 as a valid version number.
+This allows the version to be in sync with an external versioning system
+where version numbers start from zero instead of one. It has the side effect
+that documents with version number equal to zero cannot neither be updated
+using the <<docs-update-by-query,Update-By-Query API>> nor be deleted
+using the <<docs-delete-by-query,Delete By Query API>> as long as their
+version number is equal to zero.
+
 A nice side effect is that there is no need to maintain strict ordering
 of async indexing operations executed as a result of changes to a source
 database, as long as version numbers from the source database are used.
diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc
index 56ad1c7cd9a..06c20bcf07e 100644
--- a/docs/reference/docs/update-by-query.asciidoc
+++ b/docs/reference/docs/update-by-query.asciidoc
@@ -46,6 +46,10 @@ conflict if the document changes between the time when the snapshot was taken
 and when the index request is processed. When the versions match the document
 is updated and the version number is incremented.
 
+NOTE: Since `internal` versioning does not support the value 0 as a valid
+version number, documents with version equal to zero cannot be updated using
+`_update_by_query` and will fail the request.
+
 All update and query failures cause the `_update_by_query` to abort and are
 returned in the `failures` of the response. The updates that have been
 performed still stick. In other words, the process is not rolled back, only
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/40_versioning.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/40_versioning.yaml
new file mode 100644
index 00000000000..c81305e2824
--- /dev/null
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/40_versioning.yaml
@@ -0,0 +1,29 @@
+---
+"delete_by_query fails to delete documents with version number equal to zero":
+  - do:
+      index:
+        index:        index1
+        type:         type1
+        id:           1
+        version:      0 # Starting version is zero
+        version_type: external
+        body:    {"delete": 0}
+  - do:
+      indices.refresh: {}
+
+  # Delete by query uses internal versioning and will fail here
+  # because zero is not allowed as a valid version number
+  - do:
+      catch: /illegal version value \[0\] for version type \[INTERNAL\]./
+      delete_by_query:
+        index: index1
+        refresh: true
+        body:
+          query:
+            match_all: {}
+  - do:
+      get:
+        index: index1
+        type:  type1
+        id:    1
+  - match: {_version: 0}
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/40_versioning.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/40_versioning.yaml
index ac1cbe4417e..1718714defd 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/40_versioning.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/40_versioning.yaml
@@ -21,3 +21,30 @@
         type:  test
         id:    1
   - match: {_version: 2}
+
+---
+"update_by_query fails to update documents with version number equal to zero":
+  - do:
+      index:
+        index:        index1
+        type:         type1
+        id:           1
+        version:      0 # Starting version is zero
+        version_type: external
+        body:    {"update": 0}
+  - do:
+      indices.refresh: {}
+
+  # Update by query uses internal versioning and will fail here
+  # because zero is not allowed as a valid version number
+  - do:
+      catch: /illegal version value \[0\] for version type \[INTERNAL\]./
+      update_by_query:
+        index: index1
+        refresh: true
+  - do:
+      get:
+        index: index1
+        type:  type1
+        id:    1
+  - match: {_version: 0}

From dbf1f61c5b9c568960e59a8f3087a793ae8f8978 Mon Sep 17 00:00:00 2001
From: jalvar08 <jeovanny.alvarez@gmail.com>
Date: Thu, 30 Jun 2016 10:42:01 -0400
Subject: [PATCH 1156/1311] Fixing typo for path.conf location (#19098)

Changing -Ees.path.conf to -Epath.conf
---
 docs/reference/setup/configuration.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/setup/configuration.asciidoc b/docs/reference/setup/configuration.asciidoc
index 52bf5ffcbff..68f73fc96b8 100644
--- a/docs/reference/setup/configuration.asciidoc
+++ b/docs/reference/setup/configuration.asciidoc
@@ -26,7 +26,7 @@ setting, as follows:
 
 [source,sh]
 -------------------------------
-./bin/elasticsearch -Ees.path.conf=/path/to/my/config/
+./bin/elasticsearch -Epath.conf=/path/to/my/config/
 -------------------------------
 
 [float]

From afb5e6332bd9add0b05dd73d3784cfda183c7bcc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20B=C3=BCscher?= <christoph@elastic.co>
Date: Thu, 30 Jun 2016 17:05:54 +0200
Subject: [PATCH 1157/1311] Make sure TimeIntervalRounding is monotonic for
 increasing dates (#19020)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Currently there are cases when using TimeIntervalRounding#round() and date1 <
date2 that round(date2) < round(date1). These errors can happen when using a
non-fixed time zone and the values to be rounded are slightly after a time zone
offset change (e.g. DST transition).

Here is an example for the "CET" time zone with a 45 minute rounding interval.
The dates to be rounded are on the left (with utc time stamp), the rounded
values on the right. The error case is marked:

2011-10-30T01:40:00.000+02:00 1319931600000 | 2011-10-30T01:30:00.000+02:00 1319931000000
2011-10-30T02:02:30.000+02:00 1319932950000 | 2011-10-30T01:30:00.000+02:00 1319931000000
2011-10-30T02:25:00.000+02:00 1319934300000 | 2011-10-30T02:15:00.000+02:00 1319933700000
2011-10-30T02:47:30.000+02:00 1319935650000 | 2011-10-30T02:15:00.000+02:00 1319933700000
2011-10-30T02:10:00.000+01:00 1319937000000 | 2011-10-30T01:30:00.000+02:00 1319931000000 *
2011-10-30T02:32:30.000+01:00 1319938350000 | 2011-10-30T02:15:00.000+01:00 1319937300000
2011-10-30T02:55:00.000+01:00 1319939700000 | 2011-10-30T02:15:00.000+01:00 1319937300000
2011-10-30T03:17:30.000+01:00 1319941050000 | 2011-10-30T03:00:00.000+01:00 1319940000000

We should correct this by detecting that we are crossing a transition when
rounding, and in that case pick the largest valid rounded value before the
transition.

This change adds this correction logic to the rounding function and adds this
invariant to the randomized TimeIntervalRounding tests. Also adding the example
test case from above (with corrected behaviour) for illustrative purposes.
---
 .../common/rounding/TimeZoneRounding.java     |  5 ++
 .../rounding/TimeZoneRoundingTests.java       | 79 +++++++++++++++----
 2 files changed, 67 insertions(+), 17 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java b/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java
index 593b0484800..46362687e01 100644
--- a/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java
+++ b/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java
@@ -222,6 +222,11 @@ public abstract class TimeZoneRounding extends Rounding {
             long roundedUTC;
             if (isInDSTGap(rounded) == false) {
                 roundedUTC  = timeZone.convertLocalToUTC(rounded, true, utcMillis);
+                // check if we crossed DST transition, in this case we want the last rounded value before the transition
+                long transition = timeZone.previousTransition(utcMillis);
+                if (transition != utcMillis && transition > roundedUTC) {
+                    roundedUTC = roundKey(transition - 1);
+                }
             } else {
                 /*
                  * Edge case where the rounded local time is illegal and landed
diff --git a/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java b/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
index e82d37a5cf5..f9e5f6e3fbb 100644
--- a/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
+++ b/core/src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.common.rounding;
 
+import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.rounding.TimeZoneRounding.TimeIntervalRounding;
 import org.elasticsearch.common.rounding.TimeZoneRounding.TimeUnitRounding;
 import org.elasticsearch.common.unit.TimeValue;
@@ -31,10 +32,13 @@ import org.joda.time.DateTimeConstants;
 import org.joda.time.DateTimeZone;
 import org.joda.time.format.ISODateTimeFormat;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.TimeUnit;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
@@ -328,29 +332,70 @@ public class TimeZoneRoundingTests extends ESTestCase {
             long interval = unit.toMillis(randomIntBetween(1, 365));
             DateTimeZone tz = randomDateTimeZone();
             TimeZoneRounding rounding = new TimeZoneRounding.TimeIntervalRounding(interval, tz);
-            long date = Math.abs(randomLong() % (2 * (long) 10e11)); // 1970-01-01T00:00:00Z - 2033-05-18T05:33:20.000+02:00
-            try {
-                final long roundedDate = rounding.round(date);
-                final long nextRoundingValue = rounding.nextRoundingValue(roundedDate);
-                assertThat("Rounding should be idempotent", roundedDate, equalTo(rounding.round(roundedDate)));
-                assertThat("Rounded value smaller or equal than unrounded", roundedDate, lessThanOrEqualTo(date));
-                assertThat("Values smaller than rounded value should round further down", rounding.round(roundedDate - 1),
-                        lessThan(roundedDate));
+            long mainDate = Math.abs(randomLong() % (2 * (long) 10e11)); // 1970-01-01T00:00:00Z - 2033-05-18T05:33:20.000+02:00
+            if (randomBoolean()) {
+                mainDate = nastyDate(mainDate, tz, interval);
+            }
+            // check two intervals around date
+            long previousRoundedValue = Long.MIN_VALUE;
+            for (long date = mainDate - 2 * interval; date < mainDate + 2 * interval; date += interval / 2) {
+                try {
+                    final long roundedDate = rounding.round(date);
+                    final long nextRoundingValue = rounding.nextRoundingValue(roundedDate);
+                    assertThat("Rounding should be idempotent", roundedDate, equalTo(rounding.round(roundedDate)));
+                    assertThat("Rounded value smaller or equal than unrounded", roundedDate, lessThanOrEqualTo(date));
+                    assertThat("Values smaller than rounded value should round further down", rounding.round(roundedDate - 1),
+                            lessThan(roundedDate));
+                    assertThat("Rounding should be >= previous rounding value", roundedDate, greaterThanOrEqualTo(previousRoundedValue));
 
-                if (tz.isFixed()) {
-                    assertThat("NextRounding value should be greater than date", nextRoundingValue, greaterThan(roundedDate));
-                    assertThat("NextRounding value should be interval from rounded value", nextRoundingValue - roundedDate,
-                            equalTo(interval));
-                    assertThat("NextRounding value should be a rounded date", nextRoundingValue,
-                            equalTo(rounding.round(nextRoundingValue)));
+                    if (tz.isFixed()) {
+                        assertThat("NextRounding value should be greater than date", nextRoundingValue, greaterThan(roundedDate));
+                        assertThat("NextRounding value should be interval from rounded value", nextRoundingValue - roundedDate,
+                                equalTo(interval));
+                        assertThat("NextRounding value should be a rounded date", nextRoundingValue,
+                                equalTo(rounding.round(nextRoundingValue)));
+                    }
+                    previousRoundedValue = roundedDate;
+                } catch (AssertionError e) {
+                    logger.error("Rounding error at {}, timezone {}, interval: {},", new DateTime(date, tz), tz, interval);
+                    throw e;
                 }
-            } catch (AssertionError e) {
-                logger.error("Rounding error at {}, timezone {}, interval: {},", new DateTime(date, tz), tz, interval);
-                throw e;
             }
         }
     }
 
+    /**
+     * Test that rounded values are always greater or equal to last rounded value if date is increasing.
+     * The example covers an interval around 2011-10-30T02:10:00+01:00, time zone CET, interval: 2700000ms
+     */
+    public void testIntervalRoundingMonotonic_CET() {
+        long interval = TimeUnit.MINUTES.toMillis(45);
+        DateTimeZone tz = DateTimeZone.forID("CET");
+        TimeZoneRounding rounding = new TimeZoneRounding.TimeIntervalRounding(interval, tz);
+        List<Tuple<String, String>> expectedDates = new ArrayList<Tuple<String, String>>();
+        // first date is the date to be rounded, second the expected result
+        expectedDates.add(new Tuple<>("2011-10-30T01:40:00.000+02:00", "2011-10-30T01:30:00.000+02:00"));
+        expectedDates.add(new Tuple<>("2011-10-30T02:02:30.000+02:00", "2011-10-30T01:30:00.000+02:00"));
+        expectedDates.add(new Tuple<>("2011-10-30T02:25:00.000+02:00", "2011-10-30T02:15:00.000+02:00"));
+        expectedDates.add(new Tuple<>("2011-10-30T02:47:30.000+02:00", "2011-10-30T02:15:00.000+02:00"));
+        expectedDates.add(new Tuple<>("2011-10-30T02:10:00.000+01:00", "2011-10-30T02:15:00.000+02:00"));
+        expectedDates.add(new Tuple<>("2011-10-30T02:32:30.000+01:00", "2011-10-30T02:15:00.000+01:00"));
+        expectedDates.add(new Tuple<>("2011-10-30T02:55:00.000+01:00", "2011-10-30T02:15:00.000+01:00"));
+        expectedDates.add(new Tuple<>("2011-10-30T03:17:30.000+01:00", "2011-10-30T03:00:00.000+01:00"));
+
+        long previousDate = Long.MIN_VALUE;
+        for (Tuple<String, String> dates : expectedDates) {
+                final long roundedDate = rounding.round(time(dates.v1()));
+                assertThat(roundedDate, isDate(time(dates.v2()), tz));
+                assertThat(roundedDate, greaterThanOrEqualTo(previousDate));
+                previousDate = roundedDate;
+        }
+        // here's what this means for interval widths
+        assertEquals(TimeUnit.MINUTES.toMillis(45), time("2011-10-30T02:15:00.000+02:00") - time("2011-10-30T01:30:00.000+02:00"));
+        assertEquals(TimeUnit.MINUTES.toMillis(60), time("2011-10-30T02:15:00.000+01:00") - time("2011-10-30T02:15:00.000+02:00"));
+        assertEquals(TimeUnit.MINUTES.toMillis(45), time("2011-10-30T03:00:00.000+01:00") - time("2011-10-30T02:15:00.000+01:00"));
+    }
+
     /**
      * special test for DST switch from #9491
      */

From 983a64c833cb727c3e0b67ebda7184ddc00c3502 Mon Sep 17 00:00:00 2001
From: jaymode <jay.modi@elasticsearch.com>
Date: Thu, 30 Jun 2016 11:33:29 -0400
Subject: [PATCH 1158/1311] Add support for `teardown` section in REST tests

This commits adds support for a `teardown` section that can be defined in REST tests to
clean up any items that may have been created by the test and are not cleaned up by
deletion of indices and templates.
---
 .../rest-api-spec/test/README.asciidoc        | 15 ++-
 .../test/rest/ESRestTestCase.java             | 15 ++-
 .../test/rest/RestTestCandidate.java          |  5 +
 .../parser/RestTestSuiteParseContext.java     | 16 ++++
 .../test/rest/parser/RestTestSuiteParser.java |  2 +
 .../rest/parser/TeardownSectionParser.java    | 53 +++++++++++
 .../test/rest/section/RestTestSuite.java      |  9 ++
 .../test/rest/section/TeardownSection.java    | 56 +++++++++++
 .../test/rest/test/RestTestParserTests.java   | 50 ++++++++--
 .../rest/test/TeardownSectionParserTests.java | 93 +++++++++++++++++++
 10 files changed, 301 insertions(+), 13 deletions(-)
 create mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/parser/TeardownSectionParser.java
 create mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/section/TeardownSection.java
 create mode 100644 test/framework/src/test/java/org/elasticsearch/test/rest/test/TeardownSectionParserTests.java

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc b/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc
index 688d8cbdc5b..4e88cef4c9f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc
@@ -20,6 +20,7 @@ Test file structure
 
 A YAML test file consists of:
 * an optional `setup` section, followed by
+* an optional `teardown` section, followed by
 * one or more test sections
 
 For instance:
@@ -28,6 +29,10 @@ For instance:
       - do: ....
       - do: ....
 
+    ---
+    teardown:
+      - do: ....
+
     ---
     "First test":
       - do: ...
@@ -42,6 +47,11 @@ For instance:
 A `setup` section contains a list of commands to run before each test
 section in order to setup the same environment for each test section.
 
+A `teardown` section contains a list of commands to run after each test
+section in order to setup the same environment for each test section. This
+may be needed for modifications made by the testthat are not cleared by the
+deletion of indices and templates.
+
 A test section represents an independent test, containing multiple `do`
 statements and assertions. The contents of a test section must be run in
 order, but individual test sections may be run in any order, as follows:
@@ -49,9 +59,8 @@ order, but individual test sections may be run in any order, as follows:
 1. run `setup` (if any)
 2. reset the `response` var and the `stash` (see below)
 2. run test contents
-3. run teardown
-
-The `teardown` should delete all indices and all templates.
+3. run `teardown` (if any)
+4. delete all indices and all templates
 
 Dot notation:
 -------------
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index 12e46087b6c..e8895aa90db 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -359,6 +359,9 @@ public abstract class ESRestTestCase extends ESTestCase {
         //skip test if the whole suite (yaml file) is disabled
         assumeFalse(buildSkipMessage(testCandidate.getSuitePath(), testCandidate.getSetupSection().getSkipSection()),
                 testCandidate.getSetupSection().getSkipSection().skip(restTestExecutionContext.esVersion()));
+        //skip test if the whole suite (yaml file) is disabled
+        assumeFalse(buildSkipMessage(testCandidate.getSuitePath(), testCandidate.getTeardownSection().getSkipSection()),
+            testCandidate.getTeardownSection().getSkipSection().skip(restTestExecutionContext.esVersion()));
         //skip test if test section is disabled
         assumeFalse(buildSkipMessage(testCandidate.getTestPath(), testCandidate.getTestSection().getSkipSection()),
                 testCandidate.getTestSection().getSkipSection().skip(restTestExecutionContext.esVersion()));
@@ -391,8 +394,16 @@ public abstract class ESRestTestCase extends ESTestCase {
 
         restTestExecutionContext.clear();
 
-        for (ExecutableSection executableSection : testCandidate.getTestSection().getExecutableSections()) {
-            executableSection.execute(restTestExecutionContext);
+        try {
+            for (ExecutableSection executableSection : testCandidate.getTestSection().getExecutableSections()) {
+                executableSection.execute(restTestExecutionContext);
+            }
+        } finally {
+            logger.debug("start teardown test [{}]", testCandidate.getTestPath());
+            for (DoSection doSection : testCandidate.getTeardownSection().getDoSections()) {
+                doSection.execute(restTestExecutionContext);
+            }
+            logger.debug("end teardown test [{}]", testCandidate.getTestPath());
         }
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestCandidate.java b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestCandidate.java
index e454c396a3d..57c7e1b1305 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestCandidate.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestCandidate.java
@@ -20,6 +20,7 @@ package org.elasticsearch.test.rest;
 
 import org.elasticsearch.test.rest.section.RestTestSuite;
 import org.elasticsearch.test.rest.section.SetupSection;
+import org.elasticsearch.test.rest.section.TeardownSection;
 import org.elasticsearch.test.rest.section.TestSection;
 
 /**
@@ -56,6 +57,10 @@ public class RestTestCandidate {
         return restTestSuite.getSetupSection();
     }
 
+    public TeardownSection getTeardownSection() {
+        return restTestSuite.getTeardownSection();
+    }
+
     public TestSection getTestSection() {
         return testSection;
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java
index 0a0c2722020..e972aea641a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java
@@ -25,11 +25,13 @@ import java.util.Map;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.test.rest.section.DoSection;
 import org.elasticsearch.test.rest.section.ExecutableSection;
 import org.elasticsearch.test.rest.section.ResponseBodyAssertion;
 import org.elasticsearch.test.rest.section.SetupSection;
 import org.elasticsearch.test.rest.section.SkipSection;
+import org.elasticsearch.test.rest.section.TeardownSection;
 import org.elasticsearch.test.rest.section.TestSection;
 
 /**
@@ -39,6 +41,7 @@ import org.elasticsearch.test.rest.section.TestSection;
 public class RestTestSuiteParseContext {
 
     private static final SetupSectionParser SETUP_SECTION_PARSER = new SetupSectionParser();
+    private static final TeardownSectionParser TEARDOWN_SECTION_PARSER = new TeardownSectionParser();
     private static final RestTestSectionParser TEST_SECTION_PARSER = new RestTestSectionParser();
     private static final SkipSectionParser SKIP_SECTION_PARSER = new SkipSectionParser();
     private static final DoSectionParser DO_SECTION_PARSER = new DoSectionParser();
@@ -93,6 +96,19 @@ public class RestTestSuiteParseContext {
         return SetupSection.EMPTY;
     }
 
+    public TeardownSection parseTeardownSection() throws IOException, RestTestParseException {
+        advanceToFieldName();
+
+        if ("teardown".equals(parser.currentName())) {
+            parser.nextToken();
+            TeardownSection teardownSection = TEARDOWN_SECTION_PARSER.parse(this);
+            parser.nextToken();
+            return teardownSection;
+        }
+
+        return TeardownSection.EMPTY;
+    }
+
     public TestSection parseTestSection() throws IOException, RestTestParseException {
         return TEST_SECTION_PARSER.parse(this);
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java
index d3f93939c2e..c6986d3eac8 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java
@@ -21,6 +21,7 @@ package org.elasticsearch.test.rest.parser;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.yaml.YamlXContent;
 import org.elasticsearch.test.rest.section.RestTestSuite;
+import org.elasticsearch.test.rest.section.TeardownSection;
 import org.elasticsearch.test.rest.section.TestSection;
 
 import java.io.IOException;
@@ -75,6 +76,7 @@ public class RestTestSuiteParser implements RestTestFragmentParser<RestTestSuite
         RestTestSuite restTestSuite = new RestTestSuite(parseContext.getApi(), parseContext.getSuiteName());
 
         restTestSuite.setSetupSection(parseContext.parseSetupSection());
+        restTestSuite.setTeardownSection(parseContext.parseTeardownSection());
 
         while(true) {
             //the "---" section separator is not understood by the yaml parser. null is returned, same as when the parser is closed
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/TeardownSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/TeardownSectionParser.java
new file mode 100644
index 00000000000..428a21e2e06
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/TeardownSectionParser.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.rest.parser;
+
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.test.rest.section.TeardownSection;
+
+import java.io.IOException;
+
+/**
+ * Parser for teardown section
+ */
+public class TeardownSectionParser implements RestTestFragmentParser<TeardownSection> {
+
+    @Override
+    public TeardownSection parse(RestTestSuiteParseContext parseContext) throws IOException, RestTestParseException {
+        XContentParser parser = parseContext.parser();
+
+        TeardownSection teardownSection = new TeardownSection();
+        teardownSection.setSkipSection(parseContext.parseSkipSection());
+
+        while (parser.currentToken() != XContentParser.Token.END_ARRAY) {
+            parseContext.advanceToFieldName();
+            if (!"do".equals(parser.currentName())) {
+                throw new RestTestParseException("section [" + parser.currentName() + "] not supported within teardown section");
+            }
+
+            parser.nextToken();
+            teardownSection.addDoSection(parseContext.parseDoSection());
+            parser.nextToken();
+        }
+
+        parser.nextToken();
+        return teardownSection;
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/RestTestSuite.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/RestTestSuite.java
index d53671bc6bc..5c093be3fa0 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/RestTestSuite.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/RestTestSuite.java
@@ -33,6 +33,7 @@ public class RestTestSuite {
     private final String name;
 
     private SetupSection setupSection;
+    private TeardownSection teardownSection;
 
     private Set<TestSection> testSections = new TreeSet<>();
 
@@ -61,6 +62,14 @@ public class RestTestSuite {
         this.setupSection = setupSection;
     }
 
+    public TeardownSection getTeardownSection() {
+        return teardownSection;
+    }
+
+    public void setTeardownSection(TeardownSection teardownSection) {
+        this.teardownSection = teardownSection;
+    }
+
     /**
      * Adds a {@link org.elasticsearch.test.rest.section.TestSection} to the REST suite
      * @return true if the test section was not already present, false otherwise
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/TeardownSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/TeardownSection.java
new file mode 100644
index 00000000000..b3709472be5
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/TeardownSection.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.rest.section;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class TeardownSection {
+
+    public static final TeardownSection EMPTY;
+
+    static {
+        EMPTY = new TeardownSection();
+        EMPTY.setSkipSection(SkipSection.EMPTY);
+    }
+
+    private SkipSection skipSection;
+    private List<DoSection> doSections = new ArrayList<>();
+
+    public SkipSection getSkipSection() {
+        return skipSection;
+    }
+
+    public void setSkipSection(SkipSection skipSection) {
+        this.skipSection = skipSection;
+    }
+
+    public List<DoSection> getDoSections() {
+        return doSections;
+    }
+
+    public void addDoSection(DoSection doSection) {
+        this.doSections.add(doSection);
+    }
+
+    public boolean isEmpty() {
+        return EMPTY.equals(this);
+    }
+}
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/test/RestTestParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/test/RestTestParserTests.java
index 298f230d64a..76fbe8307e6 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/test/RestTestParserTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/test/RestTestParserTests.java
@@ -54,13 +54,30 @@ public class RestTestParserTests extends ESTestCase {
         parser.close();
     }
 
-    public void testParseTestSetupAndSections() throws Exception {
-        parser = YamlXContent.yamlXContent.createParser(
+    public void testParseTestSetupTeardownAndSections() throws Exception {
+        final boolean includeSetup = randomBoolean();
+        final boolean includeTeardown = randomBoolean();
+        StringBuilder testSpecBuilder = new StringBuilder();
+        if (includeSetup) {
+            testSpecBuilder
+                .append("---\n" +
                         "setup:\n" +
                         "  - do:\n" +
                         "        indices.create:\n" +
                         "          index: test_index\n" +
-                        "\n" +
+                        "\n");
+        }
+        if (includeTeardown) {
+            testSpecBuilder
+                .append("---\n" +
+                        "teardown:\n" +
+                        "  - do:\n" +
+                        "      indices.delete:\n" +
+                        "        index: test_index\n" +
+                        "\n");
+        }
+        parser = YamlXContent.yamlXContent.createParser(
+                        testSpecBuilder.toString() +
                         "---\n" +
                         "\"Get index mapping\":\n" +
                         "  - do:\n" +
@@ -92,12 +109,29 @@ public class RestTestParserTests extends ESTestCase {
         assertThat(restTestSuite, notNullValue());
         assertThat(restTestSuite.getName(), equalTo("suite"));
         assertThat(restTestSuite.getSetupSection(), notNullValue());
-        assertThat(restTestSuite.getSetupSection().getSkipSection().isEmpty(), equalTo(true));
+        if (includeSetup) {
+            assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(false));
+            assertThat(restTestSuite.getSetupSection().getSkipSection().isEmpty(), equalTo(true));
+            assertThat(restTestSuite.getSetupSection().getDoSections().size(), equalTo(1));
+            assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getApi(), equalTo("indices.create"));
+            assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getParams().size(), equalTo(1));
+            assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getParams().get("index"), equalTo("test_index"));
+        } else {
+            assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(true));
+        }
 
-        assertThat(restTestSuite.getSetupSection().getDoSections().size(), equalTo(1));
-        assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getApi(), equalTo("indices.create"));
-        assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getParams().size(), equalTo(1));
-        assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getParams().get("index"), equalTo("test_index"));
+        assertThat(restTestSuite.getTeardownSection(), notNullValue());
+        if (includeTeardown) {
+            assertThat(restTestSuite.getTeardownSection().isEmpty(), equalTo(false));
+            assertThat(restTestSuite.getTeardownSection().getSkipSection().isEmpty(), equalTo(true));
+            assertThat(restTestSuite.getTeardownSection().getDoSections().size(), equalTo(1));
+            assertThat(restTestSuite.getTeardownSection().getDoSections().get(0).getApiCallSection().getApi(), equalTo("indices.delete"));
+            assertThat(restTestSuite.getTeardownSection().getDoSections().get(0).getApiCallSection().getParams().size(), equalTo(1));
+            assertThat(restTestSuite.getTeardownSection().getDoSections().get(0).getApiCallSection().getParams().get("index"),
+                equalTo("test_index"));
+        } else {
+            assertThat(restTestSuite.getTeardownSection().isEmpty(), equalTo(true));
+        }
 
         assertThat(restTestSuite.getTestSections().size(), equalTo(2));
 
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/test/TeardownSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/test/TeardownSectionParserTests.java
new file mode 100644
index 00000000000..eeccea5f5e5
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/test/TeardownSectionParserTests.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.rest.test;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+import org.elasticsearch.test.rest.parser.RestTestSuiteParseContext;
+import org.elasticsearch.test.rest.parser.TeardownSectionParser;
+import org.elasticsearch.test.rest.section.TeardownSection;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+
+/**
+ * Unit tests for the teardown section parser
+ */
+public class TeardownSectionParserTests extends AbstractParserTestCase {
+
+    public void testParseTeardownSection() throws Exception {
+        parser = YamlXContent.yamlXContent.createParser(
+                "  - do:\n" +
+                "      delete:\n" +
+                "        index: foo\n" +
+                "        type: doc\n" +
+                "        id: 1\n" +
+                "        ignore: 404\n" +
+                "  - do:\n" +
+                "      delete2:\n" +
+                "        index: foo\n" +
+                "        type: doc\n" +
+                "        id: 1\n" +
+                "        ignore: 404"
+        );
+
+        TeardownSectionParser teardownSectionParser = new TeardownSectionParser();
+        TeardownSection section = teardownSectionParser.parse(new RestTestSuiteParseContext("api", "suite", parser));
+
+        assertThat(section, notNullValue());
+        assertThat(section.getSkipSection().isEmpty(), equalTo(true));
+        assertThat(section.getDoSections().size(), equalTo(2));
+        assertThat(section.getDoSections().get(0).getApiCallSection().getApi(), equalTo("delete"));
+        assertThat(section.getDoSections().get(1).getApiCallSection().getApi(), equalTo("delete2"));
+    }
+
+    public void testParseWithSkip() throws Exception {
+        parser = YamlXContent.yamlXContent.createParser(
+            "  - skip:\n" +
+                "      version:  \"2.0.0 - 2.3.0\"\n" +
+                "      reason:   \"there is a reason\"\n" +
+                "  - do:\n" +
+                "      delete:\n" +
+                "        index: foo\n" +
+                "        type: doc\n" +
+                "        id: 1\n" +
+                "        ignore: 404\n" +
+                "  - do:\n" +
+                "      delete2:\n" +
+                "        index: foo\n" +
+                "        type: doc\n" +
+                "        id: 1\n" +
+                "        ignore: 404"
+        );
+
+        TeardownSectionParser teardownSectionParser = new TeardownSectionParser();
+        TeardownSection section = teardownSectionParser.parse(new RestTestSuiteParseContext("api", "suite", parser));
+
+        assertThat(section, notNullValue());
+        assertThat(section.getSkipSection().isEmpty(), equalTo(false));
+        assertThat(section.getSkipSection().getLowerVersion(), equalTo(Version.V_2_0_0));
+        assertThat(section.getSkipSection().getUpperVersion(), equalTo(Version.V_2_3_0));
+        assertThat(section.getSkipSection().getReason(), equalTo("there is a reason"));
+        assertThat(section.getDoSections().size(), equalTo(2));
+        assertThat(section.getDoSections().get(0).getApiCallSection().getApi(), equalTo("delete"));
+        assertThat(section.getDoSections().get(1).getApiCallSection().getApi(), equalTo("delete2"));
+    }
+}

From e079c83020461ec58cfc24be85f58ab7ee407af1 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Thu, 30 Jun 2016 08:45:54 -0700
Subject: [PATCH 1159/1311] Fix test edge case for bytes reference

---
 .../common/bytes/AbstractBytesReferenceTestCase.java            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java b/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
index dd9d93dd22a..76757e76724 100644
--- a/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
+++ b/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
@@ -394,7 +394,7 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
     public void testSliceArrayOffset() throws IOException {
         int length = randomInt(PAGE_SIZE * randomIntBetween(2, 5));
         BytesReference pbr = newBytesReference(length);
-        int sliceOffset = randomIntBetween(0, pbr.length());
+        int sliceOffset = randomIntBetween(0, pbr.length() - 1);
         int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset);
         BytesReference slice = pbr.slice(sliceOffset, sliceLength);
         if (slice.hasArray()) {

From 04a4bcdca0374f1473b1264274954f4506e7403d Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Thu, 30 Jun 2016 08:47:55 -0700
Subject: [PATCH 1160/1311] Add comment explaining bytes reference edge case

---
 .../common/bytes/AbstractBytesReferenceTestCase.java            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java b/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
index 76757e76724..f31d7c69325 100644
--- a/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
+++ b/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
@@ -394,7 +394,7 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
     public void testSliceArrayOffset() throws IOException {
         int length = randomInt(PAGE_SIZE * randomIntBetween(2, 5));
         BytesReference pbr = newBytesReference(length);
-        int sliceOffset = randomIntBetween(0, pbr.length() - 1);
+        int sliceOffset = randomIntBetween(0, pbr.length() - 1); // an offset to the end would be len 0
         int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset);
         BytesReference slice = pbr.slice(sliceOffset, sliceLength);
         if (slice.hasArray()) {

From 09ca6d6ed21fec5c41e42642d0d956d31e1cac25 Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Thu, 30 Jun 2016 17:58:12 +0200
Subject: [PATCH 1161/1311] Add a BridgePartition to be used by
 testAckedIndexing (#19172)

We have long worked to capture different partitioning scenarios in our testing infra. This PR adds a new variant, inspired by the Jepsen blogs, which was forgotten far - namely a partition where one node can still see and be seen by all other nodes. It also updates the resiliency page to better reflect all the work that was done in this area.
---
 .../TransportMasterNodeActionTests.java       |  2 +-
 .../elasticsearch/common/BooleansTests.java   |  8 +-
 .../DiscoveryWithServiceDisruptionsIT.java    | 11 +--
 docs/resiliency/index.asciidoc                | 30 +++++++-
 .../index/reindex/ReindexScriptTests.java     |  3 +-
 .../org/elasticsearch/test/ESTestCase.java    |  9 ++-
 .../test/disruption/BridgePartition.java      | 74 +++++++++++++++++++
 7 files changed, 120 insertions(+), 17 deletions(-)
 create mode 100644 test/framework/src/main/java/org/elasticsearch/test/disruption/BridgePartition.java

diff --git a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
index 32fe6b1e408..e15c869c6ff 100644
--- a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
@@ -244,7 +244,7 @@ public class TransportMasterNodeActionTests extends ESTestCase {
         Request request = new Request();
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
 
-        setState(clusterService, ClusterStateCreationUtils.state(localNode, randomFrom(null, localNode, remoteNode), allNodes));
+        setState(clusterService, ClusterStateCreationUtils.state(localNode, randomFrom(localNode, remoteNode, null), allNodes));
 
         new Action(Settings.EMPTY, "testAction", transportService, clusterService, threadPool) {
             @Override
diff --git a/core/src/test/java/org/elasticsearch/common/BooleansTests.java b/core/src/test/java/org/elasticsearch/common/BooleansTests.java
index 6e5446cebf9..176c4c75dc7 100644
--- a/core/src/test/java/org/elasticsearch/common/BooleansTests.java
+++ b/core/src/test/java/org/elasticsearch/common/BooleansTests.java
@@ -51,9 +51,9 @@ public class BooleansTests extends ESTestCase {
         assertThat(Booleans.parseBoolean(null, false), is(false));
         assertThat(Booleans.parseBoolean(null, true), is(true));
 
-        assertThat(Booleans.parseBoolean(randomFrom("true", "on", "yes", "1"), randomFrom(null, Boolean.TRUE, Boolean.FALSE)), is(true));
-        assertThat(Booleans.parseBoolean(randomFrom("false", "off", "no", "0"), randomFrom(null, Boolean.TRUE, Boolean.FALSE)), is(false));
-        assertThat(Booleans.parseBoolean(randomFrom("true", "on", "yes").toUpperCase(Locale.ROOT),randomFrom(null, Boolean.TRUE, Boolean.FALSE)), is(true));
+        assertThat(Booleans.parseBoolean(randomFrom("true", "on", "yes", "1"), randomFrom(Boolean.TRUE, Boolean.FALSE, null)), is(true));
+        assertThat(Booleans.parseBoolean(randomFrom("false", "off", "no", "0"), randomFrom(Boolean.TRUE, Boolean.FALSE, null)), is(false));
+        assertThat(Booleans.parseBoolean(randomFrom("true", "on", "yes").toUpperCase(Locale.ROOT),randomFrom(Boolean.TRUE, Boolean.FALSE, null)), is(true));
         assertThat(Booleans.parseBoolean(null, Boolean.FALSE), is(false));
         assertThat(Booleans.parseBoolean(null, Boolean.TRUE), is(true));
         assertThat(Booleans.parseBoolean(null, null), nullValue());
@@ -70,7 +70,7 @@ public class BooleansTests extends ESTestCase {
         assertThat(Booleans.parseBooleanExact(randomFrom("true", "on", "yes", "1")), is(true));
         assertThat(Booleans.parseBooleanExact(randomFrom("false", "off", "no", "0")), is(false));
         try {
-            Booleans.parseBooleanExact(randomFrom(null, "fred", "foo", "barney"));
+            Booleans.parseBooleanExact(randomFrom("fred", "foo", "barney", null));
             fail("Expected exception while parsing invalid boolean value ");
         } catch (Exception ex) {
             assertTrue(ex instanceof IllegalArgumentException);
diff --git a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
index 0187bb28f36..7f89acd169e 100644
--- a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
+++ b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
@@ -62,6 +62,7 @@ import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.discovery.ClusterDiscoveryConfiguration;
 import org.elasticsearch.test.disruption.BlockClusterStateProcessing;
+import org.elasticsearch.test.disruption.BridgePartition;
 import org.elasticsearch.test.disruption.IntermittentLongGCDisruption;
 import org.elasticsearch.test.disruption.LongGCDisruption;
 import org.elasticsearch.test.disruption.NetworkDelaysPartition;
@@ -447,8 +448,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
         final int seconds = !(TEST_NIGHTLY && rarely()) ? 1 : 5;
         final String timeout = seconds + "s";
 
-        // TODO: add node count randomizaion
-        final List<String> nodes = startCluster(3);
+        final List<String> nodes = startCluster(rarely() ? 5 : 3);
 
         assertAcked(prepareCreate("test")
                 .setSettings(Settings.builder()
@@ -540,7 +540,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
                 logger.info("stopping disruption");
                 disruptionScheme.stopDisrupting();
                 for (String node : internalCluster().getNodeNames()) {
-                    ensureStableCluster(3, TimeValue.timeValueMillis(disruptionScheme.expectedTimeToHeal().millis() +
+                    ensureStableCluster(nodes.size(), TimeValue.timeValueMillis(disruptionScheme.expectedTimeToHeal().millis() +
                             DISRUPTION_HEALING_OVERHEAD.millis()), true, node);
                 }
                 ensureGreen("test");
@@ -548,7 +548,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
                 logger.info("validating successful docs");
                 for (String node : nodes) {
                     try {
-                        logger.debug("validating through node [{}]", node);
+                        logger.debug("validating through node [{}] ([{}] acked docs)", node, ackedDocs.size());
                         for (String id : ackedDocs.keySet()) {
                             assertTrue("doc [" + id + "] indexed via node [" + ackedDocs.get(id) + "] not found",
                                     client(node).prepareGet("test", "type", id).setPreference("_local").get().isExists());
@@ -1192,7 +1192,8 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
                 new NetworkUnresponsivePartition(random()),
                 new NetworkDelaysPartition(random()),
                 new NetworkDisconnectPartition(random()),
-                new SlowClusterStateProcessing(random())
+                new SlowClusterStateProcessing(random()),
+                new BridgePartition(random(), randomBoolean())
         );
         Collections.shuffle(list, random());
         setDisruptionScheme(list.get(0));
diff --git a/docs/resiliency/index.asciidoc b/docs/resiliency/index.asciidoc
index 6f3ed169709..802c380b4a7 100644
--- a/docs/resiliency/index.asciidoc
+++ b/docs/resiliency/index.asciidoc
@@ -55,6 +55,14 @@ If you encounter an issue, https://github.com/elastic/elasticsearch/issues[pleas
 
 We are committed to tracking down and fixing all the issues that are posted.
 
+[float]
+==== Jepsen Tests
+
+The Jepsen platform is specifically designed to test distributed systems. It is not a single test and is regularly adapted
+to create new scenarios. We have ported all published scenarios to our testing infrastructure. Of course
+as the system evolves, new scenarios can come up that are not yet covered. We are committed to investigating all new scenarios and will
+report issues that we find on this page and in our GitHub repository.
+
 [float]
 === Better request retry mechanism when nodes are disconnected (STATUS: ONGOING)
 
@@ -102,17 +110,31 @@ Indices stats and indices segments requests reach out to all nodes that have sha
 while the stats request arrives will make that part of the request fail and are just ignored in the overall stats result. {GIT}13719[#13719]
 
 [float]
-=== Jepsen Test Failures (STATUS: ONGOING)
+=== Documentation of guarantees and handling of failures (STATUS: ONGOING)
 
-We have increased our test coverage to include scenarios tested by Jepsen. We make heavy use of randomization to expand on the scenarios that can be tested and to introduce new error conditions. You can follow the work on the master branch of the https://github.com/elastic/elasticsearch/blob/master/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java[`DiscoveryWithServiceDisruptionsIT` class], where we will add more tests as time progresses.
+This status page is a start, but we can do a better job of explicitly documenting the processes at work in Elasticsearch and what happens
+in the case of each type of failure. The plan is to have a test case that validates each behavior under simulated conditions. Every test
+ will document the expected results, the associated test code, and an explicit PASS or FAIL status for each simulated case.
 
 [float]
-=== Document guarantees and handling of failure (STATUS: ONGOING)
+=== Run Jepsen (STATUS: ONGOING)
+
+We have ported all of the known scenarios in the Jepsen blogs to our testing infrastructure. The new tests are run continuously in our
+testing farm and are passing. We are also working on running Jepsen independently to verify that no failures are found.
 
-This status page is a start, but we can do a better job of explicitly documenting the processes at work in Elasticsearch, and what happens in the case of each type of failure. The plan is to have a test case that validates each behavior under simulated conditions. Every test will document the expected results, the associated test code and an explicit PASS or FAIL status for each simulated case.
 
 == Unreleased
 
+[float]
+=== Port Jepsen tests to our testing framework (STATUS: UNRELEASED, V5.0.0)
+
+We have increased our test coverage to include scenarios tested by Jepsen, as described in the Elasticsearch related blogs. We make heavy
+use of randomization to expand on the scenarios that can be tested and to introduce new error conditions.
+You can follow the work on the master branch of the
+https://github.com/elastic/elasticsearch/blob/master/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java[`DiscoveryWithServiceDisruptionsIT` class],
+where the `testAckedIndexing` test was specifically added to cover known Jepsen related scenarios.
+
+
 [float]
 === Loss of documents during network partition (STATUS: UNRELEASED, v5.0.0)
 
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java
index 74b7548cd63..c70b80b8e37 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexScriptTests.java
@@ -22,7 +22,6 @@ package org.elasticsearch.index.reindex;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.lucene.uid.Versions;
-import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.ScriptService;
 
 import java.util.Map;
@@ -106,7 +105,7 @@ public class ReindexScriptTests extends AbstractAsyncBulkIndexByScrollActionScri
     }
 
     public void testSetTimestamp() throws Exception {
-        String timestamp = randomFrom(null, "now", "1234");
+        String timestamp = randomFrom("now", "1234", null);
         IndexRequest index = applyScript((Map<String, Object> ctx) -> ctx.put("_timestamp", timestamp));
         assertEquals(timestamp, index.timestamp());
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
index d04d12304de..6d63b6a5428 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
@@ -92,6 +92,7 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.ExecutorService;
@@ -344,9 +345,15 @@ public abstract class ESTestCase extends LuceneTestCase {
 
     /** Pick a random object from the given array. The array must not be empty. */
     public static <T> T randomFrom(T... array) {
-        return RandomPicks.randomFrom(random(), array);
+        return randomFrom(random(), array);
     }
 
+    /** Pick a random object from the given array. The array must not be empty. */
+    public static <T> T randomFrom(Random random, T... array) {
+        return RandomPicks.randomFrom(random, array);
+    }
+
+
     /** Pick a random object from the given list. */
     public static <T> T randomFrom(List<T> list) {
         return RandomPicks.randomFrom(random(), list);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/BridgePartition.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/BridgePartition.java
new file mode 100644
index 00000000000..1a9c2b686c3
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/BridgePartition.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.test.disruption;
+
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.InternalTestCluster;
+import org.elasticsearch.test.transport.MockTransportService;
+
+import java.util.Random;
+
+import static org.elasticsearch.test.ESTestCase.randomFrom;
+
+/**
+ * A partition that breaks the cluster into two groups of nodes. The two groups are fully isolated
+ * with the exception of a single node that can see and be seen by all nodes in both groups.
+ */
+public class BridgePartition extends NetworkPartition {
+
+    String bridgeNode;
+    final boolean unresponsive;
+
+    public BridgePartition(Random random, boolean unresponsive) {
+        super(random);
+        this.unresponsive = unresponsive;
+    }
+
+    @Override
+    public void applyToCluster(InternalTestCluster cluster) {
+        bridgeNode = randomFrom(random, cluster.getNodeNames());
+        this.cluster = cluster;
+        for (String node: cluster.getNodeNames()) {
+            if (node.equals(bridgeNode) == false) {
+                super.applyToNode(node, cluster);
+            }
+        }
+    }
+
+    @Override
+    public TimeValue expectedTimeToHeal() {
+        return TimeValue.timeValueSeconds(0);
+    }
+
+    @Override
+    void applyDisruption(MockTransportService transportService1, MockTransportService transportService2) {
+        if (unresponsive) {
+            transportService1.addUnresponsiveRule(transportService2);
+            transportService2.addUnresponsiveRule(transportService1);
+        } else {
+            transportService1.addFailToSendNoConnectRule(transportService2);
+            transportService2.addFailToSendNoConnectRule(transportService1);
+        }
+    }
+
+    @Override
+    protected String getPartitionDescription() {
+        return "bridge (super connected node: [" + bridgeNode + "], unresponsive [" + unresponsive + "])";
+    }
+}

From e1ab3f16fd9d4372ba080b0ea7286adf97adef66 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Thu, 30 Jun 2016 18:32:15 +0200
Subject: [PATCH 1162/1311] Add link to alpha4 release notes

---
 docs/reference/release-notes.asciidoc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc
index 90492dee33e..ea535263b7d 100644
--- a/docs/reference/release-notes.asciidoc
+++ b/docs/reference/release-notes.asciidoc
@@ -5,12 +5,14 @@
 --
 This section summarizes the changes in each release.
 
+* <<release-notes-5.0.0-alpha4>>
 * <<release-notes-5.0.0-alpha3>>
 * <<release-notes-5.0.0-alpha2>>
 * <<release-notes-5.0.0-alpha1>>
 * <<release-notes-5.0.0-alpha1-2x>>
 
 --
+include::release-notes/5.0.0-alpha4.asciidoc[]
 include::release-notes/5.0.0-alpha3.asciidoc[]
 include::release-notes/5.0.0-alpha2.asciidoc[]
 include::release-notes/5.0.0-alpha1.asciidoc[]

From f5a269b029c4a357e257d7ef9b849470e76a2ddf Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 27 Jun 2016 11:21:45 -0400
Subject: [PATCH 1163/1311] Start migration away from aggregation streams

We'll migrate to NamedWriteable so we can share code with the rest
of the system. So we can work on this in multiple pull requests without
breaking Elasticsearch in between the commits this change supports
*both* old style `InternalAggregations.stream` serialization and
`NamedWriteable` style serialization. As such it creates about a
half dozen `// NORELEASE` comments that will have to be removed
once the migration is complete.

This also introduces a boolean `transportClient` flag to `SearchModule`
which is used to skip inappropriate registrations for for the
transport client while still registering the things it needs. In
this case that means that the `InternalAggregation` subclasses are
registered with the `NamedWriteableRegistry` but the `AggregationBuilder`
subclasses are not.

Finally, this moves aggregation registration from guice configuration
time to `SearchModule` construction time. This will make it simpler to
work with in the future as we further clean up Elasticsearch's
extension points.
---
 .../client/transport/TransportClient.java     |   7 +-
 .../java/org/elasticsearch/node/Node.java     |   2 +-
 .../elasticsearch/search/SearchModule.java    |  48 +++++---
 .../aggregations/InternalAggregation.java     | 111 ++++++++++++------
 .../aggregations/InternalAggregations.java    |  23 +++-
 .../metrics/InternalMetricsAggregation.java   |  10 ++
 .../InternalNumericMetricsAggregation.java    |  22 ++++
 .../metrics/avg/AvgAggregationBuilder.java    |   8 +-
 .../aggregations/metrics/avg/InternalAvg.java |  61 ++++------
 .../index/query/InnerHitBuilderTests.java     |   2 +-
 .../search/SearchModuleTests.java             |  10 +-
 .../aggregations/AggregatorParsingTests.java  |   2 +-
 .../aggregations/BaseAggregationTestCase.java |   2 +-
 .../SignificanceHeuristicTests.java           |   4 +-
 .../builder/SearchSourceBuilderTests.java     |   2 +-
 .../highlight/HighlightBuilderTests.java      |   2 +-
 .../rescore/QueryRescoreBuilderTests.java     |   2 +-
 .../search/sort/AbstractSortTestCase.java     |   2 +-
 .../search/sort/SortBuilderTests.java         |   2 +-
 .../AbstractSuggestionBuilderTestCase.java    |   2 +-
 .../messy/tests/TemplateQueryParserTests.java |   2 +-
 .../test/AbstractQueryTestCase.java           |   2 +-
 .../hamcrest/ElasticsearchAssertions.java     |   2 +-
 23 files changed, 204 insertions(+), 126 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
index c9313fc08c4..eeb9373c435 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
@@ -142,12 +142,7 @@ public class TransportClient extends AbstractClient {
                 }
                 modules.add(new NetworkModule(networkService, settings, true, namedWriteableRegistry));
                 modules.add(b -> b.bind(ThreadPool.class).toInstance(threadPool));
-                modules.add(new SearchModule(settings, namedWriteableRegistry) {
-                    @Override
-                    protected void configure() {
-                        // noop
-                    }
-                });
+                modules.add(new SearchModule(settings, namedWriteableRegistry, true));
                 modules.add(new ActionModule(false, true, settings, null, settingsModule.getClusterSettings(),
                         pluginsService.filterPlugins(ActionPlugin.class)));
 
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index ff095f749c8..fa7f090c096 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -267,7 +267,7 @@ public class Node implements Closeable {
             ClusterModule clusterModule = new ClusterModule(settings, clusterService);
             modules.add(clusterModule);
             modules.add(new IndicesModule(namedWriteableRegistry, pluginsService.filterPlugins(MapperPlugin.class)));
-            modules.add(new SearchModule(settings, namedWriteableRegistry));
+            modules.add(new SearchModule(settings, namedWriteableRegistry, false));
             modules.add(new ActionModule(DiscoveryNode.isIngestNode(settings), false, settings,
                     clusterModule.getIndexNameExpressionResolver(), settingsModule.getClusterSettings(),
                     pluginsService.filterPlugins(ActionPlugin.class)));
diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java
index 34a90894425..cd9d6f8a3f0 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -94,6 +94,7 @@ import org.elasticsearch.search.action.SearchTransportService;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.children.ChildrenAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.children.InternalChildren;
@@ -264,6 +265,7 @@ import java.util.Set;
  */
 public class SearchModule extends AbstractModule {
 
+    private final boolean transportClient;
     private final Highlighters highlighters;
     private final Suggesters suggesters;
     private final ParseFieldRegistry<ScoreFunctionParser<?>> scoreFunctionParserRegistry = new ParseFieldRegistry<>("score_function");
@@ -287,9 +289,10 @@ public class SearchModule extends AbstractModule {
     // pkg private so tests can mock
     Class<? extends SearchService> searchServiceImpl = SearchService.class;
 
-    public SearchModule(Settings settings, NamedWriteableRegistry namedWriteableRegistry) {
+    public SearchModule(Settings settings, NamedWriteableRegistry namedWriteableRegistry, boolean transportClient) {
         this.settings = settings;
         this.namedWriteableRegistry = namedWriteableRegistry;
+        this.transportClient = transportClient;
         suggesters = new Suggesters(namedWriteableRegistry);
         highlighters = new Highlighters(settings);
         registerBuiltinScoreFunctionParsers();
@@ -300,6 +303,7 @@ public class SearchModule extends AbstractModule {
         registerBuiltinSignificanceHeuristics();
         registerBuiltinMovingAverageModels();
         registerBuiltinSubFetchPhases();
+        registerBuiltinAggregations();
     }
 
     public void registerHighlighter(String key, Highlighter highligher) {
@@ -414,15 +418,27 @@ public class SearchModule extends AbstractModule {
     /**
      * Register an aggregation.
      *
-     * @param reader reads the aggregation builder from a stream
+     * @param builderReader reads the {@link AggregationBuilder} from a stream
+     * @param internalReader reads the {@link InternalAggregation} from a stream
      * @param aggregationParser reads the aggregation builder from XContent
      * @param aggregationName names by which the aggregation may be parsed. The first name is special because it is the name that the reader
      *        is registered under.
      */
-    public void registerAggregation(Writeable.Reader<? extends AggregationBuilder> reader, Aggregator.Parser aggregationParser,
-                                                                        ParseField aggregationName) {
+    public void registerAggregation(Writeable.Reader<? extends AggregationBuilder> builderReader,
+            Writeable.Reader<? extends InternalAggregation> internalReader, Aggregator.Parser aggregationParser,
+            ParseField aggregationName) {
+        if (false == transportClient) {
+            namedWriteableRegistry.register(AggregationBuilder.class, aggregationName.getPreferredName(), builderReader);
+            aggregationParserRegistry.register(aggregationParser, aggregationName);
+        }
+        namedWriteableRegistry.register(InternalAggregation.class, aggregationName.getPreferredName(), internalReader);
+    }
+
+    public void registerAggregation(Writeable.Reader<? extends AggregationBuilder> builderReader, Aggregator.Parser aggregationParser,
+            ParseField aggregationName) {
+        // NORELEASE remove me in favor of the above method
+        namedWriteableRegistry.register(AggregationBuilder.class, aggregationName.getPreferredName(), builderReader);
         aggregationParserRegistry.register(aggregationParser, aggregationName);
-        namedWriteableRegistry.register(AggregationBuilder.class, aggregationName.getPreferredName(), reader);
     }
 
     /**
@@ -441,15 +457,21 @@ public class SearchModule extends AbstractModule {
 
     @Override
     protected void configure() {
-        bind(IndicesQueriesRegistry.class).toInstance(queryParserRegistry);
-        bind(Suggesters.class).toInstance(suggesters);
-        configureSearch();
-        configureAggs();
-        configureShapes();
+        if (false == transportClient) {
+            /*
+             * Nothing is bound for transport client *but* SearchModule is still responsible for settings up the things like the
+             * NamedWriteableRegistry.
+             */
+            bind(IndicesQueriesRegistry.class).toInstance(queryParserRegistry);
+            bind(Suggesters.class).toInstance(suggesters);
+            configureSearch();
+            configureShapes();
+            bind(AggregatorParsers.class).toInstance(aggregatorParsers);
+        }
     }
 
-    protected void configureAggs() {
-        registerAggregation(AvgAggregationBuilder::new, new AvgParser(), AvgAggregationBuilder.AGGREGATION_NAME_FIELD);
+    private void registerBuiltinAggregations() {
+        registerAggregation(AvgAggregationBuilder::new, InternalAvg::new, new AvgParser(), AvgAggregationBuilder.AGGREGATION_NAME_FIELD);
         registerAggregation(SumAggregationBuilder::new, new SumParser(), SumAggregationBuilder.AGGREGATION_NAME_FIELD);
         registerAggregation(MinAggregationBuilder::new, new MinParser(), MinAggregationBuilder.AGGREGATION_NAME_FIELD);
         registerAggregation(MaxAggregationBuilder::new, new MaxParser(), MaxAggregationBuilder.AGGREGATION_NAME_FIELD);
@@ -527,7 +549,6 @@ public class SearchModule extends AbstractModule {
                 BucketSelectorPipelineAggregationBuilder.AGGREGATION_NAME_FIELD);
         registerPipelineAggregation(SerialDiffPipelineAggregationBuilder::new, SerialDiffPipelineAggregationBuilder::parse,
                 SerialDiffPipelineAggregationBuilder.AGGREGATION_NAME_FIELD);
-        bind(AggregatorParsers.class).toInstance(aggregatorParsers);
     }
 
     protected void configureSearch() {
@@ -679,7 +700,6 @@ public class SearchModule extends AbstractModule {
 
     static {
         // calcs
-        InternalAvg.registerStreams();
         InternalSum.registerStreams();
         InternalMin.registerStreams();
         InternalMax.registerStreams();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
index b4fab7093f8..b75244ce889 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
@@ -19,10 +19,10 @@
 package org.elasticsearch.search.aggregations;
 
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
@@ -43,8 +43,8 @@ import java.util.Map;
 /**
  * An internal implementation of {@link Aggregation}. Serves as a base class for all aggregation implementations.
  */
-public abstract class InternalAggregation implements Aggregation, ToXContent, Streamable {
-
+public abstract class InternalAggregation implements Aggregation, ToXContent, Streamable, NamedWriteable {
+    // NORELEASE remove Streamable
 
     /**
      * The aggregation type that holds all the string types that are associated with an aggregation:
@@ -139,15 +139,84 @@ public abstract class InternalAggregation implements Aggregation, ToXContent, St
         this.metaData = metaData;
     }
 
+    /**
+     * Read from a stream.
+     */
+    protected InternalAggregation(StreamInput in) throws IOException {
+        name = in.readString();
+        metaData = in.readMap();
+        int size = in.readVInt();
+        if (size == 0) {
+            pipelineAggregators = Collections.emptyList();
+        } else {
+            pipelineAggregators = new ArrayList<>(size);
+            for (int i = 0; i < size; i++) {
+                BytesReference type = in.readBytesReference();
+                PipelineAggregator pipelineAggregator = PipelineAggregatorStreams.stream(type).readResult(in);
+                pipelineAggregators.add(pipelineAggregator);
+            }
+        }
+    }
+
+    @Override
+    public final void readFrom(StreamInput in) throws IOException {
+        try {
+            getWriteableName(); // Throws UnsupportedOperationException if this aggregation should be read using old style Streams
+            assert false : "Used reading constructor instead";
+        } catch (UnsupportedOperationException e) {
+            // OK
+        }
+        name = in.readString();
+        metaData = in.readMap();
+        int size = in.readVInt();
+        if (size == 0) {
+            pipelineAggregators = Collections.emptyList();
+        } else {
+            pipelineAggregators = new ArrayList<>(size);
+            for (int i = 0; i < size; i++) {
+                BytesReference type = in.readBytesReference();
+                PipelineAggregator pipelineAggregator = PipelineAggregatorStreams.stream(type).readResult(in);
+                pipelineAggregators.add(pipelineAggregator);
+            }
+        }
+        doReadFrom(in);
+    }
+
+    protected void doReadFrom(StreamInput in) throws IOException {
+        throw new UnsupportedOperationException("Use reading constructor instead"); // NORELEASE remove when we remove Streamable
+    }
+
+    @Override
+    public final void writeTo(StreamOutput out) throws IOException {
+        out.writeString(name);    // NORELEASE remote writing the name - it is automatically handled with writeNamedWriteable
+        out.writeGenericValue(metaData);
+        out.writeVInt(pipelineAggregators.size());
+        for (PipelineAggregator pipelineAggregator : pipelineAggregators) {
+            out.writeBytesReference(pipelineAggregator.type().stream());
+            pipelineAggregator.writeTo(out);
+        }
+        doWriteTo(out);
+    }
+
+    protected abstract void doWriteTo(StreamOutput out) throws IOException;
+
     @Override
     public String getName() {
         return name;
     }
 
+    @Override
+    public String getWriteableName() {
+        // NORELEASE remove me when all InternalAggregations override it
+        throw new UnsupportedOperationException("Override on every class");
+    }
+
     /**
      * @return The {@link Type} of this aggregation
      */
-    public abstract Type type();
+    public Type type() {
+        throw new UnsupportedOperationException("Use getWriteableName instead"); // NORELEASE remove me
+    }
 
     /**
      * Reduces the given addAggregation to a single one and returns it. In <b>most</b> cases, the assumption will be the all given
@@ -214,40 +283,6 @@ public abstract class InternalAggregation implements Aggregation, ToXContent, St
 
     public abstract XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException;
 
-    @Override
-    public final void writeTo(StreamOutput out) throws IOException {
-        out.writeString(name);
-        out.writeGenericValue(metaData);
-        out.writeVInt(pipelineAggregators.size());
-        for (PipelineAggregator pipelineAggregator : pipelineAggregators) {
-            out.writeBytesReference(pipelineAggregator.type().stream());
-            pipelineAggregator.writeTo(out);
-        }
-        doWriteTo(out);
-    }
-
-    protected abstract void doWriteTo(StreamOutput out) throws IOException;
-
-    @Override
-    public final void readFrom(StreamInput in) throws IOException {
-        name = in.readString();
-        metaData = in.readMap();
-        int size = in.readVInt();
-        if (size == 0) {
-            pipelineAggregators = Collections.emptyList();
-        } else {
-            pipelineAggregators = new ArrayList<>(size);
-            for (int i = 0; i < size; i++) {
-                BytesReference type = in.readBytesReference();
-                PipelineAggregator pipelineAggregator = PipelineAggregatorStreams.stream(type).readResult(in);
-                pipelineAggregators.add(pipelineAggregator);
-            }
-        }
-        doReadFrom(in);
-    }
-
-    protected abstract void doReadFrom(StreamInput in) throws IOException;
-
     /**
      * Common xcontent fields that are shared among addAggregation
      */
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
index 357886a69f6..75223dcc476 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
@@ -205,9 +205,14 @@ public class InternalAggregations implements Aggregations, ToXContent, Streamabl
         } else {
             aggregations = new ArrayList<>(size);
             for (int i = 0; i < size; i++) {
-                BytesReference type = in.readBytesReference();
-                InternalAggregation aggregation = AggregationStreams.stream(type).readResult(in);
-                aggregations.add(aggregation);
+                // NORELEASE temporary hack to support old style streams and new style NamedWriteable at the same time
+                if (in.readBoolean()) {
+                    aggregations.add(in.readNamedWriteable(InternalAggregation.class));
+                } else {
+                    BytesReference type = in.readBytesReference();
+                    InternalAggregation aggregation = AggregationStreams.stream(type).readResult(in);
+                    aggregations.add(aggregation);
+                }
             }
         }
     }
@@ -217,8 +222,16 @@ public class InternalAggregations implements Aggregations, ToXContent, Streamabl
         out.writeVInt(aggregations.size());
         for (Aggregation aggregation : aggregations) {
             InternalAggregation internal = (InternalAggregation) aggregation;
-            out.writeBytesReference(internal.type().stream());
-            internal.writeTo(out);
+            // NORELEASE Temporary hack to support old style streams and new style NamedWriteable at the same time
+            try {
+                internal.getWriteableName(); // Throws UnsupportedOperationException if we should use old style streams.
+                out.writeBoolean(true);
+                out.writeNamedWriteable(internal);
+            } catch (UnsupportedOperationException e) {
+                out.writeBoolean(false);
+                out.writeBytesReference(internal.type().stream());
+                internal.writeTo(out);
+            }
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalMetricsAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalMetricsAggregation.java
index c67b7dc6b2f..5c2b1c46f5e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalMetricsAggregation.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalMetricsAggregation.java
@@ -19,9 +19,11 @@
 
 package org.elasticsearch.search.aggregations.metrics;
 
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
@@ -32,4 +34,12 @@ public abstract class InternalMetricsAggregation extends InternalAggregation {
     protected InternalMetricsAggregation(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
         super(name, pipelineAggregators, metaData);
     }
+
+    /**
+     * Read from a stream.
+     */
+    protected InternalMetricsAggregation(StreamInput in) throws IOException {
+        super(in);
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java
index 02412ba81c6..15c9d7ed59c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java
@@ -18,9 +18,11 @@
  */
 package org.elasticsearch.search.aggregations.metrics;
 
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
@@ -41,6 +43,13 @@ public abstract class InternalNumericMetricsAggregation extends InternalMetricsA
             super(name, pipelineAggregators, metaData);
         }
 
+        /**
+         * Read from a stream.
+         */
+        protected SingleValue(StreamInput in) throws IOException {
+            super(in);
+        }
+
         @Override
         public String getValueAsString() {
             return format.format(value());
@@ -67,6 +76,13 @@ public abstract class InternalNumericMetricsAggregation extends InternalMetricsA
             super(name, pipelineAggregators, metaData);
         }
 
+        /**
+         * Read from a stream.
+         */
+        protected MultiValue(StreamInput in) throws IOException {
+            super(in);
+        }
+
         public abstract double value(String name);
 
         public String valueAsString(String name) {
@@ -91,4 +107,10 @@ public abstract class InternalNumericMetricsAggregation extends InternalMetricsA
         super(name, pipelineAggregators, metaData);
     }
 
+    /**
+     * Read from a stream.
+     */
+    protected InternalNumericMetricsAggregation(StreamInput in) throws IOException {
+        super(in);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregationBuilder.java
index ce098177a0b..830ce35fde7 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregationBuilder.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
@@ -36,18 +37,19 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import java.io.IOException;
 
 public class AvgAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, AvgAggregationBuilder> {
-    public static final String NAME = InternalAvg.TYPE.name();
+    public static final String NAME = "avg";
+    private final static Type TYPE = new Type(NAME);
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     public AvgAggregationBuilder(String name) {
-        super(name, InternalAvg.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public AvgAggregationBuilder(StreamInput in) throws IOException {
-        super(in, InternalAvg.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/InternalAvg.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/InternalAvg.java
index bb4260b8189..5f0d54db003 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/InternalAvg.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/InternalAvg.java
@@ -22,7 +22,6 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -31,30 +30,9 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
-/**
-*
-*/
 public class InternalAvg extends InternalNumericMetricsAggregation.SingleValue implements Avg {
-
-    public final static Type TYPE = new Type("avg");
-
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public InternalAvg readResult(StreamInput in) throws IOException {
-            InternalAvg result = new InternalAvg();
-            result.readFrom(in);
-            return result;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
-    private double sum;
-    private long count;
-
-    InternalAvg() {} // for serialization
+    private final double sum;
+    private final long count;
 
     public InternalAvg(String name, double sum, long count, DocValueFormat format, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) {
@@ -64,6 +42,23 @@ public class InternalAvg extends InternalNumericMetricsAggregation.SingleValue i
         this.format = format;
     }
 
+    /**
+     * Read from a stream.
+     */
+    public InternalAvg(StreamInput in) throws IOException {
+        super(in);
+        format = in.readNamedWriteable(DocValueFormat.class);
+        sum = in.readDouble();
+        count = in.readVLong();
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(format);
+        out.writeDouble(sum);
+        out.writeVLong(count);
+    }
+
     @Override
     public double value() {
         return getValue();
@@ -75,8 +70,8 @@ public class InternalAvg extends InternalNumericMetricsAggregation.SingleValue i
     }
 
     @Override
-    public Type type() {
-        return TYPE;
+    public String getWriteableName() {
+        return AvgAggregationBuilder.NAME;
     }
 
     @Override
@@ -90,20 +85,6 @@ public class InternalAvg extends InternalNumericMetricsAggregation.SingleValue i
         return new InternalAvg(getName(), sum, count, format, pipelineAggregators(), getMetaData());
     }
 
-    @Override
-    protected void doReadFrom(StreamInput in) throws IOException {
-        format = in.readNamedWriteable(DocValueFormat.class);
-        sum = in.readDouble();
-        count = in.readVLong();
-    }
-
-    @Override
-    protected void doWriteTo(StreamOutput out) throws IOException {
-        out.writeNamedWriteable(format);
-        out.writeDouble(sum);
-        out.writeVLong(count);
-    }
-
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         builder.field(CommonFields.VALUE, count != 0 ? getValue() : null);
diff --git a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
index d4ba6ca9062..6460d8505ee 100644
--- a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
@@ -67,7 +67,7 @@ public class InnerHitBuilderTests extends ESTestCase {
     @BeforeClass
     public static void init() {
         namedWriteableRegistry = new NamedWriteableRegistry();
-        indicesQueriesRegistry = new SearchModule(Settings.EMPTY, namedWriteableRegistry).getQueryParserRegistry();
+        indicesQueriesRegistry = new SearchModule(Settings.EMPTY, namedWriteableRegistry, false).getQueryParserRegistry();
     }
 
     @AfterClass
diff --git a/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java b/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java
index 72873bc0d48..20a557d56a6 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchModuleTests.java
@@ -47,7 +47,7 @@ import static org.hamcrest.Matchers.notNullValue;
 public class SearchModuleTests extends ModuleTestCase {
 
    public void testDoubleRegister() {
-       SearchModule module = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry());
+       SearchModule module = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry(), false);
        try {
            module.registerHighlighter("fvh", new PlainHighlighter());
        } catch (IllegalArgumentException e) {
@@ -62,7 +62,7 @@ public class SearchModuleTests extends ModuleTestCase {
    }
 
     public void testRegisterSuggester() {
-        SearchModule module = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry());
+        SearchModule module = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry(), false);
         module.registerSuggester("custom", CustomSuggester.INSTANCE);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
                 () -> module.registerSuggester("custom", CustomSuggester.INSTANCE));
@@ -70,7 +70,7 @@ public class SearchModuleTests extends ModuleTestCase {
     }
 
     public void testRegisterHighlighter() {
-        SearchModule module = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry());
+        SearchModule module = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry(), false);
         CustomHighlighter customHighlighter = new CustomHighlighter();
         module.registerHighlighter("custom",  customHighlighter);
         IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
@@ -88,14 +88,14 @@ public class SearchModuleTests extends ModuleTestCase {
     }
 
     public void testRegisterQueryParserDuplicate() {
-        SearchModule module = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry());
+        SearchModule module = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry(), false);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> module
                 .registerQuery(TermQueryBuilder::new, TermQueryBuilder::fromXContent, TermQueryBuilder.QUERY_NAME_FIELD));
         assertThat(e.getMessage(), containsString("] already registered for [query][term] while trying to register [org.elasticsearch."));
     }
 
     public void testRegisteredQueries() throws IOException {
-        SearchModule module = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry());
+        SearchModule module = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry(), false);
         List<String> allSupportedQueries = new ArrayList<>();
         Collections.addAll(allSupportedQueries, NON_DEPRECATED_QUERIES);
         Collections.addAll(allSupportedQueries, DEPRECATED_QUERIES);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
index 1a21069623d..58b4b97db5c 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregatorParsingTests.java
@@ -119,7 +119,7 @@ public class AggregatorParsingTests extends ESTestCase {
                 protected void configure() {
                     bindMapperExtension();
                 }
-            }, new SearchModule(settings, namedWriteableRegistry) {
+            }, new SearchModule(settings, namedWriteableRegistry, false) {
                 @Override
                 protected void configureSearch() {
                     // Skip me
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
index 4e0429fca87..a4103e7ee56 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
@@ -151,7 +151,7 @@ public abstract class BaseAggregationTestCase<AB extends AbstractAggregationBuil
                     bindMapperExtension();
                 }
             },
-            new SearchModule(settings, namedWriteableRegistry) {
+            new SearchModule(settings, namedWriteableRegistry, false) {
                 @Override
                 protected void configureSearch() {
                     // Skip me
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java
index 8dc015b30ed..59aa7544d10 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/significant/SignificanceHeuristicTests.java
@@ -106,7 +106,7 @@ public class SignificanceHeuristicTests extends ESTestCase {
         ByteArrayInputStream inBuffer = new ByteArrayInputStream(outBuffer.toByteArray());
         StreamInput in = new InputStreamStreamInput(inBuffer);
         NamedWriteableRegistry registry = new NamedWriteableRegistry();
-        new SearchModule(Settings.EMPTY, registry); // populates the registry through side effects
+        new SearchModule(Settings.EMPTY, registry, false); // populates the registry through side effects
         in = new NamedWriteableAwareStreamInput(in, registry);
         in.setVersion(version);
         sigTerms[1].readFrom(in);
@@ -202,7 +202,7 @@ public class SignificanceHeuristicTests extends ESTestCase {
     // 1. The output of the builders can actually be parsed
     // 2. The parser does not swallow parameters after a significance heuristic was defined
     public void testBuilderAndParser() throws Exception {
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry());
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry(), false);
         ParseFieldRegistry<SignificanceHeuristicParser> heuristicParserMapper = searchModule.getSignificanceHeuristicParserRegistry();
         SearchContext searchContext = new SignificantTermsTestSearchContext();
 
diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index f3a78b65d78..ac90fabcb07 100644
--- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -145,7 +145,7 @@ public class SearchSourceBuilderTests extends ESTestCase {
                         bindMapperExtension();
                     }
                 },
-                new SearchModule(settings, namedWriteableRegistry) {
+                new SearchModule(settings, namedWriteableRegistry, false) {
                     @Override
                     protected void configureSearch() {
                         // Skip me
diff --git a/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java b/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java
index 9ebbb5b42e0..930a7b220e0 100644
--- a/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java
@@ -83,7 +83,7 @@ public class HighlightBuilderTests extends ESTestCase {
     @BeforeClass
     public static void init() {
         namedWriteableRegistry = new NamedWriteableRegistry();
-        indicesQueriesRegistry = new SearchModule(Settings.EMPTY, namedWriteableRegistry).getQueryParserRegistry();
+        indicesQueriesRegistry = new SearchModule(Settings.EMPTY, namedWriteableRegistry, false).getQueryParserRegistry();
     }
 
     @AfterClass
diff --git a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
index f965d3ac5fd..7c3690dcf35 100644
--- a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
@@ -70,7 +70,7 @@ public class QueryRescoreBuilderTests extends ESTestCase {
     @BeforeClass
     public static void init() {
         namedWriteableRegistry = new NamedWriteableRegistry();
-        indicesQueriesRegistry = new SearchModule(Settings.EMPTY, namedWriteableRegistry).getQueryParserRegistry();
+        indicesQueriesRegistry = new SearchModule(Settings.EMPTY, namedWriteableRegistry, false).getQueryParserRegistry();
     }
 
     @AfterClass
diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
index bdd5c76534c..b494fa4d1e6 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
@@ -106,7 +106,7 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST
         };
 
         namedWriteableRegistry = new NamedWriteableRegistry();
-        indicesQueriesRegistry = new SearchModule(Settings.EMPTY, namedWriteableRegistry).getQueryParserRegistry();
+        indicesQueriesRegistry = new SearchModule(Settings.EMPTY, namedWriteableRegistry, false).getQueryParserRegistry();
     }
 
     @AfterClass
diff --git a/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java
index f31158ff34e..54b12216302 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java
@@ -52,7 +52,7 @@ public class SortBuilderTests extends ESTestCase {
     @BeforeClass
     public static void init() {
         namedWriteableRegistry = new NamedWriteableRegistry();
-        indicesQueriesRegistry = new SearchModule(Settings.EMPTY, namedWriteableRegistry).getQueryParserRegistry();
+        indicesQueriesRegistry = new SearchModule(Settings.EMPTY, namedWriteableRegistry, false).getQueryParserRegistry();
     }
 
     @AfterClass
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
index 1f4030f487c..cd6c34497f7 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
@@ -56,7 +56,7 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
     @BeforeClass
     public static void init() throws IOException {
         namedWriteableRegistry = new NamedWriteableRegistry();
-        SearchModule searchModule = new SearchModule(Settings.EMPTY, namedWriteableRegistry);
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, namedWriteableRegistry, false);
         queriesRegistry = searchModule.getQueryParserRegistry();
         suggesters = searchModule.getSuggesters();
         parseFieldMatcher = ParseFieldMatcher.STRICT;
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
index 8d258865acb..fb8979562ff 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/messy/tests/TemplateQueryParserTests.java
@@ -118,7 +118,7 @@ public class TemplateQueryParserTests extends ESTestCase {
                     b.bind(CircuitBreakerService.class).to(NoneCircuitBreakerService.class);
                 },
                 settingsModule,
-                new SearchModule(settings, new NamedWriteableRegistry()) {
+                new SearchModule(settings, new NamedWriteableRegistry(), false) {
                     @Override
                     protected void configureSearch() {
                         // skip so we don't need transport
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
index c3693905501..e86854b34c4 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
@@ -880,7 +880,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
             scriptSettings.addAll(pluginsService.getPluginSettings());
             scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
             SettingsModule settingsModule = new SettingsModule(settings, scriptSettings, pluginsService.getPluginSettingsFilter());
-            searchModule = new SearchModule(settings, namedWriteableRegistry) {
+            searchModule = new SearchModule(settings, namedWriteableRegistry, false) {
                 @Override
                 protected void configureSearch() {
                     // Skip me
diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
index 51d15e019a4..baad3ef6f04 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
@@ -631,7 +631,7 @@ public class ElasticsearchAssertions {
             registry = ESIntegTestCase.internalCluster().getInstance(NamedWriteableRegistry.class);
         } else {
             registry = new NamedWriteableRegistry();
-            new SearchModule(Settings.EMPTY, registry);
+            new SearchModule(Settings.EMPTY, registry, false);
         }
         assertVersionSerializable(version, streamable, registry);
     }

From a24d302e73251ae4fd2d41abff1fa2cab419be66 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 30 Jun 2016 14:47:38 -0400
Subject: [PATCH 1164/1311] Fix discovery-azure-classic plugin packaging test

This commit fixes the discovery-azure-classing packaging test by fixing
the expected name of the installed plugin.
---
 .../packaging/scripts/module_and_plugin_test_cases.bash         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
index c17df96937a..21882c0ad53 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
@@ -210,7 +210,7 @@ fi
 }
 
 @test "[$GROUP] install discovery-azure-classic plugin" {
-    install_and_check_plugin discovery azure azure-core-*.jar
+    install_and_check_plugin discovery azure-classic azure-core-*.jar
 }
 
 @test "[$GROUP] install discovery-ec2 plugin" {

From d24cc65cad2f3152237df8b6c457a2d0a603f13a Mon Sep 17 00:00:00 2001
From: gfyoung <gfyoung17@gmail.com>
Date: Thu, 30 Jun 2016 23:00:10 -0400
Subject: [PATCH 1165/1311] Raised IOException on deleteBlob (#18815)

Raise IOException on deleteBlob if the blob doesn't exist

This commit raises an IOException on BlobContainer#deleteBlob
if the blob does not exist, in conformance with the BlobContainer
interface contract.  Each implementation of BlobContainer now
conforms to this contract (file system, S3, Azure, HDFS).  This
commit also contains blob container tests for each of the
repository implementations.

Closes #18530
---
 .../common/blobstore/fs/FsBlobContainer.java  |   2 +-
 .../azure/blobstore/AzureBlobContainer.java   |  10 +
 .../storage/AzureStorageServiceMock.java      |   4 +-
 .../azure/AzureBlobStoreContainerTests.java   |  47 ++++
 .../repositories/hdfs/HdfsBlobContainer.java  |  18 +-
 .../hdfs/HdfsBlobStoreContainerTests.java     | 104 +++++++++
 .../cloud/aws/blobstore/S3BlobContainer.java  |   4 +
 .../cloud/aws/blobstore/MockAmazonS3.java     | 200 ++++++++++++++++++
 .../blobstore/S3BlobStoreContainerTests.java  |  39 ++++
 .../ESBlobStoreContainerTestCase.java         |  16 ++
 10 files changed, 432 insertions(+), 12 deletions(-)
 create mode 100644 plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreContainerTests.java
 create mode 100644 plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java
 create mode 100644 plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/MockAmazonS3.java
 create mode 100644 plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreContainerTests.java

diff --git a/core/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java b/core/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java
index 822f8d1721a..67ce298becd 100644
--- a/core/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java
+++ b/core/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java
@@ -85,7 +85,7 @@ public class FsBlobContainer extends AbstractBlobContainer {
     @Override
     public void deleteBlob(String blobName) throws IOException {
         Path blobPath = path.resolve(blobName);
-        Files.deleteIfExists(blobPath);
+        Files.delete(blobPath);
     }
 
     @Override
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
index e6c3a469076..6117062fc29 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
@@ -70,6 +70,11 @@ public class AzureBlobContainer extends AbstractBlobContainer {
     @Override
     public InputStream readBlob(String blobName) throws IOException {
         logger.trace("readBlob({})", blobName);
+
+        if (!blobExists(blobName)) {
+            throw new IOException("Blob [" + blobName + "] does not exist");
+        }
+
         try {
             return blobStore.getInputStream(blobStore.container(), buildKey(blobName));
         } catch (StorageException e) {
@@ -116,6 +121,11 @@ public class AzureBlobContainer extends AbstractBlobContainer {
     @Override
     public void deleteBlob(String blobName) throws IOException {
         logger.trace("deleteBlob({})", blobName);
+
+        if (!blobExists(blobName)) {
+            throw new IOException("Blob [" + blobName + "] does not exist");
+        }
+
         try {
             blobStore.deleteBlob(blobStore.container(), buildKey(blobName));
         } catch (URISyntaxException | StorageException e) {
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java
index 8160c560325..506d574ea62 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java
@@ -95,13 +95,13 @@ public class AzureStorageServiceMock extends AbstractLifecycleComponent<AzureSto
         MapBuilder<String, BlobMetaData> blobsBuilder = MapBuilder.newMapBuilder();
         for (String blobName : blobs.keySet()) {
             final String checkBlob;
-            if (keyPath != null) {
+            if (keyPath != null && !keyPath.isEmpty()) {
                 // strip off key path from the beginning of the blob name
                 checkBlob = blobName.replace(keyPath, "");
             } else {
                 checkBlob = blobName;
             }
-            if (startsWithIgnoreCase(checkBlob, prefix)) {
+            if (prefix == null || startsWithIgnoreCase(checkBlob, prefix)) {
                 blobsBuilder.put(blobName, new PlainBlobMetaData(checkBlob, blobs.get(blobName).size()));
             }
         }
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreContainerTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreContainerTests.java
new file mode 100644
index 00000000000..5b161613c9b
--- /dev/null
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreContainerTests.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.azure;
+
+import com.microsoft.azure.storage.StorageException;
+import org.elasticsearch.cloud.azure.blobstore.AzureBlobStore;
+import org.elasticsearch.cloud.azure.storage.AzureStorageServiceMock;
+import org.elasticsearch.common.blobstore.BlobStore;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.repositories.ESBlobStoreContainerTestCase;
+import org.elasticsearch.repositories.RepositoryName;
+import org.elasticsearch.repositories.RepositorySettings;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+public class AzureBlobStoreContainerTests extends ESBlobStoreContainerTestCase {
+    @Override
+    protected BlobStore newBlobStore() throws IOException {
+        try {
+            RepositoryName repositoryName = new RepositoryName("azure", "ittest");
+            RepositorySettings repositorySettings = new RepositorySettings(
+                    Settings.EMPTY, Settings.EMPTY);
+            AzureStorageServiceMock client = new AzureStorageServiceMock(Settings.EMPTY);
+            return new AzureBlobStore(repositoryName, Settings.EMPTY, repositorySettings, client);
+        } catch (URISyntaxException | StorageException e) {
+            throw new IOException(e);
+        }
+    }
+}
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
index 6ba726e2b24..c8b3d9f7e1d 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
@@ -68,16 +68,16 @@ final class HdfsBlobContainer extends AbstractBlobContainer {
 
     @Override
     public void deleteBlob(String blobName) throws IOException {
-        try {
-            store.execute(new Operation<Boolean>() {
-                @Override
-                public Boolean run(FileContext fileContext) throws IOException {
-                    return fileContext.delete(new Path(path, blobName), true);
-                }
-            });
-        } catch (FileNotFoundException ok) {
-            // behaves like Files.deleteIfExists
+        if (!blobExists(blobName)) {
+            throw new IOException("Blob [" + blobName + "] does not exist");
         }
+
+        store.execute(new Operation<Boolean>() {
+            @Override
+            public Boolean run(FileContext fileContext) throws IOException {
+                return fileContext.delete(new Path(path, blobName), true);
+            }
+        });
     }
 
     @Override
diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java
new file mode 100644
index 00000000000..a96a8183e58
--- /dev/null
+++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.AbstractFileSystem;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.elasticsearch.common.blobstore.BlobStore;
+import org.elasticsearch.repositories.ESBlobStoreContainerTestCase;
+
+import javax.security.auth.Subject;
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.security.AccessController;
+import java.security.Principal;
+import java.security.PrivilegedAction;
+import java.util.Collections;
+
+public class HdfsBlobStoreContainerTests extends ESBlobStoreContainerTestCase {
+
+    @Override
+    protected BlobStore newBlobStore() throws IOException {
+        return AccessController.doPrivileged(
+                new PrivilegedAction<HdfsBlobStore>() {
+                    @Override
+                    public HdfsBlobStore run() {
+                        try {
+                            FileContext fileContext = createContext(new URI("hdfs:///"));
+                            return new HdfsBlobStore(fileContext, "temp", 1024);
+                        } catch (IOException | URISyntaxException e) {
+                            throw new RuntimeException(e);
+                        }
+            }
+        });
+    }
+
+    public FileContext createContext(URI uri) {
+        // mirrors HdfsRepository.java behaviour
+        Configuration cfg = new Configuration(true);
+        cfg.setClassLoader(HdfsRepository.class.getClassLoader());
+        cfg.reloadConfiguration();
+
+        Constructor<?> ctor;
+        Subject subject;
+
+        try {
+            Class<?> clazz = Class.forName("org.apache.hadoop.security.User");
+            ctor = clazz.getConstructor(String.class);
+            ctor.setAccessible(true);
+        }  catch (ClassNotFoundException | NoSuchMethodException e) {
+            throw new RuntimeException(e);
+        }
+
+        try {
+            Principal principal = (Principal) ctor.newInstance(System.getProperty("user.name"));
+            subject = new Subject(false, Collections.singleton(principal),
+                    Collections.emptySet(), Collections.emptySet());
+        } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) {
+            throw new RuntimeException(e);
+        }
+
+        // disable file system cache
+        cfg.setBoolean("fs.hdfs.impl.disable.cache", true);
+
+        // set file system to TestingFs to avoid a bunch of security
+        // checks, similar to what is done in HdfsTests.java
+        cfg.set(String.format("fs.AbstractFileSystem.%s.impl", uri.getScheme()),
+                TestingFs.class.getName());
+
+        // create the FileContext with our user
+        return Subject.doAs(subject, new PrivilegedAction<FileContext>() {
+            @Override
+            public FileContext run() {
+                try {
+                    TestingFs fs = (TestingFs) AbstractFileSystem.get(uri, cfg);
+                    return FileContext.getFileContext(fs, cfg);
+                } catch (UnsupportedFileSystemException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+        });
+    }
+}
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java
index 42df840ce40..5e014ab3ecd 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java
@@ -108,6 +108,10 @@ public class S3BlobContainer extends AbstractBlobContainer {
 
     @Override
     public void deleteBlob(String blobName) throws IOException {
+        if (!blobExists(blobName)) {
+            throw new IOException("Blob [" + blobName + "] does not exist");
+        }
+
         try {
             blobStore.client().deleteObject(blobStore.bucket(), buildKey(blobName));
         } catch (AmazonClientException e) {
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/MockAmazonS3.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/MockAmazonS3.java
new file mode 100644
index 00000000000..8124f693943
--- /dev/null
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/MockAmazonS3.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cloud.aws.blobstore;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.services.s3.AbstractAmazonS3;
+import com.amazonaws.services.s3.model.AmazonS3Exception;
+import com.amazonaws.services.s3.model.CopyObjectRequest;
+import com.amazonaws.services.s3.model.CopyObjectResult;
+import com.amazonaws.services.s3.model.DeleteObjectRequest;
+import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
+import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.services.s3.model.PutObjectResult;
+import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.model.S3ObjectInputStream;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.amazonaws.util.Base64;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.DigestInputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+class MockAmazonS3 extends AbstractAmazonS3 {
+
+    private Map<String, InputStream> blobs = new ConcurrentHashMap<>();
+
+    // in ESBlobStoreContainerTestCase.java, the maximum
+    // length of the input data is 100 bytes
+    private byte[] byteCounter = new byte[100];
+
+    @Override
+    public boolean doesBucketExist(String bucket) {
+        return true;
+    }
+
+    @Override
+    public ObjectMetadata getObjectMetadata(
+            GetObjectMetadataRequest getObjectMetadataRequest)
+            throws AmazonClientException, AmazonServiceException {
+        String blobName = getObjectMetadataRequest.getKey();
+
+        if (!blobs.containsKey(blobName)) {
+            throw new AmazonS3Exception("[" + blobName + "] does not exist.");
+        }
+
+        return new ObjectMetadata(); // nothing is done with it
+    }
+
+    @Override
+    public PutObjectResult putObject(PutObjectRequest putObjectRequest)
+            throws AmazonClientException, AmazonServiceException {
+        String blobName = putObjectRequest.getKey();
+        DigestInputStream stream = (DigestInputStream) putObjectRequest.getInputStream();
+
+        if (blobs.containsKey(blobName)) {
+            throw new AmazonS3Exception("[" + blobName + "] already exists.");
+        }
+
+        blobs.put(blobName, stream);
+
+        // input and output md5 hashes need to match to avoid an exception
+        String md5 = Base64.encodeAsString(stream.getMessageDigest().digest());
+        PutObjectResult result = new PutObjectResult();
+        result.setContentMd5(md5);
+
+        return result;
+    }
+
+    @Override
+    public S3Object getObject(GetObjectRequest getObjectRequest)
+            throws AmazonClientException, AmazonServiceException {
+        // in ESBlobStoreContainerTestCase.java, the prefix is empty,
+        // so the key and blobName are equivalent to each other
+        String blobName = getObjectRequest.getKey();
+
+        if (!blobs.containsKey(blobName)) {
+            throw new AmazonS3Exception("[" + blobName + "] does not exist.");
+        }
+
+        // the HTTP request attribute is irrelevant for reading
+        S3ObjectInputStream stream = new S3ObjectInputStream(
+                blobs.get(blobName), null, false);
+        S3Object s3Object = new S3Object();
+        s3Object.setObjectContent(stream);
+        return s3Object;
+    }
+
+    @Override
+    public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
+            throws AmazonClientException, AmazonServiceException {
+        MockObjectListing list = new MockObjectListing();
+        list.setTruncated(false);
+
+        String blobName;
+        String prefix = listObjectsRequest.getPrefix();
+
+        ArrayList<S3ObjectSummary> mockObjectSummaries = new ArrayList<>();
+
+        for (Map.Entry<String, InputStream> blob : blobs.entrySet()) {
+            blobName = blob.getKey();
+            S3ObjectSummary objectSummary = new S3ObjectSummary();
+
+            if (prefix.isEmpty() || blobName.startsWith(prefix)) {
+                objectSummary.setKey(blobName);
+
+                try {
+                    objectSummary.setSize(getSize(blob.getValue()));
+                } catch (IOException e) {
+                    throw new AmazonS3Exception("Object listing " +
+                            "failed for blob [" + blob.getKey() + "]");
+                }
+
+                mockObjectSummaries.add(objectSummary);
+            }
+        }
+
+        list.setObjectSummaries(mockObjectSummaries);
+        return list;
+    }
+
+    @Override
+    public CopyObjectResult copyObject(CopyObjectRequest copyObjectRequest)
+            throws AmazonClientException, AmazonServiceException {
+        String sourceBlobName = copyObjectRequest.getSourceKey();
+        String targetBlobName = copyObjectRequest.getDestinationKey();
+
+        if (!blobs.containsKey(sourceBlobName)) {
+            throw new AmazonS3Exception("Source blob [" +
+                    sourceBlobName + "] does not exist.");
+        }
+
+        if (blobs.containsKey(targetBlobName)) {
+            throw new AmazonS3Exception("Target blob [" +
+                    targetBlobName + "] already exists.");
+        }
+
+        blobs.put(targetBlobName, blobs.get(sourceBlobName));
+        return new CopyObjectResult(); // nothing is done with it
+    }
+
+    @Override
+    public void deleteObject(DeleteObjectRequest deleteObjectRequest)
+            throws AmazonClientException, AmazonServiceException {
+        String blobName = deleteObjectRequest.getKey();
+
+        if (!blobs.containsKey(blobName)) {
+            throw new AmazonS3Exception("[" + blobName + "] does not exist.");
+        }
+
+        blobs.remove(blobName);
+    }
+
+    private int getSize(InputStream stream) throws IOException {
+        int size = stream.read(byteCounter);
+        stream.reset(); // in case we ever need the size again
+        return size;
+    }
+
+    private class MockObjectListing extends ObjectListing {
+        // the objectSummaries attribute in ObjectListing.java
+        // is read-only, but we need to be able to write to it,
+        // so we create a mock of it to work around this
+        private List<S3ObjectSummary> mockObjectSummaries;
+
+        @Override
+        public List<S3ObjectSummary> getObjectSummaries() {
+            return mockObjectSummaries;
+        }
+
+        private void setObjectSummaries(List<S3ObjectSummary> objectSummaries) {
+            mockObjectSummaries = objectSummaries;
+        }
+    }
+}
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreContainerTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreContainerTests.java
new file mode 100644
index 00000000000..bca1c1d8a18
--- /dev/null
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreContainerTests.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.cloud.aws.blobstore;
+
+import org.elasticsearch.common.blobstore.BlobStore;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.repositories.ESBlobStoreContainerTestCase;
+
+import java.io.IOException;
+import java.util.Locale;
+
+public class S3BlobStoreContainerTests extends ESBlobStoreContainerTestCase {
+    protected BlobStore newBlobStore() throws IOException {
+        MockAmazonS3 client = new MockAmazonS3();
+        String bucket = randomAsciiOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
+
+        return new S3BlobStore(Settings.EMPTY, client, bucket, null, false,
+            new ByteSizeValue(10, ByteSizeUnit.MB), 5, "public-read-write", "standard");
+    }
+}
diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java
index 6ff0b71cdcc..67ad0eb7358 100644
--- a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java
@@ -112,6 +112,22 @@ public abstract class ESBlobStoreContainerTestCase extends ESTestCase {
         }
     }
 
+    public void testDeleteBlob() throws IOException {
+        try (final BlobStore store = newBlobStore()) {
+            final String blobName = "foobar";
+            final BlobContainer container = store.blobContainer(new BlobPath());
+            expectThrows(IOException.class, () -> container.deleteBlob(blobName));
+
+            byte[] data = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16)));
+            final BytesArray bytesArray = new BytesArray(data);
+            container.writeBlob(blobName, bytesArray);
+            container.deleteBlob(blobName); // should not raise
+
+            // blob deleted, so should raise again
+            expectThrows(IOException.class, () -> container.deleteBlob(blobName));
+        }
+    }
+
     @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/15579")
     public void testOverwriteFails() throws IOException {
         try (final BlobStore store = newBlobStore()) {

From 9bfc23e9582502b937b1dd7a7b116f931482fca9 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Tue, 28 Jun 2016 17:07:04 +0200
Subject: [PATCH 1166/1311] Add missing permission to repository-s3

Repository-S3 needs a special permission because of problems in AmazonS3Client: when no region is set on a AmazonS3Client instance, the AWS SDK loads all known partitions from a JSON file and uses a Jackson's ObjectMapper for that: this one, in version 2.5.3 with the default binding options, tries to suppress access checks of ctor/field/method and thus requires this special permission. AWS must be fixed to uses Jackson correctly and have the correct modifiers on binded classes.

This must be fixed in aws sdk (see https://github.com/aws/aws-sdk-java/issues/766) but in the meanwhile we have no choice.

closes #18539
---
 .../cloud/aws/blobstore/S3BlobContainer.java  | 29 +++++++++++++++++--
 .../plugin-metadata/plugin-security.policy    | 12 ++++++++
 2 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java
index 5e014ab3ecd..69955fb5460 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java
@@ -26,6 +26,7 @@ import com.amazonaws.services.s3.model.ObjectListing;
 import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.S3Object;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
+import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
@@ -40,6 +41,9 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.security.AccessController;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
 import java.util.Map;
 
 /**
@@ -60,8 +64,14 @@ public class S3BlobContainer extends AbstractBlobContainer {
     @Override
     public boolean blobExists(String blobName) {
         try {
-            blobStore.client().getObjectMetadata(blobStore.bucket(), buildKey(blobName));
-            return true;
+            return doPrivileged(() -> {
+                try {
+                    blobStore.client().getObjectMetadata(blobStore.bucket(), buildKey(blobName));
+                    return true;
+                } catch (AmazonS3Exception e) {
+                    return false;
+                }
+            });
         } catch (AmazonS3Exception e) {
             return false;
         } catch (Throwable e) {
@@ -180,4 +190,19 @@ public class S3BlobContainer extends AbstractBlobContainer {
         return keyPath + blobName;
     }
 
+    /**
+     * +     * Executes a {@link PrivilegedExceptionAction} with privileges enabled.
+     * +
+     */
+    <T> T doPrivileged(PrivilegedExceptionAction<T> operation) throws IOException {
+        SecurityManager sm = System.getSecurityManager();
+        if (sm != null) {
+            sm.checkPermission(new SpecialPermission());
+        }
+        try {
+            return AccessController.doPrivileged(operation);
+        } catch (PrivilegedActionException e) {
+            throw (IOException) e.getException();
+        }
+    }
 }
diff --git a/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy b/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy
index e5f26c3e9d1..1f09cada2e5 100644
--- a/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy
+++ b/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy
@@ -22,4 +22,16 @@ grant {
   // TODO: get these fixed in aws sdk
   permission java.lang.RuntimePermission "accessDeclaredMembers";
   permission java.lang.RuntimePermission "getClassLoader";
+  // Needed because of problems in AmazonS3Client:
+  // When no region is set on a AmazonS3Client instance, the
+  // AWS SDK loads all known partitions from a JSON file and
+  // uses a Jackson's ObjectMapper for that: this one, in
+  // version 2.5.3 with the default binding options, tries
+  // to suppress access checks of ctor/field/method and thus
+  // requires this special permission. AWS must be fixed to
+  // uses Jackson correctly and have the correct modifiers
+  // on binded classes.
+  // TODO: get these fixed in aws sdk
+  // See https://github.com/aws/aws-sdk-java/issues/766
+  permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
 };

From 598c36128e3ea521b3203c8addbe54067bd44b29 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 1 Jul 2016 11:00:32 +0200
Subject: [PATCH 1167/1311] Revert "Raised IOException on deleteBlob (#18815)"

This reverts commit d24cc65cad2f3152237df8b6c457a2d0a603f13a as it seems to be causing test failures.
---
 .../common/blobstore/fs/FsBlobContainer.java  |   2 +-
 .../azure/blobstore/AzureBlobContainer.java   |  10 -
 .../storage/AzureStorageServiceMock.java      |   4 +-
 .../azure/AzureBlobStoreContainerTests.java   |  47 ----
 .../repositories/hdfs/HdfsBlobContainer.java  |  18 +-
 .../hdfs/HdfsBlobStoreContainerTests.java     | 104 ---------
 .../cloud/aws/blobstore/S3BlobContainer.java  |   4 -
 .../cloud/aws/blobstore/MockAmazonS3.java     | 200 ------------------
 .../blobstore/S3BlobStoreContainerTests.java  |  39 ----
 .../ESBlobStoreContainerTestCase.java         |  16 --
 10 files changed, 12 insertions(+), 432 deletions(-)
 delete mode 100644 plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreContainerTests.java
 delete mode 100644 plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java
 delete mode 100644 plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/MockAmazonS3.java
 delete mode 100644 plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreContainerTests.java

diff --git a/core/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java b/core/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java
index 67ce298becd..822f8d1721a 100644
--- a/core/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java
+++ b/core/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java
@@ -85,7 +85,7 @@ public class FsBlobContainer extends AbstractBlobContainer {
     @Override
     public void deleteBlob(String blobName) throws IOException {
         Path blobPath = path.resolve(blobName);
-        Files.delete(blobPath);
+        Files.deleteIfExists(blobPath);
     }
 
     @Override
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
index 6117062fc29..e6c3a469076 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java
@@ -70,11 +70,6 @@ public class AzureBlobContainer extends AbstractBlobContainer {
     @Override
     public InputStream readBlob(String blobName) throws IOException {
         logger.trace("readBlob({})", blobName);
-
-        if (!blobExists(blobName)) {
-            throw new IOException("Blob [" + blobName + "] does not exist");
-        }
-
         try {
             return blobStore.getInputStream(blobStore.container(), buildKey(blobName));
         } catch (StorageException e) {
@@ -121,11 +116,6 @@ public class AzureBlobContainer extends AbstractBlobContainer {
     @Override
     public void deleteBlob(String blobName) throws IOException {
         logger.trace("deleteBlob({})", blobName);
-
-        if (!blobExists(blobName)) {
-            throw new IOException("Blob [" + blobName + "] does not exist");
-        }
-
         try {
             blobStore.deleteBlob(blobStore.container(), buildKey(blobName));
         } catch (URISyntaxException | StorageException e) {
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java
index 506d574ea62..8160c560325 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java
@@ -95,13 +95,13 @@ public class AzureStorageServiceMock extends AbstractLifecycleComponent<AzureSto
         MapBuilder<String, BlobMetaData> blobsBuilder = MapBuilder.newMapBuilder();
         for (String blobName : blobs.keySet()) {
             final String checkBlob;
-            if (keyPath != null && !keyPath.isEmpty()) {
+            if (keyPath != null) {
                 // strip off key path from the beginning of the blob name
                 checkBlob = blobName.replace(keyPath, "");
             } else {
                 checkBlob = blobName;
             }
-            if (prefix == null || startsWithIgnoreCase(checkBlob, prefix)) {
+            if (startsWithIgnoreCase(checkBlob, prefix)) {
                 blobsBuilder.put(blobName, new PlainBlobMetaData(checkBlob, blobs.get(blobName).size()));
             }
         }
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreContainerTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreContainerTests.java
deleted file mode 100644
index 5b161613c9b..00000000000
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreContainerTests.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.repositories.azure;
-
-import com.microsoft.azure.storage.StorageException;
-import org.elasticsearch.cloud.azure.blobstore.AzureBlobStore;
-import org.elasticsearch.cloud.azure.storage.AzureStorageServiceMock;
-import org.elasticsearch.common.blobstore.BlobStore;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.repositories.ESBlobStoreContainerTestCase;
-import org.elasticsearch.repositories.RepositoryName;
-import org.elasticsearch.repositories.RepositorySettings;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-
-public class AzureBlobStoreContainerTests extends ESBlobStoreContainerTestCase {
-    @Override
-    protected BlobStore newBlobStore() throws IOException {
-        try {
-            RepositoryName repositoryName = new RepositoryName("azure", "ittest");
-            RepositorySettings repositorySettings = new RepositorySettings(
-                    Settings.EMPTY, Settings.EMPTY);
-            AzureStorageServiceMock client = new AzureStorageServiceMock(Settings.EMPTY);
-            return new AzureBlobStore(repositoryName, Settings.EMPTY, repositorySettings, client);
-        } catch (URISyntaxException | StorageException e) {
-            throw new IOException(e);
-        }
-    }
-}
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
index c8b3d9f7e1d..6ba726e2b24 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
@@ -68,16 +68,16 @@ final class HdfsBlobContainer extends AbstractBlobContainer {
 
     @Override
     public void deleteBlob(String blobName) throws IOException {
-        if (!blobExists(blobName)) {
-            throw new IOException("Blob [" + blobName + "] does not exist");
+        try {
+            store.execute(new Operation<Boolean>() {
+                @Override
+                public Boolean run(FileContext fileContext) throws IOException {
+                    return fileContext.delete(new Path(path, blobName), true);
+                }
+            });
+        } catch (FileNotFoundException ok) {
+            // behaves like Files.deleteIfExists
         }
-
-        store.execute(new Operation<Boolean>() {
-            @Override
-            public Boolean run(FileContext fileContext) throws IOException {
-                return fileContext.delete(new Path(path, blobName), true);
-            }
-        });
     }
 
     @Override
diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java
deleted file mode 100644
index a96a8183e58..00000000000
--- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.repositories.hdfs;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.AbstractFileSystem;
-import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.UnsupportedFileSystemException;
-import org.elasticsearch.common.blobstore.BlobStore;
-import org.elasticsearch.repositories.ESBlobStoreContainerTestCase;
-
-import javax.security.auth.Subject;
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.security.AccessController;
-import java.security.Principal;
-import java.security.PrivilegedAction;
-import java.util.Collections;
-
-public class HdfsBlobStoreContainerTests extends ESBlobStoreContainerTestCase {
-
-    @Override
-    protected BlobStore newBlobStore() throws IOException {
-        return AccessController.doPrivileged(
-                new PrivilegedAction<HdfsBlobStore>() {
-                    @Override
-                    public HdfsBlobStore run() {
-                        try {
-                            FileContext fileContext = createContext(new URI("hdfs:///"));
-                            return new HdfsBlobStore(fileContext, "temp", 1024);
-                        } catch (IOException | URISyntaxException e) {
-                            throw new RuntimeException(e);
-                        }
-            }
-        });
-    }
-
-    public FileContext createContext(URI uri) {
-        // mirrors HdfsRepository.java behaviour
-        Configuration cfg = new Configuration(true);
-        cfg.setClassLoader(HdfsRepository.class.getClassLoader());
-        cfg.reloadConfiguration();
-
-        Constructor<?> ctor;
-        Subject subject;
-
-        try {
-            Class<?> clazz = Class.forName("org.apache.hadoop.security.User");
-            ctor = clazz.getConstructor(String.class);
-            ctor.setAccessible(true);
-        }  catch (ClassNotFoundException | NoSuchMethodException e) {
-            throw new RuntimeException(e);
-        }
-
-        try {
-            Principal principal = (Principal) ctor.newInstance(System.getProperty("user.name"));
-            subject = new Subject(false, Collections.singleton(principal),
-                    Collections.emptySet(), Collections.emptySet());
-        } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) {
-            throw new RuntimeException(e);
-        }
-
-        // disable file system cache
-        cfg.setBoolean("fs.hdfs.impl.disable.cache", true);
-
-        // set file system to TestingFs to avoid a bunch of security
-        // checks, similar to what is done in HdfsTests.java
-        cfg.set(String.format("fs.AbstractFileSystem.%s.impl", uri.getScheme()),
-                TestingFs.class.getName());
-
-        // create the FileContext with our user
-        return Subject.doAs(subject, new PrivilegedAction<FileContext>() {
-            @Override
-            public FileContext run() {
-                try {
-                    TestingFs fs = (TestingFs) AbstractFileSystem.get(uri, cfg);
-                    return FileContext.getFileContext(fs, cfg);
-                } catch (UnsupportedFileSystemException e) {
-                    throw new RuntimeException(e);
-                }
-            }
-        });
-    }
-}
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java
index 69955fb5460..7dc6f3b6a83 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java
@@ -118,10 +118,6 @@ public class S3BlobContainer extends AbstractBlobContainer {
 
     @Override
     public void deleteBlob(String blobName) throws IOException {
-        if (!blobExists(blobName)) {
-            throw new IOException("Blob [" + blobName + "] does not exist");
-        }
-
         try {
             blobStore.client().deleteObject(blobStore.bucket(), buildKey(blobName));
         } catch (AmazonClientException e) {
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/MockAmazonS3.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/MockAmazonS3.java
deleted file mode 100644
index 8124f693943..00000000000
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/MockAmazonS3.java
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.aws.blobstore;
-
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.AmazonServiceException;
-import com.amazonaws.services.s3.AbstractAmazonS3;
-import com.amazonaws.services.s3.model.AmazonS3Exception;
-import com.amazonaws.services.s3.model.CopyObjectRequest;
-import com.amazonaws.services.s3.model.CopyObjectResult;
-import com.amazonaws.services.s3.model.DeleteObjectRequest;
-import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.PutObjectResult;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.S3ObjectInputStream;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.amazonaws.util.Base64;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.security.DigestInputStream;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-class MockAmazonS3 extends AbstractAmazonS3 {
-
-    private Map<String, InputStream> blobs = new ConcurrentHashMap<>();
-
-    // in ESBlobStoreContainerTestCase.java, the maximum
-    // length of the input data is 100 bytes
-    private byte[] byteCounter = new byte[100];
-
-    @Override
-    public boolean doesBucketExist(String bucket) {
-        return true;
-    }
-
-    @Override
-    public ObjectMetadata getObjectMetadata(
-            GetObjectMetadataRequest getObjectMetadataRequest)
-            throws AmazonClientException, AmazonServiceException {
-        String blobName = getObjectMetadataRequest.getKey();
-
-        if (!blobs.containsKey(blobName)) {
-            throw new AmazonS3Exception("[" + blobName + "] does not exist.");
-        }
-
-        return new ObjectMetadata(); // nothing is done with it
-    }
-
-    @Override
-    public PutObjectResult putObject(PutObjectRequest putObjectRequest)
-            throws AmazonClientException, AmazonServiceException {
-        String blobName = putObjectRequest.getKey();
-        DigestInputStream stream = (DigestInputStream) putObjectRequest.getInputStream();
-
-        if (blobs.containsKey(blobName)) {
-            throw new AmazonS3Exception("[" + blobName + "] already exists.");
-        }
-
-        blobs.put(blobName, stream);
-
-        // input and output md5 hashes need to match to avoid an exception
-        String md5 = Base64.encodeAsString(stream.getMessageDigest().digest());
-        PutObjectResult result = new PutObjectResult();
-        result.setContentMd5(md5);
-
-        return result;
-    }
-
-    @Override
-    public S3Object getObject(GetObjectRequest getObjectRequest)
-            throws AmazonClientException, AmazonServiceException {
-        // in ESBlobStoreContainerTestCase.java, the prefix is empty,
-        // so the key and blobName are equivalent to each other
-        String blobName = getObjectRequest.getKey();
-
-        if (!blobs.containsKey(blobName)) {
-            throw new AmazonS3Exception("[" + blobName + "] does not exist.");
-        }
-
-        // the HTTP request attribute is irrelevant for reading
-        S3ObjectInputStream stream = new S3ObjectInputStream(
-                blobs.get(blobName), null, false);
-        S3Object s3Object = new S3Object();
-        s3Object.setObjectContent(stream);
-        return s3Object;
-    }
-
-    @Override
-    public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
-            throws AmazonClientException, AmazonServiceException {
-        MockObjectListing list = new MockObjectListing();
-        list.setTruncated(false);
-
-        String blobName;
-        String prefix = listObjectsRequest.getPrefix();
-
-        ArrayList<S3ObjectSummary> mockObjectSummaries = new ArrayList<>();
-
-        for (Map.Entry<String, InputStream> blob : blobs.entrySet()) {
-            blobName = blob.getKey();
-            S3ObjectSummary objectSummary = new S3ObjectSummary();
-
-            if (prefix.isEmpty() || blobName.startsWith(prefix)) {
-                objectSummary.setKey(blobName);
-
-                try {
-                    objectSummary.setSize(getSize(blob.getValue()));
-                } catch (IOException e) {
-                    throw new AmazonS3Exception("Object listing " +
-                            "failed for blob [" + blob.getKey() + "]");
-                }
-
-                mockObjectSummaries.add(objectSummary);
-            }
-        }
-
-        list.setObjectSummaries(mockObjectSummaries);
-        return list;
-    }
-
-    @Override
-    public CopyObjectResult copyObject(CopyObjectRequest copyObjectRequest)
-            throws AmazonClientException, AmazonServiceException {
-        String sourceBlobName = copyObjectRequest.getSourceKey();
-        String targetBlobName = copyObjectRequest.getDestinationKey();
-
-        if (!blobs.containsKey(sourceBlobName)) {
-            throw new AmazonS3Exception("Source blob [" +
-                    sourceBlobName + "] does not exist.");
-        }
-
-        if (blobs.containsKey(targetBlobName)) {
-            throw new AmazonS3Exception("Target blob [" +
-                    targetBlobName + "] already exists.");
-        }
-
-        blobs.put(targetBlobName, blobs.get(sourceBlobName));
-        return new CopyObjectResult(); // nothing is done with it
-    }
-
-    @Override
-    public void deleteObject(DeleteObjectRequest deleteObjectRequest)
-            throws AmazonClientException, AmazonServiceException {
-        String blobName = deleteObjectRequest.getKey();
-
-        if (!blobs.containsKey(blobName)) {
-            throw new AmazonS3Exception("[" + blobName + "] does not exist.");
-        }
-
-        blobs.remove(blobName);
-    }
-
-    private int getSize(InputStream stream) throws IOException {
-        int size = stream.read(byteCounter);
-        stream.reset(); // in case we ever need the size again
-        return size;
-    }
-
-    private class MockObjectListing extends ObjectListing {
-        // the objectSummaries attribute in ObjectListing.java
-        // is read-only, but we need to be able to write to it,
-        // so we create a mock of it to work around this
-        private List<S3ObjectSummary> mockObjectSummaries;
-
-        @Override
-        public List<S3ObjectSummary> getObjectSummaries() {
-            return mockObjectSummaries;
-        }
-
-        private void setObjectSummaries(List<S3ObjectSummary> objectSummaries) {
-            mockObjectSummaries = objectSummaries;
-        }
-    }
-}
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreContainerTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreContainerTests.java
deleted file mode 100644
index bca1c1d8a18..00000000000
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreContainerTests.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.aws.blobstore;
-
-import org.elasticsearch.common.blobstore.BlobStore;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.repositories.ESBlobStoreContainerTestCase;
-
-import java.io.IOException;
-import java.util.Locale;
-
-public class S3BlobStoreContainerTests extends ESBlobStoreContainerTestCase {
-    protected BlobStore newBlobStore() throws IOException {
-        MockAmazonS3 client = new MockAmazonS3();
-        String bucket = randomAsciiOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
-
-        return new S3BlobStore(Settings.EMPTY, client, bucket, null, false,
-            new ByteSizeValue(10, ByteSizeUnit.MB), 5, "public-read-write", "standard");
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java
index 67ad0eb7358..6ff0b71cdcc 100644
--- a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java
@@ -112,22 +112,6 @@ public abstract class ESBlobStoreContainerTestCase extends ESTestCase {
         }
     }
 
-    public void testDeleteBlob() throws IOException {
-        try (final BlobStore store = newBlobStore()) {
-            final String blobName = "foobar";
-            final BlobContainer container = store.blobContainer(new BlobPath());
-            expectThrows(IOException.class, () -> container.deleteBlob(blobName));
-
-            byte[] data = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16)));
-            final BytesArray bytesArray = new BytesArray(data);
-            container.writeBlob(blobName, bytesArray);
-            container.deleteBlob(blobName); // should not raise
-
-            // blob deleted, so should raise again
-            expectThrows(IOException.class, () -> container.deleteBlob(blobName));
-        }
-    }
-
     @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/15579")
     public void testOverwriteFails() throws IOException {
         try (final BlobStore store = newBlobStore()) {

From 62462f5d9b579c11144f5d402128776ab70d9be0 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 24 Jun 2016 12:11:48 +0200
Subject: [PATCH 1168/1311] [TEST] replace ResponseBodyAssertion with existing
 MatchAssertion

We introduced a special response_body assertion to test our docs snippets. The match assertion does the same job though and can be reused and adapted where needed. ResponseBodyAssertion contains provides much better and accurate errors though, which can be now utilized in MatchAssertion so that many more REST tests can benefit from readable error messages.

 Each response body gets always stashed and can be retrieved for later evaluations already. Instead of providing the response body as strings that get parsed to json objects separately, then converted to maps as ResponseBodyAssertion did, we parse everything once, the json is part of the yaml test, which is supported. The only downside is that json comments cannot be used, rather yaml comments should be used (// C style vs # ). There were only two docs tests that were using comments in ingest-node.asciidoc where I went ahead and remove the comments which didn't seem that useful anyways.
---
 .../doc/RestTestsFromSnippetsTask.groovy      |   5 +-
 docs/reference/ingest/ingest-node.asciidoc    |   2 -
 .../parser/RestTestSuiteParseContext.java     |  10 +-
 .../test/rest/section/MatchAssertion.java     | 132 +++++++++++++-
 .../rest/section/ResponseBodyAssertion.java   | 170 ------------------
 5 files changed, 131 insertions(+), 188 deletions(-)
 delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/section/ResponseBodyAssertion.java

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
index ba7311fee6f..c7f4316ee04 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy
@@ -131,8 +131,9 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
         }
 
         private void response(Snippet response) {
-            current.println("  - response_body: |")
-            response.contents.eachLine { current.println("      $it") }
+            current.println("  - match: ")
+            current.println("      \$body: ")
+            response.contents.eachLine { current.println("        $it") }
         }
 
         void emitDo(String method, String pathAndQuery,
diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc
index b03ed641de7..ec4f9c30e66 100644
--- a/docs/reference/ingest/ingest-node.asciidoc
+++ b/docs/reference/ingest/ingest-node.asciidoc
@@ -46,7 +46,6 @@ PUT _ingest/pipeline/my-pipeline-id
         "value": "bar"
       }
     }
-    // other processors
   ]
 }
 --------------------------------------------------
@@ -83,7 +82,6 @@ Example response:
             "value": "bar"
           }
         }
-        // other processors
       ]
     }
   } ]
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java
index e972aea641a..7edad9995bb 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java
@@ -18,22 +18,21 @@
  */
 package org.elasticsearch.test.rest.parser;
 
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.xcontent.XContentLocation;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.test.rest.section.DoSection;
 import org.elasticsearch.test.rest.section.ExecutableSection;
-import org.elasticsearch.test.rest.section.ResponseBodyAssertion;
 import org.elasticsearch.test.rest.section.SetupSection;
 import org.elasticsearch.test.rest.section.SkipSection;
 import org.elasticsearch.test.rest.section.TeardownSection;
 import org.elasticsearch.test.rest.section.TestSection;
 
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
 /**
  * Context shared across the whole tests parse phase.
  * Provides shared parse methods and holds information needed to parse the test sections (e.g. es version)
@@ -57,7 +56,6 @@ public class RestTestSuiteParseContext {
         EXECUTABLE_SECTIONS_PARSERS.put("lt", new LessThanParser());
         EXECUTABLE_SECTIONS_PARSERS.put("lte", new LessThanOrEqualToParser());
         EXECUTABLE_SECTIONS_PARSERS.put("length", new LengthParser());
-        EXECUTABLE_SECTIONS_PARSERS.put("response_body", ResponseBodyAssertion.PARSER);
     }
 
     private final String api;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/MatchAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/MatchAssertion.java
index e00fbbea01c..49b82dfafac 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/MatchAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/MatchAssertion.java
@@ -18,15 +18,21 @@
  */
 package org.elasticsearch.test.rest.section;
 
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.TreeMap;
 import java.util.regex.Pattern;
 
 import static org.elasticsearch.test.hamcrest.RegexMatcher.matches;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertThat;
 
 /**
@@ -45,7 +51,6 @@ public class MatchAssertion extends Assertion {
 
     @Override
     protected void doAssert(Object actualValue, Object expectedValue) {
-
         //if the value is wrapped into / it is a regexp (e.g. /s+d+/)
         if (expectedValue instanceof String) {
             String expValue = ((String) expectedValue).trim();
@@ -60,20 +65,131 @@ public class MatchAssertion extends Assertion {
             }
         }
 
-        assertThat(errorMessage(), actualValue, notNullValue());
+        assertNotNull("field [" + getField() + "] is null", actualValue);
         logger.trace("assert that [{}] matches [{}] (field [{}])", actualValue, expectedValue, getField());
-        if (!actualValue.getClass().equals(safeClass(expectedValue))) {
+        if (actualValue.getClass().equals(safeClass(expectedValue)) == false) {
             if (actualValue instanceof Number && expectedValue instanceof Number) {
                 //Double 1.0 is equal to Integer 1
-                assertThat(errorMessage(), ((Number) actualValue).doubleValue(), equalTo(((Number) expectedValue).doubleValue()));
+                assertThat("field [" + getField() + "] doesn't match the expected value",
+                        ((Number) actualValue).doubleValue(), equalTo(((Number) expectedValue).doubleValue()));
                 return;
             }
         }
 
-        assertThat(errorMessage(), actualValue, equalTo(expectedValue));
+        if (expectedValue.equals(actualValue) == false) {
+            FailureMessage message = new FailureMessage(getField());
+            message.compare(getField(), actualValue, expectedValue);
+            throw new AssertionError(message.message);
+        }
     }
 
-    private String errorMessage() {
-        return "field [" + getField() + "] doesn't match the expected value";
+    private static class FailureMessage {
+        private final StringBuilder message;
+        private int indent = 0;
+
+        private FailureMessage(String field) {
+            this.message = new StringBuilder(field + " didn't match the expected value:\n");
+        }
+
+        private void compareMaps(Map<String, Object> actual, Map<String, Object> expected) {
+            actual = new TreeMap<>(actual);
+            expected = new TreeMap<>(expected);
+            for (Map.Entry<String, Object> expectedEntry : expected.entrySet()) {
+                compare(expectedEntry.getKey(), actual.remove(expectedEntry.getKey()), expectedEntry.getValue());
+            }
+            for (Map.Entry<String, Object> unmatchedEntry : actual.entrySet()) {
+                field(unmatchedEntry.getKey(), "unexpected but found [" + unmatchedEntry.getValue() + "]");
+            }
+        }
+
+        private void compareLists(List<Object> actual, List<Object> expected) {
+            int i = 0;
+            while (i < actual.size() && i < expected.size()) {
+                compare(Integer.toString(i), actual.get(i), expected.get(i));
+                i++;
+            }
+            if (actual.size() == expected.size()) {
+                return;
+            }
+            indent();
+            if (actual.size() < expected.size()) {
+                message.append("expected [").append(expected.size() - i).append("] more entries\n");
+                return;
+            }
+            message.append("received [").append(actual.size() - i).append("] more entries than expected\n");
+        }
+
+        private void compare(String field, @Nullable Object actual, Object expected) {
+            if (expected instanceof Map) {
+                if (actual == null) {
+                    field(field, "expected map but not found");
+                    return;
+                }
+                if (false == actual instanceof Map) {
+                    field(field, "expected map but found [" + actual + "]");
+                    return;
+                }
+                @SuppressWarnings("unchecked")
+                Map<String, Object> expectedMap = (Map<String, Object>) expected;
+                @SuppressWarnings("unchecked")
+                Map<String, Object> actualMap = (Map<String, Object>) actual;
+                if (expectedMap.isEmpty() && actualMap.isEmpty()) {
+                    field(field, "same [empty map]");
+                    return;
+                }
+                field(field, null);
+                indent += 1;
+                compareMaps(actualMap, expectedMap);
+                indent -= 1;
+                return;
+            }
+            if (expected instanceof List) {
+                if (actual == null) {
+                    field(field, "expected list but not found");
+                    return;
+                }
+                if (false == actual instanceof List) {
+                    field(field, "expected list but found [" + actual + "]");
+                    return;
+                }
+                @SuppressWarnings("unchecked")
+                List<Object> expectedList = (List<Object>) expected;
+                @SuppressWarnings("unchecked")
+                List<Object> actualList = (List<Object>) actual;
+                if (expectedList.isEmpty() && actualList.isEmpty()) {
+                    field(field, "same [empty list]");
+                    return;
+                }
+                field(field, null);
+                indent += 1;
+                compareLists(actualList, expectedList);
+                indent -= 1;
+                return;
+            }
+            if (actual == null) {
+                field(field, "expected [" + expected + "] but not found");
+                return;
+            }
+            if (Objects.equals(expected, actual)) {
+                field(field, "same [" + expected + "]");
+                return;
+            }
+            field(field, "expected [" + expected + "] but was [" + actual + "]");
+        }
+
+        private void indent() {
+            for (int i = 0; i < indent; i++) {
+                message.append("  ");
+            }
+        }
+
+        private void field(Object name, String info) {
+            indent();
+            message.append(String.format(Locale.ROOT, "%30s: ", name));
+            if (info != null) {
+                message.append(info);
+            }
+            message.append('\n');
+        }
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/ResponseBodyAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/ResponseBodyAssertion.java
deleted file mode 100644
index 3ead65a2111..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/ResponseBodyAssertion.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.test.rest.section;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.TreeMap;
-
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.test.rest.parser.RestTestFragmentParser;
-import org.elasticsearch.test.rest.parser.RestTestParseException;
-import org.elasticsearch.test.rest.parser.RestTestSuiteParseContext;
-
-/**
- * Checks that the response body matches some text. 
- */
-public class ResponseBodyAssertion extends Assertion {
-    public static final RestTestFragmentParser<ResponseBodyAssertion> PARSER = new RestTestFragmentParser<ResponseBodyAssertion>() {
-        @Override
-        public ResponseBodyAssertion parse(RestTestSuiteParseContext parseContext) throws IOException, RestTestParseException {
-            try (XContentParser parser = JsonXContent.jsonXContent.createParser(parseContext.parseField())) {
-                return new ResponseBodyAssertion("$body", parser.map());
-            }
-        }
-    };
-
-    private ResponseBodyAssertion(String field, Map<String, Object> expectedValue) {
-        super(field, expectedValue);
-    }
-
-    @Override
-    protected void doAssert(Object actualValue, Object expectedValue) {
-        if (false == expectedValue.equals(actualValue)) {
-            @SuppressWarnings("unchecked")
-            Map<String, Object> actual = (Map<String, Object>) actualValue;
-            @SuppressWarnings("unchecked")
-            Map<String, Object> expected = (Map<String, Object>) expectedValue;
-            FailureMessage message = new FailureMessage();
-            message.compareMaps(actual, expected);
-            throw new AssertionError(message.message);
-        }
-    }
-
-    private class FailureMessage {
-        private final StringBuilder message = new StringBuilder("body didn't match the expected value:\n");
-        private int indent = 0;
-
-        private void compareMaps(Map<String, Object> actual, Map<String, Object> expected) {
-            actual = new TreeMap<>(actual);
-            expected = new TreeMap<>(expected);
-            for (Map.Entry<String, Object> expectedEntry : expected.entrySet()) {
-                compare(expectedEntry.getKey(), actual.remove(expectedEntry.getKey()), expectedEntry.getValue());
-            }
-            for (Map.Entry<String, Object> unmatchedEntry : actual.entrySet()) {
-                field(unmatchedEntry.getKey(), "unexpected but found [" + unmatchedEntry.getValue() + "]");
-            }
-        }
-
-        private void compareLists(List<Object> actual, List<Object> expected) {
-            int i = 0;
-            while (i < actual.size() && i < expected.size()) {
-                compare(i, actual.get(i), expected.get(i));
-                i++;
-            }
-            if (actual.size() == expected.size()) {
-                return;
-            }
-            indent();
-            if (actual.size() < expected.size()) {
-                message.append("expected [").append(expected.size() - i).append("] more entries\n");
-                return;
-            }
-            message.append("received [").append(actual.size() - i).append("] more entries than expected\n");
-        }
-
-        private void compare(Object field, @Nullable Object actual, Object expected) {
-            if (expected instanceof Map) {
-                if (actual == null) {
-                    field(field, "expected map but not found");
-                    return;
-                }
-                if (false == actual instanceof Map) {
-                    field(field, "expected map but found [" + actual + "]");
-                    return;
-                }
-                @SuppressWarnings("unchecked")
-                Map<String, Object> expectedMap = (Map<String, Object>) expected;
-                @SuppressWarnings("unchecked")
-                Map<String, Object> actualMap = (Map<String, Object>) actual;
-                if (expectedMap.isEmpty() && actualMap.isEmpty()) {
-                    field(field, "same [empty map]");
-                    return;
-                }
-                field(field, null);
-                indent += 1;
-                compareMaps(actualMap, expectedMap);
-                indent -= 1;
-                return;
-            }
-            if (expected instanceof List) {
-                if (actual == null) {
-                    field(field, "expected list but not found");
-                    return;
-                }
-                if (false == actual instanceof List) {
-                    field(field, "expected list but found [" + actual + "]");
-                    return;
-                }
-                @SuppressWarnings("unchecked")
-                List<Object> expectedList = (List<Object>) expected;
-                @SuppressWarnings("unchecked")
-                List<Object> actualList = (List<Object>) actual;
-                if (expectedList.isEmpty() && actualList.isEmpty()) {
-                    field(field, "same [empty list]");
-                    return;
-                }
-                field(field, null);
-                indent += 1;
-                compareLists(actualList, expectedList);
-                indent -= 1;
-                return;
-            }
-            if (actual == null) {
-                field(field, "expected [" + expected + "] but not found");
-                return;
-            }
-            if (Objects.equals(expected, actual)) {
-                field(field, "same [" + expected + "]");
-                return;
-            }
-            field(field, "expected [" + expected + "] but was [" + actual + "]");
-        }
-
-        private void indent() {
-            for (int i = 0; i < indent; i++) {
-                message.append("  ");
-            }
-        }
-
-        private void field(Object name, String info) {
-            indent();
-            message.append(String.format(Locale.ROOT, "%30s: ", name));
-            if (info != null) {
-                message.append(info);
-            }
-            message.append('\n');
-        }
-    }
-}

From 76199ce497f000c375f071d55e1e2dee99037f15 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 24 Jun 2016 13:08:21 +0200
Subject: [PATCH 1169/1311] [TEST] rename REST tests Stash methods to
 distinguish between retrieving a value and replacing values within a map

Stash#unstashMap -> replaceStashedValues
Stash#unstashValue -> getValue
---
 .../test/rest/RestTestExecutionContext.java   |  6 ++---
 .../org/elasticsearch/test/rest/Stash.java    | 24 ++++++++++---------
 .../test/rest/json/JsonPath.java              |  2 +-
 .../test/rest/section/Assertion.java          |  6 ++---
 4 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java
index 34397f03d94..d3acac8d79b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java
@@ -67,7 +67,7 @@ public class RestTestExecutionContext implements Closeable {
         HashMap<String, String> requestParams = new HashMap<>(params);
         for (Map.Entry<String, String> entry : requestParams.entrySet()) {
             if (stash.isStashedValue(entry.getValue())) {
-                entry.setValue(stash.unstashValue(entry.getValue()).toString());
+                entry.setValue(stash.getValue(entry.getValue()).toString());
             }
         }
 
@@ -90,12 +90,12 @@ public class RestTestExecutionContext implements Closeable {
         }
 
         if (bodies.size() == 1) {
-            return bodyAsString(stash.unstashMap(bodies.get(0)));
+            return bodyAsString(stash.replaceStashedValues(bodies.get(0)));
         }
 
         StringBuilder bodyBuilder = new StringBuilder();
         for (Map<String, Object> body : bodies) {
-            bodyBuilder.append(bodyAsString(stash.unstashMap(body))).append("\n");
+            bodyBuilder.append(bodyAsString(stash.replaceStashedValues(body))).append("\n");
         }
         return bodyBuilder.toString();
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java b/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
index 885df395c2b..3bac0ef4ff1 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
@@ -69,7 +69,8 @@ public class Stash implements ToXContent {
     }
 
     /**
-     * Tells whether a particular value needs to be looked up in the stash
+     * Tells whether a particular key needs to be looked up in the stash based on its name.
+     * Returns true if the string representation of the key starts with "$", false otherwise
      * The stash contains fields eventually extracted from previous responses that can be reused
      * as arguments for following requests (e.g. scroll_id)
      */
@@ -82,28 +83,29 @@ public class Stash implements ToXContent {
     }
 
     /**
-     * Extracts a value from the current stash
+     * Retrieves a value from the current stash.
      * The stash contains fields eventually extracted from previous responses that can be reused
      * as arguments for following requests (e.g. scroll_id)
      */
-    public Object unstashValue(String value) throws IOException {
-        if (value.startsWith("$body.")) {
+    public Object getValue(String key) throws IOException {
+        if (key.startsWith("$body.")) {
             if (response == null) {
                 return null;
             }
-            return response.evaluate(value.substring("$body".length()), this);
+            return response.evaluate(key.substring("$body".length()), this);
         }
-        Object stashedValue = stash.get(value.substring(1));
+        Object stashedValue = stash.get(key.substring(1));
         if (stashedValue == null) {
-            throw new IllegalArgumentException("stashed value not found for key [" + value + "]");
+            throw new IllegalArgumentException("stashed value not found for key [" + key + "]");
         }
         return stashedValue;
     }
 
     /**
-     * Recursively unstashes map values if needed
+     * Goes recursively against each map entry and replaces any string value starting with "$" with its
+     * corresponding value retrieved from the stash
      */
-    public Map<String, Object> unstashMap(Map<String, Object> map) throws IOException {
+    public Map<String, Object> replaceStashedValues(Map<String, Object> map) throws IOException {
         Map<String, Object> copy = new HashMap<>(map);
         unstashObject(copy);
         return copy;
@@ -116,7 +118,7 @@ public class Stash implements ToXContent {
             for (int i = 0; i < list.size(); i++) {
                 Object o = list.get(i);
                 if (isStashedValue(o)) {
-                    list.set(i, unstashValue(o.toString()));
+                    list.set(i, getValue(o.toString()));
                 } else {
                     unstashObject(o);
                 }
@@ -126,7 +128,7 @@ public class Stash implements ToXContent {
             Map<String, Object> map = (Map) obj;
             for (Map.Entry<String, Object> entry : map.entrySet()) {
                 if (isStashedValue(entry.getValue())) {
-                    entry.setValue(unstashValue(entry.getValue().toString()));
+                    entry.setValue(getValue(entry.getValue().toString()));
                 } else {
                     unstashObject(entry.getValue());
                 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/json/JsonPath.java b/test/framework/src/main/java/org/elasticsearch/test/rest/json/JsonPath.java
index b338d76d985..f12d5ad9ba9 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/json/JsonPath.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/json/JsonPath.java
@@ -71,7 +71,7 @@ public class JsonPath {
     @SuppressWarnings("unchecked")
     private Object evaluate(String key, Object object, Stash stash) throws IOException {
         if (stash.isStashedValue(key)) {
-            key = stash.unstashValue(key).toString();
+            key = stash.getValue(key).toString();
         }
 
         if (object instanceof Map) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/Assertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/Assertion.java
index c420309f206..de38940169f 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/Assertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/Assertion.java
@@ -48,18 +48,18 @@ public abstract class Assertion implements ExecutableSection {
         if (expectedValue instanceof Map) {
             @SuppressWarnings("unchecked")
             Map<String, Object> map = (Map<String, Object>) expectedValue;
-            return executionContext.stash().unstashMap(map);
+            return executionContext.stash().replaceStashedValues(map);
         }
 
         if (executionContext.stash().isStashedValue(expectedValue)) {
-            return executionContext.stash().unstashValue(expectedValue.toString());
+            return executionContext.stash().getValue(expectedValue.toString());
         }
         return expectedValue;
     }
 
     protected final Object getActualValue(RestTestExecutionContext executionContext) throws IOException {
         if (executionContext.stash().isStashedValue(field)) {
-            return executionContext.stash().unstashValue(field);
+            return executionContext.stash().getValue(field);
         }
         return executionContext.response(field);
     }

From 44dc801e90496fef6454fc0b9ccdd755fa07ca5b Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 24 Jun 2016 13:29:14 +0200
Subject: [PATCH 1170/1311] [TEST] make JsonPath independent of data format,
 rename to ObjectPath

The internal representation of the object that JsonPath gives access to is a map. That is independent of the initial input format, which is json but could also be yaml etc.
This commit renames JsonPath to ObjectPath and adds a static method to create an ObjectPath from an XContent
---
 .../resources/checkstyle_suppressions.xml     |  2 -
 .../{json/JsonPath.java => ObjectPath.java}   | 27 ++++-----
 .../test/rest/client/RestTestResponse.java    | 22 +++----
 ...sonPathTests.java => ObjectPathTests.java} | 58 +++++++++----------
 4 files changed, 52 insertions(+), 57 deletions(-)
 rename test/framework/src/main/java/org/elasticsearch/test/rest/{json/JsonPath.java => ObjectPath.java} (84%)
 rename test/framework/src/test/java/org/elasticsearch/test/rest/test/{JsonPathTests.java => ObjectPathTests.java} (74%)

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 51215ba6ce9..bb19be736d3 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -1263,7 +1263,6 @@
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]engine[/\\]MockEngineSupport.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]hamcrest[/\\]ElasticsearchAssertions.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]junit[/\\]listeners[/\\]LoggingListener.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]json[/\\]JsonPath.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]parser[/\\]GreaterThanEqualToParser.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]parser[/\\]GreaterThanParser.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]parser[/\\]LessThanOrEqualToParser.java" checks="LineLength" />
@@ -1281,7 +1280,6 @@
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]store[/\\]MockFSDirectoryService.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]store[/\\]MockFSIndexStore.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]test[/\\]FileUtilsTests.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]test[/\\]JsonPathTests.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]test[/\\]RestTestParserTests.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]test[/\\]InternalTestClusterTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]cli[/\\]CliTool.java" checks="LineLength" />
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/json/JsonPath.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java
similarity index 84%
rename from test/framework/src/main/java/org/elasticsearch/test/rest/json/JsonPath.java
rename to test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java
index f12d5ad9ba9..85ade469023 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/json/JsonPath.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java
@@ -16,11 +16,10 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.test.rest.json;
+package org.elasticsearch.test.rest;
 
+import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.test.rest.Stash;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -28,22 +27,20 @@ import java.util.List;
 import java.util.Map;
 
 /**
- * Holds a json object and allows to extract specific values from it
+ * Holds an object and allows to extract specific values from it given their path
  */
-public class JsonPath {
+public class ObjectPath {
 
-    final String json;
-    final Map<String, Object> jsonMap;
+    private final Map<String, Object> object;
 
-    public JsonPath(String json) throws IOException {
-        this.json = json;
-        this.jsonMap = convertToMap(json);
+    public static ObjectPath createFromXContent(String input) throws IOException {
+        try (XContentParser parser = XContentFactory.xContent(input).createParser(input)) {
+            return new ObjectPath(parser.mapOrdered());
+        }
     }
 
-    private static Map<String, Object> convertToMap(String json) throws IOException {
-        try (XContentParser parser = JsonXContent.jsonXContent.createParser(json)) {
-            return parser.mapOrdered();
-        }
+    public ObjectPath(Map<String, Object> map) throws IOException {
+        this.object = map;
     }
 
     /**
@@ -58,7 +55,7 @@ public class JsonPath {
      */
     public Object evaluate(String path, Stash stash) throws IOException {
         String[] parts = parsePath(path);
-        Object object = jsonMap;
+        Object object = this.object;
         for (String part : parts) {
             object = evaluate(part, object, stash);
             if (object == null) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java
index 5b5773d6fdc..7de312cccfe 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java
@@ -23,21 +23,21 @@ import org.apache.http.util.EntityUtils;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.test.rest.ObjectPath;
 import org.elasticsearch.test.rest.Stash;
-import org.elasticsearch.test.rest.json.JsonPath;
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 
 /**
  * Response obtained from a REST call, eagerly reads the response body into a string for later optional parsing.
- * Supports parsing the response body as json when needed and returning specific values extracted from it.
+ * Supports parsing the response body when needed and returning specific values extracted from it.
  */
 public class RestTestResponse {
 
     private final Response response;
     private final String body;
-    private JsonPath parsedResponse;
+    private ObjectPath parsedResponse;
 
     public RestTestResponse(Response response) {
         this.response = response;
@@ -74,7 +74,7 @@ public class RestTestResponse {
      */
     public Object getBody() throws IOException {
         if (isJson()) {
-            JsonPath parsedResponse = parsedResponse();
+            ObjectPath parsedResponse = parsedResponse();
             if (parsedResponse == null) {
                 return null;
             }
@@ -95,23 +95,23 @@ public class RestTestResponse {
     }
 
     /**
-     * Parses the response body as json and extracts a specific value from it (identified by the provided path)
+     * Parses the response body and extracts a specific value from it (identified by the provided path)
      */
     public Object evaluate(String path) throws IOException {
         return evaluate(path, Stash.EMPTY);
     }
 
     /**
-     * Parses the response body as json and extracts a specific value from it (identified by the provided path)
+     * Parses the response body and extracts a specific value from it (identified by the provided path)
      */
     public Object evaluate(String path, Stash stash) throws IOException {
         if (response == null) {
             return null;
         }
 
-        JsonPath jsonPath = parsedResponse();
+        ObjectPath objectPath = parsedResponse();
 
-        if (jsonPath == null) {
+        if (objectPath == null) {
             //special case: api that don't support body (e.g. exists) return true if 200, false if 404, even if no body
             //is_true: '' means the response had no body but the client returned true (caused by 200)
             //is_false: '' means the response had no body but the client returned false (caused by 404)
@@ -121,7 +121,7 @@ public class RestTestResponse {
             return null;
         }
 
-        return jsonPath.evaluate(path, stash);
+        return objectPath.evaluate(path, stash);
     }
 
     private boolean isJson() {
@@ -129,13 +129,13 @@ public class RestTestResponse {
         return contentType != null && contentType.contains("application/json");
     }
 
-    private JsonPath parsedResponse() throws IOException {
+    private ObjectPath parsedResponse() throws IOException {
         if (parsedResponse != null) {
             return parsedResponse;
         }
         if (response == null || body == null) {
             return null;
         }
-        return parsedResponse = new JsonPath(body);
+        return parsedResponse = ObjectPath.createFromXContent(body);
     }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/test/JsonPathTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java
similarity index 74%
rename from test/framework/src/test/java/org/elasticsearch/test/rest/test/JsonPathTests.java
rename to test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java
index fefcd57af79..fb8b8866d42 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/test/JsonPathTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java
@@ -20,7 +20,7 @@ package org.elasticsearch.test.rest.test;
 
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.rest.Stash;
-import org.elasticsearch.test.rest.json.JsonPath;
+import org.elasticsearch.test.rest.ObjectPath;
 
 import java.util.List;
 import java.util.Map;
@@ -33,59 +33,59 @@ import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
-public class JsonPathTests extends ESTestCase {
+public class ObjectPathTests extends ESTestCase {
     public void testEvaluateObjectPathEscape() throws Exception {
         String json = "{ \"field1\": { \"field2.field3\" : \"value2\" } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("field1.field2\\.field3");
+        ObjectPath objectPath = ObjectPath.createFromXContent(json);
+        Object object = objectPath.evaluate("field1.field2\\.field3");
         assertThat(object, instanceOf(String.class));
         assertThat((String)object, equalTo("value2"));
     }
 
     public void testEvaluateObjectPathWithDoubleDot() throws Exception {
         String json = "{ \"field1\": { \"field2\" : \"value2\" } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("field1..field2");
+        ObjectPath objectPath = ObjectPath.createFromXContent(json);
+        Object object = objectPath.evaluate("field1..field2");
         assertThat(object, instanceOf(String.class));
         assertThat((String)object, equalTo("value2"));
     }
 
     public void testEvaluateObjectPathEndsWithDot() throws Exception {
         String json = "{ \"field1\": { \"field2\" : \"value2\" } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("field1.field2.");
+        ObjectPath objectPath = ObjectPath.createFromXContent(json);
+        Object object = objectPath.evaluate("field1.field2.");
         assertThat(object, instanceOf(String.class));
         assertThat((String)object, equalTo("value2"));
     }
 
     public void testEvaluateString() throws Exception {
         String json = "{ \"field1\": { \"field2\" : \"value2\" } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("field1.field2");
+        ObjectPath objectPath = ObjectPath.createFromXContent(json);
+        Object object = objectPath.evaluate("field1.field2");
         assertThat(object, instanceOf(String.class));
         assertThat((String)object, equalTo("value2"));
     }
 
     public void testEvaluateInteger() throws Exception {
         String json = "{ \"field1\": { \"field2\" : 333 } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("field1.field2");
+        ObjectPath objectPath = ObjectPath.createFromXContent(json);
+        Object object = objectPath.evaluate("field1.field2");
         assertThat(object, instanceOf(Integer.class));
         assertThat((Integer)object, equalTo(333));
     }
 
     public void testEvaluateDouble() throws Exception {
         String json = "{ \"field1\": { \"field2\" : 3.55 } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("field1.field2");
+        ObjectPath objectPath = ObjectPath.createFromXContent(json);
+        Object object = objectPath.evaluate("field1.field2");
         assertThat(object, instanceOf(Double.class));
         assertThat((Double)object, equalTo(3.55));
     }
 
     public void testEvaluateArray() throws Exception {
         String json = "{ \"field1\": { \"array1\" : [ \"value1\", \"value2\" ] } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("field1.array1");
+        ObjectPath objectPath = ObjectPath.createFromXContent(json);
+        Object object = objectPath.evaluate("field1.array1");
         assertThat(object, instanceOf(List.class));
         List list = (List) object;
         assertThat(list.size(), equalTo(2));
@@ -97,32 +97,32 @@ public class JsonPathTests extends ESTestCase {
 
     public void testEvaluateArrayElement() throws Exception {
         String json = "{ \"field1\": { \"array1\" : [ \"value1\", \"value2\" ] } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("field1.array1.1");
+        ObjectPath objectPath = ObjectPath.createFromXContent(json);
+        Object object = objectPath.evaluate("field1.array1.1");
         assertThat(object, instanceOf(String.class));
         assertThat((String)object, equalTo("value2"));
     }
 
     public void testEvaluateArrayElementObject() throws Exception {
         String json = "{ \"field1\": { \"array1\" : [ {\"element\": \"value1\"}, {\"element\":\"value2\"} ] } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("field1.array1.1.element");
+        ObjectPath objectPath = ObjectPath.createFromXContent(json);
+        Object object = objectPath.evaluate("field1.array1.1.element");
         assertThat(object, instanceOf(String.class));
         assertThat((String)object, equalTo("value2"));
     }
 
     public void testEvaluateArrayElementObjectWrongPath() throws Exception {
         String json = "{ \"field1\": { \"array1\" : [ {\"element\": \"value1\"}, {\"element\":\"value2\"} ] } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("field1.array2.1.element");
+        ObjectPath objectPath = ObjectPath.createFromXContent(json);
+        Object object = objectPath.evaluate("field1.array2.1.element");
         assertThat(object, nullValue());
     }
 
     @SuppressWarnings("unchecked")
     public void testEvaluateObjectKeys() throws Exception {
         String json = "{ \"metadata\": { \"templates\" : {\"template_1\": { \"field\" : \"value\"}, \"template_2\": { \"field\" : \"value\"} } } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("metadata.templates");
+        ObjectPath objectPath = ObjectPath.createFromXContent(json);
+        Object object = objectPath.evaluate("metadata.templates");
         assertThat(object, instanceOf(Map.class));
         Map<String, Object> map = (Map<String, Object>)object;
         assertThat(map.size(), equalTo(2));
@@ -133,8 +133,8 @@ public class JsonPathTests extends ESTestCase {
     @SuppressWarnings("unchecked")
     public void testEvaluateEmptyPath() throws Exception {
         String json = "{ \"field1\": { \"array1\" : [ {\"element\": \"value1\"}, {\"element\":\"value2\"} ] } }";
-        JsonPath jsonPath = new JsonPath(json);
-        Object object = jsonPath.evaluate("");
+        ObjectPath objectPath = ObjectPath.createFromXContent(json);
+        Object object = objectPath.evaluate("");
         assertThat(object, notNullValue());
         assertThat(object, instanceOf(Map.class));
         assertThat(((Map<String, Object>)object).containsKey("field1"), equalTo(true));
@@ -142,9 +142,9 @@ public class JsonPathTests extends ESTestCase {
 
     public void testEvaluateStashInPropertyName() throws Exception {
         String json = "{ \"field1\": { \"elements\" : {\"element1\": \"value1\"}}}";
-        JsonPath jsonPath = new JsonPath(json);
+        ObjectPath objectPath = ObjectPath.createFromXContent(json);
         try {
-            jsonPath.evaluate("field1.$placeholder.element1");
+            objectPath.evaluate("field1.$placeholder.element1");
             fail("evaluate should have failed due to unresolved placeholder");
         } catch(IllegalArgumentException e) {
             assertThat(e.getMessage(), containsString("stashed value not found for key [$placeholder]"));
@@ -152,7 +152,7 @@ public class JsonPathTests extends ESTestCase {
 
         Stash stash = new Stash();
         stash.stashValue("placeholder", "elements");
-        Object object = jsonPath.evaluate("field1.$placeholder.element1", stash);
+        Object object = objectPath.evaluate("field1.$placeholder.element1", stash);
         assertThat(object, notNullValue());
         assertThat(object.toString(), equalTo("value1"));
     }

From bbaa23bdfdf534b9cdd2659633a5d5ebaa9fb7b8 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Sat, 25 Jun 2016 10:43:03 +0200
Subject: [PATCH 1171/1311] [TEST] extend ObjectPathTests to support also yaml
 format

---
 .../test/rest/test/ObjectPathTests.java       | 166 +++++++++++-------
 1 file changed, 104 insertions(+), 62 deletions(-)

diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java
index fb8b8866d42..f5616a957db 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java
@@ -18,10 +18,14 @@
  */
 package org.elasticsearch.test.rest.test;
 
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.rest.Stash;
 import org.elasticsearch.test.rest.ObjectPath;
+import org.elasticsearch.test.rest.Stash;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -34,94 +38,135 @@ import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
 public class ObjectPathTests extends ESTestCase {
+
+    private static XContentBuilder randomXContentBuilder() throws IOException {
+        //only string based formats are supported, no cbor nor smile
+        XContentType xContentType = randomFrom(XContentType.JSON, XContentType.YAML);
+        return XContentBuilder.builder(XContentFactory.xContent(xContentType));
+    }
+
     public void testEvaluateObjectPathEscape() throws Exception {
-        String json = "{ \"field1\": { \"field2.field3\" : \"value2\" } }";
-        ObjectPath objectPath = ObjectPath.createFromXContent(json);
+        XContentBuilder xContentBuilder = randomXContentBuilder();
+        xContentBuilder.startObject();
+        xContentBuilder.startObject("field1");
+        xContentBuilder.field("field2.field3", "value2");
+        xContentBuilder.endObject();
+        xContentBuilder.endObject();
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.string());
         Object object = objectPath.evaluate("field1.field2\\.field3");
         assertThat(object, instanceOf(String.class));
-        assertThat((String)object, equalTo("value2"));
+        assertThat(object, equalTo("value2"));
     }
 
-    public void testEvaluateObjectPathWithDoubleDot() throws Exception {
-        String json = "{ \"field1\": { \"field2\" : \"value2\" } }";
-        ObjectPath objectPath = ObjectPath.createFromXContent(json);
+    public void testEvaluateObjectPathWithDots() throws Exception {
+        XContentBuilder xContentBuilder = randomXContentBuilder();
+        xContentBuilder.startObject();
+        xContentBuilder.startObject("field1");
+        xContentBuilder.field("field2", "value2");
+        xContentBuilder.endObject();
+        xContentBuilder.endObject();
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.string());
         Object object = objectPath.evaluate("field1..field2");
         assertThat(object, instanceOf(String.class));
-        assertThat((String)object, equalTo("value2"));
-    }
-
-    public void testEvaluateObjectPathEndsWithDot() throws Exception {
-        String json = "{ \"field1\": { \"field2\" : \"value2\" } }";
-        ObjectPath objectPath = ObjectPath.createFromXContent(json);
-        Object object = objectPath.evaluate("field1.field2.");
+        assertThat(object, equalTo("value2"));
+        object = objectPath.evaluate("field1.field2.");
         assertThat(object, instanceOf(String.class));
-        assertThat((String)object, equalTo("value2"));
-    }
-
-    public void testEvaluateString() throws Exception {
-        String json = "{ \"field1\": { \"field2\" : \"value2\" } }";
-        ObjectPath objectPath = ObjectPath.createFromXContent(json);
-        Object object = objectPath.evaluate("field1.field2");
+        assertThat(object, equalTo("value2"));
+        object = objectPath.evaluate("field1.field2");
         assertThat(object, instanceOf(String.class));
-        assertThat((String)object, equalTo("value2"));
+        assertThat(object, equalTo("value2"));
     }
 
     public void testEvaluateInteger() throws Exception {
-        String json = "{ \"field1\": { \"field2\" : 333 } }";
-        ObjectPath objectPath = ObjectPath.createFromXContent(json);
+        XContentBuilder xContentBuilder = randomXContentBuilder();
+        xContentBuilder.startObject();
+        xContentBuilder.startObject("field1");
+        xContentBuilder.field("field2", 333);
+        xContentBuilder.endObject();
+        xContentBuilder.endObject();
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.string());
         Object object = objectPath.evaluate("field1.field2");
         assertThat(object, instanceOf(Integer.class));
-        assertThat((Integer)object, equalTo(333));
+        assertThat(object, equalTo(333));
     }
 
     public void testEvaluateDouble() throws Exception {
-        String json = "{ \"field1\": { \"field2\" : 3.55 } }";
-        ObjectPath objectPath = ObjectPath.createFromXContent(json);
+        XContentBuilder xContentBuilder = randomXContentBuilder();
+        xContentBuilder.startObject();
+        xContentBuilder.startObject("field1");
+        xContentBuilder.field("field2", 3.55);
+        xContentBuilder.endObject();
+        xContentBuilder.endObject();
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.string());
         Object object = objectPath.evaluate("field1.field2");
         assertThat(object, instanceOf(Double.class));
-        assertThat((Double)object, equalTo(3.55));
+        assertThat(object, equalTo(3.55));
     }
 
     public void testEvaluateArray() throws Exception {
-        String json = "{ \"field1\": { \"array1\" : [ \"value1\", \"value2\" ] } }";
-        ObjectPath objectPath = ObjectPath.createFromXContent(json);
+        XContentBuilder xContentBuilder = randomXContentBuilder();
+        xContentBuilder.startObject();
+        xContentBuilder.startObject("field1");
+        xContentBuilder.array("array1", "value1", "value2");
+        xContentBuilder.endObject();
+        xContentBuilder.endObject();
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.string());
         Object object = objectPath.evaluate("field1.array1");
         assertThat(object, instanceOf(List.class));
         List list = (List) object;
         assertThat(list.size(), equalTo(2));
         assertThat(list.get(0), instanceOf(String.class));
-        assertThat((String)list.get(0), equalTo("value1"));
+        assertThat(list.get(0), equalTo("value1"));
         assertThat(list.get(1), instanceOf(String.class));
-        assertThat((String)list.get(1), equalTo("value2"));
-    }
-
-    public void testEvaluateArrayElement() throws Exception {
-        String json = "{ \"field1\": { \"array1\" : [ \"value1\", \"value2\" ] } }";
-        ObjectPath objectPath = ObjectPath.createFromXContent(json);
-        Object object = objectPath.evaluate("field1.array1.1");
+        assertThat(list.get(1), equalTo("value2"));
+        object = objectPath.evaluate("field1.array1.1");
         assertThat(object, instanceOf(String.class));
-        assertThat((String)object, equalTo("value2"));
+        assertThat(object, equalTo("value2"));
     }
 
+    @SuppressWarnings("unchecked")
     public void testEvaluateArrayElementObject() throws Exception {
-        String json = "{ \"field1\": { \"array1\" : [ {\"element\": \"value1\"}, {\"element\":\"value2\"} ] } }";
-        ObjectPath objectPath = ObjectPath.createFromXContent(json);
+        XContentBuilder xContentBuilder = randomXContentBuilder();
+        xContentBuilder.startObject();
+        xContentBuilder.startObject("field1");
+        xContentBuilder.startArray("array1");
+        xContentBuilder.startObject();
+        xContentBuilder.field("element", "value1");
+        xContentBuilder.endObject();
+        xContentBuilder.startObject();
+        xContentBuilder.field("element", "value2");
+        xContentBuilder.endObject();
+        xContentBuilder.endArray();
+        xContentBuilder.endObject();
+        xContentBuilder.endObject();
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.string());
         Object object = objectPath.evaluate("field1.array1.1.element");
         assertThat(object, instanceOf(String.class));
-        assertThat((String)object, equalTo("value2"));
-    }
-
-    public void testEvaluateArrayElementObjectWrongPath() throws Exception {
-        String json = "{ \"field1\": { \"array1\" : [ {\"element\": \"value1\"}, {\"element\":\"value2\"} ] } }";
-        ObjectPath objectPath = ObjectPath.createFromXContent(json);
-        Object object = objectPath.evaluate("field1.array2.1.element");
+        assertThat(object, equalTo("value2"));
+        object = objectPath.evaluate("");
+        assertThat(object, notNullValue());
+        assertThat(object, instanceOf(Map.class));
+        assertThat(((Map<String, Object>)object).containsKey("field1"), equalTo(true));
+        object = objectPath.evaluate("field1.array2.1.element");
         assertThat(object, nullValue());
     }
 
     @SuppressWarnings("unchecked")
     public void testEvaluateObjectKeys() throws Exception {
-        String json = "{ \"metadata\": { \"templates\" : {\"template_1\": { \"field\" : \"value\"}, \"template_2\": { \"field\" : \"value\"} } } }";
-        ObjectPath objectPath = ObjectPath.createFromXContent(json);
+        XContentBuilder xContentBuilder = randomXContentBuilder();
+        xContentBuilder.startObject();
+        xContentBuilder.startObject("metadata");
+        xContentBuilder.startObject("templates");
+        xContentBuilder.startObject("template_1");
+        xContentBuilder.field("field", "value");
+        xContentBuilder.endObject();
+        xContentBuilder.startObject("template_2");
+        xContentBuilder.field("field", "value");
+        xContentBuilder.endObject();
+        xContentBuilder.endObject();
+        xContentBuilder.endObject();
+        xContentBuilder.endObject();
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.string());
         Object object = objectPath.evaluate("metadata.templates");
         assertThat(object, instanceOf(Map.class));
         Map<String, Object> map = (Map<String, Object>)object;
@@ -130,19 +175,16 @@ public class ObjectPathTests extends ESTestCase {
         assertThat(strings, contains("template_1", "template_2"));
     }
 
-    @SuppressWarnings("unchecked")
-    public void testEvaluateEmptyPath() throws Exception {
-        String json = "{ \"field1\": { \"array1\" : [ {\"element\": \"value1\"}, {\"element\":\"value2\"} ] } }";
-        ObjectPath objectPath = ObjectPath.createFromXContent(json);
-        Object object = objectPath.evaluate("");
-        assertThat(object, notNullValue());
-        assertThat(object, instanceOf(Map.class));
-        assertThat(((Map<String, Object>)object).containsKey("field1"), equalTo(true));
-    }
-
     public void testEvaluateStashInPropertyName() throws Exception {
-        String json = "{ \"field1\": { \"elements\" : {\"element1\": \"value1\"}}}";
-        ObjectPath objectPath = ObjectPath.createFromXContent(json);
+        XContentBuilder xContentBuilder = randomXContentBuilder();
+        xContentBuilder.startObject();
+        xContentBuilder.startObject("field1");
+        xContentBuilder.startObject("elements");
+        xContentBuilder.field("element1", "value1");
+        xContentBuilder.endObject();
+        xContentBuilder.endObject();
+        xContentBuilder.endObject();
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.string());
         try {
             objectPath.evaluate("field1.$placeholder.element1");
             fail("evaluate should have failed due to unresolved placeholder");

From d5df738538cf438d45b155e78819e43efbd1fc81 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Sat, 25 Jun 2016 15:35:20 +0200
Subject: [PATCH 1172/1311] [TEST] ObjectPath to support parsing yaml or json
 that have an array as root object

ObjectPath used a Map up until now for the internal representation of its navigable object. That works in most of the cases, but there could also be an array as root object, in which case a List needs to be used instead of a Map. This commit changes the internal representation of the object to Object which can either be a List or a Map. The change is minimal as ObjectPath already had the checks in place to verify the type of the object in the current position and navigate through it.

  Note: The new test added to ObjectPathTest uses yaml format explicitly as auto-detection of json format works only for a json object that starts with '{', not if the root object is actually an array and starts with '['.
---
 .../elasticsearch/test/rest/ObjectPath.java   |  9 ++++---
 .../test/rest/test/ObjectPathTests.java       | 24 +++++++++++++++++++
 2 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java
index 85ade469023..36755083176 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java
@@ -31,16 +31,19 @@ import java.util.Map;
  */
 public class ObjectPath {
 
-    private final Map<String, Object> object;
+    private final Object object;
 
     public static ObjectPath createFromXContent(String input) throws IOException {
         try (XContentParser parser = XContentFactory.xContent(input).createParser(input)) {
+            if (parser.nextToken() == XContentParser.Token.START_ARRAY) {
+                return new ObjectPath(parser.listOrderedMap());
+            }
             return new ObjectPath(parser.mapOrdered());
         }
     }
 
-    public ObjectPath(Map<String, Object> map) throws IOException {
-        this.object = map;
+    public ObjectPath(Object object) throws IOException {
+        this.object = object;
     }
 
     /**
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java
index f5616a957db..26dc5f928a1 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java
@@ -198,4 +198,28 @@ public class ObjectPathTests extends ESTestCase {
         assertThat(object, notNullValue());
         assertThat(object.toString(), equalTo("value1"));
     }
+
+    @SuppressWarnings("unchecked")
+    public void testEvaluateArrayAsRoot() throws Exception {
+        String input = "---\n" +
+                "- alias: \"test_alias1\"\n" +
+                "  index: \"test1\"\n" +
+                "- alias: \"test_alias2\"\n" +
+                "  index: \"test2\"";
+
+        ObjectPath objectPath = ObjectPath.createFromXContent(input);
+        Object object = objectPath.evaluate("");
+        assertThat(object, notNullValue());
+        assertThat(object, instanceOf(List.class));
+        assertThat(((List<Object>)object).size(), equalTo(2));
+        object = objectPath.evaluate("0");
+        assertThat(object, notNullValue());
+        assertThat(object, instanceOf(Map.class));
+        assertThat(((Map<String, Object>)object).get("alias"), equalTo("test_alias1"));
+        object = objectPath.evaluate("1.index");
+        assertThat(object, notNullValue());
+        assertThat(object, instanceOf(String.class));
+        assertThat(object, equalTo("test2"));
+    }
+
 }

From 34f5c50a7f9649ef73a94b740a016d63b1a8a33d Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Sat, 25 Jun 2016 15:40:41 +0200
Subject: [PATCH 1173/1311] [TEST] eagerly parse response body at ObjectPath
 initialization and read content type from response headers

We are going to parse the body anyways whenever it's in json format as it is going to be stashed. It is not useful to lazily parse it anymore. Also this allows us to not rely on automatic detection of the xcontent type based on the content of the response, but rather read the content type from the response headers.
---
 .../elasticsearch/test/rest/ObjectPath.java   |  6 +--
 .../test/rest/client/RestTestResponse.java    | 45 ++++++++-----------
 .../test/rest/test/ObjectPathTests.java       | 36 ++++++++-------
 3 files changed, 42 insertions(+), 45 deletions(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java
index 36755083176..da27c3864fc 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java
@@ -18,7 +18,7 @@
  */
 package org.elasticsearch.test.rest;
 
-import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
@@ -33,8 +33,8 @@ public class ObjectPath {
 
     private final Object object;
 
-    public static ObjectPath createFromXContent(String input) throws IOException {
-        try (XContentParser parser = XContentFactory.xContent(input).createParser(input)) {
+    public static ObjectPath createFromXContent(XContent xContent, String input) throws IOException {
+        try (XContentParser parser = xContent.createParser(input)) {
             if (parser.nextToken() == XContentParser.Token.START_ARRAY) {
                 return new ObjectPath(parser.listOrderedMap());
             }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java
index 7de312cccfe..c2d7e141b44 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java
@@ -23,6 +23,7 @@ import org.apache.http.util.EntityUtils;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.rest.ObjectPath;
 import org.elasticsearch.test.rest.Stash;
 
@@ -39,7 +40,7 @@ public class RestTestResponse {
     private final String body;
     private ObjectPath parsedResponse;
 
-    public RestTestResponse(Response response) {
+    public RestTestResponse(Response response) throws IOException {
         this.response = response;
         if (response.getEntity() != null) {
             try {
@@ -53,11 +54,24 @@ public class RestTestResponse {
         } else {
             this.body = null;
         }
+        parseResponseBody();
     }
 
-    public RestTestResponse(ResponseException responseException) {
+    public RestTestResponse(ResponseException responseException) throws IOException {
         this.response = responseException.getResponse();
         this.body = responseException.getResponseBody();
+        parseResponseBody();
+    }
+
+    private void parseResponseBody() throws IOException {
+        if (body != null) {
+            String contentType = response.getHeader("Content-Type");
+            XContentType xContentType = XContentType.fromMediaTypeOrFormat(contentType);
+            //skip parsing if we got text back (e.g. if we called _cat apis)
+            if (xContentType == XContentType.JSON) {
+                this.parsedResponse = ObjectPath.createFromXContent(xContentType.xContent(), body);
+            }
+        }
     }
 
     public int getStatusCode() {
@@ -73,11 +87,7 @@ public class RestTestResponse {
      * Might be a string or a json object parsed as a map.
      */
     public Object getBody() throws IOException {
-        if (isJson()) {
-            ObjectPath parsedResponse = parsedResponse();
-            if (parsedResponse == null) {
-                return null;
-            }
+        if (parsedResponse != null) {
             return parsedResponse.evaluate("");
         }
         return body;
@@ -109,9 +119,7 @@ public class RestTestResponse {
             return null;
         }
 
-        ObjectPath objectPath = parsedResponse();
-
-        if (objectPath == null) {
+        if (parsedResponse == null) {
             //special case: api that don't support body (e.g. exists) return true if 200, false if 404, even if no body
             //is_true: '' means the response had no body but the client returned true (caused by 200)
             //is_false: '' means the response had no body but the client returned false (caused by 404)
@@ -121,21 +129,6 @@ public class RestTestResponse {
             return null;
         }
 
-        return objectPath.evaluate(path, stash);
-    }
-
-    private boolean isJson() {
-        String contentType = response.getHeader("Content-Type");
-        return contentType != null && contentType.contains("application/json");
-    }
-
-    private ObjectPath parsedResponse() throws IOException {
-        if (parsedResponse != null) {
-            return parsedResponse;
-        }
-        if (response == null || body == null) {
-            return null;
-        }
-        return parsedResponse = ObjectPath.createFromXContent(body);
+        return parsedResponse.evaluate(path, stash);
     }
 }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java
index 26dc5f928a1..ac3a755d344 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java
@@ -52,7 +52,7 @@ public class ObjectPathTests extends ESTestCase {
         xContentBuilder.field("field2.field3", "value2");
         xContentBuilder.endObject();
         xContentBuilder.endObject();
-        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.string());
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
         Object object = objectPath.evaluate("field1.field2\\.field3");
         assertThat(object, instanceOf(String.class));
         assertThat(object, equalTo("value2"));
@@ -65,7 +65,7 @@ public class ObjectPathTests extends ESTestCase {
         xContentBuilder.field("field2", "value2");
         xContentBuilder.endObject();
         xContentBuilder.endObject();
-        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.string());
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
         Object object = objectPath.evaluate("field1..field2");
         assertThat(object, instanceOf(String.class));
         assertThat(object, equalTo("value2"));
@@ -84,7 +84,7 @@ public class ObjectPathTests extends ESTestCase {
         xContentBuilder.field("field2", 333);
         xContentBuilder.endObject();
         xContentBuilder.endObject();
-        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.string());
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
         Object object = objectPath.evaluate("field1.field2");
         assertThat(object, instanceOf(Integer.class));
         assertThat(object, equalTo(333));
@@ -97,7 +97,7 @@ public class ObjectPathTests extends ESTestCase {
         xContentBuilder.field("field2", 3.55);
         xContentBuilder.endObject();
         xContentBuilder.endObject();
-        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.string());
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
         Object object = objectPath.evaluate("field1.field2");
         assertThat(object, instanceOf(Double.class));
         assertThat(object, equalTo(3.55));
@@ -110,7 +110,7 @@ public class ObjectPathTests extends ESTestCase {
         xContentBuilder.array("array1", "value1", "value2");
         xContentBuilder.endObject();
         xContentBuilder.endObject();
-        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.string());
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
         Object object = objectPath.evaluate("field1.array1");
         assertThat(object, instanceOf(List.class));
         List list = (List) object;
@@ -139,7 +139,7 @@ public class ObjectPathTests extends ESTestCase {
         xContentBuilder.endArray();
         xContentBuilder.endObject();
         xContentBuilder.endObject();
-        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.string());
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
         Object object = objectPath.evaluate("field1.array1.1.element");
         assertThat(object, instanceOf(String.class));
         assertThat(object, equalTo("value2"));
@@ -166,7 +166,7 @@ public class ObjectPathTests extends ESTestCase {
         xContentBuilder.endObject();
         xContentBuilder.endObject();
         xContentBuilder.endObject();
-        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.string());
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
         Object object = objectPath.evaluate("metadata.templates");
         assertThat(object, instanceOf(Map.class));
         Map<String, Object> map = (Map<String, Object>)object;
@@ -184,7 +184,7 @@ public class ObjectPathTests extends ESTestCase {
         xContentBuilder.endObject();
         xContentBuilder.endObject();
         xContentBuilder.endObject();
-        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.string());
+        ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string());
         try {
             objectPath.evaluate("field1.$placeholder.element1");
             fail("evaluate should have failed due to unresolved placeholder");
@@ -201,13 +201,18 @@ public class ObjectPathTests extends ESTestCase {
 
     @SuppressWarnings("unchecked")
     public void testEvaluateArrayAsRoot() throws Exception {
-        String input = "---\n" +
-                "- alias: \"test_alias1\"\n" +
-                "  index: \"test1\"\n" +
-                "- alias: \"test_alias2\"\n" +
-                "  index: \"test2\"";
-
-        ObjectPath objectPath = ObjectPath.createFromXContent(input);
+        XContentBuilder xContentBuilder = randomXContentBuilder();
+        xContentBuilder.startArray();
+        xContentBuilder.startObject();
+        xContentBuilder.field("alias", "test_alias1");
+        xContentBuilder.field("index", "test1");
+        xContentBuilder.endObject();
+        xContentBuilder.startObject();
+        xContentBuilder.field("alias", "test_alias2");
+        xContentBuilder.field("index", "test2");
+        xContentBuilder.endObject();
+        xContentBuilder.endArray();
+        ObjectPath objectPath = ObjectPath.createFromXContent(XContentFactory.xContent(XContentType.YAML), xContentBuilder.string());
         Object object = objectPath.evaluate("");
         assertThat(object, notNullValue());
         assertThat(object, instanceOf(List.class));
@@ -221,5 +226,4 @@ public class ObjectPathTests extends ESTestCase {
         assertThat(object, instanceOf(String.class));
         assertThat(object, equalTo("test2"));
     }
-
 }

From 60bafa5d784ab4d03c222a28cd2bc3367fddf091 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Sun, 26 Jun 2016 00:04:44 +0200
Subject: [PATCH 1174/1311] [TEST] parse yaml responses too through ObjectPath
 rather than only json responses

No need to match against yaml responses via regexes in REST tests, yaml responses can be properly parsed via ObjectPath instead. Few REST tests need to be updated accordingly.
---
 .../test/cat.aliases/20_headers.yaml             | 14 +++++---------
 .../rest-api-spec/test/cat.aliases/30_yaml.yaml  | 15 +++++----------
 .../rest-api-spec/test/get/50_with_headers.yaml  | 16 ++++++----------
 .../test/rest/client/RestTestResponse.java       |  2 +-
 4 files changed, 17 insertions(+), 30 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/20_headers.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/20_headers.yaml
index b814856144b..16260151a31 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/20_headers.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/20_headers.yaml
@@ -17,12 +17,8 @@
       headers:
         Accept: application/yaml
 
-  - match:
-      $body: |
-        /^---\n
-          -\s+alias:\s+"test_alias"\s+
-              index:\s+"test"\s+
-              filter:\s+"-"\s+
-              routing.index:\s+"-"\s+
-              routing.search:\s+"-"\s+$/
-
+  - match: {0.alias: test_alias}
+  - match: {0.index: test}
+  - match: {0.filter: "-"}
+  - match: {0.routing\.index: "-"}
+  - match: {0.routing\.search: "-"}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/30_yaml.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/30_yaml.yaml
index c892891f08f..99b015f794b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/30_yaml.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/30_yaml.yaml
@@ -17,13 +17,8 @@
       cat.aliases:
         format: yaml
 
-  - match:
-      $body: |
-        /^---\n
-          -\s+alias:\s+"test_alias"\s+
-              index:\s+"test"\s+
-              filter:\s+"-"\s+
-              routing.index:\s+"-"\s+
-              routing.search:\s+"-"\s+$/
-
-
+  - match: {0.alias: test_alias}
+  - match: {0.index: test}
+  - match: {0.filter: "-"}
+  - match: {0.routing\.index: "-"}
+  - match: {0.routing\.search: "-"}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get/50_with_headers.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/get/50_with_headers.yaml
index 1bb031f0878..ed9a5b7c99f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/get/50_with_headers.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get/50_with_headers.yaml
@@ -18,13 +18,9 @@
         type: _all
         id:   1
 
-  - match:
-      $body: |
-        /^---\n
-          _index:\s+\"test_1"\n
-          _type:\s+"test"\n
-          _id:\s+"1"\n
-          _version:\s+1\n
-          found:\s+true\n
-          _source:\n
-          \s+body:\s+"foo"\n$/
+  - match: {_index: "test_1"}
+  - match: {_type: "test"}
+  - match: {_id: "1"}
+  - match: {_version: 1}
+  - match: {found: true}
+  - match: { _source: { body: foo }}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java
index c2d7e141b44..4644b87b8e7 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java
@@ -68,7 +68,7 @@ public class RestTestResponse {
             String contentType = response.getHeader("Content-Type");
             XContentType xContentType = XContentType.fromMediaTypeOrFormat(contentType);
             //skip parsing if we got text back (e.g. if we called _cat apis)
-            if (xContentType == XContentType.JSON) {
+            if (xContentType == XContentType.JSON || xContentType == XContentType.YAML) {
                 this.parsedResponse = ObjectPath.createFromXContent(xContentType.xContent(), body);
             }
         }

From 43b82ce2444ba2d656ee49448a8d68fd4a6245ff Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Mon, 27 Jun 2016 11:12:09 +0200
Subject: [PATCH 1175/1311] [TEST] remove feature yaml from REST tests

The only runner that supported it was the java runner, we can use json format instead given that the default one with cat apis is text
---
 .../resources/rest-api-spec/test/cat.aliases/30_yaml.yaml  | 7 ++-----
 .../java/org/elasticsearch/test/rest/support/Features.java | 2 +-
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/30_yaml.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/30_yaml.yaml
index 99b015f794b..178b77ce60d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/30_yaml.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/30_yaml.yaml
@@ -1,8 +1,5 @@
 ---
-"Simple alias with yaml body through format argument":
-
-  - skip:
-      features: yaml
+"Simple alias with json body through format argument":
 
   - do:
         indices.create:
@@ -15,7 +12,7 @@
 
   - do:
       cat.aliases:
-        format: yaml
+        format: json
 
   - match: {0.alias: test_alias}
   - match: {0.index: test}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java b/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java
index 66ba1528b90..0f51f72e8e5 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java
@@ -34,7 +34,7 @@ import java.util.List;
  */
 public final class Features {
 
-    private static final List<String> SUPPORTED = Arrays.asList("stash_in_path", "groovy_scripting", "headers", "yaml");
+    private static final List<String> SUPPORTED = Arrays.asList("stash_in_path", "groovy_scripting", "headers");
 
     private Features() {
 

From 0b5a5493053cba3b5084f5b44bf5886919ba82c8 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Mon, 27 Jun 2016 11:03:19 +0200
Subject: [PATCH 1176/1311] [TEST] remove special treatment for stashed $body
 in REST tests, instead always evaluate the stash through ObjectPath

When we introduced docs testing we added a special case for $body in Stash, so that the last stashed body could be evaluated, and expressions like "$body.took" could be extracted out of it. We can instead do that for any object in the stash, by simply wrapping the internal map in an ObjectPath instance. We can then drop the special stashResponse method and go back to using the ordinary stashValue too.

The downside of this change is that it adds a feature that may not be supported by other REST test runners, namely the evaluation of compouned paths from the stash. If we have "object" stashed as an object, it is now possible to extract directly each subobject of it as well e.g. "object.subobject.field1". None of the current REST tests rely on this, but our docs snippets tests do.
---
 .../org/elasticsearch/test/rest/ObjectPath.java |  2 +-
 .../test/rest/RestTestExecutionContext.java     |  2 +-
 .../java/org/elasticsearch/test/rest/Stash.java | 17 ++---------------
 .../test/rest/test/ObjectPathTests.java         |  8 ++++++++
 4 files changed, 12 insertions(+), 17 deletions(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java
index da27c3864fc..5ee1fbf521a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java
@@ -42,7 +42,7 @@ public class ObjectPath {
         }
     }
 
-    public ObjectPath(Object object) throws IOException {
+    public ObjectPath(Object object) {
         this.object = object;
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java
index d3acac8d79b..6d95c7e893f 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java
@@ -76,7 +76,7 @@ public class RestTestExecutionContext implements Closeable {
         try {
             response = callApiInternal(apiName, requestParams, body, headers);
             //we always stash the last response body
-            stash.stashResponse(response);
+            stash.stashValue("body", response.getBody());
             return response;
         } catch(ResponseException e) {
             response = new RestTestResponse(e);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java b/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
index 3bac0ef4ff1..13c5b9ff563 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
@@ -24,7 +24,6 @@ import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.test.rest.client.RestTestResponse;
 
 import java.io.IOException;
 import java.util.HashMap;
@@ -42,7 +41,7 @@ public class Stash implements ToXContent {
     public static final Stash EMPTY = new Stash();
 
     private final Map<String, Object> stash = new HashMap<>();
-    private RestTestResponse response;
+    private final ObjectPath stashObjectPath = new ObjectPath(stash);
 
     /**
      * Allows to saved a specific field in the stash as key-value pair
@@ -55,12 +54,6 @@ public class Stash implements ToXContent {
         }
     }
 
-    public void stashResponse(RestTestResponse response) throws IOException {
-        // TODO we can almost certainly save time by lazily evaluating the body
-        stashValue("body", response.getBody());
-        this.response = response;
-    }
-
     /**
      * Clears the previously stashed values
      */
@@ -88,13 +81,7 @@ public class Stash implements ToXContent {
      * as arguments for following requests (e.g. scroll_id)
      */
     public Object getValue(String key) throws IOException {
-        if (key.startsWith("$body.")) {
-            if (response == null) {
-                return null;
-            }
-            return response.evaluate(key.substring("$body".length()), this);
-        }
-        Object stashedValue = stash.get(key.substring(1));
+        Object stashedValue = stashObjectPath.evaluate(key.substring(1));
         if (stashedValue == null) {
             throw new IllegalArgumentException("stashed value not found for key [" + key + "]");
         }
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java
index ac3a755d344..4a21fefe803 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java
@@ -26,6 +26,7 @@ import org.elasticsearch.test.rest.ObjectPath;
 import org.elasticsearch.test.rest.Stash;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -197,6 +198,13 @@ public class ObjectPathTests extends ESTestCase {
         Object object = objectPath.evaluate("field1.$placeholder.element1", stash);
         assertThat(object, notNullValue());
         assertThat(object.toString(), equalTo("value1"));
+
+        Map<String, Object> stashedObject = new HashMap<>();
+        stashedObject.put("subobject", "elements");
+        stash.stashValue("object", stashedObject);
+        object = objectPath.evaluate("field1.$object\\.subobject.element1", stash);
+        assertThat(object, notNullValue());
+        assertThat(object.toString(), equalTo("value1"));
     }
 
     @SuppressWarnings("unchecked")

From dd781d410a268cd6d110d6a6942f44fa170c92fd Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Mon, 27 Jun 2016 11:43:49 +0200
Subject: [PATCH 1177/1311] fix line length problems in all classes under
 o.e.test.rest package

---
 .../main/resources/checkstyle_suppressions.xml   | 16 ----------------
 .../org/elasticsearch/test/rest/ObjectPath.java  |  3 ++-
 .../rest/parser/GreaterThanEqualToParser.java    |  3 ++-
 .../test/rest/parser/GreaterThanParser.java      |  3 ++-
 .../rest/parser/LessThanOrEqualToParser.java     |  3 ++-
 .../test/rest/parser/LessThanParser.java         |  3 ++-
 .../rest/parser/RestTestSuiteParseContext.java   |  3 ++-
 .../test/rest/parser/RestTestSuiteParser.java    |  6 ++++--
 .../test/rest/section/GreaterThanAssertion.java  |  6 ++++--
 .../section/GreaterThanEqualToAssertion.java     |  6 ++++--
 .../test/rest/section/LengthAssertion.java       |  3 ++-
 .../test/rest/section/LessThanAssertion.java     |  6 ++++--
 .../rest/section/LessThanOrEqualToAssertion.java |  6 ++++--
 .../test/rest/section/MatchAssertion.java        |  3 ++-
 .../test/rest/support/FileUtils.java             |  9 ++++++---
 .../test/rest/test/FileUtilsTests.java           |  3 ++-
 .../test/rest/test/RestTestParserTests.java      |  8 +++++---
 17 files changed, 49 insertions(+), 41 deletions(-)

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index bb19be736d3..2a361980a8e 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -1263,24 +1263,8 @@
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]engine[/\\]MockEngineSupport.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]hamcrest[/\\]ElasticsearchAssertions.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]junit[/\\]listeners[/\\]LoggingListener.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]parser[/\\]GreaterThanEqualToParser.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]parser[/\\]GreaterThanParser.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]parser[/\\]LessThanOrEqualToParser.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]parser[/\\]LessThanParser.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]parser[/\\]RestTestSuiteParseContext.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]parser[/\\]RestTestSuiteParser.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]section[/\\]GreaterThanAssertion.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]section[/\\]GreaterThanEqualToAssertion.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]section[/\\]LengthAssertion.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]section[/\\]LessThanAssertion.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]section[/\\]LessThanOrEqualToAssertion.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]section[/\\]MatchAssertion.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]spec[/\\]RestApiParser.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]support[/\\]FileUtils.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]store[/\\]MockFSDirectoryService.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]store[/\\]MockFSIndexStore.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]test[/\\]FileUtilsTests.java" checks="LineLength" />
-  <suppress files="test[/\\]framework[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]rest[/\\]test[/\\]RestTestParserTests.java" checks="LineLength" />
   <suppress files="test[/\\]framework[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]test[/\\]test[/\\]InternalTestClusterTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]cli[/\\]CliTool.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]admin[/\\]indices[/\\]settings[/\\]RestGetSettingsAction.java" checks="LineLength" />
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java
index 5ee1fbf521a..77de0f30f0c 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java
@@ -84,7 +84,8 @@ public class ObjectPath {
             } catch (NumberFormatException e) {
                 throw new IllegalArgumentException("element was a list, but [" + key + "] was not numeric", e);
             } catch (IndexOutOfBoundsException e) {
-                throw new IllegalArgumentException("element was a list with " + list.size() + " elements, but [" + key + "] was out of bounds", e);
+                throw new IllegalArgumentException("element was a list with " + list.size() +
+                        " elements, but [" + key + "] was out of bounds", e);
             }
         }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/GreaterThanEqualToParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/GreaterThanEqualToParser.java
index 68f833d35c7..7a4cd0f316a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/GreaterThanEqualToParser.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/GreaterThanEqualToParser.java
@@ -33,7 +33,8 @@ public class GreaterThanEqualToParser implements RestTestFragmentParser<GreaterT
     public GreaterThanEqualToAssertion parse(RestTestSuiteParseContext parseContext) throws IOException, RestTestParseException {
         Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
         if (! (stringObjectTuple.v2() instanceof Comparable) ) {
-            throw new RestTestParseException("gte section can only be used with objects that support natural ordering, found " + stringObjectTuple.v2().getClass().getSimpleName());
+            throw new RestTestParseException("gte section can only be used with objects that support natural ordering, found "
+                    + stringObjectTuple.v2().getClass().getSimpleName());
         }
         return new GreaterThanEqualToAssertion(stringObjectTuple.v1(), stringObjectTuple.v2());
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/GreaterThanParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/GreaterThanParser.java
index a66122138c1..7e1ca1ece7f 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/GreaterThanParser.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/GreaterThanParser.java
@@ -32,7 +32,8 @@ public class GreaterThanParser implements RestTestFragmentParser<GreaterThanAsse
     public GreaterThanAssertion parse(RestTestSuiteParseContext parseContext) throws IOException, RestTestParseException {
         Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
         if (! (stringObjectTuple.v2() instanceof Comparable) ) {
-            throw new RestTestParseException("gt section can only be used with objects that support natural ordering, found " + stringObjectTuple.v2().getClass().getSimpleName());
+            throw new RestTestParseException("gt section can only be used with objects that support natural ordering, found "
+                    + stringObjectTuple.v2().getClass().getSimpleName());
         }
         return new GreaterThanAssertion(stringObjectTuple.v1(), stringObjectTuple.v2());
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/LessThanOrEqualToParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/LessThanOrEqualToParser.java
index f2d53d05a56..a30979c6a3c 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/LessThanOrEqualToParser.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/LessThanOrEqualToParser.java
@@ -33,7 +33,8 @@ public class LessThanOrEqualToParser implements RestTestFragmentParser<LessThanO
     public LessThanOrEqualToAssertion parse(RestTestSuiteParseContext parseContext) throws IOException, RestTestParseException {
         Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
         if (! (stringObjectTuple.v2() instanceof Comparable) ) {
-            throw new RestTestParseException("lte section can only be used with objects that support natural ordering, found " + stringObjectTuple.v2().getClass().getSimpleName());
+            throw new RestTestParseException("lte section can only be used with objects that support natural ordering, found "
+                    + stringObjectTuple.v2().getClass().getSimpleName());
         }
         return new LessThanOrEqualToAssertion(stringObjectTuple.v1(), stringObjectTuple.v2());
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/LessThanParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/LessThanParser.java
index 065dd19d6a1..fc31f221758 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/LessThanParser.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/LessThanParser.java
@@ -32,7 +32,8 @@ public class LessThanParser implements RestTestFragmentParser<LessThanAssertion>
     public LessThanAssertion parse(RestTestSuiteParseContext parseContext) throws IOException, RestTestParseException {
         Tuple<String,Object> stringObjectTuple = parseContext.parseTuple();
         if (! (stringObjectTuple.v2() instanceof Comparable) ) {
-            throw new RestTestParseException("lt section can only be used with objects that support natural ordering, found " + stringObjectTuple.v2().getClass().getSimpleName());
+            throw new RestTestParseException("lt section can only be used with objects that support natural ordering, found "
+                    + stringObjectTuple.v2().getClass().getSimpleName());
         }
         return new LessThanAssertion(stringObjectTuple.v1(), stringObjectTuple.v2());
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java
index 7edad9995bb..f7325443deb 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java
@@ -158,7 +158,8 @@ public class RestTestSuiteParseContext {
             token = parser.nextToken();
         }
         if (token != XContentParser.Token.FIELD_NAME) {
-            throw new RestTestParseException("malformed test section: field name expected but found " + token + " at " + parser.getTokenLocation());
+            throw new RestTestParseException("malformed test section: field name expected but found " + token + " at "
+                    + parser.getTokenLocation());
         }
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java
index c6986d3eac8..f22f0109594 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java
@@ -71,7 +71,8 @@ public class RestTestSuiteParser implements RestTestFragmentParser<RestTestSuite
         XContentParser parser = parseContext.parser();
 
         parser.nextToken();
-        assert parser.currentToken() == XContentParser.Token.START_OBJECT : "expected token to be START_OBJECT but was " + parser.currentToken();
+        assert parser.currentToken() == XContentParser.Token.START_OBJECT : "expected token to be START_OBJECT but was "
+                + parser.currentToken();
 
         RestTestSuite restTestSuite = new RestTestSuite(parseContext.getApi(), parseContext.getSuiteName());
 
@@ -90,7 +91,8 @@ public class RestTestSuiteParser implements RestTestFragmentParser<RestTestSuite
 
             TestSection testSection = parseContext.parseTestSection();
             if (!restTestSuite.addTestSection(testSection)) {
-                throw new RestTestParseException("duplicate test section [" + testSection.getName() + "] found in [" + restTestSuite.getPath() + "]");
+                throw new RestTestParseException("duplicate test section [" + testSection.getName() + "] found in ["
+                        + restTestSuite.getPath() + "]");
             }
         }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/GreaterThanAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/GreaterThanAssertion.java
index ade7fbd59ca..63f69696653 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/GreaterThanAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/GreaterThanAssertion.java
@@ -43,8 +43,10 @@ public class GreaterThanAssertion extends Assertion {
     @SuppressWarnings("unchecked")
     protected void doAssert(Object actualValue, Object expectedValue) {
         logger.trace("assert that [{}] is greater than [{}] (field: [{}])", actualValue, expectedValue, getField());
-        assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])", actualValue, instanceOf(Comparable.class));
-        assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])", expectedValue, instanceOf(Comparable.class));
+        assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])",
+                actualValue, instanceOf(Comparable.class));
+        assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])",
+                expectedValue, instanceOf(Comparable.class));
         try {
             assertThat(errorMessage(), (Comparable) actualValue, greaterThan((Comparable) expectedValue));
         } catch (ClassCastException e) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/GreaterThanEqualToAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/GreaterThanEqualToAssertion.java
index cfdca7bc338..297eecf2d2a 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/GreaterThanEqualToAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/GreaterThanEqualToAssertion.java
@@ -43,8 +43,10 @@ public class GreaterThanEqualToAssertion extends Assertion {
     @Override
     protected void doAssert(Object actualValue, Object expectedValue) {
         logger.trace("assert that [{}] is greater than or equal to [{}] (field: [{}])", actualValue, expectedValue, getField());
-        assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])", actualValue, instanceOf(Comparable.class));
-        assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])", expectedValue, instanceOf(Comparable.class));
+        assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])",
+                actualValue, instanceOf(Comparable.class));
+        assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])",
+                expectedValue, instanceOf(Comparable.class));
         try {
             assertThat(errorMessage(), (Comparable) actualValue, greaterThanOrEqualTo((Comparable) expectedValue));
         } catch (ClassCastException e) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/LengthAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/LengthAssertion.java
index 265487a0388..eb28ba01a94 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/LengthAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/LengthAssertion.java
@@ -44,7 +44,8 @@ public class LengthAssertion extends Assertion {
     @Override
     protected void doAssert(Object actualValue, Object expectedValue) {
         logger.trace("assert that [{}] has length [{}] (field: [{}])", actualValue, expectedValue, getField());
-        assertThat("expected value of [" + getField() + "] is not numeric (got [" + expectedValue.getClass() + "]", expectedValue, instanceOf(Number.class));
+        assertThat("expected value of [" + getField() + "] is not numeric (got [" + expectedValue.getClass() + "]",
+                expectedValue, instanceOf(Number.class));
         int length = ((Number) expectedValue).intValue();
         if (actualValue instanceof String) {
             assertThat(errorMessage(), ((String) actualValue).length(), equalTo(length));
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/LessThanAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/LessThanAssertion.java
index 89387ff8952..153a7824569 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/LessThanAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/LessThanAssertion.java
@@ -44,8 +44,10 @@ public class LessThanAssertion extends Assertion {
     @SuppressWarnings("unchecked")
     protected void doAssert(Object actualValue, Object expectedValue) {
         logger.trace("assert that [{}] is less than [{}] (field: [{}])", actualValue, expectedValue, getField());
-        assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])", actualValue, instanceOf(Comparable.class));
-        assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])", expectedValue, instanceOf(Comparable.class));
+        assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])",
+                actualValue, instanceOf(Comparable.class));
+        assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])",
+                expectedValue, instanceOf(Comparable.class));
         try {
             assertThat(errorMessage(), (Comparable) actualValue, lessThan((Comparable) expectedValue));
         } catch (ClassCastException e) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/LessThanOrEqualToAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/LessThanOrEqualToAssertion.java
index 99cbf1155d5..1eb3a9fc2b2 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/LessThanOrEqualToAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/LessThanOrEqualToAssertion.java
@@ -43,8 +43,10 @@ public class LessThanOrEqualToAssertion  extends Assertion {
     @Override
     protected void doAssert(Object actualValue, Object expectedValue) {
         logger.trace("assert that [{}] is less than or equal to [{}] (field: [{}])", actualValue, expectedValue, getField());
-        assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])", actualValue, instanceOf(Comparable.class));
-        assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])", expectedValue, instanceOf(Comparable.class));
+        assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])",
+                actualValue, instanceOf(Comparable.class));
+        assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])",
+                expectedValue, instanceOf(Comparable.class));
         try {
             assertThat(errorMessage(), (Comparable) actualValue, lessThanOrEqualTo((Comparable) expectedValue));
         } catch (ClassCastException e) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/MatchAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/MatchAssertion.java
index 49b82dfafac..3a96d4532a0 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/MatchAssertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/MatchAssertion.java
@@ -55,7 +55,8 @@ public class MatchAssertion extends Assertion {
         if (expectedValue instanceof String) {
             String expValue = ((String) expectedValue).trim();
             if (expValue.length() > 2 && expValue.startsWith("/") && expValue.endsWith("/")) {
-                assertThat("field [" + getField() + "] was expected to be of type String but is an instanceof [" + safeClass(actualValue) + "]", actualValue, instanceOf(String.class));
+                assertThat("field [" + getField() + "] was expected to be of type String but is an instanceof [" +
+                        safeClass(actualValue) + "]", actualValue, instanceOf(String.class));
                 String stringValue = (String) actualValue;
                 String regex = expValue.substring(1, expValue.length() - 1);
                 logger.trace("assert that [{}] matches [{}]", stringValue, regex);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/support/FileUtils.java b/test/framework/src/main/java/org/elasticsearch/test/rest/support/FileUtils.java
index 69acae55fdc..b32308f8cd8 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/support/FileUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/support/FileUtils.java
@@ -78,7 +78,8 @@ public final class FileUtils {
      * Each input path can either be a single file (the .yaml suffix is optional) or a directory.
      * Each path is looked up in the classpath, or optionally from {@code fileSystem} if its not null.
      */
-    public static Map<String, Set<Path>> findYamlSuites(FileSystem fileSystem, String optionalPathPrefix, final String... paths) throws IOException {
+    public static Map<String, Set<Path>> findYamlSuites(FileSystem fileSystem, String optionalPathPrefix, final String... paths)
+            throws IOException {
         Map<String, Set<Path>> yamlSuites = new HashMap<>();
         for (String path : paths) {
             collectFiles(resolveFile(fileSystem, optionalPathPrefix, path, YAML_SUFFIX), YAML_SUFFIX, yamlSuites);
@@ -86,7 +87,8 @@ public final class FileUtils {
         return yamlSuites;
     }
 
-    private static Path resolveFile(FileSystem fileSystem, String optionalPathPrefix, String path, String optionalFileSuffix) throws IOException {
+    private static Path resolveFile(FileSystem fileSystem, String optionalPathPrefix, String path, String optionalFileSuffix)
+            throws IOException {
         if (fileSystem != null) {
             Path file = findFile(fileSystem, path, optionalFileSuffix);
             if (!lenientExists(file)) {
@@ -94,7 +96,8 @@ public final class FileUtils {
                 String newPath = optionalPathPrefix + "/" + path;
                 file = findFile(fileSystem, newPath, optionalFileSuffix);
                 if (!lenientExists(file)) {
-                    throw new NoSuchFileException("path prefix: " + optionalPathPrefix + ", path: " + path + ", file suffix: " + optionalFileSuffix);
+                    throw new NoSuchFileException("path prefix: " + optionalPathPrefix + ", path: " + path + ", file suffix: "
+                            + optionalFileSuffix);
                 }
             }
             return file;
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/test/FileUtilsTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/test/FileUtilsTests.java
index 128cb862e57..db41c42e16a 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/test/FileUtilsTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/test/FileUtilsTests.java
@@ -84,7 +84,8 @@ public class FileUtilsTests extends ESTestCase {
         assertSingleFile(yamlSuites.get(dir.getFileName().toString()), dir.getFileName().toString(), file.getFileName().toString());
 
         //load from external file (optional extension)
-        yamlSuites = FileUtils.findYamlSuites(dir.getFileSystem(), "/rest-api-spec/test", dir.resolve("test_loading").toAbsolutePath().toString());
+        yamlSuites = FileUtils.findYamlSuites(dir.getFileSystem(), "/rest-api-spec/test",
+                dir.resolve("test_loading").toAbsolutePath().toString());
         assertThat(yamlSuites, notNullValue());
         assertThat(yamlSuites.size(), equalTo(1));
         assertThat(yamlSuites.containsKey(dir.getFileName().toString()), equalTo(true));
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/test/RestTestParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/test/RestTestParserTests.java
index 76fbe8307e6..6b5cc3defb7 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/test/RestTestParserTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/test/RestTestParserTests.java
@@ -115,11 +115,12 @@ public class RestTestParserTests extends ESTestCase {
             assertThat(restTestSuite.getSetupSection().getDoSections().size(), equalTo(1));
             assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getApi(), equalTo("indices.create"));
             assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getParams().size(), equalTo(1));
-            assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getParams().get("index"), equalTo("test_index"));
+            assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getParams().get("index"),
+                    equalTo("test_index"));
         } else {
             assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(true));
         }
-
+        
         assertThat(restTestSuite.getTeardownSection(), notNullValue());
         if (includeTeardown) {
             assertThat(restTestSuite.getTeardownSection().isEmpty(), equalTo(false));
@@ -154,7 +155,8 @@ public class RestTestParserTests extends ESTestCase {
 
         assertThat(restTestSuite.getTestSections().get(1).getName(), equalTo("Get type mapping - pre 1.0"));
         assertThat(restTestSuite.getTestSections().get(1).getSkipSection().isEmpty(), equalTo(false));
-        assertThat(restTestSuite.getTestSections().get(1).getSkipSection().getReason(), equalTo("for newer versions the index name is always returned"));
+        assertThat(restTestSuite.getTestSections().get(1).getSkipSection().getReason(),
+                equalTo("for newer versions the index name is always returned"));
         assertThat(restTestSuite.getTestSections().get(1).getSkipSection().getLowerVersion(), equalTo(Version.V_2_0_0));
         assertThat(restTestSuite.getTestSections().get(1).getSkipSection().getUpperVersion(), equalTo(Version.CURRENT));
         assertThat(restTestSuite.getTestSections().get(1).getExecutableSections().size(), equalTo(3));

From c4b87149c2b1211287a0393b05cd1eca727155e6 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 1 Jul 2016 10:22:55 +0200
Subject: [PATCH 1178/1311] Build: set group for client and sniffer, disable
 publishing for client-test

Closes #19205
---
 client/rest/build.gradle    | 2 ++
 client/sniffer/build.gradle | 2 ++
 client/test/build.gradle    | 3 +++
 3 files changed, 7 insertions(+)

diff --git a/client/rest/build.gradle b/client/rest/build.gradle
index e578126b381..4623fbd8c2c 100644
--- a/client/rest/build.gradle
+++ b/client/rest/build.gradle
@@ -26,6 +26,8 @@ apply plugin: 'ru.vyarus.animalsniffer'
 targetCompatibility = JavaVersion.VERSION_1_7
 sourceCompatibility = JavaVersion.VERSION_1_7
 
+group = 'org.elasticsearch.client'
+
 dependencies {
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
diff --git a/client/sniffer/build.gradle b/client/sniffer/build.gradle
index 748c3e2261e..7cf16ee85d8 100644
--- a/client/sniffer/build.gradle
+++ b/client/sniffer/build.gradle
@@ -26,6 +26,8 @@ apply plugin: 'ru.vyarus.animalsniffer'
 targetCompatibility = JavaVersion.VERSION_1_7
 sourceCompatibility = JavaVersion.VERSION_1_7
 
+group = 'org.elasticsearch.client'
+
 dependencies {
   compile "org.elasticsearch.client:rest:${version}"
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
diff --git a/client/test/build.gradle b/client/test/build.gradle
index 1296b9fa5b8..05d044504ec 100644
--- a/client/test/build.gradle
+++ b/client/test/build.gradle
@@ -26,6 +26,9 @@ apply plugin: 'ru.vyarus.animalsniffer'
 targetCompatibility = JavaVersion.VERSION_1_7
 sourceCompatibility = JavaVersion.VERSION_1_7
 
+install.enabled = false
+uploadArchives.enabled = false
+
 dependencies {
   compile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   compile "junit:junit:${versions.junit}"

From 1297a707da7bdc97dfdacb59f23dd1e918c14d63 Mon Sep 17 00:00:00 2001
From: Leon Weidauer <leon@lnwdr.de>
Date: Fri, 1 Jul 2016 14:54:25 +0200
Subject: [PATCH 1179/1311] non-binary gender option in term aggr. example
 (#19188)

* non-binary gender option in term aggr. example

* replace gender with music genre for term aggregation docs
---
 .../bucket/terms-aggregation.asciidoc         | 76 ++++++++++---------
 1 file changed, 40 insertions(+), 36 deletions(-)

diff --git a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
index 3c1f4ae860a..959b93611d8 100644
--- a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
+++ b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc
@@ -9,8 +9,8 @@ Example:
 --------------------------------------------------
 {
     "aggs" : {
-        "genders" : {
-            "terms" : { "field" : "gender" }
+        "genres" : {
+            "terms" : { "field" : "genre" }
         }
     }
 }
@@ -24,16 +24,20 @@ Response:
     ...
 
     "aggregations" : {
-        "genders" : {
+        "genres" : {
             "doc_count_error_upper_bound": 0, <1>
             "sum_other_doc_count": 0, <2>
             "buckets" : [ <3>
                 {
-                    "key" : "male",
+                    "key" : "jazz",
                     "doc_count" : 10
                 },
                 {
-                    "key" : "female",
+                    "key" : "rock",
+                    "doc_count" : 10
+                },
+                {
+                    "key" : "electronic",
                     "doc_count" : 10
                 },
             ]
@@ -247,9 +251,9 @@ Ordering the buckets by their `doc_count` in an ascending manner:
 --------------------------------------------------
 {
     "aggs" : {
-        "genders" : {
+        "genres" : {
             "terms" : {
-                "field" : "gender",
+                "field" : "genre",
                 "order" : { "_count" : "asc" }
             }
         }
@@ -263,9 +267,9 @@ Ordering the buckets alphabetically by their terms in an ascending manner:
 --------------------------------------------------
 {
     "aggs" : {
-        "genders" : {
+        "genres" : {
             "terms" : {
-                "field" : "gender",
+                "field" : "genre",
                 "order" : { "_term" : "asc" }
             }
         }
@@ -280,13 +284,13 @@ Ordering the buckets by single value metrics sub-aggregation (identified by the
 --------------------------------------------------
 {
     "aggs" : {
-        "genders" : {
+        "genres" : {
             "terms" : {
-                "field" : "gender",
-                "order" : { "avg_height" : "desc" }
+                "field" : "genre",
+                "order" : { "avg_play_count" : "desc" }
             },
             "aggs" : {
-                "avg_height" : { "avg" : { "field" : "height" } }
+                "avg_play_count" : { "avg" : { "field" : "play_count" } }
             }
         }
     }
@@ -299,13 +303,13 @@ Ordering the buckets by multi value metrics sub-aggregation (identified by the a
 --------------------------------------------------
 {
     "aggs" : {
-        "genders" : {
+        "genres" : {
             "terms" : {
-                "field" : "gender",
-                "order" : { "height_stats.avg" : "desc" }
+                "field" : "genre",
+                "order" : { "playback_stats.avg" : "desc" }
             },
             "aggs" : {
-                "height_stats" : { "stats" : { "field" : "height" } }
+                "playback_stats" : { "stats" : { "field" : "play_count" } }
             }
         }
     }
@@ -343,14 +347,14 @@ PATH                :=  <AGG_NAME>[<AGG_SEPARATOR><AGG_NAME>]*[<METRIC_SEPARATOR
     "aggs" : {
         "countries" : {
             "terms" : {
-                "field" : "address.country",
-                "order" : { "females>height_stats.avg" : "desc" }
+                "field" : "artist.country",
+                "order" : { "rock>playback_stats.avg" : "desc" }
             },
             "aggs" : {
-                "females" : {
-                    "filter" : { "term" : { "gender" :  "female" }},
+                "rock" : {
+                    "filter" : { "term" : { "genre" :  "rock" }},
                     "aggs" : {
-                        "height_stats" : { "stats" : { "field" : "height" }}
+                        "playback_stats" : { "stats" : { "field" : "play_count" }}
                     }
                 }
             }
@@ -359,7 +363,7 @@ PATH                :=  <AGG_NAME>[<AGG_SEPARATOR><AGG_NAME>]*[<METRIC_SEPARATOR
 }
 --------------------------------------------------
 
-The above will sort the countries buckets based on the average height among the female population.
+The above will sort the artist's countries buckets based on the average play count among the rock songs.
 
 Multiple criteria can be used to order the buckets by providing an array of order criteria such as the following:
 
@@ -369,14 +373,14 @@ Multiple criteria can be used to order the buckets by providing an array of orde
     "aggs" : {
         "countries" : {
             "terms" : {
-                "field" : "address.country",
-                "order" : [ { "females>height_stats.avg" : "desc" }, { "_count" : "desc" } ]
+                "field" : "artist.country",
+                "order" : [ { "rock>playback_stats.avg" : "desc" }, { "_count" : "desc" } ]
             },
             "aggs" : {
-                "females" : {
-                    "filter" : { "term" : { "gender" : { "female" }}},
+                "rock" : {
+                    "filter" : { "term" : { "genre" : { "rock" }}},
                     "aggs" : {
-                        "height_stats" : { "stats" : { "field" : "height" }}
+                        "playback_stats" : { "stats" : { "field" : "play_count" }}
                     }
                 }
             }
@@ -385,7 +389,7 @@ Multiple criteria can be used to order the buckets by providing an array of orde
 }
 --------------------------------------------------
 
-The above will sort the countries buckets based on the average height among the female population and then by
+The above will sort the artist's countries buckets based on the average play count among the rock songs and then by
 their `doc_count` in descending order.
 
 NOTE: In the event that two buckets share the same values for all order criteria the bucket's term value is used as a
@@ -439,10 +443,10 @@ Generating the terms using a script:
 --------------------------------------------------
 {
     "aggs" : {
-        "genders" : {
+        "genres" : {
             "terms" : {
                 "script" : {
-                    "inline": "doc['gender'].value"
+                    "inline": "doc['genre'].value"
                     "lang": "painless"
                 }
             }
@@ -457,12 +461,12 @@ This will interpret the `script` parameter as an `inline` script with the defaul
 --------------------------------------------------
 {
     "aggs" : {
-        "genders" : {
+        "genres" : {
             "terms" : {
                 "script" : {
                     "file": "my_script",
                     "params": {
-                        "field": "gender"
+                        "field": "genre"
                     }
                 }
             }
@@ -480,11 +484,11 @@ TIP: for indexed scripts replace the `file` parameter with an `id` parameter.
 --------------------------------------------------
 {
     "aggs" : {
-        "genders" : {
+        "genres" : {
             "terms" : {
-                "field" : "gender",
+                "field" : "gendre",
                 "script" : {
-                    "inline" : "'Gender: ' +_value"
+                    "inline" : "'Genre: ' +_value"
                     "lang" : "painless"
                 }
             }

From 91b66e3cf488cc2961bab106de68eae6b2b3cf5c Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 30 Jun 2016 15:19:58 -0400
Subject: [PATCH 1180/1311] Migration stats and extended stats to
 NamedWriteable

Migrates the `stats` and `extended_stats` aggregations and pipeline
aggregations from the special purpose aggregations streams to
`NamedWriteable`. These are the first pipeline aggregations so this
adds the infrastructure to support both streams and `NamedWriteable`s
for pipeline aggregations.
---
 .../elasticsearch/search/SearchModule.java    | 65 ++++++++-----
 .../aggregations/InternalAggregation.java     | 47 +++++++---
 .../metrics/stats/InternalStats.java          | 91 +++++++------------
 .../stats/StatsAggregationBuilder.java        |  8 +-
 .../ExtendedStatsAggregationBuilder.java      |  8 +-
 .../stats/extended/InternalExtendedStats.java | 59 ++++--------
 .../pipeline/PipelineAggregator.java          | 72 ++++++++++-----
 .../pipeline/SiblingPipelineAggregator.java   |  9 ++
 .../BucketMetricsPipelineAggregator.java      | 50 +++++-----
 .../stats/InternalStatsBucket.java            | 28 ++----
 ...StatsBucketPipelineAggregationBuilder.java |  6 +-
 .../stats/StatsBucketPipelineAggregator.java  | 28 +-----
 ...StatsBucketPipelineAggregationBuilder.java |  6 +-
 ...ExtendedStatsBucketPipelineAggregator.java | 48 +++-------
 .../extended/InternalExtendedStatsBucket.java | 28 ++----
 .../search/query/QuerySearchResult.java       | 23 ++++-
 .../ExtendedStatsBucketTests.java             |  3 +-
 17 files changed, 281 insertions(+), 298 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java
index cd9d6f8a3f0..ee05a72e40a 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -91,8 +91,8 @@ import org.elasticsearch.index.query.functionscore.ScriptScoreFunctionBuilder;
 import org.elasticsearch.index.query.functionscore.WeightBuilder;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.action.SearchTransportService;
-import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
@@ -119,6 +119,7 @@ import org.elasticsearch.search.aggregations.bucket.nested.InternalNested;
 import org.elasticsearch.search.aggregations.bucket.nested.InternalReverseNested;
 import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.nested.ReverseNestedAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.range.InternalBinaryRange;
 import org.elasticsearch.search.aggregations.bucket.range.InternalRange;
 import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.range.RangeParser;
@@ -129,7 +130,6 @@ import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanc
 import org.elasticsearch.search.aggregations.bucket.range.geodistance.GeoDistanceParser;
 import org.elasticsearch.search.aggregations.bucket.range.geodistance.InternalGeoDistance;
 import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.range.InternalBinaryRange;
 import org.elasticsearch.search.aggregations.bucket.range.ip.IpRangeParser;
 import org.elasticsearch.search.aggregations.bucket.sampler.DiversifiedAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.sampler.DiversifiedSamplerParser;
@@ -200,40 +200,42 @@ import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountParser
 import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.InternalBucketMetricValue;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.avg.AvgBucketPipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.avg.AvgBucketPipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.max.MaxBucketPipelineAggregator;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.avg.AvgBucketPipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.max.MaxBucketPipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.min.MinBucketPipelineAggregator;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.max.MaxBucketPipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.min.MinBucketPipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.PercentilesBucketPipelineAggregator;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.min.MinBucketPipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.PercentilesBucketPipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.StatsBucketPipelineAggregator;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.percentile.PercentilesBucketPipelineAggregator;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.InternalStatsBucket;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.StatsBucketPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.StatsBucketPipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketParser;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.sum.SumBucketPipelineAggregator;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregator;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.InternalExtendedStatsBucket;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.sum.SumBucketPipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.pipeline.bucketscript.BucketScriptPipelineAggregator;
+import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.sum.SumBucketPipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.bucketscript.BucketScriptPipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.pipeline.bucketselector.BucketSelectorPipelineAggregator;
+import org.elasticsearch.search.aggregations.pipeline.bucketscript.BucketScriptPipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.bucketselector.BucketSelectorPipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.pipeline.cumulativesum.CumulativeSumPipelineAggregator;
+import org.elasticsearch.search.aggregations.pipeline.bucketselector.BucketSelectorPipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.cumulativesum.CumulativeSumPipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.pipeline.derivative.DerivativePipelineAggregator;
+import org.elasticsearch.search.aggregations.pipeline.cumulativesum.CumulativeSumPipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.derivative.DerivativePipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.derivative.DerivativePipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.derivative.InternalDerivative;
-import org.elasticsearch.search.aggregations.pipeline.movavg.MovAvgPipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.movavg.MovAvgPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.movavg.MovAvgPipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.EwmaModel;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.HoltLinearModel;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.HoltWintersModel;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.LinearModel;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.MovAvgModel;
 import org.elasticsearch.search.aggregations.pipeline.movavg.models.SimpleModel;
-import org.elasticsearch.search.aggregations.pipeline.serialdiff.SerialDiffPipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.serialdiff.SerialDiffPipelineAggregationBuilder;
+import org.elasticsearch.search.aggregations.pipeline.serialdiff.SerialDiffPipelineAggregator;
 import org.elasticsearch.search.controller.SearchPhaseController;
 import org.elasticsearch.search.fetch.FetchPhase;
 import org.elasticsearch.search.fetch.FetchSubPhase;
@@ -445,16 +447,31 @@ public class SearchModule extends AbstractModule {
      * Register a pipeline aggregation.
      *
      * @param reader reads the aggregation builder from a stream
+     * @param internalReader reads the {@link PipelineAggregator} from a stream
+     * @param internalReader reads the {@link InternalAggregation} that represents a bucket in this aggregation from a stream
      * @param aggregationParser reads the aggregation builder from XContent
      * @param aggregationName names by which the aggregation may be parsed. The first name is special because it is the name that the reader
      *        is registered under.
      */
     public void registerPipelineAggregation(Writeable.Reader<? extends PipelineAggregationBuilder> reader,
+            Writeable.Reader<? extends PipelineAggregator> internalReader, Writeable.Reader<? extends InternalAggregation> bucketReader,
             PipelineAggregator.Parser aggregationParser, ParseField aggregationName) {
+        if (false == transportClient) {
+            namedWriteableRegistry.register(PipelineAggregationBuilder.class, aggregationName.getPreferredName(), reader);
+            pipelineAggregationParserRegistry.register(aggregationParser, aggregationName);
+        }
+        namedWriteableRegistry.register(PipelineAggregator.class, aggregationName.getPreferredName(), internalReader);
+        namedWriteableRegistry.register(InternalAggregation.class, aggregationName.getPreferredName(), bucketReader);
+    }
+
+    public void registerPipelineAggregation(Writeable.Reader<? extends PipelineAggregationBuilder> reader,
+            PipelineAggregator.Parser aggregationParser, ParseField aggregationName) {
+        // NORELEASE remove me in favor of the above method
         pipelineAggregationParserRegistry.register(aggregationParser, aggregationName);
         namedWriteableRegistry.register(PipelineAggregationBuilder.class, aggregationName.getPreferredName(), reader);
     }
 
+
     @Override
     protected void configure() {
         if (false == transportClient) {
@@ -475,9 +492,10 @@ public class SearchModule extends AbstractModule {
         registerAggregation(SumAggregationBuilder::new, new SumParser(), SumAggregationBuilder.AGGREGATION_NAME_FIELD);
         registerAggregation(MinAggregationBuilder::new, new MinParser(), MinAggregationBuilder.AGGREGATION_NAME_FIELD);
         registerAggregation(MaxAggregationBuilder::new, new MaxParser(), MaxAggregationBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(StatsAggregationBuilder::new, new StatsParser(), StatsAggregationBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(ExtendedStatsAggregationBuilder::new, new ExtendedStatsParser(),
-        ExtendedStatsAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(StatsAggregationBuilder::new, InternalStats::new, new StatsParser(),
+                StatsAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(ExtendedStatsAggregationBuilder::new, InternalExtendedStats::new, new ExtendedStatsParser(),
+                ExtendedStatsAggregationBuilder.AGGREGATION_NAME_FIELD);
         registerAggregation(ValueCountAggregationBuilder::new, new ValueCountParser(), ValueCountAggregationBuilder.AGGREGATION_NAME_FIELD);
         registerAggregation(PercentilesAggregationBuilder::new, new PercentilesParser(),
                 PercentilesAggregationBuilder.AGGREGATION_NAME_FIELD);
@@ -522,6 +540,7 @@ public class SearchModule extends AbstractModule {
                 ScriptedMetricAggregationBuilder.AGGREGATION_NAME_FIELD);
         registerAggregation(ChildrenAggregationBuilder::new, ChildrenAggregationBuilder::parse,
                 ChildrenAggregationBuilder.AGGREGATION_NAME_FIELD);
+
         registerPipelineAggregation(DerivativePipelineAggregationBuilder::new, DerivativePipelineAggregationBuilder::parse,
                 DerivativePipelineAggregationBuilder.AGGREGATION_NAME_FIELD);
         registerPipelineAggregation(MaxBucketPipelineAggregationBuilder::new, MaxBucketPipelineAggregationBuilder.PARSER,
@@ -532,9 +551,11 @@ public class SearchModule extends AbstractModule {
                 AvgBucketPipelineAggregationBuilder.AGGREGATION_NAME_FIELD);
         registerPipelineAggregation(SumBucketPipelineAggregationBuilder::new, SumBucketPipelineAggregationBuilder.PARSER,
                 SumBucketPipelineAggregationBuilder.AGGREGATION_NAME_FIELD);
-        registerPipelineAggregation(StatsBucketPipelineAggregationBuilder::new, StatsBucketPipelineAggregationBuilder.PARSER,
+        registerPipelineAggregation(StatsBucketPipelineAggregationBuilder::new, StatsBucketPipelineAggregator::new,
+                InternalStatsBucket::new, StatsBucketPipelineAggregationBuilder.PARSER,
                 StatsBucketPipelineAggregationBuilder.AGGREGATION_NAME_FIELD);
-        registerPipelineAggregation(ExtendedStatsBucketPipelineAggregationBuilder::new, new ExtendedStatsBucketParser(),
+        registerPipelineAggregation(ExtendedStatsBucketPipelineAggregationBuilder::new, ExtendedStatsBucketPipelineAggregator::new,
+                InternalExtendedStatsBucket::new, new ExtendedStatsBucketParser(),
                 ExtendedStatsBucketPipelineAggregationBuilder.AGGREGATION_NAME_FIELD);
         registerPipelineAggregation(PercentilesBucketPipelineAggregationBuilder::new, PercentilesBucketPipelineAggregationBuilder.PARSER,
                 PercentilesBucketPipelineAggregationBuilder.AGGREGATION_NAME_FIELD);
@@ -703,8 +724,6 @@ public class SearchModule extends AbstractModule {
         InternalSum.registerStreams();
         InternalMin.registerStreams();
         InternalMax.registerStreams();
-        InternalStats.registerStreams();
-        InternalExtendedStats.registerStreams();
         InternalValueCount.registerStreams();
         InternalTDigestPercentiles.registerStreams();
         InternalTDigestPercentileRanks.registerStreams();
@@ -749,8 +768,6 @@ public class SearchModule extends AbstractModule {
         MinBucketPipelineAggregator.registerStreams();
         AvgBucketPipelineAggregator.registerStreams();
         SumBucketPipelineAggregator.registerStreams();
-        StatsBucketPipelineAggregator.registerStreams();
-        ExtendedStatsBucketPipelineAggregator.registerStreams();
         PercentilesBucketPipelineAggregator.registerStreams();
         MovAvgPipelineAggregator.registerStreams();
         CumulativeSumPipelineAggregator.registerStreams();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
index b75244ce889..93ecfa975b4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
@@ -151,9 +151,13 @@ public abstract class InternalAggregation implements Aggregation, ToXContent, St
         } else {
             pipelineAggregators = new ArrayList<>(size);
             for (int i = 0; i < size; i++) {
-                BytesReference type = in.readBytesReference();
-                PipelineAggregator pipelineAggregator = PipelineAggregatorStreams.stream(type).readResult(in);
-                pipelineAggregators.add(pipelineAggregator);
+                if (in.readBoolean()) {
+                    pipelineAggregators.add(in.readNamedWriteable(PipelineAggregator.class));
+                } else {
+                    BytesReference type = in.readBytesReference();
+                    PipelineAggregator pipelineAggregator = PipelineAggregatorStreams.stream(type).readResult(in);
+                    pipelineAggregators.add(pipelineAggregator);
+                }
             }
         }
     }
@@ -174,9 +178,13 @@ public abstract class InternalAggregation implements Aggregation, ToXContent, St
         } else {
             pipelineAggregators = new ArrayList<>(size);
             for (int i = 0; i < size; i++) {
-                BytesReference type = in.readBytesReference();
-                PipelineAggregator pipelineAggregator = PipelineAggregatorStreams.stream(type).readResult(in);
-                pipelineAggregators.add(pipelineAggregator);
+                if (in.readBoolean()) {
+                    pipelineAggregators.add(in.readNamedWriteable(PipelineAggregator.class));
+                } else {
+                    BytesReference type = in.readBytesReference();
+                    PipelineAggregator pipelineAggregator = PipelineAggregatorStreams.stream(type).readResult(in);
+                    pipelineAggregators.add(pipelineAggregator);
+                }
             }
         }
         doReadFrom(in);
@@ -188,34 +196,43 @@ public abstract class InternalAggregation implements Aggregation, ToXContent, St
 
     @Override
     public final void writeTo(StreamOutput out) throws IOException {
-        out.writeString(name);    // NORELEASE remote writing the name - it is automatically handled with writeNamedWriteable
+        out.writeString(name);    // NORELEASE remote writing the name? it is automatically handled with writeNamedWriteable
         out.writeGenericValue(metaData);
         out.writeVInt(pipelineAggregators.size());
         for (PipelineAggregator pipelineAggregator : pipelineAggregators) {
-            out.writeBytesReference(pipelineAggregator.type().stream());
-            pipelineAggregator.writeTo(out);
+            // NORELEASE temporary hack to support old style streams and new style NamedWriteable
+            try {
+                pipelineAggregator.getWriteableName(); // Throws UnsupportedOperationException if we should use old style streams.
+                out.writeBoolean(true);
+                out.writeNamedWriteable(pipelineAggregator);
+            } catch (UnsupportedOperationException e) {
+                out.writeBoolean(false);
+                out.writeBytesReference(pipelineAggregator.type().stream());
+                pipelineAggregator.writeTo(out);
+            }
         }
         doWriteTo(out);
     }
 
     protected abstract void doWriteTo(StreamOutput out) throws IOException;
 
-    @Override
-    public String getName() {
-        return name;
-    }
-
     @Override
     public String getWriteableName() {
         // NORELEASE remove me when all InternalAggregations override it
         throw new UnsupportedOperationException("Override on every class");
     }
 
+    @Override
+    public String getName() {
+        return name;
+    }
+
     /**
      * @return The {@link Type} of this aggregation
      */
     public Type type() {
-        throw new UnsupportedOperationException("Use getWriteableName instead"); // NORELEASE remove me
+        // NORELEASE remove this method
+        throw new UnsupportedOperationException(getClass().getName() + " used type but should Use getWriteableName instead");
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/InternalStats.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/InternalStats.java
index cbbb3493100..e060826c24c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/InternalStats.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/InternalStats.java
@@ -22,7 +22,6 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -31,26 +30,7 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
-/**
-*
-*/
 public class InternalStats extends InternalNumericMetricsAggregation.MultiValue implements Stats {
-
-    public final static Type TYPE = new Type("stats");
-
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public InternalStats readResult(StreamInput in) throws IOException {
-            InternalStats result = new InternalStats();
-            result.readFrom(in);
-            return result;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
     enum Metrics {
 
         count, sum, min, max, avg;
@@ -60,12 +40,10 @@ public class InternalStats extends InternalNumericMetricsAggregation.MultiValue
         }
     }
 
-    protected long count;
-    protected double min;
-    protected double max;
-    protected double sum;
-
-    protected InternalStats() {} // for serialization
+    protected final long count;
+    protected final double min;
+    protected final double max;
+    protected final double sum;
 
     public InternalStats(String name, long count, double sum, double min, double max, DocValueFormat formatter,
             List<PipelineAggregator> pipelineAggregators,
@@ -78,6 +56,36 @@ public class InternalStats extends InternalNumericMetricsAggregation.MultiValue
         this.format = formatter;
     }
 
+    /**
+     * Read from a stream.
+     */
+    public InternalStats(StreamInput in) throws IOException {
+        super(in);
+        format = in.readNamedWriteable(DocValueFormat.class);
+        count = in.readVLong();
+        min = in.readDouble();
+        max = in.readDouble();
+        sum = in.readDouble();
+    }
+
+    @Override
+    protected final void doWriteTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(format);
+        out.writeVLong(count);
+        out.writeDouble(min);
+        out.writeDouble(max);
+        out.writeDouble(sum);
+        writeOtherStatsTo(out);
+    }
+
+    protected void writeOtherStatsTo(StreamOutput out) throws IOException {
+    }
+
+    @Override
+    public String getWriteableName() {
+        return StatsAggregationBuilder.NAME;
+    }
+
     @Override
     public long getCount() {
         return count;
@@ -128,11 +136,6 @@ public class InternalStats extends InternalNumericMetricsAggregation.MultiValue
         return valueAsString(Metrics.sum.name());
     }
 
-    @Override
-    public Type type() {
-        return TYPE;
-    }
-
     @Override
     public double value(String name) {
         Metrics metrics = Metrics.valueOf(name);
@@ -163,32 +166,6 @@ public class InternalStats extends InternalNumericMetricsAggregation.MultiValue
         return new InternalStats(name, count, sum, min, max, format, pipelineAggregators(), getMetaData());
     }
 
-    @Override
-    protected void doReadFrom(StreamInput in) throws IOException {
-        format = in.readNamedWriteable(DocValueFormat.class);
-        count = in.readVLong();
-        min = in.readDouble();
-        max = in.readDouble();
-        sum = in.readDouble();
-        readOtherStatsFrom(in);
-    }
-
-    public void readOtherStatsFrom(StreamInput in) throws IOException {
-    }
-
-    @Override
-    protected void doWriteTo(StreamOutput out) throws IOException {
-        out.writeNamedWriteable(format);
-        out.writeVLong(count);
-        out.writeDouble(min);
-        out.writeDouble(max);
-        out.writeDouble(sum);
-        writeOtherStatsTo(out);
-    }
-
-    protected void writeOtherStatsTo(StreamOutput out) throws IOException {
-    }
-
     static class Fields {
         public static final String COUNT = "count";
         public static final String MIN = "min";
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregationBuilder.java
index 4fd80e04494..cac1d92101b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregationBuilder.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
@@ -36,18 +37,19 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import java.io.IOException;
 
 public class StatsAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, StatsAggregationBuilder> {
-    public static final String NAME = InternalStats.TYPE.name();
+    public static final String NAME = "stats";
+    private final static Type TYPE = new Type(NAME);
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     public StatsAggregationBuilder(String name) {
-        super(name, InternalStats.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public StatsAggregationBuilder(StreamInput in) throws IOException {
-        super(in, InternalStats.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregationBuilder.java
index a2b961f1fc3..a28508c2171 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregationBuilder.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
@@ -38,20 +39,21 @@ import java.util.Objects;
 
 public class ExtendedStatsAggregationBuilder
         extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, ExtendedStatsAggregationBuilder> {
-    public static final String NAME = InternalExtendedStats.TYPE.name();
+    public static final String NAME = "extended_stats";
+    public final static Type TYPE = new Type(NAME);
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     private double sigma = 2.0;
 
     public ExtendedStatsAggregationBuilder(String name) {
-        super(name, InternalExtendedStats.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public ExtendedStatsAggregationBuilder(StreamInput in) throws IOException {
-        super(in, InternalExtendedStats.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
         sigma = in.readDouble();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/InternalExtendedStats.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/InternalExtendedStats.java
index eff72797b58..d848001171c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/InternalExtendedStats.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/InternalExtendedStats.java
@@ -18,12 +18,10 @@
  */
 package org.elasticsearch.search.aggregations.metrics.stats.extended;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.metrics.stats.InternalStats;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -32,26 +30,7 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
-/**
-*
-*/
 public class InternalExtendedStats extends InternalStats implements ExtendedStats {
-
-    public final static Type TYPE = new Type("extended_stats", "estats");
-
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public InternalExtendedStats readResult(StreamInput in) throws IOException {
-            InternalExtendedStats result = new InternalExtendedStats();
-            result.readFrom(in);
-            return result;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
     enum Metrics {
 
         count, sum, min, max, avg, sum_of_squares, variance, std_deviation, std_upper, std_lower;
@@ -61,10 +40,8 @@ public class InternalExtendedStats extends InternalStats implements ExtendedStat
         }
     }
 
-    private double sumOfSqrs;
-    private double sigma;
-
-    protected InternalExtendedStats() {} // for serialization
+    private final double sumOfSqrs;
+    private final double sigma;
 
     public InternalExtendedStats(String name, long count, double sum, double min, double max, double sumOfSqrs, double sigma,
             DocValueFormat formatter, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
@@ -73,9 +50,24 @@ public class InternalExtendedStats extends InternalStats implements ExtendedStat
         this.sigma = sigma;
     }
 
+    /**
+     * Read from a stream.
+     */
+    public InternalExtendedStats(StreamInput in) throws IOException {
+        super(in);
+        sumOfSqrs = in.readDouble();
+        sigma = in.readDouble();
+    }
+
     @Override
-    public Type type() {
-        return TYPE;
+    protected void writeOtherStatsTo(StreamOutput out) throws IOException {
+        out.writeDouble(sumOfSqrs);
+        out.writeDouble(sigma);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ExtendedStatsAggregationBuilder.NAME;
     }
 
     @Override
@@ -157,19 +149,6 @@ public class InternalExtendedStats extends InternalStats implements ExtendedStat
                 format, pipelineAggregators(), getMetaData());
     }
 
-    @Override
-    public void readOtherStatsFrom(StreamInput in) throws IOException {
-        sumOfSqrs = in.readDouble();
-        sigma = in.readDouble();
-    }
-
-    @Override
-    protected void writeOtherStatsTo(StreamOutput out) throws IOException {
-        out.writeDouble(sumOfSqrs);
-        out.writeDouble(sigma);
-    }
-
-
     static class Fields {
         public static final String SUM_OF_SQRS = "sum_of_squares";
         public static final String SUM_OF_SQRS_AS_STRING = "sum_of_squares_as_string";
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregator.java
index 8f7ea955e91..8c43f7460dc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregator.java
@@ -21,6 +21,7 @@ package org.elasticsearch.search.aggregations.pipeline;
 
 
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
@@ -33,7 +34,8 @@ import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
 import java.io.IOException;
 import java.util.Map;
 
-public abstract class PipelineAggregator implements Streamable {
+public abstract class PipelineAggregator implements Streamable, NamedWriteable {
+    // NORELEASE remove Streamable
 
     /**
      * Parse the {@link PipelineAggregationBuilder} from a {@link QueryParseContext}.
@@ -73,6 +75,49 @@ public abstract class PipelineAggregator implements Streamable {
         this.metaData = metaData;
     }
 
+    /**
+     * Read from a stream.
+     */
+    protected PipelineAggregator(StreamInput in) throws IOException {
+        name = in.readString();
+        bucketsPaths = in.readStringArray();
+        metaData = in.readMap();
+    }
+
+    @Override
+    public final void readFrom(StreamInput in) throws IOException {
+        try {
+            getWriteableName(); // Throws UnsupportedOperationException if this aggregation should be read using old style Streams
+            assert false : "Used reading constructor instead";
+        } catch (UnsupportedOperationException e) {
+            // OK
+        }
+        name = in.readString();
+        bucketsPaths = in.readStringArray();
+        metaData = in.readMap();
+        doReadFrom(in);
+    }
+
+    protected void doReadFrom(StreamInput in) throws IOException {
+        throw new UnsupportedOperationException("Use reading constructor instead"); // NORELEASE remove when we remove Streamable
+    }
+
+    @Override
+    public final void writeTo(StreamOutput out) throws IOException {
+        out.writeString(name);  // NORELEASE remote writing the name - it is automatically handled with writeNamedWriteable
+        out.writeStringArray(bucketsPaths);
+        out.writeMap(metaData);
+        doWriteTo(out);
+    }
+
+    protected abstract void doWriteTo(StreamOutput out) throws IOException;
+
+    @Override
+    public String getWriteableName() {
+        // NORELEASE remove me when all InternalAggregations override it
+        throw new UnsupportedOperationException("Override on every class");
+    }
+
     public String name() {
         return name;
     }
@@ -85,27 +130,10 @@ public abstract class PipelineAggregator implements Streamable {
         return metaData;
     }
 
-    public abstract Type type();
+    public Type type() {
+        // NORELEASE remove this method
+        throw new UnsupportedOperationException(getClass().getName() + " used type but should Use getWriteableName instead");
+    }
 
     public abstract InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext);
-
-    @Override
-    public final void writeTo(StreamOutput out) throws IOException {
-        out.writeString(name);
-        out.writeStringArray(bucketsPaths);
-        out.writeMap(metaData);
-        doWriteTo(out);
-    }
-
-    protected abstract void doWriteTo(StreamOutput out) throws IOException;
-
-    @Override
-    public final void readFrom(StreamInput in) throws IOException {
-        name = in.readString();
-        bucketsPaths = in.readStringArray();
-        metaData = in.readMap();
-        doReadFrom(in);
-    }
-
-    protected abstract void doReadFrom(StreamInput in) throws IOException;
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/SiblingPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/SiblingPipelineAggregator.java
index cc4681c9035..f69e81fc8ba 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/SiblingPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/SiblingPipelineAggregator.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.search.aggregations.pipeline;
 
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext;
@@ -27,6 +28,7 @@ import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
 import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregation;
 import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -43,6 +45,13 @@ public abstract class SiblingPipelineAggregator extends PipelineAggregator {
         super(name, bucketsPaths, metaData);
     }
 
+    /**
+     * Read from a stream.
+     */
+    protected SiblingPipelineAggregator(StreamInput in) throws IOException {
+        super(in);
+    }
+
     @SuppressWarnings("unchecked")
     @Override
     public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsPipelineAggregator.java
index 06b093a4736..c09d303f4e4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/BucketMetricsPipelineAggregator.java
@@ -59,6 +59,35 @@ public abstract class BucketMetricsPipelineAggregator extends SiblingPipelineAgg
         this.format = format;
     }
 
+    /**
+     * Read from a stream.
+     */
+    protected BucketMetricsPipelineAggregator(StreamInput in) throws IOException {
+        super(in);
+        format = in.readNamedWriteable(DocValueFormat.class);
+        gapPolicy = GapPolicy.readFrom(in);
+    }
+
+    @Override
+    public final void doReadFrom(StreamInput in) throws IOException {
+        format = in.readNamedWriteable(DocValueFormat.class);
+        gapPolicy = GapPolicy.readFrom(in);
+        innerReadFrom(in);
+    }
+
+    protected void innerReadFrom(StreamInput in) throws IOException {
+    }
+
+    @Override
+    public final void doWriteTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(format);
+        gapPolicy.writeTo(out);
+        innerWriteTo(out);
+    }
+
+    protected void innerWriteTo(StreamOutput out) throws IOException {
+    }
+
     @Override
     public final InternalAggregation doReduce(Aggregations aggregations, ReduceContext context) {
         preCollection();
@@ -109,25 +138,4 @@ public abstract class BucketMetricsPipelineAggregator extends SiblingPipelineAgg
      *            for this bucket
      */
     protected abstract void collectBucketValue(String bucketKey, Double bucketValue);
-
-    @Override
-    public final void doReadFrom(StreamInput in) throws IOException {
-        format = in.readNamedWriteable(DocValueFormat.class);
-        gapPolicy = GapPolicy.readFrom(in);
-        innerReadFrom(in);
-    }
-
-    protected void innerReadFrom(StreamInput in) throws IOException {
-    }
-
-    @Override
-    public final void doWriteTo(StreamOutput out) throws IOException {
-        out.writeNamedWriteable(format);
-        gapPolicy.writeTo(out);
-        innerWriteTo(out);
-    }
-
-    protected void innerWriteTo(StreamOutput out) throws IOException {
-    }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/InternalStatsBucket.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/InternalStatsBucket.java
index 84f8b265756..371e5bf5e84 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/InternalStatsBucket.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/InternalStatsBucket.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats;
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.metrics.stats.InternalStats;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -31,34 +30,21 @@ import java.util.List;
 import java.util.Map;
 
 public class InternalStatsBucket extends InternalStats implements StatsBucket {
-
-    public final static Type TYPE = new Type("stats_bucket");
-
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public InternalStatsBucket readResult(StreamInput in) throws IOException {
-            InternalStatsBucket result = new InternalStatsBucket();
-            result.readFrom(in);
-            return result;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
     public InternalStatsBucket(String name, long count, double sum, double min, double max, DocValueFormat formatter,
                                List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
         super(name, count, sum, min, max, formatter, pipelineAggregators, metaData);
     }
 
-    InternalStatsBucket() {
-        // For serialization
+    /**
+     * Read from a stream.
+     */
+    public InternalStatsBucket(StreamInput in) throws IOException {
+        super(in);
     }
 
     @Override
-    public Type type() {
-        return TYPE;
+    public String getWriteableName() {
+        return StatsBucketPipelineAggregationBuilder.NAME;
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregationBuilder.java
index d3418e897ef..9e0432b2883 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregationBuilder.java
@@ -35,11 +35,11 @@ import java.util.List;
 import java.util.Map;
 
 public class StatsBucketPipelineAggregationBuilder extends BucketMetricsPipelineAggregationBuilder<StatsBucketPipelineAggregationBuilder> {
-    public static final String NAME = StatsBucketPipelineAggregator.TYPE.name();
+    public static final String NAME = "stats_bucket";
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     public StatsBucketPipelineAggregationBuilder(String name, String bucketsPath) {
-        super(name, StatsBucketPipelineAggregator.TYPE.name(), new String[] { bucketsPath });
+        super(name, NAME, new String[] { bucketsPath });
     }
 
     /**
@@ -47,7 +47,7 @@ public class StatsBucketPipelineAggregationBuilder extends BucketMetricsPipeline
      */
     public StatsBucketPipelineAggregationBuilder(StreamInput in)
             throws IOException {
-        super(in, StatsBucketPipelineAggregator.TYPE.name());
+        super(in, NAME);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregator.java
index 8be8a8462be..4b1febf4448 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/StatsBucketPipelineAggregator.java
@@ -22,10 +22,8 @@ package org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorStreams;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsPipelineAggregator;
 
 import java.io.IOException;
@@ -33,23 +31,6 @@ import java.util.List;
 import java.util.Map;
 
 public class StatsBucketPipelineAggregator extends BucketMetricsPipelineAggregator {
-
-    public final static Type TYPE = new Type("stats_bucket");
-
-    public final static PipelineAggregatorStreams.Stream STREAM = new PipelineAggregatorStreams.Stream() {
-        @Override
-        public StatsBucketPipelineAggregator readResult(StreamInput in) throws IOException {
-            StatsBucketPipelineAggregator result = new StatsBucketPipelineAggregator();
-            result.readFrom(in);
-            return result;
-        }
-    };
-
-    public static void registerStreams() {
-        PipelineAggregatorStreams.registerStream(STREAM, TYPE.stream());
-        InternalStatsBucket.registerStreams();
-    }
-
     private double sum = 0;
     private long count = 0;
     private double min = Double.POSITIVE_INFINITY;
@@ -60,13 +41,13 @@ public class StatsBucketPipelineAggregator extends BucketMetricsPipelineAggregat
         super(name, bucketsPaths, gapPolicy, formatter, metaData);
     }
 
-    StatsBucketPipelineAggregator() {
-        // For serialization
+    public StatsBucketPipelineAggregator(StreamInput in) throws IOException {
+        super(in);
     }
 
     @Override
-    public Type type() {
-        return TYPE;
+    public String getWriteableName() {
+        return StatsBucketPipelineAggregationBuilder.NAME;
     }
 
     @Override
@@ -89,5 +70,4 @@ public class StatsBucketPipelineAggregator extends BucketMetricsPipelineAggregat
     protected InternalAggregation buildAggregation(List<PipelineAggregator> pipelineAggregators, Map<String, Object> metadata) {
         return new InternalStatsBucket(name(), count, sum, min, max, format, pipelineAggregators, metadata);
     }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketPipelineAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketPipelineAggregationBuilder.java
index 792b9d1f7f1..82920be6cf1 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketPipelineAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketPipelineAggregationBuilder.java
@@ -36,20 +36,20 @@ import java.util.Objects;
 
 public class ExtendedStatsBucketPipelineAggregationBuilder
         extends BucketMetricsPipelineAggregationBuilder<ExtendedStatsBucketPipelineAggregationBuilder> {
-    public static final String NAME = ExtendedStatsBucketPipelineAggregator.TYPE.name();
+    public static final String NAME = "extended_stats_bucket";
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     private double sigma = 2.0;
 
     public ExtendedStatsBucketPipelineAggregationBuilder(String name, String bucketsPath) {
-        super(name, ExtendedStatsBucketPipelineAggregator.TYPE.name(), new String[] { bucketsPath });
+        super(name, NAME, new String[] { bucketsPath });
     }
 
     /**
      * Read from a stream.
      */
     public ExtendedStatsBucketPipelineAggregationBuilder(StreamInput in) throws IOException {
-        super(in, ExtendedStatsBucketPipelineAggregator.TYPE.name());
+        super(in, NAME);
         sigma = in.readDouble();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketPipelineAggregator.java
index d5e8628d6b5..03ed20d494f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/ExtendedStatsBucketPipelineAggregator.java
@@ -23,10 +23,8 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.InternalAggregation;
-import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
-import org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorStreams;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.BucketMetricsPipelineAggregator;
 
 import java.io.IOException;
@@ -34,29 +32,12 @@ import java.util.List;
 import java.util.Map;
 
 public class ExtendedStatsBucketPipelineAggregator extends BucketMetricsPipelineAggregator {
-
-    public final static Type TYPE = new Type("extended_stats_bucket");
-
-    public final static PipelineAggregatorStreams.Stream STREAM = new PipelineAggregatorStreams.Stream() {
-        @Override
-        public ExtendedStatsBucketPipelineAggregator readResult(StreamInput in) throws IOException {
-            ExtendedStatsBucketPipelineAggregator result = new ExtendedStatsBucketPipelineAggregator();
-            result.readFrom(in);
-            return result;
-        }
-    };
-
-    public static void registerStreams() {
-        PipelineAggregatorStreams.registerStream(STREAM, TYPE.stream());
-        InternalExtendedStatsBucket.registerStreams();
-    }
-
+    private final double sigma;
     private double sum = 0;
     private long count = 0;
     private double min = Double.POSITIVE_INFINITY;
     private double max = Double.NEGATIVE_INFINITY;
     private double sumOfSqrs = 1;
-    private double sigma;
 
     protected ExtendedStatsBucketPipelineAggregator(String name, String[] bucketsPaths, double sigma, GapPolicy gapPolicy,
                                                     DocValueFormat formatter, Map<String, Object> metaData) {
@@ -64,13 +45,22 @@ public class ExtendedStatsBucketPipelineAggregator extends BucketMetricsPipeline
         this.sigma = sigma;
     }
 
-    ExtendedStatsBucketPipelineAggregator() {
-        // For Serialization
+    /**
+     * Read from a stream.
+     */
+    public ExtendedStatsBucketPipelineAggregator(StreamInput in) throws IOException {
+        super(in);
+        sigma = in.readDouble();
     }
 
     @Override
-    public Type type() {
-        return TYPE;
+    protected void innerWriteTo(StreamOutput out) throws IOException {
+        out.writeDouble(sigma);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ExtendedStatsBucketPipelineAggregationBuilder.NAME;
     }
 
     @Override
@@ -95,14 +85,4 @@ public class ExtendedStatsBucketPipelineAggregator extends BucketMetricsPipeline
     protected InternalAggregation buildAggregation(List<PipelineAggregator> pipelineAggregators, Map<String, Object> metadata) {
         return new InternalExtendedStatsBucket(name(), count, sum, min, max, sumOfSqrs, sigma, format, pipelineAggregators, metadata);
     }
-
-    @Override
-    protected void innerReadFrom(StreamInput in) throws IOException {
-        sigma = in.readDouble();
-    }
-
-    @Override
-    protected void innerWriteTo(StreamOutput out) throws IOException {
-        out.writeDouble(sigma);
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/InternalExtendedStatsBucket.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/InternalExtendedStatsBucket.java
index 548e6443299..5589a9ebbcb 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/InternalExtendedStatsBucket.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/stats/extended/InternalExtendedStatsBucket.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.exten
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.metrics.stats.extended.InternalExtendedStats;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -31,35 +30,22 @@ import java.util.List;
 import java.util.Map;
 
 public class InternalExtendedStatsBucket extends InternalExtendedStats implements ExtendedStatsBucket {
-
-    public final static Type TYPE = new Type("extended_stats_bucket");
-
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public InternalExtendedStatsBucket readResult(StreamInput in) throws IOException {
-            InternalExtendedStatsBucket result = new InternalExtendedStatsBucket();
-            result.readFrom(in);
-            return result;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
     InternalExtendedStatsBucket(String name, long count, double sum, double min, double max, double sumOfSqrs, double sigma,
                                             DocValueFormat formatter, List<PipelineAggregator> pipelineAggregators,
                                             Map<String, Object> metaData) {
         super(name, count, sum, min, max, sumOfSqrs, sigma, formatter, pipelineAggregators, metaData);
     }
 
-    InternalExtendedStatsBucket() {
-        // for serialization
+    /**
+     * Read from a stream.
+     */
+    public InternalExtendedStatsBucket(StreamInput in) throws IOException {
+        super(in);
     }
 
     @Override
-    public Type type() {
-        return TYPE;
+    public String getWriteableName() {
+        return ExtendedStatsBucketPipelineAggregationBuilder.NAME;
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
index be8c895eecd..4fc814a0448 100644
--- a/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
+++ b/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
@@ -224,9 +224,14 @@ public class QuerySearchResult extends QuerySearchResultProvider {
             int size = in.readVInt();
             List<SiblingPipelineAggregator> pipelineAggregators = new ArrayList<>(size);
             for (int i = 0; i < size; i++) {
-                BytesReference type = in.readBytesReference();
-                PipelineAggregator pipelineAggregator = PipelineAggregatorStreams.stream(type).readResult(in);
-                pipelineAggregators.add((SiblingPipelineAggregator) pipelineAggregator);
+                // NORELEASE temporary hack to support old style streams and new style NamedWriteable at the same time
+                if (in.readBoolean()) {
+                    pipelineAggregators.add((SiblingPipelineAggregator) in.readNamedWriteable(PipelineAggregator.class));
+                } else {
+                    BytesReference type = in.readBytesReference();
+                    PipelineAggregator pipelineAggregator = PipelineAggregatorStreams.stream(type).readResult(in);
+                    pipelineAggregators.add((SiblingPipelineAggregator) pipelineAggregator);
+                }
             }
             this.pipelineAggregators = pipelineAggregators;
         }
@@ -273,8 +278,16 @@ public class QuerySearchResult extends QuerySearchResultProvider {
             out.writeBoolean(true);
             out.writeVInt(pipelineAggregators.size());
             for (PipelineAggregator pipelineAggregator : pipelineAggregators) {
-                out.writeBytesReference(pipelineAggregator.type().stream());
-                pipelineAggregator.writeTo(out);
+                // NORELEASE temporary hack to support old style streams and new style NamedWriteable
+                try {
+                    pipelineAggregator.getWriteableName(); // Throws UnsupportedOperationException if we should use old style streams.
+                    out.writeBoolean(true);
+                    out.writeNamedWriteable(pipelineAggregator);
+                } catch (UnsupportedOperationException e) {
+                    out.writeBoolean(false);
+                    out.writeBytesReference(pipelineAggregator.type().stream());
+                    pipelineAggregator.writeTo(out);
+                }
             }
         }
         if (suggest == null) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java
index e1441b0dc54..390501d2002 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java
@@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.pipeline.bucketmetrics;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryParseContext;
-import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregator;
 import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregationBuilder;
 
 import static org.hamcrest.Matchers.equalTo;
@@ -51,7 +50,7 @@ public class ExtendedStatsBucketTests extends AbstractBucketMetricsTestCase<Exte
         parser.nextToken(); // skip object start
 
         ExtendedStatsBucketPipelineAggregationBuilder builder = (ExtendedStatsBucketPipelineAggregationBuilder) aggParsers
-            .pipelineParser(ExtendedStatsBucketPipelineAggregator.TYPE.name(), parseFieldMatcher)
+            .pipelineParser(ExtendedStatsBucketPipelineAggregationBuilder.NAME, parseFieldMatcher)
             .parse("test", parseContext);
 
         assertThat(builder.sigma(), equalTo(5.0));

From 27e320d5ceb9636d0e855baaca1998afb0298561 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 29 Jun 2016 16:59:14 -0400
Subject: [PATCH 1181/1311] Migrate sum, min, and max aggs to NamedWriteable

---
 .../elasticsearch/search/SearchModule.java    |  9 +--
 .../aggregations/metrics/max/InternalMax.java | 61 +++++++-----------
 .../metrics/max/MaxAggregationBuilder.java    |  8 ++-
 .../aggregations/metrics/min/InternalMin.java | 62 +++++++------------
 .../metrics/min/MinAggregationBuilder.java    |  8 ++-
 .../aggregations/metrics/sum/InternalSum.java | 61 +++++++-----------
 .../metrics/sum/SumAggregationBuilder.java    |  8 ++-
 7 files changed, 81 insertions(+), 136 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java
index ee05a72e40a..c39848c347f 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -489,9 +489,9 @@ public class SearchModule extends AbstractModule {
 
     private void registerBuiltinAggregations() {
         registerAggregation(AvgAggregationBuilder::new, InternalAvg::new, new AvgParser(), AvgAggregationBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(SumAggregationBuilder::new, new SumParser(), SumAggregationBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(MinAggregationBuilder::new, new MinParser(), MinAggregationBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(MaxAggregationBuilder::new, new MaxParser(), MaxAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(SumAggregationBuilder::new, InternalSum::new, new SumParser(), SumAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(MinAggregationBuilder::new, InternalMin::new, new MinParser(), MinAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(MaxAggregationBuilder::new, InternalMax::new, new MaxParser(), MaxAggregationBuilder.AGGREGATION_NAME_FIELD);
         registerAggregation(StatsAggregationBuilder::new, InternalStats::new, new StatsParser(),
                 StatsAggregationBuilder.AGGREGATION_NAME_FIELD);
         registerAggregation(ExtendedStatsAggregationBuilder::new, InternalExtendedStats::new, new ExtendedStatsParser(),
@@ -721,9 +721,6 @@ public class SearchModule extends AbstractModule {
 
     static {
         // calcs
-        InternalSum.registerStreams();
-        InternalMin.registerStreams();
-        InternalMax.registerStreams();
         InternalValueCount.registerStreams();
         InternalTDigestPercentiles.registerStreams();
         InternalTDigestPercentileRanks.registerStreams();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/InternalMax.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/InternalMax.java
index 7047f28947e..c9d045a7235 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/InternalMax.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/InternalMax.java
@@ -22,7 +22,6 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -31,29 +30,8 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
-/**
-*
-*/
 public class InternalMax extends InternalNumericMetricsAggregation.SingleValue implements Max {
-
-    public final static Type TYPE = new Type("max");
-
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public InternalMax readResult(StreamInput in) throws IOException {
-            InternalMax result = new InternalMax();
-            result.readFrom(in);
-            return result;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
-    private double max;
-
-    InternalMax() {} // for serialization
+    private final double max;
 
     public InternalMax(String name, double max, DocValueFormat formatter, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) {
@@ -62,6 +40,26 @@ public class InternalMax extends InternalNumericMetricsAggregation.SingleValue i
         this.max = max;
     }
 
+    /**
+     * Read from a stream.
+     */
+    public InternalMax(StreamInput in) throws IOException {
+        super(in);
+        format = in.readNamedWriteable(DocValueFormat.class);
+        max = in.readDouble();
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(format);
+        out.writeDouble(max);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return MaxAggregationBuilder.NAME;
+    }
+
     @Override
     public double value() {
         return max;
@@ -72,11 +70,6 @@ public class InternalMax extends InternalNumericMetricsAggregation.SingleValue i
         return max;
     }
 
-    @Override
-    public Type type() {
-        return TYPE;
-    }
-
     @Override
     public InternalMax doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
         double max = Double.NEGATIVE_INFINITY;
@@ -86,18 +79,6 @@ public class InternalMax extends InternalNumericMetricsAggregation.SingleValue i
         return new InternalMax(name, max, format, pipelineAggregators(), getMetaData());
     }
 
-    @Override
-    protected void doReadFrom(StreamInput in) throws IOException {
-        format = in.readNamedWriteable(DocValueFormat.class);
-        max = in.readDouble();
-    }
-
-    @Override
-    protected void doWriteTo(StreamOutput out) throws IOException {
-        out.writeNamedWriteable(format);
-        out.writeDouble(max);
-    }
-
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         boolean hasValue = !Double.isInfinite(max);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregationBuilder.java
index 9fa919fcf9e..7158a065ac6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregationBuilder.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
@@ -36,18 +37,19 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import java.io.IOException;
 
 public class MaxAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, MaxAggregationBuilder> {
-    public static final String NAME = InternalMax.TYPE.name();
+    public static final String NAME = "max";
+    public final static Type TYPE = new Type(NAME);
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     public MaxAggregationBuilder(String name) {
-        super(name, InternalMax.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public MaxAggregationBuilder(StreamInput in) throws IOException {
-        super(in, InternalMax.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/InternalMin.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/InternalMin.java
index d729002e4d1..09f9c3915fd 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/InternalMin.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/InternalMin.java
@@ -22,7 +22,6 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -31,30 +30,8 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
-/**
-*
-*/
 public class InternalMin extends InternalNumericMetricsAggregation.SingleValue implements Min {
-
-    public final static Type TYPE = new Type("min");
-
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public InternalMin readResult(StreamInput in) throws IOException {
-            InternalMin result = new InternalMin();
-            result.readFrom(in);
-            return result;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
-
-    private double min;
-
-    InternalMin() {} // for serialization
+    private final double min;
 
     public InternalMin(String name, double min, DocValueFormat formatter, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) {
@@ -63,6 +40,26 @@ public class InternalMin extends InternalNumericMetricsAggregation.SingleValue i
         this.format = formatter;
     }
 
+    /**
+     * Read from a stream.
+     */
+    public InternalMin(StreamInput in) throws IOException {
+        super(in);
+        format = in.readNamedWriteable(DocValueFormat.class);
+        min = in.readDouble();
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(format);
+        out.writeDouble(min);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return MinAggregationBuilder.NAME;
+    }
+
     @Override
     public double value() {
         return min;
@@ -73,11 +70,6 @@ public class InternalMin extends InternalNumericMetricsAggregation.SingleValue i
         return min;
     }
 
-    @Override
-    public Type type() {
-        return TYPE;
-    }
-
     @Override
     public InternalMin doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
         double min = Double.POSITIVE_INFINITY;
@@ -87,18 +79,6 @@ public class InternalMin extends InternalNumericMetricsAggregation.SingleValue i
         return new InternalMin(getName(), min, this.format, pipelineAggregators(), getMetaData());
     }
 
-    @Override
-    protected void doReadFrom(StreamInput in) throws IOException {
-        format = in.readNamedWriteable(DocValueFormat.class);
-        min = in.readDouble();
-    }
-
-    @Override
-    protected void doWriteTo(StreamOutput out) throws IOException {
-        out.writeNamedWriteable(format);
-        out.writeDouble(min);
-    }
-
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         boolean hasValue = !Double.isInfinite(min);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregationBuilder.java
index af4f204bddb..b8a6f7ed904 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregationBuilder.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
@@ -36,18 +37,19 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import java.io.IOException;
 
 public class MinAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, MinAggregationBuilder> {
-    public static final String NAME = InternalMin.TYPE.name();
+    public static final String NAME = "min";
+    private final static Type TYPE = new Type(NAME);
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     public MinAggregationBuilder(String name) {
-        super(name, InternalMin.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public MinAggregationBuilder(StreamInput in) throws IOException {
-        super(in, InternalMin.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/InternalSum.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/InternalSum.java
index 0b76e39d208..70c69b65a17 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/InternalSum.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/InternalSum.java
@@ -22,7 +22,6 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -31,29 +30,8 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
-/**
-*
-*/
 public class InternalSum extends InternalNumericMetricsAggregation.SingleValue implements Sum {
-
-    public final static Type TYPE = new Type("sum");
-
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public InternalSum readResult(StreamInput in) throws IOException {
-            InternalSum result = new InternalSum();
-            result.readFrom(in);
-            return result;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
-    private double sum;
-
-    InternalSum() {} // for serialization
+    private final double sum;
 
     InternalSum(String name, double sum, DocValueFormat formatter, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) {
@@ -62,6 +40,26 @@ public class InternalSum extends InternalNumericMetricsAggregation.SingleValue i
         this.format = formatter;
     }
 
+    /**
+     * Read from a stream.
+     */
+    public InternalSum(StreamInput in) throws IOException {
+        super(in);
+        format = in.readNamedWriteable(DocValueFormat.class);
+        sum = in.readDouble();
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(format);
+        out.writeDouble(sum);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return SumAggregationBuilder.NAME;
+    }
+
     @Override
     public double value() {
         return sum;
@@ -72,11 +70,6 @@ public class InternalSum extends InternalNumericMetricsAggregation.SingleValue i
         return sum;
     }
 
-    @Override
-    public Type type() {
-        return TYPE;
-    }
-
     @Override
     public InternalSum doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
         double sum = 0;
@@ -86,18 +79,6 @@ public class InternalSum extends InternalNumericMetricsAggregation.SingleValue i
         return new InternalSum(name, sum, format, pipelineAggregators(), getMetaData());
     }
 
-    @Override
-    protected void doReadFrom(StreamInput in) throws IOException {
-        format = in.readNamedWriteable(DocValueFormat.class);
-        sum = in.readDouble();
-    }
-
-    @Override
-    protected void doWriteTo(StreamOutput out) throws IOException {
-        out.writeNamedWriteable(format);
-        out.writeDouble(sum);
-    }
-
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         builder.field(CommonFields.VALUE, sum);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregationBuilder.java
index 25dd1a3f214..8eafb22b6d0 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregationBuilder.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
@@ -36,18 +37,19 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import java.io.IOException;
 
 public class SumAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, SumAggregationBuilder> {
-    public static final String NAME = InternalSum.TYPE.name();
+    public static final String NAME = "sum";
+    private final static Type TYPE = new Type(NAME);
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     public SumAggregationBuilder(String name) {
-        super(name, InternalSum.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public SumAggregationBuilder(StreamInput in) throws IOException {
-        super(in, InternalSum.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     @Override

From 42addb5692ffdad4a553f4c9be2a6806a3063c36 Mon Sep 17 00:00:00 2001
From: Christoph Wurm <cwurm@users.noreply.github.com>
Date: Fri, 1 Jul 2016 15:49:43 +0200
Subject: [PATCH 1182/1311] Add ingest-useragent plugin (#19074)

---
 docs/plugins/ingest-useragent.asciidoc        |   74 +
 plugins/ingest-useragent/build.gradle         |   29 +
 .../useragent/IngestUserAgentPlugin.java      |   86 +
 .../ingest/useragent/UserAgentCache.java      |   66 +
 .../ingest/useragent/UserAgentParser.java     |  280 +
 .../ingest/useragent/UserAgentProcessor.java  |  242 +
 .../src/main/resources/regexes.yaml           | 4815 +++++++++++++++++
 .../UserAgentProcessorFactoryTests.java       |  176 +
 .../useragent/UserAgentProcessorTests.java    |  161 +
 .../ingest/useragent/UserAgentRestIT.java     |   40 +
 .../test/ingest-useragent/10_basic.yaml       |   11 +
 .../20_useragent_processor.yaml               |   86 +
 .../ingest-useragent/30_custom_regex.yaml     |   42 +
 .../ingest-useragent/test/test-regexes.yaml   |    3 +
 settings.gradle                               |    1 +
 15 files changed, 6112 insertions(+)
 create mode 100644 docs/plugins/ingest-useragent.asciidoc
 create mode 100644 plugins/ingest-useragent/build.gradle
 create mode 100644 plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java
 create mode 100644 plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentCache.java
 create mode 100644 plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java
 create mode 100644 plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
 create mode 100644 plugins/ingest-useragent/src/main/resources/regexes.yaml
 create mode 100644 plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java
 create mode 100644 plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java
 create mode 100644 plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentRestIT.java
 create mode 100644 plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/10_basic.yaml
 create mode 100644 plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yaml
 create mode 100644 plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml
 create mode 100644 plugins/ingest-useragent/test/test-regexes.yaml

diff --git a/docs/plugins/ingest-useragent.asciidoc b/docs/plugins/ingest-useragent.asciidoc
new file mode 100644
index 00000000000..8868a4cf711
--- /dev/null
+++ b/docs/plugins/ingest-useragent.asciidoc
@@ -0,0 +1,74 @@
+[[ingest-useragent]]
+=== Ingest Useragent Processor Plugin
+
+The Useragent processor extracts details from the user agent string a browser sends with its web requests. 
+This processor adds this information by default under the `useragent` field.
+
+The ingest-useragent plugin ships by default with the regexes.yaml made available by uap-java with an Apache 2.0 license. For more details see https://github.com/ua-parser/uap-core.
+
+[[ingest-useragent-install]]
+[float]
+==== Installation
+
+This plugin can be installed using the plugin manager:
+
+[source,sh]
+----------------------------------------------------------------
+sudo bin/elasticsearch-plugin install ingest-useragent
+----------------------------------------------------------------
+
+The plugin must be installed on every node in the cluster, and each node must
+be restarted after installation.
+
+[[ingest-useragent-remove]]
+[float]
+==== Removal
+
+The plugin can be removed with the following command:
+
+[source,sh]
+----------------------------------------------------------------
+sudo bin/elasticsearch-plugin remove ingest-useragent
+----------------------------------------------------------------
+
+The node must be stopped before removing the plugin.
+
+[[using-ingest-useragent]]
+==== Using the Useragent Processor in a Pipeline
+
+[[ingest-useragent-options]]
+.Useragent options
+[options="header"]
+|======
+| Name                   | Required  | Default                                                                            | Description
+| `field`                | yes       | -                                                                                  | The field containing the user agent string.
+| `target_field`         | no        | useragent                                                                          | The field that will be filled with the user agent details.
+| `regex_file`           | no        | -                                                                                  | The name of the file in the `config/ingest-useragent` directory containing the regular expressions for parsing the user agent string. Both the directory and the file have to be created before starting Elasticsearch. If not specified, ingest-useragent will use the regexes.yaml from uap-core it ships with (see below).
+| `properties`           | no        | [`name`, `major`, `minor`, `patch`, `build`, `os`, `os_name`, `os_major`, `os_minor`, `device`] | Controls what properties are added to `target_field`.
+|======
+
+Here is an example that adds the user agent details to the `useragent` field based on the `agent` field:
+
+[source,js]
+--------------------------------------------------
+{
+  "description" : "...",
+  "processors" : [
+    {
+      "useragent" : {
+        "field" : "agent"
+      }
+    }
+  ]
+}
+--------------------------------------------------
+
+===== Using a custom regex file
+To use a custom regex file for parsing the user agents, that file has to be put into the `config/ingest-useragent` directory and
+has to have a `.yaml` filename extension. The file has to be present at node startup, any changes to it or any new files added
+while the node is running will not have any effect.
+
+In practice, it will make most sense for any custom regex file to be a variant of the default file, either a more recent version
+or a customised version.
+
+The default file included in `ingest-useragent` is the `regexes.yaml` from uap-core: https://github.com/ua-parser/uap-core/blob/master/regexes.yaml 
diff --git a/plugins/ingest-useragent/build.gradle b/plugins/ingest-useragent/build.gradle
new file mode 100644
index 00000000000..86fbf243497
--- /dev/null
+++ b/plugins/ingest-useragent/build.gradle
@@ -0,0 +1,29 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ 
+esplugin {
+  description 'Ingest processor that extracts information from a user agent'
+  classname 'org.elasticsearch.ingest.useragent.IngestUserAgentPlugin'
+}
+
+integTest {
+  cluster {
+    extraConfigFile 'ingest-useragent/test-regexes.yaml', 'test/test-regexes.yaml'
+  }
+}
\ No newline at end of file
diff --git a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java
new file mode 100644
index 00000000000..e86a8443d7e
--- /dev/null
+++ b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.useragent;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.node.NodeModule;
+import org.elasticsearch.plugins.Plugin;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.PathMatcher;
+import java.nio.file.StandardOpenOption;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Stream;
+
+public class IngestUserAgentPlugin extends Plugin {
+
+    private final Setting<Long> CACHE_SIZE_SETTING = Setting.longSetting("ingest.useragent.cache_size", 1000, 0,
+            Setting.Property.NodeScope);
+
+    static final String DEFAULT_PARSER_NAME = "_default_";
+
+    public void onModule(NodeModule nodeModule) throws IOException {
+        Path userAgentConfigDirectory = nodeModule.getNode().getEnvironment().configFile().resolve("ingest-useragent");
+
+        if (Files.exists(userAgentConfigDirectory) == false && Files.isDirectory(userAgentConfigDirectory)) {
+            throw new IllegalStateException(
+                    "the user agent directory [" + userAgentConfigDirectory + "] containing the regex file doesn't exist");
+        }
+
+        long cacheSize = CACHE_SIZE_SETTING.get(nodeModule.getNode().settings());
+
+        UserAgentCache cache = new UserAgentCache(cacheSize);
+
+        Map<String, UserAgentParser> userAgentParsers = createUserAgentParsers(userAgentConfigDirectory, cache);
+
+        nodeModule.registerProcessor(UserAgentProcessor.TYPE, (registry) -> new UserAgentProcessor.Factory(userAgentParsers));
+    }
+
+    static Map<String, UserAgentParser> createUserAgentParsers(Path userAgentConfigDirectory, UserAgentCache cache) throws IOException {
+        Map<String, UserAgentParser> userAgentParsers = new HashMap<>();
+
+        UserAgentParser defaultParser = new UserAgentParser(DEFAULT_PARSER_NAME,
+                IngestUserAgentPlugin.class.getResourceAsStream("/regexes.yaml"), cache);
+        userAgentParsers.put(DEFAULT_PARSER_NAME, defaultParser);
+
+        if (Files.exists(userAgentConfigDirectory) && Files.isDirectory(userAgentConfigDirectory)) {
+            PathMatcher pathMatcher = userAgentConfigDirectory.getFileSystem().getPathMatcher("glob:**.yaml");
+    
+            try (Stream<Path> regexFiles = Files.find(userAgentConfigDirectory, 1,
+                    (path, attr) -> attr.isRegularFile() && pathMatcher.matches(path))) {
+                Iterable<Path> iterable = regexFiles::iterator;
+                for (Path path : iterable) {
+                    String parserName = path.getFileName().toString();
+                    try (InputStream regexStream = Files.newInputStream(path, StandardOpenOption.READ)) {
+                        userAgentParsers.put(parserName, new UserAgentParser(parserName, regexStream, cache));
+                    }
+                }
+            }
+        }
+
+        return Collections.unmodifiableMap(userAgentParsers);
+    }
+
+}
diff --git a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentCache.java b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentCache.java
new file mode 100644
index 00000000000..d1002f2df06
--- /dev/null
+++ b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentCache.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.useragent;
+
+import org.elasticsearch.common.cache.Cache;
+import org.elasticsearch.common.cache.CacheBuilder;
+import org.elasticsearch.ingest.useragent.UserAgentParser.Details;
+
+import java.util.Objects;
+
+class UserAgentCache {
+    private final Cache<CompositeCacheKey, Details> cache;
+    
+    UserAgentCache(long cacheSize) {
+        cache = CacheBuilder.<CompositeCacheKey, Details>builder().setMaximumWeight(cacheSize).build();
+    }
+    
+    public Details get(String parserName, String userAgent) {
+        return cache.get(new CompositeCacheKey(parserName, userAgent));
+    }
+    
+    public void put(String parserName, String userAgent, Details details) {
+        cache.put(new CompositeCacheKey(parserName, userAgent), details);
+    }
+    
+    private static final class CompositeCacheKey {
+        private final String parserName;
+        private final String userAgent;
+        
+        CompositeCacheKey(String parserName, String userAgent) {
+            this.parserName = parserName;
+            this.userAgent = userAgent;
+        }
+        
+        @Override
+        public boolean equals(Object obj) {
+            if(obj != null && obj instanceof CompositeCacheKey) {
+                CompositeCacheKey s = (CompositeCacheKey)obj;
+                return parserName.equals(s.parserName) && userAgent.equals(s.userAgent);
+            }
+            return false;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(parserName, userAgent);
+        }
+    }
+}
diff --git a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java
new file mode 100644
index 00000000000..af764d5baf2
--- /dev/null
+++ b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.useragent;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+final class UserAgentParser {
+
+    private final UserAgentCache cache;
+    private final List<UserAgentSubpattern> uaPatterns = new ArrayList<>();
+    private final List<UserAgentSubpattern> osPatterns = new ArrayList<>();
+    private final List<UserAgentSubpattern> devicePatterns = new ArrayList<>();
+    private final String name;
+    
+    public UserAgentParser(String name, InputStream regexStream, UserAgentCache cache) {
+        this.name = name;
+        this.cache = cache;
+        
+        try {
+            init(regexStream);
+        } catch (IOException e) {
+            throw new ElasticsearchParseException("error parsing regular expression file", e);
+        }
+    }
+
+    private void init(InputStream regexStream) throws IOException {
+        XContentParser yamlParser = XContentFactory.xContent(XContentType.YAML).createParser(regexStream);
+        
+        XContentParser.Token token = yamlParser.nextToken();
+        
+        if (token == XContentParser.Token.START_OBJECT) {
+            token = yamlParser.nextToken();
+            
+            for (; token != null; token = yamlParser.nextToken()) {
+                if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().equals("user_agent_parsers")) {
+                    List<Map<String, String>> parserConfigurations = readParserConfigurations(yamlParser);
+                    
+                    for (Map<String, String> map : parserConfigurations) {
+                        uaPatterns.add(new UserAgentSubpattern(compilePattern(map.get("regex"), map.get("regex_flag")),
+                                map.get("family_replacement"), map.get("v1_replacement"), map.get("v2_replacement"),
+                                map.get("v3_replacement"), map.get("v4_replacement")));
+                    }
+                }
+                else if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().equals("os_parsers")) {
+                    List<Map<String, String>> parserConfigurations = readParserConfigurations(yamlParser);
+                    
+                    for (Map<String, String> map : parserConfigurations) {
+                        osPatterns.add(new UserAgentSubpattern(compilePattern(map.get("regex"), map.get("regex_flag")),
+                                map.get("os_replacement"), map.get("os_v1_replacement"), map.get("os_v2_replacement"),
+                                map.get("os_v3_replacement"), map.get("os_v4_replacement")));
+                    }
+                }
+                else if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().equals("device_parsers")) {
+                    List<Map<String, String>> parserConfigurations = readParserConfigurations(yamlParser);
+                    
+                    for (Map<String, String> map : parserConfigurations) {
+                        devicePatterns.add(new UserAgentSubpattern(compilePattern(map.get("regex"), map.get("regex_flag")),
+                                map.get("device_replacement"), null, null, null, null));
+                    }
+                }
+            }
+        }
+        
+        if (uaPatterns.isEmpty() && osPatterns.isEmpty() && devicePatterns.isEmpty()) {
+            throw new ElasticsearchParseException("not a valid regular expression file");
+        }
+    }
+
+    private Pattern compilePattern(String regex, String regex_flag) {
+        // Only flag present in the current default regexes.yaml
+        if (regex_flag != null && regex_flag.equals("i")) {
+            return Pattern.compile(regex, Pattern.CASE_INSENSITIVE);
+        } else {
+            return Pattern.compile(regex);
+        }
+    }
+
+    private List<Map<String, String>> readParserConfigurations(XContentParser yamlParser) throws IOException {
+        List <Map<String, String>> patternList = new ArrayList<>();
+        
+        XContentParser.Token token = yamlParser.nextToken();
+        if (token != XContentParser.Token.START_ARRAY) {
+            throw new ElasticsearchParseException("malformed regular expression file, should continue with 'array' after 'object'");
+        }
+        
+        token = yamlParser.nextToken();
+        if (token != XContentParser.Token.START_OBJECT) {
+            throw new ElasticsearchParseException("malformed regular expression file, expecting 'object'");
+        }
+        
+        while (token == XContentParser.Token.START_OBJECT) {
+            token = yamlParser.nextToken();
+            
+            if (token != XContentParser.Token.FIELD_NAME) {
+                throw new ElasticsearchParseException("malformed regular expression file, should continue with 'field_name' after 'array'");
+            }
+            
+            Map<String, String> regexMap = new HashMap<>();
+            for (; token == XContentParser.Token.FIELD_NAME; token = yamlParser.nextToken()) {
+                String fieldName = yamlParser.currentName();
+                
+                token = yamlParser.nextToken();
+                String fieldValue = yamlParser.text();
+                regexMap.put(fieldName, fieldValue);
+            }
+
+            patternList.add(regexMap);
+            
+            token = yamlParser.nextToken();
+        }
+        
+        return patternList;
+    }
+
+    List<UserAgentSubpattern> getUaPatterns() {
+        return uaPatterns;
+    }
+
+    List<UserAgentSubpattern> getOsPatterns() {
+        return osPatterns;
+    }
+
+    List<UserAgentSubpattern> getDevicePatterns() {
+        return devicePatterns;
+    }
+    
+    String getName() {
+        return name;
+    }
+
+    public Details parse(String agentString) {
+        Details details = cache.get(name, agentString);;
+        
+        if (details == null) {
+            VersionedName userAgent = findMatch(uaPatterns, agentString);
+            VersionedName operatingSystem = findMatch(osPatterns, agentString);
+            VersionedName device = findMatch(devicePatterns, agentString);
+            
+            details = new Details(userAgent, operatingSystem, device);
+            
+            cache.put(name, agentString, details);
+        }
+
+        return details;
+    }
+
+    private VersionedName findMatch(List<UserAgentSubpattern> possiblePatterns, String agentString) {
+        VersionedName name;
+        for (UserAgentSubpattern pattern : possiblePatterns) {
+            name = pattern.match(agentString);
+
+            if (name != null) {
+                return name;
+            }
+        }
+        
+        return null;
+    }
+
+    static final class Details {
+        public final VersionedName userAgent;
+        public final VersionedName operatingSystem;
+        public final VersionedName device;
+        
+        public Details(VersionedName userAgent, VersionedName operatingSystem, VersionedName device) {
+            this.userAgent = userAgent;
+            this.operatingSystem = operatingSystem;
+            this.device = device;
+        }
+    }
+    
+    static final class VersionedName {
+        public final String name;
+        public final String major;
+        public final String minor;
+        public final String patch;
+        public final String build;
+        
+        public VersionedName(String name, String major, String minor, String patch, String build) {
+            this.name = name;
+            this.major = major;
+            this.minor = minor;
+            this.patch = patch;
+            this.build = build;
+        }
+    }
+    
+    /**
+     * One of: user agent, operating system, device
+     */
+    static final class UserAgentSubpattern {
+        private final Pattern pattern;
+        private final String nameReplacement, v1Replacement, v2Replacement, v3Replacement, v4Replacement;
+
+        public UserAgentSubpattern(Pattern pattern, String nameReplacement,
+                String v1Replacement, String v2Replacement, String v3Replacement, String v4Replacement) {
+          this.pattern = pattern;
+          this.nameReplacement = nameReplacement;
+          this.v1Replacement = v1Replacement;
+          this.v2Replacement = v2Replacement;
+          this.v3Replacement = v3Replacement;
+          this.v4Replacement = v4Replacement;
+        }
+
+        public VersionedName match(String agentString) {
+          String name = null, major = null, minor = null, patch = null, build = null;
+          Matcher matcher = pattern.matcher(agentString);
+
+          if (!matcher.find()) {
+            return null;
+          }
+
+          int groupCount = matcher.groupCount();
+
+          if (nameReplacement != null) {
+            if (nameReplacement.contains("$1") && groupCount >= 1 && matcher.group(1) != null) {
+              name = nameReplacement.replaceFirst("\\$1", Matcher.quoteReplacement(matcher.group(1)));
+            } else {
+              name = nameReplacement;
+            }
+          } else if (groupCount >= 1) {
+            name = matcher.group(1);
+          }
+
+          if (v1Replacement != null) {
+            major = v1Replacement;
+          } else if (groupCount >= 2) {
+            major = matcher.group(2);
+          }
+
+          if (v2Replacement != null) {
+            minor = v2Replacement;
+          } else if (groupCount >= 3) {
+            minor = matcher.group(3);
+          }
+          
+          if (v3Replacement != null) {
+              patch = v3Replacement;
+          } else if (groupCount >= 4) {
+              patch = matcher.group(4);
+          }
+          
+          if (v4Replacement != null) {
+              build = v4Replacement;
+          } else if (groupCount >= 5) {
+              build = matcher.group(5);
+          }
+          
+          return name == null ? null : new VersionedName(name, major, minor, patch, build);
+        }
+      }
+}
diff --git a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
new file mode 100644
index 00000000000..fb917b33c7c
--- /dev/null
+++ b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.useragent;
+
+import org.elasticsearch.ingest.AbstractProcessor;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.useragent.UserAgentParser.Details;
+import org.elasticsearch.ingest.useragent.UserAgentParser.VersionedName;
+
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
+import static org.elasticsearch.ingest.ConfigurationUtils.readOptionalList;
+import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
+import static org.elasticsearch.ingest.ConfigurationUtils.readOptionalStringProperty;
+
+public class UserAgentProcessor extends AbstractProcessor {
+
+    public static final String TYPE = "useragent";
+
+    private final String field;
+    private final String targetField;
+    private final Set<Property> properties;
+    
+    private final UserAgentParser parser;
+
+    public UserAgentProcessor(String tag, String field, String targetField, UserAgentParser parser, Set<Property> properties) {
+        super(tag);
+        this.field = field;
+        this.targetField = targetField;
+        this.parser = parser;
+        this.properties = properties;
+    }
+
+    @Override
+    public void execute(IngestDocument ingestDocument) throws Exception {
+        String userAgent = ingestDocument.getFieldValue(field, String.class);
+
+        Details uaClient = parser.parse(userAgent);
+
+        Map<String, Object> uaDetails = new HashMap<>();
+        for (Property property : this.properties) {
+            switch (property) {
+                case NAME:
+                    if (uaClient.userAgent != null && uaClient.userAgent.name != null) {
+                        uaDetails.put("name", uaClient.userAgent.name);
+                    }
+                    else {
+                        uaDetails.put("name", "Other");
+                    }
+                    break;
+                case MAJOR:
+                    if (uaClient.userAgent != null && uaClient.userAgent.major != null) {
+                        uaDetails.put("major", uaClient.userAgent.major);
+                    }
+                    break;
+                case MINOR:
+                    if (uaClient.userAgent != null && uaClient.userAgent.minor != null) {
+                        uaDetails.put("minor", uaClient.userAgent.minor);
+                    }
+                    break;
+                case PATCH:
+                    if (uaClient.userAgent != null && uaClient.userAgent.patch != null) {
+                        uaDetails.put("patch", uaClient.userAgent.patch);
+                    }
+                    break;
+                case BUILD:
+                    if (uaClient.userAgent != null && uaClient.userAgent.build != null) {
+                        uaDetails.put("build", uaClient.userAgent.build);
+                    }
+                    break;
+                case OS:
+                    if (uaClient.operatingSystem != null) {
+                        uaDetails.put("os", buildFullOSName(uaClient.operatingSystem));
+                    }
+                    else {
+                        uaDetails.put("os", "Other");
+                    }
+                    
+                    break;
+                case OS_NAME:
+                    if (uaClient.operatingSystem != null && uaClient.operatingSystem.name != null) {
+                        uaDetails.put("os_name", uaClient.operatingSystem.name);
+                    }
+                    else {
+                        uaDetails.put("os_name", "Other");
+                    }
+                    break;
+                case OS_MAJOR:
+                    if (uaClient.operatingSystem != null && uaClient.operatingSystem.major != null) {
+                        uaDetails.put("os_major", uaClient.operatingSystem.major);
+                    }
+                    break;
+                case OS_MINOR:
+                    if (uaClient.operatingSystem != null && uaClient.operatingSystem.minor != null) {
+                        uaDetails.put("os_minor", uaClient.operatingSystem.minor);
+                    }
+                    break;
+                case DEVICE:
+                    if (uaClient.device != null && uaClient.device.name != null) {
+                        uaDetails.put("device", uaClient.device.name);
+                    }
+                    else {
+                        uaDetails.put("device", "Other");
+                    }
+                    break;
+            }
+        }
+
+        ingestDocument.setFieldValue(targetField, uaDetails);
+    }
+
+    /** To maintain compatibility with logstash-filter-useragent */
+    private String buildFullOSName(VersionedName operatingSystem) {
+        if (operatingSystem == null || operatingSystem.name == null) {
+            return null;
+        }
+
+        StringBuilder sb = new StringBuilder(operatingSystem.name);
+
+        if (operatingSystem.major != null) {
+            sb.append(" ");
+            sb.append(operatingSystem.major);
+
+            if (operatingSystem.minor != null) {
+                sb.append(".");
+                sb.append(operatingSystem.minor);
+
+                if (operatingSystem.patch != null) {
+                    sb.append(".");
+                    sb.append(operatingSystem.patch);
+
+                    if (operatingSystem.build != null) {
+                        sb.append(".");
+                        sb.append(operatingSystem.build);
+                    }
+                }
+            }
+        }
+
+        return sb.toString();
+    }
+
+    @Override
+    public String getType() {
+        return TYPE;
+    }
+    
+    String getField() {
+        return field;
+    }
+
+    String getTargetField() {
+        return targetField;
+    }
+
+    Set<Property> getProperties() {
+        return properties;
+    }
+    
+    UserAgentParser getUaParser() {
+        return parser;
+    }
+
+    public static final class Factory extends AbstractProcessorFactory<UserAgentProcessor> {
+
+        private final Map<String, UserAgentParser> userAgentParsers;
+        
+        public Factory(Map<String, UserAgentParser> userAgentParsers) {
+            this.userAgentParsers = userAgentParsers;
+        }
+
+        @Override
+        public UserAgentProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+            String field = readStringProperty(TYPE, processorTag, config, "field");
+            String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "useragent");
+            String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME);
+            List<String> propertyNames = readOptionalList(TYPE, processorTag, config, "properties");
+
+            UserAgentParser parser = userAgentParsers.get(regexFilename);
+            if (parser == null) {
+                throw newConfigurationException(TYPE, processorTag,
+                        "regex_file", "regex file [" + regexFilename + "] doesn't exist (has to exist at node startup)");
+            }
+            
+            final Set<Property> properties;
+            if (propertyNames != null) {
+                properties = EnumSet.noneOf(Property.class);
+                for (String fieldName : propertyNames) {
+                    try {
+                        properties.add(Property.parseProperty(fieldName));
+                    } catch (IllegalArgumentException e) {
+                        throw newConfigurationException(TYPE, processorTag, "properties", e.getMessage());
+                    }
+                }
+            } else {
+                properties = EnumSet.allOf(Property.class);
+            }
+
+            return new UserAgentProcessor(processorTag, field, targetField, parser, properties);
+        }
+    }
+
+    enum Property {
+
+        NAME, MAJOR, MINOR, PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD;
+
+        public static Property parseProperty(String propertyName) {
+            try {
+                return valueOf(propertyName.toUpperCase(Locale.ROOT));
+            }
+            catch (IllegalArgumentException e) {
+                throw new IllegalArgumentException("illegal property value [" + propertyName + "]. valid values are " +
+                        Arrays.toString(EnumSet.allOf(Property.class).toArray()));
+            }
+        }
+    }
+}
diff --git a/plugins/ingest-useragent/src/main/resources/regexes.yaml b/plugins/ingest-useragent/src/main/resources/regexes.yaml
new file mode 100644
index 00000000000..cc28842308c
--- /dev/null
+++ b/plugins/ingest-useragent/src/main/resources/regexes.yaml
@@ -0,0 +1,4815 @@
+# Apache License, Version 2.0
+# ===========================
+# 
+# Copyright 2009 Google Inc.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+user_agent_parsers:
+  #### SPECIAL CASES TOP ####
+
+  # @note: iOS / OSX Applications
+  - regex: '(CFNetwork)(?:/(\d+)\.(\d+)\.?(\d+)?)?'
+    family_replacement: 'CFNetwork'
+
+  # Pingdom
+  - regex: '(Pingdom.com_bot_version_)(\d+)\.(\d+)'
+    family_replacement: 'PingdomBot'
+
+  # Facebook
+  - regex: '(facebookexternalhit)/(\d+)\.(\d+)'
+    family_replacement: 'FacebookBot'
+
+  # Google Plus
+  - regex: 'Google.*/\+/web/snippet'
+    family_replacement: 'GooglePlusBot'
+
+  # Twitter
+  - regex: '(Twitterbot)/(\d+)\.(\d+)'
+    family_replacement: 'TwitterBot'
+
+  # Bots Pattern '/name-0.0'
+  - regex: '/((?:Ant-)?Nutch|[A-z]+[Bb]ot|[A-z]+[Ss]pider|Axtaris|fetchurl|Isara|ShopSalad|Tailsweep)[ \-](\d+)(?:\.(\d+)(?:\.(\d+))?)?'
+  # Bots Pattern 'name/0.0'
+  - regex: '(008|Altresium|Argus|BaiduMobaider|BoardReader|DNSGroup|DataparkSearch|EDI|Goodzer|Grub|INGRID|Infohelfer|LinkedInBot|LOOQ|Nutch|PathDefender|Peew|PostPost|Steeler|Twitterbot|VSE|WebCrunch|WebZIP|Y!J-BR[A-Z]|YahooSeeker|envolk|sproose|wminer)/(\d+)(?:\.(\d+)(?:\.(\d+))?)?'
+
+  # MSIECrawler
+  - regex: '(MSIE) (\d+)\.(\d+)([a-z]\d?)?;.* MSIECrawler'
+    family_replacement: 'MSIECrawler'
+
+  # Downloader ...
+  - regex: '(Google-HTTP-Java-Client|Apache-HttpClient|http%20client|Python-urllib|HttpMonitor|TLSProber|WinHTTP|JNLP)(?:[ /](\d+)(?:\.(\d+)(?:\.(\d+))?)?)?'
+
+  # Bots
+  - regex: '(1470\.net crawler|50\.nu|8bo Crawler Bot|Aboundex|Accoona-[A-z]+-Agent|AdsBot-Google(?:-[a-z]+)?|altavista|AppEngine-Google|archive.*?\.org_bot|archiver|Ask Jeeves|[Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]+)*|bingbot|BingPreview|blitzbot|BlogBridge|BoardReader(?: [A-Za-z]+)*|boitho.com-dc|BotSeer|\b\w*favicon\w*\b|\bYeti(?:-[a-z]+)?|Catchpoint bot|[Cc]harlotte|Checklinks|clumboot|Comodo HTTP\(S\) Crawler|Comodo-Webinspector-Crawler|ConveraCrawler|CRAWL-E|CrawlConvera|Daumoa(?:-feedfetcher)?|Feed Seeker Bot|findlinks|Flamingo_SearchEngine|FollowSite Bot|furlbot|Genieo|gigabot|GomezAgent|gonzo1|(?:[a-zA-Z]+-)?Googlebot(?:-[a-zA-Z]+)?|Google SketchUp|grub-client|gsa-crawler|heritrix|HiddenMarket|holmes|HooWWWer|htdig|ia_archiver|ICC-Crawler|Icarus6j|ichiro(?:/mobile)?|IconSurf|IlTrovatore(?:-Setaccio)?|InfuzApp|Innovazion Crawler|InternetArchive|IP2[a-z]+Bot|jbot\b|KaloogaBot|Kraken|Kurzor|larbin|LEIA|LesnikBot|Linguee Bot|LinkAider|LinkedInBot|Lite Bot|Llaut|lycos|Mail\.RU_Bot|masidani_bot|Mediapartners-Google|Microsoft .*? Bot|mogimogi|mozDex|MJ12bot|msnbot(?:-media *)?|msrbot|netresearch|Netvibes|NewsGator[^/]*|^NING|Nutch[^/]*|Nymesis|ObjectsSearch|Orbiter|OOZBOT|PagePeeker|PagesInventory|PaxleFramework|Peeplo Screenshot Bot|PlantyNet_WebRobot|Pompos|Read%20Later|Reaper|RedCarpet|Retreiver|Riddler|Rival IQ|scooter|Scrapy|Scrubby|searchsight|seekbot|semanticdiscovery|Simpy|SimplePie|SEOstats|SimpleRSS|SiteCon|Slurp|snappy|Speedy Spider|Squrl Java|TheUsefulbot|ThumbShotsBot|Thumbshots\.ru|TwitterBot|URL2PNG|Vagabondo|VoilaBot|^vortex|Votay bot|^voyager|WASALive.Bot|Web-sniffer|WebThumb|WeSEE:[A-z]+|WhatWeb|WIRE|WordPress|Wotbox|www\.almaden\.ibm\.com|Xenu(?:.s)? Link Sleuth|Xerka [A-z]+Bot|yacy(?:bot)?|Yahoo[a-z]*Seeker|Yahoo! Slurp|Yandex\w+|YodaoBot(?:-[A-z]+)?|YottaaMonitor|Yowedo|^Zao|^Zao-Crawler|ZeBot_www\.ze\.bz|ZooShot|ZyBorg)(?:[ /]v?(\d+)(?:\.(\d+)(?:\.(\d+))?)?)?'
+
+  # Bots General matcher 'name/0.0'
+  - regex: '(?:\/[A-Za-z0-9\.]+)? *([A-Za-z0-9 \-_\!\[\]:]*(?:[Aa]rchiver|[Ii]ndexer|[Ss]craper|[Bb]ot|[Ss]pider|[Cc]rawl[a-z]*))/(\d+)(?:\.(\d+)(?:\.(\d+))?)?'
+  # Bots General matcher 'name 0.0'
+  - regex: '(?:\/[A-Za-z0-9\.]+)? *([A-Za-z0-9 _\!\[\]:]*(?:[Aa]rchiver|[Ii]ndexer|[Ss]craper|[Bb]ot|[Ss]pider|[Cc]rawl[a-z]*)) (\d+)(?:\.(\d+)(?:\.(\d+))?)?'
+  # Bots containing spider|scrape|bot(but not CUBOT)|Crawl
+  - regex: '((?:[A-z0-9]+|[A-z\-]+ ?)?(?: the )?(?:[Ss][Pp][Ii][Dd][Ee][Rr]|[Ss]crape|[A-Za-z0-9-]*(?:[^C][^Uu])[Bb]ot|[Cc][Rr][Aa][Ww][Ll])[A-z0-9]*)(?:(?:[ /]| v)(\d+)(?:\.(\d+)(?:\.(\d+))?)?)?'
+
+  # HbbTV standard defines what features the browser should understand.
+  # but it's like targeting "HTML5 browsers", effective browser support depends on the model
+  # See os_parsers if you want to target a specific TV
+  - regex: '(HbbTV)/(\d+)\.(\d+)\.(\d+) \('
+
+  # must go before Firefox to catch Chimera/SeaMonkey/Camino
+  - regex: '(Chimera|SeaMonkey|Camino)/(\d+)\.(\d+)\.?([ab]?\d+[a-z]*)?'
+
+  # Social Networks
+  # Facebook
+  - regex: '\[FB.*;(FBAV)/(\d+)(?:\.(\d+)(?:\.(\d)+)?)?'
+    family_replacement: 'Facebook'
+  # Pinterest
+  - regex: '\[(Pinterest)/[^\]]+\]'
+  - regex: '(Pinterest)(?: for Android(?: Tablet)?)?/(\d+)(?:\.(\d+)(?:\.(\d)+)?)?'
+
+  # Firefox
+  - regex: '(Pale[Mm]oon)/(\d+)\.(\d+)\.?(\d+)?'
+    family_replacement: 'Pale Moon (Firefox Variant)'
+  - regex: '(Fennec)/(\d+)\.(\d+)\.?([ab]?\d+[a-z]*)'
+    family_replacement: 'Firefox Mobile'
+  - regex: '(Fennec)/(\d+)\.(\d+)(pre)'
+    family_replacement: 'Firefox Mobile'
+  - regex: '(Fennec)/(\d+)\.(\d+)'
+    family_replacement: 'Firefox Mobile'
+  - regex: '(?:Mobile|Tablet);.*(Firefox)/(\d+)\.(\d+)'
+    family_replacement: 'Firefox Mobile'
+  - regex: '(Namoroka|Shiretoko|Minefield)/(\d+)\.(\d+)\.(\d+(?:pre)?)'
+    family_replacement: 'Firefox ($1)'
+  - regex: '(Firefox)/(\d+)\.(\d+)(a\d+[a-z]*)'
+    family_replacement: 'Firefox Alpha'
+  - regex: '(Firefox)/(\d+)\.(\d+)(b\d+[a-z]*)'
+    family_replacement: 'Firefox Beta'
+  - regex: '(Firefox)-(?:\d+\.\d+)?/(\d+)\.(\d+)(a\d+[a-z]*)'
+    family_replacement: 'Firefox Alpha'
+  - regex: '(Firefox)-(?:\d+\.\d+)?/(\d+)\.(\d+)(b\d+[a-z]*)'
+    family_replacement: 'Firefox Beta'
+  - regex: '(Namoroka|Shiretoko|Minefield)/(\d+)\.(\d+)([ab]\d+[a-z]*)?'
+    family_replacement: 'Firefox ($1)'
+  - regex: '(Firefox).*Tablet browser (\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'MicroB'
+  - regex: '(MozillaDeveloperPreview)/(\d+)\.(\d+)([ab]\d+[a-z]*)?'
+  - regex: '(FxiOS)/(\d+)\.(\d+)(\.(\d+))?(\.(\d+))?'
+    family_replacement: 'Firefox iOS'
+
+  # e.g.: Flock/2.0b2
+  - regex: '(Flock)/(\d+)\.(\d+)(b\d+?)'
+
+  # RockMelt
+  - regex: '(RockMelt)/(\d+)\.(\d+)\.(\d+)'
+
+  # e.g.: Fennec/0.9pre
+  - regex: '(Navigator)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Netscape'
+
+  - regex: '(Navigator)/(\d+)\.(\d+)([ab]\d+)'
+    family_replacement: 'Netscape'
+
+  - regex: '(Netscape6)/(\d+)\.(\d+)\.?([ab]?\d+)?'
+    family_replacement: 'Netscape'
+
+  - regex: '(MyIBrow)/(\d+)\.(\d+)'
+    family_replacement: 'My Internet Browser'
+
+  # Opera will stop at 9.80 and hide the real version in the Version string.
+  # see: http://dev.opera.com/articles/view/opera-ua-string-changes/
+  - regex: '(Opera Tablet).*Version/(\d+)\.(\d+)(?:\.(\d+))?'
+  - regex: '(Opera Mini)(?:/att)?/?(\d+)?(?:\.(\d+))?(?:\.(\d+))?'
+  - regex: '(Opera)/.+Opera Mobi.+Version/(\d+)\.(\d+)'
+    family_replacement: 'Opera Mobile'
+  - regex: '(Opera)/(\d+)\.(\d+).+Opera Mobi'
+    family_replacement: 'Opera Mobile'
+  - regex: 'Opera Mobi.+(Opera)(?:/|\s+)(\d+)\.(\d+)'
+    family_replacement: 'Opera Mobile'
+  - regex: 'Opera Mobi'
+    family_replacement: 'Opera Mobile'
+  - regex: '(Opera)/9.80.*Version/(\d+)\.(\d+)(?:\.(\d+))?'
+
+  # Opera 14 for Android uses a WebKit render engine.
+  - regex: '(?:Mobile Safari).*(OPR)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Opera Mobile'
+
+  # Opera >=15 for Desktop is similar to Chrome but includes an "OPR" Version string.
+  - regex: '(?:Chrome).*(OPR)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Opera'
+
+  # Opera Coast
+  - regex: '(Coast)/(\d+).(\d+).(\d+)'
+    family_replacement: 'Opera Coast'
+
+  # Opera Mini for iOS (from version 8.0.0)
+  - regex: '(OPiOS)/(\d+).(\d+).(\d+)'
+    family_replacement: 'Opera Mini'
+
+  # Palm WebOS looks a lot like Safari.
+  - regex: '(hpw|web)OS/(\d+)\.(\d+)(?:\.(\d+))?'
+    family_replacement: 'webOS Browser'
+
+  # LuaKit has no version info.
+  # http://luakit.org/projects/luakit/
+  - regex: '(luakit)'
+    family_replacement: 'LuaKit'
+
+  # Snowshoe
+  - regex: '(Snowshoe)/(\d+)\.(\d+).(\d+)'
+
+  # Lightning (for Thunderbird)
+  # http://www.mozilla.org/projects/calendar/lightning/
+  - regex: '(Lightning)/(\d+)\.(\d+)\.?((?:[ab]?\d+[a-z]*)|(?:\d*))'
+
+  # Swiftfox
+  - regex: '(Firefox)/(\d+)\.(\d+)\.(\d+(?:pre)?) \(Swiftfox\)'
+    family_replacement: 'Swiftfox'
+  - regex: '(Firefox)/(\d+)\.(\d+)([ab]\d+[a-z]*)? \(Swiftfox\)'
+    family_replacement: 'Swiftfox'
+
+  # Rekonq
+  - regex: '(rekonq)/(\d+)\.(\d+)\.?(\d+)? Safari'
+    family_replacement: 'Rekonq'
+  - regex: 'rekonq'
+    family_replacement: 'Rekonq'
+
+  # Conkeror lowercase/uppercase
+  # http://conkeror.org/
+  - regex: '(conkeror|Conkeror)/(\d+)\.(\d+)\.?(\d+)?'
+    family_replacement: 'Conkeror'
+
+  # catches lower case konqueror
+  - regex: '(konqueror)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Konqueror'
+
+  - regex: '(WeTab)-Browser'
+
+  - regex: '(Comodo_Dragon)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Comodo Dragon'
+
+  - regex: '(Symphony) (\d+).(\d+)'
+
+  - regex: '(Minimo)'
+
+  - regex: 'PLAYSTATION 3.+WebKit'
+    family_replacement: 'NetFront NX'
+  - regex: 'PLAYSTATION 3'
+    family_replacement: 'NetFront'
+  - regex: '(PlayStation Portable)'
+    family_replacement: 'NetFront'
+  - regex: '(PlayStation Vita)'
+    family_replacement: 'NetFront NX'
+
+  - regex: 'AppleWebKit.+ (NX)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'NetFront NX'
+  - regex: '(Nintendo 3DS)'
+    family_replacement: 'NetFront NX'
+
+  # Amazon Silk, should go before Safari and Chrome Mobile
+  - regex: '(Silk)/(\d+)\.(\d+)(?:\.([0-9\-]+))?'
+    family_replacement: 'Amazon Silk'
+
+
+  # @ref: http://www.puffinbrowser.com
+  - regex: '(Puffin)/(\d+)\.(\d+)(?:\.(\d+))?'
+
+  # Edge Mobile
+  - regex: 'Windows Phone .*(Edge)/(\d+)\.(\d+)'
+    family_replacement: 'Edge Mobile'
+
+  # Samsung Internet (based on Chrome, but lacking some features)
+  - regex: '(SamsungBrowser)/(\d+)\.(\d+)'
+    family_replacement: 'Samsung Internet'
+
+  # Chrome Mobile
+  - regex: '(CrMo)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Chrome Mobile'
+  - regex: '(CriOS)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Chrome Mobile iOS'
+  - regex: '(Chrome)/(\d+)\.(\d+)\.(\d+)\.(\d+) Mobile'
+    family_replacement: 'Chrome Mobile'
+
+  # Chrome Frame must come before MSIE.
+  - regex: '(chromeframe)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Chrome Frame'
+
+  # UC Browser
+  - regex: '(UCBrowser)[ /](\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'UC Browser'
+  - regex: '(UC Browser)[ /](\d+)\.(\d+)\.(\d+)'
+  - regex: '(UC Browser|UCBrowser|UCWEB)(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'UC Browser'
+
+  # Tizen Browser (second case included in browser/major.minor regex)
+  - regex: '(SLP Browser)/(\d+)\.(\d+)'
+    family_replacement: 'Tizen Browser'
+
+  # Sogou Explorer 2.X
+  - regex: '(SE 2\.X) MetaSr (\d+)\.(\d+)'
+    family_replacement: 'Sogou Explorer'
+
+  # Baidu Browsers (desktop spoofs chrome & IE, explorer is mobile)
+  - regex: '(baidubrowser)[/\s](\d+)'
+    family_replacement: 'Baidu Browser'
+  - regex: '(FlyFlow)/(\d+)\.(\d+)'
+    family_replacement: 'Baidu Explorer'
+
+  # QQ Browsers
+  - regex: '(MQQBrowser/Mini)(?:(\d+)(?:\.(\d+)(?:\.(\d+))?)?)?'
+    family_replacement: 'QQ Browser Mini'
+  - regex: '(MQQBrowser)(?:/(\d+)(?:\.(\d+)(?:\.(\d+))?)?)?'
+    family_replacement: 'QQ Browser Mobile'
+  - regex: '(QQBrowser)(?:/(\d+)(?:\.(\d+)\.(\d+)(?:\.(\d+))?)?)?'
+    family_replacement: 'QQ Browser'
+
+  # Rackspace Monitoring
+  - regex: '(Rackspace Monitoring)/(\d+)\.(\d+)'
+    family_replacement: 'RackspaceBot'
+
+  # PyAMF
+  - regex: '(PyAMF)/(\d+)\.(\d+)\.(\d+)'
+
+  # Yandex Browser
+  - regex: '(YaBrowser)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Yandex Browser'
+
+  # Mail.ru Amigo/Internet Browser (Chromium-based)
+  - regex: '(Chrome)/(\d+)\.(\d+)\.(\d+).* MRCHROME'
+    family_replacement: 'Mail.ru Chromium Browser'
+
+  # AOL Browser (IE-based)
+  - regex: '(AOL) (\d+)\.(\d+); AOLBuild (\d+)'
+
+  #### END SPECIAL CASES TOP ####
+
+  #### MAIN CASES - this catches > 50% of all browsers ####
+
+  # Browser/major_version.minor_version.beta_version
+  - regex: '(AdobeAIR|FireWeb|Jasmine|ANTGalio|Midori|Fresco|Lobo|PaleMoon|Maxthon|Lynx|OmniWeb|Dillo|Camino|Demeter|Fluid|Fennec|Epiphany|Shiira|Sunrise|Spotify|Flock|Netscape|Lunascape|WebPilot|NetFront|Netfront|Konqueror|SeaMonkey|Kazehakase|Vienna|Iceape|Iceweasel|IceWeasel|Iron|K-Meleon|Sleipnir|Galeon|GranParadiso|Opera Mini|iCab|NetNewsWire|ThunderBrowse|Iris|UP\.Browser|Bunjalloo|Google Earth|Raven for Mac|Openwave)/(\d+)\.(\d+)\.(\d+)'
+
+  # Outlook 2007
+  - regex: 'Microsoft Office Outlook 12\.\d+\.\d+|MSOffice 12'
+    family_replacement: 'Outlook'
+    v1_replacement: '2007'
+
+  # Outlook 2010
+  - regex: 'Microsoft Outlook 14\.\d+\.\d+|MSOffice 14'
+    family_replacement: 'Outlook'
+    v1_replacement: '2010'
+
+  # Outlook 2013
+  - regex: 'Microsoft Outlook 15\.\d+\.\d+'
+    family_replacement: 'Outlook'
+    v1_replacement: '2013'
+
+  # Outlook 2016
+  - regex: 'Microsoft Outlook (?:Mail )?16\.\d+\.\d+'
+    family_replacement: 'Outlook'
+    v1_replacement: '2016'
+
+  # Windows Live Mail
+  - regex: 'Outlook-Express\/7\.0.*'
+    family_replacement: 'Windows Live Mail'
+
+  # Apple Air Mail
+  - regex: '(Airmail) (\d+)\.(\d+)(?:\.(\d+))?'
+
+  # Thunderbird
+  - regex: '(Thunderbird)/(\d+)\.(\d+)\.(\d+(?:pre)?)'
+    family_replacement: 'Thunderbird'
+
+  # Vivaldi uses "Vivaldi"
+  - regex: '(Vivaldi)/(\d+)\.(\d+)\.(\d+)'
+
+  # Edge/major_version.minor_version
+  - regex: '(Edge)/(\d+)\.(\d+)'
+
+  # Brave Browser https://brave.com/
+  - regex: '(brave)/(\d+)\.(\d+)\.(\d+) Chrome'
+    family_replacement: 'Brave'
+
+  # Chrome/Chromium/major_version.minor_version.beta_version
+  - regex: '(Chromium|Chrome)/(\d+)\.(\d+)\.(\d+)'
+
+  # Dolphin Browser
+  # @ref: http://www.dolphin.com
+  - regex: '\b(Dolphin)(?: |HDCN/|/INT\-)(\d+)\.(\d+)\.?(\d+)?'
+
+  # Browser/major_version.minor_version
+  - regex: '(bingbot|Bolt|Jasmine|IceCat|Skyfire|Midori|Maxthon|Lynx|Arora|IBrowse|Dillo|Camino|Shiira|Fennec|Phoenix|Chrome|Flock|Netscape|Lunascape|Epiphany|WebPilot|Opera Mini|Opera|NetFront|Netfront|Konqueror|Googlebot|SeaMonkey|Kazehakase|Vienna|Iceape|Iceweasel|IceWeasel|Iron|K-Meleon|Sleipnir|Galeon|GranParadiso|iCab|iTunes|MacAppStore|NetNewsWire|Space Bison|Stainless|Orca|Dolfin|BOLT|Minimo|Tizen Browser|Polaris|Abrowser|Planetweb|ICE Browser|mDolphin|qutebrowser|Otter|QupZilla)/(\d+)\.(\d+)\.?(\d+)?'
+
+  # Chrome/Chromium/major_version.minor_version
+  - regex: '(Chromium|Chrome)/(\d+)\.(\d+)'
+
+  ##########
+  # IE Mobile needs to happen before Android to catch cases such as:
+  # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920)...
+  # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920; ANZ821)...
+  # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920; Orange)...
+  # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920; Vodafone)...
+  ##########
+
+  # IE Mobile
+  - regex: '(IEMobile)[ /](\d+)\.(\d+)'
+    family_replacement: 'IE Mobile'
+
+  # Browser major_version.minor_version.beta_version (space instead of slash)
+  - regex: '(iRider|Crazy Browser|SkipStone|iCab|Lunascape|Sleipnir|Maemo Browser) (\d+)\.(\d+)\.(\d+)'
+  # Browser major_version.minor_version (space instead of slash)
+  - regex: '(iCab|Lunascape|Opera|Android|Jasmine|Polaris) (\d+)\.(\d+)\.?(\d+)?'
+
+  # Kindle WebKit
+  - regex: '(Kindle)/(\d+)\.(\d+)'
+
+  # weird android UAs
+  - regex: '(Android) Donut'
+    v1_replacement: '1'
+    v2_replacement: '2'
+
+  - regex: '(Android) Eclair'
+    v1_replacement: '2'
+    v2_replacement: '1'
+
+  - regex: '(Android) Froyo'
+    v1_replacement: '2'
+    v2_replacement: '2'
+
+  - regex: '(Android) Gingerbread'
+    v1_replacement: '2'
+    v2_replacement: '3'
+
+  - regex: '(Android) Honeycomb'
+    v1_replacement: '3'
+
+  # desktop mode
+  # http://www.anandtech.com/show/3982/windows-phone-7-review
+  - regex: '(MSIE) (\d+)\.(\d+).*XBLWP7'
+    family_replacement: 'IE Large Screen'
+
+  #### END MAIN CASES ####
+
+  #### SPECIAL CASES ####
+  - regex: '(Obigo)InternetBrowser'
+  - regex: '(Obigo)\-Browser'
+  - regex: '(Obigo|OBIGO)[^\d]*(\d+)(?:.(\d+))?'
+    family_replacement: 'Obigo'
+
+  - regex: '(MAXTHON|Maxthon) (\d+)\.(\d+)'
+    family_replacement: 'Maxthon'
+  - regex: '(Maxthon|MyIE2|Uzbl|Shiira)'
+    v1_replacement: '0'
+
+  - regex: '(BrowseX) \((\d+)\.(\d+)\.(\d+)'
+
+  - regex: '(NCSA_Mosaic)/(\d+)\.(\d+)'
+    family_replacement: 'NCSA Mosaic'
+
+  # Polaris/d.d is above
+  - regex: '(POLARIS)/(\d+)\.(\d+)'
+    family_replacement: 'Polaris'
+  - regex: '(Embider)/(\d+)\.(\d+)'
+    family_replacement: 'Polaris'
+
+  - regex: '(BonEcho)/(\d+)\.(\d+)\.?([ab]?\d+)?'
+    family_replacement: 'Bon Echo'
+
+  # @note: iOS / OSX Applications
+  - regex: '(iPod|iPhone|iPad).+Version/(\d+)\.(\d+)(?:\.(\d+))?.* Safari'
+    family_replacement: 'Mobile Safari'
+  - regex: '(iPod|iPhone|iPad).+Version/(\d+)\.(\d+)(?:\.(\d+))?'
+    family_replacement: 'Mobile Safari UI/WKWebView'
+  - regex: '(iPod|iPhone|iPad);.*CPU.*OS (\d+)_(\d+)(?:_(\d+))?.*Mobile.* Safari'
+    family_replacement: 'Mobile Safari'
+  - regex: '(iPod|iPhone|iPad);.*CPU.*OS (\d+)_(\d+)(?:_(\d+))?.*Mobile'
+    family_replacement: 'Mobile Safari UI/WKWebView'
+  - regex: '(iPod|iPhone|iPad).* Safari'
+    family_replacement: 'Mobile Safari'
+  - regex: '(iPod|iPhone|iPad)'
+    family_replacement: 'Mobile Safari UI/WKWebView'
+
+  - regex: '(AvantGo) (\d+).(\d+)'
+
+  - regex: '(OneBrowser)/(\d+).(\d+)'
+    family_replacement: 'ONE Browser'
+
+  - regex: '(Avant)'
+    v1_replacement: '1'
+
+  # This is the Tesla Model S (see similar entry in device parsers)
+  - regex: '(QtCarBrowser)'
+    v1_replacement: '1'
+
+  - regex: '^(iBrowser/Mini)(\d+).(\d+)'
+    family_replacement: 'iBrowser Mini'
+  - regex: '^(iBrowser|iRAPP)/(\d+).(\d+)'
+
+  # nokia browsers
+  # based on: http://www.developer.nokia.com/Community/Wiki/User-Agent_headers_for_Nokia_devices
+  - regex: '^(Nokia)'
+    family_replacement: 'Nokia Services (WAP) Browser'
+  - regex: '(NokiaBrowser)/(\d+)\.(\d+).(\d+)\.(\d+)'
+    family_replacement: 'Nokia Browser'
+  - regex: '(NokiaBrowser)/(\d+)\.(\d+).(\d+)'
+    family_replacement: 'Nokia Browser'
+  - regex: '(NokiaBrowser)/(\d+)\.(\d+)'
+    family_replacement: 'Nokia Browser'
+  - regex: '(BrowserNG)/(\d+)\.(\d+).(\d+)'
+    family_replacement: 'Nokia Browser'
+  - regex: '(Series60)/5\.0'
+    family_replacement: 'Nokia Browser'
+    v1_replacement: '7'
+    v2_replacement: '0'
+  - regex: '(Series60)/(\d+)\.(\d+)'
+    family_replacement: 'Nokia OSS Browser'
+  - regex: '(S40OviBrowser)/(\d+)\.(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Ovi Browser'
+  - regex: '(Nokia)[EN]?(\d+)'
+
+  # BlackBerry devices
+  - regex: '(PlayBook).+RIM Tablet OS (\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'BlackBerry WebKit'
+  - regex: '(Black[bB]erry|BB10).+Version/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'BlackBerry WebKit'
+  - regex: '(Black[bB]erry)\s?(\d+)'
+    family_replacement: 'BlackBerry'
+
+  - regex: '(OmniWeb)/v(\d+)\.(\d+)'
+
+  - regex: '(Blazer)/(\d+)\.(\d+)'
+    family_replacement: 'Palm Blazer'
+
+  - regex: '(Pre)/(\d+)\.(\d+)'
+    family_replacement: 'Palm Pre'
+
+  # fork of Links
+  - regex: '(ELinks)/(\d+)\.(\d+)'
+  - regex: '(ELinks) \((\d+)\.(\d+)'
+  - regex: '(Links) \((\d+)\.(\d+)'
+
+  - regex: '(QtWeb) Internet Browser/(\d+)\.(\d+)'
+
+  #- regex: '\(iPad;.+(Version)/(\d+)\.(\d+)(?:\.(\d+))?.*Safari/'
+  #  family_replacement: 'iPad'
+
+  # Phantomjs, should go before Safari
+  - regex: '(PhantomJS)/(\d+)\.(\d+)\.(\d+)'
+
+  # WebKit Nightly
+  - regex: '(AppleWebKit)/(\d+)\.?(\d+)?\+ .* Safari'
+    family_replacement: 'WebKit Nightly'
+
+  # Safari
+  - regex: '(Version)/(\d+)\.(\d+)(?:\.(\d+))?.*Safari/'
+    family_replacement: 'Safari'
+  # Safari didn't provide "Version/d.d.d" prior to 3.0
+  - regex: '(Safari)/\d+'
+
+  - regex: '(OLPC)/Update(\d+)\.(\d+)'
+
+  - regex: '(OLPC)/Update()\.(\d+)'
+    v1_replacement: '0'
+
+  - regex: '(SEMC\-Browser)/(\d+)\.(\d+)'
+
+  - regex: '(Teleca)'
+    family_replacement: 'Teleca Browser'
+
+  - regex: '(Phantom)/V(\d+)\.(\d+)'
+    family_replacement: 'Phantom Browser'
+
+  - regex: 'Trident(.*)rv.(\d+)\.(\d+)'
+    family_replacement: 'IE'
+
+  # Espial
+  - regex: '(Espial)/(\d+)(?:\.(\d+))?(?:\.(\d+))?'
+
+ # Apple Mail
+
+  # apple mail - not directly detectable, have it after Safari stuff
+  - regex: '(AppleWebKit)/(\d+)\.(\d+)\.(\d+)'
+    family_replacement: 'Apple Mail'
+
+  # AFTER THE EDGE CASES ABOVE!
+  # AFTER IE11
+  # BEFORE all other IE
+  - regex: '(Firefox)/(\d+)\.(\d+)\.(\d+)'
+  - regex: '(Firefox)/(\d+)\.(\d+)(pre|[ab]\d+[a-z]*)?'
+
+  - regex: '([MS]?IE) (\d+)\.(\d+)'
+    family_replacement: 'IE'
+
+  - regex: '(python-requests)/(\d+)\.(\d+)'
+    family_replacement: 'Python Requests'
+
+  - regex: '(Java)[/ ]{0,1}\d+\.(\d+)\.(\d+)[_-]*([a-zA-Z0-9]+)*'
+
+  # Roku Digital-Video-Players https://www.roku.com/
+  - regex: '^(Roku)/DVP-(\d+)\.(\d+)'
+
+os_parsers:
+  ##########
+  # HbbTV vendors
+  ##########
+
+  # starts with the easy one : Panasonic seems consistent across years, hope it will continue
+  #HbbTV/1.1.1 (;Panasonic;VIERA 2011;f.532;0071-0802 2000-0000;)
+  #HbbTV/1.1.1 (;Panasonic;VIERA 2012;1.261;0071-3103 2000-0000;)
+  #HbbTV/1.2.1 (;Panasonic;VIERA 2013;3.672;4101-0003 0002-0000;)
+  #- regex: 'HbbTV/\d+\.\d+\.\d+ \(;(Panasonic);VIERA ([0-9]{4});'
+
+  # Sony is consistent too but do not place year like the other
+  # Opera/9.80 (Linux armv7l; HbbTV/1.1.1 (; Sony; KDL32W650A; PKG3.211EUA; 2013;); ) Presto/2.12.362 Version/12.11
+  # Opera/9.80 (Linux mips; U;  HbbTV/1.1.1 (; Sony; KDL40HX751; PKG1.902EUA; 2012;);; en) Presto/2.10.250 Version/11.60
+  # Opera/9.80 (Linux mips; U;  HbbTV/1.1.1 (; Sony; KDL22EX320; PKG4.017EUA; 2011;);; en) Presto/2.7.61 Version/11.00
+  #- regex: 'HbbTV/\d+\.\d+\.\d+ \(; (Sony);.*;.*; ([0-9]{4});\)'
+
+
+  # LG is consistent too, but we need to add manually the year model
+  #Mozilla/5.0 (Unknown; Linux armv7l) AppleWebKit/537.1+ (KHTML, like Gecko) Safari/537.1+ HbbTV/1.1.1 ( ;LGE ;NetCast 4.0 ;03.20.30 ;1.0M ;)
+  #Mozilla/5.0 (DirectFB; Linux armv7l) AppleWebKit/534.26+ (KHTML, like Gecko) Version/5.0 Safari/534.26+ HbbTV/1.1.1 ( ;LGE ;NetCast 3.0 ;1.0 ;1.0M ;)
+  - regex: 'HbbTV/\d+\.\d+\.\d+ \( ;(LG)E ;NetCast 4.0'
+    os_v1_replacement: '2013'
+  - regex: 'HbbTV/\d+\.\d+\.\d+ \( ;(LG)E ;NetCast 3.0'
+    os_v1_replacement: '2012'
+
+  # Samsung is on its way of normalizing their user-agent
+  # HbbTV/1.1.1 (;Samsung;SmartTV2013;T-FXPDEUC-1102.2;;) WebKit
+  # HbbTV/1.1.1 (;Samsung;SmartTV2013;T-MST12DEUC-1102.1;;) WebKit
+  # HbbTV/1.1.1 (;Samsung;SmartTV2012;;;) WebKit
+  # HbbTV/1.1.1 (;;;;;) Maple_2011
+  - regex: 'HbbTV/1.1.1 \(;;;;;\) Maple_2011'
+    os_replacement: 'Samsung'
+    os_v1_replacement: '2011'
+  # manage the two models of 2013
+  - regex: 'HbbTV/\d+\.\d+\.\d+ \(;(Samsung);SmartTV([0-9]{4});.*FXPDEUC'
+    os_v2_replacement: 'UE40F7000'
+  - regex: 'HbbTV/\d+\.\d+\.\d+ \(;(Samsung);SmartTV([0-9]{4});.*MST12DEUC'
+    os_v2_replacement: 'UE32F4500'
+  # generic Samsung (works starting in 2012)
+  #- regex: 'HbbTV/\d+\.\d+\.\d+ \(;(Samsung);SmartTV([0-9]{4});'
+
+  # Philips : not found any other way than a manual mapping
+  # Opera/9.80 (Linux mips; U; HbbTV/1.1.1 (; Philips; ; ; ; ) CE-HTML/1.0 NETTV/4.1.3 PHILIPSTV/1.1.1; en) Presto/2.10.250 Version/11.60
+  # Opera/9.80 (Linux mips ; U; HbbTV/1.1.1 (; Philips; ; ; ; ) CE-HTML/1.0 NETTV/3.2.1; en) Presto/2.6.33 Version/10.70
+  - regex: 'HbbTV/1.1.1 \(; (Philips);.*NETTV/4'
+    os_v1_replacement: '2013'
+  - regex: 'HbbTV/1.1.1 \(; (Philips);.*NETTV/3'
+    os_v1_replacement: '2012'
+  - regex: 'HbbTV/1.1.1 \(; (Philips);.*NETTV/2'
+    os_v1_replacement: '2011'
+
+  # the HbbTV emulator developers use HbbTV/1.1.1 (;;;;;) firetv-firefox-plugin 1.1.20
+  - regex: 'HbbTV/\d+\.\d+\.\d+.*(firetv)-firefox-plugin (\d+).(\d+).(\d+)'
+    os_replacement: 'FireHbbTV'
+
+  # generic HbbTV, hoping to catch manufacturer name (always after 2nd comma) and the first string that looks like a 2011-2019 year
+  - regex: 'HbbTV/\d+\.\d+\.\d+ \(.*; ?([a-zA-Z]+) ?;.*(201[1-9]).*\)'
+
+  ##########
+  # @note: Windows Phone needs to come before Windows NT 6.1 *and* before Android to catch cases such as:
+  # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920)...
+  # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920; ANZ821)...
+  # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920; Orange)...
+  # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920; Vodafone)...
+  ##########
+
+  - regex: '(Windows Phone) (?:OS[ /])?(\d+)\.(\d+)'
+
+  ##########
+  # Android
+  # can actually detect rooted android os. do we care?
+  ##########
+  - regex: '(Android)[ \-/](\d+)\.(\d+)(?:[.\-]([a-z0-9]+))?'
+
+  - regex: '(Android) Donut'
+    os_v1_replacement: '1'
+    os_v2_replacement: '2'
+
+  - regex: '(Android) Eclair'
+    os_v1_replacement: '2'
+    os_v2_replacement: '1'
+
+  - regex: '(Android) Froyo'
+    os_v1_replacement: '2'
+    os_v2_replacement: '2'
+
+  - regex: '(Android) Gingerbread'
+    os_v1_replacement: '2'
+    os_v2_replacement: '3'
+
+  - regex: '(Android) Honeycomb'
+    os_v1_replacement: '3'
+
+  # UCWEB
+  - regex: '^UCWEB.*; (Adr) (\d+)\.(\d+)(?:[.\-]([a-z0-9]+))?;'
+    os_replacement: 'Android'
+  - regex: '^UCWEB.*; (iPad OS|iPh OS) (\d+)_(\d+)(?:_(\d+))?;'
+    os_replacement: 'iOS'
+  - regex: '^UCWEB.*; (wds) (\d+)\.(\d+)(?:\.(\d+))?;'
+    os_replacement: 'Windows Phone'
+  # JUC
+  - regex: '^(JUC).*; ?U; ?(?:Android)?(\d+)\.(\d+)(?:[\.\-]([a-z0-9]+))?'
+    os_replacement: 'Android'
+
+  ##########
+  # Kindle Android
+  ##########
+  - regex: '(Silk-Accelerated=[a-z]{4,5})'
+    os_replacement: 'Android'
+
+  ##########
+  # Windows
+  # http://en.wikipedia.org/wiki/Windows_NT#Releases
+  # possibility of false positive when different marketing names share same NT kernel
+  # e.g. windows server 2003 and windows xp
+  # lots of ua strings have Windows NT 4.1 !?!?!?!? !?!? !? !????!?! !!! ??? !?!?! ?
+  # (very) roughly ordered in terms of frequency of occurence of regex (win xp currently most frequent, etc)
+  ##########
+
+  # ie mobile desktop mode
+  # spoofs nt 6.1. must come before windows 7
+  - regex: '(XBLWP7)'
+    os_replacement: 'Windows Phone'
+
+  # @note: This needs to come before Windows NT 6.1
+  - regex: '(Windows ?Mobile)'
+    os_replacement: 'Windows Mobile'
+
+  - regex: '(Windows (?:NT 5\.2|NT 5\.1))'
+    os_replacement: 'Windows XP'
+
+  - regex: '(Windows NT 6\.1)'
+    os_replacement: 'Windows 7'
+
+  - regex: '(Windows NT 6\.0)'
+    os_replacement: 'Windows Vista'
+
+  - regex: '(Win 9x 4\.90)'
+    os_replacement: 'Windows ME'
+
+  - regex: '(Windows 98|Windows XP|Windows ME|Windows 95|Windows CE|Windows 7|Windows NT 4\.0|Windows Vista|Windows 2000|Windows 3.1)'
+
+  - regex: '(Windows NT 6\.2; ARM;)'
+    os_replacement: 'Windows RT'
+  - regex: '(Windows NT 6\.2)'
+    os_replacement: 'Windows 8'
+
+  - regex: '(Windows NT 6\.3; ARM;)'
+    os_replacement: 'Windows RT 8.1'
+  - regex: '(Windows NT 6\.3)'
+    os_replacement: 'Windows 8.1'
+
+  - regex: '(Windows NT 6\.4)'
+    os_replacement: 'Windows 10'
+  - regex: '(Windows NT 10\.0)'
+    os_replacement: 'Windows 10'
+
+  - regex: '(Windows NT 5\.0)'
+    os_replacement: 'Windows 2000'
+
+  - regex: '(WinNT4.0)'
+    os_replacement: 'Windows NT 4.0'
+
+  - regex: '(Windows ?CE)'
+    os_replacement: 'Windows CE'
+
+  - regex: 'Win ?(95|98|3.1|NT|ME|2000)'
+    os_replacement: 'Windows $1'
+
+  - regex: 'Win16'
+    os_replacement: 'Windows 3.1'
+
+  - regex: 'Win32'
+    os_replacement: 'Windows 95'
+
+  ##########
+  # Tizen OS from Samsung
+  # spoofs Android so pushing it above
+  ##########
+  - regex: '(Tizen)/(\d+)\.(\d+)'
+
+  ##########
+  # Mac OS
+  # @ref: http://en.wikipedia.org/wiki/Mac_OS_X#Versions
+  # @ref: http://www.puredarwin.org/curious/versions
+  ##########
+  - regex: '((?:Mac ?|; )OS X)[\s/](?:(\d+)[_.](\d+)(?:[_.](\d+))?|Mach-O)'
+    os_replacement: 'Mac OS X'
+  # Leopard
+  - regex: ' (Dar)(win)/(9).(\d+).*\((?:i386|x86_64|Power Macintosh)\)'
+    os_replacement: 'Mac OS X'
+    os_v1_replacement: '10'
+    os_v2_replacement: '5'
+  # Snow Leopard
+  - regex: ' (Dar)(win)/(10).(\d+).*\((?:i386|x86_64)\)'
+    os_replacement: 'Mac OS X'
+    os_v1_replacement: '10'
+    os_v2_replacement: '6'
+  # Lion
+  - regex: ' (Dar)(win)/(11).(\d+).*\((?:i386|x86_64)\)'
+    os_replacement: 'Mac OS X'
+    os_v1_replacement: '10'
+    os_v2_replacement: '7'
+  # Mountain Lion
+  - regex: ' (Dar)(win)/(12).(\d+).*\((?:i386|x86_64)\)'
+    os_replacement: 'Mac OS X'
+    os_v1_replacement: '10'
+    os_v2_replacement: '8'
+  # Mavericks
+  - regex: ' (Dar)(win)/(13).(\d+).*\((?:i386|x86_64)\)'
+    os_replacement: 'Mac OS X'
+    os_v1_replacement: '10'
+    os_v2_replacement: '9'
+  # Yosemite is Darwin/14.x but patch versions are inconsistent in the Darwin string;
+  # more accurately covered by CFNetwork regexes downstream
+
+  # IE on Mac doesn't specify version number
+  - regex: 'Mac_PowerPC'
+    os_replacement: 'Mac OS'
+
+  # builds before tiger don't seem to specify version?
+
+  # ios devices spoof (mac os x), so including intel/ppc prefixes
+  - regex: '(?:PPC|Intel) (Mac OS X)'
+
+  ##########
+  # iOS
+  # http://en.wikipedia.org/wiki/IOS_version_history
+  ##########
+  # keep this above generic iOS, since AppleTV UAs contain 'CPU OS'
+  - regex: '(Apple\s?TV)(?:/(\d+)\.(\d+))?'
+    os_replacement: 'ATV OS X'
+
+  - regex: '(CPU OS|iPhone OS|CPU iPhone) +(\d+)[_\.](\d+)(?:[_\.](\d+))?'
+    os_replacement: 'iOS'
+
+  # remaining cases are mostly only opera uas, so catch opera as to not catch iphone spoofs
+  - regex: '(iPhone|iPad|iPod); Opera'
+    os_replacement: 'iOS'
+
+  # few more stragglers
+  - regex: '(iPhone|iPad|iPod).*Mac OS X.*Version/(\d+)\.(\d+)'
+    os_replacement: 'iOS'
+
+  # CFNetwork/Darwin - The specific CFNetwork or Darwin version determines
+  # whether the os maps to Mac OS, or iOS, or just Darwin.
+  # See: http://user-agents.me/cfnetwork-version-list
+  - regex: '(CFNetwork)/(5)48\.0\.3.* Darwin/11\.0\.0'
+    os_replacement: 'iOS'
+  - regex: '(CFNetwork)/(5)48\.(0)\.4.* Darwin/(1)1\.0\.0'
+    os_replacement: 'iOS'
+  - regex: '(CFNetwork)/(5)48\.(1)\.4'
+    os_replacement: 'iOS'
+  - regex: '(CFNetwork)/(4)85\.1(3)\.9'
+    os_replacement: 'iOS'
+  - regex: '(CFNetwork)/(6)09\.(1)\.4'
+    os_replacement: 'iOS'
+  - regex: '(CFNetwork)/(6)(0)9'
+    os_replacement: 'iOS'
+  - regex: '(CFNetwork)/6(7)2\.(1)\.13'
+    os_replacement: 'iOS'
+  - regex: '(CFNetwork)/6(7)2\.(1)\.(1)4'
+    os_replacement: 'iOS'
+  - regex: '(CF)(Network)/6(7)(2)\.1\.15'
+    os_replacement: 'iOS'
+    os_v1_replacement: '7'
+    os_v2_replacement: '1'
+  - regex: '(CFNetwork)/6(7)2\.(0)\.(?:2|8)'
+    os_replacement: 'iOS'
+  - regex: '(CFNetwork)/709\.1'
+    os_replacement: 'iOS'
+    os_v1_replacement: '8'
+    os_v2_replacement: '0.b5'
+  - regex: '(CF)(Network)/711\.(\d)'
+    os_replacement: 'iOS'
+    os_v1_replacement: '8'
+  - regex: '(CF)(Network)/(720)\.(\d)'
+    os_replacement: 'Mac OS X'
+    os_v1_replacement: '10'
+    os_v2_replacement: '10'
+  - regex: '(CF)(Network)/758\.(\d)'
+    os_replacement: 'iOS'
+    os_v1_replacement: '9'
+
+  ##########
+  # CFNetwork iOS Apps
+  # @ref: https://en.wikipedia.org/wiki/Darwin_(operating_system)#Release_history
+  ##########
+  - regex: 'CFNetwork/.* Darwin/(9)\.\d+'
+    os_replacement: 'iOS'
+    os_v1_replacement: '1'
+  - regex: 'CFNetwork/.* Darwin/(10)\.\d+'
+    os_replacement: 'iOS'
+    os_v1_replacement: '4'
+  - regex: 'CFNetwork/.* Darwin/(11)\.\d+'
+    os_replacement: 'iOS'
+    os_v1_replacement: '5'
+  - regex: 'CFNetwork/.* Darwin/(13)\.\d+'
+    os_replacement: 'iOS'
+    os_v1_replacement: '6'
+  - regex: 'CFNetwork/6.* Darwin/(14)\.\d+'
+    os_replacement: 'iOS'
+    os_v1_replacement: '7'
+  - regex: 'CFNetwork/7.* Darwin/(14)\.\d+'
+    os_replacement: 'iOS'
+    os_v1_replacement: '8'
+    os_v2_replacement: '0'
+  - regex: 'CFNetwork/7.* Darwin/(15)\.\d+'
+    os_replacement: 'iOS'
+    os_v1_replacement: '9'
+    os_v2_replacement: '0'
+  # iOS Apps
+  - regex: '\b(iOS[ /]|iPhone(?:/| v|[ _]OS[/,]|; | OS : |\d,\d/|\d,\d; )|iPad/)(\d{1,2})[_\.](\d{1,2})(?:[_\.](\d+))?'
+    os_replacement: 'iOS'
+
+  ##########
+  # Apple TV
+  ##########
+  - regex: '(tvOS)/(\d+).(\d+)'
+    os_replacement: 'tvOS'
+
+  ##########
+  # Chrome OS
+  # if version 0.0.0, probably this stuff:
+  # http://code.google.com/p/chromium-os/issues/detail?id=11573
+  # http://code.google.com/p/chromium-os/issues/detail?id=13790
+  ##########
+  - regex: '(CrOS) [a-z0-9_]+ (\d+)\.(\d+)(?:\.(\d+))?'
+    os_replacement: 'Chrome OS'
+
+  ##########
+  # Linux distros
+  ##########
+  - regex: '([Dd]ebian)'
+    os_replacement: 'Debian'
+  - regex: '(Linux Mint)(?:/(\d+))?'
+  - regex: '(Mandriva)(?: Linux)?/(?:[\d.-]+m[a-z]{2}(\d+).(\d))?'
+
+  ##########
+  # Symbian + Symbian OS
+  # http://en.wikipedia.org/wiki/History_of_Symbian
+  ##########
+  - regex: '(Symbian[Oo][Ss])[/ ](\d+)\.(\d+)'
+    os_replacement: 'Symbian OS'
+  - regex: '(Symbian/3).+NokiaBrowser/7\.3'
+    os_replacement: 'Symbian^3 Anna'
+  - regex: '(Symbian/3).+NokiaBrowser/7\.4'
+    os_replacement: 'Symbian^3 Belle'
+  - regex: '(Symbian/3)'
+    os_replacement: 'Symbian^3'
+  - regex: '\b(Series 60|SymbOS|S60Version|S60V\d|S60\b)'
+    os_replacement: 'Symbian OS'
+  - regex: '(MeeGo)'
+  - regex: 'Symbian [Oo][Ss]'
+    os_replacement: 'Symbian OS'
+  - regex: 'Series40;'
+    os_replacement: 'Nokia Series 40'
+  - regex: 'Series30Plus;'
+    os_replacement: 'Nokia Series 30 Plus'
+
+  ##########
+  # BlackBerry devices
+  ##########
+  - regex: '(BB10);.+Version/(\d+)\.(\d+)\.(\d+)'
+    os_replacement: 'BlackBerry OS'
+  - regex: '(Black[Bb]erry)[0-9a-z]+/(\d+)\.(\d+)\.(\d+)(?:\.(\d+))?'
+    os_replacement: 'BlackBerry OS'
+  - regex: '(Black[Bb]erry).+Version/(\d+)\.(\d+)\.(\d+)(?:\.(\d+))?'
+    os_replacement: 'BlackBerry OS'
+  - regex: '(RIM Tablet OS) (\d+)\.(\d+)\.(\d+)'
+    os_replacement: 'BlackBerry Tablet OS'
+  - regex: '(Play[Bb]ook)'
+    os_replacement: 'BlackBerry Tablet OS'
+  - regex: '(Black[Bb]erry)'
+    os_replacement: 'BlackBerry OS'
+
+  ##########
+  # Firefox OS
+  ##########
+  - regex: '\((?:Mobile|Tablet);.+Gecko/18.0 Firefox/\d+\.\d+'
+    os_replacement: 'Firefox OS'
+    os_v1_replacement: '1'
+    os_v2_replacement: '0'
+    os_v3_replacement: '1'
+
+  - regex: '\((?:Mobile|Tablet);.+Gecko/18.1 Firefox/\d+\.\d+'
+    os_replacement: 'Firefox OS'
+    os_v1_replacement: '1'
+    os_v2_replacement: '1'
+
+  - regex: '\((?:Mobile|Tablet);.+Gecko/26.0 Firefox/\d+\.\d+'
+    os_replacement: 'Firefox OS'
+    os_v1_replacement: '1'
+    os_v2_replacement: '2'
+
+  - regex: '\((?:Mobile|Tablet);.+Gecko/28.0 Firefox/\d+\.\d+'
+    os_replacement: 'Firefox OS'
+    os_v1_replacement: '1'
+    os_v2_replacement: '3'
+
+  - regex: '\((?:Mobile|Tablet);.+Gecko/30.0 Firefox/\d+\.\d+'
+    os_replacement: 'Firefox OS'
+    os_v1_replacement: '1'
+    os_v2_replacement: '4'
+
+  - regex: '\((?:Mobile|Tablet);.+Gecko/32.0 Firefox/\d+\.\d+'
+    os_replacement: 'Firefox OS'
+    os_v1_replacement: '2'
+    os_v2_replacement: '0'
+
+  - regex: '\((?:Mobile|Tablet);.+Gecko/34.0 Firefox/\d+\.\d+'
+    os_replacement: 'Firefox OS'
+    os_v1_replacement: '2'
+    os_v2_replacement: '1'
+
+  # Firefox OS Generic
+  - regex: '\((?:Mobile|Tablet);.+Firefox/\d+\.\d+'
+    os_replacement: 'Firefox OS'
+
+
+  ##########
+  # BREW
+  # yes, Brew is lower-cased for Brew MP
+  ##########
+  - regex: '(BREW)[ /](\d+)\.(\d+)\.(\d+)'
+  - regex: '(BREW);'
+  - regex: '(Brew MP|BMP)[ /](\d+)\.(\d+)\.(\d+)'
+    os_replacement: 'Brew MP'
+  - regex: 'BMP;'
+    os_replacement: 'Brew MP'
+
+  ##########
+  # Google TV
+  ##########
+  - regex: '(GoogleTV)(?: (\d+)\.(\d+)(?:\.(\d+))?|/[\da-z]+)'
+
+  - regex: '(WebTV)/(\d+).(\d+)'
+
+  ##########
+  # Misc mobile
+  ##########
+  - regex: '(hpw|web)OS/(\d+)\.(\d+)(?:\.(\d+))?'
+    os_replacement: 'webOS'
+  - regex: '(VRE);'
+
+  ##########
+  # Generic patterns
+  # since the majority of os cases are very specific, these go last
+  ##########
+  - regex: '(Fedora|Red Hat|PCLinuxOS|Puppy|Ubuntu|Kindle|Bada|Lubuntu|BackTrack|Slackware|(?:Free|Open|Net|\b)BSD)[/ ](\d+)\.(\d+)(?:\.(\d+)(?:\.(\d+))?)?'
+
+  # Gentoo Linux + Kernel Version
+  - regex: '(Linux)[ /](\d+)\.(\d+)(?:\.(\d+))?.*gentoo'
+    os_replacement: 'Gentoo'
+
+  # Opera Mini Bada
+  - regex: '\((Bada);'
+
+  # just os
+  - regex: '(Windows|Android|WeTab|Maemo)'
+  - regex: '(Ubuntu|Kubuntu|Arch Linux|CentOS|Slackware|Gentoo|openSUSE|SUSE|Red Hat|Fedora|PCLinuxOS|Mageia|(?:Free|Open|Net|\b)BSD)'
+  # Linux + Kernel Version
+  - regex: '(Linux)(?:[ /](\d+)\.(\d+)(?:\.(\d+))?)?'
+  - regex: 'SunOS'
+    os_replacement: 'Solaris'
+
+  # Roku Digital-Video-Players https://www.roku.com/
+  - regex: '^(Roku)/DVP-(\d+)\.(\d+)'
+
+device_parsers:
+
+  #########
+  # Mobile Spiders
+  # Catch the mobile crawler before checking for iPhones / Androids.
+  #########
+  - regex: '(?:(?:iPhone|Windows CE|Android).*(?:(?:Bot|Yeti)-Mobile|YRSpider|bots?/\d|(?:bot|spider)\.html)|AdsBot-Google-Mobile.*iPhone)'
+    regex_flag: 'i'
+    device_replacement: 'Spider'
+    brand_replacement: 'Spider'
+    model_replacement: 'Smartphone'
+  - regex: '(?:DoCoMo|\bMOT\b|\bLG\b|Nokia|Samsung|SonyEricsson).*(?:(?:Bot|Yeti)-Mobile|bots?/\d|(?:bot|crawler)\.html|(?:jump|google|Wukong)bot|ichiro/mobile|/spider|YahooSeeker)'
+    regex_flag: 'i'
+    device_replacement: 'Spider'
+    brand_replacement: 'Spider'
+    model_replacement: 'Feature Phone'
+
+  #########
+  # WebBrowser for SmartWatch
+  # @ref: https://play.google.com/store/apps/details?id=se.vaggan.webbrowser&hl=en
+  #########
+  - regex: '\bSmartWatch *\( *([^;]+) *; *([^;]+) *;'
+    device_replacement: '$1 $2'
+    brand_replacement: '$1'
+    model_replacement: '$2'
+
+  ######################################################################
+  # Android parsers
+  #
+  # @ref: https://support.google.com/googleplay/answer/1727131?hl=en
+  ######################################################################
+
+  # Android Application
+  - regex: 'Android Application[^\-]+ - (Sony) ?(Ericsson)? (.+) \w+ - '
+    device_replacement: '$1 $2'
+    brand_replacement: '$1$2'
+    model_replacement: '$3'
+  - regex: 'Android Application[^\-]+ - (?:HTC|HUAWEI|LGE|LENOVO|MEDION|TCT) (HTC|HUAWEI|LG|LENOVO|MEDION|ALCATEL)[ _\-](.+) \w+ - '
+    regex_flag: 'i'
+    device_replacement: '$1 $2'
+    brand_replacement: '$1'
+    model_replacement: '$2'
+  - regex: 'Android Application[^\-]+ - ([^ ]+) (.+) \w+ - '
+    device_replacement: '$1 $2'
+    brand_replacement: '$1'
+    model_replacement: '$2'
+
+  #########
+  # 3Q
+  # @ref: http://www.3q-int.com/
+  #########
+  - regex: '; *([BLRQ]C\d{4}[A-Z]+) +Build/'
+    device_replacement: '3Q $1'
+    brand_replacement: '3Q'
+    model_replacement: '$1'
+  - regex: '; *(?:3Q_)([^;/]+) +Build'
+    device_replacement: '3Q $1'
+    brand_replacement: '3Q'
+    model_replacement: '$1'
+
+  #########
+  # Acer
+  # @ref: http://us.acer.com/ac/en/US/content/group/tablets
+  #########
+  - regex: 'Android [34].*; *(A100|A101|A110|A200|A210|A211|A500|A501|A510|A511|A700(?: Lite| 3G)?|A701|B1-A71|A1-\d{3}|B1-\d{3}|V360|V370|W500|W500P|W501|W501P|W510|W511|W700|Slider SL101|DA22[^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Acer'
+    model_replacement: '$1'
+  - regex: '; *Acer Iconia Tab ([^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Acer'
+    model_replacement: '$1'
+  - regex: '; *(Z1[1235]0|E320[^/]*|S500|S510|Liquid[^;/]*|Iconia A\d+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Acer'
+    model_replacement: '$1'
+  - regex: '; *(Acer |ACER )([^;/]+) Build'
+    device_replacement: '$1$2'
+    brand_replacement: 'Acer'
+    model_replacement: '$2'
+
+  #########
+  # Advent
+  # @ref: https://en.wikipedia.org/wiki/Advent_Vega
+  # @note: VegaBean and VegaComb (names derived from jellybean, honeycomb) are
+  #   custom ROM builds for Vega
+  #########
+  - regex: '; *(Advent )?(Vega(?:Bean|Comb)?).* Build'
+    device_replacement: '$1$2'
+    brand_replacement: 'Advent'
+    model_replacement: '$2'
+
+  #########
+  # Ainol
+  # @ref: http://www.ainol.com/plugin.php?identifier=ainol&module=product
+  #########
+  - regex: '; *(Ainol )?((?:NOVO|[Nn]ovo)[^;/]+) Build'
+    device_replacement: '$1$2'
+    brand_replacement: 'Ainol'
+    model_replacement: '$2'
+
+  #########
+  # Airis
+  # @ref: http://airis.es/Tienda/Default.aspx?idG=001
+  #########
+  - regex: '; *AIRIS[ _\-]?([^/;\)]+) *(?:;|\)|Build)'
+    regex_flag: 'i'
+    device_replacement: '$1'
+    brand_replacement: 'Airis'
+    model_replacement: '$1'
+  - regex: '; *(OnePAD[^;/]+) Build'
+    regex_flag: 'i'
+    device_replacement: '$1'
+    brand_replacement: 'Airis'
+    model_replacement: '$1'
+
+  #########
+  # Airpad
+  # @ref: ??
+  #########
+  - regex: '; *Airpad[ \-]([^;/]+) Build'
+    device_replacement: 'Airpad $1'
+    brand_replacement: 'Airpad'
+    model_replacement: '$1'
+
+  #########
+  # Alcatel - TCT
+  # @ref: http://www.alcatelonetouch.com/global-en/products/smartphones.html
+  #########
+  - regex: '; *(one ?touch) (EVO7|T10|T20) Build'
+    device_replacement: 'Alcatel One Touch $2'
+    brand_replacement: 'Alcatel'
+    model_replacement: 'One Touch $2'
+  - regex: '; *(?:alcatel[ _])?(?:(?:one[ _]?touch[ _])|ot[ \-])([^;/]+);? Build'
+    regex_flag: 'i'
+    device_replacement: 'Alcatel One Touch $1'
+    brand_replacement: 'Alcatel'
+    model_replacement: 'One Touch $1'
+  - regex: '; *(TCL)[ _]([^;/]+) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: '$1'
+    model_replacement: '$2'
+  # operator specific models
+  - regex: '; *(Vodafone Smart II|Optimus_Madrid) Build'
+    device_replacement: 'Alcatel $1'
+    brand_replacement: 'Alcatel'
+    model_replacement: '$1'
+  - regex: '; *BASE_Lutea_3 Build'
+    device_replacement: 'Alcatel One Touch 998'
+    brand_replacement: 'Alcatel'
+    model_replacement: 'One Touch 998'
+  - regex: '; *BASE_Varia Build'
+    device_replacement: 'Alcatel One Touch 918D'
+    brand_replacement: 'Alcatel'
+    model_replacement: 'One Touch 918D'
+
+  #########
+  # Allfine
+  # @ref: http://www.myallfine.com/Products.asp
+  #########
+  - regex: '; *((?:FINE|Fine)\d[^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Allfine'
+    model_replacement: '$1'
+
+  #########
+  # Allview
+  # @ref: http://www.allview.ro/produse/droseries/lista-tablete-pc/
+  #########
+  - regex: '; *(ALLVIEW[ _]?|Allview[ _]?)((?:Speed|SPEED).*) Build/'
+    device_replacement: '$1$2'
+    brand_replacement: 'Allview'
+    model_replacement: '$2'
+  - regex: '; *(ALLVIEW[ _]?|Allview[ _]?)?(AX1_Shine|AX2_Frenzy) Build'
+    device_replacement: '$1$2'
+    brand_replacement: 'Allview'
+    model_replacement: '$2'
+  - regex: '; *(ALLVIEW[ _]?|Allview[ _]?)([^;/]*) Build'
+    device_replacement: '$1$2'
+    brand_replacement: 'Allview'
+    model_replacement: '$2'
+
+  #########
+  # Allwinner
+  # @ref: http://www.allwinner.com/
+  # @models: A31 (13.3"),A20,A10,
+  #########
+  - regex: '; *(A13-MID) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Allwinner'
+    model_replacement: '$1'
+  - regex: '; *(Allwinner)[ _\-]?([^;/]+) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Allwinner'
+    model_replacement: '$1'
+
+  #########
+  # Amaway
+  # @ref: http://www.amaway.cn/
+  #########
+  - regex: '; *(A651|A701B?|A702|A703|A705|A706|A707|A711|A712|A713|A717|A722|A785|A801|A802|A803|A901|A902|A1002|A1003|A1006|A1007|A9701|A9703|Q710|Q80) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Amaway'
+    model_replacement: '$1'
+
+  #########
+  # Amoi
+  # @ref: http://www.amoi.com/en/prd/prd_index.jspx
+  #########
+  - regex: '; *(?:AMOI|Amoi)[ _]([^;/]+) Build'
+    device_replacement: 'Amoi $1'
+    brand_replacement: 'Amoi'
+    model_replacement: '$1'
+  - regex: '^(?:AMOI|Amoi)[ _]([^;/]+) Linux'
+    device_replacement: 'Amoi $1'
+    brand_replacement: 'Amoi'
+    model_replacement: '$1'
+
+  #########
+  # Aoc
+  # @ref: http://latin.aoc.com/media_tablet
+  #########
+  - regex: '; *(MW(?:0[789]|10)[^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Aoc'
+    model_replacement: '$1'
+
+  #########
+  # Aoson
+  # @ref: http://www.luckystar.com.cn/en/mid.aspx?page=1
+  # @ref: http://www.luckystar.com.cn/en/mobiletel.aspx?page=1
+  # @note: brand owned by luckystar
+  #########
+  - regex: '; *(G7|M1013|M1015G|M11[CG]?|M-?12[B]?|M15|M19[G]?|M30[ACQ]?|M31[GQ]|M32|M33[GQ]|M36|M37|M38|M701T|M710|M712B|M713|M715G|M716G|M71(?:G|GS|T)?|M72[T]?|M73[T]?|M75[GT]?|M77G|M79T|M7L|M7LN|M81|M810|M81T|M82|M92|M92KS|M92S|M717G|M721|M722G|M723|M725G|M739|M785|M791|M92SK|M93D) Build'
+    device_replacement: 'Aoson $1'
+    brand_replacement: 'Aoson'
+    model_replacement: '$1'
+  - regex: '; *Aoson ([^;/]+) Build'
+    regex_flag: 'i'
+    device_replacement: 'Aoson $1'
+    brand_replacement: 'Aoson'
+    model_replacement: '$1'
+
+  #########
+  # Apanda
+  # @ref: http://www.apanda.com.cn/
+  #########
+  - regex: '; *[Aa]panda[ _\-]([^;/]+) Build'
+    device_replacement: 'Apanda $1'
+    brand_replacement: 'Apanda'
+    model_replacement: '$1'
+
+  #########
+  # Archos
+  # @ref: http://www.archos.com/de/products/tablets.html
+  # @ref: http://www.archos.com/de/products/smartphones/index.html
+  #########
+  - regex: '; *(?:ARCHOS|Archos) ?(GAMEPAD.*?)(?: Build|[;/\(\)\-])'
+    device_replacement: 'Archos $1'
+    brand_replacement: 'Archos'
+    model_replacement: '$1'
+  - regex: 'ARCHOS; GOGI; ([^;]+);'
+    device_replacement: 'Archos $1'
+    brand_replacement: 'Archos'
+    model_replacement: '$1'
+  - regex: '(?:ARCHOS|Archos)[ _]?(.*?)(?: Build|[;/\(\)\-]|$)'
+    device_replacement: 'Archos $1'
+    brand_replacement: 'Archos'
+    model_replacement: '$1'
+  - regex: '; *(AN(?:7|8|9|10|13)[A-Z0-9]{1,4}) Build'
+    device_replacement: 'Archos $1'
+    brand_replacement: 'Archos'
+    model_replacement: '$1'
+  - regex: '; *(A28|A32|A43|A70(?:BHT|CHT|HB|S|X)|A101(?:B|C|IT)|A7EB|A7EB-WK|101G9|80G9) Build'
+    device_replacement: 'Archos $1'
+    brand_replacement: 'Archos'
+    model_replacement: '$1'
+
+  #########
+  # A-rival
+  # @ref: http://www.a-rival.de/de/
+  #########
+  - regex: '; *(PAD-FMD[^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Arival'
+    model_replacement: '$1'
+  - regex: '; *(BioniQ) ?([^;/]+) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Arival'
+    model_replacement: '$1 $2'
+
+  #########
+  # Arnova
+  # @ref: http://arnovatech.com/
+  #########
+  - regex: '; *(AN\d[^;/]+|ARCHM\d+) Build'
+    device_replacement: 'Arnova $1'
+    brand_replacement: 'Arnova'
+    model_replacement: '$1'
+  - regex: '; *(?:ARNOVA|Arnova) ?([^;/]+) Build'
+    device_replacement: 'Arnova $1'
+    brand_replacement: 'Arnova'
+    model_replacement: '$1'
+
+  #########
+  # Assistant
+  # @ref: http://www.assistant.ua
+  #########
+  - regex: '; *(?:ASSISTANT )?(AP)-?([1789]\d{2}[A-Z]{0,2}|80104) Build'
+    device_replacement: 'Assistant $1-$2'
+    brand_replacement: 'Assistant'
+    model_replacement: '$1-$2'
+
+  #########
+  # Asus
+  # @ref: http://www.asus.com/uk/Tablets_Mobile/
+  #########
+  - regex: '; *(ME17\d[^;/]*|ME3\d{2}[^;/]+|K00[A-Z]|Nexus 10|Nexus 7(?: 2013)?|PadFone[^;/]*|Transformer[^;/]*|TF\d{3}[^;/]*|eeepc) Build'
+    device_replacement: 'Asus $1'
+    brand_replacement: 'Asus'
+    model_replacement: '$1'
+  - regex: '; *ASUS[ _]*([^;/]+) Build'
+    device_replacement: 'Asus $1'
+    brand_replacement: 'Asus'
+    model_replacement: '$1'
+
+  #########
+  # Garmin-Asus
+  #########
+  - regex: '; *Garmin-Asus ([^;/]+) Build'
+    device_replacement: 'Garmin-Asus $1'
+    brand_replacement: 'Garmin-Asus'
+    model_replacement: '$1'
+  - regex: '; *(Garminfone) Build'
+    device_replacement: 'Garmin $1'
+    brand_replacement: 'Garmin-Asus'
+    model_replacement: '$1'
+
+  #########
+  # Attab
+  # @ref: http://www.theattab.com/
+  #########
+  - regex: '; (@TAB-[^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Attab'
+    model_replacement: '$1'
+
+  #########
+  # Audiosonic
+  # @ref: ??
+  # @note: Take care with Docomo T-01 Toshiba
+  #########
+  - regex: '; *(T-(?:07|[^0]\d)[^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Audiosonic'
+    model_replacement: '$1'
+
+  #########
+  # Axioo
+  # @ref: http://www.axiooworld.com/ww/index.php
+  #########
+  - regex: '; *(?:Axioo[ _\-]([^;/]+)|(picopad)[ _\-]([^;/]+)) Build'
+    regex_flag: 'i'
+    device_replacement: 'Axioo $1$2 $3'
+    brand_replacement: 'Axioo'
+    model_replacement: '$1$2 $3'
+
+  #########
+  # Azend
+  # @ref: http://azendcorp.com/index.php/products/portable-electronics
+  #########
+  - regex: '; *(V(?:100|700|800)[^;/]*) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Azend'
+    model_replacement: '$1'
+
+  #########
+  # Bak
+  # @ref: http://www.bakinternational.com/produtos.php?cat=80
+  #########
+  - regex: '; *(IBAK\-[^;/]*) Build'
+    regex_flag: 'i'
+    device_replacement: '$1'
+    brand_replacement: 'Bak'
+    model_replacement: '$1'
+
+  #########
+  # Bedove
+  # @ref: http://www.bedove.com/product.html
+  # @models: HY6501|HY5001|X12|X21|I5
+  #########
+  - regex: '; *(HY5001|HY6501|X12|X21|I5) Build'
+    device_replacement: 'Bedove $1'
+    brand_replacement: 'Bedove'
+    model_replacement: '$1'
+
+  #########
+  # Benss
+  # @ref: http://www.benss.net/
+  #########
+  - regex: '; *(JC-[^;/]*) Build'
+    device_replacement: 'Benss $1'
+    brand_replacement: 'Benss'
+    model_replacement: '$1'
+
+  #########
+  # Blackberry
+  # @ref: http://uk.blackberry.com/
+  # @note: Android Apps seams to be used here
+  #########
+  - regex: '; *(BB) ([^;/]+) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Blackberry'
+    model_replacement: '$2'
+
+  #########
+  # Blackbird
+  # @ref: http://iblackbird.co.kr
+  #########
+  - regex: '; *(BlackBird)[ _](I8.*) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: '$1'
+    model_replacement: '$2'
+  - regex: '; *(BlackBird)[ _](.*) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: '$1'
+    model_replacement: '$2'
+
+  #########
+  # Blaupunkt
+  # @ref: http://www.blaupunkt.com
+  #########
+  # Endeavour
+  - regex: '; *([0-9]+BP[EM][^;/]*|Endeavour[^;/]+) Build'
+    device_replacement: 'Blaupunkt $1'
+    brand_replacement: 'Blaupunkt'
+    model_replacement: '$1'
+
+  #########
+  # Blu
+  # @ref: http://bluproducts.com
+  #########
+  - regex: '; *((?:BLU|Blu)[ _\-])([^;/]+) Build'
+    device_replacement: '$1$2'
+    brand_replacement: 'Blu'
+    model_replacement: '$2'
+  # BMOBILE = operator branded device
+  - regex: '; *(?:BMOBILE )?(Blu|BLU|DASH [^;/]+|VIVO 4\.3|TANK 4\.5) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Blu'
+    model_replacement: '$1'
+
+  #########
+  # Blusens
+  # @ref: http://www.blusens.com/es/?sg=1&sv=al&roc=1
+  #########
+  # tablet
+  - regex: '; *(TOUCH\d[^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Blusens'
+    model_replacement: '$1'
+
+  #########
+  # Bmobile
+  # @ref: http://bmobile.eu.com/?categoria=smartphones-2
+  # @note: Might collide with Maxx as AX is used also there.
+  #########
+  # smartphone
+  - regex: '; *(AX5\d+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Bmobile'
+    model_replacement: '$1'
+
+  #########
+  # bq
+  # @ref: http://bqreaders.com
+  #########
+  - regex: '; *([Bb]q) ([^;/]+);? Build'
+    device_replacement: '$1 $2'
+    brand_replacement: 'bq'
+    model_replacement: '$2'
+  - regex: '; *(Maxwell [^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'bq'
+    model_replacement: '$1'
+
+  #########
+  # Braun Phototechnik
+  # @ref: http://www.braun-phototechnik.de/en/products/list/~pcat.250/Tablet-PC.html
+  #########
+  - regex: '; *((?:B-Tab|B-TAB) ?\d[^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Braun'
+    model_replacement: '$1'
+
+  #########
+  # Broncho
+  # @ref: http://www.broncho.cn/
+  #########
+  - regex: '; *(Broncho) ([^;/]+) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: '$1'
+    model_replacement: '$2'
+
+  #########
+  # Captiva
+  # @ref: http://www.captiva-power.de
+  #########
+  - regex: '; *CAPTIVA ([^;/]+) Build'
+    device_replacement: 'Captiva $1'
+    brand_replacement: 'Captiva'
+    model_replacement: '$1'
+
+  #########
+  # Casio
+  # @ref: http://www.casiogzone.com/
+  #########
+  - regex: '; *(C771|CAL21|IS11CA) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Casio'
+    model_replacement: '$1'
+
+  #########
+  # Cat
+  # @ref: http://www.cat-sound.com
+  #########
+  - regex: '; *(?:Cat|CAT) ([^;/]+) Build'
+    device_replacement: 'Cat $1'
+    brand_replacement: 'Cat'
+    model_replacement: '$1'
+  - regex: '; *(?:Cat)(Nova.*) Build'
+    device_replacement: 'Cat $1'
+    brand_replacement: 'Cat'
+    model_replacement: '$1'
+  - regex: '; *(INM8002KP|ADM8000KP_[AB]) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Cat'
+    model_replacement: 'Tablet PHOENIX 8.1J0'
+
+  #########
+  # Celkon
+  # @ref: http://www.celkonmobiles.com/?_a=products
+  # @models: A10, A19Q, A101, A105, A107, A107\+, A112, A118, A119, A119Q, A15, A19, A20, A200, A220, A225, A22 Race, A27, A58, A59, A60, A62, A63, A64, A66, A67, A69, A75, A77, A79, A8\+, A83, A85, A86, A87, A89 Ultima, A9\+, A90, A900, A95, A97i, A98, AR 40, AR 45, AR 50, ML5
+  #########
+  - regex: '; *(?:[Cc]elkon[ _\*]|CELKON[ _\*])([^;/\)]+) ?(?:Build|;|\))'
+    device_replacement: '$1'
+    brand_replacement: 'Celkon'
+    model_replacement: '$1'
+  - regex: 'Build/(?:[Cc]elkon)+_?([^;/_\)]+)'
+    device_replacement: '$1'
+    brand_replacement: 'Celkon'
+    model_replacement: '$1'
+  - regex: '; *(CT)-?(\d+) Build'
+    device_replacement: '$1$2'
+    brand_replacement: 'Celkon'
+    model_replacement: '$1$2'
+  # smartphones
+  - regex: '; *(A19|A19Q|A105|A107[^;/\)]*) ?(?:Build|;|\))'
+    device_replacement: '$1'
+    brand_replacement: 'Celkon'
+    model_replacement: '$1'
+
+  #########
+  # ChangJia
+  # @ref: http://www.cjshowroom.com/eproducts.aspx?classcode=004001001
+  # @brief: China manufacturer makes tablets for different small brands
+  #         (eg. http://www.zeepad.net/index.html)
+  #########
+  - regex: '; *(TPC[0-9]{4,5}) Build'
+    device_replacement: '$1'
+    brand_replacement: 'ChangJia'
+    model_replacement: '$1'
+
+  #########
+  # Cloudfone
+  # @ref: http://www.cloudfonemobile.com/
+  #########
+  - regex: '; *(Cloudfone)[ _](Excite)([^ ][^;/]+) Build'
+    device_replacement: '$1 $2 $3'
+    brand_replacement: 'Cloudfone'
+    model_replacement: '$1 $2 $3'
+  - regex: '; *(Excite|ICE)[ _](\d+[^;/]+) Build'
+    device_replacement: 'Cloudfone $1 $2'
+    brand_replacement: 'Cloudfone'
+    model_replacement: 'Cloudfone $1 $2'
+  - regex: '; *(Cloudfone|CloudPad)[ _]([^;/]+) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Cloudfone'
+    model_replacement: '$1 $2'
+
+  #########
+  # Cmx
+  # @ref: http://cmx.at/de/
+  #########
+  - regex: '; *((?:Aquila|Clanga|Rapax)[^;/]+) Build'
+    regex_flag: 'i'
+    device_replacement: '$1'
+    brand_replacement: 'Cmx'
+    model_replacement: '$1'
+
+  #########
+  # CobyKyros
+  # @ref: http://cobykyros.com
+  # @note: Be careful with MID\d{3} from MpMan or Manta
+  #########
+  - regex: '; *(?:CFW-|Kyros )?(MID[0-9]{4}(?:[ABC]|SR|TV)?)(\(3G\)-4G| GB 8K| 3G| 8K| GB)? *(?:Build|[;\)])'
+    device_replacement: 'CobyKyros $1$2'
+    brand_replacement: 'CobyKyros'
+    model_replacement: '$1$2'
+
+  #########
+  # Coolpad
+  # @ref: ??
+  #########
+  - regex: '; *([^;/]*)Coolpad[ _]([^;/]+) Build'
+    device_replacement: '$1$2'
+    brand_replacement: 'Coolpad'
+    model_replacement: '$1$2'
+
+  #########
+  # Cube
+  # @ref: http://www.cube-tablet.com/buy-products.html
+  #########
+  - regex: '; *(CUBE[ _])?([KU][0-9]+ ?GT.*|A5300) Build'
+    regex_flag: 'i'
+    device_replacement: '$1$2'
+    brand_replacement: 'Cube'
+    model_replacement: '$2'
+
+  #########
+  # Cubot
+  # @ref: http://www.cubotmall.com/
+  #########
+  - regex: '; *CUBOT ([^;/]+) Build'
+    regex_flag: 'i'
+    device_replacement: '$1'
+    brand_replacement: 'Cubot'
+    model_replacement: '$1'
+  - regex: '; *(BOBBY) Build'
+    regex_flag: 'i'
+    device_replacement: '$1'
+    brand_replacement: 'Cubot'
+    model_replacement: '$1'
+
+  #########
+  # Danew
+  # @ref: http://www.danew.com/produits-tablette.php
+  #########
+  - regex: '; *(Dslide [^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Danew'
+    model_replacement: '$1'
+
+  #########
+  # Dell
+  # @ref: http://www.dell.com
+  # @ref: http://www.softbank.jp/mobile/support/product/101dl/
+  # @ref: http://www.softbank.jp/mobile/support/product/001dl/
+  # @ref: http://developer.emnet.ne.jp/android.html
+  # @ref: http://www.dell.com/in/p/mobile-xcd28/pd
+  # @ref: http://www.dell.com/in/p/mobile-xcd35/pd
+  #########
+  - regex: '; *(XCD)[ _]?(28|35) Build'
+    device_replacement: 'Dell $1$2'
+    brand_replacement: 'Dell'
+    model_replacement: '$1$2'
+  - regex: '; *(001DL) Build'
+    device_replacement: 'Dell $1'
+    brand_replacement: 'Dell'
+    model_replacement: 'Streak'
+  - regex: '; *(?:Dell|DELL) (Streak) Build'
+    device_replacement: 'Dell $1'
+    brand_replacement: 'Dell'
+    model_replacement: 'Streak'
+  - regex: '; *(101DL|GS01|Streak Pro[^;/]*) Build'
+    device_replacement: 'Dell $1'
+    brand_replacement: 'Dell'
+    model_replacement: 'Streak Pro'
+  - regex: '; *([Ss]treak ?7) Build'
+    device_replacement: 'Dell $1'
+    brand_replacement: 'Dell'
+    model_replacement: 'Streak 7'
+  - regex: '; *(Mini-3iX) Build'
+    device_replacement: 'Dell $1'
+    brand_replacement: 'Dell'
+    model_replacement: '$1'
+  - regex: '; *(?:Dell|DELL)[ _](Aero|Venue|Thunder|Mini.*|Streak[ _]Pro) Build'
+    device_replacement: 'Dell $1'
+    brand_replacement: 'Dell'
+    model_replacement: '$1'
+  - regex: '; *Dell[ _]([^;/]+) Build'
+    device_replacement: 'Dell $1'
+    brand_replacement: 'Dell'
+    model_replacement: '$1'
+  - regex: '; *Dell ([^;/]+) Build'
+    device_replacement: 'Dell $1'
+    brand_replacement: 'Dell'
+    model_replacement: '$1'
+
+  #########
+  # Denver
+  # @ref: http://www.denver-electronics.com/tablets1/
+  #########
+  - regex: '; *(TA[CD]-\d+[^;/]*) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Denver'
+    model_replacement: '$1'
+
+  #########
+  # Dex
+  # @ref: http://dex.ua/
+  #########
+  - regex: '; *(iP[789]\d{2}(?:-3G)?|IP10\d{2}(?:-8GB)?) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Dex'
+    model_replacement: '$1'
+
+  #########
+  # DNS AirTab
+  # @ref: http://www.dns-shop.ru/
+  #########
+  - regex: '; *(AirTab)[ _\-]([^;/]+) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: 'DNS'
+    model_replacement: '$1 $2'
+
+  #########
+  # Docomo (Operator Branded Device)
+  # @ref: http://www.ipentec.com/document/document.aspx?page=android-useragent
+  #########
+  - regex: '; *(F\-\d[^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Fujitsu'
+    model_replacement: '$1'
+  - regex: '; *(HT-03A) Build'
+    device_replacement: '$1'
+    brand_replacement: 'HTC'
+    model_replacement: 'Magic'
+  - regex: '; *(HT\-\d[^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'HTC'
+    model_replacement: '$1'
+  - regex: '; *(L\-\d[^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'LG'
+    model_replacement: '$1'
+  - regex: '; *(N\-\d[^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Nec'
+    model_replacement: '$1'
+  - regex: '; *(P\-\d[^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Panasonic'
+    model_replacement: '$1'
+  - regex: '; *(SC\-\d[^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Samsung'
+    model_replacement: '$1'
+  - regex: '; *(SH\-\d[^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Sharp'
+    model_replacement: '$1'
+  - regex: '; *(SO\-\d[^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'SonyEricsson'
+    model_replacement: '$1'
+  - regex: '; *(T\-0[12][^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Toshiba'
+    model_replacement: '$1'
+
+  #########
+  # DOOV
+  # @ref: http://www.doov.com.cn/
+  #########
+  - regex: '; *(DOOV)[ _]([^;/]+) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: 'DOOV'
+    model_replacement: '$2'
+
+  #########
+  # Enot
+  # @ref: http://www.enot.ua/
+  #########
+  - regex: '; *(Enot|ENOT)[ -]?([^;/]+) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Enot'
+    model_replacement: '$2'
+
+  #########
+  # Evercoss
+  # @ref: http://evercoss.com/android/
+  #########
+  - regex: '; *[^;/]+ Build/(?:CROSS|Cross)+[ _\-]([^\)]+)'
+    device_replacement: 'CROSS $1'
+    brand_replacement: 'Evercoss'
+    model_replacement: 'Cross $1'
+  - regex: '; *(CROSS|Cross)[ _\-]([^;/]+) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Evercoss'
+    model_replacement: 'Cross $2'
+
+  #########
+  # Explay
+  # @ref: http://explay.ru/
+  #########
+  - regex: '; *Explay[_ ](.+?)(?:[\)]| Build)'
+    device_replacement: '$1'
+    brand_replacement: 'Explay'
+    model_replacement: '$1'
+
+  #########
+  # Fly
+  # @ref: http://www.fly-phone.com/
+  #########
+  - regex: '; *(IQ.*) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Fly'
+    model_replacement: '$1'
+  - regex: '; *(Fly|FLY)[ _](IQ[^;]+|F[34]\d+[^;]*);? Build'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Fly'
+    model_replacement: '$2'
+
+  #########
+  # Fujitsu
+  # @ref: http://www.fujitsu.com/global/
+  #########
+  - regex: '; *(M532|Q572|FJL21) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Fujitsu'
+    model_replacement: '$1'
+
+  #########
+  # Galapad
+  # @ref: http://www.galapad.net/product.html
+  #########
+  - regex: '; *(G1) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Galapad'
+    model_replacement: '$1'
+
+  #########
+  # Geeksphone
+  # @ref: http://www.geeksphone.com/
+  #########
+  - regex: '; *(Geeksphone) ([^;/]+) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: '$1'
+    model_replacement: '$2'
+
+  #########
+  # Gfive
+  # @ref: http://www.gfivemobile.com/en
+  #########
+  #- regex: '; *(G\'?FIVE) ([^;/]+) Build' # there is a problem with python yaml parser here
+  - regex: '; *(G[^F]?FIVE) ([^;/]+) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Gfive'
+    model_replacement: '$2'
+
+  #########
+  # Gionee
+  # @ref: http://www.gionee.com/
+  #########
+  - regex: '; *(Gionee)[ _\-]([^;/]+)(?:/[^;/]+)? Build'
+    regex_flag: 'i'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Gionee'
+    model_replacement: '$2'
+  - regex: '; *(GN\d+[A-Z]?|INFINITY_PASSION|Ctrl_V1) Build'
+    device_replacement: 'Gionee $1'
+    brand_replacement: 'Gionee'
+    model_replacement: '$1'
+  - regex: '; *(E3) Build/JOP40D'
+    device_replacement: 'Gionee $1'
+    brand_replacement: 'Gionee'
+    model_replacement: '$1'
+
+  #########
+  # GoClever
+  # @ref: http://www.goclever.com
+  #########
+  - regex: '; *((?:FONE|QUANTUM|INSIGNIA) \d+[^;/]*|PLAYTAB) Build'
+    device_replacement: 'GoClever $1'
+    brand_replacement: 'GoClever'
+    model_replacement: '$1'
+  - regex: '; *GOCLEVER ([^;/]+) Build'
+    device_replacement: 'GoClever $1'
+    brand_replacement: 'GoClever'
+    model_replacement: '$1'
+
+  #########
+  # Google
+  # @ref: http://www.google.de/glass/start/
+  #########
+  - regex: '; *(Glass \d+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Google'
+    model_replacement: '$1'
+
+  #########
+  # Gigabyte
+  # @ref: http://gsmart.gigabytecm.com/en/
+  #########
+  - regex: '; *(GSmart)[ -]([^/]+) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Gigabyte'
+    model_replacement: '$1 $2'
+
+  #########
+  # Freescale development boards
+  # @ref: http://www.freescale.com/webapp/sps/site/prod_summary.jsp?code=IMX53QSB
+  #########
+  - regex: '; *(imx5[13]_[^/]+) Build'
+    device_replacement: 'Freescale $1'
+    brand_replacement: 'Freescale'
+    model_replacement: '$1'
+
+  #########
+  # Haier
+  # @ref: http://www.haier.com/
+  # @ref: http://www.haier.com/de/produkte/tablet/
+  #########
+  - regex: '; *Haier[ _\-]([^/]+) Build'
+    device_replacement: 'Haier $1'
+    brand_replacement: 'Haier'
+    model_replacement: '$1'
+  - regex: '; *(PAD1016) Build'
+    device_replacement: 'Haipad $1'
+    brand_replacement: 'Haipad'
+    model_replacement: '$1'
+
+  #########
+  # Haipad
+  # @ref: http://www.haipad.net/
+  # @models: V7P|M7SM7S|M9XM9X|M7XM7X|M9|M8|M7-M|M1002|M7|M701
+  #########
+  - regex: '; *(M701|M7|M8|M9) Build'
+    device_replacement: 'Haipad $1'
+    brand_replacement: 'Haipad'
+    model_replacement: '$1'
+
+  #########
+  # Hannspree
+  # @ref: http://www.hannspree.eu/
+  # @models: SN10T1|SN10T2|SN70T31B|SN70T32W
+  #########
+  - regex: '; *(SN\d+T[^;\)/]*)(?: Build|[;\)])'
+    device_replacement: 'Hannspree $1'
+    brand_replacement: 'Hannspree'
+    model_replacement: '$1'
+
+  #########
+  # HCLme
+  # @ref: http://www.hclmetablet.com/india/
+  #########
+  - regex: 'Build/HCL ME Tablet ([^;\)]+)[\);]'
+    device_replacement: 'HCLme $1'
+    brand_replacement: 'HCLme'
+    model_replacement: '$1'
+  - regex: '; *([^;\/]+) Build/HCL'
+    device_replacement: 'HCLme $1'
+    brand_replacement: 'HCLme'
+    model_replacement: '$1'
+
+  #########
+  # Hena
+  # @ref: http://www.henadigital.com/en/product/index.asp?id=6
+  #########
+  - regex: '; *(MID-?\d{4}C[EM]) Build'
+    device_replacement: 'Hena $1'
+    brand_replacement: 'Hena'
+    model_replacement: '$1'
+
+  #########
+  # Hisense
+  # @ref: http://www.hisense.com/
+  #########
+  - regex: '; *(EG\d{2,}|HS-[^;/]+|MIRA[^;/]+) Build'
+    device_replacement: 'Hisense $1'
+    brand_replacement: 'Hisense'
+    model_replacement: '$1'
+  - regex: '; *(andromax[^;/]+) Build'
+    regex_flag: 'i'
+    device_replacement: 'Hisense $1'
+    brand_replacement: 'Hisense'
+    model_replacement: '$1'
+
+  #########
+  # hitech
+  # @ref: http://www.hitech-mobiles.com/
+  #########
+  - regex: '; *(?:AMAZE[ _](S\d+)|(S\d+)[ _]AMAZE) Build'
+    device_replacement: 'AMAZE $1$2'
+    brand_replacement: 'hitech'
+    model_replacement: 'AMAZE $1$2'
+
+  #########
+  # HP
+  # @ref: http://www.hp.com/
+  #########
+  - regex: '; *(PlayBook) Build'
+    device_replacement: 'HP $1'
+    brand_replacement: 'HP'
+    model_replacement: '$1'
+  - regex: '; *HP ([^/]+) Build'
+    device_replacement: 'HP $1'
+    brand_replacement: 'HP'
+    model_replacement: '$1'
+  - regex: '; *([^/]+_tenderloin) Build'
+    device_replacement: 'HP TouchPad'
+    brand_replacement: 'HP'
+    model_replacement: 'TouchPad'
+
+  #########
+  # Huawei
+  # @ref: http://www.huaweidevice.com
+  # @note: Needs to be before HTC due to Desire HD Build on U8815
+  #########
+  - regex: '; *(HUAWEI |Huawei-)?([UY][^;/]+) Build/(?:Huawei|HUAWEI)([UY][^\);]+)\)'
+    device_replacement: '$1$2'
+    brand_replacement: 'Huawei'
+    model_replacement: '$2'
+  - regex: '; *([^;/]+) Build[/ ]Huawei(MT1-U06|[A-Z]+\d+[^\);]+)[^\);]*\)'
+    device_replacement: '$1'
+    brand_replacement: 'Huawei'
+    model_replacement: '$2'
+  - regex: '; *(S7|M860) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Huawei'
+    model_replacement: '$1'
+  - regex: '; *((?:HUAWEI|Huawei)[ \-]?)(MediaPad) Build'
+    device_replacement: '$1$2'
+    brand_replacement: 'Huawei'
+    model_replacement: '$2'
+  - regex: '; *((?:HUAWEI[ _]?|Huawei[ _])?Ascend[ _])([^;/]+) Build'
+    device_replacement: '$1$2'
+    brand_replacement: 'Huawei'
+    model_replacement: '$2'
+  - regex: '; *((?:HUAWEI|Huawei)[ _\-]?)((?:G700-|MT-)[^;/]+) Build'
+    device_replacement: '$1$2'
+    brand_replacement: 'Huawei'
+    model_replacement: '$2'
+  - regex: '; *((?:HUAWEI|Huawei)[ _\-]?)([^;/]+) Build'
+    device_replacement: '$1$2'
+    brand_replacement: 'Huawei'
+    model_replacement: '$2'
+  - regex: '; *(MediaPad[^;]+|SpringBoard) Build/Huawei'
+    device_replacement: '$1'
+    brand_replacement: 'Huawei'
+    model_replacement: '$1'
+  - regex: '; *([^;]+) Build/Huawei'
+    device_replacement: '$1'
+    brand_replacement: 'Huawei'
+    model_replacement: '$1'
+  - regex: '; *([Uu])([89]\d{3}) Build'
+    device_replacement: '$1$2'
+    brand_replacement: 'Huawei'
+    model_replacement: 'U$2'
+  - regex: '; *(?:Ideos |IDEOS )(S7) Build'
+    device_replacement: 'Huawei Ideos$1'
+    brand_replacement: 'Huawei'
+    model_replacement: 'Ideos$1'
+  - regex: '; *(?:Ideos |IDEOS )([^;/]+\s*|\s*)Build'
+    device_replacement: 'Huawei Ideos$1'
+    brand_replacement: 'Huawei'
+    model_replacement: 'Ideos$1'
+  - regex: '; *(Orange Daytona|Pulse|Pulse Mini|Vodafone 858|C8500|C8600|C8650|C8660|Nexus 6P) Build'
+    device_replacement: 'Huawei $1'
+    brand_replacement: 'Huawei'
+    model_replacement: '$1'
+
+  #########
+  # HTC
+  # @ref: http://www.htc.com/www/products/
+  # @ref: http://en.wikipedia.org/wiki/List_of_HTC_phones
+  #########
+
+  - regex: '; *HTC[ _]([^;]+); Windows Phone'
+    device_replacement: 'HTC $1'
+    brand_replacement: 'HTC'
+    model_replacement: '$1'
+
+  # Android HTC with Version Number matcher
+  # ; HTC_0P3Z11/1.12.161.3 Build
+  # ;HTC_A3335 V2.38.841.1 Build
+  - regex: '; *(?:HTC[ _/])+([^ _/]+)(?:[/\\]1\.0 | V|/| +)\d+\.\d[\d\.]*(?: *Build|\))'
+    device_replacement: 'HTC $1'
+    brand_replacement: 'HTC'
+    model_replacement: '$1'
+  - regex: '; *(?:HTC[ _/])+([^ _/]+)(?:[ _/]([^ _/]+))?(?:[/\\]1\.0 | V|/| +)\d+\.\d[\d\.]*(?: *Build|\))'
+    device_replacement: 'HTC $1 $2'
+    brand_replacement: 'HTC'
+    model_replacement: '$1 $2'
+  - regex: '; *(?:HTC[ _/])+([^ _/]+)(?:[ _/]([^ _/]+)(?:[ _/]([^ _/]+))?)?(?:[/\\]1\.0 | V|/| +)\d+\.\d[\d\.]*(?: *Build|\))'
+    device_replacement: 'HTC $1 $2 $3'
+    brand_replacement: 'HTC'
+    model_replacement: '$1 $2 $3'
+  - regex: '; *(?:HTC[ _/])+([^ _/]+)(?:[ _/]([^ _/]+)(?:[ _/]([^ _/]+)(?:[ _/]([^ _/]+))?)?)?(?:[/\\]1\.0 | V|/| +)\d+\.\d[\d\.]*(?: *Build|\))'
+    device_replacement: 'HTC $1 $2 $3 $4'
+    brand_replacement: 'HTC'
+    model_replacement: '$1 $2 $3 $4'
+
+  # Android HTC without Version Number matcher
+  - regex: '; *(?:(?:HTC|htc)(?:_blocked)*[ _/])+([^ _/;]+)(?: *Build|[;\)]| - )'
+    device_replacement: 'HTC $1'
+    brand_replacement: 'HTC'
+    model_replacement: '$1'
+  - regex: '; *(?:(?:HTC|htc)(?:_blocked)*[ _/])+([^ _/]+)(?:[ _/]([^ _/;\)]+))?(?: *Build|[;\)]| - )'
+    device_replacement: 'HTC $1 $2'
+    brand_replacement: 'HTC'
+    model_replacement: '$1 $2'
+  - regex: '; *(?:(?:HTC|htc)(?:_blocked)*[ _/])+([^ _/]+)(?:[ _/]([^ _/]+)(?:[ _/]([^ _/;\)]+))?)?(?: *Build|[;\)]| - )'
+    device_replacement: 'HTC $1 $2 $3'
+    brand_replacement: 'HTC'
+    model_replacement: '$1 $2 $3'
+  - regex: '; *(?:(?:HTC|htc)(?:_blocked)*[ _/])+([^ _/]+)(?:[ _/]([^ _/]+)(?:[ _/]([^ _/]+)(?:[ _/]([^ /;]+))?)?)?(?: *Build|[;\)]| - )'
+    device_replacement: 'HTC $1 $2 $3 $4'
+    brand_replacement: 'HTC'
+    model_replacement: '$1 $2 $3 $4'
+
+  # HTC Streaming Player
+  - regex: 'HTC Streaming Player [^\/]*/[^\/]*/ htc_([^/]+) /'
+    device_replacement: 'HTC $1'
+    brand_replacement: 'HTC'
+    model_replacement: '$1'
+  # general matcher for anything else
+  - regex: '(?:[;,] *|^)(?:htccn_chs-)?HTC[ _-]?([^;]+?)(?: *Build|clay|Android|-?Mozilla| Opera| Profile| UNTRUSTED|[;/\(\)]|$)'
+    regex_flag: 'i'
+    device_replacement: 'HTC $1'
+    brand_replacement: 'HTC'
+    model_replacement: '$1'
+  # Android matchers without HTC
+  - regex: '; *(A6277|ADR6200|ADR6300|ADR6350|ADR6400[A-Z]*|ADR6425[A-Z]*|APX515CKT|ARIA|Desire[^_ ]*|Dream|EndeavorU|Eris|Evo|Flyer|HD2|Hero|HERO200|Hero CDMA|HTL21|Incredible|Inspire[A-Z0-9]*|Legend|Liberty|Nexus ?(?:One|HD2)|One|One S C2|One[ _]?(?:S|V|X\+?)\w*|PC36100|PG06100|PG86100|S31HT|Sensation|Wildfire)(?: Build|[/;\(\)])'
+    regex_flag: 'i'
+    device_replacement: 'HTC $1'
+    brand_replacement: 'HTC'
+    model_replacement: '$1'
+  - regex: '; *(ADR6200|ADR6400L|ADR6425LVW|Amaze|DesireS?|EndeavorU|Eris|EVO|Evo\d[A-Z]+|HD2|IncredibleS?|Inspire[A-Z0-9]*|Inspire[A-Z0-9]*|Sensation[A-Z0-9]*|Wildfire)[ _-](.+?)(?:[/;\)]|Build|MIUI|1\.0)'
+    regex_flag: 'i'
+    device_replacement: 'HTC $1 $2'
+    brand_replacement: 'HTC'
+    model_replacement: '$1 $2'
+
+  #########
+  # Hyundai
+  # @ref: http://www.hyundaitechnologies.com
+  #########
+  - regex: '; *HYUNDAI (T\d[^/]*) Build'
+    device_replacement: 'Hyundai $1'
+    brand_replacement: 'Hyundai'
+    model_replacement: '$1'
+  - regex: '; *HYUNDAI ([^;/]+) Build'
+    device_replacement: 'Hyundai $1'
+    brand_replacement: 'Hyundai'
+    model_replacement: '$1'
+  # X900? http://www.amazon.com/Hyundai-X900-Retina-Android-Bluetooth/dp/B00AO07H3O
+  - regex: '; *(X700|Hold X|MB-6900) Build'
+    device_replacement: 'Hyundai $1'
+    brand_replacement: 'Hyundai'
+    model_replacement: '$1'
+
+  #########
+  # iBall
+  # @ref: http://www.iball.co.in/Category/Mobiles/22
+  #########
+  - regex: '; *(?:iBall[ _\-])?(Andi)[ _]?(\d[^;/]*) Build'
+    regex_flag: 'i'
+    device_replacement: '$1 $2'
+    brand_replacement: 'iBall'
+    model_replacement: '$1 $2'
+  - regex: '; *(IBall)(?:[ _]([^;/]+)|) Build'
+    regex_flag: 'i'
+    device_replacement: '$1 $2'
+    brand_replacement: 'iBall'
+    model_replacement: '$2'
+
+  #########
+  # IconBIT
+  # @ref: http://www.iconbit.com/catalog/tablets/
+  #########
+  - regex: '; *(NT-\d+[^ ;/]*|Net[Tt]AB [^;/]+|Mercury [A-Z]+|iconBIT)(?: S/N:[^;/]+)? Build'
+    device_replacement: '$1'
+    brand_replacement: 'IconBIT'
+    model_replacement: '$1'
+
+  #########
+  # IMO
+  # @ref: http://www.ponselimo.com/
+  #########
+  - regex: '; *(IMO)[ _]([^;/]+) Build'
+    regex_flag: 'i'
+    device_replacement: '$1 $2'
+    brand_replacement: 'IMO'
+    model_replacement: '$2'
+
+  #########
+  # i-mobile
+  # @ref: http://www.i-mobilephone.com/
+  #########
+  - regex: '; *i-?mobile[ _]([^/]+) Build/'
+    regex_flag: 'i'
+    device_replacement: 'i-mobile $1'
+    brand_replacement: 'imobile'
+    model_replacement: '$1'
+  - regex: '; *(i-(?:style|note)[^/]*) Build/'
+    regex_flag: 'i'
+    device_replacement: 'i-mobile $1'
+    brand_replacement: 'imobile'
+    model_replacement: '$1'
+
+  #########
+  # Impression
+  # @ref: http://impression.ua/planshetnye-kompyutery
+  #########
+  - regex: '; *(ImPAD) ?(\d+(?:.)*) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Impression'
+    model_replacement: '$1 $2'
+
+  #########
+  # Infinix
+  # @ref: http://www.infinixmobility.com/index.html
+  #########
+  - regex: '; *(Infinix)[ _]([^;/]+) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Infinix'
+    model_replacement: '$2'
+
+  #########
+  # Informer
+  # @ref: ??
+  #########
+  - regex: '; *(Informer)[ \-]([^;/]+) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Informer'
+    model_replacement: '$2'
+
+  #########
+  # Intenso
+  # @ref: http://www.intenso.de
+  # @models: 7":TAB 714,TAB 724;8":TAB 814,TAB 824;10":TAB 1004
+  #########
+  - regex: '; *(TAB) ?([78][12]4) Build'
+    device_replacement: 'Intenso $1'
+    brand_replacement: 'Intenso'
+    model_replacement: '$1 $2'
+
+  #########
+  # Intex
+  # @ref: http://intexmobile.in/index.aspx
+  # @note: Zync also offers a "Cloud Z5" device
+  #########
+  # smartphones
+  - regex: '; *(?:Intex[ _])?(AQUA|Aqua)([ _\.\-])([^;/]+) *(?:Build|;)'
+    device_replacement: '$1$2$3'
+    brand_replacement: 'Intex'
+    model_replacement: '$1 $3'
+  # matches "INTEX CLOUD X1"
+  - regex: '; *(?:INTEX|Intex)(?:[_ ]([^\ _;/]+))(?:[_ ]([^\ _;/]+))? *(?:Build|;)'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Intex'
+    model_replacement: '$1 $2'
+  # tablets
+  - regex: '; *([iI]Buddy)[ _]?(Connect)(?:_|\?_| )?([^;/]*) *(?:Build|;)'
+    device_replacement: '$1 $2 $3'
+    brand_replacement: 'Intex'
+    model_replacement: 'iBuddy $2 $3'
+  - regex: '; *(I-Buddy)[ _]([^;/]+) *(?:Build|;)'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Intex'
+    model_replacement: 'iBuddy $2'
+
+  #########
+  # iOCEAN
+  # @ref: http://www.iocean.cc/
+  #########
+  - regex: '; *(iOCEAN) ([^/]+) Build'
+    regex_flag: 'i'
+    device_replacement: '$1 $2'
+    brand_replacement: 'iOCEAN'
+    model_replacement: '$2'
+
+  #########
+  # i.onik
+  # @ref: http://www.i-onik.de/
+  #########
+  - regex: '; *(TP\d+(?:\.\d+)?\-\d[^;/]+) Build'
+    device_replacement: 'ionik $1'
+    brand_replacement: 'ionik'
+    model_replacement: '$1'
+
+  #########
+  # IRU.ru
+  # @ref: http://www.iru.ru/catalog/soho/planetable/
+  #########
+  - regex: '; *(M702pro) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Iru'
+    model_replacement: '$1'
+
+  #########
+  # Ivio
+  # @ref: http://www.ivio.com/mobile.php
+  # @models: DG80,DG20,DE38,DE88,MD70
+  #########
+  - regex: '; *(DE88Plus|MD70) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Ivio'
+    model_replacement: '$1'
+  - regex: '; *IVIO[_\-]([^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Ivio'
+    model_replacement: '$1'
+
+  #########
+  # Jaytech
+  # @ref: http://www.jay-tech.de/jaytech/servlet/frontend/
+  #########
+  - regex: '; *(TPC-\d+|JAY-TECH) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Jaytech'
+    model_replacement: '$1'
+
+  #########
+  # Jiayu
+  # @ref: http://www.ejiayu.com/en/Product.html
+  #########
+  - regex: '; *(JY-[^;/]+|G[234]S?) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Jiayu'
+    model_replacement: '$1'
+
+  #########
+  # JXD
+  # @ref: http://www.jxd.hk/
+  #########
+  - regex: '; *(JXD)[ _\-]([^;/]+) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: 'JXD'
+    model_replacement: '$2'
+
+  #########
+  # Karbonn
+  # @ref: http://www.karbonnmobiles.com/products_tablet.php
+  #########
+  - regex: '; *Karbonn[ _]?([^;/]+) *(?:Build|;)'
+    regex_flag: 'i'
+    device_replacement: '$1'
+    brand_replacement: 'Karbonn'
+    model_replacement: '$1'
+  - regex: '; *([^;]+) Build/Karbonn'
+    device_replacement: '$1'
+    brand_replacement: 'Karbonn'
+    model_replacement: '$1'
+  - regex: '; *(A11|A39|A37|A34|ST8|ST10|ST7|Smart Tab3|Smart Tab2|Titanium S\d) +Build'
+    device_replacement: '$1'
+    brand_replacement: 'Karbonn'
+    model_replacement: '$1'
+
+  #########
+  # KDDI (Operator Branded Device)
+  # @ref: http://www.ipentec.com/document/document.aspx?page=android-useragent
+  #########
+  - regex: '; *(IS01|IS03|IS05|IS\d{2}SH) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Sharp'
+    model_replacement: '$1'
+  - regex: '; *(IS04) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Regza'
+    model_replacement: '$1'
+  - regex: '; *(IS06|IS\d{2}PT) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Pantech'
+    model_replacement: '$1'
+  - regex: '; *(IS11S) Build'
+    device_replacement: '$1'
+    brand_replacement: 'SonyEricsson'
+    model_replacement: 'Xperia Acro'
+  - regex: '; *(IS11CA) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Casio'
+    model_replacement: 'GzOne $1'
+  - regex: '; *(IS11LG) Build'
+    device_replacement: '$1'
+    brand_replacement: 'LG'
+    model_replacement: 'Optimus X'
+  - regex: '; *(IS11N) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Medias'
+    model_replacement: '$1'
+  - regex: '; *(IS11PT) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Pantech'
+    model_replacement: 'MIRACH'
+  - regex: '; *(IS12F) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Fujitsu'
+    model_replacement: 'Arrows ES'
+  # @ref: https://ja.wikipedia.org/wiki/IS12M
+  - regex: '; *(IS12M) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Motorola'
+    model_replacement: 'XT909'
+  - regex: '; *(IS12S) Build'
+    device_replacement: '$1'
+    brand_replacement: 'SonyEricsson'
+    model_replacement: 'Xperia Acro HD'
+  - regex: '; *(ISW11F) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Fujitsu'
+    model_replacement: 'Arrowz Z'
+  - regex: '; *(ISW11HT) Build'
+    device_replacement: '$1'
+    brand_replacement: 'HTC'
+    model_replacement: 'EVO'
+  - regex: '; *(ISW11K) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Kyocera'
+    model_replacement: 'DIGNO'
+  - regex: '; *(ISW11M) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Motorola'
+    model_replacement: 'Photon'
+  - regex: '; *(ISW11SC) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Samsung'
+    model_replacement: 'GALAXY S II WiMAX'
+  - regex: '; *(ISW12HT) Build'
+    device_replacement: '$1'
+    brand_replacement: 'HTC'
+    model_replacement: 'EVO 3D'
+  - regex: '; *(ISW13HT) Build'
+    device_replacement: '$1'
+    brand_replacement: 'HTC'
+    model_replacement: 'J'
+  - regex: '; *(ISW?[0-9]{2}[A-Z]{0,2}) Build'
+    device_replacement: '$1'
+    brand_replacement: 'KDDI'
+    model_replacement: '$1'
+  - regex: '; *(INFOBAR [^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'KDDI'
+    model_replacement: '$1'
+
+  #########
+  # Kingcom
+  # @ref: http://www.e-kingcom.com
+  #########
+  - regex: '; *(JOYPAD|Joypad)[ _]([^;/]+) Build/'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Kingcom'
+    model_replacement: '$1 $2'
+
+  #########
+  # Kobo
+  # @ref: https://en.wikipedia.org/wiki/Kobo_Inc.
+  # @ref: http://www.kobo.com/devices#tablets
+  #########
+  - regex: '; *(Vox|VOX|Arc|K080) Build/'
+    regex_flag: 'i'
+    device_replacement: '$1'
+    brand_replacement: 'Kobo'
+    model_replacement: '$1'
+  - regex: '\b(Kobo Touch)\b'
+    device_replacement: '$1'
+    brand_replacement: 'Kobo'
+    model_replacement: '$1'
+
+  #########
+  # K-Touch
+  # @ref: ??
+  #########
+  - regex: '; *(K-Touch)[ _]([^;/]+) Build'
+    regex_flag: 'i'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Ktouch'
+    model_replacement: '$2'
+
+  #########
+  # KT Tech
+  # @ref: http://www.kttech.co.kr
+  #########
+  - regex: '; *((?:EV|KM)-S\d+[A-Z]?) Build'
+    regex_flag: 'i'
+    device_replacement: '$1'
+    brand_replacement: 'KTtech'
+    model_replacement: '$1'
+
+  #########
+  # Kyocera
+  # @ref: http://www.android.com/devices/?country=all&m=kyocera
+  #########
+  - regex: '; *(Zio|Hydro|Torque|Event|EVENT|Echo|Milano|Rise|URBANO PROGRESSO|WX04K|WX06K|WX10K|KYL21|101K|C5[12]\d{2}) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Kyocera'
+    model_replacement: '$1'
+
+  #########
+  # Lava
+  # @ref: http://www.lavamobiles.com/
+  #########
+  - regex: '; *(?:LAVA[ _])?IRIS[ _\-]?([^/;\)]+) *(?:;|\)|Build)'
+    regex_flag: 'i'
+    device_replacement: 'Iris $1'
+    brand_replacement: 'Lava'
+    model_replacement: 'Iris $1'
+  - regex: '; *LAVA[ _]([^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Lava'
+    model_replacement: '$1'
+
+  #########
+  # Lemon
+  # @ref: http://www.lemonmobiles.com/products.php?type=1
+  #########
+  - regex: '; *(?:(Aspire A1)|(?:LEMON|Lemon)[ _]([^;/]+))_? Build'
+    device_replacement: 'Lemon $1$2'
+    brand_replacement: 'Lemon'
+    model_replacement: '$1$2'
+
+  #########
+  # Lenco
+  # @ref: http://www.lenco.com/c/tablets/
+  #########
+  - regex: '; *(TAB-1012) Build/'
+    device_replacement: 'Lenco $1'
+    brand_replacement: 'Lenco'
+    model_replacement: '$1'
+  - regex: '; Lenco ([^;/]+) Build/'
+    device_replacement: 'Lenco $1'
+    brand_replacement: 'Lenco'
+    model_replacement: '$1'
+
+  #########
+  # Lenovo
+  # @ref: http://support.lenovo.com/en_GB/downloads/default.page?#
+  #########
+  - regex: '; *(A1_07|A2107A-H|S2005A-H|S1-37AH0) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Lenovo'
+    model_replacement: '$1'
+  - regex: '; *(Idea[Tp]ab)[ _]([^;/]+);? Build'
+    device_replacement: 'Lenovo $1 $2'
+    brand_replacement: 'Lenovo'
+    model_replacement: '$1 $2'
+  - regex: '; *(Idea(?:Tab|pad)) ?([^;/]+) Build'
+    device_replacement: 'Lenovo $1 $2'
+    brand_replacement: 'Lenovo'
+    model_replacement: '$1 $2'
+  - regex: '; *(ThinkPad) ?(Tablet) Build/'
+    device_replacement: 'Lenovo $1 $2'
+    brand_replacement: 'Lenovo'
+    model_replacement: '$1 $2'
+  - regex: '; *(?:LNV-)?(?:=?[Ll]enovo[ _\-]?|LENOVO[ _])+(.+?)(?:Build|[;/\)])'
+    device_replacement: 'Lenovo $1'
+    brand_replacement: 'Lenovo'
+    model_replacement: '$1'
+  - regex: '[;,] (?:Vodafone )?(SmartTab) ?(II) ?(\d+) Build/'
+    device_replacement: 'Lenovo $1 $2 $3'
+    brand_replacement: 'Lenovo'
+    model_replacement: '$1 $2 $3'
+  - regex: '; *(?:Ideapad )?K1 Build/'
+    device_replacement: 'Lenovo Ideapad K1'
+    brand_replacement: 'Lenovo'
+    model_replacement: 'Ideapad K1'
+  - regex: '; *(3GC101|3GW10[01]|A390) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Lenovo'
+    model_replacement: '$1'
+  - regex: '\b(?:Lenovo|LENOVO)+[ _\-]?([^,;:/ ]+)'
+    device_replacement: 'Lenovo $1'
+    brand_replacement: 'Lenovo'
+    model_replacement: '$1'
+
+  #########
+  # Lexibook
+  # @ref: http://www.lexibook.com/fr
+  #########
+  - regex: '; *(MFC\d+)[A-Z]{2}([^;,/]*),? Build'
+    device_replacement: '$1$2'
+    brand_replacement: 'Lexibook'
+    model_replacement: '$1$2'
+
+  #########
+  # LG
+  # @ref: http://www.lg.com/uk/mobile
+  #########
+  - regex: '; *(E[34][0-9]{2}|LS[6-8][0-9]{2}|VS[6-9][0-9]+[^;/]+|Nexus 4|Nexus 5X?|GT540f?|Optimus (?:2X|G|4X HD)|OptimusX4HD) *(?:Build|;)'
+    device_replacement: '$1'
+    brand_replacement: 'LG'
+    model_replacement: '$1'
+  - regex: '[;:] *(L-\d+[A-Z]|LGL\d+[A-Z]?)(?:/V\d+)? *(?:Build|[;\)])'
+    device_replacement: '$1'
+    brand_replacement: 'LG'
+    model_replacement: '$1'
+  - regex: '; *(LG-)([A-Z]{1,2}\d{2,}[^,;/\)\(]*?)(?:Build| V\d+|[,;/\)\(]|$)'
+    device_replacement: '$1$2'
+    brand_replacement: 'LG'
+    model_replacement: '$2'
+  - regex: '; *(LG[ \-]|LG)([^;/]+)[;/]? Build'
+    device_replacement: '$1$2'
+    brand_replacement: 'LG'
+    model_replacement: '$2'
+  - regex: '^(LG)-([^;/]+)/ Mozilla/.*; Android'
+    device_replacement: '$1 $2'
+    brand_replacement: 'LG'
+    model_replacement: '$2'
+
+  #########
+  # Malata
+  # @ref: http://www.malata.com/en/products.aspx?classid=680
+  #########
+  - regex: '; *((?:SMB|smb)[^;/]+) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Malata'
+    model_replacement: '$1'
+  - regex: '; *(?:Malata|MALATA) ([^;/]+) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Malata'
+    model_replacement: '$1'
+
+  #########
+  # Manta
+  # @ref: http://www.manta.com.pl/en
+  #########
+  - regex: '; *(MS[45][0-9]{3}|MID0[568][NS]?|MID[1-9]|MID[78]0[1-9]|MID970[1-9]|MID100[1-9]) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Manta'
+    model_replacement: '$1'
+
+  #########
+  # Match
+  # @ref: http://www.match.net.cn/products.asp
+  #########
+  - regex: '; *(M1052|M806|M9000|M9100|M9701|MID100|MID120|MID125|MID130|MID135|MID140|MID701|MID710|MID713|MID727|MID728|MID731|MID732|MID733|MID735|MID736|MID737|MID760|MID800|MID810|MID820|MID830|MID833|MID835|MID860|MID900|MID930|MID933|MID960|MID980) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Match'
+    model_replacement: '$1'
+
+  #########
+  # Maxx
+  # @ref: http://www.maxxmobile.in/
+  # @models: Maxx MSD7-Play, Maxx MX245+ Trance, Maxx AX8 Race, Maxx MSD7 3G- AX50, Maxx Genx Droid 7 - AX40, Maxx AX5 Duo,
+  #   Maxx AX3 Duo, Maxx AX3, Maxx AX8 Note II (Note 2), Maxx AX8 Note I, Maxx AX8, Maxx AX5 Plus, Maxx MSD7 Smarty,
+  #   Maxx AX9Z Race,
+  #   Maxx MT150, Maxx MQ601, Maxx M2020, Maxx Sleek MX463neo, Maxx MX525, Maxx MX192-Tune, Maxx Genx Droid 7 AX353,
+  # @note: Need more User-Agents!!!
+  #########
+  - regex: '; *(GenxDroid7|MSD7.*|AX\d.*|Tab 701|Tab 722) Build/'
+    device_replacement: 'Maxx $1'
+    brand_replacement: 'Maxx'
+    model_replacement: '$1'
+
+  #########
+  # Mediacom
+  # @ref: http://www.mediacomeurope.it/
+  #########
+  - regex: '; *(M-PP[^;/]+|PhonePad ?\d{2,}[^;/]+) Build'
+    device_replacement: 'Mediacom $1'
+    brand_replacement: 'Mediacom'
+    model_replacement: '$1'
+  - regex: '; *(M-MP[^;/]+|SmartPad ?\d{2,}[^;/]+) Build'
+    device_replacement: 'Mediacom $1'
+    brand_replacement: 'Mediacom'
+    model_replacement: '$1'
+
+  #########
+  # Medion
+  # @ref: http://www.medion.com/en/
+  #########
+  - regex: '; *(?:MD_)?LIFETAB[ _]([^;/]+) Build'
+    regex_flag: 'i'
+    device_replacement: 'Medion Lifetab $1'
+    brand_replacement: 'Medion'
+    model_replacement: 'Lifetab $1'
+  - regex: '; *MEDION ([^;/]+) Build'
+    device_replacement: 'Medion $1'
+    brand_replacement: 'Medion'
+    model_replacement: '$1'
+
+  #########
+  # Meizu
+  # @ref: http://www.meizu.com
+  #########
+  - regex: '; *(M030|M031|M035|M040|M065|m9) Build'
+    device_replacement: 'Meizu $1'
+    brand_replacement: 'Meizu'
+    model_replacement: '$1'
+  - regex: '; *(?:meizu_|MEIZU )(.+?) *(?:Build|[;\)])'
+    device_replacement: 'Meizu $1'
+    brand_replacement: 'Meizu'
+    model_replacement: '$1'
+
+  #########
+  # Micromax
+  # @ref: http://www.micromaxinfo.com
+  #########
+  - regex: '; *(?:Micromax[ _](A111|A240)|(A111|A240)) Build'
+    regex_flag: 'i'
+    device_replacement: 'Micromax $1$2'
+    brand_replacement: 'Micromax'
+    model_replacement: '$1$2'
+  - regex: '; *Micromax[ _](A\d{2,3}[^;/]*) Build'
+    regex_flag: 'i'
+    device_replacement: 'Micromax $1'
+    brand_replacement: 'Micromax'
+    model_replacement: '$1'
+  # be carefull here with Acer e.g. A500
+  - regex: '; *(A\d{2}|A[12]\d{2}|A90S|A110Q) Build'
+    regex_flag: 'i'
+    device_replacement: 'Micromax $1'
+    brand_replacement: 'Micromax'
+    model_replacement: '$1'
+  - regex: '; *Micromax[ _](P\d{3}[^;/]*) Build'
+    regex_flag: 'i'
+    device_replacement: 'Micromax $1'
+    brand_replacement: 'Micromax'
+    model_replacement: '$1'
+  - regex: '; *(P\d{3}|P\d{3}\(Funbook\)) Build'
+    regex_flag: 'i'
+    device_replacement: 'Micromax $1'
+    brand_replacement: 'Micromax'
+    model_replacement: '$1'
+
+  #########
+  # Mito
+  # @ref: http://new.mitomobile.com/
+  #########
+  - regex: '; *(MITO)[ _\-]?([^;/]+) Build'
+    regex_flag: 'i'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Mito'
+    model_replacement: '$2'
+
+  #########
+  # Mobistel
+  # @ref: http://www.mobistel.com/
+  #########
+  - regex: '; *(Cynus)[ _](F5|T\d|.+?) *(?:Build|[;/\)])'
+    regex_flag: 'i'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Mobistel'
+    model_replacement: '$1 $2'
+
+  #########
+  # Modecom
+  # @ref: http://www.modecom.eu/tablets/portal/
+  #########
+  - regex: '; *(MODECOM )?(FreeTab) ?([^;/]+) Build'
+    regex_flag: 'i'
+    device_replacement: '$1$2 $3'
+    brand_replacement: 'Modecom'
+    model_replacement: '$2 $3'
+  - regex: '; *(MODECOM )([^;/]+) Build'
+    regex_flag: 'i'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Modecom'
+    model_replacement: '$2'
+
+  #########
+  # Motorola
+  # @ref: http://www.motorola.com/us/shop-all-mobile-phones/
+  #########
+  - regex: '; *(MZ\d{3}\+?|MZ\d{3} 4G|Xoom|XOOM[^;/]*) Build'
+    device_replacement: 'Motorola $1'
+    brand_replacement: 'Motorola'
+    model_replacement: '$1'
+  - regex: '; *(Milestone )(XT[^;/]*) Build'
+    device_replacement: 'Motorola $1$2'
+    brand_replacement: 'Motorola'
+    model_replacement: '$2'
+  - regex: '; *(Motoroi ?x|Droid X|DROIDX) Build'
+    regex_flag: 'i'
+    device_replacement: 'Motorola $1'
+    brand_replacement: 'Motorola'
+    model_replacement: 'DROID X'
+  - regex: '; *(Droid[^;/]*|DROID[^;/]*|Milestone[^;/]*|Photon|Triumph|Devour|Titanium) Build'
+    device_replacement: 'Motorola $1'
+    brand_replacement: 'Motorola'
+    model_replacement: '$1'
+  - regex: '; *(A555|A85[34][^;/]*|A95[356]|ME[58]\d{2}\+?|ME600|ME632|ME722|MB\d{3}\+?|MT680|MT710|MT870|MT887|MT917|WX435|WX453|WX44[25]|XT\d{3,4}[A-Z\+]*|CL[iI]Q|CL[iI]Q XT) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Motorola'
+    model_replacement: '$1'
+  - regex: '; *(Motorola MOT-|Motorola[ _\-]|MOT\-?)([^;/]+) Build'
+    device_replacement: '$1$2'
+    brand_replacement: 'Motorola'
+    model_replacement: '$2'
+  - regex: '; *(Moto[_ ]?|MOT\-)([^;/]+) Build'
+    device_replacement: '$1$2'
+    brand_replacement: 'Motorola'
+    model_replacement: '$2'
+
+  #########
+  # MpMan
+  # @ref: http://www.mpmaneurope.com
+  #########
+  - regex: '; *((?:MP[DQ]C|MPG\d{1,4}|MP\d{3,4}|MID(?:(?:10[234]|114|43|7[247]|8[24]|7)C|8[01]1))[^;/]*) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Mpman'
+    model_replacement: '$1'
+
+  #########
+  # MSI
+  # @ref: http://www.msi.com/product/windpad/
+  #########
+  - regex: '; *(?:MSI[ _])?(Primo\d+|Enjoy[ _\-][^;/]+) Build'
+    regex_flag: 'i'
+    device_replacement: '$1'
+    brand_replacement: 'Msi'
+    model_replacement: '$1'
+
+  #########
+  # Multilaser
+  # http://www.multilaser.com.br/listagem_produtos.php?cat=5
+  #########
+  - regex: '; *Multilaser[ _]([^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Multilaser'
+    model_replacement: '$1'
+
+  #########
+  # MyPhone
+  # @ref: http://myphone.com.ph/
+  #########
+  - regex: '; *(My)[_]?(Pad)[ _]([^;/]+) Build'
+    device_replacement: '$1$2 $3'
+    brand_replacement: 'MyPhone'
+    model_replacement: '$1$2 $3'
+  - regex: '; *(My)\|?(Phone)[ _]([^;/]+) Build'
+    device_replacement: '$1$2 $3'
+    brand_replacement: 'MyPhone'
+    model_replacement: '$3'
+  - regex: '; *(A\d+)[ _](Duo)? Build'
+    regex_flag: 'i'
+    device_replacement: '$1 $2'
+    brand_replacement: 'MyPhone'
+    model_replacement: '$1 $2'
+
+  #########
+  # Mytab
+  # @ref: http://www.mytab.eu/en/category/mytab-products/
+  #########
+  - regex: '; *(myTab[^;/]*) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Mytab'
+    model_replacement: '$1'
+
+  #########
+  # Nabi
+  # @ref: https://www.nabitablet.com
+  #########
+  - regex: '; *(NABI2?-)([^;/]+) Build/'
+    device_replacement: '$1$2'
+    brand_replacement: 'Nabi'
+    model_replacement: '$2'
+
+  #########
+  # Nec Medias
+  # @ref: http://www.n-keitai.com/
+  #########
+  - regex: '; *(N-\d+[CDE]) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Nec'
+    model_replacement: '$1'
+  - regex: '; ?(NEC-)(.*) Build/'
+    device_replacement: '$1$2'
+    brand_replacement: 'Nec'
+    model_replacement: '$2'
+  - regex: '; *(LT-NA7) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Nec'
+    model_replacement: 'Lifetouch Note'
+
+  #########
+  # Nextbook
+  # @ref: http://nextbookusa.com
+  #########
+  - regex: '; *(NXM\d+[A-z0-9_]*|Next\d[A-z0-9_ \-]*|NEXT\d[A-z0-9_ \-]*|Nextbook [A-z0-9_ ]*|DATAM803HC|M805)(?: Build|[\);])'
+    device_replacement: '$1'
+    brand_replacement: 'Nextbook'
+    model_replacement: '$1'
+
+  #########
+  # Nokia
+  # @ref: http://www.nokia.com
+  #########
+  - regex: '; *(Nokia)([ _\-]*)([^;/]*) Build'
+    regex_flag: 'i'
+    device_replacement: '$1$2$3'
+    brand_replacement: 'Nokia'
+    model_replacement: '$3'
+
+  #########
+  # Nook
+  # @ref:
+  # TODO nook browser/1.0
+  #########
+  - regex: '; *(Nook ?|Barnes & Noble Nook |BN )([^;/]+) Build'
+    device_replacement: '$1$2'
+    brand_replacement: 'Nook'
+    model_replacement: '$2'
+  - regex: '; *(NOOK )?(BNRV200|BNRV200A|BNTV250|BNTV250A|BNTV400|BNTV600|LogicPD Zoom2) Build'
+    device_replacement: '$1$2'
+    brand_replacement: 'Nook'
+    model_replacement: '$2'
+  - regex: '; Build/(Nook)'
+    device_replacement: '$1'
+    brand_replacement: 'Nook'
+    model_replacement: 'Tablet'
+
+  #########
+  # Olivetti
+  # @ref: http://www.olivetti.de/EN/Page/t02/view_html?idp=348
+  #########
+  - regex: '; *(OP110|OliPad[^;/]+) Build'
+    device_replacement: 'Olivetti $1'
+    brand_replacement: 'Olivetti'
+    model_replacement: '$1'
+
+  #########
+  # Omega
+  # @ref: http://omega-technology.eu/en/produkty/346/tablets
+  # @note: MID tablets might get matched by CobyKyros first
+  # @models: (T107|MID(?:700[2-5]|7031|7108|7132|750[02]|8001|8500|9001|971[12])
+  #########
+  - regex: '; *OMEGA[ _\-](MID[^;/]+) Build'
+    device_replacement: 'Omega $1'
+    brand_replacement: 'Omega'
+    model_replacement: '$1'
+  - regex: '^(MID7500|MID\d+) Mozilla/5\.0 \(iPad;'
+    device_replacement: 'Omega $1'
+    brand_replacement: 'Omega'
+    model_replacement: '$1'
+
+  #########
+  # OpenPeak
+  # @ref: https://support.google.com/googleplay/answer/1727131?hl=en
+  #########
+  - regex: '; *((?:CIUS|cius)[^;/]*) Build'
+    device_replacement: 'Openpeak $1'
+    brand_replacement: 'Openpeak'
+    model_replacement: '$1'
+
+  #########
+  # Oppo
+  # @ref: http://en.oppo.com/products/
+  #########
+  - regex: '; *(Find ?(?:5|7a)|R8[012]\d{1,2}|T703\d{0,1}|U70\d{1,2}T?|X90\d{1,2}) Build'
+    device_replacement: 'Oppo $1'
+    brand_replacement: 'Oppo'
+    model_replacement: '$1'
+  - regex: '; *OPPO ?([^;/]+) Build/'
+    device_replacement: 'Oppo $1'
+    brand_replacement: 'Oppo'
+    model_replacement: '$1'
+
+  #########
+  # Odys
+  # @ref: http://odys.de
+  #########
+  - regex: '; *(?:Odys\-|ODYS\-|ODYS )([^;/]+) Build'
+    device_replacement: 'Odys $1'
+    brand_replacement: 'Odys'
+    model_replacement: '$1'
+  - regex: '; *(SELECT) ?(7) Build'
+    device_replacement: 'Odys $1 $2'
+    brand_replacement: 'Odys'
+    model_replacement: '$1 $2'
+  - regex: '; *(PEDI)_(PLUS)_(W) Build'
+    device_replacement: 'Odys $1 $2 $3'
+    brand_replacement: 'Odys'
+    model_replacement: '$1 $2 $3'
+  # Weltbild - Tablet PC 4 = Cat Phoenix = Odys Tablet PC 4?
+  - regex: '; *(AEON|BRAVIO|FUSION|FUSION2IN1|Genio|EOS10|IEOS[^;/]*|IRON|Loox|LOOX|LOOX Plus|Motion|NOON|NOON_PRO|NEXT|OPOS|PEDI[^;/]*|PRIME[^;/]*|STUDYTAB|TABLO|Tablet-PC-4|UNO_X8|XELIO[^;/]*|Xelio ?\d+ ?[Pp]ro|XENO10|XPRESS PRO) Build'
+    device_replacement: 'Odys $1'
+    brand_replacement: 'Odys'
+    model_replacement: '$1'
+
+  #########
+  # Orion
+  # @ref: http://www.orion.ua/en/products/computer-products/tablet-pcs.html
+  #########
+  - regex: '; *(TP-\d+) Build/'
+    device_replacement: 'Orion $1'
+    brand_replacement: 'Orion'
+    model_replacement: '$1'
+
+  #########
+  # PackardBell
+  # @ref: http://www.packardbell.com/pb/en/AE/content/productgroup/tablets
+  #########
+  - regex: '; *(G100W?) Build/'
+    device_replacement: 'PackardBell $1'
+    brand_replacement: 'PackardBell'
+    model_replacement: '$1'
+
+  #########
+  # Panasonic
+  # @ref: http://panasonic.jp/mobile/
+  # @models: T11, T21, T31, P11, P51, Eluga Power, Eluga DL1
+  # @models: (tab) Toughpad FZ-A1, Toughpad JT-B1
+  #########
+  - regex: '; *(Panasonic)[_ ]([^;/]+) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: '$1'
+    model_replacement: '$2'
+  # Toughpad
+  - regex: '; *(FZ-A1B|JT-B1) Build'
+    device_replacement: 'Panasonic $1'
+    brand_replacement: 'Panasonic'
+    model_replacement: '$1'
+  # Eluga Power
+  - regex: '; *(dL1|DL1) Build'
+    device_replacement: 'Panasonic $1'
+    brand_replacement: 'Panasonic'
+    model_replacement: '$1'
+
+  #########
+  # Pantech
+  # @href: http://www.pantech.co.kr/en/prod/prodList.do?gbrand=PANTECH
+  # @href: http://www.pantech.co.kr/en/prod/prodList.do?gbrand=VEGA
+  # @models: ADR8995, ADR910L, ADR930VW, C790, CDM8992, CDM8999, IS06, IS11PT, P2000, P2020, P2030, P4100, P5000, P6010, P6020, P6030, P7000, P7040, P8000, P8010, P9020, P9050, P9060, P9070, P9090, PT001, PT002, PT003, TXT8040, TXT8045, VEGA PTL21
+  #########
+  - regex: '; *(SKY[ _])?(IM\-[AT]\d{3}[^;/]+).* Build/'
+    device_replacement: 'Pantech $1$2'
+    brand_replacement: 'Pantech'
+    model_replacement: '$1$2'
+  - regex: '; *((?:ADR8995|ADR910L|ADR930L|ADR930VW|PTL21|P8000)(?: 4G)?) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Pantech'
+    model_replacement: '$1'
+  - regex: '; *Pantech([^;/]+).* Build/'
+    device_replacement: 'Pantech $1'
+    brand_replacement: 'Pantech'
+    model_replacement: '$1'
+
+  #########
+  # Papayre
+  # @ref: http://grammata.es/
+  #########
+  - regex: '; *(papyre)[ _\-]([^;/]+) Build/'
+    regex_flag: 'i'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Papyre'
+    model_replacement: '$2'
+
+  #########
+  # Pearl
+  # @ref: http://www.pearl.de/c-1540.shtml
+  #########
+  - regex: '; *(?:Touchlet )?(X10\.[^;/]+) Build/'
+    device_replacement: 'Pearl $1'
+    brand_replacement: 'Pearl'
+    model_replacement: '$1'
+
+  #########
+  # Phicomm
+  # @ref: http://www.phicomm.com.cn/
+  #########
+  - regex: '; PHICOMM (i800) Build/'
+    device_replacement: 'Phicomm $1'
+    brand_replacement: 'Phicomm'
+    model_replacement: '$1'
+  - regex: '; PHICOMM ([^;/]+) Build/'
+    device_replacement: 'Phicomm $1'
+    brand_replacement: 'Phicomm'
+    model_replacement: '$1'
+  - regex: '; *(FWS\d{3}[^;/]+) Build/'
+    device_replacement: 'Phicomm $1'
+    brand_replacement: 'Phicomm'
+    model_replacement: '$1'
+
+  #########
+  # Philips
+  # @ref: http://www.support.philips.com/support/catalog/products.jsp?_dyncharset=UTF-8&country=&categoryid=MOBILE_PHONES_SMART_SU_CN_CARE&userLanguage=en&navCount=2&groupId=PC_PRODUCTS_AND_PHONES_GR_CN_CARE&catalogType=&navAction=push&userCountry=cn&title=Smartphones&cateId=MOBILE_PHONES_CA_CN_CARE
+  # @TODO: Philips Tablets User-Agents missing!
+  # @ref: http://www.support.philips.com/support/catalog/products.jsp?_dyncharset=UTF-8&country=&categoryid=ENTERTAINMENT_TABLETS_SU_CN_CARE&userLanguage=en&navCount=0&groupId=&catalogType=&navAction=push&userCountry=cn&title=Entertainment+Tablets&cateId=TABLETS_CA_CN_CARE
+  #########
+  # @note: this a best guess according to available philips models. Need more User-Agents
+  - regex: '; *(D633|D822|D833|T539|T939|V726|W335|W336|W337|W3568|W536|W5510|W626|W632|W6350|W6360|W6500|W732|W736|W737|W7376|W820|W832|W8355|W8500|W8510|W930) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Philips'
+    model_replacement: '$1'
+  - regex: '; *(?:Philips|PHILIPS)[ _]([^;/]+) Build'
+    device_replacement: 'Philips $1'
+    brand_replacement: 'Philips'
+    model_replacement: '$1'
+
+  #########
+  # Pipo
+  # @ref: http://www.pipo.cn/En/
+  #########
+  - regex: 'Android 4\..*; *(M[12356789]|U[12368]|S[123])\ ?(pro)? Build'
+    device_replacement: 'Pipo $1$2'
+    brand_replacement: 'Pipo'
+    model_replacement: '$1$2'
+
+  #########
+  # Ployer
+  # @ref: http://en.ployer.cn/
+  #########
+  - regex: '; *(MOMO[^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Ployer'
+    model_replacement: '$1'
+
+  #########
+  # Polaroid/ Acho
+  # @ref: http://polaroidstore.com/store/start.asp?category_id=382&category_id2=0&order=title&filter1=&filter2=&filter3=&view=all
+  #########
+  - regex: '; *(?:Polaroid[ _])?((?:MIDC\d{3,}|PMID\d{2,}|PTAB\d{3,})[^;/]*)(\/[^;/]*)? Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Polaroid'
+    model_replacement: '$1'
+  - regex: '; *(?:Polaroid )(Tablet) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Polaroid'
+    model_replacement: '$1'
+
+  #########
+  # Pomp
+  # @ref: http://pompmobileshop.com/
+  #########
+  #~ TODO
+  - regex: '; *(POMP)[ _\-](.+?) *(?:Build|[;/\)])'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Pomp'
+    model_replacement: '$2'
+
+  #########
+  # Positivo
+  # @ref: http://www.positivoinformatica.com.br/www/pessoal/tablet-ypy/
+  #########
+  - regex: '; *(TB07STA|TB10STA|TB07FTA|TB10FTA) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Positivo'
+    model_replacement: '$1'
+  - regex: '; *(?:Positivo )?((?:YPY|Ypy)[^;/]+) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Positivo'
+    model_replacement: '$1'
+
+  #########
+  # POV
+  # @ref: http://www.pointofview-online.com/default2.php
+  # @TODO: Smartphone Models MOB-3515, MOB-5045-B missing
+  #########
+  - regex: '; *(MOB-[^;/]+) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'POV'
+    model_replacement: '$1'
+  - regex: '; *POV[ _\-]([^;/]+) Build/'
+    device_replacement: 'POV $1'
+    brand_replacement: 'POV'
+    model_replacement: '$1'
+  - regex: '; *((?:TAB-PLAYTAB|TAB-PROTAB|PROTAB|PlayTabPro|Mobii[ _\-]|TAB-P)[^;/]*) Build/'
+    device_replacement: 'POV $1'
+    brand_replacement: 'POV'
+    model_replacement: '$1'
+
+  #########
+  # Prestigio
+  # @ref: http://www.prestigio.com/catalogue/MultiPhones
+  # @ref: http://www.prestigio.com/catalogue/MultiPads
+  #########
+  - regex: '; *(?:Prestigio )?((?:PAP|PMP)\d[^;/]+) Build/'
+    device_replacement: 'Prestigio $1'
+    brand_replacement: 'Prestigio'
+    model_replacement: '$1'
+
+  #########
+  # Proscan
+  # @ref: http://www.proscanvideo.com/products-search.asp?itemClass=TABLET&itemnmbr=
+  #########
+  - regex: '; *(PLT[0-9]{4}.*) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Proscan'
+    model_replacement: '$1'
+
+  #########
+  # QMobile
+  # @ref: http://www.qmobile.com.pk/
+  #########
+  - regex: '; *(A2|A5|A8|A900)_?(Classic)? Build'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Qmobile'
+    model_replacement: '$1 $2'
+  - regex: '; *(Q[Mm]obile)_([^_]+)_([^_]+) Build'
+    device_replacement: 'Qmobile $2 $3'
+    brand_replacement: 'Qmobile'
+    model_replacement: '$2 $3'
+  - regex: '; *(Q\-?[Mm]obile)[_ ](A[^;/]+) Build'
+    device_replacement: 'Qmobile $2'
+    brand_replacement: 'Qmobile'
+    model_replacement: '$2'
+
+  #########
+  # Qmobilevn
+  # @ref: http://qmobile.vn/san-pham.html
+  #########
+  - regex: '; *(Q\-Smart)[ _]([^;/]+) Build/'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Qmobilevn'
+    model_replacement: '$2'
+  - regex: '; *(Q\-?[Mm]obile)[ _\-](S[^;/]+) Build/'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Qmobilevn'
+    model_replacement: '$2'
+
+  #########
+  # Quanta
+  # @ref: ?
+  #########
+  - regex: '; *(TA1013) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Quanta'
+    model_replacement: '$1'
+
+  #########
+  # Rockchip
+  # @ref: http://www.rock-chips.com/a/cn/product/index.html
+  # @note: manufacturer sells chipsets - I assume that these UAs are dev-boards
+  #########
+  - regex: '; *(RK\d+),? Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Rockchip'
+    model_replacement: '$1'
+  - regex: ' Build/(RK\d+)'
+    device_replacement: '$1'
+    brand_replacement: 'Rockchip'
+    model_replacement: '$1'
+
+  #########
+  # Samsung Android Devices
+  # @ref: http://www.samsung.com/us/mobile/cell-phones/all-products
+  #########
+  - regex: '; *(SAMSUNG |Samsung )?((?:Galaxy (?:Note II|S\d)|GT-I9082|GT-I9205|GT-N7\d{3}|SM-N9005)[^;/]*)\/?[^;/]* Build/'
+    device_replacement: 'Samsung $1$2'
+    brand_replacement: 'Samsung'
+    model_replacement: '$2'
+  - regex: '; *(Google )?(Nexus [Ss](?: 4G)?) Build/'
+    device_replacement: 'Samsung $1$2'
+    brand_replacement: 'Samsung'
+    model_replacement: '$2'
+  - regex: '; *(SAMSUNG |Samsung )([^\/]*)\/[^ ]* Build/'
+    device_replacement: 'Samsung $2'
+    brand_replacement: 'Samsung'
+    model_replacement: '$2'
+  - regex: '; *(Galaxy(?: Ace| Nexus| S ?II+|Nexus S| with MCR 1.2| Mini Plus 4G)?) Build/'
+    device_replacement: 'Samsung $1'
+    brand_replacement: 'Samsung'
+    model_replacement: '$1'
+  - regex: '; *(SAMSUNG[ _\-] *)+([^;/]+) Build'
+    device_replacement: 'Samsung $2'
+    brand_replacement: 'Samsung'
+    model_replacement: '$2'
+  - regex: '; *(SAMSUNG-)?(GT\-[BINPS]\d{4}[^\/]*)(\/[^ ]*) Build'
+    device_replacement: 'Samsung $1$2$3'
+    brand_replacement: 'Samsung'
+    model_replacement: '$2'
+  - regex: '(?:; *|^)((?:GT\-[BIiNPS]\d{4}|I9\d{2}0[A-Za-z\+]?\b)[^;/\)]*?)(?:Build|Linux|MIUI|[;/\)])'
+    device_replacement: 'Samsung $1'
+    brand_replacement: 'Samsung'
+    model_replacement: '$1'
+  - regex: '; (SAMSUNG-)([A-Za-z0-9\-]+).* Build/'
+    device_replacement: 'Samsung $1$2'
+    brand_replacement: 'Samsung'
+    model_replacement: '$2'
+  - regex: '; *((?:SCH|SGH|SHV|SHW|SPH|SC|SM)\-[A-Za-z0-9 ]+)(/?[^ ]*)? Build'
+    device_replacement: 'Samsung $1'
+    brand_replacement: 'Samsung'
+    model_replacement: '$1'
+  - regex: ' ((?:SCH)\-[A-Za-z0-9 ]+)(/?[^ ]*)? Build'
+    device_replacement: 'Samsung $1'
+    brand_replacement: 'Samsung'
+    model_replacement: '$1'
+  - regex: '; *(Behold ?(?:2|II)|YP\-G[^;/]+|EK-GC100|SCL21|I9300) Build'
+    device_replacement: 'Samsung $1'
+    brand_replacement: 'Samsung'
+    model_replacement: '$1'
+
+  #########
+  # Sharp
+  # @ref: http://www.sharp-phone.com/en/index.html
+  # @ref: http://www.android.com/devices/?country=all&m=sharp
+  #########
+  - regex: '; *(SH\-?\d\d[^;/]+|SBM\d[^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Sharp'
+    model_replacement: '$1'
+  - regex: '; *(SHARP[ -])([^;/]+) Build'
+    device_replacement: '$1$2'
+    brand_replacement: 'Sharp'
+    model_replacement: '$2'
+
+  #########
+  # Simvalley
+  # @ref: http://www.simvalley-mobile.de/
+  #########
+  - regex: '; *(SPX[_\-]\d[^;/]*) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Simvalley'
+    model_replacement: '$1'
+  - regex: '; *(SX7\-PEARL\.GmbH) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Simvalley'
+    model_replacement: '$1'
+  - regex: '; *(SP[T]?\-\d{2}[^;/]*) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Simvalley'
+    model_replacement: '$1'
+
+  #########
+  # SK Telesys
+  # @ref: http://www.sk-w.com/phone/phone_list.jsp
+  # @ref: http://www.android.com/devices/?country=all&m=sk-telesys
+  #########
+  - regex: '; *(SK\-.*) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'SKtelesys'
+    model_replacement: '$1'
+
+  #########
+  # Skytex
+  # @ref: http://skytex.com/android
+  #########
+  - regex: '; *(?:SKYTEX|SX)-([^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Skytex'
+    model_replacement: '$1'
+  - regex: '; *(IMAGINE [^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Skytex'
+    model_replacement: '$1'
+
+  #########
+  # SmartQ
+  # @ref: http://en.smartdevices.com.cn/Products/
+  # @models: Z8, X7, U7H, U7, T30, T20, Ten3, V5-II, T7-3G, SmartQ5, K7, S7, Q8, T19, Ten2, Ten, R10, T7, R7, V5, V7, SmartQ7
+  #########
+  - regex: '; *(SmartQ) ?([^;/]+) Build/'
+    device_replacement: '$1 $2'
+    brand_replacement: '$1'
+    model_replacement: '$2'
+
+  #########
+  # Smartbitt
+  # @ref: http://www.smartbitt.com/
+  # @missing: SBT Useragents
+  #########
+  - regex: '; *(WF7C|WF10C|SBT[^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Smartbitt'
+    model_replacement: '$1'
+
+  #########
+  # Softbank (Operator Branded Devices)
+  # @ref: http://www.ipentec.com/document/document.aspx?page=android-useragent
+  #########
+  - regex: '; *(SBM(?:003SH|005SH|006SH|007SH|102SH)) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Sharp'
+    model_replacement: '$1'
+  - regex: '; *(003P|101P|101P11C|102P) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Panasonic'
+    model_replacement: '$1'
+  - regex: '; *(00\dZ) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'ZTE'
+    model_replacement: '$1'
+  - regex: '; HTC(X06HT) Build'
+    device_replacement: '$1'
+    brand_replacement: 'HTC'
+    model_replacement: '$1'
+  - regex: '; *(001HT|X06HT) Build'
+    device_replacement: '$1'
+    brand_replacement: 'HTC'
+    model_replacement: '$1'
+  - regex: '; *(201M) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Motorola'
+    model_replacement: 'XT902'
+
+  #########
+  # Trekstor
+  # @ref: http://www.trekstor.co.uk/surftabs-en.html
+  # @note: Must come before SonyEricsson
+  #########
+  - regex: '; *(ST\d{4}.*)Build/ST'
+    device_replacement: 'Trekstor $1'
+    brand_replacement: 'Trekstor'
+    model_replacement: '$1'
+  - regex: '; *(ST\d{4}.*) Build/'
+    device_replacement: 'Trekstor $1'
+    brand_replacement: 'Trekstor'
+    model_replacement: '$1'
+
+  #########
+  # SonyEricsson
+  # @note: Must come before nokia since they also use symbian
+  # @ref: http://www.android.com/devices/?country=all&m=sony-ericssons
+  # @TODO: type!
+  #########
+  # android matchers
+  - regex: '; *(Sony ?Ericsson ?)([^;/]+) Build'
+    device_replacement: '$1$2'
+    brand_replacement: 'SonyEricsson'
+    model_replacement: '$2'
+  - regex: '; *((?:SK|ST|E|X|LT|MK|MT|WT)\d{2}[a-z0-9]*(?:-o)?|R800i|U20i) Build'
+    device_replacement: '$1'
+    brand_replacement: 'SonyEricsson'
+    model_replacement: '$1'
+  # TODO X\d+ is wrong
+  - regex: '; *(Xperia (?:A8|Arc|Acro|Active|Live with Walkman|Mini|Neo|Play|Pro|Ray|X\d+)[^;/]*) Build'
+    regex_flag: 'i'
+    device_replacement: '$1'
+    brand_replacement: 'SonyEricsson'
+    model_replacement: '$1'
+
+  #########
+  # Sony
+  # @ref: http://www.sonymobile.co.jp/index.html
+  # @ref: http://www.sonymobile.com/global-en/products/phones/
+  # @ref: http://www.sony.jp/tablet/
+  #########
+  - regex: '; Sony (Tablet[^;/]+) Build'
+    device_replacement: 'Sony $1'
+    brand_replacement: 'Sony'
+    model_replacement: '$1'
+  - regex: '; Sony ([^;/]+) Build'
+    device_replacement: 'Sony $1'
+    brand_replacement: 'Sony'
+    model_replacement: '$1'
+  - regex: '; *(Sony)([A-Za-z0-9\-]+) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: '$1'
+    model_replacement: '$2'
+  - regex: '; *(Xperia [^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Sony'
+    model_replacement: '$1'
+  - regex: '; *(C(?:1[0-9]|2[0-9]|53|55|6[0-9])[0-9]{2}|D[25]\d{3}|D6[56]\d{2}) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Sony'
+    model_replacement: '$1'
+  - regex: '; *(SGP\d{3}|SGPT\d{2}) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Sony'
+    model_replacement: '$1'
+  - regex: '; *(NW-Z1000Series) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Sony'
+    model_replacement: '$1'
+
+  ##########
+  # Sony PlayStation
+  # @ref: http://playstation.com
+  # The Vita spoofs the Kindle
+  ##########
+  - regex: 'PLAYSTATION 3'
+    device_replacement: 'PlayStation 3'
+    brand_replacement: 'Sony'
+    model_replacement: 'PlayStation 3'
+  - regex: '(PlayStation (?:Portable|Vita|\d+))'
+    device_replacement: '$1'
+    brand_replacement: 'Sony'
+    model_replacement: '$1'
+
+  #########
+  # Spice
+  # @ref: http://www.spicemobilephones.co.in/
+  #########
+  - regex: '; *((?:CSL_Spice|Spice|SPICE|CSL)[ _\-]?)?([Mm][Ii])([ _\-])?(\d{3}[^;/]*) Build/'
+    device_replacement: '$1$2$3$4'
+    brand_replacement: 'Spice'
+    model_replacement: 'Mi$4'
+
+  #########
+  # Sprint (Operator Branded Devices)
+  # @ref:
+  #########
+  - regex: '; *(Sprint )(.+?) *(?:Build|[;/])'
+    device_replacement: '$1$2'
+    brand_replacement: 'Sprint'
+    model_replacement: '$2'
+  - regex: '\b(Sprint)[: ]([^;,/ ]+)'
+    device_replacement: '$1$2'
+    brand_replacement: 'Sprint'
+    model_replacement: '$2'
+
+  #########
+  # Tagi
+  # @ref: ??
+  #########
+  - regex: '; *(TAGI[ ]?)(MID) ?([^;/]+) Build/'
+    device_replacement: '$1$2$3'
+    brand_replacement: 'Tagi'
+    model_replacement: '$2$3'
+
+  #########
+  # Tecmobile
+  # @ref: http://www.tecmobile.com/
+  #########
+  - regex: '; *(Oyster500|Opal 800) Build'
+    device_replacement: 'Tecmobile $1'
+    brand_replacement: 'Tecmobile'
+    model_replacement: '$1'
+
+  #########
+  # Tecno
+  # @ref: www.tecno-mobile.com/‎
+  #########
+  - regex: '; *(TECNO[ _])([^;/]+) Build/'
+    device_replacement: '$1$2'
+    brand_replacement: 'Tecno'
+    model_replacement: '$2'
+
+  #########
+  # Telechips, Techvision evaluation boards
+  # @ref:
+  #########
+  - regex: '; *Android for (Telechips|Techvision) ([^ ]+) '
+    regex_flag: 'i'
+    device_replacement: '$1 $2'
+    brand_replacement: '$1'
+    model_replacement: '$2'
+
+  #########
+  # Telstra
+  # @ref: http://www.telstra.com.au/home-phone/thub-2/
+  # @ref: https://support.google.com/googleplay/answer/1727131?hl=en
+  #########
+  - regex: '; *(T-Hub2) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Telstra'
+    model_replacement: '$1'
+
+  #########
+  # Terra
+  # @ref: http://www.wortmann.de/
+  #########
+  - regex: '; *(PAD) ?(100[12]) Build/'
+    device_replacement: 'Terra $1$2'
+    brand_replacement: 'Terra'
+    model_replacement: '$1$2'
+
+  #########
+  # Texet
+  # @ref: http://www.texet.ru/tablet/
+  #########
+  - regex: '; *(T[BM]-\d{3}[^;/]+) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Texet'
+    model_replacement: '$1'
+
+  #########
+  # Thalia
+  # @ref: http://www.thalia.de/shop/tolino-shine-ereader/show/
+  #########
+  - regex: '; *(tolino [^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Thalia'
+    model_replacement: '$1'
+  - regex: '; *Build/.* (TOLINO_BROWSER)'
+    device_replacement: '$1'
+    brand_replacement: 'Thalia'
+    model_replacement: 'Tolino Shine'
+
+  #########
+  # Thl
+  # @ref: http://en.thl.com.cn/Mobile
+  # @ref: http://thlmobilestore.com
+  #########
+  - regex: '; *(?:CJ[ -])?(ThL|THL)[ -]([^;/]+) Build/'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Thl'
+    model_replacement: '$2'
+  - regex: '; *(T100|T200|T5|W100|W200|W8s) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Thl'
+    model_replacement: '$1'
+
+  #########
+  # T-Mobile (Operator Branded Devices)
+  #########
+  # @ref: https://en.wikipedia.org/wiki/HTC_Hero
+  - regex: '; *(T-Mobile[ _]G2[ _]Touch) Build'
+    device_replacement: '$1'
+    brand_replacement: 'HTC'
+    model_replacement: 'Hero'
+  # @ref: https://en.wikipedia.org/wiki/HTC_Desire_Z
+  - regex: '; *(T-Mobile[ _]G2) Build'
+    device_replacement: '$1'
+    brand_replacement: 'HTC'
+    model_replacement: 'Desire Z'
+  - regex: '; *(T-Mobile myTouch Q) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Huawei'
+    model_replacement: 'U8730'
+  - regex: '; *(T-Mobile myTouch) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Huawei'
+    model_replacement: 'U8680'
+  - regex: '; *(T-Mobile_Espresso) Build'
+    device_replacement: '$1'
+    brand_replacement: 'HTC'
+    model_replacement: 'Espresso'
+  - regex: '; *(T-Mobile G1) Build'
+    device_replacement: '$1'
+    brand_replacement: 'HTC'
+    model_replacement: 'Dream'
+  - regex: '\b(T-Mobile ?)?(myTouch)[ _]?([34]G)[ _]?([^\/]*) (?:Mozilla|Build)'
+    device_replacement: '$1$2 $3 $4'
+    brand_replacement: 'HTC'
+    model_replacement: '$2 $3 $4'
+  - regex: '\b(T-Mobile)_([^_]+)_(.*) Build'
+    device_replacement: '$1 $2 $3'
+    brand_replacement: 'Tmobile'
+    model_replacement: '$2 $3'
+  - regex: '\b(T-Mobile)[_ ]?(.*?)Build'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Tmobile'
+    model_replacement: '$2'
+
+  #########
+  # Tomtec
+  # @ref: http://www.tom-tec.eu/pages/tablets.php
+  #########
+  - regex: ' (ATP[0-9]{4}) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Tomtec'
+    model_replacement: '$1'
+
+  #########
+  # Tooky
+  # @ref: http://www.tookymobile.com/
+  #########
+  - regex: ' *(TOOKY)[ _\-]([^;/]+) ?(?:Build|;)'
+    regex_flag: 'i'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Tooky'
+    model_replacement: '$2'
+
+  #########
+  # Toshiba
+  # @ref: http://www.toshiba.co.jp/
+  # @missing: LT170, Thrive 7, TOSHIBA STB10
+  #########
+  - regex: '\b(TOSHIBA_AC_AND_AZ|TOSHIBA_FOLIO_AND_A|FOLIO_AND_A)'
+    device_replacement: '$1'
+    brand_replacement: 'Toshiba'
+    model_replacement: 'Folio 100'
+  - regex: '; *([Ff]olio ?100) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Toshiba'
+    model_replacement: 'Folio 100'
+  - regex: '; *(AT[0-9]{2,3}(?:\-A|LE\-A|PE\-A|SE|a)?|AT7-A|AT1S0|Hikari-iFrame/WDPF-[^;/]+|THRiVE|Thrive) Build/'
+    device_replacement: 'Toshiba $1'
+    brand_replacement: 'Toshiba'
+    model_replacement: '$1'
+
+  #########
+  # Touchmate
+  # @ref: http://touchmatepc.com/new/
+  #########
+  - regex: '; *(TM-MID\d+[^;/]+|TOUCHMATE|MID-750) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Touchmate'
+    model_replacement: '$1'
+  # @todo: needs verification user-agents missing
+  - regex: '; *(TM-SM\d+[^;/]+) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Touchmate'
+    model_replacement: '$1'
+
+  #########
+  # Treq
+  # @ref: http://www.treq.co.id/product
+  #########
+  - regex: '; *(A10 [Bb]asic2?) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Treq'
+    model_replacement: '$1'
+  - regex: '; *(TREQ[ _\-])([^;/]+) Build'
+    regex_flag: 'i'
+    device_replacement: '$1$2'
+    brand_replacement: 'Treq'
+    model_replacement: '$2'
+
+  #########
+  # Umeox
+  # @ref: http://umeox.com/
+  # @models: A936|A603|X-5|X-3
+  #########
+  # @todo: guessed markers
+  - regex: '; *(X-?5|X-?3) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Umeox'
+    model_replacement: '$1'
+  # @todo: guessed markers
+  - regex: '; *(A502\+?|A936|A603|X1|X2) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Umeox'
+    model_replacement: '$1'
+
+  #########
+  # Versus
+  # @ref: http://versusuk.com/support.html
+  #########
+  - regex: '(TOUCH(?:TAB|PAD).+?) Build/'
+    regex_flag: 'i'
+    device_replacement: 'Versus $1'
+    brand_replacement: 'Versus'
+    model_replacement: '$1'
+
+  #########
+  # Vertu
+  # @ref: http://www.vertu.com/
+  #########
+  - regex: '(VERTU) ([^;/]+) Build/'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Vertu'
+    model_replacement: '$2'
+
+  #########
+  # Videocon
+  # @ref: http://www.videoconmobiles.com
+  #########
+  - regex: '; *(Videocon)[ _\-]([^;/]+) *(?:Build|;)'
+    device_replacement: '$1 $2'
+    brand_replacement: 'Videocon'
+    model_replacement: '$2'
+  - regex: ' (VT\d{2}[A-Za-z]*) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Videocon'
+    model_replacement: '$1'
+
+  #########
+  # Viewsonic
+  # @ref: http://viewsonic.com
+  #########
+  - regex: '; *((?:ViewPad|ViewPhone|VSD)[^;/]+) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Viewsonic'
+    model_replacement: '$1'
+  - regex: '; *(ViewSonic-)([^;/]+) Build/'
+    device_replacement: '$1$2'
+    brand_replacement: 'Viewsonic'
+    model_replacement: '$2'
+  - regex: '; *(GTablet.*) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Viewsonic'
+    model_replacement: '$1'
+
+  #########
+  # vivo
+  # @ref: http://vivo.cn/
+  #########
+  - regex: '; *([Vv]ivo)[ _]([^;/]+) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: 'vivo'
+    model_replacement: '$2'
+
+  #########
+  # Vodafone (Operator Branded Devices)
+  # @ref: ??
+  #########
+  - regex: '(Vodafone) (.*) Build/'
+    device_replacement: '$1 $2'
+    brand_replacement: '$1'
+    model_replacement: '$2'
+
+  #########
+  # Walton
+  # @ref: http://www.waltonbd.com/
+  #########
+  - regex: '; *(?:Walton[ _\-])?(Primo[ _\-][^;/]+) Build'
+    regex_flag: 'i'
+    device_replacement: 'Walton $1'
+    brand_replacement: 'Walton'
+    model_replacement: '$1'
+
+  #########
+  # Wiko
+  # @ref: http://fr.wikomobile.com/collection.php?s=Smartphones
+  #########
+  - regex: '; *(?:WIKO[ \-])?(CINK\+?|BARRY|BLOOM|DARKFULL|DARKMOON|DARKNIGHT|DARKSIDE|FIZZ|HIGHWAY|IGGY|OZZY|RAINBOW|STAIRWAY|SUBLIM|WAX|CINK [^;/]+) Build/'
+    regex_flag: 'i'
+    device_replacement: 'Wiko $1'
+    brand_replacement: 'Wiko'
+    model_replacement: '$1'
+
+  #########
+  # WellcoM
+  # @ref: ??
+  #########
+  - regex: '; *WellcoM-([^;/]+) Build'
+    device_replacement: 'Wellcom $1'
+    brand_replacement: 'Wellcom'
+    model_replacement: '$1'
+
+  ##########
+  # WeTab
+  # @ref: http://wetab.mobi/
+  ##########
+  - regex: '(?:(WeTab)-Browser|; (wetab) Build)'
+    device_replacement: '$1'
+    brand_replacement: 'WeTab'
+    model_replacement: 'WeTab'
+
+  #########
+  # Wolfgang
+  # @ref: http://wolfgangmobile.com/
+  #########
+  - regex: '; *(AT-AS[^;/]+) Build'
+    device_replacement: 'Wolfgang $1'
+    brand_replacement: 'Wolfgang'
+    model_replacement: '$1'
+
+  #########
+  # Woxter
+  # @ref: http://www.woxter.es/es-es/categories/index
+  #########
+  - regex: '; *(?:Woxter|Wxt) ([^;/]+) Build'
+    device_replacement: 'Woxter $1'
+    brand_replacement: 'Woxter'
+    model_replacement: '$1'
+
+  #########
+  # Yarvik Zania
+  # @ref: http://yarvik.com
+  #########
+  - regex: '; *(?:Xenta |Luna )?(TAB[234][0-9]{2}|TAB0[78]-\d{3}|TAB0?9-\d{3}|TAB1[03]-\d{3}|SMP\d{2}-\d{3}) Build/'
+    device_replacement: 'Yarvik $1'
+    brand_replacement: 'Yarvik'
+    model_replacement: '$1'
+
+  #########
+  # Yifang
+  # @note: Needs to be at the very last as manufacturer builds for other brands.
+  # @ref: http://www.yifangdigital.com/
+  # @models: M1010, M1011, M1007, M1008, M1005, M899, M899LP, M909, M8000,
+  #   M8001, M8002, M8003, M849, M815, M816, M819, M805, M878, M780LPW,
+  #   M778, M7000, M7000AD, M7000NBD, M7001, M7002, M7002KBD, M777, M767,
+  #   M789, M799, M769, M757, M755, M753, M752, M739, M729, M723, M712, M727
+  #########
+  - regex: '; *([A-Z]{2,4})(M\d{3,}[A-Z]{2})([^;\)\/]*)(?: Build|[;\)])'
+    device_replacement: 'Yifang $1$2$3'
+    brand_replacement: 'Yifang'
+    model_replacement: '$2'
+
+  #########
+  # XiaoMi
+  # @ref: http://www.xiaomi.com/event/buyphone
+  #########
+  - regex: '; *((MI|HM|MI-ONE|Redmi)[ -](NOTE |Note )?[^;/]*) (Build|MIUI)/'
+    device_replacement: 'XiaoMi $1'
+    brand_replacement: 'XiaoMi'
+    model_replacement: '$1'
+
+  #########
+  # Xolo
+  # @ref: http://www.xolo.in/
+  #########
+  - regex: '; *XOLO[ _]([^;/]*tab.*) Build'
+    regex_flag: 'i'
+    device_replacement: 'Xolo $1'
+    brand_replacement: 'Xolo'
+    model_replacement: '$1'
+  - regex: '; *XOLO[ _]([^;/]+) Build'
+    regex_flag: 'i'
+    device_replacement: 'Xolo $1'
+    brand_replacement: 'Xolo'
+    model_replacement: '$1'
+  - regex: '; *(q\d0{2,3}[a-z]?) Build'
+    regex_flag: 'i'
+    device_replacement: 'Xolo $1'
+    brand_replacement: 'Xolo'
+    model_replacement: '$1'
+
+  #########
+  # Xoro
+  # @ref: http://www.xoro.de/produkte/
+  #########
+  - regex: '; *(PAD ?[79]\d+[^;/]*|TelePAD\d+[^;/]) Build'
+    device_replacement: 'Xoro $1'
+    brand_replacement: 'Xoro'
+    model_replacement: '$1'
+
+  #########
+  # Zopo
+  # @ref: http://www.zopomobiles.com/products.html
+  #########
+  - regex: '; *(?:(?:ZOPO|Zopo)[ _]([^;/]+)|(ZP ?(?:\d{2}[^;/]+|C2))|(C[2379])) Build'
+    device_replacement: '$1$2$3'
+    brand_replacement: 'Zopo'
+    model_replacement: '$1$2$3'
+
+  #########
+  # ZiiLabs
+  # @ref: http://www.ziilabs.com/products/platforms/androidreferencetablets.php
+  #########
+  - regex: '; *(ZiiLABS) (Zii[^;/]*) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: 'ZiiLabs'
+    model_replacement: '$2'
+  - regex: '; *(Zii)_([^;/]*) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: 'ZiiLabs'
+    model_replacement: '$2'
+
+  #########
+  # ZTE
+  # @ref: http://www.ztedevices.com/
+  #########
+  - regex: '; *(ARIZONA|(?:ATLAS|Atlas) W|D930|Grand (?:[SX][^;]*|Era|Memo[^;]*)|JOE|(?:Kis|KIS)\b[^;]*|Libra|Light [^;]*|N8[056][01]|N850L|N8000|N9[15]\d{2}|N9810|NX501|Optik|(?:Vip )Racer[^;]*|RacerII|RACERII|San Francisco[^;]*|V9[AC]|V55|V881|Z[679][0-9]{2}[A-z]?) Build'
+    device_replacement: '$1'
+    brand_replacement: 'ZTE'
+    model_replacement: '$1'
+  - regex: '; *([A-Z]\d+)_USA_[^;]* Build'
+    device_replacement: '$1'
+    brand_replacement: 'ZTE'
+    model_replacement: '$1'
+  - regex: '; *(SmartTab\d+)[^;]* Build'
+    device_replacement: '$1'
+    brand_replacement: 'ZTE'
+    model_replacement: '$1'
+  - regex: '; *(?:Blade|BLADE|ZTE-BLADE)([^;/]*) Build'
+    device_replacement: 'ZTE Blade$1'
+    brand_replacement: 'ZTE'
+    model_replacement: 'Blade$1'
+  - regex: '; *(?:Skate|SKATE|ZTE-SKATE)([^;/]*) Build'
+    device_replacement: 'ZTE Skate$1'
+    brand_replacement: 'ZTE'
+    model_replacement: 'Skate$1'
+  - regex: '; *(Orange |Optimus )(Monte Carlo|San Francisco) Build'
+    device_replacement: '$1$2'
+    brand_replacement: 'ZTE'
+    model_replacement: '$1$2'
+  - regex: '; *(?:ZXY-ZTE_|ZTE\-U |ZTE[\- _]|ZTE-C[_ ])([^;/]+) Build'
+    device_replacement: 'ZTE $1'
+    brand_replacement: 'ZTE'
+    model_replacement: '$1'
+  # operator specific
+  - regex: '; (BASE) (lutea|Lutea 2|Tab[^;]*) Build'
+    device_replacement: '$1 $2'
+    brand_replacement: 'ZTE'
+    model_replacement: '$1 $2'
+  - regex: '; (Avea inTouch 2|soft stone|tmn smart a7|Movistar[ _]Link) Build'
+    regex_flag: 'i'
+    device_replacement: '$1'
+    brand_replacement: 'ZTE'
+    model_replacement: '$1'
+  - regex: '; *(vp9plus)\)'
+    device_replacement: '$1'
+    brand_replacement: 'ZTE'
+    model_replacement: '$1'
+
+  ##########
+  # Zync
+  # @ref: http://www.zync.in/index.php/our-products/tablet-phablets
+  ##########
+  - regex: '; ?(Cloud[ _]Z5|z1000|Z99 2G|z99|z930|z999|z990|z909|Z919|z900) Build/'
+    device_replacement: '$1'
+    brand_replacement: 'Zync'
+    model_replacement: '$1'
+
+  ##########
+  # Kindle
+  # @note: Needs to be after Sony Playstation Vita as this UA contains Silk/3.2
+  # @ref: https://developer.amazon.com/sdk/fire/specifications.html
+  # @ref: http://amazonsilk.wordpress.com/useful-bits/silk-user-agent/
+  ##########
+  - regex: '; ?(KFOT|Kindle Fire) Build\b'
+    device_replacement: 'Kindle Fire'
+    brand_replacement: 'Amazon'
+    model_replacement: 'Kindle Fire'
+  - regex: '; ?(KFOTE|Amazon Kindle Fire2) Build\b'
+    device_replacement: 'Kindle Fire 2'
+    brand_replacement: 'Amazon'
+    model_replacement: 'Kindle Fire 2'
+  - regex: '; ?(KFTT) Build\b'
+    device_replacement: 'Kindle Fire HD'
+    brand_replacement: 'Amazon'
+    model_replacement: 'Kindle Fire HD 7"'
+  - regex: '; ?(KFJWI) Build\b'
+    device_replacement: 'Kindle Fire HD 8.9" WiFi'
+    brand_replacement: 'Amazon'
+    model_replacement: 'Kindle Fire HD 8.9" WiFi'
+  - regex: '; ?(KFJWA) Build\b'
+    device_replacement: 'Kindle Fire HD 8.9" 4G'
+    brand_replacement: 'Amazon'
+    model_replacement: 'Kindle Fire HD 8.9" 4G'
+  - regex: '; ?(KFSOWI) Build\b'
+    device_replacement: 'Kindle Fire HD 7" WiFi'
+    brand_replacement: 'Amazon'
+    model_replacement: 'Kindle Fire HD 7" WiFi'
+  - regex: '; ?(KFTHWI) Build\b'
+    device_replacement: 'Kindle Fire HDX 7" WiFi'
+    brand_replacement: 'Amazon'
+    model_replacement: 'Kindle Fire HDX 7" WiFi'
+  - regex: '; ?(KFTHWA) Build\b'
+    device_replacement: 'Kindle Fire HDX 7" 4G'
+    brand_replacement: 'Amazon'
+    model_replacement: 'Kindle Fire HDX 7" 4G'
+  - regex: '; ?(KFAPWI) Build\b'
+    device_replacement: 'Kindle Fire HDX 8.9" WiFi'
+    brand_replacement: 'Amazon'
+    model_replacement: 'Kindle Fire HDX 8.9" WiFi'
+  - regex: '; ?(KFAPWA) Build\b'
+    device_replacement: 'Kindle Fire HDX 8.9" 4G'
+    brand_replacement: 'Amazon'
+    model_replacement: 'Kindle Fire HDX 8.9" 4G'
+  - regex: '; ?Amazon ([^;/]+) Build\b'
+    device_replacement: '$1'
+    brand_replacement: 'Amazon'
+    model_replacement: '$1'
+  - regex: '; ?(Kindle) Build\b'
+    device_replacement: 'Kindle'
+    brand_replacement: 'Amazon'
+    model_replacement: 'Kindle'
+  - regex: '; ?(Silk)/(\d+)\.(\d+)(?:\.([0-9\-]+))? Build\b'
+    device_replacement: 'Kindle Fire'
+    brand_replacement: 'Amazon'
+    model_replacement: 'Kindle Fire$2'
+  - regex: ' (Kindle)/(\d+\.\d+)'
+    device_replacement: 'Kindle'
+    brand_replacement: 'Amazon'
+    model_replacement: '$1 $2'
+  - regex: ' (Silk|Kindle)/(\d+)\.'
+    device_replacement: 'Kindle'
+    brand_replacement: 'Amazon'
+    model_replacement: 'Kindle'
+
+  #########
+  # Devices from chinese manufacturer(s)
+  # @note: identified by x-wap-profile http://218.249.47.94/Xianghe/.*
+  #########
+  - regex: '(sprd)\-([^/]+)/'
+    device_replacement: '$1 $2'
+    brand_replacement: '$1'
+    model_replacement: '$2'
+  # @ref: http://eshinechina.en.alibaba.com/
+  - regex: '; *(H\d{2}00\+?) Build'
+    device_replacement: '$1'
+    brand_replacement: 'Hero'
+    model_replacement: '$1'
+  - regex: '; *(iphone|iPhone5) Build/'
+    device_replacement: 'Xianghe $1'
+    brand_replacement: 'Xianghe'
+    model_replacement: '$1'
+  - regex: '; *(e\d{4}[a-z]?_?v\d+|v89_[^;/]+)[^;/]+ Build/'
+    device_replacement: 'Xianghe $1'
+    brand_replacement: 'Xianghe'
+    model_replacement: '$1'
+
+  #########
+  # Cellular
+  # @ref:
+  # @note: Operator branded devices
+  #########
+  - regex: '\bUSCC[_\-]?([^ ;/\)]+)'
+    device_replacement: '$1'
+    brand_replacement: 'Cellular'
+    model_replacement: '$1'
+
+  ######################################################################
+  # Windows Phone Parsers
+  ######################################################################
+
+  #########
+  # Alcatel Windows Phones
+  #########
+  - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?)?(?:ALCATEL)[^;]*; *([^;,\)]+)'
+    device_replacement: 'Alcatel $1'
+    brand_replacement: 'Alcatel'
+    model_replacement: '$1'
+
+  #########
+  # Asus Windows Phones
+  #########
+  #~ - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?|WpsLondonTest; ?)?(?:ASUS|Asus)[^;]*; *([^;,\)]+)'
+  - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?|WpsLondonTest; ?)?(?:ASUS|Asus)[^;]*; *([^;,\)]+)'
+    device_replacement: 'Asus $1'
+    brand_replacement: 'Asus'
+    model_replacement: '$1'
+
+  #########
+  # Dell Windows Phones
+  #########
+  - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?)?(?:DELL|Dell)[^;]*; *([^;,\)]+)'
+    device_replacement: 'Dell $1'
+    brand_replacement: 'Dell'
+    model_replacement: '$1'
+
+  #########
+  # HTC Windows Phones
+  #########
+  - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?|WpsLondonTest; ?)?(?:HTC|Htc|HTC_blocked[^;]*)[^;]*; *(?:HTC)?([^;,\)]+)'
+    device_replacement: 'HTC $1'
+    brand_replacement: 'HTC'
+    model_replacement: '$1'
+
+  #########
+  # Huawei Windows Phones
+  #########
+  - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?)?(?:HUAWEI)[^;]*; *(?:HUAWEI )?([^;,\)]+)'
+    device_replacement: 'Huawei $1'
+    brand_replacement: 'Huawei'
+    model_replacement: '$1'
+
+  #########
+  # LG Windows Phones
+  #########
+  - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?)?(?:LG|Lg)[^;]*; *(?:LG[ \-])?([^;,\)]+)'
+    device_replacement: 'LG $1'
+    brand_replacement: 'LG'
+    model_replacement: '$1'
+
+  #########
+  # Noka Windows Phones
+  #########
+  - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?)?(?:NOKIA|Nokia)[^;]*; *(?:NOKIA ?|Nokia ?|LUMIA ?|[Ll]umia ?)*(\d{3,}[^;\)]*)'
+    device_replacement: 'Lumia $1'
+    brand_replacement: 'Nokia'
+    model_replacement: 'Lumia $1'
+  - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?)?(?:NOKIA|Nokia)[^;]*; *(RM-\d{3,})'
+    device_replacement: 'Nokia $1'
+    brand_replacement: 'Nokia'
+    model_replacement: '$1'
+  - regex: '(?:Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)]|WPDesktop;) ?(?:ARM; ?Touch; ?|Touch; ?)?(?:NOKIA|Nokia)[^;]*; *(?:NOKIA ?|Nokia ?|LUMIA ?|[Ll]umia ?)*([^;\)]+)'
+    device_replacement: 'Nokia $1'
+    brand_replacement: 'Nokia'
+    model_replacement: '$1'
+
+  #########
+  # Microsoft Windows Phones
+  #########
+  - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?)?(?:Microsoft(?: Corporation)?)[^;]*; *([^;,\)]+)'
+    device_replacement: 'Microsoft $1'
+    brand_replacement: 'Microsoft'
+    model_replacement: '$1'
+
+  #########
+  # Samsung Windows Phones
+  #########
+  - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?|WpsLondonTest; ?)?(?:SAMSUNG)[^;]*; *(?:SAMSUNG )?([^;,\.\)]+)'
+    device_replacement: 'Samsung $1'
+    brand_replacement: 'Samsung'
+    model_replacement: '$1'
+
+  #########
+  # Toshiba Windows Phones
+  #########
+  - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?|WpsLondonTest; ?)?(?:TOSHIBA|FujitsuToshibaMobileCommun)[^;]*; *([^;,\)]+)'
+    device_replacement: 'Toshiba $1'
+    brand_replacement: 'Toshiba'
+    model_replacement: '$1'
+
+  #########
+  # Generic Windows Phones
+  #########
+  - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?|WpsLondonTest; ?)?([^;]+); *([^;,\)]+)'
+    device_replacement: '$1 $2'
+    brand_replacement: '$1'
+    model_replacement: '$2'
+
+  ######################################################################
+  # Other Devices Parser
+  ######################################################################
+
+  #########
+  # Samsung Bada Phones
+  #########
+  - regex: '(?:^|; )SAMSUNG\-([A-Za-z0-9\-]+).* Bada/'
+    device_replacement: 'Samsung $1'
+    brand_replacement: 'Samsung'
+    model_replacement: '$1'
+
+  #########
+  # Firefox OS
+  #########
+  - regex: '\(Mobile; ALCATEL ?(One|ONE) ?(Touch|TOUCH) ?([^;/]+)(?:/[^;]+)?; rv:[^\)]+\) Gecko/[^\/]+ Firefox/'
+    device_replacement: 'Alcatel $1 $2 $3'
+    brand_replacement: 'Alcatel'
+    model_replacement: 'One Touch $3'
+  - regex: '\(Mobile; (?:ZTE([^;]+)|(OpenC)); rv:[^\)]+\) Gecko/[^\/]+ Firefox/'
+    device_replacement: 'ZTE $1$2'
+    brand_replacement: 'ZTE'
+    model_replacement: '$1$2'
+
+  ##########
+  # NOKIA
+  # @note: NokiaN8-00 comes before iphone. Sometimes spoofs iphone
+  ##########
+  - regex: 'Nokia(N[0-9]+)([A-z_\-][A-z0-9_\-]*)'
+    device_replacement: 'Nokia $1'
+    brand_replacement: 'Nokia'
+    model_replacement: '$1$2'
+  - regex: '(?:NOKIA|Nokia)(?:\-| *)(?:([A-Za-z0-9]+)\-[0-9a-f]{32}|([A-Za-z0-9\-]+)(?:UCBrowser)|([A-Za-z0-9\-]+))'
+    device_replacement: 'Nokia $1$2$3'
+    brand_replacement: 'Nokia'
+    model_replacement: '$1$2$3'
+  - regex: 'Lumia ([A-Za-z0-9\-]+)'
+    device_replacement: 'Lumia $1'
+    brand_replacement: 'Nokia'
+    model_replacement: 'Lumia $1'
+  # UCWEB Browser on Symbian
+  - regex: '\(Symbian; U; S60 V5; [A-z]{2}\-[A-z]{2}; (SonyEricsson|Samsung|Nokia|LG)([^;/]+)\)'
+    device_replacement: '$1 $2'
+    brand_replacement: '$1'
+    model_replacement: '$2'
+  # Nokia Symbian
+  - regex: '\(Symbian(?:/3)?; U; ([^;]+);'
+    device_replacement: 'Nokia $1'
+    brand_replacement: 'Nokia'
+    model_replacement: '$1'
+
+  ##########
+  # BlackBerry
+  # @ref: http://www.useragentstring.com/pages/BlackBerry/
+  ##########
+  - regex: 'BB10; ([A-Za-z0-9\- ]+)\)'
+    device_replacement: 'BlackBerry $1'
+    brand_replacement: 'BlackBerry'
+    model_replacement: '$1'
+  - regex: 'Play[Bb]ook.+RIM Tablet OS'
+    device_replacement: 'BlackBerry Playbook'
+    brand_replacement: 'BlackBerry'
+    model_replacement: 'Playbook'
+  - regex: 'Black[Bb]erry ([0-9]+);'
+    device_replacement: 'BlackBerry $1'
+    brand_replacement: 'BlackBerry'
+    model_replacement: '$1'
+  - regex: 'Black[Bb]erry([0-9]+)'
+    device_replacement: 'BlackBerry $1'
+    brand_replacement: 'BlackBerry'
+    model_replacement: '$1'
+  - regex: 'Black[Bb]erry;'
+    device_replacement: 'BlackBerry'
+    brand_replacement: 'BlackBerry'
+
+  ##########
+  # PALM / HP
+  # @note: some palm devices must come before iphone. sometimes spoofs iphone in ua
+  ##########
+  - regex: '(Pre|Pixi)/\d+\.\d+'
+    device_replacement: 'Palm $1'
+    brand_replacement: 'Palm'
+    model_replacement: '$1'
+  - regex: 'Palm([0-9]+)'
+    device_replacement: 'Palm $1'
+    brand_replacement: 'Palm'
+    model_replacement: '$1'
+  - regex: 'Treo([A-Za-z0-9]+)'
+    device_replacement: 'Palm Treo $1'
+    brand_replacement: 'Palm'
+    model_replacement: 'Treo $1'
+  - regex: 'webOS.*(P160U(?:NA)?)/(\d+).(\d+)'
+    device_replacement: 'HP Veer'
+    brand_replacement: 'HP'
+    model_replacement: 'Veer'
+  - regex: '(Touch[Pp]ad)/\d+\.\d+'
+    device_replacement: 'HP TouchPad'
+    brand_replacement: 'HP'
+    model_replacement: 'TouchPad'
+  - regex: 'HPiPAQ([A-Za-z0-9]+)/\d+.\d+'
+    device_replacement: 'HP iPAQ $1'
+    brand_replacement: 'HP'
+    model_replacement: 'iPAQ $1'
+  - regex: 'PDA; (PalmOS)/sony/model ([a-z]+)/Revision'
+    device_replacement: '$1'
+    brand_replacement: 'Sony'
+    model_replacement: '$1 $2'
+
+  ##########
+  # AppleTV
+  # No built in browser that I can tell
+  # Stack Overflow indicated iTunes-AppleTV/4.1 as a known UA for app available and I'm seeing it in live traffic
+  ##########
+  - regex: '(Apple\s?TV)'
+    device_replacement: 'AppleTV'
+    brand_replacement: 'Apple'
+    model_replacement: 'AppleTV'
+
+  #########
+  # Tesla Model S
+  #########
+  - regex: '(QtCarBrowser)'
+    device_replacement: 'Tesla Model S'
+    brand_replacement: 'Tesla'
+    model_replacement: 'Model S'
+
+  ##########
+  # iSTUFF
+  # @note: complete but probably catches spoofs
+  #   ipad and ipod must be parsed before iphone
+  #   cannot determine specific device type from ua string. (3g, 3gs, 4, etc)
+  ##########
+  # @note: on some ua the device can be identified e.g. iPhone5,1
+  - regex: '((?:iPhone|iPad|iPod)\d+,\d+)'
+    device_replacement: '$1'
+    brand_replacement: 'Apple'
+    model_replacement: '$1'
+  # @note: iPad needs to be before iPhone
+  - regex: '(iPad)(?:;| Simulator;)'
+    device_replacement: '$1'
+    brand_replacement: 'Apple'
+    model_replacement: '$1'
+  - regex: '(iPod)(?:;| touch;| Simulator;)'
+    device_replacement: '$1'
+    brand_replacement: 'Apple'
+    model_replacement: '$1'
+  - regex: '(iPhone)(?:;| Simulator;)'
+    device_replacement: '$1'
+    brand_replacement: 'Apple'
+    model_replacement: '$1'
+  # @note: desktop applications show device info
+  - regex: 'CFNetwork/.* Darwin/\d.*\(((?:Mac|iMac|PowerMac|PowerBook)[^\d]*)(\d+)(?:,|%2C)(\d+)'
+    device_replacement: '$1$2,$3'
+    brand_replacement: 'Apple'
+    model_replacement: '$1$2,$3'
+  # @note: iOS applications do not show device info
+  - regex: 'CFNetwork/.* Darwin/\d'
+    device_replacement: 'iOS-Device'
+    brand_replacement: 'Apple'
+    model_replacement: 'iOS-Device'
+
+  ##########
+  # Acer
+  ##########
+  - regex: 'acer_([A-Za-z0-9]+)_'
+    device_replacement: 'Acer $1'
+    brand_replacement: 'Acer'
+    model_replacement: '$1'
+
+  ##########
+  # Alcatel
+  ##########
+  - regex: '(?:ALCATEL|Alcatel)-([A-Za-z0-9\-]+)'
+    device_replacement: 'Alcatel $1'
+    brand_replacement: 'Alcatel'
+    model_replacement: '$1'
+
+  ##########
+  # Amoi
+  ##########
+  - regex: '(?:Amoi|AMOI)\-([A-Za-z0-9]+)'
+    device_replacement: 'Amoi $1'
+    brand_replacement: 'Amoi'
+    model_replacement: '$1'
+
+  ##########
+  # Asus
+  ##########
+  - regex: '(?:; |\/|^)((?:Transformer (?:Pad|Prime) |Transformer |PadFone[ _]?)[A-Za-z0-9]*)'
+    device_replacement: 'Asus $1'
+    brand_replacement: 'Asus'
+    model_replacement: '$1'
+  - regex: '(?:asus.*?ASUS|Asus|ASUS|asus)[\- ;]*((?:Transformer (?:Pad|Prime) |Transformer |Padfone |Nexus[ _])?[A-Za-z0-9]+)'
+    device_replacement: 'Asus $1'
+    brand_replacement: 'Asus'
+    model_replacement: '$1'
+
+
+  ##########
+  # Bird
+  ##########
+  - regex: '\bBIRD[ \-\.]([A-Za-z0-9]+)'
+    device_replacement: 'Bird $1'
+    brand_replacement: 'Bird'
+    model_replacement: '$1'
+
+  ##########
+  # Dell
+  ##########
+  - regex: '\bDell ([A-Za-z0-9]+)'
+    device_replacement: 'Dell $1'
+    brand_replacement: 'Dell'
+    model_replacement: '$1'
+
+  ##########
+  # DoCoMo
+  ##########
+  - regex: 'DoCoMo/2\.0 ([A-Za-z0-9]+)'
+    device_replacement: 'DoCoMo $1'
+    brand_replacement: 'DoCoMo'
+    model_replacement: '$1'
+  - regex: '([A-Za-z0-9]+)_W;FOMA'
+    device_replacement: 'DoCoMo $1'
+    brand_replacement: 'DoCoMo'
+    model_replacement: '$1'
+  - regex: '([A-Za-z0-9]+);FOMA'
+    device_replacement: 'DoCoMo $1'
+    brand_replacement: 'DoCoMo'
+    model_replacement: '$1'
+
+  ##########
+  # htc
+  ##########
+  - regex: '\b(?:HTC/|HTC/[a-z0-9]+/)?HTC[ _\-;]? *(.*?)(?:-?Mozilla|fingerPrint|[;/\(\)]|$)'
+    device_replacement: 'HTC $1'
+    brand_replacement: 'HTC'
+    model_replacement: '$1'
+
+  ##########
+  # Huawei
+  ##########
+  - regex: 'Huawei([A-Za-z0-9]+)'
+    device_replacement: 'Huawei $1'
+    brand_replacement: 'Huawei'
+    model_replacement: '$1'
+  - regex: 'HUAWEI-([A-Za-z0-9]+)'
+    device_replacement: 'Huawei $1'
+    brand_replacement: 'Huawei'
+    model_replacement: '$1'
+  - regex: 'vodafone([A-Za-z0-9]+)'
+    device_replacement: 'Huawei Vodafone $1'
+    brand_replacement: 'Huawei'
+    model_replacement: 'Vodafone $1'
+
+  ##########
+  # i-mate
+  ##########
+  - regex: 'i\-mate ([A-Za-z0-9]+)'
+    device_replacement: 'i-mate $1'
+    brand_replacement: 'i-mate'
+    model_replacement: '$1'
+
+  ##########
+  # kyocera
+  ##########
+  - regex: 'Kyocera\-([A-Za-z0-9]+)'
+    device_replacement: 'Kyocera $1'
+    brand_replacement: 'Kyocera'
+    model_replacement: '$1'
+  - regex: 'KWC\-([A-Za-z0-9]+)'
+    device_replacement: 'Kyocera $1'
+    brand_replacement: 'Kyocera'
+    model_replacement: '$1'
+
+  ##########
+  # lenovo
+  ##########
+  - regex: 'Lenovo[_\-]([A-Za-z0-9]+)'
+    device_replacement: 'Lenovo $1'
+    brand_replacement: 'Lenovo'
+    model_replacement: '$1'
+
+  ##########
+  # HbbTV (European and Australian standard)
+  # written before the LG regexes, as LG is making HbbTV too
+  ##########
+  - regex: '(HbbTV)/[0-9]+\.[0-9]+\.[0-9]+ \([^;]*; *(LG)E *; *([^;]*) *;[^;]*;[^;]*;\)'
+    device_replacement: '$1'
+    brand_replacement: '$2'
+    model_replacement: '$3'
+  - regex: '(HbbTV)/1\.1\.1.*CE-HTML/1\.\d;(Vendor/)*(THOM[^;]*?)[;\s](?:.*SW-Version/.*)*(LF[^;]+);?'
+    device_replacement: '$1'
+    brand_replacement: 'Thomson'
+    model_replacement: '$4'
+  - regex: '(HbbTV)(?:/1\.1\.1)?(?: ?\(;;;;;\))?; *CE-HTML(?:/1\.\d)?; *([^ ]+) ([^;]+);'
+    device_replacement: '$1'
+    brand_replacement: '$2'
+    model_replacement: '$3'
+  - regex: '(HbbTV)/1\.1\.1 \(;;;;;\) Maple_2011'
+    device_replacement: '$1'
+    brand_replacement: 'Samsung'
+  - regex: '(HbbTV)/[0-9]+\.[0-9]+\.[0-9]+ \([^;]*; *(?:CUS:([^;]*)|([^;]+)) *; *([^;]*) *;.*;'
+    device_replacement: '$1'
+    brand_replacement: '$2$3'
+    model_replacement: '$4'
+  - regex: '(HbbTV)/[0-9]+\.[0-9]+\.[0-9]+'
+    device_replacement: '$1'
+
+  ##########
+  # LGE NetCast TV
+  ##########
+  - regex: 'LGE; (?:Media\/)?([^;]*);[^;]*;[^;]*;?\); "?LG NetCast(\.TV|\.Media|)-\d+'
+    device_replacement: 'NetCast$2'
+    brand_replacement: 'LG'
+    model_replacement: '$1'
+
+  ##########
+  # InettvBrowser
+  ##########
+  - regex: 'InettvBrowser/[0-9]+\.[0-9A-Z]+ \([^;]*;(Sony)([^;]*);[^;]*;[^\)]*\)'
+    device_replacement: 'Inettv'
+    brand_replacement: '$1'
+    model_replacement: '$2'
+  - regex: 'InettvBrowser/[0-9]+\.[0-9A-Z]+ \([^;]*;([^;]*);[^;]*;[^\)]*\)'
+    device_replacement: 'Inettv'
+    brand_replacement: 'Generic_Inettv'
+    model_replacement: '$1'
+  - regex: '(?:InettvBrowser|TSBNetTV|NETTV|HBBTV)'
+    device_replacement: 'Inettv'
+    brand_replacement: 'Generic_Inettv'
+
+  ##########
+  # lg
+  ##########
+  # LG Symbian Phones
+  - regex: 'Series60/\d\.\d (LG)[\-]?([A-Za-z0-9 \-]+)'
+    device_replacement: '$1 $2'
+    brand_replacement: '$1'
+    model_replacement: '$2'
+  # other LG phones
+  - regex: '\b(?:LGE[ \-]LG\-(?:AX)?|LGE |LGE?-LG|LGE?[ \-]|LG[ /\-]|lg[\-])([A-Za-z0-9]+)\b'
+    device_replacement: 'LG $1'
+    brand_replacement: 'LG'
+    model_replacement: '$1'
+  - regex: '(?:^LG[\-]?|^LGE[\-/]?)([A-Za-z]+[0-9]+[A-Za-z]*)'
+    device_replacement: 'LG $1'
+    brand_replacement: 'LG'
+    model_replacement: '$1'
+  - regex: '^LG([0-9]+[A-Za-z]*)'
+    device_replacement: 'LG $1'
+    brand_replacement: 'LG'
+    model_replacement: '$1'
+
+  ##########
+  # microsoft
+  ##########
+  - regex: '(KIN\.[^ ]+) (\d+)\.(\d+)'
+    device_replacement: 'Microsoft $1'
+    brand_replacement: 'Microsoft'
+    model_replacement: '$1'
+  - regex: '(?:MSIE|XBMC).*\b(Xbox)\b'
+    device_replacement: '$1'
+    brand_replacement: 'Microsoft'
+    model_replacement: '$1'
+  - regex: '; ARM; Trident/6\.0; Touch[\);]'
+    device_replacement: 'Microsoft Surface RT'
+    brand_replacement: 'Microsoft'
+    model_replacement: 'Surface RT'
+
+  ##########
+  # motorola
+  ##########
+  - regex: 'Motorola\-([A-Za-z0-9]+)'
+    device_replacement: 'Motorola $1'
+    brand_replacement: 'Motorola'
+    model_replacement: '$1'
+  - regex: 'MOTO\-([A-Za-z0-9]+)'
+    device_replacement: 'Motorola $1'
+    brand_replacement: 'Motorola'
+    model_replacement: '$1'
+  - regex: 'MOT\-([A-z0-9][A-z0-9\-]*)'
+    device_replacement: 'Motorola $1'
+    brand_replacement: 'Motorola'
+    model_replacement: '$1'
+
+  ##########
+  # nintendo
+  ##########
+  - regex: 'Nintendo WiiU'
+    device_replacement: 'Nintendo Wii U'
+    brand_replacement: 'Nintendo'
+    model_replacement: 'Wii U'
+  - regex: 'Nintendo (DS|3DS|DSi|Wii);'
+    device_replacement: 'Nintendo $1'
+    brand_replacement: 'Nintendo'
+    model_replacement: '$1'
+
+  ##########
+  # pantech
+  ##########
+  - regex: '(?:Pantech|PANTECH)[ _-]?([A-Za-z0-9\-]+)'
+    device_replacement: 'Pantech $1'
+    brand_replacement: 'Pantech'
+    model_replacement: '$1'
+
+  ##########
+  # philips
+  ##########
+  - regex: 'Philips([A-Za-z0-9]+)'
+    device_replacement: 'Philips $1'
+    brand_replacement: 'Philips'
+    model_replacement: '$1'
+  - regex: 'Philips ([A-Za-z0-9]+)'
+    device_replacement: 'Philips $1'
+    brand_replacement: 'Philips'
+    model_replacement: '$1'
+
+  ##########
+  # Samsung
+  ##########
+  # Samsung Symbian Devices
+  - regex: 'SymbianOS/9\.\d.* Samsung[/\-]([A-Za-z0-9 \-]+)'
+    device_replacement: 'Samsung $1'
+    brand_replacement: 'Samsung'
+    model_replacement: '$1'
+  - regex: '(Samsung)(SGH)(i[0-9]+)'
+    device_replacement: '$1 $2$3'
+    brand_replacement: '$1'
+    model_replacement: '$2-$3'
+  - regex: 'SAMSUNG-ANDROID-MMS/([^;/]+)'
+    device_replacement: '$1'
+    brand_replacement: 'Samsung'
+    model_replacement: '$1'
+  # Other Samsung
+  #- regex: 'SAMSUNG(?:; |-)([A-Za-z0-9\-]+)'
+  - regex: 'SAMSUNG(?:; |[ -/])([A-Za-z0-9\-]+)'
+    regex_flag: 'i'
+    device_replacement: 'Samsung $1'
+    brand_replacement: 'Samsung'
+    model_replacement: '$1'
+
+  ##########
+  # Sega
+  ##########
+  - regex: '(Dreamcast)'
+    device_replacement: 'Sega $1'
+    brand_replacement: 'Sega'
+    model_replacement: '$1'
+
+  ##########
+  # Siemens mobile
+  ##########
+  - regex: '^SIE-([A-Za-z0-9]+)'
+    device_replacement: 'Siemens $1'
+    brand_replacement: 'Siemens'
+    model_replacement: '$1'
+
+  ##########
+  # Softbank
+  ##########
+  - regex: 'Softbank/[12]\.0/([A-Za-z0-9]+)'
+    device_replacement: 'Softbank $1'
+    brand_replacement: 'Softbank'
+    model_replacement: '$1'
+
+  ##########
+  # SonyEricsson
+  ##########
+  - regex: 'SonyEricsson ?([A-Za-z0-9\-]+)'
+    device_replacement: 'Ericsson $1'
+    brand_replacement: 'SonyEricsson'
+    model_replacement: '$1'
+
+  ##########
+  # Sony
+  ##########
+  - regex: 'Android [^;]+; ([^ ]+) (Sony)/'
+    device_replacement: '$2 $1'
+    brand_replacement: '$2'
+    model_replacement: '$1'
+  - regex: '(Sony)(?:BDP\/|\/)?([^ /;\)]+)[ /;\)]'
+    device_replacement: '$1 $2'
+    brand_replacement: '$1'
+    model_replacement: '$2'
+
+  #########
+  # Puffin Browser Device detect
+  # A=Android, I=iOS, P=Phone, T=Tablet
+  # AT=Android+Tablet
+  #########
+  - regex: 'Puffin/[\d\.]+IT'
+    device_replacement: 'iPad'
+    brand_replacement: 'Apple'
+    model_replacement: 'iPad'
+  - regex: 'Puffin/[\d\.]+IP'
+    device_replacement: 'iPhone'
+    brand_replacement: 'Apple'
+    model_replacement: 'iPhone'
+  - regex: 'Puffin/[\d\.]+AT'
+    device_replacement: 'Generic Tablet'
+    brand_replacement: 'Generic'
+    model_replacement: 'Tablet'
+  - regex: 'Puffin/[\d\.]+AP'
+    device_replacement: 'Generic Smartphone'
+    brand_replacement: 'Generic'
+    model_replacement: 'Smartphone'
+
+  #########
+  # Android General Device Matching (far from perfect)
+  #########
+  - regex: 'Android[\- ][\d]+\.[\d]+; [A-Za-z]{2}\-[A-Za-z]{0,2}; WOWMobile (.+) Build'
+    brand_replacement: 'Generic_Android'
+    model_replacement: '$1'
+  - regex: 'Android[\- ][\d]+\.[\d]+\-update1; [A-Za-z]{2}\-[A-Za-z]{0,2} *; *(.+?) Build'
+    brand_replacement: 'Generic_Android'
+    model_replacement: '$1'
+  - regex: 'Android[\- ][\d]+(?:\.[\d]+){1,2}; *[A-Za-z]{2}[_\-][A-Za-z]{0,2}\-? *; *(.+?) Build'
+    brand_replacement: 'Generic_Android'
+    model_replacement: '$1'
+  - regex: 'Android[\- ][\d]+(?:\.[\d]+){1,2}; *[A-Za-z]{0,2}\- *; *(.+?) Build'
+    brand_replacement: 'Generic_Android'
+    model_replacement: '$1'
+  # No build info at all - "Build" follows locale immediately
+  - regex: 'Android[\- ][\d]+(?:\.[\d]+){1,2}; *[a-z]{0,2}[_\-]?[A-Za-z]{0,2};? Build'
+    device_replacement: 'Generic Smartphone'
+    brand_replacement: 'Generic'
+    model_replacement: 'Smartphone'
+  - regex: 'Android[\- ][\d]+(?:\.[\d]+){1,2}; *\-?[A-Za-z]{2}; *(.+?) Build'
+    brand_replacement: 'Generic_Android'
+    model_replacement: '$1'
+  - regex: 'Android[\- ][\d]+(?:\.[\d]+){1,2}(?:;.*)?; *(.+?) Build'
+    brand_replacement: 'Generic_Android'
+    model_replacement: '$1'
+
+  ##########
+  # Google TV
+  ##########
+  - regex: '(GoogleTV)'
+    brand_replacement: 'Generic_Inettv'
+    model_replacement: '$1'
+
+  ##########
+  # WebTV
+  ##########
+  - regex: '(WebTV)/\d+.\d+'
+    brand_replacement: 'Generic_Inettv'
+    model_replacement: '$1'
+  # Roku Digital-Video-Players https://www.roku.com/
+  - regex: '^(Roku)/DVP-\d+\.\d+'
+    brand_replacement: 'Generic_Inettv'
+    model_replacement: '$1'
+
+  ##########
+  # Generic Tablet
+  ##########
+  - regex: '(Android 3\.\d|Opera Tablet|Tablet; .+Firefox/|Android.*(?:Tab|Pad))'
+    regex_flag: 'i'
+    device_replacement: 'Generic Tablet'
+    brand_replacement: 'Generic'
+    model_replacement: 'Tablet'
+
+  ##########
+  # Generic Smart Phone
+  ##########
+  - regex: '(Symbian|\bS60(Version|V\d)|\bS60\b|\((Series 60|Windows Mobile|Palm OS|Bada); Opera Mini|Windows CE|Opera Mobi|BREW|Brew|Mobile; .+Firefox/|iPhone OS|Android|MobileSafari|Windows *Phone|\(webOS/|PalmOS)'
+    device_replacement: 'Generic Smartphone'
+    brand_replacement: 'Generic'
+    model_replacement: 'Smartphone'
+  - regex: '(hiptop|avantgo|plucker|xiino|blazer|elaine)'
+    regex_flag: 'i'
+    device_replacement: 'Generic Smartphone'
+    brand_replacement: 'Generic'
+    model_replacement: 'Smartphone'
+
+  ##########
+  # Spiders (this is hack...)
+  ##########
+  - regex: '(bot|zao|borg|DBot|oegp|silk|Xenu|zeal|^NING|CCBot|crawl|htdig|lycos|slurp|teoma|voila|yahoo|Sogou|CiBra|Nutch|^Java/|^JNLP/|Daumoa|Genieo|ichiro|larbin|pompos|Scrapy|snappy|speedy|spider|msnbot|msrbot|vortex|^vortex|crawler|favicon|indexer|Riddler|scooter|scraper|scrubby|WhatWeb|WinHTTP|bingbot|openbot|gigabot|furlbot|polybot|seekbot|^voyager|archiver|Icarus6j|mogimogi|Netvibes|blitzbot|altavista|charlotte|findlinks|Retreiver|TLSProber|WordPress|SeznamBot|ProoXiBot|wsr\-agent|Squrl Java|EtaoSpider|PaperLiBot|SputnikBot|A6\-Indexer|netresearch|searchsight|baiduspider|YisouSpider|ICC\-Crawler|http%20client|Python-urllib|dataparksearch|converacrawler|Screaming Frog|AppEngine-Google|YahooCacheSystem|fast\-webcrawler|Sogou Pic Spider|semanticdiscovery|Innovazion Crawler|facebookexternalhit|Google.*/\+/web/snippet|Google-HTTP-Java-Client|BlogBridge|IlTrovatore-Setaccio|InternetArchive|GomezAgent|WebThumbnail|heritrix|NewsGator|PagePeeker|Reaper|ZooShot|holmes)'
+    regex_flag: 'i'
+    device_replacement: 'Spider'
+    brand_replacement: 'Spider'
+    model_replacement: 'Desktop'
+
+  ##########
+  # Generic Feature Phone
+  # take care to do case insensitive matching
+  ##########
+  - regex: '^(1207|3gso|4thp|501i|502i|503i|504i|505i|506i|6310|6590|770s|802s|a wa|acer|acs\-|airn|alav|asus|attw|au\-m|aur |aus |abac|acoo|aiko|alco|alca|amoi|anex|anny|anyw|aptu|arch|argo|bmobile|bell|bird|bw\-n|bw\-u|beck|benq|bilb|blac|c55/|cdm\-|chtm|capi|comp|cond|dall|dbte|dc\-s|dica|ds\-d|ds12|dait|devi|dmob|doco|dopo|dorado|el(?:38|39|48|49|50|55|58|68)|el[3456]\d{2}dual|erk0|esl8|ex300|ez40|ez60|ez70|ezos|ezze|elai|emul|eric|ezwa|fake|fly\-|fly_|g\-mo|g1 u|g560|gf\-5|grun|gene|go.w|good|grad|hcit|hd\-m|hd\-p|hd\-t|hei\-|hp i|hpip|hs\-c|htc |htc\-|htca|htcg)'
+    regex_flag: 'i'
+    device_replacement: 'Generic Feature Phone'
+    brand_replacement: 'Generic'
+    model_replacement: 'Feature Phone'
+  - regex: '^(htcp|htcs|htct|htc_|haie|hita|huaw|hutc|i\-20|i\-go|i\-ma|i\-mobile|i230|iac|iac\-|iac/|ig01|im1k|inno|iris|jata|kddi|kgt|kgt/|kpt |kwc\-|klon|lexi|lg g|lg\-a|lg\-b|lg\-c|lg\-d|lg\-f|lg\-g|lg\-k|lg\-l|lg\-m|lg\-o|lg\-p|lg\-s|lg\-t|lg\-u|lg\-w|lg/k|lg/l|lg/u|lg50|lg54|lge\-|lge/|leno|m1\-w|m3ga|m50/|maui|mc01|mc21|mcca|medi|meri|mio8|mioa|mo01|mo02|mode|modo|mot |mot\-|mt50|mtp1|mtv |mate|maxo|merc|mits|mobi|motv|mozz|n100|n101|n102|n202|n203|n300|n302|n500|n502|n505|n700|n701|n710|nec\-|nem\-|newg|neon)'
+    regex_flag: 'i'
+    device_replacement: 'Generic Feature Phone'
+    brand_replacement: 'Generic'
+    model_replacement: 'Feature Phone'
+  - regex: '^(netf|noki|nzph|o2 x|o2\-x|opwv|owg1|opti|oran|ot\-s|p800|pand|pg\-1|pg\-2|pg\-3|pg\-6|pg\-8|pg\-c|pg13|phil|pn\-2|pt\-g|palm|pana|pire|pock|pose|psio|qa\-a|qc\-2|qc\-3|qc\-5|qc\-7|qc07|qc12|qc21|qc32|qc60|qci\-|qwap|qtek|r380|r600|raks|rim9|rove|s55/|sage|sams|sc01|sch\-|scp\-|sdk/|se47|sec\-|sec0|sec1|semc|sgh\-|shar|sie\-|sk\-0|sl45|slid|smb3|smt5|sp01|sph\-|spv |spv\-|sy01|samm|sany|sava|scoo|send|siem|smar|smit|soft|sony|t\-mo|t218|t250|t600|t610|t618|tcl\-|tdg\-|telm|tim\-|ts70|tsm\-|tsm3|tsm5|tx\-9|tagt)'
+    regex_flag: 'i'
+    device_replacement: 'Generic Feature Phone'
+    brand_replacement: 'Generic'
+    model_replacement: 'Feature Phone'
+  - regex: '^(talk|teli|topl|tosh|up.b|upg1|utst|v400|v750|veri|vk\-v|vk40|vk50|vk52|vk53|vm40|vx98|virg|vertu|vite|voda|vulc|w3c |w3c\-|wapj|wapp|wapu|wapm|wig |wapi|wapr|wapv|wapy|wapa|waps|wapt|winc|winw|wonu|x700|xda2|xdag|yas\-|your|zte\-|zeto|aste|audi|avan|blaz|brew|brvw|bumb|ccwa|cell|cldc|cmd\-|dang|eml2|fetc|hipt|http|ibro|idea|ikom|ipaq|jbro|jemu|jigs|keji|kyoc|kyok|libw|m\-cr|midp|mmef|moto|mwbp|mywa|newt|nok6|o2im|pant|pdxg|play|pluc|port|prox|rozo|sama|seri|smal|symb|treo|upsi|vx52|vx53|vx60|vx61|vx70|vx80|vx81|vx83|vx85|wap\-|webc|whit|wmlb|xda\-|xda_)'
+    regex_flag: 'i'
+    device_replacement: 'Generic Feature Phone'
+    brand_replacement: 'Generic'
+    model_replacement: 'Feature Phone'
+  - regex: '^(Ice)$'
+    device_replacement: 'Generic Feature Phone'
+    brand_replacement: 'Generic'
+    model_replacement: 'Feature Phone'
+  - regex: '(wap[\-\ ]browser|maui|netfront|obigo|teleca|up\.browser|midp|Opera Mini)'
+    regex_flag: 'i'
+    device_replacement: 'Generic Feature Phone'
+    brand_replacement: 'Generic'
+    model_replacement: 'Feature Phone'
diff --git a/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java b/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java
new file mode 100644
index 00000000000..516b7ce3206
--- /dev/null
+++ b/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.useragent;
+
+import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.ingest.AbstractProcessorFactory;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.BeforeClass;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+
+public class UserAgentProcessorFactoryTests extends ESTestCase {
+
+    private static Map<String, UserAgentParser> userAgentParsers;
+
+    private static String regexWithoutDevicesFilename = "regexes_without_devices.yaml";
+    private static Path userAgentConfigDir;
+
+    @BeforeClass
+    public static void createUserAgentParsers() throws IOException {
+        Path configDir = createTempDir();
+        userAgentConfigDir = configDir.resolve("ingest-useragent");
+        Files.createDirectories(userAgentConfigDir);
+
+        // Copy file, leaving out the device parsers at the end
+        try (BufferedReader reader = new BufferedReader(
+                new InputStreamReader(UserAgentProcessor.class.getResourceAsStream("/regexes.yaml"), StandardCharsets.UTF_8));
+                BufferedWriter writer = Files.newBufferedWriter(userAgentConfigDir.resolve(regexWithoutDevicesFilename));) {
+            String line;
+            while ((line = reader.readLine()) != null) {
+                if (line.startsWith("device_parsers:")) {
+                    break;
+                }
+
+                writer.write(line);
+                writer.newLine();
+            }
+        }
+
+        userAgentParsers = IngestUserAgentPlugin.createUserAgentParsers(userAgentConfigDir, new UserAgentCache(1000));
+    }
+
+    public void testBuildDefaults() throws Exception {
+        UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers);
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "_field");
+
+        String processorTag = randomAsciiOfLength(10);
+        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
+
+        UserAgentProcessor processor = factory.create(config);
+        assertThat(processor.getTag(), equalTo(processorTag));
+        assertThat(processor.getField(), equalTo("_field"));
+        assertThat(processor.getTargetField(), equalTo("useragent"));
+        assertThat(processor.getUaParser().getUaPatterns().size(), greaterThan(0));
+        assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0));
+        assertThat(processor.getUaParser().getDevicePatterns().size(), greaterThan(0));
+        assertThat(processor.getProperties(), equalTo(EnumSet.allOf(UserAgentProcessor.Property.class)));
+    }
+
+    public void testBuildTargetField() throws Exception {
+        UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers);
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("target_field", "_target_field");
+
+        UserAgentProcessor processor = factory.create(config);
+        assertThat(processor.getField(), equalTo("_field"));
+        assertThat(processor.getTargetField(), equalTo("_target_field"));
+    }
+
+    public void testBuildRegexFile() throws Exception {
+        UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers);
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("regex_file", regexWithoutDevicesFilename);
+
+        UserAgentProcessor processor = factory.create(config);
+        assertThat(processor.getField(), equalTo("_field"));
+        assertThat(processor.getUaParser().getUaPatterns().size(), greaterThan(0));
+        assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0));
+        assertThat(processor.getUaParser().getDevicePatterns().size(), equalTo(0));
+    }
+
+    public void testBuildNonExistingRegexFile() throws Exception {
+        UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers);
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("regex_file", "does-not-exist.yaml");
+
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        assertThat(e.getMessage(), equalTo("[regex_file] regex file [does-not-exist.yaml] doesn't exist (has to exist at node startup)"));
+    }
+
+    public void testBuildFields() throws Exception {
+        UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers);
+
+        Set<UserAgentProcessor.Property> properties = EnumSet.noneOf(UserAgentProcessor.Property.class);
+        List<String> fieldNames = new ArrayList<>();
+        int numFields = scaledRandomIntBetween(1, UserAgentProcessor.Property.values().length);
+        for (int i = 0; i < numFields; i++) {
+            UserAgentProcessor.Property property = UserAgentProcessor.Property.values()[i];
+            properties.add(property);
+            fieldNames.add(property.name().toLowerCase(Locale.ROOT));
+        }
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("properties", fieldNames);
+
+        UserAgentProcessor processor = factory.create(config);
+        assertThat(processor.getField(), equalTo("_field"));
+        assertThat(processor.getProperties(), equalTo(properties));
+    }
+
+    public void testInvalidProperty() throws Exception {
+        UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers);
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("properties", Collections.singletonList("invalid"));
+
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, MAJOR, MINOR, "
+                + "PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD]"));
+    }
+
+    public void testInvalidPropertiesType() throws Exception {
+        UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers);
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("properties", "invalid");
+
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]"));
+    }
+}
diff --git a/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java b/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java
new file mode 100644
index 00000000000..d9b5eed059a
--- /dev/null
+++ b/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.useragent;
+
+import org.elasticsearch.ingest.RandomDocumentPicks;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.useragent.UserAgentProcessor;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.is;
+
+public class UserAgentProcessorTests extends ESTestCase {
+
+    private static UserAgentProcessor processor;
+    
+    @BeforeClass
+    public static void setupProcessor() throws IOException {
+        InputStream regexStream = UserAgentProcessor.class.getResourceAsStream("/regexes.yaml");
+        assertNotNull(regexStream);
+        
+        UserAgentParser parser = new UserAgentParser(randomAsciiOfLength(10), regexStream, new UserAgentCache(1000));
+        
+        processor = new UserAgentProcessor(randomAsciiOfLength(10), "source_field", "target_field", parser,
+                EnumSet.allOf(UserAgentProcessor.Property.class));
+    }
+    
+    @SuppressWarnings("unchecked")
+    public void testCommonBrowser() throws Exception {
+        Map<String, Object> document = new HashMap<>();
+        document.put("source_field",
+            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36");
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        
+        processor.execute(ingestDocument);
+        Map<String, Object> data = ingestDocument.getSourceAndMetadata();
+
+        assertThat(data, hasKey("target_field"));
+        Map<String, Object> target = (Map<String, Object>) data.get("target_field");
+        
+        assertThat(target.get("name"), is("Chrome"));
+        assertThat(target.get("major"), is("33"));
+        assertThat(target.get("minor"), is("0"));
+        assertThat(target.get("patch"), is("1750"));
+        assertNull(target.get("build"));
+        
+        assertThat(target.get("os"), is("Mac OS X 10.9.2"));
+        assertThat(target.get("os_name"), is("Mac OS X"));
+        assertThat(target.get("os_major"), is("10"));
+        assertThat(target.get("os_minor"), is("9"));
+        
+        assertThat(target.get("device"), is("Other"));
+    }
+    
+    @SuppressWarnings("unchecked")
+    public void testUncommonDevice() throws Exception {
+        Map<String, Object> document = new HashMap<>();
+        document.put("source_field",
+                "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/525.10+ "
+                + "(KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2");
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        
+        processor.execute(ingestDocument);
+        Map<String, Object> data = ingestDocument.getSourceAndMetadata();
+
+        assertThat(data, hasKey("target_field"));
+        Map<String, Object> target = (Map<String, Object>) data.get("target_field");
+        
+        assertThat(target.get("name"), is("Android"));
+        assertThat(target.get("major"), is("3"));
+        assertThat(target.get("minor"), is("0"));
+        assertNull(target.get("patch"));
+        assertNull(target.get("build"));
+        
+        assertThat(target.get("os"), is("Android 3.0"));
+        assertThat(target.get("os_name"), is("Android"));
+        assertThat(target.get("os_major"), is("3"));
+        assertThat(target.get("os_minor"), is("0"));
+        
+        assertThat(target.get("device"), is("Motorola Xoom"));
+    }
+    
+    @SuppressWarnings("unchecked")
+    public void testSpider() throws Exception {
+        Map<String, Object> document = new HashMap<>();
+        document.put("source_field",
+            "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)");
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        
+        processor.execute(ingestDocument);
+        Map<String, Object> data = ingestDocument.getSourceAndMetadata();
+
+        assertThat(data, hasKey("target_field"));
+        Map<String, Object> target = (Map<String, Object>) data.get("target_field");
+        
+        assertThat(target.get("name"), is("EasouSpider"));
+        assertNull(target.get("major"));
+        assertNull(target.get("minor"));
+        assertNull(target.get("patch"));
+        assertNull(target.get("build"));
+        
+        assertThat(target.get("os"), is("Other"));
+        assertThat(target.get("os_name"), is("Other"));
+        assertNull(target.get("os_major"));
+        assertNull(target.get("os_minor"));
+        
+        assertThat(target.get("device"), is("Spider"));
+    }
+    
+    @SuppressWarnings("unchecked")
+    public void testUnknown() throws Exception {
+        Map<String, Object> document = new HashMap<>();
+        document.put("source_field",
+            "Something I made up v42.0.1");
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+        
+        processor.execute(ingestDocument);
+        Map<String, Object> data = ingestDocument.getSourceAndMetadata();
+
+        assertThat(data, hasKey("target_field"));
+        Map<String, Object> target = (Map<String, Object>) data.get("target_field");
+        
+        assertThat(target.get("name"), is("Other"));
+        assertNull(target.get("major"));
+        assertNull(target.get("minor"));
+        assertNull(target.get("patch"));
+        assertNull(target.get("build"));
+        
+        assertThat(target.get("os"), is("Other"));
+        assertThat(target.get("os_name"), is("Other"));
+        assertNull(target.get("os_major"));
+        assertNull(target.get("os_minor"));
+        
+        assertThat(target.get("device"), is("Other"));
+    }
+}
+
diff --git a/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentRestIT.java b/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentRestIT.java
new file mode 100644
index 00000000000..010f85f671a
--- /dev/null
+++ b/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentRestIT.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest.useragent;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.RestTestCandidate;
+import org.elasticsearch.test.rest.parser.RestTestParseException;
+
+import java.io.IOException;
+
+public class UserAgentRestIT extends ESRestTestCase {
+
+    public UserAgentRestIT(@Name("yaml") RestTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws IOException, RestTestParseException {
+        return ESRestTestCase.createParameters(0, 1);
+    }
+}
diff --git a/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/10_basic.yaml b/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/10_basic.yaml
new file mode 100644
index 00000000000..d37ad674cb9
--- /dev/null
+++ b/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/10_basic.yaml
@@ -0,0 +1,11 @@
+"ingest-useragent plugin installed":
+    - do:
+        cluster.state: {}
+
+    - set: {master_node: master}
+
+    - do:
+        nodes.info: {}
+
+    - match:  { nodes.$master.plugins.0.name: ingest-useragent }
+    - match:  { nodes.$master.ingest.processors.0.type: useragent }
diff --git a/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yaml b/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yaml
new file mode 100644
index 00000000000..13451a075d7
--- /dev/null
+++ b/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yaml
@@ -0,0 +1,86 @@
+---
+"Test user agent processor with defaults":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "useragent" : {
+                  "field" : "field1"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: {field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36"}
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
+  - match: { _source.useragent.name: "Chrome" }
+  - match: { _source.useragent.os: "Mac OS X 10.9.2" }
+  - match: { _source.useragent.os_name: "Mac OS X" }
+  - match: { _source.useragent.os_major: "10" }
+  - match: { _source.useragent.os_minor: "9" }
+  - match: { _source.useragent.major: "33" }
+  - match: { _source.useragent.minor: "0" }
+  - match: { _source.useragent.patch: "1750" }
+  - match: { _source.useragent.device: "Other" }
+
+---
+"Test user agent processor with parameters":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "useragent" : {
+                  "field" : "field1",
+                  "target_field": "field2",
+                  "properties": ["os"]
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: {field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36"}
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
+  - match: { _source.field2.os: "Mac OS X 10.9.2" }
+  - is_false: _source.useragent
+  - is_false: _source.field2.name
+  - is_false: _source.field2.os_name
+  - is_false: _source.field2.os_major
+  - is_false: _source.field2.os_minor
+  - is_false: _source.field2.major
+  - is_false: _source.field2.minor
+  - is_false: _source.field2.patch
+  - is_false: _source.field2.device
diff --git a/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml b/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml
new file mode 100644
index 00000000000..3472cd03c3f
--- /dev/null
+++ b/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml
@@ -0,0 +1,42 @@
+---
+"Test user agent processor with custom regex file":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "useragent" : {
+                  "field": "field1",
+                  "regex_file": "test-regexes.yaml"  
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        type: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: {field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36"}
+
+  - do:
+      get:
+        index: test
+        type: test
+        id: 1
+  - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
+  - match: { _source.useragent.name: "Test" }
+  - match: { _source.useragent.os: "Other" }
+  - match: { _source.useragent.os_name: "Other" }
+  - match: { _source.useragent.device: "Other" }
+  - is_false: _source.useragent.os_major
+  - is_false: _source.useragent.os_minor
+  - is_false: _source.useragent.major
+  - is_false: _source.useragent.minor
+  - is_false: _source.useragent.patch
diff --git a/plugins/ingest-useragent/test/test-regexes.yaml b/plugins/ingest-useragent/test/test-regexes.yaml
new file mode 100644
index 00000000000..e41dec700c0
--- /dev/null
+++ b/plugins/ingest-useragent/test/test-regexes.yaml
@@ -0,0 +1,3 @@
+user_agent_parsers:
+  - regex: '.*'
+    family_replacement: 'Test'
\ No newline at end of file
diff --git a/settings.gradle b/settings.gradle
index 6588b605a9d..0c1e4b01e5c 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -36,6 +36,7 @@ List projects = [
   'plugins:discovery-gce',
   'plugins:ingest-geoip',
   'plugins:ingest-attachment',
+  'plugins:ingest-useragent',
   'plugins:lang-javascript',
   'plugins:lang-python',
   'plugins:mapper-attachments',

From 5c8164a5618b4ac8f9b7a3530c68771925c96896 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Fri, 1 Jul 2016 16:09:31 +0200
Subject: [PATCH 1183/1311] Clean up BytesReference (#19196)

BytesReference should be a really simple interface, yet it has a gazillion
ways to achieve the same this. Methods like `#hasArray`, `#toBytesArray`, `#copyBytesArray`
`#toBytesRef` `#bytes` are all really duplicates. This change simplifies the interface
dramatically and makes implementations of it much simpler. All array access has been removed
and is streamlined through a single `#toBytesRef` method. Utility methods to materialize a
compact byte array has been added too for convenience.
---
 .../indices/create/CreateIndexRequest.java    |   2 +-
 .../action/termvectors/TermVectorsFields.java |   4 +-
 .../termvectors/TermVectorsRequest.java       |   3 +-
 .../termvectors/TermVectorsResponse.java      |   4 +-
 .../elasticsearch/cluster/ClusterState.java   |   4 +-
 .../common/bytes/BytesArray.java              |  68 +----
 .../common/bytes/BytesReference.java          | 232 +++++++++-------
 .../common/bytes/PagedBytesReference.java     | 176 +-----------
 .../common/compress/CompressedXContent.java   |   8 +-
 .../common/io/stream/StreamInput.java         |   7 -
 .../org/elasticsearch/common/text/Text.java   |  10 +-
 .../text/UTF8SortedAsUnicodeComparator.java   |  77 ------
 .../common/xcontent/XContentBuilder.java      |  59 ++--
 .../common/xcontent/XContentHelper.java       |  28 +-
 .../common/xcontent/cbor/CborXContent.java    |   3 -
 .../common/xcontent/json/JsonXContent.java    |   3 -
 .../xcontent/json/JsonXContentGenerator.java  |  18 +-
 .../common/xcontent/smile/SmileXContent.java  |   3 -
 .../common/xcontent/yaml/YamlXContent.java    |   3 -
 .../discovery/local/LocalDiscovery.java       |   3 +-
 .../elasticsearch/index/get/GetResult.java    |   2 +-
 .../index/mapper/SourceToParse.java           |   3 +-
 .../mapper/internal/SourceFieldMapper.java    |   7 +-
 .../index/query/WrapperQueryBuilder.java      |   3 +-
 .../index/shard/CommitPoints.java             |   3 +-
 .../elasticsearch/index/shard/IndexShard.java |   6 +-
 .../org/elasticsearch/index/store/Store.java  |   4 +-
 .../elasticsearch/indices/IndicesService.java |   2 +-
 .../indices/recovery/RecoveryTarget.java      |   9 +-
 .../ingest/InternalTemplateService.java       |   2 +-
 .../put/RestPutRepositoryAction.java          |   2 +-
 .../create/RestCreateSnapshotAction.java      |   2 +-
 .../restore/RestRestoreSnapshotAction.java    |   2 +-
 .../indices/analyze/RestAnalyzeAction.java    |   2 +-
 .../mapping/put/RestPutMappingAction.java     |   2 +-
 .../settings/RestUpdateSettingsAction.java    |   2 +-
 .../action/search/RestClearScrollAction.java  |   2 +-
 .../action/search/RestSearchScrollAction.java |   2 +-
 .../elasticsearch/script/ScriptService.java   |   2 +-
 .../org/elasticsearch/script/Template.java    |   6 +-
 .../search/internal/InternalSearchHit.java    |   2 +-
 .../internal/ShardSearchLocalRequest.java     |   3 +-
 .../completion2x/CompletionSuggestion.java    |   8 +-
 .../transport/local/LocalTransport.java       |   3 +-
 .../local/LocalTransportChannel.java          |   5 +-
 .../netty/ChannelBufferBytesReference.java    |  87 ++----
 .../org/elasticsearch/ESExceptionTests.java   |   6 +-
 .../ExceptionSerializationTests.java          |   2 +-
 .../action/OriginalIndicesTests.java          |   2 +-
 .../ClusterAllocationExplanationTests.java    |   2 +-
 .../health/ClusterHealthResponsesTests.java   |   2 +-
 .../reroute/ClusterRerouteRequestTests.java   |   2 +-
 .../cluster/reroute/ClusterRerouteTests.java  |   2 +-
 .../state/ClusterStateRequestTests.java       |   2 +-
 .../GetStoredScriptRequestTests.java          |   2 +-
 .../indices/flush/SyncedFlushUnitTests.java   |   2 +-
 .../action/bulk/BulkRequestTests.java         |   8 +-
 .../action/get/MultiGetShardRequestTests.java |   2 +-
 .../SimulateDocumentSimpleResultTests.java    |   2 +-
 .../ingest/SimulatePipelineResponseTests.java |   2 +-
 .../ingest/SimulateProcessorResultTests.java  |   2 +-
 .../ingest/WritePipelineResponseTests.java    |   4 +-
 .../ingest/WriteableIngestDocumentTests.java  |   2 +-
 .../action/main/MainActionTests.java          |   2 +-
 .../action/support/IndicesOptionsTests.java   |   2 +-
 .../GetTermVectorsCheckDocFreqIT.java         |   6 +-
 .../cluster/ClusterStateDiffIT.java           |   3 +-
 .../cluster/block/ClusterBlockTests.java      |   2 +-
 .../health/ClusterStateHealthTests.java       |   2 +-
 .../cluster/metadata/IndexGraveyardTests.java |   3 +-
 .../cluster/metadata/IndexMetaDataTests.java  |   2 +-
 .../cluster/metadata/MetaDataTests.java       |   5 +-
 .../cluster/routing/ShardRoutingTests.java    |   2 +-
 .../cluster/routing/UnassignedInfoTests.java  |   2 +-
 .../allocation/AllocationCommandsTests.java   |   2 +-
 .../ClusterSerializationTests.java            |   2 +-
 .../cluster/serialization/DiffableTests.java  |   2 +-
 .../elasticsearch/common/ChannelsTests.java   |   9 +-
 .../bytes/AbstractBytesReferenceTestCase.java | 260 +++++++++++-------
 .../common/bytes/BytesArrayTests.java         |  20 +-
 .../common/bytes/BytesReferenceTests.java     |  61 ----
 .../bytes/PagedBytesReferenceTests.java       |  52 ++--
 .../DeflateCompressedXContentTests.java       |   4 +-
 .../common/geo/GeoDistanceTests.java          |   4 +-
 .../common/geo/ShapeRelationTests.java        |  14 +-
 .../common/geo/SpatialStrategyTests.java      |  10 +-
 .../AbstractShapeBuilderTestCase.java         |   2 +-
 .../elasticsearch/common/io/StreamsTests.java |   2 +-
 .../stream/AbstractWriteableEnumTestCase.java |   4 +-
 .../common/io/stream/BytesStreamsTests.java   |  51 ++--
 .../common/io/stream/StreamTests.java         |   8 +-
 .../transport/BoundTransportAddressTests.java |   2 +-
 .../common/unit/DistanceUnitTests.java        |   2 +-
 .../common/unit/FuzzinessTests.java           |   2 +-
 .../common/unit/TimeValueTests.java           |   2 +-
 .../util/concurrent/ThreadContextTests.java   |   8 +-
 .../common/xcontent/XContentFactoryTests.java |   5 +-
 .../builder/XContentBuilderTests.java         |  15 +-
 .../common/xcontent/cbor/JsonVsCborTests.java |   3 +-
 .../xcontent/smile/JsonVsSmileTests.java      |   3 +-
 ...bstractFilteringJsonGeneratorTestCase.java |  13 +-
 .../FilterPathGeneratorFilteringTests.java    |   2 +-
 .../deps/jackson/JacksonLocationTests.java    |   3 +-
 .../elasticsearch/http/HttpServerTests.java   |   3 +-
 .../index/IndexRequestBuilderIT.java          |   3 +-
 .../index/engine/InternalEngineTests.java     |  12 +-
 .../index/engine/ShadowEngineTests.java       |  16 +-
 .../mapper/all/SimpleAllMapperTests.java      |   7 +-
 .../mapper/binary/BinaryMappingTests.java     |   3 +-
 .../timestamp/TimestampMappingTests.java      |   6 +-
 .../index/query/CombineFunctionTests.java     |  24 +-
 .../index/query/InnerHitBuilderTests.java     |   2 +-
 .../query/MoreLikeThisQueryBuilderTests.java  |   5 +-
 .../index/query/OperatorTests.java            |   8 +-
 .../index/query/ScoreModeTests.java           |  24 +-
 .../index/query/WrapperQueryBuilderTests.java |   2 +-
 ...FieldValueFactorFunctionModifierTests.java |  40 +--
 .../index/shard/IndexShardTests.java          |   6 +-
 .../snapshots/blobstore/FileInfoTests.java    |   5 +-
 .../index/store/CorruptedFileIT.java          |  11 +-
 .../index/translog/TranslogTests.java         |  52 ++--
 .../indices/IndicesRequestCacheTests.java     |  30 +-
 .../indices/TermsLookupTests.java             |   2 +-
 .../indices/recovery/RecoveryTargetTests.java |   2 +-
 .../indices/stats/IndexStatsIT.java           |   4 +-
 .../ingest/IngestStatsTests.java              |   2 +-
 .../nodesinfo/NodeInfoStreamingTests.java     |   2 +-
 .../elasticsearch/recovery/RelocationIT.java  |   6 +-
 .../rest/BytesRestResponseTests.java          |  21 +-
 .../rest/action/main/RestMainActionTests.java |   2 +-
 .../rest/action/support/RestTableTests.java   |   2 +-
 .../search/DocValueFormatTests.java           |  12 +-
 .../search/MultiValueModeTests.java           |  20 +-
 .../aggregations/BaseAggregationTestCase.java |   4 +-
 .../BasePipelineAggregationTestCase.java      |   4 +-
 .../SubAggCollectionModeTests.java            |  10 +-
 .../percentiles/PercentilesMethodTests.java   |  10 +-
 .../builder/SearchSourceBuilderTests.java     |   4 +-
 .../highlight/HighlightBuilderTests.java      |   6 +-
 .../internal/InternalSearchHitTests.java      |   2 +-
 .../matchedqueries/MatchedQueriesIT.java      |   4 +-
 .../rescore/QueryRescoreBuilderTests.java     |   2 +-
 .../searchafter/SearchAfterBuilderTests.java  |   2 +-
 .../search/slice/SliceBuilderTests.java       |   2 +-
 .../search/sort/AbstractSortTestCase.java     |   2 +-
 .../search/sort/SortOrderTests.java           |   2 +-
 .../AbstractSuggestionBuilderTestCase.java    |   2 +-
 .../suggest/completion/WritableTestCase.java  |   2 +-
 .../phrase/DirectCandidateGeneratorTests.java |   2 +-
 .../phrase/SmoothingModelTestCase.java        |   2 +-
 .../DedicatedClusterSnapshotRestoreIT.java    |   8 +-
 .../snapshots/SnapshotRequestsTests.java      |   5 +-
 .../snapshots/SnapshotTests.java              |   5 +-
 .../tasks/PersistedTaskInfoTests.java         |   4 +-
 .../org/elasticsearch/tasks/TaskIdTests.java  |   2 +-
 .../ThreadPoolSerializationTests.java         |   6 +-
 .../ChannelBufferBytesReferenceTests.java     |  36 ++-
 .../transport/netty/NettyUtilsTests.java      |  20 +-
 .../elasticsearch/update/UpdateNoopIT.java    |   4 +-
 .../messy/tests/SearchFieldsTests.java        |   2 +-
 .../template/RestSearchTemplateAction.java    |   2 +-
 .../template/MultiSearchTemplateIT.java       |   8 +-
 .../mustache/MustacheScriptEngineTests.java   |   5 +-
 .../script/mustache/MustacheTests.java        |  20 +-
 .../percolator/PercolateQuery.java            |  10 +-
 .../percolator/PercolatorFieldMapper.java     |   3 +-
 .../SimpleAttachmentMapperTests.java          |   2 +-
 .../bytes/ByteBufferBytesReference.java       | 167 -----------
 .../test/AbstractQueryTestCase.java           |   4 +-
 .../elasticsearch/test/ESIntegTestCase.java   |   3 +-
 .../org/elasticsearch/test/StreamsUtils.java  |   3 +-
 .../hamcrest/ElasticsearchAssertions.java     |   2 +-
 .../test/store/MockFSDirectoryService.java    |   4 +-
 .../test/transport/MockTransportService.java  |   2 +-
 174 files changed, 886 insertions(+), 1369 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/common/text/UTF8SortedAsUnicodeComparator.java
 delete mode 100644 core/src/test/java/org/elasticsearch/common/bytes/BytesReferenceTests.java
 delete mode 100644 test/framework/src/main/java/org/elasticsearch/common/bytes/ByteBufferBytesReference.java

diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
index 43d45672aec..b808484cef5 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java
@@ -364,7 +364,7 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>
                 throw new ElasticsearchParseException("failed to parse source for create index", e);
             }
         } else {
-            settings(new String(source.toBytes(), StandardCharsets.UTF_8));
+            settings(source.utf8ToString());
         }
         return this;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFields.java b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFields.java
index 9c745adf864..0ae8824ce8d 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFields.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsFields.java
@@ -130,7 +130,7 @@ public final class TermVectorsFields extends Fields {
      * @param termVectors Stores the actual term vectors as a {@link BytesRef}.
      */
     public TermVectorsFields(BytesReference headerRef, BytesReference termVectors) throws IOException {
-        StreamInput header = StreamInput.wrap(headerRef.toBytesArray());
+        StreamInput header = headerRef.streamInput();
         fieldMap = new ObjectLongHashMap<>();
         // here we read the header to fill the field offset map
         String headerString = header.readString();
@@ -201,7 +201,7 @@ public final class TermVectorsFields extends Fields {
         private int docCount;
 
         public TermVector(BytesReference termVectors, long readOffset) throws IOException {
-            this.perFieldTermVectorInput = StreamInput.wrap(termVectors.toBytesArray());
+            this.perFieldTermVectorInput = termVectors.streamInput();
             this.readOffset = readOffset;
             reset();
         }
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java
index 6f5bc3dccfa..4080d40e695 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java
@@ -27,6 +27,7 @@ import org.elasticsearch.action.ValidateActions;
 import org.elasticsearch.action.get.MultiGetRequest;
 import org.elasticsearch.action.support.single.shard.SingleShardRequest;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -157,7 +158,7 @@ public class TermVectorsRequest extends SingleShardRequest<TermVectorsRequest> i
         this.id = other.id();
         this.type = other.type();
         if (this.doc != null) {
-            this.doc = other.doc().copyBytesArray();
+            this.doc = new BytesArray(other.doc().toBytesRef(), true);
         }
         this.flagsEnum = other.getFlags().clone();
         this.preference = other.preference();
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java
index 0ef588f2947..964aa00b5c3 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsResponse.java
@@ -143,8 +143,8 @@ public class TermVectorsResponse extends ActionResponse implements ToXContent {
     public Fields getFields() throws IOException {
         if (hasTermVectors() && isExists()) {
             if (!sourceCopied) { // make the bytes safe
-                headerRef = headerRef.copyBytesArray();
-                termVectors = termVectors.copyBytesArray();
+                headerRef = new BytesArray(headerRef.toBytesRef(), true);
+                termVectors = new BytesArray(termVectors.toBytesRef(), true);
             }
             TermVectorsFields termVectorsFields = new TermVectorsFields(headerRef, termVectors);
             hasScores = termVectorsFields.hasScores;
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterState.java b/core/src/main/java/org/elasticsearch/cluster/ClusterState.java
index 7674874c0ba..3350169a65e 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterState.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterState.java
@@ -42,6 +42,7 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -702,7 +703,7 @@ public class ClusterState implements ToXContent, Diffable<ClusterState> {
         public static byte[] toBytes(ClusterState state) throws IOException {
             BytesStreamOutput os = new BytesStreamOutput();
             state.writeTo(os);
-            return os.bytes().toBytes();
+            return BytesReference.toBytes(os.bytes());
         }
 
         /**
@@ -711,6 +712,7 @@ public class ClusterState implements ToXContent, Diffable<ClusterState> {
          */
         public static ClusterState fromBytes(byte[] data, DiscoveryNode localNode) throws IOException {
             return readFrom(StreamInput.wrap(data), localNode);
+
         }
 
         /**
diff --git a/core/src/main/java/org/elasticsearch/common/bytes/BytesArray.java b/core/src/main/java/org/elasticsearch/common/bytes/BytesArray.java
index 1e5e76f7994..f6635d94bc6 100644
--- a/core/src/main/java/org/elasticsearch/common/bytes/BytesArray.java
+++ b/core/src/main/java/org/elasticsearch/common/bytes/BytesArray.java
@@ -27,13 +27,12 @@ import java.io.OutputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 
-public final class BytesArray implements BytesReference {
+public final class BytesArray extends BytesReference {
 
     public static final BytesArray EMPTY = new BytesArray(BytesRef.EMPTY_BYTES, 0, 0);
-
-    private byte[] bytes;
-    private int offset;
-    private int length;
+    private final byte[] bytes;
+    private final int offset;
+    private final int length;
 
     public BytesArray(String bytes) {
         BytesRef bytesRef = new BytesRef(bytes);
@@ -89,74 +88,21 @@ public final class BytesArray implements BytesReference {
         return new BytesArray(bytes, offset + from, length);
     }
 
-    @Override
-    public StreamInput streamInput() {
-        return StreamInput.wrap(bytes, offset, length);
-    }
-
-    @Override
-    public void writeTo(OutputStream os) throws IOException {
-        os.write(bytes, offset, length);
-    }
-
-    @Override
-    public byte[] toBytes() {
-        if (offset == 0 && bytes.length == length) {
-            return bytes;
-        }
-        return Arrays.copyOfRange(bytes, offset, offset + length);
-    }
-
-    @Override
-    public BytesArray toBytesArray() {
-        return this;
-    }
-
-    @Override
-    public BytesArray copyBytesArray() {
-        return new BytesArray(Arrays.copyOfRange(bytes, offset, offset + length));
-    }
-
-    @Override
-    public boolean hasArray() {
-        return true;
-    }
-
-    @Override
     public byte[] array() {
         return bytes;
     }
 
-    @Override
-    public int arrayOffset() {
+    public int offset() {
         return offset;
     }
 
-    @Override
-    public String toUtf8() {
-        if (length == 0) {
-            return "";
-        }
-        return new String(bytes, offset, length, StandardCharsets.UTF_8);
-    }
-
     @Override
     public BytesRef toBytesRef() {
         return new BytesRef(bytes, offset, length);
     }
 
     @Override
-    public BytesRef copyBytesRef() {
-        return new BytesRef(Arrays.copyOfRange(bytes, offset, offset + length));
-    }
-
-    @Override
-    public int hashCode() {
-        return Helper.bytesHashCode(this);
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        return Helper.bytesEqual(this, (BytesReference) obj);
+    public long ramBytesUsed() {
+        return bytes.length;
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java b/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java
index 97d4e22f6b2..f5491dde2e3 100644
--- a/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java
+++ b/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java
@@ -18,147 +18,74 @@
  */
 package org.elasticsearch.common.bytes;
 
+import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefIterator;
 import org.elasticsearch.common.io.stream.StreamInput;
 
 import java.io.IOException;
 import java.io.OutputStream;
+import java.util.function.ToIntBiFunction;
 
 /**
  * A reference to bytes.
  */
-public interface BytesReference {
+public abstract class BytesReference implements Accountable, Comparable<BytesReference> {
 
-    class Helper {
-
-        public static boolean bytesEqual(BytesReference a, BytesReference b) {
-            if (a == b) {
-                return true;
-            }
-            if (a.length() != b.length()) {
-                return false;
-            }
-
-            return bytesEquals(a, b);
-        }
-
-        // pkg-private for testing
-        static boolean bytesEquals(BytesReference a, BytesReference b) {
-            assert a.length() == b.length();
-            for (int i = 0, end = a.length(); i < end; ++i) {
-                if (a.get(i) != b.get(i)) {
-                    return false;
-                }
-            }
-
-            return true;
-        }
-
-        public static int bytesHashCode(BytesReference a) {
-            if (a.hasArray()) {
-                return hashCode(a.array(), a.arrayOffset(), a.length());
-            } else {
-                return slowHashCode(a);
-            }
-        }
-
-        // pkg-private for testing
-        static int hashCode(byte[] array, int offset, int length) {
-            int result = 1;
-            for (int i = offset, end = offset + length; i < end; ++i) {
-                result = 31 * result + array[i];
-            }
-            return result;
-        }
-
-        // pkg-private for testing
-        static int slowHashCode(BytesReference a) {
-            int result = 1;
-            for (int i = 0, end = a.length(); i < end; ++i) {
-                result = 31 * result + a.get(i);
-            }
-            return result;
-        }
-    }
+    private Integer hash = null; // we cache the hash of this reference since it can be quite costly to re-calculated it
 
     /**
      * Returns the byte at the specified index. Need to be between 0 and length.
      */
-    byte get(int index);
+    public abstract byte get(int index);
 
     /**
      * The length.
      */
-    int length();
+    public abstract int length();
 
     /**
      * Slice the bytes from the <tt>from</tt> index up to <tt>length</tt>.
      */
-    BytesReference slice(int from, int length);
+    public abstract BytesReference slice(int from, int length);
 
     /**
      * A stream input of the bytes.
      */
-    StreamInput streamInput();
+    public StreamInput streamInput() {
+        BytesRef ref = toBytesRef();
+        return StreamInput.wrap(ref.bytes, ref.offset, ref.length);
+    }
 
     /**
      * Writes the bytes directly to the output stream.
      */
-    void writeTo(OutputStream os) throws IOException;
-
+    public void writeTo(OutputStream os) throws IOException {
+        final BytesRefIterator iterator = iterator();
+        BytesRef ref;
+        while ((ref = iterator.next()) != null) {
+            os.write(ref.bytes, ref.offset, ref.length);
+        }
+    }
 
     /**
-     * Returns the bytes as a single byte array.
+     * Interprets the referenced bytes as UTF8 bytes, returning the resulting string
      */
-    byte[] toBytes();
-
-    /**
-     * Returns the bytes as a byte array, possibly sharing the underlying byte buffer.
-     */
-    BytesArray toBytesArray();
-
-    /**
-     * Returns the bytes copied over as a byte array.
-     */
-    BytesArray copyBytesArray();
-
-    /**
-     * Is there an underlying byte array for this bytes reference.
-     */
-    boolean hasArray();
-
-    /**
-     * The underlying byte array (if exists).
-     */
-    byte[] array();
-
-    /**
-     * The offset into the underlying byte array.
-     */
-    int arrayOffset();
-
-    /**
-     * Converts to a string based on utf8.
-     */
-    String toUtf8();
+    public String utf8ToString() {
+        return toBytesRef().utf8ToString();
+    }
 
     /**
      * Converts to Lucene BytesRef.
      */
-    BytesRef toBytesRef();
-
-    /**
-     * Converts to a copied Lucene BytesRef.
-     */
-    BytesRef copyBytesRef();
+    public abstract BytesRef toBytesRef();
 
     /**
      * Returns a BytesRefIterator for this BytesReference. This method allows
      * access to the internal pages of this reference without copying them. Use with care!
      * @see BytesRefIterator
      */
-    default BytesRefIterator iterator() {
+    public BytesRefIterator iterator() {
         return new BytesRefIterator() {
             BytesRef ref = length() == 0 ? null : toBytesRef();
             @Override
@@ -170,4 +97,115 @@ public interface BytesReference {
         };
     }
 
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+        if (other instanceof BytesReference) {
+            final BytesReference otherRef = (BytesReference) other;
+            if (length() != otherRef.length()) {
+                return false;
+            }
+            return compareIterators(this, otherRef, (a, b) ->
+                a.bytesEquals(b) ? 0 : 1 // this is a call to BytesRef#bytesEquals - this method is the hot one in the comparison
+            ) == 0;
+        }
+        return false;
+    }
+
+    @Override
+    public int hashCode() {
+        if (hash == null) {
+            final BytesRefIterator iterator = iterator();
+            BytesRef ref;
+            int result = 1;
+            try {
+                while ((ref = iterator.next()) != null) {
+                    for (int i = 0; i < ref.length; i++) {
+                        result = 31 * result + ref.bytes[ref.offset + i];
+                    }
+                }
+            } catch (IOException ex) {
+                throw new AssertionError("wont happen", ex);
+            }
+            return hash = result;
+        } else {
+            return hash.intValue();
+        }
+    }
+
+    /**
+     * Returns a compact array from the given BytesReference. The returned array won't be copied unless necessary. If you need
+     * to modify the returned array use <tt>BytesRef.deepCopyOf(reference.toBytesRef()</tt> instead
+     */
+    public static byte[] toBytes(BytesReference reference) {
+        final BytesRef bytesRef = reference.toBytesRef();
+        if (bytesRef.offset == 0 && bytesRef.length == bytesRef.bytes.length) {
+            return bytesRef.bytes;
+        }
+        return BytesRef.deepCopyOf(bytesRef).bytes;
+    }
+
+    @Override
+    public int compareTo(final BytesReference other) {
+        return compareIterators(this, other, (a, b) -> a.compareTo(b));
+    }
+
+    /**
+     * Compares the two references using the given int function.
+     */
+    private static final int compareIterators(final BytesReference a, final BytesReference b, final ToIntBiFunction<BytesRef, BytesRef> f) {
+        try {
+            // we use the iterators since it's a 0-copy comparison where possible!
+            final long lengthToCompare = Math.min(a.length(), b.length());
+            final BytesRefIterator aIter = a.iterator();
+            final BytesRefIterator bIter = b.iterator();
+            BytesRef aRef = aIter.next();
+            BytesRef bRef = bIter.next();
+            if (aRef != null && bRef != null) { // do we have any data?
+                aRef = aRef.clone(); // we clone since we modify the offsets and length in the iteration below
+                bRef = bRef.clone();
+                if (aRef.length == a.length() && bRef.length == b.length()) { // is it only one array slice we are comparing?
+                    return f.applyAsInt(aRef, bRef);
+                } else {
+                    for (int i = 0; i < lengthToCompare;) {
+                        if (aRef.length == 0) {
+                            aRef = aIter.next().clone(); // must be non null otherwise we have a bug
+                        }
+                        if (bRef.length == 0) {
+                            bRef = bIter.next().clone(); // must be non null otherwise we have a bug
+                        }
+                        final int aLength = aRef.length;
+                        final int bLength = bRef.length;
+                        final int length = Math.min(aLength, bLength); // shrink to the same length and use the fast compare in lucene
+                        aRef.length = bRef.length = length;
+                        // now we move to the fast comparison - this is the hot part of the loop
+                        int diff = f.applyAsInt(aRef, bRef);
+                        aRef.length = aLength;
+                        bRef.length = bLength;
+
+                        if (diff != 0) {
+                            return diff;
+                        }
+                        advance(aRef, length);
+                        advance(bRef, length);
+                        i += length;
+                    }
+                }
+            }
+            // One is a prefix of the other, or, they are equal:
+            return a.length() - b.length();
+        } catch (IOException ex) {
+            throw new AssertionError("can not happen", ex);
+        }
+    }
+
+    private static final void advance(final BytesRef ref, final int length) {
+        assert ref.length >= length : " ref.length: " + ref.length + " length: " + length;
+        assert ref.offset+length < ref.bytes.length || (ref.offset+length == ref.bytes.length && ref.length-length == 0)
+            : "offset: " + ref.offset + " ref.bytes.length: " + ref.bytes.length + " length: " + length + " ref.length: " + ref.length;
+        ref.length -= length;
+        ref.offset += length;
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java b/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java
index 70617f903f1..a4bdd2b2e84 100644
--- a/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java
+++ b/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java
@@ -21,7 +21,6 @@ package org.elasticsearch.common.bytes;
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefIterator;
-import org.apache.lucene.util.CharsRefBuilder;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.ByteArray;
@@ -35,7 +34,7 @@ import java.util.Arrays;
  * A page based bytes reference, internally holding the bytes in a paged
  * data structure.
  */
-public class PagedBytesReference implements BytesReference {
+public class PagedBytesReference extends BytesReference {
 
     private static final int PAGE_SIZE = BigArrays.BYTE_PAGE_SIZE;
 
@@ -80,119 +79,6 @@ public class PagedBytesReference implements BytesReference {
         return new PagedBytesReferenceStreamInput(bytearray, offset, length);
     }
 
-    @Override
-    public void writeTo(OutputStream os) throws IOException {
-        // nothing to do
-        if (length == 0) {
-            return;
-        }
-
-        BytesRef ref = new BytesRef();
-        int written = 0;
-
-        // are we a slice?
-        if (offset != 0) {
-            // remaining size of page fragment at offset
-            int fragmentSize = Math.min(length, PAGE_SIZE - (offset % PAGE_SIZE));
-            bytearray.get(offset, fragmentSize, ref);
-            os.write(ref.bytes, ref.offset, fragmentSize);
-            written += fragmentSize;
-        }
-
-        // handle remainder of pages + trailing fragment
-        while (written < length) {
-            int remaining = length - written;
-            int bulkSize = (remaining > PAGE_SIZE) ? PAGE_SIZE : remaining;
-            bytearray.get(offset + written, bulkSize, ref);
-            os.write(ref.bytes, ref.offset, bulkSize);
-            written += bulkSize;
-        }
-    }
-
-    @Override
-    public byte[] toBytes() {
-        if (length == 0) {
-            return BytesRef.EMPTY_BYTES;
-        }
-
-        BytesRef ref = new BytesRef();
-        bytearray.get(offset, length, ref);
-
-        // undo the single-page optimization by ByteArray.get(), otherwise
-        // a materialized stream will contain trailing garbage/zeros
-        byte[] result = ref.bytes;
-        if (result.length != length || ref.offset != 0) {
-            result = Arrays.copyOfRange(result, ref.offset, ref.offset + length);
-        }
-
-        return result;
-    }
-
-    @Override
-    public BytesArray toBytesArray() {
-        BytesRef ref = new BytesRef();
-        bytearray.get(offset, length, ref);
-        return new BytesArray(ref);
-    }
-
-    @Override
-    public BytesArray copyBytesArray() {
-        BytesRef ref = new BytesRef();
-        boolean copied = bytearray.get(offset, length, ref);
-
-        if (copied) {
-            // BigArray has materialized for us, no need to do it again
-            return new BytesArray(ref.bytes, ref.offset, ref.length);
-        } else {
-            // here we need to copy the bytes even when shared
-            byte[] copy = Arrays.copyOfRange(ref.bytes, ref.offset, ref.offset + ref.length);
-            return new BytesArray(copy);
-        }
-    }
-
-    @Override
-    public boolean hasArray() {
-        return (offset + length <= PAGE_SIZE);
-    }
-
-    @Override
-    public byte[] array() {
-        if (hasArray()) {
-            if (length == 0) {
-                return BytesRef.EMPTY_BYTES;
-            }
-
-            BytesRef ref = new BytesRef();
-            bytearray.get(offset, length, ref);
-            return ref.bytes;
-        }
-
-        throw new IllegalStateException("array not available");
-    }
-
-    @Override
-    public int arrayOffset() {
-        if (hasArray()) {
-            BytesRef ref = new BytesRef();
-            bytearray.get(offset, length, ref);
-            return ref.offset;
-        }
-
-        throw new IllegalStateException("array not available");
-    }
-
-    @Override
-    public String toUtf8() {
-        if (length() == 0) {
-            return "";
-        }
-
-        byte[] bytes = toBytes();
-        final CharsRefBuilder ref = new CharsRefBuilder();
-        ref.copyUTF8Bytes(bytes, offset, length);
-        return ref.toString();
-    }
-
     @Override
     public BytesRef toBytesRef() {
         BytesRef bref = new BytesRef();
@@ -201,61 +87,6 @@ public class PagedBytesReference implements BytesReference {
         return bref;
     }
 
-    @Override
-    public BytesRef copyBytesRef() {
-        byte[] bytes = toBytes();
-        return new BytesRef(bytes, offset, length);
-    }
-
-    @Override
-    public int hashCode() {
-        if (hash == 0) {
-            // TODO: delegate to BigArrays via:
-            // hash = bigarrays.hashCode(bytearray);
-            // and for slices:
-            // hash = bigarrays.hashCode(bytearray, offset, length);
-            int tmphash = 1;
-            for (int i = 0; i < length; i++) {
-                tmphash = 31 * tmphash + bytearray.get(offset + i);
-            }
-            hash = tmphash;
-        }
-        return hash;
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (this == obj) {
-            return true;
-        }
-
-        if (obj == null) {
-            return false;
-        }
-
-        if (!(obj instanceof PagedBytesReference)) {
-            return BytesReference.Helper.bytesEqual(this, (BytesReference) obj);
-        }
-
-        PagedBytesReference other = (PagedBytesReference) obj;
-        if (length != other.length) {
-            return false;
-        }
-
-        // TODO: delegate to BigArrays via:
-        // return bigarrays.equals(bytearray, other.bytearray);
-        // and for slices:
-        // return bigarrays.equals(bytearray, start, other.bytearray, otherstart, len);
-        ByteArray otherArray = other.bytearray;
-        int otherOffset = other.offset;
-        for (int i = 0; i < length; i++) {
-            if (bytearray.get(offset + i) != otherArray.get(otherOffset + i)) {
-                return false;
-            }
-        }
-        return true;
-    }
-
     private static class PagedBytesReferenceStreamInput extends StreamInput {
 
         private final ByteArray bytearray;
@@ -390,4 +221,9 @@ public class PagedBytesReference implements BytesReference {
             }
         };
     }
+
+    @Override
+    public long ramBytesUsed() {
+        return bytearray.ramBytesUsed();
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java b/core/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java
index 3864befcc04..991dfccf7f9 100644
--- a/core/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/compress/CompressedXContent.java
@@ -88,7 +88,7 @@ public final class CompressedXContent {
             xcontent.toXContent(builder, params);
             builder.endObject();
         }
-        this.bytes = bStream.bytes().toBytes();
+        this.bytes = BytesReference.toBytes(bStream.bytes());
         this.crc32 = (int) crc32.getValue();
         assertConsistent();
     }
@@ -101,14 +101,14 @@ public final class CompressedXContent {
         Compressor compressor = CompressorFactory.compressor(data);
         if (compressor != null) {
             // already compressed...
-            this.bytes = data.toBytes();
+            this.bytes = BytesReference.toBytes(data);
             this.crc32 = crc32(new BytesArray(uncompressed()));
         } else {
             BytesStreamOutput out = new BytesStreamOutput();
             try (OutputStream compressedOutput = CompressorFactory.COMPRESSOR.streamOutput(out)) {
                 data.writeTo(compressedOutput);
             }
-            this.bytes = out.bytes().toBytes();
+            this.bytes = BytesReference.toBytes(out.bytes());
             this.crc32 = crc32(data);
         }
         assertConsistent();
@@ -140,7 +140,7 @@ public final class CompressedXContent {
     /** Return the uncompressed bytes. */
     public byte[] uncompressed() {
         try {
-            return CompressorFactory.uncompress(new BytesArray(bytes)).toBytes();
+            return BytesReference.toBytes(CompressorFactory.uncompress(new BytesArray(bytes)));
         } catch (IOException e) {
             throw new IllegalStateException("Cannot decompress compressed string", e);
         }
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
index ce0bf59e9e4..ab22df57920 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
@@ -805,13 +805,6 @@ public abstract class StreamInput extends InputStream {
         return builder;
     }
 
-    public static StreamInput wrap(BytesReference reference) {
-        if (reference.hasArray() == false) {
-            reference = reference.toBytesArray();
-        }
-        return wrap(reference.array(), reference.arrayOffset(), reference.length());
-    }
-
     public static StreamInput wrap(byte[] bytes) {
         return wrap(bytes, 0, bytes.length);
     }
diff --git a/core/src/main/java/org/elasticsearch/common/text/Text.java b/core/src/main/java/org/elasticsearch/common/text/Text.java
index 6b3b18c443c..39eb817fe3c 100644
--- a/core/src/main/java/org/elasticsearch/common/text/Text.java
+++ b/core/src/main/java/org/elasticsearch/common/text/Text.java
@@ -82,13 +82,7 @@ public final class Text implements Comparable<Text> {
      * Returns a {@link String} view of the data.
      */
     public String string() {
-        if (text == null) {
-            if (!bytes.hasArray()) {
-                bytes = bytes.toBytesArray();
-            }
-            text = new String(bytes.array(), bytes.arrayOffset(), bytes.length(), StandardCharsets.UTF_8);
-        }
-        return text;
+        return text == null ? bytes.utf8ToString() : text;
     }
 
     @Override
@@ -114,6 +108,6 @@ public final class Text implements Comparable<Text> {
 
     @Override
     public int compareTo(Text text) {
-        return UTF8SortedAsUnicodeComparator.utf8SortedAsUnicodeSortOrder.compare(bytes(), text.bytes());
+        return bytes().compareTo(text.bytes());
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/text/UTF8SortedAsUnicodeComparator.java b/core/src/main/java/org/elasticsearch/common/text/UTF8SortedAsUnicodeComparator.java
deleted file mode 100644
index a330daf3800..00000000000
--- a/core/src/main/java/org/elasticsearch/common/text/UTF8SortedAsUnicodeComparator.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.common.text;
-
-import org.elasticsearch.common.bytes.BytesReference;
-
-import java.util.Comparator;
-
-// LUCENE 4 UPGRADE: Is this the right way of comparing bytesreferences inside Text instances?
-// Copied from Lucene's BytesRef comparator
-public class UTF8SortedAsUnicodeComparator implements Comparator<BytesReference> {
-
-    public final static Comparator<BytesReference> utf8SortedAsUnicodeSortOrder = new UTF8SortedAsUnicodeComparator();
-
-    // Only singleton
-    private UTF8SortedAsUnicodeComparator() {
-    }
-
-    @Override
-    public int compare(BytesReference a, BytesReference b) {
-        if (a.hasArray() && b.hasArray()) {
-            final byte[] aBytes = a.array();
-            int aUpto = a.arrayOffset();
-            final byte[] bBytes = b.array();
-            int bUpto = b.arrayOffset();
-
-            final int aStop = aUpto + Math.min(a.length(), b.length());
-            while (aUpto < aStop) {
-                int aByte = aBytes[aUpto++] & 0xff;
-                int bByte = bBytes[bUpto++] & 0xff;
-
-                int diff = aByte - bByte;
-                if (diff != 0) {
-                    return diff;
-                }
-            }
-
-            // One is a prefix of the other, or, they are equal:
-            return a.length() - b.length();
-        } else {
-            final byte[] aBytes = a.toBytes();
-            int aUpto = 0;
-            final byte[] bBytes = b.toBytes();
-            int bUpto = 0;
-
-            final int aStop = aUpto + Math.min(a.length(), b.length());
-            while (aUpto < aStop) {
-                int aByte = aBytes[aUpto++] & 0xff;
-                int bByte = bBytes[bUpto++] & 0xff;
-
-                int diff = aByte - bByte;
-                if (diff != 0) {
-                    return diff;
-                }
-            }
-
-            // One is a prefix of the other, or, they are equal:
-            return a.length() - b.length();
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
index 2891ca2608c..54b3dd47820 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
@@ -24,7 +24,6 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.math.BigDecimal;
 import java.math.RoundingMode;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.util.Calendar;
 import java.util.Collections;
@@ -35,7 +34,6 @@ import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.io.BytesStream;
@@ -323,10 +321,8 @@ public final class XContentBuilder implements BytesStream, Releasable {
      */
     public XContentBuilder field(String name, BytesReference value) throws IOException {
         field(name);
-        if (!value.hasArray()) {
-            value = value.toBytesArray();
-        }
-        generator.writeBinary(value.array(), value.arrayOffset(), value.length());
+        final BytesRef ref = value.toBytesRef();
+        generator.writeBinary(ref.bytes, ref.offset, ref.length);
         return this;
     }
 
@@ -342,17 +338,13 @@ public final class XContentBuilder implements BytesStream, Releasable {
 
     public XContentBuilder field(String name, Text value) throws IOException {
         field(name);
-        if (value.hasBytes() && value.bytes().hasArray()) {
-            generator.writeUTF8String(value.bytes().array(), value.bytes().arrayOffset(), value.bytes().length());
-            return this;
-        }
         if (value.hasString()) {
             generator.writeString(value.string());
-            return this;
+        } else {
+            // TODO: TextBytesOptimization we can use a buffer here to convert it? maybe add a request to jackson to support InputStream as well?
+            final BytesRef ref = value.bytes().toBytesRef();
+            generator.writeUTF8String(ref.bytes, ref.offset, ref.length);
         }
-        // TODO: TextBytesOptimization we can use a buffer here to convert it? maybe add a request to jackson to support InputStream as well?
-        BytesArray bytesArray = value.bytes().toBytesArray();
-        generator.writeUTF8String(bytesArray.array(), bytesArray.arrayOffset(), bytesArray.length());
         return this;
     }
 
@@ -701,27 +693,20 @@ public final class XContentBuilder implements BytesStream, Releasable {
         if (value == null) {
             return nullValue();
         }
-        if (!value.hasArray()) {
-            value = value.toBytesArray();
-        }
-        generator.writeBinary(value.array(), value.arrayOffset(), value.length());
+        BytesRef ref = value.toBytesRef();
+        generator.writeBinary(ref.bytes, ref.offset, ref.length);
         return this;
     }
 
     public XContentBuilder value(Text value) throws IOException {
         if (value == null) {
             return nullValue();
-        }
-        if (value.hasBytes() && value.bytes().hasArray()) {
-            generator.writeUTF8String(value.bytes().array(), value.bytes().arrayOffset(), value.bytes().length());
-            return this;
-        }
-        if (value.hasString()) {
+        } else if (value.hasString()) {
             generator.writeString(value.string());
-            return this;
+        } else {
+            BytesRef bytesRef = value.bytes().toBytesRef();
+            generator.writeUTF8String(bytesRef.bytes, bytesRef.offset, bytesRef.length);
         }
-        BytesArray bytesArray = value.bytes().toBytesArray();
-        generator.writeUTF8String(bytesArray.array(), bytesArray.arrayOffset(), bytesArray.length());
         return this;
     }
 
@@ -800,8 +785,7 @@ public final class XContentBuilder implements BytesStream, Releasable {
      */
     public String string() throws IOException {
         close();
-        BytesArray bytesArray = bytes().toBytesArray();
-        return new String(bytesArray.array(), bytesArray.arrayOffset(), bytesArray.length(), StandardCharsets.UTF_8);
+        return bytes().utf8ToString();
     }
 
 
@@ -885,13 +869,11 @@ public final class XContentBuilder implements BytesStream, Releasable {
         });
         map.put(Text.class, (g, v) -> {
             Text text = (Text) v;
-            if (text.hasBytes() && text.bytes().hasArray()) {
-                g.writeUTF8String(text.bytes().array(), text.bytes().arrayOffset(), text.bytes().length());
-            } else if (text.hasString()) {
-                g.writeString(text.string());
+            if (text.hasString()) {
+                    g.writeString(text.string());
             } else {
-                BytesArray bytesArray = text.bytes().toBytesArray();
-                g.writeUTF8String(bytesArray.array(), bytesArray.arrayOffset(), bytesArray.length());
+                BytesRef ref = text.bytes().toBytesRef();
+                g.writeUTF8String(ref.bytes, ref.offset, ref.length);
             }
         });
         MAP = Collections.unmodifiableMap(map);
@@ -934,11 +916,8 @@ public final class XContentBuilder implements BytesStream, Releasable {
     }
 
     private void writeBytesReference(BytesReference value) throws IOException {
-        BytesReference bytes = value;
-        if (!bytes.hasArray()) {
-            bytes = bytes.toBytesArray();
-        }
-        generator.writeBinary(bytes.array(), bytes.arrayOffset(), bytes.length());
+        BytesRef ref = value.toBytesRef();
+        generator.writeBinary(ref.bytes, ref.offset, ref.length);
     }
 
     private void writeIterable(Iterable<?> value) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java
index 59587324c33..2832527a583 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentHelper.java
@@ -21,7 +21,6 @@ package org.elasticsearch.common.xcontent;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.compress.Compressor;
@@ -31,7 +30,6 @@ import org.elasticsearch.common.xcontent.ToXContent.Params;
 import java.io.BufferedInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -93,13 +91,9 @@ public class XContentHelper {
     }
 
     public static String convertToJson(BytesReference bytes, boolean reformatJson, boolean prettyPrint) throws IOException {
-        if (bytes.hasArray()) {
-            return convertToJson(bytes.array(), bytes.arrayOffset(), bytes.length(), reformatJson, prettyPrint);
-        }
         XContentType xContentType = XContentFactory.xContentType(bytes);
         if (xContentType == XContentType.JSON && !reformatJson) {
-            BytesArray bytesArray = bytes.toBytesArray();
-            return new String(bytesArray.array(), bytesArray.arrayOffset(), bytesArray.length(), StandardCharsets.UTF_8);
+            return bytes.utf8ToString();
         }
         try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(bytes.streamInput())) {
             parser.nextToken();
@@ -112,26 +106,6 @@ public class XContentHelper {
         }
     }
 
-    public static String convertToJson(byte[] data, int offset, int length, boolean reformatJson) throws IOException {
-        return convertToJson(data, offset, length, reformatJson, false);
-    }
-
-    public static String convertToJson(byte[] data, int offset, int length, boolean reformatJson, boolean prettyPrint) throws IOException {
-        XContentType xContentType = XContentFactory.xContentType(data, offset, length);
-        if (xContentType == XContentType.JSON && !reformatJson) {
-            return new String(data, offset, length, StandardCharsets.UTF_8);
-        }
-        try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(data, offset, length)) {
-            parser.nextToken();
-            XContentBuilder builder = XContentFactory.jsonBuilder();
-            if (prettyPrint) {
-                builder.prettyPrint();
-            }
-            builder.copyCurrentStructure(parser);
-            return builder.string();
-        }
-    }
-
     /**
      * Writes serialized toXContent to pretty-printed JSON string.
      *
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java
index 1670810511a..901a9a7d717 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java
@@ -96,9 +96,6 @@ public class CborXContent implements XContent {
 
     @Override
     public XContentParser createParser(BytesReference bytes) throws IOException {
-        if (bytes.hasArray()) {
-            return createParser(bytes.array(), bytes.arrayOffset(), bytes.length());
-        }
         return createParser(bytes.streamInput());
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java
index 13cacc41538..c989aa4a28b 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java
@@ -118,9 +118,6 @@ public class JsonXContent implements XContent {
 
     @Override
     public XContentParser createParser(BytesReference bytes) throws IOException {
-        if (bytes.hasArray()) {
-            return createParser(bytes.array(), bytes.arrayOffset(), bytes.length());
-        }
         return createParser(bytes.streamInput());
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java
index f86b44b4e6f..a7c541b84fc 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java
@@ -28,6 +28,7 @@ import com.fasterxml.jackson.core.util.DefaultIndenter;
 import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -70,10 +71,6 @@ public class JsonXContentGenerator implements XContentGenerator {
     private static final DefaultPrettyPrinter.Indenter INDENTER = new DefaultIndenter("  ", LF.getValue());
     private boolean prettyPrint = false;
 
-    public JsonXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String... filters) {
-        this(jsonGenerator, os, filters, true);
-    }
-
     public JsonXContentGenerator(JsonGenerator jsonGenerator, OutputStream os, String[] filters, boolean inclusive) {
         if (jsonGenerator instanceof GeneratorBase) {
             this.base = (GeneratorBase) jsonGenerator;
@@ -355,18 +352,9 @@ public class JsonXContentGenerator implements XContentGenerator {
     }
 
     protected void copyRawValue(BytesReference content, XContent xContent) throws IOException {
-        XContentParser parser = null;
-        try {
-            if (content.hasArray()) {
-                parser = xContent.createParser(content.array(), content.arrayOffset(), content.length());
-            } else {
-                parser = xContent.createParser(content.streamInput());
-            }
+        try (StreamInput input = content.streamInput();
+             XContentParser parser = xContent.createParser(input)) {
             copyCurrentStructure(parser);
-        } finally {
-            if (parser != null) {
-                parser.close();
-            }
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java
index 3a104056b09..f1bc879f26f 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java
@@ -97,9 +97,6 @@ public class SmileXContent implements XContent {
 
     @Override
     public XContentParser createParser(BytesReference bytes) throws IOException {
-        if (bytes.hasArray()) {
-            return createParser(bytes.array(), bytes.arrayOffset(), bytes.length());
-        }
         return createParser(bytes.streamInput());
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java
index 012781cca3f..ce6e0e0e37b 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java
@@ -92,9 +92,6 @@ public class YamlXContent implements XContent {
 
     @Override
     public XContentParser createParser(BytesReference bytes) throws IOException {
-        if (bytes.hasArray()) {
-            return createParser(bytes.array(), bytes.arrayOffset(), bytes.length());
-        }
         return createParser(bytes.streamInput());
     }
 
diff --git a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
index 696eb8812c9..9b8e148550e 100644
--- a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
@@ -31,6 +31,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -318,7 +319,7 @@ public class LocalDiscovery extends AbstractLifecycleComponent<Discovery> implem
                             Diff diff = clusterState.diff(clusterChangedEvent.previousState());
                             BytesStreamOutput os = new BytesStreamOutput();
                             diff.writeTo(os);
-                            clusterStateDiffBytes = os.bytes().toBytes();
+                            clusterStateDiffBytes = BytesReference.toBytes(os.bytes());
                         }
                         try {
                             newNodeSpecificClusterState = discovery.lastProcessedClusterState.readDiffFrom(StreamInput.wrap(clusterStateDiffBytes)).apply(discovery.lastProcessedClusterState);
diff --git a/core/src/main/java/org/elasticsearch/index/get/GetResult.java b/core/src/main/java/org/elasticsearch/index/get/GetResult.java
index 428db08dbcb..0fa843adc47 100644
--- a/core/src/main/java/org/elasticsearch/index/get/GetResult.java
+++ b/core/src/main/java/org/elasticsearch/index/get/GetResult.java
@@ -116,7 +116,7 @@ public class GetResult implements Streamable, Iterable<GetField>, ToXContent {
         if (sourceAsBytes != null) {
             return sourceAsBytes;
         }
-        this.sourceAsBytes = sourceRef().toBytes();
+        this.sourceAsBytes = BytesReference.toBytes(sourceRef());
         return this.sourceAsBytes;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java b/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java
index 173b14eefef..074ce8829e3 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper;
 
 import java.util.Objects;
 
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.unit.TimeValue;
 
@@ -62,7 +63,7 @@ public class SourceToParse {
         this.id = Objects.requireNonNull(id);
         // we always convert back to byte array, since we store it and Field only supports bytes..
         // so, we might as well do it here, and improve the performance of working with direct byte arrays
-        this.source = source.toBytesArray();
+        this.source = new BytesArray(source.toBytesRef());
     }
 
     public Origin origin() {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
index ce30490968c..0f5d0516842 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java
@@ -23,6 +23,7 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -256,10 +257,8 @@ public class SourceFieldMapper extends MetadataFieldMapper {
 
             source = bStream.bytes();
         }
-        if (!source.hasArray()) {
-            source = source.toBytesArray();
-        }
-        fields.add(new StoredField(fieldType().name(), source.array(), source.arrayOffset(), source.length()));
+        BytesRef ref = source.toBytesRef();
+        fields.add(new StoredField(fieldType().name(), ref.bytes, ref.offset, ref.length));
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java
index 7e469d6bbe4..b242001846a 100644
--- a/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.query;
 
 import org.apache.lucene.search.Query;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
@@ -85,7 +86,7 @@ public class WrapperQueryBuilder extends AbstractQueryBuilder<WrapperQueryBuilde
         if (source == null || source.length() == 0) {
             throw new IllegalArgumentException("query source text cannot be null or empty");
         }
-        this.source = source.array();
+        this.source = BytesRef.deepCopyOf(source.toBytesRef()).bytes;
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/index/shard/CommitPoints.java b/core/src/main/java/org/elasticsearch/index/shard/CommitPoints.java
index 19c19b4ab21..2a5b3506028 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/CommitPoints.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/CommitPoints.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.index.shard;
 
 import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
@@ -116,7 +117,7 @@ public class CommitPoints implements Iterable<CommitPoint> {
         builder.endObject();
 
         builder.endObject();
-        return builder.bytes().toBytes();
+        return BytesReference.toBytes(builder.bytes());
     }
 
     public static CommitPoint fromXContent(byte[] data) throws Exception {
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 959b6f8f726..02eeabb803a 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -1312,7 +1312,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
             }
             out.flush();
             if (corrupt != null) {
-                logger.warn("check index [failure]\n{}", new String(os.bytes().toBytes(), StandardCharsets.UTF_8));
+                logger.warn("check index [failure]\n{}", os.bytes().utf8ToString());
                 throw corrupt;
             }
         } else {
@@ -1327,7 +1327,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
                         // ignore if closed....
                         return;
                     }
-                    logger.warn("check index [failure]\n{}", new String(os.bytes().toBytes(), StandardCharsets.UTF_8));
+                    logger.warn("check index [failure]\n{}", os.bytes().utf8ToString());
                     if ("fix".equals(checkIndexOnStartup)) {
                         if (logger.isDebugEnabled()) {
                             logger.debug("fixing index, writing new segments file ...");
@@ -1345,7 +1345,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         }
 
         if (logger.isDebugEnabled()) {
-            logger.debug("check index [success]\n{}", new String(os.bytes().toBytes(), StandardCharsets.UTF_8));
+            logger.debug("check index [success]\n{}", os.bytes().utf8ToString());
         }
 
         recoveryState.getVerifyIndex().checkIndexTime(Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - timeNS)));
diff --git a/core/src/main/java/org/elasticsearch/index/store/Store.java b/core/src/main/java/org/elasticsearch/index/store/Store.java
index 166f978a4db..51e80f0b87d 100644
--- a/core/src/main/java/org/elasticsearch/index/store/Store.java
+++ b/core/src/main/java/org/elasticsearch/index/store/Store.java
@@ -41,6 +41,7 @@ import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.SimpleFSDirectory;
 import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.Version;
@@ -1339,7 +1340,8 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
                 out.writeThrowable(exception);
                 BytesReference bytes = out.bytes();
                 output.writeVInt(bytes.length());
-                output.writeBytes(bytes.array(), bytes.arrayOffset(), bytes.length());
+                BytesRef ref = bytes.toBytesRef();
+                output.writeBytes(ref.bytes, ref.offset, ref.length);
                 CodecUtil.writeFooter(output);
             } catch (IOException ex) {
                 logger.warn("Can't mark store as corrupted", ex);
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index b8b9b827e4b..12d1966e4cd 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -1137,7 +1137,7 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService>
         BytesReference statsRef = cacheShardLevelResult(shard, searcher.getDirectoryReader(), cacheKey, out -> {
             out.writeOptionalWriteable(fieldType.stats(searcher.reader()));
         });
-        try (StreamInput in = StreamInput.wrap(statsRef)) {
+        try (StreamInput in = statsRef.streamInput()) {
             return in.readOptionalWriteable(FieldStats::readFrom);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
index 29b7df0a859..bafd5d0e58d 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
@@ -24,6 +24,8 @@ import org.apache.lucene.index.IndexFormatTooNewException;
 import org.apache.lucene.index.IndexFormatTooOldException;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefIterator;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -378,10 +380,11 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget
         } else {
             indexOutput = getOpenIndexOutput(name);
         }
-        if (content.hasArray() == false) {
-            content = content.toBytesArray();
+        BytesRefIterator iterator = content.iterator();
+        BytesRef scratch;
+        while((scratch = iterator.next()) != null) { // we iterate over all pages - this is a 0-copy for all core impls
+            indexOutput.writeBytes(scratch.bytes, scratch.offset, scratch.length);
         }
-        indexOutput.writeBytes(content.array(), content.arrayOffset(), content.length());
         indexState.addRecoveredBytesToFile(name, content.length());
         if (indexOutput.getFilePointer() >= fileMetaData.length() || lastChunk) {
             try {
diff --git a/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java b/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java
index 94b4c6fe6bf..677be3b4b0c 100644
--- a/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java
+++ b/core/src/main/java/org/elasticsearch/ingest/InternalTemplateService.java
@@ -53,7 +53,7 @@ public class InternalTemplateService implements TemplateService {
                     ExecutableScript executableScript = scriptService.executable(compiledScript, model);
                     Object result = executableScript.run();
                     if (result instanceof BytesReference) {
-                        return ((BytesReference) result).toUtf8();
+                        return ((BytesReference) result).utf8ToString();
                     }
                     return String.valueOf(result);
                 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/put/RestPutRepositoryAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/put/RestPutRepositoryAction.java
index 745dd7e0b9f..59563816dd8 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/put/RestPutRepositoryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/repositories/put/RestPutRepositoryAction.java
@@ -50,7 +50,7 @@ public class RestPutRepositoryAction extends BaseRestHandler {
     @Override
     public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         PutRepositoryRequest putRepositoryRequest = putRepositoryRequest(request.param("repository"));
-        putRepositoryRequest.source(request.content().toUtf8());
+        putRepositoryRequest.source(request.content().utf8ToString());
         putRepositoryRequest.verify(request.paramAsBoolean("verify", true));
         putRepositoryRequest.masterNodeTimeout(request.paramAsTime("master_timeout", putRepositoryRequest.masterNodeTimeout()));
         putRepositoryRequest.timeout(request.paramAsTime("timeout", putRepositoryRequest.timeout()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/create/RestCreateSnapshotAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/create/RestCreateSnapshotAction.java
index 3be27d286e2..14f02c94b61 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/create/RestCreateSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/create/RestCreateSnapshotAction.java
@@ -49,7 +49,7 @@ public class RestCreateSnapshotAction extends BaseRestHandler {
     @Override
     public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         CreateSnapshotRequest createSnapshotRequest = createSnapshotRequest(request.param("repository"), request.param("snapshot"));
-        createSnapshotRequest.source(request.content().toUtf8());
+        createSnapshotRequest.source(request.content().utf8ToString());
         createSnapshotRequest.masterNodeTimeout(request.paramAsTime("master_timeout", createSnapshotRequest.masterNodeTimeout()));
         createSnapshotRequest.waitForCompletion(request.paramAsBoolean("wait_for_completion", false));
         client.admin().cluster().createSnapshot(createSnapshotRequest, new RestToXContentListener<CreateSnapshotResponse>(channel));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/restore/RestRestoreSnapshotAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/restore/RestRestoreSnapshotAction.java
index 6a1eb0eaa50..dce7fcf6dbc 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/restore/RestRestoreSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/snapshots/restore/RestRestoreSnapshotAction.java
@@ -49,7 +49,7 @@ public class RestRestoreSnapshotAction extends BaseRestHandler {
         RestoreSnapshotRequest restoreSnapshotRequest = restoreSnapshotRequest(request.param("repository"), request.param("snapshot"));
         restoreSnapshotRequest.masterNodeTimeout(request.paramAsTime("master_timeout", restoreSnapshotRequest.masterNodeTimeout()));
         restoreSnapshotRequest.waitForCompletion(request.paramAsBoolean("wait_for_completion", false));
-        restoreSnapshotRequest.source(request.content().toUtf8());
+        restoreSnapshotRequest.source(request.content().utf8ToString());
         client.admin().cluster().restoreSnapshot(restoreSnapshotRequest, new RestToXContentListener<RestoreSnapshotResponse>(channel));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/analyze/RestAnalyzeAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/analyze/RestAnalyzeAction.java
index 588771ae1c8..4d2d66c4143 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/analyze/RestAnalyzeAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/analyze/RestAnalyzeAction.java
@@ -87,7 +87,7 @@ public class RestAnalyzeAction extends BaseRestHandler {
             XContentType type = RestActions.guessBodyContentType(request);
             if (type == null) {
                 if (texts == null || texts.length == 0) {
-                    texts = new String[]{ RestActions.getRestContent(request).toUtf8() };
+                    texts = new String[]{ RestActions.getRestContent(request).utf8ToString() };
                     analyzeRequest.text(texts);
                 }
             } else {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/mapping/put/RestPutMappingAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/mapping/put/RestPutMappingAction.java
index cb2493f4e33..7a0d07cd2ed 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/mapping/put/RestPutMappingAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/mapping/put/RestPutMappingAction.java
@@ -71,7 +71,7 @@ public class RestPutMappingAction extends BaseRestHandler {
     public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) {
         PutMappingRequest putMappingRequest = putMappingRequest(Strings.splitStringByCommaToArray(request.param("index")));
         putMappingRequest.type(request.param("type"));
-        putMappingRequest.source(request.content().toUtf8());
+        putMappingRequest.source(request.content().utf8ToString());
         putMappingRequest.updateAllTypes(request.paramAsBoolean("update_all_types", false));
         putMappingRequest.timeout(request.paramAsTime("timeout", putMappingRequest.timeout()));
         putMappingRequest.masterNodeTimeout(request.paramAsTime("master_timeout", putMappingRequest.masterNodeTimeout()));
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/settings/RestUpdateSettingsAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/settings/RestUpdateSettingsAction.java
index 70608930917..877a5a45fbe 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/settings/RestUpdateSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/settings/RestUpdateSettingsAction.java
@@ -68,7 +68,7 @@ public class RestUpdateSettingsAction extends BaseRestHandler {
         updateSettingsRequest.indicesOptions(IndicesOptions.fromRequest(request, updateSettingsRequest.indicesOptions()));
 
         Settings.Builder updateSettings = Settings.builder();
-        String bodySettingsStr = request.content().toUtf8();
+        String bodySettingsStr = request.content().utf8ToString();
         if (Strings.hasText(bodySettingsStr)) {
             Settings buildSettings = Settings.builder().loadFromSource(bodySettingsStr).build();
             for (Map.Entry<String, String> entry : buildSettings.getAsMap().entrySet()) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java
index a189a58a3ae..b057844521c 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestClearScrollAction.java
@@ -61,7 +61,7 @@ public class RestClearScrollAction extends BaseRestHandler {
         if (RestActions.hasBodyContent(request)) {
             XContentType type = RestActions.guessBodyContentType(request);
            if (type == null) {
-               scrollIds = RestActions.getRestContent(request).toUtf8();
+               scrollIds = RestActions.getRestContent(request).utf8ToString();
                clearRequest.setScrollIds(Arrays.asList(splitScrollIds(scrollIds)));
            } else {
                // NOTE: if rest request with xcontent body has request parameters, these parameters does not override xcontent value
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java
index 6b9b09545ad..45c938381b4 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java
@@ -73,7 +73,7 @@ public class RestSearchScrollAction extends BaseRestHandler {
             XContentType type = XContentFactory.xContentType(RestActions.getRestContent(request));
             if (type == null) {
                 if (scrollId == null) {
-                    scrollId = RestActions.getRestContent(request).toUtf8();
+                    scrollId = RestActions.getRestContent(request).utf8ToString();
                     searchScrollRequest.scrollId(scrollId);
                 }
             } else {
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java
index 94b4680e4d5..7d632c06391 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java
@@ -376,7 +376,7 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
                             "] lang [" + scriptLang + "]", e);
                 }
             } else {
-                throw new IllegalArgumentException("Unable to find script in : " + scriptBytes.toUtf8());
+                throw new IllegalArgumentException("Unable to find script in : " + scriptBytes.utf8ToString());
             }
         } catch (IOException e) {
             throw new IllegalArgumentException("failed to parse template script", e);
diff --git a/core/src/main/java/org/elasticsearch/script/Template.java b/core/src/main/java/org/elasticsearch/script/Template.java
index ae72a52b027..a2b0aa4b9c0 100644
--- a/core/src/main/java/org/elasticsearch/script/Template.java
+++ b/core/src/main/java/org/elasticsearch/script/Template.java
@@ -35,12 +35,12 @@ import java.util.Collections;
 import java.util.Map;
 
 public class Template extends Script {
-    
+
     /** Default templating language */
     public static final String DEFAULT_LANG = "mustache";
 
     private XContentType contentType;
-    
+
     /**
      * Constructor for simple inline template. The template will have no lang,
      * content type or params set.
@@ -173,7 +173,7 @@ public class Template extends Script {
             if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
                 contentType = parser.contentType();
                 XContentBuilder builder = XContentFactory.contentBuilder(contentType);
-                return builder.copyCurrentStructure(parser).bytes().toUtf8();
+                return builder.copyCurrentStructure(parser).bytes().utf8ToString();
             } else {
                 return parser.text();
             }
diff --git a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
index 389b1ec89b6..e52bfba2341 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
@@ -235,7 +235,7 @@ public class InternalSearchHit implements SearchHit {
         if (sourceAsBytes != null) {
             return sourceAsBytes;
         }
-        this.sourceAsBytes = sourceRef().toBytes();
+        this.sourceAsBytes = BytesReference.toBytes(sourceRef());
         return this.sourceAsBytes;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java
index 5b12c941185..f70835e0a99 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchLocalRequest.java
@@ -23,6 +23,7 @@ import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -212,7 +213,7 @@ public class ShardSearchLocalRequest implements ShardSearchRequest {
         this.innerWriteTo(out, true);
         // copy it over, most requests are small, we might as well copy to make sure we are not sliced...
         // we could potentially keep it without copying, but then pay the price of extra unused bytes up to a page
-        return out.bytes().copyBytesArray();
+        return new BytesArray(out.bytes().toBytesRef(), true);// do a deep copy
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/CompletionSuggestion.java b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/CompletionSuggestion.java
index 565be4fcd9f..cf0939b85f9 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/CompletionSuggestion.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/CompletionSuggestion.java
@@ -92,15 +92,15 @@ public class CompletionSuggestion extends Suggest.Suggestion<CompletionSuggestio
             }
 
             public String getPayloadAsString() {
-                return payload.toUtf8();
+                return payload.utf8ToString();
             }
 
             public long getPayloadAsLong() {
-                return Long.parseLong(payload.toUtf8());
+                return Long.parseLong(payload.utf8ToString());
             }
 
             public double getPayloadAsDouble() {
-                return Double.parseDouble(payload.toUtf8());
+                return Double.parseDouble(payload.utf8ToString());
             }
 
             public Map<String, Object> getPayloadAsMap() {
@@ -119,7 +119,7 @@ public class CompletionSuggestion extends Suggest.Suggestion<CompletionSuggestio
                     XContentType contentType = XContentFactory.xContentType(payload);
                     if (contentType == null) {
                         // must be a string or number
-                        builder.field("payload", payload.toUtf8());
+                        builder.field("payload", payload.utf8ToString());
                     } else {
                         builder.rawField("payload", payload);
                     }
diff --git a/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java b/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
index 4caf1a543c0..a2ffca2e804 100644
--- a/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
@@ -24,6 +24,7 @@ import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.breaker.CircuitBreaker;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.inject.Inject;
@@ -226,7 +227,7 @@ public class LocalTransport extends AbstractLifecycleComponent<Transport> implem
                 throw new NodeNotConnectedException(node, "Node not connected");
             }
 
-            final byte[] data = stream.bytes().toBytes();
+            final byte[] data = BytesReference.toBytes(stream.bytes());
             transportServiceAdapter.sent(data.length);
             transportServiceAdapter.onRequestSent(node, requestId, action, request, options);
             targetTransport.workers().execute(() -> {
diff --git a/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java b/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java
index 8ddee299d98..fc3b4a3b32f 100644
--- a/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java
+++ b/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.transport.local;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.transport.RemoteTransportException;
@@ -84,7 +85,7 @@ public class LocalTransportChannel implements TransportChannel {
             status = TransportStatus.setResponse(status);
             stream.writeByte(status); // 0 for request, 1 for response.
             response.writeTo(stream);
-            sendResponseData(stream.bytes().toBytes());
+            sendResponseData(BytesReference.toBytes(stream.bytes()));
             sourceTransportServiceAdapter.onResponseSent(requestId, action, response, options);
         }
     }
@@ -96,7 +97,7 @@ public class LocalTransportChannel implements TransportChannel {
         RemoteTransportException tx = new RemoteTransportException(targetTransport.nodeName(),
                 targetTransport.boundAddress().boundAddresses()[0], action, error);
         stream.writeThrowable(tx);
-        sendResponseData(stream.bytes().toBytes());
+        sendResponseData(BytesReference.toBytes(stream.bytes()));
         sourceTransportServiceAdapter.onResponseSent(requestId, action, error);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/ChannelBufferBytesReference.java b/core/src/main/java/org/elasticsearch/transport/netty/ChannelBufferBytesReference.java
index 60dbdd21bd1..05a5502423a 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/ChannelBufferBytesReference.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/ChannelBufferBytesReference.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.transport.netty;
 
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.jboss.netty.buffer.ChannelBuffer;
@@ -28,111 +27,65 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.charset.StandardCharsets;
 
-/**
- */
-final class ChannelBufferBytesReference implements BytesReference {
+final class ChannelBufferBytesReference extends BytesReference {
 
     private final ChannelBuffer buffer;
-    private final int size;
+    private final int length;
+    private final int offset;
 
-    ChannelBufferBytesReference(ChannelBuffer buffer, int size) {
+    ChannelBufferBytesReference(ChannelBuffer buffer, int length) {
         this.buffer = buffer;
-        this.size = size;
-        assert size <= buffer.readableBytes() : "size[" + size +"] > " + buffer.readableBytes();
+        this.length = length;
+        this.offset = buffer.readerIndex();
+        assert length <= buffer.readableBytes() : "length[" + length +"] > " + buffer.readableBytes();
     }
 
     @Override
     public byte get(int index) {
-        return buffer.getByte(buffer.readerIndex() + index);
+        return buffer.getByte(offset + index);
     }
 
     @Override
     public int length() {
-        return size;
+        return length;
     }
 
     @Override
     public BytesReference slice(int from, int length) {
-        return new ChannelBufferBytesReference(buffer.slice(buffer.readerIndex() + from, length), length);
+        return new ChannelBufferBytesReference(buffer.slice(offset + from, length), length);
     }
 
     @Override
     public StreamInput streamInput() {
-        return new ChannelBufferStreamInput(buffer.duplicate(), size);
+        return new ChannelBufferStreamInput(buffer.duplicate(), length);
     }
 
     @Override
     public void writeTo(OutputStream os) throws IOException {
-        buffer.getBytes(buffer.readerIndex(), os, size);
+        buffer.getBytes(offset, os, length);
     }
 
-    public byte[] toBytes() {
-        return copyBytesArray().toBytes();
-    }
-
-    @Override
-    public BytesArray toBytesArray() {
-        if (buffer.hasArray()) {
-            return new BytesArray(buffer.array(), buffer.arrayOffset() + buffer.readerIndex(), size);
-        }
-        return copyBytesArray();
-    }
-
-    @Override
-    public BytesArray copyBytesArray() {
-        byte[] copy = new byte[buffer.readableBytes()];
-        buffer.getBytes(buffer.readerIndex(), copy);
-        return new BytesArray(copy);
-    }
-
-    public ChannelBuffer toChannelBuffer() {
+    ChannelBuffer toChannelBuffer() {
         return buffer.duplicate();
     }
 
     @Override
-    public boolean hasArray() {
-        return buffer.hasArray();
-    }
-
-    @Override
-    public byte[] array() {
-        return buffer.array();
-    }
-
-    @Override
-    public int arrayOffset() {
-        return buffer.arrayOffset() + buffer.readerIndex();
-    }
-
-    @Override
-    public String toUtf8() {
-        return buffer.toString(StandardCharsets.UTF_8);
+    public String utf8ToString() {
+        return buffer.toString(offset, length, StandardCharsets.UTF_8);
     }
 
     @Override
     public BytesRef toBytesRef() {
         if (buffer.hasArray()) {
-            return new BytesRef(buffer.array(), buffer.arrayOffset() + buffer.readerIndex(), size);
+            return new BytesRef(buffer.array(), buffer.arrayOffset() + offset, length);
         }
-        byte[] copy = new byte[buffer.readableBytes()];
-        buffer.getBytes(buffer.readerIndex(), copy);
+        final byte[] copy = new byte[length];
+        buffer.getBytes(offset, copy);
         return new BytesRef(copy);
     }
 
     @Override
-    public BytesRef copyBytesRef() {
-        byte[] copy = new byte[size];
-        buffer.getBytes(buffer.readerIndex(), copy);
-        return new BytesRef(copy);
-    }
-
-    @Override
-    public int hashCode() {
-        return Helper.bytesHashCode(this);
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        return Helper.bytesEqual(this, (BytesReference) obj);
+    public long ramBytesUsed() {
+        return buffer.capacity();
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/ESExceptionTests.java b/core/src/test/java/org/elasticsearch/ESExceptionTests.java
index 9ac8782110e..d27790beed1 100644
--- a/core/src/test/java/org/elasticsearch/ESExceptionTests.java
+++ b/core/src/test/java/org/elasticsearch/ESExceptionTests.java
@@ -295,7 +295,7 @@ public class ESExceptionTests extends ESTestCase {
         ParsingException ex = new ParsingException(1, 2, "foobar", null);
         out.writeThrowable(ex);
 
-        StreamInput in = StreamInput.wrap(out.bytes());
+        StreamInput in = out.bytes().streamInput();
         ParsingException e = in.readThrowable();
         assertEquals(ex.getIndex(), e.getIndex());
         assertEquals(ex.getMessage(), e.getMessage());
@@ -309,7 +309,7 @@ public class ESExceptionTests extends ESTestCase {
         Throwable ex = new Throwable("eggplant", ParsingException);
         out.writeThrowable(ex);
 
-        StreamInput in = StreamInput.wrap(out.bytes());
+        StreamInput in = out.bytes().streamInput();
         Throwable throwable = in.readThrowable();
         assertEquals("throwable: eggplant", throwable.getMessage());
         assertTrue(throwable instanceof ElasticsearchException);
@@ -346,7 +346,7 @@ public class ESExceptionTests extends ESTestCase {
             BytesStreamOutput out = new BytesStreamOutput();
             ElasticsearchException ex = new ElasticsearchException("topLevel", t);
             out.writeThrowable(ex);
-            StreamInput in = StreamInput.wrap(out.bytes());
+            StreamInput in = out.bytes().streamInput();
             ElasticsearchException e = in.readThrowable();
             assertEquals(e.getMessage(), ex.getMessage());
             assertTrue("Expected: " + e.getCause().getMessage() + " to contain: " +
diff --git a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
index c0b5f2e9cf8..6a65b1d1695 100644
--- a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
+++ b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
@@ -223,7 +223,7 @@ public class ExceptionSerializationTests extends ESTestCase {
         ElasticsearchAssertions.assertVersionSerializable(VersionUtils.randomVersion(random()), exception);
         BytesStreamOutput out = new BytesStreamOutput();
         out.writeThrowable(exception);
-        StreamInput in = StreamInput.wrap(out.bytes());
+        StreamInput in = out.bytes().streamInput();
         return in.readThrowable();
     }
 
diff --git a/core/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java b/core/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java
index a19905c99d8..ceba9278a16 100644
--- a/core/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java
+++ b/core/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java
@@ -44,7 +44,7 @@ public class OriginalIndicesTests extends ESTestCase {
             out.setVersion(randomVersion(random()));
             OriginalIndices.writeOriginalIndices(originalIndices, out);
 
-            StreamInput in = StreamInput.wrap(out.bytes());
+            StreamInput in = out.bytes().streamInput();
             in.setVersion(out.getVersion());
             OriginalIndices originalIndices2 = OriginalIndices.readOriginalIndices(in);
 
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
index d0e8ef14d01..bec8bd4d2c3 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
@@ -205,7 +205,7 @@ public final class ClusterAllocationExplanationTests extends ESTestCase {
                 "assignedNode", allocationDelay, remainingDelay, null, false, nodeExplanations);
         BytesStreamOutput out = new BytesStreamOutput();
         cae.writeTo(out);
-        StreamInput in = StreamInput.wrap(out.bytes());
+        StreamInput in = out.bytes().streamInput();
         ClusterAllocationExplanation cae2 = new ClusterAllocationExplanation(in);
         assertEquals(shard, cae2.getShard());
         assertTrue(cae2.isPrimary());
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java
index 704c1348b7e..d0d452df478 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java
@@ -84,7 +84,7 @@ public class ClusterHealthResponsesTests extends ESTestCase {
         if (randomBoolean()) {
             BytesStreamOutput out = new BytesStreamOutput();
             clusterHealth.writeTo(out);
-            StreamInput in = StreamInput.wrap(out.bytes());
+            StreamInput in = out.bytes().streamInput();
             clusterHealth = ClusterHealthResponse.readResponseFrom(in);
         }
         return clusterHealth;
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java
index b736751b781..4f553dfb88a 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java
@@ -166,7 +166,7 @@ public class ClusterRerouteRequestTests extends ESTestCase {
     private ClusterRerouteRequest roundTripThroughBytes(ClusterRerouteRequest original) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             original.writeTo(output);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
                 ClusterRerouteRequest copy = new ClusterRerouteRequest();
                 copy.readFrom(in);
                 return copy;
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
index 00fcbf60a5a..dac878eefec 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
@@ -66,7 +66,7 @@ public class ClusterRerouteTests extends ESAllocationTestCase {
         BytesReference bytes = out.bytes();
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
         new NetworkModule(null, Settings.EMPTY, true, namedWriteableRegistry);
-        StreamInput wrap = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes.toBytes()),
+        StreamInput wrap = new NamedWriteableAwareStreamInput(bytes.streamInput(),
             namedWriteableRegistry);
         ClusterRerouteRequest deserializedReq = new ClusterRerouteRequest();
         deserializedReq.readFrom(wrap);
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java
index fc04de81254..b515829b72a 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java
@@ -45,7 +45,7 @@ public class ClusterStateRequestTests extends ESTestCase {
             output.setVersion(testVersion);
             clusterStateRequest.writeTo(output);
 
-            StreamInput streamInput = StreamInput.wrap(output.bytes());
+            StreamInput streamInput = output.bytes().streamInput();
             streamInput.setVersion(testVersion);
             ClusterStateRequest deserializedCSRequest = new ClusterStateRequest();
             deserializedCSRequest.readFrom(streamInput);
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java
index e4c2849b907..2e9239a2c3b 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java
@@ -37,7 +37,7 @@ public class GetStoredScriptRequestTests extends ESTestCase {
         out.setVersion(randomVersion(random()));
         request.writeTo(out);
 
-        StreamInput in = StreamInput.wrap(out.bytes());
+        StreamInput in = out.bytes().streamInput();
         in.setVersion(out.getVersion());
         GetStoredScriptRequest request2 = new GetStoredScriptRequest();
         request2.readFrom(in);
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/flush/SyncedFlushUnitTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/flush/SyncedFlushUnitTests.java
index 04f6037f64b..7040c92ec1d 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/flush/SyncedFlushUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/flush/SyncedFlushUnitTests.java
@@ -84,7 +84,7 @@ public class SyncedFlushUnitTests extends ESTestCase {
         assertThat(testPlan.result.restStatus(), equalTo(testPlan.totalCounts.failed > 0 ? RestStatus.CONFLICT : RestStatus.OK));
         BytesStreamOutput out = new BytesStreamOutput();
         testPlan.result.writeTo(out);
-        StreamInput in = StreamInput.wrap(out.bytes());
+        StreamInput in = out.bytes().streamInput();
         SyncedFlushResponse readResponse = new SyncedFlushResponse();
         readResponse.readFrom(in);
         assertThat(readResponse.totalShards(), equalTo(testPlan.totalCounts.total));
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
index 337f881d41b..142fb282c20 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java
@@ -55,9 +55,9 @@ public class BulkRequestTests extends ESTestCase {
         BulkRequest bulkRequest = new BulkRequest();
         bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null);
         assertThat(bulkRequest.numberOfActions(), equalTo(3));
-        assertThat(((IndexRequest) bulkRequest.requests().get(0)).source().toBytes(), equalTo(new BytesArray("{ \"field1\" : \"value1\" }").toBytes()));
+        assertThat(((IndexRequest) bulkRequest.requests().get(0)).source(), equalTo(new BytesArray("{ \"field1\" : \"value1\" }")));
         assertThat(bulkRequest.requests().get(1), instanceOf(DeleteRequest.class));
-        assertThat(((IndexRequest) bulkRequest.requests().get(2)).source().toBytes(), equalTo(new BytesArray("{ \"field1\" : \"value3\" }").toBytes()));
+        assertThat(((IndexRequest) bulkRequest.requests().get(2)).source(), equalTo(new BytesArray("{ \"field1\" : \"value3\" }")));
     }
 
     public void testSimpleBulk2() throws Exception {
@@ -81,7 +81,7 @@ public class BulkRequestTests extends ESTestCase {
         assertThat(bulkRequest.numberOfActions(), equalTo(4));
         assertThat(((UpdateRequest) bulkRequest.requests().get(0)).id(), equalTo("1"));
         assertThat(((UpdateRequest) bulkRequest.requests().get(0)).retryOnConflict(), equalTo(2));
-        assertThat(((UpdateRequest) bulkRequest.requests().get(0)).doc().source().toUtf8(), equalTo("{\"field\":\"value\"}"));
+        assertThat(((UpdateRequest) bulkRequest.requests().get(0)).doc().source().utf8ToString(), equalTo("{\"field\":\"value\"}"));
         assertThat(((UpdateRequest) bulkRequest.requests().get(1)).id(), equalTo("0"));
         assertThat(((UpdateRequest) bulkRequest.requests().get(1)).type(), equalTo("type1"));
         assertThat(((UpdateRequest) bulkRequest.requests().get(1)).index(), equalTo("index1"));
@@ -93,7 +93,7 @@ public class BulkRequestTests extends ESTestCase {
         assertThat(scriptParams, notNullValue());
         assertThat(scriptParams.size(), equalTo(1));
         assertThat(((Integer) scriptParams.get("param1")), equalTo(1));
-        assertThat(((UpdateRequest) bulkRequest.requests().get(1)).upsertRequest().source().toUtf8(), equalTo("{\"counter\":1}"));
+        assertThat(((UpdateRequest) bulkRequest.requests().get(1)).upsertRequest().source().utf8ToString(), equalTo("{\"counter\":1}"));
     }
 
     public void testBulkAllowExplicitIndex() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java b/core/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java
index 451ade62584..ef259463139 100644
--- a/core/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java
@@ -70,7 +70,7 @@ public class MultiGetShardRequestTests extends ESTestCase {
         out.setVersion(randomVersion(random()));
         multiGetShardRequest.writeTo(out);
 
-        StreamInput in = StreamInput.wrap(out.bytes());
+        StreamInput in = out.bytes().streamInput();
         in.setVersion(out.getVersion());
         MultiGetShardRequest multiGetShardRequest2 = new MultiGetShardRequest();
         multiGetShardRequest2.readFrom(in);
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java
index 323a8c0aaa6..544e2932b44 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java
@@ -45,7 +45,7 @@ public class SimulateDocumentSimpleResultTests extends ESTestCase {
 
         BytesStreamOutput out = new BytesStreamOutput();
         simulateDocumentBaseResult.writeTo(out);
-        StreamInput streamInput = StreamInput.wrap(out.bytes());
+        StreamInput streamInput = out.bytes().streamInput();
         SimulateDocumentBaseResult otherSimulateDocumentBaseResult = new SimulateDocumentBaseResult(streamInput);
 
         if (isFailure) {
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java
index 1376ca4280e..576e8e01724 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java
@@ -73,7 +73,7 @@ public class SimulatePipelineResponseTests extends ESTestCase {
         SimulatePipelineResponse response = new SimulatePipelineResponse(randomAsciiOfLengthBetween(1, 10), isVerbose, results);
         BytesStreamOutput out = new BytesStreamOutput();
         response.writeTo(out);
-        StreamInput streamInput = StreamInput.wrap(out.bytes());
+        StreamInput streamInput = out.bytes().streamInput();
         SimulatePipelineResponse otherResponse = new SimulatePipelineResponse();
         otherResponse.readFrom(streamInput);
 
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java
index f612f36c9d6..ccf3a674944 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java
@@ -48,7 +48,7 @@ public class SimulateProcessorResultTests extends ESTestCase {
 
         BytesStreamOutput out = new BytesStreamOutput();
         simulateProcessorResult.writeTo(out);
-        StreamInput streamInput = StreamInput.wrap(out.bytes());
+        StreamInput streamInput = out.bytes().streamInput();
         SimulateProcessorResult otherSimulateProcessorResult = new SimulateProcessorResult(streamInput);
         assertThat(otherSimulateProcessorResult.getProcessorTag(), equalTo(simulateProcessorResult.getProcessorTag()));
         if (isFailure) {
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/WritePipelineResponseTests.java b/core/src/test/java/org/elasticsearch/action/ingest/WritePipelineResponseTests.java
index 3f252c37072..00327603ba8 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/WritePipelineResponseTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/WritePipelineResponseTests.java
@@ -35,7 +35,7 @@ public class WritePipelineResponseTests extends ESTestCase {
         response = new WritePipelineResponse(isAcknowledged);
         BytesStreamOutput out = new BytesStreamOutput();
         response.writeTo(out);
-        StreamInput streamInput = StreamInput.wrap(out.bytes());
+        StreamInput streamInput = out.bytes().streamInput();
         WritePipelineResponse otherResponse = new WritePipelineResponse();
         otherResponse.readFrom(streamInput);
 
@@ -46,7 +46,7 @@ public class WritePipelineResponseTests extends ESTestCase {
         WritePipelineResponse response = new WritePipelineResponse();
         BytesStreamOutput out = new BytesStreamOutput();
         response.writeTo(out);
-        StreamInput streamInput = StreamInput.wrap(out.bytes());
+        StreamInput streamInput = out.bytes().streamInput();
         WritePipelineResponse otherResponse = new WritePipelineResponse();
         otherResponse.readFrom(streamInput);
 
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java b/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java
index a7ce842913d..b4908846e97 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java
@@ -112,7 +112,7 @@ public class WriteableIngestDocumentTests extends ESTestCase {
 
         BytesStreamOutput out = new BytesStreamOutput();
         writeableIngestDocument.writeTo(out);
-        StreamInput streamInput = StreamInput.wrap(out.bytes());
+        StreamInput streamInput = out.bytes().streamInput();
         WriteableIngestDocument otherWriteableIngestDocument = new WriteableIngestDocument(streamInput);
         assertIngestDocument(otherWriteableIngestDocument.getIngestDocument(), writeableIngestDocument.getIngestDocument());
     }
diff --git a/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java b/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java
index 2bff71d3c40..3e592d1c341 100644
--- a/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java
@@ -64,7 +64,7 @@ public class MainActionTests extends ESTestCase {
         BytesStreamOutput streamOutput = new BytesStreamOutput();
         mainResponse.writeTo(streamOutput);
         final MainResponse serialized = new MainResponse();
-        serialized.readFrom(new ByteBufferStreamInput(ByteBuffer.wrap(streamOutput.bytes().toBytes())));
+        serialized.readFrom(streamOutput.bytes().streamInput());
 
         assertThat(serialized.getNodeName(), equalTo(nodeName));
         assertThat(serialized.getClusterName(), equalTo(clusterName));
diff --git a/core/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java b/core/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
index d5ed5302b97..d656e0f62a9 100644
--- a/core/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java
@@ -38,7 +38,7 @@ public class IndicesOptionsTests extends ESTestCase {
             output.setVersion(outputVersion);
             indicesOptions.writeIndicesOptions(output);
 
-            StreamInput streamInput = StreamInput.wrap(output.bytes());
+            StreamInput streamInput = output.bytes().streamInput();
             streamInput.setVersion(randomVersion(random()));
             IndicesOptions indicesOptions2 = IndicesOptions.readIndicesOptions(streamInput);
 
diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java
index 37a1bc92e9c..1611c63d2ba 100644
--- a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java
+++ b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java
@@ -140,7 +140,7 @@ public class GetTermVectorsCheckDocFreqIT extends ESIntegTestCase {
         xBuilder.startObject();
         response.toXContent(xBuilder, null);
         xBuilder.endObject();
-        String utf8 = xBuilder.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");;
+        String utf8 = xBuilder.bytes().utf8ToString().replaceFirst("\"took\":\\d+,", "");;
         String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\""
                 + i
                 + "\",\"_version\":1,\"found\":true,\"term_vectors\":{\"field\":{\"terms\":{\"brown\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":2,\"start_offset\":10,\"end_offset\":15,\"payload\":\"d29yZA==\"}]},\"dog\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":8,\"start_offset\":40,\"end_offset\":43,\"payload\":\"d29yZA==\"}]},\"fox\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":3,\"start_offset\":16,\"end_offset\":19,\"payload\":\"d29yZA==\"}]},\"jumps\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":4,\"start_offset\":20,\"end_offset\":25,\"payload\":\"d29yZA==\"}]},\"lazy\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":7,\"start_offset\":35,\"end_offset\":39,\"payload\":\"d29yZA==\"}]},\"over\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":5,\"start_offset\":26,\"end_offset\":30,\"payload\":\"d29yZA==\"}]},\"quick\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":1,\"start_offset\":4,\"end_offset\":9,\"payload\":\"d29yZA==\"}]},\"the\":{\"doc_freq\":15,\"ttf\":30,\"term_freq\":2,\"tokens\":[{\"position\":0,\"start_offset\":0,\"end_offset\":3,\"payload\":\"d29yZA==\"},{\"position\":6,\"start_offset\":31,\"end_offset\":34,\"payload\":\"d29yZA==\"}]}}}}}";
@@ -196,7 +196,7 @@ public class GetTermVectorsCheckDocFreqIT extends ESIntegTestCase {
         xBuilder.startObject();
         response.toXContent(xBuilder, null);
         xBuilder.endObject();
-        String utf8 = xBuilder.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");;
+        String utf8 = xBuilder.bytes().utf8ToString().replaceFirst("\"took\":\\d+,", "");;
         String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\""
                 + i
                 + "\",\"_version\":1,\"found\":true,\"term_vectors\":{\"field\":{\"field_statistics\":{\"sum_doc_freq\":120,\"doc_count\":15,\"sum_ttf\":135},\"terms\":{\"brown\":{\"term_freq\":1,\"tokens\":[{\"position\":2,\"start_offset\":10,\"end_offset\":15,\"payload\":\"d29yZA==\"}]},\"dog\":{\"term_freq\":1,\"tokens\":[{\"position\":8,\"start_offset\":40,\"end_offset\":43,\"payload\":\"d29yZA==\"}]},\"fox\":{\"term_freq\":1,\"tokens\":[{\"position\":3,\"start_offset\":16,\"end_offset\":19,\"payload\":\"d29yZA==\"}]},\"jumps\":{\"term_freq\":1,\"tokens\":[{\"position\":4,\"start_offset\":20,\"end_offset\":25,\"payload\":\"d29yZA==\"}]},\"lazy\":{\"term_freq\":1,\"tokens\":[{\"position\":7,\"start_offset\":35,\"end_offset\":39,\"payload\":\"d29yZA==\"}]},\"over\":{\"term_freq\":1,\"tokens\":[{\"position\":5,\"start_offset\":26,\"end_offset\":30,\"payload\":\"d29yZA==\"}]},\"quick\":{\"term_freq\":1,\"tokens\":[{\"position\":1,\"start_offset\":4,\"end_offset\":9,\"payload\":\"d29yZA==\"}]},\"the\":{\"term_freq\":2,\"tokens\":[{\"position\":0,\"start_offset\":0,\"end_offset\":3,\"payload\":\"d29yZA==\"},{\"position\":6,\"start_offset\":31,\"end_offset\":34,\"payload\":\"d29yZA==\"}]}}}}}";
@@ -255,7 +255,7 @@ public class GetTermVectorsCheckDocFreqIT extends ESIntegTestCase {
         xBuilder.startObject();
         response.toXContent(xBuilder, ToXContent.EMPTY_PARAMS);
         xBuilder.endObject();
-        String utf8 = xBuilder.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");;
+        String utf8 = xBuilder.bytes().utf8ToString().replaceFirst("\"took\":\\d+,", "");;
         String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\""
                 + i
                 + "\",\"_version\":1,\"found\":true,\"term_vectors\":{\"field\":{\"field_statistics\":{\"sum_doc_freq\":120,\"doc_count\":15,\"sum_ttf\":135},\"terms\":{\"brown\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":2,\"start_offset\":10,\"end_offset\":15,\"payload\":\"d29yZA==\"}]},\"dog\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":8,\"start_offset\":40,\"end_offset\":43,\"payload\":\"d29yZA==\"}]},\"fox\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":3,\"start_offset\":16,\"end_offset\":19,\"payload\":\"d29yZA==\"}]},\"jumps\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":4,\"start_offset\":20,\"end_offset\":25,\"payload\":\"d29yZA==\"}]},\"lazy\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":7,\"start_offset\":35,\"end_offset\":39,\"payload\":\"d29yZA==\"}]},\"over\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":5,\"start_offset\":26,\"end_offset\":30,\"payload\":\"d29yZA==\"}]},\"quick\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":1,\"start_offset\":4,\"end_offset\":9,\"payload\":\"d29yZA==\"}]},\"the\":{\"doc_freq\":15,\"ttf\":30,\"term_freq\":2,\"tokens\":[{\"position\":0,\"start_offset\":0,\"end_offset\":3,\"payload\":\"d29yZA==\"},{\"position\":6,\"start_offset\":31,\"end_offset\":34,\"payload\":\"d29yZA==\"}]}}}}}";
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
index b82b5e0ba60..da196d5d1d3 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
@@ -30,6 +30,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.metadata.RepositoriesMetaData;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -121,7 +122,7 @@ public class ClusterStateDiffIT extends ESIntegTestCase {
                 Diff<ClusterState> diffBeforeSerialization = clusterState.diff(previousClusterState);
                 BytesStreamOutput os = new BytesStreamOutput();
                 diffBeforeSerialization.writeTo(os);
-                byte[] diffBytes = os.bytes().toBytes();
+                byte[] diffBytes = BytesReference.toBytes(os.bytes());
                 Diff<ClusterState> diff;
                 try (StreamInput input = StreamInput.wrap(diffBytes)) {
                     diff = previousClusterStateFromDiffs.readDiffFrom(input);
diff --git a/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java b/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java
index 7f2d0828128..a7fe1b918c0 100644
--- a/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java
@@ -55,7 +55,7 @@ public class ClusterBlockTests extends ESTestCase {
             out.setVersion(version);
             clusterBlock.writeTo(out);
 
-            StreamInput in = StreamInput.wrap(out.bytes());
+            StreamInput in = out.bytes().streamInput();
             in.setVersion(version);
             ClusterBlock result = ClusterBlock.readClusterBlock(in);
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
index fd1e3e62466..0918358a510 100644
--- a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
@@ -169,7 +169,7 @@ public class ClusterStateHealthTests extends ESTestCase {
         if (randomBoolean()) {
             BytesStreamOutput out = new BytesStreamOutput();
             clusterStateHealth.writeTo(out);
-            StreamInput in = StreamInput.wrap(out.bytes());
+            StreamInput in = out.bytes().streamInput();
             clusterStateHealth = new ClusterStateHealth(in);
         }
         return clusterStateHealth;
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java
index aec701052fb..8dd950ba8e6 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java
@@ -60,8 +60,7 @@ public class IndexGraveyardTests extends ESTestCase {
         final IndexGraveyard graveyard = createRandom();
         final BytesStreamOutput out = new BytesStreamOutput();
         graveyard.writeTo(out);
-        final ByteBufferStreamInput in = new ByteBufferStreamInput(ByteBuffer.wrap(out.bytes().toBytes()));
-        assertThat(IndexGraveyard.fromStream(in), equalTo(graveyard));
+        assertThat(IndexGraveyard.fromStream(out.bytes().streamInput()), equalTo(graveyard));
     }
 
     public void testXContent() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java
index 0c9827587ea..5fef33be388 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java
@@ -69,7 +69,7 @@ public class IndexMetaDataTests extends ESTestCase {
 
         final BytesStreamOutput out = new BytesStreamOutput();
         metaData.writeTo(out);
-        IndexMetaData deserialized = IndexMetaData.PROTO.readFrom(StreamInput.wrap(out.bytes()));
+        IndexMetaData deserialized = IndexMetaData.PROTO.readFrom(out.bytes().streamInput());
         assertEquals(metaData, deserialized);
         assertEquals(metaData.hashCode(), deserialized.hashCode());
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
index e1e3a39122c..cf040fb3c7f 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java
@@ -22,7 +22,6 @@ package org.elasticsearch.cluster.metadata;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -34,7 +33,6 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
-import java.nio.ByteBuffer;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
@@ -185,8 +183,7 @@ public class MetaDataTests extends ESTestCase {
         final MetaData originalMeta = MetaData.builder().indexGraveyard(graveyard).build();
         final BytesStreamOutput out = new BytesStreamOutput();
         originalMeta.writeTo(out);
-        final ByteBufferStreamInput in = new ByteBufferStreamInput(ByteBuffer.wrap(out.bytes().toBytes()));
-        final MetaData fromStreamMeta = MetaData.PROTO.readFrom(in);
+        final MetaData fromStreamMeta = MetaData.PROTO.readFrom(out.bytes().streamInput());
         assertThat(fromStreamMeta.indexGraveyard(), equalTo(fromStreamMeta.indexGraveyard()));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java
index 7267252b19f..fa9133f6d36 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java
@@ -206,7 +206,7 @@ public class ShardRoutingTests extends ESTestCase {
             if (randomBoolean()) {
                 BytesStreamOutput out = new BytesStreamOutput();
                 routing.writeTo(out);
-                routing = new ShardRouting(StreamInput.wrap(out.bytes()));
+                routing = new ShardRouting(out.bytes().streamInput());
             }
             if (routing.initializing() || routing.relocating()) {
                 assertEquals(routing.toString(), byteSize, routing.getExpectedShardSize());
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
index 75300a4beb8..ec33a3cd5fc 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java
@@ -82,7 +82,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase {
         meta.writeTo(out);
         out.close();
 
-        UnassignedInfo read = new UnassignedInfo(StreamInput.wrap(out.bytes()));
+        UnassignedInfo read = new UnassignedInfo(out.bytes().streamInput());
         assertThat(read.getReason(), equalTo(meta.getReason()));
         assertThat(read.getUnassignedTimeInMillis(), equalTo(meta.getUnassignedTimeInMillis()));
         assertThat(read.getMessage(), equalTo(meta.getMessage()));
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
index 28f27b8988c..f95fb687c76 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java
@@ -430,7 +430,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase {
         );
         BytesStreamOutput bytes = new BytesStreamOutput();
         AllocationCommands.writeTo(commands, bytes);
-        StreamInput in = StreamInput.wrap(bytes.bytes());
+        StreamInput in = bytes.bytes().streamInput();
 
         // Since the commands are named writeable we need to register them and wrap the input stream
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
diff --git a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
index 7b81d3ece27..4fa6615ac45 100644
--- a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java
@@ -80,7 +80,7 @@ public class ClusterSerializationTests extends ESAllocationTestCase {
 
         BytesStreamOutput outStream = new BytesStreamOutput();
         source.writeTo(outStream);
-        StreamInput inStream = StreamInput.wrap(outStream.bytes().toBytes());
+        StreamInput inStream = outStream.bytes().streamInput();
         RoutingTable target = RoutingTable.Builder.readFrom(inStream);
 
         assertThat(target.prettyPrint(), equalTo(source.prettyPrint()));
diff --git a/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java b/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java
index 452c6054576..611c261e334 100644
--- a/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java
@@ -310,7 +310,7 @@ public class DiffableTests extends ESTestCase {
                 logger.debug("--> serializing diff");
                 BytesStreamOutput out = new BytesStreamOutput();
                 diffMap.writeTo(out);
-                StreamInput in = StreamInput.wrap(out.bytes());
+                StreamInput in = out.bytes().streamInput();
                 logger.debug("--> reading diff back");
                 diffMap = readDiff(in);
             }
diff --git a/core/src/test/java/org/elasticsearch/common/ChannelsTests.java b/core/src/test/java/org/elasticsearch/common/ChannelsTests.java
index 4f2bad36d4a..c0cb3482b0e 100644
--- a/core/src/test/java/org/elasticsearch/common/ChannelsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/ChannelsTests.java
@@ -19,14 +19,11 @@
 
 package org.elasticsearch.common;
 
-import org.elasticsearch.common.bytes.ByteBufferBytesReference;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.Channels;
 import org.elasticsearch.test.ESTestCase;
 import org.hamcrest.Matchers;
-import org.jboss.netty.buffer.ByteBufferBackedChannelBuffer;
-import org.jboss.netty.buffer.ChannelBuffer;
 import org.junit.After;
 import org.junit.Before;
 
@@ -85,7 +82,7 @@ public class ChannelsTests extends ESTestCase {
         BytesReference source = new BytesArray(randomBytes, offset + offsetToRead, lengthToRead);
         BytesReference read = new BytesArray(readBytes, offset + offsetToRead, lengthToRead);
 
-        assertThat("read bytes didn't match written bytes", source.toBytes(), Matchers.equalTo(read.toBytes()));
+        assertThat("read bytes didn't match written bytes", BytesReference.toBytes(source), Matchers.equalTo(BytesReference.toBytes(read)));
     }
 
     public void testBufferReadPastEOFWithException() throws Exception {
@@ -157,7 +154,9 @@ public class ChannelsTests extends ESTestCase {
         copy.flip();
 
         BytesReference sourceRef = new BytesArray(randomBytes, offset + offsetToRead, lengthToRead);
-        BytesReference copyRef = new ByteBufferBytesReference(copy);
+        byte[] tmp = new byte[copy.remaining()];
+        copy.duplicate().get(tmp);
+        BytesReference copyRef = new BytesArray(tmp);
 
         assertTrue("read bytes didn't match written bytes", sourceRef.equals(copyRef));
     }
diff --git a/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java b/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
index f31d7c69325..90922327732 100644
--- a/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
+++ b/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
@@ -29,7 +29,6 @@ import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.ByteArray;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.test.ESTestCase;
-import org.hamcrest.Matchers;
 
 import java.io.EOFException;
 import java.io.IOException;
@@ -66,12 +65,9 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
         int sliceLength = Math.max(0, length - sliceOffset - 1);
         BytesReference slice = pbr.slice(sliceOffset, sliceLength);
         assertEquals(sliceLength, slice.length());
-
-        if (slice.hasArray()) {
-            assertEquals(sliceOffset, slice.arrayOffset());
-        } else {
-           expectThrows(IllegalStateException.class, () ->
-                slice.arrayOffset());
+        BytesRef singlePageOrNull = getSinglePageOrNull(slice);
+        if (singlePageOrNull != null) {
+            assertEquals(sliceOffset, singlePageOrNull.offset);
         }
     }
 
@@ -109,7 +105,7 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
 
         // bulk-read all
         si.readFully(targetBuf);
-        assertArrayEquals(pbr.toBytes(), targetBuf);
+        assertArrayEquals(BytesReference.toBytes(pbr), targetBuf);
 
         // continuing to read should now fail with EOFException
         try {
@@ -141,7 +137,7 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
         // now do NOT reset the stream - keep the stream's offset!
 
         // buffer to compare remaining bytes against bulk read
-        byte[] pbrBytesWithOffset = Arrays.copyOfRange(pbr.toBytes(), offset, length);
+        byte[] pbrBytesWithOffset = Arrays.copyOfRange(BytesReference.toBytes(pbr), offset, length);
         // randomized target buffer to ensure no stale slots
         byte[] targetBytes = new byte[pbrBytesWithOffset.length];
         random().nextBytes(targetBytes);
@@ -178,7 +174,7 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
         }
         assertEquals(pbr.length(), target.length());
         BytesRef targetBytes = target.get();
-        assertArrayEquals(pbr.toBytes(), Arrays.copyOfRange(targetBytes.bytes, targetBytes.offset, targetBytes.length));
+        assertArrayEquals(BytesReference.toBytes(pbr), Arrays.copyOfRange(targetBytes.bytes, targetBytes.offset, targetBytes.length));
     }
 
     public void testSliceStreamInput() throws IOException {
@@ -208,11 +204,11 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
         assertEquals(sliceInput.available(), 0);
 
         // compare slice content with upper half of original
-        byte[] pbrSliceBytes = Arrays.copyOfRange(pbr.toBytes(), sliceOffset, length);
+        byte[] pbrSliceBytes = Arrays.copyOfRange(BytesReference.toBytes(pbr), sliceOffset, length);
         assertArrayEquals(pbrSliceBytes, sliceBytes);
 
         // compare slice bytes with bytes read from slice via streamInput :D
-        byte[] sliceToBytes = slice.toBytes();
+        byte[] sliceToBytes = BytesReference.toBytes(slice);
         assertEquals(sliceBytes.length, sliceToBytes.length);
         assertArrayEquals(sliceBytes, sliceToBytes);
 
@@ -233,7 +229,7 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
         BytesStreamOutput out = new BytesStreamOutput();
         pbr.writeTo(out);
         assertEquals(pbr.length(), out.size());
-        assertArrayEquals(pbr.toBytes(), out.bytes().toBytes());
+        assertArrayEquals(BytesReference.toBytes(pbr), BytesReference.toBytes(out.bytes()));
         out.close();
     }
 
@@ -246,7 +242,7 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
         BytesStreamOutput sliceOut = new BytesStreamOutput(sliceLength);
         slice.writeTo(sliceOut);
         assertEquals(slice.length(), sliceOut.size());
-        assertArrayEquals(slice.toBytes(), sliceOut.bytes().toBytes());
+        assertArrayEquals(BytesReference.toBytes(slice), BytesReference.toBytes(sliceOut.bytes()));
         sliceOut.close();
     }
 
@@ -254,16 +250,16 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
         int[] sizes = {0, randomInt(PAGE_SIZE), PAGE_SIZE, randomIntBetween(2, PAGE_SIZE * randomIntBetween(2, 5))};
         for (int i = 0; i < sizes.length; i++) {
             BytesReference pbr = newBytesReference(sizes[i]);
-            byte[] bytes = pbr.toBytes();
+            byte[] bytes = BytesReference.toBytes(pbr);
             assertEquals(sizes[i], bytes.length);
         }
     }
 
-    public void testToBytesArraySharedPage() throws IOException {
+    public void testToBytesRefSharedPage() throws IOException {
         int length = randomIntBetween(10, PAGE_SIZE);
         BytesReference pbr = newBytesReference(length);
-        BytesArray ba = pbr.toBytesArray();
-        BytesArray ba2 = pbr.toBytesArray();
+        BytesArray ba = new BytesArray(pbr.toBytesRef());
+        BytesArray ba2 = new BytesArray(pbr.toBytesRef());
         assertNotNull(ba);
         assertNotNull(ba2);
         assertEquals(pbr.length(), ba.length());
@@ -272,46 +268,46 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
         assertSame(ba.array(), ba2.array());
     }
 
-    public void testToBytesArrayMaterializedPages() throws IOException {
+    public void testToBytesRefMaterializedPages() throws IOException {
         // we need a length != (n * pagesize) to avoid page sharing at boundaries
         int length = 0;
         while ((length % PAGE_SIZE) == 0) {
             length = randomIntBetween(PAGE_SIZE, PAGE_SIZE * randomIntBetween(2, 5));
         }
         BytesReference pbr = newBytesReference(length);
-        BytesArray ba = pbr.toBytesArray();
-        BytesArray ba2 = pbr.toBytesArray();
+        BytesArray ba = new BytesArray(pbr.toBytesRef());
+        BytesArray ba2 = new BytesArray(pbr.toBytesRef());
         assertNotNull(ba);
         assertNotNull(ba2);
         assertEquals(pbr.length(), ba.length());
         assertEquals(ba.length(), ba2.length());
     }
 
-    public void testCopyBytesArray() throws IOException {
+    public void testCopyBytesRefSharesBytes() throws IOException {
         // small PBR which would normally share the first page
         int length = randomIntBetween(10, PAGE_SIZE);
         BytesReference pbr = newBytesReference(length);
-        BytesArray ba = pbr.copyBytesArray();
-        BytesArray ba2 = pbr.copyBytesArray();
+        BytesArray ba = new BytesArray(pbr.toBytesRef(), true);
+        BytesArray ba2 = new BytesArray(pbr.toBytesRef(), true);
         assertNotNull(ba);
         assertNotSame(ba, ba2);
         assertNotSame(ba.array(), ba2.array());
     }
 
-    public void testSliceCopyBytesArray() throws IOException {
+    public void testSliceCopyBytesRef() throws IOException {
         int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8));
         BytesReference pbr = newBytesReference(length);
         int sliceOffset = randomIntBetween(0, pbr.length());
-        int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset);
+        int sliceLength = randomIntBetween(0, pbr.length() - sliceOffset);
         BytesReference slice = pbr.slice(sliceOffset, sliceLength);
 
-        BytesArray ba1 = slice.copyBytesArray();
-        BytesArray ba2 = slice.copyBytesArray();
+        BytesArray ba1 = new BytesArray(slice.toBytesRef(), true);
+        BytesArray ba2 = new BytesArray(slice.toBytesRef(), true);
         assertNotNull(ba1);
         assertNotNull(ba2);
         assertNotSame(ba1.array(), ba2.array());
-        assertArrayEquals(slice.toBytes(), ba1.array());
-        assertArrayEquals(slice.toBytes(), ba2.array());
+        assertArrayEquals(BytesReference.toBytes(slice), ba1.array());
+        assertArrayEquals(BytesReference.toBytes(slice), ba2.array());
         assertArrayEquals(ba1.array(), ba2.array());
     }
 
@@ -329,14 +325,14 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
         while((ref = iterator.next()) != null) {
             builder.append(ref);
         }
-        assertArrayEquals(pbr.toBytes(), BytesRef.deepCopyOf(builder.toBytesRef()).bytes);
+        assertArrayEquals(BytesReference.toBytes(pbr), BytesRef.deepCopyOf(builder.toBytesRef()).bytes);
     }
 
     public void testSliceIterator() throws IOException {
         int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8));
         BytesReference pbr = newBytesReference(length);
         int sliceOffset = randomIntBetween(0, pbr.length());
-        int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset);
+        int sliceLength = randomIntBetween(0, pbr.length() - sliceOffset);
         BytesReference slice = pbr.slice(sliceOffset, sliceLength);
         BytesRefIterator iterator = slice.iterator();
         BytesRef ref = null;
@@ -344,7 +340,7 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
         while((ref = iterator.next()) != null) {
             builder.append(ref);
         }
-        assertArrayEquals(slice.toBytes(), BytesRef.deepCopyOf(builder.toBytesRef()).bytes);
+        assertArrayEquals(BytesReference.toBytes(slice), BytesRef.deepCopyOf(builder.toBytesRef()).bytes);
     }
 
     public void testIteratorRandom() throws IOException {
@@ -352,12 +348,12 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
         BytesReference pbr = newBytesReference(length);
         if (randomBoolean()) {
             int sliceOffset = randomIntBetween(0, pbr.length());
-            int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset);
+            int sliceLength = randomIntBetween(0, pbr.length() - sliceOffset);
             pbr = pbr.slice(sliceOffset, sliceLength);
         }
 
         if (randomBoolean()) {
-            pbr = pbr.toBytesArray();
+            pbr = new BytesArray(pbr.toBytesRef());
         }
         BytesRefIterator iterator = pbr.iterator();
         BytesRef ref = null;
@@ -365,29 +361,15 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
         while((ref = iterator.next()) != null) {
             builder.append(ref);
         }
-        assertArrayEquals(pbr.toBytes(), BytesRef.deepCopyOf(builder.toBytesRef()).bytes);
-    }
-
-    public void testArray() throws IOException {
-        int[] sizes = {0, randomInt(PAGE_SIZE), PAGE_SIZE, randomIntBetween(2, PAGE_SIZE * randomIntBetween(2, 5))};
-
-        for (int i = 0; i < sizes.length; i++) {
-            BytesReference pbr = newBytesReference(sizes[i]);
-            byte[] array = pbr.array();
-            assertNotNull(array);
-            assertEquals(sizes[i], array.length);
-            assertSame(array, pbr.array());
-        }
+        assertArrayEquals(BytesReference.toBytes(pbr), BytesRef.deepCopyOf(builder.toBytesRef()).bytes);
     }
 
     public void testArrayOffset() throws IOException {
         int length = randomInt(PAGE_SIZE * randomIntBetween(2, 5));
         BytesReference pbr = newBytesReference(length);
-        if (pbr.hasArray()) {
-            assertEquals(0, pbr.arrayOffset());
-        } else {
-            expectThrows(IllegalStateException.class, () ->
-                pbr.arrayOffset());
+        BytesRef singlePageOrNull = getSinglePageOrNull(pbr);
+        if (singlePageOrNull != null) {
+            assertEquals(0, singlePageOrNull.offset);
         }
     }
 
@@ -395,20 +377,24 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
         int length = randomInt(PAGE_SIZE * randomIntBetween(2, 5));
         BytesReference pbr = newBytesReference(length);
         int sliceOffset = randomIntBetween(0, pbr.length() - 1); // an offset to the end would be len 0
-        int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset);
+        int sliceLength = randomIntBetween(0, pbr.length() - sliceOffset);
         BytesReference slice = pbr.slice(sliceOffset, sliceLength);
-        if (slice.hasArray()) {
-            assertEquals(sliceOffset, slice.arrayOffset());
-        } else {
-            expectThrows(IllegalStateException.class, () ->
-                slice.arrayOffset());
+        BytesRef singlePageOrNull = getSinglePageOrNull(slice);
+        if (singlePageOrNull != null) {
+            if (getSinglePageOrNull(pbr) == null) {
+                // original reference has pages
+                assertEquals(sliceOffset % PAGE_SIZE, singlePageOrNull.offset);
+            } else {
+                // orig ref has no pages ie. BytesArray
+                assertEquals(sliceOffset, singlePageOrNull.offset);
+            }
         }
     }
 
     public void testToUtf8() throws IOException {
         // test empty
         BytesReference pbr = newBytesReference(0);
-        assertEquals("", pbr.toUtf8());
+        assertEquals("", pbr.utf8ToString());
         // TODO: good way to test?
     }
 
@@ -417,7 +403,6 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
         BytesReference pbr = newBytesReference(length);
         BytesRef ref = pbr.toBytesRef();
         assertNotNull(ref);
-        assertEquals(pbr.arrayOffset(), ref.offset);
         assertEquals(pbr.length(), ref.length);
     }
 
@@ -426,21 +411,13 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
         BytesReference pbr = newBytesReference(length);
         // get a BytesRef from a slice
         int sliceOffset = randomIntBetween(0, pbr.length());
-        int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset);
+        int sliceLength = randomIntBetween(0, pbr.length() - sliceOffset);
         BytesRef sliceRef = pbr.slice(sliceOffset, sliceLength).toBytesRef();
         // note that these are only true if we have <= than a page, otherwise offset/length are shifted
         assertEquals(sliceOffset, sliceRef.offset);
         assertEquals(sliceLength, sliceRef.length);
     }
 
-    public void testCopyBytesRef() throws IOException {
-        int length = randomIntBetween(0, PAGE_SIZE * randomIntBetween(2, 5));
-        BytesReference pbr = newBytesReference(length);
-        BytesRef ref = pbr.copyBytesRef();
-        assertNotNull(ref);
-        assertEquals(pbr.length(), ref.length);
-    }
-
     public void testHashCode() throws IOException {
         // empty content must have hash 1 (JDK compat)
         BytesReference pbr = newBytesReference(0);
@@ -448,40 +425,36 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
 
         // test with content
         pbr = newBytesReference(randomIntBetween(0, PAGE_SIZE * randomIntBetween(2, 5)));
-        int jdkHash = Arrays.hashCode(pbr.toBytes());
+        int jdkHash = Arrays.hashCode(BytesReference.toBytes(pbr));
         int pbrHash = pbr.hashCode();
         assertEquals(jdkHash, pbrHash);
 
         // test hashes of slices
         int sliceFrom = randomIntBetween(0, pbr.length());
-        int sliceLength = randomIntBetween(pbr.length() - sliceFrom, pbr.length() - sliceFrom);
+        int sliceLength = randomIntBetween(0, pbr.length() - sliceFrom);
         BytesReference slice = pbr.slice(sliceFrom, sliceLength);
-        int sliceJdkHash = Arrays.hashCode(slice.toBytes());
+        int sliceJdkHash = Arrays.hashCode(BytesReference.toBytes(slice));
         int sliceHash = slice.hashCode();
         assertEquals(sliceJdkHash, sliceHash);
     }
 
-    public void testEquals() {
-        int length = randomIntBetween(100, PAGE_SIZE * randomIntBetween(2, 5));
-        ByteArray ba1 = bigarrays.newByteArray(length, false);
-        ByteArray ba2 = bigarrays.newByteArray(length, false);
-
-        // copy contents
-        for (long i = 0; i < length; i++) {
-            ba2.set(i, ba1.get(i));
-        }
+    public void testEquals() throws IOException {
+        BytesReference bytesReference = newBytesReference(randomIntBetween(100, PAGE_SIZE * randomIntBetween(2, 5)));
+        BytesReference copy = bytesReference.slice(0, bytesReference.length());
 
         // get refs & compare
-        BytesReference pbr = new PagedBytesReference(bigarrays, ba1, length);
-        BytesReference pbr2 = new PagedBytesReference(bigarrays, ba2, length);
-        assertEquals(pbr, pbr2);
-    }
+        assertEquals(copy, bytesReference);
+        int sliceFrom = randomIntBetween(0, bytesReference.length());
+        int sliceLength = randomIntBetween(0, bytesReference.length() - sliceFrom);
+        assertEquals(copy.slice(sliceFrom, sliceLength), bytesReference.slice(sliceFrom, sliceLength));
 
-    public void testEqualsPeerClass() throws IOException {
-        int length = randomIntBetween(100, PAGE_SIZE * randomIntBetween(2, 5));
-        BytesReference pbr = newBytesReference(length);
-        BytesReference ba = new BytesArray(pbr.toBytes());
-        assertEquals(pbr, ba);
+        BytesRef bytesRef = BytesRef.deepCopyOf(copy.toBytesRef());
+        assertEquals(new BytesArray(bytesRef), copy);
+
+        int offsetToFlip = randomIntBetween(0, bytesRef.length - 1);
+        int value = ~Byte.toUnsignedInt(bytesRef.bytes[bytesRef.offset+offsetToFlip]);
+        bytesRef.bytes[bytesRef.offset+offsetToFlip] = (byte)value;
+        assertNotEquals(new BytesArray(bytesRef), copy);
     }
 
     public void testSliceEquals() {
@@ -491,19 +464,118 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
 
         // test equality of slices
         int sliceFrom = randomIntBetween(0, pbr.length());
-        int sliceLength = randomIntBetween(pbr.length() - sliceFrom, pbr.length() - sliceFrom);
+        int sliceLength = randomIntBetween(0, pbr.length() - sliceFrom);
         BytesReference slice1 = pbr.slice(sliceFrom, sliceLength);
         BytesReference slice2 = pbr.slice(sliceFrom, sliceLength);
-        assertArrayEquals(slice1.toBytes(), slice2.toBytes());
+        assertArrayEquals(BytesReference.toBytes(slice1), BytesReference.toBytes(slice2));
 
         // test a slice with same offset but different length,
         // unless randomized testing gave us a 0-length slice.
         if (sliceLength > 0) {
             BytesReference slice3 = pbr.slice(sliceFrom, sliceLength / 2);
-            assertFalse(Arrays.equals(slice1.toBytes(), slice3.toBytes()));
+            assertFalse(Arrays.equals(BytesReference.toBytes(slice1), BytesReference.toBytes(slice3)));
         }
     }
 
     protected abstract BytesReference newBytesReference(int length) throws IOException;
 
+    public void testCompareTo() throws IOException {
+        final int iters = randomIntBetween(5, 10);
+        for (int i = 0; i < iters; i++) {
+            int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8));
+            BytesReference bytesReference = newBytesReference(length);
+            assertTrue(bytesReference.compareTo(new BytesArray("")) > 0);
+            assertTrue(new BytesArray("").compareTo(bytesReference) < 0);
+
+
+            assertEquals(0, bytesReference.compareTo(bytesReference));
+            int sliceFrom = randomIntBetween(0, bytesReference.length());
+            int sliceLength = randomIntBetween(0, bytesReference.length() - sliceFrom);
+            BytesReference slice = bytesReference.slice(sliceFrom, sliceLength);
+
+            assertEquals(bytesReference.toBytesRef().compareTo(slice.toBytesRef()),
+                new BytesArray(bytesReference.toBytesRef(), true).compareTo(new BytesArray(slice.toBytesRef(), true)));
+
+            assertEquals(bytesReference.toBytesRef().compareTo(slice.toBytesRef()),
+                bytesReference.compareTo(slice));
+            assertEquals(slice.toBytesRef().compareTo(bytesReference.toBytesRef()),
+                slice.compareTo(bytesReference));
+
+            assertEquals(0, slice.compareTo(new BytesArray(slice.toBytesRef())));
+            assertEquals(0, new BytesArray(slice.toBytesRef()).compareTo(slice));
+
+            final int crazyLength = length + randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8));
+            ReleasableBytesStreamOutput crazyStream = new ReleasableBytesStreamOutput(length, bigarrays);
+            final int offset = randomIntBetween(0, crazyLength - length);
+            for (int j = 0; j < offset; j++) {
+                crazyStream.writeByte((byte) random().nextInt(1 << 8));
+            }
+            bytesReference.writeTo(crazyStream);
+            for (int j = crazyStream.size(); j < crazyLength; j++) {
+                crazyStream.writeByte((byte) random().nextInt(1 << 8));
+            }
+            PagedBytesReference crazyReference = crazyStream.bytes();
+
+            assertFalse(crazyReference.compareTo(bytesReference) == 0);
+            assertEquals(0, crazyReference.slice(offset, length).compareTo(
+                bytesReference));
+            assertEquals(0, bytesReference.compareTo(
+                crazyReference.slice(offset, length)));
+        }
+    }
+
+    public static BytesRef getSinglePageOrNull(BytesReference ref) throws IOException {
+        if (ref.length() > 0) {
+            BytesRefIterator iterator = ref.iterator();
+            BytesRef next = iterator.next();
+            BytesRef retVal = next.clone();
+            if (iterator.next() == null) {
+                return retVal;
+            }
+        } else {
+            return new BytesRef();
+        }
+        return null;
+    }
+
+    public static int getNumPages(BytesReference ref) throws IOException {
+        int num = 0;
+        if (ref.length() > 0) {
+            BytesRefIterator iterator = ref.iterator();
+            while(iterator.next() != null) {
+                num++;
+            }
+        }
+        return num;
+    }
+
+
+    public void testBasicEquals() {
+        final int len = randomIntBetween(0, randomBoolean() ? 10: 100000);
+        final int offset1 = randomInt(5);
+        final byte[] array1 = new byte[offset1 + len + randomInt(5)];
+        random().nextBytes(array1);
+        final int offset2 = randomInt(offset1);
+        final byte[] array2 = Arrays.copyOfRange(array1, offset1 - offset2, array1.length);
+
+        final BytesArray b1 = new BytesArray(array1, offset1, len);
+        final BytesArray b2 = new BytesArray(array2, offset2, len);
+        assertEquals(b1, b2);
+        assertEquals(Arrays.hashCode(BytesReference.toBytes(b1)), b1.hashCode());
+        assertEquals(Arrays.hashCode(BytesReference.toBytes(b2)), b2.hashCode());
+
+        // test same instance
+        assertEquals(b1, b1);
+        assertEquals(b2, b2);
+
+        if (len > 0) {
+            // test different length
+            BytesArray differentLen = new BytesArray(array1, offset1, randomInt(len - 1));
+            assertNotEquals(b1, differentLen);
+
+            // test changed bytes
+            array1[offset1 + randomInt(len - 1)] += 13;
+            assertNotEquals(b1, b2);
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java b/core/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java
index 61d24ef44c3..fff030200b7 100644
--- a/core/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java
+++ b/core/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java
@@ -32,10 +32,28 @@ public class BytesArrayTests extends AbstractBytesReferenceTestCase {
             out.writeByte((byte) random().nextInt(1 << 8));
         }
         assertEquals(length, out.size());
-        BytesArray ref = out.bytes().toBytesArray();
+        BytesArray ref = new BytesArray(out.bytes().toBytesRef());
         assertEquals(length, ref.length());
         assertTrue(ref instanceof BytesArray);
         assertThat(ref.length(), Matchers.equalTo(length));
         return ref;
     }
+
+    public void testArray() throws IOException {
+        int[] sizes = {0, randomInt(PAGE_SIZE), PAGE_SIZE, randomIntBetween(2, PAGE_SIZE * randomIntBetween(2, 5))};
+
+        for (int i = 0; i < sizes.length; i++) {
+            BytesArray pbr = (BytesArray) newBytesReference(sizes[i]);
+            byte[] array = pbr.array();
+            assertNotNull(array);
+            assertEquals(sizes[i], array.length);
+            assertSame(array, pbr.array());
+        }
+    }
+
+    public void testArrayOffset() throws IOException {
+        int length = randomInt(PAGE_SIZE * randomIntBetween(2, 5));
+        BytesArray pbr = (BytesArray) newBytesReference(length);
+        assertEquals(0, pbr.offset());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/bytes/BytesReferenceTests.java b/core/src/test/java/org/elasticsearch/common/bytes/BytesReferenceTests.java
deleted file mode 100644
index 60f4983dd19..00000000000
--- a/core/src/test/java/org/elasticsearch/common/bytes/BytesReferenceTests.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.bytes;
-
-
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.Arrays;
-
-public class BytesReferenceTests extends ESTestCase {
-
-    public void testEquals() {
-        final int len = randomIntBetween(0, randomBoolean() ? 10: 100000);
-        final int offset1 = randomInt(5);
-        final byte[] array1 = new byte[offset1 + len + randomInt(5)];
-        random().nextBytes(array1);
-        final int offset2 = randomInt(offset1);
-        final byte[] array2 = Arrays.copyOfRange(array1, offset1 - offset2, array1.length);
-
-        final BytesArray b1 = new BytesArray(array1, offset1, len);
-        final BytesArray b2 = new BytesArray(array2, offset2, len);
-        assertTrue(BytesReference.Helper.bytesEqual(b1, b2));
-        assertTrue(BytesReference.Helper.bytesEquals(b1, b2));
-        assertEquals(Arrays.hashCode(b1.toBytes()), b1.hashCode());
-        assertEquals(BytesReference.Helper.bytesHashCode(b1), BytesReference.Helper.slowHashCode(b2));
-
-        // test same instance
-        assertTrue(BytesReference.Helper.bytesEqual(b1, b1));
-        assertTrue(BytesReference.Helper.bytesEquals(b1, b1));
-        assertEquals(BytesReference.Helper.bytesHashCode(b1), BytesReference.Helper.slowHashCode(b1));
-
-        if (len > 0) {
-            // test different length
-            BytesArray differentLen = new BytesArray(array1, offset1, randomInt(len - 1));
-            assertFalse(BytesReference.Helper.bytesEqual(b1, differentLen));
-
-            // test changed bytes
-            array1[offset1 + randomInt(len - 1)] += 13;
-            assertFalse(BytesReference.Helper.bytesEqual(b1, b2));
-            assertFalse(BytesReference.Helper.bytesEquals(b1, b2));
-        }
-    }
-
-}
diff --git a/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java b/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java
index 5a299d82de8..6ae2b3cf943 100644
--- a/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java
+++ b/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java
@@ -50,15 +50,15 @@ public class PagedBytesReferenceTests extends AbstractBytesReferenceTestCase {
         return ref;
     }
 
-    public void testToBytesArrayMaterializedPages() throws IOException {
+    public void testToBytesRefMaterializedPages() throws IOException {
         // we need a length != (n * pagesize) to avoid page sharing at boundaries
         int length = 0;
         while ((length % PAGE_SIZE) == 0) {
             length = randomIntBetween(PAGE_SIZE, PAGE_SIZE * randomIntBetween(2, 5));
         }
         BytesReference pbr = newBytesReference(length);
-        BytesArray ba = pbr.toBytesArray();
-        BytesArray ba2 = pbr.toBytesArray();
+        BytesArray ba = new BytesArray(pbr.toBytesRef());
+        BytesArray ba2 = new BytesArray(pbr.toBytesRef());
         assertNotNull(ba);
         assertNotNull(ba2);
         assertEquals(pbr.length(), ba.length());
@@ -67,23 +67,23 @@ public class PagedBytesReferenceTests extends AbstractBytesReferenceTestCase {
         assertNotSame(ba.array(), ba2.array());
     }
 
-    public void testArray() throws IOException {
+    public void testSinglePage() throws IOException {
         int[] sizes = {0, randomInt(PAGE_SIZE), PAGE_SIZE, randomIntBetween(2, PAGE_SIZE * randomIntBetween(2, 5))};
 
         for (int i = 0; i < sizes.length; i++) {
             BytesReference pbr = newBytesReference(sizes[i]);
             // verify that array() is cheap for small payloads
             if (sizes[i] <= PAGE_SIZE) {
-                byte[] array = pbr.array();
+                BytesRef page = getSinglePageOrNull(pbr);
+                assertNotNull(page);
+                byte[] array = page.bytes;
                 assertNotNull(array);
                 assertEquals(sizes[i], array.length);
-                assertSame(array, pbr.array());
+                assertSame(array, page.bytes);
             } else {
-                try {
-                    pbr.array();
-                    fail("expected IllegalStateException");
-                } catch (IllegalStateException isx) {
-                    // expected
+                BytesRef page = getSinglePageOrNull(pbr);
+                if (pbr.length() > 0) {
+                    assertNull(page);
                 }
             }
         }
@@ -94,22 +94,42 @@ public class PagedBytesReferenceTests extends AbstractBytesReferenceTestCase {
 
         for (int i = 0; i < sizes.length; i++) {
             BytesReference pbr = newBytesReference(sizes[i]);
-            byte[] bytes = pbr.toBytes();
+            byte[] bytes = BytesReference.toBytes(pbr);
             assertEquals(sizes[i], bytes.length);
             // verify that toBytes() is cheap for small payloads
             if (sizes[i] <= PAGE_SIZE) {
-                assertSame(bytes, pbr.toBytes());
+                assertSame(bytes, BytesReference.toBytes(pbr));
             } else {
-                assertNotSame(bytes, pbr.toBytes());
+                assertNotSame(bytes, BytesReference.toBytes(pbr));
             }
         }
     }
 
-    public void testHasArray() throws IOException {
+    public void testHasSinglePage() throws IOException {
         int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(1, 3));
         BytesReference pbr = newBytesReference(length);
         // must return true for <= pagesize
-        assertEquals(length <= PAGE_SIZE, pbr.hasArray());
+        assertEquals(length <= PAGE_SIZE, getNumPages(pbr) == 1);
+    }
+
+    public void testEquals() {
+        int length = randomIntBetween(100, PAGE_SIZE * randomIntBetween(2, 5));
+        ByteArray ba1 = bigarrays.newByteArray(length, false);
+        ByteArray ba2 = bigarrays.newByteArray(length, false);
+
+        // copy contents
+        for (long i = 0; i < length; i++) {
+            ba2.set(i, ba1.get(i));
+        }
+
+        // get refs & compare
+        BytesReference pbr = new PagedBytesReference(bigarrays, ba1, length);
+        BytesReference pbr2 = new PagedBytesReference(bigarrays, ba2, length);
+        assertEquals(pbr, pbr2);
+        int offsetToFlip = randomIntBetween(0, length - 1);
+        int value = ~Byte.toUnsignedInt(ba1.get(offsetToFlip));
+        ba2.set(offsetToFlip, (byte)value);
+        assertNotEquals(pbr, pbr2);
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/common/compress/DeflateCompressedXContentTests.java b/core/src/test/java/org/elasticsearch/common/compress/DeflateCompressedXContentTests.java
index 72866d082ae..0ce95077965 100644
--- a/core/src/test/java/org/elasticsearch/common/compress/DeflateCompressedXContentTests.java
+++ b/core/src/test/java/org/elasticsearch/common/compress/DeflateCompressedXContentTests.java
@@ -91,8 +91,8 @@ public class DeflateCompressedXContentTests extends ESTestCase {
         // of different size are being used
         assertFalse(b1.equals(b2));
         // we used the compressed representation directly and did not recompress
-        assertArrayEquals(b1.toBytes(), new CompressedXContent(b1).compressed());
-        assertArrayEquals(b2.toBytes(), new CompressedXContent(b2).compressed());
+        assertArrayEquals(BytesReference.toBytes(b1), new CompressedXContent(b1).compressed());
+        assertArrayEquals(BytesReference.toBytes(b2), new CompressedXContent(b2).compressed());
         // but compressedstring instances are still equal
         assertEquals(new CompressedXContent(b1), new CompressedXContent(b2));
     }
diff --git a/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java b/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java
index 407c9790dbe..416299f8e7e 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java
@@ -46,7 +46,7 @@ public class GeoDistanceTests extends ESTestCase {
         GeoDistance geoDistance = randomFrom(GeoDistance.PLANE, GeoDistance.FACTOR, GeoDistance.ARC, GeoDistance.SLOPPY_ARC);
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             geoDistance.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {;
+            try (StreamInput in = out.bytes().streamInput()) {;
                 GeoDistance copy = GeoDistance.readFromStream(in);
                 assertEquals(copy.toString() + " vs. " + geoDistance.toString(), copy, geoDistance);
             }
@@ -60,7 +60,7 @@ public class GeoDistanceTests extends ESTestCase {
             } else {
                 out.writeVInt(randomIntBetween(Integer.MIN_VALUE, -1));
             }
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 GeoDistance.readFromStream(in);
             } catch (IOException e) {
                 assertThat(e.getMessage(), containsString("Unknown GeoDistance ordinal ["));
diff --git a/core/src/test/java/org/elasticsearch/common/geo/ShapeRelationTests.java b/core/src/test/java/org/elasticsearch/common/geo/ShapeRelationTests.java
index 6ee6a4fff83..e4eaa17874c 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/ShapeRelationTests.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/ShapeRelationTests.java
@@ -39,21 +39,21 @@ public class ShapeRelationTests extends ESTestCase {
     public void testwriteTo() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             ShapeRelation.INTERSECTS.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(0));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             ShapeRelation.DISJOINT.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(1));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             ShapeRelation.WITHIN.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(2));
             }
         }
@@ -62,19 +62,19 @@ public class ShapeRelationTests extends ESTestCase {
     public void testReadFrom() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(0);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(ShapeRelation.readFromStream(in), equalTo(ShapeRelation.INTERSECTS));
             }
         }
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(1);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(ShapeRelation.readFromStream(in), equalTo(ShapeRelation.DISJOINT));
             }
         }
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(2);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(ShapeRelation.readFromStream(in), equalTo(ShapeRelation.WITHIN));
             }
         }
@@ -83,7 +83,7 @@ public class ShapeRelationTests extends ESTestCase {
     public void testInvalidReadFrom() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(randomIntBetween(3, Integer.MAX_VALUE));
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 ShapeRelation.readFromStream(in);
                 fail("Expected IOException");
             } catch(IOException e) {
diff --git a/core/src/test/java/org/elasticsearch/common/geo/SpatialStrategyTests.java b/core/src/test/java/org/elasticsearch/common/geo/SpatialStrategyTests.java
index c2f29e6ecd7..b6eae97932f 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/SpatialStrategyTests.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/SpatialStrategyTests.java
@@ -38,14 +38,14 @@ public class SpatialStrategyTests extends ESTestCase {
     public void testwriteTo() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             SpatialStrategy.TERM.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(0));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             SpatialStrategy.RECURSIVE.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(1));
             }
         }
@@ -54,13 +54,13 @@ public class SpatialStrategyTests extends ESTestCase {
     public void testReadFrom() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(0);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(SpatialStrategy.readFromStream(in), equalTo(SpatialStrategy.TERM));
             }
         }
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(1);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(SpatialStrategy.readFromStream(in), equalTo(SpatialStrategy.RECURSIVE));
             }
         }
@@ -69,7 +69,7 @@ public class SpatialStrategyTests extends ESTestCase {
     public void testInvalidReadFrom() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(randomIntBetween(2, Integer.MAX_VALUE));
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 SpatialStrategy.readFromStream(in);
                 fail("Expected IOException");
             } catch(IOException e) {
diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java b/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java
index 9cbd4bb769d..4003a96e26f 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java
@@ -137,7 +137,7 @@ public abstract class AbstractShapeBuilderTestCase<SB extends ShapeBuilder> exte
     static ShapeBuilder copyShape(ShapeBuilder original) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             original.writeTo(output);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
                 return namedWriteableRegistry.getReader(ShapeBuilder.class, original.getWriteableName()).read(in);
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/common/io/StreamsTests.java b/core/src/test/java/org/elasticsearch/common/io/StreamsTests.java
index 5c6c1e1789b..76b52c08a85 100644
--- a/core/src/test/java/org/elasticsearch/common/io/StreamsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/io/StreamsTests.java
@@ -84,7 +84,7 @@ public class StreamsTests extends ESTestCase {
         byte stuff[] = new byte[] { 0, 1, 2, 3 };
         BytesRef stuffRef = new BytesRef(stuff, 2, 2);
         BytesArray stuffArray = new BytesArray(stuffRef);
-        StreamInput input = StreamInput.wrap(stuffArray);
+        StreamInput input = stuffArray.streamInput();
         assertEquals(2, input.read());
         assertEquals(3, input.read());
         assertEquals(-1, input.read());
diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/AbstractWriteableEnumTestCase.java b/core/src/test/java/org/elasticsearch/common/io/stream/AbstractWriteableEnumTestCase.java
index a4d15173a7c..dc57b0c70d4 100644
--- a/core/src/test/java/org/elasticsearch/common/io/stream/AbstractWriteableEnumTestCase.java
+++ b/core/src/test/java/org/elasticsearch/common/io/stream/AbstractWriteableEnumTestCase.java
@@ -60,7 +60,7 @@ public abstract class AbstractWriteableEnumTestCase extends ESTestCase {
     protected static void assertWriteToStream(final Writeable writeableEnum, final int ordinal) throws IOException {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             writeableEnum.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(ordinal));
             }
         }
@@ -70,7 +70,7 @@ public abstract class AbstractWriteableEnumTestCase extends ESTestCase {
     protected void assertReadFromStream(final int ordinal, final Writeable expected) throws IOException {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(ordinal);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(reader.read(in), equalTo(expected));
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
index 9fcbb708156..e8a52379a11 100644
--- a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.common.io.stream;
 
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.util.BigArrays;
@@ -48,7 +49,7 @@ public class BytesStreamsTests extends ESTestCase {
 
         // test empty stream to array
         assertEquals(0, out.size());
-        assertEquals(0, out.bytes().toBytes().length);
+        assertEquals(0, out.bytes().length());
 
         out.close();
     }
@@ -63,7 +64,7 @@ public class BytesStreamsTests extends ESTestCase {
         // write single byte
         out.writeByte(expectedData[0]);
         assertEquals(expectedSize, out.size());
-        assertArrayEquals(expectedData, out.bytes().toBytes());
+        assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes()));
 
         out.close();
     }
@@ -80,7 +81,7 @@ public class BytesStreamsTests extends ESTestCase {
         }
 
         assertEquals(expectedSize, out.size());
-        assertArrayEquals(expectedData, out.bytes().toBytes());
+        assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes()));
 
         out.close();
     }
@@ -108,14 +109,14 @@ public class BytesStreamsTests extends ESTestCase {
         byte[] expectedData = randomizedByteArrayWithSize(expectedSize);
         out.writeBytes(expectedData);
         assertEquals(expectedSize, out.size());
-        assertArrayEquals(expectedData, out.bytes().toBytes());
+        assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes()));
 
         // bulk-write again with actual bytes
         expectedSize = 10;
         expectedData = randomizedByteArrayWithSize(expectedSize);
         out.writeBytes(expectedData);
         assertEquals(expectedSize, out.size());
-        assertArrayEquals(expectedData, out.bytes().toBytes());
+        assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes()));
 
         out.close();
     }
@@ -130,7 +131,7 @@ public class BytesStreamsTests extends ESTestCase {
         out.writeBytes(expectedData);
 
         assertEquals(expectedSize, out.size());
-        assertArrayEquals(expectedData, out.bytes().toBytes());
+        assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes()));
 
         out.close();
     }
@@ -149,7 +150,7 @@ public class BytesStreamsTests extends ESTestCase {
         // now write the rest - more than fits into the remaining first page
         out.writeBytes(expectedData, initialOffset, additionalLength);
         assertEquals(expectedData.length, out.size());
-        assertArrayEquals(expectedData, out.bytes().toBytes());
+        assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes()));
 
         out.close();
     }
@@ -168,7 +169,7 @@ public class BytesStreamsTests extends ESTestCase {
         // ie. we cross over into a third
         out.writeBytes(expectedData, initialOffset, additionalLength);
         assertEquals(expectedData.length, out.size());
-        assertArrayEquals(expectedData, out.bytes().toBytes());
+        assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes()));
 
         out.close();
     }
@@ -185,7 +186,7 @@ public class BytesStreamsTests extends ESTestCase {
         }
 
         assertEquals(expectedSize, out.size());
-        assertArrayEquals(expectedData, out.bytes().toBytes());
+        assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes()));
 
         out.close();
     }
@@ -202,7 +203,7 @@ public class BytesStreamsTests extends ESTestCase {
         }
 
         assertEquals(expectedSize, out.size());
-        assertArrayEquals(expectedData, out.bytes().toBytes());
+        assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes()));
 
         out.close();
     }
@@ -219,7 +220,7 @@ public class BytesStreamsTests extends ESTestCase {
         }
 
         assertEquals(expectedSize, out.size());
-        assertArrayEquals(expectedData, out.bytes().toBytes());
+        assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes()));
 
         out.close();
     }
@@ -235,7 +236,7 @@ public class BytesStreamsTests extends ESTestCase {
         out.seek(position += BigArrays.BYTE_PAGE_SIZE + 10);
         out.seek(position += BigArrays.BYTE_PAGE_SIZE * 2);
         assertEquals(position, out.position());
-        assertEquals(position, out.bytes().toBytes().length);
+        assertEquals(position, BytesReference.toBytes(out.bytes()).length);
 
         out.close();
     }
@@ -288,8 +289,8 @@ public class BytesStreamsTests extends ESTestCase {
         out.writeTimeZone(DateTimeZone.forID("CET"));
         out.writeOptionalTimeZone(DateTimeZone.getDefault());
         out.writeOptionalTimeZone(null);
-        final byte[] bytes = out.bytes().toBytes();
-        StreamInput in = StreamInput.wrap(out.bytes().toBytes());
+        final byte[] bytes = BytesReference.toBytes(out.bytes());
+        StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes()));
         assertEquals(in.available(), bytes.length);
         assertThat(in.readBoolean(), equalTo(false));
         assertThat(in.readByte(), equalTo((byte)1));
@@ -328,7 +329,7 @@ public class BytesStreamsTests extends ESTestCase {
         namedWriteableRegistry.register(BaseNamedWriteable.class, TestNamedWriteable.NAME, TestNamedWriteable::new);
         TestNamedWriteable namedWriteableIn = new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10));
         out.writeNamedWriteable(namedWriteableIn);
-        byte[] bytes = out.bytes().toBytes();
+        byte[] bytes = BytesReference.toBytes(out.bytes());
         StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry);
         assertEquals(in.available(), bytes.length);
         BaseNamedWriteable namedWriteableOut = in.readNamedWriteable(BaseNamedWriteable.class);
@@ -348,7 +349,7 @@ public class BytesStreamsTests extends ESTestCase {
     public void testNamedWriteableUnknownCategory() throws IOException {
         BytesStreamOutput out = new BytesStreamOutput();
         out.writeNamedWriteable(new TestNamedWriteable("test1", "test2"));
-        StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes().toBytes()), new NamedWriteableRegistry());
+        StreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), new NamedWriteableRegistry());
         //no named writeable registered with given name, can write but cannot read it back
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> in.readNamedWriteable(BaseNamedWriteable.class));
         assertThat(e.getMessage(), equalTo("unknown named writeable category [" + BaseNamedWriteable.class.getName() + "]"));
@@ -368,7 +369,7 @@ public class BytesStreamsTests extends ESTestCase {
             public void writeTo(StreamOutput out) throws IOException {
             }
         });
-        StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes().toBytes()), namedWriteableRegistry);
+        StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(BytesReference.toBytes(out.bytes())), namedWriteableRegistry);
         try {
             //no named writeable registered with given name under test category, can write but cannot read it back
             in.readNamedWriteable(BaseNamedWriteable.class);
@@ -382,7 +383,7 @@ public class BytesStreamsTests extends ESTestCase {
         BytesStreamOutput out = new BytesStreamOutput();
         TestNamedWriteable testNamedWriteable = new TestNamedWriteable("test1", "test2");
         out.writeNamedWriteable(testNamedWriteable);
-        StreamInput in = StreamInput.wrap(out.bytes().toBytes());
+        StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes()));
         try {
             in.readNamedWriteable(BaseNamedWriteable.class);
             fail("Expected UnsupportedOperationException");
@@ -397,7 +398,7 @@ public class BytesStreamsTests extends ESTestCase {
         namedWriteableRegistry.register(BaseNamedWriteable.class, TestNamedWriteable.NAME, (StreamInput in) -> null);
         TestNamedWriteable namedWriteableIn = new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10));
         out.writeNamedWriteable(namedWriteableIn);
-        byte[] bytes = out.bytes().toBytes();
+        byte[] bytes = BytesReference.toBytes(out.bytes());
         StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry);
         assertEquals(in.available(), bytes.length);
         IOException e = expectThrows(IOException.class, () -> in.readNamedWriteable(BaseNamedWriteable.class));
@@ -407,7 +408,7 @@ public class BytesStreamsTests extends ESTestCase {
     public void testOptionalWriteableReaderReturnsNull() throws IOException {
         BytesStreamOutput out = new BytesStreamOutput();
         out.writeOptionalWriteable(new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)));
-        StreamInput in = StreamInput.wrap(out.bytes().toBytes());
+        StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes()));
         IOException e = expectThrows(IOException.class, () -> in.readOptionalWriteable((StreamInput ignored) -> null));
         assertThat(e.getMessage(), endsWith("] returned null which is not allowed and probably means it screwed up the stream."));
     }
@@ -423,7 +424,7 @@ public class BytesStreamsTests extends ESTestCase {
         });
         TestNamedWriteable namedWriteableIn = new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10));
         out.writeNamedWriteable(namedWriteableIn);
-        byte[] bytes = out.bytes().toBytes();
+        byte[] bytes = BytesReference.toBytes(out.bytes());
         StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry);
         assertEquals(in.available(), bytes.length);
         AssertionError e = expectThrows(AssertionError.class, () -> in.readNamedWriteable(BaseNamedWriteable.class));
@@ -442,7 +443,7 @@ public class BytesStreamsTests extends ESTestCase {
         final BytesStreamOutput out = new BytesStreamOutput();
         out.writeStreamableList(expected);
 
-        final StreamInput in = StreamInput.wrap(out.bytes().toBytes());
+        final StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes()));
 
         List<TestStreamable> loaded = in.readStreamableList(TestStreamable::new);
 
@@ -537,7 +538,7 @@ public class BytesStreamsTests extends ESTestCase {
 
         // toByteArray() must fail
         try {
-            out.bytes().toBytes();
+            BytesReference.toBytes(out.bytes());
             fail("expected IllegalStateException: stream closed");
         }
         catch (IllegalStateException iex1) {
@@ -558,7 +559,7 @@ public class BytesStreamsTests extends ESTestCase {
             BytesStreamOutput out = new BytesStreamOutput();
             GeoPoint geoPoint = new GeoPoint(randomDouble(), randomDouble());
             out.writeGenericValue(geoPoint);
-            StreamInput wrap = StreamInput.wrap(out.bytes());
+            StreamInput wrap = out.bytes().streamInput();
             GeoPoint point = (GeoPoint) wrap.readGenericValue();
             assertEquals(point, geoPoint);
         }
@@ -566,7 +567,7 @@ public class BytesStreamsTests extends ESTestCase {
             BytesStreamOutput out = new BytesStreamOutput();
             GeoPoint geoPoint = new GeoPoint(randomDouble(), randomDouble());
             out.writeGeoPoint(geoPoint);
-            StreamInput wrap = StreamInput.wrap(out.bytes());
+            StreamInput wrap = out.bytes().streamInput();
             GeoPoint point = wrap.readGeoPoint();
             assertEquals(point, geoPoint);
         }
diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
index aa6016774b0..baea1d7a951 100644
--- a/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
+++ b/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
@@ -19,13 +19,13 @@
 
 package org.elasticsearch.common.io.stream;
 
-import org.elasticsearch.common.bytes.ByteBufferBytesReference;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
-import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -61,8 +61,8 @@ public class StreamTests extends ESTestCase {
         for (Tuple<Long, byte[]> value : values) {
             BytesStreamOutput out = new BytesStreamOutput();
             out.writeZLong(value.v1());
-            assertArrayEquals(Long.toString(value.v1()), value.v2(), out.bytes().toBytes());
-            ByteBufferBytesReference bytes = new ByteBufferBytesReference(ByteBuffer.wrap(value.v2()));
+            assertArrayEquals(Long.toString(value.v1()), value.v2(), BytesReference.toBytes(out.bytes()));
+            BytesReference bytes = new BytesArray(value.v2());
             assertEquals(Arrays.toString(value.v2()), (long)value.v1(), bytes.streamInput().readZLong());
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/common/transport/BoundTransportAddressTests.java b/core/src/test/java/org/elasticsearch/common/transport/BoundTransportAddressTests.java
index 45db5a33d21..1a3fa4db137 100644
--- a/core/src/test/java/org/elasticsearch/common/transport/BoundTransportAddressTests.java
+++ b/core/src/test/java/org/elasticsearch/common/transport/BoundTransportAddressTests.java
@@ -51,7 +51,7 @@ public class BoundTransportAddressTests extends ESTestCase {
         // serialize
         BytesStreamOutput streamOutput = new BytesStreamOutput();
         transportAddress.writeTo(streamOutput);
-        StreamInput in = ByteBufferStreamInput.wrap(streamOutput.bytes());
+        StreamInput in = streamOutput.bytes().streamInput();
 
         BoundTransportAddress serializedAddress;
         if (randomBoolean()) {
diff --git a/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java b/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java
index f9a4d3f22af..7c5463baed2 100644
--- a/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java
@@ -82,7 +82,7 @@ public class DistanceUnitTests extends ESTestCase {
         for (DistanceUnit unit : DistanceUnit.values()) {
           try (BytesStreamOutput out = new BytesStreamOutput()) {
               unit.writeTo(out);
-              try (StreamInput in = StreamInput.wrap(out.bytes())) {
+              try (StreamInput in = out.bytes().streamInput()) {
                   assertThat("Roundtrip serialisation failed.", DistanceUnit.readFromStream(in), equalTo(unit));
               }
           }
diff --git a/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java b/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java
index 2b5a7c00e5d..3f6f1848fd8 100644
--- a/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java
+++ b/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java
@@ -145,7 +145,7 @@ public class FuzzinessTests extends ESTestCase {
     private static Fuzziness doSerializeRoundtrip(Fuzziness in) throws IOException {
         BytesStreamOutput output = new BytesStreamOutput();
         in.writeTo(output);
-        StreamInput streamInput = StreamInput.wrap(output.bytes());
+        StreamInput streamInput = output.bytes().streamInput();
         return new Fuzziness(streamInput);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
index 78afc9e514f..003d78ce42e 100644
--- a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
+++ b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java
@@ -161,7 +161,7 @@ public class TimeValueTests extends ESTestCase {
         value.writeTo(out);
         assertEquals(expectedSize, out.size());
 
-        StreamInput in = StreamInput.wrap(out.bytes());
+        StreamInput in = out.bytes().streamInput();
         TimeValue inValue = new TimeValue(in);
 
         assertThat(inValue, equalTo(value));
diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
index 1a582d48f6b..d6797d4be26 100644
--- a/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
@@ -154,7 +154,7 @@ public class ThreadContextTests extends ESTestCase {
             assertNull(threadContext.getTransient("ctx.foo"));
             assertEquals("1", threadContext.getHeader("default"));
 
-            threadContext.readHeaders(StreamInput.wrap(out.bytes()));
+            threadContext.readHeaders(out.bytes().streamInput());
             assertEquals("bar", threadContext.getHeader("foo"));
             assertNull(threadContext.getTransient("ctx.foo"));
         }
@@ -179,14 +179,14 @@ public class ThreadContextTests extends ESTestCase {
         {
             Settings otherSettings = Settings.builder().put("request.headers.default", "5").build();
             ThreadContext otherhreadContext = new ThreadContext(otherSettings);
-            otherhreadContext.readHeaders(StreamInput.wrap(out.bytes()));
+            otherhreadContext.readHeaders(out.bytes().streamInput());
 
             assertEquals("bar", otherhreadContext.getHeader("foo"));
             assertNull(otherhreadContext.getTransient("ctx.foo"));
             assertEquals("1", otherhreadContext.getHeader("default"));
         }
     }
-    
+
     public void testSerializeInDifferentContextNoDefaults() throws IOException {
         BytesStreamOutput out = new BytesStreamOutput();
         {
@@ -202,7 +202,7 @@ public class ThreadContextTests extends ESTestCase {
         {
             Settings otherSettings = Settings.builder().put("request.headers.default", "5").build();
             ThreadContext otherhreadContext = new ThreadContext(otherSettings);
-            otherhreadContext.readHeaders(StreamInput.wrap(out.bytes()));
+            otherhreadContext.readHeaders(out.bytes().streamInput());
 
             assertEquals("bar", otherhreadContext.getHeader("foo"));
             assertNull(otherhreadContext.getTransient("ctx.foo"));
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java
index 583234461b3..8319873878a 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java
@@ -57,11 +57,10 @@ public class XContentFactoryTests extends ESTestCase {
         builder.endObject();
 
         assertThat(XContentFactory.xContentType(builder.bytes()), equalTo(type));
-        BytesArray bytesArray = builder.bytes().toBytesArray();
-        assertThat(XContentFactory.xContentType(StreamInput.wrap(bytesArray.array(), bytesArray.arrayOffset(), bytesArray.length())), equalTo(type));
+        assertThat(XContentFactory.xContentType(builder.bytes().streamInput()), equalTo(type));
 
         // CBOR is binary, cannot use String
-        if (type != XContentType.CBOR) {
+        if (type != XContentType.CBOR && type != XContentType.SMILE) {
             assertThat(XContentFactory.xContentType(builder.string()), equalTo(type));
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java
index 34944e713bd..fe69fc1f05d 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java
@@ -94,7 +94,7 @@ public class XContentBuilderTests extends ESTestCase {
             xContentBuilder.startObject();
             xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}"));
             xContentBuilder.endObject();
-            assertThat(xContentBuilder.bytes().toUtf8(), equalTo("{\"foo\":{\"test\":\"value\"}}"));
+            assertThat(xContentBuilder.bytes().utf8ToString(), equalTo("{\"foo\":{\"test\":\"value\"}}"));
         }
         {
             XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON);
@@ -102,7 +102,7 @@ public class XContentBuilderTests extends ESTestCase {
             xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}"));
             xContentBuilder.rawField("foo1", new BytesArray("{\"test\":\"value\"}"));
             xContentBuilder.endObject();
-            assertThat(xContentBuilder.bytes().toUtf8(), equalTo("{\"foo\":{\"test\":\"value\"},\"foo1\":{\"test\":\"value\"}}"));
+            assertThat(xContentBuilder.bytes().utf8ToString(), equalTo("{\"foo\":{\"test\":\"value\"},\"foo1\":{\"test\":\"value\"}}"));
         }
         {
             XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON);
@@ -110,7 +110,7 @@ public class XContentBuilderTests extends ESTestCase {
             xContentBuilder.field("test", "value");
             xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}"));
             xContentBuilder.endObject();
-            assertThat(xContentBuilder.bytes().toUtf8(), equalTo("{\"test\":\"value\",\"foo\":{\"test\":\"value\"}}"));
+            assertThat(xContentBuilder.bytes().utf8ToString(), equalTo("{\"test\":\"value\",\"foo\":{\"test\":\"value\"}}"));
         }
         {
             XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON);
@@ -119,7 +119,7 @@ public class XContentBuilderTests extends ESTestCase {
             xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}"));
             xContentBuilder.field("test1", "value1");
             xContentBuilder.endObject();
-            assertThat(xContentBuilder.bytes().toUtf8(), equalTo("{\"test\":\"value\",\"foo\":{\"test\":\"value\"},\"test1\":\"value1\"}"));
+            assertThat(xContentBuilder.bytes().utf8ToString(), equalTo("{\"test\":\"value\",\"foo\":{\"test\":\"value\"},\"test1\":\"value1\"}"));
         }
         {
             XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON);
@@ -129,7 +129,7 @@ public class XContentBuilderTests extends ESTestCase {
             xContentBuilder.rawField("foo1", new BytesArray("{\"test\":\"value\"}"));
             xContentBuilder.field("test1", "value1");
             xContentBuilder.endObject();
-            assertThat(xContentBuilder.bytes().toUtf8(), equalTo("{\"test\":\"value\",\"foo\":{\"test\":\"value\"},\"foo1\":{\"test\":\"value\"},\"test1\":\"value1\"}"));
+            assertThat(xContentBuilder.bytes().utf8ToString(), equalTo("{\"test\":\"value\",\"foo\":{\"test\":\"value\"},\"foo1\":{\"test\":\"value\"},\"test1\":\"value1\"}"));
         }
     }
 
@@ -161,15 +161,14 @@ public class XContentBuilderTests extends ESTestCase {
         gen.writeEndObject();
         gen.close();
 
-        byte[] data = bos.bytes().toBytes();
-        String sData = new String(data, "UTF8");
+        String sData = bos.bytes().utf8ToString();
         assertThat(sData, equalTo("{\"name\":\"something\", source : { test : \"value\" },\"name2\":\"something2\"}"));
     }
 
     public void testByteConversion() throws Exception {
         XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
         builder.startObject().field("test_name", (Byte)(byte)120).endObject();
-        assertThat(builder.bytes().toUtf8(), equalTo("{\"test_name\":120}"));
+        assertThat(builder.bytes().utf8ToString(), equalTo("{\"test_name\":120}"));
     }
 
     public void testDateTypesConversion() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java
index bf2dd442b64..efbca114aac 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java
@@ -63,7 +63,8 @@ public class JsonVsCborTests extends ESTestCase {
         xsonGen.close();
         jsonGen.close();
 
-        verifySameTokens(XContentFactory.xContent(XContentType.JSON).createParser(jsonOs.bytes().toBytes()), XContentFactory.xContent(XContentType.CBOR).createParser(xsonOs.bytes().toBytes()));
+        verifySameTokens(XContentFactory.xContent(XContentType.JSON).createParser(jsonOs.bytes()),
+            XContentFactory.xContent(XContentType.CBOR).createParser(xsonOs.bytes()));
     }
 
     private void verifySameTokens(XContentParser parser1, XContentParser parser2) throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/smile/JsonVsSmileTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/smile/JsonVsSmileTests.java
index 9e686fe78f1..63b19a63822 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/smile/JsonVsSmileTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/smile/JsonVsSmileTests.java
@@ -63,7 +63,8 @@ public class JsonVsSmileTests extends ESTestCase {
         xsonGen.close();
         jsonGen.close();
 
-        verifySameTokens(XContentFactory.xContent(XContentType.JSON).createParser(jsonOs.bytes().toBytes()), XContentFactory.xContent(XContentType.SMILE).createParser(xsonOs.bytes().toBytes()));
+        verifySameTokens(XContentFactory.xContent(XContentType.JSON).createParser(jsonOs.bytes()),
+            XContentFactory.xContent(XContentType.SMILE).createParser(xsonOs.bytes()));
     }
 
     private void verifySameTokens(XContentParser parser1, XContentParser parser2) throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java
index e3d8735e05e..b8b38a543f6 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java
@@ -27,7 +27,6 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.test.ESTestCase;
 
-import java.io.ByteArrayInputStream;
 import java.io.IOException;
 
 import static org.hamcrest.CoreMatchers.is;
@@ -45,7 +44,7 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
         assertNotNull(expected);
 
         // Verify that the result is equal to the expected string
-        assertThat(builder.bytes().toUtf8(), is(expected.bytes().toUtf8()));
+        assertThat(builder.bytes().utf8ToString(), is(expected.bytes().utf8ToString()));
     }
 
     protected void assertBinary(XContentBuilder expected, XContentBuilder builder) {
@@ -1166,15 +1165,15 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase
 
         // Test method: rawField(String fieldName, InputStream content)
         assertXContentBuilder(expectedRawField,
-                newXContentBuilder().startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject());
+                newXContentBuilder().startObject().field("foo", 0).rawField("raw", raw.streamInput()).endObject());
         assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilder("f*", true).startObject().field("foo", 0)
-                .rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject());
+                .rawField("raw", raw.streamInput()).endObject());
         assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilder("r*", false).startObject().field("foo", 0)
-                .rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject());
+                .rawField("raw", raw.streamInput()).endObject());
         assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("r*", true).startObject().field("foo", 0)
-                .rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject());
+                .rawField("raw", raw.streamInput()).endObject());
         assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("f*", false).startObject().field("foo", 0)
-                .rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject());
+                .rawField("raw", raw.streamInput()).endObject());
     }
 
     public void testArrays() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java
index dd2fe42eb8e..8dbefedb249 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java
@@ -142,7 +142,7 @@ public class FilterPathGeneratorFilteringTests extends ESTestCase {
                     }
                 }
             }
-            assertThat(os.bytes().toUtf8(), equalTo(replaceQuotes(expected)));
+            assertThat(os.bytes().utf8ToString(), equalTo(replaceQuotes(expected)));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/deps/jackson/JacksonLocationTests.java b/core/src/test/java/org/elasticsearch/deps/jackson/JacksonLocationTests.java
index 4efedd9154a..c25a0a6503b 100644
--- a/core/src/test/java/org/elasticsearch/deps/jackson/JacksonLocationTests.java
+++ b/core/src/test/java/org/elasticsearch/deps/jackson/JacksonLocationTests.java
@@ -56,8 +56,7 @@ public class JacksonLocationTests extends ESTestCase {
 
         gen.close();
 
-        byte[] data = os.bytes().toBytes();
-        JsonParser parser = new JsonFactory().createParser(data);
+        JsonParser parser = new JsonFactory().createParser(os.bytes().streamInput());
 
         assertThat(parser.nextToken(), equalTo(JsonToken.START_OBJECT));
         assertThat(parser.nextToken(), equalTo(JsonToken.FIELD_NAME)); // "index"
diff --git a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
index ce0c78c1c92..828336c1603 100644
--- a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
+++ b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
@@ -20,7 +20,6 @@ package org.elasticsearch.http;
 
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.breaker.CircuitBreaker;
-import org.elasticsearch.common.bytes.ByteBufferBytesReference;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
@@ -202,7 +201,7 @@ public class HttpServerTests extends ESTestCase {
 
         private TestRestRequest(String path, String content) {
             this.path = path;
-            this.content = new ByteBufferBytesReference(ByteBuffer.wrap(content.getBytes(StandardCharsets.UTF_8)));
+            this.content = new BytesArray(content);
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java b/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java
index c41051ec59c..f95e8408a87 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java
@@ -22,6 +22,7 @@ package org.elasticsearch.index;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
@@ -43,7 +44,7 @@ public class IndexRequestBuilderIT extends ESIntegTestCase {
                 client().prepareIndex("test", "test").setSource("{\"test_field\" : \"foobar\"}"),
                 client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}")),
                 client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}")),
-                client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}").toBytes()),
+                client().prepareIndex("test", "test").setSource(BytesReference.toBytes(new BytesArray("{\"test_field\" : \"foobar\"}"))),
                 client().prepareIndex("test", "test").setSource(map)
         };
         indexRandom(true, builders);
diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index a0396b7abc6..0ecf8462651 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -539,7 +539,7 @@ public class InternalEngineTests extends ESTestCase {
 
     public void testCommitStats() {
         Document document = testDocumentWithTextField();
-        document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE));
+        document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
         ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null);
         engine.index(new Engine.Index(newUid("1"), doc));
 
@@ -716,7 +716,7 @@ public class InternalEngineTests extends ESTestCase {
 
         // create a document
         Document document = testDocumentWithTextField();
-        document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE));
+        document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
         ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null);
         engine.index(new Engine.Index(newUid("1"), doc));
 
@@ -729,7 +729,7 @@ public class InternalEngineTests extends ESTestCase {
         // but, we can still get it (in realtime)
         Engine.GetResult getResult = engine.get(new Engine.Get(true, newUid("1")));
         assertThat(getResult.exists(), equalTo(true));
-        assertThat(getResult.source().source.toBytesArray(), equalTo(B_1.toBytesArray()));
+        assertThat(getResult.source().source, equalTo(B_1));
         assertThat(getResult.docIdAndVersion(), nullValue());
         getResult.release();
 
@@ -755,7 +755,7 @@ public class InternalEngineTests extends ESTestCase {
         // now do an update
         document = testDocument();
         document.add(new TextField("value", "test1", Field.Store.YES));
-        document.add(new Field(SourceFieldMapper.NAME, B_2.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE));
+        document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_2), SourceFieldMapper.Defaults.FIELD_TYPE));
         doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_2, null);
         engine.index(new Engine.Index(newUid("1"), doc));
 
@@ -769,7 +769,7 @@ public class InternalEngineTests extends ESTestCase {
         // but, we can still get it (in realtime)
         getResult = engine.get(new Engine.Get(true, newUid("1")));
         assertThat(getResult.exists(), equalTo(true));
-        assertThat(getResult.source().source.toBytesArray(), equalTo(B_2.toBytesArray()));
+        assertThat(getResult.source().source, equalTo(B_2));
         assertThat(getResult.docIdAndVersion(), nullValue());
         getResult.release();
 
@@ -808,7 +808,7 @@ public class InternalEngineTests extends ESTestCase {
 
         // add it back
         document = testDocumentWithTextField();
-        document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE));
+        document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
         doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null);
         engine.index(new Engine.Index(newUid("1"), doc, Versions.MATCH_DELETED));
 
diff --git a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
index ef443d1e102..39112ed602e 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
@@ -500,7 +500,7 @@ public class ShadowEngineTests extends ESTestCase {
     public void testShadowEngineIgnoresWriteOperations() throws Exception {
         // create a document
         ParseContext.Document document = testDocumentWithTextField();
-        document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE));
+        document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
         ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null);
         try {
             replicaEngine.index(new Engine.Index(newUid("1"), doc));
@@ -538,7 +538,7 @@ public class ShadowEngineTests extends ESTestCase {
 
         // Now, add a document to the primary so we can test shadow engine deletes
         document = testDocumentWithTextField();
-        document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE));
+        document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
         doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null);
         primaryEngine.index(new Engine.Index(newUid("1"), doc));
         primaryEngine.flush();
@@ -593,7 +593,7 @@ public class ShadowEngineTests extends ESTestCase {
 
         // create a document
         ParseContext.Document document = testDocumentWithTextField();
-        document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE));
+        document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
         ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null);
         primaryEngine.index(new Engine.Index(newUid("1"), doc));
 
@@ -612,7 +612,7 @@ public class ShadowEngineTests extends ESTestCase {
         // but, we can still get it (in realtime)
         Engine.GetResult getResult = primaryEngine.get(new Engine.Get(true, newUid("1")));
         assertThat(getResult.exists(), equalTo(true));
-        assertThat(getResult.source().source.toBytesArray(), equalTo(B_1.toBytesArray()));
+        assertThat(getResult.source().source, equalTo(B_1));
         assertThat(getResult.docIdAndVersion(), nullValue());
         getResult.release();
 
@@ -649,7 +649,7 @@ public class ShadowEngineTests extends ESTestCase {
         // now do an update
         document = testDocument();
         document.add(new TextField("value", "test1", Field.Store.YES));
-        document.add(new Field(SourceFieldMapper.NAME, B_2.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE));
+        document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_2), SourceFieldMapper.Defaults.FIELD_TYPE));
         doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_2, null);
         primaryEngine.index(new Engine.Index(newUid("1"), doc));
 
@@ -663,7 +663,7 @@ public class ShadowEngineTests extends ESTestCase {
         // but, we can still get it (in realtime)
         getResult = primaryEngine.get(new Engine.Get(true, newUid("1")));
         assertThat(getResult.exists(), equalTo(true));
-        assertThat(getResult.source().source.toBytesArray(), equalTo(B_2.toBytesArray()));
+        assertThat(getResult.source().source, equalTo(B_2));
         assertThat(getResult.docIdAndVersion(), nullValue());
         getResult.release();
 
@@ -720,7 +720,7 @@ public class ShadowEngineTests extends ESTestCase {
 
         // add it back
         document = testDocumentWithTextField();
-        document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE));
+        document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
         doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null);
         primaryEngine.index(new Engine.Index(newUid("1"), doc));
 
@@ -971,7 +971,7 @@ public class ShadowEngineTests extends ESTestCase {
 
         // create a document
         ParseContext.Document document = testDocumentWithTextField();
-        document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE));
+        document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE));
         ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null);
         pEngine.index(new Engine.Index(newUid("1"), doc));
         pEngine.flush(true, true);
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java
index 165b49d3145..817dc6e50df 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java
@@ -24,6 +24,7 @@ import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -292,17 +293,17 @@ public class SimpleAllMapperTests extends ESSingleNodeTestCase {
         }
 
         DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
-        String mapping = mappingBuilder.endObject().endObject().bytes().toUtf8();
+        String mapping = mappingBuilder.endObject().endObject().bytes().utf8ToString();
         logger.info("Mapping: {}", mapping);
         DocumentMapper docMapper = parser.parse("test", new CompressedXContent(mapping));
         String builtMapping = docMapper.mappingSource().string();
         // reparse it
         DocumentMapper builtDocMapper = parser.parse("test", new CompressedXContent(builtMapping));
 
-        byte[] json = jsonBuilder().startObject()
+        byte[] json = BytesReference.toBytes(jsonBuilder().startObject()
                 .field("foo", "bar")
                 .field("foobar", "foobar")
-                .endObject().bytes().toBytes();
+                .endObject().bytes());
         Document doc = builtDocMapper.parse("test", "test", "1", new BytesArray(json)).rootDoc();
         IndexableField[] fields = doc.getFields("_all");
         if (enabled) {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/binary/BinaryMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/binary/BinaryMappingTests.java
index fc8e2ba1872..4bf1d0c68f7 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/binary/BinaryMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/binary/BinaryMappingTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper.binary;
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.compress.CompressorFactory;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
@@ -79,7 +80,7 @@ public class BinaryMappingTests extends ESSingleNodeTestCase {
         try (StreamOutput compressed = CompressorFactory.COMPRESSOR.streamOutput(out)) {
             new BytesArray(binaryValue1).writeTo(compressed);
         }
-        final byte[] binaryValue2 = out.bytes().toBytes();
+        final byte[] binaryValue2 = BytesReference.toBytes(out.bytes());
         assertTrue(CompressorFactory.isCompressed(new BytesArray(binaryValue2)));
 
         for (byte[] value : Arrays.asList(binaryValue1, binaryValue2)) {
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
index 78da5abb746..8f38e2be576 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
@@ -297,7 +297,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
             out.close();
             BytesReference bytes = out.bytes();
 
-            MappingMetaData metaData = MappingMetaData.PROTO.readFrom(StreamInput.wrap(bytes));
+            MappingMetaData metaData = MappingMetaData.PROTO.readFrom(bytes.streamInput());
 
             assertThat(metaData, is(expected));
         }
@@ -314,7 +314,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
             out.close();
             BytesReference bytes = out.bytes();
 
-            MappingMetaData metaData = MappingMetaData.PROTO.readFrom(StreamInput.wrap(bytes));
+            MappingMetaData metaData = MappingMetaData.PROTO.readFrom(bytes.streamInput());
 
             assertThat(metaData, is(expected));
         }
@@ -331,7 +331,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase {
             out.close();
             BytesReference bytes = out.bytes();
 
-            MappingMetaData metaData = MappingMetaData.PROTO.readFrom(StreamInput.wrap(bytes));
+            MappingMetaData metaData = MappingMetaData.PROTO.readFrom(bytes.streamInput());
 
             assertThat(metaData, is(expected));
         }
diff --git a/core/src/test/java/org/elasticsearch/index/query/CombineFunctionTests.java b/core/src/test/java/org/elasticsearch/index/query/CombineFunctionTests.java
index 695330c21e2..667efbc8bac 100644
--- a/core/src/test/java/org/elasticsearch/index/query/CombineFunctionTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/CombineFunctionTests.java
@@ -40,41 +40,41 @@ public class CombineFunctionTests extends ESTestCase {
     public void testWriteTo() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             CombineFunction.MULTIPLY.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(0));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             CombineFunction.REPLACE.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(1));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             CombineFunction.SUM.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(2));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             CombineFunction.AVG.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(3));
             }
         }
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             CombineFunction.MIN.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(4));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             CombineFunction.MAX.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(5));
             }
         }
@@ -83,37 +83,37 @@ public class CombineFunctionTests extends ESTestCase {
     public void testReadFrom() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(0);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(CombineFunction.readFromStream(in), equalTo(CombineFunction.MULTIPLY));
             }
         }
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(1);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(CombineFunction.readFromStream(in), equalTo(CombineFunction.REPLACE));
             }
         }
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(2);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(CombineFunction.readFromStream(in), equalTo(CombineFunction.SUM));
             }
         }
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(3);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(CombineFunction.readFromStream(in), equalTo(CombineFunction.AVG));
             }
         }
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(4);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(CombineFunction.readFromStream(in), equalTo(CombineFunction.MIN));
             }
         }
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(5);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(CombineFunction.readFromStream(in), equalTo(CombineFunction.MAX));
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
index 6460d8505ee..090aa906456 100644
--- a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
@@ -381,7 +381,7 @@ public class InnerHitBuilderTests extends ESTestCase {
     private static InnerHitBuilder serializedCopy(InnerHitBuilder original) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             original.writeTo(output);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
                 return new InnerHitBuilder(in);
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
index 91a42d70809..3c5bfed86dd 100644
--- a/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java
@@ -33,7 +33,6 @@ import org.elasticsearch.action.termvectors.TermVectorsRequest;
 import org.elasticsearch.action.termvectors.TermVectorsResponse;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.lucene.search.MoreLikeThisQuery;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -208,7 +207,7 @@ public class MoreLikeThisQueryBuilderTests extends AbstractQueryTestCase<MoreLik
                 response.setExists(true);
                 Fields generatedFields;
                 if (request.doc() != null) {
-                    generatedFields = generateFields(randomFields, request.doc().toUtf8());
+                    generatedFields = generateFields(randomFields, request.doc().utf8ToString());
                 } else {
                     generatedFields = generateFields(request.selectedFields().toArray(new String[request.selectedFields().size()]), request.id());
                 }
@@ -292,7 +291,7 @@ public class MoreLikeThisQueryBuilderTests extends AbstractQueryTestCase<MoreLik
         Item expectedItem = generateRandomItem();
         BytesStreamOutput output = new BytesStreamOutput();
         expectedItem.writeTo(output);
-        Item newItem = new Item(StreamInput.wrap(output.bytes()));
+        Item newItem = new Item(output.bytes().streamInput());
         assertEquals(expectedItem, newItem);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/query/OperatorTests.java b/core/src/test/java/org/elasticsearch/index/query/OperatorTests.java
index 65f1b413bb6..4e7c6753ea6 100644
--- a/core/src/test/java/org/elasticsearch/index/query/OperatorTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/OperatorTests.java
@@ -37,14 +37,14 @@ public class OperatorTests extends ESTestCase {
     public void testWriteTo() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             Operator.OR.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(0));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             Operator.AND.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(1));
             }
         }
@@ -53,13 +53,13 @@ public class OperatorTests extends ESTestCase {
     public void testReadFrom() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(0);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(Operator.readFromStream(in), equalTo(Operator.OR));
             }
         }
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(1);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(Operator.readFromStream(in), equalTo(Operator.AND));
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/index/query/ScoreModeTests.java b/core/src/test/java/org/elasticsearch/index/query/ScoreModeTests.java
index 0b346d59f8b..d3298a15878 100644
--- a/core/src/test/java/org/elasticsearch/index/query/ScoreModeTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/ScoreModeTests.java
@@ -40,41 +40,41 @@ public class ScoreModeTests extends ESTestCase {
     public void testWriteTo() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             FiltersFunctionScoreQuery.ScoreMode.FIRST.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(0));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             FiltersFunctionScoreQuery.ScoreMode.AVG.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(1));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             FiltersFunctionScoreQuery.ScoreMode.MAX.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(2));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             FiltersFunctionScoreQuery.ScoreMode.SUM.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(3));
             }
         }
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             FiltersFunctionScoreQuery.ScoreMode.MIN.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(4));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             FiltersFunctionScoreQuery.ScoreMode.MULTIPLY.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(5));
             }
         }
@@ -83,37 +83,37 @@ public class ScoreModeTests extends ESTestCase {
     public void testReadFrom() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(0);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(FiltersFunctionScoreQuery.ScoreMode.readFromStream(in), equalTo(FiltersFunctionScoreQuery.ScoreMode.FIRST));
             }
         }
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(1);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(FiltersFunctionScoreQuery.ScoreMode.readFromStream(in), equalTo(FiltersFunctionScoreQuery.ScoreMode.AVG));
             }
         }
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(2);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(FiltersFunctionScoreQuery.ScoreMode.readFromStream(in), equalTo(FiltersFunctionScoreQuery.ScoreMode.MAX));
             }
         }
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(3);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(FiltersFunctionScoreQuery.ScoreMode.readFromStream(in), equalTo(FiltersFunctionScoreQuery.ScoreMode.SUM));
             }
         }
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(4);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(FiltersFunctionScoreQuery.ScoreMode.readFromStream(in), equalTo(FiltersFunctionScoreQuery.ScoreMode.MIN));
             }
         }
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(5);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(FiltersFunctionScoreQuery.ScoreMode.readFromStream(in), equalTo(FiltersFunctionScoreQuery.ScoreMode.MULTIPLY));
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/index/query/WrapperQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/WrapperQueryBuilderTests.java
index 4612aedb2c3..492b27f4957 100644
--- a/core/src/test/java/org/elasticsearch/index/query/WrapperQueryBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/WrapperQueryBuilderTests.java
@@ -45,7 +45,7 @@ public class WrapperQueryBuilderTests extends AbstractQueryTestCase<WrapperQuery
             case 0:
                 return new WrapperQueryBuilder(wrappedQuery.toString());
             case 1:
-                return new WrapperQueryBuilder(((ToXContentToBytes)wrappedQuery).buildAsBytes().toBytes());
+                return new WrapperQueryBuilder(BytesReference.toBytes(((ToXContentToBytes)wrappedQuery).buildAsBytes()));
             case 2:
                 return new WrapperQueryBuilder(((ToXContentToBytes)wrappedQuery).buildAsBytes());
             default:
diff --git a/core/src/test/java/org/elasticsearch/index/query/functionscore/FieldValueFactorFunctionModifierTests.java b/core/src/test/java/org/elasticsearch/index/query/functionscore/FieldValueFactorFunctionModifierTests.java
index d9a4d6ab7a8..cd5f20de7f2 100644
--- a/core/src/test/java/org/elasticsearch/index/query/functionscore/FieldValueFactorFunctionModifierTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/functionscore/FieldValueFactorFunctionModifierTests.java
@@ -44,70 +44,70 @@ public class FieldValueFactorFunctionModifierTests extends ESTestCase {
     public void testWriteTo() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             FieldValueFactorFunction.Modifier.NONE.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(0));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             FieldValueFactorFunction.Modifier.LOG.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(1));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             FieldValueFactorFunction.Modifier.LOG1P.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(2));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             FieldValueFactorFunction.Modifier.LOG2P.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(3));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             FieldValueFactorFunction.Modifier.LN.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(4));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             FieldValueFactorFunction.Modifier.LN1P.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(5));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             FieldValueFactorFunction.Modifier.LN2P.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(6));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             FieldValueFactorFunction.Modifier.SQUARE.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(7));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             FieldValueFactorFunction.Modifier.SQRT.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(8));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             FieldValueFactorFunction.Modifier.RECIPROCAL.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(9));
             }
         }
@@ -116,70 +116,70 @@ public class FieldValueFactorFunctionModifierTests extends ESTestCase {
     public void testReadFrom() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(0);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(FieldValueFactorFunction.Modifier.readFromStream(in), equalTo(FieldValueFactorFunction.Modifier.NONE));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(1);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(FieldValueFactorFunction.Modifier.readFromStream(in), equalTo(FieldValueFactorFunction.Modifier.LOG));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(2);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(FieldValueFactorFunction.Modifier.readFromStream(in), equalTo(FieldValueFactorFunction.Modifier.LOG1P));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(3);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(FieldValueFactorFunction.Modifier.readFromStream(in), equalTo(FieldValueFactorFunction.Modifier.LOG2P));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(4);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(FieldValueFactorFunction.Modifier.readFromStream(in), equalTo(FieldValueFactorFunction.Modifier.LN));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(5);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(FieldValueFactorFunction.Modifier.readFromStream(in), equalTo(FieldValueFactorFunction.Modifier.LN1P));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(6);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(FieldValueFactorFunction.Modifier.readFromStream(in), equalTo(FieldValueFactorFunction.Modifier.LN2P));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(7);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(FieldValueFactorFunction.Modifier.readFromStream(in), equalTo(FieldValueFactorFunction.Modifier.SQUARE));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(8);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(FieldValueFactorFunction.Modifier.readFromStream(in), equalTo(FieldValueFactorFunction.Modifier.SQRT));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(9);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(FieldValueFactorFunction.Modifier.readFromStream(in), equalTo(FieldValueFactorFunction.Modifier.RECIPROCAL));
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 40a23ee66cf..862be713030 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -653,7 +653,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         if (randomBoolean() || true) { // try to serialize it to ensure values survive the serialization
             BytesStreamOutput out = new BytesStreamOutput();
             stats.writeTo(out);
-            StreamInput in = StreamInput.wrap(out.bytes());
+            StreamInput in = out.bytes().streamInput();
             stats = ShardStats.readShardStats(in);
         }
         XContentBuilder builder = XContentFactory.jsonBuilder();
@@ -1442,7 +1442,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
         newShard.markAsRecovering("for testing", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.REPLICA, localNode, localNode));
         List<Translog.Operation> operations = new ArrayList<>();
-        operations.add(new Translog.Index("testtype", "1", jsonBuilder().startObject().field("foo", "bar").endObject().bytes().toBytes()));
+        operations.add(new Translog.Index("testtype", "1", BytesReference.toBytes(jsonBuilder().startObject().field("foo", "bar").endObject().bytes())));
         newShard.prepareForIndexRecovery();
         newShard.recoveryState().getTranslog().totalOperations(operations.size());
         newShard.skipTranslogRecovery();
@@ -1500,7 +1500,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         // Shard is still inactive since we haven't started recovering yet
         assertFalse(newShard.isActive());
         List<Translog.Operation> operations = new ArrayList<>();
-        operations.add(new Translog.Index("testtype", "1", jsonBuilder().startObject().field("foo", "bar").endObject().bytes().toBytes()));
+        operations.add(new Translog.Index("testtype", "1", BytesReference.toBytes(jsonBuilder().startObject().field("foo", "bar").endObject().bytes())));
         newShard.prepareForIndexRecovery();
         newShard.skipTranslogRecovery();
         // Shard is still inactive since we haven't started recovering yet
diff --git a/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java b/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java
index 70eacaafedb..abaebb88c5e 100644
--- a/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java
+++ b/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java
@@ -21,6 +21,7 @@ package org.elasticsearch.index.snapshots.blobstore;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Version;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -52,7 +53,7 @@ public class FileInfoTests extends ESTestCase {
             BlobStoreIndexShardSnapshot.FileInfo info = new BlobStoreIndexShardSnapshot.FileInfo("_foobar", meta, size);
             XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint();
             BlobStoreIndexShardSnapshot.FileInfo.toXContent(info, builder, ToXContent.EMPTY_PARAMS);
-            byte[] xcontent = shuffleXContent(builder).bytes().toBytes();
+            byte[] xcontent = BytesReference.toBytes(shuffleXContent(builder).bytes());
 
             final BlobStoreIndexShardSnapshot.FileInfo parsedInfo;
             try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(xcontent)) {
@@ -111,7 +112,7 @@ public class FileInfoTests extends ESTestCase {
             builder.field(FileInfo.WRITTEN_BY, Version.LATEST.toString());
             builder.field(FileInfo.CHECKSUM, "666");
             builder.endObject();
-            byte[] xContent = builder.bytes().toBytes();
+            byte[] xContent = BytesReference.toBytes(builder.bytes());
 
             if (failure == null) {
                 // No failures should read as usual
diff --git a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
index 6508336d9f8..84d50c6620f 100644
--- a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
+++ b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
@@ -22,6 +22,7 @@ import com.carrotsearch.hppc.cursors.IntObjectCursor;
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import org.apache.lucene.index.CheckIndex;
 import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
@@ -210,7 +211,7 @@ public class CorruptedFileIT extends ESIntegTestCase {
                             out.flush();
                             CheckIndex.Status status = checkIndex.checkIndex();
                             if (!status.clean) {
-                                logger.warn("check index [failure]\n{}", new String(os.bytes().toBytes(), StandardCharsets.UTF_8));
+                                logger.warn("check index [failure]\n{}", os.bytes().utf8ToString());
                                 throw new IOException("index check failure");
                             }
                         }
@@ -346,7 +347,7 @@ public class CorruptedFileIT extends ESIntegTestCase {
                 public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException {
                     if (corrupt.get() && action.equals(RecoveryTargetService.Actions.FILE_CHUNK)) {
                         RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request;
-                        byte[] array = req.content().array();
+                        byte[] array = BytesRef.deepCopyOf(req.content().toBytesRef()).bytes;
                         int i = randomIntBetween(0, req.content().length() - 1);
                         array[i] = (byte) ~array[i]; // flip one byte in the content
                         hasCorrupted.countDown();
@@ -419,10 +420,12 @@ public class CorruptedFileIT extends ESIntegTestCase {
                     if (action.equals(RecoveryTargetService.Actions.FILE_CHUNK)) {
                         RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request;
                         if (truncate && req.length() > 1) {
-                            BytesArray array = new BytesArray(req.content().array(), req.content().arrayOffset(), (int) req.length() - 1);
+                            BytesRef bytesRef = req.content().toBytesRef();
+                            BytesArray array = new BytesArray(bytesRef.bytes, bytesRef.offset, (int) req.length() - 1);
                             request = new RecoveryFileChunkRequest(req.recoveryId(), req.shardId(), req.metadata(), req.position(), array, req.lastChunk(), req.totalTranslogOps(), req.sourceThrottleTimeInNanos());
                         } else {
-                            byte[] array = req.content().array();
+                            assert req.content().toBytesRef().bytes == req.content().toBytesRef().bytes : "no internal reference!!";
+                            final byte[] array = req.content().toBytesRef().bytes;
                             int i = randomIntBetween(0, req.content().length() - 1);
                             array[i] = (byte) ~array[i]; // flip one byte in the content
                         }
diff --git a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
index c72a6ecef27..01eead9c96b 100644
--- a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
+++ b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
@@ -60,9 +60,7 @@ import java.nio.charset.Charset;
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.Files;
 import java.nio.file.InvalidPathException;
-import java.nio.file.OpenOption;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -217,24 +215,24 @@ public class TranslogTests extends ESTestCase {
         Translog.Location loc2 = translog.add(new Translog.Index("test", "2", new byte[]{2}));
         assertThat(loc2, greaterThan(loc1));
         assertThat(translog.getLastWriteLocation(), greaterThan(loc2));
-        assertThat(translog.read(loc1).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{1})));
-        assertThat(translog.read(loc2).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{2})));
+        assertThat(translog.read(loc1).getSource().source, equalTo(new BytesArray(new byte[]{1})));
+        assertThat(translog.read(loc2).getSource().source, equalTo(new BytesArray(new byte[]{2})));
 
         Translog.Location lastLocBeforeSync = translog.getLastWriteLocation();
         translog.sync();
         assertEquals(lastLocBeforeSync, translog.getLastWriteLocation());
-        assertThat(translog.read(loc1).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{1})));
-        assertThat(translog.read(loc2).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{2})));
+        assertThat(translog.read(loc1).getSource().source, equalTo(new BytesArray(new byte[]{1})));
+        assertThat(translog.read(loc2).getSource().source, equalTo(new BytesArray(new byte[]{2})));
 
         Translog.Location loc3 = translog.add(new Translog.Index("test", "2", new byte[]{3}));
         assertThat(loc3, greaterThan(loc2));
         assertThat(translog.getLastWriteLocation(), greaterThan(loc3));
-        assertThat(translog.read(loc3).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{3})));
+        assertThat(translog.read(loc3).getSource().source, equalTo(new BytesArray(new byte[]{3})));
 
         lastLocBeforeSync = translog.getLastWriteLocation();
         translog.sync();
         assertEquals(lastLocBeforeSync, translog.getLastWriteLocation());
-        assertThat(translog.read(loc3).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{3})));
+        assertThat(translog.read(loc3).getSource().source, equalTo(new BytesArray(new byte[]{3})));
         translog.prepareCommit();
         /*
          * The commit adds to the lastWriteLocation even though is isn't really a write. This is just an implementation artifact but it can
@@ -242,7 +240,7 @@ public class TranslogTests extends ESTestCase {
          * and less than the location of the next write operation.
          */
         assertThat(translog.getLastWriteLocation(), greaterThan(lastLocBeforeSync));
-        assertThat(translog.read(loc3).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{3})));
+        assertThat(translog.read(loc3).getSource().source, equalTo(new BytesArray(new byte[]{3})));
         translog.commit();
         assertNull(translog.read(loc1));
         assertNull(translog.read(loc2));
@@ -274,7 +272,7 @@ public class TranslogTests extends ESTestCase {
 
         Translog.Index index = (Translog.Index) snapshot.next();
         assertThat(index != null, equalTo(true));
-        assertThat(index.source().toBytes(), equalTo(new byte[]{1}));
+        assertThat(BytesReference.toBytes(index.source()), equalTo(new byte[]{1}));
 
         Translog.Delete delete = (Translog.Delete) snapshot.next();
         assertThat(delete != null, equalTo(true));
@@ -303,7 +301,7 @@ public class TranslogTests extends ESTestCase {
         if (randomBoolean()) {
             BytesStreamOutput out = new BytesStreamOutput();
             stats.writeTo(out);
-            StreamInput in = StreamInput.wrap(out.bytes());
+            StreamInput in = out.bytes().streamInput();
             stats = new TranslogStats();
             stats.readFrom(in);
         }
@@ -350,7 +348,7 @@ public class TranslogTests extends ESTestCase {
         BytesStreamOutput out = new BytesStreamOutput();
         total.writeTo(out);
         TranslogStats copy = new TranslogStats();
-        copy.readFrom(StreamInput.wrap(out.bytes()));
+        copy.readFrom(out.bytes().streamInput());
 
         assertEquals(6, copy.estimatedNumberOfOperations());
         assertEquals(431, copy.getTranslogSizeInBytes());
@@ -827,7 +825,7 @@ public class TranslogTests extends ESTestCase {
 
         assertEquals(max.generation, translog.currentFileGeneration());
         final Translog.Operation read = translog.read(max);
-        assertEquals(read.getSource().source.toUtf8(), Integer.toString(count));
+        assertEquals(read.getSource().source.utf8ToString(), Integer.toString(count));
     }
 
     public static Translog.Location max(Translog.Location a, Translog.Location b) {
@@ -859,7 +857,7 @@ public class TranslogTests extends ESTestCase {
                 Translog.Location location = locations.get(op);
                 if (op <= lastSynced) {
                     final Translog.Operation read = reader.read(location);
-                    assertEquals(Integer.toString(op), read.getSource().source.toUtf8());
+                    assertEquals(Integer.toString(op), read.getSource().source.utf8ToString());
                 } else {
                     try {
                         reader.read(location);
@@ -995,7 +993,7 @@ public class TranslogTests extends ESTestCase {
                 assertEquals("expected operation" + i + " to be in the previous translog but wasn't", translog.currentFileGeneration() - 1, locations.get(i).generation);
                 Translog.Operation next = snapshot.next();
                 assertNotNull("operation " + i + " must be non-null", next);
-                assertEquals(i, Integer.parseInt(next.getSource().source.toUtf8()));
+                assertEquals(i, Integer.parseInt(next.getSource().source.utf8ToString()));
             }
         }
     }
@@ -1030,7 +1028,7 @@ public class TranslogTests extends ESTestCase {
             for (int i = 0; i < upTo; i++) {
                 Translog.Operation next = snapshot.next();
                 assertNotNull("operation " + i + " must be non-null synced: " + sync, next);
-                assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.toUtf8()));
+                assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString()));
             }
         }
         if (randomBoolean()) { // recover twice
@@ -1043,7 +1041,7 @@ public class TranslogTests extends ESTestCase {
                 for (int i = 0; i < upTo; i++) {
                     Translog.Operation next = snapshot.next();
                     assertNotNull("operation " + i + " must be non-null synced: " + sync, next);
-                    assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.toUtf8()));
+                    assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString()));
                 }
             }
         }
@@ -1084,7 +1082,7 @@ public class TranslogTests extends ESTestCase {
             for (int i = 0; i < upTo; i++) {
                 Translog.Operation next = snapshot.next();
                 assertNotNull("operation " + i + " must be non-null synced: " + sync, next);
-                assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.toUtf8()));
+                assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString()));
             }
 
         }
@@ -1099,7 +1097,7 @@ public class TranslogTests extends ESTestCase {
                 for (int i = 0; i < upTo; i++) {
                     Translog.Operation next = snapshot.next();
                     assertNotNull("operation " + i + " must be non-null synced: " + sync, next);
-                    assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.toUtf8()));
+                    assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString()));
                 }
             }
         }
@@ -1143,7 +1141,7 @@ public class TranslogTests extends ESTestCase {
             for (int i = 0; i < upTo; i++) {
                 Translog.Operation next = snapshot.next();
                 assertNotNull("operation " + i + " must be non-null synced: " + sync, next);
-                assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.toUtf8()));
+                assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString()));
             }
         }
     }
@@ -1157,7 +1155,7 @@ public class TranslogTests extends ESTestCase {
             ops.add(test);
         }
         Translog.writeOperations(out, ops);
-        final List<Translog.Operation> readOperations = Translog.readOperations(StreamInput.wrap(out.bytes()));
+        final List<Translog.Operation> readOperations = Translog.readOperations(out.bytes().streamInput());
         assertEquals(ops.size(), readOperations.size());
         assertEquals(ops, readOperations);
     }
@@ -1218,7 +1216,7 @@ public class TranslogTests extends ESTestCase {
         for (int i = firstUncommitted; i < translogOperations; i++) {
             Translog.Operation next = snapshot.next();
             assertNotNull("" + i, next);
-            assertEquals(Integer.parseInt(next.getSource().source.toUtf8()), i);
+            assertEquals(Integer.parseInt(next.getSource().source.utf8ToString()), i);
         }
         assertNull(snapshot.next());
     }
@@ -1392,7 +1390,7 @@ public class TranslogTests extends ESTestCase {
                 assertEquals("expected operation" + i + " to be in the previous translog but wasn't", tlog.currentFileGeneration() - 1, locations.get(i).generation);
                 Translog.Operation next = snapshot.next();
                 assertNotNull("operation " + i + " must be non-null", next);
-                assertEquals(i, Integer.parseInt(next.getSource().source.toUtf8()));
+                assertEquals(i, Integer.parseInt(next.getSource().source.utf8ToString()));
             }
         }
     }
@@ -1716,7 +1714,7 @@ public class TranslogTests extends ESTestCase {
             for (int i = 0; i < 1; i++) {
                 Translog.Operation next = snapshot.next();
                 assertNotNull("operation " + i + " must be non-null", next);
-                assertEquals("payload missmatch", i, Integer.parseInt(next.getSource().source.toUtf8()));
+                assertEquals("payload missmatch", i, Integer.parseInt(next.getSource().source.utf8ToString()));
             }
             tlog.add(new Translog.Index("test", "" + 1, Integer.toString(1).getBytes(Charset.forName("UTF-8"))));
         }
@@ -1727,7 +1725,7 @@ public class TranslogTests extends ESTestCase {
             for (int i = 0; i < 2; i++) {
                 Translog.Operation next = snapshot.next();
                 assertNotNull("operation " + i + " must be non-null", next);
-                assertEquals("payload missmatch", i, Integer.parseInt(next.getSource().source.toUtf8()));
+                assertEquals("payload missmatch", i, Integer.parseInt(next.getSource().source.utf8ToString()));
             }
         }
     }
@@ -1771,7 +1769,7 @@ public class TranslogTests extends ESTestCase {
             for (int i = 0; i < 1; i++) {
                 Translog.Operation next = snapshot.next();
                 assertNotNull("operation " + i + " must be non-null", next);
-                assertEquals("payload missmatch", i, Integer.parseInt(next.getSource().source.toUtf8()));
+                assertEquals("payload missmatch", i, Integer.parseInt(next.getSource().source.utf8ToString()));
             }
             tlog.add(new Translog.Index("test", "" + 1, Integer.toString(1).getBytes(Charset.forName("UTF-8"))));
         }
@@ -1870,7 +1868,7 @@ public class TranslogTests extends ESTestCase {
                 assertEquals(syncedDocs.size(), snapshot.totalOperations());
                 for (int i = 0; i < syncedDocs.size(); i++) {
                     Translog.Operation next = snapshot.next();
-                    assertEquals(syncedDocs.get(i), next.getSource().source.toUtf8());
+                    assertEquals(syncedDocs.get(i), next.getSource().source.utf8ToString());
                     assertNotNull("operation " + i + " must be non-null", next);
                 }
             }
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java
index 1cca3bb7215..d43217d9785 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java
@@ -62,7 +62,7 @@ public class IndicesRequestCacheTests extends ESTestCase {
         // initial cache
         TestEntity entity = new TestEntity(requestCacheStats, reader, indexShard, 0);
         BytesReference value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes());
-        assertEquals("foo", StreamInput.wrap(value).readString());
+        assertEquals("foo", value.streamInput().readString());
         assertEquals(0, requestCacheStats.stats().getHitCount());
         assertEquals(1, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
@@ -72,7 +72,7 @@ public class IndicesRequestCacheTests extends ESTestCase {
         // cache hit
         entity = new TestEntity(requestCacheStats, reader, indexShard, 0);
         value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes());
-        assertEquals("foo", StreamInput.wrap(value).readString());
+        assertEquals("foo", value.streamInput().readString());
         assertEquals(1, requestCacheStats.stats().getHitCount());
         assertEquals(1, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
@@ -117,7 +117,7 @@ public class IndicesRequestCacheTests extends ESTestCase {
         // initial cache
         TestEntity entity = new TestEntity(requestCacheStats, reader, indexShard, 0);
         BytesReference value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes());
-        assertEquals("foo", StreamInput.wrap(value).readString());
+        assertEquals("foo", value.streamInput().readString());
         assertEquals(0, requestCacheStats.stats().getHitCount());
         assertEquals(1, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
@@ -130,7 +130,7 @@ public class IndicesRequestCacheTests extends ESTestCase {
         // cache the second
         TestEntity secondEntity = new TestEntity(requestCacheStats, secondReader, indexShard, 0);
         value = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes());
-        assertEquals("bar", StreamInput.wrap(value).readString());
+        assertEquals("bar", value.streamInput().readString());
         assertEquals(0, requestCacheStats.stats().getHitCount());
         assertEquals(2, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
@@ -141,7 +141,7 @@ public class IndicesRequestCacheTests extends ESTestCase {
 
         secondEntity = new TestEntity(requestCacheStats, secondReader, indexShard, 0);
         value = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes());
-        assertEquals("bar", StreamInput.wrap(value).readString());
+        assertEquals("bar", value.streamInput().readString());
         assertEquals(1, requestCacheStats.stats().getHitCount());
         assertEquals(2, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
@@ -150,7 +150,7 @@ public class IndicesRequestCacheTests extends ESTestCase {
 
         entity = new TestEntity(requestCacheStats, reader, indexShard, 0);
         value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes());
-        assertEquals("foo", StreamInput.wrap(value).readString());
+        assertEquals("foo", value.streamInput().readString());
         assertEquals(2, requestCacheStats.stats().getHitCount());
         assertEquals(2, requestCacheStats.stats().getMissCount());
         assertEquals(0, requestCacheStats.stats().getEvictions());
@@ -209,9 +209,9 @@ public class IndicesRequestCacheTests extends ESTestCase {
             TestEntity secondEntity = new TestEntity(requestCacheStats, secondReader, indexShard, 0);
 
             BytesReference value1 = cache.getOrCompute(entity, reader, termQuery.buildAsBytes());
-            assertEquals("foo", StreamInput.wrap(value1).readString());
+            assertEquals("foo", value1.streamInput().readString());
             BytesReference value2 = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes());
-            assertEquals("bar", StreamInput.wrap(value2).readString());
+            assertEquals("bar", value2.streamInput().readString());
             size = requestCacheStats.stats().getMemorySize();
             IOUtils.close(reader, secondReader, writer, dir, cache);
         }
@@ -240,12 +240,12 @@ public class IndicesRequestCacheTests extends ESTestCase {
         TestEntity thirddEntity = new TestEntity(requestCacheStats, thirdReader, indexShard, 0);
 
         BytesReference value1 = cache.getOrCompute(entity, reader, termQuery.buildAsBytes());
-        assertEquals("foo", StreamInput.wrap(value1).readString());
+        assertEquals("foo", value1.streamInput().readString());
         BytesReference value2 = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes());
-        assertEquals("bar", StreamInput.wrap(value2).readString());
+        assertEquals("bar", value2.streamInput().readString());
         logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize());
         BytesReference value3 = cache.getOrCompute(thirddEntity, thirdReader, termQuery.buildAsBytes());
-        assertEquals("baz", StreamInput.wrap(value3).readString());
+        assertEquals("baz", value3.streamInput().readString());
         assertEquals(2, cache.count());
         assertEquals(1, requestCacheStats.stats().getEvictions());
         IOUtils.close(reader, secondReader, thirdReader, writer, dir, cache);
@@ -277,12 +277,12 @@ public class IndicesRequestCacheTests extends ESTestCase {
         TestEntity thirddEntity = new TestEntity(requestCacheStats, thirdReader, differentIdentity, 0);
 
         BytesReference value1 = cache.getOrCompute(entity, reader, termQuery.buildAsBytes());
-        assertEquals("foo", StreamInput.wrap(value1).readString());
+        assertEquals("foo", value1.streamInput().readString());
         BytesReference value2 = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes());
-        assertEquals("bar", StreamInput.wrap(value2).readString());
+        assertEquals("bar", value2.streamInput().readString());
         logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize());
         BytesReference value3 = cache.getOrCompute(thirddEntity, thirdReader, termQuery.buildAsBytes());
-        assertEquals("baz", StreamInput.wrap(value3).readString());
+        assertEquals("baz", value3.streamInput().readString());
         assertEquals(3, cache.count());
         final long hitCount = requestCacheStats.stats().getHitCount();
         // clear all for the indexShard Idendity even though is't still open
@@ -292,7 +292,7 @@ public class IndicesRequestCacheTests extends ESTestCase {
         // third has not been validated since it's a different identity
         value3 = cache.getOrCompute(thirddEntity, thirdReader, termQuery.buildAsBytes());
         assertEquals(hitCount + 1, requestCacheStats.stats().getHitCount());
-        assertEquals("baz", StreamInput.wrap(value3).readString());
+        assertEquals("baz", value3.streamInput().readString());
 
 
         IOUtils.close(reader, secondReader, thirdReader, writer, dir, cache);
diff --git a/core/src/test/java/org/elasticsearch/indices/TermsLookupTests.java b/core/src/test/java/org/elasticsearch/indices/TermsLookupTests.java
index 59d86ddce67..fea69133377 100644
--- a/core/src/test/java/org/elasticsearch/indices/TermsLookupTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/TermsLookupTests.java
@@ -70,7 +70,7 @@ public class TermsLookupTests extends ESTestCase {
         TermsLookup termsLookup = randomTermsLookup();
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             termsLookup.writeTo(output);
-            try (StreamInput in = StreamInput.wrap(output.bytes())) {
+            try (StreamInput in = output.bytes().streamInput()) {
                 TermsLookup deserializedLookup = new TermsLookup(in);
                 assertEquals(deserializedLookup, termsLookup);
                 assertEquals(deserializedLookup.hashCode(), termsLookup.hashCode());
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java
index 8552db2d376..964a09f839d 100644
--- a/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java
@@ -84,7 +84,7 @@ public class RecoveryTargetTests extends ESTestCase {
             BytesStreamOutput out = new BytesStreamOutput();
             source.writeTo(out);
             out.close();
-            StreamInput in = StreamInput.wrap(out.bytes());
+            StreamInput in = out.bytes().streamInput();
             T obj = deserialize(in);
             lastRead = obj;
             return obj;
diff --git a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
index 6bce95af184..8e88aff523c 100644
--- a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
@@ -647,7 +647,7 @@ public class IndexStatsIT extends ESIntegTestCase {
             flags.writeTo(out);
             out.close();
             BytesReference bytes = out.bytes();
-            CommonStatsFlags readStats = CommonStatsFlags.readCommonStatsFlags(StreamInput.wrap(bytes));
+            CommonStatsFlags readStats = CommonStatsFlags.readCommonStatsFlags(bytes.streamInput());
             for (Flag flag : values) {
                 assertThat(flags.isSet(flag), equalTo(readStats.isSet(flag)));
             }
@@ -661,7 +661,7 @@ public class IndexStatsIT extends ESIntegTestCase {
             flags.writeTo(out);
             out.close();
             BytesReference bytes = out.bytes();
-            CommonStatsFlags readStats = CommonStatsFlags.readCommonStatsFlags(StreamInput.wrap(bytes));
+            CommonStatsFlags readStats = CommonStatsFlags.readCommonStatsFlags(bytes.streamInput());
             for (Flag flag : values) {
                 assertThat(flags.isSet(flag), equalTo(readStats.isSet(flag)));
             }
diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java b/core/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java
index 119e94580ad..9974dd568a8 100644
--- a/core/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java
@@ -62,7 +62,7 @@ public class IngestStatsTests extends ESTestCase {
     private IngestStats serialize(IngestStats stats) throws IOException {
         BytesStreamOutput out = new BytesStreamOutput();
         stats.writeTo(out);
-        StreamInput in = StreamInput.wrap(out.bytes());
+        StreamInput in = out.bytes().streamInput();
         return new IngestStats(in);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
index 0d07bcf0981..ea82a17f7b3 100644
--- a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
+++ b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
@@ -70,7 +70,7 @@ public class NodeInfoStreamingTests extends ESTestCase {
         out.setVersion(version);
         nodeInfo.writeTo(out);
         out.close();
-        StreamInput in = StreamInput.wrap(out.bytes());
+        StreamInput in = out.bytes().streamInput();
         in.setVersion(version);
         NodeInfo readNodeInfo = NodeInfo.readNodeInfo(in);
         assertExpectedUnchanged(nodeInfo, readNodeInfo);
diff --git a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
index 620dfeb94c2..fd5da198ed2 100644
--- a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
+++ b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
@@ -22,6 +22,7 @@ package org.elasticsearch.recovery;
 import com.carrotsearch.hppc.IntHashSet;
 import com.carrotsearch.hppc.procedures.IntProcedure;
 import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.English;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
@@ -93,8 +94,6 @@ import static org.hamcrest.Matchers.startsWith;
 public class RelocationIT extends ESIntegTestCase {
     private final TimeValue ACCEPTABLE_RELOCATION_TIME = new TimeValue(5, TimeUnit.MINUTES);
 
-
-
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return pluginList(MockTransportService.TestPlugin.class, MockIndexEventListener.TestPlugin.class);
@@ -505,7 +504,8 @@ public class RelocationIT extends ESIntegTestCase {
                 if (chunkRequest.name().startsWith(IndexFileNames.SEGMENTS)) {
                     // corrupting the segments_N files in order to make sure future recovery re-send files
                     logger.debug("corrupting [{}] to {}. file name: [{}]", action, node, chunkRequest.name());
-                    byte[] array = chunkRequest.content().array();
+                    assert chunkRequest.content().toBytesRef().bytes == chunkRequest.content().toBytesRef().bytes : "no internal reference!!";
+                    byte[] array = chunkRequest.content().toBytesRef().bytes;
                     array[0] = (byte) ~array[0]; // flip one byte in the content
                     corruptionCount.countDown();
                 }
diff --git a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java
index aa3b11e6250..6bb1716cb0f 100644
--- a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java
@@ -36,7 +36,6 @@ import java.io.IOException;
 
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.mockito.Mockito.mock;
@@ -64,7 +63,7 @@ public class BytesRestResponseTests extends ESTestCase {
 
         Throwable t = new ElasticsearchException("an error occurred reading data", new FileNotFoundException("/foo/bar"));
         BytesRestResponse response = new BytesRestResponse(channel, t);
-        String text = response.content().toUtf8();
+        String text = response.content().utf8ToString();
         assertThat(text, containsString("ElasticsearchException[an error occurred reading data]"));
         assertThat(text, not(containsString("FileNotFoundException")));
         assertThat(text, not(containsString("/foo/bar")));
@@ -77,7 +76,7 @@ public class BytesRestResponseTests extends ESTestCase {
 
         Throwable t = new ElasticsearchException("an error occurred reading data", new FileNotFoundException("/foo/bar"));
         BytesRestResponse response = new BytesRestResponse(channel, t);
-        String text = response.content().toUtf8();
+        String text = response.content().utf8ToString();
         assertThat(text, containsString("{\"type\":\"exception\",\"reason\":\"an error occurred reading data\"}"));
         assertThat(text, containsString("{\"type\":\"file_not_found_exception\",\"reason\":\"/foo/bar\"}"));
     }
@@ -88,7 +87,7 @@ public class BytesRestResponseTests extends ESTestCase {
 
         Throwable t = new Throwable("an error occurred reading data", new FileNotFoundException("/foo/bar"));
         BytesRestResponse response = new BytesRestResponse(channel, t);
-        String text = response.content().toUtf8();
+        String text = response.content().utf8ToString();
         assertThat(text, not(containsString("Throwable[an error occurred reading data]")));
         assertThat(text, not(containsString("FileNotFoundException[/foo/bar]")));
         assertThat(text, not(containsString("error_trace")));
@@ -102,7 +101,7 @@ public class BytesRestResponseTests extends ESTestCase {
 
         Throwable t = new Throwable("an error occurred reading data", new FileNotFoundException("/foo/bar"));
         BytesRestResponse response = new BytesRestResponse(channel, t);
-        String text = response.content().toUtf8();
+        String text = response.content().utf8ToString();
         assertThat(text, containsString("\"type\":\"throwable\",\"reason\":\"an error occurred reading data\""));
         assertThat(text, containsString("{\"type\":\"file_not_found_exception\""));
         assertThat(text, containsString("\"stack_trace\":\"[an error occurred reading data]"));
@@ -114,13 +113,13 @@ public class BytesRestResponseTests extends ESTestCase {
         {
             Throwable t = new ElasticsearchException("an error occurred reading data", new FileNotFoundException("/foo/bar"));
             BytesRestResponse response = new BytesRestResponse(channel, t);
-            String text = response.content().toUtf8();
+            String text = response.content().utf8ToString();
             assertThat(text, containsString("{\"root_cause\":[{\"type\":\"exception\",\"reason\":\"an error occurred reading data\"}]"));
         }
         {
             Throwable t = new FileNotFoundException("/foo/bar");
             BytesRestResponse response = new BytesRestResponse(channel, t);
-            String text = response.content().toUtf8();
+            String text = response.content().utf8ToString();
             assertThat(text, containsString("{\"root_cause\":[{\"type\":\"file_not_found_exception\",\"reason\":\"/foo/bar\"}]"));
         }
     }
@@ -130,7 +129,7 @@ public class BytesRestResponseTests extends ESTestCase {
         RestChannel channel = new SimpleExceptionRestChannel(request);
 
         BytesRestResponse response = new BytesRestResponse(channel, null);
-        String text = response.content().toUtf8();
+        String text = response.content().utf8ToString();
         assertThat(text, containsString("\"error\":\"unknown\""));
         assertThat(text, not(containsString("error_trace")));
     }
@@ -144,7 +143,7 @@ public class BytesRestResponseTests extends ESTestCase {
                 new SearchShardTarget("node_1", new Index("foo", "_na_"), 2));
         SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed",  new ShardSearchFailure[] {failure, failure1});
         BytesRestResponse response = new BytesRestResponse(channel, new RemoteTransportException("foo", ex));
-        String text = response.content().toUtf8();
+        String text = response.content().utf8ToString();
         String expected = "{\"error\":{\"root_cause\":[{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}],\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\",\"phase\":\"search\",\"grouped\":true,\"failed_shards\":[{\"shard\":1,\"index\":\"foo\",\"node\":\"node_1\",\"reason\":{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}}]},\"status\":400}";
         assertEquals(expected.trim(), text.trim());
         String stackTrace = ExceptionsHelper.stackTrace(ex);
@@ -160,7 +159,7 @@ public class BytesRestResponseTests extends ESTestCase {
         // if we try to decode the path, this will throw an IllegalArgumentException again
         final BytesRestResponse response = new BytesRestResponse(channel, e);
         assertNotNull(response.content());
-        final String content = response.content().toUtf8();
+        final String content = response.content().utf8ToString();
         assertThat(content, containsString("\"type\":\"illegal_argument_exception\""));
         assertThat(content, containsString("\"reason\":\"partial escape sequence at end of string: %a\""));
         assertThat(content, containsString("\"status\":" + 400));
@@ -171,7 +170,7 @@ public class BytesRestResponseTests extends ESTestCase {
         final RestChannel channel = new DetailedExceptionRestChannel(request);
         final BytesRestResponse response = new BytesRestResponse(channel, new ElasticsearchException("simulated"));
         assertNotNull(response.content());
-        final String content = response.content().toUtf8();
+        final String content = response.content().utf8ToString();
         assertThat(content, containsString("\"type\":\"exception\""));
         assertThat(content, containsString("\"reason\":\"simulated\""));
         assertThat(content, containsString("\"status\":" + 500));
diff --git a/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionTests.java
index ebb7dd255aa..ffefa074df7 100644
--- a/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionTests.java
@@ -95,6 +95,6 @@ public class RestMainActionTests extends ESTestCase {
         }
         mainResponse.toXContent(responseBuilder, ToXContent.EMPTY_PARAMS);
         BytesReference xcontentBytes = responseBuilder.bytes();
-        assertTrue(BytesReference.Helper.bytesEqual(xcontentBytes, response.content()));
+        assertEquals(xcontentBytes, response.content());
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java b/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java
index a7e17785d48..3dfae8cc4f8 100644
--- a/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java
@@ -169,7 +169,7 @@ public class RestTableTests extends ESTestCase {
 
     private void assertResponse(Map<String, String> headers, String mediaType, String body) throws Exception {
         RestResponse response = assertResponseContentType(headers, mediaType);
-        assertThat(response.content().toUtf8(), equalTo(body));
+        assertThat(response.content().utf8ToString(), equalTo(body));
     }
 
     private List<String> getHeaderNames(List<RestTable.DisplayHeader> headers) {
diff --git a/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java b/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java
index 2ca255ea1a3..192f40d4b2b 100644
--- a/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java
+++ b/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java
@@ -43,13 +43,13 @@ public class DocValueFormatTests extends ESTestCase {
 
         BytesStreamOutput out = new BytesStreamOutput();
         out.writeNamedWriteable(DocValueFormat.BOOLEAN);
-        StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes()), registry);
+        StreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry);
         assertSame(DocValueFormat.BOOLEAN, in.readNamedWriteable(DocValueFormat.class));
 
         DocValueFormat.Decimal decimalFormat = new DocValueFormat.Decimal("###.##");
         out = new BytesStreamOutput();
         out.writeNamedWriteable(decimalFormat);
-        in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes()), registry);
+        in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry);
         DocValueFormat vf = in.readNamedWriteable(DocValueFormat.class);
         assertEquals(DocValueFormat.Decimal.class, vf.getClass());
         assertEquals("###.##", ((DocValueFormat.Decimal) vf).pattern);
@@ -57,7 +57,7 @@ public class DocValueFormatTests extends ESTestCase {
         DocValueFormat.DateTime dateFormat = new DocValueFormat.DateTime(Joda.forPattern("epoch_second"), DateTimeZone.forOffsetHours(1));
         out = new BytesStreamOutput();
         out.writeNamedWriteable(dateFormat);
-        in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes()), registry);
+        in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry);
         vf = in.readNamedWriteable(DocValueFormat.class);
         assertEquals(DocValueFormat.DateTime.class, vf.getClass());
         assertEquals("epoch_second", ((DocValueFormat.DateTime) vf).formatter.format());
@@ -65,17 +65,17 @@ public class DocValueFormatTests extends ESTestCase {
 
         out = new BytesStreamOutput();
         out.writeNamedWriteable(DocValueFormat.GEOHASH);
-        in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes()), registry);
+        in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry);
         assertSame(DocValueFormat.GEOHASH, in.readNamedWriteable(DocValueFormat.class));
 
         out = new BytesStreamOutput();
         out.writeNamedWriteable(DocValueFormat.IP);
-        in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes()), registry);
+        in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry);
         assertSame(DocValueFormat.IP, in.readNamedWriteable(DocValueFormat.class));
 
         out = new BytesStreamOutput();
         out.writeNamedWriteable(DocValueFormat.RAW);
-        in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes()), registry);
+        in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry);
         assertSame(DocValueFormat.RAW, in.readNamedWriteable(DocValueFormat.class));
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java b/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java
index a4837e382ac..5caba0fb441 100644
--- a/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java
+++ b/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java
@@ -753,35 +753,35 @@ public class MultiValueModeTests extends ESTestCase {
     public void testWriteTo() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             MultiValueMode.SUM.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(0));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             MultiValueMode.AVG.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(1));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             MultiValueMode.MEDIAN.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(2));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             MultiValueMode.MIN.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(3));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             MultiValueMode.MAX.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(4));
             }
         }
@@ -790,35 +790,35 @@ public class MultiValueModeTests extends ESTestCase {
     public void testReadFrom() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(0);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(MultiValueMode.readMultiValueModeFrom(in), equalTo(MultiValueMode.SUM));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(1);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(MultiValueMode.readMultiValueModeFrom(in), equalTo(MultiValueMode.AVG));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(2);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(MultiValueMode.readMultiValueModeFrom(in), equalTo(MultiValueMode.MEDIAN));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(3);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(MultiValueMode.readMultiValueModeFrom(in), equalTo(MultiValueMode.MIN));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(4);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(MultiValueMode.readMultiValueModeFrom(in), equalTo(MultiValueMode.MAX));
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
index a4103e7ee56..ddde1fd9eb6 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java
@@ -215,7 +215,7 @@ public abstract class BaseAggregationTestCase<AB extends AbstractAggregationBuil
         AB testAgg = createTestAggregatorBuilder();
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             output.writeNamedWriteable(testAgg);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
                 AggregationBuilder deserialized = in.readNamedWriteable(AggregationBuilder.class);
                 assertEquals(testAgg, deserialized);
                 assertEquals(testAgg.hashCode(), deserialized.hashCode());
@@ -255,7 +255,7 @@ public abstract class BaseAggregationTestCase<AB extends AbstractAggregationBuil
     private AB copyAggregation(AB agg) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             agg.writeTo(output);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
                 @SuppressWarnings("unchecked")
                 AB secondAgg = (AB) namedWriteableRegistry.getReader(AggregationBuilder.class, agg.getWriteableName()).read(in);
                 return secondAgg;
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
index a3f9f11dc2e..9d783505e2d 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/BasePipelineAggregationTestCase.java
@@ -146,7 +146,7 @@ public abstract class BasePipelineAggregationTestCase<AF extends AbstractPipelin
         AF testAgg = createTestAggregatorFactory();
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             output.writeNamedWriteable(testAgg);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
                 PipelineAggregationBuilder deserializedQuery = in.readNamedWriteable(PipelineAggregationBuilder.class);
                 assertEquals(deserializedQuery, testAgg);
                 assertEquals(deserializedQuery.hashCode(), testAgg.hashCode());
@@ -186,7 +186,7 @@ public abstract class BasePipelineAggregationTestCase<AF extends AbstractPipelin
     private AF copyAggregation(AF agg) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             output.writeNamedWriteable(agg);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
                 @SuppressWarnings("unchecked")
                 AF secondAgg = (AF) in.readNamedWriteable(PipelineAggregationBuilder.class);
                 return secondAgg;
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/SubAggCollectionModeTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/SubAggCollectionModeTests.java
index 5ed3fe39970..9e20d313848 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/SubAggCollectionModeTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/SubAggCollectionModeTests.java
@@ -39,14 +39,14 @@ public class SubAggCollectionModeTests extends ESTestCase {
     public void testwriteTo() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             SubAggCollectionMode.DEPTH_FIRST.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(0));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             SubAggCollectionMode.BREADTH_FIRST.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(1));
             }
         }
@@ -55,13 +55,13 @@ public class SubAggCollectionModeTests extends ESTestCase {
     public void testReadFrom() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(0);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(SubAggCollectionMode.readFromStream(in), equalTo(SubAggCollectionMode.DEPTH_FIRST));
             }
         }
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(1);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(SubAggCollectionMode.readFromStream(in), equalTo(SubAggCollectionMode.BREADTH_FIRST));
             }
         }
@@ -70,7 +70,7 @@ public class SubAggCollectionModeTests extends ESTestCase {
     public void testInvalidReadFrom() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(randomIntBetween(2, Integer.MAX_VALUE));
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 SubAggCollectionMode.readFromStream(in);
                 fail("Expected IOException");
             } catch(IOException e) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesMethodTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesMethodTests.java
index 36c4caae12d..97d5cf1f9ee 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesMethodTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesMethodTests.java
@@ -38,14 +38,14 @@ public class PercentilesMethodTests extends ESTestCase {
     public void testwriteTo() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             PercentilesMethod.TDIGEST.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(0));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             PercentilesMethod.HDR.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(1));
             }
         }
@@ -54,13 +54,13 @@ public class PercentilesMethodTests extends ESTestCase {
     public void testReadFrom() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(0);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(PercentilesMethod.readFromStream(in), equalTo(PercentilesMethod.TDIGEST));
             }
         }
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(1);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(PercentilesMethod.readFromStream(in), equalTo(PercentilesMethod.HDR));
             }
         }
@@ -69,7 +69,7 @@ public class PercentilesMethodTests extends ESTestCase {
     public void testInvalidReadFrom() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             out.writeVInt(randomIntBetween(2, Integer.MAX_VALUE));
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 PercentilesMethod.readFromStream(in);
                 fail("Expected IOException");
             } catch(IOException e) {
diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index ac90fabcb07..8ddfa374ac0 100644
--- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -444,7 +444,7 @@ public class SearchSourceBuilderTests extends ESTestCase {
         SearchSourceBuilder testBuilder = createSearchSourceBuilder();
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             testBuilder.writeTo(output);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
                 SearchSourceBuilder deserializedBuilder = new SearchSourceBuilder(in);
                 assertEquals(deserializedBuilder, testBuilder);
                 assertEquals(deserializedBuilder.hashCode(), testBuilder.hashCode());
@@ -484,7 +484,7 @@ public class SearchSourceBuilderTests extends ESTestCase {
     protected SearchSourceBuilder copyBuilder(SearchSourceBuilder builder) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             builder.writeTo(output);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
                 return new SearchSourceBuilder(in);
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java b/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java
index 930a7b220e0..1e3c5453fd7 100644
--- a/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java
@@ -484,14 +484,14 @@ public class HighlightBuilderTests extends ESTestCase {
     public void testOrderSerialization() throws Exception {
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             Order.NONE.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(0));
             }
         }
 
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             Order.SCORE.writeTo(out);
-            try (StreamInput in = StreamInput.wrap(out.bytes())) {
+            try (StreamInput in = out.bytes().streamInput()) {
                 assertThat(in.readVInt(), equalTo(1));
             }
         }
@@ -738,7 +738,7 @@ public class HighlightBuilderTests extends ESTestCase {
     private static HighlightBuilder serializedCopy(HighlightBuilder original) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             original.writeTo(output);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
                 return new HighlightBuilder(in);
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/search/internal/InternalSearchHitTests.java b/core/src/test/java/org/elasticsearch/search/internal/InternalSearchHitTests.java
index 77fc2f0e6a9..dedd47d3e43 100644
--- a/core/src/test/java/org/elasticsearch/search/internal/InternalSearchHitTests.java
+++ b/core/src/test/java/org/elasticsearch/search/internal/InternalSearchHitTests.java
@@ -67,7 +67,7 @@ public class InternalSearchHitTests extends ESTestCase {
         context.streamShardTarget(InternalSearchHits.StreamContext.ShardTargetType.STREAM);
         BytesStreamOutput output = new BytesStreamOutput();
         hits.writeTo(output, context);
-        InputStream input = new ByteArrayInputStream(output.bytes().toBytes());
+        InputStream input = output.bytes().streamInput();
         context = new InternalSearchHits.StreamContext();
         context.streamShardTarget(InternalSearchHits.StreamContext.ShardTargetType.STREAM);
         InternalSearchHits results = InternalSearchHits.readSearchHits(new InputStreamStreamInput(input), context);
diff --git a/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java b/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java
index ef7a4ecc7ce..300c4f141b0 100644
--- a/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java
+++ b/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java
@@ -362,8 +362,8 @@ public class MatchedQueriesIT extends ESIntegTestCase {
         refresh();
 
         QueryBuilder[] queries = new QueryBuilder[]{
-                wrapperQuery(matchQuery("content", "amet").queryName("abc").buildAsBytes().toUtf8()),
-                constantScoreQuery(wrapperQuery(termQuery("content", "amet").queryName("abc").buildAsBytes().toUtf8()))
+                wrapperQuery(matchQuery("content", "amet").queryName("abc").buildAsBytes().utf8ToString()),
+                constantScoreQuery(wrapperQuery(termQuery("content", "amet").queryName("abc").buildAsBytes().utf8ToString()))
         };
         for (QueryBuilder query : queries) {
             SearchResponse searchResponse = client().prepareSearch()
diff --git a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
index 7c3690dcf35..28eb56bdcaf 100644
--- a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java
@@ -340,7 +340,7 @@ public class QueryRescoreBuilderTests extends ESTestCase {
     private static RescoreBuilder<?> serializedCopy(RescoreBuilder<?> original) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             output.writeNamedWriteable(original);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
                 return in.readNamedWriteable(RescoreBuilder.class);
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java b/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java
index 3c675926328..7f9b9761feb 100644
--- a/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java
@@ -164,7 +164,7 @@ public class SearchAfterBuilderTests extends ESTestCase {
     private static SearchAfterBuilder serializedCopy(SearchAfterBuilder original) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             original.writeTo(output);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
                 return new SearchAfterBuilder(in);
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
index 217f97ace0a..a0ce8f02ea3 100644
--- a/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
@@ -99,7 +99,7 @@ public class SliceBuilderTests extends ESTestCase {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             original.writeTo(output);
             try (StreamInput in =
-                     new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
+                     new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
                 return new SliceBuilder(in);
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
index b494fa4d1e6..8d46372aa42 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java
@@ -272,7 +272,7 @@ public abstract class AbstractSortTestCase<T extends SortBuilder<T>> extends EST
     private T copyItem(T original) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             original.writeTo(output);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
                 return (T) namedWriteableRegistry.getReader(SortBuilder.class, original.getWriteableName()).read(in);
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/search/sort/SortOrderTests.java b/core/src/test/java/org/elasticsearch/search/sort/SortOrderTests.java
index 2de48decbd8..208b4ed1b53 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/SortOrderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/SortOrderTests.java
@@ -37,7 +37,7 @@ public class SortOrderTests extends ESTestCase {
         for (SortOrder unit : SortOrder.values()) {
           try (BytesStreamOutput out = new BytesStreamOutput()) {
               unit.writeTo(out);
-              try (StreamInput in = StreamInput.wrap(out.bytes())) {
+              try (StreamInput in = out.bytes().streamInput()) {
                   assertThat("Roundtrip serialisation failed.", SortOrder.readFromStream(in), equalTo(unit));
               }
           }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
index cd6c34497f7..b67036e1152 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java
@@ -215,7 +215,7 @@ public abstract class AbstractSuggestionBuilderTestCase<SB extends SuggestionBui
     protected SB serializedCopy(SB original) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             output.writeNamedWriteable(original);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
                 return (SB) in.readNamedWriteable(SuggestionBuilder.class);
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/completion/WritableTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/completion/WritableTestCase.java
index 818abf64ffc..842a2395653 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/completion/WritableTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/completion/WritableTestCase.java
@@ -103,7 +103,7 @@ public abstract class WritableTestCase<M extends Writeable> extends ESTestCase {
     private M copyModel(M original) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             original.writeTo(output);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), provideNamedWritableRegistry())) {
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), provideNamedWritableRegistry())) {
                 return readFrom(in);
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java
index 3fd3850b98a..846d3193f6d 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java
@@ -206,7 +206,7 @@ public class DirectCandidateGeneratorTests extends ESTestCase{
     private static DirectCandidateGeneratorBuilder serializedCopy(DirectCandidateGeneratorBuilder original) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             original.writeTo(output);
-            try (StreamInput in = StreamInput.wrap(output.bytes())) {
+            try (StreamInput in = output.bytes().streamInput()) {
                 return new DirectCandidateGeneratorBuilder(in);
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java
index f167eefa43d..c7b883b583b 100644
--- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java
@@ -180,7 +180,7 @@ public abstract class SmoothingModelTestCase extends ESTestCase {
     static SmoothingModel copyModel(SmoothingModel original) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             original.writeTo(output);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) {
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
                 return namedWriteableRegistry.getReader(SmoothingModel.class, original.getWriteableName()).read(in);
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
index 19c508e2bb1..15623825887 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
@@ -646,8 +646,8 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
             @Override
             public void sendResponse(RestResponse response) {
                 try {
-                    assertThat(response.content().toUtf8(), containsString("notsecretusername"));
-                    assertThat(response.content().toUtf8(), not(containsString("verysecretpassword")));
+                    assertThat(response.content().utf8ToString(), containsString("notsecretusername"));
+                    assertThat(response.content().utf8ToString(), not(containsString("verysecretpassword")));
                 } catch (AssertionError ex) {
                     getRepoError.set(ex);
                 }
@@ -667,8 +667,8 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
             @Override
             public void sendResponse(RestResponse response) {
                 try {
-                    assertThat(response.content().toUtf8(), containsString("notsecretusername"));
-                    assertThat(response.content().toUtf8(), not(containsString("verysecretpassword")));
+                    assertThat(response.content().utf8ToString(), containsString("notsecretusername"));
+                    assertThat(response.content().utf8ToString(), not(containsString("verysecretpassword")));
                 } catch (AssertionError ex) {
                     clusterStateError.set(ex);
                 }
diff --git a/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java b/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java
index 38d858c49aa..c178b2a6f83 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java
@@ -22,6 +22,7 @@ package org.elasticsearch.snapshots;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.test.ESTestCase;
 
@@ -75,7 +76,7 @@ public class SnapshotRequestsTests extends ESTestCase {
             builder.endArray();
         }
 
-        byte[] bytes = builder.endObject().bytes().toBytes();
+        byte[] bytes = BytesReference.toBytes(builder.endObject().bytes());
 
 
         request.source(bytes);
@@ -134,7 +135,7 @@ public class SnapshotRequestsTests extends ESTestCase {
             builder.endArray();
         }
 
-        byte[] bytes = builder.endObject().bytes().toBytes();
+        byte[] bytes = BytesReference.toBytes(builder.endObject().bytes());
 
 
         request.source(bytes);
diff --git a/core/src/test/java/org/elasticsearch/snapshots/SnapshotTests.java b/core/src/test/java/org/elasticsearch/snapshots/SnapshotTests.java
index cb297785e4b..41cfa3d4141 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/SnapshotTests.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/SnapshotTests.java
@@ -20,12 +20,10 @@
 package org.elasticsearch.snapshots;
 
 import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
-import java.nio.ByteBuffer;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 
@@ -50,8 +48,7 @@ public class SnapshotTests extends ESTestCase {
         final Snapshot original = new Snapshot(randomAsciiOfLength(randomIntBetween(2, 8)), snapshotId);
         final BytesStreamOutput out = new BytesStreamOutput();
         original.writeTo(out);
-        final ByteBufferStreamInput in = new ByteBufferStreamInput(ByteBuffer.wrap(out.bytes().toBytes()));
-        assertThat(new Snapshot(in), equalTo(original));
+        assertThat(new Snapshot(out.bytes().streamInput()), equalTo(original));
     }
 
 }
diff --git a/core/src/test/java/org/elasticsearch/tasks/PersistedTaskInfoTests.java b/core/src/test/java/org/elasticsearch/tasks/PersistedTaskInfoTests.java
index bfbb2dff4c7..5b507436129 100644
--- a/core/src/test/java/org/elasticsearch/tasks/PersistedTaskInfoTests.java
+++ b/core/src/test/java/org/elasticsearch/tasks/PersistedTaskInfoTests.java
@@ -37,7 +37,7 @@ import java.util.Map;
 import java.util.TreeMap;
 
 /**
- * Round trip tests for {@link PersistedTaskInfo} and those classes that it includes like {@link TaskInfo} and {@link RawTaskStatus}. 
+ * Round trip tests for {@link PersistedTaskInfo} and those classes that it includes like {@link TaskInfo} and {@link RawTaskStatus}.
  */
 public class PersistedTaskInfoTests extends ESTestCase {
     public void testBinaryRoundTrip() throws IOException {
@@ -47,7 +47,7 @@ public class PersistedTaskInfoTests extends ESTestCase {
         PersistedTaskInfo read;
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             result.writeTo(out);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes()), registry)) {
+            try (StreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry)) {
                 read = new PersistedTaskInfo(in);
             }
         } catch (IOException e) {
diff --git a/core/src/test/java/org/elasticsearch/tasks/TaskIdTests.java b/core/src/test/java/org/elasticsearch/tasks/TaskIdTests.java
index b13de26b976..f7990cfacb7 100644
--- a/core/src/test/java/org/elasticsearch/tasks/TaskIdTests.java
+++ b/core/src/test/java/org/elasticsearch/tasks/TaskIdTests.java
@@ -57,7 +57,7 @@ public class TaskIdTests extends ESTestCase {
             taskId.writeTo(out);
             BytesReference bytes = out.bytes();
             assertEquals(expectedSize, bytes.length());
-            try (StreamInput in = StreamInput.wrap(bytes)) {
+            try (StreamInput in = bytes.streamInput()) {
                 return TaskId.readFromStream(in);
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java
index 486b0635c64..14cf10b8f31 100644
--- a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java
+++ b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java
@@ -58,7 +58,7 @@ public class ThreadPoolSerializationTests extends ESTestCase {
         output.setVersion(Version.CURRENT);
         info.writeTo(output);
 
-        StreamInput input = StreamInput.wrap(output.bytes());
+        StreamInput input = output.bytes().streamInput();
         ThreadPool.Info newInfo = new ThreadPool.Info();
         newInfo.readFrom(input);
 
@@ -70,7 +70,7 @@ public class ThreadPoolSerializationTests extends ESTestCase {
         output.setVersion(Version.CURRENT);
         info.writeTo(output);
 
-        StreamInput input = StreamInput.wrap(output.bytes());
+        StreamInput input = output.bytes().streamInput();
         ThreadPool.Info newInfo = new ThreadPool.Info();
         newInfo.readFrom(input);
 
@@ -125,7 +125,7 @@ public class ThreadPoolSerializationTests extends ESTestCase {
         output.setVersion(Version.CURRENT);
         info.writeTo(output);
 
-        StreamInput input = StreamInput.wrap(output.bytes());
+        StreamInput input = output.bytes().streamInput();
         ThreadPool.Info newInfo = new ThreadPool.Info();
         newInfo.readFrom(input);
 
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/ChannelBufferBytesReferenceTests.java b/core/src/test/java/org/elasticsearch/transport/netty/ChannelBufferBytesReferenceTests.java
index a284f6ea911..bc10458378f 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/ChannelBufferBytesReferenceTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/ChannelBufferBytesReferenceTests.java
@@ -18,11 +18,10 @@
  */
 package org.elasticsearch.transport.netty;
 
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.bytes.AbstractBytesReferenceTestCase;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
-import org.elasticsearch.transport.netty.NettyUtils;
 import org.jboss.netty.buffer.ChannelBuffer;
 import org.jboss.netty.buffer.ChannelBuffers;
 
@@ -38,17 +37,16 @@ public class ChannelBufferBytesReferenceTests extends AbstractBytesReferenceTest
         assertEquals(out.size(), length);
         BytesReference ref = out.bytes();
         assertEquals(ref.length(), length);
-        BytesArray bytesArray = ref.toBytesArray();
-        return NettyUtils.toBytesReference(ChannelBuffers.wrappedBuffer(bytesArray.array(), bytesArray.arrayOffset(),
-            bytesArray.length()));
+        BytesRef bytesRef = ref.toBytesRef();
+        final ChannelBuffer channelBuffer = ChannelBuffers.wrappedBuffer(bytesRef.bytes, bytesRef.offset, bytesRef.length);
+        return NettyUtils.toBytesReference(channelBuffer);
     }
 
     public void testSliceOnAdvancedBuffer() throws IOException {
         BytesReference bytesReference = newBytesReference(randomIntBetween(10, 3 * PAGE_SIZE));
-        BytesArray bytesArray = bytesReference.toBytesArray();
-
-        ChannelBuffer channelBuffer = ChannelBuffers.wrappedBuffer(bytesArray.array(), bytesArray.arrayOffset(),
-            bytesArray.length());
+        BytesRef bytesRef = bytesReference.toBytesRef();
+        ChannelBuffer channelBuffer = ChannelBuffers.wrappedBuffer(bytesRef.bytes, bytesRef.offset,
+            bytesRef.length);
         int numBytesToRead = randomIntBetween(1, 5);
         for (int i = 0; i < numBytesToRead; i++) {
             channelBuffer.readByte();
@@ -56,7 +54,25 @@ public class ChannelBufferBytesReferenceTests extends AbstractBytesReferenceTest
         BytesReference other = NettyUtils.toBytesReference(channelBuffer);
         BytesReference slice = bytesReference.slice(numBytesToRead, bytesReference.length() - numBytesToRead);
         assertEquals(other, slice);
-
         assertEquals(other.slice(3, 1), slice.slice(3, 1));
     }
+
+    public void testImmutable() throws IOException {
+        BytesReference bytesReference = newBytesReference(randomIntBetween(10, 3 * PAGE_SIZE));
+        BytesRef bytesRef = BytesRef.deepCopyOf(bytesReference.toBytesRef());
+        ChannelBuffer channelBuffer = ChannelBuffers.wrappedBuffer(bytesRef.bytes, bytesRef.offset,
+            bytesRef.length);
+        ChannelBufferBytesReference channelBufferBytesReference = new ChannelBufferBytesReference(channelBuffer, bytesRef.length);
+        assertEquals(channelBufferBytesReference, bytesReference);
+        channelBuffer.readInt(); // this advances the index of the channel buffer
+        assertEquals(channelBufferBytesReference, bytesReference);
+        assertEquals(bytesRef, channelBufferBytesReference.toBytesRef());
+
+        BytesRef unicodeBytes = new BytesRef(randomUnicodeOfCodepointLength(100));
+        channelBuffer = ChannelBuffers.wrappedBuffer(unicodeBytes.bytes, unicodeBytes.offset, unicodeBytes.length);
+        channelBufferBytesReference = new ChannelBufferBytesReference(channelBuffer, unicodeBytes.length);
+        String utf8ToString = channelBufferBytesReference.utf8ToString();
+        channelBuffer.readInt(); // this advances the index of the channel buffer
+        assertEquals(utf8ToString, channelBufferBytesReference.utf8ToString());
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyUtilsTests.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyUtilsTests.java
index fa8f30249bb..aa1db4a44c7 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/NettyUtilsTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyUtilsTests.java
@@ -18,6 +18,8 @@
  */
 package org.elasticsearch.transport.netty;
 
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.bytes.AbstractBytesReferenceTestCase;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
@@ -47,7 +49,7 @@ public class NettyUtilsTests extends ESTestCase {
         BytesReference slice = ref.slice(sliceOffset, sliceLength);
         ChannelBuffer channelBuffer = NettyUtils.toChannelBuffer(slice);
         BytesReference bytesReference = NettyUtils.toBytesReference(channelBuffer);
-        assertArrayEquals(slice.toBytes(), bytesReference.toBytes());
+        assertArrayEquals(BytesReference.toBytes(slice), BytesReference.toBytes(bytesReference));
     }
 
     public void testToChannelBufferWithSliceAfter() throws IOException {
@@ -56,7 +58,8 @@ public class NettyUtilsTests extends ESTestCase {
         int sliceLength = randomIntBetween(ref.length() - sliceOffset, ref.length() - sliceOffset);
         ChannelBuffer channelBuffer = NettyUtils.toChannelBuffer(ref);
         BytesReference bytesReference = NettyUtils.toBytesReference(channelBuffer);
-        assertArrayEquals(ref.slice(sliceOffset, sliceLength).toBytes(), bytesReference.slice(sliceOffset, sliceLength).toBytes());
+        assertArrayEquals(BytesReference.toBytes(ref.slice(sliceOffset, sliceLength)),
+            BytesReference.toBytes(bytesReference.slice(sliceOffset, sliceLength)));
     }
 
     public void testToChannelBuffer() throws IOException {
@@ -65,10 +68,10 @@ public class NettyUtilsTests extends ESTestCase {
         BytesReference bytesReference = NettyUtils.toBytesReference(channelBuffer);
         if (ref instanceof ChannelBufferBytesReference) {
             assertEquals(channelBuffer, ((ChannelBufferBytesReference) ref).toChannelBuffer());
-        } else if (ref.hasArray() == false) { // we gather the buffers into a channel buffer
+        } else if (AbstractBytesReferenceTestCase.getNumPages(ref) > 1) { // we gather the buffers into a channel buffer
             assertTrue(channelBuffer instanceof CompositeChannelBuffer);
         }
-        assertArrayEquals(ref.toBytes(), bytesReference.toBytes());
+        assertArrayEquals(BytesReference.toBytes(ref), BytesReference.toBytes(bytesReference));
     }
 
     private BytesReference getRandomizedBytesReference(int length) throws IOException {
@@ -81,13 +84,14 @@ public class NettyUtilsTests extends ESTestCase {
         BytesReference ref = out.bytes();
         assertEquals(ref.length(), length);
         if (randomBoolean()) {
-            return ref.toBytesArray();
+            return new BytesArray(ref.toBytesRef());
         } else if (randomBoolean()) {
-            BytesArray bytesArray = ref.toBytesArray();
-            return NettyUtils.toBytesReference(ChannelBuffers.wrappedBuffer(bytesArray.array(), bytesArray.arrayOffset(),
-                bytesArray.length()));
+            BytesRef bytesRef = ref.toBytesRef();
+            return NettyUtils.toBytesReference(ChannelBuffers.wrappedBuffer(bytesRef.bytes, bytesRef.offset,
+                bytesRef.length));
         } else {
             return ref;
         }
     }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java b/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java
index b4b5eefc832..f14d91465f6 100644
--- a/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java
+++ b/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java
@@ -240,12 +240,12 @@ public class UpdateNoopIT extends ESIntegTestCase {
 
     private void updateAndCheckSource(long expectedVersion, Boolean detectNoop, XContentBuilder xContentBuilder) {
         UpdateResponse updateResponse = update(detectNoop, expectedVersion, xContentBuilder);
-        assertEquals(updateResponse.getGetResult().sourceRef().toUtf8(), xContentBuilder.bytes().toUtf8());
+        assertEquals(updateResponse.getGetResult().sourceRef().utf8ToString(), xContentBuilder.bytes().utf8ToString());
     }
 
     private UpdateResponse update(Boolean detectNoop, long expectedVersion, XContentBuilder xContentBuilder) {
         UpdateRequestBuilder updateRequest = client().prepareUpdate("test", "type1", "1")
-                .setDoc(xContentBuilder.bytes().toUtf8())
+                .setDoc(xContentBuilder.bytes().utf8ToString())
                 .setDocAsUpsert(true)
                 .setFields("_source");
         if (detectNoop != null) {
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
index 28894c5c1cc..d3871e90510 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
@@ -465,7 +465,7 @@ public class SearchFieldsTests extends ESIntegTestCase {
         String dateTime = Joda.forPattern("dateOptionalTime").printer().print(new DateTime(2012, 3, 22, 0, 0, DateTimeZone.UTC));
         assertThat(searchResponse.getHits().getAt(0).fields().get("date_field").value(), equalTo((Object) dateTime));
         assertThat(searchResponse.getHits().getAt(0).fields().get("boolean_field").value(), equalTo((Object) Boolean.TRUE));
-        assertThat(((BytesReference) searchResponse.getHits().getAt(0).fields().get("binary_field").value()).toBytesArray(), equalTo((BytesReference) new BytesArray("testing text".getBytes("UTF8"))));
+        assertThat(((BytesReference) searchResponse.getHits().getAt(0).fields().get("binary_field").value()), equalTo((BytesReference) new BytesArray("testing text".getBytes("UTF8"))));
 
     }
 
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestSearchTemplateAction.java
index 131443da887..e12137ee3c5 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestSearchTemplateAction.java
@@ -73,7 +73,7 @@ public class RestSearchTemplateAction extends BaseRestHandler {
             request.setScriptType(ScriptService.ScriptType.INLINE);
             if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
                 try (XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType())) {
-                    request.setScript(builder.copyCurrentStructure(parser).bytes().toUtf8());
+                    request.setScript(builder.copyCurrentStructure(parser).bytes().utf8ToString());
                 } catch (IOException e) {
                     throw new ParsingException(parser.getTokenLocation(), "Could not parse inline template", e);
                 }
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateIT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateIT.java
index 9c28225c081..03d04e518dc 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateIT.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateIT.java
@@ -146,14 +146,14 @@ public class MultiSearchTemplateIT extends ESIntegTestCase {
         SearchTemplateResponse searchTemplateResponse1 = response1.getResponse();
         assertThat(searchTemplateResponse1.hasResponse(), is(true));
         assertHitCount(searchTemplateResponse1.getResponse(), (numDocs / 2) + (numDocs % 2));
-        assertThat(searchTemplateResponse1.getSource().toUtf8(),
+        assertThat(searchTemplateResponse1.getSource().utf8ToString(),
                 equalTo("{\"query\":{\"match\":{\"odd\":\"true\"}}}"));
 
         MultiSearchTemplateResponse.Item response2 = response.getResponses()[1];
         assertThat(response2.isFailure(), is(false));
         SearchTemplateResponse searchTemplateResponse2 = response2.getResponse();
         assertThat(searchTemplateResponse2.hasResponse(), is(false));
-        assertThat(searchTemplateResponse2.getSource().toUtf8(),
+        assertThat(searchTemplateResponse2.getSource().utf8ToString(),
                 equalTo("{\"query\":{\"match_phrase_prefix\":{\"message\":\"quick brown f\"}}}"));
 
         MultiSearchTemplateResponse.Item response3 = response.getResponses()[2];
@@ -161,7 +161,7 @@ public class MultiSearchTemplateIT extends ESIntegTestCase {
         SearchTemplateResponse searchTemplateResponse3 = response3.getResponse();
         assertThat(searchTemplateResponse3.hasResponse(), is(true));
         assertHitCount(searchTemplateResponse3.getResponse(), (numDocs / 2));
-        assertThat(searchTemplateResponse3.getSource().toUtf8(),
+        assertThat(searchTemplateResponse3.getSource().utf8ToString(),
                 equalTo("{\"query\":{\"term\":{\"odd\":\"false\"}}}"));
 
         MultiSearchTemplateResponse.Item response4 = response.getResponses()[3];
@@ -173,7 +173,7 @@ public class MultiSearchTemplateIT extends ESIntegTestCase {
         assertThat(response5.isFailure(), is(false));
         SearchTemplateResponse searchTemplateResponse5 = response5.getResponse();
         assertThat(searchTemplateResponse5.hasResponse(), is(false));
-        assertThat(searchTemplateResponse5.getSource().toUtf8(),
+        assertThat(searchTemplateResponse5.getSource().utf8ToString(),
                 equalTo("{\"query\":{\"terms\":{\"group\":[1,2,3,]}}}"));
     }
 }
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java
index 054268ef681..bdba0346a85 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java
@@ -28,7 +28,6 @@ import org.junit.Before;
 
 import java.io.IOException;
 import java.io.StringWriter;
-import java.nio.charset.Charset;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -58,7 +57,7 @@ public class MustacheScriptEngineTests extends ESTestCase {
             BytesReference o = (BytesReference) qe.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "", "mustache", qe.compile(null, template, compileParams)), vars).run();
             assertEquals("GET _search {\"query\": {\"boosting\": {\"positive\": {\"match\": {\"body\": \"gift\"}},"
                     + "\"negative\": {\"term\": {\"body\": {\"value\": \"solr\"}}}, \"negative_boost\": 0.3 } }}",
-                    new String(o.toBytes(), Charset.forName("UTF-8")));
+                    o.utf8ToString());
         }
         {
             String template = "GET _search {\"query\": " + "{\"boosting\": {" + "\"positive\": {\"match\": {\"body\": \"gift\"}},"
@@ -69,7 +68,7 @@ public class MustacheScriptEngineTests extends ESTestCase {
             BytesReference o = (BytesReference) qe.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "", "mustache", qe.compile(null, template, compileParams)), vars).run();
             assertEquals("GET _search {\"query\": {\"boosting\": {\"positive\": {\"match\": {\"body\": \"gift\"}},"
                     + "\"negative\": {\"term\": {\"body\": {\"value\": \"\\\"quick brown\\\"\"}}}, \"negative_boost\": 0.3 } }}",
-                    new String(o.toBytes(), Charset.forName("UTF-8")));
+                    o.utf8ToString());
         }
     }
 
diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java
index 8b6d0e69573..91098eb1c88 100644
--- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java
+++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java
@@ -72,7 +72,7 @@ public class MustacheTests extends ESTestCase {
                 "Mustache templating broken",
                 "GET _search {\"query\": {\"boosting\": {\"positive\": {\"match\": {\"body\": \"gift\"}},"
                         + "\"negative\": {\"term\": {\"body\": {\"value\": \"solr\"}}}, \"negative_boost\": 0.2 } }}",
-                ((BytesReference) result.run()).toUtf8()
+                ((BytesReference) result.run()).utf8ToString()
         );
     }
 
@@ -88,7 +88,7 @@ public class MustacheTests extends ESTestCase {
         assertThat(output, notNullValue());
         assertThat(output, instanceOf(BytesReference.class));
         BytesReference bytes = (BytesReference) output;
-        assertThat(bytes.toUtf8(), equalTo("foo bar"));
+        assertThat(bytes.utf8ToString(), equalTo("foo bar"));
 
         // Sets can come out in any order
         Set<String> setData = new HashSet<>();
@@ -99,7 +99,7 @@ public class MustacheTests extends ESTestCase {
         assertThat(output, notNullValue());
         assertThat(output, instanceOf(BytesReference.class));
         bytes = (BytesReference) output;
-        assertThat(bytes.toUtf8(), both(containsString("foo")).and(containsString("bar")));
+        assertThat(bytes.utf8ToString(), both(containsString("foo")).and(containsString("bar")));
     }
 
     public void testArrayInArrayAccess() throws Exception {
@@ -116,7 +116,7 @@ public class MustacheTests extends ESTestCase {
         assertThat(output, notNullValue());
         assertThat(output, instanceOf(BytesReference.class));
         BytesReference bytes = (BytesReference) output;
-        assertThat(bytes.toUtf8(), equalTo("foo bar"));
+        assertThat(bytes.utf8ToString(), equalTo("foo bar"));
     }
 
     public void testMapInArrayAccess() throws Exception {
@@ -131,7 +131,7 @@ public class MustacheTests extends ESTestCase {
         assertThat(output, notNullValue());
         assertThat(output, instanceOf(BytesReference.class));
         BytesReference bytes = (BytesReference) output;
-        assertThat(bytes.toUtf8(), equalTo("foo bar"));
+        assertThat(bytes.utf8ToString(), equalTo("foo bar"));
 
         // HashSet iteration order isn't fixed
         Set<Object> setData = new HashSet<>();
@@ -142,7 +142,7 @@ public class MustacheTests extends ESTestCase {
         assertThat(output, notNullValue());
         assertThat(output, instanceOf(BytesReference.class));
         bytes = (BytesReference) output;
-        assertThat(bytes.toUtf8(), both(containsString("foo")).and(containsString("bar")));
+        assertThat(bytes.utf8ToString(), both(containsString("foo")).and(containsString("bar")));
     }
 
     public void testEscaping() {
@@ -152,7 +152,7 @@ public class MustacheTests extends ESTestCase {
         CompiledScript compiledScript = new CompiledScript(INLINE, "name", "mustache", mustache);
         ExecutableScript executableScript = engine.executable(compiledScript, Collections.singletonMap("value", "a \"value\""));
         BytesReference rawResult = (BytesReference) executableScript.run();
-        String result = rawResult.toUtf8();
+        String result = rawResult.utf8ToString();
         assertThat(result, equalTo("{ \"field1\": \"a \\\"value\\\"\"}"));
 
         // json string escaping disabled:
@@ -160,7 +160,7 @@ public class MustacheTests extends ESTestCase {
         compiledScript = new CompiledScript(INLINE, "name", "mustache", mustache);
         executableScript = engine.executable(compiledScript, Collections.singletonMap("value", "a \"value\""));
         rawResult = (BytesReference) executableScript.run();
-        result = rawResult.toUtf8();
+        result = rawResult.utf8ToString();
         assertThat(result, equalTo("{ \"field1\": \"a \"value\"\"}"));
     }
 
@@ -182,7 +182,7 @@ public class MustacheTests extends ESTestCase {
 
         BytesReference bytes = (BytesReference) output;
         String expectedString = String.format(Locale.ROOT, "%s %s", randomArrayValues.length, randomList.size());
-        assertThat(bytes.toUtf8(), equalTo(expectedString));
+        assertThat(bytes.utf8ToString(), equalTo(expectedString));
     }
 
     public void testPrimitiveToJSON() throws Exception {
@@ -378,7 +378,7 @@ public class MustacheTests extends ESTestCase {
         Object result = engine.executable(new CompiledScript(INLINE, "inline", "mustache", compile(script)), vars).run();
         assertThat(result, notNullValue());
         assertThat(result, instanceOf(BytesReference.class));
-        assertThat(((BytesReference) result).toUtf8(), matcher);
+        assertThat(((BytesReference) result).utf8ToString(), matcher);
     }
 
     private Object compile(String script) {
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
index 311b612a18d..6cd5a568f16 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
@@ -292,19 +292,13 @@ public final class PercolateQuery extends Query implements Accountable {
 
     @Override
     public String toString(String s) {
-        return "PercolateQuery{document_type={" + documentType + "},document_source={" + documentSource.toUtf8() +
+        return "PercolateQuery{document_type={" + documentType + "},document_source={" + documentSource.utf8ToString() +
                 "},inner={" + percolatorQueriesQuery.toString(s)  + "}}";
     }
 
     @Override
     public long ramBytesUsed() {
-        long sizeInBytes = 0;
-        if (documentSource.hasArray()) {
-            sizeInBytes += documentSource.array().length;
-        } else {
-            sizeInBytes += documentSource.length();
-        }
-        return sizeInBytes;
+        return documentSource.ramBytesUsed();
     }
 
     @FunctionalInterface
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
index 53f74c97200..b2f14ab496a 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
@@ -23,6 +23,7 @@ import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -216,7 +217,7 @@ public class PercolatorFieldMapper extends FieldMapper {
         try (XContentBuilder builder = XContentFactory.contentBuilder(QUERY_BUILDER_CONTENT_TYPE)) {
             queryBuilder.toXContent(builder, new MapParams(Collections.emptyMap()));
             builder.flush();
-            byte[] queryBuilderAsBytes = builder.bytes().toBytes();
+            byte[] queryBuilderAsBytes = BytesReference.toBytes(builder.bytes());
             context.doc().add(new Field(queryBuilderField.name(), queryBuilderAsBytes, queryBuilderField.fieldType()));
         }
 
diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java
index 8c86800e52b..6b80baa7c28 100644
--- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java
+++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java
@@ -116,7 +116,7 @@ public class SimpleAttachmentMapperTests extends AttachmentUnitTestCase {
                 .endObject()
                 .endObject();
 
-        byte[] mapping = mappingBuilder.bytes().toBytes();
+        byte[] mapping = BytesReference.toBytes(mappingBuilder.bytes());
         MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper());
         DocumentMapper docMapper = mapperService.parse("mail", new CompressedXContent(mapping), true);
         // this should not throw an exception
diff --git a/test/framework/src/main/java/org/elasticsearch/common/bytes/ByteBufferBytesReference.java b/test/framework/src/main/java/org/elasticsearch/common/bytes/ByteBufferBytesReference.java
deleted file mode 100644
index a1c9da18470..00000000000
--- a/test/framework/src/main/java/org/elasticsearch/common/bytes/ByteBufferBytesReference.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.bytes;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.io.stream.ByteBufferStreamInput;
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.jboss.netty.util.CharsetUtil;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.CharBuffer;
-import java.nio.charset.CharacterCodingException;
-import java.nio.charset.CharsetDecoder;
-import java.nio.charset.CoderResult;
-import java.nio.charset.StandardCharsets;
-
-/**
- * Note: this is only used by one lone test method.
- */
-public class ByteBufferBytesReference implements BytesReference {
-
-    private final ByteBuffer buffer;
-
-    public ByteBufferBytesReference(ByteBuffer buffer) {
-        this.buffer = buffer;
-    }
-
-    @Override
-    public byte get(int index) {
-        return buffer.get(buffer.position() + index);
-    }
-
-    @Override
-    public int length() {
-        return buffer.remaining();
-    }
-
-    @Override
-    public BytesReference slice(int from, int length) {
-        ByteBuffer dup = buffer.duplicate();
-        dup.position(buffer.position() + from);
-        dup.limit(buffer.position() + from + length);
-        return new ByteBufferBytesReference(dup);
-    }
-
-    @Override
-    public StreamInput streamInput() {
-        return new ByteBufferStreamInput(buffer);
-    }
-
-    @Override
-    public void writeTo(OutputStream os) throws IOException {
-        if (buffer.hasArray()) {
-            os.write(buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining());
-        } else {
-            byte[] tmp = new byte[8192];
-            ByteBuffer buf = buffer.duplicate();
-            while (buf.hasRemaining()) {
-                buf.get(tmp, 0, Math.min(tmp.length, buf.remaining()));
-                os.write(tmp);
-            }
-        }
-    }
-
-    @Override
-    public byte[] toBytes() {
-        if (!buffer.hasRemaining()) {
-            return BytesRef.EMPTY_BYTES;
-        }
-        byte[] tmp = new byte[buffer.remaining()];
-        buffer.duplicate().get(tmp);
-        return tmp;
-    }
-
-    @Override
-    public BytesArray toBytesArray() {
-        if (buffer.hasArray()) {
-            return new BytesArray(buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining());
-        }
-        return new BytesArray(toBytes());
-    }
-
-    @Override
-    public BytesArray copyBytesArray() {
-        return new BytesArray(toBytes());
-    }
-
-    @Override
-    public boolean hasArray() {
-        return buffer.hasArray();
-    }
-
-    @Override
-    public byte[] array() {
-        return buffer.array();
-    }
-
-    @Override
-    public int arrayOffset() {
-        return buffer.arrayOffset() + buffer.position();
-    }
-
-    @Override
-    public int hashCode() {
-        return Helper.bytesHashCode(this);
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        return Helper.bytesEqual(this, (BytesReference) obj);
-    }
-
-    @Override
-    public String toUtf8() {
-        if (!buffer.hasRemaining()) {
-            return "";
-        }
-        final CharsetDecoder decoder = CharsetUtil.getDecoder(StandardCharsets.UTF_8);
-        final CharBuffer dst = CharBuffer.allocate(
-                (int) ((double) buffer.remaining() * decoder.maxCharsPerByte()));
-        try {
-            CoderResult cr = decoder.decode(buffer, dst, true);
-            if (!cr.isUnderflow()) {
-                cr.throwException();
-            }
-            cr = decoder.flush(dst);
-            if (!cr.isUnderflow()) {
-                cr.throwException();
-            }
-        } catch (CharacterCodingException x) {
-            throw new IllegalStateException(x);
-        }
-        return dst.flip().toString();
-    }
-
-    @Override
-    public BytesRef toBytesRef() {
-        if (buffer.hasArray()) {
-            return new BytesRef(buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining());
-        }
-        return new BytesRef(toBytes());
-    }
-
-    @Override
-    public BytesRef copyBytesRef() {
-        return new BytesRef(toBytes());
-    }
-}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
index e86854b34c4..d5b4cdfa554 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
@@ -513,7 +513,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
     protected QueryBuilder assertSerialization(QueryBuilder testQuery) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             output.writeNamedWriteable(testQuery);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), serviceHolder.namedWriteableRegistry)) {
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), serviceHolder.namedWriteableRegistry)) {
                 QueryBuilder deserializedQuery = in.readNamedWriteable(QueryBuilder.class);
                 assertEquals(testQuery, deserializedQuery);
                 assertEquals(testQuery.hashCode(), deserializedQuery.hashCode());
@@ -562,7 +562,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
     protected QB copyQuery(QB query) throws IOException {
         try (BytesStreamOutput output = new BytesStreamOutput()) {
             output.writeNamedWriteable(query);
-            try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), serviceHolder.namedWriteableRegistry)) {
+            try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), serviceHolder.namedWriteableRegistry)) {
                 return (QB) in.readNamedWriteable(QueryBuilder.class);
             }
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index a49c33f5be8..041b0f9929b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -106,7 +106,6 @@ import org.elasticsearch.index.MergeSchedulerConfig;
 import org.elasticsearch.index.MockEngineFactoryPlugin;
 import org.elasticsearch.index.codec.CodecService;
 import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.indices.IndicesQueryCache;
 import org.elasticsearch.indices.IndicesRequestCache;
@@ -405,7 +404,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
                     .setOrder(0)
                     .setSettings(randomSettingsBuilder);
             if (mappings != null) {
-                logger.info("test using _default_ mappings: [{}]", mappings.bytes().toUtf8());
+                logger.info("test using _default_ mappings: [{}]", mappings.bytes().utf8ToString());
                 putTemplate.addMapping("_default_", mappings);
             }
             assertAcked(putTemplate.execute().actionGet());
diff --git a/test/framework/src/main/java/org/elasticsearch/test/StreamsUtils.java b/test/framework/src/main/java/org/elasticsearch/test/StreamsUtils.java
index 10469286e1a..1d0eaa7ce51 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/StreamsUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/StreamsUtils.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.test;
 
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.Streams;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 
@@ -53,7 +54,7 @@ public class StreamsUtils {
             }
             try (BytesStreamOutput out = new BytesStreamOutput()) {
                 Streams.copy(is, out);
-                return out.bytes().toBytes();
+                return BytesReference.toBytes(out.bytes());
             }
         }
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
index baad3ef6f04..82f2346c421 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
@@ -647,7 +647,7 @@ public class ElasticsearchAssertions {
                 ((ActionRequest<?>) streamable).validate();
             }
             BytesReference orig = serialize(version, streamable);
-            StreamInput input = StreamInput.wrap(orig);
+            StreamInput input = orig.streamInput();
             if (namedWriteableRegistry != null) {
                 input = new NamedWriteableAwareStreamInput(input, namedWriteableRegistry);
             }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java
index 7b1c3fd936b..1b99d2f32cb 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java
@@ -131,11 +131,11 @@ public class MockFSDirectoryService extends FsDirectoryService {
                         ESTestCase.checkIndexFailed = true;
                         logger.warn("check index [failure] index files={}\n{}",
                                 Arrays.toString(dir.listAll()),
-                                new String(os.bytes().toBytes(), StandardCharsets.UTF_8));
+                                os.bytes().utf8ToString());
                         throw new IOException("index check failure");
                     } else {
                         if (logger.isDebugEnabled()) {
-                            logger.debug("check index [success]\n{}", new String(os.bytes().toBytes(), StandardCharsets.UTF_8));
+                            logger.debug("check index [success]\n{}", os.bytes().utf8ToString());
                         }
                     }
                 } catch (LockObtainFailedException e) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
index 7688e2842e6..7bc7f0d5254 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
@@ -383,7 +383,7 @@ public class MockTransportService extends TransportService {
                 BytesStreamOutput bStream = new BytesStreamOutput();
                 request.writeTo(bStream);
                 final TransportRequest clonedRequest = reg.newRequest();
-                clonedRequest.readFrom(StreamInput.wrap(bStream.bytes()));
+                clonedRequest.readFrom(bStream.bytes().streamInput());
 
                 threadPool.schedule(delay, ThreadPool.Names.GENERIC, new AbstractRunnable() {
                     @Override

From 4c32b025d4fe3478ffd88364acff575670f478cf Mon Sep 17 00:00:00 2001
From: Christoph Wurm <wurm@elastic.co>
Date: Fri, 1 Jul 2016 16:10:59 +0200
Subject: [PATCH 1184/1311] Fix Processor: Now implements Processor.Factory

---
 .../elasticsearch/ingest/useragent/UserAgentProcessor.java | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
index fb917b33c7c..7cb829e36e2 100644
--- a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
+++ b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
@@ -20,8 +20,8 @@
 package org.elasticsearch.ingest.useragent;
 
 import org.elasticsearch.ingest.AbstractProcessor;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
 import org.elasticsearch.ingest.useragent.UserAgentParser.Details;
 import org.elasticsearch.ingest.useragent.UserAgentParser.VersionedName;
 
@@ -36,7 +36,6 @@ import java.util.Set;
 import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
 import static org.elasticsearch.ingest.ConfigurationUtils.readOptionalList;
 import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
-import static org.elasticsearch.ingest.ConfigurationUtils.readOptionalStringProperty;
 
 public class UserAgentProcessor extends AbstractProcessor {
 
@@ -186,7 +185,7 @@ public class UserAgentProcessor extends AbstractProcessor {
         return parser;
     }
 
-    public static final class Factory extends AbstractProcessorFactory<UserAgentProcessor> {
+    public static final class Factory implements Processor.Factory {
 
         private final Map<String, UserAgentParser> userAgentParsers;
         
@@ -195,7 +194,7 @@ public class UserAgentProcessor extends AbstractProcessor {
         }
 
         @Override
-        public UserAgentProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
+        public UserAgentProcessor create(String processorTag, Map<String, Object> config) throws Exception {
             String field = readStringProperty(TYPE, processorTag, config, "field");
             String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "useragent");
             String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME);

From 368c3ccc54c839692b19f5068af335863f7c16aa Mon Sep 17 00:00:00 2001
From: Christoph Wurm <wurm@elastic.co>
Date: Fri, 1 Jul 2016 16:53:17 +0200
Subject: [PATCH 1185/1311] Fix Factory test

---
 .../UserAgentProcessorFactoryTests.java          | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java b/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java
index 516b7ce3206..52a10840387 100644
--- a/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java
+++ b/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.ingest.useragent;
 
 import org.elasticsearch.ElasticsearchParseException;
-import org.elasticsearch.ingest.AbstractProcessorFactory;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.BeforeClass;
 
@@ -81,9 +80,8 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
 
         String processorTag = randomAsciiOfLength(10);
-        config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
 
-        UserAgentProcessor processor = factory.create(config);
+        UserAgentProcessor processor = factory.create(processorTag, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("useragent"));
@@ -100,7 +98,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("target_field", "_target_field");
 
-        UserAgentProcessor processor = factory.create(config);
+        UserAgentProcessor processor = factory.create(null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("_target_field"));
     }
@@ -112,7 +110,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("regex_file", regexWithoutDevicesFilename);
 
-        UserAgentProcessor processor = factory.create(config);
+        UserAgentProcessor processor = factory.create(null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getUaParser().getUaPatterns().size(), greaterThan(0));
         assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0));
@@ -126,7 +124,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("regex_file", "does-not-exist.yaml");
 
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config));
         assertThat(e.getMessage(), equalTo("[regex_file] regex file [does-not-exist.yaml] doesn't exist (has to exist at node startup)"));
     }
 
@@ -146,7 +144,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("properties", fieldNames);
 
-        UserAgentProcessor processor = factory.create(config);
+        UserAgentProcessor processor = factory.create(null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getProperties(), equalTo(properties));
     }
@@ -158,7 +156,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("properties", Collections.singletonList("invalid"));
 
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config));
         assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, MAJOR, MINOR, "
                 + "PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD]"));
     }
@@ -170,7 +168,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("properties", "invalid");
 
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(config));
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config));
         assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]"));
     }
 }

From 93b42b8e69c2c23cb3cc6b380a33776ce05370d5 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Fri, 1 Jul 2016 16:27:44 +0200
Subject: [PATCH 1186/1311] Enable checkstyle ModifierOrder module

---
 buildSrc/src/main/resources/checkstyle.xml | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/buildSrc/src/main/resources/checkstyle.xml b/buildSrc/src/main/resources/checkstyle.xml
index de47736913f..b75f2f86ea0 100644
--- a/buildSrc/src/main/resources/checkstyle.xml
+++ b/buildSrc/src/main/resources/checkstyle.xml
@@ -39,6 +39,23 @@
 
     <module name="EqualsHashCode" />
 
+    <!-- Checks that the order of modifiers conforms to the suggestions in the
+    Java Language specification, sections 8.1.1, 8.3.1 and 8.4.3.
+    The correct order is:
+            public
+            protected
+            private
+            abstract
+            static
+            final
+            transient
+            volatile
+            synchronized
+            native
+            strictfp
+    -->
+    <module name="ModifierOrder" />
+
     <!-- We don't use Java's builtin serialization and we suppress all warning
       about it. The flip side of that coin is that we shouldn't _try_ to use
       it. We can't outright ban it with ForbiddenApis because it complain about

From 8c40b2b54eac3e3ab3c41ece5c758be75173191b Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Fri, 1 Jul 2016 16:28:59 +0200
Subject: [PATCH 1187/1311] Fix order of modifiers

---
 buildSrc/src/main/resources/checkstyle.xml    |   4 +-
 .../NamingConventionsCheckBadClasses.java     |   2 +-
 .../client/HttpDeleteWithEntity.java          |   2 +-
 .../client/HttpGetWithEntity.java             |   2 +-
 .../analyzing/XAnalyzingSuggester.java        |   2 +-
 .../cluster/allocation/NodeExplanation.java   |   2 +-
 .../TransportGetFieldMappingsIndexAction.java |   2 +-
 .../indices/rollover/MaxAgeCondition.java     |   2 +-
 .../indices/rollover/MaxDocsCondition.java    |   2 +-
 .../admin/indices/stats/CommonStatsFlags.java |   4 +-
 .../action/bulk/BulkResponse.java             |   2 +-
 .../action/bulk/TransportShardBulkAction.java |   4 +-
 .../action/fieldstats/FieldStats.java         |  24 ++--
 .../action/fieldstats/FieldStatsRequest.java  |   2 +-
 .../action/ingest/IngestActionFilter.java     |   4 +-
 .../action/search/SearchResponse.java         |   2 +-
 .../search/TransportMultiSearchAction.java    |   2 +-
 .../AbstractListenableActionFuture.java       |   2 +-
 .../action/support/ActionFilter.java          |   2 +-
 .../node/TransportBroadcastByNodeAction.java  |   2 +-
 .../replication/ReplicationOperation.java     |  26 ++--
 .../replication/ReplicationRequest.java       |   3 +-
 .../replication/ReplicationResponse.java      |   2 +-
 .../TransportReplicationAction.java           |  14 +-
 ...ransportInstanceSingleOperationAction.java |   2 +-
 .../bootstrap/BootstrapCheck.java             |   2 +-
 .../bootstrap/JNAKernel32Library.java         |   4 +-
 .../elasticsearch/bootstrap/JavaVersion.java  |   2 +-
 .../elasticsearch/cluster/ClusterState.java   |   2 +-
 .../cluster/ClusterStateObserver.java         |  14 +-
 .../cluster/ClusterStateTaskExecutor.java     |   4 +-
 .../cluster/ClusterStateUpdateTask.java       |  10 +-
 .../elasticsearch/cluster/DiffableUtils.java  |   6 +-
 .../cluster/EmptyClusterInfoService.java      |   2 +-
 .../cluster/NodeConnectionsService.java       |   2 +-
 .../cluster/metadata/IndexGraveyard.java      |  10 +-
 .../cluster/metadata/IndexMetaData.java       |   2 +-
 .../metadata/IndexNameExpressionResolver.java |   6 +-
 .../cluster/metadata/MetaData.java            |   4 +-
 .../metadata/MetaDataCreateIndexService.java  |   2 +-
 .../routing/IndexShardRoutingTable.java       |   2 +-
 .../cluster/routing/RoutingNode.java          |   3 +-
 .../AbstractAllocateAllocationCommand.java    |   2 +-
 .../command/BasePrimaryAllocationCommand.java |   2 +-
 .../cluster/service/ClusterService.java       |  22 ++--
 .../common/collect/CopyOnWriteHashMap.java    |   2 +-
 .../common/collect/HppcMaps.java              |   4 +-
 .../elasticsearch/common/geo/GeoDistance.java |   2 +-
 .../common/geo/GeoHashUtils.java              |   2 +-
 .../common/inject/internal/Errors.java        |   2 +-
 .../common/io/stream/StreamOutput.java        |   2 +-
 .../elasticsearch/common/logging/Loggers.java |   2 +-
 .../elasticsearch/common/lucene/Lucene.java   |   8 +-
 .../index/ElasticsearchDirectoryReader.java   |   2 +-
 .../lucene/index/FilterableTermsEnum.java     |   2 +-
 .../function/FiltersFunctionScoreQuery.java   |   2 +-
 .../common/rounding/Rounding.java             |   6 +-
 .../common/rounding/TimeZoneRounding.java     |   2 +-
 .../elasticsearch/common/util/BigArrays.java  |   2 +-
 .../common/util/ExtensionPoint.java           |   4 +-
 .../common/util/concurrent/KeyedLock.java     |   2 +-
 .../common/xcontent/XContentBuilder.java      |   4 +-
 .../common/xcontent/cbor/CborXContent.java    |   4 +-
 .../common/xcontent/json/JsonXContent.java    |   8 +-
 .../common/xcontent/smile/SmileXContent.java  |   4 +-
 .../common/xcontent/yaml/YamlXContent.java    |   4 +-
 .../discovery/DiscoverySettings.java          |   6 +-
 .../discovery/zen/NodeJoinController.java     |   6 +-
 .../discovery/zen/ZenDiscovery.java           |  16 +--
 .../publish/PublishClusterStateAction.java    |  16 +--
 .../java/org/elasticsearch/index/Index.java   |   2 +-
 .../org/elasticsearch/index/IndexService.java |   7 +-
 .../index/codec/CodecService.java             |   6 +-
 .../elasticsearch/index/engine/Engine.java    |   6 +-
 .../index/engine/InternalEngine.java          |   2 +-
 .../index/engine/ShadowEngine.java            |   4 +-
 .../index/fielddata/ScriptDocValues.java      |   2 +-
 .../elasticsearch/index/mapper/Mapper.java    |   2 +-
 .../index/mapper/core/BooleanFieldMapper.java |   4 +-
 .../mapper/core/LegacyNumberFieldMapper.java  |   2 +-
 .../index/query/AbstractQueryBuilder.java     |   4 +-
 .../index/query/HasChildQueryBuilder.java     |   2 +-
 .../index/query/InnerHitBuilder.java          |   2 +-
 .../index/query/TemplateQueryBuilder.java     |   2 +-
 .../functionscore/DecayFunctionBuilder.java   |   2 +-
 .../index/search/stats/ShardSearchStats.java  |   2 +-
 .../index/similarity/SimilarityService.java   |   2 +-
 .../org/elasticsearch/index/store/Store.java  |   2 +-
 .../index/translog/BaseTranslogReader.java    |   4 +-
 .../index/translog/Translog.java              |   4 +-
 .../index/translog/TranslogWriter.java        |   2 +-
 .../elasticsearch/indices/IndicesService.java |  10 +-
 .../indices/analysis/HunspellService.java     |   6 +-
 .../indices/analysis/PreBuiltAnalyzers.java   |   2 +-
 .../indices/analysis/PreBuiltCharFilters.java |   2 +-
 .../analysis/PreBuiltTokenFilters.java        |   2 +-
 .../indices/analysis/PreBuiltTokenizers.java  |   2 +-
 .../indices/flush/SyncedFlushService.java     |   4 +-
 .../recovery/RecoveriesCollection.java        |   4 +-
 .../indices/recovery/RecoveryState.java       |   2 +-
 .../indices/recovery/RecoveryTarget.java      |   2 +-
 .../elasticsearch/ingest/IngestDocument.java  |   2 +-
 .../elasticsearch/ingest/IngestMetadata.java  |   4 +-
 .../org/elasticsearch/ingest/Pipeline.java    |   8 +-
 .../ingest/PipelineConfiguration.java         |   4 +-
 .../elasticsearch/monitor/fs/FsService.java   |   2 +-
 .../monitor/jvm/JvmGcMonitorService.java      |  14 +-
 .../elasticsearch/monitor/jvm/JvmService.java |   2 +-
 .../elasticsearch/monitor/jvm/JvmStats.java   |   8 +-
 .../org/elasticsearch/monitor/os/OsProbe.java |   2 +-
 .../elasticsearch/monitor/os/OsService.java   |   2 +-
 .../monitor/process/ProcessProbe.java         |   2 +-
 .../monitor/process/ProcessService.java       |   2 +-
 .../blobstore/BlobStoreRepository.java        |   4 +-
 .../repositories/fs/FsRepository.java         |   2 +-
 .../repositories/uri/URLRepository.java       |   2 +-
 .../node/info/RestNodesInfoAction.java        |   2 +-
 .../rest/action/cat/RestThreadPoolAction.java |  10 +-
 .../termvectors/RestTermVectorsAction.java    |   4 +-
 .../java/org/elasticsearch/script/Script.java |   6 +-
 .../elasticsearch/script/ScriptMetaData.java  |  10 +-
 .../elasticsearch/script/ScriptSettings.java  |   4 +-
 .../search/aggregations/BucketCollector.java  |   2 +-
 .../aggregations/InternalAggregations.java    |   2 +-
 .../InternalMultiBucketAggregation.java       |   2 +-
 .../bucket/children/InternalChildren.java     |   2 +-
 .../bucket/filter/InternalFilter.java         |   4 +-
 .../bucket/filters/InternalFilters.java       |   6 +-
 .../bucket/global/InternalGlobal.java         |   4 +-
 .../bucket/histogram/Histogram.java           |   2 +-
 .../histogram/InternalDateHistogram.java      |   2 +-
 .../bucket/histogram/InternalHistogram.java   |  10 +-
 .../bucket/missing/InternalMissing.java       |   4 +-
 .../bucket/nested/InternalNested.java         |   2 +-
 .../bucket/nested/InternalReverseNested.java  |   2 +-
 .../nested/NestedAggregatorFactory.java       |   2 +-
 .../ReverseNestedAggregatorFactory.java       |   2 +-
 .../bucket/range/BinaryRangeAggregator.java   |   4 +-
 .../bucket/range/InternalBinaryRange.java     |   4 +-
 .../bucket/range/InternalRange.java           |  10 +-
 .../bucket/range/date/InternalDateRange.java  |   6 +-
 .../geodistance/InternalGeoDistance.java      |   2 +-
 .../bucket/sampler/InternalSampler.java       |   4 +-
 .../significant/InternalSignificantTerms.java |   4 +-
 .../significant/SignificantLongTerms.java     |   2 +-
 .../significant/SignificantStringTerms.java   |   2 +-
 .../bucket/significant/SignificantTerms.java  |   2 +-
 .../heuristics/NXYSignificanceHeuristic.java  |   2 +-
 .../bucket/terms/DoubleTerms.java             |   2 +-
 .../bucket/terms/InternalTerms.java           |   4 +-
 .../aggregations/bucket/terms/LongTerms.java  |   2 +-
 .../bucket/terms/StringTerms.java             |   2 +-
 .../aggregations/bucket/terms/Terms.java      |   4 +-
 .../bucket/terms/support/IncludeExclude.java  |   2 +-
 .../InternalNumericMetricsAggregation.java    |   4 +-
 .../metrics/NumericMetricsAggregator.java     |   4 +-
 .../metrics/avg/AvgAggregationBuilder.java    |   2 +-
 .../cardinality/CardinalityAggregator.java    |   4 +-
 .../cardinality/InternalCardinality.java      |   4 +-
 .../metrics/geobounds/InternalGeoBounds.java  |   4 +-
 .../geocentroid/InternalGeoCentroid.java      |   4 +-
 .../metrics/max/MaxAggregationBuilder.java    |   2 +-
 .../metrics/min/MinAggregationBuilder.java    |   2 +-
 .../percentiles/PercentilesParser.java        |   2 +-
 .../hdr/InternalHDRPercentileRanks.java       |   4 +-
 .../hdr/InternalHDRPercentiles.java           |   4 +-
 .../InternalTDigestPercentileRanks.java       |   4 +-
 .../tdigest/InternalTDigestPercentiles.java   |   4 +-
 .../scripted/InternalScriptedMetric.java      |   4 +-
 .../stats/StatsAggregationBuilder.java        |   2 +-
 .../ExtendedStatsAggregationBuilder.java      |   2 +-
 .../metrics/sum/SumAggregationBuilder.java    |   2 +-
 .../pipeline/InternalSimpleValue.java         |   4 +-
 .../InternalBucketMetricValue.java            |   4 +-
 .../avg/AvgBucketPipelineAggregator.java      |   4 +-
 .../max/MaxBucketPipelineAggregator.java      |   4 +-
 .../min/MinBucketPipelineAggregator.java      |   4 +-
 .../percentile/InternalPercentilesBucket.java |   4 +-
 .../PercentilesBucketPipelineAggregator.java  |   4 +-
 .../sum/SumBucketPipelineAggregator.java      |   4 +-
 .../BucketScriptPipelineAggregator.java       |   4 +-
 .../BucketSelectorPipelineAggregator.java     |   4 +-
 .../CumulativeSumPipelineAggregator.java      |   4 +-
 .../DerivativePipelineAggregator.java         |   4 +-
 .../derivative/InternalDerivative.java        |   4 +-
 .../movavg/MovAvgPipelineAggregator.java      |   4 +-
 .../SerialDiffPipelineAggregator.java         |   4 +-
 .../aggregations/support/AggregationPath.java |   2 +-
 .../aggregations/support/ValuesSource.java    |   8 +-
 .../ValuesSourceAggregationBuilder.java       |   2 +-
 .../fetch/FetchSubPhaseParseElement.java      |   2 +-
 .../fetch/innerhits/InnerHitsContext.java     |   2 +-
 .../search/highlight/HighlightBuilder.java    |   2 +-
 .../search/internal/InternalSearchHit.java    |   2 +-
 .../search/internal/SearchContext.java        |   2 +-
 .../search/internal/SubSearchContext.java     |   2 +-
 .../search/query/QuerySearchResult.java       |   2 +-
 .../search/rescore/QueryRescorer.java         |   2 +-
 .../search/slice/SliceBuilder.java            |   2 +-
 .../search/suggest/SuggestUtils.java          |   2 +-
 .../completion/CompletionSuggester.java       |   6 +-
 .../completion/CompletionSuggestion.java      |   2 +-
 .../Completion090PostingsFormat.java          |   8 +-
 .../completion2x/context/ContextMapping.java  |   4 +-
 .../snapshots/RestoreService.java             |  30 ++---
 .../tasks/LoggingTaskListener.java            |   2 +-
 .../java/org/elasticsearch/tasks/TaskId.java  |   2 +-
 .../threadpool/ExecutorBuilder.java           |   2 +-
 .../elasticsearch/threadpool/ThreadPool.java  |   2 +-
 .../transport/TransportService.java           |   6 +-
 .../transport/TransportSettings.java          |   2 +-
 .../elasticsearch/transport/Transports.java   |   2 +-
 .../transport/local/LocalTransport.java       |   2 +-
 .../transport/netty/NettyTransport.java       |   2 +-
 .../node/tasks/TransportTasksActionTests.java |   2 +-
 .../shards/IndicesShardStoreRequestIT.java    |   2 +-
 .../bulk/TransportBulkActionTookTests.java    |   2 +-
 .../ClusterStateCreationUtils.java            |   2 +-
 .../AbstractTermVectorsTestCase.java          |  24 ++--
 .../BasicAnalysisBackwardCompatibilityIT.java |   2 +-
 .../cluster/routing/PrimaryTermsTests.java    |   2 +-
 .../cluster/routing/RoutingTableTests.java    |   2 +-
 .../common/geo/GeoJSONShapeParserTests.java   |   2 +-
 .../common/io/stream/BytesStreamsTests.java   |   2 +-
 .../common/io/stream/StreamTests.java         |   2 +-
 .../common/xcontent/ObjectParserTests.java    |   2 +-
 ...usterStatePublishResponseHandlerTests.java |   2 +-
 .../DiscoveryWithServiceDisruptionsIT.java    |   2 +-
 .../zen/NodeJoinControllerTests.java          |   2 +-
 .../gateway/RecoverAfterNodesIT.java          |   2 +-
 .../index/mapper/FieldTypeTestCase.java       |   2 +-
 .../ESIndexLevelReplicationTestCase.java      |   8 +-
 .../indices/recovery/RecoveryTargetTests.java |   4 +-
 .../indices/store/IndicesStoreTests.java      |   2 +-
 .../recovery/RecoveriesCollectionTests.java   |   2 +-
 .../elasticsearch/script/StoredScriptsIT.java |   4 +-
 .../aggregations/bucket/ChildrenIT.java       |   2 +-
 .../search/fetch/FetchSubPhasePluginIT.java   |   2 +-
 .../search/functionscore/QueryRescorerIT.java |   2 +-
 .../elasticsearch/test/MockLogAppender.java   |   4 +-
 .../org/elasticsearch/ttl/SimpleTTLIT.java    |   2 +-
 .../matrix/stats/InternalMatrixStats.java     |   4 +-
 .../matrix/stats/MatrixStatsResults.java      |   4 +-
 .../MultiValuesSourceAggregationBuilder.java  |   2 +-
 .../matrix/stats/BaseMatrixStatsTestCase.java |   4 +-
 .../common/AbstractStringProcessor.java       |   2 +-
 .../ingest/common/GrokProcessor.java          |   2 +-
 .../ingest/common/JoinProcessor.java          |   2 +-
 .../ingest/common/LowercaseProcessor.java     |   2 +-
 .../ingest/common/SortProcessor.java          |   2 +-
 .../mustache/CustomMustacheFactory.java       |   2 +-
 .../CustomReflectionObjectHandler.java        |   4 +-
 .../painless/WriterConstants.java             | 124 +++++++++---------
 .../percolator/PercolateQuery.java            |   2 +-
 .../percolator/PercolateQueryBuilder.java     |   2 +-
 .../percolator/PercolateResponse.java         |   2 +-
 .../percolator/PercolatorFieldMapper.java     |   4 +-
 .../ExtractQueryTermsServiceTests.java        |   6 +-
 .../percolator/MultiPercolatorIT.java         |   4 +-
 .../percolator/PercolateQueryTests.java       |  10 +-
 .../percolator/PercolatorAggregationsIT.java  |   4 +-
 ...PercolatorBackwardsCompatibilityTests.java |   2 +-
 .../percolator/PercolatorIT.java              |   4 +-
 .../analysis/phonetic/HaasePhonetik.java      |   4 +-
 .../gce/RetryHttpInitializerWrapperTests.java |   2 +-
 .../ingest/geoip/GeoIpProcessor.java          |   2 +-
 .../repositories/azure/AzureRepository.java   |   2 +-
 .../storage/AzureStorageServiceTests.java     |   2 +-
 .../AzureStorageSettingsFilterTests.java      |   2 +-
 .../repositories/hdfs/HdfsBlobContainer.java  |   2 +-
 .../repositories/s3/S3Repository.java         |   2 +-
 .../s3/AbstractS3SnapshotRestoreTest.java     |   2 +-
 .../index/store/AbstractAzureFsTestCase.java  |   2 +-
 .../common/util/MockBigArrays.java            |   2 +-
 .../test/AbstractQueryTestCase.java           |  18 +--
 .../elasticsearch/test/ESIntegTestCase.java   |   6 +-
 .../test/InternalTestCluster.java             |   4 +-
 .../ClusterDiscoveryConfiguration.java        |   2 +-
 .../IntermittentLongGCDisruption.java         |   2 +-
 .../test/disruption/LongGCDisruption.java     |   2 +-
 .../test/engine/MockEngineSupport.java        |   2 +-
 .../test/transport/CapturingTransport.java    |  10 +-
 .../test/test/InternalTestClusterTests.java   |   2 +-
 283 files changed, 609 insertions(+), 604 deletions(-)

diff --git a/buildSrc/src/main/resources/checkstyle.xml b/buildSrc/src/main/resources/checkstyle.xml
index b75f2f86ea0..fe726062706 100644
--- a/buildSrc/src/main/resources/checkstyle.xml
+++ b/buildSrc/src/main/resources/checkstyle.xml
@@ -40,7 +40,9 @@
     <module name="EqualsHashCode" />
 
     <!-- Checks that the order of modifiers conforms to the suggestions in the
-    Java Language specification, sections 8.1.1, 8.3.1 and 8.4.3.
+    Java Language specification, sections 8.1.1, 8.3.1 and 8.4.3. It is not that
+    the standard is perfect, but having a consistent order makes the code more
+    readable and no other order is compellingly better than the standard.
     The correct order is:
             public
             protected
diff --git a/buildSrc/src/test/java/org/elasticsearch/test/NamingConventionsCheckBadClasses.java b/buildSrc/src/test/java/org/elasticsearch/test/NamingConventionsCheckBadClasses.java
index 87240534880..4fc88b3afc5 100644
--- a/buildSrc/src/test/java/org/elasticsearch/test/NamingConventionsCheckBadClasses.java
+++ b/buildSrc/src/test/java/org/elasticsearch/test/NamingConventionsCheckBadClasses.java
@@ -36,7 +36,7 @@ public class NamingConventionsCheckBadClasses {
         public void testDummy() {}
     }
 
-    public static abstract class DummyAbstractTests extends UnitTestCase {
+    public abstract static class DummyAbstractTests extends UnitTestCase {
     }
 
     public interface DummyInterfaceTests {
diff --git a/client/rest/src/main/java/org/elasticsearch/client/HttpDeleteWithEntity.java b/client/rest/src/main/java/org/elasticsearch/client/HttpDeleteWithEntity.java
index 85927b6f871..df08ae5a8d1 100644
--- a/client/rest/src/main/java/org/elasticsearch/client/HttpDeleteWithEntity.java
+++ b/client/rest/src/main/java/org/elasticsearch/client/HttpDeleteWithEntity.java
@@ -28,7 +28,7 @@ import java.net.URI;
  */
 final class HttpDeleteWithEntity extends HttpEntityEnclosingRequestBase {
 
-    final static String METHOD_NAME = HttpDelete.METHOD_NAME;
+    static final String METHOD_NAME = HttpDelete.METHOD_NAME;
 
     HttpDeleteWithEntity(final URI uri) {
         setURI(uri);
diff --git a/client/rest/src/main/java/org/elasticsearch/client/HttpGetWithEntity.java b/client/rest/src/main/java/org/elasticsearch/client/HttpGetWithEntity.java
index c25418387e4..a3846beefe4 100644
--- a/client/rest/src/main/java/org/elasticsearch/client/HttpGetWithEntity.java
+++ b/client/rest/src/main/java/org/elasticsearch/client/HttpGetWithEntity.java
@@ -28,7 +28,7 @@ import java.net.URI;
  */
 final class HttpGetWithEntity extends HttpEntityEnclosingRequestBase {
 
-    final static String METHOD_NAME = HttpGet.METHOD_NAME;
+    static final String METHOD_NAME = HttpGet.METHOD_NAME;
 
     HttpGetWithEntity(final URI uri) {
         setURI(uri);
diff --git a/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java b/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java
index 6017803b63d..c65f962dbb8 100644
--- a/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java
+++ b/core/src/main/java/org/apache/lucene/search/suggest/analyzing/XAnalyzingSuggester.java
@@ -1110,7 +1110,7 @@ public long ramBytesUsed() {
             this.analyzed.copyBytes(analyzed);
         }
 
-        private final static class SurfaceFormAndPayload implements Comparable<SurfaceFormAndPayload> {
+        private static final class SurfaceFormAndPayload implements Comparable<SurfaceFormAndPayload> {
             BytesRef payload;
             long weight;
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/NodeExplanation.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/NodeExplanation.java
index 8f467402bc4..e564711d418 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/NodeExplanation.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/NodeExplanation.java
@@ -43,7 +43,7 @@ public class NodeExplanation implements Writeable, ToXContent {
     private final String finalExplanation;
 
     public NodeExplanation(final DiscoveryNode node, final Decision nodeDecision, final Float nodeWeight,
-                           final @Nullable IndicesShardStoresResponse.StoreStatus storeStatus,
+                           @Nullable final IndicesShardStoresResponse.StoreStatus storeStatus,
                            final ClusterAllocationExplanation.FinalDecision finalDecision,
                            final String finalExplanation,
                            final ClusterAllocationExplanation.StoreCopy storeCopy) {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java
index e886af25fbb..864c6703c48 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java
@@ -130,7 +130,7 @@ public class TransportGetFieldMappingsIndexAction extends TransportSingleShardAc
 
     private static final ToXContent.Params includeDefaultsParams = new ToXContent.Params() {
 
-        final static String INCLUDE_DEFAULTS = "include_defaults";
+        static final String INCLUDE_DEFAULTS = "include_defaults";
 
         @Override
         public String param(String key) {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxAgeCondition.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxAgeCondition.java
index ee0641bbe18..9208193c984 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxAgeCondition.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxAgeCondition.java
@@ -30,7 +30,7 @@ import java.io.IOException;
  * when the index is at least {@link #value} old
  */
 public class MaxAgeCondition extends Condition<TimeValue> {
-    public final static String NAME = "max_age";
+    public static final String NAME = "max_age";
 
     public MaxAgeCondition(TimeValue value) {
         super(NAME);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxDocsCondition.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxDocsCondition.java
index 8b327929124..7c1f802389e 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxDocsCondition.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/MaxDocsCondition.java
@@ -29,7 +29,7 @@ import java.io.IOException;
  * when the index has at least {@link #value} docs
  */
 public class MaxDocsCondition extends Condition<Long> {
-    public final static String NAME = "max_docs";
+    public static final String NAME = "max_docs";
 
     public MaxDocsCondition(Long value) {
         super(NAME);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java
index 3be3de00f61..a9af50b2492 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java
@@ -31,8 +31,8 @@ import java.util.EnumSet;
  */
 public class CommonStatsFlags implements Streamable, Cloneable {
 
-    public final static CommonStatsFlags ALL = new CommonStatsFlags().all();
-    public final static CommonStatsFlags NONE = new CommonStatsFlags().clear();
+    public static final CommonStatsFlags ALL = new CommonStatsFlags().all();
+    public static final CommonStatsFlags NONE = new CommonStatsFlags().clear();
 
     private EnumSet<Flag> flags = EnumSet.allOf(Flag.class);
     private String[] types = null;
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java
index cd93735b642..e214f87ddb6 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkResponse.java
@@ -35,7 +35,7 @@ import java.util.Iterator;
  */
 public class BulkResponse extends ActionResponse implements Iterable<BulkItemResponse> {
 
-    public final static long NO_INGEST_TOOK = -1L;
+    public static final long NO_INGEST_TOOK = -1L;
 
     private BulkItemResponse[] responses;
     private long tookInMillis;
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
index 4ad1136e668..a9ec11eec4d 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
@@ -71,8 +71,8 @@ import static org.elasticsearch.action.support.replication.ReplicationOperation.
  */
 public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequest, BulkShardResponse> {
 
-    private final static String OP_TYPE_UPDATE = "update";
-    private final static String OP_TYPE_DELETE = "delete";
+    private static final String OP_TYPE_UPDATE = "update";
+    private static final String OP_TYPE_DELETE = "delete";
 
     public static final String ACTION_NAME = BulkAction.NAME + "[s]";
 
diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java
index a6978ad2a39..2c6f2539b97 100644
--- a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java
+++ b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java
@@ -615,17 +615,17 @@ public abstract class FieldStats<T> implements Writeable, ToXContent {
         }
     }
 
-    private final static class Fields {
-        final static String MAX_DOC = new String("max_doc");
-        final static String DOC_COUNT = new String("doc_count");
-        final static String DENSITY = new String("density");
-        final static String SUM_DOC_FREQ = new String("sum_doc_freq");
-        final static String SUM_TOTAL_TERM_FREQ = new String("sum_total_term_freq");
-        final static String SEARCHABLE = new String("searchable");
-        final static String AGGREGATABLE = new String("aggregatable");
-        final static String MIN_VALUE = new String("min_value");
-        final static String MIN_VALUE_AS_STRING = new String("min_value_as_string");
-        final static String MAX_VALUE = new String("max_value");
-        final static String MAX_VALUE_AS_STRING = new String("max_value_as_string");
+    private static final class Fields {
+        static final String MAX_DOC = new String("max_doc");
+        static final String DOC_COUNT = new String("doc_count");
+        static final String DENSITY = new String("density");
+        static final String SUM_DOC_FREQ = new String("sum_doc_freq");
+        static final String SUM_TOTAL_TERM_FREQ = new String("sum_total_term_freq");
+        static final String SEARCHABLE = new String("searchable");
+        static final String AGGREGATABLE = new String("aggregatable");
+        static final String MIN_VALUE = new String("min_value");
+        static final String MIN_VALUE_AS_STRING = new String("min_value_as_string");
+        static final String MAX_VALUE = new String("max_value");
+        static final String MAX_VALUE_AS_STRING = new String("max_value_as_string");
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequest.java b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequest.java
index af5fface7df..310af30d25e 100644
--- a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStatsRequest.java
@@ -39,7 +39,7 @@ import java.util.List;
  */
 public class FieldStatsRequest extends BroadcastRequest<FieldStatsRequest> {
 
-    public final static String DEFAULT_LEVEL = "cluster";
+    public static final String DEFAULT_LEVEL = "cluster";
 
     private String[] fields = Strings.EMPTY_ARRAY;
     private String level = DEFAULT_LEVEL;
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/IngestActionFilter.java b/core/src/main/java/org/elasticsearch/action/ingest/IngestActionFilter.java
index 850cac040dd..31c5ec43bd3 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/IngestActionFilter.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/IngestActionFilter.java
@@ -132,7 +132,7 @@ public final class IngestActionFilter extends AbstractComponent implements Actio
         return Integer.MAX_VALUE;
     }
 
-    final static class BulkRequestModifier implements Iterator<ActionRequest<?>> {
+    static final class BulkRequestModifier implements Iterator<ActionRequest<?>> {
 
         final BulkRequest bulkRequest;
         final Set<Integer> failedSlots;
@@ -210,7 +210,7 @@ public final class IngestActionFilter extends AbstractComponent implements Actio
 
     }
 
-    final static class IngestBulkResponseListener implements ActionListener<BulkResponse> {
+    static final class IngestBulkResponseListener implements ActionListener<BulkResponse> {
 
         private final long ingestTookInMillis;
         private final int[] originalSlots;
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java
index 8e144537c84..98d63b07b5e 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchResponse.java
@@ -168,7 +168,7 @@ public class SearchResponse extends ActionResponse implements StatusToXContent {
      *
      * @return The profile results or an empty map
      */
-    public @Nullable Map<String, ProfileShardResult> getProfileResults() {
+    @Nullable public Map<String, ProfileShardResult> getProfileResults() {
         return internalResponse.profile();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java b/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
index bd1751856b2..4d2d8695c84 100644
--- a/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
@@ -134,7 +134,7 @@ public class TransportMultiSearchAction extends HandledTransportAction<MultiSear
         });
     }
 
-    final static class SearchRequestSlot {
+    static final class SearchRequestSlot {
 
         final SearchRequest request;
         final int responseSlot;
diff --git a/core/src/main/java/org/elasticsearch/action/support/AbstractListenableActionFuture.java b/core/src/main/java/org/elasticsearch/action/support/AbstractListenableActionFuture.java
index 790689b4206..d0c5af06593 100644
--- a/core/src/main/java/org/elasticsearch/action/support/AbstractListenableActionFuture.java
+++ b/core/src/main/java/org/elasticsearch/action/support/AbstractListenableActionFuture.java
@@ -33,7 +33,7 @@ import java.util.List;
  */
 public abstract class AbstractListenableActionFuture<T, L> extends AdapterActionFuture<T, L> implements ListenableActionFuture<T> {
 
-    private final static ESLogger logger = Loggers.getLogger(AbstractListenableActionFuture.class);
+    private static final ESLogger logger = Loggers.getLogger(AbstractListenableActionFuture.class);
 
     final ThreadPool threadPool;
     volatile Object listeners;
diff --git a/core/src/main/java/org/elasticsearch/action/support/ActionFilter.java b/core/src/main/java/org/elasticsearch/action/support/ActionFilter.java
index d753eda4c69..f536d9e0ceb 100644
--- a/core/src/main/java/org/elasticsearch/action/support/ActionFilter.java
+++ b/core/src/main/java/org/elasticsearch/action/support/ActionFilter.java
@@ -55,7 +55,7 @@ public interface ActionFilter {
      * filter chain. This base class should serve any action filter implementations that doesn't require
      * to apply async filtering logic.
      */
-    public static abstract class Simple extends AbstractComponent implements ActionFilter {
+    public abstract static class Simple extends AbstractComponent implements ActionFilter {
 
         protected Simple(Settings settings) {
             super(settings);
diff --git a/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java b/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
index 3356d189143..12184d57cd8 100644
--- a/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
@@ -590,7 +590,7 @@ public abstract class TransportBroadcastByNodeAction<Request extends BroadcastRe
      * Can be used for implementations of {@link #shardOperation(BroadcastRequest, ShardRouting) shardOperation} for
      * which there is no shard-level return value.
      */
-    public final static class EmptyResult implements Streamable {
+    public static final class EmptyResult implements Streamable {
         public static EmptyResult INSTANCE = new EmptyResult();
 
         private EmptyResult() {
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
index 830f092178f..1dbe543ff2f 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
@@ -50,11 +50,11 @@ public class ReplicationOperation<
             ReplicaRequest extends ReplicationRequest<ReplicaRequest>,
             PrimaryResultT extends ReplicationOperation.PrimaryResult<ReplicaRequest>
         > {
-    final private ESLogger logger;
-    final private Request request;
-    final private Supplier<ClusterState> clusterStateSupplier;
-    final private String opType;
-    final private AtomicInteger totalShards = new AtomicInteger();
+    private final ESLogger logger;
+    private final Request request;
+    private final Supplier<ClusterState> clusterStateSupplier;
+    private final String opType;
+    private final AtomicInteger totalShards = new AtomicInteger();
     /**
      * The number of pending sub-operations in this operation. This is incremented when the following operations start and decremented when
      * they complete:
@@ -65,14 +65,14 @@ public class ReplicationOperation<
      * operations and the primary finishes.</li>
      * </ul>
      */
-    final private AtomicInteger pendingShards = new AtomicInteger();
-    final private AtomicInteger successfulShards = new AtomicInteger();
-    final private boolean executeOnReplicas;
-    final private boolean checkWriteConsistency;
-    final private Primary<Request, ReplicaRequest, PrimaryResultT> primary;
-    final private Replicas<ReplicaRequest> replicasProxy;
-    final private AtomicBoolean finished = new AtomicBoolean();
-    final protected ActionListener<PrimaryResultT> resultListener;
+    private final AtomicInteger pendingShards = new AtomicInteger();
+    private final AtomicInteger successfulShards = new AtomicInteger();
+    private final boolean executeOnReplicas;
+    private final boolean checkWriteConsistency;
+    private final Primary<Request, ReplicaRequest, PrimaryResultT> primary;
+    private final Replicas<ReplicaRequest> replicasProxy;
+    private final AtomicBoolean finished = new AtomicBoolean();
+    protected final ActionListener<PrimaryResultT> resultListener;
 
     private volatile PrimaryResultT primaryResult = null;
 
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java
index c444fd2cf39..adb44dd4964 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java
@@ -124,9 +124,8 @@ public abstract class ReplicationRequest<Request extends ReplicationRequest<Requ
      * @return the shardId of the shard where this operation should be executed on.
      * can be null if the shardID has not yet been resolved
      */
-    public
     @Nullable
-    ShardId shardId() {
+    public ShardId shardId() {
         return shardId;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
index 181e0e0ea74..f04e5604a6b 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
@@ -40,7 +40,7 @@ import java.io.IOException;
  */
 public class ReplicationResponse extends ActionResponse {
 
-    public final static ReplicationResponse.ShardInfo.Failure[] EMPTY = new ReplicationResponse.ShardInfo.Failure[0];
+    public static final ReplicationResponse.ShardInfo.Failure[] EMPTY = new ReplicationResponse.ShardInfo.Failure[0];
 
     private ShardInfo shardInfo;
 
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
index 87e602bf427..80aa23032c5 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
@@ -85,17 +85,17 @@ public abstract class TransportReplicationAction<
             Response extends ReplicationResponse
         > extends TransportAction<Request, Response> {
 
-    final protected TransportService transportService;
-    final protected ClusterService clusterService;
-    final protected IndicesService indicesService;
-    final private ShardStateAction shardStateAction;
-    final private WriteConsistencyLevel defaultWriteConsistencyLevel;
-    final private TransportRequestOptions transportOptions;
+    protected final TransportService transportService;
+    protected final ClusterService clusterService;
+    protected final IndicesService indicesService;
+    private final ShardStateAction shardStateAction;
+    private final WriteConsistencyLevel defaultWriteConsistencyLevel;
+    private final TransportRequestOptions transportOptions;
 
     // package private for testing
     final String transportReplicaAction;
     final String transportPrimaryAction;
-    final private ReplicasProxy replicasProxy;
+    private final ReplicasProxy replicasProxy;
 
     protected TransportReplicationAction(Settings settings, String actionName, TransportService transportService,
                                          ClusterService clusterService, IndicesService indicesService,
diff --git a/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java b/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java
index 847eee195e8..760b77469d2 100644
--- a/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java
@@ -205,7 +205,7 @@ public abstract class TransportInstanceSingleOperationAction<Request extends Ins
             });
         }
 
-        void retry(final @Nullable Throwable failure) {
+        void retry(@Nullable final Throwable failure) {
             if (observer.isTimedOut()) {
                 // we running as a last attempt after a timeout has happened. don't retry
                 Throwable listenFailure = failure;
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
index fc36e4fb671..db74e4b3731 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/BootstrapCheck.java
@@ -508,7 +508,7 @@ final class BootstrapCheck {
 
     }
 
-    static abstract class MightForkCheck implements BootstrapCheck.Check {
+    abstract static class MightForkCheck implements BootstrapCheck.Check {
 
         @Override
         public boolean check() {
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/JNAKernel32Library.java b/core/src/main/java/org/elasticsearch/bootstrap/JNAKernel32Library.java
index fbd6857d365..50dab6888b6 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/JNAKernel32Library.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/JNAKernel32Library.java
@@ -47,8 +47,8 @@ final class JNAKernel32Library {
     private List<NativeHandlerCallback> callbacks = new ArrayList<>();
 
     // Native library instance must be kept around for the same reason.
-    private final static class Holder {
-        private final static JNAKernel32Library instance = new JNAKernel32Library();
+    private static final class Holder {
+        private static final JNAKernel32Library instance = new JNAKernel32Library();
     }
 
     private JNAKernel32Library() {
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/JavaVersion.java b/core/src/main/java/org/elasticsearch/bootstrap/JavaVersion.java
index 4a434ebe6b7..63de83d88d0 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/JavaVersion.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/JavaVersion.java
@@ -62,7 +62,7 @@ public class JavaVersion implements Comparable<JavaVersion> {
         return value.matches("^0*[0-9]+(\\.[0-9]+)*$");
     }
 
-    private final static JavaVersion CURRENT = parse(System.getProperty("java.specification.version"));
+    private static final JavaVersion CURRENT = parse(System.getProperty("java.specification.version"));
 
     public static JavaVersion current() {
         return CURRENT;
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterState.java b/core/src/main/java/org/elasticsearch/cluster/ClusterState.java
index 3350169a65e..abad2e9a8e4 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterState.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterState.java
@@ -115,7 +115,7 @@ public class ClusterState implements ToXContent, Diffable<ClusterState> {
         String type();
     }
 
-    private final static Map<String, Custom> customPrototypes = new HashMap<>();
+    private static final Map<String, Custom> customPrototypes = new HashMap<>();
 
     /**
      * Register a custom index meta data factory. Make sure to call it from a static block.
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterStateObserver.java b/core/src/main/java/org/elasticsearch/cluster/ClusterStateObserver.java
index d79a00dc3fe..228ac3f41b9 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterStateObserver.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterStateObserver.java
@@ -274,7 +274,7 @@ public class ClusterStateObserver {
     }
 
 
-    public static abstract class ValidationPredicate implements ChangePredicate {
+    public abstract static class ValidationPredicate implements ChangePredicate {
 
         @Override
         public boolean apply(ClusterState previousState, ClusterState.ClusterStateStatus previousStatus, ClusterState newState, ClusterState.ClusterStateStatus newStatus) {
@@ -289,7 +289,7 @@ public class ClusterStateObserver {
         }
     }
 
-    public static abstract class EventPredicate implements ChangePredicate {
+    public abstract static class EventPredicate implements ChangePredicate {
         @Override
         public boolean apply(ClusterState previousState, ClusterState.ClusterStateStatus previousStatus, ClusterState newState, ClusterState.ClusterStateStatus newStatus) {
             return previousState != newState || previousStatus != newStatus;
@@ -298,8 +298,8 @@ public class ClusterStateObserver {
     }
 
     static class ObservingContext {
-        final public Listener listener;
-        final public ChangePredicate changePredicate;
+        public final Listener listener;
+        public final ChangePredicate changePredicate;
 
         public ObservingContext(Listener listener, ChangePredicate changePredicate) {
             this.listener = listener;
@@ -308,8 +308,8 @@ public class ClusterStateObserver {
     }
 
     static class ObservedState {
-        final public ClusterState clusterState;
-        final public ClusterState.ClusterStateStatus status;
+        public final ClusterState clusterState;
+        public final ClusterState.ClusterStateStatus status;
 
         public ObservedState(ClusterState clusterState) {
             this.clusterState = clusterState;
@@ -322,7 +322,7 @@ public class ClusterStateObserver {
         }
     }
 
-    private final static class ContextPreservingListener implements Listener {
+    private static final class ContextPreservingListener implements Listener {
         private final Listener delegate;
         private final ThreadContext.StoredContext tempContext;
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java b/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java
index ced8dd00982..633fce1ac96 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java
@@ -51,8 +51,8 @@ public interface ClusterStateTaskExecutor<T> {
      * @param <T> the type of the cluster state update task
      */
     class BatchResult<T> {
-        final public ClusterState resultingState;
-        final public Map<T, TaskResult> executionResults;
+        public final ClusterState resultingState;
+        public final Map<T, TaskResult> executionResults;
 
         /**
          * Construct an execution result instance with a correspondence between the tasks and their execution result
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java b/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java
index 3e2881134f8..c6988bbb8c8 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java
@@ -28,9 +28,9 @@ import java.util.List;
 /**
  * A task that can update the cluster state.
  */
-abstract public class ClusterStateUpdateTask implements ClusterStateTaskConfig, ClusterStateTaskExecutor<ClusterStateUpdateTask>, ClusterStateTaskListener {
+public abstract  class ClusterStateUpdateTask implements ClusterStateTaskConfig, ClusterStateTaskExecutor<ClusterStateUpdateTask>, ClusterStateTaskListener {
 
-    final private Priority priority;
+    private final Priority priority;
 
     public ClusterStateUpdateTask() {
         this(Priority.NORMAL);
@@ -41,7 +41,7 @@ abstract public class ClusterStateUpdateTask implements ClusterStateTaskConfig,
     }
 
     @Override
-    final public BatchResult<ClusterStateUpdateTask> execute(ClusterState currentState, List<ClusterStateUpdateTask> tasks) throws Exception {
+    public final BatchResult<ClusterStateUpdateTask> execute(ClusterState currentState, List<ClusterStateUpdateTask> tasks) throws Exception {
         ClusterState result = execute(currentState);
         return BatchResult.<ClusterStateUpdateTask>builder().successes(tasks).build(result);
     }
@@ -50,12 +50,12 @@ abstract public class ClusterStateUpdateTask implements ClusterStateTaskConfig,
      * Update the cluster state based on the current state. Return the *same instance* if no state
      * should be changed.
      */
-    abstract public ClusterState execute(ClusterState currentState) throws Exception;
+    public abstract  ClusterState execute(ClusterState currentState) throws Exception;
 
     /**
      * A callback called when execute fails.
      */
-    abstract public void onFailure(String source, Throwable t);
+    public abstract  void onFailure(String source, Throwable t);
 
     /**
      * If the cluster state update task wasn't processed by the provided timeout, call
diff --git a/core/src/main/java/org/elasticsearch/cluster/DiffableUtils.java b/core/src/main/java/org/elasticsearch/cluster/DiffableUtils.java
index fb9d7159105..1a3557890dd 100644
--- a/core/src/main/java/org/elasticsearch/cluster/DiffableUtils.java
+++ b/core/src/main/java/org/elasticsearch/cluster/DiffableUtils.java
@@ -330,7 +330,7 @@ public final class DiffableUtils {
      * @param <T> the type of map values
      * @param <M> the map implementation type
      */
-    public static abstract class MapDiff<K, T, M> implements Diff<M> {
+    public abstract static class MapDiff<K, T, M> implements Diff<M> {
 
         protected final List<K> deletes;
         protected final Map<K, Diff<T>> diffs; // incremental updates
@@ -534,7 +534,7 @@ public final class DiffableUtils {
      * @param <K> type of map keys
      * @param <V> type of map values
      */
-    public static abstract class DiffableValueSerializer<K, V extends Diffable<V>> implements ValueSerializer<K, V> {
+    public abstract static class DiffableValueSerializer<K, V extends Diffable<V>> implements ValueSerializer<K, V> {
         private static final DiffableValueSerializer WRITE_ONLY_INSTANCE = new DiffableValueSerializer() {
             @Override
             public Object read(StreamInput in, Object key) throws IOException {
@@ -577,7 +577,7 @@ public final class DiffableUtils {
      * @param <K> type of map keys
      * @param <V> type of map values
      */
-    public static abstract class NonDiffableValueSerializer<K, V> implements ValueSerializer<K, V> {
+    public abstract static class NonDiffableValueSerializer<K, V> implements ValueSerializer<K, V> {
         @Override
         public boolean supportsDiffableValues() {
             return false;
diff --git a/core/src/main/java/org/elasticsearch/cluster/EmptyClusterInfoService.java b/core/src/main/java/org/elasticsearch/cluster/EmptyClusterInfoService.java
index cc65106946a..b552508532c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/EmptyClusterInfoService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/EmptyClusterInfoService.java
@@ -26,7 +26,7 @@ import org.elasticsearch.common.settings.Settings;
  * ClusterInfoService that provides empty maps for disk usage and shard sizes
  */
 public class EmptyClusterInfoService extends AbstractComponent implements ClusterInfoService {
-    public final static EmptyClusterInfoService INSTANCE = new EmptyClusterInfoService();
+    public static final EmptyClusterInfoService INSTANCE = new EmptyClusterInfoService();
 
     private EmptyClusterInfoService() {
         super(Settings.EMPTY);
diff --git a/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java b/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
index 404ae57d5f7..5fe42a7b163 100644
--- a/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
@@ -57,7 +57,7 @@ public class NodeConnectionsService extends AbstractLifecycleComponent<NodeConne
     // if a node doesn't appear in this list it shouldn't be monitored
     private ConcurrentMap<DiscoveryNode, Integer> nodes = ConcurrentCollections.newConcurrentMap();
 
-    final private KeyedLock<DiscoveryNode> nodeLocks = new KeyedLock<>();
+    private final KeyedLock<DiscoveryNode> nodeLocks = new KeyedLock<>();
 
     private final TimeValue reconnectInterval;
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
index 6130ed2317a..94dd3c63da5 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java
@@ -54,7 +54,7 @@ import java.util.function.BiFunction;
  * tombstones remain in the cluster state for a fixed period of time, after which
  * they are purged.
  */
-final public class IndexGraveyard implements MetaData.Custom {
+public final class IndexGraveyard implements MetaData.Custom {
 
     /**
      * Setting for the maximum tombstones allowed in the cluster state;
@@ -188,7 +188,7 @@ final public class IndexGraveyard implements MetaData.Custom {
     /**
      * A class to build an IndexGraveyard.
      */
-    final public static class Builder {
+    public static final class Builder {
         private List<Tombstone> tombstones;
         private int numPurged = -1;
         private final long currentTime = System.currentTimeMillis();
@@ -273,7 +273,7 @@ final public class IndexGraveyard implements MetaData.Custom {
     /**
      * A class representing a diff of two IndexGraveyard objects.
      */
-    final public static class IndexGraveyardDiff implements Diff<MetaData.Custom> {
+    public static final class IndexGraveyardDiff implements Diff<MetaData.Custom> {
 
         private final List<Tombstone> added;
         private final int removedCount;
@@ -354,7 +354,7 @@ final public class IndexGraveyard implements MetaData.Custom {
     /**
      * An individual tombstone entry for representing a deleted index.
      */
-    final public static class Tombstone implements ToXContent, Writeable {
+    public static final class Tombstone implements ToXContent, Writeable {
 
         private static final String INDEX_KEY = "index";
         private static final String DELETE_DATE_IN_MILLIS_KEY = "delete_date_in_millis";
@@ -449,7 +449,7 @@ final public class IndexGraveyard implements MetaData.Custom {
         /**
          * A builder for building tombstone entries.
          */
-        final private static class Builder {
+        private static final class Builder {
             private Index index;
             private long deleteDateInMillis = -1L;
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
index 14d00d3fc64..b49b893f232 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
@@ -255,7 +255,7 @@ public class IndexMetaData implements Diffable<IndexMetaData>, FromXContentBuild
 
     private final ImmutableOpenIntMap<Set<String>> activeAllocationIds;
 
-    private transient final int totalNumberOfShards;
+    private final transient int totalNumberOfShards;
 
     private final DiscoveryNodeFilters requireFilters;
     private final DiscoveryNodeFilters includeFilters;
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
index 4eefac0f17f..df53395fe27 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
@@ -488,7 +488,7 @@ public class IndexNameExpressionResolver extends AbstractComponent {
         return false;
     }
 
-    final static class Context {
+    static final class Context {
 
         private final ClusterState state;
         private final IndicesOptions options;
@@ -551,7 +551,7 @@ public class IndexNameExpressionResolver extends AbstractComponent {
     /**
      * Resolves alias/index name expressions with wildcards into the corresponding concrete indices/aliases
      */
-    final static class WildcardExpressionResolver implements ExpressionResolver {
+    static final class WildcardExpressionResolver implements ExpressionResolver {
 
         @Override
         public List<String> resolve(Context context, List<String> expressions) {
@@ -738,7 +738,7 @@ public class IndexNameExpressionResolver extends AbstractComponent {
         }
     }
 
-    final static class DateMathExpressionResolver implements ExpressionResolver {
+    static final class DateMathExpressionResolver implements ExpressionResolver {
 
         private static final String EXPRESSION_LEFT_BOUND = "<";
         private static final String EXPRESSION_RIGHT_BOUND = ">";
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
index a8a9e1e1251..6fdecf542f5 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
@@ -1187,7 +1187,7 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
         }
     }
 
-    private final static ToXContent.Params FORMAT_PARAMS;
+    private static final ToXContent.Params FORMAT_PARAMS;
     static {
         Map<String, String> params = new HashMap<>(2);
         params.put("binary", "true");
@@ -1198,7 +1198,7 @@ public class MetaData implements Iterable<IndexMetaData>, Diffable<MetaData>, Fr
     /**
      * State format for {@link MetaData} to write to and load from disk
      */
-    public final static MetaDataStateFormat<MetaData> FORMAT = new MetaDataStateFormat<MetaData>(XContentType.SMILE, GLOBAL_STATE_FILE_PREFIX) {
+    public static final MetaDataStateFormat<MetaData> FORMAT = new MetaDataStateFormat<MetaData>(XContentType.SMILE, GLOBAL_STATE_FILE_PREFIX) {
 
         @Override
         public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
index 456675abcf1..4f7aa68d7de 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java
@@ -97,7 +97,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_C
  */
 public class MetaDataCreateIndexService extends AbstractComponent {
 
-    public final static int MAX_INDEX_NAME_BYTES = 255;
+    public static final int MAX_INDEX_NAME_BYTES = 255;
     private static final DefaultIndexTemplateFilter DEFAULT_INDEX_TEMPLATE_FILTER = new DefaultIndexTemplateFilter();
 
     private final ClusterService clusterService;
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
index be09f17f866..cddf6f98a54 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java
@@ -60,7 +60,7 @@ public class IndexShardRoutingTable implements Iterable<ShardRouting> {
     final List<ShardRouting> shards;
     final List<ShardRouting> activeShards;
     final List<ShardRouting> assignedShards;
-    final static List<ShardRouting> NO_SHARDS = Collections.emptyList();
+    static final List<ShardRouting> NO_SHARDS = Collections.emptyList();
     final boolean allShardsStarted;
 
     private volatile Map<AttributesKey, AttributesRoutings> activeShardsByAttributes = emptyMap();
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
index b4a04e4d104..8403f45a550 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java
@@ -80,7 +80,8 @@ public class RoutingNode implements Iterable<ShardRouting> {
         return this.node;
     }
 
-    public @Nullable ShardRouting getByShardId(ShardId id) {
+    @Nullable
+    public ShardRouting getByShardId(ShardId id) {
         return shards.get(id);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java
index 0fb27a80154..4ed8c487746 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AbstractAllocateAllocationCommand.java
@@ -62,7 +62,7 @@ public abstract class AbstractAllocateAllocationCommand implements AllocationCom
     /**
      * Works around ObjectParser not supporting constructor arguments.
      */
-    protected static abstract class Builder<T extends AbstractAllocateAllocationCommand> {
+    protected abstract static class Builder<T extends AbstractAllocateAllocationCommand> {
         protected String index;
         protected int shard = -1;
         protected String node;
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/BasePrimaryAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/BasePrimaryAllocationCommand.java
index 2eb3af9d4f9..f4dc4fba4b8 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/BasePrimaryAllocationCommand.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/BasePrimaryAllocationCommand.java
@@ -71,7 +71,7 @@ public abstract class BasePrimaryAllocationCommand extends AbstractAllocateAlloc
         return acceptDataLoss;
     }
 
-    protected static abstract class Builder<T extends BasePrimaryAllocationCommand> extends AbstractAllocateAllocationCommand.Builder<T> {
+    protected abstract static class Builder<T extends BasePrimaryAllocationCommand> extends AbstractAllocateAllocationCommand.Builder<T> {
         protected boolean acceptDataLoss;
 
         public void setAcceptDataLoss(boolean acceptDataLoss) {
diff --git a/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
index 50c91665485..1480ef9a504 100644
--- a/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
@@ -154,17 +154,17 @@ public class ClusterService extends AbstractLifecycleComponent<ClusterService> {
         this.slowTaskLoggingThreshold = slowTaskLoggingThreshold;
     }
 
-    synchronized public void setClusterStatePublisher(BiConsumer<ClusterChangedEvent, Discovery.AckListener> publisher) {
+    public synchronized void setClusterStatePublisher(BiConsumer<ClusterChangedEvent, Discovery.AckListener> publisher) {
         clusterStatePublisher = publisher;
     }
 
-    synchronized public void setLocalNode(DiscoveryNode localNode) {
+    public synchronized void setLocalNode(DiscoveryNode localNode) {
         assert clusterState.nodes().getLocalNodeId() == null : "local node is already set";
         DiscoveryNodes.Builder nodeBuilder = DiscoveryNodes.builder(clusterState.nodes()).put(localNode).localNodeId(localNode.getId());
         this.clusterState = ClusterState.builder(clusterState).nodes(nodeBuilder).build();
     }
 
-    synchronized public void setNodeConnectionsService(NodeConnectionsService nodeConnectionsService) {
+    public synchronized void setNodeConnectionsService(NodeConnectionsService nodeConnectionsService) {
         assert this.nodeConnectionsService == null : "nodeConnectionsService is already set";
         this.nodeConnectionsService = nodeConnectionsService;
     }
@@ -172,7 +172,7 @@ public class ClusterService extends AbstractLifecycleComponent<ClusterService> {
     /**
      * Adds an initial block to be set on the first cluster state created.
      */
-    synchronized public void addInitialStateBlock(ClusterBlock block) throws IllegalStateException {
+    public synchronized void addInitialStateBlock(ClusterBlock block) throws IllegalStateException {
         if (lifecycle.started()) {
             throw new IllegalStateException("can't set initial block when started");
         }
@@ -182,14 +182,14 @@ public class ClusterService extends AbstractLifecycleComponent<ClusterService> {
     /**
      * Remove an initial block to be set on the first cluster state created.
      */
-    synchronized public void removeInitialStateBlock(ClusterBlock block) throws IllegalStateException {
+    public synchronized void removeInitialStateBlock(ClusterBlock block) throws IllegalStateException {
         removeInitialStateBlock(block.id());
     }
 
     /**
      * Remove an initial block to be set on the first cluster state created.
      */
-    synchronized public void removeInitialStateBlock(int blockId) throws IllegalStateException {
+    public synchronized void removeInitialStateBlock(int blockId) throws IllegalStateException {
         if (lifecycle.started()) {
             throw new IllegalStateException("can't set initial block when started");
         }
@@ -197,7 +197,7 @@ public class ClusterService extends AbstractLifecycleComponent<ClusterService> {
     }
 
     @Override
-    synchronized protected void doStart() {
+    protected synchronized void doStart() {
         Objects.requireNonNull(clusterStatePublisher, "please set a cluster state publisher before starting");
         Objects.requireNonNull(clusterState.nodes().getLocalNode(), "please set the local node before starting");
         Objects.requireNonNull(nodeConnectionsService, "please set the node connection service before starting");
@@ -209,7 +209,7 @@ public class ClusterService extends AbstractLifecycleComponent<ClusterService> {
     }
 
     @Override
-    synchronized protected void doStop() {
+    protected synchronized void doStop() {
         for (NotifyTimeout onGoingTimeout : onGoingTimeouts) {
             onGoingTimeout.cancel();
             try {
@@ -230,7 +230,7 @@ public class ClusterService extends AbstractLifecycleComponent<ClusterService> {
     }
 
     @Override
-    synchronized protected void doClose() {
+    protected synchronized void doClose() {
     }
 
     /**
@@ -497,7 +497,7 @@ public class ClusterService extends AbstractLifecycleComponent<ClusterService> {
         return clusterName;
     }
 
-    static abstract class SourcePrioritizedRunnable extends PrioritizedRunnable {
+    abstract static class SourcePrioritizedRunnable extends PrioritizedRunnable {
         protected final String source;
 
         public SourcePrioritizedRunnable(Priority priority, String source) {
@@ -959,7 +959,7 @@ public class ClusterService extends AbstractLifecycleComponent<ClusterService> {
 
     private static class DelegetingAckListener implements Discovery.AckListener {
 
-        final private List<Discovery.AckListener> listeners;
+        private final List<Discovery.AckListener> listeners;
 
         private DelegetingAckListener(List<Discovery.AckListener> listeners) {
             this.listeners = listeners;
diff --git a/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java b/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java
index 3c0579c87e1..53167686736 100644
--- a/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java
+++ b/core/src/main/java/org/elasticsearch/common/collect/CopyOnWriteHashMap.java
@@ -75,7 +75,7 @@ public final class CopyOnWriteHashMap<K, V> extends AbstractMap<K, V> {
     /**
      * Abstraction of a node, implemented by both inner and leaf nodes.
      */
-    private static abstract class Node<K, V> {
+    private abstract static class Node<K, V> {
 
         /**
          * Recursively get the key with the given hash.
diff --git a/core/src/main/java/org/elasticsearch/common/collect/HppcMaps.java b/core/src/main/java/org/elasticsearch/common/collect/HppcMaps.java
index 5dc17bedb39..55fc67831e3 100644
--- a/core/src/main/java/org/elasticsearch/common/collect/HppcMaps.java
+++ b/core/src/main/java/org/elasticsearch/common/collect/HppcMaps.java
@@ -132,8 +132,8 @@ public final class HppcMaps {
         };
     }
 
-    public final static class Object {
-        public final static class Integer {
+    public static final class Object {
+        public static final class Integer {
             public static <V> ObjectIntHashMap<V> ensureNoNullKeys(int capacity, float loadFactor) {
                 return new ObjectIntHashMap<V>(capacity, loadFactor) {
                     @Override
diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoDistance.java b/core/src/main/java/org/elasticsearch/common/geo/GeoDistance.java
index 0eda7b92412..c6900e2ae4d 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/GeoDistance.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/GeoDistance.java
@@ -350,7 +350,7 @@ public enum GeoDistance implements Writeable {
      * Basic implementation of {@link FixedSourceDistance}. This class keeps the basic parameters for a distance
      * functions based on a fixed source. Namely latitude, longitude and unit.
      */
-    public static abstract class FixedSourceDistanceBase implements FixedSourceDistance {
+    public abstract static class FixedSourceDistanceBase implements FixedSourceDistance {
         protected final double sourceLatitude;
         protected final double sourceLongitude;
         protected final DistanceUnit unit;
diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java b/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java
index 9982a08f17f..d8fc226b617 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java
@@ -195,7 +195,7 @@ public class GeoHashUtils {
      * @param dy      delta of the second grid coordinate (must be -1, 0 or +1)
      * @return geohash of the defined cell
      */
-    public final static String neighbor(String geohash, int level, int dx, int dy) {
+    public static final String neighbor(String geohash, int level, int dx, int dy) {
         int cell = BASE_32_STRING.indexOf(geohash.charAt(level -1));
 
         // Decoding the Geohash bit pattern to determine grid coordinates
diff --git a/core/src/main/java/org/elasticsearch/common/inject/internal/Errors.java b/core/src/main/java/org/elasticsearch/common/inject/internal/Errors.java
index 795ff5380aa..24a2b4b79ec 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/internal/Errors.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/internal/Errors.java
@@ -551,7 +551,7 @@ public final class Errors {
         return root.errors == null ? 0 : root.errors.size();
     }
 
-    private static abstract class Converter<T> {
+    private abstract static class Converter<T> {
 
         final Class<T> type;
 
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
index bb9af656ea8..4ee9c80b891 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
@@ -408,7 +408,7 @@ public abstract class StreamOutput extends OutputStream {
         void write(StreamOutput o, Object value) throws IOException;
     }
 
-    private final static Map<Class<?>, Writer> WRITERS;
+    private static final Map<Class<?>, Writer> WRITERS;
 
     static {
         Map<Class<?>, Writer> writers = new HashMap<>();
diff --git a/core/src/main/java/org/elasticsearch/common/logging/Loggers.java b/core/src/main/java/org/elasticsearch/common/logging/Loggers.java
index 674522589ae..613193baee5 100644
--- a/core/src/main/java/org/elasticsearch/common/logging/Loggers.java
+++ b/core/src/main/java/org/elasticsearch/common/logging/Loggers.java
@@ -38,7 +38,7 @@ import static org.elasticsearch.common.util.CollectionUtils.asArrayList;
  */
 public class Loggers {
 
-    private final static String commonPrefix = System.getProperty("es.logger.prefix", "org.elasticsearch.");
+    private static final String commonPrefix = System.getProperty("es.logger.prefix", "org.elasticsearch.");
 
     public static final String SPACE = " ";
 
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
index 8508a8a2e40..39e67ce6458 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/Lucene.java
@@ -245,14 +245,14 @@ public class Lucene {
     /**
      * Wraps <code>delegate</code> with count based early termination collector with a threshold of <code>maxCountHits</code>
      */
-    public final static EarlyTerminatingCollector wrapCountBasedEarlyTerminatingCollector(final Collector delegate, int maxCountHits) {
+    public static final EarlyTerminatingCollector wrapCountBasedEarlyTerminatingCollector(final Collector delegate, int maxCountHits) {
         return new EarlyTerminatingCollector(delegate, maxCountHits);
     }
 
     /**
      * Wraps <code>delegate</code> with a time limited collector with a timeout of <code>timeoutInMillis</code>
      */
-    public final static TimeLimitingCollector wrapTimeLimitingCollector(final Collector delegate, final Counter counter, long timeoutInMillis) {
+    public static final TimeLimitingCollector wrapTimeLimitingCollector(final Collector delegate, final Counter counter, long timeoutInMillis) {
         return new TimeLimitingCollector(delegate, counter, timeoutInMillis);
     }
 
@@ -510,7 +510,7 @@ public class Lucene {
      * This exception is thrown when {@link org.elasticsearch.common.lucene.Lucene.EarlyTerminatingCollector}
      * reaches early termination
      * */
-    public final static class EarlyTerminationException extends ElasticsearchException {
+    public static final class EarlyTerminationException extends ElasticsearchException {
 
         public EarlyTerminationException(String msg) {
             super(msg);
@@ -525,7 +525,7 @@ public class Lucene {
      * A collector that terminates early by throwing {@link org.elasticsearch.common.lucene.Lucene.EarlyTerminationException}
      * when count of matched documents has reached <code>maxCountHits</code>
      */
-    public final static class EarlyTerminatingCollector extends SimpleCollector {
+    public static final class EarlyTerminatingCollector extends SimpleCollector {
 
         private final int maxCountHits;
         private final Collector delegate;
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/index/ElasticsearchDirectoryReader.java b/core/src/main/java/org/elasticsearch/common/lucene/index/ElasticsearchDirectoryReader.java
index c4543435917..b9caecea965 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/index/ElasticsearchDirectoryReader.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/index/ElasticsearchDirectoryReader.java
@@ -66,7 +66,7 @@ public final class ElasticsearchDirectoryReader extends FilterDirectoryReader {
         return new ElasticsearchDirectoryReader(reader, new SubReaderWrapper(shardId), shardId);
     }
 
-    private final static class SubReaderWrapper extends FilterDirectoryReader.SubReaderWrapper {
+    private static final class SubReaderWrapper extends FilterDirectoryReader.SubReaderWrapper {
         private final ShardId shardId;
         SubReaderWrapper(ShardId shardId) {
             this.shardId = shardId;
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/index/FilterableTermsEnum.java b/core/src/main/java/org/elasticsearch/common/lucene/index/FilterableTermsEnum.java
index ded912cf9e8..0ff0008a430 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/index/FilterableTermsEnum.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/index/FilterableTermsEnum.java
@@ -60,7 +60,7 @@ public class FilterableTermsEnum extends TermsEnum {
     }
 
     static final String UNSUPPORTED_MESSAGE = "This TermsEnum only supports #seekExact(BytesRef) as well as #docFreq() and #totalTermFreq()";
-    protected final static int NOT_FOUND = -1;
+    protected static final int NOT_FOUND = -1;
     private final Holder[] enums;
     protected int currentDocFreq = 0;
     protected long currentTotalTermFreq = 0;
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java
index 3927dcd518e..c8698e8223d 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FiltersFunctionScoreQuery.java
@@ -102,7 +102,7 @@ public class FiltersFunctionScoreQuery extends Query {
     final float maxBoost;
     private final Float minScore;
 
-    final protected CombineFunction combineFunction;
+    protected final CombineFunction combineFunction;
 
     public FiltersFunctionScoreQuery(Query subQuery, ScoreMode scoreMode, FilterFunction[] filterFunctions, float maxBoost, Float minScore, CombineFunction combineFunction) {
         this.subQuery = subQuery;
diff --git a/core/src/main/java/org/elasticsearch/common/rounding/Rounding.java b/core/src/main/java/org/elasticsearch/common/rounding/Rounding.java
index 7e94ebb9ee6..7905ee29d73 100644
--- a/core/src/main/java/org/elasticsearch/common/rounding/Rounding.java
+++ b/core/src/main/java/org/elasticsearch/common/rounding/Rounding.java
@@ -76,7 +76,7 @@ public abstract class Rounding implements Streamable {
      */
     public static class Interval extends Rounding {
 
-        final static byte ID = 0;
+        static final byte ID = 0;
 
         public static final ParseField INTERVAL_FIELD = new ParseField("interval");
 
@@ -157,7 +157,7 @@ public abstract class Rounding implements Streamable {
 
     public static class FactorRounding extends Rounding {
 
-        final static byte ID = 7;
+        static final byte ID = 7;
 
         public static final ParseField FACTOR_FIELD = new ParseField("factor");
 
@@ -226,7 +226,7 @@ public abstract class Rounding implements Streamable {
 
     public static class OffsetRounding extends Rounding {
 
-        final static byte ID = 8;
+        static final byte ID = 8;
 
         public static final ParseField OFFSET_FIELD = new ParseField("offset");
 
diff --git a/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java b/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java
index 46362687e01..e52769d7f36 100644
--- a/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java
+++ b/core/src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java
@@ -195,7 +195,7 @@ public abstract class TimeZoneRounding extends Rounding {
 
     static class TimeIntervalRounding extends TimeZoneRounding {
 
-        final static byte ID = 2;
+        static final byte ID = 2;
 
         private long interval;
         private DateTimeZone timeZone;
diff --git a/core/src/main/java/org/elasticsearch/common/util/BigArrays.java b/core/src/main/java/org/elasticsearch/common/util/BigArrays.java
index c2c8e242acc..da49aac5ed0 100644
--- a/core/src/main/java/org/elasticsearch/common/util/BigArrays.java
+++ b/core/src/main/java/org/elasticsearch/common/util/BigArrays.java
@@ -89,7 +89,7 @@ public class BigArrays implements Releasable {
         recycler.close();
     }
 
-    private static abstract class AbstractArrayWrapper extends AbstractArray implements BigArray {
+    private abstract static class AbstractArrayWrapper extends AbstractArray implements BigArray {
 
         protected static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(ByteArrayWrapper.class);
 
diff --git a/core/src/main/java/org/elasticsearch/common/util/ExtensionPoint.java b/core/src/main/java/org/elasticsearch/common/util/ExtensionPoint.java
index db167c5daea..490209bd8d2 100644
--- a/core/src/main/java/org/elasticsearch/common/util/ExtensionPoint.java
+++ b/core/src/main/java/org/elasticsearch/common/util/ExtensionPoint.java
@@ -162,7 +162,7 @@ public abstract class ExtensionPoint {
     /**
      * A set based extension point which allows to register extended classes that might be used to chain additional functionality etc.
      */
-    public final static class ClassSet<T> extends ExtensionPoint {
+    public static final class ClassSet<T> extends ExtensionPoint {
         protected final Class<T> extensionClass;
         private final Set<Class<? extends T>> extensions = new HashSet<>();
 
@@ -205,7 +205,7 @@ public abstract class ExtensionPoint {
      * A an instance of a map, mapping one instance value to another. Both key and value are instances, not classes
      * like with other extension points.
      */
-    public final static class InstanceMap<K, V> extends ExtensionPoint {
+    public static final class InstanceMap<K, V> extends ExtensionPoint {
         private final Map<K, V> map = new HashMap<>();
         private final Class<K> keyType;
         private final Class<V> valueType;
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/KeyedLock.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/KeyedLock.java
index 0007903be02..e9b67c65657 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/KeyedLock.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/KeyedLock.java
@@ -110,7 +110,7 @@ public class KeyedLock<T> {
     }
 
     @SuppressWarnings("serial")
-    private final static class KeyLock extends ReentrantLock {
+    private static final class KeyLock extends ReentrantLock {
         KeyLock(boolean fair) {
             super(fair);
         }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
index 54b3dd47820..1d8ce366b96 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java
@@ -52,7 +52,7 @@ import org.joda.time.format.ISODateTimeFormat;
  */
 public final class XContentBuilder implements BytesStream, Releasable {
 
-    public final static DateTimeFormatter defaultDatePrinter = ISODateTimeFormat.dateTime().withZone(DateTimeZone.UTC);
+    public static final DateTimeFormatter defaultDatePrinter = ISODateTimeFormat.dateTime().withZone(DateTimeZone.UTC);
 
     public static XContentBuilder builder(XContent xContent) throws IOException {
         return new XContentBuilder(xContent, new BytesStreamOutput());
@@ -809,7 +809,7 @@ public final class XContentBuilder implements BytesStream, Releasable {
         void write(XContentGenerator g, Object v) throws IOException;
     }
 
-    private final static Map<Class<?>, Writer> MAP;
+    private static final Map<Class<?>, Writer> MAP;
 
     static {
         Map<Class<?>, Writer> map = new HashMap<>();
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java
index 901a9a7d717..c8715d03c3e 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java
@@ -45,8 +45,8 @@ public class CborXContent implements XContent {
         return XContentBuilder.builder(cborXContent);
     }
 
-    final static CBORFactory cborFactory;
-    public final static CborXContent cborXContent;
+    static final CBORFactory cborFactory;
+    public static final CborXContent cborXContent;
 
     static {
         cborFactory = new CBORFactory();
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java
index c989aa4a28b..4828d8a752e 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java
@@ -45,10 +45,10 @@ public class JsonXContent implements XContent {
         return XContentBuilder.builder(jsonXContent);
     }
 
-    private final static JsonFactory jsonFactory;
-    public final static String JSON_ALLOW_UNQUOTED_FIELD_NAMES = "elasticsearch.json.allow_unquoted_field_names";
-    public final static JsonXContent jsonXContent;
-    public final static boolean unquotedFieldNamesSet;
+    private static final JsonFactory jsonFactory;
+    public static final String JSON_ALLOW_UNQUOTED_FIELD_NAMES = "elasticsearch.json.allow_unquoted_field_names";
+    public static final JsonXContent jsonXContent;
+    public static final boolean unquotedFieldNamesSet;
 
     static {
         jsonFactory = new JsonFactory();
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java
index f1bc879f26f..629f8612ea6 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java
@@ -45,8 +45,8 @@ public class SmileXContent implements XContent {
         return XContentBuilder.builder(smileXContent);
     }
 
-    final static SmileFactory smileFactory;
-    public final static SmileXContent smileXContent;
+    static final SmileFactory smileFactory;
+    public static final SmileXContent smileXContent;
 
     static {
         smileFactory = new SmileFactory();
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java
index ce6e0e0e37b..27a0dd46e00 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java
@@ -44,8 +44,8 @@ public class YamlXContent implements XContent {
         return XContentBuilder.builder(yamlXContent);
     }
 
-    final static YAMLFactory yamlFactory;
-    public final static YamlXContent yamlXContent;
+    static final YAMLFactory yamlFactory;
+    public static final YamlXContent yamlXContent;
 
     static {
         yamlFactory = new YAMLFactory();
diff --git a/core/src/main/java/org/elasticsearch/discovery/DiscoverySettings.java b/core/src/main/java/org/elasticsearch/discovery/DiscoverySettings.java
index ca7ab342cd5..6f5a6c9a745 100644
--- a/core/src/main/java/org/elasticsearch/discovery/DiscoverySettings.java
+++ b/core/src/main/java/org/elasticsearch/discovery/DiscoverySettings.java
@@ -36,9 +36,9 @@ import java.util.EnumSet;
  */
 public class DiscoverySettings extends AbstractComponent {
 
-    public final static int NO_MASTER_BLOCK_ID = 2;
-    public final static ClusterBlock NO_MASTER_BLOCK_ALL = new ClusterBlock(NO_MASTER_BLOCK_ID, "no master", true, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL);
-    public final static ClusterBlock NO_MASTER_BLOCK_WRITES = new ClusterBlock(NO_MASTER_BLOCK_ID, "no master", true, false, RestStatus.SERVICE_UNAVAILABLE, EnumSet.of(ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA_WRITE));
+    public static final int NO_MASTER_BLOCK_ID = 2;
+    public static final ClusterBlock NO_MASTER_BLOCK_ALL = new ClusterBlock(NO_MASTER_BLOCK_ID, "no master", true, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL);
+    public static final ClusterBlock NO_MASTER_BLOCK_WRITES = new ClusterBlock(NO_MASTER_BLOCK_ID, "no master", true, false, RestStatus.SERVICE_UNAVAILABLE, EnumSet.of(ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA_WRITE));
     /**
      * sets the timeout for a complete publishing cycle, including both sending and committing. the master
      * will continue to process the next cluster state update after this time has elapsed
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
index 9659434e03d..f4551ecb016 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
@@ -344,7 +344,7 @@ public class NodeJoinController extends AbstractComponent {
 
     static class JoinTaskListener implements ClusterStateTaskListener {
         final List<MembershipAction.JoinCallback> callbacks;
-        final private ESLogger logger;
+        private final ESLogger logger;
 
         JoinTaskListener(MembershipAction.JoinCallback callback, ESLogger logger) {
             this(Collections.singletonList(callback), logger);
@@ -379,12 +379,12 @@ public class NodeJoinController extends AbstractComponent {
     }
 
     // a task indicated that the current node should become master, if no current master is known
-    private final static DiscoveryNode BECOME_MASTER_TASK = new DiscoveryNode("_BECOME_MASTER_TASK_", DummyTransportAddress.INSTANCE,
+    private static final DiscoveryNode BECOME_MASTER_TASK = new DiscoveryNode("_BECOME_MASTER_TASK_", DummyTransportAddress.INSTANCE,
         Collections.emptyMap(), Collections.emptySet(), Version.CURRENT);
 
     // a task that is used to process pending joins without explicitly becoming a master and listening to the results
     // this task is used when election is stop without the local node becoming a master per se (though it might
-    private final static DiscoveryNode FINISH_ELECTION_NOT_MASTER_TASK = new DiscoveryNode("_NOT_MASTER_TASK_",
+    private static final DiscoveryNode FINISH_ELECTION_NOT_MASTER_TASK = new DiscoveryNode("_NOT_MASTER_TASK_",
         DummyTransportAddress.INSTANCE, Collections.emptyMap(), Collections.emptySet(), Version.CURRENT);
 
     class JoinTaskExecutor implements ClusterStateTaskExecutor<DiscoveryNode> {
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
index 54ef9928585..eca34389d90 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
@@ -88,25 +88,25 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
  */
 public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implements Discovery, PingContextProvider {
 
-    public final static Setting<TimeValue> PING_TIMEOUT_SETTING =
+    public static final Setting<TimeValue> PING_TIMEOUT_SETTING =
         Setting.positiveTimeSetting("discovery.zen.ping_timeout", timeValueSeconds(3), Property.NodeScope);
-    public final static Setting<TimeValue> JOIN_TIMEOUT_SETTING =
+    public static final Setting<TimeValue> JOIN_TIMEOUT_SETTING =
         Setting.timeSetting("discovery.zen.join_timeout",
             settings -> TimeValue.timeValueMillis(PING_TIMEOUT_SETTING.get(settings).millis() * 20).toString(),
             TimeValue.timeValueMillis(0), Property.NodeScope);
-    public final static Setting<Integer> JOIN_RETRY_ATTEMPTS_SETTING =
+    public static final Setting<Integer> JOIN_RETRY_ATTEMPTS_SETTING =
         Setting.intSetting("discovery.zen.join_retry_attempts", 3, 1, Property.NodeScope);
-    public final static Setting<TimeValue> JOIN_RETRY_DELAY_SETTING =
+    public static final Setting<TimeValue> JOIN_RETRY_DELAY_SETTING =
         Setting.positiveTimeSetting("discovery.zen.join_retry_delay", TimeValue.timeValueMillis(100), Property.NodeScope);
-    public final static Setting<Integer> MAX_PINGS_FROM_ANOTHER_MASTER_SETTING =
+    public static final Setting<Integer> MAX_PINGS_FROM_ANOTHER_MASTER_SETTING =
         Setting.intSetting("discovery.zen.max_pings_from_another_master", 3, 1, Property.NodeScope);
-    public final static Setting<Boolean> SEND_LEAVE_REQUEST_SETTING =
+    public static final Setting<Boolean> SEND_LEAVE_REQUEST_SETTING =
         Setting.boolSetting("discovery.zen.send_leave_request", true, Property.NodeScope);
-    public final static Setting<TimeValue> MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING =
+    public static final Setting<TimeValue> MASTER_ELECTION_WAIT_FOR_JOINS_TIMEOUT_SETTING =
         Setting.timeSetting("discovery.zen.master_election.wait_for_joins_timeout",
             settings -> TimeValue.timeValueMillis(JOIN_TIMEOUT_SETTING.get(settings).millis() / 2).toString(), TimeValue.timeValueMillis(0),
             Property.NodeScope);
-    public final static Setting<Boolean> MASTER_ELECTION_IGNORE_NON_MASTER_PINGS_SETTING =
+    public static final Setting<Boolean> MASTER_ELECTION_IGNORE_NON_MASTER_PINGS_SETTING =
             Setting.boolSetting("discovery.zen.master_election.ignore_non_master_pings", false, Property.NodeScope);
 
     public static final String DISCOVERY_REJOIN_ACTION_NAME = "internal:discovery/zen/rejoin";
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java b/core/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java
index 48be9f64d5c..7714f34d0e2 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java
@@ -543,11 +543,11 @@ public class PublishClusterStateAction extends AbstractComponent {
             }
         }
 
-        synchronized public boolean isCommitted() {
+        public synchronized boolean isCommitted() {
             return committed;
         }
 
-        synchronized public void onNodeSendAck(DiscoveryNode node) {
+        public synchronized void onNodeSendAck(DiscoveryNode node) {
             if (committed) {
                 assert sendAckedBeforeCommit.isEmpty();
                 sendCommitToNode(node, clusterState, this);
@@ -570,7 +570,7 @@ public class PublishClusterStateAction extends AbstractComponent {
          * check if enough master node responded to commit the change. fails the commit
          * if there are no more pending master nodes but not enough acks to commit.
          */
-        synchronized private void checkForCommitOrFailIfNoPending(DiscoveryNode masterNode) {
+        private synchronized void checkForCommitOrFailIfNoPending(DiscoveryNode masterNode) {
             logger.trace("master node {} acked cluster state version [{}]. processing ... (current pending [{}], needed [{}])",
                     masterNode, clusterState.version(), pendingMasterNodes, neededMastersToCommit);
             neededMastersToCommit--;
@@ -585,14 +585,14 @@ public class PublishClusterStateAction extends AbstractComponent {
             decrementPendingMasterAcksAndChangeForFailure();
         }
 
-        synchronized private void decrementPendingMasterAcksAndChangeForFailure() {
+        private synchronized void decrementPendingMasterAcksAndChangeForFailure() {
             pendingMasterNodes--;
             if (pendingMasterNodes == 0 && neededMastersToCommit > 0) {
                 markAsFailed("no more pending master nodes, but failed to reach needed acks ([" + neededMastersToCommit + "] left)");
             }
         }
 
-        synchronized public void onNodeSendFailed(DiscoveryNode node, Throwable t) {
+        public synchronized void onNodeSendFailed(DiscoveryNode node, Throwable t) {
             if (node.isMasterNode()) {
                 logger.trace("master node {} failed to ack cluster state version [{}]. processing ... (current pending [{}], needed [{}])",
                         node, clusterState.version(), pendingMasterNodes, neededMastersToCommit);
@@ -606,7 +606,7 @@ public class PublishClusterStateAction extends AbstractComponent {
          *
          * @return true if successful
          */
-        synchronized private boolean markAsCommitted() {
+        private synchronized boolean markAsCommitted() {
             if (committedOrFailed()) {
                 return committed;
             }
@@ -621,7 +621,7 @@ public class PublishClusterStateAction extends AbstractComponent {
          *
          * @return true if the publishing was failed and the cluster state is *not* committed
          **/
-        synchronized private boolean markAsFailed(String details, Throwable reason) {
+        private synchronized boolean markAsFailed(String details, Throwable reason) {
             if (committedOrFailed()) {
                 return committed == false;
             }
@@ -636,7 +636,7 @@ public class PublishClusterStateAction extends AbstractComponent {
          *
          * @return true if the publishing was failed and the cluster state is *not* committed
          **/
-        synchronized private boolean markAsFailed(String reason) {
+        private synchronized boolean markAsFailed(String reason) {
             if (committedOrFailed()) {
                 return committed == false;
             }
diff --git a/core/src/main/java/org/elasticsearch/index/Index.java b/core/src/main/java/org/elasticsearch/index/Index.java
index 2fc62c5ea7f..25b293ad387 100644
--- a/core/src/main/java/org/elasticsearch/index/Index.java
+++ b/core/src/main/java/org/elasticsearch/index/Index.java
@@ -128,7 +128,7 @@ public class Index implements Writeable, ToXContent {
     /**
      * Builder for Index objects.  Used by ObjectParser instances only.
      */
-    final private static class Builder {
+    private static final class Builder {
         private String name;
         private String uuid;
 
diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java
index 1902f24c4b7..4e8caa7d5e9 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexService.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexService.java
@@ -186,7 +186,8 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
      * Return the shard with the provided id, or null if there is no such shard.
      */
     @Override
-    public @Nullable IndexShard getShardOrNull(int shardId) {
+    @Nullable
+    public IndexShard getShardOrNull(int shardId) {
         return shards.get(shardId);
     }
 
@@ -730,7 +731,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
         }
     }
 
-    static abstract class BaseAsyncTask implements Runnable, Closeable {
+    abstract static class BaseAsyncTask implements Runnable, Closeable {
         protected final IndexService indexService;
         protected final ThreadPool threadPool;
         private final TimeValue interval;
@@ -827,7 +828,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
     /**
      * FSyncs the translog for all shards of this index in a defined interval.
      */
-    final static class AsyncTranslogFSync extends BaseAsyncTask {
+    static final class AsyncTranslogFSync extends BaseAsyncTask {
 
         AsyncTranslogFSync(IndexService indexService) {
             super(indexService, indexService.getIndexSettings().getTranslogSyncInterval());
diff --git a/core/src/main/java/org/elasticsearch/index/codec/CodecService.java b/core/src/main/java/org/elasticsearch/index/codec/CodecService.java
index 3edc509b7eb..f8d19be3c8d 100644
--- a/core/src/main/java/org/elasticsearch/index/codec/CodecService.java
+++ b/core/src/main/java/org/elasticsearch/index/codec/CodecService.java
@@ -40,10 +40,10 @@ public class CodecService {
 
     private final Map<String, Codec> codecs;
 
-    public final static String DEFAULT_CODEC = "default";
-    public final static String BEST_COMPRESSION_CODEC = "best_compression";
+    public static final String DEFAULT_CODEC = "default";
+    public static final String BEST_COMPRESSION_CODEC = "best_compression";
     /** the raw unfiltered lucene default. useful for testing */
-    public final static String LUCENE_DEFAULT_CODEC = "lucene_default";
+    public static final String LUCENE_DEFAULT_CODEC = "lucene_default";
 
     public CodecService(@Nullable MapperService mapperService, ESLogger logger) {
         final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
diff --git a/core/src/main/java/org/elasticsearch/index/engine/Engine.java b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
index 5c1a20c16e3..d169e495a3d 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -297,7 +297,7 @@ public abstract class Engine implements Closeable {
         PENDING_OPERATIONS
     }
 
-    final protected GetResult getFromSearcher(Get get, Function<String, Searcher> searcherFactory) throws EngineException {
+    protected final GetResult getFromSearcher(Get get, Function<String, Searcher> searcherFactory) throws EngineException {
         final Searcher searcher = searcherFactory.apply("get");
         final Versions.DocIdAndVersion docIdAndVersion;
         try {
@@ -506,7 +506,7 @@ public abstract class Engine implements Closeable {
     }
 
     /** How much heap is used that would be freed by a refresh.  Note that this may throw {@link AlreadyClosedException}. */
-    abstract public long getIndexBufferRAMBytesUsed();
+    public abstract  long getIndexBufferRAMBytesUsed();
 
     protected Segment[] getSegmentInfo(SegmentInfos lastCommittedSegmentInfos, boolean verbose) {
         ensureOpen();
@@ -757,7 +757,7 @@ public abstract class Engine implements Closeable {
         }
     }
 
-    public static abstract class Operation {
+    public abstract static class Operation {
         private final Term uid;
         private long version;
         private final VersionType versionType;
diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index 449c04ce4d5..6052de7f586 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -995,7 +995,7 @@ public class InternalEngine extends Engine {
     }
 
     /** Extended SearcherFactory that warms the segments if needed when acquiring a new searcher */
-    final static class SearchFactory extends EngineSearcherFactory {
+    static final class SearchFactory extends EngineSearcherFactory {
         private final Engine.Warmer warmer;
         private final ESLogger logger;
         private final AtomicBoolean isEngineClosed;
diff --git a/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java b/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java
index 0a55803a5ec..c8ee7d5b8b0 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java
@@ -58,8 +58,8 @@ import java.util.function.Function;
 public class ShadowEngine extends Engine {
 
     /** how long to wait for an index to exist */
-    public final static String NONEXISTENT_INDEX_RETRY_WAIT = "index.shadow.wait_for_initial_commit";
-    public final static TimeValue DEFAULT_NONEXISTENT_INDEX_RETRY_WAIT = TimeValue.timeValueSeconds(5);
+    public static final String NONEXISTENT_INDEX_RETRY_WAIT = "index.shadow.wait_for_initial_commit";
+    public static final TimeValue DEFAULT_NONEXISTENT_INDEX_RETRY_WAIT = TimeValue.timeValueSeconds(5);
 
     private volatile SearcherManager searcherManager;
 
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
index afaaa9a1df8..408950c3edb 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
@@ -50,7 +50,7 @@ public interface ScriptDocValues<T> extends List<T> {
      */
     List<T> getValues();
 
-    public final static class Strings extends AbstractList<String> implements ScriptDocValues<String> {
+    public static final class Strings extends AbstractList<String> implements ScriptDocValues<String> {
 
         private final SortedBinaryDocValues values;
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java
index 9233dc30189..a6c3cdb9013 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/Mapper.java
@@ -62,7 +62,7 @@ public abstract class Mapper implements ToXContent, Iterable<Mapper> {
         }
     }
 
-    public static abstract class Builder<T extends Builder, Y extends Mapper> {
+    public abstract static class Builder<T extends Builder, Y extends Mapper> {
 
         public String name;
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java
index 7237381159b..17e8712f571 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java
@@ -72,8 +72,8 @@ public class BooleanFieldMapper extends FieldMapper {
     }
 
     public static class Values {
-        public final static BytesRef TRUE = new BytesRef("T");
-        public final static BytesRef FALSE = new BytesRef("F");
+        public static final BytesRef TRUE = new BytesRef("T");
+        public static final BytesRef FALSE = new BytesRef("F");
     }
 
     public static class Builder extends FieldMapper.Builder<Builder, BooleanFieldMapper> {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyNumberFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyNumberFieldMapper.java
index 7278e17a3c1..2b0cf0968ad 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyNumberFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/core/LegacyNumberFieldMapper.java
@@ -120,7 +120,7 @@ public abstract class LegacyNumberFieldMapper extends FieldMapper implements All
         protected abstract int maxPrecisionStep();
     }
 
-    public static abstract class NumberFieldType extends TermBasedFieldType {
+    public abstract static class NumberFieldType extends TermBasedFieldType {
 
         public NumberFieldType(LegacyNumericType numericType) {
             setTokenized(false);
diff --git a/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java
index e883166f900..0dff9a98863 100644
--- a/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java
@@ -239,14 +239,14 @@ public abstract class AbstractQueryBuilder<QB extends AbstractQueryBuilder<QB>>
         return getWriteableName();
     }
 
-    protected final static void writeQueries(StreamOutput out, List<? extends QueryBuilder> queries) throws IOException {
+    protected static final void writeQueries(StreamOutput out, List<? extends QueryBuilder> queries) throws IOException {
         out.writeVInt(queries.size());
         for (QueryBuilder query : queries) {
             out.writeNamedWriteable(query);
         }
     }
 
-    protected final static List<QueryBuilder> readQueries(StreamInput in) throws IOException {
+    protected static final List<QueryBuilder> readQueries(StreamInput in) throws IOException {
         List<QueryBuilder> queries = new ArrayList<>();
         int size = in.readVInt();
         for (int i = 0; i < size; i++) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
index 389e3812e33..9319719e81a 100644
--- a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java
@@ -357,7 +357,7 @@ public class HasChildQueryBuilder extends AbstractQueryBuilder<HasChildQueryBuil
                                     parentType, scoreMode, parentChildIndexFieldData, context.getSearchSimilarity());
     }
 
-    final static class LateParsingQuery extends Query {
+    static final class LateParsingQuery extends Query {
 
         private final Query toQuery;
         private final Query innerQuery;
diff --git a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
index b54d7713cd3..631bc163156 100644
--- a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
@@ -62,7 +62,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
     public static final ParseField INNER_HITS_FIELD = new ParseField("inner_hits");
     public static final QueryBuilder DEFAULT_INNER_HIT_QUERY = new MatchAllQueryBuilder();
 
-    private final static ObjectParser<InnerHitBuilder, QueryParseContext> PARSER = new ObjectParser<>("inner_hits", InnerHitBuilder::new);
+    private static final ObjectParser<InnerHitBuilder, QueryParseContext> PARSER = new ObjectParser<>("inner_hits", InnerHitBuilder::new);
 
     static {
         PARSER.declareString(InnerHitBuilder::setName, NAME_FIELD);
diff --git a/core/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java
index 3b47378d1e8..236d9751ab1 100644
--- a/core/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/TemplateQueryBuilder.java
@@ -48,7 +48,7 @@ public class TemplateQueryBuilder extends AbstractQueryBuilder<TemplateQueryBuil
     public static final String NAME = "template";
     public static final ParseField QUERY_NAME_FIELD = new ParseField(NAME);
 
-    private final static Map<String, ScriptService.ScriptType> parametersToTypes = new HashMap<>();
+    private static final Map<String, ScriptService.ScriptType> parametersToTypes = new HashMap<>();
     static {
         parametersToTypes.put("query", ScriptService.ScriptType.INLINE);
         parametersToTypes.put("file", ScriptService.ScriptType.FILE);
diff --git a/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java b/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java
index 2df87a0ad64..7ddf11202df 100644
--- a/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java
@@ -501,7 +501,7 @@ public abstract class DecayFunctionBuilder<DFB extends DecayFunctionBuilder<DFB>
      * This is the base class for scoring a single field.
      *
      * */
-    public static abstract class AbstractDistanceScoreFunction extends ScoreFunction {
+    public abstract static class AbstractDistanceScoreFunction extends ScoreFunction {
 
         private final double scale;
         protected final double offset;
diff --git a/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java b/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java
index 13d72a74b9c..68a2a30a2e9 100644
--- a/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java
+++ b/core/src/main/java/org/elasticsearch/index/search/stats/ShardSearchStats.java
@@ -179,7 +179,7 @@ public final class ShardSearchStats implements SearchOperationListener {
         totalStats.scrollMetric.inc(System.nanoTime() - context.getOriginNanoTime());
     }
 
-    final static class StatsHolder {
+    static final class StatsHolder {
         public final MeanMetric queryMetric = new MeanMetric();
         public final MeanMetric fetchMetric = new MeanMetric();
         public final MeanMetric scrollMetric = new MeanMetric();
diff --git a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
index 4ba2e0d0a38..7bb81a0109b 100644
--- a/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
+++ b/core/src/main/java/org/elasticsearch/index/similarity/SimilarityService.java
@@ -36,7 +36,7 @@ import java.util.function.BiFunction;
 
 public final class SimilarityService extends AbstractIndexComponent {
 
-    public final static String DEFAULT_SIMILARITY = "BM25";
+    public static final String DEFAULT_SIMILARITY = "BM25";
     private final Similarity defaultSimilarity;
     private final Similarity baseSimilarity;
     private final Map<String, SimilarityProvider> similarities;
diff --git a/core/src/main/java/org/elasticsearch/index/store/Store.java b/core/src/main/java/org/elasticsearch/index/store/Store.java
index 51e80f0b87d..2f647933ea2 100644
--- a/core/src/main/java/org/elasticsearch/index/store/Store.java
+++ b/core/src/main/java/org/elasticsearch/index/store/Store.java
@@ -716,7 +716,7 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
      *
      * @see StoreFileMetaData
      */
-    public final static class MetadataSnapshot implements Iterable<StoreFileMetaData>, Writeable {
+    public static final class MetadataSnapshot implements Iterable<StoreFileMetaData>, Writeable {
         private final Map<String, StoreFileMetaData> metadata;
 
         public static final MetadataSnapshot EMPTY = new MetadataSnapshot();
diff --git a/core/src/main/java/org/elasticsearch/index/translog/BaseTranslogReader.java b/core/src/main/java/org/elasticsearch/index/translog/BaseTranslogReader.java
index 4cdbb28a68e..49e8249e9d3 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/BaseTranslogReader.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/BaseTranslogReader.java
@@ -52,7 +52,7 @@ public abstract class BaseTranslogReader implements Comparable<BaseTranslogReade
 
     public abstract long sizeInBytes();
 
-    abstract public int totalOperations();
+    public abstract  int totalOperations();
 
     public final long getFirstOperationOffset() {
         return firstOperationOffset;
@@ -117,7 +117,7 @@ public abstract class BaseTranslogReader implements Comparable<BaseTranslogReade
     /**
      * reads bytes at position into the given buffer, filling it.
      */
-    abstract protected void readBytes(ByteBuffer buffer, long position) throws IOException;
+    protected abstract  void readBytes(ByteBuffer buffer, long position) throws IOException;
 
     @Override
     public String toString() {
diff --git a/core/src/main/java/org/elasticsearch/index/translog/Translog.java b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
index 788ee547f78..fe970dcbc5f 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/Translog.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
@@ -122,7 +122,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
     private final Path location;
     private TranslogWriter current;
 
-    private final static long NOT_SET_GENERATION = -1; // -1 is safe as it will not cause a translog deletion.
+    private static final long NOT_SET_GENERATION = -1; // -1 is safe as it will not cause a translog deletion.
 
     private volatile long currentCommittingGeneration = NOT_SET_GENERATION;
     private volatile long lastCommittedTranslogFileGeneration = NOT_SET_GENERATION;
@@ -1261,7 +1261,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
     /**
      * References a transaction log generation
      */
-    public final static class TranslogGeneration {
+    public static final class TranslogGeneration {
         public final String translogUUID;
         public final long translogFileGeneration;
 
diff --git a/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java b/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
index 59135684e02..b873c414b4d 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
@@ -114,7 +114,7 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
         return tragedy;
     }
 
-    private synchronized final void closeWithTragicEvent(Throwable throwable) throws IOException {
+    private synchronized void closeWithTragicEvent(Throwable throwable) throws IOException {
         assert throwable != null : "throwable must not be null in a tragic event";
         if (tragedy == null) {
             tragedy = throwable;
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index 12d1966e4cd..27387d7daaa 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -331,7 +331,8 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService>
      * Returns an IndexService for the specified index if exists otherwise returns <code>null</code>.
      */
     @Override
-    public @Nullable IndexService indexService(Index index) {
+    @Nullable
+    public IndexService indexService(Index index) {
         return indices.get(index.getUUID());
     }
 
@@ -737,7 +738,8 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService>
      * @return IndexMetaData for the index loaded from disk
      */
     @Override
-    public @Nullable IndexMetaData verifyIndexIsDeleted(final Index index, final ClusterState clusterState) {
+    @Nullable
+    public IndexMetaData verifyIndexIsDeleted(final Index index, final ClusterState clusterState) {
         // this method should only be called when we know the index (name + uuid) is not part of the cluster state
         if (clusterState.metaData().index(index) != null) {
             throw new IllegalStateException("Cannot delete index [" + index + "], it is still part of the cluster state.");
@@ -997,7 +999,7 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService>
      * has an entry invalidated may not clean up the entry if it is not read from
      * or written to after invalidation.
      */
-    private final static class CacheCleaner implements Runnable, Releasable {
+    private static final class CacheCleaner implements Runnable, Releasable {
 
         private final IndicesFieldDataCache cache;
         private final ESLogger logger;
@@ -1160,7 +1162,7 @@ public class IndicesService extends AbstractLifecycleComponent<IndicesService>
         return indicesRequestCache.getOrCompute(cacheEntity, reader, cacheKey);
     }
 
-    final static class IndexShardCacheEntity extends AbstractIndexShardCacheEntity {
+    static final class IndexShardCacheEntity extends AbstractIndexShardCacheEntity {
         private final IndexShard indexShard;
 
         protected IndexShardCacheEntity(IndexShard indexShard, Loader loader) {
diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java b/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java
index 97667fcdac6..e7ba2bfd70f 100644
--- a/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java
+++ b/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java
@@ -75,11 +75,11 @@ import java.util.function.Function;
  */
 public class HunspellService extends AbstractComponent {
 
-    public final static Setting<Boolean> HUNSPELL_LAZY_LOAD =
+    public static final Setting<Boolean> HUNSPELL_LAZY_LOAD =
         Setting.boolSetting("indices.analysis.hunspell.dictionary.lazy", Boolean.FALSE, Property.NodeScope);
-    public final static Setting<Boolean> HUNSPELL_IGNORE_CASE =
+    public static final Setting<Boolean> HUNSPELL_IGNORE_CASE =
         Setting.boolSetting("indices.analysis.hunspell.dictionary.ignore_case", Boolean.FALSE, Property.NodeScope);
-    public final static Setting<Settings> HUNSPELL_DICTIONARY_OPTIONS =
+    public static final Setting<Settings> HUNSPELL_DICTIONARY_OPTIONS =
         Setting.groupSetting("indices.analysis.hunspell.dictionary.", Property.NodeScope);
     private final ConcurrentHashMap<String, Dictionary> dictionaries = new ConcurrentHashMap<>();
     private final Map<String, Dictionary> knownDictionaries;
diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzers.java b/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzers.java
index 6d5c3a8ed18..ca954619927 100644
--- a/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzers.java
+++ b/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltAnalyzers.java
@@ -465,7 +465,7 @@ public enum PreBuiltAnalyzers {
         }
     };
 
-    abstract protected Analyzer create(Version version);
+    protected abstract  Analyzer create(Version version);
 
     protected final PreBuiltCacheFactory.PreBuiltCache<Analyzer> cache;
 
diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltCharFilters.java b/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltCharFilters.java
index 3e8647892e5..dcf6c295303 100644
--- a/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltCharFilters.java
+++ b/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltCharFilters.java
@@ -38,7 +38,7 @@ public enum PreBuiltCharFilters {
         }
     };
 
-    abstract public Reader create(Reader tokenStream, Version version);
+    public abstract  Reader create(Reader tokenStream, Version version);
 
     protected final PreBuiltCacheFactory.PreBuiltCache<CharFilterFactory> cache;
 
diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltTokenFilters.java b/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltTokenFilters.java
index 027c75073f6..6b69658b341 100644
--- a/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltTokenFilters.java
+++ b/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltTokenFilters.java
@@ -393,7 +393,7 @@ public enum PreBuiltTokenFilters {
 
     ;
 
-    abstract public TokenStream create(TokenStream tokenStream, Version version);
+    public abstract  TokenStream create(TokenStream tokenStream, Version version);
 
     protected final PreBuiltCacheFactory.PreBuiltCache<TokenFilterFactory> cache;
 
diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltTokenizers.java b/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltTokenizers.java
index c6a8255e4c3..424b5e4534f 100644
--- a/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltTokenizers.java
+++ b/core/src/main/java/org/elasticsearch/indices/analysis/PreBuiltTokenizers.java
@@ -129,7 +129,7 @@ public enum PreBuiltTokenizers {
 
     ;
 
-    abstract protected Tokenizer create(Version version);
+    protected abstract  Tokenizer create(Version version);
 
     protected final PreBuiltCacheFactory.PreBuiltCache<TokenizerFactory> cache;
 
diff --git a/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java b/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
index 16e4d62a721..0fcdd16aba7 100644
--- a/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
+++ b/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
@@ -443,7 +443,7 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL
         return new InFlightOpsResponse(opCount);
     }
 
-    public final static class PreShardSyncedFlushRequest extends TransportRequest {
+    public static final class PreShardSyncedFlushRequest extends TransportRequest {
         private ShardId shardId;
 
         public PreShardSyncedFlushRequest() {
@@ -480,7 +480,7 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL
     /**
      * Response for first step of synced flush (flush) for one shard copy
      */
-    final static class PreSyncedFlushResponse extends TransportResponse {
+    static final class PreSyncedFlushResponse extends TransportResponse {
 
         Engine.CommitId commitId;
 
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java
index 34bd35dfeda..4a2b93de084 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java
@@ -45,8 +45,8 @@ public class RecoveriesCollection {
     /** This is the single source of truth for ongoing recoveries. If it's not here, it was canceled or done */
     private final ConcurrentMap<Long, RecoveryTarget> onGoingRecoveries = ConcurrentCollections.newConcurrentMap();
 
-    final private ESLogger logger;
-    final private ThreadPool threadPool;
+    private final ESLogger logger;
+    private final ThreadPool threadPool;
 
     public RecoveriesCollection(ESLogger logger, ThreadPool threadPool) {
         this.logger = logger;
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java
index 6e2b300c00c..a742c582a9d 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryState.java
@@ -706,7 +706,7 @@ public class RecoveryState implements ToXContent, Streamable {
 
         private Map<String, File> fileDetails = new HashMap<>();
 
-        public final static long UNKNOWN = -1L;
+        public static final long UNKNOWN = -1L;
 
         private long version = UNKNOWN;
         private long sourceThrottlingInNanos = UNKNOWN;
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
index bafd5d0e58d..a006c8de921 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
@@ -63,7 +63,7 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget
 
     private final ESLogger logger;
 
-    private final static AtomicLong idGenerator = new AtomicLong();
+    private static final AtomicLong idGenerator = new AtomicLong();
 
     private final String RECOVERY_PREFIX = "recovery.";
 
diff --git a/core/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/core/src/main/java/org/elasticsearch/ingest/IngestDocument.java
index 80bbd52bcf6..baa0f3acd0d 100644
--- a/core/src/main/java/org/elasticsearch/ingest/IngestDocument.java
+++ b/core/src/main/java/org/elasticsearch/ingest/IngestDocument.java
@@ -47,7 +47,7 @@ import java.util.TimeZone;
  */
 public final class IngestDocument {
 
-    public final static String INGEST_KEY = "_ingest";
+    public static final String INGEST_KEY = "_ingest";
     private static final String INGEST_KEY_PREFIX = INGEST_KEY + ".";
     private static final String SOURCE_PREFIX = SourceFieldMapper.NAME + ".";
 
diff --git a/core/src/main/java/org/elasticsearch/ingest/IngestMetadata.java b/core/src/main/java/org/elasticsearch/ingest/IngestMetadata.java
index 2a9c9fa7584..9ad369e22d4 100644
--- a/core/src/main/java/org/elasticsearch/ingest/IngestMetadata.java
+++ b/core/src/main/java/org/elasticsearch/ingest/IngestMetadata.java
@@ -44,8 +44,8 @@ import java.util.Map;
  */
 public final class IngestMetadata implements MetaData.Custom {
 
-    public final static String TYPE = "ingest";
-    public final static IngestMetadata PROTO = new IngestMetadata();
+    public static final String TYPE = "ingest";
+    public static final IngestMetadata PROTO = new IngestMetadata();
     private static final ParseField PIPELINES_FIELD = new ParseField("pipeline");
     private static final ObjectParser<List<PipelineConfiguration>, ParseFieldMatcherSupplier> INGEST_METADATA_PARSER = new ObjectParser<>(
             "ingest_metadata", ArrayList::new);
diff --git a/core/src/main/java/org/elasticsearch/ingest/Pipeline.java b/core/src/main/java/org/elasticsearch/ingest/Pipeline.java
index d568f5521a4..df386166e7c 100644
--- a/core/src/main/java/org/elasticsearch/ingest/Pipeline.java
+++ b/core/src/main/java/org/elasticsearch/ingest/Pipeline.java
@@ -31,9 +31,9 @@ import java.util.Map;
  */
 public final class Pipeline {
 
-    final static String DESCRIPTION_KEY = "description";
-    final static String PROCESSORS_KEY = "processors";
-    final static String ON_FAILURE_KEY = "on_failure";
+    static final String DESCRIPTION_KEY = "description";
+    static final String PROCESSORS_KEY = "processors";
+    static final String ON_FAILURE_KEY = "on_failure";
 
     private final String id;
     private final String description;
@@ -96,7 +96,7 @@ public final class Pipeline {
         return compoundProcessor.flattenProcessors();
     }
 
-    public final static class Factory {
+    public static final class Factory {
 
         public Pipeline create(String id, Map<String, Object> config, ProcessorsRegistry processorRegistry) throws Exception {
             String description = ConfigurationUtils.readOptionalStringProperty(null, null, config, DESCRIPTION_KEY);
diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java b/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java
index a6bff3ff5cb..56f4e2c6a79 100644
--- a/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java
+++ b/core/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java
@@ -40,12 +40,12 @@ import java.util.function.BiFunction;
  */
 public final class PipelineConfiguration extends AbstractDiffable<PipelineConfiguration> implements ToXContent {
 
-    final static PipelineConfiguration PROTOTYPE = new PipelineConfiguration(null, null);
+    static final PipelineConfiguration PROTOTYPE = new PipelineConfiguration(null, null);
 
     public static PipelineConfiguration readPipelineConfiguration(StreamInput in) throws IOException {
         return PROTOTYPE.readFrom(in);
     }
-    private final static ObjectParser<Builder, ParseFieldMatcherSupplier> PARSER = new ObjectParser<>("pipeline_config", Builder::new);
+    private static final ObjectParser<Builder, ParseFieldMatcherSupplier> PARSER = new ObjectParser<>("pipeline_config", Builder::new);
     static {
         PARSER.declareString(Builder::setId, new ParseField("id"));
         PARSER.declareField((parser, builder, aVoid) -> {
diff --git a/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java b/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java
index e04a06fc460..9f38538f269 100644
--- a/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/fs/FsService.java
@@ -36,7 +36,7 @@ public class FsService extends AbstractComponent {
     private final TimeValue refreshInterval;
     private final SingleObjectCache<FsInfo> cache;
 
-    public final static Setting<TimeValue> REFRESH_INTERVAL_SETTING =
+    public static final Setting<TimeValue> REFRESH_INTERVAL_SETTING =
         Setting.timeSetting(
             "monitor.fs.refresh_interval",
             TimeValue.timeValueSeconds(1),
diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
index ac75eb93aaa..69ec81c38e9 100644
--- a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
@@ -50,20 +50,20 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent<JvmGcMonitor
 
     private volatile ScheduledFuture scheduledFuture;
 
-    public final static Setting<Boolean> ENABLED_SETTING =
+    public static final Setting<Boolean> ENABLED_SETTING =
         Setting.boolSetting("monitor.jvm.gc.enabled", true, Property.NodeScope);
-    public final static Setting<TimeValue> REFRESH_INTERVAL_SETTING =
+    public static final Setting<TimeValue> REFRESH_INTERVAL_SETTING =
         Setting.timeSetting("monitor.jvm.gc.refresh_interval", TimeValue.timeValueSeconds(1), TimeValue.timeValueSeconds(1),
             Property.NodeScope);
 
     private static String GC_COLLECTOR_PREFIX = "monitor.jvm.gc.collector.";
-    public final static Setting<Settings> GC_SETTING = Setting.groupSetting(GC_COLLECTOR_PREFIX, Property.NodeScope);
+    public static final Setting<Settings> GC_SETTING = Setting.groupSetting(GC_COLLECTOR_PREFIX, Property.NodeScope);
 
-    public final static Setting<Integer> GC_OVERHEAD_WARN_SETTING =
+    public static final Setting<Integer> GC_OVERHEAD_WARN_SETTING =
         Setting.intSetting("monitor.jvm.gc.overhead.warn", 50, 0, 100, Property.NodeScope);
-    public final static Setting<Integer> GC_OVERHEAD_INFO_SETTING =
+    public static final Setting<Integer> GC_OVERHEAD_INFO_SETTING =
         Setting.intSetting("monitor.jvm.gc.overhead.info", 25, 0, 100, Property.NodeScope);
-    public final static Setting<Integer> GC_OVERHEAD_DEBUG_SETTING =
+    public static final Setting<Integer> GC_OVERHEAD_DEBUG_SETTING =
         Setting.intSetting("monitor.jvm.gc.overhead.debug", 10, 0, 100, Property.NodeScope);
 
     static class GcOverheadThreshold {
@@ -341,7 +341,7 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent<JvmGcMonitor
     protected void doClose() {
     }
 
-    static abstract class JvmMonitor implements Runnable {
+    abstract static class JvmMonitor implements Runnable {
 
         enum Threshold { DEBUG, INFO, WARN }
 
diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmService.java b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmService.java
index 78dcf13ae8b..f0d71db6992 100644
--- a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmService.java
@@ -36,7 +36,7 @@ public class JvmService extends AbstractComponent {
 
     private JvmStats jvmStats;
 
-    public final static Setting<TimeValue> REFRESH_INTERVAL_SETTING =
+    public static final Setting<TimeValue> REFRESH_INTERVAL_SETTING =
         Setting.timeSetting("monitor.jvm.refresh_interval", TimeValue.timeValueSeconds(1), TimeValue.timeValueSeconds(1),
             Property.NodeScope);
 
diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java
index 43b675f34a2..7122f1ea382 100644
--- a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java
+++ b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java
@@ -48,10 +48,10 @@ import java.util.concurrent.TimeUnit;
  */
 public class JvmStats implements Streamable, ToXContent {
 
-    private final static RuntimeMXBean runtimeMXBean;
-    private final static MemoryMXBean memoryMXBean;
-    private final static ThreadMXBean threadMXBean;
-    private final static ClassLoadingMXBean classLoadingMXBean;
+    private static final RuntimeMXBean runtimeMXBean;
+    private static final MemoryMXBean memoryMXBean;
+    private static final ThreadMXBean threadMXBean;
+    private static final ClassLoadingMXBean classLoadingMXBean;
 
     static {
         runtimeMXBean = ManagementFactory.getRuntimeMXBean();
diff --git a/core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java b/core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
index c6b4f080b91..f41afb90e05 100644
--- a/core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
+++ b/core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
@@ -153,7 +153,7 @@ public class OsProbe {
     }
 
     private static class OsProbeHolder {
-        private final static OsProbe INSTANCE = new OsProbe();
+        private static final OsProbe INSTANCE = new OsProbe();
     }
 
     public static OsProbe getInstance() {
diff --git a/core/src/main/java/org/elasticsearch/monitor/os/OsService.java b/core/src/main/java/org/elasticsearch/monitor/os/OsService.java
index eb535141d83..248b49f21cb 100644
--- a/core/src/main/java/org/elasticsearch/monitor/os/OsService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/os/OsService.java
@@ -38,7 +38,7 @@ public class OsService extends AbstractComponent {
 
     private SingleObjectCache<OsStats> osStatsCache;
 
-    public final static Setting<TimeValue> REFRESH_INTERVAL_SETTING =
+    public static final Setting<TimeValue> REFRESH_INTERVAL_SETTING =
         Setting.timeSetting("monitor.os.refresh_interval", TimeValue.timeValueSeconds(1), TimeValue.timeValueSeconds(1),
             Property.NodeScope);
 
diff --git a/core/src/main/java/org/elasticsearch/monitor/process/ProcessProbe.java b/core/src/main/java/org/elasticsearch/monitor/process/ProcessProbe.java
index 4cd005982c2..019a00bd4c8 100644
--- a/core/src/main/java/org/elasticsearch/monitor/process/ProcessProbe.java
+++ b/core/src/main/java/org/elasticsearch/monitor/process/ProcessProbe.java
@@ -47,7 +47,7 @@ public class ProcessProbe {
     }
 
     private static class ProcessProbeHolder {
-        private final static ProcessProbe INSTANCE = new ProcessProbe();
+        private static final ProcessProbe INSTANCE = new ProcessProbe();
     }
 
     public static ProcessProbe getInstance() {
diff --git a/core/src/main/java/org/elasticsearch/monitor/process/ProcessService.java b/core/src/main/java/org/elasticsearch/monitor/process/ProcessService.java
index dbab25037d9..38593534480 100644
--- a/core/src/main/java/org/elasticsearch/monitor/process/ProcessService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/process/ProcessService.java
@@ -35,7 +35,7 @@ public final class ProcessService extends AbstractComponent {
     private final ProcessInfo info;
     private final SingleObjectCache<ProcessStats> processStatsCache;
 
-    public final static Setting<TimeValue> REFRESH_INTERVAL_SETTING =
+    public static final Setting<TimeValue> REFRESH_INTERVAL_SETTING =
         Setting.timeSetting("monitor.process.refresh_interval", TimeValue.timeValueSeconds(1), TimeValue.timeValueSeconds(1),
             Property.NodeScope);
 
diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index 506e64cb9a5..c1efcb7801c 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -233,12 +233,12 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
      *
      * @return blob store
      */
-    abstract protected BlobStore blobStore();
+    protected abstract  BlobStore blobStore();
 
     /**
      * Returns base path of the repository
      */
-    abstract protected BlobPath basePath();
+    protected abstract  BlobPath basePath();
 
     /**
      * Returns true if metadata and snapshot files should be compressed
diff --git a/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java b/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
index 56e7e08c2c3..4e57c431f14 100644
--- a/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
@@ -50,7 +50,7 @@ import java.util.function.Function;
  */
 public class FsRepository extends BlobStoreRepository {
 
-    public final static String TYPE = "fs";
+    public static final String TYPE = "fs";
 
     public static final Setting<String> LOCATION_SETTING =
         new Setting<>("location", "", Function.identity(), Property.NodeScope);
diff --git a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java b/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
index baea7fe28f9..b805727fd9c 100644
--- a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
@@ -55,7 +55,7 @@ import java.util.function.Predicate;
  */
 public class URLRepository extends BlobStoreRepository {
 
-    public final static String TYPE = "url";
+    public static final String TYPE = "url";
 
     public static final Setting<List<String>> SUPPORTED_PROTOCOLS_SETTING =
         Setting.listSetting("repositories.url.supported_protocols", Arrays.asList("http", "https", "ftp", "file", "jar"),
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java
index 3df457efb9d..8826efef225 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/info/RestNodesInfoAction.java
@@ -43,7 +43,7 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 public class RestNodesInfoAction extends BaseRestHandler {
 
     private final SettingsFilter settingsFilter;
-    private final static Set<String> ALLOWED_METRICS = Sets.newHashSet("http", "jvm", "os", "plugins", "process", "settings", "thread_pool", "transport", "ingest", "indices");
+    private static final Set<String> ALLOWED_METRICS = Sets.newHashSet("http", "jvm", "os", "plugins", "process", "settings", "thread_pool", "transport", "ingest", "indices");
 
     @Inject
     public RestNodesInfoAction(Settings settings, RestController controller, SettingsFilter settingsFilter) {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java
index caf28a9b018..8987fd27413 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java
@@ -55,7 +55,7 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 
 public class RestThreadPoolAction extends AbstractCatAction {
 
-    private final static String[] SUPPORTED_NAMES = new String[]{
+    private static final String[] SUPPORTED_NAMES = new String[]{
             ThreadPool.Names.BULK,
             ThreadPool.Names.FLUSH,
             ThreadPool.Names.GENERIC,
@@ -69,7 +69,7 @@ public class RestThreadPoolAction extends AbstractCatAction {
             ThreadPool.Names.WARMER
     };
 
-    private final static String[] SUPPORTED_ALIASES = new String[]{
+    private static final String[] SUPPORTED_ALIASES = new String[]{
             "b",
             "f",
             "ge",
@@ -87,14 +87,14 @@ public class RestThreadPoolAction extends AbstractCatAction {
         assert SUPPORTED_ALIASES.length == SUPPORTED_NAMES.length: "SUPPORTED_NAMES/ALIASES mismatch";
     }
 
-    private final static String[] DEFAULT_THREAD_POOLS = new String[]{
+    private static final String[] DEFAULT_THREAD_POOLS = new String[]{
             ThreadPool.Names.BULK,
             ThreadPool.Names.INDEX,
             ThreadPool.Names.SEARCH,
     };
 
-    private final static Map<String, String> ALIAS_TO_THREAD_POOL;
-    private final static Map<String, String> THREAD_POOL_TO_ALIAS;
+    private static final Map<String, String> ALIAS_TO_THREAD_POOL;
+    private static final Map<String, String> THREAD_POOL_TO_ALIAS;
 
     static {
         ALIAS_TO_THREAD_POOL = new HashMap<>(SUPPORTED_NAMES.length);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/termvectors/RestTermVectorsAction.java b/core/src/main/java/org/elasticsearch/rest/action/termvectors/RestTermVectorsAction.java
index 6aa2d1bdb14..30ee18300b1 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/termvectors/RestTermVectorsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/termvectors/RestTermVectorsAction.java
@@ -75,7 +75,7 @@ public class RestTermVectorsAction extends BaseRestHandler {
         client.termVectors(termVectorsRequest, new RestToXContentListener<TermVectorsResponse>(channel));
     }
 
-    static public void readURIParameters(TermVectorsRequest termVectorsRequest, RestRequest request) {
+    public static void readURIParameters(TermVectorsRequest termVectorsRequest, RestRequest request) {
         String fields = request.param("fields");
         addFieldStringsFromParameter(termVectorsRequest, fields);
         termVectorsRequest.offsets(request.paramAsBoolean("offsets", termVectorsRequest.offsets()));
@@ -93,7 +93,7 @@ public class RestTermVectorsAction extends BaseRestHandler {
         termVectorsRequest.fieldStatistics(request.paramAsBoolean("field_statistics", termVectorsRequest.fieldStatistics()));
     }
 
-    static public void addFieldStringsFromParameter(TermVectorsRequest termVectorsRequest, String fields) {
+    public static void addFieldStringsFromParameter(TermVectorsRequest termVectorsRequest, String fields) {
         Set<String> selectedFields = termVectorsRequest.selectedFields();
         if (fields != null) {
             String[] paramFieldStrings = Strings.commaDelimitedListToStringArray(fields);
diff --git a/core/src/main/java/org/elasticsearch/script/Script.java b/core/src/main/java/org/elasticsearch/script/Script.java
index 54b3cd9328d..f5debcd67da 100644
--- a/core/src/main/java/org/elasticsearch/script/Script.java
+++ b/core/src/main/java/org/elasticsearch/script/Script.java
@@ -44,9 +44,9 @@ public class Script implements ToXContent, Writeable {
     private static final ScriptParser PARSER = new ScriptParser();
 
     private String script;
-    private @Nullable ScriptType type;
-    private @Nullable String lang;
-    private @Nullable Map<String, Object> params;
+    @Nullable private ScriptType type;
+    @Nullable private String lang;
+    @Nullable private Map<String, Object> params;
 
     /**
      * Constructor for simple inline script. The script will have no lang or
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java b/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java
index 5c0f8cee04c..78f33bbc605 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptMetaData.java
@@ -47,8 +47,8 @@ import java.util.Optional;
 
 public final class ScriptMetaData implements MetaData.Custom {
 
-    public final static String TYPE = "stored_scripts";
-    public final static ScriptMetaData PROTO = new ScriptMetaData(Collections.emptyMap());
+    public static final String TYPE = "stored_scripts";
+    public static final ScriptMetaData PROTO = new ScriptMetaData(Collections.emptyMap());
 
     private final Map<String, ScriptAsBytes> scripts;
 
@@ -205,7 +205,7 @@ public final class ScriptMetaData implements MetaData.Custom {
         return language + "#" + id;
     }
 
-    final public static class Builder {
+    public static final class Builder {
 
         private Map<String, ScriptAsBytes> scripts;
 
@@ -234,7 +234,7 @@ public final class ScriptMetaData implements MetaData.Custom {
         }
     }
 
-    final static class ScriptMetadataDiff implements Diff<MetaData.Custom> {
+    static final class ScriptMetadataDiff implements Diff<MetaData.Custom> {
 
         final Diff<Map<String, ScriptAsBytes>> pipelines;
 
@@ -257,7 +257,7 @@ public final class ScriptMetaData implements MetaData.Custom {
         }
     }
 
-    final static class ScriptAsBytes extends AbstractDiffable<ScriptAsBytes> {
+    static final class ScriptAsBytes extends AbstractDiffable<ScriptAsBytes> {
 
         public ScriptAsBytes(BytesReference script) {
             this.script = script;
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java b/core/src/main/java/org/elasticsearch/script/ScriptSettings.java
index 57fa8d58f2c..41b19a3e572 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptSettings.java
@@ -35,9 +35,9 @@ import java.util.stream.Collectors;
 
 public class ScriptSettings {
 
-    public final static String DEFAULT_LANG = "groovy";
+    public static final String DEFAULT_LANG = "groovy";
 
-    private final static Map<ScriptService.ScriptType, Setting<Boolean>> SCRIPT_TYPE_SETTING_MAP;
+    private static final Map<ScriptService.ScriptType, Setting<Boolean>> SCRIPT_TYPE_SETTING_MAP;
 
     static {
         Map<ScriptService.ScriptType, Setting<Boolean>> scriptTypeSettingMap = new HashMap<>();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/BucketCollector.java b/core/src/main/java/org/elasticsearch/search/aggregations/BucketCollector.java
index c1c1bff1adb..2de6ae0cf93 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/BucketCollector.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/BucketCollector.java
@@ -34,7 +34,7 @@ import java.util.stream.StreamSupport;
  */
 public abstract class BucketCollector implements Collector {
 
-    public final static BucketCollector NO_OP_COLLECTOR = new BucketCollector() {
+    public static final BucketCollector NO_OP_COLLECTOR = new BucketCollector() {
 
         @Override
         public LeafBucketCollector getLeafCollector(LeafReaderContext reader) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
index 75223dcc476..3628133580b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregations.java
@@ -43,7 +43,7 @@ import static java.util.Collections.unmodifiableMap;
  */
 public class InternalAggregations implements Aggregations, ToXContent, Streamable {
 
-    public final static InternalAggregations EMPTY = new InternalAggregations();
+    public static final InternalAggregations EMPTY = new InternalAggregations();
 
     private List<InternalAggregation> aggregations = Collections.emptyList();
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
index e46f24c2425..c579f349b61 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
@@ -71,7 +71,7 @@ public abstract class InternalMultiBucketAggregation<A extends InternalMultiBuck
         }
     }
 
-    public static abstract class InternalBucket implements Bucket {
+    public abstract static class InternalBucket implements Bucket {
         @Override
         public Object getProperty(String containingAggName, List<String> path) {
             if (path.isEmpty()) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/InternalChildren.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/InternalChildren.java
index 324f32a3813..a1febde4380 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/InternalChildren.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/children/InternalChildren.java
@@ -35,7 +35,7 @@ public class InternalChildren extends InternalSingleBucketAggregation implements
 
     public static final Type TYPE = new Type("children");
 
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalChildren readResult(StreamInput in) throws IOException {
             InternalChildren result = new InternalChildren();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/InternalFilter.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/InternalFilter.java
index 19f3b5a7e94..b7896b4eb17 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/InternalFilter.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/InternalFilter.java
@@ -33,9 +33,9 @@ import java.util.Map;
 */
 public class InternalFilter extends InternalSingleBucketAggregation implements Filter {
 
-    public final static Type TYPE = new Type("filter");
+    public static final Type TYPE = new Type("filter");
 
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalFilter readResult(StreamInput in) throws IOException {
             InternalFilter result = new InternalFilter();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/InternalFilters.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/InternalFilters.java
index aed398d51df..4cace0a50d0 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/InternalFilters.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/InternalFilters.java
@@ -42,9 +42,9 @@ import java.util.Map;
  */
 public class InternalFilters extends InternalMultiBucketAggregation<InternalFilters, InternalFilters.InternalBucket> implements Filters {
 
-    public final static Type TYPE = new Type("filters");
+    public static final Type TYPE = new Type("filters");
 
-    private final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    private static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalFilters readResult(StreamInput in) throws IOException {
             InternalFilters filters = new InternalFilters();
@@ -53,7 +53,7 @@ public class InternalFilters extends InternalMultiBucketAggregation<InternalFilt
         }
     };
 
-    private final static BucketStreams.Stream<InternalBucket> BUCKET_STREAM = new BucketStreams.Stream<InternalBucket>() {
+    private static final BucketStreams.Stream<InternalBucket> BUCKET_STREAM = new BucketStreams.Stream<InternalBucket>() {
         @Override
         public InternalBucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
             InternalBucket filters = new InternalBucket(context.keyed());
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/InternalGlobal.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/InternalGlobal.java
index abd5479ba42..e8d4d6fd849 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/InternalGlobal.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/InternalGlobal.java
@@ -34,9 +34,9 @@ import java.util.Map;
  */
 public class InternalGlobal extends InternalSingleBucketAggregation implements Global {
 
-    public final static Type TYPE = new Type("global");
+    public static final Type TYPE = new Type("global");
 
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalGlobal readResult(StreamInput in) throws IOException {
             InternalGlobal result = new InternalGlobal();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/Histogram.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/Histogram.java
index 0c126947197..6e6ac058204 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/Histogram.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/Histogram.java
@@ -46,7 +46,7 @@ public interface Histogram extends MultiBucketsAggregation {
     /**
      * A strategy defining the order in which the buckets in this histogram are ordered.
      */
-    static abstract class Order implements ToXContent {
+    abstract static class Order implements ToXContent {
 
         public static final Order KEY_ASC = new InternalOrder((byte) 1, "_key", true, new Comparator<InternalHistogram.Bucket>() {
             @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java
index 84cc98515dd..6f359708f5a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalDateHistogram.java
@@ -32,7 +32,7 @@ import org.joda.time.DateTimeZone;
 public class InternalDateHistogram {
 
     public static final Factory HISTOGRAM_FACTORY = new Factory();
-    final static Type TYPE = new Type("date_histogram", "dhisto");
+    static final Type TYPE = new Type("date_histogram", "dhisto");
 
     static class Bucket extends InternalHistogram.Bucket {
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java
index a5ee0728b4a..2ea2eaecde8 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java
@@ -52,9 +52,9 @@ public class InternalHistogram<B extends InternalHistogram.Bucket> extends Inter
         Histogram {
 
     public static final Factory<Bucket> HISTOGRAM_FACTORY = new Factory<Bucket>();
-    final static Type TYPE = new Type("histogram", "histo");
+    static final Type TYPE = new Type("histogram", "histo");
 
-    private final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    private static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @SuppressWarnings("rawtypes")
         @Override
         public InternalHistogram readResult(StreamInput in) throws IOException {
@@ -64,7 +64,7 @@ public class InternalHistogram<B extends InternalHistogram.Bucket> extends Inter
         }
     };
 
-    private final static BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
+    private static final BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
         @Override
         public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
             Factory<?> factory = (Factory<?>) context.attributes().get("factory");
@@ -96,8 +96,8 @@ public class InternalHistogram<B extends InternalHistogram.Bucket> extends Inter
         long key;
         long docCount;
         InternalAggregations aggregations;
-        private transient final boolean keyed;
-        protected transient final DocValueFormat format;
+        private final transient boolean keyed;
+        protected final transient DocValueFormat format;
         private Factory<?> factory;
 
         public Bucket(boolean keyed, DocValueFormat formatter, Factory<?> factory) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/InternalMissing.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/InternalMissing.java
index 9931f29613f..78293dd5832 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/InternalMissing.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/InternalMissing.java
@@ -33,9 +33,9 @@ import java.util.Map;
  */
 public class InternalMissing extends InternalSingleBucketAggregation implements Missing {
 
-    public final static Type TYPE = new Type("missing");
+    public static final Type TYPE = new Type("missing");
 
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalMissing readResult(StreamInput in) throws IOException {
             InternalMissing missing = new InternalMissing();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/InternalNested.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/InternalNested.java
index badd63eb2ce..cdceade65ad 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/InternalNested.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/InternalNested.java
@@ -35,7 +35,7 @@ public class InternalNested extends InternalSingleBucketAggregation implements N
 
     public static final Type TYPE = new Type("nested");
 
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalNested readResult(StreamInput in) throws IOException {
             InternalNested result = new InternalNested();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/InternalReverseNested.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/InternalReverseNested.java
index 78fb6ecab5e..60d4920e177 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/InternalReverseNested.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/InternalReverseNested.java
@@ -35,7 +35,7 @@ public class InternalReverseNested extends InternalSingleBucketAggregation imple
 
     public static final Type TYPE = new Type("reverse_nested");
 
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalReverseNested readResult(StreamInput in) throws IOException {
             InternalReverseNested result = new InternalReverseNested();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorFactory.java
index 5a8f47c0f25..b09e3d4d40b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorFactory.java
@@ -60,7 +60,7 @@ public class NestedAggregatorFactory extends AggregatorFactory<NestedAggregatorF
         return new NestedAggregator(name, factories, objectMapper, context, parent, pipelineAggregators, metaData);
     }
 
-    private final static class Unmapped extends NonCollectingAggregator {
+    private static final class Unmapped extends NonCollectingAggregator {
 
         public Unmapped(String name, AggregationContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
                 Map<String, Object> metaData) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorFactory.java
index b1a4ce799b8..b8be6479436 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorFactory.java
@@ -79,7 +79,7 @@ public class ReverseNestedAggregatorFactory extends AggregatorFactory<ReverseNes
         return null;
     }
 
-    private final static class Unmapped extends NonCollectingAggregator {
+    private static final class Unmapped extends NonCollectingAggregator {
 
         public Unmapped(String name, AggregationContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
                 Map<String, Object> metaData) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java
index 9909d6eca06..1928a1c0c74 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/BinaryRangeAggregator.java
@@ -117,7 +117,7 @@ public final class BinaryRangeAggregator extends BucketsAggregator {
         }
     }
 
-    static abstract class SortedSetRangeLeafCollector extends LeafBucketCollectorBase {
+    abstract static class SortedSetRangeLeafCollector extends LeafBucketCollectorBase {
 
         final long[] froms, tos, maxTos;
         final SortedSetDocValues values;
@@ -223,7 +223,7 @@ public final class BinaryRangeAggregator extends BucketsAggregator {
         protected abstract void doCollect(LeafBucketCollector sub, int doc, long bucket) throws IOException;
     }
 
-    static abstract class SortedBinaryRangeLeafCollector extends LeafBucketCollectorBase {
+    abstract static class SortedBinaryRangeLeafCollector extends LeafBucketCollectorBase {
 
         final Range[] ranges;
         final BytesRef[] maxTos;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
index f3f1763113e..568033d2fed 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
@@ -44,7 +44,7 @@ public final class InternalBinaryRange
 
     public static final Type TYPE = new Type("binary_range");
 
-    private final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    private static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalBinaryRange readResult(StreamInput in) throws IOException {
             InternalBinaryRange range = new InternalBinaryRange();
@@ -53,7 +53,7 @@ public final class InternalBinaryRange
         }
     };
 
-    private final static BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
+    private static final BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
         @Override
         public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
             Bucket bucket = new Bucket(context.format(), context.keyed());
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
index 0eebd6df150..a6c1dc91b5d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
@@ -46,9 +46,9 @@ public class InternalRange<B extends InternalRange.Bucket, R extends InternalRan
 
     static final Factory FACTORY = new Factory();
 
-    public final static Type TYPE = new Type("range");
+    public static final Type TYPE = new Type("range");
 
-    private final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    private static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalRange readResult(StreamInput in) throws IOException {
             InternalRange ranges = new InternalRange();
@@ -57,7 +57,7 @@ public class InternalRange<B extends InternalRange.Bucket, R extends InternalRan
         }
     };
 
-    private final static BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
+    private static final BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
         @Override
         public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
             Bucket buckets = new Bucket(context.keyed(), context.format());
@@ -81,8 +81,8 @@ public class InternalRange<B extends InternalRange.Bucket, R extends InternalRan
 
     public static class Bucket extends InternalMultiBucketAggregation.InternalBucket implements Range.Bucket {
 
-        protected transient final boolean keyed;
-        protected transient final DocValueFormat format;
+        protected final transient boolean keyed;
+        protected final transient DocValueFormat format;
         protected double from;
         protected double to;
         private long docCount;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/InternalDateRange.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/InternalDateRange.java
index 80411c59045..550e49797df 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/InternalDateRange.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/InternalDateRange.java
@@ -40,9 +40,9 @@ import java.util.Map;
  */
 public class InternalDateRange extends InternalRange<InternalDateRange.Bucket, InternalDateRange> {
 
-    public final static Type TYPE = new Type("date_range", "drange");
+    public static final Type TYPE = new Type("date_range", "drange");
 
-    private final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    private static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalDateRange readResult(StreamInput in) throws IOException {
             InternalDateRange ranges = new InternalDateRange();
@@ -51,7 +51,7 @@ public class InternalDateRange extends InternalRange<InternalDateRange.Bucket, I
         }
     };
 
-    private final static BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
+    private static final BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
         @Override
         public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
             Bucket buckets = new Bucket(context.keyed(), context.format());
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/InternalGeoDistance.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/InternalGeoDistance.java
index 13ed3a44fb9..54244ca65bd 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/InternalGeoDistance.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/InternalGeoDistance.java
@@ -50,7 +50,7 @@ public class InternalGeoDistance extends InternalRange<InternalGeoDistance.Bucke
         }
     };
 
-    private final static BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
+    private static final BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
         @Override
         public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
             Bucket buckets = new Bucket(context.keyed());
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java
index f33cdb5f846..c7bbeba38eb 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java
@@ -33,9 +33,9 @@ import java.util.Map;
 */
 public class InternalSampler extends InternalSingleBucketAggregation implements Sampler {
 
-    public final static Type TYPE = new Type("sampler");
+    public static final Type TYPE = new Type("sampler");
 
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalSampler readResult(StreamInput in) throws IOException {
             InternalSampler result = new InternalSampler();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java
index d42aa731887..d76f46b45bf 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/InternalSignificantTerms.java
@@ -53,12 +53,12 @@ public abstract class InternalSignificantTerms<A extends InternalSignificantTerm
     protected InternalSignificantTerms() {} // for serialization
 
     @SuppressWarnings("PMD.ConstructorCallsOverridableMethod")
-    public static abstract class Bucket extends SignificantTerms.Bucket {
+    public abstract static class Bucket extends SignificantTerms.Bucket {
 
         long bucketOrd;
         protected InternalAggregations aggregations;
         double score;
-        transient final DocValueFormat format;
+        final transient DocValueFormat format;
 
         protected Bucket(long subsetSize, long supersetSize, DocValueFormat format) {
             // for serialization
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java
index f2172082ae4..b8dca9cd380 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java
@@ -52,7 +52,7 @@ public class SignificantLongTerms extends InternalSignificantTerms<SignificantLo
         }
     };
 
-    private final static BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
+    private static final BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
         @Override
         public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
             Bucket buckets = new Bucket((long) context.attributes().get("subsetSize"), (long) context.attributes().get("supersetSize"),
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java
index b951c351702..616a6e163f8 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java
@@ -53,7 +53,7 @@ public class SignificantStringTerms extends InternalSignificantTerms<Significant
         }
     };
 
-    private final static BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
+    private static final BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
         @Override
         public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
             Bucket buckets = new Bucket((long) context.attributes().get("subsetSize"),
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTerms.java
index ef0a4f2d469..80d2ed070bf 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTerms.java
@@ -29,7 +29,7 @@ import java.util.List;
 public interface SignificantTerms extends MultiBucketsAggregation, Iterable<SignificantTerms.Bucket> {
 
 
-    static abstract class Bucket extends InternalMultiBucketAggregation.InternalBucket {
+    abstract static class Bucket extends InternalMultiBucketAggregation.InternalBucket {
 
         long subsetDf;
         long subsetSize;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/NXYSignificanceHeuristic.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/NXYSignificanceHeuristic.java
index 9b2a26fa00c..3036c57865d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/NXYSignificanceHeuristic.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/heuristics/NXYSignificanceHeuristic.java
@@ -149,7 +149,7 @@ public abstract class NXYSignificanceHeuristic extends SignificanceHeuristic {
                 backgroundIsSuperset);
     }
 
-    public static abstract class NXYParser implements SignificanceHeuristicParser {
+    public abstract static class NXYParser implements SignificanceHeuristicParser {
 
         @Override
         public SignificanceHeuristic parse(XContentParser parser, ParseFieldMatcher parseFieldMatcher)
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java
index 425f877caf7..fa7ff8345cf 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java
@@ -50,7 +50,7 @@ public class DoubleTerms extends InternalTerms<DoubleTerms, DoubleTerms.Bucket>
         }
     };
 
-    private final static BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
+    private static final BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
         @Override
         public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
             Bucket buckets = new Bucket(context.format(), (boolean) context.attributes().get("showDocCountError"));
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
index fcbd66dced6..e134595c594 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
@@ -44,7 +44,7 @@ public abstract class InternalTerms<A extends InternalTerms, B extends InternalT
     protected static final String DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME = "doc_count_error_upper_bound";
     protected static final String SUM_OF_OTHER_DOC_COUNTS = "sum_other_doc_count";
 
-    public static abstract class Bucket extends Terms.Bucket {
+    public abstract static class Bucket extends Terms.Bucket {
 
         long bucketOrd;
 
@@ -52,7 +52,7 @@ public abstract class InternalTerms<A extends InternalTerms, B extends InternalT
         protected long docCountError;
         protected InternalAggregations aggregations;
         protected boolean showDocCountError;
-        transient final DocValueFormat format;
+        final transient DocValueFormat format;
 
         protected Bucket(DocValueFormat formatter, boolean showDocCountError) {
             // for serialization
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java
index 91aec24529e..a893fef3856 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java
@@ -50,7 +50,7 @@ public class LongTerms extends InternalTerms<LongTerms, LongTerms.Bucket> {
         }
     };
 
-    private final static BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
+    private static final BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
         @Override
         public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
             Bucket buckets = new Bucket(context.format(), (boolean) context.attributes().get("showDocCountError"));
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java
index ddaaebe8999..9f646aefe12 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java
@@ -52,7 +52,7 @@ public class StringTerms extends InternalTerms<StringTerms, StringTerms.Bucket>
         }
     };
 
-    private final static BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
+    private static final BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
         @Override
         public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
             Bucket buckets = new Bucket(
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/Terms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/Terms.java
index 16fb7f1d81e..aa70309da95 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/Terms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/Terms.java
@@ -62,7 +62,7 @@ public interface Terms extends MultiBucketsAggregation {
     /**
      * A bucket that is associated with a single term
      */
-    static abstract class Bucket extends InternalMultiBucketAggregation.InternalBucket {
+    abstract static class Bucket extends InternalMultiBucketAggregation.InternalBucket {
 
         public abstract Number getKeyAsNumber();
 
@@ -97,7 +97,7 @@ public interface Terms extends MultiBucketsAggregation {
     /**
      * Determines the order by which the term buckets will be sorted
      */
-    static abstract class Order implements ToXContent {
+    abstract static class Order implements ToXContent {
 
         /**
          * @return a bucket ordering strategy that sorts buckets by their document counts (ascending or descending)
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java
index 209700b86d9..f9ef22f2361 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java
@@ -135,7 +135,7 @@ public class IncludeExclude implements Writeable, ToXContent {
         }
     }
 
-    public static abstract class OrdinalsFilter {
+    public abstract static class OrdinalsFilter {
         public abstract LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals, ValuesSource.Bytes.WithOrdinals valueSource)
                 throws IOException;
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java
index 15c9d7ed59c..ffa41a9ed8c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java
@@ -35,7 +35,7 @@ public abstract class InternalNumericMetricsAggregation extends InternalMetricsA
 
     protected DocValueFormat format = DEFAULT_FORMAT;
 
-    public static abstract class SingleValue extends InternalNumericMetricsAggregation implements NumericMetricsAggregation.SingleValue {
+    public abstract static class SingleValue extends InternalNumericMetricsAggregation implements NumericMetricsAggregation.SingleValue {
 
         protected SingleValue() {}
 
@@ -68,7 +68,7 @@ public abstract class InternalNumericMetricsAggregation extends InternalMetricsA
 
     }
 
-    public static abstract class MultiValue extends InternalNumericMetricsAggregation implements NumericMetricsAggregation.MultiValue {
+    public abstract static class MultiValue extends InternalNumericMetricsAggregation implements NumericMetricsAggregation.MultiValue {
 
         protected MultiValue() {}
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/NumericMetricsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/NumericMetricsAggregator.java
index e7306e9667a..3ffd7f79750 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/NumericMetricsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/NumericMetricsAggregator.java
@@ -36,7 +36,7 @@ public abstract class NumericMetricsAggregator extends MetricsAggregator {
         super(name, context, parent, pipelineAggregators, metaData);
     }
 
-    public static abstract class SingleValue extends NumericMetricsAggregator {
+    public abstract static class SingleValue extends NumericMetricsAggregator {
 
         protected SingleValue(String name, AggregationContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
                 Map<String, Object> metaData) throws IOException {
@@ -46,7 +46,7 @@ public abstract class NumericMetricsAggregator extends MetricsAggregator {
         public abstract double metric(long owningBucketOrd);
     }
 
-    public static abstract class MultiValue extends NumericMetricsAggregator {
+    public abstract static class MultiValue extends NumericMetricsAggregator {
 
         protected MultiValue(String name, AggregationContext context, Aggregator parent, List<PipelineAggregator> pipelineAggregators,
                 Map<String, Object> metaData) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregationBuilder.java
index 830ce35fde7..287b437f136 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/avg/AvgAggregationBuilder.java
@@ -38,7 +38,7 @@ import java.io.IOException;
 
 public class AvgAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, AvgAggregationBuilder> {
     public static final String NAME = "avg";
-    private final static Type TYPE = new Type(NAME);
+    private static final Type TYPE = new Type(NAME);
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     public AvgAggregationBuilder(String name) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java
index 0793bacf722..99922cc6aec 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregator.java
@@ -156,7 +156,7 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue
         Releasables.close(counts, collector);
     }
 
-    private static abstract class Collector extends LeafBucketCollector implements Releasable {
+    private abstract static class Collector extends LeafBucketCollector implements Releasable {
 
         public abstract void postCollect();
 
@@ -293,7 +293,7 @@ public class CardinalityAggregator extends NumericMetricsAggregator.SingleValue
     /**
      * Representation of a list of hash values. There might be dups and there is no guarantee on the order.
      */
-    static abstract class MurmurHash3Values {
+    abstract static class MurmurHash3Values {
 
         public abstract void setDocument(int docId);
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinality.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinality.java
index 2d320eda818..02fe800adf3 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinality.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinality.java
@@ -35,9 +35,9 @@ import java.util.Map;
 
 public final class InternalCardinality extends InternalNumericMetricsAggregation.SingleValue implements Cardinality {
 
-    public final static Type TYPE = new Type("cardinality");
+    public static final Type TYPE = new Type("cardinality");
 
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalCardinality readResult(StreamInput in) throws IOException {
             InternalCardinality result = new InternalCardinality();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBounds.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBounds.java
index c3bd5deb5b3..0b41d5d4939 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBounds.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geobounds/InternalGeoBounds.java
@@ -34,8 +34,8 @@ import java.util.Map;
 
 public class InternalGeoBounds extends InternalMetricsAggregation implements GeoBounds {
 
-    public final static Type TYPE = new Type("geo_bounds");
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    public static final Type TYPE = new Type("geo_bounds");
+    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalGeoBounds readResult(StreamInput in) throws IOException {
             InternalGeoBounds result = new InternalGeoBounds();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java
index 2bb3056ca66..6b15aab8142 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java
@@ -38,8 +38,8 @@ import java.util.Map;
  */
 public class InternalGeoCentroid extends InternalMetricsAggregation implements GeoCentroid {
 
-    public final static Type TYPE = new Type("geo_centroid");
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    public static final Type TYPE = new Type("geo_centroid");
+    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalGeoCentroid readResult(StreamInput in) throws IOException {
             InternalGeoCentroid result = new InternalGeoCentroid();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregationBuilder.java
index 7158a065ac6..cb45e73db41 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/max/MaxAggregationBuilder.java
@@ -38,7 +38,7 @@ import java.io.IOException;
 
 public class MaxAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, MaxAggregationBuilder> {
     public static final String NAME = "max";
-    public final static Type TYPE = new Type(NAME);
+    public static final Type TYPE = new Type(NAME);
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     public MaxAggregationBuilder(String name) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregationBuilder.java
index b8a6f7ed904..10edbe9d3eb 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/MinAggregationBuilder.java
@@ -38,7 +38,7 @@ import java.io.IOException;
 
 public class MinAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, MinAggregationBuilder> {
     public static final String NAME = "min";
-    private final static Type TYPE = new Type(NAME);
+    private static final Type TYPE = new Type(NAME);
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     public MinAggregationBuilder(String name) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesParser.java
index ffdc4d9a631..806fb26cd3f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesParser.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesParser.java
@@ -33,7 +33,7 @@ public class PercentilesParser extends AbstractPercentilesParser {
         super(true);
     }
 
-    public final static double[] DEFAULT_PERCENTS = new double[] { 1, 5, 25, 50, 75, 95, 99 };
+    public static final double[] DEFAULT_PERCENTS = new double[] { 1, 5, 25, 50, 75, 95, 99 };
 
     @Override
     protected ParseField keysField() {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentileRanks.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentileRanks.java
index 6e072c08388..0bd6b0322a0 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentileRanks.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentileRanks.java
@@ -37,9 +37,9 @@ import java.util.Map;
 */
 public class InternalHDRPercentileRanks extends AbstractInternalHDRPercentiles implements PercentileRanks {
 
-    public final static Type TYPE = new Type(PercentileRanks.TYPE_NAME, "hdr_percentile_ranks");
+    public static final Type TYPE = new Type(PercentileRanks.TYPE_NAME, "hdr_percentile_ranks");
 
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalHDRPercentileRanks readResult(StreamInput in) throws IOException {
             InternalHDRPercentileRanks result = new InternalHDRPercentileRanks();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentiles.java
index b8ecade437e..62fe2cdc951 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentiles.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentiles.java
@@ -37,9 +37,9 @@ import java.util.Map;
 */
 public class InternalHDRPercentiles extends AbstractInternalHDRPercentiles implements Percentiles {
 
-    public final static Type TYPE = new Type(Percentiles.TYPE_NAME, "hdr_percentiles");
+    public static final Type TYPE = new Type(Percentiles.TYPE_NAME, "hdr_percentiles");
 
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalHDRPercentiles readResult(StreamInput in) throws IOException {
             InternalHDRPercentiles result = new InternalHDRPercentiles();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentileRanks.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentileRanks.java
index a293d773c83..f6622e0ab25 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentileRanks.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentileRanks.java
@@ -36,9 +36,9 @@ import java.util.Map;
 */
 public class InternalTDigestPercentileRanks extends AbstractInternalTDigestPercentiles implements PercentileRanks {
 
-    public final static Type TYPE = new Type(PercentileRanks.TYPE_NAME, "t_digest_percentile_ranks");
+    public static final Type TYPE = new Type(PercentileRanks.TYPE_NAME, "t_digest_percentile_ranks");
 
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalTDigestPercentileRanks readResult(StreamInput in) throws IOException {
             InternalTDigestPercentileRanks result = new InternalTDigestPercentileRanks();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentiles.java
index 999c85df0c0..ce299d4d90b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentiles.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentiles.java
@@ -36,9 +36,9 @@ import java.util.Map;
 */
 public class InternalTDigestPercentiles extends AbstractInternalTDigestPercentiles implements Percentiles {
 
-    public final static Type TYPE = new Type(Percentiles.TYPE_NAME, "t_digest_percentiles");
+    public static final Type TYPE = new Type(Percentiles.TYPE_NAME, "t_digest_percentiles");
 
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalTDigestPercentiles readResult(StreamInput in) throws IOException {
             InternalTDigestPercentiles result = new InternalTDigestPercentiles();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java
index 2c09d58afdb..3bcbc0cf845 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java
@@ -40,9 +40,9 @@ import java.util.Map;
 
 public class InternalScriptedMetric extends InternalMetricsAggregation implements ScriptedMetric {
 
-    public final static Type TYPE = new Type("scripted_metric");
+    public static final Type TYPE = new Type("scripted_metric");
 
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalScriptedMetric readResult(StreamInput in) throws IOException {
             InternalScriptedMetric result = new InternalScriptedMetric();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregationBuilder.java
index cac1d92101b..e1ecc8d4f7b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/StatsAggregationBuilder.java
@@ -38,7 +38,7 @@ import java.io.IOException;
 
 public class StatsAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, StatsAggregationBuilder> {
     public static final String NAME = "stats";
-    private final static Type TYPE = new Type(NAME);
+    private static final Type TYPE = new Type(NAME);
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     public StatsAggregationBuilder(String name) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregationBuilder.java
index a28508c2171..ca90fe17f5e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/stats/extended/ExtendedStatsAggregationBuilder.java
@@ -40,7 +40,7 @@ import java.util.Objects;
 public class ExtendedStatsAggregationBuilder
         extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, ExtendedStatsAggregationBuilder> {
     public static final String NAME = "extended_stats";
-    public final static Type TYPE = new Type(NAME);
+    public static final Type TYPE = new Type(NAME);
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     private double sigma = 2.0;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregationBuilder.java
index 8eafb22b6d0..04422ed8137 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/sum/SumAggregationBuilder.java
@@ -38,7 +38,7 @@ import java.io.IOException;
 
 public class SumAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.Numeric, SumAggregationBuilder> {
     public static final String NAME = "sum";
-    private final static Type TYPE = new Type(NAME);
+    private static final Type TYPE = new Type(NAME);
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     public SumAggregationBuilder(String name) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValue.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValue.java
index 13cfd9487cd..24f2b88b952 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValue.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/InternalSimpleValue.java
@@ -34,9 +34,9 @@ import java.util.Map;
 
 public class InternalSimpleValue extends InternalNumericMetricsAggregation.SingleValue implements SimpleValue {
 
-    public final static Type TYPE = new Type("simple_value");
+    public static final Type TYPE = new Type("simple_value");
 
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalSimpleValue readResult(StreamInput in) throws IOException {
             InternalSimpleValue result = new InternalSimpleValue();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/InternalBucketMetricValue.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/InternalBucketMetricValue.java
index dab7235e9d1..683620b672d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/InternalBucketMetricValue.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/InternalBucketMetricValue.java
@@ -34,9 +34,9 @@ import java.util.Map;
 
 public class InternalBucketMetricValue extends InternalNumericMetricsAggregation.SingleValue {
 
-    public final static Type TYPE = new Type("bucket_metric_value");
+    public static final Type TYPE = new Type("bucket_metric_value");
 
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalBucketMetricValue readResult(StreamInput in) throws IOException {
             InternalBucketMetricValue result = new InternalBucketMetricValue();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/avg/AvgBucketPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/avg/AvgBucketPipelineAggregator.java
index 74ae7325051..56d4ed41704 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/avg/AvgBucketPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/avg/AvgBucketPipelineAggregator.java
@@ -35,9 +35,9 @@ import java.util.Map;
 
 public class AvgBucketPipelineAggregator extends BucketMetricsPipelineAggregator {
 
-    public final static Type TYPE = new Type("avg_bucket");
+    public static final Type TYPE = new Type("avg_bucket");
 
-    public final static PipelineAggregatorStreams.Stream STREAM = new PipelineAggregatorStreams.Stream() {
+    public static final PipelineAggregatorStreams.Stream STREAM = new PipelineAggregatorStreams.Stream() {
         @Override
         public AvgBucketPipelineAggregator readResult(StreamInput in) throws IOException {
             AvgBucketPipelineAggregator result = new AvgBucketPipelineAggregator();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/max/MaxBucketPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/max/MaxBucketPipelineAggregator.java
index fa10de557ae..ac3cd069585 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/max/MaxBucketPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/max/MaxBucketPipelineAggregator.java
@@ -37,9 +37,9 @@ import java.util.Map;
 
 public class MaxBucketPipelineAggregator extends BucketMetricsPipelineAggregator {
 
-    public final static Type TYPE = new Type("max_bucket");
+    public static final Type TYPE = new Type("max_bucket");
 
-    public final static PipelineAggregatorStreams.Stream STREAM = new PipelineAggregatorStreams.Stream() {
+    public static final PipelineAggregatorStreams.Stream STREAM = new PipelineAggregatorStreams.Stream() {
         @Override
         public MaxBucketPipelineAggregator readResult(StreamInput in) throws IOException {
             MaxBucketPipelineAggregator result = new MaxBucketPipelineAggregator();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/min/MinBucketPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/min/MinBucketPipelineAggregator.java
index 37e4a89f7c1..e4b86ee9f8d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/min/MinBucketPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/min/MinBucketPipelineAggregator.java
@@ -37,9 +37,9 @@ import java.util.Map;
 
 public class MinBucketPipelineAggregator extends BucketMetricsPipelineAggregator {
 
-    public final static Type TYPE = new Type("min_bucket");
+    public static final Type TYPE = new Type("min_bucket");
 
-    public final static PipelineAggregatorStreams.Stream STREAM = new PipelineAggregatorStreams.Stream() {
+    public static final PipelineAggregatorStreams.Stream STREAM = new PipelineAggregatorStreams.Stream() {
         @Override
         public MinBucketPipelineAggregator readResult(StreamInput in) throws IOException {
             MinBucketPipelineAggregator result = new MinBucketPipelineAggregator();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucket.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucket.java
index 65d04ee12ad..badbb9411a9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucket.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/InternalPercentilesBucket.java
@@ -39,9 +39,9 @@ import java.util.Map;
 
 public class InternalPercentilesBucket extends InternalNumericMetricsAggregation.MultiValue implements PercentilesBucket {
 
-    public final static Type TYPE = new Type("percentiles_bucket");
+    public static final Type TYPE = new Type("percentiles_bucket");
 
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalPercentilesBucket readResult(StreamInput in) throws IOException {
             InternalPercentilesBucket result = new InternalPercentilesBucket();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregator.java
index d21fa53b734..12769680e9f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/percentile/PercentilesBucketPipelineAggregator.java
@@ -38,10 +38,10 @@ import java.util.Map;
 
 public class PercentilesBucketPipelineAggregator extends BucketMetricsPipelineAggregator {
 
-    public final static Type TYPE = new Type("percentiles_bucket");
+    public static final Type TYPE = new Type("percentiles_bucket");
     public final ParseField PERCENTS_FIELD = new ParseField("percents");
 
-    public final static PipelineAggregatorStreams.Stream STREAM = new PipelineAggregatorStreams.Stream() {
+    public static final PipelineAggregatorStreams.Stream STREAM = new PipelineAggregatorStreams.Stream() {
         @Override
         public PercentilesBucketPipelineAggregator readResult(StreamInput in) throws IOException {
             PercentilesBucketPipelineAggregator result = new PercentilesBucketPipelineAggregator();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/sum/SumBucketPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/sum/SumBucketPipelineAggregator.java
index edd912ee8ff..89d0ca9ccf6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/sum/SumBucketPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/sum/SumBucketPipelineAggregator.java
@@ -35,9 +35,9 @@ import java.util.Map;
 
 public class SumBucketPipelineAggregator extends BucketMetricsPipelineAggregator {
 
-    public final static Type TYPE = new Type("sum_bucket");
+    public static final Type TYPE = new Type("sum_bucket");
 
-    public final static PipelineAggregatorStreams.Stream STREAM = new PipelineAggregatorStreams.Stream() {
+    public static final PipelineAggregatorStreams.Stream STREAM = new PipelineAggregatorStreams.Stream() {
         @Override
         public SumBucketPipelineAggregator readResult(StreamInput in) throws IOException {
             SumBucketPipelineAggregator result = new SumBucketPipelineAggregator();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java
index 719f2c2ffe7..35893c7d09e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketscript/BucketScriptPipelineAggregator.java
@@ -51,9 +51,9 @@ import static org.elasticsearch.search.aggregations.pipeline.BucketHelpers.resol
 
 public class BucketScriptPipelineAggregator extends PipelineAggregator {
 
-    public final static Type TYPE = new Type("bucket_script");
+    public static final Type TYPE = new Type("bucket_script");
 
-    public final static PipelineAggregatorStreams.Stream STREAM = in -> {
+    public static final PipelineAggregatorStreams.Stream STREAM = in -> {
         BucketScriptPipelineAggregator result = new BucketScriptPipelineAggregator();
         result.readFrom(in);
         return result;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java
index 28fded6f559..7fff7f04db9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/bucketselector/BucketSelectorPipelineAggregator.java
@@ -46,9 +46,9 @@ import static org.elasticsearch.search.aggregations.pipeline.BucketHelpers.resol
 
 public class BucketSelectorPipelineAggregator extends PipelineAggregator {
 
-    public final static Type TYPE = new Type("bucket_selector");
+    public static final Type TYPE = new Type("bucket_selector");
 
-    public final static PipelineAggregatorStreams.Stream STREAM = new PipelineAggregatorStreams.Stream() {
+    public static final PipelineAggregatorStreams.Stream STREAM = new PipelineAggregatorStreams.Stream() {
         @Override
         public BucketSelectorPipelineAggregator readResult(StreamInput in) throws IOException {
             BucketSelectorPipelineAggregator result = new BucketSelectorPipelineAggregator();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregator.java
index 5f003481f17..9ba381f2a5f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/cumulativesum/CumulativeSumPipelineAggregator.java
@@ -43,9 +43,9 @@ import static org.elasticsearch.search.aggregations.pipeline.BucketHelpers.resol
 
 public class CumulativeSumPipelineAggregator extends PipelineAggregator {
 
-    public final static Type TYPE = new Type("cumulative_sum");
+    public static final Type TYPE = new Type("cumulative_sum");
 
-    public final static PipelineAggregatorStreams.Stream STREAM = in -> {
+    public static final PipelineAggregatorStreams.Stream STREAM = in -> {
         CumulativeSumPipelineAggregator result = new CumulativeSumPipelineAggregator();
         result.readFrom(in);
         return result;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregator.java
index 964c82a4d94..b40666128cb 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/DerivativePipelineAggregator.java
@@ -44,9 +44,9 @@ import static org.elasticsearch.search.aggregations.pipeline.BucketHelpers.resol
 
 public class DerivativePipelineAggregator extends PipelineAggregator {
 
-    public final static Type TYPE = new Type("derivative");
+    public static final Type TYPE = new Type("derivative");
 
-    public final static PipelineAggregatorStreams.Stream STREAM = new PipelineAggregatorStreams.Stream() {
+    public static final PipelineAggregatorStreams.Stream STREAM = new PipelineAggregatorStreams.Stream() {
         @Override
         public DerivativePipelineAggregator readResult(StreamInput in) throws IOException {
             DerivativePipelineAggregator result = new DerivativePipelineAggregator();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/InternalDerivative.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/InternalDerivative.java
index 24579d65a30..dea181ff25c 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/InternalDerivative.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/derivative/InternalDerivative.java
@@ -33,9 +33,9 @@ import java.util.Map;
 
 public class InternalDerivative extends InternalSimpleValue implements Derivative {
 
-    public final static Type TYPE = new Type("derivative");
+    public static final Type TYPE = new Type("derivative");
 
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalDerivative readResult(StreamInput in) throws IOException {
             InternalDerivative result = new InternalDerivative();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregator.java
index 586424de53a..880b4e4e6a0 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/MovAvgPipelineAggregator.java
@@ -48,9 +48,9 @@ import static org.elasticsearch.search.aggregations.pipeline.BucketHelpers.resol
 
 public class MovAvgPipelineAggregator extends PipelineAggregator {
 
-    public final static Type TYPE = new Type("moving_avg");
+    public static final Type TYPE = new Type("moving_avg");
 
-    public final static PipelineAggregatorStreams.Stream STREAM = new PipelineAggregatorStreams.Stream() {
+    public static final PipelineAggregatorStreams.Stream STREAM = new PipelineAggregatorStreams.Stream() {
         @Override
         public MovAvgPipelineAggregator readResult(StreamInput in) throws IOException {
             MovAvgPipelineAggregator result = new MovAvgPipelineAggregator();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregator.java
index 2ebf740bb6f..e18a06e2b20 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/serialdiff/SerialDiffPipelineAggregator.java
@@ -45,9 +45,9 @@ import static org.elasticsearch.search.aggregations.pipeline.BucketHelpers.resol
 
 public class SerialDiffPipelineAggregator extends PipelineAggregator {
 
-    public final static Type TYPE = new Type("serial_diff");
+    public static final Type TYPE = new Type("serial_diff");
 
-    public final static PipelineAggregatorStreams.Stream STREAM = in -> {
+    public static final PipelineAggregatorStreams.Stream STREAM = in -> {
         SerialDiffPipelineAggregator result = new SerialDiffPipelineAggregator();
         result.readFrom(in);
         return result;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/AggregationPath.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/AggregationPath.java
index 84fd26a74f3..746e0e5e161 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/AggregationPath.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/AggregationPath.java
@@ -63,7 +63,7 @@ import java.util.List;
  */
 public class AggregationPath {
 
-    private final static String AGG_DELIM = ">";
+    private static final String AGG_DELIM = ">";
 
     public static AggregationPath parse(String path) {
         String[] elements = Strings.tokenizeToStringArray(path, AGG_DELIM);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java
index 70262096cb0..36999e61279 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java
@@ -67,7 +67,7 @@ public abstract class ValuesSource {
         return false;
     }
 
-    public static abstract class Bytes extends ValuesSource {
+    public abstract static class Bytes extends ValuesSource {
 
         public static final WithOrdinals EMPTY = new WithOrdinals() {
 
@@ -98,7 +98,7 @@ public abstract class ValuesSource {
             }
         }
 
-        public static abstract class WithOrdinals extends Bytes {
+        public abstract static class WithOrdinals extends Bytes {
 
             @Override
             public Bits docsWithValue(LeafReaderContext context) {
@@ -224,7 +224,7 @@ public abstract class ValuesSource {
 
     }
 
-    public static abstract class Numeric extends ValuesSource {
+    public abstract static class Numeric extends ValuesSource {
 
         public static final Numeric EMPTY = new Numeric() {
 
@@ -487,7 +487,7 @@ public abstract class ValuesSource {
         }
     }
 
-    public static abstract class GeoPoint extends ValuesSource {
+    public abstract static class GeoPoint extends ValuesSource {
 
         public static final GeoPoint EMPTY = new GeoPoint() {
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
index a55b9035c03..8f14a1ffaf9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceAggregationBuilder.java
@@ -49,7 +49,7 @@ import java.util.Objects;
 public abstract class ValuesSourceAggregationBuilder<VS extends ValuesSource, AB extends ValuesSourceAggregationBuilder<VS, AB>>
         extends AbstractAggregationBuilder<AB> {
 
-    public static abstract class LeafOnly<VS extends ValuesSource, AB extends ValuesSourceAggregationBuilder<VS, AB>>
+    public abstract static class LeafOnly<VS extends ValuesSource, AB extends ValuesSourceAggregationBuilder<VS, AB>>
             extends ValuesSourceAggregationBuilder<VS, AB> {
 
         protected LeafOnly(String name, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType) {
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseParseElement.java b/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseParseElement.java
index f43ed68d769..752aeecf04e 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseParseElement.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/FetchSubPhaseParseElement.java
@@ -29,7 +29,7 @@ import org.elasticsearch.search.internal.SearchContext;
 public abstract class FetchSubPhaseParseElement<SubPhaseContext extends FetchSubPhaseContext> implements SearchParseElement {
 
     @Override
-    final public void parse(XContentParser parser, SearchContext context) throws Exception {
+    public final void parse(XContentParser parser, SearchContext context) throws Exception {
         SubPhaseContext fetchSubPhaseContext = context.getFetchSubPhaseContext(getContextFactory());
         // this is to make sure that the SubFetchPhase knows it should execute
         fetchSubPhaseContext.setHitExecutionNeeded(true);
diff --git a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java
index f34da5301d5..577d2d64943 100644
--- a/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java
+++ b/core/src/main/java/org/elasticsearch/search/fetch/innerhits/InnerHitsContext.java
@@ -86,7 +86,7 @@ public final class InnerHitsContext {
         innerHits.put(innerHit.getName(), innerHit);
     }
 
-    public static abstract class BaseInnerHits extends SubSearchContext {
+    public abstract static class BaseInnerHits extends SubSearchContext {
 
         private final String name;
         private InnerHitsContext childInnerHits;
diff --git a/core/src/main/java/org/elasticsearch/search/highlight/HighlightBuilder.java b/core/src/main/java/org/elasticsearch/search/highlight/HighlightBuilder.java
index 046bea64c91..92bdf6e8aa8 100644
--- a/core/src/main/java/org/elasticsearch/search/highlight/HighlightBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/highlight/HighlightBuilder.java
@@ -91,7 +91,7 @@ public class HighlightBuilder extends AbstractHighlighterBuilder<HighlightBuilde
     /**
      * a {@link FieldOptions} with default settings
      */
-    final static FieldOptions defaultOptions = new SearchContextHighlight.FieldOptions.Builder()
+    static final FieldOptions defaultOptions = new SearchContextHighlight.FieldOptions.Builder()
             .preTags(DEFAULT_PRE_TAGS).postTags(DEFAULT_POST_TAGS).scoreOrdered(DEFAULT_SCORE_ORDERED)
             .highlightFilter(DEFAULT_HIGHLIGHT_FILTER).requireFieldMatch(DEFAULT_REQUIRE_FIELD_MATCH)
             .forceSource(DEFAULT_FORCE_SOURCE).fragmentCharSize(DEFAULT_FRAGMENT_CHAR_SIZE)
diff --git a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
index e52bfba2341..191537b4de5 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/InternalSearchHit.java
@@ -762,7 +762,7 @@ public class InternalSearchHit implements SearchHit {
         }
     }
 
-    public final static class InternalNestedIdentity implements NestedIdentity, Streamable, ToXContent {
+    public static final class InternalNestedIdentity implements NestedIdentity, Streamable, ToXContent {
 
         private Text field;
         private int offset;
diff --git a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
index 7b29cabdf16..b711748b7a8 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/SearchContext.java
@@ -70,7 +70,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 public abstract class SearchContext implements Releasable {
 
     private static ThreadLocal<SearchContext> current = new ThreadLocal<>();
-    public final static int DEFAULT_TERMINATE_AFTER = 0;
+    public static final int DEFAULT_TERMINATE_AFTER = 0;
 
     public static void setCurrent(SearchContext value) {
         current.set(value);
diff --git a/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java b/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
index 363b6ad076a..9276e86edda 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java
@@ -43,7 +43,7 @@ public class SubSearchContext extends FilteredSearchContext {
 
     // By default return 3 hits per bucket. A higher default would make the response really large by default, since
     // the to hits are returned per bucket.
-    private final static int DEFAULT_SIZE = 3;
+    private static final int DEFAULT_SIZE = 3;
 
     private int from;
     private int size = DEFAULT_SIZE;
diff --git a/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java b/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
index 4fc814a0448..772f91a525b 100644
--- a/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
+++ b/core/src/main/java/org/elasticsearch/search/query/QuerySearchResult.java
@@ -143,7 +143,7 @@ public class QuerySearchResult extends QuerySearchResultProvider {
      * Returns the profiled results for this search, or potentially null if result was empty
      * @return The profiled results, or null
      */
-    public @Nullable ProfileShardResult profileResults() {
+    @Nullable public ProfileShardResult profileResults() {
         return profileShardResults;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java
index 29d62f3c7d3..2a57e41cfaf 100644
--- a/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java
+++ b/core/src/main/java/org/elasticsearch/search/rescore/QueryRescorer.java
@@ -120,7 +120,7 @@ public final class QueryRescorer implements Rescorer {
         }
     }
 
-    private final static Comparator<ScoreDoc> SCORE_DOC_COMPARATOR = new Comparator<ScoreDoc>() {
+    private static final Comparator<ScoreDoc> SCORE_DOC_COMPARATOR = new Comparator<ScoreDoc>() {
         @Override
         public int compare(ScoreDoc o1, ScoreDoc o2) {
             int cmp = Float.compare(o2.score, o1.score);
diff --git a/core/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java b/core/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java
index 97c79aefa71..b6b3a4613d8 100644
--- a/core/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java
@@ -55,7 +55,7 @@ public class SliceBuilder extends ToXContentToBytes implements Writeable {
     public static final ParseField FIELD_FIELD = new ParseField("field");
     public static final ParseField ID_FIELD = new ParseField("id");
     public static final ParseField MAX_FIELD = new ParseField("max");
-    private final static ObjectParser<SliceBuilder, QueryParseContext> PARSER =
+    private static final ObjectParser<SliceBuilder, QueryParseContext> PARSER =
         new ObjectParser<>("slice", SliceBuilder::new);
 
     static {
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java b/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java
index 5cf599beda4..5b4dcafe383 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java
@@ -99,7 +99,7 @@ public final class SuggestUtils {
         return result.get();
     }
 
-    public static abstract class TokenConsumer {
+    public abstract static class TokenConsumer {
         protected CharTermAttribute charTermAttr;
         protected PositionIncrementAttribute posIncAttr;
         protected OffsetAttribute offsetAttr;
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
index 16a2804b378..759ab8d206b 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggester.java
@@ -201,12 +201,12 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext>
 
     // TODO: this should be refactored and moved to lucene
     // see https://issues.apache.org/jira/browse/LUCENE-6880
-    private final static class TopDocumentsCollector extends TopSuggestDocsCollector {
+    private static final class TopDocumentsCollector extends TopSuggestDocsCollector {
 
         /**
          * Holds a list of suggest meta data for a doc
          */
-        private final static class SuggestDoc extends TopSuggestDocs.SuggestScoreDoc {
+        private static final class SuggestDoc extends TopSuggestDocs.SuggestScoreDoc {
 
             private List<TopSuggestDocs.SuggestScoreDoc> suggestScoreDocs;
 
@@ -252,7 +252,7 @@ public class CompletionSuggester extends Suggester<CompletionSuggestionContext>
             }
         }
 
-        private final static class SuggestDocPriorityQueue extends PriorityQueue<SuggestDoc> {
+        private static final class SuggestDocPriorityQueue extends PriorityQueue<SuggestDoc> {
 
             public SuggestDocPriorityQueue(int maxSize) {
                 super(maxSize);
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java
index ed3ebead7f5..2d10a5b7030 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java
@@ -128,7 +128,7 @@ public final class CompletionSuggestion extends Suggest.Suggestion<CompletionSug
         return new Entry();
     }
 
-    public final static class Entry extends Suggest.Suggestion.Entry<CompletionSuggestion.Entry.Option> {
+    public static final class Entry extends Suggest.Suggestion.Entry<CompletionSuggestion.Entry.Option> {
 
         public Entry(Text text, int offset, int length) {
             super(text, offset, length);
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/Completion090PostingsFormat.java b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/Completion090PostingsFormat.java
index 8e9e51597dc..b3b73306f97 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/Completion090PostingsFormat.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/Completion090PostingsFormat.java
@@ -74,9 +74,9 @@ public class Completion090PostingsFormat extends PostingsFormat {
     public static final int SUGGEST_VERSION_CURRENT = SUGGEST_CODEC_VERSION;
     public static final String EXTENSION = "cmp";
 
-    private final static ESLogger logger = Loggers.getLogger(Completion090PostingsFormat.class);
+    private static final ESLogger logger = Loggers.getLogger(Completion090PostingsFormat.class);
     private PostingsFormat delegatePostingsFormat;
-    private final static Map<String, CompletionLookupProvider> providers;
+    private static final Map<String, CompletionLookupProvider> providers;
     private CompletionLookupProvider writeProvider;
 
 
@@ -267,7 +267,7 @@ public class Completion090PostingsFormat extends PostingsFormat {
         }
     }
 
-    public static abstract class CompletionLookupProvider implements PayloadProcessor, CompletionTokenStream.ToFiniteStrings {
+    public abstract static class CompletionLookupProvider implements PayloadProcessor, CompletionTokenStream.ToFiniteStrings {
 
         public static final char UNIT_SEPARATOR = '\u001f';
 
@@ -344,7 +344,7 @@ public class Completion090PostingsFormat extends PostingsFormat {
         return completionStats;
     }
 
-    public static abstract class LookupFactory implements Accountable {
+    public abstract static class LookupFactory implements Accountable {
         public abstract Lookup getLookup(CompletionFieldMapper2x.CompletionFieldType fieldType,
                                          CompletionSuggestionContext suggestionContext);
         public abstract CompletionStats stats(String ... fields);
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/context/ContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/context/ContextMapping.java
index dd23bdecb0e..b92d2e1b614 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/context/ContextMapping.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/context/ContextMapping.java
@@ -204,7 +204,7 @@ public abstract class ContextMapping implements ToXContent {
      *  form a {@link ContextConfig} which is used to build the underlying {@link FST}. This class hold
      *  a simple method wrapping a {@link TokenStream} by provided document informations.
      */
-    public static abstract class ContextConfig {
+    public abstract static class ContextConfig {
 
         /**
          * Wrap a {@link TokenStream} for building suggestions to use context informations
@@ -225,7 +225,7 @@ public abstract class ContextMapping implements ToXContent {
      * {@link ContextMapping} such a query is used to wrap the {@link TokenStream} of the actual
      * suggestion request into a {@link TokenStream} with the context settings
      */
-    public static abstract class ContextQuery implements ToXContent {
+    public abstract static class ContextQuery implements ToXContent {
 
         protected final String name;
 
diff --git a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
index 74434413ae9..4c4d5ed95bf 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
@@ -496,7 +496,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
                 UPDATE_RESTORE_ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME);
     }
 
-    public final static class RestoreCompletionResponse {
+    public static final class RestoreCompletionResponse {
         private final Snapshot snapshot;
         private final RestoreInfo restoreInfo;
 
@@ -869,33 +869,33 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
      */
     public static class RestoreRequest {
 
-        final private String cause;
+        private final String cause;
 
-        final private String repositoryName;
+        private final String repositoryName;
 
-        final private String snapshotName;
+        private final String snapshotName;
 
-        final private String[] indices;
+        private final String[] indices;
 
-        final private String renamePattern;
+        private final String renamePattern;
 
-        final private String renameReplacement;
+        private final String renameReplacement;
 
-        final private IndicesOptions indicesOptions;
+        private final IndicesOptions indicesOptions;
 
-        final private Settings settings;
+        private final Settings settings;
 
-        final private TimeValue masterNodeTimeout;
+        private final TimeValue masterNodeTimeout;
 
-        final private boolean includeGlobalState;
+        private final boolean includeGlobalState;
 
-        final private boolean partial;
+        private final boolean partial;
 
-        final private boolean includeAliases;
+        private final boolean includeAliases;
 
-        final private Settings indexSettings;
+        private final Settings indexSettings;
 
-        final private String[] ignoreIndexSettings;
+        private final String[] ignoreIndexSettings;
 
         /**
          * Constructs new restore request
diff --git a/core/src/main/java/org/elasticsearch/tasks/LoggingTaskListener.java b/core/src/main/java/org/elasticsearch/tasks/LoggingTaskListener.java
index b2016f094f5..002d0b6a468 100644
--- a/core/src/main/java/org/elasticsearch/tasks/LoggingTaskListener.java
+++ b/core/src/main/java/org/elasticsearch/tasks/LoggingTaskListener.java
@@ -27,7 +27,7 @@ import org.elasticsearch.common.logging.Loggers;
  * need a listener but aren't returning the result to the user.
  */
 public final class LoggingTaskListener<Response> implements TaskListener<Response> {
-    private final static ESLogger logger = Loggers.getLogger(LoggingTaskListener.class);
+    private static final ESLogger logger = Loggers.getLogger(LoggingTaskListener.class);
 
     /**
      * Get the instance of NoopActionListener cast appropriately.
diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskId.java b/core/src/main/java/org/elasticsearch/tasks/TaskId.java
index eaab8d16b4d..1aeceef247f 100644
--- a/core/src/main/java/org/elasticsearch/tasks/TaskId.java
+++ b/core/src/main/java/org/elasticsearch/tasks/TaskId.java
@@ -31,7 +31,7 @@ import java.io.IOException;
  */
 public final class TaskId implements Writeable {
 
-    public final static TaskId EMPTY_TASK_ID = new TaskId();
+    public static final TaskId EMPTY_TASK_ID = new TaskId();
 
     private final String nodeId;
     private final long id;
diff --git a/core/src/main/java/org/elasticsearch/threadpool/ExecutorBuilder.java b/core/src/main/java/org/elasticsearch/threadpool/ExecutorBuilder.java
index 5f36d391eba..0f9b79156e6 100644
--- a/core/src/main/java/org/elasticsearch/threadpool/ExecutorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/threadpool/ExecutorBuilder.java
@@ -78,7 +78,7 @@ public abstract class ExecutorBuilder<U extends ExecutorBuilder.ExecutorSettings
      */
     abstract String formatInfo(ThreadPool.Info info);
 
-    static abstract class ExecutorSettings {
+    abstract static class ExecutorSettings {
 
         protected final String nodeName;
 
diff --git a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
index 551eeea838d..c465f98c182 100644
--- a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
+++ b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
@@ -93,7 +93,7 @@ public class ThreadPool extends AbstractComponent implements Closeable {
             this.type = type;
         }
 
-        private final static Map<String, ThreadPoolType> TYPE_MAP;
+        private static final Map<String, ThreadPoolType> TYPE_MAP;
 
         static {
             Map<String, ThreadPoolType> typeMap = new HashMap<>();
diff --git a/core/src/main/java/org/elasticsearch/transport/TransportService.java b/core/src/main/java/org/elasticsearch/transport/TransportService.java
index 21a43560eba..e2f3ab44eea 100644
--- a/core/src/main/java/org/elasticsearch/transport/TransportService.java
+++ b/core/src/main/java/org/elasticsearch/transport/TransportService.java
@@ -909,7 +909,7 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
      * This handler wrapper ensures that the response thread executes with the correct thread context. Before any of the4 handle methods
      * are invoked we restore the context.
      */
-    private final static class ContextRestoreResponseHandler<T extends TransportResponse> implements TransportResponseHandler<T> {
+    private static final class ContextRestoreResponseHandler<T extends TransportResponse> implements TransportResponseHandler<T> {
         private final TransportResponseHandler<T> delegate;
         private final ThreadContext.StoredContext threadContext;
 
@@ -944,8 +944,8 @@ public class TransportService extends AbstractLifecycleComponent<TransportServic
     static class DirectResponseChannel implements TransportChannel {
         final ESLogger logger;
         final DiscoveryNode localNode;
-        final private String action;
-        final private long requestId;
+        private final String action;
+        private final long requestId;
         final TransportServiceAdapter adapter;
         final ThreadPool threadPool;
 
diff --git a/core/src/main/java/org/elasticsearch/transport/TransportSettings.java b/core/src/main/java/org/elasticsearch/transport/TransportSettings.java
index e36d4e403c4..6965e4ed24f 100644
--- a/core/src/main/java/org/elasticsearch/transport/TransportSettings.java
+++ b/core/src/main/java/org/elasticsearch/transport/TransportSettings.java
@@ -34,7 +34,7 @@ import static org.elasticsearch.common.settings.Setting.listSetting;
  * a collection of settings related to transport components, which are also needed in org.elasticsearch.bootstrap.Security
  * This class should only contain static code which is *safe* to load before the security manager is enforced.
  */
-final public class TransportSettings {
+public final class TransportSettings {
 
     public static final Setting<List<String>> HOST =
         listSetting("transport.host", emptyList(), Function.identity(), Property.NodeScope);
diff --git a/core/src/main/java/org/elasticsearch/transport/Transports.java b/core/src/main/java/org/elasticsearch/transport/Transports.java
index 1186c821309..15bf2c833f6 100644
--- a/core/src/main/java/org/elasticsearch/transport/Transports.java
+++ b/core/src/main/java/org/elasticsearch/transport/Transports.java
@@ -27,7 +27,7 @@ public enum Transports {
     ;
 
     /** threads whose name is prefixed by this string will be considered network threads, even though they aren't */
-    public final static String TEST_MOCK_TRANSPORT_THREAD_PREFIX = "__mock_network_thread";
+    public static final String TEST_MOCK_TRANSPORT_THREAD_PREFIX = "__mock_network_thread";
 
     /**
      * Utility method to detect whether a thread is a network thread. Typically
diff --git a/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java b/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
index a2ffca2e804..ebb29d6feae 100644
--- a/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
@@ -81,7 +81,7 @@ public class LocalTransport extends AbstractLifecycleComponent<Transport> implem
     private volatile TransportServiceAdapter transportServiceAdapter;
     private volatile BoundTransportAddress boundAddress;
     private volatile LocalTransportAddress localAddress;
-    private final static ConcurrentMap<LocalTransportAddress, LocalTransport> transports = newConcurrentMap();
+    private static final ConcurrentMap<LocalTransportAddress, LocalTransport> transports = newConcurrentMap();
     private static final AtomicLong transportAddressIdGenerator = new AtomicLong();
     private final ConcurrentMap<DiscoveryNode, LocalTransport> connectedNodes = newConcurrentMap();
     protected final NamedWriteableRegistry namedWriteableRegistry;
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
index 3c75f68eeb0..d591a75d051 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
@@ -668,7 +668,7 @@ public class NettyTransport extends TcpTransport<Channel> {
         }
     }
 
-    private final static class ChannelBufferStreamOutput extends StreamOutput {
+    private static final class ChannelBufferStreamOutput extends StreamOutput {
 
         private final ChannelBuffer buffer;
         private int offset;
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
index c4d49d899b9..56756ad9fdb 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
@@ -249,7 +249,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
     /**
      * Test class for testing task operations
      */
-    static abstract class TestTasksAction extends TransportTasksAction<Task, TestTasksRequest, TestTasksResponse, TestTaskResponse> {
+    abstract static class TestTasksAction extends TransportTasksAction<Task, TestTasksRequest, TestTasksResponse, TestTaskResponse> {
 
         protected TestTasksAction(Settings settings, String actionName, ThreadPool threadPool,
                 ClusterService clusterService, TransportService transportService) {
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
index 6e3e5d76224..1cd1704e164 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java
@@ -215,7 +215,7 @@ public class IndicesShardStoreRequestIT extends ESIntegTestCase {
         client().admin().indices().prepareFlush().setForce(true).setWaitIfOngoing(true).execute().actionGet();
     }
 
-    private final static class IndexNodePredicate implements Predicate<Settings> {
+    private static final class IndexNodePredicate implements Predicate<Settings> {
         private final Set<String> nodesWithShard;
 
         public IndexNodePredicate(String index) {
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
index 6ae7559ba62..6111c4c9953 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
@@ -59,7 +59,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 
 public class TransportBulkActionTookTests extends ESTestCase {
 
-    static private ThreadPool threadPool;
+    private static ThreadPool threadPool;
     private ClusterService clusterService;
 
     @BeforeClass
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
index dc40fda3f8e..b524082718a 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
@@ -260,7 +260,7 @@ public class ClusterStateCreationUtils {
                 new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
     }
 
-    static private String selectAndRemove(Set<String> strings) {
+    private static String selectAndRemove(Set<String> strings) {
         String selection = randomFrom(strings.toArray(new String[strings.size()]));
         strings.remove(selection);
         return selection;
diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java b/core/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java
index 208945a6179..d9f351120b2 100644
--- a/core/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java
+++ b/core/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java
@@ -68,10 +68,10 @@ import static org.hamcrest.Matchers.equalTo;
 public abstract class AbstractTermVectorsTestCase extends ESIntegTestCase {
 
     protected static class TestFieldSetting {
-        final public String name;
-        final public boolean storedOffset;
-        final public boolean storedPayloads;
-        final public boolean storedPositions;
+        public final String name;
+        public final boolean storedOffset;
+        public final boolean storedPayloads;
+        public final boolean storedPositions;
 
         public TestFieldSetting(String name, boolean storedOffset, boolean storedPayloads, boolean storedPositions) {
             this.name = name;
@@ -124,9 +124,9 @@ public abstract class AbstractTermVectorsTestCase extends ESIntegTestCase {
     }
 
     protected static class TestDoc {
-        final public String id;
-        final public TestFieldSetting[] fieldSettings;
-        final public String[] fieldContent;
+        public final String id;
+        public final TestFieldSetting[] fieldSettings;
+        public final String[] fieldContent;
         public String index = "test";
         public String alias = "alias";
         public String type = "type1";
@@ -163,11 +163,11 @@ public abstract class AbstractTermVectorsTestCase extends ESIntegTestCase {
     }
 
     protected static class TestConfig {
-        final public TestDoc doc;
-        final public String[] selectedFields;
-        final public boolean requestPositions;
-        final public boolean requestOffsets;
-        final public boolean requestPayloads;
+        public final TestDoc doc;
+        public final String[] selectedFields;
+        public final boolean requestPositions;
+        public final boolean requestOffsets;
+        public final boolean requestPayloads;
         public Class expectedException = null;
 
         public TestConfig(TestDoc doc, String[] selectedFields, boolean requestPositions, boolean requestOffsets, boolean requestPayloads) {
diff --git a/core/src/test/java/org/elasticsearch/bwcompat/BasicAnalysisBackwardCompatibilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/BasicAnalysisBackwardCompatibilityIT.java
index 40995ff778b..be0848ed4dc 100644
--- a/core/src/test/java/org/elasticsearch/bwcompat/BasicAnalysisBackwardCompatibilityIT.java
+++ b/core/src/test/java/org/elasticsearch/bwcompat/BasicAnalysisBackwardCompatibilityIT.java
@@ -41,7 +41,7 @@ import static org.hamcrest.Matchers.equalTo;
 public class BasicAnalysisBackwardCompatibilityIT extends ESBackcompatTestCase {
 
     // This pattern match characters with Line_Break = Complex_Content.
-    final static Pattern complexUnicodeChars = Pattern.compile("[\u17B4\u17B5\u17D3\u17CB-\u17D1\u17DD\u1036\u17C6\u1A74\u1038\u17C7\u0E4E\u0E47-\u0E4D\u0EC8-\u0ECD\uAABF\uAAC1\u1037\u17C8-\u17CA\u1A75-\u1A7C\u1AA8-\u1AAB\uAADE\uAADF\u1AA0-\u1AA6\u1AAC\u1AAD\u109E\u109F\uAA77-\uAA79\u0E46\u0EC6\u17D7\u1AA7\uA9E6\uAA70\uAADD\u19DA\u0E01-\u0E3A\u0E40-\u0E45\u0EDE\u0E81\u0E82\u0E84\u0E87\u0E88\u0EAA\u0E8A\u0EDF\u0E8D\u0E94-\u0E97\u0E99-\u0E9F\u0EA1-\u0EA3\u0EA5\u0EA7\u0EAB\u0EDC\u0EDD\u0EAD-\u0EB9\u0EBB-\u0EBD\u0EC0-\u0EC4\uAA80-\uAABE\uAAC0\uAAC2\uAADB\uAADC\u1000\u1075\u1001\u1076\u1002\u1077\uAA60\uA9E9\u1003\uA9E0\uA9EA\u1004\u105A\u1005\u1078\uAA61\u1006\uA9E1\uAA62\uAA7E\u1007\uAA63\uA9EB\u1079\uAA72\u1008\u105B\uA9E2\uAA64\uA9EC\u1061\uAA7F\u1009\u107A\uAA65\uA9E7\u100A\u100B\uAA66\u100C\uAA67\u100D\uAA68\uA9ED\u100E\uAA69\uA9EE\u100F\u106E\uA9E3\uA9EF\u1010-\u1012\u107B\uA9FB\u1013\uAA6A\uA9FC\u1014\u107C\uAA6B\u105E\u1015\u1016\u107D\u107E\uAA6F\u108E\uA9E8\u1017\u107F\uA9FD\u1018\uA9E4\uA9FE\u1019\u105F\u101A\u103B\u101B\uAA73\uAA7A\u103C\u101C\u1060\u101D\u103D\u1082\u1080\u1050\u1051\u1065\u101E\u103F\uAA6C\u101F\u1081\uAA6D\u103E\uAA6E\uAA71\u1020\uA9FA\u105C\u105D\u106F\u1070\u1066\u1021-\u1026\u1052-\u1055\u1027-\u102A\u102C\u102B\u1083\u1072\u109C\u102D\u1071\u102E\u1033\u102F\u1073\u1074\u1030\u1056-\u1059\u1031\u1084\u1035\u1085\u1032\u109D\u1034\u1062\u1067\u1068\uA9E5\u1086\u1039\u103A\u1063\u1064\u1069-\u106D\u1087\u108B\u1088\u108C\u108D\u1089\u108A\u108F\u109A\u109B\uAA7B-\uAA7D\uAA74-\uAA76\u1780-\u17A2\u17DC\u17A3-\u17B3\u17B6-\u17C5\u17D2\u1950-\u196D\u1970-\u1974\u1980-\u199C\u19DE\u19DF\u199D-\u19AB\u19B0-\u19C9\u1A20-\u1A26\u1A58\u1A59\u1A27-\u1A3B\u1A5A\u1A5B\u1A3C-\u1A46\u1A54\u1A47-\u1A4C\u1A53\u1A6B\u1A55-\u1A57\u1A5C-\u1A5E\u1A4D-\u1A52\u1A61\u1A6C\u1A62-\u1A6A\u1A6E\u1A6F\u1A73\u1A70-\u1A72\u1A6D\u1A60]");
+    static final Pattern complexUnicodeChars = Pattern.compile("[\u17B4\u17B5\u17D3\u17CB-\u17D1\u17DD\u1036\u17C6\u1A74\u1038\u17C7\u0E4E\u0E47-\u0E4D\u0EC8-\u0ECD\uAABF\uAAC1\u1037\u17C8-\u17CA\u1A75-\u1A7C\u1AA8-\u1AAB\uAADE\uAADF\u1AA0-\u1AA6\u1AAC\u1AAD\u109E\u109F\uAA77-\uAA79\u0E46\u0EC6\u17D7\u1AA7\uA9E6\uAA70\uAADD\u19DA\u0E01-\u0E3A\u0E40-\u0E45\u0EDE\u0E81\u0E82\u0E84\u0E87\u0E88\u0EAA\u0E8A\u0EDF\u0E8D\u0E94-\u0E97\u0E99-\u0E9F\u0EA1-\u0EA3\u0EA5\u0EA7\u0EAB\u0EDC\u0EDD\u0EAD-\u0EB9\u0EBB-\u0EBD\u0EC0-\u0EC4\uAA80-\uAABE\uAAC0\uAAC2\uAADB\uAADC\u1000\u1075\u1001\u1076\u1002\u1077\uAA60\uA9E9\u1003\uA9E0\uA9EA\u1004\u105A\u1005\u1078\uAA61\u1006\uA9E1\uAA62\uAA7E\u1007\uAA63\uA9EB\u1079\uAA72\u1008\u105B\uA9E2\uAA64\uA9EC\u1061\uAA7F\u1009\u107A\uAA65\uA9E7\u100A\u100B\uAA66\u100C\uAA67\u100D\uAA68\uA9ED\u100E\uAA69\uA9EE\u100F\u106E\uA9E3\uA9EF\u1010-\u1012\u107B\uA9FB\u1013\uAA6A\uA9FC\u1014\u107C\uAA6B\u105E\u1015\u1016\u107D\u107E\uAA6F\u108E\uA9E8\u1017\u107F\uA9FD\u1018\uA9E4\uA9FE\u1019\u105F\u101A\u103B\u101B\uAA73\uAA7A\u103C\u101C\u1060\u101D\u103D\u1082\u1080\u1050\u1051\u1065\u101E\u103F\uAA6C\u101F\u1081\uAA6D\u103E\uAA6E\uAA71\u1020\uA9FA\u105C\u105D\u106F\u1070\u1066\u1021-\u1026\u1052-\u1055\u1027-\u102A\u102C\u102B\u1083\u1072\u109C\u102D\u1071\u102E\u1033\u102F\u1073\u1074\u1030\u1056-\u1059\u1031\u1084\u1035\u1085\u1032\u109D\u1034\u1062\u1067\u1068\uA9E5\u1086\u1039\u103A\u1063\u1064\u1069-\u106D\u1087\u108B\u1088\u108C\u108D\u1089\u108A\u108F\u109A\u109B\uAA7B-\uAA7D\uAA74-\uAA76\u1780-\u17A2\u17DC\u17A3-\u17B3\u17B6-\u17C5\u17D2\u1950-\u196D\u1970-\u1974\u1980-\u199C\u19DE\u19DF\u199D-\u19AB\u19B0-\u19C9\u1A20-\u1A26\u1A58\u1A59\u1A27-\u1A3B\u1A5A\u1A5B\u1A3C-\u1A46\u1A54\u1A47-\u1A4C\u1A53\u1A6B\u1A55-\u1A57\u1A5C-\u1A5E\u1A4D-\u1A52\u1A61\u1A6C\u1A62-\u1A6A\u1A6E\u1A6F\u1A73\u1A70-\u1A72\u1A6D\u1A60]");
 
     /**
      * Simple upgrade test for analyzers to make sure they analyze to the same tokens after upgrade
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java
index 2d3e44db68a..32072282d6f 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java
@@ -50,7 +50,7 @@ public class PrimaryTermsTests extends ESAllocationTestCase {
     private RoutingTable testRoutingTable;
     private int numberOfShards;
     private int numberOfReplicas;
-    private final static Settings DEFAULT_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
+    private static final Settings DEFAULT_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
     private AllocationService allocationService;
     private ClusterState clusterState;
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java
index 2d1a467a001..9da5e76ed1f 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java
@@ -47,7 +47,7 @@ public class RoutingTableTests extends ESAllocationTestCase {
     private int numberOfReplicas;
     private int shardsPerIndex;
     private int totalNumberOfShards;
-    private final static Settings DEFAULT_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
+    private static final Settings DEFAULT_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build();
     private final AllocationService ALLOCATION_SERVICE = createAllocationService(Settings.builder()
             .put("cluster.routing.allocation.node_concurrent_recoveries", Integer.MAX_VALUE) // don't limit recoveries
             .put("cluster.routing.allocation.node_initial_primaries_recoveries", Integer.MAX_VALUE)
diff --git a/core/src/test/java/org/elasticsearch/common/geo/GeoJSONShapeParserTests.java b/core/src/test/java/org/elasticsearch/common/geo/GeoJSONShapeParserTests.java
index 566d2148cae..76376a4d30d 100644
--- a/core/src/test/java/org/elasticsearch/common/geo/GeoJSONShapeParserTests.java
+++ b/core/src/test/java/org/elasticsearch/common/geo/GeoJSONShapeParserTests.java
@@ -56,7 +56,7 @@ import static org.elasticsearch.common.geo.builders.ShapeBuilder.SPATIAL_CONTEXT
  */
 public class GeoJSONShapeParserTests extends ESTestCase {
 
-    private final static GeometryFactory GEOMETRY_FACTORY = SPATIAL_CONTEXT.getGeometryFactory();
+    private static final GeometryFactory GEOMETRY_FACTORY = SPATIAL_CONTEXT.getGeometryFactory();
 
     public void testParse_simplePoint() throws IOException {
         String pointGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "Point")
diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
index e8a52379a11..94f07369770 100644
--- a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
@@ -459,7 +459,7 @@ public class BytesStreamsTests extends ESTestCase {
         out.close();
     }
 
-    private static abstract class BaseNamedWriteable implements NamedWriteable {
+    private abstract static class BaseNamedWriteable implements NamedWriteable {
 
     }
 
diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
index baea1d7a951..06d39398c8e 100644
--- a/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
+++ b/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java
@@ -143,7 +143,7 @@ public class StreamTests extends ESTestCase {
         assertThat(targetArray, equalTo(sourceArray));
     }
 
-    final static class WriteableString implements Writeable {
+    static final class WriteableString implements Writeable {
         final String string;
 
         public WriteableString(String string) {
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
index 159d8a97be4..a8d26e87ecf 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java
@@ -35,7 +35,7 @@ import org.elasticsearch.test.ESTestCase;
 
 public class ObjectParserTests extends ESTestCase {
 
-    private final static ParseFieldMatcherSupplier STRICT_PARSING = () -> ParseFieldMatcher.STRICT;
+    private static final ParseFieldMatcherSupplier STRICT_PARSING = () -> ParseFieldMatcher.STRICT;
 
     public void testBasics() throws IOException {
         XContentParser parser = XContentType.JSON.xContent().createParser(
diff --git a/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java b/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java
index f6aac190c4b..bb38e329103 100644
--- a/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java
@@ -40,7 +40,7 @@ import static org.hamcrest.Matchers.not;
 
 public class BlockingClusterStatePublishResponseHandlerTests extends ESTestCase {
 
-    static private class PublishResponder extends AbstractRunnable {
+    private static class PublishResponder extends AbstractRunnable {
 
         final boolean fail;
         final DiscoveryNode node;
diff --git a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
index 7f89acd169e..13e19e84978 100644
--- a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
+++ b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
@@ -170,7 +170,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
         return nodes;
     }
 
-    final static Settings DEFAULT_SETTINGS = Settings.builder()
+    static final Settings DEFAULT_SETTINGS = Settings.builder()
             .put(FaultDetection.PING_TIMEOUT_SETTING.getKey(), "1s") // for hitting simulated network failures quickly
             .put(FaultDetection.PING_RETRIES_SETTING.getKey(), "1") // for hitting simulated network failures quickly
             .put("discovery.zen.join_timeout", "10s")  // still long to induce failures but to long so test won't time out
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
index 135352343b6..15c8d312952 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
@@ -581,7 +581,7 @@ public class NodeJoinControllerTests extends ESTestCase {
         }
     }
 
-    final static AtomicInteger joinId = new AtomicInteger();
+    static final AtomicInteger joinId = new AtomicInteger();
 
     private SimpleFuture joinNodeAsync(final DiscoveryNode node) throws InterruptedException {
         final SimpleFuture future = new SimpleFuture("join of " + node + " (id [" + joinId.incrementAndGet() + "]");
diff --git a/core/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java b/core/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java
index d6e8d61a7a6..1e35bcdd469 100644
--- a/core/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java
+++ b/core/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java
@@ -36,7 +36,7 @@ import static org.hamcrest.Matchers.hasItem;
 
 @ClusterScope(scope = Scope.TEST, numDataNodes = 0)
 public class RecoverAfterNodesIT extends ESIntegTestCase {
-    private final static TimeValue BLOCK_WAIT_TIMEOUT = TimeValue.timeValueSeconds(10);
+    private static final TimeValue BLOCK_WAIT_TIMEOUT = TimeValue.timeValueSeconds(10);
 
     public Set<ClusterBlock> waitForNoBlocksOnNode(TimeValue timeout, Client nodeClient) throws InterruptedException {
         long start = System.currentTimeMillis();
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
index 68e59527982..37d0436c9db 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java
@@ -32,7 +32,7 @@ import java.util.List;
 public abstract class FieldTypeTestCase extends ESTestCase {
 
     /** Abstraction for mutating a property of a MappedFieldType */
-    public static abstract class Modifier {
+    public abstract static class Modifier {
         /** The name of the property that is being modified. Used in test failure messages. */
         public final String property;
         /** true if this modifier only makes types incompatible in strict mode, false otherwise */
diff --git a/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
index edf5a0d28f2..1191806bc8e 100644
--- a/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
@@ -107,10 +107,10 @@ import static org.hamcrest.Matchers.equalTo;
 public abstract class ESIndexLevelReplicationTestCase extends ESTestCase {
 
     private ThreadPool threadPool;
-    final private Index index = new Index("test", "uuid");
-    final private ShardId shardId = new ShardId(index, 0);
-    final private Map<String, String> indexMapping = Collections.singletonMap("type", "{ \"type\": {} }");
-    protected final static RecoveryTargetService.RecoveryListener recoveryListener = new RecoveryTargetService.RecoveryListener() {
+    private final Index index = new Index("test", "uuid");
+    private final ShardId shardId = new ShardId(index, 0);
+    private final Map<String, String> indexMapping = Collections.singletonMap("type", "{ \"type\": {} }");
+    protected static final RecoveryTargetService.RecoveryListener recoveryListener = new RecoveryTargetService.RecoveryListener() {
         @Override
         public void onRecoveryDone(RecoveryState state) {
 
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java
index 964a09f839d..bcd614121b6 100644
--- a/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java
@@ -57,8 +57,8 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo;
 public class RecoveryTargetTests extends ESTestCase {
     abstract class Streamer<T extends Streamable> extends Thread {
         private T lastRead;
-        final private AtomicBoolean shouldStop;
-        final private T source;
+        private final AtomicBoolean shouldStop;
+        private final T source;
         final AtomicReference<Throwable> error = new AtomicReference<>();
         final Version streamVersion;
 
diff --git a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
index e558f0f2a12..96af4ef3671 100644
--- a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java
@@ -57,7 +57,7 @@ import static org.elasticsearch.test.VersionUtils.randomVersion;
 /**
  */
 public class IndicesStoreTests extends ESTestCase {
-    private final static ShardRoutingState[] NOT_STARTED_STATES;
+    private static final ShardRoutingState[] NOT_STARTED_STATES;
 
     static {
         Set<ShardRoutingState> set = new HashSet<>();
diff --git a/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java b/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java
index 2564b31488b..740a027aecb 100644
--- a/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java
+++ b/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java
@@ -47,7 +47,7 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.lessThan;
 
 public class RecoveriesCollectionTests extends ESSingleNodeTestCase {
-    final static RecoveryTargetService.RecoveryListener listener = new RecoveryTargetService.RecoveryListener() {
+    static final RecoveryTargetService.RecoveryListener listener = new RecoveryTargetService.RecoveryListener() {
         @Override
         public void onRecoveryDone(RecoveryState state) {
 
diff --git a/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java b/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java
index d8d6b0f5409..658a3bf5658 100644
--- a/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java
+++ b/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java
@@ -29,8 +29,8 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke
 
 public class StoredScriptsIT extends ESIntegTestCase {
 
-    private final static int SCRIPT_MAX_SIZE_IN_BYTES = 64;
-    private final static String LANG = MockScriptEngine.NAME;
+    private static final int SCRIPT_MAX_SIZE_IN_BYTES = 64;
+    private static final String LANG = MockScriptEngine.NAME;
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java
index b22f7524e32..f4234423531 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenIT.java
@@ -61,7 +61,7 @@ import static org.hamcrest.Matchers.sameInstance;
 @ESIntegTestCase.SuiteScopeTestCase
 public class ChildrenIT extends ESIntegTestCase {
 
-    private final static Map<String, Control> categoryToControl = new HashMap<>();
+    private static final Map<String, Control> categoryToControl = new HashMap<>();
 
     @Override
     public void setupSuiteScopeCluster() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
index 39df053c862..b8e572b9dcc 100644
--- a/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
+++ b/core/src/test/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java
@@ -105,7 +105,7 @@ public class FetchSubPhasePluginIT extends ESIntegTestCase {
         }
     }
 
-    public final static class TermVectorsFetchSubPhase implements FetchSubPhase {
+    public static final class TermVectorsFetchSubPhase implements FetchSubPhase {
 
         public static final ContextFactory<TermVectorsFetchContext> CONTEXT_FACTORY = new ContextFactory<TermVectorsFetchContext>() {
 
diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java
index 14d0fc959c3..4ba1b902fef 100644
--- a/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java
+++ b/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java
@@ -355,7 +355,7 @@ public class QueryRescorerIT extends ESIntegTestCase {
     // and shard id are equal during merging shard results.
     // This comparator uses a custom tie in case the scores are equal, so that both regular hits and rescored hits
     // are sorted equally. This is fine since tests only care about the fact the scores should be equal, not ordering.
-    private final static Comparator<SearchHit> searchHitsComparator = new Comparator<SearchHit>() {
+    private static final Comparator<SearchHit> searchHitsComparator = new Comparator<SearchHit>() {
         @Override
         public int compare(SearchHit hit1, SearchHit hit2) {
             int cmp = Float.compare(hit2.getScore(), hit1.getScore());
diff --git a/core/src/test/java/org/elasticsearch/test/MockLogAppender.java b/core/src/test/java/org/elasticsearch/test/MockLogAppender.java
index 9e4a881b25b..8f10ccd6537 100644
--- a/core/src/test/java/org/elasticsearch/test/MockLogAppender.java
+++ b/core/src/test/java/org/elasticsearch/test/MockLogAppender.java
@@ -34,7 +34,7 @@ import static org.hamcrest.MatcherAssert.assertThat;
  */
 public class MockLogAppender extends AppenderSkeleton {
 
-    private final static String COMMON_PREFIX = System.getProperty("es.logger.prefix", "org.elasticsearch.");
+    private static final String COMMON_PREFIX = System.getProperty("es.logger.prefix", "org.elasticsearch.");
 
     private List<LoggingExpectation> expectations;
 
@@ -75,7 +75,7 @@ public class MockLogAppender extends AppenderSkeleton {
         void assertMatched();
     }
 
-    public static abstract class AbstractEventExpectation implements LoggingExpectation {
+    public abstract static class AbstractEventExpectation implements LoggingExpectation {
         protected final String name;
         protected final String logger;
         protected final Level level;
diff --git a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
index 9e08ecde6fa..583d8a0288d 100644
--- a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
+++ b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java
@@ -59,7 +59,7 @@ import static org.hamcrest.Matchers.nullValue;
 @ClusterScope(scope= Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1)
 public class SimpleTTLIT extends ESIntegTestCase {
 
-    static private final long PURGE_INTERVAL = 200;
+    private static final long PURGE_INTERVAL = 200;
 
     @Override
     protected int numberOfShards() {
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java
index edef75389c8..20be7e72888 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java
@@ -36,8 +36,8 @@ import java.util.Map;
  */
 public class InternalMatrixStats extends InternalMetricsAggregation implements MatrixStats {
 
-    public final static Type TYPE = new Type("matrix_stats");
-    public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
+    public static final Type TYPE = new Type("matrix_stats");
+    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
         @Override
         public InternalMatrixStats readResult(StreamInput in) throws IOException {
             InternalMatrixStats result = new InternalMatrixStats();
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java
index 96b7b74ab4e..1ae29e65761 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java
@@ -34,9 +34,9 @@ import java.util.Map;
  */
 class MatrixStatsResults implements Writeable {
     /** object holding results - computes results in place */
-    final protected RunningStats results;
+    protected final RunningStats results;
     /** pearson product correlation coefficients */
-    final protected Map<String, HashMap<String, Double>> correlation;
+    protected final Map<String, HashMap<String, Double>> correlation;
 
     /** Base ctor */
     public MatrixStatsResults() {
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
index 51e5ce1cf27..e3aa171fe3d 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java
@@ -52,7 +52,7 @@ public abstract class MultiValuesSourceAggregationBuilder<VS extends ValuesSourc
 
     public static final ParseField MULTIVALUE_MODE_FIELD = new ParseField("mode");
 
-    public static abstract class LeafOnly<VS extends ValuesSource, AB extends MultiValuesSourceAggregationBuilder<VS, AB>>
+    public abstract static class LeafOnly<VS extends ValuesSource, AB extends MultiValuesSourceAggregationBuilder<VS, AB>>
             extends MultiValuesSourceAggregationBuilder<VS, AB> {
 
         protected LeafOnly(String name, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType) {
diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/BaseMatrixStatsTestCase.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/BaseMatrixStatsTestCase.java
index b1296bb1146..2e4fa4313bd 100644
--- a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/BaseMatrixStatsTestCase.java
+++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/BaseMatrixStatsTestCase.java
@@ -34,8 +34,8 @@ public abstract class BaseMatrixStatsTestCase extends ESTestCase {
     protected final ArrayList<Double> fieldA = new ArrayList<>(numObs);
     protected final ArrayList<Double> fieldB = new ArrayList<>(numObs);
     protected final MultiPassStats actualStats = new MultiPassStats();
-    protected final static String fieldAKey = "fieldA";
-    protected final static String fieldBKey = "fieldB";
+    protected static final String fieldAKey = "fieldA";
+    protected static final String fieldBKey = "fieldB";
 
     @Before
     public void setup() {
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
index f35f3fec534..a0fcdb1e0d1 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
@@ -53,7 +53,7 @@ abstract class AbstractStringProcessor extends AbstractProcessor {
 
     protected abstract String process(String value);
 
-    static abstract class Factory implements Processor.Factory {
+    abstract static class Factory implements Processor.Factory {
         protected final String processorType;
 
         protected Factory(String processorType) {
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
index 32edc665d58..39d17ec09bf 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
@@ -114,7 +114,7 @@ public final class GrokProcessor extends AbstractProcessor {
         return combinedPattern;
     }
 
-    public final static class Factory implements Processor.Factory {
+    public static final class Factory implements Processor.Factory {
 
         private final Map<String, String> builtinPatterns;
 
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java
index f8bc2afffdd..da0011d20c8 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java
@@ -70,7 +70,7 @@ public final class JoinProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public final static class Factory implements Processor.Factory {
+    public static final class Factory implements Processor.Factory {
         @Override
         public JoinProcessor create(String processorTag, Map<String, Object> config) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java
index e7a8f3f3e6a..a0ae8e13158 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java
@@ -44,7 +44,7 @@ public final class LowercaseProcessor extends AbstractStringProcessor {
         return TYPE;
     }
 
-    public final static class Factory extends AbstractStringProcessor.Factory {
+    public static final class Factory extends AbstractStringProcessor.Factory {
 
         public Factory() {
             super(TYPE);
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java
index 5ab4016921d..d5994d411c9 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java
@@ -111,7 +111,7 @@ public final class SortProcessor extends AbstractProcessor {
         return TYPE;
     }
 
-    public final static class Factory implements Processor.Factory {
+    public static final class Factory implements Processor.Factory {
 
         @Override
         public SortProcessor create(String processorTag, Map<String, Object> config) throws Exception {
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java
index ceb187bcc63..8419730dc1c 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java
@@ -92,7 +92,7 @@ public class CustomMustacheFactory extends DefaultMustacheFactory {
     /**
      * Base class for custom Mustache functions
      */
-    static abstract class CustomCode extends IterableCode {
+    abstract static class CustomCode extends IterableCode {
 
         private final String code;
 
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java
index 45d3d8c182d..dd3055ba8e8 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java
@@ -49,7 +49,7 @@ final class CustomReflectionObjectHandler extends ReflectionObjectHandler {
         }
     }
 
-    final static class ArrayMap extends AbstractMap<Object, Object> implements Iterable<Object> {
+    static final class ArrayMap extends AbstractMap<Object, Object> implements Iterable<Object> {
 
         private final Object array;
         private final int length;
@@ -109,7 +109,7 @@ final class CustomReflectionObjectHandler extends ReflectionObjectHandler {
 
     }
 
-    final static class CollectionMap extends AbstractMap<Object, Object> implements Iterable<Object> {
+    static final class CollectionMap extends AbstractMap<Object, Object> implements Iterable<Object> {
 
         private final Collection<Object> col;
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
index ca549522018..c546207b1ee 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -43,37 +43,37 @@ import java.util.regex.Pattern;
  */
 public final class WriterConstants {
 
-    public final static int CLASS_VERSION = Opcodes.V1_8;
-    public final static int ASM_VERSION = Opcodes.ASM5;
-    public final static String BASE_CLASS_NAME = Executable.class.getName();
-    public final static Type BASE_CLASS_TYPE   = Type.getType(Executable.class);
+    public static final int CLASS_VERSION = Opcodes.V1_8;
+    public static final int ASM_VERSION = Opcodes.ASM5;
+    public static final String BASE_CLASS_NAME = Executable.class.getName();
+    public static final Type BASE_CLASS_TYPE   = Type.getType(Executable.class);
 
-    public final static String CLASS_NAME      = BASE_CLASS_NAME + "$Script";
-    public final static Type CLASS_TYPE        = Type.getObjectType(CLASS_NAME.replace('.', '/'));
+    public static final String CLASS_NAME      = BASE_CLASS_NAME + "$Script";
+    public static final Type CLASS_TYPE        = Type.getObjectType(CLASS_NAME.replace('.', '/'));
 
-    public final static Method CONSTRUCTOR = getAsmMethod(void.class, "<init>", String.class, String.class, BitSet.class);
-    public final static Method CLINIT      = getAsmMethod(void.class, "<clinit>");
-    public final static Method EXECUTE     =
+    public static final Method CONSTRUCTOR = getAsmMethod(void.class, "<init>", String.class, String.class, BitSet.class);
+    public static final Method CLINIT      = getAsmMethod(void.class, "<clinit>");
+    public static final Method EXECUTE     =
         getAsmMethod(Object.class, "execute", Map.class, Scorer.class, LeafDocLookup.class, Object.class);
 
-    public final static Type PAINLESS_ERROR_TYPE = Type.getType(PainlessError.class);
+    public static final Type PAINLESS_ERROR_TYPE = Type.getType(PainlessError.class);
 
-    public final static Type NEEDS_SCORE_TYPE = Type.getType(NeedsScore.class);
-    public final static Type SCORER_TYPE = Type.getType(Scorer.class);
-    public final static Method SCORER_SCORE = getAsmMethod(float.class, "score");
+    public static final Type NEEDS_SCORE_TYPE = Type.getType(NeedsScore.class);
+    public static final Type SCORER_TYPE = Type.getType(Scorer.class);
+    public static final Method SCORER_SCORE = getAsmMethod(float.class, "score");
 
-    public final static Type MAP_TYPE  = Type.getType(Map.class);
-    public final static Method MAP_GET = getAsmMethod(Object.class, "get", Object.class);
+    public static final Type MAP_TYPE  = Type.getType(Map.class);
+    public static final Method MAP_GET = getAsmMethod(Object.class, "get", Object.class);
 
-    public final static Type ITERATOR_TYPE = Type.getType(Iterator.class);
-    public final static Method ITERATOR_HASNEXT = getAsmMethod(boolean.class, "hasNext");
-    public final static Method ITERATOR_NEXT = getAsmMethod(Object.class, "next");
+    public static final Type ITERATOR_TYPE = Type.getType(Iterator.class);
+    public static final Method ITERATOR_HASNEXT = getAsmMethod(boolean.class, "hasNext");
+    public static final Method ITERATOR_NEXT = getAsmMethod(Object.class, "next");
 
-    public final static Type UTILITY_TYPE = Type.getType(Utility.class);
-    public final static Method STRING_TO_CHAR = getAsmMethod(char.class, "StringTochar", String.class);
-    public final static Method CHAR_TO_STRING = getAsmMethod(String.class, "charToString", char.class);
+    public static final Type UTILITY_TYPE = Type.getType(Utility.class);
+    public static final Method STRING_TO_CHAR = getAsmMethod(char.class, "StringTochar", String.class);
+    public static final Method CHAR_TO_STRING = getAsmMethod(String.class, "charToString", char.class);
     
-    public final static Type METHOD_HANDLE_TYPE = Type.getType(MethodHandle.class);
+    public static final Type METHOD_HANDLE_TYPE = Type.getType(MethodHandle.class);
     
     public static final Type AUGMENTATION_TYPE = Type.getType(Augmentation.class);
 
@@ -83,46 +83,46 @@ public final class WriterConstants {
      * because it can do it statically. This is both faster and prevents the script from doing something super slow like building a regex
      * per time it is run.
      */
-    public final static Method PATTERN_COMPILE = getAsmMethod(Pattern.class, "compile", String.class, int.class);
-    public final static Method PATTERN_MATCHER = getAsmMethod(Matcher.class, "matcher", CharSequence.class);
-    public final static Method MATCHER_MATCHES = getAsmMethod(boolean.class, "matches");
-    public final static Method MATCHER_FIND = getAsmMethod(boolean.class, "find");
+    public static final Method PATTERN_COMPILE = getAsmMethod(Pattern.class, "compile", String.class, int.class);
+    public static final Method PATTERN_MATCHER = getAsmMethod(Matcher.class, "matcher", CharSequence.class);
+    public static final Method MATCHER_MATCHES = getAsmMethod(boolean.class, "matches");
+    public static final Method MATCHER_FIND = getAsmMethod(boolean.class, "find");
 
     /** dynamic callsite bootstrap signature */
-    final static MethodType DEF_BOOTSTRAP_TYPE =
+    static final MethodType DEF_BOOTSTRAP_TYPE =
         MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, 
                               int.class, int.class, Object[].class);
-    final static Handle DEF_BOOTSTRAP_HANDLE =
+    static final Handle DEF_BOOTSTRAP_HANDLE =
         new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(DefBootstrap.class),
             "bootstrap", DEF_BOOTSTRAP_TYPE.toMethodDescriptorString(), false);
 
-    public final static Type DEF_UTIL_TYPE = Type.getType(Def.class);
-    public final static Method DEF_TO_BOOLEAN         = getAsmMethod(boolean.class, "DefToboolean"       , Object.class);
-    public final static Method DEF_TO_BYTE_IMPLICIT   = getAsmMethod(byte.class   , "DefTobyteImplicit"  , Object.class);
-    public final static Method DEF_TO_SHORT_IMPLICIT  = getAsmMethod(short.class  , "DefToshortImplicit" , Object.class);
-    public final static Method DEF_TO_CHAR_IMPLICIT   = getAsmMethod(char.class   , "DefTocharImplicit"  , Object.class);
-    public final static Method DEF_TO_INT_IMPLICIT    = getAsmMethod(int.class    , "DefTointImplicit"   , Object.class);
-    public final static Method DEF_TO_LONG_IMPLICIT   = getAsmMethod(long.class   , "DefTolongImplicit"  , Object.class);
-    public final static Method DEF_TO_FLOAT_IMPLICIT  = getAsmMethod(float.class  , "DefTofloatImplicit" , Object.class);
-    public final static Method DEF_TO_DOUBLE_IMPLICIT = getAsmMethod(double.class , "DefTodoubleImplicit", Object.class);
-    public final static Method DEF_TO_BYTE_EXPLICIT   = getAsmMethod(byte.class   , "DefTobyteExplicit"  , Object.class);
-    public final static Method DEF_TO_SHORT_EXPLICIT  = getAsmMethod(short.class  , "DefToshortExplicit" , Object.class);
-    public final static Method DEF_TO_CHAR_EXPLICIT   = getAsmMethod(char.class   , "DefTocharExplicit"  , Object.class);
-    public final static Method DEF_TO_INT_EXPLICIT    = getAsmMethod(int.class    , "DefTointExplicit"   , Object.class);
-    public final static Method DEF_TO_LONG_EXPLICIT   = getAsmMethod(long.class   , "DefTolongExplicit"  , Object.class);
-    public final static Method DEF_TO_FLOAT_EXPLICIT  = getAsmMethod(float.class  , "DefTofloatExplicit" , Object.class);
-    public final static Method DEF_TO_DOUBLE_EXPLICIT = getAsmMethod(double.class , "DefTodoubleExplicit", Object.class);
+    public static final Type DEF_UTIL_TYPE = Type.getType(Def.class);
+    public static final Method DEF_TO_BOOLEAN         = getAsmMethod(boolean.class, "DefToboolean"       , Object.class);
+    public static final Method DEF_TO_BYTE_IMPLICIT   = getAsmMethod(byte.class   , "DefTobyteImplicit"  , Object.class);
+    public static final Method DEF_TO_SHORT_IMPLICIT  = getAsmMethod(short.class  , "DefToshortImplicit" , Object.class);
+    public static final Method DEF_TO_CHAR_IMPLICIT   = getAsmMethod(char.class   , "DefTocharImplicit"  , Object.class);
+    public static final Method DEF_TO_INT_IMPLICIT    = getAsmMethod(int.class    , "DefTointImplicit"   , Object.class);
+    public static final Method DEF_TO_LONG_IMPLICIT   = getAsmMethod(long.class   , "DefTolongImplicit"  , Object.class);
+    public static final Method DEF_TO_FLOAT_IMPLICIT  = getAsmMethod(float.class  , "DefTofloatImplicit" , Object.class);
+    public static final Method DEF_TO_DOUBLE_IMPLICIT = getAsmMethod(double.class , "DefTodoubleImplicit", Object.class);
+    public static final Method DEF_TO_BYTE_EXPLICIT   = getAsmMethod(byte.class   , "DefTobyteExplicit"  , Object.class);
+    public static final Method DEF_TO_SHORT_EXPLICIT  = getAsmMethod(short.class  , "DefToshortExplicit" , Object.class);
+    public static final Method DEF_TO_CHAR_EXPLICIT   = getAsmMethod(char.class   , "DefTocharExplicit"  , Object.class);
+    public static final Method DEF_TO_INT_EXPLICIT    = getAsmMethod(int.class    , "DefTointExplicit"   , Object.class);
+    public static final Method DEF_TO_LONG_EXPLICIT   = getAsmMethod(long.class   , "DefTolongExplicit"  , Object.class);
+    public static final Method DEF_TO_FLOAT_EXPLICIT  = getAsmMethod(float.class  , "DefTofloatExplicit" , Object.class);
+    public static final Method DEF_TO_DOUBLE_EXPLICIT = getAsmMethod(double.class , "DefTodoubleExplicit", Object.class);
 
     /** invokedynamic bootstrap for lambda expression/method references */
-    public final static MethodType LAMBDA_BOOTSTRAP_TYPE =
+    public static final MethodType LAMBDA_BOOTSTRAP_TYPE =
             MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class,
                                   MethodType.class, Object[].class);
-    public final static Handle LAMBDA_BOOTSTRAP_HANDLE =
+    public static final Handle LAMBDA_BOOTSTRAP_HANDLE =
             new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(LambdaMetafactory.class),
                 "altMetafactory", LAMBDA_BOOTSTRAP_TYPE.toMethodDescriptorString(), false);
 
     /** dynamic invokedynamic bootstrap for indy string concats (Java 9+) */
-    public final static Handle INDY_STRING_CONCAT_BOOTSTRAP_HANDLE;
+    public static final Handle INDY_STRING_CONCAT_BOOTSTRAP_HANDLE;
     static {
         Handle bs;
         try {
@@ -139,24 +139,24 @@ public final class WriterConstants {
         INDY_STRING_CONCAT_BOOTSTRAP_HANDLE = bs;
     }
 
-    public final static int MAX_INDY_STRING_CONCAT_ARGS = 200;
+    public static final int MAX_INDY_STRING_CONCAT_ARGS = 200;
 
-    public final static Type STRING_TYPE = Type.getType(String.class);
-    public final static Type STRINGBUILDER_TYPE = Type.getType(StringBuilder.class);
+    public static final Type STRING_TYPE = Type.getType(String.class);
+    public static final Type STRINGBUILDER_TYPE = Type.getType(StringBuilder.class);
 
-    public final static Method STRINGBUILDER_CONSTRUCTOR    = getAsmMethod(void.class, "<init>");
-    public final static Method STRINGBUILDER_APPEND_BOOLEAN = getAsmMethod(StringBuilder.class, "append", boolean.class);
-    public final static Method STRINGBUILDER_APPEND_CHAR    = getAsmMethod(StringBuilder.class, "append", char.class);
-    public final static Method STRINGBUILDER_APPEND_INT     = getAsmMethod(StringBuilder.class, "append", int.class);
-    public final static Method STRINGBUILDER_APPEND_LONG    = getAsmMethod(StringBuilder.class, "append", long.class);
-    public final static Method STRINGBUILDER_APPEND_FLOAT   = getAsmMethod(StringBuilder.class, "append", float.class);
-    public final static Method STRINGBUILDER_APPEND_DOUBLE  = getAsmMethod(StringBuilder.class, "append", double.class);
-    public final static Method STRINGBUILDER_APPEND_STRING  = getAsmMethod(StringBuilder.class, "append", String.class);
-    public final static Method STRINGBUILDER_APPEND_OBJECT  = getAsmMethod(StringBuilder.class, "append", Object.class);
-    public final static Method STRINGBUILDER_TOSTRING       = getAsmMethod(String.class, "toString");
+    public static final Method STRINGBUILDER_CONSTRUCTOR    = getAsmMethod(void.class, "<init>");
+    public static final Method STRINGBUILDER_APPEND_BOOLEAN = getAsmMethod(StringBuilder.class, "append", boolean.class);
+    public static final Method STRINGBUILDER_APPEND_CHAR    = getAsmMethod(StringBuilder.class, "append", char.class);
+    public static final Method STRINGBUILDER_APPEND_INT     = getAsmMethod(StringBuilder.class, "append", int.class);
+    public static final Method STRINGBUILDER_APPEND_LONG    = getAsmMethod(StringBuilder.class, "append", long.class);
+    public static final Method STRINGBUILDER_APPEND_FLOAT   = getAsmMethod(StringBuilder.class, "append", float.class);
+    public static final Method STRINGBUILDER_APPEND_DOUBLE  = getAsmMethod(StringBuilder.class, "append", double.class);
+    public static final Method STRINGBUILDER_APPEND_STRING  = getAsmMethod(StringBuilder.class, "append", String.class);
+    public static final Method STRINGBUILDER_APPEND_OBJECT  = getAsmMethod(StringBuilder.class, "append", Object.class);
+    public static final Method STRINGBUILDER_TOSTRING       = getAsmMethod(String.class, "toString");
 
-    public final static Type OBJECTS_TYPE = Type.getType(Objects.class);
-    public final static Method EQUALS = getAsmMethod(boolean.class, "equals", Object.class, Object.class);
+    public static final Type OBJECTS_TYPE = Type.getType(Objects.class);
+    public static final Method EQUALS = getAsmMethod(boolean.class, "equals", Object.class, Object.class);
 
     private static Method getAsmMethod(final Class<?> rtype, final String name, final Class<?>... ptypes) {
         return new Method(name, MethodType.methodType(rtype, ptypes).toMethodDescriptorString());
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
index 6cd5a568f16..76bc136656c 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
@@ -315,7 +315,7 @@ public final class PercolateQuery extends Query implements Accountable {
 
     }
 
-    static abstract class BaseScorer extends Scorer {
+    abstract static class BaseScorer extends Scorer {
 
         final Scorer approximation;
         final QueryStore.Leaf percolatorQueries;
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
index 3ce3726aa5e..a5db24a71ef 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
@@ -541,7 +541,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
         };
     }
 
-    private final static class LegacyQueryFieldVisitor extends StoredFieldVisitor {
+    private static final class LegacyQueryFieldVisitor extends StoredFieldVisitor {
 
         private BytesArray source;
 
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateResponse.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateResponse.java
index 622c27fca92..3d82cb12d2c 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateResponse.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateResponse.java
@@ -49,7 +49,7 @@ public class PercolateResponse extends BroadcastResponse implements Iterable<Per
 
     public static final Match[] EMPTY = new Match[0];
     // PercolateQuery emits this score if no 'query' is defined in the percolate request
-    public final static float NO_SCORE = 0.0f;
+    public static final float NO_SCORE = 0.0f;
 
     private long tookInMillis;
     private Match[] matches;
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
index b2f14ab496a..1428b8116a8 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
@@ -52,8 +52,8 @@ import java.util.Map;
 
 public class PercolatorFieldMapper extends FieldMapper {
 
-    public final static XContentType QUERY_BUILDER_CONTENT_TYPE = XContentType.SMILE;
-    public final static Setting<Boolean> INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING =
+    public static final XContentType QUERY_BUILDER_CONTENT_TYPE = XContentType.SMILE;
+    public static final Setting<Boolean> INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING =
             Setting.boolSetting("index.percolator.map_unmapped_fields_as_string", false, Setting.Property.IndexScope);
     public static final String CONTENT_TYPE = "percolator";
     private static final PercolatorFieldType FIELD_TYPE = new PercolatorFieldType();
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java
index b9486a89f49..a050c8eb420 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java
@@ -71,9 +71,9 @@ import static org.hamcrest.Matchers.sameInstance;
 
 public class ExtractQueryTermsServiceTests extends ESTestCase {
 
-    public final static String QUERY_TERMS_FIELD = "extracted_terms";
-    public final static String EXTRACTION_RESULT_FIELD = "extraction_result";
-    public final static FieldType QUERY_TERMS_FIELD_TYPE = new FieldType();
+    public static final String QUERY_TERMS_FIELD = "extracted_terms";
+    public static final String EXTRACTION_RESULT_FIELD = "extraction_result";
+    public static final FieldType QUERY_TERMS_FIELD_TYPE = new FieldType();
 
     static {
         QUERY_TERMS_FIELD_TYPE.setTokenized(false);
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java
index 15d531467d4..0a359376f7b 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java
@@ -53,8 +53,8 @@ import static org.hamcrest.Matchers.nullValue;
 
 public class MultiPercolatorIT extends ESIntegTestCase {
 
-    private final static String INDEX_NAME = "queries";
-    private final static String TYPE_NAME = "query";
+    private static final String INDEX_NAME = "queries";
+    private static final String TYPE_NAME = "query";
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java
index 79904f422bb..c2c2a641a71 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java
@@ -82,9 +82,9 @@ import static org.hamcrest.Matchers.is;
 
 public class PercolateQueryTests extends ESTestCase {
 
-    public final static String EXTRACTED_TERMS_FIELD_NAME = "extracted_terms";
-    public final static String UNKNOWN_QUERY_FIELD_NAME = "unknown_query";
-    public final static FieldType EXTRACTED_TERMS_FIELD_TYPE = new FieldType();
+    public static final String EXTRACTED_TERMS_FIELD_NAME = "extracted_terms";
+    public static final String UNKNOWN_QUERY_FIELD_NAME = "unknown_query";
+    public static final FieldType EXTRACTED_TERMS_FIELD_TYPE = new FieldType();
 
     static {
         EXTRACTED_TERMS_FIELD_TYPE.setTokenized(false);
@@ -433,7 +433,7 @@ public class PercolateQueryTests extends ESTestCase {
         }
     }
 
-    private final static class CustomQuery extends Query {
+    private static final class CustomQuery extends Query {
 
         private final Term term;
 
@@ -462,7 +462,7 @@ public class PercolateQueryTests extends ESTestCase {
         }
     }
 
-    private final static class ControlQuery extends Query {
+    private static final class ControlQuery extends Query {
 
         private final MemoryIndex memoryIndex;
         private final PercolateQuery.QueryStore queryStore;
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java
index 75d4f408774..f4a436e7c40 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java
@@ -50,8 +50,8 @@ import static org.hamcrest.Matchers.notNullValue;
 
 public class PercolatorAggregationsIT extends ESIntegTestCase {
 
-    private final static String INDEX_NAME = "queries";
-    private final static String TYPE_NAME = "query";
+    private static final String INDEX_NAME = "queries";
+    private static final String TYPE_NAME = "query";
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java
index 701fd1b660f..7a51d8a7ab2 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java
@@ -48,7 +48,7 @@ import static org.hamcrest.Matchers.notNullValue;
 // Can'r run as IT as the test cluster is immutable and this test adds nodes during the test
 public class PercolatorBackwardsCompatibilityTests extends ESIntegTestCase {
 
-    private final static String INDEX_NAME = "percolator_index";
+    private static final String INDEX_NAME = "percolator_index";
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
index ff9bf6634cd..e4e379c8f60 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
@@ -91,8 +91,8 @@ import static org.hamcrest.Matchers.nullValue;
 
 public class PercolatorIT extends ESIntegTestCase {
 
-    private final static String INDEX_NAME = "queries";
-    private final static String TYPE_NAME = "query";
+    private static final String INDEX_NAME = "queries";
+    private static final String TYPE_NAME = "query";
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
diff --git a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/phonetic/HaasePhonetik.java b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/phonetic/HaasePhonetik.java
index 728a9354d97..e8b49a50edb 100644
--- a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/phonetic/HaasePhonetik.java
+++ b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/phonetic/HaasePhonetik.java
@@ -37,9 +37,9 @@ package org.elasticsearch.index.analysis.phonetic;
  */
 public class HaasePhonetik extends KoelnerPhonetik {
 
-    private final static String[] HAASE_VARIATIONS_PATTERNS = {"OWN", "RB", "WSK", "A$", "O$", "SCH",
+    private static final String[] HAASE_VARIATIONS_PATTERNS = {"OWN", "RB", "WSK", "A$", "O$", "SCH",
         "GLI", "EAU$", "^CH", "AUX", "EUX", "ILLE"};
-    private final static String[] HAASE_VARIATIONS_REPLACEMENTS = {"AUN", "RW", "RSK", "AR", "OW", "CH",
+    private static final String[] HAASE_VARIATIONS_REPLACEMENTS = {"AUN", "RW", "RSK", "AR", "OW", "CH",
         "LI", "O", "SCH", "O", "O", "I"};
 
     @Override
diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapperTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapperTests.java
index aa6e91fabdf..a49124749fe 100644
--- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapperTests.java
+++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapperTests.java
@@ -44,7 +44,7 @@ import static org.hamcrest.Matchers.lessThan;
 
 public class RetryHttpInitializerWrapperTests extends ESTestCase {
 
-    static private class FailThenSuccessBackoffTransport extends MockHttpTransport {
+    private static class FailThenSuccessBackoffTransport extends MockHttpTransport {
 
         public int lowLevelExecCalls;
         int errorStatusCode;
diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
index 92c4785dbaf..64f0994b7d9 100644
--- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
+++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
@@ -272,7 +272,7 @@ public final class GeoIpProcessor extends AbstractProcessor {
     // Geoip2's AddressNotFoundException is checked and due to the fact that we need run their code
     // inside a PrivilegedAction code block, we are forced to catch any checked exception and rethrow
     // it with an unchecked exception.
-    private final static class AddressNotFoundRuntimeException extends RuntimeException {
+    private static final class AddressNotFoundRuntimeException extends RuntimeException {
 
         public AddressNotFoundRuntimeException(Throwable cause) {
             super(cause);
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
index 4d3459cdcd4..1333c755e7c 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
@@ -65,7 +65,7 @@ public class AzureRepository extends BlobStoreRepository {
 
     private static final ByteSizeValue MAX_CHUNK_SIZE = new ByteSizeValue(64, ByteSizeUnit.MB);
 
-    public final static String TYPE = "azure";
+    public static final String TYPE = "azure";
 
     public static final class Repository {
         public static final Setting<String> ACCOUNT_SETTING = Setting.simpleString("account", Property.NodeScope);
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java
index 5959f964644..ad2b14e368b 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java
@@ -32,7 +32,7 @@ import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 
 public class AzureStorageServiceTests extends ESTestCase {
-    final static Settings settings = Settings.builder()
+    static final Settings settings = Settings.builder()
             .put("cloud.azure.storage.azure1.account", "myaccount1")
             .put("cloud.azure.storage.azure1.key", "mykey1")
             .put("cloud.azure.storage.azure1.default", true)
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java
index db3bd617da4..4b74d2aa0d1 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java
@@ -35,7 +35,7 @@ import java.io.IOException;
 import static org.hamcrest.Matchers.contains;
 
 public class AzureStorageSettingsFilterTests extends ESTestCase {
-    final static Settings settings = Settings.builder()
+    static final Settings settings = Settings.builder()
             .put("cloud.azure.storage.azure1.account", "myaccount1")
             .put("cloud.azure.storage.azure1.key", "mykey1")
             .put("cloud.azure.storage.azure1.default", true)
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
index 6ba726e2b24..a22178315f7 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java
@@ -131,7 +131,7 @@ final class HdfsBlobContainer extends AbstractBlobContainer {
     }
 
     @Override
-    public Map<String, BlobMetaData> listBlobsByPrefix(final @Nullable String prefix) throws IOException {
+    public Map<String, BlobMetaData> listBlobsByPrefix(@Nullable final String prefix) throws IOException {
         FileStatus[] files = store.execute(new Operation<FileStatus[]>() {
             @Override
             public FileStatus[] run(FileContext fileContext) throws IOException {
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
index fe184a06aef..e3b7e8296e0 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
@@ -57,7 +57,7 @@ import java.util.function.Function;
  */
 public class S3Repository extends BlobStoreRepository {
 
-    public final static String TYPE = "s3";
+    public static final String TYPE = "s3";
 
     /**
      * Global S3 repositories settings. Starting with: repositories.s3
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
index 2e196610c1a..93c19df9c04 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
@@ -52,7 +52,7 @@ import static org.hamcrest.Matchers.notNullValue;
 /**
  */
 @ClusterScope(scope = Scope.SUITE, numDataNodes = 2, numClientNodes = 0, transportClientRatio = 0.0)
-abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase {
+public abstract  class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase {
 
     @Override
     public Settings nodeSettings(int nodeOrdinal) {
diff --git a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/AbstractAzureFsTestCase.java b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/AbstractAzureFsTestCase.java
index 9e29d6f091a..6016b0ca335 100644
--- a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/AbstractAzureFsTestCase.java
+++ b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/AbstractAzureFsTestCase.java
@@ -28,7 +28,7 @@ import java.util.Collection;
 
 import static org.hamcrest.Matchers.is;
 
-abstract public class AbstractAzureFsTestCase extends ESIntegTestCase {
+public abstract  class AbstractAzureFsTestCase extends ESIntegTestCase {
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return pluginList(SMBStorePlugin.class);
diff --git a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
index 613445c2271..22a1e2660b6 100644
--- a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
+++ b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java
@@ -246,7 +246,7 @@ public class MockBigArrays extends BigArrays {
         return arr;
     }
 
-    private static abstract class AbstractArrayWrapper {
+    private abstract static class AbstractArrayWrapper {
 
         final BigArray in;
         boolean clearOnResize;
diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
index d5b4cdfa554..0d9ed196838 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java
@@ -326,11 +326,11 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
     /**
      * Parses the query provided as string argument and compares it with the expected result provided as argument as a {@link QueryBuilder}
      */
-    protected final static void assertParsedQuery(String queryAsString, QueryBuilder expectedQuery) throws IOException {
+    protected static final void assertParsedQuery(String queryAsString, QueryBuilder expectedQuery) throws IOException {
         assertParsedQuery(queryAsString, expectedQuery, ParseFieldMatcher.STRICT);
     }
 
-    protected final static void assertParsedQuery(String queryAsString, QueryBuilder expectedQuery, ParseFieldMatcher matcher) throws IOException {
+    protected static final void assertParsedQuery(String queryAsString, QueryBuilder expectedQuery, ParseFieldMatcher matcher) throws IOException {
         QueryBuilder newQuery = parseQuery(queryAsString, matcher);
         assertNotSame(newQuery, expectedQuery);
         assertEquals(expectedQuery, newQuery);
@@ -340,31 +340,31 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
     /**
      * Parses the query provided as bytes argument and compares it with the expected result provided as argument as a {@link QueryBuilder}
      */
-    protected final static void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery) throws IOException {
+    protected static final void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery) throws IOException {
         assertParsedQuery(queryAsBytes, expectedQuery, ParseFieldMatcher.STRICT);
     }
 
-    protected final static void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery, ParseFieldMatcher matcher) throws IOException {
+    protected static final void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery, ParseFieldMatcher matcher) throws IOException {
         QueryBuilder newQuery = parseQuery(queryAsBytes, matcher);
         assertNotSame(newQuery, expectedQuery);
         assertEquals(expectedQuery, newQuery);
         assertEquals(expectedQuery.hashCode(), newQuery.hashCode());
     }
 
-    protected final static QueryBuilder parseQuery(String queryAsString) throws IOException {
+    protected static final QueryBuilder parseQuery(String queryAsString) throws IOException {
         return parseQuery(queryAsString, ParseFieldMatcher.STRICT);
     }
 
-    protected final static QueryBuilder parseQuery(String queryAsString, ParseFieldMatcher matcher) throws IOException {
+    protected static final QueryBuilder parseQuery(String queryAsString, ParseFieldMatcher matcher) throws IOException {
         XContentParser parser = XContentFactory.xContent(queryAsString).createParser(queryAsString);
         return parseQuery(parser, matcher);
     }
 
-    protected final static QueryBuilder parseQuery(BytesReference queryAsBytes) throws IOException {
+    protected static final QueryBuilder parseQuery(BytesReference queryAsBytes) throws IOException {
         return parseQuery(queryAsBytes, ParseFieldMatcher.STRICT);
     }
 
-    protected final static QueryBuilder parseQuery(BytesReference queryAsBytes, ParseFieldMatcher matcher) throws IOException {
+    protected static final QueryBuilder parseQuery(BytesReference queryAsBytes, ParseFieldMatcher matcher) throws IOException {
         XContentParser parser = XContentFactory.xContent(queryAsBytes).createParser(queryAsBytes);
         return parseQuery(parser, matcher);
     }
@@ -690,7 +690,7 @@ public abstract class AbstractQueryTestCase<QB extends AbstractQueryBuilder<QB>>
     }
 
     public static class GeohashGenerator extends CodepointSetGenerator {
-        private final static char[] ASCII_SET = "0123456789bcdefghjkmnpqrstuvwxyz".toCharArray();
+        private static final char[] ASCII_SET = "0123456789bcdefghjkmnpqrstuvwxyz".toCharArray();
 
         public GeohashGenerator() {
             super(ASCII_SET);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index 041b0f9929b..d13f69a3765 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -924,7 +924,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
      *                This saves on unneeded searches.
      * @return the actual number of docs seen.
      */
-    public long waitForDocs(final long numDocs, final @Nullable BackgroundIndexer indexer) throws InterruptedException {
+    public long waitForDocs(final long numDocs, @Nullable final BackgroundIndexer indexer) throws InterruptedException {
         // indexing threads can wait for up to ~1m before retrying when they first try to index into a shard which is not STARTED.
         return waitForDocs(numDocs, 90, TimeUnit.SECONDS, indexer);
     }
@@ -939,7 +939,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
      *                        This saves on unneeded searches.
      * @return the actual number of docs seen.
      */
-    public long waitForDocs(final long numDocs, int maxWaitTime, TimeUnit maxWaitTimeUnit, final @Nullable BackgroundIndexer indexer)
+    public long waitForDocs(final long numDocs, int maxWaitTime, TimeUnit maxWaitTimeUnit, @Nullable final BackgroundIndexer indexer)
             throws InterruptedException {
         final AtomicLong lastKnownCount = new AtomicLong(-1);
         long lastStartCount = -1;
@@ -2034,7 +2034,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
      * The returned client gets automatically closed when needed, it shouldn't be closed as part of tests otherwise
      * it cannot be reused by other tests anymore.
      */
-    protected synchronized static RestClient getRestClient() {
+    protected static synchronized RestClient getRestClient() {
         if (restClient == null) {
             restClient = createRestClient(null);
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index b6b75b1ec64..11bad415764 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -1337,7 +1337,7 @@ public final class InternalTestCluster extends TestCluster {
     /**
      * Restarts a node and calls the callback during restart.
      */
-    synchronized public void restartNode(String nodeName, RestartCallback callback) throws Exception {
+    public synchronized void restartNode(String nodeName, RestartCallback callback) throws Exception {
         ensureOpen();
         NodeAndClient nodeAndClient = nodes.get(nodeName);
         if (nodeAndClient != null) {
@@ -1346,7 +1346,7 @@ public final class InternalTestCluster extends TestCluster {
         }
     }
 
-    synchronized private void restartAllNodes(boolean rollingRestart, RestartCallback callback) throws Exception {
+    private synchronized void restartAllNodes(boolean rollingRestart, RestartCallback callback) throws Exception {
         ensureOpen();
         List<NodeAndClient> toRemove = new ArrayList<>();
         try {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java b/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java
index 7c832118ca0..48b726dc3cc 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java
@@ -126,7 +126,7 @@ public class ClusterDiscoveryConfiguration extends NodeConfigurationSource {
         }
 
         @SuppressForbidden(reason = "we know we pass a IP address")
-        protected synchronized static int[] unicastHostPorts(int numHosts) {
+        protected static synchronized int[] unicastHostPorts(int numHosts) {
             int[] unicastHostPorts = new int[numHosts];
 
             final int basePort = calcBasePort();
diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java
index d957220c6df..caab35e4b42 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java
@@ -61,7 +61,7 @@ public class IntermittentLongGCDisruption extends LongGCDisruption {
         this.delayDurationMax = delayDurationMax;
     }
 
-    final static AtomicInteger thread_ids = new AtomicInteger();
+    static final AtomicInteger thread_ids = new AtomicInteger();
 
     @Override
     public void startDisrupting() {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/LongGCDisruption.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/LongGCDisruption.java
index 591540e72b9..b210a20cf70 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/disruption/LongGCDisruption.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/LongGCDisruption.java
@@ -33,7 +33,7 @@ import java.util.regex.Pattern;
  */
 public class LongGCDisruption extends SingleNodeDisruption {
 
-    private final static Pattern[] unsafeClasses = new Pattern[]{
+    private static final Pattern[] unsafeClasses = new Pattern[]{
             // logging has shared JVM locks - we may suspend a thread and block other nodes from doing their thing
             Pattern.compile("Logger")
     };
diff --git a/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java b/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java
index bf32b6b8575..304e3047496 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java
@@ -172,7 +172,7 @@ public final class MockEngineSupport {
         return reader;
     }
 
-    public static abstract class DirectoryReaderWrapper extends FilterDirectoryReader {
+    public abstract static class DirectoryReaderWrapper extends FilterDirectoryReader {
         protected final SubReaderWrapper subReaderWrapper;
 
         public DirectoryReaderWrapper(DirectoryReader in, SubReaderWrapper subReaderWrapper) throws IOException {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java
index 654a1c971ca..94c30f496dd 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java
@@ -51,11 +51,11 @@ public class CapturingTransport implements Transport {
 
     private TransportServiceAdapter adapter;
 
-    static public class CapturedRequest {
-        final public DiscoveryNode node;
-        final public long requestId;
-        final public String action;
-        final public TransportRequest request;
+    public static class CapturedRequest {
+        public final DiscoveryNode node;
+        public final long requestId;
+        public final String action;
+        public final TransportRequest request;
 
         public CapturedRequest(DiscoveryNode node, long requestId, String action, TransportRequest request) {
             this.node = node;
diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java
index 63c5eb01fb0..7253c4d26e7 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java
@@ -90,7 +90,7 @@ public class InternalTestClusterTests extends ESTestCase {
      * a set of settings that are expected to have different values betweem clusters, even they have been initialized with the same
      * base settins.
      */
-    final static Set<String> clusterUniqueSettings = new HashSet<>();
+    static final Set<String> clusterUniqueSettings = new HashSet<>();
 
     static {
         clusterUniqueSettings.add(ClusterName.CLUSTER_NAME_SETTING.getKey());

From 0a293fad29311da1a7472e2faf6790fc34f26ae2 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Fri, 1 Jul 2016 13:05:41 +0200
Subject: [PATCH 1188/1311] Remove some unused code

---
 .../lucene/store/StoreRateLimiting.java       |  2 +-
 .../action/bulk/BulkShardRequest.java         |  2 +-
 .../action/bulk/TransportBulkAction.java      |  2 +-
 .../explain/TransportExplainAction.java       |  2 +-
 .../action/support/WriteResponse.java         |  2 +-
 .../replication/ReplicationResponse.java      |  1 -
 .../termvectors/TermVectorsRequest.java       |  4 +-
 .../action/update/UpdateHelper.java           |  6 +--
 .../cluster/ClusterInfoService.java           |  8 ++--
 .../cluster/metadata/IndexTemplateFilter.java |  2 +-
 .../cluster/metadata/MappingMetaData.java     |  7 ---
 .../MetaDataIndexTemplateService.java         | 10 ++---
 .../cluster/routing/OperationRouting.java     |  7 ---
 .../decider/DiskThresholdDecider.java         |  1 -
 .../elasticsearch/common/UUIDGenerator.java   |  2 +-
 .../elasticsearch/common/geo/GeoDistance.java |  8 ++--
 .../common/geo/builders/PolygonBuilder.java   |  9 ++--
 .../elasticsearch/common/inject/Scope.java    |  2 +-
 .../elasticsearch/common/inject/State.java    |  2 +-
 .../inject/internal/InternalFactory.java      |  2 +-
 .../common/lucene/all/AllTokenStream.java     |  3 --
 .../common/settings/SettingsFilter.java       |  1 -
 .../settings/loader/SettingsLoader.java       |  2 +-
 .../common/transport/PortsRange.java          |  2 +-
 .../common/transport/TransportAddress.java    |  2 +-
 .../elasticsearch/common/util/BigArray.java   |  2 +-
 .../common/util/DoubleArray.java              |  8 ++--
 .../elasticsearch/common/util/FloatArray.java |  8 ++--
 .../elasticsearch/common/util/IntArray.java   |  8 ++--
 .../elasticsearch/common/util/LongArray.java  |  8 ++--
 .../common/util/concurrent/ThreadContext.java |  3 +-
 .../common/xcontent/ToXContent.java           |  8 ++--
 .../support/AbstractXContentParser.java       | 16 ++-----
 .../discovery/local/LocalDiscovery.java       |  3 --
 .../discovery/zen/ZenDiscovery.java           |  2 +-
 .../zen/membership/MembershipAction.java      | 10 ++---
 .../org/elasticsearch/gateway/Gateway.java    |  6 +--
 .../elasticsearch/gateway/GatewayService.java |  5 +--
 .../index/MergePolicyConfig.java              |  1 -
 .../fielddata/ordinals/MultiOrdinals.java     |  1 -
 .../index/get/ShardGetService.java            |  8 ++--
 .../index/mapper/DocumentParser.java          |  5 +--
 .../index/mapper/MapperService.java           |  8 ++--
 .../index/mapper/ParseContext.java            |  2 -
 .../mapper/internal/TimestampFieldMapper.java |  5 +--
 .../index/query/CommonTermsQueryBuilder.java  |  2 -
 .../index/shard/SnapshotStatus.java           |  6 ---
 .../index/termvectors/TermVectorsService.java |  2 +-
 .../index/translog/Translog.java              |  4 --
 .../indices/IndexingMemoryController.java     |  8 +---
 .../elasticsearch/monitor/jvm/HotThreads.java |  5 ---
 .../repositories/RepositoriesService.java     |  4 +-
 .../rest/action/cat/AbstractCatAction.java    |  3 +-
 .../rest/action/cat/RestAliasAction.java      |  2 +-
 .../rest/action/cat/RestAllocationAction.java |  2 +-
 .../rest/action/cat/RestCountAction.java      |  2 +-
 .../rest/action/cat/RestFielddataAction.java  |  2 +-
 .../rest/action/cat/RestHealthAction.java     |  2 +-
 .../rest/action/cat/RestIndicesAction.java    |  2 +-
 .../rest/action/cat/RestMasterAction.java     |  2 +-
 .../rest/action/cat/RestNodeAttrsAction.java  |  2 +-
 .../rest/action/cat/RestNodesAction.java      |  2 +-
 .../cat/RestPendingClusterTasksAction.java    |  2 +-
 .../rest/action/cat/RestPluginsAction.java    |  2 +-
 .../rest/action/cat/RestRecoveryAction.java   |  2 +-
 .../action/cat/RestRepositoriesAction.java    |  2 +-
 .../rest/action/cat/RestSegmentsAction.java   |  2 +-
 .../rest/action/cat/RestShardsAction.java     |  2 +-
 .../rest/action/cat/RestSnapshotAction.java   |  2 +-
 .../rest/action/cat/RestTasksAction.java      |  2 +-
 .../rest/action/cat/RestThreadPoolAction.java |  2 +-
 .../rest/action/index/RestIndexAction.java    |  4 +-
 .../elasticsearch/script/ScriptService.java   | 12 +++---
 .../elasticsearch/search/SearchService.java   | 43 ++++++-------------
 .../bucket/BestBucketsDeferringCollector.java |  1 +
 .../bucket/BestDocsDeferringCollector.java    |  1 +
 .../bucket/DeferringBucketCollector.java      |  9 +---
 .../bucket/range/InternalRange.java           |  2 -
 .../scripted/ScriptedMetricAggregator.java    |  3 +-
 .../support/ValuesSourceConfig.java           |  1 -
 .../highlight/FastVectorHighlighter.java      |  6 +--
 .../SourceScoreOrderFragmentsBuilder.java     |  8 +---
 .../SourceSimpleFragmentsBuilder.java         |  8 +---
 .../internal/ShardSearchTransportRequest.java |  7 ---
 .../completion/CompletionSuggestion.java      |  1 -
 .../time/format/StrictISODateTimeFormat.java  |  6 +--
 .../ClusterAllocationExplainIT.java           | 12 ++----
 .../ClusterAllocationExplainTests.java        |  2 -
 .../ClusterAllocationExplanationTests.java    |  3 --
 .../action/bulk/BulkShardRequestTests.java    |  6 +--
 .../ClusterStateCreationUtils.java            |  1 -
 .../action/termvectors/GetTermVectorsIT.java  | 17 --------
 .../action/update/UpdateRequestTests.java     |  2 +-
 .../elasticsearch/blocks/SimpleBlocksIT.java  |  9 ----
 .../NodesStatsBasicBackwardsCompatIT.java     |  5 +--
 .../common/network/NetworkModuleTests.java    |  2 +-
 .../gateway/GatewayServiceTests.java          |  3 +-
 .../elasticsearch/index/IndexModuleTests.java |  1 -
 .../index/IndexServiceTests.java              |  4 --
 .../index/SettingsListenerIT.java             |  2 -
 .../IndexingMemoryControllerTests.java        |  3 +-
 .../plugin/example/ExampleCatAction.java      |  3 +-
 102 files changed, 148 insertions(+), 317 deletions(-)

diff --git a/core/src/main/java/org/apache/lucene/store/StoreRateLimiting.java b/core/src/main/java/org/apache/lucene/store/StoreRateLimiting.java
index ea504f7688c..e1ae7b938b3 100644
--- a/core/src/main/java/org/apache/lucene/store/StoreRateLimiting.java
+++ b/core/src/main/java/org/apache/lucene/store/StoreRateLimiting.java
@@ -26,7 +26,7 @@ import org.elasticsearch.common.unit.ByteSizeValue;
  */
 public class StoreRateLimiting {
 
-    public static interface Provider {
+    public interface Provider {
 
         StoreRateLimiting rateLimiting();
     }
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java
index 321b7e2a8e5..2c16bcb5e9c 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java
@@ -38,7 +38,7 @@ public class BulkShardRequest extends ReplicatedWriteRequest<BulkShardRequest> {
     public BulkShardRequest() {
     }
 
-    BulkShardRequest(BulkRequest bulkRequest, ShardId shardId, RefreshPolicy refreshPolicy, BulkItemRequest[] items) {
+    BulkShardRequest(ShardId shardId, RefreshPolicy refreshPolicy, BulkItemRequest[] items) {
         super(shardId);
         this.items = items;
         setRefreshPolicy(refreshPolicy);
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
index c6b046a4758..022a427081a 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
@@ -344,7 +344,7 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
         for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) {
             final ShardId shardId = entry.getKey();
             final List<BulkItemRequest> requests = entry.getValue();
-            BulkShardRequest bulkShardRequest = new BulkShardRequest(bulkRequest, shardId, bulkRequest.getRefreshPolicy(),
+            BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, bulkRequest.getRefreshPolicy(),
                     requests.toArray(new BulkItemRequest[requests.size()]));
             bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel());
             bulkShardRequest.timeout(bulkRequest.timeout());
diff --git a/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java b/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
index 1084cf6c237..dc49f2f7ad6 100644
--- a/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
+++ b/core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java
@@ -131,7 +131,7 @@ public class TransportExplainAction extends TransportSingleShardAction<ExplainRe
                 // Advantage is that we're not opening a second searcher to retrieve the _source. Also
                 // because we are working in the same searcher in engineGetResult we can be sure that a
                 // doc isn't deleted between the initial get and this call.
-                GetResult getResult = indexShard.getService().get(result, request.id(), request.type(), request.fields(), request.fetchSourceContext(), false);
+                GetResult getResult = indexShard.getService().get(result, request.id(), request.type(), request.fields(), request.fetchSourceContext());
                 return new ExplainResponse(shardId.getIndexName(), request.type(), request.id(), true, explanation, getResult);
             } else {
                 return new ExplainResponse(shardId.getIndexName(), request.type(), request.id(), true, explanation);
diff --git a/core/src/main/java/org/elasticsearch/action/support/WriteResponse.java b/core/src/main/java/org/elasticsearch/action/support/WriteResponse.java
index 07f5ea695d9..e0e4d4f4c98 100644
--- a/core/src/main/java/org/elasticsearch/action/support/WriteResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/support/WriteResponse.java
@@ -36,5 +36,5 @@ public interface WriteResponse {
      * {@link RefreshPolicy#IMMEDIATE} should always mark this as true. Requests that set it to {@link RefreshPolicy#WAIT_UNTIL} will only
      * set this to true if they run out of refresh listener slots (see {@link IndexSettings#MAX_REFRESH_LISTENERS_PER_SHARD}).
      */
-    public abstract void setForcedRefresh(boolean forcedRefresh);
+    void setForcedRefresh(boolean forcedRefresh);
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
index f04e5604a6b..b75cde3f765 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
@@ -298,7 +298,6 @@ public class ReplicationResponse extends ActionResponse {
             private static final String _SHARDS = "_shards";
             private static final String TOTAL = "total";
             private static final String SUCCESSFUL = "successful";
-            private static final String PENDING = "pending";
             private static final String FAILED = "failed";
             private static final String FAILURES = "failures";
 
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java
index 4080d40e695..3f33b2e3901 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/TermVectorsRequest.java
@@ -595,7 +595,7 @@ public class TermVectorsRequest extends SingleShardRequest<TermVectorsRequest> i
                 } else if (currentFieldName.equals("per_field_analyzer") || currentFieldName.equals("perFieldAnalyzer")) {
                     termVectorsRequest.perFieldAnalyzer(readPerFieldAnalyzer(parser.map()));
                 } else if (currentFieldName.equals("filter")) {
-                    termVectorsRequest.filterSettings(readFilterSettings(parser, termVectorsRequest));
+                    termVectorsRequest.filterSettings(readFilterSettings(parser));
                 } else if ("_index".equals(currentFieldName)) { // the following is important for multi request parsing.
                     termVectorsRequest.index = parser.text();
                 } else if ("_type".equals(currentFieldName)) {
@@ -641,7 +641,7 @@ public class TermVectorsRequest extends SingleShardRequest<TermVectorsRequest> i
         return mapStrStr;
     }
 
-    private static FilterSettings readFilterSettings(XContentParser parser, TermVectorsRequest termVectorsRequest) throws IOException {
+    private static FilterSettings readFilterSettings(XContentParser parser) throws IOException {
         FilterSettings settings = new FilterSettings();
         XContentParser.Token token;
         String currentFieldName = null;
diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java b/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
index e2a035d3b5c..aa92510e38b 100644
--- a/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
+++ b/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java
@@ -22,8 +22,6 @@ package org.elasticsearch.action.update;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.client.Requests;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
@@ -63,13 +61,11 @@ import java.util.Map;
 public class UpdateHelper extends AbstractComponent {
 
     private final ScriptService scriptService;
-    private final ClusterService clusterService;
 
     @Inject
-    public UpdateHelper(Settings settings, ScriptService scriptService, ClusterService clusterService) {
+    public UpdateHelper(Settings settings, ScriptService scriptService) {
         super(settings);
         this.scriptService = scriptService;
-        this.clusterService = clusterService;
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterInfoService.java b/core/src/main/java/org/elasticsearch/cluster/ClusterInfoService.java
index d4ceb844ec2..c17bc08ac0a 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterInfoService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterInfoService.java
@@ -26,16 +26,16 @@ package org.elasticsearch.cluster;
 public interface ClusterInfoService {
 
     /** The latest cluster information */
-    public ClusterInfo getClusterInfo();
+    ClusterInfo getClusterInfo();
 
     /** Add a listener that will be called every time new information is gathered */
-    public void addListener(Listener listener);
+    void addListener(Listener listener);
 
     /**
      * Interface for listeners to implement in order to perform actions when
      * new information about the cluster has been gathered
      */
-    public interface Listener {
-        public void onNewInfo(ClusterInfo info);
+    interface Listener {
+        void onNewInfo(ClusterInfo info);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateFilter.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateFilter.java
index 95accc4db80..0ec3c5f863b 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateFilter.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateFilter.java
@@ -32,7 +32,7 @@ public interface IndexTemplateFilter {
      */
     boolean apply(CreateIndexClusterStateUpdateRequest request, IndexTemplateMetaData template);
 
-    static class Compound implements IndexTemplateFilter {
+    class Compound implements IndexTemplateFilter {
 
         private IndexTemplateFilter[] filters;
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
index 981e6dc1914..78206cd1182 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
@@ -19,11 +19,8 @@
 
 package org.elasticsearch.cluster.metadata;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.TimestampParsingException;
 import org.elasticsearch.cluster.AbstractDiffable;
-import org.elasticsearch.common.Nullable;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -34,11 +31,9 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
@@ -82,8 +77,6 @@ public class MappingMetaData extends AbstractDiffable<MappingMetaData> {
 
     public static class Timestamp {
 
-        private static final FormatDateTimeFormatter EPOCH_MILLIS_PARSER = Joda.forPattern("epoch_millis");
-
         public static String parseStringTimestamp(String timestampAsString, FormatDateTimeFormatter dateTimeFormatter) throws TimestampParsingException {
             try {
                 return Long.toString(dateTimeFormatter.parser().parseMillis(timestampAsString));
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
index 8f59ea3a527..625f1a0bb56 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
@@ -35,8 +35,8 @@ import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.indices.IndexTemplateAlreadyExistsException;
@@ -160,7 +160,7 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
                     throw new IndexTemplateAlreadyExistsException(request.name);
                 }
 
-                validateAndAddTemplate(request, templateBuilder, indicesService, nodeServicesProvider, metaDataCreateIndexService);
+                validateAndAddTemplate(request, templateBuilder, indicesService, nodeServicesProvider);
 
                 for (Alias alias : request.aliases) {
                     AliasMetaData aliasMetaData = AliasMetaData.builder(alias.name()).filter(alias.filter())
@@ -185,7 +185,7 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
     }
 
     private static void validateAndAddTemplate(final PutRequest request, IndexTemplateMetaData.Builder templateBuilder, IndicesService indicesService,
-                                               NodeServicesProvider nodeServicesProvider, MetaDataCreateIndexService metaDataCreateIndexService) throws Exception {
+                                               NodeServicesProvider nodeServicesProvider) throws Exception {
         Index createdIndex = null;
         final String temporaryIndexName = UUIDs.randomBase64UUID();
         try {
@@ -276,7 +276,7 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
         }
     }
 
-    public static interface PutListener {
+    public interface PutListener {
 
         void onResponse(PutResponse response);
 
@@ -391,7 +391,7 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
         }
     }
 
-    public static interface RemoveListener {
+    public interface RemoveListener {
 
         void onResponse(RemoveResponse response);
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
index 5b24ac7edff..ef3fae48301 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java
@@ -229,11 +229,4 @@ public class OperationRouting extends AbstractComponent {
         // of original index to hash documents
         return Math.floorMod(hash, indexMetaData.getRoutingNumShards()) / indexMetaData.getRoutingFactor();
     }
-
-    private void ensureNodeIdExists(DiscoveryNodes nodes, String nodeId) {
-        if (!nodes.getDataNodes().keys().contains(nodeId)) {
-            throw new IllegalArgumentException("No data node with id[" + nodeId + "] found");
-        }
-    }
-
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
index ca341451fed..edeb609a9c7 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java
@@ -505,7 +505,6 @@ public class DiskThresholdDecider extends AllocationDecider {
     }
 
     private DiskUsage getDiskUsage(RoutingNode node, RoutingAllocation allocation, ImmutableOpenMap<String, DiskUsage> usages) {
-        ClusterInfo clusterInfo = allocation.clusterInfo();
         DiskUsage usage = usages.get(node.nodeId());
         if (usage == null) {
             // If there is no usage, and we have other nodes in the cluster,
diff --git a/core/src/main/java/org/elasticsearch/common/UUIDGenerator.java b/core/src/main/java/org/elasticsearch/common/UUIDGenerator.java
index 0cdcfec1cf7..a74c5fbec30 100644
--- a/core/src/main/java/org/elasticsearch/common/UUIDGenerator.java
+++ b/core/src/main/java/org/elasticsearch/common/UUIDGenerator.java
@@ -23,5 +23,5 @@ package org.elasticsearch.common;
  * Generates opaque unique strings.
  */
 interface UUIDGenerator {
-    public String getBase64UUID();
+    String getBase64UUID();
 }
diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoDistance.java b/core/src/main/java/org/elasticsearch/common/geo/GeoDistance.java
index c6900e2ae4d..efd8d529d4b 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/GeoDistance.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/GeoDistance.java
@@ -80,7 +80,7 @@ public enum GeoDistance implements Writeable {
 
         @Override
         public FixedSourceDistance fixedSourceDistance(double sourceLatitude, double sourceLongitude, DistanceUnit unit) {
-            return new FactorFixedSourceDistance(sourceLatitude, sourceLongitude, unit);
+            return new FactorFixedSourceDistance(sourceLatitude, sourceLongitude);
         }
     },
     /**
@@ -217,12 +217,12 @@ public enum GeoDistance implements Writeable {
         throw new IllegalArgumentException("No geo distance for [" + name + "]");
     }
 
-    public static interface FixedSourceDistance {
+    public interface FixedSourceDistance {
 
         double calculate(double targetLatitude, double targetLongitude);
     }
 
-    public static interface DistanceBoundingCheck {
+    public interface DistanceBoundingCheck {
 
         boolean isWithin(double targetLatitude, double targetLongitude);
 
@@ -331,7 +331,7 @@ public enum GeoDistance implements Writeable {
         private final double sinA;
         private final double cosA;
 
-        public FactorFixedSourceDistance(double sourceLatitude, double sourceLongitude, DistanceUnit unit) {
+        public FactorFixedSourceDistance(double sourceLatitude, double sourceLongitude) {
             this.sourceLongitude = sourceLongitude;
             this.a = Math.toRadians(90D - sourceLatitude);
             this.sinA = Math.sin(a);
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java
index 9fad4fb8ef3..589829f661e 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java
@@ -19,20 +19,19 @@
 
 package org.elasticsearch.common.geo.builders;
 
-import org.locationtech.spatial4j.exception.InvalidShapeException;
-import org.locationtech.spatial4j.shape.Shape;
 import com.vividsolutions.jts.geom.Coordinate;
 import com.vividsolutions.jts.geom.Geometry;
 import com.vividsolutions.jts.geom.GeometryFactory;
 import com.vividsolutions.jts.geom.LinearRing;
 import com.vividsolutions.jts.geom.MultiPolygon;
 import com.vividsolutions.jts.geom.Polygon;
-
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.locationtech.spatial4j.exception.InvalidShapeException;
+import org.locationtech.spatial4j.shape.Shape;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -579,7 +578,7 @@ public class PolygonBuilder extends ShapeBuilder {
         boolean direction = (component == 0 ^ orientation == Orientation.RIGHT);
         // set the points array accordingly (shell or hole)
         Coordinate[] points = (hole != null) ? hole.coordinates(false) : shell.coordinates(false);
-        ring(component, direction, orientation == Orientation.LEFT, shell, points, 0, edges, offset, points.length-1, translated);
+        ring(component, direction, orientation == Orientation.LEFT, points, 0, edges, offset, points.length-1, translated);
         return points.length-1;
     }
 
@@ -594,7 +593,7 @@ public class PolygonBuilder extends ShapeBuilder {
      *            number of points
      * @return Array of edges
      */
-    private static Edge[] ring(int component, boolean direction, boolean handedness, LineStringBuilder shell,
+    private static Edge[] ring(int component, boolean direction, boolean handedness,
                                  Coordinate[] points, int offset, Edge[] edges, int toffset, int length, final AtomicBoolean translated) {
         // calculate the direction of the points:
         // find the point a the top of the set and check its
diff --git a/core/src/main/java/org/elasticsearch/common/inject/Scope.java b/core/src/main/java/org/elasticsearch/common/inject/Scope.java
index dacb105fa3a..073ce7d5fd9 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/Scope.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/Scope.java
@@ -47,7 +47,7 @@ public interface Scope {
      *         when an instance of the requested object doesn't already exist in this
      *         scope
      */
-    public <T> Provider<T> scope(Key<T> key, Provider<T> unscoped);
+    <T> Provider<T> scope(Key<T> key, Provider<T> unscoped);
 
     /**
      * A short but useful description of this scope.  For comparison, the standard
diff --git a/core/src/main/java/org/elasticsearch/common/inject/State.java b/core/src/main/java/org/elasticsearch/common/inject/State.java
index 2337816b01c..761a7e4b523 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/State.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/State.java
@@ -36,7 +36,7 @@ import static java.util.Collections.emptySet;
  */
 interface State {
 
-    static final State NONE = new State() {
+    State NONE = new State() {
         @Override
         public State parent() {
             throw new UnsupportedOperationException();
diff --git a/core/src/main/java/org/elasticsearch/common/inject/internal/InternalFactory.java b/core/src/main/java/org/elasticsearch/common/inject/internal/InternalFactory.java
index b2203ed3b64..41933472ced 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/internal/InternalFactory.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/internal/InternalFactory.java
@@ -29,7 +29,7 @@ public interface InternalFactory<T> {
      * ES:
      * An factory that returns a pre created instance.
      */
-    public static class Instance<T> implements InternalFactory<T> {
+    class Instance<T> implements InternalFactory<T> {
 
         private final T object;
 
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/all/AllTokenStream.java b/core/src/main/java/org/elasticsearch/common/lucene/all/AllTokenStream.java
index e22fb0f665c..6a49034d0b3 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/all/AllTokenStream.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/all/AllTokenStream.java
@@ -22,7 +22,6 @@ package org.elasticsearch.common.lucene.all;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.SmallFloat;
@@ -38,12 +37,10 @@ public final class AllTokenStream extends TokenFilter {
     }
 
     private final BytesRef payloadSpare = new BytesRef(new byte[1]);
-    private final OffsetAttribute offsetAttribute;
     private final PayloadAttribute payloadAttribute;
 
     AllTokenStream(TokenStream input, float boost) {
         super(input);
-        offsetAttribute = addAttribute(OffsetAttribute.class);
         payloadAttribute = addAttribute(PayloadAttribute.class);
         payloadSpare.bytes[0] = SmallFloat.floatToByte315(boost);
     }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/SettingsFilter.java b/core/src/main/java/org/elasticsearch/common/settings/SettingsFilter.java
index 636b1e40afc..32c5e7a0da3 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/SettingsFilter.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/SettingsFilter.java
@@ -49,7 +49,6 @@ public final class SettingsFilter extends AbstractComponent {
 
     public SettingsFilter(Settings settings, Collection<String> patterns) {
         super(settings);
-        HashSet<String> set = new HashSet<>();
         for (String pattern : patterns) {
             if (isValidPattern(pattern) == false) {
                 throw new IllegalArgumentException("invalid pattern: " + pattern);
diff --git a/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoader.java b/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoader.java
index 67c09e42db2..080824befb3 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoader.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/loader/SettingsLoader.java
@@ -33,7 +33,7 @@ import java.util.Map;
  */
 public interface SettingsLoader {
 
-    static class Helper {
+    class Helper {
 
         public static Map<String, String> loadNestedFromMap(@Nullable Map map) {
             Map<String, String> settings = new HashMap<>();
diff --git a/core/src/main/java/org/elasticsearch/common/transport/PortsRange.java b/core/src/main/java/org/elasticsearch/common/transport/PortsRange.java
index 4f5a3966d43..f88f1de8fe0 100644
--- a/core/src/main/java/org/elasticsearch/common/transport/PortsRange.java
+++ b/core/src/main/java/org/elasticsearch/common/transport/PortsRange.java
@@ -80,7 +80,7 @@ public class PortsRange {
         return success;
     }
 
-    public static interface PortCallback {
+    public interface PortCallback {
         boolean onPortNumber(int portNumber);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/transport/TransportAddress.java b/core/src/main/java/org/elasticsearch/common/transport/TransportAddress.java
index 99ba37a4f60..08e8af2bffe 100644
--- a/core/src/main/java/org/elasticsearch/common/transport/TransportAddress.java
+++ b/core/src/main/java/org/elasticsearch/common/transport/TransportAddress.java
@@ -48,5 +48,5 @@ public interface TransportAddress extends Writeable {
 
     boolean isLoopbackOrLinkLocalAddress();
 
-    public String toString();
+    String toString();
 }
diff --git a/core/src/main/java/org/elasticsearch/common/util/BigArray.java b/core/src/main/java/org/elasticsearch/common/util/BigArray.java
index 6fc2b10ed59..5c108a9d503 100644
--- a/core/src/main/java/org/elasticsearch/common/util/BigArray.java
+++ b/core/src/main/java/org/elasticsearch/common/util/BigArray.java
@@ -26,6 +26,6 @@ import org.elasticsearch.common.lease.Releasable;
 public interface BigArray extends Releasable, Accountable {
 
     /** Return the length of this array. */
-    public long size();
+    long size();
 
 }
diff --git a/core/src/main/java/org/elasticsearch/common/util/DoubleArray.java b/core/src/main/java/org/elasticsearch/common/util/DoubleArray.java
index f38b6a5641d..819d913bbd3 100644
--- a/core/src/main/java/org/elasticsearch/common/util/DoubleArray.java
+++ b/core/src/main/java/org/elasticsearch/common/util/DoubleArray.java
@@ -27,21 +27,21 @@ public interface DoubleArray extends BigArray {
     /**
      * Get an element given its index.
      */
-    public abstract double get(long index);
+    double get(long index);
 
     /**
      * Set a value at the given index and return the previous value.
      */
-    public abstract double set(long index, double value);
+    double set(long index, double value);
 
     /**
      * Increment value at the given index by <code>inc</code> and return the value.
      */
-    public abstract double increment(long index, double inc);
+    double increment(long index, double inc);
 
     /**
      * Fill slots between <code>fromIndex</code> inclusive to <code>toIndex</code> exclusive with <code>value</code>.
      */
-    public abstract void fill(long fromIndex, long toIndex, double value);
+    void fill(long fromIndex, long toIndex, double value);
 
 }
diff --git a/core/src/main/java/org/elasticsearch/common/util/FloatArray.java b/core/src/main/java/org/elasticsearch/common/util/FloatArray.java
index a90298e8ed5..a07bb35dfb2 100644
--- a/core/src/main/java/org/elasticsearch/common/util/FloatArray.java
+++ b/core/src/main/java/org/elasticsearch/common/util/FloatArray.java
@@ -27,21 +27,21 @@ public interface FloatArray extends BigArray {
     /**
      * Get an element given its index.
      */
-    public abstract float get(long index);
+    float get(long index);
 
     /**
      * Set a value at the given index and return the previous value.
      */
-    public abstract float set(long index, float value);
+    float set(long index, float value);
 
     /**
      * Increment value at the given index by <code>inc</code> and return the value.
      */
-    public abstract float increment(long index, float inc);
+    float increment(long index, float inc);
 
     /**
      * Fill slots between <code>fromIndex</code> inclusive to <code>toIndex</code> exclusive with <code>value</code>.
      */
-    public abstract void fill(long fromIndex, long toIndex, float value);
+    void fill(long fromIndex, long toIndex, float value);
 
 }
diff --git a/core/src/main/java/org/elasticsearch/common/util/IntArray.java b/core/src/main/java/org/elasticsearch/common/util/IntArray.java
index 096b81be498..2a7214b0f70 100644
--- a/core/src/main/java/org/elasticsearch/common/util/IntArray.java
+++ b/core/src/main/java/org/elasticsearch/common/util/IntArray.java
@@ -27,21 +27,21 @@ public interface IntArray extends BigArray {
     /**
      * Get an element given its index.
      */
-    public abstract int get(long index);
+    int get(long index);
 
     /**
      * Set a value at the given index and return the previous value.
      */
-    public abstract int set(long index, int value);
+    int set(long index, int value);
 
     /**
      * Increment value at the given index by <code>inc</code> and return the value.
      */
-    public abstract int increment(long index, int inc);
+    int increment(long index, int inc);
 
     /**
      * Fill slots between <code>fromIndex</code> inclusive to <code>toIndex</code> exclusive with <code>value</code>.
      */
-    public abstract void fill(long fromIndex, long toIndex, int value);
+    void fill(long fromIndex, long toIndex, int value);
 
 }
diff --git a/core/src/main/java/org/elasticsearch/common/util/LongArray.java b/core/src/main/java/org/elasticsearch/common/util/LongArray.java
index 670f729da1c..fe8a40008d5 100644
--- a/core/src/main/java/org/elasticsearch/common/util/LongArray.java
+++ b/core/src/main/java/org/elasticsearch/common/util/LongArray.java
@@ -27,21 +27,21 @@ public interface LongArray extends BigArray {
     /**
      * Get an element given its index.
      */
-    public abstract long get(long index);
+    long get(long index);
 
     /**
      * Set a value at the given index and return the previous value.
      */
-    public abstract long set(long index, long value);
+    long set(long index, long value);
 
     /**
      * Increment value at the given index by <code>inc</code> and return the value.
      */
-    public abstract long increment(long index, long inc);
+    long increment(long index, long inc);
 
     /**
      * Fill slots between <code>fromIndex</code> inclusive to <code>toIndex</code> exclusive with <code>value</code>.
      */
-    public abstract void fill(long fromIndex, long toIndex, long value);
+    void fill(long fromIndex, long toIndex, long value);
 
 }
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
index 512690595ef..c715fceb84a 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
@@ -263,8 +263,7 @@ public final class ThreadContext implements Closeable, Writeable {
         }
 
         private void putSingleHeader(String key, String value, Map<String, String> newHeaders) {
-            final String existingValue;
-            if ((existingValue = newHeaders.putIfAbsent(key, value)) != null) {
+            if (newHeaders.putIfAbsent(key, value) != null) {
                 throw new IllegalArgumentException("value for key [" + key + "] already present");
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java b/core/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java
index 2d39578ea9c..01111fa940a 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/ToXContent.java
@@ -29,7 +29,7 @@ import java.util.Map;
  */
 public interface ToXContent {
 
-    public static interface Params {
+    interface Params {
         String param(String key);
 
         String param(String key, String defaultValue);
@@ -39,7 +39,7 @@ public interface ToXContent {
         Boolean paramAsBoolean(String key, Boolean defaultValue);
     }
 
-    public static final Params EMPTY_PARAMS = new Params() {
+    Params EMPTY_PARAMS = new Params() {
         @Override
         public String param(String key) {
             return null;
@@ -62,7 +62,7 @@ public interface ToXContent {
 
     };
 
-    public static class MapParams implements Params {
+    class MapParams implements Params {
 
         private final Map<String, String> params;
 
@@ -95,7 +95,7 @@ public interface ToXContent {
         }
     }
 
-    public static class DelegatingMapParams extends MapParams {
+    class DelegatingMapParams extends MapParams {
 
         private final Params delegate;
 
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java
index 39a6fca1095..eaaa98167bd 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java
@@ -228,23 +228,13 @@ public abstract class AbstractXContentParser implements XContentParser {
         return readListOrderedMap(this);
     }
 
-    static interface MapFactory {
+    interface MapFactory {
         Map<String, Object> newMap();
     }
 
-    static final MapFactory SIMPLE_MAP_FACTORY = new MapFactory() {
-        @Override
-        public Map<String, Object> newMap() {
-            return new HashMap<>();
-        }
-    };
+    static final MapFactory SIMPLE_MAP_FACTORY = HashMap::new;
 
-    static final MapFactory ORDERED_MAP_FACTORY = new MapFactory() {
-        @Override
-        public Map<String, Object> newMap() {
-            return new LinkedHashMap<>();
-        }
-    };
+    static final MapFactory ORDERED_MAP_FACTORY = LinkedHashMap::new;
 
     static Map<String, Object> readMap(XContentParser parser) throws IOException {
         return readMap(parser, SIMPLE_MAP_FACTORY);
diff --git a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
index 9b8e148550e..a711427de77 100644
--- a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
@@ -50,7 +50,6 @@ import java.util.HashSet;
 import java.util.Queue;
 import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.elasticsearch.cluster.ClusterState.Builder;
 
@@ -69,8 +68,6 @@ public class LocalDiscovery extends AbstractLifecycleComponent<Discovery> implem
 
     private volatile boolean master = false;
 
-    private final AtomicBoolean initialStateSent = new AtomicBoolean();
-
     private static final ConcurrentMap<ClusterName, ClusterGroup> clusterGroups = ConcurrentCollections.newConcurrentMap();
 
     private volatile ClusterState lastProcessedClusterState;
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
index eca34389d90..0efbe64196e 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
@@ -197,7 +197,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implemen
                         discoverySettings,
                         clusterService.getClusterName());
         this.pingService.setPingContextProvider(this);
-        this.membership = new MembershipAction(settings, clusterService, transportService, this, new MembershipListener());
+        this.membership = new MembershipAction(settings, transportService, this, new MembershipListener());
 
         this.joinThreadControl = new JoinThreadControl(threadPool);
 
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/membership/MembershipAction.java b/core/src/main/java/org/elasticsearch/discovery/zen/membership/MembershipAction.java
index 46686965741..1de3805d67e 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/membership/MembershipAction.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/membership/MembershipAction.java
@@ -21,7 +21,6 @@ package org.elasticsearch.discovery.zen.membership;
 
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -48,13 +47,13 @@ public class MembershipAction extends AbstractComponent {
     public static final String DISCOVERY_JOIN_VALIDATE_ACTION_NAME = "internal:discovery/zen/join/validate";
     public static final String DISCOVERY_LEAVE_ACTION_NAME = "internal:discovery/zen/leave";
 
-    public static interface JoinCallback {
+    public interface JoinCallback {
         void onSuccess();
 
         void onFailure(Throwable t);
     }
 
-    public static interface MembershipListener {
+    public interface MembershipListener {
         void onJoin(DiscoveryNode node, JoinCallback callback);
 
         void onLeave(DiscoveryNode node);
@@ -66,14 +65,11 @@ public class MembershipAction extends AbstractComponent {
 
     private final MembershipListener listener;
 
-    private final ClusterService clusterService;
-
-    public MembershipAction(Settings settings, ClusterService clusterService, TransportService transportService, DiscoveryNodesProvider nodesProvider, MembershipListener listener) {
+    public MembershipAction(Settings settings, TransportService transportService, DiscoveryNodesProvider nodesProvider, MembershipListener listener) {
         super(settings);
         this.transportService = transportService;
         this.nodesProvider = nodesProvider;
         this.listener = listener;
-        this.clusterService = clusterService;
 
         transportService.registerRequestHandler(DISCOVERY_JOIN_ACTION_NAME, JoinRequest::new, ThreadPool.Names.GENERIC, new JoinRequestRequestHandler());
         transportService.registerRequestHandler(DISCOVERY_JOIN_VALIDATE_ACTION_NAME, ValidateJoinRequest::new, ThreadPool.Names.GENERIC, new ValidateJoinRequestRequestHandler());
diff --git a/core/src/main/java/org/elasticsearch/gateway/Gateway.java b/core/src/main/java/org/elasticsearch/gateway/Gateway.java
index 919db87d386..ee2abc17ab6 100644
--- a/core/src/main/java/org/elasticsearch/gateway/Gateway.java
+++ b/core/src/main/java/org/elasticsearch/gateway/Gateway.java
@@ -32,7 +32,6 @@ import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.Discovery;
-import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.indices.IndicesService;
@@ -47,8 +46,6 @@ public class Gateway extends AbstractComponent implements ClusterStateListener {
 
     private final ClusterService clusterService;
 
-    private final NodeEnvironment nodeEnv;
-
     private final GatewayMetaState metaState;
 
     private final TransportNodesListGatewayMetaState listGatewayMetaState;
@@ -57,14 +54,13 @@ public class Gateway extends AbstractComponent implements ClusterStateListener {
     private final IndicesService indicesService;
     private final NodeServicesProvider nodeServicesProvider;
 
-    public Gateway(Settings settings, ClusterService clusterService, NodeEnvironment nodeEnv, GatewayMetaState metaState,
+    public Gateway(Settings settings, ClusterService clusterService, GatewayMetaState metaState,
                    TransportNodesListGatewayMetaState listGatewayMetaState, Discovery discovery,
                    NodeServicesProvider nodeServicesProvider, IndicesService indicesService) {
         super(settings);
         this.nodeServicesProvider = nodeServicesProvider;
         this.indicesService = indicesService;
         this.clusterService = clusterService;
-        this.nodeEnv = nodeEnv;
         this.metaState = metaState;
         this.listGatewayMetaState = listGatewayMetaState;
         this.minimumMasterNodesProvider = discovery::getMinimumMasterNodes;
diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayService.java b/core/src/main/java/org/elasticsearch/gateway/GatewayService.java
index 7261f812696..8bd1dcbd6ea 100644
--- a/core/src/main/java/org/elasticsearch/gateway/GatewayService.java
+++ b/core/src/main/java/org/elasticsearch/gateway/GatewayService.java
@@ -42,7 +42,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.discovery.Discovery;
-import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.index.NodeServicesProvider;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.rest.RestStatus;
@@ -96,11 +95,11 @@ public class GatewayService extends AbstractLifecycleComponent<GatewayService> i
 
     @Inject
     public GatewayService(Settings settings, AllocationService allocationService, ClusterService clusterService,
-                          ThreadPool threadPool, NodeEnvironment nodeEnvironment, GatewayMetaState metaState,
+                          ThreadPool threadPool, GatewayMetaState metaState,
                           TransportNodesListGatewayMetaState listGatewayMetaState, Discovery discovery,
                           NodeServicesProvider nodeServicesProvider, IndicesService indicesService) {
         super(settings);
-        this.gateway = new Gateway(settings, clusterService, nodeEnvironment, metaState, listGatewayMetaState, discovery,
+        this.gateway = new Gateway(settings, clusterService, metaState, listGatewayMetaState, discovery,
             nodeServicesProvider, indicesService);
         this.allocationService = allocationService;
         this.clusterService = clusterService;
diff --git a/core/src/main/java/org/elasticsearch/index/MergePolicyConfig.java b/core/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
index c8d82eae888..b5856d606e4 100644
--- a/core/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
+++ b/core/src/main/java/org/elasticsearch/index/MergePolicyConfig.java
@@ -157,7 +157,6 @@ public final class MergePolicyConfig {
 
     MergePolicyConfig(ESLogger logger, IndexSettings indexSettings) {
         this.logger = logger;
-        IndexScopedSettings scopedSettings = indexSettings.getScopedSettings();
         double forceMergeDeletesPctAllowed = indexSettings.getValue(INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING); // percentage
         ByteSizeValue floorSegment = indexSettings.getValue(INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING);
         int maxMergeAtOnce = indexSettings.getValue(INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING);
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java
index bafa00b7c52..d85073a0688 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiOrdinals.java
@@ -41,7 +41,6 @@ import java.util.List;
 public class MultiOrdinals extends Ordinals {
 
     private static final int OFFSETS_PAGE_SIZE = 1024;
-    private static final int OFFSET_INIT_PAGE_COUNT = 16;
 
     /**
      * Return true if this impl is going to be smaller than {@link SinglePackedOrdinals} by at least 20%.
diff --git a/core/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/core/src/main/java/org/elasticsearch/index/get/ShardGetService.java
index 10c2e636e9a..8004b91296f 100644
--- a/core/src/main/java/org/elasticsearch/index/get/ShardGetService.java
+++ b/core/src/main/java/org/elasticsearch/index/get/ShardGetService.java
@@ -110,7 +110,7 @@ public final class ShardGetService extends AbstractIndexShardComponent {
      * <p>
      * Note: Call <b>must</b> release engine searcher associated with engineGetResult!
      */
-    public GetResult get(Engine.GetResult engineGetResult, String id, String type, String[] fields, FetchSourceContext fetchSourceContext, boolean ignoreErrorsOnGeneratedFields) {
+    public GetResult get(Engine.GetResult engineGetResult, String id, String type, String[] fields, FetchSourceContext fetchSourceContext) {
         if (!engineGetResult.exists()) {
             return new GetResult(shardId.getIndexName(), type, id, -1, false, null, null);
         }
@@ -119,7 +119,7 @@ public final class ShardGetService extends AbstractIndexShardComponent {
         try {
             long now = System.nanoTime();
             fetchSourceContext = normalizeFetchSourceContent(fetchSourceContext, fields);
-            GetResult getResult = innerGetLoadFromStoredFields(type, id, fields, fetchSourceContext, engineGetResult, mapperService, ignoreErrorsOnGeneratedFields);
+            GetResult getResult = innerGetLoadFromStoredFields(type, id, fields, fetchSourceContext, engineGetResult, mapperService);
             if (getResult.isExists()) {
                 existsMetric.inc(System.nanoTime() - now);
             } else {
@@ -183,7 +183,7 @@ public final class ShardGetService extends AbstractIndexShardComponent {
         try {
             // break between having loaded it from translog (so we only have _source), and having a document to load
             if (get.docIdAndVersion() != null) {
-                return innerGetLoadFromStoredFields(type, id, gFields, fetchSourceContext, get, mapperService, ignoreErrorsOnGeneratedFields);
+                return innerGetLoadFromStoredFields(type, id, gFields, fetchSourceContext, get, mapperService);
             } else {
                 Translog.Source source = get.source();
 
@@ -316,7 +316,7 @@ public final class ShardGetService extends AbstractIndexShardComponent {
         }
     }
 
-    private GetResult innerGetLoadFromStoredFields(String type, String id, String[] gFields, FetchSourceContext fetchSourceContext, Engine.GetResult get, MapperService mapperService, boolean ignoreErrorsOnGeneratedFields) {
+    private GetResult innerGetLoadFromStoredFields(String type, String id, String[] gFields, FetchSourceContext fetchSourceContext, Engine.GetResult get, MapperService mapperService) {
         Map<String, GetField> fields = null;
         BytesReference source = null;
         Versions.DocIdAndVersion docIdAndVersion = get.docIdAndVersion();
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
index 41f8963b593..48199a49b36 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
@@ -76,7 +76,7 @@ final class DocumentParser {
                     docMapperParser, docMapper, source, parser);
             validateStart(parser);
             internalParseDocument(mapping, context, parser);
-            validateEnd(source, parser);
+            validateEnd(parser);
         } catch (Throwable t) {
             throw wrapInMapperParsingException(source, t);
         }
@@ -128,7 +128,7 @@ final class DocumentParser {
         }
     }
 
-    private static void validateEnd(SourceToParse source, XContentParser parser) throws IOException {
+    private static void validateEnd(XContentParser parser) throws IOException {
         XContentParser.Token token;// only check for end of tokens if we created the parser here
         // try to parse the next token, this should be null if the object is ended properly
         // but will throw a JSON exception if the extra tokens is not valid JSON (this will be handled by the catch)
@@ -368,7 +368,6 @@ final class DocumentParser {
             token = parser.nextToken();
         }
 
-        ObjectMapper update = null;
         innerParseObject(context, mapper, parser, currentFieldName, token);
         // restore the enable path flag
         if (nested.isNested()) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
index 34a0ead0596..c3c23d82f70 100755
--- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.index.mapper;
 
 import com.carrotsearch.hppc.ObjectHashSet;
-
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;
@@ -35,7 +34,6 @@ import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.AbstractIndexComponent;
-import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.mapper.Mapper.BuilderContext;
@@ -316,7 +314,7 @@ public class MapperService extends AbstractIndexComponent {
         Collections.addAll(fieldMappers, newMapper.mapping().metadataMappers);
         MapperUtils.collect(newMapper.mapping().root(), objectMappers, fieldMappers);
         checkFieldUniqueness(newMapper.type(), objectMappers, fieldMappers);
-        checkObjectsCompatibility(newMapper.type(), objectMappers, fieldMappers, updateAllTypes);
+        checkObjectsCompatibility(objectMappers, updateAllTypes);
 
         // 3. update lookup data-structures
         // this will in particular make sure that the merged fields are compatible with other types
@@ -381,7 +379,7 @@ public class MapperService extends AbstractIndexComponent {
         for (DocumentMapper mapper : docMappers(false)) {
             List<FieldMapper> fieldMappers = new ArrayList<>();
             Collections.addAll(fieldMappers, mapper.mapping().metadataMappers);
-            MapperUtils.collect(mapper.root(), new ArrayList<ObjectMapper>(), fieldMappers);
+            MapperUtils.collect(mapper.root(), new ArrayList<>(), fieldMappers);
             for (FieldMapper fieldMapper : fieldMappers) {
                 assert fieldMapper.fieldType() == fieldTypes.get(fieldMapper.name()) : fieldMapper.name();
             }
@@ -449,7 +447,7 @@ public class MapperService extends AbstractIndexComponent {
         }
     }
 
-    private void checkObjectsCompatibility(String type, Collection<ObjectMapper> objectMappers, Collection<FieldMapper> fieldMappers, boolean updateAllTypes) {
+    private void checkObjectsCompatibility(Collection<ObjectMapper> objectMappers, boolean updateAllTypes) {
         assert Thread.holdsLock(this);
 
         for (ObjectMapper newObjectMapper : objectMappers) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java b/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
index 27b983060a3..64be912733f 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ParseContext.java
@@ -310,8 +310,6 @@ public abstract class ParseContext {
 
         private Field version;
 
-        private StringBuilder stringBuilder = new StringBuilder();
-
         private final AllEntries allEntries;
 
         private final List<Mapper> dynamicMappers;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
index 24a86b11392..6889cd90fd0 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
@@ -24,7 +24,6 @@ import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.index.IndexOptions;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.TimestampParsingException;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
 import org.elasticsearch.common.joda.Joda;
 import org.elasticsearch.common.lucene.Lucene;
@@ -82,7 +81,7 @@ public class TimestampFieldMapper extends MetadataFieldMapper {
         private String defaultTimestamp = Defaults.DEFAULT_TIMESTAMP;
         private Boolean ignoreMissing = null;
 
-        public Builder(MappedFieldType existing, Settings settings) {
+        public Builder(MappedFieldType existing) {
             super(Defaults.NAME, existing == null ? Defaults.FIELD_TYPE : existing, Defaults.FIELD_TYPE);
         }
 
@@ -130,7 +129,7 @@ public class TimestampFieldMapper extends MetadataFieldMapper {
             if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0_alpha4)) {
                 throw new IllegalArgumentException("[_timestamp] is removed in 5.0. As a replacement, you can use an ingest pipeline to add a field with the current timestamp to your documents.");
             }
-            Builder builder = new Builder(parserContext.mapperService().fullName(NAME), parserContext.mapperService().getIndexSettings().getSettings());
+            Builder builder = new Builder(parserContext.mapperService().fullName(NAME));
             boolean defaultSet = false;
             Boolean ignoreMissing = null;
             for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
diff --git a/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
index 692c157c1b1..69ba746c883 100644
--- a/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java
@@ -405,7 +405,6 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQue
     private static Query parseQueryString(ExtendedCommonTermsQuery query, Object queryString, String field, Analyzer analyzer,
                                          String lowFreqMinimumShouldMatch, String highFreqMinimumShouldMatch) throws IOException {
         // Logic similar to QueryParser#getFieldQuery
-        int count = 0;
         try (TokenStream source = analyzer.tokenStream(field, queryString.toString())) {
             source.reset();
             CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
@@ -414,7 +413,6 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder<CommonTermsQue
                 // UTF-8
                 builder.copyChars(termAtt);
                 query.add(new Term(field, builder.toBytesRef()));
-                count++;
             }
         }
 
diff --git a/core/src/main/java/org/elasticsearch/index/shard/SnapshotStatus.java b/core/src/main/java/org/elasticsearch/index/shard/SnapshotStatus.java
index f4b26cd988d..a72cfc48b65 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/SnapshotStatus.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/SnapshotStatus.java
@@ -43,8 +43,6 @@ public class SnapshotStatus {
 
     private Translog translog = new Translog();
 
-    private Throwable failure;
-
     public Stage stage() {
         return this.stage;
     }
@@ -70,10 +68,6 @@ public class SnapshotStatus {
         this.time = time;
     }
 
-    public void failed(Throwable failure) {
-        this.failure = failure;
-    }
-
     public Index index() {
         return index;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
index e4534e59d80..6b8bc607679 100644
--- a/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
+++ b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
@@ -202,7 +202,7 @@ public class TermVectorsService  {
 
         /* generate term vectors from fetched document fields */
         GetResult getResult = indexShard.getService().get(
-                get, request.id(), request.type(), validFields.toArray(Strings.EMPTY_ARRAY), null, false);
+                get, request.id(), request.type(), validFields.toArray(Strings.EMPTY_ARRAY), null);
         Fields generatedTermVectors = generateTermVectors(indexShard, getResult.getFields().values(), request.offsets(), request.perFieldAnalyzer(), validFields);
 
         /* merge with existing Fields */
diff --git a/core/src/main/java/org/elasticsearch/index/translog/Translog.java b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
index fe970dcbc5f..083e011516c 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/Translog.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
@@ -572,10 +572,6 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
         }
     }
 
-    private boolean isReferencedGeneration(long generation) { // used to make decisions if a file can be deleted
-        return generation >= lastCommittedTranslogFileGeneration;
-    }
-
     public TranslogConfig getConfig() {
         return config;
     }
diff --git a/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java b/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
index 27877614300..2a09362e871 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
@@ -20,8 +20,8 @@
 package org.elasticsearch.indices;
 
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
@@ -34,8 +34,6 @@ import org.elasticsearch.index.engine.FlushNotAllowedEngineException;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.IndexingOperationListener;
-import org.elasticsearch.index.translog.Translog;
-import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.Closeable;
@@ -94,10 +92,6 @@ public class IndexingMemoryController extends AbstractComponent implements Index
     private final ShardsIndicesStatusChecker statusChecker;
 
     IndexingMemoryController(Settings settings, ThreadPool threadPool, Iterable<IndexShard> indexServices) {
-        this(settings, threadPool, indexServices, JvmInfo.jvmInfo().getMem().getHeapMax().bytes());
-    }
-
-    IndexingMemoryController(Settings settings, ThreadPool threadPool, Iterable<IndexShard> indexServices, long jvmMemoryInBytes) {
         super(settings);
         this.indexShards = indexServices;
 
diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/HotThreads.java b/core/src/main/java/org/elasticsearch/monitor/jvm/HotThreads.java
index c143e0762d5..181b94f13db 100644
--- a/core/src/main/java/org/elasticsearch/monitor/jvm/HotThreads.java
+++ b/core/src/main/java/org/elasticsearch/monitor/jvm/HotThreads.java
@@ -20,13 +20,9 @@
 package org.elasticsearch.monitor.jvm;
 
 import org.apache.lucene.util.CollectionUtil;
-import org.apache.lucene.util.Constants;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.common.joda.FormatDateTimeFormatter;
 import org.elasticsearch.common.joda.Joda;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.unit.TimeValue;
 
 import java.lang.management.ManagementFactory;
@@ -45,7 +41,6 @@ import java.util.concurrent.TimeUnit;
 public class HotThreads {
 
     private static final Object mutex = new Object();
-    private static final ESLogger logger = Loggers.getLogger(HotThreads.class);
 
     private static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("dateOptionalTime");
 
diff --git a/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
index 951e7760239..383ded2b2de 100644
--- a/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
+++ b/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
@@ -409,7 +409,7 @@ public class RepositoriesService extends AbstractComponent implements ClusterSta
             Repository repository = repositoryInjector.getInstance(Repository.class);
             IndexShardRepository indexShardRepository = repositoryInjector.getInstance(IndexShardRepository.class);
             repository.start();
-            return new RepositoryHolder(repositoryMetaData.type(), repositoryMetaData.settings(), repositoryInjector, repository, indexShardRepository);
+            return new RepositoryHolder(repositoryMetaData.type(), repositoryMetaData.settings(), repository, indexShardRepository);
         } catch (Throwable t) {
             logger.warn("failed to create repository [{}][{}]", t, repositoryMetaData.type(), repositoryMetaData.name());
             throw new RepositoryException(repositoryMetaData.name(), "failed to create repository", t);
@@ -473,7 +473,7 @@ public class RepositoriesService extends AbstractComponent implements ClusterSta
         private final Repository repository;
         private final IndexShardRepository indexShardRepository;
 
-        public RepositoryHolder(String type, Settings settings, Injector injector, Repository repository, IndexShardRepository indexShardRepository) {
+        public RepositoryHolder(String type, Settings settings,Repository repository, IndexShardRepository indexShardRepository) {
             this.type = type;
             this.settings = settings;
             this.repository = repository;
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/AbstractCatAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/AbstractCatAction.java
index a40463e3e27..6c6a4592a9f 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/AbstractCatAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/AbstractCatAction.java
@@ -26,7 +26,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestChannel;
-import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestStatus;
 
@@ -38,7 +37,7 @@ import static org.elasticsearch.rest.action.support.RestTable.pad;
  */
 public abstract class AbstractCatAction extends BaseRestHandler {
 
-    public AbstractCatAction(Settings settings, RestController controller) {
+    public AbstractCatAction(Settings settings) {
         super(settings);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
index 9c68d0abbc7..deb11b13cd3 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestAliasAction.java
@@ -45,7 +45,7 @@ public class RestAliasAction extends AbstractCatAction {
 
     @Inject
     public RestAliasAction(Settings settings, RestController controller) {
-        super(settings, controller);
+        super(settings);
         controller.registerHandler(GET, "/_cat/aliases", this);
         controller.registerHandler(GET, "/_cat/aliases/{alias}", this);
     }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestAllocationAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestAllocationAction.java
index 146b52a63a0..33a5841480c 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestAllocationAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestAllocationAction.java
@@ -49,7 +49,7 @@ public class RestAllocationAction extends AbstractCatAction {
 
     @Inject
     public RestAllocationAction(Settings settings, RestController controller) {
-        super(settings, controller);
+        super(settings);
         controller.registerHandler(GET, "/_cat/allocation", this);
         controller.registerHandler(GET, "/_cat/allocation/{nodes}", this);
     }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java
index 032fd5c3aba..27d29ed63b6 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestCountAction.java
@@ -46,7 +46,7 @@ public class RestCountAction extends AbstractCatAction {
 
     @Inject
     public RestCountAction(Settings settings, RestController restController, RestController controller, IndicesQueriesRegistry indicesQueriesRegistry) {
-        super(settings, controller);
+        super(settings);
         restController.registerHandler(GET, "/_cat/count", this);
         restController.registerHandler(GET, "/_cat/count/{index}", this);
         this.indicesQueriesRegistry = indicesQueriesRegistry;
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestFielddataAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestFielddataAction.java
index d3458b40d77..36ce8247da8 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestFielddataAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestFielddataAction.java
@@ -44,7 +44,7 @@ public class RestFielddataAction extends AbstractCatAction {
 
     @Inject
     public RestFielddataAction(Settings settings, RestController controller) {
-        super(settings, controller);
+        super(settings);
         controller.registerHandler(GET, "/_cat/fielddata", this);
         controller.registerHandler(GET, "/_cat/fielddata/{fields}", this);
     }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestHealthAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestHealthAction.java
index 07f36cc2f6d..cd38e6591b6 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestHealthAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestHealthAction.java
@@ -40,7 +40,7 @@ public class RestHealthAction extends AbstractCatAction {
 
     @Inject
     public RestHealthAction(Settings settings, RestController controller) {
-        super(settings, controller);
+        super(settings);
         controller.registerHandler(GET, "/_cat/health", this);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
index fcdfe009e18..08d45e3311d 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
@@ -58,7 +58,7 @@ public class RestIndicesAction extends AbstractCatAction {
 
     @Inject
     public RestIndicesAction(Settings settings, RestController controller, IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, controller);
+        super(settings);
         this.indexNameExpressionResolver = indexNameExpressionResolver;
         controller.registerHandler(GET, "/_cat/indices", this);
         controller.registerHandler(GET, "/_cat/indices/{index}", this);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestMasterAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestMasterAction.java
index 164c2f79dac..5edd764fd10 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestMasterAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestMasterAction.java
@@ -40,7 +40,7 @@ public class RestMasterAction extends AbstractCatAction {
 
     @Inject
     public RestMasterAction(Settings settings, RestController controller) {
-        super(settings, controller);
+        super(settings);
         controller.registerHandler(GET, "/_cat/master", this);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java
index 815895a2679..e7d3b159189 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodeAttrsAction.java
@@ -48,7 +48,7 @@ public class RestNodeAttrsAction extends AbstractCatAction {
 
     @Inject
     public RestNodeAttrsAction(Settings settings, RestController controller) {
-        super(settings, controller);
+        super(settings);
         controller.registerHandler(GET, "/_cat/nodeattrs", this);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
index cb7bd59be3c..26536ab6003 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
@@ -73,7 +73,7 @@ public class RestNodesAction extends AbstractCatAction {
 
     @Inject
     public RestNodesAction(Settings settings, RestController controller) {
-        super(settings, controller);
+        super(settings);
         controller.registerHandler(GET, "/_cat/nodes", this);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java
index b85906e25b4..341a8bd5f82 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestPendingClusterTasksAction.java
@@ -38,7 +38,7 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 public class RestPendingClusterTasksAction extends AbstractCatAction {
     @Inject
     public RestPendingClusterTasksAction(Settings settings, RestController controller) {
-        super(settings, controller);
+        super(settings);
         controller.registerHandler(GET, "/_cat/pending_tasks", this);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java
index 13e1f2176d7..4850ff312c3 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestPluginsAction.java
@@ -45,7 +45,7 @@ public class RestPluginsAction extends AbstractCatAction {
 
     @Inject
     public RestPluginsAction(Settings settings, RestController controller) {
-        super(settings, controller);
+        super(settings);
         controller.registerHandler(GET, "/_cat/plugins", this);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java
index 6308ce7e689..a891bbc36f8 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRecoveryAction.java
@@ -52,7 +52,7 @@ public class RestRecoveryAction extends AbstractCatAction {
 
     @Inject
     public RestRecoveryAction(Settings settings, RestController restController, RestController controller) {
-        super(settings, controller);
+        super(settings);
         restController.registerHandler(GET, "/_cat/recovery", this);
         restController.registerHandler(GET, "/_cat/recovery/{index}", this);
     }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRepositoriesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRepositoriesAction.java
index ee53e1241c7..0b1580723b6 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestRepositoriesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestRepositoriesAction.java
@@ -41,7 +41,7 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 public class RestRepositoriesAction extends AbstractCatAction {
     @Inject
     public RestRepositoriesAction(Settings settings, RestController controller) {
-        super(settings, controller);
+        super(settings);
         controller.registerHandler(GET, "/_cat/repositories", this);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestSegmentsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestSegmentsAction.java
index 2540e194d84..95351e71352 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestSegmentsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestSegmentsAction.java
@@ -50,7 +50,7 @@ public class RestSegmentsAction extends AbstractCatAction {
 
     @Inject
     public RestSegmentsAction(Settings settings, RestController controller) {
-        super(settings, controller);
+        super(settings);
         controller.registerHandler(GET, "/_cat/segments", this);
         controller.registerHandler(GET, "/_cat/segments/{index}", this);
     }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
index 214979aef66..c98518ef17f 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
@@ -50,7 +50,7 @@ public class RestShardsAction extends AbstractCatAction {
 
     @Inject
     public RestShardsAction(Settings settings, RestController controller) {
-        super(settings, controller);
+        super(settings);
         controller.registerHandler(GET, "/_cat/shards", this);
         controller.registerHandler(GET, "/_cat/shards/{index}", this);
     }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java
index e503118fbab..025205174a8 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestSnapshotAction.java
@@ -48,7 +48,7 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 public class RestSnapshotAction extends AbstractCatAction {
     @Inject
     public RestSnapshotAction(Settings settings, RestController controller) {
-        super(settings, controller);
+        super(settings);
         controller.registerHandler(GET, "/_cat/snapshots/{repository}", this);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java
index 7486acfbb80..94116e30ad1 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestTasksAction.java
@@ -52,7 +52,7 @@ public class RestTasksAction extends AbstractCatAction {
 
     @Inject
     public RestTasksAction(Settings settings, RestController controller, ClusterService clusterService) {
-        super(settings, controller);
+        super(settings);
         controller.registerHandler(GET, "/_cat/tasks", this);
         this.clusterService = clusterService;
     }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java
index 8987fd27413..880995ec5da 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestThreadPoolAction.java
@@ -109,7 +109,7 @@ public class RestThreadPoolAction extends AbstractCatAction {
 
     @Inject
     public RestThreadPoolAction(Settings settings, RestController controller) {
-        super(settings, controller);
+        super(settings);
         controller.registerHandler(GET, "/_cat/thread_pool", this);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java
index 884adcf0d81..c3c4549aa63 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java
@@ -51,13 +51,13 @@ public class RestIndexAction extends BaseRestHandler {
         controller.registerHandler(POST, "/{index}/{type}", this); // auto id creation
         controller.registerHandler(PUT, "/{index}/{type}/{id}", this);
         controller.registerHandler(POST, "/{index}/{type}/{id}", this);
-        CreateHandler createHandler = new CreateHandler(settings, controller);
+        CreateHandler createHandler = new CreateHandler(settings);
         controller.registerHandler(PUT, "/{index}/{type}/{id}/_create", createHandler);
         controller.registerHandler(POST, "/{index}/{type}/{id}/_create", createHandler);
     }
 
     final class CreateHandler extends BaseRestHandler {
-        protected CreateHandler(Settings settings, RestController controller) {
+        protected CreateHandler(Settings settings) {
             super(settings);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java
index 7d632c06391..e42984aaf9a 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java
@@ -236,7 +236,7 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
         }
 
         ScriptEngineService scriptEngineService = getScriptEngineServiceForLang(lang);
-        if (canExecuteScript(lang, scriptEngineService, script.getType(), scriptContext) == false) {
+        if (canExecuteScript(lang, script.getType(), scriptContext) == false) {
             throw new IllegalStateException("scripts of type [" + script.getType() + "], operation [" + scriptContext.getKey() + "] and lang [" + lang + "] are disabled");
         }
 
@@ -357,7 +357,7 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
                     ScriptEngineService scriptEngineService = getScriptEngineServiceForLang(scriptLang);
                     //we don't know yet what the script will be used for, but if all of the operations for this lang with
                     //indexed scripts are disabled, it makes no sense to even compile it.
-                    if (isAnyScriptContextEnabled(scriptLang, scriptEngineService, ScriptType.STORED)) {
+                    if (isAnyScriptContextEnabled(scriptLang, ScriptType.STORED)) {
                         Object compiled = scriptEngineService.compile(id, template.getScript(), Collections.emptyMap());
                         if (compiled == null) {
                             throw new IllegalArgumentException("Unable to parse [" + template.getScript() +
@@ -466,16 +466,16 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
         return getScriptEngineServiceForLang(compiledScript.lang()).search(compiledScript, lookup, script.getParams());
     }
 
-    private boolean isAnyScriptContextEnabled(String lang, ScriptEngineService scriptEngineService, ScriptType scriptType) {
+    private boolean isAnyScriptContextEnabled(String lang, ScriptType scriptType) {
         for (ScriptContext scriptContext : scriptContextRegistry.scriptContexts()) {
-            if (canExecuteScript(lang, scriptEngineService, scriptType, scriptContext)) {
+            if (canExecuteScript(lang, scriptType, scriptContext)) {
                 return true;
             }
         }
         return false;
     }
 
-    private boolean canExecuteScript(String lang, ScriptEngineService scriptEngineService, ScriptType scriptType, ScriptContext scriptContext) {
+    private boolean canExecuteScript(String lang, ScriptType scriptType, ScriptContext scriptContext) {
         assert lang != null;
         if (scriptContextRegistry.isSupportedContext(scriptContext) == false) {
             throw new IllegalArgumentException("script context [" + scriptContext.getKey() + "] not supported");
@@ -556,7 +556,7 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
                 try {
                     //we don't know yet what the script will be used for, but if all of the operations for this lang
                     // with file scripts are disabled, it makes no sense to even compile it and cache it.
-                    if (isAnyScriptContextEnabled(engineService.getType(), engineService, ScriptType.FILE)) {
+                    if (isAnyScriptContextEnabled(engineService.getType(), ScriptType.FILE)) {
                         logger.info("compiling script file [{}]", file.toAbsolutePath());
                         try (InputStreamReader reader = new InputStreamReader(Files.newInputStream(file), StandardCharsets.UTF_8)) {
                             String script = Streams.copyToString(reader);
diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java
index 3c17a506eb9..bc77a01d072 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchService.java
@@ -19,19 +19,7 @@
 
 package org.elasticsearch.search;
 
-import static java.util.Collections.unmodifiableMap;
-import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
-import static org.elasticsearch.common.unit.TimeValue.timeValueMinutes;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.atomic.AtomicLong;
-
+import com.carrotsearch.hppc.ObjectFloatHashMap;
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.TopDocs;
 import org.elasticsearch.ExceptionsHelper;
@@ -57,8 +45,8 @@ import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
-import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.shard.IndexEventListener;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.SearchOperationListener;
@@ -101,7 +89,18 @@ import org.elasticsearch.search.sort.SortAndFormats;
 import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 
-import com.carrotsearch.hppc.ObjectFloatHashMap;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static java.util.Collections.unmodifiableMap;
+import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
+import static org.elasticsearch.common.unit.TimeValue.timeValueMinutes;
 
 /**
  *
@@ -844,20 +843,6 @@ public class SearchService extends AbstractLifecycleComponent<SearchService> imp
         }
     }
 
-    private void shortcutDocIdsToLoadForScanning(SearchContext context) {
-        TopDocs topDocs = context.queryResult().topDocs();
-        if (topDocs.scoreDocs.length == 0) {
-            // no more docs...
-            context.docIdsToLoad(EMPTY_DOC_IDS, 0, 0);
-            return;
-        }
-        int[] docIdsToLoad = new int[topDocs.scoreDocs.length];
-        for (int i = 0; i < docIdsToLoad.length; i++) {
-            docIdsToLoad[i] = topDocs.scoreDocs[i].doc;
-        }
-        context.docIdsToLoad(docIdsToLoad, 0, docIdsToLoad.length);
-    }
-
     private void processScroll(InternalScrollSearchRequest request, SearchContext context) {
         // process scroll
         context.from(context.from() + context.size());
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java
index 4dba4000da6..b7cab31974e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java
@@ -82,6 +82,7 @@ public class BestBucketsDeferringCollector extends DeferringBucketCollector {
     }
 
     /** Set the deferred collectors. */
+    @Override
     public void setDeferredCollector(Iterable<BucketCollector> deferredCollectors) {
         this.collector = BucketCollector.wrap(deferredCollectors);
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestDocsDeferringCollector.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestDocsDeferringCollector.java
index 0d5bb5ce50d..23369234676 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestDocsDeferringCollector.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestDocsDeferringCollector.java
@@ -79,6 +79,7 @@ public class BestDocsDeferringCollector extends DeferringBucketCollector impleme
     }
 
     /** Set the deferred collectors. */
+    @Override
     public void setDeferredCollector(Iterable<BucketCollector> deferredCollectors) {
         this.deferred = BucketCollector.wrap(deferredCollectors);
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/DeferringBucketCollector.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/DeferringBucketCollector.java
index 4f37ec3cf8a..7ecf7672bfd 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/DeferringBucketCollector.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/DeferringBucketCollector.java
@@ -34,18 +34,13 @@ import java.io.IOException;
  */
 public abstract class DeferringBucketCollector extends BucketCollector {
 
-    private BucketCollector collector;
     /** Sole constructor. */
     public DeferringBucketCollector() {}
 
     /** Set the deferred collectors. */
-    public void setDeferredCollector(Iterable<BucketCollector> deferredCollectors) {
-        this.collector = BucketCollector.wrap(deferredCollectors);
-    }
+    public abstract void setDeferredCollector(Iterable<BucketCollector> deferredCollectors);
 
-
-    public final void replay(long... selectedBuckets) throws IOException
-    {
+    public final void replay(long... selectedBuckets) throws IOException {
         prepareSelectedBuckets(selectedBuckets);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
index a6c1dc91b5d..bbdd85b969a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
@@ -260,7 +260,6 @@ public class InternalRange<B extends InternalRange.Bucket, R extends InternalRan
     }
 
     private List<B> ranges;
-    private Map<String, B> rangeMap;
     protected DocValueFormat format;
     protected boolean keyed;
 
@@ -333,7 +332,6 @@ public class InternalRange<B extends InternalRange.Bucket, R extends InternalRan
             ranges.add(getFactory().createBucket(key, in.readDouble(), in.readDouble(), in.readVLong(), InternalAggregations.readAggregations(in), keyed, format));
         }
         this.ranges = ranges;
-        this.rangeMap = null;
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java
index 44708eba409..e2d3034fa11 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregator.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.search.aggregations.metrics.scripted;
 
 import org.apache.lucene.index.LeafReaderContext;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.LeafSearchScript;
 import org.elasticsearch.script.Script;
@@ -34,6 +33,7 @@ import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.metrics.MetricsAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
+
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
@@ -52,7 +52,6 @@ public class ScriptedMetricAggregator extends MetricsAggregator {
         super(name, context, parent, pipelineAggregators, metaData);
         this.params = params;
         ScriptService scriptService = context.searchContext().scriptService();
-        ClusterState state = context.searchContext().getQueryShardContext().getClusterState();
         if (initScript != null) {
             scriptService.executable(initScript, ScriptContext.Standard.AGGS, Collections.emptyMap()).run();
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
index 5e53812e726..cb6f76037cf 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java
@@ -32,7 +32,6 @@ public class ValuesSourceConfig<VS extends ValuesSource> {
     private SearchScript script;
     private ValueType scriptValueType;
     private boolean unmapped = false;
-    private String formatPattern;
     private DocValueFormat format = DocValueFormat.RAW;
     private Object missing;
     private DateTimeZone timeZone;
diff --git a/core/src/main/java/org/elasticsearch/search/highlight/FastVectorHighlighter.java b/core/src/main/java/org/elasticsearch/search/highlight/FastVectorHighlighter.java
index 51c56e676c1..3d08019539c 100644
--- a/core/src/main/java/org/elasticsearch/search/highlight/FastVectorHighlighter.java
+++ b/core/src/main/java/org/elasticsearch/search/highlight/FastVectorHighlighter.java
@@ -110,7 +110,7 @@ public class FastVectorHighlighter implements Highlighter {
                     if (!forceSource && mapper.fieldType().stored()) {
                         fragmentsBuilder = new SimpleFragmentsBuilder(mapper, field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner);
                     } else {
-                        fragmentsBuilder = new SourceSimpleFragmentsBuilder(mapper, context, hitContext, field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner);
+                        fragmentsBuilder = new SourceSimpleFragmentsBuilder(mapper, context, field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner);
                     }
                 } else {
                     fragListBuilder = field.fieldOptions().fragmentOffset() == -1 ? new SimpleFragListBuilder() : new SimpleFragListBuilder(field.fieldOptions().fragmentOffset());
@@ -118,13 +118,13 @@ public class FastVectorHighlighter implements Highlighter {
                         if (!forceSource && mapper.fieldType().stored()) {
                             fragmentsBuilder = new ScoreOrderFragmentsBuilder(field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner);
                         } else {
-                            fragmentsBuilder = new SourceScoreOrderFragmentsBuilder(mapper, context, hitContext, field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner);
+                            fragmentsBuilder = new SourceScoreOrderFragmentsBuilder(mapper, context, field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner);
                         }
                     } else {
                         if (!forceSource && mapper.fieldType().stored()) {
                             fragmentsBuilder = new SimpleFragmentsBuilder(mapper, field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner);
                         } else {
-                            fragmentsBuilder = new SourceSimpleFragmentsBuilder(mapper, context, hitContext, field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner);
+                            fragmentsBuilder = new SourceSimpleFragmentsBuilder(mapper, context, field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner);
                         }
                     }
                 }
diff --git a/core/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceScoreOrderFragmentsBuilder.java b/core/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceScoreOrderFragmentsBuilder.java
index 8de58f062a7..1220a76c120 100644
--- a/core/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceScoreOrderFragmentsBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceScoreOrderFragmentsBuilder.java
@@ -27,7 +27,6 @@ import org.apache.lucene.search.vectorhighlight.BoundaryScanner;
 import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo;
 import org.apache.lucene.search.vectorhighlight.ScoreOrderFragmentsBuilder;
 import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.lookup.SourceLookup;
 
@@ -43,14 +42,11 @@ public class SourceScoreOrderFragmentsBuilder extends ScoreOrderFragmentsBuilder
 
     private final SearchContext searchContext;
 
-    private final FetchSubPhase.HitContext hitContext;
-
-    public SourceScoreOrderFragmentsBuilder(FieldMapper mapper, SearchContext searchContext,
-                                            FetchSubPhase.HitContext hitContext, String[] preTags, String[] postTags, BoundaryScanner boundaryScanner) {
+    public SourceScoreOrderFragmentsBuilder(FieldMapper mapper, SearchContext searchContext, String[] preTags, String[] postTags,
+                                            BoundaryScanner boundaryScanner) {
         super(preTags, postTags, boundaryScanner);
         this.mapper = mapper;
         this.searchContext = searchContext;
-        this.hitContext = hitContext;
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceSimpleFragmentsBuilder.java b/core/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceSimpleFragmentsBuilder.java
index b9fbd929f42..7ec6fe457de 100644
--- a/core/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceSimpleFragmentsBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceSimpleFragmentsBuilder.java
@@ -24,7 +24,6 @@ import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.vectorhighlight.BoundaryScanner;
 import org.elasticsearch.index.mapper.FieldMapper;
-import org.elasticsearch.search.fetch.FetchSubPhase;
 import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.search.lookup.SourceLookup;
 
@@ -38,13 +37,10 @@ public class SourceSimpleFragmentsBuilder extends SimpleFragmentsBuilder {
 
     private final SearchContext searchContext;
 
-    private final FetchSubPhase.HitContext hitContext;
-
-    public SourceSimpleFragmentsBuilder(FieldMapper mapper, SearchContext searchContext,
-                                        FetchSubPhase.HitContext hitContext, String[] preTags, String[] postTags, BoundaryScanner boundaryScanner) {
+    public SourceSimpleFragmentsBuilder(FieldMapper mapper, SearchContext searchContext, String[] preTags, String[] postTags,
+                                        BoundaryScanner boundaryScanner) {
         super(mapper, preTags, postTags, boundaryScanner);
         this.searchContext = searchContext;
-        this.hitContext = hitContext;
     }
 
     public static final Field[] EMPTY_FIELDS = new Field[0];
diff --git a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java
index 45124dc8b52..93013b94b36 100644
--- a/core/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java
+++ b/core/src/main/java/org/elasticsearch/search/internal/ShardSearchTransportRequest.java
@@ -157,11 +157,4 @@ public class ShardSearchTransportRequest extends TransportRequest implements Sha
     public void rewrite(QueryShardContext context) throws IOException {
         shardSearchLocalRequest.rewrite(context);
     }
-
-    private ShardSearchTransportRequest shallowCopy(ShardSearchLocalRequest rewritten) {
-        ShardSearchTransportRequest newRequest = new ShardSearchTransportRequest();
-        newRequest.originalIndices = originalIndices;
-        newRequest.shardSearchLocalRequest = rewritten;
-        return newRequest;
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java
index 2d10a5b7030..0c209e00a79 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestion.java
@@ -98,7 +98,6 @@ public final class CompletionSuggestion extends Suggest.Suggestion<CompletionSug
             // combine suggestion entries from participating shards on the coordinating node
             // the global top <code>size</code> entries are collected from the shard results
             // using a priority queue
-            Comparator<Suggest.Suggestion.Entry.Option> optionComparator = sortComparator();
             OptionPriorityQueue priorityQueue = new OptionPriorityQueue(size, sortComparator());
             for (Suggest.Suggestion<Entry> entries : toReduce) {
                 assert entries.getEntries().size() == 1 : "CompletionSuggestion must have only one entry";
diff --git a/core/src/main/java/org/joda/time/format/StrictISODateTimeFormat.java b/core/src/main/java/org/joda/time/format/StrictISODateTimeFormat.java
index 9768b9a8559..a417fa8f103 100644
--- a/core/src/main/java/org/joda/time/format/StrictISODateTimeFormat.java
+++ b/core/src/main/java/org/joda/time/format/StrictISODateTimeFormat.java
@@ -177,7 +177,7 @@ public class StrictISODateTimeFormat {
         if (workingFields.contains(DateTimeFieldType.monthOfYear())) {
             reducedPrec = dateByMonth(bld, workingFields, extended, strictISO);
         } else if (workingFields.contains(DateTimeFieldType.dayOfYear())) {
-            reducedPrec = dateByOrdinal(bld, workingFields, extended, strictISO);
+            reducedPrec = dateByOrdinal(bld, workingFields, extended);
         } else if (workingFields.contains(DateTimeFieldType.weekOfWeekyear())) {
             reducedPrec = dateByWeek(bld, workingFields, extended, strictISO);
         } else if (workingFields.contains(DateTimeFieldType.dayOfMonth())) {
@@ -288,14 +288,12 @@ public class StrictISODateTimeFormat {
      * @param bld  the builder
      * @param fields  the fields
      * @param extended  true to use extended format
-     * @param strictISO  true to only allow ISO formats
      * @since 1.1
      */
     private static boolean dateByOrdinal(
             DateTimeFormatterBuilder bld,
             Collection<DateTimeFieldType> fields,
-            boolean extended,
-            boolean strictISO) {
+            boolean extended) {
 
         boolean reducedPrec = false;
         if (fields.remove(DateTimeFieldType.year())) {
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
index 9b19a34b2ff..eff436d3a0e 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainIT.java
@@ -48,12 +48,9 @@ public final class ClusterAllocationExplainIT extends ESIntegTestCase {
 
         // Wait for all 3 nodes to be up
         logger.info("--> waiting for 3 nodes to be up");
-        assertBusy(new Runnable() {
-            @Override
-            public void run() {
-                NodesStatsResponse resp = client().admin().cluster().prepareNodesStats().get();
-                assertThat(resp.getNodes().size(), equalTo(3));
-            }
+        assertBusy(() -> {
+            NodesStatsResponse resp = client().admin().cluster().prepareNodesStats().get();
+            assertThat(resp.getNodes().size(), equalTo(3));
         });
 
         logger.info("--> creating 'test' index");
@@ -126,7 +123,6 @@ public final class ClusterAllocationExplainIT extends ESIntegTestCase {
 
         Map<DiscoveryNode, NodeExplanation> explanations = cae.getNodeExplanations();
 
-        Float noAttrWeight = -1f;
         Float barAttrWeight = -1f;
         Float fooBarAttrWeight = -1f;
         for (Map.Entry<DiscoveryNode, NodeExplanation> entry : explanations.entrySet()) {
@@ -134,7 +130,6 @@ public final class ClusterAllocationExplainIT extends ESIntegTestCase {
             String nodeName = node.getName();
             NodeExplanation explanation = entry.getValue();
             ClusterAllocationExplanation.FinalDecision finalDecision = explanation.getFinalDecision();
-            String finalExplanation = explanation.getFinalExplanation();
             ClusterAllocationExplanation.StoreCopy storeCopy = explanation.getStoreCopy();
             Decision d = explanation.getDecision();
             float weight = explanation.getWeight();
@@ -143,7 +138,6 @@ public final class ClusterAllocationExplainIT extends ESIntegTestCase {
             assertEquals(d.type(), Decision.Type.NO);
             if (noAttrNode.equals(nodeName)) {
                 assertThat(d.toString(), containsString("node does not match index include filters [foo:\"bar\"]"));
-                noAttrWeight = weight;
                 assertNull(storeStatus);
                 assertEquals("the shard cannot be assigned because one or more allocation decider returns a 'NO' decision",
                         explanation.getFinalExplanation());
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java
index d5cefc6d1f3..6c23d1604b8 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java
@@ -47,7 +47,6 @@ public final class ClusterAllocationExplainTests extends ESSingleNodeTestCase {
         NodeExplanation explanation = cae.getNodeExplanations().values().iterator().next();
         ClusterAllocationExplanation.FinalDecision fd = explanation.getFinalDecision();
         ClusterAllocationExplanation.StoreCopy storeCopy = explanation.getStoreCopy();
-        String finalExplanation = explanation.getFinalExplanation();
         Decision d = explanation.getDecision();
         assertNotNull("should have a decision", d);
         assertEquals(Decision.Type.NO, d.type());
@@ -76,7 +75,6 @@ public final class ClusterAllocationExplainTests extends ESSingleNodeTestCase {
         d = explanation.getDecision();
         fd = explanation.getFinalDecision();
         storeCopy = explanation.getStoreCopy();
-        finalExplanation = explanation.getFinalExplanation();
         assertNotNull("should have a decision", d);
         assertEquals(Decision.Type.NO, d.type());
         assertEquals(ClusterAllocationExplanation.FinalDecision.ALREADY_ASSIGNED, fd);
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
index bec8bd4d2c3..1561ab7a77c 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
@@ -41,7 +41,6 @@ import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -215,9 +214,7 @@ public final class ClusterAllocationExplanationTests extends ESTestCase {
         assertEquals(allocationDelay, cae2.getAllocationDelayMillis());
         assertEquals(remainingDelay, cae2.getRemainingDelayMillis());
         for (Map.Entry<DiscoveryNode, NodeExplanation> entry : cae2.getNodeExplanations().entrySet()) {
-            DiscoveryNode node = entry.getKey();
             NodeExplanation explanation = entry.getValue();
-            IndicesShardStoresResponse.StoreStatus status = explanation.getStoreStatus();
             assertNotNull(explanation.getStoreStatus());
             assertNotNull(explanation.getDecision());
             assertEquals(nodeWeight, explanation.getWeight());
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java
index b26d2531ff0..bb406366d25 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java
@@ -29,11 +29,11 @@ public class BulkShardRequestTests extends ESTestCase {
     public void testToString() {
         String index = randomSimpleString(random(), 10);
         int count = between(1, 100);
-        BulkShardRequest r = new BulkShardRequest(null, new ShardId(index, "ignored", 0), RefreshPolicy.NONE, new BulkItemRequest[count]);
+        BulkShardRequest r = new BulkShardRequest(new ShardId(index, "ignored", 0), RefreshPolicy.NONE, new BulkItemRequest[count]);
         assertEquals("BulkShardRequest to [" + index + "] containing [" + count + "] requests", r.toString());
-        r = new BulkShardRequest(null, new ShardId(index, "ignored", 0), RefreshPolicy.IMMEDIATE, new BulkItemRequest[count]);
+        r = new BulkShardRequest(new ShardId(index, "ignored", 0), RefreshPolicy.IMMEDIATE, new BulkItemRequest[count]);
         assertEquals("BulkShardRequest to [" + index + "] containing [" + count + "] requests and a refresh", r.toString());
-        r = new BulkShardRequest(null, new ShardId(index, "ignored", 0), RefreshPolicy.WAIT_UNTIL, new BulkItemRequest[count]);
+        r = new BulkShardRequest(new ShardId(index, "ignored", 0), RefreshPolicy.WAIT_UNTIL, new BulkItemRequest[count]);
         assertEquals("BulkShardRequest to [" + index + "] containing [" + count + "] requests blocking until refresh", r.toString());
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
index b524082718a..d0e26c22cfb 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
@@ -220,7 +220,6 @@ public class ClusterStateCreationUtils {
      * Creates a cluster state with no index
      */
     public static ClusterState stateWithNoShard() {
-        int numberOfNodes = 2;
         DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder();
         discoBuilder.localNodeId(newNode(0).getId());
         discoBuilder.masterNodeId(newNode(1).getId());
diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
index 12af9f8a2c2..6ba31fdb88e 100644
--- a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
+++ b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
@@ -39,7 +39,6 @@ import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.mapper.FieldMapper;
-import org.hamcrest.Matcher;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -55,7 +54,6 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -963,21 +961,6 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase {
         return randomBoolean() ? "test" : "alias";
     }
 
-    private Map<String, Integer> getFieldStatistics(Map<String, Object> stats, String fieldName) throws IOException {
-        return (Map<String, Integer>) ((Map<String, Object>) stats.get(fieldName)).get("field_statistics");
-    }
-
-    private Map<String, Integer> getTermStatistics(Map<String, Object> stats, String fieldName, String term) {
-        return (Map<String, Integer>) ((Map<String, Object>) ((Map<String, Object>) stats.get(fieldName)).get("terms")).get(term);
-    }
-
-    private Matcher<Integer> equalOrLessThanTo(Integer value, boolean isEqual) {
-        if (isEqual) {
-            return equalTo(value);
-        }
-        return lessThan(value);
-    }
-
     public void testTermVectorsWithVersion() {
         assertAcked(prepareCreate("test").addAlias(new Alias("alias"))
                 .setSettings(Settings.builder().put("index.refresh_interval", -1)));
diff --git a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java
index 597a2a4db39..d105a4bf63b 100644
--- a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java
@@ -135,7 +135,7 @@ public class UpdateRequestTests extends ESTestCase {
         TimeValue providedTTLValue = TimeValue.parseTimeValue(randomTimeValue(), null, "ttl");
         Settings settings = settings(Version.CURRENT).build();
 
-        UpdateHelper updateHelper = new UpdateHelper(settings, null, null);
+        UpdateHelper updateHelper = new UpdateHelper(settings, null);
 
         // We just upsert one document with ttl
         IndexRequest indexRequest = new IndexRequest("test", "type1", "1")
diff --git a/core/src/test/java/org/elasticsearch/blocks/SimpleBlocksIT.java b/core/src/test/java/org/elasticsearch/blocks/SimpleBlocksIT.java
index ffe82f9388d..699b919cf05 100644
--- a/core/src/test/java/org/elasticsearch/blocks/SimpleBlocksIT.java
+++ b/core/src/test/java/org/elasticsearch/blocks/SimpleBlocksIT.java
@@ -141,15 +141,6 @@ public class SimpleBlocksIT extends ESIntegTestCase {
         }
     }
 
-    private void canNotIndexExists(String index) {
-        try {
-            IndicesExistsResponse r = client().admin().indices().prepareExists(index).execute().actionGet();
-            fail();
-        } catch (ClusterBlockException e) {
-            // all is well
-        }
-    }
-
     private void setIndexReadOnly(String index, Object value) {
         HashMap<String, Object> newSettings = new HashMap<>();
         newSettings.put(IndexMetaData.SETTING_READ_ONLY, value);
diff --git a/core/src/test/java/org/elasticsearch/bwcompat/NodesStatsBasicBackwardsCompatIT.java b/core/src/test/java/org/elasticsearch/bwcompat/NodesStatsBasicBackwardsCompatIT.java
index c9d5f0b622e..4601c1bcfcf 100644
--- a/core/src/test/java/org/elasticsearch/bwcompat/NodesStatsBasicBackwardsCompatIT.java
+++ b/core/src/test/java/org/elasticsearch/bwcompat/NodesStatsBasicBackwardsCompatIT.java
@@ -22,7 +22,6 @@ package org.elasticsearch.bwcompat;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestBuilder;
-import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESBackcompatTestCase;
@@ -46,7 +45,7 @@ public class NodesStatsBasicBackwardsCompatIT extends ESBackcompatTestCase {
         for (NodeInfo n : nodesInfo.getNodes()) {
             TransportClient tc = TransportClient.builder().settings(settings).build().addTransportAddress(n.getNode().getAddress());
             // Just verify that the NS can be sent and serialized/deserialized between nodes with basic indices
-            NodesStatsResponse ns = tc.admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
+            tc.admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet();
             tc.close();
         }
     }
@@ -78,7 +77,7 @@ public class NodesStatsBasicBackwardsCompatIT extends ESBackcompatTestCase {
                     method.invoke(nsBuilder);
                 }
             }
-            NodesStatsResponse ns = nsBuilder.execute().actionGet();
+            nsBuilder.execute().actionGet();
             tc.close();
 
         }
diff --git a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
index 9ae0beadf59..245520d65f3 100644
--- a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
@@ -95,7 +95,7 @@ public class NetworkModuleTests extends ModuleTestCase {
 
     static class FakeCatRestHandler extends AbstractCatAction {
         public FakeCatRestHandler() {
-            super(null, null);
+            super(null);
         }
         @Override
         protected void doRequest(RestRequest request, RestChannel channel, NodeClient client) {}
diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java b/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
index 4f65c5fafdd..1b62f5d330a 100644
--- a/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.gateway;
 
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
@@ -40,7 +39,7 @@ public class GatewayServiceTests extends ESTestCase {
                 .put("http.enabled", "false")
                 .put("discovery.type", "local")
                 .put(settings.build()).build(),
-                null, clusterService, null, null, null, null, new NoopDiscovery(), null, null);
+                null, clusterService, null, null, null, new NoopDiscovery(), null, null);
     }
 
     public void testDefaultRecoverAfterTime() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
index 2769534aee0..91e9bb2c016 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java
@@ -193,7 +193,6 @@ public class IndexModuleTests extends ESTestCase {
         IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings);
         IndexModule module = new IndexModule(indexSettings, null,
                 new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap()));
-        Consumer<Settings> listener = (s) -> {};
         module.addIndexEventListener(eventListener);
         IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry,
             new IndicesFieldDataCache(settings, this.listener));
diff --git a/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java
index 97258b12a3b..22324e1ff2b 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java
@@ -77,7 +77,6 @@ public class IndexServiceTests extends ESSingleNodeTestCase {
 
     public void testFilteringAliases() throws Exception {
         IndexService indexService = createIndex("test", Settings.EMPTY);
-        IndexShard shard = indexService.getShard(0);
         add(indexService, "cats", filter(termQuery("animal", "cat")));
         add(indexService, "dogs", filter(termQuery("animal", "dog")));
         add(indexService, "all", null);
@@ -101,7 +100,6 @@ public class IndexServiceTests extends ESSingleNodeTestCase {
 
     public void testAliasFilters() throws Exception {
         IndexService indexService = createIndex("test", Settings.EMPTY);
-        IndexShard shard = indexService.getShard(0);
 
         add(indexService, "cats", filter(termQuery("animal", "cat")));
         add(indexService, "dogs", filter(termQuery("animal", "dog")));
@@ -118,7 +116,6 @@ public class IndexServiceTests extends ESSingleNodeTestCase {
 
     public void testRemovedAliasFilter() throws Exception {
         IndexService indexService = createIndex("test", Settings.EMPTY);
-        IndexShard shard = indexService.getShard(0);
 
         add(indexService, "cats", filter(termQuery("animal", "cat")));
         remove(indexService, "cats");
@@ -132,7 +129,6 @@ public class IndexServiceTests extends ESSingleNodeTestCase {
 
     public void testUnknownAliasFilter() throws Exception {
         IndexService indexService = createIndex("test", Settings.EMPTY);
-        IndexShard shard = indexService.getShard(0);
 
         add(indexService, "cats", filter(termQuery("animal", "cat")));
         add(indexService, "dogs", filter(termQuery("animal", "dog")));
diff --git a/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java b/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java
index 000d2509ea8..8e2b8f68963 100644
--- a/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java
+++ b/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java
@@ -45,8 +45,6 @@ public class SettingsListenerIT extends ESIntegTestCase {
 
     public static class SettingsListenerPlugin extends Plugin {
         private final SettingsTestingService service = new SettingsTestingService();
-        private static final Setting<Integer> SETTING = Setting.intSetting("index.test.new.setting", 0,
-            Property.Dynamic, Property.IndexScope);
 
         @Override
         public List<Setting<?>> getSettings() {
diff --git a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
index 1f1b758f349..995beb1742c 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
@@ -70,8 +70,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase {
             super(Settings.builder()
                             .put("indices.memory.interval", "200h") // disable it
                             .put(settings)
-                            .build(),
-                    null, null, 100 * 1024 * 1024); // fix jvm mem size to 100mb
+                            .build(), null, null);
         }
 
         public void deleteShard(IndexShard shard) {
diff --git a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java
index 75ae79d5f7d..8bcc0ca1fc8 100644
--- a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java
+++ b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java
@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.plugin.example;
 
-import org.elasticsearch.client.Client;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.inject.Inject;
@@ -40,7 +39,7 @@ public class ExampleCatAction extends AbstractCatAction {
 
     @Inject
     public ExampleCatAction(Settings settings, RestController controller, ExamplePluginConfiguration config) {
-        super(settings, controller);
+        super(settings);
         this.config = config;
         controller.registerHandler(GET, "/_cat/configured_example", this);
     }

From 5c7ca5cc0c734827b7c3decf30966e7e800a466d Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Fri, 1 Jul 2016 17:24:43 +0200
Subject: [PATCH 1189/1311] Fix checkstyle violations

---
 .../cluster/AbstractIndicesClusterStateServiceTestCase.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java b/core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java
index 69bee510710..4bf12bd9138 100644
--- a/core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java
+++ b/core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java
@@ -181,7 +181,8 @@ public abstract class AbstractIndicesClusterStateServiceTestCase extends ESTestC
         }
 
         @Override
-        public @Nullable MockIndexService indexService(Index index) {
+        @Nullable
+        public MockIndexService indexService(Index index) {
             return indices.get(index.getUUID());
         }
 

From b66fc308fc478bd374807358321b068c789cb061 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 1 Jul 2016 11:28:55 -0400
Subject: [PATCH 1190/1311] Add ingest-useragent to vagrant tests

---
 .../packaging/scripts/module_and_plugin_test_cases.bash   | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
index 21882c0ad53..eac0a80e4f4 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
@@ -228,6 +228,10 @@ fi
     install_and_check_plugin ingest geoip geoip2-*.jar jackson-annotations-*.jar jackson-databind-*.jar maxmind-db-*.jar
 }
 
+@test "[$GROUP] install ingest-useragent plugin" {
+    install_and_check_plugin ingest useragent
+}
+
 @test "[$GROUP] check ingest-common module" {
     check_module ingest-common jcodings-*.jar joni-*.jar
 }
@@ -357,6 +361,10 @@ fi
     remove_plugin ingest-geoip
 }
 
+@test "[$GROUP] remove ingest-useragent plugin" {
+    remove_plugin ingest-useragent
+}
+
 @test "[$GROUP] remove javascript plugin" {
     remove_plugin lang-javascript
 }

From 01d7020ee37fd71ac9b5e5f1d9e7c912d74415ce Mon Sep 17 00:00:00 2001
From: Tal Levy <JubBoy333@gmail.com>
Date: Fri, 1 Jul 2016 09:15:05 -0700
Subject: [PATCH 1191/1311] Skip the execution of an empty pipeline (#19200)

main optimization: `sourceToMap` is not called, therefore avoiding creation of Map of Maps
Closes #19192.
---
 .../ingest/PipelineExecutionService.java      |  4 ++++
 .../ingest/PipelineExecutionServiceTests.java | 23 +++++++++++++++++--
 .../ingest/PipelineFactoryTests.java          | 12 ++++++++++
 3 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java b/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java
index ef726e44843..e47c4db2340 100644
--- a/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java
+++ b/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java
@@ -143,6 +143,10 @@ public class PipelineExecutionService implements ClusterStateListener {
     }
 
     private void innerExecute(IndexRequest indexRequest, Pipeline pipeline) throws Exception {
+        if (pipeline.getProcessors().isEmpty()) {
+            return;
+        }
+
         long startTimeInNanos = System.nanoTime();
         // the pipeline specific stat holder may not exist and that is fine:
         // (e.g. the pipeline may have been removed while we're ingesting a document
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
index 8bf6f77a026..fcc6e04c6c1 100644
--- a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
@@ -134,8 +134,25 @@ public class PipelineExecutionServiceTests extends ESTestCase {
         verify(completionHandler, times(1)).accept(true);
     }
 
+    public void testExecuteEmptyPipeline() throws Exception {
+        CompoundProcessor processor = mock(CompoundProcessor.class);
+        when(store.get("_id")).thenReturn(new Pipeline("_id", "_description", processor));
+        when(processor.getProcessors()).thenReturn(Collections.emptyList());
+
+        IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id");
+        @SuppressWarnings("unchecked")
+        Consumer<Throwable> failureHandler = mock(Consumer.class);
+        @SuppressWarnings("unchecked")
+        Consumer<Boolean> completionHandler = mock(Consumer.class);
+        executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler);
+        verify(processor, never()).execute(any());
+        verify(failureHandler, never()).accept(any());
+        verify(completionHandler, times(1)).accept(true);
+    }
+
     public void testExecutePropagateAllMetaDataUpdates() throws Exception {
         CompoundProcessor processor = mock(CompoundProcessor.class);
+        when(processor.getProcessors()).thenReturn(Collections.singletonList(mock(Processor.class)));
         doAnswer((InvocationOnMock invocationOnMock) -> {
             IngestDocument ingestDocument = (IngestDocument) invocationOnMock.getArguments()[0];
             for (IngestDocument.MetaData metaData : IngestDocument.MetaData.values()) {
@@ -171,6 +188,7 @@ public class PipelineExecutionServiceTests extends ESTestCase {
 
     public void testExecuteFailure() throws Exception {
         CompoundProcessor processor = mock(CompoundProcessor.class);
+        when(processor.getProcessors()).thenReturn(Collections.singletonList(mock(Processor.class)));
         when(store.get("_id")).thenReturn(new Pipeline("_id", "_description", processor));
         IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id");
         doThrow(new RuntimeException()).when(processor).execute(eqID("_index", "_type", "_id", Collections.emptyMap()));
@@ -313,6 +331,7 @@ public class PipelineExecutionServiceTests extends ESTestCase {
         }
 
         CompoundProcessor processor = mock(CompoundProcessor.class);
+        when(processor.getProcessors()).thenReturn(Collections.singletonList(mock(Processor.class)));
         Exception error = new RuntimeException();
         doThrow(error).when(processor).execute(any());
         when(store.get(pipelineId)).thenReturn(new Pipeline(pipelineId, null, processor));
@@ -356,8 +375,8 @@ public class PipelineExecutionServiceTests extends ESTestCase {
         assertThat(ingestStats.getTotalStats().getIngestFailedCount(), equalTo(0L));
         assertThat(ingestStats.getTotalStats().getIngestTimeInMillis(), equalTo(0L));
 
-        when(store.get("_id1")).thenReturn(new Pipeline("_id1", null, new CompoundProcessor()));
-        when(store.get("_id2")).thenReturn(new Pipeline("_id2", null, new CompoundProcessor()));
+        when(store.get("_id1")).thenReturn(new Pipeline("_id1", null, new CompoundProcessor(mock(Processor.class))));
+        when(store.get("_id2")).thenReturn(new Pipeline("_id2", null, new CompoundProcessor(mock(Processor.class))));
 
         Map<String, PipelineConfiguration> configurationMap = new HashMap<>();
         configurationMap.put("_id1", new PipelineConfiguration("_id1", new BytesArray("{}")));
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java
index 8fe7ddd84cb..ced25419d92 100644
--- a/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java
@@ -31,6 +31,7 @@ import java.util.List;
 import java.util.Map;
 
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 import static org.mockito.Mockito.mock;
@@ -69,6 +70,17 @@ public class PipelineFactoryTests extends ESTestCase {
         }
     }
 
+    public void testCreateWithEmptyProcessorsField() throws Exception {
+        Map<String, Object> pipelineConfig = new HashMap<>();
+        pipelineConfig.put(Pipeline.DESCRIPTION_KEY, "_description");
+        pipelineConfig.put(Pipeline.PROCESSORS_KEY, Collections.emptyList());
+        Pipeline.Factory factory = new Pipeline.Factory();
+        Pipeline pipeline = factory.create("_id", pipelineConfig, null);
+        assertThat(pipeline.getId(), equalTo("_id"));
+        assertThat(pipeline.getDescription(), equalTo("_description"));
+        assertThat(pipeline.getProcessors(), is(empty()));
+    }
+
     public void testCreateWithPipelineOnFailure() throws Exception {
         Map<String, Object> processorConfig = new HashMap<>();
         Map<String, Object> pipelineConfig = new HashMap<>();

From 76ba10bab67c432b3ea3fddbaa897b0d3dad11f2 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Fri, 1 Jul 2016 09:31:41 -0700
Subject: [PATCH 1192/1311] Remove unnecessary optional injection of
 ScriptService into NodeService

---
 .../org/elasticsearch/node/service/NodeService.java    | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/node/service/NodeService.java b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
index 79fbb24a177..103ee73a1e9 100644
--- a/core/src/main/java/org/elasticsearch/node/service/NodeService.java
+++ b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
@@ -74,7 +74,7 @@ public class NodeService extends AbstractComponent implements Closeable {
     @Inject
     public NodeService(Settings settings, ThreadPool threadPool, MonitorService monitorService,
                        Discovery discovery, TransportService transportService, IndicesService indicesService,
-                       PluginsService pluginService, CircuitBreakerService circuitBreakerService,
+                       PluginsService pluginService, CircuitBreakerService circuitBreakerService, ScriptService scriptService,
                        IngestService ingestService, ClusterService clusterService, SettingsFilter settingsFilter) {
         super(settings);
         this.threadPool = threadPool;
@@ -87,17 +87,11 @@ public class NodeService extends AbstractComponent implements Closeable {
         this.clusterService = clusterService;
         this.ingestService = ingestService;
         this.settingsFilter = settingsFilter;
+        this.scriptService = scriptService;
         clusterService.add(ingestService.getPipelineStore());
         clusterService.add(ingestService.getPipelineExecutionService());
     }
 
-    // can not use constructor injection or there will be a circular dependency
-    // nocommit: try removing this...
-    @Inject(optional = true)
-    public void setScriptService(ScriptService scriptService) {
-        this.scriptService = scriptService;
-    }
-
     public void setHttpServer(@Nullable HttpServer httpServer) {
         this.httpServer = httpServer;
     }

From dbbfadeefaffa71c46a7640437524779be611bcb Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Fri, 1 Jul 2016 09:36:44 -0700
Subject: [PATCH 1193/1311] Internal: Remove cyclic dependency between
 HttpServer and NodeService

NodeService has an "service attributes" map, which is only
set by HttpServer on start/stop. But the only thing it puts in this map
is already available as part of the HttpServer info which is added to
node info requests. This change removes the attributes map and removes
the dependency in HttpServer on NodeService.
---
 .../admin/cluster/node/info/NodeInfo.java     | 38 ++++---------
 .../cluster/node/info/NodesInfoResponse.java  |  6 --
 .../org/elasticsearch/http/HttpServer.java    |  8 +--
 .../node/service/NodeService.java             | 56 ++++---------------
 .../nodesinfo/NodeInfoStreamingTests.java     |  8 +--
 5 files changed, 26 insertions(+), 90 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
index 64292cb9064..685ab703fbc 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
@@ -46,8 +46,6 @@ import static java.util.Collections.unmodifiableMap;
  * Node information (static, does not change over time).
  */
 public class NodeInfo extends BaseNodeResponse {
-    @Nullable
-    private Map<String, String> serviceAttributes;
 
     private Version version;
     private Build build;
@@ -85,14 +83,13 @@ public class NodeInfo extends BaseNodeResponse {
     public NodeInfo() {
     }
 
-    public NodeInfo(Version version, Build build, DiscoveryNode node, @Nullable Map<String, String> serviceAttributes, @Nullable Settings settings,
+    public NodeInfo(Version version, Build build, DiscoveryNode node, @Nullable Settings settings,
                     @Nullable OsInfo os, @Nullable ProcessInfo process, @Nullable JvmInfo jvm, @Nullable ThreadPoolInfo threadPool,
                     @Nullable TransportInfo transport, @Nullable HttpInfo http, @Nullable PluginsAndModules plugins, @Nullable IngestInfo ingest,
                     @Nullable ByteSizeValue totalIndexingBuffer) {
         super(node);
         this.version = version;
         this.build = build;
-        this.serviceAttributes = serviceAttributes;
         this.settings = settings;
         this.os = os;
         this.process = process;
@@ -127,14 +124,6 @@ public class NodeInfo extends BaseNodeResponse {
         return this.build;
     }
 
-    /**
-     * The service attributes of the node.
-     */
-    @Nullable
-    public Map<String, String> getServiceAttributes() {
-        return this.serviceAttributes;
-    }
-
     /**
      * The settings of the node.
      */
@@ -213,13 +202,15 @@ public class NodeInfo extends BaseNodeResponse {
         } else {
             totalIndexingBuffer = null;
         }
-        if (in.readBoolean()) {
-            Map<String, String> builder = new HashMap<>();
-            int size = in.readVInt();
-            for (int i = 0; i < size; i++) {
-                builder.put(in.readString(), in.readString());
+        if (version.onOrBefore(Version.V_5_0_0_alpha4)) {
+            // service attributes were removed
+            if (in.readBoolean()) {
+                int size = in.readVInt();
+                for (int i = 0; i < size; i++) {
+                    in.readString(); // key
+                    in.readString(); // value
+                }
             }
-            serviceAttributes = unmodifiableMap(builder);
         }
         if (in.readBoolean()) {
             settings = Settings.readSettingsFromStream(in);
@@ -262,15 +253,8 @@ public class NodeInfo extends BaseNodeResponse {
             out.writeBoolean(true);
             out.writeLong(totalIndexingBuffer.bytes());
         }
-        if (getServiceAttributes() == null) {
-            out.writeBoolean(false);
-        } else {
-            out.writeBoolean(true);
-            out.writeVInt(serviceAttributes.size());
-            for (Map.Entry<String, String> entry : serviceAttributes.entrySet()) {
-                out.writeString(entry.getKey());
-                out.writeString(entry.getValue());
-            }
+        if (version.onOrBefore(Version.V_5_0_0_alpha4)) {
+            out.writeBoolean(false); // service attributes removed
         }
         if (settings == null) {
             out.writeBoolean(false);
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java
index a9dc0ff2e45..2d273bef2c0 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoResponse.java
@@ -73,12 +73,6 @@ public class NodesInfoResponse extends BaseNodesResponse<NodeInfo> implements To
                 builder.byteSizeField("total_indexing_buffer", "total_indexing_buffer_in_bytes", nodeInfo.getTotalIndexingBuffer());
             }
 
-            if (nodeInfo.getServiceAttributes() != null) {
-                for (Map.Entry<String, String> nodeAttribute : nodeInfo.getServiceAttributes().entrySet()) {
-                    builder.field(nodeAttribute.getKey(), nodeAttribute.getValue());
-                }
-            }
-
             builder.startArray("roles");
             for (DiscoveryNode.Role role : nodeInfo.getNode().getRoles()) {
                 builder.value(role.getRoleName());
diff --git a/core/src/main/java/org/elasticsearch/http/HttpServer.java b/core/src/main/java/org/elasticsearch/http/HttpServer.java
index f44b839b2a3..63bcd761eb3 100644
--- a/core/src/main/java/org/elasticsearch/http/HttpServer.java
+++ b/core/src/main/java/org/elasticsearch/http/HttpServer.java
@@ -56,22 +56,18 @@ public class HttpServer extends AbstractLifecycleComponent<HttpServer> implement
 
     private final RestController restController;
 
-    private final NodeService nodeService;
-
     private final NodeClient client;
 
     private final CircuitBreakerService circuitBreakerService;
 
     @Inject
-    public HttpServer(Settings settings, HttpServerTransport transport, RestController restController, NodeService nodeService,
+    public HttpServer(Settings settings, HttpServerTransport transport, RestController restController,
                       NodeClient client, CircuitBreakerService circuitBreakerService) {
         super(settings);
         this.transport = transport;
         this.restController = restController;
-        this.nodeService = nodeService;
         this.client = client;
         this.circuitBreakerService = circuitBreakerService;
-        nodeService.setHttpServer(this);
         transport.httpServerAdapter(this);
     }
 
@@ -82,12 +78,10 @@ public class HttpServer extends AbstractLifecycleComponent<HttpServer> implement
         if (logger.isInfoEnabled()) {
             logger.info("{}", transport.boundAddress());
         }
-        nodeService.putAttribute("http_address", transport.boundAddress().publishAddress().toString());
     }
 
     @Override
     protected void doStop() {
-        nodeService.removeAttribute("http_address");
         transport.stop();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/node/service/NodeService.java b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
index 0920a84bfd6..f3743a46d1e 100644
--- a/core/src/main/java/org/elasticsearch/node/service/NodeService.java
+++ b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
@@ -19,13 +19,15 @@
 
 package org.elasticsearch.node.service;
 
+import java.io.Closeable;
+import java.io.IOException;
+
 import org.elasticsearch.Build;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -42,14 +44,6 @@ import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.unmodifiableMap;
-
 /**
  */
 public class NodeService extends AbstractComponent implements Closeable {
@@ -64,18 +58,14 @@ public class NodeService extends AbstractComponent implements Closeable {
     private final SettingsFilter settingsFilter;
     private ClusterService clusterService;
     private ScriptService scriptService;
-
-    @Nullable
-    private HttpServer httpServer;
-
-    private volatile Map<String, String> serviceAttributes = emptyMap();
+    private final HttpServer httpServer;
 
     private final Discovery discovery;
 
     @Inject
     public NodeService(Settings settings, ThreadPool threadPool, MonitorService monitorService,
                        Discovery discovery, TransportService transportService, IndicesService indicesService,
-                       PluginsService pluginService, CircuitBreakerService circuitBreakerService,
+                       PluginsService pluginService, CircuitBreakerService circuitBreakerService, HttpServer httpServer,
                        ProcessorsRegistry.Builder processorsRegistryBuilder, ClusterService clusterService, SettingsFilter settingsFilter) {
         super(settings);
         this.threadPool = threadPool;
@@ -85,6 +75,7 @@ public class NodeService extends AbstractComponent implements Closeable {
         this.discovery = discovery;
         this.pluginService = pluginService;
         this.circuitBreakerService = circuitBreakerService;
+        this.httpServer = httpServer;
         this.clusterService = clusterService;
         this.ingestService = new IngestService(settings, threadPool, processorsRegistryBuilder);
         this.settingsFilter = settingsFilter;
@@ -99,38 +90,15 @@ public class NodeService extends AbstractComponent implements Closeable {
         this.ingestService.buildProcessorsFactoryRegistry(scriptService, clusterService);
     }
 
-    public void setHttpServer(@Nullable HttpServer httpServer) {
-        this.httpServer = httpServer;
-    }
-
-    public synchronized void putAttribute(String key, String value) {
-        Map<String, String> newServiceAttributes = new HashMap<>(serviceAttributes);
-        newServiceAttributes.put(key, value);
-        serviceAttributes = unmodifiableMap(newServiceAttributes);
-    }
-
-    public synchronized void removeAttribute(String key) {
-        Map<String, String> newServiceAttributes = new HashMap<>(serviceAttributes);
-        newServiceAttributes.remove(key);
-        serviceAttributes = unmodifiableMap(newServiceAttributes);
-    }
-
-    /**
-     * Attributes different services in the node can add to be reported as part of the node info (for example).
-     */
-    public Map<String, String> attributes() {
-        return this.serviceAttributes;
-    }
-
     public NodeInfo info() {
-        return new NodeInfo(Version.CURRENT, Build.CURRENT, discovery.localNode(), serviceAttributes,
+        return new NodeInfo(Version.CURRENT, Build.CURRENT, discovery.localNode(),
                 settings,
                 monitorService.osService().info(),
                 monitorService.processService().info(),
                 monitorService.jvmService().info(),
                 threadPool.info(),
                 transportService.info(),
-                httpServer == null ? null : httpServer.info(),
+                httpServer.info(),
                 pluginService == null ? null : pluginService.info(),
                 ingestService == null ? null : ingestService.info(),
                 indicesService.getTotalIndexingBufferBytes()
@@ -139,14 +107,14 @@ public class NodeService extends AbstractComponent implements Closeable {
 
     public NodeInfo info(boolean settings, boolean os, boolean process, boolean jvm, boolean threadPool,
                 boolean transport, boolean http, boolean plugin, boolean ingest, boolean indices) {
-        return new NodeInfo(Version.CURRENT, Build.CURRENT, discovery.localNode(), serviceAttributes,
+        return new NodeInfo(Version.CURRENT, Build.CURRENT, discovery.localNode(),
                 settings ? settingsFilter.filter(this.settings) : null,
                 os ? monitorService.osService().info() : null,
                 process ? monitorService.processService().info() : null,
                 jvm ? monitorService.jvmService().info() : null,
                 threadPool ? this.threadPool.info() : null,
                 transport ? transportService.info() : null,
-                http ? (httpServer == null ? null : httpServer.info()) : null,
+                http ? httpServer.info() : null,
                 plugin ? (pluginService == null ? null : pluginService.info()) : null,
                 ingest ? (ingestService == null ? null : ingestService.info()) : null,
                 indices ? indicesService.getTotalIndexingBufferBytes() : null
@@ -164,7 +132,7 @@ public class NodeService extends AbstractComponent implements Closeable {
                 threadPool.stats(),
                 monitorService.fsService().stats(),
                 transportService.stats(),
-                httpServer == null ? null : httpServer.stats(),
+                httpServer.stats(),
                 circuitBreakerService.stats(),
                 scriptService.stats(),
                 discovery.stats(),
@@ -185,7 +153,7 @@ public class NodeService extends AbstractComponent implements Closeable {
                 threadPool ? this.threadPool.stats() : null,
                 fs ? monitorService.fsService().stats() : null,
                 transport ? transportService.stats() : null,
-                http ? (httpServer == null ? null : httpServer.stats()) : null,
+                http ? httpServer.stats() : null,
                 circuitBreaker ? circuitBreakerService.stats() : null,
                 script ? scriptService.stats() : null,
                 discoveryStats ? discovery.stats() : null,
diff --git a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
index ea82a17f7b3..739dcd8b2c6 100644
--- a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
+++ b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
@@ -81,11 +81,6 @@ public class NodeInfoStreamingTests extends ESTestCase {
         assertThat(nodeInfo.getBuild().toString(), equalTo(readNodeInfo.getBuild().toString()));
         assertThat(nodeInfo.getHostname(), equalTo(readNodeInfo.getHostname()));
         assertThat(nodeInfo.getVersion(), equalTo(readNodeInfo.getVersion()));
-        assertThat(nodeInfo.getServiceAttributes().size(), equalTo(readNodeInfo.getServiceAttributes().size()));
-        for (Map.Entry<String, String> entry : nodeInfo.getServiceAttributes().entrySet()) {
-            assertNotNull(readNodeInfo.getServiceAttributes().get(entry.getKey()));
-            assertThat(readNodeInfo.getServiceAttributes().get(entry.getKey()), equalTo(entry.getValue()));
-        }
         compareJsonOutput(nodeInfo.getHttp(), readNodeInfo.getHttp());
         compareJsonOutput(nodeInfo.getJvm(), readNodeInfo.getJvm());
         compareJsonOutput(nodeInfo.getProcess(), readNodeInfo.getProcess());
@@ -149,6 +144,7 @@ public class NodeInfoStreamingTests extends ESTestCase {
             // pick a random long that sometimes exceeds an int:
             indexingBuffer = new ByteSizeValue(random().nextLong() & ((1L<<40)-1));
         }
-        return new NodeInfo(VersionUtils.randomVersion(random()), build, node, serviceAttributes, settings, osInfo, process, jvm, threadPoolInfo, transport, htttpInfo, plugins, ingestInfo, indexingBuffer);
+        return new NodeInfo(VersionUtils.randomVersion(random()), build, node, settings, osInfo, process, jvm,
+            threadPoolInfo, transport, htttpInfo, plugins, ingestInfo, indexingBuffer);
     }
 }

From a206c7a14988c7967b312be04403186a605bf257 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Fri, 1 Jul 2016 10:16:56 -0700
Subject: [PATCH 1194/1311] Remove unnecessary transport level bwc

---
 .../action/admin/cluster/node/info/NodeInfo.java    | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
index 685ab703fbc..d7ce899792f 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodeInfo.java
@@ -202,16 +202,6 @@ public class NodeInfo extends BaseNodeResponse {
         } else {
             totalIndexingBuffer = null;
         }
-        if (version.onOrBefore(Version.V_5_0_0_alpha4)) {
-            // service attributes were removed
-            if (in.readBoolean()) {
-                int size = in.readVInt();
-                for (int i = 0; i < size; i++) {
-                    in.readString(); // key
-                    in.readString(); // value
-                }
-            }
-        }
         if (in.readBoolean()) {
             settings = Settings.readSettingsFromStream(in);
         }
@@ -253,9 +243,6 @@ public class NodeInfo extends BaseNodeResponse {
             out.writeBoolean(true);
             out.writeLong(totalIndexingBuffer.bytes());
         }
-        if (version.onOrBefore(Version.V_5_0_0_alpha4)) {
-            out.writeBoolean(false); // service attributes removed
-        }
         if (settings == null) {
             out.writeBoolean(false);
         } else {

From ff42d7cfc69bc985fc56133efba2fea6bb8413b2 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 1 Jul 2016 12:18:47 -0400
Subject: [PATCH 1195/1311] Add embedded stash key support to rest tests

This allowes embedding stash keys in string like `t${key}est`. This
allows simple string concatenation like acitons.

The test for this is in `ObjectPathTests` because `Stash` doesn't seem
to have a test on its own and it is simple enough to test embedded
stashes this way. And this is a way I expect them to be used eventually.
---
 .../elasticsearch/test/rest/ObjectPath.java   |  2 +-
 .../test/rest/RestTestExecutionContext.java   |  2 +-
 .../org/elasticsearch/test/rest/Stash.java    | 39 ++++++++++++++++---
 .../test/rest/section/Assertion.java          |  4 +-
 .../test/rest/support/Features.java           |  2 +-
 .../test/rest/test/ObjectPathTests.java       | 30 +++++++++++++-
 6 files changed, 68 insertions(+), 11 deletions(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java
index 77de0f30f0c..8c492d279b0 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java
@@ -70,7 +70,7 @@ public class ObjectPath {
 
     @SuppressWarnings("unchecked")
     private Object evaluate(String key, Object object, Stash stash) throws IOException {
-        if (stash.isStashedValue(key)) {
+        if (stash.containsStashedValue(key)) {
             key = stash.getValue(key).toString();
         }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java
index 6d95c7e893f..d7295e1dca7 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java
@@ -66,7 +66,7 @@ public class RestTestExecutionContext implements Closeable {
         //makes a copy of the parameters before modifying them for this specific request
         HashMap<String, String> requestParams = new HashMap<>(params);
         for (Map.Entry<String, String> entry : requestParams.entrySet()) {
-            if (stash.isStashedValue(entry.getValue())) {
+            if (stash.containsStashedValue(entry.getValue())) {
                 entry.setValue(stash.getValue(entry.getValue()).toString());
             }
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java b/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
index 13c5b9ff563..575f6f337ba 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
@@ -29,12 +29,15 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * Allows to cache the last obtained test response and or part of it within variables
  * that can be used as input values in following requests and assertions.
  */
 public class Stash implements ToXContent {
+    private static final Pattern EXTENDED_KEY = Pattern.compile("\\$\\{([^}]+)\\}");
 
     private static final ESLogger logger = Loggers.getLogger(Stash.class);
 
@@ -67,12 +70,18 @@ public class Stash implements ToXContent {
      * The stash contains fields eventually extracted from previous responses that can be reused
      * as arguments for following requests (e.g. scroll_id)
      */
-    public boolean isStashedValue(Object key) {
+    public boolean containsStashedValue(Object key) {
         if (key == null) {
             return false;
         }
         String stashKey = key.toString();
-        return Strings.hasLength(stashKey) && stashKey.startsWith("$");
+        if (false == Strings.hasLength(stashKey)) {
+            return false;
+        }
+        if (stashKey.startsWith("$")) {
+            return true;
+        }
+        return EXTENDED_KEY.matcher(stashKey).find();
     }
 
     /**
@@ -81,7 +90,27 @@ public class Stash implements ToXContent {
      * as arguments for following requests (e.g. scroll_id)
      */
     public Object getValue(String key) throws IOException {
-        Object stashedValue = stashObjectPath.evaluate(key.substring(1));
+        if (key.charAt(0) == '$' && key.charAt(1) != '{') {
+            return unstash(key.substring(1));
+        }
+        Matcher matcher = EXTENDED_KEY.matcher(key);
+        /*
+         * String*Buffer* because that is what the Matcher API takes. In modern versions of java the uncontended synchronization is very,
+         * very cheap so that should be a problem.
+         */
+        StringBuffer result = new StringBuffer(key.length());
+        if (false == matcher.find()) {
+            throw new IllegalArgumentException("Doesn't contain any stash keys [" + key + "]");
+        }
+        do {
+            matcher.appendReplacement(result, Matcher.quoteReplacement(unstash(matcher.group(1)).toString()));
+        } while (matcher.find());
+        matcher.appendTail(result);
+        return result.toString();
+    }
+
+    private Object unstash(String key) throws IOException {
+        Object stashedValue = stashObjectPath.evaluate(key);
         if (stashedValue == null) {
             throw new IllegalArgumentException("stashed value not found for key [" + key + "]");
         }
@@ -104,7 +133,7 @@ public class Stash implements ToXContent {
             List list = (List) obj;
             for (int i = 0; i < list.size(); i++) {
                 Object o = list.get(i);
-                if (isStashedValue(o)) {
+                if (containsStashedValue(o)) {
                     list.set(i, getValue(o.toString()));
                 } else {
                     unstashObject(o);
@@ -114,7 +143,7 @@ public class Stash implements ToXContent {
         if (obj instanceof Map) {
             Map<String, Object> map = (Map) obj;
             for (Map.Entry<String, Object> entry : map.entrySet()) {
-                if (isStashedValue(entry.getValue())) {
+                if (containsStashedValue(entry.getValue())) {
                     entry.setValue(getValue(entry.getValue().toString()));
                 } else {
                     unstashObject(entry.getValue());
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/Assertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/Assertion.java
index de38940169f..fbba9de163b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/Assertion.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/Assertion.java
@@ -51,14 +51,14 @@ public abstract class Assertion implements ExecutableSection {
             return executionContext.stash().replaceStashedValues(map);
         }
 
-        if (executionContext.stash().isStashedValue(expectedValue)) {
+        if (executionContext.stash().containsStashedValue(expectedValue)) {
             return executionContext.stash().getValue(expectedValue.toString());
         }
         return expectedValue;
     }
 
     protected final Object getActualValue(RestTestExecutionContext executionContext) throws IOException {
-        if (executionContext.stash().isStashedValue(field)) {
+        if (executionContext.stash().containsStashedValue(field)) {
             return executionContext.stash().getValue(field);
         }
         return executionContext.response(field);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java b/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java
index 0f51f72e8e5..cf33570ef28 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java
@@ -34,7 +34,7 @@ import java.util.List;
  */
 public final class Features {
 
-    private static final List<String> SUPPORTED = Arrays.asList("stash_in_path", "groovy_scripting", "headers");
+    private static final List<String> SUPPORTED = Arrays.asList("stash_in_path", "groovy_scripting", "headers", "embedded_stash_key");
 
     private Features() {
 
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java
index 4a21fefe803..1d99a73c767 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java
@@ -190,21 +190,49 @@ public class ObjectPathTests extends ESTestCase {
             objectPath.evaluate("field1.$placeholder.element1");
             fail("evaluate should have failed due to unresolved placeholder");
         } catch(IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("stashed value not found for key [$placeholder]"));
+            assertThat(e.getMessage(), containsString("stashed value not found for key [placeholder]"));
         }
 
+        // Stashed value is whole property name
         Stash stash = new Stash();
         stash.stashValue("placeholder", "elements");
         Object object = objectPath.evaluate("field1.$placeholder.element1", stash);
         assertThat(object, notNullValue());
         assertThat(object.toString(), equalTo("value1"));
 
+        // Stash key has dots
         Map<String, Object> stashedObject = new HashMap<>();
         stashedObject.put("subobject", "elements");
         stash.stashValue("object", stashedObject);
         object = objectPath.evaluate("field1.$object\\.subobject.element1", stash);
         assertThat(object, notNullValue());
         assertThat(object.toString(), equalTo("value1"));
+
+        // Stashed value is part of property name
+        stash.stashValue("placeholder", "ele");
+        object = objectPath.evaluate("field1.${placeholder}ments.element1", stash);
+        assertThat(object, notNullValue());
+        assertThat(object.toString(), equalTo("value1"));
+
+        // Stashed value is inside of property name
+        stash.stashValue("placeholder", "le");
+        object = objectPath.evaluate("field1.e${placeholder}ments.element1", stash);
+        assertThat(object, notNullValue());
+        assertThat(object.toString(), equalTo("value1"));
+
+        // Multiple stashed values in property name
+        stash.stashValue("placeholder", "le");
+        stash.stashValue("placeholder2", "nts");
+        object = objectPath.evaluate("field1.e${placeholder}me${placeholder2}.element1", stash);
+        assertThat(object, notNullValue());
+        assertThat(object.toString(), equalTo("value1"));
+
+        // Stashed value is part of property name and has dots
+        stashedObject.put("subobject", "ele");
+        stash.stashValue("object", stashedObject);
+        object = objectPath.evaluate("field1.${object\\.subobject}ments.element1", stash);
+        assertThat(object, notNullValue());
+        assertThat(object.toString(), equalTo("value1"));
     }
 
     @SuppressWarnings("unchecked")

From cb207764391b25a1a9236a7be15c26f7972dde3b Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Fri, 1 Jul 2016 01:46:05 -0400
Subject: [PATCH 1196/1311] Includes the index UUID in the _cat/indices API and
 adds tests for the _cat/indices functionality.

Closes #19204
Closes #19132
---
 .../rest/action/cat/RestIndicesAction.java    |  19 +-
 .../indices/stats/IndicesStatsTests.java      |  11 ++
 .../action/cat/RestIndicesActionTests.java    | 162 ++++++++++++++++++
 .../test/cat.indices/10_basic.yaml            |   2 +
 4 files changed, 187 insertions(+), 7 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java

diff --git a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
index 08d45e3311d..e24c9d13aa6 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
@@ -38,6 +38,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.Table;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
@@ -84,7 +85,7 @@ public class RestIndicesAction extends AbstractCatAction {
             @Override
             public void processResponse(final ClusterStateResponse clusterStateResponse) {
                 final ClusterState state = clusterStateResponse.getState();
-                final String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, strictExpandIndicesOptions, indices);
+                final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, strictExpandIndicesOptions, indices);
                 // concreteIndices should contain exactly the indices in state.metaData() that were selected by clusterStateRequest using
                 // IndicesOptions.strictExpand(). We select the indices again here so that they can be displayed in the resulting table
                 // in the requesting order.
@@ -129,6 +130,7 @@ public class RestIndicesAction extends AbstractCatAction {
         table.addCell("health", "alias:h;desc:current health status");
         table.addCell("status", "alias:s;desc:open/close status");
         table.addCell("index", "alias:i,idx;desc:index name");
+        table.addCell("uuid", "alias:id,uuid;desc:index uuid");
         table.addCell("pri", "alias:p,shards.primary,shardsPrimary;text-align:right;desc:number of primary shards");
         table.addCell("rep", "alias:r,shards.replica,shardsReplica;text-align:right;desc:number of replica shards");
         table.addCell("docs.count", "alias:dc,docsCount;text-align:right;desc:available docs");
@@ -312,19 +314,22 @@ public class RestIndicesAction extends AbstractCatAction {
         return table;
     }
 
-    private Table buildTable(RestRequest request, String[] indices, ClusterHealthResponse health, IndicesStatsResponse stats, MetaData indexMetaDatas) {
+    // package private for testing
+    Table buildTable(RestRequest request, Index[] indices, ClusterHealthResponse health, IndicesStatsResponse stats, MetaData indexMetaDatas) {
         Table table = getTableWithHeader(request);
 
-        for (String index : indices) {
-            ClusterIndexHealth indexHealth = health.getIndices().get(index);
-            IndexStats indexStats = stats.getIndices().get(index);
-            IndexMetaData indexMetaData = indexMetaDatas.getIndices().get(index);
+        for (final Index index : indices) {
+            final String indexName = index.getName();
+            ClusterIndexHealth indexHealth = health.getIndices().get(indexName);
+            IndexStats indexStats = stats.getIndices().get(indexName);
+            IndexMetaData indexMetaData = indexMetaDatas.getIndices().get(indexName);
             IndexMetaData.State state = indexMetaData.getState();
 
             table.startRow();
             table.addCell(state == IndexMetaData.State.OPEN ? (indexHealth == null ? "red*" : indexHealth.getStatus().toString().toLowerCase(Locale.ROOT)) : null);
             table.addCell(state.toString().toLowerCase(Locale.ROOT));
-            table.addCell(index);
+            table.addCell(indexName);
+            table.addCell(index.getUUID());
             table.addCell(indexHealth == null ? null : indexHealth.getNumberOfShards());
             table.addCell(indexHealth == null ? null : indexHealth.getNumberOfReplicas());
             table.addCell(indexStats == null ? null : indexStats.getPrimaries().getDocs().getCount());
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java
index 726dccee597..dfc10169e70 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.admin.indices.stats;
 
+import org.elasticsearch.action.ShardOperationFailedException;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.engine.CommitStats;
@@ -26,6 +27,8 @@ import org.elasticsearch.index.engine.SegmentsStats;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 
+import java.util.List;
+
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasKey;
@@ -108,4 +111,12 @@ public class IndicesStatsTests extends ESSingleNodeTestCase {
         }
     }
 
+    /**
+     * Gives access to package private IndicesStatsResponse constructor for test purpose.
+     **/
+    public static IndicesStatsResponse newIndicesStatsResponse(ShardStats[] shards, int totalShards, int successfulShards,
+                                                               int failedShards, List<ShardOperationFailedException> shardFailures) {
+        return new IndicesStatsResponse(shards, totalShards, successfulShards, failedShards, shardFailures);
+    }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java
new file mode 100644
index 00000000000..db2148b9f61
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.action.cat;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.elasticsearch.action.admin.indices.stats.CommonStats;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsTests;
+import org.elasticsearch.action.admin.indices.stats.ShardStats;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.routing.ShardRouting;
+import org.elasticsearch.cluster.routing.UnassignedInfo;
+import org.elasticsearch.common.Table;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.cache.query.QueryCacheStats;
+import org.elasticsearch.index.cache.request.RequestCacheStats;
+import org.elasticsearch.index.engine.SegmentsStats;
+import org.elasticsearch.index.fielddata.FieldDataStats;
+import org.elasticsearch.index.flush.FlushStats;
+import org.elasticsearch.index.get.GetStats;
+import org.elasticsearch.index.merge.MergeStats;
+import org.elasticsearch.index.refresh.RefreshStats;
+import org.elasticsearch.index.search.stats.SearchStats;
+import org.elasticsearch.index.shard.DocsStats;
+import org.elasticsearch.index.shard.IndexingStats;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.index.shard.ShardPath;
+import org.elasticsearch.index.store.StoreStats;
+import org.elasticsearch.index.warmer.WarmerStats;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.search.suggest.completion.CompletionStats;
+import org.elasticsearch.test.ESTestCase;
+
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+import static java.util.Collections.emptyList;
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * Tests for {@link RestIndicesAction}
+ */
+public class RestIndicesActionTests extends ESTestCase {
+
+    public void testBuildTable() {
+        final Settings settings = Settings.EMPTY;
+        final RestController restController = new RestController(settings);
+        final RestIndicesAction action = new RestIndicesAction(settings, restController, new IndexNameExpressionResolver(settings));
+
+        // build a (semi-)random table
+        final int numIndices = randomIntBetween(0, 5);
+        Index[] indices = new Index[numIndices];
+        for (int i = 0; i < numIndices; i++) {
+            indices[i] = new Index(randomAsciiOfLength(5), UUIDs.randomBase64UUID());
+        }
+
+        final MetaData.Builder metaDataBuilder = MetaData.builder();
+        for (final Index index : indices) {
+            metaDataBuilder.put(IndexMetaData.builder(index.getName())
+                                    .settings(Settings.builder()
+                                                  .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
+                                                  .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID()))
+                                    .creationDate(System.currentTimeMillis())
+                                    .numberOfShards(1)
+                                    .numberOfReplicas(1)
+                                    .state(IndexMetaData.State.OPEN));
+        }
+        final MetaData metaData = metaDataBuilder.build();
+
+        final ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+                                              .metaData(metaData)
+                                              .build();
+        final String[] indicesStr = new String[indices.length];
+        for (int i = 0; i < indices.length; i++) {
+            indicesStr[i] = indices[i].getName();
+        }
+        final ClusterHealthResponse clusterHealth = new ClusterHealthResponse(
+            clusterState.getClusterName().value(), indicesStr, clusterState, 0, 0, 0, TimeValue.timeValueMillis(1000L)
+        );
+
+        final Table table = action.buildTable(null, indices, clusterHealth, randomIndicesStatsResponse(indices), metaData);
+
+        // now, verify the table is correct
+        int count = 0;
+        List<Table.Cell> headers = table.getHeaders();
+        assertThat(headers.get(count++).value, equalTo("health"));
+        assertThat(headers.get(count++).value, equalTo("status"));
+        assertThat(headers.get(count++).value, equalTo("index"));
+        assertThat(headers.get(count++).value, equalTo("uuid"));
+
+        List<List<Table.Cell>> rows = table.getRows();
+        assertThat(rows.size(), equalTo(indices.length));
+        // TODO: more to verify (e.g. randomize cluster health, num primaries, num replicas, etc)
+        for (int i = 0; i < rows.size(); i++) {
+            count = 0;
+            final List<Table.Cell> row = rows.get(i);
+            assertThat(row.get(count++).value, equalTo("red*")); // all are red because cluster state doesn't have routing entries
+            assertThat(row.get(count++).value, equalTo("open")); // all are OPEN for now
+            assertThat(row.get(count++).value, equalTo(indices[i].getName()));
+            assertThat(row.get(count++).value, equalTo(indices[i].getUUID()));
+        }
+    }
+
+    private IndicesStatsResponse randomIndicesStatsResponse(final Index[] indices) {
+        List<ShardStats> shardStats = new ArrayList<>();
+        for (final Index index : indices) {
+            for (int i = 0; i < 2; i++) {
+                ShardId shardId = new ShardId(index, i);
+                Path path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve(String.valueOf(i));
+                ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, null, i == 0,
+                    new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null));
+                shardRouting = shardRouting.initialize("node-0", null, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
+                shardRouting = shardRouting.moveToStarted();
+                CommonStats stats = new CommonStats();
+                stats.fieldData = new FieldDataStats();
+                stats.queryCache = new QueryCacheStats();
+                stats.docs = new DocsStats();
+                stats.store = new StoreStats();
+                stats.indexing = new IndexingStats();
+                stats.search = new SearchStats();
+                stats.segments = new SegmentsStats();
+                stats.merge = new MergeStats();
+                stats.refresh = new RefreshStats();
+                stats.completion = new CompletionStats();
+                stats.requestCache = new RequestCacheStats();
+                stats.get = new GetStats();
+                stats.flush = new FlushStats();
+                stats.warmer = new WarmerStats();
+                shardStats.add(new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null));
+            }
+        }
+        return IndicesStatsTests.newIndicesStatsResponse(
+            shardStats.toArray(new ShardStats[shardStats.size()]), shardStats.size(), shardStats.size(), 0, emptyList()
+        );
+    }
+}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yaml
index 51f8fe9ed4c..1e485fc6d30 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yaml
@@ -26,6 +26,7 @@
                /^(green  \s+
                   open   \s+
                   index1 \s+
+                  ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
                   1      \s+
                   0      \s+
                   0      \s+
@@ -62,6 +63,7 @@
                /^(       \s+
                   close  \s+
                   index1 \s+
+                  ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+
                          \s+
                          \s+
                          \s+

From f30a70c51f09fba861bcf94d7ddd0f2a35805dd1 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 1 Jul 2016 14:14:08 -0400
Subject: [PATCH 1197/1311] Fix comment

I forgot a word....
---
 .../src/main/java/org/elasticsearch/test/rest/Stash.java        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java b/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
index 575f6f337ba..a4218f83340 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
@@ -96,7 +96,7 @@ public class Stash implements ToXContent {
         Matcher matcher = EXTENDED_KEY.matcher(key);
         /*
          * String*Buffer* because that is what the Matcher API takes. In modern versions of java the uncontended synchronization is very,
-         * very cheap so that should be a problem.
+         * very cheap so that should not be a problem.
          */
         StringBuffer result = new StringBuffer(key.length());
         if (false == matcher.find()) {

From de71a9abb3c410c58c76e9fd03ebbfbd6fe6cc25 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 30 Jun 2016 15:54:09 -0400
Subject: [PATCH 1198/1311] Migrate value_count, percentiles, and
 percentile_ranks aggregations to NamedWriteable

These are the first aggregations with multiple `InternalAggregation`s
backing the same `AggregationBuilder`. This required a change in the
register method's signature.
---
 .../elasticsearch/search/SearchModule.java    | 101 ++++++++++++------
 .../PercentileRanksAggregationBuilder.java    |   9 +-
 .../PercentilesAggregationBuilder.java        |   9 +-
 .../hdr/AbstractInternalHDRPercentiles.java   |  75 ++++++-------
 .../hdr/InternalHDRPercentileRanks.java       |  40 +++----
 .../hdr/InternalHDRPercentiles.java           |  40 +++----
 .../AbstractInternalTDigestPercentiles.java   |  49 ++++-----
 .../InternalTDigestPercentileRanks.java       |  39 +++----
 .../tdigest/InternalTDigestPercentiles.java   |  40 +++----
 .../valuecount/InternalValueCount.java        |  54 ++++------
 .../ValueCountAggregationBuilder.java         |   8 +-
 11 files changed, 214 insertions(+), 250 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java
index c39848c347f..ea3348e87fa 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -27,6 +27,7 @@ import org.elasticsearch.common.inject.AbstractModule;
 import org.elasticsearch.common.io.stream.NamedWriteable;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.io.stream.Writeable.Reader;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ParseFieldRegistry;
@@ -93,6 +94,7 @@ import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.search.action.SearchTransportService;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.Aggregator.Parser;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
@@ -259,8 +261,10 @@ import org.elasticsearch.search.suggest.Suggester;
 import org.elasticsearch.search.suggest.Suggesters;
 
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.TreeMap;
 
 /**
  *
@@ -419,21 +423,54 @@ public class SearchModule extends AbstractModule {
 
     /**
      * Register an aggregation.
-     *
-     * @param builderReader reads the {@link AggregationBuilder} from a stream
-     * @param internalReader reads the {@link InternalAggregation} from a stream
-     * @param aggregationParser reads the aggregation builder from XContent
-     * @param aggregationName names by which the aggregation may be parsed. The first name is special because it is the name that the reader
-     *        is registered under.
      */
-    public void registerAggregation(Writeable.Reader<? extends AggregationBuilder> builderReader,
-            Writeable.Reader<? extends InternalAggregation> internalReader, Aggregator.Parser aggregationParser,
-            ParseField aggregationName) {
+    public void registerAggregation(AggregationSpec spec) {
         if (false == transportClient) {
-            namedWriteableRegistry.register(AggregationBuilder.class, aggregationName.getPreferredName(), builderReader);
-            aggregationParserRegistry.register(aggregationParser, aggregationName);
+            namedWriteableRegistry.register(AggregationBuilder.class, spec.aggregationName.getPreferredName(), spec.builderReader);
+            aggregationParserRegistry.register(spec.aggregationParser, spec.aggregationName);
+        }
+        for (Map.Entry<String, Writeable.Reader<? extends InternalAggregation>> t : spec.internalReaders.entrySet()) {
+            String writeableName = t.getKey();
+            Writeable.Reader<? extends InternalAggregation> internalReader = t.getValue();
+            namedWriteableRegistry.register(InternalAggregation.class, writeableName, internalReader);
+        }
+    }
+
+    public static class AggregationSpec {
+        private final Map<String, Writeable.Reader<? extends InternalAggregation>> internalReaders = new TreeMap<>();
+        private final Writeable.Reader<? extends AggregationBuilder> builderReader;
+        private final Aggregator.Parser aggregationParser;
+        private final ParseField aggregationName;
+
+        /**
+         * Register an aggregation.
+         *
+         * @param builderReader reads the {@link AggregationBuilder} from a stream
+         * @param aggregationParser reads the aggregation builder from XContent
+         * @param aggregationName names by which the aggregation may be parsed. The first name is special because it is the name that the
+         *          reader is registered under.
+         */
+        public AggregationSpec(Reader<? extends AggregationBuilder> builderReader, Parser aggregationParser, ParseField aggregationName) {
+            this.builderReader = builderReader;
+            this.aggregationParser = aggregationParser;
+            this.aggregationName = aggregationName;
+        }
+
+        /**
+         * Add a reader for the shard level results of the aggregation with {@linkplain aggregationName}'s
+         * {@link ParseField#getPreferredName()} as the {@link NamedWriteable#getWriteableName()}.
+         */
+        public AggregationSpec addResultReader(Writeable.Reader<? extends InternalAggregation> resultReader) {
+            return addResultReader(aggregationName.getPreferredName(), resultReader);
+        }
+
+        /**
+         * Add a reader for the shard level results of the aggregation.
+         */
+        public AggregationSpec addResultReader(String writeableName, Writeable.Reader<? extends InternalAggregation> resultReader) {
+            internalReaders.put(writeableName, resultReader);
+            return this;
         }
-        namedWriteableRegistry.register(InternalAggregation.class, aggregationName.getPreferredName(), internalReader);
     }
 
     public void registerAggregation(Writeable.Reader<? extends AggregationBuilder> builderReader, Aggregator.Parser aggregationParser,
@@ -488,19 +525,28 @@ public class SearchModule extends AbstractModule {
     }
 
     private void registerBuiltinAggregations() {
-        registerAggregation(AvgAggregationBuilder::new, InternalAvg::new, new AvgParser(), AvgAggregationBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(SumAggregationBuilder::new, InternalSum::new, new SumParser(), SumAggregationBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(MinAggregationBuilder::new, InternalMin::new, new MinParser(), MinAggregationBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(MaxAggregationBuilder::new, InternalMax::new, new MaxParser(), MaxAggregationBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(StatsAggregationBuilder::new, InternalStats::new, new StatsParser(),
-                StatsAggregationBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(ExtendedStatsAggregationBuilder::new, InternalExtendedStats::new, new ExtendedStatsParser(),
-                ExtendedStatsAggregationBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(ValueCountAggregationBuilder::new, new ValueCountParser(), ValueCountAggregationBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(PercentilesAggregationBuilder::new, new PercentilesParser(),
-                PercentilesAggregationBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(PercentileRanksAggregationBuilder::new, new PercentileRanksParser(),
-                PercentileRanksAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(new AggregationSpec(AvgAggregationBuilder::new, new AvgParser(), AvgAggregationBuilder.AGGREGATION_NAME_FIELD)
+                .addResultReader(InternalAvg::new));
+        registerAggregation(new AggregationSpec(SumAggregationBuilder::new, new SumParser(), SumAggregationBuilder.AGGREGATION_NAME_FIELD)
+                .addResultReader(InternalSum::new));
+        registerAggregation(new AggregationSpec(MinAggregationBuilder::new, new MinParser(), MinAggregationBuilder.AGGREGATION_NAME_FIELD)
+                .addResultReader(InternalMin::new));
+        registerAggregation(new AggregationSpec(MaxAggregationBuilder::new, new MaxParser(), MaxAggregationBuilder.AGGREGATION_NAME_FIELD)
+                .addResultReader(InternalMax::new));
+        registerAggregation(new AggregationSpec(StatsAggregationBuilder::new, new StatsParser(),
+                StatsAggregationBuilder.AGGREGATION_NAME_FIELD).addResultReader(InternalStats::new));
+        registerAggregation(new AggregationSpec(ExtendedStatsAggregationBuilder::new, new ExtendedStatsParser(),
+                ExtendedStatsAggregationBuilder.AGGREGATION_NAME_FIELD).addResultReader(InternalExtendedStats::new));
+        registerAggregation(new AggregationSpec(ValueCountAggregationBuilder::new, new ValueCountParser(),
+                ValueCountAggregationBuilder.AGGREGATION_NAME_FIELD).addResultReader(InternalValueCount::new));
+        registerAggregation(new AggregationSpec(PercentilesAggregationBuilder::new, new PercentilesParser(),
+                PercentilesAggregationBuilder.AGGREGATION_NAME_FIELD)
+                    .addResultReader(InternalTDigestPercentiles.NAME, InternalTDigestPercentiles::new)
+                    .addResultReader(InternalHDRPercentiles.NAME, InternalHDRPercentiles::new));
+        registerAggregation(new AggregationSpec(PercentileRanksAggregationBuilder::new, new PercentileRanksParser(),
+                PercentileRanksAggregationBuilder.AGGREGATION_NAME_FIELD)
+                    .addResultReader(InternalTDigestPercentileRanks.NAME, InternalTDigestPercentileRanks::new)
+                    .addResultReader(InternalHDRPercentileRanks.NAME, InternalHDRPercentileRanks::new));
         registerAggregation(CardinalityAggregationBuilder::new, new CardinalityParser(),
                 CardinalityAggregationBuilder.AGGREGATION_NAME_FIELD);
         registerAggregation(GlobalAggregationBuilder::new, GlobalAggregationBuilder::parse,
@@ -721,11 +767,6 @@ public class SearchModule extends AbstractModule {
 
     static {
         // calcs
-        InternalValueCount.registerStreams();
-        InternalTDigestPercentiles.registerStreams();
-        InternalTDigestPercentileRanks.registerStreams();
-        InternalHDRPercentiles.registerStreams();
-        InternalHDRPercentileRanks.registerStreams();
         InternalCardinality.registerStreams();
         InternalScriptedMetric.registerStreams();
         InternalGeoCentroid.registerStreams();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksAggregationBuilder.java
index d36dcdecb7b..576406ad2c1 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanksAggregationBuilder.java
@@ -25,8 +25,8 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.HDRPercentileRanksAggregatorFactory;
-import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentileRanks;
 import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.TDigestPercentileRanksAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
@@ -42,7 +42,8 @@ import java.util.Arrays;
 import java.util.Objects;
 
 public class PercentileRanksAggregationBuilder extends LeafOnly<ValuesSource.Numeric, PercentileRanksAggregationBuilder> {
-    public static final String NAME = InternalTDigestPercentileRanks.TYPE.name();
+    public static final String NAME = PercentileRanks.TYPE_NAME;
+    public static final Type TYPE = new Type(NAME);
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     private double[] values;
@@ -52,14 +53,14 @@ public class PercentileRanksAggregationBuilder extends LeafOnly<ValuesSource.Num
     private boolean keyed = true;
 
     public PercentileRanksAggregationBuilder(String name) {
-        super(name, InternalTDigestPercentileRanks.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public PercentileRanksAggregationBuilder(StreamInput in) throws IOException {
-        super(in, InternalTDigestPercentileRanks.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
         values = in.readDoubleArray();
         keyed = in.readBoolean();
         numberOfSignificantValueDigits = in.readVInt();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesAggregationBuilder.java
index d2d1e8d9f8a..ca7a877c373 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesAggregationBuilder.java
@@ -25,8 +25,8 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.metrics.percentiles.hdr.HDRPercentilesAggregatorFactory;
-import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.InternalTDigestPercentiles;
 import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.TDigestPercentilesAggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
@@ -42,7 +42,8 @@ import java.util.Arrays;
 import java.util.Objects;
 
 public class PercentilesAggregationBuilder extends LeafOnly<ValuesSource.Numeric, PercentilesAggregationBuilder> {
-    public static final String NAME = InternalTDigestPercentiles.TYPE.name();
+    public static final String NAME = Percentiles.TYPE_NAME;
+    public static final Type TYPE = new Type(NAME);
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     private double[] percents = PercentilesParser.DEFAULT_PERCENTS;
@@ -52,14 +53,14 @@ public class PercentilesAggregationBuilder extends LeafOnly<ValuesSource.Numeric
     private boolean keyed = true;
 
     public PercentilesAggregationBuilder(String name) {
-        super(name, InternalTDigestPercentiles.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(name, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
     }
 
     /**
      * Read from a stream.
      */
     public PercentilesAggregationBuilder(StreamInput in) throws IOException {
-        super(in, InternalTDigestPercentiles.TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
+        super(in, TYPE, ValuesSourceType.NUMERIC, ValueType.NUMERIC);
         percents = in.readDoubleArray();
         keyed = in.readBoolean();
         numberOfSignificantValueDigits = in.readVInt();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractInternalHDRPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractInternalHDRPercentiles.java
index 7d4ea18afcd..ba76f8b9863 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractInternalHDRPercentiles.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractInternalHDRPercentiles.java
@@ -36,11 +36,9 @@ import java.util.zip.DataFormatException;
 
 abstract class AbstractInternalHDRPercentiles extends InternalNumericMetricsAggregation.MultiValue {
 
-    protected double[] keys;
-    protected DoubleHistogram state;
-    private boolean keyed;
-
-    AbstractInternalHDRPercentiles() {} // for serialization
+    protected final double[] keys;
+    protected final DoubleHistogram state;
+    private final boolean keyed;
 
     public AbstractInternalHDRPercentiles(String name, double[] keys, DoubleHistogram state, boolean keyed, DocValueFormat format,
             List<PipelineAggregator> pipelineAggregators,
@@ -52,6 +50,38 @@ abstract class AbstractInternalHDRPercentiles extends InternalNumericMetricsAggr
         this.format = format;
     }
 
+    /**
+     * Read from a stream.
+     */
+    protected AbstractInternalHDRPercentiles(StreamInput in) throws IOException {
+        super(in);
+        format = in.readNamedWriteable(DocValueFormat.class);
+        keys = in.readDoubleArray();
+        long minBarForHighestToLowestValueRatio = in.readLong();
+        final int serializedLen = in.readVInt();
+        byte[] bytes = new byte[serializedLen];
+        in.readBytes(bytes, 0, serializedLen);
+        ByteBuffer stateBuffer = ByteBuffer.wrap(bytes);
+        try {
+            state = DoubleHistogram.decodeFromCompressedByteBuffer(stateBuffer, minBarForHighestToLowestValueRatio);
+        } catch (DataFormatException e) {
+            throw new IOException("Failed to decode DoubleHistogram for aggregation [" + name + "]", e);
+        }
+        keyed = in.readBoolean();
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(format);
+        out.writeDoubleArray(keys);
+        out.writeLong(state.getHighestToLowestValueRatio());
+        ByteBuffer stateBuffer = ByteBuffer.allocate(state.getNeededByteBufferCapacity());
+        final int serializedLen = state.encodeIntoCompressedByteBuffer(stateBuffer);
+        out.writeVInt(serializedLen);
+        out.writeBytes(stateBuffer.array(), 0, serializedLen);
+        out.writeBoolean(keyed);
+    }
+
     @Override
     public double value(String name) {
         return value(Double.parseDouble(name));
@@ -80,41 +110,6 @@ abstract class AbstractInternalHDRPercentiles extends InternalNumericMetricsAggr
     protected abstract AbstractInternalHDRPercentiles createReduced(String name, double[] keys, DoubleHistogram merged, boolean keyed,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData);
 
-    @Override
-    protected void doReadFrom(StreamInput in) throws IOException {
-        format = in.readNamedWriteable(DocValueFormat.class);
-        keys = new double[in.readInt()];
-        for (int i = 0; i < keys.length; ++i) {
-            keys[i] = in.readDouble();
-        }
-        long minBarForHighestToLowestValueRatio = in.readLong();
-        final int serializedLen = in.readVInt();
-        byte[] bytes = new byte[serializedLen];
-        in.readBytes(bytes, 0, serializedLen);
-        ByteBuffer stateBuffer = ByteBuffer.wrap(bytes);
-        try {
-            state = DoubleHistogram.decodeFromCompressedByteBuffer(stateBuffer, minBarForHighestToLowestValueRatio);
-        } catch (DataFormatException e) {
-            throw new IOException("Failed to decode DoubleHistogram for aggregation [" + name + "]", e);
-        }
-        keyed = in.readBoolean();
-    }
-
-    @Override
-    protected void doWriteTo(StreamOutput out) throws IOException {
-        out.writeNamedWriteable(format);
-        out.writeInt(keys.length);
-        for (int i = 0 ; i < keys.length; ++i) {
-            out.writeDouble(keys[i]);
-        }
-        out.writeLong(state.getHighestToLowestValueRatio());
-        ByteBuffer stateBuffer = ByteBuffer.allocate(state.getNeededByteBufferCapacity());
-        final int serializedLen = state.encodeIntoCompressedByteBuffer(stateBuffer);
-        out.writeVInt(serializedLen);
-        out.writeBytes(stateBuffer.array(), 0, serializedLen);
-        out.writeBoolean(keyed);
-    }
-
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         if (keyed) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentileRanks.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentileRanks.java
index 0bd6b0322a0..35234d73acc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentileRanks.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentileRanks.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.aggregations.metrics.percentiles.hdr;
 import org.HdrHistogram.DoubleHistogram;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentile;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
 import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanks;
@@ -32,34 +31,26 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-/**
-*
-*/
 public class InternalHDRPercentileRanks extends AbstractInternalHDRPercentiles implements PercentileRanks {
-
-    public static final Type TYPE = new Type(PercentileRanks.TYPE_NAME, "hdr_percentile_ranks");
-
-    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public InternalHDRPercentileRanks readResult(StreamInput in) throws IOException {
-            InternalHDRPercentileRanks result = new InternalHDRPercentileRanks();
-            result.readFrom(in);
-            return result;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
-    InternalHDRPercentileRanks() {
-    } // for serialization
+    public static final String NAME = "hdr_percentile_ranks";
 
     public InternalHDRPercentileRanks(String name, double[] cdfValues, DoubleHistogram state, boolean keyed, DocValueFormat formatter,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
         super(name, cdfValues, state, keyed, formatter, pipelineAggregators, metaData);
     }
 
+    /**
+     * Read from a stream.
+     */
+    public InternalHDRPercentileRanks(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
     @Override
     public Iterator<Percentile> iterator() {
         return new Iter(keys, state);
@@ -86,11 +77,6 @@ public class InternalHDRPercentileRanks extends AbstractInternalHDRPercentiles i
         return new InternalHDRPercentileRanks(name, keys, merged, keyed, format, pipelineAggregators, metaData);
     }
 
-    @Override
-    public Type type() {
-        return TYPE;
-    }
-
     static double percentileRank(DoubleHistogram state, double value) {
         if (state.getTotalCount() == 0) {
             return Double.NaN;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentiles.java
index 62fe2cdc951..579f25c1666 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentiles.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/InternalHDRPercentiles.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.aggregations.metrics.percentiles.hdr;
 import org.HdrHistogram.DoubleHistogram;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentile;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles;
@@ -32,34 +31,26 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-/**
-*
-*/
 public class InternalHDRPercentiles extends AbstractInternalHDRPercentiles implements Percentiles {
-
-    public static final Type TYPE = new Type(Percentiles.TYPE_NAME, "hdr_percentiles");
-
-    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public InternalHDRPercentiles readResult(StreamInput in) throws IOException {
-            InternalHDRPercentiles result = new InternalHDRPercentiles();
-            result.readFrom(in);
-            return result;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
-    InternalHDRPercentiles() {
-    } // for serialization
+    public static final String NAME = "hdr_percentiles";
 
     public InternalHDRPercentiles(String name, double[] percents, DoubleHistogram state, boolean keyed, DocValueFormat formatter,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
         super(name, percents, state, keyed, formatter, pipelineAggregators, metaData);
     }
 
+    /**
+     * Read from a stream.
+     */
+    public InternalHDRPercentiles(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
     @Override
     public Iterator<Percentile> iterator() {
         return new Iter(keys, state);
@@ -89,11 +80,6 @@ public class InternalHDRPercentiles extends AbstractInternalHDRPercentiles imple
         return new InternalHDRPercentiles(name, keys, merged, keyed, format, pipelineAggregators, metaData);
     }
 
-    @Override
-    public Type type() {
-        return TYPE;
-    }
-
     public static class Iter implements Iterator<Percentile> {
 
         private final double[] percents;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractInternalTDigestPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractInternalTDigestPercentiles.java
index b55efcde92d..43bee101f4a 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractInternalTDigestPercentiles.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/AbstractInternalTDigestPercentiles.java
@@ -33,11 +33,9 @@ import java.util.Map;
 
 abstract class AbstractInternalTDigestPercentiles extends InternalNumericMetricsAggregation.MultiValue {
 
-    protected double[] keys;
-    protected TDigestState state;
-    private boolean keyed;
-
-    AbstractInternalTDigestPercentiles() {} // for serialization
+    protected final double[] keys;
+    protected final TDigestState state;
+    private final boolean keyed;
 
     public AbstractInternalTDigestPercentiles(String name, double[] keys, TDigestState state, boolean keyed, DocValueFormat formatter,
             List<PipelineAggregator> pipelineAggregators,
@@ -49,6 +47,25 @@ abstract class AbstractInternalTDigestPercentiles extends InternalNumericMetrics
         this.format = formatter;
     }
 
+    /**
+     * Read from a stream.
+     */
+    protected AbstractInternalTDigestPercentiles(StreamInput in) throws IOException {
+        super(in);
+        format = in.readNamedWriteable(DocValueFormat.class);
+        keys = in.readDoubleArray();
+        state = TDigestState.read(in);
+        keyed = in.readBoolean();
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(format);
+        out.writeDoubleArray(keys);
+        TDigestState.write(state, out);
+        out.writeBoolean(keyed);
+    }
+
     @Override
     public double value(String name) {
         return value(Double.parseDouble(name));
@@ -76,28 +93,6 @@ abstract class AbstractInternalTDigestPercentiles extends InternalNumericMetrics
     protected abstract AbstractInternalTDigestPercentiles createReduced(String name, double[] keys, TDigestState merged, boolean keyed,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData);
 
-    @Override
-    protected void doReadFrom(StreamInput in) throws IOException {
-        format = in.readNamedWriteable(DocValueFormat.class);
-        keys = new double[in.readInt()];
-        for (int i = 0; i < keys.length; ++i) {
-            keys[i] = in.readDouble();
-        }
-        state = TDigestState.read(in);
-        keyed = in.readBoolean();
-    }
-
-    @Override
-    protected void doWriteTo(StreamOutput out) throws IOException {
-        out.writeNamedWriteable(format);
-        out.writeInt(keys.length);
-        for (int i = 0 ; i < keys.length; ++i) {
-            out.writeDouble(keys[i]);
-        }
-        TDigestState.write(state, out);
-        out.writeBoolean(keyed);
-    }
-
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         if (keyed) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentileRanks.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentileRanks.java
index f6622e0ab25..9e24ba5d86e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentileRanks.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentileRanks.java
@@ -20,7 +20,6 @@ package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest;
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentile;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
 import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanks;
@@ -31,33 +30,26 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-/**
-*
-*/
 public class InternalTDigestPercentileRanks extends AbstractInternalTDigestPercentiles implements PercentileRanks {
-
-    public static final Type TYPE = new Type(PercentileRanks.TYPE_NAME, "t_digest_percentile_ranks");
-
-    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public InternalTDigestPercentileRanks readResult(StreamInput in) throws IOException {
-            InternalTDigestPercentileRanks result = new InternalTDigestPercentileRanks();
-            result.readFrom(in);
-            return result;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
-    InternalTDigestPercentileRanks() {} // for serialization
+    public static final String NAME = "tdigest_percentile_ranks";
 
     public InternalTDigestPercentileRanks(String name, double[] cdfValues, TDigestState state, boolean keyed, DocValueFormat formatter,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
         super(name, cdfValues, state, keyed, formatter, pipelineAggregators, metaData);
     }
 
+    /**
+     * Read from a stream.
+     */
+    public InternalTDigestPercentileRanks(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
     @Override
     public Iterator<Percentile> iterator() {
         return new Iter(keys, state);
@@ -84,11 +76,6 @@ public class InternalTDigestPercentileRanks extends AbstractInternalTDigestPerce
         return new InternalTDigestPercentileRanks(name, keys, merged, keyed, format, pipelineAggregators, metaData);
     }
 
-    @Override
-    public Type type() {
-        return TYPE;
-    }
-
     static double percentileRank(TDigestState state, double value) {
         double percentileRank = state.cdf(value);
         if (percentileRank < 0) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentiles.java
index ce299d4d90b..ec619219111 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentiles.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/tdigest/InternalTDigestPercentiles.java
@@ -20,7 +20,6 @@ package org.elasticsearch.search.aggregations.metrics.percentiles.tdigest;
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentile;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles;
@@ -31,34 +30,26 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-/**
-*
-*/
 public class InternalTDigestPercentiles extends AbstractInternalTDigestPercentiles implements Percentiles {
-
-    public static final Type TYPE = new Type(Percentiles.TYPE_NAME, "t_digest_percentiles");
-
-    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public InternalTDigestPercentiles readResult(StreamInput in) throws IOException {
-            InternalTDigestPercentiles result = new InternalTDigestPercentiles();
-            result.readFrom(in);
-            return result;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
-    InternalTDigestPercentiles() {
-    } // for serialization
+    public static final String NAME = "tdigest_percentiles";
 
     public InternalTDigestPercentiles(String name, double[] percents, TDigestState state, boolean keyed, DocValueFormat formatter,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
         super(name, percents, state, keyed, formatter, pipelineAggregators, metaData);
     }
 
+    /**
+     * Read from a stream.
+     */
+    public InternalTDigestPercentiles(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
     @Override
     public Iterator<Percentile> iterator() {
         return new Iter(keys, state);
@@ -85,11 +76,6 @@ public class InternalTDigestPercentiles extends AbstractInternalTDigestPercentil
         return new InternalTDigestPercentiles(name, keys, merged, keyed, format, pipelineAggregators, metaData);
     }
 
-    @Override
-    public Type type() {
-        return TYPE;
-    }
-
     public static class Iter implements Iterator<Percentile> {
 
         private final double[] percents;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCount.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCount.java
index 807c7ed78e2..bfd186df5f2 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCount.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/InternalValueCount.java
@@ -21,7 +21,6 @@ package org.elasticsearch.search.aggregations.metrics.valuecount;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -34,25 +33,7 @@ import java.util.Map;
  * An internal implementation of {@link ValueCount}.
  */
 public class InternalValueCount extends InternalNumericMetricsAggregation.SingleValue implements ValueCount {
-
-    public static final Type TYPE = new Type("value_count", "vcount");
-
-    private static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public InternalValueCount readResult(StreamInput in) throws IOException {
-            InternalValueCount count = new InternalValueCount();
-            count.readFrom(in);
-            return count;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
-    private long value;
-
-    InternalValueCount() {} // for serialization
+    private final long value;
 
     public InternalValueCount(String name, long value, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) {
@@ -60,6 +41,24 @@ public class InternalValueCount extends InternalNumericMetricsAggregation.Single
         this.value = value;
     }
 
+    /**
+     * Read from a stream.
+     */
+    public InternalValueCount(StreamInput in) throws IOException {
+        super(in);
+        value = in.readVLong();
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeVLong(value);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ValueCountAggregationBuilder.NAME;
+    }
+
     @Override
     public long getValue() {
         return value;
@@ -70,11 +69,6 @@ public class InternalValueCount extends InternalNumericMetricsAggregation.Single
         return value;
     }
 
-    @Override
-    public Type type() {
-        return TYPE;
-    }
-
     @Override
     public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
         long valueCount = 0;
@@ -84,16 +78,6 @@ public class InternalValueCount extends InternalNumericMetricsAggregation.Single
         return new InternalValueCount(name, valueCount, pipelineAggregators(), getMetaData());
     }
 
-    @Override
-    protected void doReadFrom(StreamInput in) throws IOException {
-        value = in.readVLong();
-    }
-
-    @Override
-    protected void doWriteTo(StreamOutput out) throws IOException {
-        out.writeVLong(value);
-    }
-
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         builder.field(CommonFields.VALUE, value);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregationBuilder.java
index 8cc7a5b39c8..067d16db9f8 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/valuecount/ValueCountAggregationBuilder.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSource;
@@ -35,18 +36,19 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import java.io.IOException;
 
 public class ValueCountAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource, ValueCountAggregationBuilder> {
-    public static final String NAME = InternalValueCount.TYPE.name();
+    public static final String NAME = "value_count";
+    public static final Type TYPE = new Type(NAME);
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     public ValueCountAggregationBuilder(String name, ValueType targetValueType) {
-        super(name, InternalValueCount.TYPE, ValuesSourceType.ANY, targetValueType);
+        super(name, TYPE, ValuesSourceType.ANY, targetValueType);
     }
 
     /**
      * Read from a stream.
      */
     public ValueCountAggregationBuilder(StreamInput in) throws IOException {
-        super(in, InternalValueCount.TYPE, ValuesSourceType.ANY);
+        super(in, TYPE, ValuesSourceType.ANY);
     }
 
     @Override

From f6491047a0a7182d5c55660acc57add812ce64c1 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Fri, 1 Jul 2016 12:36:44 -0700
Subject: [PATCH 1199/1311] Fix line lengths for ingest config utils

---
 .../ingest/ConfigurationUtils.java            | 21 +++++++++++--------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java b/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
index 5bbef2a2551..6eb4b0fe608 100644
--- a/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
+++ b/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
@@ -39,8 +39,8 @@ public final class ConfigurationUtils {
      *
      * If the property value isn't of type string a {@link ElasticsearchParseException} is thrown.
      */
-    public static String readOptionalStringProperty(String processorType, String processorTag, Map<String, Object> configuration,
-                                                    String propertyName) {
+    public static String readOptionalStringProperty(String processorType, String processorTag,
+                                                    Map<String, Object> configuration, String propertyName) {
         Object value = configuration.remove(propertyName);
         return readString(processorType, processorTag, propertyName, value);
     }
@@ -112,8 +112,8 @@ public final class ConfigurationUtils {
      * If the property value isn't of type int a {@link ElasticsearchParseException} is thrown.
      * If the property is missing an {@link ElasticsearchParseException} is thrown
      */
-    public static int readIntProperty(String processorType, String processorTag, Map<String, Object> configuration, String propertyName,
-                                      int defaultValue) {
+    public static int readIntProperty(String processorType, String processorTag, Map<String, Object> configuration,
+                                      String propertyName, int defaultValue) {
         Object value = configuration.remove(propertyName);
         if (value == null) {
             return defaultValue;
@@ -146,7 +146,8 @@ public final class ConfigurationUtils {
      * If the property value isn't of type list an {@link ElasticsearchParseException} is thrown.
      * If the property is missing an {@link ElasticsearchParseException} is thrown
      */
-    public static <T> List<T> readList(String processorType, String processorTag, Map<String, Object> configuration, String propertyName) {
+    public static <T> List<T> readList(String processorType, String processorTag, Map<String, Object> configuration,
+                                       String propertyName) {
         Object value = configuration.remove(propertyName);
         if (value == null) {
             throw newConfigurationException(processorType, processorTag, propertyName, "required property is missing");
@@ -211,7 +212,8 @@ public final class ConfigurationUtils {
     /**
      * Returns and removes the specified property as an {@link Object} from the specified configuration map.
      */
-    public static Object readObject(String processorType, String processorTag, Map<String, Object> configuration, String propertyName) {
+    public static Object readObject(String processorType, String processorTag, Map<String, Object> configuration,
+                                    String propertyName) {
         Object value = configuration.remove(propertyName);
         if (value == null) {
             throw newConfigurationException(processorType, processorTag, propertyName, "required property is missing");
@@ -219,8 +221,8 @@ public final class ConfigurationUtils {
         return value;
     }
 
-    public static ElasticsearchParseException newConfigurationException(String processorType, String processorTag, String propertyName,
-                                                                        String reason) {
+    public static ElasticsearchParseException newConfigurationException(String processorType, String processorTag,
+                                                                        String propertyName, String reason) {
         ElasticsearchParseException exception = new ElasticsearchParseException("[" + propertyName + "] " + reason);
 
         if (processorType != null) {
@@ -249,7 +251,8 @@ public final class ConfigurationUtils {
         return processors;
     }
 
-    private static Processor readProcessor(Map<String, Processor.Factory> processorFactories, String type, Map<String, Object> config) throws Exception {
+    private static Processor readProcessor(Map<String, Processor.Factory> processorFactories,
+                                           String type, Map<String, Object> config) throws Exception {
         Processor.Factory factory = processorFactories.get(type);
         if (factory != null) {
             boolean ignoreFailure = ConfigurationUtils.readBooleanProperty(null, null, config, "ignore_failure", false);

From 6110dcc7104a12d62c407c13a3f808fb70b6949d Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Fri, 1 Jul 2016 12:45:43 -0700
Subject: [PATCH 1200/1311] Fix unit tests for http server that no longer need
 node service

---
 .../org/elasticsearch/http/HttpServerTests.java    | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
index 828336c1603..9ed680a6fbe 100644
--- a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
+++ b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
@@ -18,7 +18,8 @@
  */
 package org.elasticsearch.http;
 
-import org.elasticsearch.cluster.service.ClusterService;
+import java.util.Map;
+
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -32,7 +33,6 @@ import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
-import org.elasticsearch.node.service.NodeService;
 import org.elasticsearch.rest.AbstractRestChannel;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestController;
@@ -42,10 +42,6 @@ import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.Before;
 
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.util.Map;
-
 public class HttpServerTests extends ESTestCase {
     private static final ByteSizeValue BREAKER_LIMIT = new ByteSizeValue(20);
     private HttpServer httpServer;
@@ -71,11 +67,7 @@ public class HttpServerTests extends ESTestCase {
                 throw new IllegalArgumentException("test error");
             });
 
-        ClusterService clusterService = new ClusterService(Settings.EMPTY,
-            new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), null);
-        NodeService nodeService = new NodeService(Settings.EMPTY, null, null, null, null, null, null, null, null,
-            clusterService, null);
-        httpServer = new HttpServer(settings, httpServerTransport, restController, nodeService, null, circuitBreakerService);
+        httpServer = new HttpServer(settings, httpServerTransport, restController, null, circuitBreakerService);
         httpServer.start();
     }
 

From 62397c0c3d662932f0069e0d56f258f0754cc8b8 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Fri, 1 Jul 2016 12:55:41 -0700
Subject: [PATCH 1201/1311] Fix node service to still work when http is
 disabled

---
 .../elasticsearch/common/network/NetworkModule.java |  3 +++
 .../org/elasticsearch/node/service/NodeService.java | 13 ++++++++-----
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
index 2b94f224b9c..b65f568af8a 100644
--- a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
+++ b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
@@ -32,6 +32,7 @@ import org.elasticsearch.cluster.routing.allocation.command.CancelAllocationComm
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.inject.AbstractModule;
+import org.elasticsearch.common.inject.util.Providers;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Setting;
@@ -166,6 +167,8 @@ public class NetworkModule extends AbstractModule {
             if (HTTP_ENABLED.get(settings)) {
                 bind(HttpServer.class).asEagerSingleton();
                 httpTransportTypes.bindType(binder(), settings, HTTP_TYPE_SETTING.getKey(), NETTY_TRANSPORT);
+            } else {
+                bind(HttpServer.class).toProvider(Providers.of(null));
             }
             // Bind the AllocationCommandRegistry so RestClusterRerouteAction can get it.
             bind(AllocationCommandRegistry.class).toInstance(allocationCommandRegistry);
diff --git a/core/src/main/java/org/elasticsearch/node/service/NodeService.java b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
index f3743a46d1e..8f938bcb22d 100644
--- a/core/src/main/java/org/elasticsearch/node/service/NodeService.java
+++ b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
@@ -28,6 +28,7 @@ import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -58,14 +59,16 @@ public class NodeService extends AbstractComponent implements Closeable {
     private final SettingsFilter settingsFilter;
     private ClusterService clusterService;
     private ScriptService scriptService;
-    private final HttpServer httpServer;
+
+    @Nullable
+    private HttpServer httpServer;
 
     private final Discovery discovery;
 
     @Inject
     public NodeService(Settings settings, ThreadPool threadPool, MonitorService monitorService,
                        Discovery discovery, TransportService transportService, IndicesService indicesService,
-                       PluginsService pluginService, CircuitBreakerService circuitBreakerService, HttpServer httpServer,
+                       PluginsService pluginService, CircuitBreakerService circuitBreakerService, @Nullable HttpServer httpServer,
                        ProcessorsRegistry.Builder processorsRegistryBuilder, ClusterService clusterService, SettingsFilter settingsFilter) {
         super(settings);
         this.threadPool = threadPool;
@@ -98,7 +101,7 @@ public class NodeService extends AbstractComponent implements Closeable {
                 monitorService.jvmService().info(),
                 threadPool.info(),
                 transportService.info(),
-                httpServer.info(),
+                httpServer == null ? null : httpServer.info(),
                 pluginService == null ? null : pluginService.info(),
                 ingestService == null ? null : ingestService.info(),
                 indicesService.getTotalIndexingBufferBytes()
@@ -114,7 +117,7 @@ public class NodeService extends AbstractComponent implements Closeable {
                 jvm ? monitorService.jvmService().info() : null,
                 threadPool ? this.threadPool.info() : null,
                 transport ? transportService.info() : null,
-                http ? httpServer.info() : null,
+                http ? (httpServer == null ? null : httpServer.info()) : null,
                 plugin ? (pluginService == null ? null : pluginService.info()) : null,
                 ingest ? (ingestService == null ? null : ingestService.info()) : null,
                 indices ? indicesService.getTotalIndexingBufferBytes() : null
@@ -132,7 +135,7 @@ public class NodeService extends AbstractComponent implements Closeable {
                 threadPool.stats(),
                 monitorService.fsService().stats(),
                 transportService.stats(),
-                httpServer.stats(),
+                httpServer == null ? null : httpServer.stats(),
                 circuitBreakerService.stats(),
                 scriptService.stats(),
                 discovery.stats(),

From f9eed91f33998c775fbc41dff763b9e845b35ce6 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Fri, 1 Jul 2016 13:00:33 -0700
Subject: [PATCH 1202/1311] Fix test oops

---
 .../SimulatePipelineRequestParsingTests.java  | 23 ++++++++-----------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
index 6f335aef634..8418c886be9 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
@@ -19,18 +19,6 @@
 
 package org.elasticsearch.action.ingest;
 
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.ingest.PipelineStore;
-import org.elasticsearch.ingest.Processor;
-import org.elasticsearch.ingest.TestProcessor;
-import org.elasticsearch.ingest.CompoundProcessor;
-import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.Pipeline;
-import org.elasticsearch.ingest.Processor;
-import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.test.ESTestCase;
-import org.junit.Before;
-
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -39,6 +27,15 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
+import org.elasticsearch.ingest.CompoundProcessor;
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Pipeline;
+import org.elasticsearch.ingest.PipelineStore;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.TestProcessor;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
 import static org.elasticsearch.action.ingest.SimulatePipelineRequest.Fields;
 import static org.elasticsearch.action.ingest.SimulatePipelineRequest.SIMULATED_PIPELINE_ID;
 import static org.elasticsearch.ingest.IngestDocument.MetaData.ID;
@@ -59,7 +56,7 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
         CompoundProcessor pipelineCompoundProcessor = new CompoundProcessor(processor);
         Pipeline pipeline = new Pipeline(SIMULATED_PIPELINE_ID, null, pipelineCompoundProcessor);
         Map<String, Processor.Factory> registry =
-            Collections.singletonMap("mock_processor", (factories, config) -> processor);
+            Collections.singletonMap("mock_processor", (factories, tag, config) -> processor);
         store = mock(PipelineStore.class);
         when(store.get(SIMULATED_PIPELINE_ID)).thenReturn(pipeline);
         when(store.getProcessorFactories()).thenReturn(registry);

From 40e1fe7d9e09bcb5d9de2228873c6d71761804fc Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Fri, 1 Jul 2016 13:17:12 -0700
Subject: [PATCH 1203/1311] Remove unused ClusterService member of NodeService

---
 .../main/java/org/elasticsearch/node/service/NodeService.java   | 2 --
 1 file changed, 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/node/service/NodeService.java b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
index 7954dfa2238..e3989e26d05 100644
--- a/core/src/main/java/org/elasticsearch/node/service/NodeService.java
+++ b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
@@ -58,7 +58,6 @@ public class NodeService extends AbstractComponent implements Closeable {
     private final CircuitBreakerService circuitBreakerService;
     private final IngestService ingestService;
     private final SettingsFilter settingsFilter;
-    private ClusterService clusterService;
     private ScriptService scriptService;
 
     @Nullable
@@ -80,7 +79,6 @@ public class NodeService extends AbstractComponent implements Closeable {
         this.pluginService = pluginService;
         this.circuitBreakerService = circuitBreakerService;
         this.httpServer = httpServer;
-        this.clusterService = clusterService;
         this.ingestService = ingestService;
         this.settingsFilter = settingsFilter;
         this.scriptService = scriptService;

From 10261a615b981d3eeafc75d113b2de716b3e449e Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Fri, 1 Jul 2016 14:16:09 -0700
Subject: [PATCH 1204/1311] Update ingest useragent plugin to use new ingest
 plugin

---
 .../ingest/geoip/IngestGeoIpPlugin.java       |  2 +-
 .../useragent/IngestUserAgentPlugin.java      | 36 +++++++++++++------
 .../ingest/useragent/UserAgentProcessor.java  |  3 +-
 .../UserAgentProcessorFactoryTests.java       | 14 ++++----
 4 files changed, 35 insertions(+), 20 deletions(-)

diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
index fb83f595e4c..52fbad2c90f 100644
--- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
+++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java
@@ -48,7 +48,7 @@ public class IngestGeoIpPlugin extends Plugin implements IngestPlugin, Closeable
     private Map<String, DatabaseReader> databaseReaders;
 
     @Override
-    public synchronized Map<String, Processor.Factory> getProcessors(
+    public Map<String, Processor.Factory> getProcessors(
         Environment env, ScriptService scriptService, TemplateService templateService) {
         if (databaseReaders != null) {
             throw new IllegalStateException("called onModule twice for geoip plugin!!");
diff --git a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java
index e86a8443d7e..0257c267eb4 100644
--- a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java
+++ b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java
@@ -20,8 +20,14 @@
 package org.elasticsearch.ingest.useragent;
 
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.TemplateService;
 import org.elasticsearch.node.NodeModule;
+import org.elasticsearch.plugins.IngestPlugin;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.ScriptService;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -34,28 +40,36 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.stream.Stream;
 
-public class IngestUserAgentPlugin extends Plugin {
+public class IngestUserAgentPlugin extends Plugin implements IngestPlugin {
 
     private final Setting<Long> CACHE_SIZE_SETTING = Setting.longSetting("ingest.useragent.cache_size", 1000, 0,
             Setting.Property.NodeScope);
 
     static final String DEFAULT_PARSER_NAME = "_default_";
+    private final Settings settings;
 
-    public void onModule(NodeModule nodeModule) throws IOException {
-        Path userAgentConfigDirectory = nodeModule.getNode().getEnvironment().configFile().resolve("ingest-useragent");
+    IngestUserAgentPlugin(Settings settings) {
+        this.settings = settings;
+    }
+
+    @Override
+    public Map<String, Processor.Factory> getProcessors(
+        Environment env, ScriptService scriptService, TemplateService templateService) {
+        Path userAgentConfigDirectory = env.configFile().resolve("ingest-useragent");
 
         if (Files.exists(userAgentConfigDirectory) == false && Files.isDirectory(userAgentConfigDirectory)) {
             throw new IllegalStateException(
-                    "the user agent directory [" + userAgentConfigDirectory + "] containing the regex file doesn't exist");
+                "the user agent directory [" + userAgentConfigDirectory + "] containing the regex file doesn't exist");
         }
 
-        long cacheSize = CACHE_SIZE_SETTING.get(nodeModule.getNode().settings());
-
-        UserAgentCache cache = new UserAgentCache(cacheSize);
-
-        Map<String, UserAgentParser> userAgentParsers = createUserAgentParsers(userAgentConfigDirectory, cache);
-
-        nodeModule.registerProcessor(UserAgentProcessor.TYPE, (registry) -> new UserAgentProcessor.Factory(userAgentParsers));
+        long cacheSize = CACHE_SIZE_SETTING.get(settings);
+        Map<String, UserAgentParser> userAgentParsers;
+        try {
+            userAgentParsers = createUserAgentParsers(userAgentConfigDirectory, new UserAgentCache(cacheSize));
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        return Collections.singletonMap(UserAgentProcessor.TYPE, new UserAgentProcessor.Factory(userAgentParsers));
     }
 
     static Map<String, UserAgentParser> createUserAgentParsers(Path userAgentConfigDirectory, UserAgentCache cache) throws IOException {
diff --git a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
index 7cb829e36e2..e52d0966a44 100644
--- a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
+++ b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
@@ -194,7 +194,8 @@ public class UserAgentProcessor extends AbstractProcessor {
         }
 
         @Override
-        public UserAgentProcessor create(String processorTag, Map<String, Object> config) throws Exception {
+        public UserAgentProcessor create(Map<String, Processor.Factory> factories, String processorTag,
+                                         Map<String, Object> config) throws Exception {
             String field = readStringProperty(TYPE, processorTag, config, "field");
             String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "useragent");
             String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME);
diff --git a/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java b/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java
index 52a10840387..2fbdb6723b3 100644
--- a/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java
+++ b/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java
@@ -81,7 +81,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
 
         String processorTag = randomAsciiOfLength(10);
 
-        UserAgentProcessor processor = factory.create(processorTag, config);
+        UserAgentProcessor processor = factory.create(null, processorTag, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("useragent"));
@@ -98,7 +98,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("target_field", "_target_field");
 
-        UserAgentProcessor processor = factory.create(null, config);
+        UserAgentProcessor processor = factory.create(null, null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("_target_field"));
     }
@@ -110,7 +110,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("regex_file", regexWithoutDevicesFilename);
 
-        UserAgentProcessor processor = factory.create(null, config);
+        UserAgentProcessor processor = factory.create(null, null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getUaParser().getUaPatterns().size(), greaterThan(0));
         assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0));
@@ -124,7 +124,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("regex_file", "does-not-exist.yaml");
 
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config));
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
         assertThat(e.getMessage(), equalTo("[regex_file] regex file [does-not-exist.yaml] doesn't exist (has to exist at node startup)"));
     }
 
@@ -144,7 +144,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("properties", fieldNames);
 
-        UserAgentProcessor processor = factory.create(null, config);
+        UserAgentProcessor processor = factory.create(null, null, config);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getProperties(), equalTo(properties));
     }
@@ -156,7 +156,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("properties", Collections.singletonList("invalid"));
 
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config));
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
         assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, MAJOR, MINOR, "
                 + "PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD]"));
     }
@@ -168,7 +168,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("properties", "invalid");
 
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config));
+        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
         assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]"));
     }
 }

From e707f0ea6e996f325afe07cfa1a8e75a816eb676 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Fri, 1 Jul 2016 14:21:41 -0700
Subject: [PATCH 1205/1311] Simplify ingest useragent construction

---
 .../ingest/useragent/IngestUserAgentPlugin.java            | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java
index 0257c267eb4..3e9a42fb8d9 100644
--- a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java
+++ b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java
@@ -46,11 +46,6 @@ public class IngestUserAgentPlugin extends Plugin implements IngestPlugin {
             Setting.Property.NodeScope);
 
     static final String DEFAULT_PARSER_NAME = "_default_";
-    private final Settings settings;
-
-    IngestUserAgentPlugin(Settings settings) {
-        this.settings = settings;
-    }
 
     @Override
     public Map<String, Processor.Factory> getProcessors(
@@ -62,7 +57,7 @@ public class IngestUserAgentPlugin extends Plugin implements IngestPlugin {
                 "the user agent directory [" + userAgentConfigDirectory + "] containing the regex file doesn't exist");
         }
 
-        long cacheSize = CACHE_SIZE_SETTING.get(settings);
+        long cacheSize = CACHE_SIZE_SETTING.get(env.settings());
         Map<String, UserAgentParser> userAgentParsers;
         try {
             userAgentParsers = createUserAgentParsers(userAgentConfigDirectory, new UserAgentCache(cacheSize));

From 05998224d8ecc11bef0b5a2432044e6a54585fa3 Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Fri, 1 Jul 2016 17:26:16 -0400
Subject: [PATCH 1206/1311] Adding repository index generational files

Before, a repository would maintain an index file (named 'index') per
repository, that contained the current snapshots in the repository.
This file was not atomically written, so repositories had to depend on
listing the blobs in the repository to determine what the current
snapshots are, and only rely on the index file if the repository does
not support the listBlobs operation.  This could cause an incorrect view
of the current snapshots in the repository if any prior snapshot delete
operations failed to delete snapshot metadata files.

This commit introduces the atomic writing of the index file, and because
atomic writes are not guaranteed if the file already exists, we write to
a generational index file (index-N, where N is the current generation).
We also maintain an index-latest file that contains the current
generation, for those repositories that cannot list blobs.

Closes #19002
Relates #18156
---
 .../common/settings/ClusterSettings.java      |   1 -
 .../blobstore/BlobStoreRepository.java        | 327 ++++++++++--------
 .../repositories/uri/URLRepository.java       |  23 --
 .../blobstore/BlobStoreRepositoryTests.java   | 171 +++++----
 .../DedicatedClusterSnapshotRestoreIT.java    |  11 +-
 .../SharedClusterSnapshotRestoreIT.java       |   5 +-
 .../cloud/azure/blobstore/AzureBlobStore.java |   3 +-
 .../azure/storage/AzureStorageService.java    |   3 +-
 .../storage/AzureStorageServiceMock.java      |   7 +-
 9 files changed, 288 insertions(+), 263 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index 9469ac8b509..01bd5e8a818 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -366,7 +366,6 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     Node.NODE_INGEST_SETTING,
                     Node.NODE_ATTRIBUTES,
                     URLRepository.ALLOWED_URLS_SETTING,
-                    URLRepository.REPOSITORIES_LIST_DIRECTORIES_SETTING,
                     URLRepository.REPOSITORIES_URL_SETTING,
                     URLRepository.SUPPORTED_PROTOCOLS_SETTING,
                     TransportMasterNodeReadAction.FORCE_LOCAL_SETTING,
diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index c1efcb7801c..52372706d34 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -24,6 +24,7 @@ import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.Numbers;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
@@ -34,7 +35,6 @@ import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.compress.NotXContentException;
 import org.elasticsearch.common.io.Streams;
@@ -83,7 +83,8 @@ import java.util.stream.Collectors;
  * <pre>
  * {@code
  *   STORE_ROOT
- *   |- index             - list of all snapshot name as JSON array
+ *   |- index-N           - list of all snapshot name as JSON array, N is the generation of the file
+ *   |- index-latest      - contains the numeric value of the latest generation of the index file (i.e. N from above)
  *   |- snapshot-20131010 - JSON serialized Snapshot for snapshot "20131010"
  *   |- meta-20131010.dat - JSON serialized MetaData for snapshot "20131010" (includes only global metadata)
  *   |- snapshot-20131011 - JSON serialized Snapshot for snapshot "20131011"
@@ -127,12 +128,14 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
 
     private static final String SNAPSHOT_SUFFIX = ".dat";
 
-    private static final String COMMON_SNAPSHOT_PREFIX = "snap";
-
     private static final String SNAPSHOT_CODEC = "snapshot";
 
     static final String SNAPSHOTS_FILE = "index"; // package private for unit testing
 
+    private static final String SNAPSHOTS_FILE_PREFIX = "index-";
+
+    private static final String SNAPSHOTS_INDEX_LATEST_BLOB = "index.latest";
+
     private static final String TESTS_FILE = "tests-";
 
     private static final String METADATA_NAME_FORMAT = "meta-%s.dat";
@@ -147,7 +150,6 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
 
     private static final String LEGACY_SNAPSHOT_NAME_FORMAT = LEGACY_SNAPSHOT_PREFIX + "%s";
 
-
     private final BlobStoreIndexShardRepository indexShardRepository;
 
     private final RateLimiter snapshotRateLimiter;
@@ -334,7 +336,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
             }
             // Delete snapshot from the snapshot list
             List<SnapshotId> snapshotIds = snapshots().stream().filter(id -> snapshotId.equals(id) == false).collect(Collectors.toList());
-            writeSnapshotList(snapshotIds);
+            writeSnapshotsToIndexGen(snapshotIds);
+
             // Now delete all indices
             for (String index : indices) {
                 BlobPath indexPath = basePath().add("indices").add(index);
@@ -386,8 +389,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
                 snapshotIds = new ArrayList<>(snapshotIds);
                 snapshotIds.add(snapshotId);
                 snapshotIds = Collections.unmodifiableList(snapshotIds);
+                writeSnapshotsToIndexGen(snapshotIds);
             }
-            writeSnapshotList(snapshotIds);
             return blobStoreSnapshot;
         } catch (IOException ex) {
             throw new RepositoryException(this.repositoryName, "failed to update snapshot in repository", ex);
@@ -400,40 +403,12 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
     @Override
     public List<SnapshotId> snapshots() {
         try {
-            List<SnapshotId> snapshots = new ArrayList<>();
-            Map<String, BlobMetaData> blobs;
-            try {
-                blobs = snapshotsBlobContainer.listBlobsByPrefix(COMMON_SNAPSHOT_PREFIX);
-            } catch (UnsupportedOperationException ex) {
-                // Fall back in case listBlobsByPrefix isn't supported by the blob store
-                return readSnapshotList();
-            }
-            int prefixLength = SNAPSHOT_PREFIX.length();
-            int suffixLength = SNAPSHOT_SUFFIX.length();
-            int legacyPrefixLength = LEGACY_SNAPSHOT_PREFIX.length();
-            for (BlobMetaData md : blobs.values()) {
-                String blobName = md.name();
-                final String name;
-                String uuid;
-                if (blobName.startsWith(SNAPSHOT_PREFIX) && blobName.length() > legacyPrefixLength) {
-                    final String str = blobName.substring(prefixLength, blobName.length() - suffixLength);
-                    // TODO: this will go away once we make the snapshot file writes atomic and
-                    // use it as the source of truth for the snapshots list instead of listing blobs
-                    Tuple<String, String> pair = parseNameUUIDFromBlobName(str);
-                    name = pair.v1();
-                    uuid = pair.v2();
-                } else if (blobName.startsWith(LEGACY_SNAPSHOT_PREFIX) && blobName.length() > suffixLength + prefixLength) {
-                    name = blobName.substring(legacyPrefixLength);
-                    uuid = SnapshotId.UNASSIGNED_UUID;
-                } else {
-                    // not sure what it was - ignore
-                    continue;
-                }
-                snapshots.add(new SnapshotId(name, uuid));
-            }
-            return Collections.unmodifiableList(snapshots);
-        } catch (IOException ex) {
-            throw new RepositoryException(repositoryName, "failed to list snapshots in repository", ex);
+            return Collections.unmodifiableList(readSnapshotsFromIndex());
+        } catch (NoSuchFileException | FileNotFoundException e) {
+            // its a fresh repository, no index file exists, so return an empty list
+            return Collections.emptyList();
+        } catch (IOException ioe) {
+            throw new RepositoryException(repositoryName, "failed to list snapshots in repository", ioe);
         }
     }
 
@@ -564,89 +539,6 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
     private static final String NAME = "name";
     private static final String UUID = "uuid";
 
-    /**
-     * Writes snapshot index file
-     * <p>
-     * This file can be used by read-only repositories that are unable to list files in the repository
-     *
-     * @param snapshots list of snapshot ids
-     * @throws IOException I/O errors
-     */
-    protected void writeSnapshotList(List<SnapshotId> snapshots) throws IOException {
-        final BytesReference bRef;
-        try(BytesStreamOutput bStream = new BytesStreamOutput()) {
-            try(StreamOutput stream = new OutputStreamStreamOutput(bStream)) {
-                XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream);
-                builder.startObject();
-                builder.startArray(SNAPSHOTS);
-                for (SnapshotId snapshot : snapshots) {
-                    builder.startObject();
-                    builder.field(NAME, snapshot.getName());
-                    builder.field(UUID, snapshot.getUUID());
-                    builder.endObject();
-                }
-                builder.endArray();
-                builder.endObject();
-                builder.close();
-            }
-            bRef = bStream.bytes();
-        }
-        if (snapshotsBlobContainer.blobExists(SNAPSHOTS_FILE)) {
-            snapshotsBlobContainer.deleteBlob(SNAPSHOTS_FILE);
-        }
-        snapshotsBlobContainer.writeBlob(SNAPSHOTS_FILE, bRef);
-    }
-
-    /**
-     * Reads snapshot index file
-     * <p>
-     * This file can be used by read-only repositories that are unable to list files in the repository
-     *
-     * @return list of snapshots in the repository
-     * @throws IOException I/O errors
-     */
-    protected List<SnapshotId> readSnapshotList() throws IOException {
-        try (InputStream blob = snapshotsBlobContainer.readBlob(SNAPSHOTS_FILE)) {
-            BytesStreamOutput out = new BytesStreamOutput();
-            Streams.copy(blob, out);
-            ArrayList<SnapshotId> snapshots = new ArrayList<>();
-            try (XContentParser parser = XContentHelper.createParser(out.bytes())) {
-                if (parser.nextToken() == XContentParser.Token.START_OBJECT) {
-                    if (parser.nextToken() == XContentParser.Token.FIELD_NAME) {
-                        String currentFieldName = parser.currentName();
-                        if (SNAPSHOTS.equals(currentFieldName)) {
-                            if (parser.nextToken() == XContentParser.Token.START_ARRAY) {
-                                while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
-                                    // the new format from 5.0 which contains the snapshot name and uuid
-                                    String name = null;
-                                    String uuid = null;
-                                    if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
-                                        while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
-                                            currentFieldName = parser.currentName();
-                                            parser.nextToken();
-                                            if (NAME.equals(currentFieldName)) {
-                                                name = parser.text();
-                                            } else if (UUID.equals(currentFieldName)) {
-                                                uuid = parser.text();
-                                            }
-                                        }
-                                        snapshots.add(new SnapshotId(name, uuid));
-                                    }
-                                    // the old format pre 5.0 that only contains the snapshot name, use the name as the uuid too
-                                    else {
-                                        name = parser.text();
-                                        snapshots.add(new SnapshotId(name, SnapshotId.UNASSIGNED_UUID));
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-            return Collections.unmodifiableList(snapshots);
-        }
-    }
-
     @Override
     public void onRestorePause(long nanos) {
         restoreRateLimitingTimeInNanos.inc(nanos);
@@ -714,27 +606,99 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
         return snapshotsBlobContainer;
     }
 
-    // TODO: this will go away once readSnapshotsList uses the index file instead of listing blobs
-    // to know which snapshots are part of a repository.  See #18156
-    // Package private for testing.
-    static Tuple<String, String> parseNameUUIDFromBlobName(final String str) {
-        final String name;
-        final String uuid;
-        final int sizeOfUUID = 22; // uuid is 22 chars in length
-        // unreliable, but highly unlikely to have a snapshot name with a dash followed by 22 characters,
-        // and this will go away before a release (see #18156).
-        //norelease
-        if (str.length() > sizeOfUUID + 1 && str.charAt(str.length() - sizeOfUUID - 1) == '-') {
-            // new naming convention, snapshot blob id has name and uuid
-            final int idx = str.length() - sizeOfUUID - 1;
-            name = str.substring(0, idx);
-            uuid = str.substring(idx + 1);
-        } else {
-            // old naming convention, before snapshots had UUIDs
-            name = str;
-            uuid = SnapshotId.UNASSIGNED_UUID;
+    protected void writeSnapshotsToIndexGen(final List<SnapshotId> snapshots) throws IOException {
+        assert readOnly() == false; // can not write to a read only repository
+        final BytesReference snapshotsBytes;
+        try (BytesStreamOutput bStream = new BytesStreamOutput()) {
+            try (StreamOutput stream = new OutputStreamStreamOutput(bStream)) {
+                XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON, stream);
+                builder.startObject();
+                builder.startArray(SNAPSHOTS);
+                for (SnapshotId snapshot : snapshots) {
+                    builder.startObject();
+                    builder.field(NAME, snapshot.getName());
+                    builder.field(UUID, snapshot.getUUID());
+                    builder.endObject();
+                }
+                builder.endArray();
+                builder.endObject();
+                builder.close();
+            }
+            snapshotsBytes = bStream.bytes();
+        }
+        final long gen = latestIndexBlobId() + 1;
+        // write the index file
+        writeAtomic(SNAPSHOTS_FILE_PREFIX + Long.toString(gen), snapshotsBytes);
+        // delete the N-2 index file if it exists, keep the previous one around as a backup
+        if (readOnly() == false && gen - 2 >= 0) {
+            final String oldSnapshotIndexFile = SNAPSHOTS_FILE_PREFIX + Long.toString(gen - 2);
+            if (snapshotsBlobContainer.blobExists(oldSnapshotIndexFile)) {
+                snapshotsBlobContainer.deleteBlob(oldSnapshotIndexFile);
+            }
+        }
+
+        // write the current generation to the index-latest file
+        final BytesReference genBytes;
+        try (BytesStreamOutput bStream = new BytesStreamOutput()) {
+            bStream.writeLong(gen);
+            genBytes = bStream.bytes();
+        }
+        if (snapshotsBlobContainer.blobExists(SNAPSHOTS_INDEX_LATEST_BLOB)) {
+            snapshotsBlobContainer.deleteBlob(SNAPSHOTS_INDEX_LATEST_BLOB);
+        }
+        writeAtomic(SNAPSHOTS_INDEX_LATEST_BLOB, genBytes);
+    }
+
+    protected List<SnapshotId> readSnapshotsFromIndex() throws IOException {
+        final long indexGen = latestIndexBlobId();
+        final String snapshotsIndexBlobName;
+        if (indexGen == -1) {
+            // index-N file doesn't exist, either its a fresh repository, or its in the
+            // old format, so look for the older index file before returning an empty list
+            snapshotsIndexBlobName = SNAPSHOTS_FILE;
+        } else {
+            snapshotsIndexBlobName = SNAPSHOTS_FILE_PREFIX + Long.toString(indexGen);
+        }
+
+        try (InputStream blob = snapshotsBlobContainer.readBlob(snapshotsIndexBlobName)) {
+            BytesStreamOutput out = new BytesStreamOutput();
+            Streams.copy(blob, out);
+            ArrayList<SnapshotId> snapshots = new ArrayList<>();
+            try (XContentParser parser = XContentHelper.createParser(out.bytes())) {
+                if (parser.nextToken() == XContentParser.Token.START_OBJECT) {
+                    if (parser.nextToken() == XContentParser.Token.FIELD_NAME) {
+                        String currentFieldName = parser.currentName();
+                        if (SNAPSHOTS.equals(currentFieldName)) {
+                            if (parser.nextToken() == XContentParser.Token.START_ARRAY) {
+                                while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
+                                    // the new format from 5.0 which contains the snapshot name and uuid
+                                    String name = null;
+                                    String uuid = null;
+                                    if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
+                                        while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+                                            currentFieldName = parser.currentName();
+                                            parser.nextToken();
+                                            if (NAME.equals(currentFieldName)) {
+                                                name = parser.text();
+                                            } else if (UUID.equals(currentFieldName)) {
+                                                uuid = parser.text();
+                                            }
+                                        }
+                                        snapshots.add(new SnapshotId(name, uuid));
+                                    }
+                                    // the old format pre 5.0 that only contains the snapshot name, use the name as the uuid too
+                                    else {
+                                        name = parser.text();
+                                        snapshots.add(new SnapshotId(name, SnapshotId.UNASSIGNED_UUID));
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            return Collections.unmodifiableList(snapshots);
         }
-        return Tuple.tuple(name, uuid);
     }
 
     // Package private for testing
@@ -746,4 +710,79 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Rep
         }
         return snapshotId.getName() + "-" + uuid;
     }
+
+    /**
+     * Get the latest snapshot index blob id.  Snapshot index blobs are named index-N, where N is
+     * the next version number from when the index blob was written.  Each individual index-N blob is
+     * only written once and never overwritten.  The highest numbered index-N blob is the latest one
+     * that contains the current snapshots in the repository.
+     *
+     * Package private for testing
+     */
+    long latestIndexBlobId() throws IOException {
+        try {
+            // first, try listing the blobs and determining which index blob is the latest
+            return listBlobsToGetLatestIndexId();
+        } catch (UnsupportedOperationException e) {
+            // could not list the blobs because the repository does not support the operation,
+            // try reading from the index-latest file
+            try {
+                return readSnapshotIndexLatestBlob();
+            } catch (IOException ioe) {
+                // we likely could not find the blob, this can happen in two scenarios:
+                //  (1) its an empty repository
+                //  (2) when writing the index-latest blob, if the blob already exists,
+                //      we first delete it, then atomically write the new blob.  there is
+                //      a small window in time when the blob is deleted and the new one
+                //      written - if the node crashes during that time, we won't have an
+                //      index-latest blob
+                // in a read-only repository, we can't know which of the two scenarios it is,
+                // but we will assume (1) because we can't do anything about (2) anyway
+                return -1;
+            }
+        }
+    }
+
+    // package private for testing
+    long readSnapshotIndexLatestBlob() throws IOException {
+        try (InputStream blob = snapshotsBlobContainer.readBlob(SNAPSHOTS_INDEX_LATEST_BLOB)) {
+            BytesStreamOutput out = new BytesStreamOutput();
+            Streams.copy(blob, out);
+            return Numbers.bytesToLong(out.bytes().toBytesRef());
+        }
+    }
+
+    private long listBlobsToGetLatestIndexId() throws IOException {
+        Map<String, BlobMetaData> blobs = snapshotsBlobContainer.listBlobsByPrefix(SNAPSHOTS_FILE_PREFIX);
+        long latest = -1;
+        if (blobs.isEmpty()) {
+            // no snapshot index blobs have been written yet
+            return latest;
+        }
+        for (final BlobMetaData blobMetaData : blobs.values()) {
+            final String blobName = blobMetaData.name();
+            try {
+                final long curr = Long.parseLong(blobName.substring(SNAPSHOTS_FILE_PREFIX.length()));
+                latest = Math.max(latest, curr);
+            } catch (NumberFormatException nfe) {
+                // the index- blob wasn't of the format index-N where N is a number,
+                // no idea what this blob is but it doesn't belong in the repository!
+                logger.debug("[{}] Unknown blob in the repository: {}", repositoryName, blobName);
+            }
+        }
+        return latest;
+    }
+
+    private void writeAtomic(final String blobName, final BytesReference bytesRef) throws IOException {
+        final String tempBlobName = "pending-" + blobName;
+        snapshotsBlobContainer.writeBlob(tempBlobName, bytesRef);
+        try {
+            snapshotsBlobContainer.move(tempBlobName, blobName);
+        } catch (IOException ex) {
+            // Move failed - try cleaning up
+            snapshotsBlobContainer.deleteBlob(tempBlobName);
+            throw ex;
+        }
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java b/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
index b805727fd9c..0eb3006c5c3 100644
--- a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.repositories.uri;
 
-import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.blobstore.url.URLBlobStore;
@@ -42,7 +41,6 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.function.Function;
-import java.util.function.Predicate;
 
 /**
  * Read-only URL-based implementation of the BlobStoreRepository
@@ -69,11 +67,6 @@ public class URLRepository extends BlobStoreRepository {
         new Setting<>("repositories.url.url", (s) -> s.get("repositories.uri.url", "http:"), URLRepository::parseURL,
             Property.NodeScope);
 
-    public static final Setting<Boolean> LIST_DIRECTORIES_SETTING =
-        Setting.boolSetting("list_directories", true, Property.NodeScope);
-    public static final Setting<Boolean> REPOSITORIES_LIST_DIRECTORIES_SETTING =
-        Setting.boolSetting("repositories.uri.list_directories", true, Property.NodeScope);
-
     private final List<String> supportedProtocols;
 
     private final URIPattern[] urlWhiteList;
@@ -84,8 +77,6 @@ public class URLRepository extends BlobStoreRepository {
 
     private final BlobPath basePath;
 
-    private boolean listDirectories;
-
     /**
      * Constructs new read-only URL-based repository
      *
@@ -103,7 +94,6 @@ public class URLRepository extends BlobStoreRepository {
         supportedProtocols = SUPPORTED_PROTOCOLS_SETTING.get(settings);
         urlWhiteList = ALLOWED_URLS_SETTING.get(settings).toArray(new URIPattern[]{});
         this.environment = environment;
-        listDirectories = LIST_DIRECTORIES_SETTING.exists(repositorySettings.settings()) ? LIST_DIRECTORIES_SETTING.get(repositorySettings.settings()) : REPOSITORIES_LIST_DIRECTORIES_SETTING.get(settings);
 
         URL url = URL_SETTING.exists(repositorySettings.settings()) ? URL_SETTING.get(repositorySettings.settings()) : REPOSITORIES_URL_SETTING.get(settings);
         URL normalizedURL = checkURL(url);
@@ -124,19 +114,6 @@ public class URLRepository extends BlobStoreRepository {
         return basePath;
     }
 
-    @Override
-    public List<SnapshotId> snapshots() {
-        if (listDirectories) {
-            return super.snapshots();
-        } else {
-            try {
-                return readSnapshotList();
-            } catch (IOException ex) {
-                throw new RepositoryException(repositoryName, "failed to get snapshot list in repository", ex);
-            }
-        }
-    }
-
     /**
      * Makes sure that the url is white listed or if it points to the local file system it matches one on of the root path in path.repo
      */
diff --git a/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java b/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
index 3d46c0bbacf..17e2481b7d0 100644
--- a/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
+++ b/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRes
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.OutputStreamStreamOutput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -46,7 +45,6 @@ import java.util.List;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.blobId;
-import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.parseNameUUIDFromBlobName;
 import static org.hamcrest.Matchers.equalTo;
 
 /**
@@ -108,7 +106,90 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
         assertThat(snapshotIds, equalTo(originalSnapshots));
     }
 
-    public void testSnapshotIndexFile() throws Exception {
+    public void testReadAndWriteSnapshotsThroughIndexFile() throws Exception {
+        final BlobStoreRepository repository = setupRepo();
+
+        // write to and read from a snapshot file with no entries
+        assertThat(repository.snapshots().size(), equalTo(0));
+        repository.writeSnapshotsToIndexGen(Collections.emptyList());
+        assertThat(repository.snapshots().size(), equalTo(0));
+
+        // write to and read from a snapshot file with a random number of entries
+        final int numSnapshots = randomIntBetween(1, 1000);
+        final List<SnapshotId> snapshotIds = new ArrayList<>(numSnapshots);
+        for (int i = 0; i < numSnapshots; i++) {
+            snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
+        }
+        repository.writeSnapshotsToIndexGen(snapshotIds);
+        assertThat(repository.snapshots(), equalTo(snapshotIds));
+    }
+
+    public void testIndexGenerationalFiles() throws Exception {
+        final BlobStoreRepository repository = setupRepo();
+
+        // write to index generational file
+        final int numSnapshots = randomIntBetween(1, 1000);
+        final List<SnapshotId> snapshotIds = new ArrayList<>(numSnapshots);
+        for (int i = 0; i < numSnapshots; i++) {
+            snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
+        }
+        repository.writeSnapshotsToIndexGen(snapshotIds);
+        assertThat(Sets.newHashSet(repository.readSnapshotsFromIndex()), equalTo(Sets.newHashSet(snapshotIds)));
+        assertThat(repository.latestIndexBlobId(), equalTo(0L));
+        assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(0L));
+
+        // adding more and writing to a new index generational file
+        for (int i = 0; i < 10; i++) {
+            snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
+        }
+        repository.writeSnapshotsToIndexGen(snapshotIds);
+        assertThat(Sets.newHashSet(repository.readSnapshotsFromIndex()), equalTo(Sets.newHashSet(snapshotIds)));
+        assertThat(repository.latestIndexBlobId(), equalTo(1L));
+        assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(1L));
+
+        // removing a snapshot adn writing to a new index generational file
+        snapshotIds.remove(0);
+        repository.writeSnapshotsToIndexGen(snapshotIds);
+        assertThat(Sets.newHashSet(repository.readSnapshotsFromIndex()), equalTo(Sets.newHashSet(snapshotIds)));
+        assertThat(repository.latestIndexBlobId(), equalTo(2L));
+        assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(2L));
+    }
+
+    public void testOldIndexFileFormat() throws Exception {
+        final BlobStoreRepository repository = setupRepo();
+
+        // write old index file format
+        final int numOldSnapshots = randomIntBetween(1, 50);
+        final List<SnapshotId> snapshotIds = new ArrayList<>();
+        for (int i = 0; i < numOldSnapshots; i++) {
+            snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), SnapshotId.UNASSIGNED_UUID));
+        }
+        writeOldFormat(repository, snapshotIds.stream().map(SnapshotId::getName).collect(Collectors.toList()));
+        assertThat(Sets.newHashSet(repository.snapshots()), equalTo(Sets.newHashSet(snapshotIds)));
+
+        // write to and read from a snapshot file with a random number of new entries added
+        final int numSnapshots = randomIntBetween(1, 1000);
+        for (int i = 0; i < numSnapshots; i++) {
+            snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
+        }
+        repository.writeSnapshotsToIndexGen(snapshotIds);
+        assertThat(Sets.newHashSet(repository.snapshots()), equalTo(Sets.newHashSet(snapshotIds)));
+    }
+
+    public void testBlobId() {
+        SnapshotId snapshotId = new SnapshotId("abc123", SnapshotId.UNASSIGNED_UUID);
+        assertThat(blobId(snapshotId), equalTo("abc123")); // just the snapshot name
+        snapshotId = new SnapshotId("abc-123", SnapshotId.UNASSIGNED_UUID);
+        assertThat(blobId(snapshotId), equalTo("abc-123")); // just the snapshot name
+        String uuid = UUIDs.randomBase64UUID();
+        snapshotId = new SnapshotId("abc123", uuid);
+        assertThat(blobId(snapshotId), equalTo("abc123-" + uuid)); // snapshot name + '-' + uuid
+        uuid = UUIDs.randomBase64UUID();
+        snapshotId = new SnapshotId("abc-123", uuid);
+        assertThat(blobId(snapshotId), equalTo("abc-123-" + uuid)); // snapshot name + '-' + uuid
+    }
+
+    private BlobStoreRepository setupRepo() {
         final Client client = client();
         final Path location = ESIntegTestCase.randomRepoPath(node().settings());
         final String repositoryName = "test-repo";
@@ -123,89 +204,7 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
         final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class);
         @SuppressWarnings("unchecked") final BlobStoreRepository repository =
             (BlobStoreRepository) repositoriesService.repository(repositoryName);
-
-        // write to and read from a snapshot file with no entries
-        repository.writeSnapshotList(Collections.emptyList());
-        List<SnapshotId> readSnapshotIds = repository.readSnapshotList();
-        assertThat(readSnapshotIds.size(), equalTo(0));
-
-        // write to and read from a snapshot file with a random number of entries
-        final int numSnapshots = randomIntBetween(1, 1000);
-        final List<SnapshotId> snapshotIds = new ArrayList<>(numSnapshots);
-        for (int i = 0; i < numSnapshots; i++) {
-            snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
-        }
-        repository.writeSnapshotList(snapshotIds);
-        readSnapshotIds = repository.readSnapshotList();
-        assertThat(readSnapshotIds, equalTo(snapshotIds));
-    }
-
-    public void testOldIndexFileFormat() throws Exception {
-        final Client client = client();
-        final Path location = ESIntegTestCase.randomRepoPath(node().settings());
-        final String repositoryName = "test-repo";
-
-        PutRepositoryResponse putRepositoryResponse =
-            client.admin().cluster().preparePutRepository(repositoryName)
-                .setType("fs")
-                .setSettings(Settings.builder().put(node().settings()).put("location", location))
-                .get();
-        assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
-
-        final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class);
-        @SuppressWarnings("unchecked") final BlobStoreRepository repository =
-            (BlobStoreRepository) repositoriesService.repository(repositoryName);
-
-        // write old index file format
-        final int numOldSnapshots = randomIntBetween(1, 50);
-        final List<SnapshotId> snapshotIds = new ArrayList<>();
-        for (int i = 0; i < numOldSnapshots; i++) {
-            snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), SnapshotId.UNASSIGNED_UUID));
-        }
-        writeOldFormat(repository, snapshotIds.stream().map(SnapshotId::getName).collect(Collectors.toList()));
-        List<SnapshotId> readSnapshotIds = repository.readSnapshotList();
-        assertThat(Sets.newHashSet(readSnapshotIds), equalTo(Sets.newHashSet(snapshotIds)));
-
-        // write to and read from a snapshot file with a random number of new entries added
-        final int numSnapshots = randomIntBetween(1, 1000);
-        for (int i = 0; i < numSnapshots; i++) {
-            snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
-        }
-        repository.writeSnapshotList(snapshotIds);
-        readSnapshotIds = repository.readSnapshotList();
-        assertThat(Sets.newHashSet(readSnapshotIds), equalTo(Sets.newHashSet(snapshotIds)));
-    }
-
-    public void testParseUUIDFromBlobName() {
-        String blobStr = "abc123";
-        Tuple<String, String> pair = parseNameUUIDFromBlobName(blobStr);
-        assertThat(pair.v1(), equalTo(blobStr)); // snapshot name
-        assertThat(pair.v2(), equalTo(SnapshotId.UNASSIGNED_UUID)); // snapshot uuid
-        blobStr = "abcefghijklmnopqrstuvwxyz";
-        pair = parseNameUUIDFromBlobName(blobStr);
-        assertThat(pair.v1(), equalTo(blobStr));
-        assertThat(pair.v2(), equalTo(SnapshotId.UNASSIGNED_UUID));
-        blobStr = "abc123-xyz"; // not enough characters after '-' to have a uuid
-        pair = parseNameUUIDFromBlobName(blobStr);
-        assertThat(pair.v1(), equalTo(blobStr));
-        assertThat(pair.v2(), equalTo(SnapshotId.UNASSIGNED_UUID));
-        blobStr = "abc123-a1b2c3d4e5f6g7h8i9j0k1";
-        pair = parseNameUUIDFromBlobName(blobStr);
-        assertThat(pair.v1(), equalTo("abc123"));
-        assertThat(pair.v2(), equalTo("a1b2c3d4e5f6g7h8i9j0k1"));
-    }
-
-    public void testBlobId() {
-        SnapshotId snapshotId = new SnapshotId("abc123", SnapshotId.UNASSIGNED_UUID);
-        assertThat(blobId(snapshotId), equalTo("abc123")); // just the snapshot name
-        snapshotId = new SnapshotId("abc-123", SnapshotId.UNASSIGNED_UUID);
-        assertThat(blobId(snapshotId), equalTo("abc-123")); // just the snapshot name
-        String uuid = UUIDs.randomBase64UUID();
-        snapshotId = new SnapshotId("abc123", uuid);
-        assertThat(blobId(snapshotId), equalTo("abc123-" + uuid)); // snapshot name + '-' + uuid
-        uuid = UUIDs.randomBase64UUID();
-        snapshotId = new SnapshotId("abc-123", uuid);
-        assertThat(blobId(snapshotId), equalTo("abc-123-" + uuid)); // snapshot name + '-' + uuid
+        return repository;
     }
 
     private void writeOldFormat(final BlobStoreRepository repository, final List<String> snapshotNames) throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
index 15623825887..a25c8e0593e 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
@@ -284,8 +284,8 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
         countDownLatch.await();
     }
 
-    private static interface ClusterStateUpdater {
-        public ClusterState execute(ClusterState currentState) throws Exception;
+    private interface ClusterStateUpdater {
+        ClusterState execute(ClusterState currentState) throws Exception;
     }
 
     public void testSnapshotDuringNodeShutdown() throws Exception {
@@ -392,8 +392,11 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
 
         logger.info("--> making sure that snapshot no longer exists");
         assertThrows(client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").execute(), SnapshotMissingException.class);
-        // Subtract index file from the count
-        assertThat("not all files were deleted during snapshot cancellation", numberOfFilesBeforeSnapshot, equalTo(numberOfFiles(repo) - 1));
+        // Subtract three files that will remain in the repository:
+        //   (1) index-1
+        //   (2) index-0 (because we keep the previous version) and
+        //   (3) index-latest
+        assertThat("not all files were deleted during snapshot cancellation", numberOfFilesBeforeSnapshot, equalTo(numberOfFiles(repo) - 3));
         logger.info("--> done");
     }
 
diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
index 19b46710fea..e1efbdfaf81 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -818,8 +818,9 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
 
         logger.info("--> delete the last snapshot");
         client.admin().cluster().prepareDeleteSnapshot("test-repo", lastSnapshot).get();
-        logger.info("--> make sure that number of files is back to what it was when the first snapshot was made");
-        assertThat(numberOfFiles(repo), equalTo(numberOfFiles[0]));
+        logger.info("--> make sure that number of files is back to what it was when the first snapshot was made, " +
+                    "plus one because one backup index-N file should remain");
+        assertThat(numberOfFiles(repo), equalTo(numberOfFiles[0] + 1));
     }
 
     public void testDeleteSnapshotWithMissingIndexAndShardMetadata() throws Exception {
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
index 2809b8588f1..510b6d20f07 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
@@ -34,6 +34,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.repositories.RepositoryName;
 import org.elasticsearch.repositories.RepositorySettings;
 
+import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.URISyntaxException;
@@ -127,7 +128,7 @@ public class AzureBlobStore extends AbstractComponent implements BlobStore {
         this.client.deleteBlob(this.accountName, this.locMode, container, blob);
     }
 
-    public InputStream getInputStream(String container, String blob) throws URISyntaxException, StorageException
+    public InputStream getInputStream(String container, String blob) throws URISyntaxException, StorageException, IOException
     {
         return this.client.getInputStream(this.accountName, this.locMode, container, blob);
     }
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
index 13db36aeb5a..137ff20efbe 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
@@ -29,6 +29,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 
+import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.URISyntaxException;
@@ -75,7 +76,7 @@ public interface AzureStorageService {
     void deleteBlob(String account, LocationMode mode, String container, String blob) throws URISyntaxException, StorageException;
 
     InputStream getInputStream(String account, LocationMode mode, String container, String blob)
-        throws URISyntaxException, StorageException;
+        throws URISyntaxException, StorageException, IOException;
 
     OutputStream getOutputStream(String account, LocationMode mode, String container, String blob)
         throws URISyntaxException, StorageException;
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java
index 8160c560325..5d4f9c6f69b 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java
@@ -31,6 +31,8 @@ import org.elasticsearch.common.settings.Settings;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.URISyntaxException;
@@ -79,7 +81,10 @@ public class AzureStorageServiceMock extends AbstractLifecycleComponent<AzureSto
     }
 
     @Override
-    public InputStream getInputStream(String account, LocationMode mode, String container, String blob) {
+    public InputStream getInputStream(String account, LocationMode mode, String container, String blob) throws IOException {
+        if (!blobExists(account, mode, container, blob)) {
+            throw new FileNotFoundException("missing blob [" + blob + "]");
+        }
         return new ByteArrayInputStream(blobs.get(blob).toByteArray());
     }
 

From 822c99536765f66758ecd42ae8669603940acbfb Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Fri, 1 Jul 2016 16:17:42 -0700
Subject: [PATCH 1207/1311] Internal: Remove generics from LifecycleComponent

The only reason for LifecycleComponent taking a generic type was so that
it could return that type on its start and stop methods. However, this
chaining has no practical necessity. Instead, start and stop can be
void, and a whole bunch of confusing generics disappear.
---
 .../cluster/NodeConnectionsService.java              |  2 +-
 .../cluster/routing/DelayedAllocationService.java    |  2 +-
 .../cluster/routing/RoutingService.java              |  2 +-
 .../cluster/service/ClusterService.java              |  2 +-
 .../common/component/AbstractLifecycleComponent.java | 12 +++++-------
 .../common/component/LifecycleComponent.java         |  6 +++---
 .../java/org/elasticsearch/discovery/Discovery.java  |  2 +-
 .../discovery/local/LocalDiscovery.java              |  2 +-
 .../elasticsearch/discovery/zen/ZenDiscovery.java    |  2 +-
 .../elasticsearch/discovery/zen/ping/ZenPing.java    |  2 +-
 .../discovery/zen/ping/ZenPingService.java           |  2 +-
 .../discovery/zen/ping/unicast/UnicastZenPing.java   |  2 +-
 .../org/elasticsearch/gateway/GatewayService.java    |  2 +-
 .../main/java/org/elasticsearch/http/HttpServer.java |  2 +-
 .../org/elasticsearch/http/HttpServerTransport.java  |  2 +-
 .../http/netty/NettyHttpServerTransport.java         |  2 +-
 .../org/elasticsearch/indices/IndicesService.java    |  2 +-
 .../indices/breaker/CircuitBreakerService.java       |  2 +-
 .../indices/cluster/IndicesClusterStateService.java  |  2 +-
 .../elasticsearch/indices/ttl/IndicesTTLService.java |  2 +-
 .../org/elasticsearch/monitor/MonitorService.java    |  2 +-
 .../monitor/jvm/JvmGcMonitorService.java             |  2 +-
 .../org/elasticsearch/repositories/Repository.java   |  2 +-
 .../repositories/blobstore/BlobStoreRepository.java  |  2 +-
 .../java/org/elasticsearch/rest/RestController.java  |  2 +-
 .../java/org/elasticsearch/search/SearchService.java |  2 +-
 .../snapshots/SnapshotShardsService.java             |  2 +-
 .../elasticsearch/snapshots/SnapshotsService.java    |  2 +-
 .../org/elasticsearch/transport/TcpTransport.java    |  2 +-
 .../java/org/elasticsearch/transport/Transport.java  |  2 +-
 .../elasticsearch/transport/TransportService.java    |  2 +-
 .../transport/local/LocalTransport.java              |  2 +-
 .../java/org/elasticsearch/tribe/TribeService.java   |  2 +-
 .../watcher/ResourceWatcherService.java              |  2 +-
 .../client/transport/FailAndRetryMockTransport.java  | 12 +++---------
 .../cluster/NodeConnectionsServiceTests.java         | 12 +++---------
 .../cluster/service/ClusterServiceIT.java            |  2 +-
 .../common/network/NetworkModuleTests.java           |  2 +-
 .../java/org/elasticsearch/http/HttpServerTests.java |  2 +-
 .../http/netty/NettyHttpServerPipeliningTests.java   |  4 ++--
 .../java/org/elasticsearch/test/NoopDiscovery.java   | 12 +++---------
 .../classic/management/AzureComputeServiceImpl.java  |  2 +-
 .../management/AzureComputeServiceAbstractMock.java  |  2 +-
 .../elasticsearch/cloud/aws/AwsEc2ServiceImpl.java   |  2 +-
 .../discovery/ec2/AwsEc2ServiceMock.java             |  2 +-
 .../elasticsearch/cloud/gce/GceComputeService.java   |  2 +-
 .../cloud/gce/GceComputeServiceImpl.java             |  2 +-
 .../cloud/azure/blobstore/AzureBlobStore.java        |  3 ++-
 .../cloud/azure/storage/AzureStorageService.java     |  2 +-
 .../cloud/azure/storage/AzureStorageServiceImpl.java |  2 +-
 .../cloud/azure/storage/AzureStorageServiceMock.java |  2 +-
 .../org/elasticsearch/cloud/aws/AwsS3Service.java    |  2 +-
 .../cloud/aws/InternalAwsS3Service.java              |  2 +-
 .../test/transport/CapturingTransport.java           | 12 +++---------
 .../test/transport/MockTransportService.java         |  6 ++----
 55 files changed, 72 insertions(+), 99 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java b/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
index 5fe42a7b163..79c4db9dcad 100644
--- a/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
@@ -46,7 +46,7 @@ import static org.elasticsearch.common.settings.Setting.positiveTimeSetting;
  * to pings. This is done by {@link org.elasticsearch.discovery.zen.fd.NodesFaultDetection}. Master fault detection
  * is done by {@link org.elasticsearch.discovery.zen.fd.MasterFaultDetection}.
  */
-public class NodeConnectionsService extends AbstractLifecycleComponent<NodeConnectionsService> {
+public class NodeConnectionsService extends AbstractLifecycleComponent {
 
     public static final Setting<TimeValue> CLUSTER_NODE_RECONNECT_INTERVAL_SETTING =
             positiveTimeSetting("cluster.nodes.reconnect_interval", TimeValue.timeValueSeconds(10), Property.NodeScope);
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/DelayedAllocationService.java b/core/src/main/java/org/elasticsearch/cluster/routing/DelayedAllocationService.java
index e79884cf9c9..ea0b7e3094c 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/DelayedAllocationService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/DelayedAllocationService.java
@@ -51,7 +51,7 @@ import java.util.concurrent.atomic.AtomicReference;
  * {@link AllocationService#removeDelayMarkers(RoutingAllocation)}, triggering yet
  * another cluster change event.
  */
-public class DelayedAllocationService extends AbstractLifecycleComponent<DelayedAllocationService> implements ClusterStateListener {
+public class DelayedAllocationService extends AbstractLifecycleComponent implements ClusterStateListener {
 
     static final String CLUSTER_UPDATE_TASK_SOURCE = "delayed_allocation_reroute";
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
index 78e7e15d389..7bd2c2219b5 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
@@ -44,7 +44,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
  * actions.
  * </p>
  */
-public class RoutingService extends AbstractLifecycleComponent<RoutingService> {
+public class RoutingService extends AbstractLifecycleComponent {
 
     private static final String CLUSTER_UPDATE_TASK_SOURCE = "cluster_reroute";
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
index 1480ef9a504..accb9304d25 100644
--- a/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
@@ -88,7 +88,7 @@ import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadF
 /**
  *
  */
-public class ClusterService extends AbstractLifecycleComponent<ClusterService> {
+public class ClusterService extends AbstractLifecycleComponent {
 
     public static final Setting<TimeValue> CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING =
             Setting.positiveTimeSetting("cluster.service.slow_task_logging_threshold", TimeValue.timeValueSeconds(30),
diff --git a/core/src/main/java/org/elasticsearch/common/component/AbstractLifecycleComponent.java b/core/src/main/java/org/elasticsearch/common/component/AbstractLifecycleComponent.java
index 11dbca72973..6f1534b57d8 100644
--- a/core/src/main/java/org/elasticsearch/common/component/AbstractLifecycleComponent.java
+++ b/core/src/main/java/org/elasticsearch/common/component/AbstractLifecycleComponent.java
@@ -27,7 +27,7 @@ import java.util.concurrent.CopyOnWriteArrayList;
 /**
  *
  */
-public abstract class AbstractLifecycleComponent<T> extends AbstractComponent implements LifecycleComponent<T> {
+public abstract class AbstractLifecycleComponent extends AbstractComponent implements LifecycleComponent {
 
     protected final Lifecycle lifecycle = new Lifecycle();
 
@@ -58,9 +58,9 @@ public abstract class AbstractLifecycleComponent<T> extends AbstractComponent im
 
     @SuppressWarnings({"unchecked"})
     @Override
-    public T start() {
+    public void start() {
         if (!lifecycle.canMoveToStarted()) {
-            return (T) this;
+            return;
         }
         for (LifecycleListener listener : listeners) {
             listener.beforeStart();
@@ -70,16 +70,15 @@ public abstract class AbstractLifecycleComponent<T> extends AbstractComponent im
         for (LifecycleListener listener : listeners) {
             listener.afterStart();
         }
-        return (T) this;
     }
 
     protected abstract void doStart();
 
     @SuppressWarnings({"unchecked"})
     @Override
-    public T stop() {
+    public void stop() {
         if (!lifecycle.canMoveToStopped()) {
-            return (T) this;
+            return;
         }
         for (LifecycleListener listener : listeners) {
             listener.beforeStop();
@@ -89,7 +88,6 @@ public abstract class AbstractLifecycleComponent<T> extends AbstractComponent im
         for (LifecycleListener listener : listeners) {
             listener.afterStop();
         }
-        return (T) this;
     }
 
     protected abstract void doStop();
diff --git a/core/src/main/java/org/elasticsearch/common/component/LifecycleComponent.java b/core/src/main/java/org/elasticsearch/common/component/LifecycleComponent.java
index 2933bffa630..b57486105d2 100644
--- a/core/src/main/java/org/elasticsearch/common/component/LifecycleComponent.java
+++ b/core/src/main/java/org/elasticsearch/common/component/LifecycleComponent.java
@@ -24,7 +24,7 @@ import org.elasticsearch.common.lease.Releasable;
 /**
  *
  */
-public interface LifecycleComponent<T> extends Releasable {
+public interface LifecycleComponent extends Releasable {
 
     Lifecycle.State lifecycleState();
 
@@ -32,7 +32,7 @@ public interface LifecycleComponent<T> extends Releasable {
 
     void removeLifecycleListener(LifecycleListener listener);
 
-    T start();
+    void start();
 
-    T stop();
+    void stop();
 }
diff --git a/core/src/main/java/org/elasticsearch/discovery/Discovery.java b/core/src/main/java/org/elasticsearch/discovery/Discovery.java
index 778e2d15053..751f5480ca8 100644
--- a/core/src/main/java/org/elasticsearch/discovery/Discovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/Discovery.java
@@ -34,7 +34,7 @@ import java.io.IOException;
  * state to all nodes, electing a master of the cluster that raises cluster state change
  * events.
  */
-public interface Discovery extends LifecycleComponent<Discovery> {
+public interface Discovery extends LifecycleComponent {
 
     DiscoveryNode localNode();
 
diff --git a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
index a711427de77..7ba207284b2 100644
--- a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
@@ -56,7 +56,7 @@ import static org.elasticsearch.cluster.ClusterState.Builder;
 /**
  *
  */
-public class LocalDiscovery extends AbstractLifecycleComponent<Discovery> implements Discovery {
+public class LocalDiscovery extends AbstractLifecycleComponent implements Discovery {
 
     private static final LocalDiscovery[] NO_MEMBERS = new LocalDiscovery[0];
 
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
index 0efbe64196e..6ebdfa7e6e6 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
@@ -86,7 +86,7 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 /**
  *
  */
-public class ZenDiscovery extends AbstractLifecycleComponent<Discovery> implements Discovery, PingContextProvider {
+public class ZenDiscovery extends AbstractLifecycleComponent implements Discovery, PingContextProvider {
 
     public static final Setting<TimeValue> PING_TIMEOUT_SETTING =
         Setting.positiveTimeSetting("discovery.zen.ping_timeout", timeValueSeconds(3), Property.NodeScope);
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ping/ZenPing.java b/core/src/main/java/org/elasticsearch/discovery/zen/ping/ZenPing.java
index 06db6ab3d5c..5a9f5f463e2 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ping/ZenPing.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ping/ZenPing.java
@@ -32,7 +32,7 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicLong;
 
-public interface ZenPing extends LifecycleComponent<ZenPing> {
+public interface ZenPing extends LifecycleComponent {
 
     void setPingContextProvider(PingContextProvider contextProvider);
 
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ping/ZenPingService.java b/core/src/main/java/org/elasticsearch/discovery/zen/ping/ZenPingService.java
index 6933d8c37b5..bd5855666ac 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ping/ZenPingService.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ping/ZenPingService.java
@@ -33,7 +33,7 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
-public class ZenPingService extends AbstractLifecycleComponent<ZenPing> implements ZenPing {
+public class ZenPingService extends AbstractLifecycleComponent implements ZenPing {
 
     private List<ZenPing> zenPings = Collections.emptyList();
 
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
index 06fc9eae13d..d010587dda4 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
@@ -86,7 +86,7 @@ import static org.elasticsearch.discovery.zen.ping.ZenPing.PingResponse.readPing
 /**
  *
  */
-public class UnicastZenPing extends AbstractLifecycleComponent<ZenPing> implements ZenPing {
+public class UnicastZenPing extends AbstractLifecycleComponent implements ZenPing {
 
     public static final String ACTION_NAME = "internal:discovery/zen/unicast";
     public static final Setting<List<String>> DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING =
diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayService.java b/core/src/main/java/org/elasticsearch/gateway/GatewayService.java
index 8bd1dcbd6ea..746d70db009 100644
--- a/core/src/main/java/org/elasticsearch/gateway/GatewayService.java
+++ b/core/src/main/java/org/elasticsearch/gateway/GatewayService.java
@@ -52,7 +52,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 /**
  *
  */
-public class GatewayService extends AbstractLifecycleComponent<GatewayService> implements ClusterStateListener {
+public class GatewayService extends AbstractLifecycleComponent implements ClusterStateListener {
 
     public static final Setting<Integer> EXPECTED_NODES_SETTING =
         Setting.intSetting("gateway.expected_nodes", -1, -1, Property.NodeScope);
diff --git a/core/src/main/java/org/elasticsearch/http/HttpServer.java b/core/src/main/java/org/elasticsearch/http/HttpServer.java
index 63bcd761eb3..8e3f15c5a24 100644
--- a/core/src/main/java/org/elasticsearch/http/HttpServer.java
+++ b/core/src/main/java/org/elasticsearch/http/HttpServer.java
@@ -51,7 +51,7 @@ import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR;
 /**
  * A component to serve http requests, backed by rest handlers.
  */
-public class HttpServer extends AbstractLifecycleComponent<HttpServer> implements HttpServerAdapter {
+public class HttpServer extends AbstractLifecycleComponent implements HttpServerAdapter {
     private final HttpServerTransport transport;
 
     private final RestController restController;
diff --git a/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java b/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java
index fa75f1dcb1e..ab8b2bd1797 100644
--- a/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java
+++ b/core/src/main/java/org/elasticsearch/http/HttpServerTransport.java
@@ -25,7 +25,7 @@ import org.elasticsearch.common.transport.BoundTransportAddress;
 /**
  *
  */
-public interface HttpServerTransport extends LifecycleComponent<HttpServerTransport> {
+public interface HttpServerTransport extends LifecycleComponent {
 
     BoundTransportAddress boundAddress();
 
diff --git a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java
index 77c140ce7ce..6578628107e 100644
--- a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java
+++ b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpServerTransport.java
@@ -113,7 +113,7 @@ import static org.elasticsearch.http.netty.cors.CorsHandler.ANY_ORIGIN;
 /**
  *
  */
-public class NettyHttpServerTransport extends AbstractLifecycleComponent<HttpServerTransport> implements HttpServerTransport {
+public class NettyHttpServerTransport extends AbstractLifecycleComponent implements HttpServerTransport {
 
     static {
         NettyUtils.setup();
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index 27387d7daaa..fadd10eae58 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -135,7 +135,7 @@ import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList;
 /**
  *
  */
-public class IndicesService extends AbstractLifecycleComponent<IndicesService>
+public class IndicesService extends AbstractLifecycleComponent
     implements IndicesClusterStateService.AllocatedIndices<IndexShard, IndexService>, IndexService.ShardStoreDeleter {
 
     public static final String INDICES_SHARDS_CLOSED_TIMEOUT = "indices.shards_closed_timeout";
diff --git a/core/src/main/java/org/elasticsearch/indices/breaker/CircuitBreakerService.java b/core/src/main/java/org/elasticsearch/indices/breaker/CircuitBreakerService.java
index 05bf8472ac9..93e5ad74ad8 100644
--- a/core/src/main/java/org/elasticsearch/indices/breaker/CircuitBreakerService.java
+++ b/core/src/main/java/org/elasticsearch/indices/breaker/CircuitBreakerService.java
@@ -27,7 +27,7 @@ import org.elasticsearch.common.settings.Settings;
  * Interface for Circuit Breaker services, which provide breakers to classes
  * that load field data.
  */
-public abstract class CircuitBreakerService extends AbstractLifecycleComponent<CircuitBreakerService> {
+public abstract class CircuitBreakerService extends AbstractLifecycleComponent {
 
     protected CircuitBreakerService(Settings settings) {
         super(settings);
diff --git a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
index c0bfedc47ca..b5644ac3bb0 100644
--- a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
+++ b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
@@ -82,7 +82,7 @@ import java.util.concurrent.TimeUnit;
 /**
  *
  */
-public class IndicesClusterStateService extends AbstractLifecycleComponent<IndicesClusterStateService> implements ClusterStateListener {
+public class IndicesClusterStateService extends AbstractLifecycleComponent implements ClusterStateListener {
 
     final AllocatedIndices<? extends Shard, ? extends AllocatedIndex<? extends Shard>> indicesService;
     private final ClusterService clusterService;
diff --git a/core/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java b/core/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java
index 8cc479aa4e7..b837839ce86 100644
--- a/core/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java
+++ b/core/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java
@@ -66,7 +66,7 @@ import java.util.concurrent.locks.ReentrantLock;
 /**
  * A node level service that delete expired docs on node primary shards.
  */
-public class IndicesTTLService extends AbstractLifecycleComponent<IndicesTTLService> {
+public class IndicesTTLService extends AbstractLifecycleComponent {
 
     public static final Setting<TimeValue> INDICES_TTL_INTERVAL_SETTING =
         Setting.positiveTimeSetting("indices.ttl.interval", TimeValue.timeValueSeconds(60),
diff --git a/core/src/main/java/org/elasticsearch/monitor/MonitorService.java b/core/src/main/java/org/elasticsearch/monitor/MonitorService.java
index 6ae9e2a5b67..59434be7d93 100644
--- a/core/src/main/java/org/elasticsearch/monitor/MonitorService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/MonitorService.java
@@ -31,7 +31,7 @@ import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
 
-public class MonitorService extends AbstractLifecycleComponent<MonitorService> {
+public class MonitorService extends AbstractLifecycleComponent {
 
     private final JvmGcMonitorService jvmGcMonitorService;
     private final OsService osService;
diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
index 69ec81c38e9..4c89c6a2a50 100644
--- a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
@@ -40,7 +40,7 @@ import java.util.function.BiFunction;
 
 import static java.util.Collections.unmodifiableMap;
 
-public class JvmGcMonitorService extends AbstractLifecycleComponent<JvmGcMonitorService> {
+public class JvmGcMonitorService extends AbstractLifecycleComponent {
 
     private final ThreadPool threadPool;
     private final boolean enabled;
diff --git a/core/src/main/java/org/elasticsearch/repositories/Repository.java b/core/src/main/java/org/elasticsearch/repositories/Repository.java
index 0cfe74a0e34..47a87edd7f9 100644
--- a/core/src/main/java/org/elasticsearch/repositories/Repository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/Repository.java
@@ -47,7 +47,7 @@ import java.util.function.Predicate;
  * with possible list of failures</li>
  * </ul>
  */
-public interface Repository extends LifecycleComponent<Repository> {
+public interface Repository extends LifecycleComponent {
 
     /**
      * Reads snapshot description from repository.
diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index c1efcb7801c..6518bd1df56 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -115,7 +115,7 @@ import java.util.stream.Collectors;
  * }
  * </pre>
  */
-public abstract class BlobStoreRepository extends AbstractLifecycleComponent<Repository> implements Repository, RateLimiterListener {
+public abstract class BlobStoreRepository extends AbstractLifecycleComponent implements Repository, RateLimiterListener {
 
     private BlobContainer snapshotsBlobContainer;
 
diff --git a/core/src/main/java/org/elasticsearch/rest/RestController.java b/core/src/main/java/org/elasticsearch/rest/RestController.java
index 8059dfd7ea5..29dce98ca49 100644
--- a/core/src/main/java/org/elasticsearch/rest/RestController.java
+++ b/core/src/main/java/org/elasticsearch/rest/RestController.java
@@ -44,7 +44,7 @@ import static org.elasticsearch.rest.RestStatus.OK;
 /**
  *
  */
-public class RestController extends AbstractLifecycleComponent<RestController> {
+public class RestController extends AbstractLifecycleComponent {
     private final PathTrie<RestHandler> getHandlers = new PathTrie<>(RestUtils.REST_DECODER);
     private final PathTrie<RestHandler> postHandlers = new PathTrie<>(RestUtils.REST_DECODER);
     private final PathTrie<RestHandler> putHandlers = new PathTrie<>(RestUtils.REST_DECODER);
diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java
index bc77a01d072..ec75ae37254 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchService.java
@@ -105,7 +105,7 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueMinutes;
 /**
  *
  */
-public class SearchService extends AbstractLifecycleComponent<SearchService> implements IndexEventListener {
+public class SearchService extends AbstractLifecycleComponent implements IndexEventListener {
 
     // we can have 5 minutes here, since we make sure to clean with search requests and when shard/index closes
     public static final Setting<TimeValue> DEFAULT_KEEPALIVE_SETTING =
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
index b9d8c729223..460eaf9f819 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
@@ -75,7 +75,7 @@ import static org.elasticsearch.cluster.SnapshotsInProgress.completed;
  * This service runs on data and master nodes and controls currently snapshotted shards on these nodes. It is responsible for
  * starting and stopping shard level snapshots
  */
-public class SnapshotShardsService extends AbstractLifecycleComponent<SnapshotShardsService> implements ClusterStateListener {
+public class SnapshotShardsService extends AbstractLifecycleComponent implements ClusterStateListener {
 
     public static final String UPDATE_SNAPSHOT_ACTION_NAME = "internal:cluster/snapshot/update_snapshot";
 
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
index d6e610a184f..e242d1be732 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
@@ -98,7 +98,7 @@ import static org.elasticsearch.cluster.SnapshotsInProgress.completed;
  * notifies all {@link #snapshotCompletionListeners} that snapshot is completed, and finally calls {@link #removeSnapshotFromClusterState(Snapshot, SnapshotInfo, Throwable)} to remove snapshot from cluster state</li>
  * </ul>
  */
-public class SnapshotsService extends AbstractLifecycleComponent<SnapshotsService> implements ClusterStateListener {
+public class SnapshotsService extends AbstractLifecycleComponent implements ClusterStateListener {
 
     private final ClusterService clusterService;
 
diff --git a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
index f9b5eceb4b8..cd8063969d4 100644
--- a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
@@ -105,7 +105,7 @@ import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.new
 
 /**
  */
-public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent<Transport> implements Transport {
+public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent implements Transport {
 
     public static final String HTTP_SERVER_WORKER_THREAD_NAME_PREFIX = "http_server_worker";
     public static final String HTTP_SERVER_BOSS_THREAD_NAME_PREFIX = "http_server_boss";
diff --git a/core/src/main/java/org/elasticsearch/transport/Transport.java b/core/src/main/java/org/elasticsearch/transport/Transport.java
index 754d2105e83..6e36363741d 100644
--- a/core/src/main/java/org/elasticsearch/transport/Transport.java
+++ b/core/src/main/java/org/elasticsearch/transport/Transport.java
@@ -35,7 +35,7 @@ import java.util.Map;
 /**
  *
  */
-public interface Transport extends LifecycleComponent<Transport> {
+public interface Transport extends LifecycleComponent {
 
 
     Setting<Boolean> TRANSPORT_TCP_COMPRESS = Setting.boolSetting("transport.tcp.compress", false, Property.NodeScope);
diff --git a/core/src/main/java/org/elasticsearch/transport/TransportService.java b/core/src/main/java/org/elasticsearch/transport/TransportService.java
index e2f3ab44eea..5a62c8bad31 100644
--- a/core/src/main/java/org/elasticsearch/transport/TransportService.java
+++ b/core/src/main/java/org/elasticsearch/transport/TransportService.java
@@ -69,7 +69,7 @@ import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
 /**
  *
  */
-public class TransportService extends AbstractLifecycleComponent<TransportService> {
+public class TransportService extends AbstractLifecycleComponent {
 
     public static final String DIRECT_RESPONSE_PROFILE = ".direct";
     private static final String HANDSHAKE_ACTION_NAME = "internal:transport/handshake";
diff --git a/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java b/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
index ebb29d6feae..930e3951098 100644
--- a/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
@@ -73,7 +73,7 @@ import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.new
 /**
  *
  */
-public class LocalTransport extends AbstractLifecycleComponent<Transport> implements Transport {
+public class LocalTransport extends AbstractLifecycleComponent implements Transport {
 
     public static final String LOCAL_TRANSPORT_THREAD_NAME_PREFIX = "local_transport";
     final ThreadPool threadPool;
diff --git a/core/src/main/java/org/elasticsearch/tribe/TribeService.java b/core/src/main/java/org/elasticsearch/tribe/TribeService.java
index 259a9d443a3..5c7d6d93f7c 100644
--- a/core/src/main/java/org/elasticsearch/tribe/TribeService.java
+++ b/core/src/main/java/org/elasticsearch/tribe/TribeService.java
@@ -86,7 +86,7 @@ import static java.util.Collections.unmodifiableMap;
  * in another cluster, the conflict one will be discarded. This happens because we need to have the correct index name
  * to propagate to the relevant cluster.
  */
-public class TribeService extends AbstractLifecycleComponent<TribeService> {
+public class TribeService extends AbstractLifecycleComponent {
 
     public static final ClusterBlock TRIBE_METADATA_BLOCK = new ClusterBlock(10, "tribe node, metadata not allowed", false, false,
             RestStatus.BAD_REQUEST, EnumSet.of(ClusterBlockLevel.METADATA_READ, ClusterBlockLevel.METADATA_WRITE));
diff --git a/core/src/main/java/org/elasticsearch/watcher/ResourceWatcherService.java b/core/src/main/java/org/elasticsearch/watcher/ResourceWatcherService.java
index e66269484e0..f5c7ea56cae 100644
--- a/core/src/main/java/org/elasticsearch/watcher/ResourceWatcherService.java
+++ b/core/src/main/java/org/elasticsearch/watcher/ResourceWatcherService.java
@@ -40,7 +40,7 @@ import java.util.concurrent.ScheduledFuture;
  * registered watcher periodically. The frequency of checks can be specified using {@code resource.reload.interval} setting, which
  * defaults to {@code 60s}. The service can be disabled by setting {@code resource.reload.enabled} setting to {@code false}.
  */
-public class ResourceWatcherService extends AbstractLifecycleComponent<ResourceWatcherService> {
+public class ResourceWatcherService extends AbstractLifecycleComponent {
 
     public enum Frequency {
 
diff --git a/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java b/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java
index c085c3164a0..9d2c176dffb 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/FailAndRetryMockTransport.java
@@ -187,19 +187,13 @@ abstract class FailAndRetryMockTransport<Response extends TransportResponse> imp
     }
 
     @Override
-    public Transport start() {
-        return null;
-    }
+    public void start() {}
 
     @Override
-    public Transport stop() {
-        return null;
-    }
+    public void stop() {}
 
     @Override
-    public void close() {
-
-    }
+    public void close() {}
 
     @Override
     public Map<String, BoundTransportAddress> profileBoundAddresses() {
diff --git a/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
index 5eb5a34c44f..95dd97b4702 100644
--- a/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
@@ -253,18 +253,12 @@ public class NodeConnectionsServiceTests extends ESTestCase {
         }
 
         @Override
-        public Transport start() {
-            return null;
-        }
+        public void start() {}
 
         @Override
-        public Transport stop() {
-            return null;
-        }
+        public void stop() {}
 
         @Override
-        public void close() {
-
-        }
+        public void close() {}
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java
index 991f11a4493..7907ad3e63c 100644
--- a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java
@@ -591,7 +591,7 @@ public class ClusterServiceIT extends ESIntegTestCase {
     }
 
     @Singleton
-    public static class MasterAwareService extends AbstractLifecycleComponent<MasterAwareService> implements LocalNodeMasterListener {
+    public static class MasterAwareService extends AbstractLifecycleComponent implements LocalNodeMasterListener {
 
         private final ClusterService clusterService;
         private volatile boolean master;
diff --git a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
index 245520d65f3..131636e2011 100644
--- a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java
@@ -59,7 +59,7 @@ public class NetworkModuleTests extends ModuleTestCase {
         }
     }
 
-    static class FakeHttpTransport extends AbstractLifecycleComponent<HttpServerTransport> implements HttpServerTransport {
+    static class FakeHttpTransport extends AbstractLifecycleComponent implements HttpServerTransport {
         public FakeHttpTransport() {
             super(null);
         }
diff --git a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
index 9ed680a6fbe..9a93cb4d7ea 100644
--- a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
+++ b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java
@@ -120,7 +120,7 @@ public class HttpServerTests extends ESTestCase {
         assertEquals(0, inFlightRequestsBreaker.getUsed());
     }
 
-    private static final class TestHttpServerTransport extends AbstractLifecycleComponent<HttpServerTransport> implements
+    private static final class TestHttpServerTransport extends AbstractLifecycleComponent implements
         HttpServerTransport {
 
         public TestHttpServerTransport() {
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java
index 6fc9a4e674a..ef8621dfcd3 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpServerPipeliningTests.java
@@ -142,9 +142,9 @@ public class NettyHttpServerPipeliningTests extends ESTestCase {
         }
 
         @Override
-        public HttpServerTransport stop() {
+        public void stop() {
             executorService.shutdownNow();
-            return super.stop();
+            super.stop();
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/test/NoopDiscovery.java b/core/src/test/java/org/elasticsearch/test/NoopDiscovery.java
index 3193aaf458e..21c13816293 100644
--- a/core/src/test/java/org/elasticsearch/test/NoopDiscovery.java
+++ b/core/src/test/java/org/elasticsearch/test/NoopDiscovery.java
@@ -86,17 +86,11 @@ public class NoopDiscovery implements Discovery {
     }
 
     @Override
-    public Discovery start() {
-        return null;
-    }
+    public void start() {}
 
     @Override
-    public Discovery stop() {
-        return null;
-    }
+    public void stop() {}
 
     @Override
-    public void close() {
-
-    }
+    public void close() {}
 }
diff --git a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java
index 7d6f81ad0b6..2375db2502b 100644
--- a/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java
+++ b/plugins/discovery-azure-classic/src/main/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceImpl.java
@@ -39,7 +39,7 @@ import java.util.ServiceLoader;
 /**
  *
  */
-public class AzureComputeServiceImpl extends AbstractLifecycleComponent<AzureComputeServiceImpl>
+public class AzureComputeServiceImpl extends AbstractLifecycleComponent
     implements AzureComputeService {
 
     private final ComputeManagementClient client;
diff --git a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceAbstractMock.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceAbstractMock.java
index 33f40a9159a..02de9db6d69 100644
--- a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceAbstractMock.java
+++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/cloud/azure/classic/management/AzureComputeServiceAbstractMock.java
@@ -26,7 +26,7 @@ import org.elasticsearch.common.settings.Settings;
 /**
  *
  */
-public abstract class AzureComputeServiceAbstractMock extends AbstractLifecycleComponent<AzureComputeServiceAbstractMock>
+public abstract class AzureComputeServiceAbstractMock extends AbstractLifecycleComponent
     implements AzureComputeService {
 
     protected AzureComputeServiceAbstractMock(Settings settings) {
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java
index 5ce110487c4..2fc82a01f09 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java
@@ -49,7 +49,7 @@ import java.util.Random;
 /**
  *
  */
-public class AwsEc2ServiceImpl extends AbstractLifecycleComponent<AwsEc2Service> implements AwsEc2Service {
+public class AwsEc2ServiceImpl extends AbstractLifecycleComponent implements AwsEc2Service {
 
     public static final String EC2_METADATA_URL = "http://169.254.169.254/latest/meta-data/";
 
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceMock.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceMock.java
index f14a80f01cc..19a5c8d3b96 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceMock.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceMock.java
@@ -27,7 +27,7 @@ import org.elasticsearch.common.settings.Settings;
 
 import java.util.List;
 
-public class AwsEc2ServiceMock extends AbstractLifecycleComponent<AwsEc2Service> implements AwsEc2Service {
+public class AwsEc2ServiceMock extends AbstractLifecycleComponent implements AwsEc2Service {
 
     private int nodes;
     private List<List<Tag>> tagsList;
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java
index a6faa390e5d..00a44cd56af 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeService.java
@@ -31,7 +31,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.function.Function;
 
-public interface GceComputeService extends LifecycleComponent<GceComputeService> {
+public interface GceComputeService extends LifecycleComponent {
 
     /**
      * GCE API Version: Elasticsearch/GceCloud/1.0
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java
index 63a19919a7b..8d1090dd82c 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/cloud/gce/GceComputeServiceImpl.java
@@ -56,7 +56,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.function.Function;
 
-public class GceComputeServiceImpl extends AbstractLifecycleComponent<GceComputeService>
+public class GceComputeServiceImpl extends AbstractLifecycleComponent
     implements GceComputeService {
 
     // all settings just used for testing - not registered by default
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
index 2809b8588f1..e11797219ef 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
@@ -56,7 +56,8 @@ public class AzureBlobStore extends AbstractComponent implements BlobStore {
     public AzureBlobStore(RepositoryName name, Settings settings, RepositorySettings repositorySettings,
                           AzureStorageService client) throws URISyntaxException, StorageException {
         super(settings);
-        this.client = client.start();
+        this.client = client;
+        client.start();
         this.container = getValue(repositorySettings, Repository.CONTAINER_SETTING, Storage.CONTAINER_SETTING);
         this.repositoryName = name.getName();
         this.accountName = getValue(repositorySettings, Repository.ACCOUNT_SETTING, Storage.ACCOUNT_SETTING);
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
index 13db36aeb5a..aa0c1c2cf83 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
@@ -86,5 +86,5 @@ public interface AzureStorageService {
     void moveBlob(String account, LocationMode mode, String container, String sourceBlob, String targetBlob)
         throws URISyntaxException, StorageException;
 
-    AzureStorageService start();
+    void start();
 }
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
index 09dbc6520a1..80e62b90ad3 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
@@ -45,7 +45,7 @@ import java.net.URISyntaxException;
 import java.util.HashMap;
 import java.util.Map;
 
-public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureStorageServiceImpl>
+public class AzureStorageServiceImpl extends AbstractLifecycleComponent
     implements AzureStorageService {
 
     final AzureStorageSettings primaryStorageSettings;
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java
index 8160c560325..36351b3db88 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java
@@ -41,7 +41,7 @@ import java.util.concurrent.ConcurrentHashMap;
 /**
  * In memory storage for unit tests
  */
-public class AzureStorageServiceMock extends AbstractLifecycleComponent<AzureStorageServiceMock>
+public class AzureStorageServiceMock extends AbstractLifecycleComponent
         implements AzureStorageService {
 
     protected Map<String, ByteArrayOutputStream> blobs = new ConcurrentHashMap<>();
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
index 15cd55c823d..752a2abc3ce 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java
@@ -31,7 +31,7 @@ import java.util.function.Function;
 /**
  *
  */
-public interface AwsS3Service extends LifecycleComponent<AwsS3Service> {
+public interface AwsS3Service extends LifecycleComponent {
 
     // Global AWS settings (shared between discovery-ec2 and repository-s3)
     // Each setting starting with `cloud.aws` also exists in discovery-ec2 project. Don't forget to update
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
index 287973a3a90..1a0c2992b15 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
@@ -44,7 +44,7 @@ import java.util.Map;
 /**
  *
  */
-public class InternalAwsS3Service extends AbstractLifecycleComponent<AwsS3Service> implements AwsS3Service {
+public class InternalAwsS3Service extends AbstractLifecycleComponent implements AwsS3Service {
 
     /**
      * (acceskey, endpoint) -&gt; client
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java
index 94c30f496dd..f0f85b24f72 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java
@@ -242,19 +242,13 @@ public class CapturingTransport implements Transport {
     }
 
     @Override
-    public Transport start() {
-        return null;
-    }
+    public void start() {}
 
     @Override
-    public Transport stop() {
-        return null;
-    }
+    public void stop() {}
 
     @Override
-    public void close() {
-
-    }
+    public void close() {}
 
     @Override
     public List<String> getLocalAddresses() {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
index 7bc7f0d5254..41c27ecbf97 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
@@ -558,15 +558,13 @@ public class MockTransportService extends TransportService {
         }
 
         @Override
-        public Transport start() {
+        public void start() {
             transport.start();
-            return this;
         }
 
         @Override
-        public Transport stop() {
+        public void stop() {
             transport.stop();
-            return this;
         }
 
         @Override

From 50b97ba5f5007d6370b21b7a3eeb4d81604855a1 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Sat, 2 Jul 2016 09:27:07 +0200
Subject: [PATCH 1208/1311] Fix test assertion matching exception message

Newer versions of the URL class in JDK 9 use a different exception message when throwing a MalformedURLException due to an
unknown protocol.
---
 .../test/java/org/elasticsearch/snapshots/RepositoriesIT.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java b/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java
index 79d1497912a..980912ecb6d 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java
@@ -153,7 +153,8 @@ public class RepositoriesIT extends AbstractSnapshotIntegTestCase {
                     .get();
             fail("Shouldn't be here");
         } catch (RepositoryException ex) {
-            assertThat(ex.toString(), containsString("unsupported url protocol [netdoc]"));
+            assertThat(ex.toString(), containsString("Unable to parse URL repository setting"));
+            assertThat(ex.toString(), containsString("netdoc"));
         }
 
         logger.info("--> trying creating url repository with location that is not registered in path.repo setting");

From b4064ce43fc43fecc2b63c63c0d27d52f4656f16 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Sat, 2 Jul 2016 09:35:54 +0200
Subject: [PATCH 1209/1311] Make primary relocation handoff non-blocking
 (#19013)

Primary relocation and indexing concurrently can currently lead to a deadlock situation as indexing operations are blocked on a (bounded) thread pool during the hand-off phase between old and new primary. This change replaces blocking of indexing operations by putting operations that cannot be executed during relocation hand-off in a queue to be executed once relocation completes.

Closes #18553.
---
 .../AbstractListenableActionFuture.java       |   4 +-
 .../support/ThreadedActionListener.java       |  17 +-
 .../TransportReplicationAction.java           | 126 ++++++----
 .../concurrent/SuspendableRefContainer.java   | 117 ----------
 .../elasticsearch/index/shard/IndexShard.java |  58 +++--
 .../index/shard/IndexShardOperationsLock.java | 166 ++++++++++++++
 .../recovery/RecoverySourceHandler.java       |  17 +-
 .../recovery/RecoveryTargetService.java       |   6 -
 .../TransportMasterNodeActionTests.java       |   2 +-
 .../TransportReplicationActionTests.java      |  67 +++---
 .../SuspendableRefContainerTests.java         | 115 ----------
 .../shard/IndexShardOperationsLockTests.java  | 215 ++++++++++++++++++
 .../index/shard/IndexShardTests.java          |  97 ++++++--
 .../flush/SyncedFlushSingleNodeTests.java     |   9 +-
 .../elasticsearch/recovery/RelocationIT.java  |  17 +-
 15 files changed, 656 insertions(+), 377 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/common/util/concurrent/SuspendableRefContainer.java
 create mode 100644 core/src/main/java/org/elasticsearch/index/shard/IndexShardOperationsLock.java
 delete mode 100644 core/src/test/java/org/elasticsearch/common/util/concurrent/SuspendableRefContainerTests.java
 create mode 100644 core/src/test/java/org/elasticsearch/index/shard/IndexShardOperationsLockTests.java

diff --git a/core/src/main/java/org/elasticsearch/action/support/AbstractListenableActionFuture.java b/core/src/main/java/org/elasticsearch/action/support/AbstractListenableActionFuture.java
index d0c5af06593..faaf7183467 100644
--- a/core/src/main/java/org/elasticsearch/action/support/AbstractListenableActionFuture.java
+++ b/core/src/main/java/org/elasticsearch/action/support/AbstractListenableActionFuture.java
@@ -53,7 +53,7 @@ public abstract class AbstractListenableActionFuture<T, L> extends AdapterAction
     }
 
     public void internalAddListener(ActionListener<T> listener) {
-        listener = new ThreadedActionListener<>(logger, threadPool, ThreadPool.Names.LISTENER, listener);
+        listener = new ThreadedActionListener<>(logger, threadPool, ThreadPool.Names.LISTENER, listener, false);
         boolean executeImmediate = false;
         synchronized (this) {
             if (executedListeners) {
@@ -106,4 +106,4 @@ public abstract class AbstractListenableActionFuture<T, L> extends AdapterAction
             listener.onFailure(e);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/main/java/org/elasticsearch/action/support/ThreadedActionListener.java b/core/src/main/java/org/elasticsearch/action/support/ThreadedActionListener.java
index 1eec4c8d9e9..e7ab536387e 100644
--- a/core/src/main/java/org/elasticsearch/action/support/ThreadedActionListener.java
+++ b/core/src/main/java/org/elasticsearch/action/support/ThreadedActionListener.java
@@ -64,7 +64,7 @@ public final class ThreadedActionListener<Response> implements ActionListener<Re
             if (listener instanceof ThreadedActionListener) {
                 return listener;
             }
-            return new ThreadedActionListener<>(logger, threadPool, ThreadPool.Names.LISTENER, listener);
+            return new ThreadedActionListener<>(logger, threadPool, ThreadPool.Names.LISTENER, listener, false);
         }
     }
 
@@ -72,17 +72,25 @@ public final class ThreadedActionListener<Response> implements ActionListener<Re
     private final ThreadPool threadPool;
     private final String executor;
     private final ActionListener<Response> listener;
+    private final boolean forceExecution;
 
-    public ThreadedActionListener(ESLogger logger, ThreadPool threadPool, String executor, ActionListener<Response> listener) {
+    public ThreadedActionListener(ESLogger logger, ThreadPool threadPool, String executor, ActionListener<Response> listener,
+                                  boolean forceExecution) {
         this.logger = logger;
         this.threadPool = threadPool;
         this.executor = executor;
         this.listener = listener;
+        this.forceExecution = forceExecution;
     }
 
     @Override
     public void onResponse(final Response response) {
         threadPool.executor(executor).execute(new AbstractRunnable() {
+            @Override
+            public boolean isForceExecution() {
+                return forceExecution;
+            }
+
             @Override
             protected void doRun() throws Exception {
                 listener.onResponse(response);
@@ -98,6 +106,11 @@ public final class ThreadedActionListener<Response> implements ActionListener<Re
     @Override
     public void onFailure(final Throwable e) {
         threadPool.executor(executor).execute(new AbstractRunnable() {
+            @Override
+            public boolean isForceExecution() {
+                return forceExecution;
+            }
+
             @Override
             protected void doRun() throws Exception {
                 listener.onFailure(e);
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
index 80aa23032c5..f562224d9f7 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
@@ -42,6 +42,7 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
@@ -91,6 +92,7 @@ public abstract class TransportReplicationAction<
     private final ShardStateAction shardStateAction;
     private final WriteConsistencyLevel defaultWriteConsistencyLevel;
     private final TransportRequestOptions transportOptions;
+    private final String executor;
 
     // package private for testing
     final String transportReplicaAction;
@@ -108,6 +110,7 @@ public abstract class TransportReplicationAction<
         this.clusterService = clusterService;
         this.indicesService = indicesService;
         this.shardStateAction = shardStateAction;
+        this.executor = executor;
 
         this.transportPrimaryAction = actionName + "[p]";
         this.transportReplicaAction = actionName + "[r]";
@@ -157,7 +160,7 @@ public abstract class TransportReplicationAction<
 
     /**
      * Synchronous replica operation on nodes with replica copies. This is done under the lock form
-     * {@link #acquireReplicaOperationLock(ShardId, long)}.
+     * {@link #acquireReplicaOperationLock(ShardId, long, ActionListener)}.
      */
     protected abstract ReplicaResult shardOperationOnReplica(ReplicaRequest shardRequest);
 
@@ -235,12 +238,33 @@ public abstract class TransportReplicationAction<
         }
 
         @Override
-        public void messageReceived(Request request, TransportChannel channel, Task task) throws Exception {
-            ReplicationTask replicationTask = (ReplicationTask) task;
-            boolean success = false;
-            PrimaryShardReference primaryShardReference = getPrimaryShardReference(request.shardId());
+        public void messageReceived(Request request, TransportChannel channel, Task task) {
+            new AsyncPrimaryAction(request, channel, (ReplicationTask) task).run();
+        }
+    }
+
+    class AsyncPrimaryAction extends AbstractRunnable implements ActionListener<PrimaryShardReference> {
+
+        private final Request request;
+        private final TransportChannel channel;
+        private final ReplicationTask replicationTask;
+
+        AsyncPrimaryAction(Request request, TransportChannel channel, ReplicationTask replicationTask) {
+            this.request = request;
+            this.channel = channel;
+            this.replicationTask = replicationTask;
+        }
+
+        @Override
+        protected void doRun() throws Exception {
+            acquirePrimaryShardReference(request.shardId(), this);
+        }
+
+        @Override
+        public void onResponse(PrimaryShardReference primaryShardReference) {
             try {
                 if (primaryShardReference.isRelocated()) {
+                    primaryShardReference.close(); // release shard operation lock as soon as possible
                     setPhase(replicationTask, "primary_delegation");
                     // delegate primary phase to relocation target
                     // it is safe to execute primary phase on relocation target as there are no more in-flight operations where primary
@@ -268,7 +292,7 @@ public abstract class TransportReplicationAction<
                     setPhase(replicationTask, "primary");
                     final IndexMetaData indexMetaData = clusterService.state().getMetaData().index(request.shardId().getIndex());
                     final boolean executeOnReplicas = (indexMetaData == null) || shouldExecuteReplication(indexMetaData.getSettings());
-                    final ActionListener<Response> listener = createResponseListener(channel, replicationTask, primaryShardReference);
+                    final ActionListener<Response> listener = createResponseListener(primaryShardReference);
                     createReplicatedOperation(request, new ActionListener<PrimaryResult>() {
                         @Override
                         public void onResponse(PrimaryResult result) {
@@ -280,29 +304,30 @@ public abstract class TransportReplicationAction<
                             listener.onFailure(e);
                         }
                     }, primaryShardReference, executeOnReplicas).execute();
-                    success = true;
-                }
-            } finally {
-                if (success == false) {
-                    primaryShardReference.close();
                 }
+            } catch (Throwable t) {
+                Releasables.closeWhileHandlingException(primaryShardReference); // release shard operation lock before responding to caller
+                onFailure(t);
             }
         }
 
-        protected ReplicationOperation<Request, ReplicaRequest, PrimaryResult> createReplicatedOperation(
-                Request request, ActionListener<PrimaryResult> listener,
-                PrimaryShardReference primaryShardReference, boolean executeOnReplicas) {
-            return new ReplicationOperation<>(request, primaryShardReference, listener,
-                executeOnReplicas, checkWriteConsistency(), replicasProxy, clusterService::state, logger, actionName
-            );
+        @Override
+        public void onFailure(Throwable t) {
+            setPhase(replicationTask, "finished");
+            try {
+                channel.sendResponse(t);
+            } catch (IOException e) {
+                e.addSuppressed(t);
+                logger.warn("failed to send response", e);
+            }
         }
 
-        private ActionListener<Response> createResponseListener(final TransportChannel channel, final ReplicationTask replicationTask,
-                                                                final PrimaryShardReference primaryShardReference) {
+        private ActionListener<Response> createResponseListener(final PrimaryShardReference primaryShardReference) {
             return new ActionListener<Response>() {
                 @Override
                 public void onResponse(Response response) {
-                    finish();
+                    primaryShardReference.close(); // release shard operation lock before responding to caller
+                    setPhase(replicationTask, "finished");
                     try {
                         channel.sendResponse(response);
                     } catch (IOException e) {
@@ -310,15 +335,10 @@ public abstract class TransportReplicationAction<
                     }
                 }
 
-                private void finish() {
-                    primaryShardReference.close();
-                    setPhase(replicationTask, "finished");
-                }
-
                 @Override
                 public void onFailure(Throwable e) {
+                    primaryShardReference.close(); // release shard operation lock before responding to caller
                     setPhase(replicationTask, "finished");
-                    primaryShardReference.close();
                     try {
                         channel.sendResponse(e);
                     } catch (IOException e1) {
@@ -327,6 +347,14 @@ public abstract class TransportReplicationAction<
                 }
             };
         }
+
+        protected ReplicationOperation<Request, ReplicaRequest, PrimaryResult> createReplicatedOperation(
+            Request request, ActionListener<PrimaryResult> listener,
+            PrimaryShardReference primaryShardReference, boolean executeOnReplicas) {
+            return new ReplicationOperation<>(request, primaryShardReference, listener,
+                executeOnReplicas, checkWriteConsistency(), replicasProxy, clusterService::state, logger, actionName
+            );
+        }
     }
 
     protected class PrimaryResult implements ReplicationOperation.PrimaryResult<ReplicaRequest> {
@@ -388,7 +416,7 @@ public abstract class TransportReplicationAction<
         }
     }
 
-    private final class AsyncReplicaAction extends AbstractRunnable {
+    private final class AsyncReplicaAction extends AbstractRunnable implements ActionListener<Releasable> {
         private final ReplicaRequest request;
         private final TransportChannel channel;
         /**
@@ -405,6 +433,18 @@ public abstract class TransportReplicationAction<
             this.task = task;
         }
 
+        @Override
+        public void onResponse(Releasable releasable) {
+            try {
+                ReplicaResult replicaResult = shardOperationOnReplica(request);
+                releasable.close(); // release shard operation lock before responding to caller
+                replicaResult.respond(new ResponseListener());
+            } catch (Throwable t) {
+                Releasables.closeWhileHandlingException(releasable); // release shard operation lock before responding to caller
+                AsyncReplicaAction.this.onFailure(t);
+            }
+        }
+
         @Override
         public void onFailure(Throwable t) {
             if (t instanceof RetryOnReplicaException) {
@@ -451,11 +491,7 @@ public abstract class TransportReplicationAction<
         protected void doRun() throws Exception {
             setPhase(task, "replica");
             assert request.shardId() != null : "request shardId must be set";
-            ReplicaResult result;
-            try (Releasable ignored = acquireReplicaOperationLock(request.shardId(), request.primaryTerm())) {
-                result = shardOperationOnReplica(request);
-            }
-            result.respond(new ResponseListener());
+            acquireReplicaOperationLock(request.shardId(), request.primaryTerm(), this);
         }
 
         /**
@@ -733,10 +769,10 @@ public abstract class TransportReplicationAction<
     }
 
     /**
-     * returns a new reference to {@link IndexShard} to perform a primary operation. Released after performing primary operation locally
+     * tries to acquire reference to {@link IndexShard} to perform a primary operation. Released after performing primary operation locally
      * and replication of the operation to all replica shards is completed / failed (see {@link ReplicationOperation}).
      */
-    protected PrimaryShardReference getPrimaryShardReference(ShardId shardId) {
+    protected void acquirePrimaryShardReference(ShardId shardId, ActionListener<PrimaryShardReference> onReferenceAcquired) {
         IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
         IndexShard indexShard = indexService.getShard(shardId.id());
         // we may end up here if the cluster state used to route the primary is so stale that the underlying
@@ -746,17 +782,29 @@ public abstract class TransportReplicationAction<
             throw new ReplicationOperation.RetryOnPrimaryException(indexShard.shardId(),
                 "actual shard is not a primary " + indexShard.routingEntry());
         }
-        return new PrimaryShardReference(indexShard, indexShard.acquirePrimaryOperationLock());
+
+        ActionListener<Releasable> onAcquired = new ActionListener<Releasable>() {
+            @Override
+            public void onResponse(Releasable releasable) {
+                onReferenceAcquired.onResponse(new PrimaryShardReference(indexShard, releasable));
+            }
+
+            @Override
+            public void onFailure(Throwable e) {
+                onReferenceAcquired.onFailure(e);
+            }
+        };
+
+        indexShard.acquirePrimaryOperationLock(onAcquired, executor);
     }
 
     /**
-     * Acquire an operation on replicas. The lock is closed as soon as
-     * replication is completed on the node.
+     * tries to acquire an operation on replicas. The lock is closed as soon as replication is completed on the node.
      */
-    protected Releasable acquireReplicaOperationLock(ShardId shardId, long primaryTerm) {
+    protected void acquireReplicaOperationLock(ShardId shardId, long primaryTerm, ActionListener<Releasable> onLockAcquired) {
         IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
         IndexShard indexShard = indexService.getShard(shardId.id());
-        return indexShard.acquireReplicaOperationLock(primaryTerm);
+        indexShard.acquireReplicaOperationLock(primaryTerm, onLockAcquired, executor);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/SuspendableRefContainer.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/SuspendableRefContainer.java
deleted file mode 100644
index 2afb78591dd..00000000000
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/SuspendableRefContainer.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.util.concurrent;
-
-import org.elasticsearch.common.lease.Releasable;
-
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * Container that represents a resource with reference counting capabilities. Provides operations to suspend acquisition of new references.
- * This is useful for resource management when resources are intermittently unavailable.
- *
- * Assumes less than Integer.MAX_VALUE references are concurrently being held at one point in time.
- */
-public final class SuspendableRefContainer {
-    private static final int TOTAL_PERMITS = Integer.MAX_VALUE;
-    private final Semaphore semaphore;
-
-    public SuspendableRefContainer() {
-        // fair semaphore to ensure that blockAcquisition() does not starve under thread contention
-        this.semaphore = new Semaphore(TOTAL_PERMITS, true);
-    }
-
-    /**
-     * Tries acquiring a reference. Returns reference holder if reference acquisition is not blocked at the time of invocation (see
-     * {@link #blockAcquisition()}). Returns null if reference acquisition is blocked at the time of invocation.
-     *
-     * @return reference holder if reference acquisition is not blocked, null otherwise
-     * @throws InterruptedException if the current thread is interrupted
-     */
-    public Releasable tryAcquire() throws InterruptedException {
-        if (semaphore.tryAcquire(1, 0, TimeUnit.SECONDS)) { // the untimed tryAcquire methods do not honor the fairness setting
-            return idempotentRelease(1);
-        } else {
-            return null;
-        }
-    }
-
-    /**
-     * Acquires a reference. Blocks if reference acquisition is blocked at the time of invocation.
-     *
-     * @return reference holder
-     * @throws InterruptedException if the current thread is interrupted
-     */
-    public Releasable acquire() throws InterruptedException {
-        semaphore.acquire();
-        return idempotentRelease(1);
-    }
-
-    /**
-     * Acquires a reference. Blocks if reference acquisition is blocked at the time of invocation.
-     *
-     * @return reference holder
-     */
-    public Releasable acquireUninterruptibly() {
-        semaphore.acquireUninterruptibly();
-        return idempotentRelease(1);
-    }
-
-    /**
-     * Disables reference acquisition and waits until all existing references are released.
-     * When released, reference acquisition is enabled again.
-     * This guarantees that between successful acquisition and release, no one is holding a reference.
-     *
-     * @return references holder to all references
-     */
-    public Releasable blockAcquisition() {
-        semaphore.acquireUninterruptibly(TOTAL_PERMITS);
-        return idempotentRelease(TOTAL_PERMITS);
-    }
-
-    /**
-     * Helper method that ensures permits are only released once
-     *
-     * @return reference holder
-     */
-    private Releasable idempotentRelease(int permits) {
-        AtomicBoolean closed = new AtomicBoolean();
-        return () -> {
-            if (closed.compareAndSet(false, true)) {
-                semaphore.release(permits);
-            }
-        };
-    }
-
-    /**
-     * Returns the number of references currently being held.
-     */
-    public int activeRefs() {
-        int availablePermits = semaphore.availablePermits();
-        if (availablePermits == 0) {
-            // when blockAcquisition is holding all permits
-            return 0;
-        } else {
-            return TOTAL_PERMITS - availablePermits;
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 02eeabb803a..59851898d3c 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -32,6 +32,7 @@ import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.ThreadInterruptedException;
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.admin.indices.upgrade.post.UpgradeRequest;
@@ -54,7 +55,6 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.Callback;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
-import org.elasticsearch.common.util.concurrent.SuspendableRefContainer;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexNotFoundException;
@@ -128,6 +128,7 @@ import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
@@ -187,7 +188,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
 
     private final ShardPath path;
 
-    private final SuspendableRefContainer suspendableRefContainer;
+    private final IndexShardOperationsLock indexShardOperationsLock;
 
     private static final EnumSet<IndexShardState> readAllowedStates = EnumSet.of(IndexShardState.STARTED, IndexShardState.RELOCATED, IndexShardState.POST_RECOVERY);
     // for primaries, we only allow to write when actually started (so the cluster has decided we started)
@@ -260,7 +261,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         } else {
             cachingPolicy = new UsageTrackingQueryCachingPolicy();
         }
-        suspendableRefContainer = new SuspendableRefContainer();
+        indexShardOperationsLock = new IndexShardOperationsLock(shardId, logger, threadPool);
         searcherWrapper = indexSearcherWrapper;
         primaryTerm = indexSettings.getIndexMetaData().primaryTerm(shardId.id());
         refreshListeners = buildRefreshListeners();
@@ -436,20 +437,29 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         }
     }
 
-    public IndexShard relocated(String reason) throws IndexShardNotStartedException {
-        try (Releasable block = suspendableRefContainer.blockAcquisition()) {
-            // no shard operation locks are being held here, move state from started to relocated
-            synchronized (mutex) {
-                if (state != IndexShardState.STARTED) {
-                    throw new IndexShardNotStartedException(shardId, state);
+    public void relocated(String reason) throws IllegalIndexShardStateException, InterruptedException {
+        try {
+            indexShardOperationsLock.blockOperations(30, TimeUnit.MINUTES, () -> {
+                // no shard operation locks are being held here, move state from started to relocated
+                assert indexShardOperationsLock.getActiveOperationsCount() == 0 :
+                    "in-flight operations in progress while moving shard state to relocated";
+                synchronized (mutex) {
+                    if (state != IndexShardState.STARTED) {
+                        throw new IndexShardNotStartedException(shardId, state);
+                    }
+                    changeState(IndexShardState.RELOCATED, reason);
                 }
-                changeState(IndexShardState.RELOCATED, reason);
-            }
+            });
+        } catch (TimeoutException e) {
+            logger.warn("timed out waiting for relocation hand-off to complete");
+            // This is really bad as ongoing replication operations are preventing this shard from completing relocation hand-off.
+            // Fail primary relocation source and target shards.
+            failShard("timed out waiting for relocation hand-off to complete", null);
+            throw new IndexShardClosedException(shardId(), "timed out waiting for relocation hand-off to complete");
         }
-
-        return this;
     }
 
+
     public IndexShardState state() {
         return state;
     }
@@ -842,6 +852,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
                     }
                 } finally { // playing safe here and close the engine even if the above succeeds - close can be called multiple times
                     IOUtils.close(engine);
+                    indexShardOperationsLock.close();
                 }
             }
         }
@@ -1534,17 +1545,23 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
             IndexingMemoryController.SHARD_INACTIVE_TIME_SETTING.get(indexSettings.getSettings()), refreshListeners);
     }
 
-    public Releasable acquirePrimaryOperationLock() {
+    /**
+     * Acquire a primary operation lock whenever the shard is ready for indexing. If the lock is directly available, the provided
+     * ActionListener will be called on the calling thread. During relocation hand-off, lock acquisition can be delayed. The provided
+     * ActionListener will then be called using the provided executor.
+     */
+    public void acquirePrimaryOperationLock(ActionListener<Releasable> onLockAcquired, String executorOnDelay) {
         verifyNotClosed();
         verifyPrimary();
-        return suspendableRefContainer.acquireUninterruptibly();
+
+        indexShardOperationsLock.acquire(onLockAcquired, executorOnDelay, false);
     }
 
     /**
-     * acquires operation log. If the given primary term is lower then the one in {@link #shardRouting}
-     * an {@link IllegalArgumentException} is thrown.
+     * Acquire a replica operation lock whenever the shard is ready for indexing (see acquirePrimaryOperationLock). If the given primary
+     * term is lower then the one in {@link #shardRouting} an {@link IllegalArgumentException} is thrown.
      */
-    public Releasable acquireReplicaOperationLock(long opPrimaryTerm) {
+    public void acquireReplicaOperationLock(long opPrimaryTerm, ActionListener<Releasable> onLockAcquired, String executorOnDelay) {
         verifyNotClosed();
         verifyReplicationTarget();
         if (primaryTerm > opPrimaryTerm) {
@@ -1552,11 +1569,12 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
             throw new IllegalArgumentException(LoggerMessageFormat.format("{} operation term [{}] is too old (current [{}])",
                 shardId, opPrimaryTerm, primaryTerm));
         }
-        return suspendableRefContainer.acquireUninterruptibly();
+
+        indexShardOperationsLock.acquire(onLockAcquired, executorOnDelay, true);
     }
 
     public int getActiveOperationsCount() {
-        return suspendableRefContainer.activeRefs(); // refCount is incremented on creation and decremented on close
+        return indexShardOperationsLock.getActiveOperationsCount(); // refCount is incremented on successful acquire and decremented on close
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShardOperationsLock.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShardOperationsLock.java
new file mode 100644
index 00000000000..a0f2714a3a7
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShardOperationsLock.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.shard;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ThreadedActionListener;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.Closeable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class IndexShardOperationsLock implements Closeable {
+    private final ShardId shardId;
+    private final ESLogger logger;
+    private final ThreadPool threadPool;
+
+    private static final int TOTAL_PERMITS = Integer.MAX_VALUE;
+    // fair semaphore to ensure that blockOperations() does not starve under thread contention
+    final Semaphore semaphore = new Semaphore(TOTAL_PERMITS, true);
+    @Nullable private List<ActionListener<Releasable>> delayedOperations; // operations that are delayed due to relocation hand-off
+    private volatile boolean closed;
+
+    public IndexShardOperationsLock(ShardId shardId, ESLogger logger, ThreadPool threadPool) {
+        this.shardId = shardId;
+        this.logger = logger;
+        this.threadPool = threadPool;
+    }
+
+    @Override
+    public void close() {
+        closed = true;
+    }
+
+    /**
+     * Wait for in-flight operations to finish and executes onBlocked under the guarantee that no new operations are started. Queues
+     * operations that are occurring in the meanwhile and runs them once onBlocked has executed.
+     *
+     * @param timeout the maximum time to wait for the in-flight operations block
+     * @param timeUnit the time unit of the {@code timeout} argument
+     * @param onBlocked the action to run once the block has been acquired
+     * @throws InterruptedException if calling thread is interrupted
+     * @throws TimeoutException if timed out waiting for in-flight operations to finish
+     * @throws IndexShardClosedException if operation lock has been closed
+     */
+    public void blockOperations(long timeout, TimeUnit timeUnit, Runnable onBlocked) throws InterruptedException, TimeoutException {
+        if (closed) {
+            throw new IndexShardClosedException(shardId);
+        }
+        try {
+            if (semaphore.tryAcquire(TOTAL_PERMITS, timeout, timeUnit)) {
+                try {
+                    onBlocked.run();
+                } finally {
+                    semaphore.release(TOTAL_PERMITS);
+                }
+            } else {
+                throw new TimeoutException("timed out during blockOperations");
+            }
+        } finally {
+            final List<ActionListener<Releasable>> queuedActions;
+            synchronized (this) {
+                queuedActions = delayedOperations;
+                delayedOperations = null;
+            }
+            if (queuedActions != null) {
+                // Try acquiring permits on fresh thread (for two reasons):
+                // - blockOperations is called on recovery thread which can be expected to be interrupted when recovery is cancelled.
+                //   Interruptions are bad here as permit acquisition will throw an InterruptedException which will be swallowed by
+                //   ThreadedActionListener if the queue of the thread pool on which it submits is full.
+                // - if permit is acquired and queue of the thread pool which the ThreadedActionListener uses is full, the onFailure
+                //   handler is executed on the calling thread. This should not be the recovery thread as it would delay the recovery.
+                threadPool.executor(ThreadPool.Names.GENERIC).execute(() -> {
+                    for (ActionListener<Releasable> queuedAction : queuedActions) {
+                        acquire(queuedAction, null, false);
+                    }
+                });
+            }
+        }
+    }
+
+    /**
+     * Acquires a lock whenever lock acquisition is not blocked. If the lock is directly available, the provided
+     * ActionListener will be called on the calling thread. During calls of {@link #blockOperations(long, TimeUnit, Runnable)}, lock
+     * acquisition can be delayed. The provided ActionListener will then be called using the provided executor once blockOperations
+     * terminates.
+     *
+     * @param onAcquired ActionListener that is invoked once acquisition is successful or failed
+     * @param executorOnDelay executor to use for delayed call
+     * @param forceExecution whether the runnable should force its execution in case it gets rejected
+     */
+    public void acquire(ActionListener<Releasable> onAcquired, String executorOnDelay, boolean forceExecution) {
+        if (closed) {
+            onAcquired.onFailure(new IndexShardClosedException(shardId));
+            return;
+        }
+        Releasable releasable;
+        try {
+            synchronized (this) {
+                releasable = tryAcquire();
+                if (releasable == null) {
+                    // blockOperations is executing, this operation will be retried by blockOperations once it finishes
+                    if (delayedOperations == null) {
+                        delayedOperations = new ArrayList<>();
+                    }
+                    if (executorOnDelay != null) {
+                        delayedOperations.add(
+                            new ThreadedActionListener<>(logger, threadPool, executorOnDelay, onAcquired, forceExecution));
+                    } else {
+                        delayedOperations.add(onAcquired);
+                    }
+                    return;
+                }
+            }
+        } catch (InterruptedException e) {
+            onAcquired.onFailure(e);
+            return;
+        }
+        onAcquired.onResponse(releasable);
+    }
+
+    @Nullable private Releasable tryAcquire() throws InterruptedException {
+        if (semaphore.tryAcquire(1, 0, TimeUnit.SECONDS)) { // the untimed tryAcquire methods do not honor the fairness setting
+            AtomicBoolean closed = new AtomicBoolean();
+            return () -> {
+                if (closed.compareAndSet(false, true)) {
+                    semaphore.release(1);
+                }
+            };
+        }
+        return null;
+    }
+
+    public int getActiveOperationsCount() {
+        int availablePermits = semaphore.availablePermits();
+        if (availablePermits == 0) {
+            // when blockOperations is holding all permits
+            return 0;
+        } else {
+            return TOTAL_PERMITS - availablePermits;
+        }
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
index b609eb5d08a..78d9bcf53fd 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
@@ -39,7 +39,6 @@ import org.elasticsearch.common.lucene.store.InputStreamIndexInput;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.CancellableThreads;
 import org.elasticsearch.index.engine.RecoveryEngineException;
-import org.elasticsearch.index.shard.IllegalIndexShardStateException;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardClosedException;
 import org.elasticsearch.index.shard.IndexShardState;
@@ -360,22 +359,20 @@ public class RecoverySourceHandler {
         cancellableThreads.checkForCancel();
         StopWatch stopWatch = new StopWatch().start();
         logger.trace("[{}][{}] finalizing recovery to {}", indexName, shardId, request.targetNode());
-
-
         cancellableThreads.execute(recoveryTarget::finalizeRecovery);
 
         if (isPrimaryRelocation()) {
+            logger.trace("[{}][{}] performing relocation hand-off to {}", indexName, shardId, request.targetNode());
+            try {
+                cancellableThreads.execute(() -> shard.relocated("to " + request.targetNode()));
+            } catch (Throwable e) {
+                logger.debug("[{}][{}] completing relocation hand-off to {} failed", e, indexName, shardId, request.targetNode());
+                throw e;
+            }
             /**
              * if the recovery process fails after setting the shard state to RELOCATED, both relocation source and
              * target are failed (see {@link IndexShard#updateRoutingEntry}).
              */
-            try {
-                shard.relocated("to " + request.targetNode());
-            } catch (IllegalIndexShardStateException e) {
-                // we can ignore this exception since, on the other node, when it moved to phase3
-                // it will also send shard started, which might cause the index shard we work against
-                // to move be closed by the time we get to the relocated method
-            }
         }
         stopWatch.stop();
         logger.trace("[{}][{}] finalizing recovery to {}: took [{}]",
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetService.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetService.java
index 53feb8e6135..ec4e74ab8fb 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetService.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetService.java
@@ -263,12 +263,6 @@ public class RecoveryTargetService extends AbstractComponent implements IndexEve
                 return;
             }
 
-            if (cause instanceof IndexShardClosedException) {
-                onGoingRecoveries.failRecovery(recoveryTarget.recoveryId(), new RecoveryFailedException(request, "source shard is " +
-                        "closed", cause), false);
-                return;
-            }
-
             if (cause instanceof AlreadyClosedException) {
                 onGoingRecoveries.failRecovery(recoveryTarget.recoveryId(), new RecoveryFailedException(request, "source shard is " +
                         "closed", cause), false);
diff --git a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
index e15c869c6ff..37cd74e00d9 100644
--- a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
@@ -136,7 +136,7 @@ public class TransportMasterNodeActionTests extends ESTestCase {
         @Override
         protected void doExecute(Task task, final Request request, ActionListener<Response> listener) {
             // remove unneeded threading by wrapping listener with SAME to prevent super.doExecute from wrapping it with LISTENER
-            super.doExecute(task, request, new ThreadedActionListener<>(logger, threadPool, ThreadPool.Names.SAME, listener));
+            super.doExecute(task, request, new ThreadedActionListener<>(logger, threadPool, ThreadPool.Names.SAME, listener, false));
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
index 229e3b6635e..faa6a031f3b 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
@@ -390,7 +390,15 @@ public class TransportReplicationActionTests extends ESTestCase {
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
         ReplicationTask task = maybeTask();
         AtomicBoolean executed = new AtomicBoolean();
-        Action.PrimaryOperationTransportHandler primaryPhase = action.new PrimaryOperationTransportHandler() {
+
+        ShardRouting primaryShard = state.getRoutingTable().shardRoutingTable(shardId).primaryShard();
+        boolean executeOnPrimary = true;
+        // whether shard has been marked as relocated already (i.e. relocation completed)
+        if (primaryShard.relocating() && randomBoolean()) {
+            isRelocated.set(true);
+            executeOnPrimary = false;
+        }
+        action.new AsyncPrimaryAction(request, createTransportChannel(listener), task) {
             @Override
             protected ReplicationOperation<Request, Request, Action.PrimaryResult> createReplicatedOperation(Request request,
                     ActionListener<Action.PrimaryResult> actionListener, Action.PrimaryShardReference primaryShardReference,
@@ -403,15 +411,7 @@ public class TransportReplicationActionTests extends ESTestCase {
                     }
                 };
             }
-        };
-        ShardRouting primaryShard = state.getRoutingTable().shardRoutingTable(shardId).primaryShard();
-        boolean executeOnPrimary = true;
-        // whether shard has been marked as relocated already (i.e. relocation completed)
-        if (primaryShard.relocating() && randomBoolean()) {
-            isRelocated.set(true);
-            executeOnPrimary = false;
-        }
-        primaryPhase.messageReceived(request, createTransportChannel(listener), task);
+        }.run();
         if (executeOnPrimary) {
             assertTrue(executed.get());
             assertTrue(listener.isDone());
@@ -445,7 +445,7 @@ public class TransportReplicationActionTests extends ESTestCase {
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
         ReplicationTask task = maybeTask();
         AtomicBoolean executed = new AtomicBoolean();
-        Action.PrimaryOperationTransportHandler primaryPhase = action.new PrimaryOperationTransportHandler() {
+        action.new AsyncPrimaryAction(request, createTransportChannel(listener), task) {
             @Override
             protected ReplicationOperation<Request, Request, Action.PrimaryResult> createReplicatedOperation(Request request,
                     ActionListener<Action.PrimaryResult> actionListener, Action.PrimaryShardReference primaryShardReference,
@@ -458,8 +458,7 @@ public class TransportReplicationActionTests extends ESTestCase {
                     }
                 };
             }
-        };
-        primaryPhase.messageReceived(request, createTransportChannel(listener), task);
+        }.run();
         assertThat(executed.get(), equalTo(true));
         assertPhase(task, "finished");
     }
@@ -579,16 +578,18 @@ public class TransportReplicationActionTests extends ESTestCase {
         metaData.put(IndexMetaData.builder(metaData.get(index)).settings(settings));
         state = ClusterState.builder(state).metaData(metaData).build();
         setState(clusterService, state);
-        Action.PrimaryOperationTransportHandler primaryPhase = action.new PrimaryOperationTransportHandler() {
+        AtomicBoolean executed = new AtomicBoolean();
+        action.new AsyncPrimaryAction(new Request(shardId), createTransportChannel(new PlainActionFuture<>()), null) {
             @Override
             protected ReplicationOperation<Request, Request, Action.PrimaryResult> createReplicatedOperation(Request request,
                     ActionListener<Action.PrimaryResult> actionListener, Action.PrimaryShardReference primaryShardReference,
                     boolean executeOnReplicas) {
                 assertFalse(executeOnReplicas);
+                assertFalse(executed.getAndSet(true));
                 return new NoopReplicationOperation(request, actionListener);
             }
-        };
-        primaryPhase.messageReceived(new Request(shardId), createTransportChannel(new PlainActionFuture<>()), null);
+        }.run();
+        assertThat(executed.get(), equalTo(true));
     }
 
     public void testCounterOnPrimary() throws Exception {
@@ -604,17 +605,16 @@ public class TransportReplicationActionTests extends ESTestCase {
         final boolean throwExceptionOnCreation = i == 1;
         final boolean throwExceptionOnRun = i == 2;
         final boolean respondWithError = i == 3;
-        Action.PrimaryOperationTransportHandler primaryPhase = action.new PrimaryOperationTransportHandler() {
-
+        action.new AsyncPrimaryAction(request, createTransportChannel(listener), task) {
             @Override
             protected ReplicationOperation<Request, Request, Action.PrimaryResult> createReplicatedOperation(Request request,
-                    ActionListener<Action.PrimaryResult> listener, Action.PrimaryShardReference primaryShardReference,
+                    ActionListener<Action.PrimaryResult> actionListener, Action.PrimaryShardReference primaryShardReference,
                     boolean executeOnReplicas) {
                 assertIndexShardCounter(1);
                 if (throwExceptionOnCreation) {
                     throw new ElasticsearchException("simulated exception, during createReplicatedOperation");
                 }
-                return new NoopReplicationOperation(request, listener) {
+                return new NoopReplicationOperation(request, actionListener) {
                     @Override
                     public void execute() throws Exception {
                         assertIndexShardCounter(1);
@@ -629,18 +629,7 @@ public class TransportReplicationActionTests extends ESTestCase {
                     }
                 };
             }
-        };
-        try {
-            primaryPhase.messageReceived(request, createTransportChannel(listener), task);
-        } catch (ElasticsearchException e) {
-            if (throwExceptionOnCreation || throwExceptionOnRun) {
-                assertThat(e.getMessage(), containsString("simulated"));
-                assertIndexShardCounter(0);
-                return; // early terminate
-            } else {
-                throw e;
-            }
-        }
+        }.run();
         assertIndexShardCounter(0);
         assertTrue(listener.isDone());
         assertPhase(task, "finished");
@@ -648,7 +637,7 @@ public class TransportReplicationActionTests extends ESTestCase {
         try {
             listener.get();
         } catch (ExecutionException e) {
-            if (respondWithError) {
+            if (throwExceptionOnCreation || throwExceptionOnRun || respondWithError) {
                 Throwable cause = e.getCause();
                 assertThat(cause, instanceOf(ElasticsearchException.class));
                 assertThat(cause.getMessage(), containsString("simulated"));
@@ -787,9 +776,9 @@ public class TransportReplicationActionTests extends ESTestCase {
         }
 
         @Override
-        protected PrimaryShardReference getPrimaryShardReference(ShardId shardId) {
+        protected void acquirePrimaryShardReference(ShardId shardId, ActionListener<PrimaryShardReference> onReferenceAcquired) {
             count.incrementAndGet();
-            return new PrimaryShardReference(null, null) {
+            PrimaryShardReference primaryShardReference = new PrimaryShardReference(null, null) {
                 @Override
                 public boolean isRelocated() {
                     return isRelocated.get();
@@ -812,13 +801,15 @@ public class TransportReplicationActionTests extends ESTestCase {
                 public void close() {
                     count.decrementAndGet();
                 }
-
             };
+
+            onReferenceAcquired.onResponse(primaryShardReference);
         }
 
-        protected Releasable acquireReplicaOperationLock(ShardId shardId, long primaryTerm) {
+        @Override
+        protected void acquireReplicaOperationLock(ShardId shardId, long primaryTerm, ActionListener<Releasable> onLockAcquired) {
             count.incrementAndGet();
-            return count::decrementAndGet;
+            onLockAcquired.onResponse(count::decrementAndGet);
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/SuspendableRefContainerTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/SuspendableRefContainerTests.java
deleted file mode 100644
index 83db2d4a7c6..00000000000
--- a/core/src/test/java/org/elasticsearch/common/util/concurrent/SuspendableRefContainerTests.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.util.concurrent;
-
-import org.elasticsearch.common.lease.Releasable;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.nullValue;
-
-public class SuspendableRefContainerTests extends ESTestCase {
-
-    public void testBasicAcquire() throws InterruptedException {
-        SuspendableRefContainer refContainer = new SuspendableRefContainer();
-        assertThat(refContainer.activeRefs(), equalTo(0));
-
-        Releasable lock1 = randomLockingMethod(refContainer);
-        assertThat(refContainer.activeRefs(), equalTo(1));
-        Releasable lock2 = randomLockingMethod(refContainer);
-        assertThat(refContainer.activeRefs(), equalTo(2));
-        lock1.close();
-        assertThat(refContainer.activeRefs(), equalTo(1));
-        lock1.close(); // check idempotence
-        assertThat(refContainer.activeRefs(), equalTo(1));
-        lock2.close();
-        assertThat(refContainer.activeRefs(), equalTo(0));
-    }
-
-    public void testAcquisitionBlockingBlocksNewAcquisitions() throws InterruptedException {
-        SuspendableRefContainer refContainer = new SuspendableRefContainer();
-        assertThat(refContainer.activeRefs(), equalTo(0));
-
-        try (Releasable block = refContainer.blockAcquisition()) {
-            assertThat(refContainer.activeRefs(), equalTo(0));
-            assertThat(refContainer.tryAcquire(), nullValue());
-            assertThat(refContainer.activeRefs(), equalTo(0));
-        }
-        try (Releasable lock = refContainer.tryAcquire()) {
-            assertThat(refContainer.activeRefs(), equalTo(1));
-        }
-
-        // same with blocking acquire
-        AtomicBoolean acquired = new AtomicBoolean();
-        Thread t = new Thread(() -> {
-            try (Releasable lock = randomBoolean() ? refContainer.acquire() : refContainer.acquireUninterruptibly()) {
-                acquired.set(true);
-                assertThat(refContainer.activeRefs(), equalTo(1));
-            } catch (InterruptedException e) {
-                fail("Interrupted");
-            }
-        });
-        try (Releasable block = refContainer.blockAcquisition()) {
-            assertThat(refContainer.activeRefs(), equalTo(0));
-            t.start();
-            // check that blocking acquire really blocks
-            assertThat(acquired.get(), equalTo(false));
-            assertThat(refContainer.activeRefs(), equalTo(0));
-        }
-        t.join();
-        assertThat(acquired.get(), equalTo(true));
-        assertThat(refContainer.activeRefs(), equalTo(0));
-    }
-
-    public void testAcquisitionBlockingWaitsOnExistingAcquisitions() throws InterruptedException {
-        SuspendableRefContainer refContainer = new SuspendableRefContainer();
-
-        AtomicBoolean acquired = new AtomicBoolean();
-        Thread t = new Thread(() -> {
-            try (Releasable block = refContainer.blockAcquisition()) {
-                acquired.set(true);
-                assertThat(refContainer.activeRefs(), equalTo(0));
-            }
-        });
-        try (Releasable lock = randomLockingMethod(refContainer)) {
-            assertThat(refContainer.activeRefs(), equalTo(1));
-            t.start();
-            assertThat(acquired.get(), equalTo(false));
-            assertThat(refContainer.activeRefs(), equalTo(1));
-        }
-        t.join();
-        assertThat(acquired.get(), equalTo(true));
-        assertThat(refContainer.activeRefs(), equalTo(0));
-    }
-
-    private Releasable randomLockingMethod(SuspendableRefContainer refContainer) throws InterruptedException {
-        switch (randomInt(2)) {
-            case 0: return refContainer.tryAcquire();
-            case 1: return refContainer.acquire();
-            case 2: return refContainer.acquireUninterruptibly();
-        }
-        throw new IllegalArgumentException("randomLockingMethod inconsistent");
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardOperationsLockTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardOperationsLockTests.java
new file mode 100644
index 00000000000..9ac9e86fec8
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardOperationsLockTests.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.index.shard;
+
+import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.common.inject.internal.Nullable;
+import org.elasticsearch.common.lease.Releasable;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class IndexShardOperationsLockTests extends ESTestCase {
+
+    private static ThreadPool threadPool;
+
+    private IndexShardOperationsLock block;
+
+    @BeforeClass
+    public static void setupThreadPool() {
+        threadPool = new TestThreadPool("IndexShardOperationsLockTests");
+    }
+
+    @AfterClass
+    public static void shutdownThreadPool() {
+        ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS);
+        threadPool = null;
+    }
+
+    @Before
+    public void createIndexShardOperationsLock() {
+         block = new IndexShardOperationsLock(new ShardId("blubb", "id", 0), logger, threadPool);
+    }
+
+    @After
+    public void checkNoInflightOperations() {
+        assertThat(block.semaphore.availablePermits(), equalTo(Integer.MAX_VALUE));
+        assertThat(block.getActiveOperationsCount(), equalTo(0));
+    }
+
+    public void testAllOperationsInvoked() throws InterruptedException, TimeoutException, ExecutionException {
+        int numThreads = 10;
+
+        List<PlainActionFuture<Releasable>> futures = new ArrayList<>();
+        List<Thread> operationThreads = new ArrayList<>();
+        CountDownLatch latch = new CountDownLatch(numThreads / 2);
+        for (int i = 0; i < numThreads; i++) {
+            PlainActionFuture<Releasable> future = new PlainActionFuture<Releasable>() {
+                @Override
+                public void onResponse(Releasable releasable) {
+                    releasable.close();
+                    super.onResponse(releasable);
+                }
+            };
+            Thread thread = new Thread() {
+                public void run() {
+                    latch.countDown();
+                    block.acquire(future, ThreadPool.Names.GENERIC, true);
+                }
+            };
+            futures.add(future);
+            operationThreads.add(thread);
+        }
+
+        threadPool.generic().execute(() -> {
+            try {
+                latch.await();
+                blockAndWait().close();
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+        });
+
+        for (Thread thread : operationThreads) {
+            thread.start();
+        }
+
+        for (PlainActionFuture<Releasable> future : futures) {
+            assertNotNull(future.get(1, TimeUnit.MINUTES));
+        }
+
+        for (Thread thread : operationThreads) {
+            thread.join();
+        }
+    }
+
+
+    public void testOperationsInvokedImmediatelyIfNoBlock() throws ExecutionException, InterruptedException {
+        PlainActionFuture<Releasable> future = new PlainActionFuture<>();
+        block.acquire(future, ThreadPool.Names.GENERIC, true);
+        assertTrue(future.isDone());
+        future.get().close();
+    }
+
+    public void testOperationsIfClosed() throws ExecutionException, InterruptedException {
+        PlainActionFuture<Releasable> future = new PlainActionFuture<>();
+        block.close();
+        block.acquire(future, ThreadPool.Names.GENERIC, true);
+        ExecutionException exception = expectThrows(ExecutionException.class, future::get);
+        assertThat(exception.getCause(), instanceOf(IndexShardClosedException.class));
+    }
+
+    public void testBlockIfClosed() throws ExecutionException, InterruptedException {
+        block.close();
+        expectThrows(IndexShardClosedException.class, () -> block.blockOperations(randomInt(10), TimeUnit.MINUTES,
+            () -> { throw new IllegalArgumentException("fake error"); }));
+    }
+
+    public void testOperationsDelayedIfBlock() throws ExecutionException, InterruptedException, TimeoutException {
+        PlainActionFuture<Releasable> future = new PlainActionFuture<>();
+        try (Releasable releasable = blockAndWait()) {
+            block.acquire(future, ThreadPool.Names.GENERIC, true);
+            assertFalse(future.isDone());
+        }
+        future.get(1, TimeUnit.MINUTES).close();
+    }
+
+    protected Releasable blockAndWait() throws InterruptedException {
+        CountDownLatch blockAcquired = new CountDownLatch(1);
+        CountDownLatch releaseBlock = new CountDownLatch(1);
+        CountDownLatch blockReleased = new CountDownLatch(1);
+        boolean throwsException = randomBoolean();
+        IndexShardClosedException exception = new IndexShardClosedException(new ShardId("blubb", "id", 0));
+        threadPool.generic().execute(() -> {
+                try {
+                    block.blockOperations(1, TimeUnit.MINUTES, () -> {
+                        try {
+                            blockAcquired.countDown();
+                            releaseBlock.await();
+                            if (throwsException) {
+                                throw exception;
+                            }
+                        } catch (InterruptedException e) {
+                            throw new RuntimeException();
+                        }
+                    });
+                } catch (Exception e) {
+                    if (e != exception) {
+                        throw new RuntimeException(e);
+                    }
+                } finally {
+                    blockReleased.countDown();
+                }
+            });
+        blockAcquired.await();
+        return () -> {
+            releaseBlock.countDown();
+            try {
+                blockReleased.await();
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+        };
+    }
+
+    public void testActiveOperationsCount() throws ExecutionException, InterruptedException {
+        PlainActionFuture<Releasable> future1 = new PlainActionFuture<>();
+        block.acquire(future1, ThreadPool.Names.GENERIC, true);
+        assertTrue(future1.isDone());
+        assertThat(block.getActiveOperationsCount(), equalTo(1));
+
+        PlainActionFuture<Releasable> future2 = new PlainActionFuture<>();
+        block.acquire(future2, ThreadPool.Names.GENERIC, true);
+        assertTrue(future2.isDone());
+        assertThat(block.getActiveOperationsCount(), equalTo(2));
+
+        future1.get().close();
+        assertThat(block.getActiveOperationsCount(), equalTo(1));
+        future1.get().close(); // check idempotence
+        assertThat(block.getActiveOperationsCount(), equalTo(1));
+        future2.get().close();
+        assertThat(block.getActiveOperationsCount(), equalTo(0));
+
+        try (Releasable releasable = blockAndWait()) {
+            assertThat(block.getActiveOperationsCount(), equalTo(0));
+        }
+
+        PlainActionFuture<Releasable> future3 = new PlainActionFuture<>();
+        block.acquire(future3, ThreadPool.Names.GENERIC, true);
+        assertTrue(future3.isDone());
+        assertThat(block.getActiveOperationsCount(), equalTo(1));
+        future3.get().close();
+        assertThat(block.getActiveOperationsCount(), equalTo(0));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 862be713030..6674d6859ab 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -44,6 +44,7 @@ import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.TransportIndexAction;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.ClusterInfoService;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.InternalClusterInfoService;
@@ -105,6 +106,7 @@ import org.elasticsearch.test.FieldMaskingReader;
 import org.elasticsearch.test.IndexSettingsModule;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.test.VersionUtils;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -121,6 +123,7 @@ import java.util.concurrent.BrokenBarrierException;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
@@ -295,13 +298,13 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         client().admin().indices().prepareDelete("test").get();
         assertThat(indexShard.getActiveOperationsCount(), equalTo(0));
         try {
-            indexShard.acquirePrimaryOperationLock();
+            indexShard.acquirePrimaryOperationLock(null, ThreadPool.Names.INDEX);
             fail("we should not be able to increment anymore");
         } catch (IndexShardClosedException e) {
             // expected
         }
         try {
-            indexShard.acquireReplicaOperationLock(indexShard.getPrimaryTerm());
+            indexShard.acquireReplicaOperationLock(indexShard.getPrimaryTerm(), null, ThreadPool.Names.INDEX);
             fail("we should not be able to increment anymore");
         } catch (IndexShardClosedException e) {
             // expected
@@ -339,21 +342,33 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         assertEquals(0, indexShard.getActiveOperationsCount());
         if (newPrimaryShardRouting.isRelocationTarget() == false) {
             try {
-                indexShard.acquireReplicaOperationLock(primaryTerm);
+                indexShard.acquireReplicaOperationLock(primaryTerm, null, ThreadPool.Names.INDEX);
                 fail("shard shouldn't accept operations as replica");
             } catch (IllegalStateException ignored) {
 
             }
         }
-        Releasable operation1 = indexShard.acquirePrimaryOperationLock();
+        Releasable operation1 = acquirePrimaryOperationLockBlockingly(indexShard);
         assertEquals(1, indexShard.getActiveOperationsCount());
-        Releasable operation2 = indexShard.acquirePrimaryOperationLock();
+        Releasable operation2 = acquirePrimaryOperationLockBlockingly(indexShard);
         assertEquals(2, indexShard.getActiveOperationsCount());
 
         Releasables.close(operation1, operation2);
         assertEquals(0, indexShard.getActiveOperationsCount());
     }
 
+    private Releasable acquirePrimaryOperationLockBlockingly(IndexShard indexShard) throws ExecutionException, InterruptedException {
+        PlainActionFuture<Releasable> fut = new PlainActionFuture<>();
+        indexShard.acquirePrimaryOperationLock(fut, ThreadPool.Names.INDEX);
+        return fut.get();
+    }
+
+    private Releasable acquireReplicaOperationLockBlockingly(IndexShard indexShard, long opPrimaryTerm) throws ExecutionException, InterruptedException {
+        PlainActionFuture<Releasable> fut = new PlainActionFuture<>();
+        indexShard.acquireReplicaOperationLock(opPrimaryTerm, fut, ThreadPool.Names.INDEX);
+        return fut.get();
+    }
+
     public void testOperationLocksOnReplicaShards() throws InterruptedException, ExecutionException, IOException {
         assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)).get());
         ensureGreen("test");
@@ -399,20 +414,20 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         assertEquals(0, indexShard.getActiveOperationsCount());
         if (newShardRouting.primary() == false) {
             try {
-                indexShard.acquirePrimaryOperationLock();
+                indexShard.acquirePrimaryOperationLock(null, ThreadPool.Names.INDEX);
                 fail("shard shouldn't accept primary ops");
             } catch (IllegalStateException ignored) {
 
             }
         }
 
-        Releasable operation1 = indexShard.acquireReplicaOperationLock(primaryTerm);
+        Releasable operation1 = acquireReplicaOperationLockBlockingly(indexShard, primaryTerm);
         assertEquals(1, indexShard.getActiveOperationsCount());
-        Releasable operation2 = indexShard.acquireReplicaOperationLock(primaryTerm);
+        Releasable operation2 = acquireReplicaOperationLockBlockingly(indexShard, primaryTerm);
         assertEquals(2, indexShard.getActiveOperationsCount());
 
         try {
-            indexShard.acquireReplicaOperationLock(primaryTerm - 1);
+            indexShard.acquireReplicaOperationLock(primaryTerm - 1, null, ThreadPool.Names.INDEX);
             fail("you can not increment the operation counter with an older primary term");
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), containsString("operation term"));
@@ -420,7 +435,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         }
 
         // but you can increment with a newer one..
-        indexShard.acquireReplicaOperationLock(primaryTerm + 1 + randomInt(20)).close();
+        acquireReplicaOperationLockBlockingly(indexShard, primaryTerm + 1 + randomInt(20)).close();
         Releasables.close(operation1, operation2);
         assertEquals(0, indexShard.getActiveOperationsCount());
     }
@@ -882,13 +897,18 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
         IndexService test = indicesService.indexService(resolveIndex("test"));
         final IndexShard shard = test.getShardOrNull(0);
+        assertBusy(() -> assertThat(shard.state(), equalTo(IndexShardState.STARTED)));
         CountDownLatch latch = new CountDownLatch(1);
         Thread recoveryThread = new Thread(() -> {
             latch.countDown();
-            shard.relocated("simulated recovery");
+            try {
+                shard.relocated("simulated recovery");
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
         });
 
-        try (Releasable ignored = shard.acquirePrimaryOperationLock()) {
+        try (Releasable ignored = acquirePrimaryOperationLockBlockingly(shard)) {
             // start finalization of recovery
             recoveryThread.start();
             latch.await();
@@ -898,12 +918,50 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         // recovery can be now finalized
         recoveryThread.join();
         assertThat(shard.state(), equalTo(IndexShardState.RELOCATED));
-        try (Releasable ignored = shard.acquirePrimaryOperationLock()) {
+        try (Releasable ignored = acquirePrimaryOperationLockBlockingly(shard)) {
             // lock can again be acquired
             assertThat(shard.state(), equalTo(IndexShardState.RELOCATED));
         }
     }
 
+    public void testDelayedOperationsBeforeAndAfterRelocated() throws Exception {
+        assertAcked(client().admin().indices().prepareCreate("test").setSettings(
+            Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)
+        ).get());
+        ensureGreen();
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        IndexService test = indicesService.indexService(resolveIndex("test"));
+        final IndexShard shard = test.getShardOrNull(0);
+        assertBusy(() -> assertThat(shard.state(), equalTo(IndexShardState.STARTED)));
+        Thread recoveryThread = new Thread(() -> {
+            try {
+                shard.relocated("simulated recovery");
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
+        });
+
+        recoveryThread.start();
+        List<PlainActionFuture<Releasable>> onLockAcquiredActions = new ArrayList<>();
+        for (int i = 0; i < 10; i++) {
+            PlainActionFuture<Releasable> onLockAcquired = new PlainActionFuture<Releasable>() {
+                @Override
+                public void onResponse(Releasable releasable) {
+                    releasable.close();
+                    super.onResponse(releasable);
+                }
+            };
+            shard.acquirePrimaryOperationLock(onLockAcquired, ThreadPool.Names.INDEX);
+            onLockAcquiredActions.add(onLockAcquired);
+        }
+
+        for (PlainActionFuture<Releasable> onLockAcquired : onLockAcquiredActions) {
+            assertNotNull(onLockAcquired.get(30, TimeUnit.SECONDS));
+        }
+
+        recoveryThread.join();
+    }
+
     public void testStressRelocated() throws Exception {
         assertAcked(client().admin().indices().prepareCreate("test").setSettings(
             Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)
@@ -920,10 +978,10 @@ public class IndexShardTests extends ESSingleNodeTestCase {
             indexThreads[i] = new Thread() {
                 @Override
                 public void run() {
-                    try (Releasable operationLock = shard.acquirePrimaryOperationLock()) {
+                    try (Releasable operationLock = acquirePrimaryOperationLockBlockingly(shard)) {
                         allPrimaryOperationLocksAcquired.countDown();
                         barrier.await();
-                    } catch (InterruptedException | BrokenBarrierException e) {
+                    } catch (InterruptedException | BrokenBarrierException | ExecutionException e) {
                         throw new RuntimeException(e);
                     }
                 }
@@ -932,7 +990,11 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         }
         AtomicBoolean relocated = new AtomicBoolean();
         final Thread recoveryThread = new Thread(() -> {
-            shard.relocated("simulated recovery");
+            try {
+                shard.relocated("simulated recovery");
+            } catch (InterruptedException e) {
+                throw new RuntimeException(e);
+            }
             relocated.set(true);
         });
         // ensure we wait for all primary operation locks to be acquired
@@ -1410,8 +1472,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndexShard newShard = new IndexShard(initializingShardRouting, indexService.getIndexSettings(), shard.shardPath(),
             shard.store(), indexService.cache(), indexService.mapperService(), indexService.similarityService(),
             indexService.fieldData(), shard.getEngineFactory(), indexService.getIndexEventListener(), wrapper,
-            indexService.getThreadPool(), indexService.getBigArrays(), null, Collections.emptyList(), Arrays.asList(listeners)
-        );
+            indexService.getThreadPool(), indexService.getBigArrays(), null, Collections.emptyList(), Arrays.asList(listeners));
         return newShard;
     }
 
diff --git a/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java b/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java
index 5932434438c..ea2a80bada5 100644
--- a/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushSingleNodeTests.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.indices.flush;
 
+import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -31,9 +32,11 @@ import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardNotFoundException;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ExecutionException;
 
 /**
  */
@@ -103,7 +106,7 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase {
         assertTrue(response.success());
     }
 
-    public void testSyncFailsIfOperationIsInFlight() throws InterruptedException {
+    public void testSyncFailsIfOperationIsInFlight() throws InterruptedException, ExecutionException {
         createIndex("test");
         client().prepareIndex("test", "test", "1").setSource("{}").get();
         IndexService test = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test"));
@@ -111,7 +114,9 @@ public class SyncedFlushSingleNodeTests extends ESSingleNodeTestCase {
 
         SyncedFlushService flushService = getInstanceFromNode(SyncedFlushService.class);
         final ShardId shardId = shard.shardId();
-        try (Releasable operationLock = shard.acquirePrimaryOperationLock()) {
+        PlainActionFuture<Releasable> fut = new PlainActionFuture<>();
+        shard.acquirePrimaryOperationLock(fut, ThreadPool.Names.INDEX);
+        try (Releasable operationLock = fut.get()) {
             SyncedFlushUtil.LatchedListener<ShardsSyncedFlushResult> listener = new SyncedFlushUtil.LatchedListener<>();
             flushService.attemptSyncedFlush(shardId, listener);
             listener.latch.await();
diff --git a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
index fd5da198ed2..0c224238a90 100644
--- a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
+++ b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
@@ -432,22 +432,23 @@ public class RelocationIT extends ESIntegTestCase {
         }
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/18553")
     public void testIndexAndRelocateConcurrently() throws ExecutionException, InterruptedException {
+        int halfNodes = randomIntBetween(1, 3);
         Settings blueSetting = Settings.builder().put("node.attr.color", "blue").build();
-        InternalTestCluster.Async<List<String>> blueFuture = internalCluster().startNodesAsync(blueSetting, blueSetting);
+        InternalTestCluster.Async<List<String>> blueFuture = internalCluster().startNodesAsync(halfNodes, blueSetting);
         Settings redSetting = Settings.builder().put("node.attr.color", "red").build();
-        InternalTestCluster.Async<java.util.List<String>> redFuture = internalCluster().startNodesAsync(redSetting, redSetting);
+        InternalTestCluster.Async<java.util.List<String>> redFuture = internalCluster().startNodesAsync(halfNodes, redSetting);
         blueFuture.get();
         redFuture.get();
         logger.info("blue nodes: {}", blueFuture.get());
         logger.info("red nodes: {}", redFuture.get());
-        ensureStableCluster(4);
+        ensureStableCluster(halfNodes * 2);
 
         assertAcked(prepareCreate("test").setSettings(Settings.builder()
             .put("index.routing.allocation.exclude.color", "blue")
-            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1)
-            .put(indexSettings())));
+            .put(indexSettings())
+            .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) // NORELEASE: set to randomInt(halfNodes - 1) once replica data loss is fixed
+        ));
         ensureYellow();
         assertAllShardsOnNodes("test", redFuture.get().toArray(new String[2]));
         int numDocs = randomIntBetween(100, 150);
@@ -479,9 +480,11 @@ public class RelocationIT extends ESIntegTestCase {
         numDocs *= 2;
 
         logger.info(" --> waiting for relocation to complete");
-        ensureGreen("test");// move all shards to the new node (it waits on relocation)
+        ensureGreen("test"); // move all shards to the new nodes (it waits on relocation)
+
         final int numIters = randomIntBetween(10, 20);
         for (int i = 0; i < numIters; i++) {
+            logger.info(" --> checking iteration {}", i);
             SearchResponse afterRelocation = client().prepareSearch().setSize(ids.size()).get();
             assertNoFailures(afterRelocation);
             assertSearchHits(afterRelocation, ids.toArray(new String[ids.size()]));

From 3221c9d9701a65a486ddaf00e411cdc9b17ad33e Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Sat, 2 Jul 2016 10:07:12 +0200
Subject: [PATCH 1210/1311] Match exception message more exactly in test
 assertion

---
 .../java/org/elasticsearch/snapshots/RepositoriesIT.java    | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java b/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java
index 980912ecb6d..48a1cc6081e 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java
@@ -40,6 +40,7 @@ import java.util.List;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
@@ -153,8 +154,9 @@ public class RepositoriesIT extends AbstractSnapshotIntegTestCase {
                     .get();
             fail("Shouldn't be here");
         } catch (RepositoryException ex) {
-            assertThat(ex.toString(), containsString("Unable to parse URL repository setting"));
-            assertThat(ex.toString(), containsString("netdoc"));
+            assertThat(ex.toString(),
+                either(containsString("unsupported url protocol [netdoc]"))
+                    .or(containsString("unknown protocol: netdoc"))); // newer versions of JDK 9
         }
 
         logger.info("--> trying creating url repository with location that is not registered in path.repo setting");

From 3e199b1dffcad8236fcde5fd34e18acf186a09c8 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Sat, 2 Jul 2016 11:54:52 +0200
Subject: [PATCH 1211/1311] Make testAllOperationsInvoked properly clean up
 after itself

---
 .../index/shard/IndexShardOperationsLockTests.java            | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardOperationsLockTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardOperationsLockTests.java
index 9ac9e86fec8..c9bb9e19866 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardOperationsLockTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardOperationsLockTests.java
@@ -92,10 +92,12 @@ public class IndexShardOperationsLockTests extends ESTestCase {
             operationThreads.add(thread);
         }
 
+        CountDownLatch blockFinished = new CountDownLatch(1);
         threadPool.generic().execute(() -> {
             try {
                 latch.await();
                 blockAndWait().close();
+                blockFinished.countDown();
             } catch (InterruptedException e) {
                 throw new RuntimeException(e);
             }
@@ -112,6 +114,8 @@ public class IndexShardOperationsLockTests extends ESTestCase {
         for (Thread thread : operationThreads) {
             thread.join();
         }
+
+        blockFinished.await();
     }
 
 

From 5cf7583bded93c855f9fca0ba3b4e7730e03da1a Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Sat, 2 Jul 2016 11:33:16 -0700
Subject: [PATCH 1212/1311] Internal: Remove unused methods and classes

This change removes a handful of classes and methods that were simply
unused. Some of the classes were intermediate abstract classes that
added nothing to the base class they extended.
---
 .../action/ActionListenerResponseHandler.java |  4 +-
 .../tasks/get/TransportGetTaskAction.java     |  4 +-
 .../broadcast/TransportBroadcastAction.java   |  4 +-
 .../node/TransportBroadcastByNodeAction.java  |  4 +-
 .../support/nodes/TransportNodesAction.java   |  4 +-
 .../TransportReplicationAction.java           |  4 +-
 ...ransportInstanceSingleOperationAction.java |  4 +-
 .../shard/TransportSingleShardAction.java     |  6 +-
 .../support/tasks/TransportTasksAction.java   |  4 +-
 .../TransportClientNodesService.java          |  4 +-
 .../common/io/CharSequenceReader.java         | 28 ------
 .../common/io/FastStringReader.java           |  2 +-
 .../common/util/CollectionUtils.java          | 85 ++-----------------
 .../zen/fd/MasterFaultDetection.java          |  4 +-
 .../discovery/zen/fd/NodesFaultDetection.java |  4 +-
 .../zen/ping/unicast/UnicastZenPing.java      |  4 +-
 .../indices/flush/SyncedFlushService.java     |  8 +-
 .../node/service/NodeService.java             | 45 ++--------
 .../BaseTransportResponseHandler.java         | 27 ------
 .../FutureTransportResponseHandler.java       |  2 +-
 .../TransportClientNodesServiceTests.java     |  4 +-
 .../common/settings/bar/BarTestClass.java     | 24 ------
 .../AbstractSimpleTransportTestCase.java      | 32 +++----
 .../netty/NettyScheduledPingTests.java        |  4 +-
 24 files changed, 66 insertions(+), 249 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/common/io/CharSequenceReader.java
 delete mode 100644 core/src/main/java/org/elasticsearch/transport/BaseTransportResponseHandler.java
 delete mode 100644 core/src/test/java/org/elasticsearch/common/settings/bar/BarTestClass.java

diff --git a/core/src/main/java/org/elasticsearch/action/ActionListenerResponseHandler.java b/core/src/main/java/org/elasticsearch/action/ActionListenerResponseHandler.java
index 6cdc1c3194f..e0e04652315 100644
--- a/core/src/main/java/org/elasticsearch/action/ActionListenerResponseHandler.java
+++ b/core/src/main/java/org/elasticsearch/action/ActionListenerResponseHandler.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.action;
 
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.BaseTransportResponseHandler;
+import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportResponse;
 
@@ -31,7 +31,7 @@ import java.util.function.Supplier;
  * A simple base class for action response listeners, defaulting to using the SAME executor (as its
  * very common on response handlers).
  */
-public class ActionListenerResponseHandler<Response extends TransportResponse> extends BaseTransportResponseHandler<Response> {
+public class ActionListenerResponseHandler<Response extends TransportResponse> implements TransportResponseHandler<Response> {
 
     private final ActionListener<Response> listener;
     private final Supplier<Response> responseSupplier;
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
index 477f646fe39..1138792d97e 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
@@ -44,9 +44,9 @@ import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskInfo;
 import org.elasticsearch.tasks.TaskPersistenceService;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.BaseTransportResponseHandler;
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequestOptions;
+import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
@@ -111,7 +111,7 @@ public class TransportGetTaskAction extends HandledTransportAction<GetTaskReques
         GetTaskRequest nodeRequest = request.nodeRequest(clusterService.localNode().getId(), thisTask.getId());
         taskManager.registerChildTask(thisTask, node.getId());
         transportService.sendRequest(node, GetTaskAction.NAME, nodeRequest, builder.build(),
-                new BaseTransportResponseHandler<GetTaskResponse>() {
+                new TransportResponseHandler<GetTaskResponse>() {
                     @Override
                     public GetTaskResponse newInstance() {
                         return new GetTaskResponse();
diff --git a/core/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java b/core/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java
index 182d922fc39..5d0ca7eacf7 100644
--- a/core/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java
@@ -37,7 +37,7 @@ import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.BaseTransportResponseHandler;
+import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequestHandler;
@@ -177,7 +177,7 @@ public abstract class TransportBroadcastAction<Request extends BroadcastRequest<
                         onOperation(shard, shardIt, shardIndex, new NoShardAvailableActionException(shardIt.shardId()));
                     } else {
                         taskManager.registerChildTask(task, node.getId());
-                        transportService.sendRequest(node, transportShardAction, shardRequest, new BaseTransportResponseHandler<ShardResponse>() {
+                        transportService.sendRequest(node, transportShardAction, shardRequest, new TransportResponseHandler<ShardResponse>() {
                             @Override
                             public ShardResponse newInstance() {
                                 return newShardResponse();
diff --git a/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java b/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
index 12184d57cd8..cb6a10fbf79 100644
--- a/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
@@ -46,7 +46,7 @@ import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.BaseTransportResponseHandler;
+import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.NodeShouldNotConnectException;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportException;
@@ -318,7 +318,7 @@ public abstract class TransportBroadcastByNodeAction<Request extends BroadcastRe
                     nodeRequest.setParentTask(clusterService.localNode().getId(), task.getId());
                     taskManager.registerChildTask(task, node.getId());
                 }
-                transportService.sendRequest(node, transportNodeBroadcastAction, nodeRequest, new BaseTransportResponseHandler<NodeResponse>() {
+                transportService.sendRequest(node, transportNodeBroadcastAction, nodeRequest, new TransportResponseHandler<NodeResponse>() {
                     @Override
                     public NodeResponse newInstance() {
                         return new NodeResponse();
diff --git a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
index 1609a08bb14..d26c5c6d0ca 100644
--- a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
@@ -31,7 +31,7 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.BaseTransportResponseHandler;
+import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.NodeShouldNotConnectException;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportException;
@@ -204,7 +204,7 @@ public abstract class TransportNodesAction<NodesRequest extends BaseNodesRequest
                         }
 
                         transportService.sendRequest(node, transportNodeAction, nodeRequest, builder.build(),
-                                                     new BaseTransportResponseHandler<NodeResponse>() {
+                                                     new TransportResponseHandler<NodeResponse>() {
                             @Override
                             public NodeResponse newInstance() {
                                 return newNodeResponse();
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
index 80aa23032c5..13bff24ec1d 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
@@ -55,7 +55,7 @@ import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.BaseTransportResponseHandler;
+import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportChannelResponseHandler;
@@ -628,7 +628,7 @@ public abstract class TransportReplicationAction<
         }
 
         private void performAction(final DiscoveryNode node, final String action, final boolean isPrimaryAction) {
-            transportService.sendRequest(node, action, request, transportOptions, new BaseTransportResponseHandler<Response>() {
+            transportService.sendRequest(node, action, request, transportOptions, new TransportResponseHandler<Response>() {
 
                 @Override
                 public Response newInstance() {
diff --git a/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java b/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java
index 760b77469d2..a1483967e1f 100644
--- a/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java
@@ -39,7 +39,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.BaseTransportResponseHandler;
+import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportException;
@@ -174,7 +174,7 @@ public abstract class TransportInstanceSingleOperationAction<Request extends Ins
 
             request.shardId = shardIt.shardId();
             DiscoveryNode node = nodes.get(shard.currentNodeId());
-            transportService.sendRequest(node, shardActionName, request, transportOptions(), new BaseTransportResponseHandler<Response>() {
+            transportService.sendRequest(node, shardActionName, request, transportOptions(), new TransportResponseHandler<Response>() {
 
                 @Override
                 public Response newInstance() {
diff --git a/core/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java b/core/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java
index 0c7f0627c66..a625ed224b9 100644
--- a/core/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java
@@ -39,7 +39,7 @@ import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.BaseTransportResponseHandler;
+import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequestHandler;
@@ -159,7 +159,7 @@ public abstract class TransportSingleShardAction<Request extends SingleShardRequ
         public void start() {
             if (shardIt == null) {
                 // just execute it on the local node
-                transportService.sendRequest(clusterService.localNode(), transportShardAction, internalRequest.request(), new BaseTransportResponseHandler<Response>() {
+                transportService.sendRequest(clusterService.localNode(), transportShardAction, internalRequest.request(), new TransportResponseHandler<Response>() {
                     @Override
                     public Response newInstance() {
                         return newResponse();
@@ -224,7 +224,7 @@ public abstract class TransportSingleShardAction<Request extends SingleShardRequ
                             node
                     );
                 }
-                transportService.sendRequest(node, transportShardAction, internalRequest.request(), new BaseTransportResponseHandler<Response>() {
+                transportService.sendRequest(node, transportShardAction, internalRequest.request(), new TransportResponseHandler<Response>() {
 
                     @Override
                     public Response newInstance() {
diff --git a/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java b/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
index 0a45497e1fd..0782475ec06 100644
--- a/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
@@ -38,7 +38,7 @@ import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.BaseTransportResponseHandler;
+import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.NodeShouldNotConnectException;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportException;
@@ -237,7 +237,7 @@ public abstract class TransportTasksAction<
                             nodeRequest.setParentTask(clusterService.localNode().getId(), task.getId());
                             taskManager.registerChildTask(task, node.getId());
                             transportService.sendRequest(node, transportNodeAction, nodeRequest, builder.build(),
-                                new BaseTransportResponseHandler<NodeTasksResponse>() {
+                                new TransportResponseHandler<NodeTasksResponse>() {
                                     @Override
                                     public NodeTasksResponse newInstance() {
                                         return new NodeTasksResponse();
diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
index 120a4711217..7aaf80762be 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
@@ -41,7 +41,7 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.BaseTransportResponseHandler;
+import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.FutureTransportResponseHandler;
 import org.elasticsearch.transport.TransportException;
@@ -459,7 +459,7 @@ public class TransportClientNodesService extends AbstractComponent implements Cl
                                     Requests.clusterStateRequest().clear().nodes(true).local(true),
                                     TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STATE)
                                             .withTimeout(pingTimeout).build(),
-                                    new BaseTransportResponseHandler<ClusterStateResponse>() {
+                                    new TransportResponseHandler<ClusterStateResponse>() {
 
                                         @Override
                                         public ClusterStateResponse newInstance() {
diff --git a/core/src/main/java/org/elasticsearch/common/io/CharSequenceReader.java b/core/src/main/java/org/elasticsearch/common/io/CharSequenceReader.java
deleted file mode 100644
index cb6c3bc6af1..00000000000
--- a/core/src/main/java/org/elasticsearch/common/io/CharSequenceReader.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.io;
-
-import java.io.Reader;
-
-/**
- *
- */
-public abstract class CharSequenceReader extends Reader implements CharSequence {
-}
diff --git a/core/src/main/java/org/elasticsearch/common/io/FastStringReader.java b/core/src/main/java/org/elasticsearch/common/io/FastStringReader.java
index 25019149501..17398b7139b 100644
--- a/core/src/main/java/org/elasticsearch/common/io/FastStringReader.java
+++ b/core/src/main/java/org/elasticsearch/common/io/FastStringReader.java
@@ -28,7 +28,7 @@ import java.io.Reader;
  * (shay.banon
  * )
  */
-public class FastStringReader extends CharSequenceReader {
+public class FastStringReader extends Reader implements CharSequence {
 
     private String str;
     private int length;
diff --git a/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java b/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java
index 36e0b19c782..5171be0ee91 100644
--- a/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java
+++ b/core/src/main/java/org/elasticsearch/common/util/CollectionUtils.java
@@ -19,16 +19,6 @@
 
 package org.elasticsearch.common.util;
 
-import com.carrotsearch.hppc.DoubleArrayList;
-import com.carrotsearch.hppc.FloatArrayList;
-import com.carrotsearch.hppc.LongArrayList;
-import com.carrotsearch.hppc.ObjectArrayList;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefArray;
-import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.lucene.util.InPlaceMergeSorter;
-import org.apache.lucene.util.IntroSorter;
-
 import java.util.AbstractList;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -40,11 +30,15 @@ import java.util.List;
 import java.util.Objects;
 import java.util.RandomAccess;
 
+import com.carrotsearch.hppc.ObjectArrayList;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefArray;
+import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.lucene.util.InPlaceMergeSorter;
+import org.apache.lucene.util.IntroSorter;
+
 /** Collections-related utility methods. */
 public class CollectionUtils {
-    public static void sort(LongArrayList list) {
-        sort(list.buffer, list.size());
-    }
 
     public static void sort(final long[] array, int len) {
         new IntroSorter() {
@@ -76,29 +70,6 @@ public class CollectionUtils {
         }.sort(0, len);
     }
 
-    public static void sortAndDedup(LongArrayList list) {
-        list.elementsCount = sortAndDedup(list.buffer, list.elementsCount);
-    }
-
-    /** Sort and deduplicate values in-place, then return the unique element count. */
-    public static int sortAndDedup(long[] array, int len) {
-        if (len <= 1) {
-            return len;
-        }
-        sort(array, len);
-        int uniqueCount = 1;
-        for (int i = 1; i < len; ++i) {
-            if (array[i] != array[i - 1]) {
-                array[uniqueCount++] = array[i];
-            }
-        }
-        return uniqueCount;
-    }
-
-    public static void sort(FloatArrayList list) {
-        sort(list.buffer, list.size());
-    }
-
     public static void sort(final float[] array, int len) {
         new IntroSorter() {
 
@@ -129,29 +100,6 @@ public class CollectionUtils {
         }.sort(0, len);
     }
 
-    public static void sortAndDedup(FloatArrayList list) {
-        list.elementsCount = sortAndDedup(list.buffer, list.elementsCount);
-    }
-
-    /** Sort and deduplicate values in-place, then return the unique element count. */
-    public static int sortAndDedup(float[] array, int len) {
-        if (len <= 1) {
-            return len;
-        }
-        sort(array, len);
-        int uniqueCount = 1;
-        for (int i = 1; i < len; ++i) {
-            if (Float.compare(array[i], array[i - 1]) != 0) {
-                array[uniqueCount++] = array[i];
-            }
-        }
-        return uniqueCount;
-    }
-
-    public static void sort(DoubleArrayList list) {
-        sort(list.buffer, list.size());
-    }
-
     public static void sort(final double[] array, int len) {
         new IntroSorter() {
 
@@ -182,25 +130,6 @@ public class CollectionUtils {
         }.sort(0, len);
     }
 
-    public static void sortAndDedup(DoubleArrayList list) {
-        list.elementsCount = sortAndDedup(list.buffer, list.elementsCount);
-    }
-
-    /** Sort and deduplicate values in-place, then return the unique element count. */
-    public static int sortAndDedup(double[] array, int len) {
-        if (len <= 1) {
-            return len;
-        }
-        sort(array, len);
-        int uniqueCount = 1;
-        for (int i = 1; i < len; ++i) {
-            if (Double.compare(array[i], array[i - 1]) != 0) {
-                array[uniqueCount++] = array[i];
-            }
-        }
-        return uniqueCount;
-    }
-
     /**
      * Checks if the given array contains any elements.
      *
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
index 86c9ea86a80..6694018eb31 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
@@ -34,7 +34,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.BaseTransportResponseHandler;
+import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportException;
@@ -232,7 +232,7 @@ public class MasterFaultDetection extends FaultDetection {
             }
             final MasterPingRequest request = new MasterPingRequest(clusterService.localNode().getId(), masterToPing.getId(), clusterName);
             final TransportRequestOptions options = TransportRequestOptions.builder().withType(TransportRequestOptions.Type.PING).withTimeout(pingRetryTimeout).build();
-            transportService.sendRequest(masterToPing, MASTER_PING_ACTION_NAME, request, options, new BaseTransportResponseHandler<MasterPingResponseResponse>() {
+            transportService.sendRequest(masterToPing, MASTER_PING_ACTION_NAME, request, options, new TransportResponseHandler<MasterPingResponseResponse>() {
 
                         @Override
                         public MasterPingResponseResponse newInstance() {
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java b/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java
index 6fb1c82b268..97703e94e31 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java
@@ -28,7 +28,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.BaseTransportResponseHandler;
+import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportException;
@@ -203,7 +203,7 @@ public class NodesFaultDetection extends FaultDetection {
             }
             final PingRequest pingRequest = new PingRequest(node.getId(), clusterName, localNode, clusterStateVersion);
             final TransportRequestOptions options = TransportRequestOptions.builder().withType(TransportRequestOptions.Type.PING).withTimeout(pingRetryTimeout).build();
-            transportService.sendRequest(node, PING_ACTION_NAME, pingRequest, options, new BaseTransportResponseHandler<PingResponse>() {
+            transportService.sendRequest(node, PING_ACTION_NAME, pingRequest, options, new TransportResponseHandler<PingResponse>() {
                         @Override
                         public PingResponse newInstance() {
                             return new PingResponse();
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
index d010587dda4..87e099332ee 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
@@ -46,7 +46,7 @@ import org.elasticsearch.discovery.zen.elect.ElectMasterService;
 import org.elasticsearch.discovery.zen.ping.PingContextProvider;
 import org.elasticsearch.discovery.zen.ping.ZenPing;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.BaseTransportResponseHandler;
+import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.RemoteTransportException;
 import org.elasticsearch.transport.TransportChannel;
@@ -444,7 +444,7 @@ public class UnicastZenPing extends AbstractLifecycleComponent implements ZenPin
                                        final CountDownLatch latch, final DiscoveryNode node, final DiscoveryNode nodeToSend) {
         logger.trace("[{}] sending to {}", id, nodeToSend);
         transportService.sendRequest(nodeToSend, ACTION_NAME, pingRequest, TransportRequestOptions.builder()
-                .withTimeout((long) (timeout.millis() * 1.25)).build(), new BaseTransportResponseHandler<UnicastPingResponse>() {
+                .withTimeout((long) (timeout.millis() * 1.25)).build(), new TransportResponseHandler<UnicastPingResponse>() {
 
             @Override
             public UnicastPingResponse newInstance() {
diff --git a/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java b/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
index 0fcdd16aba7..ac7efcf0b5d 100644
--- a/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
+++ b/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
@@ -50,7 +50,7 @@ import org.elasticsearch.index.shard.ShardNotFoundException;
 import org.elasticsearch.indices.IndexClosedException;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.BaseTransportResponseHandler;
+import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
@@ -270,7 +270,7 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL
             }
             logger.trace("{} retrieving in flight operation count", shardId);
             transportService.sendRequest(primaryNode, IN_FLIGHT_OPS_ACTION_NAME, new InFlightOpsRequest(shardId),
-                    new BaseTransportResponseHandler<InFlightOpsResponse>() {
+                    new TransportResponseHandler<InFlightOpsResponse>() {
                         @Override
                         public InFlightOpsResponse newInstance() {
                             return new InFlightOpsResponse();
@@ -319,7 +319,7 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL
             }
             logger.trace("{} sending synced flush request to {}. sync id [{}].", shardId, shard, syncId);
             transportService.sendRequest(node, SYNCED_FLUSH_ACTION_NAME, new ShardSyncedFlushRequest(shard.shardId(), syncId, expectedCommitId),
-                    new BaseTransportResponseHandler<ShardSyncedFlushResponse>() {
+                    new TransportResponseHandler<ShardSyncedFlushResponse>() {
                         @Override
                         public ShardSyncedFlushResponse newInstance() {
                             return new ShardSyncedFlushResponse();
@@ -373,7 +373,7 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL
                 }
                 continue;
             }
-            transportService.sendRequest(node, PRE_SYNCED_FLUSH_ACTION_NAME, new PreShardSyncedFlushRequest(shard.shardId()), new BaseTransportResponseHandler<PreSyncedFlushResponse>() {
+            transportService.sendRequest(node, PRE_SYNCED_FLUSH_ACTION_NAME, new PreShardSyncedFlushRequest(shard.shardId()), new TransportResponseHandler<PreSyncedFlushResponse>() {
                 @Override
                 public PreSyncedFlushResponse newInstance() {
                     return new PreSyncedFlushResponse();
diff --git a/core/src/main/java/org/elasticsearch/node/service/NodeService.java b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
index 8f938bcb22d..13c9b46d29e 100644
--- a/core/src/main/java/org/elasticsearch/node/service/NodeService.java
+++ b/core/src/main/java/org/elasticsearch/node/service/NodeService.java
@@ -61,15 +61,16 @@ public class NodeService extends AbstractComponent implements Closeable {
     private ScriptService scriptService;
 
     @Nullable
-    private HttpServer httpServer;
+    private final HttpServer httpServer;
 
     private final Discovery discovery;
 
     @Inject
-    public NodeService(Settings settings, ThreadPool threadPool, MonitorService monitorService,
-                       Discovery discovery, TransportService transportService, IndicesService indicesService,
-                       PluginsService pluginService, CircuitBreakerService circuitBreakerService, @Nullable HttpServer httpServer,
-                       ProcessorsRegistry.Builder processorsRegistryBuilder, ClusterService clusterService, SettingsFilter settingsFilter) {
+    public NodeService(Settings settings, ThreadPool threadPool, MonitorService monitorService, Discovery discovery,
+                       TransportService transportService, IndicesService indicesService, PluginsService pluginService,
+                       CircuitBreakerService circuitBreakerService, @Nullable HttpServer httpServer,
+                       ProcessorsRegistry.Builder processorsRegistryBuilder, ClusterService clusterService,
+                       SettingsFilter settingsFilter) {
         super(settings);
         this.threadPool = threadPool;
         this.monitorService = monitorService;
@@ -93,21 +94,6 @@ public class NodeService extends AbstractComponent implements Closeable {
         this.ingestService.buildProcessorsFactoryRegistry(scriptService, clusterService);
     }
 
-    public NodeInfo info() {
-        return new NodeInfo(Version.CURRENT, Build.CURRENT, discovery.localNode(),
-                settings,
-                monitorService.osService().info(),
-                monitorService.processService().info(),
-                monitorService.jvmService().info(),
-                threadPool.info(),
-                transportService.info(),
-                httpServer == null ? null : httpServer.info(),
-                pluginService == null ? null : pluginService.info(),
-                ingestService == null ? null : ingestService.info(),
-                indicesService.getTotalIndexingBufferBytes()
-        );
-    }
-
     public NodeInfo info(boolean settings, boolean os, boolean process, boolean jvm, boolean threadPool,
                 boolean transport, boolean http, boolean plugin, boolean ingest, boolean indices) {
         return new NodeInfo(Version.CURRENT, Build.CURRENT, discovery.localNode(),
@@ -124,25 +110,6 @@ public class NodeService extends AbstractComponent implements Closeable {
         );
     }
 
-    public NodeStats stats() throws IOException {
-        // for indices stats we want to include previous allocated shards stats as well (it will
-        // only be applied to the sensible ones to use, like refresh/merge/flush/indexing stats)
-        return new NodeStats(discovery.localNode(), System.currentTimeMillis(),
-                indicesService.stats(true),
-                monitorService.osService().stats(),
-                monitorService.processService().stats(),
-                monitorService.jvmService().stats(),
-                threadPool.stats(),
-                monitorService.fsService().stats(),
-                transportService.stats(),
-                httpServer == null ? null : httpServer.stats(),
-                circuitBreakerService.stats(),
-                scriptService.stats(),
-                discovery.stats(),
-                ingestService.getPipelineExecutionService().stats()
-        );
-    }
-
     public NodeStats stats(CommonStatsFlags indices, boolean os, boolean process, boolean jvm, boolean threadPool,
                            boolean fs, boolean transport, boolean http, boolean circuitBreaker,
                            boolean script, boolean discoveryStats, boolean ingest) {
diff --git a/core/src/main/java/org/elasticsearch/transport/BaseTransportResponseHandler.java b/core/src/main/java/org/elasticsearch/transport/BaseTransportResponseHandler.java
deleted file mode 100644
index df2362dd47b..00000000000
--- a/core/src/main/java/org/elasticsearch/transport/BaseTransportResponseHandler.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.transport;
-
-/**
- * A simple based class that always spawns.
- */
-public abstract class BaseTransportResponseHandler<T extends TransportResponse> implements TransportResponseHandler<T> {
-
-}
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/transport/FutureTransportResponseHandler.java b/core/src/main/java/org/elasticsearch/transport/FutureTransportResponseHandler.java
index 57679ac5b1b..36b69de2326 100644
--- a/core/src/main/java/org/elasticsearch/transport/FutureTransportResponseHandler.java
+++ b/core/src/main/java/org/elasticsearch/transport/FutureTransportResponseHandler.java
@@ -24,7 +24,7 @@ import org.elasticsearch.threadpool.ThreadPool;
 /**
  * A response handler to be used when all interaction will be done through the {@link TransportFuture}.
  */
-public abstract class FutureTransportResponseHandler<T extends TransportResponse> extends BaseTransportResponseHandler<T> {
+public abstract class FutureTransportResponseHandler<T extends TransportResponse> implements TransportResponseHandler<T> {
 
     @Override
     public void handleResponse(T response) {
diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
index 5c07f5e6f25..a431d6d2d4e 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
@@ -30,7 +30,7 @@ import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.BaseTransportResponseHandler;
+import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
@@ -188,7 +188,7 @@ public class TransportClientNodesServiceTests extends ESTestCase {
                     }
 
                     iteration.transportService.sendRequest(node, "action", new TestRequest(),
-                            TransportRequestOptions.EMPTY, new BaseTransportResponseHandler<TestResponse>() {
+                            TransportRequestOptions.EMPTY, new TransportResponseHandler<TestResponse>() {
                         @Override
                         public TestResponse newInstance() {
                             return new TestResponse();
diff --git a/core/src/test/java/org/elasticsearch/common/settings/bar/BarTestClass.java b/core/src/test/java/org/elasticsearch/common/settings/bar/BarTestClass.java
deleted file mode 100644
index 8c7b0c1f255..00000000000
--- a/core/src/test/java/org/elasticsearch/common/settings/bar/BarTestClass.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.settings.bar;
-
-//used in SettingsTest
-public class BarTestClass {
-}
diff --git a/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
index 9ae029a4aa4..c33db33b1f8 100644
--- a/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
+++ b/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
@@ -153,7 +153,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         });
 
         TransportFuture<StringMessageResponse> res = serviceB.submitRequest(nodeA, "sayHello",
-                new StringMessageRequest("moshe"), new BaseTransportResponseHandler<StringMessageResponse>() {
+                new StringMessageRequest("moshe"), new TransportResponseHandler<StringMessageResponse>() {
                     @Override
                     public StringMessageResponse newInstance() {
                         return new StringMessageResponse();
@@ -184,7 +184,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         }
 
         res = serviceB.submitRequest(nodeA, "sayHello", new StringMessageRequest("moshe"),
-            TransportRequestOptions.builder().withCompress(true).build(), new BaseTransportResponseHandler<StringMessageResponse>() {
+            TransportRequestOptions.builder().withCompress(true).build(), new TransportResponseHandler<StringMessageResponse>() {
                     @Override
                     public StringMessageResponse newInstance() {
                         return new StringMessageResponse();
@@ -233,7 +233,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         });
         final Object context = new Object();
         final String executor = randomFrom(ThreadPool.THREAD_POOL_TYPES.keySet().toArray(new String[0]));
-        TransportResponseHandler<StringMessageResponse> responseHandler = new BaseTransportResponseHandler<StringMessageResponse>() {
+        TransportResponseHandler<StringMessageResponse> responseHandler = new TransportResponseHandler<StringMessageResponse>() {
             @Override
             public StringMessageResponse newInstance() {
                 return new StringMessageResponse();
@@ -336,7 +336,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
         TransportFuture<TransportResponse.Empty> res = serviceB.submitRequest(nodeA, "sayHello",
                 TransportRequest.Empty.INSTANCE, TransportRequestOptions.builder().withCompress(true).build(),
-            new BaseTransportResponseHandler<TransportResponse.Empty>() {
+            new TransportResponseHandler<TransportResponse.Empty>() {
                 @Override
                 public TransportResponse.Empty newInstance() {
                     return TransportResponse.Empty.INSTANCE;
@@ -386,7 +386,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
         TransportFuture<StringMessageResponse> res = serviceB.submitRequest(nodeA, "sayHello",
                 new StringMessageRequest("moshe"), TransportRequestOptions.builder().withCompress(true).build(),
-            new BaseTransportResponseHandler<StringMessageResponse>() {
+            new TransportResponseHandler<StringMessageResponse>() {
                 @Override
                 public StringMessageResponse newInstance() {
                     return new StringMessageResponse();
@@ -430,7 +430,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         });
 
         TransportFuture<StringMessageResponse> res = serviceB.submitRequest(nodeA, "sayHelloException",
-                new StringMessageRequest("moshe"), new BaseTransportResponseHandler<StringMessageResponse>() {
+                new StringMessageRequest("moshe"), new TransportResponseHandler<StringMessageResponse>() {
                     @Override
                     public StringMessageResponse newInstance() {
                         return new StringMessageResponse();
@@ -520,7 +520,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
         TransportFuture<StringMessageResponse> res = serviceB.submitRequest(nodeA, "sayHelloTimeoutNoResponse",
                 new StringMessageRequest("moshe"), TransportRequestOptions.builder().withTimeout(100).build(),
-            new BaseTransportResponseHandler<StringMessageResponse>() {
+            new TransportResponseHandler<StringMessageResponse>() {
                 @Override
                 public StringMessageResponse newInstance() {
                     return new StringMessageResponse();
@@ -575,7 +575,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         final CountDownLatch latch = new CountDownLatch(1);
         TransportFuture<StringMessageResponse> res = serviceB.submitRequest(nodeA, "sayHelloTimeoutDelayedResponse",
                 new StringMessageRequest("2m"), TransportRequestOptions.builder().withTimeout(100).build(),
-            new BaseTransportResponseHandler<StringMessageResponse>() {
+            new TransportResponseHandler<StringMessageResponse>() {
                 @Override
                 public StringMessageResponse newInstance() {
                     return new StringMessageResponse();
@@ -612,7 +612,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
             // now, try and send another request, this times, with a short timeout
             res = serviceB.submitRequest(nodeA, "sayHelloTimeoutDelayedResponse",
                     new StringMessageRequest(counter + "ms"), TransportRequestOptions.builder().withTimeout(3000).build(),
-                new BaseTransportResponseHandler<StringMessageResponse>() {
+                new TransportResponseHandler<StringMessageResponse>() {
                     @Override
                     public StringMessageResponse newInstance() {
                         return new StringMessageResponse();
@@ -664,7 +664,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         };
 
         final Semaphore requestCompleted = new Semaphore(0);
-        TransportResponseHandler noopResponseHandler = new BaseTransportResponseHandler<StringMessageResponse>() {
+        TransportResponseHandler noopResponseHandler = new TransportResponseHandler<StringMessageResponse>() {
 
             @Override
             public StringMessageResponse newInstance() {
@@ -967,7 +967,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         Version0Request version0Request = new Version0Request();
         version0Request.value1 = 1;
         Version0Response version0Response = serviceA.submitRequest(nodeB, "/version", version0Request,
-            new BaseTransportResponseHandler<Version0Response>() {
+            new TransportResponseHandler<Version0Response>() {
                 @Override
                 public Version0Response newInstance() {
                     return new Version0Response();
@@ -1009,7 +1009,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         version1Request.value1 = 1;
         version1Request.value2 = 2;
         Version1Response version1Response = serviceB.submitRequest(nodeA, "/version", version1Request,
-            new BaseTransportResponseHandler<Version1Response>() {
+            new TransportResponseHandler<Version1Response>() {
                 @Override
                 public Version1Response newInstance() {
                     return new Version1Response();
@@ -1055,7 +1055,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         version1Request.value1 = 1;
         version1Request.value2 = 2;
         Version1Response version1Response = serviceB.submitRequest(nodeB, "/version", version1Request,
-            new BaseTransportResponseHandler<Version1Response>() {
+            new TransportResponseHandler<Version1Response>() {
                 @Override
                 public Version1Response newInstance() {
                     return new Version1Response();
@@ -1098,7 +1098,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         Version0Request version0Request = new Version0Request();
         version0Request.value1 = 1;
         Version0Response version0Response = serviceA.submitRequest(nodeA, "/version", version0Request,
-            new BaseTransportResponseHandler<Version0Response>() {
+            new TransportResponseHandler<Version0Response>() {
                 @Override
                 public Version0Response newInstance() {
                     return new Version0Response();
@@ -1137,7 +1137,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         serviceB.addFailToSendNoConnectRule(serviceA);
 
         TransportFuture<StringMessageResponse> res = serviceB.submitRequest(nodeA, "sayHello",
-                new StringMessageRequest("moshe"), new BaseTransportResponseHandler<StringMessageResponse>() {
+                new StringMessageRequest("moshe"), new TransportResponseHandler<StringMessageResponse>() {
                     @Override
                     public StringMessageResponse newInstance() {
                         return new StringMessageResponse();
@@ -1197,7 +1197,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
         TransportFuture<StringMessageResponse> res = serviceB.submitRequest(nodeA, "sayHello",
                 new StringMessageRequest("moshe"), TransportRequestOptions.builder().withTimeout(100).build(),
-            new BaseTransportResponseHandler<StringMessageResponse>() {
+            new TransportResponseHandler<StringMessageResponse>() {
                 @Override
                 public StringMessageResponse newInstance() {
                     return new StringMessageResponse();
diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java
index c69f56c2cbd..f089e9068be 100644
--- a/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java
+++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyScheduledPingTests.java
@@ -31,7 +31,7 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.BaseTransportResponseHandler;
+import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.TransportChannel;
 import org.elasticsearch.transport.TransportException;
@@ -114,7 +114,7 @@ public class NettyScheduledPingTests extends ESTestCase {
         for (int i = 0; i < rounds; i++) {
             serviceB.submitRequest(nodeA, "sayHello",
                     TransportRequest.Empty.INSTANCE, TransportRequestOptions.builder().withCompress(randomBoolean()).build(),
-                new BaseTransportResponseHandler<TransportResponse.Empty>() {
+                new TransportResponseHandler<TransportResponse.Empty>() {
                         @Override
                         public TransportResponse.Empty newInstance() {
                             return TransportResponse.Empty.INSTANCE;

From c8a9b28d1b7c545ff04cdcd58fe79f096bbb408f Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Thu, 30 Jun 2016 11:41:25 +0200
Subject: [PATCH 1213/1311] Rename Search Template REST tests

---
 .../10_basic.yaml                             |  6 +-
 .../20_stored_template.yaml                   |  2 +-
 .../30_file_template.yaml                     |  2 +-
 ...ate.msearch.json => msearch_template.json} |  2 +-
 .../rest-api-spec/api/template.search.json    | 56 -------------------
 5 files changed, 6 insertions(+), 62 deletions(-)
 rename modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/{template.msearch => msearch_template}/10_basic.yaml (97%)
 rename modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/{template.msearch => msearch_template}/20_stored_template.yaml (98%)
 rename modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/{template.msearch => msearch_template}/30_file_template.yaml (98%)
 rename rest-api-spec/src/main/resources/rest-api-spec/api/{template.msearch.json => msearch_template.json} (97%)
 delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/template.search.json

diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/10_basic.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/10_basic.yaml
similarity index 97%
rename from modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/10_basic.yaml
rename to modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/10_basic.yaml
index 2a32e75331e..ab42133a2b2 100644
--- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/10_basic.yaml
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/10_basic.yaml
@@ -36,7 +36,7 @@ setup:
 "Basic multi-search template":
 
   - do:
-      template.msearch:
+      msearch_template:
         body:
           - index: index_*
           - inline: '{"query": {"match": {"foo": "{{value}}"} } }'
@@ -71,7 +71,7 @@ setup:
 "Multi-search template with errors":
 
   - do:
-      template.msearch:
+      msearch_template:
         body:
           # Search 0 is OK
           - index: index_*
@@ -109,7 +109,7 @@ setup:
 
   - do:
       catch: /(.)*action_request_validation_exception(.)*template.is.missing(.)*/
-      template.msearch:
+      msearch_template:
         body:
           # Search 0 is OK
           - index: index_*
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/20_stored_template.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/20_stored_template.yaml
similarity index 98%
rename from modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/20_stored_template.yaml
rename to modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/20_stored_template.yaml
index 5d04449e541..ffbe9200375 100644
--- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/20_stored_template.yaml
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/20_stored_template.yaml
@@ -42,7 +42,7 @@ setup:
   - match: { acknowledged: true }
 
   - do:
-      template.msearch:
+      msearch_template:
         body:
           - index: index_*
           - id: stored_template_1
diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/30_file_template.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/30_file_template.yaml
similarity index 98%
rename from modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/30_file_template.yaml
rename to modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/30_file_template.yaml
index 0c6ad8021e1..a5d11898baf 100644
--- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/template.msearch/30_file_template.yaml
+++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/msearch_template/30_file_template.yaml
@@ -44,7 +44,7 @@ setup:
   - match: { template_output.size: 20 }
 
   - do:
-      template.msearch:
+      msearch_template:
         body:
           - index: index_*
           - file: template_1
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/template.msearch.json b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json
similarity index 97%
rename from rest-api-spec/src/main/resources/rest-api-spec/api/template.msearch.json
rename to rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json
index 379b7bdf362..39aa53b2572 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/template.msearch.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json
@@ -1,5 +1,5 @@
 {
-  "template.msearch": {
+  "msearch_template": {
     "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/search-template.html",
     "methods": ["GET", "POST"],
     "url": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/template.search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/template.search.json
deleted file mode 100644
index 6a2a8c1d7e2..00000000000
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/template.search.json
+++ /dev/null
@@ -1,56 +0,0 @@
-{
-  "template.search": {
-    "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/current/search-template.html",
-    "methods": ["GET", "POST"],
-    "url": {
-      "path": "/_search/template",
-      "paths": ["/_search/template", "/{index}/_search/template", "/{index}/{type}/_search/template"],
-      "parts": {
-        "index": {
-         "type" : "list",
-         "description" : "A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices"
-        },
-        "type": {
-          "type" : "list",
-          "description" : "A comma-separated list of document types to search; leave empty to perform the operation on all types"
-        }
-      },
-      "params" : {
-        "ignore_unavailable": {
-            "type" : "boolean",
-            "description" : "Whether specified concrete indices should be ignored when unavailable (missing or closed)"
-        },
-        "allow_no_indices": {
-            "type" : "boolean",
-            "description" : "Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)"
-        },
-        "expand_wildcards": {
-            "type" : "enum",
-            "options" : ["open","closed","none","all"],
-            "default" : "open",
-            "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both."
-        },
-        "preference": {
-          "type" : "string",
-          "description" : "Specify the node or shard the operation should be performed on (default: random)"
-        },
-        "routing": {
-          "type" : "list",
-          "description" : "A comma-separated list of specific routing values"
-        },
-        "scroll": {
-          "type" : "duration",
-          "description" : "Specify how long a consistent view of the index should be maintained for scrolled search"
-        },
-        "search_type": {
-          "type" : "enum",
-          "options" : ["query_then_fetch", "query_and_fetch", "dfs_query_then_fetch", "dfs_query_and_fetch"],
-          "description" : "Search operation type"
-        }
-      }
-    },
-    "body": {
-      "description": "The search definition template and its params"
-    }
-  }
-}

From 4131476db1da90ca7a0409f67127e0feab2eef4d Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@elastic.co>
Date: Mon, 4 Jul 2016 09:45:25 +0200
Subject: [PATCH 1214/1311] Propagate canTripCircuitBreaker for all broadcasted
 actions

With this commit we also propagate the `canTripCircuitBreaker`
setting for the main action in TransportBroadcastByNodeAction.

Previously, we set it only on the additional action added by
this handler.
---
 .../support/broadcast/node/TransportBroadcastByNodeAction.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java b/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
index 12184d57cd8..178e2b69a7b 100644
--- a/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
@@ -109,7 +109,8 @@ public abstract class TransportBroadcastByNodeAction<Request extends BroadcastRe
             Supplier<Request> request,
             String executor,
             boolean canTripCircuitBreaker) {
-        super(settings, actionName, threadPool, transportService, actionFilters, indexNameExpressionResolver, request);
+        super(settings, actionName, canTripCircuitBreaker, threadPool, transportService, actionFilters, indexNameExpressionResolver,
+            request);
 
         this.clusterService = clusterService;
         this.transportService = transportService;

From 25881c265bab7835271dc0b4afcc5a8ecfbbc6c1 Mon Sep 17 00:00:00 2001
From: Daniel Mitterdorfer <daniel.mitterdorfer@elastic.co>
Date: Mon, 4 Jul 2016 09:52:16 +0200
Subject: [PATCH 1215/1311] Improve error checking and reporting in CBSIT

---
 .../breaker/CircuitBreakerServiceIT.java      | 30 ++++++++++++-------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
index 0ae8f71c742..b448f35c21b 100644
--- a/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/memory/breaker/CircuitBreakerServiceIT.java
@@ -28,6 +28,8 @@ import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
@@ -62,7 +64,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.instanceOf;
-import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.startsWith;
@@ -70,7 +71,7 @@ import static org.hamcrest.Matchers.startsWith;
 /**
  * Integration tests for InternalCircuitBreakerService
  */
-@ClusterScope(scope = TEST, randomDynamicTemplates = false)
+@ClusterScope(scope = TEST, randomDynamicTemplates = false, numClientNodes = 0, maxNumDataNodes = 1)
 public class CircuitBreakerServiceIT extends ESIntegTestCase {
     /** Reset all breaker settings back to their defaults */
     private void reset() {
@@ -266,17 +267,26 @@ public class CircuitBreakerServiceIT extends ESIntegTestCase {
 
         // Perform a search to load field data for the "test" field
         try {
-            client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC).get();
-            fail("should have thrown an exception");
+            SearchResponse searchResponse = client.prepareSearch("cb-test").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC).get();
+            if (searchResponse.getShardFailures().length > 0) {
+                // each shard must have failed with CircuitBreakingException
+                for (ShardSearchFailure shardSearchFailure : searchResponse.getShardFailures()) {
+                    Throwable cause = ExceptionsHelper.unwrap(shardSearchFailure.getCause(), CircuitBreakingException.class);
+                    assertThat(cause, instanceOf(CircuitBreakingException.class));
+                    assertEquals(((CircuitBreakingException) cause).getByteLimit(), 500L);
+                }
+            } else {
+                fail("should have thrown a CircuitBreakingException");
+            }
         } catch (Exception e) {
-            final Throwable cause = ExceptionsHelper.unwrap(e, CircuitBreakingException.class);
-            assertNotNull("CircuitBreakingException is not the cause of " + e, cause);
-            String errMsg = "would be larger than limit of [500/500b]]";
-            assertThat("Exception: [" + cause.toString() + "] should contain a CircuitBreakingException",
+            Throwable cause = ExceptionsHelper.unwrap(e, CircuitBreakingException.class);
+            assertThat(cause, instanceOf(CircuitBreakingException.class));
+            assertEquals(((CircuitBreakingException) cause).getByteLimit(), 500L);
+            assertThat("Exception: [" + cause.toString() + "] should be caused by the parent circuit breaker",
                 cause.toString(), startsWith("CircuitBreakingException[[parent] Data too large"));
-            assertThat("Exception: [" + cause.toString() + "] should contain a CircuitBreakingException",
-                cause.toString(), endsWith(errMsg));
         }
+
+        reset();
     }
 
     public void testRequestBreaker() throws Exception {

From afe99fcdcde62af9184658b60646d3af0aec9233 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Tue, 21 Jun 2016 11:27:27 +0200
Subject: [PATCH 1216/1311] Restore reverted change now that alpha4 is out:

Rename `fields` to `stored_fields` and add `docvalue_fields`

`stored_fields` parameter will no longer try to retrieve fields from the _source but will only return stored fields.
`fields` will throw an exception if the user uses it.
Add `docvalue_fields` as an adjunct to `fielddata_fields` which is deprecated. `docvalue_fields` will try to load the value from the docvalue and fallback to fielddata cache if docvalues are not enabled on that field.

Closes #18943
---
 .../action/search/SearchRequestBuilder.java   |  41 +++-
 .../index/query/InnerHitBuilder.java          | 228 ++++++++++++------
 .../rest/action/search/RestSearchAction.java  |  31 ++-
 .../elasticsearch/search/SearchService.java   |   8 +-
 .../tophits/TopHitsAggregationBuilder.java    |  12 +-
 .../search/builder/SearchSourceBuilder.java   | 174 ++++++-------
 .../index/IndexWithShadowReplicasIT.java      |   2 +-
 .../TokenCountFieldMapperIntegrationIT.java   |   4 +-
 .../mapper/geo/GeoPointFieldMapperTests.java  |   8 +-
 .../index/query/InnerHitBuilderTests.java     |  14 +-
 .../index/store/ExceptionRetryIT.java         |   2 +-
 .../template/SimpleIndexTemplateIT.java       |   4 +-
 .../elasticsearch/recovery/RelocationIT.java  |   2 +-
 .../metrics/AbstractGeoTestCase.java          |   2 +-
 .../builder/SearchSourceBuilderTests.java     |   4 +-
 .../search/child/ChildQuerySearchIT.java      |   8 +-
 .../elasticsearch/search/geo/GeoFilterIT.java |   4 +-
 .../search/innerhits/InnerHitsIT.java         |   4 +-
 .../search/source/SourceFetchingIT.java       |   4 +-
 .../update/TimestampTTLBWIT.java              |   2 +-
 docs/plugins/mapper-attachments.asciidoc      |   4 +-
 .../metrics/tophits-aggregation.asciidoc      |   2 +-
 docs/reference/mapping/params/store.asciidoc  |   2 +-
 .../migration/migrate_5_0/search.asciidoc     |  10 +-
 .../modules/scripting/painless.asciidoc       |   2 +-
 docs/reference/search/request-body.asciidoc   |   4 +-
 .../search/request/docvalue-fields.asciidoc   |  23 ++
 .../search/request/fielddata-fields.asciidoc  |  23 --
 .../search/request/highlighting.asciidoc      |   2 +-
 .../search/request/inner-hits.asciidoc        |   2 +-
 ...fields.asciidoc => stored-fields.asciidoc} |   6 +-
 docs/reference/search/uri-request.asciidoc    |   2 +-
 .../messy/tests/GeoDistanceTests.java         |  16 +-
 .../messy/tests/SearchFieldsTests.java        |  66 ++---
 .../reindex/AbstractBulkByScrollRequest.java  |   4 +-
 .../test/delete_by_query/20_validation.yaml   |   4 +-
 .../test/reindex/20_validation.yaml           |   4 +-
 .../test/update_by_query/20_validation.yaml   |   4 +-
 .../test/mapper_attachments/30_mapping.yaml   |   2 +-
 .../test/mapper_attachments/40_highlight.yaml |   2 +-
 .../50_files_supported.yaml                   |   4 +-
 .../size/SizeFieldMapperUpgradeTests.java     |   4 +-
 .../resources/rest-api-spec/api/search.json   |  10 +-
 .../test/search/10_source_filtering.yaml      |   7 +-
 .../rest-api-spec/test/search/issue4895.yaml  |   2 +-
 45 files changed, 450 insertions(+), 319 deletions(-)
 create mode 100644 docs/reference/search/request/docvalue-fields.asciidoc
 delete mode 100644 docs/reference/search/request/fielddata-fields.asciidoc
 rename docs/reference/search/request/{fields.asciidoc => stored-fields.asciidoc} (91%)

diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
index 2297c98b636..5c08acb99ea 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java
@@ -252,8 +252,8 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
     /**
      * Sets no fields to be loaded, resulting in only id and type to be returned per field.
      */
-    public SearchRequestBuilder setNoFields() {
-        sourceBuilder().noFields();
+    public SearchRequestBuilder setNoStoredFields() {
+        sourceBuilder().noStoredFields();
         return this;
     }
 
@@ -289,13 +289,23 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
         return this;
     }
 
+    /**
+     * Adds a docvalue based field to load and return. The field does not have to be stored,
+     * but its recommended to use non analyzed or numeric fields.
+     *
+     * @param name The field to get from the docvalue
+     */
+    public SearchRequestBuilder addDocValueField(String name) {
+        sourceBuilder().docValueField(name);
+        return this;
+    }
 
     /**
-     * Adds a field to load and return (note, it must be stored) as part of the search request.
+     * Adds a stored field to load and return (note, it must be stored) as part of the search request.
      * If none are specified, the source of the document will be return.
      */
-    public SearchRequestBuilder addField(String field) {
-        sourceBuilder().field(field);
+    public SearchRequestBuilder addStoredField(String field) {
+        sourceBuilder().storedField(field);
         return this;
     }
 
@@ -304,12 +314,15 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
      * but its recommended to use non analyzed or numeric fields.
      *
      * @param name The field to get from the field data cache
+     * @deprecated Use {@link SearchRequestBuilder#addDocValueField(String)} instead.
      */
+    @Deprecated
     public SearchRequestBuilder addFieldDataField(String name) {
-        sourceBuilder().fieldDataField(name);
+        sourceBuilder().docValueField(name);
         return this;
     }
 
+
     /**
      * Adds a script based field to load and return. The field does not have to be stored,
      * but its recommended to use non analyzed or numeric fields.
@@ -366,12 +379,24 @@ public class SearchRequestBuilder extends ActionRequestBuilder<SearchRequest, Se
         return this;
     }
 
+    /**
+     * Sets the stored fields to load and return as part of the search request. If none
+     * are specified, the source of the document will be returned.
+     *
+     * @deprecated Use {@link SearchRequestBuilder#storedFields(String...)} instead.
+     */
+    @Deprecated
+    public SearchRequestBuilder fields(String... fields) {
+        sourceBuilder().storedFields(Arrays.asList(fields));
+        return this;
+    }
+
     /**
      * Sets the fields to load and return as part of the search request. If none
      * are specified, the source of the document will be returned.
      */
-    public SearchRequestBuilder fields(String... fields) {
-        sourceBuilder().fields(Arrays.asList(fields));
+    public SearchRequestBuilder storedFields(String... fields) {
+        sourceBuilder().storedFields(Arrays.asList(fields));
         return this;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
index 631bc163156..5afb9ddc24c 100644
--- a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
@@ -71,8 +71,14 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         PARSER.declareBoolean(InnerHitBuilder::setExplain, SearchSourceBuilder.EXPLAIN_FIELD);
         PARSER.declareBoolean(InnerHitBuilder::setVersion, SearchSourceBuilder.VERSION_FIELD);
         PARSER.declareBoolean(InnerHitBuilder::setTrackScores, SearchSourceBuilder.TRACK_SCORES_FIELD);
-        PARSER.declareStringArray(InnerHitBuilder::setFieldNames, SearchSourceBuilder.FIELDS_FIELD);
-        PARSER.declareStringArray(InnerHitBuilder::setFieldDataFields, SearchSourceBuilder.FIELDDATA_FIELDS_FIELD);
+        PARSER.declareStringArray(InnerHitBuilder::setStoredFieldNames, SearchSourceBuilder.STORED_FIELDS_FIELD);
+        PARSER.declareField((p, i, c) -> {
+            throw new ParsingException(p.getTokenLocation(), "The field [" +
+                SearchSourceBuilder.FIELDS_FIELD + "] is not longer supported, please use [" +
+                SearchSourceBuilder.STORED_FIELDS_FIELD + "] to retrieve stored fields or _source filtering " +
+                "if the field is not stored");
+        }, SearchSourceBuilder.FIELDS_FIELD, ObjectParser.ValueType.STRING_ARRAY);
+        PARSER.declareStringArray(InnerHitBuilder::setDocValueFields, SearchSourceBuilder.DOCVALUE_FIELDS_FIELD);
         PARSER.declareField((p, i, c) -> {
             try {
                 Set<ScriptField> scriptFields = new HashSet<>();
@@ -131,10 +137,10 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
     private boolean version;
     private boolean trackScores;
 
-    private List<String> fieldNames;
+    private List<String> storedFieldNames;
     private QueryBuilder query = DEFAULT_INNER_HIT_QUERY;
     private List<SortBuilder<?>> sorts;
-    private List<String> fieldDataFields;
+    private List<String> docValueFields;
     private Set<ScriptField> scriptFields;
     private HighlightBuilder highlightBuilder;
     private FetchSourceContext fetchSourceContext;
@@ -143,46 +149,6 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
     public InnerHitBuilder() {
     }
 
-    /**
-     * Read from a stream.
-     */
-    public InnerHitBuilder(StreamInput in) throws IOException {
-        name = in.readOptionalString();
-        nestedPath = in.readOptionalString();
-        parentChildType = in.readOptionalString();
-        from = in.readVInt();
-        size = in.readVInt();
-        explain = in.readBoolean();
-        version = in.readBoolean();
-        trackScores = in.readBoolean();
-        fieldNames = (List<String>) in.readGenericValue();
-        fieldDataFields = (List<String>) in.readGenericValue();
-        if (in.readBoolean()) {
-            int size = in.readVInt();
-            scriptFields = new HashSet<>(size);
-            for (int i = 0; i < size; i++) {
-                scriptFields.add(new ScriptField(in));
-            }
-        }
-        fetchSourceContext = in.readOptionalStreamable(FetchSourceContext::new);
-        if (in.readBoolean()) {
-            int size = in.readVInt();
-            sorts = new ArrayList<>(size);
-            for (int i = 0; i < size; i++) {
-                sorts.add(in.readNamedWriteable(SortBuilder.class));
-            }
-        }
-        highlightBuilder = in.readOptionalWriteable(HighlightBuilder::new);
-        query = in.readNamedWriteable(QueryBuilder.class);
-        if (in.readBoolean()) {
-            int size = in.readVInt();
-            childInnerHits = new HashMap<>(size);
-            for (int i = 0; i < size; i++) {
-                childInnerHits.put(in.readString(), new InnerHitBuilder(in));
-            }
-        }
-    }
-
     private InnerHitBuilder(InnerHitBuilder other) {
         name = other.name;
         from = other.from;
@@ -190,11 +156,11 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         explain = other.explain;
         version = other.version;
         trackScores = other.trackScores;
-        if (other.fieldNames != null) {
-            fieldNames = new ArrayList<>(other.fieldNames);
+        if (other.storedFieldNames != null) {
+            storedFieldNames = new ArrayList<>(other.storedFieldNames);
         }
-        if (other.fieldDataFields != null) {
-            fieldDataFields = new ArrayList<>(other.fieldDataFields);
+        if (other.docValueFields != null) {
+            docValueFields = new ArrayList<>(other.docValueFields);
         }
         if (other.scriptFields != null) {
             scriptFields = new HashSet<>(other.scriptFields);
@@ -232,6 +198,46 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         }
     }
 
+    /**
+     * Read from a stream.
+     */
+    public InnerHitBuilder(StreamInput in) throws IOException {
+        name = in.readOptionalString();
+        nestedPath = in.readOptionalString();
+        parentChildType = in.readOptionalString();
+        from = in.readVInt();
+        size = in.readVInt();
+        explain = in.readBoolean();
+        version = in.readBoolean();
+        trackScores = in.readBoolean();
+        storedFieldNames = (List<String>) in.readGenericValue();
+        docValueFields = (List<String>) in.readGenericValue();
+        if (in.readBoolean()) {
+            int size = in.readVInt();
+            scriptFields = new HashSet<>(size);
+            for (int i = 0; i < size; i++) {
+                scriptFields.add(new ScriptField(in));
+            }
+        }
+        fetchSourceContext = in.readOptionalStreamable(FetchSourceContext::new);
+        if (in.readBoolean()) {
+            int size = in.readVInt();
+            sorts = new ArrayList<>(size);
+            for (int i = 0; i < size; i++) {
+                sorts.add(in.readNamedWriteable(SortBuilder.class));
+            }
+        }
+        highlightBuilder = in.readOptionalWriteable(HighlightBuilder::new);
+        query = in.readNamedWriteable(QueryBuilder.class);
+        if (in.readBoolean()) {
+            int size = in.readVInt();
+            childInnerHits = new HashMap<>(size);
+            for (int i = 0; i < size; i++) {
+                childInnerHits.put(in.readString(), new InnerHitBuilder(in));
+            }
+        }
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(name);
@@ -242,8 +248,8 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         out.writeBoolean(explain);
         out.writeBoolean(version);
         out.writeBoolean(trackScores);
-        out.writeGenericValue(fieldNames);
-        out.writeGenericValue(fieldDataFields);
+        out.writeGenericValue(storedFieldNames);
+        out.writeGenericValue(docValueFields);
         boolean hasScriptFields = scriptFields != null;
         out.writeBoolean(hasScriptFields);
         if (hasScriptFields) {
@@ -334,29 +340,103 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         return this;
     }
 
+    /**
+     * Gets the stored fields to load and return.
+     *
+     * @deprecated Use {@link InnerHitBuilder#getStoredFieldNames()} instead.
+     */
+    @Deprecated
     public List<String> getFieldNames() {
-        return fieldNames;
+        return storedFieldNames;
     }
 
+    /**
+     * Sets the stored fields to load and return. If none
+     * are specified, the source of the document will be returned.
+     *
+     * @deprecated Use {@link InnerHitBuilder#setStoredFieldNames(List)} instead.
+     */
+    @Deprecated
     public InnerHitBuilder setFieldNames(List<String> fieldNames) {
-        this.fieldNames = fieldNames;
+        this.storedFieldNames = fieldNames;
         return this;
     }
 
+
+    /**
+     * Gets the stored fields to load and return.
+     */
+    public List<String> getStoredFieldNames() {
+        return storedFieldNames;
+    }
+
+    /**
+     * Sets the stored fields to load and return. If none
+     * are specified, the source of the document will be returned.
+     */
+    public InnerHitBuilder setStoredFieldNames(List<String> fieldNames) {
+        this.storedFieldNames = fieldNames;
+        return this;
+    }
+
+    /**
+     * Gets the docvalue fields.
+     *
+     * @deprecated Use {@link InnerHitBuilder#getDocValueFields()} instead.
+     */
+    @Deprecated
     public List<String> getFieldDataFields() {
-        return fieldDataFields;
+        return docValueFields;
     }
 
+    /**
+     * Sets the stored fields to load from the docvalue and return.
+     *
+     * @deprecated Use {@link InnerHitBuilder#setDocValueFields(List)} instead.
+     */
+    @Deprecated
     public InnerHitBuilder setFieldDataFields(List<String> fieldDataFields) {
-        this.fieldDataFields = fieldDataFields;
+        this.docValueFields = fieldDataFields;
         return this;
     }
 
+    /**
+     * Adds a field to load from the docvalue and return.
+     *
+     * @deprecated Use {@link InnerHitBuilder#addDocValueField(String)} instead.
+     */
+    @Deprecated
     public InnerHitBuilder addFieldDataField(String field) {
-        if (fieldDataFields == null) {
-            fieldDataFields = new ArrayList<>();
+        if (docValueFields == null) {
+            docValueFields = new ArrayList<>();
         }
-        fieldDataFields.add(field);
+        docValueFields.add(field);
+        return this;
+    }
+
+    /**
+     * Gets the docvalue fields.
+     */
+    public List<String> getDocValueFields() {
+        return docValueFields;
+    }
+
+    /**
+     * Sets the stored fields to load from the docvalue and return.
+     */
+    public InnerHitBuilder setDocValueFields(List<String> docValueFields) {
+        this.docValueFields = docValueFields;
+        return this;
+    }
+
+    /**
+     * Adds a field to load from the docvalue and return.
+     */
+    public InnerHitBuilder addDocValueField(String field) {
+        if (docValueFields == null) {
+            docValueFields = new ArrayList<>();
+        }
+        docValueFields.add(field);
         return this;
     }
 
@@ -484,19 +564,19 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         innerHitsContext.explain(explain);
         innerHitsContext.version(version);
         innerHitsContext.trackScores(trackScores);
-        if (fieldNames != null) {
-            if (fieldNames.isEmpty()) {
+        if (storedFieldNames != null) {
+            if (storedFieldNames.isEmpty()) {
                 innerHitsContext.emptyFieldNames();
             } else {
-                for (String fieldName : fieldNames) {
+                for (String fieldName : storedFieldNames) {
                     innerHitsContext.fieldNames().add(fieldName);
                 }
             }
         }
-        if (fieldDataFields != null) {
+        if (docValueFields != null) {
             FieldDataFieldsContext fieldDataFieldsContext = innerHitsContext
                     .getFetchSubPhaseContext(FieldDataFieldsFetchSubPhase.CONTEXT_FACTORY);
-            for (String field : fieldDataFields) {
+            for (String field : docValueFields) {
                 fieldDataFieldsContext.add(new FieldDataFieldsContext.FieldDataField(field));
             }
             fieldDataFieldsContext.setHitExecutionNeeded(true);
@@ -553,20 +633,20 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         if (fetchSourceContext != null) {
             builder.field(SearchSourceBuilder._SOURCE_FIELD.getPreferredName(), fetchSourceContext, params);
         }
-        if (fieldNames != null) {
-            if (fieldNames.size() == 1) {
-                builder.field(SearchSourceBuilder.FIELDS_FIELD.getPreferredName(), fieldNames.get(0));
+        if (storedFieldNames != null) {
+            if (storedFieldNames.size() == 1) {
+                builder.field(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), storedFieldNames.get(0));
             } else {
-                builder.startArray(SearchSourceBuilder.FIELDS_FIELD.getPreferredName());
-                for (String fieldName : fieldNames) {
+                builder.startArray(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName());
+                for (String fieldName : storedFieldNames) {
                     builder.value(fieldName);
                 }
                 builder.endArray();
             }
         }
-        if (fieldDataFields != null) {
-            builder.startArray(SearchSourceBuilder.FIELDDATA_FIELDS_FIELD.getPreferredName());
-            for (String fieldDataField : fieldDataFields) {
+        if (docValueFields != null) {
+            builder.startArray(SearchSourceBuilder.DOCVALUE_FIELDS_FIELD.getPreferredName());
+            for (String fieldDataField : docValueFields) {
                 builder.value(fieldDataField);
             }
             builder.endArray();
@@ -613,8 +693,8 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
                 Objects.equals(explain, that.explain) &&
                 Objects.equals(version, that.version) &&
                 Objects.equals(trackScores, that.trackScores) &&
-                Objects.equals(fieldNames, that.fieldNames) &&
-                Objects.equals(fieldDataFields, that.fieldDataFields) &&
+                Objects.equals(storedFieldNames, that.storedFieldNames) &&
+                Objects.equals(docValueFields, that.docValueFields) &&
                 Objects.equals(scriptFields, that.scriptFields) &&
                 Objects.equals(fetchSourceContext, that.fetchSourceContext) &&
                 Objects.equals(sorts, that.sorts) &&
@@ -625,8 +705,8 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
 
     @Override
     public int hashCode() {
-        return Objects.hash(name, nestedPath, parentChildType, from, size, explain, version, trackScores, fieldNames,
-                fieldDataFields, scriptFields, fetchSourceContext, sorts, highlightBuilder, query, childInnerHits);
+        return Objects.hash(name, nestedPath, parentChildType, from, size, explain, version, trackScores, storedFieldNames,
+            docValueFields, scriptFields, fetchSourceContext, sorts, highlightBuilder, query, childInnerHits);
     }
 
     public static InnerHitBuilder fromXContent(QueryParseContext context) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
index 870d4a9eb20..80a9f2bd4e5 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
@@ -24,6 +24,7 @@ import org.elasticsearch.action.search.SearchType;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
@@ -175,27 +176,35 @@ public class RestSearchAction extends BaseRestHandler {
             }
         }
 
-        String sField = request.param("fields");
+        if (request.param("fields") != null) {
+            throw new IllegalArgumentException("The parameter [" +
+                SearchSourceBuilder.FIELDS_FIELD + "] is not longer supported, please use [" +
+                SearchSourceBuilder.STORED_FIELDS_FIELD + "] to retrieve stored fields or _source filtering " +
+                "if the field is not stored");
+        }
+
+        String sField = request.param("stored_fields");
         if (sField != null) {
             if (!Strings.hasText(sField)) {
-                searchSourceBuilder.noFields();
+                searchSourceBuilder.noStoredFields();
             } else {
                 String[] sFields = Strings.splitStringByCommaToArray(sField);
                 if (sFields != null) {
                     for (String field : sFields) {
-                        searchSourceBuilder.field(field);
+                        searchSourceBuilder.storedField(field);
                     }
                 }
             }
         }
-        String sFieldDataFields = request.param("fielddata_fields");
-        if (sFieldDataFields != null) {
-            if (Strings.hasText(sFieldDataFields)) {
-                String[] sFields = Strings.splitStringByCommaToArray(sFieldDataFields);
-                if (sFields != null) {
-                    for (String field : sFields) {
-                        searchSourceBuilder.fieldDataField(field);
-                    }
+        String sDocValueFields = request.param("docvalue_fields");
+        if (sDocValueFields == null) {
+            sDocValueFields = request.param("fielddata_fields");
+        }
+        if (sDocValueFields != null) {
+            if (Strings.hasText(sDocValueFields)) {
+                String[] sFields = Strings.splitStringByCommaToArray(sDocValueFields);
+                for (String field : sFields) {
+                    searchSourceBuilder.docValueField(field);
                 }
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java
index ec75ae37254..d8069575c5f 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchService.java
@@ -712,8 +712,8 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
                 throw new SearchContextException(context, "failed to create RescoreSearchContext", e);
             }
         }
-        if (source.fields() != null) {
-            context.fieldNames().addAll(source.fields());
+        if (source.storedFields() != null) {
+            context.fieldNames().addAll(source.storedFields());
         }
         if (source.explain() != null) {
             context.explain(source.explain());
@@ -721,9 +721,9 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
         if (source.fetchSource() != null) {
             context.fetchSourceContext(source.fetchSource());
         }
-        if (source.fieldDataFields() != null) {
+        if (source.docValueFields() != null) {
             FieldDataFieldsContext fieldDataFieldsContext = context.getFetchSubPhaseContext(FieldDataFieldsFetchSubPhase.CONTEXT_FACTORY);
-            for (String field : source.fieldDataFields()) {
+            for (String field : source.docValueFields()) {
                 fieldDataFieldsContext.add(new FieldDataField(field));
             }
             fieldDataFieldsContext.setHitExecutionNeeded(true);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java
index 00d783c12e3..c9ed41c6ddc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregationBuilder.java
@@ -567,9 +567,9 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
         }
         if (fieldNames != null) {
             if (fieldNames.size() == 1) {
-                builder.field(SearchSourceBuilder.FIELDS_FIELD.getPreferredName(), fieldNames.get(0));
+                builder.field(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), fieldNames.get(0));
             } else {
-                builder.startArray(SearchSourceBuilder.FIELDS_FIELD.getPreferredName());
+                builder.startArray(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName());
                 for (String fieldName : fieldNames) {
                     builder.value(fieldName);
                 }
@@ -577,7 +577,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
             }
         }
         if (fieldDataFields != null) {
-            builder.startArray(SearchSourceBuilder.FIELDDATA_FIELDS_FIELD.getPreferredName());
+            builder.startArray(SearchSourceBuilder.DOCVALUE_FIELDS_FIELD.getPreferredName());
             for (String fieldDataField : fieldDataFields) {
                 builder.value(fieldDataField);
             }
@@ -628,7 +628,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
                     factory.trackScores(parser.booleanValue());
                 } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder._SOURCE_FIELD)) {
                     factory.fetchSource(FetchSourceContext.parse(context));
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.FIELDS_FIELD)) {
+                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.STORED_FIELDS_FIELD)) {
                     List<String> fieldNames = new ArrayList<>();
                     fieldNames.add(parser.text());
                     factory.fields(fieldNames);
@@ -694,7 +694,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
 
-                if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.FIELDS_FIELD)) {
+                if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.STORED_FIELDS_FIELD)) {
                     List<String> fieldNames = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token == XContentParser.Token.VALUE_STRING) {
@@ -705,7 +705,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
                         }
                     }
                     factory.fields(fieldNames);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.FIELDDATA_FIELDS_FIELD)) {
+                } else if (context.getParseFieldMatcher().match(currentFieldName, SearchSourceBuilder.DOCVALUE_FIELDS_FIELD)) {
                     List<String> fieldDataFields = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token == XContentParser.Token.VALUE_STRING) {
diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index ebf8880d4aa..d56119edce4 100644
--- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -83,7 +83,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
     public static final ParseField EXPLAIN_FIELD = new ParseField("explain");
     public static final ParseField _SOURCE_FIELD = new ParseField("_source");
     public static final ParseField FIELDS_FIELD = new ParseField("fields");
-    public static final ParseField FIELDDATA_FIELDS_FIELD = new ParseField("fielddata_fields");
+    public static final ParseField STORED_FIELDS_FIELD = new ParseField("stored_fields");
+    public static final ParseField DOCVALUE_FIELDS_FIELD = new ParseField("docvalue_fields", "fielddata_fields");
     public static final ParseField SCRIPT_FIELDS_FIELD = new ParseField("script_fields");
     public static final ParseField SCRIPT_FIELD = new ParseField("script");
     public static final ParseField IGNORE_FAILURE_FIELD = new ParseField("ignore_failure");
@@ -146,8 +147,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
     private TimeValue timeout = null;
     private int terminateAfter = SearchContext.DEFAULT_TERMINATE_AFTER;
 
-    private List<String> fieldNames;
-    private List<String> fieldDataFields;
+    private List<String> storedFieldNames;
+    private List<String> docValueFields;
     private List<ScriptField> scriptFields;
     private FetchSourceContext fetchSourceContext;
 
@@ -181,22 +182,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         aggregations = in.readOptionalWriteable(AggregatorFactories.Builder::new);
         explain = in.readOptionalBoolean();
         fetchSourceContext = in.readOptionalStreamable(FetchSourceContext::new);
-        boolean hasFieldDataFields = in.readBoolean();
-        if (hasFieldDataFields) {
-            int size = in.readVInt();
-            fieldDataFields = new ArrayList<>(size);
-            for (int i = 0; i < size; i++) {
-                fieldDataFields.add(in.readString());
-            }
-        }
-        boolean hasFieldNames = in.readBoolean();
-        if (hasFieldNames) {
-            int size = in.readVInt();
-            fieldNames = new ArrayList<>(size);
-            for (int i = 0; i < size; i++) {
-                fieldNames.add(in.readString());
-            }
-        }
+        docValueFields = (List<String>) in.readGenericValue();
+        storedFieldNames = (List<String>) in.readGenericValue();
         from = in.readVInt();
         highlightBuilder = in.readOptionalWriteable(HighlightBuilder::new);
         boolean hasIndexBoost = in.readBoolean();
@@ -255,22 +242,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         out.writeOptionalWriteable(aggregations);
         out.writeOptionalBoolean(explain);
         out.writeOptionalStreamable(fetchSourceContext);
-        boolean hasFieldDataFields = fieldDataFields != null;
-        out.writeBoolean(hasFieldDataFields);
-        if (hasFieldDataFields) {
-            out.writeVInt(fieldDataFields.size());
-            for (String field : fieldDataFields) {
-                out.writeString(field);
-            }
-        }
-        boolean hasFieldNames = fieldNames != null;
-        out.writeBoolean(hasFieldNames);
-        if (hasFieldNames) {
-            out.writeVInt(fieldNames.size());
-            for (String field : fieldNames) {
-                out.writeString(field);
-            }
-        }
+        out.writeGenericValue(docValueFields);
+        out.writeGenericValue(storedFieldNames);
         out.writeVInt(from);
         out.writeOptionalWriteable(highlightBuilder);
         boolean hasIndexBoost = indexBoost != null;
@@ -732,62 +705,89 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
     }
 
     /**
-     * Adds a field to load and return (note, it must be stored) as part of the
+     * Adds a stored field to load and return as part of the
      * search request. If none are specified, the source of the document will be
      * return.
      */
-    public SearchSourceBuilder field(String name) {
-        if (fieldNames == null) {
-            fieldNames = new ArrayList<>();
+    public SearchSourceBuilder storedField(String name) {
+        if (storedFieldNames == null) {
+            storedFieldNames = new ArrayList<>();
         }
-        fieldNames.add(name);
+        storedFieldNames.add(name);
         return this;
     }
 
     /**
-     * Sets the fields to load and return as part of the search request. If none
+     * Sets the stored fields to load and return as part of the search request. If none
      * are specified, the source of the document will be returned.
      */
-    public SearchSourceBuilder fields(List<String> fields) {
-        this.fieldNames = fields;
+    public SearchSourceBuilder storedFields(List<String> fields) {
+        this.storedFieldNames = fields;
         return this;
     }
 
     /**
-     * Sets no fields to be loaded, resulting in only id and type to be returned
+     * Sets no stored fields to be loaded, resulting in only id and type to be returned
      * per field.
      */
-    public SearchSourceBuilder noFields() {
-        this.fieldNames = Collections.emptyList();
+    public SearchSourceBuilder noStoredFields() {
+        this.storedFieldNames = Collections.emptyList();
         return this;
     }
 
     /**
-     * Gets the fields to load and return as part of the search request.
+     * Gets the stored fields to load and return as part of the search request.
      */
-    public List<String> fields() {
-        return fieldNames;
+    public List<String> storedFields() {
+        return storedFieldNames;
+    }
+
+
+    /**
+     * Adds a field to load from the docvalue and return as part of the
+     * search request.
+     *
+     * @deprecated Use {@link SearchSourceBuilder#docValueField(String)} instead.
+     */
+    @Deprecated
+    public SearchSourceBuilder fieldDataField(String name) {
+        if (docValueFields == null) {
+            docValueFields = new ArrayList<>();
+        }
+        docValueFields.add(name);
+        return this;
     }
 
     /**
-     * Adds a field to load from the field data cache and return as part of the
+     * Gets the docvalue fields.
+     *
+     * @deprecated Use {@link SearchSourceBuilder#docValueFields()} instead.
+     */
+    @Deprecated
+    public List<String> fieldDataFields() {
+        return docValueFields;
+    }
+
+
+    /**
+     * Gets the docvalue fields.
+     */
+    public List<String> docValueFields() {
+        return docValueFields;
+    }
+
+    /**
+     * Adds a field to load from the docvalue and return as part of the
      * search request.
      */
-    public SearchSourceBuilder fieldDataField(String name) {
-        if (fieldDataFields == null) {
-            fieldDataFields = new ArrayList<>();
+    public SearchSourceBuilder docValueField(String name) {
+        if (docValueFields == null) {
+            docValueFields = new ArrayList<>();
         }
-        fieldDataFields.add(name);
+        docValueFields.add(name);
         return this;
     }
 
-    /**
-     * Gets the field-data fields.
-     */
-    public List<String> fieldDataFields() {
-        return fieldDataFields;
-    }
-
     /**
      * Adds a script field under the given name with the provided script.
      *
@@ -910,8 +910,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
             rewrittenBuilder.explain = explain;
             rewrittenBuilder.ext = ext;
             rewrittenBuilder.fetchSourceContext = fetchSourceContext;
-            rewrittenBuilder.fieldDataFields = fieldDataFields;
-            rewrittenBuilder.fieldNames = fieldNames;
+            rewrittenBuilder.docValueFields = docValueFields;
+            rewrittenBuilder.storedFieldNames = storedFieldNames;
             rewrittenBuilder.from = from;
             rewrittenBuilder.highlightBuilder = highlightBuilder;
             rewrittenBuilder.indexBoost = indexBoost;
@@ -971,12 +971,16 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                     trackScores = parser.booleanValue();
                 } else if (context.getParseFieldMatcher().match(currentFieldName, _SOURCE_FIELD)) {
                     fetchSourceContext = FetchSourceContext.parse(context);
-                } else if (context.getParseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) {
-                    field(parser.text());
+                } else if (context.getParseFieldMatcher().match(currentFieldName, STORED_FIELDS_FIELD)) {
+                    storedField(parser.text());
                 } else if (context.getParseFieldMatcher().match(currentFieldName, SORT_FIELD)) {
                     sort(parser.text());
                 } else if (context.getParseFieldMatcher().match(currentFieldName, PROFILE_FIELD)) {
                     profile = parser.booleanValue();
+                } else if (context.getParseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) {
+                    throw new ParsingException(parser.getTokenLocation(), "Deprecated field [" +
+                        SearchSourceBuilder.FIELDS_FIELD + "] used, expected [" +
+                        SearchSourceBuilder.STORED_FIELDS_FIELD + "] instead");
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].",
                             parser.getTokenLocation());
@@ -1026,22 +1030,21 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                             parser.getTokenLocation());
                 }
             } else if (token == XContentParser.Token.START_ARRAY) {
-
-                if (context.getParseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) {
-                    fieldNames = new ArrayList<>();
+                if (context.getParseFieldMatcher().match(currentFieldName, STORED_FIELDS_FIELD)) {
+                    storedFieldNames = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token == XContentParser.Token.VALUE_STRING) {
-                            fieldNames.add(parser.text());
+                            storedFieldNames.add(parser.text());
                         } else {
                             throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.VALUE_STRING + "] in ["
                                     + currentFieldName + "] but found [" + token + "]", parser.getTokenLocation());
                         }
                     }
-                } else if (context.getParseFieldMatcher().match(currentFieldName, FIELDDATA_FIELDS_FIELD)) {
-                    fieldDataFields = new ArrayList<>();
+                } else if (context.getParseFieldMatcher().match(currentFieldName, DOCVALUE_FIELDS_FIELD)) {
+                    docValueFields = new ArrayList<>();
                     while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                         if (token == XContentParser.Token.VALUE_STRING) {
-                            fieldDataFields.add(parser.text());
+                            docValueFields.add(parser.text());
                         } else {
                             throw new ParsingException(parser.getTokenLocation(), "Expected [" + XContentParser.Token.VALUE_STRING + "] in ["
                                     + currentFieldName + "] but found [" + token + "]", parser.getTokenLocation());
@@ -1068,6 +1071,11 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                     fetchSourceContext = FetchSourceContext.parse(context);
                 } else if (context.getParseFieldMatcher().match(currentFieldName, SEARCH_AFTER)) {
                     searchAfterBuilder = SearchAfterBuilder.fromXContent(parser, context.getParseFieldMatcher());
+                } else if (context.getParseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) {
+                    throw new ParsingException(parser.getTokenLocation(), "The field [" +
+                        SearchSourceBuilder.FIELDS_FIELD + "] is not longer supported, please use [" +
+                        SearchSourceBuilder.STORED_FIELDS_FIELD + "] to retrieve stored fields or _source filtering " +
+                        "if the field is not stored");
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].",
                             parser.getTokenLocation());
@@ -1131,21 +1139,21 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
             builder.field(_SOURCE_FIELD.getPreferredName(), fetchSourceContext);
         }
 
-        if (fieldNames != null) {
-            if (fieldNames.size() == 1) {
-                builder.field(FIELDS_FIELD.getPreferredName(), fieldNames.get(0));
+        if (storedFieldNames != null) {
+            if (storedFieldNames.size() == 1) {
+                builder.field(STORED_FIELDS_FIELD.getPreferredName(), storedFieldNames.get(0));
             } else {
-                builder.startArray(FIELDS_FIELD.getPreferredName());
-                for (String fieldName : fieldNames) {
+                builder.startArray(STORED_FIELDS_FIELD.getPreferredName());
+                for (String fieldName : storedFieldNames) {
                     builder.value(fieldName);
                 }
                 builder.endArray();
             }
         }
 
-        if (fieldDataFields != null) {
-            builder.startArray(FIELDDATA_FIELDS_FIELD.getPreferredName());
-            for (String fieldDataField : fieldDataFields) {
+        if (docValueFields != null) {
+            builder.startArray(DOCVALUE_FIELDS_FIELD.getPreferredName());
+            for (String fieldDataField : docValueFields) {
                 builder.value(fieldDataField);
             }
             builder.endArray();
@@ -1339,7 +1347,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
 
     @Override
     public int hashCode() {
-        return Objects.hash(aggregations, explain, fetchSourceContext, fieldDataFields, fieldNames, from,
+        return Objects.hash(aggregations, explain, fetchSourceContext, docValueFields, storedFieldNames, from,
                 highlightBuilder, indexBoost, minScore, postQueryBuilder, queryBuilder, rescoreBuilders, scriptFields,
                 size, sorts, searchAfterBuilder, sliceBuilder, stats, suggestBuilder, terminateAfter, timeout, trackScores, version, profile);
     }
@@ -1356,8 +1364,8 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
         return Objects.equals(aggregations, other.aggregations)
                 && Objects.equals(explain, other.explain)
                 && Objects.equals(fetchSourceContext, other.fetchSourceContext)
-                && Objects.equals(fieldDataFields, other.fieldDataFields)
-                && Objects.equals(fieldNames, other.fieldNames)
+                && Objects.equals(docValueFields, other.docValueFields)
+                && Objects.equals(storedFieldNames, other.storedFieldNames)
                 && Objects.equals(from, other.from)
                 && Objects.equals(highlightBuilder, other.highlightBuilder)
                 && Objects.equals(indexBoost, other.indexBoost)
diff --git a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
index f0e12abeac8..596714b1c5b 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
@@ -677,7 +677,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
         client().prepareIndex(IDX, "doc", "4").setSource("foo", "eggplant").get();
         flushAndRefresh(IDX);
 
-        SearchResponse resp = client().prepareSearch(IDX).setQuery(matchAllQuery()).addFieldDataField("foo").addSort("foo", SortOrder.ASC).get();
+        SearchResponse resp = client().prepareSearch(IDX).setQuery(matchAllQuery()).addDocValueField("foo").addSort("foo", SortOrder.ASC).get();
         assertHitCount(resp, 4);
         assertOrderedSearchHits(resp, "2", "3", "4", "1");
         SearchHit[] hits = resp.getHits().hits();
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java
index 31e4e2d0923..da0c3d081af 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/core/TokenCountFieldMapperIntegrationIT.java
@@ -166,9 +166,9 @@ public class TokenCountFieldMapperIntegrationIT extends ESIntegTestCase {
 
     private SearchRequestBuilder prepareSearch() {
         SearchRequestBuilder request = client().prepareSearch("test").setTypes("test");
-        request.addField("foo.token_count");
+        request.addStoredField("foo.token_count");
         if (loadCountedFields) {
-            request.addField("foo");
+            request.addStoredField("foo");
         }
         return request;
     }
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java
index 202afd7a4b1..14dd370fbfd 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperTests.java
@@ -816,7 +816,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
                 .field("lon", -74.0059731).endObject().endObject()).setRefreshPolicy(IMMEDIATE).get();
 
         // match all search with geohash field
-        SearchResponse searchResponse = client().prepareSearch().addField("location.geohash").setQuery(matchAllQuery()).execute().actionGet();
+        SearchResponse searchResponse = client().prepareSearch().addStoredField("location.geohash").setQuery(matchAllQuery()).execute().actionGet();
         Map<String, SearchHitField> m = searchResponse.getHits().getAt(0).getFields();
 
         // ensure single geohash was indexed
@@ -841,7 +841,7 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
                 .field("lon", -74.0059731).endObject().endObject()).setRefreshPolicy(IMMEDIATE).get();
 
         // match all search with geohash field (includes prefixes)
-        SearchResponse searchResponse = client().prepareSearch().addField("location.geohash").setQuery(matchAllQuery()).execute().actionGet();
+        SearchResponse searchResponse = client().prepareSearch().addStoredField("location.geohash").setQuery(matchAllQuery()).execute().actionGet();
         Map<String, SearchHitField> m = searchResponse.getHits().getAt(0).getFields();
 
         List<Object> hashes = m.get("location.geohash").values();
@@ -872,11 +872,11 @@ public class GeoPointFieldMapperTests extends ESSingleNodeTestCase {
         }
 
         // query by geohash subfield
-        SearchResponse searchResponse = client().prepareSearch().addField("location.geohash").setQuery(matchAllQuery()).execute().actionGet();
+        SearchResponse searchResponse = client().prepareSearch().addStoredField("location.geohash").setQuery(matchAllQuery()).execute().actionGet();
         assertEquals(numDocs, searchResponse.getHits().totalHits());
 
         // query by latlon subfield
-        searchResponse = client().prepareSearch().addField("location.latlon").setQuery(matchAllQuery()).execute().actionGet();
+        searchResponse = client().prepareSearch().addStoredField("location.latlon").setQuery(matchAllQuery()).execute().actionGet();
         assertEquals(numDocs, searchResponse.getHits().totalHits());
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
index 090aa906456..da8610f29f6 100644
--- a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
@@ -218,8 +218,8 @@ public class InnerHitBuilderTests extends ESTestCase {
         innerHits.setExplain(randomBoolean());
         innerHits.setVersion(randomBoolean());
         innerHits.setTrackScores(randomBoolean());
-        innerHits.setFieldNames(randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16)));
-        innerHits.setFieldDataFields(randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16)));
+        innerHits.setStoredFieldNames(randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16)));
+        innerHits.setDocValueFields(randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16)));
         // Random script fields deduped on their field name.
         Map<String, SearchSourceBuilder.ScriptField> scriptFields = new HashMap<>();
         for (SearchSourceBuilder.ScriptField field: randomListStuff(16, InnerHitBuilderTests::randomScript)) {
@@ -294,11 +294,11 @@ public class InnerHitBuilderTests extends ESTestCase {
                 break;
             case 6:
                 if (randomBoolean()) {
-                    instance.setFieldDataFields(randomValueOtherThan(instance.getFieldDataFields(), () -> {
+                    instance.setDocValueFields(randomValueOtherThan(instance.getDocValueFields(), () -> {
                         return randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16));
                     }));
                 } else {
-                    instance.addFieldDataField(randomAsciiOfLengthBetween(1, 16));
+                    instance.addDocValueField(randomAsciiOfLengthBetween(1, 16));
                 }
                 break;
             case 7:
@@ -341,12 +341,12 @@ public class InnerHitBuilderTests extends ESTestCase {
                         HighlightBuilderTests::randomHighlighterBuilder));
                 break;
             case 11:
-                if (instance.getFieldNames() == null || randomBoolean()) {
-                    instance.setFieldNames(randomValueOtherThan(instance.getFieldNames(), () -> {
+                if (instance.getStoredFieldNames() == null || randomBoolean()) {
+                    instance.setStoredFieldNames(randomValueOtherThan(instance.getStoredFieldNames(), () -> {
                         return randomListStuff(16, () -> randomAsciiOfLengthBetween(1, 16));
                     }));
                 } else {
-                    instance.getFieldNames().add(randomAsciiOfLengthBetween(1, 16));
+                    instance.getStoredFieldNames().add(randomAsciiOfLengthBetween(1, 16));
                 }
                 break;
             default:
diff --git a/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java b/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java
index 163d72f4553..590c5c624f1 100644
--- a/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java
+++ b/core/src/test/java/org/elasticsearch/index/store/ExceptionRetryIT.java
@@ -113,7 +113,7 @@ public class ExceptionRetryIT extends ESIntegTestCase {
         }
 
         refresh();
-        SearchResponse searchResponse = client().prepareSearch("index").setSize(numDocs * 2).addField("_id").get();
+        SearchResponse searchResponse = client().prepareSearch("index").setSize(numDocs * 2).addStoredField("_id").get();
 
         Set<String> uniqueIds = new HashSet();
         long dupCounter = 0;
diff --git a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
index 493f8b74e04..acdfdd12266 100644
--- a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
@@ -116,7 +116,7 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
         ensureGreen();
         SearchResponse searchResponse = client().prepareSearch("test_index")
                 .setQuery(termQuery("field1", "value1"))
-                .addField("field1").addField("field2")
+                .addStoredField("field1").addStoredField("field2")
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 1);
@@ -130,7 +130,7 @@ public class SimpleIndexTemplateIT extends ESIntegTestCase {
         // now only match on one template (template_1)
         searchResponse = client().prepareSearch("text_index")
                 .setQuery(termQuery("field1", "value1"))
-                .addField("field1").addField("field2")
+                .addStoredField("field1").addStoredField("field2")
                 .execute().actionGet();
         if (searchResponse.getFailedShards() > 0) {
             logger.warn("failed search {}", Arrays.toString(searchResponse.getShardFailures()));
diff --git a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
index 0c224238a90..2d778de37c9 100644
--- a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
+++ b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java
@@ -218,7 +218,7 @@ public class RelocationIT extends ESIntegTestCase {
             for (int i = 0; i < 10; i++) {
                 try {
                     logger.info("--> START search test round {}", i + 1);
-                    SearchHits hits = client().prepareSearch("test").setQuery(matchAllQuery()).setSize((int) indexer.totalIndexedDocs()).setNoFields().execute().actionGet().getHits();
+                    SearchHits hits = client().prepareSearch("test").setQuery(matchAllQuery()).setSize((int) indexer.totalIndexedDocs()).setNoStoredFields().execute().actionGet().getHits();
                     ranOnce = true;
                     if (hits.totalHits() != indexer.totalIndexedDocs()) {
                         int[] hitIds = new int[(int) indexer.totalIndexedDocs()];
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java
index 49ef9e1b6b5..5cc6ec58630 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java
@@ -181,7 +181,7 @@ public abstract class AbstractGeoTestCase extends ESIntegTestCase {
         // Added to debug a test failure where the terms aggregation seems to be reporting two documents with the same value for NUMBER_FIELD_NAME.  This will check that after
         // random indexing each document only has 1 value for NUMBER_FIELD_NAME and it is the correct value. Following this initial change its seems that this call was getting
         // more that 2000 hits (actual value was 2059) so now it will also check to ensure all hits have the correct index and type
-        SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME).addField(NUMBER_FIELD_NAME).addSort(SortBuilders.fieldSort(NUMBER_FIELD_NAME)
+        SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME).addStoredField(NUMBER_FIELD_NAME).addSort(SortBuilders.fieldSort(NUMBER_FIELD_NAME)
                 .order(SortOrder.ASC)).setSize(5000).get();
         assertSearchResponse(response);
         long totalHits = response.getHits().totalHits();
diff --git a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
index 8ddfa374ac0..5a8b9fc767f 100644
--- a/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java
@@ -221,12 +221,12 @@ public class SearchSourceBuilderTests extends ESTestCase {
             for (int i = 0; i < fieldsSize; i++) {
                 fields.add(randomAsciiOfLengthBetween(5, 50));
             }
-            builder.fields(fields);
+            builder.storedFields(fields);
         }
         if (randomBoolean()) {
             int fieldDataFieldsSize = randomInt(25);
             for (int i = 0; i < fieldDataFieldsSize; i++) {
-                builder.fieldDataField(randomAsciiOfLengthBetween(5, 50));
+                builder.docValueField(randomAsciiOfLengthBetween(5, 50));
             }
         }
         if (randomBoolean()) {
diff --git a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
index f8ca1e1aaf7..68679e89ae6 100644
--- a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
+++ b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java
@@ -202,7 +202,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         refresh();
 
         // TEST FETCHING _parent from child
-        SearchResponse searchResponse = client().prepareSearch("test").setQuery(idsQuery("child").addIds("c1")).fields("_parent").execute()
+        SearchResponse searchResponse = client().prepareSearch("test").setQuery(idsQuery("child").addIds("c1")).storedFields("_parent").execute()
                 .actionGet();
         assertNoFailures(searchResponse);
         assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
@@ -210,7 +210,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         assertThat(searchResponse.getHits().getAt(0).field("_parent").value().toString(), equalTo("p1"));
 
         // TEST matching on parent
-        searchResponse = client().prepareSearch("test").setQuery(termQuery("_parent#parent", "p1")).fields("_parent").get();
+        searchResponse = client().prepareSearch("test").setQuery(termQuery("_parent#parent", "p1")).storedFields("_parent").get();
         assertNoFailures(searchResponse);
         assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
         assertThat(searchResponse.getHits().getAt(0).id(), anyOf(equalTo("c1"), equalTo("c2")));
@@ -218,7 +218,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
         assertThat(searchResponse.getHits().getAt(1).id(), anyOf(equalTo("c1"), equalTo("c2")));
         assertThat(searchResponse.getHits().getAt(1).field("_parent").value().toString(), equalTo("p1"));
 
-        searchResponse = client().prepareSearch("test").setQuery(queryStringQuery("_parent#parent:p1")).fields("_parent").get();
+        searchResponse = client().prepareSearch("test").setQuery(queryStringQuery("_parent#parent:p1")).storedFields("_parent").get();
         assertNoFailures(searchResponse);
         assertThat(searchResponse.getHits().totalHits(), equalTo(2L));
         assertThat(searchResponse.getHits().getAt(0).id(), anyOf(equalTo("c1"), equalTo("c2")));
@@ -1394,7 +1394,7 @@ public class ChildQuerySearchIT extends ESIntegTestCase {
             SearchResponse scrollResponse = client().prepareSearch("test")
                     .setScroll(TimeValue.timeValueSeconds(30))
                     .setSize(1)
-                    .addField("_id")
+                    .addStoredField("_id")
                     .setQuery(query)
                     .execute()
                     .actionGet();
diff --git a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java
index 0debdb263af..985605c4e65 100644
--- a/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java
+++ b/core/src/test/java/org/elasticsearch/search/geo/GeoFilterIT.java
@@ -415,13 +415,13 @@ public class GeoFilterIT extends ESIntegTestCase {
             assertThat(hit.getId(), equalTo(key));
         }
 
-        SearchResponse world = client().prepareSearch().addField("pin").setQuery(
+        SearchResponse world = client().prepareSearch().addStoredField("pin").setQuery(
                 geoBoundingBoxQuery("pin").setCorners(90, -179.99999, -90, 179.99999)
         ).execute().actionGet();
 
         assertHitCount(world, 53);
 
-        SearchResponse distance = client().prepareSearch().addField("pin").setQuery(
+        SearchResponse distance = client().prepareSearch().addStoredField("pin").setQuery(
                 geoDistanceQuery("pin").distance("425km").point(51.11, 9.851)
                 ).execute().actionGet();
 
diff --git a/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java b/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
index 01f98564814..56b33c6007e 100644
--- a/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
@@ -156,7 +156,7 @@ public class InnerHitsIT extends ESIntegTestCase {
                 .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg).innerHit(
                         new InnerHitBuilder().setHighlightBuilder(new HighlightBuilder().field("comments.message"))
                                 .setExplain(true)
-                                .addFieldDataField("comments.message")
+                                .addDocValueField("comments.message")
                                 .addScriptField("script", new Script("5", ScriptService.ScriptType.INLINE, MockScriptEngine.NAME, Collections.emptyMap()))
                                 .setSize(1)
                 )).get();
@@ -287,7 +287,7 @@ public class InnerHitsIT extends ESIntegTestCase {
                 .setQuery(
                         hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit(
                                 new InnerHitBuilder()
-                                        .addFieldDataField("message")
+                                        .addDocValueField("message")
                                         .setHighlightBuilder(new HighlightBuilder().field("message"))
                                         .setExplain(true).setSize(1)
                                         .addScriptField("script", new Script("5", ScriptService.ScriptType.INLINE,
diff --git a/core/src/test/java/org/elasticsearch/search/source/SourceFetchingIT.java b/core/src/test/java/org/elasticsearch/search/source/SourceFetchingIT.java
index 33fcb55cada..df147ce106f 100644
--- a/core/src/test/java/org/elasticsearch/search/source/SourceFetchingIT.java
+++ b/core/src/test/java/org/elasticsearch/search/source/SourceFetchingIT.java
@@ -37,10 +37,10 @@ public class SourceFetchingIT extends ESIntegTestCase {
         SearchResponse response = client().prepareSearch("test").get();
         assertThat(response.getHits().getAt(0).getSourceAsString(), notNullValue());
 
-        response = client().prepareSearch("test").addField("bla").get();
+        response = client().prepareSearch("test").addStoredField("bla").get();
         assertThat(response.getHits().getAt(0).getSourceAsString(), nullValue());
 
-        response = client().prepareSearch("test").addField("_source").get();
+        response = client().prepareSearch("test").addStoredField("_source").get();
         assertThat(response.getHits().getAt(0).getSourceAsString(), notNullValue());
 
     }
diff --git a/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java b/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java
index 3573089fcaa..e81b4decb2d 100644
--- a/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java
+++ b/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java
@@ -87,7 +87,7 @@ public class TimestampTTLBWIT extends ESIntegTestCase {
                 .setQuery(matchAllQuery())
                 .setSize(randomIntBetween(1, numDocs + 5))
                 .addSort("_timestamp", order)
-                .addField("_timestamp")
+                .addStoredField("_timestamp")
                 .execute().actionGet();
         assertNoFailures(searchResponse);
         SearchHit[] hits = searchResponse.getHits().hits();
diff --git a/docs/plugins/mapper-attachments.asciidoc b/docs/plugins/mapper-attachments.asciidoc
index b287decd4cf..f2c034a317e 100644
--- a/docs/plugins/mapper-attachments.asciidoc
+++ b/docs/plugins/mapper-attachments.asciidoc
@@ -246,7 +246,7 @@ PUT /test/person/1?refresh=true
 }
 GET /test/person/_search
 {
-  "fields": [ "file.content_type" ],
+  "stored_fields": [ "file.content_type" ],
   "query": {
     "match": {
       "file.content_type": "text plain"
@@ -367,7 +367,7 @@ PUT /test/person/1?refresh=true
 }
 GET /test/person/_search
 {
-  "fields": [],
+  "stored_fields": [],
   "query": {
     "match": {
       "file.content": "king queen"
diff --git a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
index 83855a8aae5..1b955d2a898 100644
--- a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
@@ -22,7 +22,7 @@ The top_hits aggregation returns regular search hits, because of this many per h
 * <<search-request-named-queries-and-filters,Named filters and queries>>
 * <<search-request-source-filtering,Source filtering>>
 * <<search-request-script-fields,Script fields>>
-* <<search-request-fielddata-fields,Fielddata fields>>
+* <<search-request-docvalue-fields,Doc value fields>>
 * <<search-request-version,Include versions>>
 
 ==== Example
diff --git a/docs/reference/mapping/params/store.asciidoc b/docs/reference/mapping/params/store.asciidoc
index d34d1016546..53cac7493ff 100644
--- a/docs/reference/mapping/params/store.asciidoc
+++ b/docs/reference/mapping/params/store.asciidoc
@@ -48,7 +48,7 @@ PUT my_index/my_type/1
 
 GET my_index/_search
 {
-  "fields": [ "title", "date" ] <2>
+  "stored_fields": [ "title", "date" ] <2>
 }
 --------------------------------------------------
 // CONSOLE
diff --git a/docs/reference/migration/migrate_5_0/search.asciidoc b/docs/reference/migration/migrate_5_0/search.asciidoc
index 72b29e8a9e7..09478ee3251 100644
--- a/docs/reference/migration/migrate_5_0/search.asciidoc
+++ b/docs/reference/migration/migrate_5_0/search.asciidoc
@@ -64,11 +64,15 @@ characteristics as the former `scan` search type.
 
 ==== `fields` parameter
 
-The `fields` parameter used to try to retrieve field values from stored
-fields, and fall back to extracting from the `_source` if a field is not
-marked as stored.  Now, the `fields` parameter will only return stored fields
+The `fields` parameter has been replaced by `stored_fields`.
+The `stored_fields` parameter will only return stored fields
 -- it will no longer extract values from the `_source`.
 
+==== `fielddata_fields` parameter
+
+The `fielddata_fields` has been deprecated, use parameter `docvalue_fields` instead.
+
+
 ==== search-exists API removed
 
 The search exists api has been removed in favour of using the search api with
diff --git a/docs/reference/modules/scripting/painless.asciidoc b/docs/reference/modules/scripting/painless.asciidoc
index 93fb136913b..ad36cdd6df4 100644
--- a/docs/reference/modules/scripting/painless.asciidoc
+++ b/docs/reference/modules/scripting/painless.asciidoc
@@ -143,7 +143,7 @@ First, let's look at the source data for a player by submitting the following re
 ----------------------------------------------------------------
 GET hockey/_search
 {
-  "fields": [
+  "stored_fields": [
     "_id",
     "_source"
   ],
diff --git a/docs/reference/search/request-body.asciidoc b/docs/reference/search/request-body.asciidoc
index 958320ea110..a9adc157bd3 100644
--- a/docs/reference/search/request-body.asciidoc
+++ b/docs/reference/search/request-body.asciidoc
@@ -143,11 +143,11 @@ include::request/sort.asciidoc[]
 
 include::request/source-filtering.asciidoc[]
 
-include::request/fields.asciidoc[]
+include::request/stored-fields.asciidoc[]
 
 include::request/script-fields.asciidoc[]
 
-include::request/fielddata-fields.asciidoc[]
+include::request/docvalue-fields.asciidoc[]
 
 include::request/post-filter.asciidoc[]
 
diff --git a/docs/reference/search/request/docvalue-fields.asciidoc b/docs/reference/search/request/docvalue-fields.asciidoc
new file mode 100644
index 00000000000..bd090a3bd8d
--- /dev/null
+++ b/docs/reference/search/request/docvalue-fields.asciidoc
@@ -0,0 +1,23 @@
+[[search-request-docvalue-fields]]
+=== Doc value Fields
+
+Allows to return the <<docvalue,doc value>> representation of a field for each hit, for
+example:
+
+[source,js]
+--------------------------------------------------
+GET /_search
+{
+    "query" : {
+        "match_all": {}
+    },
+    "docvalue_fields" : ["test1", "test2"]
+}
+--------------------------------------------------
+// CONSOLE
+
+Doc value fields can work on fields that are not stored.
+
+Note that if the fields parameter specifies fields without docvalues it will try to load the value from the fielddata cache
+causing the terms for that field to be loaded to memory (cached), which will result in more memory consumption.
+
diff --git a/docs/reference/search/request/fielddata-fields.asciidoc b/docs/reference/search/request/fielddata-fields.asciidoc
deleted file mode 100644
index f3a3508b144..00000000000
--- a/docs/reference/search/request/fielddata-fields.asciidoc
+++ /dev/null
@@ -1,23 +0,0 @@
-[[search-request-fielddata-fields]]
-=== Field Data Fields
-
-Allows to return the <<fielddata,field data>> representation of a field for each hit, for
-example:
-
-[source,js]
---------------------------------------------------
-GET /_search
-{
-    "query" : {
-        "match_all": {}
-    },
-    "fielddata_fields" : ["test1", "test2"]
-}
---------------------------------------------------
-// CONSOLE
-
-Field data fields can work on fields that are not stored.
-
-It's important to understand that using the `fielddata_fields` parameter will
-cause the terms for that field to be loaded to memory (cached), which will
-result in more memory consumption.
diff --git a/docs/reference/search/request/highlighting.asciidoc b/docs/reference/search/request/highlighting.asciidoc
index c235c37b338..efb7053c179 100644
--- a/docs/reference/search/request/highlighting.asciidoc
+++ b/docs/reference/search/request/highlighting.asciidoc
@@ -372,7 +372,7 @@ query and the rescore query in `highlight_query`.
 --------------------------------------------------
 GET /_search
 {
-    "fields": [ "_id" ],
+    "stored_fields": [ "_id" ],
     "query" : {
         "match": {
             "content": {
diff --git a/docs/reference/search/request/inner-hits.asciidoc b/docs/reference/search/request/inner-hits.asciidoc
index 345bc9abde2..3c8e0e9f00e 100644
--- a/docs/reference/search/request/inner-hits.asciidoc
+++ b/docs/reference/search/request/inner-hits.asciidoc
@@ -72,7 +72,7 @@ Inner hits also supports the following per document features:
 * <<search-request-explain,Explain>>
 * <<search-request-source-filtering,Source filtering>>
 * <<search-request-script-fields,Script fields>>
-* <<search-request-fielddata-fields,Fielddata fields>>
+* <<search-request-docvalue-fields,Doc value fields>>
 * <<search-request-version,Include versions>>
 
 [[nested-inner-hits]]
diff --git a/docs/reference/search/request/fields.asciidoc b/docs/reference/search/request/stored-fields.asciidoc
similarity index 91%
rename from docs/reference/search/request/fields.asciidoc
rename to docs/reference/search/request/stored-fields.asciidoc
index 3483d470ee2..3d5b8c01b47 100644
--- a/docs/reference/search/request/fields.asciidoc
+++ b/docs/reference/search/request/stored-fields.asciidoc
@@ -1,7 +1,7 @@
 [[search-request-fields]]
 === Fields
 
-WARNING: The `fields` parameter is about fields that are explicitly marked as
+WARNING: The `stored_fields` parameter is about fields that are explicitly marked as
 stored in the mapping, which is off by default and generally not recommended.
 Use <<search-request-source-filtering,source filtering>> instead to select
 subsets of the original source document to be returned.
@@ -13,7 +13,7 @@ by a search hit.
 --------------------------------------------------
 GET /_search
 {
-    "fields" : ["user", "postDate"],
+    "stored_fields" : ["user", "postDate"],
     "query" : {
         "term" : { "user" : "kimchy" }
     }
@@ -30,7 +30,7 @@ returned, for example:
 --------------------------------------------------
 GET /_search
 {
-    "fields" : [],
+    "stored_fields" : [],
     "query" : {
         "term" : { "user" : "kimchy" }
     }
diff --git a/docs/reference/search/uri-request.asciidoc b/docs/reference/search/uri-request.asciidoc
index 496f04ea4af..ba36992f6fb 100644
--- a/docs/reference/search/uri-request.asciidoc
+++ b/docs/reference/search/uri-request.asciidoc
@@ -83,7 +83,7 @@ hits was computed.
 part of the document by using `_source_include` & `_source_exclude` (see the <<search-request-source-filtering, request body>>
 documentation for more details)
 
-|`fields` |The selective stored fields of the document to return for each hit,
+|`stored_fields` |The selective stored fields of the document to return for each hit,
 comma delimited. Not specifying any value will cause no fields to return.
 
 |`sort` |Sorting to perform. Can either be in the form of `fieldName`, or
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java
index e9c4bf6e359..0d7dd4e12e2 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java
@@ -72,28 +72,28 @@ public class GeoDistanceTests extends ESIntegTestCase {
 
         refresh();
 
-        SearchResponse searchResponse1 = client().prepareSearch().addField("_source")
+        SearchResponse searchResponse1 = client().prepareSearch().addStoredField("_source")
                 .addScriptField("distance", new Script("doc['location'].arcDistance(" + target_lat + "," + target_long + ")")).execute()
                 .actionGet();
         Double resultDistance1 = searchResponse1.getHits().getHits()[0].getFields().get("distance").getValue();
         assertThat(resultDistance1,
                 closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.DEFAULT), 0.01d));
 
-        SearchResponse searchResponse2 = client().prepareSearch().addField("_source")
+        SearchResponse searchResponse2 = client().prepareSearch().addStoredField("_source")
                 .addScriptField("distance", new Script("doc['location'].distance(" + target_lat + "," + target_long + ")")).execute()
                 .actionGet();
         Double resultDistance2 = searchResponse2.getHits().getHits()[0].getFields().get("distance").getValue();
         assertThat(resultDistance2,
                 closeTo(GeoDistance.PLANE.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.DEFAULT), 0.01d));
 
-        SearchResponse searchResponse3 = client().prepareSearch().addField("_source")
+        SearchResponse searchResponse3 = client().prepareSearch().addStoredField("_source")
                 .addScriptField("distance", new Script("doc['location'].arcDistanceInKm(" + target_lat + "," + target_long + ")"))
                 .execute().actionGet();
         Double resultArcDistance3 = searchResponse3.getHits().getHits()[0].getFields().get("distance").getValue();
         assertThat(resultArcDistance3,
                 closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS), 0.01d));
 
-        SearchResponse searchResponse4 = client().prepareSearch().addField("_source")
+        SearchResponse searchResponse4 = client().prepareSearch().addStoredField("_source")
                 .addScriptField("distance", new Script("doc['location'].distanceInKm(" + target_lat + "," + target_long + ")")).execute()
                 .actionGet();
         Double resultDistance4 = searchResponse4.getHits().getHits()[0].getFields().get("distance").getValue();
@@ -102,7 +102,7 @@ public class GeoDistanceTests extends ESIntegTestCase {
 
         SearchResponse searchResponse5 = client()
                 .prepareSearch()
-                .addField("_source")
+                .addStoredField("_source")
                 .addScriptField("distance", new Script("doc['location'].arcDistanceInKm(" + (target_lat) + "," + (target_long + 360) + ")"))
                 .execute().actionGet();
         Double resultArcDistance5 = searchResponse5.getHits().getHits()[0].getFields().get("distance").getValue();
@@ -111,21 +111,21 @@ public class GeoDistanceTests extends ESIntegTestCase {
 
         SearchResponse searchResponse6 = client()
                 .prepareSearch()
-                .addField("_source")
+                .addStoredField("_source")
                 .addScriptField("distance", new Script("doc['location'].arcDistanceInKm(" + (target_lat + 360) + "," + (target_long) + ")"))
                 .execute().actionGet();
         Double resultArcDistance6 = searchResponse6.getHits().getHits()[0].getFields().get("distance").getValue();
         assertThat(resultArcDistance6,
                 closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS), 0.01d));
 
-        SearchResponse searchResponse7 = client().prepareSearch().addField("_source")
+        SearchResponse searchResponse7 = client().prepareSearch().addStoredField("_source")
                 .addScriptField("distance", new Script("doc['location'].arcDistanceInMiles(" + target_lat + "," + target_long + ")"))
                 .execute().actionGet();
         Double resultDistance7 = searchResponse7.getHits().getHits()[0].getFields().get("distance").getValue();
         assertThat(resultDistance7,
                 closeTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.MILES), 0.01d));
 
-        SearchResponse searchResponse8 = client().prepareSearch().addField("_source")
+        SearchResponse searchResponse8 = client().prepareSearch().addStoredField("_source")
                 .addScriptField("distance", new Script("doc['location'].distanceInMiles(" + target_lat + "," + target_long + ")"))
                 .execute().actionGet();
         Double resultDistance8 = searchResponse8.getHits().getHits()[0].getFields().get("distance").getValue();
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
index d3871e90510..b7be9693210 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java
@@ -102,33 +102,33 @@ public class SearchFieldsTests extends ESIntegTestCase {
 
         client().admin().indices().prepareRefresh().execute().actionGet();
 
-        SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("field1").execute().actionGet();
+        SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field1").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field1").value().toString(), equalTo("value1"));
 
         // field2 is not stored, check that it is not extracted from source.
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("field2").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field2").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(0));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field2"), nullValue());
 
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("field3").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field3").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3"));
 
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("*3").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("*3").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3"));
 
 
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("*3").addField("field1").addField("field2").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("*3").addStoredField("field1").addStoredField("field2").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(2));
@@ -136,20 +136,20 @@ public class SearchFieldsTests extends ESIntegTestCase {
         assertThat(searchResponse.getHits().getAt(0).fields().get("field1").value().toString(), equalTo("value1"));
 
 
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("field*").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field*").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(2));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3"));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field1").value().toString(), equalTo("value1"));
 
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("f*3").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("f*3").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().size(), equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3"));
 
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("*").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("*").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).source(), nullValue());
@@ -157,7 +157,7 @@ public class SearchFieldsTests extends ESIntegTestCase {
         assertThat(searchResponse.getHits().getAt(0).fields().get("field1").value().toString(), equalTo("value1"));
         assertThat(searchResponse.getHits().getAt(0).fields().get("field3").value().toString(), equalTo("value3"));
 
-        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addField("*").addField("_source").execute().actionGet();
+        searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("*").addStoredField("_source").execute().actionGet();
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().hits().length, equalTo(1));
         assertThat(searchResponse.getHits().getAt(0).source(), notNullValue());
@@ -437,15 +437,15 @@ public class SearchFieldsTests extends ESIntegTestCase {
         client().admin().indices().prepareRefresh().execute().actionGet();
 
         SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery())
-                .addField("byte_field")
-                .addField("short_field")
-                .addField("integer_field")
-                .addField("long_field")
-                .addField("float_field")
-                .addField("double_field")
-                .addField("date_field")
-                .addField("boolean_field")
-                .addField("binary_field")
+                .addStoredField("byte_field")
+                .addStoredField("short_field")
+                .addStoredField("integer_field")
+                .addStoredField("long_field")
+                .addStoredField("float_field")
+                .addStoredField("double_field")
+                .addStoredField("date_field")
+                .addStoredField("boolean_field")
+                .addStoredField("binary_field")
                 .execute().actionGet();
 
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
@@ -478,7 +478,7 @@ public class SearchFieldsTests extends ESIntegTestCase {
 
         SearchResponse searchResponse = client().prepareSearch("my-index")
                 .setTypes("my-type1")
-                .addField("field1").addField("_routing")
+                .addStoredField("field1").addStoredField("_routing")
                 .get();
 
         assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
@@ -493,7 +493,7 @@ public class SearchFieldsTests extends ESIntegTestCase {
                 .setRefreshPolicy(IMMEDIATE)
                 .get();
 
-        assertFailures(client().prepareSearch("my-index").setTypes("my-type1").addField("field1"),
+        assertFailures(client().prepareSearch("my-index").setTypes("my-type1").addStoredField("field1"),
                 RestStatus.BAD_REQUEST,
                 containsString("field [field1] isn't a leaf field"));
     }
@@ -557,14 +557,14 @@ public class SearchFieldsTests extends ESIntegTestCase {
 
 
         String field = "field1.field2.field3.field4";
-        SearchResponse searchResponse = client().prepareSearch("my-index").setTypes("my-type1").addField(field).get();
+        SearchResponse searchResponse = client().prepareSearch("my-index").setTypes("my-type1").addStoredField(field).get();
         assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().getAt(0).field(field).isMetadataField(), equalTo(false));
         assertThat(searchResponse.getHits().getAt(0).field(field).getValues().size(), equalTo(2));
         assertThat(searchResponse.getHits().getAt(0).field(field).getValues().get(0).toString(), equalTo("value1"));
         assertThat(searchResponse.getHits().getAt(0).field(field).getValues().get(1).toString(), equalTo("value2"));
 
-        searchResponse = client().prepareSearch("my-index").setTypes("my-type2").addField(field).get();
+        searchResponse = client().prepareSearch("my-index").setTypes("my-type2").addStoredField(field).get();
         assertThat(searchResponse.getHits().totalHits(), equalTo(1L));
         assertThat(searchResponse.getHits().getAt(0).field(field).isMetadataField(), equalTo(false));
         assertThat(searchResponse.getHits().getAt(0).field(field).getValues().size(), equalTo(2));
@@ -621,16 +621,16 @@ public class SearchFieldsTests extends ESIntegTestCase {
         client().admin().indices().prepareRefresh().execute().actionGet();
 
         SearchRequestBuilder builder = client().prepareSearch().setQuery(matchAllQuery())
-                .addFieldDataField("text_field")
-                .addFieldDataField("keyword_field")
-                .addFieldDataField("byte_field")
-                .addFieldDataField("short_field")
-                .addFieldDataField("integer_field")
-                .addFieldDataField("long_field")
-                .addFieldDataField("float_field")
-                .addFieldDataField("double_field")
-                .addFieldDataField("date_field")
-                .addFieldDataField("boolean_field");
+                .addDocValueField("text_field")
+                .addDocValueField("keyword_field")
+                .addDocValueField("byte_field")
+                .addDocValueField("short_field")
+                .addDocValueField("integer_field")
+                .addDocValueField("long_field")
+                .addDocValueField("float_field")
+                .addDocValueField("double_field")
+                .addDocValueField("date_field")
+                .addDocValueField("boolean_field");
         SearchResponse searchResponse = builder.execute().actionGet();
 
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(1L));
@@ -704,7 +704,7 @@ public class SearchFieldsTests extends ESIntegTestCase {
                     .setParent("parent_1")
                     .setSource(jsonBuilder().startObject().field("field1", "value").endObject()));
 
-        SearchResponse response = client().prepareSearch("test").addField("field1").get();
+        SearchResponse response = client().prepareSearch("test").addStoredField("field1").get();
         assertSearchResponse(response);
         assertHitCount(response, 1);
 
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
index 80a6ff891da..7725ee7f519 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByScrollRequest.java
@@ -123,8 +123,8 @@ public abstract class AbstractBulkByScrollRequest<Self extends AbstractBulkByScr
         if (searchRequest.source().from() != -1) {
             e = addValidationError("from is not supported in this context", e);
         }
-        if (searchRequest.source().fields() != null) {
-            e = addValidationError("fields is not supported in this context", e);
+        if (searchRequest.source().storedFields() != null) {
+            e = addValidationError("stored_fields is not supported in this context", e);
         }
         if (maxRetries < 0) {
             e = addValidationError("retries cannnot be negative", e);
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml
index 8648c9034ee..7f17befe909 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/delete_by_query/20_validation.yaml
@@ -59,11 +59,11 @@
 ---
 "source fields may not be modified":
   - do:
-      catch: /fields is not supported in this context/
+      catch: /stored_fields is not supported in this context/
       delete_by_query:
         index: test
         body:
-          fields: [_id]
+          stored_fields: [_id]
 
 ---
 "requests_per_second cannot be an empty string":
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
index a7a5198e430..b3e35d2f49e 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
@@ -216,11 +216,11 @@
 ---
 "source fields may not be modified":
   - do:
-      catch: /fields is not supported in this context/
+      catch: /stored_fields is not supported in this context/
       reindex:
         body:
           source:
             index: test
-            fields: [_id]
+            stored_fields: [_id]
           dest:
             index: dest
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml
index ea487eb54e0..08ab3cbcc10 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yaml
@@ -67,11 +67,11 @@
 ---
 "source fields may not be modified":
   - do:
-      catch: /fields is not supported in this context/
+      catch: /stored_fields is not supported in this context/
       update_by_query:
         index: test
         body:
-          fields: [_id]
+          stored_fields: [_id]
 
 ---
 "requests_per_second cannot be an empty string":
diff --git a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/30_mapping.yaml b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/30_mapping.yaml
index 458990cc90c..f180f51dfc5 100644
--- a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/30_mapping.yaml
+++ b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/30_mapping.yaml
@@ -54,7 +54,7 @@
         search:
             index: test
             body:
-                fields: [file.content_type,file.name]
+                stored_fields: [file.content_type,file.name]
 
     - match: { hits.total: 1 }
     - match: { hits.hits.0.fields: { file.content_type: ["text/my-dummy-content-type"],  file.name: ["my-dummy-name-txt"] }}
diff --git a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/40_highlight.yaml b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/40_highlight.yaml
index dc6f800c078..c2d42be3b9a 100644
--- a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/40_highlight.yaml
+++ b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/40_highlight.yaml
@@ -57,7 +57,7 @@ setup:
                 query:
                     match:
                         file.content: "apache tika"
-                fields: []
+                stored_fields: []
                 highlight:
                   fields:
                     file.content: {}
diff --git a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/50_files_supported.yaml b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/50_files_supported.yaml
index 69991b9d0c0..99f427c1ab9 100644
--- a/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/50_files_supported.yaml
+++ b/plugins/mapper-attachments/src/test/resources/rest-api-spec/test/mapper_attachments/50_files_supported.yaml
@@ -38,7 +38,7 @@ setup:
       search:
           index: test
           body:
-              fields: [file.content, file.author, file.date, file.content_length, file.content_type]
+              stored_fields: [file.content, file.author, file.date, file.content_length, file.content_type]
   - match: { hits.total: 1 }
   - match: { hits.hits.0.fields: {
       file.content: ["Test elasticsearch\n"],
@@ -65,7 +65,7 @@ setup:
       search:
           index: test
           body:
-              fields: [file.content, file.author, file.date, file.content_length, file.content_type]
+              stored_fields: [file.content, file.author, file.date, file.content_length, file.content_type]
   - match: { hits.total: 1 }
   - match: { hits.hits.0.fields: {
       file.content: ["Test elasticsearch\n"],
diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java
index 761fb5fd144..956645aebd2 100644
--- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java
+++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java
@@ -83,8 +83,8 @@ public class SizeFieldMapperUpgradeTests extends ESIntegTestCase {
         ElasticsearchAssertions.assertHitCount(countResponse, 3L);
 
         final SearchResponse sizeResponse = client().prepareSearch(indexName)
-                .addField("_source")
-                .addField("_size")
+                .addStoredField("_source")
+                .addStoredField("_size")
                 .get();
         ElasticsearchAssertions.assertHitCount(sizeResponse, 3L);
         for (SearchHit hit : sizeResponse.getHits().getHits()) {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
index d2b9b8cf9b4..21fda8dc805 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
@@ -38,13 +38,17 @@
           "type" : "boolean",
           "description" : "Specify whether to return detailed information about score computation as part of a hit"
         },
-        "fields": {
+        "stored_fields": {
           "type" : "list",
-          "description" : "A comma-separated list of fields to return as part of a hit"
+          "description" : "A comma-separated list of stored fields to return as part of a hit"
+        },
+        "docvalue_fields": {
+          "type" : "list",
+          "description" : "A comma-separated list of fields to return as the docvalue representation of a field for each hit"
         },
         "fielddata_fields": {
           "type" : "list",
-          "description" : "A comma-separated list of fields to return as the field data representation of a field for each hit"
+          "description" : "A comma-separated list of fields to return as the docvalue representation of a field for each hit"
         },
         "from": {
           "type" : "number",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml
index 84bf44f7392..bb5346ca280 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml
@@ -95,7 +95,7 @@ setup:
   - do:
       search:
         body:
-          fields: [ include.field2 ]
+          stored_fields: [ include.field2 ]
           query: { match_all: {} }
   - is_false:  hits.hits.0._source
 
@@ -104,14 +104,15 @@ setup:
   - do:
         search:
           body:
-            fields: [ include.field2, _source ]
+            stored_fields: [ include.field2, _source ]
             query: { match_all: {} }
   - match: { hits.hits.0._source.include.field2: v2 }
   - is_true:  hits.hits.0._source
 
 ---
 "fielddata_fields":
+=======
   - do:
       search:
-        fielddata_fields: [ "count" ]
+        docvalue_fields: [ "count" ]
   - match: { hits.hits.0.fields.count: [1] }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yaml
index df7322f12c8..993cbed2647 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/issue4895.yaml
@@ -31,6 +31,6 @@ setup:
                   term:
                       data: some
                   preference: _local
-              fields: [user,amount]
+              stored_fields: [user,amount]
 
 

From f572f8cc17feeb41155cfdc7289f80048e1a1e40 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Mon, 4 Jul 2016 11:02:06 +0200
Subject: [PATCH 1217/1311] Bad asciidoc link

---
 docs/reference/search/request/docvalue-fields.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/search/request/docvalue-fields.asciidoc b/docs/reference/search/request/docvalue-fields.asciidoc
index bd090a3bd8d..b4d2493d853 100644
--- a/docs/reference/search/request/docvalue-fields.asciidoc
+++ b/docs/reference/search/request/docvalue-fields.asciidoc
@@ -1,7 +1,7 @@
 [[search-request-docvalue-fields]]
 === Doc value Fields
 
-Allows to return the <<docvalue,doc value>> representation of a field for each hit, for
+Allows to return the <<doc-values,doc value>> representation of a field for each hit, for
 example:
 
 [source,js]

From 1924b46dbbdfa860441a02005b9a5bb1d6ef92b6 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Mon, 4 Jul 2016 12:33:44 +0200
Subject: [PATCH 1218/1311] Fix typo in rest test

---
 .../resources/rest-api-spec/test/search/10_source_filtering.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml
index bb5346ca280..424153aa573 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yaml
@@ -111,7 +111,6 @@ setup:
 
 ---
 "fielddata_fields":
-=======
   - do:
       search:
         docvalue_fields: [ "count" ]

From 86d2e883620ef44afb5fed7ed86854061d49f966 Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Fri, 17 Jun 2016 17:32:38 +0200
Subject: [PATCH 1219/1311] re-introduce: Inline reroute with process of node
 join/master election (#18938)

---
 .../cluster/routing/RoutingService.java       |  4 --
 .../elasticsearch/discovery/Discovery.java    |  4 +-
 .../discovery/local/LocalDiscovery.java       | 24 ++++++------
 .../discovery/zen/NodeJoinController.java     | 39 +++++++------------
 .../discovery/zen/ZenDiscovery.java           | 17 ++++----
 .../java/org/elasticsearch/node/Node.java     |  3 +-
 .../zen/NodeJoinControllerTests.java          | 35 +++++++----------
 .../org/elasticsearch/test/NoopDiscovery.java |  4 +-
 8 files changed, 51 insertions(+), 79 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
index 7bd2c2219b5..163057b9b46 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
@@ -72,10 +72,6 @@ public class RoutingService extends AbstractLifecycleComponent {
     protected void doClose() {
     }
 
-    public AllocationService getAllocationService() {
-        return this.allocationService;
-    }
-
     /**
      * Initiates a reroute.
      */
diff --git a/core/src/main/java/org/elasticsearch/discovery/Discovery.java b/core/src/main/java/org/elasticsearch/discovery/Discovery.java
index 751f5480ca8..d7cfb2eeae6 100644
--- a/core/src/main/java/org/elasticsearch/discovery/Discovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/Discovery.java
@@ -22,7 +22,7 @@ package org.elasticsearch.discovery;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RoutingService;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -44,7 +44,7 @@ public interface Discovery extends LifecycleComponent {
      * Another hack to solve dep injection problem..., note, this will be called before
      * any start is called.
      */
-    void setRoutingService(RoutingService routingService);
+    void setAllocationService(AllocationService allocationService);
 
     /**
      * Publish all the changes to the cluster from the master (can be called just by the master). The publish
diff --git a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
index 7ba207284b2..65767e5e86d 100644
--- a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
@@ -28,7 +28,7 @@ import org.elasticsearch.cluster.IncompatibleClusterStateVersionException;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.RoutingService;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -61,7 +61,7 @@ public class LocalDiscovery extends AbstractLifecycleComponent implements Discov
     private static final LocalDiscovery[] NO_MEMBERS = new LocalDiscovery[0];
 
     private final ClusterService clusterService;
-    private RoutingService routingService;
+    private AllocationService allocationService;
     private final ClusterName clusterName;
 
     private final DiscoverySettings discoverySettings;
@@ -81,8 +81,8 @@ public class LocalDiscovery extends AbstractLifecycleComponent implements Discov
     }
 
     @Override
-    public void setRoutingService(RoutingService routingService) {
-        this.routingService = routingService;
+    public void setAllocationService(AllocationService allocationService) {
+        this.allocationService = allocationService;
     }
 
     @Override
@@ -154,7 +154,12 @@ public class LocalDiscovery extends AbstractLifecycleComponent implements Discov
                             nodesBuilder.put(discovery.localNode());
                         }
                         nodesBuilder.localNodeId(master.localNode().getId()).masterNodeId(master.localNode().getId());
-                        return ClusterState.builder(currentState).nodes(nodesBuilder).build();
+                        currentState = ClusterState.builder(currentState).nodes(nodesBuilder).build();
+                        RoutingAllocation.Result result =  master.allocationService.reroute(currentState, "node_add");
+                        if (result.changed()) {
+                            currentState = ClusterState.builder(currentState).routingResult(result).build();
+                        }
+                        return currentState;
                     }
 
                     @Override
@@ -162,13 +167,6 @@ public class LocalDiscovery extends AbstractLifecycleComponent implements Discov
                         logger.error("unexpected failure during [{}]", t, source);
                     }
 
-                    @Override
-                    public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
-                        // we reroute not in the same cluster state update since in certain areas we rely on
-                        // the node to be in the cluster state (sampled from ClusterService#state) to be there, also
-                        // shard transitions need to better be handled in such cases
-                        master.routingService.reroute("post_node_add");
-                    }
                 });
             }
         } // else, no master node, the next node that will start will fill things in...
@@ -224,7 +222,7 @@ public class LocalDiscovery extends AbstractLifecycleComponent implements Discov
                         }
                         // reroute here, so we eagerly remove dead nodes from the routing
                         ClusterState updatedState = ClusterState.builder(currentState).nodes(newNodes).build();
-                        RoutingAllocation.Result routingResult = master.routingService.getAllocationService().reroute(
+                        RoutingAllocation.Result routingResult = master.allocationService.reroute(
                                 ClusterState.builder(updatedState).build(), "elected as master");
                         return ClusterState.builder(updatedState).routingResult(routingResult).build();
                     }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
index f4551ecb016..57cfe7e1da5 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
@@ -29,7 +29,7 @@ import org.elasticsearch.cluster.NotMasterException;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.RoutingService;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
@@ -58,7 +58,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 public class NodeJoinController extends AbstractComponent {
 
     private final ClusterService clusterService;
-    private final RoutingService routingService;
+    private final AllocationService allocationService;
     private final ElectMasterService electMaster;
     private final DiscoverySettings discoverySettings;
     private final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor();
@@ -68,10 +68,11 @@ public class NodeJoinController extends AbstractComponent {
     private ElectionContext electionContext = null;
 
 
-    public NodeJoinController(ClusterService clusterService, RoutingService routingService, ElectMasterService electMaster, DiscoverySettings discoverySettings, Settings settings) {
+    public NodeJoinController(ClusterService clusterService, AllocationService allocationService, ElectMasterService electMaster,
+                              DiscoverySettings discoverySettings, Settings settings) {
         super(settings);
         this.clusterService = clusterService;
-        this.routingService = routingService;
+        this.allocationService = allocationService;
         this.electMaster = electMaster;
         this.discoverySettings = discoverySettings;
     }
@@ -406,21 +407,7 @@ public class NodeJoinController extends AbstractComponent {
                 ClusterBlocks clusterBlocks = ClusterBlocks.builder().blocks(currentState.blocks())
                     .removeGlobalBlock(discoverySettings.getNoMasterBlock()).build();
                 newState.blocks(clusterBlocks);
-                newState.nodes(nodesBuilder);
                 nodesChanged = true;
-
-                // reroute now to remove any dead nodes (master may have stepped down when they left and didn't update the routing table)
-                // Note: also do it now to avoid assigning shards to these nodes. We will have another reroute after the cluster
-                // state is published.
-                // TODO: this publishing of a cluster state with no nodes assigned to joining nodes shouldn't be needed anymore. remove.
-
-                final ClusterState tmpState = newState.build();
-                RoutingAllocation.Result result = routingService.getAllocationService().reroute(tmpState, "nodes joined");
-                newState = ClusterState.builder(tmpState);
-                if (result.changed()) {
-                    newState.routingResult(result);
-                }
-                nodesBuilder = DiscoveryNodes.builder(tmpState.nodes());
             }
 
             if (nodesBuilder.isLocalNodeElectedMaster() == false) {
@@ -439,7 +426,8 @@ public class NodeJoinController extends AbstractComponent {
                     for (DiscoveryNode existingNode : currentNodes) {
                         if (node.getAddress().equals(existingNode.getAddress())) {
                             nodesBuilder.remove(existingNode.getId());
-                            logger.warn("received join request from node [{}], but found existing node {} with same address, removing existing node", node, existingNode);
+                            logger.warn("received join request from node [{}], but found existing node {} with same address, " +
+                                "removing existing node", node, existingNode);
                         }
                     }
                 }
@@ -448,6 +436,12 @@ public class NodeJoinController extends AbstractComponent {
 
             if (nodesChanged) {
                 newState.nodes(nodesBuilder);
+                final ClusterState tmpState = newState.build();
+                RoutingAllocation.Result result = allocationService.reroute(tmpState, "node_join");
+                newState = ClusterState.builder(tmpState);
+                if (result.changed()) {
+                    newState.routingResult(result);
+                }
             }
 
             // we must return a new cluster state instance to force publishing. This is important
@@ -463,13 +457,6 @@ public class NodeJoinController extends AbstractComponent {
 
         @Override
         public void clusterStatePublished(ClusterChangedEvent event) {
-            if (event.nodesDelta().hasChanges()) {
-                // we reroute not in the same cluster state update since in certain areas we rely on
-                // the node to be in the cluster state (sampled from ClusterService#state) to be there, also
-                // shard transitions need to better be handled in such cases
-                routingService.reroute("post_node_add");
-            }
-
             NodeJoinController.this.electMaster.logMinimumMasterNodesWarningIfNecessary(event.previousState(), event.state());
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
index 6ebdfa7e6e6..598eb2d6875 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
@@ -32,7 +32,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.RoutingService;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Priority;
@@ -113,7 +113,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
 
     private final TransportService transportService;
     private final ClusterService clusterService;
-    private RoutingService routingService;
+    private AllocationService allocationService;
     private final ClusterName clusterName;
     private final DiscoverySettings discoverySettings;
     private final ZenPingService pingService;
@@ -146,7 +146,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
     /** counts the time this node has joined the cluster or have elected it self as master */
     private final AtomicLong clusterJoinsCounter = new AtomicLong();
 
-    // must initialized in doStart(), when we have the routingService set
+    // must initialized in doStart(), when we have the allocationService set
     private volatile NodeJoinController nodeJoinController;
 
     @Inject
@@ -206,8 +206,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
     }
 
     @Override
-    public void setRoutingService(RoutingService routingService) {
-        this.routingService = routingService;
+    public void setAllocationService(AllocationService allocationService) {
+        this.allocationService = allocationService;
     }
 
     @Override
@@ -215,7 +215,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
         nodesFD.setLocalNode(clusterService.localNode());
         joinThreadControl.start();
         pingService.start();
-        this.nodeJoinController = new NodeJoinController(clusterService, routingService, electMaster, discoverySettings, settings);
+        this.nodeJoinController = new NodeJoinController(clusterService, allocationService, electMaster, discoverySettings, settings);
     }
 
     @Override
@@ -516,8 +516,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
                         return rejoin(currentState, "not enough master nodes");
                     }
                     // eagerly run reroute to remove dead nodes from routing table
-                    RoutingAllocation.Result routingResult = routingService.getAllocationService().reroute(
-                            ClusterState.builder(currentState).build(),
+                    RoutingAllocation.Result routingResult = allocationService.reroute(ClusterState.builder(currentState).build(),
                             "[" + node + "] left");
                     return ClusterState.builder(currentState).routingResult(routingResult).build();
                 }
@@ -561,7 +560,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
                     return rejoin(currentState, "not enough master nodes");
                 }
                 // eagerly run reroute to remove dead nodes from routing table
-                RoutingAllocation.Result routingResult = routingService.getAllocationService().reroute(
+                RoutingAllocation.Result routingResult = allocationService.reroute(
                         ClusterState.builder(currentState).build(),
                         "[" + node + "] failed");
                 return ClusterState.builder(currentState).routingResult(routingResult).build();
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index c466490939e..ce1c21943f3 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -39,6 +39,7 @@ import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeService;
 import org.elasticsearch.cluster.routing.RoutingService;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.StopWatch;
 import org.elasticsearch.common.component.Lifecycle;
@@ -350,7 +351,7 @@ public class Node implements Closeable {
         ESLogger logger = Loggers.getLogger(Node.class, NODE_NAME_SETTING.get(settings));
         logger.info("starting ...");
         // hack around dependency injection problem (for now...)
-        injector.getInstance(Discovery.class).setRoutingService(injector.getInstance(RoutingService.class));
+        injector.getInstance(Discovery.class).setAllocationService(injector.getInstance(AllocationService.class));
         for (Class<? extends LifecycleComponent> plugin : pluginsService.nodeServices()) {
             injector.getInstance(plugin).start();
         }
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
index 15c8d312952..9825cd3284e 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
@@ -24,7 +24,6 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.NotMasterException;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
@@ -99,10 +98,9 @@ public class NodeJoinControllerTests extends ESTestCase {
         // make sure we have a master
         setState(clusterService, ClusterState.builder(clusterService.state()).nodes(
             DiscoveryNodes.builder(initialNodes).masterNodeId(localNode.getId())));
-        nodeJoinController = new NodeJoinController(clusterService, new NoopRoutingService(Settings.EMPTY),
-            new ElectMasterService(Settings.EMPTY),
-            new DiscoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)),
-            Settings.EMPTY);
+        nodeJoinController = new NodeJoinController(clusterService, new NoopAllocationService(Settings.EMPTY),
+            new ElectMasterService(Settings.EMPTY), new DiscoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY,
+            ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), Settings.EMPTY);
     }
 
     @After
@@ -298,7 +296,8 @@ public class NodeJoinControllerTests extends ESTestCase {
         }
 
         logger.debug("--> asserting master election didn't finish yet");
-        assertThat("election finished after [" + initialJoins + "] master nodes but required joins is [" + requiredJoins + "]", electionFuture.isDone(), equalTo(false));
+        assertThat("election finished after [" + initialJoins + "] master nodes but required joins is [" + requiredJoins + "]",
+            electionFuture.isDone(), equalTo(false));
 
         final int finalJoins = requiredJoins - initialJoins + randomInt(5);
         nodesToJoin.clear();
@@ -374,7 +373,8 @@ public class NodeJoinControllerTests extends ESTestCase {
         nodeJoinController.waitToBeElectedAsMaster(requiredJoins, TimeValue.timeValueMillis(1), new NodeJoinController.ElectionCallback() {
             @Override
             public void onElectedAsMaster(ClusterState state) {
-                assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(), equalTo(true));
+                assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(),
+                    equalTo(true));
                 latch.countDown();
             }
 
@@ -492,7 +492,8 @@ public class NodeJoinControllerTests extends ESTestCase {
         nodeJoinController.waitToBeElectedAsMaster(requiredJoins, TimeValue.timeValueHours(30), new NodeJoinController.ElectionCallback() {
             @Override
             public void onElectedAsMaster(ClusterState state) {
-                assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(), equalTo(true));
+                assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(),
+                    equalTo(true));
                 latch.countDown();
             }
 
@@ -516,18 +517,6 @@ public class NodeJoinControllerTests extends ESTestCase {
     }
 
 
-    static class NoopRoutingService extends RoutingService {
-
-        public NoopRoutingService(Settings settings) {
-            super(settings, null, new NoopAllocationService(settings));
-        }
-
-        @Override
-        protected void performReroute(String reason) {
-
-        }
-    }
-
     static class NoopAllocationService extends AllocationService {
 
         public NoopAllocationService(Settings settings) {
@@ -535,12 +524,14 @@ public class NodeJoinControllerTests extends ESTestCase {
         }
 
         @Override
-        public RoutingAllocation.Result applyStartedShards(ClusterState clusterState, List<? extends ShardRouting> startedShards, boolean withReroute) {
+        public RoutingAllocation.Result applyStartedShards(ClusterState clusterState, List<? extends ShardRouting> startedShards,
+                                                           boolean withReroute) {
             return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData());
         }
 
         @Override
-        public RoutingAllocation.Result applyFailedShards(ClusterState clusterState, List<FailedRerouteAllocation.FailedShard> failedShards) {
+        public RoutingAllocation.Result applyFailedShards(ClusterState clusterState,
+                                                          List<FailedRerouteAllocation.FailedShard> failedShards) {
             return new RoutingAllocation.Result(false, clusterState.routingTable(), clusterState.metaData());
         }
 
diff --git a/core/src/test/java/org/elasticsearch/test/NoopDiscovery.java b/core/src/test/java/org/elasticsearch/test/NoopDiscovery.java
index 21c13816293..c36082f1475 100644
--- a/core/src/test/java/org/elasticsearch/test/NoopDiscovery.java
+++ b/core/src/test/java/org/elasticsearch/test/NoopDiscovery.java
@@ -20,7 +20,7 @@ package org.elasticsearch.test;
 
 import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.RoutingService;
+import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.component.LifecycleListener;
 import org.elasticsearch.discovery.Discovery;
@@ -41,7 +41,7 @@ public class NoopDiscovery implements Discovery {
     }
 
     @Override
-    public void setRoutingService(RoutingService routingService) {
+    public void setAllocationService(AllocationService allocationService) {
 
     }
 

From 3343ceeae44f3d28e3c1ba7861886848df74e390 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 4 Jul 2016 08:41:06 -0400
Subject: [PATCH 1220/1311] Do not catch throwable

Today throughout the codebase, catch throwable is used with reckless
abandon. This is dangerous because the throwable could be a fatal
virtual machine error resulting from an internal error in the JVM, or an
out of memory error or a stack overflow error that leaves the virtual
machine in an unstable and unpredictable state. This commit removes
catch throwable from the codebase and removes the temptation to use it
by modifying listener APIs to receive instances of Exception instead of
the top-level Throwable.

Relates #19231
---
 .../elasticsearch/ElasticsearchException.java |  12 +-
 .../ElasticsearchSecurityException.java       |   2 +-
 .../org/elasticsearch/ExceptionsHelper.java   |  24 ++--
 .../elasticsearch/action/ActionListener.java  |   2 +-
 .../elasticsearch/action/ActionRunnable.java  |  10 +-
 .../action/LatchedActionListener.java         |   2 +-
 .../action/TaskOperationFailure.java          |  10 +-
 ...ansportClusterAllocationExplainAction.java |   4 +-
 .../health/TransportClusterHealthAction.java  |   6 +-
 .../tasks/get/TransportGetTaskAction.java     |  10 +-
 .../TransportDeleteRepositoryAction.java      |   2 +-
 .../put/TransportPutRepositoryAction.java     |   2 +-
 .../TransportVerifyRepositoryAction.java      |   2 +-
 .../TransportClusterRerouteAction.java        |   6 +-
 .../TransportClusterUpdateSettingsAction.java |  16 +--
 .../create/TransportCreateSnapshotAction.java |   8 +-
 .../delete/TransportDeleteSnapshotAction.java |   4 +-
 .../get/TransportGetSnapshotsAction.java      |   4 +-
 .../TransportRestoreSnapshotAction.java       |   4 +-
 .../TransportSnapshotsStatusAction.java       |   4 +-
 .../alias/TransportIndicesAliasesAction.java  |   2 +-
 .../close/TransportCloseIndexAction.java      |   2 +-
 .../create/TransportCreateIndexAction.java    |   2 +-
 .../delete/TransportDeleteIndexAction.java    |   2 +-
 .../get/TransportGetFieldMappingsAction.java  |   2 +-
 .../put/TransportPutMappingAction.java        |   2 +-
 .../open/TransportOpenIndexAction.java        |   2 +-
 .../rollover/TransportRolloverAction.java     |   7 +-
 .../put/TransportUpdateSettingsAction.java    |   2 +-
 .../shards/IndicesShardStoresResponse.java    |   8 +-
 .../indices/shrink/TransportShrinkAction.java |   4 +-
 .../TransportDeleteIndexTemplateAction.java   |   6 +-
 .../put/TransportPutIndexTemplateAction.java  |   6 +-
 .../upgrade/post/TransportUpgradeAction.java  |   8 +-
 .../post/TransportUpgradeSettingsAction.java  |   2 +-
 .../action/bulk/BulkItemResponse.java         |   8 +-
 .../action/bulk/BulkRequestHandler.java       |  14 +-
 .../org/elasticsearch/action/bulk/Retry.java  |   6 +-
 .../action/bulk/TransportBulkAction.java      |  15 ++-
 .../action/bulk/TransportShardBulkAction.java |  56 ++++----
 .../action/delete/TransportDeleteAction.java  |   2 +-
 .../action/get/MultiGetResponse.java          |  18 +--
 .../action/get/TransportMultiGetAction.java   |   2 +-
 .../get/TransportShardMultiGetAction.java     |  10 +-
 .../action/index/TransportIndexAction.java    |   9 +-
 .../action/ingest/IngestActionFilter.java     |  20 +--
 .../ingest/PutPipelineTransportAction.java    |   2 +-
 .../ingest/SimulateDocumentBaseResult.java    |   4 +-
 .../ingest/SimulateProcessorResult.java       |   4 +-
 .../search/AbstractSearchAsyncAction.java     |  57 ++++----
 .../action/search/MultiSearchResponse.java    |  21 ++-
 .../SearchDfsQueryAndFetchAsyncAction.java    |  14 +-
 .../SearchDfsQueryThenFetchAsyncAction.java   |  22 +--
 .../search/SearchPhaseExecutionException.java |   3 -
 .../SearchQueryAndFetchAsyncAction.java       |   4 +-
 .../SearchQueryThenFetchAsyncAction.java      |  12 +-
 .../SearchScrollQueryAndFetchAsyncAction.java |  12 +-
 ...SearchScrollQueryThenFetchAsyncAction.java |  23 ++--
 .../action/search/ShardSearchFailure.java     |  12 +-
 .../search/TransportClearScrollAction.java    |   4 +-
 .../search/TransportMultiSearchAction.java    |   3 +-
 .../search/TransportSearchScrollAction.java   |   2 +-
 .../AbstractListenableActionFuture.java       |   2 +-
 .../action/support/AdapterActionFuture.java   |   2 +-
 .../DefaultShardOperationFailedException.java |   8 +-
 .../support/DelegatingActionListener.java     |   2 +-
 .../support/HandledTransportAction.java       |   4 +-
 .../support/PlainListenableActionFuture.java  |   4 +-
 .../support/ThreadedActionListener.java       |  10 +-
 .../action/support/TransportAction.java       |  33 ++---
 .../action/support/TransportActions.java      |  21 +--
 .../broadcast/TransportBroadcastAction.java   |  36 ++---
 .../node/TransportBroadcastByNodeAction.java  |  31 ++---
 .../master/TransportMasterNodeAction.java     |   2 +-
 .../support/nodes/TransportNodesAction.java   |  13 +-
 .../replication/ReplicationOperation.java     |  32 ++---
 .../replication/ReplicationResponse.java      |   6 +-
 .../TransportBroadcastReplicationAction.java  |   2 +-
 .../TransportReplicationAction.java           |  78 +++++------
 ...ransportInstanceSingleOperationAction.java |  21 +--
 .../shard/TransportSingleShardAction.java     |  14 +-
 .../support/tasks/TransportTasksAction.java   |  16 +--
 .../termvectors/MultiTermVectorsResponse.java |  10 +-
 .../TransportMultiTermVectorsAction.java      |   2 +-
 .../TransportShardMultiTermsVectorAction.java |   2 +-
 .../action/update/TransportUpdateAction.java  |  38 +++---
 .../elasticsearch/bootstrap/Bootstrap.java    |   6 +-
 .../elasticsearch/bootstrap/JNANatives.java   |   6 +-
 .../org/elasticsearch/bootstrap/Seccomp.java  |   2 +-
 .../TransportClientNodesService.java          |  19 +--
 .../AckedClusterStateTaskListener.java        |   4 +-
 .../cluster/AckedClusterStateUpdateTask.java  |   8 +-
 .../cluster/ClusterStateTaskConfig.java       |   2 +-
 .../cluster/ClusterStateTaskExecutor.java     |  18 +--
 .../cluster/ClusterStateTaskListener.java     |   2 +-
 .../cluster/ClusterStateUpdateTask.java       |   6 +-
 .../cluster/InternalClusterInfoService.java   |   6 +-
 .../cluster/NodeConnectionsService.java       |   6 +-
 .../action/shard/ShardStateAction.java        |  48 +++----
 .../cluster/metadata/AliasValidator.java      |   6 +-
 .../MetaDataIndexTemplateService.java         |  14 +-
 .../metadata/MetaDataMappingService.java      |  16 +--
 .../MetaDataUpdateSettingsService.java        |   2 +-
 .../routing/DelayedAllocationService.java     |   8 +-
 .../cluster/routing/RoutingService.java       |   8 +-
 .../cluster/routing/UnassignedInfo.java       |  11 +-
 .../allocation/FailedRerouteAllocation.java   |   4 +-
 .../cluster/service/ClusterService.java       |  42 +++---
 .../common/MacAddressProvider.java            |   2 +-
 .../org/elasticsearch/common/Randomness.java  |   2 +-
 .../common/geo/ShapesAvailability.java        |   4 +-
 .../common/io/stream/StreamInput.java         |  50 ++++---
 .../common/io/stream/StreamOutput.java        |  17 +--
 .../common/lease/Releasables.java             |   5 +-
 .../common/logging/LoggerMessageFormat.java   |   2 +-
 .../elasticsearch/common/network/Cidrs.java   |   4 +-
 .../common/util/CancellableThreads.java       |   7 +-
 .../util/concurrent/AbstractRunnable.java     |  10 +-
 .../common/util/concurrent/ThreadBarrier.java |  24 ++--
 .../common/util/concurrent/ThreadContext.java |   8 +-
 ...AckClusterStatePublishResponseHandler.java |  15 ++-
 ...ingClusterStatePublishResponseHandler.java |   2 +-
 .../elasticsearch/discovery/Discovery.java    |   2 +-
 .../discovery/local/LocalDiscovery.java       |  18 +--
 .../discovery/zen/NodeJoinController.java     |  18 +--
 .../discovery/zen/ZenDiscovery.java           |  65 ++++-----
 .../zen/fd/MasterFaultDetection.java          |  13 +-
 .../zen/membership/MembershipAction.java      |  15 ++-
 .../zen/ping/unicast/UnicastZenPing.java      |  11 +-
 .../publish/PendingClusterStatesQueue.java    |   9 +-
 .../publish/PublishClusterStateAction.java    |  39 +++---
 .../org/elasticsearch/env/ESFileStore.java    |   4 +-
 .../elasticsearch/env/NodeEnvironment.java    |  22 ++-
 .../gateway/AsyncShardFetch.java              |   2 +-
 .../gateway/DanglingIndicesState.java         |   2 +-
 .../gateway/GatewayMetaState.java             |   9 +-
 .../elasticsearch/gateway/GatewayService.java |  10 +-
 .../gateway/LocalAllocateDangledIndices.java  |  11 +-
 .../gateway/MetaDataStateFormat.java          |   2 +-
 .../gateway/MetaStateService.java             |   4 +-
 ...ransportNodesListGatewayStartedShards.java |   8 +-
 .../org/elasticsearch/http/HttpServer.java    |  14 +-
 .../index/CompositeIndexEventListener.java    | 106 +++++++--------
 .../org/elasticsearch/index/IndexService.java |  14 +-
 .../org/elasticsearch/index/IndexWarmer.java  |   4 +-
 .../index/cache/bitset/BitsetFilterCache.java |   8 +-
 .../elasticsearch/index/engine/Engine.java    |  25 ++--
 .../index/engine/InternalEngine.java          | 125 +++++++++++-------
 .../index/engine/ShadowEngine.java            |  18 ++-
 .../fielddata/IndexFieldDataService.java      |  30 +----
 .../plain/AbstractIndexFieldData.java         |   4 +-
 .../plain/AbstractIndexOrdinalsFieldData.java |   2 +-
 .../plain/ParentChildIndexFieldData.java      |   2 +-
 .../SortedSetDVOrdinalsIndexFieldData.java    |   4 +-
 .../index/mapper/DocumentParser.java          |   6 +-
 .../index/mapper/MapperService.java           |   2 +-
 .../elasticsearch/index/mapper/Mapping.java   |   3 +-
 .../index/query/QueryParseContext.java        |   2 +-
 .../elasticsearch/index/shard/IndexShard.java |  67 +++++-----
 .../shard/IndexingOperationListener.java      |  34 ++---
 .../index/shard/InternalIndexingStats.java    |   4 +-
 .../index/shard/RefreshListeners.java         |   4 +-
 .../index/shard/SearchOperationListener.java  |  40 +++---
 .../index/shard/StoreRecovery.java            |  13 +-
 .../shard/TranslogRecoveryPerformer.java      |   4 +-
 .../BlobStoreIndexShardRepository.java        |  23 ++--
 .../org/elasticsearch/index/store/Store.java  |  20 +--
 .../index/termvectors/TermVectorsService.java |  12 +-
 .../index/translog/Translog.java              |  42 ++++--
 .../index/translog/TranslogWriter.java        |  58 +++++---
 .../indices/IndexingMemoryController.java     |   4 +-
 .../elasticsearch/indices/IndicesService.java |  14 +-
 .../indices/analysis/HunspellService.java     |   4 +-
 .../cluster/IndicesClusterStateService.java   |  47 ++++---
 .../cache/IndicesFieldDataCache.java          |   8 +-
 .../indices/flush/SyncedFlushService.java     |  16 +--
 .../recovery/RecoveriesCollection.java        |   5 +-
 .../recovery/RecoverySourceHandler.java       |  22 +--
 .../indices/recovery/RecoveryTarget.java      |   6 +-
 .../recovery/RecoveryTargetService.java       |  13 +-
 .../SharedFSRecoverySourceHandler.java        |   8 +-
 .../indices/store/IndicesStore.java           |   6 +-
 .../indices/ttl/IndicesTTLService.java        |   4 +-
 .../ingest/ConfigurationUtils.java            |   2 +-
 .../ingest/PipelineExecutionService.java      |  16 +--
 .../org/elasticsearch/monitor/Probes.java     |   2 +-
 .../monitor/jvm/JvmGcMonitorService.java      |  10 +-
 .../elasticsearch/monitor/jvm/JvmInfo.java    |   2 +-
 .../elasticsearch/monitor/jvm/JvmStats.java   |   6 +-
 .../org/elasticsearch/monitor/os/OsProbe.java |  12 +-
 .../monitor/process/ProcessProbe.java         |  14 +-
 .../elasticsearch/plugins/PluginsService.java |   2 +-
 .../repositories/RepositoriesService.java     |  39 +++---
 .../repositories/VerificationFailure.java     |   6 +-
 .../VerifyNodeRepositoryAction.java           |   6 +-
 .../blobstore/BlobStoreRepository.java        |   2 +-
 .../elasticsearch/rest/BytesRestResponse.java |  26 ++--
 .../elasticsearch/rest/RestController.java    |   7 +-
 .../node/tasks/RestListTasksAction.java       |   2 +-
 .../alias/head/RestAliasesExistAction.java    |   9 +-
 .../action/support/RestActionListener.java    |  11 +-
 .../elasticsearch/script/ScriptService.java   |   2 +-
 .../elasticsearch/search/MultiValueMode.java  |   2 +-
 .../elasticsearch/search/SearchService.java   |  48 ++++---
 .../search/action/SearchTransportService.java |   2 +-
 .../support/AggregationContext.java           |   8 +-
 .../search/query/QueryPhase.java              |   2 +-
 .../snapshots/RestoreService.java             |  16 +--
 .../snapshots/SnapshotShardsService.java      |  24 ++--
 .../snapshots/SnapshotsService.java           | 105 ++++++++-------
 .../tasks/PersistedTaskInfo.java              |   6 +-
 .../java/org/elasticsearch/tasks/Task.java    |   2 +-
 .../org/elasticsearch/tasks/TaskManager.java  |   6 +-
 .../tasks/TaskPersistenceService.java         |  11 +-
 .../elasticsearch/threadpool/ThreadPool.java  |   6 +-
 .../transport/DelegatingTransportChannel.java |   4 +-
 .../transport/PlainTransportFuture.java       |   4 +-
 .../transport/RequestHandlerRegistry.java     |   4 +-
 ...ponseHandlerFailureTransportException.java |   2 -
 .../elasticsearch/transport/TcpTransport.java |  58 ++++----
 .../transport/TcpTransportChannel.java        |   4 +-
 .../transport/TransportChannel.java           |   2 +-
 .../transport/TransportService.java           |  59 ++++-----
 .../transport/TransportServiceAdapter.java    |   2 +-
 .../transport/local/LocalTransport.java       |  44 +++---
 .../local/LocalTransportChannel.java          |   6 +-
 .../transport/netty/NettyTransport.java       |  10 +-
 .../org/elasticsearch/tribe/TribeService.java |  17 +--
 .../elasticsearch/watcher/FileWatcher.java    |  20 +--
 .../org/elasticsearch/ESExceptionTests.java   |  48 ++++---
 .../ExceptionSerializationTests.java          |  18 ++-
 .../action/ListenerActionIT.java              |   2 +-
 .../action/RejectionActionIT.java             |   4 +-
 .../action/admin/HotThreadsIT.java            |   2 +-
 .../node/tasks/CancellableTasksTests.java     |   4 +-
 .../admin/cluster/node/tasks/TasksIT.java     |   4 +-
 .../node/tasks/TransportTasksActionTests.java |   4 +-
 .../cluster/reroute/ClusterRerouteTests.java  |   2 +-
 .../admin/indices/create/CreateIndexIT.java   |   5 +-
 .../MetaDataIndexTemplateServiceTests.java    |   9 +-
 .../elasticsearch/action/bulk/RetryTests.java |   2 +-
 .../bulk/TransportBulkActionTookTests.java    |   2 +-
 .../ingest/BulkRequestModifierTests.java      |   4 +-
 .../action/main/MainActionTests.java          |   2 +-
 .../support/ListenableActionFutureTests.java  |   6 +-
 .../TransportActionFilterChainTests.java      |  14 +-
 .../TransportBroadcastByNodeActionTests.java  |   2 +-
 .../TransportMasterNodeActionTests.java       |   4 +-
 .../ReplicationOperationTests.java            |  26 ++--
 .../TransportReplicationActionTests.java      |   8 +-
 .../TransportWriteActionTests.java            |   2 +-
 ...ortInstanceSingleOperationActionTests.java |  12 +-
 .../action/termvectors/GetTermVectorsIT.java  |  20 ++-
 .../termvectors/MultiTermVectorsIT.java       |  21 ++-
 .../client/AbstractClientHeadersTestCase.java |   2 +-
 .../TransportClientNodesServiceTests.java     |   3 +-
 .../cluster/MinimumMasterNodesIT.java         |   4 +-
 .../action/shard/ShardStateActionTests.java   |  18 +--
 .../health/ClusterStateHealthTests.java       |   4 +-
 .../cluster/service/ClusterServiceIT.java     |  32 ++---
 .../cluster/service/ClusterServiceTests.java  |  56 ++++----
 .../breaker/MemoryCircuitBreakerTests.java    |  14 +-
 .../common/util/CancellableThreadsTests.java  |  16 +--
 .../AbstractLifecycleRunnableTests.java       |   8 +-
 .../concurrent/AbstractRunnableTests.java     |  33 ++---
 .../util/concurrent/EsExecutorsTests.java     |   8 +-
 .../util/concurrent/RefCountedTests.java      |   4 +-
 .../util/concurrent/ThreadContextTests.java   |   4 +-
 .../ConstructingObjectParserTests.java        |   2 +-
 ...usterStatePublishResponseHandlerTests.java |   4 +-
 .../DiscoveryWithServiceDisruptionsIT.java    |  10 +-
 .../zen/NodeJoinControllerTests.java          |  30 ++---
 .../discovery/zen/ZenDiscoveryIT.java         |   8 +-
 .../PendingClusterStatesQueueTests.java       |   4 +-
 .../PublishClusterStateActionTests.java       |  10 +-
 .../env/NodeEnvironmentTests.java             |   9 +-
 .../gateway/AsyncShardFetchTests.java         |   2 +-
 .../gateway/MetaDataWriteDataNodesIT.java     |   4 +-
 .../gateway/PrimaryShardAllocatorTests.java   |   4 +-
 .../index/IndexWithShadowReplicasIT.java      |   8 +-
 .../index/engine/InternalEngineTests.java     |  12 +-
 .../index/engine/ShadowEngineTests.java       |   2 +-
 .../mapper/DynamicMappingDisabledTests.java   |   3 +-
 .../mapper/DynamicMappingIntegrationIT.java   |   4 +-
 .../index/mapper/MapperServiceTests.java      |   2 +-
 .../mapper/merge/TestMergeMapperTests.java    |   6 +-
 .../ESIndexLevelReplicationTestCase.java      |   8 +-
 .../index/shard/IndexShardTests.java          |  18 +--
 .../shard/IndexingOperationListenerTests.java |   8 +-
 .../index/shard/RefreshListenersTests.java    |  10 +-
 .../index/store/CorruptedFileIT.java          |   6 +-
 .../index/translog/TranslogTests.java         |  33 +++--
 .../elasticsearch/indexing/IndexActionIT.java |  14 +-
 .../indices/analyze/AnalyzeActionIT.java      |   6 +-
 .../elasticsearch/indices/flush/FlushIT.java  |   4 +-
 .../indices/flush/SyncedFlushUtil.java        |   4 +-
 .../mapping/ConcurrentDynamicTemplateIT.java  |   4 +-
 .../mapping/UpdateMappingIntegrationIT.java   |  11 +-
 .../indices/recovery/RecoveryTargetTests.java |   8 +-
 .../indices/state/RareClusterStateIT.java     |  12 +-
 .../store/IndicesStoreIntegrationIT.java      |   2 +-
 .../ingest/PipelineExecutionServiceTests.java |  38 +++---
 .../jvm/JvmGcMonitorServiceSettingsTests.java |  40 +++---
 .../monitor/jvm/JvmMonitorTests.java          |  12 +-
 .../rest/BytesRestResponseTests.java          |  31 +++--
 .../search/SearchWithRejectionsIT.java        |   2 +-
 .../search/aggregations/bucket/FilterIT.java  |   3 -
 .../search/aggregations/bucket/FiltersIT.java |   3 -
 .../search/basic/SearchWhileRelocatingIT.java |   6 +-
 .../AbstractSnapshotIntegTestCase.java        |   4 +-
 .../DedicatedClusterSnapshotRestoreIT.java    |   2 +-
 .../SharedClusterSnapshotRestoreIT.java       |   2 +-
 .../test/geo/RandomShapeGenerator.java        |  21 ++-
 .../threadpool/SimpleThreadPoolIT.java        |  15 +--
 .../org/elasticsearch/update/UpdateIT.java    |  13 +-
 .../ConcurrentDocumentOperationIT.java        |   2 +-
 .../groovy/GroovyScriptEngineService.java     |   4 +-
 .../messy/tests/MinDocCountTests.java         |   4 +-
 .../script/groovy/GroovyScriptTests.java      |   6 +-
 .../template/MultiSearchTemplateResponse.java |  20 +--
 .../TransportMultiSearchTemplateAction.java   |   2 +-
 .../TransportSearchTemplateAction.java        |   9 +-
 .../percolator/MultiPercolateResponse.java    |  20 +--
 .../TransportMultiPercolateAction.java        |   4 +-
 .../percolator/TransportPercolateAction.java  |   4 +-
 .../AbstractAsyncBulkByScrollAction.java      |  25 ++--
 .../index/reindex/BulkByScrollTask.java       |   8 +-
 .../reindex/AsyncBulkByScrollActionTests.java |   2 +-
 .../index/reindex/BulkByScrollTaskTests.java  |   6 +-
 .../UpdateByQueryWhileModifyingTests.java     |   6 +-
 .../index/analysis/IcuTokenizerFactory.java   |   7 +-
 .../gce/GceUnicastHostsProvider.java          |   4 +-
 .../attachment/AttachmentProcessor.java       |   2 +-
 .../ingest/attachment/TikaDocTests.java       |   2 +-
 .../plugin/example/ExampleCatAction.java      |   7 +-
 .../JavaScriptScriptMultiThreadedTests.java   |  15 +--
 .../PythonScriptMultiThreadedTests.java       |  12 +-
 .../mapper/attachments/AttachmentMapper.java  |   6 +-
 .../mapper/attachments/TikaDocTests.java      |   2 +-
 .../cloud/aws/blobstore/S3BlobContainer.java  |   5 +-
 .../s3/AbstractS3SnapshotRestoreTest.java     |   2 +-
 .../elasticsearch/bootstrap/SeccompTests.java |   2 +-
 .../bootstrap/ESElasticsearchCliTestCase.java |   4 +-
 .../elasticsearch/test/BackgroundIndexer.java |   4 +-
 .../test/ClusterServiceUtils.java             |   5 +-
 .../elasticsearch/test/ESIntegTestCase.java   |  20 +--
 .../test/InternalTestCluster.java             |   2 +-
 .../org/elasticsearch/test/VersionUtils.java  |   2 +-
 .../elasticsearch/test/client/NoOpClient.java |   4 +-
 .../BlockClusterStateProcessing.java          |   4 +-
 .../SlowClusterStateProcessing.java           |   2 +-
 .../hamcrest/ElasticsearchAssertions.java     |  35 +++--
 .../test/rest/spec/RestSpec.java              |   2 +-
 .../test/tasks/MockTaskManager.java           |  12 +-
 .../test/transport/MockTransportService.java  |  11 +-
 355 files changed, 2146 insertions(+), 2142 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/ElasticsearchException.java b/core/src/main/java/org/elasticsearch/ElasticsearchException.java
index aa92fc176cf..1a695490585 100644
--- a/core/src/main/java/org/elasticsearch/ElasticsearchException.java
+++ b/core/src/main/java/org/elasticsearch/ElasticsearchException.java
@@ -100,7 +100,7 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
     }
 
     public ElasticsearchException(StreamInput in) throws IOException {
-        super(in.readOptionalString(), in.readThrowable());
+        super(in.readOptionalString(), in.readException());
         readStackTrace(this, in);
         int numKeys = in.readVInt();
         for (int i = 0; i < numKeys; i++) {
@@ -162,7 +162,7 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
      * Unwraps the actual cause from the exception for cases when the exception is a
      * {@link ElasticsearchWrapperException}.
      *
-     * @see org.elasticsearch.ExceptionsHelper#unwrapCause(Throwable)
+     * @see ExceptionsHelper#unwrapCause(Throwable)
      */
     public Throwable unwrapCause() {
         return ExceptionsHelper.unwrapCause(this);
@@ -415,7 +415,7 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
 
         int numSuppressed = in.readVInt();
         for (int i = 0; i < numSuppressed; i++) {
-            throwable.addSuppressed(in.readThrowable());
+            throwable.addSuppressed(in.readException());
         }
         return throwable;
     }
@@ -794,9 +794,9 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
         return null;
     }
 
-    public static void renderThrowable(XContentBuilder builder, Params params, Throwable t) throws IOException {
+    public static void renderException(XContentBuilder builder, Params params, Exception e) throws IOException {
         builder.startObject("error");
-        final ElasticsearchException[] rootCauses = ElasticsearchException.guessRootCauses(t);
+        final ElasticsearchException[] rootCauses = ElasticsearchException.guessRootCauses(e);
         builder.field("root_cause");
         builder.startArray();
         for (ElasticsearchException rootCause : rootCauses) {
@@ -806,7 +806,7 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
             builder.endObject();
         }
         builder.endArray();
-        ElasticsearchException.toXContent(builder, params, t);
+        ElasticsearchException.toXContent(builder, params, e);
         builder.endObject();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/ElasticsearchSecurityException.java b/core/src/main/java/org/elasticsearch/ElasticsearchSecurityException.java
index f4878fe6f9b..b6cd420c856 100644
--- a/core/src/main/java/org/elasticsearch/ElasticsearchSecurityException.java
+++ b/core/src/main/java/org/elasticsearch/ElasticsearchSecurityException.java
@@ -36,7 +36,7 @@ public class ElasticsearchSecurityException extends ElasticsearchException {
         this.status = status ;
     }
 
-    public ElasticsearchSecurityException(String msg, Throwable cause, Object... args) {
+    public ElasticsearchSecurityException(String msg, Exception cause, Object... args) {
         this(msg, ExceptionsHelper.status(cause), cause, args);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/ExceptionsHelper.java b/core/src/main/java/org/elasticsearch/ExceptionsHelper.java
index 28f8d4391ec..e2af52ccd2c 100644
--- a/core/src/main/java/org/elasticsearch/ExceptionsHelper.java
+++ b/core/src/main/java/org/elasticsearch/ExceptionsHelper.java
@@ -37,25 +37,22 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-/**
- *
- */
 public final class ExceptionsHelper {
 
     private static final ESLogger logger = Loggers.getLogger(ExceptionsHelper.class);
 
-    public static RuntimeException convertToRuntime(Throwable t) {
-        if (t instanceof RuntimeException) {
-            return (RuntimeException) t;
+    public static RuntimeException convertToRuntime(Exception e) {
+        if (e instanceof RuntimeException) {
+            return (RuntimeException) e;
         }
-        return new ElasticsearchException(t);
+        return new ElasticsearchException(e);
     }
 
-    public static ElasticsearchException convertToElastic(Throwable t) {
-        if (t instanceof ElasticsearchException) {
-            return (ElasticsearchException) t;
+    public static ElasticsearchException convertToElastic(Exception e) {
+        if (e instanceof ElasticsearchException) {
+            return (ElasticsearchException) e;
         }
-        return new ElasticsearchException(t);
+        return new ElasticsearchException(e);
     }
 
     public static RestStatus status(Throwable t) {
@@ -164,8 +161,8 @@ public final class ExceptionsHelper {
     }
 
     public static IOException unwrapCorruption(Throwable t) {
-        return (IOException) unwrap(t, CorruptIndexException.class, 
-                                       IndexFormatTooOldException.class, 
+        return (IOException) unwrap(t, CorruptIndexException.class,
+                                       IndexFormatTooOldException.class,
                                        IndexFormatTooNewException.class);
     }
 
@@ -209,7 +206,6 @@ public final class ExceptionsHelper {
         return true;
     }
 
-
     /**
      * Deduplicate the failures by exception message and index.
      */
diff --git a/core/src/main/java/org/elasticsearch/action/ActionListener.java b/core/src/main/java/org/elasticsearch/action/ActionListener.java
index 8447d6cef08..e7d5ecd8d64 100644
--- a/core/src/main/java/org/elasticsearch/action/ActionListener.java
+++ b/core/src/main/java/org/elasticsearch/action/ActionListener.java
@@ -32,5 +32,5 @@ public interface ActionListener<Response> {
     /**
      * A failure caused by an exception at some phase of the task.
      */
-    void onFailure(Throwable e);
+    void onFailure(Exception e);
 }
diff --git a/core/src/main/java/org/elasticsearch/action/ActionRunnable.java b/core/src/main/java/org/elasticsearch/action/ActionRunnable.java
index 36c3f4f17fa..78e2249d6f4 100644
--- a/core/src/main/java/org/elasticsearch/action/ActionRunnable.java
+++ b/core/src/main/java/org/elasticsearch/action/ActionRunnable.java
@@ -22,11 +22,11 @@ package org.elasticsearch.action;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 
 /**
- * Base class for {@link Runnable}s that need to call {@link ActionListener#onFailure(Throwable)} in case an uncaught
+ * Base class for {@link Runnable}s that need to call {@link ActionListener#onFailure(Exception)} in case an uncaught
  * exception or error is thrown while the actual action is run.
  */
 public abstract class ActionRunnable<Response> extends AbstractRunnable {
-    
+
     protected final ActionListener<Response> listener;
 
     public ActionRunnable(ActionListener<Response> listener) {
@@ -34,11 +34,11 @@ public abstract class ActionRunnable<Response> extends AbstractRunnable {
     }
 
     /**
-     * Calls the action listeners {@link ActionListener#onFailure(Throwable)} method with the given exception.
+     * Calls the action listeners {@link ActionListener#onFailure(Exception)} method with the given exception.
      * This method is invoked for all exception thrown by {@link #doRun()}
      */
     @Override
-    public void onFailure(Throwable t) {
-        listener.onFailure(t);
+    public void onFailure(Exception e) {
+        listener.onFailure(e);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/LatchedActionListener.java b/core/src/main/java/org/elasticsearch/action/LatchedActionListener.java
index fb0fd81a7be..e5e0af93072 100644
--- a/core/src/main/java/org/elasticsearch/action/LatchedActionListener.java
+++ b/core/src/main/java/org/elasticsearch/action/LatchedActionListener.java
@@ -45,7 +45,7 @@ public class LatchedActionListener<T> implements ActionListener<T> {
     }
 
     @Override
-    public void onFailure(Throwable e) {
+    public void onFailure(Exception e) {
         try {
             delegate.onFailure(e);
         } finally {
diff --git a/core/src/main/java/org/elasticsearch/action/TaskOperationFailure.java b/core/src/main/java/org/elasticsearch/action/TaskOperationFailure.java
index 67436f31772..f14b0606c71 100644
--- a/core/src/main/java/org/elasticsearch/action/TaskOperationFailure.java
+++ b/core/src/main/java/org/elasticsearch/action/TaskOperationFailure.java
@@ -43,15 +43,15 @@ public final class TaskOperationFailure implements Writeable, ToXContent {
 
     private final long taskId;
 
-    private final Throwable reason;
+    private final Exception reason;
 
     private final RestStatus status;
 
-    public TaskOperationFailure(String nodeId, long taskId, Throwable t) {
+    public TaskOperationFailure(String nodeId, long taskId, Exception e) {
         this.nodeId = nodeId;
         this.taskId = taskId;
-        this.reason = t;
-        status = ExceptionsHelper.status(t);
+        this.reason = e;
+        status = ExceptionsHelper.status(e);
     }
 
     /**
@@ -60,7 +60,7 @@ public final class TaskOperationFailure implements Writeable, ToXContent {
     public TaskOperationFailure(StreamInput in) throws IOException {
         nodeId = in.readString();
         taskId = in.readLong();
-        reason = in.readThrowable();
+        reason = in.readException();
         status = RestStatus.readFrom(in);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
index 121dfe6f682..d63a7ff8968 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java
@@ -145,7 +145,7 @@ public class TransportClusterAllocationExplainAction
             // No copies of the data
             storeCopy = ClusterAllocationExplanation.StoreCopy.NONE;
         } else {
-            final Throwable storeErr = storeStatus.getStoreException();
+            final Exception storeErr = storeStatus.getStoreException();
             if (storeErr != null) {
                 if (ExceptionsHelper.unwrapCause(storeErr) instanceof CorruptIndexException) {
                     storeCopy = ClusterAllocationExplanation.StoreCopy.CORRUPT;
@@ -323,7 +323,7 @@ public class TransportClusterAllocationExplainAction
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 listener.onFailure(e);
             }
         });
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
index de616c61907..7d0a6288922 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthAction.java
@@ -104,9 +104,9 @@ public class TransportClusterHealthAction extends TransportMasterNodeReadAction<
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
-                    logger.error("unexpected failure during [{}]", t, source);
-                    listener.onFailure(t);
+                public void onFailure(String source, Exception e) {
+                    logger.error("unexpected failure during [{}]", e, source);
+                    listener.onFailure(e);
                 }
 
                 @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
index 477f646fe39..318ea031792 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/node/tasks/get/TransportGetTaskAction.java
@@ -154,8 +154,8 @@ public class TransportGetTaskAction extends HandledTransportAction<GetTaskReques
                     }
 
                     @Override
-                    public void onFailure(Throwable t) {
-                        listener.onFailure(t);
+                    public void onFailure(Exception e) {
+                        listener.onFailure(e);
                     }
                 });
             } else {
@@ -179,7 +179,7 @@ public class TransportGetTaskAction extends HandledTransportAction<GetTaskReques
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 /*
                  * We couldn't load the task from the task index. Instead of 404 we should use the snapshot we took after it finished. If
                  * the error isn't a 404 then we'll just throw it back to the user.
@@ -207,13 +207,13 @@ public class TransportGetTaskAction extends HandledTransportAction<GetTaskReques
             public void onResponse(GetResponse getResponse) {
                 try {
                     onGetFinishedTaskFromIndex(getResponse, listener);
-                } catch (Throwable e) {
+                } catch (Exception e) {
                     listener.onFailure(e);
                 }
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 if (ExceptionsHelper.unwrap(e, IndexNotFoundException.class) != null) {
                     // We haven't yet created the index for the task results so it can't be found.
                     listener.onFailure(new ResourceNotFoundException("task [{}] isn't running or persisted", e, request.getTaskId()));
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java
index a17d2aac892..6baf834278b 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java
@@ -76,7 +76,7 @@ public class TransportDeleteRepositoryAction extends TransportMasterNodeAction<D
                     }
 
                     @Override
-                    public void onFailure(Throwable e) {
+                    public void onFailure(Exception e) {
                         listener.onFailure(e);
                     }
                 });
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java
index d1639001352..2ac4950c327 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java
@@ -80,7 +80,7 @@ public class TransportPutRepositoryAction extends TransportMasterNodeAction<PutR
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 listener.onFailure(e);
             }
         });
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java
index 704cc511ccc..3bd2bef1274 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java
@@ -78,7 +78,7 @@ public class TransportVerifyRepositoryAction extends TransportMasterNodeAction<V
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 listener.onFailure(e);
             }
         });
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java
index b0b676f6e2e..875562ad64a 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java
@@ -102,9 +102,9 @@ public class TransportClusterRerouteAction extends TransportMasterNodeAction<Clu
         }
 
         @Override
-        public void onFailure(String source, Throwable t) {
-            logger.debug("failed to perform [{}]", t, source);
-            super.onFailure(source, t);
+        public void onFailure(String source, Exception e) {
+            logger.debug("failed to perform [{}]", e, source);
+            super.onFailure(source, e);
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
index 1abca63794b..4464b5d793f 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
@@ -93,11 +93,11 @@ public class TransportClusterUpdateSettingsAction extends TransportMasterNodeAct
             }
 
             @Override
-            public void onAllNodesAcked(@Nullable Throwable t) {
+            public void onAllNodesAcked(@Nullable Exception e) {
                 if (changed) {
                     reroute(true);
                 } else {
-                    super.onAllNodesAcked(t);
+                    super.onAllNodesAcked(e);
                 }
             }
 
@@ -146,10 +146,10 @@ public class TransportClusterUpdateSettingsAction extends TransportMasterNodeAct
                     }
 
                     @Override
-                    public void onFailure(String source, Throwable t) {
+                    public void onFailure(String source, Exception e) {
                         //if the reroute fails we only log
-                        logger.debug("failed to perform [{}]", t, source);
-                        listener.onFailure(new ElasticsearchException("reroute after update settings failed", t));
+                        logger.debug("failed to perform [{}]", e, source);
+                        listener.onFailure(new ElasticsearchException("reroute after update settings failed", e));
                     }
 
                     @Override
@@ -165,9 +165,9 @@ public class TransportClusterUpdateSettingsAction extends TransportMasterNodeAct
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
-                logger.debug("failed to perform [{}]", t, source);
-                super.onFailure(source, t);
+            public void onFailure(String source, Exception e) {
+                logger.debug("failed to perform [{}]", e, source);
+                super.onFailure(source, e);
             }
 
             @Override
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java
index 53b1bf86f66..269edfc401b 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java
@@ -94,10 +94,10 @@ public class TransportCreateSnapshotAction extends TransportMasterNodeAction<Cre
                         }
 
                         @Override
-                        public void onSnapshotFailure(Snapshot snapshot, Throwable t) {
+                        public void onSnapshotFailure(Snapshot snapshot, Exception e) {
                             if (snapshot.getRepository().equals(request.repository()) &&
                                     snapshot.getSnapshotId().getName().equals(request.snapshot())) {
-                                listener.onFailure(t);
+                                listener.onFailure(e);
                                 snapshotsService.removeListener(this);
                             }
                         }
@@ -108,8 +108,8 @@ public class TransportCreateSnapshotAction extends TransportMasterNodeAction<Cre
             }
 
             @Override
-            public void onFailure(Throwable t) {
-                listener.onFailure(t);
+            public void onFailure(Exception e) {
+                listener.onFailure(e);
             }
         });
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
index d2857dc9e1a..3c37d1870e5 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java
@@ -72,8 +72,8 @@ public class TransportDeleteSnapshotAction extends TransportMasterNodeAction<Del
             }
 
             @Override
-            public void onFailure(Throwable t) {
-                listener.onFailure(t);
+            public void onFailure(Exception e) {
+                listener.onFailure(e);
             }
         });
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
index f734201e7bd..ad8cb1ae88e 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
@@ -120,8 +120,8 @@ public class TransportGetSnapshotsAction extends TransportMasterNodeAction<GetSn
                 snapshotInfoBuilder.addAll(snapshotsService.snapshots(repository, new ArrayList<>(toResolve), request.ignoreUnavailable()));
             }
             listener.onResponse(new GetSnapshotsResponse(snapshotInfoBuilder));
-        } catch (Throwable t) {
-            listener.onFailure(t);
+        } catch (Exception e) {
+            listener.onFailure(e);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java
index a38fbce46c2..070db6c5248 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java
@@ -94,7 +94,7 @@ public class TransportRestoreSnapshotAction extends TransportMasterNodeAction<Re
                         }
 
                         @Override
-                        public void onFailure(Throwable e) {
+                        public void onFailure(Exception e) {
                             listener.onFailure(e);
                         }
                     });
@@ -104,7 +104,7 @@ public class TransportRestoreSnapshotAction extends TransportMasterNodeAction<Re
             }
 
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception t) {
                 listener.onFailure(t);
             }
         });
diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
index afbb2179c0a..dbcc093ec1b 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
@@ -125,13 +125,13 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction<Sn
                                 List<SnapshotsInProgress.Entry> currentSnapshots =
                                         snapshotsService.currentSnapshots(request.repository(), Arrays.asList(request.snapshots()));
                                 listener.onResponse(buildResponse(request, currentSnapshots, nodeSnapshotStatuses));
-                            } catch (Throwable e) {
+                            } catch (Exception e) {
                                 listener.onFailure(e);
                             }
                         }
 
                         @Override
-                        public void onFailure(Throwable e) {
+                        public void onFailure(Exception e) {
                             listener.onFailure(e);
                         }
                     });
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java
index 218b84e68ae..3ae7d7ebb7c 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/alias/TransportIndicesAliasesAction.java
@@ -119,7 +119,7 @@ public class TransportIndicesAliasesAction extends TransportMasterNodeAction<Ind
             }
 
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception t) {
                 logger.debug("failed to perform aliases", t);
                 listener.onFailure(t);
             }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java
index 4fbfc7e72ab..06810c4dcd1 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/close/TransportCloseIndexAction.java
@@ -107,7 +107,7 @@ public class TransportCloseIndexAction extends TransportMasterNodeAction<CloseIn
             }
 
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception t) {
                 logger.debug("failed to close indices [{}]", t, (Object)concreteIndices);
                 listener.onFailure(t);
             }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java
index 7b47a46a236..444da8df082 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java
@@ -87,7 +87,7 @@ public class TransportCreateIndexAction extends TransportMasterNodeAction<Create
             }
 
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception t) {
                 if (t instanceof IndexAlreadyExistsException) {
                     logger.trace("[{}] failed to create", t, request.index());
                 } else {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java
index 0ae8444ea7c..947936bddc7 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java
@@ -99,7 +99,7 @@ public class TransportDeleteIndexAction extends TransportMasterNodeAction<Delete
             }
 
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception t) {
                 logger.debug("failed to delete indices [{}]", t, concreteIndices);
                 listener.onFailure(t);
             }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java
index d9031807ae8..a69dd2ed437 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java
@@ -77,7 +77,7 @@ public class TransportGetFieldMappingsAction extends HandledTransportAction<GetF
                     }
 
                     @Override
-                    public void onFailure(Throwable e) {
+                    public void onFailure(Exception e) {
                         int index = indexCounter.getAndIncrement();
                         indexResponses.set(index, e);
                         if (completionCounter.decrementAndGet() == 0) {
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java
index 46535350154..a03472262e7 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java
@@ -91,7 +91,7 @@ public class TransportPutMappingAction extends TransportMasterNodeAction<PutMapp
                 }
 
                 @Override
-                public void onFailure(Throwable t) {
+                public void onFailure(Exception t) {
                     logger.debug("failed to put mappings on indices [{}], type [{}]", t, concreteIndices, request.type());
                     listener.onFailure(t);
                 }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java
index 50e79036694..d672d55b322 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/open/TransportOpenIndexAction.java
@@ -92,7 +92,7 @@ public class TransportOpenIndexAction extends TransportMasterNodeAction<OpenInde
             }
 
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception t) {
                 logger.debug("failed to open indices [{}]", t, (Object)concreteIndices);
                 listener.onFailure(t);
             }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
index 952448af5e7..af81e5cf8c9 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java
@@ -43,7 +43,6 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.shard.DocsStats;
-import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
@@ -132,14 +131,14 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
                                             }
 
                                             @Override
-                                            public void onFailure(Throwable e) {
+                                            public void onFailure(Exception e) {
                                                 listener.onFailure(e);
                                             }
                                         });
                                 }
 
                                 @Override
-                                public void onFailure(Throwable t) {
+                                public void onFailure(Exception t) {
                                     listener.onFailure(t);
                                 }
                             });
@@ -152,7 +151,7 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     listener.onFailure(e);
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java
index b20f636977a..5655400465f 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java
@@ -91,7 +91,7 @@ public class TransportUpdateSettingsAction extends TransportMasterNodeAction<Upd
             }
 
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception t) {
                 logger.debug("failed to update settings on indices [{}]", t, (Object)concreteIndices);
                 listener.onFailure(t);
             }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java
index a05090c5084..e6530c7baea 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java
@@ -57,7 +57,7 @@ public class IndicesShardStoresResponse extends ActionResponse implements ToXCon
         private DiscoveryNode node;
         private long legacyVersion;
         private String allocationId;
-        private Throwable storeException;
+        private Exception storeException;
         private AllocationStatus allocationStatus;
 
         /**
@@ -116,7 +116,7 @@ public class IndicesShardStoresResponse extends ActionResponse implements ToXCon
         private StoreStatus() {
         }
 
-        public StoreStatus(DiscoveryNode node, long legacyVersion, String allocationId, AllocationStatus allocationStatus, Throwable storeException) {
+        public StoreStatus(DiscoveryNode node, long legacyVersion, String allocationId, AllocationStatus allocationStatus, Exception storeException) {
             this.node = node;
             this.legacyVersion = legacyVersion;
             this.allocationId = allocationId;
@@ -150,7 +150,7 @@ public class IndicesShardStoresResponse extends ActionResponse implements ToXCon
          * Exception while trying to open the
          * shard index or from when the shard failed
          */
-        public Throwable getStoreException() {
+        public Exception getStoreException() {
             return storeException;
         }
 
@@ -177,7 +177,7 @@ public class IndicesShardStoresResponse extends ActionResponse implements ToXCon
             allocationId = in.readOptionalString();
             allocationStatus = AllocationStatus.readFrom(in);
             if (in.readBoolean()) {
-                storeException = in.readThrowable();
+                storeException = in.readException();
             }
         }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java
index a00dd23b764..57dc3e82e18 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java
@@ -100,7 +100,7 @@ public class TransportShrinkAction extends TransportMasterNodeAction<ShrinkReque
                     }
 
                     @Override
-                    public void onFailure(Throwable t) {
+                    public void onFailure(Exception t) {
                         if (t instanceof IndexAlreadyExistsException) {
                             logger.trace("[{}] failed to create shrink index", t, updateRequest.index());
                         } else {
@@ -112,7 +112,7 @@ public class TransportShrinkAction extends TransportMasterNodeAction<ShrinkReque
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 listener.onFailure(e);
             }
         });
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java
index 0763f232711..33addcb8440 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java
@@ -72,9 +72,9 @@ public class TransportDeleteIndexTemplateAction extends TransportMasterNodeActio
             }
 
             @Override
-            public void onFailure(Throwable t) {
-                logger.debug("failed to delete templates [{}]", t, request.name());
-                listener.onFailure(t);
+            public void onFailure(Exception e) {
+                logger.debug("failed to delete templates [{}]", e, request.name());
+                listener.onFailure(e);
             }
         });
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java
index b773efc2b55..0d14c4d24df 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java
@@ -93,9 +93,9 @@ public class TransportPutIndexTemplateAction extends TransportMasterNodeAction<P
                     }
 
                     @Override
-                    public void onFailure(Throwable t) {
-                        logger.debug("failed to put template [{}]", t, request.name());
-                        listener.onFailure(t);
+                    public void onFailure(Exception e) {
+                        logger.debug("failed to put template [{}]", e, request.name());
+                        listener.onFailure(e);
                     }
                 });
     }
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeAction.java
index cdf6f585e53..87f39336047 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeAction.java
@@ -190,13 +190,13 @@ public class TransportUpgradeAction extends TransportBroadcastByNodeAction<Upgra
                     } else {
                         updateSettings(upgradeResponse, listener);
                     }
-                } catch (Throwable t) {
-                    listener.onFailure(t);
+                } catch (Exception e) {
+                    listener.onFailure(e);
                 }
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 listener.onFailure(e);
             }
         };
@@ -212,7 +212,7 @@ public class TransportUpgradeAction extends TransportBroadcastByNodeAction<Upgra
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 listener.onFailure(e);
             }
         });
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java
index 403456cb903..3b77892086b 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java
@@ -78,7 +78,7 @@ public class TransportUpgradeSettingsAction extends TransportMasterNodeAction<Up
             }
 
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception t) {
                 logger.debug("failed to upgrade minimum compatibility version settings on indices [{}]", t, request.versions().keySet());
                 listener.onFailure(t);
             }
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java
index 512d2c505a9..1f875ee29d4 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java
@@ -91,12 +91,12 @@ public class BulkItemResponse implements Streamable, StatusToXContent {
         private final Throwable cause;
         private final RestStatus status;
 
-        public Failure(String index, String type, String id, Throwable t) {
+        public Failure(String index, String type, String id, Throwable cause) {
             this.index = index;
             this.type = type;
             this.id = id;
-            this.cause = t;
-            this.status = ExceptionsHelper.status(t);
+            this.cause = cause;
+            this.status = ExceptionsHelper.status(cause);
         }
 
         /**
@@ -106,7 +106,7 @@ public class BulkItemResponse implements Streamable, StatusToXContent {
             index = in.readString();
             type = in.readString();
             id = in.readOptionalString();
-            cause = in.readThrowable();
+            cause = in.readException();
             status = ExceptionsHelper.status(cause);
         }
 
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestHandler.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestHandler.java
index dc98a16c578..a829e4b0292 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestHandler.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestHandler.java
@@ -80,10 +80,10 @@ abstract class BulkRequestHandler {
                 if (!afterCalled) {
                     listener.afterBulk(executionId, bulkRequest, e);
                 }
-            } catch (Throwable t) {
-                logger.warn("Failed to execute bulk request {}.", t, executionId);
+            } catch (Exception e) {
+                logger.warn("Failed to execute bulk request {}.", e, executionId);
                 if (!afterCalled) {
-                    listener.afterBulk(executionId, bulkRequest, t);
+                    listener.afterBulk(executionId, bulkRequest, e);
                 }
             }
         }
@@ -131,7 +131,7 @@ abstract class BulkRequestHandler {
                             }
 
                             @Override
-                            public void onFailure(Throwable e) {
+                            public void onFailure(Exception e) {
                                 try {
                                     listener.afterBulk(executionId, bulkRequest, e);
                                 } finally {
@@ -144,9 +144,9 @@ abstract class BulkRequestHandler {
                 Thread.currentThread().interrupt();
                 logger.info("Bulk request {} has been cancelled.", e, executionId);
                 listener.afterBulk(executionId, bulkRequest, e);
-            } catch (Throwable t) {
-                logger.warn("Failed to execute bulk request {}.", t, executionId);
-                listener.afterBulk(executionId, bulkRequest, t);
+            } catch (Exception e) {
+                logger.warn("Failed to execute bulk request {}.", e, executionId);
+                listener.afterBulk(executionId, bulkRequest, e);
             } finally {
                 if (!bulkRequestSetupSuccessful && acquired) {  // if we fail on client.bulk() release the semaphore
                     semaphore.release();
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/Retry.java b/core/src/main/java/org/elasticsearch/action/bulk/Retry.java
index acaa784ac87..95778785ab9 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/Retry.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/Retry.java
@@ -130,7 +130,7 @@ public class Retry {
         }
 
         @Override
-        public void onFailure(Throwable e) {
+        public void onFailure(Exception e) {
             try {
                 listener.onFailure(e);
             } finally {
@@ -163,8 +163,8 @@ public class Retry {
             }
             for (BulkItemResponse bulkItemResponse : bulkItemResponses) {
                 if (bulkItemResponse.isFailed()) {
-                    Throwable cause = bulkItemResponse.getFailure().getCause();
-                    Throwable rootCause = ExceptionsHelper.unwrapCause(cause);
+                    final Throwable cause = bulkItemResponse.getFailure().getCause();
+                    final Throwable rootCause = ExceptionsHelper.unwrapCause(cause);
                     if (!rootCause.getClass().equals(retryOnThrowable)) {
                         return false;
                     }
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
index 022a427081a..f23c91be201 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
@@ -150,14 +150,14 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
                             if (counter.decrementAndGet() == 0) {
                                 try {
                                     executeBulk(task, bulkRequest, startTime, listener, responses);
-                                } catch (Throwable t) {
-                                    listener.onFailure(t);
+                                } catch (Exception e) {
+                                    listener.onFailure(e);
                                 }
                             }
                         }
 
                         @Override
-                        public void onFailure(Throwable e) {
+                        public void onFailure(Exception e) {
                             if (!(ExceptionsHelper.unwrapCause(e) instanceof IndexAlreadyExistsException)) {
                                 // fail all requests involving this index, if create didnt work
                                 for (int i = 0; i < bulkRequest.requests.size(); i++) {
@@ -170,8 +170,9 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
                             if (counter.decrementAndGet() == 0) {
                                 try {
                                     executeBulk(task, bulkRequest, startTime, listener, responses);
-                                } catch (Throwable t) {
-                                    listener.onFailure(t);
+                                } catch (Exception inner) {
+                                    inner.addSuppressed(e);
+                                    listener.onFailure(inner);
                                 }
                             }
                         }
@@ -195,7 +196,7 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
         return autoCreateIndex.shouldAutoCreate(index, state);
     }
 
-    private boolean setResponseFailureIfIndexMatches(AtomicArray<BulkItemResponse> responses, int idx, ActionRequest request, String index, Throwable e) {
+    private boolean setResponseFailureIfIndexMatches(AtomicArray<BulkItemResponse> responses, int idx, ActionRequest request, String index, Exception e) {
         if (request instanceof IndexRequest) {
             IndexRequest indexRequest = (IndexRequest) request;
             if (index.equals(indexRequest.index())) {
@@ -367,7 +368,7 @@ public class TransportBulkAction extends HandledTransportAction<BulkRequest, Bul
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     // create failures for all relevant requests
                     for (BulkItemRequest request : requests) {
                         final String indexName = concreteIndices.getConcreteIndex(request.index()).getName();
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
index a9ec11eec4d..45fc97dca98 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java
@@ -158,7 +158,7 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
             // add the response
             IndexResponse indexResponse = result.getResponse();
             setResponse(item, new BulkItemResponse(item.id(), indexRequest.opType().lowercase(), indexResponse));
-        } catch (Throwable e) {
+        } catch (Exception e) {
             // rethrow the failure if we are going to retry on primary and let parent failure to handle it
             if (retryPrimaryException(e)) {
                 // restore updated versions...
@@ -181,11 +181,11 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
         return location;
     }
 
-    private <ReplicationRequestT extends ReplicationRequest<ReplicationRequestT>> void logFailure(Throwable e, String operation, ShardId shardId, ReplicationRequest<ReplicationRequestT> request) {
-        if (ExceptionsHelper.status(e) == RestStatus.CONFLICT) {
-            logger.trace("{} failed to execute bulk item ({}) {}", e, shardId, operation, request);
+    private <ReplicationRequestT extends ReplicationRequest<ReplicationRequestT>> void logFailure(Throwable t, String operation, ShardId shardId, ReplicationRequest<ReplicationRequestT> request) {
+        if (ExceptionsHelper.status(t) == RestStatus.CONFLICT) {
+            logger.trace("{} failed to execute bulk item ({}) {}", t, shardId, operation, request);
         } else {
-            logger.debug("{} failed to execute bulk item ({}) {}", e, shardId, operation, request);
+            logger.debug("{} failed to execute bulk item ({}) {}", t, shardId, operation, request);
         }
     }
 
@@ -200,7 +200,7 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
             DeleteResponse deleteResponse = writeResult.getResponse();
             location = locationToSync(location, writeResult.getLocation());
             setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_DELETE, deleteResponse));
-        } catch (Throwable e) {
+        } catch (Exception e) {
             // rethrow the failure if we are going to retry on primary and let parent failure to handle it
             if (retryPrimaryException(e)) {
                 // restore updated versions...
@@ -232,7 +232,7 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
             UpdateResult updateResult;
             try {
                 updateResult = shardUpdateOperation(metaData, request, updateRequest, indexShard);
-            } catch (Throwable t) {
+            } catch (Exception t) {
                 updateResult = new UpdateResult(null, null, false, t, null);
             }
             if (updateResult.success()) {
@@ -275,43 +275,43 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
                 // NOTE: Breaking out of the retry_on_conflict loop!
                 break;
             } else if (updateResult.failure()) {
-                Throwable t = updateResult.error;
+                Throwable e = updateResult.error;
                 if (updateResult.retry) {
                     // updateAttemptCount is 0 based and marks current attempt, if it's equal to retryOnConflict we are going out of the iteration
                     if (updateAttemptsCount >= updateRequest.retryOnConflict()) {
                         setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE,
-                            new BulkItemResponse.Failure(request.index(), updateRequest.type(), updateRequest.id(), t)));
+                            new BulkItemResponse.Failure(request.index(), updateRequest.type(), updateRequest.id(), e)));
                     }
                 } else {
                     // rethrow the failure if we are going to retry on primary and let parent failure to handle it
-                    if (retryPrimaryException(t)) {
+                    if (retryPrimaryException(e)) {
                         // restore updated versions...
                         for (int j = 0; j < requestIndex; j++) {
                             applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]);
                         }
-                        throw (ElasticsearchException) t;
+                        throw (ElasticsearchException) e;
                     }
                     // if its a conflict failure, and we already executed the request on a primary (and we execute it
                     // again, due to primary relocation and only processing up to N bulk items when the shard gets closed)
                     // then just use the response we got from the successful execution
-                    if (item.getPrimaryResponse() != null && isConflictException(t)) {
+                    if (item.getPrimaryResponse() != null && isConflictException(e)) {
                         setResponse(item, item.getPrimaryResponse());
                     } else if (updateResult.result == null) {
-                        setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, new BulkItemResponse.Failure(request.index(), updateRequest.type(), updateRequest.id(), t)));
+                        setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, new BulkItemResponse.Failure(request.index(), updateRequest.type(), updateRequest.id(), e)));
                     } else {
                         switch (updateResult.result.operation()) {
                             case UPSERT:
                             case INDEX:
                                 IndexRequest indexRequest = updateResult.request();
-                                logFailure(t, "index", request.shardId(), indexRequest);
+                                logFailure(e, "index", request.shardId(), indexRequest);
                                 setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE,
-                                    new BulkItemResponse.Failure(request.index(), indexRequest.type(), indexRequest.id(), t)));
+                                    new BulkItemResponse.Failure(request.index(), indexRequest.type(), indexRequest.id(), e)));
                                 break;
                             case DELETE:
                                 DeleteRequest deleteRequest = updateResult.request();
-                                logFailure(t, "delete", request.shardId(), deleteRequest);
+                                logFailure(e, "delete", request.shardId(), deleteRequest);
                                 setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_DELETE,
-                                    new BulkItemResponse.Failure(request.index(), deleteRequest.type(), deleteRequest.id(), t)));
+                                    new BulkItemResponse.Failure(request.index(), deleteRequest.type(), deleteRequest.id(), e)));
                                 break;
                         }
                     }
@@ -335,7 +335,7 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
     }
 
     private WriteResult<IndexResponse> shardIndexOperation(BulkShardRequest request, IndexRequest indexRequest, IndexMetaData metaData,
-            IndexShard indexShard, boolean processed) throws Throwable {
+            IndexShard indexShard, boolean processed) throws Exception {
 
         MappingMetaData mappingMd = metaData.mappingOrDefault(indexRequest.type());
         if (!processed) {
@@ -406,26 +406,26 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
                 try {
                     WriteResult result = shardIndexOperation(bulkShardRequest, indexRequest, metaData, indexShard, false);
                     return new UpdateResult(translate, indexRequest, result);
-                } catch (Throwable t) {
-                    t = ExceptionsHelper.unwrapCause(t);
+                } catch (Exception e) {
+                    final Throwable cause = ExceptionsHelper.unwrapCause(e);
                     boolean retry = false;
-                    if (t instanceof VersionConflictEngineException) {
+                    if (cause instanceof VersionConflictEngineException) {
                         retry = true;
                     }
-                    return new UpdateResult(translate, indexRequest, retry, t, null);
+                    return new UpdateResult(translate, indexRequest, retry, cause, null);
                 }
             case DELETE:
                 DeleteRequest deleteRequest = translate.action();
                 try {
                     WriteResult<DeleteResponse> result = TransportDeleteAction.executeDeleteRequestOnPrimary(deleteRequest, indexShard);
                     return new UpdateResult(translate, deleteRequest, result);
-                } catch (Throwable t) {
-                    t = ExceptionsHelper.unwrapCause(t);
+                } catch (Exception e) {
+                    final Throwable cause = ExceptionsHelper.unwrapCause(e);
                     boolean retry = false;
-                    if (t instanceof VersionConflictEngineException) {
+                    if (cause instanceof VersionConflictEngineException) {
                         retry = true;
                     }
-                    return new UpdateResult(translate, deleteRequest, retry, t, null);
+                    return new UpdateResult(translate, deleteRequest, retry, cause, null);
                 }
             case NONE:
                 UpdateResponse updateResponse = translate.action();
@@ -449,7 +449,7 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
                 try {
                     Engine.Index operation = TransportIndexAction.executeIndexRequestOnReplica(indexRequest, indexShard);
                     location = locationToSync(location, operation.getTranslogLocation());
-                } catch (Throwable e) {
+                } catch (Exception e) {
                     // if its not an ignore replica failure, we need to make sure to bubble up the failure
                     // so we will fail the shard
                     if (!ignoreReplicaException(e)) {
@@ -462,7 +462,7 @@ public class TransportShardBulkAction extends TransportWriteAction<BulkShardRequ
                     Engine.Delete delete = TransportDeleteAction.executeDeleteRequestOnReplica(deleteRequest, indexShard);
                     indexShard.delete(delete);
                     location = locationToSync(location, delete.getTranslogLocation());
-                } catch (Throwable e) {
+                } catch (Exception e) {
                     // if its not an ignore replica failure, we need to make sure to bubble up the failure
                     // so we will fail the shard
                     if (!ignoreReplicaException(e)) {
diff --git a/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java b/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java
index beced23c338..4bf2bb47992 100644
--- a/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java
+++ b/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java
@@ -77,7 +77,7 @@ public class TransportDeleteAction extends TransportWriteAction<DeleteRequest, D
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     if (ExceptionsHelper.unwrapCause(e) instanceof IndexAlreadyExistsException) {
                         // we have the index, do it
                         innerExecute(task, request, listener);
diff --git a/core/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java b/core/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java
index 891f1cf6597..24069db645c 100644
--- a/core/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java
@@ -40,17 +40,17 @@ public class MultiGetResponse extends ActionResponse implements Iterable<MultiGe
         private String index;
         private String type;
         private String id;
-        private Throwable throwable;
+        private Exception exception;
 
         Failure() {
 
         }
 
-        public Failure(String index, String type, String id, Throwable throwable) {
+        public Failure(String index, String type, String id, Exception exception) {
             this.index = index;
             this.type = type;
             this.id = id;
-            this.throwable = throwable;
+            this.exception = exception;
         }
 
         /**
@@ -78,7 +78,7 @@ public class MultiGetResponse extends ActionResponse implements Iterable<MultiGe
          * The failure message.
          */
         public String getMessage() {
-            return throwable != null ? throwable.getMessage() : null;
+            return exception != null ? exception.getMessage() : null;
         }
 
         public static Failure readFailure(StreamInput in) throws IOException {
@@ -92,7 +92,7 @@ public class MultiGetResponse extends ActionResponse implements Iterable<MultiGe
             index = in.readString();
             type = in.readOptionalString();
             id = in.readString();
-            throwable = in.readThrowable();
+            exception = in.readException();
         }
 
         @Override
@@ -100,11 +100,11 @@ public class MultiGetResponse extends ActionResponse implements Iterable<MultiGe
             out.writeString(index);
             out.writeOptionalString(type);
             out.writeString(id);
-            out.writeThrowable(throwable);
+            out.writeThrowable(exception);
         }
 
-        public Throwable getFailure() {
-            return throwable;
+        public Exception getFailure() {
+            return exception;
         }
     }
 
@@ -136,7 +136,7 @@ public class MultiGetResponse extends ActionResponse implements Iterable<MultiGe
                 builder.field(Fields._INDEX, failure.getIndex());
                 builder.field(Fields._TYPE, failure.getType());
                 builder.field(Fields._ID, failure.getId());
-                ElasticsearchException.renderThrowable(builder, params, failure.getFailure());
+                ElasticsearchException.renderException(builder, params, failure.getFailure());
                 builder.endObject();
             } else {
                 GetResponse getResponse = response.getResponse();
diff --git a/core/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java b/core/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java
index f4037e40863..4e62030d329 100644
--- a/core/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java
+++ b/core/src/main/java/org/elasticsearch/action/get/TransportMultiGetAction.java
@@ -105,7 +105,7 @@ public class TransportMultiGetAction extends HandledTransportAction<MultiGetRequ
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     // create failures for all relevant requests
                     for (int i = 0; i < shardRequest.locations.size(); i++) {
                         MultiGetRequest.Item item = shardRequest.items.get(i);
diff --git a/core/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java b/core/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
index 5fbcda40983..d9e5eaa2e1b 100644
--- a/core/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
+++ b/core/src/main/java/org/elasticsearch/action/get/TransportShardMultiGetAction.java
@@ -88,12 +88,12 @@ public class TransportShardMultiGetAction extends TransportSingleShardAction<Mul
             try {
                 GetResult getResult = indexShard.getService().get(item.type(), item.id(), item.fields(), request.realtime(), item.version(), item.versionType(), item.fetchSourceContext(), request.ignoreErrorsOnGeneratedFields());
                 response.add(request.locations.get(i), new GetResponse(getResult));
-            } catch (Throwable t) {
-                if (TransportActions.isShardNotAvailableException(t)) {
-                    throw (ElasticsearchException) t;
+            } catch (Exception e) {
+                if (TransportActions.isShardNotAvailableException(e)) {
+                    throw (ElasticsearchException) e;
                 } else {
-                    logger.debug("{} failed to execute multi_get for [{}]/[{}]", t, shardId, item.type(), item.id());
-                    response.add(request.locations.get(i), new MultiGetResponse.Failure(request.index(), item.type(), item.id(), t));
+                    logger.debug("{} failed to execute multi_get for [{}]/[{}]", e, shardId, item.type(), item.id());
+                    response.add(request.locations.get(i), new MultiGetResponse.Failure(request.index(), item.type(), item.id(), e));
                 }
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java b/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
index 00be64757ae..fb42ef236d8 100644
--- a/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
+++ b/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java
@@ -101,13 +101,14 @@ public class TransportIndexAction extends TransportWriteAction<IndexRequest, Ind
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     if (ExceptionsHelper.unwrapCause(e) instanceof IndexAlreadyExistsException) {
                         // we have the index, do it
                         try {
                             innerExecute(task, request, listener);
-                        } catch (Throwable e1) {
-                            listener.onFailure(e1);
+                        } catch (Exception inner) {
+                            inner.addSuppressed(e);
+                            listener.onFailure(inner);
                         }
                     } else {
                         listener.onFailure(e);
@@ -196,7 +197,7 @@ public class TransportIndexAction extends TransportWriteAction<IndexRequest, Ind
 
         assert request.versionType().validateVersionForWrites(request.version());
 
-        IndexResponse response = new IndexResponse(shardId, request.type(), request.id(), request.version(), created); 
+        IndexResponse response = new IndexResponse(shardId, request.type(), request.id(), request.version(), created);
         return new WriteResult<>(response, operation.getTranslogLocation());
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/IngestActionFilter.java b/core/src/main/java/org/elasticsearch/action/ingest/IngestActionFilter.java
index 31c5ec43bd3..87d45f6ccd1 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/IngestActionFilter.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/IngestActionFilter.java
@@ -104,13 +104,13 @@ public final class IngestActionFilter extends AbstractComponent implements Actio
     void processBulkIndexRequest(Task task, BulkRequest original, String action, ActionFilterChain chain, ActionListener<BulkResponse> listener) {
         long ingestStartTimeInNanos = System.nanoTime();
         BulkRequestModifier bulkRequestModifier = new BulkRequestModifier(original);
-        executionService.executeBulkRequest(() -> bulkRequestModifier, (indexRequest, throwable) -> {
-            logger.debug("failed to execute pipeline [{}] for document [{}/{}/{}]", throwable, indexRequest.getPipeline(), indexRequest.index(), indexRequest.type(), indexRequest.id());
-            bulkRequestModifier.markCurrentItemAsFailed(throwable);
-        }, (throwable) -> {
-            if (throwable != null) {
-                logger.error("failed to execute pipeline for a bulk request", throwable);
-                listener.onFailure(throwable);
+        executionService.executeBulkRequest(() -> bulkRequestModifier, (indexRequest, exception) -> {
+            logger.debug("failed to execute pipeline [{}] for document [{}/{}/{}]", exception, indexRequest.getPipeline(), indexRequest.index(), indexRequest.type(), indexRequest.id());
+            bulkRequestModifier.markCurrentItemAsFailed(exception);
+        }, (exception) -> {
+            if (exception != null) {
+                logger.error("failed to execute pipeline for a bulk request", exception);
+                listener.onFailure(exception);
             } else {
                 long ingestTookInMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - ingestStartTimeInNanos);
                 BulkRequest bulkRequest = bulkRequestModifier.getBulkRequest();
@@ -188,7 +188,7 @@ public final class IngestActionFilter extends AbstractComponent implements Actio
                     }
 
                     @Override
-                    public void onFailure(Throwable e) {
+                    public void onFailure(Exception e) {
                         actionListener.onFailure(e);
                     }
                 };
@@ -197,7 +197,7 @@ public final class IngestActionFilter extends AbstractComponent implements Actio
             }
         }
 
-        void markCurrentItemAsFailed(Throwable e) {
+        void markCurrentItemAsFailed(Exception e) {
             IndexRequest indexRequest = (IndexRequest) bulkRequest.requests().get(currentSlot);
             // We hit a error during preprocessing a request, so we:
             // 1) Remember the request item slot from the bulk, so that we're done processing all requests we know what failed
@@ -233,7 +233,7 @@ public final class IngestActionFilter extends AbstractComponent implements Actio
         }
 
         @Override
-        public void onFailure(Throwable e) {
+        public void onFailure(Exception e) {
             actionListener.onFailure(e);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
index df92567b98d..82cd8d8eb7b 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/PutPipelineTransportAction.java
@@ -90,7 +90,7 @@ public class PutPipelineTransportAction extends TransportMasterNodeAction<PutPip
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 listener.onFailure(e);
             }
         });
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java
index 8521484370c..a7fb9237523 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java
@@ -49,7 +49,7 @@ public final class SimulateDocumentBaseResult implements SimulateDocumentResult
     public SimulateDocumentBaseResult(StreamInput in) throws IOException {
         if (in.readBoolean()) {
             ingestDocument = null;
-            failure = in.readThrowable();
+            failure = in.readException();
         } else {
             ingestDocument = new WriteableIngestDocument(in);
             failure = null;
@@ -84,7 +84,7 @@ public final class SimulateDocumentBaseResult implements SimulateDocumentResult
         if (failure == null) {
             ingestDocument.toXContent(builder, params);
         } else {
-            ElasticsearchException.renderThrowable(builder, params, failure);
+            ElasticsearchException.renderException(builder, params, failure);
         }
         builder.endObject();
         return builder;
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
index ce98767bcc8..8e4725abecc 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
@@ -52,7 +52,7 @@ public class SimulateProcessorResult implements Writeable, ToXContent {
     public SimulateProcessorResult(StreamInput in) throws IOException {
         this.processorTag = in.readString();
         if (in.readBoolean()) {
-            this.failure = in.readThrowable();
+            this.failure = in.readException();
             this.ingestDocument = null;
         } else {
             this.ingestDocument = new WriteableIngestDocument(in);
@@ -96,7 +96,7 @@ public class SimulateProcessorResult implements Writeable, ToXContent {
         if (failure == null) {
             ingestDocument.toXContent(builder, params);
         } else {
-            ElasticsearchException.renderThrowable(builder, params, failure);
+            ElasticsearchException.renderException(builder, params, failure);
         }
         builder.endObject();
         return builder;
diff --git a/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
index d843d8c0630..642748bd031 100644
--- a/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java
@@ -167,7 +167,7 @@ abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult>
                         }
 
                         @Override
-                        public void onFailure(Throwable t) {
+                        public void onFailure(Exception t) {
                             onFirstPhaseResult(shardIndex, shard, node.getId(), shardIt, t);
                         }
                     });
@@ -188,7 +188,7 @@ abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult>
         if (xTotalOps == expectedTotalOps) {
             try {
                 innerMoveToSecondPhase();
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 if (logger.isDebugEnabled()) {
                     logger.debug("{}: Failed to execute [{}] while moving to second phase", e, shardIt.shardId(), request);
                 }
@@ -201,33 +201,34 @@ abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult>
     }
 
     void onFirstPhaseResult(final int shardIndex, @Nullable ShardRouting shard, @Nullable String nodeId,
-                            final ShardIterator shardIt, Throwable t) {
+                            final ShardIterator shardIt, Exception e) {
         // we always add the shard failure for a specific shard instance
         // we do make sure to clean it on a successful response from a shard
         SearchShardTarget shardTarget = new SearchShardTarget(nodeId, shardIt.shardId().getIndex(), shardIt.shardId().getId());
-        addShardFailure(shardIndex, shardTarget, t);
+        addShardFailure(shardIndex, shardTarget, e);
 
         if (totalOps.incrementAndGet() == expectedTotalOps) {
             if (logger.isDebugEnabled()) {
-                if (t != null && !TransportActions.isShardNotAvailableException(t)) {
-                    logger.debug("{}: Failed to execute [{}]", t, shard != null ? shard.shortSummary() : shardIt.shardId(), request);
+                if (e != null && !TransportActions.isShardNotAvailableException(e)) {
+                    logger.debug("{}: Failed to execute [{}]", e, shard != null ? shard.shortSummary() : shardIt.shardId(), request);
                 } else if (logger.isTraceEnabled()) {
-                    logger.trace("{}: Failed to execute [{}]", t, shard, request);
+                    logger.trace("{}: Failed to execute [{}]", e, shard, request);
                 }
             }
             final ShardSearchFailure[] shardSearchFailures = buildShardFailures();
             if (successfulOps.get() == 0) {
                 if (logger.isDebugEnabled()) {
-                    logger.debug("All shards failed for phase: [{}]", t, firstPhaseName());
+                    logger.debug("All shards failed for phase: [{}]", e, firstPhaseName());
                 }
 
                 // no successful ops, raise an exception
-                raiseEarlyFailure(new SearchPhaseExecutionException(firstPhaseName(), "all shards failed", t, shardSearchFailures));
+                raiseEarlyFailure(new SearchPhaseExecutionException(firstPhaseName(), "all shards failed", e, shardSearchFailures));
             } else {
                 try {
                     innerMoveToSecondPhase();
-                } catch (Throwable e) {
-                    raiseEarlyFailure(new ReduceSearchPhaseException(firstPhaseName(), "", e, shardSearchFailures));
+                } catch (Exception inner) {
+                    inner.addSuppressed(e);
+                    raiseEarlyFailure(new ReduceSearchPhaseException(firstPhaseName(), "", inner, shardSearchFailures));
                 }
             }
         } else {
@@ -235,20 +236,21 @@ abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult>
             final boolean lastShard = nextShard == null;
             // trace log this exception
             if (logger.isTraceEnabled()) {
-                logger.trace("{}: Failed to execute [{}] lastShard [{}]", t, shard != null ? shard.shortSummary() : shardIt.shardId(),
+                logger.trace("{}: Failed to execute [{}] lastShard [{}]", e, shard != null ? shard.shortSummary() : shardIt.shardId(),
                     request, lastShard);
             }
             if (!lastShard) {
                 try {
                     performFirstPhase(shardIndex, shardIt, nextShard);
-                } catch (Throwable t1) {
-                    onFirstPhaseResult(shardIndex, shard, shard.currentNodeId(), shardIt, t1);
+                } catch (Exception inner) {
+                    inner.addSuppressed(e);
+                    onFirstPhaseResult(shardIndex, shard, shard.currentNodeId(), shardIt, inner);
                 }
             } else {
                 // no more shards active, add a failure
                 if (logger.isDebugEnabled() && !logger.isTraceEnabled()) { // do not double log this exception
-                    if (t != null && !TransportActions.isShardNotAvailableException(t)) {
-                        logger.debug("{}: Failed to execute [{}] lastShard [{}]", t,
+                    if (e != null && !TransportActions.isShardNotAvailableException(e)) {
+                        logger.debug("{}: Failed to execute [{}] lastShard [{}]", e,
                             shard != null ? shard.shortSummary() : shardIt.shardId(), request, lastShard);
                     }
                 }
@@ -269,9 +271,9 @@ abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult>
         return failures;
     }
 
-    protected final void addShardFailure(final int shardIndex, @Nullable SearchShardTarget shardTarget, Throwable t) {
+    protected final void addShardFailure(final int shardIndex, @Nullable SearchShardTarget shardTarget, Exception e) {
         // we don't aggregate shard failures on non active shards (but do keep the header counts right)
-        if (TransportActions.isShardNotAvailableException(t)) {
+        if (TransportActions.isShardNotAvailableException(e)) {
             return;
         }
 
@@ -285,26 +287,27 @@ abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult>
         }
         ShardSearchFailure failure = shardFailures.get(shardIndex);
         if (failure == null) {
-            shardFailures.set(shardIndex, new ShardSearchFailure(t, shardTarget));
+            shardFailures.set(shardIndex, new ShardSearchFailure(e, shardTarget));
         } else {
             // the failure is already present, try and not override it with an exception that is less meaningless
             // for example, getting illegal shard state
-            if (TransportActions.isReadOverrideException(t)) {
-                shardFailures.set(shardIndex, new ShardSearchFailure(t, shardTarget));
+            if (TransportActions.isReadOverrideException(e)) {
+                shardFailures.set(shardIndex, new ShardSearchFailure(e, shardTarget));
             }
         }
     }
 
-    private void raiseEarlyFailure(Throwable t) {
+    private void raiseEarlyFailure(Exception e) {
         for (AtomicArray.Entry<FirstResult> entry : firstResults.asList()) {
             try {
                 DiscoveryNode node = nodes.get(entry.value.shardTarget().nodeId());
                 sendReleaseSearchContext(entry.value.id(), node);
-            } catch (Throwable t1) {
-                logger.trace("failed to release context", t1);
+            } catch (Exception inner) {
+                inner.addSuppressed(e);
+                logger.trace("failed to release context", inner);
             }
         }
-        listener.onFailure(t);
+        listener.onFailure(e);
     }
 
     /**
@@ -324,8 +327,8 @@ abstract class AbstractSearchAsyncAction<FirstResult extends SearchPhaseResult>
                     try {
                         DiscoveryNode node = nodes.get(entry.value.queryResult().shardTarget().nodeId());
                         sendReleaseSearchContext(entry.value.queryResult().id(), node);
-                    } catch (Throwable t1) {
-                        logger.trace("failed to release context", t1);
+                    } catch (Exception e) {
+                        logger.trace("failed to release context", e);
                     }
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java b/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java
index bc23c11ceac..f1b84812019 100644
--- a/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java
@@ -29,7 +29,6 @@ import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -45,22 +44,22 @@ public class MultiSearchResponse extends ActionResponse implements Iterable<Mult
      */
     public static class Item implements Streamable {
         private SearchResponse response;
-        private Throwable throwable;
+        private Exception exception;
 
         Item() {
 
         }
 
-        public Item(SearchResponse response, Throwable throwable) {
+        public Item(SearchResponse response, Exception exception) {
             this.response = response;
-            this.throwable = throwable;
+            this.exception = exception;
         }
 
         /**
          * Is it a failed search?
          */
         public boolean isFailure() {
-            return throwable != null;
+            return exception != null;
         }
 
         /**
@@ -68,7 +67,7 @@ public class MultiSearchResponse extends ActionResponse implements Iterable<Mult
          */
         @Nullable
         public String getFailureMessage() {
-            return throwable == null ? null : throwable.getMessage();
+            return exception == null ? null : exception.getMessage();
         }
 
         /**
@@ -91,7 +90,7 @@ public class MultiSearchResponse extends ActionResponse implements Iterable<Mult
                 this.response = new SearchResponse();
                 response.readFrom(in);
             } else {
-                throwable = in.readThrowable();
+                exception = in.readException();
             }
         }
 
@@ -102,12 +101,12 @@ public class MultiSearchResponse extends ActionResponse implements Iterable<Mult
                 response.writeTo(out);
             } else {
                 out.writeBoolean(false);
-                out.writeThrowable(throwable);
+                out.writeThrowable(exception);
             }
         }
 
-        public Throwable getFailure() {
-            return throwable;
+        public Exception getFailure() {
+            return exception;
         }
     }
 
@@ -156,7 +155,7 @@ public class MultiSearchResponse extends ActionResponse implements Iterable<Mult
         for (Item item : items) {
             builder.startObject();
             if (item.isFailure()) {
-                ElasticsearchException.renderThrowable(builder, params, item.getFailure());
+                ElasticsearchException.renderException(builder, params, item.getFailure());
                 builder.field(Fields.STATUS, ExceptionsHelper.status(item.getFailure()).getStatus());
             } else {
                 item.getResponse().toXContent(builder, params);
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryAndFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryAndFetchAsyncAction.java
index 7cf608559ce..e19540e26d5 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryAndFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryAndFetchAsyncAction.java
@@ -89,7 +89,7 @@ class SearchDfsQueryAndFetchAsyncAction extends AbstractSearchAsyncAction<DfsSea
             }
 
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception t) {
                 try {
                     onSecondPhaseFailure(t, querySearchRequest, shardIndex, dfsResult, counter);
                 } finally {
@@ -102,12 +102,12 @@ class SearchDfsQueryAndFetchAsyncAction extends AbstractSearchAsyncAction<DfsSea
         });
     }
 
-    void onSecondPhaseFailure(Throwable t, QuerySearchRequest querySearchRequest, int shardIndex, DfsSearchResult dfsResult,
+    void onSecondPhaseFailure(Exception e, QuerySearchRequest querySearchRequest, int shardIndex, DfsSearchResult dfsResult,
                               AtomicInteger counter) {
         if (logger.isDebugEnabled()) {
-            logger.debug("[{}] Failed to execute query phase", t, querySearchRequest.id());
+            logger.debug("[{}] Failed to execute query phase", e, querySearchRequest.id());
         }
-        this.addShardFailure(shardIndex, dfsResult.shardTarget(), t);
+        this.addShardFailure(shardIndex, dfsResult.shardTarget(), e);
         successfulOps.decrementAndGet();
         if (counter.decrementAndGet() == 0) {
             finishHim();
@@ -130,12 +130,12 @@ class SearchDfsQueryAndFetchAsyncAction extends AbstractSearchAsyncAction<DfsSea
             }
 
             @Override
-            public void onFailure(Throwable t) {
-                ReduceSearchPhaseException failure = new ReduceSearchPhaseException("query_fetch", "", t, buildShardFailures());
+            public void onFailure(Exception e) {
+                ReduceSearchPhaseException failure = new ReduceSearchPhaseException("query_fetch", "", e, buildShardFailures());
                 if (logger.isDebugEnabled()) {
                     logger.debug("failed to reduce search", failure);
                 }
-                super.onFailure(t);
+                super.onFailure(e);
             }
         });
 
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
index 954630b9295..cf3f9716710 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java
@@ -97,7 +97,7 @@ class SearchDfsQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<DfsSe
             }
 
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception t) {
                 try {
                     onQueryFailure(t, querySearchRequest, shardIndex, dfsResult, counter);
                 } finally {
@@ -110,12 +110,12 @@ class SearchDfsQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<DfsSe
         });
     }
 
-    void onQueryFailure(Throwable t, QuerySearchRequest querySearchRequest, int shardIndex, DfsSearchResult dfsResult,
+    void onQueryFailure(Exception e, QuerySearchRequest querySearchRequest, int shardIndex, DfsSearchResult dfsResult,
                         AtomicInteger counter) {
         if (logger.isDebugEnabled()) {
-            logger.debug("[{}] Failed to execute query phase", t, querySearchRequest.id());
+            logger.debug("[{}] Failed to execute query phase", e, querySearchRequest.id());
         }
-        this.addShardFailure(shardIndex, dfsResult.shardTarget(), t);
+        this.addShardFailure(shardIndex, dfsResult.shardTarget(), e);
         successfulOps.decrementAndGet();
         if (counter.decrementAndGet() == 0) {
             if (successfulOps.get() == 0) {
@@ -129,7 +129,7 @@ class SearchDfsQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<DfsSe
     void executeFetchPhase() {
         try {
             innerExecuteFetchPhase();
-        } catch (Throwable e) {
+        } catch (Exception e) {
             listener.onFailure(new ReduceSearchPhaseException("query", "", e, buildShardFailures()));
         }
     }
@@ -169,7 +169,7 @@ class SearchDfsQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<DfsSe
             }
 
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception t) {
                 // the search context might not be cleared on the node where the fetch was executed for example
                 // because the action was rejected by the thread pool. in this case we need to send a dedicated
                 // request to clear the search context. by setting docIdsToLoad to null, the context will be cleared
@@ -180,12 +180,12 @@ class SearchDfsQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<DfsSe
         });
     }
 
-    void onFetchFailure(Throwable t, ShardFetchSearchRequest fetchSearchRequest, int shardIndex,
+    void onFetchFailure(Exception e, ShardFetchSearchRequest fetchSearchRequest, int shardIndex,
                         SearchShardTarget shardTarget, AtomicInteger counter) {
         if (logger.isDebugEnabled()) {
-            logger.debug("[{}] Failed to execute fetch phase", t, fetchSearchRequest.id());
+            logger.debug("[{}] Failed to execute fetch phase", e, fetchSearchRequest.id());
         }
-        this.addShardFailure(shardIndex, shardTarget, t);
+        this.addShardFailure(shardIndex, shardTarget, e);
         successfulOps.decrementAndGet();
         if (counter.decrementAndGet() == 0) {
             finishHim();
@@ -208,9 +208,9 @@ class SearchDfsQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<DfsSe
             }
 
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception e) {
                 try {
-                    ReduceSearchPhaseException failure = new ReduceSearchPhaseException("merge", "", t, buildShardFailures());
+                    ReduceSearchPhaseException failure = new ReduceSearchPhaseException("merge", "", e, buildShardFailures());
                     if (logger.isDebugEnabled()) {
                         logger.debug("failed to reduce search", failure);
                     }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java b/core/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java
index ebf3144a540..3aaa6442451 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java
@@ -32,9 +32,6 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
-/**
- *
- */
 public class SearchPhaseExecutionException extends ElasticsearchException {
     private final String phaseName;
     private final ShardSearchFailure[] shardFailures;
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchQueryAndFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchQueryAndFetchAsyncAction.java
index ee955f2c894..5d55dd468a5 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchQueryAndFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchQueryAndFetchAsyncAction.java
@@ -73,8 +73,8 @@ class SearchQueryAndFetchAsyncAction extends AbstractSearchAsyncAction<QueryFetc
             }
 
             @Override
-            public void onFailure(Throwable t) {
-                ReduceSearchPhaseException failure = new ReduceSearchPhaseException("merge", "", t, buildShardFailures());
+            public void onFailure(Exception e) {
+                ReduceSearchPhaseException failure = new ReduceSearchPhaseException("merge", "", e, buildShardFailures());
                 if (logger.isDebugEnabled()) {
                     logger.debug("failed to reduce search", failure);
                 }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
index 59a6f790e2c..a6f9aa26f59 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java
@@ -102,7 +102,7 @@ class SearchQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<QuerySea
             }
 
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception t) {
                 // the search context might not be cleared on the node where the fetch was executed for example
                 // because the action was rejected by the thread pool. in this case we need to send a dedicated
                 // request to clear the search context. by setting docIdsToLoad to null, the context will be cleared
@@ -113,12 +113,12 @@ class SearchQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<QuerySea
         });
     }
 
-    void onFetchFailure(Throwable t, ShardFetchSearchRequest fetchSearchRequest, int shardIndex, SearchShardTarget shardTarget,
+    void onFetchFailure(Exception e, ShardFetchSearchRequest fetchSearchRequest, int shardIndex, SearchShardTarget shardTarget,
                         AtomicInteger counter) {
         if (logger.isDebugEnabled()) {
-            logger.debug("[{}] Failed to execute fetch phase", t, fetchSearchRequest.id());
+            logger.debug("[{}] Failed to execute fetch phase", e, fetchSearchRequest.id());
         }
-        this.addShardFailure(shardIndex, shardTarget, t);
+        this.addShardFailure(shardIndex, shardTarget, e);
         successfulOps.decrementAndGet();
         if (counter.decrementAndGet() == 0) {
             finishHim();
@@ -141,9 +141,9 @@ class SearchQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<QuerySea
             }
 
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception e) {
                 try {
-                    ReduceSearchPhaseException failure = new ReduceSearchPhaseException("fetch", "", t, buildShardFailures());
+                    ReduceSearchPhaseException failure = new ReduceSearchPhaseException("fetch", "", e, buildShardFailures());
                     if (logger.isDebugEnabled()) {
                         logger.debug("failed to reduce search", failure);
                     }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java
index 90bb0937327..94ce1887c34 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryAndFetchAsyncAction.java
@@ -138,21 +138,21 @@ class SearchScrollQueryAndFetchAsyncAction extends AbstractAsyncAction {
             }
 
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception t) {
                 onPhaseFailure(t, searchId, shardIndex);
             }
         });
     }
 
-    private void onPhaseFailure(Throwable t, long searchId, int shardIndex) {
+    private void onPhaseFailure(Exception e, long searchId, int shardIndex) {
         if (logger.isDebugEnabled()) {
-            logger.debug("[{}] Failed to execute query phase", t, searchId);
+            logger.debug("[{}] Failed to execute query phase", e, searchId);
         }
-        addShardFailure(shardIndex, new ShardSearchFailure(t));
+        addShardFailure(shardIndex, new ShardSearchFailure(e));
         successfulOps.decrementAndGet();
         if (counter.decrementAndGet() == 0) {
             if (successfulOps.get() == 0) {
-                listener.onFailure(new SearchPhaseExecutionException("query_fetch", "all shards failed", t, buildShardFailures()));
+                listener.onFailure(new SearchPhaseExecutionException("query_fetch", "all shards failed", e, buildShardFailures()));
             } else {
                 finishHim();
             }
@@ -162,7 +162,7 @@ class SearchScrollQueryAndFetchAsyncAction extends AbstractAsyncAction {
     private void finishHim() {
         try {
             innerFinishHim();
-        } catch (Throwable e) {
+        } catch (Exception e) {
             listener.onFailure(new ReduceSearchPhaseException("fetch", "", e, buildShardFailures()));
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java b/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
index 694ee16beac..ac8715eeb9f 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchScrollQueryThenFetchAsyncAction.java
@@ -113,7 +113,7 @@ class SearchScrollQueryThenFetchAsyncAction extends AbstractAsyncAction {
                 if (counter.decrementAndGet() == 0) {
                     try {
                         executeFetchPhase();
-                    } catch (Throwable e) {
+                    } catch (Exception e) {
                         listener.onFailure(new SearchPhaseExecutionException("query", "Fetch failed", e, ShardSearchFailure.EMPTY_ARRAY));
                         return;
                     }
@@ -131,32 +131,33 @@ class SearchScrollQueryThenFetchAsyncAction extends AbstractAsyncAction {
                 if (counter.decrementAndGet() == 0) {
                     try {
                         executeFetchPhase();
-                    } catch (Throwable e) {
+                    } catch (Exception e) {
                         onFailure(e);
                     }
                 }
             }
 
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception t) {
                 onQueryPhaseFailure(shardIndex, counter, searchId, t);
             }
         });
     }
 
-    void onQueryPhaseFailure(final int shardIndex, final AtomicInteger counter, final long searchId, Throwable t) {
+    void onQueryPhaseFailure(final int shardIndex, final AtomicInteger counter, final long searchId, Exception failure) {
         if (logger.isDebugEnabled()) {
-            logger.debug("[{}] Failed to execute query phase", t, searchId);
+            logger.debug("[{}] Failed to execute query phase", failure, searchId);
         }
-        addShardFailure(shardIndex, new ShardSearchFailure(t));
+        addShardFailure(shardIndex, new ShardSearchFailure(failure));
         successfulOps.decrementAndGet();
         if (counter.decrementAndGet() == 0) {
             if (successfulOps.get() == 0) {
-                listener.onFailure(new SearchPhaseExecutionException("query", "all shards failed", t, buildShardFailures()));
+                listener.onFailure(new SearchPhaseExecutionException("query", "all shards failed", failure, buildShardFailures()));
             } else {
                 try {
                     executeFetchPhase();
-                } catch (Throwable e) {
+                } catch (Exception e) {
+                    e.addSuppressed(failure);
                     listener.onFailure(new SearchPhaseExecutionException("query", "Fetch failed", e, ShardSearchFailure.EMPTY_ARRAY));
                 }
             }
@@ -193,7 +194,7 @@ class SearchScrollQueryThenFetchAsyncAction extends AbstractAsyncAction {
                 }
 
                 @Override
-                public void onFailure(Throwable t) {
+                public void onFailure(Exception t) {
                     if (logger.isDebugEnabled()) {
                         logger.debug("Failed to execute fetch phase", t);
                     }
@@ -209,8 +210,8 @@ class SearchScrollQueryThenFetchAsyncAction extends AbstractAsyncAction {
     private void finishHim() {
         try {
             innerFinishHim();
-        } catch (Throwable e) {
-            listener.onFailure(new ReduceSearchPhaseException("fetch", "", e, buildShardFailures()));
+        } catch (Exception e) {
+            listener.onFailure(new ReduceSearchPhaseException("fetch", "inner finish failed", e, buildShardFailures()));
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java b/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java
index 2a01eb4e1c6..6c24234acef 100644
--- a/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java
+++ b/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java
@@ -48,19 +48,19 @@ public class ShardSearchFailure implements ShardOperationFailedException {
 
     }
 
-    public ShardSearchFailure(Throwable t) {
-        this(t, null);
+    public ShardSearchFailure(Exception e) {
+        this(e, null);
     }
 
-    public ShardSearchFailure(Throwable t, @Nullable SearchShardTarget shardTarget) {
-        Throwable actual = ExceptionsHelper.unwrapCause(t);
+    public ShardSearchFailure(Exception e, @Nullable SearchShardTarget shardTarget) {
+        final Throwable actual = ExceptionsHelper.unwrapCause(e);
         if (actual != null && actual instanceof SearchException) {
             this.shardTarget = ((SearchException) actual).shard();
         } else if (shardTarget != null) {
             this.shardTarget = shardTarget;
         }
         status = ExceptionsHelper.status(actual);
-        this.reason = ExceptionsHelper.detailedMessage(t);
+        this.reason = ExceptionsHelper.detailedMessage(e);
         this.cause = actual;
     }
 
@@ -135,7 +135,7 @@ public class ShardSearchFailure implements ShardOperationFailedException {
         }
         reason = in.readString();
         status = RestStatus.readFrom(in);
-        cause = in.readThrowable();
+        cause = in.readException();
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java b/core/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java
index c2922bf6deb..092b69fc936 100644
--- a/core/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/TransportClearScrollAction.java
@@ -103,7 +103,7 @@ public class TransportClearScrollAction extends HandledTransportAction<ClearScro
                         }
 
                         @Override
-                        public void onFailure(Throwable e) {
+                        public void onFailure(Exception e) {
                             onFailedFreedContext(e, node);
                         }
                     });
@@ -124,7 +124,7 @@ public class TransportClearScrollAction extends HandledTransportAction<ClearScro
                             }
 
                             @Override
-                            public void onFailure(Throwable e) {
+                            public void onFailure(Exception e) {
                                 onFailedFreedContext(e, node);
                             }
                         });
diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java b/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
index 4d2d8695c84..efd04035276 100644
--- a/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/TransportMultiSearchAction.java
@@ -30,7 +30,6 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AtomicArray;
-import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -119,7 +118,7 @@ public class TransportMultiSearchAction extends HandledTransportAction<MultiSear
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 responses.set(request.responseSlot, new MultiSearchResponse.Item(null, e));
                 handleResponse();
             }
diff --git a/core/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java b/core/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java
index 3bcadda1725..485baaa022b 100644
--- a/core/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java
+++ b/core/src/main/java/org/elasticsearch/action/search/TransportSearchScrollAction.java
@@ -74,7 +74,7 @@ public class TransportSearchScrollAction extends HandledTransportAction<SearchSc
                     throw new IllegalArgumentException("Scroll id type [" + scrollId.getType() + "] unrecognized");
             }
             action.start();
-        } catch (Throwable e) {
+        } catch (Exception e) {
             listener.onFailure(e);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/action/support/AbstractListenableActionFuture.java b/core/src/main/java/org/elasticsearch/action/support/AbstractListenableActionFuture.java
index faaf7183467..78a61ebfe16 100644
--- a/core/src/main/java/org/elasticsearch/action/support/AbstractListenableActionFuture.java
+++ b/core/src/main/java/org/elasticsearch/action/support/AbstractListenableActionFuture.java
@@ -102,7 +102,7 @@ public abstract class AbstractListenableActionFuture<T, L> extends AdapterAction
             // we use a timeout of 0 to by pass assertion forbidding to call actionGet() (blocking) on a network thread.
             // here we know we will never block
             listener.onResponse(actionGet(0));
-        } catch (Throwable e) {
+        } catch (Exception e) {
             listener.onFailure(e);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/action/support/AdapterActionFuture.java b/core/src/main/java/org/elasticsearch/action/support/AdapterActionFuture.java
index ec2db7a22b2..eab486f4929 100644
--- a/core/src/main/java/org/elasticsearch/action/support/AdapterActionFuture.java
+++ b/core/src/main/java/org/elasticsearch/action/support/AdapterActionFuture.java
@@ -98,7 +98,7 @@ public abstract class AdapterActionFuture<T, L> extends BaseFuture<T> implements
     }
 
     @Override
-    public void onFailure(Throwable e) {
+    public void onFailure(Exception e) {
         setException(e);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java b/core/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java
index 676949e367d..e584b2d8037 100644
--- a/core/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java
+++ b/core/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java
@@ -54,11 +54,11 @@ public class DefaultShardOperationFailedException implements ShardOperationFaile
         this.status = e.status();
     }
 
-    public DefaultShardOperationFailedException(String index, int shardId, Throwable t) {
+    public DefaultShardOperationFailedException(String index, int shardId, Throwable reason) {
         this.index = index;
         this.shardId = shardId;
-        this.reason = t;
-        status = ExceptionsHelper.status(t);
+        this.reason = reason;
+        this.status = ExceptionsHelper.status(reason);
     }
 
     @Override
@@ -98,7 +98,7 @@ public class DefaultShardOperationFailedException implements ShardOperationFaile
             index = in.readString();
         }
         shardId = in.readVInt();
-        reason = in.readThrowable();
+        reason = in.readException();
         status = RestStatus.readFrom(in);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/support/DelegatingActionListener.java b/core/src/main/java/org/elasticsearch/action/support/DelegatingActionListener.java
index 9e7c2205270..c02de8410cf 100644
--- a/core/src/main/java/org/elasticsearch/action/support/DelegatingActionListener.java
+++ b/core/src/main/java/org/elasticsearch/action/support/DelegatingActionListener.java
@@ -41,7 +41,7 @@ public abstract class DelegatingActionListener<Instigator extends ActionResponse
     }
 
     @Override
-    public final void onFailure(Throwable e) {
+    public final void onFailure(Exception e) {
         delegatedActionListener.onFailure(e);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java b/core/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java
index 997ed69f3fa..752ae41cee1 100644
--- a/core/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/HandledTransportAction.java
@@ -65,13 +65,13 @@ public abstract class HandledTransportAction<Request extends ActionRequest<Reque
                 public void onResponse(Response response) {
                     try {
                         channel.sendResponse(response);
-                    } catch (Throwable e) {
+                    } catch (Exception e) {
                         onFailure(e);
                     }
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     try {
                         channel.sendResponse(e);
                     } catch (Exception e1) {
diff --git a/core/src/main/java/org/elasticsearch/action/support/PlainListenableActionFuture.java b/core/src/main/java/org/elasticsearch/action/support/PlainListenableActionFuture.java
index 1ec30606312..c9b0cf9d82f 100644
--- a/core/src/main/java/org/elasticsearch/action/support/PlainListenableActionFuture.java
+++ b/core/src/main/java/org/elasticsearch/action/support/PlainListenableActionFuture.java
@@ -21,9 +21,6 @@ package org.elasticsearch.action.support;
 
 import org.elasticsearch.threadpool.ThreadPool;
 
-/**
- *
- */
 public class PlainListenableActionFuture<T> extends AbstractListenableActionFuture<T, T> {
 
     public PlainListenableActionFuture(ThreadPool threadPool) {
@@ -34,4 +31,5 @@ public class PlainListenableActionFuture<T> extends AbstractListenableActionFutu
     protected T convert(T response) {
         return response;
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/ThreadedActionListener.java b/core/src/main/java/org/elasticsearch/action/support/ThreadedActionListener.java
index e7ab536387e..4ff7cdaa7bd 100644
--- a/core/src/main/java/org/elasticsearch/action/support/ThreadedActionListener.java
+++ b/core/src/main/java/org/elasticsearch/action/support/ThreadedActionListener.java
@@ -97,14 +97,14 @@ public final class ThreadedActionListener<Response> implements ActionListener<Re
             }
 
             @Override
-            public void onFailure(Throwable t) {
-                listener.onFailure(t);
+            public void onFailure(Exception e) {
+                listener.onFailure(e);
             }
         });
     }
 
     @Override
-    public void onFailure(final Throwable e) {
+    public void onFailure(final Exception e) {
         threadPool.executor(executor).execute(new AbstractRunnable() {
             @Override
             public boolean isForceExecution() {
@@ -117,8 +117,8 @@ public final class ThreadedActionListener<Response> implements ActionListener<Re
             }
 
             @Override
-            public void onFailure(Throwable t) {
-                logger.warn("failed to execute failure callback on [{}], failure [{}]", t, listener, e);
+            public void onFailure(Exception e) {
+                logger.warn("failed to execute failure callback on [{}], failure [{}]", e, listener, e);
             }
         });
     }
diff --git a/core/src/main/java/org/elasticsearch/action/support/TransportAction.java b/core/src/main/java/org/elasticsearch/action/support/TransportAction.java
index 75542784997..60bd6100089 100644
--- a/core/src/main/java/org/elasticsearch/action/support/TransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/TransportAction.java
@@ -92,7 +92,7 @@ public abstract class TransportAction<Request extends ActionRequest<Request>, Re
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     taskManager.unregister(task);
                     listener.onFailure(e);
                 }
@@ -113,7 +113,7 @@ public abstract class TransportAction<Request extends ActionRequest<Request>, Re
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 if (task != null) {
                     taskManager.unregister(task);
                 }
@@ -140,9 +140,9 @@ public abstract class TransportAction<Request extends ActionRequest<Request>, Re
         if (filters.length == 0) {
             try {
                 doExecute(task, request, listener);
-            } catch(Throwable t) {
-                logger.trace("Error during transport action execution.", t);
-                listener.onFailure(t);
+            } catch(Exception e) {
+                logger.trace("Error during transport action execution.", e);
+                listener.onFailure(e);
             }
         } else {
             RequestFilterChain<Request, Response> requestFilterChain = new RequestFilterChain<>(this, logger);
@@ -180,9 +180,9 @@ public abstract class TransportAction<Request extends ActionRequest<Request>, Re
                 } else {
                     listener.onFailure(new IllegalStateException("proceed was called too many times"));
                 }
-            } catch(Throwable t) {
-                logger.trace("Error during transport action execution.", t);
-                listener.onFailure(t);
+            } catch(Exception e) {
+                logger.trace("Error during transport action execution.", e);
+                listener.onFailure(e);
             }
         }
 
@@ -221,9 +221,9 @@ public abstract class TransportAction<Request extends ActionRequest<Request>, Re
                 } else {
                     listener.onFailure(new IllegalStateException("proceed was called too many times"));
                 }
-            } catch (Throwable t) {
-                logger.trace("Error during transport action execution.", t);
-                listener.onFailure(t);
+            } catch (Exception e) {
+                logger.trace("Error during transport action execution.", e);
+                listener.onFailure(e);
             }
         }
     }
@@ -246,7 +246,7 @@ public abstract class TransportAction<Request extends ActionRequest<Request>, Re
         }
 
         @Override
-        public void onFailure(Throwable e) {
+        public void onFailure(Exception e) {
             listener.onFailure(e);
         }
     }
@@ -269,17 +269,18 @@ public abstract class TransportAction<Request extends ActionRequest<Request>, Re
         public void onResponse(Response response) {
             try {
                 taskManager.persistResult(task, response, delegate);
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 delegate.onFailure(e);
             }
         }
 
         @Override
-        public void onFailure(Throwable e) {
+        public void onFailure(Exception e) {
             try {
                 taskManager.persistResult(task, e, delegate);
-            } catch (Throwable e1) {
-                delegate.onFailure(e1);
+            } catch (Exception inner) {
+                inner.addSuppressed(e);
+                delegate.onFailure(inner);
             }
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/action/support/TransportActions.java b/core/src/main/java/org/elasticsearch/action/support/TransportActions.java
index 950596cb0b6..d27fbaf04f6 100644
--- a/core/src/main/java/org/elasticsearch/action/support/TransportActions.java
+++ b/core/src/main/java/org/elasticsearch/action/support/TransportActions.java
@@ -27,30 +27,23 @@ import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.shard.IllegalIndexShardStateException;
 import org.elasticsearch.index.shard.ShardNotFoundException;
 
-/**
- */
 public class TransportActions {
 
-    public static boolean isShardNotAvailableException(Throwable t) {
-        Throwable actual = ExceptionsHelper.unwrapCause(t);
-        if (actual instanceof ShardNotFoundException ||
+    public static boolean isShardNotAvailableException(final Throwable e) {
+        final Throwable actual = ExceptionsHelper.unwrapCause(e);
+        return (actual instanceof ShardNotFoundException ||
                 actual instanceof IndexNotFoundException ||
                 actual instanceof IllegalIndexShardStateException ||
                 actual instanceof NoShardAvailableActionException ||
                 actual instanceof UnavailableShardsException ||
-                actual instanceof AlreadyClosedException) {
-            return true;
-        }
-        return false;
+                actual instanceof AlreadyClosedException);
     }
 
     /**
      * If a failure is already present, should this failure override it or not for read operations.
      */
-    public static boolean isReadOverrideException(Throwable t) {
-        if (isShardNotAvailableException(t)) {
-            return false;
-        }
-        return true;
+    public static boolean isReadOverrideException(Exception e) {
+        return !isShardNotAvailableException(e);
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java b/core/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java
index 182d922fc39..f31e7f74eb8 100644
--- a/core/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java
@@ -144,7 +144,7 @@ public abstract class TransportBroadcastAction<Request extends BroadcastRequest<
                 // no shards
                 try {
                     listener.onResponse(newResponse(request, new AtomicReferenceArray(0), clusterState));
-                } catch (Throwable e) {
+                } catch (Exception e) {
                     listener.onFailure(e);
                 }
                 return;
@@ -199,7 +199,7 @@ public abstract class TransportBroadcastAction<Request extends BroadcastRequest<
                             }
                         });
                     }
-                } catch (Throwable e) {
+                } catch (Exception e) {
                     onOperation(shard, shardIt, shardIndex, e);
                 }
             }
@@ -215,25 +215,25 @@ public abstract class TransportBroadcastAction<Request extends BroadcastRequest<
         }
 
         @SuppressWarnings({"unchecked"})
-        void onOperation(@Nullable ShardRouting shard, final ShardIterator shardIt, int shardIndex, Throwable t) {
+        void onOperation(@Nullable ShardRouting shard, final ShardIterator shardIt, int shardIndex, Exception e) {
             // we set the shard failure always, even if its the first in the replication group, and the next one
             // will work (it will just override it...)
-            setFailure(shardIt, shardIndex, t);
+            setFailure(shardIt, shardIndex, e);
             ShardRouting nextShard = shardIt.nextOrNull();
             if (nextShard != null) {
-                if (t != null) {
+                if (e != null) {
                     if (logger.isTraceEnabled()) {
-                        if (!TransportActions.isShardNotAvailableException(t)) {
-                            logger.trace("{}: failed to execute [{}]", t, shard != null ? shard.shortSummary() : shardIt.shardId(), request);
+                        if (!TransportActions.isShardNotAvailableException(e)) {
+                            logger.trace("{}: failed to execute [{}]", e, shard != null ? shard.shortSummary() : shardIt.shardId(), request);
                         }
                     }
                 }
                 performOperation(shardIt, nextShard, shardIndex);
             } else {
                 if (logger.isDebugEnabled()) {
-                    if (t != null) {
-                        if (!TransportActions.isShardNotAvailableException(t)) {
-                            logger.debug("{}: failed to execute [{}]", t, shard != null ? shard.shortSummary() : shardIt.shardId(), request);
+                    if (e != null) {
+                        if (!TransportActions.isShardNotAvailableException(e)) {
+                            logger.debug("{}: failed to execute [{}]", e, shard != null ? shard.shortSummary() : shardIt.shardId(), request);
                         }
                     }
                 }
@@ -246,25 +246,25 @@ public abstract class TransportBroadcastAction<Request extends BroadcastRequest<
         protected void finishHim() {
             try {
                 listener.onResponse(newResponse(request, shardsResponses, clusterState));
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 listener.onFailure(e);
             }
         }
 
-        void setFailure(ShardIterator shardIt, int shardIndex, Throwable t) {
+        void setFailure(ShardIterator shardIt, int shardIndex, Exception e) {
             // we don't aggregate shard failures on non active shards (but do keep the header counts right)
-            if (TransportActions.isShardNotAvailableException(t)) {
+            if (TransportActions.isShardNotAvailableException(e)) {
                 return;
             }
 
-            if (!(t instanceof BroadcastShardOperationFailedException)) {
-                t = new BroadcastShardOperationFailedException(shardIt.shardId(), t);
+            if (!(e instanceof BroadcastShardOperationFailedException)) {
+                e = new BroadcastShardOperationFailedException(shardIt.shardId(), e);
             }
 
             Object response = shardsResponses.get(shardIndex);
             if (response == null) {
                 // just override it and return
-                shardsResponses.set(shardIndex, t);
+                shardsResponses.set(shardIndex, e);
             }
 
             if (!(response instanceof Throwable)) {
@@ -274,8 +274,8 @@ public abstract class TransportBroadcastAction<Request extends BroadcastRequest<
 
             // the failure is already present, try and not override it with an exception that is less meaningless
             // for example, getting illegal shard state
-            if (TransportActions.isReadOverrideException(t)) {
-                shardsResponses.set(shardIndex, t);
+            if (TransportActions.isReadOverrideException(e)) {
+                shardsResponses.set(shardIndex, e);
             }
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java b/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
index 178e2b69a7b..6fa0605828b 100644
--- a/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeAction.java
@@ -299,7 +299,7 @@ public abstract class TransportBroadcastByNodeAction<Request extends BroadcastRe
             if (nodeIds.size() == 0) {
                 try {
                     onCompletion();
-                } catch (Throwable e) {
+                } catch (Exception e) {
                     listener.onFailure(e);
                 }
             } else {
@@ -340,7 +340,7 @@ public abstract class TransportBroadcastByNodeAction<Request extends BroadcastRe
                         return ThreadPool.Names.SAME;
                     }
                 });
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 onNodeFailure(node, nodeIndex, e);
             }
         }
@@ -380,15 +380,15 @@ public abstract class TransportBroadcastByNodeAction<Request extends BroadcastRe
             Response response = null;
             try {
                 response = newResponse(request, responses, unavailableShardExceptions, nodeIds, clusterState);
-            } catch (Throwable t) {
-                logger.debug("failed to combine responses from nodes", t);
-                listener.onFailure(t);
+            } catch (Exception e) {
+                logger.debug("failed to combine responses from nodes", e);
+                listener.onFailure(e);
             }
             if (response != null) {
                 try {
                     listener.onResponse(response);
-                } catch (Throwable t) {
-                    listener.onFailure(t);
+                } catch (Exception e) {
+                    listener.onFailure(e);
                 }
             }
         }
@@ -433,18 +433,19 @@ public abstract class TransportBroadcastByNodeAction<Request extends BroadcastRe
                 if (logger.isTraceEnabled()) {
                     logger.trace("[{}]  completed operation for shard [{}]", actionName, shardRouting.shortSummary());
                 }
-            } catch (Throwable t) {
-                BroadcastShardOperationFailedException e = new BroadcastShardOperationFailedException(shardRouting.shardId(), "operation " + actionName + " failed", t);
-                e.setIndex(shardRouting.getIndexName());
-                e.setShard(shardRouting.shardId());
-                shardResults[shardIndex] = e;
-                if (TransportActions.isShardNotAvailableException(t)) {
+            } catch (Exception e) {
+                BroadcastShardOperationFailedException failure =
+                    new BroadcastShardOperationFailedException(shardRouting.shardId(), "operation " + actionName + " failed", e);
+                failure.setIndex(shardRouting.getIndexName());
+                failure.setShard(shardRouting.shardId());
+                shardResults[shardIndex] = failure;
+                if (TransportActions.isShardNotAvailableException(e)) {
                     if (logger.isTraceEnabled()) {
-                        logger.trace("[{}] failed to execute operation for shard [{}]", t, actionName, shardRouting.shortSummary());
+                        logger.trace("[{}] failed to execute operation for shard [{}]", e, actionName, shardRouting.shortSummary());
                     }
                 } else {
                     if (logger.isDebugEnabled()) {
-                        logger.debug("[{}] failed to execute operation for shard [{}]", t, actionName, shardRouting.shortSummary());
+                        logger.debug("[{}] failed to execute operation for shard [{}]", e, actionName, shardRouting.shortSummary());
                     }
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java b/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
index f0ea4ea14f2..7d6d4375731 100644
--- a/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java
@@ -152,7 +152,7 @@ public abstract class TransportMasterNodeAction<Request extends MasterNodeReques
                         }
 
                         @Override
-                        public void onFailure(Throwable t) {
+                        public void onFailure(Exception t) {
                             if (t instanceof Discovery.FailedToCommitClusterStateException
                                     || (t instanceof NotMasterException)) {
                                 logger.debug("master could not publish cluster state or stepped down before publishing action [{}], scheduling a retry", t, actionName);
diff --git a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
index 1609a08bb14..93d5a9c36e1 100644
--- a/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java
@@ -48,9 +48,6 @@ import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReferenceArray;
 import java.util.function.Supplier;
 
-/**
- *
- */
 public abstract class TransportNodesAction<NodesRequest extends BaseNodesRequest<NodesRequest>,
                                            NodesResponse extends BaseNodesResponse,
                                            NodeRequest extends BaseNodeRequest,
@@ -226,8 +223,8 @@ public abstract class TransportNodesAction<NodesRequest extends BaseNodesRequest
                             }
                         });
                     }
-                } catch (Throwable t) {
-                    onFailure(idx, nodeId, t);
+                } catch (Exception e) {
+                    onFailure(idx, nodeId, e);
                 }
             }
         }
@@ -255,9 +252,9 @@ public abstract class TransportNodesAction<NodesRequest extends BaseNodesRequest
             NodesResponse finalResponse;
             try {
                 finalResponse = newResponse(request, responses);
-            } catch (Throwable t) {
-                logger.debug("failed to combine responses from nodes", t);
-                listener.onFailure(t);
+            } catch (Exception e) {
+                logger.debug("failed to combine responses from nodes", e);
+                listener.onFailure(e);
                 return;
             }
             listener.onResponse(finalResponse);
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
index 1dbe543ff2f..dc7846a74de 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java
@@ -159,7 +159,7 @@ public class ReplicationOperation<
             }
 
             @Override
-            public void onFailure(Throwable replicaException) {
+            public void onFailure(Exception replicaException) {
                 logger.trace("[{}] failure while performing [{}] on replica {}, request [{}]", replicaException, shard.shardId(), opType,
                     shard, replicaRequest);
                 if (ignoreReplicaException(replicaException)) {
@@ -180,7 +180,7 @@ public class ReplicationOperation<
         });
     }
 
-    private void onPrimaryDemoted(Throwable demotionFailure) {
+    private void onPrimaryDemoted(Exception demotionFailure) {
         String primaryFail = String.format(Locale.ROOT,
             "primary shard [%s] was demoted while failing replica shard",
             primary.routingEntry());
@@ -266,9 +266,9 @@ public class ReplicationOperation<
         }
     }
 
-    private void finishAsFailed(Throwable throwable) {
+    private void finishAsFailed(Exception exception) {
         if (finished.compareAndSet(false, true)) {
-            resultListener.onFailure(throwable);
+            resultListener.onFailure(exception);
         }
     }
 
@@ -276,7 +276,7 @@ public class ReplicationOperation<
     /**
      * Should an exception be ignored when the operation is performed on the replica.
      */
-    public static boolean ignoreReplicaException(Throwable e) {
+    public static boolean ignoreReplicaException(Exception e) {
         if (TransportActions.isShardNotAvailableException(e)) {
             return true;
         }
@@ -288,14 +288,11 @@ public class ReplicationOperation<
         return false;
     }
 
-    public static boolean isConflictException(Throwable e) {
-        Throwable cause = ExceptionsHelper.unwrapCause(e);
+    public static boolean isConflictException(Throwable t) {
+        final Throwable cause = ExceptionsHelper.unwrapCause(t);
         // on version conflict or document missing, it means
         // that a new change has crept into the replica, and it's fine
-        if (cause instanceof VersionConflictEngineException) {
-            return true;
-        }
-        return false;
+        return cause instanceof VersionConflictEngineException;
     }
 
 
@@ -313,7 +310,7 @@ public class ReplicationOperation<
         /**
          * fail the primary, typically due to the fact that the operation has learned the primary has been demoted by the master
          */
-        void failShard(String message, Throwable throwable);
+        void failShard(String message, Exception exception);
 
         /**
          * Performs the given request on this primary. Yes, this returns as soon as it can with the request for the replicas and calls a
@@ -340,20 +337,17 @@ public class ReplicationOperation<
 
         /**
          * Fail the specified shard, removing it from the current set of active shards
-         *
          * @param replica          shard to fail
          * @param primary          the primary shard that requested the failure
          * @param message          a (short) description of the reason
-         * @param throwable        the original exception which caused the ReplicationOperation to request the shard to be failed
+         * @param exception        the original exception which caused the ReplicationOperation to request the shard to be failed
          * @param onSuccess        a callback to call when the shard has been successfully removed from the active set.
          * @param onPrimaryDemoted a callback to call when the shard can not be failed because the current primary has been demoted
-         *                         by the master.
+*                         by the master.
          * @param onIgnoredFailure a callback to call when failing a shard has failed, but it that failure can be safely ignored and the
-         *                         replication operation can finish processing
-         *                         Note: this callback should be used in extreme situations, typically node shutdown.
          */
-        void failShard(ShardRouting replica, ShardRouting primary, String message, Throwable throwable, Runnable onSuccess,
-                       Consumer<Throwable> onPrimaryDemoted, Consumer<Throwable> onIgnoredFailure);
+        void failShard(ShardRouting replica, ShardRouting primary, String message, Exception exception, Runnable onSuccess,
+                       Consumer<Exception> onPrimaryDemoted, Consumer<Exception> onIgnoredFailure);
     }
 
     public static class RetryOnPrimaryException extends ElasticsearchException {
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
index b75cde3f765..c79f8bb7302 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
@@ -175,11 +175,11 @@ public class ReplicationResponse extends ActionResponse {
 
             private ShardId shardId;
             private String nodeId;
-            private Throwable cause;
+            private Exception cause;
             private RestStatus status;
             private boolean primary;
 
-            public Failure(ShardId  shardId, @Nullable String nodeId, Throwable cause, RestStatus status, boolean primary) {
+            public Failure(ShardId  shardId, @Nullable String nodeId, Exception cause, RestStatus status, boolean primary) {
                 this.shardId = shardId;
                 this.nodeId = nodeId;
                 this.cause = cause;
@@ -251,7 +251,7 @@ public class ReplicationResponse extends ActionResponse {
             public void readFrom(StreamInput in) throws IOException {
                 shardId = ShardId.readShardId(in);
                 nodeId = in.readOptionalString();
-                cause = in.readThrowable();
+                cause = in.readException();
                 status = RestStatus.readFrom(in);
                 primary = in.readBoolean();
             }
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java
index 2cab7d78317..e33d10eaa25 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java
@@ -93,7 +93,7 @@ public abstract class TransportBroadcastReplicationAction<Request extends Broadc
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     logger.trace("{}: got failure from {}", actionName, shardId);
                     int totalNumCopies = clusterState.getMetaData().getIndexSafe(shardId.getIndex()).getNumberOfReplicas() + 1;
                     ShardResponse shardResponse = newShardResponse();
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
index f562224d9f7..da1d2e68917 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java
@@ -196,7 +196,7 @@ public abstract class TransportReplicationAction<
         return TransportRequestOptions.EMPTY;
     }
 
-    protected boolean retryPrimaryException(Throwable e) {
+    protected boolean retryPrimaryException(final Throwable e) {
         return e.getClass() == ReplicationOperation.RetryOnPrimaryException.class
             || TransportActions.isShardNotAvailableException(e);
     }
@@ -209,17 +209,18 @@ public abstract class TransportReplicationAction<
                 public void onResponse(Response result) {
                     try {
                         channel.sendResponse(result);
-                    } catch (Throwable e) {
+                    } catch (Exception e) {
                         onFailure(e);
                     }
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     try {
                         channel.sendResponse(e);
-                    } catch (Throwable e1) {
-                        logger.warn("Failed to send response for {}", e1, actionName);
+                    } catch (Exception inner) {
+                        inner.addSuppressed(e);
+                        logger.warn("Failed to send response for {}", inner, actionName);
                     }
                 }
             });
@@ -300,25 +301,25 @@ public abstract class TransportReplicationAction<
                         }
 
                         @Override
-                        public void onFailure(Throwable e) {
+                        public void onFailure(Exception e) {
                             listener.onFailure(e);
                         }
                     }, primaryShardReference, executeOnReplicas).execute();
                 }
-            } catch (Throwable t) {
+            } catch (Exception e) {
                 Releasables.closeWhileHandlingException(primaryShardReference); // release shard operation lock before responding to caller
-                onFailure(t);
+                onFailure(e);
             }
         }
 
         @Override
-        public void onFailure(Throwable t) {
+        public void onFailure(Exception e) {
             setPhase(replicationTask, "finished");
             try {
-                channel.sendResponse(t);
-            } catch (IOException e) {
-                e.addSuppressed(t);
-                logger.warn("failed to send response", e);
+                channel.sendResponse(e);
+            } catch (IOException inner) {
+                inner.addSuppressed(e);
+                logger.warn("failed to send response", inner);
             }
         }
 
@@ -336,7 +337,7 @@ public abstract class TransportReplicationAction<
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     primaryShardReference.close(); // release shard operation lock before responding to caller
                     setPhase(replicationTask, "finished");
                     try {
@@ -439,16 +440,16 @@ public abstract class TransportReplicationAction<
                 ReplicaResult replicaResult = shardOperationOnReplica(request);
                 releasable.close(); // release shard operation lock before responding to caller
                 replicaResult.respond(new ResponseListener());
-            } catch (Throwable t) {
+            } catch (Exception e) {
                 Releasables.closeWhileHandlingException(releasable); // release shard operation lock before responding to caller
-                AsyncReplicaAction.this.onFailure(t);
+                AsyncReplicaAction.this.onFailure(e);
             }
         }
 
         @Override
-        public void onFailure(Throwable t) {
-            if (t instanceof RetryOnReplicaException) {
-                logger.trace("Retrying operation on replica, action [{}], request [{}]", t, transportReplicaAction, request);
+        public void onFailure(Exception e) {
+            if (e instanceof RetryOnReplicaException) {
+                logger.trace("Retrying operation on replica, action [{}], request [{}]", e, transportReplicaAction, request);
                 final ThreadContext.StoredContext context = threadPool.getThreadContext().newStoredContext();
                 observer.waitForNextChange(new ClusterStateObserver.Listener() {
                     @Override
@@ -473,17 +474,17 @@ public abstract class TransportReplicationAction<
                     }
                 });
             } else {
-                responseWithFailure(t);
+                responseWithFailure(e);
             }
         }
 
-        protected void responseWithFailure(Throwable t) {
+        protected void responseWithFailure(Exception e) {
             try {
                 setPhase(task, "finished");
-                channel.sendResponse(t);
+                channel.sendResponse(e);
             } catch (IOException responseException) {
+                responseException.addSuppressed(e);
                 logger.warn("failed to send error message back to client for action [{}]", responseException, transportReplicaAction);
-                logger.warn("actual Exception", t);
             }
         }
 
@@ -513,7 +514,7 @@ public abstract class TransportReplicationAction<
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 responseWithFailure(e);
             }
         }
@@ -544,7 +545,7 @@ public abstract class TransportReplicationAction<
         }
 
         @Override
-        public void onFailure(Throwable e) {
+        public void onFailure(Exception e) {
             finishWithUnexpectedFailure(e);
         }
 
@@ -694,14 +695,15 @@ public abstract class TransportReplicationAction<
                         } else {
                             finishAsFailed(exp);
                         }
-                    } catch (Throwable t) {
-                        finishWithUnexpectedFailure(t);
+                    } catch (Exception e) {
+                        e.addSuppressed(exp);
+                        finishWithUnexpectedFailure(e);
                     }
                 }
             });
         }
 
-        void retry(Throwable failure) {
+        void retry(Exception failure) {
             assert failure != null;
             if (observer.isTimedOut()) {
                 // we running as a last attempt after a timeout has happened. don't retry
@@ -731,7 +733,7 @@ public abstract class TransportReplicationAction<
             });
         }
 
-        void finishAsFailed(Throwable failure) {
+        void finishAsFailed(Exception failure) {
             if (finished.compareAndSet(false, true)) {
                 setPhase(task, "failed");
                 logger.trace("operation failed. action [{}], request [{}]", failure, actionName, request);
@@ -741,7 +743,7 @@ public abstract class TransportReplicationAction<
             }
         }
 
-        void finishWithUnexpectedFailure(Throwable failure) {
+        void finishWithUnexpectedFailure(Exception failure) {
             logger.warn("unexpected error during the primary phase for action [{}], request [{}]", failure, actionName, request);
             if (finished.compareAndSet(false, true)) {
                 setPhase(task, "failed");
@@ -790,7 +792,7 @@ public abstract class TransportReplicationAction<
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 onReferenceAcquired.onFailure(e);
             }
         };
@@ -835,11 +837,11 @@ public abstract class TransportReplicationAction<
         }
 
         @Override
-        public void failShard(String reason, Throwable e) {
+        public void failShard(String reason, Exception e) {
             try {
                 indexShard.failShard(reason, e);
-            } catch (Throwable suppressed) {
-                e.addSuppressed(suppressed);
+            } catch (Exception inner) {
+                e.addSuppressed(inner);
             }
         }
 
@@ -871,10 +873,10 @@ public abstract class TransportReplicationAction<
         }
 
         @Override
-        public void failShard(ShardRouting replica, ShardRouting primary, String message, Throwable throwable,
-                              Runnable onSuccess, Consumer<Throwable> onFailure, Consumer<Throwable> onIgnoredFailure) {
+        public void failShard(ShardRouting replica, ShardRouting primary, String message, Exception exception,
+                              Runnable onSuccess, Consumer<Exception> onFailure, Consumer<Exception> onIgnoredFailure) {
             shardStateAction.shardFailed(
-                replica, primary, message, throwable,
+                replica, primary, message, exception,
                 new ShardStateAction.Listener() {
                     @Override
                     public void onSuccess() {
@@ -882,7 +884,7 @@ public abstract class TransportReplicationAction<
                     }
 
                     @Override
-                    public void onFailure(Throwable shardFailedError) {
+                    public void onFailure(Exception shardFailedError) {
                         if (shardFailedError instanceof ShardStateAction.NoLongerPrimaryShardException) {
                             onFailure.accept(shardFailedError);
                         } else {
diff --git a/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java b/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java
index 760b77469d2..db786cad63f 100644
--- a/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationAction.java
@@ -95,7 +95,7 @@ public abstract class TransportInstanceSingleOperationAction<Request extends Ins
      */
     protected abstract void resolveRequest(ClusterState state, Request request);
 
-    protected boolean retryOnFailure(Throwable e) {
+    protected boolean retryOnFailure(Exception e) {
         return false;
     }
 
@@ -150,7 +150,7 @@ public abstract class TransportInstanceSingleOperationAction<Request extends Ins
                     }
                 }
                 shardIt = shards(observer.observedState(), request);
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 listener.onFailure(e);
                 return;
             }
@@ -193,11 +193,11 @@ public abstract class TransportInstanceSingleOperationAction<Request extends Ins
 
                 @Override
                 public void handleException(TransportException exp) {
-                    Throwable cause = exp.unwrapCause();
+                    final Throwable cause = exp.unwrapCause();
                     // if we got disconnected from the node, or the node / shard is not in the right state (being closed)
                     if (cause instanceof ConnectTransportException || cause instanceof NodeClosedException ||
                             retryOnFailure(exp)) {
-                        retry(cause);
+                        retry((Exception) cause);
                     } else {
                         listener.onFailure(exp);
                     }
@@ -205,10 +205,10 @@ public abstract class TransportInstanceSingleOperationAction<Request extends Ins
             });
         }
 
-        void retry(@Nullable final Throwable failure) {
+        void retry(@Nullable final Exception failure) {
             if (observer.isTimedOut()) {
                 // we running as a last attempt after a timeout has happened. don't retry
-                Throwable listenFailure = failure;
+                Exception listenFailure = failure;
                 if (listenFailure == null) {
                     if (shardIt == null) {
                         listenFailure = new UnavailableShardsException(request.concreteIndex(), -1, "Timeout waiting for [{}], request: {}", request.timeout(), actionName);
@@ -249,17 +249,18 @@ public abstract class TransportInstanceSingleOperationAction<Request extends Ins
                 public void onResponse(Response response) {
                     try {
                         channel.sendResponse(response);
-                    } catch (Throwable e) {
+                    } catch (Exception e) {
                         onFailure(e);
                     }
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     try {
                         channel.sendResponse(e);
-                    } catch (Exception e1) {
-                        logger.warn("failed to send response for get", e1);
+                    } catch (Exception inner) {
+                        inner.addSuppressed(e);
+                        logger.warn("failed to send response for get", inner);
                     }
                 }
             });
diff --git a/core/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java b/core/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java
index 0c7f0627c66..0e1c26e5f87 100644
--- a/core/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java
@@ -124,7 +124,7 @@ public abstract class TransportSingleShardAction<Request extends SingleShardRequ
         private final ShardsIterator shardIt;
         private final InternalRequest internalRequest;
         private final DiscoveryNodes nodes;
-        private volatile Throwable lastFailure;
+        private volatile Exception lastFailure;
 
         private AsyncSingleAction(Request request, ActionListener<Response> listener) {
             this.listener = listener;
@@ -185,22 +185,22 @@ public abstract class TransportSingleShardAction<Request extends SingleShardRequ
             }
         }
 
-        private void onFailure(ShardRouting shardRouting, Throwable e) {
+        private void onFailure(ShardRouting shardRouting, Exception e) {
             if (logger.isTraceEnabled() && e != null) {
                 logger.trace("{}: failed to execute [{}]", e, shardRouting, internalRequest.request());
             }
             perform(e);
         }
 
-        private void perform(@Nullable final Throwable currentFailure) {
-            Throwable lastFailure = this.lastFailure;
+        private void perform(@Nullable final Exception currentFailure) {
+            Exception lastFailure = this.lastFailure;
             if (lastFailure == null || TransportActions.isReadOverrideException(currentFailure)) {
                 lastFailure = currentFailure;
                 this.lastFailure = currentFailure;
             }
             final ShardRouting shardRouting = shardIt.nextOrNull();
             if (shardRouting == null) {
-                Throwable failure = lastFailure;
+                Exception failure = lastFailure;
                 if (failure == null || isShardNotAvailableException(failure)) {
                     failure = new NoShardAvailableActionException(null, LoggerMessageFormat.format("No shard available for [{}]", internalRequest.request()), failure);
                 } else {
@@ -261,13 +261,13 @@ public abstract class TransportSingleShardAction<Request extends SingleShardRequ
                 public void onResponse(Response result) {
                     try {
                         channel.sendResponse(result);
-                    } catch (Throwable e) {
+                    } catch (Exception e) {
                         onFailure(e);
                     }
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     try {
                         channel.sendResponse(e);
                     } catch (Exception e1) {
diff --git a/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java b/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
index 0a45497e1fd..a907fb86af5 100644
--- a/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/tasks/TransportTasksAction.java
@@ -215,9 +215,9 @@ public abstract class TransportTasksAction<
                 // nothing to do
                 try {
                     listener.onResponse(newResponse(request, responses));
-                } catch (Throwable t) {
-                    logger.debug("failed to generate empty response", t);
-                    listener.onFailure(t);
+                } catch (Exception e) {
+                    logger.debug("failed to generate empty response", e);
+                    listener.onFailure(e);
                 }
             } else {
                 TransportRequestOptions.Builder builder = TransportRequestOptions.builder();
@@ -259,8 +259,8 @@ public abstract class TransportTasksAction<
                                     }
                                 });
                         }
-                    } catch (Throwable t) {
-                        onFailure(idx, nodeId, t);
+                    } catch (Exception e) {
+                        onFailure(idx, nodeId, e);
                     }
                 }
             }
@@ -289,9 +289,9 @@ public abstract class TransportTasksAction<
             TasksResponse finalResponse;
             try {
                 finalResponse = newResponse(request, responses);
-            } catch (Throwable t) {
-                logger.debug("failed to combine responses from nodes", t);
-                listener.onFailure(t);
+            } catch (Exception e) {
+                logger.debug("failed to combine responses from nodes", e);
+                listener.onFailure(e);
                 return;
             }
             listener.onResponse(finalResponse);
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java b/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
index 8e442538d77..7d8294ecbd4 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
@@ -40,13 +40,13 @@ public class MultiTermVectorsResponse extends ActionResponse implements Iterable
         private String index;
         private String type;
         private String id;
-        private Throwable cause;
+        private Exception cause;
 
         Failure() {
 
         }
 
-        public Failure(String index, String type, String id, Throwable cause) {
+        public Failure(String index, String type, String id, Exception cause) {
             this.index = index;
             this.type = type;
             this.id = id;
@@ -77,7 +77,7 @@ public class MultiTermVectorsResponse extends ActionResponse implements Iterable
         /**
          * The failure cause.
          */
-        public Throwable getCause() {
+        public Exception getCause() {
             return this.cause;
         }
 
@@ -92,7 +92,7 @@ public class MultiTermVectorsResponse extends ActionResponse implements Iterable
             index = in.readString();
             type = in.readOptionalString();
             id = in.readString();
-            cause = in.readThrowable();
+            cause = in.readException();
         }
 
         @Override
@@ -132,7 +132,7 @@ public class MultiTermVectorsResponse extends ActionResponse implements Iterable
                 builder.field(Fields._INDEX, failure.getIndex());
                 builder.field(Fields._TYPE, failure.getType());
                 builder.field(Fields._ID, failure.getId());
-                ElasticsearchException.renderThrowable(builder, params, failure.getCause());
+                ElasticsearchException.renderException(builder, params, failure.getCause());
                 builder.endObject();
             } else {
                 TermVectorsResponse getResponse = response.getResponse();
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java b/core/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java
index 8c3dc496296..fc44ba64a98 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/TransportMultiTermVectorsAction.java
@@ -107,7 +107,7 @@ public class TransportMultiTermVectorsAction extends HandledTransportAction<Mult
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     // create failures for all relevant requests
                     for (int i = 0; i < shardRequest.locations.size(); i++) {
                         TermVectorsRequest termVectorsRequest = shardRequest.requests.get(i);
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java b/core/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java
index afc63010841..da12831f1c7 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/TransportShardMultiTermsVectorAction.java
@@ -83,7 +83,7 @@ public class TransportShardMultiTermsVectorAction extends TransportSingleShardAc
             try {
                 TermVectorsResponse termVectorsResponse = TermVectorsService.getTermVectors(indexShard, termVectorsRequest);
                 response.add(request.locations.get(i), termVectorsResponse);
-            } catch (Throwable t) {
+            } catch (Exception t) {
                 if (TransportActions.isShardNotAvailableException(t)) {
                     throw (ElasticsearchException) t;
                 } else {
diff --git a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
index ae6a15411f5..3bff01cb8a6 100644
--- a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
+++ b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java
@@ -46,6 +46,7 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.stream.NotSerializableExceptionWrapper;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -61,6 +62,8 @@ import org.elasticsearch.transport.TransportService;
 import java.util.Collections;
 import java.util.Map;
 
+import static org.elasticsearch.ExceptionsHelper.unwrapCause;
+
 /**
  */
 public class TransportUpdateAction extends TransportInstanceSingleOperationAction<UpdateRequest, UpdateResponse> {
@@ -97,7 +100,7 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
     }
 
     @Override
-    protected boolean retryOnFailure(Throwable e) {
+    protected boolean retryOnFailure(Exception e) {
         return TransportActions.isShardNotAvailableException(e);
     }
 
@@ -125,13 +128,14 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
-                    if (ExceptionsHelper.unwrapCause(e) instanceof IndexAlreadyExistsException) {
+                public void onFailure(Exception e) {
+                    if (unwrapCause(e) instanceof IndexAlreadyExistsException) {
                         // we have the index, do it
                         try {
                             innerExecute(request, listener);
-                        } catch (Throwable e1) {
-                            listener.onFailure(e1);
+                        } catch (Exception inner) {
+                            inner.addSuppressed(e);
+                            listener.onFailure(inner);
                         }
                     } else {
                         listener.onFailure(e);
@@ -193,9 +197,9 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
                     }
 
                     @Override
-                    public void onFailure(Throwable e) {
-                        e = ExceptionsHelper.unwrapCause(e);
-                        if (e instanceof VersionConflictEngineException) {
+                    public void onFailure(Exception e) {
+                        final Throwable cause = ExceptionsHelper.unwrapCause(e);
+                        if (cause instanceof VersionConflictEngineException) {
                             if (retryCount < request.retryOnConflict()) {
                                 logger.trace("Retry attempt [{}] of [{}] on version conflict on [{}][{}][{}]",
                                         retryCount + 1, request.retryOnConflict(), request.index(), request.getShardId(), request.id());
@@ -208,7 +212,7 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
                                 return;
                             }
                         }
-                        listener.onFailure(e);
+                        listener.onFailure(cause instanceof Exception ? (Exception) cause : new NotSerializableExceptionWrapper(cause));
                     }
                 });
                 break;
@@ -226,9 +230,9 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
                     }
 
                     @Override
-                    public void onFailure(Throwable e) {
-                        e = ExceptionsHelper.unwrapCause(e);
-                        if (e instanceof VersionConflictEngineException) {
+                    public void onFailure(Exception e) {
+                        final Throwable cause = unwrapCause(e);
+                        if (cause instanceof VersionConflictEngineException) {
                             if (retryCount < request.retryOnConflict()) {
                                 threadPool.executor(executor()).execute(new ActionRunnable<UpdateResponse>(listener) {
                                     @Override
@@ -239,7 +243,7 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
                                 return;
                             }
                         }
-                        listener.onFailure(e);
+                        listener.onFailure(cause instanceof Exception ? (Exception) cause : new NotSerializableExceptionWrapper(cause));
                     }
                 });
                 break;
@@ -255,9 +259,9 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
                     }
 
                     @Override
-                    public void onFailure(Throwable e) {
-                        e = ExceptionsHelper.unwrapCause(e);
-                        if (e instanceof VersionConflictEngineException) {
+                    public void onFailure(Exception e) {
+                        final Throwable cause = unwrapCause(e);
+                        if (cause instanceof VersionConflictEngineException) {
                             if (retryCount < request.retryOnConflict()) {
                                 threadPool.executor(executor()).execute(new ActionRunnable<UpdateResponse>(listener) {
                                     @Override
@@ -268,7 +272,7 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
                                 return;
                             }
                         }
-                        listener.onFailure(e);
+                        listener.onFailure(cause instanceof Exception ? (Exception) cause : new NotSerializableExceptionWrapper(cause));
                     }
                 });
                 break;
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
index 54cea62e12e..4168e34a74b 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
@@ -125,7 +125,7 @@ final class Bootstrap {
         // force remainder of JNA to be loaded (if available).
         try {
             JNAKernel32Library.getInstance();
-        } catch (Throwable ignored) {
+        } catch (Exception ignored) {
             // we've already logged this.
         }
 
@@ -216,7 +216,7 @@ final class Bootstrap {
     static void init(
             final boolean foreground,
             final String pidFile,
-            final Map<String, String> esSettings) throws Throwable {
+            final Map<String, String> esSettings) throws Exception {
         // Set the system property before anything has a chance to trigger its use
         initLoggerPrefix();
 
@@ -254,7 +254,7 @@ final class Bootstrap {
             if (!foreground) {
                 closeSysError();
             }
-        } catch (Throwable e) {
+        } catch (Exception e) {
             // disable console logging, so user does not see the exception twice (jvm will show it already)
             if (foreground) {
                 Loggers.disableConsoleLogging();
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java b/core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java
index e55d38a0f72..5a8693b3137 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java
@@ -217,13 +217,13 @@ class JNANatives {
             if (ret == 1) {
                 LOCAL_SECCOMP_ALL = true;
             }
-        } catch (Throwable t) {
+        } catch (Exception e) {
             // this is likely to happen unless the kernel is newish, its a best effort at the moment
             // so we log stacktrace at debug for now...
             if (logger.isDebugEnabled()) {
-                logger.debug("unable to install syscall filter", t);
+                logger.debug("unable to install syscall filter", e);
             }
-            logger.warn("unable to install syscall filter: ", t);
+            logger.warn("unable to install syscall filter: ", e);
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Seccomp.java b/core/src/main/java/org/elasticsearch/bootstrap/Seccomp.java
index 46908e60642..2ea6179e200 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Seccomp.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Seccomp.java
@@ -611,7 +611,7 @@ final class Seccomp {
      * This is best effort and OS and architecture dependent. It may throw any Throwable.
      * @return 0 if we can do this for application threads, 1 for the entire process
      */
-    static int init(Path tmpFile) throws Throwable {
+    static int init(Path tmpFile) throws Exception {
         if (Constants.LINUX) {
             return linuxImpl();
         } else if (Constants.MAC_OS_X) {
diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
index 120a4711217..c751b9c43c1 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
@@ -230,9 +230,9 @@ public class TransportClientNodesService extends AbstractComponent implements Cl
         DiscoveryNode node = nodes.get((index) % nodes.size());
         try {
             callback.doWithNode(node, retryListener);
-        } catch (Throwable t) {
+        } catch (Exception e) {
             //this exception can't come from the TransportService as it doesn't throw exception at all
-            listener.onFailure(t);
+            listener.onFailure(e);
         }
     }
 
@@ -258,7 +258,7 @@ public class TransportClientNodesService extends AbstractComponent implements Cl
         }
 
         @Override
-        public void onFailure(Throwable e) {
+        public void onFailure(Exception e) {
             if (ExceptionsHelper.unwrapCause(e) instanceof ConnectTransportException) {
                 int i = ++this.i;
                 if (i >= nodes.size()) {
@@ -266,9 +266,10 @@ public class TransportClientNodesService extends AbstractComponent implements Cl
                 } else {
                     try {
                         callback.doWithNode(nodes.get((index + i) % nodes.size()), this);
-                    } catch(final Throwable t) {
+                    } catch(final Exception inner) {
+                        inner.addSuppressed(e);
                         // this exception can't come from the TransportService as it doesn't throw exceptions at all
-                        listener.onFailure(t);
+                        listener.onFailure(inner);
                     }
                 }
             } else {
@@ -335,7 +336,7 @@ public class TransportClientNodesService extends AbstractComponent implements Cl
                     try {
                         logger.trace("connecting to node [{}]", node);
                         transportService.connectToNode(node);
-                    } catch (Throwable e) {
+                    } catch (Exception e) {
                         it.remove();
                         logger.debug("failed to connect to discovered node [{}]", e, node);
                     }
@@ -373,7 +374,7 @@ public class TransportClientNodesService extends AbstractComponent implements Cl
                         // its a listed node, light connect to it...
                         logger.trace("connecting to listed node (light) [{}]", listedNode);
                         transportService.connectToNodeLight(listedNode);
-                    } catch (Throwable e) {
+                    } catch (Exception e) {
                         logger.debug("failed to connect to node [{}], removed from nodes list", e, listedNode);
                         newFilteredNodes.add(listedNode);
                         continue;
@@ -405,7 +406,7 @@ public class TransportClientNodesService extends AbstractComponent implements Cl
                         logger.debug("node {} didn't return any discovery info, temporarily using transport discovery node", listedNode);
                         newNodes.add(listedNode);
                     }
-                } catch (Throwable e) {
+                } catch (Exception e) {
                     logger.info("failed to get node info for {}, disconnecting...", e, listedNode);
                     transportService.disconnectFromNode(listedNode);
                 }
@@ -484,7 +485,7 @@ public class TransportClientNodesService extends AbstractComponent implements Cl
                                             latch.countDown();
                                         }
                                     });
-                        } catch (Throwable e) {
+                        } catch (Exception e) {
                             logger.info("failed to get local cluster state info for {}, disconnecting...", e, listedNode);
                             transportService.disconnectFromNode(listedNode);
                             latch.countDown();
diff --git a/core/src/main/java/org/elasticsearch/cluster/AckedClusterStateTaskListener.java b/core/src/main/java/org/elasticsearch/cluster/AckedClusterStateTaskListener.java
index cdd9b2204ff..148a1dea309 100644
--- a/core/src/main/java/org/elasticsearch/cluster/AckedClusterStateTaskListener.java
+++ b/core/src/main/java/org/elasticsearch/cluster/AckedClusterStateTaskListener.java
@@ -36,9 +36,9 @@ public interface AckedClusterStateTaskListener extends ClusterStateTaskListener
      * Called once all the nodes have acknowledged the cluster state update request. Must be
      * very lightweight execution, since it gets executed on the cluster service thread.
      *
-     * @param t optional error that might have been thrown
+     * @param e optional error that might have been thrown
      */
-    void onAllNodesAcked(@Nullable Throwable t);
+    void onAllNodesAcked(@Nullable Exception e);
 
     /**
      * Called once the acknowledgement timeout defined by
diff --git a/core/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java b/core/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java
index b833f6e1879..faf2f30bb3e 100644
--- a/core/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java
+++ b/core/src/main/java/org/elasticsearch/cluster/AckedClusterStateUpdateTask.java
@@ -58,9 +58,9 @@ public abstract class AckedClusterStateUpdateTask<Response> extends ClusterState
      * Called once all the nodes have acknowledged the cluster state update request. Must be
      * very lightweight execution, since it gets executed on the cluster service thread.
      *
-     * @param t optional error that might have been thrown
+     * @param e optional error that might have been thrown
      */
-    public void onAllNodesAcked(@Nullable Throwable t) {
+    public void onAllNodesAcked(@Nullable Exception e) {
         listener.onResponse(newResponse(true));
     }
 
@@ -75,8 +75,8 @@ public abstract class AckedClusterStateUpdateTask<Response> extends ClusterState
     }
 
     @Override
-    public void onFailure(String source, Throwable t) {
-        listener.onFailure(t);
+    public void onFailure(String source, Exception e) {
+        listener.onFailure(e);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskConfig.java b/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskConfig.java
index 2ef2438991e..0724c05ac21 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskConfig.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskConfig.java
@@ -29,7 +29,7 @@ public interface ClusterStateTaskConfig {
     /**
      * The timeout for this cluster state update task configuration. If
      * the cluster state update task isn't processed within this
-     * timeout, the associated {@link ClusterStateTaskListener#onFailure(String, Throwable)}
+     * timeout, the associated {@link ClusterStateTaskListener#onFailure(String, Exception)}
      * is invoked.
      *
      * @return the timeout, or null if one is not set
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java b/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java
index 633fce1ac96..c801ff9e95f 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java
@@ -82,13 +82,13 @@ public interface ClusterStateTaskExecutor<T> {
                 return this;
             }
 
-            public Builder<T> failure(T task, Throwable t) {
-                return result(task, TaskResult.failure(t));
+            public Builder<T> failure(T task, Exception e) {
+                return result(task, TaskResult.failure(e));
             }
 
-            public Builder<T> failures(Iterable<T> tasks, Throwable t) {
+            public Builder<T> failures(Iterable<T> tasks, Exception e) {
                 for (T task : tasks) {
-                    failure(task, t);
+                    failure(task, e);
                 }
                 return this;
             }
@@ -106,7 +106,7 @@ public interface ClusterStateTaskExecutor<T> {
     }
 
     final class TaskResult {
-        private final Throwable failure;
+        private final Exception failure;
 
         private static final TaskResult SUCCESS = new TaskResult(null);
 
@@ -114,11 +114,11 @@ public interface ClusterStateTaskExecutor<T> {
             return SUCCESS;
         }
 
-        public static TaskResult failure(Throwable failure) {
+        public static TaskResult failure(Exception failure) {
             return new TaskResult(failure);
         }
 
-        private TaskResult(Throwable failure) {
+        private TaskResult(Exception failure) {
             this.failure = failure;
         }
 
@@ -126,7 +126,7 @@ public interface ClusterStateTaskExecutor<T> {
             return this == SUCCESS;
         }
 
-        public Throwable getFailure() {
+        public Exception getFailure() {
             assert !isSuccess();
             return failure;
         }
@@ -136,7 +136,7 @@ public interface ClusterStateTaskExecutor<T> {
          * @param onSuccess handler to invoke on success
          * @param onFailure handler to invoke on failure; the throwable passed through will not be null
          */
-        public void handle(Runnable onSuccess, Consumer<Throwable> onFailure) {
+        public void handle(Runnable onSuccess, Consumer<Exception> onFailure) {
             if (failure == null) {
                 onSuccess.run();
             } else {
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskListener.java b/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskListener.java
index 3bf7887cd1c..757c8b0c82e 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskListener.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskListener.java
@@ -25,7 +25,7 @@ public interface ClusterStateTaskListener {
     /**
      * A callback called when execute fails.
      */
-    void onFailure(String source, Throwable t);
+    void onFailure(String source, Exception e);
 
     /**
      * called when the task was rejected because the local node is no longer master
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java b/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java
index c6988bbb8c8..52e4b8dcff1 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java
@@ -50,16 +50,16 @@ public abstract  class ClusterStateUpdateTask implements ClusterStateTaskConfig,
      * Update the cluster state based on the current state. Return the *same instance* if no state
      * should be changed.
      */
-    public abstract  ClusterState execute(ClusterState currentState) throws Exception;
+    public abstract ClusterState execute(ClusterState currentState) throws Exception;
 
     /**
      * A callback called when execute fails.
      */
-    public abstract  void onFailure(String source, Throwable t);
+    public abstract void onFailure(String source, Exception e);
 
     /**
      * If the cluster state update task wasn't processed by the provided timeout, call
-     * {@link #onFailure(String, Throwable)}. May return null to indicate no timeout is needed (default).
+     * {@link ClusterStateTaskListener#onFailure(String, Exception)}. May return null to indicate no timeout is needed (default).
      */
     @Nullable
     public TimeValue timeout() {
diff --git a/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
index 3101e700327..2e26bad4684 100644
--- a/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.cluster;
 
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.LatchedActionListener;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
@@ -30,7 +29,6 @@ import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -309,7 +307,7 @@ public class InternalClusterInfoService extends AbstractComponent implements Clu
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 if (e instanceof ReceiveTimeoutTransportException) {
                     logger.error("NodeStatsAction timed out for ClusterInfoUpdateJob", e);
                 } else {
@@ -339,7 +337,7 @@ public class InternalClusterInfoService extends AbstractComponent implements Clu
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 if (e instanceof ReceiveTimeoutTransportException) {
                     logger.error("IndicesStatsAction timed out for ClusterInfoUpdateJob", e);
                 } else {
diff --git a/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java b/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
index 79c4db9dcad..a487bda0db4 100644
--- a/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/NodeConnectionsService.java
@@ -90,7 +90,7 @@ public class NodeConnectionsService extends AbstractLifecycleComponent {
                 assert current != null : "node " + node + " was removed in event but not in internal nodes";
                 try {
                     transportService.disconnectFromNode(node);
-                } catch (Throwable e) {
+                } catch (Exception e) {
                     logger.warn("failed to disconnect to node [{}]", e, node);
                 }
             }
@@ -123,8 +123,8 @@ public class NodeConnectionsService extends AbstractLifecycleComponent {
     class ConnectionChecker extends AbstractRunnable {
 
         @Override
-        public void onFailure(Throwable t) {
-            logger.warn("unexpected error while checking for node reconnects", t);
+        public void onFailure(Exception e) {
+            logger.warn("unexpected error while checking for node reconnects", e);
         }
 
         protected void doRun() {
diff --git a/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
index 630852361b4..1668f3f57ed 100644
--- a/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
+++ b/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
@@ -111,7 +111,7 @@ public class ShardStateAction extends AbstractComponent {
                             waitForNewMasterAndRetry(actionName, observer, shardRoutingEntry, listener);
                         } else {
                             logger.warn("{} unexpected failure while sending request [{}] to [{}] for shard [{}]", exp, shardRoutingEntry.getShardRouting().shardId(), actionName, masterNode, shardRoutingEntry);
-                            listener.onFailure(exp instanceof RemoteTransportException ? exp.getCause() : exp);
+                            listener.onFailure(exp instanceof RemoteTransportException ? (Exception) (exp.getCause() instanceof Exception ? exp.getCause() : new ElasticsearchException(exp.getCause())) : exp);
                         }
                     }
                 });
@@ -131,14 +131,13 @@ public class ShardStateAction extends AbstractComponent {
     /**
      * Send a shard failed request to the master node to update the
      * cluster state.
-     *
-     * @param shardRouting       the shard to fail
+     *  @param shardRouting       the shard to fail
      * @param sourceShardRouting the source shard requesting the failure (must be the shard itself, or the primary shard)
      * @param message            the reason for the failure
      * @param failure            the underlying cause of the failure
      * @param listener           callback upon completion of the request
      */
-    public void shardFailed(final ShardRouting shardRouting, ShardRouting sourceShardRouting, final String message, @Nullable final Throwable failure, Listener listener) {
+    public void shardFailed(final ShardRouting shardRouting, ShardRouting sourceShardRouting, final String message, @Nullable final Exception failure, Listener listener) {
         ClusterStateObserver observer = new ClusterStateObserver(clusterService, null, logger, threadPool.getThreadContext());
         ShardRoutingEntry shardRoutingEntry = new ShardRoutingEntry(shardRouting, sourceShardRouting, message, failure);
         sendShardAction(SHARD_FAILED_ACTION_NAME, observer, shardRoutingEntry, listener);
@@ -190,12 +189,13 @@ public class ShardStateAction extends AbstractComponent {
                 shardFailedClusterStateTaskExecutor,
                 new ClusterStateTaskListener() {
                     @Override
-                    public void onFailure(String source, Throwable t) {
-                        logger.error("{} unexpected failure while failing shard [{}]", t, request.shardRouting.shardId(), request.shardRouting);
+                    public void onFailure(String source, Exception e) {
+                        logger.error("{} unexpected failure while failing shard [{}]", e, request.shardRouting.shardId(), request.shardRouting);
                         try {
-                            channel.sendResponse(t);
-                        } catch (Throwable channelThrowable) {
-                            logger.warn("{} failed to send failure [{}] while failing shard [{}]", channelThrowable, request.shardRouting.shardId(), t, request.shardRouting);
+                            channel.sendResponse(e);
+                        } catch (Exception channelException) {
+                            channelException.addSuppressed(e);
+                            logger.warn("{} failed to send failure [{}] while failing shard [{}]", channelException, request.shardRouting.shardId(), e, request.shardRouting);
                         }
                     }
 
@@ -204,8 +204,8 @@ public class ShardStateAction extends AbstractComponent {
                         logger.error("{} no longer master while failing shard [{}]", request.shardRouting.shardId(), request.shardRouting);
                         try {
                             channel.sendResponse(new NotMasterException(source));
-                        } catch (Throwable channelThrowable) {
-                            logger.warn("{} failed to send no longer master while failing shard [{}]", channelThrowable, request.shardRouting.shardId(), request.shardRouting);
+                        } catch (Exception channelException) {
+                            logger.warn("{} failed to send no longer master while failing shard [{}]", channelException, request.shardRouting.shardId(), request.shardRouting);
                         }
                     }
 
@@ -213,8 +213,8 @@ public class ShardStateAction extends AbstractComponent {
                     public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                         try {
                             channel.sendResponse(TransportResponse.Empty.INSTANCE);
-                        } catch (Throwable channelThrowable) {
-                            logger.warn("{} failed to send response while failing shard [{}]", channelThrowable, request.shardRouting.shardId(), request.shardRouting);
+                        } catch (Exception channelException) {
+                            logger.warn("{} failed to send response while failing shard [{}]", channelException, request.shardRouting.shardId(), request.shardRouting);
                         }
                     }
                 }
@@ -259,10 +259,10 @@ public class ShardStateAction extends AbstractComponent {
                     maybeUpdatedState = ClusterState.builder(currentState).routingResult(result).build();
                 }
                 batchResultBuilder.successes(tasksToFail);
-            } catch (Throwable t) {
+            } catch (Exception e) {
                 // failures are communicated back to the requester
                 // cluster state will not be updated in this case
-                batchResultBuilder.failures(tasksToFail, t);
+                batchResultBuilder.failures(tasksToFail, e);
             }
 
             partition
@@ -379,16 +379,16 @@ public class ShardStateAction extends AbstractComponent {
                     maybeUpdatedState = ClusterState.builder(currentState).routingResult(result).build();
                 }
                 builder.successes(tasks);
-            } catch (Throwable t) {
-                builder.failures(tasks, t);
+            } catch (Exception e) {
+                builder.failures(tasks, e);
             }
 
             return builder.build(maybeUpdatedState);
         }
 
         @Override
-        public void onFailure(String source, Throwable t) {
-            logger.error("unexpected failure during [{}]", t, source);
+        public void onFailure(String source, Exception e) {
+            logger.error("unexpected failure during [{}]", e, source);
         }
     }
 
@@ -396,12 +396,12 @@ public class ShardStateAction extends AbstractComponent {
         ShardRouting shardRouting;
         ShardRouting sourceShardRouting;
         String message;
-        Throwable failure;
+        Exception failure;
 
         public ShardRoutingEntry() {
         }
 
-        ShardRoutingEntry(ShardRouting shardRouting, ShardRouting sourceShardRouting, String message, @Nullable Throwable failure) {
+        ShardRoutingEntry(ShardRouting shardRouting, ShardRouting sourceShardRouting, String message, @Nullable Exception failure) {
             this.shardRouting = shardRouting;
             this.sourceShardRouting = sourceShardRouting;
             this.message = message;
@@ -418,7 +418,7 @@ public class ShardStateAction extends AbstractComponent {
             shardRouting = new ShardRouting(in);
             sourceShardRouting = new ShardRouting(in);
             message = in.readString();
-            failure = in.readThrowable();
+            failure = in.readException();
         }
 
         @Override
@@ -461,9 +461,9 @@ public class ShardStateAction extends AbstractComponent {
          * Any other exception is communicated to the requester via
          * this notification.
          *
-         * @param t the unexpected cause of the failure on the master
+         * @param e the unexpected cause of the failure on the master
          */
-        default void onFailure(final Throwable t) {
+        default void onFailure(final Exception e) {
         }
 
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java
index 647e30cd85c..d98187cc6c1 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java
@@ -84,7 +84,7 @@ public class AliasValidator extends AbstractComponent {
         if (Strings.hasLength(alias.filter())) {
             try (XContentParser parser = XContentFactory.xContent(alias.filter()).createParser(alias.filter())) {
                 parser.map();
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 throw new IllegalArgumentException("failed to parse filter for alias [" + alias.name() + "]", e);
             }
         }
@@ -121,7 +121,7 @@ public class AliasValidator extends AbstractComponent {
         assert queryShardContext != null;
         try (XContentParser parser = XContentFactory.xContent(filter).createParser(filter)) {
             validateAliasFilter(parser, queryShardContext);
-        } catch (Throwable e) {
+        } catch (Exception e) {
             throw new IllegalArgumentException("failed to parse filter for alias [" + alias + "]", e);
         }
     }
@@ -135,7 +135,7 @@ public class AliasValidator extends AbstractComponent {
         assert queryShardContext != null;
         try (XContentParser parser = XContentFactory.xContent(filter).createParser(filter)) {
             validateAliasFilter(parser, queryShardContext);
-        } catch (Throwable e) {
+        } catch (Exception e) {
             throw new IllegalArgumentException("failed to parse filter for alias [" + alias + "]", e);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
index 625f1a0bb56..101f59f3aec 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java
@@ -83,8 +83,8 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
-                listener.onFailure(t);
+            public void onFailure(String source, Exception e) {
+                listener.onFailure(e);
             }
 
             @Override
@@ -134,7 +134,7 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
 
         try {
             validate(request);
-        } catch (Throwable e) {
+        } catch (Exception e) {
             listener.onFailure(e);
             return;
         }
@@ -150,8 +150,8 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
-                listener.onFailure(t);
+            public void onFailure(String source, Exception e) {
+                listener.onFailure(e);
             }
 
             @Override
@@ -280,7 +280,7 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
 
         void onResponse(PutResponse response);
 
-        void onFailure(Throwable t);
+        void onFailure(Exception e);
     }
 
     public static class PutRequest {
@@ -395,6 +395,6 @@ public class MetaDataIndexTemplateService extends AbstractComponent {
 
         void onResponse(RemoveResponse response);
 
-        void onFailure(Throwable t);
+        void onFailure(Exception e);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
index a98f2148da8..f22ac75ef67 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
@@ -187,8 +187,8 @@ public class MetaDataMappingService extends AbstractComponent {
                     builder.putMapping(new MappingMetaData(mapper));
                 }
             }
-        } catch (Throwable t) {
-            logger.warn("[{}] failed to refresh-mapping in cluster state", t, index);
+        } catch (Exception e) {
+            logger.warn("[{}] failed to refresh-mapping in cluster state", e, index);
         }
         return dirty;
     }
@@ -202,7 +202,7 @@ public class MetaDataMappingService extends AbstractComponent {
             refreshTask,
             ClusterStateTaskConfig.build(Priority.HIGH),
             refreshExecutor,
-            (source, t) -> logger.warn("failure during [{}]", t, source)
+            (source, e) -> logger.warn("failure during [{}]", e, source)
         );
     }
 
@@ -233,8 +233,8 @@ public class MetaDataMappingService extends AbstractComponent {
                         }
                         currentState = applyRequest(currentState, request);
                         builder.success(request);
-                    } catch (Throwable t) {
-                        builder.failure(request, t);
+                    } catch (Exception e) {
+                        builder.failure(request, e);
                     }
                 }
                 return builder.build(currentState);
@@ -357,8 +357,8 @@ public class MetaDataMappingService extends AbstractComponent {
                 new AckedClusterStateTaskListener() {
 
                     @Override
-                    public void onFailure(String source, Throwable t) {
-                        listener.onFailure(t);
+                    public void onFailure(String source, Exception e) {
+                        listener.onFailure(e);
                     }
 
                     @Override
@@ -367,7 +367,7 @@ public class MetaDataMappingService extends AbstractComponent {
                     }
 
                     @Override
-                    public void onAllNodesAcked(@Nullable Throwable t) {
+                    public void onAllNodesAcked(@Nullable Exception e) {
                         listener.onResponse(new ClusterStateUpdateResponse(true));
                     }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
index 8bf7b5edd94..9db777a4794 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java
@@ -146,7 +146,7 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
                     }
 
                     @Override
-                    public void onFailure(Throwable t) {
+                    public void onFailure(Exception t) {
                         for (Index index : indices) {
                             logger.warn("{} fail to auto expand replicas to [{}]", index, fNumberOfReplicas);
                         }
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/DelayedAllocationService.java b/core/src/main/java/org/elasticsearch/cluster/routing/DelayedAllocationService.java
index ea0b7e3094c..29d74dd8933 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/DelayedAllocationService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/DelayedAllocationService.java
@@ -96,8 +96,8 @@ public class DelayedAllocationService extends AbstractLifecycleComponent impleme
                 }
 
                 @Override
-                public void onFailure(Throwable t) {
-                    logger.warn("failed to submit schedule/execute reroute post unassigned shard", t);
+                public void onFailure(Exception e) {
+                    logger.warn("failed to submit schedule/execute reroute post unassigned shard", e);
                     removeIfSameTask(DelayedRerouteTask.this);
                 }
             });
@@ -125,9 +125,9 @@ public class DelayedAllocationService extends AbstractLifecycleComponent impleme
         }
 
         @Override
-        public void onFailure(String source, Throwable t) {
+        public void onFailure(String source, Exception e) {
             removeIfSameTask(this);
-            logger.warn("failed to schedule/execute reroute post unassigned shard", t);
+            logger.warn("failed to schedule/execute reroute post unassigned shard", e);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
index 163057b9b46..cfe48dd711e 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java
@@ -109,17 +109,17 @@ public class RoutingService extends AbstractLifecycleComponent {
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
+                public void onFailure(String source, Exception e) {
                     rerouting.set(false);
                     ClusterState state = clusterService.state();
                     if (logger.isTraceEnabled()) {
-                        logger.error("unexpected failure during [{}], current state:\n{}", t, source, state.prettyPrint());
+                        logger.error("unexpected failure during [{}], current state:\n{}", e, source, state.prettyPrint());
                     } else {
-                        logger.error("unexpected failure during [{}], current state version [{}]", t, source, state.version());
+                        logger.error("unexpected failure during [{}], current state version [{}]", e, source, state.version());
                     }
                 }
             });
-        } catch (Throwable e) {
+        } catch (Exception e) {
             rerouting.set(false);
             ClusterState state = clusterService.state();
             logger.warn("failed to reroute routing table, current state:\n{}", e, state.prettyPrint());
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java b/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
index 5d6333972b3..cfa7c771d50 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
@@ -21,7 +21,6 @@ package org.elasticsearch.cluster.routing;
 
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -114,7 +113,7 @@ public final class UnassignedInfo implements ToXContent, Writeable {
     private final long unassignedTimeNanos; // in nanoseconds, used to calculate delay for delayed shard allocation
     private final boolean delayed; // if allocation of this shard is delayed due to INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING
     private final String message;
-    private final Throwable failure;
+    private final Exception failure;
     private final int failedAllocations;
 
     /**
@@ -135,8 +134,8 @@ public final class UnassignedInfo implements ToXContent, Writeable {
      * @param unassignedTimeMillis the time of unassignment used to display to in our reporting.
      * @param delayed              if allocation of this shard is delayed due to INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.
      */
-    public UnassignedInfo(Reason reason, @Nullable String message, @Nullable Throwable failure, int failedAllocations,
-        long unassignedTimeNanos, long unassignedTimeMillis, boolean delayed) {
+    public UnassignedInfo(Reason reason, @Nullable String message, @Nullable Exception failure, int failedAllocations,
+                          long unassignedTimeNanos, long unassignedTimeMillis, boolean delayed) {
         this.reason = reason;
         this.unassignedTimeMillis = unassignedTimeMillis;
         this.unassignedTimeNanos = unassignedTimeNanos;
@@ -158,7 +157,7 @@ public final class UnassignedInfo implements ToXContent, Writeable {
         this.unassignedTimeNanos = System.nanoTime();
         this.delayed = in.readBoolean();
         this.message = in.readOptionalString();
-        this.failure = in.readThrowable();
+        this.failure = in.readException();
         this.failedAllocations = in.readVInt();
     }
 
@@ -226,7 +225,7 @@ public final class UnassignedInfo implements ToXContent, Writeable {
      * Returns additional failure exception details if exists.
      */
     @Nullable
-    public Throwable getFailure() {
+    public Exception getFailure() {
         return failure;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java
index b1b0dfce1fe..c3a397a785b 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java
@@ -41,9 +41,9 @@ public class FailedRerouteAllocation extends RoutingAllocation {
     public static class FailedShard {
         public final ShardRouting shard;
         public final String message;
-        public final Throwable failure;
+        public final Exception failure;
 
-        public FailedShard(ShardRouting shard, String message, Throwable failure) {
+        public FailedShard(ShardRouting shard, String message, Exception failure) {
             this.shard = shard;
             this.message = message;
             this.failure = failure;
diff --git a/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
index accb9304d25..10cf6216a86 100644
--- a/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
@@ -547,7 +547,7 @@ public class ClusterService extends AbstractLifecycleComponent {
         try {
             List<T> inputs = toExecute.stream().map(tUpdateTask -> tUpdateTask.task).collect(Collectors.toList());
             batchResult = executor.execute(previousClusterState, inputs);
-        } catch (Throwable e) {
+        } catch (Exception e) {
             TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
             if (logger.isTraceEnabled()) {
                 logger.trace("failed to execute cluster state update in [{}], state:\nversion [{}], source [{}]\n{}{}{}", e, executionTime,
@@ -707,8 +707,8 @@ public class ClusterService extends AbstractLifecycleComponent {
             if (newClusterState.nodes().isLocalNodeElectedMaster()) {
                 try {
                     ackListener.onNodeAck(newClusterState.nodes().getLocalNode(), null);
-                } catch (Throwable t) {
-                    logger.debug("error while processing ack for master node [{}]", t, newClusterState.nodes().getLocalNode());
+                } catch (Exception e) {
+                    logger.debug("error while processing ack for master node [{}]", e, newClusterState.nodes().getLocalNode());
                 }
             }
 
@@ -726,9 +726,9 @@ public class ClusterService extends AbstractLifecycleComponent {
             logger.debug("processing [{}]: took [{}] done applying updated cluster_state (version: {}, uuid: {})", source, executionTime,
                     newClusterState.version(), newClusterState.stateUUID());
             warnAboutSlowTaskIfNeeded(executionTime, source);
-        } catch (Throwable t) {
+        } catch (Exception e) {
             TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
-            logger.warn("failed to apply updated cluster state in [{}]:\nversion [{}], uuid [{}], source [{}]\n{}", t, executionTime,
+            logger.warn("failed to apply updated cluster state in [{}]:\nversion [{}], uuid [{}], source [{}]\n{}", e, executionTime,
                     newClusterState.version(), newClusterState.stateUUID(), source, newClusterState.prettyPrint());
             // TODO: do we want to call updateTask.onFailure here?
         }
@@ -756,11 +756,12 @@ public class ClusterService extends AbstractLifecycleComponent {
         }
 
         @Override
-        public void onFailure(String source, Throwable t) {
+        public void onFailure(String source, Exception e) {
             try {
-                listener.onFailure(source, t);
-            } catch (Exception e) {
-                logger.error("exception thrown by listener notifying of failure [{}] from [{}]", e, t, source);
+                listener.onFailure(source, e);
+            } catch (Exception inner) {
+                inner.addSuppressed(e);
+                logger.error("exception thrown by listener notifying of failure from [{}]", inner, source);
             }
         }
 
@@ -805,11 +806,12 @@ public class ClusterService extends AbstractLifecycleComponent {
         }
 
         @Override
-        public void onAllNodesAcked(@Nullable Throwable t) {
+        public void onAllNodesAcked(@Nullable Exception e) {
             try {
-                listener.onAllNodesAcked(t);
-            } catch (Exception e) {
-                logger.error("exception thrown by listener while notifying on all nodes acked [{}]", e, t);
+                listener.onAllNodesAcked(e);
+            } catch (Exception inner) {
+                inner.addSuppressed(e);
+                logger.error("exception thrown by listener while notifying on all nodes acked", inner);
             }
         }
 
@@ -966,9 +968,9 @@ public class ClusterService extends AbstractLifecycleComponent {
         }
 
         @Override
-        public void onNodeAck(DiscoveryNode node, @Nullable Throwable t) {
+        public void onNodeAck(DiscoveryNode node, @Nullable Exception e) {
             for (Discovery.AckListener listener : listeners) {
-                listener.onNodeAck(node, t);
+                listener.onNodeAck(node, e);
             }
         }
 
@@ -987,7 +989,7 @@ public class ClusterService extends AbstractLifecycleComponent {
         private final DiscoveryNodes nodes;
         private final long clusterStateVersion;
         private final Future<?> ackTimeoutCallback;
-        private Throwable lastFailure;
+        private Exception lastFailure;
 
         AckCountDownListener(AckedClusterStateTaskListener ackedTaskListener, long clusterStateVersion, DiscoveryNodes nodes,
                              ThreadPool threadPool) {
@@ -1013,18 +1015,18 @@ public class ClusterService extends AbstractLifecycleComponent {
         }
 
         @Override
-        public void onNodeAck(DiscoveryNode node, @Nullable Throwable t) {
+        public void onNodeAck(DiscoveryNode node, @Nullable Exception e) {
             if (!ackedTaskListener.mustAck(node)) {
                 //we always wait for the master ack anyway
                 if (!node.equals(nodes.getMasterNode())) {
                     return;
                 }
             }
-            if (t == null) {
+            if (e == null) {
                 logger.trace("ack received from node [{}], cluster_state update (version: {})", node, clusterStateVersion);
             } else {
-                this.lastFailure = t;
-                logger.debug("ack received from node [{}], cluster_state update (version: {})", t, node, clusterStateVersion);
+                this.lastFailure = e;
+                logger.debug("ack received from node [{}], cluster_state update (version: {})", e, node, clusterStateVersion);
             }
 
             if (countDown.countDown()) {
diff --git a/core/src/main/java/org/elasticsearch/common/MacAddressProvider.java b/core/src/main/java/org/elasticsearch/common/MacAddressProvider.java
index 52ebc283f3c..1c6d7b3945b 100644
--- a/core/src/main/java/org/elasticsearch/common/MacAddressProvider.java
+++ b/core/src/main/java/org/elasticsearch/common/MacAddressProvider.java
@@ -59,7 +59,7 @@ public class MacAddressProvider {
         byte[] address = null;
         try {
             address = getMacAddress();
-        } catch (Throwable t) {
+        } catch (SocketException e) {
             // address will be set below
         }
 
diff --git a/core/src/main/java/org/elasticsearch/common/Randomness.java b/core/src/main/java/org/elasticsearch/common/Randomness.java
index ddcd3ea90f7..295d7a6bcda 100644
--- a/core/src/main/java/org/elasticsearch/common/Randomness.java
+++ b/core/src/main/java/org/elasticsearch/common/Randomness.java
@@ -54,7 +54,7 @@ public final class Randomness {
             Class<?> clazz = Class.forName("com.carrotsearch.randomizedtesting.RandomizedContext");
             maybeCurrentMethod = clazz.getMethod("current");
             maybeGetRandomMethod = clazz.getMethod("getRandom");
-        } catch (Throwable t) {
+        } catch (Exception e) {
             maybeCurrentMethod = null;
             maybeGetRandomMethod = null;
         }
diff --git a/core/src/main/java/org/elasticsearch/common/geo/ShapesAvailability.java b/core/src/main/java/org/elasticsearch/common/geo/ShapesAvailability.java
index dff4277e96f..fe33fadb857 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/ShapesAvailability.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/ShapesAvailability.java
@@ -31,7 +31,7 @@ public class ShapesAvailability {
         try {
             Class.forName("org.locationtech.spatial4j.shape.impl.PointImpl");
             xSPATIAL4J_AVAILABLE = true;
-        } catch (Throwable t) {
+        } catch (ClassNotFoundException ignored) {
             xSPATIAL4J_AVAILABLE = false;
         }
         SPATIAL4J_AVAILABLE = xSPATIAL4J_AVAILABLE;
@@ -40,7 +40,7 @@ public class ShapesAvailability {
         try {
             Class.forName("com.vividsolutions.jts.geom.GeometryFactory");
             xJTS_AVAILABLE = true;
-        } catch (Throwable t) {
+        } catch (ClassNotFoundException ignored) {
             xJTS_AVAILABLE = false;
         }
         JTS_AVAILABLE = xJTS_AVAILABLE;
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
index ab22df57920..1b28adb1cb1 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
@@ -27,6 +27,7 @@ import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.util.BitUtil;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CharsRefBuilder;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
@@ -60,7 +61,6 @@ import java.util.Map;
 import java.util.function.IntFunction;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.ElasticsearchException.readException;
 import static org.elasticsearch.ElasticsearchException.readStackTrace;
 
 /**
@@ -648,17 +648,17 @@ public abstract class StreamInput extends InputStream {
         }
     }
 
-    public <T extends Throwable> T readThrowable() throws IOException {
+    public <T extends Exception> T readException() throws IOException {
         if (readBoolean()) {
             int key = readVInt();
             switch (key) {
                 case 0:
                     final int ord = readVInt();
-                    return (T) readException(this, ord);
+                    return (T) ElasticsearchException.readException(this, ord);
                 case 1:
                     String msg1 = readOptionalString();
                     String resource1 = readOptionalString();
-                    return (T) readStackTrace(new CorruptIndexException(msg1, resource1, readThrowable()), this);
+                    return (T) readStackTrace(new CorruptIndexException(msg1, resource1, readException()), this);
                 case 2:
                     String resource2 = readOptionalString();
                     int version2 = readInt();
@@ -681,67 +681,63 @@ public abstract class StreamInput extends InputStream {
                 case 5:
                     return (T) readStackTrace(new NumberFormatException(readOptionalString()), this);
                 case 6:
-                    return (T) readStackTrace(new IllegalArgumentException(readOptionalString(), readThrowable()), this);
+                    return (T) readStackTrace(new IllegalArgumentException(readOptionalString(), readException()), this);
                 case 7:
-                    return (T) readStackTrace(new AlreadyClosedException(readOptionalString(), readThrowable()), this);
+                    return (T) readStackTrace(new AlreadyClosedException(readOptionalString(), readException()), this);
                 case 8:
                     return (T) readStackTrace(new EOFException(readOptionalString()), this);
                 case 9:
-                    return (T) readStackTrace(new SecurityException(readOptionalString(), readThrowable()), this);
+                    return (T) readStackTrace(new SecurityException(readOptionalString(), readException()), this);
                 case 10:
                     return (T) readStackTrace(new StringIndexOutOfBoundsException(readOptionalString()), this);
                 case 11:
                     return (T) readStackTrace(new ArrayIndexOutOfBoundsException(readOptionalString()), this);
                 case 12:
-                    return (T) readStackTrace(new AssertionError(readOptionalString(), readThrowable()), this);
-                case 13:
                     return (T) readStackTrace(new FileNotFoundException(readOptionalString()), this);
-                case 14:
+                case 13:
                     final int subclass = readVInt();
                     final String file = readOptionalString();
                     final String other = readOptionalString();
                     final String reason = readOptionalString();
                     readOptionalString(); // skip the msg - it's composed from file, other and reason
-                    final Throwable throwable;
+                    final Exception exception;
                     switch (subclass) {
                         case 0:
-                            throwable = new NoSuchFileException(file, other, reason);
+                            exception = new NoSuchFileException(file, other, reason);
                             break;
                         case 1:
-                            throwable = new NotDirectoryException(file);
+                            exception = new NotDirectoryException(file);
                             break;
                         case 2:
-                            throwable = new DirectoryNotEmptyException(file);
+                            exception = new DirectoryNotEmptyException(file);
                             break;
                         case 3:
-                            throwable = new AtomicMoveNotSupportedException(file, other, reason);
+                            exception = new AtomicMoveNotSupportedException(file, other, reason);
                             break;
                         case 4:
-                            throwable = new FileAlreadyExistsException(file, other, reason);
+                            exception = new FileAlreadyExistsException(file, other, reason);
                             break;
                         case 5:
-                            throwable = new AccessDeniedException(file, other, reason);
+                            exception = new AccessDeniedException(file, other, reason);
                             break;
                         case 6:
-                            throwable = new FileSystemLoopException(file);
+                            exception = new FileSystemLoopException(file);
                             break;
                         case 7:
-                            throwable = new FileSystemException(file, other, reason);
+                            exception = new FileSystemException(file, other, reason);
                             break;
                         default:
                             throw new IllegalStateException("unknown FileSystemException with index " + subclass);
                     }
-                    return (T) readStackTrace(throwable, this);
+                    return (T) readStackTrace(exception, this);
+                case 14:
+                    return (T) readStackTrace(new IllegalStateException(readOptionalString(), readException()), this);
                 case 15:
-                    return (T) readStackTrace(new OutOfMemoryError(readOptionalString()), this);
+                    return (T) readStackTrace(new LockObtainFailedException(readOptionalString(), readException()), this);
                 case 16:
-                    return (T) readStackTrace(new IllegalStateException(readOptionalString(), readThrowable()), this);
-                case 17:
-                    return (T) readStackTrace(new LockObtainFailedException(readOptionalString(), readThrowable()), this);
-                case 18:
                     return (T) readStackTrace(new InterruptedException(readOptionalString()), this);
-                case 19:
-                    return (T) readStackTrace(new IOException(readOptionalString(), readThrowable()), this);
+                case 17:
+                    return (T) readStackTrace(new IOException(readOptionalString(), readException()), this);
                 default:
                     assert false : "no such exception for id: " + key;
             }
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
index 4ee9c80b891..653dd9676b3 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
@@ -687,13 +687,11 @@ public abstract class StreamOutput extends OutputStream {
             } else if (throwable instanceof ArrayIndexOutOfBoundsException) {
                 writeVInt(11);
                 writeCause = false;
-            } else if (throwable instanceof AssertionError) {
-                writeVInt(12);
             } else if (throwable instanceof FileNotFoundException) {
-                writeVInt(13);
+                writeVInt(12);
                 writeCause = false;
             } else if (throwable instanceof FileSystemException) {
-                writeVInt(14);
+                writeVInt(13);
                 if (throwable instanceof NoSuchFileException) {
                     writeVInt(0);
                 } else if (throwable instanceof NotDirectoryException) {
@@ -715,18 +713,15 @@ public abstract class StreamOutput extends OutputStream {
                 writeOptionalString(((FileSystemException) throwable).getOtherFile());
                 writeOptionalString(((FileSystemException) throwable).getReason());
                 writeCause = false;
-            } else if (throwable instanceof OutOfMemoryError) {
-                writeVInt(15);
-                writeCause = false;
             } else if (throwable instanceof IllegalStateException) {
-                writeVInt(16);
+                writeVInt(14);
             } else if (throwable instanceof LockObtainFailedException) {
-                writeVInt(17);
+                writeVInt(15);
             } else if (throwable instanceof InterruptedException) {
-                writeVInt(18);
+                writeVInt(16);
                 writeCause = false;
             } else if (throwable instanceof IOException) {
-                writeVInt(19);
+                writeVInt(17);
             } else {
                 ElasticsearchException ex;
                 if (throwable instanceof ElasticsearchException && ElasticsearchException.isRegistered(throwable.getClass())) {
diff --git a/core/src/main/java/org/elasticsearch/common/lease/Releasables.java b/core/src/main/java/org/elasticsearch/common/lease/Releasables.java
index 5e2d5af522d..bfabd20976d 100644
--- a/core/src/main/java/org/elasticsearch/common/lease/Releasables.java
+++ b/core/src/main/java/org/elasticsearch/common/lease/Releasables.java
@@ -22,6 +22,7 @@ package org.elasticsearch.common.lease;
 import org.apache.lucene.util.IOUtils;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.Arrays;
 
 /** Utility methods to work with {@link Releasable}s. */
@@ -32,9 +33,9 @@ public enum Releasables {
         try {
             // this does the right thing with respect to add suppressed and not wrapping errors etc.
             IOUtils.close(releasables);
-        } catch (Throwable t) {
+        } catch (IOException e) {
             if (ignoreException == false) {
-                IOUtils.reThrowUnchecked(t);
+                throw new UncheckedIOException(e);
             }
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/common/logging/LoggerMessageFormat.java b/core/src/main/java/org/elasticsearch/common/logging/LoggerMessageFormat.java
index f1e135d645c..3b85a42ce96 100644
--- a/core/src/main/java/org/elasticsearch/common/logging/LoggerMessageFormat.java
+++ b/core/src/main/java/org/elasticsearch/common/logging/LoggerMessageFormat.java
@@ -153,7 +153,7 @@ public class LoggerMessageFormat {
         try {
             String oAsString = o.toString();
             sbuf.append(oAsString);
-        } catch (Throwable t) {
+        } catch (Exception e) {
             sbuf.append("[FAILED toString()]");
         }
 
diff --git a/core/src/main/java/org/elasticsearch/common/network/Cidrs.java b/core/src/main/java/org/elasticsearch/common/network/Cidrs.java
index f0bd4fb74c8..1bdd7bf562b 100644
--- a/core/src/main/java/org/elasticsearch/common/network/Cidrs.java
+++ b/core/src/main/java/org/elasticsearch/common/network/Cidrs.java
@@ -52,9 +52,9 @@ public final class Cidrs {
         byte[] addressBytes;
         try {
             addressBytes = InetAddresses.forString(fields[0]).getAddress();
-        } catch (Throwable t) {
+        } catch (Exception e) {
             throw new IllegalArgumentException(
-                    String.format(Locale.ROOT, "invalid IPv4/CIDR; unable to parse [%s] as an IP address literal", fields[0]), t
+                    String.format(Locale.ROOT, "invalid IPv4/CIDR; unable to parse [%s] as an IP address literal", fields[0]), e
             );
         }
         long accumulator =
diff --git a/core/src/main/java/org/elasticsearch/common/util/CancellableThreads.java b/core/src/main/java/org/elasticsearch/common/util/CancellableThreads.java
index b6edb2db8da..2e5ee110d60 100644
--- a/core/src/main/java/org/elasticsearch/common/util/CancellableThreads.java
+++ b/core/src/main/java/org/elasticsearch/common/util/CancellableThreads.java
@@ -42,7 +42,7 @@ public class CancellableThreads {
     }
 
 
-    /** call this will throw an exception if operation was cancelled. Override {@link #onCancel(String, java.lang.Throwable)} for custom failure logic */
+    /** call this will throw an exception if operation was cancelled. Override {@link #onCancel(String, Exception)} for custom failure logic */
     public synchronized void checkForCancel() {
         if (isCancelled()) {
             onCancel(reason, null);
@@ -53,11 +53,10 @@ public class CancellableThreads {
      * called if {@link #checkForCancel()} was invoked after the operation was cancelled.
      * the default implementation always throws an {@link ExecutionCancelledException}, suppressing
      * any other exception that occurred before cancellation
-     *
-     * @param reason              reason for failure supplied by the caller of {@link #cancel}
+     *  @param reason              reason for failure supplied by the caller of {@link #cancel}
      * @param suppressedException any error that was encountered during the execution before the operation was cancelled.
      */
-    protected void onCancel(String reason, @Nullable Throwable suppressedException) {
+    protected void onCancel(String reason, @Nullable Exception suppressedException) {
         RuntimeException e = new ExecutionCancelledException("operation was cancelled reason [" + reason + "]");
         if (suppressedException != null) {
             e.addSuppressed(suppressedException);
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/AbstractRunnable.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/AbstractRunnable.java
index b63cc2e31ca..e2bde346f6f 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/AbstractRunnable.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/AbstractRunnable.java
@@ -35,7 +35,7 @@ public abstract class AbstractRunnable implements Runnable {
     public final void run() {
         try {
             doRun();
-        } catch (Throwable t) {
+        } catch (Exception t) {
             onFailure(t);
         } finally {
             onAfter();
@@ -53,14 +53,14 @@ public abstract class AbstractRunnable implements Runnable {
     /**
      * This method is invoked for all exception thrown by {@link #doRun()}
      */
-    public abstract void onFailure(Throwable t);
+    public abstract void onFailure(Exception e);
 
     /**
      * This should be executed if the thread-pool executing this action rejected the execution.
-     * The default implementation forwards to {@link #onFailure(Throwable)}
+     * The default implementation forwards to {@link #onFailure(Exception)}
      */
-    public void onRejection(Throwable t) {
-        onFailure(t);
+    public void onRejection(Exception e) {
+        onFailure(e);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadBarrier.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadBarrier.java
index a3d10ff825f..967f0c890d2 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadBarrier.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadBarrier.java
@@ -24,7 +24,6 @@ import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
-
 /**
  * A synchronization aid that allows a set of threads to all wait for each other
  * to reach a common barrier point. Barriers are useful in programs involving a
@@ -55,9 +54,9 @@ import java.util.concurrent.TimeoutException;
  *                  process();
  *                  barrier.await();    //wait for all threads to process
  *                }
- *              catch(Throwable t){
- *                  log(&quot;Worker thread caught exception&quot;, t);
- *                  barrier.reset(t);
+ *              catch(Exception e){
+ *                  log(&quot;Worker thread caught exception&quot;, e);
+ *                  barrier.reset(e);
  *                }
  *            }
  *        }
@@ -88,9 +87,9 @@ import java.util.concurrent.TimeoutException;
  *               Assert.assertFalse(&quot;Exceeded notification count&quot;,
  *                                          actualNotificationCount &gt; EXPECTED_COUNT);
  *            }
- *          catch(Throwable t) {
- *              log(&quot;Worker thread caught exception&quot;, t);
- *              barrier.reset(t);
+ *          catch(Exception e) {
+ *              log(&quot;Worker thread caught exception&quot;, e);
+ *              barrier.reset(e);
  *            }
  *        }
  *
@@ -118,9 +117,9 @@ import java.util.concurrent.TimeoutException;
 public class ThreadBarrier extends CyclicBarrier {
     /**
      * The cause of a {@link BrokenBarrierException} and {@link TimeoutException}
-     * thrown from an await() when {@link #reset(Throwable)} was invoked.
+     * thrown from an await() when {@link #reset(Exception)} was invoked.
      */
-    private Throwable cause;
+    private Exception cause;
 
     public ThreadBarrier(int parties) {
         super(parties);
@@ -166,7 +165,7 @@ public class ThreadBarrier extends CyclicBarrier {
      *
      * @param cause The cause of the BrokenBarrierException
      */
-    public synchronized void reset(Throwable cause) {
+    public synchronized void reset(Exception cause) {
         if (!isBroken()) {
             super.reset();
         }
@@ -178,7 +177,7 @@ public class ThreadBarrier extends CyclicBarrier {
 
     /**
      * Queries if this barrier is in a broken state. Note that if
-     * {@link #reset(Throwable)} is invoked the barrier will remain broken, while
+     * {@link #reset(Exception)} is invoked the barrier will remain broken, while
      * {@link #reset()} will reset the barrier to its initial state and
      * {@link #isBroken()} will return false.
      *
@@ -229,7 +228,7 @@ public class ThreadBarrier extends CyclicBarrier {
 
     /**
      * Initializes the cause of this throwable to the specified value. The cause
-     * is the throwable that was initialized by {@link #reset(Throwable)}.
+     * is the throwable that was initialized by {@link #reset(Exception)}.
      *
      * @param t throwable.
      */
@@ -301,4 +300,5 @@ public class ThreadBarrier extends CyclicBarrier {
             return (time) / 1000000000.0;
         }
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
index c715fceb84a..457dcad07cf 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
@@ -433,13 +433,13 @@ public final class ThreadContext implements Closeable, Writeable {
         }
 
         @Override
-        public void onFailure(Throwable t) {
-            in.onFailure(t);
+        public void onFailure(Exception e) {
+            in.onFailure(e);
         }
 
         @Override
-        public void onRejection(Throwable t) {
-            in.onRejection(t);
+        public void onRejection(Exception e) {
+            in.onRejection(e);
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/discovery/AckClusterStatePublishResponseHandler.java b/core/src/main/java/org/elasticsearch/discovery/AckClusterStatePublishResponseHandler.java
index 24657981c43..26e78989591 100644
--- a/core/src/main/java/org/elasticsearch/discovery/AckClusterStatePublishResponseHandler.java
+++ b/core/src/main/java/org/elasticsearch/discovery/AckClusterStatePublishResponseHandler.java
@@ -55,19 +55,20 @@ public class AckClusterStatePublishResponseHandler extends BlockingClusterStateP
     }
 
     @Override
-    public void onFailure(DiscoveryNode node, Throwable t) {
+    public void onFailure(DiscoveryNode node, Exception e) {
         try {
-            super.onFailure(node, t);
+            super.onFailure(node, e);
         } finally {
-            onNodeAck(ackListener, node, t);
+            onNodeAck(ackListener, node, e);
         }
     }
 
-    private void onNodeAck(final Discovery.AckListener ackListener, DiscoveryNode node, Throwable t) {
+    private void onNodeAck(final Discovery.AckListener ackListener, DiscoveryNode node, Exception e) {
         try {
-            ackListener.onNodeAck(node, t);
-        } catch (Throwable t1) {
-            logger.debug("error while processing ack for node [{}]", t1, node);
+            ackListener.onNodeAck(node, e);
+        } catch (Exception inner) {
+            inner.addSuppressed(e);
+            logger.debug("error while processing ack for node [{}]", inner, node);
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandler.java b/core/src/main/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandler.java
index 3fe99face52..72e59675c1b 100644
--- a/core/src/main/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandler.java
+++ b/core/src/main/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandler.java
@@ -61,7 +61,7 @@ public class BlockingClusterStatePublishResponseHandler {
      * Called for each failure obtained from non master nodes
      * @param node the node that replied to the publish event
      */
-    public void onFailure(DiscoveryNode node, Throwable t) {
+    public void onFailure(DiscoveryNode node, Exception e) {
         boolean found = pendingNodes.remove(node);
         assert found : "node [" + node + "] already responded or failed";
         latch.countDown();
diff --git a/core/src/main/java/org/elasticsearch/discovery/Discovery.java b/core/src/main/java/org/elasticsearch/discovery/Discovery.java
index d7cfb2eeae6..e4addfdd02f 100644
--- a/core/src/main/java/org/elasticsearch/discovery/Discovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/Discovery.java
@@ -59,7 +59,7 @@ public interface Discovery extends LifecycleComponent {
     void publish(ClusterChangedEvent clusterChangedEvent, AckListener ackListener);
 
     interface AckListener {
-        void onNodeAck(DiscoveryNode node, @Nullable Throwable t);
+        void onNodeAck(DiscoveryNode node, @Nullable Exception e);
         void onTimeout();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
index 65767e5e86d..5f8a81ec994 100644
--- a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
@@ -134,8 +134,8 @@ public class LocalDiscovery extends AbstractLifecycleComponent implements Discov
                     }
 
                     @Override
-                    public void onFailure(String source, Throwable t) {
-                        logger.error("unexpected failure during [{}]", t, source);
+                    public void onFailure(String source, Exception e) {
+                        logger.error("unexpected failure during [{}]", e, source);
                     }
                 });
             } else if (firstMaster != null) {
@@ -163,8 +163,8 @@ public class LocalDiscovery extends AbstractLifecycleComponent implements Discov
                     }
 
                     @Override
-                    public void onFailure(String source, Throwable t) {
-                        logger.error("unexpected failure during [{}]", t, source);
+                    public void onFailure(String source, Exception e) {
+                        logger.error("unexpected failure during [{}]", e, source);
                     }
 
                 });
@@ -228,8 +228,8 @@ public class LocalDiscovery extends AbstractLifecycleComponent implements Discov
                     }
 
                     @Override
-                    public void onFailure(String source, Throwable t) {
-                        logger.error("unexpected failure during [{}]", t, source);
+                    public void onFailure(String source, Exception e) {
+                        logger.error("unexpected failure during [{}]", e, source);
                     }
                 });
             }
@@ -370,9 +370,9 @@ public class LocalDiscovery extends AbstractLifecycleComponent implements Discov
                         }
 
                         @Override
-                        public void onFailure(String source, Throwable t) {
-                            logger.error("unexpected failure during [{}]", t, source);
-                            publishResponseHandler.onFailure(discovery.localNode(), t);
+                        public void onFailure(String source, Exception e) {
+                            logger.error("unexpected failure during [{}]", e, source);
+                            publishResponseHandler.onFailure(discovery.localNode(), e);
                         }
 
                         @Override
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
index 57cfe7e1da5..f0b192d01d0 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
@@ -131,10 +131,10 @@ public class NodeJoinController extends AbstractComponent {
                 logger.trace("timed out waiting to be elected. waited [{}]. pending master node joins [{}]", timeValue, pendingNodes);
             }
             failContextIfNeeded(myElectionContext, "timed out waiting to be elected");
-        } catch (Throwable t) {
-            logger.error("unexpected failure while waiting for incoming joins", t);
+        } catch (Exception e) {
+            logger.error("unexpected failure while waiting for incoming joins", e);
             if (myElectionContext != null) {
-                failContextIfNeeded(myElectionContext, "unexpected failure while waiting for pending joins [" + t.getMessage() + "]");
+                failContextIfNeeded(myElectionContext, "unexpected failure while waiting for pending joins [" + e.getMessage() + "]");
             }
         }
     }
@@ -336,8 +336,8 @@ public class NodeJoinController extends AbstractComponent {
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
-                ElectionContext.this.onFailure(t);
+            public void onFailure(String source, Exception e) {
+                ElectionContext.this.onFailure(e);
             }
         };
 
@@ -357,12 +357,12 @@ public class NodeJoinController extends AbstractComponent {
         }
 
         @Override
-        public void onFailure(String source, Throwable t) {
+        public void onFailure(String source, Exception e) {
             for (MembershipAction.JoinCallback callback : callbacks) {
                 try {
-                    callback.onFailure(t);
-                } catch (Exception e) {
-                    logger.error("error during task failure", e);
+                    callback.onFailure(e);
+                } catch (Exception inner) {
+                    logger.error("error handling task failure [{}]", inner, e);
                 }
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
index 598eb2d6875..e052362b5e2 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
@@ -236,8 +236,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
             }
 
             @Override
-            public void onFailure(String source, @org.elasticsearch.common.Nullable Throwable t) {
-                logger.warn("failed to start initial join process", t);
+            public void onFailure(String source, @org.elasticsearch.common.Nullable Exception e) {
+                logger.warn("failed to start initial join process", e);
             }
         });
     }
@@ -326,8 +326,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
-                    logger.error("unexpected failure during [{}]", t, source);
+                public void onFailure(String source, Exception e) {
+                    logger.error("unexpected failure during [{}]", e, source);
                 }
 
             });
@@ -446,8 +446,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
                 }
 
                 @Override
-                public void onFailure(String source, @Nullable Throwable t) {
-                    logger.error("unexpected error while trying to finalize cluster join", t);
+                public void onFailure(String source, @Nullable Exception e) {
+                    logger.error("unexpected error while trying to finalize cluster join", e);
                     joinThreadControl.markThreadAsDoneAndStartNew(currentThread);
                 }
             });
@@ -473,20 +473,20 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
                 logger.trace("joining master {}", masterNode);
                 membership.sendJoinRequestBlocking(masterNode, clusterService.localNode(), joinTimeout);
                 return true;
-            } catch (Throwable t) {
-                Throwable unwrap = ExceptionsHelper.unwrapCause(t);
+            } catch (Exception e) {
+                final Throwable unwrap = ExceptionsHelper.unwrapCause(e);
                 if (unwrap instanceof NotMasterException) {
                     if (++joinAttempt == this.joinRetryAttempts) {
-                        logger.info("failed to send join request to master [{}], reason [{}], tried [{}] times", masterNode, ExceptionsHelper.detailedMessage(t), joinAttempt);
+                        logger.info("failed to send join request to master [{}], reason [{}], tried [{}] times", masterNode, ExceptionsHelper.detailedMessage(e), joinAttempt);
                         return false;
                     } else {
-                        logger.trace("master {} failed with [{}]. retrying... (attempts done: [{}])", masterNode, ExceptionsHelper.detailedMessage(t), joinAttempt);
+                        logger.trace("master {} failed with [{}]. retrying... (attempts done: [{}])", masterNode, ExceptionsHelper.detailedMessage(e), joinAttempt);
                     }
                 } else {
                     if (logger.isTraceEnabled()) {
-                        logger.trace("failed to send join request to master [{}]", t, masterNode);
+                        logger.trace("failed to send join request to master [{}]", e, masterNode);
                     } else {
-                        logger.info("failed to send join request to master [{}], reason [{}]", masterNode, ExceptionsHelper.detailedMessage(t));
+                        logger.info("failed to send join request to master [{}], reason [{}]", masterNode, ExceptionsHelper.detailedMessage(e));
                     }
                     return false;
                 }
@@ -527,8 +527,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
-                    logger.error("unexpected failure during [{}]", t, source);
+                public void onFailure(String source, Exception e) {
+                    logger.error("unexpected failure during [{}]", e, source);
                 }
             });
         } else if (node.equals(nodes().getMasterNode())) {
@@ -572,8 +572,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
-                logger.error("unexpected failure during [{}]", t, source);
+            public void onFailure(String source, Exception e) {
+                logger.error("unexpected failure during [{}]", e, source);
             }
 
             @Override
@@ -610,8 +610,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
-                logger.error("unexpected failure during [{}]", t, source);
+            public void onFailure(String source, Exception e) {
+                logger.error("unexpected failure during [{}]", e, source);
             }
 
             @Override
@@ -659,8 +659,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
-                logger.error("unexpected failure during [{}]", t, source);
+            public void onFailure(String source, Exception e) {
+                logger.error("unexpected failure during [{}]", e, source);
             }
 
             @Override
@@ -744,13 +744,14 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
-                logger.error("unexpected failure during [{}]", t, source);
+            public void onFailure(String source, Exception e) {
+                logger.error("unexpected failure during [{}]", e, source);
                 if (newClusterState != null) {
                     try {
-                        publishClusterState.pendingStatesQueue().markAsFailed(newClusterState, t);
-                    } catch (Throwable unexpected) {
-                        logger.error("unexpected exception while failing [{}]", unexpected, source);
+                        publishClusterState.pendingStatesQueue().markAsFailed(newClusterState, e);
+                    } catch (Exception inner) {
+                        inner.addSuppressed(e);
+                        logger.error("unexpected exception while failing [{}]", inner, source);
                     }
                 }
             }
@@ -761,8 +762,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
                     if (newClusterState != null) {
                         publishClusterState.pendingStatesQueue().markAsProcessed(newClusterState);
                     }
-                } catch (Throwable t) {
-                    onFailure(source, t);
+                } catch (Exception e) {
+                    onFailure(source, e);
                 }
             }
         });
@@ -832,7 +833,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
             // node calling the join request
             try {
                 membership.sendValidateJoinRequestBlocking(node, state, joinTimeout);
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 logger.warn("failed to validate incoming join request from node [{}]", e, node);
                 callback.onFailure(new IllegalStateException("failure when sending a validation request to node", e));
                 return;
@@ -1049,8 +1050,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
-                    logger.debug("unexpected error during cluster state update task after pings from another master", t);
+                public void onFailure(String source, Exception e) {
+                    logger.debug("unexpected error during cluster state update task after pings from another master", e);
                 }
             });
         }
@@ -1109,8 +1110,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
-                    logger.error("unexpected failure during [{}]", t, source);
+                public void onFailure(String source, Exception e) {
+                    logger.error("unexpected failure during [{}]", e, source);
                 }
             });
         }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
index 86c9ea86a80..36a9bd426e8 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
@@ -369,14 +369,15 @@ public class MasterFaultDetection extends FaultDetection {
                     }
 
                     @Override
-                    public void onFailure(String source, @Nullable Throwable t) {
-                        if (t == null) {
-                            t = new ElasticsearchException("unknown error while processing ping");
+                    public void onFailure(String source, @Nullable Exception e) {
+                        if (e == null) {
+                            e = new ElasticsearchException("unknown error while processing ping");
                         }
                         try {
-                            channel.sendResponse(t);
-                        } catch (IOException e) {
-                            logger.warn("error while sending ping response", e);
+                            channel.sendResponse(e);
+                        } catch (IOException inner) {
+                            inner.addSuppressed(e);
+                            logger.warn("error while sending ping response", inner);
                         }
                     }
 
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/membership/MembershipAction.java b/core/src/main/java/org/elasticsearch/discovery/zen/membership/MembershipAction.java
index 1de3805d67e..961b8d79728 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/membership/MembershipAction.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/membership/MembershipAction.java
@@ -50,7 +50,7 @@ public class MembershipAction extends AbstractComponent {
     public interface JoinCallback {
         void onSuccess();
 
-        void onFailure(Throwable t);
+        void onFailure(Exception e);
     }
 
     public interface MembershipListener {
@@ -137,17 +137,18 @@ public class MembershipAction extends AbstractComponent {
                 public void onSuccess() {
                     try {
                         channel.sendResponse(TransportResponse.Empty.INSTANCE);
-                    } catch (Throwable t) {
-                        onFailure(t);
+                    } catch (Exception e) {
+                        onFailure(e);
                     }
                 }
 
                 @Override
-                public void onFailure(Throwable t) {
+                public void onFailure(Exception e) {
                     try {
-                        channel.sendResponse(t);
-                    } catch (Throwable e) {
-                        logger.warn("failed to send back failure on join request", e);
+                        channel.sendResponse(e);
+                    } catch (Exception inner) {
+                        inner.addSuppressed(e);
+                        logger.warn("failed to send back failure on join request", inner);
                     }
                 }
             });
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
index d010587dda4..98ccf137b13 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.discovery.zen.ping.unicast;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
-
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
@@ -262,16 +261,16 @@ public class UnicastZenPing extends AbstractLifecycleComponent implements ZenPin
                         }
 
                         @Override
-                        public void onFailure(Throwable t) {
-                            logger.debug("Ping execution failed", t);
+                        public void onFailure(Exception e) {
+                            logger.debug("Ping execution failed", e);
                             sendPingsHandler.close();
                         }
                     });
                 }
 
                 @Override
-                public void onFailure(Throwable t) {
-                    logger.debug("Ping execution failed", t);
+                public void onFailure(Exception e) {
+                    logger.debug("Ping execution failed", e);
                     sendPingsHandler.close();
                 }
             });
@@ -418,7 +417,7 @@ public class UnicastZenPing extends AbstractLifecycleComponent implements ZenPin
                             // something went wrong on the other side
                             logger.debug("[{}] received a remote error as a response to ping {}", e,
                                     sendPingsHandler.id(), finalNodeToSend);
-                        } catch (Throwable e) {
+                        } catch (Exception e) {
                             logger.warn("[{}] failed send ping to {}", e, sendPingsHandler.id(), finalNodeToSend);
                         } finally {
                             if (!success) {
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueue.java b/core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueue.java
index e591f44d309..24b093627b6 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueue.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueue.java
@@ -51,7 +51,7 @@ public class PendingClusterStatesQueue {
 
         void onNewClusterStateProcessed();
 
-        void onNewClusterStateFailed(Throwable t);
+        void onNewClusterStateFailed(Exception e);
     }
 
     final ArrayList<ClusterStateContext> pendingStates = new ArrayList<>();
@@ -97,7 +97,7 @@ public class PendingClusterStatesQueue {
      * mark that the processing of the given state has failed. All committed states that are {@link ClusterState#supersedes(ClusterState)}-ed
      * by this failed state, will be failed as well
      */
-    public synchronized void markAsFailed(ClusterState state, Throwable reason) {
+    public synchronized void markAsFailed(ClusterState state, Exception reason) {
         final ClusterStateContext failedContext = findState(state.stateUUID());
         if (failedContext == null) {
             throw new IllegalArgumentException("can't resolve failed cluster state with uuid [" + state.stateUUID() + "], version [" + state.version() + "]");
@@ -194,8 +194,9 @@ public class PendingClusterStatesQueue {
         return null;
     }
 
-    /** clear the incoming queue. any committed state will be failed */
-    public synchronized void failAllStatesAndClear(Throwable reason) {
+    /** clear the incoming queue. any committed state will be failed
+     */
+    public synchronized void failAllStatesAndClear(Exception reason) {
         for (ClusterStateContext pendingState : pendingStates) {
             if (pendingState.committed()) {
                 pendingState.listener.onNewClusterStateFailed(reason);
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java b/core/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java
index 7714f34d0e2..bcdd2f25d4d 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java
@@ -152,21 +152,21 @@ public class PublishClusterStateAction extends AbstractComponent {
 
             final BlockingClusterStatePublishResponseHandler publishResponseHandler = new AckClusterStatePublishResponseHandler(nodesToPublishTo, ackListener);
             sendingController = new SendingController(clusterChangedEvent.state(), minMasterNodes, totalMasterNodes, publishResponseHandler);
-        } catch (Throwable t) {
-            throw new Discovery.FailedToCommitClusterStateException("unexpected error while preparing to publish", t);
+        } catch (Exception e) {
+            throw new Discovery.FailedToCommitClusterStateException("unexpected error while preparing to publish", e);
         }
 
         try {
             innerPublish(clusterChangedEvent, nodesToPublishTo, sendingController, sendFullVersion, serializedStates, serializedDiffs);
         } catch (Discovery.FailedToCommitClusterStateException t) {
             throw t;
-        } catch (Throwable t) {
+        } catch (Exception e) {
             // try to fail committing, in cause it's still on going
-            if (sendingController.markAsFailed("unexpected error", t)) {
+            if (sendingController.markAsFailed("unexpected error", e)) {
                 // signal the change should be rejected
-                throw new Discovery.FailedToCommitClusterStateException("unexpected error", t);
+                throw new Discovery.FailedToCommitClusterStateException("unexpected error", e);
             } else {
-                throw t;
+                throw e;
             }
         }
     }
@@ -243,7 +243,7 @@ public class PublishClusterStateAction extends AbstractComponent {
             try {
                 bytes = serializeFullClusterState(clusterState, node.getVersion());
                 serializedStates.put(node.getVersion(), bytes);
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 logger.warn("failed to serialize cluster_state before publishing it to node {}", e, node);
                 sendingController.onNodeSendFailed(node, e);
                 return;
@@ -295,9 +295,9 @@ public class PublishClusterStateAction extends AbstractComponent {
                             }
                         }
                     });
-        } catch (Throwable t) {
-            logger.warn("error sending cluster state to {}", t, node);
-            sendingController.onNodeSendFailed(node, t);
+        } catch (Exception e) {
+            logger.warn("error sending cluster state to {}", e, node);
+            sendingController.onNodeSendFailed(node, e);
         }
     }
 
@@ -326,7 +326,7 @@ public class PublishClusterStateAction extends AbstractComponent {
                             sendingController.getPublishResponseHandler().onFailure(node, exp);
                         }
                     });
-        } catch (Throwable t) {
+        } catch (Exception t) {
             logger.warn("error sending cluster state commit (uuid [{}], version [{}]) to {}", t, clusterState.stateUUID(), clusterState.version(), node);
             sendingController.getPublishResponseHandler().onFailure(node, t);
         }
@@ -428,18 +428,19 @@ public class PublishClusterStateAction extends AbstractComponent {
                 try {
                     // send a response to the master to indicate that this cluster state has been processed post committing it.
                     channel.sendResponse(TransportResponse.Empty.INSTANCE);
-                } catch (Throwable e) {
+                } catch (Exception e) {
                     logger.debug("failed to send response on cluster state processed", e);
                     onNewClusterStateFailed(e);
                 }
             }
 
             @Override
-            public void onNewClusterStateFailed(Throwable t) {
+            public void onNewClusterStateFailed(Exception e) {
                 try {
-                    channel.sendResponse(t);
-                } catch (Throwable e) {
-                    logger.debug("failed to send response on cluster state processed", e);
+                    channel.sendResponse(e);
+                } catch (Exception inner) {
+                    inner.addSuppressed(e);
+                    logger.debug("failed to send response on cluster state processed", inner);
                 }
             }
         });
@@ -592,13 +593,13 @@ public class PublishClusterStateAction extends AbstractComponent {
             }
         }
 
-        public synchronized void onNodeSendFailed(DiscoveryNode node, Throwable t) {
+        public synchronized void onNodeSendFailed(DiscoveryNode node, Exception e) {
             if (node.isMasterNode()) {
                 logger.trace("master node {} failed to ack cluster state version [{}]. processing ... (current pending [{}], needed [{}])",
                         node, clusterState.version(), pendingMasterNodes, neededMastersToCommit);
                 decrementPendingMasterAcksAndChangeForFailure();
             }
-            publishResponseHandler.onFailure(node, t);
+            publishResponseHandler.onFailure(node, e);
         }
 
         /**
@@ -621,7 +622,7 @@ public class PublishClusterStateAction extends AbstractComponent {
          *
          * @return true if the publishing was failed and the cluster state is *not* committed
          **/
-        private synchronized boolean markAsFailed(String details, Throwable reason) {
+        private synchronized boolean markAsFailed(String details, Exception reason) {
             if (committedOrFailed()) {
                 return committed == false;
             }
diff --git a/core/src/main/java/org/elasticsearch/env/ESFileStore.java b/core/src/main/java/org/elasticsearch/env/ESFileStore.java
index 6df242f7ce1..8051c522d03 100644
--- a/core/src/main/java/org/elasticsearch/env/ESFileStore.java
+++ b/core/src/main/java/org/elasticsearch/env/ESFileStore.java
@@ -169,7 +169,7 @@ class ESFileStore extends FileStore {
                 if (Character.isAlphabetic(driveLetter) == false || root.charAt(1) != ':') {
                     throw new RuntimeException("root isn't a drive letter: " + root);
                 }
-            } catch (Throwable checkFailed) {
+            } catch (Exception checkFailed) {
                 // something went wrong, 
                 possibleBug.addSuppressed(checkFailed);
                 throw possibleBug;
@@ -188,7 +188,7 @@ class ESFileStore extends FileStore {
                     }
                 }
                 throw new RuntimeException("no filestores matched");
-            } catch (Throwable weTried) {
+            } catch (Exception weTried) {
                 IOException newException = new IOException("Unable to retrieve filestore for '" + path + "', tried matching against " + Arrays.toString(fileStores), weTried);
                 newException.addSuppressed(possibleBug);
                 throw newException;
diff --git a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
index 8815d18769d..6a805584f5c 100644
--- a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
+++ b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
@@ -670,10 +670,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
      * @throws IllegalStateException if the node is not configured to store local locations
      */
     public Path[] nodeDataPaths() {
-        assert assertEnvIsLocked();
-        if (nodePaths == null || locks == null) {
-            throw new IllegalStateException("node is not configured to store local location");
-        }
+        assertEnvIsLocked();
         Path[] paths = new Path[nodePaths.length];
         for(int i=0;i<paths.length;i++) {
             paths[i] = nodePaths[i].path;
@@ -685,7 +682,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
      * Returns an array of all of the {@link NodePath}s.
      */
     public NodePath[] nodePaths() {
-        assert assertEnvIsLocked();
+        assertEnvIsLocked();
         if (nodePaths == null || locks == null) {
             throw new IllegalStateException("node is not configured to store local location");
         }
@@ -696,7 +693,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
      * Returns all index paths.
      */
     public Path[] indexPaths(Index index) {
-        assert assertEnvIsLocked();
+        assertEnvIsLocked();
         Path[] indexPaths = new Path[nodePaths.length];
         for (int i = 0; i < nodePaths.length; i++) {
             indexPaths[i] = nodePaths[i].resolve(index);
@@ -713,7 +710,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
      *
      */
     public Path[] availableShardPaths(ShardId shardId) {
-        assert assertEnvIsLocked();
+        assertEnvIsLocked();
         final NodePath[] nodePaths = nodePaths();
         final Path[] shardLocations = new Path[nodePaths.length];
         for (int i = 0; i < nodePaths.length; i++) {
@@ -729,7 +726,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
         if (nodePaths == null || locks == null) {
             throw new IllegalStateException("node is not configured to store local location");
         }
-        assert assertEnvIsLocked();
+        assertEnvIsLocked();
         Set<String> indexFolders = new HashSet<>();
         for (NodePath nodePath : nodePaths) {
             Path indicesLocation = nodePath.indicesPath;
@@ -754,7 +751,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
         if (nodePaths == null || locks == null) {
             throw new IllegalStateException("node is not configured to store local location");
         }
-        assert assertEnvIsLocked();
+        assertEnvIsLocked();
         List<Path> paths = new ArrayList<>(nodePaths.length);
         for (NodePath nodePath : nodePaths) {
             Path indexFolder = nodePath.indicesPath.resolve(indexFolderName);
@@ -778,7 +775,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
         if (nodePaths == null || locks == null) {
             throw new IllegalStateException("node is not configured to store local location");
         }
-        assert assertEnvIsLocked();
+        assertEnvIsLocked();
         final Set<ShardId> shardIds = new HashSet<>();
         final String indexUniquePathId = index.getUUID();
         for (final NodePath nodePath : nodePaths) {
@@ -829,18 +826,17 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
     }
 
 
-    private boolean assertEnvIsLocked() {
+    private void assertEnvIsLocked() {
         if (!closed.get() && locks != null) {
             for (Lock lock : locks) {
                 try {
                     lock.ensureValid();
                 } catch (IOException e) {
                     logger.warn("lock assertion failed", e);
-                    return false;
+                    throw new IllegalStateException("environment is not locked", e);
                 }
             }
         }
-        return true;
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java b/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java
index 737ed7d591c..dc7194b949c 100644
--- a/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java
+++ b/core/src/main/java/org/elasticsearch/gateway/AsyncShardFetch.java
@@ -274,7 +274,7 @@ public abstract class AsyncShardFetch<T extends BaseNodeResponse> implements Rel
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 List<FailedNodeException> failures = new ArrayList<>(nodes.length);
                 for (final DiscoveryNode node: nodes) {
                     failures.add(new FailedNodeException(node.getId(), "total failure in fetching", e));
diff --git a/core/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java b/core/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java
index 2c652fc1481..370778898fc 100644
--- a/core/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java
+++ b/core/src/main/java/org/elasticsearch/gateway/DanglingIndicesState.java
@@ -170,7 +170,7 @@ public class DanglingIndicesState extends AbstractComponent {
                     }
                 }
             );
-        } catch (Throwable e) {
+        } catch (Exception e) {
             logger.warn("failed to send allocate dangled", e);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
index 534a73355af..a96b03ee560 100644
--- a/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
+++ b/core/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java
@@ -50,9 +50,6 @@ import java.util.Set;
 import static java.util.Collections.emptySet;
 import static java.util.Collections.unmodifiableSet;
 
-/**
- *
- */
 public class GatewayMetaState extends AbstractComponent implements ClusterStateListener {
 
     private final NodeEnvironment nodeEnv;
@@ -141,7 +138,7 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
                         newPreviouslyWrittenIndices.addAll(previouslyWrittenIndices);
                         previouslyWrittenIndices = unmodifiableSet(newPreviouslyWrittenIndices);
                     }
-                } catch (Throwable e) {
+                } catch (Exception e) {
                     success = false;
                 }
             }
@@ -149,7 +146,7 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
             if (previousMetaData == null || !MetaData.isGlobalStateEquals(previousMetaData, newMetaData)) {
                 try {
                     metaStateService.writeGlobalState("changed", newMetaData);
-                } catch (Throwable e) {
+                } catch (Exception e) {
                     success = false;
                 }
             }
@@ -161,7 +158,7 @@ public class GatewayMetaState extends AbstractComponent implements ClusterStateL
             for (IndexMetaWriteInfo indexMetaWrite : writeInfo) {
                 try {
                     metaStateService.writeIndex(indexMetaWrite.reason, indexMetaWrite.newMetaData);
-                } catch (Throwable e) {
+                } catch (Exception e) {
                     success = false;
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/gateway/GatewayService.java b/core/src/main/java/org/elasticsearch/gateway/GatewayService.java
index 746d70db009..3282a8f2a4e 100644
--- a/core/src/main/java/org/elasticsearch/gateway/GatewayService.java
+++ b/core/src/main/java/org/elasticsearch/gateway/GatewayService.java
@@ -216,11 +216,11 @@ public class GatewayService extends AbstractLifecycleComponent implements Cluste
             if (recovered.compareAndSet(false, true)) {
                 threadPool.generic().execute(new AbstractRunnable() {
                     @Override
-                    public void onFailure(Throwable t) {
-                        logger.warn("Recovery failed", t);
+                    public void onFailure(Exception e) {
+                        logger.warn("Recovery failed", e);
                         // we reset `recovered` in the listener don't reset it here otherwise there might be a race
                         // that resets it to false while a new recover is already running?
-                        recoveryListener.onFailure("state recovery failed: " + t.getMessage());
+                        recoveryListener.onFailure("state recovery failed: " + e.getMessage());
                     }
 
                     @Override
@@ -288,8 +288,8 @@ public class GatewayService extends AbstractLifecycleComponent implements Cluste
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
-                    logger.error("unexpected failure during [{}]", t, source);
+                public void onFailure(String source, Exception e) {
+                    logger.error("unexpected failure during [{}]", e, source);
                     GatewayRecoveryListener.this.onFailure("failed to updated cluster state");
                 }
 
diff --git a/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java b/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
index 3ee6851f8a5..af8709ccd96 100644
--- a/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
+++ b/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
@@ -175,12 +175,13 @@ public class LocalAllocateDangledIndices extends AbstractComponent {
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
-                    logger.error("unexpected failure during [{}]", t, source);
+                public void onFailure(String source, Exception e) {
+                    logger.error("unexpected failure during [{}]", e, source);
                     try {
-                        channel.sendResponse(t);
-                    } catch (Exception e) {
-                        logger.warn("failed send response for allocating dangled", e);
+                        channel.sendResponse(e);
+                    } catch (Exception inner) {
+                        inner.addSuppressed(e);
+                        logger.warn("failed send response for allocating dangled", inner);
                     }
                 }
 
diff --git a/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java b/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java
index 9a78925368e..9c2d6c80011 100644
--- a/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java
+++ b/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java
@@ -312,7 +312,7 @@ public abstract class MetaDataStateFormat<T> {
                     logger.trace("state id [{}] read from [{}]", id, stateFile.getFileName());
                 }
                 return state;
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 exceptions.add(new IOException("failed to read " + pathAndStateId.toString(), e));
                 logger.debug("{}: failed to read [{}], ignoring...", e, pathAndStateId.file.toAbsolutePath(), prefix);
             }
diff --git a/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java
index b6f415a9049..b6166c9d545 100644
--- a/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java
+++ b/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java
@@ -127,7 +127,7 @@ public class MetaStateService extends AbstractComponent {
         try {
             IndexMetaData.FORMAT.write(indexMetaData, indexMetaData.getVersion(),
                 nodeEnv.indexPaths(indexMetaData.getIndex()));
-        } catch (Throwable ex) {
+        } catch (Exception ex) {
             logger.warn("[{}]: failed to write index state", ex, index);
             throw new IOException("failed to write state for [" + index + "]", ex);
         }
@@ -140,7 +140,7 @@ public class MetaStateService extends AbstractComponent {
         logger.trace("[_global] writing state, reason [{}]",  reason);
         try {
             MetaData.FORMAT.write(metaData, metaData.version(), nodeEnv.nodeDataPaths());
-        } catch (Throwable ex) {
+        } catch (Exception ex) {
             logger.warn("[_global]: failed to write global state", ex);
             throw new IOException("failed to write global state", ex);
         }
diff --git a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
index 42fc8163706..2ecabf778c7 100644
--- a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
+++ b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
@@ -245,7 +245,7 @@ public class TransportNodesListGatewayStartedShards extends
         private long legacyVersion = ShardStateMetaData.NO_VERSION; // for pre-3.0 shards that have not yet been active
         private String allocationId = null;
         private boolean primary = false;
-        private Throwable storeException = null;
+        private Exception storeException = null;
 
         public NodeGatewayStartedShards() {
         }
@@ -255,7 +255,7 @@ public class TransportNodesListGatewayStartedShards extends
         }
 
         public NodeGatewayStartedShards(DiscoveryNode node, long legacyVersion, String allocationId, boolean primary,
-                                        Throwable storeException) {
+                                        Exception storeException) {
             super(node);
             this.legacyVersion = legacyVersion;
             this.allocationId = allocationId;
@@ -275,7 +275,7 @@ public class TransportNodesListGatewayStartedShards extends
             return this.primary;
         }
 
-        public Throwable storeException() {
+        public Exception storeException() {
             return this.storeException;
         }
 
@@ -286,7 +286,7 @@ public class TransportNodesListGatewayStartedShards extends
             allocationId = in.readOptionalString();
             primary = in.readBoolean();
             if (in.readBoolean()) {
-                storeException = in.readThrowable();
+                storeException = in.readException();
             }
         }
 
diff --git a/core/src/main/java/org/elasticsearch/http/HttpServer.java b/core/src/main/java/org/elasticsearch/http/HttpServer.java
index 8e3f15c5a24..ccf2d764c05 100644
--- a/core/src/main/java/org/elasticsearch/http/HttpServer.java
+++ b/core/src/main/java/org/elasticsearch/http/HttpServer.java
@@ -19,11 +19,6 @@
 
 package org.elasticsearch.http;
 
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.concurrent.atomic.AtomicBoolean;
-
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.breaker.CircuitBreaker;
@@ -45,6 +40,11 @@ import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
 import org.elasticsearch.rest.RestStatus;
 
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.concurrent.atomic.AtomicBoolean;
+
 import static org.elasticsearch.rest.RestStatus.FORBIDDEN;
 import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR;
 
@@ -114,8 +114,8 @@ public class HttpServer extends AbstractLifecycleComponent implements HttpServer
             // iff we could reserve bytes for the request we need to send the response also over this channel
             responseChannel = new ResourceHandlingHttpChannel(channel, circuitBreakerService, contentLength);
             restController.dispatchRequest(request, responseChannel, client, threadContext);
-        } catch (Throwable t) {
-            restController.sendErrorResponse(request, responseChannel, t);
+        } catch (Exception e) {
+            restController.sendErrorResponse(request, responseChannel, e);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/CompositeIndexEventListener.java b/core/src/main/java/org/elasticsearch/index/CompositeIndexEventListener.java
index acc0d3f8370..97e00b98df9 100644
--- a/core/src/main/java/org/elasticsearch/index/CompositeIndexEventListener.java
+++ b/core/src/main/java/org/elasticsearch/index/CompositeIndexEventListener.java
@@ -57,8 +57,8 @@ final class CompositeIndexEventListener implements IndexEventListener {
         for (IndexEventListener listener : listeners) {
             try {
                 listener.shardRoutingChanged(indexShard, oldRouting, newRouting);
-            } catch (Throwable t) {
-                logger.warn("[{}] failed to invoke shard touring changed callback", t, indexShard.shardId().getId());
+            } catch (Exception e) {
+                logger.warn("[{}] failed to invoke shard touring changed callback", e, indexShard.shardId().getId());
             }
         }
     }
@@ -68,9 +68,9 @@ final class CompositeIndexEventListener implements IndexEventListener {
         for (IndexEventListener listener : listeners) {
             try {
                 listener.afterIndexShardCreated(indexShard);
-            } catch (Throwable t) {
-                logger.warn("[{}] failed to invoke after shard created callback", t, indexShard.shardId().getId());
-                throw t;
+            } catch (Exception e) {
+                logger.warn("[{}] failed to invoke after shard created callback", e, indexShard.shardId().getId());
+                throw e;
             }
         }
     }
@@ -80,9 +80,9 @@ final class CompositeIndexEventListener implements IndexEventListener {
         for (IndexEventListener listener : listeners) {
             try {
                 listener.afterIndexShardStarted(indexShard);
-            } catch (Throwable t) {
-                logger.warn("[{}] failed to invoke after shard started callback", t, indexShard.shardId().getId());
-                throw t;
+            } catch (Exception e) {
+                logger.warn("[{}] failed to invoke after shard started callback", e, indexShard.shardId().getId());
+                throw e;
             }
         }
     }
@@ -93,9 +93,9 @@ final class CompositeIndexEventListener implements IndexEventListener {
         for (IndexEventListener listener : listeners) {
             try {
                 listener.beforeIndexShardClosed(shardId, indexShard, indexSettings);
-            } catch (Throwable t) {
-                logger.warn("[{}] failed to invoke before shard closed callback", t, shardId.getId());
-                throw t;
+            } catch (Exception e) {
+                logger.warn("[{}] failed to invoke before shard closed callback", e, shardId.getId());
+                throw e;
             }
         }
     }
@@ -106,9 +106,9 @@ final class CompositeIndexEventListener implements IndexEventListener {
         for (IndexEventListener listener : listeners) {
             try {
                 listener.afterIndexShardClosed(shardId, indexShard, indexSettings);
-            } catch (Throwable t) {
-                logger.warn("[{}] failed to invoke after shard closed callback", t, shardId.getId());
-                throw t;
+            } catch (Exception e) {
+                logger.warn("[{}] failed to invoke after shard closed callback", e, shardId.getId());
+                throw e;
             }
         }
     }
@@ -118,9 +118,9 @@ final class CompositeIndexEventListener implements IndexEventListener {
         for (IndexEventListener listener : listeners) {
             try {
                 listener.onShardInactive(indexShard);
-            } catch (Throwable t) {
-                logger.warn("[{}] failed to invoke on shard inactive callback", t, indexShard.shardId().getId());
-                throw t;
+            } catch (Exception e) {
+                logger.warn("[{}] failed to invoke on shard inactive callback", e, indexShard.shardId().getId());
+                throw e;
             }
         }
     }
@@ -130,9 +130,9 @@ final class CompositeIndexEventListener implements IndexEventListener {
         for (IndexEventListener listener : listeners) {
             try {
                 listener.indexShardStateChanged(indexShard, previousState, indexShard.state(), reason);
-            } catch (Throwable t) {
-                logger.warn("[{}] failed to invoke index shard state changed callback", t, indexShard.shardId().getId());
-                throw t;
+            } catch (Exception e) {
+                logger.warn("[{}] failed to invoke index shard state changed callback", e, indexShard.shardId().getId());
+                throw e;
             }
         }
     }
@@ -142,9 +142,9 @@ final class CompositeIndexEventListener implements IndexEventListener {
         for (IndexEventListener listener : listeners) {
             try {
                 listener.beforeIndexCreated(index, indexSettings);
-            } catch (Throwable t) {
-                logger.warn("failed to invoke before index created callback", t);
-                throw t;
+            } catch (Exception e) {
+                logger.warn("failed to invoke before index created callback", e);
+                throw e;
             }
         }
     }
@@ -154,9 +154,9 @@ final class CompositeIndexEventListener implements IndexEventListener {
         for (IndexEventListener listener : listeners) {
             try {
                 listener.afterIndexCreated(indexService);
-            } catch (Throwable t) {
-                logger.warn("failed to invoke after index created callback", t);
-                throw t;
+            } catch (Exception e) {
+                logger.warn("failed to invoke after index created callback", e);
+                throw e;
             }
         }
     }
@@ -166,9 +166,9 @@ final class CompositeIndexEventListener implements IndexEventListener {
         for (IndexEventListener listener : listeners) {
             try {
                 listener.beforeIndexShardCreated(shardId, indexSettings);
-            } catch (Throwable t) {
-                logger.warn("[{}] failed to invoke before shard created callback", t, shardId);
-                throw t;
+            } catch (Exception e) {
+                logger.warn("[{}] failed to invoke before shard created callback", e, shardId);
+                throw e;
             }
         }
     }
@@ -178,9 +178,9 @@ final class CompositeIndexEventListener implements IndexEventListener {
         for (IndexEventListener listener : listeners) {
             try {
                 listener.beforeIndexClosed(indexService);
-            } catch (Throwable t) {
-                logger.warn("failed to invoke before index closed callback", t);
-                throw t;
+            } catch (Exception e) {
+                logger.warn("failed to invoke before index closed callback", e);
+                throw e;
             }
         }
     }
@@ -190,9 +190,9 @@ final class CompositeIndexEventListener implements IndexEventListener {
         for (IndexEventListener listener : listeners) {
             try {
                 listener.beforeIndexDeleted(indexService);
-            } catch (Throwable t) {
-                logger.warn("failed to invoke before index deleted callback", t);
-                throw t;
+            } catch (Exception e) {
+                logger.warn("failed to invoke before index deleted callback", e);
+                throw e;
             }
         }
     }
@@ -202,9 +202,9 @@ final class CompositeIndexEventListener implements IndexEventListener {
         for (IndexEventListener listener : listeners) {
             try {
                 listener.afterIndexDeleted(index, indexSettings);
-            } catch (Throwable t) {
-                logger.warn("failed to invoke after index deleted callback", t);
-                throw t;
+            } catch (Exception e) {
+                logger.warn("failed to invoke after index deleted callback", e);
+                throw e;
             }
         }
     }
@@ -214,9 +214,9 @@ final class CompositeIndexEventListener implements IndexEventListener {
         for (IndexEventListener listener : listeners) {
             try {
                 listener.afterIndexClosed(index, indexSettings);
-            } catch (Throwable t) {
-                logger.warn("failed to invoke after index closed callback", t);
-                throw t;
+            } catch (Exception e) {
+                logger.warn("failed to invoke after index closed callback", e);
+                throw e;
             }
         }
     }
@@ -227,9 +227,9 @@ final class CompositeIndexEventListener implements IndexEventListener {
         for (IndexEventListener listener : listeners) {
             try {
                 listener.beforeIndexShardDeleted(shardId, indexSettings);
-            } catch (Throwable t) {
-                logger.warn("[{}] failed to invoke before shard deleted callback", t, shardId.getId());
-                throw t;
+            } catch (Exception e) {
+                logger.warn("[{}] failed to invoke before shard deleted callback", e, shardId.getId());
+                throw e;
             }
         }
     }
@@ -240,9 +240,9 @@ final class CompositeIndexEventListener implements IndexEventListener {
         for (IndexEventListener listener : listeners) {
             try {
                 listener.afterIndexShardDeleted(shardId, indexSettings);
-            } catch (Throwable t) {
-                logger.warn("[{}] failed to invoke after shard deleted callback", t, shardId.getId());
-                throw t;
+            } catch (Exception e) {
+                logger.warn("[{}] failed to invoke after shard deleted callback", e, shardId.getId());
+                throw e;
             }
         }
     }
@@ -252,9 +252,9 @@ final class CompositeIndexEventListener implements IndexEventListener {
         for (IndexEventListener listener  : listeners) {
             try {
                 listener.beforeIndexAddedToCluster(index, indexSettings);
-            } catch (Throwable t) {
-                logger.warn("failed to invoke before index added to cluster callback", t);
-                throw t;
+            } catch (Exception e) {
+                logger.warn("failed to invoke before index added to cluster callback", e);
+                throw e;
             }
         }
     }
@@ -264,9 +264,9 @@ final class CompositeIndexEventListener implements IndexEventListener {
         for (IndexEventListener listener  : listeners) {
             try {
                 listener.onStoreClosed(shardId);
-            } catch (Throwable t) {
-                logger.warn("failed to invoke on store closed", t);
-                throw t;
+            } catch (Exception e) {
+                logger.warn("failed to invoke on store closed", e);
+                throw e;
             }
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java
index 4e8caa7d5e9..0ba37418792 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexService.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexService.java
@@ -234,8 +234,8 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
                 for (final int shardId : shardIds) {
                     try {
                         removeShard(shardId, reason);
-                    } catch (Throwable t) {
-                        logger.warn("failed to close shard", t);
+                    } catch (Exception e) {
+                        logger.warn("failed to close shard", e);
                     }
                 }
             } finally {
@@ -290,9 +290,9 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
                 try {
                     ShardPath.deleteLeftoverShardDirectory(logger, nodeEnv, lock, this.indexSettings);
                     path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings);
-                } catch (Throwable t) {
-                    t.addSuppressed(ex);
-                    throw t;
+                } catch (Exception inner) {
+                    ex.addSuppressed(inner);
+                    throw ex;
                 }
             }
 
@@ -390,7 +390,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
                         // only flush we are we closed (closed index or shutdown) and if we are not deleted
                         final boolean flushEngine = deleted.get() == false && closed.get();
                         indexShard.close(reason, flushEngine);
-                    } catch (Throwable e) {
+                    } catch (Exception e) {
                         logger.debug("[{}] failed to close index shard", e, shardId);
                         // ignore
                     }
@@ -401,7 +401,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
         } finally {
             try {
                 store.close();
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 logger.warn("[{}] failed to close store on shard removal (reason: [{}])", e, shardId, reason);
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/index/IndexWarmer.java b/core/src/main/java/org/elasticsearch/index/IndexWarmer.java
index 499031af970..ba48adb71a8 100644
--- a/core/src/main/java/org/elasticsearch/index/IndexWarmer.java
+++ b/core/src/main/java/org/elasticsearch/index/IndexWarmer.java
@@ -146,8 +146,8 @@ public final class IndexWarmer extends AbstractComponent {
                             indexShard.warmerService().logger().trace("warmed global ordinals for [{}], took [{}]", fieldType.name(),
                                 TimeValue.timeValueNanos(System.nanoTime() - start));
                         }
-                    } catch (Throwable t) {
-                        indexShard.warmerService().logger().warn("failed to warm-up global ordinals for [{}]", t, fieldType.name());
+                    } catch (Exception e) {
+                        indexShard.warmerService().logger().warn("failed to warm-up global ordinals for [{}]", e, fieldType.name());
                     } finally {
                         latch.countDown();
                     }
diff --git a/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java b/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java
index 2452e8147c2..5eb2ec8ab70 100644
--- a/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java
+++ b/core/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java
@@ -42,6 +42,8 @@ import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexWarmer;
+import org.elasticsearch.index.IndexWarmer.TerminationHandle;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperService;
@@ -49,8 +51,6 @@ import org.elasticsearch.index.mapper.object.ObjectMapper;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardUtils;
-import org.elasticsearch.index.IndexWarmer;
-import org.elasticsearch.index.IndexWarmer.TerminationHandle;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.Closeable;
@@ -257,8 +257,8 @@ public final class BitsetFilterCache extends AbstractIndexComponent implements L
                             if (indexShard.warmerService().logger().isTraceEnabled()) {
                                 indexShard.warmerService().logger().trace("warmed bitset for [{}], took [{}]", filterToWarm, TimeValue.timeValueNanos(System.nanoTime() - start));
                             }
-                        } catch (Throwable t) {
-                            indexShard.warmerService().logger().warn("failed to load bitset for [{}]", t, filterToWarm);
+                        } catch (Exception e) {
+                            indexShard.warmerService().logger().warn("failed to load bitset for [{}]", e, filterToWarm);
                         } finally {
                             latch.countDown();
                         }
diff --git a/core/src/main/java/org/elasticsearch/index/engine/Engine.java b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
index d169e495a3d..f9186185e61 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/Engine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/Engine.java
@@ -102,7 +102,7 @@ public abstract class Engine implements Closeable {
     protected final ReentrantReadWriteLock rwl = new ReentrantReadWriteLock();
     protected final ReleasableLock readLock = new ReleasableLock(rwl.readLock());
     protected final ReleasableLock writeLock = new ReleasableLock(rwl.writeLock());
-    protected volatile Throwable failedEngine = null;
+    protected volatile Exception failedEngine = null;
     /*
      * on <tt>lastWriteNanos</tt> we use System.nanoTime() to initialize this since:
      *  - we use the value for figuring out if the shard / engine is active so if we startup and no write has happened yet we still consider it active
@@ -302,7 +302,7 @@ public abstract class Engine implements Closeable {
         final Versions.DocIdAndVersion docIdAndVersion;
         try {
             docIdAndVersion = Versions.loadDocIdAndVersion(searcher.reader(), get.uid());
-        } catch (Throwable e) {
+        } catch (Exception e) {
             Releasables.closeWhileHandlingException(searcher);
             //TODO: A better exception goes here
             throw new EngineException(shardId, "Couldn't resolve version", e);
@@ -362,7 +362,7 @@ public abstract class Engine implements Closeable {
             }
         } catch (EngineClosedException ex) {
             throw ex;
-        } catch (Throwable ex) {
+        } catch (Exception ex) {
             ensureOpen(); // throw EngineCloseException here if we are already closed
             logger.error("failed to acquire searcher, source {}", ex, source);
             throw new EngineException(shardId, "failed to acquire searcher, source " + source, ex);
@@ -660,7 +660,7 @@ public abstract class Engine implements Closeable {
      * fail engine due to some error. the engine will also be closed.
      * The underlying store is marked corrupted iff failure is caused by index corruption
      */
-    public void failEngine(String reason, @Nullable Throwable failure) {
+    public void failEngine(String reason, @Nullable Exception failure) {
         if (failEngineLock.tryLock()) {
             store.incRef();
             try {
@@ -688,9 +688,10 @@ public abstract class Engine implements Closeable {
                     }
                     eventListener.onFailedEngine(reason, failure);
                 }
-            } catch (Throwable t) {
+            } catch (Exception inner) {
+                if (failure != null) inner.addSuppressed(failure);
                 // don't bubble up these exceptions up
-                logger.warn("failEngine threw exception", t);
+                logger.warn("failEngine threw exception", inner);
             } finally {
                 store.decRef();
             }
@@ -700,12 +701,12 @@ public abstract class Engine implements Closeable {
     }
 
     /** Check whether the engine should be failed */
-    protected boolean maybeFailEngine(String source, Throwable t) {
-        if (Lucene.isCorruptionException(t)) {
-            failEngine("corrupt file (source: [" + source + "])", t);
+    protected boolean maybeFailEngine(String source, Exception e) {
+        if (Lucene.isCorruptionException(e)) {
+            failEngine("corrupt file (source: [" + source + "])", e);
             return true;
-        } else if (ExceptionsHelper.isOOM(t)) {
-            failEngine("out of memory (source: [" + source + "])", t);
+        } else if (ExceptionsHelper.isOOM(e)) {
+            failEngine("out of memory (source: [" + source + "])", e);
             return true;
         }
         return false;
@@ -716,7 +717,7 @@ public abstract class Engine implements Closeable {
         /**
          * Called when a fatal exception occurred
          */
-        default void onFailedEngine(String reason, @Nullable Throwable t) {}
+        default void onFailedEngine(String reason, @Nullable Exception e) {}
     }
 
     public static class Searcher implements Releasable {
diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index 6052de7f586..779d7629686 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -185,10 +185,14 @@ public class InternalEngine extends Engine {
             }
             try {
                 recoverFromTranslog(engineConfig.getTranslogRecoveryPerformer());
-            } catch (Throwable t) {
-                allowCommits.set(false); // just play safe and never allow commits on this
-                failEngine("failed to recover from translog", t);
-                throw t;
+            } catch (Exception e) {
+                try {
+                    allowCommits.set(false); // just play safe and never allow commits on this
+                    failEngine("failed to recover from translog", e);
+                } catch (Exception inner) {
+                    e.addSuppressed(inner);
+                }
+                throw e;
             }
         } finally {
             flushLock.unlock();
@@ -202,7 +206,7 @@ public class InternalEngine extends Engine {
         try {
             Translog.Snapshot snapshot = translog.newSnapshot();
             opsRecovered = handler.recoveryFromSnapshot(this, snapshot);
-        } catch (Throwable e) {
+        } catch (Exception e) {
             throw new EngineException(shardId, "failed to recover from translog", e);
         }
         // flush if we recovered something or if we have references to older translogs
@@ -297,8 +301,8 @@ public class InternalEngine extends Engine {
                 maybeFailEngine("start", e);
                 try {
                     indexWriter.rollback();
-                } catch (IOException e1) { // iw is closed below
-                    e.addSuppressed(e1);
+                } catch (IOException inner) { // iw is closed below
+                    e.addSuppressed(inner);
                 }
                 throw new EngineCreationFailureException(shardId, "failed to open reader on writer", e);
             }
@@ -402,9 +406,13 @@ public class InternalEngine extends Engine {
                     created = innerIndex(index);
                 }
             }
-        } catch (OutOfMemoryError | IllegalStateException | IOException t) {
-            maybeFailEngine("index", t);
-            throw new IndexFailedEngineException(shardId, index.type(), index.id(), t);
+        } catch (IllegalStateException | IOException e) {
+            try {
+                maybeFailEngine("index", e);
+            } catch (Exception inner) {
+                e.addSuppressed(inner);
+            }
+            throw new IndexFailedEngineException(shardId, index.type(), index.id(), e);
         }
         return created;
     }
@@ -483,9 +491,13 @@ public class InternalEngine extends Engine {
             ensureOpen();
             // NOTE: we don't throttle this when merges fall behind because delete-by-id does not create new segments:
             innerDelete(delete);
-        } catch (OutOfMemoryError | IllegalStateException | IOException t) {
-            maybeFailEngine("delete", t);
-            throw new DeleteFailedEngineException(shardId, delete, t);
+        } catch (IllegalStateException | IOException e) {
+            try {
+                maybeFailEngine("delete", e);
+            } catch (Exception inner) {
+                e.addSuppressed(inner);
+            }
+            throw new DeleteFailedEngineException(shardId, delete, e);
         }
 
         maybePruneDeletedTombstones();
@@ -554,9 +566,13 @@ public class InternalEngine extends Engine {
             maybeFailEngine("refresh", e);
         } catch (EngineClosedException e) {
             throw e;
-        } catch (Throwable t) {
-            failEngine("refresh failed", t);
-            throw new RefreshFailedEngineException(shardId, t);
+        } catch (Exception e) {
+            try {
+                failEngine("refresh failed", e);
+            } catch (Exception inner) {
+                e.addSuppressed(inner);
+            }
+            throw new RefreshFailedEngineException(shardId, e);
         }
 
         // TODO: maybe we should just put a scheduled job in threadPool?
@@ -598,9 +614,13 @@ public class InternalEngine extends Engine {
             maybeFailEngine("writeIndexingBuffer", e);
         } catch (EngineClosedException e) {
             throw e;
-        } catch (Throwable t) {
-            failEngine("writeIndexingBuffer failed", t);
-            throw new RefreshFailedEngineException(shardId, t);
+        } catch (Exception e) {
+            try {
+                failEngine("writeIndexingBuffer failed", e);
+            } catch (Exception inner) {
+                e.addSuppressed(inner);
+            }
+            throw new RefreshFailedEngineException(shardId, e);
         }
     }
 
@@ -703,7 +723,7 @@ public class InternalEngine extends Engine {
                         refresh("version_table_flush");
                         // after refresh documents can be retrieved from the index so we can now commit the translog
                         translog.commit();
-                    } catch (Throwable e) {
+                    } catch (Exception e) {
                         throw new FlushFailedEngineException(shardId, e);
                     }
                 }
@@ -717,9 +737,13 @@ public class InternalEngine extends Engine {
                 try {
                     // reread the last committed segment infos
                     lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo();
-                } catch (Throwable e) {
+                } catch (Exception e) {
                     if (isClosed.get() == false) {
-                        logger.warn("failed to read latest segment infos on flush", e);
+                        try {
+                            logger.warn("failed to read latest segment infos on flush", e);
+                        } catch (Exception inner) {
+                            e.addSuppressed(inner);
+                        }
                         if (Lucene.isCorruptionException(e)) {
                             throw new FlushFailedEngineException(shardId, e);
                         }
@@ -811,9 +835,13 @@ public class InternalEngine extends Engine {
             } finally {
                 store.decRef();
             }
-        } catch (Throwable t) {
-            maybeFailEngine("force merge", t);
-            throw t;
+        } catch (Exception e) {
+            try {
+                maybeFailEngine("force merge", e);
+            } catch (Exception inner) {
+                e.addSuppressed(inner);
+            }
+            throw e;
         } finally {
             try {
                 mp.setUpgradeInProgress(false, false); // reset it just to make sure we reset it in a case of an error
@@ -842,29 +870,32 @@ public class InternalEngine extends Engine {
     }
 
     @Override
-    protected boolean maybeFailEngine(String source, Throwable t) {
-        boolean shouldFail = super.maybeFailEngine(source, t);
+    protected boolean maybeFailEngine(String source, Exception e) {
+        boolean shouldFail = super.maybeFailEngine(source, e);
         if (shouldFail) {
             return true;
         }
 
         // Check for AlreadyClosedException
-        if (t instanceof AlreadyClosedException) {
+        if (e instanceof AlreadyClosedException) {
             // if we are already closed due to some tragic exception
             // we need to fail the engine. it might have already been failed before
             // but we are double-checking it's failed and closed
             if (indexWriter.isOpen() == false && indexWriter.getTragicException() != null) {
-                failEngine("already closed by tragic event on the index writer", indexWriter.getTragicException());
+                final Exception tragedy = indexWriter.getTragicException() instanceof Exception ?
+                        (Exception) indexWriter.getTragicException() :
+                        new Exception(indexWriter.getTragicException());
+                failEngine("already closed by tragic event on the index writer", tragedy);
             } else if (translog.isOpen() == false && translog.getTragicException() != null) {
                 failEngine("already closed by tragic event on the translog", translog.getTragicException());
             }
             return true;
-        } else if (t != null &&
-            ((indexWriter.isOpen() == false && indexWriter.getTragicException() == t)
-                || (translog.isOpen() == false && translog.getTragicException() == t))) {
+        } else if (e != null &&
+            ((indexWriter.isOpen() == false && indexWriter.getTragicException() == e)
+                || (translog.isOpen() == false && translog.getTragicException() == e))) {
             // this spot on - we are handling the tragic event exception here so we have to fail the engine
             // right away
-            failEngine(source, t);
+            failEngine(source, e);
             return true;
         }
         return false;
@@ -920,13 +951,13 @@ public class InternalEngine extends Engine {
                 this.versionMap.clear();
                 try {
                     IOUtils.close(searcherManager);
-                } catch (Throwable t) {
-                    logger.warn("Failed to close SearcherManager", t);
+                } catch (Exception e) {
+                    logger.warn("Failed to close SearcherManager", e);
                 }
                 try {
                     IOUtils.close(translog);
-                } catch (Throwable t) {
-                    logger.warn("Failed to close translog", t);
+                } catch (Exception e) {
+                    logger.warn("Failed to close translog", e);
                 }
                 // no need to commit in this case!, we snapshot before we close the shard, so translog and all sync'ed
                 logger.trace("rollback indexWriter");
@@ -936,7 +967,7 @@ public class InternalEngine extends Engine {
                     // ignore
                 }
                 logger.trace("rollback indexWriter done");
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 logger.warn("failed to rollback writer on close", e);
             } finally {
                 store.decRef();
@@ -974,7 +1005,7 @@ public class InternalEngine extends Engine {
             boolean verbose = false;
             try {
                 verbose = Boolean.parseBoolean(System.getProperty("tests.verbose"));
-            } catch (Throwable ignore) {
+            } catch (Exception ignore) {
             }
             iwc.setInfoStream(verbose ? InfoStream.getDefault() : new LoggerInfoStream(logger));
             iwc.setMergeScheduler(mergeScheduler);
@@ -1020,7 +1051,7 @@ public class InternalEngine extends Engine {
                 try {
                     assert searcher.getIndexReader() instanceof ElasticsearchDirectoryReader : "this class needs an ElasticsearchDirectoryReader but got: " + searcher.getIndexReader().getClass();
                     warmer.warm(new Searcher("top_reader_warming", searcher));
-                } catch (Throwable e) {
+                } catch (Exception e) {
                     if (isEngineClosed.get() == false) {
                         logger.warn("failed to prepare/warm", e);
                     }
@@ -1093,7 +1124,7 @@ public class InternalEngine extends Engine {
                 // we deadlock on engine#close for instance.
                 engineConfig.getThreadPool().executor(ThreadPool.Names.FLUSH).execute(new AbstractRunnable() {
                     @Override
-                    public void onFailure(Throwable t) {
+                    public void onFailure(Exception e) {
                         if (isClosed.get() == false) {
                             logger.warn("failed to flush after merge has finished");
                         }
@@ -1121,8 +1152,8 @@ public class InternalEngine extends Engine {
             logger.error("failed to merge", exc);
             engineConfig.getThreadPool().generic().execute(new AbstractRunnable() {
                 @Override
-                public void onFailure(Throwable t) {
-                    logger.debug("merge failure action rejected", t);
+                public void onFailure(Exception e) {
+                    logger.debug("merge failure action rejected", e);
                 }
 
                 @Override
@@ -1147,8 +1178,12 @@ public class InternalEngine extends Engine {
             }
             indexWriter.setCommitData(commitData);
             writer.commit();
-        } catch (Throwable ex) {
-            failEngine("lucene commit failed", ex);
+        } catch (Exception ex) {
+            try {
+                failEngine("lucene commit failed", ex);
+            } catch (Exception inner) {
+                ex.addSuppressed(inner);
+            }
             throw ex;
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java b/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java
index c8ee7d5b8b0..83f9d466f0e 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java
@@ -89,7 +89,7 @@ public class ShadowEngine extends Engine {
                             nonexistentRetryTime + "ms, " +
                             "directory is not an index");
                 }
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 logger.warn("failed to create new reader", e);
                 throw e;
             } finally {
@@ -140,7 +140,7 @@ public class ShadowEngine extends Engine {
         try (ReleasableLock lock = readLock.acquire()) {
             // reread the last committed segment infos
             lastCommittedSegmentInfos = readLastCommittedSegmentInfos(searcherManager, store);
-        } catch (Throwable e) {
+        } catch (Exception e) {
             if (isClosed.get() == false) {
                 logger.warn("failed to read latest segment infos on flush", e);
                 if (Lucene.isCorruptionException(e)) {
@@ -194,9 +194,13 @@ public class ShadowEngine extends Engine {
             ensureOpen();
         } catch (EngineClosedException e) {
             throw e;
-        } catch (Throwable t) {
-            failEngine("refresh failed", t);
-            throw new RefreshFailedEngineException(shardId, t);
+        } catch (Exception e) {
+            try {
+                failEngine("refresh failed", e);
+            } catch (Exception inner) {
+                e.addSuppressed(inner);
+            }
+            throw new RefreshFailedEngineException(shardId, e);
         }
     }
 
@@ -216,8 +220,8 @@ public class ShadowEngine extends Engine {
             try {
                 logger.debug("shadow replica close searcher manager refCount: {}", store.refCount());
                 IOUtils.close(searcherManager);
-            } catch (Throwable t) {
-                logger.warn("shadow replica failed to close searcher manager", t);
+            } catch (Exception e) {
+                logger.warn("shadow replica failed to close searcher manager", e);
             } finally {
                 store.decRef();
             }
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
index 30e1086e187..2577d967e62 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
@@ -21,26 +21,12 @@ package org.elasticsearch.index.fielddata;
 
 import org.apache.lucene.util.Accountable;
 import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.common.collect.MapBuilder;
-import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.fielddata.plain.AbstractGeoPointDVIndexFieldData;
-import org.elasticsearch.index.fielddata.plain.BytesBinaryDVIndexFieldData;
-import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
-import org.elasticsearch.index.fielddata.plain.GeoPointArrayIndexFieldData;
-import org.elasticsearch.index.fielddata.plain.IndexIndexFieldData;
-import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
-import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.core.BooleanFieldMapper;
-import org.elasticsearch.index.mapper.core.KeywordFieldMapper;
-import org.elasticsearch.index.mapper.core.TextFieldMapper;
-import org.elasticsearch.index.mapper.internal.IndexFieldMapper;
-import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
@@ -53,10 +39,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import static java.util.Collections.unmodifiableMap;
-
-/**
- */
 public class IndexFieldDataService extends AbstractIndexComponent implements Closeable {
     public static final String FIELDDATA_CACHE_VALUE_NODE = "node";
     public static final String FIELDDATA_CACHE_KEY = "index.fielddata.cache";
@@ -97,13 +79,13 @@ public class IndexFieldDataService extends AbstractIndexComponent implements Clo
     }
 
     public synchronized void clear() {
-        List<Throwable> exceptions = new ArrayList<>(0);
+        List<Exception> exceptions = new ArrayList<>(0);
         final Collection<IndexFieldDataCache> fieldDataCacheValues = fieldDataCaches.values();
         for (IndexFieldDataCache cache : fieldDataCacheValues) {
             try {
                 cache.clear();
-            } catch (Throwable t) {
-                exceptions.add(t);
+            } catch (Exception e) {
+                exceptions.add(e);
             }
         }
         fieldDataCacheValues.clear();
@@ -111,13 +93,13 @@ public class IndexFieldDataService extends AbstractIndexComponent implements Clo
     }
 
     public synchronized void clearField(final String fieldName) {
-        List<Throwable> exceptions = new ArrayList<>(0);
+        List<Exception> exceptions = new ArrayList<>(0);
         final IndexFieldDataCache cache = fieldDataCaches.remove(fieldName);
         if (cache != null) {
             try {
                 cache.clear();
-            } catch (Throwable t) {
-                exceptions.add(t);
+            } catch (Exception e) {
+                exceptions.add(e);
             }
         }
         ExceptionsHelper.maybeThrowRuntimeAndSuppress(exceptions);
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java
index 604e866de02..6c1f3e6fc9e 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java
@@ -33,8 +33,6 @@ import org.elasticsearch.index.fielddata.RamAccountingTermsEnum;
 
 import java.io.IOException;
 
-/**
- */
 public abstract class AbstractIndexFieldData<FD extends AtomicFieldData> extends AbstractIndexComponent implements IndexFieldData<FD> {
 
     private final String fieldName;
@@ -69,7 +67,7 @@ public abstract class AbstractIndexFieldData<FD extends AtomicFieldData> extends
         try {
             FD fd = cache.load(context, this);
             return fd;
-        } catch (Throwable e) {
+        } catch (Exception e) {
             if (e instanceof ElasticsearchException) {
                 throw (ElasticsearchException) e;
             } else {
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
index 9c71f5bad9b..0223c1a0b63 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java
@@ -86,7 +86,7 @@ public abstract class AbstractIndexOrdinalsFieldData extends AbstractIndexFieldD
         }
         try {
             return cache.load(indexReader, this);
-        } catch (Throwable e) {
+        } catch (Exception e) {
             if (e instanceof ElasticsearchException) {
                 throw (ElasticsearchException) e;
             } else {
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java
index caeb92a7d44..1fc82e31c60 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java
@@ -158,7 +158,7 @@ public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicPare
         }
         try {
             return cache.load(indexReader, this);
-        } catch (Throwable e) {
+        } catch (Exception e) {
             if (e instanceof ElasticsearchException) {
                 throw (ElasticsearchException) e;
             } else {
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java
index 87dc1a2eb08..f0c8aa3d076 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java
@@ -30,8 +30,8 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
 import org.elasticsearch.index.fielddata.ordinals.GlobalOrdinalsBuilder;
-import org.elasticsearch.search.MultiValueMode;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.search.MultiValueMode;
 
 import java.io.IOException;
 
@@ -89,7 +89,7 @@ public class SortedSetDVOrdinalsIndexFieldData extends DocValuesIndexFieldData i
         }
         try {
             return cache.load(indexReader, this);
-        } catch (Throwable e) {
+        } catch (Exception e) {
             if (e instanceof ElasticsearchException) {
                 throw (ElasticsearchException) e;
             } else {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
index 48199a49b36..fc20ef50cf6 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
@@ -77,8 +77,8 @@ final class DocumentParser {
             validateStart(parser);
             internalParseDocument(mapping, context, parser);
             validateEnd(parser);
-        } catch (Throwable t) {
-            throw wrapInMapperParsingException(source, t);
+        } catch (Exception e) {
+            throw wrapInMapperParsingException(source, e);
         }
         String remainingPath = context.path().pathAsText("");
         if (remainingPath.isEmpty() == false) {
@@ -173,7 +173,7 @@ final class DocumentParser {
     }
 
 
-    private static MapperParsingException wrapInMapperParsingException(SourceToParse source, Throwable e) {
+    private static MapperParsingException wrapInMapperParsingException(SourceToParse source, Exception e) {
         // if its already a mapper parsing exception, no need to wrap it...
         if (e instanceof MapperParsingException) {
             return (MapperParsingException) e;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
index c3c23d82f70..d706823d157 100755
--- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java
@@ -216,7 +216,7 @@ public class MapperService extends AbstractIndexComponent {
                         requireRefresh = true;
                     }
                 }
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 logger.warn("[{}] failed to add mapping [{}], source [{}]", e, index(), mappingType, mappingSource);
                 throw e;
             }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/Mapping.java b/core/src/main/java/org/elasticsearch/index/mapper/Mapping.java
index 6f2fea6a3d4..8c451c93774 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/Mapping.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/Mapping.java
@@ -26,6 +26,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.mapper.object.RootObjectMapper;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -136,7 +137,7 @@ public final class Mapping implements ToXContent {
             toXContent(builder, new ToXContent.MapParams(emptyMap()));
             return builder.endObject().string();
         } catch (IOException bogus) {
-            throw new AssertionError(bogus);
+            throw new UncheckedIOException(bogus);
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
index 93bd07c7bf9..510a8049630 100644
--- a/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
+++ b/core/src/main/java/org/elasticsearch/index/query/QueryParseContext.java
@@ -79,7 +79,7 @@ public class QueryParseContext implements ParseFieldMatcherSupplier {
             return queryBuilder;
         } catch (ParsingException e) {
             throw e;
-        } catch (Throwable e) {
+        } catch (Exception e) {
             throw new ParsingException(parser == null ? null : parser.getTokenLocation(), "Failed to parse", e);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 59851898d3c..ed4072b6b29 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -379,8 +379,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
                 // we want to refresh *before* we move to internal STARTED state
                 try {
                     getEngine().refresh("cluster_state_started");
-                } catch (Throwable t) {
-                    logger.debug("failed to refresh due to move to cluster wide started", t);
+                } catch (Exception e) {
+                    logger.debug("failed to refresh due to move to cluster wide started", e);
                 }
 
                 boolean movedToStarted = false;
@@ -483,9 +483,9 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         try {
             verifyPrimary();
             return prepareIndex(docMapper(source.type()), source, version, versionType, Engine.Operation.Origin.PRIMARY);
-        } catch (Throwable t) {
-            verifyNotClosed(t);
-            throw t;
+        } catch (Exception e) {
+            verifyNotClosed(e);
+            throw e;
         }
     }
 
@@ -493,9 +493,9 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         try {
             verifyReplicationTarget();
             return prepareIndex(docMapper(source.type()), source, version, versionType, Engine.Operation.Origin.REPLICA);
-        } catch (Throwable t) {
-            verifyNotClosed(t);
-            throw t;
+        } catch (Exception e) {
+            verifyNotClosed(e);
+            throw e;
         }
     }
 
@@ -531,9 +531,9 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
             }
             created = engine.index(index);
             index.endTime(System.nanoTime());
-        } catch (Throwable ex) {
-            indexingOperationListeners.postIndex(index, ex);
-            throw ex;
+        } catch (Exception e) {
+            indexingOperationListeners.postIndex(index, e);
+            throw e;
         }
 
         indexingOperationListeners.postIndex(index, created);
@@ -578,9 +578,9 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
             }
             engine.delete(delete);
             delete.endTime(System.nanoTime());
-        } catch (Throwable ex) {
-            indexingOperationListeners.postDelete(delete, ex);
-            throw ex;
+        } catch (Exception e) {
+            indexingOperationListeners.postDelete(delete, e);
+            throw e;
         }
 
         indexingOperationListeners.postDelete(delete);
@@ -816,7 +816,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
      * Fails the shard and marks the shard store as corrupted if
      * <code>e</code> is caused by index corruption
      */
-    public void failShard(String reason, @Nullable Throwable e) {
+    public void failShard(String reason, @Nullable Exception e) {
         // fail the engine. This will cause this shard to also be removed from the node's index service.
         getEngine().failEngine(reason, e);
     }
@@ -1074,7 +1074,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         verifyNotClosed(null);
     }
 
-    private void verifyNotClosed(Throwable suppressed) throws IllegalIndexShardStateException {
+    private void verifyNotClosed(Exception suppressed) throws IllegalIndexShardStateException {
         IndexShardState state = this.state; // one time volatile read
         if (state == IndexShardState.CLOSED) {
             final IllegalIndexShardStateException exc = new IndexShardClosedException(shardId, "operation only allowed when not closed");
@@ -1388,7 +1388,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
                 try {
                     markAsRecovering("from " + recoveryState.getSourceNode(), recoveryState);
                     recoveryTargetService.startRecovery(this, recoveryState.getType(), recoveryState.getSourceNode(), recoveryListener);
-                } catch (Throwable e) {
+                } catch (Exception e) {
                     failShard("corrupted preexisting index", e);
                     recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, e), true);
                 }
@@ -1400,8 +1400,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
                         if (recoverFromStore()) {
                             recoveryListener.onRecoveryDone(recoveryState);
                         }
-                    } catch (Throwable t) {
-                        recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, t), true);
+                    } catch (Exception e) {
+                        recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, e), true);
                     }
                 });
                 break;
@@ -1428,21 +1428,21 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
                                 .filter((s) -> shards.contains(s.shardId())).collect(Collectors.toList()))) {
                                 recoveryListener.onRecoveryDone(recoveryState);
                             }
-                        } catch (Throwable t) {
+                        } catch (Exception e) {
                             recoveryListener.onRecoveryFailure(recoveryState,
-                                new RecoveryFailedException(recoveryState, null, t), true);
+                                new RecoveryFailedException(recoveryState, null, e), true);
                         }
                     });
                 } else {
-                    final Throwable t;
+                    final Exception e;
                     if (numShards == -1) {
-                        t = new IndexNotFoundException(mergeSourceIndex);
+                        e = new IndexNotFoundException(mergeSourceIndex);
                     } else {
-                        t = new IllegalStateException("not all shards from index " + mergeSourceIndex
+                        e = new IllegalStateException("not all shards from index " + mergeSourceIndex
                             + " are started yet, expected " + numShards + " found " + startedShards.size() + " can't recover shard "
                             + shardId());
                     }
-                    recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, t), true);
+                    recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, e), true);
                 }
                 break;
             case SNAPSHOT:
@@ -1454,7 +1454,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
                         if (restoreFromRepository(indexShardRepository)) {
                             recoveryListener.onRecoveryDone(recoveryState);
                         }
-                    } catch (Throwable first) {
+                    } catch (Exception first) {
                         recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, first), true);
                     }
                 });
@@ -1469,13 +1469,14 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
 
         // called by the current engine
         @Override
-        public void onFailedEngine(String reason, @Nullable Throwable failure) {
+        public void onFailedEngine(String reason, @Nullable Exception failure) {
             final ShardFailure shardFailure = new ShardFailure(shardRouting, reason, failure);
             for (Callback<ShardFailure> listener : delegates) {
                 try {
                     listener.handle(shardFailure);
-                } catch (Exception e) {
-                    logger.warn("exception while notifying engine failure", e);
+                } catch (Exception inner) {
+                    inner.addSuppressed(failure);
+                    logger.warn("exception while notifying engine failure", inner);
                 }
             }
         }
@@ -1620,9 +1621,9 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
                     logger.debug("submitting async flush request");
                     final AbstractRunnable abstractRunnable = new AbstractRunnable() {
                         @Override
-                        public void onFailure(Throwable t) {
+                        public void onFailure(Exception e) {
                             if (state != IndexShardState.CLOSED) {
-                                logger.warn("failed to flush index", t);
+                                logger.warn("failed to flush index", e);
                             }
                         }
 
@@ -1665,9 +1666,9 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         public final ShardRouting routing;
         public final String reason;
         @Nullable
-        public final Throwable cause;
+        public final Exception cause;
 
-        public ShardFailure(ShardRouting routing, String reason, @Nullable Throwable cause) {
+        public ShardFailure(ShardRouting routing, String reason, @Nullable Exception cause) {
             this.routing = routing;
             this.reason = reason;
             this.cause = cause;
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexingOperationListener.java b/core/src/main/java/org/elasticsearch/index/shard/IndexingOperationListener.java
index aedc8554624..13ff87d4187 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexingOperationListener.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexingOperationListener.java
@@ -43,7 +43,7 @@ public interface IndexingOperationListener {
     /**
      * Called after the indexing operation occurred with exception.
      */
-    default void postIndex(Engine.Index index, Throwable ex) {}
+    default void postIndex(Engine.Index index, Exception ex) {}
 
     /**
      * Called before the delete occurs.
@@ -61,7 +61,7 @@ public interface IndexingOperationListener {
     /**
      * Called after the delete operation occurred with exception.
      */
-    default void postDelete(Engine.Delete delete, Throwable ex) {}
+    default void postDelete(Engine.Delete delete, Exception ex) {}
 
     /**
      * A Composite listener that multiplexes calls to each of the listeners methods.
@@ -81,8 +81,8 @@ public interface IndexingOperationListener {
             for (IndexingOperationListener listener : listeners) {
                 try {
                     listener.preIndex(operation);
-                } catch (Throwable t) {
-                    logger.warn("preIndex listener [{}] failed", t, listener);
+                } catch (Exception e) {
+                    logger.warn("preIndex listener [{}] failed", e, listener);
                 }
             }
             return operation;
@@ -94,20 +94,21 @@ public interface IndexingOperationListener {
             for (IndexingOperationListener listener : listeners) {
                 try {
                     listener.postIndex(index, created);
-                } catch (Throwable t) {
-                    logger.warn("postIndex listener [{}] failed", t, listener);
+                } catch (Exception e) {
+                    logger.warn("postIndex listener [{}] failed", e, listener);
                 }
             }
         }
 
         @Override
-        public void postIndex(Engine.Index index, Throwable ex) {
+        public void postIndex(Engine.Index index, Exception ex) {
             assert index != null && ex != null;
             for (IndexingOperationListener listener : listeners) {
                 try {
                     listener.postIndex(index, ex);
-                } catch (Throwable t) {
-                    logger.warn("postIndex listener [{}] failed", t, listener);
+                } catch (Exception inner) {
+                    inner.addSuppressed(ex);
+                    logger.warn("postIndex listener [{}] failed", inner, listener);
                 }
             }
         }
@@ -118,8 +119,8 @@ public interface IndexingOperationListener {
             for (IndexingOperationListener listener : listeners) {
                 try {
                     listener.preDelete(delete);
-                } catch (Throwable t) {
-                    logger.warn("preDelete listener [{}] failed", t, listener);
+                } catch (Exception e) {
+                    logger.warn("preDelete listener [{}] failed", e, listener);
                 }
             }
             return delete;
@@ -131,20 +132,21 @@ public interface IndexingOperationListener {
             for (IndexingOperationListener listener : listeners) {
                 try {
                     listener.postDelete(delete);
-                } catch (Throwable t) {
-                    logger.warn("postDelete listener [{}] failed", t, listener);
+                } catch (Exception e) {
+                    logger.warn("postDelete listener [{}] failed", e, listener);
                 }
             }
         }
 
         @Override
-        public void postDelete(Engine.Delete delete, Throwable ex) {
+        public void postDelete(Engine.Delete delete, Exception ex) {
             assert delete != null && ex != null;
             for (IndexingOperationListener listener : listeners) {
                 try {
                     listener.postDelete(delete, ex);
-                } catch (Throwable t) {
-                    logger.warn("postDelete listener [{}] failed", t, listener);
+                } catch (Exception inner) {
+                    inner.addSuppressed(ex);
+                    logger.warn("postDelete listener [{}] failed", inner, listener);
                 }
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java b/core/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java
index b85ca7e6a9d..f62b8f7fe3c 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/InternalIndexingStats.java
@@ -86,7 +86,7 @@ final class InternalIndexingStats implements IndexingOperationListener {
     }
 
     @Override
-    public void postIndex(Engine.Index index, Throwable ex) {
+    public void postIndex(Engine.Index index, Exception ex) {
         if (!index.origin().isRecovery()) {
             totalStats.indexCurrent.dec();
             typeStats(index.type()).indexCurrent.dec();
@@ -118,7 +118,7 @@ final class InternalIndexingStats implements IndexingOperationListener {
     }
 
     @Override
-    public void postDelete(Engine.Delete delete, Throwable ex) {
+    public void postDelete(Engine.Delete delete, Exception ex) {
         if (!delete.origin().isRecovery()) {
             totalStats.deleteCurrent.dec();
             typeStats(delete.type()).deleteCurrent.dec();
diff --git a/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java b/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
index b594b31abb8..76352e79bb4 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java
@@ -200,8 +200,8 @@ public final class RefreshListeners implements ReferenceManager.RefreshListener
                 for (Consumer<Boolean> listener : finalListenersToFire) {
                     try {
                         listener.accept(false);
-                    } catch (Throwable t) {
-                        logger.warn("Error firing refresh listener", t);
+                    } catch (Exception e) {
+                        logger.warn("Error firing refresh listener", e);
                     }
                 }
             });
diff --git a/core/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java b/core/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
index 012617edcd0..5a4ac1297f7 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/SearchOperationListener.java
@@ -119,8 +119,8 @@ public interface SearchOperationListener {
             for (SearchOperationListener listener : listeners) {
                 try {
                     listener.onPreQueryPhase(searchContext);
-                } catch (Throwable t) {
-                    logger.warn("onPreQueryPhase listener [{}] failed", t, listener);
+                } catch (Exception e) {
+                    logger.warn("onPreQueryPhase listener [{}] failed", e, listener);
                 }
             }
         }
@@ -130,8 +130,8 @@ public interface SearchOperationListener {
             for (SearchOperationListener listener : listeners) {
                 try {
                     listener.onFailedQueryPhase(searchContext);
-                } catch (Throwable t) {
-                    logger.warn("onFailedQueryPhase listener [{}] failed", t, listener);
+                } catch (Exception e) {
+                    logger.warn("onFailedQueryPhase listener [{}] failed", e, listener);
                 }
             }
         }
@@ -141,8 +141,8 @@ public interface SearchOperationListener {
             for (SearchOperationListener listener : listeners) {
                 try {
                     listener.onQueryPhase(searchContext, tookInNanos);
-                } catch (Throwable t) {
-                    logger.warn("onQueryPhase listener [{}] failed", t, listener);
+                } catch (Exception e) {
+                    logger.warn("onQueryPhase listener [{}] failed", e, listener);
                 }
             }
         }
@@ -152,8 +152,8 @@ public interface SearchOperationListener {
             for (SearchOperationListener listener : listeners) {
                 try {
                     listener.onPreFetchPhase(searchContext);
-                } catch (Throwable t) {
-                    logger.warn("onPreFetchPhase listener [{}] failed", t, listener);
+                } catch (Exception e) {
+                    logger.warn("onPreFetchPhase listener [{}] failed", e, listener);
                 }
             }
         }
@@ -163,8 +163,8 @@ public interface SearchOperationListener {
             for (SearchOperationListener listener : listeners) {
                 try {
                     listener.onFailedFetchPhase(searchContext);
-                } catch (Throwable t) {
-                    logger.warn("onFailedFetchPhase listener [{}] failed", t, listener);
+                } catch (Exception e) {
+                    logger.warn("onFailedFetchPhase listener [{}] failed", e, listener);
                 }
             }
         }
@@ -174,8 +174,8 @@ public interface SearchOperationListener {
             for (SearchOperationListener listener : listeners) {
                 try {
                     listener.onFetchPhase(searchContext, tookInNanos);
-                } catch (Throwable t) {
-                    logger.warn("onFetchPhase listener [{}] failed", t, listener);
+                } catch (Exception e) {
+                    logger.warn("onFetchPhase listener [{}] failed", e, listener);
                 }
             }
         }
@@ -185,8 +185,8 @@ public interface SearchOperationListener {
             for (SearchOperationListener listener : listeners) {
                 try {
                     listener.onNewContext(context);
-                } catch (Throwable t) {
-                    logger.warn("onNewContext listener [{}] failed", t, listener);
+                } catch (Exception e) {
+                    logger.warn("onNewContext listener [{}] failed", e, listener);
                 }
             }
         }
@@ -196,8 +196,8 @@ public interface SearchOperationListener {
             for (SearchOperationListener listener : listeners) {
                 try {
                     listener.onFreeContext(context);
-                } catch (Throwable t) {
-                    logger.warn("onFreeContext listener [{}] failed", t, listener);
+                } catch (Exception e) {
+                    logger.warn("onFreeContext listener [{}] failed", e, listener);
                 }
             }
         }
@@ -207,8 +207,8 @@ public interface SearchOperationListener {
             for (SearchOperationListener listener : listeners) {
                 try {
                     listener.onNewScrollContext(context);
-                } catch (Throwable t) {
-                    logger.warn("onNewScrollContext listener [{}] failed", t, listener);
+                } catch (Exception e) {
+                    logger.warn("onNewScrollContext listener [{}] failed", e, listener);
                 }
             }
         }
@@ -218,8 +218,8 @@ public interface SearchOperationListener {
             for (SearchOperationListener listener : listeners) {
                 try {
                     listener.onFreeScrollContext(context);
-                } catch (Throwable t) {
-                    logger.warn("onFreeScrollContext listener [{}] failed", t, listener);
+                } catch (Exception e) {
+                    logger.warn("onFreeScrollContext listener [{}] failed", e, listener);
                 }
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
index dbfcad6048a..4a9284d8155 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
@@ -319,12 +319,13 @@ final class StoreRecovery {
                 store.failIfCorrupted();
                 try {
                     si = store.readLastCommittedSegmentsInfo();
-                } catch (Throwable e) {
+                } catch (Exception e) {
                     String files = "_unknown_";
                     try {
                         files = Arrays.toString(store.directory().listAll());
-                    } catch (Throwable e1) {
-                        files += " (failure=" + ExceptionsHelper.detailedMessage(e1) + ")";
+                    } catch (Exception inner) {
+                        inner.addSuppressed(e);
+                        files += " (failure=" + ExceptionsHelper.detailedMessage(inner) + ")";
                     }
                     if (indexShouldExists) {
                         throw new IndexShardRecoveryException(shardId, "shard allocated for local recovery (post api), should exist, but doesn't, current files: " + files, e);
@@ -340,7 +341,7 @@ final class StoreRecovery {
                         Lucene.cleanLuceneIndex(store.directory());
                     }
                 }
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 throw new IndexShardRecoveryException(shardId, "failed to fetch index version after copying it over", e);
             }
             recoveryState.getIndex().updateVersion(version);
@@ -400,8 +401,8 @@ final class StoreRecovery {
             indexShard.skipTranslogRecovery();
             indexShard.finalizeRecovery();
             indexShard.postRecovery("restore done");
-        } catch (Throwable t) {
-            throw new IndexShardRestoreFailedException(shardId, "restore failed", t);
+        } catch (Exception e) {
+            throw new IndexShardRestoreFailedException(shardId, "restore failed", e);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java
index 54cfa80926c..78628a02c49 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java
@@ -72,8 +72,8 @@ public class TranslogRecoveryPerformer {
                 numOps++;
             }
             engine.getTranslog().sync();
-        } catch (Throwable t) {
-            throw new BatchOperationException(shardId, "failed to apply batch translog operation", numOps, t);
+        } catch (Exception e) {
+            throw new BatchOperationException(shardId, "failed to apply batch translog operation", numOps, e);
         }
         return numOps;
     }
diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java
index 96230459934..526aab7581b 100644
--- a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java
+++ b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java
@@ -181,7 +181,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements
             snapshotContext.snapshot(snapshotIndexCommit);
             snapshotStatus.time(System.currentTimeMillis() - snapshotStatus.startTime());
             snapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.DONE);
-        } catch (Throwable e) {
+        } catch (Exception e) {
             snapshotStatus.time(System.currentTimeMillis() - snapshotStatus.startTime());
             snapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.FAILURE);
             snapshotStatus.failure(ExceptionsHelper.detailedMessage(e));
@@ -201,7 +201,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements
         final RestoreContext snapshotContext = new RestoreContext(snapshotId, version, shardId, snapshotShardId, recoveryState);
         try {
             snapshotContext.restore();
-        } catch (Throwable e) {
+        } catch (Exception e) {
             throw new IndexShardRestoreFailedException(shardId, "failed to restore snapshot [" + snapshotId + "]", e);
         }
     }
@@ -559,7 +559,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements
                                 // don't have this hash we try to read that hash from the blob store
                                 // in a bwc compatible way.
                                 maybeRecalculateMetadataHash(blobContainer, fileInfo, metadata);
-                            } catch (Throwable e) {
+                            } catch (Exception e) {
                                 logger.warn("{} Can't calculate hash from blob for file [{}] [{}]", e, shardId, fileInfo.physicalName(), fileInfo.metadata());
                             }
                             if (fileInfo.isSame(md) && snapshotFileExistsInBlobs(fileInfo, blobs)) {
@@ -651,19 +651,20 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements
                 }
                 Store.verify(indexInput);
                 snapshotStatus.addProcessedFile(fileInfo.length());
-            } catch (Throwable t) {
+            } catch (Exception t) {
                 failStoreIfCorrupted(t);
                 snapshotStatus.addProcessedFile(0);
                 throw t;
             }
         }
 
-        private void failStoreIfCorrupted(Throwable t) {
-            if (t instanceof CorruptIndexException || t instanceof IndexFormatTooOldException || t instanceof IndexFormatTooNewException) {
+        private void failStoreIfCorrupted(Exception e) {
+            if (e instanceof CorruptIndexException || e instanceof IndexFormatTooOldException || e instanceof IndexFormatTooNewException) {
                 try {
-                    store.markStoreCorrupted((IOException) t);
-                } catch (IOException e) {
-                    logger.warn("store cannot be marked as corrupted", e);
+                    store.markStoreCorrupted((IOException) e);
+                } catch (IOException inner) {
+                    inner.addSuppressed(e);
+                    logger.warn("store cannot be marked as corrupted", inner);
                 }
             }
         }
@@ -730,7 +731,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements
      * The new logic for StoreFileMetaData reads the entire <tt>.si</tt> and <tt>segments.n</tt> files to strengthen the
      * comparison of the files on a per-segment / per-commit level.
      */
-    private static void maybeRecalculateMetadataHash(final BlobContainer blobContainer, final FileInfo fileInfo, Store.MetadataSnapshot snapshot) throws Throwable {
+    private static void maybeRecalculateMetadataHash(final BlobContainer blobContainer, final FileInfo fileInfo, Store.MetadataSnapshot snapshot) throws Exception {
         final StoreFileMetaData metadata;
         if (fileInfo != null && (metadata = snapshot.get(fileInfo.physicalName())) != null) {
             if (metadata.hash().length > 0 && fileInfo.metadata().hash().length == 0) {
@@ -836,7 +837,7 @@ public class BlobStoreIndexShardRepository extends AbstractComponent implements
                         // don't have this hash we try to read that hash from the blob store
                         // in a bwc compatible way.
                         maybeRecalculateMetadataHash(blobContainer, fileInfo, recoveryTargetMetadata);
-                    } catch (Throwable e) {
+                    } catch (Exception e) {
                         // if the index is broken we might not be able to read it
                         logger.warn("{} Can't calculate hash from blog for file [{}] [{}]", e, shardId, fileInfo.physicalName(), fileInfo.metadata());
                     }
diff --git a/core/src/main/java/org/elasticsearch/index/store/Store.java b/core/src/main/java/org/elasticsearch/index/store/Store.java
index 2f647933ea2..211f97d8916 100644
--- a/core/src/main/java/org/elasticsearch/index/store/Store.java
+++ b/core/src/main/java/org/elasticsearch/index/store/Store.java
@@ -302,7 +302,7 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
                 try {
                     directory.deleteFile(origFile);
                 } catch (FileNotFoundException | NoSuchFileException e) {
-                } catch (Throwable ex) {
+                } catch (Exception ex) {
                     logger.debug("failed to delete file [{}]", ex, origFile);
                 }
                 // now, rename the files... and fail it it won't work
@@ -569,7 +569,7 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
                         final byte[] buffer = new byte[size];
                         input.readBytes(buffer, 0, buffer.length);
                         StreamInput in = StreamInput.wrap(buffer);
-                        Throwable t = in.readThrowable();
+                        Exception t = in.readException();
                         if (t instanceof CorruptIndexException) {
                             ex.add((CorruptIndexException) t);
                         } else {
@@ -838,7 +838,7 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
             } catch (CorruptIndexException | IndexNotFoundException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
                 // we either know the index is corrupted or it's just not there
                 throw ex;
-            } catch (Throwable ex) {
+            } catch (Exception ex) {
                 try {
                     // Lucene checks the checksum after it tries to lookup the codec etc.
                     // in that case we might get only IAE or similar exceptions while we are really corrupt...
@@ -846,14 +846,14 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
                     logger.warn("failed to build store metadata. checking segment info integrity (with commit [{}])",
                             ex, commit == null ? "no" : "yes");
                     Lucene.checkSegmentInfoIntegrity(directory);
+                    throw ex;
                 } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException cex) {
                     cex.addSuppressed(ex);
                     throw cex;
-                } catch (Throwable e) {
-                    // ignore...
+                } catch (Exception inner) {
+                    ex.addSuppressed(inner);
+                    throw ex;
                 }
-
-                throw ex;
             }
             return new LoadedMetadata(unmodifiableMap(builder), unmodifiableMap(commitUserDataBuilder), numDocs);
         }
@@ -878,7 +878,7 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
                         checksum = digestToString(CodecUtil.retrieveChecksum(in));
                     }
 
-                } catch (Throwable ex) {
+                } catch (Exception ex) {
                     logger.debug("Can retrieve checksum from file [{}]", ex, file);
                     throw ex;
                 }
@@ -1320,8 +1320,8 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
         for (String file : files) {
             try {
                 directory().deleteFile(file);
-            } catch (Throwable ex) {
-                // ignore
+            } catch (Exception ex) {
+                // ignore :(
             }
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
index 6b8bc607679..fb013f5c31f 100644
--- a/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
+++ b/core/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java
@@ -65,9 +65,6 @@ import java.util.function.LongSupplier;
 
 import static org.elasticsearch.index.mapper.SourceToParse.source;
 
-/**
- */
-
 public class TermVectorsService  {
 
 
@@ -149,7 +146,7 @@ public class TermVectorsService  {
                 termVectorsResponse.setFields(termVectorsByField, request.selectedFields(), request.getFlags(), topLevelFields, dfs, termVectorsFilter);
             }
             termVectorsResponse.setTookInMillis(TimeUnit.NANOSECONDS.toMillis(nanoTimeSupplier.getAsLong() - startTime));
-        } catch (Throwable ex) {
+        } catch (Exception ex) {
             throw new ElasticsearchException("failed to execute term vector request", ex);
         } finally {
             searcher.close();
@@ -237,8 +234,7 @@ public class TermVectorsService  {
         return selectedFields;
     }
 
-    private static Fields generateTermVectors(IndexShard indexShard, Collection<GetField> getFields, boolean withOffsets, @Nullable Map<String, String> perFieldAnalyzer, Set<String> fields)
-            throws IOException {
+    private static Fields generateTermVectors(IndexShard indexShard, Collection<GetField> getFields, boolean withOffsets, @Nullable Map<String, String> perFieldAnalyzer, Set<String> fields) throws IOException {
         /* store document in memory index */
         MemoryIndex index = new MemoryIndex(withOffsets);
         for (GetField getField : getFields) {
@@ -256,7 +252,7 @@ public class TermVectorsService  {
         return MultiFields.getFields(index.createSearcher().getIndexReader());
     }
 
-    private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVectorsRequest request, boolean doAllFields) throws Throwable {
+    private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVectorsRequest request, boolean doAllFields) throws IOException {
         // parse the document, at the moment we do update the mapping, just like percolate
         ParsedDocument parsedDocument = parseDocument(indexShard, indexShard.shardId().getIndexName(), request.type(), request.doc());
 
@@ -287,7 +283,7 @@ public class TermVectorsService  {
         return generateTermVectors(indexShard, getFields, request.offsets(), request.perFieldAnalyzer(), seenFields);
     }
 
-    private static ParsedDocument parseDocument(IndexShard indexShard, String index, String type, BytesReference doc) throws Throwable {
+    private static ParsedDocument parseDocument(IndexShard indexShard, String index, String type, BytesReference doc) {
         MapperService mapperService = indexShard.mapperService();
         DocumentMapperForType docMapper = mapperService.documentMapperWithAutoCreate(type);
         ParsedDocument parsedDocument = docMapper.getDocumentMapper().parse(source(index, type, "_id_for_tv_api", doc));
diff --git a/core/src/main/java/org/elasticsearch/index/translog/Translog.java b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
index 083e011516c..cbb54bd9d50 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/Translog.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
@@ -206,11 +206,11 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
 
             }
             // now that we know which files are there, create a new current one.
-        } catch (Throwable t) {
+        } catch (Exception e) {
             // close the opened translog files if we fail to create a new translog...
             IOUtils.closeWhileHandlingException(current);
             IOUtils.closeWhileHandlingException(readers);
-            throw t;
+            throw e;
         }
     }
 
@@ -437,10 +437,18 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
                 return location;
             }
         } catch (AlreadyClosedException | IOException ex) {
-            closeOnTragicEvent(ex);
+            try {
+                closeOnTragicEvent(ex);
+            } catch (Exception inner) {
+                ex.addSuppressed(inner);
+            }
             throw ex;
-        } catch (Throwable e) {
-            closeOnTragicEvent(e);
+        } catch (Exception e) {
+            try {
+                closeOnTragicEvent(e);
+            } catch (Exception inner) {
+                e.addSuppressed(inner);
+            }
             throw new TranslogException(shardId, "Failed to write operation [" + operation + "]", e);
         } finally {
             Releasables.close(out.bytes());
@@ -508,8 +516,12 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
             if (closed.get() == false) {
                 current.sync();
             }
-        } catch (Throwable ex) {
-            closeOnTragicEvent(ex);
+        } catch (Exception ex) {
+            try {
+                closeOnTragicEvent(ex);
+            } catch (Exception inner) {
+                ex.addSuppressed(inner);
+            }
             throw ex;
         }
     }
@@ -541,14 +553,18 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
                 ensureOpen();
                 return current.syncUpTo(location.translogLocation + location.size);
             }
-        } catch (Throwable ex) {
-            closeOnTragicEvent(ex);
+        } catch (Exception ex) {
+            try {
+                closeOnTragicEvent(ex);
+            } catch (Exception inner) {
+                ex.addSuppressed(inner);
+            }
             throw ex;
         }
         return false;
     }
 
-    private void closeOnTragicEvent(Throwable ex) {
+    private void closeOnTragicEvent(Exception ex) {
         if (current.getTragicException() != null) {
             try {
                 close();
@@ -1193,9 +1209,9 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
             current = createWriter(current.getGeneration() + 1);
             logger.trace("current translog set to [{}]", current.getGeneration());
 
-        } catch (Throwable t) {
+        } catch (Exception e) {
             IOUtils.closeWhileHandlingException(this); // tragic event
-            throw t;
+            throw e;
         }
     }
 
@@ -1318,7 +1334,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
      * e.g. disk full while flushing a new segment, this returns the root cause exception.
      * Otherwise (no tragic exception has occurred) it returns null.
      */
-    public Throwable getTragicException() {
+    public Exception getTragicException() {
         return current.getTragicException();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java b/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
index b873c414b4d..89d12983b07 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java
@@ -26,7 +26,6 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.Channels;
-import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.index.shard.ShardId;
 
@@ -36,7 +35,6 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
-import java.nio.file.OpenOption;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -55,7 +53,7 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
     /* the number of translog operations written to this file */
     private volatile int operationCounter;
     /* if we hit an exception that we can't recover from we assign it to this var and ship it with every AlreadyClosedException we throw */
-    private volatile Throwable tragedy;
+    private volatile Exception tragedy;
     /* A buffered outputstream what writes to the writers channel */
     private final OutputStream outputStream;
     /* the total offset of this file including the bytes written to the file as well as into the buffer */
@@ -97,11 +95,11 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
             writeCheckpoint(channelFactory, headerLength, 0, file.getParent(), fileGeneration);
             final TranslogWriter writer = new TranslogWriter(channelFactory, shardId, fileGeneration, channel, file, bufferSize);
             return writer;
-        } catch (Throwable throwable) {
+        } catch (Exception exception) {
             // if we fail to bake the file-generation into the checkpoint we stick with the file and once we recover and that
             // file exists we remove it. We only apply this logic to the checkpoint.generation+1 any other file with a higher generation is an error condition
             IOUtils.closeWhileHandlingException(channel);
-            throw throwable;
+            throw exception;
         }
     }
 
@@ -110,18 +108,18 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
      * e.g. disk full while flushing a new segment, this returns the root cause exception.
      * Otherwise (no tragic exception has occurred) it returns null.
      */
-    public Throwable getTragicException() {
+    public Exception getTragicException() {
         return tragedy;
     }
 
-    private synchronized void closeWithTragicEvent(Throwable throwable) throws IOException {
-        assert throwable != null : "throwable must not be null in a tragic event";
+    private synchronized void closeWithTragicEvent(Exception exception) throws IOException {
+        assert exception != null;
         if (tragedy == null) {
-            tragedy = throwable;
-        } else if (tragedy != throwable) {
+            tragedy = exception;
+        } else if (tragedy != exception) {
             // it should be safe to call closeWithTragicEvents on multiple layers without
             // worrying about self suppression.
-            tragedy.addSuppressed(throwable);
+            tragedy.addSuppressed(exception);
         }
         close();
     }
@@ -134,8 +132,12 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
         final long offset = totalOffset;
         try {
             data.writeTo(outputStream);
-        } catch (Throwable ex) {
-            closeWithTragicEvent(ex);
+        } catch (Exception ex) {
+            try {
+                closeWithTragicEvent(ex);
+            } catch (Exception inner) {
+                ex.addSuppressed(inner);
+            }
             throw ex;
         }
         totalOffset += data.length();
@@ -184,7 +186,11 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
                 try {
                     sync(); // sync before we close..
                 } catch (IOException e) {
-                    closeWithTragicEvent(e);
+                    try {
+                        closeWithTragicEvent(e);
+                    } catch (Exception inner) {
+                        e.addSuppressed(inner);
+                    }
                     throw e;
                 }
                 if (closed.compareAndSet(false, true)) {
@@ -247,8 +253,12 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
                             outputStream.flush();
                             offsetToSync = totalOffset;
                             opsCounter = operationCounter;
-                        } catch (Throwable ex) {
-                            closeWithTragicEvent(ex);
+                        } catch (Exception ex) {
+                            try {
+                                closeWithTragicEvent(ex);
+                            } catch (Exception inner) {
+                                ex.addSuppressed(inner);
+                            }
                             throw ex;
                         }
                     }
@@ -257,8 +267,12 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
                     try {
                         channel.force(false);
                         writeCheckpoint(channelFactory, offsetToSync, opsCounter, path.getParent(), generation);
-                    } catch (Throwable ex) {
-                        closeWithTragicEvent(ex);
+                    } catch (Exception ex) {
+                        try {
+                            closeWithTragicEvent(ex);
+                        } catch (Exception inner) {
+                            ex.addSuppressed(inner);
+                        }
                         throw ex;
                     }
                     assert lastSyncedOffset <= offsetToSync : "illegal state: " + lastSyncedOffset + " <= " + offsetToSync;
@@ -324,8 +338,12 @@ public class TranslogWriter extends BaseTranslogReader implements Closeable {
                 try {
                     ensureOpen();
                     super.flush();
-                } catch (Throwable ex) {
-                    closeWithTragicEvent(ex);
+                } catch (Exception ex) {
+                    try {
+                        closeWithTragicEvent(ex);
+                    } catch (Exception inner) {
+                        ex.addSuppressed(inner);
+                    }
                     throw ex;
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java b/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
index 2a09362e871..514060de844 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java
@@ -176,8 +176,8 @@ public class IndexingMemoryController extends AbstractComponent implements Index
             }
 
             @Override
-            public void onFailure(Throwable t) {
-                logger.warn("failed to write indexing buffer for shard [{}]; ignoring", t, shard.shardId());
+            public void onFailure(Exception e) {
+                logger.warn("failed to write indexing buffer for shard [{}]; ignoring", e, shard.shardId());
             }
         });
     }
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
index fadd10eae58..e0b6bb3f0c7 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java
@@ -20,15 +20,14 @@
 package org.elasticsearch.indices;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
-
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.util.CollectionUtil;
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
-import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
+import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
 import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.action.fieldstats.FieldStats;
@@ -53,8 +52,8 @@ import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
-import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
@@ -132,9 +131,6 @@ import static java.util.Collections.unmodifiableMap;
 import static org.elasticsearch.common.collect.MapBuilder.newMapBuilder;
 import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList;
 
-/**
- *
- */
 public class IndicesService extends AbstractLifecycleComponent
     implements IndicesClusterStateService.AllocatedIndices<IndexShard, IndexService>, IndexService.ShardStoreDeleter {
 
@@ -221,7 +217,7 @@ public class IndicesService extends AbstractLifecycleComponent
             indicesStopExecutor.execute(() -> {
                 try {
                     removeIndex(index, "shutdown", false);
-                } catch (Throwable e) {
+                } catch (Exception e) {
                     logger.warn("failed to remove index on stop [{}]", e, index);
                 } finally {
                     latch.countDown();
@@ -477,7 +473,7 @@ public class IndicesService extends AbstractLifecycleComponent
     public void removeIndex(Index index, String reason) {
         try {
             removeIndex(index, reason, false);
-        } catch (Throwable e) {
+        } catch (Exception e) {
             logger.warn("failed to remove index ({})", e, reason);
         }
     }
@@ -568,7 +564,7 @@ public class IndicesService extends AbstractLifecycleComponent
     public void deleteIndex(Index index, String reason) {
         try {
             removeIndex(index, reason, true);
-        } catch (Throwable e) {
+        } catch (Exception e) {
             logger.warn("failed to delete index ({})", e, reason);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java b/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java
index e7ba2bfd70f..f3812f6900e 100644
--- a/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java
+++ b/core/src/main/java/org/elasticsearch/indices/analysis/HunspellService.java
@@ -96,7 +96,7 @@ public class HunspellService extends AbstractComponent {
         this.loadingFunction = (locale) -> {
             try {
                 return loadDictionary(locale, settings, env);
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 throw new IllegalStateException("failed to load hunspell dictionary for locale: " + locale, e);
             }
         };
@@ -135,7 +135,7 @@ public class HunspellService extends AbstractComponent {
                             if (inner.iterator().hasNext()) { // just making sure it's indeed a dictionary dir
                                 try {
                                     getDictionary(file.getFileName().toString());
-                                } catch (Throwable e) {
+                                } catch (Exception e) {
                                     // The cache loader throws unchecked exception (see #loadDictionary()),
                                     // here we simply report the exception and continue loading the dictionaries
                                     logger.error("exception while loading dictionary {}", e, file.getFileName());
diff --git a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
index b5644ac3bb0..3338f43ab4a 100644
--- a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
+++ b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
@@ -79,9 +79,6 @@ import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
 
-/**
- *
- */
 public class IndicesClusterStateService extends AbstractLifecycleComponent implements ClusterStateListener {
 
     final AllocatedIndices<? extends Shard, ? extends AllocatedIndex<? extends Shard>> indicesService;
@@ -268,8 +265,8 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
             if (indexSettings != null) {
                 threadPool.generic().execute(new AbstractRunnable() {
                     @Override
-                    public void onFailure(Throwable t) {
-                        logger.warn("[{}] failed to complete pending deletion for index", t, index);
+                    public void onFailure(Exception e) {
+                        logger.warn("[{}] failed to complete pending deletion for index", e, index);
                     }
 
                     @Override
@@ -427,7 +424,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
                             indexMetaData.getIndexUUID(), state.nodes().getLocalNodeId())
                     );
                 }
-            } catch (Throwable t) {
+            } catch (Exception e) {
                 final String failShardReason;
                 if (indexService == null) {
                     failShardReason = "failed to create index";
@@ -436,7 +433,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
                     indicesService.removeIndex(index, "removing index (mapping update failed)");
                 }
                 for (ShardRouting shardRouting : entry.getValue()) {
-                    sendFailShard(shardRouting, failShardReason, t);
+                    sendFailShard(shardRouting, failShardReason, e);
                 }
             }
         }
@@ -461,7 +458,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
                                 newIndexMetaData.getIndexUUID(), state.nodes().getLocalNodeId())
                         );
                     }
-                } catch (Throwable t) {
+                } catch (Exception e) {
                     indicesService.removeIndex(indexService.index(), "removing index (mapping update failed)");
 
                     // fail shards that would be created or updated by createOrUpdateShards
@@ -469,7 +466,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
                     if (localRoutingNode != null) {
                         for (final ShardRouting shardRouting : localRoutingNode) {
                             if (shardRouting.index().equals(index) && failedShardsCache.containsKey(shardRouting.shardId()) == false) {
-                                sendFailShard(shardRouting, "failed to update mapping for index", t);
+                                sendFailShard(shardRouting, "failed to update mapping for index", e);
                             }
                         }
                     }
@@ -526,7 +523,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
             // ignore this, the method call can happen several times
             logger.debug("Trying to create shard that already exists", e);
             assert false;
-        } catch (Throwable e) {
+        } catch (Exception e) {
             failAndRemoveShard(shardRouting, true, "failed to create shard", e);
         }
     }
@@ -539,7 +536,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
 
         try {
             shard.updateRoutingEntry(shardRouting);
-        } catch (Throwable e) {
+        } catch (Exception e) {
             failAndRemoveShard(shardRouting, true, "failed updating shard routing entry", e);
             return;
         }
@@ -627,7 +624,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
                     if (Lucene.isCorruptionException(e.getCause())) {
                         restoreService.failRestore(state.getRestoreSource().snapshot(), shardRouting.shardId());
                     }
-                } catch (Throwable inner) {
+                } catch (Exception inner) {
                     e.addSuppressed(inner);
                 } finally {
                     handleRecoveryFailure(shardRouting, sendShardFailure, e);
@@ -659,11 +656,11 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
         }
     }
 
-    private synchronized void handleRecoveryFailure(ShardRouting shardRouting, boolean sendShardFailure, Throwable failure) {
+    private synchronized void handleRecoveryFailure(ShardRouting shardRouting, boolean sendShardFailure, Exception failure) {
         failAndRemoveShard(shardRouting, sendShardFailure, "failed recovery", failure);
     }
 
-    private void failAndRemoveShard(ShardRouting shardRouting, boolean sendShardFailure, String message, @Nullable Throwable failure) {
+    private void failAndRemoveShard(ShardRouting shardRouting, boolean sendShardFailure, String message, @Nullable Exception failure) {
         try {
             AllocatedIndex<? extends Shard> indexService = indicesService.indexService(shardRouting.shardId().getIndex());
             if (indexService != null) {
@@ -671,8 +668,13 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
             }
         } catch (ShardNotFoundException e) {
             // the node got closed on us, ignore it
-        } catch (Throwable e1) {
-            logger.warn("[{}][{}] failed to remove shard after failure ([{}])", e1, shardRouting.getIndexName(), shardRouting.getId(),
+        } catch (Exception inner) {
+            inner.addSuppressed(failure);
+            logger.warn(
+                "[{}][{}] failed to remove shard after failure ([{}])",
+                inner,
+                shardRouting.getIndexName(),
+                shardRouting.getId(),
                 message);
         }
         if (sendShardFailure) {
@@ -680,14 +682,19 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent imple
         }
     }
 
-    private void sendFailShard(ShardRouting shardRouting, String message, @Nullable Throwable failure) {
+    private void sendFailShard(ShardRouting shardRouting, String message, @Nullable Exception failure) {
         try {
             logger.warn("[{}] marking and sending shard failed due to [{}]", failure, shardRouting.shardId(), message);
             failedShardsCache.put(shardRouting.shardId(), shardRouting);
             shardStateAction.shardFailed(shardRouting, shardRouting, message, failure, SHARD_STATE_ACTION_LISTENER);
-        } catch (Throwable e1) {
-            logger.warn("[{}][{}] failed to mark shard as failed (because of [{}])", e1, shardRouting.getIndexName(), shardRouting.getId(),
-                message);
+        } catch (Exception inner) {
+            if (failure != null) inner.addSuppressed(failure);
+            logger.warn(
+                    "[{}][{}] failed to mark shard as failed (because of [{}])",
+                    inner,
+                    shardRouting.getIndexName(),
+                    shardRouting.getId(),
+                    message);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java b/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java
index 4f7d482f8a9..3ab18dd1bd0 100644
--- a/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java
+++ b/core/src/main/java/org/elasticsearch/indices/fielddata/cache/IndicesFieldDataCache.java
@@ -48,8 +48,6 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.function.ToLongBiFunction;
 
-/**
- */
 public class IndicesFieldDataCache extends AbstractComponent implements RemovalListener<IndicesFieldDataCache.Key, Accountable>, Releasable{
 
     public static final Setting<ByteSizeValue> INDICES_FIELDDATA_CACHE_SIZE_KEY =
@@ -91,7 +89,7 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL
         for (IndexFieldDataCache.Listener listener : key.listeners) {
             try {
                 listener.onRemoval(key.shardId, indexCache.fieldName, notification.getRemovalReason() == RemovalNotification.RemovalReason.EVICTED, value.ramBytesUsed());
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 // load anyway since listeners should not throw exceptions
                 logger.error("Failed to call listener on field data cache unloading", e);
             }
@@ -138,7 +136,7 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL
                 for (Listener listener : k.listeners) {
                     try {
                         listener.onCache(shardId, fieldName, fieldData);
-                    } catch (Throwable e) {
+                    } catch (Exception e) {
                         // load anyway since listeners should not throw exceptions
                         logger.error("Failed to call listener on atomic field data loading", e);
                     }
@@ -162,7 +160,7 @@ public class IndicesFieldDataCache extends AbstractComponent implements RemovalL
                 for (Listener listener : k.listeners) {
                     try {
                         listener.onCache(shardId, fieldName, ifd);
-                    } catch (Throwable e) {
+                    } catch (Exception e) {
                         // load anyway since listeners should not throw exceptions
                         logger.error("Failed to call listener on global ordinals loading", e);
                     }
diff --git a/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java b/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
index 0fcdd16aba7..66963a4c59c 100644
--- a/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
+++ b/core/src/main/java/org/elasticsearch/indices/flush/SyncedFlushService.java
@@ -99,7 +99,7 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     logger.debug("{} sync flush on inactive shard failed", e, indexShard.shardId());
                 }
             });
@@ -143,7 +143,7 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL
                     }
 
                     @Override
-                    public void onFailure(Throwable e) {
+                    public void onFailure(Exception e) {
                         logger.debug("{} unexpected error while executing synced flush", shardId);
                         final int totalShards = indexMetaData.getNumberOfReplicas() + 1;
                         results.get(index).add(new ShardsSyncedFlushResult(shardId, totalShards, e.getMessage()));
@@ -219,7 +219,7 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL
                         }
 
                         @Override
-                        public void onFailure(Throwable e) {
+                        public void onFailure(Exception e) {
                             actionListener.onFailure(e);
                         }
                     };
@@ -228,15 +228,15 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     actionListener.onFailure(e);
                 }
             };
 
             // 1. send pre-sync flushes to all replicas
             sendPreSyncRequests(activeShards, state, shardId, commitIdsListener);
-        } catch (Throwable t) {
-            actionListener.onFailure(t);
+        } catch (Exception e) {
+            actionListener.onFailure(e);
         }
     }
 
@@ -292,8 +292,8 @@ public class SyncedFlushService extends AbstractComponent implements IndexEventL
                             return ThreadPool.Names.SAME;
                         }
                     });
-        } catch (Throwable t) {
-            listener.onFailure(t);
+        } catch (Exception e) {
+            listener.onFailure(e);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java
index 4a2b93de084..19d7a4edf93 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveriesCollection.java
@@ -32,7 +32,6 @@ import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.Predicate;
 
 /**
  * This class holds a collection of all on going recoveries on the current node (i.e., the node is the target node
@@ -198,8 +197,8 @@ public class RecoveriesCollection {
         }
 
         @Override
-        public void onFailure(Throwable t) {
-            logger.error("unexpected error while monitoring recovery [{}]", t, recoveryId);
+        public void onFailure(Exception e) {
+            logger.error("unexpected error while monitoring recovery [{}]", e, recoveryId);
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
index 78d9bcf53fd..db2f97787d1 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
@@ -82,7 +82,7 @@ public class RecoverySourceHandler {
 
     private final CancellableThreads cancellableThreads = new CancellableThreads() {
         @Override
-        protected void onCancel(String reason, @Nullable Throwable suppressedException) {
+        protected void onCancel(String reason, @Nullable Exception suppressedException) {
             RuntimeException e;
             if (shard.state() == IndexShardState.CLOSED) { // check if the shard got closed on us
                 e = new IndexShardClosedException(shard.shardId(), "shard is closed and recovery was canceled reason [" + reason + "]");
@@ -119,14 +119,14 @@ public class RecoverySourceHandler {
             final IndexCommit phase1Snapshot;
             try {
                 phase1Snapshot = shard.snapshotIndex(false);
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 IOUtils.closeWhileHandlingException(translogView);
                 throw new RecoveryEngineException(shard.shardId(), 1, "Snapshot failed", e);
             }
 
             try {
                 phase1(phase1Snapshot, translogView);
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 throw new RecoveryEngineException(shard.shardId(), 1, "phase1 failed", e);
             } finally {
                 try {
@@ -139,7 +139,7 @@ public class RecoverySourceHandler {
             logger.trace("{} snapshot translog for recovery. current size is [{}]", shard.shardId(), translogView.totalOperations());
             try {
                 phase2(translogView.snapshot());
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 throw new RecoveryEngineException(shard.shardId(), 2, "phase2 failed", e);
             }
 
@@ -301,7 +301,7 @@ public class RecoverySourceHandler {
 
             logger.trace("[{}][{}] recovery [phase1] to {}: took [{}]", indexName, shardId, request.targetNode(), stopWatch.totalTime());
             response.phase1Time = stopWatch.totalTime().millis();
-        } catch (Throwable e) {
+        } catch (Exception e) {
             throw new RecoverFilesRecoveryException(request.shardId(), response.phase1FileNames.size(), new ByteSizeValue(totalSize), e);
         } finally {
             store.decRef();
@@ -365,7 +365,7 @@ public class RecoverySourceHandler {
             logger.trace("[{}][{}] performing relocation hand-off to {}", indexName, shardId, request.targetNode());
             try {
                 cancellableThreads.execute(() -> shard.relocated("to " + request.targetNode()));
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 logger.debug("[{}][{}] completing relocation hand-off to {} failed", e, indexName, shardId, request.targetNode());
                 throw e;
             }
@@ -508,7 +508,7 @@ public class RecoverySourceHandler {
         }
     }
 
-    void sendFiles(Store store, StoreFileMetaData[] files, Function<StoreFileMetaData, OutputStream> outputStreamFactory) throws Throwable {
+    void sendFiles(Store store, StoreFileMetaData[] files, Function<StoreFileMetaData, OutputStream> outputStreamFactory) throws Exception {
         store.incRef();
         try {
             ArrayUtil.timSort(files, (a, b) -> Long.compare(a.length(), b.length())); // send smallest first
@@ -518,9 +518,9 @@ public class RecoverySourceHandler {
                     // it's fine that we are only having the indexInput in the try/with block. The copy methods handles
                     // exceptions during close correctly and doesn't hide the original exception.
                     Streams.copy(new InputStreamIndexInput(indexInput, md.length()), outputStreamFactory.apply(md));
-                } catch (Throwable t) {
+                } catch (Exception e) {
                     final IOException corruptIndexException;
-                    if ((corruptIndexException = ExceptionsHelper.unwrapCorruption(t)) != null) {
+                    if ((corruptIndexException = ExceptionsHelper.unwrapCorruption(e)) != null) {
                         if (store.checkIntegrityNoException(md) == false) { // we are corrupted on the primary -- fail!
                             logger.warn("{} Corrupted file detected {} checksum mismatch", shardId, md);
                             failEngine(corruptIndexException);
@@ -528,13 +528,13 @@ public class RecoverySourceHandler {
                         } else { // corruption has happened on the way to replica
                             RemoteTransportException exception = new RemoteTransportException("File corruption occurred on recovery but " +
                                     "checksums are ok", null);
-                            exception.addSuppressed(t);
+                            exception.addSuppressed(e);
                             logger.warn("{} Remote file corruption on node {}, recovering {}. local checksum OK",
                                     corruptIndexException, shardId, request.targetNode(), md);
                             throw exception;
                         }
                     } else {
-                        throw t;
+                        throw e;
                     }
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
index a006c8de921..4a091f53ed1 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java
@@ -267,8 +267,8 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget
             logger.trace("closing IndexOutput file [{}]", entry.getValue());
             try {
                 entry.getValue().close();
-            } catch (Throwable t) {
-                logger.debug("error while closing recovery output [{}]", t, entry.getValue());
+            } catch (Exception e) {
+                logger.debug("error while closing recovery output [{}]", e, entry.getValue());
             }
             iterator.remove();
         }
@@ -353,7 +353,7 @@ public class RecoveryTarget extends AbstractRefCounted implements RecoveryTarget
                 } finally {
                     Lucene.cleanLuceneIndex(store.directory()); // clean up and delete all files
                 }
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 logger.debug("Failed to clean lucene index", e);
                 ex.addSuppressed(e);
             }
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetService.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetService.java
index ec4e74ab8fb..d54d7859089 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetService.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTargetService.java
@@ -58,7 +58,6 @@ import org.elasticsearch.transport.TransportService;
 import java.io.IOException;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Predicate;
 
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 
@@ -152,7 +151,7 @@ public class RecoveryTargetService extends AbstractComponent implements IndexEve
     private void retryRecovery(final RecoveryTarget recoveryTarget, TimeValue retryAfter, final StartRecoveryRequest currentRequest) {
         try {
             recoveryTarget.resetRecovery();
-        } catch (Throwable e) {
+        } catch (Exception e) {
             onGoingRecoveries.failRecovery(recoveryTarget.recoveryId(), new RecoveryFailedException(currentRequest, e), true);
         }
         threadPool.schedule(retryAfter, ThreadPool.Names.GENERIC, new RecoveryRunner(recoveryTarget.recoveryId()));
@@ -220,7 +219,7 @@ public class RecoveryTargetService extends AbstractComponent implements IndexEve
             }
         } catch (CancellableThreads.ExecutionCancelledException e) {
             logger.trace("recovery cancelled", e);
-        } catch (Throwable e) {
+        } catch (Exception e) {
             if (logger.isTraceEnabled()) {
                 logger.trace("[{}][{}] Got exception on recovery", e, request.shardId().getIndex().getName(), request.shardId().id());
             }
@@ -432,16 +431,16 @@ public class RecoveryTargetService extends AbstractComponent implements IndexEve
         }
 
         @Override
-        public void onFailure(Throwable t) {
+        public void onFailure(Exception e) {
             try (RecoveriesCollection.RecoveryRef recoveryRef = onGoingRecoveries.getRecovery(recoveryId)) {
                 if (recoveryRef != null) {
-                    logger.error("unexpected error during recovery [{}], failing shard", t, recoveryId);
+                    logger.error("unexpected error during recovery [{}], failing shard", e, recoveryId);
                     onGoingRecoveries.failRecovery(recoveryId,
-                            new RecoveryFailedException(recoveryRef.status().state(), "unexpected error", t),
+                            new RecoveryFailedException(recoveryRef.status().state(), "unexpected error", e),
                             true // be safe
                     );
                 } else {
-                    logger.debug("unexpected error during recovery, but recovery id [{}] is finished", t, recoveryId);
+                    logger.debug("unexpected error during recovery, but recovery id [{}] is finished", e, recoveryId);
                 }
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java b/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java
index 868b5dba9b9..6794475e0a3 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/SharedFSRecoverySourceHandler.java
@@ -61,7 +61,7 @@ public class SharedFSRecoverySourceHandler extends RecoverySourceHandler {
             prepareTargetForTranslog(0);
             finalizeRecovery();
             return response;
-        } catch (Throwable t) {
+        } catch (Exception e) {
             if (engineClosed) {
                 // If the relocation fails then the primary is closed and can't be
                 // used anymore... (because it's closed) that's a problem, so in
@@ -69,11 +69,11 @@ public class SharedFSRecoverySourceHandler extends RecoverySourceHandler {
                 // create a new IndexWriter
                 logger.info("recovery failed for primary shadow shard, failing shard");
                 // pass the failure as null, as we want to ensure the store is not marked as corrupted
-                shard.failShard("primary relocation failed on shared filesystem", t);
+                shard.failShard("primary relocation failed on shared filesystem", e);
             } else {
-                logger.info("recovery failed on shared filesystem", t);
+                logger.info("recovery failed on shared filesystem", e);
             }
-            throw t;
+            throw e;
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java b/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
index 8766d23d3ee..bc7e7f59fc0 100644
--- a/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
+++ b/core/src/main/java/org/elasticsearch/indices/store/IndicesStore.java
@@ -265,15 +265,15 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe
                     }
                     try {
                         indicesService.deleteShardStore("no longer used", shardId, currentState);
-                    } catch (Throwable ex) {
+                    } catch (Exception ex) {
                         logger.debug("{} failed to delete unallocated shard, ignoring", ex, shardId);
                     }
                     return currentState;
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
-                    logger.error("{} unexpected error during deletion of unallocated shard", t, shardId);
+                public void onFailure(String source, Exception e) {
+                    logger.error("{} unexpected error during deletion of unallocated shard", e, shardId);
                 }
             });
         }
diff --git a/core/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java b/core/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java
index b837839ce86..a752c226429 100644
--- a/core/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java
+++ b/core/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java
@@ -140,7 +140,7 @@ public class IndicesTTLService extends AbstractLifecycleComponent {
                     try {
                         List<IndexShard> shardsToPurge = getShardsToPurge();
                         purgeShards(shardsToPurge);
-                    } catch (Throwable e) {
+                    } catch (Exception e) {
                         if (running.get()) {
                             logger.warn("failed to execute ttl purge", e);
                         }
@@ -295,7 +295,7 @@ public class IndicesTTLService extends AbstractLifecycleComponent {
                     }
 
                     @Override
-                    public void onFailure(Throwable e) {
+                    public void onFailure(Exception e) {
                         if (logger.isTraceEnabled()) {
                             logger.trace("failed to execute bulk", e);
                         } else {
diff --git a/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java b/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
index 54fbc4a0237..86c8644134c 100644
--- a/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
+++ b/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
@@ -120,7 +120,7 @@ public final class ConfigurationUtils {
         }
         try {
             return Integer.parseInt(value.toString());
-        } catch (Throwable t) {
+        } catch (Exception e) {
             throw newConfigurationException(processorType, processorTag, propertyName,
                 "property cannot be converted to an int [" + value.toString() + "]");
         }
diff --git a/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java b/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java
index e47c4db2340..e7146636534 100644
--- a/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java
+++ b/core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java
@@ -51,13 +51,13 @@ public class PipelineExecutionService implements ClusterStateListener {
         this.threadPool = threadPool;
     }
 
-    public void executeIndexRequest(IndexRequest request, Consumer<Throwable> failureHandler, Consumer<Boolean> completionHandler) {
+    public void executeIndexRequest(IndexRequest request, Consumer<Exception> failureHandler, Consumer<Boolean> completionHandler) {
         Pipeline pipeline = getPipeline(request.getPipeline());
         threadPool.executor(ThreadPool.Names.INDEX).execute(new AbstractRunnable() {
 
             @Override
-            public void onFailure(Throwable t) {
-                failureHandler.accept(t);
+            public void onFailure(Exception e) {
+                failureHandler.accept(e);
             }
 
             @Override
@@ -69,13 +69,13 @@ public class PipelineExecutionService implements ClusterStateListener {
     }
 
     public void executeBulkRequest(Iterable<ActionRequest<?>> actionRequests,
-                                   BiConsumer<IndexRequest, Throwable> itemFailureHandler,
-                                   Consumer<Throwable> completionHandler) {
+                                   BiConsumer<IndexRequest, Exception> itemFailureHandler,
+                                   Consumer<Exception> completionHandler) {
         threadPool.executor(ThreadPool.Names.BULK).execute(new AbstractRunnable() {
 
             @Override
-            public void onFailure(Throwable t) {
-                completionHandler.accept(t);
+            public void onFailure(Exception e) {
+                completionHandler.accept(e);
             }
 
             @Override
@@ -89,7 +89,7 @@ public class PipelineExecutionService implements ClusterStateListener {
                                 //this shouldn't be needed here but we do it for consistency with index api
                                 // which requires it to prevent double execution
                                 indexRequest.setPipeline(null);
-                            } catch (Throwable e) {
+                            } catch (Exception e) {
                                 itemFailureHandler.accept(indexRequest, e);
                             }
                         }
diff --git a/core/src/main/java/org/elasticsearch/monitor/Probes.java b/core/src/main/java/org/elasticsearch/monitor/Probes.java
index e31903c0bce..fbdd022736b 100644
--- a/core/src/main/java/org/elasticsearch/monitor/Probes.java
+++ b/core/src/main/java/org/elasticsearch/monitor/Probes.java
@@ -30,7 +30,7 @@ public class Probes {
                 if (load >= 0) {
                     return (short) (load * 100);
                 }
-            } catch (Throwable t) {
+            } catch (Exception e) {
                 return -1;
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
index 4c89c6a2a50..1f769e1dc96 100644
--- a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
+++ b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmGcMonitorService.java
@@ -185,8 +185,8 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent {
         }
         scheduledFuture = threadPool.scheduleWithFixedDelay(new JvmMonitor(gcThresholds, gcOverheadThreshold) {
             @Override
-            void onMonitorFailure(Throwable t) {
-                logger.debug("failed to monitor", t);
+            void onMonitorFailure(Exception e) {
+                logger.debug("failed to monitor", e);
             }
 
             @Override
@@ -389,12 +389,12 @@ public class JvmGcMonitorService extends AbstractLifecycleComponent {
         public void run() {
             try {
                 monitorGc();
-            } catch (Throwable t) {
-                onMonitorFailure(t);
+            } catch (Exception e) {
+                onMonitorFailure(e);
             }
         }
 
-        abstract void onMonitorFailure(Throwable t);
+        abstract void onMonitorFailure(Exception e);
 
         synchronized void monitorGc() {
             seq++;
diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java
index 69abd0752f6..1619ecee23a 100644
--- a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java
+++ b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmInfo.java
@@ -76,7 +76,7 @@ public class JvmInfo implements Streamable, ToXContent {
         try {
             Class<?> vmClass = Class.forName("sun.misc.VM");
             info.mem.directMemoryMax = (Long) vmClass.getMethod("maxDirectMemory").invoke(null);
-        } catch (Throwable t) {
+        } catch (Exception t) {
             // ignore
         }
         info.inputArguments = runtimeMXBean.getInputArguments().toArray(new String[runtimeMXBean.getInputArguments().size()]);
diff --git a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java
index 7122f1ea382..353b4178259 100644
--- a/core/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java
+++ b/core/src/main/java/org/elasticsearch/monitor/jvm/JvmStats.java
@@ -88,9 +88,7 @@ public class JvmStats implements Streamable, ToXContent {
                         peakUsage.getUsed() < 0 ? 0 : peakUsage.getUsed(),
                         peakUsage.getMax() < 0 ? 0 : peakUsage.getMax()
                 ));
-            } catch (OutOfMemoryError err) {
-                throw err; // rethrow
-            } catch (Throwable ex) {
+            } catch (Exception ex) {
                 /* ignore some JVMs might barf here with:
                  * java.lang.InternalError: Memory Pool not found
                  * we just omit the pool in that case!*/
@@ -119,7 +117,7 @@ public class JvmStats implements Streamable, ToXContent {
             for (BufferPoolMXBean bufferPool : bufferPools) {
                 stats.bufferPools.add(new BufferPool(bufferPool.getName(), bufferPool.getCount(), bufferPool.getTotalCapacity(), bufferPool.getMemoryUsed()));
             }
-        } catch (Throwable t) {
+        } catch (Exception e) {
             // buffer pools are not available
         }
 
diff --git a/core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java b/core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
index f41afb90e05..017f961a315 100644
--- a/core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
+++ b/core/src/main/java/org/elasticsearch/monitor/os/OsProbe.java
@@ -60,7 +60,7 @@ public class OsProbe {
         }
         try {
             return (long) getFreePhysicalMemorySize.invoke(osMxBean);
-        } catch (Throwable t) {
+        } catch (Exception e) {
             return -1;
         }
     }
@@ -74,7 +74,7 @@ public class OsProbe {
         }
         try {
             return (long) getTotalPhysicalMemorySize.invoke(osMxBean);
-        } catch (Throwable t) {
+        } catch (Exception e) {
             return -1;
         }
     }
@@ -88,7 +88,7 @@ public class OsProbe {
         }
         try {
             return (long) getFreeSwapSpaceSize.invoke(osMxBean);
-        } catch (Throwable t) {
+        } catch (Exception e) {
             return -1;
         }
     }
@@ -102,7 +102,7 @@ public class OsProbe {
         }
         try {
             return (long) getTotalSwapSpaceSize.invoke(osMxBean);
-        } catch (Throwable t) {
+        } catch (Exception e) {
             return -1;
         }
     }
@@ -128,7 +128,7 @@ public class OsProbe {
         try {
             double oneMinuteLoadAverage = (double) getSystemLoadAverage.invoke(osMxBean);
             return new double[] { oneMinuteLoadAverage >= 0 ? oneMinuteLoadAverage : -1, -1, -1 };
-        } catch (Throwable t) {
+        } catch (Exception e) {
             return null;
         }
     }
@@ -199,7 +199,7 @@ public class OsProbe {
     private static Method getMethod(String methodName) {
         try {
             return Class.forName("com.sun.management.OperatingSystemMXBean").getMethod(methodName);
-        } catch (Throwable t) {
+        } catch (Exception e) {
             // not available
             return null;
         }
diff --git a/core/src/main/java/org/elasticsearch/monitor/process/ProcessProbe.java b/core/src/main/java/org/elasticsearch/monitor/process/ProcessProbe.java
index 019a00bd4c8..b19b54a9478 100644
--- a/core/src/main/java/org/elasticsearch/monitor/process/ProcessProbe.java
+++ b/core/src/main/java/org/elasticsearch/monitor/process/ProcessProbe.java
@@ -66,7 +66,7 @@ public class ProcessProbe {
         }
         try {
             return (Long) getMaxFileDescriptorCountField.invoke(osMxBean);
-        } catch (Throwable t) {
+        } catch (Exception t) {
             return -1;
         }
     }
@@ -80,7 +80,7 @@ public class ProcessProbe {
         }
         try {
             return (Long) getOpenFileDescriptorCountField.invoke(osMxBean);
-        } catch (Throwable t) {
+        } catch (Exception t) {
             return -1;
         }
     }
@@ -102,7 +102,7 @@ public class ProcessProbe {
                 if (time >= 0) {
                     return (time / 1_000_000L);
                 }
-            } catch (Throwable t) {
+            } catch (Exception t) {
                 return -1;
             }
         }
@@ -119,7 +119,7 @@ public class ProcessProbe {
                 if (virtual >= 0) {
                     return virtual;
                 }
-            } catch (Throwable t) {
+            } catch (Exception t) {
                 return -1;
             }
         }
@@ -155,12 +155,12 @@ public class ProcessProbe {
     private static Method getMethod(String methodName) {
         try {
             return Class.forName("com.sun.management.OperatingSystemMXBean").getMethod(methodName);
-        } catch (Throwable t) {
+        } catch (Exception t) {
             // not available
             return null;
         }
     }
-    
+
     /**
      * Returns a given method of the UnixOperatingSystemMXBean,
      * or null if the method is not found or unavailable.
@@ -168,7 +168,7 @@ public class ProcessProbe {
     private static Method getUnixMethod(String methodName) {
         try {
             return Class.forName("com.sun.management.UnixOperatingSystemMXBean").getMethod(methodName);
-        } catch (Throwable t) {
+        } catch (Exception t) {
             // not available
             return null;
         }
diff --git a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
index 4769b5954ba..077f65d3da8 100644
--- a/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
+++ b/core/src/main/java/org/elasticsearch/plugins/PluginsService.java
@@ -457,7 +457,7 @@ public class PluginsService extends AbstractComponent {
                         "Settings instance");
                 }
             }
-        } catch (Throwable e) {
+        } catch (Exception e) {
             throw new ElasticsearchException("Failed to load plugin class [" + pluginClass.getName() + "]", e);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
index 383ded2b2de..0ed53c0a9c5 100644
--- a/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
+++ b/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
@@ -147,9 +147,9 @@ public class RepositoriesService extends AbstractComponent implements ClusterSta
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
-                logger.warn("failed to create repository [{}]", t, request.name);
-                super.onFailure(source, t);
+            public void onFailure(String source, Exception e) {
+                logger.warn("failed to create repository [{}]", e, request.name);
+                super.onFailure(source, e);
             }
 
             @Override
@@ -221,32 +221,33 @@ public class RepositoriesService extends AbstractComponent implements ClusterSta
                         public void onResponse(VerifyResponse verifyResponse) {
                             try {
                                 repository.endVerification(verificationToken);
-                            } catch (Throwable t) {
-                                logger.warn("[{}] failed to finish repository verification", t, repositoryName);
-                                listener.onFailure(t);
+                            } catch (Exception e) {
+                                logger.warn("[{}] failed to finish repository verification", e, repositoryName);
+                                listener.onFailure(e);
                                 return;
                             }
                             listener.onResponse(verifyResponse);
                         }
 
                         @Override
-                        public void onFailure(Throwable e) {
+                        public void onFailure(Exception e) {
                             listener.onFailure(e);
                         }
                     });
-                } catch (Throwable t) {
+                } catch (Exception e) {
                     try {
                         repository.endVerification(verificationToken);
-                    } catch (Throwable t1) {
-                        logger.warn("[{}] failed to finish repository verification", t1, repositoryName);
+                    } catch (Exception inner) {
+                        inner.addSuppressed(e);
+                        logger.warn("[{}] failed to finish repository verification", inner, repositoryName);
                     }
-                    listener.onFailure(t);
+                    listener.onFailure(e);
                 }
             } else {
                 listener.onResponse(new VerifyResponse(new DiscoveryNode[0], new VerificationFailure[0]));
             }
-        } catch (Throwable t) {
-            listener.onFailure(t);
+        } catch (Exception e) {
+            listener.onFailure(e);
         }
     }
 
@@ -313,7 +314,7 @@ public class RepositoriesService extends AbstractComponent implements ClusterSta
                 }
             }
             repositories = unmodifiableMap(builder);
-        } catch (Throwable ex) {
+        } catch (Exception ex) {
             logger.warn("failure updating cluster state ", ex);
         }
     }
@@ -410,9 +411,9 @@ public class RepositoriesService extends AbstractComponent implements ClusterSta
             IndexShardRepository indexShardRepository = repositoryInjector.getInstance(IndexShardRepository.class);
             repository.start();
             return new RepositoryHolder(repositoryMetaData.type(), repositoryMetaData.settings(), repository, indexShardRepository);
-        } catch (Throwable t) {
-            logger.warn("failed to create repository [{}][{}]", t, repositoryMetaData.type(), repositoryMetaData.name());
-            throw new RepositoryException(repositoryMetaData.name(), "failed to create repository", t);
+        } catch (Exception e) {
+            logger.warn("failed to create repository [{}][{}]", e, repositoryMetaData.type(), repositoryMetaData.name());
+            throw new RepositoryException(repositoryMetaData.name(), "failed to create repository", e);
         }
     }
 
@@ -448,7 +449,7 @@ public class RepositoriesService extends AbstractComponent implements ClusterSta
                     }
 
                     @Override
-                    public void onFailure(Throwable e) {
+                    public void onFailure(Exception e) {
                         listener.onFailure(e);
                     }
                 });
@@ -458,7 +459,7 @@ public class RepositoriesService extends AbstractComponent implements ClusterSta
         }
 
         @Override
-        public void onFailure(Throwable e) {
+        public void onFailure(Exception e) {
             listener.onFailure(e);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/repositories/VerificationFailure.java b/core/src/main/java/org/elasticsearch/repositories/VerificationFailure.java
index c5c974715c4..5672506b7fe 100644
--- a/core/src/main/java/org/elasticsearch/repositories/VerificationFailure.java
+++ b/core/src/main/java/org/elasticsearch/repositories/VerificationFailure.java
@@ -31,13 +31,13 @@ public class VerificationFailure implements Streamable {
 
     private String nodeId;
 
-    private Throwable cause;
+    private Exception cause;
 
     VerificationFailure() {
 
     }
 
-    public VerificationFailure(String nodeId, Throwable cause) {
+    public VerificationFailure(String nodeId, Exception cause) {
         this.nodeId = nodeId;
         this.cause = cause;
     }
@@ -53,7 +53,7 @@ public class VerificationFailure implements Streamable {
     @Override
     public void readFrom(StreamInput in) throws IOException {
         nodeId = in.readOptionalString();
-        cause = in.readThrowable();
+        cause = in.readException();
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java b/core/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java
index 93bcca1c375..796c65e9b46 100644
--- a/core/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java
+++ b/core/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java
@@ -83,9 +83,9 @@ public class VerifyNodeRepositoryAction  extends AbstractComponent {
             if (node.equals(localNode)) {
                 try {
                     doVerify(repository, verificationToken);
-                } catch (Throwable t) {
-                    logger.warn("[{}] failed to verify repository", t, repository);
-                    errors.add(new VerificationFailure(node.getId(), t));
+                } catch (Exception e) {
+                    logger.warn("[{}] failed to verify repository", e, repository);
+                    errors.add(new VerificationFailure(node.getId(), e));
                 }
                 if (counter.decrementAndGet() == 0) {
                     finishVerification(listener, nodes, errors);
diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index d0d6503a6f9..b1f1ff66cf5 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -223,7 +223,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
     protected void doClose() {
         try {
             blobStore().close();
-        } catch (Throwable t) {
+        } catch (Exception t) {
             logger.warn("cannot close blob store", t);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java b/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java
index 8cdbca512e7..fcf79962b6b 100644
--- a/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java
+++ b/core/src/main/java/org/elasticsearch/rest/BytesRestResponse.java
@@ -77,22 +77,22 @@ public class BytesRestResponse extends RestResponse {
         this.contentType = contentType;
     }
 
-    public BytesRestResponse(RestChannel channel, Throwable t) throws IOException {
-        this(channel, ExceptionsHelper.status(t), t);
+    public BytesRestResponse(RestChannel channel, Exception e) throws IOException {
+        this(channel, ExceptionsHelper.status(e), e);
     }
 
-    public BytesRestResponse(RestChannel channel, RestStatus status, Throwable t) throws IOException {
+    public BytesRestResponse(RestChannel channel, RestStatus status, Exception e) throws IOException {
         this.status = status;
         if (channel.request().method() == RestRequest.Method.HEAD) {
             this.content = BytesArray.EMPTY;
             this.contentType = TEXT_CONTENT_TYPE;
         } else {
-            XContentBuilder builder = convert(channel, status, t);
+            XContentBuilder builder = convert(channel, status, e);
             this.content = builder.bytes();
             this.contentType = builder.contentType().mediaType();
         }
-        if (t instanceof ElasticsearchException) {
-            copyHeaders(((ElasticsearchException) t));
+        if (e instanceof ElasticsearchException) {
+            copyHeaders(((ElasticsearchException) e));
         }
     }
 
@@ -113,9 +113,9 @@ public class BytesRestResponse extends RestResponse {
 
     private static final ESLogger SUPPRESSED_ERROR_LOGGER = ESLoggerFactory.getLogger("rest.suppressed");
 
-    private static XContentBuilder convert(RestChannel channel, RestStatus status, Throwable t) throws IOException {
+    private static XContentBuilder convert(RestChannel channel, RestStatus status, Exception e) throws IOException {
         XContentBuilder builder = channel.newErrorBuilder().startObject();
-        if (t == null) {
+        if (e == null) {
             builder.field("error", "unknown");
         } else if (channel.detailedErrorsEnabled()) {
             final ToXContent.Params params;
@@ -123,15 +123,15 @@ public class BytesRestResponse extends RestResponse {
                 params =  new ToXContent.DelegatingMapParams(Collections.singletonMap(ElasticsearchException.REST_EXCEPTION_SKIP_STACK_TRACE, "false"), channel.request());
             } else {
                 if (status.getStatus() < 500) {
-                    SUPPRESSED_ERROR_LOGGER.debug("path: {}, params: {}", t, channel.request().rawPath(), channel.request().params());
+                    SUPPRESSED_ERROR_LOGGER.debug("path: {}, params: {}", e, channel.request().rawPath(), channel.request().params());
                 } else {
-                    SUPPRESSED_ERROR_LOGGER.warn("path: {}, params: {}", t, channel.request().rawPath(), channel.request().params());
+                    SUPPRESSED_ERROR_LOGGER.warn("path: {}, params: {}", e, channel.request().rawPath(), channel.request().params());
                 }
                 params = channel.request();
             }
             builder.field("error");
             builder.startObject();
-            final ElasticsearchException[] rootCauses = ElasticsearchException.guessRootCauses(t);
+            final ElasticsearchException[] rootCauses = ElasticsearchException.guessRootCauses(e);
             builder.field("root_cause");
             builder.startArray();
             for (ElasticsearchException rootCause : rootCauses){
@@ -141,10 +141,10 @@ public class BytesRestResponse extends RestResponse {
             }
             builder.endArray();
 
-            ElasticsearchException.toXContent(builder, params, t);
+            ElasticsearchException.toXContent(builder, params, e);
             builder.endObject();
         } else {
-            builder.field("error", simpleMessage(t));
+            builder.field("error", simpleMessage(e));
         }
         builder.field("status", status.getStatus());
         builder.endObject();
diff --git a/core/src/main/java/org/elasticsearch/rest/RestController.java b/core/src/main/java/org/elasticsearch/rest/RestController.java
index 29dce98ca49..03d2c267c58 100644
--- a/core/src/main/java/org/elasticsearch/rest/RestController.java
+++ b/core/src/main/java/org/elasticsearch/rest/RestController.java
@@ -171,11 +171,12 @@ public class RestController extends AbstractLifecycleComponent {
         }
     }
 
-    public void sendErrorResponse(RestRequest request, RestChannel channel, Throwable e) {
+    public void sendErrorResponse(RestRequest request, RestChannel channel, Exception e) {
         try {
             channel.sendResponse(new BytesRestResponse(channel, e));
-        } catch (Throwable e1) {
-            logger.error("failed to send failure response for uri [{}]", e1, request.uri());
+        } catch (Exception inner) {
+            inner.addSuppressed(e);
+            logger.error("failed to send failure response for uri [{}]", inner, request.uri());
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java
index 5d1c617e8e3..d2f1769041e 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/cluster/node/tasks/RestListTasksAction.java
@@ -86,7 +86,7 @@ public class RestListTasksAction extends BaseRestHandler {
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 channelListener.onFailure(e);
             }
         };
diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/head/RestAliasesExistAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/head/RestAliasesExistAction.java
index 7c539e1be48..54f9a649168 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/head/RestAliasesExistAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/alias/head/RestAliasesExistAction.java
@@ -70,18 +70,19 @@ public class RestAliasesExistAction extends BaseRestHandler {
                     } else {
                         channel.sendResponse(new BytesRestResponse(NOT_FOUND, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
                     }
-                } catch (Throwable e) {
+                } catch (Exception e) {
                     onFailure(e);
                 }
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 try {
                     channel.sendResponse(
                         new BytesRestResponse(ExceptionsHelper.status(e), BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY));
-                } catch (Exception e1) {
-                    logger.error("Failed to send failure response", e1);
+                } catch (Exception inner) {
+                    inner.addSuppressed(e);
+                    logger.error("Failed to send failure response", inner);
                 }
             }
         });
diff --git a/core/src/main/java/org/elasticsearch/rest/action/support/RestActionListener.java b/core/src/main/java/org/elasticsearch/rest/action/support/RestActionListener.java
index 9ebdc7b5d07..15b0736a790 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/support/RestActionListener.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/support/RestActionListener.java
@@ -45,19 +45,20 @@ public abstract class RestActionListener<Response> implements ActionListener<Res
     public final void onResponse(Response response) {
         try {
             processResponse(response);
-        } catch (Throwable t) {
-            onFailure(t);
+        } catch (Exception e) {
+            onFailure(e);
         }
     }
 
     protected abstract void processResponse(Response response) throws Exception;
 
     @Override
-    public final void onFailure(Throwable e) {
+    public final void onFailure(Exception e) {
         try {
             channel.sendResponse(new BytesRestResponse(channel, e));
-        } catch (Throwable e1) {
-            logger.error("failed to send failure response", e1);
+        } catch (Exception inner) {
+            inner.addSuppressed(e);
+            logger.error("failed to send failure response", inner);
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptService.java b/core/src/main/java/org/elasticsearch/script/ScriptService.java
index e42984aaf9a..597a13bd1e6 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptService.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptService.java
@@ -571,7 +571,7 @@ public class ScriptService extends AbstractComponent implements Closeable, Clust
                     } else {
                         logger.warn("skipping compile of script file [{}] as all scripted operations are disabled for file scripts", file.toAbsolutePath());
                     }
-                } catch (Throwable e) {
+                } catch (Exception e) {
                     logger.warn("failed to load/compile script [{}]", e, scriptNameExt.v1());
                 }
             }
diff --git a/core/src/main/java/org/elasticsearch/search/MultiValueMode.java b/core/src/main/java/org/elasticsearch/search/MultiValueMode.java
index 483426fce16..90e3417ad1f 100644
--- a/core/src/main/java/org/elasticsearch/search/MultiValueMode.java
+++ b/core/src/main/java/org/elasticsearch/search/MultiValueMode.java
@@ -519,7 +519,7 @@ public enum MultiValueMode implements Writeable {
     public static MultiValueMode fromString(String sortMode) {
         try {
             return valueOf(sortMode.toUpperCase(Locale.ROOT));
-        } catch (Throwable t) {
+        } catch (Exception e) {
             throw new IllegalArgumentException("Illegal sort mode: " + sortMode);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/search/SearchService.java b/core/src/main/java/org/elasticsearch/search/SearchService.java
index d8069575c5f..5aa761a3479 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchService.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchService.java
@@ -22,6 +22,7 @@ package org.elasticsearch.search;
 import com.carrotsearch.hppc.ObjectFloatHashMap;
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.TopDocs;
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -233,7 +234,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
             dfsPhase.execute(context);
             contextProcessedSuccessfully(context);
             return context.dfsResult();
-        } catch (Throwable e) {
+        } catch (Exception e) {
             logger.trace("Dfs phase failed", e);
             processFailure(context, e);
             throw ExceptionsHelper.convertToRuntime(e);
@@ -272,10 +273,11 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
             operationListener.onQueryPhase(context, System.nanoTime() - time);
 
             return context.queryResult();
-        } catch (Throwable e) {
+        } catch (Exception e) {
             // execution exception can happen while loading the cache, strip it
             if (e instanceof ExecutionException) {
-                e = e.getCause();
+                e = (e.getCause() == null || e.getCause() instanceof Exception) ?
+                        (Exception) e.getCause() : new ElasticsearchException(e.getCause());
             }
             operationListener.onFailedQueryPhase(context);
             logger.trace("Query phase failed", e);
@@ -298,7 +300,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
             contextProcessedSuccessfully(context);
             operationListener.onQueryPhase(context, System.nanoTime() - time);
             return new ScrollQuerySearchResult(context.queryResult(), context.shardTarget());
-        } catch (Throwable e) {
+        } catch (Exception e) {
             operationListener.onFailedQueryPhase(context);
             logger.trace("Query phase failed", e);
             processFailure(context, e);
@@ -326,7 +328,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
             }
             operationListener.onQueryPhase(context, System.nanoTime() - time);
             return context.queryResult();
-        } catch (Throwable e) {
+        } catch (Exception e) {
             operationListener.onFailedQueryPhase(context);
             logger.trace("Query phase failed", e);
             processFailure(context, e);
@@ -355,7 +357,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
             long time = System.nanoTime();
             try {
                 loadOrExecuteQueryPhase(request, context);
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 operationListener.onFailedQueryPhase(context);
                 throw ExceptionsHelper.convertToRuntime(e);
             }
@@ -370,13 +372,13 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
                 } else {
                     contextProcessedSuccessfully(context);
                 }
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 operationListener.onFailedFetchPhase(context);
                 throw ExceptionsHelper.convertToRuntime(e);
             }
             operationListener.onFetchPhase(context, System.nanoTime() - time2);
             return new QueryFetchSearchResult(context.queryResult(), context.fetchResult());
-        } catch (Throwable e) {
+        } catch (Exception e) {
             logger.trace("Fetch phase failed", e);
             processFailure(context, e);
             throw ExceptionsHelper.convertToRuntime(e);
@@ -395,7 +397,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
             long time = System.nanoTime();
             try {
                 queryPhase.execute(context);
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 operationListener.onFailedQueryPhase(context);
                 throw ExceptionsHelper.convertToRuntime(e);
             }
@@ -410,13 +412,13 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
                 } else {
                     contextProcessedSuccessfully(context);
                 }
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 operationListener.onFailedFetchPhase(context);
                 throw ExceptionsHelper.convertToRuntime(e);
             }
             operationListener.onFetchPhase(context, System.nanoTime() - time2);
             return new QueryFetchSearchResult(context.queryResult(), context.fetchResult());
-        } catch (Throwable e) {
+        } catch (Exception e) {
             logger.trace("Fetch phase failed", e);
             processFailure(context, e);
             throw ExceptionsHelper.convertToRuntime(e);
@@ -435,7 +437,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
             long time = System.nanoTime();
             try {
                 queryPhase.execute(context);
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 operationListener.onFailedQueryPhase(context);
                 throw ExceptionsHelper.convertToRuntime(e);
             }
@@ -450,13 +452,13 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
                 } else {
                     contextProcessedSuccessfully(context);
                 }
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 operationListener.onFailedFetchPhase(context);
                 throw ExceptionsHelper.convertToRuntime(e);
             }
             operationListener.onFetchPhase(context, System.nanoTime() - time2);
             return new ScrollQueryFetchSearchResult(new QueryFetchSearchResult(context.queryResult(), context.fetchResult()), context.shardTarget());
-        } catch (Throwable e) {
+        } catch (Exception e) {
             logger.trace("Fetch phase failed", e);
             processFailure(context, e);
             throw ExceptionsHelper.convertToRuntime(e);
@@ -484,7 +486,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
             }
             operationListener.onFetchPhase(context, System.nanoTime() - time);
             return context.fetchResult();
-        } catch (Throwable e) {
+        } catch (Exception e) {
             operationListener.onFailedFetchPhase(context);
             logger.trace("Fetch phase failed", e);
             processFailure(context, e);
@@ -566,7 +568,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
                 keepAlive = request.scroll().keepAlive().millis();
             }
             context.keepAlive(keepAlive);
-        } catch (Throwable e) {
+        } catch (Exception e) {
             context.close();
             throw ExceptionsHelper.convertToRuntime(e);
         }
@@ -623,14 +625,15 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
         SearchContext.removeCurrent();
     }
 
-    private void processFailure(SearchContext context, Throwable t) {
+    private void processFailure(SearchContext context, Exception e) {
         freeContext(context.id());
         try {
-            if (Lucene.isCorruptionException(t)) {
-                context.indexShard().failShard("search execution corruption failure", t);
+            if (Lucene.isCorruptionException(e)) {
+                context.indexShard().failShard("search execution corruption failure", e);
             }
-        } catch (Throwable e) {
-            logger.warn("failed to process shard failure to (potentially) send back shard failure on corruption", e);
+        } catch (Exception inner) {
+            inner.addSuppressed(e);
+            logger.warn("failed to process shard failure to (potentially) send back shard failure on corruption", inner);
         }
     }
 
@@ -771,7 +774,8 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
                 String sSource = "_na_";
                 try {
                     sSource = source.toString();
-                } catch (Throwable e1) {
+                } catch (Exception inner) {
+                    e.addSuppressed(inner);
                     // ignore
                 }
                 XContentLocation location = extParser != null ? extParser.getTokenLocation() : null;
diff --git a/core/src/main/java/org/elasticsearch/search/action/SearchTransportService.java b/core/src/main/java/org/elasticsearch/search/action/SearchTransportService.java
index b6ac92c42da..8552d21b5c3 100644
--- a/core/src/main/java/org/elasticsearch/search/action/SearchTransportService.java
+++ b/core/src/main/java/org/elasticsearch/search/action/SearchTransportService.java
@@ -115,7 +115,7 @@ public class SearchTransportService extends AbstractComponent {
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
 
             }
         }, SearchFreeContextResponse::new));
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java
index c4d76304d09..97e2362bdeb 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java
@@ -35,9 +35,6 @@ import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
 
-/**
- *
- */
 public class AggregationContext {
 
     private final SearchContext searchContext;
@@ -58,7 +55,10 @@ public class AggregationContext {
      *  no value source could be built. */
     @Nullable
     public <VS extends ValuesSource> VS valuesSource(ValuesSourceConfig<VS> config, SearchContext context) throws IOException {
-        assert config.valid() : "value source config is invalid - must have either a field context or a script or marked as unmapped";
+        if (!config.valid()) {
+            throw new IllegalStateException(
+                    "value source config is invalid; must have either a field context or a script or marked as unwrapped");
+        }
 
         final VS vs;
         if (config.unmapped()) {
diff --git a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
index 22e6c40d338..189fead7813 100644
--- a/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
+++ b/core/src/main/java/org/elasticsearch/search/query/QueryPhase.java
@@ -391,7 +391,7 @@ public class QueryPhase implements SearchPhase {
 
             return rescore;
 
-        } catch (Throwable e) {
+        } catch (Exception e) {
             throw new QueryPhaseExecutionException(searchContext, "Failed to execute main query", e);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
index 4c4d5ed95bf..53fb6f4781e 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
@@ -458,9 +458,9 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
 
 
                 @Override
-                public void onFailure(String source, Throwable t) {
-                    logger.warn("[{}] failed to restore snapshot", t, snapshotId);
-                    listener.onFailure(t);
+                public void onFailure(String source, Exception e) {
+                    logger.warn("[{}] failed to restore snapshot", e, snapshotId);
+                    listener.onFailure(e);
                 }
 
                 @Override
@@ -475,7 +475,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
             });
 
 
-        } catch (Throwable e) {
+        } catch (Exception e) {
             logger.warn("[{}] failed to restore snapshot", e, request.repositoryName + ":" + request.snapshotName);
             listener.onFailure(e);
         }
@@ -598,9 +598,9 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
             }
 
             @Override
-            public void onFailure(String source, @Nullable Throwable t) {
+            public void onFailure(String source, @Nullable Exception e) {
                 for (UpdateIndexShardRestoreStatusRequest request : drainedRequests) {
-                    logger.warn("[{}][{}] failed to update snapshot status to [{}]", t, request.snapshot(), request.shardId(), request.status());
+                    logger.warn("[{}][{}] failed to update snapshot status to [{}]", e, request.snapshot(), request.shardId(), request.status());
                 }
             }
 
@@ -667,7 +667,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
                 for (ActionListener<RestoreCompletionResponse> listener : listeners) {
                     try {
                         listener.onResponse(new RestoreCompletionResponse(snapshot, restoreInfo));
-                    } catch (Throwable e) {
+                    } catch (Exception e) {
                         logger.warn("failed to update snapshot status for [{}]", e, listener);
                     }
                 }
@@ -840,7 +840,7 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
             if (event.localNodeMaster()) {
                 processDeletedIndices(event);
             }
-        } catch (Throwable t) {
+        } catch (Exception t) {
             logger.warn("Failed to update restore state ", t);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
index 460eaf9f819..91db508b7e3 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
@@ -163,8 +163,8 @@ public class SnapshotShardsService extends AbstractLifecycleComponent implements
                 syncShardStatsOnNewMaster(event);
             }
 
-        } catch (Throwable t) {
-            logger.warn("Failed to update snapshot state ", t);
+        } catch (Exception e) {
+            logger.warn("Failed to update snapshot state ", e);
         }
     }
 
@@ -301,14 +301,14 @@ public class SnapshotShardsService extends AbstractLifecycleComponent implements
                             }
 
                             @Override
-                            public void onFailure(Throwable t) {
-                                logger.warn("[{}] [{}] failed to create snapshot", t, shardId, entry.getKey());
-                                updateIndexShardSnapshotStatus(entry.getKey(), shardId, new SnapshotsInProgress.ShardSnapshotStatus(localNodeId, SnapshotsInProgress.State.FAILED, ExceptionsHelper.detailedMessage(t)));
+                            public void onFailure(Exception e) {
+                                logger.warn("[{}] [{}] failed to create snapshot", e, shardId, entry.getKey());
+                                updateIndexShardSnapshotStatus(entry.getKey(), shardId, new SnapshotsInProgress.ShardSnapshotStatus(localNodeId, SnapshotsInProgress.State.FAILED, ExceptionsHelper.detailedMessage(e)));
                             }
 
                         });
-                    } catch (Throwable t) {
-                        updateIndexShardSnapshotStatus(entry.getKey(), shardId, new SnapshotsInProgress.ShardSnapshotStatus(localNodeId, SnapshotsInProgress.State.FAILED, ExceptionsHelper.detailedMessage(t)));
+                    } catch (Exception e) {
+                        updateIndexShardSnapshotStatus(entry.getKey(), shardId, new SnapshotsInProgress.ShardSnapshotStatus(localNodeId, SnapshotsInProgress.State.FAILED, ExceptionsHelper.detailedMessage(e)));
                     }
                 }
             }
@@ -354,7 +354,7 @@ public class SnapshotShardsService extends AbstractLifecycleComponent implements
             throw e;
         } catch (IndexShardSnapshotFailedException e) {
             throw e;
-        } catch (Throwable e) {
+        } catch (Exception e) {
             throw new IndexShardSnapshotFailedException(shardId, "Failed to snapshot", e);
         }
     }
@@ -483,8 +483,8 @@ public class SnapshotShardsService extends AbstractLifecycleComponent implements
                 transportService.sendRequest(clusterService.state().nodes().getMasterNode(),
                         UPDATE_SNAPSHOT_ACTION_NAME, request, EmptyTransportResponseHandler.INSTANCE_SAME);
             }
-        } catch (Throwable t) {
-            logger.warn("[{}] [{}] failed to update snapshot state", t, request.snapshot(), request.status());
+        } catch (Exception e) {
+            logger.warn("[{}] [{}] failed to update snapshot state", e, request.snapshot(), request.status());
         }
     }
 
@@ -566,9 +566,9 @@ public class SnapshotShardsService extends AbstractLifecycleComponent implements
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
+            public void onFailure(String source, Exception e) {
                 for (UpdateIndexShardSnapshotStatusRequest request : drainedRequests) {
-                    logger.warn("[{}][{}] failed to update snapshot status to [{}]", t, request.snapshot(), request.shardId(), request.status());
+                    logger.warn("[{}][{}] failed to update snapshot status to [{}]", e, request.snapshot(), request.shardId(), request.status());
                 }
             }
         });
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
index e242d1be732..5f5ac65e38e 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
@@ -95,7 +95,7 @@ import static org.elasticsearch.cluster.SnapshotsInProgress.completed;
  * <li>Once shard snapshot is created data node updates state of the shard in the cluster state using the {@link SnapshotShardsService#updateIndexShardSnapshotStatus} method</li>
  * <li>When last shard is completed master node in {@link SnapshotShardsService#innerUpdateSnapshotState} method marks the snapshot as completed</li>
  * <li>After cluster state is updated, the {@link #endSnapshot(SnapshotsInProgress.Entry)} finalizes snapshot in the repository,
- * notifies all {@link #snapshotCompletionListeners} that snapshot is completed, and finally calls {@link #removeSnapshotFromClusterState(Snapshot, SnapshotInfo, Throwable)} to remove snapshot from cluster state</li>
+ * notifies all {@link #snapshotCompletionListeners} that snapshot is completed, and finally calls {@link #removeSnapshotFromClusterState(Snapshot, SnapshotInfo, Exception)} to remove snapshot from cluster state</li>
  * </ul>
  */
 public class SnapshotsService extends AbstractLifecycleComponent implements ClusterStateListener {
@@ -249,10 +249,10 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
-                logger.warn("[{}][{}] failed to create snapshot", t, repositoryName, snapshotName);
+            public void onFailure(String source, Exception e) {
+                logger.warn("[{}][{}] failed to create snapshot", e, repositoryName, snapshotName);
                 newSnapshot = null;
-                listener.onFailure(t);
+                listener.onFailure(e);
             }
 
             @Override
@@ -400,9 +400,9 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
-                    logger.warn("[{}] failed to create snapshot", t, snapshot.snapshot().getSnapshotId());
-                    removeSnapshotFromClusterState(snapshot.snapshot(), null, t, new CleanupAfterErrorListener(snapshot, true, userCreateSnapshotListener, t));
+                public void onFailure(String source, Exception e) {
+                    logger.warn("[{}] failed to create snapshot", e, snapshot.snapshot().getSnapshotId());
+                    removeSnapshotFromClusterState(snapshot.snapshot(), null, e, new CleanupAfterErrorListener(snapshot, true, userCreateSnapshotListener, e));
                 }
 
                 @Override
@@ -422,9 +422,9 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                     }
                 }
             });
-        } catch (Throwable t) {
-            logger.warn("failed to create snapshot [{}]", t, snapshot.snapshot().getSnapshotId());
-            removeSnapshotFromClusterState(snapshot.snapshot(), null, t, new CleanupAfterErrorListener(snapshot, snapshotCreated, userCreateSnapshotListener, t));
+        } catch (Exception e) {
+            logger.warn("failed to create snapshot [{}]", e, snapshot.snapshot().getSnapshotId());
+            removeSnapshotFromClusterState(snapshot.snapshot(), null, e, new CleanupAfterErrorListener(snapshot, snapshotCreated, userCreateSnapshotListener, e));
         }
     }
 
@@ -433,40 +433,42 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
         private final SnapshotsInProgress.Entry snapshot;
         private final boolean snapshotCreated;
         private final CreateSnapshotListener userCreateSnapshotListener;
-        private final Throwable t;
+        private final Exception e;
 
-        public CleanupAfterErrorListener(SnapshotsInProgress.Entry snapshot, boolean snapshotCreated, CreateSnapshotListener userCreateSnapshotListener, Throwable t) {
+        public CleanupAfterErrorListener(SnapshotsInProgress.Entry snapshot, boolean snapshotCreated, CreateSnapshotListener userCreateSnapshotListener, Exception e) {
             this.snapshot = snapshot;
             this.snapshotCreated = snapshotCreated;
             this.userCreateSnapshotListener = userCreateSnapshotListener;
-            this.t = t;
+            this.e = e;
         }
 
         @Override
         public void onResponse(SnapshotInfo snapshotInfo) {
-            cleanupAfterError();
+            cleanupAfterError(this.e);
         }
 
         @Override
-        public void onFailure(Throwable e) {
-            cleanupAfterError();
+        public void onFailure(Exception e) {
+            e.addSuppressed(this.e);
+            cleanupAfterError(e);
         }
 
-        private void cleanupAfterError() {
+        private void cleanupAfterError(Exception exception) {
             if(snapshotCreated) {
                 try {
                     repositoriesService.repository(snapshot.snapshot().getRepository())
                                        .finalizeSnapshot(snapshot.snapshot().getSnapshotId(),
                                                          snapshot.indices(),
                                                          snapshot.startTime(),
-                                                         ExceptionsHelper.detailedMessage(t),
+                                                         ExceptionsHelper.detailedMessage(exception),
                                                          0,
                                                          Collections.emptyList());
-                } catch (Throwable t2) {
-                    logger.warn("[{}] failed to close snapshot in repository", snapshot.snapshot());
+                } catch (Exception inner) {
+                    inner.addSuppressed(exception);
+                    logger.warn("[{}] failed to close snapshot in repository", inner, snapshot.snapshot());
                 }
             }
-            userCreateSnapshotListener.onFailure(t);
+            userCreateSnapshotListener.onFailure(e);
         }
 
     }
@@ -591,8 +593,8 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                     processStartedShards(event);
                 }
             }
-        } catch (Throwable t) {
-            logger.warn("Failed to update snapshot state ", t);
+        } catch (Exception e) {
+            logger.warn("Failed to update snapshot state ", e);
         }
     }
 
@@ -653,7 +655,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                                 }
 
                                 @Override
-                                public void onFailure(Throwable t) {
+                                public void onFailure(Exception e) {
                                     logger.warn("failed to clean up abandoned snapshot {} in INIT state", snapshot.snapshot());
                                 }
                             });
@@ -670,7 +672,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
+                public void onFailure(String source, Exception e) {
                     logger.warn("failed to update snapshot state after node removal");
                 }
             });
@@ -712,8 +714,8 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
-                    logger.warn("failed to update snapshot state after shards started from [{}] ", t, source);
+                public void onFailure(String source, Exception e) {
+                    logger.warn("failed to update snapshot state after shards started from [{}] ", e, source);
                 }
             });
         }
@@ -866,9 +868,9 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                     }
                     SnapshotInfo snapshotInfo = repository.finalizeSnapshot(snapshot.getSnapshotId(), entry.indices(), entry.startTime(), failure, entry.shards().size(), Collections.unmodifiableList(shardFailures));
                     removeSnapshotFromClusterState(snapshot, snapshotInfo, null);
-                } catch (Throwable t) {
-                    logger.warn("[{}] failed to finalize snapshot", t, snapshot);
-                    removeSnapshotFromClusterState(snapshot, null, t);
+                } catch (Exception e) {
+                    logger.warn("[{}] failed to finalize snapshot", e, snapshot);
+                    removeSnapshotFromClusterState(snapshot, null, e);
                 }
             }
         });
@@ -876,24 +878,21 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
 
     /**
      * Removes record of running snapshot from cluster state
-     *
-     * @param snapshot       snapshot
+     *  @param snapshot       snapshot
      * @param snapshotInfo   snapshot info if snapshot was successful
-     * @param t              exception if snapshot failed
+     * @param e              exception if snapshot failed
      */
-    private void removeSnapshotFromClusterState(final Snapshot snapshot, final SnapshotInfo snapshotInfo, final Throwable t) {
-        removeSnapshotFromClusterState(snapshot, snapshotInfo, t, null);
+    private void removeSnapshotFromClusterState(final Snapshot snapshot, final SnapshotInfo snapshotInfo, final Exception e) {
+        removeSnapshotFromClusterState(snapshot, snapshotInfo, e, null);
     }
 
     /**
      * Removes record of running snapshot from cluster state and notifies the listener when this action is complete
-     *
-     * @param snapshot   snapshot
-     * @param snapshot   snapshot info if snapshot was successful
-     * @param t          exception if snapshot failed
+     *  @param snapshot   snapshot
+     * @param failure          exception if snapshot failed
      * @param listener   listener to notify when snapshot information is removed from the cluster state
      */
-    private void removeSnapshotFromClusterState(final Snapshot snapshot, final SnapshotInfo snapshotInfo, final Throwable t,
+    private void removeSnapshotFromClusterState(final Snapshot snapshot, final SnapshotInfo snapshotInfo, final Exception failure,
                                                 @Nullable ActionListener<SnapshotInfo> listener) {
         clusterService.submitStateUpdateTask("remove snapshot metadata", new ClusterStateUpdateTask() {
             @Override
@@ -918,10 +917,10 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
-                logger.warn("[{}] failed to remove snapshot metadata", t, snapshot);
+            public void onFailure(String source, Exception e) {
+                logger.warn("[{}] failed to remove snapshot metadata", e, snapshot);
                 if (listener != null) {
-                    listener.onFailure(t);
+                    listener.onFailure(e);
                 }
             }
 
@@ -932,9 +931,9 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                         if (snapshotInfo != null) {
                             listener.onSnapshotCompletion(snapshot, snapshotInfo);
                         } else {
-                            listener.onSnapshotFailure(snapshot, t);
+                            listener.onSnapshotFailure(snapshot, failure);
                         }
-                    } catch (Throwable t) {
+                    } catch (Exception t) {
                         logger.warn("failed to notify listener [{}]", t, listener);
                     }
                 }
@@ -1045,8 +1044,8 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
-                listener.onFailure(t);
+            public void onFailure(String source, Exception e) {
+                listener.onFailure(e);
             }
 
             @Override
@@ -1064,9 +1063,9 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                         }
 
                         @Override
-                        public void onSnapshotFailure(Snapshot failedSnapshot, Throwable t) {
+                        public void onSnapshotFailure(Snapshot failedSnapshot, Exception e) {
                             if (failedSnapshot.equals(snapshot)) {
-                                logger.trace("deleted snapshot failed - deleting files", t);
+                                logger.trace("deleted snapshot failed - deleting files", e);
                                 removeListener(this);
                                 deleteSnapshotFromRepository(snapshot, listener);
                             }
@@ -1111,7 +1110,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                 Repository repository = repositoriesService.repository(snapshot.getRepository());
                 repository.deleteSnapshot(snapshot.getSnapshotId());
                 listener.onResponse();
-            } catch (Throwable t) {
+            } catch (Exception t) {
                 listener.onFailure(t);
             }
         });
@@ -1272,7 +1271,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
         /**
          * Called if a snapshot operation couldn't start
          */
-        void onFailure(Throwable t);
+        void onFailure(Exception e);
     }
 
     /**
@@ -1288,14 +1287,14 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
         /**
          * Called if delete operation failed
          */
-        void onFailure(Throwable t);
+        void onFailure(Exception e);
     }
 
     public interface SnapshotCompletionListener {
 
         void onSnapshotCompletion(Snapshot snapshot, SnapshotInfo snapshotInfo);
 
-        void onSnapshotFailure(Snapshot snapshot, Throwable t);
+        void onSnapshotFailure(Snapshot snapshot, Exception e);
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/tasks/PersistedTaskInfo.java b/core/src/main/java/org/elasticsearch/tasks/PersistedTaskInfo.java
index 4e92cf2e874..ceea2774419 100644
--- a/core/src/main/java/org/elasticsearch/tasks/PersistedTaskInfo.java
+++ b/core/src/main/java/org/elasticsearch/tasks/PersistedTaskInfo.java
@@ -67,7 +67,7 @@ public final class PersistedTaskInfo implements Writeable, ToXContent {
     /**
      * Construct a {@linkplain PersistedTaskInfo} for a task that completed with an error.
      */
-    public PersistedTaskInfo(TaskInfo task, Throwable error) throws IOException {
+    public PersistedTaskInfo(TaskInfo task, Exception error) throws IOException {
         this(true, task, toXContent(error), null);
     }
 
@@ -228,12 +228,12 @@ public final class PersistedTaskInfo implements Writeable, ToXContent {
         }
     }
 
-    private static BytesReference toXContent(Throwable error) throws IOException {
+    private static BytesReference toXContent(Exception error) throws IOException {
         try (XContentBuilder builder = XContentFactory.contentBuilder(Requests.INDEX_CONTENT_TYPE)) {
             builder.startObject();
             ElasticsearchException.toXContent(builder, ToXContent.EMPTY_PARAMS, error);
             builder.endObject();
-            return builder.bytes(); 
+            return builder.bytes();
         }
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/tasks/Task.java b/core/src/main/java/org/elasticsearch/tasks/Task.java
index 8c0b599e933..93fee29d249 100644
--- a/core/src/main/java/org/elasticsearch/tasks/Task.java
+++ b/core/src/main/java/org/elasticsearch/tasks/Task.java
@@ -135,7 +135,7 @@ public class Task {
 
     public interface Status extends ToXContent, NamedWriteable {}
 
-    public PersistedTaskInfo result(DiscoveryNode node, Throwable error) throws IOException {
+    public PersistedTaskInfo result(DiscoveryNode node, Exception error) throws IOException {
         return new PersistedTaskInfo(taskInfo(node, true), error);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskManager.java b/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
index 7a7a71174a6..a6724d16f25 100644
--- a/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
+++ b/core/src/main/java/org/elasticsearch/tasks/TaskManager.java
@@ -155,7 +155,7 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen
     /**
      * Stores the task failure
      */
-    public <Response extends ActionResponse> void persistResult(Task task, Throwable error, ActionListener<Response> listener) {
+    public <Response extends ActionResponse> void persistResult(Task task, Exception error, ActionListener<Response> listener) {
         DiscoveryNode localNode = lastDiscoveryNodes.getLocalNode();
         if (localNode == null) {
             // too early to persist anything, shouldn't really be here - just pass the error along
@@ -177,7 +177,7 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 logger.warn("couldn't persist error {}", e, ExceptionsHelper.detailedMessage(error));
                 listener.onFailure(e);
             }
@@ -211,7 +211,7 @@ public class TaskManager extends AbstractComponent implements ClusterStateListen
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 logger.warn("couldn't persist response {}", e, response);
                 listener.onFailure(e);
             }
diff --git a/core/src/main/java/org/elasticsearch/tasks/TaskPersistenceService.java b/core/src/main/java/org/elasticsearch/tasks/TaskPersistenceService.java
index e1a26663e7b..992a7de4b72 100644
--- a/core/src/main/java/org/elasticsearch/tasks/TaskPersistenceService.java
+++ b/core/src/main/java/org/elasticsearch/tasks/TaskPersistenceService.java
@@ -90,13 +90,14 @@ public class TaskPersistenceService extends AbstractComponent {
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     if (ExceptionsHelper.unwrapCause(e) instanceof IndexAlreadyExistsException) {
                         // we have the index, do it
                         try {
                             doPersist(taskResult, listener);
-                        } catch (Throwable e1) {
-                            listener.onFailure(e1);
+                        } catch (Exception inner) {
+                            inner.addSuppressed(e);
+                            listener.onFailure(inner);
                         }
                     } else {
                         listener.onFailure(e);
@@ -115,7 +116,7 @@ public class TaskPersistenceService extends AbstractComponent {
                                  }
 
                                  @Override
-                                 public void onFailure(Throwable e) {
+                                 public void onFailure(Exception e) {
                                      listener.onFailure(e);
                                  }
                              }
@@ -142,7 +143,7 @@ public class TaskPersistenceService extends AbstractComponent {
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 listener.onFailure(e);
             }
         });
diff --git a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
index c465f98c182..f0519080803 100644
--- a/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
+++ b/core/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
@@ -405,9 +405,9 @@ public class ThreadPool extends AbstractComponent implements Closeable {
         public void run() {
             try {
                 runnable.run();
-            } catch (Throwable t) {
-                logger.warn("failed to run {}", t, runnable.toString());
-                throw t;
+            } catch (Exception e) {
+                logger.warn("failed to run {}", e, runnable.toString());
+                throw e;
             }
         }
 
diff --git a/core/src/main/java/org/elasticsearch/transport/DelegatingTransportChannel.java b/core/src/main/java/org/elasticsearch/transport/DelegatingTransportChannel.java
index f6b178dba8d..af85b438618 100644
--- a/core/src/main/java/org/elasticsearch/transport/DelegatingTransportChannel.java
+++ b/core/src/main/java/org/elasticsearch/transport/DelegatingTransportChannel.java
@@ -64,8 +64,8 @@ public class DelegatingTransportChannel implements TransportChannel {
     }
 
     @Override
-    public void sendResponse(Throwable error) throws IOException {
-        channel.sendResponse(error);
+    public void sendResponse(Exception exception) throws IOException {
+        channel.sendResponse(exception);
     }
 
     public TransportChannel getChannel() {
diff --git a/core/src/main/java/org/elasticsearch/transport/PlainTransportFuture.java b/core/src/main/java/org/elasticsearch/transport/PlainTransportFuture.java
index 350d556536b..3d0e603195b 100644
--- a/core/src/main/java/org/elasticsearch/transport/PlainTransportFuture.java
+++ b/core/src/main/java/org/elasticsearch/transport/PlainTransportFuture.java
@@ -87,8 +87,8 @@ public class PlainTransportFuture<V extends TransportResponse> extends BaseFutur
         try {
             handler.handleResponse(response);
             set(response);
-        } catch (Throwable t) {
-            handleException(new ResponseHandlerFailureTransportException(t));
+        } catch (Exception e) {
+            handleException(new ResponseHandlerFailureTransportException(e));
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/transport/RequestHandlerRegistry.java b/core/src/main/java/org/elasticsearch/transport/RequestHandlerRegistry.java
index 9c783c57380..27342af1bdd 100644
--- a/core/src/main/java/org/elasticsearch/transport/RequestHandlerRegistry.java
+++ b/core/src/main/java/org/elasticsearch/transport/RequestHandlerRegistry.java
@@ -118,9 +118,9 @@ public class RequestHandlerRegistry<Request extends TransportRequest> {
         }
 
         @Override
-        public void sendResponse(Throwable error) throws IOException {
+        public void sendResponse(Exception exception) throws IOException {
             endTask();
-            super.sendResponse(error);
+            super.sendResponse(exception);
         }
 
         private void endTask() {
diff --git a/core/src/main/java/org/elasticsearch/transport/ResponseHandlerFailureTransportException.java b/core/src/main/java/org/elasticsearch/transport/ResponseHandlerFailureTransportException.java
index 375fbb8bfcb..8597edb8761 100644
--- a/core/src/main/java/org/elasticsearch/transport/ResponseHandlerFailureTransportException.java
+++ b/core/src/main/java/org/elasticsearch/transport/ResponseHandlerFailureTransportException.java
@@ -25,8 +25,6 @@ import java.io.IOException;
 
 /**
  * A failure to handle the response of a transaction action.
- *
- *
  */
 public class ResponseHandlerFailureTransportException extends TransportException {
 
diff --git a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
index cd8063969d4..6814e7ad5f1 100644
--- a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
@@ -103,8 +103,6 @@ import static org.elasticsearch.common.transport.NetworkExceptionHelper.isCloseC
 import static org.elasticsearch.common.transport.NetworkExceptionHelper.isConnectException;
 import static org.elasticsearch.common.util.concurrent.ConcurrentCollections.newConcurrentMap;
 
-/**
- */
 public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent implements Transport {
 
     public static final String HTTP_SERVER_WORKER_THREAD_NAME_PREFIX = "http_server_worker";
@@ -257,12 +255,12 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
                 for (Channel channel : channels.allChannels) {
                     try {
                         sendMessage(channel, pingHeader, successfulPings::inc, false);
-                    } catch (Throwable t) {
+                    } catch (Exception e) {
                         if (isOpen(channel)) {
-                            logger.debug("[{}] failed to send ping transport message", t, node);
+                            logger.debug("[{}] failed to send ping transport message", e, node);
                             failedPings.inc();
                         } else {
-                            logger.trace("[{}] failed to send ping transport message (channel closed)", t, node);
+                            logger.trace("[{}] failed to send ping transport message (channel closed)", e, node);
                         }
                     }
                 }
@@ -283,11 +281,11 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
         }
 
         @Override
-        public void onFailure(Throwable t) {
+        public void onFailure(Exception e) {
             if (lifecycle.stoppedOrClosed()) {
-                logger.trace("failed to send ping transport message", t);
+                logger.trace("failed to send ping transport message", e);
             } else {
-                logger.warn("failed to send ping transport message", t);
+                logger.warn("failed to send ping transport message", e);
             }
         }
     }
@@ -393,7 +391,7 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
                     } else {
                         try {
                             nodeChannels = connectToChannels(node);
-                        } catch (Throwable e) {
+                        } catch (Exception e) {
                             logger.trace("failed to connect to [{}], cleaning dangling connections", e, node);
                             throw e;
                         }
@@ -445,7 +443,7 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
     /**
      * Disconnects from a node if a channel is found as part of that nodes channels.
      */
-    protected final void disconnectFromNodeChannel(final Channel channel, final Throwable failure) {
+    protected final void disconnectFromNodeChannel(final Channel channel, final Exception failure) {
         threadPool.generic().execute(() -> {
             try {
                 closeChannels(Collections.singletonList(channel));
@@ -769,8 +767,8 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
                 for (Map.Entry<String, List<Channel>> entry : serverChannels.entrySet()) {
                     try {
                         closeChannels(entry.getValue());
-                    } catch (Throwable t) {
-                        logger.debug("Error closing serverChannel for profile [{}]", t, entry.getKey());
+                    } catch (Exception e) {
+                        logger.debug("Error closing serverChannel for profile [{}]", e, entry.getKey());
                     }
                 }
                 try {
@@ -797,7 +795,7 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
         }
     }
 
-    protected void onException(Channel channel, Throwable e) {
+    protected void onException(Channel channel, Exception e) {
         if (!lifecycle.started()) {
             // ignore
             return;
@@ -937,7 +935,7 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
      * @param requestId the request ID this response replies to
      * @param action the action this response replies to
      */
-    public void sendErrorResponse(Version nodeVersion, Channel channel, final Throwable error, final long requestId,
+    public void sendErrorResponse(Version nodeVersion, Channel channel, final Exception error, final long requestId,
                                   final String action) throws IOException {
         BytesStreamOutput stream = new BytesStreamOutput();
         stream.setVersion(nodeVersion);
@@ -964,7 +962,7 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
     /**
      * Sends the response to the given channel. This method should be used to send {@link TransportResponse} objects back to the caller.
      *
-     * @see #sendErrorResponse(Version, Object, Throwable, long, String) for sending back errors to the caller
+     * @see #sendErrorResponse(Version, Object, Exception, long, String) for sending back errors to the caller
      */
     public void sendResponse(Version nodeVersion, Channel channel, final TransportResponse response, final long requestId,
                              final String action, TransportResponseOptions options) throws IOException {
@@ -1216,15 +1214,15 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
         response.remoteAddress(new InetSocketTransportAddress(remoteAddress));
         try {
             response.readFrom(stream);
-        } catch (Throwable e) {
+        } catch (Exception e) {
             handleException(handler, new TransportSerializationException(
                 "Failed to deserialize response of type [" + response.getClass().getName() + "]", e));
             return;
         }
         threadPool.executor(handler.executor()).execute(new AbstractRunnable() {
             @Override
-            public void onFailure(Throwable t) {
-                handleException(handler, new ResponseHandlerFailureTransportException(t));
+            public void onFailure(Exception e) {
+                handleException(handler, new ResponseHandlerFailureTransportException(e));
             }
 
             @Override
@@ -1238,10 +1236,10 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
      * Executed for a received response error
      */
     private void handlerResponseError(StreamInput stream, final TransportResponseHandler handler) {
-        Throwable error;
+        Exception error;
         try {
-            error = stream.readThrowable();
-        } catch (Throwable e) {
+            error = stream.readException();
+        } catch (Exception e) {
             error = new TransportSerializationException("Failed to deserialize exception response from stream", e);
         }
         handleException(handler, error);
@@ -1255,7 +1253,7 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
         threadPool.executor(handler.executor()).execute(() -> {
             try {
                 handler.handleException(rtx);
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 logger.error("failed to handle exception response [{}]", e, handler);
             }
         });
@@ -1284,16 +1282,16 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
             // in case we throw an exception, i.e. when the limit is hit, we don't want to verify
             validateRequest(stream, requestId, action);
             threadPool.executor(reg.getExecutor()).execute(new RequestHandler(reg, request, transportChannel));
-        } catch (Throwable e) {
+        } catch (Exception e) {
             // the circuit breaker tripped
             if (transportChannel == null) {
                 transportChannel = new TcpTransportChannel<>(this, channel, transportName, action, requestId, version, profileName, 0);
             }
             try {
                 transportChannel.sendResponse(e);
-            } catch (IOException e1) {
-                logger.warn("Failed to send error message back to client for action [{}]", e, action);
-                logger.warn("Actual Exception", e1);
+            } catch (IOException inner) {
+                inner.addSuppressed(e);
+                logger.warn("Failed to send error message back to client for action [{}]", inner, action);
             }
         }
         return action;
@@ -1332,14 +1330,14 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
         }
 
         @Override
-        public void onFailure(Throwable e) {
+        public void onFailure(Exception e) {
             if (lifecycleState() == Lifecycle.State.STARTED) {
                 // we can only send a response transport is started....
                 try {
                     transportChannel.sendResponse(e);
-                } catch (Throwable e1) {
-                    logger.warn("Failed to send error message back to client for action [{}]", e1, reg.getAction());
-                    logger.warn("Actual Exception", e);
+                } catch (Exception inner) {
+                    inner.addSuppressed(e);
+                    logger.warn("Failed to send error message back to client for action [{}]", inner, reg.getAction());
                 }
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java b/core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java
index 74bdad48746..dc8f26f1864 100644
--- a/core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java
+++ b/core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java
@@ -72,9 +72,9 @@ public final class TcpTransportChannel<Channel> implements TransportChannel {
     }
 
     @Override
-    public void sendResponse(Throwable error) throws IOException {
+    public void sendResponse(Exception exception) throws IOException {
         release();
-       transport.sendErrorResponse(version, channel, error, requestId, action);
+       transport.sendErrorResponse(version, channel, exception, requestId, action);
     }
 
     private void release() {
diff --git a/core/src/main/java/org/elasticsearch/transport/TransportChannel.java b/core/src/main/java/org/elasticsearch/transport/TransportChannel.java
index 53fd4ebe91e..d9fbcad30e0 100644
--- a/core/src/main/java/org/elasticsearch/transport/TransportChannel.java
+++ b/core/src/main/java/org/elasticsearch/transport/TransportChannel.java
@@ -38,5 +38,5 @@ public interface TransportChannel {
 
     void sendResponse(TransportResponse response, TransportResponseOptions options) throws IOException;
 
-    void sendResponse(Throwable error) throws IOException;
+    void sendResponse(Exception exception) throws IOException;
 }
diff --git a/core/src/main/java/org/elasticsearch/transport/TransportService.java b/core/src/main/java/org/elasticsearch/transport/TransportService.java
index 5a62c8bad31..765bb988777 100644
--- a/core/src/main/java/org/elasticsearch/transport/TransportService.java
+++ b/core/src/main/java/org/elasticsearch/transport/TransportService.java
@@ -64,7 +64,6 @@ import java.util.function.Supplier;
 
 import static java.util.Collections.emptyList;
 import static org.elasticsearch.common.settings.Setting.listSetting;
-import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
 
 /**
  *
@@ -204,13 +203,13 @@ public class TransportService extends AbstractLifecycleComponent {
                     // want handlers to worry about stack overflows
                     threadPool.generic().execute(new AbstractRunnable() {
                         @Override
-                        public void onRejection(Throwable t) {
+                        public void onRejection(Exception e) {
                             // if we get rejected during node shutdown we don't wanna bubble it up
-                            logger.debug("failed to notify response handler on rejection, action: {}", t, holderToNotify.action());
+                            logger.debug("failed to notify response handler on rejection, action: {}", e, holderToNotify.action());
                         }
                         @Override
-                        public void onFailure(Throwable t) {
-                            logger.warn("failed to notify response handler on exception, action: {}", t, holderToNotify.action());
+                        public void onFailure(Exception e) {
+                            logger.warn("failed to notify response handler on exception, action: {}", e, holderToNotify.action());
                         }
                         @Override
                         public void doRun() {
@@ -470,7 +469,7 @@ public class TransportService extends AbstractLifecycleComponent {
             } else {
                 transport.sendRequest(node, requestId, action, request, options);
             }
-        } catch (final Throwable e) {
+        } catch (final Exception e) {
             // usually happen either because we failed to connect to the node
             // or because we failed serializing the message
             final RequestHolder holderToNotify = clientHandlers.remove(requestId);
@@ -482,13 +481,13 @@ public class TransportService extends AbstractLifecycleComponent {
                 final SendRequestTransportException sendRequestException = new SendRequestTransportException(node, action, e);
                 threadPool.executor(ThreadPool.Names.GENERIC).execute(new AbstractRunnable() {
                     @Override
-                    public void onRejection(Throwable t) {
+                    public void onRejection(Exception e) {
                         // if we get rejected during node shutdown we don't wanna bubble it up
-                        logger.debug("failed to notify response handler on rejection, action: {}", t, holderToNotify.action());
+                        logger.debug("failed to notify response handler on rejection, action: {}", e, holderToNotify.action());
                     }
                     @Override
-                    public void onFailure(Throwable t) {
-                        logger.warn("failed to notify response handler on exception, action: {}", t, holderToNotify.action());
+                    public void onFailure(Exception e) {
+                        logger.warn("failed to notify response handler on exception, action: {}", e, holderToNotify.action());
                     }
                     @Override
                     protected void doRun() throws Exception {
@@ -524,23 +523,23 @@ public class TransportService extends AbstractLifecycleComponent {
                     }
 
                     @Override
-                    public void onFailure(Throwable e) {
+                    public void onFailure(Exception e) {
                         try {
                             channel.sendResponse(e);
-                        } catch (Throwable e1) {
-                            logger.warn("failed to notify channel of error message for action [{}]", e1, action);
-                            logger.warn("actual exception", e);
+                        } catch (Exception inner) {
+                            inner.addSuppressed(e);
+                            logger.warn("failed to notify channel of error message for action [{}]", inner, action);
                         }
                     }
                 });
             }
 
-        } catch (Throwable e) {
+        } catch (Exception e) {
             try {
                 channel.sendResponse(e);
-            } catch (Throwable e1) {
-                logger.warn("failed to notify channel of error message for action [{}]", e1, action);
-                logger.warn("actual exception", e1);
+            } catch (Exception inner) {
+                inner.addSuppressed(e);
+                logger.warn("failed to notify channel of error message for action [{}]", inner, action);
             }
         }
 
@@ -655,14 +654,14 @@ public class TransportService extends AbstractLifecycleComponent {
         }
 
         @Override
-        public void onResponseSent(long requestId, String action, Throwable t) {
+        public void onResponseSent(long requestId, String action, Exception e) {
             if (traceEnabled() && shouldTraceAction(action)) {
-                traceResponseSent(requestId, action, t);
+                traceResponseSent(requestId, action, e);
             }
         }
 
-        protected void traceResponseSent(long requestId, String action, Throwable t) {
-            tracerLog.trace("[{}][{}] sent error response", t, requestId, action);
+        protected void traceResponseSent(long requestId, String action, Exception e) {
+            tracerLog.trace("[{}][{}] sent error response", e, requestId, action);
         }
 
         @Override
@@ -998,17 +997,17 @@ public class TransportService extends AbstractLifecycleComponent {
         protected void processResponse(TransportResponseHandler handler, TransportResponse response) {
             try {
                 handler.handleResponse(response);
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 processException(handler, wrapInRemote(new ResponseHandlerFailureTransportException(e)));
             }
         }
 
         @Override
-        public void sendResponse(Throwable error) throws IOException {
+        public void sendResponse(Exception exception) throws IOException {
             final TransportResponseHandler handler = adapter.onResponseReceived(requestId);
             // ignore if its null, the adapter logs it
             if (handler != null) {
-                final RemoteTransportException rtx = wrapInRemote(error);
+                final RemoteTransportException rtx = wrapInRemote(exception);
                 final String executor = handler.executor();
                 if (ThreadPool.Names.SAME.equals(executor)) {
                     processException(handler, rtx);
@@ -1024,17 +1023,17 @@ public class TransportService extends AbstractLifecycleComponent {
             }
         }
 
-        protected RemoteTransportException wrapInRemote(Throwable t) {
-            if (t instanceof RemoteTransportException) {
-                return (RemoteTransportException) t;
+        protected RemoteTransportException wrapInRemote(Exception e) {
+            if (e instanceof RemoteTransportException) {
+                return (RemoteTransportException) e;
             }
-            return new RemoteTransportException(localNode.getName(), localNode.getAddress(), action, t);
+            return new RemoteTransportException(localNode.getName(), localNode.getAddress(), action, e);
         }
 
         protected void processException(final TransportResponseHandler handler, final RemoteTransportException rtx) {
             try {
                 handler.handleException(rtx);
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 logger.error("failed to handle exception for action [{}], handler [{}]", e, action, handler);
             }
         }
diff --git a/core/src/main/java/org/elasticsearch/transport/TransportServiceAdapter.java b/core/src/main/java/org/elasticsearch/transport/TransportServiceAdapter.java
index 382a4192d1c..5a1b9e9b349 100644
--- a/core/src/main/java/org/elasticsearch/transport/TransportServiceAdapter.java
+++ b/core/src/main/java/org/elasticsearch/transport/TransportServiceAdapter.java
@@ -37,7 +37,7 @@ public interface TransportServiceAdapter {
     void onResponseSent(long requestId, String action, TransportResponse response, TransportResponseOptions options);
 
     /** called by the {@link Transport} implementation after an exception was sent as a response to an incoming request */
-    void onResponseSent(long requestId, String action, Throwable t);
+    void onResponseSent(long requestId, String action, Exception e);
 
     /**
      * called by the {@link Transport} implementation when a response or an exception has been received for a previously
diff --git a/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java b/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
index 930e3951098..f6519cbeeba 100644
--- a/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
@@ -274,7 +274,7 @@ public class LocalTransport extends AbstractLifecycleComponent implements Transp
                     }
                 }
             }
-        } catch (Throwable e) {
+        } catch (Exception e) {
             if (sendRequestId != null) {
                 TransportResponseHandler handler = sourceTransport.transportServiceAdapter.onResponseReceived(sendRequestId);
                 if (handler != null) {
@@ -329,25 +329,25 @@ public class LocalTransport extends AbstractLifecycleComponent implements Transp
                     }
 
                     @Override
-                    public void onFailure(Throwable e) {
+                    public void onFailure(Exception e) {
                         if (lifecycleState() == Lifecycle.State.STARTED) {
                             // we can only send a response transport is started....
                             try {
                                 transportChannel.sendResponse(e);
-                            } catch (Throwable e1) {
-                                logger.warn("Failed to send error message back to client for action [{}]", e1, action);
-                                logger.warn("Actual Exception", e);
+                            } catch (Exception inner) {
+                                inner.addSuppressed(e);
+                                logger.warn("Failed to send error message back to client for action [{}]", inner, action);
                             }
                         }
                     }
                 });
             }
-        } catch (Throwable e) {
+        } catch (Exception e) {
             try {
                 transportChannel.sendResponse(e);
-            } catch (Throwable e1) {
-                logger.warn("Failed to send error message back to client for action [{}]", e, action);
-                logger.warn("Actual Exception", e1);
+            } catch (Exception inner) {
+                inner.addSuppressed(e);
+                logger.warn("Failed to send error message back to client for action [{}]", inner, action);
             }
 
         }
@@ -359,7 +359,7 @@ public class LocalTransport extends AbstractLifecycleComponent implements Transp
         response.remoteAddress(sourceTransport.boundAddress.publishAddress());
         try {
             response.readFrom(buffer);
-        } catch (Throwable e) {
+        } catch (Exception e) {
             handleException(handler, new TransportSerializationException(
                     "Failed to deserialize response of type [" + response.getClass().getName() + "]", e));
             return;
@@ -371,31 +371,31 @@ public class LocalTransport extends AbstractLifecycleComponent implements Transp
         threadPool.executor(handler.executor()).execute(() -> {
             try {
                 handler.handleResponse(response);
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 handleException(handler, new ResponseHandlerFailureTransportException(e));
             }
         });
     }
 
     private void handleResponseError(StreamInput buffer, final TransportResponseHandler handler) {
-        Throwable error;
+        Exception exception;
         try {
-            error = buffer.readThrowable();
-        } catch (Throwable e) {
-            error = new TransportSerializationException("Failed to deserialize exception response from stream", e);
+            exception = buffer.readException();
+        } catch (Exception e) {
+            exception = new TransportSerializationException("Failed to deserialize exception response from stream", e);
         }
-        handleException(handler, error);
+        handleException(handler, exception);
     }
 
-    private void handleException(final TransportResponseHandler handler, Throwable error) {
-        if (!(error instanceof RemoteTransportException)) {
-            error = new RemoteTransportException("None remote transport exception", null, null, error);
+    private void handleException(final TransportResponseHandler handler, Exception exception) {
+        if (!(exception instanceof RemoteTransportException)) {
+            exception = new RemoteTransportException("None remote transport exception", null, null, exception);
         }
-        final RemoteTransportException rtx = (RemoteTransportException) error;
+        final RemoteTransportException rtx = (RemoteTransportException) exception;
         try {
             handler.handleException(rtx);
-        } catch (Throwable t) {
-            logger.error("failed to handle exception response [{}]", t, handler);
+        } catch (Exception e) {
+            logger.error("failed to handle exception response [{}]", e, handler);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java b/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java
index fc3b4a3b32f..129d35c8ebf 100644
--- a/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java
+++ b/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java
@@ -91,14 +91,14 @@ public class LocalTransportChannel implements TransportChannel {
     }
 
     @Override
-    public void sendResponse(Throwable error) throws IOException {
+    public void sendResponse(Exception exception) throws IOException {
         BytesStreamOutput stream = new BytesStreamOutput();
         writeResponseExceptionHeader(stream);
         RemoteTransportException tx = new RemoteTransportException(targetTransport.nodeName(),
-                targetTransport.boundAddress().boundAddresses()[0], action, error);
+                targetTransport.boundAddress().boundAddresses()[0], action, exception);
         stream.writeThrowable(tx);
         sendResponseData(BytesReference.toBytes(stream.bytes()));
-        sourceTransportServiceAdapter.onResponseSent(requestId, action, error);
+        sourceTransportServiceAdapter.onResponseSent(requestId, action, exception);
     }
 
     private void sendResponseData(byte[] data) {
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
index d591a75d051..804d84e8374 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.transport.netty;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Booleans;
@@ -321,7 +322,10 @@ public class NettyTransport extends TcpTransport<Channel> {
     }
 
     protected final void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
-       onException(ctx.getChannel(), e.getCause());
+        onException(
+                ctx.getChannel(),
+                e.getCause() == null || e.getCause() instanceof Exception ?
+                        (Exception)e.getCause() : new ElasticsearchException(e.getCause()));
     }
 
     @Override
@@ -602,8 +606,8 @@ public class NettyTransport extends TcpTransport<Channel> {
                 ServerBootstrap serverBootstrap = entry.getValue();
                 try {
                     serverBootstrap.releaseExternalResources();
-                } catch (Throwable t) {
-                    logger.debug("Error closing serverBootstrap for profile [{}]", t, name);
+                } catch (Exception e) {
+                    logger.debug("Error closing serverBootstrap for profile [{}]", e, name);
                 }
             }
             serverBootstraps.clear();
diff --git a/core/src/main/java/org/elasticsearch/tribe/TribeService.java b/core/src/main/java/org/elasticsearch/tribe/TribeService.java
index 5c7d6d93f7c..43e36f2993b 100644
--- a/core/src/main/java/org/elasticsearch/tribe/TribeService.java
+++ b/core/src/main/java/org/elasticsearch/tribe/TribeService.java
@@ -255,13 +255,14 @@ public class TribeService extends AbstractLifecycleComponent {
             try {
                 node.injector().getInstance(ClusterService.class).add(new TribeClusterStateListener(node));
                 node.start();
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 // calling close is safe for non started nodes, we can just iterate over all
                 for (Node otherNode : nodes) {
                     try {
                         otherNode.close();
-                    } catch (Throwable t) {
-                        logger.warn("failed to close node {} on failed start", t, otherNode);
+                    } catch (Exception inner) {
+                        inner.addSuppressed(e);
+                        logger.warn("failed to close node {} on failed start", inner, otherNode);
                     }
                 }
                 if (e instanceof RuntimeException) {
@@ -282,8 +283,8 @@ public class TribeService extends AbstractLifecycleComponent {
         for (Node node : nodes) {
             try {
                 node.close();
-            } catch (Throwable t) {
-                logger.warn("failed to close node {}", t, node);
+            } catch (Exception e) {
+                logger.warn("failed to close node {}", e, node);
             }
         }
     }
@@ -307,7 +308,7 @@ public class TribeService extends AbstractLifecycleComponent {
                     event,
                     ClusterStateTaskConfig.build(Priority.NORMAL),
                     executor,
-                    (source, t) -> logger.warn("failed to process [{}]", t, source));
+                    (source, e) -> logger.warn("failed to process [{}]", e, source));
         }
     }
 
@@ -333,8 +334,8 @@ public class TribeService extends AbstractLifecycleComponent {
                 // we only need to apply the latest cluster state update
                 accumulator = applyUpdate(accumulator, tasks.get(tasks.size() - 1));
                 builder.successes(tasks);
-            } catch (Throwable t) {
-                builder.failures(tasks, t);
+            } catch (Exception e) {
+                builder.failures(tasks, e);
             }
 
             return builder.build(accumulator);
diff --git a/core/src/main/java/org/elasticsearch/watcher/FileWatcher.java b/core/src/main/java/org/elasticsearch/watcher/FileWatcher.java
index 2b498fed130..a6b0bdd8401 100644
--- a/core/src/main/java/org/elasticsearch/watcher/FileWatcher.java
+++ b/core/src/main/java/org/elasticsearch/watcher/FileWatcher.java
@@ -257,8 +257,8 @@ public class FileWatcher extends AbstractResourceWatcher<FileChangesListener> {
                     } else {
                         listener.onFileCreated(file);
                     }
-                } catch (Throwable t) {
-                    logger.warn("cannot notify file changes listener", t);
+                } catch (Exception e) {
+                    logger.warn("cannot notify file changes listener", e);
                 }
             }
         }
@@ -267,8 +267,8 @@ public class FileWatcher extends AbstractResourceWatcher<FileChangesListener> {
             for (FileChangesListener listener : listeners()) {
                 try {
                     listener.onFileDeleted(file);
-                } catch (Throwable t) {
-                    logger.warn("cannot notify file changes listener", t);
+                } catch (Exception e) {
+                    logger.warn("cannot notify file changes listener", e);
                 }
             }
         }
@@ -277,8 +277,8 @@ public class FileWatcher extends AbstractResourceWatcher<FileChangesListener> {
             for (FileChangesListener listener : listeners()) {
                 try {
                     listener.onFileChanged(file);
-                } catch (Throwable t) {
-                    logger.warn("cannot notify file changes listener", t);
+                } catch (Exception e) {
+                    logger.warn("cannot notify file changes listener", e);
                 }
 
             }
@@ -292,8 +292,8 @@ public class FileWatcher extends AbstractResourceWatcher<FileChangesListener> {
                     } else {
                         listener.onDirectoryCreated(file);
                     }
-                } catch (Throwable t) {
-                    logger.warn("cannot notify file changes listener", t);
+                } catch (Exception e) {
+                    logger.warn("cannot notify file changes listener", e);
                 }
             }
             children = listChildren(initial);
@@ -307,8 +307,8 @@ public class FileWatcher extends AbstractResourceWatcher<FileChangesListener> {
             for (FileChangesListener listener : listeners()) {
                 try {
                     listener.onDirectoryDeleted(file);
-                } catch (Throwable t) {
-                    logger.warn("cannot notify file changes listener", t);
+                } catch (Exception e) {
+                    logger.warn("cannot notify file changes listener", e);
                 }
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/ESExceptionTests.java b/core/src/test/java/org/elasticsearch/ESExceptionTests.java
index d27790beed1..5e761a97c63 100644
--- a/core/src/test/java/org/elasticsearch/ESExceptionTests.java
+++ b/core/src/test/java/org/elasticsearch/ESExceptionTests.java
@@ -57,6 +57,14 @@ import static org.hamcrest.Matchers.equalTo;
 public class ESExceptionTests extends ESTestCase {
     private static final ToXContent.Params PARAMS = ToXContent.EMPTY_PARAMS;
 
+    private class UnknownException extends Exception {
+
+        UnknownException(final String message, final Exception cause) {
+            super(message, cause);
+        }
+
+    }
+
     public void testStatus() {
         ElasticsearchException exception = new ElasticsearchException("test");
         assertThat(exception.status(), equalTo(RestStatus.INTERNAL_SERVER_ERROR));
@@ -296,7 +304,7 @@ public class ESExceptionTests extends ESTestCase {
         out.writeThrowable(ex);
 
         StreamInput in = out.bytes().streamInput();
-        ParsingException e = in.readThrowable();
+        ParsingException e = in.readException();
         assertEquals(ex.getIndex(), e.getIndex());
         assertEquals(ex.getMessage(), e.getMessage());
         assertEquals(ex.getLineNumber(), e.getLineNumber());
@@ -305,23 +313,27 @@ public class ESExceptionTests extends ESTestCase {
 
     public void testSerializeUnknownException() throws IOException {
         BytesStreamOutput out = new BytesStreamOutput();
-        ParsingException ParsingException = new ParsingException(1, 2, "foobar", null);
-        Throwable ex = new Throwable("eggplant", ParsingException);
+        ParsingException parsingException = new ParsingException(1, 2, "foobar", null);
+        final Exception ex = new UnknownException("eggplant", parsingException);
         out.writeThrowable(ex);
 
         StreamInput in = out.bytes().streamInput();
-        Throwable throwable = in.readThrowable();
-        assertEquals("throwable: eggplant", throwable.getMessage());
+        Throwable throwable = in.readException();
+        assertEquals("unknown_exception: eggplant", throwable.getMessage());
         assertTrue(throwable instanceof ElasticsearchException);
         ParsingException e = (ParsingException)throwable.getCause();
-                assertEquals(ParsingException.getIndex(), e.getIndex());
-        assertEquals(ParsingException.getMessage(), e.getMessage());
-        assertEquals(ParsingException.getLineNumber(), e.getLineNumber());
-        assertEquals(ParsingException.getColumnNumber(), e.getColumnNumber());
+                assertEquals(parsingException.getIndex(), e.getIndex());
+        assertEquals(parsingException.getMessage(), e.getMessage());
+        assertEquals(parsingException.getLineNumber(), e.getLineNumber());
+        assertEquals(parsingException.getColumnNumber(), e.getColumnNumber());
     }
 
     public void testWriteThrowable() throws IOException {
-        Throwable[] causes = new Throwable[] {
+
+        final QueryShardException queryShardException = new QueryShardException(new Index("foo", "_na_"), "foobar", null);
+        final UnknownException unknownException = new UnknownException("this exception is unknown", queryShardException);
+
+        final Exception[] causes = new Exception[]{
                 new IllegalStateException("foobar"),
                 new IllegalArgumentException("alalaal"),
                 new NullPointerException("boom"),
@@ -336,32 +348,30 @@ public class ESExceptionTests extends ESTestCase {
                 new StringIndexOutOfBoundsException("booom"),
                 new FileNotFoundException("booom"),
                 new NoSuchFileException("booom"),
-                new AssertionError("booom", new NullPointerException()),
-                new OutOfMemoryError("no memory left"),
                 new AlreadyClosedException("closed!!", new NullPointerException()),
                 new LockObtainFailedException("can't lock directory", new NullPointerException()),
-                new Throwable("this exception is unknown", new QueryShardException(new Index("foo", "_na_"), "foobar", null) ), // somethin unknown
-        };
-        for (Throwable t : causes) {
+                unknownException};
+        for (final Exception cause : causes) {
             BytesStreamOutput out = new BytesStreamOutput();
-            ElasticsearchException ex = new ElasticsearchException("topLevel", t);
+            ElasticsearchException ex = new ElasticsearchException("topLevel", cause);
             out.writeThrowable(ex);
             StreamInput in = out.bytes().streamInput();
-            ElasticsearchException e = in.readThrowable();
+            ElasticsearchException e = in.readException();
             assertEquals(e.getMessage(), ex.getMessage());
             assertTrue("Expected: " + e.getCause().getMessage() + " to contain: " +
                             ex.getCause().getClass().getName() + " but it didn't",
                     e.getCause().getMessage().contains(ex.getCause().getMessage()));
-            if (ex.getCause().getClass() != Throwable.class) { // throwable is not directly mapped
+            if (ex.getCause().getClass() != UnknownException.class) { // unknown exception is not directly mapped
                 assertEquals(e.getCause().getClass(), ex.getCause().getClass());
             } else {
                 assertEquals(e.getCause().getClass(), NotSerializableExceptionWrapper.class);
             }
             assertArrayEquals(e.getStackTrace(), ex.getStackTrace());
             assertTrue(e.getStackTrace().length > 1);
-            ElasticsearchAssertions.assertVersionSerializable(VersionUtils.randomVersion(random()), t);
+            ElasticsearchAssertions.assertVersionSerializable(VersionUtils.randomVersion(random()), cause);
             ElasticsearchAssertions.assertVersionSerializable(VersionUtils.randomVersion(random()), ex);
             ElasticsearchAssertions.assertVersionSerializable(VersionUtils.randomVersion(random()), e);
         }
     }
+
 }
diff --git a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
index 6a65b1d1695..17ff2047238 100644
--- a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
+++ b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
@@ -219,12 +219,12 @@ public class ExceptionSerializationTests extends ESTestCase {
         }
     }
 
-    private <T extends Throwable> T serialize(T exception) throws IOException {
+    private <T extends Exception> T serialize(T exception) throws IOException {
         ElasticsearchAssertions.assertVersionSerializable(VersionUtils.randomVersion(random()), exception);
         BytesStreamOutput out = new BytesStreamOutput();
         out.writeThrowable(exception);
         StreamInput in = out.bytes().streamInput();
-        return in.readThrowable();
+        return in.readException();
     }
 
     public void testIllegalShardRoutingStateException() throws IOException {
@@ -546,12 +546,20 @@ public class ExceptionSerializationTests extends ESTestCase {
         ex = serialize(new NotSerializableExceptionWrapper(new IllegalArgumentException("nono!")));
         assertEquals("{\"type\":\"illegal_argument_exception\",\"reason\":\"illegal_argument_exception: nono!\"}", toXContent(ex));
 
-        Throwable[] unknowns = new Throwable[]{
+        class UnknownException extends Exception {
+
+            public UnknownException(final String message) {
+                super(message);
+            }
+
+        }
+
+        Exception[] unknowns = new Exception[]{
                 new Exception("foobar"),
                 new ClassCastException("boom boom boom"),
-                new UnsatisfiedLinkError("booom")
+                new UnknownException("boom")
         };
-        for (Throwable t : unknowns) {
+        for (Exception t : unknowns) {
             if (randomBoolean()) {
                 t.addSuppressed(new UnsatisfiedLinkError("suppressed"));
                 t.addSuppressed(new NullPointerException());
diff --git a/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java b/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java
index 3976f665df9..93389f898e9 100644
--- a/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java
+++ b/core/src/test/java/org/elasticsearch/action/ListenerActionIT.java
@@ -51,7 +51,7 @@ public class ListenerActionIT extends ESIntegTestCase {
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 threadName.set(Thread.currentThread().getName());
                 failure.set(e);
                 latch.countDown();
diff --git a/core/src/test/java/org/elasticsearch/action/RejectionActionIT.java b/core/src/test/java/org/elasticsearch/action/RejectionActionIT.java
index 6f100170250..6745906488a 100644
--- a/core/src/test/java/org/elasticsearch/action/RejectionActionIT.java
+++ b/core/src/test/java/org/elasticsearch/action/RejectionActionIT.java
@@ -75,7 +75,7 @@ public class RejectionActionIT extends ESIntegTestCase {
                         }
 
                         @Override
-                        public void onFailure(Throwable e) {
+                        public void onFailure(Exception e) {
                             responses.add(e);
                             latch.countDown();
                         }
@@ -92,7 +92,7 @@ public class RejectionActionIT extends ESIntegTestCase {
                     assertTrue("got unexpected reason..." + failure.reason(), failure.reason().toLowerCase(Locale.ENGLISH).contains("rejected"));
                 }
             } else {
-                Throwable t = (Throwable) response;
+                Exception t = (Exception) response;
                 Throwable unwrap = ExceptionsHelper.unwrapCause(t);
                 if (unwrap instanceof SearchPhaseExecutionException) {
                     SearchPhaseExecutionException e = (SearchPhaseExecutionException) unwrap;
diff --git a/core/src/test/java/org/elasticsearch/action/admin/HotThreadsIT.java b/core/src/test/java/org/elasticsearch/action/admin/HotThreadsIT.java
index 480c103df81..5d512538821 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/HotThreadsIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/HotThreadsIT.java
@@ -100,7 +100,7 @@ public class HotThreadsIT extends ESIntegTestCase {
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     logger.error("FAILED", e);
                     hasErrors.set(true);
                     latch.countDown();
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
index 594028f4e6f..9027b3d372e 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java
@@ -230,7 +230,7 @@ public class CancellableTasksTests extends TaskManagerTestCase {
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 throwableReference.set(e);
                 responseLatch.countDown();
             }
@@ -308,7 +308,7 @@ public class CancellableTasksTests extends TaskManagerTestCase {
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 throwableReference.set(e);
                 responseLatch.countDown();
             }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
index 6f8e3fda156..f9045b58413 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java
@@ -738,12 +738,12 @@ public class TasksIT extends ESIntegTestCase {
                     }
 
                     @Override
-                    public void onFailure(Throwable e) {
+                    public void onFailure(Exception e) {
                         throw new RuntimeException(e);
                     }
                 });
         b.await();
- 
+
         // Now we can find it!
         GetTaskResponse response = expectFinishedTask(new TaskId("fake:1"));
         assertEquals("test", response.getTask().getTask().getAction());
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
index 56756ad9fdb..2c78786ab04 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java
@@ -338,7 +338,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 logger.warn("Couldn't get list of tasks", e);
                 responseLatch.countDown();
             }
@@ -526,7 +526,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase {
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 responseLatch.countDown();
             }
         });
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
index dac878eefec..657fec558b8 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java
@@ -94,7 +94,7 @@ public class ClusterRerouteTests extends ESAllocationTestCase {
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
 
             }
         };
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
index 3e7323dceeb..428e859e342 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java
@@ -41,14 +41,11 @@ import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.index.query.TermsQueryBuilder;
-import org.elasticsearch.node.service.NodeService;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
-import org.junit.Ignore;
 
 import java.util.HashMap;
-import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
 
@@ -232,7 +229,7 @@ public class CreateIndexIT extends ESIntegTestCase {
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     throw new RuntimeException(e);
                 }
             }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
index d62fe30f6fa..58784fdb7df 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/template/put/MetaDataIndexTemplateServiceTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.action.admin.indices.template.put;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.cluster.metadata.AliasValidator;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -172,8 +171,8 @@ public class MetaDataIndexTemplateServiceTests extends ESSingleNodeTestCase {
             }
 
             @Override
-            public void onFailure(Throwable t) {
-                throwables.add(t);
+            public void onFailure(Exception e) {
+                throwables.add(e);
             }
         });
         return throwables;
@@ -205,8 +204,8 @@ public class MetaDataIndexTemplateServiceTests extends ESSingleNodeTestCase {
             }
 
             @Override
-            public void onFailure(Throwable t) {
-                throwables.add(t);
+            public void onFailure(Exception e) {
+                throwables.add(e);
                 latch.countDown();
             }
         });
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/RetryTests.java b/core/src/test/java/org/elasticsearch/action/bulk/RetryTests.java
index 6d9987394f9..4fa640b3adc 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/RetryTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/RetryTests.java
@@ -149,7 +149,7 @@ public class RetryTests extends ESTestCase {
         }
 
         @Override
-        public void onFailure(Throwable e) {
+        public void onFailure(Exception e) {
             this.lastFailure = e;
             latch.countDown();
         }
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
index 6111c4c9953..7c39adc76f6 100644
--- a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
+++ b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java
@@ -201,7 +201,7 @@ public class TransportBulkActionTookTests extends ESTestCase {
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
 
             }
         });
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/BulkRequestModifierTests.java b/core/src/test/java/org/elasticsearch/action/ingest/BulkRequestModifierTests.java
index 3286c07e06c..9ee5036131d 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/BulkRequestModifierTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/BulkRequestModifierTests.java
@@ -111,7 +111,7 @@ public class BulkRequestModifierTests extends ESTestCase {
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
             }
         });
 
@@ -157,7 +157,7 @@ public class BulkRequestModifierTests extends ESTestCase {
         }
 
         @Override
-        public void onFailure(Throwable e) {
+        public void onFailure(Exception e) {
         }
 
         public BulkResponse getResponse() {
diff --git a/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java b/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java
index 3e592d1c341..a8c550e01c5 100644
--- a/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java
@@ -121,7 +121,7 @@ public class MainActionTests extends ESTestCase {
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 logger.error("unexpected error", e);
             }
         });
diff --git a/core/src/test/java/org/elasticsearch/action/support/ListenableActionFutureTests.java b/core/src/test/java/org/elasticsearch/action/support/ListenableActionFutureTests.java
index 80492f0be61..8169a674bed 100644
--- a/core/src/test/java/org/elasticsearch/action/support/ListenableActionFutureTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/ListenableActionFutureTests.java
@@ -45,15 +45,15 @@ public class ListenableActionFutureTests extends ESTestCase {
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     error.set(e);
                     listenerCalled.countDown();
                 }
             });
             Thread networkThread = new Thread(new AbstractRunnable() {
                 @Override
-                public void onFailure(Throwable t) {
-                    error.set(t);
+                public void onFailure(Exception e) {
+                    error.set(e);
                     listenerCalled.countDown();
                 }
 
diff --git a/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java b/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java
index 00068c05efe..b78425cd2a5 100644
--- a/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java
@@ -41,6 +41,8 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+import java.util.stream.IntStream;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
@@ -102,8 +104,8 @@ public class TransportActionFilterChainTests extends ESTestCase {
         try {
             assertThat(future.get(), notNullValue());
             assertThat("shouldn't get here if an error is expected", errorExpected, equalTo(false));
-        } catch(Throwable t) {
-            assertThat("shouldn't get here if an error is not expected " + t.getMessage(), errorExpected, equalTo(true));
+        } catch (ExecutionException e) {
+            assertThat("shouldn't get here if an error is not expected " + e.getMessage(), errorExpected, equalTo(true));
         }
 
         List<RequestTestFilter> testFiltersByLastExecution = new ArrayList<>();
@@ -182,8 +184,8 @@ public class TransportActionFilterChainTests extends ESTestCase {
         try {
             assertThat(future.get(), notNullValue());
             assertThat("shouldn't get here if an error is expected", errorExpected, equalTo(false));
-        } catch(Throwable t) {
-            assertThat("shouldn't get here if an error is not expected " + t.getMessage(), errorExpected, equalTo(true));
+        } catch(ExecutionException e) {
+            assertThat("shouldn't get here if an error is not expected " + e.getMessage(), errorExpected, equalTo(true));
         }
 
         List<ResponseTestFilter> testFiltersByLastExecution = new ArrayList<>();
@@ -252,7 +254,7 @@ public class TransportActionFilterChainTests extends ESTestCase {
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 failures.add(e);
                 latch.countDown();
             }
@@ -309,7 +311,7 @@ public class TransportActionFilterChainTests extends ESTestCase {
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 failures.add(e);
                 latch.countDown();
             }
diff --git a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
index 1d65f277e3c..5e49518bd5a 100644
--- a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
@@ -491,7 +491,7 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase {
         }
 
         @Override
-        public void sendResponse(Throwable error) throws IOException {
+        public void sendResponse(Exception exception) throws IOException {
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
index 37cd74e00d9..c6d17605eb4 100644
--- a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
@@ -167,7 +167,7 @@ public class TransportMasterNodeActionTests extends ESTestCase {
         Request request = new Request();
         PlainActionFuture<Response> listener = new PlainActionFuture<>();
 
-        final Throwable exception = new Throwable();
+        final Exception exception = new Exception();
         final Response response = new Response();
 
         setState(clusterService, ClusterStateCreationUtils.state(localNode, localNode, allNodes));
@@ -342,7 +342,7 @@ public class TransportMasterNodeActionTests extends ESTestCase {
             protected void masterOperation(Request request, ClusterState state, ActionListener<Response> listener) throws Exception {
                 // The other node has become master, simulate failures of this node while publishing cluster state through ZenDiscovery
                 setState(clusterService, ClusterStateCreationUtils.state(localNode, remoteNode, allNodes));
-                Throwable failure = randomBoolean()
+                Exception failure = randomBoolean()
                         ? new Discovery.FailedToCommitClusterStateException("Fake error")
                         : new NotMasterException("Fake error");
                 listener.onFailure(failure);
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java
index 55e2a9d3cf2..9f41f0e37c2 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java
@@ -81,11 +81,11 @@ public class ReplicationOperationTests extends ESTestCase {
 
         final Set<ShardRouting> expectedReplicas = getExpectedReplicas(shardId, state);
 
-        final Map<ShardRouting, Throwable> expectedFailures = new HashMap<>();
+        final Map<ShardRouting, Exception> expectedFailures = new HashMap<>();
         final Set<ShardRouting> expectedFailedShards = new HashSet<>();
         for (ShardRouting replica : expectedReplicas) {
             if (randomBoolean()) {
-                Throwable t;
+                Exception t;
                 boolean criticalFailure = randomBoolean();
                 if (criticalFailure) {
                     t = new CorruptIndexException("simulated", (String) null);
@@ -166,7 +166,7 @@ public class ReplicationOperationTests extends ESTestCase {
 
         final Set<ShardRouting> expectedReplicas = getExpectedReplicas(shardId, state);
 
-        final Map<ShardRouting, Throwable> expectedFailures = new HashMap<>();
+        final Map<ShardRouting, Exception> expectedFailures = new HashMap<>();
         final ShardRouting failedReplica = randomFrom(new ArrayList<>(expectedReplicas));
         expectedFailures.put(failedReplica, new CorruptIndexException("simulated", (String) null));
 
@@ -175,9 +175,9 @@ public class ReplicationOperationTests extends ESTestCase {
         final ClusterState finalState = state;
         final TestReplicaProxy replicasProxy = new TestReplicaProxy(expectedFailures) {
             @Override
-            public void failShard(ShardRouting replica, ShardRouting primary, String message, Throwable throwable,
-                                  Runnable onSuccess, Consumer<Throwable> onPrimaryDemoted,
-                                  Consumer<Throwable> onIgnoredFailure) {
+            public void failShard(ShardRouting replica, ShardRouting primary, String message, Exception exception,
+                                  Runnable onSuccess, Consumer<Exception> onPrimaryDemoted,
+                                  Consumer<Exception> onIgnoredFailure) {
                 assertThat(replica, equalTo(failedReplica));
                 onPrimaryDemoted.accept(new ElasticsearchException("the king is dead"));
             }
@@ -185,7 +185,7 @@ public class ReplicationOperationTests extends ESTestCase {
         AtomicBoolean primaryFailed = new AtomicBoolean();
         final TestPrimary primary = new TestPrimary(primaryShard, primaryTerm) {
             @Override
-            public void failShard(String message, Throwable throwable) {
+            public void failShard(String message, Exception exception) {
                 assertTrue(primaryFailed.compareAndSet(false, true));
             }
         };
@@ -376,8 +376,8 @@ public class ReplicationOperationTests extends ESTestCase {
         }
 
         @Override
-        public void failShard(String message, Throwable throwable) {
-            throw new AssertionError("should shouldn't be failed with [" + message + "]", throwable);
+        public void failShard(String message, Exception exception) {
+            throw new AssertionError("should shouldn't be failed with [" + message + "]", exception);
         }
 
         @Override
@@ -415,7 +415,7 @@ public class ReplicationOperationTests extends ESTestCase {
 
     static class TestReplicaProxy implements ReplicationOperation.Replicas<Request> {
 
-        final Map<ShardRouting, Throwable> opFailures;
+        final Map<ShardRouting, Exception> opFailures;
 
         final Set<ShardRouting> failedReplicas = ConcurrentCollections.newConcurrentSet();
 
@@ -423,7 +423,7 @@ public class ReplicationOperationTests extends ESTestCase {
             this(Collections.emptyMap());
         }
 
-        TestReplicaProxy(Map<ShardRouting, Throwable> opFailures) {
+        TestReplicaProxy(Map<ShardRouting, Exception> opFailures) {
             this.opFailures = opFailures;
         }
 
@@ -438,8 +438,8 @@ public class ReplicationOperationTests extends ESTestCase {
         }
 
         @Override
-        public void failShard(ShardRouting replica, ShardRouting primary, String message, Throwable throwable, Runnable onSuccess,
-                              Consumer<Throwable> onPrimaryDemoted, Consumer<Throwable> onIgnoredFailure) {
+        public void failShard(ShardRouting replica, ShardRouting primary, String message, Exception exception, Runnable onSuccess,
+                              Consumer<Exception> onPrimaryDemoted, Consumer<Exception> onIgnoredFailure) {
             if (failedReplicas.add(replica) == false) {
                 fail("replica [" + replica + "] was failed twice");
             }
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
index faa6a031f3b..de2ddabb0fe 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java
@@ -785,7 +785,7 @@ public class TransportReplicationActionTests extends ESTestCase {
                 }
 
                 @Override
-                public void failShard(String reason, @Nullable Throwable e) {
+                public void failShard(String reason, @Nullable Exception e) {
                     throw new UnsupportedOperationException();
                 }
 
@@ -856,9 +856,9 @@ public class TransportReplicationActionTests extends ESTestCase {
             }
 
             @Override
-            public void sendResponse(Throwable error) throws IOException {
-                consumer.accept(error);
-                listener.onFailure(error);
+            public void sendResponse(Exception exception) throws IOException {
+                consumer.accept(exception);
+                listener.onFailure(exception);
             }
 
             @Override
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
index 7b312959631..80e689743fd 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java
@@ -179,7 +179,7 @@ public class TransportWriteActionTests extends ESTestCase {
         }
 
         @Override
-        public void onFailure(Throwable e) {
+        public void onFailure(Exception e) {
             throw new RuntimeException(e);
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
index c26d376b587..37abc4d5eed 100644
--- a/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/single/instance/TransportInstanceSingleOperationActionTests.java
@@ -179,9 +179,9 @@ public class TransportInstanceSingleOperationActionTests extends ESTestCase {
             action.new AsyncSingleAction(request, listener).start();
             listener.get();
             fail("expected ClusterBlockException");
-        } catch (Throwable t) {
-            if (ExceptionsHelper.unwrap(t, ClusterBlockException.class) == null) {
-                logger.info("expected ClusterBlockException  but got ", t);
+        } catch (Exception e) {
+            if (ExceptionsHelper.unwrap(e, ClusterBlockException.class) == null) {
+                logger.info("expected ClusterBlockException  but got ", e);
                 fail("expected ClusterBlockException");
             }
         }
@@ -317,9 +317,9 @@ public class TransportInstanceSingleOperationActionTests extends ESTestCase {
         assertThat(transport.capturedRequests().length, equalTo(0));
         try {
             listener.get();
-        } catch (Throwable t) {
-            if (ExceptionsHelper.unwrap(t, IllegalStateException.class) == null) {
-                logger.info("expected IllegalStateException  but got ", t);
+        } catch (Exception e) {
+            if (ExceptionsHelper.unwrap(e, IllegalStateException.class) == null) {
+                logger.info("expected IllegalStateException  but got ", e);
                 fail("expected and IllegalStateException");
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
index 6ba31fdb88e..d8fd7916b5a 100644
--- a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
+++ b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java
@@ -389,19 +389,15 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase {
         TestConfig[] testConfigs = generateTestConfigs(20, testDocs, testFieldSettings);
 
         for (TestConfig test : testConfigs) {
-            try {
-                TermVectorsRequestBuilder request = getRequestForConfig(test);
-                if (test.expectedException != null) {
-                    assertThrows(request, test.expectedException);
-                    continue;
-                }
-
-                TermVectorsResponse response = request.get();
-                Fields luceneTermVectors = getTermVectorsFromLucene(directoryReader, test.doc);
-                validateResponse(response, luceneTermVectors, test);
-            } catch (Throwable t) {
-                throw new Exception("Test exception while running " + test.toString(), t);
+            TermVectorsRequestBuilder request = getRequestForConfig(test);
+            if (test.expectedException != null) {
+                assertThrows(request, test.expectedException);
+                continue;
             }
+
+            TermVectorsResponse response = request.get();
+            Fields luceneTermVectors = getTermVectorsFromLucene(directoryReader, test.doc);
+            validateResponse(response, luceneTermVectors, test);
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java b/core/src/test/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java
index 57a89c82cc8..5ed4f3252d5 100644
--- a/core/src/test/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java
+++ b/core/src/test/java/org/elasticsearch/action/termvectors/MultiTermVectorsIT.java
@@ -56,21 +56,16 @@ public class MultiTermVectorsIT extends AbstractTermVectorsTestCase {
 
         for (int i = 0; i < testConfigs.length; i++) {
             TestConfig test = testConfigs[i];
-            try {
-                MultiTermVectorsItemResponse item = responseItems[i];
-                if (test.expectedException != null) {
-                    assertTrue(item.isFailed());
-                    continue;
-                } else if (item.isFailed()) {
-                    fail(item.getFailure().getCause().getMessage());
-                }
-                Fields luceneTermVectors = getTermVectorsFromLucene(directoryReader, test.doc);
-                validateResponse(item.getResponse(), luceneTermVectors, test);
-            } catch (Throwable t) {
-                throw new Exception("Test exception while running " + test.toString(), t);
+            MultiTermVectorsItemResponse item = responseItems[i];
+            if (test.expectedException != null) {
+                assertTrue(item.isFailed());
+                continue;
+            } else if (item.isFailed()) {
+                fail(item.getFailure().getCause().getMessage());
             }
+            Fields luceneTermVectors = getTermVectorsFromLucene(directoryReader, test.doc);
+            validateResponse(item.getResponse(), luceneTermVectors, test);
         }
-
     }
 
     public void testMissingIndexThrowsMissingIndex() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java b/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java
index 196b053ff82..276a43581a6 100644
--- a/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java
+++ b/core/src/test/java/org/elasticsearch/client/AbstractClientHeadersTestCase.java
@@ -179,7 +179,7 @@ public abstract class AbstractClientHeadersTestCase extends ESTestCase {
         }
 
         @Override
-        public void onFailure(Throwable t) {
+        public void onFailure(Exception t) {
             Throwable e = unwrap(t, InternalException.class);
             assertThat("expected action [" + action + "] to throw an internal exception", e, notNullValue());
             assertThat(action, equalTo(((InternalException) e).action));
diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
index 5c07f5e6f25..42ec724ed52 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.client.transport;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.node.liveness.LivenessResponse;
 import org.elasticsearch.action.admin.cluster.node.liveness.TransportLivenessAction;
@@ -171,7 +170,7 @@ public class TransportClientNodesServiceTests extends ESTestCase {
                     }
 
                     @Override
-                    public void onFailure(Throwable e) {
+                    public void onFailure(Exception e) {
                         finalFailures.incrementAndGet();
                         finalFailure.set(e);
                         latch.countDown();
diff --git a/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java b/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java
index 61bb898acc2..aad2aa212a1 100644
--- a/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java
@@ -395,8 +395,8 @@ public class MinimumMasterNodesIT extends ESIntegTestCase {
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
-                failure.set(t);
+            public void onFailure(String source, Exception e) {
+                failure.set(e);
                 latch.countDown();
             }
         });
diff --git a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
index 7f0ac1a6e45..d387d6f7d43 100644
--- a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardStateActionTests.java
@@ -148,7 +148,7 @@ public class ShardStateActionTests extends ESTestCase {
             }
 
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception e) {
                 success.set(false);
                 latch.countDown();
                 assert false;
@@ -196,7 +196,7 @@ public class ShardStateActionTests extends ESTestCase {
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 success.set(false);
                 latch.countDown();
                 assert false;
@@ -245,9 +245,9 @@ public class ShardStateActionTests extends ESTestCase {
             }
 
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception e) {
                 success.set(false);
-                throwable.set(t);
+                throwable.set(e);
                 latch.countDown();
                 assert false;
             }
@@ -281,7 +281,7 @@ public class ShardStateActionTests extends ESTestCase {
             }
 
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception e) {
                 failure.set(true);
             }
         });
@@ -313,7 +313,7 @@ public class ShardStateActionTests extends ESTestCase {
             }
 
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception e) {
                 success.set(false);
                 latch.countDown();
                 assert false;
@@ -348,8 +348,8 @@ public class ShardStateActionTests extends ESTestCase {
             }
 
             @Override
-            public void onFailure(Throwable t) {
-                failure.set(t);
+            public void onFailure(Exception e) {
+                failure.set(e);
                 latch.countDown();
             }
         });
@@ -401,7 +401,7 @@ public class ShardStateActionTests extends ESTestCase {
         }
     }
 
-    private Throwable getSimulatedFailure() {
+    private Exception getSimulatedFailure() {
         return new CorruptIndexException("simulated", (String) null);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
index 0918358a510..8718c479216 100644
--- a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java
@@ -118,8 +118,8 @@ public class ClusterStateHealthTests extends ESTestCase {
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
-                logger.warn("unexpected failure", t);
+            public void onFailure(String source, Exception e) {
+                logger.warn("unexpected failure", e);
             }
         });
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java
index 7907ad3e63c..23713832edf 100644
--- a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceIT.java
@@ -94,7 +94,7 @@ public class ClusterServiceIT extends ESIntegTestCase {
             }
 
             @Override
-            public void onAllNodesAcked(@Nullable Throwable t) {
+            public void onAllNodesAcked(@Nullable Exception e) {
                 allNodesAcked.set(true);
                 latch.countDown();
             }
@@ -127,8 +127,8 @@ public class ClusterServiceIT extends ESIntegTestCase {
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
-                logger.error("failed to execute callback in test {}", t, source);
+            public void onFailure(String source, Exception e) {
+                logger.error("failed to execute callback in test {}", e, source);
                 onFailure.set(true);
                 latch.countDown();
             }
@@ -165,7 +165,7 @@ public class ClusterServiceIT extends ESIntegTestCase {
             }
 
             @Override
-            public void onAllNodesAcked(@Nullable Throwable t) {
+            public void onAllNodesAcked(@Nullable Exception e) {
                 allNodesAcked.set(true);
                 latch.countDown();
             }
@@ -198,8 +198,8 @@ public class ClusterServiceIT extends ESIntegTestCase {
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
-                logger.error("failed to execute callback in test {}", t, source);
+            public void onFailure(String source, Exception e) {
+                logger.error("failed to execute callback in test {}", e, source);
                 onFailure.set(true);
                 latch.countDown();
             }
@@ -240,7 +240,7 @@ public class ClusterServiceIT extends ESIntegTestCase {
             }
 
             @Override
-            public void onAllNodesAcked(@Nullable Throwable t) {
+            public void onAllNodesAcked(@Nullable Exception e) {
                 allNodesAcked.set(true);
                 latch.countDown();
             }
@@ -272,8 +272,8 @@ public class ClusterServiceIT extends ESIntegTestCase {
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
-                logger.error("failed to execute callback in test {}", t, source);
+            public void onFailure(String source, Exception e) {
+                logger.error("failed to execute callback in test {}", e, source);
                 onFailure.set(true);
                 latch.countDown();
             }
@@ -313,7 +313,7 @@ public class ClusterServiceIT extends ESIntegTestCase {
             }
 
             @Override
-            public void onAllNodesAcked(@Nullable Throwable t) {
+            public void onAllNodesAcked(@Nullable Exception e) {
                 allNodesAcked.set(true);
                 latch.countDown();
             }
@@ -346,8 +346,8 @@ public class ClusterServiceIT extends ESIntegTestCase {
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
-                logger.error("failed to execute callback in test {}", t, source);
+            public void onFailure(String source, Exception e) {
+                logger.error("failed to execute callback in test {}", e, source);
                 onFailure.set(true);
                 latch.countDown();
             }
@@ -388,7 +388,7 @@ public class ClusterServiceIT extends ESIntegTestCase {
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
+            public void onFailure(String source, Exception e) {
                 invoked1.countDown();
                 fail();
             }
@@ -403,7 +403,7 @@ public class ClusterServiceIT extends ESIntegTestCase {
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
+                public void onFailure(String source, Exception e) {
                     fail();
                 }
 
@@ -458,7 +458,7 @@ public class ClusterServiceIT extends ESIntegTestCase {
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
+            public void onFailure(String source, Exception e) {
                 invoked3.countDown();
                 fail();
             }
@@ -473,7 +473,7 @@ public class ClusterServiceIT extends ESIntegTestCase {
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
+                public void onFailure(String source, Exception e) {
                     fail();
                 }
             });
diff --git a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
index 66f96f8cd3a..c94e441aea3 100644
--- a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
@@ -149,8 +149,8 @@ public class ClusterServiceTests extends ESTestCase {
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
-                throw new RuntimeException(t);
+            public void onFailure(String source, Exception e) {
+                throw new RuntimeException(e);
             }
         });
 
@@ -163,7 +163,7 @@ public class ClusterServiceTests extends ESTestCase {
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
+            public void onFailure(String source, Exception e) {
                 timedOut.countDown();
             }
 
@@ -183,8 +183,8 @@ public class ClusterServiceTests extends ESTestCase {
         final CountDownLatch allProcessed = new CountDownLatch(1);
         clusterService.submitStateUpdateTask("test3", new ClusterStateUpdateTask() {
             @Override
-            public void onFailure(String source, Throwable t) {
-                throw new RuntimeException(t);
+            public void onFailure(String source, Exception e) {
+                throw new RuntimeException(e);
             }
 
             @Override
@@ -212,7 +212,7 @@ public class ClusterServiceTests extends ESTestCase {
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
+            public void onFailure(String source, Exception e) {
                 taskFailed[0] = true;
                 latch1.countDown();
             }
@@ -237,7 +237,7 @@ public class ClusterServiceTests extends ESTestCase {
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
+            public void onFailure(String source, Exception e) {
                 taskFailed[0] = true;
                 latch2.countDown();
             }
@@ -286,7 +286,7 @@ public class ClusterServiceTests extends ESTestCase {
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
+                public void onFailure(String source, Exception e) {
                 }
             }
         );
@@ -326,9 +326,9 @@ public class ClusterServiceTests extends ESTestCase {
 
         ClusterStateTaskListener listener = new ClusterStateTaskListener() {
             @Override
-            public void onFailure(String source, Throwable t) {
-                logger.error("unexpected failure: [{}]", t, source);
-                failures.add(new Tuple<>(source, t));
+            public void onFailure(String source, Exception e) {
+                logger.error("unexpected failure: [{}]", e, source);
+                failures.add(new Tuple<>(source, e));
                 updateLatch.countDown();
             }
 
@@ -387,8 +387,8 @@ public class ClusterServiceTests extends ESTestCase {
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
-                    fail(ExceptionsHelper.detailedMessage(t));
+                public void onFailure(String source, Exception e) {
+                    fail(ExceptionsHelper.detailedMessage(e));
                 }
             })) ;
         }
@@ -523,8 +523,8 @@ public class ClusterServiceTests extends ESTestCase {
         final CountDownLatch updateLatch = new CountDownLatch(totalTaskCount);
         final ClusterStateTaskListener listener = new ClusterStateTaskListener() {
             @Override
-            public void onFailure(String source, Throwable t) {
-                fail(ExceptionsHelper.detailedMessage(t));
+            public void onFailure(String source, Exception e) {
+                fail(ExceptionsHelper.detailedMessage(e));
             }
 
             @Override
@@ -647,8 +647,8 @@ public class ClusterServiceTests extends ESTestCase {
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
-                    fail(ExceptionsHelper.detailedMessage(t));
+                public void onFailure(String source, Exception e) {
+                    fail(ExceptionsHelper.detailedMessage(e));
                 }
             };
 
@@ -693,7 +693,7 @@ public class ClusterServiceTests extends ESTestCase {
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
+                public void onFailure(String source, Exception e) {
                     fail();
                 }
             });
@@ -710,7 +710,7 @@ public class ClusterServiceTests extends ESTestCase {
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
+                public void onFailure(String source, Exception e) {
                     latch.countDown();
                 }
             });
@@ -727,7 +727,7 @@ public class ClusterServiceTests extends ESTestCase {
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
+                public void onFailure(String source, Exception e) {
                     fail();
                 }
             });
@@ -745,7 +745,7 @@ public class ClusterServiceTests extends ESTestCase {
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
+                public void onFailure(String source, Exception e) {
                     fail();
                 }
             });
@@ -788,7 +788,7 @@ public class ClusterServiceTests extends ESTestCase {
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
+                public void onFailure(String source, Exception e) {
                     fail();
                 }
             });
@@ -807,7 +807,7 @@ public class ClusterServiceTests extends ESTestCase {
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
+                public void onFailure(String source, Exception e) {
                     latch.countDown();
                 }
             });
@@ -824,7 +824,7 @@ public class ClusterServiceTests extends ESTestCase {
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
+                public void onFailure(String source, Exception e) {
                     fail();
                 }
             });
@@ -841,7 +841,7 @@ public class ClusterServiceTests extends ESTestCase {
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
+                public void onFailure(String source, Exception e) {
                     fail();
                 }
             });
@@ -859,7 +859,7 @@ public class ClusterServiceTests extends ESTestCase {
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
+                public void onFailure(String source, Exception e) {
                     fail();
                 }
             });
@@ -902,7 +902,7 @@ public class ClusterServiceTests extends ESTestCase {
         }
 
         @Override
-        public void onFailure(String source, Throwable t) {
+        public void onFailure(String source, Exception e) {
         }
 
         public void close() {
@@ -930,7 +930,7 @@ public class ClusterServiceTests extends ESTestCase {
         }
 
         @Override
-        public void onFailure(String source, Throwable t) {
+        public void onFailure(String source, Exception e) {
             latch.countDown();
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/common/breaker/MemoryCircuitBreakerTests.java b/core/src/test/java/org/elasticsearch/common/breaker/MemoryCircuitBreakerTests.java
index bb9d23db1cb..f10a0da3029 100644
--- a/core/src/test/java/org/elasticsearch/common/breaker/MemoryCircuitBreakerTests.java
+++ b/core/src/test/java/org/elasticsearch/common/breaker/MemoryCircuitBreakerTests.java
@@ -43,7 +43,7 @@ public class MemoryCircuitBreakerTests extends ESTestCase {
         final int BYTES_PER_THREAD = scaledRandomIntBetween(500, 4500);
         final Thread[] threads = new Thread[NUM_THREADS];
         final AtomicBoolean tripped = new AtomicBoolean(false);
-        final AtomicReference<Throwable> lastException = new AtomicReference<>(null);
+        final AtomicReference<Exception> lastException = new AtomicReference<>(null);
 
         final MemoryCircuitBreaker breaker = new MemoryCircuitBreaker(new ByteSizeValue((BYTES_PER_THREAD * NUM_THREADS) - 1), 1.0, logger);
 
@@ -60,8 +60,8 @@ public class MemoryCircuitBreakerTests extends ESTestCase {
                             } else {
                                 assertThat(tripped.compareAndSet(false, true), equalTo(true));
                             }
-                        } catch (Throwable e2) {
-                            lastException.set(e2);
+                        } catch (Exception e) {
+                            lastException.set(e);
                         }
                     }
                 }
@@ -117,8 +117,8 @@ public class MemoryCircuitBreakerTests extends ESTestCase {
                             } else {
                                 assertThat(tripped.compareAndSet(false, true), equalTo(true));
                             }
-                        } catch (Throwable e2) {
-                            lastException.set(e2);
+                        } catch (Exception e) {
+                            lastException.set(e);
                         }
                     }
                 }
@@ -178,8 +178,8 @@ public class MemoryCircuitBreakerTests extends ESTestCase {
                             breaker.addEstimateBytesAndMaybeBreak(1L, "test");
                         } catch (CircuitBreakingException e) {
                             tripped.incrementAndGet();
-                        } catch (Throwable e2) {
-                            lastException.set(e2);
+                        } catch (Exception e) {
+                            lastException.set(e);
                         }
                     }
                 }
diff --git a/core/src/test/java/org/elasticsearch/common/util/CancellableThreadsTests.java b/core/src/test/java/org/elasticsearch/common/util/CancellableThreadsTests.java
index a89cb48c37a..729c431d2b2 100644
--- a/core/src/test/java/org/elasticsearch/common/util/CancellableThreadsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/CancellableThreadsTests.java
@@ -132,7 +132,7 @@ public class CancellableThreadsTests extends ESTestCase {
     public void testCancellableThreads() throws InterruptedException {
         Thread[] threads = new Thread[randomIntBetween(3, 10)];
         final TestPlan[] plans = new TestPlan[threads.length];
-        final Throwable[] throwables = new Throwable[threads.length];
+        final Exception[] exceptions = new Exception[threads.length];
         final boolean[] interrupted = new boolean[threads.length];
         final CancellableThreads cancellableThreads = new CancellableThreads();
         final CountDownLatch readyForCancel = new CountDownLatch(threads.length);
@@ -153,8 +153,8 @@ public class CancellableThreadsTests extends ESTestCase {
                     } else {
                         cancellableThreads.execute(new TestRunnable(plan, readyForCancel));
                     }
-                } catch (Throwable t) {
-                    throwables[plan.id] = t;
+                } catch (Exception e) {
+                    exceptions[plan.id] = e;
                 }
                 if (plan.exceptBeforeCancel || plan.exitBeforeCancel) {
                     // we have to mark we're ready now (actually done).
@@ -176,19 +176,19 @@ public class CancellableThreadsTests extends ESTestCase {
             TestPlan plan = plans[i];
             final Class<?> exceptionClass = plan.ioException ? IOCustomException.class : CustomException.class;
             if (plan.exceptBeforeCancel) {
-                assertThat(throwables[i], Matchers.instanceOf(exceptionClass));
+                assertThat(exceptions[i], Matchers.instanceOf(exceptionClass));
             } else if (plan.exitBeforeCancel) {
-                assertNull(throwables[i]);
+                assertNull(exceptions[i]);
             } else {
                 // in all other cases, we expect a cancellation exception.
-                assertThat(throwables[i], Matchers.instanceOf(CancellableThreads.ExecutionCancelledException.class));
+                assertThat(exceptions[i], Matchers.instanceOf(CancellableThreads.ExecutionCancelledException.class));
                 if (plan.exceptAfterCancel) {
-                    assertThat(throwables[i].getSuppressed(),
+                    assertThat(exceptions[i].getSuppressed(),
                             Matchers.arrayContaining(
                                     Matchers.instanceOf(exceptionClass)
                             ));
                 } else {
-                    assertThat(throwables[i].getSuppressed(), Matchers.emptyArray());
+                    assertThat(exceptions[i].getSuppressed(), Matchers.emptyArray());
                 }
             }
             assertThat(interrupted[plan.id], Matchers.equalTo(plan.presetInterrupt));
diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/AbstractLifecycleRunnableTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/AbstractLifecycleRunnableTests.java
index 4c2e4700943..02adb783197 100644
--- a/core/src/test/java/org/elasticsearch/common/util/concurrent/AbstractLifecycleRunnableTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/AbstractLifecycleRunnableTests.java
@@ -48,7 +48,7 @@ public class AbstractLifecycleRunnableTests extends ESTestCase {
 
         AbstractLifecycleRunnable runnable = new AbstractLifecycleRunnable(lifecycle, logger) {
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception e) {
                 fail("It should not fail");
             }
 
@@ -77,7 +77,7 @@ public class AbstractLifecycleRunnableTests extends ESTestCase {
 
         AbstractLifecycleRunnable runnable = new AbstractLifecycleRunnable(lifecycle, logger) {
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception e) {
                 fail("It should not fail");
             }
 
@@ -106,7 +106,7 @@ public class AbstractLifecycleRunnableTests extends ESTestCase {
 
         AbstractLifecycleRunnable runnable = new AbstractLifecycleRunnable(lifecycle, logger) {
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception e) {
                 fail("It should not fail");
             }
 
@@ -145,7 +145,7 @@ public class AbstractLifecycleRunnableTests extends ESTestCase {
 
         AbstractLifecycleRunnable runnable = new AbstractLifecycleRunnable(lifecycle, logger) {
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception e) {
                 fail("It should not fail");
             }
 
diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/AbstractRunnableTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/AbstractRunnableTests.java
index 54491aade6f..2373b30e1b2 100644
--- a/core/src/test/java/org/elasticsearch/common/util/concurrent/AbstractRunnableTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/AbstractRunnableTests.java
@@ -37,8 +37,8 @@ public class AbstractRunnableTests extends ESTestCase {
 
         AbstractRunnable runnable = new AbstractRunnable() {
             @Override
-            public void onFailure(Throwable t) {
-                fail("It should not fail");
+            public void onFailure(Exception e) {
+                fail(e.toString());
             }
 
             @Override
@@ -57,8 +57,8 @@ public class AbstractRunnableTests extends ESTestCase {
 
         AbstractRunnable runnable = new AbstractRunnable() {
             @Override
-            public void onFailure(Throwable t) {
-                assertSame(exception, t);
+            public void onFailure(Exception e) {
+                assertSame(exception, e);
             }
 
             @Override
@@ -76,8 +76,8 @@ public class AbstractRunnableTests extends ESTestCase {
 
         AbstractRunnable runnable = new AbstractRunnable() {
             @Override
-            public void onFailure(Throwable t) {
-                fail("It should not fail");
+            public void onFailure(Exception e) {
+                fail(e.toString());
             }
 
             @Override
@@ -91,7 +91,7 @@ public class AbstractRunnableTests extends ESTestCase {
                     afterCallable.call();
                 }
                 catch (Exception e) {
-                    fail("Unexpected for mock.");
+                    fail(e.toString());
                 }
             }
         };
@@ -111,8 +111,8 @@ public class AbstractRunnableTests extends ESTestCase {
 
         AbstractRunnable runnable = new AbstractRunnable() {
             @Override
-            public void onFailure(Throwable t) {
-                assertSame(exception, t);
+            public void onFailure(Exception e) {
+                assertSame(exception, e);
             }
 
             @Override
@@ -126,7 +126,7 @@ public class AbstractRunnableTests extends ESTestCase {
                     afterCallable.call();
                 }
                 catch (Exception e) {
-                    fail("Unexpected for mock.");
+                    fail(e.toString());
                 }
             }
         };
@@ -142,14 +142,15 @@ public class AbstractRunnableTests extends ESTestCase {
 
         AbstractRunnable runnable = new AbstractRunnable() {
             @Override
-            public void onFailure(Throwable t) {
-                assertSame(exception, t);
+            public void onFailure(Exception e) {
+                assertSame(exception, e);
 
                 try {
                     failureCallable.call();
                 }
-                catch (Exception e) {
-                    fail("Unexpected for mock.");
+                catch (Exception inner) {
+                    inner.addSuppressed(e);
+                    fail(inner.toString());
                 }
             }
 
@@ -165,8 +166,8 @@ public class AbstractRunnableTests extends ESTestCase {
     public void testIsForceExecutuonDefaultsFalse() {
         AbstractRunnable runnable = new AbstractRunnable() {
             @Override
-            public void onFailure(Throwable t) {
-                fail("Not tested");
+            public void onFailure(Exception e) {
+                fail(e.toString());
             }
 
             @Override
diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java
index 57da614e689..72db2911fc0 100644
--- a/core/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java
@@ -88,8 +88,8 @@ public class EsExecutorsTests extends ESTestCase {
             }
 
             @Override
-            public void onFailure(Throwable t) {
-                throw new AssertionError(t);
+            public void onFailure(Exception e) {
+                throw new AssertionError(e);
             }
         });
 
@@ -178,7 +178,7 @@ public class EsExecutorsTests extends ESTestCase {
                     try {
                         barrier.await();
                         barrier.await();
-                    } catch (Throwable e) {
+                    } catch (Exception e) {
                         barrier.reset(e);
                     }
                 }
@@ -214,7 +214,7 @@ public class EsExecutorsTests extends ESTestCase {
                     try {
                         barrier.await();
                         barrier.await();
-                    } catch (Throwable e) {
+                    } catch (Exception e) {
                         barrier.reset(e);
                     }
                 }
diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/RefCountedTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/RefCountedTests.java
index 9338beccb9a..c5d0ec4257e 100644
--- a/core/src/test/java/org/elasticsearch/common/util/concurrent/RefCountedTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/RefCountedTests.java
@@ -88,7 +88,7 @@ public class RefCountedTests extends ESTestCase {
         final MyRefCounted counted = new MyRefCounted();
         Thread[] threads = new Thread[randomIntBetween(2, 5)];
         final CountDownLatch latch = new CountDownLatch(1);
-        final CopyOnWriteArrayList<Throwable> exceptions = new CopyOnWriteArrayList<>();
+        final CopyOnWriteArrayList<Exception> exceptions = new CopyOnWriteArrayList<>();
         for (int i = 0; i < threads.length; i++) {
             threads[i] = new Thread() {
                 @Override
@@ -103,7 +103,7 @@ public class RefCountedTests extends ESTestCase {
                                 counted.decRef();
                             }
                         }
-                    } catch (Throwable e) {
+                    } catch (Exception e) {
                         exceptions.add(e);
                     }
                 }
diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
index d6797d4be26..e6726879513 100644
--- a/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
@@ -294,8 +294,8 @@ public class ThreadContextTests extends ESTestCase {
         }
         return new AbstractRunnable() {
             @Override
-            public void onFailure(Throwable t) {
-                throw new RuntimeException(t);
+            public void onFailure(Exception e) {
+                throw new RuntimeException(e);
             }
 
             @Override
diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java
index f3592936765..bef4a047ef5 100644
--- a/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java
+++ b/core/src/test/java/org/elasticsearch/common/xcontent/ConstructingObjectParserTests.java
@@ -68,7 +68,7 @@ public class ConstructingObjectParserTests extends ESTestCase {
             assertEquals(expected.b, parsed.b);
             assertEquals(expected.c, parsed.c);
             assertEquals(expected.d, parsed.d);
-        } catch (Throwable e) {
+        } catch (Exception e) {
             // It is convenient to decorate the error message with the json
             throw new Exception("Error parsing: [" + builder.string() + "]", e);
         }
diff --git a/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java b/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java
index bb38e329103..0f34ff181a7 100644
--- a/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java
@@ -58,8 +58,8 @@ public class BlockingClusterStatePublishResponseHandlerTests extends ESTestCase
         }
 
         @Override
-        public void onFailure(Throwable t) {
-            logger.error("unexpected error", t);
+        public void onFailure(Exception e) {
+            logger.error("unexpected error", e);
         }
 
         @Override
diff --git a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
index 13e19e84978..a1bac928daf 100644
--- a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
+++ b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
@@ -503,8 +503,8 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
                             }
                         } catch (InterruptedException e) {
                             // fine - semaphore interrupt
-                        } catch (Throwable t) {
-                            logger.info("unexpected exception in background thread of [{}]", t, node);
+                        } catch (AssertionError | Exception e) {
+                            logger.info("unexpected exception in background thread of [{}]", e, node);
                         }
                     }
                 });
@@ -690,8 +690,8 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
-                logger.warn("failure [{}]", t, source);
+            public void onFailure(String source, Exception e) {
+                logger.warn("failure [{}]", e, source);
             }
         });
 
@@ -960,7 +960,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
             }
 
             @Override
-            public void onFailure(Throwable t) {
+            public void onFailure(Exception e) {
                 success.set(false);
                 latch.countDown();
                 assert false;
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
index 9825cd3284e..531b20aa3c9 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
@@ -196,9 +196,9 @@ public class NodeJoinControllerTests extends ESTestCase {
         final SimpleFuture electionFuture = new SimpleFuture("master election");
         final Thread masterElection = new Thread(new AbstractRunnable() {
             @Override
-            public void onFailure(Throwable t) {
-                logger.error("unexpected error from waitToBeElectedAsMaster", t);
-                electionFuture.markAsFailed(t);
+            public void onFailure(Exception e) {
+                logger.error("unexpected error from waitToBeElectedAsMaster", e);
+                electionFuture.markAsFailed(e);
             }
 
             @Override
@@ -244,9 +244,9 @@ public class NodeJoinControllerTests extends ESTestCase {
         final SimpleFuture electionFuture = new SimpleFuture("master election");
         final Thread masterElection = new Thread(new AbstractRunnable() {
             @Override
-            public void onFailure(Throwable t) {
-                logger.error("unexpected error from waitToBeElectedAsMaster", t);
-                electionFuture.markAsFailed(t);
+            public void onFailure(Exception e) {
+                logger.error("unexpected error from waitToBeElectedAsMaster", e);
+                electionFuture.markAsFailed(e);
             }
 
             @Override
@@ -425,9 +425,9 @@ public class NodeJoinControllerTests extends ESTestCase {
             nodes.add(node);
             threads[i] = new Thread(new AbstractRunnable() {
                 @Override
-                public void onFailure(Throwable t) {
-                    logger.error("unexpected error in join thread", t);
-                    backgroundExceptions.add(t);
+                public void onFailure(Exception e) {
+                    logger.error("unexpected error in join thread", e);
+                    backgroundExceptions.add(e);
                 }
 
                 @Override
@@ -468,9 +468,9 @@ public class NodeJoinControllerTests extends ESTestCase {
             nodes.add(node);
             threads[i] = new Thread(new AbstractRunnable() {
                 @Override
-                public void onFailure(Throwable t) {
-                    logger.error("unexpected error in join thread", t);
-                    backgroundExceptions.add(t);
+                public void onFailure(Exception e) {
+                    logger.error("unexpected error in join thread", e);
+                    backgroundExceptions.add(e);
                 }
 
                 @Override
@@ -587,9 +587,9 @@ public class NodeJoinControllerTests extends ESTestCase {
             }
 
             @Override
-            public void onFailure(Throwable t) {
-                logger.error("unexpected error for {}", t, future);
-                future.markAsFailed(t);
+            public void onFailure(Exception e) {
+                logger.error("unexpected error for {}", e, future);
+                future.markAsFailed(e);
             }
         });
         return future;
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java
index fd0b11eae01..3d0d9ddd8b1 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryIT.java
@@ -261,8 +261,8 @@ public class ZenDiscoveryIT extends ESIntegTestCase {
             }
 
             @Override
-            public void onFailure(Throwable t) {
-                holder.set((IllegalStateException) t);
+            public void onFailure(Exception e) {
+                holder.set((IllegalStateException) e);
             }
         });
 
@@ -309,8 +309,8 @@ public class ZenDiscoveryIT extends ESIntegTestCase {
             }
 
             @Override
-            public void onFailure(Throwable t) {
-                holder.set((IllegalStateException) t);
+            public void onFailure(Exception e) {
+                holder.set((IllegalStateException) e);
             }
         });
 
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueueTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueueTests.java
index ab9aed6ba44..073e0fe1478 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueueTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueueTests.java
@@ -259,8 +259,8 @@ public class PendingClusterStatesQueueTests extends ESTestCase {
         }
 
         @Override
-        public void onNewClusterStateFailed(Throwable t) {
-            failure = t;
+        public void onNewClusterStateFailed(Exception e) {
+            failure = e;
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java
index 61374cc0d8f..8271f2dff70 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java
@@ -797,9 +797,9 @@ public class PublishClusterStateActionTests extends ESTestCase {
         }
 
         @Override
-        public void onNodeAck(DiscoveryNode node, @Nullable Throwable t) {
-            if (t != null) {
-                errors.add(new Tuple<>(node, t));
+        public void onNodeAck(DiscoveryNode node, @Nullable Exception e) {
+            if (e != null) {
+                errors.add(new Tuple<>(node, e));
             }
             countDown.countDown();
         }
@@ -910,8 +910,8 @@ public class PublishClusterStateActionTests extends ESTestCase {
         }
 
         @Override
-        public void sendResponse(Throwable error) throws IOException {
-            this.error.set(error);
+        public void sendResponse(Exception exception) throws IOException {
+            this.error.set(exception);
             assertThat(response.get(), nullValue());
         }
 
diff --git a/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
index ad425d8afc9..75bfed85333 100644
--- a/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
+++ b/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
@@ -22,7 +22,6 @@ import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.io.PathUtils;
@@ -48,8 +47,6 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists;
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileNotExists;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.containsString;
@@ -269,9 +266,9 @@ public class NodeEnvironmentTests extends ESTestCase {
         if (randomBoolean()) {
             Thread t = new Thread(new AbstractRunnable() {
                 @Override
-                public void onFailure(Throwable t) {
-                    logger.error("unexpected error", t);
-                    threadException.set(t);
+                public void onFailure(Exception e) {
+                    logger.error("unexpected error", e);
+                    threadException.set(e);
                     latch.countDown();
                     blockLatch.countDown();
                 }
diff --git a/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java b/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
index 948f4820439..c5a386976bd 100644
--- a/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
@@ -292,7 +292,7 @@ public class AsyncShardFetchTests extends ESTestCase {
                             } else {
                                 processAsyncFetch(shardId, Collections.singletonList(entry.response), null);
                             }
-                        } catch (Throwable e) {
+                        } catch (Exception e) {
                             logger.error("unexpected failure", e);
                         } finally {
                             if (entry != null) {
diff --git a/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java b/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java
index 4999ef5eac5..795046ba10c 100644
--- a/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java
+++ b/core/src/test/java/org/elasticsearch/gateway/MetaDataWriteDataNodesIT.java
@@ -161,8 +161,8 @@ public class MetaDataWriteDataNodesIT extends ESIntegTestCase {
             logger.info("checking if meta state exists...");
             try {
                 assertTrue("Expecting meta state of index " + indexName + " to be on node " + nodeName, getIndicesMetaDataOnNode(nodeName).containsKey(indexName));
-            } catch (Throwable t) {
-                logger.info("failed to load meta state", t);
+            } catch (Exception e) {
+                logger.info("failed to load meta state", e);
                 fail("could not load meta state");
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
index 96e360550af..e64c816c4bf 100644
--- a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java
@@ -24,7 +24,6 @@ import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RestoreSource;
@@ -41,6 +40,7 @@ import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardStateMetaData;
 import org.elasticsearch.snapshots.Snapshot;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.test.ESAllocationTestCase;
 import org.junit.Before;
 
@@ -547,7 +547,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase {
             return addData(node, version, allocationId, primary, null);
         }
 
-        public TestAllocator addData(DiscoveryNode node, long version, String allocationId, boolean primary, @Nullable Throwable storeException) {
+        public TestAllocator addData(DiscoveryNode node, long version, String allocationId, boolean primary, @Nullable Exception storeException) {
             if (data == null) {
                 data = new HashMap<>();
             }
diff --git a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
index 596714b1c5b..f67ee09cf0f 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
@@ -379,7 +379,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
         assertThat(gResp2.getField("foo").getValue().toString(), equalTo("bar"));
     }
 
-    public void testPrimaryRelocationWithConcurrentIndexing() throws Throwable {
+    public void testPrimaryRelocationWithConcurrentIndexing() throws Exception {
         Path dataPath = createTempDir();
         Settings nodeSettings = nodeSettings(dataPath);
 
@@ -408,7 +408,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
         final int numPhase2Docs = scaledRandomIntBetween(25, 200);
         final CountDownLatch phase1finished = new CountDownLatch(1);
         final CountDownLatch phase2finished = new CountDownLatch(1);
-        final CopyOnWriteArrayList<Throwable> exceptions = new CopyOnWriteArrayList<>();
+        final CopyOnWriteArrayList<Exception> exceptions = new CopyOnWriteArrayList<>();
         Thread thread = new Thread() {
             @Override
             public void run() {
@@ -418,8 +418,8 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
                         final IndexResponse indexResponse = client().prepareIndex(IDX, "doc",
                                 Integer.toString(counter.incrementAndGet())).setSource("foo", "bar").get();
                         assertTrue(indexResponse.isCreated());
-                    } catch (Throwable t) {
-                        exceptions.add(t);
+                    } catch (Exception e) {
+                        exceptions.add(e);
                     }
                     final int docCount = counter.get();
                     if (docCount == numPhase1Docs) {
diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
index 0ecf8462651..a3fd266f603 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
@@ -297,7 +297,7 @@ public class InternalEngineTests extends ESTestCase {
         }
         Engine.EventListener listener = new Engine.EventListener() {
             @Override
-            public void onFailedEngine(String reason, @Nullable Throwable t) {
+            public void onFailedEngine(String reason, @Nullable Exception e) {
                 // we don't need to notify anybody in this test
             }
         };
@@ -2092,7 +2092,7 @@ public class InternalEngineTests extends ESTestCase {
     }
 
     public void testShardNotAvailableExceptionWhenEngineClosedConcurrently() throws IOException, InterruptedException {
-        AtomicReference<Throwable> throwable = new AtomicReference<>();
+        AtomicReference<Exception> exception = new AtomicReference<>();
         String operation = randomFrom("optimize", "refresh", "flush");
         Thread mergeThread = new Thread() {
             @Override
@@ -2115,8 +2115,8 @@ public class InternalEngineTests extends ESTestCase {
                                 break;
                             }
                         }
-                    } catch (Throwable t) {
-                        throwable.set(t);
+                    } catch (Exception e) {
+                        exception.set(e);
                         stop = true;
                     }
                 }
@@ -2125,8 +2125,8 @@ public class InternalEngineTests extends ESTestCase {
         mergeThread.start();
         engine.close();
         mergeThread.join();
-        logger.info("exception caught: ", throwable.get());
-        assertTrue("expected an Exception that signals shard is not available", TransportActions.isShardNotAvailableException(throwable.get()));
+        logger.info("exception caught: ", exception.get());
+        assertTrue("expected an Exception that signals shard is not available", TransportActions.isShardNotAvailableException(exception.get()));
     }
 
     public void testCurrentTranslogIDisCommitted() throws IOException {
diff --git a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
index 39112ed602e..672686926bd 100644
--- a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
+++ b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
@@ -242,7 +242,7 @@ public class ShadowEngineTests extends ESTestCase {
         }
         Engine.EventListener eventListener = new Engine.EventListener() {
             @Override
-            public void onFailedEngine(String reason, @Nullable Throwable t) {
+            public void onFailedEngine(String reason, @Nullable Exception e) {
                 // we don't need to notify anybody in this test
             }
         };
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
index f9fb5e77b70..9a8815e9398 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingDisabledTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.index.mapper;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.index.IndexResponse;
@@ -116,7 +115,7 @@ public class DynamicMappingDisabledTests extends ESSingleNodeTestCase {
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 onFailureCalled.set(true);
                 assertThat(e, instanceOf(IndexNotFoundException.class));
                 assertEquals(e.getMessage(), "no such index");
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingIntegrationIT.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingIntegrationIT.java
index 8afdea27451..71628c06128 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingIntegrationIT.java
@@ -96,8 +96,8 @@ public class DynamicMappingIntegrationIT extends ESIntegTestCase {
                     try {
                         startLatch.await();
                         assertTrue(client().prepareIndex("index", "type", id).setSource("field" + id, "bar").get().isCreated());
-                    } catch (Throwable t) {
-                        error.compareAndSet(null, t);
+                    } catch (Exception e) {
+                        error.compareAndSet(null, e);
                     }
                 }
             });
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
index c93c181f860..2afeb02499d 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java
@@ -103,7 +103,7 @@ public class MapperServiceTests extends ESSingleNodeTestCase {
 
         // 2. already existing index
         IndexService indexService = createIndex("index2");
-        expectThrows(ExecutionException.class, () -> {
+        e = expectThrows(ExecutionException.class, () -> {
             client().prepareIndex("index1", MapperService.DEFAULT_MAPPING, "2").setSource().execute().get();
         });
         throwable = ExceptionsHelper.unwrapCause(e.getCause());
diff --git a/core/src/test/java/org/elasticsearch/index/mapper/merge/TestMergeMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/merge/TestMergeMapperTests.java
index 627f268545a..0133d3e5943 100644
--- a/core/src/test/java/org/elasticsearch/index/mapper/merge/TestMergeMapperTests.java
+++ b/core/src/test/java/org/elasticsearch/index/mapper/merge/TestMergeMapperTests.java
@@ -159,7 +159,7 @@ public class TestMergeMapperTests extends ESSingleNodeTestCase {
         final AtomicBoolean stopped = new AtomicBoolean(false);
         final CyclicBarrier barrier = new CyclicBarrier(2);
         final AtomicReference<String> lastIntroducedFieldName = new AtomicReference<>();
-        final AtomicReference<Throwable> error = new AtomicReference<>();
+        final AtomicReference<Exception> error = new AtomicReference<>();
         final Thread updater = new Thread() {
             @Override
             public void run() {
@@ -173,8 +173,8 @@ public class TestMergeMapperTests extends ESSingleNodeTestCase {
                         lastIntroducedFieldName.set(fieldName);
                         mapperService.merge("test", new CompressedXContent(update.toString()), MapperService.MergeReason.MAPPING_UPDATE, false);
                     }
-                } catch (Throwable t) {
-                    error.set(t);
+                } catch (Exception e) {
+                    error.set(e);
                 } finally {
                     stopped.set(true);
                 }
diff --git a/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
index 1191806bc8e..1c6a973192a 100644
--- a/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
@@ -408,7 +408,7 @@ public abstract class ESIndexLevelReplicationTestCase extends ESTestCase {
         }
 
         @Override
-        public void failShard(String message, Throwable throwable) {
+        public void failShard(String message, Exception exception) {
             throw new UnsupportedOperationException();
         }
 
@@ -436,14 +436,14 @@ public abstract class ESIndexLevelReplicationTestCase extends ESTestCase {
                     .filter(s -> replicaRouting.isSameAllocation(s.routingEntry())).findFirst().get();
                 TransportIndexAction.executeIndexRequestOnReplica(request, replica);
                 listener.onResponse(TransportResponse.Empty.INSTANCE);
-            } catch (Throwable t) {
+            } catch (Exception t) {
                 listener.onFailure(t);
             }
         }
 
         @Override
-        public void failShard(ShardRouting replica, ShardRouting primary, String message, Throwable throwable, Runnable onSuccess,
-                              Consumer<Throwable> onPrimaryDemoted, Consumer<Throwable> onIgnoredFailure) {
+        public void failShard(ShardRouting replica, ShardRouting primary, String message, Exception exception, Runnable onSuccess,
+                              Consumer<Exception> onPrimaryDemoted, Consumer<Exception> onIgnoredFailure) {
             throw new UnsupportedOperationException();
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 6674d6859ab..9bc49ad626e 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -725,7 +725,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
             }
 
             @Override
-            public void postIndex(Engine.Index index, Throwable ex) {
+            public void postIndex(Engine.Index index, Exception ex) {
                 postIndexException.incrementAndGet();
             }
 
@@ -741,7 +741,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
             }
 
             @Override
-            public void postDelete(Engine.Delete delete, Throwable ex) {
+            public void postDelete(Engine.Delete delete, Exception ex) {
                 postDeleteException.incrementAndGet();
 
             }
@@ -1383,7 +1383,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {};
         shard.close("simon says", false);
         AtomicReference<IndexShard> shardRef = new AtomicReference<>();
-        List<Throwable> failures = new ArrayList<>();
+        List<Exception> failures = new ArrayList<>();
         IndexingOperationListener listener = new IndexingOperationListener() {
 
             @Override
@@ -1393,9 +1393,9 @@ public class IndexShardTests extends ESSingleNodeTestCase {
                     // this is all IMC needs to do - check current memory and refresh
                     assertTrue(shardRef.get().getIndexBufferRAMBytesUsed() > 0);
                     shardRef.get().refresh("test");
-                } catch (Throwable t) {
-                    failures.add(t);
-                    throw t;
+                } catch (Exception e) {
+                    failures.add(e);
+                    throw e;
                 }
             }
 
@@ -1407,9 +1407,9 @@ public class IndexShardTests extends ESSingleNodeTestCase {
                     // this is all IMC needs to do - check current memory and refresh
                     assertTrue(shardRef.get().getIndexBufferRAMBytesUsed() > 0);
                     shardRef.get().refresh("test");
-                } catch (Throwable t) {
-                    failures.add(t);
-                    throw t;
+                } catch (Exception e) {
+                    failures.add(e);
+                    throw e;
                 }
             }
         };
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java
index 8d86e64a391..d1cf8b32f58 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexingOperationListenerTests.java
@@ -51,7 +51,7 @@ public class IndexingOperationListenerTests extends ESTestCase{
             }
 
             @Override
-            public void postIndex(Engine.Index index, Throwable ex) {
+            public void postIndex(Engine.Index index, Exception ex) {
                 postIndexException.incrementAndGet();
             }
 
@@ -67,7 +67,7 @@ public class IndexingOperationListenerTests extends ESTestCase{
             }
 
             @Override
-            public void postDelete(Engine.Delete delete, Throwable ex) {
+            public void postDelete(Engine.Delete delete, Exception ex) {
                 postDeleteException.incrementAndGet();
             }
         };
@@ -83,7 +83,7 @@ public class IndexingOperationListenerTests extends ESTestCase{
                 throw new RuntimeException();            }
 
             @Override
-            public void postIndex(Engine.Index index, Throwable ex) {
+            public void postIndex(Engine.Index index, Exception ex) {
                 throw new RuntimeException();            }
 
             @Override
@@ -96,7 +96,7 @@ public class IndexingOperationListenerTests extends ESTestCase{
                 throw new RuntimeException();            }
 
             @Override
-            public void postDelete(Engine.Delete delete, Throwable ex) {
+            public void postDelete(Engine.Delete delete, Exception ex) {
                 throw new RuntimeException();
             }
         };
diff --git a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
index 4938f686f60..79b0773481e 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java
@@ -115,7 +115,7 @@ public class RefreshListenersTests extends ESTestCase {
                 BigArrays.NON_RECYCLING_INSTANCE);
         Engine.EventListener eventListener = new Engine.EventListener() {
             @Override
-            public void onFailedEngine(String reason, @Nullable Throwable t) {
+            public void onFailedEngine(String reason, @Nullable Exception e) {
                 // we don't need to notify anybody in this test
             }
         };
@@ -251,7 +251,7 @@ public class RefreshListenersTests extends ESTestCase {
                             getResult.docIdAndVersion().context.reader().document(getResult.docIdAndVersion().docId, visitor);
                             assertEquals(Arrays.asList(testFieldValue), visitor.fields().get("test"));
                         }
-                    } catch (Throwable t) {
+                    } catch (Exception t) {
                         throw new RuntimeException("failure on the [" + iteration + "] iteration of thread [" + threadId + "]", t);
                     }
                 }
@@ -279,7 +279,7 @@ public class RefreshListenersTests extends ESTestCase {
         document.add(uidField);
         document.add(versionField);
         BytesReference source = new BytesArray(new byte[] { 1 });
-        ParsedDocument doc = new ParsedDocument(versionField, id, type, null, -1, -1, Arrays.asList(document), source, null); 
+        ParsedDocument doc = new ParsedDocument(versionField, id, type, null, -1, -1, Arrays.asList(document), source, null);
         Engine.Index index = new Engine.Index(new Term("_uid", uid), doc);
         engine.index(index);
         return index;
@@ -290,7 +290,7 @@ public class RefreshListenersTests extends ESTestCase {
          * When the listener is called this captures it's only argument.
          */
         AtomicReference<Boolean> forcedRefresh = new AtomicReference<>();
-        private volatile Throwable error;
+        private volatile Exception error;
 
         @Override
         public void accept(Boolean forcedRefresh) {
@@ -298,7 +298,7 @@ public class RefreshListenersTests extends ESTestCase {
                 assertNotNull(forcedRefresh);
                 Boolean oldValue = this.forcedRefresh.getAndSet(forcedRefresh);
                 assertNull("Listener called twice", oldValue);
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 error = e;
             }
         }
diff --git a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
index 84d50c6620f..eb70069b0b7 100644
--- a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
+++ b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
@@ -193,7 +193,7 @@ public class CorruptedFileIT extends ESIntegTestCase {
          * run the checkindex. if the corruption is still there we will catch it.
          */
         final CountDownLatch latch = new CountDownLatch(numShards * 3); // primary + 2 replicas
-        final CopyOnWriteArrayList<Throwable> exception = new CopyOnWriteArrayList<>();
+        final CopyOnWriteArrayList<Exception> exception = new CopyOnWriteArrayList<>();
         final IndexEventListener listener = new IndexEventListener() {
             @Override
             public void afterIndexShardClosed(ShardId sid, @Nullable IndexShard indexShard, Settings indexSettings) {
@@ -215,8 +215,8 @@ public class CorruptedFileIT extends ESIntegTestCase {
                                 throw new IOException("index check failure");
                             }
                         }
-                    } catch (Throwable t) {
-                        exception.add(t);
+                    } catch (Exception e) {
+                        exception.add(e);
                     } finally {
                         store.decRef();
                         latch.countDown();
diff --git a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
index 01eead9c96b..c9b3daa806a 100644
--- a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
+++ b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.index.translog;
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-
 import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.mockfile.FilterFileChannel;
@@ -456,7 +455,7 @@ public class TranslogTests extends ESTestCase {
         final BlockingQueue<LocationOperation> writtenOperations = new ArrayBlockingQueue<>(threadCount * opsPerThread);
 
         Thread[] threads = new Thread[threadCount];
-        final Throwable[] threadExceptions = new Throwable[threadCount];
+        final Exception[] threadExceptions = new Exception[threadCount];
         final CountDownLatch downLatch = new CountDownLatch(1);
         for (int i = 0; i < threadCount; i++) {
             final int threadId = i;
@@ -624,7 +623,7 @@ public class TranslogTests extends ESTestCase {
         final AtomicBoolean run = new AtomicBoolean(true);
 
         // any errors on threads
-        final List<Throwable> errors = new CopyOnWriteArrayList<>();
+        final List<Exception> errors = new CopyOnWriteArrayList<>();
         logger.debug("using [{}] readers. [{}] writers. flushing every ~[{}] ops.", readers.length, writers.length, flushEveryOps);
         for (int i = 0; i < writers.length; i++) {
             final String threadName = "writer_" + i;
@@ -663,9 +662,9 @@ public class TranslogTests extends ESTestCase {
                 }
 
                 @Override
-                public void onFailure(Throwable t) {
-                    logger.error("--> writer [{}] had an error", t, threadName);
-                    errors.add(t);
+                public void onFailure(Exception e) {
+                    logger.error("--> writer [{}] had an error", e, threadName);
+                    errors.add(e);
                 }
             }, threadName);
             writers[i].start();
@@ -678,14 +677,14 @@ public class TranslogTests extends ESTestCase {
                 Set<Translog.Operation> writtenOpsAtView;
 
                 @Override
-                public void onFailure(Throwable t) {
-                    logger.error("--> reader [{}] had an error", t, threadId);
-                    errors.add(t);
+                public void onFailure(Exception e) {
+                    logger.error("--> reader [{}] had an error", e, threadId);
+                    errors.add(e);
                     try {
                         closeView();
-                    } catch (IOException e) {
-                        logger.error("unexpected error while closing view, after failure");
-                        t.addSuppressed(e);
+                    } catch (IOException inner) {
+                        inner.addSuppressed(e);
+                        logger.error("unexpected error while closing view, after failure", inner);
                     }
                 }
 
@@ -1240,7 +1239,7 @@ public class TranslogTests extends ESTestCase {
         final BlockingQueue<LocationOperation> writtenOperations = new ArrayBlockingQueue<>(threadCount * opsPerThread);
 
         Thread[] threads = new Thread[threadCount];
-        final Throwable[] threadExceptions = new Throwable[threadCount];
+        final Exception[] threadExceptions = new Exception[threadCount];
         final CountDownLatch downLatch = new CountDownLatch(1);
         for (int i = 0; i < threadCount; i++) {
             final int threadId = i;
@@ -1267,10 +1266,10 @@ public class TranslogTests extends ESTestCase {
         private final int opsPerThread;
         private final int threadId;
         private final Collection<LocationOperation> writtenOperations;
-        private final Throwable[] threadExceptions;
+        private final Exception[] threadExceptions;
         private final Translog translog;
 
-        public TranslogThread(Translog translog, CountDownLatch downLatch, int opsPerThread, int threadId, Collection<LocationOperation> writtenOperations, Throwable[] threadExceptions) {
+        public TranslogThread(Translog translog, CountDownLatch downLatch, int opsPerThread, int threadId, Collection<LocationOperation> writtenOperations, Exception[] threadExceptions) {
             this.translog = translog;
             this.downLatch = downLatch;
             this.opsPerThread = opsPerThread;
@@ -1304,7 +1303,7 @@ public class TranslogTests extends ESTestCase {
                     writtenOperations.add(new LocationOperation(op, loc));
                     afterAdd();
                 }
-            } catch (Throwable t) {
+            } catch (Exception t) {
                 threadExceptions[threadId] = t;
             }
         }
@@ -1446,7 +1445,7 @@ public class TranslogTests extends ESTestCase {
 
         final int threadCount = randomIntBetween(1, 5);
         Thread[] threads = new Thread[threadCount];
-        final Throwable[] threadExceptions = new Throwable[threadCount];
+        final Exception[] threadExceptions = new Exception[threadCount];
         final CountDownLatch downLatch = new CountDownLatch(1);
         final CountDownLatch added = new CountDownLatch(randomIntBetween(10, 100));
         List<LocationOperation> writtenOperations = Collections.synchronizedList(new ArrayList<>());
diff --git a/core/src/test/java/org/elasticsearch/indexing/IndexActionIT.java b/core/src/test/java/org/elasticsearch/indexing/IndexActionIT.java
index 23925f574ff..60e062c0d1c 100644
--- a/core/src/test/java/org/elasticsearch/indexing/IndexActionIT.java
+++ b/core/src/test/java/org/elasticsearch/indexing/IndexActionIT.java
@@ -50,7 +50,7 @@ public class IndexActionIT extends ESIntegTestCase {
     public void testAutoGenerateIdNoDuplicates() throws Exception {
         int numberOfIterations = scaledRandomIntBetween(10, 50);
         for (int i = 0; i < numberOfIterations; i++) {
-            Throwable firstError = null;
+            Exception firstError = null;
             createIndex("test");
             int numOfDocs = randomIntBetween(10, 100);
             logger.info("indexing [{}] docs", numOfDocs);
@@ -66,19 +66,19 @@ public class IndexActionIT extends ESIntegTestCase {
                 try {
                     logger.debug("running search with all types");
                     assertHitCount(client().prepareSearch("test").get(), numOfDocs);
-                } catch (Throwable t) {
-                    logger.error("search for all docs types failed", t);
+                } catch (Exception e) {
+                    logger.error("search for all docs types failed", e);
                     if (firstError == null) {
-                        firstError = t;
+                        firstError = e;
                     }
                 }
                 try {
                     logger.debug("running search with a specific type");
                     assertHitCount(client().prepareSearch("test").setTypes("type").get(), numOfDocs);
-                } catch (Throwable t) {
-                    logger.error("search for all docs of a specific type failed", t);
+                } catch (Exception e) {
+                    logger.error("search for all docs of a specific type failed", e);
                     if (firstError == null) {
-                        firstError = t;
+                        firstError = e;
                     }
                 }
             }
diff --git a/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java b/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java
index 4312dd6105e..5e636bed939 100644
--- a/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/analyze/AnalyzeActionIT.java
@@ -456,9 +456,9 @@ public class AnalyzeActionIT extends ESIntegTestCase {
                 .setAnalyzer("not_exist_analyzer")
                 .get();
             fail("shouldn't get here");
-        } catch (Throwable t) {
-            assertThat(t, instanceOf(IllegalArgumentException.class));
-            assertThat(t.getMessage(), startsWith("failed to find global analyzer"));
+        } catch (Exception e) {
+            assertThat(e, instanceOf(IllegalArgumentException.class));
+            assertThat(e.getMessage(), startsWith("failed to find global analyzer"));
 
         }
 
diff --git a/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java b/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
index 8c724efdfc7..5f7f26cd38c 100644
--- a/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/flush/FlushIT.java
@@ -68,14 +68,14 @@ public class FlushIT extends ESIntegTestCase {
                             // don't use assertAllSuccessful it uses a randomized context that belongs to a different thread
                             assertThat("Unexpected ShardFailures: " + Arrays.toString(flushResponse.getShardFailures()), flushResponse.getFailedShards(), equalTo(0));
                             latch.countDown();
-                        } catch (Throwable ex) {
+                        } catch (Exception ex) {
                             onFailure(ex);
                         }
 
                     }
 
                     @Override
-                    public void onFailure(Throwable e) {
+                    public void onFailure(Exception e) {
                         errors.add(e);
                         latch.countDown();
                     }
diff --git a/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushUtil.java b/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushUtil.java
index 485ec020c3f..b71ba63a157 100644
--- a/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushUtil.java
+++ b/core/src/test/java/org/elasticsearch/indices/flush/SyncedFlushUtil.java
@@ -57,7 +57,7 @@ public class SyncedFlushUtil {
 
     public static final class LatchedListener<T> implements ActionListener<T> {
         public volatile T result;
-        public volatile Throwable error;
+        public volatile Exception error;
         public final CountDownLatch latch = new CountDownLatch(1);
 
         @Override
@@ -67,7 +67,7 @@ public class SyncedFlushUtil {
         }
 
         @Override
-        public void onFailure(Throwable e) {
+        public void onFailure(Exception e) {
             error = e;
             latch.countDown();
         }
diff --git a/core/src/test/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java b/core/src/test/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java
index 2981f2d110c..eeaeb84d9a9 100644
--- a/core/src/test/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/mapping/ConcurrentDynamicTemplateIT.java
@@ -69,7 +69,7 @@ public class ConcurrentDynamicTemplateIT extends ESIntegTestCase {
                     }
 
                     @Override
-                    public void onFailure(Throwable e) {
+                    public void onFailure(Exception e) {
                         throwable.add(e);
                         latch.countDown();
                     }
@@ -83,4 +83,4 @@ public class ConcurrentDynamicTemplateIT extends ESIntegTestCase {
 
         }
     }
-}
\ No newline at end of file
+}
diff --git a/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java
index 68a176e22c3..91fd7bb972b 100644
--- a/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java
@@ -43,6 +43,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_METADATA;
 import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_BLOCKS_READ;
@@ -258,7 +259,7 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase {
         // not all shards are allocated with the initial create index. Wait for it..
         ensureYellow();
 
-        final Throwable[] threadException = new Throwable[1];
+        final AtomicReference<Exception> threadException = new AtomicReference<>();
         final AtomicBoolean stop = new AtomicBoolean(false);
         Thread[] threads = new Thread[3];
         final CyclicBarrier barrier = new CyclicBarrier(threads.length);
@@ -298,8 +299,8 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase {
                             assertThat(mappings.containsKey(typeName), equalTo(true));
                             assertThat(((Map<String, Object>) mappings.get(typeName).getSourceAsMap().get("properties")).keySet(), Matchers.hasItem(fieldName));
                         }
-                    } catch (Throwable t) {
-                        threadException[0] = t;
+                    } catch (Exception e) {
+                        threadException.set(e);
                         stop.set(true);
                     }
                 }
@@ -311,8 +312,8 @@ public class UpdateMappingIntegrationIT extends ESIntegTestCase {
 
         for (Thread t : threads) t.join();
 
-        if (threadException[0] != null) {
-            throw threadException[0];
+        if (threadException.get() != null) {
+            throw threadException.get();
         }
 
     }
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java
index bcd614121b6..c59eea6af6d 100644
--- a/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java
@@ -59,7 +59,7 @@ public class RecoveryTargetTests extends ESTestCase {
         private T lastRead;
         private final AtomicBoolean shouldStop;
         private final T source;
-        final AtomicReference<Throwable> error = new AtomicReference<>();
+        final AtomicReference<Exception> error = new AtomicReference<>();
         final Version streamVersion;
 
         Streamer(AtomicBoolean shouldStop, T source) {
@@ -73,7 +73,7 @@ public class RecoveryTargetTests extends ESTestCase {
         }
 
         public T lastRead() throws Throwable {
-            Throwable t = error.get();
+            Exception t = error.get();
             if (t != null) {
                 throw t;
             }
@@ -105,8 +105,8 @@ public class RecoveryTargetTests extends ESTestCase {
                     serializeDeserialize();
                 }
                 serializeDeserialize();
-            } catch (Throwable t) {
-                error.set(t);
+            } catch (Exception e) {
+                error.set(e);
             }
         }
     }
diff --git a/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java b/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java
index 81c50cc4f9c..a5b8bf63422 100644
--- a/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java
@@ -145,7 +145,7 @@ public class RareClusterStateIT extends ESIntegTestCase {
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
+            public void onFailure(String source, Exception e) {
 
             }
         });
@@ -165,7 +165,7 @@ public class RareClusterStateIT extends ESIntegTestCase {
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
+            public void onFailure(String source, Exception e) {
 
             }
         });
@@ -260,7 +260,7 @@ public class RareClusterStateIT extends ESIntegTestCase {
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 putMappingResponse.set(e);
             }
         });
@@ -292,7 +292,7 @@ public class RareClusterStateIT extends ESIntegTestCase {
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 docIndexResponse.set(e);
             }
         });
@@ -376,7 +376,7 @@ public class RareClusterStateIT extends ESIntegTestCase {
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 putMappingResponse.set(e);
             }
         });
@@ -403,7 +403,7 @@ public class RareClusterStateIT extends ESIntegTestCase {
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 docIndexResponse.set(e);
             }
         });
diff --git a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java
index b4f66c2e17b..ad26ec71226 100644
--- a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java
@@ -426,7 +426,7 @@ public class IndicesStoreIntegrationIT extends ESIntegTestCase {
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
+            public void onFailure(String source, Exception e) {
             }
         });
         waitNoPendingTasksOnAll();
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
index fcc6e04c6c1..53964132abe 100644
--- a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java
@@ -74,7 +74,7 @@ public class PipelineExecutionServiceTests extends ESTestCase {
     public void testExecuteIndexPipelineDoesNotExist() {
         IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id");
         @SuppressWarnings("unchecked")
-        Consumer<Throwable> failureHandler = mock(Consumer.class);
+        Consumer<Exception> failureHandler = mock(Consumer.class);
         @SuppressWarnings("unchecked")
         Consumer<Boolean> completionHandler = mock(Consumer.class);
         try {
@@ -83,7 +83,7 @@ public class PipelineExecutionServiceTests extends ESTestCase {
         } catch (IllegalArgumentException e) {
             assertThat(e.getMessage(), equalTo("pipeline with id [_id] does not exist"));
         }
-        verify(failureHandler, never()).accept(any(Throwable.class));
+        verify(failureHandler, never()).accept(any(Exception.class));
         verify(completionHandler, never()).accept(anyBoolean());
     }
 
@@ -98,9 +98,9 @@ public class PipelineExecutionServiceTests extends ESTestCase {
                 new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("does_not_exist");
         bulkRequest.add(indexRequest2);
         @SuppressWarnings("unchecked")
-        BiConsumer<IndexRequest, Throwable> failureHandler = mock(BiConsumer.class);
+        BiConsumer<IndexRequest, Exception> failureHandler = mock(BiConsumer.class);
         @SuppressWarnings("unchecked")
-        Consumer<Throwable> completionHandler = mock(Consumer.class);
+        Consumer<Exception> completionHandler = mock(Consumer.class);
         executionService.executeBulkRequest(bulkRequest.requests(), failureHandler, completionHandler);
         verify(failureHandler, times(1)).accept(
             argThat(new CustomTypeSafeMatcher<IndexRequest>("failure handler was not called with the expected arguments") {
@@ -126,7 +126,7 @@ public class PipelineExecutionServiceTests extends ESTestCase {
 
         IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id");
         @SuppressWarnings("unchecked")
-        Consumer<Throwable> failureHandler = mock(Consumer.class);
+        Consumer<Exception> failureHandler = mock(Consumer.class);
         @SuppressWarnings("unchecked")
         Consumer<Boolean> completionHandler = mock(Consumer.class);
         executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler);
@@ -141,7 +141,7 @@ public class PipelineExecutionServiceTests extends ESTestCase {
 
         IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id");
         @SuppressWarnings("unchecked")
-        Consumer<Throwable> failureHandler = mock(Consumer.class);
+        Consumer<Exception> failureHandler = mock(Consumer.class);
         @SuppressWarnings("unchecked")
         Consumer<Boolean> completionHandler = mock(Consumer.class);
         executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler);
@@ -169,7 +169,7 @@ public class PipelineExecutionServiceTests extends ESTestCase {
 
         IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id");
         @SuppressWarnings("unchecked")
-        Consumer<Throwable> failureHandler = mock(Consumer.class);
+        Consumer<Exception> failureHandler = mock(Consumer.class);
         @SuppressWarnings("unchecked")
         Consumer<Boolean> completionHandler = mock(Consumer.class);
         executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler);
@@ -193,7 +193,7 @@ public class PipelineExecutionServiceTests extends ESTestCase {
         IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id");
         doThrow(new RuntimeException()).when(processor).execute(eqID("_index", "_type", "_id", Collections.emptyMap()));
         @SuppressWarnings("unchecked")
-        Consumer<Throwable> failureHandler = mock(Consumer.class);
+        Consumer<Exception> failureHandler = mock(Consumer.class);
         @SuppressWarnings("unchecked")
         Consumer<Boolean> completionHandler = mock(Consumer.class);
         executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler);
@@ -213,7 +213,7 @@ public class PipelineExecutionServiceTests extends ESTestCase {
         IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id");
         doThrow(new RuntimeException()).when(processor).execute(eqID("_index", "_type", "_id", Collections.emptyMap()));
         @SuppressWarnings("unchecked")
-        Consumer<Throwable> failureHandler = mock(Consumer.class);
+        Consumer<Exception> failureHandler = mock(Consumer.class);
         @SuppressWarnings("unchecked")
         Consumer<Boolean> completionHandler = mock(Consumer.class);
         executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler);
@@ -231,7 +231,7 @@ public class PipelineExecutionServiceTests extends ESTestCase {
         doThrow(new RuntimeException()).when(processor).execute(eqID("_index", "_type", "_id", Collections.emptyMap()));
         doThrow(new RuntimeException()).when(onFailureProcessor).execute(eqID("_index", "_type", "_id", Collections.emptyMap()));
         @SuppressWarnings("unchecked")
-        Consumer<Throwable> failureHandler = mock(Consumer.class);
+        Consumer<Exception> failureHandler = mock(Consumer.class);
         @SuppressWarnings("unchecked")
         Consumer<Boolean> completionHandler = mock(Consumer.class);
         executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler);
@@ -253,7 +253,7 @@ public class PipelineExecutionServiceTests extends ESTestCase {
         doThrow(new RuntimeException()).when(onFailureProcessor).execute(eqID("_index", "_type", "_id", Collections.emptyMap()));
         doThrow(new RuntimeException()).when(processor).execute(eqID("_index", "_type", "_id", Collections.emptyMap()));
         @SuppressWarnings("unchecked")
-        Consumer<Throwable> failureHandler = mock(Consumer.class);
+        Consumer<Exception> failureHandler = mock(Consumer.class);
         @SuppressWarnings("unchecked")
         Consumer<Boolean> completionHandler = mock(Consumer.class);
         executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler);
@@ -268,7 +268,7 @@ public class PipelineExecutionServiceTests extends ESTestCase {
 
         IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id");
         @SuppressWarnings("unchecked")
-        Consumer<Throwable> failureHandler = mock(Consumer.class);
+        Consumer<Exception> failureHandler = mock(Consumer.class);
         @SuppressWarnings("unchecked")
         Consumer<Boolean> completionHandler = mock(Consumer.class);
         executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler);
@@ -284,7 +284,7 @@ public class PipelineExecutionServiceTests extends ESTestCase {
 
         IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id");
         @SuppressWarnings("unchecked")
-        Consumer<Throwable> failureHandler = mock(Consumer.class);
+        Consumer<Exception> failureHandler = mock(Consumer.class);
         @SuppressWarnings("unchecked")
         Consumer<Boolean> completionHandler = mock(Consumer.class);
         executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler);
@@ -298,7 +298,7 @@ public class PipelineExecutionServiceTests extends ESTestCase {
         IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").setPipeline("_id")
                 .source(Collections.emptyMap())
                 .ttl(1000L);
-        Consumer<Throwable> failureHandler = mock(Consumer.class);
+        Consumer<Exception> failureHandler = mock(Consumer.class);
         Consumer<Boolean> completionHandler = mock(Consumer.class);
         executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler);
 
@@ -336,8 +336,8 @@ public class PipelineExecutionServiceTests extends ESTestCase {
         doThrow(error).when(processor).execute(any());
         when(store.get(pipelineId)).thenReturn(new Pipeline(pipelineId, null, processor));
 
-        BiConsumer<IndexRequest, Throwable> requestItemErrorHandler = mock(BiConsumer.class);
-        Consumer<Throwable> completionHandler = mock(Consumer.class);
+        BiConsumer<IndexRequest, Exception> requestItemErrorHandler = mock(BiConsumer.class);
+        Consumer<Exception> completionHandler = mock(Consumer.class);
         executionService.executeBulkRequest(bulkRequest.requests(), requestItemErrorHandler, completionHandler);
 
         verify(requestItemErrorHandler, times(numIndexRequests)).accept(any(IndexRequest.class), eq(error));
@@ -358,9 +358,9 @@ public class PipelineExecutionServiceTests extends ESTestCase {
         when(store.get(pipelineId)).thenReturn(new Pipeline(pipelineId, null, new CompoundProcessor()));
 
         @SuppressWarnings("unchecked")
-        BiConsumer<IndexRequest, Throwable> requestItemErrorHandler = mock(BiConsumer.class);
+        BiConsumer<IndexRequest, Exception> requestItemErrorHandler = mock(BiConsumer.class);
         @SuppressWarnings("unchecked")
-        Consumer<Throwable> completionHandler = mock(Consumer.class);
+        Consumer<Exception> completionHandler = mock(Consumer.class);
         executionService.executeBulkRequest(bulkRequest.requests(), requestItemErrorHandler, completionHandler);
 
         verify(requestItemErrorHandler, never()).accept(any(), any());
@@ -383,7 +383,7 @@ public class PipelineExecutionServiceTests extends ESTestCase {
         configurationMap.put("_id2", new PipelineConfiguration("_id2", new BytesArray("{}")));
         executionService.updatePipelineStats(new IngestMetadata(configurationMap));
 
-        Consumer<Throwable> failureHandler = mock(Consumer.class);
+        Consumer<Exception> failureHandler = mock(Consumer.class);
         Consumer<Boolean> completionHandler = mock(Consumer.class);
 
         IndexRequest indexRequest = new IndexRequest("_index");
diff --git a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java
index a1f4d381911..94a56d11933 100644
--- a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java
+++ b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmGcMonitorServiceSettingsTests.java
@@ -54,9 +54,9 @@ public class JvmGcMonitorServiceSettingsTests extends ESTestCase {
     public void testNegativeSetting() throws InterruptedException {
         String collector = randomAsciiOfLength(5);
         Settings settings = Settings.builder().put("monitor.jvm.gc.collector." + collector + ".warn", "-" + randomTimeValue()).build();
-        execute(settings, (command, interval) -> null, t -> {
-            assertThat(t, instanceOf(IllegalArgumentException.class));
-            assertThat(t.getMessage(), allOf(containsString("invalid gc_threshold"), containsString("for [monitor.jvm.gc.collector." + collector + ".")));
+        execute(settings, (command, interval) -> null, e -> {
+            assertThat(e, instanceOf(IllegalArgumentException.class));
+            assertThat(e.getMessage(), allOf(containsString("invalid gc_threshold"), containsString("for [monitor.jvm.gc.collector." + collector + ".")));
         }, true, null);
     }
 
@@ -74,9 +74,9 @@ public class JvmGcMonitorServiceSettingsTests extends ESTestCase {
         }
 
         // we should get an exception that a setting is missing
-        execute(builder.build(), (command, interval) -> null, t -> {
-            assertThat(t, instanceOf(IllegalArgumentException.class));
-            assertThat(t.getMessage(), containsString("missing gc_threshold for [monitor.jvm.gc.collector." + collector + "."));
+        execute(builder.build(), (command, interval) -> null, e -> {
+            assertThat(e, instanceOf(IllegalArgumentException.class));
+            assertThat(e.getMessage(), containsString("missing gc_threshold for [monitor.jvm.gc.collector." + collector + "."));
         }, true, null);
     }
 
@@ -84,18 +84,18 @@ public class JvmGcMonitorServiceSettingsTests extends ESTestCase {
         for (final String threshold : new String[] { "warn", "info", "debug" }) {
             final Settings.Builder builder = Settings.builder();
             builder.put("monitor.jvm.gc.overhead." + threshold, randomIntBetween(Integer.MIN_VALUE, -1));
-            execute(builder.build(), (command, interval) -> null, t -> {
-                assertThat(t, instanceOf(IllegalArgumentException.class));
-                assertThat(t.getMessage(), containsString("setting [monitor.jvm.gc.overhead." + threshold + "] must be >= 0"));
+            execute(builder.build(), (command, interval) -> null, e -> {
+                assertThat(e, instanceOf(IllegalArgumentException.class));
+                assertThat(e.getMessage(), containsString("setting [monitor.jvm.gc.overhead." + threshold + "] must be >= 0"));
             }, true, null);
         }
 
         for (final String threshold : new String[] { "warn", "info", "debug" }) {
             final Settings.Builder builder = Settings.builder();
             builder.put("monitor.jvm.gc.overhead." + threshold, randomIntBetween(100 + 1, Integer.MAX_VALUE));
-            execute(builder.build(), (command, interval) -> null, t -> {
-                assertThat(t, instanceOf(IllegalArgumentException.class));
-                assertThat(t.getMessage(), containsString("setting [monitor.jvm.gc.overhead." + threshold + "] must be <= 100"));
+            execute(builder.build(), (command, interval) -> null, e -> {
+                assertThat(e, instanceOf(IllegalArgumentException.class));
+                assertThat(e.getMessage(), containsString("setting [monitor.jvm.gc.overhead." + threshold + "] must be <= 100"));
             }, true, null);
         }
 
@@ -104,9 +104,9 @@ public class JvmGcMonitorServiceSettingsTests extends ESTestCase {
         infoWarnOutOfOrderBuilder.put("monitor.jvm.gc.overhead.info", info);
         final int warn = randomIntBetween(1, info - 1);
         infoWarnOutOfOrderBuilder.put("monitor.jvm.gc.overhead.warn", warn);
-        execute(infoWarnOutOfOrderBuilder.build(), (command, interval) -> null, t -> {
-            assertThat(t, instanceOf(IllegalArgumentException.class));
-            assertThat(t.getMessage(), containsString("[monitor.jvm.gc.overhead.warn] must be greater than [monitor.jvm.gc.overhead.info] [" + info + "] but was [" + warn + "]"));
+        execute(infoWarnOutOfOrderBuilder.build(), (command, interval) -> null, e -> {
+            assertThat(e, instanceOf(IllegalArgumentException.class));
+            assertThat(e.getMessage(), containsString("[monitor.jvm.gc.overhead.warn] must be greater than [monitor.jvm.gc.overhead.info] [" + info + "] but was [" + warn + "]"));
         }, true, null);
 
         final Settings.Builder debugInfoOutOfOrderBuilder = Settings.builder();
@@ -114,9 +114,9 @@ public class JvmGcMonitorServiceSettingsTests extends ESTestCase {
         final int debug = randomIntBetween(info + 1, 99);
         debugInfoOutOfOrderBuilder.put("monitor.jvm.gc.overhead.debug", debug);
         debugInfoOutOfOrderBuilder.put("monitor.jvm.gc.overhead.warn", randomIntBetween(debug + 1, 100)); // or the test will fail for the wrong reason
-        execute(debugInfoOutOfOrderBuilder.build(), (command, interval) -> null, t -> {
-            assertThat(t, instanceOf(IllegalArgumentException.class));
-            assertThat(t.getMessage(), containsString("[monitor.jvm.gc.overhead.info] must be greater than [monitor.jvm.gc.overhead.debug] [" + debug + "] but was [" + info + "]"));
+        execute(debugInfoOutOfOrderBuilder.build(), (command, interval) -> null, e -> {
+            assertThat(e, instanceOf(IllegalArgumentException.class));
+            assertThat(e.getMessage(), containsString("[monitor.jvm.gc.overhead.info] must be greater than [monitor.jvm.gc.overhead.debug] [" + debug + "] but was [" + info + "]"));
         }, true, null);
     }
 
@@ -124,7 +124,7 @@ public class JvmGcMonitorServiceSettingsTests extends ESTestCase {
         execute(settings, scheduler, null, false, asserts);
     }
 
-    private static void execute(Settings settings, BiFunction<Runnable, TimeValue, ScheduledFuture<?>> scheduler, Consumer<Throwable> consumer, boolean constructionShouldFail, Runnable asserts) throws InterruptedException {
+    private static void execute(Settings settings, BiFunction<Runnable, TimeValue, ScheduledFuture<?>> scheduler, Consumer<Exception> consumer, boolean constructionShouldFail, Runnable asserts) throws InterruptedException {
         assert constructionShouldFail == (consumer != null);
         assert constructionShouldFail == (asserts == null);
         ThreadPool threadPool = null;
@@ -143,7 +143,7 @@ public class JvmGcMonitorServiceSettingsTests extends ESTestCase {
                 service.doStart();
                 asserts.run();
                 service.doStop();
-            } catch (Throwable t) {
+            } catch (Exception t) {
                 consumer.accept(t);
             }
         } finally {
diff --git a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmMonitorTests.java b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmMonitorTests.java
index 91862e9cd18..278a47ed21f 100644
--- a/core/src/test/java/org/elasticsearch/monitor/jvm/JvmMonitorTests.java
+++ b/core/src/test/java/org/elasticsearch/monitor/jvm/JvmMonitorTests.java
@@ -48,10 +48,10 @@ public class JvmMonitorTests extends ESTestCase {
         AtomicBoolean invoked = new AtomicBoolean();
         JvmGcMonitorService.JvmMonitor monitor = new JvmGcMonitorService.JvmMonitor(Collections.emptyMap(), IGNORE) {
             @Override
-            void onMonitorFailure(Throwable t) {
+            void onMonitorFailure(Exception e) {
                 invoked.set(true);
-                assertThat(t, instanceOf(RuntimeException.class));
-                assertThat(t, hasToString(containsString("simulated")));
+                assertThat(e, instanceOf(RuntimeException.class));
+                assertThat(e, hasToString(containsString("simulated")));
             }
 
             @Override
@@ -174,7 +174,7 @@ public class JvmMonitorTests extends ESTestCase {
 
         JvmGcMonitorService.JvmMonitor monitor = new JvmGcMonitorService.JvmMonitor(gcThresholds, IGNORE) {
             @Override
-            void onMonitorFailure(Throwable t) {
+            void onMonitorFailure(Exception e) {
             }
 
             @Override
@@ -284,7 +284,7 @@ public class JvmMonitorTests extends ESTestCase {
         final JvmGcMonitorService.JvmMonitor monitor = new JvmGcMonitorService.JvmMonitor(Collections.emptyMap(), IGNORE) {
 
             @Override
-            void onMonitorFailure(Throwable t) {
+            void onMonitorFailure(Exception e) {
             }
 
             @Override
@@ -358,7 +358,7 @@ public class JvmMonitorTests extends ESTestCase {
         final JvmGcMonitorService.JvmMonitor monitor = new JvmGcMonitorService.JvmMonitor(Collections.emptyMap(), gcOverheadThreshold) {
 
             @Override
-            void onMonitorFailure(final Throwable t) {
+            void onMonitorFailure(final Exception e) {
             }
 
             @Override
diff --git a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java
index 6bb1716cb0f..051159b448b 100644
--- a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java
@@ -41,11 +41,16 @@ import static org.hamcrest.Matchers.notNullValue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-/**
- *
- */
 public class BytesRestResponseTests extends ESTestCase {
 
+    class UnknownException extends Exception {
+
+        public UnknownException(final String message, final Throwable cause) {
+            super(message, cause);
+        }
+
+    }
+
     public void testWithHeaders() throws Exception {
         RestRequest request = new FakeRestRequest();
         RestChannel channel = randomBoolean() ? new DetailedExceptionRestChannel(request) : new SimpleExceptionRestChannel(request);
@@ -61,7 +66,7 @@ public class BytesRestResponseTests extends ESTestCase {
         RestRequest request = new FakeRestRequest();
         RestChannel channel = new SimpleExceptionRestChannel(request);
 
-        Throwable t = new ElasticsearchException("an error occurred reading data", new FileNotFoundException("/foo/bar"));
+        Exception t = new ElasticsearchException("an error occurred reading data", new FileNotFoundException("/foo/bar"));
         BytesRestResponse response = new BytesRestResponse(channel, t);
         String text = response.content().utf8ToString();
         assertThat(text, containsString("ElasticsearchException[an error occurred reading data]"));
@@ -74,7 +79,7 @@ public class BytesRestResponseTests extends ESTestCase {
         RestRequest request = new FakeRestRequest();
         RestChannel channel = new DetailedExceptionRestChannel(request);
 
-        Throwable t = new ElasticsearchException("an error occurred reading data", new FileNotFoundException("/foo/bar"));
+        Exception t = new ElasticsearchException("an error occurred reading data", new FileNotFoundException("/foo/bar"));
         BytesRestResponse response = new BytesRestResponse(channel, t);
         String text = response.content().utf8ToString();
         assertThat(text, containsString("{\"type\":\"exception\",\"reason\":\"an error occurred reading data\"}"));
@@ -85,10 +90,10 @@ public class BytesRestResponseTests extends ESTestCase {
         RestRequest request = new FakeRestRequest();
         RestChannel channel = new SimpleExceptionRestChannel(request);
 
-        Throwable t = new Throwable("an error occurred reading data", new FileNotFoundException("/foo/bar"));
+        Exception t = new UnknownException("an error occurred reading data", new FileNotFoundException("/foo/bar"));
         BytesRestResponse response = new BytesRestResponse(channel, t);
         String text = response.content().utf8ToString();
-        assertThat(text, not(containsString("Throwable[an error occurred reading data]")));
+        assertThat(text, not(containsString("UnknownException[an error occurred reading data]")));
         assertThat(text, not(containsString("FileNotFoundException[/foo/bar]")));
         assertThat(text, not(containsString("error_trace")));
         assertThat(text, containsString("\"error\":\"No ElasticsearchException found\""));
@@ -99,10 +104,10 @@ public class BytesRestResponseTests extends ESTestCase {
         request.params().put("error_trace", "true");
         RestChannel channel = new DetailedExceptionRestChannel(request);
 
-        Throwable t = new Throwable("an error occurred reading data", new FileNotFoundException("/foo/bar"));
+        Exception t = new UnknownException("an error occurred reading data", new FileNotFoundException("/foo/bar"));
         BytesRestResponse response = new BytesRestResponse(channel, t);
         String text = response.content().utf8ToString();
-        assertThat(text, containsString("\"type\":\"throwable\",\"reason\":\"an error occurred reading data\""));
+        assertThat(text, containsString("\"type\":\"unknown_exception\",\"reason\":\"an error occurred reading data\""));
         assertThat(text, containsString("{\"type\":\"file_not_found_exception\""));
         assertThat(text, containsString("\"stack_trace\":\"[an error occurred reading data]"));
     }
@@ -111,14 +116,14 @@ public class BytesRestResponseTests extends ESTestCase {
         RestRequest request = new FakeRestRequest();
         RestChannel channel = new DetailedExceptionRestChannel(request);
         {
-            Throwable t = new ElasticsearchException("an error occurred reading data", new FileNotFoundException("/foo/bar"));
-            BytesRestResponse response = new BytesRestResponse(channel, t);
+            Exception e = new ElasticsearchException("an error occurred reading data", new FileNotFoundException("/foo/bar"));
+            BytesRestResponse response = new BytesRestResponse(channel, e);
             String text = response.content().utf8ToString();
             assertThat(text, containsString("{\"root_cause\":[{\"type\":\"exception\",\"reason\":\"an error occurred reading data\"}]"));
         }
         {
-            Throwable t = new FileNotFoundException("/foo/bar");
-            BytesRestResponse response = new BytesRestResponse(channel, t);
+            Exception e = new FileNotFoundException("/foo/bar");
+            BytesRestResponse response = new BytesRestResponse(channel, e);
             String text = response.content().utf8ToString();
             assertThat(text, containsString("{\"root_cause\":[{\"type\":\"file_not_found_exception\",\"reason\":\"/foo/bar\"}]"));
         }
diff --git a/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java b/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java
index 2bb39ad10ea..6542bad5b8a 100644
--- a/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/SearchWithRejectionsIT.java
@@ -65,7 +65,7 @@ public class SearchWithRejectionsIT extends ESIntegTestCase {
         for (int i = 0; i < numSearches; i++) {
             try {
                 responses[i].get();
-            } catch (Throwable t) {
+            } catch (Exception t) {
             }
         }
         awaitBusy(() -> client().admin().indices().prepareStats().execute().actionGet().getTotal().getSearch().getOpenContexts() == 0, 1, TimeUnit.SECONDS);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java
index a48facc4d66..0c8b9a22c37 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FilterIT.java
@@ -52,9 +52,6 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.core.IsNull.notNullValue;
 
-/**
- *
- */
 @ESIntegTestCase.SuiteScopeTestCase
 public class FilterIT extends ESIntegTestCase {
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java
index a95df3ff5e6..592861ccce2 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersIT.java
@@ -58,9 +58,6 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.core.IsNull.notNullValue;
 
-/**
- *
- */
 @ESIntegTestCase.SuiteScopeTestCase
 public class FiltersIT extends ESIntegTestCase {
 
diff --git a/core/src/test/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java b/core/src/test/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java
index 98ae3241dbb..eb6322c151e 100644
--- a/core/src/test/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java
+++ b/core/src/test/java/org/elasticsearch/search/basic/SearchWhileRelocatingIT.java
@@ -98,11 +98,11 @@ public class SearchWhileRelocatingIT extends ESIntegTestCase {
                             if (numberOfReplicas == 1 || !ex.getMessage().contains("all shards failed")) {
                                 thrownExceptions.add(ex);
                             }
-                        } catch (Throwable t) {
+                        } catch (Exception ex) {
                             if (!criticalException) {
-                                nonCriticalExceptions.add(t);
+                                nonCriticalExceptions.add(ex);
                             } else {
-                                thrownExceptions.add(t);
+                                thrownExceptions.add(ex);
                             }
                         }
                     }
diff --git a/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java b/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
index 50fb3f9074b..bbd1bec1d45 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/AbstractSnapshotIntegTestCase.java
@@ -279,8 +279,8 @@ public abstract class AbstractSnapshotIntegTestCase extends ESIntegTestCase {
                 }
 
                 @Override
-                public void onFailure(String source, Throwable t) {
-                    logger.warn("failed to execute [{}]", t, source);
+                public void onFailure(String source, Exception e) {
+                    logger.warn("failed to execute [{}]", e, source);
                 }
             });
 
diff --git a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
index a25c8e0593e..9d571c02c90 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
@@ -272,7 +272,7 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest
             }
 
             @Override
-            public void onFailure(String source, @Nullable Throwable t) {
+            public void onFailure(String source, @Nullable Exception e) {
                 countDownLatch.countDown();
             }
 
diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
index e1efbdfaf81..f6049002852 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -2042,7 +2042,7 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
+            public void onFailure(String source, Exception e) {
                 fail();
             }
 
diff --git a/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java b/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java
index 897fa44b593..20c82e6f518 100644
--- a/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java
+++ b/core/src/test/java/org/elasticsearch/test/geo/RandomShapeGenerator.java
@@ -20,16 +20,9 @@
 package org.elasticsearch.test.geo;
 
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
-import org.locationtech.spatial4j.context.jts.JtsSpatialContext;
-import org.locationtech.spatial4j.distance.DistanceUtils;
-import org.locationtech.spatial4j.exception.InvalidShapeException;
-import org.locationtech.spatial4j.shape.Point;
-import org.locationtech.spatial4j.shape.Rectangle;
-import org.locationtech.spatial4j.shape.impl.Range;
 import com.vividsolutions.jts.algorithm.ConvexHull;
 import com.vividsolutions.jts.geom.Coordinate;
 import com.vividsolutions.jts.geom.Geometry;
-
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.geo.builders.CoordinateCollection;
 import org.elasticsearch.common.geo.builders.CoordinatesBuilder;
@@ -42,6 +35,12 @@ import org.elasticsearch.common.geo.builders.PolygonBuilder;
 import org.elasticsearch.common.geo.builders.ShapeBuilder;
 import org.elasticsearch.search.geo.GeoShapeQueryTests;
 import org.junit.Assert;
+import org.locationtech.spatial4j.context.jts.JtsSpatialContext;
+import org.locationtech.spatial4j.distance.DistanceUtils;
+import org.locationtech.spatial4j.exception.InvalidShapeException;
+import org.locationtech.spatial4j.shape.Point;
+import org.locationtech.spatial4j.shape.Rectangle;
+import org.locationtech.spatial4j.shape.impl.Range;
 
 import java.util.Random;
 
@@ -230,14 +229,10 @@ public class RandomShapeGenerator extends RandomGeoGenerator {
                     // The validate flag will check for these possibilities and bail if an incorrect geometry is created
                     try {
                         pgb.build();
-                    } catch (Throwable e) {
+                    } catch (AssertionError | InvalidShapeException e) {
                         // jts bug may occasionally misinterpret coordinate order causing an unhelpful ('geom' assertion)
                         // or InvalidShapeException
-                        if (e instanceof InvalidShapeException || e instanceof AssertionError) {
-                            return null;
-                        }
-                        // throw any other exception
-                        throw e;
+                        return null;
                     }
                 }
                 return pgb;
diff --git a/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java b/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java
index 28267e9beb7..974929dddf2 100644
--- a/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java
+++ b/core/src/test/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java
@@ -19,8 +19,6 @@
 
 package org.elasticsearch.threadpool;
 
-import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
-import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -36,7 +34,6 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.hamcrest.RegexMatcher;
-import org.elasticsearch.threadpool.ThreadPool.Names;
 import org.elasticsearch.tribe.TribeIT;
 
 import java.io.IOException;
@@ -46,19 +43,11 @@ import java.lang.management.ThreadMXBean;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.BrokenBarrierException;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.Executor;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
 import java.util.regex.Pattern;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.not;
-import static org.hamcrest.Matchers.sameInstance;
 
 /**
  */
@@ -136,9 +125,9 @@ public class SimpleThreadPoolIT extends ESIntegTestCase {
         try {
             new Node(settings);
             fail("The node startup is supposed to fail");
-        } catch(Throwable t) {
+        } catch(Exception e) {
             //all good
-            assertThat(t.getMessage(), containsString("mandatory plugins [non_existing]"));
+            assertThat(e.getMessage(), containsString("mandatory plugins [non_existing]"));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/update/UpdateIT.java b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
index 55834c181b0..c8cacbc36c3 100644
--- a/core/src/test/java/org/elasticsearch/update/UpdateIT.java
+++ b/core/src/test/java/org/elasticsearch/update/UpdateIT.java
@@ -65,11 +65,8 @@ import java.util.concurrent.TimeUnit;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows;
-import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -800,7 +797,7 @@ public class UpdateIT extends ESIntegTestCase {
         final CountDownLatch latch = new CountDownLatch(numberOfThreads);
         final CountDownLatch startLatch = new CountDownLatch(1);
         final int numberOfUpdatesPerThread = scaledRandomIntBetween(100, 500);
-        final List<Throwable> failures = new CopyOnWriteArrayList<>();
+        final List<Exception> failures = new CopyOnWriteArrayList<>();
 
         for (int i = 0; i < numberOfThreads; i++) {
             Runnable r = new Runnable() {
@@ -832,7 +829,7 @@ public class UpdateIT extends ESIntegTestCase {
                         logger.warn("Test was forcefully stopped. Client [{}] may still have outstanding requests.", Thread.currentThread().getName());
                         failures.add(e);
                         Thread.currentThread().interrupt();
-                    } catch (Throwable e) {
+                    } catch (Exception e) {
                         failures.add(e);
                     } finally {
                         latch.countDown();
@@ -900,7 +897,7 @@ public class UpdateIT extends ESIntegTestCase {
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     synchronized (failedMap) {
                         incrementMapValue(id, failedMap);
                     }
@@ -922,7 +919,7 @@ public class UpdateIT extends ESIntegTestCase {
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     synchronized (failedMap) {
                         incrementMapValue(id, failedMap);
                     }
@@ -976,7 +973,7 @@ public class UpdateIT extends ESIntegTestCase {
                             }
                         }
                     }
-                } catch (Throwable e) {
+                } catch (Exception e) {
                     logger.error("Something went wrong", e);
                     failures.add(e);
                 } finally {
diff --git a/core/src/test/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java b/core/src/test/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java
index b2cc794ac6c..e2c572f783a 100644
--- a/core/src/test/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java
+++ b/core/src/test/java/org/elasticsearch/versioning/ConcurrentDocumentOperationIT.java
@@ -53,7 +53,7 @@ public class ConcurrentDocumentOperationIT extends ESIntegTestCase {
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     logger.error("Unexpected exception while indexing", e);
                     failure.set(e);
                     latch.countDown();
diff --git a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
index 105d42c8c86..a83dd93a17e 100644
--- a/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
+++ b/modules/lang-groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java
@@ -155,7 +155,7 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
 
                 GroovyClassLoader groovyClassLoader = new GroovyClassLoader(loader, configuration);
                 return groovyClassLoader.parseClass(codeSource);
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 if (logger.isTraceEnabled()) {
                     logger.trace("Exception compiling Groovy script:", e);
                 }
@@ -293,7 +293,7 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri
                 // NOTE: we truncate the stack because IndyInterface has security issue (needs getClassLoader)
                 // we don't do a security check just as a tradeoff, it cannot really escalate to anything.
                 return AccessController.doPrivileged((PrivilegedAction<Object>) script::run);
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 if (logger.isTraceEnabled()) {
                     logger.trace("failed to run {}", e, compiledScript);
                 }
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java
index 662d4d2f30c..640c00b291d 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MinDocCountTests.java
@@ -22,7 +22,6 @@ package org.elasticsearch.messy.tests;
 import com.carrotsearch.hppc.LongHashSet;
 import com.carrotsearch.hppc.LongSet;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
-
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
@@ -316,7 +315,8 @@ public class MinDocCountTests extends AbstractTermsTestCase {
                     Thread.sleep(60000);
                     logger.debug("1m passed. retrying.");
                     testMinDocCountOnTerms(field, script, order, include, false);
-                } catch (Throwable secondFailure) {
+                } catch (Exception secondFailure) {
+                    secondFailure.addSuppressed(ae);
                     logger.error("exception on retry (will re-throw the original in a sec)", secondFailure);
                 }
                 throw ae;
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java
index 4a7b4350d23..f2eee2bb408 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyScriptTests.java
@@ -99,15 +99,15 @@ public class GroovyScriptTests extends ESIntegTestCase {
 
         try {
             client().prepareSearch("test")
-                    .setQuery(constantScoreQuery(scriptQuery(new Script("assert false", ScriptType.INLINE, "groovy", null)))).get();
+                    .setQuery(constantScoreQuery(scriptQuery(new Script("null.foo", ScriptType.INLINE, "groovy", null)))).get();
             fail("should have thrown an exception");
         } catch (SearchPhaseExecutionException e) {
             assertThat(e.toString() + "should not contained NotSerializableTransportException",
                     e.toString().contains("NotSerializableTransportException"), equalTo(false));
             assertThat(e.toString() + "should have contained ScriptException",
                     e.toString().contains("ScriptException"), equalTo(true));
-            assertThat(e.toString()+ "should have contained an assert error",
-                    e.toString().contains("AssertionError[assert false"), equalTo(true));
+            assertThat(e.toString()+ "should have contained a NullPointerException",
+                    e.toString().contains("NullPointerException[Cannot get property 'foo' on null object]"), equalTo(true));
         }
     }
 
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateResponse.java
index c779757f61b..5ca5078661c 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateResponse.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateResponse.java
@@ -40,21 +40,21 @@ public class MultiSearchTemplateResponse extends ActionResponse implements Itera
      */
     public static class Item implements Streamable {
         private SearchTemplateResponse response;
-        private Throwable throwable;
+        private Exception exception;
 
         Item() {
         }
 
-        public Item(SearchTemplateResponse response, Throwable throwable) {
+        public Item(SearchTemplateResponse response, Exception exception) {
             this.response = response;
-            this.throwable = throwable;
+            this.exception = exception;
         }
 
         /**
          * Is it a failed search?
          */
         public boolean isFailure() {
-            return throwable != null;
+            return exception != null;
         }
 
         /**
@@ -62,7 +62,7 @@ public class MultiSearchTemplateResponse extends ActionResponse implements Itera
          */
         @Nullable
         public String getFailureMessage() {
-            return throwable == null ? null : throwable.getMessage();
+            return exception == null ? null : exception.getMessage();
         }
 
         /**
@@ -85,7 +85,7 @@ public class MultiSearchTemplateResponse extends ActionResponse implements Itera
                 this.response = new SearchTemplateResponse();
                 response.readFrom(in);
             } else {
-                throwable = in.readThrowable();
+                exception = in.readException();
             }
         }
 
@@ -96,12 +96,12 @@ public class MultiSearchTemplateResponse extends ActionResponse implements Itera
                 response.writeTo(out);
             } else {
                 out.writeBoolean(false);
-                out.writeThrowable(throwable);
+                out.writeThrowable(exception);
             }
         }
 
-        public Throwable getFailure() {
-            return throwable;
+        public Exception getFailure() {
+            return exception;
         }
     }
 
@@ -150,7 +150,7 @@ public class MultiSearchTemplateResponse extends ActionResponse implements Itera
         for (Item item : items) {
             builder.startObject();
             if (item.isFailure()) {
-                ElasticsearchException.renderThrowable(builder, params, item.getFailure());
+                ElasticsearchException.renderException(builder, params, item.getFailure());
             } else {
                 item.getResponse().toXContent(builder, params);
             }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportMultiSearchTemplateAction.java
index 1ffb19b5fc4..642fe7648da 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportMultiSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportMultiSearchTemplateAction.java
@@ -61,7 +61,7 @@ public class TransportMultiSearchTemplateAction extends HandledTransportAction<M
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     responses.set(index, new MultiSearchTemplateResponse.Item(null, e));
                     if (counter.decrementAndGet() == 0) {
                         finishHim();
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportSearchTemplateAction.java
index 9e1dab3af00..0c55a63596a 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportSearchTemplateAction.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/TransportSearchTemplateAction.java
@@ -26,7 +26,6 @@ import org.elasticsearch.action.search.TransportSearchAction;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -44,8 +43,6 @@ import org.elasticsearch.search.suggest.Suggesters;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
-import java.io.IOException;
-
 import static java.util.Collections.emptyMap;
 import static org.elasticsearch.script.ScriptContext.Standard.SEARCH;
 
@@ -102,18 +99,18 @@ public class TransportSearchTemplateAction extends HandledTransportAction<Search
                         try {
                             response.setResponse(searchResponse);
                             listener.onResponse(response);
-                        } catch (Throwable t) {
+                        } catch (Exception t) {
                             listener.onFailure(t);
                         }
                     }
 
                     @Override
-                    public void onFailure(Throwable t) {
+                    public void onFailure(Exception t) {
                         listener.onFailure(t);
                     }
                 });
             }
-        } catch (Throwable t) {
+        } catch (Exception t) {
             listener.onFailure(t);
         }
     }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateResponse.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateResponse.java
index c05c0097c90..66b61ff0350 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateResponse.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateResponse.java
@@ -77,7 +77,7 @@ public class MultiPercolateResponse extends ActionResponse implements Iterable<M
         for (MultiPercolateResponse.Item item : items) {
             builder.startObject();
             if (item.isFailure()) {
-                ElasticsearchException.renderThrowable(builder, params, item.getFailure());
+                ElasticsearchException.renderException(builder, params, item.getFailure());
             } else {
                 item.getResponse().toXContent(builder, params);
             }
@@ -113,14 +113,14 @@ public class MultiPercolateResponse extends ActionResponse implements Iterable<M
     public static class Item implements Streamable {
 
         private PercolateResponse response;
-        private Throwable throwable;
+        private Exception exception;
 
         Item(PercolateResponse response) {
             this.response = response;
         }
 
-        Item(Throwable Throwable) {
-            this.throwable = Throwable;
+        Item(Exception exception) {
+            this.exception = exception;
         }
 
         Item() {
@@ -140,7 +140,7 @@ public class MultiPercolateResponse extends ActionResponse implements Iterable<M
          */
         @Nullable
         public String getErrorMessage() {
-            return throwable == null ? null : throwable.getMessage();
+            return exception == null ? null : exception.getMessage();
         }
 
         /**
@@ -148,11 +148,11 @@ public class MultiPercolateResponse extends ActionResponse implements Iterable<M
          * <code>false</code> is returned.
          */
         public boolean isFailure() {
-            return throwable != null;
+            return exception != null;
         }
 
-        public Throwable getFailure() {
-            return throwable;
+        public Exception getFailure() {
+            return exception;
         }
 
         @Override
@@ -161,7 +161,7 @@ public class MultiPercolateResponse extends ActionResponse implements Iterable<M
                 response = new PercolateResponse();
                 response.readFrom(in);
             } else {
-                throwable = in.readThrowable();
+                exception = in.readException();
             }
         }
 
@@ -172,7 +172,7 @@ public class MultiPercolateResponse extends ActionResponse implements Iterable<M
                 response.writeTo(out);
             } else {
                 out.writeBoolean(false);
-                out.writeThrowable(throwable);
+                out.writeThrowable(exception);
             }
         }
     }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java
index 2b9f77ab82a..9968035ec85 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java
@@ -107,7 +107,7 @@ public class TransportMultiPercolateAction extends HandledTransportAction<MultiP
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     listener.onFailure(e);
                 }
             });
@@ -135,7 +135,7 @@ public class TransportMultiPercolateAction extends HandledTransportAction<MultiP
                     }
 
                     @Override
-                    public void onFailure(Throwable e) {
+                    public void onFailure(Exception e) {
                         listener.onFailure(e);
                     }
                 });
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java
index 9e86ae44cdd..beca222b755 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java
@@ -89,7 +89,7 @@ public class TransportPercolateAction extends HandledTransportAction<PercolateRe
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     listener.onFailure(e);
                 }
             });
@@ -117,7 +117,7 @@ public class TransportPercolateAction extends HandledTransportAction<PercolateRe
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 listener.onFailure(e);
             }
         });
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
index 3403a8077b9..c8a791a7b71 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
@@ -138,8 +138,8 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
                         firstSearchRequest.types() == null || firstSearchRequest.types().length == 0 ? ""
                                 : firstSearchRequest.types());
             }
-        } catch (Throwable t) {
-            finishHim(t);
+        } catch (Exception e) {
+            finishHim(e);
             return;
         }
         searchWithRetry(listener -> client.search(firstSearchRequest, listener), (SearchResponse response) -> {
@@ -185,8 +185,8 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
             }
 
             @Override
-            public void onFailure(Throwable t) {
-                finishHim(t);
+            public void onFailure(Exception e) {
+                finishHim(e);
             }
         };
         prepareBulkRequestRunnable = (AbstractRunnable) threadPool.getThreadContext().preserveContext(prepareBulkRequestRunnable);
@@ -250,7 +250,7 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 finishHim(e);
             }
         });
@@ -308,7 +308,7 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
             }
 
             startNextScroll(thisBatchStartTime, response.getItems().length);
-        } catch (Throwable t) {
+        } catch (Exception t) {
             finishHim(t);
         }
     }
@@ -360,7 +360,7 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 finishHim(e);
             }
         });
@@ -371,19 +371,18 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
      *
      * @param failure if non null then the request failed catastrophically with this exception
      */
-    void finishHim(Throwable failure) {
+    void finishHim(Exception failure) {
         finishHim(failure, emptyList(), emptyList(), false);
     }
 
     /**
      * Finish the request.
-     *
-     * @param failure if non null then the request failed catastrophically with this exception
+     *  @param failure if non null then the request failed catastrophically with this exception
      * @param indexingFailures any indexing failures accumulated during the request
      * @param searchFailures any search failures accumulated during the request
      * @param timedOut have any of the sub-requests timed out?
      */
-    void finishHim(Throwable failure, List<Failure> indexingFailures, List<ShardSearchFailure> searchFailures, boolean timedOut) {
+    void finishHim(Exception failure, List<Failure> indexingFailures, List<ShardSearchFailure> searchFailures, boolean timedOut) {
         String scrollId = scroll.get();
         if (Strings.hasLength(scrollId)) {
             /*
@@ -403,7 +402,7 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
                 }
 
                 @Override
-                public void onFailure(Throwable e) {
+                public void onFailure(Exception e) {
                     logger.warn("Failed to clear scroll [{}]", e, scrollId);
                 }
             });
@@ -487,7 +486,7 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
             }
 
             @Override
-            public void onFailure(Throwable e) {
+            public void onFailure(Exception e) {
                 if (ExceptionsHelper.unwrap(e, EsRejectedExecutionException.class) != null) {
                     if (retries.hasNext()) {
                         TimeValue delay = retries.next();
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
index b778210611e..8fe5035c4cb 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkByScrollTask.java
@@ -475,8 +475,8 @@ public class BulkByScrollTask extends CancellableTask {
                 }
 
                 @Override
-                public void onFailure(Throwable t) {
-                    command.onFailure(t);
+                public void onFailure(Exception e) {
+                    command.onFailure(e);
                 }
             });
         }
@@ -546,8 +546,8 @@ public class BulkByScrollTask extends CancellableTask {
         }
 
         @Override
-        public void onFailure(Throwable t) {
-            delegate.onFailure(t);
+        public void onFailure(Exception e) {
+            delegate.onFailure(e);
         }
     }
 }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
index 9b094a9e2d3..61c0f2157b5 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
@@ -805,7 +805,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
             super.doExecute(action, request, listener);
         }
 
-        private Throwable wrappedRejectedException() {
+        private Exception wrappedRejectedException() {
             Exception e = new EsRejectedExecutionException();
             int wraps = randomIntBetween(0, 4);
             for (int i = 0; i < wraps; i++) {
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
index 05699c6f7af..735c3aa5b64 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkByScrollTaskTests.java
@@ -190,8 +190,8 @@ public class BulkByScrollTaskTests extends ESTestCase {
                 }
 
                 @Override
-                public void onFailure(Throwable t) {
-                    errors.add(t);
+                public void onFailure(Exception e) {
+                    errors.add(e);
                 }
             });
 
@@ -271,7 +271,7 @@ public class BulkByScrollTaskTests extends ESTestCase {
                 protected void doRun() throws Exception {
                 }
                 @Override
-                public void onFailure(Throwable t) {
+                public void onFailure(Exception e) {
                     throw new UnsupportedOperationException();
                 }
             });
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java
index faea69b870f..6bbcbd6e643 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWhileModifyingTests.java
@@ -44,7 +44,7 @@ public class UpdateByQueryWhileModifyingTests extends ReindexTestCase {
         AtomicReference<String> value = new AtomicReference<>(randomSimpleString(random()));
         indexRandom(true, client().prepareIndex("test", "test", "test").setSource("test", value.get()));
 
-        AtomicReference<Throwable> failure = new AtomicReference<>();
+        AtomicReference<Exception> failure = new AtomicReference<>();
         AtomicBoolean keepUpdating = new AtomicBoolean(true);
         Thread updater = new Thread(() -> {
             while (keepUpdating.get()) {
@@ -52,8 +52,8 @@ public class UpdateByQueryWhileModifyingTests extends ReindexTestCase {
                     BulkIndexByScrollResponse response = updateByQuery().source("test").refresh(true).abortOnVersionConflict(false).get();
                     assertThat(response, matcher().updated(either(equalTo(0L)).or(equalTo(1L)))
                             .versionConflicts(either(equalTo(0L)).or(equalTo(1L))));
-                } catch (Throwable t) {
-                    failure.set(t);
+                } catch (Exception e) {
+                    failure.set(e);
                 }
             }
         });
diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTokenizerFactory.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTokenizerFactory.java
index e04724ee370..eac3ceebc16 100644
--- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTokenizerFactory.java
+++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/analysis/IcuTokenizerFactory.java
@@ -40,9 +40,6 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.stream.Collectors;
 
-
-/**
- */
 public class IcuTokenizerFactory extends AbstractTokenizerFactory {
 
     private final ICUTokenizerConfig config;
@@ -101,8 +98,8 @@ public class IcuTokenizerFactory extends AbstractTokenizerFactory {
                 };
                 return config;
             }
-        } catch (Throwable t) {
-            throw new ElasticsearchException("failed to load ICU rule files", t);
+        } catch (Exception e) {
+            throw new ElasticsearchException("failed to load ICU rule files", e);
         }
     }
 
diff --git a/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java
index c9dd2263245..be3d737b919 100644
--- a/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java
+++ b/plugins/discovery-gce/src/main/java/org/elasticsearch/discovery/gce/GceUnicastHostsProvider.java
@@ -249,8 +249,8 @@ public class GceUnicastHostsProvider extends AbstractComponent implements Unicas
                 }
 
             }
-        } catch (Throwable e) {
-            logger.warn("Exception caught during discovery: {}", e, e.getMessage());
+        } catch (Exception e) {
+            logger.warn("exception caught during discovery", e);
         }
 
         logger.debug("{} node(s) added", cachedDiscoNodes.size());
diff --git a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
index fb4dc37d641..b06232e1c41 100644
--- a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
+++ b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
@@ -122,7 +122,7 @@ public final class AttachmentProcessor extends AbstractProcessor {
                 String length = Strings.hasLength(contentLength) ? contentLength : String.valueOf(parsedContent.length());
                 additionalFields.put(Property.CONTENT_LENGTH.toLowerCase(), length);
             }
-        } catch (Throwable e) {
+        } catch (Exception e) {
             throw new ElasticsearchParseException("Error parsing document in field [{}]", e, field);
         }
 
diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java
index 0c63f65c247..4b9a40dd8a9 100644
--- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java
+++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/TikaDocTests.java
@@ -58,7 +58,7 @@ public class TikaDocTests extends ESTestCase {
             assertNotNull(parsedContent);
             assertFalse(parsedContent.isEmpty());
             logger.debug("extracted content: {}", parsedContent);
-        } catch (Throwable e) {
+        } catch (Exception e) {
             throw new RuntimeException("parsing of filename: " + fileName.getFileName() + " failed", e);
         }
     }
diff --git a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java
index 8bcc0ca1fc8..b27c3fad2b8 100644
--- a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java
+++ b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java
@@ -52,11 +52,12 @@ public class ExampleCatAction extends AbstractCatAction {
         table.endRow();
         try {
             channel.sendResponse(RestTable.buildResponse(table, channel));
-        } catch (Throwable e) {
+        } catch (Exception e) {
             try {
                 channel.sendResponse(new BytesRestResponse(channel, e));
-            } catch (Throwable e1) {
-                logger.error("failed to send failure response", e1);
+            } catch (Exception inner) {
+                inner.addSuppressed(e);
+                logger.error("failed to send failure response", inner);
             }
         }
     }
diff --git a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptMultiThreadedTests.java b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptMultiThreadedTests.java
index c2f1214e72b..634a4ca6dfa 100644
--- a/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptMultiThreadedTests.java
+++ b/plugins/lang-javascript/src/test/java/org/elasticsearch/script/javascript/JavaScriptScriptMultiThreadedTests.java
@@ -35,9 +35,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.hamcrest.Matchers.equalTo;
 
-/**
- *
- */
 public class JavaScriptScriptMultiThreadedTests extends ESTestCase {
     public void testExecutableNoRuntimeParams() throws Exception {
         final JavaScriptScriptEngineService se = new JavaScriptScriptEngineService(Settings.Builder.EMPTY_SETTINGS);
@@ -64,9 +61,9 @@ public class JavaScriptScriptMultiThreadedTests extends ESTestCase {
                             long result = ((Number) script.run()).longValue();
                             assertThat(result, equalTo(addition));
                         }
-                    } catch (Throwable t) {
+                    } catch (Exception e) {
                         failed.set(true);
-                        logger.error("failed", t);
+                        logger.error("failed", e);
                     } finally {
                         latch.countDown();
                     }
@@ -106,9 +103,9 @@ public class JavaScriptScriptMultiThreadedTests extends ESTestCase {
                             long result = ((Number) script.run()).longValue();
                             assertThat(result, equalTo(addition));
                         }
-                    } catch (Throwable t) {
+                    } catch (Exception e) {
                         failed.set(true);
-                        logger.error("failed", t);
+                        logger.error("failed", e);
                     } finally {
                         latch.countDown();
                     }
@@ -147,9 +144,9 @@ public class JavaScriptScriptMultiThreadedTests extends ESTestCase {
                             long result = ((Number) se.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "testExecutableNoRuntimeParams", "js", compiled), runtimeVars).run()).longValue();
                             assertThat(result, equalTo(addition));
                         }
-                    } catch (Throwable t) {
+                    } catch (Exception e) {
                         failed.set(true);
-                        logger.error("failed", t);
+                        logger.error("failed", e);
                     } finally {
                         latch.countDown();
                     }
diff --git a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTests.java b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTests.java
index abf9f661a6d..0a887bc9a7e 100644
--- a/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTests.java
+++ b/plugins/lang-python/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTests.java
@@ -66,9 +66,9 @@ public class PythonScriptMultiThreadedTests extends ESTestCase {
                             long result = ((Number) script.run()).longValue();
                             assertThat(result, equalTo(addition));
                         }
-                    } catch (Throwable t) {
+                    } catch (Exception e) {
                         failed.set(true);
-                        logger.error("failed", t);
+                        logger.error("failed", e);
                     } finally {
                         latch.countDown();
                     }
@@ -109,9 +109,9 @@ public class PythonScriptMultiThreadedTests extends ESTestCase {
 //                            long result = ((Number) script.run(runtimeVars)).longValue();
 //                            assertThat(result, equalTo(addition));
 //                        }
-//                    } catch (Throwable t) {
+//                    } catch (Exception e) {
 //                        failed.set(true);
-//                        logger.error("failed", t);
+//                        logger.error("failed", e);
 //                    } finally {
 //                        latch.countDown();
 //                    }
@@ -151,9 +151,9 @@ public class PythonScriptMultiThreadedTests extends ESTestCase {
                             long result = ((Number) se.executable(compiledScript, runtimeVars).run()).longValue();
                             assertThat(result, equalTo(addition));
                         }
-                    } catch (Throwable t) {
+                    } catch (Exception e) {
                         failed.set(true);
-                        logger.error("failed", t);
+                        logger.error("failed", e);
                     } finally {
                         latch.countDown();
                     }
diff --git a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java
index 79174e54c62..06e51686823 100644
--- a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java
+++ b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/AttachmentMapper.java
@@ -482,7 +482,7 @@ public class AttachmentMapper extends FieldMapper {
         String parsedContent;
         try {
             parsedContent = TikaImpl.parse(content, metadata, indexedChars);
-        } catch (Throwable e) {
+        } catch (Exception e) {
             // #18: we could ignore errors when Tika does not parse data
             if (!ignoreErrors) {
                 logger.trace("exception caught", e);
@@ -508,8 +508,8 @@ public class AttachmentMapper extends FieldMapper {
                 }
                 context = context.createExternalValueContext(language);
                 languageMapper.parse(context);
-            } catch(Throwable t) {
-                logger.debug("Cannot detect language: [{}]", t.getMessage());
+            } catch(Exception e) {
+                logger.debug("Cannot detect language: [{}]", e.getMessage());
             }
         }
 
diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaDocTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaDocTests.java
index fbbdeb83a7d..b32a6ab79a0 100644
--- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaDocTests.java
+++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/TikaDocTests.java
@@ -58,7 +58,7 @@ public class TikaDocTests extends ESTestCase {
             assertNotNull(parsedContent);
             assertFalse(parsedContent.isEmpty());
             logger.debug("extracted content: {}", parsedContent);
-        } catch (Throwable e) {
+        } catch (Exception e) {
             throw new RuntimeException("parsing of filename: " + fileName.getFileName() + " failed", e);
         }
     }
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java
index 7dc6f3b6a83..ea71dc152f9 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java
@@ -46,9 +46,6 @@ import java.security.PrivilegedActionException;
 import java.security.PrivilegedExceptionAction;
 import java.util.Map;
 
-/**
- *
- */
 public class S3BlobContainer extends AbstractBlobContainer {
 
     protected final S3BlobStore blobStore;
@@ -74,7 +71,7 @@ public class S3BlobContainer extends AbstractBlobContainer {
             });
         } catch (AmazonS3Exception e) {
             return false;
-        } catch (Throwable e) {
+        } catch (Exception e) {
             throw new BlobStoreException("failed to check if blob exists", e);
         }
     }
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
index 93c19df9c04..d1c43f15adb 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
@@ -512,7 +512,7 @@ public abstract  class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase
                     multiObjectDeleteRequest.setKeys(keys);
                     client.deleteObjects(multiObjectDeleteRequest);
                 }
-            } catch (Throwable ex) {
+            } catch (Exception ex) {
                 logger.warn("Failed to delete S3 repository [{}] in [{}]", ex, bucketName, region);
             }
         }
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/SeccompTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/SeccompTests.java
index a319aaabb70..d028dfd573a 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/SeccompTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/SeccompTests.java
@@ -40,7 +40,7 @@ public class SeccompTests extends ESTestCase {
         if (!JNANatives.LOCAL_SECCOMP_ALL) {
             try {
                 Seccomp.init(createTempDir());
-            } catch (Throwable e) {
+            } catch (Exception e) {
                 throw new RuntimeException("unable to forcefully apply seccomp to test thread", e);
             }
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java b/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java
index aa327ae2546..f08f15f236b 100644
--- a/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java
@@ -53,12 +53,12 @@ abstract class ESElasticsearchCliTestCase extends ESTestCase {
             assertThat(status, equalTo(expectedStatus));
             assertThat(init.get(), equalTo(expectedInit));
             outputConsumer.accept(terminal.getOutput());
-        } catch (Throwable t) {
+        } catch (Exception e) {
             // if an unexpected exception is thrown, we log
             // terminal output to aid debugging
             logger.info(terminal.getOutput());
             // rethrow so the test fails
-            throw t;
+            throw e;
         }
     }
 
diff --git a/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java b/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java
index 933f26e6e81..4440fbe117d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java
@@ -49,7 +49,7 @@ public class BackgroundIndexer implements AutoCloseable {
 
     final Thread[] writers;
     final CountDownLatch stopLatch;
-    final CopyOnWriteArrayList<Throwable> failures;
+    final CopyOnWriteArrayList<Exception> failures;
     final AtomicBoolean stop = new AtomicBoolean(false);
     final AtomicLong idGenerator = new AtomicLong();
     final AtomicLong indexCounter = new AtomicLong();
@@ -169,7 +169,7 @@ public class BackgroundIndexer implements AutoCloseable {
                             }
                         }
                         logger.info("**** done indexing thread {}  stop: {} numDocsIndexed: {}", indexerId, stop.get(), indexCounter.get());
-                    } catch (Throwable e) {
+                    } catch (Exception e) {
                         failures.add(e);
                         logger.warn("**** failed indexing thread {} on doc id {}", e, indexerId, id);
                     } finally {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
index fe7ba74a327..a86b55c81db 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
@@ -21,7 +21,6 @@ package org.elasticsearch.test;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterChangedEvent;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.NodeConnectionsService;
@@ -98,8 +97,8 @@ public class ClusterServiceUtils {
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
-                fail("unexpected exception" + t);
+            public void onFailure(String source, Exception e) {
+                fail("unexpected exception" + e);
             }
         });
         try {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
index d13f69a3765..6d07a85e50b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
@@ -955,7 +955,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
                         client().admin().indices().prepareRefresh().get();
                     }
                     lastKnownCount.set(count);
-                } catch (Throwable e) { // count now acts like search and barfs if all shards failed...
+                } catch (Exception e) { // count now acts like search and barfs if all shards failed...
                     logger.debug("failed to executed count", e);
                     return false;
                 }
@@ -1333,7 +1333,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
         }
         final String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
         Collections.shuffle(builders, random());
-        final CopyOnWriteArrayList<Tuple<IndexRequestBuilder, Throwable>> errors = new CopyOnWriteArrayList<>();
+        final CopyOnWriteArrayList<Tuple<IndexRequestBuilder, Exception>> errors = new CopyOnWriteArrayList<>();
         List<CountDownLatch> inFlightAsyncOperations = new ArrayList<>();
         // If you are indexing just a few documents then frequently do it one at a time.  If many then frequently in bulk.
         if (builders.size() < FREQUENT_BULK_THRESHOLD ? frequently() : builders.size() < ALWAYS_BULK_THRESHOLD ? rarely() : false) {
@@ -1366,8 +1366,8 @@ public abstract class ESIntegTestCase extends ESTestCase {
         for (CountDownLatch operation : inFlightAsyncOperations) {
             operation.await();
         }
-        final List<Throwable> actualErrors = new ArrayList<>();
-        for (Tuple<IndexRequestBuilder, Throwable> tuple : errors) {
+        final List<Exception> actualErrors = new ArrayList<>();
+        for (Tuple<IndexRequestBuilder, Exception> tuple : errors) {
             if (ExceptionsHelper.unwrapCause(tuple.v2()) instanceof EsRejectedExecutionException) {
                 tuple.v1().execute().actionGet(); // re-index if rejected
             } else {
@@ -1525,7 +1525,7 @@ public abstract class ESIntegTestCase extends ESTestCase {
         }
 
         @Override
-        public final void onFailure(Throwable t) {
+        public final void onFailure(Exception t) {
             try {
                 logger.info("Action Failed", t);
                 addError(t);
@@ -1534,24 +1534,24 @@ public abstract class ESIntegTestCase extends ESTestCase {
             }
         }
 
-        protected void addError(Throwable t) {
+        protected void addError(Exception e) {
         }
 
     }
 
     private class PayloadLatchedActionListener<Response, T> extends LatchedActionListener<Response> {
-        private final CopyOnWriteArrayList<Tuple<T, Throwable>> errors;
+        private final CopyOnWriteArrayList<Tuple<T, Exception>> errors;
         private final T builder;
 
-        public PayloadLatchedActionListener(T builder, CountDownLatch latch, CopyOnWriteArrayList<Tuple<T, Throwable>> errors) {
+        public PayloadLatchedActionListener(T builder, CountDownLatch latch, CopyOnWriteArrayList<Tuple<T, Exception>> errors) {
             super(latch);
             this.errors = errors;
             this.builder = builder;
         }
 
         @Override
-        protected void addError(Throwable t) {
-            errors.add(new Tuple<>(builder, t));
+        protected void addError(Exception e) {
+            errors.add(new Tuple<>(builder, e));
         }
 
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index 11bad415764..489cad5b085 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -1481,7 +1481,7 @@ public final class InternalTestCluster extends TestCluster {
             Client client = viaNode != null ? client(viaNode) : client();
             ClusterState state = client.admin().cluster().prepareState().execute().actionGet().getState();
             return state.nodes().getMasterNode().getName();
-        } catch (Throwable e) {
+        } catch (Exception e) {
             logger.warn("Can't fetch cluster state", e);
             throw new RuntimeException("Can't get master node " + e.getMessage(), e);
         }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java
index 5525baf4206..d09c763322c 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/VersionUtils.java
@@ -44,7 +44,7 @@ public class VersionUtils {
                     try {
                         Version object = (Version) field.get(null);
                         ids.add(object.id);
-                    } catch (Throwable e) {
+                    } catch (IllegalAccessException e) {
                         throw new RuntimeException(e);
                     }
                 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/client/NoOpClient.java b/test/framework/src/main/java/org/elasticsearch/test/client/NoOpClient.java
index 6ff45608700..1d91b0980e4 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/client/NoOpClient.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/client/NoOpClient.java
@@ -50,8 +50,8 @@ public class NoOpClient extends AbstractClient {
     public void close() {
         try {
             ThreadPool.terminate(threadPool(), 10, TimeUnit.SECONDS);
-        } catch (Throwable t) {
-            throw new ElasticsearchException(t.getMessage(), t);
+        } catch (Exception e) {
+            throw new ElasticsearchException(e.getMessage(), e);
         }
     }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java
index cbcb9766943..956088f0fd1 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/BlockClusterStateProcessing.java
@@ -76,8 +76,8 @@ public class BlockClusterStateProcessing extends SingleNodeDisruption {
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
-                logger.error("unexpected error during disruption", t);
+            public void onFailure(String source, Exception e) {
+                logger.error("unexpected error during disruption", e);
             }
         });
         try {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java
index be0b69a8e8b..f69c0a3085d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/SlowClusterStateProcessing.java
@@ -124,7 +124,7 @@ public class SlowClusterStateProcessing extends SingleNodeDisruption {
             }
 
             @Override
-            public void onFailure(String source, Throwable t) {
+            public void onFailure(String source, Exception e) {
                 countDownLatch.countDown();
             }
         });
diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
index 82f2346c421..988d4b9bb87 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
@@ -95,9 +95,6 @@ import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-/**
- *
- */
 public class ElasticsearchAssertions {
 
     public static void assertAcked(AcknowledgedRequestBuilder<?, ?, ?> builder) {
@@ -555,7 +552,6 @@ public class ElasticsearchAssertions {
             extraInfo += " with status [" + status + "]";
         }
 
-
         try {
             future.actionGet();
             fail = true;
@@ -565,7 +561,7 @@ public class ElasticsearchAssertions {
             if (status != null) {
                 assertThat(extraInfo, ExceptionsHelper.status(esException), equalTo(status));
             }
-        } catch (Throwable e) {
+        } catch (Exception e) {
             assertThat(extraInfo, e, instanceOf(exceptionClass));
             if (status != null) {
                 assertThat(extraInfo, ExceptionsHelper.status(e), equalTo(status));
@@ -597,7 +593,7 @@ public class ElasticsearchAssertions {
         try {
             future.actionGet();
             fail = true;
-        } catch (Throwable e) {
+        } catch (Exception e) {
             assertThat(extraInfo, ExceptionsHelper.status(e), equalTo(status));
         }
         // has to be outside catch clause to get a proper message
@@ -657,35 +653,38 @@ public class ElasticsearchAssertions {
                     equalTo(0));
             assertThat("Serialization failed with version [" + version + "] bytes should be equal for streamable [" + streamable + "]",
                     serialize(version, streamable), equalTo(orig));
-        } catch (Throwable ex) {
+        } catch (Exception ex) {
             throw new RuntimeException("failed to check serialization - version [" + version + "] for streamable [" + streamable + "]", ex);
         }
 
     }
 
-    public static void assertVersionSerializable(Version version, final Throwable t) {
-        ElasticsearchAssertions.assertVersionSerializable(version, new ThrowableWrapper(t));
+    public static void assertVersionSerializable(Version version, final Exception e) {
+        ElasticsearchAssertions.assertVersionSerializable(version, new ExceptionWrapper(e));
     }
 
-    public static final class ThrowableWrapper implements Streamable {
-        Throwable throwable;
-        public ThrowableWrapper(Throwable t) {
-            throwable = t;
+    public static final class ExceptionWrapper implements Streamable {
+
+        private Exception exception;
+
+        public ExceptionWrapper(Exception e) {
+            exception = e;
         }
 
-        public ThrowableWrapper() {
-            throwable = null;
+        public ExceptionWrapper() {
+            exception = null;
         }
 
         @Override
         public void readFrom(StreamInput in) throws IOException {
-            throwable = in.readThrowable();
+            exception = in.readException();
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            out.writeThrowable(throwable);
+            out.writeThrowable(exception);
         }
+
     }
 
 
@@ -697,7 +696,7 @@ public class ElasticsearchAssertions {
             assertThat(constructor, Matchers.notNullValue());
             Streamable newInstance = constructor.newInstance();
             return newInstance;
-        } catch (Throwable e) {
+        } catch (Exception e) {
             return null;
         }
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/spec/RestSpec.java b/test/framework/src/main/java/org/elasticsearch/test/rest/spec/RestSpec.java
index 106ff5176c7..c6ea48fd6ef 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/spec/RestSpec.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/spec/RestSpec.java
@@ -75,7 +75,7 @@ public class RestSpec {
                         }
                         restSpec.addApi(restApi);
                     }
-                } catch (Throwable ex) {
+                } catch (Exception ex) {
                     throw new IOException("Can't parse rest spec file: [" + jsonFile + "]", ex);
                 }
             }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java b/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java
index b0d16d10c49..ec695e8bd41 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/tasks/MockTaskManager.java
@@ -50,8 +50,8 @@ public class MockTaskManager extends TaskManager {
             for (MockTaskManagerListener listener : listeners) {
                 try {
                     listener.onTaskRegistered(task);
-                } catch (Throwable t) {
-                    logger.warn("failed to notify task manager listener about unregistering the task with id {}", t, task.getId());
+                } catch (Exception e) {
+                    logger.warn("failed to notify task manager listener about unregistering the task with id {}", e, task.getId());
                 }
             }
         }
@@ -65,8 +65,8 @@ public class MockTaskManager extends TaskManager {
             for (MockTaskManagerListener listener : listeners) {
                 try {
                     listener.onTaskUnregistered(task);
-                } catch (Throwable t) {
-                    logger.warn("failed to notify task manager listener about unregistering the task with id {}", t, task.getId());
+                } catch (Exception e) {
+                    logger.warn("failed to notify task manager listener about unregistering the task with id {}", e, task.getId());
                 }
             }
         } else {
@@ -80,8 +80,8 @@ public class MockTaskManager extends TaskManager {
         for (MockTaskManagerListener listener : listeners) {
             try {
                 listener.waitForTaskCompletion(task);
-            } catch (Throwable t) {
-                logger.warn("failed to notify task manager listener about waitForTaskCompletion the task with id {}", t, task.getId());
+            } catch (Exception e) {
+                logger.warn("failed to notify task manager listener about waitForTaskCompletion the task with id {}", e, task.getId());
             }
         }
         super.waitForTaskCompletion(task, untilInNanos);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
index 41c27ecbf97..bc371ca02d1 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.test.transport;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.transport.TransportService;
 
@@ -34,7 +33,6 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.common.transport.BoundTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
@@ -52,7 +50,6 @@ import org.elasticsearch.transport.Transport;
 import org.elasticsearch.transport.TransportException;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportRequestOptions;
-import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.TransportServiceAdapter;
 import org.elasticsearch.transport.local.LocalTransport;
 import org.elasticsearch.transport.netty.NettyTransport;
@@ -387,7 +384,7 @@ public class MockTransportService extends TransportService {
 
                 threadPool.schedule(delay, ThreadPool.Names.GENERIC, new AbstractRunnable() {
                     @Override
-                    public void onFailure(Throwable e) {
+                    public void onFailure(Exception e) {
                         logger.debug("failed to send delayed request", e);
                     }
 
@@ -639,10 +636,10 @@ public class MockTransportService extends TransportService {
         }
 
         @Override
-        protected void traceResponseSent(long requestId, String action, Throwable t) {
-            super.traceResponseSent(requestId, action, t);
+        protected void traceResponseSent(long requestId, String action, Exception e) {
+            super.traceResponseSent(requestId, action, e);
             for (Tracer tracer : activeTracers) {
-                tracer.responseSent(requestId, action, t);
+                tracer.responseSent(requestId, action, e);
             }
         }
 

From c02de9227cafbdda69578519a9f07d5ff65ef890 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 1 Jul 2016 15:12:38 -0400
Subject: [PATCH 1221/1311] Migrate remaining calc aggs to NamedWriteable

Once all of these are migrated we'll be able to remove aggregation's
custom "streams" which function that same as NamedWriteable. It also
allows us to make most of the fields on aggregations final which is
rather nice.
---
 .../resources/checkstyle_suppressions.xml     |  1 -
 .../elasticsearch/search/SearchModule.java    | 23 ++---
 .../CardinalityAggregationBuilder.java        |  8 +-
 .../cardinality/InternalCardinality.java      | 58 +++++--------
 .../GeoCentroidAggregationBuilder.java        |  8 +-
 .../geocentroid/InternalGeoCentroid.java      | 84 ++++++++-----------
 .../scripted/InternalScriptedMetric.java      | 77 ++++++-----------
 .../ScriptedMetricAggregationBuilder.java     |  8 +-
 8 files changed, 105 insertions(+), 162 deletions(-)

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 2a361980a8e..6f65428c2bd 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -626,7 +626,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]percentiles[/\\]tdigest[/\\]AbstractTDigestPercentilesAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]percentiles[/\\]tdigest[/\\]TDigestPercentileRanksAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]percentiles[/\\]tdigest[/\\]TDigestPercentilesAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]scripted[/\\]InternalScriptedMetric.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]scripted[/\\]ScriptedMetricAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]stats[/\\]extended[/\\]ExtendedStatsAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]tophits[/\\]TopHitsAggregator.java" checks="LineLength" />
diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java
index ea3348e87fa..764a87589cc 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -429,7 +429,7 @@ public class SearchModule extends AbstractModule {
             namedWriteableRegistry.register(AggregationBuilder.class, spec.aggregationName.getPreferredName(), spec.builderReader);
             aggregationParserRegistry.register(spec.aggregationParser, spec.aggregationName);
         }
-        for (Map.Entry<String, Writeable.Reader<? extends InternalAggregation>> t : spec.internalReaders.entrySet()) {
+        for (Map.Entry<String, Writeable.Reader<? extends InternalAggregation>> t : spec.resultReaders.entrySet()) {
             String writeableName = t.getKey();
             Writeable.Reader<? extends InternalAggregation> internalReader = t.getValue();
             namedWriteableRegistry.register(InternalAggregation.class, writeableName, internalReader);
@@ -437,7 +437,7 @@ public class SearchModule extends AbstractModule {
     }
 
     public static class AggregationSpec {
-        private final Map<String, Writeable.Reader<? extends InternalAggregation>> internalReaders = new TreeMap<>();
+        private final Map<String, Writeable.Reader<? extends InternalAggregation>> resultReaders = new TreeMap<>();
         private final Writeable.Reader<? extends AggregationBuilder> builderReader;
         private final Aggregator.Parser aggregationParser;
         private final ParseField aggregationName;
@@ -468,7 +468,7 @@ public class SearchModule extends AbstractModule {
          * Add a reader for the shard level results of the aggregation.
          */
         public AggregationSpec addResultReader(String writeableName, Writeable.Reader<? extends InternalAggregation> resultReader) {
-            internalReaders.put(writeableName, resultReader);
+            resultReaders.put(writeableName, resultReader);
             return this;
         }
     }
@@ -547,8 +547,8 @@ public class SearchModule extends AbstractModule {
                 PercentileRanksAggregationBuilder.AGGREGATION_NAME_FIELD)
                     .addResultReader(InternalTDigestPercentileRanks.NAME, InternalTDigestPercentileRanks::new)
                     .addResultReader(InternalHDRPercentileRanks.NAME, InternalHDRPercentileRanks::new));
-        registerAggregation(CardinalityAggregationBuilder::new, new CardinalityParser(),
-                CardinalityAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(new AggregationSpec(CardinalityAggregationBuilder::new, new CardinalityParser(),
+                CardinalityAggregationBuilder.AGGREGATION_NAME_FIELD).addResultReader(InternalCardinality::new));
         registerAggregation(GlobalAggregationBuilder::new, GlobalAggregationBuilder::parse,
                 GlobalAggregationBuilder.AGGREGATION_NAME_FIELD);
         registerAggregation(MissingAggregationBuilder::new, new MissingParser(), MissingAggregationBuilder.AGGREGATION_NAME_FIELD);
@@ -580,10 +580,10 @@ public class SearchModule extends AbstractModule {
         registerAggregation(TopHitsAggregationBuilder::new, TopHitsAggregationBuilder::parse,
                 TopHitsAggregationBuilder.AGGREGATION_NAME_FIELD);
         registerAggregation(GeoBoundsAggregationBuilder::new, new GeoBoundsParser(), GeoBoundsAggregationBuilder.AGGREGATION_NAME_FIED);
-        registerAggregation(GeoCentroidAggregationBuilder::new, new GeoCentroidParser(),
-                GeoCentroidAggregationBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(ScriptedMetricAggregationBuilder::new, ScriptedMetricAggregationBuilder::parse,
-                ScriptedMetricAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(new AggregationSpec(GeoCentroidAggregationBuilder::new, new GeoCentroidParser(),
+                GeoCentroidAggregationBuilder.AGGREGATION_NAME_FIELD).addResultReader(InternalGeoCentroid::new));
+        registerAggregation(new AggregationSpec(ScriptedMetricAggregationBuilder::new, ScriptedMetricAggregationBuilder::parse,
+                ScriptedMetricAggregationBuilder.AGGREGATION_NAME_FIELD).addResultReader(InternalScriptedMetric::new));
         registerAggregation(ChildrenAggregationBuilder::new, ChildrenAggregationBuilder::parse,
                 ChildrenAggregationBuilder.AGGREGATION_NAME_FIELD);
 
@@ -766,11 +766,6 @@ public class SearchModule extends AbstractModule {
     }
 
     static {
-        // calcs
-        InternalCardinality.registerStreams();
-        InternalScriptedMetric.registerStreams();
-        InternalGeoCentroid.registerStreams();
-
         // buckets
         InternalGlobal.registerStreams();
         InternalFilter.registerStreams();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregationBuilder.java
index a7850c23475..2d573c48e8e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/CardinalityAggregationBuilder.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
@@ -38,7 +39,8 @@ import java.util.Objects;
 public final class CardinalityAggregationBuilder
     extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource, CardinalityAggregationBuilder> {
 
-    public static final String NAME = InternalCardinality.TYPE.name();
+    public static final String NAME = "cardinality";
+    private static final Type TYPE = new Type(NAME);
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     public static final ParseField PRECISION_THRESHOLD_FIELD = new ParseField("precision_threshold");
@@ -46,14 +48,14 @@ public final class CardinalityAggregationBuilder
     private Long precisionThreshold = null;
 
     public CardinalityAggregationBuilder(String name, ValueType targetValueType) {
-        super(name, InternalCardinality.TYPE, ValuesSourceType.ANY, targetValueType);
+        super(name, TYPE, ValuesSourceType.ANY, targetValueType);
     }
 
     /**
      * Read from a stream.
      */
     public CardinalityAggregationBuilder(StreamInput in) throws IOException {
-        super(in, InternalCardinality.TYPE, ValuesSourceType.ANY);
+        super(in, TYPE, ValuesSourceType.ANY);
         if (in.readBoolean()) {
             precisionThreshold = in.readLong();
         }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinality.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinality.java
index 02fe800adf3..7bfd54eb640 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinality.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/InternalCardinality.java
@@ -34,23 +34,7 @@ import java.util.List;
 import java.util.Map;
 
 public final class InternalCardinality extends InternalNumericMetricsAggregation.SingleValue implements Cardinality {
-
-    public static final Type TYPE = new Type("cardinality");
-
-    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public InternalCardinality readResult(StreamInput in) throws IOException {
-            InternalCardinality result = new InternalCardinality();
-            result.readFrom(in);
-            return result;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
-    private HyperLogLogPlusPlus counts;
+    private final HyperLogLogPlusPlus counts;
 
     InternalCardinality(String name, HyperLogLogPlusPlus counts, List<PipelineAggregator> pipelineAggregators,
             Map<String, Object> metaData) {
@@ -58,26 +42,11 @@ public final class InternalCardinality extends InternalNumericMetricsAggregation
         this.counts = counts;
     }
 
-    private InternalCardinality() {
-    }
-
-    @Override
-    public double value() {
-        return getValue();
-    }
-
-    @Override
-    public long getValue() {
-        return counts == null ? 0 : counts.cardinality(0);
-    }
-
-    @Override
-    public Type type() {
-        return TYPE;
-    }
-
-    @Override
-    protected void doReadFrom(StreamInput in) throws IOException {
+    /**
+     * Read from a stream.
+     */
+    public InternalCardinality(StreamInput in) throws IOException {
+        super(in);
         format = in.readNamedWriteable(DocValueFormat.class);
         if (in.readBoolean()) {
             counts = HyperLogLogPlusPlus.readFrom(in, BigArrays.NON_RECYCLING_INSTANCE);
@@ -97,6 +66,21 @@ public final class InternalCardinality extends InternalNumericMetricsAggregation
         }
     }
 
+    @Override
+    public String getWriteableName() {
+        return CardinalityAggregationBuilder.NAME;
+    }
+
+    @Override
+    public double value() {
+        return getValue();
+    }
+
+    @Override
+    public long getValue() {
+        return counts == null ? 0 : counts.cardinality(0);
+    }
+
     @Override
     public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
         InternalCardinality reduced = null;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregationBuilder.java
index f9bf2e0a346..3e61f34eef6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/GeoCentroidAggregationBuilder.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
@@ -36,18 +37,19 @@ import java.io.IOException;
 
 public class GeoCentroidAggregationBuilder
         extends ValuesSourceAggregationBuilder.LeafOnly<ValuesSource.GeoPoint, GeoCentroidAggregationBuilder> {
-    public static final String NAME = InternalGeoCentroid.TYPE.name();
+    public static final String NAME = "geo_centroid";
+    public static final Type TYPE = new Type(NAME);
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     public GeoCentroidAggregationBuilder(String name) {
-        super(name, InternalGeoCentroid.TYPE, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
+        super(name, TYPE, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
     }
 
     /**
      * Read from a stream.
      */
     public GeoCentroidAggregationBuilder(StreamInput in) throws IOException {
-        super(in, InternalGeoCentroid.TYPE, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
+        super(in, TYPE, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java
index 6b15aab8142..06d9d369029 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/geocentroid/InternalGeoCentroid.java
@@ -24,7 +24,6 @@ import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.metrics.InternalMetricsAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -34,29 +33,11 @@ import java.util.List;
 import java.util.Map;
 
 /**
- * Serialization and merge logic for {@link org.elasticsearch.search.aggregations.metrics.geocentroid.GeoCentroidAggregator}
+ * Serialization and merge logic for {@link GeoCentroidAggregator}.
  */
 public class InternalGeoCentroid extends InternalMetricsAggregation implements GeoCentroid {
-
-    public static final Type TYPE = new Type("geo_centroid");
-    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public InternalGeoCentroid readResult(StreamInput in) throws IOException {
-            InternalGeoCentroid result = new InternalGeoCentroid();
-            result.readFrom(in);
-            return result;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
-    protected GeoPoint centroid;
-    protected long count;
-
-    protected InternalGeoCentroid() {
-    }
+    protected final GeoPoint centroid;
+    protected final long count;
 
     public InternalGeoCentroid(String name, GeoPoint centroid, long count, List<PipelineAggregator>
             pipelineAggregators, Map<String, Object> metaData) {
@@ -67,6 +48,37 @@ public class InternalGeoCentroid extends InternalMetricsAggregation implements G
         this.count = count;
     }
 
+    /**
+     * Read from a stream.
+     */
+    public InternalGeoCentroid(StreamInput in) throws IOException {
+        super(in);
+        count = in.readVLong();
+        if (in.readBoolean()) {
+            final long hash = in.readLong();
+            centroid = new GeoPoint(GeoPointField.decodeLatitude(hash), GeoPointField.decodeLongitude(hash));
+        } else {
+            centroid = null;
+        }
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeVLong(count);
+        if (centroid != null) {
+            out.writeBoolean(true);
+            // should we just write lat and lon separately?
+            out.writeLong(GeoPointField.encodeLatLon(centroid.lat(), centroid.lon()));
+        } else {
+            out.writeBoolean(false);
+        }
+    }
+
+    @Override
+    public String getWriteableName() {
+        return GeoCentroidAggregationBuilder.NAME;
+    }
+
     @Override
     public GeoPoint centroid() {
         return centroid;
@@ -77,11 +89,6 @@ public class InternalGeoCentroid extends InternalMetricsAggregation implements G
         return count;
     }
 
-    @Override
-    public Type type() {
-        return TYPE;
-    }
-
     @Override
     public InternalGeoCentroid doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
         double lonSum = Double.NaN;
@@ -125,29 +132,6 @@ public class InternalGeoCentroid extends InternalMetricsAggregation implements G
         }
     }
 
-    @Override
-    protected void doReadFrom(StreamInput in) throws IOException {
-        count = in.readVLong();
-        if (in.readBoolean()) {
-            final long hash = in.readLong();
-            centroid = new GeoPoint(GeoPointField.decodeLatitude(hash), GeoPointField.decodeLongitude(hash));
-        } else {
-            centroid = null;
-        }
-    }
-
-    @Override
-    protected void doWriteTo(StreamOutput out) throws IOException {
-        out.writeVLong(count);
-        if (centroid != null) {
-            out.writeBoolean(true);
-            // should we just write lat and lon separately?
-            out.writeLong(GeoPointField.encodeLatLon(centroid.lat(), centroid.lon()));
-        } else {
-            out.writeBoolean(false);
-        }
-    }
-
     static class Fields {
         public static final String CENTROID = "location";
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java
index 3bcbc0cf845..c5704d9f2b6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/InternalScriptedMetric.java
@@ -26,7 +26,6 @@ import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.metrics.InternalMetricsAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -39,38 +38,36 @@ import java.util.List;
 import java.util.Map;
 
 public class InternalScriptedMetric extends InternalMetricsAggregation implements ScriptedMetric {
+    private final Script reduceScript;
+    private final Object aggregation;
 
-    public static final Type TYPE = new Type("scripted_metric");
-
-    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public InternalScriptedMetric readResult(StreamInput in) throws IOException {
-            InternalScriptedMetric result = new InternalScriptedMetric();
-            result.readFrom(in);
-            return result;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
-    private Script reduceScript;
-    private Object aggregation;
-
-    private InternalScriptedMetric() {
-    }
-
-    private InternalScriptedMetric(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
+    public InternalScriptedMetric(String name, Object aggregation, Script reduceScript, List<PipelineAggregator> pipelineAggregators,
+            Map<String, Object> metaData) {
         super(name, pipelineAggregators, metaData);
-    }
-
-    public InternalScriptedMetric(String name, Object aggregation, Script reduceScript, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
-        this(name, pipelineAggregators, metaData);
         this.aggregation = aggregation;
         this.reduceScript = reduceScript;
     }
 
+    /**
+     * Read from a stream.
+     */
+    public InternalScriptedMetric(StreamInput in) throws IOException {
+        super(in);
+        reduceScript = in.readOptionalWriteable(Script::new);
+        aggregation = in.readGenericValue();
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeOptionalWriteable(reduceScript);
+        out.writeGenericValue(aggregation);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return ScriptedMetricAggregationBuilder.NAME;
+    }
+
     @Override
     public Object aggregation() {
         return aggregation;
@@ -98,15 +95,11 @@ public class InternalScriptedMetric extends InternalMetricsAggregation implement
         } else {
             aggregation = aggregationObjects;
         }
-        return new InternalScriptedMetric(firstAggregation.getName(), aggregation, firstAggregation.reduceScript, pipelineAggregators(), getMetaData());
+        return new InternalScriptedMetric(firstAggregation.getName(), aggregation, firstAggregation.reduceScript, pipelineAggregators(),
+                getMetaData());
 
     }
 
-    @Override
-    public Type type() {
-        return TYPE;
-    }
-
     @Override
     public Object getProperty(List<String> path) {
         if (path.isEmpty()) {
@@ -118,24 +111,6 @@ public class InternalScriptedMetric extends InternalMetricsAggregation implement
         }
     }
 
-    @Override
-    protected void doReadFrom(StreamInput in) throws IOException {
-        if (in.readBoolean()) {
-            reduceScript = new Script(in);
-        }
-        aggregation = in.readGenericValue();
-    }
-
-    @Override
-    protected void doWriteTo(StreamOutput out) throws IOException {
-        boolean hasScript = reduceScript != null;
-        out.writeBoolean(hasScript);
-        if (hasScript) {
-            reduceScript.writeTo(out);
-        }
-        out.writeGenericValue(aggregation);
-    }
-
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         return builder.field("value", aggregation);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java
index 5a5f9894f33..cfd50ce9fbd 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregationBuilder.java
@@ -31,6 +31,7 @@ import org.elasticsearch.script.ScriptParameterParser;
 import org.elasticsearch.script.ScriptParameterParser.ScriptParameterValue;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 
@@ -42,7 +43,8 @@ import java.util.Set;
 
 public class ScriptedMetricAggregationBuilder extends AbstractAggregationBuilder<ScriptedMetricAggregationBuilder> {
 
-    public static final String NAME = InternalScriptedMetric.TYPE.name();
+    public static final String NAME = "scripted_metric";
+    private static final Type TYPE = new Type(NAME);
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     private static final ParseField INIT_SCRIPT_FIELD = new ParseField("init_script");
@@ -59,14 +61,14 @@ public class ScriptedMetricAggregationBuilder extends AbstractAggregationBuilder
     private Map<String, Object> params;
 
     public ScriptedMetricAggregationBuilder(String name) {
-        super(name, InternalScriptedMetric.TYPE);
+        super(name, TYPE);
     }
 
     /**
      * Read from a stream.
      */
     public ScriptedMetricAggregationBuilder(StreamInput in) throws IOException {
-        super(in, InternalScriptedMetric.TYPE);
+        super(in, TYPE);
         initScript = in.readOptionalWriteable(Script::new);
         mapScript = in.readOptionalWriteable(Script::new);
         combineScript = in.readOptionalWriteable(Script::new);

From a728c4bb5c5e963310e8c73b47239a627640c764 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Fri, 1 Jul 2016 15:58:24 -0400
Subject: [PATCH 1222/1311] Migrate global, filter, and filters aggs to
 NamedWriteable

Once all of these are migrated we'll be able to remove aggregation's
custom "streams" which function that same as NamedWriteable. It also
allows us to make most of the fields on aggregations final which is
rather nice.

Also starts to migrate MultiBucketAggregation.Bucket to Writeable,
allowing the buckets to have immutable parts.
---
 .../elasticsearch/search/SearchModule.java    |  15 +-
 .../InternalMultiBucketAggregation.java       |   9 ++
 .../InternalSingleBucketAggregation.java      |  33 +++--
 .../bucket/MultiBucketsAggregation.java       |  14 +-
 .../filter/FilterAggregationBuilder.java      |   8 +-
 .../bucket/filter/InternalFilter.java         |  33 ++---
 .../filters/FiltersAggregationBuilder.java    |  10 +-
 .../bucket/filters/InternalFilters.java       | 134 ++++++------------
 .../global/GlobalAggregationBuilder.java      |   8 +-
 .../bucket/global/InternalGlobal.java         |  30 ++--
 10 files changed, 127 insertions(+), 167 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java
index 764a87589cc..369750a214b 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -549,13 +549,13 @@ public class SearchModule extends AbstractModule {
                     .addResultReader(InternalHDRPercentileRanks.NAME, InternalHDRPercentileRanks::new));
         registerAggregation(new AggregationSpec(CardinalityAggregationBuilder::new, new CardinalityParser(),
                 CardinalityAggregationBuilder.AGGREGATION_NAME_FIELD).addResultReader(InternalCardinality::new));
-        registerAggregation(GlobalAggregationBuilder::new, GlobalAggregationBuilder::parse,
-                GlobalAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(new AggregationSpec(GlobalAggregationBuilder::new, GlobalAggregationBuilder::parse,
+                GlobalAggregationBuilder.AGGREGATION_NAME_FIELD).addResultReader(InternalGlobal::new));
         registerAggregation(MissingAggregationBuilder::new, new MissingParser(), MissingAggregationBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(FilterAggregationBuilder::new, FilterAggregationBuilder::parse,
-                FilterAggregationBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(FiltersAggregationBuilder::new, FiltersAggregationBuilder::parse,
-                FiltersAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(new AggregationSpec(FilterAggregationBuilder::new, FilterAggregationBuilder::parse,
+                FilterAggregationBuilder.AGGREGATION_NAME_FIELD).addResultReader(InternalFilter::new));
+        registerAggregation(new AggregationSpec(FiltersAggregationBuilder::new, FiltersAggregationBuilder::parse,
+                FiltersAggregationBuilder.AGGREGATION_NAME_FIELD).addResultReader(InternalFilters::new));
         registerAggregation(SamplerAggregationBuilder::new, SamplerAggregationBuilder::parse,
                 SamplerAggregationBuilder.AGGREGATION_NAME_FIELD);
         registerAggregation(DiversifiedAggregationBuilder::new, new DiversifiedSamplerParser(),
@@ -767,9 +767,6 @@ public class SearchModule extends AbstractModule {
 
     static {
         // buckets
-        InternalGlobal.registerStreams();
-        InternalFilter.registerStreams();
-        InternalFilters.registerStream();
         InternalSampler.registerStreams();
         UnmappedSampler.registerStreams();
         InternalMissing.registerStreams();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
index c579f349b61..46867d24f43 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/InternalMultiBucketAggregation.java
@@ -19,9 +19,11 @@
 
 package org.elasticsearch.search.aggregations;
 
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
@@ -35,6 +37,13 @@ public abstract class InternalMultiBucketAggregation<A extends InternalMultiBuck
         super(name, pipelineAggregators, metaData);
     }
 
+    /**
+     * Read from a stream.
+     */
+    protected InternalMultiBucketAggregation(StreamInput in) throws IOException {
+        super(in);
+    }
+
     /**
      * Create a new copy of this {@link Aggregation} with the same settings as
      * this {@link Aggregation} and contains the provided buckets.
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java
index 303d6c784e3..9b88ab47053 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/InternalSingleBucketAggregation.java
@@ -54,6 +54,27 @@ public abstract class InternalSingleBucketAggregation extends InternalAggregatio
         this.aggregations = aggregations;
     }
 
+    /**
+     * Read from a stream.
+     */
+    protected InternalSingleBucketAggregation(StreamInput in) throws IOException {
+        super(in);
+        docCount = in.readVLong();
+        aggregations = InternalAggregations.readAggregations(in);
+    }
+
+    @Override
+    protected void doReadFrom(StreamInput in) throws IOException {
+        docCount = in.readVLong();
+        aggregations = InternalAggregations.readAggregations(in);
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeVLong(docCount);
+        aggregations.writeTo(out);
+    }
+
     @Override
     public long getDocCount() {
         return docCount;
@@ -114,18 +135,6 @@ public abstract class InternalSingleBucketAggregation extends InternalAggregatio
         }
     }
 
-    @Override
-    protected void doReadFrom(StreamInput in) throws IOException {
-        docCount = in.readVLong();
-        aggregations = InternalAggregations.readAggregations(in);
-    }
-
-    @Override
-    protected void doWriteTo(StreamOutput out) throws IOException {
-        out.writeVLong(docCount);
-        aggregations.writeTo(out);
-    }
-
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         builder.field(CommonFields.DOC_COUNT, docCount);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/MultiBucketsAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/MultiBucketsAggregation.java
index 19a62b8ac4a..39603c9ab45 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/MultiBucketsAggregation.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/MultiBucketsAggregation.java
@@ -19,7 +19,9 @@
 
 package org.elasticsearch.search.aggregations.bucket;
 
+import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.util.Comparators;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.search.aggregations.Aggregation;
@@ -27,19 +29,19 @@ import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.HasAggregations;
 import org.elasticsearch.search.aggregations.support.AggregationPath;
 
+import java.io.IOException;
 import java.util.List;
 
 /**
  * An aggregation that returns multiple buckets
  */
 public interface MultiBucketsAggregation extends Aggregation {
-
-
     /**
      * A bucket represents a criteria to which all documents that fall in it adhere to. It is also uniquely identified
      * by a key, and can potentially hold sub-aggregations computed over all documents in it.
      */
-    public interface Bucket extends HasAggregations, ToXContent, Streamable {
+    public interface Bucket extends HasAggregations, ToXContent, Streamable, Writeable {
+        // NORELEASE remove Streamable
 
         /**
          * @return The key associated with the bucket
@@ -64,6 +66,12 @@ public interface MultiBucketsAggregation extends Aggregation {
 
         Object getProperty(String containingAggName, List<String> path);
 
+        @Override
+        default void readFrom(StreamInput in) throws IOException {
+            // NORELEASE remove me when no Buckets override it
+            throw new UnsupportedOperationException("Prefer the Writeable interface");
+        }
+
         static class SubAggregationComparator<B extends Bucket> implements java.util.Comparator<B> {
 
             private final AggregationPath path;
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
index 4026f846e6d..40334fc75eb 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java
@@ -29,13 +29,15 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 
 import java.io.IOException;
 import java.util.Objects;
 
 public class FilterAggregationBuilder extends AbstractAggregationBuilder<FilterAggregationBuilder> {
-    public static final String NAME = InternalFilter.TYPE.name();
+    public static final String NAME = "filter";
+    private static final Type TYPE = new Type(NAME);
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     private final QueryBuilder filter;
@@ -49,7 +51,7 @@ public class FilterAggregationBuilder extends AbstractAggregationBuilder<FilterA
      *            {@link Filter} aggregation.
      */
     public FilterAggregationBuilder(String name, QueryBuilder filter) {
-        super(name, InternalFilter.TYPE);
+        super(name, TYPE);
         if (filter == null) {
             throw new IllegalArgumentException("[filter] must not be null: [" + name + "]");
         }
@@ -60,7 +62,7 @@ public class FilterAggregationBuilder extends AbstractAggregationBuilder<FilterA
      * Read from a stream.
      */
     public FilterAggregationBuilder(StreamInput in) throws IOException {
-        super(in, InternalFilter.TYPE);
+        super(in, TYPE);
         filter = in.readNamedWriteable(QueryBuilder.class);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/InternalFilter.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/InternalFilter.java
index b7896b4eb17..09f61142324 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/InternalFilter.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/InternalFilter.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.search.aggregations.bucket.filter;
 
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -28,35 +27,21 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
-/**
-*
-*/
 public class InternalFilter extends InternalSingleBucketAggregation implements Filter {
-
-    public static final Type TYPE = new Type("filter");
-
-    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public InternalFilter readResult(StreamInput in) throws IOException {
-            InternalFilter result = new InternalFilter();
-            result.readFrom(in);
-            return result;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
-    InternalFilter() {} // for serialization
-
     InternalFilter(String name, long docCount, InternalAggregations subAggregations, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
         super(name, docCount, subAggregations, pipelineAggregators, metaData);
     }
 
+    /**
+     * Stream from a stream.
+     */
+    public InternalFilter(StreamInput in) throws IOException {
+        super(in);
+    }
+
     @Override
-    public Type type() {
-        return TYPE;
+    public String getWriteableName() {
+        return FilterAggregationBuilder.NAME;
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregationBuilder.java
index 9b2b9cee7ff..018963a2ca7 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/FiltersAggregationBuilder.java
@@ -29,6 +29,7 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.bucket.filters.FiltersAggregator.KeyedFilter;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
@@ -43,7 +44,8 @@ import java.util.Objects;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 
 public class FiltersAggregationBuilder extends AbstractAggregationBuilder<FiltersAggregationBuilder> {
-    public static final String NAME = InternalFilters.TYPE.name();
+    public static final String NAME = "filters";
+    private static final Type TYPE = new Type(NAME);
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     private static final ParseField FILTERS_FIELD = new ParseField("filters");
@@ -66,7 +68,7 @@ public class FiltersAggregationBuilder extends AbstractAggregationBuilder<Filter
     }
 
     private FiltersAggregationBuilder(String name, List<KeyedFilter> filters) {
-        super(name, InternalFilters.TYPE);
+        super(name, TYPE);
         // internally we want to have a fixed order of filters, regardless of the order of the filters in the request
         this.filters = new ArrayList<>(filters);
         Collections.sort(this.filters, (KeyedFilter kf1, KeyedFilter kf2) -> kf1.key().compareTo(kf2.key()));
@@ -80,7 +82,7 @@ public class FiltersAggregationBuilder extends AbstractAggregationBuilder<Filter
      *            the filters to use with this aggregation
      */
     public FiltersAggregationBuilder(String name, QueryBuilder... filters) {
-        super(name, InternalFilters.TYPE);
+        super(name, TYPE);
         List<KeyedFilter> keyedFilters = new ArrayList<>(filters.length);
         for (int i = 0; i < filters.length; i++) {
             keyedFilters.add(new KeyedFilter(String.valueOf(i), filters[i]));
@@ -93,7 +95,7 @@ public class FiltersAggregationBuilder extends AbstractAggregationBuilder<Filter
      * Read from a stream.
      */
     public FiltersAggregationBuilder(StreamInput in) throws IOException {
-        super(in, InternalFilters.TYPE);
+        super(in, TYPE);
         keyed = in.readBoolean();
         int filtersSize = in.readVInt();
         filters = new ArrayList<>(filtersSize);
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/InternalFilters.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/InternalFilters.java
index 4cace0a50d0..bd33f1608bc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/InternalFilters.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/InternalFilters.java
@@ -22,13 +22,10 @@ package org.elasticsearch.search.aggregations.bucket.filters;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
-import org.elasticsearch.search.aggregations.bucket.BucketStreamContext;
-import org.elasticsearch.search.aggregations.bucket.BucketStreams;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
@@ -37,55 +34,14 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-/**
- *
- */
 public class InternalFilters extends InternalMultiBucketAggregation<InternalFilters, InternalFilters.InternalBucket> implements Filters {
-
-    public static final Type TYPE = new Type("filters");
-
-    private static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public InternalFilters readResult(StreamInput in) throws IOException {
-            InternalFilters filters = new InternalFilters();
-            filters.readFrom(in);
-            return filters;
-        }
-    };
-
-    private static final BucketStreams.Stream<InternalBucket> BUCKET_STREAM = new BucketStreams.Stream<InternalBucket>() {
-        @Override
-        public InternalBucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
-            InternalBucket filters = new InternalBucket(context.keyed());
-            filters.readFrom(in);
-            return filters;
-        }
-
-        @Override
-        public BucketStreamContext getBucketStreamContext(InternalBucket bucket) {
-            BucketStreamContext context = new BucketStreamContext();
-            context.keyed(bucket.keyed);
-            return context;
-        }
-    };
-
-    public static void registerStream() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-        BucketStreams.registerStream(BUCKET_STREAM, TYPE.stream());
-    }
-
     public static class InternalBucket extends InternalMultiBucketAggregation.InternalBucket implements Filters.Bucket {
 
         private final boolean keyed;
-        private String key;
+        private final String key;
         private long docCount;
         InternalAggregations aggregations;
 
-        private InternalBucket(boolean keyed) {
-            // for serialization
-            this.keyed = keyed;
-        }
-
         public InternalBucket(String key, long docCount, InternalAggregations aggregations, boolean keyed) {
             this.key = key;
             this.docCount = docCount;
@@ -93,6 +49,23 @@ public class InternalFilters extends InternalMultiBucketAggregation<InternalFilt
             this.keyed = keyed;
         }
 
+        /**
+         * Read from a stream.
+         */
+        public InternalBucket(StreamInput in, boolean keyed) throws IOException {
+            this.keyed = keyed;
+            key = in.readOptionalString();
+            docCount = in.readVLong();
+            aggregations = InternalAggregations.readAggregations(in);
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeOptionalString(key);
+            out.writeVLong(docCount);
+            aggregations.writeTo(out);
+        }
+
         @Override
         public String getKey() {
             return key;
@@ -140,27 +113,11 @@ public class InternalFilters extends InternalMultiBucketAggregation<InternalFilt
             builder.endObject();
             return builder;
         }
-
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            key = in.readOptionalString();
-            docCount = in.readVLong();
-            aggregations = InternalAggregations.readAggregations(in);
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeOptionalString(key);
-            out.writeVLong(docCount);
-            aggregations.writeTo(out);
-        }
     }
 
-    private List<InternalBucket> buckets;
+    private final List<InternalBucket> buckets;
+    private final boolean keyed;
     private Map<String, InternalBucket> bucketMap;
-    private boolean keyed;
-
-    public InternalFilters() {} // for serialization
 
     public InternalFilters(String name, List<InternalBucket> buckets, boolean keyed, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
         super(name, pipelineAggregators, metaData);
@@ -168,9 +125,33 @@ public class InternalFilters extends InternalMultiBucketAggregation<InternalFilt
         this.keyed = keyed;
     }
 
+    /**
+     * Read from a stream.
+     */
+    public InternalFilters(StreamInput in) throws IOException {
+        super(in);
+        keyed = in.readBoolean();
+        int size = in.readVInt();
+        List<InternalBucket> buckets = new ArrayList<>(size);
+        for (int i = 0; i < size; i++) {
+            buckets.add(new InternalBucket(in, keyed));
+        }
+        this.buckets = buckets;
+        this.bucketMap = null;
+    }
+
     @Override
-    public Type type() {
-        return TYPE;
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        out.writeBoolean(keyed);
+        out.writeVInt(buckets.size());
+        for (InternalBucket bucket : buckets) {
+            bucket.writeTo(out);
+        }
+    }
+
+    @Override
+    public String getWriteableName() {
+        return FiltersAggregationBuilder.NAME;
     }
 
     @Override
@@ -226,29 +207,6 @@ public class InternalFilters extends InternalMultiBucketAggregation<InternalFilt
         return reduced;
     }
 
-    @Override
-    protected void doReadFrom(StreamInput in) throws IOException {
-        keyed = in.readBoolean();
-        int size = in.readVInt();
-        List<InternalBucket> buckets = new ArrayList<>(size);
-        for (int i = 0; i < size; i++) {
-            InternalBucket bucket = new InternalBucket(keyed);
-            bucket.readFrom(in);
-            buckets.add(bucket);
-        }
-        this.buckets = buckets;
-        this.bucketMap = null;
-    }
-
-    @Override
-    protected void doWriteTo(StreamOutput out) throws IOException {
-        out.writeBoolean(keyed);
-        out.writeVInt(buckets.size());
-        for (InternalBucket bucket : buckets) {
-            bucket.writeTo(out);
-        }
-    }
-
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
         if (keyed) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java
index 2772f6aa880..189eb9bebfc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/GlobalAggregationBuilder.java
@@ -26,24 +26,26 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 
 import java.io.IOException;
 
 public class GlobalAggregationBuilder extends AbstractAggregationBuilder<GlobalAggregationBuilder> {
-    public static final String NAME = InternalGlobal.TYPE.name();
+    public static final String NAME = "global";
+    private static final Type TYPE = new Type(NAME);
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     public GlobalAggregationBuilder(String name) {
-        super(name, InternalGlobal.TYPE);
+        super(name, TYPE);
     }
 
     /**
      * Read from a stream.
      */
     public GlobalAggregationBuilder(StreamInput in) throws IOException {
-        super(in, InternalGlobal.TYPE);
+        super(in, TYPE);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/InternalGlobal.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/InternalGlobal.java
index e8d4d6fd849..f278e5e72b1 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/InternalGlobal.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/global/InternalGlobal.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.search.aggregations.bucket.global;
 
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -33,31 +32,20 @@ import java.util.Map;
  * regardless the query.
  */
 public class InternalGlobal extends InternalSingleBucketAggregation implements Global {
-
-    public static final Type TYPE = new Type("global");
-
-    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public InternalGlobal readResult(StreamInput in) throws IOException {
-            InternalGlobal result = new InternalGlobal();
-            result.readFrom(in);
-            return result;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
-    InternalGlobal() {} // for serialization
-
     InternalGlobal(String name, long docCount, InternalAggregations aggregations, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
         super(name, docCount, aggregations, pipelineAggregators, metaData);
     }
 
+    /**
+     * Read from a stream.
+     */
+    public InternalGlobal(StreamInput in) throws IOException {
+        super(in);
+    }
+
     @Override
-    public Type type() {
-        return TYPE;
+    public String getWriteableName() {
+        return GlobalAggregationBuilder.NAME;
     }
 
     @Override

From 0e7faf1005f13dd36aea5279e772de3eaa8890cd Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Mon, 4 Jul 2016 09:51:28 +0200
Subject: [PATCH 1223/1311] Enable Checkstyle RedundantModifier

---
 buildSrc/src/main/resources/checkstyle.xml    |  2 +
 .../search/SearchPhaseExecutionException.java |  2 +-
 .../org/elasticsearch/bootstrap/Seccomp.java  |  8 +--
 .../cluster/InternalClusterInfoService.java   |  2 +-
 .../common/bytes/BytesReference.java          |  4 +-
 .../common/collect/ImmutableOpenIntMap.java   |  4 +-
 .../common/collect/ImmutableOpenMap.java      |  4 +-
 .../common/geo/GeoHashUtils.java              |  2 +-
 .../elasticsearch/common/geo/GeoPoint.java    | 12 ++---
 .../common/geo/builders/PolygonBuilder.java   |  4 +-
 .../common/lucene/all/AllTermQuery.java       |  4 +-
 .../common/lucene/all/AllTokenStream.java     |  2 +-
 .../settings/AbstractScopedSettings.java      |  2 +-
 .../common/settings/IndexScopedSettings.java  |  2 +-
 .../common/settings/Setting.java              |  2 +-
 .../loader/PropertiesSettingsLoader.java      |  2 +-
 .../common/util/ExtensionPoint.java           |  6 +--
 .../zen/fd/MasterFaultDetection.java          |  2 +-
 .../elasticsearch/env/NodeEnvironment.java    |  2 +-
 .../gateway/LocalAllocateDangledIndices.java  |  2 +-
 .../index/analysis/CharMatcher.java           |  4 +-
 .../analysis/MultiTermAwareComponent.java     |  2 +-
 .../index/fielddata/FieldData.java            |  2 +-
 .../index/fielddata/ordinals/Ordinals.java    |  5 +-
 .../plain/AbstractIndexFieldData.java         |  6 +--
 .../index/mapper/DocumentParser.java          |  2 +-
 .../index/mapper/MappedFieldType.java         |  2 +-
 .../index/mapper/MetadataFieldMapper.java     |  2 +-
 .../mapper/geo/GeoPointFieldMapperLegacy.java |  6 +--
 .../mapper/ip/LegacyIpIndexFieldData.java     |  8 +--
 .../index/query/MoreLikeThisQueryBuilder.java |  2 +-
 .../elasticsearch/index/shard/IndexShard.java |  2 +-
 .../index/shard/StoreRecovery.java            |  2 +-
 .../index/store/DirectoryUtils.java           |  2 +-
 .../org/elasticsearch/index/store/Store.java  |  8 +--
 .../index/translog/Translog.java              |  2 +-
 .../indices/IndicesRequestCache.java          |  4 +-
 .../recovery/RecoverySourceHandler.java       |  4 +-
 .../elasticsearch/script/ScriptContext.java   |  8 +--
 .../elasticsearch/search/DocValueFormat.java  | 12 ++---
 .../org/elasticsearch/search/SearchHit.java   | 10 ++--
 .../org/elasticsearch/search/SearchHits.java  |  2 +-
 .../aggregations/AggregationStreams.java      |  2 +-
 .../aggregations/bucket/BucketStreams.java    |  2 +-
 .../aggregations/bucket/filters/Filters.java  |  2 +-
 .../bucket/histogram/Histogram.java           |  4 +-
 .../aggregations/bucket/range/Range.java      |  2 +-
 .../bucket/range/RangeAggregator.java         |  4 +-
 .../metrics/NumericMetricsAggregation.java    |  4 +-
 .../metrics/cardinality/Cardinality.java      |  2 +-
 .../metrics/percentiles/PercentileRanks.java  |  2 +-
 .../metrics/percentiles/Percentiles.java      |  2 +-
 .../pipeline/PipelineAggregator.java          |  8 +--
 .../pipeline/PipelineAggregatorBuilders.java  | 30 +++++------
 .../pipeline/PipelineAggregatorStreams.java   |  2 +-
 .../movavg/models/MovAvgModelBuilder.java     |  2 +-
 .../search/rescore/Rescorer.java              | 10 ++--
 .../completion2x/CompletionTokenStream.java   | 51 ++-----------------
 .../context/GeolocationContextMapping.java    |  2 +-
 .../DirectCandidateGeneratorBuilder.java      |  6 +--
 .../search/suggest/phrase/Laplace.java        |  2 +-
 .../suggest/phrase/LinearInterpolation.java   |  2 +-
 .../MultiCandidateGeneratorWrapper.java       | 10 +---
 .../search/suggest/phrase/StupidBackoff.java  |  2 +-
 .../transport/TcpTransportChannel.java        | 12 ++---
 .../apache/lucene/util/SloppyMathTests.java   |  4 +-
 .../TransportActionFilterChainTests.java      |  4 +-
 .../NodeVersionAllocationDeciderTests.java    |  2 +-
 .../http/netty/NettyHttpClient.java           |  2 +-
 .../index/store/CorruptedFileIT.java          |  4 +-
 .../rest/RestFilterChainTests.java            |  4 +-
 .../search/query/MultiMatchQueryIT.java       |  2 +-
 .../searchafter/SearchAfterBuilderTests.java  |  4 +-
 .../search/slice/SliceBuilderTests.java       |  2 +-
 .../support/MultiValuesSourceParser.java      |  4 +-
 .../ingest/common/ConvertProcessor.java       |  2 +-
 .../painless/CompilerSettings.java            |  4 +-
 .../elasticsearch/painless/Definition.java    | 12 ++---
 .../python/PythonScriptEngineService.java     |  2 +-
 .../engine/ThrowingLeafReaderWrapper.java     |  6 +--
 80 files changed, 171 insertions(+), 219 deletions(-)

diff --git a/buildSrc/src/main/resources/checkstyle.xml b/buildSrc/src/main/resources/checkstyle.xml
index fe726062706..706ef46ffa1 100644
--- a/buildSrc/src/main/resources/checkstyle.xml
+++ b/buildSrc/src/main/resources/checkstyle.xml
@@ -58,6 +58,8 @@
     -->
     <module name="ModifierOrder" />
 
+    <module name="RedundantModifier" />
+
     <!-- We don't use Java's builtin serialization and we suppress all warning
       about it. The flip side of that coin is that we shouldn't _try_ to use
       it. We can't outright ban it with ForbiddenApis because it complain about
diff --git a/core/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java b/core/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java
index 3aaa6442451..515d3204fb6 100644
--- a/core/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java
+++ b/core/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java
@@ -66,7 +66,7 @@ public class SearchPhaseExecutionException extends ElasticsearchException {
         }
     }
 
-    private static final Throwable deduplicateCause(Throwable cause, ShardSearchFailure[] shardFailures) {
+    private static Throwable deduplicateCause(Throwable cause, ShardSearchFailure[] shardFailures) {
         if (shardFailures == null) {
             throw new IllegalArgumentException("shardSearchFailures must not be null");
         }
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Seccomp.java b/core/src/main/java/org/elasticsearch/bootstrap/Seccomp.java
index 2ea6179e200..cfae761452f 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Seccomp.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Seccomp.java
@@ -97,7 +97,7 @@ final class Seccomp {
     // Linux implementation, based on seccomp(2) or prctl(2) with bpf filtering
 
     /** Access to non-standard Linux libc methods */
-    static interface LinuxLibrary extends Library {
+    interface LinuxLibrary extends Library {
         /**
          * maps to prctl(2)
          */
@@ -107,7 +107,7 @@ final class Seccomp {
          * this is the only way, DON'T use it on some other architecture unless you know wtf you are doing
          */
         NativeLong syscall(NativeLong number, Object... args);
-    };
+    }
 
     // null if unavailable or something goes wrong.
     private static final LinuxLibrary linux_libc;
@@ -415,7 +415,7 @@ final class Seccomp {
     // OS X implementation via sandbox(7)
 
     /** Access to non-standard OS X libc methods */
-    static interface MacLibrary extends Library {
+    interface MacLibrary extends Library {
         /**
          * maps to sandbox_init(3), since Leopard
          */
@@ -489,7 +489,7 @@ final class Seccomp {
     // Solaris implementation via priv_set(3C)
 
     /** Access to non-standard Solaris libc methods */
-    static interface SolarisLibrary extends Library {
+    interface SolarisLibrary extends Library {
         /**
          * see priv_set(3C), a convenience method for setppriv(2).
          */
diff --git a/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java b/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
index 2e26bad4684..6c22ff171dd 100644
--- a/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/InternalClusterInfoService.java
@@ -278,7 +278,7 @@ public class InternalClusterInfoService extends AbstractComponent implements Clu
         return latch;
     }
 
-    private final void maybeRefresh() {
+    private void maybeRefresh() {
         // Short-circuit if not enabled
         if (enabled) {
             refresh();
diff --git a/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java b/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java
index f5491dde2e3..6fcb783dd30 100644
--- a/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java
+++ b/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java
@@ -155,7 +155,7 @@ public abstract class BytesReference implements Accountable, Comparable<BytesRef
     /**
      * Compares the two references using the given int function.
      */
-    private static final int compareIterators(final BytesReference a, final BytesReference b, final ToIntBiFunction<BytesRef, BytesRef> f) {
+    private static int compareIterators(final BytesReference a, final BytesReference b, final ToIntBiFunction<BytesRef, BytesRef> f) {
         try {
             // we use the iterators since it's a 0-copy comparison where possible!
             final long lengthToCompare = Math.min(a.length(), b.length());
@@ -201,7 +201,7 @@ public abstract class BytesReference implements Accountable, Comparable<BytesRef
         }
     }
 
-    private static final void advance(final BytesRef ref, final int length) {
+    private static void advance(final BytesRef ref, final int length) {
         assert ref.length >= length : " ref.length: " + ref.length + " length: " + length;
         assert ref.offset+length < ref.bytes.length || (ref.offset+length == ref.bytes.length && ref.length-length == 0)
             : "offset: " + ref.offset + " ref.bytes.length: " + ref.bytes.length + " length: " + length + " ref.length: " + ref.length;
diff --git a/core/src/main/java/org/elasticsearch/common/collect/ImmutableOpenIntMap.java b/core/src/main/java/org/elasticsearch/common/collect/ImmutableOpenIntMap.java
index 0b53614723b..43e3552909b 100644
--- a/core/src/main/java/org/elasticsearch/common/collect/ImmutableOpenIntMap.java
+++ b/core/src/main/java/org/elasticsearch/common/collect/ImmutableOpenIntMap.java
@@ -132,7 +132,7 @@ public final class ImmutableOpenIntMap<VType> implements Iterable<IntObjectCurso
             }
 
             @Override
-            public final void remove() {
+            public void remove() {
                 throw new UnsupportedOperationException();
             }
         };
@@ -162,7 +162,7 @@ public final class ImmutableOpenIntMap<VType> implements Iterable<IntObjectCurso
             }
 
             @Override
-            public final void remove() {
+            public void remove() {
                 throw new UnsupportedOperationException();
             }
         };
diff --git a/core/src/main/java/org/elasticsearch/common/collect/ImmutableOpenMap.java b/core/src/main/java/org/elasticsearch/common/collect/ImmutableOpenMap.java
index c5210b1c2dc..85178283dfa 100644
--- a/core/src/main/java/org/elasticsearch/common/collect/ImmutableOpenMap.java
+++ b/core/src/main/java/org/elasticsearch/common/collect/ImmutableOpenMap.java
@@ -136,7 +136,7 @@ public final class ImmutableOpenMap<KType, VType> implements Iterable<ObjectObje
             }
 
             @Override
-            public final void remove() {
+            public void remove() {
                 throw new UnsupportedOperationException();
             }
         };
@@ -164,7 +164,7 @@ public final class ImmutableOpenMap<KType, VType> implements Iterable<ObjectObje
             }
 
             @Override
-            public final void remove() {
+            public void remove() {
                 throw new UnsupportedOperationException();
             }
         };
diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java b/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java
index d8fc226b617..3c5d1415624 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java
@@ -172,7 +172,7 @@ public class GeoHashUtils {
         return BitUtil.flipFlop(((geoHashLong >>> 4) << odd) << (((12 - level) * 5) + (MORTON_OFFSET - odd)));
     }
 
-    private static final char encode(int x, int y) {
+    private static char encode(int x, int y) {
         return BASE_32[((x & 1) + ((y & 1) * 2) + ((x & 2) * 2) + ((y & 2) * 4) + ((x & 4) * 4)) % 32];
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java b/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java
index 96fe2826da8..504fc413137 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/GeoPoint.java
@@ -98,27 +98,27 @@ public final class GeoPoint {
         return this.resetFromIndexHash(BitUtil.flipFlop((geohashLong >>> 4) << ((level * 5) + 2)));
     }
 
-    public final double lat() {
+    public double lat() {
         return this.lat;
     }
 
-    public final double getLat() {
+    public double getLat() {
         return this.lat;
     }
 
-    public final double lon() {
+    public double lon() {
         return this.lon;
     }
 
-    public final double getLon() {
+    public double getLon() {
         return this.lon;
     }
 
-    public final String geohash() {
+    public String geohash() {
         return stringEncode(lon, lat);
     }
 
-    public final String getGeohash() {
+    public String getGeohash() {
         return stringEncode(lon, lat);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java b/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java
index 589829f661e..f88a246dd0b 100644
--- a/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/geo/builders/PolygonBuilder.java
@@ -631,7 +631,7 @@ public class PolygonBuilder extends ShapeBuilder {
         return concat(component, direction ^ orientation, points, offset, edges, toffset, length);
     }
 
-    private static final int top(Coordinate[] points, int offset, int length) {
+    private static int top(Coordinate[] points, int offset, int length) {
         int top = 0; // we start at 1 here since top points to 0
         for (int i = 1; i < length; i++) {
             if (points[offset + i].y < points[offset + top].y) {
@@ -645,7 +645,7 @@ public class PolygonBuilder extends ShapeBuilder {
         return top;
     }
 
-    private static final double[] range(Coordinate[] points, int offset, int length) {
+    private static double[] range(Coordinate[] points, int offset, int length) {
         double minX = points[0].x;
         double maxX = points[0].x;
         double minY = points[0].y;
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java
index 75f400fdc9d..500fa206c96 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java
@@ -117,12 +117,12 @@ public final class AllTermQuery extends Query {
         return new Weight(this) {
 
             @Override
-            public final float getValueForNormalization() throws IOException {
+            public float getValueForNormalization() throws IOException {
                 return stats.getValueForNormalization();
             }
 
             @Override
-            public final void normalize(float norm, float topLevelBoost) {
+            public void normalize(float norm, float topLevelBoost) {
                 stats.normalize(norm, topLevelBoost);
             }
 
diff --git a/core/src/main/java/org/elasticsearch/common/lucene/all/AllTokenStream.java b/core/src/main/java/org/elasticsearch/common/lucene/all/AllTokenStream.java
index 6a49034d0b3..045a4badc45 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/all/AllTokenStream.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/all/AllTokenStream.java
@@ -46,7 +46,7 @@ public final class AllTokenStream extends TokenFilter {
     }
 
     @Override
-    public final boolean incrementToken() throws IOException {
+    public boolean incrementToken() throws IOException {
         if (!input.incrementToken()) {
             return false;
         }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
index efa4c1316ac..152a5629dd8 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java
@@ -457,7 +457,7 @@ public abstract class AbstractScopedSettings extends AbstractComponent {
         return changed;
     }
 
-    private static final boolean applyDeletes(Set<String> deletes, Settings.Builder builder) {
+    private static boolean applyDeletes(Set<String> deletes, Settings.Builder builder) {
         boolean changed = false;
         for (String entry : deletes) {
             Set<String> keysToRemove = new HashSet<>();
diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
index bbbbe57618b..43db77a1e95 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
@@ -178,7 +178,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
     }
 
     @Override
-    protected final boolean isPrivateSetting(String key) {
+    protected boolean isPrivateSetting(String key) {
         switch (key) {
             case IndexMetaData.SETTING_CREATION_DATE:
             case IndexMetaData.SETTING_INDEX_UUID:
diff --git a/core/src/main/java/org/elasticsearch/common/settings/Setting.java b/core/src/main/java/org/elasticsearch/common/settings/Setting.java
index 2c3aad38c63..920e1d7f5fc 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/Setting.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/Setting.java
@@ -465,7 +465,7 @@ public class Setting<T> extends ToXContentToBytes {
         }
 
         @Override
-        public final void apply(T value, Settings current, Settings previous) {
+        public void apply(T value, Settings current, Settings previous) {
             logger.info("updating [{}] from [{}] to [{}]", key, getRaw(previous), getRaw(current));
             consumer.accept(value);
         }
diff --git a/core/src/main/java/org/elasticsearch/common/settings/loader/PropertiesSettingsLoader.java b/core/src/main/java/org/elasticsearch/common/settings/loader/PropertiesSettingsLoader.java
index 6ee1f58cf4b..7192afa3ac3 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/loader/PropertiesSettingsLoader.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/loader/PropertiesSettingsLoader.java
@@ -46,7 +46,7 @@ public class PropertiesSettingsLoader implements SettingsLoader {
         return load(() -> StreamInput.wrap(source), (inStream, props) -> props.load(inStream));
     }
 
-    private final <T extends Closeable> Map<String, String> load(
+    private <T extends Closeable> Map<String, String> load(
             Supplier<T> supplier,
             IOExceptionThrowingBiConsumer<T, Properties> properties
     ) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/common/util/ExtensionPoint.java b/core/src/main/java/org/elasticsearch/common/util/ExtensionPoint.java
index 490209bd8d2..4b68512502d 100644
--- a/core/src/main/java/org/elasticsearch/common/util/ExtensionPoint.java
+++ b/core/src/main/java/org/elasticsearch/common/util/ExtensionPoint.java
@@ -184,7 +184,7 @@ public abstract class ExtensionPoint {
          * @param extension the extension to register
          * @throws IllegalArgumentException iff the class is already registered
          */
-        public final void registerExtension(Class<? extends T> extension) {
+        public void registerExtension(Class<? extends T> extension) {
             if (extensions.contains(extension)) {
                 throw new IllegalArgumentException("Can't register the same [" + this.name + "] more than once for [" + extension.getName() + "]");
             }
@@ -192,7 +192,7 @@ public abstract class ExtensionPoint {
         }
 
         @Override
-        protected final void bindExtensions(Binder binder) {
+        protected void bindExtensions(Binder binder) {
             Multibinder<T> allocationMultibinder = Multibinder.newSetBinder(binder, extensionClass);
             for (Class<? extends T> clazz : extensions) {
                 binder.bind(clazz).asEagerSingleton();
@@ -227,7 +227,7 @@ public abstract class ExtensionPoint {
          *
          * @throws IllegalArgumentException iff the key is already registered
          */
-        public final void registerExtension(K key, V value) {
+        public void registerExtension(K key, V value) {
             V old = map.put(key, value);
             if (old != null) {
                 throw new IllegalArgumentException("Cannot register [" + this.name + "] with key [" + key + "] to [" + value + "], already registered to [" + old + "]");
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
index 36a9bd426e8..c5d22b44fc9 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
@@ -55,7 +55,7 @@ public class MasterFaultDetection extends FaultDetection {
 
     public static final String MASTER_PING_ACTION_NAME = "internal:discovery/zen/fd/master_ping";
 
-    public static interface Listener {
+    public interface Listener {
 
         /** called when pinging the master failed, like a timeout, transport disconnects etc */
         void onMasterFailure(DiscoveryNode masterNode, Throwable cause, String reason);
diff --git a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
index 6a805584f5c..c091788911f 100644
--- a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
+++ b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
@@ -812,7 +812,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
     }
 
     @Override
-    public final void close() {
+    public void close() {
         if (closed.compareAndSet(false, true) && locks != null) {
             for (Lock lock : locks) {
                 try {
diff --git a/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java b/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
index af8709ccd96..ee987b8665f 100644
--- a/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
+++ b/core/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java
@@ -105,7 +105,7 @@ public class LocalAllocateDangledIndices extends AbstractComponent {
         });
     }
 
-    public static interface Listener {
+    public interface Listener {
         void onResponse(AllocateDangledResponse response);
 
         void onFailure(Throwable e);
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/CharMatcher.java b/core/src/main/java/org/elasticsearch/index/analysis/CharMatcher.java
index 90c6ad3f3b4..238fde5e6b3 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/CharMatcher.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/CharMatcher.java
@@ -27,7 +27,7 @@ import java.util.Set;
  */
 public interface CharMatcher {
 
-    public static class ByUnicodeCategory implements CharMatcher {
+    class ByUnicodeCategory implements CharMatcher {
 
         public static CharMatcher of(byte unicodeCategory) {
             return new ByUnicodeCategory(unicodeCategory);
@@ -134,5 +134,5 @@ public interface CharMatcher {
     }
 
     /** Returns true if, and only if, the provided character matches this character class. */
-    public boolean isTokenChar(int c);
+    boolean isTokenChar(int c);
 }
diff --git a/core/src/main/java/org/elasticsearch/index/analysis/MultiTermAwareComponent.java b/core/src/main/java/org/elasticsearch/index/analysis/MultiTermAwareComponent.java
index 0662352e79c..ca1d4b9e820 100644
--- a/core/src/main/java/org/elasticsearch/index/analysis/MultiTermAwareComponent.java
+++ b/core/src/main/java/org/elasticsearch/index/analysis/MultiTermAwareComponent.java
@@ -25,6 +25,6 @@ public interface MultiTermAwareComponent {
     /** Returns an analysis component to handle analysis if multi-term queries.
      * The returned component must be a TokenizerFactory, TokenFilterFactory or CharFilterFactory.
      */
-    public Object getMultiTermComponent();
+    Object getMultiTermComponent();
 
 }
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java
index e64499e8a3c..fce29c09ba4 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/FieldData.java
@@ -502,7 +502,7 @@ public enum FieldData {
         };
     }
 
-    private static interface ToStringValues {
+    private interface ToStringValues {
 
         void get(int docID, List<CharSequence> values);
 
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/Ordinals.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/Ordinals.java
index f00f266ee1d..77c0fd6bf5b 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/Ordinals.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/Ordinals.java
@@ -47,10 +47,9 @@ public abstract class Ordinals implements Accountable {
         return ordinals(NO_VALUES);
     }
 
-    public static interface ValuesHolder {
-
-        public abstract BytesRef lookupOrd(long ord);
+    public interface ValuesHolder {
 
+        BytesRef lookupOrd(long ord);
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java
index 6c1f3e6fc9e..c519c6634a2 100644
--- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexFieldData.java
@@ -96,7 +96,7 @@ public abstract class AbstractIndexFieldData<FD extends AtomicFieldData> extends
         /**
          * @return the number of bytes for the given term
          */
-        public long bytesPerValue(BytesRef term);
+        long bytesPerValue(BytesRef term);
 
         /**
          * Execute any pre-loading estimations for the terms. May also
@@ -107,7 +107,7 @@ public abstract class AbstractIndexFieldData<FD extends AtomicFieldData> extends
          * @param terms terms to be estimated
          * @return A TermsEnum for the given terms
          */
-        public TermsEnum beforeLoad(Terms terms) throws IOException;
+        TermsEnum beforeLoad(Terms terms) throws IOException;
 
         /**
          * Possibly adjust a circuit breaker after field data has been loaded,
@@ -116,6 +116,6 @@ public abstract class AbstractIndexFieldData<FD extends AtomicFieldData> extends
          * @param termsEnum  terms that were loaded
          * @param actualUsed actual field data memory usage
          */
-        public void afterLoad(TermsEnum termsEnum, long actualUsed);
+        void afterLoad(TermsEnum termsEnum, long actualUsed);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
index fc20ef50cf6..07c18060fbe 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
@@ -66,7 +66,7 @@ final class DocumentParser {
         this.docMapper = docMapper;
     }
 
-    final ParsedDocument parseDocument(SourceToParse source) throws MapperParsingException {
+    ParsedDocument parseDocument(SourceToParse source) throws MapperParsingException {
         validateType(source);
 
         final Mapping mapping = docMapper.mapping();
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
index 3b4dd3bb5c6..9a434cc8a36 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java
@@ -139,7 +139,7 @@ public abstract class MappedFieldType extends FieldType {
     public abstract String typeName();
 
     /** Checks this type is the same type as other. Adds a conflict if they are different. */
-    private final void checkTypeName(MappedFieldType other) {
+    private void checkTypeName(MappedFieldType other) {
         if (typeName().equals(other.typeName()) == false) {
             throw new IllegalArgumentException("mapper [" + name + "] cannot be changed from type [" + typeName() + "] to [" + other.typeName() + "]");
         } else if (getClass() != other.getClass()) {
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java
index 622c7729dd4..3ec3991837f 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java
@@ -31,7 +31,7 @@ import java.util.Map;
  */
 public abstract class MetadataFieldMapper extends FieldMapper {
 
-    public static interface TypeParser extends Mapper.TypeParser {
+    public interface TypeParser extends Mapper.TypeParser {
 
         @Override
         MetadataFieldMapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException;
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperLegacy.java b/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperLegacy.java
index 73b91ffcfb2..7eddea679bc 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperLegacy.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapperLegacy.java
@@ -155,7 +155,7 @@ public class GeoPointFieldMapperLegacy extends BaseGeoPointFieldMapper implement
         }
 
         /** Get an instance based on the number of bytes that has been used to encode values. */
-        public static final Encoding of(int numBytesPerValue) {
+        public static Encoding of(int numBytesPerValue) {
             final Encoding instance = INSTANCES[numBytesPerValue];
             if (instance == null) {
                 throw new IllegalStateException("No encoding for " + numBytesPerValue + " bytes per value");
@@ -170,7 +170,7 @@ public class GeoPointFieldMapperLegacy extends BaseGeoPointFieldMapper implement
          *  <li>1m: 8 bytes</li>
          *  <li>1cm: 8 bytes</li>
          *  <li>1mm: 10 bytes</li></ul> */
-        public static final Encoding of(DistanceUnit.Distance precision) {
+        public static Encoding of(DistanceUnit.Distance precision) {
             for (Encoding encoding : INSTANCES) {
                 if (encoding != null && encoding.precision().compareTo(precision) <= 0) {
                     return encoding;
@@ -206,7 +206,7 @@ public class GeoPointFieldMapperLegacy extends BaseGeoPointFieldMapper implement
         }
 
         /** The number of bytes required to encode a single geo point. */
-        public final int numBytes() {
+        public int numBytes() {
             return numBytes;
         }
 
diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpIndexFieldData.java
index b94855d25bf..8435f639b7f 100644
--- a/core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpIndexFieldData.java
+++ b/core/src/main/java/org/elasticsearch/index/mapper/ip/LegacyIpIndexFieldData.java
@@ -52,19 +52,19 @@ final class LegacyIpIndexFieldData implements IndexFieldData<AtomicFieldData> {
         this.logger = Loggers.getLogger(getClass());
     }
 
-    public final String getFieldName() {
+    public String getFieldName() {
         return fieldName;
     }
 
-    public final void clear() {
+    public void clear() {
         // nothing to do
     }
 
-    public final void clear(IndexReader reader) {
+    public void clear(IndexReader reader) {
         // nothing to do
     }
 
-    public final Index index() {
+    public Index index() {
         return index;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
index 5fa36ec9777..e4ee3cd171c 100644
--- a/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilder.java
@@ -450,7 +450,7 @@ public class MoreLikeThisQueryBuilder extends AbstractQueryBuilder<MoreLikeThisQ
         }
 
         @Override
-        public final String toString() {
+        public String toString() {
             try {
                 XContentBuilder builder = XContentFactory.jsonBuilder();
                 builder.prettyPrint();
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index ed4072b6b29..7843aabb49f 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -1538,7 +1538,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         return mapperService.documentMapperWithAutoCreate(type);
     }
 
-    private final EngineConfig newEngineConfig(EngineConfig.OpenMode openMode) {
+    private EngineConfig newEngineConfig(EngineConfig.OpenMode openMode) {
         final IndexShardRecoveryPerformer translogRecoveryPerformer = new IndexShardRecoveryPerformer(shardId, mapperService, logger);
         return new EngineConfig(openMode, shardId,
             threadPool, indexSettings, warmer, store, deletionPolicy, indexSettings.getMergePolicy(),
diff --git a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
index 4a9284d8155..01513aa6bc6 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
@@ -130,7 +130,7 @@ final class StoreRecovery {
         return false;
     }
 
-    final void addIndices(RecoveryState.Index indexRecoveryStats, Directory target, Directory... sources) throws IOException {
+    void addIndices(RecoveryState.Index indexRecoveryStats, Directory target, Directory... sources) throws IOException {
         target = new org.apache.lucene.store.HardlinkCopyDirectoryWrapper(target);
         try (IndexWriter writer = new IndexWriter(new StatsDirectoryWrapper(target, indexRecoveryStats),
             new IndexWriterConfig(null)
diff --git a/core/src/main/java/org/elasticsearch/index/store/DirectoryUtils.java b/core/src/main/java/org/elasticsearch/index/store/DirectoryUtils.java
index c99bc322708..11203a81987 100644
--- a/core/src/main/java/org/elasticsearch/index/store/DirectoryUtils.java
+++ b/core/src/main/java/org/elasticsearch/index/store/DirectoryUtils.java
@@ -30,7 +30,7 @@ public final class DirectoryUtils {
 
     private DirectoryUtils() {} // no instance
 
-    static final <T extends Directory> Directory getLeafDirectory(FilterDirectory dir, Class<T> targetClass) {
+    static <T extends Directory> Directory getLeafDirectory(FilterDirectory dir, Class<T> targetClass) {
         Directory current = dir.getDelegate();
         while (true) {
             if ((current instanceof FilterDirectory)) {
diff --git a/core/src/main/java/org/elasticsearch/index/store/Store.java b/core/src/main/java/org/elasticsearch/index/store/Store.java
index 211f97d8916..d52ab16459e 100644
--- a/core/src/main/java/org/elasticsearch/index/store/Store.java
+++ b/core/src/main/java/org/elasticsearch/index/store/Store.java
@@ -556,7 +556,7 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
         failIfCorrupted(directory, shardId);
     }
 
-    private static final void failIfCorrupted(Directory directory, ShardId shardId) throws IOException {
+    private static void failIfCorrupted(Directory directory, ShardId shardId) throws IOException {
         final String[] files = directory.listAll();
         List<CorruptIndexException> ex = new ArrayList<>();
         for (String file : files) {
@@ -1033,7 +1033,7 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
             return null;
         }
 
-        private final int numSegmentFiles() { // only for asserts
+        private int numSegmentFiles() { // only for asserts
             int count = 0;
             for (StoreFileMetaData file : this) {
                 if (file.name().startsWith(IndexFileNames.SEGMENTS)) {
@@ -1353,8 +1353,8 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
     /**
      * A listener that is executed once the store is closed and all references to it are released
      */
-    public static interface OnClose extends Callback<ShardLock> {
-        static final OnClose EMPTY = new OnClose() {
+    public interface OnClose extends Callback<ShardLock> {
+        OnClose EMPTY = new OnClose() {
             /**
              * This method is called while the provided {@link org.elasticsearch.env.ShardLock} is held.
              * This method is only called once after all resources for a store are released.
diff --git a/core/src/main/java/org/elasticsearch/index/translog/Translog.java b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
index cbb54bd9d50..d26a06a5ce6 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/Translog.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
@@ -215,7 +215,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
     }
 
     /** recover all translog files found on disk */
-    private final ArrayList<TranslogReader> recoverFromFiles(TranslogGeneration translogGeneration, Checkpoint checkpoint) throws IOException {
+    private ArrayList<TranslogReader> recoverFromFiles(TranslogGeneration translogGeneration, Checkpoint checkpoint) throws IOException {
         boolean success = false;
         ArrayList<TranslogReader> foundTranslogs = new ArrayList<>();
         final Path tempFile = Files.createTempFile(location, TRANSLOG_FILE_PREFIX, TRANSLOG_FILE_SUFFIX); // a temp file to copy checkpoint to - note it must be in on the same FS otherwise atomic move won't work
diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java b/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
index 4b4aa4e8df2..5229de3f16c 100644
--- a/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
+++ b/core/src/main/java/org/elasticsearch/indices/IndicesRequestCache.java
@@ -328,11 +328,11 @@ public final class IndicesRequestCache extends AbstractComponent implements Remo
     /**
      * Returns the current size of the cache
      */
-    final int count() {
+    int count() {
         return cache.count();
     }
 
-    final int numRegisteredCloseListeners() { // for testing
+    int numRegisteredCloseListeners() { // for testing
         return registeredClosedListeners.size();
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
index db2f97787d1..6c126c58397 100644
--- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
+++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java
@@ -486,12 +486,12 @@ public class RecoverySourceHandler {
         }
 
         @Override
-        public final void write(int b) throws IOException {
+        public void write(int b) throws IOException {
             throw new UnsupportedOperationException("we can't send single bytes over the wire");
         }
 
         @Override
-        public final void write(byte[] b, int offset, int length) throws IOException {
+        public void write(byte[] b, int offset, int length) throws IOException {
             sendNextChunk(position, new BytesArray(b, offset, length), md.length() == position + length);
             position += length;
             assert md.length() >= position : "length: " + md.length() + " but positions was: " + position;
diff --git a/core/src/main/java/org/elasticsearch/script/ScriptContext.java b/core/src/main/java/org/elasticsearch/script/ScriptContext.java
index 3ab2bb52c9b..cc2408786e9 100644
--- a/core/src/main/java/org/elasticsearch/script/ScriptContext.java
+++ b/core/src/main/java/org/elasticsearch/script/ScriptContext.java
@@ -84,21 +84,21 @@ public interface ScriptContext {
             this.key = pluginName + "_" + operation;
         }
 
-        public final String getPluginName() {
+        public String getPluginName() {
             return pluginName;
         }
 
-        public final String getOperation() {
+        public String getOperation() {
             return operation;
         }
 
         @Override
-        public final String getKey() {
+        public String getKey() {
             return key;
         }
 
         @Override
-        public final String toString() {
+        public String toString() {
             return getKey();
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/search/DocValueFormat.java b/core/src/main/java/org/elasticsearch/search/DocValueFormat.java
index 4fc1967d7ec..4cbb8720d77 100644
--- a/core/src/main/java/org/elasticsearch/search/DocValueFormat.java
+++ b/core/src/main/java/org/elasticsearch/search/DocValueFormat.java
@@ -73,7 +73,7 @@ public interface DocValueFormat extends NamedWriteable {
      *  to the original BytesRef. */
     BytesRef parseBytesRef(String value);
 
-    public static final DocValueFormat RAW = new DocValueFormat() {
+    DocValueFormat RAW = new DocValueFormat() {
 
         @Override
         public String getWriteableName() {
@@ -121,7 +121,7 @@ public interface DocValueFormat extends NamedWriteable {
         }
     };
 
-    public static final class DateTime implements DocValueFormat {
+    final class DateTime implements DocValueFormat {
 
         public static final String NAME = "date_time";
 
@@ -181,7 +181,7 @@ public interface DocValueFormat extends NamedWriteable {
         }
     }
 
-    public static final DocValueFormat GEOHASH = new DocValueFormat() {
+    DocValueFormat GEOHASH = new DocValueFormat() {
 
         @Override
         public String getWriteableName() {
@@ -223,7 +223,7 @@ public interface DocValueFormat extends NamedWriteable {
         }
     };
 
-    public static final DocValueFormat BOOLEAN = new DocValueFormat() {
+    DocValueFormat BOOLEAN = new DocValueFormat() {
 
         @Override
         public String getWriteableName() {
@@ -271,7 +271,7 @@ public interface DocValueFormat extends NamedWriteable {
         }
     };
 
-    public static final DocValueFormat IP = new DocValueFormat() {
+    DocValueFormat IP = new DocValueFormat() {
 
         @Override
         public String getWriteableName() {
@@ -315,7 +315,7 @@ public interface DocValueFormat extends NamedWriteable {
         }
     };
 
-    public static final class Decimal implements DocValueFormat {
+    final class Decimal implements DocValueFormat {
 
         public static final String NAME = "decimal";
         private static final DecimalFormatSymbols SYMBOLS = new DecimalFormatSymbols(Locale.ROOT);
diff --git a/core/src/main/java/org/elasticsearch/search/SearchHit.java b/core/src/main/java/org/elasticsearch/search/SearchHit.java
index c4625a8e5d8..b149a4162fe 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchHit.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchHit.java
@@ -146,7 +146,7 @@ public interface SearchHit extends Streamable, ToXContent, Iterable<SearchHitFie
     /**
      * The hit field matching the given field name.
      */
-    public SearchHitField field(String fieldName);
+    SearchHitField field(String fieldName);
 
     /**
      * A map of hit fields (from field name to hit fields) if additional fields
@@ -208,23 +208,23 @@ public interface SearchHit extends Streamable, ToXContent, Iterable<SearchHitFie
     /**
      * Encapsulates the nested identity of a hit.
      */
-    public interface NestedIdentity {
+    interface NestedIdentity {
 
         /**
          * Returns the nested field in the source this hit originates from
          */
-        public Text getField();
+        Text getField();
 
         /**
          * Returns the offset in the nested array of objects in the source this hit
          */
-        public int getOffset();
+        int getOffset();
 
         /**
          * Returns the next child nested level if there is any, otherwise <code>null</code> is returned.
          *
          * In the case of mappings with multiple levels of nested object fields
          */
-        public NestedIdentity getChild();
+        NestedIdentity getChild();
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/SearchHits.java b/core/src/main/java/org/elasticsearch/search/SearchHits.java
index 08d13bf701c..400e2ebc44f 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchHits.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchHits.java
@@ -62,5 +62,5 @@ public interface SearchHits extends Streamable, ToXContent, Iterable<SearchHit>
     /**
      * The hits of the search request (based on the search type, and from / size provided).
      */
-    public SearchHit[] getHits();
+    SearchHit[] getHits();
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationStreams.java b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationStreams.java
index 2ebe2ddc6a4..81228be2606 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/AggregationStreams.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/AggregationStreams.java
@@ -38,7 +38,7 @@ public class AggregationStreams {
     /**
      * A stream that knows how to read an aggregation from the input.
      */
-    public static interface Stream {
+    public interface Stream {
          InternalAggregation readResult(StreamInput in) throws IOException;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketStreams.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketStreams.java
index ffbf8262cd7..2025d62ef6e 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketStreams.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketStreams.java
@@ -35,7 +35,7 @@ public class BucketStreams {
     /**
      * A stream that knows how to read a bucket from the input.
      */
-    public static interface Stream<B extends MultiBucketsAggregation.Bucket> {
+    public interface Stream<B extends MultiBucketsAggregation.Bucket> {
         B readResult(StreamInput in, BucketStreamContext context) throws IOException;
         BucketStreamContext getBucketStreamContext(B bucket);
     }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/Filters.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/Filters.java
index 7dc0fdc6b11..4b8c893c163 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/Filters.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/filters/Filters.java
@@ -32,7 +32,7 @@ public interface Filters extends MultiBucketsAggregation {
     /**
      * A bucket associated with a specific filter (identified by its key)
      */
-    public static interface Bucket extends MultiBucketsAggregation.Bucket {
+    interface Bucket extends MultiBucketsAggregation.Bucket {
     }
 
     /**
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/Histogram.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/Histogram.java
index 6e6ac058204..a70e5419096 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/Histogram.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/Histogram.java
@@ -32,7 +32,7 @@ public interface Histogram extends MultiBucketsAggregation {
     /**
      * A bucket in the histogram where documents fall in
      */
-    static interface Bucket extends MultiBucketsAggregation.Bucket {
+    interface Bucket extends MultiBucketsAggregation.Bucket {
 
     }
 
@@ -46,7 +46,7 @@ public interface Histogram extends MultiBucketsAggregation {
     /**
      * A strategy defining the order in which the buckets in this histogram are ordered.
      */
-    abstract static class Order implements ToXContent {
+    abstract class Order implements ToXContent {
 
         public static final Order KEY_ASC = new InternalOrder((byte) 1, "_key", true, new Comparator<InternalHistogram.Bucket>() {
             @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/Range.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/Range.java
index 8280e558a00..b16fc81cdb1 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/Range.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/Range.java
@@ -31,7 +31,7 @@ public interface Range extends MultiBucketsAggregation {
     /**
      * A bucket associated with a specific range
      */
-    public static interface Bucket extends MultiBucketsAggregation.Bucket {
+    interface Bucket extends MultiBucketsAggregation.Bucket {
 
         /**
          * @return  The lower bound of the range
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java
index 6a99295f754..3aa5fdca17f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregator.java
@@ -305,7 +305,7 @@ public class RangeAggregator extends BucketsAggregator {
         };
     }
 
-    private final long subBucketOrdinal(long owningBucketOrdinal, int rangeOrd) {
+    private long subBucketOrdinal(long owningBucketOrdinal, int rangeOrd) {
         return owningBucketOrdinal * ranges.length + rangeOrd;
     }
 
@@ -337,7 +337,7 @@ public class RangeAggregator extends BucketsAggregator {
         return rangeFactory.create(name, buckets, format, keyed, pipelineAggregators(), metaData());
     }
 
-    private static final void sortRanges(final Range[] ranges) {
+    private static void sortRanges(final Range[] ranges) {
         new InPlaceMergeSorter() {
 
             @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/NumericMetricsAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/NumericMetricsAggregation.java
index 11083c9a061..046b9825acc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/NumericMetricsAggregation.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/NumericMetricsAggregation.java
@@ -23,7 +23,7 @@ import org.elasticsearch.search.aggregations.Aggregation;
 
 public interface NumericMetricsAggregation extends Aggregation {
 
-    public static interface SingleValue extends NumericMetricsAggregation {
+    interface SingleValue extends NumericMetricsAggregation {
 
         double value();
 
@@ -31,6 +31,6 @@ public interface NumericMetricsAggregation extends Aggregation {
 
     }
 
-    public static interface MultiValue extends NumericMetricsAggregation {
+    interface MultiValue extends NumericMetricsAggregation {
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/Cardinality.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/Cardinality.java
index f9d8e10730a..92f3b8bb261 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/Cardinality.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/cardinality/Cardinality.java
@@ -29,6 +29,6 @@ public interface Cardinality extends NumericMetricsAggregation.SingleValue {
     /**
      * The number of unique terms.
      */
-    public long getValue();
+    long getValue();
 
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanks.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanks.java
index 19ad169b8f7..8a2dc9d9026 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanks.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentileRanks.java
@@ -26,7 +26,7 @@ import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
  */
 public interface PercentileRanks extends NumericMetricsAggregation.MultiValue, Iterable<Percentile> {
 
-    public static final String TYPE_NAME = "percentile_ranks";
+    String TYPE_NAME = "percentile_ranks";
 
     /**
      * Return the percentile for the given value.
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/Percentiles.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/Percentiles.java
index 30fdb3f0002..a9052536dc4 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/Percentiles.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/Percentiles.java
@@ -25,7 +25,7 @@ import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
  */
 public interface Percentiles extends NumericMetricsAggregation.MultiValue, Iterable<Percentile> {
 
-    public static final String TYPE_NAME = "percentiles";
+    String TYPE_NAME = "percentiles";
 
     /**
      * Return the value associated with the provided percentile.
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregator.java
index 8c43f7460dc..446b56d07e6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregator.java
@@ -41,10 +41,10 @@ public abstract class PipelineAggregator implements Streamable, NamedWriteable {
      * Parse the {@link PipelineAggregationBuilder} from a {@link QueryParseContext}.
      */
     @FunctionalInterface
-    public static interface Parser {
-        public static final ParseField BUCKETS_PATH = new ParseField("buckets_path");
-        public static final ParseField FORMAT = new ParseField("format");
-        public static final ParseField GAP_POLICY = new ParseField("gap_policy");
+    public interface Parser {
+        ParseField BUCKETS_PATH = new ParseField("buckets_path");
+        ParseField FORMAT = new ParseField("format");
+        ParseField GAP_POLICY = new ParseField("gap_policy");
 
         /**
          * Returns the pipeline aggregator factory with which this parser is
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregatorBuilders.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregatorBuilders.java
index 3dc33188495..ce738d7a618 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregatorBuilders.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregatorBuilders.java
@@ -41,70 +41,70 @@ public final class PipelineAggregatorBuilders {
     private PipelineAggregatorBuilders() {
     }
 
-    public static final DerivativePipelineAggregationBuilder derivative(String name, String bucketsPath) {
+    public static DerivativePipelineAggregationBuilder derivative(String name, String bucketsPath) {
         return new DerivativePipelineAggregationBuilder(name, bucketsPath);
     }
 
-    public static final MaxBucketPipelineAggregationBuilder maxBucket(String name, String bucketsPath) {
+    public static MaxBucketPipelineAggregationBuilder maxBucket(String name, String bucketsPath) {
         return new MaxBucketPipelineAggregationBuilder(name, bucketsPath);
     }
 
-    public static final MinBucketPipelineAggregationBuilder minBucket(String name, String bucketsPath) {
+    public static MinBucketPipelineAggregationBuilder minBucket(String name, String bucketsPath) {
         return new MinBucketPipelineAggregationBuilder(name, bucketsPath);
     }
 
-    public static final AvgBucketPipelineAggregationBuilder avgBucket(String name, String bucketsPath) {
+    public static AvgBucketPipelineAggregationBuilder avgBucket(String name, String bucketsPath) {
         return new AvgBucketPipelineAggregationBuilder(name, bucketsPath);
     }
 
-    public static final SumBucketPipelineAggregationBuilder sumBucket(String name, String bucketsPath) {
+    public static SumBucketPipelineAggregationBuilder sumBucket(String name, String bucketsPath) {
         return new SumBucketPipelineAggregationBuilder(name, bucketsPath);
     }
 
-    public static final StatsBucketPipelineAggregationBuilder statsBucket(String name, String bucketsPath) {
+    public static StatsBucketPipelineAggregationBuilder statsBucket(String name, String bucketsPath) {
         return new StatsBucketPipelineAggregationBuilder(name, bucketsPath);
     }
 
-    public static final ExtendedStatsBucketPipelineAggregationBuilder extendedStatsBucket(String name,
+    public static ExtendedStatsBucketPipelineAggregationBuilder extendedStatsBucket(String name,
             String bucketsPath) {
         return new ExtendedStatsBucketPipelineAggregationBuilder(name, bucketsPath);
     }
 
-    public static final PercentilesBucketPipelineAggregationBuilder percentilesBucket(String name,
+    public static PercentilesBucketPipelineAggregationBuilder percentilesBucket(String name,
             String bucketsPath) {
         return new PercentilesBucketPipelineAggregationBuilder(name, bucketsPath);
     }
 
-    public static final MovAvgPipelineAggregationBuilder movingAvg(String name, String bucketsPath) {
+    public static MovAvgPipelineAggregationBuilder movingAvg(String name, String bucketsPath) {
         return new MovAvgPipelineAggregationBuilder(name, bucketsPath);
     }
 
-    public static final BucketScriptPipelineAggregationBuilder bucketScript(String name,
+    public static BucketScriptPipelineAggregationBuilder bucketScript(String name,
             Map<String, String> bucketsPathsMap, Script script) {
         return new BucketScriptPipelineAggregationBuilder(name, bucketsPathsMap, script);
     }
 
-    public static final BucketScriptPipelineAggregationBuilder bucketScript(String name, Script script,
+    public static BucketScriptPipelineAggregationBuilder bucketScript(String name, Script script,
             String... bucketsPaths) {
         return new BucketScriptPipelineAggregationBuilder(name, script, bucketsPaths);
     }
 
-    public static final BucketSelectorPipelineAggregationBuilder bucketSelector(String name,
+    public static BucketSelectorPipelineAggregationBuilder bucketSelector(String name,
             Map<String, String> bucketsPathsMap, Script script) {
         return new BucketSelectorPipelineAggregationBuilder(name, bucketsPathsMap, script);
     }
 
-    public static final BucketSelectorPipelineAggregationBuilder bucketSelector(String name, Script script,
+    public static BucketSelectorPipelineAggregationBuilder bucketSelector(String name, Script script,
             String... bucketsPaths) {
         return new BucketSelectorPipelineAggregationBuilder(name, script, bucketsPaths);
     }
 
-    public static final CumulativeSumPipelineAggregationBuilder cumulativeSum(String name,
+    public static CumulativeSumPipelineAggregationBuilder cumulativeSum(String name,
             String bucketsPath) {
         return new CumulativeSumPipelineAggregationBuilder(name, bucketsPath);
     }
 
-    public static final SerialDiffPipelineAggregationBuilder diff(String name, String bucketsPath) {
+    public static SerialDiffPipelineAggregationBuilder diff(String name, String bucketsPath) {
         return new SerialDiffPipelineAggregationBuilder(name, bucketsPath);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregatorStreams.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregatorStreams.java
index 71046092879..ad1dc73f6e3 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregatorStreams.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/PipelineAggregatorStreams.java
@@ -38,7 +38,7 @@ public class PipelineAggregatorStreams {
     /**
      * A stream that knows how to read an aggregation from the input.
      */
-    public static interface Stream {
+    public interface Stream {
         PipelineAggregator readResult(StreamInput in) throws IOException;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/MovAvgModelBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/MovAvgModelBuilder.java
index 759c493d858..5d858b48d16 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/MovAvgModelBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/pipeline/movavg/models/MovAvgModelBuilder.java
@@ -27,5 +27,5 @@ import org.elasticsearch.common.xcontent.ToXContent;
  */
 public interface MovAvgModelBuilder extends ToXContent {
 
-    public MovAvgModel build();
+    MovAvgModel build();
 }
diff --git a/core/src/main/java/org/elasticsearch/search/rescore/Rescorer.java b/core/src/main/java/org/elasticsearch/search/rescore/Rescorer.java
index 5e824aadc6c..bf15e06bcb3 100644
--- a/core/src/main/java/org/elasticsearch/search/rescore/Rescorer.java
+++ b/core/src/main/java/org/elasticsearch/search/rescore/Rescorer.java
@@ -37,7 +37,7 @@ public interface Rescorer {
     /**
      * Returns the name of this rescorer
      */
-    public String name();
+    String name();
 
     /**
      * Modifies the result of the previously executed search ({@link TopDocs})
@@ -48,7 +48,7 @@ public interface Rescorer {
      * @param rescoreContext the {@link RescoreSearchContext}. This will never be <code>null</code>
      * @throws IOException if an {@link IOException} occurs during rescoring
      */
-    public TopDocs rescore(TopDocs topDocs, SearchContext context, RescoreSearchContext rescoreContext) throws IOException;
+    TopDocs rescore(TopDocs topDocs, SearchContext context, RescoreSearchContext rescoreContext) throws IOException;
 
     /**
      * Executes an {@link Explanation} phase on the rescorer.
@@ -60,8 +60,8 @@ public interface Rescorer {
      * @return the explain for the given top level document ID.
      * @throws IOException if an {@link IOException} occurs
      */
-    public Explanation explain(int topLevelDocId, SearchContext context, RescoreSearchContext rescoreContext,
-            Explanation sourceExplanation) throws IOException;
+    Explanation explain(int topLevelDocId, SearchContext context, RescoreSearchContext rescoreContext,
+                        Explanation sourceExplanation) throws IOException;
 
     /**
      * Extracts all terms needed to execute this {@link Rescorer}. This method
@@ -69,7 +69,7 @@ public interface Rescorer {
      * {@link SearchType#DFS_QUERY_AND_FETCH} and
      * {@link SearchType#DFS_QUERY_THEN_FETCH}
      */
-    public void extractTerms(SearchContext context, RescoreSearchContext rescoreContext, Set<Term> termsSet);
+    void extractTerms(SearchContext context, RescoreSearchContext rescoreContext, Set<Term> termsSet);
 
     /*
      * TODO: At this point we only have one implementation which modifies the
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/CompletionTokenStream.java b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/CompletionTokenStream.java
index cbf34be8389..2c752d64175 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/CompletionTokenStream.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/CompletionTokenStream.java
@@ -23,11 +23,8 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
-import org.apache.lucene.util.AttributeImpl;
-import org.apache.lucene.util.AttributeReflector;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.lucene.util.CharsRefBuilder;
 import org.apache.lucene.util.IntsRef;
 import org.apache.lucene.util.fst.Util;
 
@@ -108,8 +105,8 @@ public final class CompletionTokenStream extends TokenStream {
         input.close();
     }
 
-    public static interface ToFiniteStrings {
-        public Set<IntsRef> toFiniteStrings(TokenStream stream) throws IOException;
+    public interface ToFiniteStrings {
+        Set<IntsRef> toFiniteStrings(TokenStream stream) throws IOException;
     }
 
     @Override
@@ -129,48 +126,8 @@ public final class CompletionTokenStream extends TokenStream {
         /**
          * Return the builder from which the term is derived.
          */
-        public BytesRefBuilder builder();
+        BytesRefBuilder builder();
 
-        public CharSequence toUTF16();
-    }
-
-    public static final class ByteTermAttributeImpl extends AttributeImpl implements ByteTermAttribute, TermToBytesRefAttribute {
-        private final BytesRefBuilder bytes = new BytesRefBuilder();
-        private CharsRefBuilder charsRef;
-
-        @Override
-        public BytesRefBuilder builder() {
-            return bytes;
-        }
-
-        @Override
-        public BytesRef getBytesRef() {
-            return bytes.get();
-        }
-
-        @Override
-        public void clear() {
-            bytes.clear();
-        }
-
-        @Override
-        public void reflectWith(AttributeReflector reflector) {
-
-        }
-
-        @Override
-        public void copyTo(AttributeImpl target) {
-            ByteTermAttributeImpl other = (ByteTermAttributeImpl) target;
-            other.bytes.copyBytes(bytes);
-        }
-
-        @Override
-        public CharSequence toUTF16() {
-            if (charsRef == null) {
-                charsRef = new CharsRefBuilder();
-            }
-            charsRef.copyUTF8Bytes(getBytesRef());
-            return charsRef.get();
-        }
+        CharSequence toUTF16();
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/context/GeolocationContextMapping.java b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/context/GeolocationContextMapping.java
index e131efce9e8..a5ed47db7c4 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/context/GeolocationContextMapping.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/context/GeolocationContextMapping.java
@@ -323,7 +323,7 @@ public class GeolocationContextMapping extends ContextMapping {
         return new GeoQuery(name, geohash, precisions);
     }
 
-    private static final int parsePrecision(XContentParser parser) throws IOException, ElasticsearchParseException {
+    private static int parsePrecision(XContentParser parser) throws IOException, ElasticsearchParseException {
         switch (parser.currentToken()) {
         case VALUE_STRING:
             return GeoUtils.geoHashLevelsForPrecision(parser.text());
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java
index fb59f2eddb0..b65d6fd3bb4 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorBuilder.java
@@ -428,7 +428,7 @@ public final class DirectCandidateGeneratorBuilder implements CandidateGenerator
      }
 
     @Override
-    public final String toString() {
+    public String toString() {
         try {
             XContentBuilder builder = XContentFactory.jsonBuilder();
             builder.prettyPrint();
@@ -440,14 +440,14 @@ public final class DirectCandidateGeneratorBuilder implements CandidateGenerator
     }
 
     @Override
-    public final int hashCode() {
+    public int hashCode() {
         return Objects.hash(field, preFilter, postFilter, suggestMode, accuracy,
                 size, sort, stringDistance, maxEdits, maxInspections,
                 maxTermFreq, prefixLength, minWordLength, minDocFreq);
     }
 
     @Override
-    public final boolean equals(Object obj) {
+    public boolean equals(Object obj) {
         if (this == obj) {
             return true;
         }
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java
index 89b2e763540..6c781a8ee91 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/Laplace.java
@@ -99,7 +99,7 @@ public final class Laplace extends SmoothingModel {
     }
 
     @Override
-    protected final int doHashCode() {
+    protected int doHashCode() {
         return Objects.hash(alpha);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java
index ba25766c8e0..72802e66eaa 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/LinearInterpolation.java
@@ -127,7 +127,7 @@ public final class LinearInterpolation extends SmoothingModel {
     }
 
     @Override
-    protected final int doHashCode() {
+    protected int doHashCode() {
         return Objects.hash(trigramLambda, bigramLambda, unigramLambda);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/MultiCandidateGeneratorWrapper.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/MultiCandidateGeneratorWrapper.java
index 4936973b237..18174d351e7 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/MultiCandidateGeneratorWrapper.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/MultiCandidateGeneratorWrapper.java
@@ -54,16 +54,10 @@ public final class MultiCandidateGeneratorWrapper extends CandidateGenerator {
         return reduce(set, numCandidates);
     }
 
-    private final CandidateSet reduce(CandidateSet set, int numCandidates) {
+    private CandidateSet reduce(CandidateSet set, int numCandidates) {
         if (set.candidates.length > numCandidates) {
             Candidate[] candidates = set.candidates;
-            Arrays.sort(candidates, new Comparator<Candidate>() {
-
-                @Override
-                public int compare(Candidate left, Candidate right) {
-                 return Double.compare(right.score, left.score);
-                }
-            });
+            Arrays.sort(candidates, (left, right) -> Double.compare(right.score, left.score));
             Candidate[] newSet = new Candidate[numCandidates];
             System.arraycopy(candidates, 0, newSet, 0, numCandidates);
             set.candidates = newSet;
diff --git a/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java b/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java
index 4160fcb42d4..3e67110ce7d 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/phrase/StupidBackoff.java
@@ -102,7 +102,7 @@ public final class StupidBackoff extends SmoothingModel {
     }
 
     @Override
-    protected final int doHashCode() {
+    protected int doHashCode() {
         return Objects.hash(discount);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java b/core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java
index dc8f26f1864..87e14c0366e 100644
--- a/core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java
+++ b/core/src/main/java/org/elasticsearch/transport/TcpTransportChannel.java
@@ -50,22 +50,22 @@ public final class TcpTransportChannel<Channel> implements TransportChannel {
     }
 
     @Override
-    public final String getProfileName() {
+    public String getProfileName() {
         return profileName;
     }
 
     @Override
-    public final String action() {
+    public String action() {
         return this.action;
     }
 
     @Override
-    public final void sendResponse(TransportResponse response) throws IOException {
+    public void sendResponse(TransportResponse response) throws IOException {
         sendResponse(response, TransportResponseOptions.EMPTY);
     }
 
     @Override
-    public final void sendResponse(TransportResponse response, TransportResponseOptions options) throws IOException {
+    public void sendResponse(TransportResponse response, TransportResponseOptions options) throws IOException {
         release();
         transport.sendResponse(version, channel, response, requestId, action, options);
 
@@ -86,12 +86,12 @@ public final class TcpTransportChannel<Channel> implements TransportChannel {
     }
 
     @Override
-    public final long getRequestId() {
+    public long getRequestId() {
         return requestId;
     }
 
     @Override
-    public final String getChannelType() {
+    public String getChannelType() {
         return channelType;
     }
 
diff --git a/core/src/test/java/org/apache/lucene/util/SloppyMathTests.java b/core/src/test/java/org/apache/lucene/util/SloppyMathTests.java
index abfc7c005e3..502d11e3b44 100644
--- a/core/src/test/java/org/apache/lucene/util/SloppyMathTests.java
+++ b/core/src/test/java/org/apache/lucene/util/SloppyMathTests.java
@@ -76,13 +76,13 @@ public class SloppyMathTests extends ESTestCase {
         assertThat("distance between("+lat1+", "+lon1+") and ("+lat2+", "+lon2+"))", sloppy, closeTo(accurate, maxError(accurate)));
     }
 
-    private static final double randomLatitude() {
+    private static double randomLatitude() {
         // crop pole areas, sine we now there the function
         // is not accurate around lat(89°, 90°) and lat(-90°, -89°)
         return (random().nextDouble() - 0.5) * 178.0;
     }
 
-    private static final double randomLongitude() {
+    private static double randomLongitude() {
         return (random().nextDouble() - 0.5) * 360.0;
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java b/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java
index b78425cd2a5..bbf1d2f1942 100644
--- a/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/TransportActionFilterChainTests.java
@@ -447,12 +447,12 @@ public class TransportActionFilterChainTests extends ESTestCase {
         }
     }
 
-    private static interface RequestCallback {
+    private interface RequestCallback {
         <Request extends ActionRequest<Request>, Response extends ActionResponse> void execute(Task task, String action, Request request,
                 ActionListener<Response> listener, ActionFilterChain<Request, Response> actionFilterChain);
     }
 
-    private static interface ResponseCallback {
+    private interface ResponseCallback {
         <Response extends ActionResponse> void execute(String action, Response response, ActionListener<Response> listener,
                 ActionFilterChain<?, Response> chain);
     }
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
index 6dd4df59060..f3ac9c8eff0 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
@@ -407,7 +407,7 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase {
         return clusterState;
     }
 
-    private final void assertRecoveryNodeVersions(RoutingNodes routingNodes) {
+    private void assertRecoveryNodeVersions(RoutingNodes routingNodes) {
         logger.trace("RoutingNodes: {}", routingNodes.prettyPrint());
 
         List<ShardRouting> mutableShardRoutings = routingNodes.shardsWithState(ShardRoutingState.RELOCATING);
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpClient.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpClient.java
index 264876b7963..0aeb00914e2 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpClient.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpClient.java
@@ -104,7 +104,7 @@ public class NettyHttpClient implements Closeable {
     }
 
     @SafeVarargs // Safe not because it doesn't do anything with the type parameters but because it won't leak them into other methods.
-    private final Collection<HttpResponse> processRequestsWithBody(HttpMethod method, SocketAddress remoteAddress, Tuple<String,
+    final Collection<HttpResponse> processRequestsWithBody(HttpMethod method, SocketAddress remoteAddress, Tuple<String,
         CharSequence>... urisAndBodies) throws InterruptedException {
         Collection<HttpRequest> requests = new ArrayList<>(urisAndBodies.length);
         for (Tuple<String, CharSequence> uriAndBody : urisAndBodies) {
diff --git a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
index eb70069b0b7..a8f8a9f802d 100644
--- a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
+++ b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java
@@ -646,12 +646,12 @@ public class CorruptedFileIT extends ESIntegTestCase {
         return shardRouting;
     }
 
-    private static final boolean isPerCommitFile(String fileName) {
+    private static boolean isPerCommitFile(String fileName) {
         // .liv and segments_N are per commit files and might change after corruption
         return fileName.startsWith("segments") || fileName.endsWith(".liv");
     }
 
-    private static final boolean isPerSegmentFile(String fileName) {
+    private static boolean isPerSegmentFile(String fileName) {
         return isPerCommitFile(fileName) == false;
     }
 
diff --git a/core/src/test/java/org/elasticsearch/rest/RestFilterChainTests.java b/core/src/test/java/org/elasticsearch/rest/RestFilterChainTests.java
index dd6d1dac47b..19b9051dd7e 100644
--- a/core/src/test/java/org/elasticsearch/rest/RestFilterChainTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/RestFilterChainTests.java
@@ -142,7 +142,7 @@ public class RestFilterChainTests extends ESTestCase {
         assertThat(fakeRestChannel.errors().get(), equalTo(additionalContinueCount));
     }
 
-    private static enum Operation implements Callback {
+    private enum Operation implements Callback {
         CONTINUE_PROCESSING {
             @Override
             public void execute(RestRequest request, RestChannel channel, NodeClient client, RestFilterChain filterChain) throws Exception {
@@ -157,7 +157,7 @@ public class RestFilterChainTests extends ESTestCase {
         }
     }
 
-    private static interface Callback {
+    private interface Callback {
         void execute(RestRequest request, RestChannel channel, NodeClient client, RestFilterChain filterChain) throws Exception;
     }
 
diff --git a/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java b/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java
index 875256a0f92..766aff8d274 100644
--- a/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java
+++ b/core/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java
@@ -623,7 +623,7 @@ public class MultiMatchQueryIT extends ESIntegTestCase {
         assertFirstHit(searchResponse, hasId("ultimate1"));
     }
 
-    private static final void assertEquivalent(String query, SearchResponse left, SearchResponse right) {
+    private static void assertEquivalent(String query, SearchResponse left, SearchResponse right) {
         assertNoFailures(left);
         assertNoFailures(right);
         SearchHits leftHits = left.getHits();
diff --git a/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java b/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java
index 7f9b9761feb..88af1ddf2b6 100644
--- a/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java
@@ -65,7 +65,7 @@ public class SearchAfterBuilderTests extends ESTestCase {
         indicesQueriesRegistry = null;
     }
 
-    private final SearchAfterBuilder randomSearchFromBuilder() throws IOException {
+    private SearchAfterBuilder randomSearchFromBuilder() throws IOException {
         int numSearchFrom = randomIntBetween(1, 10);
         SearchAfterBuilder searchAfterBuilder = new SearchAfterBuilder();
         Object[] values = new Object[numSearchFrom];
@@ -112,7 +112,7 @@ public class SearchAfterBuilderTests extends ESTestCase {
     // ensure that every number type remain the same before/after xcontent (de)serialization.
     // This is not a problem because the final type of each field value is extracted from associated sort field.
     // This little trick ensure that equals and hashcode are the same when using the xcontent serialization.
-    private final SearchAfterBuilder randomJsonSearchFromBuilder() throws IOException {
+    private SearchAfterBuilder randomJsonSearchFromBuilder() throws IOException {
         int numSearchAfter = randomIntBetween(1, 10);
         XContentBuilder jsonBuilder = XContentFactory.jsonBuilder();
         jsonBuilder.startObject();
diff --git a/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
index a0ce8f02ea3..668351c6db9 100644
--- a/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
+++ b/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java
@@ -88,7 +88,7 @@ public class SliceBuilderTests extends ESTestCase {
         indicesQueriesRegistry = null;
     }
 
-    private final SliceBuilder randomSliceBuilder() throws IOException {
+    private SliceBuilder randomSliceBuilder() throws IOException {
         int max = randomIntBetween(2, MAX_SLICE);
         int id = randomInt(max - 1);
         String field = randomAsciiOfLengthBetween(5, 20);
diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParser.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParser.java
index dd2b69696f3..0de5e13c058 100644
--- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParser.java
+++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceParser.java
@@ -165,8 +165,8 @@ public abstract class MultiValuesSourceParser<VS extends ValuesSource> implement
         return factory;
     }
 
-    private final void parseMissingAndAdd(final String aggregationName, final String currentFieldName,
-                                          XContentParser parser, final Map<String, Object> missing) throws IOException {
+    private void parseMissingAndAdd(final String aggregationName, final String currentFieldName,
+                                    XContentParser parser, final Map<String, Object> missing) throws IOException {
         XContentParser.Token token = parser.currentToken();
         if (token == null) {
             token = parser.nextToken();
diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
index 558bdde96a2..9b087fc208f 100644
--- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
+++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
@@ -93,7 +93,7 @@ public final class ConvertProcessor extends AbstractProcessor {
         };
 
         @Override
-        public final String toString() {
+        public String toString() {
             return name().toLowerCase(Locale.ROOT);
         }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java
index 94d7a1305d6..f0e1bde74d0 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/CompilerSettings.java
@@ -60,7 +60,7 @@ public final class CompilerSettings {
      * in a script before an exception is thrown.  This attempts to prevent infinite loops.  Note if
      * the counter is set to 0, no loop counter will be written.
      */
-    public final int getMaxLoopCounter() {
+    public int getMaxLoopCounter() {
         return maxLoopCounter;
     }
 
@@ -68,7 +68,7 @@ public final class CompilerSettings {
      * Set the cumulative total number of statements that can be made in all loops.
      * @see #getMaxLoopCounter
      */
-    public final void setMaxLoopCounter(int max) {
+    public void setMaxLoopCounter(int max) {
         this.maxLoopCounter = max;
     }
 
diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
index e57aad862aa..69a74301bc8 100644
--- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
+++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -646,7 +646,7 @@ public final class Definition {
         }
     }
 
-    private final void addStruct(final String name, final Class<?> clazz) {
+    private void addStruct(final String name, final Class<?> clazz) {
         if (!name.matches("^[_a-zA-Z][\\.,_a-zA-Z0-9]*$")) {
             throw new IllegalArgumentException("Invalid struct name [" + name + "].");
         }
@@ -661,7 +661,7 @@ public final class Definition {
         simpleTypesMap.put(name, getTypeInternal(name));
     }
 
-    private final void addConstructorInternal(final String struct, final String name, final Type[] args) {
+    private void addConstructorInternal(final String struct, final String name, final Type[] args) {
         final Struct owner = structsMap.get(struct);
 
         if (owner == null) {
@@ -734,7 +734,7 @@ public final class Definition {
      * </ul>
      * no spaces allowed.
      */
-    private final void addSignature(String className, String signature) {
+    private void addSignature(String className, String signature) {
         String elements[] = signature.split("\u0020");
         if (elements.length != 2) {
             throw new IllegalArgumentException("Malformed signature: " + signature);
@@ -774,8 +774,8 @@ public final class Definition {
         }
     }
 
-    private final void addMethodInternal(String struct, String name, boolean augmentation,
-                                         Type rtn, Type[] args) {
+    private void addMethodInternal(String struct, String name, boolean augmentation,
+                                   Type rtn, Type[] args) {
         final Struct owner = structsMap.get(struct);
 
         if (owner == null) {
@@ -858,7 +858,7 @@ public final class Definition {
         }
     }
 
-    private final void addFieldInternal(String struct, String name, Type type) {
+    private void addFieldInternal(String struct, String name, Type type) {
         final Struct owner = structsMap.get(struct);
 
         if (owner == null) {
diff --git a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
index b642b7b0a73..5a16c06d4dc 100644
--- a/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
+++ b/plugins/lang-python/src/main/java/org/elasticsearch/script/python/PythonScriptEngineService.java
@@ -261,7 +261,7 @@ public class PythonScriptEngineService extends AbstractComponent implements Scri
     }
 
     /** Evaluates with reduced privileges */
-    private final PyObject evalRestricted(final PyCode code) {
+    private PyObject evalRestricted(final PyCode code) {
         // eval the script with reduced privileges
         return AccessController.doPrivileged(new PrivilegedAction<PyObject>() {
             @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java b/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java
index 73281b3f6ea..37ed43b9450 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/engine/ThrowingLeafReaderWrapper.java
@@ -62,18 +62,18 @@ public class ThrowingLeafReaderWrapper extends FilterLeafReader {
      * A callback interface that allows to throw certain exceptions for
      * methods called on the IndexReader that is wrapped by {@link ThrowingLeafReaderWrapper}
      */
-    public static interface Thrower {
+    public interface Thrower {
         /**
          * Maybe throws an exception ;)
          */
-        public void maybeThrow(Flags flag) throws IOException;
+        void maybeThrow(Flags flag) throws IOException;
 
         /**
          * If this method returns true the {@link Terms} instance for the given field
          * is wrapped with Thrower support otherwise no exception will be thrown for
          * the current {@link Terms} instance or any other instance obtained from it.
          */
-        public boolean wrapTerms(String field);
+        boolean wrapTerms(String field);
     }
 
     public ThrowingLeafReaderWrapper(LeafReader in, Thrower thrower) {

From d72134b46eba627cc45b5646620172d9bd686e9b Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Mon, 4 Jul 2016 15:58:27 +0200
Subject: [PATCH 1224/1311] Fix CompletionSuggestSearchIT and
 CompletionSuggestSearch2xIT

---
 .../completion2x/CompletionTokenStream.java   | 47 ++++++++++++++++++-
 1 file changed, 45 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/CompletionTokenStream.java b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/CompletionTokenStream.java
index 2c752d64175..bdfcb9eff11 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/CompletionTokenStream.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/CompletionTokenStream.java
@@ -23,8 +23,11 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.AttributeReflector;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.lucene.util.CharsRefBuilder;
 import org.apache.lucene.util.IntsRef;
 import org.apache.lucene.util.fst.Util;
 
@@ -105,8 +108,8 @@ public final class CompletionTokenStream extends TokenStream {
         input.close();
     }
 
-    public interface ToFiniteStrings {
-        Set<IntsRef> toFiniteStrings(TokenStream stream) throws IOException;
+    public static interface ToFiniteStrings {
+        public Set<IntsRef> toFiniteStrings(TokenStream stream) throws IOException;
     }
 
     @Override
@@ -130,4 +133,44 @@ public final class CompletionTokenStream extends TokenStream {
 
         CharSequence toUTF16();
     }
+
+    public static final class ByteTermAttributeImpl extends AttributeImpl implements ByteTermAttribute, TermToBytesRefAttribute {
+        private final BytesRefBuilder bytes = new BytesRefBuilder();
+        private CharsRefBuilder charsRef;
+
+        @Override
+        public BytesRefBuilder builder() {
+            return bytes;
+        }
+
+        @Override
+        public BytesRef getBytesRef() {
+            return bytes.get();
+        }
+
+        @Override
+        public void clear() {
+            bytes.clear();
+        }
+
+        @Override
+        public void reflectWith(AttributeReflector reflector) {
+
+        }
+
+        @Override
+        public void copyTo(AttributeImpl target) {
+            ByteTermAttributeImpl other = (ByteTermAttributeImpl) target;
+            other.bytes.copyBytes(bytes);
+        }
+
+        @Override
+        public CharSequence toUTF16() {
+            if (charsRef == null) {
+                charsRef = new CharsRefBuilder();
+            }
+            charsRef.copyUTF8Bytes(getBytesRef());
+            return charsRef.get();
+        }
+    }
 }

From ed444cd276cafb3d537141300fce032ad5078827 Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Mon, 4 Jul 2016 16:29:26 +0200
Subject: [PATCH 1225/1311] Fix CompletionTokenStream modifier redundancy

---
 .../search/suggest/completion2x/CompletionTokenStream.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/CompletionTokenStream.java b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/CompletionTokenStream.java
index bdfcb9eff11..9cb9891add0 100644
--- a/core/src/main/java/org/elasticsearch/search/suggest/completion2x/CompletionTokenStream.java
+++ b/core/src/main/java/org/elasticsearch/search/suggest/completion2x/CompletionTokenStream.java
@@ -108,8 +108,8 @@ public final class CompletionTokenStream extends TokenStream {
         input.close();
     }
 
-    public static interface ToFiniteStrings {
-        public Set<IntsRef> toFiniteStrings(TokenStream stream) throws IOException;
+    public interface ToFiniteStrings {
+        Set<IntsRef> toFiniteStrings(TokenStream stream) throws IOException;
     }
 
     @Override

From 6861d3571e6ded1b9718aea55d20072bcbf632d2 Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Mon, 4 Jul 2016 21:09:25 +0200
Subject: [PATCH 1226/1311] Persistent Node Ids (#19140)

Node IDs are currently randomly generated during node startup. That means they change every time the node is restarted. While this doesn't matter for ES proper, it makes it hard for external services to track nodes. Another, more minor, side effect is that indexing the output of, say, the node stats API results in creating new fields due to node ID being used as keys.

The first approach I considered was to use the node's published address as the base for the id. We already [treat nodes with the same address as the same](https://github.com/elastic/elasticsearch/blob/master/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java#L387) so this is a simple change (see [here](https://github.com/elastic/elasticsearch/compare/master...bleskes:node_persistent_id_based_on_address)). While this is simple and it works for probably most cases, it is not perfect. For example, if after a node restart, the node is not able to bind to the same port (because it's not yet freed by the OS), it will cause the node to still change identity. Also in environments where the host IP can change due to a host restart, identity will not be the same.

Due to those limitation, I opted to go with a different approach where the node id will be persisted in the node's data folder. This has the upside of connecting the id to the nodes data. It also means that the host can be adapted in any way (replace network cards, attach storage to a new VM). I

It does however also have downsides - we now run the risk of two nodes having the same id, if someone copies clones a data folder from one node to another. To mitigate this I changed the semantics of the protection against multiple nodes with the same address to be stricter - it will now reject the incoming join if a node exists with the same id but a different address. Note that if the existing node doesn't respond to pings (i.e., it's not alive) it will be removed and the new node will be accepted when it tries another join.

Last, and most importantly, this change requires that *all* nodes persist data to disk. This is a change from current behavior where only data & master nodes store local files. This is the main reason for marking this PR as breaking.

Other less important notes:
- DummyTransportAddress is removed as we need a unique network address per node. Use `LocalTransportAddress.buildUnique()` instead.
- I renamed `node.add_lid_to_custom_path` to `node.add_lock_id_to_custom_path` to avoid confusion with the node ID which is now part of the `NodeEnvironment` logic.
- I removed the `version` paramater from `MetaDataStateFormat#write` , it wasn't really used and was just in the way :)
- TribeNodes are special in the sense that they do start multiple sub-nodes (previously known as client nodes). Those sub-nodes do not store local files but derive their ID from the parent node id, so they are generated consistently.
---
 .../routing/allocation/Allocators.java        |   4 +-
 .../resources/checkstyle_suppressions.xml     |   6 -
 .../TransportClientNodesService.java          |   6 +-
 .../cluster/node/DiscoveryNode.java           | 111 ++++++++++-------
 .../cluster/node/DiscoveryNodeService.java    |  21 +---
 .../cluster/node/DiscoveryNodes.java          |  80 ++++++++++--
 .../common/hash/MurmurHash3.java              |   5 +-
 .../common/settings/ClusterSettings.java      |  10 +-
 .../transport/DummyTransportAddress.java      |  74 -----------
 .../transport/LocalTransportAddress.java      |  10 ++
 .../TransportAddressSerializers.java          |   1 -
 .../discovery/local/LocalDiscovery.java       |  11 +-
 .../discovery/zen/NodeJoinController.java     |  22 ++--
 .../discovery/zen/ZenDiscovery.java           |   9 +-
 .../zen/fd/MasterFaultDetection.java          |  57 +++++----
 .../discovery/zen/fd/NodesFaultDetection.java |  39 +++---
 .../zen/ping/unicast/UnicastZenPing.java      |   9 +-
 .../publish/PublishClusterStateAction.java    |   4 +-
 .../elasticsearch/env/NodeEnvironment.java    |  87 ++++++++++---
 .../org/elasticsearch/env/NodeMetaData.java   | 116 ++++++++++++++++++
 .../gateway/MetaDataStateFormat.java          |  28 +++--
 .../gateway/MetaStateService.java             |   4 +-
 .../elasticsearch/index/shard/IndexShard.java |   2 +-
 .../java/org/elasticsearch/node/Node.java     |  13 +-
 .../org/elasticsearch/tribe/TribeService.java |  23 +++-
 .../ClusterAllocationExplanationTests.java    |   4 +-
 .../IndicesShardStoreResponseTests.java       |   8 +-
 .../shrink/TransportShrinkActionTests.java    |   4 +-
 .../ingest/IngestProxyActionFilterTests.java  |   4 +-
 .../TransportBroadcastByNodeActionTests.java  |   4 +-
 .../TransportMasterNodeActionTests.java       |   6 +-
 .../nodes/TransportNodesActionTests.java      |   4 +-
 .../ClusterStateCreationUtils.java            |   4 +-
 .../TransportClientNodesServiceTests.java     |   2 +-
 .../cluster/ClusterChangedEventTests.java     |   5 +-
 .../cluster/ClusterStateDiffIT.java           |   4 +-
 .../cluster/ClusterStateTests.java            |   7 +-
 .../elasticsearch/cluster/DiskUsageTests.java |  16 ++-
 .../cluster/NodeConnectionsServiceTests.java  |   4 +-
 ...rdFailedClusterStateTaskExecutorTests.java |   5 +-
 .../MetaDataCreateIndexServiceTests.java      |   4 +-
 .../node/DiscoveryNodeFiltersTests.java       |  48 ++++----
 .../node/DiscoveryNodeServiceTests.java       |  10 +-
 .../cluster/node/DiscoveryNodesTests.java     |  96 +++++++++++++--
 .../NodeVersionAllocationDeciderTests.java    |  18 +--
 .../allocation/SameShardRoutingTests.java     |   8 +-
 .../DiskThresholdDeciderUnitTests.java        |   9 +-
 .../ClusterStateToStringTests.java            |   4 +-
 .../cluster/service/ClusterServiceTests.java  |   4 +-
 .../common/util/IndexFolderUpgraderTests.java |  14 +--
 ...usterStatePublishResponseHandlerTests.java |   4 +-
 .../zen/ElectMasterServiceTests.java          |   4 +-
 .../zen/NodeJoinControllerTests.java          |  55 +++++++--
 .../discovery/zen/ZenDiscoveryUnitTests.java  |  11 +-
 .../discovery/zen/ZenPingTests.java           |   4 +-
 .../PendingClusterStatesQueueTests.java       |   4 +-
 .../PublishClusterStateActionTests.java       |  10 +-
 .../env/NodeEnvironmentTests.java             |  24 +++-
 .../gateway/AsyncShardFetchTests.java         |   6 +-
 .../gateway/GatewayIndexStateIT.java          |   8 +-
 .../gateway/MetaDataStateFormatTests.java     |  10 +-
 .../index/IndexWithShadowReplicasIT.java      |   4 +-
 .../ESIndexLevelReplicationTestCase.java      |   4 +-
 .../index/shard/IndexShardTests.java          |  22 ++--
 .../index/shard/ShardPathTests.java           |  10 +-
 .../IndexingMemoryControllerTests.java        |   5 +-
 ...dicesLifecycleListenerSingleNodeTests.java |   4 +-
 ...ClusterStateServiceRandomUpdatesTests.java |   6 +-
 .../recovery/RecoverySourceHandlerTests.java  |  14 +--
 .../indices/recovery/RecoveryTargetTests.java |   5 +-
 .../indices/state/RareClusterStateIT.java     |   4 +-
 .../template/SimpleIndexTemplateIT.java       |   1 -
 .../ingest/ConfigurationUtilsTests.java       |   2 -
 .../ingest/PipelineStoreTests.java            |   2 +-
 .../nodesinfo/NodeInfoStreamingTests.java     |   6 +-
 .../nodesinfo/SimpleNodesInfoIT.java          |   2 +-
 .../recovery/RecoveriesCollectionTests.java   |   8 +-
 .../tribe/TribeServiceTests.java              |  14 ++-
 .../indices/shadow-replicas.asciidoc          |   6 +-
 .../migration/migrate_5_0/fs.asciidoc         |   6 +
 .../migration/migrate_5_0/settings.asciidoc   |   2 +
 .../elasticsearch/tribe/TribeUnitTests.java   |   6 +-
 .../org/elasticsearch/tribe/elasticsearch.yml |   4 +-
 .../MockInternalClusterInfoService.java       |   4 +-
 .../test/ClusterServiceUtils.java             |   4 +-
 .../test/ESAllocationTestCase.java            |  10 +-
 .../test/InternalTestCluster.java             |   8 +-
 .../test/test/InternalTestClusterTests.java   |   5 +-
 88 files changed, 862 insertions(+), 494 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/common/transport/DummyTransportAddress.java
 create mode 100644 core/src/main/java/org/elasticsearch/env/NodeMetaData.java

diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java
index aba7fda1021..97fbda80dc6 100644
--- a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java
+++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/Allocators.java
@@ -31,7 +31,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.gateway.GatewayAllocator;
 
@@ -102,7 +102,7 @@ public final class Allocators {
     }
 
     public static DiscoveryNode newNode(String nodeId, Map<String, String> attributes) {
-        return new DiscoveryNode("", nodeId, DummyTransportAddress.INSTANCE, attributes, Sets.newHashSet(DiscoveryNode.Role.MASTER,
+        return new DiscoveryNode("", nodeId, LocalTransportAddress.buildUnique(), attributes, Sets.newHashSet(DiscoveryNode.Role.MASTER,
             DiscoveryNode.Role.DATA), Version.CURRENT);
     }
 }
diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 6f65428c2bd..c73545c8a7c 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -266,7 +266,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]MetaDataMappingService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]MetaDataUpdateSettingsService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]metadata[/\\]RepositoriesMetaData.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]node[/\\]DiscoveryNodes.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]IndexRoutingTable.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]IndexShardRoutingTable.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cluster[/\\]routing[/\\]OperationRouting.java" checks="LineLength" />
@@ -341,12 +340,9 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]DiscoveryService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]DiscoverySettings.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]local[/\\]LocalDiscovery.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]NodeJoinController.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]ZenDiscovery.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]elect[/\\]ElectMasterService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]fd[/\\]FaultDetection.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]fd[/\\]MasterFaultDetection.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]fd[/\\]NodesFaultDetection.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]membership[/\\]MembershipAction.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]ping[/\\]ZenPing.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]publish[/\\]PendingClusterStatesQueue.java" checks="LineLength" />
@@ -357,7 +353,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]GatewayMetaState.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]GatewayService.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]LocalAllocateDangledIndices.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]MetaDataStateFormat.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]PrimaryShardAllocator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]ReplicaShardAllocator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]TransportNodesListGatewayMetaState.java" checks="LineLength" />
@@ -847,7 +842,6 @@
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]BlockingClusterStatePublishResponseHandlerTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]DiscoveryWithServiceDisruptionsIT.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]ZenUnicastDiscoveryIT.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]NodeJoinControllerTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]ZenDiscoveryUnitTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]discovery[/\\]zen[/\\]publish[/\\]PublishClusterStateActionTests.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]document[/\\]DocumentActionsIT.java" checks="LineLength" />
diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
index c751b9c43c1..7c2efa5fff9 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
@@ -397,9 +397,9 @@ public class TransportClientNodesService extends AbstractComponent implements Cl
                         // use discovered information but do keep the original transport address,
                         // so people can control which address is exactly used.
                         DiscoveryNode nodeWithInfo = livenessResponse.getDiscoveryNode();
-                        newNodes.add(new DiscoveryNode(nodeWithInfo.getName(), nodeWithInfo.getId(), nodeWithInfo.getHostName(),
-                                nodeWithInfo.getHostAddress(), listedNode.getAddress(), nodeWithInfo.getAttributes(),
-                                nodeWithInfo.getRoles(), nodeWithInfo.getVersion()));
+                        newNodes.add(new DiscoveryNode(nodeWithInfo.getName(), nodeWithInfo.getId(), nodeWithInfo.getEphemeralId(),
+                            nodeWithInfo.getHostName(), nodeWithInfo.getHostAddress(), listedNode.getAddress(),
+                            nodeWithInfo.getAttributes(), nodeWithInfo.getRoles(), nodeWithInfo.getVersion()));
                     } else {
                         // although we asked for one node, our target may not have completed
                         // initialization yet and doesn't have cluster nodes
diff --git a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
index ff37b6e6d4c..1cc79f3525e 100644
--- a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
+++ b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
@@ -20,7 +20,7 @@
 package org.elasticsearch.cluster.node;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -64,7 +64,15 @@ public class DiscoveryNode implements Writeable, ToXContent {
     }
 
     public static boolean nodeRequiresLocalStorage(Settings settings) {
-        return Node.NODE_DATA_SETTING.get(settings) || Node.NODE_MASTER_SETTING.get(settings);
+        boolean localStorageEnable = Node.NODE_LOCAL_STORAGE_SETTING.get(settings);
+        if (localStorageEnable == false &&
+            (Node.NODE_DATA_SETTING.get(settings) ||
+                Node.NODE_MASTER_SETTING.get(settings))
+            ) {
+            // TODO: make this a proper setting validation logic, requiring multi-settings validation
+            throw new IllegalArgumentException("storage can not be disabled for master and data nodes");
+        }
+        return localStorageEnable;
     }
 
     public static boolean isMasterNode(Settings settings) {
@@ -81,6 +89,7 @@ public class DiscoveryNode implements Writeable, ToXContent {
 
     private final String nodeName;
     private final String nodeId;
+    private final String ephemeralId;
     private final String hostName;
     private final String hostAddress;
     private final TransportAddress address;
@@ -97,14 +106,15 @@ public class DiscoveryNode implements Writeable, ToXContent {
      * and updated.
      * </p>
      *
-     * @param nodeId     the nodes unique id.
-     * @param address    the nodes transport address
-     * @param attributes node attributes
-     * @param roles      node roles
-     * @param version    the version of the node.
+     * @param id               the nodes unique (persistent) node id. This constructor will auto generate a random ephemeral id.
+     * @param address          the nodes transport address
+     * @param attributes       node attributes
+     * @param roles            node roles
+     * @param version          the version of the node
      */
-    public DiscoveryNode(String nodeId, TransportAddress address, Map<String, String> attributes, Set<Role> roles, Version version) {
-        this("", nodeId, address.getHost(), address.getAddress(), address, attributes, roles, version);
+    public DiscoveryNode(String id, TransportAddress address, Map<String, String> attributes, Set<Role> roles,
+                         Version version) {
+        this("", id, address, attributes, roles, version);
     }
 
     /**
@@ -116,16 +126,16 @@ public class DiscoveryNode implements Writeable, ToXContent {
      * and updated.
      * </p>
      *
-     * @param nodeName   the nodes name
-     * @param nodeId     the nodes unique id.
-     * @param address    the nodes transport address
-     * @param attributes node attributes
-     * @param roles      node roles
-     * @param version    the version of the node.
+     * @param nodeName         the nodes name
+     * @param nodeId           the nodes unique persistent id. An ephemeral id will be auto generated.
+     * @param address          the nodes transport address
+     * @param attributes       node attributes
+     * @param roles            node roles
+     * @param version          the version of the node
      */
-    public DiscoveryNode(String nodeName, String nodeId, TransportAddress address, Map<String, String> attributes,
-                         Set<Role> roles, Version version) {
-        this(nodeName, nodeId, address.getHost(), address.getAddress(), address, attributes, roles, version);
+    public DiscoveryNode(String nodeName, String nodeId, TransportAddress address,
+                         Map<String, String> attributes, Set<Role> roles, Version version) {
+        this(nodeName, nodeId, UUIDs.randomBase64UUID(), address.getHost(), address.getAddress(), address, attributes, roles, version);
     }
 
     /**
@@ -137,23 +147,24 @@ public class DiscoveryNode implements Writeable, ToXContent {
      * and updated.
      * </p>
      *
-     * @param nodeName    the nodes name
-     * @param nodeId      the nodes unique id.
-     * @param hostName    the nodes hostname
-     * @param hostAddress the nodes host address
-     * @param address     the nodes transport address
-     * @param attributes  node attributes
-     * @param roles       node roles
-     * @param version     the version of the node.
+     * @param nodeName         the nodes name
+     * @param nodeId           the nodes unique persistent id
+     * @param ephemeralId      the nodes unique ephemeral id
+     * @param hostAddress      the nodes host address
+     * @param address          the nodes transport address
+     * @param attributes       node attributes
+     * @param roles            node roles
+     * @param version          the version of the node
      */
-    public DiscoveryNode(String nodeName, String nodeId, String hostName, String hostAddress, TransportAddress address,
-                         Map<String, String> attributes, Set<Role> roles, Version version) {
+    public DiscoveryNode(String nodeName, String nodeId, String ephemeralId, String hostName, String hostAddress,
+                         TransportAddress address, Map<String, String> attributes, Set<Role> roles, Version version) {
         if (nodeName != null) {
             this.nodeName = nodeName.intern();
         } else {
             this.nodeName = "";
         }
         this.nodeId = nodeId.intern();
+        this.ephemeralId = ephemeralId.intern();
         this.hostName = hostName.intern();
         this.hostAddress = hostAddress.intern();
         this.address = address;
@@ -184,6 +195,7 @@ public class DiscoveryNode implements Writeable, ToXContent {
     public DiscoveryNode(StreamInput in) throws IOException {
         this.nodeName = in.readString().intern();
         this.nodeId = in.readString().intern();
+        this.ephemeralId = in.readString().intern();
         this.hostName = in.readString().intern();
         this.hostAddress = in.readString().intern();
         this.address = TransportAddressSerializers.addressFromStream(in);
@@ -208,6 +220,7 @@ public class DiscoveryNode implements Writeable, ToXContent {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(nodeName);
         out.writeString(nodeId);
+        out.writeString(ephemeralId);
         out.writeString(hostName);
         out.writeString(hostAddress);
         addressToStream(out, address);
@@ -237,6 +250,17 @@ public class DiscoveryNode implements Writeable, ToXContent {
         return nodeId;
     }
 
+    /**
+     * The unique ephemeral id of the node. Ephemeral ids are meant to be attached the the life span
+     * of a node process. When ever a node is restarted, it's ephemeral id is required to change (while it's {@link #getId()}
+     * will be read from the data folder and will remain the same across restarts). Since all node attributes and addresses
+     * are maintained during the life span of a node process, we can (and are) using the ephemeralId in
+     * {@link DiscoveryNode#equals(Object)}.
+     */
+    public String getEphemeralId() {
+        return ephemeralId;
+    }
+
     /**
      * The name of the node.
      */
@@ -293,18 +317,25 @@ public class DiscoveryNode implements Writeable, ToXContent {
     }
 
     @Override
-    public boolean equals(Object obj) {
-        if (!(obj instanceof DiscoveryNode)) {
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
             return false;
         }
 
-        DiscoveryNode other = (DiscoveryNode) obj;
-        return this.nodeId.equals(other.nodeId);
+        DiscoveryNode that = (DiscoveryNode) o;
+
+        return ephemeralId.equals(that.ephemeralId);
     }
 
     @Override
     public int hashCode() {
-        return nodeId.hashCode();
+        // we only need to hash the id because it's highly unlikely that two nodes
+        // in our system will have the same id but be different
+        // This is done so that this class can be used efficiently as a key in a map
+        return ephemeralId.hashCode();
     }
 
     @Override
@@ -313,15 +344,10 @@ public class DiscoveryNode implements Writeable, ToXContent {
         if (nodeName.length() > 0) {
             sb.append('{').append(nodeName).append('}');
         }
-        if (nodeId != null) {
-            sb.append('{').append(nodeId).append('}');
-        }
-        if (Strings.hasLength(hostName)) {
-            sb.append('{').append(hostName).append('}');
-        }
-        if (address != null) {
-            sb.append('{').append(address).append('}');
-        }
+        sb.append('{').append(nodeId).append('}');
+        sb.append('{').append(ephemeralId).append('}');
+        sb.append('{').append(hostName).append('}');
+        sb.append('{').append(address).append('}');
         if (!attributes.isEmpty()) {
             sb.append(attributes);
         }
@@ -332,6 +358,7 @@ public class DiscoveryNode implements Writeable, ToXContent {
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject(getId());
         builder.field("name", getName());
+        builder.field("ephemeral_id", getEphemeralId());
         builder.field("transport_address", getAddress().toString());
 
         builder.startObject("attributes");
diff --git a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeService.java b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeService.java
index 06d0c7dbf24..a91b216eac5 100644
--- a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeService.java
@@ -20,12 +20,8 @@
 package org.elasticsearch.cluster.node;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.common.Randomness;
-import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.node.Node;
@@ -34,17 +30,14 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.function.Supplier;
 
 /**
  */
 public class DiscoveryNodeService extends AbstractComponent {
 
-    public static final Setting<Long> NODE_ID_SEED_SETTING =
-            // don't use node.id.seed so it won't be seen as an attribute
-            Setting.longSetting("node_id.seed", 0L, Long.MIN_VALUE, Property.NodeScope);
     private final List<CustomAttributesProvider> customAttributesProviders = new CopyOnWriteArrayList<>();
 
     @Inject
@@ -52,18 +45,12 @@ public class DiscoveryNodeService extends AbstractComponent {
         super(settings);
     }
 
-    public static String generateNodeId(Settings settings) {
-        Random random = Randomness.get(settings, NODE_ID_SEED_SETTING);
-        return UUIDs.randomBase64UUID(random);
-    }
-
     public DiscoveryNodeService addCustomAttributeProvider(CustomAttributesProvider customAttributesProvider) {
         customAttributesProviders.add(customAttributesProvider);
         return this;
     }
 
-    public DiscoveryNode buildLocalNode(TransportAddress publishAddress) {
-        final String nodeId = generateNodeId(settings);
+    public DiscoveryNode buildLocalNode(TransportAddress publishAddress, Supplier<String> nodeIdSupplier) {
         Map<String, String> attributes = new HashMap<>(Node.NODE_ATTRIBUTES.get(this.settings).getAsMap());
         Set<DiscoveryNode.Role> roles = new HashSet<>();
         if (Node.NODE_INGEST_SETTING.get(settings)) {
@@ -90,8 +77,8 @@ public class DiscoveryNodeService extends AbstractComponent {
                 logger.warn("failed to build custom attributes from provider [{}]", e, provider);
             }
         }
-        return new DiscoveryNode(Node.NODE_NAME_SETTING.get(settings), nodeId, publishAddress, attributes,
-                roles, Version.CURRENT);
+        return new DiscoveryNode(Node.NODE_NAME_SETTING.get(settings), nodeIdSupplier.get(), publishAddress, attributes, roles,
+            Version.CURRENT);
     }
 
     public interface CustomAttributesProvider {
diff --git a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
index 43d71de726a..68dedc433da 100644
--- a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
+++ b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java
@@ -153,7 +153,7 @@ public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements
     }
 
     /**
-     * Determine if a given node exists
+     * Determine if a given node id exists
      *
      * @param nodeId id of the node which existence should be verified
      * @return <code>true</code> if the node exists. Otherwise <code>false</code>
@@ -162,6 +162,18 @@ public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements
         return nodes.containsKey(nodeId);
     }
 
+    /**
+     * Determine if a given node exists
+     *
+     * @param node of the node which existence should be verified
+     * @return <code>true</code> if the node exists. Otherwise <code>false</code>
+     */
+    public boolean nodeExists(DiscoveryNode node) {
+        DiscoveryNode existing = nodes.get(node.getId());
+        return existing != null && existing.equals(node);
+    }
+
+
     /**
      * Get the id of the master node
      *
@@ -225,7 +237,7 @@ public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements
      * @return the oldest version in the cluster
      */
     public Version getSmallestVersion() {
-       return minNodeVersion;
+        return minNodeVersion;
     }
 
     /**
@@ -247,7 +259,8 @@ public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements
     public DiscoveryNode resolveNode(String node) {
         String[] resolvedNodeIds = resolveNodes(node);
         if (resolvedNodeIds.length > 1) {
-            throw new IllegalArgumentException("resolved [" + node + "] into [" + resolvedNodeIds.length + "] nodes, where expected to be resolved to a single node");
+            throw new IllegalArgumentException("resolved [" + node + "] into [" + resolvedNodeIds.length
+                + "] nodes, where expected to be resolved to a single node");
         }
         if (resolvedNodeIds.length == 0) {
             throw new IllegalArgumentException("failed to resolve [" + node + "], no matching nodes");
@@ -361,12 +374,12 @@ public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements
         List<DiscoveryNode> removed = new ArrayList<>();
         List<DiscoveryNode> added = new ArrayList<>();
         for (DiscoveryNode node : other) {
-            if (!this.nodeExists(node.getId())) {
+            if (!this.nodeExists(node)) {
                 removed.add(node);
             }
         }
         for (DiscoveryNode node : this) {
-            if (!other.nodeExists(node.getId())) {
+            if (!other.nodeExists(node)) {
                 added.add(node);
             }
         }
@@ -378,7 +391,8 @@ public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements
                 newMasterNode = getMasterNode();
             }
         }
-        return new Delta(previousMasterNode, newMasterNode, localNodeId, Collections.unmodifiableList(removed), Collections.unmodifiableList(added));
+        return new Delta(previousMasterNode, newMasterNode, localNodeId, Collections.unmodifiableList(removed),
+            Collections.unmodifiableList(added));
     }
 
     @Override
@@ -420,7 +434,8 @@ public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements
             this(null, null, localNodeId, removed, added);
         }
 
-        public Delta(@Nullable DiscoveryNode previousMasterNode, @Nullable DiscoveryNode newMasterNode, String localNodeId, List<DiscoveryNode> removed, List<DiscoveryNode> added) {
+        public Delta(@Nullable DiscoveryNode previousMasterNode, @Nullable DiscoveryNode newMasterNode, String localNodeId,
+                     List<DiscoveryNode> removed, List<DiscoveryNode> added) {
             this.previousMasterNode = previousMasterNode;
             this.newMasterNode = newMasterNode;
             this.localNodeId = localNodeId;
@@ -538,7 +553,10 @@ public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements
                 // reuse the same instance of our address and local node id for faster equality
                 node = localNode;
             }
-            builder.put(node);
+            // some one already built this and validated it's OK, skip the n2 scans
+            assert builder.validatePut(node) == null : "building disco nodes from network doesn't pass preflight: "
+                + builder.validatePut(node);
+            builder.putUnsafe(node);
         }
         return builder.build();
     }
@@ -572,16 +590,36 @@ public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements
             this.nodes = ImmutableOpenMap.builder(nodes.getNodes());
         }
 
+        /**
+         * adds a disco node to the builder. Will throw an {@link IllegalArgumentException} if
+         * the supplied node doesn't pass the pre-flight checks performed by {@link #validatePut(DiscoveryNode)}
+         */
         public Builder put(DiscoveryNode node) {
-            nodes.put(node.getId(), node);
+            final String preflight = validatePut(node);
+            if (preflight != null) {
+                throw new IllegalArgumentException(preflight);
+            }
+            putUnsafe(node);
             return this;
         }
 
+        private void putUnsafe(DiscoveryNode node) {
+            nodes.put(node.getId(), node);
+        }
+
         public Builder remove(String nodeId) {
             nodes.remove(nodeId);
             return this;
         }
 
+        public Builder remove(DiscoveryNode node) {
+            if (node.equals(nodes.get(node.getId()))) {
+                nodes.remove(node.getId());
+            }
+            return this;
+        }
+
+
         public Builder masterNodeId(String masterNodeId) {
             this.masterNodeId = masterNodeId;
             return this;
@@ -592,6 +630,30 @@ public class DiscoveryNodes extends AbstractDiffable<DiscoveryNodes> implements
             return this;
         }
 
+        /**
+         * Checks that a node can be safely added to this node collection.
+         *
+         * @return null if all is OK or an error message explaining why a node can not be added.
+         *
+         * Note: if this method returns a non-null value, calling {@link #put(DiscoveryNode)} will fail with an
+         * exception
+         */
+        private String validatePut(DiscoveryNode node) {
+            for (ObjectCursor<DiscoveryNode> cursor : nodes.values()) {
+                final DiscoveryNode existingNode = cursor.value;
+                if (node.getAddress().equals(existingNode.getAddress()) &&
+                    node.getId().equals(existingNode.getId()) == false) {
+                    return "can't add node " + node + ", found existing node " + existingNode + " with same address";
+                }
+                if (node.getId().equals(existingNode.getId()) &&
+                    node.getAddress().equals(existingNode.getAddress()) == false) {
+                    return "can't add node " + node + ", found existing node " + existingNode
+                        + " with the same id, but a different address";
+                }
+            }
+            return null;
+        }
+
         public DiscoveryNodes build() {
             ImmutableOpenMap.Builder<String, DiscoveryNode> dataNodesBuilder = ImmutableOpenMap.builder();
             ImmutableOpenMap.Builder<String, DiscoveryNode> masterNodesBuilder = ImmutableOpenMap.builder();
diff --git a/core/src/main/java/org/elasticsearch/common/hash/MurmurHash3.java b/core/src/main/java/org/elasticsearch/common/hash/MurmurHash3.java
index ba159f30a22..a52f0e8acc4 100644
--- a/core/src/main/java/org/elasticsearch/common/hash/MurmurHash3.java
+++ b/core/src/main/java/org/elasticsearch/common/hash/MurmurHash3.java
@@ -32,7 +32,10 @@ public enum MurmurHash3 {
      * A 128-bits hash.
      */
     public static class Hash128 {
-        public long h1, h2;
+        /** lower 64 bits part **/
+        public long h1;
+        /** higher 64 bits part **/
+        public long h2;
     }
 
     private static long C1 = 0x87c37b91114253d5L;
diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
index 01bd5e8a818..9f8eda54fa1 100644
--- a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
+++ b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -24,16 +24,14 @@ import org.elasticsearch.action.support.AutoCreateIndex;
 import org.elasticsearch.action.support.DestructiveOperations;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
 import org.elasticsearch.bootstrap.BootstrapSettings;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.transport.TransportClientNodesService;
 import org.elasticsearch.cluster.ClusterModule;
+import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.InternalClusterInfoService;
 import org.elasticsearch.cluster.NodeConnectionsService;
 import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.node.DiscoveryNodeService;
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
@@ -49,6 +47,7 @@ import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.discovery.DiscoveryModule;
@@ -336,7 +335,7 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     Environment.PATH_SCRIPTS_SETTING,
                     Environment.PATH_SHARED_DATA_SETTING,
                     Environment.PIDFILE_SETTING,
-                    DiscoveryNodeService.NODE_ID_SEED_SETTING,
+                    NodeEnvironment.NODE_ID_SEED_SETTING,
                     DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING,
                     DiscoveryModule.DISCOVERY_TYPE_SETTING,
                     DiscoveryModule.ZEN_MASTER_SERVICE_TYPE_SETTING,
@@ -365,6 +364,7 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     Node.NODE_MODE_SETTING,
                     Node.NODE_INGEST_SETTING,
                     Node.NODE_ATTRIBUTES,
+                    Node.NODE_LOCAL_STORAGE_SETTING,
                     URLRepository.ALLOWED_URLS_SETTING,
                     URLRepository.REPOSITORIES_URL_SETTING,
                     URLRepository.SUPPORTED_PROTOCOLS_SETTING,
@@ -387,7 +387,7 @@ public final class ClusterSettings extends AbstractScopedSettings {
                     TribeService.TRIBE_NAME_SETTING,
                     NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING,
                     NodeEnvironment.ENABLE_LUCENE_SEGMENT_INFOS_TRACE_SETTING,
-                    NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH,
+                    NodeEnvironment.ADD_NODE_LOCK_ID_TO_CUSTOM_PATH,
                     OsService.REFRESH_INTERVAL_SETTING,
                     ProcessService.REFRESH_INTERVAL_SETTING,
                     JvmService.REFRESH_INTERVAL_SETTING,
diff --git a/core/src/main/java/org/elasticsearch/common/transport/DummyTransportAddress.java b/core/src/main/java/org/elasticsearch/common/transport/DummyTransportAddress.java
deleted file mode 100644
index dae4a37765d..00000000000
--- a/core/src/main/java/org/elasticsearch/common/transport/DummyTransportAddress.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.transport;
-
-import org.elasticsearch.common.io.stream.StreamOutput;
-
-import java.io.IOException;
-
-/**
- *
- */
-public class DummyTransportAddress implements TransportAddress {
-
-    public static final DummyTransportAddress INSTANCE = new DummyTransportAddress();
-
-    private DummyTransportAddress() {
-    }
-
-    @Override
-    public short uniqueAddressTypeId() {
-        return 0;
-    }
-
-    @Override
-    public boolean sameHost(TransportAddress other) {
-        return other == INSTANCE;
-    }
-
-    @Override
-    public boolean isLoopbackOrLinkLocalAddress() {
-        return false;
-    }
-
-    @Override
-    public String getHost() {
-        return "dummy";
-    }
-
-    @Override
-    public String getAddress() {
-        return "0.0.0.0"; // see https://en.wikipedia.org/wiki/0.0.0.0
-    }
-
-    @Override
-    public int getPort() {
-        return 42;
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-    }
-
-    @Override
-    public String toString() {
-        return "_dummy_addr_";
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/common/transport/LocalTransportAddress.java b/core/src/main/java/org/elasticsearch/common/transport/LocalTransportAddress.java
index 9ded8dd23e1..48becc832da 100644
--- a/core/src/main/java/org/elasticsearch/common/transport/LocalTransportAddress.java
+++ b/core/src/main/java/org/elasticsearch/common/transport/LocalTransportAddress.java
@@ -23,6 +23,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 
 import java.io.IOException;
+import java.util.concurrent.atomic.AtomicLong;
 
 /**
  *
@@ -30,6 +31,15 @@ import java.io.IOException;
 public final class LocalTransportAddress implements TransportAddress {
     public static final short TYPE_ID = 2;
 
+    private static final AtomicLong transportAddressIdGenerator = new AtomicLong();
+
+    /**
+     * generates a new unique address
+     */
+    public static LocalTransportAddress buildUnique() {
+        return new LocalTransportAddress(Long.toString(transportAddressIdGenerator.incrementAndGet()));
+    }
+
     private String id;
 
     public LocalTransportAddress(String id) {
diff --git a/core/src/main/java/org/elasticsearch/common/transport/TransportAddressSerializers.java b/core/src/main/java/org/elasticsearch/common/transport/TransportAddressSerializers.java
index d5db2558163..784bee52d63 100644
--- a/core/src/main/java/org/elasticsearch/common/transport/TransportAddressSerializers.java
+++ b/core/src/main/java/org/elasticsearch/common/transport/TransportAddressSerializers.java
@@ -37,7 +37,6 @@ public abstract class TransportAddressSerializers {
 
     static {
         Map<Short, Writeable.Reader<TransportAddress>> registry = new HashMap<>();
-        addAddressType(registry, DummyTransportAddress.INSTANCE.uniqueAddressTypeId(), (in) -> DummyTransportAddress.INSTANCE);
         addAddressType(registry, InetSocketTransportAddress.TYPE_ID, InetSocketTransportAddress::new);
         addAddressType(registry, LocalTransportAddress.TYPE_ID, LocalTransportAddress::new);
         ADDRESS_REGISTRY = unmodifiableMap(registry);
diff --git a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
index 5f8a81ec994..790d40b8f59 100644
--- a/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java
@@ -47,6 +47,7 @@ import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.discovery.DiscoveryStats;
 
 import java.util.HashSet;
+import java.util.Optional;
 import java.util.Queue;
 import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
@@ -100,6 +101,14 @@ public class LocalDiscovery extends AbstractLifecycleComponent implements Discov
             }
             logger.debug("Connected to cluster [{}]", clusterName);
 
+            Optional<LocalDiscovery> current = clusterGroup.members().stream().filter(other -> (
+                other.localNode().equals(this.localNode()) || other.localNode().getId().equals(this.localNode().getId())
+            )).findFirst();
+            if (current.isPresent()) {
+                throw new IllegalStateException("current cluster group already contains a node with the same id. current "
+                    + current.get().localNode() + ", this node " + localNode());
+            }
+
             clusterGroup.members().add(this);
 
             LocalDiscovery firstMaster = null;
@@ -308,7 +317,7 @@ public class LocalDiscovery extends AbstractLifecycleComponent implements Discov
                 synchronized (this) {
                     // we do the marshaling intentionally, to check it works well...
                     // check if we published cluster state at least once and node was in the cluster when we published cluster state the last time
-                    if (discovery.lastProcessedClusterState != null && clusterChangedEvent.previousState().nodes().nodeExists(discovery.localNode().getId())) {
+                    if (discovery.lastProcessedClusterState != null && clusterChangedEvent.previousState().nodes().nodeExists(discovery.localNode())) {
                         // both conditions are true - which means we can try sending cluster state as diffs
                         if (clusterStateDiffBytes == null) {
                             Diff diff = clusterState.diff(clusterChangedEvent.previousState());
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
index f0b192d01d0..def67d97cc6 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
@@ -36,7 +36,7 @@ import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.discovery.zen.elect.ElectMasterService;
@@ -380,13 +380,13 @@ public class NodeJoinController extends AbstractComponent {
     }
 
     // a task indicated that the current node should become master, if no current master is known
-    private static final DiscoveryNode BECOME_MASTER_TASK = new DiscoveryNode("_BECOME_MASTER_TASK_", DummyTransportAddress.INSTANCE,
+    private static final DiscoveryNode BECOME_MASTER_TASK = new DiscoveryNode("_BECOME_MASTER_TASK_", LocalTransportAddress.buildUnique(),
         Collections.emptyMap(), Collections.emptySet(), Version.CURRENT);
 
     // a task that is used to process pending joins without explicitly becoming a master and listening to the results
     // this task is used when election is stop without the local node becoming a master per se (though it might
     private static final DiscoveryNode FINISH_ELECTION_NOT_MASTER_TASK = new DiscoveryNode("_NOT_MASTER_TASK_",
-        DummyTransportAddress.INSTANCE, Collections.emptyMap(), Collections.emptySet(), Version.CURRENT);
+        LocalTransportAddress.buildUnique(), Collections.emptyMap(), Collections.emptySet(), Version.CURRENT);
 
     class JoinTaskExecutor implements ClusterStateTaskExecutor<DiscoveryNode> {
 
@@ -418,17 +418,15 @@ public class NodeJoinController extends AbstractComponent {
             for (final DiscoveryNode node : joiningNodes) {
                 if (node.equals(BECOME_MASTER_TASK) || node.equals(FINISH_ELECTION_NOT_MASTER_TASK)) {
                     // noop
-                } else if (currentNodes.nodeExists(node.getId())) {
+                } else if (currentNodes.nodeExists(node)) {
                     logger.debug("received a join request for an existing node [{}]", node);
                 } else {
-                    nodesChanged = true;
-                    nodesBuilder.put(node);
-                    for (DiscoveryNode existingNode : currentNodes) {
-                        if (node.getAddress().equals(existingNode.getAddress())) {
-                            nodesBuilder.remove(existingNode.getId());
-                            logger.warn("received join request from node [{}], but found existing node {} with same address, " +
-                                "removing existing node", node, existingNode);
-                        }
+                    try {
+                        nodesBuilder.put(node);
+                        nodesChanged = true;
+                    } catch (IllegalArgumentException e) {
+                        results.failure(node, e);
+                        continue;
                     }
                 }
                 results.success(node);
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
index e052362b5e2..f8e730fa298 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
@@ -548,12 +548,11 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
         clusterService.submitStateUpdateTask("zen-disco-node_failed(" + node + "), reason " + reason, new ClusterStateUpdateTask(Priority.IMMEDIATE) {
             @Override
             public ClusterState execute(ClusterState currentState) {
-                if (currentState.nodes().get(node.getId()) == null) {
+                if (currentState.nodes().nodeExists(node) == false) {
                     logger.debug("node [{}] already removed from cluster state. ignoring.", node);
                     return currentState;
                 }
-                DiscoveryNodes.Builder builder = DiscoveryNodes.builder(currentState.nodes())
-                        .remove(node.getId());
+                DiscoveryNodes.Builder builder = DiscoveryNodes.builder(currentState.nodes()).remove(node);
                 currentState = ClusterState.builder(currentState).nodes(builder).build();
                 // check if we have enough master nodes, if not, we need to move into joining the cluster again
                 if (!electMaster.hasEnoughMasterNodes(currentState.nodes())) {
@@ -642,14 +641,14 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
 
             @Override
             public ClusterState execute(ClusterState currentState) {
-                if (!masterNode.getId().equals(currentState.nodes().getMasterNodeId())) {
+                if (!masterNode.equals(currentState.nodes().getMasterNode())) {
                     // master got switched on us, no need to send anything
                     return currentState;
                 }
 
                 DiscoveryNodes discoveryNodes = DiscoveryNodes.builder(currentState.nodes())
                         // make sure the old master node, which has failed, is not part of the nodes we publish
-                        .remove(masterNode.getId())
+                        .remove(masterNode)
                         .masterNodeId(null).build();
 
                 // flush any pending cluster states from old master, so it will not be set as master again
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
index c5d22b44fc9..e5cb7b3b1dd 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/fd/MasterFaultDetection.java
@@ -80,7 +80,8 @@ public class MasterFaultDetection extends FaultDetection {
         super(settings, threadPool, transportService, clusterService.getClusterName());
         this.clusterService = clusterService;
 
-        logger.debug("[master] uses ping_interval [{}], ping_timeout [{}], ping_retries [{}]", pingInterval, pingRetryTimeout, pingRetryCount);
+        logger.debug("[master] uses ping_interval [{}], ping_timeout [{}], ping_retries [{}]", pingInterval, pingRetryTimeout,
+            pingRetryCount);
 
         transportService.registerRequestHandler(
             MASTER_PING_ACTION_NAME, MasterPingRequest::new, ThreadPool.Names.SAME, false, false, new MasterPingRequestHandler());
@@ -230,9 +231,11 @@ public class MasterFaultDetection extends FaultDetection {
                 threadPool.schedule(pingInterval, ThreadPool.Names.SAME, MasterPinger.this);
                 return;
             }
-            final MasterPingRequest request = new MasterPingRequest(clusterService.localNode().getId(), masterToPing.getId(), clusterName);
-            final TransportRequestOptions options = TransportRequestOptions.builder().withType(TransportRequestOptions.Type.PING).withTimeout(pingRetryTimeout).build();
-            transportService.sendRequest(masterToPing, MASTER_PING_ACTION_NAME, request, options, new BaseTransportResponseHandler<MasterPingResponseResponse>() {
+            final MasterPingRequest request = new MasterPingRequest(clusterService.localNode(), masterToPing, clusterName);
+            final TransportRequestOptions options = TransportRequestOptions.builder().withType(TransportRequestOptions.Type.PING)
+                .withTimeout(pingRetryTimeout).build();
+            transportService.sendRequest(masterToPing, MASTER_PING_ACTION_NAME, request, options,
+                new BaseTransportResponseHandler<MasterPingResponseResponse>() {
 
                         @Override
                         public MasterPingResponseResponse newInstance() {
@@ -273,17 +276,21 @@ public class MasterFaultDetection extends FaultDetection {
                                         notifyMasterFailure(masterToPing, exp,"not master");
                                         return;
                                     } else if (exp.getCause() instanceof NodeDoesNotExistOnMasterException) {
-                                        logger.debug("[master] pinging a master {} but we do not exists on it, act as if its master failure", masterNode);
+                                        logger.debug("[master] pinging a master {} but we do not exists on it, act as if its master failure"
+                                            , masterNode);
                                         notifyMasterFailure(masterToPing, exp,"do not exists on master, act as master failure");
                                         return;
                                     }
 
                                     int retryCount = ++MasterFaultDetection.this.retryCount;
-                                    logger.trace("[master] failed to ping [{}], retry [{}] out of [{}]", exp, masterNode, retryCount, pingRetryCount);
+                                    logger.trace("[master] failed to ping [{}], retry [{}] out of [{}]", exp, masterNode, retryCount,
+                                        pingRetryCount);
                                     if (retryCount >= pingRetryCount) {
-                                        logger.debug("[master] failed to ping [{}], tried [{}] times, each with maximum [{}] timeout", masterNode, pingRetryCount, pingRetryTimeout);
+                                        logger.debug("[master] failed to ping [{}], tried [{}] times, each with maximum [{}] timeout",
+                                            masterNode, pingRetryCount, pingRetryTimeout);
                                         // not good, failure
-                                        notifyMasterFailure(masterToPing, null, "failed to ping, tried [" + pingRetryCount + "] times, each with  maximum [" + pingRetryTimeout + "] timeout");
+                                        notifyMasterFailure(masterToPing, null, "failed to ping, tried [" + pingRetryCount
+                                            + "] times, each with  maximum [" + pingRetryTimeout + "] timeout");
                                     } else {
                                         // resend the request, not reschedule, rely on send timeout
                                         transportService.sendRequest(masterToPing, MASTER_PING_ACTION_NAME, request, options, this);
@@ -331,14 +338,16 @@ public class MasterFaultDetection extends FaultDetection {
             final DiscoveryNodes nodes = clusterService.state().nodes();
             // check if we are really the same master as the one we seemed to be think we are
             // this can happen if the master got "kill -9" and then another node started using the same port
-            if (!request.masterNodeId.equals(nodes.getLocalNodeId())) {
+            if (!request.masterNode.equals(nodes.getLocalNode())) {
                 throw new ThisIsNotTheMasterYouAreLookingForException();
             }
 
             // ping from nodes of version < 1.4.0 will have the clustername set to null
             if (request.clusterName != null && !request.clusterName.equals(clusterName)) {
-                logger.trace("master fault detection ping request is targeted for a different [{}] cluster then us [{}]", request.clusterName, clusterName);
-                throw new ThisIsNotTheMasterYouAreLookingForException("master fault detection ping request is targeted for a different [" + request.clusterName + "] cluster then us [" + clusterName + "]");
+                logger.trace("master fault detection ping request is targeted for a different [{}] cluster then us [{}]",
+                    request.clusterName, clusterName);
+                throw new ThisIsNotTheMasterYouAreLookingForException("master fault detection ping request is targeted for a different ["
+                    + request.clusterName + "] cluster then us [" + clusterName + "]");
             }
 
             // when we are elected as master or when a node joins, we use a cluster state update thread
@@ -349,15 +358,15 @@ public class MasterFaultDetection extends FaultDetection {
             // all processing is finished.
             //
 
-            if (!nodes.isLocalNodeElectedMaster() || !nodes.nodeExists(request.nodeId)) {
-                logger.trace("checking ping from [{}] under a cluster state thread", request.nodeId);
-                clusterService.submitStateUpdateTask("master ping (from: [" + request.nodeId + "])", new ClusterStateUpdateTask() {
+            if (!nodes.isLocalNodeElectedMaster() || !nodes.nodeExists(request.sourceNode)) {
+                logger.trace("checking ping from {} under a cluster state thread", request.sourceNode);
+                clusterService.submitStateUpdateTask("master ping (from: " + request.sourceNode + ")", new ClusterStateUpdateTask() {
 
                     @Override
                     public ClusterState execute(ClusterState currentState) throws Exception {
                         // if we are no longer master, fail...
                         DiscoveryNodes nodes = currentState.nodes();
-                        if (!nodes.nodeExists(request.nodeId)) {
+                        if (!nodes.nodeExists(request.sourceNode)) {
                             throw new NodeDoesNotExistOnMasterException();
                         }
                         return currentState;
@@ -400,33 +409,33 @@ public class MasterFaultDetection extends FaultDetection {
 
     public static class MasterPingRequest extends TransportRequest {
 
-        private String nodeId;
+        private DiscoveryNode sourceNode;
 
-        private String masterNodeId;
+        private DiscoveryNode masterNode;
         private ClusterName clusterName;
 
         public MasterPingRequest() {
         }
 
-        private MasterPingRequest(String nodeId, String masterNodeId, ClusterName clusterName) {
-            this.nodeId = nodeId;
-            this.masterNodeId = masterNodeId;
+        private MasterPingRequest(DiscoveryNode sourceNode, DiscoveryNode masterNode, ClusterName clusterName) {
+            this.sourceNode = sourceNode;
+            this.masterNode = masterNode;
             this.clusterName = clusterName;
         }
 
         @Override
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
-            nodeId = in.readString();
-            masterNodeId = in.readString();
+            sourceNode = new DiscoveryNode(in);
+            masterNode = new DiscoveryNode(in);
             clusterName = new ClusterName(in);
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            out.writeString(nodeId);
-            out.writeString(masterNodeId);
+            sourceNode.writeTo(out);
+            masterNode.writeTo(out);
             clusterName.writeTo(out);
         }
     }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java b/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java
index 6fb1c82b268..8c261b988f9 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/fd/NodesFaultDetection.java
@@ -70,7 +70,8 @@ public class NodesFaultDetection extends FaultDetection {
     public NodesFaultDetection(Settings settings, ThreadPool threadPool, TransportService transportService, ClusterName clusterName) {
         super(settings, threadPool, transportService, clusterName);
 
-        logger.debug("[node  ] uses ping_interval [{}], ping_timeout [{}], ping_retries [{}]", pingInterval, pingRetryTimeout, pingRetryCount);
+        logger.debug("[node  ] uses ping_interval [{}], ping_timeout [{}], ping_retries [{}]", pingInterval, pingRetryTimeout,
+            pingRetryCount);
 
         transportService.registerRequestHandler(
             PING_ACTION_NAME, PingRequest::new, ThreadPool.Names.SAME, false, false, new PingRequestHandler());
@@ -95,7 +96,7 @@ public class NodesFaultDetection extends FaultDetection {
     public void updateNodesAndPing(ClusterState clusterState) {
         // remove any nodes we don't need, this will cause their FD to stop
         for (DiscoveryNode monitoredNode : nodesFD.keySet()) {
-            if (!clusterState.nodes().nodeExists(monitoredNode.getId())) {
+            if (!clusterState.nodes().nodeExists(monitoredNode)) {
                 nodesFD.remove(monitoredNode);
             }
         }
@@ -201,8 +202,9 @@ public class NodesFaultDetection extends FaultDetection {
             if (!running()) {
                 return;
             }
-            final PingRequest pingRequest = new PingRequest(node.getId(), clusterName, localNode, clusterStateVersion);
-            final TransportRequestOptions options = TransportRequestOptions.builder().withType(TransportRequestOptions.Type.PING).withTimeout(pingRetryTimeout).build();
+            final PingRequest pingRequest = new PingRequest(node, clusterName, localNode, clusterStateVersion);
+            final TransportRequestOptions options = TransportRequestOptions.builder().withType(TransportRequestOptions.Type.PING)
+                .withTimeout(pingRetryTimeout).build();
             transportService.sendRequest(node, PING_ACTION_NAME, pingRequest, options, new BaseTransportResponseHandler<PingResponse>() {
                         @Override
                         public PingResponse newInstance() {
@@ -231,10 +233,12 @@ public class NodesFaultDetection extends FaultDetection {
                             retryCount++;
                             logger.trace("[node  ] failed to ping [{}], retry [{}] out of [{}]", exp, node, retryCount, pingRetryCount);
                             if (retryCount >= pingRetryCount) {
-                                logger.debug("[node  ] failed to ping [{}], tried [{}] times, each with  maximum [{}] timeout", node, pingRetryCount, pingRetryTimeout);
+                                logger.debug("[node  ] failed to ping [{}], tried [{}] times, each with  maximum [{}] timeout", node,
+                                    pingRetryCount, pingRetryTimeout);
                                 // not good, failure
                                 if (nodesFD.remove(node, NodeFD.this)) {
-                                    notifyNodeFailure(node, "failed to ping, tried [" + pingRetryCount + "] times, each with maximum [" + pingRetryTimeout + "] timeout");
+                                    notifyNodeFailure(node, "failed to ping, tried [" + pingRetryCount + "] times, each with maximum ["
+                                        + pingRetryTimeout + "] timeout");
                                 }
                             } else {
                                 // resend the request, not reschedule, rely on send timeout
@@ -256,14 +260,15 @@ public class NodesFaultDetection extends FaultDetection {
         public void messageReceived(PingRequest request, TransportChannel channel) throws Exception {
             // if we are not the node we are supposed to be pinged, send an exception
             // this can happen when a kill -9 is sent, and another node is started using the same port
-            if (!localNode.getId().equals(request.nodeId)) {
-                throw new IllegalStateException("Got pinged as node [" + request.nodeId + "], but I am node [" + localNode.getId() + "]");
+            if (!localNode.equals(request.targetNode())) {
+                throw new IllegalStateException("Got pinged as node " + request.targetNode() + "], but I am node " + localNode );
             }
 
             // PingRequest will have clusterName set to null if it came from a node of version <1.4.0
             if (request.clusterName != null && !request.clusterName.equals(clusterName)) {
                 // Don't introduce new exception for bwc reasons
-                throw new IllegalStateException("Got pinged with cluster name [" + request.clusterName + "], but I'm part of cluster [" + clusterName + "]");
+                throw new IllegalStateException("Got pinged with cluster name [" + request.clusterName + "], but I'm part of cluster ["
+                    + clusterName + "]");
             }
 
             notifyPingReceived(request);
@@ -275,8 +280,8 @@ public class NodesFaultDetection extends FaultDetection {
 
     public static class PingRequest extends TransportRequest {
 
-        // the (assumed) node id we are pinging
-        private String nodeId;
+        // the (assumed) node we are pinging
+        private DiscoveryNode targetNode;
 
         private ClusterName clusterName;
 
@@ -287,15 +292,15 @@ public class NodesFaultDetection extends FaultDetection {
         public PingRequest() {
         }
 
-        PingRequest(String nodeId, ClusterName clusterName, DiscoveryNode masterNode, long clusterStateVersion) {
-            this.nodeId = nodeId;
+        PingRequest(DiscoveryNode targetNode, ClusterName clusterName, DiscoveryNode masterNode, long clusterStateVersion) {
+            this.targetNode = targetNode;
             this.clusterName = clusterName;
             this.masterNode = masterNode;
             this.clusterStateVersion = clusterStateVersion;
         }
 
-        public String nodeId() {
-            return nodeId;
+        public DiscoveryNode targetNode() {
+            return targetNode;
         }
 
         public ClusterName clusterName() {
@@ -313,7 +318,7 @@ public class NodesFaultDetection extends FaultDetection {
         @Override
         public void readFrom(StreamInput in) throws IOException {
             super.readFrom(in);
-            nodeId = in.readString();
+            targetNode = new DiscoveryNode(in);
             clusterName = new ClusterName(in);
             masterNode = new DiscoveryNode(in);
             clusterStateVersion = in.readLong();
@@ -322,7 +327,7 @@ public class NodesFaultDetection extends FaultDetection {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            out.writeString(nodeId);
+            targetNode.writeTo(out);
             clusterName.writeTo(out);
             masterNode.writeTo(out);
             out.writeLong(clusterStateVersion);
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
index 98ccf137b13..ce55723ad24 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPing.java
@@ -27,6 +27,7 @@ import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -372,9 +373,9 @@ public class UnicastZenPing extends AbstractLifecycleComponent implements ZenPin
                 if (!nodeFoundByAddress) {
                     if (!nodeToSend.getId().startsWith(UNICAST_NODE_PREFIX)) {
                         DiscoveryNode tempNode = new DiscoveryNode("",
-                                UNICAST_NODE_PREFIX + unicastNodeIdGenerator.incrementAndGet() + "_" + nodeToSend.getId() + "#",
-                                nodeToSend.getHostName(), nodeToSend.getHostAddress(), nodeToSend.getAddress(), nodeToSend.getAttributes(),
-                                nodeToSend.getRoles(), nodeToSend.getVersion());
+                            UNICAST_NODE_PREFIX + unicastNodeIdGenerator.incrementAndGet() + "_" + nodeToSend.getId() + "#",
+                            UUIDs.randomBase64UUID(), nodeToSend.getHostName(), nodeToSend.getHostAddress(), nodeToSend.getAddress(),
+                            nodeToSend.getAttributes(), nodeToSend.getRoles(), nodeToSend.getVersion());
 
                         logger.trace("replacing {} with temp node {}", nodeToSend, tempNode);
                         nodeToSend = tempNode;
@@ -461,7 +462,7 @@ public class UnicastZenPing extends AbstractLifecycleComponent implements ZenPin
                 try {
                     DiscoveryNodes discoveryNodes = contextProvider.nodes();
                     for (PingResponse pingResponse : response.pingResponses) {
-                        if (pingResponse.node().getId().equals(discoveryNodes.getLocalNodeId())) {
+                        if (pingResponse.node().equals(discoveryNodes.getLocalNode())) {
                             // that's us, ignore
                             continue;
                         }
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java b/core/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java
index bcdd2f25d4d..10f874923da 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateAction.java
@@ -185,7 +185,7 @@ public class PublishClusterStateAction extends AbstractComponent {
             // try and serialize the cluster state once (or per version), so we don't serialize it
             // per node when we send it over the wire, compress it while we are at it...
             // we don't send full version if node didn't exist in the previous version of cluster state
-            if (sendFullVersion || !previousState.nodes().nodeExists(node.getId())) {
+            if (sendFullVersion || !previousState.nodes().nodeExists(node)) {
                 sendFullClusterState(clusterState, serializedStates, node, publishTimeout, sendingController);
             } else {
                 sendClusterStateDiff(clusterState, serializedDiffs, serializedStates, node, publishTimeout, sendingController);
@@ -216,7 +216,7 @@ public class PublishClusterStateAction extends AbstractComponent {
         Diff<ClusterState> diff = null;
         for (final DiscoveryNode node : nodesToPublishTo) {
             try {
-                if (sendFullVersion || !previousState.nodes().nodeExists(node.getId())) {
+                if (sendFullVersion || !previousState.nodes().nodeExists(node)) {
                     // will send a full reference
                     if (serializedStates.containsKey(node.getVersion()) == false) {
                         serializedStates.put(node.getVersion(), serializeFullClusterState(clusterState, node.getVersion()));
diff --git a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
index c091788911f..dc8b9289d0f 100644
--- a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
+++ b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
@@ -31,10 +31,13 @@ import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -65,6 +68,7 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
@@ -76,6 +80,7 @@ import static java.util.Collections.unmodifiableSet;
  * A component that holds all data paths for a single node.
  */
 public final class NodeEnvironment extends AbstractComponent implements Closeable {
+
     public static class NodePath {
         /* ${data.paths}/nodes/{node.id} */
         public final Path path;
@@ -134,23 +139,34 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
     private final Path sharedDataPath;
     private final Lock[] locks;
 
-    private final boolean addNodeId;
+    private final boolean addLockIdToCustomPath;
 
-    private final int localNodeId;
+    private final int nodeLockId;
     private final AtomicBoolean closed = new AtomicBoolean(false);
     private final Map<ShardId, InternalShardLock> shardLocks = new HashMap<>();
 
+    private final NodeMetaData nodeMetaData;
+
     /**
      * Maximum number of data nodes that should run in an environment.
      */
     public static final Setting<Integer> MAX_LOCAL_STORAGE_NODES_SETTING = Setting.intSetting("node.max_local_storage_nodes", 50, 1,
-            Property.NodeScope);
+        Property.NodeScope);
 
     /**
-     * If true automatically append node id to custom data paths.
+     * If true automatically append node lock id to custom data paths.
      */
-    public static final Setting<Boolean> ADD_NODE_ID_TO_CUSTOM_PATH =
-        Setting.boolSetting("node.add_id_to_custom_path", true, Property.NodeScope);
+    public static final Setting<Boolean> ADD_NODE_LOCK_ID_TO_CUSTOM_PATH =
+        Setting.boolSetting("node.add_lock_id_to_custom_path", true, Property.NodeScope);
+
+
+    /**
+     * Seed for determining a persisted unique uuid of this node. If the node has already a persisted uuid on disk,
+     * this seed will be ignored and the uuid from disk will be reused.
+     */
+    public static final Setting<Long> NODE_ID_SEED_SETTING =
+        Setting.longSetting("node.id.seed", 0L, Long.MIN_VALUE, Property.NodeScope);
+
 
     /**
      * If true the [verbose] SegmentInfos.infoStream logging is sent to System.out.
@@ -166,22 +182,23 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
     public NodeEnvironment(Settings settings, Environment environment) throws IOException {
         super(settings);
 
-        this.addNodeId = ADD_NODE_ID_TO_CUSTOM_PATH.get(settings);
+        this.addLockIdToCustomPath = ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.get(settings);
 
         if (!DiscoveryNode.nodeRequiresLocalStorage(settings)) {
             nodePaths = null;
             sharedDataPath = null;
             locks = null;
-            localNodeId = -1;
+            nodeLockId = -1;
+            nodeMetaData = new NodeMetaData(generateNodeId(settings));
             return;
         }
-        final NodePath[] nodePaths = new NodePath[environment.dataFiles().length];
+        final NodePath[] nodePaths = new NodePath[environment.dataWithClusterFiles().length];
         final Lock[] locks = new Lock[nodePaths.length];
         boolean success = false;
 
         try {
             sharedDataPath = environment.sharedDataFile();
-            int localNodeId = -1;
+            int nodeLockId = -1;
             IOException lastException = null;
             int maxLocalStorageNodes = MAX_LOCAL_STORAGE_NODES_SETTING.get(settings);
             for (int possibleLockId = 0; possibleLockId < maxLocalStorageNodes; possibleLockId++) {
@@ -203,7 +220,7 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
                         try {
                             locks[dirIndex] = luceneDir.obtainLock(NODE_LOCK_FILENAME);
                             nodePaths[dirIndex] = new NodePath(dir);
-                            localNodeId = possibleLockId;
+                            nodeLockId = possibleLockId;
                         } catch (LockObtainFailedException ex) {
                             logger.trace("failed to obtain node lock on {}", dir.toAbsolutePath());
                             // release all the ones that were obtained up until now
@@ -227,20 +244,22 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
 
             if (locks[0] == null) {
                 throw new IllegalStateException("Failed to obtain node lock, is the following location writable?: "
-                    + Arrays.toString(environment.dataFiles()), lastException);
+                    + Arrays.toString(environment.dataWithClusterFiles()), lastException);
             }
 
-            this.localNodeId = localNodeId;
+            this.nodeLockId = nodeLockId;
             this.locks = locks;
             this.nodePaths = nodePaths;
 
             if (logger.isDebugEnabled()) {
-                logger.debug("using node location [{}], local_node_id [{}]", nodePaths, localNodeId);
+                logger.debug("using node location [{}], local_lock_id [{}]", nodePaths, nodeLockId);
             }
 
             maybeLogPathDetails();
             maybeLogHeapDetails();
 
+            this.nodeMetaData = loadOrCreateNodeMetaData(settings, logger, nodePaths);
+
             applySegmentInfosTrace(settings);
             assertCanWrite();
             success = true;
@@ -360,6 +379,27 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
     }
 
 
+    /**
+     * scans the node paths and loads existing metaData file. If not found a new meta data will be generated
+     * and persisted into the nodePaths
+     */
+    private static NodeMetaData loadOrCreateNodeMetaData(Settings settings, ESLogger logger,
+                                                         NodePath... nodePaths) throws IOException {
+        final Path[] paths = Arrays.stream(nodePaths).map(np -> np.path).toArray(Path[]::new);
+        NodeMetaData metaData = NodeMetaData.FORMAT.loadLatestState(logger, paths);
+        if (metaData == null) {
+            metaData = new NodeMetaData(generateNodeId(settings));
+        }
+        // we write again to make sure all paths have the latest state file
+        NodeMetaData.FORMAT.write(metaData, paths);
+        return metaData;
+    }
+
+    public static String generateNodeId(Settings settings) {
+        Random random = Randomness.get(settings, NODE_ID_SEED_SETTING);
+        return UUIDs.randomBase64UUID(random);
+    }
+
     @SuppressForbidden(reason = "System.out.*")
     static void applySegmentInfosTrace(Settings settings) {
         if (ENABLE_LUCENE_SEGMENT_INFOS_TRACE_SETTING.get(settings)) {
@@ -657,10 +697,6 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
         }
     }
 
-    public int localNodeId() {
-        return this.localNodeId;
-    }
-
     public boolean hasNodeFile() {
         return nodePaths != null && locks != null;
     }
@@ -678,6 +714,17 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
         return paths;
     }
 
+    /**
+     * returns the unique uuid describing this node. The uuid is persistent in the data folder of this node
+     * and remains across restarts.
+     **/
+    public String nodeId() {
+        // we currently only return the ID and hide the underlying nodeMetaData implementation in order to avoid
+        // confusion with other "metadata" like node settings found in elasticsearch.yml. In future
+        // we can encapsulate both (and more) in one NodeMetaData (or NodeSettings) object ala IndexSettings
+        return nodeMetaData.nodeId();
+    }
+
     /**
      * Returns an array of all of the {@link NodePath}s.
      */
@@ -881,8 +928,8 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
         if (customDataDir != null) {
             // This assert is because this should be caught by MetaDataCreateIndexService
             assert sharedDataPath != null;
-            if (addNodeId) {
-                return sharedDataPath.resolve(customDataDir).resolve(Integer.toString(this.localNodeId));
+            if (addLockIdToCustomPath) {
+                return sharedDataPath.resolve(customDataDir).resolve(Integer.toString(this.nodeLockId));
             } else {
                 return sharedDataPath.resolve(customDataDir);
             }
diff --git a/core/src/main/java/org/elasticsearch/env/NodeMetaData.java b/core/src/main/java/org/elasticsearch/env/NodeMetaData.java
new file mode 100644
index 00000000000..60625b1852d
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/env/NodeMetaData.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.env;
+
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParseFieldMatcherSupplier;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.gateway.MetaDataStateFormat;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Objects;
+
+/**
+ * Metadata associated with this node. Currently only contains the unique uuid describing this node.
+ * The metadata is persisted in the data folder of this node and is reused across restarts.
+ */
+public final class NodeMetaData {
+
+    private static final String NODE_ID_KEY = "node_id";
+
+    private final String nodeId;
+
+    public NodeMetaData(final String nodeId) {
+        this.nodeId = Objects.requireNonNull(nodeId);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+
+        NodeMetaData that = (NodeMetaData) o;
+
+        return Objects.equals(this.nodeId, that.nodeId);
+    }
+
+    @Override
+    public int hashCode() {
+        return this.nodeId.hashCode();
+    }
+
+    @Override
+    public String toString() {
+        return "node_id [" + nodeId + "]";
+    }
+
+    private static ObjectParser<Builder, ParseFieldMatcherSupplier> PARSER = new ObjectParser<>("node_meta_data",
+        Builder::new);
+
+    static {
+        PARSER.declareString(Builder::setNodeId, new ParseField(NODE_ID_KEY));
+    }
+
+    public String nodeId() {
+        return nodeId;
+    }
+
+    private static class Builder {
+        String nodeId;
+
+        public void setNodeId(String nodeId) {
+            this.nodeId = nodeId;
+        }
+
+        public NodeMetaData build() {
+            return new NodeMetaData(nodeId);
+        }
+    }
+
+
+    public static final MetaDataStateFormat<NodeMetaData> FORMAT = new MetaDataStateFormat<NodeMetaData>(XContentType.SMILE, "node-") {
+
+        @Override
+        protected XContentBuilder newXContentBuilder(XContentType type, OutputStream stream) throws IOException {
+            XContentBuilder xContentBuilder = super.newXContentBuilder(type, stream);
+            xContentBuilder.prettyPrint();
+            return xContentBuilder;
+        }
+
+        @Override
+        public void toXContent(XContentBuilder builder, NodeMetaData nodeMetaData) throws IOException {
+            builder.field(NODE_ID_KEY, nodeMetaData.nodeId);
+        }
+
+        @Override
+        public NodeMetaData fromXContent(XContentParser parser) throws IOException {
+            return PARSER.apply(parser, () -> ParseFieldMatcher.STRICT).build();
+        }
+    };
+}
diff --git a/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java b/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java
index 9c2d6c80011..cc6a48b855b 100644
--- a/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java
+++ b/core/src/main/java/org/elasticsearch/gateway/MetaDataStateFormat.java
@@ -64,7 +64,9 @@ public abstract class MetaDataStateFormat<T> {
     public static final String STATE_DIR_NAME = "_state";
     public static final String STATE_FILE_EXTENSION = ".st";
     private static final String STATE_FILE_CODEC = "state";
-    private static final int STATE_FILE_VERSION = 0;
+    private static final int MIN_COMPATIBLE_STATE_FILE_VERSION = 0;
+    private static final int STATE_FILE_VERSION = 1;
+    private static final int STATE_FILE_VERSION_ES_2X_AND_BELOW = 0;
     private static final int BUFFER_SIZE = 4096;
     private final XContentType format;
     private final String prefix;
@@ -96,11 +98,10 @@ public abstract class MetaDataStateFormat<T> {
      * it's target filename of the pattern <tt>{prefix}{version}.st</tt>.
      *
      * @param state the state object to write
-     * @param version the version of the state
      * @param locations the locations where the state should be written to.
      * @throws IOException if an IOException occurs
      */
-    public final void write(final T state, final long version, final Path... locations) throws IOException {
+    public final void write(final T state, final Path... locations) throws IOException {
         if (locations == null) {
             throw new IllegalArgumentException("Locations must not be null");
         }
@@ -116,10 +117,10 @@ public abstract class MetaDataStateFormat<T> {
         final Path finalStatePath = stateLocation.resolve(fileName);
         try {
             final String resourceDesc = "MetaDataStateFormat.write(path=\"" + tmpStatePath + "\")";
-            try (OutputStreamIndexOutput out = new OutputStreamIndexOutput(resourceDesc, fileName, Files.newOutputStream(tmpStatePath), BUFFER_SIZE)) {
+            try (OutputStreamIndexOutput out =
+                     new OutputStreamIndexOutput(resourceDesc, fileName, Files.newOutputStream(tmpStatePath), BUFFER_SIZE)) {
                 CodecUtil.writeHeader(out, STATE_FILE_CODEC, STATE_FILE_VERSION);
                 out.writeInt(format.index());
-                out.writeLong(version);
                 try (XContentBuilder builder = newXContentBuilder(format, new IndexOutputOutputStream(out) {
                     @Override
                     public void close() throws IOException {
@@ -145,7 +146,8 @@ public abstract class MetaDataStateFormat<T> {
                 Path finalPath = stateLocation.resolve(fileName);
                 try {
                     Files.copy(finalStatePath, tmpPath);
-                    Files.move(tmpPath, finalPath, StandardCopyOption.ATOMIC_MOVE); // we are on the same FileSystem / Partition here we can do an atomic move
+                    // we are on the same FileSystem / Partition here we can do an atomic move
+                    Files.move(tmpPath, finalPath, StandardCopyOption.ATOMIC_MOVE);
                     IOUtils.fsync(stateLocation, true); // we just fsync the dir here..
                 } finally {
                     Files.deleteIfExists(tmpPath);
@@ -182,13 +184,18 @@ public abstract class MetaDataStateFormat<T> {
             try (final IndexInput indexInput = dir.openInput(file.getFileName().toString(), IOContext.DEFAULT)) {
                  // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here.
                 CodecUtil.checksumEntireFile(indexInput);
-                CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, STATE_FILE_VERSION, STATE_FILE_VERSION);
+                final int fileVersion = CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, MIN_COMPATIBLE_STATE_FILE_VERSION,
+                    STATE_FILE_VERSION);
                 final XContentType xContentType = XContentType.values()[indexInput.readInt()];
-                indexInput.readLong(); // version currently unused
+                if (fileVersion == STATE_FILE_VERSION_ES_2X_AND_BELOW) {
+                    // format version 0, wrote a version that always came from the content state file and was never used
+                    indexInput.readLong(); // version currently unused
+                }
                 long filePointer = indexInput.getFilePointer();
                 long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
                 try (IndexInput slice = indexInput.slice("state_xcontent", filePointer, contentSize)) {
-                    try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(new InputStreamIndexInput(slice, contentSize))) {
+                    try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(new InputStreamIndexInput(slice,
+                        contentSize))) {
                         return fromXContent(parser);
                     }
                 }
@@ -261,7 +268,8 @@ public abstract class MetaDataStateFormat<T> {
                 // now, iterate over the current versions, and find latest one
                 // we don't check if the stateDir is present since it could be deleted
                 // after the check. Also if there is a _state file and it's not a dir something is really wrong
-                try (DirectoryStream<Path> paths = Files.newDirectoryStream(stateDir)) { // we don't pass a glob since we need the group part for parsing
+                // we don't pass a glob since we need the group part for parsing
+                try (DirectoryStream<Path> paths = Files.newDirectoryStream(stateDir)) {
                     for (Path stateFile : paths) {
                         final Matcher matcher = stateFilePattern.matcher(stateFile.getFileName().toString());
                         if (matcher.matches()) {
diff --git a/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java
index b6166c9d545..9a36df43672 100644
--- a/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java
+++ b/core/src/main/java/org/elasticsearch/gateway/MetaStateService.java
@@ -125,7 +125,7 @@ public class MetaStateService extends AbstractComponent {
         final Index index = indexMetaData.getIndex();
         logger.trace("[{}] writing state, reason [{}]", index, reason);
         try {
-            IndexMetaData.FORMAT.write(indexMetaData, indexMetaData.getVersion(),
+            IndexMetaData.FORMAT.write(indexMetaData,
                 nodeEnv.indexPaths(indexMetaData.getIndex()));
         } catch (Exception ex) {
             logger.warn("[{}]: failed to write index state", ex, index);
@@ -139,7 +139,7 @@ public class MetaStateService extends AbstractComponent {
     void writeGlobalState(String reason, MetaData metaData) throws Exception {
         logger.trace("[_global] writing state, reason [{}]",  reason);
         try {
-            MetaData.FORMAT.write(metaData, metaData.version(), nodeEnv.nodeDataPaths());
+            MetaData.FORMAT.write(metaData, nodeEnv.nodeDataPaths());
         } catch (Exception ex) {
             logger.warn("[_global]: failed to write global state", ex);
             throw new IOException("failed to write global state", ex);
diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index 7843aabb49f..ab2fe683e81 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -1524,7 +1524,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
             }
             logger.trace("{} writing shard state, reason [{}]", shardId, writeReason);
             final ShardStateMetaData newShardStateMetadata = new ShardStateMetaData(newRouting.primary(), getIndexUUID(), newRouting.allocationId());
-            ShardStateMetaData.FORMAT.write(newShardStateMetadata, newShardStateMetadata.legacyVersion, shardPath().getShardStatePath());
+            ShardStateMetaData.FORMAT.write(newShardStateMetadata, shardPath().getShardStatePath());
         } else {
             logger.trace("{} skip writing shard state, has been written before", shardId);
         }
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index ce1c21943f3..a0bb85c8ba7 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -149,6 +149,15 @@ public class Node implements Closeable {
         new Setting<>("node.mode", "network", Function.identity(), Property.NodeScope);
     public static final Setting<Boolean> NODE_INGEST_SETTING =
         Setting.boolSetting("node.ingest", true, Property.NodeScope);
+
+    /**
+    * controls whether the node is allowed to persist things like metadata to disk
+    * Note that this does not control whether the node stores actual indices (see
+    * {@link #NODE_DATA_SETTING}). However, if this is false, {@link #NODE_DATA_SETTING}
+    * and {@link #NODE_MASTER_SETTING} must also be false.
+    *
+    */
+    public static final Setting<Boolean> NODE_LOCAL_STORAGE_SETTING = Setting.boolSetting("node.local_storage", true, Property.NodeScope);
     public static final Setting<String> NODE_NAME_SETTING = Setting.simpleString("node.name", Property.NodeScope);
     public static final Setting<Settings> NODE_ATTRIBUTES = Setting.groupSetting("node.attr.", Property.NodeScope);
     public static final Setting<String> BREAKER_TYPE_KEY = new Setting<>("indices.breaker.type", "hierarchy", (s) -> {
@@ -253,7 +262,7 @@ public class Node implements Closeable {
             final ClusterService clusterService = new ClusterService(settings, settingsModule.getClusterSettings(), threadPool);
             clusterService.add(scriptModule.getScriptService());
             resourcesToClose.add(clusterService);
-            final TribeService tribeService = new TribeService(settings, clusterService);
+            final TribeService tribeService = new TribeService(settings, clusterService, nodeEnvironment.nodeId());
             resourcesToClose.add(tribeService);
             NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
             ModulesBuilder modules = new ModulesBuilder();
@@ -394,7 +403,7 @@ public class Node implements Closeable {
         validateNodeBeforeAcceptingRequests(settings, transportService.boundAddress());
 
         DiscoveryNode localNode = injector.getInstance(DiscoveryNodeService.class)
-                .buildLocalNode(transportService.boundAddress().publishAddress());
+                .buildLocalNode(transportService.boundAddress().publishAddress(), injector.getInstance(NodeEnvironment.class)::nodeId);
 
         // TODO: need to find a cleaner way to start/construct a service with some initial parameters,
         // playing nice with the life cycle interfaces
diff --git a/core/src/main/java/org/elasticsearch/tribe/TribeService.java b/core/src/main/java/org/elasticsearch/tribe/TribeService.java
index 43e36f2993b..69e9027ddf1 100644
--- a/core/src/main/java/org/elasticsearch/tribe/TribeService.java
+++ b/core/src/main/java/org/elasticsearch/tribe/TribeService.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.tribe;
 
+import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.support.master.TransportMasterNodeReadAction;
 import org.elasticsearch.cluster.ClusterChangedEvent;
@@ -40,6 +41,7 @@ import org.elasticsearch.common.Priority;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.common.hash.MurmurHash3;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.regex.Regex;
@@ -51,6 +53,7 @@ import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.rest.RestStatus;
@@ -174,14 +177,14 @@ public class TribeService extends AbstractLifecycleComponent {
 
     private final List<Node> nodes = new CopyOnWriteArrayList<>();
 
-    public TribeService(Settings settings, ClusterService clusterService) {
+    public TribeService(Settings settings, ClusterService clusterService, final String tribeNodeId) {
         super(settings);
         this.clusterService = clusterService;
         Map<String, Settings> nodesSettings = new HashMap<>(settings.getGroups("tribe", true));
         nodesSettings.remove("blocks"); // remove prefix settings that don't indicate a client
         nodesSettings.remove("on_conflict"); // remove prefix settings that don't indicate a client
         for (Map.Entry<String, Settings> entry : nodesSettings.entrySet()) {
-            Settings clientSettings = buildClientSettings(entry.getKey(), settings, entry.getValue());
+            Settings clientSettings = buildClientSettings(entry.getKey(), tribeNodeId, settings, entry.getValue());
             nodes.add(new TribeClientNode(clientSettings));
         }
 
@@ -206,7 +209,7 @@ public class TribeService extends AbstractLifecycleComponent {
      * Builds node settings for a tribe client node from the tribe node's global settings,
      * combined with tribe specific settings.
      */
-    static Settings buildClientSettings(String tribeName, Settings globalSettings, Settings tribeSettings) {
+    static Settings buildClientSettings(String tribeName, String parentNodeId, Settings globalSettings, Settings tribeSettings) {
         for (String tribeKey : tribeSettings.getAsMap().keySet()) {
             if (tribeKey.startsWith("path.")) {
                 throw new IllegalArgumentException("Setting [" + tribeKey + "] not allowed in tribe client [" + tribeName + "]");
@@ -236,6 +239,12 @@ public class TribeService extends AbstractLifecycleComponent {
         sb.put(Node.NODE_DATA_SETTING.getKey(), false);
         sb.put(Node.NODE_MASTER_SETTING.getKey(), false);
         sb.put(Node.NODE_INGEST_SETTING.getKey(), false);
+
+        // node id of a tribe client node is determined by node id of parent node and tribe name
+        final BytesRef seedAsString = new BytesRef(parentNodeId + "/" + tribeName);
+        long nodeIdSeed = MurmurHash3.hash128(seedAsString.bytes, seedAsString.offset, seedAsString.length, 0, new MurmurHash3.Hash128()).h1;
+        sb.put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), nodeIdSeed);
+        sb.put(Node.NODE_LOCAL_STORAGE_SETTING.getKey(), false);
         return sb.build();
     }
 
@@ -359,14 +368,16 @@ public class TribeService extends AbstractLifecycleComponent {
             }
             // go over tribe nodes, and see if they need to be added
             for (DiscoveryNode tribe : tribeState.nodes()) {
-                if (currentState.nodes().get(tribe.getId()) == null) {
+                if (currentState.nodes().nodeExists(tribe) == false) {
                     // a new node, add it, but also add the tribe name to the attributes
                     Map<String, String> tribeAttr = new HashMap<>(tribe.getAttributes());
                     tribeAttr.put(TRIBE_NAME_SETTING.getKey(), tribeName);
-                    DiscoveryNode discoNode = new DiscoveryNode(tribe.getName(), tribe.getId(), tribe.getHostName(), tribe.getHostAddress(),
-                            tribe.getAddress(), unmodifiableMap(tribeAttr), tribe.getRoles(), tribe.getVersion());
+                    DiscoveryNode discoNode = new DiscoveryNode(tribe.getName(), tribe.getId(), tribe.getEphemeralId(),
+                        tribe.getHostName(), tribe.getHostAddress(), tribe.getAddress(), unmodifiableMap(tribeAttr), tribe.getRoles(),
+                        tribe.getVersion());
                     clusterStateChanged = true;
                     logger.info("[{}] adding node [{}]", tribeName, discoNode);
+                    nodes.remove(tribe.getId()); // remove any existing node with the same id but different ephemeral id
                     nodes.put(discoNode);
                 }
             }
diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
index 1561ab7a77c..5b39c5d34dc 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java
@@ -31,7 +31,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.Decision;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -65,7 +65,7 @@ public final class ClusterAllocationExplanationTests extends ESTestCase {
             .numberOfShards(1)
             .numberOfReplicas(1)
             .build();
-    private DiscoveryNode node = new DiscoveryNode("node-0", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+    private DiscoveryNode node = new DiscoveryNode("node-0", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
     private static Decision.Multi yesDecision = new Decision.Multi();
     private static Decision.Multi noDecision = new Decision.Multi();
 
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java
index 3c12d7d9b10..9705009a044 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreResponseTests.java
@@ -26,7 +26,7 @@ import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.ImmutableOpenIntMap;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -53,8 +53,8 @@ public class IndicesShardStoreResponseTests extends ESTestCase {
         List<IndicesShardStoresResponse.Failure> failures = new ArrayList<>();
         ImmutableOpenIntMap.Builder<List<IndicesShardStoresResponse.StoreStatus>> storeStatuses = ImmutableOpenIntMap.builder();
 
-        DiscoveryNode node1 = new DiscoveryNode("node1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
-        DiscoveryNode node2 = new DiscoveryNode("node2", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        DiscoveryNode node1 = new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
+        DiscoveryNode node2 = new DiscoveryNode("node2", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
         List<IndicesShardStoresResponse.StoreStatus> storeStatusList = new ArrayList<>();
         storeStatusList.add(new IndicesShardStoresResponse.StoreStatus(node1, 3, null, IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null));
         storeStatusList.add(new IndicesShardStoresResponse.StoreStatus(node2, ShardStateMetaData.NO_VERSION, UUIDs.randomBase64UUID(), IndicesShardStoresResponse.StoreStatus.AllocationStatus.REPLICA, null));
@@ -122,7 +122,7 @@ public class IndicesShardStoreResponseTests extends ESTestCase {
     }
 
     public void testStoreStatusOrdering() throws Exception {
-        DiscoveryNode node1 = new DiscoveryNode("node1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        DiscoveryNode node1 = new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
         List<IndicesShardStoresResponse.StoreStatus> orderedStoreStatuses = new ArrayList<>();
         orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, ShardStateMetaData.NO_VERSION, UUIDs.randomBase64UUID(), IndicesShardStoresResponse.StoreStatus.AllocationStatus.PRIMARY, null));
         orderedStoreStatuses.add(new IndicesShardStoresResponse.StoreStatus(node1, ShardStateMetaData.NO_VERSION, UUIDs.randomBase64UUID(), IndicesShardStoresResponse.StoreStatus.AllocationStatus.REPLICA, null));
diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java
index d78374d446f..3236de4aaeb 100644
--- a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java
@@ -38,7 +38,7 @@ import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllo
 import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.gateway.NoopGatewayAllocator;
@@ -140,7 +140,7 @@ public class TransportShrinkActionTests extends ESTestCase {
     }
 
     private DiscoveryNode newNode(String nodeId) {
-        return new DiscoveryNode(nodeId, DummyTransportAddress.INSTANCE, emptyMap(),
+        return new DiscoveryNode(nodeId, LocalTransportAddress.buildUnique(), emptyMap(),
             Collections.unmodifiableSet(new HashSet<>(Arrays.asList(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA))), Version.CURRENT);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/action/ingest/IngestProxyActionFilterTests.java b/core/src/test/java/org/elasticsearch/action/ingest/IngestProxyActionFilterTests.java
index a62946bf0f6..3d1a1a1c69d 100644
--- a/core/src/test/java/org/elasticsearch/action/ingest/IngestProxyActionFilterTests.java
+++ b/core/src/test/java/org/elasticsearch/action/ingest/IngestProxyActionFilterTests.java
@@ -33,7 +33,7 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.VersionUtils;
@@ -78,7 +78,7 @@ public class IngestProxyActionFilterTests extends ESTestCase {
             if (i < ingestNodes) {
                 roles.add(DiscoveryNode.Role.INGEST);
             }
-            DiscoveryNode node = new DiscoveryNode(nodeId, nodeId, DummyTransportAddress.INSTANCE, attributes, roles, VersionUtils.randomVersion(random()));
+            DiscoveryNode node = new DiscoveryNode(nodeId, nodeId, LocalTransportAddress.buildUnique(), attributes, roles, VersionUtils.randomVersion(random()));
             builder.put(node);
             if (i == totalNodes - 1) {
                 localNode = node;
diff --git a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
index 5e49518bd5a..603ad664ec3 100644
--- a/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/broadcast/node/TransportBroadcastByNodeActionTests.java
@@ -49,7 +49,7 @@ import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
@@ -247,7 +247,7 @@ public class TransportBroadcastByNodeActionTests extends ESTestCase {
     }
 
     static DiscoveryNode newNode(int nodeId) {
-        return new DiscoveryNode("node_" + nodeId, DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        return new DiscoveryNode("node_" + nodeId, LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
     }
 
     @AfterClass
diff --git a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
index c6d17605eb4..5d01aa369b0 100644
--- a/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/master/TransportMasterNodeActionTests.java
@@ -37,7 +37,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.discovery.MasterNotDiscoveredException;
@@ -89,9 +89,9 @@ public class TransportMasterNodeActionTests extends ESTestCase {
         transportService = new TransportService(clusterService.getSettings(), transport, threadPool);
         transportService.start();
         transportService.acceptIncomingRequests();
-        localNode = new DiscoveryNode("local_node", DummyTransportAddress.INSTANCE, Collections.emptyMap(),
+        localNode = new DiscoveryNode("local_node", LocalTransportAddress.buildUnique(), Collections.emptyMap(),
                 Collections.singleton(DiscoveryNode.Role.MASTER), Version.CURRENT);
-        remoteNode = new DiscoveryNode("remote_node", DummyTransportAddress.INSTANCE, Collections.emptyMap(),
+        remoteNode = new DiscoveryNode("remote_node", LocalTransportAddress.buildUnique(), Collections.emptyMap(),
                 Collections.singleton(DiscoveryNode.Role.MASTER), Version.CURRENT);
         allNodes = new DiscoveryNode[]{localNode, remoteNode};
     }
diff --git a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
index a15f89bced4..ae8ea4a0b95 100644
--- a/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
+++ b/core/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java
@@ -32,7 +32,7 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.transport.CapturingTransport;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -236,7 +236,7 @@ public class TransportNodesActionTests extends ESTestCase {
 
     private static DiscoveryNode newNode(int nodeId, Map<String, String> attributes, Set<DiscoveryNode.Role> roles) {
         String node = "node_" + nodeId;
-        return new DiscoveryNode(node, node, DummyTransportAddress.INSTANCE, attributes, roles, Version.CURRENT);
+        return new DiscoveryNode(node, node, LocalTransportAddress.buildUnique(), attributes, roles, Version.CURRENT);
     }
 
     private static class TestTransportNodesAction
diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
index d0e26c22cfb..7496bb85faf 100644
--- a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
+++ b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java
@@ -34,7 +34,7 @@ import org.elasticsearch.cluster.routing.ShardRoutingState;
 import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.index.shard.ShardId;
 
 import java.util.Arrays;
@@ -255,7 +255,7 @@ public class ClusterStateCreationUtils {
     }
 
     private static DiscoveryNode newNode(int nodeId) {
-        return new DiscoveryNode("node_" + nodeId, DummyTransportAddress.INSTANCE, Collections.emptyMap(),
+        return new DiscoveryNode("node_" + nodeId, LocalTransportAddress.buildUnique(), Collections.emptyMap(),
                 new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
index 42ec724ed52..3d50e2e44ac 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientNodesServiceTests.java
@@ -122,7 +122,7 @@ public class TransportClientNodesServiceTests extends ESTestCase {
                 @SuppressWarnings("unchecked")
                 public void handleResponse(T response) {
                     LivenessResponse livenessResponse = new LivenessResponse(clusterName,
-                            new DiscoveryNode(node.getName(), node.getId(), "liveness-hostname" + node.getId(),
+                            new DiscoveryNode(node.getName(), node.getId(), node.getEphemeralId(), "liveness-hostname" + node.getId(),
                                     "liveness-hostaddress" + node.getId(),
                                     new LocalTransportAddress("liveness-address-" + node.getId()), node.getAttributes(), node.getRoles(),
                                     node.getVersion()));
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java b/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
index 72748a59986..555f23813cb 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterChangedEventTests.java
@@ -30,7 +30,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.gateway.GatewayService;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.test.ESTestCase;
@@ -320,7 +320,8 @@ public class ClusterChangedEventTests extends ESTestCase {
 
     // Create a new DiscoveryNode
     private static DiscoveryNode newNode(final String nodeId, Set<DiscoveryNode.Role> roles) {
-        return new DiscoveryNode(nodeId, nodeId, DummyTransportAddress.INSTANCE, Collections.emptyMap(), roles, Version.CURRENT);
+        return new DiscoveryNode(nodeId, nodeId, nodeId, "host", "host_address", new LocalTransportAddress("_test_" + nodeId),
+            Collections.emptyMap(), roles, Version.CURRENT);
     }
 
     // Create the metadata for a cluster state.
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
index da196d5d1d3..68a0f73eb34 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
@@ -53,6 +53,7 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.snapshots.Snapshot;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.Collections;
@@ -191,9 +192,8 @@ public class ClusterStateDiffIT extends ESIntegTestCase {
         List<String> nodeIds = randomSubsetOf(randomInt(clusterState.nodes().getNodes().size() - 1), clusterState.nodes().getNodes().keys().toArray(String.class));
         for (String nodeId : nodeIds) {
             if (nodeId.startsWith("node-")) {
+                nodes.remove(nodeId);
                 if (randomBoolean()) {
-                    nodes.remove(nodeId);
-                } else {
                     nodes.put(new DiscoveryNode(nodeId, new LocalTransportAddress(randomAsciiOfLength(10)), emptyMap(),
                             emptySet(), randomVersion(random())));
                 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/core/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java
index 5e272a27459..6b99e525cb2 100644
--- a/core/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java
@@ -22,7 +22,7 @@ import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.test.ESTestCase;
 
 import static java.util.Collections.emptyMap;
@@ -32,8 +32,9 @@ import static org.hamcrest.Matchers.equalTo;
 public class ClusterStateTests extends ESTestCase {
 
     public void testSupersedes() {
-        final DiscoveryNode node1 = new DiscoveryNode("node1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
-        final DiscoveryNode node2 = new DiscoveryNode("node2", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        final Version version = Version.CURRENT;
+        final DiscoveryNode node1 = new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), version);
+        final DiscoveryNode node2 = new DiscoveryNode("node2", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), version);
         final DiscoveryNodes nodes = DiscoveryNodes.builder().put(node1).put(node2).build();
         ClusterName name = ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY);
         ClusterState noMaster1 = ClusterState.builder(name).version(randomInt(5)).nodes(nodes).build();
diff --git a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
index 9210e2a56bd..2073235af98 100644
--- a/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java
@@ -23,8 +23,6 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.ShardStats;
-import org.elasticsearch.cluster.ClusterName;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -33,7 +31,7 @@ import org.elasticsearch.cluster.routing.ShardRoutingHelper;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.ShardPath;
@@ -200,11 +198,11 @@ public class DiskUsageTests extends ESTestCase {
                 new FsInfo.Path("/most", "/dev/sda", 100, 90, 80),
         };
         List<NodeStats> nodeStats = Arrays.asList(
-                new NodeStats(new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0,
+                new NodeStats(new DiscoveryNode("node_1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT), 0,
                         null,null,null,null,null,new FsInfo(0, null, node1FSInfo), null,null,null,null,null, null),
-                new NodeStats(new DiscoveryNode("node_2", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0,
+                new NodeStats(new DiscoveryNode("node_2", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT), 0,
                         null,null,null,null,null, new FsInfo(0, null, node2FSInfo), null,null,null,null,null, null),
-                new NodeStats(new DiscoveryNode("node_3", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0,
+                new NodeStats(new DiscoveryNode("node_3", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT), 0,
                         null,null,null,null,null, new FsInfo(0, null, node3FSInfo), null,null,null,null,null, null)
         );
         InternalClusterInfoService.fillDiskUsagePerNode(logger, nodeStats, newLeastAvaiableUsages, newMostAvaiableUsages);
@@ -241,11 +239,11 @@ public class DiskUsageTests extends ESTestCase {
                 new FsInfo.Path("/least", "/dev/sda", 10, -8, 0),
         };
         List<NodeStats> nodeStats = Arrays.asList(
-                new NodeStats(new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0,
+                new NodeStats(new DiscoveryNode("node_1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT), 0,
                         null,null,null,null,null,new FsInfo(0, null, node1FSInfo), null,null,null,null,null, null),
-                new NodeStats(new DiscoveryNode("node_2", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0,
+                new NodeStats(new DiscoveryNode("node_2", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT), 0,
                         null,null,null,null,null, new FsInfo(0, null, node2FSInfo), null,null,null,null,null, null),
-                new NodeStats(new DiscoveryNode("node_3", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT), 0,
+                new NodeStats(new DiscoveryNode("node_3", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT), 0,
                         null,null,null,null,null, new FsInfo(0, null, node3FSInfo), null,null,null,null,null, null)
         );
         InternalClusterInfoService.fillDiskUsagePerNode(logger, nodeStats, newLeastAvailableUsages, newMostAvailableUsages);
diff --git a/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
index 95dd97b4702..b0bc3ee2e4e 100644
--- a/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/NodeConnectionsServiceTests.java
@@ -26,7 +26,7 @@ import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.component.LifecycleListener;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.test.ESTestCase;
@@ -64,7 +64,7 @@ public class NodeConnectionsServiceTests extends ESTestCase {
         List<DiscoveryNode> nodes = new ArrayList<>();
         for (int i = randomIntBetween(20, 50); i > 0; i--) {
             Set<DiscoveryNode.Role> roles = new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values())));
-            nodes.add(new DiscoveryNode("node_" + i, "" + i, DummyTransportAddress.INSTANCE, Collections.emptyMap(),
+            nodes.add(new DiscoveryNode("node_" + i, "" + i, LocalTransportAddress.buildUnique(), Collections.emptyMap(),
                     roles, Version.CURRENT));
         }
         return nodes;
diff --git a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java
index 1faac874114..d12b6b563b3 100644
--- a/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/action/shard/ShardFailedClusterStateTaskExecutorTests.java
@@ -27,7 +27,6 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.node.DiscoveryNodeService;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
@@ -41,6 +40,7 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
@@ -305,7 +305,8 @@ public class ShardFailedClusterStateTaskExecutorTests extends ESAllocationTestCa
             return randomSubsetOf(1, shards.toArray(new ShardRouting[0])).get(0);
         } else {
             return
-                    TestShardRouting.newShardRouting(shardRouting.shardId(), DiscoveryNodeService.generateNodeId(Settings.EMPTY), randomBoolean(), randomFrom(ShardRoutingState.values()));
+                    TestShardRouting.newShardRouting(shardRouting.shardId(), UUIDs.randomBase64UUID(random()), randomBoolean(),
+                        randomFrom(ShardRoutingState.values()));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
index 04d27020273..ff31a72a5c6 100644
--- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java
@@ -34,7 +34,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.indices.InvalidIndexNameException;
@@ -181,7 +181,7 @@ public class MetaDataCreateIndexServiceTests extends ESTestCase {
     }
 
     private DiscoveryNode newNode(String nodeId) {
-        return new DiscoveryNode(nodeId, DummyTransportAddress.INSTANCE, emptyMap(),
+        return new DiscoveryNode(nodeId, LocalTransportAddress.buildUnique(), emptyMap(),
             Collections.unmodifiableSet(new HashSet<>(Arrays.asList(DiscoveryNode.Role.MASTER, DiscoveryNode.Role.DATA))), Version.CURRENT);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeFiltersTests.java b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeFiltersTests.java
index 38aa73a9935..59f058a95fb 100644
--- a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeFiltersTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeFiltersTests.java
@@ -21,8 +21,8 @@ package org.elasticsearch.cluster.node;
 
 import org.elasticsearch.Version;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -64,10 +64,11 @@ public class DiscoveryNodeFiltersTests extends ESTestCase {
                 .build();
         DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(OR, "xxx.", settings);
 
-        DiscoveryNode node = new DiscoveryNode("name1", "id1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        DiscoveryNode node = new DiscoveryNode("name1", "id1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(),
+            Version.CURRENT);
         assertThat(filters.match(node), equalTo(true));
 
-        node = new DiscoveryNode("name2", "id2", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        node = new DiscoveryNode("name2", "id2", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
         assertThat(filters.match(node), equalTo(false));
     }
 
@@ -77,10 +78,11 @@ public class DiscoveryNodeFiltersTests extends ESTestCase {
                 .build();
         DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(OR, "xxx.", settings);
 
-        DiscoveryNode node = new DiscoveryNode("name1", "id1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        DiscoveryNode node = new DiscoveryNode("name1", "id1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(),
+            Version.CURRENT);
         assertThat(filters.match(node), equalTo(true));
 
-        node = new DiscoveryNode("name2", "id2", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        node = new DiscoveryNode("name2", "id2", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
         assertThat(filters.match(node), equalTo(false));
     }
 
@@ -91,13 +93,14 @@ public class DiscoveryNodeFiltersTests extends ESTestCase {
                 .build());
         DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(OR, "xxx.", settings);
 
-        DiscoveryNode node = new DiscoveryNode("name1", "id1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        final Version version = Version.CURRENT;
+        DiscoveryNode node = new DiscoveryNode("name1", "id1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), version);
         assertThat(filters.match(node), equalTo(true));
 
-        node = new DiscoveryNode("name2", "id2", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        node = new DiscoveryNode("name2", "id2", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), version);
         assertThat(filters.match(node), equalTo(true));
 
-        node = new DiscoveryNode("name3", "id3", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        node = new DiscoveryNode("name3", "id3", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), version);
         assertThat(filters.match(node), equalTo(false));
     }
 
@@ -111,7 +114,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase {
         Map<String, String> attributes = new HashMap<>();
         attributes.put("tag", "A");
         attributes.put("group", "B");
-        DiscoveryNode node = new DiscoveryNode("name1", "id1", DummyTransportAddress.INSTANCE,
+        DiscoveryNode node = new DiscoveryNode("name1", "id1", LocalTransportAddress.buildUnique(),
                 attributes, emptySet(), Version.CURRENT);
         assertThat(filters.match(node), equalTo(true));
 
@@ -119,7 +122,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase {
         attributes.put("tag", "A");
         attributes.put("group", "B");
         attributes.put("name", "X");
-        node = new DiscoveryNode("name2", "id2", DummyTransportAddress.INSTANCE,
+        node = new DiscoveryNode("name2", "id2", LocalTransportAddress.buildUnique(),
                 attributes, emptySet(), Version.CURRENT);
         assertThat(filters.match(node), equalTo(true));
 
@@ -127,11 +130,11 @@ public class DiscoveryNodeFiltersTests extends ESTestCase {
         attributes.put("tag", "A");
         attributes.put("group", "F");
         attributes.put("name", "X");
-        node = new DiscoveryNode("name3", "id3", DummyTransportAddress.INSTANCE,
+        node = new DiscoveryNode("name3", "id3", LocalTransportAddress.buildUnique(),
                 attributes, emptySet(), Version.CURRENT);
         assertThat(filters.match(node), equalTo(false));
 
-        node = new DiscoveryNode("name4", "id4", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        node = new DiscoveryNode("name4", "id4", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
         assertThat(filters.match(node), equalTo(false));
     }
 
@@ -141,7 +144,8 @@ public class DiscoveryNodeFiltersTests extends ESTestCase {
                 .build();
         DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(OR, "xxx.", settings);
 
-        DiscoveryNode node = new DiscoveryNode("name1", "id1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        DiscoveryNode node = new DiscoveryNode("name1", "id1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(),
+            Version.CURRENT);
         assertThat(filters.match(node), equalTo(true));
     }
 
@@ -152,7 +156,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase {
                 .build());
         DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(AND, "xxx.", settings);
 
-        DiscoveryNode node = new DiscoveryNode("", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null);
+        DiscoveryNode node = new DiscoveryNode("", "", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null);
         assertThat(filters.match(node), equalTo(true));
     }
 
@@ -163,7 +167,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase {
                 .build());
         DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(AND, "xxx.", settings);
 
-        DiscoveryNode node = new DiscoveryNode("", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null);
+        DiscoveryNode node = new DiscoveryNode("", "", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null);
         assertThat(filters.match(node), equalTo(false));
     }
 
@@ -174,7 +178,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase {
                 .build());
         DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(AND, "xxx.", settings);
 
-        DiscoveryNode node = new DiscoveryNode("", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null);
+        DiscoveryNode node = new DiscoveryNode("", "", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null);
         assertThat(filters.match(node), equalTo(false));
     }
 
@@ -185,7 +189,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase {
                 .build());
         DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(OR, "xxx.", settings);
 
-        DiscoveryNode node = new DiscoveryNode("", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null);
+        DiscoveryNode node = new DiscoveryNode("", "", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null);
         assertThat(filters.match(node), equalTo(true));
     }
 
@@ -196,7 +200,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase {
                 .build());
         DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(OR, "xxx.", settings);
 
-        DiscoveryNode node = new DiscoveryNode("", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null);
+        DiscoveryNode node = new DiscoveryNode("", "", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null);
         assertThat(filters.match(node), equalTo(true));
     }
 
@@ -207,7 +211,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase {
                 .build());
         DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(AND, "xxx.", settings);
 
-        DiscoveryNode node = new DiscoveryNode("", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null);
+        DiscoveryNode node = new DiscoveryNode("", "", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null);
         assertThat(filters.match(node), equalTo(true));
     }
 
@@ -218,7 +222,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase {
                 .build());
         DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(AND, "xxx.", settings);
 
-        DiscoveryNode node = new DiscoveryNode("", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null);
+        DiscoveryNode node = new DiscoveryNode("", "", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null);
         assertThat(filters.match(node), equalTo(false));
     }
 
@@ -229,7 +233,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase {
                 .build());
         DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(OR, "xxx.", settings);
 
-        DiscoveryNode node = new DiscoveryNode("", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null);
+        DiscoveryNode node = new DiscoveryNode("", "", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null);
         assertThat(filters.match(node), equalTo(true));
     }
 
@@ -240,7 +244,7 @@ public class DiscoveryNodeFiltersTests extends ESTestCase {
                 .build());
         DiscoveryNodeFilters filters = DiscoveryNodeFilters.buildFromSettings(OR, "xxx.", settings);
 
-        DiscoveryNode node = new DiscoveryNode("", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null);
+        DiscoveryNode node = new DiscoveryNode("", "", "", "", "192.1.1.54", localAddress, singletonMap("tag", "A"), emptySet(), null);
         assertThat(filters.match(node), equalTo(true));
     }
 
diff --git a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeServiceTests.java
index fb38a428a76..e9f80426df6 100644
--- a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeServiceTests.java
@@ -19,9 +19,9 @@
 
 package org.elasticsearch.cluster.node;
 
-import org.elasticsearch.Version;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.HashMap;
@@ -55,7 +55,8 @@ public class DiscoveryNodeServiceTests extends ESTestCase {
             }
         }
         DiscoveryNodeService discoveryNodeService = new DiscoveryNodeService(builder.build());
-        DiscoveryNode discoveryNode = discoveryNodeService.buildLocalNode(DummyTransportAddress.INSTANCE);
+        DiscoveryNode discoveryNode = discoveryNodeService.buildLocalNode(LocalTransportAddress.buildUnique(),
+            () -> UUIDs.randomBase64UUID(random()));
         assertThat(discoveryNode.getRoles(), equalTo(selectedRoles));
         assertThat(discoveryNode.getAttributes(), equalTo(expectedAttributes));
     }
@@ -77,7 +78,8 @@ public class DiscoveryNodeServiceTests extends ESTestCase {
         expectedAttributes.putAll(customAttributes);
         discoveryNodeService.addCustomAttributeProvider(() -> customAttributes);
 
-        DiscoveryNode discoveryNode = discoveryNodeService.buildLocalNode(DummyTransportAddress.INSTANCE);
+        DiscoveryNode discoveryNode = discoveryNodeService.buildLocalNode(LocalTransportAddress.buildUnique(),
+            () -> UUIDs.randomBase64UUID(random()));
         assertThat(discoveryNode.getAttributes(), equalTo(expectedAttributes));
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java
index 984cd31b7a0..ec741a908c5 100644
--- a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodesTests.java
@@ -19,8 +19,9 @@
 
 package org.elasticsearch.cluster.node;
 
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
 import org.elasticsearch.Version;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
@@ -30,10 +31,15 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.nullValue;
 
 public class DiscoveryNodesTests extends ESTestCase {
 
@@ -53,7 +59,7 @@ public class DiscoveryNodesTests extends ESTestCase {
             DiscoveryNode resolvedNode = discoveryNodes.resolveNode(nodeSelector.selector);
             assertThat(matchingNodeIds.size(), equalTo(1));
             assertThat(resolvedNode.getId(), equalTo(matchingNodeIds.iterator().next()));
-        } catch(IllegalArgumentException e) {
+        } catch (IllegalArgumentException e) {
             if (matchingNodeIds.size() == 0) {
                 assertThat(e.getMessage(), equalTo("failed to resolve [" + nodeSelector.selector + "], no matching nodes"));
             } else if (matchingNodeIds.size() > 1) {
@@ -98,26 +104,98 @@ public class DiscoveryNodesTests extends ESTestCase {
         assertThat(resolvedNodesIds, equalTo(expectedNodesIds));
     }
 
-    private static DiscoveryNodes buildDiscoveryNodes() {
-        int numNodes = randomIntBetween(1, 10);
-        DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder();
+    public void testDeltas() {
+        Set<DiscoveryNode> nodesA = new HashSet<>();
+        nodesA.addAll(randomNodes(1 + randomInt(10)));
+        Set<DiscoveryNode> nodesB = new HashSet<>();
+        nodesB.addAll(randomNodes(1 + randomInt(5)));
+        for (DiscoveryNode node : randomSubsetOf(nodesA)) {
+            if (randomBoolean()) {
+                // change an attribute
+                Map<String, String> attrs = new HashMap<>(node.getAttributes());
+                attrs.put("new", "new");
+                node = new DiscoveryNode(node.getName(), node.getId(), node.getAddress(), attrs, node.getRoles(), node.getVersion());
+            }
+            nodesB.add(node);
+        }
+
+        DiscoveryNode masterA = randomBoolean() ? null : RandomPicks.randomFrom(random(), nodesA);
+        DiscoveryNode masterB = randomBoolean() ? null : RandomPicks.randomFrom(random(), nodesB);
+
+        DiscoveryNodes.Builder builderA = DiscoveryNodes.builder();
+        nodesA.stream().forEach(builderA::put);
+        final String masterAId = masterA == null ? null : masterA.getId();
+        builderA.masterNodeId(masterAId);
+        builderA.localNodeId(RandomPicks.randomFrom(random(), nodesA).getId());
+
+        DiscoveryNodes.Builder builderB = DiscoveryNodes.builder();
+        nodesB.stream().forEach(builderB::put);
+        final String masterBId = masterB == null ? null : masterB.getId();
+        builderB.masterNodeId(masterBId);
+        builderB.localNodeId(RandomPicks.randomFrom(random(), nodesB).getId());
+
+        final DiscoveryNodes discoNodesA = builderA.build();
+        final DiscoveryNodes discoNodesB = builderB.build();
+        logger.info("nodes A: {}", discoNodesA.prettyPrint());
+        logger.info("nodes B: {}", discoNodesB.prettyPrint());
+
+        DiscoveryNodes.Delta delta = discoNodesB.delta(discoNodesA);
+
+        if (masterB == null || Objects.equals(masterAId, masterBId)) {
+            assertFalse(delta.masterNodeChanged());
+            assertThat(delta.previousMasterNode(), nullValue());
+            assertThat(delta.newMasterNode(), nullValue());
+        } else {
+            assertTrue(delta.masterNodeChanged());
+            assertThat(delta.newMasterNode().getId(), equalTo(masterBId));
+            assertThat(delta.previousMasterNode() != null ? delta.previousMasterNode().getId() : null,
+                equalTo(masterAId));
+        }
+
+        Set<DiscoveryNode> newNodes = new HashSet<>(nodesB);
+        newNodes.removeAll(nodesA);
+        assertThat(delta.added(), equalTo(newNodes.isEmpty() == false));
+        assertThat(delta.addedNodes(), containsInAnyOrder(newNodes.stream().collect(Collectors.toList()).toArray()));
+        assertThat(delta.addedNodes().size(), equalTo(newNodes.size()));
+
+        Set<DiscoveryNode> removedNodes = new HashSet<>(nodesA);
+        removedNodes.removeAll(nodesB);
+        assertThat(delta.removed(), equalTo(removedNodes.isEmpty() == false));
+        assertThat(delta.removedNodes(), containsInAnyOrder(removedNodes.stream().collect(Collectors.toList()).toArray()));
+        assertThat(delta.removedNodes().size(), equalTo(removedNodes.size()));
+    }
+
+    private static AtomicInteger idGenerator = new AtomicInteger();
+
+    private static List<DiscoveryNode> randomNodes(final int numNodes) {
         List<DiscoveryNode> nodesList = new ArrayList<>();
         for (int i = 0; i < numNodes; i++) {
             Map<String, String> attributes = new HashMap<>();
             if (frequently()) {
                 attributes.put("custom", randomBoolean() ? "match" : randomAsciiOfLengthBetween(3, 5));
             }
-            final DiscoveryNode node = newNode(i, attributes, new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))));
-            discoBuilder = discoBuilder.put(node);
+            final DiscoveryNode node = newNode(idGenerator.getAndIncrement(), attributes,
+                new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))));
             nodesList.add(node);
         }
+        return nodesList;
+    }
+
+    private static DiscoveryNodes buildDiscoveryNodes() {
+        int numNodes = randomIntBetween(1, 10);
+        DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder();
+        List<DiscoveryNode> nodesList = randomNodes(numNodes);
+        for (DiscoveryNode node : nodesList) {
+            discoBuilder = discoBuilder.put(node);
+        }
         discoBuilder.localNodeId(randomFrom(nodesList).getId());
         discoBuilder.masterNodeId(randomFrom(nodesList).getId());
         return discoBuilder.build();
     }
 
     private static DiscoveryNode newNode(int nodeId, Map<String, String> attributes, Set<DiscoveryNode.Role> roles) {
-        return new DiscoveryNode("name_" + nodeId, "node_" + nodeId, DummyTransportAddress.INSTANCE, attributes, roles, Version.CURRENT);
+        return new DiscoveryNode("name_" + nodeId, "node_" + nodeId, LocalTransportAddress.buildUnique(), attributes, roles,
+            Version.CURRENT);
     }
 
     private enum NodeSelector {
@@ -152,7 +230,7 @@ public class DiscoveryNodesTests extends ESTestCase {
                 nodes.getIngestNodes().keysIt().forEachRemaining(ids::add);
                 return ids;
             }
-        },CUSTOM_ATTRIBUTE("attr:value") {
+        }, CUSTOM_ATTRIBUTE("attr:value") {
             @Override
             Set<String> matchingNodeIds(DiscoveryNodes nodes) {
                 Set<String> ids = new HashSet<>();
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
index f3ac9c8eff0..9859cd4d570 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/NodeVersionAllocationDeciderTests.java
@@ -25,8 +25,6 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.EmptyClusterInfoService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
@@ -44,12 +42,14 @@ import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider;
+import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.snapshots.Snapshot;
+import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.test.ESAllocationTestCase;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.gateway.NoopGatewayAllocator;
@@ -307,11 +307,11 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase {
     public void testRebalanceDoesNotAllocatePrimaryAndReplicasOnDifferentVersionNodes() {
         ShardId shard1 = new ShardId("test1", "_na_", 0);
         ShardId shard2 = new ShardId("test2", "_na_", 0);
-        final DiscoveryNode newNode = new DiscoveryNode("newNode", DummyTransportAddress.INSTANCE, emptyMap(),
+        final DiscoveryNode newNode = new DiscoveryNode("newNode", LocalTransportAddress.buildUnique(), emptyMap(),
                 MASTER_DATA_ROLES, Version.CURRENT);
-        final DiscoveryNode oldNode1 = new DiscoveryNode("oldNode1", DummyTransportAddress.INSTANCE, emptyMap(),
+        final DiscoveryNode oldNode1 = new DiscoveryNode("oldNode1", LocalTransportAddress.buildUnique(), emptyMap(),
                 MASTER_DATA_ROLES, VersionUtils.getPreviousVersion());
-        final DiscoveryNode oldNode2 = new DiscoveryNode("oldNode2", DummyTransportAddress.INSTANCE, emptyMap(),
+        final DiscoveryNode oldNode2 = new DiscoveryNode("oldNode2", LocalTransportAddress.buildUnique(), emptyMap(),
                 MASTER_DATA_ROLES, VersionUtils.getPreviousVersion());
         MetaData metaData = MetaData.builder()
             .put(IndexMetaData.builder(shard1.getIndexName()).settings(settings(Version.CURRENT).put(Settings.EMPTY)).numberOfShards(1).numberOfReplicas(1))
@@ -347,11 +347,11 @@ public class NodeVersionAllocationDeciderTests extends ESAllocationTestCase {
     }
 
     public void testRestoreDoesNotAllocateSnapshotOnOlderNodes() {
-        final DiscoveryNode newNode = new DiscoveryNode("newNode", DummyTransportAddress.INSTANCE, emptyMap(),
+        final DiscoveryNode newNode = new DiscoveryNode("newNode", LocalTransportAddress.buildUnique(), emptyMap(),
                 MASTER_DATA_ROLES, Version.CURRENT);
-        final DiscoveryNode oldNode1 = new DiscoveryNode("oldNode1", DummyTransportAddress.INSTANCE, emptyMap(),
+        final DiscoveryNode oldNode1 = new DiscoveryNode("oldNode1", LocalTransportAddress.buildUnique(), emptyMap(),
                 MASTER_DATA_ROLES, VersionUtils.getPreviousVersion());
-        final DiscoveryNode oldNode2 = new DiscoveryNode("oldNode2", DummyTransportAddress.INSTANCE, emptyMap(),
+        final DiscoveryNode oldNode2 = new DiscoveryNode("oldNode2", LocalTransportAddress.buildUnique(), emptyMap(),
                 MASTER_DATA_ROLES, VersionUtils.getPreviousVersion());
 
         int numberOfShards = randomIntBetween(1, 3);
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java
index 3d0475ed137..e09d9790651 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/SameShardRoutingTests.java
@@ -33,7 +33,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationD
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.test.ESAllocationTestCase;
 
 import static java.util.Collections.emptyMap;
@@ -63,9 +63,9 @@ public class SameShardRoutingTests extends ESAllocationTestCase {
         logger.info("--> adding two nodes with the same host");
         clusterState = ClusterState.builder(clusterState).nodes(
                 DiscoveryNodes.builder()
-                .put(new DiscoveryNode("node1", "node1", "test1", "test1", DummyTransportAddress.INSTANCE, emptyMap(),
+                .put(new DiscoveryNode("node1", "node1", "node1", "test1", "test1", LocalTransportAddress.buildUnique(), emptyMap(),
                         MASTER_DATA_ROLES, Version.CURRENT))
-                .put(new DiscoveryNode("node2", "node2", "test1", "test1", DummyTransportAddress.INSTANCE, emptyMap(),
+                .put(new DiscoveryNode("node2", "node2", "node2", "test1", "test1", LocalTransportAddress.buildUnique(), emptyMap(),
                         MASTER_DATA_ROLES, Version.CURRENT))).build();
         routingTable = strategy.reroute(clusterState, "reroute").routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
@@ -82,7 +82,7 @@ public class SameShardRoutingTests extends ESAllocationTestCase {
 
         logger.info("--> add another node, with a different host, replicas will be allocating");
         clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())
-                .put(new DiscoveryNode("node3", "node3", "test2", "test2", DummyTransportAddress.INSTANCE, emptyMap(),
+                .put(new DiscoveryNode("node3", "node3", "node3", "test2", "test2", LocalTransportAddress.buildUnique(), emptyMap(),
                         MASTER_DATA_ROLES, Version.CURRENT))).build();
         routingTable = strategy.reroute(clusterState, "reroute").routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
index be50c5f5331..56ca6381af9 100644
--- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java
@@ -41,7 +41,6 @@ import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.index.Index;
@@ -110,9 +109,9 @@ public class DiskThresholdDeciderUnitTests extends ESAllocationTestCase {
         final Index index = metaData.index("test").getIndex();
 
         ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
-        DiscoveryNode node_0 = new DiscoveryNode("node_0", DummyTransportAddress.INSTANCE, Collections.emptyMap(),
+        DiscoveryNode node_0 = new DiscoveryNode("node_0", LocalTransportAddress.buildUnique(), Collections.emptyMap(),
                 new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
-        DiscoveryNode node_1 = new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, Collections.emptyMap(),
+        DiscoveryNode node_1 = new DiscoveryNode("node_1", LocalTransportAddress.buildUnique(), Collections.emptyMap(),
                 new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
 
         RoutingTable routingTable = RoutingTable.builder()
@@ -149,9 +148,9 @@ public class DiskThresholdDeciderUnitTests extends ESAllocationTestCase {
         DiskThresholdDecider decider = new DiskThresholdDecider(Settings.EMPTY, nss, cis, null);
         ImmutableOpenMap.Builder<ShardRouting, String> shardRoutingMap = ImmutableOpenMap.builder();
 
-        DiscoveryNode node_0 = new DiscoveryNode("node_0", DummyTransportAddress.INSTANCE, Collections.emptyMap(),
+        DiscoveryNode node_0 = new DiscoveryNode("node_0", LocalTransportAddress.buildUnique(), Collections.emptyMap(),
                 new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
-        DiscoveryNode node_1 = new DiscoveryNode("node_1", DummyTransportAddress.INSTANCE, Collections.emptyMap(),
+        DiscoveryNode node_1 = new DiscoveryNode("node_1", LocalTransportAddress.buildUnique(), Collections.emptyMap(),
                 new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
 
         MetaData metaData = MetaData.builder()
diff --git a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java
index 99cde60f086..9957a6d3603 100644
--- a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterStateToStringTests.java
@@ -29,7 +29,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.test.ESAllocationTestCase;
 
 import static java.util.Collections.emptyMap;
@@ -50,7 +50,7 @@ public class ClusterStateToStringTests extends ESAllocationTestCase {
                 .addAsNew(metaData.index("test_idx"))
                 .build();
 
-        DiscoveryNodes nodes = DiscoveryNodes.builder().put(new DiscoveryNode("node_foo", DummyTransportAddress.INSTANCE,
+        DiscoveryNodes nodes = DiscoveryNodes.builder().put(new DiscoveryNode("node_foo", LocalTransportAddress.buildUnique(),
                 emptyMap(), emptySet(), Version.CURRENT)).localNodeId("node_foo").masterNodeId("node_foo").build();
 
         ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).nodes(nodes)
diff --git a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
index c94e441aea3..54f6233631b 100644
--- a/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/cluster/service/ClusterServiceTests.java
@@ -37,7 +37,7 @@ import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.MockLogAppender;
@@ -109,7 +109,7 @@ public class ClusterServiceTests extends ESTestCase {
         TimedClusterService timedClusterService = new TimedClusterService(Settings.builder().put("cluster.name",
             "ClusterServiceTests").build(), new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             threadPool);
-        timedClusterService.setLocalNode(new DiscoveryNode("node1", DummyTransportAddress.INSTANCE, emptyMap(),
+        timedClusterService.setLocalNode(new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), emptyMap(),
             emptySet(), Version.CURRENT));
         timedClusterService.setNodeConnectionsService(new NodeConnectionsService(Settings.EMPTY, null, null) {
             @Override
diff --git a/core/src/test/java/org/elasticsearch/common/util/IndexFolderUpgraderTests.java b/core/src/test/java/org/elasticsearch/common/util/IndexFolderUpgraderTests.java
index 26d6af1cd5f..5302ba8d55c 100644
--- a/core/src/test/java/org/elasticsearch/common/util/IndexFolderUpgraderTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/IndexFolderUpgraderTests.java
@@ -67,7 +67,7 @@ public class IndexFolderUpgraderTests extends ESTestCase {
     public void testUpgradeCustomDataPath() throws IOException {
         Path customPath = createTempDir();
         final Settings nodeSettings = Settings.builder()
-            .put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH.getKey(), randomBoolean())
+            .put(NodeEnvironment.ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.getKey(), randomBoolean())
             .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), customPath.toAbsolutePath().toString()).build();
         try (NodeEnvironment nodeEnv = newNodeEnvironment(nodeSettings)) {
             final Index index = new Index(randomAsciiOfLength(10), UUIDs.randomBase64UUID());
@@ -96,7 +96,7 @@ public class IndexFolderUpgraderTests extends ESTestCase {
     public void testPartialUpgradeCustomDataPath() throws IOException {
         Path customPath = createTempDir();
         final Settings nodeSettings = Settings.builder()
-            .put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH.getKey(), randomBoolean())
+            .put(NodeEnvironment.ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.getKey(), randomBoolean())
             .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), customPath.toAbsolutePath().toString()).build();
         try (NodeEnvironment nodeEnv = newNodeEnvironment(nodeSettings)) {
             final Index index = new Index(randomAsciiOfLength(10), UUIDs.randomBase64UUID());
@@ -136,7 +136,7 @@ public class IndexFolderUpgraderTests extends ESTestCase {
 
     public void testUpgrade() throws IOException {
         final Settings nodeSettings = Settings.builder()
-            .put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH.getKey(), randomBoolean()).build();
+            .put(NodeEnvironment.ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.getKey(), randomBoolean()).build();
         try (NodeEnvironment nodeEnv = newNodeEnvironment(nodeSettings)) {
             final Index index = new Index(randomAsciiOfLength(10), UUIDs.randomBase64UUID());
             Settings settings = Settings.builder()
@@ -159,7 +159,7 @@ public class IndexFolderUpgraderTests extends ESTestCase {
 
     public void testUpgradeIndices() throws IOException {
         final Settings nodeSettings = Settings.builder()
-            .put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH.getKey(), randomBoolean()).build();
+            .put(NodeEnvironment.ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.getKey(), randomBoolean()).build();
         try (NodeEnvironment nodeEnv = newNodeEnvironment(nodeSettings)) {
             Map<IndexSettings, Tuple<Integer, Integer>>  indexSettingsMap = new HashMap<>();
             for (int i = 0; i < randomIntBetween(2, 5); i++) {
@@ -256,7 +256,7 @@ public class IndexFolderUpgraderTests extends ESTestCase {
             .numberOfReplicas(0)
             .build();
         try (NodeEnvironment nodeEnvironment = newNodeEnvironment()) {
-            IndexMetaData.FORMAT.write(indexState, 1, nodeEnvironment.indexPaths(index));
+            IndexMetaData.FORMAT.write(indexState, nodeEnvironment.indexPaths(index));
             assertFalse(IndexFolderUpgrader.needsUpgrade(index, index.getUUID()));
         }
     }
@@ -305,7 +305,7 @@ public class IndexFolderUpgraderTests extends ESTestCase {
         for (int i = 0; i < nodePaths.length; i++) {
             oldIndexPaths[i] = nodePaths[i].indicesPath.resolve(indexSettings.getIndex().getName());
         }
-        IndexMetaData.FORMAT.write(indexSettings.getIndexMetaData(), 1, oldIndexPaths);
+        IndexMetaData.FORMAT.write(indexSettings.getIndexMetaData(), oldIndexPaths);
         for (int id = 0; id < indexSettings.getNumberOfShards(); id++) {
             Path oldIndexPath = randomFrom(oldIndexPaths);
             ShardId shardId = new ShardId(indexSettings.getIndex(), id);
@@ -316,7 +316,7 @@ public class IndexFolderUpgraderTests extends ESTestCase {
                 writeShard(shardId, oldIndexPath, numIdxFiles, numTranslogFiles);
             }
             ShardStateMetaData state = new ShardStateMetaData(true, indexSettings.getUUID(), AllocationId.newInitializing());
-            ShardStateMetaData.FORMAT.write(state, 1, oldIndexPath.resolve(String.valueOf(shardId.getId())));
+            ShardStateMetaData.FORMAT.write(state, oldIndexPath.resolve(String.valueOf(shardId.getId())));
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java b/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java
index 0f34ff181a7..4ff4c4cd035 100644
--- a/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java
@@ -21,7 +21,7 @@ package org.elasticsearch.discovery;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.test.ESTestCase;
@@ -77,7 +77,7 @@ public class BlockingClusterStatePublishResponseHandlerTests extends ESTestCase
         int nodeCount = scaledRandomIntBetween(10, 20);
         DiscoveryNode[] allNodes = new DiscoveryNode[nodeCount];
         for (int i = 0; i < nodeCount; i++) {
-            DiscoveryNode node = new DiscoveryNode("node_" + i, DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+            DiscoveryNode node = new DiscoveryNode("node_" + i, LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
             allNodes[i] = node;
         }
 
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java
index 0f93e5d460c..b31b0cbaa55 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java
@@ -22,7 +22,7 @@ package org.elasticsearch.discovery.zen;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.discovery.zen.elect.ElectMasterService;
 import org.elasticsearch.test.ESTestCase;
 
@@ -46,7 +46,7 @@ public class ElectMasterServiceTests extends ESTestCase {
             if (randomBoolean()) {
                 roles.add(DiscoveryNode.Role.MASTER);
             }
-            DiscoveryNode node = new DiscoveryNode("n_" + i, "n_" + i, DummyTransportAddress.INSTANCE, Collections.emptyMap(),
+            DiscoveryNode node = new DiscoveryNode("n_" + i, "n_" + i, LocalTransportAddress.buildUnique(), Collections.emptyMap(),
                     roles, Version.CURRENT);
             nodes.add(node);
         }
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
index 531b20aa3c9..cd2b4eaf2e4 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java
@@ -31,7 +31,6 @@ import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
 import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
@@ -40,6 +39,7 @@ import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.discovery.zen.elect.ElectMasterService;
 import org.elasticsearch.discovery.zen.membership.MembershipAction;
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -49,6 +49,8 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -67,6 +69,7 @@ import static java.util.Collections.emptySet;
 import static java.util.Collections.shuffle;
 import static org.elasticsearch.test.ClusterServiceUtils.createClusterService;
 import static org.elasticsearch.test.ClusterServiceUtils.setState;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -203,10 +206,12 @@ public class NodeJoinControllerTests extends ESTestCase {
 
             @Override
             protected void doRun() throws Exception {
-                nodeJoinController.waitToBeElectedAsMaster(requiredJoins, TimeValue.timeValueHours(30), new NodeJoinController.ElectionCallback() {
+                nodeJoinController.waitToBeElectedAsMaster(requiredJoins, TimeValue.timeValueHours(30),
+                    new NodeJoinController.ElectionCallback() {
                     @Override
                     public void onElectedAsMaster(ClusterState state) {
-                        assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(), equalTo(true));
+                        assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(),
+                            equalTo(true));
                         electionFuture.markAsDone();
                     }
 
@@ -251,10 +256,12 @@ public class NodeJoinControllerTests extends ESTestCase {
 
             @Override
             protected void doRun() throws Exception {
-                nodeJoinController.waitToBeElectedAsMaster(requiredJoins, TimeValue.timeValueHours(30), new NodeJoinController.ElectionCallback() {
+                nodeJoinController.waitToBeElectedAsMaster(requiredJoins, TimeValue.timeValueHours(30),
+                    new NodeJoinController.ElectionCallback() {
                     @Override
                     public void onElectedAsMaster(ClusterState state) {
-                        assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(), equalTo(true));
+                        assertThat("callback called with elected as master, but state disagrees", state.nodes().isLocalNodeElectedMaster(),
+                            equalTo(true));
                         electionFuture.markAsDone();
                     }
 
@@ -403,7 +410,7 @@ public class NodeJoinControllerTests extends ESTestCase {
     public void testNewClusterStateOnExistingNodeJoin() throws InterruptedException, ExecutionException {
         ClusterState state = clusterService.state();
         final DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(state.nodes());
-        final DiscoveryNode other_node = new DiscoveryNode("other_node", DummyTransportAddress.INSTANCE,
+        final DiscoveryNode other_node = new DiscoveryNode("other_node", LocalTransportAddress.buildUnique(),
             emptyMap(), emptySet(), Version.CURRENT);
         nodesBuilder.put(other_node);
         setState(clusterService, ClusterState.builder(state).nodes(nodesBuilder));
@@ -516,6 +523,38 @@ public class NodeJoinControllerTests extends ESTestCase {
         assertNodesInCurrentState(nodes);
     }
 
+    public void testRejectingJoinWithSameAddressButDifferentId() throws InterruptedException, ExecutionException {
+        ClusterState state = clusterService.state();
+        final DiscoveryNode other_node = new DiscoveryNode("other_node", state.nodes().getLocalNode().getAddress(),
+            emptyMap(), emptySet(), Version.CURRENT);
+
+        ExecutionException e = expectThrows(ExecutionException.class, () -> joinNode(other_node));
+        assertThat(e.getMessage(), containsString("found existing node"));
+    }
+
+    public void testRejectingJoinWithSameIdButDifferentAddress() throws InterruptedException, ExecutionException {
+        ClusterState state = clusterService.state();
+        final DiscoveryNode other_node = new DiscoveryNode(state.nodes().getLocalNode().getId(),
+            new LocalTransportAddress(randomAsciiOfLength(20)), emptyMap(), emptySet(), Version.CURRENT);
+
+        ExecutionException e = expectThrows(ExecutionException.class, () -> joinNode(other_node));
+        assertThat(e.getMessage(), containsString("found existing node"));
+    }
+
+    public void testJoinWithSameIdSameAddressButDifferentMeta() throws InterruptedException, ExecutionException {
+        ClusterState state = clusterService.state();
+        final DiscoveryNode localNode = state.nodes().getLocalNode();
+        final DiscoveryNode other_node = new DiscoveryNode(
+            randomBoolean() ? localNode.getName() : "other_name",
+            localNode.getId(), localNode.getAddress(),
+            randomBoolean() ? localNode.getAttributes() : Collections.singletonMap("attr", "other"),
+            randomBoolean() ? localNode.getRoles() : new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values()))),
+            randomBoolean() ? localNode.getVersion() : VersionUtils.randomVersion(random()));
+
+        joinNode(other_node);
+
+        assertThat(clusterService.localNode(), equalTo(other_node));
+    }
 
     static class NoopAllocationService extends AllocationService {
 
@@ -599,8 +638,8 @@ public class NodeJoinControllerTests extends ESTestCase {
      * creates an object clone of node, so it will be a different object instance
      */
     private DiscoveryNode cloneNode(DiscoveryNode node) {
-        return new DiscoveryNode(node.getName(), node.getId(), node.getHostName(), node.getHostAddress(), node.getAddress(),
-            node.getAttributes(), node.getRoles(), node.getVersion());
+        return new DiscoveryNode(node.getName(), node.getId(), node.getEphemeralId(), node.getHostName(), node.getHostAddress(),
+            node.getAddress(), node.getAttributes(), node.getRoles(), node.getVersion());
     }
 
     private void joinNode(final DiscoveryNode node) throws InterruptedException, ExecutionException {
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java
index a6638eb19cf..9db83f48f0e 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenDiscoveryUnitTests.java
@@ -24,8 +24,7 @@ import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.discovery.zen.ping.ZenPing;
 import org.elasticsearch.test.ESTestCase;
 
@@ -51,9 +50,9 @@ public class ZenDiscoveryUnitTests extends ESTestCase {
         ClusterName clusterName = new ClusterName("abc");
 
         DiscoveryNodes.Builder currentNodes = DiscoveryNodes.builder();
-        currentNodes.masterNodeId("a").put(new DiscoveryNode("a", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT));
+        currentNodes.masterNodeId("a").put(new DiscoveryNode("a", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT));
         DiscoveryNodes.Builder newNodes = DiscoveryNodes.builder();
-        newNodes.masterNodeId("a").put(new DiscoveryNode("a", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT));
+        newNodes.masterNodeId("a").put(new DiscoveryNode("a", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT));
 
         ClusterState.Builder currentState = ClusterState.builder(clusterName);
         currentState.nodes(currentNodes);
@@ -71,7 +70,7 @@ public class ZenDiscoveryUnitTests extends ESTestCase {
         assertFalse("should not ignore, because new state's version is higher to current state's version", shouldIgnoreOrRejectNewClusterState(logger, currentState.build(), newState.build()));
 
         currentNodes = DiscoveryNodes.builder();
-        currentNodes.masterNodeId("b").put(new DiscoveryNode("b", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT));
+        currentNodes.masterNodeId("b").put(new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT));
         ;
         // version isn't taken into account, so randomize it to ensure this.
         if (randomBoolean()) {
@@ -109,7 +108,7 @@ public class ZenDiscoveryUnitTests extends ESTestCase {
         ArrayList<DiscoveryNode> allNodes = new ArrayList<>();
         for (int i = randomIntBetween(10, 20); i >= 0; i--) {
             Set<DiscoveryNode.Role> roles = new HashSet<>(randomSubsetOf(Arrays.asList(DiscoveryNode.Role.values())));
-            DiscoveryNode node = new DiscoveryNode("node_" + i, "id_" + i, DummyTransportAddress.INSTANCE, Collections.emptyMap(),
+            DiscoveryNode node = new DiscoveryNode("node_" + i, "id_" + i, LocalTransportAddress.buildUnique(), Collections.emptyMap(),
                     roles, Version.CURRENT);
             responses.add(new ZenPing.PingResponse(node, randomBoolean() ? null : node, new ClusterName("test"), randomBoolean()));
             allNodes.add(node);
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java
index 8aa5114c387..72674f44e3d 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ZenPingTests.java
@@ -23,7 +23,7 @@ import org.elasticsearch.Version;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.discovery.zen.ping.ZenPing;
 import org.elasticsearch.test.ESTestCase;
 
@@ -42,7 +42,7 @@ public class ZenPingTests extends ESTestCase {
         boolean hasJoinedOncePerNode[] = new boolean[nodes.length];
         ArrayList<ZenPing.PingResponse> pings = new ArrayList<>();
         for (int i = 0; i < nodes.length; i++) {
-            nodes[i] = new DiscoveryNode("" + i, DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+            nodes[i] = new DiscoveryNode("" + i, LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
         }
 
         for (int pingCount = scaledRandomIntBetween(10, nodes.length * 10); pingCount > 0; pingCount--) {
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueueTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueueTests.java
index 073e0fe1478..42aa792c95f 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueueTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PendingClusterStatesQueueTests.java
@@ -25,7 +25,7 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.discovery.zen.publish.PendingClusterStatesQueue.ClusterStateContext;
 import org.elasticsearch.test.ESTestCase;
 
@@ -237,7 +237,7 @@ public class PendingClusterStatesQueueTests extends ESTestCase {
             ClusterState state = lastClusterStatePerMaster[masterIndex];
             if (state == null) {
                 state = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).nodes(DiscoveryNodes.builder()
-                                .put(new DiscoveryNode(masters[masterIndex], DummyTransportAddress.INSTANCE,
+                                .put(new DiscoveryNode(masters[masterIndex], LocalTransportAddress.buildUnique(),
                                         emptyMap(), emptySet(),Version.CURRENT)).masterNodeId(masters[masterIndex]).build()
                 ).build();
             } else {
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java
index 8271f2dff70..0d4274a5a53 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java
@@ -43,6 +43,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.discovery.Discovery;
 import org.elasticsearch.discovery.DiscoverySettings;
 import org.elasticsearch.discovery.zen.DiscoveryNodesProvider;
+import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.junit.annotations.TestLogging;
@@ -152,16 +153,17 @@ public class PublishClusterStateActionTests extends ESTestCase {
         return createMockNode(name, settings, null);
     }
 
-    public MockNode createMockNode(String name, Settings settings, @Nullable ClusterStateListener listener) throws Exception {
-        settings = Settings.builder()
+    public MockNode createMockNode(String name, final Settings basSettings, @Nullable ClusterStateListener listener) throws Exception {
+        final Settings settings = Settings.builder()
                 .put("name", name)
                 .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "", TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
-                .put(settings)
+                .put(basSettings)
                 .build();
 
         MockTransportService service = buildTransportService(settings);
         DiscoveryNodeService discoveryNodeService = new DiscoveryNodeService(settings);
-        DiscoveryNode discoveryNode = discoveryNodeService.buildLocalNode(service.boundAddress().publishAddress());
+        DiscoveryNode discoveryNode = discoveryNodeService.buildLocalNode(service.boundAddress().publishAddress(),
+            () -> NodeEnvironment.generateNodeId(settings));
         MockNode node = new MockNode(discoveryNode, service, listener, logger);
         node.action = buildPublishClusterStateAction(settings, service, () -> node.clusterState, node);
         final CountDownLatch latch = new CountDownLatch(nodes.size() * 2 + 1);
diff --git a/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
index 75bfed85333..50e05d97985 100644
--- a/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
+++ b/core/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java
@@ -51,6 +51,7 @@ import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.not;
 
 @LuceneTestCase.SuppressFileSystems("ExtrasFS") // TODO: fix test to allow extras
 public class NodeEnvironmentTests extends ESTestCase {
@@ -389,7 +390,7 @@ public class NodeEnvironmentTests extends ESTestCase {
 
         env.close();
         NodeEnvironment env2 = newNodeEnvironment(dataPaths, "/tmp",
-                Settings.builder().put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH.getKey(), false).build());
+                Settings.builder().put(NodeEnvironment.ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.getKey(), false).build());
 
         assertThat(env2.availableShardPaths(sid), equalTo(env2.availableShardPaths(sid)));
         assertThat(env2.resolveCustomLocation(s2, sid), equalTo(PathUtils.get("/tmp/foo/" + index.getUUID() + "/0")));
@@ -447,6 +448,27 @@ public class NodeEnvironmentTests extends ESTestCase {
         }
     }
 
+    public void testPersistentNodeId() throws IOException {
+        String[] paths = tmpPaths();
+        NodeEnvironment env = newNodeEnvironment(paths, Settings.builder()
+            .put("node.local_storage", false)
+            .put("node.master", false)
+            .put("node.data", false)
+            .build());
+        String nodeID = env.nodeId();
+        env.close();
+        env = newNodeEnvironment(paths, Settings.EMPTY);
+        assertThat("previous node didn't have local storage enabled, id should change", env.nodeId(), not(equalTo(nodeID)));
+        nodeID = env.nodeId();
+        env.close();
+        env = newNodeEnvironment(paths, Settings.EMPTY);
+        assertThat(env.nodeId(), equalTo(nodeID));
+        env.close();
+        env = newNodeEnvironment(Settings.EMPTY);
+        assertThat(env.nodeId(), not(equalTo(nodeID)));
+        env.close();
+    }
+
     /** Converts an array of Strings to an array of Paths, adding an additional child if specified */
     private Path[] stringsToPaths(String[] strings, String additional) {
         Path[] locations = new Path[strings.length];
diff --git a/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java b/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
index c5a386976bd..092e6eaff8a 100644
--- a/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/AsyncShardFetchTests.java
@@ -24,7 +24,7 @@ import org.elasticsearch.action.support.nodes.BaseNodeResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.TestThreadPool;
@@ -45,11 +45,11 @@ import static org.hamcrest.Matchers.sameInstance;
 /**
  */
 public class AsyncShardFetchTests extends ESTestCase {
-    private final DiscoveryNode node1 = new DiscoveryNode("node1", DummyTransportAddress.INSTANCE, Collections.emptyMap(),
+    private final DiscoveryNode node1 = new DiscoveryNode("node1", LocalTransportAddress.buildUnique(), Collections.emptyMap(),
             Collections.singleton(DiscoveryNode.Role.DATA), Version.CURRENT);
     private final Response response1 = new Response(node1);
     private final Throwable failure1 = new Throwable("simulated failure 1");
-    private final DiscoveryNode node2 = new DiscoveryNode("node2", DummyTransportAddress.INSTANCE, Collections.emptyMap(),
+    private final DiscoveryNode node2 = new DiscoveryNode("node2", LocalTransportAddress.buildUnique(), Collections.emptyMap(),
             Collections.singleton(DiscoveryNode.Role.DATA), Version.CURRENT);
     private final Response response2 = new Response(node2);
     private final Throwable failure2 = new Throwable("simulate failure 2");
diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
index 129495ea15e..f86b56f1052 100644
--- a/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
+++ b/core/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
@@ -346,7 +346,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
             logger.info("--> created temp data path for shadow replicas [{}]", dataPath);
             logger.info("--> starting a cluster with " + numNodes + " nodes");
             final Settings nodeSettings = Settings.builder()
-                                                  .put("node.add_id_to_custom_path", false)
+                                                  .put("node.add_lock_id_to_custom_path", false)
                                                   .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), dataPath.toString())
                                                   .put("index.store.fs.fs_lock", randomFrom("native", "simple"))
                                                   .build();
@@ -426,7 +426,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
                  // this one is not validated ahead of time and breaks allocation
                 .put("index.analysis.filter.myCollator.type", "icu_collation")
             ).build();
-            IndexMetaData.FORMAT.write(brokenMeta, brokenMeta.getVersion(), services.indexPaths(brokenMeta.getIndex()));
+            IndexMetaData.FORMAT.write(brokenMeta, services.indexPaths(brokenMeta.getIndex()));
         }
         internalCluster().fullRestart();
         // ensureGreen(closedIndex) waits for the index to show up in the metadata
@@ -483,7 +483,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
         for (NodeEnvironment services : internalCluster().getInstances(NodeEnvironment.class)) {
             IndexMetaData brokenMeta = IndexMetaData.builder(metaData).settings(metaData.getSettings()
                 .filter((s) -> "index.analysis.analyzer.test.tokenizer".equals(s) == false)).build();
-            IndexMetaData.FORMAT.write(brokenMeta, brokenMeta.getVersion(), services.indexPaths(brokenMeta.getIndex()));
+            IndexMetaData.FORMAT.write(brokenMeta, services.indexPaths(brokenMeta.getIndex()));
         }
         internalCluster().fullRestart();
         // ensureGreen(closedIndex) waits for the index to show up in the metadata
@@ -521,7 +521,7 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
             MetaData brokenMeta = MetaData.builder(metaData).persistentSettings(Settings.builder()
                 .put(metaData.persistentSettings()).put("this.is.unknown", true)
                 .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), "broken").build()).build();
-            MetaData.FORMAT.write(brokenMeta, metaData.version(), nodeEnv.nodeDataPaths());
+            MetaData.FORMAT.write(brokenMeta, nodeEnv.nodeDataPaths());
         }
         internalCluster().fullRestart();
         ensureYellow("test"); // wait for state recovery
diff --git a/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java b/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java
index 41eba406009..4cf505d839a 100644
--- a/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/MetaDataStateFormatTests.java
@@ -104,7 +104,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
         Format format = new Format(randomFrom(XContentType.values()), "foo-");
         DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean());
         int version = between(0, Integer.MAX_VALUE/2);
-        format.write(state, version, dirs);
+        format.write(state, dirs);
         for (Path file : dirs) {
             Path[] list = content("*", file);
             assertEquals(list.length, 1);
@@ -119,7 +119,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
         }
         final int version2 = between(version, Integer.MAX_VALUE);
         DummyState state2 = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean());
-        format.write(state2, version2, dirs);
+        format.write(state2, dirs);
 
         for (Path file : dirs) {
             Path[] list = content("*", file);
@@ -146,7 +146,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
         Format format = new Format(randomFrom(XContentType.values()), "foo-");
         DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean());
         int version = between(0, Integer.MAX_VALUE/2);
-        format.write(state, version, dirs);
+        format.write(state, dirs);
         for (Path file : dirs) {
             Path[] list = content("*", file);
             assertEquals(list.length, 1);
@@ -170,7 +170,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
         Format format = new Format(randomFrom(XContentType.values()), "foo-");
         DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean());
         int version = between(0, Integer.MAX_VALUE/2);
-        format.write(state, version, dirs);
+        format.write(state, dirs);
         for (Path file : dirs) {
             Path[] list = content("*", file);
             assertEquals(list.length, 1);
@@ -261,7 +261,7 @@ public class MetaDataStateFormatTests extends ESTestCase {
                 }
             }
             for (int j = numLegacy; j < numStates; j++) {
-                format.write(meta.get(j), j, dirs[i]);
+                format.write(meta.get(j), dirs[i]);
                 if (randomBoolean() && (j < numStates - 1 || dirs.length > 0 && i != 0)) {  // corrupt a file that we do not necessarily need here....
                     Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + j + ".st");
                     corruptedFiles.add(file);
diff --git a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
index f67ee09cf0f..bd78607c617 100644
--- a/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
+++ b/core/src/test/java/org/elasticsearch/index/IndexWithShadowReplicasIT.java
@@ -94,7 +94,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
 
     private Settings nodeSettings(String dataPath) {
         return Settings.builder()
-                .put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH.getKey(), false)
+                .put(NodeEnvironment.ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.getKey(), false)
                 .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), dataPath)
                 .put(FsDirectoryService.INDEX_LOCK_FACTOR_SETTING.getKey(), randomFrom("native", "simple"))
                 .build();
@@ -454,7 +454,7 @@ public class IndexWithShadowReplicasIT extends ESIntegTestCase {
     public void testPrimaryRelocationWhereRecoveryFails() throws Exception {
         Path dataPath = createTempDir();
         Settings nodeSettings = Settings.builder()
-                .put("node.add_id_to_custom_path", false)
+                .put("node.add_lock_id_to_custom_path", false)
                 .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), dataPath)
                 .build();
 
diff --git a/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
index 1c6a973192a..2ccae5a287b 100644
--- a/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
+++ b/core/src/test/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java
@@ -46,7 +46,7 @@ import org.elasticsearch.cluster.routing.TestShardRouting;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.env.NodeEnvironment;
@@ -198,7 +198,7 @@ public abstract class ESIndexLevelReplicationTestCase extends ESTestCase {
     }
 
     private DiscoveryNode getDiscoveryNode(String id) {
-        return new DiscoveryNode(id, id, DummyTransportAddress.INSTANCE, Collections.emptyMap(),
+        return new DiscoveryNode(id, id, LocalTransportAddress.buildUnique(), Collections.emptyMap(),
             Collections.singleton(DiscoveryNode.Role.DATA), Version.CURRENT);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 9bc49ad626e..a0813fb572f 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -69,7 +69,7 @@ import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -463,7 +463,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
 
     public static void write(ShardStateMetaData shardStateMetaData,
                              Path... shardPaths) throws IOException {
-        ShardStateMetaData.FORMAT.write(shardStateMetaData, shardStateMetaData.legacyVersion, shardPaths);
+        ShardStateMetaData.FORMAT.write(shardStateMetaData, shardPaths);
     }
 
     public void testDurableFlagHasEffect() {
@@ -1036,7 +1036,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         routing = ShardRoutingHelper.reinit(routing);
         IndexShard newShard = test.createShard(routing);
         newShard.updateRoutingEntry(routing);
-        DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
         newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode));
         assertTrue(newShard.recoverFromStore());
         assertEquals(translogOps, newShard.recoveryState().getTranslog().recoveredOperations());
@@ -1063,7 +1063,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         routing = ShardRoutingHelper.reinit(routing, UnassignedInfo.Reason.INDEX_CREATED);
         IndexShard newShard = test.createShard(routing);
         newShard.updateRoutingEntry(routing);
-        DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
         newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode,
             localNode));
         assertTrue(newShard.recoverFromStore());
@@ -1080,7 +1080,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         createIndex("test");
         ensureGreen();
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
         IndexService test = indicesService.indexService(resolveIndex("test"));
         final IndexShard shard = test.getShardOrNull(0);
 
@@ -1176,7 +1176,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         Store targetStore = test_target_shard.store();
 
         test_target_shard.updateRoutingEntry(routing);
-        DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
         test_target_shard.markAsRecovering("store", new RecoveryState(routing.shardId(), routing.primary(), RecoveryState.Type.SNAPSHOT, routing.restoreSource(), localNode));
         assertTrue(test_target_shard.restoreFromRepository(new IndexShardRepository() {
             @Override
@@ -1460,7 +1460,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
     }
 
     public  static final IndexShard recoverShard(IndexShard newShard) throws IOException {
-        DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
         newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), newShard.routingEntry().primary(), RecoveryState.Type.STORE, localNode, localNode));
         assertTrue(newShard.recoverFromStore());
         newShard.updateRoutingEntry(newShard.routingEntry().moveToStarted());
@@ -1500,7 +1500,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         ShardRouting routing = getInitializingShardRouting(shard.routingEntry());
         test.removeShard(0, "b/c britta says so");
         IndexShard newShard = test.createShard(routing);
-        DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
         newShard.markAsRecovering("for testing", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.REPLICA, localNode, localNode));
         List<Translog.Operation> operations = new ArrayList<>();
         operations.add(new Translog.Index("testtype", "1", BytesReference.toBytes(jsonBuilder().startObject().field("foo", "bar").endObject().bytes())));
@@ -1528,7 +1528,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         test.removeShard(0, "b/c britta says so");
         IndexShard newShard = test.createShard(routing);
         newShard.shardRouting = routing;
-        DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
         newShard.markAsRecovering("for testing", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.REPLICA, localNode, localNode));
         // Shard is still inactive since we haven't started recovering yet
         assertFalse(newShard.isActive());
@@ -1556,7 +1556,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         ShardRouting routing = getInitializingShardRouting(shard.routingEntry());
         test.removeShard(0, "b/c britta says so");
         IndexShard newShard = test.createShard(routing);
-        DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
         newShard.markAsRecovering("for testing", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.REPLICA, localNode, localNode));
         // Shard is still inactive since we haven't started recovering yet
         assertFalse(newShard.isActive());
@@ -1591,7 +1591,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         IndexShard shard = test.getShardOrNull(0);
         ShardRouting routing = ShardRoutingHelper.initWithSameId(shard.routingEntry());
         test.removeShard(0, "b/c simon says so");
-        DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
         {
             final IndexShard newShard = test.createShard(routing);
             newShard.updateRoutingEntry(routing);
diff --git a/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java b/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java
index b2bd7e1f9ff..749b1621e4d 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java
@@ -45,7 +45,7 @@ public class ShardPathTests extends ESTestCase {
             ShardId shardId = new ShardId("foo", "0xDEADBEEF", 0);
             Path[] paths = env.availableShardPaths(shardId);
             Path path = randomFrom(paths);
-            ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, "0xDEADBEEF", AllocationId.newInitializing()), 2, path);
+            ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, "0xDEADBEEF", AllocationId.newInitializing()), path);
             ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), settings));
             assertEquals(path, shardPath.getDataPath());
             assertEquals("0xDEADBEEF", shardPath.getShardId().getIndex().getUUID());
@@ -65,7 +65,7 @@ public class ShardPathTests extends ESTestCase {
             Path[] paths = env.availableShardPaths(shardId);
             assumeTrue("This test tests multi data.path but we only got one", paths.length > 1);
             int id = randomIntBetween(1, 10);
-            ShardStateMetaData.FORMAT.write(new ShardStateMetaData(id, true, indexUUID, AllocationId.newInitializing()), id, paths);
+            ShardStateMetaData.FORMAT.write(new ShardStateMetaData(id, true, indexUUID, AllocationId.newInitializing()), paths);
             ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), settings));
             fail("Expected IllegalStateException");
         } catch (IllegalStateException e) {
@@ -82,7 +82,7 @@ public class ShardPathTests extends ESTestCase {
             Path[] paths = env.availableShardPaths(shardId);
             Path path = randomFrom(paths);
             int id = randomIntBetween(1, 10);
-            ShardStateMetaData.FORMAT.write(new ShardStateMetaData(id, true, "0xDEADBEEF", AllocationId.newInitializing()), id, path);
+            ShardStateMetaData.FORMAT.write(new ShardStateMetaData(id, true, "0xDEADBEEF", AllocationId.newInitializing()), path);
             ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), settings));
             fail("Expected IllegalStateException");
         } catch (IllegalStateException e) {
@@ -124,7 +124,7 @@ public class ShardPathTests extends ESTestCase {
             final boolean includeNodeId = randomBoolean();
             indexSettings = indexSettingsBuilder.put(IndexMetaData.SETTING_DATA_PATH, "custom").build();
             nodeSettings = Settings.builder().put(Environment.PATH_SHARED_DATA_SETTING.getKey(), path.toAbsolutePath().toAbsolutePath())
-                    .put(NodeEnvironment.ADD_NODE_ID_TO_CUSTOM_PATH.getKey(), includeNodeId).build();
+                    .put(NodeEnvironment.ADD_NODE_LOCK_ID_TO_CUSTOM_PATH.getKey(), includeNodeId).build();
             if (includeNodeId) {
                 customPath = path.resolve("custom").resolve("0");
             } else {
@@ -139,7 +139,7 @@ public class ShardPathTests extends ESTestCase {
             ShardId shardId = new ShardId("foo", indexUUID, 0);
             Path[] paths = env.availableShardPaths(shardId);
             Path path = randomFrom(paths);
-            ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, indexUUID, AllocationId.newInitializing()), 2, path);
+            ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, indexUUID, AllocationId.newInitializing()), path);
             ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.getIndex(), indexSettings));
             boolean found = false;
             for (Path p : env.nodeDataPaths()) {
diff --git a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
index 995beb1742c..7558fbd66fe 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java
@@ -22,9 +22,8 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.ShardRouting;
-import org.elasticsearch.cluster.routing.ShardRoutingHelper;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.index.IndexService;
@@ -448,7 +447,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase {
         try {
             assertEquals(0, imc.availableShards().size());
             ShardRouting routing = newShard.routingEntry();
-            DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+            DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
             newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode));
 
             assertEquals(1, imc.availableShards().size());
diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java
index 92a411a95de..17a4b93c240 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java
@@ -25,7 +25,7 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingHelper;
 import org.elasticsearch.cluster.routing.UnassignedInfo;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.NodeServicesProvider;
@@ -103,7 +103,7 @@ public class IndicesLifecycleListenerSingleNodeTests extends ESSingleNodeTestCas
             newRouting = ShardRoutingHelper.initialize(newRouting, nodeId);
             IndexShard shard = index.createShard(newRouting);
             shard.updateRoutingEntry(newRouting);
-            final DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE,
+            final DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(),
                     emptyMap(), emptySet(), Version.CURRENT);
             shard.markAsRecovering("store", new RecoveryState(shard.shardId(), newRouting.primary(), RecoveryState.Type.SNAPSHOT, newRouting.restoreSource(), localNode));
             shard.recoverFromStore();
diff --git a/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
index 78ef13dde56..ad20cb577bb 100644
--- a/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
@@ -37,7 +37,7 @@ import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation.FailedShard;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.indices.recovery.RecoveryTargetService;
 import org.elasticsearch.repositories.RepositoriesService;
@@ -123,7 +123,7 @@ public class IndicesClusterStateServiceRandomUpdatesTests extends AbstractIndice
 
         for (Iterator<Entry<DiscoveryNode, IndicesClusterStateService>> it = clusterStateServiceMap.entrySet().iterator(); it.hasNext(); ) {
             DiscoveryNode node = it.next().getKey();
-            if (state.nodes().nodeExists(node.getId()) == false) {
+            if (state.nodes().nodeExists(node) == false) {
                 it.remove();
             }
         }
@@ -254,7 +254,7 @@ public class IndicesClusterStateServiceRandomUpdatesTests extends AbstractIndice
         for (DiscoveryNode.Role mustHaveRole : mustHaveRoles) {
             roles.add(mustHaveRole);
         }
-        return new DiscoveryNode("node_" + randomAsciiOfLength(8), DummyTransportAddress.INSTANCE, Collections.emptyMap(), roles,
+        return new DiscoveryNode("node_" + randomAsciiOfLength(8), LocalTransportAddress.buildUnique(), Collections.emptyMap(), roles,
             Version.CURRENT);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
index 0d6d5122006..2c52cd33015 100644
--- a/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java
@@ -38,7 +38,7 @@ import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.lucene.store.IndexOutputOutputStream;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.store.DirectoryService;
@@ -69,8 +69,8 @@ public class RecoverySourceHandlerTests extends ESTestCase {
                 put("indices.recovery.concurrent_small_file_streams", 1).build();
         final RecoverySettings recoverySettings = new RecoverySettings(settings, service);
         StartRecoveryRequest request = new StartRecoveryRequest(shardId,
-                new DiscoveryNode("b", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT),
-                new DiscoveryNode("b", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT),
+                new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
+                new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
             null, RecoveryState.Type.STORE, randomLong());
         Store store = newStore(createTempDir());
         RecoverySourceHandler handler = new RecoverySourceHandler(null, null, request, recoverySettings.getChunkSize().bytesAsInt(),
@@ -119,8 +119,8 @@ public class RecoverySourceHandlerTests extends ESTestCase {
                 put("indices.recovery.concurrent_small_file_streams", 1).build();
         final RecoverySettings recoverySettings = new RecoverySettings(settings, service);
         StartRecoveryRequest request = new StartRecoveryRequest(shardId,
-                new DiscoveryNode("b", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT),
-                new DiscoveryNode("b", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT),
+                new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
+                new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
             null, RecoveryState.Type.STORE, randomLong());
         Path tempDir = createTempDir();
         Store store = newStore(tempDir, false);
@@ -182,8 +182,8 @@ public class RecoverySourceHandlerTests extends ESTestCase {
                 put("indices.recovery.concurrent_small_file_streams", 1).build();
         final RecoverySettings recoverySettings = new RecoverySettings(settings, service);
         StartRecoveryRequest request = new StartRecoveryRequest(shardId,
-                new DiscoveryNode("b", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT),
-                new DiscoveryNode("b", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT),
+                new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
+                new DiscoveryNode("b", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
             null, RecoveryState.Type.STORE, randomLong());
         Path tempDir = createTempDir();
         Store store = newStore(tempDir, false);
diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java
index c59eea6af6d..d0401196b95 100644
--- a/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java
@@ -24,7 +24,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.recovery.RecoveryState.File;
 import org.elasticsearch.indices.recovery.RecoveryState.Index;
@@ -339,7 +339,8 @@ public class RecoveryTargetTests extends ESTestCase {
     }
 
     public void testStageSequenceEnforcement() {
-        final DiscoveryNode discoveryNode = new DiscoveryNode("1", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        final DiscoveryNode discoveryNode = new DiscoveryNode("1", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(),
+            Version.CURRENT);
         Stage[] stages = Stage.values();
         int i = randomIntBetween(0, stages.length - 1);
         int j;
diff --git a/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java b/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java
index a5b8bf63422..2ad8ebb52f9 100644
--- a/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java
@@ -42,7 +42,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.discovery.DiscoverySettings;
@@ -126,7 +126,7 @@ public class RareClusterStateIT extends ESIntegTestCase {
                 // inject a node
                 ClusterState.Builder builder = ClusterState.builder(currentState);
                 builder.nodes(DiscoveryNodes.builder(currentState.nodes()).put(new DiscoveryNode("_non_existent",
-                        DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT)));
+                        LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT)));
 
                 // open index
                 final IndexMetaData indexMetaData = IndexMetaData.builder(currentState.metaData().index(index)).state(IndexMetaData.State.OPEN).build();
diff --git a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
index acdfdd12266..a5ec8e4ecd7 100644
--- a/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
+++ b/core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java
@@ -35,7 +35,6 @@ import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.indices.IndexTemplateAlreadyExistsException;
 import org.elasticsearch.indices.InvalidAliasNameException;
-import org.elasticsearch.indices.InvalidIndexNameException;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.test.ESIntegTestCase;
 
diff --git a/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java b/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java
index 82886628c85..99c407c18b7 100644
--- a/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java
@@ -21,10 +21,8 @@ package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.ingest.ProcessorsRegistry;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.threadpool.TestThreadPool;
 import org.junit.Before;
 
 import java.util.ArrayList;
diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
index ec41eda47b0..600e6dbb51e 100644
--- a/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
+++ b/core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
@@ -259,7 +259,7 @@ public class PipelineStoreTests extends ESTestCase {
             store.validatePipeline(ingestInfos, putRequest);
             fail("exception expected");
         } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("Processor type [remove] is not installed on node [{_node_id2}{local}{local[_id]}]"));
+            assertThat(e.getMessage(), equalTo("Processor type [remove] is not installed on node [" + node2 + "]"));
         }
 
         ingestInfos.put(node2, new IngestInfo(Arrays.asList(new ProcessorInfo("set"), new ProcessorInfo("remove"))));
diff --git a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
index 739dcd8b2c6..090517adfcd 100644
--- a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
+++ b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java
@@ -28,7 +28,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.BoundTransportAddress;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
@@ -117,7 +117,7 @@ public class NodeInfoStreamingTests extends ESTestCase {
 
     private NodeInfo createNodeInfo() {
         Build build = Build.CURRENT;
-        DiscoveryNode node = new DiscoveryNode("test_node", DummyTransportAddress.INSTANCE,
+        DiscoveryNode node = new DiscoveryNode("test_node", LocalTransportAddress.buildUnique(),
                 emptyMap(), emptySet(), VersionUtils.randomVersion(random()));
         Map<String, String> serviceAttributes = new HashMap<>();
         serviceAttributes.put("test", "attribute");
@@ -129,7 +129,7 @@ public class NodeInfoStreamingTests extends ESTestCase {
         threadPoolInfos.add(new ThreadPool.Info("test_threadpool", ThreadPool.ThreadPoolType.FIXED, 5));
         ThreadPoolInfo threadPoolInfo = new ThreadPoolInfo(threadPoolInfos);
         Map<String, BoundTransportAddress> profileAddresses = new HashMap<>();
-        BoundTransportAddress dummyBoundTransportAddress = new BoundTransportAddress(new TransportAddress[]{DummyTransportAddress.INSTANCE}, DummyTransportAddress.INSTANCE);
+        BoundTransportAddress dummyBoundTransportAddress = new BoundTransportAddress(new TransportAddress[]{LocalTransportAddress.buildUnique()}, LocalTransportAddress.buildUnique());
         profileAddresses.put("test_address", dummyBoundTransportAddress);
         TransportInfo transport = new TransportInfo(dummyBoundTransportAddress, profileAddresses);
         HttpInfo htttpInfo = new HttpInfo(dummyBoundTransportAddress, randomLong());
diff --git a/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java b/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java
index 0cc30f8d569..0916cad60d5 100644
--- a/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java
+++ b/core/src/test/java/org/elasticsearch/nodesinfo/SimpleNodesInfoIT.java
@@ -24,9 +24,9 @@ import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
-import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.List;
 
diff --git a/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java b/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java
index 740a027aecb..d56e1341165 100644
--- a/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java
+++ b/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java
@@ -22,7 +22,7 @@ import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
@@ -30,16 +30,13 @@ import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.recovery.RecoveriesCollection;
 import org.elasticsearch.indices.recovery.RecoveryFailedException;
 import org.elasticsearch.indices.recovery.RecoveryState;
-import org.elasticsearch.indices.recovery.RecoveryTarget;
 import org.elasticsearch.indices.recovery.RecoveryTargetService;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 
-import java.util.ArrayList;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.function.Predicate;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
@@ -135,7 +132,8 @@ public class RecoveriesCollectionTests extends ESSingleNodeTestCase {
     long startRecovery(RecoveriesCollection collection, RecoveryTargetService.RecoveryListener listener, TimeValue timeValue) {
         IndicesService indexServices = getInstanceFromNode(IndicesService.class);
         IndexShard indexShard = indexServices.indexServiceSafe(resolveIndex("test")).getShardOrNull(0);
-        final DiscoveryNode sourceNode = new DiscoveryNode("id", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT);
+        final DiscoveryNode sourceNode = new DiscoveryNode("id", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(),
+            Version.CURRENT);
         return collection.startRecovery(indexShard, sourceNode, listener, timeValue);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java b/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java
index 2bbedd8784b..f361496c537 100644
--- a/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/tribe/TribeServiceTests.java
@@ -27,7 +27,7 @@ public class TribeServiceTests extends ESTestCase {
         Settings globalSettings = Settings.builder()
             .put("node.name", "nodename")
             .put("path.home", "some/path").build();
-        Settings clientSettings = TribeService.buildClientSettings("tribe1", globalSettings, Settings.EMPTY);
+        Settings clientSettings = TribeService.buildClientSettings("tribe1", "parent_id", globalSettings, Settings.EMPTY);
         assertEquals("some/path", clientSettings.get("path.home"));
         assertEquals("nodename/tribe1", clientSettings.get("node.name"));
         assertEquals("tribe1", clientSettings.get("tribe.name"));
@@ -35,7 +35,9 @@ public class TribeServiceTests extends ESTestCase {
         assertEquals("false", clientSettings.get("node.master"));
         assertEquals("false", clientSettings.get("node.data"));
         assertEquals("false", clientSettings.get("node.ingest"));
-        assertEquals(7, clientSettings.getAsMap().size());
+        assertEquals("false", clientSettings.get("node.local_storage"));
+        assertEquals("3707202549613653169", clientSettings.get("node.id.seed")); // should be fixed by the parent id and tribe name
+        assertEquals(9, clientSettings.getAsMap().size());
     }
 
     public void testEnvironmentSettings() {
@@ -45,7 +47,7 @@ public class TribeServiceTests extends ESTestCase {
             .put("path.conf", "conf/path")
             .put("path.scripts", "scripts/path")
             .put("path.logs", "logs/path").build();
-        Settings clientSettings = TribeService.buildClientSettings("tribe1", globalSettings, Settings.EMPTY);
+        Settings clientSettings = TribeService.buildClientSettings("tribe1", "parent_id", globalSettings, Settings.EMPTY);
         assertEquals("some/path", clientSettings.get("path.home"));
         assertEquals("conf/path", clientSettings.get("path.conf"));
         assertEquals("scripts/path", clientSettings.get("path.scripts"));
@@ -54,7 +56,7 @@ public class TribeServiceTests extends ESTestCase {
         Settings tribeSettings = Settings.builder()
             .put("path.home", "alternate/path").build();
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
-            TribeService.buildClientSettings("tribe1", globalSettings, tribeSettings);
+            TribeService.buildClientSettings("tribe1", "parent_id", globalSettings, tribeSettings);
         });
         assertTrue(e.getMessage(), e.getMessage().contains("Setting [path.home] not allowed in tribe client"));
     }
@@ -69,7 +71,7 @@ public class TribeServiceTests extends ESTestCase {
             .put("transport.host", "3.3.3.3")
             .put("transport.bind_host", "4.4.4.4")
             .put("transport.publish_host", "5.5.5.5").build();
-        Settings clientSettings = TribeService.buildClientSettings("tribe1", globalSettings, Settings.EMPTY);
+        Settings clientSettings = TribeService.buildClientSettings("tribe1", "parent_id", globalSettings, Settings.EMPTY);
         assertEquals("0.0.0.0", clientSettings.get("network.host"));
         assertEquals("1.1.1.1", clientSettings.get("network.bind_host"));
         assertEquals("2.2.2.2", clientSettings.get("network.publish_host"));
@@ -85,7 +87,7 @@ public class TribeServiceTests extends ESTestCase {
             .put("transport.host", "6.6.6.6")
             .put("transport.bind_host", "7.7.7.7")
             .put("transport.publish_host", "8.8.8.8").build();
-        clientSettings = TribeService.buildClientSettings("tribe1", globalSettings, tribeSettings);
+        clientSettings = TribeService.buildClientSettings("tribe1", "parent_id", globalSettings, tribeSettings);
         assertEquals("3.3.3.3", clientSettings.get("network.host"));
         assertEquals("4.4.4.4", clientSettings.get("network.bind_host"));
         assertEquals("5.5.5.5", clientSettings.get("network.publish_host"));
diff --git a/docs/reference/indices/shadow-replicas.asciidoc b/docs/reference/indices/shadow-replicas.asciidoc
index 60360c147b5..3a0b23852b0 100644
--- a/docs/reference/indices/shadow-replicas.asciidoc
+++ b/docs/reference/indices/shadow-replicas.asciidoc
@@ -10,12 +10,12 @@ index.
 
 In order to fully utilize the `index.data_path` and `index.shadow_replicas`
 settings, you need to allow Elasticsearch to use the same data directory for
-multiple instances by setting `node.add_id_to_custom_path` to false in
+multiple instances by setting `node.add_lock_id_to_custom_path` to false in
 elasticsearch.yml:
 
 [source,yaml]
 --------------------------------------------------
-node.add_id_to_custom_path: false
+node.add_lock_id_to_custom_path: false
 --------------------------------------------------
 
 You will also need to indicate to the security manager where the custom indices
@@ -114,7 +114,7 @@ settings API:
 
 These are non-dynamic settings that need to be configured in `elasticsearch.yml`
 
-`node.add_id_to_custom_path`::
+`node.add_lock_id_to_custom_path`::
     Boolean setting indicating whether Elasticsearch should append the node's
     ordinal to the custom data path. For example, if this is enabled and a path
     of "/tmp/foo" is used, the first locally-running node will use "/tmp/foo/0",
diff --git a/docs/reference/migration/migrate_5_0/fs.asciidoc b/docs/reference/migration/migrate_5_0/fs.asciidoc
index 859f3092823..42c8b4ddcea 100644
--- a/docs/reference/migration/migrate_5_0/fs.asciidoc
+++ b/docs/reference/migration/migrate_5_0/fs.asciidoc
@@ -23,3 +23,9 @@ behavior will be removed.
 If you are using a multi-cluster setup with both instances of Elasticsearch
 pointing to the same data path, you will need to add the cluster name to the
 data path so that different clusters do not overwrite data.
+
+==== Local files
+
+Prior to 5.0, nodes that were marked with both `node.data: false` and `node.master: false` (or the now removed `node.client: true`)
+didn't write any files or folder to disk. 5.x added persistent node ids, requiring nodes to store that information. As such, all
+node types will write a small state file to their data folders.
\ No newline at end of file
diff --git a/docs/reference/migration/migrate_5_0/settings.asciidoc b/docs/reference/migration/migrate_5_0/settings.asciidoc
index ffe69aa3cfb..7bfa9dc875c 100644
--- a/docs/reference/migration/migrate_5_0/settings.asciidoc
+++ b/docs/reference/migration/migrate_5_0/settings.asciidoc
@@ -26,6 +26,8 @@ should be used instead.
 The `name` setting has been removed and is replaced by `node.name`. Usage of
 `-Dname=some_node_name` is not supported anymore.
 
+The `node.add_id_to_custom_path` was renamed to `add_lock_id_to_custom_path`.
+
 ==== Node attribute settings
 
 Node level attributes used for allocation filtering, forced awareness or other node identification / grouping
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java
index 4199a5d67cd..e7b5d1c4501 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/tribe/TribeUnitTests.java
@@ -23,10 +23,10 @@ import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodeService;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESTestCase;
@@ -65,14 +65,14 @@ public class TribeUnitTests extends ESTestCase {
                 .put(baseSettings)
                 .put("cluster.name", "tribe1")
                 .put("node.name", "tribe1_node")
-                    .put(DiscoveryNodeService.NODE_ID_SEED_SETTING.getKey(), random().nextLong())
+                    .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), random().nextLong())
                 .build()).start();
         tribe2 = new TribeClientNode(
             Settings.builder()
                 .put(baseSettings)
                 .put("cluster.name", "tribe2")
                 .put("node.name", "tribe2_node")
-                    .put(DiscoveryNodeService.NODE_ID_SEED_SETTING.getKey(), random().nextLong())
+                    .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), random().nextLong())
                 .build()).start();
     }
 
diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/tribe/elasticsearch.yml b/qa/evil-tests/src/test/resources/org/elasticsearch/tribe/elasticsearch.yml
index d4fa8d8d130..19b2a7b5dd9 100644
--- a/qa/evil-tests/src/test/resources/org/elasticsearch/tribe/elasticsearch.yml
+++ b/qa/evil-tests/src/test/resources/org/elasticsearch/tribe/elasticsearch.yml
@@ -1,5 +1,5 @@
 cluster.name: tribe_node_cluster
 tribe.t1.cluster.name: tribe1
 tribe.t2.cluster.name: tribe2
-tribe.t1.node_id.seed: 1
-tribe.t2.node_id.seed: 2
+tribe.t1.node.id.seed: 1
+tribe.t2.node.id.seed: 2
diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
index 128b0d0e315..45edbd8bcb2 100644
--- a/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
+++ b/test/framework/src/main/java/org/elasticsearch/cluster/MockInternalClusterInfoService.java
@@ -32,7 +32,7 @@ import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.monitor.fs.FsInfo;
 import org.elasticsearch.plugins.Plugin;
@@ -67,7 +67,7 @@ public class MockInternalClusterInfoService extends InternalClusterInfoService {
             usage.getTotalBytes(), usage.getFreeBytes(), usage.getFreeBytes());
         paths[0] = path;
         FsInfo fsInfo = new FsInfo(System.currentTimeMillis(), null, paths);
-        return new NodeStats(new DiscoveryNode(nodeName, DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT),
+        return new NodeStats(new DiscoveryNode(nodeName, LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT),
             System.currentTimeMillis(),
             null, null, null, null, null,
             fsInfo,
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
index a86b55c81db..a6d35930e6b 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ClusterServiceUtils.java
@@ -29,7 +29,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Arrays;
@@ -45,7 +45,7 @@ public class ClusterServiceUtils {
         ClusterService clusterService = new ClusterService(Settings.builder().put("cluster.name", "ClusterServiceTests").build(),
                 new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
                 threadPool);
-        clusterService.setLocalNode(new DiscoveryNode("node", DummyTransportAddress.INSTANCE, Collections.emptyMap(),
+        clusterService.setLocalNode(new DiscoveryNode("node", LocalTransportAddress.buildUnique(), Collections.emptyMap(),
                 new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())),Version.CURRENT));
         clusterService.setNodeConnectionsService(new NodeConnectionsService(Settings.EMPTY, null, null) {
             @Override
diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java
index 5704a178f48..1aa0428454e 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java
@@ -42,7 +42,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationD
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.DummyTransportAddress;
+import org.elasticsearch.common.transport.LocalTransportAddress;
 import org.elasticsearch.gateway.AsyncShardFetch;
 import org.elasticsearch.gateway.GatewayAllocator;
 import org.elasticsearch.gateway.ReplicaShardAllocator;
@@ -133,19 +133,19 @@ public abstract class ESAllocationTestCase extends ESTestCase {
     }
 
     protected static DiscoveryNode newNode(String nodeName, String nodeId, Map<String, String> attributes) {
-        return new DiscoveryNode(nodeName, nodeId, DummyTransportAddress.INSTANCE, attributes, MASTER_DATA_ROLES, Version.CURRENT);
+        return new DiscoveryNode(nodeName, nodeId, LocalTransportAddress.buildUnique(), attributes, MASTER_DATA_ROLES, Version.CURRENT);
     }
 
     protected static DiscoveryNode newNode(String nodeId, Map<String, String> attributes) {
-        return new DiscoveryNode(nodeId, DummyTransportAddress.INSTANCE, attributes, MASTER_DATA_ROLES, Version.CURRENT);
+        return new DiscoveryNode(nodeId, LocalTransportAddress.buildUnique(), attributes, MASTER_DATA_ROLES, Version.CURRENT);
     }
 
     protected static DiscoveryNode newNode(String nodeId, Set<DiscoveryNode.Role> roles) {
-        return new DiscoveryNode(nodeId, DummyTransportAddress.INSTANCE, emptyMap(), roles, Version.CURRENT);
+        return new DiscoveryNode(nodeId, LocalTransportAddress.buildUnique(), emptyMap(), roles, Version.CURRENT);
     }
 
     protected static DiscoveryNode newNode(String nodeId, Version version) {
-        return new DiscoveryNode(nodeId, DummyTransportAddress.INSTANCE, emptyMap(), MASTER_DATA_ROLES, version);
+        return new DiscoveryNode(nodeId, LocalTransportAddress.buildUnique(), emptyMap(), MASTER_DATA_ROLES, version);
     }
 
     protected  static ClusterState startRandomInitializingShard(ClusterState clusterState, AllocationService strategy) {
diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
index 489cad5b085..620727e255d 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
@@ -40,7 +40,6 @@ import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNode.Role;
-import org.elasticsearch.cluster.node.DiscoveryNodeService;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.routing.OperationRouting;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -606,7 +605,7 @@ public final class InternalTestCluster extends TestCluster {
             .put(Environment.PATH_HOME_SETTING.getKey(), baseDir) // allow overriding path.home
             .put(settings)
             .put("node.name", name)
-            .put(DiscoveryNodeService.NODE_ID_SEED_SETTING.getKey(), seed)
+            .put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), seed)
             .build();
         MockNode node = new MockNode(finalSettings, plugins);
         return new NodeAndClient(name, node, nodeId);
@@ -898,8 +897,8 @@ public final class InternalTestCluster extends TestCluster {
         }
 
         private void createNewNode(final Settings newSettings) {
-            final long newIdSeed = DiscoveryNodeService.NODE_ID_SEED_SETTING.get(node.settings()) + 1; // use a new seed to make sure we have new node id
-            Settings finalSettings = Settings.builder().put(node.settings()).put(newSettings).put(DiscoveryNodeService.NODE_ID_SEED_SETTING.getKey(), newIdSeed).build();
+            final long newIdSeed = NodeEnvironment.NODE_ID_SEED_SETTING.get(node.settings()) + 1; // use a new seed to make sure we have new node id
+            Settings finalSettings = Settings.builder().put(node.settings()).put(newSettings).put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), newIdSeed).build();
             Collection<Class<? extends Plugin>> plugins = node.getPlugins();
             node = new MockNode(finalSettings, plugins);
             markNodeDataDirsAsNotEligableForWipe(node);
@@ -1393,7 +1392,6 @@ public final class InternalTestCluster extends TestCluster {
                 // delete data folders now, before we start other nodes that may claim it
                 nodeAndClient.clearDataIfNeeded(callback);
 
-
                 DiscoveryNode discoveryNode = getInstanceFromNode(ClusterService.class, nodeAndClient.node()).localNode();
                 nodesRoleOrder[nodeAndClient.nodeAndClientId()] = discoveryNode.getRoles();
                 nodesByRoles.computeIfAbsent(discoveryNode.getRoles(), k -> new ArrayList<>()).add(nodeAndClient);
diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java
index 7253c4d26e7..5bf11e4dc98 100644
--- a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java
+++ b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java
@@ -178,7 +178,7 @@ public class InternalTestClusterTests extends ESTestCase {
         int maxNumDataNodes = 2;
         final String clusterName1 = "shared1";
         NodeConfigurationSource nodeConfigurationSource = NodeConfigurationSource.EMPTY;
-        int numClientNodes = 0;
+        int numClientNodes = randomIntBetween(0, 2);
         boolean enableHttpPipelining = randomBoolean();
         String nodePrefix = "test";
         Path baseDir = createTempDir();
@@ -218,8 +218,7 @@ public class InternalTestClusterTests extends ESTestCase {
             assertFileNotExists(testMarker); // a new unknown node used this path, it should be cleaned
             assertFileExists(stableTestMarker); // but leaving the structure of existing, reused nodes
             for (String name: cluster.getNodeNames()) {
-                assertThat("data paths for " + name + " changed", getNodePaths(cluster, name),
-                    equalTo(shardNodePaths.get(name)));
+                assertThat("data paths for " + name + " changed", getNodePaths(cluster, name), equalTo(shardNodePaths.get(name)));
             }
 
             cluster.beforeTest(random(), 0.0);

From 37c8c0fa034bc9f86a1026db01020becd603924c Mon Sep 17 00:00:00 2001
From: Boaz Leskes <b.leskes@gmail.com>
Date: Mon, 4 Jul 2016 22:54:43 +0200
Subject: [PATCH 1227/1311] Improve logging for batched cluster state updates
 (#19255)

We've been slowly improving batch support in `ClusterService` so service won't need to implement this tricky logic themselves. These good changes are blessed but our logging infra didn't catch up and we now log things like:

```
[2016-07-04 21:51:22,318][DEBUG][cluster.service          ] [node_sm0] processing [put-mapping [type1],put-mapping [type1]]:
```

Depending on the `source` string this can get quite ugly (mostly in the ZenDiscovery area).

This PR adds some infra to improve logging, keeping the non-batched task the same. As result the above line looks like:

```
[2016-07-04 21:44:45,047][DEBUG][cluster.service          ] [node_s0] processing [put-mapping[type0, type0, type0]]: execute
```

ZenDiscovery waiting on join moved from:

```
[2016-07-04 17:09:45,111][DEBUG][cluster.service          ] [node_t0] processing [elected_as_master, [1] nodes joined),elected_as_master, [1] nodes joined)]: execute
```

To

```
[2016-07-04 22:03:30,142][DEBUG][cluster.service          ] [node_t3] processing [elected_as_master ([3] nodes joined)[{node_t2}{R3hu3uoSQee0B6bkuw8pjw}{p9n28HDJQdiDMdh3tjxA5g}{127.0.0.1}{127.0.0.1:30107}, {node_t1}{ynYQfk7uR8qR5wKIysFlQg}{wa_OKuJHSl-Oyl9Gis-GXg}{127.0.0.1}{127.0.0.1:30106}, {node_t0}{pweq-2T4TlKPrEVAVW6bJw}{NPBSLXSTTguT1So0JsZY8g}{127.0.0.1}{127.0.0.1:30105}]]: execute
```

As a bonus, I removed all `zen-disco` prefixes to sources from that area.
---
 .../cluster/ClusterStateTaskExecutor.java     | 19 +++++++
 .../cluster/ClusterStateUpdateTask.java       |  5 ++
 .../action/shard/ShardStateAction.java        | 14 +++++-
 .../metadata/MetaDataMappingService.java      | 14 +++++-
 .../cluster/service/ClusterService.java       | 49 ++++++++++---------
 .../discovery/zen/NodeJoinController.java     | 20 ++++++--
 .../discovery/zen/ZenDiscovery.java           | 11 +++--
 .../org/elasticsearch/tribe/TribeService.java |  8 ++-
 8 files changed, 101 insertions(+), 39 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java b/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java
index c801ff9e95f..9324e075e90 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterStateTaskExecutor.java
@@ -46,6 +46,25 @@ public interface ClusterStateTaskExecutor<T> {
     default void clusterStatePublished(ClusterChangedEvent clusterChangedEvent) {
     }
 
+    /**
+     * Builds a concise description of a list of tasks (to be used in logging etc.).
+     *
+     * Note that the tasks given are not necessarily the same as those that will be passed to {@link #execute(ClusterState, List)}.
+     * but are guaranteed to be a subset of them. This method can be called multiple times with different lists before execution.
+     * This allows groupd task description but the submitting source.
+     */
+    default String describeTasks(List<T> tasks) {
+        return tasks.stream().map(T::toString).reduce((s1,s2) -> {
+            if (s1.isEmpty()) {
+                return s2;
+            } else if (s2.isEmpty()) {
+                return s1;
+            } else {
+                return s1 + ", " + s2;
+            }
+        }).orElse("");
+    }
+
     /**
      * Represents the result of a batched execution of cluster state update tasks
      * @param <T> the type of the cluster state update task
diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java b/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java
index 52e4b8dcff1..a679d098616 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterStateUpdateTask.java
@@ -46,6 +46,11 @@ public abstract  class ClusterStateUpdateTask implements ClusterStateTaskConfig,
         return BatchResult.<ClusterStateUpdateTask>builder().successes(tasks).build(result);
     }
 
+    @Override
+    public String describeTasks(List<ClusterStateUpdateTask> tasks) {
+        return ""; // one of task, source is enough
+    }
+
     /**
      * Update the cluster state based on the current state. Return the *same instance* if no state
      * should be changed.
diff --git a/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
index 1668f3f57ed..edd4e7db865 100644
--- a/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
+++ b/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
@@ -183,7 +183,7 @@ public class ShardStateAction extends AbstractComponent {
         public void messageReceived(ShardRoutingEntry request, TransportChannel channel) throws Exception {
             logger.warn("{} received shard failed for {}", request.failure, request.shardRouting.shardId(), request);
             clusterService.submitStateUpdateTask(
-                "shard-failed (" + request.shardRouting + "), message [" + request.message + "]",
+                "shard-failed",
                 request,
                 ClusterStateTaskConfig.build(Priority.HIGH),
                 shardFailedClusterStateTaskExecutor,
@@ -233,6 +233,11 @@ public class ShardStateAction extends AbstractComponent {
             this.logger = logger;
         }
 
+        @Override
+        public String describeTasks(List<ShardRoutingEntry> tasks) {
+            return tasks.stream().map(entry -> entry.getShardRouting().toString()).reduce((s1, s2) -> s1 + ", " + s2).orElse("");
+        }
+
         @Override
         public BatchResult<ShardRoutingEntry> execute(ClusterState currentState, List<ShardRoutingEntry> tasks) throws Exception {
             BatchResult.Builder<ShardRoutingEntry> batchResultBuilder = BatchResult.builder();
@@ -346,7 +351,7 @@ public class ShardStateAction extends AbstractComponent {
         public void messageReceived(ShardRoutingEntry request, TransportChannel channel) throws Exception {
             logger.debug("{} received shard started for [{}]", request.shardRouting.shardId(), request);
             clusterService.submitStateUpdateTask(
-                "shard-started (" + request.shardRouting + "), reason [" + request.message + "]",
+                "shard-started",
                 request,
                 ClusterStateTaskConfig.build(Priority.URGENT),
                 shardStartedClusterStateTaskExecutor,
@@ -364,6 +369,11 @@ public class ShardStateAction extends AbstractComponent {
             this.logger = logger;
         }
 
+        @Override
+        public String describeTasks(List<ShardRoutingEntry> tasks) {
+            return tasks.stream().map(entry -> entry.getShardRouting().toString()).reduce((s1, s2) -> s1 + ", " + s2).orElse("");
+        }
+
         @Override
         public BatchResult<ShardRoutingEntry> execute(ClusterState currentState, List<ShardRoutingEntry> tasks) throws Exception {
             BatchResult.Builder<ShardRoutingEntry> builder = BatchResult.builder();
diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
index f22ac75ef67..c44fee0fb2b 100644
--- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
@@ -82,6 +82,11 @@ public class MetaDataMappingService extends AbstractComponent {
             this.index = index;
             this.indexUUID = indexUUID;
         }
+
+        @Override
+        public String toString() {
+            return "[" + index + "][" + indexUUID + "]";
+        }
     }
 
     class RefreshTaskExecutor implements ClusterStateTaskExecutor<RefreshTask> {
@@ -198,7 +203,7 @@ public class MetaDataMappingService extends AbstractComponent {
      */
     public void refreshMapping(final String index, final String indexUUID) {
         final RefreshTask refreshTask = new RefreshTask(index, indexUUID);
-        clusterService.submitStateUpdateTask("refresh-mapping [" + index + "]",
+        clusterService.submitStateUpdateTask("refresh-mapping",
             refreshTask,
             ClusterStateTaskConfig.build(Priority.HIGH),
             refreshExecutor,
@@ -347,10 +352,15 @@ public class MetaDataMappingService extends AbstractComponent {
 
             return ClusterState.builder(currentState).metaData(builder).build();
         }
+
+        @Override
+        public String describeTasks(List<PutMappingClusterStateUpdateRequest> tasks) {
+            return tasks.stream().map(PutMappingClusterStateUpdateRequest::type).reduce((s1, s2) -> s1 + ", " + s2).orElse("");
+        }
     }
 
     public void putMapping(final PutMappingClusterStateUpdateRequest request, final ActionListener<ClusterStateUpdateResponse> listener) {
-        clusterService.submitStateUpdateTask("put-mapping [" + request.type() + "]",
+        clusterService.submitStateUpdateTask("put-mapping",
                 request,
                 ClusterStateTaskConfig.build(Priority.HIGH, request.masterNodeTimeout()),
                 putMappingExecutor,
diff --git a/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
index 10cf6216a86..f8eee7071f6 100644
--- a/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
+++ b/core/src/main/java/org/elasticsearch/cluster/service/ClusterService.java
@@ -42,7 +42,6 @@ import org.elasticsearch.cluster.routing.OperationRouting;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Priority;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
@@ -512,17 +511,17 @@ public class ClusterService extends AbstractLifecycleComponent {
 
     <T> void runTasksForExecutor(ClusterStateTaskExecutor<T> executor) {
         final ArrayList<UpdateTask<T>> toExecute = new ArrayList<>();
-        final ArrayList<String> sources = new ArrayList<>();
+        final Map<String, ArrayList<T>> processTasksBySource = new HashMap<>();
         synchronized (updateTasksPerExecutor) {
             List<UpdateTask> pending = updateTasksPerExecutor.remove(executor);
             if (pending != null) {
                 for (UpdateTask<T> task : pending) {
                     if (task.processed.getAndSet(true) == false) {
-                        logger.trace("will process [{}]", task.source);
+                        logger.trace("will process [{}[{}]]", task.source, task.task);
                         toExecute.add(task);
-                        sources.add(task.source);
+                        processTasksBySource.computeIfAbsent(task.source, s -> new ArrayList<>()).add(task.task);
                     } else {
-                        logger.trace("skipping [{}], already processed", task.source);
+                        logger.trace("skipping [{}[{}]], already processed", task.source, task.task);
                     }
                 }
             }
@@ -530,15 +529,19 @@ public class ClusterService extends AbstractLifecycleComponent {
         if (toExecute.isEmpty()) {
             return;
         }
-        final String source = Strings.collectionToCommaDelimitedString(sources);
+        final String tasksSummary = processTasksBySource.entrySet().stream().map(entry -> {
+            String tasks = executor.describeTasks(entry.getValue());
+            return tasks.isEmpty() ? entry.getKey() : entry.getKey() + "[" + tasks + "]";
+        }).reduce((s1, s2) -> s1 + ", " + s2).orElse("");
+
         if (!lifecycle.started()) {
-            logger.debug("processing [{}]: ignoring, cluster_service not started", source);
+            logger.debug("processing [{}]: ignoring, cluster_service not started", tasksSummary);
             return;
         }
-        logger.debug("processing [{}]: execute", source);
+        logger.debug("processing [{}]: execute", tasksSummary);
         ClusterState previousClusterState = clusterState;
         if (!previousClusterState.nodes().isLocalNodeElectedMaster() && executor.runOnlyOnMaster()) {
-            logger.debug("failing [{}]: local node is no longer master", source);
+            logger.debug("failing [{}]: local node is no longer master", tasksSummary);
             toExecute.stream().forEach(task -> task.listener.onNoLongerMaster(task.source));
             return;
         }
@@ -551,10 +554,10 @@ public class ClusterService extends AbstractLifecycleComponent {
             TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
             if (logger.isTraceEnabled()) {
                 logger.trace("failed to execute cluster state update in [{}], state:\nversion [{}], source [{}]\n{}{}{}", e, executionTime,
-                        previousClusterState.version(), source, previousClusterState.nodes().prettyPrint(),
+                        previousClusterState.version(), tasksSummary, previousClusterState.nodes().prettyPrint(),
                         previousClusterState.routingTable().prettyPrint(), previousClusterState.getRoutingNodes().prettyPrint());
             }
-            warnAboutSlowTaskIfNeeded(executionTime, source);
+            warnAboutSlowTaskIfNeeded(executionTime, tasksSummary);
             batchResult = ClusterStateTaskExecutor.BatchResult.<T>builder()
                     .failures(toExecute.stream().map(updateTask -> updateTask.task)::iterator, e)
                     .build(previousClusterState);
@@ -597,8 +600,8 @@ public class ClusterService extends AbstractLifecycleComponent {
                 task.listener.clusterStateProcessed(task.source, previousClusterState, newClusterState);
             }
             TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
-            logger.debug("processing [{}]: took [{}] no change in cluster_state", source, executionTime);
-            warnAboutSlowTaskIfNeeded(executionTime, source);
+            logger.debug("processing [{}]: took [{}] no change in cluster_state", tasksSummary, executionTime);
+            warnAboutSlowTaskIfNeeded(executionTime, tasksSummary);
             return;
         }
 
@@ -640,18 +643,18 @@ public class ClusterService extends AbstractLifecycleComponent {
             newClusterState.status(ClusterState.ClusterStateStatus.BEING_APPLIED);
 
             if (logger.isTraceEnabled()) {
-                logger.trace("cluster state updated, source [{}]\n{}", source, newClusterState.prettyPrint());
+                logger.trace("cluster state updated, source [{}]\n{}", tasksSummary, newClusterState.prettyPrint());
             } else if (logger.isDebugEnabled()) {
-                logger.debug("cluster state updated, version [{}], source [{}]", newClusterState.version(), source);
+                logger.debug("cluster state updated, version [{}], source [{}]", newClusterState.version(), tasksSummary);
             }
 
-            ClusterChangedEvent clusterChangedEvent = new ClusterChangedEvent(source, newClusterState, previousClusterState);
+            ClusterChangedEvent clusterChangedEvent = new ClusterChangedEvent(tasksSummary, newClusterState, previousClusterState);
             // new cluster state, notify all listeners
             final DiscoveryNodes.Delta nodesDelta = clusterChangedEvent.nodesDelta();
             if (nodesDelta.hasChanges() && logger.isInfoEnabled()) {
                 String summary = nodesDelta.shortSummary();
                 if (summary.length() > 0) {
-                    logger.info("{}, reason: {}", summary, source);
+                    logger.info("{}, reason: {}", summary, tasksSummary);
                 }
             }
 
@@ -665,7 +668,7 @@ public class ClusterService extends AbstractLifecycleComponent {
                 try {
                     clusterStatePublisher.accept(clusterChangedEvent, ackListener);
                 } catch (Discovery.FailedToCommitClusterStateException t) {
-                    logger.warn("failing [{}]: failed to commit cluster state version [{}]", t, source, newClusterState.version());
+                    logger.warn("failing [{}]: failed to commit cluster state version [{}]", t, tasksSummary, newClusterState.version());
                     proccessedListeners.forEach(task -> task.listener.onFailure(task.source, t));
                     return;
                 }
@@ -719,17 +722,17 @@ public class ClusterService extends AbstractLifecycleComponent {
             try {
                 executor.clusterStatePublished(clusterChangedEvent);
             } catch (Exception e) {
-                logger.error("exception thrown while notifying executor of new cluster state publication [{}]", e, source);
+                logger.error("exception thrown while notifying executor of new cluster state publication [{}]", e, tasksSummary);
             }
 
             TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
-            logger.debug("processing [{}]: took [{}] done applying updated cluster_state (version: {}, uuid: {})", source, executionTime,
-                    newClusterState.version(), newClusterState.stateUUID());
-            warnAboutSlowTaskIfNeeded(executionTime, source);
+            logger.debug("processing [{}]: took [{}] done applying updated cluster_state (version: {}, uuid: {})", tasksSummary,
+                executionTime, newClusterState.version(), newClusterState.stateUUID());
+            warnAboutSlowTaskIfNeeded(executionTime, tasksSummary);
         } catch (Exception e) {
             TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
             logger.warn("failed to apply updated cluster state in [{}]:\nversion [{}], uuid [{}], source [{}]\n{}", e, executionTime,
-                    newClusterState.version(), newClusterState.stateUUID(), source, newClusterState.prettyPrint());
+                    newClusterState.version(), newClusterState.stateUUID(), tasksSummary, newClusterState.prettyPrint());
             // TODO: do we want to call updateTask.onFailure here?
         }
 
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
index def67d97cc6..c86e80e289b 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/NodeJoinController.java
@@ -180,7 +180,7 @@ public class NodeJoinController extends AbstractComponent {
             electionContext.addIncomingJoin(node, callback);
             checkPendingJoinsAndElectIfNeeded();
         } else {
-            clusterService.submitStateUpdateTask("zen-disco-join(node " + node + "])",
+            clusterService.submitStateUpdateTask("zen-disco-node-join",
                 node, ClusterStateTaskConfig.build(Priority.URGENT),
                 joinTaskExecutor, new JoinTaskListener(callback, logger));
         }
@@ -279,7 +279,7 @@ public class NodeJoinController extends AbstractComponent {
             innerClose();
 
             Map<DiscoveryNode, ClusterStateTaskListener> tasks = getPendingAsTasks();
-            final String source = "zen-disco-join(elected_as_master, [" + tasks.size() + "] nodes joined)";
+            final String source = "zen-disco-elected-as-master ([" + tasks.size() + "] nodes joined)";
 
             tasks.put(BECOME_MASTER_TASK, joinProcessedListener);
             clusterService.submitStateUpdateTasks(source, tasks, ClusterStateTaskConfig.build(Priority.URGENT), joinTaskExecutor);
@@ -288,7 +288,7 @@ public class NodeJoinController extends AbstractComponent {
         public synchronized void closeAndProcessPending(String reason) {
             innerClose();
             Map<DiscoveryNode, ClusterStateTaskListener> tasks = getPendingAsTasks();
-            final String source = "zen-disco-join(election stopped [" + reason + "] nodes joined";
+            final String source = "zen-disco-process-pending-joins [" +  reason + "]";
 
             tasks.put(FINISH_ELECTION_NOT_MASTER_TASK, joinProcessedListener);
             clusterService.submitStateUpdateTasks(source, tasks, ClusterStateTaskConfig.build(Priority.URGENT), joinTaskExecutor);
@@ -381,12 +381,22 @@ public class NodeJoinController extends AbstractComponent {
 
     // a task indicated that the current node should become master, if no current master is known
     private static final DiscoveryNode BECOME_MASTER_TASK = new DiscoveryNode("_BECOME_MASTER_TASK_", LocalTransportAddress.buildUnique(),
-        Collections.emptyMap(), Collections.emptySet(), Version.CURRENT);
+        Collections.emptyMap(), Collections.emptySet(), Version.CURRENT) {
+        @Override
+        public String toString() {
+            return ""; // this is not really task , so don't log anything about it...
+        }
+    };
 
     // a task that is used to process pending joins without explicitly becoming a master and listening to the results
     // this task is used when election is stop without the local node becoming a master per se (though it might
     private static final DiscoveryNode FINISH_ELECTION_NOT_MASTER_TASK = new DiscoveryNode("_NOT_MASTER_TASK_",
-        LocalTransportAddress.buildUnique(), Collections.emptyMap(), Collections.emptySet(), Version.CURRENT);
+        LocalTransportAddress.buildUnique(), Collections.emptyMap(), Collections.emptySet(), Version.CURRENT) {
+            @Override
+            public String toString() {
+                return ""; // this is not really task , so don't log anything about it...
+            }
+    };
 
     class JoinTaskExecutor implements ClusterStateTaskExecutor<DiscoveryNode> {
 
diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
index f8e730fa298..3388347d6c2 100644
--- a/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
+++ b/core/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java
@@ -406,7 +406,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
             );
         } else {
             // process any incoming joins (they will fail because we are not the master)
-            nodeJoinController.stopElectionContext("not master");
+            nodeJoinController.stopElectionContext(masterNode + " elected");
 
             // send join request
             final boolean success = joinElectedMaster(masterNode);
@@ -506,7 +506,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
             return;
         }
         if (localNodeMaster()) {
-            clusterService.submitStateUpdateTask("zen-disco-node_left(" + node + ")", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
+            clusterService.submitStateUpdateTask("zen-disco-node-left(" + node + ")", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
                 @Override
                 public ClusterState execute(ClusterState currentState) {
                     DiscoveryNodes.Builder builder = DiscoveryNodes.builder(currentState.nodes()).remove(node.getId());
@@ -545,7 +545,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
             // nothing to do here...
             return;
         }
-        clusterService.submitStateUpdateTask("zen-disco-node_failed(" + node + "), reason " + reason, new ClusterStateUpdateTask(Priority.IMMEDIATE) {
+        clusterService.submitStateUpdateTask("zen-disco-node-failed(" + node + "), reason " + reason,
+            new ClusterStateUpdateTask(Priority.IMMEDIATE) {
             @Override
             public ClusterState execute(ClusterState currentState) {
                 if (currentState.nodes().nodeExists(node) == false) {
@@ -592,7 +593,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
             // We only set the new value. If the master doesn't see enough nodes it will revoke it's mastership.
             return;
         }
-        clusterService.submitStateUpdateTask("zen-disco-minimum_master_nodes_changed", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
+        clusterService.submitStateUpdateTask("zen-disco-mini-master-nodes-changed", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
             @Override
             public ClusterState execute(ClusterState currentState) {
                 // check if we have enough master nodes, if not, we need to move into joining the cluster again
@@ -632,7 +633,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implements Discover
 
         logger.info("master_left [{}], reason [{}]", cause, masterNode, reason);
 
-        clusterService.submitStateUpdateTask("zen-disco-master_failed (" + masterNode + ")", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
+        clusterService.submitStateUpdateTask("master_failed (" + masterNode + ")", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
 
             @Override
             public boolean runOnlyOnMaster() {
diff --git a/core/src/main/java/org/elasticsearch/tribe/TribeService.java b/core/src/main/java/org/elasticsearch/tribe/TribeService.java
index 69e9027ddf1..572f5e0ce13 100644
--- a/core/src/main/java/org/elasticsearch/tribe/TribeService.java
+++ b/core/src/main/java/org/elasticsearch/tribe/TribeService.java
@@ -313,7 +313,7 @@ public class TribeService extends AbstractLifecycleComponent {
         public void clusterChanged(final ClusterChangedEvent event) {
             logger.debug("[{}] received cluster event, [{}]", tribeName, event.source());
             clusterService.submitStateUpdateTask(
-                    "cluster event from " + tribeName + ", " + event.source(),
+                    "cluster event from " + tribeName,
                     event,
                     ClusterStateTaskConfig.build(Priority.NORMAL),
                     executor,
@@ -328,12 +328,16 @@ public class TribeService extends AbstractLifecycleComponent {
             this.tribeName = tribeName;
         }
 
-
         @Override
         public boolean runOnlyOnMaster() {
             return false;
         }
 
+        @Override
+        public String describeTasks(List<ClusterChangedEvent> tasks) {
+            return tasks.stream().map(ClusterChangedEvent::source).reduce((s1, s2) -> s1 + ", " + s2).orElse("");
+        }
+
         @Override
         public BatchResult<ClusterChangedEvent> execute(ClusterState currentState, List<ClusterChangedEvent> tasks) throws Exception {
             ClusterState accumulator = ClusterState.builder(currentState).build();

From f5b07e438fd6368af4a2014fb1971cb1cfcd7ca2 Mon Sep 17 00:00:00 2001
From: Ali Beyad <ali@elastic.co>
Date: Mon, 4 Jul 2016 15:49:10 -0400
Subject: [PATCH 1228/1311] Fixes getting snapshot status checks for preventing
 duplicate entries from both current snapshots (cluster state) and repository
 snapshots (on storage).

---
 .../status/TransportSnapshotsStatusAction.java      | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
index dbcc093ec1b..76fe9510ef5 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
@@ -207,15 +207,14 @@ public class TransportSnapshotsStatusAction extends TransportMasterNodeAction<Sn
                 .filter(s -> requestedSnapshotNames.contains(s.getName()))
                 .collect(Collectors.toMap(SnapshotId::getName, Function.identity()));
             for (final String snapshotName : request.snapshots()) {
+                if (currentSnapshotNames.contains(snapshotName)) {
+                    // we've already found this snapshot in the current snapshot entries, so skip over
+                    continue;
+                }
                 SnapshotId snapshotId = matchedSnapshotIds.get(snapshotName);
                 if (snapshotId == null) {
-                    if (currentSnapshotNames.contains(snapshotName)) {
-                        // we've already found this snapshot in the current snapshot entries, so skip over
-                        continue;
-                    } else {
-                        // neither in the current snapshot entries nor found in the repository
-                        throw new SnapshotMissingException(repositoryName, snapshotName);
-                    }
+                    // neither in the current snapshot entries nor found in the repository
+                    throw new SnapshotMissingException(repositoryName, snapshotName);
                 }
                 SnapshotInfo snapshotInfo = snapshotsService.snapshot(repositoryName, snapshotId);
                 List<SnapshotIndexShardStatus> shardStatusBuilder = new ArrayList<>();

From 36b887ee7c13044e0dbfefc536d9108e25e57ba2 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 4 Jul 2016 19:22:12 -0400
Subject: [PATCH 1229/1311] Throw translog corrupted exception on malformed op

Today when reading a malformed operation from the translog, we throw an
assertion error that is immediately caught and wrapped into a translog
corrupted exception. This commit replaces this by electing to directly
throw a translog corrupted exception instead.

Additionally, this cleanup also addressed a double-wrapped translog
corrupted exception. Namely, verifying the checksum can throw a translog
corrupted exception which the existing code would catch and wrap again
in a translog corrupted exception.

Relates #19256
---
 .../java/org/elasticsearch/index/translog/Translog.java     | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/translog/Translog.java b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
index d26a06a5ce6..eb22c84cf0e 100644
--- a/core/src/main/java/org/elasticsearch/index/translog/Translog.java
+++ b/core/src/main/java/org/elasticsearch/index/translog/Translog.java
@@ -1128,7 +1128,7 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
         try {
             final int opSize = in.readInt();
             if (opSize < 4) { // 4byte for the checksum
-                throw new AssertionError("operation size must be at least 4 but was: " + opSize);
+                throw new TranslogCorruptedException("operation size must be at least 4 but was: " + opSize);
             }
             in.resetDigest(); // size is not part of the checksum!
             if (in.markSupported()) { // if we can we validate the checksum first
@@ -1143,10 +1143,10 @@ public class Translog extends AbstractIndexShardComponent implements IndexShardC
             }
             operation = Translog.Operation.readType(in);
             verifyChecksum(in);
+        } catch (TranslogCorruptedException e) {
+            throw e;
         } catch (EOFException e) {
             throw new TruncatedTranslogException("reached premature end of file, translog is truncated", e);
-        } catch (AssertionError | Exception e) {
-            throw new TranslogCorruptedException("translog corruption while reading from stream", e);
         }
         return operation;
     }

From f9d55be1ed013e93cd732d01161ff2a34d49db13 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 4 Jul 2016 19:22:29 -0400
Subject: [PATCH 1230/1311] Rename UserError

The top-level class Throwable represents all errors and exceptions in
Java. This hierarchy is divided into Error and Exception, the former
being serious problems that applications should not try to catch and the
latter representing exceptional conditions that an application might
want to catch and handle. This commit renames
org.elasticsearch.cli.UserError to org.elasticsearch.UserException to
make its name consistent with where it falls in this hierarchy.

Relates #19254
---
 .../bootstrap/Elasticsearch.java              |  6 ++---
 .../java/org/elasticsearch/cli/Command.java   |  6 ++---
 .../org/elasticsearch/cli/MultiCommand.java   |  4 +--
 .../org/elasticsearch/cli/SettingCommand.java |  2 +-
 .../{UserError.java => UserException.java}    |  6 ++---
 .../plugins/InstallPluginCommand.java         | 26 +++++++++----------
 .../plugins/RemovePluginCommand.java          |  9 +++----
 .../org/elasticsearch/cli/CommandTests.java   |  6 ++---
 .../elasticsearch/cli/MultiCommandTests.java  |  4 +--
 .../plugins/InstallPluginCommandTests.java    | 26 +++++++++----------
 .../plugins/RemovePluginCommandTests.java     |  6 ++---
 11 files changed, 50 insertions(+), 51 deletions(-)
 rename core/src/main/java/org/elasticsearch/cli/{UserError.java => UserException.java} (85%)

diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
index b3259129473..94ff59ee9ab 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
@@ -25,7 +25,7 @@ import org.elasticsearch.Build;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.SettingCommand;
 import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.cli.UserError;
+import org.elasticsearch.cli.UserException;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 
 import java.io.IOException;
@@ -73,11 +73,11 @@ class Elasticsearch extends SettingCommand {
     @Override
     protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
         if (options.nonOptionArguments().isEmpty() == false) {
-            throw new UserError(ExitCodes.USAGE, "Positional arguments not allowed, found " + options.nonOptionArguments());
+            throw new UserException(ExitCodes.USAGE, "Positional arguments not allowed, found " + options.nonOptionArguments());
         }
         if (options.has(versionOption)) {
             if (options.has(daemonizeOption) || options.has(pidfileOption)) {
-                throw new UserError(ExitCodes.USAGE, "Elasticsearch version option is mutually exclusive with any other option");
+                throw new UserException(ExitCodes.USAGE, "Elasticsearch version option is mutually exclusive with any other option");
             }
             terminal.println("Version: " + org.elasticsearch.Version.CURRENT
                     + ", Build: " + Build.CURRENT.shortHash() + "/" + Build.CURRENT.date()
diff --git a/core/src/main/java/org/elasticsearch/cli/Command.java b/core/src/main/java/org/elasticsearch/cli/Command.java
index 3e2faf13657..0b615c695e9 100644
--- a/core/src/main/java/org/elasticsearch/cli/Command.java
+++ b/core/src/main/java/org/elasticsearch/cli/Command.java
@@ -55,7 +55,7 @@ public abstract class Command {
             printHelp(terminal);
             terminal.println(Terminal.Verbosity.SILENT, "ERROR: " + e.getMessage());
             return ExitCodes.USAGE;
-        } catch (UserError e) {
+        } catch (UserException e) {
             if (e.exitCode == ExitCodes.USAGE) {
                 printHelp(terminal);
             }
@@ -79,7 +79,7 @@ public abstract class Command {
         if (options.has(silentOption)) {
             if (options.has(verboseOption)) {
                 // mutually exclusive, we can remove this with jopt-simple 5.0, which natively supports it
-                throw new UserError(ExitCodes.USAGE, "Cannot specify -s and -v together");
+                throw new UserException(ExitCodes.USAGE, "Cannot specify -s and -v together");
             }
             terminal.setVerbosity(Terminal.Verbosity.SILENT);
         } else if (options.has(verboseOption)) {
@@ -110,7 +110,7 @@ public abstract class Command {
     /**
      * Executes this command.
      *
-     * Any runtime user errors (like an input file that does not exist), should throw a {@link UserError}. */
+     * Any runtime user errors (like an input file that does not exist), should throw a {@link UserException}. */
     protected abstract void execute(Terminal terminal, OptionSet options) throws Exception;
 
 }
diff --git a/core/src/main/java/org/elasticsearch/cli/MultiCommand.java b/core/src/main/java/org/elasticsearch/cli/MultiCommand.java
index a9feee0c9bf..16754cd7bf1 100644
--- a/core/src/main/java/org/elasticsearch/cli/MultiCommand.java
+++ b/core/src/main/java/org/elasticsearch/cli/MultiCommand.java
@@ -60,11 +60,11 @@ public class MultiCommand extends Command {
         }
         String[] args = arguments.values(options).toArray(new String[0]);
         if (args.length == 0) {
-            throw new UserError(ExitCodes.USAGE, "Missing command");
+            throw new UserException(ExitCodes.USAGE, "Missing command");
         }
         Command subcommand = subcommands.get(args[0]);
         if (subcommand == null) {
-            throw new UserError(ExitCodes.USAGE, "Unknown command [" + args[0] + "]");
+            throw new UserException(ExitCodes.USAGE, "Unknown command [" + args[0] + "]");
         }
         subcommand.mainWithoutErrorHandling(Arrays.copyOfRange(args, 1, args.length), terminal);
     }
diff --git a/core/src/main/java/org/elasticsearch/cli/SettingCommand.java b/core/src/main/java/org/elasticsearch/cli/SettingCommand.java
index 868975ac6ff..17f7c9e5204 100644
--- a/core/src/main/java/org/elasticsearch/cli/SettingCommand.java
+++ b/core/src/main/java/org/elasticsearch/cli/SettingCommand.java
@@ -41,7 +41,7 @@ public abstract class SettingCommand extends Command {
         final Map<String, String> settings = new HashMap<>();
         for (final KeyValuePair kvp : settingOption.values(options)) {
             if (kvp.value.isEmpty()) {
-                throw new UserError(ExitCodes.USAGE, "Setting [" + kvp.key + "] must not be empty");
+                throw new UserException(ExitCodes.USAGE, "Setting [" + kvp.key + "] must not be empty");
             }
             settings.put(kvp.key, kvp.value);
         }
diff --git a/core/src/main/java/org/elasticsearch/cli/UserError.java b/core/src/main/java/org/elasticsearch/cli/UserException.java
similarity index 85%
rename from core/src/main/java/org/elasticsearch/cli/UserError.java
rename to core/src/main/java/org/elasticsearch/cli/UserException.java
index 2a4f2bf1233..a7f88ccab4a 100644
--- a/core/src/main/java/org/elasticsearch/cli/UserError.java
+++ b/core/src/main/java/org/elasticsearch/cli/UserException.java
@@ -22,13 +22,13 @@ package org.elasticsearch.cli;
 /**
  * An exception representing a user fixable problem in {@link Command} usage.
  */
-public class UserError extends Exception {
+public class UserException extends Exception {
 
     /** The exist status the cli should use when catching this user error. */
     public final int exitCode;
 
-    /** Constructs a UserError with an exit status and message to show the user. */
-    public UserError(int exitCode, String msg) {
+    /** Constructs a UserException with an exit status and message to show the user. */
+    public UserException(int exitCode, String msg) {
         super(msg);
         this.exitCode = exitCode;
     }
diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
index e9ea2d11e37..8eaf29be3a6 100644
--- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
@@ -29,7 +29,7 @@ import org.elasticsearch.bootstrap.JarHell;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.SettingCommand;
 import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.cli.UserError;
+import org.elasticsearch.cli.UserException;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.hash.MessageDigests;
 import org.elasticsearch.common.io.FileSystemUtils;
@@ -188,7 +188,7 @@ class InstallPluginCommand extends SettingCommand {
         // TODO: in jopt-simple 5.0 we can enforce a min/max number of positional args
         List<String> args = arguments.values(options);
         if (args.size() != 1) {
-            throw new UserError(ExitCodes.USAGE, "Must supply a single plugin id argument");
+            throw new UserException(ExitCodes.USAGE, "Must supply a single plugin id argument");
         }
         String pluginId = args.get(0);
         boolean isBatch = options.has(batchOption) || System.console() == null;
@@ -250,7 +250,7 @@ class InstallPluginCommand extends SettingCommand {
             if (plugins.isEmpty() == false) {
                 msg += ", did you mean " + (plugins.size() == 1 ? "[" + plugins.get(0) + "]": "any of " + plugins.toString()) + "?";
             }
-            throw new UserError(ExitCodes.USAGE, msg);
+            throw new UserException(ExitCodes.USAGE, msg);
         }
         terminal.println("-> Downloading " + URLDecoder.decode(pluginId, "UTF-8"));
         return downloadZip(terminal, pluginId, tmpDir);
@@ -328,20 +328,20 @@ class InstallPluginCommand extends SettingCommand {
             BufferedReader checksumReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
             expectedChecksum = checksumReader.readLine();
             if (checksumReader.readLine() != null) {
-                throw new UserError(ExitCodes.IO_ERROR, "Invalid checksum file at " + checksumUrl);
+                throw new UserException(ExitCodes.IO_ERROR, "Invalid checksum file at " + checksumUrl);
             }
         }
 
         byte[] zipbytes = Files.readAllBytes(zip);
         String gotChecksum = MessageDigests.toHexString(MessageDigests.sha1().digest(zipbytes));
         if (expectedChecksum.equals(gotChecksum) == false) {
-            throw new UserError(ExitCodes.IO_ERROR, "SHA1 mismatch, expected " + expectedChecksum + " but got " + gotChecksum);
+            throw new UserException(ExitCodes.IO_ERROR, "SHA1 mismatch, expected " + expectedChecksum + " but got " + gotChecksum);
         }
 
         return zip;
     }
 
-    private Path unzip(Path zip, Path pluginsDir) throws IOException, UserError {
+    private Path unzip(Path zip, Path pluginsDir) throws IOException, UserException {
         // unzip plugin to a staging temp dir
 
         final Path target = stagingDirectory(pluginsDir);
@@ -386,7 +386,7 @@ class InstallPluginCommand extends SettingCommand {
         Files.delete(zip);
         if (hasEsDir == false) {
             IOUtils.rm(target);
-            throw new UserError(ExitCodes.DATA_ERROR, "`elasticsearch` directory is missing in the plugin zip");
+            throw new UserException(ExitCodes.DATA_ERROR, "`elasticsearch` directory is missing in the plugin zip");
         }
         return target;
     }
@@ -425,7 +425,7 @@ class InstallPluginCommand extends SettingCommand {
         // don't let luser install plugin as a module...
         // they might be unavoidably in maven central and are packaged up the same way)
         if (MODULES.contains(info.getName())) {
-            throw new UserError(ExitCodes.USAGE, "plugin '" + info.getName() + "' cannot be installed like this, it is a system module");
+            throw new UserException(ExitCodes.USAGE, "plugin '" + info.getName() + "' cannot be installed like this, it is a system module");
         }
 
         // check for jar hell before any copying
@@ -475,7 +475,7 @@ class InstallPluginCommand extends SettingCommand {
 
             final Path destination = env.pluginsFile().resolve(info.getName());
             if (Files.exists(destination)) {
-                throw new UserError(
+                throw new UserException(
                     ExitCodes.USAGE,
                     "plugin directory " + destination.toAbsolutePath() +
                         " already exists. To update the plugin, uninstall it first using 'remove " + info.getName() + "' command");
@@ -520,7 +520,7 @@ class InstallPluginCommand extends SettingCommand {
     /** Copies the files from {@code tmpBinDir} into {@code destBinDir}, along with permissions from dest dirs parent. */
     private void installBin(PluginInfo info, Path tmpBinDir, Path destBinDir) throws Exception {
         if (Files.isDirectory(tmpBinDir) == false) {
-            throw new UserError(ExitCodes.IO_ERROR, "bin in plugin " + info.getName() + " is not a directory");
+            throw new UserException(ExitCodes.IO_ERROR, "bin in plugin " + info.getName() + " is not a directory");
         }
         Files.createDirectory(destBinDir);
         setFileAttributes(destBinDir, DIR_AND_EXECUTABLE_PERMS);
@@ -528,7 +528,7 @@ class InstallPluginCommand extends SettingCommand {
         try (DirectoryStream<Path> stream = Files.newDirectoryStream(tmpBinDir)) {
             for (Path srcFile : stream) {
                 if (Files.isDirectory(srcFile)) {
-                    throw new UserError(
+                    throw new UserException(
                         ExitCodes.DATA_ERROR,
                         "Directories not allowed in bin dir for plugin " + info.getName() + ", found " + srcFile.getFileName());
                 }
@@ -547,7 +547,7 @@ class InstallPluginCommand extends SettingCommand {
      */
     private void installConfig(PluginInfo info, Path tmpConfigDir, Path destConfigDir) throws Exception {
         if (Files.isDirectory(tmpConfigDir) == false) {
-            throw new UserError(ExitCodes.IO_ERROR, "config in plugin " + info.getName() + " is not a directory");
+            throw new UserException(ExitCodes.IO_ERROR, "config in plugin " + info.getName() + " is not a directory");
         }
 
         Files.createDirectories(destConfigDir);
@@ -563,7 +563,7 @@ class InstallPluginCommand extends SettingCommand {
         try (DirectoryStream<Path> stream = Files.newDirectoryStream(tmpConfigDir)) {
             for (Path srcFile : stream) {
                 if (Files.isDirectory(srcFile)) {
-                    throw new UserError(ExitCodes.DATA_ERROR, "Directories not allowed in config dir for plugin " + info.getName());
+                    throw new UserException(ExitCodes.DATA_ERROR, "Directories not allowed in config dir for plugin " + info.getName());
                 }
 
                 Path destFile = destConfigDir.resolve(tmpConfigDir.relativize(srcFile));
diff --git a/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java
index 8279cc5c51f..d6d39e39ed4 100644
--- a/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java
@@ -29,10 +29,9 @@ import java.util.Map;
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
 import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.cli.Command;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.SettingCommand;
-import org.elasticsearch.cli.UserError;
+import org.elasticsearch.cli.UserException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.common.settings.Settings;
@@ -58,7 +57,7 @@ class RemovePluginCommand extends SettingCommand {
         // TODO: in jopt-simple 5.0 we can enforce a min/max number of positional args
         List<String> args = arguments.values(options);
         if (args.size() != 1) {
-            throw new UserError(ExitCodes.USAGE, "Must supply a single plugin id argument");
+            throw new UserException(ExitCodes.USAGE, "Must supply a single plugin id argument");
         }
         execute(terminal, args.get(0), settings);
     }
@@ -71,7 +70,7 @@ class RemovePluginCommand extends SettingCommand {
 
         Path pluginDir = env.pluginsFile().resolve(pluginName);
         if (Files.exists(pluginDir) == false) {
-            throw new UserError(ExitCodes.USAGE, "plugin " + pluginName + " not found; run 'elasticsearch-plugin list' to get list of installed plugins");
+            throw new UserException(ExitCodes.USAGE, "plugin " + pluginName + " not found; run 'elasticsearch-plugin list' to get list of installed plugins");
         }
 
         List<Path> pluginPaths = new ArrayList<>();
@@ -79,7 +78,7 @@ class RemovePluginCommand extends SettingCommand {
         Path pluginBinDir = env.binFile().resolve(pluginName);
         if (Files.exists(pluginBinDir)) {
             if (Files.isDirectory(pluginBinDir) == false) {
-                throw new UserError(ExitCodes.IO_ERROR, "Bin dir for " + pluginName + " is not a directory");
+                throw new UserException(ExitCodes.IO_ERROR, "Bin dir for " + pluginName + " is not a directory");
             }
             pluginPaths.add(pluginBinDir);
             terminal.println(VERBOSE, "Removing: " + pluginBinDir);
diff --git a/core/src/test/java/org/elasticsearch/cli/CommandTests.java b/core/src/test/java/org/elasticsearch/cli/CommandTests.java
index 1f50ad4c13b..376c883c074 100644
--- a/core/src/test/java/org/elasticsearch/cli/CommandTests.java
+++ b/core/src/test/java/org/elasticsearch/cli/CommandTests.java
@@ -30,7 +30,7 @@ public class CommandTests extends ESTestCase {
         }
         @Override
         protected void execute(Terminal terminal, OptionSet options) throws Exception {
-            throw new UserError(ExitCodes.DATA_ERROR, "Bad input");
+            throw new UserException(ExitCodes.DATA_ERROR, "Bad input");
         }
     }
 
@@ -40,7 +40,7 @@ public class CommandTests extends ESTestCase {
         }
         @Override
         protected void execute(Terminal terminal, OptionSet options) throws Exception {
-            throw new UserError(ExitCodes.USAGE, "something was no good");
+            throw new UserException(ExitCodes.USAGE, "something was no good");
         }
     }
 
@@ -87,7 +87,7 @@ public class CommandTests extends ESTestCase {
         MockTerminal terminal = new MockTerminal();
         NoopCommand command = new NoopCommand();
         String[] args = {"-v", "-s"};
-        UserError e = expectThrows(UserError.class, () -> {
+        UserException e = expectThrows(UserException.class, () -> {
             command.mainWithoutErrorHandling(args, terminal);
         });
         assertTrue(e.getMessage(), e.getMessage().contains("Cannot specify -s and -v together"));
diff --git a/core/src/test/java/org/elasticsearch/cli/MultiCommandTests.java b/core/src/test/java/org/elasticsearch/cli/MultiCommandTests.java
index 4f91d378440..f4680492028 100644
--- a/core/src/test/java/org/elasticsearch/cli/MultiCommandTests.java
+++ b/core/src/test/java/org/elasticsearch/cli/MultiCommandTests.java
@@ -61,7 +61,7 @@ public class MultiCommandTests extends CommandTestCase {
 
     public void testUnknownCommand() throws Exception {
         multiCommand.subcommands.put("something", new DummySubCommand());
-        UserError e = expectThrows(UserError.class, () -> {
+        UserException e = expectThrows(UserException.class, () -> {
             execute("somethingelse");
         });
         assertEquals(ExitCodes.USAGE, e.exitCode);
@@ -70,7 +70,7 @@ public class MultiCommandTests extends CommandTestCase {
 
     public void testMissingCommand() throws Exception {
         multiCommand.subcommands.put("command1", new DummySubCommand());
-        UserError e = expectThrows(UserError.class, () -> {
+        UserException e = expectThrows(UserException.class, () -> {
             execute();
         });
         assertEquals(ExitCodes.USAGE, e.exitCode);
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
index e5117fa0aa0..af1f311dd23 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java
@@ -26,7 +26,7 @@ import org.apache.lucene.util.LuceneTestCase;
 import org.elasticsearch.Version;
 import org.elasticsearch.cli.MockTerminal;
 import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.cli.UserError;
+import org.elasticsearch.cli.UserException;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.io.PathUtils;
@@ -320,7 +320,7 @@ public class InstallPluginCommandTests extends ESTestCase {
 
     public void testUnknownPlugin() throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
-        UserError e = expectThrows(UserError.class, () -> installPlugin("foo", env.v1()));
+        UserException e = expectThrows(UserException.class, () -> installPlugin("foo", env.v1()));
         assertTrue(e.getMessage(), e.getMessage().contains("Unknown plugin foo"));
     }
 
@@ -350,7 +350,7 @@ public class InstallPluginCommandTests extends ESTestCase {
         Tuple<Path, Environment> env = createEnv(fs, temp);
         Path pluginDir = createPluginDir(temp);
         String pluginZip = createPlugin("lang-groovy", pluginDir);
-        UserError e = expectThrows(UserError.class, () -> installPlugin(pluginZip, env.v1()));
+        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
         assertTrue(e.getMessage(), e.getMessage().contains("is a system module"));
         assertInstallCleaned(env.v2());
     }
@@ -385,7 +385,7 @@ public class InstallPluginCommandTests extends ESTestCase {
         Path pluginDir = createPluginDir(temp);
         String pluginZip = createPlugin("fake", pluginDir);
         installPlugin(pluginZip, env.v1());
-        UserError e = expectThrows(UserError.class, () -> installPlugin(pluginZip, env.v1()));
+        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
         assertTrue(e.getMessage(), e.getMessage().contains("already exists"));
         assertInstallCleaned(env.v2());
     }
@@ -407,7 +407,7 @@ public class InstallPluginCommandTests extends ESTestCase {
         Path binDir = pluginDir.resolve("bin");
         Files.createFile(binDir);
         String pluginZip = createPlugin("fake", pluginDir);
-        UserError e = expectThrows(UserError.class, () -> installPlugin(pluginZip, env.v1()));
+        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
         assertTrue(e.getMessage(), e.getMessage().contains("not a directory"));
         assertInstallCleaned(env.v2());
     }
@@ -419,7 +419,7 @@ public class InstallPluginCommandTests extends ESTestCase {
         Files.createDirectories(dirInBinDir);
         Files.createFile(dirInBinDir.resolve("somescript"));
         String pluginZip = createPlugin("fake", pluginDir);
-        UserError e = expectThrows(UserError.class, () -> installPlugin(pluginZip, env.v1()));
+        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
         assertTrue(e.getMessage(), e.getMessage().contains("Directories not allowed in bin dir for plugin"));
         assertInstallCleaned(env.v2());
     }
@@ -490,7 +490,7 @@ public class InstallPluginCommandTests extends ESTestCase {
         Path configDir = pluginDir.resolve("config");
         Files.createFile(configDir);
         String pluginZip = createPlugin("fake", pluginDir);
-        UserError e = expectThrows(UserError.class, () -> installPlugin(pluginZip, env.v1()));
+        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
         assertTrue(e.getMessage(), e.getMessage().contains("not a directory"));
         assertInstallCleaned(env.v2());
     }
@@ -502,7 +502,7 @@ public class InstallPluginCommandTests extends ESTestCase {
         Files.createDirectories(dirInConfigDir);
         Files.createFile(dirInConfigDir.resolve("myconfig.yml"));
         String pluginZip = createPlugin("fake", pluginDir);
-        UserError e = expectThrows(UserError.class, () -> installPlugin(pluginZip, env.v1()));
+        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
         assertTrue(e.getMessage(), e.getMessage().contains("Directories not allowed in config dir for plugin"));
         assertInstallCleaned(env.v2());
     }
@@ -534,7 +534,7 @@ public class InstallPluginCommandTests extends ESTestCase {
         Path pluginDir = createPluginDir(temp);
         Files.createFile(pluginDir.resolve(PluginInfo.ES_PLUGIN_PROPERTIES));
         String pluginZip = writeZip(pluginDir, null);
-        UserError e = expectThrows(UserError.class, () -> installPlugin(pluginZip, env.v1()));
+        UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1()));
         assertTrue(e.getMessage(), e.getMessage().contains("`elasticsearch` directory is missing in the plugin zip"));
         assertInstallCleaned(env.v2());
     }
@@ -580,16 +580,16 @@ public class InstallPluginCommandTests extends ESTestCase {
 
     public void testInstallMisspelledOfficialPlugins() throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
-        UserError e = expectThrows(UserError.class, () -> installPlugin("xpack", env.v1()));
+        UserException e = expectThrows(UserException.class, () -> installPlugin("xpack", env.v1()));
         assertThat(e.getMessage(), containsString("Unknown plugin xpack, did you mean [x-pack]?"));
 
-        e = expectThrows(UserError.class, () -> installPlugin("analysis-smartnc", env.v1()));
+        e = expectThrows(UserException.class, () -> installPlugin("analysis-smartnc", env.v1()));
         assertThat(e.getMessage(), containsString("Unknown plugin analysis-smartnc, did you mean [analysis-smartcn]?"));
 
-        e = expectThrows(UserError.class, () -> installPlugin("repository", env.v1()));
+        e = expectThrows(UserException.class, () -> installPlugin("repository", env.v1()));
         assertThat(e.getMessage(), containsString("Unknown plugin repository, did you mean any of [repository-s3, repository-gcs]?"));
 
-        e = expectThrows(UserError.class, () -> installPlugin("unknown_plugin", env.v1()));
+        e = expectThrows(UserException.class, () -> installPlugin("unknown_plugin", env.v1()));
         assertThat(e.getMessage(), containsString("Unknown plugin unknown_plugin"));
     }
 
diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java
index 3a4639fa839..e2910be64f0 100644
--- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java
+++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/RemovePluginCommandTests.java
@@ -27,7 +27,7 @@ import java.util.HashMap;
 import java.util.Map;
 
 import org.apache.lucene.util.LuceneTestCase;
-import org.elasticsearch.cli.UserError;
+import org.elasticsearch.cli.UserException;
 import org.elasticsearch.cli.MockTerminal;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
@@ -72,7 +72,7 @@ public class RemovePluginCommandTests extends ESTestCase {
     }
 
     public void testMissing() throws Exception {
-        UserError e = expectThrows(UserError.class, () -> removePlugin("dne", home));
+        UserException e = expectThrows(UserException.class, () -> removePlugin("dne", home));
         assertTrue(e.getMessage(), e.getMessage().contains("plugin dne not found"));
         assertRemoveCleaned(env);
     }
@@ -102,7 +102,7 @@ public class RemovePluginCommandTests extends ESTestCase {
 
     public void testBinNotDir() throws Exception {
         Files.createDirectories(env.pluginsFile().resolve("elasticsearch"));
-        UserError e = expectThrows(UserError.class, () -> removePlugin("elasticsearch", home));
+        UserException e = expectThrows(UserException.class, () -> removePlugin("elasticsearch", home));
         assertTrue(e.getMessage(), e.getMessage().contains("not a directory"));
         assertTrue(Files.exists(env.pluginsFile().resolve("elasticsearch"))); // did not remove
         assertTrue(Files.exists(env.binFile().resolve("elasticsearch")));

From a00a54ebda186650b757e307e8d273da3bfa29c4 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Mon, 4 Jul 2016 19:45:46 -0400
Subject: [PATCH 1231/1311] Fix style violation in InstallPluginCommand.java

This commit fixes a checkstyle violation in InstallPluginCommand.java
added after renaming UserError to UserException in
f9d55be1ed013e93cd732d01161ff2a34d49db13.
---
 .../java/org/elasticsearch/plugins/InstallPluginCommand.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
index 8eaf29be3a6..d0adda942a3 100644
--- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
@@ -425,7 +425,8 @@ class InstallPluginCommand extends SettingCommand {
         // don't let luser install plugin as a module...
         // they might be unavoidably in maven central and are packaged up the same way)
         if (MODULES.contains(info.getName())) {
-            throw new UserException(ExitCodes.USAGE, "plugin '" + info.getName() + "' cannot be installed like this, it is a system module");
+            throw new UserException(
+                    ExitCodes.USAGE, "plugin '" + info.getName() + "' cannot be installed like this, it is a system module");
         }
 
         // check for jar hell before any copying

From 44ccf67e33e6d01847485e6fe2bf5b4c881130fd Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Tue, 5 Jul 2016 08:33:19 +0200
Subject: [PATCH 1232/1311] Simplify TcpTransport interface by reducing send
 code to a single send method (#19223)

Due to some optimization on the netty layer we had quite some code / cruft
added to the TcpTransport to allow for those optimizations. After cleaning
up BytesReference we can now move this optimization into TcpTransport and
have a simple send method on the implementation layer instead. This commit
adds a CompositeBytesReference that also allows message headers to be written
separately which simplify the header code as well since no skips are needed
anymore.
---
 .../common/bytes/BytesArray.java              |  22 +-
 .../common/bytes/BytesReference.java          |  75 ++++++-
 .../bytes/BytesReferenceStreamInput.java      | 136 +++++++++++++
 .../common/bytes/CompositeBytesReference.java | 151 ++++++++++++++
 .../common/bytes/PagedBytesReference.java     | 113 -----------
 .../common/io/stream/StreamInput.java         |   6 -
 .../elasticsearch/transport/TcpHeader.java    |   2 +-
 .../elasticsearch/transport/TcpTransport.java | 140 +++++++------
 .../transport/netty/NettyTransport.java       | 191 ++----------------
 .../bytes/AbstractBytesReferenceTestCase.java |  50 +++++
 .../bytes/CompositeBytesReferenceTests.java   | 110 ++++++++++
 .../zen/ping/unicast/UnicastZenPingIT.java    |   1 -
 12 files changed, 614 insertions(+), 383 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/common/bytes/BytesReferenceStreamInput.java
 create mode 100644 core/src/main/java/org/elasticsearch/common/bytes/CompositeBytesReference.java
 create mode 100644 core/src/test/java/org/elasticsearch/common/bytes/CompositeBytesReferenceTests.java

diff --git a/core/src/main/java/org/elasticsearch/common/bytes/BytesArray.java b/core/src/main/java/org/elasticsearch/common/bytes/BytesArray.java
index f6635d94bc6..43c1df588b1 100644
--- a/core/src/main/java/org/elasticsearch/common/bytes/BytesArray.java
+++ b/core/src/main/java/org/elasticsearch/common/bytes/BytesArray.java
@@ -35,10 +35,7 @@ public final class BytesArray extends BytesReference {
     private final int length;
 
     public BytesArray(String bytes) {
-        BytesRef bytesRef = new BytesRef(bytes);
-        this.bytes = bytesRef.bytes;
-        this.offset = bytesRef.offset;
-        this.length = bytesRef.length;
+        this(new BytesRef(bytes));
     }
 
     public BytesArray(BytesRef bytesRef) {
@@ -47,21 +44,15 @@ public final class BytesArray extends BytesReference {
 
     public BytesArray(BytesRef bytesRef, boolean deepCopy) {
         if (deepCopy) {
-            BytesRef copy = BytesRef.deepCopyOf(bytesRef);
-            bytes = copy.bytes;
-            offset = copy.offset;
-            length = copy.length;
-        } else {
-            bytes = bytesRef.bytes;
-            offset = bytesRef.offset;
-            length = bytesRef.length;
+            bytesRef = BytesRef.deepCopyOf(bytesRef);
         }
+        bytes = bytesRef.bytes;
+        offset = bytesRef.offset;
+        length = bytesRef.length;
     }
 
     public BytesArray(byte[] bytes) {
-        this.bytes = bytes;
-        this.offset = 0;
-        this.length = bytes.length;
+        this(bytes, 0, bytes.length);
     }
 
     public BytesArray(byte[] bytes, int offset, int length) {
@@ -105,4 +96,5 @@ public final class BytesArray extends BytesReference {
     public long ramBytesUsed() {
         return bytes.length;
     }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java b/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java
index 6fcb783dd30..f31ea2bbf82 100644
--- a/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java
+++ b/core/src/main/java/org/elasticsearch/common/bytes/BytesReference.java
@@ -24,6 +24,7 @@ import org.apache.lucene.util.BytesRefIterator;
 import org.elasticsearch.common.io.stream.StreamInput;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.function.ToIntBiFunction;
 
@@ -52,9 +53,8 @@ public abstract class BytesReference implements Accountable, Comparable<BytesRef
     /**
      * A stream input of the bytes.
      */
-    public StreamInput streamInput() {
-        BytesRef ref = toBytesRef();
-        return StreamInput.wrap(ref.bytes, ref.offset, ref.length);
+    public StreamInput streamInput() throws IOException {
+        return new MarkSupportingStreamInputWrapper(this);
     }
 
     /**
@@ -208,4 +208,73 @@ public abstract class BytesReference implements Accountable, Comparable<BytesRef
         ref.length -= length;
         ref.offset += length;
     }
+
+    /**
+     * Instead of adding the complexity of {@link InputStream#reset()} etc to the actual impl
+     * this wrapper builds it on top of the BytesReferenceStreamInput which is much simpler
+     * that way.
+     */
+    private static final class MarkSupportingStreamInputWrapper extends StreamInput {
+        private final BytesReference reference;
+        private BytesReferenceStreamInput input;
+        private int mark = 0;
+
+        private MarkSupportingStreamInputWrapper(BytesReference reference) throws IOException {
+            this.reference = reference;
+            this.input = new BytesReferenceStreamInput(reference.iterator(), reference.length());
+        }
+
+        @Override
+        public byte readByte() throws IOException {
+            return input.readByte();
+        }
+
+        @Override
+        public void readBytes(byte[] b, int offset, int len) throws IOException {
+            input.readBytes(b, offset, len);
+        }
+
+        @Override
+        public int read(byte[] b, int off, int len) throws IOException {
+            return input.read(b, off, len);
+        }
+
+        @Override
+        public void close() throws IOException {
+            input.close();
+        }
+
+        @Override
+        public int read() throws IOException {
+            return input.read();
+        }
+
+        @Override
+        public int available() throws IOException {
+            return input.available();
+        }
+
+        @Override
+        public void reset() throws IOException {
+            input = new BytesReferenceStreamInput(reference.iterator(), reference.length());
+            input.skip(mark);
+        }
+
+        @Override
+        public boolean markSupported() {
+            return true;
+        }
+
+        @Override
+        public void mark(int readLimit) {
+            // readLimit is optional it only guarantees that the stream remembers data upto this limit but it can remember more
+            // which we do in our case
+            this.mark = input.getOffset();
+        }
+
+        @Override
+        public long skip(long n) throws IOException {
+            return input.skip(n);
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/bytes/BytesReferenceStreamInput.java b/core/src/main/java/org/elasticsearch/common/bytes/BytesReferenceStreamInput.java
new file mode 100644
index 00000000000..4426ea53efa
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/bytes/BytesReferenceStreamInput.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.common.bytes;
+
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefIterator;
+import org.elasticsearch.common.io.stream.StreamInput;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+/**
+ * A StreamInput that reads off a {@link BytesRefIterator}. This is used to provide
+ * generic stream access to {@link BytesReference} instances without materializing the
+ * underlying bytes reference.
+ */
+final class BytesReferenceStreamInput extends StreamInput {
+    private final BytesRefIterator iterator;
+    private int sliceOffset;
+    private BytesRef slice;
+    private final int length; // the total size of the stream
+    private int offset; // the current position of the stream
+
+    public BytesReferenceStreamInput(BytesRefIterator iterator, final int length) throws IOException {
+        this.iterator = iterator;
+        this.slice = iterator.next();
+        this.length = length;
+        this.offset = 0;
+        this.sliceOffset = 0;
+    }
+
+    @Override
+    public byte readByte() throws IOException {
+        if (offset >= length) {
+            throw new EOFException();
+        }
+        maybeNextSlice();
+        byte b = slice.bytes[slice.offset + (sliceOffset++)];
+        offset++;
+        return b;
+    }
+
+    private void maybeNextSlice() throws IOException {
+        while (sliceOffset == slice.length) {
+            slice = iterator.next();
+            sliceOffset = 0;
+            if (slice == null) {
+                throw new EOFException();
+            }
+        }
+    }
+
+    @Override
+    public void readBytes(byte[] b, int bOffset, int len) throws IOException {
+        if (offset + len > length) {
+            throw new IndexOutOfBoundsException("Cannot read " + len + " bytes from stream with length " + length + " at offset " + offset);
+        }
+        read(b, bOffset, len);
+    }
+
+    @Override
+    public int read() throws IOException {
+        if (offset >= length) {
+            return -1;
+        }
+        return Byte.toUnsignedInt(readByte());
+    }
+
+    @Override
+    public int read(final byte[] b, final int bOffset, final int len) throws IOException {
+        if (offset >= length) {
+            return -1;
+        }
+        final int numBytesToCopy =  Math.min(len, length - offset);
+        int remaining = numBytesToCopy; // copy the full length or the remaining part
+        int destOffset = bOffset;
+        while (remaining > 0) {
+            maybeNextSlice();
+            final int currentLen = Math.min(remaining, slice.length - sliceOffset);
+            assert currentLen > 0 : "length has to be > 0 to make progress but was: " + currentLen;
+            System.arraycopy(slice.bytes, slice.offset + sliceOffset, b, destOffset, currentLen);
+            destOffset += currentLen;
+            remaining -= currentLen;
+            sliceOffset += currentLen;
+            offset += currentLen;
+            assert remaining >= 0 : "remaining: " + remaining;
+        }
+        return numBytesToCopy;
+    }
+
+    @Override
+    public void close() throws IOException {
+        // do nothing
+    }
+
+    @Override
+    public int available() throws IOException {
+        return length - offset;
+    }
+
+    @Override
+    public long skip(long n) throws IOException {
+        final int skip = (int) Math.min(Integer.MAX_VALUE, n);
+        final int numBytesSkipped =  Math.min(skip, length - offset);
+        int remaining = numBytesSkipped;
+        while (remaining > 0) {
+            maybeNextSlice();
+            int currentLen = Math.min(remaining, slice.length - (slice.offset + sliceOffset));
+            remaining -= currentLen;
+            sliceOffset += currentLen;
+            offset += currentLen;
+            assert remaining >= 0 : "remaining: " + remaining;
+        }
+        return numBytesSkipped;
+    }
+
+    int getOffset() {
+        return offset;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/bytes/CompositeBytesReference.java b/core/src/main/java/org/elasticsearch/common/bytes/CompositeBytesReference.java
new file mode 100644
index 00000000000..279275b92e8
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/common/bytes/CompositeBytesReference.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.bytes;
+
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.lucene.util.BytesRefIterator;
+import org.apache.lucene.util.RamUsageEstimator;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Objects;
+
+/**
+ * A composite {@link BytesReference} that allows joining multiple bytes references
+ * into one without copying.
+ *
+ * Note, {@link #toBytesRef()} will materialize all pages in this BytesReference.
+ */
+public final class CompositeBytesReference extends BytesReference {
+
+    private final BytesReference[] references;
+    private final int[] offsets;
+    private final int length;
+    private final long ramBytesUsed;
+
+    public CompositeBytesReference(BytesReference... references) {
+        this.references = Objects.requireNonNull(references, "references must not be null");
+        this.offsets = new int[references.length];
+        long ramBytesUsed = 0;
+        int offset = 0;
+        for (int i = 0; i < references.length; i++) {
+            BytesReference reference = references[i];
+            if (reference == null) {
+                throw new IllegalArgumentException("references must not be null");
+            }
+            offsets[i] = offset; // we use the offsets to seek into the right BytesReference for random access and slicing
+            offset += reference.length();
+            ramBytesUsed += reference.ramBytesUsed();
+        }
+        this.ramBytesUsed = ramBytesUsed
+            + (Integer.BYTES * offsets.length + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER) // offsets
+            + (references.length * RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER) // references
+            + Integer.BYTES // length
+            + Long.BYTES; // ramBytesUsed
+        length = offset;
+    }
+
+
+    @Override
+    public byte get(int index) {
+        final int i = getOffsetIndex(index);
+        return references[i].get(index - offsets[i]);
+    }
+
+    @Override
+    public int length() {
+        return length;
+    }
+
+    @Override
+    public BytesReference slice(int from, int length) {
+        // for slices we only need to find the start and the end reference
+        // adjust them and pass on the references in between as they are fully contained
+        final int to = from + length;
+        final int limit = getOffsetIndex(from + length);
+        final int start = getOffsetIndex(from);
+        final BytesReference[] inSlice = new BytesReference[1 + (limit - start)];
+        for (int i = 0, j = start; i < inSlice.length; i++) {
+            inSlice[i] = references[j++];
+        }
+        int inSliceOffset = from - offsets[start];
+        if (inSlice.length == 1) {
+            return inSlice[0].slice(inSliceOffset, length);
+        }
+        // now adjust slices in front and at the end
+        inSlice[0] = inSlice[0].slice(inSliceOffset, inSlice[0].length() - inSliceOffset);
+        inSlice[inSlice.length-1] = inSlice[inSlice.length-1].slice(0, to - offsets[limit]);
+        return new CompositeBytesReference(inSlice);
+    }
+
+    private final int getOffsetIndex(int offset) {
+        final int i = Arrays.binarySearch(offsets, offset);
+        return i < 0 ? (-(i + 1)) - 1 : i;
+    }
+
+    @Override
+    public BytesRef toBytesRef() {
+        BytesRefBuilder builder = new BytesRefBuilder();
+        builder.grow(length());
+        BytesRef spare;
+        BytesRefIterator iterator = iterator();
+        try {
+            while ((spare = iterator.next()) != null) {
+                builder.append(spare);
+            }
+        } catch (IOException ex) {
+            throw new AssertionError("won't happen", ex); // this is really an error since we don't do IO in our bytesreferences
+        }
+        return builder.toBytesRef();
+    }
+
+    @Override
+    public BytesRefIterator iterator() {
+        if (references.length > 0) {
+            return new BytesRefIterator() {
+                int index = 0;
+                private BytesRefIterator current = references[index++].iterator();
+                @Override
+                public BytesRef next() throws IOException {
+                    BytesRef next = current.next();
+                    if (next == null) {
+                        while (index < references.length) {
+                            current = references[index++].iterator();
+                            next = current.next();
+                            if (next != null) {
+                                break;
+                            }
+                        }
+                    }
+                    return next;
+                }
+            };
+        } else {
+            return () -> null;
+        }
+
+    }
+
+    @Override
+    public long ramBytesUsed() {
+       return ramBytesUsed;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java b/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java
index a4bdd2b2e84..ef1102326d6 100644
--- a/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java
+++ b/core/src/main/java/org/elasticsearch/common/bytes/PagedBytesReference.java
@@ -25,10 +25,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.ByteArray;
 
-import java.io.EOFException;
 import java.io.IOException;
-import java.io.OutputStream;
-import java.util.Arrays;
 
 /**
  * A page based bytes reference, internally holding the bytes in a paged
@@ -42,7 +39,6 @@ public class PagedBytesReference extends BytesReference {
     protected final ByteArray bytearray;
     private final int offset;
     private final int length;
-    private int hash = 0;
 
     public PagedBytesReference(BigArrays bigarrays, ByteArray bytearray, int length) {
         this(bigarrays, bytearray, 0, length);
@@ -70,15 +66,9 @@ public class PagedBytesReference extends BytesReference {
         if (from < 0 || (from + length) > length()) {
             throw new IllegalArgumentException("can't slice a buffer with length [" + length() + "], with slice parameters from [" + from + "], length [" + length + "]");
         }
-
         return new PagedBytesReference(bigarrays, bytearray, offset + from, length);
     }
 
-    @Override
-    public StreamInput streamInput() {
-        return new PagedBytesReferenceStreamInput(bytearray, offset, length);
-    }
-
     @Override
     public BytesRef toBytesRef() {
         BytesRef bref = new BytesRef();
@@ -87,109 +77,6 @@ public class PagedBytesReference extends BytesReference {
         return bref;
     }
 
-    private static class PagedBytesReferenceStreamInput extends StreamInput {
-
-        private final ByteArray bytearray;
-        private final BytesRef ref;
-        private final int offset;
-        private final int length;
-        private int pos;
-        private int mark;
-
-        public PagedBytesReferenceStreamInput(ByteArray bytearray, int offset, int length) {
-            this.bytearray = bytearray;
-            this.ref = new BytesRef();
-            this.offset = offset;
-            this.length = length;
-            this.pos = 0;
-
-            if (offset + length > bytearray.size()) {
-                throw new IndexOutOfBoundsException("offset+length >= bytearray.size()");
-            }
-        }
-
-        @Override
-        public byte readByte() throws IOException {
-            if (pos >= length) {
-                throw new EOFException();
-            }
-
-            return bytearray.get(offset + pos++);
-        }
-
-        @Override
-        public void readBytes(byte[] b, int bOffset, int len) throws IOException {
-            if (len > offset + length) {
-                throw new IndexOutOfBoundsException("Cannot read " + len + " bytes from stream with length " + length + " at pos " + pos);
-            }
-
-            read(b, bOffset, len);
-        }
-
-        @Override
-        public int read() throws IOException {
-            return (pos < length) ? Byte.toUnsignedInt(bytearray.get(offset + pos++)) : -1;
-        }
-
-        @Override
-        public int read(final byte[] b, final int bOffset, final int len) throws IOException {
-            if (len == 0) {
-                return 0;
-            }
-
-            if (pos >= offset + length) {
-                return -1;
-            }
-
-            final int numBytesToCopy = Math.min(len, length - pos); // copy the full length or the remaining part
-
-            // current offset into the underlying ByteArray
-            long byteArrayOffset = offset + pos;
-
-            // bytes already copied
-            int copiedBytes = 0;
-
-            while (copiedBytes < numBytesToCopy) {
-                long pageFragment = PAGE_SIZE - (byteArrayOffset % PAGE_SIZE); // how much can we read until hitting N*PAGE_SIZE?
-                int bulkSize = (int) Math.min(pageFragment, numBytesToCopy - copiedBytes); // we cannot copy more than a page fragment
-                boolean copied = bytearray.get(byteArrayOffset, bulkSize, ref); // get the fragment
-                assert (copied == false); // we should never ever get back a materialized byte[]
-                System.arraycopy(ref.bytes, ref.offset, b, bOffset + copiedBytes, bulkSize); // copy fragment contents
-                copiedBytes += bulkSize; // count how much we copied
-                byteArrayOffset += bulkSize; // advance ByteArray index
-            }
-
-            pos += copiedBytes; // finally advance our stream position
-            return copiedBytes;
-        }
-
-        @Override
-        public boolean markSupported() {
-            return true;
-        }
-
-        @Override
-        public void mark(int readlimit) {
-            this.mark = pos;
-        }
-
-        @Override
-        public void reset() throws IOException {
-            pos = mark;
-        }
-
-        @Override
-        public void close() throws IOException {
-            // do nothing
-        }
-
-        @Override
-        public int available() throws IOException {
-            return length - pos;
-        }
-
-    }
-
     @Override
     public final BytesRefIterator iterator() {
         final int offset = this.offset;
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
index 1b28adb1cb1..80ff214cbc9 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
@@ -380,12 +380,6 @@ public abstract class StreamInput extends InputStream {
         return false;
     }
 
-    /**
-     * Resets the stream.
-     */
-    @Override
-    public abstract void reset() throws IOException;
-
     /**
      * Closes the stream to further operations.
      */
diff --git a/core/src/main/java/org/elasticsearch/transport/TcpHeader.java b/core/src/main/java/org/elasticsearch/transport/TcpHeader.java
index bbc54c0eb62..d18f62f60ee 100644
--- a/core/src/main/java/org/elasticsearch/transport/TcpHeader.java
+++ b/core/src/main/java/org/elasticsearch/transport/TcpHeader.java
@@ -41,7 +41,7 @@ public class TcpHeader {
         output.writeByte((byte)'E');
         output.writeByte((byte)'S');
         // write the size, the size indicates the remaining message size, not including the size int
-        output.writeInt(messageSize - TcpHeader.MARKER_BYTES_SIZE - TcpHeader.MESSAGE_LENGTH_SIZE);
+        output.writeInt(messageSize + REQUEST_ID_SIZE + STATUS_SIZE + VERSION_ID_SIZE);
         output.writeLong(requestId);
         output.writeByte(status);
         output.writeInt(version.id);
diff --git a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
index 6814e7ad5f1..6bfebd1f35d 100644
--- a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
@@ -29,6 +29,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.bytes.CompositeBytesReference;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.component.Lifecycle;
 import org.elasticsearch.common.compress.Compressor;
@@ -347,6 +348,10 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
             }
         }
 
+        public List<Channel[]> getChannelArrays() {
+            return Arrays.asList(recovery, bulk, reg, state, ping);
+        }
+
         public synchronized void close() {
             closeChannels(allChannels);
         }
@@ -869,7 +874,7 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
     protected void stopInternal() {}
 
     public boolean canCompress(TransportRequest request) {
-        return compress;
+        return compress && (!(request instanceof BytesTransportRequest));
     }
 
     @Override
@@ -885,9 +890,8 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
         status = TransportStatus.setRequest(status);
         ReleasableBytesStreamOutput bStream = new ReleasableBytesStreamOutput(bigArrays);
         boolean addedReleaseListener = false;
+        StreamOutput stream = bStream;
         try {
-            bStream.skip(TcpHeader.HEADER_SIZE);
-            StreamOutput stream = bStream;
             // only compress if asked, and, the request is not bytes, since then only
             // the header part is compressed, and the "body" can't be extracted as compressed
             if (options.compress() && canCompress(request)) {
@@ -903,12 +907,7 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
             stream.setVersion(version);
             threadPool.getThreadContext().writeTo(stream);
             stream.writeString(action);
-
-            Message<Channel> writeable = prepareSend(node.getVersion(), request, stream, bStream);
-            try (StreamOutput headerOutput = writeable.getHeaderOutput()) {
-                TcpHeader.writeHeader(headerOutput, requestId, status, version,
-                    writeable.size());
-            }
+            BytesReference message = buildMessage(requestId, status, node.getVersion(), request, stream, bStream);
             final TransportRequestOptions finalOptions = options;
             Runnable onRequestSent = () -> {
                 try {
@@ -917,10 +916,10 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
                     transportServiceAdapter.onRequestSent(node, requestId, action, request, finalOptions);
                 }
             };
-            writeable.send(targetChannel, onRequestSent);
+            sendMessage(targetChannel, message, onRequestSent, false);
             addedReleaseListener = true;
-
         } finally {
+            IOUtils.close(stream);
             if (!addedReleaseListener) {
                 Releasables.close(bStream.bytes());
             }
@@ -937,26 +936,19 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
      */
     public void sendErrorResponse(Version nodeVersion, Channel channel, final Exception error, final long requestId,
                                   final String action) throws IOException {
-        BytesStreamOutput stream = new BytesStreamOutput();
-        stream.setVersion(nodeVersion);
-        stream.skip(TcpHeader.HEADER_SIZE);
-        RemoteTransportException tx = new RemoteTransportException(
-            nodeName(), new InetSocketTransportAddress(getLocalAddress(channel)), action, error);
-        stream.writeThrowable(tx);
-        byte status = 0;
-        status = TransportStatus.setResponse(status);
-        status = TransportStatus.setError(status);
-
-        final BytesReference bytes = stream.bytes();
-        Message<Channel> writeable = prepareSend(nodeVersion, bytes);
-        try (StreamOutput headerOutput = writeable.getHeaderOutput()) {
-            TcpHeader.writeHeader(headerOutput, requestId, status, nodeVersion,
-                writeable.size());
+        try(BytesStreamOutput stream = new BytesStreamOutput()) {
+            stream.setVersion(nodeVersion);
+            RemoteTransportException tx = new RemoteTransportException(
+                nodeName(), new InetSocketTransportAddress(getLocalAddress(channel)), action, error);
+            stream.writeThrowable(tx);
+            byte status = 0;
+            status = TransportStatus.setResponse(status);
+            status = TransportStatus.setError(status);
+            final BytesReference bytes = stream.bytes();
+            final BytesReference header = buildHeader(requestId, status, nodeVersion, bytes.length());
+            Runnable onRequestSent = () -> transportServiceAdapter.onResponseSent(requestId, action, error);
+            sendMessage(channel, new CompositeBytesReference(header, bytes), onRequestSent, false);
         }
-        Runnable onRequestSent = () -> {
-            transportServiceAdapter.onResponseSent(requestId, action, error);
-        };
-        writeable.send(channel, onRequestSent);
     }
 
     /**
@@ -974,19 +966,15 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
         status = TransportStatus.setResponse(status); // TODO share some code with sendRequest
         ReleasableBytesStreamOutput bStream = new ReleasableBytesStreamOutput(bigArrays);
         boolean addedReleaseListener = false;
+        StreamOutput stream = bStream;
         try {
-            bStream.skip(TcpHeader.HEADER_SIZE);
-            StreamOutput stream = bStream;
             if (options.compress()) {
                 status = TransportStatus.setCompress(status);
                 stream = CompressorFactory.COMPRESSOR.streamOutput(stream);
             }
             stream.setVersion(nodeVersion);
-            Message<Channel> writeable = prepareSend(nodeVersion, response, stream, bStream);
-            try (StreamOutput headerOutput = writeable.getHeaderOutput()) {
-                TcpHeader.writeHeader(headerOutput, requestId, status, nodeVersion,
-                    writeable.size());
-            }
+            BytesReference reference = buildMessage(requestId, status,nodeVersion, response, stream, bStream);
+
             final TransportResponseOptions finalOptions = options;
             Runnable onRequestSent = () -> {
                 try {
@@ -995,10 +983,11 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
                     transportServiceAdapter.onResponseSent(requestId, action, response, finalOptions);
                 }
             };
-            writeable.send(channel, onRequestSent);
+            sendMessage(channel, reference, onRequestSent, false);
             addedReleaseListener = true;
 
         } finally {
+            IOUtils.close(stream);
             if (!addedReleaseListener) {
                 Releasables.close(bStream.bytes());
             }
@@ -1006,44 +995,51 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
     }
 
     /**
-     * Serializes the given message into a bytes representation and forwards to {@link #prepareSend(Version, TransportMessage,
-     * StreamOutput, ReleasableBytesStream)}
+     * Writes the Tcp message header into a bytes reference.
+     *
+     * @param requestId       the request ID
+     * @param status          the request status
+     * @param protocolVersion the protocol version used to serialize the data in the message
+     * @param length          the payload length in bytes
+     * @see TcpHeader
      */
-    protected Message<Channel> prepareSend(Version nodeVersion, TransportMessage message, StreamOutput stream,
-                                                    ReleasableBytesStream writtenBytes) throws IOException {
-        message.writeTo(stream);
+    private BytesReference buildHeader(long requestId, byte status, Version protocolVersion, int length) throws IOException {
+        try (BytesStreamOutput headerOutput = new BytesStreamOutput(TcpHeader.HEADER_SIZE)) {
+            headerOutput.setVersion(protocolVersion);
+            TcpHeader.writeHeader(headerOutput, requestId, status, protocolVersion,
+                length);
+            final BytesReference bytes = headerOutput.bytes();
+            assert bytes.length() == TcpHeader.HEADER_SIZE : "header size mismatch expected: " + TcpHeader.HEADER_SIZE + " but was: "
+                + bytes.length();
+            return bytes;
+        }
+    }
+
+    /**
+     * Serializes the given message into a bytes representation
+     */
+    private BytesReference buildMessage(long requestId, byte status, Version nodeVersion, TransportMessage message, StreamOutput stream,
+                                        ReleasableBytesStream writtenBytes) throws IOException {
+        final BytesReference zeroCopyBuffer;
+        if (message instanceof BytesTransportRequest) { // what a shitty optimization - we should use a direct send method instead
+            BytesTransportRequest bRequest = (BytesTransportRequest) message;
+            assert nodeVersion.equals(bRequest.version());
+            bRequest.writeThin(stream);
+            zeroCopyBuffer = bRequest.bytes;
+        } else {
+            message.writeTo(stream);
+            zeroCopyBuffer = BytesArray.EMPTY;
+        }
+        // we have to close the stream here - flush is not enough since we might be compressing the content
+        // and if we do that the close method will write some marker bytes (EOS marker) and otherwise
+        // we barf on the decompressing end when we read past EOF on purpose in the #validateRequest method.
+        // this might be a problem in deflate after all but it's important to close it for now.
         stream.close();
-        return prepareSend(nodeVersion, writtenBytes.bytes());
+        final BytesReference messageBody = writtenBytes.bytes();
+        final BytesReference header = buildHeader(requestId, status, stream.getVersion(), messageBody.length() + zeroCopyBuffer.length());
+        return new CompositeBytesReference(header, messageBody, zeroCopyBuffer);
     }
 
-    /**
-     * prepares a implementation specific message to send across the network
-     */
-    protected abstract Message<Channel> prepareSend(Version nodeVersion, BytesReference bytesReference) throws IOException;
-
-    /**
-     * Allows implementations to transform TransportMessages into implementation specific messages
-     */
-    protected interface Message<Channel> {
-        /**
-         * Creates an output to write the message header to.
-         */
-        StreamOutput getHeaderOutput();
-
-        /**
-         * Returns the size of the message in bytes
-         */
-        int size();
-
-        /**
-         * sends the message to the channel
-         * @param channel the channe to send the message to
-         * @param onRequestSent a callback executed once the message has been fully send
-         */
-        void send(Channel channel, Runnable onRequestSent);
-    }
-
-
     /**
      * Validates the first N bytes of the message header and returns <code>true</code> if the message is
      * a ping message and has no payload ie. isn't a real user level message.
diff --git a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
index 804d84e8374..31e04c56e71 100644
--- a/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/netty/NettyTransport.java
@@ -20,15 +20,11 @@
 package org.elasticsearch.transport.netty;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.bytes.ReleasablePagedBytesReference;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.io.ReleasableBytesStream;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lease.Releasables;
 import org.elasticsearch.common.network.NetworkService;
 import org.elasticsearch.common.network.NetworkService.TcpSettings;
@@ -43,17 +39,12 @@ import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.transport.BytesTransportRequest;
 import org.elasticsearch.transport.ConnectTransportException;
 import org.elasticsearch.transport.TcpTransport;
-import org.elasticsearch.transport.TransportMessage;
-import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.transport.TransportServiceAdapter;
 import org.elasticsearch.transport.TransportSettings;
 import org.jboss.netty.bootstrap.ClientBootstrap;
 import org.jboss.netty.bootstrap.ServerBootstrap;
-import org.jboss.netty.buffer.ChannelBuffer;
-import org.jboss.netty.buffer.ChannelBuffers;
 import org.jboss.netty.channel.AdaptiveReceiveBufferSizePredictorFactory;
 import org.jboss.netty.channel.Channel;
 import org.jboss.netty.channel.ChannelFuture;
@@ -72,11 +63,10 @@ import org.jboss.netty.channel.socket.oio.OioClientSocketChannelFactory;
 import org.jboss.netty.channel.socket.oio.OioServerSocketChannelFactory;
 import org.jboss.netty.util.HashedWheelTimer;
 
-import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Executors;
@@ -346,81 +336,33 @@ public class NettyTransport extends TcpTransport<Channel> {
         channels[0].getCloseFuture().addListener(new ChannelCloseListener(node));
         return new NodeChannels(channels, channels, channels, channels, channels);
     }
-
     protected NodeChannels connectToChannels(DiscoveryNode node) {
         final NodeChannels nodeChannels = new NodeChannels(new Channel[connectionsPerNodeRecovery], new Channel[connectionsPerNodeBulk],
             new Channel[connectionsPerNodeReg], new Channel[connectionsPerNodeState],
             new Channel[connectionsPerNodePing]);
         boolean success = false;
         try {
-            ChannelFuture[] connectRecovery = new ChannelFuture[nodeChannels.recovery.length];
-            ChannelFuture[] connectBulk = new ChannelFuture[nodeChannels.bulk.length];
-            ChannelFuture[] connectReg = new ChannelFuture[nodeChannels.reg.length];
-            ChannelFuture[] connectState = new ChannelFuture[nodeChannels.state.length];
-            ChannelFuture[] connectPing = new ChannelFuture[nodeChannels.ping.length];
+            int numConnections = connectionsPerNodeBulk + connectionsPerNodePing + connectionsPerNodeRecovery + connectionsPerNodeReg
+                + connectionsPerNodeState;
+            ArrayList<ChannelFuture> connections = new ArrayList<>();
             InetSocketAddress address = ((InetSocketTransportAddress) node.getAddress()).address();
-            for (int i = 0; i < connectRecovery.length; i++) {
-                connectRecovery[i] = clientBootstrap.connect(address);
+            for (int i = 0; i < numConnections; i++) {
+                connections.add(clientBootstrap.connect(address));
             }
-            for (int i = 0; i < connectBulk.length; i++) {
-                connectBulk[i] = clientBootstrap.connect(address);
-            }
-            for (int i = 0; i < connectReg.length; i++) {
-                connectReg[i] = clientBootstrap.connect(address);
-            }
-            for (int i = 0; i < connectState.length; i++) {
-                connectState[i] = clientBootstrap.connect(address);
-            }
-            for (int i = 0; i < connectPing.length; i++) {
-                connectPing[i] = clientBootstrap.connect(address);
-            }
-
+            final Iterator<ChannelFuture> iterator = connections.iterator();
             try {
-                for (int i = 0; i < connectRecovery.length; i++) {
-                    connectRecovery[i].awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
-                    if (!connectRecovery[i].isSuccess()) {
-                        throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connectRecovery[i].getCause());
+                for (Channel[] channels : nodeChannels.getChannelArrays()) {
+                    for (int i = 0; i < channels.length; i++) {
+                        assert iterator.hasNext();
+                        ChannelFuture future = iterator.next();
+                        future.awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
+                        if (!future.isSuccess()) {
+                            throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", future.getCause());
+                        }
+                        channels[i] = future.getChannel();
+                        channels[i].getCloseFuture().addListener(new ChannelCloseListener(node));
                     }
-                    nodeChannels.recovery[i] = connectRecovery[i].getChannel();
-                    nodeChannels.recovery[i].getCloseFuture().addListener(new ChannelCloseListener(node));
                 }
-
-                for (int i = 0; i < connectBulk.length; i++) {
-                    connectBulk[i].awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
-                    if (!connectBulk[i].isSuccess()) {
-                        throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connectBulk[i].getCause());
-                    }
-                    nodeChannels.bulk[i] = connectBulk[i].getChannel();
-                    nodeChannels.bulk[i].getCloseFuture().addListener(new ChannelCloseListener(node));
-                }
-
-                for (int i = 0; i < connectReg.length; i++) {
-                    connectReg[i].awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
-                    if (!connectReg[i].isSuccess()) {
-                        throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connectReg[i].getCause());
-                    }
-                    nodeChannels.reg[i] = connectReg[i].getChannel();
-                    nodeChannels.reg[i].getCloseFuture().addListener(new ChannelCloseListener(node));
-                }
-
-                for (int i = 0; i < connectState.length; i++) {
-                    connectState[i].awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
-                    if (!connectState[i].isSuccess()) {
-                        throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connectState[i].getCause());
-                    }
-                    nodeChannels.state[i] = connectState[i].getChannel();
-                    nodeChannels.state[i].getCloseFuture().addListener(new ChannelCloseListener(node));
-                }
-
-                for (int i = 0; i < connectPing.length; i++) {
-                    connectPing[i].awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
-                    if (!connectPing[i].isSuccess()) {
-                        throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connectPing[i].getCause());
-                    }
-                    nodeChannels.ping[i] = connectPing[i].getChannel();
-                    nodeChannels.ping[i].getCloseFuture().addListener(new ChannelCloseListener(node));
-                }
-
                 if (nodeChannels.recovery.length == 0) {
                     if (nodeChannels.bulk.length > 0) {
                         nodeChannels.recovery = nodeChannels.bulk;
@@ -432,14 +374,7 @@ public class NettyTransport extends TcpTransport<Channel> {
                     nodeChannels.bulk = nodeChannels.reg;
                 }
             } catch (RuntimeException e) {
-                // clean the futures
-                List<ChannelFuture> futures = new ArrayList<>();
-                futures.addAll(Arrays.asList(connectRecovery));
-                futures.addAll(Arrays.asList(connectBulk));
-                futures.addAll(Arrays.asList(connectReg));
-                futures.addAll(Arrays.asList(connectState));
-                futures.addAll(Arrays.asList(connectPing));
-                for (ChannelFuture future : Collections.unmodifiableList(futures)) {
+                for (ChannelFuture future : Collections.unmodifiableList(connections)) {
                     future.cancel();
                     if (future.getChannel() != null && future.getChannel().isOpen()) {
                         try {
@@ -546,6 +481,7 @@ public class NettyTransport extends TcpTransport<Channel> {
         }
     }
 
+    @Override
     protected void sendMessage(Channel channel, BytesReference reference, Runnable sendListener, boolean close) {
         final ChannelFuture future = channel.write(NettyUtils.toChannelBuffer(reference));
         if (close) {
@@ -617,93 +553,4 @@ public class NettyTransport extends TcpTransport<Channel> {
             }
         });
     }
-
-    @Override
-    public Message<Channel> prepareSend(Version nodeVersion, TransportMessage message, StreamOutput stream,
-                                        ReleasableBytesStream writtenBytes) throws IOException {
-            // it might be nice to somehow generalize this optimization, maybe a smart "paged" bytes output
-            // that create paged channel buffers, but its tricky to know when to do it (where this option is
-            // more explicit).
-            if (message instanceof BytesTransportRequest) {
-                BytesTransportRequest bRequest = (BytesTransportRequest) message;
-                assert nodeVersion.equals(bRequest.version());
-                bRequest.writeThin(stream);
-                stream.close();
-                ReleasablePagedBytesReference bytes = writtenBytes.bytes();
-                ChannelBuffer headerBuffer = NettyUtils.toChannelBuffer(bytes);
-                ChannelBuffer contentBuffer = NettyUtils.toChannelBuffer(bRequest.bytes());
-                ChannelBuffer buffer = ChannelBuffers.wrappedBuffer(NettyUtils.DEFAULT_GATHERING, headerBuffer, contentBuffer);
-                return new NettyMessage(buffer);
-            } else {
-                return super.prepareSend(nodeVersion, message, stream, writtenBytes);
-            }
-    }
-
-    @Override
-    public Message<Channel> prepareSend(Version nodeVersion, BytesReference bytesReference) {
-        return new NettyMessage(NettyUtils.toChannelBuffer(bytesReference));
-    }
-
-    @Override
-    public boolean canCompress(TransportRequest request) {
-        return super.canCompress(request) && (!(request instanceof BytesTransportRequest));
-    }
-
-    private class NettyMessage implements Message<Channel> {
-        private final ChannelBuffer buffer;
-
-        public NettyMessage(ChannelBuffer buffer) {
-            this.buffer = buffer;
-        }
-
-        public StreamOutput getHeaderOutput() {
-            return new ChannelBufferStreamOutput(buffer);
-        }
-
-        public int size() {
-            return buffer.readableBytes();
-        }
-
-        @Override
-        public void send(Channel channel, Runnable onRequestSent) {
-            ChannelFuture future = channel.write(buffer);
-            ChannelFutureListener channelFutureListener = f -> onRequestSent.run();
-            future.addListener(channelFutureListener);
-        }
-    }
-
-    private static final class ChannelBufferStreamOutput extends StreamOutput {
-
-        private final ChannelBuffer buffer;
-        private int offset;
-
-        public ChannelBufferStreamOutput(ChannelBuffer buffer) {
-            this.buffer = buffer;
-            this.offset = buffer.readerIndex();
-        }
-
-        @Override
-        public void writeByte(byte b) throws IOException {
-            buffer.setByte(offset++, b);
-        }
-
-        @Override
-        public void writeBytes(byte[] b, int offset, int length) throws IOException {
-            buffer.setBytes(this.offset, b, offset, length);
-            this.offset += length;
-        }
-
-        @Override
-        public void flush() throws IOException {
-        }
-
-        @Override
-        public void close() throws IOException {
-        }
-
-        @Override
-        public void reset() throws IOException {
-            throw new UnsupportedOperationException();
-        }
-    }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java b/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
index 90922327732..c998146ed74 100644
--- a/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
+++ b/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
@@ -42,11 +42,20 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
     public void testGet() throws IOException {
         int length = randomIntBetween(1, PAGE_SIZE * 3);
         BytesReference pbr = newBytesReference(length);
+
         int sliceOffset = randomIntBetween(0, length / 2);
         int sliceLength = Math.max(1, length - sliceOffset - 1);
         BytesReference slice = pbr.slice(sliceOffset, sliceLength);
         assertEquals(pbr.get(sliceOffset), slice.get(0));
         assertEquals(pbr.get(sliceOffset + sliceLength - 1), slice.get(sliceLength - 1));
+        final int probes = randomIntBetween(20, 100);
+        BytesReference copy = new BytesArray(pbr.toBytesRef(), true);
+        for (int i = 0; i < probes; i++) {
+            int index = randomIntBetween(0, copy.length() - 1);
+            assertEquals(pbr.get(index), copy.get(index));
+            index = randomIntBetween(sliceOffset, sliceOffset + sliceLength);
+            assertEquals(pbr.get(index), slice.get(index - sliceOffset));
+        }
     }
 
     public void testLength() throws IOException {
@@ -121,6 +130,26 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
             si.readBytes(targetBuf, 0, length * 2));
     }
 
+    public void testStreamInputMarkAndReset() throws IOException {
+        int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20));
+        BytesReference pbr = newBytesReference(length);
+        StreamInput si = pbr.streamInput();
+        assertNotNull(si);
+
+        StreamInput wrap = StreamInput.wrap(BytesReference.toBytes(pbr));
+        while(wrap.available() > 0) {
+            if (rarely()) {
+                wrap.mark(Integer.MAX_VALUE);
+                si.mark(Integer.MAX_VALUE);
+            } else if (rarely()) {
+                wrap.reset();
+                si.reset();
+            }
+            assertEquals(si.readByte(), wrap.readByte());
+            assertEquals(si.available(), wrap.available());
+        }
+    }
+
     public void testStreamInputBulkReadWithOffset() throws IOException {
         final int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20));
         BytesReference pbr = newBytesReference(length);
@@ -233,6 +262,24 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
         out.close();
     }
 
+    public void testInputStreamSkip() throws IOException {
+        int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20));
+        BytesReference pbr = newBytesReference(length);
+        final int iters = randomIntBetween(5, 50);
+        for (int i = 0; i < iters; i++) {
+            try (StreamInput input = pbr.streamInput()) {
+                final int offset = randomIntBetween(0, length-1);
+                assertEquals(offset, input.skip(offset));
+                assertEquals(pbr.get(offset), input.readByte());
+                final int nextOffset = randomIntBetween(offset, length-2);
+                assertEquals(nextOffset - offset, input.skip(nextOffset - offset));
+                assertEquals(pbr.get(nextOffset+1), input.readByte()); // +1 for the one byte we read above
+                assertEquals(length - (nextOffset+2), input.skip(Long.MAX_VALUE));
+                assertEquals(0, input.skip(randomIntBetween(0, Integer.MAX_VALUE)));
+            }
+        }
+    }
+
     public void testSliceWriteToOutputStream() throws IOException {
         int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 5));
         BytesReference pbr = newBytesReference(length);
@@ -252,6 +299,9 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
             BytesReference pbr = newBytesReference(sizes[i]);
             byte[] bytes = BytesReference.toBytes(pbr);
             assertEquals(sizes[i], bytes.length);
+            for (int j = 0; j  < bytes.length; j++) {
+                assertEquals(bytes[j], pbr.get(j));
+            }
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/common/bytes/CompositeBytesReferenceTests.java b/core/src/test/java/org/elasticsearch/common/bytes/CompositeBytesReferenceTests.java
new file mode 100644
index 00000000000..aec957aba68
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/bytes/CompositeBytesReferenceTests.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.common.bytes;
+
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.lucene.util.BytesRefIterator;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
+import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class CompositeBytesReferenceTests extends AbstractBytesReferenceTestCase {
+    @Override
+    protected BytesReference newBytesReference(int length) throws IOException {
+        // we know bytes stream output always creates a paged bytes reference, we use it to create randomized content
+        List<BytesReference> referenceList = newRefList(length);
+        BytesReference ref = new CompositeBytesReference(referenceList.toArray(new BytesReference[0]));
+        assertEquals(length, ref.length());
+        return ref;
+    }
+
+    private List<BytesReference> newRefList(int length) throws IOException {
+        List<BytesReference> referenceList = new ArrayList<>();
+        for (int i = 0; i < length;) {
+            int remaining = length-i;
+            int sliceLength = randomIntBetween(1, remaining);
+            ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(sliceLength, bigarrays);
+            for (int j = 0; j < sliceLength; j++) {
+                out.writeByte((byte) random().nextInt(1 << 8));
+            }
+            assertEquals(sliceLength, out.size());
+            referenceList.add(out.bytes());
+            i+=sliceLength;
+        }
+        return referenceList;
+    }
+
+    public void testCompositeBuffer() throws IOException {
+        List<BytesReference> referenceList = newRefList(randomIntBetween(1, PAGE_SIZE * 2));
+        BytesReference ref = new CompositeBytesReference(referenceList.toArray(new BytesReference[0]));
+        BytesRefIterator iterator = ref.iterator();
+        BytesRefBuilder builder = new BytesRefBuilder();
+
+        for (BytesReference reference : referenceList) {
+            BytesRefIterator innerIter = reference.iterator(); // sometimes we have a paged ref - pull an iter and walk all pages!
+            BytesRef scratch;
+            while ((scratch = innerIter.next()) != null) {
+                BytesRef next = iterator.next();
+                assertNotNull(next);
+                assertEquals(next, scratch);
+                builder.append(next);
+            }
+
+        }
+        assertNull(iterator.next());
+
+        int offset = 0;
+        for (BytesReference reference : referenceList) {
+            assertEquals(reference, ref.slice(offset, reference.length()));
+            int probes = randomIntBetween(Math.min(10, reference.length()), reference.length());
+            for (int i = 0; i < probes; i++) {
+                int index = randomIntBetween(0, reference.length()-1);
+                assertEquals(ref.get(offset + index), reference.get(index));
+            }
+            offset += reference.length();
+        }
+
+        BytesArray array = new BytesArray(builder.toBytesRef());
+        assertEquals(array, ref);
+        assertEquals(array.hashCode(), ref.hashCode());
+
+        BytesStreamOutput output = new BytesStreamOutput();
+        ref.writeTo(output);
+        assertEquals(array, output.bytes());
+    }
+
+    @Override
+    public void testToBytesRefSharedPage() throws IOException {
+       // CompositeBytesReference doesn't share pages
+    }
+
+    @Override
+    public void testSliceArrayOffset() throws IOException {
+        // the assertions in this test only work on no-composite buffers
+    }
+
+    @Override
+    public void testSliceToBytesRef() throws IOException {
+        // CompositeBytesReference shifts offsets
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java b/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java
index f072c5faf8a..06ac6aeb570 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java
@@ -20,7 +20,6 @@
 package org.elasticsearch.discovery.zen.ping.unicast;
 
 import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;

From cbbc8790a562a9a20655c49bb07f55dfbbbd2545 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Tue, 5 Jul 2016 08:39:08 +0200
Subject: [PATCH 1233/1311] Remove redundant modifier

---
 .../org/elasticsearch/common/bytes/CompositeBytesReference.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/common/bytes/CompositeBytesReference.java b/core/src/main/java/org/elasticsearch/common/bytes/CompositeBytesReference.java
index 279275b92e8..3538cba869c 100644
--- a/core/src/main/java/org/elasticsearch/common/bytes/CompositeBytesReference.java
+++ b/core/src/main/java/org/elasticsearch/common/bytes/CompositeBytesReference.java
@@ -96,7 +96,7 @@ public final class CompositeBytesReference extends BytesReference {
         return new CompositeBytesReference(inSlice);
     }
 
-    private final int getOffsetIndex(int offset) {
+    private int getOffsetIndex(int offset) {
         final int i = Arrays.binarySearch(offsets, offset);
         return i < 0 ? (-(i + 1)) - 1 : i;
     }

From 84efaf4579b297bf8fb0e41dbe7b1ff19fc4e965 Mon Sep 17 00:00:00 2001
From: Colin Goodheart-Smithe <colings86@users.noreply.github.com>
Date: Tue, 5 Jul 2016 09:01:08 +0100
Subject: [PATCH 1234/1311] [TEST] fix histogram test when extended bounds
 overlaps data

Closes #19221
---
 .../test/java/org/elasticsearch/messy/tests/HistogramTests.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java
index a6349db0d07..b4260cf5530 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java
@@ -858,7 +858,7 @@ public class HistogramTests extends ESIntegTestCase {
 
         // constructing the newly expected bucket list
         int bucketsCount = (int) ((boundsMaxKey - boundsMinKey) / interval) + 1;
-        long[] extendedValueCounts = new long[bucketsCount];
+        long[] extendedValueCounts = new long[valueCounts.length + addedBucketsLeft + addedBucketsRight];
         System.arraycopy(valueCounts, 0, extendedValueCounts, addedBucketsLeft, valueCounts.length);
 
         SearchResponse response = null;

From a4ec0ac22f153750b80a9b1de9788c2fb80cd67e Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Tue, 5 Jul 2016 11:11:55 +0200
Subject: [PATCH 1235/1311] Upgrade to netty 3.10.6.Final (#19235)

---
 core/build.gradle                                             | 2 +-
 .../discovery/zen/ping/unicast/UnicastZenPingIT.java          | 4 ++--
 distribution/licenses/netty-3.10.5.Final.jar.sha1             | 1 -
 distribution/licenses/netty-3.10.6.Final.jar.sha1             | 1 +
 4 files changed, 4 insertions(+), 4 deletions(-)
 delete mode 100644 distribution/licenses/netty-3.10.5.Final.jar.sha1
 create mode 100644 distribution/licenses/netty-3.10.6.Final.jar.sha1

diff --git a/core/build.gradle b/core/build.gradle
index 88c5cc2a16d..22318801766 100644
--- a/core/build.gradle
+++ b/core/build.gradle
@@ -75,7 +75,7 @@ dependencies {
   compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
 
   // network stack
-  compile 'io.netty:netty:3.10.5.Final'
+  compile 'io.netty:netty:3.10.6.Final'
   // percentiles aggregation
   compile 'com.tdunning:t-digest:3.0'
   // precentil ranks aggregation
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java b/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java
index 06ac6aeb570..7715749fdf6 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/ping/unicast/UnicastZenPingIT.java
@@ -30,6 +30,7 @@ import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.discovery.zen.elect.ElectMasterService;
 import org.elasticsearch.discovery.zen.ping.PingContextProvider;
 import org.elasticsearch.discovery.zen.ping.ZenPing;
@@ -42,7 +43,6 @@ import org.elasticsearch.transport.TransportConnectionListener;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.TransportSettings;
 import org.elasticsearch.transport.netty.NettyTransport;
-import org.jboss.netty.util.internal.ConcurrentHashMap;
 
 import java.net.InetSocketAddress;
 import java.util.concurrent.ConcurrentMap;
@@ -208,7 +208,7 @@ public class UnicastZenPingIT extends ESTestCase {
         final TransportService transportService = new TransportService(settings, transport, threadPool);
         transportService.start();
         transportService.acceptIncomingRequests();
-        ConcurrentMap<TransportAddress, AtomicInteger> counters = new ConcurrentHashMap<>();
+        ConcurrentMap<TransportAddress, AtomicInteger> counters = ConcurrentCollections.newConcurrentMap();
         transportService.addConnectionListener(new TransportConnectionListener() {
             @Override
             public void onNodeConnected(DiscoveryNode node) {
diff --git a/distribution/licenses/netty-3.10.5.Final.jar.sha1 b/distribution/licenses/netty-3.10.5.Final.jar.sha1
deleted file mode 100644
index 6f190752e9e..00000000000
--- a/distribution/licenses/netty-3.10.5.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9ca7d55d246092bddd29b867706e2f6c7db701a0
diff --git a/distribution/licenses/netty-3.10.6.Final.jar.sha1 b/distribution/licenses/netty-3.10.6.Final.jar.sha1
new file mode 100644
index 00000000000..35872846658
--- /dev/null
+++ b/distribution/licenses/netty-3.10.6.Final.jar.sha1
@@ -0,0 +1 @@
+18ed04a0e502896552854926e908509db2987a00
\ No newline at end of file

From d08812d8394629ed1a7a0a75aafcf88e1774d39b Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Tue, 5 Jul 2016 11:23:26 +0200
Subject: [PATCH 1236/1311] [TEST] fix test to account for internal empyt
 reference optimization

---
 .../common/bytes/AbstractBytesReferenceTestCase.java  | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java b/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
index c998146ed74..9a9e2a4e615 100644
--- a/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
+++ b/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
@@ -462,9 +462,16 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
         // get a BytesRef from a slice
         int sliceOffset = randomIntBetween(0, pbr.length());
         int sliceLength = randomIntBetween(0, pbr.length() - sliceOffset);
+
         BytesRef sliceRef = pbr.slice(sliceOffset, sliceLength).toBytesRef();
-        // note that these are only true if we have <= than a page, otherwise offset/length are shifted
-        assertEquals(sliceOffset, sliceRef.offset);
+
+        if (sliceLength == 0 && sliceOffset != sliceRef.offset) {
+            // some impls optimize this to an empty instance then the offset will be 0
+            assertEquals(0, sliceRef.offset);
+        } else {
+            // note that these are only true if we have <= than a page, otherwise offset/length are shifted
+            assertEquals(sliceOffset, sliceRef.offset);
+        }
         assertEquals(sliceLength, sliceRef.length);
     }
 

From d7b478cd041295ae54bdc168fede8e9d07e89cbd Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 5 Jul 2016 07:10:15 -0400
Subject: [PATCH 1237/1311] Change Debian repository for Vagrant debian-8 box

This commit modifies the Debian repository to work around some upstream
reliability issues with the previous repository.
---
 Vagrantfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Vagrantfile b/Vagrantfile
index 4f8ee7164f6..5b51f612eb3 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -42,7 +42,7 @@ Vagrant.configure(2) do |config|
   # debian and it works fine.
   config.vm.define "debian-8" do |config|
     config.vm.box = "elastic/debian-8-x86_64"
-    deb_common config, 'echo deb http://http.debian.net/debian jessie-backports main > /etc/apt/sources.list.d/backports.list', 'backports'
+    deb_common config, 'echo deb http://cloudfront.debian.net/debian jessie-backports main > /etc/apt/sources.list.d/backports.list', 'backports'
   end
   config.vm.define "centos-6" do |config|
     config.vm.box = "elastic/centos-6-x86_64"

From d1727653dd158ca8715a5e5f0096ec8d7f50a0ab Mon Sep 17 00:00:00 2001
From: Christoph Wurm <cwurm@users.noreply.github.com>
Date: Tue, 5 Jul 2016 13:34:58 +0200
Subject: [PATCH 1238/1311] Update shrink-index.asciidoc

Fix half-finished sentence
---
 docs/reference/indices/shrink-index.asciidoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/reference/indices/shrink-index.asciidoc b/docs/reference/indices/shrink-index.asciidoc
index 39c5134e23c..2975ef903f3 100644
--- a/docs/reference/indices/shrink-index.asciidoc
+++ b/docs/reference/indices/shrink-index.asciidoc
@@ -76,8 +76,8 @@ Indices can only be shrunk if they satisfy the following requirements:
 * The index must have more primary shards than the target index.
 
 * The number of primary shards in the target index must be a factor of the
-  number of primary shards in the source index. must have more primary shards
-   than the target index.
+  number of primary shards in the source index. The source index must have
+  more primary shards than the target index.
 
 * The index must not contain more than `2,147,483,519` documents in total
   across all shards that will be shrunk into a single shard on the target index

From 768beea6c7d904e12ba91cb2abfb5cf2cc5cea05 Mon Sep 17 00:00:00 2001
From: Christoph Wurm <cwurm@users.noreply.github.com>
Date: Tue, 5 Jul 2016 13:49:25 +0200
Subject: [PATCH 1239/1311] Update refresh.asciidoc

Fix grammar and example
---
 docs/reference/docs/refresh.asciidoc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/reference/docs/refresh.asciidoc b/docs/reference/docs/refresh.asciidoc
index dd829e19bc3..6bf6d0885bc 100644
--- a/docs/reference/docs/refresh.asciidoc
+++ b/docs/reference/docs/refresh.asciidoc
@@ -17,7 +17,7 @@ indexing and a search standpoint.
 
 Wait for the changes made by the request to be made visible by a refresh before
 replying. This doesn't force an immediate refresh, rather, it waits for a
-refresh happen. Elasticsearch automatically refreshes shards that have changed
+refresh to happen. Elasticsearch automatically refreshes shards that have changed
 every `index.refresh_interval` which defaults to one second. That setting is
 <<dynamic-index-settings,dynamic>>. The <<indices-refresh>> API will also
 cause the request to return, as will setting `refresh` to `true` on any of the
@@ -36,7 +36,7 @@ use `refresh=false`, or, because that is the default, just leave the `refresh`
 parameter out of the URL. That is the simplest and fastest choice.
 
 If you absolutely must have the changes made by a request visible synchronously
-with the request then you must get to pick between putting more load on
+with the request then you must pick between putting more load on
 Elasticsearch (`true`) and waiting longer for the response (`wait_for`). Here
 are a few points that should inform that decision:
 
@@ -97,7 +97,7 @@ search:
 --------------------------------------------------
 PUT /test/test/3
 {"test": "test"}
-PUT /test/test/4?refresh=true
+PUT /test/test/4?refresh=false
 {"test": "test"}
 --------------------------------------------------
 // CONSOLE

From 4b0d317e63ea5f02bb3fde5eb976b961f553dc96 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Mon, 4 Jul 2016 14:55:51 +0200
Subject: [PATCH 1240/1311] Bump version to 5.0.0-alpha5.

---
 buildSrc/version.properties                       | 2 +-
 core/src/main/java/org/elasticsearch/Version.java | 6 +++++-
 qa/backwards-5.0/build.gradle                     | 2 +-
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index 7e23db81bc4..7565488d4ab 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,4 +1,4 @@
-elasticsearch     = 5.0.0-alpha4
+elasticsearch     = 5.0.0-alpha5
 lucene            = 6.1.0
 
 # optional dependencies
diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java
index 7e2ffd1dfa0..34982b679c7 100644
--- a/core/src/main/java/org/elasticsearch/Version.java
+++ b/core/src/main/java/org/elasticsearch/Version.java
@@ -77,7 +77,9 @@ public class Version {
     public static final Version V_5_0_0_alpha3 = new Version(V_5_0_0_alpha3_ID, org.apache.lucene.util.Version.LUCENE_6_0_0);
     public static final int V_5_0_0_alpha4_ID = 5000004;
     public static final Version V_5_0_0_alpha4 = new Version(V_5_0_0_alpha4_ID, org.apache.lucene.util.Version.LUCENE_6_1_0);
-    public static final Version CURRENT = V_5_0_0_alpha4;
+    public static final int V_5_0_0_alpha5_ID = 5000005;
+    public static final Version V_5_0_0_alpha5 = new Version(V_5_0_0_alpha5_ID, org.apache.lucene.util.Version.LUCENE_6_1_0);
+    public static final Version CURRENT = V_5_0_0_alpha5;
 
     static {
         assert CURRENT.luceneVersion.equals(org.apache.lucene.util.Version.LATEST) : "Version must be upgraded to ["
@@ -90,6 +92,8 @@ public class Version {
 
     public static Version fromId(int id) {
         switch (id) {
+            case V_5_0_0_alpha5_ID:
+                return V_5_0_0_alpha5;
             case V_5_0_0_alpha4_ID:
                 return V_5_0_0_alpha4;
             case V_5_0_0_alpha3_ID:
diff --git a/qa/backwards-5.0/build.gradle b/qa/backwards-5.0/build.gradle
index fbce12f8126..657a6b7c078 100644
--- a/qa/backwards-5.0/build.gradle
+++ b/qa/backwards-5.0/build.gradle
@@ -18,6 +18,6 @@ integTest {
   cluster {
     numNodes = 2
     numBwcNodes = 1
-    bwcVersion = "5.0.0-alpha4-SNAPSHOT" // this is the same as the current version until we released the first RC
+    bwcVersion = "5.0.0-alpha5-SNAPSHOT" // this is the same as the current version until we released the first RC
   }
 }

From dcf6a967256c4051c82dec61baaddbb2256c47bb Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Fri, 1 Jul 2016 18:13:29 +0200
Subject: [PATCH 1241/1311] Add doc values support to the _size field in the
 mapper-size plugin

This change activates the doc_values on the _size field for indices created after 5.0.0-alpha4.
It also adds a note in the breaking changes that explain the situation and how to get around it.

Closes #18334
---
 .../resources/checkstyle_suppressions.xml     |   4 -
 docs/plugins/mapper-size.asciidoc             |  23 ++-
 .../migration/migrate_5_0/plugins.asciidoc    |   8 ++
 .../index/mapper/size/SizeFieldMapper.java    |  54 +++++--
 .../size/SizeFieldMapperUpgradeTests.java     |   3 +-
 .../index/mapper/size/SizeMappingIT.java      |  25 ++--
 .../index/mapper/size/SizeMappingTests.java   | 133 +++++++++++-------
 7 files changed, 167 insertions(+), 83 deletions(-)

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index c73545c8a7c..7aa77a26458 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -1200,10 +1200,6 @@
   <suppress files="plugins[/\\]mapper-murmur3[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]murmur3[/\\]Murmur3FieldMapper.java" checks="LineLength" />
   <suppress files="plugins[/\\]mapper-murmur3[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]murmur3[/\\]Murmur3FieldMapperTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]mapper-murmur3[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]murmur3[/\\]Murmur3FieldMapperUpgradeTests.java" checks="LineLength" />
-  <suppress files="plugins[/\\]mapper-size[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]size[/\\]SizeFieldMapper.java" checks="LineLength" />
-  <suppress files="plugins[/\\]mapper-size[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]size[/\\]SizeFieldMapperUpgradeTests.java" checks="LineLength" />
-  <suppress files="plugins[/\\]mapper-size[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]size[/\\]SizeMappingIT.java" checks="LineLength" />
-  <suppress files="plugins[/\\]mapper-size[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]size[/\\]SizeMappingTests.java" checks="LineLength" />
   <suppress files="plugins[/\\]repository-azure[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]blobstore[/\\]AzureBlobContainer.java" checks="LineLength" />
   <suppress files="plugins[/\\]repository-azure[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]blobstore[/\\]AzureBlobStore.java" checks="LineLength" />
   <suppress files="plugins[/\\]repository-azure[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]cloud[/\\]azure[/\\]storage[/\\]AzureStorageServiceImpl.java" checks="LineLength" />
diff --git a/docs/plugins/mapper-size.asciidoc b/docs/plugins/mapper-size.asciidoc
index 800a640890a..7707d2544c4 100644
--- a/docs/plugins/mapper-size.asciidoc
+++ b/docs/plugins/mapper-size.asciidoc
@@ -52,7 +52,8 @@ PUT my_index
 --------------------------
 // CONSOLE
 
-The value of the `_size` field is accessible in queries:
+The value of the `_size` field is accessible in queries, aggregations, scripts,
+and when sorting:
 
 [source,js]
 --------------------------
@@ -75,6 +76,26 @@ GET my_index/_search
         "gt": 10
       }
     }
+  },
+  "aggs": {
+    "sizes": {
+      "terms": {
+        "field": "_size", <2>
+        "size": 10
+      }
+    }
+  },
+  "sort": [
+    {
+      "_size": { <3>
+        "order": "desc"
+      }
+    }
+  ],
+  "script_fields": {
+    "size": {
+      "script": "doc['_size']"  <4>
+    }
   }
 }
 --------------------------
diff --git a/docs/reference/migration/migrate_5_0/plugins.asciidoc b/docs/reference/migration/migrate_5_0/plugins.asciidoc
index e1ff497a8f3..79583c6b925 100644
--- a/docs/reference/migration/migrate_5_0/plugins.asciidoc
+++ b/docs/reference/migration/migrate_5_0/plugins.asciidoc
@@ -140,3 +140,11 @@ remove their `onModule(ActionModule)` implementation.
 
 Plugins that register custom `RestHandler`s should implement `ActionPlugin` and
 remove their `onModule(NetworkModule)` implemnetation.
+
+==== Mapper-Size plugin
+
+The metadata field `_size` is not accessible in aggregations, scripts and when
+sorting for indices created in 2.x even if the index has been upgraded using the <<indices-upgrade,`upgrade`>> API.
+If these features are needed in your application it is required to reindex the data with Elasticsearch 5.x.
+The easiest way to reindex old indices is to use the `reindex` API, or the reindex UI provided by
+the <<migration-plugin,Migration Plugin>>.
diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java
index 9c07c7b3eb3..c9c8972c62d 100644
--- a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java
+++ b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java
@@ -42,15 +42,15 @@ import java.util.Map;
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.lenientNodeBooleanValue;
 
 public class SizeFieldMapper extends MetadataFieldMapper {
-
     public static final String NAME = "_size";
-    public static final String CONTENT_TYPE = "_size";
 
     public static class Defaults  {
         public static final EnabledAttributeMapper ENABLED_STATE = EnabledAttributeMapper.UNSET_DISABLED;
 
-        public static final MappedFieldType SIZE_FIELD_TYPE = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER);
-        public static final MappedFieldType LEGACY_SIZE_FIELD_TYPE = LegacyIntegerFieldMapper.Defaults.FIELD_TYPE.clone();
+        public static final MappedFieldType SIZE_FIELD_TYPE =
+            new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.INTEGER);
+        public static final MappedFieldType LEGACY_SIZE_FIELD_TYPE =
+            LegacyIntegerFieldMapper.Defaults.FIELD_TYPE.clone();
 
         static {
             SIZE_FIELD_TYPE.setStored(true);
@@ -68,14 +68,31 @@ public class SizeFieldMapper extends MetadataFieldMapper {
         }
     }
 
+    private static MappedFieldType defaultFieldType(Version indexCreated) {
+        MappedFieldType defaultFieldType;
+        if (indexCreated.before(Version.V_5_0_0_alpha2)) {
+            defaultFieldType = Defaults.LEGACY_SIZE_FIELD_TYPE.clone();
+            // doc_values are disabled for bwc with indices created before V_5_0_0_alpha4
+            defaultFieldType.setHasDocValues(false);
+        } else {
+            defaultFieldType = Defaults.SIZE_FIELD_TYPE.clone();
+            if (indexCreated.onOrBefore(Version.V_5_0_0_alpha4)) {
+                // doc_values are disabled for bwc with indices created before V_5_0_0_alpha4
+                defaultFieldType.setHasDocValues(false);
+            } else {
+                defaultFieldType.setHasDocValues(true);
+            }
+        }
+        return defaultFieldType;
+    }
+
     public static class Builder extends MetadataFieldMapper.Builder<Builder, SizeFieldMapper> {
 
         protected EnabledAttributeMapper enabledState = EnabledAttributeMapper.UNSET_DISABLED;
 
         private Builder(MappedFieldType existing, Version indexCreated) {
-            super(NAME, existing == null
-                    ? indexCreated.before(Version.V_5_0_0_alpha2) ? Defaults.LEGACY_SIZE_FIELD_TYPE : Defaults.SIZE_FIELD_TYPE
-                    : existing, Defaults.LEGACY_SIZE_FIELD_TYPE);
+            super(NAME, existing == null ? defaultFieldType(indexCreated) : existing.clone(),
+                defaultFieldType(indexCreated));
             builder = this;
         }
 
@@ -87,21 +104,27 @@ public class SizeFieldMapper extends MetadataFieldMapper {
         @Override
         public SizeFieldMapper build(BuilderContext context) {
             setupFieldType(context);
-            fieldType.setHasDocValues(false);
+            if (context.indexCreatedVersion().onOrBefore(Version.V_5_0_0_alpha4)) {
+                // Make sure that the doc_values are disabled on indices created before V_5_0_0_alpha4
+                fieldType.setHasDocValues(false);
+            }
             return new SizeFieldMapper(enabledState, fieldType, context.indexSettings());
         }
     }
 
     public static class TypeParser implements MetadataFieldMapper.TypeParser {
         @Override
-        public MetadataFieldMapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
-            Builder builder = new Builder(parserContext.mapperService().fullName(NAME), parserContext.indexVersionCreated());
+        public MetadataFieldMapper.Builder<?, ?> parse(String name, Map<String, Object> node,
+                                                       ParserContext parserContext) throws MapperParsingException {
+            Builder builder = new Builder(parserContext.mapperService().fullName(NAME),
+                parserContext.indexVersionCreated());
             for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
                 Map.Entry<String, Object> entry = iterator.next();
                 String fieldName = entry.getKey();
                 Object fieldNode = entry.getValue();
                 if (fieldName.equals("enabled")) {
-                    builder.enabled(lenientNodeBooleanValue(fieldNode) ? EnabledAttributeMapper.ENABLED : EnabledAttributeMapper.DISABLED);
+                    builder.enabled(lenientNodeBooleanValue(fieldNode) ?
+                        EnabledAttributeMapper.ENABLED : EnabledAttributeMapper.DISABLED);
                     iterator.remove();
                 }
             }
@@ -116,14 +139,15 @@ public class SizeFieldMapper extends MetadataFieldMapper {
 
     private EnabledAttributeMapper enabledState;
 
-    private SizeFieldMapper(Settings indexSettings, MappedFieldType mappedFieldType) {
-        this(Defaults.ENABLED_STATE, mappedFieldType == null ? Defaults.LEGACY_SIZE_FIELD_TYPE : mappedFieldType, indexSettings);
+    private SizeFieldMapper(Settings indexSettings, MappedFieldType existing) {
+        this(Defaults.ENABLED_STATE,
+            existing == null ? defaultFieldType(Version.indexCreated(indexSettings)) : existing.clone(),
+            indexSettings);
     }
 
     private SizeFieldMapper(EnabledAttributeMapper enabled, MappedFieldType fieldType, Settings indexSettings) {
-        super(NAME, fieldType, Defaults.LEGACY_SIZE_FIELD_TYPE, indexSettings);
+        super(NAME, fieldType, defaultFieldType(Version.indexCreated(indexSettings)), indexSettings);
         this.enabledState = enabled;
-
     }
 
     @Override
diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java
index 956645aebd2..7cbce102c57 100644
--- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java
+++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeFieldMapperUpgradeTests.java
@@ -67,7 +67,8 @@ public class SizeFieldMapperUpgradeTests extends ESIntegTestCase {
         Settings settings = Settings.builder()
                 .put(Environment.PATH_DATA_SETTING.getKey(), dataPath)
                 .build();
-        final String node = internalCluster().startDataOnlyNode(settings); // workaround for dangling index loading issue when node is master
+        // workaround for dangling index loading issue when node is master
+        final String node = internalCluster().startDataOnlyNode(settings);
         Path[] nodePaths = internalCluster().getInstance(NodeEnvironment.class, node).nodeDataPaths();
         assertEquals(1, nodePaths.length);
         dataPath = nodePaths[0].resolve(NodeEnvironment.INDICES_FOLDER);
diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java
index b6c341c0601..279c5c96091 100644
--- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java
+++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java
@@ -49,15 +49,19 @@ public class SizeMappingIT extends ESIntegTestCase {
         String index = "foo";
         String type = "mytype";
 
-        XContentBuilder builder = jsonBuilder().startObject().startObject("_size").field("enabled", true).endObject().endObject();
+        XContentBuilder builder =
+            jsonBuilder().startObject().startObject("_size").field("enabled", true).endObject().endObject();
         assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
 
         // check mapping again
         assertSizeMappingEnabled(index, type, true);
 
         // update some field in the mapping
-        XContentBuilder updateMappingBuilder = jsonBuilder().startObject().startObject("properties").startObject("otherField").field("type", "text").endObject().endObject().endObject();
-        PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping(index).setType(type).setSource(updateMappingBuilder).get();
+        XContentBuilder updateMappingBuilder =
+            jsonBuilder().startObject().startObject("properties").startObject("otherField").field("type", "text")
+                .endObject().endObject().endObject();
+        PutMappingResponse putMappingResponse =
+            client().admin().indices().preparePutMapping(index).setType(type).setSource(updateMappingBuilder).get();
         assertAcked(putMappingResponse);
 
         // make sure size field is still in mapping
@@ -68,15 +72,18 @@ public class SizeMappingIT extends ESIntegTestCase {
         String index = "foo";
         String type = "mytype";
 
-        XContentBuilder builder = jsonBuilder().startObject().startObject("_size").field("enabled", true).endObject().endObject();
+        XContentBuilder builder =
+            jsonBuilder().startObject().startObject("_size").field("enabled", true).endObject().endObject();
         assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder));
 
         // check mapping again
         assertSizeMappingEnabled(index, type, true);
 
         // update some field in the mapping
-        XContentBuilder updateMappingBuilder = jsonBuilder().startObject().startObject("_size").field("enabled", false).endObject().endObject();
-        PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping(index).setType(type).setSource(updateMappingBuilder).get();
+        XContentBuilder updateMappingBuilder =
+            jsonBuilder().startObject().startObject("_size").field("enabled", false).endObject().endObject();
+        PutMappingResponse putMappingResponse =
+            client().admin().indices().preparePutMapping(index).setType(type).setSource(updateMappingBuilder).get();
         assertAcked(putMappingResponse);
 
         // make sure size field is still in mapping
@@ -84,8 +91,10 @@ public class SizeMappingIT extends ESIntegTestCase {
     }
 
     private void assertSizeMappingEnabled(String index, String type, boolean enabled) throws IOException {
-        String errMsg = String.format(Locale.ROOT, "Expected size field mapping to be " + (enabled ? "enabled" : "disabled") + " for %s/%s", index, type);
-        GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(index).addTypes(type).get();
+        String errMsg = String.format(Locale.ROOT,
+            "Expected size field mapping to be " + (enabled ? "enabled" : "disabled") + " for %s/%s", index, type);
+        GetMappingsResponse getMappingsResponse =
+            client().admin().indices().prepareGetMappings(index).addTypes(type).get();
         Map<String, Object> mappingSource = getMappingsResponse.getMappings().get(index).get(type).getSourceAsMap();
         assertThat(errMsg, mappingSource, hasKey("_size"));
         String sizeAsString = mappingSource.get("_size").toString();
diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java
index b0802a955df..8cc01aba4bb 100644
--- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java
+++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java
@@ -19,58 +19,48 @@
 
 package org.elasticsearch.index.mapper.size;
 
-import java.util.Collections;
-import java.util.Map;
+import java.util.Collection;
 
+import org.elasticsearch.Version;
+import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.DocumentMapperParser;
-import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.SourceToParse;
-import org.elasticsearch.indices.IndicesModule;
-import org.elasticsearch.plugins.MapperPlugin;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.core.LegacyNumberFieldMapper;
+import org.elasticsearch.index.mapper.core.NumberFieldMapper;
+import org.elasticsearch.plugin.mapper.MapperSizePlugin;
+import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
-import org.junit.Before;
-
+import org.elasticsearch.test.InternalSettingsPlugin;
 
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.instanceOf;
 
 import org.apache.lucene.index.IndexableField;
 
 public class SizeMappingTests extends ESSingleNodeTestCase {
-
-    IndexService indexService;
-    MapperService mapperService;
-    DocumentMapperParser parser;
-
-    @Before
-    public void before() {
-        indexService = createIndex("test");
-        IndicesModule indices = newTestIndicesModule(Collections.emptyMap(),
-            Collections.singletonMap(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser())
-        );
-        mapperService = new MapperService(indexService.getIndexSettings(), indexService.analysisService(), indexService.similarityService(), indices.getMapperRegistry(), indexService::newQueryShardContext);
-        parser = mapperService.documentMapperParser();
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(MapperSizePlugin.class, InternalSettingsPlugin.class);
     }
 
     public void testSizeEnabled() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("_size").field("enabled", true).endObject()
-                .endObject().endObject().string();
-        DocumentMapper docMapper = parser.parse("type", new CompressedXContent(mapping));
+        IndexService service = createIndex("test", Settings.EMPTY, "type", "_size", "enabled=true");
+        DocumentMapper docMapper = service.mapperService().documentMapper("type");
 
         BytesReference source = XContentFactory.jsonBuilder()
-                .startObject()
-                .field("field", "value")
-                .endObject()
-                .bytes();
+            .startObject()
+            .field("field", "value")
+            .endObject()
+            .bytes();
         ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source));
 
         boolean stored = false;
@@ -84,47 +74,82 @@ public class SizeMappingTests extends ESSingleNodeTestCase {
     }
 
     public void testSizeDisabled() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("_size").field("enabled", false).endObject()
-                .endObject().endObject().string();
-        DocumentMapper docMapper = parser.parse("type", new CompressedXContent(mapping));
+        IndexService service = createIndex("test", Settings.EMPTY, "type", "_size", "enabled=false");
+        DocumentMapper docMapper = service.mapperService().documentMapper("type");
 
         BytesReference source = XContentFactory.jsonBuilder()
-                .startObject()
-                .field("field", "value")
-                .endObject()
-                .bytes();
+            .startObject()
+            .field("field", "value")
+            .endObject()
+            .bytes();
         ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source));
 
         assertThat(doc.rootDoc().getField("_size"), nullValue());
     }
 
     public void testSizeNotSet() throws Exception {
-        String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .endObject().endObject().string();
-        DocumentMapper docMapper = parser.parse("type", new CompressedXContent(mapping));
+        IndexService service = createIndex("test", Settings.EMPTY, "type");
+        DocumentMapper docMapper = service.mapperService().documentMapper("type");
 
         BytesReference source = XContentFactory.jsonBuilder()
-                .startObject()
-                .field("field", "value")
-                .endObject()
-                .bytes();
+            .startObject()
+            .field("field", "value")
+            .endObject()
+            .bytes();
         ParsedDocument doc = docMapper.parse(SourceToParse.source("test", "type", "1", source));
 
         assertThat(doc.rootDoc().getField("_size"), nullValue());
     }
 
     public void testThatDisablingWorksWhenMerging() throws Exception {
-        String enabledMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("_size").field("enabled", true).endObject()
-                .endObject().endObject().string();
-        DocumentMapper enabledMapper = mapperService.merge("type", new CompressedXContent(enabledMapping), MapperService.MergeReason.MAPPING_UPDATE, false);
+        IndexService service = createIndex("test", Settings.EMPTY, "type", "_size", "enabled=true");
+        DocumentMapper docMapper = service.mapperService().documentMapper("type");
+        assertThat(docMapper.metadataMapper(SizeFieldMapper.class).enabled(), is(true));
 
         String disabledMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
-                .startObject("_size").field("enabled", false).endObject()
-                .endObject().endObject().string();
-        DocumentMapper disabledMapper = mapperService.merge("type", new CompressedXContent(disabledMapping), MapperService.MergeReason.MAPPING_UPDATE, false);
+            .startObject("_size").field("enabled", false).endObject()
+            .endObject().endObject().string();
+        docMapper = service.mapperService().merge("type", new CompressedXContent(disabledMapping),
+            MapperService.MergeReason.MAPPING_UPDATE, false);
 
-        assertThat(disabledMapper.metadataMapper(SizeFieldMapper.class).enabled(), is(false));
+        assertThat(docMapper.metadataMapper(SizeFieldMapper.class).enabled(), is(false));
     }
+
+    public void testBWCMapper() throws Exception {
+        {
+            // IntPoint && docvalues=true for V_5_0_0_alpha5
+            IndexService service = createIndex("foo", Settings.EMPTY, "bar", "_size", "enabled=true");
+            DocumentMapper docMapper = service.mapperService().documentMapper("bar");
+            SizeFieldMapper mapper = docMapper.metadataMapper(SizeFieldMapper.class);
+            assertThat(mapper.enabled(), is(true));
+            MappedFieldType ft = mapper.fieldType();
+            assertThat(ft.hasDocValues(), is(true));
+            assertThat(mapper.fieldType(), instanceOf(NumberFieldMapper.NumberFieldType.class));
+        }
+
+        {
+            // IntPoint with docvalues=false if version > V_5_0_0_alpha2 && version < V_5_0_0_beta1
+            IndexService service = createIndex("foo2",
+                Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_5_0_0_alpha4.id).build(),
+                "bar", "_size", "enabled=true");
+            DocumentMapper docMapper = service.mapperService().documentMapper("bar");
+            SizeFieldMapper mapper = docMapper.metadataMapper(SizeFieldMapper.class);
+            assertThat(mapper.enabled(), is(true));
+            assertThat(mapper.fieldType().hasDocValues(), is(false));
+            assertThat(mapper.fieldType(), instanceOf(NumberFieldMapper.NumberFieldType.class));
+        }
+
+        {
+            // LegacyIntField with docvalues=false if version < V_5_0_0_alpha2
+            IndexService service = createIndex("foo3",
+                Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_5_0_0_alpha1.id).build(),
+                "bar", "_size", "enabled=true");
+            DocumentMapper docMapper = service.mapperService().documentMapper("bar");
+            SizeFieldMapper mapper = docMapper.metadataMapper(SizeFieldMapper.class);
+            assertThat(mapper.enabled(), is(true));
+            assertThat(mapper.fieldType().hasDocValues(), is(false));
+            assertThat(mapper.fieldType(), instanceOf(LegacyNumberFieldMapper.NumberFieldType.class));
+        }
+    }
+
 }

From 3269c84c7a7c4bcc5972468e18388ab7ca1692b2 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 4 Jul 2016 22:26:27 -0400
Subject: [PATCH 1242/1311] Remote BucketStreams

It isn't used.
---
 .../bucket/BucketStreamContext.java           | 77 -------------------
 .../aggregations/bucket/BucketStreams.java    | 67 ----------------
 .../bucket/geogrid/InternalGeoHashGrid.java   | 21 +----
 .../bucket/histogram/InternalHistogram.java   | 24 ------
 .../bucket/range/InternalBinaryRange.java     | 30 ++------
 .../bucket/range/InternalRange.java           | 20 -----
 .../bucket/range/date/InternalDateRange.java  | 20 -----
 .../geodistance/InternalGeoDistance.java      | 20 -----
 .../significant/SignificantLongTerms.java     | 26 -------
 .../significant/SignificantStringTerms.java   | 24 ------
 .../bucket/terms/DoubleTerms.java             | 23 ------
 .../aggregations/bucket/terms/LongTerms.java  | 23 ------
 .../bucket/terms/StringTerms.java             | 24 ------
 13 files changed, 6 insertions(+), 393 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketStreamContext.java
 delete mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketStreams.java

diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketStreamContext.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketStreamContext.java
deleted file mode 100644
index 5916baa3f8e..00000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketStreamContext.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.aggregations.bucket;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.search.DocValueFormat;
-
-import java.io.IOException;
-import java.util.Map;
-
-public class BucketStreamContext implements Streamable {
-
-    private DocValueFormat format;
-    private boolean keyed;
-    private Map<String, Object> attributes;
-
-    public BucketStreamContext() {
-    }
-
-    public void format(DocValueFormat format) {
-        this.format = format;
-    }
-
-    public DocValueFormat format() {
-        return format;
-    }
-
-    public void keyed(boolean keyed) {
-        this.keyed = keyed;
-    }
-
-    public boolean keyed() {
-        return keyed;
-    }
-
-    public void attributes(Map<String, Object> attributes) {
-        this.attributes = attributes;
-    }
-
-    public Map<String, Object> attributes() {
-        return attributes;
-    }
-
-    @Override
-    public void readFrom(StreamInput in) throws IOException {
-        format = in.readNamedWriteable(DocValueFormat.class);
-        keyed = in.readBoolean();
-        attributes = in.readMap();
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeNamedWriteable(format);
-        out.writeBoolean(keyed);
-        out.writeMap(attributes);
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketStreams.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketStreams.java
deleted file mode 100644
index 2025d62ef6e..00000000000
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/BucketStreams.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.search.aggregations.bucket;
-
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.io.stream.StreamInput;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.unmodifiableMap;
-
-public class BucketStreams {
-    private static Map<BytesReference, Stream> streams = emptyMap();
-
-    /**
-     * A stream that knows how to read a bucket from the input.
-     */
-    public interface Stream<B extends MultiBucketsAggregation.Bucket> {
-        B readResult(StreamInput in, BucketStreamContext context) throws IOException;
-        BucketStreamContext getBucketStreamContext(B bucket);
-    }
-
-    /**
-     * Registers the given stream and associate it with the given types.
-     *
-     * @param stream    The streams to register
-     * @param types     The types associated with the streams
-     */
-    public static synchronized void registerStream(Stream stream, BytesReference... types) {
-        Map<BytesReference, Stream> newStreams = new HashMap<>(streams);
-        for (BytesReference type : types) {
-            newStreams.put(type, stream);
-        }
-        streams = unmodifiableMap(newStreams);
-    }
-
-    /**
-     * Returns the stream that is registered for the given type
-     *
-     * @param   type The given type
-     * @return  The associated stream
-     */
-    public static Stream stream(BytesReference type) {
-        return streams.get(type);
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java
index 0d056e1efbf..493461968df 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java
@@ -18,8 +18,8 @@
  */
 package org.elasticsearch.search.aggregations.bucket.geogrid;
 
-import org.elasticsearch.common.geo.GeoHashUtils;
 import org.apache.lucene.util.PriorityQueue;
+import org.elasticsearch.common.geo.GeoHashUtils;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -30,8 +30,6 @@ import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
-import org.elasticsearch.search.aggregations.bucket.BucketStreamContext;
-import org.elasticsearch.search.aggregations.bucket.BucketStreams;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
@@ -60,25 +58,8 @@ public class InternalGeoHashGrid extends InternalMultiBucketAggregation<Internal
         }
     };
 
-
-    public static final BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
-        @Override
-        public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
-            Bucket bucket = new Bucket();
-            bucket.readFrom(in);
-            return bucket;
-        }
-
-        @Override
-        public BucketStreamContext getBucketStreamContext(Bucket bucket) {
-            BucketStreamContext context = new BucketStreamContext();
-            return context;
-        }
-    };
-
     public static void registerStreams() {
         AggregationStreams.registerStream(STREAM, TYPE.stream());
-        BucketStreams.registerStream(BUCKET_STREAM, TYPE.stream());
     }
 
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java
index 2ea2eaecde8..323de70fc7f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalHistogram.java
@@ -31,8 +31,6 @@ import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
-import org.elasticsearch.search.aggregations.bucket.BucketStreamContext;
-import org.elasticsearch.search.aggregations.bucket.BucketStreams;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.ValueType;
 
@@ -64,31 +62,9 @@ public class InternalHistogram<B extends InternalHistogram.Bucket> extends Inter
         }
     };
 
-    private static final BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
-        @Override
-        public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
-            Factory<?> factory = (Factory<?>) context.attributes().get("factory");
-            if (factory == null) {
-                throw new IllegalStateException("No factory found for histogram buckets");
-            }
-            Bucket histogram = new Bucket(context.keyed(), context.format(), factory);
-            histogram.readFrom(in);
-            return histogram;
-        }
-
-        @Override
-        public BucketStreamContext getBucketStreamContext(Bucket bucket) {
-            BucketStreamContext context = new BucketStreamContext();
-            context.format(bucket.format);
-            context.keyed(bucket.keyed);
-            return context;
-        }
-    };
-
     public static void registerStream() {
 
         AggregationStreams.registerStream(STREAM, TYPE.stream());
-        BucketStreams.registerStream(BUCKET_STREAM, TYPE.stream());
     }
 
     public static class Bucket extends InternalMultiBucketAggregation.InternalBucket implements Histogram.Bucket {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
index 568033d2fed..bb8840be7fa 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalBinaryRange.java
@@ -18,11 +18,6 @@
  */
 package org.elasticsearch.search.aggregations.bucket.range;
 
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -33,10 +28,13 @@ import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
-import org.elasticsearch.search.aggregations.bucket.BucketStreamContext;
-import org.elasticsearch.search.aggregations.bucket.BucketStreams;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
 /** A range aggregation for data that is encoded in doc values using a binary representation. */
 public final class InternalBinaryRange
         extends InternalMultiBucketAggregation<InternalBinaryRange, InternalBinaryRange.Bucket>
@@ -53,26 +51,8 @@ public final class InternalBinaryRange
         }
     };
 
-    private static final BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
-        @Override
-        public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
-            Bucket bucket = new Bucket(context.format(), context.keyed());
-            bucket.readFrom(in);
-            return bucket;
-        }
-
-        @Override
-        public BucketStreamContext getBucketStreamContext(Bucket bucket) {
-            BucketStreamContext context = new BucketStreamContext();
-            context.format(bucket.format);
-            context.keyed(bucket.keyed);
-            return context;
-        }
-    };
-
     public static void registerStream() {
         AggregationStreams.registerStream(STREAM, TYPE.stream());
-        BucketStreams.registerStream(BUCKET_STREAM, TYPE.stream());
     }
 
     public static class Bucket extends InternalMultiBucketAggregation.InternalBucket implements Range.Bucket {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
index bbdd85b969a..788246e786b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/InternalRange.java
@@ -27,8 +27,6 @@ import org.elasticsearch.search.aggregations.Aggregations;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
-import org.elasticsearch.search.aggregations.bucket.BucketStreamContext;
-import org.elasticsearch.search.aggregations.bucket.BucketStreams;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.ValueType;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
@@ -57,26 +55,8 @@ public class InternalRange<B extends InternalRange.Bucket, R extends InternalRan
         }
     };
 
-    private static final BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
-        @Override
-        public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
-            Bucket buckets = new Bucket(context.keyed(), context.format());
-            buckets.readFrom(in);
-            return buckets;
-        }
-
-        @Override
-        public BucketStreamContext getBucketStreamContext(Bucket bucket) {
-            BucketStreamContext context = new BucketStreamContext();
-            context.format(bucket.format);
-            context.keyed(bucket.keyed);
-            return context;
-        }
-    };
-
     public static void registerStream() {
         AggregationStreams.registerStream(STREAM, TYPE.stream());
-        BucketStreams.registerStream(BUCKET_STREAM, TYPE.stream());
     }
 
     public static class Bucket extends InternalMultiBucketAggregation.InternalBucket implements Range.Bucket {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/InternalDateRange.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/InternalDateRange.java
index 550e49797df..559da45aa75 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/InternalDateRange.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/date/InternalDateRange.java
@@ -23,8 +23,6 @@ import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.aggregations.bucket.BucketStreamContext;
-import org.elasticsearch.search.aggregations.bucket.BucketStreams;
 import org.elasticsearch.search.aggregations.bucket.range.InternalRange;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.ValueType;
@@ -51,26 +49,8 @@ public class InternalDateRange extends InternalRange<InternalDateRange.Bucket, I
         }
     };
 
-    private static final BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
-        @Override
-        public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
-            Bucket buckets = new Bucket(context.keyed(), context.format());
-            buckets.readFrom(in);
-            return buckets;
-        }
-
-        @Override
-        public BucketStreamContext getBucketStreamContext(Bucket bucket) {
-            BucketStreamContext context = new BucketStreamContext();
-            context.format(bucket.format());
-            context.keyed(bucket.keyed());
-            return context;
-        }
-    };
-
     public static void registerStream() {
         AggregationStreams.registerStream(STREAM, TYPE.stream());
-        BucketStreams.registerStream(BUCKET_STREAM, TYPE.stream());
     }
 
     public static final Factory FACTORY = new Factory();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/InternalGeoDistance.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/InternalGeoDistance.java
index 54244ca65bd..2d5e63ae15d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/InternalGeoDistance.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/geodistance/InternalGeoDistance.java
@@ -23,8 +23,6 @@ import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.aggregations.bucket.BucketStreamContext;
-import org.elasticsearch.search.aggregations.bucket.BucketStreams;
 import org.elasticsearch.search.aggregations.bucket.range.InternalRange;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.ValueType;
@@ -50,26 +48,8 @@ public class InternalGeoDistance extends InternalRange<InternalGeoDistance.Bucke
         }
     };
 
-    private static final BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
-        @Override
-        public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
-            Bucket buckets = new Bucket(context.keyed());
-            buckets.readFrom(in);
-            return buckets;
-        }
-
-        @Override
-        public BucketStreamContext getBucketStreamContext(Bucket bucket) {
-            BucketStreamContext context = new BucketStreamContext();
-            context.format(DocValueFormat.RAW);
-            context.keyed(bucket.keyed());
-            return context;
-        }
-    };
-
     public static void registerStream() {
         AggregationStreams.registerStream(STREAM, TYPE.stream());
-        BucketStreams.registerStream(BUCKET_STREAM, TYPE.stream());
     }
 
     public static final Factory FACTORY = new Factory();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java
index b8dca9cd380..f05cc440804 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantLongTerms.java
@@ -24,17 +24,13 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.aggregations.bucket.BucketStreamContext;
-import org.elasticsearch.search.aggregations.bucket.BucketStreams;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 
 /**
  *
@@ -52,30 +48,8 @@ public class SignificantLongTerms extends InternalSignificantTerms<SignificantLo
         }
     };
 
-    private static final BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
-        @Override
-        public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
-            Bucket buckets = new Bucket((long) context.attributes().get("subsetSize"), (long) context.attributes().get("supersetSize"),
-                    context.format());
-            buckets.readFrom(in);
-            return buckets;
-        }
-
-        @Override
-        public BucketStreamContext getBucketStreamContext(Bucket bucket) {
-            BucketStreamContext context = new BucketStreamContext();
-            Map<String, Object> attributes = new HashMap<>();
-            attributes.put("subsetSize", bucket.subsetSize);
-            attributes.put("supersetSize", bucket.supersetSize);
-            context.attributes(attributes);
-            context.format(bucket.format);
-            return context;
-        }
-    };
-
     public static void registerStreams() {
         AggregationStreams.registerStream(STREAM, TYPE.stream());
-        BucketStreams.registerStream(BUCKET_STREAM, TYPE.stream());
     }
 
     static class Bucket extends InternalSignificantTerms.Bucket {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java
index 616a6e163f8..560be5bcdb6 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantStringTerms.java
@@ -26,14 +26,11 @@ import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.aggregations.bucket.BucketStreamContext;
-import org.elasticsearch.search.aggregations.bucket.BucketStreams;
 import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -53,29 +50,8 @@ public class SignificantStringTerms extends InternalSignificantTerms<Significant
         }
     };
 
-    private static final BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
-        @Override
-        public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
-            Bucket buckets = new Bucket((long) context.attributes().get("subsetSize"),
-                    (long) context.attributes().get("supersetSize"), context.format());
-            buckets.readFrom(in);
-            return buckets;
-        }
-
-        @Override
-        public BucketStreamContext getBucketStreamContext(Bucket bucket) {
-            BucketStreamContext context = new BucketStreamContext();
-            Map<String, Object> attributes = new HashMap<>();
-            attributes.put("subsetSize", bucket.subsetSize);
-            attributes.put("supersetSize", bucket.supersetSize);
-            context.attributes(attributes);
-            return context;
-        }
-    };
-
     public static void registerStream() {
         AggregationStreams.registerStream(STREAM, TYPE.stream());
-        BucketStreams.registerStream(BUCKET_STREAM, TYPE.stream());
     }
 
     public static void registerStreams() {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java
index fa7ff8345cf..ebc06b48ffa 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java
@@ -24,13 +24,10 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.aggregations.bucket.BucketStreamContext;
-import org.elasticsearch.search.aggregations.bucket.BucketStreams;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -50,28 +47,8 @@ public class DoubleTerms extends InternalTerms<DoubleTerms, DoubleTerms.Bucket>
         }
     };
 
-    private static final BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
-        @Override
-        public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
-            Bucket buckets = new Bucket(context.format(), (boolean) context.attributes().get("showDocCountError"));
-            buckets.readFrom(in);
-            return buckets;
-        }
-
-        @Override
-        public BucketStreamContext getBucketStreamContext(Bucket bucket) {
-            BucketStreamContext context = new BucketStreamContext();
-            Map<String, Object> attributes = new HashMap<>();
-            attributes.put("showDocCountError", bucket.showDocCountError);
-            context.attributes(attributes);
-            context.format(bucket.format);
-            return context;
-        }
-    };
-
     public static void registerStreams() {
         AggregationStreams.registerStream(STREAM, TYPE.stream());
-        BucketStreams.registerStream(BUCKET_STREAM, TYPE.stream());
     }
 
     static class Bucket extends InternalTerms.Bucket {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java
index a893fef3856..4d95c8b4024 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java
@@ -24,13 +24,10 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.aggregations.bucket.BucketStreamContext;
-import org.elasticsearch.search.aggregations.bucket.BucketStreams;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -50,28 +47,8 @@ public class LongTerms extends InternalTerms<LongTerms, LongTerms.Bucket> {
         }
     };
 
-    private static final BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
-        @Override
-        public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
-            Bucket buckets = new Bucket(context.format(), (boolean) context.attributes().get("showDocCountError"));
-            buckets.readFrom(in);
-            return buckets;
-        }
-
-        @Override
-        public BucketStreamContext getBucketStreamContext(Bucket bucket) {
-            BucketStreamContext context = new BucketStreamContext();
-            Map<String, Object> attributes = new HashMap<>();
-            attributes.put("showDocCountError", bucket.showDocCountError);
-            context.attributes(attributes);
-            context.format(bucket.format);
-            return context;
-        }
-    };
-
     public static void registerStreams() {
         AggregationStreams.registerStream(STREAM, TYPE.stream());
-        BucketStreams.registerStream(BUCKET_STREAM, TYPE.stream());
     }
 
     static class Bucket extends InternalTerms.Bucket {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java
index 9f646aefe12..45da502cf28 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java
@@ -26,13 +26,10 @@ import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
-import org.elasticsearch.search.aggregations.bucket.BucketStreamContext;
-import org.elasticsearch.search.aggregations.bucket.BucketStreams;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -52,29 +49,8 @@ public class StringTerms extends InternalTerms<StringTerms, StringTerms.Bucket>
         }
     };
 
-    private static final BucketStreams.Stream<Bucket> BUCKET_STREAM = new BucketStreams.Stream<Bucket>() {
-        @Override
-        public Bucket readResult(StreamInput in, BucketStreamContext context) throws IOException {
-            Bucket buckets = new Bucket(
-                    context.format(),
-                    (boolean) context.attributes().get("showDocCountError"));
-            buckets.readFrom(in);
-            return buckets;
-        }
-
-        @Override
-        public BucketStreamContext getBucketStreamContext(Bucket bucket) {
-            BucketStreamContext context = new BucketStreamContext();
-            Map<String, Object> attributes = new HashMap<>();
-            attributes.put("showDocCountError", bucket.showDocCountError);
-            context.attributes(attributes);
-            return context;
-        }
-    };
-
     public static void registerStreams() {
         AggregationStreams.registerStream(STREAM, TYPE.stream());
-        BucketStreams.registerStream(BUCKET_STREAM, TYPE.stream());
     }
 
     public static class Bucket extends InternalTerms.Bucket {

From e3fe5c962555547d39321c84a6ce21b3e4430224 Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Tue, 5 Jul 2016 15:12:18 +0200
Subject: [PATCH 1243/1311] Add missing footer notes in mapper size docs

---
 docs/plugins/mapper-size.asciidoc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/plugins/mapper-size.asciidoc b/docs/plugins/mapper-size.asciidoc
index 7707d2544c4..df16d7eb857 100644
--- a/docs/plugins/mapper-size.asciidoc
+++ b/docs/plugins/mapper-size.asciidoc
@@ -103,3 +103,7 @@ GET my_index/_search
 // TEST[continued]
 
 <1> Querying on the `_size` field
+<2> Aggregating on the `_size` field
+<3> Sorting on the `_size` field
+<4> Accessing the `_size` field in scripts (inline scripts must be modules-security-scripting.html#enable-dynamic-scripting[enabled] for this example to work)
+

From 036b8ff17709efda67a0b928282c7f9d17e260fd Mon Sep 17 00:00:00 2001
From: LeeDr <lee.drengenberg@elastic.co>
Date: Tue, 5 Jul 2016 09:11:29 -0500
Subject: [PATCH 1244/1311] Fix stored_fields message

---
 .../java/org/elasticsearch/index/query/InnerHitBuilder.java     | 2 +-
 .../org/elasticsearch/rest/action/search/RestSearchAction.java  | 2 +-
 .../org/elasticsearch/search/builder/SearchSourceBuilder.java   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
index 5afb9ddc24c..5792db9197b 100644
--- a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
@@ -74,7 +74,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl
         PARSER.declareStringArray(InnerHitBuilder::setStoredFieldNames, SearchSourceBuilder.STORED_FIELDS_FIELD);
         PARSER.declareField((p, i, c) -> {
             throw new ParsingException(p.getTokenLocation(), "The field [" +
-                SearchSourceBuilder.FIELDS_FIELD + "] is not longer supported, please use [" +
+                SearchSourceBuilder.FIELDS_FIELD + "] is no longer supported, please use [" +
                 SearchSourceBuilder.STORED_FIELDS_FIELD + "] to retrieve stored fields or _source filtering " +
                 "if the field is not stored");
         }, SearchSourceBuilder.FIELDS_FIELD, ObjectParser.ValueType.STRING_ARRAY);
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
index 80a9f2bd4e5..c34a9391d8a 100644
--- a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
+++ b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
@@ -178,7 +178,7 @@ public class RestSearchAction extends BaseRestHandler {
 
         if (request.param("fields") != null) {
             throw new IllegalArgumentException("The parameter [" +
-                SearchSourceBuilder.FIELDS_FIELD + "] is not longer supported, please use [" +
+                SearchSourceBuilder.FIELDS_FIELD + "] is no longer supported, please use [" +
                 SearchSourceBuilder.STORED_FIELDS_FIELD + "] to retrieve stored fields or _source filtering " +
                 "if the field is not stored");
         }
diff --git a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
index d56119edce4..1fc66cf45dd 100644
--- a/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java
@@ -1073,7 +1073,7 @@ public final class SearchSourceBuilder extends ToXContentToBytes implements Writ
                     searchAfterBuilder = SearchAfterBuilder.fromXContent(parser, context.getParseFieldMatcher());
                 } else if (context.getParseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) {
                     throw new ParsingException(parser.getTokenLocation(), "The field [" +
-                        SearchSourceBuilder.FIELDS_FIELD + "] is not longer supported, please use [" +
+                        SearchSourceBuilder.FIELDS_FIELD + "] is no longer supported, please use [" +
                         SearchSourceBuilder.STORED_FIELDS_FIELD + "] to retrieve stored fields or _source filtering " +
                         "if the field is not stored");
                 } else {

From f36c1b4e6035a29c445d7caae6e56a04235d8d10 Mon Sep 17 00:00:00 2001
From: Britta Weber <brwe@users.noreply.github.com>
Date: Tue, 5 Jul 2016 15:53:25 +0200
Subject: [PATCH 1245/1311] Update fielddata.asciidoc

---
 docs/reference/mapping/params/fielddata.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/mapping/params/fielddata.asciidoc b/docs/reference/mapping/params/fielddata.asciidoc
index 92151c56d49..2e6b63698c7 100644
--- a/docs/reference/mapping/params/fielddata.asciidoc
+++ b/docs/reference/mapping/params/fielddata.asciidoc
@@ -81,7 +81,7 @@ can move the loading time from the first search request, to the refresh itself.
 Fielddata filtering can be used to reduce the number of terms loaded into
 memory, and thus reduce memory usage. Terms can be filtered by _frequency_:
 
-The frequency filter allows you to only load terms whose term frequency falls
+The frequency filter allows you to only load terms whose document frequency falls
 between a `min` and `max` value, which can be expressed an absolute
 number (when the number is bigger than 1.0) or as a percentage
 (eg `0.01` is `1%` and `1.0` is `100%`). Frequency is calculated

From c9da56dc807353ce3638968991c1c455c1280360 Mon Sep 17 00:00:00 2001
From: Christoph Wurm <cwurm@users.noreply.github.com>
Date: Tue, 5 Jul 2016 18:37:28 +0200
Subject: [PATCH 1246/1311] Reword Refresh API reference (#19270)

---
 docs/reference/docs/refresh.asciidoc | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/reference/docs/refresh.asciidoc b/docs/reference/docs/refresh.asciidoc
index 6bf6d0885bc..90c5d4e3afe 100644
--- a/docs/reference/docs/refresh.asciidoc
+++ b/docs/reference/docs/refresh.asciidoc
@@ -19,9 +19,10 @@ Wait for the changes made by the request to be made visible by a refresh before
 replying. This doesn't force an immediate refresh, rather, it waits for a
 refresh to happen. Elasticsearch automatically refreshes shards that have changed
 every `index.refresh_interval` which defaults to one second. That setting is
-<<dynamic-index-settings,dynamic>>. The <<indices-refresh>> API will also
-cause the request to return, as will setting `refresh` to `true` on any of the
-APIs that support it.
+<<dynamic-index-settings,dynamic>>. Calling the <<indices-refresh>> API or
+setting `refresh` to `true` on any of the APIs that support it will also
+cause a refresh, in turn causing already running requests with `refresh=wait_for`
+to return.
 
 `false` (the default)::
 

From ebe616988a4e2118b768783fb9d1e7fe1e0ee5e0 Mon Sep 17 00:00:00 2001
From: Justin Patrin <papercrane@reversefold.com>
Date: Tue, 5 Jul 2016 10:17:17 -0700
Subject: [PATCH 1247/1311] Start transport client round-robin randomly

This commit modifies the initial value of the transport client
round-robin index to a random value so that initial requests are more
likely to not all hit the same node.

Relates #14143
---
 .../client/transport/TransportClientNodesService.java         | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
index 7c2efa5fff9..17907e37b6c 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.client.transport;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
+
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
@@ -33,6 +34,7 @@ import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
@@ -95,7 +97,7 @@ public class TransportClientNodesService extends AbstractComponent implements Cl
 
     private volatile ScheduledFuture nodesSamplerFuture;
 
-    private final AtomicInteger randomNodeGenerator = new AtomicInteger();
+    private final AtomicInteger randomNodeGenerator = new AtomicInteger(Randomness.get().nextInt());
 
     private final boolean ignoreClusterName;
 

From 96f283c195ce7275ae9233d4156126ba325c05ce Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Tue, 5 Jul 2016 14:37:01 -0400
Subject: [PATCH 1248/1311] Rename writeThrowable to writeException

This commit renames writeThrowable to writeException. The situation here
stems from the fact that the StreamOutput method for serializing
Exceptions needs to accept Throwables too as Throwables can be the cause
of serialized Exceptions. Yet, we do not serialize Throwables in the
Error sub-hierarchy in a way that they can be deserialized into their
initial type. This leads to an asymmetry in the StreamOutput method for
serializing Exceptions and the StreamInput method for writing
Excpetions. Namely, the former will accept Throwables but the latter
will only return Exceptions. A goal with the stream methods has always
been symmetry in the method names so that serialization/deserialization
routines appear symmetrical in code. It is this asymmetry on the
input/output types for Exceptions on StreamOutput/StreamInput that
clashes with the desired symmetry of naming. Despite this, we should
favor symmetry in the naming of the methods. This commit renames
StreamOutput#writeThrowable to StreamOutput#writeException which leaves
us with Exception StreamInput#readException and void
StreamOutput#writeException(Throwable).
---
 .../main/java/org/elasticsearch/ElasticsearchException.java | 4 ++--
 .../java/org/elasticsearch/action/TaskOperationFailure.java | 2 +-
 .../admin/indices/shards/IndicesShardStoresResponse.java    | 2 +-
 .../org/elasticsearch/action/bulk/BulkItemResponse.java     | 2 +-
 .../java/org/elasticsearch/action/get/MultiGetResponse.java | 2 +-
 .../action/ingest/SimulateDocumentBaseResult.java           | 2 +-
 .../action/ingest/SimulateProcessorResult.java              | 2 +-
 .../elasticsearch/action/search/MultiSearchResponse.java    | 2 +-
 .../org/elasticsearch/action/search/ShardSearchFailure.java | 2 +-
 .../support/DefaultShardOperationFailedException.java       | 2 +-
 .../action/support/replication/ReplicationResponse.java     | 2 +-
 .../action/termvectors/MultiTermVectorsResponse.java        | 2 +-
 .../cluster/action/shard/ShardStateAction.java              | 2 +-
 .../org/elasticsearch/cluster/routing/UnassignedInfo.java   | 2 +-
 .../common/io/stream/NotSerializableExceptionWrapper.java   | 2 +-
 .../org/elasticsearch/common/io/stream/StreamOutput.java    | 4 ++--
 .../gateway/TransportNodesListGatewayStartedShards.java     | 2 +-
 core/src/main/java/org/elasticsearch/index/store/Store.java | 2 +-
 .../org/elasticsearch/repositories/VerificationFailure.java | 2 +-
 .../main/java/org/elasticsearch/transport/TcpTransport.java | 2 +-
 .../transport/local/LocalTransportChannel.java              | 2 +-
 core/src/test/java/org/elasticsearch/ESExceptionTests.java  | 6 +++---
 .../java/org/elasticsearch/ExceptionSerializationTests.java | 2 +-
 .../action/search/template/MultiSearchTemplateResponse.java | 2 +-
 .../elasticsearch/percolator/MultiPercolateResponse.java    | 2 +-
 .../test/hamcrest/ElasticsearchAssertions.java              | 2 +-
 26 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/ElasticsearchException.java b/core/src/main/java/org/elasticsearch/ElasticsearchException.java
index 1a695490585..801939ba737 100644
--- a/core/src/main/java/org/elasticsearch/ElasticsearchException.java
+++ b/core/src/main/java/org/elasticsearch/ElasticsearchException.java
@@ -204,7 +204,7 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(this.getMessage());
-        out.writeThrowable(this.getCause());
+        out.writeException(this.getCause());
         writeStackTraces(this, out);
         out.writeVInt(headers.size());
         for (Map.Entry<String, List<String>> entry : headers.entrySet()) {
@@ -435,7 +435,7 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
         Throwable[] suppressed = throwable.getSuppressed();
         out.writeVInt(suppressed.length);
         for (Throwable t : suppressed) {
-            out.writeThrowable(t);
+            out.writeException(t);
         }
         return throwable;
     }
diff --git a/core/src/main/java/org/elasticsearch/action/TaskOperationFailure.java b/core/src/main/java/org/elasticsearch/action/TaskOperationFailure.java
index f14b0606c71..6704f610ec0 100644
--- a/core/src/main/java/org/elasticsearch/action/TaskOperationFailure.java
+++ b/core/src/main/java/org/elasticsearch/action/TaskOperationFailure.java
@@ -68,7 +68,7 @@ public final class TaskOperationFailure implements Writeable, ToXContent {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(nodeId);
         out.writeLong(taskId);
-        out.writeThrowable(reason);
+        out.writeException(reason);
         RestStatus.writeTo(out, status);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java
index e6530c7baea..f1df6d53e18 100644
--- a/core/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoresResponse.java
@@ -189,7 +189,7 @@ public class IndicesShardStoresResponse extends ActionResponse implements ToXCon
             allocationStatus.writeTo(out);
             if (storeException != null) {
                 out.writeBoolean(true);
-                out.writeThrowable(storeException);
+                out.writeException(storeException);
             } else {
                 out.writeBoolean(false);
             }
diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java
index 1f875ee29d4..ad45ace84c9 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java
@@ -115,7 +115,7 @@ public class BulkItemResponse implements Streamable, StatusToXContent {
             out.writeString(getIndex());
             out.writeString(getType());
             out.writeOptionalString(getId());
-            out.writeThrowable(getCause());
+            out.writeException(getCause());
         }
 
 
diff --git a/core/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java b/core/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java
index 24069db645c..e1fe435fd10 100644
--- a/core/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/get/MultiGetResponse.java
@@ -100,7 +100,7 @@ public class MultiGetResponse extends ActionResponse implements Iterable<MultiGe
             out.writeString(index);
             out.writeOptionalString(type);
             out.writeString(id);
-            out.writeThrowable(exception);
+            out.writeException(exception);
         }
 
         public Exception getFailure() {
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java
index a7fb9237523..82b39ac8972 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulateDocumentBaseResult.java
@@ -63,7 +63,7 @@ public final class SimulateDocumentBaseResult implements SimulateDocumentResult
             ingestDocument.writeTo(out);
         } else {
             out.writeBoolean(true);
-            out.writeThrowable(failure);
+            out.writeException(failure);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java b/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
index 8e4725abecc..c2703f0224d 100644
--- a/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
+++ b/core/src/main/java/org/elasticsearch/action/ingest/SimulateProcessorResult.java
@@ -68,7 +68,7 @@ public class SimulateProcessorResult implements Writeable, ToXContent {
             ingestDocument.writeTo(out);
         } else {
             out.writeBoolean(true);
-            out.writeThrowable(failure);
+            out.writeException(failure);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java b/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java
index f1b84812019..317b775a403 100644
--- a/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/search/MultiSearchResponse.java
@@ -101,7 +101,7 @@ public class MultiSearchResponse extends ActionResponse implements Iterable<Mult
                 response.writeTo(out);
             } else {
                 out.writeBoolean(false);
-                out.writeThrowable(exception);
+                out.writeException(exception);
             }
         }
 
diff --git a/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java b/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java
index 6c24234acef..8070081dcd8 100644
--- a/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java
+++ b/core/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java
@@ -148,7 +148,7 @@ public class ShardSearchFailure implements ShardOperationFailedException {
         }
         out.writeString(reason);
         RestStatus.writeTo(out, status);
-        out.writeThrowable(cause);
+        out.writeException(cause);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java b/core/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java
index e584b2d8037..0fe3be1ad63 100644
--- a/core/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java
+++ b/core/src/main/java/org/elasticsearch/action/support/DefaultShardOperationFailedException.java
@@ -111,7 +111,7 @@ public class DefaultShardOperationFailedException implements ShardOperationFaile
             out.writeString(index);
         }
         out.writeVInt(shardId);
-        out.writeThrowable(reason);
+        out.writeException(reason);
         RestStatus.writeTo(out, status);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
index c79f8bb7302..e6ce0a5aad4 100644
--- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java
@@ -260,7 +260,7 @@ public class ReplicationResponse extends ActionResponse {
             public void writeTo(StreamOutput out) throws IOException {
                 shardId.writeTo(out);
                 out.writeOptionalString(nodeId);
-                out.writeThrowable(cause);
+                out.writeException(cause);
                 RestStatus.writeTo(out, status);
                 out.writeBoolean(primary);
             }
diff --git a/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java b/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
index 7d8294ecbd4..233d4b0c638 100644
--- a/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
+++ b/core/src/main/java/org/elasticsearch/action/termvectors/MultiTermVectorsResponse.java
@@ -100,7 +100,7 @@ public class MultiTermVectorsResponse extends ActionResponse implements Iterable
             out.writeString(index);
             out.writeOptionalString(type);
             out.writeString(id);
-            out.writeThrowable(cause);
+            out.writeException(cause);
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
index edd4e7db865..e6a6dea7def 100644
--- a/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
+++ b/core/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java
@@ -437,7 +437,7 @@ public class ShardStateAction extends AbstractComponent {
             shardRouting.writeTo(out);
             sourceShardRouting.writeTo(out);
             out.writeString(message);
-            out.writeThrowable(failure);
+            out.writeException(failure);
         }
 
         @Override
diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java b/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
index cfa7c771d50..c66c4e0bd3d 100644
--- a/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
+++ b/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java
@@ -167,7 +167,7 @@ public final class UnassignedInfo implements ToXContent, Writeable {
         // Do not serialize unassignedTimeNanos as System.nanoTime() cannot be compared across different JVMs
         out.writeBoolean(delayed);
         out.writeOptionalString(message);
-        out.writeThrowable(failure);
+        out.writeException(failure);
         out.writeVInt(failedAllocations);
     }
 
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/NotSerializableExceptionWrapper.java b/core/src/main/java/org/elasticsearch/common/io/stream/NotSerializableExceptionWrapper.java
index 64cbf7d8211..b25d4bc9b72 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/NotSerializableExceptionWrapper.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/NotSerializableExceptionWrapper.java
@@ -27,7 +27,7 @@ import java.io.IOException;
 
 /**
  * This exception can be used to wrap a given, not serializable exception
- * to serialize via {@link StreamOutput#writeThrowable(Throwable)}.
+ * to serialize via {@link StreamOutput#writeException(Throwable)}.
  * This class will preserve the stacktrace as well as the suppressed exceptions of
  * the throwable it was created with instead of it's own. The stacktrace has no indication
  * of where this exception was created.
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
index 653dd9676b3..dee3ea24f00 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
@@ -631,7 +631,7 @@ public abstract class StreamOutput extends OutputStream {
         }
     }
 
-    public void writeThrowable(Throwable throwable) throws IOException {
+    public void writeException(Throwable throwable) throws IOException {
         if (throwable == null) {
             writeBoolean(false);
         } else {
@@ -739,7 +739,7 @@ public abstract class StreamOutput extends OutputStream {
                 writeOptionalString(throwable.getMessage());
             }
             if (writeCause) {
-                writeThrowable(throwable.getCause());
+                writeException(throwable.getCause());
             }
             ElasticsearchException.writeStackTraces(throwable, this);
         }
diff --git a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
index 2ecabf778c7..6d9cbd6ea51 100644
--- a/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
+++ b/core/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java
@@ -298,7 +298,7 @@ public class TransportNodesListGatewayStartedShards extends
             out.writeBoolean(primary);
             if (storeException != null) {
                 out.writeBoolean(true);
-                out.writeThrowable(storeException);
+                out.writeException(storeException);
             } else {
                 out.writeBoolean(false);
             }
diff --git a/core/src/main/java/org/elasticsearch/index/store/Store.java b/core/src/main/java/org/elasticsearch/index/store/Store.java
index d52ab16459e..dfbad39ad39 100644
--- a/core/src/main/java/org/elasticsearch/index/store/Store.java
+++ b/core/src/main/java/org/elasticsearch/index/store/Store.java
@@ -1337,7 +1337,7 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
             try (IndexOutput output = this.directory().createOutput(uuid, IOContext.DEFAULT)) {
                 CodecUtil.writeHeader(output, CODEC, VERSION);
                 BytesStreamOutput out = new BytesStreamOutput();
-                out.writeThrowable(exception);
+                out.writeException(exception);
                 BytesReference bytes = out.bytes();
                 output.writeVInt(bytes.length());
                 BytesRef ref = bytes.toBytesRef();
diff --git a/core/src/main/java/org/elasticsearch/repositories/VerificationFailure.java b/core/src/main/java/org/elasticsearch/repositories/VerificationFailure.java
index 5672506b7fe..b9cc46e850c 100644
--- a/core/src/main/java/org/elasticsearch/repositories/VerificationFailure.java
+++ b/core/src/main/java/org/elasticsearch/repositories/VerificationFailure.java
@@ -59,7 +59,7 @@ public class VerificationFailure implements Streamable {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(nodeId);
-        out.writeThrowable(cause);
+        out.writeException(cause);
     }
 
     public static VerificationFailure readNode(StreamInput in) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
index 6bfebd1f35d..b56b91061ed 100644
--- a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
@@ -940,7 +940,7 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
             stream.setVersion(nodeVersion);
             RemoteTransportException tx = new RemoteTransportException(
                 nodeName(), new InetSocketTransportAddress(getLocalAddress(channel)), action, error);
-            stream.writeThrowable(tx);
+            stream.writeException(tx);
             byte status = 0;
             status = TransportStatus.setResponse(status);
             status = TransportStatus.setError(status);
diff --git a/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java b/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java
index 129d35c8ebf..81d99b8d493 100644
--- a/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java
+++ b/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java
@@ -96,7 +96,7 @@ public class LocalTransportChannel implements TransportChannel {
         writeResponseExceptionHeader(stream);
         RemoteTransportException tx = new RemoteTransportException(targetTransport.nodeName(),
                 targetTransport.boundAddress().boundAddresses()[0], action, exception);
-        stream.writeThrowable(tx);
+        stream.writeException(tx);
         sendResponseData(BytesReference.toBytes(stream.bytes()));
         sourceTransportServiceAdapter.onResponseSent(requestId, action, exception);
     }
diff --git a/core/src/test/java/org/elasticsearch/ESExceptionTests.java b/core/src/test/java/org/elasticsearch/ESExceptionTests.java
index 5e761a97c63..44ccc4ef342 100644
--- a/core/src/test/java/org/elasticsearch/ESExceptionTests.java
+++ b/core/src/test/java/org/elasticsearch/ESExceptionTests.java
@@ -301,7 +301,7 @@ public class ESExceptionTests extends ESTestCase {
     public void testSerializeElasticsearchException() throws IOException {
         BytesStreamOutput out = new BytesStreamOutput();
         ParsingException ex = new ParsingException(1, 2, "foobar", null);
-        out.writeThrowable(ex);
+        out.writeException(ex);
 
         StreamInput in = out.bytes().streamInput();
         ParsingException e = in.readException();
@@ -315,7 +315,7 @@ public class ESExceptionTests extends ESTestCase {
         BytesStreamOutput out = new BytesStreamOutput();
         ParsingException parsingException = new ParsingException(1, 2, "foobar", null);
         final Exception ex = new UnknownException("eggplant", parsingException);
-        out.writeThrowable(ex);
+        out.writeException(ex);
 
         StreamInput in = out.bytes().streamInput();
         Throwable throwable = in.readException();
@@ -354,7 +354,7 @@ public class ESExceptionTests extends ESTestCase {
         for (final Exception cause : causes) {
             BytesStreamOutput out = new BytesStreamOutput();
             ElasticsearchException ex = new ElasticsearchException("topLevel", cause);
-            out.writeThrowable(ex);
+            out.writeException(ex);
             StreamInput in = out.bytes().streamInput();
             ElasticsearchException e = in.readException();
             assertEquals(e.getMessage(), ex.getMessage());
diff --git a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
index 17ff2047238..9086bb4222f 100644
--- a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
+++ b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java
@@ -222,7 +222,7 @@ public class ExceptionSerializationTests extends ESTestCase {
     private <T extends Exception> T serialize(T exception) throws IOException {
         ElasticsearchAssertions.assertVersionSerializable(VersionUtils.randomVersion(random()), exception);
         BytesStreamOutput out = new BytesStreamOutput();
-        out.writeThrowable(exception);
+        out.writeException(exception);
         StreamInput in = out.bytes().streamInput();
         return in.readException();
     }
diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateResponse.java b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateResponse.java
index 5ca5078661c..f9e99ffc7e3 100644
--- a/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateResponse.java
+++ b/modules/lang-mustache/src/main/java/org/elasticsearch/action/search/template/MultiSearchTemplateResponse.java
@@ -96,7 +96,7 @@ public class MultiSearchTemplateResponse extends ActionResponse implements Itera
                 response.writeTo(out);
             } else {
                 out.writeBoolean(false);
-                out.writeThrowable(exception);
+                out.writeException(exception);
             }
         }
 
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateResponse.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateResponse.java
index 66b61ff0350..afc5b7ab6c7 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateResponse.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateResponse.java
@@ -172,7 +172,7 @@ public class MultiPercolateResponse extends ActionResponse implements Iterable<M
                 response.writeTo(out);
             } else {
                 out.writeBoolean(false);
-                out.writeThrowable(exception);
+                out.writeException(exception);
             }
         }
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
index 988d4b9bb87..24af2b72bd1 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java
@@ -682,7 +682,7 @@ public class ElasticsearchAssertions {
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            out.writeThrowable(exception);
+            out.writeException(exception);
         }
 
     }

From b3c015e2bb6324e48d02dd3770f371eb98a3a756 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 19 May 2016 12:40:22 -0400
Subject: [PATCH 1249/1311] Reindex from remote

This adds a remote option to reindex that looks like

```
curl -POST 'localhost:9200/_reindex?pretty' -d'{
  "source": {
    "remote": {
      "host": "http://otherhost:9200"
    },
    "index": "target",
    "query": {
      "match": {
        "foo": "bar"
      }
    }
  },
  "dest": {
    "index": "target"
  }
}'
```

This reindex has all of the features of local reindex:
* Using queries to filter what is copied
* Retry on rejection
* Throttle/rethottle
The big advantage of this version is that it goes over the HTTP API
which can be made backwards compatible.

Some things are different:

The query field is sent directly to the other node rather than parsed
on the coordinating node. This should allow it to support constructs
that are invalid on the coordinating node but are valid on the target
node. Mostly, that means old syntax.
---
 .../action/bulk/BackoffPolicy.java            |  46 ++
 .../common/xcontent/ObjectParser.java         |   1 +
 .../action/bulk/BackoffPolicyTests.java       |  65 ++
 docs/build.gradle                             |  14 +
 docs/reference/docs/reindex.asciidoc          |  54 ++
 modules/reindex/build.gradle                  |  38 ++
 .../licenses/commons-codec-1.10.jar.sha1      |   1 +
 .../licenses/commons-codec-LICENSE.txt        | 202 +++++++
 .../reindex/licenses/commons-codec-NOTICE.txt |  17 +
 .../licenses/commons-logging-1.1.3.jar.sha1   |   1 +
 .../licenses/commons-logging-LICENSE.txt      | 202 +++++++
 .../licenses/commons-logging-NOTICE.txt       |   5 +
 .../licenses/httpclient-4.5.2.jar.sha1        |   1 +
 .../reindex/licenses/httpclient-LICENSE.txt   | 558 ++++++++++++++++++
 .../reindex/licenses/httpclient-NOTICE.txt    |   5 +
 .../reindex/licenses/httpcore-4.4.4.jar.sha1  |   1 +
 modules/reindex/licenses/httpcore-LICENSE.txt | 241 ++++++++
 modules/reindex/licenses/httpcore-NOTICE.txt  |   8 +
 .../AbstractAsyncBulkByScrollAction.java      | 219 ++-----
 .../AbstractAsyncBulkIndexByScrollAction.java |  69 +--
 .../AbstractBulkIndexByScrollRequest.java     |   2 +-
 .../reindex/BulkIndexByScrollResponse.java    |  58 +-
 ...kIndexByScrollResponseContentListener.java |   8 +-
 .../reindex/ClientScrollableHitSource.java    | 251 ++++++++
 .../index/reindex/ReindexPlugin.java          |   8 +
 .../index/reindex/ReindexRequest.java         |  23 +
 .../index/reindex/ReindexRequestBuilder.java  |   9 +
 .../index/reindex/RestReindexAction.java      |  82 ++-
 .../index/reindex/ScrollableHitSource.java    | 357 +++++++++++
 .../reindex/TransportDeleteByQueryAction.java |  25 +-
 .../index/reindex/TransportReindexAction.java |  93 ++-
 .../reindex/TransportRethrottleAction.java    |   1 -
 .../reindex/TransportUpdateByQueryAction.java |  17 +-
 .../index/reindex/remote/RemoteInfo.java      | 113 ++++
 .../reindex/remote/RemoteRequestBuilders.java | 163 +++++
 .../reindex/remote/RemoteResponseParsers.java | 301 ++++++++++
 .../remote/RemoteScrollableHitSource.java     | 242 ++++++++
 ...BulkIndexByScrollActionScriptTestCase.java |  10 +-
 ...lkIndexbyScrollActionMetadataTestCase.java |  23 +-
 .../reindex/AsyncBulkByScrollActionTests.java | 110 ++--
 .../BulkIndexByScrollResponseMatcher.java     |   2 +-
 .../index/reindex/CancelTests.java            |   2 +-
 .../index/reindex/ReindexFailureTests.java    |   4 +-
 .../ReindexFromRemoteWhitelistTests.java      |  92 +++
 .../index/reindex/ReindexMetadataTests.java   |  11 +-
 .../index/reindex/ReindexRequestTests.java    |  60 ++
 ...> ReindexSourceTargetValidationTests.java} |  57 +-
 .../index/reindex/RestReindexActionTests.java | 121 ++++
 .../index/reindex/RetryTests.java             |  68 ++-
 .../index/reindex/RoundTripTests.java         |  64 +-
 .../reindex/UpdateByQueryMetadataTests.java   |   3 +-
 .../reindex/UpdateByQueryWithScriptTests.java |   3 +-
 .../index/reindex/remote/RemoteInfoTests.java |  36 ++
 .../remote/RemoteRequestBuildersTests.java    | 181 ++++++
 .../RemoteScrollableHitSourceTests.java       | 381 ++++++++++++
 .../responses/failure_with_status.json        |  28 +
 .../test/resources/responses/main/0_20_5.json |  10 +
 .../resources/responses/main/0_90_13.json     |  13 +
 .../test/resources/responses/main/1_7_5.json  |  13 +
 .../test/resources/responses/main/2_3_3.json  |  12 +
 .../responses/main/5_0_0_alpha_3.json         |  12 +
 .../test/resources/responses/rejection.json   |  34 ++
 .../resources/responses/request_failure.json  |  15 +
 .../responses/scroll_fully_loaded.json        |  30 +
 .../test/resources/responses/scroll_ok.json   |  26 +
 .../test/resources/responses/start_ok.json    |  25 +
 .../test/resources/responses/start_scan.json  |  15 +
 .../test/reindex/20_validation.yaml           |  43 ++
 .../rest-api-spec/test/reindex/90_remote.yaml | 207 +++++++
 .../org/elasticsearch/test/rest/Stash.java    |   2 +-
 .../test/rest/test/StashTests.java            |  46 ++
 71 files changed, 4791 insertions(+), 469 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/action/bulk/BackoffPolicyTests.java
 create mode 100644 modules/reindex/licenses/commons-codec-1.10.jar.sha1
 create mode 100644 modules/reindex/licenses/commons-codec-LICENSE.txt
 create mode 100644 modules/reindex/licenses/commons-codec-NOTICE.txt
 create mode 100644 modules/reindex/licenses/commons-logging-1.1.3.jar.sha1
 create mode 100644 modules/reindex/licenses/commons-logging-LICENSE.txt
 create mode 100644 modules/reindex/licenses/commons-logging-NOTICE.txt
 create mode 100644 modules/reindex/licenses/httpclient-4.5.2.jar.sha1
 create mode 100644 modules/reindex/licenses/httpclient-LICENSE.txt
 create mode 100644 modules/reindex/licenses/httpclient-NOTICE.txt
 create mode 100644 modules/reindex/licenses/httpcore-4.4.4.jar.sha1
 create mode 100644 modules/reindex/licenses/httpcore-LICENSE.txt
 create mode 100644 modules/reindex/licenses/httpcore-NOTICE.txt
 create mode 100644 modules/reindex/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java
 create mode 100644 modules/reindex/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java
 create mode 100644 modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteInfo.java
 create mode 100644 modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java
 create mode 100644 modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java
 create mode 100644 modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java
 create mode 100644 modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWhitelistTests.java
 create mode 100644 modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java
 rename modules/reindex/src/test/java/org/elasticsearch/index/reindex/{ReindexSameIndexTests.java => ReindexSourceTargetValidationTests.java} (62%)
 create mode 100644 modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java
 create mode 100644 modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteInfoTests.java
 create mode 100644 modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java
 create mode 100644 modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java
 create mode 100644 modules/reindex/src/test/resources/responses/failure_with_status.json
 create mode 100644 modules/reindex/src/test/resources/responses/main/0_20_5.json
 create mode 100644 modules/reindex/src/test/resources/responses/main/0_90_13.json
 create mode 100644 modules/reindex/src/test/resources/responses/main/1_7_5.json
 create mode 100644 modules/reindex/src/test/resources/responses/main/2_3_3.json
 create mode 100644 modules/reindex/src/test/resources/responses/main/5_0_0_alpha_3.json
 create mode 100644 modules/reindex/src/test/resources/responses/rejection.json
 create mode 100644 modules/reindex/src/test/resources/responses/request_failure.json
 create mode 100644 modules/reindex/src/test/resources/responses/scroll_fully_loaded.json
 create mode 100644 modules/reindex/src/test/resources/responses/scroll_ok.json
 create mode 100644 modules/reindex/src/test/resources/responses/start_ok.json
 create mode 100644 modules/reindex/src/test/resources/responses/start_scan.json
 create mode 100644 modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yaml
 create mode 100644 test/framework/src/test/java/org/elasticsearch/test/rest/test/StashTests.java

diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BackoffPolicy.java b/core/src/main/java/org/elasticsearch/action/bulk/BackoffPolicy.java
index a0ccca0fb5c..bc8f8c347ab 100644
--- a/core/src/main/java/org/elasticsearch/action/bulk/BackoffPolicy.java
+++ b/core/src/main/java/org/elasticsearch/action/bulk/BackoffPolicy.java
@@ -89,6 +89,13 @@ public abstract class BackoffPolicy implements Iterable<TimeValue> {
         return new ExponentialBackoff((int) checkDelay(initialDelay).millis(), maxNumberOfRetries);
     }
 
+    /**
+     * Wraps the backoff policy in one that calls a method every time a new backoff is taken from the policy.
+     */
+    public static BackoffPolicy wrap(BackoffPolicy delegate, Runnable onBackoff) {
+        return new WrappedBackoffPolicy(delegate, onBackoff);
+    }
+
     private static TimeValue checkDelay(TimeValue delay) {
         if (delay.millis() > Integer.MAX_VALUE) {
             throw new IllegalArgumentException("delay must be <= " + Integer.MAX_VALUE + " ms");
@@ -200,4 +207,43 @@ public abstract class BackoffPolicy implements Iterable<TimeValue> {
             return delay;
         }
     }
+
+    private static final class WrappedBackoffPolicy extends BackoffPolicy {
+        private final BackoffPolicy delegate;
+        private final Runnable onBackoff;
+
+        public WrappedBackoffPolicy(BackoffPolicy delegate, Runnable onBackoff) {
+            this.delegate = delegate;
+            this.onBackoff = onBackoff;
+        }
+
+        @Override
+        public Iterator<TimeValue> iterator() {
+            return new WrappedBackoffIterator(delegate.iterator(), onBackoff);
+        }
+    }
+
+    private static final class WrappedBackoffIterator implements Iterator<TimeValue> {
+        private final Iterator<TimeValue> delegate;
+        private final Runnable onBackoff;
+
+        public WrappedBackoffIterator(Iterator<TimeValue> delegate, Runnable onBackoff) {
+            this.delegate = delegate;
+            this.onBackoff = onBackoff;
+        }
+
+        @Override
+        public boolean hasNext() {
+            return delegate.hasNext();
+        }
+
+        @Override
+        public TimeValue next() {
+            if (false == delegate.hasNext()) {
+                throw new NoSuchElementException();
+            }
+            onBackoff.run();
+            return delegate.next();
+        }
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java b/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
index 41cb3bca82b..3e595ae0d18 100644
--- a/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
+++ b/core/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java
@@ -413,6 +413,7 @@ public final class ObjectParser<Value, Context extends ParseFieldMatcherSupplier
         STRING(VALUE_STRING),
         STRING_OR_NULL(VALUE_STRING, VALUE_NULL),
         FLOAT(VALUE_NUMBER, VALUE_STRING),
+        FLOAT_OR_NULL(VALUE_NUMBER, VALUE_STRING, VALUE_NULL),
         DOUBLE(VALUE_NUMBER, VALUE_STRING),
         LONG(VALUE_NUMBER, VALUE_STRING),
         INT(VALUE_NUMBER, VALUE_STRING),
diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BackoffPolicyTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BackoffPolicyTests.java
new file mode 100644
index 00000000000..e3cfeb2a4ac
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/action/bulk/BackoffPolicyTests.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.bulk;
+
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.test.ESTestCase;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
+
+public class BackoffPolicyTests extends ESTestCase {
+    public void testWrapBackoffPolicy() {
+        TimeValue timeValue = timeValueMillis(between(0, Integer.MAX_VALUE));
+        int maxNumberOfRetries = between(1, 1000);
+        BackoffPolicy policy = BackoffPolicy.constantBackoff(timeValue, maxNumberOfRetries);
+        AtomicInteger retries = new AtomicInteger();
+        policy = BackoffPolicy.wrap(policy, retries::getAndIncrement);
+
+        int expectedRetries = 0;
+        {
+            // Fetching the iterator doesn't call the callback
+            Iterator<TimeValue> itr = policy.iterator();
+            assertEquals(expectedRetries, retries.get());
+
+            while (itr.hasNext()) {
+                // hasNext doesn't trigger the callback
+                assertEquals(expectedRetries, retries.get());
+                // next does
+                itr.next();
+                expectedRetries += 1;
+                assertEquals(expectedRetries, retries.get());
+            }
+            // next doesn't call the callback when there isn't a backoff available
+            expectThrows(NoSuchElementException.class, () -> itr.next());
+            assertEquals(expectedRetries, retries.get());
+        }
+        {
+            // The second iterator also calls the callback
+            Iterator<TimeValue> itr = policy.iterator();
+            itr.next();
+            expectedRetries += 1;
+            assertEquals(expectedRetries, retries.get());
+        }
+    }
+}
diff --git a/docs/build.gradle b/docs/build.gradle
index 660755a1c65..41e3e352b8d 100644
--- a/docs/build.gradle
+++ b/docs/build.gradle
@@ -30,6 +30,8 @@ integTest {
     configFile 'scripts/my_script.py'
     configFile 'userdict_ja.txt'
     configFile 'KeywordTokenizer.rbbi'
+    // Whitelist reindexing from the local node so we can test it.
+    setting 'reindex.remote.whitelist', 'myself'
   }
 }
 
@@ -81,3 +83,15 @@ Closure setupTwitter = { String name, int count ->
   }
 setupTwitter('twitter', 5)
 setupTwitter('big_twitter', 120)
+
+buildRestTests.setups['host'] = '''
+  # Fetch the http host. We use the host of the master because we know there will always be a master.
+  - do:
+      cluster.state: {}
+  - set: { master_node: master }
+  - do:
+      nodes.info:
+        metric: [ http ]
+  - is_true: nodes.$master.http.publish_address
+  - set: {nodes.$master.http.publish_address: host}
+'''
diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc
index aac722e96a3..2afc8e86bb1 100644
--- a/docs/reference/docs/reindex.asciidoc
+++ b/docs/reference/docs/reindex.asciidoc
@@ -361,6 +361,60 @@ POST _reindex
 // CONSOLE
 // TEST[s/^/PUT source\nGET _cluster\/health?wait_for_status=yellow\n/]
 
+[float]
+=== Reindex from Remote
+
+Reindex supports reindexing from a remote Elasticsearch cluster:
+
+[source,js]
+--------------------------------------------------
+POST _reindex
+{
+  "source": {
+    "remote": {
+      "host": "http://otherhost:9200",
+      "username": "user",
+      "password": "pass"
+    },
+    "index": "source",
+    "query": {
+      "match": {
+        "test": "data"
+      }
+    }
+  },
+  "dest": {
+    "index": "dest"
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:host]
+// TEST[s/^/PUT source\nGET _cluster\/health?wait_for_status=yellow\n/]
+// TEST[s/otherhost:9200",/\${host}"/]
+// TEST[s/"username": "user",//]
+// TEST[s/"password": "pass"//]
+
+The `host` parameter must contain a scheme, host, and port (e.g.
+`https://otherhost:9200`). The `username` and `password` parameters are
+optional and when they are present reindex will connect to the remote
+Elasticsearch node using using basic auth. Be sure to use `https` when using
+basic auth or the password will be sent in plain text.
+
+Remote hosts have to be explicitly whitelisted in elasticsearch.yaml using the
+`reindex.remote.whitelist` property. It can be set to a comma delimited list
+of allowed remote `host` and `port` combinations (e.g.
+`otherhost:9200, another:9200`). Scheme is ignored by the whitelist - only host
+and port are used.
+
+This feature should work with remote clusters of any version of Elasticsearch
+you are likely to find. This should allow you to upgrade from any version of
+Elasticsearch to the current version by reindexing from a cluster of the old
+version.
+
+To enable queries sent to older versions of Elasticsearch the `query` parameter
+is sent directly to the remote host without validation or modification.
+
 [float]
 === URL Parameters
 
diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle
index 5bd5aeb3dc0..c6def124326 100644
--- a/modules/reindex/build.gradle
+++ b/modules/reindex/build.gradle
@@ -21,3 +21,41 @@ esplugin {
   description 'The Reindex module adds APIs to reindex from one index to another or update documents in place.'
   classname 'org.elasticsearch.index.reindex.ReindexPlugin'
 }
+
+integTest {
+  cluster {
+    // Whitelist reindexing from the local node so we can test it.
+    setting 'reindex.remote.whitelist', 'myself'
+  }
+}
+
+run {
+  // Whitelist reindexing from the local node so we can test it.
+  setting 'reindex.remote.whitelist', 'myself'
+}
+
+
+dependencies {
+  compile "org.elasticsearch.client:rest:${version}"
+  // dependencies of the rest client
+  compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
+  compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
+  compile "commons-codec:commons-codec:${versions.commonscodec}"
+  compile "commons-logging:commons-logging:${versions.commonslogging}"
+}
+
+dependencyLicenses {
+  // Don't check the client's license. We know it.
+  dependencies = project.configurations.runtime.fileCollection {
+    it.group.startsWith('org.elasticsearch') == false
+  } - project.configurations.provided
+}
+
+thirdPartyAudit.excludes = [
+  // Commons logging
+  'javax.servlet.ServletContextEvent',
+  'javax.servlet.ServletContextListener',
+  'org.apache.avalon.framework.logger.Logger',
+  'org.apache.log.Hierarchy',
+  'org.apache.log.Logger',
+]
diff --git a/modules/reindex/licenses/commons-codec-1.10.jar.sha1 b/modules/reindex/licenses/commons-codec-1.10.jar.sha1
new file mode 100644
index 00000000000..3fe8682a1b0
--- /dev/null
+++ b/modules/reindex/licenses/commons-codec-1.10.jar.sha1
@@ -0,0 +1 @@
+4b95f4897fa13f2cd904aee711aeafc0c5295cd8
\ No newline at end of file
diff --git a/modules/reindex/licenses/commons-codec-LICENSE.txt b/modules/reindex/licenses/commons-codec-LICENSE.txt
new file mode 100644
index 00000000000..d6456956733
--- /dev/null
+++ b/modules/reindex/licenses/commons-codec-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/modules/reindex/licenses/commons-codec-NOTICE.txt b/modules/reindex/licenses/commons-codec-NOTICE.txt
new file mode 100644
index 00000000000..56916449bbe
--- /dev/null
+++ b/modules/reindex/licenses/commons-codec-NOTICE.txt
@@ -0,0 +1,17 @@
+Apache Commons Codec
+Copyright 2002-2015 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java
+contains test data from http://aspell.net/test/orig/batch0.tab.
+Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org)
+
+===============================================================================
+
+The content of package org.apache.commons.codec.language.bm has been translated
+from the original php source code available at http://stevemorse.org/phoneticinfo.htm
+with permission from the original authors.
+Original source copyright:
+Copyright (c) 2008 Alexander Beider & Stephen P. Morse.
diff --git a/modules/reindex/licenses/commons-logging-1.1.3.jar.sha1 b/modules/reindex/licenses/commons-logging-1.1.3.jar.sha1
new file mode 100644
index 00000000000..c8756c43832
--- /dev/null
+++ b/modules/reindex/licenses/commons-logging-1.1.3.jar.sha1
@@ -0,0 +1 @@
+f6f66e966c70a83ffbdb6f17a0919eaf7c8aca7f
diff --git a/modules/reindex/licenses/commons-logging-LICENSE.txt b/modules/reindex/licenses/commons-logging-LICENSE.txt
new file mode 100644
index 00000000000..57bc88a15a0
--- /dev/null
+++ b/modules/reindex/licenses/commons-logging-LICENSE.txt
@@ -0,0 +1,202 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
diff --git a/modules/reindex/licenses/commons-logging-NOTICE.txt b/modules/reindex/licenses/commons-logging-NOTICE.txt
new file mode 100644
index 00000000000..72eb32a9024
--- /dev/null
+++ b/modules/reindex/licenses/commons-logging-NOTICE.txt
@@ -0,0 +1,5 @@
+Apache Commons CLI
+Copyright 2001-2009 The Apache Software Foundation
+
+This product includes software developed by
+The Apache Software Foundation (http://www.apache.org/).
diff --git a/modules/reindex/licenses/httpclient-4.5.2.jar.sha1 b/modules/reindex/licenses/httpclient-4.5.2.jar.sha1
new file mode 100644
index 00000000000..6937112a09f
--- /dev/null
+++ b/modules/reindex/licenses/httpclient-4.5.2.jar.sha1
@@ -0,0 +1 @@
+733db77aa8d9b2d68015189df76ab06304406e50
\ No newline at end of file
diff --git a/modules/reindex/licenses/httpclient-LICENSE.txt b/modules/reindex/licenses/httpclient-LICENSE.txt
new file mode 100644
index 00000000000..32f01eda18f
--- /dev/null
+++ b/modules/reindex/licenses/httpclient-LICENSE.txt
@@ -0,0 +1,558 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+=========================================================================
+
+This project includes Public Suffix List copied from
+<https://publicsuffix.org/list/effective_tld_names.dat>
+licensed under the terms of the Mozilla Public License, v. 2.0
+
+Full license text: <http://mozilla.org/MPL/2.0/>
+
+Mozilla Public License Version 2.0
+==================================
+
+1. Definitions
+--------------
+
+1.1. "Contributor"
+    means each individual or legal entity that creates, contributes to
+    the creation of, or owns Covered Software.
+
+1.2. "Contributor Version"
+    means the combination of the Contributions of others (if any) used
+    by a Contributor and that particular Contributor's Contribution.
+
+1.3. "Contribution"
+    means Covered Software of a particular Contributor.
+
+1.4. "Covered Software"
+    means Source Code Form to which the initial Contributor has attached
+    the notice in Exhibit A, the Executable Form of such Source Code
+    Form, and Modifications of such Source Code Form, in each case
+    including portions thereof.
+
+1.5. "Incompatible With Secondary Licenses"
+    means
+
+    (a) that the initial Contributor has attached the notice described
+        in Exhibit B to the Covered Software; or
+
+    (b) that the Covered Software was made available under the terms of
+        version 1.1 or earlier of the License, but not also under the
+        terms of a Secondary License.
+
+1.6. "Executable Form"
+    means any form of the work other than Source Code Form.
+
+1.7. "Larger Work"
+    means a work that combines Covered Software with other material, in
+    a separate file or files, that is not Covered Software.
+
+1.8. "License"
+    means this document.
+
+1.9. "Licensable"
+    means having the right to grant, to the maximum extent possible,
+    whether at the time of the initial grant or subsequently, any and
+    all of the rights conveyed by this License.
+
+1.10. "Modifications"
+    means any of the following:
+
+    (a) any file in Source Code Form that results from an addition to,
+        deletion from, or modification of the contents of Covered
+        Software; or
+
+    (b) any new file in Source Code Form that contains any Covered
+        Software.
+
+1.11. "Patent Claims" of a Contributor
+    means any patent claim(s), including without limitation, method,
+    process, and apparatus claims, in any patent Licensable by such
+    Contributor that would be infringed, but for the grant of the
+    License, by the making, using, selling, offering for sale, having
+    made, import, or transfer of either its Contributions or its
+    Contributor Version.
+
+1.12. "Secondary License"
+    means either the GNU General Public License, Version 2.0, the GNU
+    Lesser General Public License, Version 2.1, the GNU Affero General
+    Public License, Version 3.0, or any later versions of those
+    licenses.
+
+1.13. "Source Code Form"
+    means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your")
+    means an individual or a legal entity exercising rights under this
+    License. For legal entities, "You" includes any entity that
+    controls, is controlled by, or is under common control with You. For
+    purposes of this definition, "control" means (a) the power, direct
+    or indirect, to cause the direction or management of such entity,
+    whether by contract or otherwise, or (b) ownership of more than
+    fifty percent (50%) of the outstanding shares or beneficial
+    ownership of such entity.
+
+2. License Grants and Conditions
+--------------------------------
+
+2.1. Grants
+
+Each Contributor hereby grants You a world-wide, royalty-free,
+non-exclusive license:
+
+(a) under intellectual property rights (other than patent or trademark)
+    Licensable by such Contributor to use, reproduce, make available,
+    modify, display, perform, distribute, and otherwise exploit its
+    Contributions, either on an unmodified basis, with Modifications, or
+    as part of a Larger Work; and
+
+(b) under Patent Claims of such Contributor to make, use, sell, offer
+    for sale, have made, import, and otherwise transfer either its
+    Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+The licenses granted in Section 2.1 with respect to any Contribution
+become effective for each Contribution on the date the Contributor first
+distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+The licenses granted in this Section 2 are the only rights granted under
+this License. No additional rights or licenses will be implied from the
+distribution or licensing of Covered Software under this License.
+Notwithstanding Section 2.1(b) above, no patent license is granted by a
+Contributor:
+
+(a) for any code that a Contributor has removed from Covered Software;
+    or
+
+(b) for infringements caused by: (i) Your and any other third party's
+    modifications of Covered Software, or (ii) the combination of its
+    Contributions with other software (except as part of its Contributor
+    Version); or
+
+(c) under Patent Claims infringed by Covered Software in the absence of
+    its Contributions.
+
+This License does not grant any rights in the trademarks, service marks,
+or logos of any Contributor (except as may be necessary to comply with
+the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+No Contributor makes additional grants as a result of Your choice to
+distribute the Covered Software under a subsequent version of this
+License (see Section 10.2) or under the terms of a Secondary License (if
+permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+Each Contributor represents that the Contributor believes its
+Contributions are its original creation(s) or it has sufficient rights
+to grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+This License is not intended to limit any rights You have under
+applicable copyright doctrines of fair use, fair dealing, or other
+equivalents.
+
+2.7. Conditions
+
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+in Section 2.1.
+
+3. Responsibilities
+-------------------
+
+3.1. Distribution of Source Form
+
+All distribution of Covered Software in Source Code Form, including any
+Modifications that You create or to which You contribute, must be under
+the terms of this License. You must inform recipients that the Source
+Code Form of the Covered Software is governed by the terms of this
+License, and how they can obtain a copy of this License. You may not
+attempt to alter or restrict the recipients' rights in the Source Code
+Form.
+
+3.2. Distribution of Executable Form
+
+If You distribute Covered Software in Executable Form then:
+
+(a) such Covered Software must also be made available in Source Code
+    Form, as described in Section 3.1, and You must inform recipients of
+    the Executable Form how they can obtain a copy of such Source Code
+    Form by reasonable means in a timely manner, at a charge no more
+    than the cost of distribution to the recipient; and
+
+(b) You may distribute such Executable Form under the terms of this
+    License, or sublicense it under different terms, provided that the
+    license for the Executable Form does not attempt to limit or alter
+    the recipients' rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+You may create and distribute a Larger Work under terms of Your choice,
+provided that You also comply with the requirements of this License for
+the Covered Software. If the Larger Work is a combination of Covered
+Software with a work governed by one or more Secondary Licenses, and the
+Covered Software is not Incompatible With Secondary Licenses, this
+License permits You to additionally distribute such Covered Software
+under the terms of such Secondary License(s), so that the recipient of
+the Larger Work may, at their option, further distribute the Covered
+Software under the terms of either this License or such Secondary
+License(s).
+
+3.4. Notices
+
+You may not remove or alter the substance of any license notices
+(including copyright notices, patent notices, disclaimers of warranty,
+or limitations of liability) contained within the Source Code Form of
+the Covered Software, except that You may alter any license notices to
+the extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+You may choose to offer, and to charge a fee for, warranty, support,
+indemnity or liability obligations to one or more recipients of Covered
+Software. However, You may do so only on Your own behalf, and not on
+behalf of any Contributor. You must make it absolutely clear that any
+such warranty, support, indemnity, or liability obligation is offered by
+You alone, and You hereby agree to indemnify every Contributor for any
+liability incurred by such Contributor as a result of warranty, support,
+indemnity or liability terms You offer. You may include additional
+disclaimers of warranty and limitations of liability specific to any
+jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+---------------------------------------------------
+
+If it is impossible for You to comply with any of the terms of this
+License with respect to some or all of the Covered Software due to
+statute, judicial order, or regulation then You must: (a) comply with
+the terms of this License to the maximum extent possible; and (b)
+describe the limitations and the code they affect. Such description must
+be placed in a text file included with all distributions of the Covered
+Software under this License. Except to the extent prohibited by statute
+or regulation, such description must be sufficiently detailed for a
+recipient of ordinary skill to be able to understand it.
+
+5. Termination
+--------------
+
+5.1. The rights granted under this License will terminate automatically
+if You fail to comply with any of its terms. However, if You become
+compliant, then the rights granted under this License from a particular
+Contributor are reinstated (a) provisionally, unless and until such
+Contributor explicitly and finally terminates Your grants, and (b) on an
+ongoing basis, if such Contributor fails to notify You of the
+non-compliance by some reasonable means prior to 60 days after You have
+come back into compliance. Moreover, Your grants from a particular
+Contributor are reinstated on an ongoing basis if such Contributor
+notifies You of the non-compliance by some reasonable means, this is the
+first time You have received notice of non-compliance with this License
+from such Contributor, and You become compliant prior to 30 days after
+Your receipt of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+infringement claim (excluding declaratory judgment actions,
+counter-claims, and cross-claims) alleging that a Contributor Version
+directly or indirectly infringes any patent, then the rights granted to
+You by any and all Contributors for the Covered Software under Section
+2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+end user license agreements (excluding distributors and resellers) which
+have been validly granted by You or Your distributors under this License
+prior to termination shall survive termination.
+
+************************************************************************
+*                                                                      *
+*  6. Disclaimer of Warranty                                           *
+*  -------------------------                                           *
+*                                                                      *
+*  Covered Software is provided under this License on an "as is"       *
+*  basis, without warranty of any kind, either expressed, implied, or  *
+*  statutory, including, without limitation, warranties that the       *
+*  Covered Software is free of defects, merchantable, fit for a        *
+*  particular purpose or non-infringing. The entire risk as to the     *
+*  quality and performance of the Covered Software is with You.        *
+*  Should any Covered Software prove defective in any respect, You     *
+*  (not any Contributor) assume the cost of any necessary servicing,   *
+*  repair, or correction. This disclaimer of warranty constitutes an   *
+*  essential part of this License. No use of any Covered Software is   *
+*  authorized under this License except under this disclaimer.         *
+*                                                                      *
+************************************************************************
+
+************************************************************************
+*                                                                      *
+*  7. Limitation of Liability                                          *
+*  --------------------------                                          *
+*                                                                      *
+*  Under no circumstances and under no legal theory, whether tort      *
+*  (including negligence), contract, or otherwise, shall any           *
+*  Contributor, or anyone who distributes Covered Software as          *
+*  permitted above, be liable to You for any direct, indirect,         *
+*  special, incidental, or consequential damages of any character      *
+*  including, without limitation, damages for lost profits, loss of    *
+*  goodwill, work stoppage, computer failure or malfunction, or any    *
+*  and all other commercial damages or losses, even if such party      *
+*  shall have been informed of the possibility of such damages. This   *
+*  limitation of liability shall not apply to liability for death or   *
+*  personal injury resulting from such party's negligence to the       *
+*  extent applicable law prohibits such limitation. Some               *
+*  jurisdictions do not allow the exclusion or limitation of           *
+*  incidental or consequential damages, so this exclusion and          *
+*  limitation may not apply to You.                                    *
+*                                                                      *
+************************************************************************
+
+8. Litigation
+-------------
+
+Any litigation relating to this License may be brought only in the
+courts of a jurisdiction where the defendant maintains its principal
+place of business and such litigation shall be governed by laws of that
+jurisdiction, without reference to its conflict-of-law provisions.
+Nothing in this Section shall prevent a party's ability to bring
+cross-claims or counter-claims.
+
+9. Miscellaneous
+----------------
+
+This License represents the complete agreement concerning the subject
+matter hereof. If any provision of this License is held to be
+unenforceable, such provision shall be reformed only to the extent
+necessary to make it enforceable. Any law or regulation which provides
+that the language of a contract shall be construed against the drafter
+shall not be used to construe this License against a Contributor.
+
+10. Versions of the License
+---------------------------
+
+10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in Section
+10.3, no one other than the license steward has the right to modify or
+publish new versions of this License. Each version will be given a
+distinguishing version number.
+
+10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version
+of the License under which You originally received the Covered Software,
+or under the terms of any subsequent version published by the license
+steward.
+
+10.3. Modified Versions
+
+If you create software not governed by this License, and you want to
+create a new license for such software, you may create and use a
+modified version of this License if you rename the license and remove
+any references to the name of the license steward (except to note that
+such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With
+Secondary Licenses under the terms of this version of the License, the
+notice described in Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+-------------------------------------------
+
+  This Source Code Form is subject to the terms of the Mozilla Public
+  License, v. 2.0. If a copy of the MPL was not distributed with this
+  file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular
+file, then You may include the notice in a location (such as a LICENSE
+file in a relevant directory) where a recipient would be likely to look
+for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+---------------------------------------------------------
+
+  This Source Code Form is "Incompatible With Secondary Licenses", as
+  defined by the Mozilla Public License, v. 2.0.
diff --git a/modules/reindex/licenses/httpclient-NOTICE.txt b/modules/reindex/licenses/httpclient-NOTICE.txt
new file mode 100644
index 00000000000..4f6058178b2
--- /dev/null
+++ b/modules/reindex/licenses/httpclient-NOTICE.txt
@@ -0,0 +1,5 @@
+Apache HttpComponents Client
+Copyright 1999-2015 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
diff --git a/modules/reindex/licenses/httpcore-4.4.4.jar.sha1 b/modules/reindex/licenses/httpcore-4.4.4.jar.sha1
new file mode 100644
index 00000000000..ef0c257e012
--- /dev/null
+++ b/modules/reindex/licenses/httpcore-4.4.4.jar.sha1
@@ -0,0 +1 @@
+b31526a230871fbe285fbcbe2813f9c0839ae9b0
\ No newline at end of file
diff --git a/modules/reindex/licenses/httpcore-LICENSE.txt b/modules/reindex/licenses/httpcore-LICENSE.txt
new file mode 100644
index 00000000000..72819a9f06f
--- /dev/null
+++ b/modules/reindex/licenses/httpcore-LICENSE.txt
@@ -0,0 +1,241 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+=========================================================================
+
+This project contains annotations in the package org.apache.http.annotation
+which are derived from JCIP-ANNOTATIONS
+Copyright (c) 2005 Brian Goetz and Tim Peierls.
+See http://www.jcip.net and the Creative Commons Attribution License
+(http://creativecommons.org/licenses/by/2.5)
+Full text: http://creativecommons.org/licenses/by/2.5/legalcode
+
+License
+
+THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED.
+
+BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE BOUND BY THE TERMS OF THIS LICENSE. THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS.
+
+1. Definitions
+
+    "Collective Work" means a work, such as a periodical issue, anthology or encyclopedia, in which the Work in its entirety in unmodified form, along with a number of other contributions, constituting separate and independent works in themselves, are assembled into a collective whole. A work that constitutes a Collective Work will not be considered a Derivative Work (as defined below) for the purposes of this License.
+    "Derivative Work" means a work based upon the Work or upon the Work and other pre-existing works, such as a translation, musical arrangement, dramatization, fictionalization, motion picture version, sound recording, art reproduction, abridgment, condensation, or any other form in which the Work may be recast, transformed, or adapted, except that a work that constitutes a Collective Work will not be considered a Derivative Work for the purpose of this License. For the avoidance of doubt, where the Work is a musical composition or sound recording, the synchronization of the Work in timed-relation with a moving image ("synching") will be considered a Derivative Work for the purpose of this License.
+    "Licensor" means the individual or entity that offers the Work under the terms of this License.
+    "Original Author" means the individual or entity who created the Work.
+    "Work" means the copyrightable work of authorship offered under the terms of this License.
+    "You" means an individual or entity exercising rights under this License who has not previously violated the terms of this License with respect to the Work, or who has received express permission from the Licensor to exercise rights under this License despite a previous violation.
+
+2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or restrict any rights arising from fair use, first sale or other limitations on the exclusive rights of the copyright owner under copyright law or other applicable laws.
+
+3. License Grant. Subject to the terms and conditions of this License, Licensor hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the duration of the applicable copyright) license to exercise the rights in the Work as stated below:
+
+    to reproduce the Work, to incorporate the Work into one or more Collective Works, and to reproduce the Work as incorporated in the Collective Works;
+    to create and reproduce Derivative Works;
+    to distribute copies or phonorecords of, display publicly, perform publicly, and perform publicly by means of a digital audio transmission the Work including as incorporated in Collective Works;
+    to distribute copies or phonorecords of, display publicly, perform publicly, and perform publicly by means of a digital audio transmission Derivative Works.
+
+    For the avoidance of doubt, where the work is a musical composition:
+        Performance Royalties Under Blanket Licenses. Licensor waives the exclusive right to collect, whether individually or via a performance rights society (e.g. ASCAP, BMI, SESAC), royalties for the public performance or public digital performance (e.g. webcast) of the Work.
+        Mechanical Rights and Statutory Royalties. Licensor waives the exclusive right to collect, whether individually or via a music rights agency or designated agent (e.g. Harry Fox Agency), royalties for any phonorecord You create from the Work ("cover version") and distribute, subject to the compulsory license created by 17 USC Section 115 of the US Copyright Act (or the equivalent in other jurisdictions).
+    Webcasting Rights and Statutory Royalties. For the avoidance of doubt, where the Work is a sound recording, Licensor waives the exclusive right to collect, whether individually or via a performance-rights society (e.g. SoundExchange), royalties for the public digital performance (e.g. webcast) of the Work, subject to the compulsory license created by 17 USC Section 114 of the US Copyright Act (or the equivalent in other jurisdictions).
+
+The above rights may be exercised in all media and formats whether now known or hereafter devised. The above rights include the right to make such modifications as are technically necessary to exercise the rights in other media and formats. All rights not expressly granted by Licensor are hereby reserved.
+
+4. Restrictions.The license granted in Section 3 above is expressly made subject to and limited by the following restrictions:
+
+    You may distribute, publicly display, publicly perform, or publicly digitally perform the Work only under the terms of this License, and You must include a copy of, or the Uniform Resource Identifier for, this License with every copy or phonorecord of the Work You distribute, publicly display, publicly perform, or publicly digitally perform. You may not offer or impose any terms on the Work that alter or restrict the terms of this License or the recipients' exercise of the rights granted hereunder. You may not sublicense the Work. You must keep intact all notices that refer to this License and to the disclaimer of warranties. You may not distribute, publicly display, publicly perform, or publicly digitally perform the Work with any technological measures that control access or use of the Work in a manner inconsistent with the terms of this License Agreement. The above applies to the Work as incorporated in a Collective Work, but this does not require the Collective Work apart from the Work itself to be made subject to the terms of this License. If You create a Collective Work, upon notice from any Licensor You must, to the extent practicable, remove from the Collective Work any credit as required by clause 4(b), as requested. If You create a Derivative Work, upon notice from any Licensor You must, to the extent practicable, remove from the Derivative Work any credit as required by clause 4(b), as requested.
+    If you distribute, publicly display, publicly perform, or publicly digitally perform the Work or any Derivative Works or Collective Works, You must keep intact all copyright notices for the Work and provide, reasonable to the medium or means You are utilizing: (i) the name of the Original Author (or pseudonym, if applicable) if supplied, and/or (ii) if the Original Author and/or Licensor designate another party or parties (e.g. a sponsor institute, publishing entity, journal) for attribution in Licensor's copyright notice, terms of service or by other reasonable means, the name of such party or parties; the title of the Work if supplied; to the extent reasonably practicable, the Uniform Resource Identifier, if any, that Licensor specifies to be associated with the Work, unless such URI does not refer to the copyright notice or licensing information for the Work; and in the case of a Derivative Work, a credit identifying the use of the Work in the Derivative Work (e.g., "French translation of the Work by Original Author," or "Screenplay based on original Work by Original Author"). Such credit may be implemented in any reasonable manner; provided, however, that in the case of a Derivative Work or Collective Work, at a minimum such credit will appear where any other comparable authorship credit appears and in a manner at least as prominent as such other comparable authorship credit.
+
+5. Representations, Warranties and Disclaimer
+
+UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU.
+
+6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+7. Termination
+
+    This License and the rights granted hereunder will terminate automatically upon any breach by You of the terms of this License. Individuals or entities who have received Derivative Works or Collective Works from You under this License, however, will not have their licenses terminated provided such individuals or entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will survive any termination of this License.
+    Subject to the above terms and conditions, the license granted here is perpetual (for the duration of the applicable copyright in the Work). Notwithstanding the above, Licensor reserves the right to release the Work under different license terms or to stop distributing the Work at any time; provided, however that any such election will not serve to withdraw this License (or any other license that has been, or is required to be, granted under the terms of this License), and this License will continue in full force and effect unless terminated as stated above.
+
+8. Miscellaneous
+
+    Each time You distribute or publicly digitally perform the Work or a Collective Work, the Licensor offers to the recipient a license to the Work on the same terms and conditions as the license granted to You under this License.
+    Each time You distribute or publicly digitally perform a Derivative Work, Licensor offers to the recipient a license to the original Work on the same terms and conditions as the license granted to You under this License.
+    If any provision of this License is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this License, and without further action by the parties to this agreement, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable.
+    No term or provision of this License shall be deemed waived and no breach consented to unless such waiver or consent shall be in writing and signed by the party to be charged with such waiver or consent.
+    This License constitutes the entire agreement between the parties with respect to the Work licensed here. There are no understandings, agreements or representations with respect to the Work not specified here. Licensor shall not be bound by any additional provisions that may appear in any communication from You. This License may not be modified without the mutual written agreement of the Licensor and You.
diff --git a/modules/reindex/licenses/httpcore-NOTICE.txt b/modules/reindex/licenses/httpcore-NOTICE.txt
new file mode 100644
index 00000000000..c0be50a505e
--- /dev/null
+++ b/modules/reindex/licenses/httpcore-NOTICE.txt
@@ -0,0 +1,8 @@
+Apache HttpComponents Core
+Copyright 2005-2014 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+This project contains annotations derived from JCIP-ANNOTATIONS
+Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
index c8a791a7b71..584dd022932 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkByScrollAction.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.index.reindex;
 
-import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
@@ -30,34 +29,24 @@ import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.bulk.Retry;
 import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.action.search.ClearScrollRequest;
-import org.elasticsearch.action.search.ClearScrollResponse;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchScrollRequest;
-import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.client.ParentTaskAssigningClient;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
-import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
+import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Consumer;
 
 import static java.lang.Math.max;
 import static java.lang.Math.min;
@@ -74,46 +63,57 @@ import static org.elasticsearch.search.sort.SortBuilders.fieldSort;
  * their tests can use them. Most methods run in the listener thread pool because the are meant to be fast and don't expect to block.
  */
 public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBulkByScrollRequest<Request>> {
+    protected final ESLogger logger;
+    protected final BulkByScrollTask task;
+    protected final ThreadPool threadPool;
     /**
      * The request for this action. Named mainRequest because we create lots of <code>request</code> variables all representing child
      * requests of this mainRequest.
      */
     protected final Request mainRequest;
-    protected final BulkByScrollTask task;
 
     private final AtomicLong startTime = new AtomicLong(-1);
-    private final AtomicReference<String> scroll = new AtomicReference<>();
     private final Set<String> destinationIndices = Collections.newSetFromMap(new ConcurrentHashMap<>());
 
-    private final ESLogger logger;
     private final ParentTaskAssigningClient client;
-    private final ThreadPool threadPool;
-    private final SearchRequest firstSearchRequest;
     private final ActionListener<BulkIndexByScrollResponse> listener;
-    private final BackoffPolicy backoffPolicy;
     private final Retry bulkRetry;
+    private final ScrollableHitSource scrollSource; 
 
     public AbstractAsyncBulkByScrollAction(BulkByScrollTask task, ESLogger logger, ParentTaskAssigningClient client,
-                                           ThreadPool threadPool, Request mainRequest, SearchRequest firstSearchRequest,
-                                           ActionListener<BulkIndexByScrollResponse> listener) {
+                                           ThreadPool threadPool, Request mainRequest, ActionListener<BulkIndexByScrollResponse> listener) {
         this.task = task;
         this.logger = logger;
         this.client = client;
         this.threadPool = threadPool;
         this.mainRequest = mainRequest;
-        this.firstSearchRequest = firstSearchRequest;
         this.listener = listener;
-        backoffPolicy = buildBackoffPolicy();
-        bulkRetry = Retry.on(EsRejectedExecutionException.class).policy(wrapBackoffPolicy(backoffPolicy));
+        BackoffPolicy backoffPolicy = buildBackoffPolicy();
+        bulkRetry = Retry.on(EsRejectedExecutionException.class).policy(BackoffPolicy.wrap(backoffPolicy, task::countBulkRetry));
+        scrollSource = buildScrollableResultSource(backoffPolicy);
+        /*
+         * Default to sorting by doc. We can't do this in the request itself because it is normal to *add* to the sorts rather than replace
+         * them and if we add _doc as the first sort by default then sorts will never work.... So we add it here, only if there isn't
+         * another sort.
+         */
+        List<SortBuilder<?>> sorts = mainRequest.getSearchRequest().source().sorts();
+        if (sorts == null || sorts.isEmpty()) {
+            mainRequest.getSearchRequest().source().sort(fieldSort("_doc"));
+        }
     }
 
-    protected abstract BulkRequest buildBulk(Iterable<SearchHit> docs);
+    protected abstract BulkRequest buildBulk(Iterable<? extends ScrollableHitSource.Hit> docs);
+
+    protected ScrollableHitSource buildScrollableResultSource(BackoffPolicy backoffPolicy) {
+        return new ClientScrollableHitSource(logger, backoffPolicy, threadPool, task::countSearchRetry, this::finishHim, client,
+                mainRequest.getSearchRequest());
+    }
 
     /**
      * Build the response for reindex actions.
      */
     protected BulkIndexByScrollResponse buildResponse(TimeValue took, List<BulkItemResponse.Failure> indexingFailures,
-                                                      List<ShardSearchFailure> searchFailures, boolean timedOut) {
+                                                      List<SearchFailure> searchFailures, boolean timedOut) {
         return new BulkIndexByScrollResponse(took, task.getStatus(), indexingFailures, searchFailures, timedOut);
     }
 
@@ -126,50 +126,33 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
             return;
         }
         try {
-            // Default to sorting by _doc if it hasn't been changed.
-            if (firstSearchRequest.source().sorts() == null) {
-                firstSearchRequest.source().sort(fieldSort("_doc"));
-            }
             startTime.set(System.nanoTime());
-            if (logger.isDebugEnabled()) {
-                logger.debug("executing initial scroll against {}{}",
-                        firstSearchRequest.indices() == null || firstSearchRequest.indices().length == 0 ? "all indices"
-                                : firstSearchRequest.indices(),
-                        firstSearchRequest.types() == null || firstSearchRequest.types().length == 0 ? ""
-                                : firstSearchRequest.types());
-            }
+            scrollSource.start(response -> onScrollResponse(timeValueNanos(System.nanoTime()), 0, response));
         } catch (Exception e) {
             finishHim(e);
-            return;
         }
-        searchWithRetry(listener -> client.search(firstSearchRequest, listener), (SearchResponse response) -> {
-            logger.debug("[{}] documents match query", response.getHits().getTotalHits());
-            onScrollResponse(timeValueNanos(System.nanoTime()), 0, response);
-        });
     }
 
     /**
      * Process a scroll response.
      * @param lastBatchStartTime the time when the last batch started. Used to calculate the throttling delay.
      * @param lastBatchSize the size of the last batch. Used to calculate the throttling delay.
-     * @param searchResponse the scroll response to process
+     * @param response the scroll response to process
      */
-    void onScrollResponse(TimeValue lastBatchStartTime, int lastBatchSize, SearchResponse searchResponse) {
+    void onScrollResponse(TimeValue lastBatchStartTime, int lastBatchSize, ScrollableHitSource.Response response) {
         if (task.isCancelled()) {
             finishHim(null);
             return;
         }
-        setScroll(searchResponse.getScrollId());
         if (    // If any of the shards failed that should abort the request.
-                (searchResponse.getShardFailures() != null && searchResponse.getShardFailures().length > 0)
+                (response.getFailures().size() > 0)
                 // Timeouts aren't shard failures but we still need to pass them back to the user.
-                || searchResponse.isTimedOut()
+                || response.isTimedOut()
                 ) {
-            startNormalTermination(emptyList(), unmodifiableList(Arrays.asList(searchResponse.getShardFailures())),
-                    searchResponse.isTimedOut());
+            refreshAndFinish(emptyList(), response.getFailures(), response.isTimedOut());
             return;
         }
-        long total = searchResponse.getHits().totalHits();
+        long total = response.getTotalHits();
         if (mainRequest.getSize() > 0) {
             total = min(total, mainRequest.getSize());
         }
@@ -181,7 +164,7 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
                  * It is important that the batch start time be calculated from here, scroll response to scroll response. That way the time
                  * waiting on the scroll doesn't count against this batch in the throttle.
                  */
-                prepareBulkRequest(timeValueNanos(System.nanoTime()), searchResponse);
+                prepareBulkRequest(timeValueNanos(System.nanoTime()), response);
             }
 
             @Override
@@ -198,27 +181,25 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
      * delay has been slept. Uses the generic thread pool because reindex is rare enough not to need its own thread pool and because the
      * thread may be blocked by the user script.
      */
-    void prepareBulkRequest(TimeValue thisBatchStartTime, SearchResponse searchResponse) {
+    void prepareBulkRequest(TimeValue thisBatchStartTime, ScrollableHitSource.Response response) {
         if (task.isCancelled()) {
             finishHim(null);
             return;
         }
-        SearchHit[] docs = searchResponse.getHits().getHits();
-        logger.debug("scroll returned [{}] documents with a scroll id of [{}]", docs.length, searchResponse.getScrollId());
-        if (docs.length == 0) {
-            startNormalTermination(emptyList(), emptyList(), false);
+        if (response.getHits().isEmpty()) {
+            refreshAndFinish(emptyList(), emptyList(), false);
             return;
         }
         task.countBatch();
-        List<SearchHit> docsIterable = Arrays.asList(docs);
+        List<? extends ScrollableHitSource.Hit> hits = response.getHits();
         if (mainRequest.getSize() != SIZE_ALL_MATCHES) {
-            // Truncate the docs if we have more than the request size
+            // Truncate the hits if we have more than the request size
             long remaining = max(0, mainRequest.getSize() - task.getSuccessfullyProcessed());
-            if (remaining < docs.length) {
-                docsIterable = docsIterable.subList(0, (int) remaining);
+            if (remaining < hits.size()) {
+                hits = hits.subList(0, (int) remaining);
             }
         }
-        BulkRequest request = buildBulk(docsIterable);
+        BulkRequest request = buildBulk(hits);
         if (request.requests().isEmpty()) {
             /*
              * If we noop-ed the entire batch then just skip to the next batch or the BulkRequest would fail validation.
@@ -297,13 +278,13 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
             addDestinationIndices(destinationIndicesThisBatch);
 
             if (false == failures.isEmpty()) {
-                startNormalTermination(unmodifiableList(failures), emptyList(), false);
+                refreshAndFinish(unmodifiableList(failures), emptyList(), false);
                 return;
             }
 
             if (mainRequest.getSize() != SIZE_ALL_MATCHES && task.getSuccessfullyProcessed() >= mainRequest.getSize()) {
                 // We've processed all the requested docs.
-                startNormalTermination(emptyList(), emptyList(), false);
+                refreshAndFinish(emptyList(), emptyList(), false);
                 return;
             }
 
@@ -324,11 +305,8 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
             finishHim(null);
             return;
         }
-        SearchScrollRequest request = new SearchScrollRequest();
-        // Add the wait time into the scroll timeout so it won't timeout while we wait for throttling
-        request.scrollId(scroll.get()).scroll(timeValueNanos(
-                firstSearchRequest.scroll().keepAlive().nanos() + task.throttleWaitTime(lastBatchStartTime, lastBatchSize).nanos()));
-        searchWithRetry(listener -> client.searchScroll(request, listener), (SearchResponse response) -> {
+        TimeValue extraKeepAlive = task.throttleWaitTime(lastBatchStartTime, lastBatchSize);
+        scrollSource.startNextScroll(extraKeepAlive, response -> {
             onScrollResponse(lastBatchStartTime, lastBatchSize, response);
         });
     }
@@ -344,9 +322,10 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
     }
 
     /**
-     * Start terminating a request that finished non-catastrophically.
+     * Start terminating a request that finished non-catastrophically by refreshing the modified indices and then proceeding to
+     * {@link #finishHim(Exception, List, List, boolean)}.
      */
-    void startNormalTermination(List<Failure> indexingFailures, List<ShardSearchFailure> searchFailures, boolean timedOut) {
+    void refreshAndFinish(List<Failure> indexingFailures, List<SearchFailure> searchFailures, boolean timedOut) {
         if (task.isCancelled() || false == mainRequest.isRefresh() || destinationIndices.isEmpty()) {
             finishHim(null, indexingFailures, searchFailures, timedOut);
             return;
@@ -377,36 +356,13 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
 
     /**
      * Finish the request.
-     *  @param failure if non null then the request failed catastrophically with this exception
+     * @param failure if non null then the request failed catastrophically with this exception
      * @param indexingFailures any indexing failures accumulated during the request
      * @param searchFailures any search failures accumulated during the request
      * @param timedOut have any of the sub-requests timed out?
      */
-    void finishHim(Exception failure, List<Failure> indexingFailures, List<ShardSearchFailure> searchFailures, boolean timedOut) {
-        String scrollId = scroll.get();
-        if (Strings.hasLength(scrollId)) {
-            /*
-             * Fire off the clear scroll but don't wait for it it return before
-             * we send the use their response.
-             */
-            ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
-            clearScrollRequest.addScrollId(scrollId);
-            /*
-             * Unwrap the client so we don't set our task as the parent. If we *did* set our ID then the clear scroll would be cancelled as
-             * if this task is cancelled. But we want to clear the scroll regardless of whether or not the main request was cancelled.
-             */
-            client.unwrap().clearScroll(clearScrollRequest, new ActionListener<ClearScrollResponse>() {
-                @Override
-                public void onResponse(ClearScrollResponse response) {
-                    logger.debug("Freed [{}] contexts", response.getNumFreed());
-                }
-
-                @Override
-                public void onFailure(Exception e) {
-                    logger.warn("Failed to clear scroll [{}]", e, scrollId);
-                }
-            });
-        }
+    void finishHim(Exception failure, List<Failure> indexingFailures, List<SearchFailure> searchFailures, boolean timedOut) {
+        scrollSource.close();
         if (failure == null) {
             listener.onResponse(
                     buildResponse(timeValueNanos(System.nanoTime() - startTime.get()), indexingFailures, searchFailures, timedOut));
@@ -434,75 +390,6 @@ public abstract class AbstractAsyncBulkByScrollAction<Request extends AbstractBu
      * Set the last returned scrollId. Exists entirely for testing.
      */
     void setScroll(String scroll) {
-        this.scroll.set(scroll);
-    }
-
-    /**
-     * Wraps a backoffPolicy in another policy that counts the number of backoffs acquired. Used to count bulk backoffs.
-     */
-    private BackoffPolicy wrapBackoffPolicy(BackoffPolicy backoffPolicy) {
-        return new BackoffPolicy() {
-            @Override
-            public Iterator<TimeValue> iterator() {
-                return new Iterator<TimeValue>() {
-                    private final Iterator<TimeValue> delegate = backoffPolicy.iterator();
-                    @Override
-                    public boolean hasNext() {
-                        return delegate.hasNext();
-                    }
-
-                    @Override
-                    public TimeValue next() {
-                        if (false == delegate.hasNext()) {
-                            return null;
-                        }
-                        task.countBulkRetry();
-                        return delegate.next();
-                    }
-                };
-            }
-        };
-    }
-
-    /**
-     * Run a search action and call onResponse when a the response comes in, retrying if the action fails with an exception caused by
-     * rejected execution.
-     *
-     * @param action consumes a listener and starts the action. The listener it consumes is rigged to retry on failure.
-     * @param onResponse consumes the response from the action
-     */
-    private <T> void searchWithRetry(Consumer<ActionListener<T>> action, Consumer<T> onResponse) {
-        class RetryHelper extends AbstractRunnable implements ActionListener<T> {
-            private final Iterator<TimeValue> retries = backoffPolicy.iterator();
-
-            @Override
-            public void onResponse(T response) {
-                onResponse.accept(response);
-            }
-
-            @Override
-            protected void doRun() throws Exception {
-                action.accept(this);
-            }
-
-            @Override
-            public void onFailure(Exception e) {
-                if (ExceptionsHelper.unwrap(e, EsRejectedExecutionException.class) != null) {
-                    if (retries.hasNext()) {
-                        TimeValue delay = retries.next();
-                        logger.trace("retrying rejected search after [{}]", e, delay);
-                        threadPool.schedule(delay, ThreadPool.Names.SAME, this);
-                        task.countSearchRetry();
-                    } else {
-                        logger.warn("giving up on search because we retried {} times without success", e, retries);
-                        finishHim(e);
-                    }
-                } else {
-                    logger.warn("giving up on search because it failed with a non-retryable exception", e);
-                    finishHim(e);
-                }
-            }
-        }
-        new RetryHelper().run();
+        scrollSource.setScroll(scroll);
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java
index 60ab088d76c..4b87df46312 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.ParentTaskAssigningClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.common.logging.ESLogger;
@@ -44,8 +43,6 @@ import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.SearchHitField;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Arrays;
@@ -72,13 +69,13 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
      * from copying search hit metadata (parent, routing, etc) to potentially transforming the
      * {@link RequestWrapper} completely.
      */
-    private final BiFunction<RequestWrapper<?>, SearchHit, RequestWrapper<?>> scriptApplier;
+    private final BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> scriptApplier;
 
     public AbstractAsyncBulkIndexByScrollAction(BulkByScrollTask task, ESLogger logger, ParentTaskAssigningClient client,
-                                                ThreadPool threadPool, Request mainRequest, SearchRequest firstSearchRequest,
+                                                ThreadPool threadPool, Request mainRequest,
                                                 ActionListener<BulkIndexByScrollResponse> listener,
                                                 ScriptService scriptService, ClusterState clusterState) {
-        super(task, logger, client, threadPool, mainRequest, firstSearchRequest, listener);
+        super(task, logger, client, threadPool, mainRequest, listener);
         this.scriptService = scriptService;
         this.clusterState = clusterState;
         this.scriptApplier = Objects.requireNonNull(buildScriptApplier(), "script applier must not be null");
@@ -87,15 +84,15 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
     /**
      * Build the {@link BiFunction} to apply to all {@link RequestWrapper}.
      */
-    protected BiFunction<RequestWrapper<?>, SearchHit, RequestWrapper<?>> buildScriptApplier() {
+    protected BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> buildScriptApplier() {
         // The default script applier executes a no-op
         return (request, searchHit) -> request;
     }
 
     @Override
-    protected BulkRequest buildBulk(Iterable<SearchHit> docs) {
+    protected BulkRequest buildBulk(Iterable<? extends ScrollableHitSource.Hit> docs) {
         BulkRequest bulkRequest = new BulkRequest();
-        for (SearchHit doc : docs) {
+        for (ScrollableHitSource.Hit doc : docs) {
             if (accept(doc)) {
                 RequestWrapper<?> request = scriptApplier.apply(copyMetadata(buildRequest(doc), doc), doc);
                 if (request != null) {
@@ -111,14 +108,14 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
      * from the bulk request. It is also where we fail on invalid search hits, like
      * when the document has no source but it's required.
      */
-    protected boolean accept(SearchHit doc) {
-        if (doc.hasSource()) {
+    protected boolean accept(ScrollableHitSource.Hit doc) {
+        if (doc.getSource() == null) {
             /*
              * Either the document didn't store _source or we didn't fetch it for some reason. Since we don't allow the user to
              * change the "fields" part of the search request it is unlikely that we got here because we didn't fetch _source.
              * Thus the error message assumes that it wasn't stored.
              */
-            throw new IllegalArgumentException("[" + doc.index() + "][" + doc.type() + "][" + doc.id() + "] didn't store _source");
+            throw new IllegalArgumentException("[" + doc.getIndex() + "][" + doc.getType() + "][" + doc.getId() + "] didn't store _source");
         }
         return true;
     }
@@ -128,34 +125,27 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
      * metadata or scripting. That will be handled by copyMetadata and
      * apply functions that can be overridden.
      */
-    protected abstract RequestWrapper<?> buildRequest(SearchHit doc);
+    protected abstract RequestWrapper<?> buildRequest(ScrollableHitSource.Hit doc);
 
     /**
      * Copies the metadata from a hit to the request.
      */
-    protected RequestWrapper<?> copyMetadata(RequestWrapper<?> request, SearchHit doc) {
-        copyParent(request, fieldValue(doc, ParentFieldMapper.NAME));
-        copyRouting(request, fieldValue(doc, RoutingFieldMapper.NAME));
+    protected RequestWrapper<?> copyMetadata(RequestWrapper<?> request, ScrollableHitSource.Hit doc) {
+        request.setParent(doc.getParent());
+        copyRouting(request, doc.getRouting());
 
         // Comes back as a Long but needs to be a string
-        Long timestamp = fieldValue(doc, TimestampFieldMapper.NAME);
+        Long timestamp = doc.getTimestamp();
         if (timestamp != null) {
             request.setTimestamp(timestamp.toString());
         }
-        Long ttl = fieldValue(doc, TTLFieldMapper.NAME);
+        Long ttl = doc.getTTL();
         if (ttl != null) {
             request.setTtl(ttl);
         }
         return request;
     }
 
-    /**
-     * Copy the parent from a search hit to the request.
-     */
-    protected void copyParent(RequestWrapper<?> request, String parent) {
-        request.setParent(parent);
-    }
-
     /**
      * Copy the routing from a search hit to the request.
      */
@@ -163,11 +153,6 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
         request.setRouting(routing);
     }
 
-    protected <T> T fieldValue(SearchHit doc, String fieldName) {
-        SearchHitField field = doc.field(fieldName);
-        return field == null ? null : field.value();
-    }
-
     /**
      * Wrapper for the {@link ActionRequest} that are used in this action class.
      */
@@ -435,7 +420,7 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
     /**
      * Apply a {@link Script} to a {@link RequestWrapper}
      */
-    public abstract class ScriptApplier implements BiFunction<RequestWrapper<?>, SearchHit, RequestWrapper<?>> {
+    public abstract class ScriptApplier implements BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> {
 
         private final BulkByScrollTask task;
         private final ScriptService scriptService;
@@ -455,7 +440,7 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
 
         @Override
         @SuppressWarnings("unchecked")
-        public RequestWrapper<?> apply(RequestWrapper<?> request, SearchHit doc) {
+        public RequestWrapper<?> apply(RequestWrapper<?> request, ScrollableHitSource.Hit doc) {
             if (script == null) {
                 return request;
             }
@@ -467,18 +452,18 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
                 context = new HashMap<>();
             }
 
-            context.put(IndexFieldMapper.NAME, doc.index());
-            context.put(TypeFieldMapper.NAME, doc.type());
-            context.put(IdFieldMapper.NAME, doc.id());
+            context.put(IndexFieldMapper.NAME, doc.getIndex());
+            context.put(TypeFieldMapper.NAME, doc.getType());
+            context.put(IdFieldMapper.NAME, doc.getId());
             Long oldVersion = doc.getVersion();
             context.put(VersionFieldMapper.NAME, oldVersion);
-            String oldParent = fieldValue(doc, ParentFieldMapper.NAME);
+            String oldParent = doc.getParent();
             context.put(ParentFieldMapper.NAME, oldParent);
-            String oldRouting = fieldValue(doc, RoutingFieldMapper.NAME);
+            String oldRouting = doc.getRouting();
             context.put(RoutingFieldMapper.NAME, oldRouting);
-            Long oldTimestamp = fieldValue(doc, TimestampFieldMapper.NAME);
+            Long oldTimestamp = doc.getTimestamp();
             context.put(TimestampFieldMapper.NAME, oldTimestamp);
-            Long oldTTL = fieldValue(doc, TTLFieldMapper.NAME);
+            Long oldTTL = doc.getTTL();
             context.put(TTLFieldMapper.NAME, oldTTL);
             context.put(SourceFieldMapper.NAME, request.getSource());
 
@@ -501,15 +486,15 @@ public abstract class AbstractAsyncBulkIndexByScrollAction<Request extends Abstr
             request.setSource((Map<String, Object>) resultCtx.remove(SourceFieldMapper.NAME));
 
             Object newValue = context.remove(IndexFieldMapper.NAME);
-            if (false == doc.index().equals(newValue)) {
+            if (false == doc.getIndex().equals(newValue)) {
                 scriptChangedIndex(request, newValue);
             }
             newValue = context.remove(TypeFieldMapper.NAME);
-            if (false == doc.type().equals(newValue)) {
+            if (false == doc.getType().equals(newValue)) {
                 scriptChangedType(request, newValue);
             }
             newValue = context.remove(IdFieldMapper.NAME);
-            if (false == doc.id().equals(newValue)) {
+            if (false == doc.getId().equals(newValue)) {
                 scriptChangedId(request, newValue);
             }
             newValue = context.remove(VersionFieldMapper.NAME);
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequest.java
index 13b8b9780b3..10fb0bc676e 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequest.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkIndexByScrollRequest.java
@@ -74,7 +74,7 @@ public abstract class AbstractBulkIndexByScrollRequest<Self extends AbstractBulk
     protected void searchToString(StringBuilder b) {
         super.searchToString(b);
         if (script != null) {
-            b.append(" updated with [").append(script).append(']');
+            b.append(" updated with ").append(script);
         }
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java
index bfd139746ca..d717c1ad4de 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponse.java
@@ -21,21 +21,18 @@ package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
-import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.List;
 
 import static java.lang.Math.min;
-import static java.util.Collections.unmodifiableList;
 import static java.util.Objects.requireNonNull;
-import static org.elasticsearch.action.search.ShardSearchFailure.readShardSearchFailure;
 
 /**
  * Response used for actions that index many documents using a scroll request.
@@ -43,18 +40,18 @@ import static org.elasticsearch.action.search.ShardSearchFailure.readShardSearch
 public class BulkIndexByScrollResponse extends ActionResponse implements ToXContent {
     private TimeValue took;
     private BulkByScrollTask.Status status;
-    private List<Failure> indexingFailures;
-    private List<ShardSearchFailure> searchFailures;
+    private List<Failure> bulkFailures;
+    private List<SearchFailure> searchFailures;
     private boolean timedOut;
 
     public BulkIndexByScrollResponse() {
     }
 
-    public BulkIndexByScrollResponse(TimeValue took, BulkByScrollTask.Status status, List<Failure> indexingFailures,
-                                     List<ShardSearchFailure> searchFailures, boolean timedOut) {
+    public BulkIndexByScrollResponse(TimeValue took, BulkByScrollTask.Status status, List<Failure> bulkFailures,
+                                     List<SearchFailure> searchFailures, boolean timedOut) {
         this.took = took;
         this.status = requireNonNull(status, "Null status not supported");
-        this.indexingFailures = indexingFailures;
+        this.bulkFailures = bulkFailures;
         this.searchFailures = searchFailures;
         this.timedOut = timedOut;
     }
@@ -113,17 +110,16 @@ public class BulkIndexByScrollResponse extends ActionResponse implements ToXCont
     }
 
     /**
-     * All of the indexing failures. Version conflicts are only included if the request sets abortOnVersionConflict to true (the
-     * default).
+     * All of the bulk failures. Version conflicts are only included if the request sets abortOnVersionConflict to true (the default).
      */
-    public List<Failure> getIndexingFailures() {
-        return indexingFailures;
+    public List<Failure> getBulkFailures() {
+        return bulkFailures;
     }
 
     /**
      * All search failures.
      */
-    public List<ShardSearchFailure> getSearchFailures() {
+    public List<SearchFailure> getSearchFailures() {
         return searchFailures;
     }
 
@@ -139,14 +135,8 @@ public class BulkIndexByScrollResponse extends ActionResponse implements ToXCont
         super.writeTo(out);
         took.writeTo(out);
         status.writeTo(out);
-        out.writeVInt(indexingFailures.size());
-        for (Failure failure: indexingFailures) {
-            failure.writeTo(out);
-        }
-        out.writeVInt(searchFailures.size());
-        for (ShardSearchFailure failure: searchFailures) {
-            failure.writeTo(out);
-        }
+        out.writeList(bulkFailures);
+        out.writeList(searchFailures);
         out.writeBoolean(timedOut);
     }
 
@@ -155,19 +145,9 @@ public class BulkIndexByScrollResponse extends ActionResponse implements ToXCont
         super.readFrom(in);
         took = new TimeValue(in);
         status = new BulkByScrollTask.Status(in);
-        int indexingFailuresCount = in.readVInt();
-        List<Failure> indexingFailures = new ArrayList<>(indexingFailuresCount);
-        for (int i = 0; i < indexingFailuresCount; i++) {
-            indexingFailures.add(new Failure(in));
-        }
-        this.indexingFailures = unmodifiableList(indexingFailures);
-        int searchFailuresCount = in.readVInt();
-        List<ShardSearchFailure> searchFailures = new ArrayList<>(searchFailuresCount);
-        for (int i = 0; i < searchFailuresCount; i++) {
-            searchFailures.add(readShardSearchFailure(in));
-        }
-        this.searchFailures = unmodifiableList(searchFailures);
-        this.timedOut = in.readBoolean();
+        bulkFailures = in.readList(Failure::new);
+        searchFailures = in.readList(SearchFailure::new);
+        timedOut = in.readBoolean();
     }
 
     @Override
@@ -176,15 +156,13 @@ public class BulkIndexByScrollResponse extends ActionResponse implements ToXCont
         builder.field("timed_out", timedOut);
         status.innerXContent(builder, params);
         builder.startArray("failures");
-        for (Failure failure: indexingFailures) {
+        for (Failure failure: bulkFailures) {
             builder.startObject();
             failure.toXContent(builder, params);
             builder.endObject();
         }
-        for (ShardSearchFailure failure: searchFailures) {
-            builder.startObject();
+        for (SearchFailure failure: searchFailures) {
             failure.toXContent(builder, params);
-            builder.endObject();
         }
         builder.endArray();
         return builder;
@@ -197,7 +175,7 @@ public class BulkIndexByScrollResponse extends ActionResponse implements ToXCont
         builder.append("took=").append(took).append(',');
         builder.append("timed_out=").append(timedOut).append(',');
         status.innerToString(builder);
-        builder.append(",indexing_failures=").append(getIndexingFailures().subList(0, min(3, getIndexingFailures().size())));
+        builder.append(",bulk_failures=").append(getBulkFailures().subList(0, min(3, getBulkFailures().size())));
         builder.append(",search_failures=").append(getSearchFailures().subList(0, min(3, getSearchFailures().size())));
         return builder.append(']').toString();
     }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java
index 72bf6957e12..6cfba3a302d 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java
@@ -21,9 +21,9 @@ package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
-import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestResponse;
@@ -61,13 +61,13 @@ public class BulkIndexByScrollResponseContentListener<R extends BulkIndexByScrol
         if (response.isTimedOut()) {
             status = RestStatus.REQUEST_TIMEOUT;
         }
-        for (Failure failure : response.getIndexingFailures()) {
+        for (Failure failure : response.getBulkFailures()) {
             if (failure.getStatus().getStatus() > status.getStatus()) {
                 status = failure.getStatus();
             }
         }
-        for (ShardSearchFailure failure: response.getSearchFailures()) {
-            RestStatus failureStatus = ExceptionsHelper.status(failure.getCause());
+        for (SearchFailure failure: response.getSearchFailures()) {
+            RestStatus failureStatus = ExceptionsHelper.status(failure.getReason());
             if (failureStatus.getStatus() > status.getStatus()) {
                 status = failureStatus;
             }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java
new file mode 100644
index 00000000000..5e694e2cf26
--- /dev/null
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ClientScrollableHitSource.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.bulk.BackoffPolicy;
+import org.elasticsearch.action.search.ClearScrollRequest;
+import org.elasticsearch.action.search.ClearScrollResponse;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchScrollRequest;
+import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.ParentTaskAssigningClient;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
+import org.elasticsearch.index.mapper.internal.RoutingFieldMapper;
+import org.elasticsearch.index.mapper.internal.TTLFieldMapper;
+import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHitField;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.function.Consumer;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.unmodifiableList;
+import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
+import static org.elasticsearch.common.util.CollectionUtils.isEmpty;
+
+/**
+ * A scrollable source of hits from a {@linkplain Client} instance.
+ */
+public class ClientScrollableHitSource extends ScrollableHitSource {
+    private final ParentTaskAssigningClient client;
+    private final SearchRequest firstSearchRequest;
+
+    public ClientScrollableHitSource(ESLogger logger, BackoffPolicy backoffPolicy, ThreadPool threadPool, Runnable countSearchRetry,
+            Consumer<Exception> fail, ParentTaskAssigningClient client, SearchRequest firstSearchRequest) {
+        super(logger, backoffPolicy, threadPool, countSearchRetry, fail);
+        this.client = client;
+        this.firstSearchRequest = firstSearchRequest;
+    }
+
+    @Override
+    public void doStart(Consumer<? super Response> onResponse) {
+        if (logger.isDebugEnabled()) {
+            logger.debug("executing initial scroll against {}{}",
+                    isEmpty(firstSearchRequest.indices()) ? "all indices" : firstSearchRequest.indices(),
+                    isEmpty(firstSearchRequest.types()) ? "" : firstSearchRequest.types());
+        }
+        searchWithRetry(listener -> client.search(firstSearchRequest, listener), r -> consume(r, onResponse));
+    }
+
+    @Override
+    protected void doStartNextScroll(String scrollId, TimeValue extraKeepAlive, Consumer<? super Response> onResponse) {
+        SearchScrollRequest request = new SearchScrollRequest();
+        // Add the wait time into the scroll timeout so it won't timeout while we wait for throttling
+        request.scrollId(scrollId).scroll(timeValueNanos(firstSearchRequest.scroll().keepAlive().nanos() + extraKeepAlive.nanos()));
+        searchWithRetry(listener -> client.searchScroll(request, listener), r -> consume(r, onResponse));
+    }
+
+    @Override
+    public void clearScroll(String scrollId) {
+        /*
+         * Fire off the clear scroll but don't wait for it it return before
+         * we send the use their response.
+         */
+        ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
+        clearScrollRequest.addScrollId(scrollId);
+        /*
+         * Unwrap the client so we don't set our task as the parent. If we *did* set our ID then the clear scroll would be cancelled as
+         * if this task is cancelled. But we want to clear the scroll regardless of whether or not the main request was cancelled.
+         */
+        client.unwrap().clearScroll(clearScrollRequest, new ActionListener<ClearScrollResponse>() {
+            @Override
+            public void onResponse(ClearScrollResponse response) {
+                logger.debug("Freed [{}] contexts", response.getNumFreed());
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                logger.warn("Failed to clear scroll [{}]", e, scrollId);
+            }
+        });
+    }
+
+    /**
+     * Run a search action and call onResponse when a the response comes in, retrying if the action fails with an exception caused by
+     * rejected execution.
+     *
+     * @param action consumes a listener and starts the action. The listener it consumes is rigged to retry on failure.
+     * @param onResponse consumes the response from the action
+     */
+    private void searchWithRetry(Consumer<ActionListener<SearchResponse>> action, Consumer<SearchResponse> onResponse) {
+        /*
+         * RetryHelper is both an AbstractRunnable and an ActionListener<SearchResponse> - meaning that it both starts the search and
+         * handles reacts to the results. The complexity is all in onFailure which either adapts the failure to the "fail" listener or
+         * retries the search. Since both AbstractRunnable and ActionListener define the onFailure method it is called for either failure
+         * to run the action (either while running or before starting) and for failure on the response from the action.
+         */
+        class RetryHelper extends AbstractRunnable implements ActionListener<SearchResponse> {
+            private final Iterator<TimeValue> retries = backoffPolicy.iterator();
+            private volatile int retryCount = 0;
+
+            @Override
+            protected void doRun() throws Exception {
+                action.accept(this);
+            }
+
+            @Override
+            public void onResponse(SearchResponse response) {
+                onResponse.accept(response);
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                if (ExceptionsHelper.unwrap(e, EsRejectedExecutionException.class) != null) {
+                    if (retries.hasNext()) {
+                        retryCount += 1;
+                        TimeValue delay = retries.next();
+                        logger.trace("retrying rejected search after [{}]", e, delay);
+                        countSearchRetry.run();
+                        threadPool.schedule(delay, ThreadPool.Names.SAME, this);
+                    } else {
+                        logger.warn("giving up on search because we retried [{}] times without success", e, retryCount);
+                        fail.accept(e);
+                    }
+                } else {
+                    logger.warn("giving up on search because it failed with a non-retryable exception", e);
+                    fail.accept(e);
+                }
+            }
+        }
+        new RetryHelper().run();
+    }
+
+    private void consume(SearchResponse response, Consumer<? super Response> onResponse) {
+        onResponse.accept(wrap(response));
+    }
+
+    private Response wrap(SearchResponse response) {
+        List<SearchFailure> failures;
+        if (response.getShardFailures() == null) {
+            failures = emptyList();
+        } else {
+            failures = new ArrayList<>(response.getShardFailures().length);
+            for (ShardSearchFailure failure: response.getShardFailures()) {
+                String nodeId = failure.shard() == null ? null : failure.shard().nodeId();
+                failures.add(new SearchFailure(failure.getCause(), failure.index(), failure.shardId(), nodeId));
+            }
+        }
+        List<Hit> hits;
+        if (response.getHits().getHits() == null || response.getHits().getHits().length == 0) {
+            hits = emptyList();
+        } else {
+            hits = new ArrayList<>(response.getHits().getHits().length);
+            for (SearchHit hit: response.getHits().getHits()) {
+                hits.add(new ClientHit(hit));
+            }
+            hits = unmodifiableList(hits);
+        }
+        return new Response(response.isTimedOut(), failures, response.getHits().getTotalHits(),
+                hits, response.getScrollId());
+    }
+
+    private static class ClientHit implements Hit {
+        private final SearchHit delegate;
+        private final BytesReference source;
+
+        public ClientHit(SearchHit delegate) {
+            this.delegate = delegate;
+            source = delegate.hasSource() ? null : delegate.getSourceRef();
+        }
+
+        @Override
+        public String getIndex() {
+            return delegate.getIndex();
+        }
+
+        @Override
+        public String getType() {
+            return delegate.getType();
+        }
+
+        @Override
+        public String getId() {
+            return delegate.getId();
+        }
+
+        @Override
+        public BytesReference getSource() {
+            return source;
+        }
+
+        @Override
+        public long getVersion() {
+            return delegate.getVersion();
+        }
+
+        @Override
+        public String getParent() {
+            return fieldValue(ParentFieldMapper.NAME);
+        }
+
+        @Override
+        public String getRouting() {
+            return fieldValue(RoutingFieldMapper.NAME);
+        }
+
+        @Override
+        public Long getTimestamp() {
+            return fieldValue(TimestampFieldMapper.NAME);
+        }
+
+        @Override
+        public Long getTTL() {
+            return fieldValue(TTLFieldMapper.NAME);
+        }
+        
+        private <T> T fieldValue(String fieldName) {
+            SearchHitField field = delegate.field(fieldName);
+            return field == null ? null : field.value();
+        }
+    }
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
index 5a7b81a7124..4f2cb2578ac 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexPlugin.java
@@ -23,12 +23,15 @@ import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestHandler;
 
 import java.util.Arrays;
 import java.util.List;
 
+import static java.util.Collections.singletonList;
+
 public class ReindexPlugin extends Plugin implements ActionPlugin {
     public static final String NAME = "reindex";
 
@@ -49,4 +52,9 @@ public class ReindexPlugin extends Plugin implements ActionPlugin {
     public void onModule(NetworkModule networkModule) {
         networkModule.registerTaskStatus(BulkByScrollTask.Status.NAME, BulkByScrollTask.Status::new);
     }
+
+    @Override
+    public List<Setting<?>> getSettings() {
+        return singletonList(TransportReindexAction.REMOTE_CLUSTER_WHITELIST);
+    }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
index 660815bbf52..8c11cd3430f 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequest.java
@@ -27,11 +27,13 @@ import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.uid.Versions;
+import org.elasticsearch.index.reindex.remote.RemoteInfo;
 
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
 
+import static java.util.Collections.singletonList;
 import static java.util.Collections.unmodifiableList;
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 import static org.elasticsearch.index.VersionType.INTERNAL;
@@ -48,6 +50,8 @@ public class ReindexRequest extends AbstractBulkIndexByScrollRequest<ReindexRequ
      */
     private IndexRequest destination;
 
+    private RemoteInfo remoteInfo;
+
     public ReindexRequest() {
     }
 
@@ -90,6 +94,9 @@ public class ReindexRequest extends AbstractBulkIndexByScrollRequest<ReindexRequ
         if (destination.timestamp() != null) {
             e = addValidationError("setting timestamp on destination isn't supported. use scripts instead.", e);
         }
+        if (getRemoteInfo() != null && getSearchRequest().source().query() != null) {
+            e = addValidationError("reindex from remote sources should use RemoteInfo's query instead of source's query", e);
+        }
         return e;
     }
 
@@ -110,23 +117,36 @@ public class ReindexRequest extends AbstractBulkIndexByScrollRequest<ReindexRequ
         return destination;
     }
 
+    public void setRemoteInfo(RemoteInfo remoteInfo) {
+        this.remoteInfo = remoteInfo;
+    }
+
+    public RemoteInfo getRemoteInfo() {
+        return remoteInfo;
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
         destination = new IndexRequest();
         destination.readFrom(in);
+        remoteInfo = in.readOptionalWriteable(RemoteInfo::new);
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         destination.writeTo(out);
+        out.writeOptionalWriteable(remoteInfo);
     }
 
     @Override
     public String toString() {
         StringBuilder b = new StringBuilder();
         b.append("reindex from ");
+        if (remoteInfo != null) {
+            b.append('[').append(remoteInfo).append(']');
+        }
         searchToString(b);
         b.append(" to [").append(destination.index()).append(']');
         if (destination.type() != null) {
@@ -148,6 +168,9 @@ public class ReindexRequest extends AbstractBulkIndexByScrollRequest<ReindexRequ
     public List<? extends IndicesRequest> subRequests() {
         assert getSearchRequest() != null;
         assert getDestination() != null;
+        if (remoteInfo != null) {
+            return singletonList(getDestination());
+        }
         return unmodifiableList(Arrays.asList(getSearchRequest(), getDestination()));
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java
index 4f814dbc49d..1eadf2c15bc 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ReindexRequestBuilder.java
@@ -25,6 +25,7 @@ import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.index.reindex.remote.RemoteInfo;
 
 public class ReindexRequestBuilder extends
         AbstractBulkIndexByScrollRequestBuilder<ReindexRequest, ReindexRequestBuilder> {
@@ -67,4 +68,12 @@ public class ReindexRequestBuilder extends
         destination.setIndex(index).setType(type);
         return this;
     }
+
+    /**
+     * Setup reindexing from a remote cluster.
+     */
+    public ReindexRequestBuilder setRemoteInfo(RemoteInfo remoteInfo) {
+        request().setRemoteInfo(remoteInfo);
+        return this;
+    }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
index af0bd168ff0..3bbfebf68a7 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
@@ -27,16 +27,21 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.ParseFieldMatcherSupplier;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.reindex.remote.RemoteInfo;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
@@ -48,36 +53,39 @@ import org.elasticsearch.search.suggest.Suggesters;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
+import static java.util.Objects.requireNonNull;
 import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
 /**
- * Expose IndexBySearchRequest over rest.
+ * Expose reindex over rest.
  */
 public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexRequest, TransportReindexAction> {
-
-    private static final ObjectParser<ReindexRequest, ReindexParseContext> PARSER = new ObjectParser<>("reindex");
+    static final ObjectParser<ReindexRequest, ReindexParseContext> PARSER = new ObjectParser<>("reindex");
+    private static final Pattern HOST_PATTERN = Pattern.compile("(?<scheme>[^:]+)://(?<host>[^:]+):(?<port>\\d+)");
 
     static {
-        ObjectParser.Parser<SearchRequest, ReindexParseContext> sourceParser = (parser, search, context) -> {
-            /*
-             * Extract the parameters that we need from the source sent to the parser. We could do away with this hack when search source
-             * has an ObjectParser.
-             */
+        ObjectParser.Parser<ReindexRequest, ReindexParseContext> sourceParser = (parser, request, context) -> {
+            // Funky hack to work around Search not having a proper ObjectParser and us wanting to extract query if using remote.
             Map<String, Object> source = parser.map();
             String[] indices = extractStringArray(source, "index");
             if (indices != null) {
-                search.indices(indices);
+                request.getSearchRequest().indices(indices);
             }
             String[] types = extractStringArray(source, "type");
             if (types != null) {
-                search.types(types);
+                request.getSearchRequest().types(types);
             }
+            request.setRemoteInfo(buildRemoteInfo(source));
             XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType());
             builder.map(source);
             try (XContentParser innerParser = parser.contentType().xContent().createParser(builder.bytes())) {
-                search.source().parseXContent(context.queryParseContext(innerParser), context.aggParsers, context.suggesters);
+                request.getSearchRequest().source().parseXContent(context.queryParseContext(innerParser), context.aggParsers,
+                        context.suggesters);
             }
         };
 
@@ -94,7 +102,7 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexReq
         destParser.declareString((i, ttl) -> i.ttl(parseTimeValue(ttl, TimeValue.timeValueMillis(-1), "ttl").millis()),
                 new ParseField("ttl"));
 
-        PARSER.declareField((p, v, c) -> sourceParser.parse(p, v.getSearchRequest(), c), new ParseField("source"), ValueType.OBJECT);
+        PARSER.declareField((p, v, c) -> sourceParser.parse(p, v, c), new ParseField("source"), ValueType.OBJECT);
         PARSER.declareField((p, v, c) -> destParser.parse(p, v.getDestination(), c), new ParseField("dest"), ValueType.OBJECT);
         PARSER.declareInt(ReindexRequest::setSize, new ParseField("size"));
         PARSER.declareField((p, v, c) -> v.setScript(Script.parse(p, c.getParseFieldMatcher())), new ParseField("script"),
@@ -127,6 +135,29 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexReq
         return internal;
     }
 
+    static RemoteInfo buildRemoteInfo(Map<String, Object> source) throws IOException {
+        @SuppressWarnings("unchecked")
+        Map<String, Object> remote = (Map<String, Object>) source.remove("remote");
+        if (remote == null) {
+            return null;
+        }
+        String username = extractString(remote, "username");
+        String password = extractString(remote, "password");
+        String hostInRequest = requireNonNull(extractString(remote, "host"), "[host] must be specified to reindex from a remote cluster");
+        Matcher hostMatcher = HOST_PATTERN.matcher(hostInRequest);
+        if (false == hostMatcher.matches()) {
+            throw new IllegalArgumentException("[host] must be of the form [scheme]://[host]:[port] but was [" + hostInRequest + "]");
+        }
+        String scheme = hostMatcher.group("scheme");
+        String host = hostMatcher.group("host");
+        int port = Integer.parseInt(hostMatcher.group("port"));
+        if (false == remote.isEmpty()) {
+            throw new IllegalArgumentException(
+                    "Unsupported fields in [remote]: [" + Strings.collectionToCommaDelimitedString(remote.keySet()) + "]");
+        }
+        return new RemoteInfo(scheme, host, port, queryForRemote(source), username, password);
+    }
+
     /**
      * Yank a string array from a map. Emulates XContent's permissive String to
      * String array conversions.
@@ -147,7 +178,32 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexReq
         }
     }
 
-    private class ReindexParseContext implements ParseFieldMatcherSupplier {
+    private static String extractString(Map<String, Object> source, String name) {
+        Object value = source.remove(name);
+        if (value == null) {
+            return null;
+        }
+        if (value instanceof String) {
+            return (String) value;
+        }
+        throw new IllegalArgumentException("Expected [" + name + "] to be a string but was [" + value + "]");
+    }
+
+    private static BytesReference queryForRemote(Map<String, Object> source) throws IOException {
+        XContentBuilder builder = JsonXContent.contentBuilder().prettyPrint();
+        Object query = source.remove("query");
+        if (query == null) {
+            return matchAllQuery().toXContent(builder, ToXContent.EMPTY_PARAMS).bytes();
+        }
+        if (!(query instanceof Map)) {
+            throw new IllegalArgumentException("Expected [query] to be an object but was [" + query + "]");
+        }
+        @SuppressWarnings("unchecked")
+        Map<String, Object> map = (Map<String, Object>) query;
+        return builder.map(map).bytes();
+    }
+
+    static class ReindexParseContext implements ParseFieldMatcherSupplier {
         private final IndicesQueriesRegistry indicesQueryRegistry;
         private final ParseFieldMatcher parseFieldMatcher;
         private final AggregatorParsers aggParsers;
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java
new file mode 100644
index 00000000000..b03496df7a7
--- /dev/null
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/ScrollableHitSource.java
@@ -0,0 +1,357 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.bulk.BackoffPolicy;
+import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.reindex.remote.RemoteScrollableHitSource;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * A scrollable source of results.
+ */
+public abstract class ScrollableHitSource implements Closeable {
+    private final AtomicReference<String> scrollId = new AtomicReference<>();
+
+    protected final ESLogger logger;
+    protected final BackoffPolicy backoffPolicy;
+    protected final ThreadPool threadPool;
+    protected final Runnable countSearchRetry;
+    protected final Consumer<Exception> fail;
+
+    public ScrollableHitSource(ESLogger logger, BackoffPolicy backoffPolicy, ThreadPool threadPool, Runnable countSearchRetry,
+            Consumer<Exception> fail) {
+        this.logger = logger;
+        this.backoffPolicy = backoffPolicy;
+        this.threadPool = threadPool;
+        this.countSearchRetry = countSearchRetry;
+        this.fail = fail;
+    }
+
+    public final void start(Consumer<Response> onResponse) {
+        doStart(response -> {
+           setScroll(response.getScrollId());
+           logger.debug("scroll returned [{}] documents with a scroll id of [{}]", response.getHits().size(), response.getScrollId());
+           onResponse.accept(response);
+        });
+    }
+    protected abstract void doStart(Consumer<? super Response> onResponse);
+
+    public final void startNextScroll(TimeValue extraKeepAlive, Consumer<Response> onResponse) {
+        doStartNextScroll(scrollId.get(), extraKeepAlive, response -> {
+            setScroll(response.getScrollId());
+            onResponse.accept(response);
+        });
+    }
+    protected abstract void doStartNextScroll(String scrollId, TimeValue extraKeepAlive, Consumer<? super Response> onResponse);
+    
+    @Override
+    public void close() {
+        String scrollId = this.scrollId.get();
+        if (Strings.hasLength(scrollId)) {
+            clearScroll(scrollId);
+        }
+    }
+    protected abstract void clearScroll(String scrollId);
+
+    /**
+     * Set the id of the last scroll. Used for debugging.
+     */
+    final void setScroll(String scrollId) {
+        this.scrollId.set(scrollId);
+    }
+
+    /**
+     * Response from each scroll batch.
+     */
+    public static class Response {
+        private final boolean timedOut;
+        private final List<SearchFailure> failures;
+        private final long totalHits;
+        private final List<? extends Hit> hits;
+        private final String scrollId;
+
+        public Response(boolean timedOut, List<SearchFailure> failures, long totalHits, List<? extends Hit> hits, String scrollId) {
+            this.timedOut = timedOut;
+            this.failures = failures;
+            this.totalHits = totalHits;
+            this.hits = hits;
+            this.scrollId = scrollId;
+        }
+
+        /**
+         * Did this batch time out?
+         */
+        public boolean isTimedOut() {
+            return timedOut;
+        }
+
+        /**
+         * Where there any search failures?
+         */
+        public final List<SearchFailure> getFailures() {
+            return failures;
+        }
+
+        /**
+         * What were the total number of documents matching the search?
+         */
+        public long getTotalHits() {
+            return totalHits;
+        }
+
+        /**
+         * The documents returned in this batch.
+         */
+        public List<? extends Hit> getHits() {
+            return hits;
+        }
+
+        /**
+         * The scroll id used to fetch the next set of documents.
+         */
+        public String getScrollId() {
+            return scrollId;
+        }
+    }
+
+    /**
+     * A document returned as part of the response. Think of it like {@link SearchHit} but with all the things reindex needs in convenient
+     * methods.
+     */
+    public interface Hit {
+        String getIndex();
+        String getType();
+        String getId();
+        long getVersion();
+        /**
+         * The source of the hit. Returns null if the source didn't come back from the search, usually because it source wasn't stored at
+         * all.
+         */
+        @Nullable BytesReference getSource();
+        @Nullable String getParent();
+        @Nullable String getRouting();
+        @Nullable Long getTimestamp();
+        @Nullable Long getTTL();
+    }
+
+    /**
+     * An implementation of {@linkplain Hit} that uses getters and setters. Primarily used for testing and {@link RemoteScrollableHitSource}
+     * .
+     */
+    public static class BasicHit implements Hit {
+        private final String index;
+        private final String type;
+        private final String id;
+        private final long version;
+
+        private BytesReference source;
+        private String parent;
+        private String routing;
+        private Long timestamp;
+        private Long ttl;
+
+        public BasicHit(String index, String type, String id, long version) {
+            this.index = index;
+            this.type = type;
+            this.id = id;
+            this.version = version;
+        }
+
+        @Override
+        public String getIndex() {
+            return index;
+        }
+
+        @Override
+        public String getType() {
+            return type;
+        }
+
+        @Override
+        public String getId() {
+            return id;
+        }
+
+        @Override
+        public long getVersion() {
+            return version;
+        }
+
+        @Override
+        public BytesReference getSource() {
+            return source;
+        }
+
+        public BasicHit setSource(BytesReference source) {
+            this.source = source;
+            return this;
+        }
+
+        @Override
+        public String getParent() {
+            return parent;
+        }
+
+        public BasicHit setParent(String parent) {
+            this.parent = parent;
+            return this;
+        }
+
+        @Override
+        public String getRouting() {
+            return routing;
+        }
+
+        public BasicHit setRouting(String routing) {
+            this.routing = routing;
+            return this;
+        }
+
+        @Override
+        public Long getTimestamp() {
+            return timestamp;
+        }
+
+        public BasicHit setTimestamp(Long timestamp) {
+            this.timestamp = timestamp;
+            return this;
+        }
+
+        @Override
+        public Long getTTL() {
+            return ttl;
+        }
+
+        public BasicHit setTTL(Long ttl) {
+            this.ttl = ttl;
+            return this;
+        }
+    }
+
+    /**
+     * A failure during search. Like {@link ShardSearchFailure} but useful for reindex from remote as well.
+     */
+    public static class SearchFailure implements Writeable, ToXContent {
+        private final Throwable reason;
+        @Nullable
+        private final String index;
+        @Nullable
+        private final Integer shardId;
+        @Nullable
+        private final String nodeId;
+
+        public SearchFailure(Throwable reason, @Nullable String index, @Nullable Integer shardId, @Nullable String nodeId) {
+            this.index = index;
+            this.shardId = shardId;
+            this.reason = requireNonNull(reason, "reason cannot be null");
+            this.nodeId = nodeId;
+        }
+
+        /**
+         * Build a search failure that doesn't have shard information available.
+         */
+        public SearchFailure(Throwable reason) {
+            this(reason, null, null, null);
+        }
+
+        /**
+         * Read from a stream.
+         */
+        public SearchFailure(StreamInput in) throws IOException {
+            reason = in.readException();
+            index = in.readOptionalString();
+            shardId = in.readOptionalVInt();
+            nodeId = in.readOptionalString();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeException(reason);
+            out.writeOptionalString(index);
+            out.writeOptionalVInt(shardId);
+            out.writeOptionalString(nodeId);
+        }
+
+        public String getIndex() {
+            return index;
+        }
+
+        public Integer getShardId() {
+            return shardId;
+        }
+
+        public Throwable getReason() {
+            return reason;
+        }
+
+        @Nullable
+        public String getNodeId() {
+            return nodeId;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            if (index != null) {
+                builder.field("index", index);
+            }
+            if (shardId != null) {
+                builder.field("shard", shardId);
+            }
+            if (nodeId != null) {
+                builder.field("node", nodeId);
+            }
+            builder.field("reason");
+            {
+                builder.startObject();
+                ElasticsearchException.toXContent(builder, params, reason);
+                builder.endObject();
+            }
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+    }
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java
index 471bd066f94..c3847ab2125 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportDeleteByQueryAction.java
@@ -31,10 +31,7 @@ import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
-import org.elasticsearch.index.mapper.internal.RoutingFieldMapper;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -74,35 +71,35 @@ public class TransportDeleteByQueryAction extends HandledTransportAction<DeleteB
         public AsyncDeleteBySearchAction(BulkByScrollTask task, ESLogger logger, ParentTaskAssigningClient client, ThreadPool threadPool,
                                          DeleteByQueryRequest request, ActionListener<BulkIndexByScrollResponse> listener,
                                          ScriptService scriptService, ClusterState clusterState) {
-            super(task, logger, client, threadPool, request, request.getSearchRequest(), listener, scriptService, clusterState);
+            super(task, logger, client, threadPool, request, listener, scriptService, clusterState);
         }
 
         @Override
-        protected boolean accept(SearchHit doc) {
+        protected boolean accept(ScrollableHitSource.Hit doc) {
             // Delete-by-query does not require the source to delete a document
             // and the default implementation checks for it
             return true;
         }
 
         @Override
-        protected RequestWrapper<DeleteRequest> buildRequest(SearchHit doc) {
+        protected RequestWrapper<DeleteRequest> buildRequest(ScrollableHitSource.Hit doc) {
             DeleteRequest delete = new DeleteRequest();
-            delete.index(doc.index());
-            delete.type(doc.type());
-            delete.id(doc.id());
-            delete.version(doc.version());
+            delete.index(doc.getIndex());
+            delete.type(doc.getType());
+            delete.id(doc.getId());
+            delete.version(doc.getVersion());
             return wrap(delete);
         }
 
         /**
-         * Overrides the parent {@link AbstractAsyncBulkIndexByScrollAction#copyMetadata(RequestWrapper, SearchHit)}
+         * Overrides the parent {@link AbstractAsyncBulkIndexByScrollAction#copyMetadata(RequestWrapper, ScrollableHitSource.Hit)}
          * method that is much more Update/Reindex oriented and so also copies things like timestamp/ttl which we
          * don't care for a deletion.
          */
         @Override
-        protected RequestWrapper<?> copyMetadata(RequestWrapper<?> request, SearchHit doc) {
-            copyParent(request, fieldValue(doc, ParentFieldMapper.NAME));
-            copyRouting(request, fieldValue(doc, RoutingFieldMapper.NAME));
+        protected RequestWrapper<?> copyMetadata(RequestWrapper<?> request, ScrollableHitSource.Hit doc) {
+            request.setParent(doc.getParent());
+            request.setRouting(doc.getRouting());
             return request;
         }
     }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
index e98c45595c7..04ccfa1ba49 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
@@ -19,8 +19,10 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.apache.http.HttpHost;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.bulk.BackoffPolicy;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.ActionFilters;
@@ -28,51 +30,72 @@ import org.elasticsearch.action.support.AutoCreateIndex;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.ParentTaskAssigningClient;
+import org.elasticsearch.client.RestClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.lucene.uid.Versions;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.http.HttpInfo;
 import org.elasticsearch.index.mapper.internal.TTLFieldMapper;
 import org.elasticsearch.index.mapper.internal.VersionFieldMapper;
+import org.elasticsearch.index.reindex.remote.RemoteInfo;
+import org.elasticsearch.index.reindex.remote.RemoteScrollableHitSource;
+import org.elasticsearch.node.service.NodeService;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.function.BiFunction;
+import java.util.function.Function;
 
+import static java.util.Collections.emptyList;
 import static java.util.Objects.requireNonNull;
 import static org.elasticsearch.index.VersionType.INTERNAL;
 
 public class TransportReindexAction extends HandledTransportAction<ReindexRequest, BulkIndexByScrollResponse> {
+    public static final Setting<List<String>> REMOTE_CLUSTER_WHITELIST =
+            Setting.listSetting("reindex.remote.whitelist", emptyList(), Function.identity(), Property.NodeScope);
+
     private final ClusterService clusterService;
     private final ScriptService scriptService;
     private final AutoCreateIndex autoCreateIndex;
     private final Client client;
+    private final Set<String> remoteWhitelist;
+    private final NodeService nodeService;
 
     @Inject
     public TransportReindexAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters,
             IndexNameExpressionResolver indexNameExpressionResolver, ClusterService clusterService, ScriptService scriptService,
-            AutoCreateIndex autoCreateIndex, Client client, TransportService transportService) {
+            AutoCreateIndex autoCreateIndex, Client client, TransportService transportService, NodeService nodeService) {
         super(settings, ReindexAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver,
                 ReindexRequest::new);
         this.clusterService = clusterService;
         this.scriptService = scriptService;
         this.autoCreateIndex = autoCreateIndex;
         this.client = client;
+        remoteWhitelist = new HashSet<>(REMOTE_CLUSTER_WHITELIST.get(settings));
+        this.nodeService = nodeService;
     }
 
     @Override
     protected void doExecute(Task task, ReindexRequest request, ActionListener<BulkIndexByScrollResponse> listener) {
+        checkRemoteWhitelist(request.getRemoteInfo());
         ClusterState state = clusterService.state();
-        validateAgainstAliases(request.getSearchRequest(), request.getDestination(), indexNameExpressionResolver, autoCreateIndex, state);
+        validateAgainstAliases(request.getSearchRequest(), request.getDestination(), request.getRemoteInfo(), indexNameExpressionResolver,
+                autoCreateIndex, state);
         ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task);
         new AsyncIndexBySearchAction((BulkByScrollTask) task, logger, client, threadPool, request, listener, scriptService, state).start();
     }
@@ -82,15 +105,43 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
         throw new UnsupportedOperationException("task required");
     }
 
+    private void checkRemoteWhitelist(RemoteInfo remoteInfo) {
+        TransportAddress publishAddress = null;
+        HttpInfo httpInfo = nodeService.info().getHttp();
+        if (httpInfo != null && httpInfo.getAddress() != null) {
+            publishAddress = httpInfo.getAddress().publishAddress();
+        }
+        checkRemoteWhitelist(remoteWhitelist, remoteInfo, publishAddress);
+    }
+
+    static void checkRemoteWhitelist(Set<String> whitelist, RemoteInfo remoteInfo, TransportAddress publishAddress) {
+        if (remoteInfo == null) return;
+        String check = remoteInfo.getHost() + ':' + remoteInfo.getPort();
+        if (whitelist.contains(check)) return;
+        /*
+         * For testing we support the key "myself" to allow connecting to the local node. We can't just change the setting to include the
+         * local node because it is intentionally not a dynamic setting for security purposes. We can't use something like "localhost:9200"
+         * because we don't know up front which port we'll get because the tests bind to port 0. Instead we try to resolve it here, taking
+         * "myself" to mean "my published http address".
+         */
+        if (whitelist.contains("myself") && publishAddress != null && publishAddress.toString().equals(check)) {
+            return;
+        }
+        throw new IllegalArgumentException('[' + check + "] not whitelisted in " + REMOTE_CLUSTER_WHITELIST.getKey());
+    }
+
     /**
      * Throws an ActionRequestValidationException if the request tries to index
      * back into the same index or into an index that points to two indexes.
      * This cannot be done during request validation because the cluster state
      * isn't available then. Package private for testing.
      */
-    static String validateAgainstAliases(SearchRequest source, IndexRequest destination,
+    static void validateAgainstAliases(SearchRequest source, IndexRequest destination, RemoteInfo remoteInfo,
                                          IndexNameExpressionResolver indexNameExpressionResolver, AutoCreateIndex autoCreateIndex,
                                          ClusterState clusterState) {
+        if (remoteInfo != null) {
+            return;
+        }
         String target = destination.index();
         if (false == autoCreateIndex.shouldAutoCreate(target, clusterState)) {
             /*
@@ -107,7 +158,6 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
                 throw e;
             }
         }
-        return target;
     }
 
     /**
@@ -121,11 +171,30 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
         public AsyncIndexBySearchAction(BulkByScrollTask task, ESLogger logger, ParentTaskAssigningClient client, ThreadPool threadPool,
                                         ReindexRequest request, ActionListener<BulkIndexByScrollResponse> listener,
                                         ScriptService scriptService, ClusterState clusterState) {
-            super(task, logger, client, threadPool, request, request.getSearchRequest(), listener, scriptService, clusterState);
+            super(task, logger, client, threadPool, request, listener, scriptService, clusterState);
         }
 
         @Override
-        protected BiFunction<RequestWrapper<?>, SearchHit, RequestWrapper<?>> buildScriptApplier() {
+        protected ScrollableHitSource buildScrollableResultSource(BackoffPolicy backoffPolicy) {
+            if (mainRequest.getRemoteInfo() != null) {
+                // NORELEASE track 500-level retries that are builtin to the client
+                RemoteInfo remoteInfo = mainRequest.getRemoteInfo();
+                if (remoteInfo.getUsername() != null) {
+                    // NORELEASE support auth
+                    throw new UnsupportedOperationException("Auth is unsupported");
+                }
+                RestClient restClient = RestClient.builder(new HttpHost(remoteInfo.getHost(), remoteInfo.getPort(), remoteInfo.getScheme()))
+                        .build();
+                RemoteScrollableHitSource.AsyncClient client = new RemoteScrollableHitSource.AsynchronizingRestClient(threadPool,
+                        restClient);
+                return new RemoteScrollableHitSource(logger, backoffPolicy, threadPool, task::countSearchRetry, this::finishHim, client,
+                        remoteInfo.getQuery(), mainRequest.getSearchRequest());
+            }
+            return super.buildScrollableResultSource(backoffPolicy);
+        }
+
+        @Override
+        protected BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> buildScriptApplier() {
             Script script = mainRequest.getScript();
             if (script != null) {
                 return new ReindexScriptApplier(task, scriptService, script, script.getParams());
@@ -134,7 +203,7 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
         }
 
         @Override
-        protected RequestWrapper<IndexRequest> buildRequest(SearchHit doc) {
+        protected RequestWrapper<IndexRequest> buildRequest(ScrollableHitSource.Hit doc) {
             IndexRequest index = new IndexRequest();
 
             // Copy the index from the request so we always write where it asked to write
@@ -142,7 +211,7 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
 
             // If the request override's type then the user wants all documents in that type. Otherwise keep the doc's type.
             if (mainRequest.getDestination().type() == null) {
-                index.type(doc.type());
+                index.type(doc.getType());
             } else {
                 index.type(mainRequest.getDestination().type());
             }
@@ -155,12 +224,12 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
             if (index.versionType() == INTERNAL) {
                 index.version(mainRequest.getDestination().version());
             } else {
-                index.version(doc.version());
+                index.version(doc.getVersion());
             }
 
             // id and source always come from the found doc. Scripts can change them but they operate on the index request.
-            index.id(doc.id());
-            index.source(doc.sourceRef());
+            index.id(doc.getId());
+            index.source(doc.getSource());
 
             /*
              * The rest of the index request just has to be copied from the template. It may be changed later from scripts or the superclass
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java
index 2990461c0a9..0ef55fd58ff 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportRethrottleAction.java
@@ -24,7 +24,6 @@ import org.elasticsearch.action.TaskOperationFailure;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.tasks.TransportTasksAction;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
index e30f0d2b140..3765c77927c 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java
@@ -41,7 +41,6 @@ import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
 import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -85,11 +84,11 @@ public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateB
         public AsyncIndexBySearchAction(BulkByScrollTask task, ESLogger logger, ParentTaskAssigningClient client, ThreadPool threadPool,
                                         UpdateByQueryRequest request, ActionListener<BulkIndexByScrollResponse> listener,
                                         ScriptService scriptService, ClusterState clusterState) {
-            super(task, logger, client, threadPool, request, request.getSearchRequest(), listener, scriptService, clusterState);
+            super(task, logger, client, threadPool, request, listener, scriptService, clusterState);
         }
 
         @Override
-        protected BiFunction<RequestWrapper<?>, SearchHit, RequestWrapper<?>> buildScriptApplier() {
+        protected BiFunction<RequestWrapper<?>, ScrollableHitSource.Hit, RequestWrapper<?>> buildScriptApplier() {
             Script script = mainRequest.getScript();
             if (script != null) {
                 return new UpdateByQueryScriptApplier(task, scriptService, script, script.getParams());
@@ -98,14 +97,14 @@ public class TransportUpdateByQueryAction extends HandledTransportAction<UpdateB
         }
 
         @Override
-        protected RequestWrapper<IndexRequest> buildRequest(SearchHit doc) {
+        protected RequestWrapper<IndexRequest> buildRequest(ScrollableHitSource.Hit doc) {
             IndexRequest index = new IndexRequest();
-            index.index(doc.index());
-            index.type(doc.type());
-            index.id(doc.id());
-            index.source(doc.sourceRef());
+            index.index(doc.getIndex());
+            index.type(doc.getType());
+            index.id(doc.getId());
+            index.source(doc.getSource());
             index.versionType(VersionType.INTERNAL);
-            index.version(doc.version());
+            index.version(doc.getVersion());
             index.setPipeline(mainRequest.getPipeline());
             return wrap(index);
         }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteInfo.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteInfo.java
new file mode 100644
index 00000000000..89d6cb18401
--- /dev/null
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteInfo.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex.remote;
+
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+
+import java.io.IOException;
+
+import static java.util.Objects.requireNonNull;
+
+public class RemoteInfo implements Writeable {
+    private final String scheme;
+    private final String host;
+    private final int port;
+    private final BytesReference query;
+    private final String username;
+    private final String password;
+
+    public RemoteInfo(String scheme, String host, int port, BytesReference query, String username, String password) {
+        this.scheme = requireNonNull(scheme, "[scheme] must be specified to reindex from a remote cluster");
+        this.host = requireNonNull(host, "[host] must be specified to reindex from a remote cluster");
+        this.port = port;
+        this.query = requireNonNull(query, "[query] must be specified to reindex from a remote cluster");
+        this.username = username;
+        this.password = password;
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public RemoteInfo(StreamInput in) throws IOException {
+        scheme = in.readString();
+        host = in.readString();
+        port = in.readVInt();
+        query = in.readBytesReference();
+        username = in.readOptionalString();
+        password = in.readOptionalString();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(scheme);
+        out.writeString(host);
+        out.writeVInt(port);
+        out.writeBytesReference(query);
+        out.writeOptionalString(username);
+        out.writeOptionalString(password);
+    }
+
+    public String getScheme() {
+        return scheme;
+    }
+
+    public String getHost() {
+        return host;
+    }
+
+    public int getPort() {
+        return port;
+    }
+
+    public BytesReference getQuery() {
+        return query;
+    }
+
+    @Nullable
+    public String getUsername() {
+        return username;
+    }
+
+    @Nullable
+    public String getPassword() {
+        return password;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder b = new StringBuilder();
+        if (false == "http".equals(scheme)) {
+            // http is the default so it isn't worth taking up space if it is the scheme
+            b.append("scheme=").append(scheme).append(' ');
+        }
+        b.append("host=").append(host).append(" port=").append(port).append(" query=").append(query.utf8ToString());
+        if (username != null) {
+            b.append(" username=").append(username);
+        }
+        if (password != null) {
+            b.append(" password=<<>>");
+        }
+        return b.toString();
+    }
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java
new file mode 100644
index 00000000000..00c9f0ae509
--- /dev/null
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex.remote;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.SortBuilder;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static java.util.Collections.singletonMap;
+
+final class RemoteRequestBuilders {
+    private RemoteRequestBuilders() {}
+
+    static String initialSearchPath(SearchRequest searchRequest) {
+        // It is nasty to build paths with StringBuilder but we'll be careful....
+        StringBuilder path = new StringBuilder("/");
+        addIndexesOrTypes(path, "Index", searchRequest.indices());
+        addIndexesOrTypes(path, "Type", searchRequest.types());
+        path.append("_search");
+        return path.toString();
+    }
+
+    static Map<String, String> initialSearchParams(SearchRequest searchRequest, Version remoteVersion) {
+        Map<String, String> params = new HashMap<>();
+        if (searchRequest.scroll() != null) {
+            params.put("scroll", searchRequest.scroll().keepAlive().toString());
+        }
+        params.put("size", Integer.toString(searchRequest.source().size()));
+        if (searchRequest.source().version() == null || searchRequest.source().version() == true) {
+            // false is the only value that makes it false. Null defaults to true....
+            params.put("version", null);
+        }
+        if (searchRequest.source().sorts() != null) {
+            boolean useScan = false;
+            // Detect if we should use search_type=scan rather than a sort
+            if (remoteVersion.before(Version.V_2_1_0)) {
+                for (SortBuilder<?> sort : searchRequest.source().sorts()) {
+                    if (sort instanceof FieldSortBuilder) {
+                        FieldSortBuilder f = (FieldSortBuilder) sort;
+                        if (f.getFieldName().equals(FieldSortBuilder.DOC_FIELD_NAME)) {
+                            useScan = true;
+                            break;
+                        }
+                    }
+                }
+            }
+            if (useScan) {
+                params.put("search_type", "scan");
+            } else {
+                StringBuilder sorts = new StringBuilder(sortToUri(searchRequest.source().sorts().get(0)));
+                for (int i = 1; i < searchRequest.source().sorts().size(); i++) {
+                    sorts.append(',').append(sortToUri(searchRequest.source().sorts().get(i)));
+                }
+                params.put("sorts", sorts.toString());
+            }
+        }
+        if (searchRequest.source().storedFields() != null && false == searchRequest.source().storedFields().isEmpty()) {
+            StringBuilder fields = new StringBuilder(searchRequest.source().storedFields().get(0));
+            for (int i = 1; i < searchRequest.source().storedFields().size(); i++) {
+                fields.append(',').append(searchRequest.source().storedFields().get(i));
+            }
+            String storedFieldsParamName = remoteVersion.before(Version.V_5_0_0_alpha4) ? "fields" : "stored_fields";
+            params.put(storedFieldsParamName, fields.toString());
+        }
+        return params;
+    }
+
+    static HttpEntity initialSearchEntity(BytesReference query) {
+        try (XContentBuilder entity = JsonXContent.contentBuilder(); XContentParser queryParser = XContentHelper.createParser(query)) {
+            entity.startObject();
+            entity.field("query");
+            /*
+             * We're intentionally a bit paranoid here - copying the query as xcontent rather than writing a raw field. We don't want poorly
+             * written queries to escape. Ever.
+             */
+            entity.copyCurrentStructure(queryParser);
+            XContentParser.Token shouldBeEof = queryParser.nextToken();
+            if (shouldBeEof != null) {
+                throw new ElasticsearchException(
+                        "query was more than a single object. This first token after the object is [" + shouldBeEof + "]");
+            }
+            entity.endObject();
+            BytesRef bytes = entity.bytes().toBytesRef();
+            return new ByteArrayEntity(bytes.bytes, bytes.offset, bytes.length, ContentType.APPLICATION_JSON);
+        } catch (IOException e) {
+            throw new ElasticsearchException("unexpected error building entity", e);
+        }
+    }
+
+    private static void addIndexesOrTypes(StringBuilder path, String name, String[] indicesOrTypes) {
+        if (indicesOrTypes == null || indicesOrTypes.length == 0) {
+            return;
+        }
+        for (String indexOrType : indicesOrTypes) {
+            checkIndexOrType(name, indexOrType);
+        }
+        path.append(Strings.arrayToCommaDelimitedString(indicesOrTypes)).append('/');
+    }
+
+    private static void checkIndexOrType(String name, String indexOrType) {
+        if (indexOrType.indexOf(',') >= 0) {
+            throw new IllegalArgumentException(name + " containing [,] not supported but got [" + indexOrType + "]");
+        }
+        if (indexOrType.indexOf('/') >= 0) {
+            throw new IllegalArgumentException(name + " containing [/] not supported but got [" + indexOrType + "]");
+        }
+    }
+
+    private static String sortToUri(SortBuilder<?> sort) {
+        if (sort instanceof FieldSortBuilder) {
+            FieldSortBuilder f = (FieldSortBuilder) sort;
+            return f.getFieldName() + ":" + f.order();
+        }
+        throw new IllegalArgumentException("Unsupported sort [" + sort + "]");
+    }
+
+    static String scrollPath() {
+        return "/_search/scroll";
+    }
+
+    static Map<String, String> scrollParams(TimeValue keepAlive) {
+        return singletonMap("scroll", keepAlive.toString());
+    }
+
+    static HttpEntity scrollEntity(String scroll) {
+        return new StringEntity(scroll, ContentType.TEXT_PLAIN);
+    }
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java
new file mode 100644
index 00000000000..0a467593a2c
--- /dev/null
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteResponseParsers.java
@@ -0,0 +1,301 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex.remote;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcherSupplier;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentLocation;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.reindex.ScrollableHitSource.BasicHit;
+import org.elasticsearch.index.reindex.ScrollableHitSource.Hit;
+import org.elasticsearch.index.reindex.ScrollableHitSource.Response;
+import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.function.BiFunction;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+import static java.util.Objects.requireNonNull;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
+
+/**
+ * Parsers to convert the response from the remote host into objects useful for {@link RemoteScrollableHitSource}. Lots of data is
+ * intentionally thrown on the floor because we don't need it but ObjectParser and friends are strict about blowing up when they see
+ * elements they don't understand. So you'll see a lot of BiConsumers that look like "(b, v) -&gt; {}". That means "I don't care about the
+ * value here, just throw it away and don't blow up.
+ */
+final class RemoteResponseParsers {
+    private RemoteResponseParsers() {}
+
+    /**
+     * Parser for an individual {@code hit} element.
+     */
+    public static final ConstructingObjectParser<BasicHit, ParseFieldMatcherSupplier> HIT_PARSER = new ConstructingObjectParser<>("hit",
+            a -> {
+                int i = 0;
+                String index = (String) a[i++];
+                String type = (String) a[i++];
+                String id = (String) a[i++];
+                long version = (long) a[i++];
+                return new BasicHit(index, type, id, version);
+            });
+    static {
+        HIT_PARSER.declareString(constructorArg(), new ParseField("_index"));
+        HIT_PARSER.declareString(constructorArg(), new ParseField("_type"));
+        HIT_PARSER.declareString(constructorArg(), new ParseField("_id"));
+        HIT_PARSER.declareLong(constructorArg(), new ParseField("_version"));
+        HIT_PARSER.declareObject(BasicHit::setSource, (p, s) -> {
+            try {
+                /*
+                 * We spool the data from the remote back into xcontent so we can get bytes to send. There ought to be a better way but for
+                 * now this should do.
+                 */
+                try (XContentBuilder b = JsonXContent.contentBuilder()) {
+                    b.copyCurrentStructure(p);
+                    return b.bytes();
+                }
+            } catch (IOException e) {
+                throw new ParsingException(p.getTokenLocation(), "[hit] failed to parse [_source]", e);
+            }
+        }, new ParseField("_source"));
+        HIT_PARSER.declareString(BasicHit::setRouting, new ParseField("_routing"));
+        HIT_PARSER.declareString(BasicHit::setParent, new ParseField("_parent"));
+        HIT_PARSER.declareLong(BasicHit::setTTL, new ParseField("_ttl"));
+        HIT_PARSER.declareLong(BasicHit::setTimestamp, new ParseField("_timestamp"));
+        HIT_PARSER.declareField((b, v) -> {}, p -> null, new ParseField("_score"), ValueType.FLOAT_OR_NULL);
+        HIT_PARSER.declareStringArray((b, v) -> {}, new ParseField("sort"));
+    }
+
+    /**
+     * Parser for the {@code hits} element. Parsed to an array of {@code [total (Long), hits (List<Hit>)]}.
+     */
+    public static final ConstructingObjectParser<Object[], ParseFieldMatcherSupplier> HITS_PARSER = new ConstructingObjectParser<>("hits",
+            a -> a);
+    static {
+        HITS_PARSER.declareLong(constructorArg(), new ParseField("total"));
+        HITS_PARSER.declareObjectArray(constructorArg(), HIT_PARSER, new ParseField("hits"));
+        HITS_PARSER.declareField((b, v) -> {}, p -> null, new ParseField("max_score"), ValueType.FLOAT_OR_NULL);
+    }
+
+    /**
+     * Parser for {@code failed} shards in the {@code _shards} elements.
+     */
+    public static final ConstructingObjectParser<SearchFailure, ParseFieldMatcherSupplier> SEARCH_FAILURE_PARSER =
+            new ConstructingObjectParser<>("failure", a -> {
+                int i = 0;
+                String index = (String) a[i++];
+                Integer shardId = (Integer) a[i++];
+                String nodeId = (String) a[i++];
+                Object reason = a[i++];
+
+                Throwable reasonThrowable;
+                if (reason instanceof String) {
+                    reasonThrowable = new RuntimeException("Unknown remote exception with reason=[" + (String) reason + "]");
+                } else {
+                    reasonThrowable = (Throwable) reason;
+                }
+                return new SearchFailure(reasonThrowable, index, shardId, nodeId);
+            });
+    static {
+        SEARCH_FAILURE_PARSER.declareString(optionalConstructorArg(), new ParseField("index"));
+        SEARCH_FAILURE_PARSER.declareInt(optionalConstructorArg(), new ParseField("shard"));
+        SEARCH_FAILURE_PARSER.declareString(optionalConstructorArg(), new ParseField("node"));
+        SEARCH_FAILURE_PARSER.declareField(constructorArg(), (p, c) -> {
+            if (p.currentToken() == XContentParser.Token.START_OBJECT) {
+                return ThrowableBuilder.PARSER.apply(p, c);
+            } else {
+                return p.text();
+            }
+        }, new ParseField("reason"), ValueType.OBJECT_OR_STRING);
+        SEARCH_FAILURE_PARSER.declareInt((b, v) -> {}, new ParseField("status"));
+    }
+
+    /**
+     * Parser for the {@code _shards} element. Throws everything out except the errors array if there is one. If there isn't one then it
+     * parses to an empty list.
+     */
+    public static final ConstructingObjectParser<List<Throwable>, ParseFieldMatcherSupplier> SHARDS_PARSER =
+            new ConstructingObjectParser<>("_shards", a -> {
+                @SuppressWarnings("unchecked")
+                List<Throwable> failures = (List<Throwable>) a[0];
+                failures = failures == null ? emptyList() : failures;
+                return failures;
+            });
+    static {
+        SHARDS_PARSER.declareObjectArray(optionalConstructorArg(), SEARCH_FAILURE_PARSER, new ParseField("failures"));
+        SHARDS_PARSER.declareInt((b, v) -> {}, new ParseField("total"));
+        SHARDS_PARSER.declareInt((b, v) -> {}, new ParseField("successful"));
+        SHARDS_PARSER.declareInt((b, v) -> {}, new ParseField("failed"));
+    }
+
+    public static final ConstructingObjectParser<Response, ParseFieldMatcherSupplier> RESPONSE_PARSER =
+            new ConstructingObjectParser<>("search_response", a -> {
+                int i = 0;
+                Throwable catastrophicFailure = (Throwable) a[i++];
+                if (catastrophicFailure != null) {
+                    return new Response(false, singletonList(new SearchFailure(catastrophicFailure)), 0, emptyList(), null);
+                }
+                boolean timedOut = (boolean) a[i++];
+                String scroll = (String) a[i++];
+                Object[] hitsElement = (Object[]) a[i++];
+                @SuppressWarnings("unchecked")
+                List<SearchFailure> failures = (List<SearchFailure>) a[i++];
+
+                long totalHits = 0;
+                List<Hit> hits = emptyList();
+
+                // Pull apart the hits element if we got it
+                if (hitsElement != null) {
+                    i = 0;
+                    totalHits = (long) hitsElement[i++];
+                    @SuppressWarnings("unchecked")
+                    List<Hit> h = (List<Hit>) hitsElement[i++];
+                    hits = h;
+                }
+
+                return new Response(timedOut, failures, totalHits, hits, scroll);
+            });
+    static {
+        RESPONSE_PARSER.declareObject(optionalConstructorArg(), ThrowableBuilder.PARSER, new ParseField("error"));
+        RESPONSE_PARSER.declareBoolean(optionalConstructorArg(), new ParseField("timed_out"));
+        RESPONSE_PARSER.declareString(optionalConstructorArg(), new ParseField("_scroll_id"));
+        RESPONSE_PARSER.declareObject(optionalConstructorArg(), HITS_PARSER, new ParseField("hits"));
+        RESPONSE_PARSER.declareObject(optionalConstructorArg(), SHARDS_PARSER, new ParseField("_shards"));
+        RESPONSE_PARSER.declareInt((b, v) -> {}, new ParseField("took"));
+        RESPONSE_PARSER.declareBoolean((b, v) -> {}, new ParseField("terminated_early"));
+        RESPONSE_PARSER.declareInt((b, v) -> {}, new ParseField("status"));
+    }
+
+    /**
+     * Collects stuff about Throwables and attempts to rebuild them.
+     */
+    public static class ThrowableBuilder {
+        public static final BiFunction<XContentParser, ParseFieldMatcherSupplier, Throwable> PARSER;
+        static {
+            ObjectParser<ThrowableBuilder, ParseFieldMatcherSupplier> parser = new ObjectParser<>("reason", ThrowableBuilder::new);
+            PARSER = parser.andThen(ThrowableBuilder::build);
+            parser.declareString(ThrowableBuilder::setType, new ParseField("type"));
+            parser.declareString(ThrowableBuilder::setReason, new ParseField("reason"));
+            parser.declareObject(ThrowableBuilder::setCausedBy, PARSER, new ParseField("caused_by"));
+
+            // So we can give a nice error for parsing exceptions
+            parser.declareInt(ThrowableBuilder::setLine, new ParseField("line"));
+            parser.declareInt(ThrowableBuilder::setColumn, new ParseField("col"));
+
+            // So we don't blow up on search exceptions
+            parser.declareString((b, v) -> {}, new ParseField("phase"));
+            parser.declareBoolean((b, v) -> {}, new ParseField("grouped"));
+            parser.declareField((p, v, c) -> p.skipChildren(), new ParseField("failed_shards"), ValueType.OBJECT_ARRAY);
+
+            // Just throw away the root_cause
+            parser.declareField((p, v, c) -> p.skipChildren(), new ParseField("root_cause"), ValueType.OBJECT_ARRAY);
+        }
+
+        private String type;
+        private String reason;
+        private Integer line;
+        private Integer column;
+        private Throwable causedBy;
+
+        public Throwable build() {
+            Throwable t = buildWithoutCause();
+            if (causedBy != null) { 
+                t.initCause(causedBy);
+            }
+            return t;
+        }
+
+        private Throwable buildWithoutCause() {
+            requireNonNull(type, "[type] is required");
+            requireNonNull(reason, "[reason] is required");
+            switch (type) {
+            // Make some effort to use the right exceptions
+            case "es_rejected_execution_exception":
+                return new EsRejectedExecutionException(reason);
+            case "parsing_exception":
+                XContentLocation location = null;
+                if (line != null && column != null) {
+                    location = new XContentLocation(line, column);
+                }
+                return new ParsingException(location, reason);
+            // But it isn't worth trying to get it perfect....
+            default:
+                return new RuntimeException(type + ": " + reason);
+            }
+        }
+
+        public void setType(String type) {
+            this.type = type;
+        }
+        public void setReason(String reason) {
+            this.reason = reason;
+        }
+        public void setLine(Integer line) {
+            this.line = line;
+        }
+        public void setColumn(Integer column) {
+            this.column = column;
+        }
+        public void setCausedBy(Throwable causedBy) {
+            this.causedBy = causedBy;
+        }
+    }
+
+    /**
+     * Parses the {@code version} field of the main action. There are a surprising number of fields in this that we don't need!
+     */
+    public static final ConstructingObjectParser<Version, ParseFieldMatcherSupplier> VERSION_PARSER = new ConstructingObjectParser<>(
+            "version", a -> Version.fromString((String) a[0]));
+    static {
+        VERSION_PARSER.declareString(constructorArg(), new ParseField("number"));
+        VERSION_PARSER.declareBoolean((p, v) -> {}, new ParseField("snapshot_build"));
+        VERSION_PARSER.declareBoolean((p, v) -> {}, new ParseField("build_snapshot"));
+        VERSION_PARSER.declareString((p, v) -> {}, new ParseField("build_hash"));
+        VERSION_PARSER.declareString((p, v) -> {}, new ParseField("build_date"));
+        VERSION_PARSER.declareString((p, v) -> {}, new ParseField("build_timestamp"));
+        VERSION_PARSER.declareString((p, v) -> {}, new ParseField("lucene_version"));
+    }
+
+    /**
+     * Parses the main action to return just the {@linkplain Version} that it returns. We throw everything else out.
+     */
+    public static final ConstructingObjectParser<Version, ParseFieldMatcherSupplier> MAIN_ACTION_PARSER = new ConstructingObjectParser<>(
+            "/", a -> (Version) a[0]);
+    static {
+        MAIN_ACTION_PARSER.declareBoolean((p, v) -> {}, new ParseField("ok"));
+        MAIN_ACTION_PARSER.declareInt((p, v) -> {}, new ParseField("status"));
+        MAIN_ACTION_PARSER.declareString((p, v) -> {}, new ParseField("name"));
+        MAIN_ACTION_PARSER.declareString((p, v) -> {}, new ParseField("cluster_name"));
+        MAIN_ACTION_PARSER.declareString((p, v) -> {}, new ParseField("name"));
+        MAIN_ACTION_PARSER.declareString((p, v) -> {}, new ParseField("tagline"));
+        MAIN_ACTION_PARSER.declareObject(constructorArg(), VERSION_PARSER, new ParseField("version"));
+    }
+}
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java
new file mode 100644
index 00000000000..62dbd59f80a
--- /dev/null
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSource.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex.remote;
+
+import org.apache.http.HttpEntity;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.bulk.BackoffPolicy;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParseFieldMatcherSupplier;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.common.xcontent.XContent;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.reindex.ScrollableHitSource;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.threadpool.ThreadPool;
+
+import java.io.BufferedInputStream;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.function.BiFunction;
+import java.util.function.Consumer;
+
+import static java.util.Collections.emptyMap;
+import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
+import static org.elasticsearch.common.unit.TimeValue.timeValueNanos;
+import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.initialSearchEntity;
+import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.initialSearchParams;
+import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.initialSearchPath;
+import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.scrollEntity;
+import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.scrollParams;
+import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.scrollPath;
+import static org.elasticsearch.index.reindex.remote.RemoteResponseParsers.MAIN_ACTION_PARSER;
+import static org.elasticsearch.index.reindex.remote.RemoteResponseParsers.RESPONSE_PARSER;
+
+public class RemoteScrollableHitSource extends ScrollableHitSource {
+    private final AsyncClient client;
+    private final BytesReference query;
+    private final SearchRequest searchRequest;
+    Version remoteVersion;
+
+    public RemoteScrollableHitSource(ESLogger logger, BackoffPolicy backoffPolicy, ThreadPool threadPool, Runnable countSearchRetry,
+            Consumer<Exception> fail, AsyncClient client, BytesReference query, SearchRequest searchRequest) {
+        super(logger, backoffPolicy, threadPool, countSearchRetry, fail);
+        this.query = query;
+        this.searchRequest = searchRequest;
+        this.client = client;
+    }
+
+    @Override
+    public void close() {
+        try {
+            client.close();
+        } catch (IOException e) {
+            fail.accept(new IOException("couldn't close the remote connection", e));
+        }
+    }
+
+    @Override
+    protected void doStart(Consumer<? super Response> onResponse) {
+        lookupRemoteVersion(version -> {
+            remoteVersion = version;
+            execute("POST", initialSearchPath(searchRequest), initialSearchParams(searchRequest, version),
+                    initialSearchEntity(query), RESPONSE_PARSER, r -> onStartResponse(onResponse, r));
+        });
+    }
+
+    void lookupRemoteVersion(Consumer<Version> onVersion) {
+        execute("GET", "", emptyMap(), null, MAIN_ACTION_PARSER, onVersion);
+        
+    }
+
+    void onStartResponse(Consumer<? super Response> onResponse, Response response) {
+        if (Strings.hasLength(response.getScrollId()) && response.getHits().isEmpty()) {
+            logger.debug("First response looks like a scan response. Jumping right to the second. scroll=[{}]", response.getScrollId());
+            doStartNextScroll(response.getScrollId(), timeValueMillis(0), onResponse);
+        } else {
+            onResponse.accept(response);
+        }
+    }
+
+    @Override
+    protected void doStartNextScroll(String scrollId, TimeValue extraKeepAlive, Consumer<? super Response> onResponse) {
+        execute("POST", scrollPath(), scrollParams(timeValueNanos(searchRequest.scroll().keepAlive().nanos() + extraKeepAlive.nanos())),
+                scrollEntity(scrollId), RESPONSE_PARSER, onResponse);
+    }
+
+    @Override
+    protected void clearScroll(String scrollId) {
+        // Need to throw out response....
+        client.performRequest("DELETE", scrollPath(), emptyMap(), scrollEntity(scrollId), new ResponseListener() {
+            @Override
+            public void onResponse(InputStream response) {
+                logger.debug("Successfully cleared [{}]", scrollId);
+            }
+
+            @Override
+            public void onRetryableFailure(Exception t) {
+                onFailure(t);
+            }
+
+            @Override
+            public void onFailure(Exception t) {
+                logger.warn("Failed to clear scroll [{}]", t, scrollId);
+            }
+        });
+    }
+
+    <T> void execute(String method, String uri, Map<String, String> params, HttpEntity entity,
+            BiFunction<XContentParser, ParseFieldMatcherSupplier, T> parser, Consumer<? super T> listener) {
+        class RetryHelper extends AbstractRunnable {
+            private final Iterator<TimeValue> retries = backoffPolicy.iterator();
+
+            @Override
+            protected void doRun() throws Exception {
+                client.performRequest(method, uri, params, entity, new ResponseListener() {
+                    @Override
+                    public void onResponse(InputStream content) {
+                        T response;
+                        try {
+                            XContent xContent = XContentFactory.xContentType(content).xContent();
+                            try(XContentParser xContentParser = xContent.createParser(content)) {
+                                response = parser.apply(xContentParser, () -> ParseFieldMatcher.STRICT);
+                            }
+                        } catch (IOException e) {
+                            throw new ElasticsearchException("Error deserializing response", e);
+                        }
+                        listener.accept(response);
+                    }
+
+                    @Override
+                    public void onFailure(Exception e) {
+                        fail.accept(e);
+                    }
+
+                    @Override
+                    public void onRetryableFailure(Exception t) {
+                        if (retries.hasNext()) {
+                            TimeValue delay = retries.next();
+                            logger.trace("retrying rejected search after [{}]", t, delay);
+                            countSearchRetry.run();
+                            threadPool.schedule(delay, ThreadPool.Names.SAME, RetryHelper.this);
+                        } else {
+                            fail.accept(t);
+                        }
+                    }
+                });
+            }
+
+            @Override
+            public void onFailure(Exception t) {
+                fail.accept(t);
+            }
+        }
+        new RetryHelper().run();
+    }
+
+    public interface AsyncClient extends Closeable {
+        void performRequest(String method, String uri, Map<String, String> params, HttpEntity entity, ResponseListener listener);
+    }
+
+    public interface ResponseListener extends ActionListener<InputStream> {
+        void onRetryableFailure(Exception t);
+    }
+
+    public static class AsynchronizingRestClient implements AsyncClient {
+        private final ThreadPool threadPool;
+        private final RestClient restClient;
+
+        public AsynchronizingRestClient(ThreadPool threadPool, RestClient restClient) {
+            this.threadPool = threadPool;
+            this.restClient = restClient;
+        }
+
+        @Override
+        public void performRequest(String method, String uri, Map<String, String> params, HttpEntity entity,
+                ResponseListener listener) {
+            /*
+             * We use the generic thread pool here because this client is blocking the generic thread pool is sized appropriately for some
+             * of the threads on it to be blocked, waiting on IO. It'd be a disaster if this ran on the listener thread pool, eating
+             * valuable threads needed to handle responses. Most other thread pool would probably not mind running this either, but the
+             * generic thread pool is the "most right" place for it to run. We could make our own thread pool for this but the generic
+             * thread pool already has plenty of capacity.
+             */
+            threadPool.generic().execute(new AbstractRunnable() {
+                @Override
+                protected void doRun() throws Exception {
+                    try (org.elasticsearch.client.Response response = restClient.performRequest(method, uri, params, entity)) {
+                        InputStream markSupportedInputStream = new BufferedInputStream(response.getEntity().getContent());
+                        listener.onResponse(markSupportedInputStream);
+                    }
+                }
+
+                @Override
+                public void onFailure(Exception t) {
+                    if (t instanceof ResponseException) {
+                        ResponseException re = (ResponseException) t;
+                        if (RestStatus.TOO_MANY_REQUESTS.getStatus() == re.getResponse().getStatusLine().getStatusCode()) {
+                            listener.onRetryableFailure(t);
+                            return;
+                        }
+                    }
+                    listener.onFailure(t);
+                }
+            });
+        }
+
+        @Override
+        public void close() throws IOException {
+            restClient.close();
+        }
+    }
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionScriptTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionScriptTestCase.java
index 0318e4ddb01..f8351b262fc 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionScriptTestCase.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexByScrollActionScriptTestCase.java
@@ -22,21 +22,15 @@ package org.elasticsearch.index.reindex;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.common.text.Text;
-import org.elasticsearch.index.Index;
 import org.elasticsearch.index.reindex.AbstractAsyncBulkIndexByScrollAction.OpType;
 import org.elasticsearch.index.reindex.AbstractAsyncBulkIndexByScrollAction.RequestWrapper;
 import org.elasticsearch.script.CompiledScript;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
-import org.elasticsearch.search.SearchHitField;
-import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.internal.InternalSearchHit;
 import org.junit.Before;
 import org.mockito.Matchers;
 
-import java.util.HashMap;
 import java.util.Map;
 import java.util.function.Consumer;
 
@@ -63,9 +57,7 @@ public abstract class AbstractAsyncBulkIndexByScrollActionScriptTestCase<
     @SuppressWarnings("unchecked")
     protected <T extends ActionRequest<?>> T applyScript(Consumer<Map<String, Object>> scriptBody) {
         IndexRequest index = new IndexRequest("index", "type", "1").source(singletonMap("foo", "bar"));
-        Map<String, SearchHitField> fields = new HashMap<>();
-        InternalSearchHit doc = new InternalSearchHit(0, "id", new Text("type"), fields);
-        doc.shardTarget(new SearchShardTarget("nodeid", new Index("index", "uuid"), 1));
+        ScrollableHitSource.Hit doc = new ScrollableHitSource.BasicHit("test", "type", "id", 0);
         ExecutableScript executableScript = new SimpleExecutableScript(scriptBody);
 
         when(scriptService.executable(any(CompiledScript.class), Matchers.<Map<String, Object>>any()))
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexbyScrollActionMetadataTestCase.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexbyScrollActionMetadataTestCase.java
index 5a9976fc005..4cc10334223 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexbyScrollActionMetadataTestCase.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AbstractAsyncBulkIndexbyScrollActionMetadataTestCase.java
@@ -20,16 +20,7 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.common.text.Text;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.index.mapper.internal.TTLFieldMapper;
-import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
-import org.elasticsearch.search.SearchShardTarget;
-import org.elasticsearch.search.internal.InternalSearchHit;
-import org.elasticsearch.search.internal.InternalSearchHitField;
 
-import static java.util.Collections.singletonList;
-import static java.util.Collections.singletonMap;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
 
 public abstract class AbstractAsyncBulkIndexbyScrollActionMetadataTestCase<
@@ -37,25 +28,19 @@ public abstract class AbstractAsyncBulkIndexbyScrollActionMetadataTestCase<
                 Response extends BulkIndexByScrollResponse>
         extends AbstractAsyncBulkIndexByScrollActionTestCase<Request, Response> {
 
-    /**
-     * Create a doc with some metadata.
-     */
-    protected InternalSearchHit doc(String field, Object value) {
-        InternalSearchHit doc = new InternalSearchHit(0, "id", new Text("type"), singletonMap(field,
-                new InternalSearchHitField(field, singletonList(value))));
-        doc.shardTarget(new SearchShardTarget("node", new Index("index", "uuid"), 0));
-        return doc;
+    protected ScrollableHitSource.BasicHit doc() {
+        return new ScrollableHitSource.BasicHit("index", "type", "id", 0);
     }
 
     public void testTimestampIsCopied() {
         IndexRequest index = new IndexRequest();
-        action().copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc(TimestampFieldMapper.NAME, 10L));
+        action().copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc().setTimestamp(10L));
         assertEquals("10", index.timestamp());
     }
 
     public void testTTL() throws Exception {
         IndexRequest index = new IndexRequest();
-        action().copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc(TTLFieldMapper.NAME, 10L));
+        action().copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc().setTTL(10L));
         assertEquals(timeValueMillis(10), index.ttl());
     }
 
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
index 61c0f2157b5..77e792b8333 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/AsyncBulkByScrollActionTests.java
@@ -62,9 +62,10 @@ import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.engine.VersionConflictEngineException;
+import org.elasticsearch.index.reindex.ScrollableHitSource.Hit;
+import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.internal.InternalSearchHit;
 import org.elasticsearch.search.internal.InternalSearchHits;
 import org.elasticsearch.search.internal.InternalSearchResponse;
@@ -95,6 +96,7 @@ import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static java.util.Collections.singleton;
+import static java.util.Collections.singletonList;
 import static org.apache.lucene.util.TestUtil.randomSimpleString;
 import static org.elasticsearch.action.bulk.BackoffPolicy.constantBackoff;
 import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
@@ -103,7 +105,7 @@ import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.either;
-import static org.hamcrest.Matchers.emptyCollectionOf;
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasSize;
@@ -155,7 +157,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
      * random scroll id so it is checked instead.
      */
     private String scrollId() {
-        scrollId = randomSimpleString(random(), 1, 1000); // Empty strings get special behavior we don't want
+        scrollId = randomSimpleString(random(), 1, 10); // Empty strings get special behavior we don't want
         return scrollId;
     }
 
@@ -216,10 +218,8 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         assertEquals(0, testTask.getStatus().getTotal());
 
         long total = randomIntBetween(0, Integer.MAX_VALUE);
-        InternalSearchHits hits = new InternalSearchHits(null, total, 0);
-        InternalSearchResponse searchResponse = new InternalSearchResponse(hits, null, null, null, false, false);
-        new DummyAbstractAsyncBulkByScrollAction().onScrollResponse(timeValueSeconds(0), 0,
-                new SearchResponse(searchResponse, scrollId(), 5, 4, randomLong(), null));
+        ScrollableHitSource.Response response = new ScrollableHitSource.Response(false, emptyList(), total, emptyList(), null);
+        simulateScrollResponse(new DummyAbstractAsyncBulkByScrollAction(), timeValueSeconds(0), 0, response);
         assertEquals(total, testTask.getStatus().getTotal());
     }
 
@@ -229,12 +229,10 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
     public void testScrollResponseBatchingBehavior() throws Exception {
         int maxBatches = randomIntBetween(0, 100);
         for (int batches = 1; batches < maxBatches; batches++) {
-            InternalSearchHit hit = new InternalSearchHit(0, "id", new Text("type"), emptyMap());
-            InternalSearchHits hits = new InternalSearchHits(new InternalSearchHit[] { hit }, 0, 0);
-            InternalSearchResponse searchResponse = new InternalSearchResponse(hits, null, null, null, false, false);
+            Hit hit = new ScrollableHitSource.BasicHit("index", "type", "id", 0);
+            ScrollableHitSource.Response response = new ScrollableHitSource.Response(false, emptyList(), 1, singletonList(hit), null);
             DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction();
-            action.onScrollResponse(timeValueNanos(System.nanoTime()), 0,
-                    new SearchResponse(searchResponse, scrollId(), 5, 4, randomLong(), null));
+            simulateScrollResponse(action, timeValueNanos(System.nanoTime()), 0, response);
 
             // Use assert busy because the update happens on another thread
             final int expectedBatches = batches;
@@ -314,16 +312,10 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
                 return null;
             }
         };
-        InternalSearchHits hits = new InternalSearchHits(null, 0, 0);
-        InternalSearchResponse searchResponse = new InternalSearchResponse(hits, null, null, null, false, false);
-        new DummyAbstractAsyncBulkByScrollAction().onScrollResponse(timeValueNanos(System.nanoTime()), 10,
-                new SearchResponse(searchResponse, scrollId(), 5, 4, randomLong(), null));
-        try {
-            listener.get();
-            fail("Expected a failure");
-        } catch (ExecutionException e) {
-            assertThat(e.getMessage(), equalTo("EsRejectedExecutionException[test]"));
-        }
+        ScrollableHitSource.Response response = new ScrollableHitSource.Response(false, emptyList(), 0, emptyList(), null);
+        simulateScrollResponse(new DummyAbstractAsyncBulkByScrollAction(), timeValueNanos(System.nanoTime()), 10, response);
+        ExecutionException e = expectThrows(ExecutionException.class, () -> listener.get()); 
+        assertThat(e.getMessage(), equalTo("EsRejectedExecutionException[test]"));
         assertThat(client.scrollsCleared, contains(scrollId));
 
         // When the task is rejected we don't increment the throttled timer
@@ -335,12 +327,12 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
      * scroll request going down.
      */
     public void testShardFailuresAbortRequest() throws Exception {
-        ShardSearchFailure shardFailure = new ShardSearchFailure(new RuntimeException("test"));
-        InternalSearchResponse internalResponse = new InternalSearchResponse(null, null, null, null, false, null);
-        new DummyAbstractAsyncBulkByScrollAction().onScrollResponse(timeValueNanos(System.nanoTime()), 0,
-                new SearchResponse(internalResponse, scrollId(), 5, 4, randomLong(), new ShardSearchFailure[] { shardFailure }));
+        SearchFailure shardFailure = new SearchFailure(new RuntimeException("test"));
+        ScrollableHitSource.Response scrollResponse = new ScrollableHitSource.Response(false, singletonList(shardFailure), 0,
+                emptyList(), null);
+        simulateScrollResponse(new DummyAbstractAsyncBulkByScrollAction(), timeValueNanos(System.nanoTime()), 0, scrollResponse);
         BulkIndexByScrollResponse response = listener.get();
-        assertThat(response.getIndexingFailures(), emptyCollectionOf(Failure.class));
+        assertThat(response.getBulkFailures(), empty());
         assertThat(response.getSearchFailures(), contains(shardFailure));
         assertFalse(response.isTimedOut());
         assertNull(response.getReasonCancelled());
@@ -351,12 +343,11 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
      * Mimicks search timeouts.
      */
     public void testSearchTimeoutsAbortRequest() throws Exception {
-        InternalSearchResponse internalResponse = new InternalSearchResponse(null, null, null, null, true, null);
-        new DummyAbstractAsyncBulkByScrollAction().onScrollResponse(timeValueNanos(System.nanoTime()), 0,
-                new SearchResponse(internalResponse, scrollId(), 5, 4, randomLong(), new ShardSearchFailure[0]));
+        ScrollableHitSource.Response scrollResponse = new ScrollableHitSource.Response(true, emptyList(), 0, emptyList(), null);
+        simulateScrollResponse(new DummyAbstractAsyncBulkByScrollAction(), timeValueNanos(System.nanoTime()), 0, scrollResponse);
         BulkIndexByScrollResponse response = listener.get();
-        assertThat(response.getIndexingFailures(), emptyCollectionOf(Failure.class));
-        assertThat(response.getSearchFailures(), emptyCollectionOf(ShardSearchFailure.class));
+        assertThat(response.getBulkFailures(), empty());
+        assertThat(response.getSearchFailures(), empty());
         assertTrue(response.isTimedOut());
         assertNull(response.getReasonCancelled());
         assertThat(client.scrollsCleared, contains(scrollId));
@@ -371,8 +362,8 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         BulkResponse bulkResponse = new BulkResponse(new BulkItemResponse[] {new BulkItemResponse(0, "index", failure)}, randomLong());
         action.onBulkResponse(timeValueNanos(System.nanoTime()), bulkResponse);
         BulkIndexByScrollResponse response = listener.get();
-        assertThat(response.getIndexingFailures(), contains(failure));
-        assertThat(response.getSearchFailures(), emptyCollectionOf(ShardSearchFailure.class));
+        assertThat(response.getBulkFailures(), contains(failure));
+        assertThat(response.getSearchFailures(), empty());
         assertNull(response.getReasonCancelled());
     }
 
@@ -382,15 +373,13 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
     public void testListenerReceiveBuildBulkExceptions() throws Exception {
         DummyAbstractAsyncBulkByScrollAction action = new DummyAbstractAsyncBulkByScrollAction() {
             @Override
-            protected BulkRequest buildBulk(Iterable<SearchHit> docs) {
+            protected BulkRequest buildBulk(Iterable<? extends ScrollableHitSource.Hit> docs) {
                 throw new RuntimeException("surprise");
             }
         };
-        InternalSearchHit hit = new InternalSearchHit(0, "id", new Text("type"), emptyMap());
-        InternalSearchHits hits = new InternalSearchHits(new InternalSearchHit[] {hit}, 0, 0);
-        InternalSearchResponse internalResponse = new InternalSearchResponse(hits, null, null, null, false, false);
-        SearchResponse searchResponse = new SearchResponse(internalResponse, scrollId(), 5, 4, randomLong(), null);
-        action.onScrollResponse(timeValueNanos(System.nanoTime()), 0, searchResponse);
+        Hit hit = new ScrollableHitSource.BasicHit("index", "type", "id", 0);
+        ScrollableHitSource.Response response = new ScrollableHitSource.Response(false, emptyList(), 1, singletonList(hit), null);
+        simulateScrollResponse(action, timeValueNanos(System.nanoTime()), 0, response);
         ExecutionException e = expectThrows(ExecutionException.class, () -> listener.get());
         assertThat(e.getCause(), instanceOf(RuntimeException.class));
         assertThat(e.getCause().getMessage(), equalTo("surprise"));
@@ -499,9 +488,9 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         action.sendBulkRequest(timeValueNanos(System.nanoTime()), request);
         if (failWithRejection) {
             BulkIndexByScrollResponse response = listener.get();
-            assertThat(response.getIndexingFailures(), hasSize(1));
-            assertEquals(response.getIndexingFailures().get(0).getStatus(), RestStatus.TOO_MANY_REQUESTS);
-            assertThat(response.getSearchFailures(), emptyCollectionOf(ShardSearchFailure.class));
+            assertThat(response.getBulkFailures(), hasSize(1));
+            assertEquals(response.getBulkFailures().get(0).getStatus(), RestStatus.TOO_MANY_REQUESTS);
+            assertThat(response.getSearchFailures(), empty());
             assertNull(response.getReasonCancelled());
         } else {
             successLatch.await(10, TimeUnit.SECONDS);
@@ -549,7 +538,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         if (addDestinationIndexes) {
             action.addDestinationIndices(singleton("foo"));
         }
-        action.startNormalTermination(emptyList(), emptyList(), false);
+        action.refreshAndFinish(emptyList(), emptyList(), false);
         if (shouldRefresh) {
             assertArrayEquals(new String[] {"foo"}, client.lastRefreshRequest.get().indices());
         } else {
@@ -563,7 +552,7 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
 
     public void testCancelBeforeScrollResponse() throws Exception {
         // We bail so early we don't need to pass in a half way valid response.
-        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.onScrollResponse(timeValueNanos(System.nanoTime()), 1,
+        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> simulateScrollResponse(action, timeValueNanos(System.nanoTime()), 1,
                 null));
     }
 
@@ -582,10 +571,10 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.startNextScroll(timeValueNanos(System.nanoTime()), 0));
     }
 
-    public void testCancelBeforeStartNormalTermination() throws Exception {
+    public void testCancelBeforeRefreshAndFinish() throws Exception {
         // Refresh or not doesn't matter - we don't try to refresh.
         testRequest.setRefresh(usually());
-        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.startNormalTermination(emptyList(), emptyList(), false));
+        cancelTaskCase((DummyAbstractAsyncBulkByScrollAction action) -> action.refreshAndFinish(emptyList(), emptyList(), false));
         assertNull("No refresh was attempted", client.lastRefreshRequest.get());
     }
 
@@ -625,12 +614,10 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
             action.setScroll(scrollId());
         }
         long total = randomIntBetween(0, Integer.MAX_VALUE);
-        InternalSearchHits hits = new InternalSearchHits(null, total, 0);
-        InternalSearchResponse searchResponse = new InternalSearchResponse(hits, null, null, null, false, false);
+        ScrollableHitSource.Response response = new ScrollableHitSource.Response(false, emptyList(), total, emptyList(), null);
         // Use a long delay here so the test will time out if the cancellation doesn't reschedule the throttled task
-        SearchResponse scrollResponse = new SearchResponse(searchResponse, scrollId(), 5, 4, randomLong(), null);
         testTask.rethrottle(1);
-        action.onScrollResponse(timeValueNanos(System.nanoTime()), 1000, scrollResponse);
+        simulateScrollResponse(action, timeValueNanos(System.nanoTime()), 1000, response);
 
         // Now that we've got our cancel we'll just verify that it all came through all right
         assertEquals(reason, listener.get(10, TimeUnit.SECONDS).getReasonCancelled());
@@ -656,23 +643,26 @@ public class AsyncBulkByScrollActionTests extends ESTestCase {
         }
     }
 
+    /**
+     * Simulate a scroll response by setting the scroll id and firing the onScrollResponse method.
+     */
+    private void simulateScrollResponse(DummyAbstractAsyncBulkByScrollAction action, TimeValue lastBatchTime, int lastBatchSize,
+            ScrollableHitSource.Response response) {
+        action.setScroll(scrollId());
+        action.onScrollResponse(lastBatchTime, lastBatchSize, response);
+    }
+
     private class DummyAbstractAsyncBulkByScrollAction
             extends AbstractAsyncBulkByScrollAction<DummyAbstractBulkByScrollRequest> {
         public DummyAbstractAsyncBulkByScrollAction() {
-            super(testTask, logger, new ParentTaskAssigningClient(client, localNode, testTask), threadPool, testRequest, firstSearchRequest,
-                    listener);
+            super(testTask, AsyncBulkByScrollActionTests.this.logger, new ParentTaskAssigningClient(client, localNode, testTask),
+                    AsyncBulkByScrollActionTests.this.threadPool, testRequest, listener);
         }
 
         @Override
-        protected BulkRequest buildBulk(Iterable<SearchHit> docs) {
+        protected BulkRequest buildBulk(Iterable<? extends ScrollableHitSource.Hit> docs) {
             return new BulkRequest();
         }
-
-        @Override
-        protected BulkIndexByScrollResponse buildResponse(TimeValue took, List<Failure> indexingFailures,
-                List<ShardSearchFailure> searchFailures, boolean timedOut) {
-            return new BulkIndexByScrollResponse(took, task.getStatus(), indexingFailures, searchFailures, timedOut);
-        }
     }
 
     /**
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseMatcher.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseMatcher.java
index 4ef16c59141..c0c06b14d55 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseMatcher.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseMatcher.java
@@ -124,7 +124,7 @@ public class BulkIndexByScrollResponseMatcher extends TypeSafeMatcher<BulkIndexB
                 deletedMatcher.matches(item.getDeleted()) &&
                 (batchesMatcher == null || batchesMatcher.matches(item.getBatches())) &&
                 versionConflictsMatcher.matches(item.getVersionConflicts()) &&
-                failuresMatcher.matches(item.getIndexingFailures().size()) &&
+                failuresMatcher.matches(item.getBulkFailures().size()) &&
                 reasonCancelledMatcher.matches(item.getReasonCancelled());
     }
 
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
index bbe7dd612e8..59093252d7e 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/CancelTests.java
@@ -141,7 +141,7 @@ public class CancelTests extends ReindexTestCase {
         // And check the status of the response
         BulkIndexByScrollResponse response = future.get();
         assertThat(response.getReasonCancelled(), equalTo("by user request"));
-        assertThat(response.getIndexingFailures(), emptyIterable());
+        assertThat(response.getBulkFailures(), emptyIterable());
         assertThat(response.getSearchFailures(), emptyIterable());
 
         flushAndRefresh(INDEX);
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java
index ed60b3e7ddf..45ac5640221 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFailureTests.java
@@ -61,7 +61,7 @@ public class ReindexFailureTests extends ReindexTestCase {
         assertThat(response, matcher()
                 .batches(1)
                 .failures(both(greaterThan(0)).and(lessThanOrEqualTo(maximumNumberOfShards()))));
-        for (Failure failure: response.getIndexingFailures()) {
+        for (Failure failure: response.getBulkFailures()) {
             assertThat(failure.getMessage(), containsString("NumberFormatException[For input string: \"words words\"]"));
         }
     }
@@ -79,7 +79,7 @@ public class ReindexFailureTests extends ReindexTestCase {
 
         BulkIndexByScrollResponse response = copy.get();
         assertThat(response, matcher().batches(1).versionConflicts(1).failures(1).created(99));
-        for (Failure failure: response.getIndexingFailures()) {
+        for (Failure failure: response.getBulkFailures()) {
             assertThat(failure.getMessage(), containsString("VersionConflictEngineException[[test]["));
         }
     }
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWhitelistTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWhitelistTests.java
new file mode 100644
index 00000000000..b88ed135515
--- /dev/null
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWhitelistTests.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.index.reindex.remote.RemoteInfo;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.HashSet;
+import java.util.Set;
+
+import static java.util.Collections.emptySet;
+import static org.elasticsearch.index.reindex.TransportReindexAction.checkRemoteWhitelist;
+
+/**
+ * Tests the reindex-from-remote whitelist of remotes.
+ */
+public class ReindexFromRemoteWhitelistTests extends ESTestCase {
+    private TransportAddress localhost;
+
+    @Before
+    public void setupLocalhost() throws UnknownHostException {
+        localhost = new InetSocketTransportAddress(InetAddress.getByAddress(new byte[] { 0x7f, 0x00, 0x00, 0x01 }), 9200);
+    }
+
+    public void testLocalRequestWithoutWhitelist() {
+        checkRemoteWhitelist(emptySet(), null, localhostOrNone());
+    }
+
+    public void testLocalRequestWithWhitelist() {
+        checkRemoteWhitelist(randomWhitelist(), null, localhostOrNone());
+    }
+
+    public void testWhitelistedRemote() {
+        Set<String> whitelist = randomWhitelist();
+        String[] inList = whitelist.iterator().next().split(":");
+        String host = inList[0];
+        int port = Integer.valueOf(inList[1]);
+        checkRemoteWhitelist(whitelist, new RemoteInfo(randomAsciiOfLength(5), host, port, new BytesArray("test"), null, null),
+                localhostOrNone());
+    }
+
+    public void testMyselfInWhitelistRemote() throws UnknownHostException {
+        Set<String> whitelist = randomWhitelist();
+        whitelist.add("myself");
+        TransportAddress publishAddress = new InetSocketTransportAddress(InetAddress.getByAddress(new byte[] {0x7f,0x00,0x00,0x01}), 9200);
+        checkRemoteWhitelist(whitelist, new RemoteInfo(randomAsciiOfLength(5), "127.0.0.1", 9200, new BytesArray("test"), null, null),
+                publishAddress);
+    }
+
+    public void testUnwhitelistedRemote() {
+        int port = between(1, Integer.MAX_VALUE);
+        Exception e = expectThrows(IllegalArgumentException.class, () -> checkRemoteWhitelist(randomWhitelist(),
+                new RemoteInfo(randomAsciiOfLength(5), "not in list", port, new BytesArray("test"), null, null), localhostOrNone()));
+        assertEquals("[not in list:" + port + "] not whitelisted in reindex.remote.whitelist", e.getMessage());
+    }
+
+    private Set<String> randomWhitelist() {
+        int size = between(1, 100);
+        Set<String> set = new HashSet<>(size);
+        while (set.size() < size) {
+            set.add(randomAsciiOfLength(5) + ':' + between(1, Integer.MAX_VALUE));
+        }
+        return set;
+    }
+
+    private TransportAddress localhostOrNone() {
+        return randomFrom(random(), null, localhost);
+    }
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java
index 670fcefbf55..dab0cab8d8a 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexMetadataTests.java
@@ -21,7 +21,6 @@ package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.index.mapper.internal.RoutingFieldMapper;
 
 /**
  * Index-by-search test for ttl, timestamp, and routing.
@@ -29,7 +28,7 @@ import org.elasticsearch.index.mapper.internal.RoutingFieldMapper;
 public class ReindexMetadataTests extends AbstractAsyncBulkIndexbyScrollActionMetadataTestCase<ReindexRequest, BulkIndexByScrollResponse> {
     public void testRoutingCopiedByDefault() throws Exception {
         IndexRequest index = new IndexRequest();
-        action().copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc(RoutingFieldMapper.NAME, "foo"));
+        action().copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc().setRouting("foo"));
         assertEquals("foo", index.routing());
     }
 
@@ -37,7 +36,7 @@ public class ReindexMetadataTests extends AbstractAsyncBulkIndexbyScrollActionMe
         TransportReindexAction.AsyncIndexBySearchAction action = action();
         action.mainRequest.getDestination().routing("keep");
         IndexRequest index = new IndexRequest();
-        action.copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc(RoutingFieldMapper.NAME, "foo"));
+        action.copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc().setRouting("foo"));
         assertEquals("foo", index.routing());
     }
 
@@ -45,7 +44,7 @@ public class ReindexMetadataTests extends AbstractAsyncBulkIndexbyScrollActionMe
         TransportReindexAction.AsyncIndexBySearchAction action = action();
         action.mainRequest.getDestination().routing("discard");
         IndexRequest index = new IndexRequest();
-        action.copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc(RoutingFieldMapper.NAME, "foo"));
+        action.copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc().setRouting("foo"));
         assertEquals(null, index.routing());
     }
 
@@ -53,7 +52,7 @@ public class ReindexMetadataTests extends AbstractAsyncBulkIndexbyScrollActionMe
         TransportReindexAction.AsyncIndexBySearchAction action = action();
         action.mainRequest.getDestination().routing("=cat");
         IndexRequest index = new IndexRequest();
-        action.copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc(RoutingFieldMapper.NAME, "foo"));
+        action.copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc().setRouting("foo"));
         assertEquals("cat", index.routing());
     }
 
@@ -61,7 +60,7 @@ public class ReindexMetadataTests extends AbstractAsyncBulkIndexbyScrollActionMe
         TransportReindexAction.AsyncIndexBySearchAction action = action();
         action.mainRequest.getDestination().routing("==]");
         IndexRequest index = new IndexRequest();
-        action.copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc(RoutingFieldMapper.NAME, "foo"));
+        action.copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc().setRouting("foo"));
         assertEquals("=]", index.routing());
     }
 
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java
new file mode 100644
index 00000000000..efaf5e627ad
--- /dev/null
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexRequestTests.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.index.reindex.remote.RemoteInfo;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+
+/**
+ * Tests some of the validation of {@linkplain ReindexRequest}. See reindex's rest tests for much more.
+ */
+public class ReindexRequestTests extends ESTestCase {
+    public void testTimestampAndTtlNotAllowed() {
+        ReindexRequest reindex = request();
+        reindex.getDestination().ttl("1s").timestamp("now");
+        ActionRequestValidationException e = reindex.validate();
+        assertEquals("Validation Failed: 1: setting ttl on destination isn't supported. use scripts instead.;"
+                + "2: setting timestamp on destination isn't supported. use scripts instead.;",
+                e.getMessage());
+    }
+
+    public void testReindexFromRemoteDoesNotSupportSearchQuery() {
+        ReindexRequest reindex = request();
+        reindex.setRemoteInfo(new RemoteInfo(randomAsciiOfLength(5), randomAsciiOfLength(5), between(1, Integer.MAX_VALUE),
+                new BytesArray("real_query"), null, null));
+        reindex.getSearchRequest().source().query(matchAllQuery()); // Unsupported place to put query
+        ActionRequestValidationException e = reindex.validate();
+        assertEquals("Validation Failed: 1: reindex from remote sources should use RemoteInfo's query instead of source's query;",
+                e.getMessage());
+    }
+
+    private ReindexRequest request() {
+        ReindexRequest reindex = new ReindexRequest(new SearchRequest(), new IndexRequest());
+        reindex.getSearchRequest().indices("source");
+        reindex.getDestination().index("dest");
+        return reindex;
+    }
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSameIndexTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java
similarity index 62%
rename from modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSameIndexTests.java
rename to modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java
index f1218414af7..66896406c66 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSameIndexTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexSourceTargetValidationTests.java
@@ -30,15 +30,20 @@ import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.reindex.remote.RemoteInfo;
 import org.elasticsearch.test.ESTestCase;
 
 import static org.hamcrest.Matchers.containsString;
 
 /**
- * Tests that indexing from an index back into itself fails the request.
+ * Tests source and target index validation of reindex. Mostly that means testing that indexing from an index back into itself fails the
+ * request. Note that we can't catch you trying to remotely reindex from yourself into yourself. We actually assert here that reindexes
+ * from remote don't need to come from existing indexes. It'd be silly to fail requests if the source index didn't exist on the target
+ * cluster....
  */
-public class ReindexSameIndexTests extends ESTestCase {
+public class ReindexSourceTargetValidationTests extends ESTestCase {
     private static final ClusterState STATE = ClusterState.builder(new ClusterName("test")).metaData(MetaData.builder()
                 .put(index("target", "target_alias", "target_multi"), true)
                 .put(index("target2", "target_multi"), true)
@@ -50,7 +55,7 @@ public class ReindexSameIndexTests extends ESTestCase {
     private static final IndexNameExpressionResolver INDEX_NAME_EXPRESSION_RESOLVER = new IndexNameExpressionResolver(Settings.EMPTY);
     private static final AutoCreateIndex AUTO_CREATE_INDEX = new AutoCreateIndex(Settings.EMPTY, INDEX_NAME_EXPRESSION_RESOLVER);
 
-    public void testObviousCases() throws Exception {
+    public void testObviousCases() {
         fails("target", "target");
         fails("target", "foo", "bar", "target", "baz");
         fails("target", "foo", "bar", "target", "baz", "target");
@@ -58,7 +63,7 @@ public class ReindexSameIndexTests extends ESTestCase {
         succeeds("target", "source", "source2");
     }
 
-    public void testAliasesContainTarget() throws Exception {
+    public void testAliasesContainTarget() {
         fails("target", "target_alias");
         fails("target_alias", "target");
         fails("target", "foo", "bar", "target_alias", "baz");
@@ -71,31 +76,33 @@ public class ReindexSameIndexTests extends ESTestCase {
         succeeds("target", "source", "source2", "source_multi");
     }
 
-    public void testTargetIsAlias() throws Exception {
-        try {
-            succeeds("target_multi", "foo");
-            fail("Expected failure");
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString("Alias [target_multi] has more than one indices associated with it [["));
-            // The index names can come in either order
-            assertThat(e.getMessage(), containsString("target"));
-            assertThat(e.getMessage(), containsString("target2"));
-        }
+    public void testTargetIsAlias() {
+        Exception e = expectThrows(IllegalArgumentException.class, () -> succeeds("target_multi", "foo"));
+        assertThat(e.getMessage(), containsString("Alias [target_multi] has more than one indices associated with it [["));
+        // The index names can come in either order
+        assertThat(e.getMessage(), containsString("target"));
+        assertThat(e.getMessage(), containsString("target2"));
     }
 
-    private void fails(String target, String... sources) throws Exception {
-        try {
-            succeeds(target, sources);
-            fail("Expected an exception");
-        } catch (ActionRequestValidationException e) {
-            assertThat(e.getMessage(),
-                    containsString("reindex cannot write into an index its reading from [target]"));
-        }
+    public void testRemoteInfoSkipsValidation() {
+        // The index doesn't have to exist
+        succeeds(new RemoteInfo(randomAsciiOfLength(5), "test", 9200, new BytesArray("test"), null, null), "does_not_exist", "target");
+        // And it doesn't matter if they are the same index. They are considered to be different because the remote one is, well, remote.
+        succeeds(new RemoteInfo(randomAsciiOfLength(5), "test", 9200, new BytesArray("test"), null, null), "target", "target");
     }
 
-    private void succeeds(String target, String... sources) throws Exception {
-        TransportReindexAction.validateAgainstAliases(new SearchRequest(sources), new IndexRequest(target), INDEX_NAME_EXPRESSION_RESOLVER,
-                AUTO_CREATE_INDEX, STATE);
+    private void fails(String target, String... sources) {
+        Exception e = expectThrows(ActionRequestValidationException.class, () -> succeeds(target, sources));
+        assertThat(e.getMessage(), containsString("reindex cannot write into an index its reading from [target]"));
+    }
+
+    private void succeeds(String target, String... sources) {
+        succeeds(null, target, sources);
+    }
+
+    private void succeeds(RemoteInfo remoteInfo, String target, String... sources) {
+        TransportReindexAction.validateAgainstAliases(new SearchRequest(sources), new IndexRequest(target), remoteInfo,
+                INDEX_NAME_EXPRESSION_RESOLVER, AUTO_CREATE_INDEX, STATE);
     }
 
     private static IndexMetaData index(String name, String... aliases) {
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java
new file mode 100644
index 00000000000..1cbec59c49d
--- /dev/null
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RestReindexActionTests.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex;
+
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.index.reindex.RestReindexAction.ReindexParseContext;
+import org.elasticsearch.index.reindex.remote.RemoteInfo;
+import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class RestReindexActionTests extends ESTestCase {
+    public void testBuildRemoteInfoNoRemote() throws IOException {
+        assertNull(RestReindexAction.buildRemoteInfo(new HashMap<>()));
+    }
+
+    public void testBuildRemoteInfoFullyLoaded() throws IOException {
+        Map<String, Object> remote = new HashMap<>();
+        remote.put("host", "https://example.com:9200");
+        remote.put("username", "testuser");
+        remote.put("password", "testpass");
+
+        Map<String, Object> query = new HashMap<>();
+        query.put("a", "b");
+
+        Map<String, Object> source = new HashMap<>();
+        source.put("remote", remote);
+        source.put("query", query);
+
+        RemoteInfo remoteInfo = RestReindexAction.buildRemoteInfo(source);
+        assertEquals("https", remoteInfo.getScheme());
+        assertEquals("example.com", remoteInfo.getHost());
+        assertEquals(9200, remoteInfo.getPort());
+        assertEquals("{\n  \"a\" : \"b\"\n}", remoteInfo.getQuery().utf8ToString());
+        assertEquals("testuser", remoteInfo.getUsername());
+        assertEquals("testpass", remoteInfo.getPassword());
+    }
+
+    public void testBuildRemoteInfoWithoutAllParts() throws IOException {
+        expectThrows(IllegalArgumentException.class, () -> buildRemoteInfoHostTestCase("example.com"));
+        expectThrows(IllegalArgumentException.class, () -> buildRemoteInfoHostTestCase("example.com:9200"));
+        expectThrows(IllegalArgumentException.class, () -> buildRemoteInfoHostTestCase("http://example.com"));
+    }
+
+    public void testBuildRemoteInfoWithAllParts() throws IOException {
+        RemoteInfo info = buildRemoteInfoHostTestCase("http://example.com:9200");
+        assertEquals("http", info.getScheme());
+        assertEquals("example.com", info.getHost());
+        assertEquals(9200, info.getPort());
+
+        info = buildRemoteInfoHostTestCase("https://other.example.com:9201");
+        assertEquals("https", info.getScheme());
+        assertEquals("other.example.com", info.getHost());
+        assertEquals(9201, info.getPort());
+    }
+
+    public void testReindexFromRemoteRequestParsing() throws IOException {
+        BytesReference request;
+        try (XContentBuilder b = JsonXContent.contentBuilder()) {
+            b.startObject(); {
+                b.startObject("source"); {
+                    b.startObject("remote"); {
+                        b.field("host", "http://localhost:9200");
+                    }
+                    b.endObject();
+                    b.field("index", "source");
+                }
+                b.endObject();
+                b.startObject("dest"); {
+                    b.field("index", "dest");
+                }
+                b.endObject();
+            }
+            b.endObject();
+            request = b.bytes();
+        }
+        try (XContentParser p = JsonXContent.jsonXContent.createParser(request)) {
+            ReindexRequest r = new ReindexRequest(new SearchRequest(), new IndexRequest());
+            RestReindexAction.PARSER.parse(p, r,
+                    new ReindexParseContext(new IndicesQueriesRegistry(), null, null, ParseFieldMatcher.STRICT));
+            assertEquals("localhost", r.getRemoteInfo().getHost());
+            assertArrayEquals(new String[] {"source"}, r.getSearchRequest().indices());
+        }
+    }
+
+    private RemoteInfo buildRemoteInfoHostTestCase(String hostInRest) throws IOException {
+        Map<String, Object> remote = new HashMap<>();
+        remote.put("host", hostInRest);
+
+        Map<String, Object> source = new HashMap<>();
+        source.put("remote", remote);
+
+        return RestReindexAction.buildRemoteInfo(source);
+    }
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
index 09945c9372b..fd5ddaedd69 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java
@@ -20,13 +20,18 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.ListenableActionFuture;
+import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse;
 import org.elasticsearch.action.bulk.BackoffPolicy;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.bulk.Retry;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.index.reindex.remote.RemoteInfo;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -39,6 +44,7 @@ import java.util.List;
 import java.util.concurrent.CyclicBarrier;
 
 import static org.elasticsearch.index.reindex.ReindexTestCase.matcher;
+import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.hasSize;
 
@@ -68,6 +74,10 @@ public class RetryTests extends ESSingleNodeTestCase {
         // Use queues of size 1 because size 0 is broken and because search requests need the queue to function
         settings.put("thread_pool.bulk.queue_size", 1);
         settings.put("thread_pool.search.queue_size", 1);
+        // Enable http so we can test retries on reindex from remote. In this case the "remote" cluster is just this cluster.
+        settings.put(NetworkModule.HTTP_ENABLED.getKey(), true);
+        // Whitelist reindexing from the http host we're going to use
+        settings.put(TransportReindexAction.REMOTE_CLUSTER_WHITELIST.getKey(), "myself");
         return settings.build();
     }
 
@@ -97,6 +107,15 @@ public class RetryTests extends ESSingleNodeTestCase {
                 matcher().created(DOC_COUNT));
     }
 
+    public void testReindexFromRemote() throws Exception {
+        NodeInfo nodeInfo = client().admin().cluster().prepareNodesInfo().get().getNodes().get(0);
+        TransportAddress address = nodeInfo.getHttp().getAddress().publishAddress();
+        RemoteInfo remote = new RemoteInfo("http", address.getHost(), address.getPort(), new BytesArray("{\"match_all\":{}}"), null, null);
+        ReindexRequestBuilder request = ReindexAction.INSTANCE.newRequestBuilder(client()).source("source").destination("dest")
+                .setRemoteInfo(remote);
+        testCase(ReindexAction.NAME, request, matcher().created(DOC_COUNT));
+    }
+
     public void testUpdateByQuery() throws Exception {
         testCase(UpdateByQueryAction.NAME, UpdateByQueryAction.INSTANCE.newRequestBuilder(client()).source("source"),
                 matcher().updated(DOC_COUNT));
@@ -118,34 +137,41 @@ public class RetryTests extends ESSingleNodeTestCase {
         logger.info("Starting request");
         ListenableActionFuture<BulkIndexByScrollResponse> responseListener = request.execute();
 
-        logger.info("Waiting for search rejections on the initial search");
-        assertBusy(() -> assertThat(taskStatus(action).getSearchRetries(), greaterThan(0L)));
+        try {
+            logger.info("Waiting for search rejections on the initial search");
+            assertBusy(() -> assertThat(taskStatus(action).getSearchRetries(), greaterThan(0L)));
 
-        logger.info("Blocking bulk and unblocking search so we start to get bulk rejections");
-        CyclicBarrier bulkBlock = blockExecutor(ThreadPool.Names.BULK);
-        initialSearchBlock.await();
+            logger.info("Blocking bulk and unblocking search so we start to get bulk rejections");
+            CyclicBarrier bulkBlock = blockExecutor(ThreadPool.Names.BULK);
+            initialSearchBlock.await();
 
-        logger.info("Waiting for bulk rejections");
-        assertBusy(() -> assertThat(taskStatus(action).getBulkRetries(), greaterThan(0L)));
+            logger.info("Waiting for bulk rejections");
+            assertBusy(() -> assertThat(taskStatus(action).getBulkRetries(), greaterThan(0L)));
 
-        // Keep a copy of the current number of search rejections so we can assert that we get more when we block the scroll
-        long initialSearchRejections = taskStatus(action).getSearchRetries();
+            // Keep a copy of the current number of search rejections so we can assert that we get more when we block the scroll
+            long initialSearchRejections = taskStatus(action).getSearchRetries();
 
-        logger.info("Blocking search and unblocking bulk so we should get search rejections for the scroll");
-        CyclicBarrier scrollBlock = blockExecutor(ThreadPool.Names.SEARCH);
-        bulkBlock.await();
+            logger.info("Blocking search and unblocking bulk so we should get search rejections for the scroll");
+            CyclicBarrier scrollBlock = blockExecutor(ThreadPool.Names.SEARCH);
+            bulkBlock.await();
 
-        logger.info("Waiting for search rejections for the scroll");
-        assertBusy(() -> assertThat(taskStatus(action).getSearchRetries(), greaterThan(initialSearchRejections)));
+            logger.info("Waiting for search rejections for the scroll");
+            assertBusy(() -> assertThat(taskStatus(action).getSearchRetries(), greaterThan(initialSearchRejections)));
 
-        logger.info("Unblocking the scroll");
-        scrollBlock.await();
+            logger.info("Unblocking the scroll");
+            scrollBlock.await();
 
-        logger.info("Waiting for the request to finish");
-        BulkIndexByScrollResponse response = responseListener.get();
-        assertThat(response, matcher);
-        assertThat(response.getBulkRetries(), greaterThan(0L));
-        assertThat(response.getSearchRetries(), greaterThan(initialSearchRejections));
+            logger.info("Waiting for the request to finish");
+            BulkIndexByScrollResponse response = responseListener.get();
+            assertThat(response, matcher);
+            assertThat(response.getBulkRetries(), greaterThan(0L));
+            assertThat(response.getSearchRetries(), greaterThan(initialSearchRejections));
+        } finally {
+            // Fetch the response just in case we blew up half way through. This will make sure the failure is thrown up to the top level.
+            BulkIndexByScrollResponse response = responseListener.get();
+            assertThat(response.getSearchFailures(), empty());
+            assertThat(response.getBulkFailures(), empty());
+        }
     }
 
     /**
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
index d1cb77361bb..3e3b3a63d62 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java
@@ -19,20 +19,21 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.WriteConsistencyLevel;
 import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.lucene.uid.Versions;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.index.Index;
-import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
+import org.elasticsearch.index.reindex.remote.RemoteInfo;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService.ScriptType;
-import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.test.ESTestCase;
 
@@ -56,11 +57,28 @@ public class RoundTripTests extends ESTestCase {
         randomRequest(reindex);
         reindex.getDestination().version(randomFrom(Versions.MATCH_ANY, Versions.MATCH_DELETED, 12L, 1L, 123124L, 12L));
         reindex.getDestination().index("test");
+        if (randomBoolean()) {
+            int port = between(1, Integer.MAX_VALUE);
+            BytesReference query = new BytesArray(randomAsciiOfLength(5));
+            String username = randomBoolean() ? randomAsciiOfLength(5) : null;
+            String password = username != null && randomBoolean() ? randomAsciiOfLength(5) : null;
+            reindex.setRemoteInfo(new RemoteInfo(randomAsciiOfLength(5), randomAsciiOfLength(5), port, query, username, password));
+        }
         ReindexRequest tripped = new ReindexRequest();
         roundTrip(reindex, tripped);
         assertRequestEquals(reindex, tripped);
         assertEquals(reindex.getDestination().version(), tripped.getDestination().version());
         assertEquals(reindex.getDestination().index(), tripped.getDestination().index());
+        if (reindex.getRemoteInfo() == null) {
+            assertNull(tripped.getRemoteInfo());
+        } else {
+            assertNotNull(tripped.getRemoteInfo());
+            assertEquals(reindex.getRemoteInfo().getScheme(), tripped.getRemoteInfo().getScheme());
+            assertEquals(reindex.getRemoteInfo().getHost(), tripped.getRemoteInfo().getHost());
+            assertEquals(reindex.getRemoteInfo().getQuery(), tripped.getRemoteInfo().getQuery());
+            assertEquals(reindex.getRemoteInfo().getUsername(), tripped.getRemoteInfo().getUsername());
+            assertEquals(reindex.getRemoteInfo().getPassword(), tripped.getRemoteInfo().getPassword());
+        }
     }
 
     public void testUpdateByQueryRequest() throws IOException {
@@ -149,13 +167,19 @@ public class RoundTripTests extends ESTestCase {
                         randomSimpleString(random()), new IllegalArgumentException("test")));
     }
 
-    private List<ShardSearchFailure> randomSearchFailures() {
-        if (usually()) {
+    private List<SearchFailure> randomSearchFailures() {
+        if (randomBoolean()) {
             return emptyList();
         }
-        Index index = new Index(randomSimpleString(random()), "uuid");
-        return singletonList(new ShardSearchFailure(randomSimpleString(random()),
-                new SearchShardTarget(randomSimpleString(random()), index, randomInt()), randomFrom(RestStatus.values())));
+        String index = null;
+        Integer shardId = null;
+        String nodeId = null;
+        if (randomBoolean()) {
+            index = randomAsciiOfLength(5);
+            shardId = randomInt();
+            nodeId = usually() ? randomAsciiOfLength(5) : null;
+        }
+        return singletonList(new SearchFailure(new ElasticsearchException("foo"), index, shardId, nodeId));
     }
 
     private void roundTrip(Streamable example, Streamable empty) throws IOException {
@@ -182,10 +206,10 @@ public class RoundTripTests extends ESTestCase {
     private void assertResponseEquals(BulkIndexByScrollResponse expected, BulkIndexByScrollResponse actual) {
         assertEquals(expected.getTook(), actual.getTook());
         assertTaskStatusEquals(expected.getStatus(), actual.getStatus());
-        assertEquals(expected.getIndexingFailures().size(), actual.getIndexingFailures().size());
-        for (int i = 0; i < expected.getIndexingFailures().size(); i++) {
-            Failure expectedFailure = expected.getIndexingFailures().get(i);
-            Failure actualFailure = actual.getIndexingFailures().get(i);
+        assertEquals(expected.getBulkFailures().size(), actual.getBulkFailures().size());
+        for (int i = 0; i < expected.getBulkFailures().size(); i++) {
+            Failure expectedFailure = expected.getBulkFailures().get(i);
+            Failure actualFailure = actual.getBulkFailures().get(i);
             assertEquals(expectedFailure.getIndex(), actualFailure.getIndex());
             assertEquals(expectedFailure.getType(), actualFailure.getType());
             assertEquals(expectedFailure.getId(), actualFailure.getId());
@@ -194,13 +218,15 @@ public class RoundTripTests extends ESTestCase {
         }
         assertEquals(expected.getSearchFailures().size(), actual.getSearchFailures().size());
         for (int i = 0; i < expected.getSearchFailures().size(); i++) {
-            ShardSearchFailure expectedFailure = expected.getSearchFailures().get(i);
-            ShardSearchFailure actualFailure = actual.getSearchFailures().get(i);
-            assertEquals(expectedFailure.shard(), actualFailure.shard());
-            assertEquals(expectedFailure.status(), actualFailure.status());
-            // We can't use getCause because throwable doesn't implement equals
-            assertEquals(expectedFailure.reason(), actualFailure.reason());
+            SearchFailure expectedFailure = expected.getSearchFailures().get(i);
+            SearchFailure actualFailure = actual.getSearchFailures().get(i);
+            assertEquals(expectedFailure.getIndex(), actualFailure.getIndex());
+            assertEquals(expectedFailure.getShardId(), actualFailure.getShardId());
+            assertEquals(expectedFailure.getNodeId(), actualFailure.getNodeId());
+            assertEquals(expectedFailure.getReason().getClass(), actualFailure.getReason().getClass());
+            assertEquals(expectedFailure.getReason().getMessage(), actualFailure.getReason().getMessage());
         }
+        
     }
 
     private void assertTaskStatusEquals(BulkByScrollTask.Status expected, BulkByScrollTask.Status actual) {
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java
index bb6a33b593a..6ebb0749792 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryMetadataTests.java
@@ -21,13 +21,12 @@ package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.index.mapper.internal.RoutingFieldMapper;
 
 public class UpdateByQueryMetadataTests
         extends AbstractAsyncBulkIndexbyScrollActionMetadataTestCase<UpdateByQueryRequest, BulkIndexByScrollResponse> {
     public void testRoutingIsCopied() throws Exception {
         IndexRequest index = new IndexRequest();
-        action().copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc(RoutingFieldMapper.NAME, "foo"));
+        action().copyMetadata(AbstractAsyncBulkIndexByScrollAction.wrap(index), doc().setRouting("foo"));
         assertEquals("foo", index.routing());
     }
 
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java
index 1c57c202766..c5b9d4da64f 100644
--- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/UpdateByQueryWithScriptTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.script.ScriptService;
 
 import java.util.Date;
@@ -48,7 +49,7 @@ public class UpdateByQueryWithScriptTests
 
     @Override
     protected UpdateByQueryRequest request() {
-        return new UpdateByQueryRequest();
+        return new UpdateByQueryRequest(new SearchRequest());
     }
 
     @Override
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteInfoTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteInfoTests.java
new file mode 100644
index 00000000000..5492a05986c
--- /dev/null
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteInfoTests.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex.remote;
+
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.test.ESTestCase;
+
+public class RemoteInfoTests extends ESTestCase {
+    public void testToString() {
+        RemoteInfo info = new RemoteInfo("http", "testhost", 12344, new BytesArray("testquery"), null, null);
+        assertEquals("host=testhost port=12344 query=testquery", info.toString());
+        info = new RemoteInfo("http", "testhost", 12344, new BytesArray("testquery"), "testuser", null);
+        assertEquals("host=testhost port=12344 query=testquery username=testuser", info.toString());
+        info = new RemoteInfo("http", "testhost", 12344, new BytesArray("testquery"), "testuser", "testpass");
+        assertEquals("host=testhost port=12344 query=testquery username=testuser password=<<>>", info.toString());
+        info = new RemoteInfo("https", "testhost", 12344, new BytesArray("testquery"), "testuser", "testpass");
+        assertEquals("scheme=https host=testhost port=12344 query=testquery username=testuser password=<<>>", info.toString());
+    }
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java
new file mode 100644
index 00000000000..9bbfd175a79
--- /dev/null
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex.remote;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.entity.ContentType;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+
+import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.initialSearchEntity;
+import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.initialSearchParams;
+import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.initialSearchPath;
+import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.scrollEntity;
+import static org.elasticsearch.index.reindex.remote.RemoteRequestBuilders.scrollParams;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.either;
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.not;
+
+public class RemoteRequestBuildersTests extends ESTestCase {
+    public void testIntialSearchPath() {
+        SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder());
+
+        assertEquals("/_search", initialSearchPath(searchRequest));
+        searchRequest.indices("a");
+        searchRequest.types("b");
+        assertEquals("/a/b/_search", initialSearchPath(searchRequest));
+        searchRequest.indices("a", "b");
+        searchRequest.types("c", "d");
+        assertEquals("/a,b/c,d/_search", initialSearchPath(searchRequest));
+
+        searchRequest.indices("cat,");
+        expectBadStartRequest(searchRequest, "Index", ",", "cat,");
+        searchRequest.indices("cat,", "dog");
+        expectBadStartRequest(searchRequest, "Index", ",", "cat,");
+        searchRequest.indices("dog", "cat,");
+        expectBadStartRequest(searchRequest, "Index", ",", "cat,");
+        searchRequest.indices("cat/");
+        expectBadStartRequest(searchRequest, "Index", "/", "cat/");
+        searchRequest.indices("cat/", "dog");
+        expectBadStartRequest(searchRequest, "Index", "/", "cat/");
+        searchRequest.indices("dog", "cat/");
+        expectBadStartRequest(searchRequest, "Index", "/", "cat/");
+
+        searchRequest.indices("ok");
+        searchRequest.types("cat,");
+        expectBadStartRequest(searchRequest, "Type", ",", "cat,");
+        searchRequest.types("cat,", "dog");
+        expectBadStartRequest(searchRequest, "Type", ",", "cat,");
+        searchRequest.types("dog", "cat,");
+        expectBadStartRequest(searchRequest, "Type", ",", "cat,");
+        searchRequest.types("cat/");
+        expectBadStartRequest(searchRequest, "Type", "/", "cat/");
+        searchRequest.types("cat/", "dog");
+        expectBadStartRequest(searchRequest, "Type", "/", "cat/");
+        searchRequest.types("dog", "cat/");
+        expectBadStartRequest(searchRequest, "Type", "/", "cat/");
+    }
+
+    private void expectBadStartRequest(SearchRequest searchRequest, String type, String bad, String failed) {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> initialSearchPath(searchRequest));
+        assertEquals(type + " containing [" + bad + "] not supported but got [" + failed + "]", e.getMessage());
+    }
+
+    public void testInitialSearchParamsSort() {
+        SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder());
+
+        // Test sort:_doc for versions that support it.
+        Version remoteVersion = Version.fromId(between(Version.V_2_1_0_ID, Version.CURRENT.id));
+        searchRequest.source().sort("_doc");
+        assertThat(initialSearchParams(searchRequest, remoteVersion), hasEntry("sorts", "_doc:asc"));
+
+        // Test search_type scan for versions that don't support sort:_doc.
+        remoteVersion = Version.fromId(between(0, Version.V_2_1_0_ID - 1));
+        assertThat(initialSearchParams(searchRequest, remoteVersion), hasEntry("search_type", "scan"));
+
+        // Test sorting by some field. Version doesn't matter.
+        remoteVersion = Version.fromId(between(0, Version.CURRENT.id));
+        searchRequest.source().sorts().clear();
+        searchRequest.source().sort("foo");
+        assertThat(initialSearchParams(searchRequest, remoteVersion), hasEntry("sorts", "foo:asc"));
+    }
+
+    public void testInitialSearchParamsFields() {
+        SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder());
+
+        // Test request without any fields
+        Version remoteVersion = Version.fromId(between(0, Version.CURRENT.id));
+        assertThat(initialSearchParams(searchRequest, remoteVersion),
+                not(either(hasKey("stored_fields")).or(hasKey("fields"))));
+
+        // Setup some fields for the next two tests
+        searchRequest.source().storedField("_source").storedField("_id");
+
+        // Test stored_fields for versions that support it
+        remoteVersion = Version.fromId(between(Version.V_5_0_0_alpha4_ID, Version.CURRENT.id));
+        assertThat(initialSearchParams(searchRequest, remoteVersion), hasEntry("stored_fields", "_source,_id"));
+
+        // Test fields for versions that support it
+        remoteVersion = Version.fromId(between(0, Version.V_5_0_0_alpha4_ID - 1));
+        assertThat(initialSearchParams(searchRequest, remoteVersion), hasEntry("fields", "_source,_id"));
+    }
+
+    public void testInitialSearchParamsMisc() {
+        SearchRequest searchRequest = new SearchRequest().source(new SearchSourceBuilder());
+        Version remoteVersion = Version.fromId(between(0, Version.CURRENT.id));
+
+        TimeValue scroll = null;
+        if (randomBoolean()) {
+            scroll = TimeValue.parseTimeValue(randomPositiveTimeValue(), "test");
+            searchRequest.scroll(scroll);
+        }
+        int size = between(0, Integer.MAX_VALUE);
+        searchRequest.source().size(size);
+        Boolean fetchVersion = null;
+        if (randomBoolean()) {
+            fetchVersion = randomBoolean();
+            searchRequest.source().version(fetchVersion);
+        }
+
+        Map<String, String> params = initialSearchParams(searchRequest, remoteVersion);
+
+        assertThat(params, scroll == null ? not(hasKey("scroll")) : hasEntry("scroll", scroll.toString()));
+        assertThat(params, hasEntry("size", Integer.toString(size)));
+        assertThat(params, fetchVersion == null || fetchVersion == true ? hasEntry("version", null) : not(hasEntry("version", null)));
+    }
+
+    public void testInitialSearchEntity() throws IOException {
+        String query = "{\"match_all\":{}}";
+        HttpEntity entity = initialSearchEntity(new BytesArray(query));
+        assertEquals(ContentType.APPLICATION_JSON.toString(), entity.getContentType().getValue());
+        assertEquals("{\"query\":" + query + "}",
+                Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)));
+
+        // Invalid XContent fails
+        RuntimeException e = expectThrows(RuntimeException.class, () -> initialSearchEntity(new BytesArray("{}, \"trailing\": {}")));
+        assertThat(e.getCause().getMessage(), containsString("Unexpected character (',' (code 44))"));
+        e = expectThrows(RuntimeException.class, () -> initialSearchEntity(new BytesArray("{")));
+        assertThat(e.getCause().getMessage(), containsString("Unexpected end-of-input"));
+    }
+
+    public void testScrollParams() {
+        TimeValue scroll = TimeValue.parseTimeValue(randomPositiveTimeValue(), "test");
+        assertThat(scrollParams(scroll), hasEntry("scroll", scroll.toString()));
+    }
+
+    public void testScrollEntity() throws IOException {
+        String scroll = randomAsciiOfLength(30);
+        HttpEntity entity = scrollEntity(scroll);
+        assertEquals(ContentType.TEXT_PLAIN.toString(), entity.getContentType().getValue());
+        assertEquals(scroll, Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8)));
+    }
+}
diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java
new file mode 100644
index 00000000000..f8f3e82b4bb
--- /dev/null
+++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteScrollableHitSourceTests.java
@@ -0,0 +1,381 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.reindex.remote;
+
+import org.apache.http.HttpEntity;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.bulk.BackoffPolicy;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.Streams;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.index.reindex.ScrollableHitSource.Response;
+import org.elasticsearch.index.reindex.remote.RemoteScrollableHitSource.ResponseListener;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.threadpool.TestThreadPool;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
+
+import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
+import static org.elasticsearch.common.unit.TimeValue.timeValueMinutes;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class RemoteScrollableHitSourceTests extends ESTestCase {
+    private final String FAKE_SCROLL_ID = "DnF1ZXJ5VGhlbkZldGNoBQAAAfakescroll";
+    private int retries;
+    private ThreadPool threadPool;
+    private SearchRequest searchRequest;
+    private int retriesAllowed;
+
+    @Before
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+        threadPool = new TestThreadPool(getTestName()) {
+            @Override
+            public Executor executor(String name) {
+                return r -> r.run();
+            }
+
+            @Override
+            public ScheduledFuture<?> schedule(TimeValue delay, String name, Runnable command) {
+                command.run();
+                return null;
+            }
+        };
+        retries = 0;
+        searchRequest = new SearchRequest();
+        searchRequest.scroll(timeValueMinutes(5));
+        searchRequest.source(new SearchSourceBuilder().size(10).version(true).sort("_doc").size(123));
+        retriesAllowed = 0;
+    }
+
+    @After
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        terminate(threadPool);
+    }
+
+    public void testLookupRemoteVersion() throws Exception {
+        sourceWithMockedRemoteCall(false, "main/0_20_5.json").lookupRemoteVersion(v -> assertEquals(Version.fromString("0.20.5"), v));
+        sourceWithMockedRemoteCall(false, "main/0_90_13.json").lookupRemoteVersion(v -> assertEquals(Version.fromString("0.90.13"), v));
+        sourceWithMockedRemoteCall(false, "main/1_7_5.json").lookupRemoteVersion(v -> assertEquals(Version.fromString("1.7.5"), v));
+        sourceWithMockedRemoteCall(false, "main/2_3_3.json").lookupRemoteVersion(v -> assertEquals(Version.V_2_3_3, v));
+        sourceWithMockedRemoteCall(false, "main/5_0_0_alpha_3.json").lookupRemoteVersion(v -> assertEquals(Version.V_5_0_0_alpha3, v));
+    }
+
+    public void testParseStartOk() throws Exception {
+        AtomicBoolean called = new AtomicBoolean();
+        sourceWithMockedRemoteCall("start_ok.json").doStart(r -> {
+            assertFalse(r.isTimedOut());
+            assertEquals(FAKE_SCROLL_ID, r.getScrollId());
+            assertEquals(4, r.getTotalHits());
+            assertThat(r.getFailures(), empty());
+            assertThat(r.getHits(), hasSize(1));
+            assertEquals("test", r.getHits().get(0).getIndex());
+            assertEquals("test", r.getHits().get(0).getType());
+            assertEquals("AVToMiC250DjIiBO3yJ_", r.getHits().get(0).getId());
+            assertEquals("{\"test\":\"test2\"}", r.getHits().get(0).getSource().utf8ToString());
+            assertNull(r.getHits().get(0).getTTL());
+            assertNull(r.getHits().get(0).getTimestamp());
+            assertNull(r.getHits().get(0).getRouting());
+            called.set(true);
+        });
+        assertTrue(called.get());
+    }
+
+    public void testParseScrollOk() throws Exception {
+        AtomicBoolean called = new AtomicBoolean();
+        sourceWithMockedRemoteCall("scroll_ok.json").doStartNextScroll("", timeValueMillis(0), r -> {
+            assertFalse(r.isTimedOut());
+            assertEquals(FAKE_SCROLL_ID, r.getScrollId());
+            assertEquals(4, r.getTotalHits());
+            assertThat(r.getFailures(), empty());
+            assertThat(r.getHits(), hasSize(1));
+            assertEquals("test", r.getHits().get(0).getIndex());
+            assertEquals("test", r.getHits().get(0).getType());
+            assertEquals("AVToMiDL50DjIiBO3yKA", r.getHits().get(0).getId());
+            assertEquals("{\"test\":\"test3\"}", r.getHits().get(0).getSource().utf8ToString());
+            assertNull(r.getHits().get(0).getTTL());
+            assertNull(r.getHits().get(0).getTimestamp());
+            assertNull(r.getHits().get(0).getRouting());
+            called.set(true);
+        });
+        assertTrue(called.get());
+    }
+
+    /**
+     * Test for parsing _ttl, _timestamp, and _routing.
+     */
+    public void testParseScrollFullyLoaded() throws Exception {
+        AtomicBoolean called = new AtomicBoolean();
+        sourceWithMockedRemoteCall("scroll_fully_loaded.json").doStartNextScroll("", timeValueMillis(0), r -> {
+            assertEquals("AVToMiDL50DjIiBO3yKA", r.getHits().get(0).getId());
+            assertEquals("{\"test\":\"test3\"}", r.getHits().get(0).getSource().utf8ToString());
+            assertEquals((Long) 1234L, r.getHits().get(0).getTTL());
+            assertEquals((Long) 123444L, r.getHits().get(0).getTimestamp());
+            assertEquals("testrouting", r.getHits().get(0).getRouting());
+            assertEquals("testparent", r.getHits().get(0).getParent());
+            called.set(true);
+        });
+        assertTrue(called.get());
+    }
+
+    /**
+     * Versions of Elasticsearch before 2.1.0 don't support sort:_doc and instead need to use search_type=scan. Scan doesn't return
+     * documents the first iteration but reindex doesn't like that. So we jump start strait to the next iteration.
+     */
+    public void testScanJumpStart() throws Exception {
+        AtomicBoolean called = new AtomicBoolean();
+        sourceWithMockedRemoteCall("start_scan.json", "scroll_ok.json").doStart(r -> {
+            assertFalse(r.isTimedOut());
+            assertEquals(FAKE_SCROLL_ID, r.getScrollId());
+            assertEquals(4, r.getTotalHits());
+            assertThat(r.getFailures(), empty());
+            assertThat(r.getHits(), hasSize(1));
+            assertEquals("test", r.getHits().get(0).getIndex());
+            assertEquals("test", r.getHits().get(0).getType());
+            assertEquals("AVToMiDL50DjIiBO3yKA", r.getHits().get(0).getId());
+            assertEquals("{\"test\":\"test3\"}", r.getHits().get(0).getSource().utf8ToString());
+            assertNull(r.getHits().get(0).getTTL());
+            assertNull(r.getHits().get(0).getTimestamp());
+            assertNull(r.getHits().get(0).getRouting());
+            called.set(true);
+        });
+        assertTrue(called.get());
+    }
+
+    public void testParseRejection() throws Exception {
+        // The rejection comes through in the handler because the mocked http response isn't marked as an error
+        AtomicBoolean called = new AtomicBoolean();
+        // Handling a scroll rejection is the same as handling a search rejection so we reuse the verification code
+        Consumer<Response> checkResponse = r -> {
+            assertFalse(r.isTimedOut());
+            assertEquals(FAKE_SCROLL_ID, r.getScrollId());
+            assertEquals(4, r.getTotalHits());
+            assertThat(r.getFailures(), hasSize(1));
+            assertEquals("test", r.getFailures().get(0).getIndex());
+            assertEquals((Integer) 0, r.getFailures().get(0).getShardId());
+            assertEquals("87A7NvevQxSrEwMbtRCecg", r.getFailures().get(0).getNodeId());
+            assertThat(r.getFailures().get(0).getReason(), instanceOf(EsRejectedExecutionException.class));
+            assertEquals("rejected execution of org.elasticsearch.transport.TransportService$5@52d06af2 on "
+                    + "EsThreadPoolExecutor[search, queue capacity = 1000, org.elasticsearch.common.util.concurrent."
+                    + "EsThreadPoolExecutor@778ea553[Running, pool size = 7, active threads = 7, queued tasks = 1000, "
+                    + "completed tasks = 4182]]", r.getFailures().get(0).getReason().getMessage());
+            assertThat(r.getHits(), hasSize(1));
+            assertEquals("test", r.getHits().get(0).getIndex());
+            assertEquals("test", r.getHits().get(0).getType());
+            assertEquals("AVToMiC250DjIiBO3yJ_", r.getHits().get(0).getId());
+            assertEquals("{\"test\":\"test1\"}", r.getHits().get(0).getSource().utf8ToString());
+            called.set(true);
+        };
+        sourceWithMockedRemoteCall("rejection.json").doStart(checkResponse);
+        assertTrue(called.get());
+        called.set(false);
+        sourceWithMockedRemoteCall("rejection.json").doStartNextScroll("scroll", timeValueMillis(0), checkResponse);
+        assertTrue(called.get());
+    }
+
+    public void testParseFailureWithStatus() throws Exception {
+        // The rejection comes through in the handler because the mocked http response isn't marked as an error
+        AtomicBoolean called = new AtomicBoolean();
+        // Handling a scroll rejection is the same as handling a search rejection so we reuse the verification code
+        Consumer<Response> checkResponse = r -> {
+            assertFalse(r.isTimedOut());
+            assertEquals(FAKE_SCROLL_ID, r.getScrollId());
+            assertEquals(10000, r.getTotalHits());
+            assertThat(r.getFailures(), hasSize(1));
+            assertEquals(null, r.getFailures().get(0).getIndex());
+            assertEquals(null, r.getFailures().get(0).getShardId());
+            assertEquals(null, r.getFailures().get(0).getNodeId());
+            assertThat(r.getFailures().get(0).getReason(), instanceOf(RuntimeException.class));
+            assertEquals("Unknown remote exception with reason=[SearchContextMissingException[No search context found for id [82]]]",
+                    r.getFailures().get(0).getReason().getMessage());
+            assertThat(r.getHits(), hasSize(1));
+            assertEquals("test", r.getHits().get(0).getIndex());
+            assertEquals("test", r.getHits().get(0).getType());
+            assertEquals("10000", r.getHits().get(0).getId());
+            assertEquals("{\"test\":\"test10000\"}", r.getHits().get(0).getSource().utf8ToString());
+            called.set(true);
+        };
+        sourceWithMockedRemoteCall("failure_with_status.json").doStart(checkResponse);
+        assertTrue(called.get());
+        called.set(false);
+        sourceWithMockedRemoteCall("failure_with_status.json").doStartNextScroll("scroll", timeValueMillis(0), checkResponse);
+        assertTrue(called.get());
+    }
+
+    public void testParseRequestFailure() throws Exception {
+        AtomicBoolean called = new AtomicBoolean();
+        Consumer<Response> checkResponse = r -> {
+            assertFalse(r.isTimedOut());
+            assertNull(r.getScrollId());
+            assertEquals(0, r.getTotalHits());
+            assertThat(r.getFailures(), hasSize(1));
+            assertThat(r.getFailures().get(0).getReason(), instanceOf(ParsingException.class));
+            ParsingException failure = (ParsingException) r.getFailures().get(0).getReason();
+            assertEquals("Unknown key for a VALUE_STRING in [invalid].", failure.getMessage());
+            assertEquals(2, failure.getLineNumber());
+            assertEquals(14, failure.getColumnNumber());
+            called.set(true);
+        };
+        sourceWithMockedRemoteCall("request_failure.json").doStart(checkResponse);
+        assertTrue(called.get());
+        called.set(false);
+        sourceWithMockedRemoteCall("request_failure.json").doStartNextScroll("scroll", timeValueMillis(0), checkResponse);
+        assertTrue(called.get());
+    }
+
+    public void testRetryAndSucceed() throws Exception {
+        AtomicBoolean called = new AtomicBoolean();
+        Consumer<Response> checkResponse = r -> {
+            assertThat(r.getFailures(), hasSize(0));
+            called.set(true);
+        };
+        retriesAllowed = between(1, Integer.MAX_VALUE);
+        sourceWithMockedRemoteCall("fail:rejection.json", "start_ok.json").doStart(checkResponse);
+        assertTrue(called.get());
+        assertEquals(1, retries);
+        retries = 0;
+        called.set(false);
+        sourceWithMockedRemoteCall("fail:rejection.json", "scroll_ok.json").doStartNextScroll("scroll", timeValueMillis(0),
+                checkResponse);
+        assertTrue(called.get());
+        assertEquals(1, retries);
+    }
+
+    public void testRetryUntilYouRunOutOfTries() throws Exception {
+        AtomicBoolean called = new AtomicBoolean();
+        Consumer<Response> checkResponse = r -> called.set(true);
+        retriesAllowed = between(0, 10);
+        String[] paths = new String[retriesAllowed + 2];
+        for (int i = 0; i < retriesAllowed + 2; i++) {
+            paths[i] = "fail:rejection.json";
+        }
+        RuntimeException e = expectThrows(RuntimeException.class, () -> sourceWithMockedRemoteCall(paths).doStart(checkResponse));
+        assertEquals("failed", e.getMessage());
+        assertFalse(called.get());
+        assertEquals(retriesAllowed, retries);
+        retries = 0;
+        e = expectThrows(RuntimeException.class,
+                () -> sourceWithMockedRemoteCall(paths).doStartNextScroll("scroll", timeValueMillis(0), checkResponse));
+        assertEquals("failed", e.getMessage());
+        assertFalse(called.get());
+        assertEquals(retriesAllowed, retries);
+    }
+
+    private RemoteScrollableHitSource sourceWithMockedRemoteCall(String... paths) throws Exception {
+        return sourceWithMockedRemoteCall(true, paths);
+    }
+
+    /**
+     * Creates a hit source that doesn't make the remote request and instead returns data from some files. Also requests are always returned
+     * synchronously rather than asynchronously.
+     */
+    private RemoteScrollableHitSource sourceWithMockedRemoteCall(boolean mockRemoteVersion, String... paths) throws Exception {
+        URL[] resources = new URL[paths.length];
+        for (int i = 0; i < paths.length; i++) {
+            resources[i] = Thread.currentThread().getContextClassLoader().getResource("responses/" + paths[i].replace("fail:", ""));
+            if (resources[i] == null) {
+                throw new IllegalArgumentException("Couldn't find [" + paths[i] + "]");
+            }
+        }
+        RemoteScrollableHitSource.AsyncClient client = new RemoteScrollableHitSource.AsyncClient() {
+            int responseCount = 0;
+            @Override
+            public void performRequest(String method, String uri, Map<String, String> params, HttpEntity entity,
+                    ResponseListener listener) {
+                try {
+                    URL resource = resources[responseCount];
+                    String path = paths[responseCount++];
+                    InputStream stream = resource.openStream();
+                    if (path.startsWith("fail:")) {
+                        String body = Streams.copyToString(new InputStreamReader(entity.getContent(), StandardCharsets.UTF_8));
+                        if (path.equals("fail:rejection.json")) {
+                            listener.onRetryableFailure(new RuntimeException(body));
+                        } else {
+                            listener.onFailure(new RuntimeException(body));
+                        }
+                    } else {
+                        listener.onResponse(stream);
+                    }
+                } catch (IOException e) {
+                    listener.onFailure(e);
+                }
+            }
+
+            @Override
+            public void close() throws IOException {
+            }
+        };
+        TestRemoteScrollableHitSource hitSource = new TestRemoteScrollableHitSource(client) {
+            @Override
+            void lookupRemoteVersion(Consumer<Version> onVersion) {
+                if (mockRemoteVersion) {
+                    onVersion.accept(Version.CURRENT);
+                } else {
+                    super.lookupRemoteVersion(onVersion);
+                }
+            }
+        };
+        if (mockRemoteVersion) {
+            hitSource.remoteVersion = Version.CURRENT;
+        }
+        return hitSource;
+    }
+
+    private BackoffPolicy backoff() {
+        return BackoffPolicy.constantBackoff(timeValueMillis(0), retriesAllowed);
+    }
+
+    private void countRetry() {
+        retries += 1;
+    }
+
+    private void failRequest(Throwable t) {
+        throw new RuntimeException("failed", t);
+    }
+
+    private class TestRemoteScrollableHitSource extends RemoteScrollableHitSource {
+        public TestRemoteScrollableHitSource(RemoteScrollableHitSource.AsyncClient client) {
+            super(RemoteScrollableHitSourceTests.this.logger, backoff(), RemoteScrollableHitSourceTests.this.threadPool,
+                    RemoteScrollableHitSourceTests.this::countRetry, RemoteScrollableHitSourceTests.this::failRequest, client,
+                    new BytesArray("{}"), RemoteScrollableHitSourceTests.this.searchRequest);
+        }
+    }
+}
diff --git a/modules/reindex/src/test/resources/responses/failure_with_status.json b/modules/reindex/src/test/resources/responses/failure_with_status.json
new file mode 100644
index 00000000000..314de37a679
--- /dev/null
+++ b/modules/reindex/src/test/resources/responses/failure_with_status.json
@@ -0,0 +1,28 @@
+{
+  "_scroll_id": "DnF1ZXJ5VGhlbkZldGNoBQAAAfakescroll",
+  "took": 3,
+  "timed_out": false,
+  "_shards": {
+    "total": 5,
+    "successful": 4,
+    "failed": 1,
+    "failures": [ {
+      "status": 404,
+      "reason": "SearchContextMissingException[No search context found for id [82]]"
+    } ]
+  },
+  "hits": {
+    "total": 10000,
+    "max_score": 0.0,
+    "hits": [ {
+      "_index": "test",
+      "_type": "test",
+      "_id": "10000",
+      "_version": 1,
+      "_score": 0.0,
+      "_source": {
+        "test": "test10000"
+      }
+    } ]
+  }
+}
diff --git a/modules/reindex/src/test/resources/responses/main/0_20_5.json b/modules/reindex/src/test/resources/responses/main/0_20_5.json
new file mode 100644
index 00000000000..58ca8e9428f
--- /dev/null
+++ b/modules/reindex/src/test/resources/responses/main/0_20_5.json
@@ -0,0 +1,10 @@
+{
+  "ok" : true,
+  "status" : 200,
+  "name" : "Techno",
+  "version" : {
+    "number" : "0.20.5",
+    "snapshot_build" : false
+  },
+  "tagline" : "You Know, for Search"
+}
diff --git a/modules/reindex/src/test/resources/responses/main/0_90_13.json b/modules/reindex/src/test/resources/responses/main/0_90_13.json
new file mode 100644
index 00000000000..1b104e068d9
--- /dev/null
+++ b/modules/reindex/src/test/resources/responses/main/0_90_13.json
@@ -0,0 +1,13 @@
+{
+  "ok" : true,
+  "status" : 200,
+  "name" : "Mogul of the Mystic Mountain",
+  "version" : {
+    "number" : "0.90.13",
+    "build_hash" : "249c9c5e06765c9e929e92b1d235e1ba4dc679fa",
+    "build_timestamp" : "2014-03-25T15:27:12Z",
+    "build_snapshot" : false,
+    "lucene_version" : "4.6"
+  },
+  "tagline" : "You Know, for Search"
+}
diff --git a/modules/reindex/src/test/resources/responses/main/1_7_5.json b/modules/reindex/src/test/resources/responses/main/1_7_5.json
new file mode 100644
index 00000000000..0fe721defee
--- /dev/null
+++ b/modules/reindex/src/test/resources/responses/main/1_7_5.json
@@ -0,0 +1,13 @@
+{
+  "status" : 200,
+  "name" : "Robert Kelly",
+  "cluster_name" : "elasticsearch",
+  "version" : {
+    "number" : "1.7.5",
+    "build_hash" : "00f95f4ffca6de89d68b7ccaf80d148f1f70e4d4",
+    "build_timestamp" : "2016-02-02T09:55:30Z",
+    "build_snapshot" : false,
+    "lucene_version" : "4.10.4"
+  },
+  "tagline" : "You Know, for Search"
+}
diff --git a/modules/reindex/src/test/resources/responses/main/2_3_3.json b/modules/reindex/src/test/resources/responses/main/2_3_3.json
new file mode 100644
index 00000000000..8cd90b3b637
--- /dev/null
+++ b/modules/reindex/src/test/resources/responses/main/2_3_3.json
@@ -0,0 +1,12 @@
+{
+  "name" : "Ezekiel Stane",
+  "cluster_name" : "elasticsearch",
+  "version" : {
+    "number" : "2.3.3",
+    "build_hash" : "218bdf10790eef486ff2c41a3df5cfa32dadcfde",
+    "build_timestamp" : "2016-05-17T15:40:04Z",
+    "build_snapshot" : false,
+    "lucene_version" : "5.5.0"
+  },
+  "tagline" : "You Know, for Search"
+}
diff --git a/modules/reindex/src/test/resources/responses/main/5_0_0_alpha_3.json b/modules/reindex/src/test/resources/responses/main/5_0_0_alpha_3.json
new file mode 100644
index 00000000000..6911f61c3e9
--- /dev/null
+++ b/modules/reindex/src/test/resources/responses/main/5_0_0_alpha_3.json
@@ -0,0 +1,12 @@
+{
+  "name" : "Paibo",
+  "cluster_name" : "distribution_run",
+  "version" : {
+    "number" : "5.0.0-alpha3",
+    "build_hash" : "42e092f",
+    "build_date" : "2016-05-26T16:55:45.405Z",
+    "build_snapshot" : true,
+    "lucene_version" : "6.0.0"
+  },
+  "tagline" : "You Know, for Search"
+}
diff --git a/modules/reindex/src/test/resources/responses/rejection.json b/modules/reindex/src/test/resources/responses/rejection.json
new file mode 100644
index 00000000000..36120fbf888
--- /dev/null
+++ b/modules/reindex/src/test/resources/responses/rejection.json
@@ -0,0 +1,34 @@
+{
+  "_scroll_id" : "DnF1ZXJ5VGhlbkZldGNoBQAAAfakescroll",
+  "took" : 6,
+  "timed_out" : false,
+  "_shards" : {
+    "total" : 5,
+    "successful" : 4,
+    "failed" : 1,
+    "failures" : [ {
+      "shard" : 0,
+      "index" : "test",
+      "node" : "87A7NvevQxSrEwMbtRCecg",
+      "reason" : {
+        "type" : "es_rejected_execution_exception",
+        "reason" : "rejected execution of org.elasticsearch.transport.TransportService$5@52d06af2 on EsThreadPoolExecutor[search, queue capacity = 1000, org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor@778ea553[Running, pool size = 7, active threads = 7, queued tasks = 1000, completed tasks = 4182]]"
+      }
+    } ]
+  },
+  "hits" : {
+    "total" : 4,
+    "max_score" : null,
+    "hits" : [ {
+      "_index" : "test",
+      "_type" : "test",
+      "_id" : "AVToMiC250DjIiBO3yJ_",
+      "_version" : 1,
+      "_score" : null,
+      "_source" : {
+        "test" : "test1"
+      },
+      "sort" : [ 0 ]
+    } ]
+  }
+}
diff --git a/modules/reindex/src/test/resources/responses/request_failure.json b/modules/reindex/src/test/resources/responses/request_failure.json
new file mode 100644
index 00000000000..6f6de78c060
--- /dev/null
+++ b/modules/reindex/src/test/resources/responses/request_failure.json
@@ -0,0 +1,15 @@
+{
+  "error" : {
+    "root_cause" : [ {
+      "type" : "parsing_exception",
+      "reason" : "Unknown key for a VALUE_STRING in [invalid].",
+      "line" : 2,
+      "col" : 14
+    } ],
+    "type" : "parsing_exception",
+    "reason" : "Unknown key for a VALUE_STRING in [invalid].",
+    "line" : 2,
+    "col" : 14
+  },
+  "status" : 400
+}
diff --git a/modules/reindex/src/test/resources/responses/scroll_fully_loaded.json b/modules/reindex/src/test/resources/responses/scroll_fully_loaded.json
new file mode 100644
index 00000000000..a2c1be34e5c
--- /dev/null
+++ b/modules/reindex/src/test/resources/responses/scroll_fully_loaded.json
@@ -0,0 +1,30 @@
+{
+  "_scroll_id" : "DnF1ZXJ5VGhlbkZldGNoBQAAAfakescroll",
+  "took" : 3,
+  "timed_out" : false,
+  "terminated_early" : true,
+  "_shards" : {
+    "total" : 5,
+    "successful" : 5,
+    "failed" : 0
+  },
+  "hits" : {
+    "total" : 4,
+    "max_score" : null,
+    "hits" : [ {
+      "_index" : "test",
+      "_type" : "test",
+      "_id" : "AVToMiDL50DjIiBO3yKA",
+      "_version" : 1,
+      "_score" : null,
+      "_source" : {
+        "test" : "test3"
+      },
+      "sort" : [ 0 ],
+      "_routing": "testrouting",
+      "_parent": "testparent",
+      "_ttl" : 1234,
+      "_timestamp": 123444
+    } ]
+  }
+}
diff --git a/modules/reindex/src/test/resources/responses/scroll_ok.json b/modules/reindex/src/test/resources/responses/scroll_ok.json
new file mode 100644
index 00000000000..5cdc4a400cb
--- /dev/null
+++ b/modules/reindex/src/test/resources/responses/scroll_ok.json
@@ -0,0 +1,26 @@
+{
+  "_scroll_id" : "DnF1ZXJ5VGhlbkZldGNoBQAAAfakescroll",
+  "took" : 3,
+  "timed_out" : false,
+  "terminated_early" : true,
+  "_shards" : {
+    "total" : 5,
+    "successful" : 5,
+    "failed" : 0
+  },
+  "hits" : {
+    "total" : 4,
+    "max_score" : null,
+    "hits" : [ {
+      "_index" : "test",
+      "_type" : "test",
+      "_id" : "AVToMiDL50DjIiBO3yKA",
+      "_version" : 1,
+      "_score" : null,
+      "_source" : {
+        "test" : "test3"
+      },
+      "sort" : [ 0 ]
+    } ]
+  }
+}
diff --git a/modules/reindex/src/test/resources/responses/start_ok.json b/modules/reindex/src/test/resources/responses/start_ok.json
new file mode 100644
index 00000000000..a2988341f8c
--- /dev/null
+++ b/modules/reindex/src/test/resources/responses/start_ok.json
@@ -0,0 +1,25 @@
+{
+  "_scroll_id" : "DnF1ZXJ5VGhlbkZldGNoBQAAAfakescroll",
+  "took" : 6,
+  "timed_out" : false,
+  "_shards" : {
+    "total" : 5,
+    "successful" : 5,
+    "failed" : 0
+  },
+  "hits" : {
+    "total" : 4,
+    "max_score" : null,
+    "hits" : [ {
+      "_index" : "test",
+      "_type" : "test",
+      "_id" : "AVToMiC250DjIiBO3yJ_",
+      "_version" : 1,
+      "_score" : null,
+      "_source" : {
+        "test" : "test2"
+      },
+      "sort" : [ 0 ]
+    } ]
+  }
+}
diff --git a/modules/reindex/src/test/resources/responses/start_scan.json b/modules/reindex/src/test/resources/responses/start_scan.json
new file mode 100644
index 00000000000..5576d708b30
--- /dev/null
+++ b/modules/reindex/src/test/resources/responses/start_scan.json
@@ -0,0 +1,15 @@
+{
+  "_scroll_id" : "c2Nhbjs1OzQ0Ojd5aUZoUm5hU2lDZ3ZvUHMzMXdGQ1E7NDU6N3lpRmhSbmFTaUNndm9QczMxd0ZDUTs0Mjo3eWlGaFJuYVNpQ2d2b1BzMzF3RkNROzQzOjd5aUZoUm5hU2lDZ3ZvUHMzMXdGQ1E7NDE6N3lpRmhSbmFTaUNndm9QczMxd0ZDUTsxO3RvdGFsX2hpdHM6MTAwMDA7",
+  "took" : 13,
+  "timed_out" : false,
+  "_shards" : {
+    "total" : 5,
+    "successful" : 5,
+    "failed" : 0
+  },
+  "hits" : {
+    "total" : 10000,
+    "max_score" : 0.0,
+    "hits" : [ ]
+  }
+}
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
index b3e35d2f49e..e31c8f84cec 100644
--- a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/20_validation.yaml
@@ -224,3 +224,46 @@
             stored_fields: [_id]
           dest:
             index: dest
+
+---
+"unwhitelisted remote host fails":
+  - do:
+      catch: /\[badremote:9200\] not whitelisted in reindex.remote.whitelist/
+      reindex:
+        body:
+          source:
+            remote:
+              host: http://badremote:9200
+            index: test
+          dest:
+            index: dest
+
+---
+"badly formatted remote host fails":
+  - do:
+      catch: /\[host\] must be of the form \[scheme\].//\[host\].\[port\]/
+      reindex:
+        body:
+          source:
+            remote:
+              host: badremote
+              weird: stuff
+              badkey: is bad
+            index: test
+          dest:
+            index: dest
+
+---
+"junk in remote fails":
+  - do:
+      catch: /Unsupported fields in \[remote\]. \[weird,badkey\]/
+      reindex:
+        body:
+          source:
+            remote:
+              host: http://okremote:9200
+              weird: stuff
+              badkey: is bad
+            index: test
+          dest:
+            index: dest
diff --git a/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yaml b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yaml
new file mode 100644
index 00000000000..6adac98ad77
--- /dev/null
+++ b/modules/reindex/src/test/resources/rest-api-spec/test/reindex/90_remote.yaml
@@ -0,0 +1,207 @@
+---
+"Basic reindex from remote":
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+        refresh: true
+
+  # Fetch the http host. We use the host of the master because we know there will always be a master.
+  - do:
+      cluster.state: {}
+  - set: { master_node: master }
+  - do:
+      nodes.info:
+        metric: [ http ]
+  - is_true: nodes.$master.http.publish_address
+  - set: {nodes.$master.http.publish_address: host}
+  - do:
+      reindex:
+        refresh: true
+        body:
+          source:
+            remote:
+              host: http://${host}
+            index: source
+          dest:
+            index: dest
+  - match: {created: 1}
+  - match: {updated: 0}
+  - match: {version_conflicts: 0}
+  - match: {batches: 1}
+  - match: {failures: []}
+  - match: {throttled_millis: 0}
+  - gte: { took: 0 }
+  - is_false: task
+  - is_false: deleted
+
+  - do:
+      search:
+        index: dest
+        body:
+          query:
+            match:
+              text: test
+  - match: {hits.total: 1}
+
+---
+"Reindex from remote with query":
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      2
+        body:    { "text": "test2" }
+  - do:
+      indices.refresh: {}
+
+  # Fetch the http host. We use the host of the master because we know there will always be a master.
+  - do:
+      cluster.state: {}
+  - set: { master_node: master }
+  - do:
+      nodes.info:
+        metric: [ http ]
+  - is_true: nodes.$master.http.publish_address
+  - set: {nodes.$master.http.publish_address: host}
+  - do:
+      reindex:
+        refresh: true
+        body:
+          source:
+            remote:
+              host: http://${host}
+            index: source
+            query:
+              match:
+                text: test2
+          dest:
+            index: dest
+  - match: {created: 1}
+
+  - do:
+      search:
+        index: dest
+        body:
+          query:
+            match_all: {}
+  - match: {hits.total: 1}
+
+---
+"Reindex from remote with routing":
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+        routing: foo
+        refresh: true
+
+  # Fetch the http host. We use the host of the master because we know there will always be a master.
+  - do:
+      cluster.state: {}
+  - set: { master_node: master }
+  - do:
+      nodes.info:
+        metric: [ http ]
+  - is_true: nodes.$master.http.publish_address
+  - set: {nodes.$master.http.publish_address: host}
+  - do:
+      reindex:
+        refresh: true
+        body:
+          source:
+            remote:
+              host: http://${host}
+            index: source
+          dest:
+            index: dest
+  - match: {created: 1}
+
+  - do:
+      search:
+        index: dest
+        routing: foo
+        body:
+          query:
+            match:
+              text: test
+  - match: {hits.total: 1}
+
+---
+"Reindex from remote with parent/child":
+  - do:
+      indices.create:
+          index: source
+          body:
+              mappings:
+                foo: {}
+                bar:
+                  _parent:
+                    type:  foo
+  - do:
+      indices.create:
+          index: dest
+          body:
+              mappings:
+                foo: {}
+                bar:
+                  _parent:
+                    type:  foo
+  - do:
+      index:
+        index:   source
+        type:    foo
+        id:      1
+        body:    { "text": "test" }
+  - do:
+      index:
+        index:   source
+        type:    bar
+        id:      1
+        parent:  1
+        body:    { "text": "test2" }
+  - do:
+      indices.refresh: {}
+
+  # Fetch the http host. We use the host of the master because we know there will always be a master.
+  - do:
+      cluster.state: {}
+  - set: { master_node: master }
+  - do:
+      nodes.info:
+        metric: [ http ]
+  - is_true: nodes.$master.http.publish_address
+  - set: {nodes.$master.http.publish_address: host}
+  - do:
+      reindex:
+        refresh: true
+        body:
+          source:
+            remote:
+              host: http://${host}
+            index: source
+          dest:
+            index: dest
+  - match: {created: 2}
+
+  - do:
+      search:
+        index: dest
+        body:
+          query:
+            has_parent:
+              parent_type: foo
+              query:
+                match:
+                  text: test
+  - match: {hits.total: 1}
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java b/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
index a4218f83340..f687f2b39bf 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java
@@ -71,7 +71,7 @@ public class Stash implements ToXContent {
      * as arguments for following requests (e.g. scroll_id)
      */
     public boolean containsStashedValue(Object key) {
-        if (key == null) {
+        if (key == null || false == key instanceof CharSequence) {
             return false;
         }
         String stashKey = key.toString();
diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/test/StashTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/test/StashTests.java
new file mode 100644
index 00000000000..7d0c0598f09
--- /dev/null
+++ b/test/framework/src/test/java/org/elasticsearch/test/rest/test/StashTests.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.rest.test;
+
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.rest.Stash;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static java.util.Collections.singletonMap;
+
+public class StashTests extends ESTestCase {
+    public void testReplaceStashedValuesEmbeddedStashKey() throws IOException {
+        Stash stash = new Stash();
+        stash.stashValue("stashed", "bar");
+        
+        Map<String, Object> expected = new HashMap<>();
+        expected.put("key", singletonMap("a", "foobar"));
+        Map<String, Object> map = new HashMap<>();
+        Map<String, Object> map2 = new HashMap<>();
+        map2.put("a", "foo${stashed}");
+        map.put("key", map2);
+
+        Map<String, Object> actual = stash.replaceStashedValues(map);
+        assertEquals(expected, actual);
+    }
+}

From 74af0e36f3bc6b7f3315ba23463fca78997d60aa Mon Sep 17 00:00:00 2001
From: Igor Motov <igor@motovs.org>
Date: Tue, 5 Jul 2016 16:40:03 -0400
Subject: [PATCH 1250/1311] [TEST] Fix rare OBOE in
 AbstractBytesReferenceTestCase

---
 .../common/bytes/AbstractBytesReferenceTestCase.java            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java b/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
index 9a9e2a4e615..c521314f92d 100644
--- a/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
+++ b/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
@@ -53,7 +53,7 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
         for (int i = 0; i < probes; i++) {
             int index = randomIntBetween(0, copy.length() - 1);
             assertEquals(pbr.get(index), copy.get(index));
-            index = randomIntBetween(sliceOffset, sliceOffset + sliceLength);
+            index = randomIntBetween(sliceOffset, sliceOffset + sliceLength - 1);
             assertEquals(pbr.get(index), slice.get(index - sliceOffset));
         }
     }

From 14eefb760742fc005a0ead9f1512f5cbb54b4f02 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Tue, 5 Jul 2016 18:04:50 -0700
Subject: [PATCH 1251/1311] Internal: Remove guice from transport client helper
 classes

This change removes injection for constructors of
TransportClientNodesService and TransportProxyClient.
---
 .../elasticsearch/action/ActionModule.java    | 24 ++++++-------
 .../action/TransportActionNodeProxy.java      |  1 -
 .../client/transport/TransportClient.java     | 35 ++++++++++---------
 .../TransportClientNodesService.java          |  2 +-
 .../support/TransportProxyClient.java         | 14 +++-----
 .../common/network/NetworkModule.java         |  5 +--
 .../client/transport/TransportClientIT.java   | 11 +++---
 7 files changed, 41 insertions(+), 51 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/ActionModule.java b/core/src/main/java/org/elasticsearch/action/ActionModule.java
index 5ce7aaa4d95..46494f4ebdc 100644
--- a/core/src/main/java/org/elasticsearch/action/ActionModule.java
+++ b/core/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -19,6 +19,12 @@
 
 package org.elasticsearch.action;
 
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 import org.elasticsearch.action.admin.cluster.allocation.ClusterAllocationExplainAction;
 import org.elasticsearch.action.admin.cluster.allocation.TransportClusterAllocationExplainAction;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
@@ -303,12 +309,6 @@ import org.elasticsearch.rest.action.termvectors.RestMultiTermVectorsAction;
 import org.elasticsearch.rest.action.termvectors.RestTermVectorsAction;
 import org.elasticsearch.rest.action.update.RestUpdateAction;
 
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
 import static java.util.Collections.unmodifiableList;
 import static java.util.Collections.unmodifiableMap;
 
@@ -338,6 +338,10 @@ public class ActionModule extends AbstractModule {
         restController = new RestController(settings);
     }
 
+    public Map<String, ActionHandler<?, ?>> getActions() {
+        return actions;
+    }
+
     static Map<String, ActionHandler<?, ?>> setupActions(List<ActionPlugin> actionPlugins) {
         // Subclass NamedRegistry for easy registration
         class ActionRegistry extends NamedRegistry<ActionHandler<?, ?>> {
@@ -621,14 +625,6 @@ public class ActionModule extends AbstractModule {
         bind(ActionFilters.class).asEagerSingleton();
         bind(DestructiveOperations.class).toInstance(destructiveOperations);
 
-        // register Name -> GenericAction Map that can be injected to instances.
-        @SuppressWarnings("rawtypes")
-        MapBinder<String, GenericAction> actionsBinder
-                = MapBinder.newMapBinder(binder(), String.class, GenericAction.class);
-
-        for (Map.Entry<String, ActionHandler<?, ?>> entry : actions.entrySet()) {
-            actionsBinder.addBinding(entry.getKey()).toInstance(entry.getValue().getAction());
-        }
         if (false == transportClient) {
             // Supporting classes only used when not a transport client
             bind(AutoCreateIndex.class).toInstance(autoCreateIndex);
diff --git a/core/src/main/java/org/elasticsearch/action/TransportActionNodeProxy.java b/core/src/main/java/org/elasticsearch/action/TransportActionNodeProxy.java
index b12af818c2e..7d57e5bd60a 100644
--- a/core/src/main/java/org/elasticsearch/action/TransportActionNodeProxy.java
+++ b/core/src/main/java/org/elasticsearch/action/TransportActionNodeProxy.java
@@ -35,7 +35,6 @@ public class TransportActionNodeProxy<Request extends ActionRequest, Response ex
     private final GenericAction<Request, Response> action;
     private final TransportRequestOptions transportOptions;
 
-    @Inject
     public TransportActionNodeProxy(Settings settings, GenericAction<Request, Response> action, TransportService transportService) {
         super(settings);
         this.action = action;
diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
index eeb9373c435..5421691d515 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClient.java
@@ -19,6 +19,12 @@
 
 package org.elasticsearch.client.transport;
 
+import java.io.Closeable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
 import org.apache.lucene.util.IOUtils;
 import org.elasticsearch.action.Action;
 import org.elasticsearch.action.ActionListener;
@@ -53,11 +59,6 @@ import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TcpTransport;
 import org.elasticsearch.transport.TransportService;
 
-import java.io.Closeable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
 /**
  * The transport client allows to create a client that is not part of the cluster, but simply connects to one
  * or more nodes directly by adding their respective addresses using {@link #addTransportAddress(org.elasticsearch.common.transport.TransportAddress)}.
@@ -143,8 +144,9 @@ public class TransportClient extends AbstractClient {
                 modules.add(new NetworkModule(networkService, settings, true, namedWriteableRegistry));
                 modules.add(b -> b.bind(ThreadPool.class).toInstance(threadPool));
                 modules.add(new SearchModule(settings, namedWriteableRegistry, true));
-                modules.add(new ActionModule(false, true, settings, null, settingsModule.getClusterSettings(),
-                        pluginsService.filterPlugins(ActionPlugin.class)));
+                ActionModule actionModule = new ActionModule(false, true, settings, null, settingsModule.getClusterSettings(),
+                    pluginsService.filterPlugins(ActionPlugin.class));
+                modules.add(actionModule);
 
                 pluginsService.processModules(modules);
                 CircuitBreakerService circuitBreakerService = Node.createCircuitBreakerService(settingsModule.getSettings(),
@@ -161,9 +163,14 @@ public class TransportClient extends AbstractClient {
 
                 Injector injector = modules.createInjector();
                 final TransportService transportService = injector.getInstance(TransportService.class);
+                final TransportClientNodesService nodesService =
+                    new TransportClientNodesService(settings, transportService, threadPool);
+                final TransportProxyClient proxy = new TransportProxyClient(settings, transportService, nodesService,
+                    actionModule.getActions().values().stream().map(x -> x.getAction()).collect(Collectors.toList()));
+
                 transportService.start();
                 transportService.acceptIncomingRequests();
-                TransportClient transportClient = new TransportClient(injector);
+                TransportClient transportClient = new TransportClient(injector, nodesService, proxy);
                 resourcesToClose.clear();
                 return transportClient;
             } finally {
@@ -179,15 +186,11 @@ public class TransportClient extends AbstractClient {
     private final TransportClientNodesService nodesService;
     private final TransportProxyClient proxy;
 
-    private TransportClient(Injector injector) {
+    private TransportClient(Injector injector, TransportClientNodesService nodesService, TransportProxyClient proxy) {
         super(injector.getInstance(Settings.class), injector.getInstance(ThreadPool.class));
         this.injector = injector;
-        nodesService = injector.getInstance(TransportClientNodesService.class);
-        proxy = injector.getInstance(TransportProxyClient.class);
-    }
-
-    TransportClientNodesService nodeService() {
-        return nodesService;
+        this.nodesService = nodesService;
+        this.proxy = proxy;
     }
 
     /**
@@ -263,7 +266,7 @@ public class TransportClient extends AbstractClient {
     @Override
     public void close() {
         List<Closeable> closeables = new ArrayList<>();
-        closeables.add(injector.getInstance(TransportClientNodesService.class));
+        closeables.add(nodesService);
         closeables.add(injector.getInstance(TransportService.class));
 
         for (Class<? extends LifecycleComponent> plugin : injector.getInstance(PluginsService.class).nodeServices()) {
diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
index 17907e37b6c..2ca4767a713 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java
@@ -113,7 +113,6 @@ public class TransportClientNodesService extends AbstractComponent implements Cl
     public static final Setting<Boolean> CLIENT_TRANSPORT_SNIFF =
         Setting.boolSetting("client.transport.sniff", false, Property.NodeScope);
 
-    @Inject
     public TransportClientNodesService(Settings settings,TransportService transportService,
                                        ThreadPool threadPool) {
         super(settings);
@@ -282,6 +281,7 @@ public class TransportClientNodesService extends AbstractComponent implements Cl
 
     }
 
+    @Override
     public void close() {
         synchronized (mutex) {
             if (closed) {
diff --git a/core/src/main/java/org/elasticsearch/client/transport/support/TransportProxyClient.java b/core/src/main/java/org/elasticsearch/client/transport/support/TransportProxyClient.java
index 900876415e3..34833e9400a 100644
--- a/core/src/main/java/org/elasticsearch/client/transport/support/TransportProxyClient.java
+++ b/core/src/main/java/org/elasticsearch/client/transport/support/TransportProxyClient.java
@@ -33,6 +33,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.transport.TransportService;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import static java.util.Collections.unmodifiableMap;
@@ -45,11 +46,11 @@ public class TransportProxyClient {
     private final TransportClientNodesService nodesService;
     private final Map<Action, TransportActionNodeProxy> proxies;
 
-    @Inject
-    public TransportProxyClient(Settings settings, TransportService transportService, TransportClientNodesService nodesService, Map<String, GenericAction> actions) {
+    public TransportProxyClient(Settings settings, TransportService transportService,
+                                TransportClientNodesService nodesService, List<GenericAction> actions) {
         this.nodesService = nodesService;
         Map<Action, TransportActionNodeProxy> proxies = new HashMap<>();
-        for (GenericAction action : actions.values()) {
+        for (GenericAction action : actions) {
             if (action instanceof Action) {
                 proxies.put((Action) action, new TransportActionNodeProxy(settings, action, transportService));
             }
@@ -59,11 +60,6 @@ public class TransportProxyClient {
 
     public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(final Action<Request, Response, RequestBuilder> action, final Request request, ActionListener<Response> listener) {
         final TransportActionNodeProxy<Request, Response> proxy = proxies.get(action);
-        nodesService.execute(new TransportClientNodesService.NodeListenerCallback<Response>() {
-            @Override
-            public void doWithNode(DiscoveryNode node, ActionListener<Response> listener) {
-                proxy.execute(node, request, listener);
-            }
-        }, listener);
+        nodesService.execute((n, l) -> proxy.execute(n, request, l), listener);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
index b65f568af8a..fc65ed6c5b2 100644
--- a/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
+++ b/core/src/main/java/org/elasticsearch/common/network/NetworkModule.java
@@ -160,10 +160,7 @@ public class NetworkModule extends AbstractModule {
         String defaultTransport = DiscoveryNode.isLocalNode(settings) ? LOCAL_TRANSPORT : NETTY_TRANSPORT;
         transportTypes.bindType(binder(), settings, TRANSPORT_TYPE_KEY, defaultTransport);
 
-        if (transportClient) {
-            bind(TransportProxyClient.class).asEagerSingleton();
-            bind(TransportClientNodesService.class).asEagerSingleton();
-        } else {
+        if (transportClient == false) {
             if (HTTP_ENABLED.get(settings)) {
                 bind(HttpServer.class).asEagerSingleton();
                 httpTransportTypes.bindType(binder(), settings, HTTP_TYPE_SETTING.getKey(), NETTY_TRANSPORT);
diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientIT.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientIT.java
index 9cdeef2a7ff..5c3cb6d6339 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientIT.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientIT.java
@@ -51,7 +51,6 @@ public class TransportClientIT extends ESIntegTestCase {
 
     public void testNodeVersionIsUpdated() throws IOException {
         TransportClient client = (TransportClient)  internalCluster().client();
-        TransportClientNodesService nodeService = client.nodeService();
         Node node = new Node(Settings.builder()
                 .put(internalCluster().getDefaultSettings())
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
@@ -65,19 +64,19 @@ public class TransportClientIT extends ESIntegTestCase {
             TransportAddress transportAddress = node.injector().getInstance(TransportService.class).boundAddress().publishAddress();
             client.addTransportAddress(transportAddress);
             // since we force transport clients there has to be one node started that we connect to.
-            assertThat(nodeService.connectedNodes().size(), greaterThanOrEqualTo(1));
+            assertThat(client.connectedNodes().size(), greaterThanOrEqualTo(1));
             // connected nodes have updated version
-            for (DiscoveryNode discoveryNode : nodeService.connectedNodes()) {
+            for (DiscoveryNode discoveryNode : client.connectedNodes()) {
                 assertThat(discoveryNode.getVersion(), equalTo(Version.CURRENT));
             }
 
-            for (DiscoveryNode discoveryNode : nodeService.listedNodes()) {
+            for (DiscoveryNode discoveryNode : client.listedNodes()) {
                 assertThat(discoveryNode.getId(), startsWith("#transport#-"));
                 assertThat(discoveryNode.getVersion(), equalTo(Version.CURRENT.minimumCompatibilityVersion()));
             }
 
-            assertThat(nodeService.filteredNodes().size(), equalTo(1));
-            for (DiscoveryNode discoveryNode : nodeService.filteredNodes()) {
+            assertThat(client.filteredNodes().size(), equalTo(1));
+            for (DiscoveryNode discoveryNode : client.filteredNodes()) {
                 assertThat(discoveryNode.getVersion(), equalTo(Version.CURRENT.minimumCompatibilityVersion()));
             }
         } finally {

From f49ce8e6fe2ea427e72fc0d1468844c0964acdbd Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Tue, 5 Jul 2016 21:03:39 -0700
Subject: [PATCH 1252/1311] Add basic tests for ingest plugins setup

---
 .../ingest/IngestServiceTests.java            | 53 +++++++++++++++++++
 1 file changed, 53 insertions(+)
 create mode 100644 core/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java

diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/core/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java
new file mode 100644
index 00000000000..e44dc42cbe8
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.ingest;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.plugins.IngestPlugin;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.test.ESTestCase;
+
+public class IngestServiceTests extends ESTestCase {
+    private final IngestPlugin DUMMY_PLUGIN = new IngestPlugin() {
+        @Override
+        public Map<String, Processor.Factory> getProcessors(Environment env, ScriptService scriptService, TemplateService templateService) {
+            return Collections.singletonMap("foo", (factories, tag, config) -> null);
+        }
+    };
+
+    public void testIngestPlugin() {
+        IngestService ingestService = new IngestService(Settings.EMPTY, null, null, null, Collections.singletonList(DUMMY_PLUGIN));
+        Map<String, Processor.Factory> factories = ingestService.getPipelineStore().getProcessorFactories();
+        assertTrue(factories.containsKey("foo"));
+        assertEquals(1, factories.size());
+    }
+
+    public void testIngestPluginDuplicate() {
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
+            new IngestService(Settings.EMPTY, null, null, null, Arrays.asList(DUMMY_PLUGIN, DUMMY_PLUGIN))
+        );
+        assertTrue(e.getMessage(), e.getMessage().contains("already registered"));
+    }
+}

From f815799bd48fc9982ecdf25c71a85690f6bc1053 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Tue, 5 Jul 2016 22:16:22 -0700
Subject: [PATCH 1253/1311] Fix reindex action to depend on HttpServer instead
 of NodeService for http info

---
 .../index/reindex/TransportReindexAction.java          | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
index 04ccfa1ba49..d58bd8ea37a 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
@@ -34,6 +34,7 @@ import org.elasticsearch.client.RestClient;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.lucene.uid.Versions;
@@ -42,6 +43,7 @@ import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.TransportAddress;
 import org.elasticsearch.http.HttpInfo;
+import org.elasticsearch.http.HttpServer;
 import org.elasticsearch.index.mapper.internal.TTLFieldMapper;
 import org.elasticsearch.index.mapper.internal.VersionFieldMapper;
 import org.elasticsearch.index.reindex.remote.RemoteInfo;
@@ -74,12 +76,12 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
     private final AutoCreateIndex autoCreateIndex;
     private final Client client;
     private final Set<String> remoteWhitelist;
-    private final NodeService nodeService;
+    private final HttpServer httpServer;
 
     @Inject
     public TransportReindexAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters,
             IndexNameExpressionResolver indexNameExpressionResolver, ClusterService clusterService, ScriptService scriptService,
-            AutoCreateIndex autoCreateIndex, Client client, TransportService transportService, NodeService nodeService) {
+            AutoCreateIndex autoCreateIndex, Client client, TransportService transportService, @Nullable HttpServer httpServer) {
         super(settings, ReindexAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver,
                 ReindexRequest::new);
         this.clusterService = clusterService;
@@ -87,7 +89,7 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
         this.autoCreateIndex = autoCreateIndex;
         this.client = client;
         remoteWhitelist = new HashSet<>(REMOTE_CLUSTER_WHITELIST.get(settings));
-        this.nodeService = nodeService;
+        this.httpServer = httpServer;
     }
 
     @Override
@@ -107,7 +109,7 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
 
     private void checkRemoteWhitelist(RemoteInfo remoteInfo) {
         TransportAddress publishAddress = null;
-        HttpInfo httpInfo = nodeService.info().getHttp();
+        HttpInfo httpInfo = httpServer.info();
         if (httpInfo != null && httpInfo.getAddress() != null) {
             publishAddress = httpInfo.getAddress().publishAddress();
         }

From e7818f75e1c9761782a9608053256cff7713d3e0 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Tue, 5 Jul 2016 22:33:08 -0700
Subject: [PATCH 1254/1311] Fix checkstyle for TestProcessor

---
 .../src/main/java/org/elasticsearch/ingest/TestProcessor.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java b/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java
index 7e7343539b7..4e4c5a24c0c 100644
--- a/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java
+++ b/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java
@@ -66,7 +66,8 @@ public class TestProcessor implements Processor {
 
     public static final class Factory implements Processor.Factory {
         @Override
-        public TestProcessor create(Map<String, Processor.Factory> registry, String processorTag, Map<String, Object> config) throws Exception {
+        public TestProcessor create(Map<String, Processor.Factory> registry, String processorTag,
+                                    Map<String, Object> config) throws Exception {
             return new TestProcessor(processorTag, "test-processor", ingestDocument -> {});
         }
     }

From 4ea5f51a9cfcb0d3059b027a83a32227bf5e526c Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Tue, 5 Jul 2016 23:59:29 -0700
Subject: [PATCH 1255/1311] Fix reindex NPE when http is disabled

---
 .../org/elasticsearch/index/reindex/TransportReindexAction.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
index d58bd8ea37a..3e6f806d293 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportReindexAction.java
@@ -109,7 +109,7 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
 
     private void checkRemoteWhitelist(RemoteInfo remoteInfo) {
         TransportAddress publishAddress = null;
-        HttpInfo httpInfo = httpServer.info();
+        HttpInfo httpInfo = httpServer == null ? null : httpServer.info();
         if (httpInfo != null && httpInfo.getAddress() != null) {
             publishAddress = httpInfo.getAddress().publishAddress();
         }

From 3e6769f237f925100744acd30529d855898d6e25 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Wed, 6 Jul 2016 14:31:58 +0200
Subject: [PATCH 1256/1311] Add otto-de/flummi client to plugins

Closes #19266
---
 docs/community-clients/index.asciidoc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/community-clients/index.asciidoc b/docs/community-clients/index.asciidoc
index 6ee3398fb06..707d976536b 100644
--- a/docs/community-clients/index.asciidoc
+++ b/docs/community-clients/index.asciidoc
@@ -84,6 +84,8 @@ See the {client}/groovy-api/current/index.html[official Elasticsearch Groovy cli
 
 Also see the {client}/java-api/current/index.html[official Elasticsearch Java client].
 
+* [Flummi](https://github.com/otto-de/flummi):
+  Java Rest client with comprehensive query DSL API
 * https://github.com/searchbox-io/Jest[Jest]:
   Java Rest client.
 

From 37725f640c9563f606995db9f862ff599df781ad Mon Sep 17 00:00:00 2001
From: Jim Ferenczi <jim.ferenczi@elastic.co>
Date: Mon, 4 Jul 2016 09:25:46 +0200
Subject: [PATCH 1257/1311] Add missing field type in the FieldStats response.

This change adds the type of the field in the fieldstats response.
It can be one of the following:
  * "integer" for byte, short, integer and long
  * "float" for float, half-float and double
  * "date" for date
  * "ip" for ip
  * "text" for string, keyword and text.

Closes #17750
---
 .../action/fieldstats/FieldStats.java         | 86 +++++++++----------
 .../fieldstats/TransportFieldStatsAction.java |  4 +-
 .../fieldstats/FieldStatsIntegrationIT.java   | 19 +++-
 .../fieldstats/FieldStatsTests.java           | 26 +++++-
 .../test/field_stats/10_basics.yaml           | 12 ++-
 5 files changed, 98 insertions(+), 49 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java
index 2c6f2539b97..4a4f106b085 100644
--- a/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java
+++ b/core/src/main/java/org/elasticsearch/action/fieldstats/FieldStats.java
@@ -68,6 +68,23 @@ public abstract class FieldStats<T> implements Writeable, ToXContent {
         return this.type;
     }
 
+    public String getDisplayType() {
+        switch (type) {
+            case 0:
+                return "integer";
+            case 1:
+                return "float";
+            case 2:
+                return "date";
+            case 3:
+                return "string";
+            case 4:
+                return "ip";
+            default:
+                throw new IllegalArgumentException("Unknown type.");
+        }
+    }
+
     /**
      * @return the total number of documents.
      *
@@ -220,23 +237,24 @@ public abstract class FieldStats<T> implements Writeable, ToXContent {
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
-        builder.field(Fields.MAX_DOC, maxDoc);
-        builder.field(Fields.DOC_COUNT, docCount);
-        builder.field(Fields.DENSITY, getDensity());
-        builder.field(Fields.SUM_DOC_FREQ, sumDocFreq);
-        builder.field(Fields.SUM_TOTAL_TERM_FREQ, sumTotalTermFreq);
-        builder.field(Fields.SEARCHABLE, isSearchable);
-        builder.field(Fields.AGGREGATABLE, isAggregatable);
+        builder.field(TYPE_FIELD, getDisplayType());
+        builder.field(MAX_DOC_FIELD, maxDoc);
+        builder.field(DOC_COUNT_FIELD, docCount);
+        builder.field(DENSITY_FIELD, getDensity());
+        builder.field(SUM_DOC_FREQ_FIELD, sumDocFreq);
+        builder.field(SUM_TOTAL_TERM_FREQ_FIELD, sumTotalTermFreq);
+        builder.field(SEARCHABLE_FIELD, isSearchable);
+        builder.field(AGGREGATABLE_FIELD, isAggregatable);
         toInnerXContent(builder);
         builder.endObject();
         return builder;
     }
 
     protected void toInnerXContent(XContentBuilder builder) throws IOException {
-        builder.field(Fields.MIN_VALUE, getMinValue());
-        builder.field(Fields.MIN_VALUE_AS_STRING, getMinValueAsString());
-        builder.field(Fields.MAX_VALUE, getMaxValue());
-        builder.field(Fields.MAX_VALUE_AS_STRING, getMaxValueAsString());
+        builder.field(MIN_VALUE_FIELD, getMinValue());
+        builder.field(MIN_VALUE_AS_STRING_FIELD, getMinValueAsString());
+        builder.field(MAX_VALUE_FIELD, getMaxValue());
+        builder.field(MAX_VALUE_AS_STRING_FIELD, getMaxValueAsString());
     }
 
     @Override
@@ -484,8 +502,8 @@ public abstract class FieldStats<T> implements Writeable, ToXContent {
 
         @Override
         protected void toInnerXContent(XContentBuilder builder) throws IOException {
-            builder.field(Fields.MIN_VALUE, getMinValueAsString());
-            builder.field(Fields.MAX_VALUE, getMaxValueAsString());
+            builder.field(MIN_VALUE_FIELD, getMinValueAsString());
+            builder.field(MAX_VALUE_FIELD, getMaxValueAsString());
         }
     }
 
@@ -598,34 +616,16 @@ public abstract class FieldStats<T> implements Writeable, ToXContent {
         }
     }
 
-    public static String typeName(byte type) {
-        switch (type) {
-            case 0:
-                return "whole-number";
-            case 1:
-                return "floating-point";
-            case 2:
-                return "date";
-            case 3:
-                return "text";
-            case 4:
-                return "ip";
-            default:
-                throw new IllegalArgumentException("Unknown type.");
-        }
-    }
-
-    private static final class Fields {
-        static final String MAX_DOC = new String("max_doc");
-        static final String DOC_COUNT = new String("doc_count");
-        static final String DENSITY = new String("density");
-        static final String SUM_DOC_FREQ = new String("sum_doc_freq");
-        static final String SUM_TOTAL_TERM_FREQ = new String("sum_total_term_freq");
-        static final String SEARCHABLE = new String("searchable");
-        static final String AGGREGATABLE = new String("aggregatable");
-        static final String MIN_VALUE = new String("min_value");
-        static final String MIN_VALUE_AS_STRING = new String("min_value_as_string");
-        static final String MAX_VALUE = new String("max_value");
-        static final String MAX_VALUE_AS_STRING = new String("max_value_as_string");
-    }
+    static final String TYPE_FIELD = new String("type");
+    static final String MAX_DOC_FIELD = new String("max_doc");
+    static final String DOC_COUNT_FIELD = new String("doc_count");
+    static final String DENSITY_FIELD = new String("density");
+    static final String SUM_DOC_FREQ_FIELD = new String("sum_doc_freq");
+    static final String SUM_TOTAL_TERM_FREQ_FIELD = new String("sum_total_term_freq");
+    static final String SEARCHABLE_FIELD = new String("searchable");
+    static final String AGGREGATABLE_FIELD = new String("aggregatable");
+    static final String MIN_VALUE_FIELD = new String("min_value");
+    static final String MIN_VALUE_AS_STRING_FIELD = new String("min_value_as_string");
+    static final String MAX_VALUE_FIELD = new String("max_value");
+    static final String MAX_VALUE_AS_STRING_FIELD = new String("max_value_as_string");
 }
diff --git a/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsAction.java b/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsAction.java
index f70e5dda114..e65f6951432 100644
--- a/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsAction.java
+++ b/core/src/main/java/org/elasticsearch/action/fieldstats/TransportFieldStatsAction.java
@@ -116,9 +116,9 @@ public class TransportFieldStatsAction extends
                                 Arrays.sort(fields, (o1, o2) -> Byte.compare(o1.getType(), o2.getType()));
                                 conflicts.put(entry.getKey(),
                                     "Field [" + entry.getKey() + "] of type [" +
-                                        FieldStats.typeName(fields[0].getType()) +
+                                        fields[0].getDisplayType() +
                                         "] conflicts with existing field of type [" +
-                                        FieldStats.typeName(fields[1].getType()) +
+                                        fields[1].getDisplayType() +
                                         "] in other index.");
                             }
                         } else {
diff --git a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java
index e87cc5f6571..6240a13a0bd 100644
--- a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java
+++ b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java
@@ -169,18 +169,25 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
 
         assertThat(response.getAllFieldStats().get("byte").getMinValue(), equalTo(minByte));
         assertThat(response.getAllFieldStats().get("byte").getMaxValue(), equalTo(maxByte));
+        assertThat(response.getAllFieldStats().get("byte").getDisplayType(), equalTo("integer"));
         assertThat(response.getAllFieldStats().get("short").getMinValue(), equalTo(minShort));
         assertThat(response.getAllFieldStats().get("short").getMaxValue(), equalTo(maxShort));
+        assertThat(response.getAllFieldStats().get("short").getDisplayType(), equalTo("integer"));
         assertThat(response.getAllFieldStats().get("integer").getMinValue(), equalTo(minInt));
         assertThat(response.getAllFieldStats().get("integer").getMaxValue(), equalTo(maxInt));
+        assertThat(response.getAllFieldStats().get("integer").getDisplayType(), equalTo("integer"));
         assertThat(response.getAllFieldStats().get("long").getMinValue(), equalTo(minLong));
         assertThat(response.getAllFieldStats().get("long").getMaxValue(), equalTo(maxLong));
+        assertThat(response.getAllFieldStats().get("long").getDisplayType(), equalTo("integer"));
         assertThat(response.getAllFieldStats().get("half_float").getMinValue(), equalTo(minHalfFloat));
         assertThat(response.getAllFieldStats().get("half_float").getMaxValue(), equalTo(maxHalfFloat));
+        assertThat(response.getAllFieldStats().get("half_float").getDisplayType(), equalTo("float"));
         assertThat(response.getAllFieldStats().get("float").getMinValue(), equalTo(minFloat));
         assertThat(response.getAllFieldStats().get("float").getMaxValue(), equalTo(maxFloat));
+        assertThat(response.getAllFieldStats().get("float").getDisplayType(), equalTo("float"));
         assertThat(response.getAllFieldStats().get("double").getMinValue(), equalTo(minDouble));
         assertThat(response.getAllFieldStats().get("double").getMaxValue(), equalTo(maxDouble));
+        assertThat(response.getAllFieldStats().get("double").getDisplayType(), equalTo("float"));
     }
 
     public void testFieldStatsIndexLevel() throws Exception {
@@ -207,6 +214,8 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
         assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1));
         assertThat(response.getIndicesMergedFieldStats().get("_all").get("value").getMinValue(), equalTo(-10L));
         assertThat(response.getIndicesMergedFieldStats().get("_all").get("value").getMaxValue(), equalTo(300L));
+        assertThat(response.getIndicesMergedFieldStats().get("_all").get("value").getDisplayType(),
+            equalTo("integer"));
 
         // Level: cluster
         response = client().prepareFieldStats().setFields("value").setLevel("cluster").get();
@@ -216,6 +225,8 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
         assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1));
         assertThat(response.getIndicesMergedFieldStats().get("_all").get("value").getMinValue(), equalTo(-10L));
         assertThat(response.getIndicesMergedFieldStats().get("_all").get("value").getMaxValue(), equalTo(300L));
+        assertThat(response.getIndicesMergedFieldStats().get("_all").get("value").getDisplayType(),
+            equalTo("integer"));
 
         // Level: indices
         response = client().prepareFieldStats().setFields("value").setLevel("indices").get();
@@ -228,6 +239,8 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
         assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMaxValue(), equalTo(200L));
         assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMinValue(), equalTo(201L));
         assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getMaxValue(), equalTo(300L));
+        assertThat(response.getIndicesMergedFieldStats().get("test3").get("value").getDisplayType(),
+            equalTo("integer"));
 
         // Illegal level option:
         try {
@@ -259,7 +272,7 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
         assertThat(response.getIndicesMergedFieldStats().get("_all").size(), equalTo(0));
         assertThat(response.getConflicts().size(), equalTo(1));
         assertThat(response.getConflicts().get("value"),
-            equalTo("Field [value] of type [whole-number] conflicts with existing field of type [text] " +
+            equalTo("Field [value] of type [integer] conflicts with existing field of type [string] " +
                 "in other index."));
 
         response = client().prepareFieldStats().setFields("value").setLevel("indices").get();
@@ -296,7 +309,7 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
         assertThat(response.getIndicesMergedFieldStats().get("_all").get("value2").getMaxValue(), equalTo(1L));
         assertThat(response.getConflicts().size(), equalTo(1));
         assertThat(response.getConflicts().get("value"),
-            equalTo("Field [value] of type [whole-number] conflicts with existing field of type [text] " +
+            equalTo("Field [value] of type [integer] conflicts with existing field of type [string] " +
                 "in other index."));
 
         response = client().prepareFieldStats().setFields("value", "value2").setLevel("indices").get();
@@ -310,6 +323,8 @@ public class FieldStatsIntegrationIT extends ESIntegTestCase {
             equalTo(new BytesRef("a")));
         assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMaxValue(),
             equalTo(new BytesRef("b")));
+        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getDisplayType(),
+            equalTo("string"));
     }
 
     public void testFieldStatsFiltering() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java
index 20db1bf20c4..bfee11f7f0e 100644
--- a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java
+++ b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java
@@ -89,6 +89,8 @@ public class FieldStatsTests extends ESSingleNodeTestCase {
             equalTo(String.format(Locale.ENGLISH, "%03d", 0)));
         assertThat(result.getAllFieldStats().get("field").getMaxValueAsString(),
             equalTo(String.format(Locale.ENGLISH, "%03d", 10)));
+        assertThat(result.getAllFieldStats().get("field").getDisplayType(),
+            equalTo("string"));
     }
 
     public void testDouble() {
@@ -106,6 +108,7 @@ public class FieldStatsTests extends ESSingleNodeTestCase {
         assertThat(result.getAllFieldStats().get(fieldName).getMinValue(), equalTo(-1d));
         assertThat(result.getAllFieldStats().get(fieldName).getMaxValue(), equalTo(9d));
         assertThat(result.getAllFieldStats().get(fieldName).getMinValueAsString(), equalTo(Double.toString(-1)));
+        assertThat(result.getAllFieldStats().get(fieldName).getDisplayType(), equalTo("float"));
     }
 
     public void testHalfFloat() {
@@ -124,6 +127,7 @@ public class FieldStatsTests extends ESSingleNodeTestCase {
         assertThat(result.getAllFieldStats().get(fieldName).getMaxValue(), equalTo(9d));
         assertThat(result.getAllFieldStats().get(fieldName).getMinValueAsString(), equalTo(Float.toString(-1)));
         assertThat(result.getAllFieldStats().get(fieldName).getMaxValueAsString(), equalTo(Float.toString(9)));
+        assertThat(result.getAllFieldStats().get(fieldName).getDisplayType(), equalTo("float"));
     }
 
     public void testFloat() {
@@ -169,6 +173,11 @@ public class FieldStatsTests extends ESSingleNodeTestCase {
             equalTo(java.lang.Long.toString(max)));
         assertThat(result.getAllFieldStats().get(fieldName).isSearchable(), equalTo(true));
         assertThat(result.getAllFieldStats().get(fieldName).isAggregatable(), equalTo(true));
+        if (fieldType.equals("float") || fieldType.equals("double") || fieldType.equals("half-float")) {
+            assertThat(result.getAllFieldStats().get(fieldName).getDisplayType(), equalTo("float"));
+        } else {
+            assertThat(result.getAllFieldStats().get(fieldName).getDisplayType(), equalTo("integer"));
+        }
 
         client().admin().indices().prepareDelete("test").get();
         client().admin().indices().prepareDelete("test1").get();
@@ -191,6 +200,7 @@ public class FieldStatsTests extends ESSingleNodeTestCase {
         assertThat(stat.getSumTotalTermFreq(), equalTo(4L));
         assertThat(stat.isSearchable(), equalTo(true));
         assertThat(stat.isAggregatable(), equalTo(false));
+        assertThat(stat.getDisplayType(), equalTo("integer"));
     }
 
     public void testMerge_notAvailable() {
@@ -209,6 +219,7 @@ public class FieldStatsTests extends ESSingleNodeTestCase {
         assertThat(stat.getSumTotalTermFreq(), equalTo(-1L));
         assertThat(stat.isSearchable(), equalTo(true));
         assertThat(stat.isAggregatable(), equalTo(true));
+        assertThat(stat.getDisplayType(), equalTo("integer"));
 
         stats.add(new FieldStats.Long(1, -1L, -1L, -1L, true, true, 1L, 1L));
         stat = stats.remove(0);
@@ -221,6 +232,7 @@ public class FieldStatsTests extends ESSingleNodeTestCase {
         assertThat(stat.getSumTotalTermFreq(), equalTo(-1L));
         assertThat(stat.isSearchable(), equalTo(true));
         assertThat(stat.isAggregatable(), equalTo(true));
+        assertThat(stat.getDisplayType(), equalTo("integer"));
     }
 
     public void testNumberFiltering() {
@@ -350,6 +362,8 @@ public class FieldStatsTests extends ESSingleNodeTestCase {
             equalTo(dateTime1Str));
         assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValueAsString(),
             equalTo(dateTime2Str));
+        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getDisplayType(),
+            equalTo("date"));
 
         response = client().prepareFieldStats()
                 .setFields("value")
@@ -370,6 +384,8 @@ public class FieldStatsTests extends ESSingleNodeTestCase {
             equalTo(dateTime1.getMillis()));
         assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getMinValueAsString(),
             equalTo(dateTime1Str));
+        assertThat(response.getIndicesMergedFieldStats().get("test1").get("value").getDisplayType(),
+            equalTo("date"));
 
         response = client().prepareFieldStats()
                 .setFields("value")
@@ -402,6 +418,8 @@ public class FieldStatsTests extends ESSingleNodeTestCase {
             equalTo(dateTime2.getMillis()));
         assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValueAsString(),
             equalTo(dateTime2Str));
+        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getDisplayType(),
+            equalTo("date"));
 
         response = client().prepareFieldStats()
                 .setFields("value")
@@ -417,6 +435,8 @@ public class FieldStatsTests extends ESSingleNodeTestCase {
             equalTo(dateTime1Str));
         assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValueAsString(),
             equalTo(dateTime2Str));
+        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getDisplayType(),
+            equalTo("date"));
 
         response = client().prepareFieldStats()
                 .setFields("value")
@@ -432,6 +452,8 @@ public class FieldStatsTests extends ESSingleNodeTestCase {
             equalTo(dateTime1Str));
         assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValueAsString(),
             equalTo(dateTime2Str));
+        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getDisplayType(),
+            equalTo("date"));
     }
 
     public void testDateFiltering_optionalFormat() {
@@ -453,6 +475,8 @@ public class FieldStatsTests extends ESSingleNodeTestCase {
         assertThat(response.getIndicesMergedFieldStats().size(), equalTo(1));
         assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getMinValueAsString(),
             equalTo("2014-01-02T00:00:00.000Z"));
+        assertThat(response.getIndicesMergedFieldStats().get("test2").get("value").getDisplayType(),
+            equalTo("date"));
 
         try {
             client().prepareFieldStats()
@@ -487,6 +511,6 @@ public class FieldStatsTests extends ESSingleNodeTestCase {
             .get();
         assertThat(response.getAllFieldStats().size(), equalTo(1));
         assertThat(response.getAllFieldStats().get("_type").isSearchable(), equalTo(true));
-        // assertThat(response.getAllFieldStats().get("_type").isAggregatable(), equalTo(true));
+        assertThat(response.getAllFieldStats().get("_type").isAggregatable(), equalTo(true));
     }
 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml
index eb1d1c6f758..35cc19224ec 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/field_stats/10_basics.yaml
@@ -54,6 +54,7 @@ setup:
   - match: { indices._all.fields.foo.doc_count: 2 }
   - match: { indices._all.fields.foo.min_value: "babar" }
   - match: { indices._all.fields.foo.max_value: "bar" }
+  - match: { indices._all.fields.foo.type: "string" }
   - is_false: indices._all.fields.foo.min_value_as_string
   - is_false: indices._all.fields.foo.max_value_as_string
   - match: { indices._all.fields.foo.searchable: true }
@@ -66,6 +67,7 @@ setup:
   - match: { indices._all.fields.number.min_value_as_string: "123" }
   - match: { indices._all.fields.number.max_value: 456 }
   - match: { indices._all.fields.number.max_value_as_string: "456" }
+  - match: { indices._all.fields.number.type: "integer" }
   - is_false: conflicts
 
 ---
@@ -83,6 +85,7 @@ setup:
   - is_false: indices.test_1.fields.foo.max_value_as_string
   - match: { indices.test_1.fields.foo.searchable: true }
   - match: { indices.test_1.fields.foo.aggregatable: false }
+  - match: { indices.test_1.fields.foo.type: "string" }
   - match: { indices.test_1.fields.number.max_doc: 1 }
   - match: { indices.test_1.fields.number.doc_count: 1 }
   - match: { indices.test_1.fields.number.searchable: true }
@@ -91,14 +94,17 @@ setup:
   - match: { indices.test_1.fields.number.min_value_as_string: "123" }
   - match: { indices.test_1.fields.number.max_value: 123 }
   - match: { indices.test_1.fields.number.max_value_as_string: "123" }
+  - match: { indices.test_1.fields.number.type: "integer" }
   - match: { indices.test_2.fields.foo.max_doc: 1 }
   - match: { indices.test_2.fields.foo.doc_count: 1 }
   - match: { indices.test_2.fields.foo.min_value: "babar" }
   - match: { indices.test_2.fields.foo.max_value: "babar" }
+  - match: { indices.test_2.fields.foo.type: "string" }
   - is_false: indices.test_2.fields.foo.min_value_as_string
   - is_false: indices.test_2.fields.foo.max_value_as_string
   - match: { indices.test_2.fields.foo.searchable: true }
   - match: { indices.test_2.fields.foo.aggregatable: false }
+  - match: { indices.test_2.fields.foo.type: "string" }
   - match: { indices.test_2.fields.number.max_doc: 1 }
   - match: { indices.test_2.fields.number.doc_count: 1 }
   - match: { indices.test_2.fields.number.searchable: true }
@@ -107,6 +113,7 @@ setup:
   - match: { indices.test_2.fields.number.min_value_as_string: "456" }
   - match: { indices.test_2.fields.number.max_value: 456 }
   - match: { indices.test_2.fields.number.max_value_as_string: "456" }
+  - match: { indices.test_2.fields.number.type: "integer" }
   - is_false: conflicts
 
 ---
@@ -124,6 +131,7 @@ setup:
   - match: { indices.test_1.fields.foo.aggregatable: false }
   - match: { indices.test_1.fields.foo.min_value: "bar" }
   - match: { indices.test_1.fields.foo.max_value: "bar" }
+  - match: { indices.test_1.fields.foo.type: "string" }
   - is_false: indices.test_1.fields.number
   - is_false: conflicts
 
@@ -156,6 +164,7 @@ setup:
   - match: { indices._all.fields.foo.max_value: "bar" }
   - match: { indices._all.fields.foo.searchable: true }
   - match: { indices._all.fields.foo.aggregatable: false }
+  - match: { indices._all.fields.foo.type: "string" }
   - match: { indices._all.fields.number.max_doc: 2 }
   - match: { indices._all.fields.number.doc_count: 2 }
   - match: { indices._all.fields.number.searchable: true }
@@ -164,5 +173,6 @@ setup:
   - match: { indices._all.fields.number.min_value_as_string: "123" }
   - match: { indices._all.fields.number.max_value: 456 }
   - match: { indices._all.fields.number.max_value_as_string: "456" }
-  - match: { conflicts.bar: "Field [bar] of type [whole-number] conflicts with existing field of type [text] in other index." }
+  - match: { indices._all.fields.number.type: "integer" }
+  - match: { conflicts.bar: "Field [bar] of type [integer] conflicts with existing field of type [string] in other index." }
   - is_false: indices._all.fields.bar

From 76057e6b0532959332d516aff5d7d215fab97b23 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Wed, 6 Jul 2016 16:51:22 +0200
Subject: [PATCH 1258/1311] Fix test issue where index is explicitly deleted
 during cluster state update

Calling indicesService.deleteIndex() can trip an assertion if there is an ongoing cluster state applied in
IndicesClusterStateService. This means that the index is possibly deleted after the failMissingShards
check and before we try creating new and updated shards, tripping an assertion that non-existing shards must
have shard state initializing (started in this case).
---
 .../java/org/elasticsearch/indices/IndicesServiceTests.java    | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
index 75c968e1d67..27bb2cccff4 100644
--- a/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java
@@ -285,9 +285,6 @@ public class IndicesServiceTests extends ESSingleNodeTestCase {
         listener.latch.await();
         assertThat(clusterService.state(), not(originalState));
         assertNotNull(clusterService.state().getMetaData().index(alias));
-
-        // cleanup
-        indicesService.deleteIndex(test.index(), "finished with test");
     }
 
     /**

From b927cfe1de6f573f443ebd58f492b759cf939093 Mon Sep 17 00:00:00 2001
From: Chris Earle <chris.earle@elastic.co>
Date: Fri, 15 Apr 2016 20:58:00 -0400
Subject: [PATCH 1259/1311] Add DeprecationRestHandler to automatically log
 deprecated REST calls

This adds a new proxy for RestHandlers and RestControllers so that requests made
to deprecated REST APIs can be automatically logged in the ES logs via the
DeprecationLogger as well as via a "Warning" header (RFC-7234) for all responses.
---
 .../elasticsearch/ElasticsearchException.java |  21 +-
 .../common/io/stream/StreamInput.java         |  16 ++
 .../common/io/stream/StreamOutput.java        |  15 ++
 .../common/logging/DeprecationLogger.java     |  91 +++++++-
 .../common/util/concurrent/ThreadContext.java | 123 +++++++---
 .../http/netty/HttpRequestHandler.java        |   2 +-
 .../http/netty/NettyHttpChannel.java          |  12 +-
 .../java/org/elasticsearch/node/Node.java     |   6 +
 .../rest/DeprecationRestHandler.java          | 108 +++++++++
 .../elasticsearch/rest/RestController.java    |  23 +-
 .../elasticsearch/transport/TcpTransport.java |  11 +-
 .../transport/local/LocalTransport.java       |   7 +-
 .../local/LocalTransportChannel.java          |   7 +-
 .../common/io/stream/BytesStreamsTests.java   |  48 +++-
 .../logging/DeprecationLoggerTests.java       | 155 +++++++++++++
 .../util/concurrent/ThreadContextTests.java   | 105 +++++++--
 .../http/netty/NettyHttpChannelTests.java     |   4 +-
 .../elasticsearch/rest/DeprecationHttpIT.java | 216 ++++++++++++++++++
 .../rest/DeprecationRestHandlerTests.java     | 139 +++++++++++
 .../rest/RestControllerTests.java             |  26 +++
 .../plugins/TestDeprecatedQueryBuilder.java   | 104 +++++++++
 .../TestDeprecationHeaderRestAction.java      | 108 +++++++++
 .../rest/plugins/TestDeprecationPlugin.java   |  55 +++++
 23 files changed, 1308 insertions(+), 94 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/rest/DeprecationRestHandler.java
 create mode 100644 core/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java
 create mode 100644 core/src/test/java/org/elasticsearch/rest/DeprecationHttpIT.java
 create mode 100644 core/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java
 create mode 100644 core/src/test/java/org/elasticsearch/rest/plugins/TestDeprecatedQueryBuilder.java
 create mode 100644 core/src/test/java/org/elasticsearch/rest/plugins/TestDeprecationHeaderRestAction.java
 create mode 100644 core/src/test/java/org/elasticsearch/rest/plugins/TestDeprecationPlugin.java

diff --git a/core/src/main/java/org/elasticsearch/ElasticsearchException.java b/core/src/main/java/org/elasticsearch/ElasticsearchException.java
index 801939ba737..54bbfc851d2 100644
--- a/core/src/main/java/org/elasticsearch/ElasticsearchException.java
+++ b/core/src/main/java/org/elasticsearch/ElasticsearchException.java
@@ -33,7 +33,6 @@ import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.transport.TcpTransport;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -102,16 +101,7 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
     public ElasticsearchException(StreamInput in) throws IOException {
         super(in.readOptionalString(), in.readException());
         readStackTrace(this, in);
-        int numKeys = in.readVInt();
-        for (int i = 0; i < numKeys; i++) {
-            final String key = in.readString();
-            final int numValues = in.readVInt();
-            final ArrayList<String> values = new ArrayList<>(numValues);
-            for (int j = 0; j < numValues; j++) {
-                values.add(in.readString());
-            }
-            headers.put(key, values);
-        }
+        headers.putAll(in.readMapOfLists());
     }
 
     /**
@@ -206,14 +196,7 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
         out.writeOptionalString(this.getMessage());
         out.writeException(this.getCause());
         writeStackTraces(this, out);
-        out.writeVInt(headers.size());
-        for (Map.Entry<String, List<String>> entry : headers.entrySet()) {
-            out.writeString(entry.getKey());
-            out.writeVInt(entry.getValue().size());
-            for (String v : entry.getValue()) {
-                out.writeString(v);
-            }
-        }
+        out.writeMapOfLists(headers);
     }
 
     public static ElasticsearchException readException(StreamInput input, int id) throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
index 80ff214cbc9..2d3aebb320d 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
@@ -53,6 +53,7 @@ import java.nio.file.FileSystemLoopException;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.NotDirectoryException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -414,6 +415,21 @@ public abstract class StreamInput extends InputStream {
         return (Map<String, Object>) readGenericValue();
     }
 
+    /**
+     * Read a map of strings to string lists.
+     */
+    public Map<String, List<String>> readMapOfLists() throws IOException {
+        int size = readVInt();
+        if (size == 0) {
+            return Collections.emptyMap();
+        }
+        Map<String, List<String>> map = new HashMap<>(size);
+        for (int i = 0; i < size; ++i) {
+            map.put(readString(), readList(StreamInput::readString));
+        }
+        return map;
+    }
+
     @SuppressWarnings({"unchecked"})
     @Nullable
     public Object readGenericValue() throws IOException {
diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
index dee3ea24f00..64ed6ed3904 100644
--- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
+++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
@@ -403,6 +403,21 @@ public abstract class StreamOutput extends OutputStream {
         writeGenericValue(map);
     }
 
+    /**
+     * Writes a map of strings to string lists.
+     */
+    public void writeMapOfLists(Map<String, List<String>> map) throws IOException {
+        writeVInt(map.size());
+
+        for (Map.Entry<String, List<String>> entry : map.entrySet()) {
+            writeString(entry.getKey());
+            writeVInt(entry.getValue().size());
+            for (String v : entry.getValue()) {
+                writeString(v);
+            }
+        }
+    }
+
     @FunctionalInterface
     interface Writer {
         void write(StreamOutput o, Object value) throws IOException;
diff --git a/core/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java b/core/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java
index b792a85d34c..5970f917322 100644
--- a/core/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java
+++ b/core/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java
@@ -20,12 +20,70 @@
 package org.elasticsearch.common.logging;
 
 import org.elasticsearch.common.SuppressLoggerChecks;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+
+import java.util.Iterator;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArraySet;
 
 /**
  * A logger that logs deprecation notices.
  */
 public class DeprecationLogger {
 
+    /**
+     * The "Warning" Header comes from RFC-7234. As the RFC describes, it's generally used for caching purposes, but it can be
+     * used for <em>any</em> warning.
+     *
+     * https://tools.ietf.org/html/rfc7234#section-5.5
+     */
+    public static final String DEPRECATION_HEADER = "Warning";
+
+    /**
+     * This is set once by the {@code Node} constructor, but it uses {@link CopyOnWriteArraySet} to ensure that tests can run in parallel.
+     * <p>
+     * Integration tests will create separate nodes within the same classloader, thus leading to a shared, {@code static} state.
+     * In order for all tests to appropriately be handled, this must be able to remember <em>all</em> {@link ThreadContext}s that it is
+     * given in a thread safe manner.
+     * <p>
+     * For actual usage, multiple nodes do not share the same JVM and therefore this will only be set once in practice.
+     */
+    private static final CopyOnWriteArraySet<ThreadContext> THREAD_CONTEXT = new CopyOnWriteArraySet<>();
+
+    /**
+     * Set the {@link ThreadContext} used to add deprecation headers to network responses.
+     * <p>
+     * This is expected to <em>only</em> be invoked by the {@code Node}'s constructor (therefore once outside of tests).
+     *
+     * @param threadContext The thread context owned by the {@code ThreadPool} (and implicitly a {@code Node})
+     * @throws IllegalStateException if this {@code threadContext} has already been set
+     */
+    public static void setThreadContext(ThreadContext threadContext) {
+        assert threadContext != null;
+
+        // add returning false means it _did_ have it already
+        if (THREAD_CONTEXT.add(threadContext) == false) {
+            throw new IllegalStateException("Double-setting ThreadContext not allowed!");
+        }
+    }
+
+    /**
+     * Remove the {@link ThreadContext} used to add deprecation headers to network responses.
+     * <p>
+     * This is expected to <em>only</em> be invoked by the {@code Node}'s {@code close} method (therefore once outside of tests).
+     *
+     * @param threadContext The thread context owned by the {@code ThreadPool} (and implicitly a {@code Node})
+     * @throws IllegalStateException if this {@code threadContext} is unknown (and presumably already unset before)
+     */
+    public static void removeThreadContext(ThreadContext threadContext) {
+        assert threadContext != null;
+
+        // remove returning false means it did not have it already
+        if (THREAD_CONTEXT.remove(threadContext) == false) {
+            throw new IllegalStateException("Removing unknown ThreadContext not allowed!");
+        }
+    }
+
     private final ESLogger logger;
 
     /**
@@ -47,8 +105,37 @@ public class DeprecationLogger {
     /**
      * Logs a deprecated message.
      */
-    @SuppressLoggerChecks(reason = "safely delegates to logger")
     public void deprecated(String msg, Object... params) {
-        logger.debug(msg, params);
+        deprecated(THREAD_CONTEXT, msg, params);
     }
+
+    /**
+     * Logs a deprecated message to the deprecation log, as well as to the local {@link ThreadContext}.
+     *
+     * @param threadContexts The node's {@link ThreadContext} (outside of concurrent tests, this should only ever have one context).
+     * @param msg The deprecation message.
+     * @param params The parameters used to fill in the message, if any exist.
+     */
+    @SuppressLoggerChecks(reason = "safely delegates to logger")
+    void deprecated(Set<ThreadContext> threadContexts, String msg, Object... params) {
+        Iterator<ThreadContext> iterator = threadContexts.iterator();
+
+        if (iterator.hasNext()) {
+            final String formattedMsg = LoggerMessageFormat.format(msg, params);
+
+            while (iterator.hasNext()) {
+                try {
+                    iterator.next().addResponseHeader(DEPRECATION_HEADER, formattedMsg);
+                } catch (IllegalStateException e) {
+                    // ignored; it should be removed shortly
+                }
+            }
+
+            logger.debug(formattedMsg);
+        } else {
+            logger.debug(msg, params);
+        }
+
+    }
+
 }
diff --git a/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java b/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
index 457dcad07cf..b816ea76dec 100644
--- a/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
+++ b/core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java
@@ -28,8 +28,10 @@ import org.elasticsearch.common.settings.Settings;
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -64,8 +66,8 @@ public final class ThreadContext implements Closeable, Writeable {
 
     public static final String PREFIX = "request.headers";
     public static final Setting<Settings> DEFAULT_HEADERS_SETTING = Setting.groupSetting(PREFIX + ".", Property.NodeScope);
+    private static final ThreadContextStruct DEFAULT_CONTEXT = new ThreadContextStruct();
     private final Map<String, String> defaultHeader;
-    private static final ThreadContextStruct DEFAULT_CONTEXT = new ThreadContextStruct(Collections.emptyMap());
     private final ContextThreadLocal threadLocal;
 
     /**
@@ -110,7 +112,7 @@ public final class ThreadContext implements Closeable, Writeable {
     public StoredContext stashAndMergeHeaders(Map<String, String> headers) {
         final ThreadContextStruct context = threadLocal.get();
         Map<String, String> newHeader = new HashMap<>(headers);
-        newHeader.putAll(context.headers);
+        newHeader.putAll(context.requestHeaders);
         threadLocal.set(DEFAULT_CONTEXT.putHeaders(newHeader));
         return () -> {
             threadLocal.set(context);
@@ -143,7 +145,7 @@ public final class ThreadContext implements Closeable, Writeable {
      * Returns the header for the given key or <code>null</code> if not present
      */
     public String getHeader(String key) {
-        String value = threadLocal.get().headers.get(key);
+        String value = threadLocal.get().requestHeaders.get(key);
         if (value == null)  {
             return defaultHeader.get(key);
         }
@@ -151,11 +153,27 @@ public final class ThreadContext implements Closeable, Writeable {
     }
 
     /**
-     * Returns all of the current contexts headers
+     * Returns all of the request contexts headers
      */
     public Map<String, String> getHeaders() {
         HashMap<String, String> map = new HashMap<>(defaultHeader);
-        map.putAll(threadLocal.get().headers);
+        map.putAll(threadLocal.get().requestHeaders);
+        return Collections.unmodifiableMap(map);
+    }
+
+    /**
+     * Get a copy of all <em>response</em> headers.
+     *
+     * @return Never {@code null}.
+     */
+    public Map<String, List<String>> getResponseHeaders() {
+        Map<String, List<String>> responseHeaders = threadLocal.get().responseHeaders;
+        HashMap<String, List<String>> map = new HashMap<>(responseHeaders.size());
+
+        for (Map.Entry<String, List<String>> entry : responseHeaders.entrySet()) {
+            map.put(entry.getKey(), Collections.unmodifiableList(entry.getValue()));
+        }
+
         return Collections.unmodifiableMap(map);
     }
 
@@ -170,7 +188,7 @@ public final class ThreadContext implements Closeable, Writeable {
      * Puts a header into the context
      */
     public void putHeader(String key, String value) {
-        threadLocal.set(threadLocal.get().putPersistent(key, value));
+        threadLocal.set(threadLocal.get().putRequest(key, value));
     }
 
     /**
@@ -190,10 +208,20 @@ public final class ThreadContext implements Closeable, Writeable {
     /**
      * Returns a transient header object or <code>null</code> if there is no header for the given key
      */
+    @SuppressWarnings("unchecked") // (T)object
     public <T> T getTransient(String key) {
         return (T) threadLocal.get().transientHeaders.get(key);
     }
 
+    /**
+     * Add the <em>unique</em> response {@code value} for the specified {@code key}.
+     * <p>
+     * Any duplicate {@code value} is ignored.
+     */
+    public void addResponseHeader(String key, String value) {
+        threadLocal.set(threadLocal.get().putResponse(key, value));
+    }
+
     /**
      * Saves the current thread context and wraps command in a Runnable that restores that context before running command. If
      * <code>command</code> has already been passed through this method then it is returned unaltered rather than wrapped twice.
@@ -234,32 +262,41 @@ public final class ThreadContext implements Closeable, Writeable {
     }
 
     static final class ThreadContextStruct {
-        private final Map<String,String> headers;
+        private final Map<String, String> requestHeaders;
         private final Map<String, Object> transientHeaders;
+        private final Map<String, List<String>> responseHeaders;
 
         private ThreadContextStruct(StreamInput in) throws IOException {
-            int numValues = in.readVInt();
-            Map<String, String> headers = numValues == 0 ? Collections.emptyMap() : new HashMap<>(numValues);
-            for (int i = 0; i < numValues; i++) {
-                headers.put(in.readString(), in.readString());
+            final int numRequest = in.readVInt();
+            Map<String, String> requestHeaders = numRequest == 0 ? Collections.emptyMap() : new HashMap<>(numRequest);
+            for (int i = 0; i < numRequest; i++) {
+                requestHeaders.put(in.readString(), in.readString());
             }
-            this.headers = headers;
+
+            this.requestHeaders = requestHeaders;
+            this.responseHeaders = in.readMapOfLists();
             this.transientHeaders = Collections.emptyMap();
         }
 
-        private ThreadContextStruct(Map<String, String> headers, Map<String, Object> transientHeaders) {
-            this.headers = headers;
+        private ThreadContextStruct(Map<String, String> requestHeaders,
+                                    Map<String, List<String>> responseHeaders,
+                                    Map<String, Object> transientHeaders) {
+            this.requestHeaders = requestHeaders;
+            this.responseHeaders = responseHeaders;
             this.transientHeaders = transientHeaders;
         }
 
-        private ThreadContextStruct(Map<String, String> headers) {
-            this(headers, Collections.emptyMap());
+        /**
+         * This represents the default context and it should only ever be called by {@link #DEFAULT_CONTEXT}.
+         */
+        private ThreadContextStruct() {
+            this(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
         }
 
-        private ThreadContextStruct putPersistent(String key, String value) {
-            Map<String, String> newHeaders = new HashMap<>(this.headers);
-            putSingleHeader(key, value, newHeaders);
-            return new ThreadContextStruct(newHeaders, transientHeaders);
+        private ThreadContextStruct putRequest(String key, String value) {
+            Map<String, String> newRequestHeaders = new HashMap<>(this.requestHeaders);
+            putSingleHeader(key, value, newRequestHeaders);
+            return new ThreadContextStruct(newRequestHeaders, responseHeaders, transientHeaders);
         }
 
         private void putSingleHeader(String key, String value, Map<String, String> newHeaders) {
@@ -276,24 +313,45 @@ public final class ThreadContext implements Closeable, Writeable {
                 for (Map.Entry<String, String> entry : headers.entrySet()) {
                     putSingleHeader(entry.getKey(), entry.getValue(), newHeaders);
                 }
-                newHeaders.putAll(this.headers);
-                return new ThreadContextStruct(newHeaders, transientHeaders);
+                newHeaders.putAll(this.requestHeaders);
+                return new ThreadContextStruct(newHeaders, responseHeaders, transientHeaders);
             }
         }
 
+        private ThreadContextStruct putResponse(String key, String value) {
+            assert value != null;
+
+            final Map<String, List<String>> newResponseHeaders = new HashMap<>(this.responseHeaders);
+            final List<String> existingValues = newResponseHeaders.get(key);
+
+            if (existingValues != null) {
+                if (existingValues.contains(value)) {
+                    return this;
+                }
+
+                final List<String> newValues = new ArrayList<>(existingValues);
+                newValues.add(value);
+
+                newResponseHeaders.put(key, Collections.unmodifiableList(newValues));
+            } else {
+                newResponseHeaders.put(key, Collections.singletonList(value));
+            }
+
+            return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders);
+        }
+
         private ThreadContextStruct putTransient(String key, Object value) {
             Map<String, Object> newTransient = new HashMap<>(this.transientHeaders);
             if (newTransient.putIfAbsent(key, value) != null) {
                 throw new IllegalArgumentException("value for key [" + key + "] already present");
             }
-            return new ThreadContextStruct(headers, newTransient);
+            return new ThreadContextStruct(requestHeaders, responseHeaders, newTransient);
         }
 
         boolean isEmpty() {
-            return headers.isEmpty() && transientHeaders.isEmpty();
+            return requestHeaders.isEmpty() && responseHeaders.isEmpty() && transientHeaders.isEmpty();
         }
 
-
         private ThreadContextStruct copyHeaders(Iterable<Map.Entry<String, String>> headers) {
             Map<String, String> newHeaders = new HashMap<>();
             for (Map.Entry<String, String> header : headers) {
@@ -303,20 +361,21 @@ public final class ThreadContext implements Closeable, Writeable {
         }
 
         private void writeTo(StreamOutput out, Map<String, String> defaultHeaders) throws IOException {
-            final Map<String, String> headers;
+            final Map<String, String> requestHeaders;
             if (defaultHeaders.isEmpty()) {
-                headers = this.headers;
+                requestHeaders = this.requestHeaders;
             } else {
-                headers = new HashMap<>(defaultHeaders);
-                headers.putAll(this.headers);
+                requestHeaders = new HashMap<>(defaultHeaders);
+                requestHeaders.putAll(this.requestHeaders);
             }
 
-            int keys = headers.size();
-            out.writeVInt(keys);
-            for (Map.Entry<String, String> entry : headers.entrySet()) {
+            out.writeVInt(requestHeaders.size());
+            for (Map.Entry<String, String> entry : requestHeaders.entrySet()) {
                 out.writeString(entry.getKey());
                 out.writeString(entry.getValue());
             }
+
+            out.writeMapOfLists(responseHeaders);
         }
 
     }
diff --git a/core/src/main/java/org/elasticsearch/http/netty/HttpRequestHandler.java b/core/src/main/java/org/elasticsearch/http/netty/HttpRequestHandler.java
index 376ca738fab..859708eff71 100644
--- a/core/src/main/java/org/elasticsearch/http/netty/HttpRequestHandler.java
+++ b/core/src/main/java/org/elasticsearch/http/netty/HttpRequestHandler.java
@@ -61,7 +61,7 @@ public class HttpRequestHandler extends SimpleChannelUpstreamHandler {
         // the netty HTTP handling always copy over the buffer to its own buffer, either in NioWorker internally
         // when reading, or using a cumalation buffer
         NettyHttpRequest httpRequest = new NettyHttpRequest(request, e.getChannel());
-        NettyHttpChannel channel = new NettyHttpChannel(serverTransport, httpRequest, oue, detailedErrorsEnabled);
+        NettyHttpChannel channel = new NettyHttpChannel(serverTransport, httpRequest, oue, detailedErrorsEnabled, threadContext);
         serverTransport.dispatchRequest(httpRequest, channel);
         super.messageReceived(ctx, e);
     }
diff --git a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpChannel.java b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpChannel.java
index c4253df2860..8df31b26dc8 100644
--- a/core/src/main/java/org/elasticsearch/http/netty/NettyHttpChannel.java
+++ b/core/src/main/java/org/elasticsearch/http/netty/NettyHttpChannel.java
@@ -25,6 +25,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput;
 import org.elasticsearch.common.lease.Releasable;
 import org.elasticsearch.transport.netty.NettyUtils;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.http.netty.cors.CorsHandler;
 import org.elasticsearch.http.netty.pipelining.OrderedDownstreamChannelEvent;
 import org.elasticsearch.http.netty.pipelining.OrderedUpstreamMessageEvent;
@@ -60,6 +61,7 @@ public final class NettyHttpChannel extends AbstractRestChannel {
     private final Channel channel;
     private final org.jboss.netty.handler.codec.http.HttpRequest nettyRequest;
     private final OrderedUpstreamMessageEvent orderedUpstreamMessageEvent;
+    private final ThreadContext threadContext;
 
     /**
      * @param transport                   The corresponding <code>NettyHttpServerTransport</code> where this channel belongs to.
@@ -70,12 +72,13 @@ public final class NettyHttpChannel extends AbstractRestChannel {
      */
     public NettyHttpChannel(NettyHttpServerTransport transport, NettyHttpRequest request,
                             @Nullable OrderedUpstreamMessageEvent orderedUpstreamMessageEvent,
-                            boolean detailedErrorsEnabled) {
+                            boolean detailedErrorsEnabled, ThreadContext threadContext) {
         super(request, detailedErrorsEnabled);
         this.transport = transport;
         this.channel = request.getChannel();
         this.nettyRequest = request.request();
         this.orderedUpstreamMessageEvent = orderedUpstreamMessageEvent;
+        this.threadContext = threadContext;
     }
 
     @Override
@@ -83,7 +86,6 @@ public final class NettyHttpChannel extends AbstractRestChannel {
         return new ReleasableBytesStreamOutput(transport.bigArrays);
     }
 
-
     @Override
     public void sendResponse(RestResponse response) {
         // if the response object was created upstream, then use it;
@@ -99,7 +101,8 @@ public final class NettyHttpChannel extends AbstractRestChannel {
         }
 
         // Add all custom headers
-        addCustomHeaders(response, resp);
+        addCustomHeaders(resp, response.getHeaders());
+        addCustomHeaders(resp, threadContext.getResponseHeaders());
 
         BytesReference content = response.content();
         ChannelBuffer buffer;
@@ -170,8 +173,7 @@ public final class NettyHttpChannel extends AbstractRestChannel {
         }
     }
 
-    private void addCustomHeaders(RestResponse response, HttpResponse resp) {
-        Map<String, List<String>> customHeaders = response.getHeaders();
+    private void addCustomHeaders(HttpResponse resp, Map<String, List<String>> customHeaders) {
         if (customHeaders != null) {
             for (Map.Entry<String, List<String>> headerEntry : customHeaders.entrySet()) {
                 for (String headerValue : headerEntry.getValue()) {
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index 3d2283bf33f..c3c64818fb0 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -238,6 +238,10 @@ public class Node implements Closeable {
         try {
             final ThreadPool threadPool = new ThreadPool(settings, executorBuilders.toArray(new ExecutorBuilder[0]));
             resourcesToClose.add(() -> ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS));
+            // adds the context to the DeprecationLogger so that it does not need to be injected everywhere
+            DeprecationLogger.setThreadContext(threadPool.getThreadContext());
+            resourcesToClose.add(() -> DeprecationLogger.removeThreadContext(threadPool.getThreadContext()));
+
             final List<Setting<?>> additionalSettings = new ArrayList<>();
             final List<String> additionalSettingsFilter = new ArrayList<>();
             additionalSettings.addAll(pluginsService.getPluginSettings());
@@ -313,7 +317,9 @@ public class Node implements Closeable {
                 }
             );
             injector = modules.createInjector();
+
             client.intialize(injector.getInstance(new Key<Map<GenericAction, TransportAction>>() {}));
+
             success = true;
         } catch (IOException ex) {
             throw new ElasticsearchException("failed to bind service", ex);
diff --git a/core/src/main/java/org/elasticsearch/rest/DeprecationRestHandler.java b/core/src/main/java/org/elasticsearch/rest/DeprecationRestHandler.java
new file mode 100644
index 00000000000..1a78a4c8c04
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/rest/DeprecationRestHandler.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.rest;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.logging.DeprecationLogger;
+
+import java.util.Objects;
+
+/**
+ * {@code DeprecationRestHandler} provides a proxy for any existing {@link RestHandler} so that usage of the handler can be
+ * logged using the {@link DeprecationLogger}.
+ */
+public class DeprecationRestHandler implements RestHandler {
+
+    private final RestHandler handler;
+    private final String deprecationMessage;
+    private final DeprecationLogger deprecationLogger;
+
+    /**
+     * Create a {@link DeprecationRestHandler} that encapsulates the {@code handler} using the {@code deprecationLogger} to log
+     * deprecation {@code warning}.
+     *
+     * @param handler The rest handler to deprecate (it's possible that the handler is reused with a different name!)
+     * @param deprecationMessage The message to warn users with when they use the {@code handler}
+     * @param deprecationLogger The deprecation logger
+     * @throws NullPointerException if any parameter except {@code deprecationMessage} is {@code null}
+     * @throws IllegalArgumentException if {@code deprecationMessage} is not a valid header
+     */
+    public DeprecationRestHandler(RestHandler handler, String deprecationMessage, DeprecationLogger deprecationLogger) {
+        this.handler = Objects.requireNonNull(handler);
+        this.deprecationMessage = requireValidHeader(deprecationMessage);
+        this.deprecationLogger = Objects.requireNonNull(deprecationLogger);
+    }
+
+    /**
+     * {@inheritDoc}
+     * <p>
+     * Usage is logged via the {@link DeprecationLogger} so that the actual response can be notified of deprecation as well.
+     */
+    @Override
+    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
+        deprecationLogger.deprecated(deprecationMessage);
+
+        handler.handleRequest(request, channel, client);
+    }
+
+    /**
+     * This does a very basic pass at validating that a header's value contains only expected characters according to RFC-5987, and those
+     * that it references.
+     * <p>
+     * https://tools.ietf.org/html/rfc5987
+     * <p>
+     * This is only expected to be used for assertions. The idea is that only readable US-ASCII characters are expected; the rest must be
+     * encoded with percent encoding, which makes checking for a valid character range very simple.
+     *
+     * @param value The header value to check
+     * @return {@code true} if the {@code value} is not obviously wrong.
+     */
+    public static boolean validHeaderValue(String value) {
+        if (Strings.hasText(value) == false) {
+            return false;
+        }
+
+        for (int i = 0; i < value.length(); i++) {
+            char c = value.charAt(i);
+
+            // 32 = ' ' (31 = unit separator); 126 = '~' (127 = DEL)
+            if (c < 32 || c > 126) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    /**
+     * Throw an exception if the {@code value} is not a {@link #validHeaderValue(String) valid header}.
+     *
+     * @param value The header value to check
+     * @return Always {@code value}.
+     * @throws IllegalArgumentException if {@code value} is not a {@link #validHeaderValue(String) valid header}.
+     */
+    public static String requireValidHeader(String value) {
+        if (validHeaderValue(value) == false) {
+            throw new IllegalArgumentException("header value must contain only US ASCII text");
+        }
+
+        return value;
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/rest/RestController.java b/core/src/main/java/org/elasticsearch/rest/RestController.java
index 03d2c267c58..4eb11700f96 100644
--- a/core/src/main/java/org/elasticsearch/rest/RestController.java
+++ b/core/src/main/java/org/elasticsearch/rest/RestController.java
@@ -23,6 +23,7 @@ import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.path.PathTrie;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
@@ -112,7 +113,27 @@ public class RestController extends AbstractLifecycleComponent {
     }
 
     /**
-     * Registers a rest handler to be executed when the provided method and path match the request.
+     * Registers a REST handler to be executed when the provided {@code method} and {@code path} match the request.
+     *
+     * @param method GET, POST, etc.
+     * @param path Path to handle (e.g., "/{index}/{type}/_bulk")
+     * @param handler The handler to actually execute
+     * @param deprecationMessage The message to log and send as a header in the response
+     * @param logger The existing deprecation logger to use
+     */
+    public void registerAsDeprecatedHandler(RestRequest.Method method, String path, RestHandler handler,
+                                            String deprecationMessage, DeprecationLogger logger) {
+        assert (handler instanceof DeprecationRestHandler) == false;
+
+        registerHandler(method, path, new DeprecationRestHandler(handler, deprecationMessage, logger));
+    }
+
+    /**
+     * Registers a REST handler to be executed when the provided method and path match the request.
+     *
+     * @param method GET, POST, etc.
+     * @param path Path to handle (e.g., "/{index}/{type}/_bulk")
+     * @param handler The handler to actually execute
      */
     public void registerHandler(RestRequest.Method method, String path, RestHandler handler) {
         PathTrie<RestHandler> handlers = getHandlersForMethod(method);
diff --git a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
index b56b91061ed..799f6df7902 100644
--- a/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/TcpTransport.java
@@ -936,10 +936,11 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
      */
     public void sendErrorResponse(Version nodeVersion, Channel channel, final Exception error, final long requestId,
                                   final String action) throws IOException {
-        try(BytesStreamOutput stream = new BytesStreamOutput()) {
+        try (BytesStreamOutput stream = new BytesStreamOutput()) {
             stream.setVersion(nodeVersion);
             RemoteTransportException tx = new RemoteTransportException(
                 nodeName(), new InetSocketTransportAddress(getLocalAddress(channel)), action, error);
+            threadPool.getThreadContext().writeTo(stream);
             stream.writeException(tx);
             byte status = 0;
             status = TransportStatus.setResponse(status);
@@ -972,8 +973,9 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
                 status = TransportStatus.setCompress(status);
                 stream = CompressorFactory.COMPRESSOR.streamOutput(stream);
             }
+            threadPool.getThreadContext().writeTo(stream);
             stream.setVersion(nodeVersion);
-            BytesReference reference = buildMessage(requestId, status,nodeVersion, response, stream, bStream);
+            BytesReference reference = buildMessage(requestId, status, nodeVersion, response, stream, bStream);
 
             final TransportResponseOptions finalOptions = options;
             Runnable onRequestSent = () -> {
@@ -1006,8 +1008,7 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
     private BytesReference buildHeader(long requestId, byte status, Version protocolVersion, int length) throws IOException {
         try (BytesStreamOutput headerOutput = new BytesStreamOutput(TcpHeader.HEADER_SIZE)) {
             headerOutput.setVersion(protocolVersion);
-            TcpHeader.writeHeader(headerOutput, requestId, status, protocolVersion,
-                length);
+            TcpHeader.writeHeader(headerOutput, requestId, status, protocolVersion, length);
             final BytesReference bytes = headerOutput.bytes();
             assert bytes.length() == TcpHeader.HEADER_SIZE : "header size mismatch expected: " + TcpHeader.HEADER_SIZE + " but was: "
                 + bytes.length();
@@ -1174,8 +1175,8 @@ public abstract class TcpTransport<Channel> extends AbstractLifecycleComponent i
             }
             streamIn = new NamedWriteableAwareStreamInput(streamIn, namedWriteableRegistry);
             streamIn.setVersion(version);
+            threadPool.getThreadContext().readHeaders(streamIn);
             if (TransportStatus.isRequest(status)) {
-                threadPool.getThreadContext().readHeaders(streamIn);
                 handleRequest(channel, profileName, streamIn, requestId, messageLengthBytes, version, remoteAddress);
             } else {
                 final TransportResponseHandler<?> handler = transportServiceAdapter.onResponseReceived(requestId);
diff --git a/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java b/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
index f6519cbeeba..eba5fd57734 100644
--- a/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
+++ b/core/src/main/java/org/elasticsearch/transport/local/LocalTransport.java
@@ -259,9 +259,8 @@ public class LocalTransport extends AbstractLifecycleComponent implements Transp
             long requestId = stream.readLong();
             byte status = stream.readByte();
             boolean isRequest = TransportStatus.isRequest(status);
+            threadPool.getThreadContext().readHeaders(stream);
             if (isRequest) {
-                ThreadContext threadContext = threadPool.getThreadContext();
-                threadContext.readHeaders(stream);
                 handleRequest(stream, requestId, data.length, sourceTransport, version);
             } else {
                 final TransportResponseHandler handler = transportServiceAdapter.onResponseReceived(requestId);
@@ -304,7 +303,7 @@ public class LocalTransport extends AbstractLifecycleComponent implements Transp
             inFlightRequestsBreaker().addWithoutBreaking(messageLengthBytes);
         }
         final LocalTransportChannel transportChannel = new LocalTransportChannel(this, transportServiceAdapter, sourceTransport, action,
-            requestId, version, messageLengthBytes);
+            requestId, version, messageLengthBytes, threadPool.getThreadContext());
         try {
             if (reg == null) {
                 throw new ActionNotFoundTransportException("Action [" + action + "] not found");
@@ -389,7 +388,7 @@ public class LocalTransport extends AbstractLifecycleComponent implements Transp
 
     private void handleException(final TransportResponseHandler handler, Exception exception) {
         if (!(exception instanceof RemoteTransportException)) {
-            exception = new RemoteTransportException("None remote transport exception", null, null, exception);
+            exception = new RemoteTransportException("Not a remote transport exception", null, null, exception);
         }
         final RemoteTransportException rtx = (RemoteTransportException) exception;
         try {
diff --git a/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java b/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java
index 81d99b8d493..fc748b96aea 100644
--- a/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java
+++ b/core/src/main/java/org/elasticsearch/transport/local/LocalTransportChannel.java
@@ -48,10 +48,12 @@ public class LocalTransportChannel implements TransportChannel {
     private final long requestId;
     private final Version version;
     private final long reservedBytes;
+    private final ThreadContext threadContext;
     private final AtomicBoolean closed = new AtomicBoolean();
 
     public LocalTransportChannel(LocalTransport sourceTransport, TransportServiceAdapter sourceTransportServiceAdapter,
-                                 LocalTransport targetTransport, String action, long requestId, Version version, long reservedBytes) {
+                                 LocalTransport targetTransport, String action, long requestId, Version version, long reservedBytes,
+                                 ThreadContext threadContext) {
         this.sourceTransport = sourceTransport;
         this.sourceTransportServiceAdapter = sourceTransportServiceAdapter;
         this.targetTransport = targetTransport;
@@ -59,6 +61,7 @@ public class LocalTransportChannel implements TransportChannel {
         this.requestId = requestId;
         this.version = version;
         this.reservedBytes = reservedBytes;
+        this.threadContext = threadContext;
     }
 
     @Override
@@ -84,6 +87,7 @@ public class LocalTransportChannel implements TransportChannel {
             byte status = 0;
             status = TransportStatus.setResponse(status);
             stream.writeByte(status); // 0 for request, 1 for response.
+            threadContext.writeTo(stream);
             response.writeTo(stream);
             sendResponseData(BytesReference.toBytes(stream.bytes()));
             sourceTransportServiceAdapter.onResponseSent(requestId, action, response, options);
@@ -135,5 +139,6 @@ public class LocalTransportChannel implements TransportChannel {
         status = TransportStatus.setResponse(status);
         status = TransportStatus.setError(status);
         stream.writeByte(status);
+        threadContext.writeTo(stream);
     }
 }
diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
index 94f07369770..ee16eb82a2c 100644
--- a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
@@ -30,7 +30,9 @@ import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 
 import static org.hamcrest.Matchers.closeTo;
@@ -445,7 +447,7 @@ public class BytesStreamsTests extends ESTestCase {
 
         final StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes()));
 
-        List<TestStreamable> loaded = in.readStreamableList(TestStreamable::new);
+        final List<TestStreamable> loaded = in.readStreamableList(TestStreamable::new);
 
         assertThat(loaded, hasSize(expected.size()));
 
@@ -459,7 +461,49 @@ public class BytesStreamsTests extends ESTestCase {
         out.close();
     }
 
-    private abstract static class BaseNamedWriteable implements NamedWriteable {
+    public void testWriteMapOfLists() throws IOException {
+        final int size = randomIntBetween(0, 5);
+        final Map<String, List<String>> expected = new HashMap<>(size);
+
+        for (int i = 0; i < size; ++i) {
+            int listSize = randomIntBetween(0, 5);
+            List<String> list = new ArrayList<>(listSize);
+
+            for (int j = 0; j < listSize; ++j) {
+                list.add(randomAsciiOfLength(5));
+            }
+
+            expected.put(randomAsciiOfLength(2), list);
+        }
+
+        final BytesStreamOutput out = new BytesStreamOutput();
+        out.writeMapOfLists(expected);
+
+        final StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes()));
+
+        final Map<String, List<String>> loaded = in.readMapOfLists();
+
+        assertThat(loaded.size(), equalTo(expected.size()));
+
+        for (Map.Entry<String, List<String>> entry : expected.entrySet()) {
+            assertThat(loaded.containsKey(entry.getKey()), equalTo(true));
+
+            List<String> loadedList = loaded.get(entry.getKey());
+
+            assertThat(loadedList, hasSize(entry.getValue().size()));
+
+            for (int i = 0; i < loadedList.size(); ++i) {
+                assertEquals(entry.getValue().get(i), loadedList.get(i));
+            }
+        }
+
+        assertEquals(0, in.available());
+
+        in.close();
+        out.close();
+    }
+
+    private static abstract class BaseNamedWriteable implements NamedWriteable {
 
     }
 
diff --git a/core/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java b/core/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java
new file mode 100644
index 00000000000..f75e73ced2c
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.common.logging;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasSize;
+
+/**
+ * Tests {@link DeprecationLogger}
+ */
+public class DeprecationLoggerTests extends ESTestCase {
+
+    private final DeprecationLogger logger = new DeprecationLogger(Loggers.getLogger(getClass()));
+
+    public void testAddsHeaderWithThreadContext() throws IOException {
+        String msg = "A simple message [{}]";
+        String param = randomAsciiOfLengthBetween(1, 5);
+        String formatted = LoggerMessageFormat.format(msg, (Object)param);
+
+        try (ThreadContext threadContext = new ThreadContext(Settings.EMPTY)) {
+            Set<ThreadContext> threadContexts = Collections.singleton(threadContext);
+
+            logger.deprecated(threadContexts, msg, param);
+
+            Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
+
+            assertEquals(1, responseHeaders.size());
+            assertEquals(formatted, responseHeaders.get(DeprecationLogger.DEPRECATION_HEADER).get(0));
+        }
+    }
+
+    public void testAddsCombinedHeaderWithThreadContext() throws IOException {
+        String msg = "A simple message [{}]";
+        String param = randomAsciiOfLengthBetween(1, 5);
+        String formatted = LoggerMessageFormat.format(msg, (Object)param);
+        String formatted2 = randomAsciiOfLengthBetween(1, 10);
+
+        try (ThreadContext threadContext = new ThreadContext(Settings.EMPTY)) {
+            Set<ThreadContext> threadContexts = Collections.singleton(threadContext);
+
+            logger.deprecated(threadContexts, msg, param);
+            logger.deprecated(threadContexts, formatted2);
+
+            Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
+
+            assertEquals(1, responseHeaders.size());
+
+            List<String> responses = responseHeaders.get(DeprecationLogger.DEPRECATION_HEADER);
+
+            assertEquals(2, responses.size());
+            assertEquals(formatted, responses.get(0));
+            assertEquals(formatted2, responses.get(1));
+        }
+    }
+
+    public void testCanRemoveThreadContext() throws IOException {
+        final String expected = "testCanRemoveThreadContext";
+        final String unexpected = "testCannotRemoveThreadContext";
+
+        try (ThreadContext threadContext = new ThreadContext(Settings.EMPTY)) {
+            // NOTE: by adding it to the logger, we allow any concurrent test to write to it (from their own threads)
+            DeprecationLogger.setThreadContext(threadContext);
+
+            logger.deprecated(expected);
+
+            Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
+            List<String> responses = responseHeaders.get(DeprecationLogger.DEPRECATION_HEADER);
+
+            // ensure it works (note: concurrent tests may be adding to it, but in different threads, so it should have no impact)
+            assertThat(responses, hasSize(atLeast(1)));
+            assertThat(responses, hasItem(equalTo(expected)));
+
+            DeprecationLogger.removeThreadContext(threadContext);
+
+            logger.deprecated(unexpected);
+
+            responseHeaders = threadContext.getResponseHeaders();
+            responses = responseHeaders.get(DeprecationLogger.DEPRECATION_HEADER);
+
+            assertThat(responses, hasSize(atLeast(1)));
+            assertThat(responses, hasItem(expected));
+            assertThat(responses, not(hasItem(unexpected)));
+        }
+    }
+
+    public void testIgnoresClosedThreadContext() throws IOException {
+        ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+        Set<ThreadContext> threadContexts = new HashSet<>(1);
+
+        threadContexts.add(threadContext);
+
+        threadContext.close();
+
+        logger.deprecated(threadContexts, "Ignored logger message");
+
+        assertTrue(threadContexts.contains(threadContext));
+    }
+
+    public void testSafeWithoutThreadContext() {
+        logger.deprecated(Collections.emptySet(), "Ignored");
+    }
+
+    public void testFailsWithoutThreadContextSet() {
+        expectThrows(NullPointerException.class, () -> logger.deprecated((Set<ThreadContext>)null, "Does not explode"));
+    }
+
+    public void testFailsWhenDoubleSettingSameThreadContext() throws IOException {
+        try (ThreadContext threadContext = new ThreadContext(Settings.EMPTY)) {
+            DeprecationLogger.setThreadContext(threadContext);
+
+            try {
+                expectThrows(IllegalStateException.class, () -> DeprecationLogger.setThreadContext(threadContext));
+            } finally {
+                // cleanup after ourselves
+                DeprecationLogger.removeThreadContext(threadContext);
+            }
+        }
+    }
+
+    public void testFailsWhenRemovingUnknownThreadContext() throws IOException {
+        try (ThreadContext threadContext = new ThreadContext(Settings.EMPTY)) {
+            expectThrows(IllegalStateException.class, () -> DeprecationLogger.removeThreadContext(threadContext));
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
index e6726879513..d402f09f07d 100644
--- a/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
+++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java
@@ -19,14 +19,18 @@
 package org.elasticsearch.common.util.concurrent;
 
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.sameInstance;
 
 public class ThreadContextTests extends ESTestCase {
@@ -35,7 +39,7 @@ public class ThreadContextTests extends ESTestCase {
         Settings build = Settings.builder().put("request.headers.default", "1").build();
         ThreadContext threadContext = new ThreadContext(build);
         threadContext.putHeader("foo", "bar");
-        threadContext.putTransient("ctx.foo", new Integer(1));
+        threadContext.putTransient("ctx.foo", 1);
         assertEquals("bar", threadContext.getHeader("foo"));
         assertEquals(new Integer(1), threadContext.getTransient("ctx.foo"));
         assertEquals("1", threadContext.getHeader("default"));
@@ -46,7 +50,7 @@ public class ThreadContextTests extends ESTestCase {
         }
 
         assertEquals("bar", threadContext.getHeader("foo"));
-        assertEquals(new Integer(1), threadContext.getTransient("ctx.foo"));
+        assertEquals(Integer.valueOf(1), threadContext.getTransient("ctx.foo"));
         assertEquals("1", threadContext.getHeader("default"));
     }
 
@@ -54,7 +58,7 @@ public class ThreadContextTests extends ESTestCase {
         Settings build = Settings.builder().put("request.headers.default", "1").build();
         ThreadContext threadContext = new ThreadContext(build);
         threadContext.putHeader("foo", "bar");
-        threadContext.putTransient("ctx.foo", new Integer(1));
+        threadContext.putTransient("ctx.foo", 1);
         assertEquals("bar", threadContext.getHeader("foo"));
         assertEquals(new Integer(1), threadContext.getTransient("ctx.foo"));
         assertEquals("1", threadContext.getHeader("default"));
@@ -70,7 +74,7 @@ public class ThreadContextTests extends ESTestCase {
 
         assertNull(threadContext.getHeader("simon"));
         assertEquals("bar", threadContext.getHeader("foo"));
-        assertEquals(new Integer(1), threadContext.getTransient("ctx.foo"));
+        assertEquals(Integer.valueOf(1), threadContext.getTransient("ctx.foo"));
         assertEquals("1", threadContext.getHeader("default"));
     }
 
@@ -78,9 +82,9 @@ public class ThreadContextTests extends ESTestCase {
         Settings build = Settings.builder().put("request.headers.default", "1").build();
         ThreadContext threadContext = new ThreadContext(build);
         threadContext.putHeader("foo", "bar");
-        threadContext.putTransient("ctx.foo", new Integer(1));
+        threadContext.putTransient("ctx.foo", 1);
         assertEquals("bar", threadContext.getHeader("foo"));
-        assertEquals(new Integer(1), threadContext.getTransient("ctx.foo"));
+        assertEquals(Integer.valueOf(1), threadContext.getTransient("ctx.foo"));
         assertEquals("1", threadContext.getHeader("default"));
         ThreadContext.StoredContext storedContext = threadContext.newStoredContext();
         threadContext.putHeader("foo.bar", "baz");
@@ -91,7 +95,7 @@ public class ThreadContextTests extends ESTestCase {
         }
 
         assertEquals("bar", threadContext.getHeader("foo"));
-        assertEquals(new Integer(1), threadContext.getTransient("ctx.foo"));
+        assertEquals(Integer.valueOf(1), threadContext.getTransient("ctx.foo"));
         assertEquals("1", threadContext.getHeader("default"));
         assertEquals("baz", threadContext.getHeader("foo.bar"));
         if (randomBoolean()) {
@@ -100,11 +104,44 @@ public class ThreadContextTests extends ESTestCase {
             storedContext.close();
         }
         assertEquals("bar", threadContext.getHeader("foo"));
-        assertEquals(new Integer(1), threadContext.getTransient("ctx.foo"));
+        assertEquals(Integer.valueOf(1), threadContext.getTransient("ctx.foo"));
         assertEquals("1", threadContext.getHeader("default"));
         assertNull(threadContext.getHeader("foo.bar"));
     }
 
+    public void testResponseHeaders() {
+        final boolean expectThird = randomBoolean();
+
+        final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+
+        threadContext.addResponseHeader("foo", "bar");
+        // pretend that another thread created the same response
+        if (randomBoolean()) {
+            threadContext.addResponseHeader("foo", "bar");
+        }
+
+        threadContext.addResponseHeader("Warning", "One is the loneliest number");
+        threadContext.addResponseHeader("Warning", "Two can be as bad as one");
+        if (expectThird) {
+            threadContext.addResponseHeader("Warning", "No is the saddest experience");
+        }
+
+        final Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
+        final List<String> foo = responseHeaders.get("foo");
+        final List<String> warnings = responseHeaders.get("Warning");
+        final int expectedWarnings = expectThird ? 3 : 2;
+
+        assertThat(foo, hasSize(1));
+        assertEquals("bar", foo.get(0));
+        assertThat(warnings, hasSize(expectedWarnings));
+        assertThat(warnings, hasItem(equalTo("One is the loneliest number")));
+        assertThat(warnings, hasItem(equalTo("Two can be as bad as one")));
+
+        if (expectThird) {
+            assertThat(warnings, hasItem(equalTo("No is the saddest experience")));
+        }
+    }
+
     public void testCopyHeaders() {
         Settings build = Settings.builder().put("request.headers.default", "1").build();
         ThreadContext threadContext = new ThreadContext(build);
@@ -117,7 +154,7 @@ public class ThreadContextTests extends ESTestCase {
         Settings build = Settings.builder().put("request.headers.default", "1").build();
         ThreadContext threadContext = new ThreadContext(build);
         threadContext.putHeader("foo", "bar");
-        threadContext.putTransient("ctx.foo", new Integer(1));
+        threadContext.putTransient("ctx.foo", 1);
 
         threadContext.close();
         try {
@@ -146,20 +183,35 @@ public class ThreadContextTests extends ESTestCase {
         Settings build = Settings.builder().put("request.headers.default", "1").build();
         ThreadContext threadContext = new ThreadContext(build);
         threadContext.putHeader("foo", "bar");
-        threadContext.putTransient("ctx.foo", new Integer(1));
+        threadContext.putTransient("ctx.foo", 1);
+        threadContext.addResponseHeader("Warning", "123456");
+        if (rarely()) {
+            threadContext.addResponseHeader("Warning", "123456");
+        }
+        threadContext.addResponseHeader("Warning", "234567");
+
         BytesStreamOutput out = new BytesStreamOutput();
         threadContext.writeTo(out);
         try (ThreadContext.StoredContext ctx = threadContext.stashContext()) {
             assertNull(threadContext.getHeader("foo"));
             assertNull(threadContext.getTransient("ctx.foo"));
+            assertTrue(threadContext.getResponseHeaders().isEmpty());
             assertEquals("1", threadContext.getHeader("default"));
 
             threadContext.readHeaders(out.bytes().streamInput());
             assertEquals("bar", threadContext.getHeader("foo"));
             assertNull(threadContext.getTransient("ctx.foo"));
+
+            final Map<String, List<String>> responseHeaders = threadContext.getResponseHeaders();
+            final List<String> warnings = responseHeaders.get("Warning");
+
+            assertThat(responseHeaders.keySet(), hasSize(1));
+            assertThat(warnings, hasSize(2));
+            assertThat(warnings, hasItem(equalTo("123456")));
+            assertThat(warnings, hasItem(equalTo("234567")));
         }
         assertEquals("bar", threadContext.getHeader("foo"));
-        assertEquals(new Integer(1), threadContext.getTransient("ctx.foo"));
+        assertEquals(Integer.valueOf(1), threadContext.getTransient("ctx.foo"));
         assertEquals("1", threadContext.getHeader("default"));
     }
 
@@ -169,21 +221,35 @@ public class ThreadContextTests extends ESTestCase {
             Settings build = Settings.builder().put("request.headers.default", "1").build();
             ThreadContext threadContext = new ThreadContext(build);
             threadContext.putHeader("foo", "bar");
-            threadContext.putTransient("ctx.foo", new Integer(1));
+            threadContext.putTransient("ctx.foo", 1);
+            threadContext.addResponseHeader("Warning", "123456");
+            if (rarely()) {
+                threadContext.addResponseHeader("Warning", "123456");
+            }
+            threadContext.addResponseHeader("Warning", "234567");
 
             assertEquals("bar", threadContext.getHeader("foo"));
             assertNotNull(threadContext.getTransient("ctx.foo"));
             assertEquals("1", threadContext.getHeader("default"));
+            assertThat(threadContext.getResponseHeaders().keySet(), hasSize(1));
             threadContext.writeTo(out);
         }
         {
             Settings otherSettings = Settings.builder().put("request.headers.default", "5").build();
-            ThreadContext otherhreadContext = new ThreadContext(otherSettings);
-            otherhreadContext.readHeaders(out.bytes().streamInput());
+            ThreadContext otherThreadContext = new ThreadContext(otherSettings);
+            otherThreadContext.readHeaders(out.bytes().streamInput());
 
-            assertEquals("bar", otherhreadContext.getHeader("foo"));
-            assertNull(otherhreadContext.getTransient("ctx.foo"));
-            assertEquals("1", otherhreadContext.getHeader("default"));
+            assertEquals("bar", otherThreadContext.getHeader("foo"));
+            assertNull(otherThreadContext.getTransient("ctx.foo"));
+            assertEquals("1", otherThreadContext.getHeader("default"));
+
+            final Map<String, List<String>> responseHeaders = otherThreadContext.getResponseHeaders();
+            final List<String> warnings = responseHeaders.get("Warning");
+
+            assertThat(responseHeaders.keySet(), hasSize(1));
+            assertThat(warnings, hasSize(2));
+            assertThat(warnings, hasItem(equalTo("123456")));
+            assertThat(warnings, hasItem(equalTo("234567")));
         }
     }
 
@@ -192,7 +258,7 @@ public class ThreadContextTests extends ESTestCase {
         {
             ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
             threadContext.putHeader("foo", "bar");
-            threadContext.putTransient("ctx.foo", new Integer(1));
+            threadContext.putTransient("ctx.foo", 1);
 
             assertEquals("bar", threadContext.getHeader("foo"));
             assertNotNull(threadContext.getTransient("ctx.foo"));
@@ -210,7 +276,6 @@ public class ThreadContextTests extends ESTestCase {
         }
     }
 
-
     public void testCanResetDefault() {
         Settings build = Settings.builder().put("request.headers.default", "1").build();
         ThreadContext threadContext = new ThreadContext(build);
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java
index a56e9993434..72dd6c8c28b 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpChannelTests.java
@@ -181,7 +181,7 @@ public class NettyHttpChannelTests extends ESTestCase {
         NettyHttpRequest request = new NettyHttpRequest(httpRequest, writeCapturingChannel);
 
         // send a response
-        NettyHttpChannel channel = new NettyHttpChannel(httpServerTransport, request, null, randomBoolean());
+        NettyHttpChannel channel = new NettyHttpChannel(httpServerTransport, request, null, randomBoolean(), threadPool.getThreadContext());
         TestReponse resp = new TestReponse();
         final String customHeader = "custom-header";
         final String customHeaderValue = "xyz";
@@ -207,7 +207,7 @@ public class NettyHttpChannelTests extends ESTestCase {
         WriteCapturingChannel writeCapturingChannel = new WriteCapturingChannel();
         NettyHttpRequest request = new NettyHttpRequest(httpRequest, writeCapturingChannel);
 
-        NettyHttpChannel channel = new NettyHttpChannel(httpServerTransport, request, null, randomBoolean());
+        NettyHttpChannel channel = new NettyHttpChannel(httpServerTransport, request, null, randomBoolean(), threadPool.getThreadContext());
         channel.sendResponse(new TestReponse());
 
         // get the response
diff --git a/core/src/test/java/org/elasticsearch/rest/DeprecationHttpIT.java b/core/src/test/java/org/elasticsearch/rest/DeprecationHttpIT.java
new file mode 100644
index 00000000000..3bb61f64e36
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/rest/DeprecationHttpIT.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.rest;
+
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.entity.StringEntity;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+import org.elasticsearch.common.logging.LoggerMessageFormat;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.plugins.TestDeprecatedQueryBuilder;
+import org.elasticsearch.rest.plugins.TestDeprecationHeaderRestAction;
+import org.elasticsearch.rest.plugins.TestDeprecationPlugin;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import org.hamcrest.Matcher;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.rest.RestStatus.OK;
+import static org.elasticsearch.rest.plugins.TestDeprecationHeaderRestAction.TEST_DEPRECATED_SETTING_TRUE1;
+import static org.elasticsearch.rest.plugins.TestDeprecationHeaderRestAction.TEST_DEPRECATED_SETTING_TRUE2;
+import static org.elasticsearch.rest.plugins.TestDeprecationHeaderRestAction.TEST_NOT_DEPRECATED_SETTING;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasSize;
+
+/**
+ * Tests {@code DeprecationLogger} uses the {@code ThreadContext} to add response headers.
+ */
+public class DeprecationHttpIT extends ESIntegTestCase {
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder()
+                .put(super.nodeSettings(nodeOrdinal))
+                .put("force.http.enabled", true)
+                // change values of deprecated settings so that accessing them is logged
+                .put(TEST_DEPRECATED_SETTING_TRUE1.getKey(), ! TEST_DEPRECATED_SETTING_TRUE1.getDefault(Settings.EMPTY))
+                .put(TEST_DEPRECATED_SETTING_TRUE2.getKey(), ! TEST_DEPRECATED_SETTING_TRUE2.getDefault(Settings.EMPTY))
+                // non-deprecated setting to ensure not everything is logged
+                .put(TEST_NOT_DEPRECATED_SETTING.getKey(), ! TEST_NOT_DEPRECATED_SETTING.getDefault(Settings.EMPTY))
+                .build();
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return pluginList(TestDeprecationPlugin.class);
+    }
+
+    /**
+     * Attempts to do a scatter/gather request that expects unique responses per sub-request.
+     */
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/19222")
+    public void testUniqueDeprecationResponsesMergedTogether() throws IOException {
+        final String[] indices = new String[randomIntBetween(2, 5)];
+
+        // add at least one document for each index
+        for (int i = 0; i < indices.length; ++i) {
+            indices[i] = "test" + i;
+
+            // create indices with a single shard to reduce noise; the query only deprecates uniquely by index anyway
+            assertTrue(prepareCreate(indices[i]).setSettings(Settings.builder().put("number_of_shards", 1)).get().isAcknowledged());
+
+            int randomDocCount = randomIntBetween(1, 2);
+
+            for (int j = 0; j < randomDocCount; ++j) {
+                index(indices[i], "type", Integer.toString(j), "{\"field\":" + j + "}");
+            }
+        }
+
+        refresh(indices);
+
+        final String commaSeparatedIndices = Stream.of(indices).collect(Collectors.joining(","));
+
+        final String body =
+            "{\"query\":{\"bool\":{\"filter\":[{\"" + TestDeprecatedQueryBuilder.NAME +  "\":{}}]}}}";
+
+        // trigger all index deprecations
+        try (Response response = getRestClient().performRequest("GET",
+                                                                "/" + commaSeparatedIndices + "/_search",
+                                                                Collections.emptyMap(),
+                                                                new StringEntity(body, RestClient.JSON_CONTENT_TYPE))) {
+            assertThat(response.getStatusLine().getStatusCode(), equalTo(OK.getStatus()));
+
+            final List<String> deprecatedWarnings = getWarningHeaders(response.getHeaders());
+            final List<Matcher<String>> headerMatchers = new ArrayList<>(indices.length);
+
+            for (String index : indices) {
+                headerMatchers.add(containsString(LoggerMessageFormat.format("[{}] index", (Object)index)));
+            }
+
+            assertThat(deprecatedWarnings, hasSize(headerMatchers.size()));
+            for (Matcher<String> headerMatcher : headerMatchers) {
+                assertThat(deprecatedWarnings, hasItem(headerMatcher));
+            }
+        }
+    }
+
+    public void testDeprecationWarningsAppearInHeaders() throws IOException {
+        doTestDeprecationWarningsAppearInHeaders();
+    }
+
+    public void testDeprecationHeadersDoNotGetStuck() throws IOException {
+        doTestDeprecationWarningsAppearInHeaders();
+        doTestDeprecationWarningsAppearInHeaders();
+        if (rarely()) {
+            doTestDeprecationWarningsAppearInHeaders();
+        }
+    }
+
+    /**
+     * Run a request that receives a predictably randomized number of deprecation warnings.
+     * <p>
+     * Re-running this back-to-back helps to ensure that warnings are not being maintained across requests.
+     */
+    private void doTestDeprecationWarningsAppearInHeaders() throws IOException {
+        final boolean useDeprecatedField = randomBoolean();
+        final boolean useNonDeprecatedSetting = randomBoolean();
+
+        // deprecated settings should also trigger a deprecation warning
+        final List<Setting<Boolean>> settings = new ArrayList<>(3);
+        settings.add(TEST_DEPRECATED_SETTING_TRUE1);
+
+        if (randomBoolean()) {
+            settings.add(TEST_DEPRECATED_SETTING_TRUE2);
+        }
+
+        if (useNonDeprecatedSetting) {
+            settings.add(TEST_NOT_DEPRECATED_SETTING);
+        }
+
+        Collections.shuffle(settings, random());
+
+        // trigger all deprecations
+        try (Response response = getRestClient().performRequest("GET",
+                                                                "/_test_cluster/deprecated_settings",
+                                                                Collections.emptyMap(),
+                                                                buildSettingsRequest(settings, useDeprecatedField))) {
+            assertThat(response.getStatusLine().getStatusCode(), equalTo(OK.getStatus()));
+
+            final List<String> deprecatedWarnings = getWarningHeaders(response.getHeaders());
+            final List<Matcher<String>> headerMatchers = new ArrayList<>(4);
+
+            headerMatchers.add(equalTo(TestDeprecationHeaderRestAction.DEPRECATED_ENDPOINT));
+            if (useDeprecatedField) {
+                headerMatchers.add(equalTo(TestDeprecationHeaderRestAction.DEPRECATED_USAGE));
+            }
+            for (Setting<?> setting : settings) {
+                if (setting.isDeprecated()) {
+                    headerMatchers.add(containsString(LoggerMessageFormat.format("[{}] setting was deprecated", (Object)setting.getKey())));
+                }
+            }
+
+            assertThat(deprecatedWarnings, hasSize(headerMatchers.size()));
+            for (Matcher<String> headerMatcher : headerMatchers) {
+                assertThat(deprecatedWarnings, hasItem(headerMatcher));
+            }
+        }
+    }
+
+    private List<String> getWarningHeaders(Header[] headers) {
+        List<String> warnings = new ArrayList<>();
+
+        for (Header header : headers) {
+            if (header.getName().equals("Warning")) {
+                warnings.add(header.getValue());
+            }
+        }
+
+        return warnings;
+    }
+
+    private HttpEntity buildSettingsRequest(List<Setting<Boolean>> settings, boolean useDeprecatedField) throws IOException {
+        XContentBuilder builder = JsonXContent.contentBuilder();
+
+        builder.startObject().startArray(useDeprecatedField ? "deprecated_settings" : "settings");
+
+        for (Setting<Boolean> setting : settings) {
+            builder.value(setting.getKey());
+        }
+
+        builder.endArray().endObject();
+
+        return new StringEntity(builder.string(), RestClient.JSON_CONTENT_TYPE);
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java b/core/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java
new file mode 100644
index 00000000000..be0f3b15115
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/rest/DeprecationRestHandlerTests.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.rest;
+
+import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.test.ESTestCase;
+
+import org.mockito.InOrder;
+
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+
+/**
+ * Tests {@link DeprecationRestHandler}.
+ */
+public class DeprecationRestHandlerTests extends ESTestCase {
+
+    private final RestHandler handler = mock(RestHandler.class);
+    /**
+     * Note: Headers should only use US ASCII (and this inevitably becomes one!).
+     */
+    private final String deprecationMessage = randomAsciiOfLengthBetween(1, 30);
+    private final DeprecationLogger deprecationLogger = mock(DeprecationLogger.class);
+
+    public void testNullHandler() {
+        expectThrows(NullPointerException.class, () -> new DeprecationRestHandler(null, deprecationMessage, deprecationLogger));
+    }
+
+    public void testInvalidDeprecationMessageThrowsException() {
+        String invalidDeprecationMessage = randomFrom("", null, "     ");
+
+        expectThrows(IllegalArgumentException.class,
+                     () -> new DeprecationRestHandler(handler, invalidDeprecationMessage, deprecationLogger));
+    }
+
+    public void testNullDeprecationLogger() {
+        expectThrows(NullPointerException.class, () -> new DeprecationRestHandler(handler, deprecationMessage, null));
+    }
+
+    public void testHandleRequestLogsWarningThenForwards() throws Exception {
+        RestRequest request = mock(RestRequest.class);
+        RestChannel channel = mock(RestChannel.class);
+        NodeClient client = mock(NodeClient.class);
+
+        DeprecationRestHandler deprecatedHandler = new DeprecationRestHandler(handler, deprecationMessage, deprecationLogger);
+
+        // test it
+        deprecatedHandler.handleRequest(request, channel, client);
+
+        InOrder inOrder = inOrder(handler, request, channel, deprecationLogger);
+
+        // log, then forward
+        inOrder.verify(deprecationLogger).deprecated(deprecationMessage);
+        inOrder.verify(handler).handleRequest(request, channel, client);
+        inOrder.verifyNoMoreInteractions();
+    }
+
+    public void testValidHeaderValue() {
+        ASCIIHeaderGenerator generator = new ASCIIHeaderGenerator();
+        String value = generator.ofCodeUnitsLength(random(), 1, 50);
+
+        assertTrue(DeprecationRestHandler.validHeaderValue(value));
+        assertSame(value, DeprecationRestHandler.requireValidHeader(value));
+    }
+
+    public void testInvalidHeaderValue() {
+        ASCIIHeaderGenerator generator = new ASCIIHeaderGenerator();
+        String value = generator.ofCodeUnitsLength(random(), 0, 25) +
+                       randomFrom('\t', '\0', '\n', (char)27 /* ESC */, (char)31 /* unit separator*/, (char)127 /* DEL */) +
+                       generator.ofCodeUnitsLength(random(), 0, 25);
+
+        assertFalse(DeprecationRestHandler.validHeaderValue(value));
+
+        expectThrows(IllegalArgumentException.class, () -> DeprecationRestHandler.requireValidHeader(value));
+    }
+
+    public void testInvalidHeaderValueNull() {
+        assertFalse(DeprecationRestHandler.validHeaderValue(null));
+
+        expectThrows(IllegalArgumentException.class, () -> DeprecationRestHandler.requireValidHeader(null));
+    }
+
+    public void testInvalidHeaderValueEmpty() {
+        String blank = randomFrom("", "\t", "    ");
+
+        assertFalse(DeprecationRestHandler.validHeaderValue(blank));
+
+        expectThrows(IllegalArgumentException.class, () -> DeprecationRestHandler.requireValidHeader(blank));
+    }
+
+    /**
+     * {@code ASCIIHeaderGenerator} only uses characters expected to be valid in headers (simplified US-ASCII).
+     */
+    private static class ASCIIHeaderGenerator extends CodepointSetGenerator {
+        /**
+         * Create a character array for characters [{@code from}, {@code to}].
+         *
+         * @param from Starting code point (inclusive).
+         * @param to Ending code point (inclusive).
+         * @return Never {@code null}.
+         */
+        static char[] asciiFromTo(int from, int to) {
+            char[] chars = new char[to - from + 1];
+
+            for (int i = from; i <= to; ++i) {
+                chars[i - from] = (char)i;
+            }
+
+            return chars;
+        }
+
+        /**
+         * Create a generator for characters [32, 126].
+         */
+        public ASCIIHeaderGenerator() {
+            super(asciiFromTo(32, 126));
+        }
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java
index 834afe5d5cd..de4e2db6a21 100644
--- a/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java
+++ b/core/src/test/java/org/elasticsearch/rest/RestControllerTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.rest;
 
 import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.ESTestCase;
@@ -35,6 +36,11 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doCallRealMethod;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
 
 public class RestControllerTests extends ESTestCase {
 
@@ -110,6 +116,26 @@ public class RestControllerTests extends ESTestCase {
         assertFalse(controller.canTripCircuitBreaker(new FakeRestRequest.Builder().withPath("/do-not-trip").build()));
     }
 
+    public void testRegisterHandlerAsDeprecationHandler() {
+        RestController controller = mock(RestController.class);
+
+        RestRequest.Method method = randomFrom(RestRequest.Method.values());
+        String path = "/_" + randomAsciiOfLengthBetween(1, 6);
+        RestHandler handler = mock(RestHandler.class);
+        String deprecationMessage = randomAsciiOfLengthBetween(1, 10);
+        DeprecationLogger logger = mock(DeprecationLogger.class);
+
+        // don't want to test everything -- just that it actually wraps the handler
+        doCallRealMethod().when(controller).registerAsDeprecatedHandler(method, path, handler, deprecationMessage, logger);
+
+        controller.registerAsDeprecatedHandler(method, path, handler, deprecationMessage, logger);
+
+        verify(controller).registerHandler(eq(method), eq(path), any(DeprecationRestHandler.class));
+    }
+
+    /**
+     * Useful for testing with deprecation handler.
+     */
     private static class FakeRestHandler implements RestHandler {
         private final boolean canTripCircuitBreaker;
 
diff --git a/core/src/test/java/org/elasticsearch/rest/plugins/TestDeprecatedQueryBuilder.java b/core/src/test/java/org/elasticsearch/rest/plugins/TestDeprecatedQueryBuilder.java
new file mode 100644
index 00000000000..f3b02d1f6bc
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/rest/plugins/TestDeprecatedQueryBuilder.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.rest.plugins;
+
+import org.apache.lucene.search.Query;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.AbstractQueryBuilder;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.index.query.QueryShardContext;
+
+import java.io.IOException;
+import java.util.Optional;
+
+/**
+ * A query that performs a <code>match_all</code> query, but with each <em>index</em> touched getting a unique deprecation warning.
+ * <p>
+ * This makes it easy to test multiple unique responses for a single request.
+ */
+public class TestDeprecatedQueryBuilder extends AbstractQueryBuilder<TestDeprecatedQueryBuilder> {
+
+    public static final String NAME = "deprecated_match_all";
+    public static final ParseField QUERY_NAME_FIELD = new ParseField(NAME);
+
+    private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(TestDeprecatedQueryBuilder.class));
+
+    public TestDeprecatedQueryBuilder() {
+        // nothing to do
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public TestDeprecatedQueryBuilder(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    protected void doWriteTo(StreamOutput out) throws IOException {
+        // nothing to do
+    }
+
+    @Override
+    protected void doXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(NAME).endObject();
+    }
+
+    public static Optional<TestDeprecatedQueryBuilder> fromXContent(QueryParseContext parseContext) throws IOException, ParsingException {
+        XContentParser parser = parseContext.parser();
+
+        if (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+            throw new ParsingException(parser.getTokenLocation(), "[{}] query does not have any fields", NAME);
+        }
+
+        return Optional.of(new TestDeprecatedQueryBuilder());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    protected Query doToQuery(QueryShardContext context) throws IOException {
+        DEPRECATION_LOGGER.deprecated("[{}] query is deprecated, but used on [{}] index", NAME, context.index().getName());
+
+        return Queries.newMatchAllQuery();
+    }
+
+    @Override
+    public int doHashCode() {
+        return 0;
+    }
+
+    @Override
+    protected boolean doEquals(TestDeprecatedQueryBuilder other) {
+        return true;
+    }
+
+}
diff --git a/core/src/test/java/org/elasticsearch/rest/plugins/TestDeprecationHeaderRestAction.java b/core/src/test/java/org/elasticsearch/rest/plugins/TestDeprecationHeaderRestAction.java
new file mode 100644
index 00000000000..da1d165b6cf
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/rest/plugins/TestDeprecationHeaderRestAction.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.rest.plugins;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestStatus;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Enables testing {@code DeprecationRestHandler} via integration tests by guaranteeing a deprecated REST endpoint.
+ * <p>
+ * This adds an endpoint named <code>/_test_cluster/deprecated_settings</code> that touches specified settings given their names
+ * and returns their values.
+ */
+public class TestDeprecationHeaderRestAction extends BaseRestHandler {
+
+    public static final Setting<Boolean> TEST_DEPRECATED_SETTING_TRUE1 =
+        Setting.boolSetting("test.setting.deprecated.true1", true,
+                            Setting.Property.NodeScope, Setting.Property.Deprecated, Setting.Property.Dynamic);
+    public static final Setting<Boolean> TEST_DEPRECATED_SETTING_TRUE2 =
+        Setting.boolSetting("test.setting.deprecated.true2", true,
+                            Setting.Property.NodeScope, Setting.Property.Deprecated, Setting.Property.Dynamic);
+    public static final Setting<Boolean> TEST_NOT_DEPRECATED_SETTING =
+        Setting.boolSetting("test.setting.not_deprecated", false,
+                            Setting.Property.NodeScope, Setting.Property.Dynamic);
+
+    private static final Map<String, Setting<?>> SETTINGS;
+
+    static {
+        Map<String, Setting<?>> settingsMap = new HashMap<>(3);
+
+        settingsMap.put(TEST_DEPRECATED_SETTING_TRUE1.getKey(), TEST_DEPRECATED_SETTING_TRUE1);
+        settingsMap.put(TEST_DEPRECATED_SETTING_TRUE2.getKey(), TEST_DEPRECATED_SETTING_TRUE2);
+        settingsMap.put(TEST_NOT_DEPRECATED_SETTING.getKey(), TEST_NOT_DEPRECATED_SETTING);
+
+        SETTINGS = Collections.unmodifiableMap(settingsMap);
+    }
+
+    public static final String DEPRECATED_ENDPOINT = "[/_test_cluster/deprecated_settings] exists for deprecated tests";
+    public static final String DEPRECATED_USAGE = "[deprecated_settings] usage is deprecated. use [settings] instead";
+
+    @Inject
+    public TestDeprecationHeaderRestAction(Settings settings, RestController controller) {
+        super(settings);
+
+        controller.registerAsDeprecatedHandler(RestRequest.Method.GET, "/_test_cluster/deprecated_settings", this,
+                                               DEPRECATED_ENDPOINT, deprecationLogger);
+    }
+
+    @SuppressWarnings("unchecked") // List<String> casts
+    @Override
+    public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
+        final List<String> settings;
+
+        try (XContentParser parser = XContentFactory.xContent(request.content()).createParser(request.content())) {
+            final Map<String, Object> source = parser.map();
+
+            if (source.containsKey("deprecated_settings")) {
+                deprecationLogger.deprecated(DEPRECATED_USAGE);
+
+                settings = (List<String>)source.get("deprecated_settings");
+            } else {
+                settings = (List<String>)source.get("settings");
+            }
+        }
+
+        final XContentBuilder builder = channel.newBuilder();
+
+        builder.startObject().startArray("settings");
+        for (String setting : settings) {
+            builder.startObject().field(setting, SETTINGS.get(setting).getRaw(this.settings)).endObject();
+        }
+        builder.endArray().endObject();
+
+        channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder));
+    }
+}
diff --git a/core/src/test/java/org/elasticsearch/rest/plugins/TestDeprecationPlugin.java b/core/src/test/java/org/elasticsearch/rest/plugins/TestDeprecationPlugin.java
new file mode 100644
index 00000000000..c85674a1962
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/rest/plugins/TestDeprecationPlugin.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.rest.plugins;
+
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestHandler;
+import org.elasticsearch.search.SearchModule;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Adds {@link TestDeprecationHeaderRestAction} for testing deprecation requests via HTTP.
+ */
+public class TestDeprecationPlugin extends Plugin implements ActionPlugin {
+
+    @Override
+    public List<Class<? extends RestHandler>> getRestHandlers() {
+        return Collections.singletonList(TestDeprecationHeaderRestAction.class);
+    }
+
+    @Override
+    public List<Setting<?>> getSettings() {
+        return Arrays.asList(
+            TestDeprecationHeaderRestAction.TEST_DEPRECATED_SETTING_TRUE1,
+            TestDeprecationHeaderRestAction.TEST_DEPRECATED_SETTING_TRUE2,
+            TestDeprecationHeaderRestAction.TEST_NOT_DEPRECATED_SETTING);
+    }
+
+    public void onModule(SearchModule module) {
+        module.registerQuery(TestDeprecatedQueryBuilder::new,
+                             TestDeprecatedQueryBuilder::fromXContent,
+                             TestDeprecatedQueryBuilder.QUERY_NAME_FIELD);
+    }
+
+}

From dd7be74bcf12cd66621acc2bc86336bd694ca738 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Tue, 5 Jul 2016 22:12:24 -0700
Subject: [PATCH 1260/1311] Plugins: Simplified repository api for
 snapshot/restore

The api for snapshot/restore was split up between two interfaces,
Repository and IndexShardRepository. There was also complex
initialization and injection between the two. However, there is always a
one to one relationship between the two.

This change moves the IndexShardRepository api into Repository, as well
as updates the API so as not to require any services to be injected for
sublcasses.
---
 .../elasticsearch/index/shard/IndexShard.java |   8 +-
 .../index/shard/StoreRecovery.java            |  10 +-
 .../index/snapshots/IndexShardRepository.java |  81 --
 .../BlobStoreIndexShardRepository.java        | 972 ------------------
 .../repositories/RepositoriesModule.java      |  14 +-
 .../repositories/RepositoriesService.java     |  32 +-
 .../repositories/Repository.java              |  60 +-
 .../repositories/RepositoryModule.java        |  61 --
 .../repositories/RepositoryNameModule.java    |  39 -
 .../repositories/RepositoryTypesRegistry.java |   9 +-
 .../VerifyNodeRepositoryAction.java           |  24 +-
 .../blobstore/BlobStoreRepository.java        | 905 ++++++++++++++--
 .../repositories/fs/FsRepository.java         |   6 +-
 .../uri/URLIndexShardRepository.java          |  43 -
 .../repositories/uri/URLRepository.java       |   9 +-
 .../snapshots/RestoreService.java             |   3 +-
 .../snapshots/SnapshotShardsService.java      |   8 +-
 .../snapshots/SnapshotsService.java           |   4 +-
 .../index/shard/IndexShardTests.java          | 124 ++-
 .../snapshots/RepositoriesIT.java             |   2 -
 .../snapshots/mockstore/MockRepository.java   |  49 +-
 .../azure/AzureRepositoryPlugin.java          |  13 +-
 .../repositories/azure/AzureRepository.java   |   4 +-
 .../gcs/GoogleCloudStoragePlugin.java         |  14 +-
 .../gcs/GoogleCloudStorageRepository.java     |   4 +-
 .../repositories/hdfs/HdfsPlugin.java         |   3 +-
 .../repositories/hdfs/HdfsRepository.java     |   5 +-
 .../repository/s3/S3RepositoryPlugin.java     |  24 +-
 .../repositories/s3/S3Repository.java         |   6 +-
 .../cloud/aws/RepositoryS3SettingsTests.java  |   2 +-
 30 files changed, 1087 insertions(+), 1451 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/index/snapshots/IndexShardRepository.java
 delete mode 100644 core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java
 delete mode 100644 core/src/main/java/org/elasticsearch/repositories/RepositoryModule.java
 delete mode 100644 core/src/main/java/org/elasticsearch/repositories/RepositoryNameModule.java
 delete mode 100644 core/src/main/java/org/elasticsearch/repositories/uri/URLIndexShardRepository.java

diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index ab2fe683e81..a97aad7abfa 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -94,7 +94,6 @@ import org.elasticsearch.index.refresh.RefreshStats;
 import org.elasticsearch.index.search.stats.SearchStats;
 import org.elasticsearch.index.search.stats.ShardSearchStats;
 import org.elasticsearch.index.similarity.SimilarityService;
-import org.elasticsearch.index.snapshots.IndexShardRepository;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.store.Store.MetadataSnapshot;
 import org.elasticsearch.index.store.StoreFileMetaData;
@@ -111,6 +110,7 @@ import org.elasticsearch.indices.recovery.RecoveryFailedException;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.indices.recovery.RecoveryTargetService;
 import org.elasticsearch.repositories.RepositoriesService;
+import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.search.suggest.completion.CompletionFieldStats;
 import org.elasticsearch.search.suggest.completion.CompletionStats;
 import org.elasticsearch.search.suggest.completion2x.Completion090PostingsFormat;
@@ -1161,7 +1161,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         return storeRecovery.recoverFromStore(this, shouldExist);
     }
 
-    public boolean restoreFromRepository(IndexShardRepository repository) {
+    public boolean restoreFromRepository(Repository repository) {
         assert shardRouting.primary() : "recover from store only makes sense if the shard is a primary shard";
         StoreRecovery storeRecovery = new StoreRecovery(shardId, logger);
         return storeRecovery.recoverFromRepository(this, repository);
@@ -1449,9 +1449,9 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
                 markAsRecovering("from snapshot", recoveryState); // mark the shard as recovering on the cluster state thread
                 threadPool.generic().execute(() -> {
                     try {
-                        final IndexShardRepository indexShardRepository = repositoriesService.indexShardRepository(
+                        final Repository repository = repositoriesService.repository(
                             recoveryState.getRestoreSource().snapshot().getRepository());
-                        if (restoreFromRepository(indexShardRepository)) {
+                        if (restoreFromRepository(repository)) {
                             recoveryListener.onRecoveryDone(recoveryState);
                         }
                     } catch (Exception first) {
diff --git a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
index 01513aa6bc6..c1016f0f4e3 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
@@ -40,10 +40,10 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.engine.EngineException;
 import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.snapshots.IndexShardRepository;
 import org.elasticsearch.index.snapshots.IndexShardRestoreFailedException;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.indices.recovery.RecoveryState;
+import org.elasticsearch.repositories.Repository;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -219,14 +219,14 @@ final class StoreRecovery {
     }
 
     /**
-     * Recovers an index from a given {@link IndexShardRepository}. This method restores a
+     * Recovers an index from a given {@link Repository}. This method restores a
      * previously created index snapshot into an existing initializing shard.
      * @param indexShard the index shard instance to recovery the snapshot from
      * @param repository the repository holding the physical files the shard should be recovered from
      * @return <code>true</code> if the shard has been recovered successfully, <code>false</code> if the recovery
      * has been ignored due to a concurrent modification of if the clusters state has changed due to async updates.
      */
-    boolean recoverFromRepository(final IndexShard indexShard, IndexShardRepository repository) {
+    boolean recoverFromRepository(final IndexShard indexShard, Repository repository) {
         if (canRecover(indexShard)) {
             final ShardRouting shardRouting = indexShard.routingEntry();
             if (shardRouting.restoreSource() == null) {
@@ -380,7 +380,7 @@ final class StoreRecovery {
     /**
      * Restores shard from {@link RestoreSource} associated with this shard in routing table
      */
-    private void restore(final IndexShard indexShard, final IndexShardRepository indexShardRepository) {
+    private void restore(final IndexShard indexShard, final Repository repository) {
         RestoreSource restoreSource = indexShard.routingEntry().restoreSource();
         final RecoveryState.Translog translogState = indexShard.recoveryState().getTranslog();
         if (restoreSource == null) {
@@ -397,7 +397,7 @@ final class StoreRecovery {
             if (!shardId.getIndexName().equals(restoreSource.index())) {
                 snapshotShardId = new ShardId(restoreSource.index(), IndexMetaData.INDEX_UUID_NA_VALUE, shardId.id());
             }
-            indexShardRepository.restore(restoreSource.snapshot().getSnapshotId(), restoreSource.version(), shardId, snapshotShardId, indexShard.recoveryState());
+            repository.restore(indexShard, restoreSource.snapshot().getSnapshotId(), restoreSource.version(), snapshotShardId, indexShard.recoveryState());
             indexShard.skipTranslogRecovery();
             indexShard.finalizeRecovery();
             indexShard.postRecovery("restore done");
diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardRepository.java b/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardRepository.java
deleted file mode 100644
index 5988d82def2..00000000000
--- a/core/src/main/java/org/elasticsearch/index/snapshots/IndexShardRepository.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.snapshots;
-
-import org.apache.lucene.index.IndexCommit;
-import org.elasticsearch.Version;
-import org.elasticsearch.snapshots.SnapshotId;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.indices.recovery.RecoveryState;
-
-/**
- * Shard-level snapshot repository
- * <p>
- * IndexShardRepository is used on data node to create snapshots of individual shards. See {@link org.elasticsearch.repositories.Repository}
- * for more information.
- */
-public interface IndexShardRepository {
-
-    /**
-     * Creates a snapshot of the shard based on the index commit point.
-     * <p>
-     * The index commit point can be obtained by using {@link org.elasticsearch.index.engine.Engine#snapshotIndex} method.
-     * IndexShardRepository implementations shouldn't release the snapshot index commit point. It is done by the method caller.
-     * <p>
-     * As snapshot process progresses, implementation of this method should update {@link IndexShardSnapshotStatus} object and check
-     * {@link IndexShardSnapshotStatus#aborted()} to see if the snapshot process should be aborted.
-     *
-     * @param snapshotId          snapshot id
-     * @param shardId             shard to be snapshotted
-     * @param snapshotIndexCommit commit point
-     * @param snapshotStatus      snapshot status
-     */
-    void snapshot(SnapshotId snapshotId, ShardId shardId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus);
-
-    /**
-     * Restores snapshot of the shard.
-     * <p>
-     * The index can be renamed on restore, hence different {@code shardId} and {@code snapshotShardId} are supplied.
-     *
-     * @param snapshotId      snapshot id
-     * @param shardId         shard id (in the current index)
-     * @param version   version of elasticsearch that created this snapshot
-     * @param snapshotShardId shard id (in the snapshot)
-     * @param recoveryState   recovery state
-     */
-    void restore(SnapshotId snapshotId, Version version, ShardId shardId, ShardId snapshotShardId, RecoveryState recoveryState);
-
-    /**
-     * Retrieve shard snapshot status for the stored snapshot
-     *
-     * @param snapshotId snapshot id
-     * @param version   version of elasticsearch that created this snapshot
-     * @param shardId    shard id
-     * @return snapshot status
-     */
-    IndexShardSnapshotStatus snapshotStatus(SnapshotId snapshotId, Version version, ShardId shardId);
-
-    /**
-     * Verifies repository settings on data node
-     * @param verificationToken value returned by {@link org.elasticsearch.repositories.Repository#startVerification()}
-     */
-    void verify(String verificationToken);
-
-}
diff --git a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java b/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java
deleted file mode 100644
index 526aab7581b..00000000000
--- a/core/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardRepository.java
+++ /dev/null
@@ -1,972 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.snapshots.blobstore;
-
-import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.IndexCommit;
-import org.apache.lucene.index.IndexFormatTooNewException;
-import org.apache.lucene.index.IndexFormatTooOldException;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.SegmentInfos;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.RateLimiter;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
-import org.elasticsearch.ExceptionsHelper;
-import org.elasticsearch.Version;
-import org.elasticsearch.snapshots.SnapshotId;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.ParseFieldMatcher;
-import org.elasticsearch.common.blobstore.BlobContainer;
-import org.elasticsearch.common.blobstore.BlobMetaData;
-import org.elasticsearch.common.blobstore.BlobPath;
-import org.elasticsearch.common.blobstore.BlobStore;
-import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.collect.Tuple;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.common.lucene.store.InputStreamIndexInput;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.util.iterable.Iterables;
-import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.index.snapshots.IndexShardRepository;
-import org.elasticsearch.index.snapshots.IndexShardRestoreFailedException;
-import org.elasticsearch.index.snapshots.IndexShardSnapshotException;
-import org.elasticsearch.index.snapshots.IndexShardSnapshotFailedException;
-import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
-import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo;
-import org.elasticsearch.index.store.Store;
-import org.elasticsearch.index.store.StoreFileMetaData;
-import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.indices.recovery.RecoveryState;
-import org.elasticsearch.repositories.RepositoryName;
-import org.elasticsearch.repositories.RepositoryVerificationException;
-import org.elasticsearch.repositories.blobstore.BlobStoreFormat;
-import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
-import org.elasticsearch.repositories.blobstore.ChecksumBlobStoreFormat;
-import org.elasticsearch.repositories.blobstore.LegacyBlobStoreFormat;
-
-import java.io.FilterInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.unmodifiableMap;
-import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.testBlobPrefix;
-
-/**
- * Blob store based implementation of IndexShardRepository
- */
-public class BlobStoreIndexShardRepository extends AbstractComponent implements IndexShardRepository {
-
-    private static final int BUFFER_SIZE = 4096;
-    private BlobStore blobStore;
-
-    private BlobPath basePath;
-
-    private final String repositoryName;
-
-    private ByteSizeValue chunkSize;
-
-    private final IndicesService indicesService;
-
-    private final ClusterService clusterService;
-
-    private RateLimiter snapshotRateLimiter;
-
-    private RateLimiter restoreRateLimiter;
-
-    private RateLimitingInputStream.Listener snapshotThrottleListener;
-
-    private RateLimitingInputStream.Listener restoreThrottleListener;
-
-    private boolean compress;
-
-    private final ParseFieldMatcher parseFieldMatcher;
-
-    protected static final String LEGACY_SNAPSHOT_PREFIX = "snapshot-";
-
-    protected static final String LEGACY_SNAPSHOT_NAME_FORMAT = LEGACY_SNAPSHOT_PREFIX + "%s";
-
-    protected static final String SNAPSHOT_PREFIX = "snap-";
-
-    protected static final String SNAPSHOT_NAME_FORMAT = SNAPSHOT_PREFIX + "%s.dat";
-
-    protected static final String SNAPSHOT_CODEC = "snapshot";
-
-    protected static final String SNAPSHOT_INDEX_PREFIX = "index-";
-
-    protected static final String SNAPSHOT_INDEX_NAME_FORMAT = SNAPSHOT_INDEX_PREFIX + "%s";
-
-    protected static final String SNAPSHOT_INDEX_CODEC = "snapshots";
-
-    protected static final String DATA_BLOB_PREFIX = "__";
-
-    private ChecksumBlobStoreFormat<BlobStoreIndexShardSnapshot> indexShardSnapshotFormat;
-
-    private LegacyBlobStoreFormat<BlobStoreIndexShardSnapshot> indexShardSnapshotLegacyFormat;
-
-    private ChecksumBlobStoreFormat<BlobStoreIndexShardSnapshots> indexShardSnapshotsFormat;
-
-    @Inject
-    public BlobStoreIndexShardRepository(Settings settings, RepositoryName repositoryName, IndicesService indicesService, ClusterService clusterService) {
-        super(settings);
-        this.parseFieldMatcher = new ParseFieldMatcher(settings);
-        this.repositoryName = repositoryName.name();
-        this.indicesService = indicesService;
-        this.clusterService = clusterService;
-    }
-
-    /**
-     * Called by {@link org.elasticsearch.repositories.blobstore.BlobStoreRepository} on repository startup
-     *
-     * @param blobStore blob store
-     * @param basePath  base path to blob store
-     * @param chunkSize chunk size
-     */
-    public void initialize(BlobStore blobStore, BlobPath basePath, ByteSizeValue chunkSize,
-                           RateLimiter snapshotRateLimiter, RateLimiter restoreRateLimiter,
-                           final RateLimiterListener rateLimiterListener, boolean compress) {
-        this.blobStore = blobStore;
-        this.basePath = basePath;
-        this.chunkSize = chunkSize;
-        this.snapshotRateLimiter = snapshotRateLimiter;
-        this.restoreRateLimiter = restoreRateLimiter;
-        this.snapshotThrottleListener = nanos -> rateLimiterListener.onSnapshotPause(nanos);
-        this.restoreThrottleListener = nanos -> rateLimiterListener.onRestorePause(nanos);
-        this.compress = compress;
-        indexShardSnapshotFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_CODEC, SNAPSHOT_NAME_FORMAT, BlobStoreIndexShardSnapshot.PROTO, parseFieldMatcher, isCompress());
-        indexShardSnapshotLegacyFormat = new LegacyBlobStoreFormat<>(LEGACY_SNAPSHOT_NAME_FORMAT, BlobStoreIndexShardSnapshot.PROTO, parseFieldMatcher);
-        indexShardSnapshotsFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_INDEX_CODEC, SNAPSHOT_INDEX_NAME_FORMAT, BlobStoreIndexShardSnapshots.PROTO, parseFieldMatcher, isCompress());
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public void snapshot(SnapshotId snapshotId, ShardId shardId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus) {
-        SnapshotContext snapshotContext = new SnapshotContext(snapshotId, shardId, snapshotStatus);
-        snapshotStatus.startTime(System.currentTimeMillis());
-
-        try {
-            snapshotContext.snapshot(snapshotIndexCommit);
-            snapshotStatus.time(System.currentTimeMillis() - snapshotStatus.startTime());
-            snapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.DONE);
-        } catch (Exception e) {
-            snapshotStatus.time(System.currentTimeMillis() - snapshotStatus.startTime());
-            snapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.FAILURE);
-            snapshotStatus.failure(ExceptionsHelper.detailedMessage(e));
-            if (e instanceof IndexShardSnapshotFailedException) {
-                throw (IndexShardSnapshotFailedException) e;
-            } else {
-                throw new IndexShardSnapshotFailedException(shardId, e);
-            }
-        }
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public void restore(SnapshotId snapshotId, Version version, ShardId shardId, ShardId snapshotShardId, RecoveryState recoveryState) {
-        final RestoreContext snapshotContext = new RestoreContext(snapshotId, version, shardId, snapshotShardId, recoveryState);
-        try {
-            snapshotContext.restore();
-        } catch (Exception e) {
-            throw new IndexShardRestoreFailedException(shardId, "failed to restore snapshot [" + snapshotId + "]", e);
-        }
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    public IndexShardSnapshotStatus snapshotStatus(SnapshotId snapshotId, Version version, ShardId shardId) {
-        Context context = new Context(snapshotId, version, shardId);
-        BlobStoreIndexShardSnapshot snapshot = context.loadSnapshot();
-        IndexShardSnapshotStatus status = new IndexShardSnapshotStatus();
-        status.updateStage(IndexShardSnapshotStatus.Stage.DONE);
-        status.startTime(snapshot.startTime());
-        status.files(snapshot.numberOfFiles(), snapshot.totalSize());
-        // The snapshot is done which means the number of processed files is the same as total
-        status.processedFiles(snapshot.numberOfFiles(), snapshot.totalSize());
-        status.time(snapshot.time());
-        return status;
-    }
-
-    @Override
-    public void verify(String seed) {
-        BlobContainer testBlobContainer = blobStore.blobContainer(basePath.add(testBlobPrefix(seed)));
-        DiscoveryNode localNode = clusterService.localNode();
-        if (testBlobContainer.blobExists("master.dat")) {
-            try  {
-                testBlobContainer.writeBlob("data-" + localNode.getId() + ".dat", new BytesArray(seed));
-            } catch (IOException exp) {
-                throw new RepositoryVerificationException(repositoryName, "store location [" + blobStore + "] is not accessible on the node [" + localNode + "]", exp);
-            }
-        } else {
-            throw new RepositoryVerificationException(repositoryName, "a file written by master to the store [" + blobStore + "] cannot be accessed on the node [" + localNode + "]. "
-                    + "This might indicate that the store [" + blobStore + "] is not shared between this node and the master node or "
-                    + "that permissions on the store don't allow reading files written by the master node");
-        }
-    }
-
-    /**
-     * Delete shard snapshot
-     *
-     * @param snapshotId snapshot id
-     * @param shardId    shard id
-     */
-    public void delete(SnapshotId snapshotId, Version version, ShardId shardId) {
-        Context context = new Context(snapshotId, version, shardId, shardId);
-        context.delete();
-    }
-
-    @Override
-    public String toString() {
-        return "BlobStoreIndexShardRepository[" +
-                "[" + repositoryName +
-                "], [" + blobStore + ']' +
-                ']';
-    }
-
-    /**
-     * Returns true if metadata files should be compressed
-     *
-     * @return true if compression is needed
-     */
-    protected boolean isCompress() {
-        return compress;
-    }
-
-    BlobStoreFormat<BlobStoreIndexShardSnapshot> indexShardSnapshotFormat(Version version) {
-        if (BlobStoreRepository.legacyMetaData(version)) {
-            return indexShardSnapshotLegacyFormat;
-        } else {
-            return indexShardSnapshotFormat;
-        }
-    }
-
-    /**
-     * Context for snapshot/restore operations
-     */
-    private class Context {
-
-        protected final SnapshotId snapshotId;
-
-        protected final ShardId shardId;
-
-        protected final BlobContainer blobContainer;
-
-        protected final Version version;
-
-        public Context(SnapshotId snapshotId, Version version, ShardId shardId) {
-            this(snapshotId, version, shardId, shardId);
-        }
-
-        public Context(SnapshotId snapshotId, Version version, ShardId shardId, ShardId snapshotShardId) {
-            this.snapshotId = snapshotId;
-            this.version = version;
-            this.shardId = shardId;
-            blobContainer = blobStore.blobContainer(basePath.add("indices").add(snapshotShardId.getIndexName()).add(Integer.toString(snapshotShardId.getId())));
-        }
-
-        /**
-         * Delete shard snapshot
-         */
-        public void delete() {
-            final Map<String, BlobMetaData> blobs;
-            try {
-                blobs = blobContainer.listBlobs();
-            } catch (IOException e) {
-                throw new IndexShardSnapshotException(shardId, "Failed to list content of gateway", e);
-            }
-
-            Tuple<BlobStoreIndexShardSnapshots, Integer> tuple = buildBlobStoreIndexShardSnapshots(blobs);
-            BlobStoreIndexShardSnapshots snapshots = tuple.v1();
-            int fileListGeneration = tuple.v2();
-
-            try {
-                indexShardSnapshotFormat(version).delete(blobContainer, snapshotId.getName());
-            } catch (IOException e) {
-                logger.debug("[{}] [{}] failed to delete shard snapshot file", shardId, snapshotId);
-            }
-
-            // Build a list of snapshots that should be preserved
-            List<SnapshotFiles> newSnapshotsList = new ArrayList<>();
-            for (SnapshotFiles point : snapshots) {
-                if (!point.snapshot().equals(snapshotId.getName())) {
-                    newSnapshotsList.add(point);
-                }
-            }
-            // finalize the snapshot and rewrite the snapshot index with the next sequential snapshot index
-            finalize(newSnapshotsList, fileListGeneration + 1, blobs);
-        }
-
-        /**
-         * Loads information about shard snapshot
-         */
-        public BlobStoreIndexShardSnapshot loadSnapshot() {
-            try {
-                return indexShardSnapshotFormat(version).read(blobContainer, snapshotId.getName());
-            } catch (IOException ex) {
-                throw new IndexShardRestoreFailedException(shardId, "failed to read shard snapshot file", ex);
-            }
-        }
-
-        /**
-         * Removes all unreferenced files from the repository and writes new index file
-         *
-         * We need to be really careful in handling index files in case of failures to make sure we have index file that
-         * points to files that were deleted.
-         *
-         *
-         * @param snapshots list of active snapshots in the container
-         * @param fileListGeneration the generation number of the snapshot index file
-         * @param blobs     list of blobs in the container
-         */
-        protected void finalize(List<SnapshotFiles> snapshots, int fileListGeneration, Map<String, BlobMetaData> blobs) {
-            BlobStoreIndexShardSnapshots newSnapshots = new BlobStoreIndexShardSnapshots(snapshots);
-            List<String> blobsToDelete = new ArrayList<>();
-            // delete old index files first
-            for (String blobName : blobs.keySet()) {
-                // delete old file lists
-                if (indexShardSnapshotsFormat.isTempBlobName(blobName) || blobName.startsWith(SNAPSHOT_INDEX_PREFIX)) {
-                    blobsToDelete.add(blobName);
-                }
-            }
-
-            try {
-                blobContainer.deleteBlobs(blobsToDelete);
-            } catch (IOException e) {
-                // We cannot delete index file - this is fatal, we cannot continue, otherwise we might end up
-                // with references to non-existing files
-                throw new IndexShardSnapshotFailedException(shardId, "error deleting index files during cleanup", e);
-            }
-
-            blobsToDelete = new ArrayList<>();
-            // now go over all the blobs, and if they don't exists in a snapshot, delete them
-            for (String blobName : blobs.keySet()) {
-                // delete unused files
-                if (blobName.startsWith(DATA_BLOB_PREFIX)) {
-                    if (newSnapshots.findNameFile(FileInfo.canonicalName(blobName)) == null) {
-                        blobsToDelete.add(blobName);
-                    }
-                }
-            }
-            try {
-                blobContainer.deleteBlobs(blobsToDelete);
-            } catch (IOException e) {
-                logger.debug("[{}] [{}] error deleting some of the blobs [{}] during cleanup", e, snapshotId, shardId, blobsToDelete);
-            }
-
-            // If we deleted all snapshots - we don't need to create the index file
-            if (snapshots.size() > 0) {
-                try {
-                    indexShardSnapshotsFormat.writeAtomic(newSnapshots, blobContainer, Integer.toString(fileListGeneration));
-                } catch (IOException e) {
-                    throw new IndexShardSnapshotFailedException(shardId, "Failed to write file list", e);
-                }
-            }
-        }
-
-        /**
-         * Generates blob name
-         *
-         * @param generation the blob number
-         * @return the blob name
-         */
-        protected String fileNameFromGeneration(long generation) {
-            return DATA_BLOB_PREFIX + Long.toString(generation, Character.MAX_RADIX);
-        }
-
-        /**
-         * Finds the next available blob number
-         *
-         * @param blobs list of blobs in the repository
-         * @return next available blob number
-         */
-        protected long findLatestFileNameGeneration(Map<String, BlobMetaData> blobs) {
-            long generation = -1;
-            for (String name : blobs.keySet()) {
-                if (!name.startsWith(DATA_BLOB_PREFIX)) {
-                    continue;
-                }
-                name = FileInfo.canonicalName(name);
-                try {
-                    long currentGen = Long.parseLong(name.substring(DATA_BLOB_PREFIX.length()), Character.MAX_RADIX);
-                    if (currentGen > generation) {
-                        generation = currentGen;
-                    }
-                } catch (NumberFormatException e) {
-                    logger.warn("file [{}] does not conform to the '{}' schema", name, DATA_BLOB_PREFIX);
-                }
-            }
-            return generation;
-        }
-
-        /**
-         * Loads all available snapshots in the repository
-         *
-         * @param blobs list of blobs in repository
-         * @return tuple of BlobStoreIndexShardSnapshots and the last snapshot index generation
-         */
-        protected Tuple<BlobStoreIndexShardSnapshots, Integer> buildBlobStoreIndexShardSnapshots(Map<String, BlobMetaData> blobs) {
-            int latest = -1;
-            for (String name : blobs.keySet()) {
-                if (name.startsWith(SNAPSHOT_INDEX_PREFIX)) {
-                    try {
-                        int gen = Integer.parseInt(name.substring(SNAPSHOT_INDEX_PREFIX.length()));
-                        if (gen > latest) {
-                            latest = gen;
-                        }
-                    } catch (NumberFormatException ex) {
-                        logger.warn("failed to parse index file name [{}]", name);
-                    }
-                }
-            }
-            if (latest >= 0) {
-                try {
-                    final BlobStoreIndexShardSnapshots shardSnapshots =
-                        indexShardSnapshotsFormat.read(blobContainer, Integer.toString(latest));
-                    return new Tuple<>(shardSnapshots, latest);
-                } catch (IOException e) {
-                    logger.warn("failed to read index file  [{}]", e, SNAPSHOT_INDEX_PREFIX + latest);
-                }
-            }
-
-            // We couldn't load the index file - falling back to loading individual snapshots
-            List<SnapshotFiles> snapshots = new ArrayList<>();
-            for (String name : blobs.keySet()) {
-                try {
-                    BlobStoreIndexShardSnapshot snapshot = null;
-                    if (name.startsWith(SNAPSHOT_PREFIX)) {
-                        snapshot = indexShardSnapshotFormat.readBlob(blobContainer, name);
-                    } else if (name.startsWith(LEGACY_SNAPSHOT_PREFIX)) {
-                        snapshot = indexShardSnapshotLegacyFormat.readBlob(blobContainer, name);
-                    }
-                    if (snapshot != null) {
-                        snapshots.add(new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles()));
-                    }
-                } catch (IOException e) {
-                    logger.warn("failed to read commit point [{}]", e, name);
-                }
-            }
-            return new Tuple<>(new BlobStoreIndexShardSnapshots(snapshots), -1);
-        }
-    }
-
-    /**
-     * Context for snapshot operations
-     */
-    private class SnapshotContext extends Context {
-
-        private final Store store;
-
-        private final IndexShardSnapshotStatus snapshotStatus;
-
-        /**
-         * Constructs new context
-         *
-         * @param snapshotId     snapshot id
-         * @param shardId        shard to be snapshotted
-         * @param snapshotStatus snapshot status to report progress
-         */
-        public SnapshotContext(SnapshotId snapshotId, ShardId shardId, IndexShardSnapshotStatus snapshotStatus) {
-            super(snapshotId, Version.CURRENT, shardId);
-            this.snapshotStatus = snapshotStatus;
-            store = indicesService.indexServiceSafe(shardId.getIndex()).getShardOrNull(shardId.id()).store();
-        }
-
-        /**
-         * Create snapshot from index commit point
-         *
-         * @param snapshotIndexCommit snapshot commit point
-         */
-        public void snapshot(IndexCommit snapshotIndexCommit) {
-            logger.debug("[{}] [{}] snapshot to [{}] ...", shardId, snapshotId, repositoryName);
-            store.incRef();
-            try {
-                final Map<String, BlobMetaData> blobs;
-                try {
-                    blobs = blobContainer.listBlobs();
-                } catch (IOException e) {
-                    throw new IndexShardSnapshotFailedException(shardId, "failed to list blobs", e);
-                }
-
-                long generation = findLatestFileNameGeneration(blobs);
-                Tuple<BlobStoreIndexShardSnapshots, Integer> tuple = buildBlobStoreIndexShardSnapshots(blobs);
-                BlobStoreIndexShardSnapshots snapshots = tuple.v1();
-                int fileListGeneration = tuple.v2();
-
-                final List<BlobStoreIndexShardSnapshot.FileInfo> indexCommitPointFiles = new ArrayList<>();
-
-                int indexNumberOfFiles = 0;
-                long indexTotalFilesSize = 0;
-                ArrayList<FileInfo> filesToSnapshot = new ArrayList<>();
-                final Store.MetadataSnapshot metadata;
-                // TODO apparently we don't use the MetadataSnapshot#.recoveryDiff(...) here but we should
-                final Collection<String> fileNames;
-                try {
-                    metadata = store.getMetadata(snapshotIndexCommit);
-                    fileNames = snapshotIndexCommit.getFileNames();
-                } catch (IOException e) {
-                    throw new IndexShardSnapshotFailedException(shardId, "Failed to get store file metadata", e);
-                }
-                for (String fileName : fileNames) {
-                    if (snapshotStatus.aborted()) {
-                        logger.debug("[{}] [{}] Aborted on the file [{}], exiting", shardId, snapshotId, fileName);
-                        throw new IndexShardSnapshotFailedException(shardId, "Aborted");
-                    }
-                    logger.trace("[{}] [{}] Processing [{}]", shardId, snapshotId, fileName);
-                    final StoreFileMetaData md = metadata.get(fileName);
-                    FileInfo existingFileInfo = null;
-                    List<FileInfo> filesInfo = snapshots.findPhysicalIndexFiles(fileName);
-                    if (filesInfo != null) {
-                        for (FileInfo fileInfo : filesInfo) {
-                            try {
-                                // in 1.3.3 we added additional hashes for .si / segments_N files
-                                // to ensure we don't double the space in the repo since old snapshots
-                                // don't have this hash we try to read that hash from the blob store
-                                // in a bwc compatible way.
-                                maybeRecalculateMetadataHash(blobContainer, fileInfo, metadata);
-                            } catch (Exception e) {
-                                logger.warn("{} Can't calculate hash from blob for file [{}] [{}]", e, shardId, fileInfo.physicalName(), fileInfo.metadata());
-                            }
-                            if (fileInfo.isSame(md) && snapshotFileExistsInBlobs(fileInfo, blobs)) {
-                                // a commit point file with the same name, size and checksum was already copied to repository
-                                // we will reuse it for this snapshot
-                                existingFileInfo = fileInfo;
-                                break;
-                            }
-                        }
-                    }
-                    if (existingFileInfo == null) {
-                        indexNumberOfFiles++;
-                        indexTotalFilesSize += md.length();
-                        // create a new FileInfo
-                        BlobStoreIndexShardSnapshot.FileInfo snapshotFileInfo = new BlobStoreIndexShardSnapshot.FileInfo(fileNameFromGeneration(++generation), md, chunkSize);
-                        indexCommitPointFiles.add(snapshotFileInfo);
-                        filesToSnapshot.add(snapshotFileInfo);
-                    } else {
-                        indexCommitPointFiles.add(existingFileInfo);
-                    }
-                }
-
-                snapshotStatus.files(indexNumberOfFiles, indexTotalFilesSize);
-
-                if (snapshotStatus.aborted()) {
-                    logger.debug("[{}] [{}] Aborted during initialization", shardId, snapshotId);
-                    throw new IndexShardSnapshotFailedException(shardId, "Aborted");
-                }
-
-                snapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.STARTED);
-
-                for (FileInfo snapshotFileInfo : filesToSnapshot) {
-                    try {
-                        snapshotFile(snapshotFileInfo);
-                    } catch (IOException e) {
-                        throw new IndexShardSnapshotFailedException(shardId, "Failed to perform snapshot (index files)", e);
-                    }
-                }
-
-                snapshotStatus.indexVersion(snapshotIndexCommit.getGeneration());
-                // now create and write the commit point
-                snapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.FINALIZE);
-
-                BlobStoreIndexShardSnapshot snapshot = new BlobStoreIndexShardSnapshot(snapshotId.getName(),
-                        snapshotIndexCommit.getGeneration(), indexCommitPointFiles, snapshotStatus.startTime(),
-                        // snapshotStatus.startTime() is assigned on the same machine, so it's safe to use with VLong
-                        System.currentTimeMillis() - snapshotStatus.startTime(), indexNumberOfFiles, indexTotalFilesSize);
-                //TODO: The time stored in snapshot doesn't include cleanup time.
-                logger.trace("[{}] [{}] writing shard snapshot file", shardId, snapshotId);
-                try {
-                    indexShardSnapshotFormat.write(snapshot, blobContainer, snapshotId.getName());
-                } catch (IOException e) {
-                    throw new IndexShardSnapshotFailedException(shardId, "Failed to write commit point", e);
-                }
-
-                // delete all files that are not referenced by any commit point
-                // build a new BlobStoreIndexShardSnapshot, that includes this one and all the saved ones
-                List<SnapshotFiles> newSnapshotsList = new ArrayList<>();
-                newSnapshotsList.add(new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles()));
-                for (SnapshotFiles point : snapshots) {
-                    newSnapshotsList.add(point);
-                }
-                // finalize the snapshot and rewrite the snapshot index with the next sequential snapshot index
-                finalize(newSnapshotsList, fileListGeneration + 1, blobs);
-                snapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.DONE);
-            } finally {
-                store.decRef();
-            }
-        }
-
-        /**
-         * Snapshot individual file
-         * <p>
-         * This is asynchronous method. Upon completion of the operation latch is getting counted down and any failures are
-         * added to the {@code failures} list
-         *
-         * @param fileInfo file to be snapshotted
-         */
-        private void snapshotFile(final BlobStoreIndexShardSnapshot.FileInfo fileInfo) throws IOException {
-            final String file = fileInfo.physicalName();
-            try (IndexInput indexInput = store.openVerifyingInput(file, IOContext.READONCE, fileInfo.metadata())) {
-                for (int i = 0; i < fileInfo.numberOfParts(); i++) {
-                    final long partBytes = fileInfo.partBytes(i);
-
-                    final InputStreamIndexInput inputStreamIndexInput = new InputStreamIndexInput(indexInput, partBytes);
-                    InputStream inputStream = snapshotRateLimiter == null ? inputStreamIndexInput : new RateLimitingInputStream(inputStreamIndexInput, snapshotRateLimiter, snapshotThrottleListener);
-                    inputStream = new AbortableInputStream(inputStream, fileInfo.physicalName());
-                    blobContainer.writeBlob(fileInfo.partName(i), inputStream, partBytes);
-                }
-                Store.verify(indexInput);
-                snapshotStatus.addProcessedFile(fileInfo.length());
-            } catch (Exception t) {
-                failStoreIfCorrupted(t);
-                snapshotStatus.addProcessedFile(0);
-                throw t;
-            }
-        }
-
-        private void failStoreIfCorrupted(Exception e) {
-            if (e instanceof CorruptIndexException || e instanceof IndexFormatTooOldException || e instanceof IndexFormatTooNewException) {
-                try {
-                    store.markStoreCorrupted((IOException) e);
-                } catch (IOException inner) {
-                    inner.addSuppressed(e);
-                    logger.warn("store cannot be marked as corrupted", inner);
-                }
-            }
-        }
-
-        /**
-         * Checks if snapshot file already exists in the list of blobs
-         *
-         * @param fileInfo file to check
-         * @param blobs    list of blobs
-         * @return true if file exists in the list of blobs
-         */
-        private boolean snapshotFileExistsInBlobs(BlobStoreIndexShardSnapshot.FileInfo fileInfo, Map<String, BlobMetaData> blobs) {
-            BlobMetaData blobMetaData = blobs.get(fileInfo.name());
-            if (blobMetaData != null) {
-                return blobMetaData.length() == fileInfo.length();
-            } else if (blobs.containsKey(fileInfo.partName(0))) {
-                // multi part file sum up the size and check
-                int part = 0;
-                long totalSize = 0;
-                while (true) {
-                    blobMetaData = blobs.get(fileInfo.partName(part++));
-                    if (blobMetaData == null) {
-                        break;
-                    }
-                    totalSize += blobMetaData.length();
-                }
-                return totalSize == fileInfo.length();
-            }
-            // no file, not exact and not multipart
-            return false;
-        }
-
-        private class AbortableInputStream extends FilterInputStream {
-            private final String fileName;
-
-            public AbortableInputStream(InputStream delegate, String fileName) {
-                super(delegate);
-                this.fileName = fileName;
-            }
-
-            @Override
-            public int read() throws IOException {
-                checkAborted();
-                return in.read();
-            }
-
-            @Override
-            public int read(byte[] b, int off, int len) throws IOException {
-                checkAborted();
-                return in.read(b, off, len);
-            }
-
-            private void checkAborted() {
-                if (snapshotStatus.aborted()) {
-                    logger.debug("[{}] [{}] Aborted on the file [{}], exiting", shardId, snapshotId, fileName);
-                    throw new IndexShardSnapshotFailedException(shardId, "Aborted");
-                }
-            }
-        }
-    }
-
-    /**
-     * This is a BWC layer to ensure we update the snapshots metadata with the corresponding hashes before we compare them.
-     * The new logic for StoreFileMetaData reads the entire <tt>.si</tt> and <tt>segments.n</tt> files to strengthen the
-     * comparison of the files on a per-segment / per-commit level.
-     */
-    private static void maybeRecalculateMetadataHash(final BlobContainer blobContainer, final FileInfo fileInfo, Store.MetadataSnapshot snapshot) throws Exception {
-        final StoreFileMetaData metadata;
-        if (fileInfo != null && (metadata = snapshot.get(fileInfo.physicalName())) != null) {
-            if (metadata.hash().length > 0 && fileInfo.metadata().hash().length == 0) {
-                // we have a hash - check if our repo has a hash too otherwise we have
-                // to calculate it.
-                // we might have multiple parts even though the file is small... make sure we read all of it.
-                try (final InputStream stream = new PartSliceStream(blobContainer, fileInfo)) {
-                    BytesRefBuilder builder = new BytesRefBuilder();
-                    Store.MetadataSnapshot.hashFile(builder, stream, fileInfo.length());
-                    BytesRef hash = fileInfo.metadata().hash(); // reset the file infos metadata hash
-                    assert hash.length == 0;
-                    hash.bytes = builder.bytes();
-                    hash.offset = 0;
-                    hash.length = builder.length();
-                }
-            }
-        }
-    }
-
-    private static final class PartSliceStream extends SlicedInputStream {
-
-        private final BlobContainer container;
-        private final FileInfo info;
-
-        public PartSliceStream(BlobContainer container, FileInfo info) {
-            super(info.numberOfParts());
-            this.info = info;
-            this.container = container;
-        }
-
-        @Override
-        protected InputStream openSlice(long slice) throws IOException {
-            return container.readBlob(info.partName(slice));
-        }
-    }
-
-    /**
-     * Context for restore operations
-     */
-    private class RestoreContext extends Context {
-
-        private final Store store;
-
-        private final RecoveryState recoveryState;
-
-        /**
-         * Constructs new restore context
-         *
-         * @param snapshotId      snapshot id
-         * @param shardId         shard to be restored
-         * @param snapshotShardId shard in the snapshot that data should be restored from
-         * @param recoveryState   recovery state to report progress
-         */
-        public RestoreContext(SnapshotId snapshotId, Version version, ShardId shardId, ShardId snapshotShardId, RecoveryState recoveryState) {
-            super(snapshotId, version, shardId, snapshotShardId);
-            this.recoveryState = recoveryState;
-            store = indicesService.indexServiceSafe(shardId.getIndex()).getShardOrNull(shardId.id()).store();
-        }
-
-        /**
-         * Performs restore operation
-         */
-        public void restore() throws IOException {
-            store.incRef();
-            try {
-                logger.debug("[{}] [{}] restoring to [{}] ...", snapshotId, repositoryName, shardId);
-                BlobStoreIndexShardSnapshot snapshot = loadSnapshot();
-
-                if (snapshot.indexFiles().size() == 1
-                    && snapshot.indexFiles().get(0).physicalName().startsWith("segments_")
-                    && snapshot.indexFiles().get(0).hasUnknownChecksum()) {
-                    // If the shard has no documents, it will only contain a single segments_N file for the
-                    // shard's snapshot.  If we are restoring a snapshot created by a previous supported version,
-                    // it is still possible that in that version, an empty shard has a segments_N file with an unsupported
-                    // version (and no checksum), because we don't know the Lucene version to assign segments_N until we
-                    // have written some data.  Since the segments_N for an empty shard could have an incompatible Lucene
-                    // version number and no checksum, even though the index itself is perfectly fine to restore, this
-                    // empty shard would cause exceptions to be thrown.  Since there is no data to restore from an empty
-                    // shard anyway, we just create the empty shard here and then exit.
-                    IndexWriter writer = new IndexWriter(store.directory(), new IndexWriterConfig(null)
-                                                                                .setOpenMode(IndexWriterConfig.OpenMode.CREATE)
-                                                                                .setCommitOnClose(true));
-                    writer.close();
-                    return;
-                }
-
-                SnapshotFiles snapshotFiles = new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles());
-                final Store.MetadataSnapshot recoveryTargetMetadata;
-                try {
-                    recoveryTargetMetadata = store.getMetadataOrEmpty();
-                } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException e) {
-                    logger.warn("{} Can't read metadata from store", e, shardId);
-                    throw new IndexShardRestoreFailedException(shardId, "Can't restore corrupted shard", e);
-                }
-
-                final List<FileInfo> filesToRecover = new ArrayList<>();
-                final Map<String, StoreFileMetaData> snapshotMetaData = new HashMap<>();
-                final Map<String, FileInfo> fileInfos = new HashMap<>();
-                for (final FileInfo fileInfo : snapshot.indexFiles()) {
-                    try {
-                        // in 1.3.3 we added additional hashes for .si / segments_N files
-                        // to ensure we don't double the space in the repo since old snapshots
-                        // don't have this hash we try to read that hash from the blob store
-                        // in a bwc compatible way.
-                        maybeRecalculateMetadataHash(blobContainer, fileInfo, recoveryTargetMetadata);
-                    } catch (Exception e) {
-                        // if the index is broken we might not be able to read it
-                        logger.warn("{} Can't calculate hash from blog for file [{}] [{}]", e, shardId, fileInfo.physicalName(), fileInfo.metadata());
-                    }
-                    snapshotMetaData.put(fileInfo.metadata().name(), fileInfo.metadata());
-                    fileInfos.put(fileInfo.metadata().name(), fileInfo);
-                }
-                final Store.MetadataSnapshot sourceMetaData = new Store.MetadataSnapshot(unmodifiableMap(snapshotMetaData), emptyMap(), 0);
-                final Store.RecoveryDiff diff = sourceMetaData.recoveryDiff(recoveryTargetMetadata);
-                for (StoreFileMetaData md : diff.identical) {
-                    FileInfo fileInfo = fileInfos.get(md.name());
-                    recoveryState.getIndex().addFileDetail(fileInfo.name(), fileInfo.length(), true);
-                    if (logger.isTraceEnabled()) {
-                        logger.trace("[{}] [{}] not_recovering [{}] from [{}], exists in local store and is same", shardId, snapshotId, fileInfo.physicalName(), fileInfo.name());
-                    }
-                }
-
-                for (StoreFileMetaData md : Iterables.concat(diff.different, diff.missing)) {
-                    FileInfo fileInfo = fileInfos.get(md.name());
-                    filesToRecover.add(fileInfo);
-                    recoveryState.getIndex().addFileDetail(fileInfo.name(), fileInfo.length(), false);
-                    if (logger.isTraceEnabled()) {
-                        if (md == null) {
-                            logger.trace("[{}] [{}] recovering [{}] from [{}], does not exists in local store", shardId, snapshotId, fileInfo.physicalName(), fileInfo.name());
-                        } else {
-                            logger.trace("[{}] [{}] recovering [{}] from [{}], exists in local store but is different", shardId, snapshotId, fileInfo.physicalName(), fileInfo.name());
-                        }
-                    }
-                }
-                final RecoveryState.Index index = recoveryState.getIndex();
-                if (filesToRecover.isEmpty()) {
-                    logger.trace("no files to recover, all exists within the local store");
-                }
-
-                if (logger.isTraceEnabled()) {
-                    logger.trace("[{}] [{}] recovering_files [{}] with total_size [{}], reusing_files [{}] with reused_size [{}]", shardId, snapshotId,
-                            index.totalRecoverFiles(), new ByteSizeValue(index.totalRecoverBytes()), index.reusedFileCount(), new ByteSizeValue(index.reusedFileCount()));
-                }
-                try {
-                    for (final FileInfo fileToRecover : filesToRecover) {
-                        logger.trace("[{}] [{}] restoring file [{}]", shardId, snapshotId, fileToRecover.name());
-                        restoreFile(fileToRecover);
-                    }
-                } catch (IOException ex) {
-                    throw new IndexShardRestoreFailedException(shardId, "Failed to recover index", ex);
-                }
-                final StoreFileMetaData restoredSegmentsFile = sourceMetaData.getSegmentsFile();
-                if (recoveryTargetMetadata == null) {
-                    throw new IndexShardRestoreFailedException(shardId, "Snapshot has no segments file");
-                }
-                assert restoredSegmentsFile != null;
-                // read the snapshot data persisted
-                final SegmentInfos segmentCommitInfos;
-                try {
-                    segmentCommitInfos = Lucene.pruneUnreferencedFiles(restoredSegmentsFile.name(), store.directory());
-                } catch (IOException e) {
-                    throw new IndexShardRestoreFailedException(shardId, "Failed to fetch index version after copying it over", e);
-                }
-                recoveryState.getIndex().updateVersion(segmentCommitInfos.getVersion());
-
-                /// now, go over and clean files that are in the store, but were not in the snapshot
-                try {
-                    for (String storeFile : store.directory().listAll()) {
-                        if (Store.isAutogenerated(storeFile) || snapshotFiles.containPhysicalIndexFile(storeFile)) {
-                            continue; //skip write.lock, checksum files and files that exist in the snapshot
-                        }
-                        try {
-                            store.deleteQuiet("restore", storeFile);
-                            store.directory().deleteFile(storeFile);
-                        } catch (IOException e) {
-                            logger.warn("[{}] failed to delete file [{}] during snapshot cleanup", snapshotId, storeFile);
-                        }
-                    }
-                } catch (IOException e) {
-                    logger.warn("[{}] failed to list directory - some of files might not be deleted", snapshotId);
-                }
-            } finally {
-                store.decRef();
-            }
-        }
-
-        /**
-         * Restores a file
-         * This is asynchronous method. Upon completion of the operation latch is getting counted down and any failures are
-         * added to the {@code failures} list
-         *
-         * @param fileInfo file to be restored
-         */
-        private void restoreFile(final FileInfo fileInfo) throws IOException {
-            boolean success = false;
-
-            try (InputStream partSliceStream = new PartSliceStream(blobContainer, fileInfo)) {
-                final InputStream stream;
-                if (restoreRateLimiter == null) {
-                    stream = partSliceStream;
-                } else {
-                    stream = new RateLimitingInputStream(partSliceStream, restoreRateLimiter, restoreThrottleListener);
-                }
-                try (final IndexOutput indexOutput = store.createVerifyingOutput(fileInfo.physicalName(), fileInfo.metadata(), IOContext.DEFAULT)) {
-                    final byte[] buffer = new byte[BUFFER_SIZE];
-                    int length;
-                    while ((length = stream.read(buffer)) > 0) {
-                        indexOutput.writeBytes(buffer, 0, length);
-                        recoveryState.getIndex().addRecoveredBytesToFile(fileInfo.name(), length);
-                    }
-                    Store.verify(indexOutput);
-                    indexOutput.close();
-                    store.directory().sync(Collections.singleton(fileInfo.physicalName()));
-                    success = true;
-                } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
-                    try {
-                        store.markStoreCorrupted(ex);
-                    } catch (IOException e) {
-                        logger.warn("store cannot be marked as corrupted", e);
-                    }
-                    throw ex;
-                } finally {
-                    if (success == false) {
-                        store.deleteQuiet(fileInfo.physicalName());
-                    }
-                }
-            }
-        }
-
-    }
-
-    public interface RateLimiterListener {
-        void onRestorePause(long nanos);
-
-        void onSnapshotPause(long nanos);
-    }
-
-}
diff --git a/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java b/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java
index acad2753f7a..0ba9cccddaf 100644
--- a/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java
+++ b/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java
@@ -21,8 +21,6 @@ package org.elasticsearch.repositories;
 
 import org.elasticsearch.action.admin.cluster.snapshots.status.TransportNodesSnapshotsStatus;
 import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.index.snapshots.IndexShardRepository;
-import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository;
 import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.repositories.uri.URLRepository;
 import org.elasticsearch.snapshots.RestoreService;
@@ -32,20 +30,20 @@ import org.elasticsearch.snapshots.SnapshotsService;
 /**
  * Sets up classes for Snapshot/Restore.
  *
- * Plugins can add custom repository types by calling {@link #registerRepository(String, Class, Class)}.
+ * Plugins can add custom repository types by calling {@link #registerRepository(String, Class)}.
  */
 public class RepositoriesModule extends AbstractModule {
 
     private final RepositoryTypesRegistry repositoryTypes = new RepositoryTypesRegistry();
 
     public RepositoriesModule() {
-        registerRepository(FsRepository.TYPE, FsRepository.class, BlobStoreIndexShardRepository.class);
-        registerRepository(URLRepository.TYPE, URLRepository.class, BlobStoreIndexShardRepository.class);
+        registerRepository(FsRepository.TYPE, FsRepository.class);
+        registerRepository(URLRepository.TYPE, URLRepository.class);
     }
 
-    /** Registers a custom repository type to the given {@link Repository} and {@link IndexShardRepository}. */
-    public void registerRepository(String type, Class<? extends Repository> repositoryType, Class<? extends IndexShardRepository> shardRepositoryType) {
-        repositoryTypes.registerRepository(type, repositoryType, shardRepositoryType);
+    /** Registers a custom repository type to the given {@link Repository}. */
+    public void registerRepository(String type, Class<? extends Repository> repositoryType) {
+        repositoryTypes.registerRepository(type, repositoryType);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
index 0ed53c0a9c5..6efba054c40 100644
--- a/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
+++ b/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
@@ -37,7 +37,6 @@ import org.elasticsearch.common.inject.Injector;
 import org.elasticsearch.common.inject.ModulesBuilder;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.snapshots.IndexShardRepository;
 import org.elasticsearch.snapshots.RestoreService;
 import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.transport.TransportService;
@@ -336,23 +335,6 @@ public class RepositoriesService extends AbstractComponent implements ClusterSta
         throw new RepositoryMissingException(repository);
     }
 
-    /**
-     * Returns registered index shard repository
-     * <p>
-     * This method is called only on data nodes
-     *
-     * @param repository repository name
-     * @return registered repository
-     * @throws RepositoryMissingException if repository with such name isn't registered
-     */
-    public IndexShardRepository indexShardRepository(String repository) {
-        RepositoryHolder holder = repositories.get(repository);
-        if (holder != null) {
-            return holder.indexShardRepository;
-        }
-        throw new RepositoryMissingException(repository);
-    }
-
     /**
      * Creates a new repository and adds it to the list of registered repositories.
      * <p>
@@ -403,14 +385,16 @@ public class RepositoriesService extends AbstractComponent implements ClusterSta
         try {
             ModulesBuilder modules = new ModulesBuilder();
             RepositoryName name = new RepositoryName(repositoryMetaData.type(), repositoryMetaData.name());
-            modules.add(new RepositoryNameModule(name));
-            modules.add(new RepositoryModule(name, repositoryMetaData.settings(), this.settings, typesRegistry));
+            modules.add(b -> {
+                b.bind(RepositoryName.class).toInstance(name);
+                typesRegistry.bindType(b, repositoryMetaData.type());
+                b.bind(RepositorySettings.class).toInstance(new RepositorySettings(settings, repositoryMetaData.settings()));
+            });
 
             repositoryInjector = modules.createChildInjector(injector);
             Repository repository = repositoryInjector.getInstance(Repository.class);
-            IndexShardRepository indexShardRepository = repositoryInjector.getInstance(IndexShardRepository.class);
             repository.start();
-            return new RepositoryHolder(repositoryMetaData.type(), repositoryMetaData.settings(), repository, indexShardRepository);
+            return new RepositoryHolder(repositoryMetaData.type(), repositoryMetaData.settings(), repository);
         } catch (Exception e) {
             logger.warn("failed to create repository [{}][{}]", e, repositoryMetaData.type(), repositoryMetaData.name());
             throw new RepositoryException(repositoryMetaData.name(), "failed to create repository", e);
@@ -472,13 +456,11 @@ public class RepositoriesService extends AbstractComponent implements ClusterSta
         private final String type;
         private final Settings settings;
         private final Repository repository;
-        private final IndexShardRepository indexShardRepository;
 
-        public RepositoryHolder(String type, Settings settings,Repository repository, IndexShardRepository indexShardRepository) {
+        public RepositoryHolder(String type, Settings settings,Repository repository) {
             this.type = type;
             this.settings = settings;
             this.repository = repository;
-            this.indexShardRepository = indexShardRepository;
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/repositories/Repository.java b/core/src/main/java/org/elasticsearch/repositories/Repository.java
index 47a87edd7f9..6c4cf93a7a4 100644
--- a/core/src/main/java/org/elasticsearch/repositories/Repository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/Repository.java
@@ -19,7 +19,11 @@
 package org.elasticsearch.repositories;
 
 import org.apache.lucene.index.IndexCommit;
+import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.common.component.LifecycleComponent;
 import org.elasticsearch.index.shard.ShardId;
@@ -29,22 +33,19 @@ import org.elasticsearch.snapshots.SnapshotShardFailure;
 
 import java.io.IOException;
 import java.util.List;
-import java.util.function.Predicate;
 
 /**
  * Snapshot repository interface.
  * <p>
- * Responsible for index and cluster level operations. It's called only on master.
- * Shard-level operations are performed using {@link org.elasticsearch.index.snapshots.IndexShardRepository}
- * interface on data nodes.
+ * Responsible for index and cluster and shard level operations.
  * <p>
  * Typical snapshot usage pattern:
  * <ul>
  * <li>Master calls {@link #initializeSnapshot(SnapshotId, List, org.elasticsearch.cluster.metadata.MetaData)}
  * with list of indices that will be included into the snapshot</li>
- * <li>Data nodes call {@link org.elasticsearch.index.snapshots.IndexShardRepository#snapshot(SnapshotId, ShardId, IndexCommit, IndexShardSnapshotStatus)} for each shard</li>
- * <li>When all shard calls return master calls {@link #finalizeSnapshot}
- * with possible list of failures</li>
+ * <li>Data nodes call {@link Repository#snapshot(IndexShard, SnapshotId, IndexCommit, IndexShardSnapshotStatus)}
+ * for each shard</li>
+ * <li>When all shard calls return master calls {@link #finalizeSnapshot} with possible list of failures</li>
  * </ul>
  */
 public interface Repository extends LifecycleComponent {
@@ -141,4 +142,49 @@ public interface Repository extends LifecycleComponent {
      */
     boolean readOnly();
 
+    /**
+     * Creates a snapshot of the shard based on the index commit point.
+     * <p>
+     * The index commit point can be obtained by using {@link org.elasticsearch.index.engine.Engine#snapshotIndex} method.
+     * Repository implementations shouldn't release the snapshot index commit point. It is done by the method caller.
+     * <p>
+     * As snapshot process progresses, implementation of this method should update {@link IndexShardSnapshotStatus} object and check
+     * {@link IndexShardSnapshotStatus#aborted()} to see if the snapshot process should be aborted.
+     *
+     * @param shard               shard to be snapshotted
+     * @param snapshotId          snapshot id
+     * @param snapshotIndexCommit commit point
+     * @param snapshotStatus      snapshot status
+     */
+    void snapshot(IndexShard shard, SnapshotId snapshotId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus);
+
+    /**
+     * Restores snapshot of the shard.
+     * <p>
+     * The index can be renamed on restore, hence different {@code shardId} and {@code snapshotShardId} are supplied.
+     *
+     * @param shard           the shard to restore the index into
+     * @param snapshotId      snapshot id
+     * @param version         version of elasticsearch that created this snapshot
+     * @param snapshotShardId shard id (in the snapshot)
+     * @param recoveryState   recovery state
+     */
+    void restore(IndexShard shard, SnapshotId snapshotId, Version version, ShardId snapshotShardId, RecoveryState recoveryState);
+
+    /**
+     * Retrieve shard snapshot status for the stored snapshot
+     *
+     * @param snapshotId snapshot id
+     * @param version   version of elasticsearch that created this snapshot
+     * @param shardId    shard id
+     * @return snapshot status
+     */
+    IndexShardSnapshotStatus snapshotStatus(SnapshotId snapshotId, Version version, ShardId shardId);
+
+    /**
+     * Verifies repository settings on data node.
+     * @param verificationToken value returned by {@link org.elasticsearch.repositories.Repository#startVerification()}
+     * @param localNode         the local node information, for inclusion in verification errors
+     */
+    void verify(String verificationToken, DiscoveryNode localNode);
 }
diff --git a/core/src/main/java/org/elasticsearch/repositories/RepositoryModule.java b/core/src/main/java/org/elasticsearch/repositories/RepositoryModule.java
deleted file mode 100644
index 3ed3a78c0d1..00000000000
--- a/core/src/main/java/org/elasticsearch/repositories/RepositoryModule.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.repositories;
-
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.settings.Settings;
-
-/**
- * Binds repository classes for the specific repository type.
- */
-public class RepositoryModule extends AbstractModule {
-
-    private RepositoryName repositoryName;
-
-    private final Settings globalSettings;
-
-    private final Settings settings;
-
-    private final RepositoryTypesRegistry typesRegistry;
-
-    /**
-     * Spawns module for repository with specified name, type and settings
-     *
-     * @param repositoryName repository name and type
-     * @param settings       repository settings
-     * @param globalSettings global settings
-     * @param typesRegistry  registry of repository types
-     */
-    public RepositoryModule(RepositoryName repositoryName, Settings settings, Settings globalSettings, RepositoryTypesRegistry typesRegistry) {
-        this.repositoryName = repositoryName;
-        this.globalSettings = globalSettings;
-        this.settings = settings;
-        this.typesRegistry = typesRegistry;
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override
-    protected void configure() {
-        typesRegistry.bindType(binder(), repositoryName.type());
-        bind(RepositorySettings.class).toInstance(new RepositorySettings(globalSettings, settings));
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/repositories/RepositoryNameModule.java b/core/src/main/java/org/elasticsearch/repositories/RepositoryNameModule.java
deleted file mode 100644
index 47be67df34c..00000000000
--- a/core/src/main/java/org/elasticsearch/repositories/RepositoryNameModule.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.repositories;
-
-import org.elasticsearch.common.inject.AbstractModule;
-
-/**
- * Binds specific instance of RepositoryName for injection to repository module
- */
-public class RepositoryNameModule extends AbstractModule {
-
-    private final RepositoryName repositoryName;
-
-    public RepositoryNameModule(RepositoryName repositoryName) {
-        this.repositoryName = repositoryName;
-    }
-
-    @Override
-    protected void configure() {
-        bind(RepositoryName.class).toInstance(repositoryName);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/repositories/RepositoryTypesRegistry.java b/core/src/main/java/org/elasticsearch/repositories/RepositoryTypesRegistry.java
index d2f02aa5795..63a790b77fd 100644
--- a/core/src/main/java/org/elasticsearch/repositories/RepositoryTypesRegistry.java
+++ b/core/src/main/java/org/elasticsearch/repositories/RepositoryTypesRegistry.java
@@ -22,22 +22,18 @@ package org.elasticsearch.repositories;
 import org.elasticsearch.common.inject.Binder;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.ExtensionPoint;
-import org.elasticsearch.index.snapshots.IndexShardRepository;
 
 /**
- * A mapping from type name to implementations of {@link Repository} and {@link IndexShardRepository}.
+ * A mapping from type name to implementations of {@link Repository}.
  */
 public class RepositoryTypesRegistry {
     // invariant: repositories and shardRepositories have the same keyset
     private final ExtensionPoint.SelectedType<Repository> repositoryTypes =
         new ExtensionPoint.SelectedType<>("repository", Repository.class);
-    private final ExtensionPoint.SelectedType<IndexShardRepository> shardRepositoryTypes =
-        new ExtensionPoint.SelectedType<>("index_repository", IndexShardRepository.class);
 
     /** Adds a new repository type to the registry, bound to the given implementation classes. */
-    public void registerRepository(String name, Class<? extends Repository> repositoryType, Class<? extends IndexShardRepository> shardRepositoryType) {
+    public void registerRepository(String name, Class<? extends Repository> repositoryType) {
         repositoryTypes.registerExtension(name, repositoryType);
-        shardRepositoryTypes.registerExtension(name, shardRepositoryType);
     }
 
     /**
@@ -47,6 +43,5 @@ public class RepositoryTypesRegistry {
     public void bindType(Binder binder, String type) {
         Settings settings = Settings.builder().put("type", type).build();
         repositoryTypes.bindType(binder, settings, "type", null);
-        shardRepositoryTypes.bindType(binder, settings, "type", null);
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java b/core/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java
index 796c65e9b46..65544421c8c 100644
--- a/core/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java
+++ b/core/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java
@@ -19,6 +19,12 @@
 
 package org.elasticsearch.repositories;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicInteger;
+
 import com.carrotsearch.hppc.ObjectContainer;
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import org.elasticsearch.action.ActionListener;
@@ -29,7 +35,6 @@ import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.snapshots.IndexShardRepository;
 import org.elasticsearch.repositories.RepositoriesService.VerifyResponse;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.EmptyTransportResponseHandler;
@@ -40,12 +45,6 @@ import org.elasticsearch.transport.TransportRequestHandler;
 import org.elasticsearch.transport.TransportResponse;
 import org.elasticsearch.transport.TransportService;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.atomic.AtomicInteger;
-
 public class VerifyNodeRepositoryAction  extends AbstractComponent {
     public static final String ACTION_NAME = "internal:admin/repository/verify";
 
@@ -82,7 +81,7 @@ public class VerifyNodeRepositoryAction  extends AbstractComponent {
         for (final DiscoveryNode node : nodes) {
             if (node.equals(localNode)) {
                 try {
-                    doVerify(repository, verificationToken);
+                    doVerify(repository, verificationToken, localNode);
                 } catch (Exception e) {
                     logger.warn("[{}] failed to verify repository", e, repository);
                     errors.add(new VerificationFailure(node.getId(), e));
@@ -115,9 +114,9 @@ public class VerifyNodeRepositoryAction  extends AbstractComponent {
         listener.onResponse(new RepositoriesService.VerifyResponse(nodes.toArray(new DiscoveryNode[nodes.size()]), errors.toArray(new VerificationFailure[errors.size()])));
     }
 
-    private void doVerify(String repository, String verificationToken) {
-        IndexShardRepository blobStoreIndexShardRepository = repositoriesService.indexShardRepository(repository);
-        blobStoreIndexShardRepository.verify(verificationToken);
+    private void doVerify(String repositoryName, String verificationToken, DiscoveryNode localNode) {
+        Repository repository = repositoriesService.repository(repositoryName);
+        repository.verify(verificationToken, localNode);
     }
 
     public static class VerifyNodeRepositoryRequest extends TransportRequest {
@@ -151,8 +150,9 @@ public class VerifyNodeRepositoryAction  extends AbstractComponent {
     class VerifyNodeRepositoryRequestHandler implements TransportRequestHandler<VerifyNodeRepositoryRequest> {
         @Override
         public void messageReceived(VerifyNodeRepositoryRequest request, TransportChannel channel) throws Exception {
+            DiscoveryNode localNode = clusterService.state().nodes().getLocalNode();
             try {
-                doVerify(request.repository, request.verificationToken);
+                doVerify(request.repository, request.verificationToken, localNode);
             } catch (Exception ex) {
                 logger.warn("[{}] failed to verify repository", ex, request.repository);
                 throw ex;
diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index b1f1ff66cf5..4c328872742 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -19,12 +19,43 @@
 
 package org.elasticsearch.repositories.blobstore;
 
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.IndexCommit;
+import org.apache.lucene.index.IndexFormatTooNewException;
+import org.apache.lucene.index.IndexFormatTooOldException;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.SegmentInfos;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.RateLimiter;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Numbers;
+import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.lucene.store.InputStreamIndexInput;
+import org.elasticsearch.common.util.iterable.Iterables;
+import org.elasticsearch.index.shard.IndexShard;
+import org.elasticsearch.index.snapshots.IndexShardRestoreFailedException;
+import org.elasticsearch.index.snapshots.IndexShardSnapshotException;
+import org.elasticsearch.index.snapshots.IndexShardSnapshotFailedException;
+import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
+import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot;
+import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshots;
+import org.elasticsearch.index.snapshots.blobstore.RateLimitingInputStream;
+import org.elasticsearch.index.snapshots.blobstore.SlicedInputStream;
+import org.elasticsearch.index.snapshots.blobstore.SnapshotFiles;
+import org.elasticsearch.index.store.Store;
+import org.elasticsearch.index.store.StoreFileMetaData;
+import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.snapshots.SnapshotId;
 import org.elasticsearch.common.ParseFieldMatcher;
 import org.elasticsearch.common.Strings;
@@ -50,9 +81,6 @@ import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.index.snapshots.IndexShardRepository;
-import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository;
-import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository.RateLimiterListener;
 import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.RepositorySettings;
@@ -64,15 +92,21 @@ import org.elasticsearch.snapshots.SnapshotMissingException;
 import org.elasticsearch.snapshots.SnapshotShardFailure;
 
 import java.io.FileNotFoundException;
+import java.io.FilterInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.NoSuchFileException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.unmodifiableMap;
+
 /**
  * BlobStore - based implementation of Snapshot Repository
  * <p>
@@ -116,19 +150,19 @@ import java.util.stream.Collectors;
  * }
  * </pre>
  */
-public abstract class BlobStoreRepository extends AbstractLifecycleComponent implements Repository, RateLimiterListener {
+public abstract class BlobStoreRepository extends AbstractLifecycleComponent implements Repository {
 
     private BlobContainer snapshotsBlobContainer;
 
     protected final String repositoryName;
 
+    private static final int BUFFER_SIZE = 4096;
+
     private static final String LEGACY_SNAPSHOT_PREFIX = "snapshot-";
 
     private static final String SNAPSHOT_PREFIX = "snap-";
 
-    private static final String SNAPSHOT_SUFFIX = ".dat";
-
-    private static final String SNAPSHOT_CODEC = "snapshot";
+    protected static final String SNAPSHOT_CODEC = "snapshot";
 
     static final String SNAPSHOTS_FILE = "index"; // package private for unit testing
 
@@ -146,11 +180,17 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     private static final String INDEX_METADATA_CODEC = "index-metadata";
 
-    private static final String SNAPSHOT_NAME_FORMAT = SNAPSHOT_PREFIX + "%s" + SNAPSHOT_SUFFIX;
+    protected static final String LEGACY_SNAPSHOT_NAME_FORMAT = LEGACY_SNAPSHOT_PREFIX + "%s";
 
-    private static final String LEGACY_SNAPSHOT_NAME_FORMAT = LEGACY_SNAPSHOT_PREFIX + "%s";
+    protected static final String SNAPSHOT_NAME_FORMAT = SNAPSHOT_PREFIX + "%s.dat";
 
-    private final BlobStoreIndexShardRepository indexShardRepository;
+    protected static final String SNAPSHOT_INDEX_PREFIX = "index-";
+
+    protected static final String SNAPSHOT_INDEX_NAME_FORMAT = SNAPSHOT_INDEX_PREFIX + "%s";
+
+    protected static final String SNAPSHOT_INDEX_CODEC = "snapshots";
+
+    protected static final String DATA_BLOB_PREFIX = "__";
 
     private final RateLimiter snapshotRateLimiter;
 
@@ -174,29 +214,36 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     private final boolean readOnly;
 
+    private final ParseFieldMatcher parseFieldMatcher;
+
+    private final ChecksumBlobStoreFormat<BlobStoreIndexShardSnapshot> indexShardSnapshotFormat;
+
+    private final LegacyBlobStoreFormat<BlobStoreIndexShardSnapshot> indexShardSnapshotLegacyFormat;
+
+    private final ChecksumBlobStoreFormat<BlobStoreIndexShardSnapshots> indexShardSnapshotsFormat;
+
     /**
      * Constructs new BlobStoreRepository
      *
      * @param repositoryName       repository name
      * @param repositorySettings   repository settings
-     * @param indexShardRepository an instance of IndexShardRepository
      */
-    protected BlobStoreRepository(String repositoryName, RepositorySettings repositorySettings, IndexShardRepository indexShardRepository) {
+    protected BlobStoreRepository(String repositoryName, RepositorySettings repositorySettings) {
         super(repositorySettings.globalSettings());
         this.repositoryName = repositoryName;
-        this.indexShardRepository = (BlobStoreIndexShardRepository) indexShardRepository;
+        parseFieldMatcher = new ParseFieldMatcher(settings);
         snapshotRateLimiter = getRateLimiter(repositorySettings, "max_snapshot_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB));
         restoreRateLimiter = getRateLimiter(repositorySettings, "max_restore_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB));
         readOnly = repositorySettings.settings().getAsBoolean("readonly", false);
+        indexShardSnapshotFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_CODEC, SNAPSHOT_NAME_FORMAT, BlobStoreIndexShardSnapshot.PROTO, parseFieldMatcher, isCompress());
+        indexShardSnapshotLegacyFormat = new LegacyBlobStoreFormat<>(LEGACY_SNAPSHOT_NAME_FORMAT, BlobStoreIndexShardSnapshot.PROTO, parseFieldMatcher);
+        indexShardSnapshotsFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_INDEX_CODEC, SNAPSHOT_INDEX_NAME_FORMAT, BlobStoreIndexShardSnapshots.PROTO, parseFieldMatcher, isCompress());
+
     }
 
-    /**
-     * {@inheritDoc}
-     */
     @Override
     protected void doStart() {
         this.snapshotsBlobContainer = blobStore().blobContainer(basePath());
-        indexShardRepository.initialize(blobStore(), basePath(), chunkSize(), snapshotRateLimiter, restoreRateLimiter, this, isCompress());
 
         ParseFieldMatcher parseFieldMatcher = new ParseFieldMatcher(settings);
         globalMetaDataFormat = new ChecksumBlobStoreFormat<>(METADATA_CODEC, METADATA_NAME_FORMAT, MetaData.PROTO, parseFieldMatcher, isCompress());
@@ -209,16 +256,9 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         snapshotLegacyFormat = new LegacyBlobStoreFormat<>(LEGACY_SNAPSHOT_NAME_FORMAT, SnapshotInfo.PROTO, parseFieldMatcher);
     }
 
-    /**
-     * {@inheritDoc}
-     */
     @Override
-    protected void doStop() {
-    }
+    protected void doStop() {}
 
-    /**
-     * {@inheritDoc}
-     */
     @Override
     protected void doClose() {
         try {
@@ -229,18 +269,14 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
     }
 
     /**
-     * Returns initialized and ready to use BlobStore
-     * <p>
-     * This method is first called in the {@link #doStart()} method.
-     *
-     * @return blob store
+     * Returns the BlobStore to read and write data.
      */
-    protected abstract  BlobStore blobStore();
+    protected abstract BlobStore blobStore();
 
     /**
      * Returns base path of the repository
      */
-    protected abstract  BlobPath basePath();
+    protected abstract BlobPath basePath();
 
     /**
      * Returns true if metadata and snapshot files should be compressed
@@ -262,9 +298,6 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         return null;
     }
 
-    /**
-     * {@inheritDoc}
-     */
     @Override
     public void initializeSnapshot(SnapshotId snapshotId, List<String> indices, MetaData metaData) {
         if (readOnly()) {
@@ -293,9 +326,6 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         }
     }
 
-    /**
-     * {@inheritDoc}
-     */
     @Override
     public void deleteSnapshot(SnapshotId snapshotId) {
         if (readOnly()) {
@@ -352,7 +382,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
                     if (indexMetaData != null) {
                         for (int shardId = 0; shardId < indexMetaData.getNumberOfShards(); shardId++) {
                             try {
-                                indexShardRepository.delete(snapshotId, snapshot.version(), new ShardId(indexMetaData.getIndex(), shardId));
+                                delete(snapshotId, snapshot.version(), new ShardId(indexMetaData.getIndex(), shardId));
                             } catch (SnapshotException ex) {
                                 logger.warn("[{}] failed to delete shard data for shard [{}][{}]", ex, snapshotId, index, shardId);
                             }
@@ -365,9 +395,6 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         }
     }
 
-    /**
-     * {@inheritDoc}
-     */
     @Override
     public SnapshotInfo finalizeSnapshot(final SnapshotId snapshotId,
                                          final List<String> indices,
@@ -397,9 +424,6 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         }
     }
 
-    /**
-     * {@inheritDoc}
-     */
     @Override
     public List<SnapshotId> snapshots() {
         try {
@@ -412,9 +436,6 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         }
     }
 
-    /**
-     * {@inheritDoc}
-     */
     @Override
     public MetaData readSnapshotMetaData(SnapshotInfo snapshot, List<String> indices) throws IOException {
         return readSnapshotMetaData(snapshot.snapshotId(), snapshot.version(), indices, false);
@@ -539,16 +560,6 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
     private static final String NAME = "name";
     private static final String UUID = "uuid";
 
-    @Override
-    public void onRestorePause(long nanos) {
-        restoreRateLimitingTimeInNanos.inc(nanos);
-    }
-
-    @Override
-    public void onSnapshotPause(long nanos) {
-        snapshotRateLimitingTimeInNanos.inc(nanos);
-    }
-
     @Override
     public long snapshotThrottleTimeInNanos() {
         return snapshotRateLimitingTimeInNanos.count();
@@ -785,4 +796,782 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         }
     }
 
+    @Override
+    public void snapshot(IndexShard shard, SnapshotId snapshotId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus) {
+        SnapshotContext snapshotContext = new SnapshotContext(shard, snapshotId, snapshotStatus);
+        snapshotStatus.startTime(System.currentTimeMillis());
+
+        try {
+            snapshotContext.snapshot(snapshotIndexCommit);
+            snapshotStatus.time(System.currentTimeMillis() - snapshotStatus.startTime());
+            snapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.DONE);
+        } catch (Exception e) {
+            snapshotStatus.time(System.currentTimeMillis() - snapshotStatus.startTime());
+            snapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.FAILURE);
+            snapshotStatus.failure(ExceptionsHelper.detailedMessage(e));
+            if (e instanceof IndexShardSnapshotFailedException) {
+                throw (IndexShardSnapshotFailedException) e;
+            } else {
+                throw new IndexShardSnapshotFailedException(shard.shardId(), e);
+            }
+        }
+    }
+
+    @Override
+    public void restore(IndexShard shard, SnapshotId snapshotId, Version version, ShardId snapshotShardId, RecoveryState recoveryState) {
+        final RestoreContext snapshotContext = new RestoreContext(shard, snapshotId, version, snapshotShardId, recoveryState);
+        try {
+            snapshotContext.restore();
+        } catch (Exception e) {
+            throw new IndexShardRestoreFailedException(shard.shardId(), "failed to restore snapshot [" + snapshotId + "]", e);
+        }
+    }
+
+    @Override
+    public IndexShardSnapshotStatus snapshotStatus(SnapshotId snapshotId, Version version, ShardId shardId) {
+        Context context = new Context(snapshotId, version, shardId);
+        BlobStoreIndexShardSnapshot snapshot = context.loadSnapshot();
+        IndexShardSnapshotStatus status = new IndexShardSnapshotStatus();
+        status.updateStage(IndexShardSnapshotStatus.Stage.DONE);
+        status.startTime(snapshot.startTime());
+        status.files(snapshot.numberOfFiles(), snapshot.totalSize());
+        // The snapshot is done which means the number of processed files is the same as total
+        status.processedFiles(snapshot.numberOfFiles(), snapshot.totalSize());
+        status.time(snapshot.time());
+        return status;
+    }
+
+    @Override
+    public void verify(String seed, DiscoveryNode localNode) {
+        BlobContainer testBlobContainer = blobStore().blobContainer(basePath().add(testBlobPrefix(seed)));
+        if (testBlobContainer.blobExists("master.dat")) {
+            try  {
+                testBlobContainer.writeBlob("data-" + localNode.getId() + ".dat", new BytesArray(seed));
+            } catch (IOException exp) {
+                throw new RepositoryVerificationException(repositoryName, "store location [" + blobStore() + "] is not accessible on the node [" + localNode + "]", exp);
+            }
+        } else {
+            throw new RepositoryVerificationException(repositoryName, "a file written by master to the store [" + blobStore() + "] cannot be accessed on the node [" + localNode + "]. "
+                + "This might indicate that the store [" + blobStore() + "] is not shared between this node and the master node or "
+                + "that permissions on the store don't allow reading files written by the master node");
+        }
+    }
+
+    /**
+     * Delete shard snapshot
+     *
+     * @param snapshotId snapshot id
+     * @param shardId    shard id
+     */
+    public void delete(SnapshotId snapshotId, Version version, ShardId shardId) {
+        Context context = new Context(snapshotId, version, shardId, shardId);
+        context.delete();
+    }
+
+    @Override
+    public String toString() {
+        return "BlobStoreRepository[" +
+            "[" + repositoryName +
+            "], [" + blobStore() + ']' +
+            ']';
+    }
+
+    BlobStoreFormat<BlobStoreIndexShardSnapshot> indexShardSnapshotFormat(Version version) {
+        if (BlobStoreRepository.legacyMetaData(version)) {
+            return indexShardSnapshotLegacyFormat;
+        } else {
+            return indexShardSnapshotFormat;
+        }
+    }
+
+    /**
+     * Context for snapshot/restore operations
+     */
+    private class Context {
+
+        protected final SnapshotId snapshotId;
+
+        protected final ShardId shardId;
+
+        protected final BlobContainer blobContainer;
+
+        protected final Version version;
+
+        public Context(SnapshotId snapshotId, Version version, ShardId shardId) {
+            this(snapshotId, version, shardId, shardId);
+        }
+
+        public Context(SnapshotId snapshotId, Version version, ShardId shardId, ShardId snapshotShardId) {
+            this.snapshotId = snapshotId;
+            this.version = version;
+            this.shardId = shardId;
+            blobContainer = blobStore().blobContainer(basePath().add("indices").add(snapshotShardId.getIndexName()).add(Integer.toString(snapshotShardId.getId())));
+        }
+
+        /**
+         * Delete shard snapshot
+         */
+        public void delete() {
+            final Map<String, BlobMetaData> blobs;
+            try {
+                blobs = blobContainer.listBlobs();
+            } catch (IOException e) {
+                throw new IndexShardSnapshotException(shardId, "Failed to list content of gateway", e);
+            }
+
+            Tuple<BlobStoreIndexShardSnapshots, Integer> tuple = buildBlobStoreIndexShardSnapshots(blobs);
+            BlobStoreIndexShardSnapshots snapshots = tuple.v1();
+            int fileListGeneration = tuple.v2();
+
+            try {
+                indexShardSnapshotFormat(version).delete(blobContainer, snapshotId.getName());
+            } catch (IOException e) {
+                logger.debug("[{}] [{}] failed to delete shard snapshot file", shardId, snapshotId);
+            }
+
+            // Build a list of snapshots that should be preserved
+            List<SnapshotFiles> newSnapshotsList = new ArrayList<>();
+            for (SnapshotFiles point : snapshots) {
+                if (!point.snapshot().equals(snapshotId.getName())) {
+                    newSnapshotsList.add(point);
+                }
+            }
+            // finalize the snapshot and rewrite the snapshot index with the next sequential snapshot index
+            finalize(newSnapshotsList, fileListGeneration + 1, blobs);
+        }
+
+        /**
+         * Loads information about shard snapshot
+         */
+        public BlobStoreIndexShardSnapshot loadSnapshot() {
+            try {
+                return indexShardSnapshotFormat(version).read(blobContainer, snapshotId.getName());
+            } catch (IOException ex) {
+                throw new IndexShardRestoreFailedException(shardId, "failed to read shard snapshot file", ex);
+            }
+        }
+
+        /**
+         * Removes all unreferenced files from the repository and writes new index file
+         *
+         * We need to be really careful in handling index files in case of failures to make sure we have index file that
+         * points to files that were deleted.
+         *
+         *
+         * @param snapshots list of active snapshots in the container
+         * @param fileListGeneration the generation number of the snapshot index file
+         * @param blobs     list of blobs in the container
+         */
+        protected void finalize(List<SnapshotFiles> snapshots, int fileListGeneration, Map<String, BlobMetaData> blobs) {
+            BlobStoreIndexShardSnapshots newSnapshots = new BlobStoreIndexShardSnapshots(snapshots);
+            List<String> blobsToDelete = new ArrayList<>();
+            // delete old index files first
+            for (String blobName : blobs.keySet()) {
+                // delete old file lists
+                if (indexShardSnapshotsFormat.isTempBlobName(blobName) || blobName.startsWith(SNAPSHOT_INDEX_PREFIX)) {
+                    blobsToDelete.add(blobName);
+                }
+            }
+
+            try {
+                blobContainer.deleteBlobs(blobsToDelete);
+            } catch (IOException e) {
+                // We cannot delete index file - this is fatal, we cannot continue, otherwise we might end up
+                // with references to non-existing files
+                throw new IndexShardSnapshotFailedException(shardId, "error deleting index files during cleanup", e);
+            }
+
+            blobsToDelete = new ArrayList<>();
+            // now go over all the blobs, and if they don't exists in a snapshot, delete them
+            for (String blobName : blobs.keySet()) {
+                // delete unused files
+                if (blobName.startsWith(DATA_BLOB_PREFIX)) {
+                    if (newSnapshots.findNameFile(BlobStoreIndexShardSnapshot.FileInfo.canonicalName(blobName)) == null) {
+                        blobsToDelete.add(blobName);
+                    }
+                }
+            }
+            try {
+                blobContainer.deleteBlobs(blobsToDelete);
+            } catch (IOException e) {
+                logger.debug("[{}] [{}] error deleting some of the blobs [{}] during cleanup", e, snapshotId, shardId, blobsToDelete);
+            }
+
+            // If we deleted all snapshots - we don't need to create the index file
+            if (snapshots.size() > 0) {
+                try {
+                    indexShardSnapshotsFormat.writeAtomic(newSnapshots, blobContainer, Integer.toString(fileListGeneration));
+                } catch (IOException e) {
+                    throw new IndexShardSnapshotFailedException(shardId, "Failed to write file list", e);
+                }
+            }
+        }
+
+        /**
+         * Generates blob name
+         *
+         * @param generation the blob number
+         * @return the blob name
+         */
+        protected String fileNameFromGeneration(long generation) {
+            return DATA_BLOB_PREFIX + Long.toString(generation, Character.MAX_RADIX);
+        }
+
+        /**
+         * Finds the next available blob number
+         *
+         * @param blobs list of blobs in the repository
+         * @return next available blob number
+         */
+        protected long findLatestFileNameGeneration(Map<String, BlobMetaData> blobs) {
+            long generation = -1;
+            for (String name : blobs.keySet()) {
+                if (!name.startsWith(DATA_BLOB_PREFIX)) {
+                    continue;
+                }
+                name = BlobStoreIndexShardSnapshot.FileInfo.canonicalName(name);
+                try {
+                    long currentGen = Long.parseLong(name.substring(DATA_BLOB_PREFIX.length()), Character.MAX_RADIX);
+                    if (currentGen > generation) {
+                        generation = currentGen;
+                    }
+                } catch (NumberFormatException e) {
+                    logger.warn("file [{}] does not conform to the '{}' schema", name, DATA_BLOB_PREFIX);
+                }
+            }
+            return generation;
+        }
+
+        /**
+         * Loads all available snapshots in the repository
+         *
+         * @param blobs list of blobs in repository
+         * @return tuple of BlobStoreIndexShardSnapshots and the last snapshot index generation
+         */
+        protected Tuple<BlobStoreIndexShardSnapshots, Integer> buildBlobStoreIndexShardSnapshots(Map<String, BlobMetaData> blobs) {
+            int latest = -1;
+            for (String name : blobs.keySet()) {
+                if (name.startsWith(SNAPSHOT_INDEX_PREFIX)) {
+                    try {
+                        int gen = Integer.parseInt(name.substring(SNAPSHOT_INDEX_PREFIX.length()));
+                        if (gen > latest) {
+                            latest = gen;
+                        }
+                    } catch (NumberFormatException ex) {
+                        logger.warn("failed to parse index file name [{}]", name);
+                    }
+                }
+            }
+            if (latest >= 0) {
+                try {
+                    final BlobStoreIndexShardSnapshots shardSnapshots =
+                        indexShardSnapshotsFormat.read(blobContainer, Integer.toString(latest));
+                    return new Tuple<>(shardSnapshots, latest);
+                } catch (IOException e) {
+                    logger.warn("failed to read index file  [{}]", e, SNAPSHOT_INDEX_PREFIX + latest);
+                }
+            }
+
+            // We couldn't load the index file - falling back to loading individual snapshots
+            List<SnapshotFiles> snapshots = new ArrayList<>();
+            for (String name : blobs.keySet()) {
+                try {
+                    BlobStoreIndexShardSnapshot snapshot = null;
+                    if (name.startsWith(SNAPSHOT_PREFIX)) {
+                        snapshot = indexShardSnapshotFormat.readBlob(blobContainer, name);
+                    } else if (name.startsWith(LEGACY_SNAPSHOT_PREFIX)) {
+                        snapshot = indexShardSnapshotLegacyFormat.readBlob(blobContainer, name);
+                    }
+                    if (snapshot != null) {
+                        snapshots.add(new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles()));
+                    }
+                } catch (IOException e) {
+                    logger.warn("failed to read commit point [{}]", e, name);
+                }
+            }
+            return new Tuple<>(new BlobStoreIndexShardSnapshots(snapshots), -1);
+        }
+    }
+
+    /**
+     * Context for snapshot operations
+     */
+    private class SnapshotContext extends Context {
+
+        private final Store store;
+
+        private final IndexShardSnapshotStatus snapshotStatus;
+
+        /**
+         * Constructs new context
+         *
+         * @param shard          shard to be snapshotted
+         * @param snapshotId     snapshot id
+         * @param snapshotStatus snapshot status to report progress
+         */
+        public SnapshotContext(IndexShard shard, SnapshotId snapshotId, IndexShardSnapshotStatus snapshotStatus) {
+            super(snapshotId, Version.CURRENT, shard.shardId());
+            this.snapshotStatus = snapshotStatus;
+            this.store = shard.store();
+        }
+
+        /**
+         * Create snapshot from index commit point
+         *
+         * @param snapshotIndexCommit snapshot commit point
+         */
+        public void snapshot(IndexCommit snapshotIndexCommit) {
+            logger.debug("[{}] [{}] snapshot to [{}] ...", shardId, snapshotId, repositoryName);
+            store.incRef();
+            try {
+                final Map<String, BlobMetaData> blobs;
+                try {
+                    blobs = blobContainer.listBlobs();
+                } catch (IOException e) {
+                    throw new IndexShardSnapshotFailedException(shardId, "failed to list blobs", e);
+                }
+
+                long generation = findLatestFileNameGeneration(blobs);
+                Tuple<BlobStoreIndexShardSnapshots, Integer> tuple = buildBlobStoreIndexShardSnapshots(blobs);
+                BlobStoreIndexShardSnapshots snapshots = tuple.v1();
+                int fileListGeneration = tuple.v2();
+
+                final List<BlobStoreIndexShardSnapshot.FileInfo> indexCommitPointFiles = new ArrayList<>();
+
+                int indexNumberOfFiles = 0;
+                long indexTotalFilesSize = 0;
+                ArrayList<BlobStoreIndexShardSnapshot.FileInfo> filesToSnapshot = new ArrayList<>();
+                final Store.MetadataSnapshot metadata;
+                // TODO apparently we don't use the MetadataSnapshot#.recoveryDiff(...) here but we should
+                final Collection<String> fileNames;
+                try {
+                    metadata = store.getMetadata(snapshotIndexCommit);
+                    fileNames = snapshotIndexCommit.getFileNames();
+                } catch (IOException e) {
+                    throw new IndexShardSnapshotFailedException(shardId, "Failed to get store file metadata", e);
+                }
+                for (String fileName : fileNames) {
+                    if (snapshotStatus.aborted()) {
+                        logger.debug("[{}] [{}] Aborted on the file [{}], exiting", shardId, snapshotId, fileName);
+                        throw new IndexShardSnapshotFailedException(shardId, "Aborted");
+                    }
+                    logger.trace("[{}] [{}] Processing [{}]", shardId, snapshotId, fileName);
+                    final StoreFileMetaData md = metadata.get(fileName);
+                    BlobStoreIndexShardSnapshot.FileInfo existingFileInfo = null;
+                    List<BlobStoreIndexShardSnapshot.FileInfo> filesInfo = snapshots.findPhysicalIndexFiles(fileName);
+                    if (filesInfo != null) {
+                        for (BlobStoreIndexShardSnapshot.FileInfo fileInfo : filesInfo) {
+                            try {
+                                // in 1.3.3 we added additional hashes for .si / segments_N files
+                                // to ensure we don't double the space in the repo since old snapshots
+                                // don't have this hash we try to read that hash from the blob store
+                                // in a bwc compatible way.
+                                maybeRecalculateMetadataHash(blobContainer, fileInfo, metadata);
+                            } catch (Exception e) {
+                                logger.warn("{} Can't calculate hash from blob for file [{}] [{}]", e, shardId, fileInfo.physicalName(), fileInfo.metadata());
+                            }
+                            if (fileInfo.isSame(md) && snapshotFileExistsInBlobs(fileInfo, blobs)) {
+                                // a commit point file with the same name, size and checksum was already copied to repository
+                                // we will reuse it for this snapshot
+                                existingFileInfo = fileInfo;
+                                break;
+                            }
+                        }
+                    }
+                    if (existingFileInfo == null) {
+                        indexNumberOfFiles++;
+                        indexTotalFilesSize += md.length();
+                        // create a new FileInfo
+                        BlobStoreIndexShardSnapshot.FileInfo snapshotFileInfo = new BlobStoreIndexShardSnapshot.FileInfo(fileNameFromGeneration(++generation), md, chunkSize());
+                        indexCommitPointFiles.add(snapshotFileInfo);
+                        filesToSnapshot.add(snapshotFileInfo);
+                    } else {
+                        indexCommitPointFiles.add(existingFileInfo);
+                    }
+                }
+
+                snapshotStatus.files(indexNumberOfFiles, indexTotalFilesSize);
+
+                if (snapshotStatus.aborted()) {
+                    logger.debug("[{}] [{}] Aborted during initialization", shardId, snapshotId);
+                    throw new IndexShardSnapshotFailedException(shardId, "Aborted");
+                }
+
+                snapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.STARTED);
+
+                for (BlobStoreIndexShardSnapshot.FileInfo snapshotFileInfo : filesToSnapshot) {
+                    try {
+                        snapshotFile(snapshotFileInfo);
+                    } catch (IOException e) {
+                        throw new IndexShardSnapshotFailedException(shardId, "Failed to perform snapshot (index files)", e);
+                    }
+                }
+
+                snapshotStatus.indexVersion(snapshotIndexCommit.getGeneration());
+                // now create and write the commit point
+                snapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.FINALIZE);
+
+                BlobStoreIndexShardSnapshot snapshot = new BlobStoreIndexShardSnapshot(snapshotId.getName(),
+                    snapshotIndexCommit.getGeneration(), indexCommitPointFiles, snapshotStatus.startTime(),
+                    // snapshotStatus.startTime() is assigned on the same machine, so it's safe to use with VLong
+                    System.currentTimeMillis() - snapshotStatus.startTime(), indexNumberOfFiles, indexTotalFilesSize);
+                //TODO: The time stored in snapshot doesn't include cleanup time.
+                logger.trace("[{}] [{}] writing shard snapshot file", shardId, snapshotId);
+                try {
+                    indexShardSnapshotFormat.write(snapshot, blobContainer, snapshotId.getName());
+                } catch (IOException e) {
+                    throw new IndexShardSnapshotFailedException(shardId, "Failed to write commit point", e);
+                }
+
+                // delete all files that are not referenced by any commit point
+                // build a new BlobStoreIndexShardSnapshot, that includes this one and all the saved ones
+                List<SnapshotFiles> newSnapshotsList = new ArrayList<>();
+                newSnapshotsList.add(new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles()));
+                for (SnapshotFiles point : snapshots) {
+                    newSnapshotsList.add(point);
+                }
+                // finalize the snapshot and rewrite the snapshot index with the next sequential snapshot index
+                finalize(newSnapshotsList, fileListGeneration + 1, blobs);
+                snapshotStatus.updateStage(IndexShardSnapshotStatus.Stage.DONE);
+            } finally {
+                store.decRef();
+            }
+        }
+
+        /**
+         * Snapshot individual file
+         * <p>
+         * This is asynchronous method. Upon completion of the operation latch is getting counted down and any failures are
+         * added to the {@code failures} list
+         *
+         * @param fileInfo file to be snapshotted
+         */
+        private void snapshotFile(final BlobStoreIndexShardSnapshot.FileInfo fileInfo) throws IOException {
+            final String file = fileInfo.physicalName();
+            try (IndexInput indexInput = store.openVerifyingInput(file, IOContext.READONCE, fileInfo.metadata())) {
+                for (int i = 0; i < fileInfo.numberOfParts(); i++) {
+                    final long partBytes = fileInfo.partBytes(i);
+
+                    final InputStreamIndexInput inputStreamIndexInput = new InputStreamIndexInput(indexInput, partBytes);
+                    InputStream inputStream = inputStreamIndexInput;
+                    if (snapshotRateLimiter != null) {
+                        inputStream = new RateLimitingInputStream(inputStreamIndexInput, snapshotRateLimiter,
+                                                                  snapshotRateLimitingTimeInNanos::inc);
+                    }
+                    inputStream = new AbortableInputStream(inputStream, fileInfo.physicalName());
+                    blobContainer.writeBlob(fileInfo.partName(i), inputStream, partBytes);
+                }
+                Store.verify(indexInput);
+                snapshotStatus.addProcessedFile(fileInfo.length());
+            } catch (Exception t) {
+                failStoreIfCorrupted(t);
+                snapshotStatus.addProcessedFile(0);
+                throw t;
+            }
+        }
+
+        private void failStoreIfCorrupted(Exception e) {
+            if (e instanceof CorruptIndexException || e instanceof IndexFormatTooOldException || e instanceof IndexFormatTooNewException) {
+                try {
+                    store.markStoreCorrupted((IOException) e);
+                } catch (IOException inner) {
+                    inner.addSuppressed(e);
+                    logger.warn("store cannot be marked as corrupted", inner);
+                }
+            }
+        }
+
+        /**
+         * Checks if snapshot file already exists in the list of blobs
+         *
+         * @param fileInfo file to check
+         * @param blobs    list of blobs
+         * @return true if file exists in the list of blobs
+         */
+        private boolean snapshotFileExistsInBlobs(BlobStoreIndexShardSnapshot.FileInfo fileInfo, Map<String, BlobMetaData> blobs) {
+            BlobMetaData blobMetaData = blobs.get(fileInfo.name());
+            if (blobMetaData != null) {
+                return blobMetaData.length() == fileInfo.length();
+            } else if (blobs.containsKey(fileInfo.partName(0))) {
+                // multi part file sum up the size and check
+                int part = 0;
+                long totalSize = 0;
+                while (true) {
+                    blobMetaData = blobs.get(fileInfo.partName(part++));
+                    if (blobMetaData == null) {
+                        break;
+                    }
+                    totalSize += blobMetaData.length();
+                }
+                return totalSize == fileInfo.length();
+            }
+            // no file, not exact and not multipart
+            return false;
+        }
+
+        private class AbortableInputStream extends FilterInputStream {
+            private final String fileName;
+
+            public AbortableInputStream(InputStream delegate, String fileName) {
+                super(delegate);
+                this.fileName = fileName;
+            }
+
+            @Override
+            public int read() throws IOException {
+                checkAborted();
+                return in.read();
+            }
+
+            @Override
+            public int read(byte[] b, int off, int len) throws IOException {
+                checkAborted();
+                return in.read(b, off, len);
+            }
+
+            private void checkAborted() {
+                if (snapshotStatus.aborted()) {
+                    logger.debug("[{}] [{}] Aborted on the file [{}], exiting", shardId, snapshotId, fileName);
+                    throw new IndexShardSnapshotFailedException(shardId, "Aborted");
+                }
+            }
+        }
+    }
+
+    /**
+     * This is a BWC layer to ensure we update the snapshots metadata with the corresponding hashes before we compare them.
+     * The new logic for StoreFileMetaData reads the entire <tt>.si</tt> and <tt>segments.n</tt> files to strengthen the
+     * comparison of the files on a per-segment / per-commit level.
+     */
+    private static void maybeRecalculateMetadataHash(final BlobContainer blobContainer, final BlobStoreIndexShardSnapshot.FileInfo fileInfo, Store.MetadataSnapshot snapshot) throws Exception {
+        final StoreFileMetaData metadata;
+        if (fileInfo != null && (metadata = snapshot.get(fileInfo.physicalName())) != null) {
+            if (metadata.hash().length > 0 && fileInfo.metadata().hash().length == 0) {
+                // we have a hash - check if our repo has a hash too otherwise we have
+                // to calculate it.
+                // we might have multiple parts even though the file is small... make sure we read all of it.
+                try (final InputStream stream = new PartSliceStream(blobContainer, fileInfo)) {
+                    BytesRefBuilder builder = new BytesRefBuilder();
+                    Store.MetadataSnapshot.hashFile(builder, stream, fileInfo.length());
+                    BytesRef hash = fileInfo.metadata().hash(); // reset the file infos metadata hash
+                    assert hash.length == 0;
+                    hash.bytes = builder.bytes();
+                    hash.offset = 0;
+                    hash.length = builder.length();
+                }
+            }
+        }
+    }
+
+    private static final class PartSliceStream extends SlicedInputStream {
+
+        private final BlobContainer container;
+        private final BlobStoreIndexShardSnapshot.FileInfo info;
+
+        public PartSliceStream(BlobContainer container, BlobStoreIndexShardSnapshot.FileInfo info) {
+            super(info.numberOfParts());
+            this.info = info;
+            this.container = container;
+        }
+
+        @Override
+        protected InputStream openSlice(long slice) throws IOException {
+            return container.readBlob(info.partName(slice));
+        }
+    }
+
+    /**
+     * Context for restore operations
+     */
+    private class RestoreContext extends Context {
+
+        private final Store store;
+
+        private final RecoveryState recoveryState;
+
+        /**
+         * Constructs new restore context
+         *
+         * @param shard           shard to restore into
+         * @param snapshotId      snapshot id
+         * @param snapshotShardId shard in the snapshot that data should be restored from
+         * @param recoveryState   recovery state to report progress
+         */
+        public RestoreContext(IndexShard shard, SnapshotId snapshotId, Version version, ShardId snapshotShardId, RecoveryState recoveryState) {
+            super(snapshotId, version, shard.shardId(), snapshotShardId);
+            this.recoveryState = recoveryState;
+            store = shard.store();
+        }
+
+        /**
+         * Performs restore operation
+         */
+        public void restore() throws IOException {
+            store.incRef();
+            try {
+                logger.debug("[{}] [{}] restoring to [{}] ...", snapshotId, repositoryName, shardId);
+                BlobStoreIndexShardSnapshot snapshot = loadSnapshot();
+
+                if (snapshot.indexFiles().size() == 1
+                    && snapshot.indexFiles().get(0).physicalName().startsWith("segments_")
+                    && snapshot.indexFiles().get(0).hasUnknownChecksum()) {
+                    // If the shard has no documents, it will only contain a single segments_N file for the
+                    // shard's snapshot.  If we are restoring a snapshot created by a previous supported version,
+                    // it is still possible that in that version, an empty shard has a segments_N file with an unsupported
+                    // version (and no checksum), because we don't know the Lucene version to assign segments_N until we
+                    // have written some data.  Since the segments_N for an empty shard could have an incompatible Lucene
+                    // version number and no checksum, even though the index itself is perfectly fine to restore, this
+                    // empty shard would cause exceptions to be thrown.  Since there is no data to restore from an empty
+                    // shard anyway, we just create the empty shard here and then exit.
+                    IndexWriter writer = new IndexWriter(store.directory(), new IndexWriterConfig(null)
+                        .setOpenMode(IndexWriterConfig.OpenMode.CREATE)
+                        .setCommitOnClose(true));
+                    writer.close();
+                    return;
+                }
+
+                SnapshotFiles snapshotFiles = new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles());
+                final Store.MetadataSnapshot recoveryTargetMetadata;
+                try {
+                    recoveryTargetMetadata = store.getMetadataOrEmpty();
+                } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException e) {
+                    logger.warn("{} Can't read metadata from store", e, shardId);
+                    throw new IndexShardRestoreFailedException(shardId, "Can't restore corrupted shard", e);
+                }
+
+                final List<BlobStoreIndexShardSnapshot.FileInfo> filesToRecover = new ArrayList<>();
+                final Map<String, StoreFileMetaData> snapshotMetaData = new HashMap<>();
+                final Map<String, BlobStoreIndexShardSnapshot.FileInfo> fileInfos = new HashMap<>();
+                for (final BlobStoreIndexShardSnapshot.FileInfo fileInfo : snapshot.indexFiles()) {
+                    try {
+                        // in 1.3.3 we added additional hashes for .si / segments_N files
+                        // to ensure we don't double the space in the repo since old snapshots
+                        // don't have this hash we try to read that hash from the blob store
+                        // in a bwc compatible way.
+                        maybeRecalculateMetadataHash(blobContainer, fileInfo, recoveryTargetMetadata);
+                    } catch (Exception e) {
+                        // if the index is broken we might not be able to read it
+                        logger.warn("{} Can't calculate hash from blog for file [{}] [{}]", e, shardId, fileInfo.physicalName(), fileInfo.metadata());
+                    }
+                    snapshotMetaData.put(fileInfo.metadata().name(), fileInfo.metadata());
+                    fileInfos.put(fileInfo.metadata().name(), fileInfo);
+                }
+                final Store.MetadataSnapshot sourceMetaData = new Store.MetadataSnapshot(unmodifiableMap(snapshotMetaData), emptyMap(), 0);
+                final Store.RecoveryDiff diff = sourceMetaData.recoveryDiff(recoveryTargetMetadata);
+                for (StoreFileMetaData md : diff.identical) {
+                    BlobStoreIndexShardSnapshot.FileInfo fileInfo = fileInfos.get(md.name());
+                    recoveryState.getIndex().addFileDetail(fileInfo.name(), fileInfo.length(), true);
+                    if (logger.isTraceEnabled()) {
+                        logger.trace("[{}] [{}] not_recovering [{}] from [{}], exists in local store and is same", shardId, snapshotId, fileInfo.physicalName(), fileInfo.name());
+                    }
+                }
+
+                for (StoreFileMetaData md : Iterables.concat(diff.different, diff.missing)) {
+                    BlobStoreIndexShardSnapshot.FileInfo fileInfo = fileInfos.get(md.name());
+                    filesToRecover.add(fileInfo);
+                    recoveryState.getIndex().addFileDetail(fileInfo.name(), fileInfo.length(), false);
+                    if (logger.isTraceEnabled()) {
+                        if (md == null) {
+                            logger.trace("[{}] [{}] recovering [{}] from [{}], does not exists in local store", shardId, snapshotId, fileInfo.physicalName(), fileInfo.name());
+                        } else {
+                            logger.trace("[{}] [{}] recovering [{}] from [{}], exists in local store but is different", shardId, snapshotId, fileInfo.physicalName(), fileInfo.name());
+                        }
+                    }
+                }
+                final RecoveryState.Index index = recoveryState.getIndex();
+                if (filesToRecover.isEmpty()) {
+                    logger.trace("no files to recover, all exists within the local store");
+                }
+
+                if (logger.isTraceEnabled()) {
+                    logger.trace("[{}] [{}] recovering_files [{}] with total_size [{}], reusing_files [{}] with reused_size [{}]", shardId, snapshotId,
+                        index.totalRecoverFiles(), new ByteSizeValue(index.totalRecoverBytes()), index.reusedFileCount(), new ByteSizeValue(index.reusedFileCount()));
+                }
+                try {
+                    for (final BlobStoreIndexShardSnapshot.FileInfo fileToRecover : filesToRecover) {
+                        logger.trace("[{}] [{}] restoring file [{}]", shardId, snapshotId, fileToRecover.name());
+                        restoreFile(fileToRecover);
+                    }
+                } catch (IOException ex) {
+                    throw new IndexShardRestoreFailedException(shardId, "Failed to recover index", ex);
+                }
+                final StoreFileMetaData restoredSegmentsFile = sourceMetaData.getSegmentsFile();
+                if (recoveryTargetMetadata == null) {
+                    throw new IndexShardRestoreFailedException(shardId, "Snapshot has no segments file");
+                }
+                assert restoredSegmentsFile != null;
+                // read the snapshot data persisted
+                final SegmentInfos segmentCommitInfos;
+                try {
+                    segmentCommitInfos = Lucene.pruneUnreferencedFiles(restoredSegmentsFile.name(), store.directory());
+                } catch (IOException e) {
+                    throw new IndexShardRestoreFailedException(shardId, "Failed to fetch index version after copying it over", e);
+                }
+                recoveryState.getIndex().updateVersion(segmentCommitInfos.getVersion());
+
+                /// now, go over and clean files that are in the store, but were not in the snapshot
+                try {
+                    for (String storeFile : store.directory().listAll()) {
+                        if (Store.isAutogenerated(storeFile) || snapshotFiles.containPhysicalIndexFile(storeFile)) {
+                            continue; //skip write.lock, checksum files and files that exist in the snapshot
+                        }
+                        try {
+                            store.deleteQuiet("restore", storeFile);
+                            store.directory().deleteFile(storeFile);
+                        } catch (IOException e) {
+                            logger.warn("[{}] failed to delete file [{}] during snapshot cleanup", snapshotId, storeFile);
+                        }
+                    }
+                } catch (IOException e) {
+                    logger.warn("[{}] failed to list directory - some of files might not be deleted", snapshotId);
+                }
+            } finally {
+                store.decRef();
+            }
+        }
+
+        /**
+         * Restores a file
+         * This is asynchronous method. Upon completion of the operation latch is getting counted down and any failures are
+         * added to the {@code failures} list
+         *
+         * @param fileInfo file to be restored
+         */
+        private void restoreFile(final BlobStoreIndexShardSnapshot.FileInfo fileInfo) throws IOException {
+            boolean success = false;
+
+            try (InputStream partSliceStream = new PartSliceStream(blobContainer, fileInfo)) {
+                final InputStream stream;
+                if (restoreRateLimiter == null) {
+                    stream = partSliceStream;
+                } else {
+                    stream = new RateLimitingInputStream(partSliceStream, restoreRateLimiter, restoreRateLimitingTimeInNanos::inc);
+                }
+                try (final IndexOutput indexOutput = store.createVerifyingOutput(fileInfo.physicalName(), fileInfo.metadata(), IOContext.DEFAULT)) {
+                    final byte[] buffer = new byte[BUFFER_SIZE];
+                    int length;
+                    while ((length = stream.read(buffer)) > 0) {
+                        indexOutput.writeBytes(buffer, 0, length);
+                        recoveryState.getIndex().addRecoveredBytesToFile(fileInfo.name(), length);
+                    }
+                    Store.verify(indexOutput);
+                    indexOutput.close();
+                    store.directory().sync(Collections.singleton(fileInfo.physicalName()));
+                    success = true;
+                } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
+                    try {
+                        store.markStoreCorrupted(ex);
+                    } catch (IOException e) {
+                        logger.warn("store cannot be marked as corrupted", e);
+                    }
+                    throw ex;
+                } finally {
+                    if (success == false) {
+                        store.deleteQuiet(fileInfo.physicalName());
+                    }
+                }
+            }
+        }
+
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java b/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
index 4e57c431f14..5be7115a25b 100644
--- a/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
@@ -27,7 +27,6 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.index.snapshots.IndexShardRepository;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.RepositoryName;
 import org.elasticsearch.repositories.RepositorySettings;
@@ -77,11 +76,10 @@ public class FsRepository extends BlobStoreRepository {
      *
      * @param name                 repository name
      * @param repositorySettings   repository settings
-     * @param indexShardRepository index shard repository
      */
     @Inject
-    public FsRepository(RepositoryName name, RepositorySettings repositorySettings, IndexShardRepository indexShardRepository, Environment environment) throws IOException {
-        super(name.getName(), repositorySettings, indexShardRepository);
+    public FsRepository(RepositoryName name, RepositorySettings repositorySettings, Environment environment) throws IOException {
+        super(name.getName(), repositorySettings);
         Path locationFile;
         String location = REPOSITORIES_LOCATION_SETTING.get(repositorySettings.settings());
         if (location.isEmpty()) {
diff --git a/core/src/main/java/org/elasticsearch/repositories/uri/URLIndexShardRepository.java b/core/src/main/java/org/elasticsearch/repositories/uri/URLIndexShardRepository.java
deleted file mode 100644
index 616a36d5066..00000000000
--- a/core/src/main/java/org/elasticsearch/repositories/uri/URLIndexShardRepository.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.repositories.uri;
-
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository;
-import org.elasticsearch.indices.IndicesService;
-import org.elasticsearch.repositories.RepositoryName;
-
-/**
- */
-public class URLIndexShardRepository extends BlobStoreIndexShardRepository {
-
-    @Inject
-    public URLIndexShardRepository(Settings settings, RepositoryName repositoryName, IndicesService indicesService, ClusterService clusterService) {
-        super(settings, repositoryName, indicesService, clusterService);
-    }
-
-
-    @Override
-    public void verify(String seed) {
-        //TODO: Add verification that URL is accessible
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java b/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
index 0eb3006c5c3..f08faa2c0d6 100644
--- a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
@@ -27,7 +27,6 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.util.URIPattern;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.index.snapshots.IndexShardRepository;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.RepositoryName;
 import org.elasticsearch.repositories.RepositorySettings;
@@ -82,11 +81,10 @@ public class URLRepository extends BlobStoreRepository {
      *
      * @param name                 repository name
      * @param repositorySettings   repository settings
-     * @param indexShardRepository shard repository
      */
     @Inject
-    public URLRepository(RepositoryName name, RepositorySettings repositorySettings, IndexShardRepository indexShardRepository, Environment environment) throws IOException {
-        super(name.getName(), repositorySettings, indexShardRepository);
+    public URLRepository(RepositoryName name, RepositorySettings repositorySettings, Environment environment) throws IOException {
+        super(name.getName(), repositorySettings);
 
         if (URL_SETTING.exists(repositorySettings.settings()) == false && REPOSITORIES_URL_SETTING.exists(settings) ==  false) {
             throw new RepositoryException(name.name(), "missing url");
@@ -101,9 +99,6 @@ public class URLRepository extends BlobStoreRepository {
         basePath = BlobPath.cleanPath();
     }
 
-    /**
-     * {@inheritDoc}
-     */
     @Override
     protected BlobStore blobStore() {
         return blobStore;
diff --git a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
index 53fb6f4781e..67a5ae361bb 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
@@ -63,7 +63,6 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.index.snapshots.IndexShardRepository;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -112,7 +111,7 @@ import static org.elasticsearch.common.util.set.Sets.newHashSet;
  * method.
  * <p>
  * Individual shards are getting restored as part of normal recovery process in
- * {@link IndexShard#restoreFromRepository(IndexShardRepository)} )}
+ * {@link IndexShard#restoreFromRepository(Repository)} )}
  * method, which detects that shard should be restored from snapshot rather than recovered from gateway by looking
  * at the {@link org.elasticsearch.cluster.routing.ShardRouting#restoreSource()} property.
  * <p>
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
index 91db508b7e3..b24627bb634 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
@@ -43,10 +43,10 @@ import org.elasticsearch.index.engine.SnapshotFailedEngineException;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.shard.IndexShardState;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.index.snapshots.IndexShardRepository;
 import org.elasticsearch.index.snapshots.IndexShardSnapshotFailedException;
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.EmptyTransportResponseHandler;
 import org.elasticsearch.transport.TransportChannel;
@@ -322,7 +322,7 @@ public class SnapshotShardsService extends AbstractLifecycleComponent implements
      * @param snapshotStatus snapshot status
      */
     private void snapshot(final IndexShard indexShard, final Snapshot snapshot, final IndexShardSnapshotStatus snapshotStatus) {
-        IndexShardRepository indexShardRepository = snapshotsService.getRepositoriesService().indexShardRepository(snapshot.getRepository());
+        Repository repository = snapshotsService.getRepositoriesService().repository(snapshot.getRepository());
         ShardId shardId = indexShard.shardId();
         if (!indexShard.routingEntry().primary()) {
             throw new IndexShardSnapshotFailedException(shardId, "snapshot should be performed only on primary");
@@ -340,11 +340,11 @@ public class SnapshotShardsService extends AbstractLifecycleComponent implements
             // we flush first to make sure we get the latest writes snapshotted
             IndexCommit snapshotIndexCommit = indexShard.snapshotIndex(true);
             try {
-                indexShardRepository.snapshot(snapshot.getSnapshotId(), shardId, snapshotIndexCommit, snapshotStatus);
+                repository.snapshot(indexShard, snapshot.getSnapshotId(), snapshotIndexCommit, snapshotStatus);
                 if (logger.isDebugEnabled()) {
                     StringBuilder sb = new StringBuilder();
                     sb.append("    index    : version [").append(snapshotStatus.indexVersion()).append("], number_of_files [").append(snapshotStatus.numberOfFiles()).append("] with total_size [").append(new ByteSizeValue(snapshotStatus.totalSize())).append("]\n");
-                    logger.debug("snapshot ({}) completed to {}, took [{}]\n{}", snapshot, indexShardRepository,
+                    logger.debug("snapshot ({}) completed to {}, took [{}]\n{}", snapshot, repository,
                         TimeValue.timeValueMillis(snapshotStatus.time()), sb);
                 }
             } finally {
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
index 5f5ac65e38e..8d2407dd2e4 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
@@ -55,7 +55,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.index.snapshots.IndexShardRepository;
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.Repository;
@@ -547,7 +546,6 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                                                                  final SnapshotInfo snapshotInfo) throws IOException {
         Map<ShardId, IndexShardSnapshotStatus> shardStatus = new HashMap<>();
         Repository repository = repositoriesService.repository(repositoryName);
-        IndexShardRepository indexShardRepository = repositoriesService.indexShardRepository(repositoryName);
         MetaData metaData = repository.readSnapshotMetaData(snapshotInfo, snapshotInfo.indices());
         for (String index : snapshotInfo.indices()) {
             IndexMetaData indexMetaData = metaData.indices().get(index);
@@ -563,7 +561,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                         shardStatus.put(shardId, shardSnapshotStatus);
                     } else {
                         IndexShardSnapshotStatus shardSnapshotStatus =
-                            indexShardRepository.snapshotStatus(snapshotInfo.snapshotId(), snapshotInfo.version(), shardId);
+                            repository.snapshotStatus(snapshotInfo.snapshotId(), snapshotInfo.version(), shardId);
                         shardStatus.put(shardId, shardSnapshotStatus);
                     }
                 }
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index a0813fb572f..acec53da24b 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -18,6 +18,27 @@
  */
 package org.elasticsearch.index.shard;
 
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.BiConsumer;
+
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.index.CorruptIndexException;
@@ -50,6 +71,7 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.InternalClusterInfoService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.AllocationId;
 import org.elasticsearch.cluster.routing.RestoreSource;
@@ -63,6 +85,7 @@ import org.elasticsearch.common.UUIDs;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.lease.Releasable;
@@ -91,15 +114,17 @@ import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.index.mapper.Uid;
 import org.elasticsearch.index.mapper.internal.UidFieldMapper;
-import org.elasticsearch.index.snapshots.IndexShardRepository;
 import org.elasticsearch.index.snapshots.IndexShardSnapshotStatus;
 import org.elasticsearch.index.store.Store;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.snapshots.Snapshot;
 import org.elasticsearch.snapshots.SnapshotId;
+import org.elasticsearch.snapshots.SnapshotInfo;
+import org.elasticsearch.snapshots.SnapshotShardFailure;
 import org.elasticsearch.test.DummyShardLock;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.elasticsearch.test.FieldMaskingReader;
@@ -108,27 +133,6 @@ import org.elasticsearch.test.InternalSettingsPlugin;
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.threadpool.ThreadPool;
 
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.BrokenBarrierException;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.BiConsumer;
-
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
@@ -1178,13 +1182,9 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         test_target_shard.updateRoutingEntry(routing);
         DiscoveryNode localNode = new DiscoveryNode("foo", LocalTransportAddress.buildUnique(), emptyMap(), emptySet(), Version.CURRENT);
         test_target_shard.markAsRecovering("store", new RecoveryState(routing.shardId(), routing.primary(), RecoveryState.Type.SNAPSHOT, routing.restoreSource(), localNode));
-        assertTrue(test_target_shard.restoreFromRepository(new IndexShardRepository() {
+        assertTrue(test_target_shard.restoreFromRepository(new RestoreOnlyRepository() {
             @Override
-            public void snapshot(SnapshotId snapshotId, ShardId shardId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus) {
-            }
-
-            @Override
-            public void restore(SnapshotId snapshotId, Version version, ShardId shardId, ShardId snapshotShardId, RecoveryState recoveryState) {
+            public void restore(IndexShard shard, SnapshotId snapshotId, Version version, ShardId snapshotShardId, RecoveryState recoveryState) {
                 try {
                     cleanLuceneIndex(targetStore.directory());
                     for (String file : sourceStore.directory().listAll()) {
@@ -1197,15 +1197,6 @@ public class IndexShardTests extends ESSingleNodeTestCase {
                     throw new RuntimeException(ex);
                 }
             }
-
-            @Override
-            public IndexShardSnapshotStatus snapshotStatus(SnapshotId snapshotId, Version version, ShardId shardId) {
-                return null;
-            }
-
-            @Override
-            public void verify(String verificationToken) {
-            }
         }));
 
         test_target_shard.updateRoutingEntry(routing.moveToStarted());
@@ -1649,4 +1640,63 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         assertEquals(mappings.get("index_1").get("test").get().source().string(), "{\"test\":{\"properties\":{\"foo\":{\"type\":\"text\"}}}}");
 
     }
+
+    /** A dummy repository for testing which just needs restore overridden */
+    private static abstract class RestoreOnlyRepository extends AbstractLifecycleComponent implements Repository {
+        public RestoreOnlyRepository() {
+            super(Settings.EMPTY);
+        }
+        @Override
+        protected void doStart() {}
+        @Override
+        protected void doStop() {}
+        @Override
+        protected void doClose() {}
+        @Override
+        public SnapshotInfo readSnapshot(SnapshotId snapshotId) {
+            return null;
+        }
+        @Override
+        public MetaData readSnapshotMetaData(SnapshotInfo snapshot, List<String> indices) throws IOException {
+            return null;
+        }
+        @Override
+        public List<SnapshotId> snapshots() {
+            return null;
+        }
+        @Override
+        public void initializeSnapshot(SnapshotId snapshotId, List<String> indices, MetaData metaData) {}
+        @Override
+        public SnapshotInfo finalizeSnapshot(SnapshotId snapshotId, List<String> indices, long startTime, String failure, int totalShards, List<SnapshotShardFailure> shardFailures) {
+            return null;
+        }
+        @Override
+        public void deleteSnapshot(SnapshotId snapshotId) {}
+        @Override
+        public long snapshotThrottleTimeInNanos() {
+            return 0;
+        }
+        @Override
+        public long restoreThrottleTimeInNanos() {
+            return 0;
+        }
+        @Override
+        public String startVerification() {
+            return null;
+        }
+        @Override
+        public void endVerification(String verificationToken) {}
+        @Override
+        public boolean readOnly() {
+            return false;
+        }
+        @Override
+        public void snapshot(IndexShard shard, SnapshotId snapshotId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus) {}
+        @Override
+        public IndexShardSnapshotStatus snapshotStatus(SnapshotId snapshotId, Version version, ShardId shardId) {
+            return null;
+        }
+        @Override
+        public void verify(String verificationToken, DiscoveryNode localNode) {}
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java b/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java
index 48a1cc6081e..639b60d6d09 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java
@@ -44,8 +44,6 @@ import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 
-/**
- */
 @ESIntegTestCase.ClusterScope(minNumDataNodes = 2)
 public class RepositoriesIT extends AbstractSnapshotIntegTestCase {
     public void testRepositoryCreation() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java b/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
index bb6f5cc4f74..762be7a3684 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
@@ -19,29 +19,6 @@
 
 package org.elasticsearch.snapshots.mockstore;
 
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.snapshots.SnapshotId;
-import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.blobstore.BlobContainer;
-import org.elasticsearch.common.blobstore.BlobMetaData;
-import org.elasticsearch.common.blobstore.BlobPath;
-import org.elasticsearch.common.blobstore.BlobStore;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.io.PathUtils;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Property;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.index.snapshots.IndexShardRepository;
-import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository;
-import org.elasticsearch.repositories.RepositoriesModule;
-import org.elasticsearch.repositories.RepositoryName;
-import org.elasticsearch.repositories.RepositorySettings;
-import org.elasticsearch.repositories.fs.FsRepository;
-
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
@@ -55,6 +32,26 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicLong;
 
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.blobstore.BlobContainer;
+import org.elasticsearch.common.blobstore.BlobMetaData;
+import org.elasticsearch.common.blobstore.BlobPath;
+import org.elasticsearch.common.blobstore.BlobStore;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.repositories.RepositoriesModule;
+import org.elasticsearch.repositories.RepositoryName;
+import org.elasticsearch.repositories.RepositorySettings;
+import org.elasticsearch.repositories.fs.FsRepository;
+import org.elasticsearch.snapshots.SnapshotId;
+
 public class MockRepository extends FsRepository {
 
     public static class Plugin extends org.elasticsearch.plugins.Plugin {
@@ -64,7 +61,7 @@ public class MockRepository extends FsRepository {
             Setting.simpleString("secret.mock.password", Property.NodeScope, Property.Filtered);
 
         public void onModule(RepositoriesModule repositoriesModule) {
-            repositoriesModule.registerRepository("mock", MockRepository.class, BlobStoreIndexShardRepository.class);
+            repositoriesModule.registerRepository("mock", MockRepository.class);
         }
 
         @Override
@@ -100,8 +97,8 @@ public class MockRepository extends FsRepository {
     private volatile boolean blocked = false;
 
     @Inject
-    public MockRepository(RepositoryName name, RepositorySettings repositorySettings, IndexShardRepository indexShardRepository, ClusterService clusterService, Environment environment) throws IOException {
-        super(name, overrideSettings(repositorySettings, clusterService), indexShardRepository, environment);
+    public MockRepository(RepositoryName name, RepositorySettings repositorySettings, ClusterService clusterService, Environment environment) throws IOException {
+        super(name, overrideSettings(repositorySettings, clusterService), environment);
         randomControlIOExceptionRate = repositorySettings.settings().getAsDouble("random_control_io_exception_rate", 0.0);
         randomDataFileIOExceptionRate = repositorySettings.settings().getAsDouble("random_data_file_io_exception_rate", 0.0);
         maximumNumberOfFailures = repositorySettings.settings().getAsLong("max_failure_number", 100L);
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java b/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
index b04b613df21..e7701a828ac 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
@@ -19,6 +19,11 @@
 
 package org.elasticsearch.plugin.repository.azure;
 
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
 import org.elasticsearch.cloud.azure.AzureRepositoryModule;
 import org.elasticsearch.cloud.azure.storage.AzureStorageService;
 import org.elasticsearch.common.inject.Module;
@@ -26,16 +31,10 @@ import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.repositories.RepositoriesModule;
 import org.elasticsearch.repositories.azure.AzureRepository;
 
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
 /**
  *
  */
@@ -56,7 +55,7 @@ public class AzureRepositoryPlugin extends Plugin {
 
     public void onModule(RepositoriesModule module) {
         logger.debug("registering repository type [{}]", AzureRepository.TYPE);
-        module.registerRepository(AzureRepository.TYPE, AzureRepository.class, BlobStoreIndexShardRepository.class);
+        module.registerRepository(AzureRepository.TYPE, AzureRepository.class);
     }
 
     @Override
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
index 1333c755e7c..47ec4128c7c 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
@@ -34,7 +34,6 @@ import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.index.snapshots.IndexShardRepository;
 import org.elasticsearch.repositories.RepositoryName;
 import org.elasticsearch.repositories.RepositorySettings;
 import org.elasticsearch.repositories.RepositoryVerificationException;
@@ -86,9 +85,8 @@ public class AzureRepository extends BlobStoreRepository {
 
     @Inject
     public AzureRepository(RepositoryName name, RepositorySettings repositorySettings,
-                           IndexShardRepository indexShardRepository,
                            AzureBlobStore azureBlobStore) throws IOException, URISyntaxException, StorageException {
-        super(name.getName(), repositorySettings, indexShardRepository);
+        super(name.getName(), repositorySettings);
 
         String container = getValue(repositorySettings, Repository.CONTAINER_SETTING, Storage.CONTAINER_SETTING);
 
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStoragePlugin.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStoragePlugin.java
index 77d8e23c9e5..33022ff449e 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStoragePlugin.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStoragePlugin.java
@@ -19,6 +19,11 @@
 
 package org.elasticsearch.plugin.repository.gcs;
 
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.Collection;
+import java.util.Collections;
+
 import com.google.api.client.auth.oauth2.TokenRequest;
 import com.google.api.client.auth.oauth2.TokenResponse;
 import com.google.api.client.googleapis.json.GoogleJsonError;
@@ -35,16 +40,10 @@ import com.google.api.services.storage.model.Objects;
 import com.google.api.services.storage.model.StorageObject;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.inject.Module;
-import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.repositories.RepositoriesModule;
 import org.elasticsearch.repositories.gcs.GoogleCloudStorageRepository;
 
-import java.security.AccessController;
-import java.security.PrivilegedAction;
-import java.util.Collection;
-import java.util.Collections;
-
 public class GoogleCloudStoragePlugin extends Plugin {
 
     public static final String NAME = "repository-gcs";
@@ -115,7 +114,6 @@ public class GoogleCloudStoragePlugin extends Plugin {
     }
 
     public void onModule(RepositoriesModule repositoriesModule) {
-        repositoriesModule.registerRepository(GoogleCloudStorageRepository.TYPE,
-                GoogleCloudStorageRepository.class, BlobStoreIndexShardRepository.class);
+        repositoriesModule.registerRepository(GoogleCloudStorageRepository.TYPE, GoogleCloudStorageRepository.class);
     }
 }
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
index 3616e18e83d..df2f054fa91 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
@@ -29,7 +29,6 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.index.snapshots.IndexShardRepository;
 import org.elasticsearch.plugin.repository.gcs.GoogleCloudStoragePlugin;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.RepositoryName;
@@ -75,9 +74,8 @@ public class GoogleCloudStorageRepository extends BlobStoreRepository {
 
     @Inject
     public GoogleCloudStorageRepository(RepositoryName repositoryName, RepositorySettings repositorySettings,
-                                        IndexShardRepository indexShardRepository,
                                         GoogleCloudStorageService storageService) throws Exception {
-        super(repositoryName.getName(), repositorySettings, indexShardRepository);
+        super(repositoryName.getName(), repositorySettings);
 
         String bucket = get(BUCKET, repositoryName, repositorySettings);
         String application = get(APPLICATION_NAME, repositoryName, repositorySettings);
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java
index ab539aeea1c..3e8f484e3e7 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java
@@ -26,7 +26,6 @@ import java.security.PrivilegedAction;
 
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.SuppressForbidden;
-import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.repositories.RepositoriesModule;
 
@@ -85,6 +84,6 @@ public final class HdfsPlugin extends Plugin {
     }
 
     public void onModule(RepositoriesModule repositoriesModule) {
-        repositoriesModule.registerRepository("hdfs", HdfsRepository.class, BlobStoreIndexShardRepository.class);
+        repositoriesModule.registerRepository("hdfs", HdfsRepository.class);
     }
 }
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
index 1e8e267bd41..de8fb155abb 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
@@ -44,7 +44,6 @@ import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.index.snapshots.IndexShardRepository;
 import org.elasticsearch.repositories.RepositoryName;
 import org.elasticsearch.repositories.RepositorySettings;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
@@ -63,8 +62,8 @@ public final class HdfsRepository extends BlobStoreRepository {
     private static final ByteSizeValue DEFAULT_BUFFER_SIZE = new ByteSizeValue(100, ByteSizeUnit.KB);
 
     @Inject
-    public HdfsRepository(RepositoryName name, RepositorySettings repositorySettings, IndexShardRepository indexShardRepository) throws IOException {
-        super(name.getName(), repositorySettings, indexShardRepository);
+    public HdfsRepository(RepositoryName name, RepositorySettings repositorySettings) throws IOException {
+        super(name.getName(), repositorySettings);
         this.repositorySettings = repositorySettings;
 
         this.chunkSize = repositorySettings.settings().getAsBytesSize("chunk_size", null);
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
index 7d52a067b65..fc55e1b0ff1 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
@@ -19,18 +19,6 @@
 
 package org.elasticsearch.plugin.repository.s3;
 
-import org.elasticsearch.SpecialPermission;
-import org.elasticsearch.cloud.aws.AwsS3Service;
-import org.elasticsearch.cloud.aws.S3Module;
-import org.elasticsearch.common.component.LifecycleComponent;
-import org.elasticsearch.common.inject.Module;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.SettingsModule;
-import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardRepository;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.repositories.RepositoriesModule;
-import org.elasticsearch.repositories.s3.S3Repository;
-
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.ArrayList;
@@ -39,6 +27,16 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 
+import org.elasticsearch.SpecialPermission;
+import org.elasticsearch.cloud.aws.AwsS3Service;
+import org.elasticsearch.cloud.aws.S3Module;
+import org.elasticsearch.common.component.LifecycleComponent;
+import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.repositories.RepositoriesModule;
+import org.elasticsearch.repositories.s3.S3Repository;
+
 /**
  *
  */
@@ -78,7 +76,7 @@ public class S3RepositoryPlugin extends Plugin {
     }
 
     public void onModule(RepositoriesModule repositoriesModule) {
-        repositoriesModule.registerRepository(S3Repository.TYPE, S3Repository.class, BlobStoreIndexShardRepository.class);
+        repositoriesModule.registerRepository(S3Repository.TYPE, S3Repository.class);
     }
 
     @Override
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
index e3b7e8296e0..182612041f1 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
@@ -32,7 +32,6 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.index.snapshots.IndexShardRepository;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.RepositoryName;
 import org.elasticsearch.repositories.RepositorySettings;
@@ -250,12 +249,11 @@ public class S3Repository extends BlobStoreRepository {
      *
      * @param name                 repository name
      * @param repositorySettings   repository settings
-     * @param indexShardRepository index shard repository
      * @param s3Service            S3 service
      */
     @Inject
-    public S3Repository(RepositoryName name, RepositorySettings repositorySettings, IndexShardRepository indexShardRepository, AwsS3Service s3Service) throws IOException {
-        super(name.getName(), repositorySettings, indexShardRepository);
+    public S3Repository(RepositoryName name, RepositorySettings repositorySettings, AwsS3Service s3Service) throws IOException {
+        super(name.getName(), repositorySettings);
 
         String bucket = getValue(repositorySettings, Repository.BUCKET_SETTING, Repositories.BUCKET_SETTING);
         if (bucket == null) {
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java
index 4cb8e4d3abb..c11cb969570 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java
@@ -336,7 +336,7 @@ public class RepositoryS3SettingsTests extends ESTestCase {
             .build());
 
         try {
-            new S3Repository(new RepositoryName("s3", "s3repo"), s3RepositorySettings, null, null);
+            new S3Repository(new RepositoryName("s3", "s3repo"), s3RepositorySettings, null);
             fail("We should either raise a NPE or a RepositoryException or a IllegalArgumentException");
         } catch (RepositoryException e) {
             assertThat(e.getDetailedMessage(), containsString(expectedMessage));

From b3105bd316f29666d0dbbbc36630911a6505c64e Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Wed, 6 Jul 2016 20:09:59 -0400
Subject: [PATCH 1261/1311] Fix modifier order in BytesStreamsTests

This commit fixes an issue with the ordering of modifiers on a static
nested class in BytesStreamsTests.
---
 .../org/elasticsearch/common/io/stream/BytesStreamsTests.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
index ee16eb82a2c..f0cc617299a 100644
--- a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
+++ b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java
@@ -503,7 +503,7 @@ public class BytesStreamsTests extends ESTestCase {
         out.close();
     }
 
-    private static abstract class BaseNamedWriteable implements NamedWriteable {
+    private abstract static class BaseNamedWriteable implements NamedWriteable {
 
     }
 

From 71b48fb16c1ba61820a7d87011a9e4641ed2d060 Mon Sep 17 00:00:00 2001
From: Alexander Reelsen <alexander@reelsen.net>
Date: Thu, 7 Jul 2016 08:50:10 +0200
Subject: [PATCH 1262/1311] Dependencies: Update to jopt-5.0 (#19278)

The new version of jopt allows us to remove a couple of TODOs in the code.

Closes #12368
---
 core/build.gradle                             |  2 +-
 .../elasticsearch/bootstrap/Bootstrap.java    |  7 +++---
 .../bootstrap/Elasticsearch.java              | 23 +++++++++++--------
 .../java/org/elasticsearch/cli/Command.java   |  7 ++----
 .../plugins/InstallPluginCommand.java         |  7 +-----
 .../plugins/RemovePluginCommand.java          |  8 ++-----
 .../bootstrap/ElasticsearchCliTests.java      | 19 ++++++++++-----
 .../org/elasticsearch/cli/CommandTests.java   |  6 +++--
 .../licenses/jopt-simple-4.9.jar.sha1         |  1 -
 .../licenses/jopt-simple-5.0.2.jar.sha1       |  1 +
 .../bootstrap/ESElasticsearchCliTestCase.java |  5 ++--
 11 files changed, 44 insertions(+), 42 deletions(-)
 delete mode 100644 distribution/licenses/jopt-simple-4.9.jar.sha1
 create mode 100644 distribution/licenses/jopt-simple-5.0.2.jar.sha1

diff --git a/core/build.gradle b/core/build.gradle
index 22318801766..513e998ebdc 100644
--- a/core/build.gradle
+++ b/core/build.gradle
@@ -59,7 +59,7 @@ dependencies {
   compile 'org.elasticsearch:securesm:1.0'
 
   // utilities
-  compile 'net.sf.jopt-simple:jopt-simple:4.9'
+  compile 'net.sf.jopt-simple:jopt-simple:5.0.2'
   compile 'com.carrotsearch:hppc:0.7.1'
 
   // time handling, remove with java 8 time
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
index 4168e34a74b..3d5347307d6 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
@@ -26,7 +26,6 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.Version;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.common.PidFile;
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.inject.CreationException;
 import org.elasticsearch.common.logging.ESLogger;
@@ -180,10 +179,10 @@ final class Bootstrap {
         };
     }
 
-    private static Environment initialSettings(boolean foreground, String pidFile, Map<String, String> esSettings) {
+    private static Environment initialSettings(boolean foreground, Path pidFile, Map<String, String> esSettings) {
         Terminal terminal = foreground ? Terminal.DEFAULT : null;
         Settings.Builder builder = Settings.builder();
-        if (Strings.hasLength(pidFile)) {
+        if (pidFile != null) {
             builder.put(Environment.PIDFILE_SETTING.getKey(), pidFile);
         }
         return InternalSettingsPreparer.prepareEnvironment(builder.build(), terminal, esSettings);
@@ -215,7 +214,7 @@ final class Bootstrap {
      */
     static void init(
             final boolean foreground,
-            final String pidFile,
+            final Path pidFile,
             final Map<String, String> esSettings) throws Exception {
         // Set the system property before anything has a chance to trigger its use
         initLoggerPrefix();
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
index 94ff59ee9ab..9c76fdfb030 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java
@@ -21,6 +21,9 @@ package org.elasticsearch.bootstrap;
 
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
+import joptsimple.OptionSpecBuilder;
+import joptsimple.util.PathConverter;
+import joptsimple.util.PathProperties;
 import org.elasticsearch.Build;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.cli.SettingCommand;
@@ -29,6 +32,7 @@ import org.elasticsearch.cli.UserException;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 
 import java.io.IOException;
+import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.Map;
 
@@ -37,22 +41,23 @@ import java.util.Map;
  */
 class Elasticsearch extends SettingCommand {
 
-    private final OptionSpec<Void> versionOption;
-    private final OptionSpec<Void> daemonizeOption;
-    private final OptionSpec<String> pidfileOption;
+    private final OptionSpecBuilder versionOption;
+    private final OptionSpecBuilder daemonizeOption;
+    private final OptionSpec<Path> pidfileOption;
 
     // visible for testing
     Elasticsearch() {
         super("starts elasticsearch");
-        // TODO: in jopt-simple 5.0, make this mutually exclusive with all other options
         versionOption = parser.acceptsAll(Arrays.asList("V", "version"),
             "Prints elasticsearch version information and exits");
         daemonizeOption = parser.acceptsAll(Arrays.asList("d", "daemonize"),
-            "Starts Elasticsearch in the background");
-        // TODO: in jopt-simple 5.0 this option type can be a Path
+            "Starts Elasticsearch in the background")
+            .availableUnless(versionOption);
         pidfileOption = parser.acceptsAll(Arrays.asList("p", "pidfile"),
             "Creates a pid file in the specified path on start")
-            .withRequiredArg();
+            .availableUnless(versionOption)
+            .withRequiredArg()
+            .withValuesConvertedBy(new PathConverter());
     }
 
     /**
@@ -86,12 +91,12 @@ class Elasticsearch extends SettingCommand {
         }
 
         final boolean daemonize = options.has(daemonizeOption);
-        final String pidFile = pidfileOption.value(options);
+        final Path pidFile = pidfileOption.value(options);
 
         init(daemonize, pidFile, settings);
     }
 
-    void init(final boolean daemonize, final String pidFile, final Map<String, String> esSettings) {
+    void init(final boolean daemonize, final Path pidFile, final Map<String, String> esSettings) {
         try {
             Bootstrap.init(!daemonize, pidFile, esSettings);
         } catch (final Throwable t) {
diff --git a/core/src/main/java/org/elasticsearch/cli/Command.java b/core/src/main/java/org/elasticsearch/cli/Command.java
index 0b615c695e9..2e896759ebb 100644
--- a/core/src/main/java/org/elasticsearch/cli/Command.java
+++ b/core/src/main/java/org/elasticsearch/cli/Command.java
@@ -41,7 +41,8 @@ public abstract class Command {
 
     private final OptionSpec<Void> helpOption = parser.acceptsAll(Arrays.asList("h", "help"), "show help").forHelp();
     private final OptionSpec<Void> silentOption = parser.acceptsAll(Arrays.asList("s", "silent"), "show minimal output");
-    private final OptionSpec<Void> verboseOption = parser.acceptsAll(Arrays.asList("v", "verbose"), "show verbose output");
+    private final OptionSpec<Void> verboseOption = parser.acceptsAll(Arrays.asList("v", "verbose"), "show verbose output")
+            .availableUnless(silentOption);
 
     public Command(String description) {
         this.description = description;
@@ -77,10 +78,6 @@ public abstract class Command {
         }
 
         if (options.has(silentOption)) {
-            if (options.has(verboseOption)) {
-                // mutually exclusive, we can remove this with jopt-simple 5.0, which natively supports it
-                throw new UserException(ExitCodes.USAGE, "Cannot specify -s and -v together");
-            }
             terminal.setVerbosity(Terminal.Verbosity.SILENT);
         } else if (options.has(verboseOption)) {
             terminal.setVerbosity(Terminal.Verbosity.VERBOSE);
diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
index d0adda942a3..50ff0acb628 100644
--- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java
@@ -185,12 +185,7 @@ class InstallPluginCommand extends SettingCommand {
 
     @Override
     protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
-        // TODO: in jopt-simple 5.0 we can enforce a min/max number of positional args
-        List<String> args = arguments.values(options);
-        if (args.size() != 1) {
-            throw new UserException(ExitCodes.USAGE, "Must supply a single plugin id argument");
-        }
-        String pluginId = args.get(0);
+        String pluginId = arguments.value(options);
         boolean isBatch = options.has(batchOption) || System.console() == null;
         execute(terminal, pluginId, isBatch, settings);
     }
diff --git a/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java
index d6d39e39ed4..0b4f8d281d0 100644
--- a/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java
+++ b/core/src/main/java/org/elasticsearch/plugins/RemovePluginCommand.java
@@ -54,12 +54,8 @@ class RemovePluginCommand extends SettingCommand {
 
     @Override
     protected void execute(Terminal terminal, OptionSet options, Map<String, String> settings) throws Exception {
-        // TODO: in jopt-simple 5.0 we can enforce a min/max number of positional args
-        List<String> args = arguments.values(options);
-        if (args.size() != 1) {
-            throw new UserException(ExitCodes.USAGE, "Must supply a single plugin id argument");
-        }
-        execute(terminal, args.get(0), settings);
+        String arg = arguments.value(options);
+        execute(terminal, arg, settings);
     }
 
     // pkg private for testing
diff --git a/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java b/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java
index 8b8a4d947a9..f8bdf244999 100644
--- a/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java
+++ b/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchCliTests.java
@@ -24,10 +24,12 @@ import org.elasticsearch.Version;
 import org.elasticsearch.cli.ExitCodes;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 
+import java.nio.file.Path;
 import java.util.function.Consumer;
 
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.hasEntry;
 
 public class ElasticsearchCliTests extends ESElasticsearchCliTestCase {
@@ -50,7 +52,8 @@ public class ElasticsearchCliTests extends ESElasticsearchCliTestCase {
                 ExitCodes.USAGE,
                 output -> assertThat(
                         output,
-                        containsString("ERROR: Elasticsearch version option is mutually exclusive with any other option")),
+                        allOf(containsString("ERROR:"),
+                              containsString("are unavailable given other options on the command line"))),
                 args);
     }
 
@@ -91,18 +94,22 @@ public class ElasticsearchCliTests extends ESElasticsearchCliTestCase {
     }
 
     public void testThatPidFileCanBeConfigured() throws Exception {
-        runPidFileTest(ExitCodes.USAGE, false, output -> assertThat(output, containsString("Option p/pidfile requires an argument")), "-p");
-        runPidFileTest(ExitCodes.OK, true, output -> {}, "-p", "/tmp/pid");
-        runPidFileTest(ExitCodes.OK, true, output -> {}, "--pidfile", "/tmp/pid");
+        Path tmpDir = createTempDir();
+        Path pidFile = tmpDir.resolve("pid");
+        runPidFileTest(ExitCodes.USAGE, false,
+                output -> assertThat(output, containsString("Option p/pidfile requires an argument")), pidFile, "-p");
+        runPidFileTest(ExitCodes.OK, true, output -> {}, pidFile, "-p", pidFile.toString());
+        runPidFileTest(ExitCodes.OK, true, output -> {}, pidFile, "--pidfile", tmpDir.toString() + "/pid");
     }
 
-    private void runPidFileTest(final int expectedStatus, final boolean expectedInit, Consumer<String> outputConsumer, final String... args)
+    private void runPidFileTest(final int expectedStatus, final boolean expectedInit, Consumer<String> outputConsumer,
+                                Path expectedPidFile, final String... args)
             throws Exception {
         runTest(
                 expectedStatus,
                 expectedInit,
                 outputConsumer,
-                (foreground, pidFile, esSettings) -> assertThat(pidFile, equalTo("/tmp/pid")),
+                (foreground, pidFile, esSettings) -> assertThat(pidFile.toString(), equalTo(expectedPidFile.toString())),
                 args);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/cli/CommandTests.java b/core/src/test/java/org/elasticsearch/cli/CommandTests.java
index 376c883c074..0da487e7b9d 100644
--- a/core/src/test/java/org/elasticsearch/cli/CommandTests.java
+++ b/core/src/test/java/org/elasticsearch/cli/CommandTests.java
@@ -19,6 +19,7 @@
 
 package org.elasticsearch.cli;
 
+import joptsimple.OptionException;
 import joptsimple.OptionSet;
 import org.elasticsearch.test.ESTestCase;
 
@@ -87,10 +88,11 @@ public class CommandTests extends ESTestCase {
         MockTerminal terminal = new MockTerminal();
         NoopCommand command = new NoopCommand();
         String[] args = {"-v", "-s"};
-        UserException e = expectThrows(UserException.class, () -> {
+        OptionException e = expectThrows(OptionException.class, () -> {
             command.mainWithoutErrorHandling(args, terminal);
         });
-        assertTrue(e.getMessage(), e.getMessage().contains("Cannot specify -s and -v together"));
+        assertTrue(e.getMessage(),
+                e.getMessage().contains("Option(s) [v/verbose] are unavailable given other options on the command line"));
     }
 
     public void testSilentVerbosity() throws Exception {
diff --git a/distribution/licenses/jopt-simple-4.9.jar.sha1 b/distribution/licenses/jopt-simple-4.9.jar.sha1
deleted file mode 100644
index b86fa62ac20..00000000000
--- a/distribution/licenses/jopt-simple-4.9.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ee9e9eaa0a35360dcfeac129ff4923215fd65904
\ No newline at end of file
diff --git a/distribution/licenses/jopt-simple-5.0.2.jar.sha1 b/distribution/licenses/jopt-simple-5.0.2.jar.sha1
new file mode 100644
index 00000000000..b50ed4fea3b
--- /dev/null
+++ b/distribution/licenses/jopt-simple-5.0.2.jar.sha1
@@ -0,0 +1 @@
+98cafc6081d5632b61be2c9e60650b64ddbc637c
\ No newline at end of file
diff --git a/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java b/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java
index f08f15f236b..2a04a5be97f 100644
--- a/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/bootstrap/ESElasticsearchCliTestCase.java
@@ -22,6 +22,7 @@ package org.elasticsearch.bootstrap;
 import org.elasticsearch.cli.MockTerminal;
 import org.elasticsearch.test.ESTestCase;
 
+import java.nio.file.Path;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
@@ -31,7 +32,7 @@ import static org.hamcrest.CoreMatchers.equalTo;
 abstract class ESElasticsearchCliTestCase extends ESTestCase {
 
     interface InitConsumer {
-        void accept(final boolean foreground, final String pidFile, final Map<String, String> esSettings);
+        void accept(final boolean foreground, final Path pidFile, final Map<String, String> esSettings);
     }
 
     void runTest(
@@ -45,7 +46,7 @@ abstract class ESElasticsearchCliTestCase extends ESTestCase {
             final AtomicBoolean init = new AtomicBoolean();
             final int status = Elasticsearch.main(args, new Elasticsearch() {
                 @Override
-                void init(final boolean daemonize, final String pidFile, final Map<String, String> esSettings) {
+                void init(final boolean daemonize, final Path pidFile, final Map<String, String> esSettings) {
                     init.set(true);
                     initConsumer.accept(!daemonize, pidFile, esSettings);
                 }

From b4defafcb26ab86843bbe3464a4cf54cdc978696 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Tue, 5 Jul 2016 09:42:29 +0200
Subject: [PATCH 1263/1311] ingest: Renamed from `ingest-useragent` to
 `ingest-user-agent` and processor from `useragent` to `user_agent`

and on some other places did similar renaming. This is consistent with ES naming. Also made sure that the docs are navigable from the reference guide.
---
 ...nt.asciidoc => ingest-user-agent.asciidoc} | 38 +++++++++----------
 docs/plugins/ingest.asciidoc                  | 10 ++++-
 .../build.gradle                              |  2 +-
 .../useragent/IngestUserAgentPlugin.java      |  4 +-
 .../ingest/useragent/UserAgentCache.java      |  0
 .../ingest/useragent/UserAgentParser.java     |  0
 .../ingest/useragent/UserAgentProcessor.java  |  4 +-
 .../src/main/resources/regexes.yaml           |  0
 .../UserAgentProcessorFactoryTests.java       |  4 +-
 .../useragent/UserAgentProcessorTests.java    |  0
 .../ingest/useragent/UserAgentRestIT.java     |  0
 .../test/ingest-useragent/10_basic.yaml       | 11 ++++++
 .../20_useragent_processor.yaml               | 24 ++++++------
 .../ingest-useragent/30_custom_regex.yaml     | 20 +++++-----
 .../test/test-regexes.yaml                    |  0
 .../test/ingest-useragent/10_basic.yaml       | 11 ------
 .../scripts/module_and_plugin_test_cases.bash |  8 ++--
 settings.gradle                               |  2 +-
 18 files changed, 72 insertions(+), 66 deletions(-)
 rename docs/plugins/{ingest-useragent.asciidoc => ingest-user-agent.asciidoc} (56%)
 rename plugins/{ingest-useragent => ingest-user-agent}/build.gradle (92%)
 rename plugins/{ingest-useragent => ingest-user-agent}/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java (98%)
 rename plugins/{ingest-useragent => ingest-user-agent}/src/main/java/org/elasticsearch/ingest/useragent/UserAgentCache.java (100%)
 rename plugins/{ingest-useragent => ingest-user-agent}/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java (100%)
 rename plugins/{ingest-useragent => ingest-user-agent}/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java (98%)
 rename plugins/{ingest-useragent => ingest-user-agent}/src/main/resources/regexes.yaml (100%)
 rename plugins/{ingest-useragent => ingest-user-agent}/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java (98%)
 rename plugins/{ingest-useragent => ingest-user-agent}/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java (100%)
 rename plugins/{ingest-useragent => ingest-user-agent}/src/test/java/org/elasticsearch/ingest/useragent/UserAgentRestIT.java (100%)
 create mode 100644 plugins/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/10_basic.yaml
 rename plugins/{ingest-useragent => ingest-user-agent}/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yaml (79%)
 rename plugins/{ingest-useragent => ingest-user-agent}/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml (67%)
 rename plugins/{ingest-useragent => ingest-user-agent}/test/test-regexes.yaml (100%)
 delete mode 100644 plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/10_basic.yaml

diff --git a/docs/plugins/ingest-useragent.asciidoc b/docs/plugins/ingest-user-agent.asciidoc
similarity index 56%
rename from docs/plugins/ingest-useragent.asciidoc
rename to docs/plugins/ingest-user-agent.asciidoc
index 8868a4cf711..1110a0d859c 100644
--- a/docs/plugins/ingest-useragent.asciidoc
+++ b/docs/plugins/ingest-user-agent.asciidoc
@@ -1,12 +1,12 @@
-[[ingest-useragent]]
-=== Ingest Useragent Processor Plugin
+[[ingest-user-agent]]
+=== Ingest user agent processor plugin
 
-The Useragent processor extracts details from the user agent string a browser sends with its web requests. 
-This processor adds this information by default under the `useragent` field.
+The `user_agent` processor extracts details from the user agent string a browser sends with its web requests.
+This processor adds this information by default under the `user_agent` field.
 
-The ingest-useragent plugin ships by default with the regexes.yaml made available by uap-java with an Apache 2.0 license. For more details see https://github.com/ua-parser/uap-core.
+The ingest-user-agent plugin ships by default with the regexes.yaml made available by uap-java with an Apache 2.0 license. For more details see https://github.com/ua-parser/uap-core.
 
-[[ingest-useragent-install]]
+[[ingest-user-agent-install]]
 [float]
 ==== Installation
 
@@ -14,13 +14,13 @@ This plugin can be installed using the plugin manager:
 
 [source,sh]
 ----------------------------------------------------------------
-sudo bin/elasticsearch-plugin install ingest-useragent
+sudo bin/elasticsearch-plugin install ingest-user-agent
 ----------------------------------------------------------------
 
 The plugin must be installed on every node in the cluster, and each node must
 be restarted after installation.
 
-[[ingest-useragent-remove]]
+[[ingest-user-agent-remove]]
 [float]
 ==== Removal
 
@@ -28,26 +28,26 @@ The plugin can be removed with the following command:
 
 [source,sh]
 ----------------------------------------------------------------
-sudo bin/elasticsearch-plugin remove ingest-useragent
+sudo bin/elasticsearch-plugin remove ingest-user-agent
 ----------------------------------------------------------------
 
 The node must be stopped before removing the plugin.
 
-[[using-ingest-useragent]]
-==== Using the Useragent Processor in a Pipeline
+[[using-ingest-user-agent]]
+==== Using the user_agent Processor in a Pipeline
 
-[[ingest-useragent-options]]
-.Useragent options
+[[ingest-user-agent-options]]
+.User-agent options
 [options="header"]
 |======
 | Name                   | Required  | Default                                                                            | Description
 | `field`                | yes       | -                                                                                  | The field containing the user agent string.
-| `target_field`         | no        | useragent                                                                          | The field that will be filled with the user agent details.
-| `regex_file`           | no        | -                                                                                  | The name of the file in the `config/ingest-useragent` directory containing the regular expressions for parsing the user agent string. Both the directory and the file have to be created before starting Elasticsearch. If not specified, ingest-useragent will use the regexes.yaml from uap-core it ships with (see below).
+| `target_field`         | no        | user_agent                                                                          | The field that will be filled with the user agent details.
+| `regex_file`           | no        | -                                                                                  | The name of the file in the `config/ingest-user-agent` directory containing the regular expressions for parsing the user agent string. Both the directory and the file have to be created before starting Elasticsearch. If not specified, ingest-user-agent will use the regexes.yaml from uap-core it ships with (see below).
 | `properties`           | no        | [`name`, `major`, `minor`, `patch`, `build`, `os`, `os_name`, `os_major`, `os_minor`, `device`] | Controls what properties are added to `target_field`.
 |======
 
-Here is an example that adds the user agent details to the `useragent` field based on the `agent` field:
+Here is an example that adds the user agent details to the `user_agent` field based on the `agent` field:
 
 [source,js]
 --------------------------------------------------
@@ -55,7 +55,7 @@ Here is an example that adds the user agent details to the `useragent` field bas
   "description" : "...",
   "processors" : [
     {
-      "useragent" : {
+      "user_agent" : {
         "field" : "agent"
       }
     }
@@ -64,11 +64,11 @@ Here is an example that adds the user agent details to the `useragent` field bas
 --------------------------------------------------
 
 ===== Using a custom regex file
-To use a custom regex file for parsing the user agents, that file has to be put into the `config/ingest-useragent` directory and
+To use a custom regex file for parsing the user agents, that file has to be put into the `config/ingest-user-agent` directory and
 has to have a `.yaml` filename extension. The file has to be present at node startup, any changes to it or any new files added
 while the node is running will not have any effect.
 
 In practice, it will make most sense for any custom regex file to be a variant of the default file, either a more recent version
 or a customised version.
 
-The default file included in `ingest-useragent` is the `regexes.yaml` from uap-core: https://github.com/ua-parser/uap-core/blob/master/regexes.yaml 
+The default file included in `ingest-user-agent` is the `regexes.yaml` from uap-core: https://github.com/ua-parser/uap-core/blob/master/regexes.yaml
diff --git a/docs/plugins/ingest.asciidoc b/docs/plugins/ingest.asciidoc
index 7488cab6565..490d3d1362b 100644
--- a/docs/plugins/ingest.asciidoc
+++ b/docs/plugins/ingest.asciidoc
@@ -19,8 +19,14 @@ The GeoIP processor adds information about the geographical location of IP addre
 This processor adds this information by default under the `geoip` field.
 +
 The ingest-geoip plugin ships by default with the GeoLite2 City and GeoLite2 Country geoip2 databases from Maxmind made available
-under the CCA-ShareAlike 3.0 license. For more details see, http://dev.maxmind.com/geoip/geoip2/geolite2/. 
+under the CCA-ShareAlike 3.0 license. For more details see, http://dev.maxmind.com/geoip/geoip2/geolite2/.
+
+<<ingest-user-agent>>::
+
+A processor that extracts details from the User-Agent header value.
 
 include::ingest-attachment.asciidoc[]
 
-include::ingest-geoip.asciidoc[]
\ No newline at end of file
+include::ingest-geoip.asciidoc[]
+
+include::ingest-user-agent.asciidoc[]
\ No newline at end of file
diff --git a/plugins/ingest-useragent/build.gradle b/plugins/ingest-user-agent/build.gradle
similarity index 92%
rename from plugins/ingest-useragent/build.gradle
rename to plugins/ingest-user-agent/build.gradle
index 86fbf243497..ec599874d15 100644
--- a/plugins/ingest-useragent/build.gradle
+++ b/plugins/ingest-user-agent/build.gradle
@@ -24,6 +24,6 @@ esplugin {
 
 integTest {
   cluster {
-    extraConfigFile 'ingest-useragent/test-regexes.yaml', 'test/test-regexes.yaml'
+    extraConfigFile 'ingest-user-agent/test-regexes.yaml', 'test/test-regexes.yaml'
   }
 }
\ No newline at end of file
diff --git a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java b/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java
similarity index 98%
rename from plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java
rename to plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java
index 3e9a42fb8d9..100183b4532 100644
--- a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java
+++ b/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java
@@ -42,7 +42,7 @@ import java.util.stream.Stream;
 
 public class IngestUserAgentPlugin extends Plugin implements IngestPlugin {
 
-    private final Setting<Long> CACHE_SIZE_SETTING = Setting.longSetting("ingest.useragent.cache_size", 1000, 0,
+    private final Setting<Long> CACHE_SIZE_SETTING = Setting.longSetting("ingest.user_agent.cache_size", 1000, 0,
             Setting.Property.NodeScope);
 
     static final String DEFAULT_PARSER_NAME = "_default_";
@@ -76,7 +76,7 @@ public class IngestUserAgentPlugin extends Plugin implements IngestPlugin {
 
         if (Files.exists(userAgentConfigDirectory) && Files.isDirectory(userAgentConfigDirectory)) {
             PathMatcher pathMatcher = userAgentConfigDirectory.getFileSystem().getPathMatcher("glob:**.yaml");
-    
+
             try (Stream<Path> regexFiles = Files.find(userAgentConfigDirectory, 1,
                     (path, attr) -> attr.isRegularFile() && pathMatcher.matches(path))) {
                 Iterable<Path> iterable = regexFiles::iterator;
diff --git a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentCache.java b/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentCache.java
similarity index 100%
rename from plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentCache.java
rename to plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentCache.java
diff --git a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java b/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java
similarity index 100%
rename from plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java
rename to plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java
diff --git a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java b/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
similarity index 98%
rename from plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
rename to plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
index e52d0966a44..ec18126457b 100644
--- a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
+++ b/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
@@ -39,7 +39,7 @@ import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty;
 
 public class UserAgentProcessor extends AbstractProcessor {
 
-    public static final String TYPE = "useragent";
+    public static final String TYPE = "user_agent";
 
     private final String field;
     private final String targetField;
@@ -197,7 +197,7 @@ public class UserAgentProcessor extends AbstractProcessor {
         public UserAgentProcessor create(Map<String, Processor.Factory> factories, String processorTag,
                                          Map<String, Object> config) throws Exception {
             String field = readStringProperty(TYPE, processorTag, config, "field");
-            String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "useragent");
+            String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "user_agent");
             String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME);
             List<String> propertyNames = readOptionalList(TYPE, processorTag, config, "properties");
 
diff --git a/plugins/ingest-useragent/src/main/resources/regexes.yaml b/plugins/ingest-user-agent/src/main/resources/regexes.yaml
similarity index 100%
rename from plugins/ingest-useragent/src/main/resources/regexes.yaml
rename to plugins/ingest-user-agent/src/main/resources/regexes.yaml
diff --git a/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java b/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java
similarity index 98%
rename from plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java
rename to plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java
index 2fbdb6723b3..10c6ccd7941 100644
--- a/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java
+++ b/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java
@@ -52,7 +52,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
     @BeforeClass
     public static void createUserAgentParsers() throws IOException {
         Path configDir = createTempDir();
-        userAgentConfigDir = configDir.resolve("ingest-useragent");
+        userAgentConfigDir = configDir.resolve("ingest-user-agent");
         Files.createDirectories(userAgentConfigDir);
 
         // Copy file, leaving out the device parsers at the end
@@ -84,7 +84,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         UserAgentProcessor processor = factory.create(null, processorTag, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo("_field"));
-        assertThat(processor.getTargetField(), equalTo("useragent"));
+        assertThat(processor.getTargetField(), equalTo("user_agent"));
         assertThat(processor.getUaParser().getUaPatterns().size(), greaterThan(0));
         assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0));
         assertThat(processor.getUaParser().getDevicePatterns().size(), greaterThan(0));
diff --git a/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java b/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java
similarity index 100%
rename from plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java
rename to plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java
diff --git a/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentRestIT.java b/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentRestIT.java
similarity index 100%
rename from plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentRestIT.java
rename to plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentRestIT.java
diff --git a/plugins/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/10_basic.yaml b/plugins/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/10_basic.yaml
new file mode 100644
index 00000000000..fee3173f393
--- /dev/null
+++ b/plugins/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/10_basic.yaml
@@ -0,0 +1,11 @@
+"ingest-user-agent plugin installed":
+    - do:
+        cluster.state: {}
+
+    - set: {master_node: master}
+
+    - do:
+        nodes.info: {}
+
+    - match:  { nodes.$master.plugins.0.name: ingest-user-agent }
+    - match:  { nodes.$master.ingest.processors.0.type: user_agent }
diff --git a/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yaml b/plugins/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yaml
similarity index 79%
rename from plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yaml
rename to plugins/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yaml
index 13451a075d7..0964e69a99b 100644
--- a/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yaml
+++ b/plugins/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yaml
@@ -8,7 +8,7 @@
             "description": "_description",
             "processors": [
               {
-                "useragent" : {
+                "user_agent" : {
                   "field" : "field1"
                 }
               }
@@ -30,15 +30,15 @@
         type: test
         id: 1
   - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
-  - match: { _source.useragent.name: "Chrome" }
-  - match: { _source.useragent.os: "Mac OS X 10.9.2" }
-  - match: { _source.useragent.os_name: "Mac OS X" }
-  - match: { _source.useragent.os_major: "10" }
-  - match: { _source.useragent.os_minor: "9" }
-  - match: { _source.useragent.major: "33" }
-  - match: { _source.useragent.minor: "0" }
-  - match: { _source.useragent.patch: "1750" }
-  - match: { _source.useragent.device: "Other" }
+  - match: { _source.user_agent.name: "Chrome" }
+  - match: { _source.user_agent.os: "Mac OS X 10.9.2" }
+  - match: { _source.user_agent.os_name: "Mac OS X" }
+  - match: { _source.user_agent.os_major: "10" }
+  - match: { _source.user_agent.os_minor: "9" }
+  - match: { _source.user_agent.major: "33" }
+  - match: { _source.user_agent.minor: "0" }
+  - match: { _source.user_agent.patch: "1750" }
+  - match: { _source.user_agent.device: "Other" }
 
 ---
 "Test user agent processor with parameters":
@@ -50,7 +50,7 @@
             "description": "_description",
             "processors": [
               {
-                "useragent" : {
+                "user_agent" : {
                   "field" : "field1",
                   "target_field": "field2",
                   "properties": ["os"]
@@ -75,7 +75,7 @@
         id: 1
   - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
   - match: { _source.field2.os: "Mac OS X 10.9.2" }
-  - is_false: _source.useragent
+  - is_false: _source.user_agent
   - is_false: _source.field2.name
   - is_false: _source.field2.os_name
   - is_false: _source.field2.os_major
diff --git a/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml b/plugins/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml
similarity index 67%
rename from plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml
rename to plugins/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml
index 3472cd03c3f..5613145b664 100644
--- a/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml
+++ b/plugins/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml
@@ -8,7 +8,7 @@
             "description": "_description",
             "processors": [
               {
-                "useragent" : {
+                "user_agent" : {
                   "field": "field1",
                   "regex_file": "test-regexes.yaml"  
                 }
@@ -31,12 +31,12 @@
         type: test
         id: 1
   - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
-  - match: { _source.useragent.name: "Test" }
-  - match: { _source.useragent.os: "Other" }
-  - match: { _source.useragent.os_name: "Other" }
-  - match: { _source.useragent.device: "Other" }
-  - is_false: _source.useragent.os_major
-  - is_false: _source.useragent.os_minor
-  - is_false: _source.useragent.major
-  - is_false: _source.useragent.minor
-  - is_false: _source.useragent.patch
+  - match: { _source.user_agent.name: "Test" }
+  - match: { _source.user_agent.os: "Other" }
+  - match: { _source.user_agent.os_name: "Other" }
+  - match: { _source.user_agent.device: "Other" }
+  - is_false: _source.user_agent.os_major
+  - is_false: _source.user_agent.os_minor
+  - is_false: _source.user_agent.major
+  - is_false: _source.user_agent.minor
+  - is_false: _source.user_agent.patch
diff --git a/plugins/ingest-useragent/test/test-regexes.yaml b/plugins/ingest-user-agent/test/test-regexes.yaml
similarity index 100%
rename from plugins/ingest-useragent/test/test-regexes.yaml
rename to plugins/ingest-user-agent/test/test-regexes.yaml
diff --git a/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/10_basic.yaml b/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/10_basic.yaml
deleted file mode 100644
index d37ad674cb9..00000000000
--- a/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/10_basic.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-"ingest-useragent plugin installed":
-    - do:
-        cluster.state: {}
-
-    - set: {master_node: master}
-
-    - do:
-        nodes.info: {}
-
-    - match:  { nodes.$master.plugins.0.name: ingest-useragent }
-    - match:  { nodes.$master.ingest.processors.0.type: useragent }
diff --git a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
index eac0a80e4f4..041150d7958 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
@@ -228,8 +228,8 @@ fi
     install_and_check_plugin ingest geoip geoip2-*.jar jackson-annotations-*.jar jackson-databind-*.jar maxmind-db-*.jar
 }
 
-@test "[$GROUP] install ingest-useragent plugin" {
-    install_and_check_plugin ingest useragent
+@test "[$GROUP] install ingest-user-agent plugin" {
+    install_and_check_plugin ingest user-agent
 }
 
 @test "[$GROUP] check ingest-common module" {
@@ -361,8 +361,8 @@ fi
     remove_plugin ingest-geoip
 }
 
-@test "[$GROUP] remove ingest-useragent plugin" {
-    remove_plugin ingest-useragent
+@test "[$GROUP] remove ingest-user0agent plugin" {
+    remove_plugin ingest-user-agent
 }
 
 @test "[$GROUP] remove javascript plugin" {
diff --git a/settings.gradle b/settings.gradle
index 0c1e4b01e5c..3896082d9d8 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -36,7 +36,7 @@ List projects = [
   'plugins:discovery-gce',
   'plugins:ingest-geoip',
   'plugins:ingest-attachment',
-  'plugins:ingest-useragent',
+  'plugins:ingest-user-agent',
   'plugins:lang-javascript',
   'plugins:lang-python',
   'plugins:mapper-attachments',

From ee86a9f6340db535dae611453513dab44e447a3d Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Thu, 7 Jul 2016 14:54:50 +0200
Subject: [PATCH 1264/1311] Update field-stats.asciidoc

Change use of index constraints to correctly identify any indices containing relevant docs

Closes #19232
---
 docs/reference/search/field-stats.asciidoc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/reference/search/field-stats.asciidoc b/docs/reference/search/field-stats.asciidoc
index 4c38363bee9..90dbd539d65 100644
--- a/docs/reference/search/field-stats.asciidoc
+++ b/docs/reference/search/field-stats.asciidoc
@@ -311,10 +311,10 @@ curl -XPOST "http://localhost:9200/_field_stats?level=indices" -d '{
    "fields" : ["answer_count"] <1>
    "index_constraints" : { <2>
       "creation_date" : { <3>
-         "min_value" : { <4>
+         "max_value" : { <4>
             "gte" : "2014-01-01T00:00:00.000Z"
          },
-         "max_value" : {
+         "min_value" : { <4>
             "lt" : "2015-01-01T00:00:00.000Z"
          }
       }
@@ -325,7 +325,7 @@ curl -XPOST "http://localhost:9200/_field_stats?level=indices" -d '{
 <1> The fields to compute and return field stats for.
 <2> The set index constraints. Note that index constrains can be defined for fields that aren't defined in the `fields` option.
 <3> Index constraints for the field `creation_date`.
-<4> An index constraint on the `min_value` property of a field statistic.
+<4> Index constraints on the `max_value` and `min_value` property of a field statistic.
 
 For a field, index constraints can be defined on the `min_value` statistic, `max_value` statistic or both.
 Each index constraint support the following comparisons:
@@ -345,11 +345,11 @@ curl -XPOST "http://localhost:9200/_field_stats?level=indices" -d '{
    "fields" : ["answer_count"]
    "index_constraints" : {
       "creation_date" : {
-         "min_value" : {
+         "max_value" : {
             "gte" : "2014-01-01",
             "format" : "date_optional_time" <1>
          },
-         "max_value" : {
+         "min_value" : {
             "lt" : "2015-01-01",
             "format" : "date_optional_time"
          }

From a13717633cd65a9b7f582a69d2b860f8b4c83ba8 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 7 Jul 2016 09:09:11 -0400
Subject: [PATCH 1265/1311] Fix typo in name of remove ingest-user-agent test

This commit fixes a simple typo in the name of the test for testing the
removal of the ingest-user-agent plugin.
---
 .../packaging/scripts/module_and_plugin_test_cases.bash         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
index 041150d7958..b44e5885ff8 100644
--- a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
+++ b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash
@@ -361,7 +361,7 @@ fi
     remove_plugin ingest-geoip
 }
 
-@test "[$GROUP] remove ingest-user0agent plugin" {
+@test "[$GROUP] remove ingest-user-agent plugin" {
     remove_plugin ingest-user-agent
 }
 

From b58f2eb5c286abc65bbd620c165487b8f1fa2fdb Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Wed, 6 Jul 2016 12:55:20 +0200
Subject: [PATCH 1266/1311] Move back some messy tests from Groovy plugin to
 core

This commit moves back some messy tests that have been placed in lang-groovy module in https://github.com/elastic/elasticsearch/pull/13834. It removes the dependency on Groovy plugin as well as change back the tests to integration tests (IT suffix).

It also changes the current MockScriptEngine and MockScriptPlugin to make it easier to use.
---
 .../resources/checkstyle_suppressions.xml     |   3 -
 .../elasticsearch/script/FileScriptTests.java |   2 +-
 .../elasticsearch/script/StoredScriptsIT.java |  12 +-
 .../AggregationTestScriptsPlugin.java         |  85 +++++++
 .../metrics/HDRPercentileRanksIT.java         | 102 ++++++---
 .../metrics/HDRPercentilesIT.java             | 117 +++++++---
 .../aggregations/metrics/TopHitsIT.java       |  12 +-
 .../aggregations/pipeline/BucketScriptIT.java | 116 +++++++---
 .../pipeline/BucketSelectorIT.java            | 216 +++++++++++++-----
 .../search/innerhits/InnerHitsIT.java         |  13 +-
 .../messy/tests/package-info.java             |   4 -
 .../script/MockScriptEngine.java              | 181 ++++++++++-----
 .../script/MockScriptPlugin.java              |  46 ++++
 13 files changed, 699 insertions(+), 210 deletions(-)
 create mode 100644 core/src/test/java/org/elasticsearch/search/aggregations/AggregationTestScriptsPlugin.java
 rename modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentileRanksTests.java => core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java (82%)
 rename modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentilesTests.java => core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java (81%)
 rename modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketScriptTests.java => core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java (84%)
 rename modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketSelectorTests.java => core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java (68%)
 create mode 100644 test/framework/src/main/java/org/elasticsearch/script/MockScriptPlugin.java

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 7aa77a26458..574bb1e3b30 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -1125,12 +1125,9 @@
   <suppress files="modules[/\\]lang-expression[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]expression[/\\]MoreExpressionTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]groovy[/\\]GroovyPlugin.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]groovy[/\\]GroovyScriptEngineService.java" checks="LineLength" />
-  <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]BucketScriptTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]BulkTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]DoubleTermsTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]EquivalenceTests.java" checks="LineLength" />
-  <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]HDRPercentileRanksTests.java" checks="LineLength" />
-  <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]HDRPercentilesTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]HistogramTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]IPv4RangeTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]IndexLookupTests.java" checks="LineLength" />
diff --git a/core/src/test/java/org/elasticsearch/script/FileScriptTests.java b/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
index 9fbd4a81f19..f932317085b 100644
--- a/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/script/FileScriptTests.java
@@ -58,7 +58,7 @@ public class FileScriptTests extends ESTestCase {
         CompiledScript compiledScript = scriptService.compile(script, ScriptContext.Standard.SEARCH, Collections.emptyMap());
         assertNotNull(compiledScript);
         MockCompiledScript executable = (MockCompiledScript) compiledScript.compiled();
-        assertEquals("script1.mockscript", executable.name);
+        assertEquals("script1.mockscript", executable.getName());
     }
 
     public void testAllOpsDisabled() throws Exception {
diff --git a/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java b/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java
index 658a3bf5658..6ae607e7b8e 100644
--- a/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java
+++ b/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java
@@ -24,6 +24,9 @@ import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.function.Function;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 
@@ -41,7 +44,7 @@ public class StoredScriptsIT extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return pluginList(MockScriptEngine.TestPlugin.class);
+        return pluginList(CustomScriptPlugin.class);
     }
 
     public void testBasics() {
@@ -79,4 +82,11 @@ public class StoredScriptsIT extends ESIntegTestCase {
         assertEquals("Limit of script size in bytes [64] has been exceeded for script [foobar] with size [65]", e.getMessage());
     }
 
+    public static class CustomScriptPlugin extends MockScriptPlugin {
+
+        @Override
+        protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
+            return Collections.emptyMap();
+        }
+    }
 }
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationTestScriptsPlugin.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationTestScriptsPlugin.java
new file mode 100644
index 00000000000..218c3eae0c5
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationTestScriptsPlugin.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations;
+
+import org.elasticsearch.index.fielddata.ScriptDocValues;
+import org.elasticsearch.script.MockScriptPlugin;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Function;
+
+/**
+ * This class contains various mocked scripts that are used in aggregations integration tests.
+ */
+public class AggregationTestScriptsPlugin extends MockScriptPlugin {
+
+    @Override
+    protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
+        Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>();
+
+        scripts.put("20 - _value", vars -> {
+            double value = (double) vars.get("_value");
+            return 20.0d - value;
+        });
+
+        scripts.put("_value - 1", vars -> {
+            double value = (double) vars.get("_value");
+            return value - 1.0d;
+        });
+
+        scripts.put("_value - dec", vars -> {
+            double value = (double) vars.get("_value");
+            int dec = (int) vars.get("dec");
+            return value - dec;
+        });
+
+        scripts.put("doc['value'].value", vars -> {
+            Map<?, ?> doc = (Map) vars.get("doc");
+            return doc.get("value");
+        });
+
+        scripts.put("doc['value'].value - dec", vars -> {
+            int dec = (int) vars.get("dec");
+            Map<?, ?> doc = (Map) vars.get("doc");
+            ScriptDocValues.Longs value = (ScriptDocValues.Longs) doc.get("value");
+            return value.getValue() - dec;
+        });
+
+        scripts.put("doc['values'].values", vars -> {
+            Map<?, ?> doc = (Map) vars.get("doc");
+            return doc.get("values");
+        });
+
+        scripts.put("decrement all values", vars -> {
+            int dec = (int) vars.get("dec");
+            Map<?, ?> doc = (Map) vars.get("doc");
+            ScriptDocValues.Longs values = (ScriptDocValues.Longs) doc.get("values");
+
+            double[] res = new double[values.size()];
+            for (int i = 0; i < res.length; i++) {
+                res[i] = values.get(i) - dec;
+            }
+            return res;
+        });
+
+        return scripts;
+    }
+}
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentileRanksTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java
similarity index 82%
rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentileRanksTests.java
rename to core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java
index 21fb5be6b9d..c1f6847e2b6 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentileRanksTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java
@@ -16,20 +16,19 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.messy.tests;
+package org.elasticsearch.search.aggregations.metrics;
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService.ScriptType;
-import org.elasticsearch.script.groovy.GroovyPlugin;
+import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.global.Global;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Order;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
-import org.elasticsearch.search.aggregations.metrics.AbstractNumericTestCase;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
 import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanks;
 import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesMethod;
@@ -41,6 +40,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static java.util.Collections.emptyMap;
 import static org.elasticsearch.common.util.CollectionUtils.iterableAsArrayList;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
@@ -56,13 +56,11 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.sameInstance;
 
-/**
- *
- */
-public class HDRPercentileRanksTests extends AbstractNumericTestCase {
+public class HDRPercentileRanksIT extends AbstractNumericTestCase {
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(GroovyPlugin.class);
+        return Collections.singleton(AggregationTestScriptsPlugin.class);
     }
 
     private static double[] randomPercents(long minValue, long maxValue) {
@@ -82,7 +80,7 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase {
             }
         }
         Arrays.sort(percents);
-        Loggers.getLogger(HDRPercentileRanksTests.class).info("Using values={}", Arrays.toString(percents));
+        Loggers.getLogger(HDRPercentileRanksIT.class).info("Using values={}", Arrays.toString(percents));
         return percents;
     }
 
@@ -208,7 +206,7 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase {
         PercentileRanks values = global.getAggregations().get("percentile_ranks");
         assertThat(values, notNullValue());
         assertThat(values.getName(), equalTo("percentile_ranks"));
-        assertThat((PercentileRanks) global.getProperty("percentile_ranks"), sameInstance(values));
+        assertThat(global.getProperty("percentile_ranks"), sameInstance(values));
 
     }
 
@@ -255,8 +253,12 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase {
                 .prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(
-                        percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
-                        .field("value").script(new Script("_value - 1")).values(pcts))
+                        percentileRanks("percentile_ranks")
+                                .method(PercentilesMethod.HDR)
+                                .numberOfSignificantValueDigits(sigDigits)
+                                .field("value")
+                                .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .values(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -275,8 +277,12 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase {
                 .prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(
-                        percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
-                        .field("value").script(new Script("_value - dec", ScriptType.INLINE, null, params)).values(pcts))
+                        percentileRanks("percentile_ranks")
+                                .method(PercentilesMethod.HDR)
+                                .numberOfSignificantValueDigits(sigDigits)
+                                .field("value")
+                                .script(new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params))
+                                .values(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -311,8 +317,12 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase {
                 .prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(
-                        percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
-                        .field("values").script(new Script("_value - 1")).values(pcts))
+                        percentileRanks("percentile_ranks")
+                                .method(PercentilesMethod.HDR)
+                                .numberOfSignificantValueDigits(sigDigits)
+                                .field("values")
+                                .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .values(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -328,8 +338,12 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase {
                 .prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(
-                        percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
-                        .field("values").script(new Script("20 - _value")).values(pcts))
+                        percentileRanks("percentile_ranks")
+                                .method(PercentilesMethod.HDR)
+                                .numberOfSignificantValueDigits(sigDigits)
+                                .field("values")
+                                .script(new Script("20 - _value", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .values(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -348,8 +362,12 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase {
                 .prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(
-                        percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
-                        .field("values").script(new Script("_value - dec", ScriptType.INLINE, null, params)).values(pcts))
+                        percentileRanks("percentile_ranks")
+                                .method(PercentilesMethod.HDR)
+                                .numberOfSignificantValueDigits(sigDigits)
+                                .field("values")
+                                .script(new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params))
+                                .values(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -366,8 +384,11 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase {
                 .prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(
-                        percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
-                        .script(new Script("doc['value'].value")).values(pcts))
+                        percentileRanks("percentile_ranks")
+                                .method(PercentilesMethod.HDR)
+                                .numberOfSignificantValueDigits(sigDigits)
+                                .script(new Script("doc['value'].value", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .values(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -381,13 +402,19 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase {
         int sigDigits = randomSignificantDigits();
         Map<String, Object> params = new HashMap<>();
         params.put("dec", 1);
+
+        Script script = new Script("doc['value'].value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params);
+
         final double[] pcts = randomPercents(minValue - 1, maxValue - 1);
         SearchResponse searchResponse = client()
                 .prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(
-                        percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
-                        .script(new Script("doc['value'].value - dec", ScriptType.INLINE, null, params)).values(pcts))
+                        percentileRanks("percentile_ranks")
+                                .method(PercentilesMethod.HDR)
+                                .numberOfSignificantValueDigits(sigDigits)
+                                .script(script)
+                                .values(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -400,12 +427,18 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase {
     public void testScriptMultiValued() throws Exception {
         int sigDigits = randomSignificantDigits();
         final double[] pcts = randomPercents(minValues, maxValues);
+
+        Script script = new Script("doc['values'].values", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap());
+
         SearchResponse searchResponse = client()
                 .prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(
-                        percentileRanks("percentile_ranks").method(PercentilesMethod.HDR).numberOfSignificantValueDigits(sigDigits)
-                        .script(new Script("doc['values'].values")).values(pcts))
+                        percentileRanks("percentile_ranks")
+                                .method(PercentilesMethod.HDR)
+                                .numberOfSignificantValueDigits(sigDigits)
+                                .script(script)
+                                .values(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -419,6 +452,17 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase {
         int sigDigits = randomSignificantDigits();
         Map<String, Object> params = new HashMap<>();
         params.put("dec", 1);
+
+        // Equivalent to:
+        //
+        // List values = doc['values'].values;
+        // double[] res = new double[values.size()];
+        // for (int i = 0; i < res.length; i++) {
+        //      res[i] = values.get(i) - dec;
+        // };
+        // return res;
+        Script script = new Script("decrement all values", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params);
+
         final double[] pcts = randomPercents(minValues - 1, maxValues - 1);
         SearchResponse searchResponse = client()
                 .prepareSearch("idx")
@@ -427,10 +471,8 @@ public class HDRPercentileRanksTests extends AbstractNumericTestCase {
                         percentileRanks("percentile_ranks")
                                 .method(PercentilesMethod.HDR)
                                 .numberOfSignificantValueDigits(sigDigits)
-                                .script(new Script(
-                                        "List values = doc['values'].values; double[] res = new double[values.size()]; for (int i = 0; i < res.length; i++) { res[i] = values.get(i) - dec; }; return res;",
-                                ScriptType.INLINE, null, params))
-                        .values(pcts))
+                                .script(script)
+                                .values(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentilesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java
similarity index 81%
rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentilesTests.java
rename to core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java
index 9c21928798a..0dd0ee6799c 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentilesTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.messy.tests;
+package org.elasticsearch.search.aggregations.metrics;
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.logging.Loggers;
@@ -24,13 +24,12 @@ import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService.ScriptType;
-import org.elasticsearch.script.groovy.GroovyPlugin;
+import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.global.Global;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Order;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
-import org.elasticsearch.search.aggregations.metrics.AbstractNumericTestCase;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles;
 import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesMethod;
@@ -42,6 +41,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static java.util.Collections.emptyMap;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.filter;
@@ -57,13 +57,11 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.sameInstance;
 
-/**
- *
- */
-public class HDRPercentilesTests extends AbstractNumericTestCase {
+public class HDRPercentilesIT extends AbstractNumericTestCase {
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(GroovyPlugin.class);
+        return Collections.singleton(AggregationTestScriptsPlugin.class);
     }
 
     private static double[] randomPercentiles() {
@@ -83,7 +81,7 @@ public class HDRPercentilesTests extends AbstractNumericTestCase {
             }
         }
         Arrays.sort(percentiles);
-        Loggers.getLogger(HDRPercentilesTests.class).info("Using percentiles={}", Arrays.toString(percentiles));
+        Loggers.getLogger(HDRPercentilesIT.class).info("Using percentiles={}", Arrays.toString(percentiles));
         return percentiles;
     }
 
@@ -130,8 +128,8 @@ public class HDRPercentilesTests extends AbstractNumericTestCase {
                                         percentiles("percentiles").field("value")
                                                 .numberOfSignificantValueDigits(sigDigits)
                                                 .method(PercentilesMethod.HDR)
-                                                .percentiles(10,
-                                                15))).execute().actionGet();
+                                                .percentiles(10, 15)))
+                .execute().actionGet();
 
         assertThat(searchResponse.getHits().getTotalHits(), equalTo(2L));
         Histogram histo = searchResponse.getAggregations().get("histo");
@@ -210,7 +208,7 @@ public class HDRPercentilesTests extends AbstractNumericTestCase {
         Percentiles percentiles = global.getAggregations().get("percentiles");
         assertThat(percentiles, notNullValue());
         assertThat(percentiles.getName(), equalTo("percentiles"));
-        assertThat((Percentiles) global.getProperty("percentiles"), sameInstance(percentiles));
+        assertThat(global.getProperty("percentiles"), sameInstance(percentiles));
 
     }
 
@@ -240,8 +238,13 @@ public class HDRPercentilesTests extends AbstractNumericTestCase {
                 .prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(
-                        percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR).field("value")
-                                .script(new Script("_value - 1")).percentiles(pcts)).execute().actionGet();
+                        percentiles("percentiles")
+                                .numberOfSignificantValueDigits(sigDigits)
+                                .method(PercentilesMethod.HDR)
+                                .field("value")
+                                .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .percentiles(pcts))
+                .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
 
@@ -253,15 +256,20 @@ public class HDRPercentilesTests extends AbstractNumericTestCase {
     public void testSingleValuedFieldWithValueScriptWithParams() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("dec", 1);
+
         final double[] pcts = randomPercentiles();
         int sigDigits = randomSignificantDigits();
         SearchResponse searchResponse = client()
                 .prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(
-                        percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR).field("value")
-                                .script(new Script("_value - dec", ScriptType.INLINE, null, params)).percentiles(pcts)).execute()
-                .actionGet();
+                        percentiles("percentiles")
+                                .numberOfSignificantValueDigits(sigDigits)
+                                .method(PercentilesMethod.HDR)
+                                .field("value")
+                                .script(new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params))
+                                .percentiles(pcts))
+                .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
 
@@ -295,8 +303,13 @@ public class HDRPercentilesTests extends AbstractNumericTestCase {
                 .prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(
-                        percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR).field("values")
-                                .script(new Script("_value - 1")).percentiles(pcts)).execute().actionGet();
+                        percentiles("percentiles")
+                                .numberOfSignificantValueDigits(sigDigits)
+                                .method(PercentilesMethod.HDR)
+                                .field("values")
+                                .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .percentiles(pcts))
+                .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
 
@@ -311,8 +324,13 @@ public class HDRPercentilesTests extends AbstractNumericTestCase {
                 .prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(
-                        percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR).field("values")
-                                .script(new Script("20 - _value")).percentiles(pcts)).execute().actionGet();
+                        percentiles("percentiles")
+                                .numberOfSignificantValueDigits(sigDigits)
+                                .method(PercentilesMethod.HDR)
+                                .field("values")
+                                .script(new Script("20 - _value", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .percentiles(pcts))
+                .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
 
@@ -324,15 +342,20 @@ public class HDRPercentilesTests extends AbstractNumericTestCase {
     public void testMultiValuedFieldWithValueScriptWithParams() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("dec", 1);
+
         final double[] pcts = randomPercentiles();
         int sigDigits = randomSignificantDigits();
         SearchResponse searchResponse = client()
                 .prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(
-                        percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR).field("values")
-                                .script(new Script("_value - dec", ScriptType.INLINE, null, params)).percentiles(pcts)).execute()
-                .actionGet();
+                        percentiles("percentiles")
+                                .numberOfSignificantValueDigits(sigDigits)
+                                .method(PercentilesMethod.HDR)
+                                .field("values")
+                                .script(new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params))
+                                .percentiles(pcts))
+                .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
 
@@ -348,8 +371,12 @@ public class HDRPercentilesTests extends AbstractNumericTestCase {
                 .prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(
-                        percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR)
-                                .script(new Script("doc['value'].value")).percentiles(pcts)).execute().actionGet();
+                        percentiles("percentiles")
+                                .numberOfSignificantValueDigits(sigDigits)
+                                .method(PercentilesMethod.HDR)
+                                .script(new Script("doc['value'].value", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .percentiles(pcts))
+                .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
 
@@ -361,14 +388,20 @@ public class HDRPercentilesTests extends AbstractNumericTestCase {
     public void testScriptSingleValuedWithParams() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("dec", 1);
+
+        Script script = new Script("doc['value'].value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params);
+
         final double[] pcts = randomPercentiles();
         int sigDigits = randomSignificantDigits();
         SearchResponse searchResponse = client()
                 .prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(
-                        percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR)
-                                .script(new Script("doc['value'].value - dec", ScriptType.INLINE, null, params)).percentiles(pcts))
+                        percentiles("percentiles")
+                                .numberOfSignificantValueDigits(sigDigits)
+                                .method(PercentilesMethod.HDR)
+                                .script(script)
+                                .percentiles(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -381,12 +414,19 @@ public class HDRPercentilesTests extends AbstractNumericTestCase {
     public void testScriptMultiValued() throws Exception {
         final double[] pcts = randomPercentiles();
         int sigDigits = randomSignificantDigits();
+
+        Script script = new Script("doc['values'].values", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap());
+
         SearchResponse searchResponse = client()
                 .prepareSearch("idx")
                 .setQuery(matchAllQuery())
                 .addAggregation(
-                        percentiles("percentiles").numberOfSignificantValueDigits(sigDigits).method(PercentilesMethod.HDR)
-                                .script(new Script("doc['values'].values")).percentiles(pcts)).execute().actionGet();
+                        percentiles("percentiles")
+                                .numberOfSignificantValueDigits(sigDigits)
+                                .method(PercentilesMethod.HDR)
+                                .script(script)
+                                .percentiles(pcts))
+                .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
 
@@ -398,6 +438,17 @@ public class HDRPercentilesTests extends AbstractNumericTestCase {
     public void testScriptMultiValuedWithParams() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("dec", 1);
+
+        // Equivalent to:
+        //
+        // List values = doc['values'].values;
+        // double[] res = new double[values.size()];
+        // for (int i = 0; i < res.length; i++) {
+        //      res[i] = values.get(i) - dec;
+        // };
+        // return res;
+        Script script = new Script("decrement all values", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params);
+
         final double[] pcts = randomPercentiles();
         int sigDigits = randomSignificantDigits();
         SearchResponse searchResponse = client()
@@ -407,9 +458,9 @@ public class HDRPercentilesTests extends AbstractNumericTestCase {
                         percentiles("percentiles")
                                 .numberOfSignificantValueDigits(sigDigits)
                                 .method(PercentilesMethod.HDR)
-                                .script(new Script(
-                                        "List values = doc['values'].values; double[] res = new double[values.size()]; for (int i = 0; i < res.length; i++) { res[i] = values.get(i) - dec; }; return res;",
-                                        ScriptType.INLINE, null, params)).percentiles(pcts)).execute().actionGet();
+                                .script(script)
+                                .percentiles(pcts))
+                .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
index 84455c231aa..bfd4ed8ada9 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
@@ -30,6 +30,7 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptEngine;
+import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchHit;
@@ -54,6 +55,8 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.common.xcontent.XContentFactory.smileBuilder;
@@ -92,7 +95,14 @@ public class TopHitsIT extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(MockScriptEngine.TestPlugin.class);
+        return Collections.singleton(CustomScriptPlugin.class);
+    }
+
+    public static class CustomScriptPlugin extends MockScriptPlugin {
+        @Override
+        protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
+            return Collections.emptyMap();
+        }
     }
 
     public static String randomExecutionHint() {
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketScriptTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java
similarity index 84%
rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketScriptTests.java
rename to core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java
index e481e2f59ae..220dfd29817 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketScriptTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java
@@ -17,23 +17,21 @@
  * under the License.
  */
 
-package org.elasticsearch.messy.tests;
+package org.elasticsearch.search.aggregations.pipeline;
 
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService.ScriptType;
-import org.elasticsearch.script.groovy.GroovyPlugin;
-import org.elasticsearch.script.groovy.GroovyScriptEngineService;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram;
 import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram.Bucket;
 import org.elasticsearch.search.aggregations.metrics.sum.Sum;
 import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy;
-import org.elasticsearch.search.aggregations.pipeline.SimpleValue;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.io.IOException;
@@ -43,18 +41,20 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.bucketScript;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
 @ESIntegTestCase.SuiteScopeTestCase
-public class BucketScriptTests extends ESIntegTestCase {
+public class BucketScriptIT extends ESIntegTestCase {
 
     private static final String FIELD_1_NAME = "field1";
     private static final String FIELD_2_NAME = "field2";
@@ -68,7 +68,54 @@ public class BucketScriptTests extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(GroovyPlugin.class);
+        return Collections.singleton(CustomScriptPlugin.class);
+    }
+
+    public static class CustomScriptPlugin extends MockScriptPlugin {
+
+        @Override
+        protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
+            Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>();
+
+            scripts.put("_value0 + _value1 + _value2", vars -> {
+                double value0 = (double) vars.get("_value0");
+                double value1 = (double) vars.get("_value1");
+                double value2 = (double) vars.get("_value2");
+                return value0 + value1 + value2;
+            });
+
+            scripts.put("_value0 + _value1 / _value2", vars -> {
+                double value0 = (double) vars.get("_value0");
+                double value1 = (double) vars.get("_value1");
+                double value2 = (double) vars.get("_value2");
+                return value0 + value1 / value2;
+            });
+
+            scripts.put("_value0", vars -> vars.get("_value0"));
+
+            scripts.put("foo + bar + baz", vars -> {
+                double foo = (double) vars.get("foo");
+                double bar = (double) vars.get("bar");
+                double baz = (double) vars.get("baz");
+                return foo + bar + baz;
+            });
+
+            scripts.put("(_value0 + _value1 + _value2) * factor", vars -> {
+                double value0 = (double) vars.get("_value0");
+                double value1 = (double) vars.get("_value1");
+                double value2 = (double) vars.get("_value2");
+                return (value0 + value1 + value2) * (int) vars.get("factor");
+            });
+
+            scripts.put("my_script", vars -> {
+                double value0 = (double) vars.get("_value0");
+                double value1 = (double) vars.get("_value1");
+                double value2 = (double) vars.get("_value2");
+                return value0 + value1 + value2;
+            });
+
+            return scripts;
+        }
     }
 
     @Override
@@ -86,11 +133,6 @@ public class BucketScriptTests extends ESIntegTestCase {
             builders.add(client().prepareIndex("idx", "type").setSource(newDocBuilder()));
         }
 
-        client().admin().cluster().preparePutStoredScript()
-                .setId("my_script")
-                .setScriptLang(GroovyScriptEngineService.NAME)
-                .setSource(new BytesArray("{ \"script\": \"_value0 + _value1 + _value2\" }")).get();
-
         indexRandom(true, builders);
         ensureSearchable();
     }
@@ -117,8 +159,10 @@ public class BucketScriptTests extends ESIntegTestCase {
                                 .subAggregation(sum("field3Sum").field(FIELD_3_NAME))
                                 .subAggregation(sum("field4Sum").field(FIELD_4_NAME))
                                 .subAggregation(
-                                        bucketScript("seriesArithmetic", new Script("_value0 + _value1 + _value2", ScriptType.INLINE, null, null)
-                                                , "field2Sum", "field3Sum", "field4Sum"))).execute().actionGet();
+                                        bucketScript("seriesArithmetic",
+                                                new Script("_value0 + _value1 + _value2", ScriptType.INLINE, CustomScriptPlugin.NAME, null)
+                                                , "field2Sum", "field3Sum", "field4Sum")))
+                .execute().actionGet();
 
         assertSearchResponse(response);
 
@@ -161,8 +205,10 @@ public class BucketScriptTests extends ESIntegTestCase {
                                 .subAggregation(sum("field3Sum").field(FIELD_3_NAME))
                                 .subAggregation(sum("field4Sum").field(FIELD_4_NAME))
                                 .subAggregation(
-                                        bucketScript("seriesArithmetic", new Script("_value0 + _value1 / _value2", ScriptType.INLINE, null, null),
-                                                "field2Sum", "field3Sum", "field4Sum"))).execute().actionGet();
+                                        bucketScript("seriesArithmetic",
+                                                new Script("_value0 + _value1 / _value2", ScriptType.INLINE, CustomScriptPlugin.NAME, null),
+                                                "field2Sum", "field3Sum", "field4Sum")))
+                .execute().actionGet();
 
         assertSearchResponse(response);
 
@@ -203,8 +249,10 @@ public class BucketScriptTests extends ESIntegTestCase {
                                 .interval(interval)
                                 .subAggregation(sum("field2Sum").field(FIELD_2_NAME))
                                 .subAggregation(
-                                        bucketScript("seriesArithmetic", new Script("_value0", ScriptType.INLINE, null, null),
-                                                "field2Sum"))).execute().actionGet();
+                                        bucketScript("seriesArithmetic",
+                                                new Script("_value0", ScriptType.INLINE, CustomScriptPlugin.NAME, null),
+                                                "field2Sum")))
+                .execute().actionGet();
 
         assertSearchResponse(response);
 
@@ -246,7 +294,8 @@ public class BucketScriptTests extends ESIntegTestCase {
                                 .subAggregation(sum("field4Sum").field(FIELD_4_NAME))
                                 .subAggregation(
                                         bucketScript("seriesArithmetic", bucketsPathsMap,
-                                                new Script("foo + bar + baz", ScriptType.INLINE, null, null)))).execute().actionGet();
+                                                new Script("foo + bar + baz", ScriptType.INLINE, CustomScriptPlugin.NAME, null))))
+                .execute().actionGet();
 
         assertSearchResponse(response);
 
@@ -281,6 +330,9 @@ public class BucketScriptTests extends ESIntegTestCase {
     public void testInlineScriptWithParams() {
         Map<String, Object> params = new HashMap<>();
         params.put("factor", 3);
+
+        Script script = new Script("(_value0 + _value1 + _value2) * factor", ScriptType.INLINE, CustomScriptPlugin.NAME, params);
+
         SearchResponse response = client()
                 .prepareSearch("idx")
                 .addAggregation(
@@ -290,9 +342,8 @@ public class BucketScriptTests extends ESIntegTestCase {
                                 .subAggregation(sum("field2Sum").field(FIELD_2_NAME))
                                 .subAggregation(sum("field3Sum").field(FIELD_3_NAME))
                                 .subAggregation(sum("field4Sum").field(FIELD_4_NAME))
-                                .subAggregation(
-                                        bucketScript("seriesArithmetic", new Script("(_value0 + _value1 + _value2) * factor", ScriptType.INLINE, null, params),
-                                                "field2Sum", "field3Sum", "field4Sum"))).execute().actionGet();
+                                .subAggregation(bucketScript("seriesArithmetic", script, "field2Sum", "field3Sum", "field4Sum")))
+                .execute().actionGet();
 
         assertSearchResponse(response);
 
@@ -335,8 +386,10 @@ public class BucketScriptTests extends ESIntegTestCase {
                                 .subAggregation(sum("field3Sum").field(FIELD_3_NAME))
                                 .subAggregation(sum("field4Sum").field(FIELD_4_NAME))
                                 .subAggregation(
-                                        bucketScript("seriesArithmetic", new Script("_value0 + _value1 + _value2", ScriptType.INLINE, null, null),
-                                                "field2Sum", "field3Sum", "field4Sum").gapPolicy(GapPolicy.INSERT_ZEROS))).execute().actionGet();
+                                        bucketScript("seriesArithmetic",
+                                                new Script("_value0 + _value1 + _value2", ScriptType.INLINE, CustomScriptPlugin.NAME, null),
+                                                "field2Sum", "field3Sum", "field4Sum").gapPolicy(GapPolicy.INSERT_ZEROS)))
+                .execute().actionGet();
 
         assertSearchResponse(response);
 
@@ -370,7 +423,13 @@ public class BucketScriptTests extends ESIntegTestCase {
         }
     }
 
-    public void testIndexedScript() {
+    public void testStoredScript() {
+        assertAcked(client().admin().cluster().preparePutStoredScript()
+                .setId("my_script")
+                .setScriptLang(CustomScriptPlugin.NAME)
+                // Script source is not interpreted but it references a pre-defined script from CustomScriptPlugin
+                .setSource(new BytesArray("{ \"script\": \"my_script\" }")));
+
         SearchResponse response = client()
                 .prepareSearch("idx")
                 .addAggregation(
@@ -381,7 +440,8 @@ public class BucketScriptTests extends ESIntegTestCase {
                                 .subAggregation(sum("field3Sum").field(FIELD_3_NAME))
                                 .subAggregation(sum("field4Sum").field(FIELD_4_NAME))
                                 .subAggregation(
-                                        bucketScript("seriesArithmetic", new Script("my_script", ScriptType.STORED, null, null),
+                                        bucketScript("seriesArithmetic",
+                                                new Script("my_script", ScriptType.STORED, CustomScriptPlugin.NAME, null),
                                                 "field2Sum", "field3Sum", "field4Sum"))).execute().actionGet();
 
         assertSearchResponse(response);
@@ -425,7 +485,8 @@ public class BucketScriptTests extends ESIntegTestCase {
                                 .subAggregation(sum("field3Sum").field(FIELD_3_NAME))
                                 .subAggregation(sum("field4Sum").field(FIELD_4_NAME))
                                 .subAggregation(
-                                        bucketScript("seriesArithmetic", new Script("_value0 + _value1 + _value2", ScriptType.INLINE, null, null),
+                                        bucketScript("seriesArithmetic",
+                                                new Script("_value0 + _value1 + _value2", ScriptType.INLINE, CustomScriptPlugin.NAME, null),
                                                 "field2Sum", "field3Sum", "field4Sum")))
                                 .execute().actionGet();
 
@@ -448,7 +509,8 @@ public class BucketScriptTests extends ESIntegTestCase {
                                 .subAggregation(sum("field3Sum").field(FIELD_3_NAME))
                                 .subAggregation(sum("field4Sum").field(FIELD_4_NAME))
                                 .subAggregation(
-                                        bucketScript("seriesArithmetic", new Script("_value0 + _value1 + _value2", ScriptType.INLINE, null, null),
+                                        bucketScript("seriesArithmetic",
+                                                new Script("_value0 + _value1 + _value2", ScriptType.INLINE, CustomScriptPlugin.NAME, null),
                                                 "field2Sum", "field3Sum", "field4Sum"))).execute().actionGet();
 
         assertSearchResponse(response);
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketSelectorTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java
similarity index 68%
rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketSelectorTests.java
rename to core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java
index 5b796a92d7f..64dc7e50bcc 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketSelectorTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java
@@ -17,17 +17,16 @@
  * under the License.
  */
 
-package org.elasticsearch.messy.tests;
+package org.elasticsearch.search.aggregations.pipeline;
 
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService.ScriptType;
-import org.elasticsearch.script.groovy.GroovyPlugin;
-import org.elasticsearch.script.groovy.GroovyScriptEngineService;
 import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.histogram.InternalHistogram;
@@ -43,12 +42,14 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.sum;
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.bucketSelector;
 import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregatorBuilders.derivative;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -57,7 +58,7 @@ import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
 @ESIntegTestCase.SuiteScopeTestCase
-public class BucketSelectorTests extends ESIntegTestCase {
+public class BucketSelectorIT extends ESIntegTestCase {
 
     private static final String FIELD_1_NAME = "field1";
     private static final String FIELD_2_NAME = "field2";
@@ -71,7 +72,69 @@ public class BucketSelectorTests extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(GroovyPlugin.class);
+        return Collections.singleton(CustomScriptPlugin.class);
+    }
+
+    public static class CustomScriptPlugin extends MockScriptPlugin {
+
+        @Override
+        protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
+            Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>();
+
+            scripts.put("Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)", vars -> {
+                double value0 = (double) vars.get("_value0");
+                double value1 = (double) vars.get("_value1");
+                return Double.isNaN(value0) ? false : (value0 + value1 > 100);
+            });
+
+            scripts.put("Double.isNaN(_value0) ? true : (_value0 < 10000)", vars -> {
+                double value0 = (double) vars.get("_value0");
+                return Double.isNaN(value0) ? true : (value0 < 10000);
+            });
+
+            scripts.put("Double.isNaN(_value0) ? false : (_value0 > 10000)", vars -> {
+                double value0 = (double) vars.get("_value0");
+                return Double.isNaN(value0) ? false : (value0 > 10000);
+            });
+
+            scripts.put("Double.isNaN(_value0) ? false : (_value0 < _value1)", vars -> {
+                double value0 = (double) vars.get("_value0");
+                double value1 = (double) vars.get("_value1");
+                return Double.isNaN(value0) ? false : (value0 < value1);
+            });
+
+            scripts.put("Double.isNaN(_value0) ? false : (_value0 > 100)", vars -> {
+                double value0 = (double) vars.get("_value0");
+                return Double.isNaN(value0) ? false : (value0 > 10000);
+            });
+
+            scripts.put("Double.isNaN(my_value1) ? false : (my_value1 + my_value2 > 100)", vars -> {
+                double myValue1 = (double) vars.get("my_value1");
+                double myValue2 = (double) vars.get("my_value2");
+                return Double.isNaN(myValue1) ? false : (myValue1 + myValue2 > 100);
+            });
+
+            scripts.put("Double.isNaN(_value0) ? false : (_value0 + _value1 > threshold)", vars -> {
+                double value0 = (double) vars.get("_value0");
+                double value1 = (double) vars.get("_value1");
+                int threshold = (int) vars.get("threshold");
+                return Double.isNaN(value0) ? false : (value0 + value1 > threshold);
+            });
+
+            scripts.put("_value0 + _value1 > 100", vars -> {
+                double value0 = (double) vars.get("_value0");
+                double value1 = (double) vars.get("_value1");
+                return (value0 + value1 > 100);
+            });
+
+            scripts.put("my_script", vars -> {
+                double value0 = (double) vars.get("_value0");
+                double value1 = (double) vars.get("_value1");
+                return Double.isNaN(value0) ? false : (value0 + value1 > 100);
+            });
+
+            return scripts;
+        }
     }
 
     @Override
@@ -94,9 +157,6 @@ public class BucketSelectorTests extends ESIntegTestCase {
         builders.add(client().prepareIndex("idx_with_gaps", "type").setSource(newDocBuilder(3, 1, 0, 0)));
         builders.add(client().prepareIndex("idx_with_gaps", "type").setSource(newDocBuilder(3, 3, 0, 0)));
 
-        client().admin().cluster().preparePutStoredScript().setId("my_script").setScriptLang(GroovyScriptEngineService.NAME)
-                .setSource(new BytesArray("{ \"script\": \"Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)\" }")).get();
-
         indexRandom(true, builders);
         ensureSearchable();
     }
@@ -118,12 +178,13 @@ public class BucketSelectorTests extends ESIntegTestCase {
     }
 
     public void testInlineScript() {
+        Script script =
+                new Script("Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+
         SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(histogram("histo").field(FIELD_1_NAME).interval(interval)
                         .subAggregation(sum("field2Sum").field(FIELD_2_NAME)).subAggregation(sum("field3Sum").field(FIELD_3_NAME))
-                        .subAggregation(bucketSelector("bucketSelector",
-                                new Script("Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)", ScriptType.INLINE, null, null),
-                                "field2Sum", "field3Sum")))
+                        .subAggregation(bucketSelector("bucketSelector", script, "field2Sum", "field3Sum")))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -146,6 +207,8 @@ public class BucketSelectorTests extends ESIntegTestCase {
     }
 
     public void testInlineScriptNoBucketsPruned() {
+        Script script = new Script("Double.isNaN(_value0) ? true : (_value0 < 10000)", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+
         SearchResponse response = client()
                 .prepareSearch("idx")
                 .addAggregation(
@@ -154,10 +217,8 @@ public class BucketSelectorTests extends ESIntegTestCase {
                                 .interval(interval)
                                 .subAggregation(sum("field2Sum").field(FIELD_2_NAME))
                                 .subAggregation(sum("field3Sum").field(FIELD_3_NAME))
-                                .subAggregation(
-                                        bucketSelector("bucketSelector", new Script("Double.isNaN(_value0) ? true : (_value0 < 10000)",
-                                                ScriptType.INLINE, null, null), "field2Sum", "field3Sum"))).execute()
-                .actionGet();
+                                .subAggregation(bucketSelector("bucketSelector", script, "field2Sum", "field3Sum")))
+                .execute().actionGet();
 
         assertSearchResponse(response);
 
@@ -179,6 +240,8 @@ public class BucketSelectorTests extends ESIntegTestCase {
     }
 
     public void testInlineScriptNoBucketsLeft() {
+        Script script = new Script("Double.isNaN(_value0) ? false : (_value0 > 10000)", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+
         SearchResponse response = client()
                 .prepareSearch("idx")
                 .addAggregation(
@@ -187,9 +250,8 @@ public class BucketSelectorTests extends ESIntegTestCase {
                                 .interval(interval)
                                 .subAggregation(sum("field2Sum").field(FIELD_2_NAME))
                                 .subAggregation(sum("field3Sum").field(FIELD_3_NAME))
-                                .subAggregation(
-                                        bucketSelector("bucketSelector", new Script("Double.isNaN(_value0) ? false : (_value0 > 10000)",
-                                                ScriptType.INLINE, null, null), "field2Sum", "field3Sum"))).execute().actionGet();
+                                .subAggregation(bucketSelector("bucketSelector", script, "field2Sum", "field3Sum")))
+                .execute().actionGet();
 
         assertSearchResponse(response);
 
@@ -201,6 +263,8 @@ public class BucketSelectorTests extends ESIntegTestCase {
     }
 
     public void testInlineScript2() {
+        Script script = new Script("Double.isNaN(_value0) ? false : (_value0 < _value1)", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+
         SearchResponse response = client()
                 .prepareSearch("idx")
                 .addAggregation(
@@ -209,9 +273,8 @@ public class BucketSelectorTests extends ESIntegTestCase {
                                 .interval(interval)
                                 .subAggregation(sum("field2Sum").field(FIELD_2_NAME))
                                 .subAggregation(sum("field3Sum").field(FIELD_3_NAME))
-                                .subAggregation(
-                                        bucketSelector("bucketSelector", new Script("Double.isNaN(_value0) ? false : (_value0 < _value1)",
-                                                ScriptType.INLINE, null, null), "field2Sum", "field3Sum"))).execute().actionGet();
+                                .subAggregation(bucketSelector("bucketSelector", script, "field2Sum", "field3Sum")))
+                .execute().actionGet();
 
         assertSearchResponse(response);
 
@@ -233,6 +296,8 @@ public class BucketSelectorTests extends ESIntegTestCase {
     }
 
     public void testInlineScriptSingleVariable() {
+        Script script = new Script("Double.isNaN(_value0) ? false : (_value0 > 100)", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+
         SearchResponse response = client()
                 .prepareSearch("idx")
                 .addAggregation(
@@ -240,9 +305,8 @@ public class BucketSelectorTests extends ESIntegTestCase {
                                 .field(FIELD_1_NAME)
                                 .interval(interval)
                                 .subAggregation(sum("field2Sum").field(FIELD_2_NAME))
-                                .subAggregation(
-                                        bucketSelector("bucketSelector", new Script("Double.isNaN(_value0) ? false : (_value0 > 100)",
-                                                ScriptType.INLINE,null, null), "field2Sum"))).execute().actionGet();
+                                .subAggregation(bucketSelector("bucketSelector", script, "field2Sum")))
+                .execute().actionGet();
 
         assertSearchResponse(response);
 
@@ -261,15 +325,21 @@ public class BucketSelectorTests extends ESIntegTestCase {
     }
 
     public void testInlineScriptNamedVars() {
+        Script script = new Script("Double.isNaN(my_value1) ? false : (my_value1 + my_value2 > 100)", ScriptType.INLINE,
+                CustomScriptPlugin.NAME, null);
+
         Map<String, String> bucketPathsMap = new HashMap<>();
         bucketPathsMap.put("my_value1", "field2Sum");
         bucketPathsMap.put("my_value2", "field3Sum");
 
         SearchResponse response = client().prepareSearch("idx")
-                .addAggregation(histogram("histo").field(FIELD_1_NAME).interval(interval)
-                        .subAggregation(sum("field2Sum").field(FIELD_2_NAME)).subAggregation(sum("field3Sum").field(FIELD_3_NAME))
-                        .subAggregation(bucketSelector("bucketSelector", bucketPathsMap, new Script(
-                                "Double.isNaN(my_value1) ? false : (my_value1 + my_value2 > 100)", ScriptType.INLINE, null, null))))
+                .addAggregation(
+                        histogram("histo")
+                                .field(FIELD_1_NAME)
+                                .interval(interval)
+                                .subAggregation(sum("field2Sum").field(FIELD_2_NAME))
+                                .subAggregation(sum("field3Sum").field(FIELD_3_NAME))
+                                .subAggregation(bucketSelector("bucketSelector", bucketPathsMap, script)))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -292,13 +362,17 @@ public class BucketSelectorTests extends ESIntegTestCase {
     }
 
     public void testInlineScriptWithParams() {
-        Map<String, Object> params = new HashMap<>();
-        params.put("threshold", 100);
-        SearchResponse response = client().prepareSearch("idx").addAggregation(histogram("histo").field(FIELD_1_NAME).interval(interval)
-                .subAggregation(sum("field2Sum").field(FIELD_2_NAME)).subAggregation(sum("field3Sum").field(FIELD_3_NAME))
-                .subAggregation(bucketSelector("bucketSelector",
-                        new Script("Double.isNaN(_value0) ? false : (_value0 + _value1 > threshold)", ScriptType.INLINE, null, params),
-                        "field2Sum", "field3Sum")))
+        Script script = new Script("Double.isNaN(_value0) ? false : (_value0 + _value1 > threshold)", ScriptType.INLINE,
+                CustomScriptPlugin.NAME, Collections.singletonMap("threshold", 100));
+
+        SearchResponse response = client().prepareSearch("idx")
+                .addAggregation(
+                        histogram("histo")
+                                .field(FIELD_1_NAME)
+                                .interval(interval)
+                                .subAggregation(sum("field2Sum").field(FIELD_2_NAME))
+                                .subAggregation(sum("field3Sum").field(FIELD_3_NAME))
+                                .subAggregation(bucketSelector("bucketSelector", script, "field2Sum", "field3Sum")))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -321,13 +395,17 @@ public class BucketSelectorTests extends ESIntegTestCase {
     }
 
     public void testInlineScriptInsertZeros() {
+        Script script = new Script("_value0 + _value1 > 100", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
+
         SearchResponse response = client().prepareSearch("idx")
                 .addAggregation(
-                        histogram("histo").field(FIELD_1_NAME).interval(interval).subAggregation(sum("field2Sum").field(FIELD_2_NAME))
+                        histogram("histo")
+                                .field(FIELD_1_NAME)
+                                .interval(interval)
+                                .subAggregation(sum("field2Sum").field(FIELD_2_NAME))
                                 .subAggregation(sum("field3Sum").field(FIELD_3_NAME))
-                                .subAggregation(bucketSelector("bucketSelector",
-                                        new Script("_value0 + _value1 > 100", ScriptType.INLINE, null, null), "field2Sum", "field3Sum")
-                                                .gapPolicy(GapPolicy.INSERT_ZEROS)))
+                                .subAggregation(bucketSelector("bucketSelector", script , "field2Sum", "field3Sum")
+                        .gapPolicy(GapPolicy.INSERT_ZEROS)))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -349,7 +427,15 @@ public class BucketSelectorTests extends ESIntegTestCase {
         }
     }
 
-    public void testIndexedScript() {
+    public void testStoredScript() {
+        assertAcked(client().admin().cluster().preparePutStoredScript()
+                .setId("my_script")
+                .setScriptLang(CustomScriptPlugin.NAME)
+                // Source is not interpreted but my_script is defined in CustomScriptPlugin
+                .setSource(new BytesArray("{ \"script\": \"Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)\" }")));
+
+        Script script = new Script("my_script", ScriptType.STORED, CustomScriptPlugin.NAME, null);
+
         SearchResponse response = client()
                 .prepareSearch("idx")
                 .addAggregation(
@@ -358,9 +444,8 @@ public class BucketSelectorTests extends ESIntegTestCase {
                                 .interval(interval)
                                 .subAggregation(sum("field2Sum").field(FIELD_2_NAME))
                                 .subAggregation(sum("field3Sum").field(FIELD_3_NAME))
-                                .subAggregation(
-                                        bucketSelector("bucketSelector", new Script("my_script", ScriptType.STORED, null, null),
-                                                "field2Sum", "field3Sum"))).execute().actionGet();
+                                .subAggregation(bucketSelector("bucketSelector", script, "field2Sum", "field3Sum")))
+                .execute().actionGet();
 
         assertSearchResponse(response);
 
@@ -382,12 +467,17 @@ public class BucketSelectorTests extends ESIntegTestCase {
     }
 
     public void testUnmapped() throws Exception {
+        Script script = new Script("Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)", ScriptType.INLINE,
+                CustomScriptPlugin.NAME, null);
+
         SearchResponse response = client().prepareSearch("idx_unmapped")
-                .addAggregation(histogram("histo").field(FIELD_1_NAME).interval(interval)
-                        .subAggregation(sum("field2Sum").field(FIELD_2_NAME)).subAggregation(sum("field3Sum").field(FIELD_3_NAME))
-                        .subAggregation(bucketSelector("bucketSelector",
-                                new Script("Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)", ScriptType.INLINE, null, null),
-                                "field2Sum", "field3Sum")))
+                .addAggregation(
+                        histogram("histo")
+                                .field(FIELD_1_NAME)
+                                .interval(interval)
+                                .subAggregation(sum("field2Sum").field(FIELD_2_NAME))
+                                .subAggregation(sum("field3Sum").field(FIELD_3_NAME))
+                                .subAggregation(bucketSelector("bucketSelector", script, "field2Sum", "field3Sum")))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -399,12 +489,17 @@ public class BucketSelectorTests extends ESIntegTestCase {
     }
 
     public void testPartiallyUnmapped() throws Exception {
+        Script script = new Script("Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)", ScriptType.INLINE,
+                CustomScriptPlugin.NAME, null);
+
         SearchResponse response = client().prepareSearch("idx", "idx_unmapped")
-                .addAggregation(histogram("histo").field(FIELD_1_NAME).interval(interval)
-                        .subAggregation(sum("field2Sum").field(FIELD_2_NAME)).subAggregation(sum("field3Sum").field(FIELD_3_NAME))
-                        .subAggregation(bucketSelector("bucketSelector",
-                                new Script("Double.isNaN(_value0) ? false : (_value0 + _value1 > 100)", ScriptType.INLINE, null, null),
-                                "field2Sum", "field3Sum")))
+                .addAggregation(
+                        histogram("histo")
+                                .field(FIELD_1_NAME)
+                                .interval(interval)
+                                .subAggregation(sum("field2Sum").field(FIELD_2_NAME))
+                                .subAggregation(sum("field3Sum").field(FIELD_3_NAME))
+                                .subAggregation(bucketSelector("bucketSelector", script, "field2Sum", "field3Sum")))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -428,9 +523,18 @@ public class BucketSelectorTests extends ESIntegTestCase {
 
     public void testEmptyBuckets() {
         SearchResponse response = client().prepareSearch("idx_with_gaps")
-                .addAggregation(histogram("histo").field(FIELD_1_NAME).interval(1)
-                        .subAggregation(histogram("inner_histo").field(FIELD_1_NAME).interval(1).extendedBounds(new ExtendedBounds(1L, 4L))
-                                .minDocCount(0).subAggregation(derivative("derivative", "_count").gapPolicy(GapPolicy.INSERT_ZEROS))))
+                .addAggregation(
+                        histogram("histo")
+                                .field(FIELD_1_NAME)
+                                .interval(1)
+                                .subAggregation(
+                                        histogram("inner_histo")
+                                                .field(FIELD_1_NAME)
+                                                .interval(1)
+                                                .extendedBounds(new ExtendedBounds(1L, 4L))
+                                .minDocCount(0)
+                                .subAggregation(derivative("derivative", "_count")
+                                .gapPolicy(GapPolicy.INSERT_ZEROS))))
                 .execute().actionGet();
 
         assertSearchResponse(response);
diff --git a/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java b/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
index 56b33c6007e..780e8cbcad4 100644
--- a/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
+++ b/core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java
@@ -31,6 +31,7 @@ import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.InnerHitBuilder;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.MockScriptEngine;
+import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.SearchHit;
@@ -47,6 +48,8 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
+import java.util.function.Function;
 
 import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -71,9 +74,17 @@ import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
 public class InnerHitsIT extends ESIntegTestCase {
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return pluginList(MockScriptEngine.TestPlugin.class);
+        return Collections.singleton(CustomScriptPlugin.class);
+    }
+
+    public static class CustomScriptPlugin extends MockScriptPlugin {
+        @Override
+        protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
+            return Collections.emptyMap();
+        }
     }
 
     public void testSimpleNested() throws Exception {
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java
index ebc03bea2b3..a9bf09344f2 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java
@@ -36,8 +36,6 @@
  */
 /* List of renames that took place:
   renamed:    core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/AvgTests.java
-  renamed:    core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketScriptIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketScriptTests.java
-  renamed:    core/src/test/java/org/elasticsearch/search/aggregations/pipeline/BucketSelectorIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BucketSelectorTests.java
   renamed:    core/src/test/java/org/elasticsearch/document/BulkIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java
   renamed:    core/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/CardinalityTests.java
   renamed:    core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ChildQuerySearchTests.java
@@ -50,8 +48,6 @@
   renamed:    core/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ExtendedStatsTests.java
   renamed:    core/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/FunctionScoreTests.java
   renamed:    core/src/test/java/org/elasticsearch/search/geo/GeoDistanceIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java
-  renamed:    core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentileRanksTests.java
-  renamed:    core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HDRPercentilesTests.java
   renamed:    core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java
   renamed:    core/src/test/java/org/elasticsearch/search/aggregations/bucket/IPv4RangeIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IPv4RangeTests.java
   renamed:    core/src/test/java/org/elasticsearch/script/IndexLookupIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexLookupTests.java
diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
index 61a0cf6c236..519e52074da 100644
--- a/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
+++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptEngine.java
@@ -22,92 +22,60 @@ package org.elasticsearch.script;
 import org.apache.lucene.index.LeafReaderContext;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.plugins.ScriptPlugin;
+import org.elasticsearch.search.lookup.LeafSearchLookup;
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.Map;
+import java.util.function.Function;
 
 /**
- * A dummy script engine used for testing. Scripts must be a number. Many
- * tests rely on the fact this thing returns a String as its compiled form.
- * they even try to serialize it over the network!
+ * A mocked script engine that can be used for testing purpose.
  */
 public class MockScriptEngine implements ScriptEngineService {
 
     public static final String NAME = "mockscript";
 
-    /** A compiled script, just holds the scripts name, source, and params that were passed in */
-    public static class MockCompiledScript {
-        public final String name;
-        public final String source;
-        public final Map<String,String> params;
+    private final String type;
+    private final Map<String, Function<Map<String, Object>, Object>> scripts;
 
-        MockCompiledScript(String name, String source, Map<String,String> params) {
-            this.name = name;
-            this.source = source;
-            this.params = params;
-        }
+    public MockScriptEngine(String type, Map<String, Function<Map<String, Object>, Object>> scripts) {
+        this.type = type;
+        this.scripts = Collections.unmodifiableMap(scripts);
     }
 
-    public static class TestPlugin extends Plugin implements ScriptPlugin {
-        @Override
-        public ScriptEngineService getScriptEngineService(Settings settings) {
-            return new MockScriptEngine();
-        }
+    public MockScriptEngine() {
+        this(NAME, Collections.emptyMap());
     }
 
     @Override
     public String getType() {
-        return NAME;
+        return type;
     }
 
     @Override
     public String getExtension() {
-        return NAME;
+        return getType();
     }
 
     @Override
-    public Object compile(String scriptName, String scriptSource, Map<String, String> params) {
-        return new MockCompiledScript(scriptName, scriptSource, params);
+    public Object compile(String name, String source, Map<String, String> params) {
+        Function<Map<String, Object>, Object> script = scripts.get(source);
+        return new MockCompiledScript(name, params, source, script);
     }
 
     @Override
     public ExecutableScript executable(CompiledScript compiledScript, @Nullable Map<String, Object> vars) {
-        assert compiledScript.compiled() instanceof MockCompiledScript
-          : "do NOT pass compiled scripts from other engines to me, I will fail your test, got: " + compiledScript;
-        return new AbstractExecutableScript() {
-            @Override
-            public Object run() {
-                return new BytesArray(((MockCompiledScript)compiledScript.compiled()).source);
-            }
-        };
+        MockCompiledScript compiled = (MockCompiledScript) compiledScript.compiled();
+        return compiled.createExecutableScript(vars);
     }
 
     @Override
     public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, @Nullable Map<String, Object> vars) {
-        return new SearchScript() {
-            @Override
-            public LeafSearchScript getLeafSearchScript(LeafReaderContext context) throws IOException {
-                AbstractSearchScript leafSearchScript = new AbstractSearchScript() {
-
-                    @Override
-                    public Object run() {
-                        return ((MockCompiledScript)compiledScript.compiled()).source;
-                    }
-
-                };
-                leafSearchScript.setLookup(lookup.getLeafSearchLookup(context));
-                return leafSearchScript;
-            }
-
-            @Override
-            public boolean needsScores() {
-                return false;
-            }
-        };
+        MockCompiledScript compiled = (MockCompiledScript) compiledScript.compiled();
+        return compiled.createSearchScript(vars, lookup);
     }
 
     @Override
@@ -118,4 +86,111 @@ public class MockScriptEngine implements ScriptEngineService {
     public boolean isInlineScriptEnabled() {
         return true;
     }
+
+
+    public class MockCompiledScript {
+
+        private final String name;
+        private final String source;
+        private final Map<String, String> params;
+        private final Function<Map<String, Object>, Object> script;
+
+        public MockCompiledScript(String name, Map<String, String> params, String source, Function<Map<String, Object>, Object> script) {
+            this.name = name;
+            this.source = source;
+            this.params = params;
+            this.script = script;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public ExecutableScript createExecutableScript(Map<String, Object> vars) {
+            Map<String, Object> context = new HashMap<>();
+            if (params != null) {
+                context.putAll(params);
+            }
+            if (vars != null) {
+                context.putAll(vars);
+            }
+            return new MockExecutableScript(context, script != null ? script : ctx -> new BytesArray(source));
+        }
+
+        public SearchScript createSearchScript(Map<String, Object> vars, SearchLookup lookup) {
+            Map<String, Object> context = new HashMap<>();
+            if (params != null) {
+                context.putAll(params);
+            }
+            if (vars != null) {
+                context.putAll(vars);
+            }
+            return new MockSearchScript(lookup, context, script != null ? script : ctx -> source);
+        }
+    }
+
+    public class MockExecutableScript implements ExecutableScript {
+
+        private final Function<Map<String, Object>, Object> script;
+        private final Map<String, Object> vars;
+
+        public MockExecutableScript(Map<String, Object> vars, Function<Map<String, Object>, Object> script) {
+            this.vars = vars;
+            this.script = script;
+        }
+
+        @Override
+        public void setNextVar(String name, Object value) {
+            vars.put(name, value);
+        }
+
+        @Override
+        public Object run() {
+            return script.apply(vars);
+        }
+    }
+
+    public class MockSearchScript implements SearchScript {
+
+        private final Function<Map<String, Object>, Object> script;
+        private final Map<String, Object> vars;
+        private final SearchLookup lookup;
+
+        public MockSearchScript(SearchLookup lookup, Map<String, Object> vars, Function<Map<String, Object>, Object> script) {
+            this.lookup = lookup;
+            this.vars = vars;
+            this.script = script;
+        }
+
+        @Override
+        public LeafSearchScript getLeafSearchScript(LeafReaderContext context) throws IOException {
+            LeafSearchLookup leafLookup = lookup.getLeafSearchLookup(context);
+
+            Map<String, Object> ctx = new HashMap<>();
+            ctx.putAll(leafLookup.asMap());
+            if (vars != null) {
+                ctx.putAll(vars);
+            }
+
+            AbstractSearchScript leafSearchScript = new AbstractSearchScript() {
+
+                @Override
+                public Object run() {
+                    return script.apply(ctx);
+                }
+
+                @Override
+                public void setNextVar(String name, Object value) {
+                    ctx.put(name, value);
+                }
+            };
+            leafSearchScript.setLookup(leafLookup);
+            return leafSearchScript;
+        }
+
+        @Override
+        public boolean needsScores() {
+            return false;
+        }
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/script/MockScriptPlugin.java b/test/framework/src/main/java/org/elasticsearch/script/MockScriptPlugin.java
new file mode 100644
index 00000000000..dc397bd0b14
--- /dev/null
+++ b/test/framework/src/main/java/org/elasticsearch/script/MockScriptPlugin.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.script;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.ScriptPlugin;
+
+import java.util.Map;
+import java.util.function.Function;
+
+/**
+ * A script plugin that uses {@link MockScriptEngine} as the script engine for tests.
+ */
+public abstract class MockScriptPlugin extends Plugin implements ScriptPlugin {
+
+    public static final String NAME = MockScriptEngine.NAME;
+
+    @Override
+    public ScriptEngineService getScriptEngineService(Settings settings) {
+        return new MockScriptEngine(pluginScriptLang(), pluginScripts());
+    }
+
+    protected abstract Map<String, Function<Map<String, Object>, Object>> pluginScripts();
+
+    public String pluginScriptLang() {
+        return NAME;
+    }
+}

From ec3807d4266d05bd20ae484aeab600af78024a53 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Thu, 7 Jul 2016 16:36:23 +0200
Subject: [PATCH 1267/1311] Added 2.3.4 version and bwc indices

---
 .../main/java/org/elasticsearch/Version.java    |   4 ++++
 .../test/resources/indices/bwc/index-2.3.4.zip  | Bin 0 -> 79323 bytes
 .../test/resources/indices/bwc/repo-2.3.4.zip   | Bin 0 -> 77302 bytes
 3 files changed, 4 insertions(+)
 create mode 100644 core/src/test/resources/indices/bwc/index-2.3.4.zip
 create mode 100644 core/src/test/resources/indices/bwc/repo-2.3.4.zip

diff --git a/core/src/main/java/org/elasticsearch/Version.java b/core/src/main/java/org/elasticsearch/Version.java
index 34982b679c7..da876730b9a 100644
--- a/core/src/main/java/org/elasticsearch/Version.java
+++ b/core/src/main/java/org/elasticsearch/Version.java
@@ -69,6 +69,8 @@ public class Version {
     public static final Version V_2_3_2 = new Version(V_2_3_2_ID, org.apache.lucene.util.Version.LUCENE_5_5_0);
     public static final int V_2_3_3_ID = 2030399;
     public static final Version V_2_3_3 = new Version(V_2_3_3_ID, org.apache.lucene.util.Version.LUCENE_5_5_0);
+    public static final int V_2_3_4_ID = 2030499;
+    public static final Version V_2_3_4 = new Version(V_2_3_4_ID, org.apache.lucene.util.Version.LUCENE_5_5_0);
     public static final int V_5_0_0_alpha1_ID = 5000001;
     public static final Version V_5_0_0_alpha1 = new Version(V_5_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_6_0_0);
     public static final int V_5_0_0_alpha2_ID = 5000002;
@@ -102,6 +104,8 @@ public class Version {
                 return V_5_0_0_alpha2;
             case V_5_0_0_alpha1_ID:
                 return V_5_0_0_alpha1;
+            case V_2_3_4_ID:
+                return V_2_3_4;
             case V_2_3_3_ID:
                 return V_2_3_3;
             case V_2_3_2_ID:
diff --git a/core/src/test/resources/indices/bwc/index-2.3.4.zip b/core/src/test/resources/indices/bwc/index-2.3.4.zip
new file mode 100644
index 0000000000000000000000000000000000000000..2d8514724b4b614a2267a4a79b14c657132b8764
GIT binary patch
literal 79323
zcmbrlWl$x-mMwg6IB4UI)401kG*090?(WXP8h3Yhx5nM2ad(%--Qnvy_nY@7Zp6Hq
zH?JagMg6E;EBCHkD>Evq<fS1Xfq;J;P=QAh|90{JpP&GE0AmAZ0|pgkH~<9D)K`t{
zZ@9R_1Hd5f!2keA^uIUC|F7oI|I!@e-<un{8R=Qr8k@N5G1Ifqv;Gqf;r|v6_dkQP
zwKFzx`fq{)|D})k|7TFff5M{uSA9kM7g*B&3Rcg_*}&Q4pXmQ6>VHz$e-s!N@DKD5
zoiBO%e{0_IHyHk#=w{Y-h6dJjtn^OK|Mmp`6C$0sDQR#a^kI(g0U!PsSyIcv3rMp?
zlxPD)tc`oj7yK^feoQPGQjqv>qSW|pZb*4ZA5hsO2;B4C(>d}iK~blkCpw#ZuLPH`
zX&$dR52l&S1A-bunu$w~uI#%VmjIFSVC>xuX(Dv53v2yVFcZN@q8?Q$A5BM^SR3@I
zy6ozMF^?u?TXau!?0(y*n<H~g65U#PR^zqou+;TJAtiRY>HOc&hLOhcno?^RKG4y2
z(zIjZ6|=Bf9p_)*M;e>sk-w5gyW_TFvOZ_wa$<}gsvv5L59GtNq42>xe*_$6G;55*
zWVs>!ff|6^gu+s>uKg8+BA{bz^u5T0f;PTGY4Dfvr_Y43T%rS4oRXdtTH39p;lqn7
zI~Dns5@A{26BWImHGd3Me{5gqrl+?*e#0h?g5W^qr_$gRHC~sh2n&63F?R~7p<vG=
zr71x#syJ}&{I%wod945Z#~JefR>;4OKC=JnLH<>Zh0(tc?*DQW{wqAwe;5AW#7Xxb
z#QA?s#qvL)`tPdiU-RPrpOO7X&HWz|{HNyr!I=qcv@-guK<~ft{r`#c-_%aW`0qme
z2m3$bU#zsDJDX%uQ#)QgehdUmEpJ6fPaIQ>2xqyyO+>Fmk8~CUhZ0R|4HHcw@{iA-
zRjySx)s?ao^|r7u1xl!oZjg*OjP<P1E%$A0w088YuvQh4E-QppJb;Gwu^9L4WHesj
z{)1b8`4`LL`ky8EZ$sywK1Tds`Pe@wZQt<`OdJ7#%p3qf@c)HU&(MS3)cBvq_Mb2Z
zzr&>B4L0#3skxM9pQK%^GFo61Cp86aq)l#sH=1MM#i{ORHo!qk>em8@QS>2%B|s3A
z;LsGcqszt#OcLLYh=`fTzU!@hKHkCleef-9u}uXQR+n8=+yBXY<U3M!*^;JE4)OL|
z{34#^th~H#bU0c<;If~%7tOsh@>K~bgUM;ufCbqo@|sU>b??^8%)Of@{rx#@i6Q-k
zhusWGZ5c0^YH0VGMZo*wXGzVFo|diq;h$@YZA&s8zQ+$+wU~_OfhutUpZk^-%FH(B
z^@*D7c7xT0J+nExmVDgghn%Ygj_ytY8;j59C#!>-507U$4p}YUkGGm%uE?~60yJ_{
zcpSLhh}Ii%axd4R{c`-zL0|TM{^oUARNz6vWimP}8_se@s1^?zh1DkXxT)-y<NOo*
zl_G~5E!bju?fVWKMUJx!g;($GlqH&Yw)cR9z;;4O{Uw3VG8{{B-6m(&kY~Ix4(>tQ
zTq+v9>GNhTSJFbkptUldoQW(ehG-a1(mmu3o3%2!jEPF(8exMcXVN|SQ)kd;mN7NU
zDM5MbiL$CJzoD!rF-O*w(NM_H+_D7nR5l*#^8i*Wn^aso(|R{9E(Uo!)9^;rLDn6~
zkY~r?g3N0y8~CM&>DU+G<@MEs;|p=c3zq$>h$=Gu_WdzVv6dx?pw^A7va-SpFV-z*
z=kK$qy+QW}Dk&;1K^I<WwyZm&4n#GQxjU%Mt1B-dSrG;2T+JIXDz+)r^N1E+d6OGy
z3>!;&u~Rvfn`k%0J=d{&y$ab|5>xJ82;inkMbWX);I$1e_Y3U%Eup-#b3YGM^MK#N
zb+uArTo{(qa8?MD#MJE9dr@3c99vJsRXla`hEHNKB)JFrZKz>6&WiS7;5)Etr<LI<
zK)ZseHORpRPg6$j!lhuV^Z~tUpO6M8+Yd24p(8Y-Z>zLOKXKPx&C+}u9Wh2@Lh4%7
z{kbfhDrd13k5(Oy!G7JzxnXIz`_HIx6?)|i-ebHr*!!U)Y&oP=1NdAopzaJg3sS!@
zK1Jzk%9K;FPTs<Lgp;aB$;W(W#fl(K&O)(3q{S1lt}EEl4Is(Q(9k$Vu!b?y#jSN`
z@IqrrQ0!Z_+j_v*S!j$*!6l2vm*Tx61xB#<7m}YP&&A0U9zCyH^QoBj5<u?NkY0Bg
zGZLLopU`mzrl9PYu99wJlE6b-8QUJ^fs&K;&8Vbxr&e!K8qS%8$k7P4p<n55vZB>$
zGuf~lgML^=SF9TjMiW2m5p|QefpZ`9cy?w82Kgcg)R<^7SSI0VaDEosqOrMNH@pX^
zd-SIsx{m^VL?UAaBGbXgLA(jaVpMMt7vHsh2cTu<Gmc;{eTdJ?>u*^~PT0pBWv4P2
z*^JIPceLv;dUW@V5l8(#@f6A~-}c8c$YfZ`Jl^;AAKJJ;CwVvRtsI1B8_7pQ9v5-!
zR&07iq~MpQ8~FfFz;aX_Do-rev7fC`_GH7kwYIfBBP1rpvCGLxXx5VPLgvs3>&^3B
z+6YQ{Lze%>#D2{g8TFY|o|9RV*Awm}>)tLnlQ5|G(j=86!=<=$1D%l65m+CPe@A@2
z69~kmewyORw1*DLRoK4~OE^*gzOx3UG|P&|kXpSg`-H%qCM>~@?LLH1gqd9-T}j2I
zcC^@@?rUpChevHKkrk3xTQh%WXF$|dgt5N~opLEmB&(2l>{>xYI3|)QZn(<$ZRRvq
z!;VaI7a5~yySW^#Gc-(Ew&&`Da%$NH+E1`7FB9{bk8I>zuH~1aTL29_lm<If$`;kJ
z9|r{tT+VLd*&u@96V*vc=UMz?r7nDk`U4;#=j-|h)zSl7<`tUiz&ztKS~Qsq)GH23
zz=F9@ZG9<B8ydlk?N|&@J3G`R*r52D{UGAlLoWM7JfrFAv0Me~2D0~{AxlO;i1UPS
z4gO{db6A`49aNuC-KXA}WaOw9sS<8m?AH&epZn*6D%`hSaw%3eGND7p&?n<(LbdH^
zY)4z56*8qPH<Q`~IzCS2R)pFc9!V=<o4*D>TuOJ~U13%SK?jV@6}s{jsd=DN1$J!B
z*vq8~#YP2uzk1MHKWDAqvds|FId+EZx@+;^={3MeNM&$3QDQV3X(P2`Yz4lJJ8IO0
zOQ|dTV{>`X`?8XK<j^Xs7-C^yzW}&s>`$FE6Sz+fv04ZHtkdUsc~mL)cdEsxlOa@E
zR!VLQ8V$>W8s-5C`@EsBb4#W$YZPD#qPLHAtPi}l+w-vvpPnF%9diox5`~(=EDlaZ
zSSzFE>r-d%=nbWLlTF>Pgzpion2c$)p~E+|aVK&R{Sat`zmXn2P!svO)~WP68yw=n
z_*d-LHLe&;wLlo_Q!3F9^walE=Ckz*-cOCeE>4DfA#GKljEq&&q3rQ|QDm4L@z|09
z<z|i3GskGEmXhmX^ioa6Y?Z@vJJ86@ZARX@MxT;@j)+K|`zQHtAyE%!D7K$_8!9ee
zBZY9=>?wX4H0keZn^!<Ewf8f*Z_ya=MwG_$P>M%sjk)uk?VWA(y4Rv1D~3o6gTD9x
z#SBqNjoHO6a`A8vM77w4+p(nYeuNY>$eSN6%hsouj+t4goDJOH<{~$XjIZ;p<))<z
zAIKGmH7mx#?6=^li6OI@;Ec0bTUCBTE~kDl!e7-tA&qg>mrUkPTdx^WFDcjly>I@9
zSWK}ikLCT>jw~~9Y&Wo1`$Nd&z#Q<a1RwmPz1(mC!8N%GFMn*PHvbJB^LKmkz(d6x
zqfdZ|GxFjBT~^1OSM99Cy49MDN5_7rt&@$eN4*gMo8tQL%7)S=TLY&ZHS|xxByuqw
zp_{7jI1GgJ>lApcW>cfz;S#0xDzbO**2`3W`rTSGGH3uayD6S@D<Hh%sO&+|9hLL$
zVnGX>V&)M5@Mb(OV76st2ha#xRD8foMSOBdIc@lfTjVt(77d<s7c|0-*HGN~=dT~K
zE=pX9=S7bbFr*Jb8>;IY!JRt2|IG*W6vEORKm$e7--{DGDwkZBy{t7P7NU-a^uaLl
z42geu3%qcc<<L)oXq(ajE+5fB0s47vWa}YKK2X`Id)CN91nv03?~OzaF_z|}IW_9V
zLfNw1-j+r@Gs~1wOMNa4)fcg+P9+1Dp3DUXsILT|qVEdMlXNY+pfLzDg#s3eOjnB{
z)z|`zu~;FSX0*%$9qr-JOY$&0JMH<bHy_3*r#5<Uy1EnXw>-gJYHq{zRI;{RqHWnz
z^5ynC21RoEmaZw^B31DV{+J1W`i-4QdfFY+5x@63(~6+?UgX8dlY@L_$h@slbo<VW
zj{Ax-S8A@qy@~^x5DsLad-bLUaF(NZ;H0)OJ>Z7zi%w_STl1$gAX6Nljh3C<cx#?<
zQ~9#TeOUQfd1dKJ)N6Y!1;0Hyjj*Viah)g(y=>yHYeG`iLl={rD~|WNgO+g{^S#j!
zW1!0p+-p~16ZVu5;5CDWncig+3xmA*8b?2Z!dMK@LgK+z(PEIu|Cn4hw~FoS!Z4!I
z^#yiTX>##ns$Q$uj?v+uKe~o<idcNd7LU}_R65NziVP1lS`Z6j;*i`z{83(hO*O7l
zq7g%NgXiB)zmtSMePDja56;}wY>5Huk46}^2=&fQ7#@d$wO#b(?;0H;Mir_p1e|g<
zOt3d@bnL?=9nH-^W`hfab^Rc<BETq4hCP4X!G?4ko^V#$m=2#u+wN9olyK}~+g&hj
z1qN;Bulcn$Q7b%(w7SgW8S}n@c<cjwLvnLU<XKy6(LF*{)=LzTY>}zfYz?f@P$dxp
z_2vBfNkN`WU|fVlccKh1TkCHUuj-n?7z=Ya9S0WAB7#<@AuTYW<)i%}MH7avuPXp4
z?3FN$5D5ONZB@9Vd3-NK?S`6Ont52L<jiZpxI&+#($PWw3OFYQzj5R0a0%sRN4ik0
zdd|%C-ROTp1gl+zVG&O_{obO~jV{swzw;H2u~v@<jWEC@6K4M8T>RE2oOrXbESFLt
zBhry<y@!-~M0$ojveaFwjO^9oMyj)bpx_0Soy*}lv3<>&a{d~x_>v|nEG6#`))C8~
zpdFR?aD|3{!XzlFwR2=Z>iGWMAAj!LK*K?r>_aexAvE9(?WLJzVv!&@AvvP>yU009
z;*2et6)hiHzqe9!!lJ<{npE==uTtH^BHk8J`Ax2Z)Gtd)UQV-Kb*SNO1zcMDI5Q!~
z^l{K#F$8MWss2kDG!O}D7M=}$Lwo8@sdV3=o|;v32czwyrM@aCSCa3Aw1!YoY&1ns
zRXt@E23S=p+XDH5Y^3z)R~cFmA^QXZ6QrHSlgUqAItgl+Ieg@9c~kILc@wgnv<}g6
zc`Lb!)Zx6+`hr|k_fi-Tl4*E6T!DyHGtl;wFm}Eo#r2*bl^;aBQ3v8`8=xht$<WUx
zFg0n(8^OmYKB_}VRt7o}gQ=-m%H>~QCoop6Ef4|cuoIK_d<fCD<1rs}_g?H)753nu
z+L8pb+F6#S;-Vk<SdAF~A*Zy(ISL|(_16O6*zgo5yVy;bHK-4lj*21EWT}PkYwaDE
zhgCev&ul1Y^VRmj3EB~HzM77xKJOMQSMfZc+=s=M&Ok-rN#|%>>hc1pV%Qms0x!|2
z<8nVD9uP6%@Xo2SIR`yIH+W#mTCf<58%>Q_7=pX_X97gdTdtlG0nPH>H@RpREq;4k
zgZW}NlmEJAZO+}i0{vA0Ot@P$<#Eq{Ni)2&MpephW+zulp=-Vj5PnM0Y?31D&QjhU
zpVSHC@0*e<)ZI%vRId}+bKg3fIE4t-!6!&VF=fET@KYC-G1XPKqNV0mo9GNbNrw5C
zTIv&%Dl0MnBLe!HCCXfkD8nr17=JpxL)xUazBPr%MxDhCn7CrMN?bxiJG@jSY<gjw
z^P7!QwfniV1CFGhm`-~QDV}ALmfM{qAqK&WJGtU<wl*MWG*R48=bb;r&mz?a&N{?+
z7JJ~nwGS=?R~anUm7xkxw?@IA>PX&>^@rQf*<*}@Yn@ViO`&eTwR=Lq82>Rqhe!h`
zE?7y@*HHrgE*j%cyFVf-Sih%dtM>NHn=*GsAUcWLXHX%fVAi?@@5WvwcYA~Ac`w|F
zx0&daA4T$`wSDmXHPX?Ujkac**;?-ri-l=2vHxfOtB4SduVMHFIK2=&Pb;r9vy1&w
z&li$T%G3g5cLdl8(g`+SU4qopKOZZEwon~7&A(sSUP%Jf1QoUTQ@fmbcbLGnJZ^%c
zGE%l~KE?+3iQ2&g5s1#;{SV%c{5GCH%Fk+WVUnYBg11!j!ZmOj!mpOAIAqpV`zb!*
zW4@%m_<^OJzS{|64t_$9EJFRzrvW$CMZXDyu%-aDS$x(1`sW8!Ca|QwD^xChEK&@`
z{|ge-udnCDknIA&Hi&|JfOWvon@!KSGvJ#%bcG>p;Fr~&pNsF(Pn3i=bD?1RwJmUs
z+m8zL1`U;z{+O4NAa^m3RDRlod&t&JBD0~Q4!n+D-2JZc@g*iIxF!H~|FtWz4Ekm1
z9pr^4JNGx`pyJf1#JdCeWkCnLWNrIQ?!;ImR(~{bzk5%{8yE9b;1$YF_C)mjb&Qlu
zF=f9lk4SkoVD`F4N6_x|aR%Yrz=JLbGR6QOtQZ36e8MZ>3PTs^5qzq&+|({;r;GyY
zwq8KN?n&3}P{WW^+~koi@ks#8Tv|o5-|iG6R~lvtG@={1bPg5>(G%|_Q<w9lGp-vR
z^3GAm4^jRU&miG89$=Q<F(lmuymdtCQT^@?u3i)mnB7U?eS(m|fLZrea`RRPYJ+Hf
zfDe-Dw+_ee%{{dcDrk@aIv`aIQ2GGN-GL^_6@eOvT1Y=vrpJfh0Cgjtr+4y#vgHF<
z;}Ju(xX+t3Ur+${stdJ}fsC--UFP&^D`Gt)F>Q6*dgfxhk%z=Uwk!=A71*RK;bZgx
zka+Oovi!=5Z{2wtmA`G(9%>tBLQHOls^I#lo7D*aq;7jZjKO&2{*i6w7%4}!8aF@P
zhq+b(vo)3y&(>1%j9wWcJ+k5@cC2>Lz%2^ScyC-=PFG0%8Xtv^gofOsdZT+p`7L8l
zNN#b)%HVJaTIfgc1mM&0RW_F`)oDTjOe87qH!zee5=NP~4je1MRms5-<(5kA@cbTx
z#ARqDM@5FJ&5$ZbKWj1o52o^w>=fXKt^oMZ9KL)h!*1Wm)q9`VKZnp_414i~$R^O$
zXZ9efoYE;zz-aMPX~~+Cy4E=E;6VgB;LqQ60XXmn)0;Qzy)gcipHbE;&Wigk6bXXI
z03bF)xl8go)eIPi3?Or`R#mutyManrGfGp6cOjAV+e)XknQxW&bMjQl3Aa#JN^!}o
z&G6|U>Z0*|zO0q@E5{d>tZEaag}TK1E%Um|U_Yi@tE@x5RIP2+Vjrx47Fs=L@BrFg
z^8$4KDq}K75~4h>++q$=)gkBQ>c(6l6#E3~%AbU9(7Ze2uCPzi`RGUr4X6Oc24(2F
zgg^tsenP+(Pd+KC)rEm6a^DX&gCVS-53*zlDz!xMLrzR&`gVH@Kk1Z9kq<tlxjKT5
zV0Pyz!R%2%#=01CtW~IhU+d6<vN>2cWx@*C^Tn}bbB_!4(qu9Bjqq4{+s|sNP;kB!
zpUTE<b3bJxDTO|>=7&IPsYt7bF)&P_BNTTI!6!AU?QamPRg#{+yw{gnIB`_6DN`L8
zA^^;Dd^llQGD^0hz0KUIEt8J9PmIi;BJplH#2`+c8>EHs)NiFB6*M{@Wn@}Q`n#<g
zC`ng)9qTgrt$}^kd2rjg$TYQjAn^wq%TWfbcArNA5+(GM*Z@9}FCqxT7yt>gKq#`5
zKs|ry7MNdH2VsIkEK<|tN3g>Jor}UrJIu7xlAd&&YMQY!y9+mU;H>M>;b@P@^Z+-e
zKY}Ydg-g4qbgy*u&8SpoDgzrm>&Pj8T)M@-Pz`-F6A_IPA?%zKAv{vJO;@@?C=Cgm
zpo=K9!;J0o0yt=~9T1E!Wpk_5WTHI|JsIpjpZk0h#+@v+FJhPkSdT~_V;deVXiYEa
zs_m-po+Z7-sN@8e-XHH2+sy6zLzqF^MnYp|WOJ9e;qkD7pS`~-y>_h_Gl(sRu3I~E
zR3Ch+7G@V_!AD!ZgkGIfqjBVqVFk)?VOE@~gbp9IkcVUO0)N*#QILR1>YdY$96=>K
zHu)!fDM)8qPmIa4;5R)Sain|l-Q?E{CQz=Nf=&_%6%jn%T#KGQmy>y~qAv904?>qu
zS*hup&8+6!@s6M6VKl$Bv9E+dfN89x%)8si5_D-!!Q}_$Dm>Jtt}7cD9!-pgc!3iB
ze30s>iT^4e$+OWGW;sDdJiqO~vZT0d=}`hidHx`q3%$^m8hTM8@0yFl7=h;@Zp)~w
z=fP%EHJJFL6aN0)-#dDSO}!uG6hWdj$D#@L$|_GbF+ZEu)RRIJa|ZT}dFo7;_`=dD
z_xOhgg(j1bnP=;A=zBXN>n={}DpUAVgkr?jbQL7+DW+qA_Q*mr51{3ZGqb=H6rrzt
z^g2b0@5b*@R*lbbTS3Pmqs43cutWx;?o%~_Q_celE=<HoZOqlDUz&AQPd+w3w-wJM
z<zSf7*Ep=`jh6x6T~wieGrKL%f3ATgi+$qS1#fomR3+)wBHBGWd`BvcZS;L(kO}^#
zPGTOA2jDCi`O3}}`MfWum0f>7T?&EzU1sF<zy?CQ)3yVl1po?v6ILOj_xgHD#=ta4
zHr~T(1edH{b31|Y-46Fas6Z}SoBvCP3C`*9buZ4HoI1%3v)+;+I1cfSfE!bGOX>;_
z7(&b*A3csvcnNB}7t~?w06G~)ZZ&mo5Mt1QspbblEcJE-<2QTpXxjP@uI;FE^hORM
z^FSDnl&h{Dmr+Ty0;Buf^5j{3^hA%Zv83)GN0z(iL<)2iH+teMSz!8V8tt~jHnbVl
z15wiV7Nm7b07>73!}nUs>?t--P>)Wj5wV|4ZsST(az0;^szwcrLRHBnAjBsi?hR7n
z*g+A;?zE_dxD7&WsWQTyMma#Ve52YGMpv|HnEFQaILaY23Yb5WTkC_evl6hD^5<Zm
zjJb`<ez|IUl(0^5-ekc4h@oI4Ia8<tN1m}GR3GezpEGeDXhv!Ue_GHSdj<<3*SII(
zDdeoc<PCNZpB+%$HFm&1ozh!PG(Te$KC{#&TkgJ8-4Tzvxrw3Q?*lJw?^bn24?3wr
zd!)Flqmegg1DMy_E}PM~*vZksV;`IK<8TC#Y>hVngCvlGH{lsUjyl<44H$}okP4Dz
z0tEn)siTFf?+aZ*PTz9F^Z20|!Hz9ZV@R<!-EL~Je#~6$;Wl+xGdOM&7x$w-1u8GA
zJVztB&=IlbC<5|{<QQ}1Ka3Oi>`wRRX-9GtV2?!`5KPX4^!gK@A+}A&)uHO45KJD5
zPp-@N@N1Z>3YKAo;oa$<*+J96`k#K3Lqe&%W$Jd06dpMp(Q<~)$P){FWtd$0eK_cn
z5~qgY^d;azaenOS2FEokvWys<6M@XldOI(X!Z|r&!=<ssd~DFza+_dp9cXRt1iE*?
z%ZE80l~>IlNaD!5Sq8t?G845+==Vn?;oh=oEPuz`EIv#}#q{%jPE%5tQOY_cF&>;#
z>yL~QWj6nPJ8q<}huW~zTI(1^UfU^&yv_c6gLxu*i*|Kh(A)g#g3b!{btB*{sIM_c
z-7rsf2^;c{v}3U%(x%X87}4@lJiMjck2hH6T7V%iQeONt6n+#6J&!EB%Zz6ZH)}(P
zQ3z%^iD#|!ww}rpY7_R%E@c2xW?%-LrVU@LL=9vufW-1$ciP+wDzm$X`3n()`3*by
zE(|nUoQ73)-t<*04TjBeLsoG(n5%^YIruKuynVXauPg57LP-^_A1Tn!SB5_)$3HeT
ztHMTu($ndL7B~HqdPu?Fu=bUJDQ-<n@0+NlCuV}am?MGBG31B0IMwIvDjw2tYp>=n
z@85M91g-X3YX=LgWGaVX4;|nYP`CN6D@%9MYhSIIC*{fPgz<b8m>OWgRy1y2p&0;a
z?}&DAhtA;^m6+W_RTKvEPO&jqGRdt)d$scH2cysw3|r>S9@!(eLhRRLhCB&+;Q4RN
zmjkbs#dt~(%k@sXh-*T_b?$&=39bmE1<j5aQzH+V;tpU>jd|BOY0yh3m$+N`OQoh8
z2;2&0Y__NGO$$lW1SU5xUCNOZs%O*fb{B3?^5@soX4_y#y4nvuA!O!*ukqtN%GEV9
zS;p|teUlDc6WcHS!J0P)1l!r$-}x|e+of5MmUl}-T$mJ8tVW?!jy+4IZSUO@MN}%t
z<Rb+XndK$v?U_w2f&+@@7@Hok4?q$u7$6VkXMW}HPrhA0QBRdpI}1iy7!1H)sM+#n
zMhuOGS@Q~#RaF2vm0W)}<f}Nq(GBP&Z!nJ|u_-V{xU#f|OtWoAYEgP;Pl;CVrgbYz
zf=1TC@l&-ng{s*m)cLSAXZ^BHk#y{(Wj>fyXS;Mp>6z+o-hpMGr>w<MM^(a<xf+7$
zQ%3JOOoc+CP^~|o2IvNXEy;za6lB@BX)m|fkE_!G>x3e&qR~mh#{<&`n61MY5<$N1
zUzXGW!5bjru<H@53fU(OFCiRIs(F@R#Aa=TBQy#*xk|h6B~k`TD@)Qu8qJYPns5FL
zY=&Q1+H}6W3);OSg79V`>`{)i_65w~YYhGTvLeSpp3b^%q$w|X7NY<-aA%x3K@K5s
zx$mu&0=u~5R~FAsm3#$ydAROjF>k!~yZTjC{3A7#2g+an1un^|wFR0>ZSw1kZtFP9
zD?ERLF~|IVf@g!vI-VaMF+BBQKmMwO6|;3Xk^a80Xd5ne2xk~)b(*of&mw4!Ze-os
zsoK1*+hT|5o3ZMcT{ZD$b%{(19ZEvW(RILE0DBZQ_hDy_S@emUPPWr6)G(ZHr)?mF
z(2@v7PedD|ElB{zihULt5m!@40eSW{Yd414HxPltFvd^{<uNYN##Q2&?Rg-pRVX_@
zdZtVu$>0x#92wEV>5DsnxO#?vWlaw5+s@%8KyFF*z~T6rCxQPgDqT9whl%jqQ<_;^
zp^M;B?AVT}A}Ggr)S5hv*%Jn>=88Jo(1m9IBZStO24k6;;v?v4TFziDT7F#6t?6()
zbC&x|j@d*hJX)Sug~90*jTN9%31N}apW;!ap=c5HnHW1iuFX&uRm_9bbNG$=SYBN0
zLWZabn{RwPL&bTd`zP3ftce5mPyq%ghc-N1OjQDx^mAZWZpZ+R02=j$Y9YY1PWLcc
zpd$F2^o6Xa9=6Fx&vp*TS69;HMNu_EcY&_5>K<Ki6=}$uM}4kdN*i5=p+ZLA8Dy4c
zJ&Dp|htY($F6Pf44xRu%vsIc@5T!}-jrPPH9Xp!CA_=?EA;I(kI;?tOqnp~~vIYoM
z2#oKGtX0Ig5~1*r8f^wgAKkDd+kjFw#@b}Yt8FcqWkkkyp&EAVs~BqkefW_cElC!f
z{Sh{Ct(^bUCoe}-dbeBHB8?F|;Wldevq?1(GR1#u+;+oW9z?AXYR||m^kqTMDtHqm
zs-nOFL+VWs18A|}pFB2tZHdTM$&d#o6w)yL+ege5tqG1noQyZIr+H=f<Wk6>Yto9c
zb}=Cb1D4)N`aDExi@dfK;l-1mLk#z3Y9JQ=>79^^oF808$e*_|v=s+&o;io77y3$s
z;r35wO@x+HlfHFtTckT@iV*_+WS{*-3_&aF?g_&ka7X|PexeA?x5x#E;I^ndMOuRl
zB|64MA))`-&*6a?M?;}!(PS@fVbdei4Y~Gr5g1m(xUFZf?+N=Orzp{fcS=t%u?6n%
z9ZX(CeUi<)2CMCf&T-1XxpNM2xB@`&;^Xcp!pyX2?tU_DsH+Ru*Bwius=U0k5fY=)
zuCE9q6Wig2YB}ZXIem1#f#KTWPnL8S>)TiQQ1hm+rkc>_TM0{<n`#uxe1o*wH(^Z5
zaME5wEdJ2HVMij_JqNy@M~kJdi<w@TfPZ+I@2YfMOE8-?3+)>9@VpylC+V|efHob=
zd)xT+moHcAkz+Oxi&>z5%BV)<7wXUIh>|}vS+<CsBV2jd-7wRlsZfp7tg_51M~~i^
zJ@b3TId_lJfbILS6jKZe#_Nf14W}m;dXG^Rb!f*QIT8eC6&r2KfyG)gK-^mo82Q?m
zK*>6z#x<BFYLi$-Glo0To6P!1FV)N{Ft(VFSpZ7p&0pxrffq%*rCSQ>?GyJF9uz*v
z`gb)@^UH(tA*Hfykok~cMvnGhNVW<_q<NS$l<natbSX+yesX`2$VA$VyC^UL6ZBBx
z!JOO+e^Y>q>4vYM^O2c(^)I0mcDKSxfwxg!)^ajLG)>`Udj+YBj>Bqzg?35(w1U9=
z-h_FnTQ`Kstc2OMN@Z#wJeIl&_scM6p4|0NgFFRhjnWS}qpf`rb`HaQ;E_8wAzPlh
zt9%s774v&JOn)R$2IQUp)1p`n5~k7WTr{#zs5PL&y4J9J=)T`R3K!n7PhU3%^SJ1W
zAA+pC=a8SI1wAlTHxTJIbgd<oNrRr{-45Ftxpq@hs%LngX{BKIJAw|+&-L6C`>dO9
zZx;bn1+u!xF$yjcQqFLUL6W5|RA@Yus422Cl6pNEFH<~i`H0Craa7$Ky;P8;t>7{4
z(2#@p`O90ARw4)#nyOT$+1>jH8sK-w5pU+LNmJmU2A~WN*pJ7@AVY3E_jd8)63kp5
z1`3-S8rWkIsBjX_-o4qx1bliunA9$vqzax0*;S!Gg?dX^YWBm)$=rzmZBI2^;D!IT
zj;6{PS0CtbxqO99V$2L>i?qZdWu>&Y4o{#;Kbt4j_bT*)5_?umOd4|(ZfYP}%2%Z%
z%S6TDl}hvzwI2=1!pbCc^xKlJ`0i`K3Uv#ZzJj@|6O7Dw1bNC*B;?Wy2GAB8(xYVg
zDS@q+kR7}kL1g@)U?Pf;=0*MPmq(cnQFmJhWNPK~CVng7QB{fzSBrF0&8&-t?e~kK
z*q9@Kg-?azCuq0zHzK8K2=|~1OqipdfLx#1tnx<eG*`H`Bk=ZdH>%%zD%GcfSM#IX
z2##hO0+c0}{HPeHDDff0i(W{EmNp36YWqV@cL=^ZSspE_^!R@K+e>3bdy``VkG{}#
z_AYe2HIF3<`tkKWLsK86Nt6#R7BVy<XFfz#(my^Sw&?@(Xle)qrV!dn=r?{Y4dr}V
zTAVdnk?3>jGfN#2wp*Seb$osbyxn|qyxe^7p*#z>A$sS@dC%I?J+UR3z&-L?A5#*h
zRze$^>hyam9VdC#rbdyYs|H^H>Mrl=4gtc+f)+oF<^sq)emlz6vxr-;f?!%SdaS6%
zAH@U-?o%r{P~`h>nZ@yvedQ2D^n|`b8BwNL!I7QG1p-%eR}O0>>nVS;j(JtK);Qpv
zj`JyjT@^g;_IX8NdCuwc8KckRf6CcHd+#Cv{ayP^M6SiT^27JxZmALor|3ci6i1%T
zOs6i1J1F#_cM1LR#Cfh&!_#pCL`+2Y19KTm!NQ+NJY}5BBxY#58^r~gyvdvdc6!Cj
zF^B}frA1Ej@=f?6QW8YFQqOixJfRbJg91R}<yi$H>UHZ?<tmC<`e{QpqG%fIo0EwP
zvSCrVaVFc-V!Np2k$Ox1p?(OiS8x?%pa|(aa}JQlM^J>0GquIRA%FQinE5>zqH($|
zQlq6GLO;8h1t~Gq_s*Qy)hg`y9@X-5PW6)z3>o{Yxd|%^DLNPgIL2}T@cv%`jk~Oh
zkUUO#hRG-7gyd}SKG4#QqZSErTsq4eK*mXC1PhdoYWULg`Hd{a?N?atYEX9BS6NO%
zDK{=mr&<!m?Pwsa^I_B3mqMe~&4T&9@sZv`QnZnr`C%}rtYgia1wK)1<6otMy850!
zT2WjZ#`Rwkq!2MG$DJuiIj>y_Dzg(w4-gf*k~;Wdcp4L(v*1wSe|vT)+&~#9|EUBU
zdpiAw)#J6a$7YapldT)9A|TEACGPqPDImFo27?w<DOE3T=18Nw$>T`}Ua?7ls!Be!
zsF9x;yPhcz;_kv{=K*V5pyEULHRL~%KZ&D5JNtL)WcxAMzy+ocvXoprMwJTWtjiTJ
zdmQ|Ih}hb}E3%`{^n%y$g=1@0i$0edvL-PzFE{k8M@q=nBphS{1@(+~b_}fwM51&Z
z{%!aLlhMg-xmJGg+aK?w3bP8!rWIH|oP?T)@J>;h_IFP@gdeA}D%o`Cug$GecV;Bb
z@WTNqDoJJvZ%;5k3puYOMh~3yfAfGDiSCUezHl!Oed~nWkNQPn#B*B@FK<gdhXZ~&
z&lav|lqNIyLSoy@Fu@Z1=chdV^)Mw0qr?<osZe;Xou}OhJ$KNN71r0gRre+rqqK7R
zh9i*5^g}6}4Cd3?=%y64-$nxijV`dMMV6+>O<!jHT@o~^;1u^;RGi5TuK7Y9|C8X`
zbzWLkptSkige?2MRfTFeCb(+R<pI2L;{j>}EY}PAJDT9BAue9{TflFj^0~~-uHPG8
z;&7g)2rCl(2eG9ln3UgZzD3h<Sun80#yW7SFHoFDaT2kjBN97!M5OFSxjUn5CHD6f
zipDhM_iIC)`m_A(+0EG)y=#{)jeh};m&o4D@DEGYh~g{+$iLBLt&t%3;0NXOjbB)^
zK9j0XHepckld4UGYO^Y1YzYU0lkYry`*{c@q+#ap#KK47s@R}Dx}|qojOvvg>!ohK
zacyJ5i#SI^91<&Ib?NG(vJeJ!&}~9MS6mV+{iue{*R43k>o!Ep<v|N|*p)K1Yin|8
zxM@E(W$|uXk3D!qeKF<ip`^w{;&2ml{970|@&m?_=3e%K6}!zFEiIg(vEyw9g&w{K
z2wL9U>(S&;il5afU$*R(*sl93#0i6E6n^0@hYq}1HpDccdG#;#n1Z{JAqdGY3S>4b
zmgB#aWWSrjB^Tv=KS?!B*e!asOr$$e3O51wBwcf=#kWgUHXKtR3;i7O9$h+(TK0pO
z-OZ=CKSt&vaY|@*1@&&4RIk)DE$(MAaEe|FK-hX_vAhOG*ZX1V3eDZ1FVAuHr+!2r
zQSh6oD;(0&n=KK2Z@<!HtTH4l85{KYX;{34P_3q9fh#TLFe(UJS`FCfpG^Tq8q%Z4
zzU9s+Vs=lR@LZ>Jlp9~#D3mr{{L$p%(ya23%|JBwamil$Yb($a^NiksKdEoNv8-9-
zen|*^=zn6Y8*+VnUCk97^%PAv%@hN53L*%}ejObsjbAsCS0D(^`|+dl#Bfq_N#r4K
z-~d^i#peKpqRZP};X?7?=GV6(!}(brm78r(o-~;c%vC4e<M&I)V!Tqy2XHOtGG`6@
z*;7rpssKmR@xcIZ^F~8HUCLoCAu*hKqGqH%W4zQ;*vT&tiR%PYU+4(c1@dZC#)qfI
z?YLZV{A;Vya!y46<lzj-ZB!Eyw)F=q>>~^N<Eyh$%@k9Mi47BqkMo1?0CSL)yVN01
z#=DE4<SQK^_8`kJe}X8LUFL`1Yft=VED|R9XExH|&ozDZOmgp=?hgG<bceIBT`+cT
z$5B`LWu}2{PjCB|ID)GMUCD+~3VNg5KRpL1d{)%BSvIvnZ>jgNob*9IYhtnJAhhX=
z>DIDW!ouw2S?AAwjIbo<jvZ6xB?%oeORvRr!MQ^xZXNs_AE9cF{F(?OkbUHZW<KHR
zU92e8toE`S+85oa>jdBs{xGO(umRuh?Vz~l&~JUbwygJG8Lp6Z?-=Vrk@~Pch~<>>
z@Jq3jfARhWbBhLa_7%>P7(={ql@}BJKw>N&4#3fsNkh@G<O-r2ukSQc|3RUP+~8KZ
zDiOfPHWqLAm2=Jy{OSsO-qw>fPI(p4zEzeicoUG=bRkfjn6UJ;Ov<lQM}B<68qBkE
zl51vHpsECl<S#c4byN;VVmFVF--L6HLggoFFxo?xzoSg5=WURw)|vs4i#jSf#$Qgf
zF6+ka`#d_FL&i|m#&`Vj5vGE2EJ%j3z9(@|I{e)ZyG^+-2p2<bWaNieqGQ6~^iqNM
z9<;Pe2!CpHgYP4(q3VWvP+PgUyEbpF28KW_=43qW4f)%e0J$1xHvDn;R<U0DR5l$~
zGuopMKQ&m<zC9g7GnM4>M`sd!&KU?x+%7iB?G2A|KaqFh4OhFGsL%;_zPF)K)DCMi
z)W~q>G)?$K;={A2p4h#p{p&c!V^SwiZ{BFrdC=u_^+E`+=n7{3$LbH1i&A5!JP|{*
z&l+Gdm&Jy0sAj-n44S~Pg0&-DA7J$s4kmw9DO~c|(J^jP>Y9U0Rw|5j^VxLtlX)rS
z>Dri2PUFUfqa^M!z%iR@G`fC%og(RCK%lqQ|FW~4*z5IZ?4p9P#)V|Ov<f2r#LEhW
zDE#paW!<)XkP4T?9zjY<6sy@%rk#K-9r6u49$_r8Ye+aurO?^R7R#~7{u58^AtJL$
zKIJGdO*f6g!*yK^@JBGDA#>n~sp!x*u0c)>FNc|c3ggm*f93uuu_k;Vqbd`eF+`<3
z(nL};kutOFa%7T=Ao$PQ!oJf+QcB=>?w;GhB5o$LeXCEaE)qv=`Vu;5QbW*`%lfb^
z_+sFd($3@<HS1h)P*?M}6>R!xqUiw4*=_eETgTOR=clS@+8-%zF`KA>*f7szqEZfS
z+%R!TdL~ohY-7ec)tWre{z&)Ih<=+xJQ$Cq6q1PRPh&BrkhC7-nG49(!Cj%-_C}aJ
zm`0NuCG|)n6>6di;W9t$oXkb)C-zR)S}A(zHrMruruMin4)@>hHY=+NLKvbSwF3iz
zV*L?*U=Z)QU{Fl`*yrpN_7vmAcKS?63``Jvb!}+74_~LnzfH$Nuu;4LdX9Qg#ll^!
ztaC9Ww9xoThvGu(wI;#7ej<;4`X>(G)lbXO`SVAC`0l2_;=AM%y&=A#OrEli!U3(O
z20q*`_)tiMY|D(o54K$rV8bN2>>W#}(O881EPG-bpNBWXQuc=H^lu@vb&0)O2sugA
zohi?vwSUDPxc!3Cw>$4kFlPu1mV8Ht@~1TB5p&DqJa%Wfo#xrJTLTN6iM{RB`jCvC
zR*F>hwf4;T^Ifl5$rU|iZ#v$r5N(0dRd(5heCQf37-&EKZRA#S1GKAN*c?Cjr&nOy
z<nzbz##b?ayr_AggMK@jACk6{^w)iGXNj%y(F&gM=+E{C+GwOFtq4M9P**)Eh7=AK
z_R+RQQ4)*n8q)YSut{Gd+d1*wxQf6#FATRz421wFZ*SkOXqR0jFYQSTF~gENd4RyM
zxNnw}&kLAvv!8h~tHykEu+rdP8z3UIgt&Ocjt!5k=7N0+z}wiz=>}9#5M8pkL!4!J
z<2XKG1&-H1Gu5F6_#hu1n3kq1GhXLUbez|V*)ErP<T*E1d*u-lA@6=)2pi78&N^r8
znvpEuWb2bOMWP#t=1$>*s_q^=c!b6(T<kq)qn;x?`y(50;_(Xjfkf}*<tq6aeO`zR
z0DI*lBbS2+eqa3~o8gy_9lOV;p`X^KjQn*zw=bY03@~+VIojFYqP80O6!anMCGZ<s
zHOH|3*S;8b?h|Tld3JVjED!pAtJJ%}4cX*Ch)jIME4oy(G*_&j+Nw%%C<j#yn<t!f
zwiK;;wp=b)z`cx~eqcwpDW<s`2pvQH?3Ziea%_31BT})|1ZZd#5{`&#y`t~#o+Rt;
z@m!RG+)lz%EDu$&?@=gKs$D0}EiuiL<9R}nBiqOtUL#(_nham9?qJBj=v_mR|9Jwo
z>E5imfp+H%`b{j>8sp&HuIQMwwMVnekfTx)=Xc><L(v+$iS6IL{#hZR$DXG5lysrT
z@&hGPh0T>u#HIDm#;lvIhEW3eT^oBkmxND3^#bKLU1SM38jJ!S54g1bP14LopjPy^
z7WPFeQ{r6Rm|wbz`*lGScVrfVUu0vZ=S!@O<LUG2W5*~%#S@z%anLP-2G8%`{H@XQ
zV!lbVH?ds)fIAuN;Z3s&aLqLNqC8*fz6OL%yqli%kMniH+Du_@AdN+z_(hR7!7jB`
zLi<H9)5xMO=7^*-7Eb}rV?=Xw<5UAqxK1tJ-Fvr=`rwy`9A#orzj=%@xRvQc+i(0t
z?up}L*fBnlXjx(1E|f!0X8?XIk7^U#u99m@cbtV*Qy1x|;^O**!+oNnB!i8CI7M2a
zSq3Jx5TV_<Qx7$T%=$3ZdsO`FO>~Y)OsvsNPE3+-Fw#M0f$=99LZy}U9|LNDQE4)V
z1_Ok1<d4x3xLtxT5zpFteP}oIrF_fG-06Zd42K(3V$tje@;8JA3n(Bi8=Q0v<cq?6
zg<`!DHpR=5ZQ=LEAF<KH>IH6d+L{6fE73%7KDtt`_hLBOH6|GurwUAJ(dG%}OyV=>
z*?b-hRSbxI!Q4aH`MD>ZKV2)rDKH#U*J+tzzojP`n}HjNjzzfUj-?gP`lOL4<Xz1=
z69o}s?~9U7g&2M&rYHtKAS?DJb{H0bgY4)!RGR1y9Yg(H;Eqf0Vdsf+|0u3=c}g8m
zrBkke+I<m3>H@a`MY0lkX)I8&{^en7^P29>LRHdBR$qfg<BcW3Gp{6))?%o4Rpr+=
zPUJ%gG3)o~u6@R@y9Jw56l6SLityqdO^~aCdv=YK$4Db>m2MTM;v$kxhOwhqy}o~c
z#gm1;&!|~?3yi=7-MesQ!$o#gT;#&KOcol^GitnK<3y1ZhQv3Fo|*6;OS|(&L{Vfk
zI?6+RN6bj%5*4$E{C>+}CF_jYa+jU{?fl!^$;&rfR5x)e28Yq6@F>=DEX^GwMj{`w
zpH%rh!H5rDe6)Q4g_Nc1{7%+Eex(@}y@k^rUn!m%ioI6OCF=o#;E|ld#)5Yz&!Q@!
zj)g>q1crczQvTpPa&d4QM=vSjFSbGID9;mc_iWgAc&YbG=9yrUEFpw15dn+rAD)UM
zVPT%CM98FsQvkI3<L}Wz8n>ApK`)wbzC`FZHa3|Llqq4l9*q+Pm-~5;m3VV?@g20<
z;*;&&2j&$KykDO-ywnu~+PmuT;f>>MDSlat96b)xd09ofA)JkGHe$KQ%Myw4D>E{+
zX;1dSQZNEPBta>;@@c|(oN0b|+v1v*M(E?TuUo&D`lMU$c^nD;{>Pi6=ubIYyeda)
zyOkI26tt%kO-7(=l-KY{87>mhq`Y({hh5|@@~#~K*F6>MR8i%8PJ2GxnJOnOvYA7J
zD+|oF$f0u%wR%C5`HhWmI*kLlEkRktlsYlN;M+{>T{^5alhYt^2xhV#Q;7pS+hF5{
z@71lYb1JuKwL|{nl<^~+epn7dwWE*1V^hOM5VLu1au3Qg8JwCuppYHX#^3hYLypz*
zy5iSHij81t$??)0cxn1C1Qf&}neZ%Ylqv=%kDHpA?4<g~<cooZnVu~`4l78`wXVTx
zl_kt9lZ1KqrMX)8Avprt37<%StN6EX)_4pw)wKT$t2BWO636N+ZpY;4Z2%CeRj;)P
z0=9Nal5&L@AEDZjilJT4QkJ>UIR2vEDZMWBE!TWsC$-k2@$?10W0!tGmuw#^R%{ql
zK6L6DTkM?$$;~?0hXf?dPcjzHCsQSB4U*Ob$B~~s0>x$xzHPNp<~`99t+o5F8``hl
zVaAqEe#3rtppMm~+;?7ueZ{=HcOF_6_c#ZSdo`Qf4DuKC_bwZAG(z+s9^$;%zp=@v
zE1pkgX$9u90BT=6vg(+ALcca>_UhbR<1{1B4}Cuorf-*qdhQZQjbhTvYyU-Kz?5lV
z|0*BFH~q7H-#~u5#z?YyC-OH{>0F)cjfYY;PtZ68;t<kO>1_XgyLTaCK<i}XK!l*u
zCTend;IEKre+~`oou=4#g?;Q!{Jndn(ehqras{Hxz*&*684B|+9;(9S77zC9Lst2n
z-7raCdtyDiXQS#D?f1kZJXR=L-z|lXJwIA!9@ZL7G7*5UUb=zC8$mqmTtUuJd@jQo
z;Ndq8No&aE1$*xNQ<ppHJ@aNXjNs0Nu*z!KXF&%*5MtQOg&K@l{>o0KwpGl*lQcgy
zKF_=6D(niM&$Am?X>>-HH*kBQ4q{0*&FqCxMsL&aglH8B)LnY$P7P_?K7ZEXz(&lN
z+hA2PZC9c72v+!;CGfm7)|aGN%W98Ie&f$4ZsFi3SmE>Mh0-g9QytJLXm&a*lwAX-
zGswx)aD<D9hV01>!frg5t+vYxHZ$@%V_ZG;>f>ahFa5qJ;ta7eUe;v6nYSoJLB5s(
z8nYB~-^t3B2u71L>%Jv*c*eIeGb(ERU_bVhtVE|KB@au=LJKKwOD4jHr9$y%h(MO>
zm+v&SsA}6M8DPomN|jD*qRlVQ$BZioJ+OhMH3zd$G7$y7x?xK<bp&ntbo*83$D5W#
zQe4N!TAN`%D+I+qn3LB=_ML)W%x!XU(5Kb^;82mAe9=ws8I3t=Qzxh$t`AgXCBLfU
zU$xMgY^ZW_dDMWa-=$9Q$&usIE*L!IlToN^9m8(+$XL7zq^rf*DcEmaljf*i!u4%|
zvNwj><IBU!HPO;fNMULC0vRKCE;}qujUx{*e@w!gHbuZ}_9JVYPFgSIt+txUOG&Z>
z<vY_r)B692Y0iI7@St0kZK78+)(x@HgvolT-_3<3r_TS<Uy$&Aaz}PEYLt2KeKL||
zK`d6Lkaa76^7MBEx)D@@T5Q>R08U-sAh&rc6g2don)#5tRdaz3iM($UQEvX<rj(hq
zjt=)b)jzsJn7grpjwBbZH;W-Z$3xY+OO-oVekFY;jvVDHz0uDG>eH8z<yK&nejTQn
z%?`y`d{^uh8@|Bl>DlTZ@_bKC{GnWy6L^223n(5e@<~y>W)JvUGo~?YUXM)cQr%a+
zqnIwV&X!R5&ZNmU0R~2g>EXe}C2m%V(OhZ5!&#^a__M57+Hz^%lf^@c@J1!Ncri^q
z(=XV)=MR@%uz%OovHNhV=^SJ+^;@zkcE-q$?n3Gi+(3Lvm_kthkh<yEelER&^CBA?
zzOi_>6M{=-W^P+ss%c6szpE19d8O*yh-OqLo`<Pdvj*SNfpiY{FgpmYyn=?0oIK~b
zT%a!luG4`?;*C9cDm#fgMb+fau(3I@|1DfQk(Hz|)1B0L_knQ-&BM0PGArhU&`2bR
zI1b(0BU~8TgxeM@A7$18Q5khIKp{=>yB(_I0w1`uWPDVx4qT9~boR)KRSU1&C{lA7
zDR;P7StFotoQdl7S00WS3$e=?6>o-Avp##gz8YiEu^eN^sLaAL#=2H;U#vjBkEULu
zivRV#G&aVkSMVaHVc<&#n6wdXD=DF3)OgFAN(R+ZU!mY0PJxr`?l#$69VSHJ*A|W}
z1S7F_HY3rRAjG0%Bqch^XY|rZFII}=I{qd23u-Wi3>jZ}{TGvt9Z%$frR#TeCjvNm
z*ZipbgZ+9hutjSUG@Dxd%`jJ?^xEf{1AuhDO5#M(Eg?ZJn|(O7uj+^t@TQdaPltZu
z#_zU0+a-&YKpU><p>lcl*4FS&;goJld#<VNRT|OZ^NRtXx(NEN{SR+Ni<XZm13BL5
z$SeHHr6d1`HtU2Q=o1zumJl3c;07O&jJSCvUY3OoO`qdXI?~_Iu}N1?KhoLx){R&+
zC|Tk#6>ijWWACbY6LM5jQ9PSZyT;FNF`mCA7jj5dk;arFZIE|Q3%%2l>$2J}af5cT
zs2r|<wwXVHrxF-vG0j_cBBH<YhV1OePZF10+Ne9CZJ<(`X&z$cExo}ka`FpBeX(hq
zKYfVJ@}}fHu+K37jTJM#HClASV6NfC2Q5b25spoR2R<GwuSG;y@Rh*PfnA_|ftMaX
zgVnyKw@}$r+Au1K>OzY*0MYTxHl^cE<W`mwX#a#77<P+Z^2|{hS14;1d_bJ7p4c+p
zrIcVJ2+`hrK;-%VvG$J9l?Csc=!uPv&5mu`I<ajh-Lc(C$F{AG&5m<o+qUi7zqzy4
zT{G*>%!m1Z*!B5+-l|o#_pS%#e~K-wjTP-`hQ?0tXs>Y*R8)))w*5fjb0>YT<b%!&
zs21y-uoP#jq&0NZgOALrjWYMyM0YsT^TC{Kt3`>(Ee(@SJb33%cr#WdOR(Vld9b>9
zD~|C^_uzwgg`<fbt({)ncuxc*chYNq-`)!<Sl)Olv$9HdyFDAh4v>OJm|RhV3pp^V
zsCxv#HF!NGogv6Ab_Mht7=@1n;-;FSQoA}yrv8Xr8Toz8<5>8UL?!lQE;XY-J^E|R
zaMfR0ykcV(Tz;QMe55VT;-n!xfHj(RR0sSDW80%rXMS>3N+--PxnPBioPh&pP8^)}
zL0R>vKvy9<BV}i!D&OsrZNZ;BN~s#j6md8XsH`>S`DQ;O3k(Y4(b{`e$k-zisvz+`
zLF{M}jVk9@KUSE(w)U)UOsfNv{3|sw7ECyJ8fdRK<y3ljPFmrEKd_^~c!~3iZxIQa
z7@GAmbiA6Cy7{zdO1McdX>O<mBB4#0pK2?}EMzby+wJ1{?k{q?CKItDwGC!14(g(C
z%KDZ-?LEE~6#e5)<ADo@+e6bHTu7A;kfQ?z%aTjJaJ(T;ovmZXhc_vTnINalhoI~D
zFCo?&abMQwUt+OGjRe17(E9IbSx5OYI<P+6y<b(&7TsI}=86$xGu1*pLqijRSkh}r
z61%56lAM%9NU1#<i`AV=Fwo%l;M=%H`NLF9c?`B)BF?<o4^Vul^4rnNhQPoMxGa=G
zx}oovlvvO{WIrZLqJ%7|7GmJYR!f`XMbn-eh;}4?f~R3I6c*9pKy3HeV9lTJt+~ZP
zJKoJC@od3%(z8E{nPhf2u@GylU+rQ1x!4m3Yzc_HnGvU2hW+rzJ?e4^#lxrzhQTo7
z`#$Qe#GzOGO+0f1=@FHs=T~QfOWr6RKh@*g$o-F>jHlmiJq!{`fK=But)ZH8(c`bA
zNh{3B$4tBz*%iF`;J=dvtU`2da@ETKyhdy<Eh+irA=1XHfk^pCYa9^mR0J|@N14K;
zhqUU{<UA_H_J!3dv6K<A_2CQGHf>O=_2QdNJ!JrbD22>!6V<+3>Dde^>mf4XmZt3m
zk9vOiL;vg_v#yO@=g946`ohZyQFyYT_P;Q_h68kf@6i~eu$3npcD5xpdPY$S{n!}Z
zJY#l&zQG1asSUJk=X5%y3M@Cz2whlP=_}4}p2f(~N-{>R%4VZd#^xVw5jX$DSk~Y&
z!k&6HY~GayH;+N#Fi=@T_5G*oEv+ncM@5Oppye|f3@_Z8A;+6_^K~zTc`L2vkzMe@
z-L6PKJ;|{b+4vDM64!oPpy<>rR2(ud$=UJyGP6%4I_mLW_7FMGv3(y2e)voGMCAw;
z^E2ltLm}FgPS>f!f-_3lV_QTb8d)}Ayj>^$NR*F~OW2klx~dNMLLfC^maxprA-hsd
z|DG~0aiv~$5E&}1qxTO5MEe1)Q#!I+Wnz7n1U~yzFc0F^;SkQ4jKw$M47ugMVsv3i
zS`d0x&5y`oQ?%w!Y8-SyU4;{PN3<qE!oB2eDLg4#1#{4^5-msei*W24@s^xLA7$?b
zIY+lN?yHYBvkhjd0bYY79Xp<iIykl80OY;>|K9sKe%u*4u?v!e4xW&iyu)mfXz4&g
zD_Lb-9>9^yVCqYSz$gXzq+9Bldo0W9Febd$2bT!w0rax;vGk-o^x48dc(8^<cWOqT
zB8_m%Jh8Y&pOg>FO0Qp{KA+yq>|3@F1jCJRv6J^&FnJPEvCwPUA?W=6VKMpj<v%`n
z(Xo&E(8dc)g)<|-UT%lcdcbRwuQ1b|ec?C`X&HXzk_lhu*1qx!X-gxw<1!<?WsIkR
z=|0c_!fMLI437)wwl1JN=ZK0&@d!IIaWMb9>s&_$NqAj5)7M#>BR;yrBQbGrGLAOw
zC#*uca-!rtL1(|;)FydkFIOR{JM`#Gp}Tx4C4KCj>;&IFhKc&YY~RW+e^aVHq^vFf
z)kv{_)Yqz7$)S)a4u4S;eYU_|7?o1WeY5HDHi}Fc)ADqi@$G7hQ40V1)*@w*R0GVv
zXizj|oQ9=C3tV$H|0hbtN)C$7#RM`h3f9EU<{{>nz)mY6q!8@cn`XGI2fWLIdV%ym
z2~cR7u4`xX0l2a<Wf;KP2WeiH5yWI&%NXYLjJChq!o2AMu)dc1QK0WhZda|ss&Dgm
zaIhDzIMKpN9l4UFPG71YVTxoV6Hwcf8zRXsk$g5*TjZU-?m)d@rpfpvTz%qh<!yf0
zS`{zBqecz|iv;Pl%_ZWT<}hj~N4+Hi=t3!LtAsC-wb)Oy>(@`!s1-a6692$bC)rlV
z*12=SL&m}PmIV65PKIa3uN_(t&4!mwf_MUG0Bfb9+y*sf!!Twr)G~sw)FF6#crkzJ
z*pJ18g<8OoE|{_dJYH#91vkLKUdj|6QRqM%E<U?In!tM@FR`F_iiQr<p{}JBQqjXO
zDcTLg7XOHss=S&`OpblCdb#RI`>C)Jo|$ws?&=_9gX^619%B4ReCNd0I4Q={GgrUo
z8>FoT7uC!aO){5#O&}XwBA#FwInE7P=qJ}LZL47*h-l@J4{BPVxd4z;iwK86u(3xq
ze<{I!>IEMVsDFZudS>*S6aFPuxq38F=&K#IWOGspHzxJx0qKyK@utO?ja8|AKQ7A>
zfGo33(*Ya#LxIlya<w@${()v^^oTm#VWAX1Uii^m_AKxCorvqsDY^IDx};9rJ>iC(
z;)puksRr`y7kW0*!%2?4*#)};bmAM##dV+4h<kp`kr97x85W!6h*vgl18>$bN?C0Z
zP&}}=RY)_^uS}<Zeq7{xrbNjDoDmxn@?jb-X9wrq4|B%4JbeFe^JGUWd9y5)sWC#4
zBsv<uOcF!ta<pZ0>P$60Go_;Q2nQKr2|;nIbt|f{#|mxif1CLFEqa&jwkRv!Q^`l(
z5B-<JUamgtTXwhce;f{ydX|v+^B(LdS{4etSAK(1!)cw<u8Fm+0|!|_(cQlqtqG{{
z^WJJGNv^>2YJ&HaQ^lMZyw6C_p8R1n-jWO}Q_$W`+}zs>AX)0wF%+qdGIJqwEMnvt
z`ThEW9*EA#mgi!`Mg0>eIsf@64X_=pfe=8Jt^chXc9k7v+#|<CA<w?wR2?mb#Sgv6
zl8CT-Hg*^$6V4d8rNJ7D7Ux%#V*CT&k1bDvIsSaZ2{pc{bIP$j5^%jc{2PI3cb6MU
zV)F%q4v9J%!A9P2pyLGC0azhRHF!?bk;R_v5|%+NF>M@AO8q!WUUuwzj+-T6J3G90
zI;QazEG`wU%9gsUijcGb{ak9KuX)cgGS!pz<X9f~qis`zEmW|WSg#R1!XV<%Jsz=V
zNYZ`WhdBB|ZHsHykHf>VLgYel;jhII#<n2HG^lmxEg<}d*C_4y2xs<anj+VSG+?@I
zjpKzh+ip>@bR<lai3%H^Iclp!&(sjCpd_(`GB2!JQPEwhJRQIMJem#pMnCT)7cUed
zQ%-;btzU+}cpBf!Hm`0b)C~9R4(&ZXHGuiCAp)!?ae7f6Ts6R?t(xC{9yGQkCgcwv
z(T)o(=ov>wjYM&GjDu1E{DDwxpfAT`{-Je3Hv*PK?x}oUqVl-4TSqK@ks%<%4W>c*
z$w>7!1E(IgBe+F6d_d$ykwXnA@Ry$1RJW4vkldq3p|TZDNX>&ESb-xSwf2sI)|rbA
z$9cKJR;q<XZw&hRA+a4=8yPdx2^h8SAYgxUYr~WM6NyNrC#5SP3Oao2gBw<|J7&{b
zNJV074}>%1)Pe5(NvgcxUZLUvsWW#&>I!a_ei=VJldd^zC5ajFC`wM=|M0u^?M~Lg
zf6fbr8();NCbTb>tUuh1)&as<#7ZrwP(*6vw@<#aQ~XY8;Ypl&pX82U`*T}QXL0A3
z#CRr3ngi4JF6(cJJAvBWDJ%c^gs1`!jHlLK$fT*@P^Ph65@08aUBg$KK`)4RbA}nb
zMR<4W`<x3mJTjHk-k5OCPoWxdp=N?BGQ2T&BlKL5dVwX_bwLvei#SE7*cT9dyhzMr
z|B_kK!=7R5x=ZZFfY<z1^map8>~IJ71{^$<7uy+gcnV%a_-mI(Qki$3JK=BYK8fk>
z{A6e@X6HuC&!8kYX$g&MJ{RacM#%!nM(p#q6CPw>Z&b1U{@+1f|D0JtE|xu-NceiG
z`8jON8=bnHVYO@S{9e8$+1CV>NUr9Uj&Gt4rjKt0yH!$!mrV$_P_kSAj}On{wb-lC
zMCfOp?6cRxtX8m238wx|o^RIa&t!$W2+YA9#`s&-M+u$!0mg9|hQ=YA)|NrtIX~m*
zN0Eem$<-ODeQLi|IF*vw$Fsyt@Cm-O3kHC#O@(lu$2q^F1v0`@J_n>;HelFHV4D#`
z71U+PHyoyNoN)oF?Ohx;#T|rAg0?a!m-Wkc?Q<)kC?<4(4&qHvrDE3%Nh@sXFZR6k
z=f>&)dzdZoYn)<}xZGw|Vq!oN&0o!Jr<PuMJDeyHAPZk?T!YT>5RrY8^zox8qjgeY
zZY2DDjvjLn^$|7;ui<g>kCan!%vG{CD6g-N-tjb@yr3Wg*-en2!+Kt#*diqDp_4on
z+ht4k0E{3nn<i_!Z90TI(&Cw5T*(o}jR91Ioxq|4xILZWxzW&e=N@B7{RXz4H56Nk
zTGlfp;s+fx6;<ueiS2(<0|(-&5k;6_io9&dybr=Wo9@}Rago&u9fp_Uo(UlLT&O;n
zo@Ax68t7dQjeZf9-ZN7*X)lbLGfLa*Pv}?>0d-!gyY&#SY#|<$xLaw7R&aY1kjAlh
zDlshB*$>vbxF6nf2KhigGHQKpXr@@__2*!PRlW63&l%W1O%K0xY%=e^!HpG3Ud!I`
zqmB@}h^z@jb_=eK3_NZ($GT+PIyd^nKux@2H_^o*L$#oZIx$57%-agOzOnPk2p~ce
z{){Z)28qjv;**9!Od**T=of6e@Ry10X^F}kB3*QRBe_ov=3&@fS24N6m3DuxQJ`;e
zqbfI6`{GSX)Sa?YanQ5uA~GNqzQiy8f?-h1F~#HmASfR5w<;14kuEdyEDPoI#eRJz
z#`)WS-A!Q6OzQ?S(bNa-4J$l)V9lW+L-|{$uYtQ4N}cj3%4oF=xXmmCcZ8wCZTDXM
z!a)4T0Jt-3%1W&L<NUN(b+XTpP8Sq8jagwAX>MUCH~I)@T(^2g;4<rp26p$THsiPY
zjTXD8ea)tP(K{#4__q3<pMC}P&wg!xkNgs7J7-Iw+yd0~*x!cFHS`<9)SJEM?{N9i
zGBdEP{g~f_(2Wl7JYlZ&L9=so1Y~X|0?4BCr7baS`f_^U=#{#e?IFu{{bl*JWRFG}
zjZOQ`(G~WqZ-a_G6p4xVO7@>e0mOhj2w#zth9cj<4$$6VV6{ddjgIoL85l+8--+Q6
z!D(&TGk<P7^>)}RT_7;wpz(t(e~a1~U3kLzq<b4<)-QEV`E!MnSZSWu=02oX)Bx_Q
zV6pRC#Y2{8Z;*j(Q{}%KZc%+0-k@|obq;>>KiEO2{6)Mhq(mghB1FHAnQRO`<C~!a
zN)y(`A)0y;R;9P1GXQp;ko2RlqZ3qQiptqVY;=Jc%gIq(-=^AJmQO71L&0q^)Ndt&
zK#I_=0ud^UHQSi(8=Qq?Ml2uLNj8m8rf2~f1%TRTeMT%}0s37n&{l!3C^jCZ%N(3R
z(Q|?B?7lq5-3d%fe6wH>(b_QG2wx`_{Wccf$>c<W&L?<^d)FrH`<;>B@B9)WbW9^r
zs#dWv>NIp=UA&?={?4uf8Z8Xy#`Cd8+yH6#Mh}XR12_ZD^DbAaF7ScecOU%pPKb6_
z;@90t*km%kg?s0XTTH4;Xl+P?GV!Z$o#zCwAj%OE<uYzJ%v<bcR5J}U{-Q<U$B`DX
zYFNbuy7<bdheG}EmB8wsZ})l>-RhMbTP4>8J|jx!36j@R3p0|MG;qfl{!J>4|D0Kz
z2MvLS<^ZJ>>;!z<RvRC^vX{LGRI3+~fKP@DZUGn*Z))}tSrhjLr|;|AAS-M_Zj?`5
zxL#3H_yAAa-$+x+w3iS@{k`Vtk!TQbvd5g$YitaSyK;%Qa;+Yk0G(+AFK~ur{t;e!
zI-*;~YN`m?#<0!biKy_myJV@v=#as714&=sPS}&MsfQkzRo~_W5<yek5;%&wKv5&+
z<*EI}>=#*SjxrK(qWGs(r4s~C1|F3L!8x{JdR{dd1Hb`c)$DHpN3&gHf?x{w;<Ba0
z7EjDQCV#th?Uc7l$mGpmvg-w^HpA*rgOtqP9f#kqBIG4Snve*Ye7RwatemLc8l}=a
zIdVyhm;)%+;affF@^=#R&4t30CX}ghG7cD5{O5;SCn(_^NNrhSZ*$zk5_*pVu#`d?
za||e~#7fWy2Mao5nr9&FeGvLZ1&yl{#p~fgVd%IxnQ>V5K9-vRNW(d=FufS}%j%^*
z5l+d?-qWFewd`;K7pip#kBcLhAPG2+ecO`|@eDzGauk<Ae>jroIZni9Gm$L^)hH3q
zT{kFLZeRBjtMfv>8G)YjZ&ech2y8jy-yrE!NEC9w5U)1Ht%aEH9Weqk?9;L}5vi&n
zCJABK*qevMAtB_JN{xRwFzgh^mN(t&JNSMlaJLTR*_1pU{dmArLR}Tt0~RcEGp_~e
z#civo$WmUO+k|Y^iOaz<zN0~ZHKWNAozsIVB_xR+pgNiEXB(;(xzx^cam;IqT*E#;
zvYV&N<^Jv8iIL`cdM;$I^<n-$>&^+KEZ2SDaFz^}&-<I4*d<(k!N{K3NeGHi-PNcK
z$Vpa4K=48;Q>3oy`qUHO(%glb8GDV-63gk`H@W{xZPHC!2WP(f5JvAvJ&`9=@_Ats
zo*+gtAA-AYKeLYPF(b=5^H`Cn>~l>n@achuh2TTe1fw8=BfMi?Jh!gE36d!mtBN_>
zby`QI%&B|r13)e&DJQS>tQ0Ng6m_*r&9>3{k47#KMl<6e4YZ&hdvK#}SuOm`4V^cY
z06E`;DjY~2J6PFL{9gF*w*UE1dpYepRBuss`^-CqrCAUEJu_<?V?x9<Z_r`-h7Ml3
za47dolz2Zx4UnKayJa&R<%1l^r>T5C*7jUAN9a|FX{Gog6S!s{*Hal1jMlKj84mC@
zce+q?U7y#cHRJ5pNA7y=F7ve1PGa9_DV&vpq6F4=^dDOD%$goBBCnG46t07bzI%&+
z=^HWMEGyj~FOXraoJ{xmLp8xqsoPv0QPRv!K^tdJsMCrF6<zKbPg4ugF?&ZdR}bf9
zcTA{yTuECr?$!CHmb~A<G9K-gdn4O~Oz88_h&e&kXbhlZ$wG_d`D+9W1RKJtyJx_R
z1!b-E9?a<ffj?ui{zj<W2MW?Ik+K0g6MD{r%0IJ6OH8jbF?L2kR5>%a_GZGyB6t%t
zpP5{C>SOql{^=rE{^zT!0$s@;P|qB1wzr{8yRq6iT8CU>$$`aN6h%#}YvG=V8^|He
zykd?R$61yZuvM6Ib|URxL4CVlKWMd+K0CPRAHJaaX1V)Cz&slP4MX35B795V|JoG`
zj2(*8jAX3}$V<Td6LyJn5E@`++#pj-^8zA!XQ8CvJ^+T1rHN-^+)T4x+CuLCblzo%
z;?R4y3H_FfLAOsDnPhOg0dmdq5glt_VmhEP{*9uLQpd8{yk~XS)!jnBhOb@>ArG-i
zEjH=ro|CW>jY`p#^A~)KM#+gu6k`c(v%iDdzdY*eEr>l$nhFR&co^6rGoXCZjmuG#
zF;kN?=$SFTW(?Dt-F&>$eJnr(g`G^TTe(Cemq&Iqbu-^m+sRfJ^4dwQjGaX@2UR<W
zD@f;bg{$9OuH!j;#WFU0&$~;K^(mD+A+P2CJHzI$N1@%1uYAuh(NE3PD`Ysru<OKf
zKmciW^)y#$#olh4@YX^^Sb0F{bUSyD&CQqXhHc8Og@t&>k}(IK>v?^yUDcz!_+A!&
zhF}^S+K*(Ybo#8h;yuUx9Z^8=CbjQ}p38j6oZFVj)W?|hZ)$$%1nrxcS(_dC1zDt3
z?1B9e&xUUW|J*WIwXkGRuY_S#!WA8E4FRzp+X>p??r_Bw<7#v`6jXMlHGBpl-oiJ6
zd+?0q@iq9x@Ls_pg^UzjeF6SbYN}!|cI;p)t^5OKL&U6W;C#rNR!qOLjDBY)<^7go
zc(@wVEZ}1=NPO_xrx~g=GVE1WYRRO@tIxv4E+pZHCpo1=CwxmbfXh+GvvyNqeNcCx
zX8!OPKlId47L@VEVPqAC!INpltAeLNk9s%sP+`C|Y@7n@5rCahyMS+M&orZdoLeT#
zK9hLd?l41N#UtdG{~r7$nRc%%9<GLFhtR7)B@kps4yADIv3jZ;2?qbxaBk>uRgR7$
zO)X}IX|4;HE=5^Q`e`xRW#v9sofYt>vjkA?^ZW7I_K)mQ-bR74Pmqq3MSBDb1wmq#
zy$HC!6~2d+Pv8u^b*C9K1SA!u%U6RN;(VUZAdFidEd$6nY3;qu;HDZtSJ2shOsjRF
z2n%gsHLqmy!Nc@aLQX4HG306wjCzn7=}O8FC(<52)-m{w4>>nF%GlJPUqIo0)R3pS
z;yZF2^Y3@@BBcDmS6cd0E1GH_rGg&A)IkL2aZGNw<bDqF^pCWCCwFQ`9p+04iA0=;
ztOdAk|LR%7wrZ37Ekp3s;nIq>XnJ><8EAS$>V3Bg*%51I+(zX%RvbpgS*;OT5Z}Z_
zU;v#&+dqMh4?M{VbkqpJui4+xHRF;PN`F>My(+DpODilT*2soX3c<7$jUX==aPB!7
zMQB0Z*h=FrC|${Li>m$lx~_=IaFTaC|0po%?*&Gq<@W12Gulv|iMm#XC_MgiwhuOq
z7!%KNXcSTawe#x8gLd?+4H53N_MjBI0G-hkiEF9R-SmHKz)QO#n&NGNvRThYJ%3`I
z+vRGqpKzWoIpMjOuLUJ?1~$C+8Q?ytH#&FU7j&S-*{TKjk_yq}(*7n|)lLpo5;Yt*
zT9mfMqoP)EF7={LM`Uj(f5$lj{9u7-zcgFu)((brHlgTK$79^)OPQkm#VR)-#vwlk
z!Sly(1)8~i)ac+DCo=|`Nz7A^5fBxzn}-JPhM8nAbYbl^m}|?uS$R>#3<KYQ9muRW
zz)M>60xth4J$-zHfzSOr&y7_DlcOV3C4{D8O#nk@T63WF@z3wt!780a(T^7s@>|O?
ziBj!QYF}t-eh5g%P+1u^N6l9LT5%E4b+@11SdEKTDD#jGosox(;JkT?IU^&%+Lwem
z-Z)~gz8>7X&CXC`=^(C|=QxwRCkS;#;(3xt<IyVKW7b1OfZ1~ht~+tj^3us}W4;xV
z{#MMPua3p8Be0SLVH0Ev!}1d#%RfpOMbq#pX_ec+t2Dc!9``58!GRtha7WY&a+aD#
zhP3F(feuT`^EkmBK1%-?NS2C7H`)^B9I=3Z*7}|;roEQs1RgyMXde0KC>W=KsolMT
zY2yabjCS1g`MNUt(d>r>kI+?`#eKrf3&9?r{lu(rfSrf&uzto2zX3x|+@c~`ULpK7
zMfWHr+Ko)~a^wX~QusqC+b3E+NAqBJJS*5%wVNF>&ryi*RJXcc?v(|m#gzqOh)PG5
z#e9wWxJ&4W4(#yy{M$a?)N7dlCx#Mz#5>O-myBs}P~8-Lt`o)d+<h^IIKRchKQ^L|
zLXX`)BV~wJUNn)4lO*^@Bpv*Yi2A#e2u>5fTGq<hE(T}+*QR@Qil84!?9k#V0(REW
zgP&s-W7|UeH&mZr!2_Sk>{v4^1uu5F=8E$K0GvQ=tD=3GQ`Z|+H+IS5l~IVbXN5&A
zNBHT*L{UQZImd)c)6^aU#)dBJk99jpGn-J#r0^6w3u6$KgT*2qa5I+I&M(=`TC&q0
z?`7%OV_7iASCZN<a&l&rdYET4Ee-iZ`p{C^gl6?P`e?x=2Cc+r6xt$~9Y!m^3*!Z!
z5OS$rQoj5a5JqPeMtx4Sy|SEk`=QBvtJsi#KO=BO+VpKWsgFo?)vq-LpM1WKB`?pF
z(Ww82)nB-*TV&YH#@kZXXrS@ZA?#}iS#RYJ9(0t7=<jb?b8AH~n}>yot>x@!v9l|k
zV;;?hvsQC>`+Sgo1+AHUVpZa7KMchg!T%97+ovxvYaOXb8FHca$e><ZiGy8uTBORH
z+;`tItz)4EOR0l-AWD)wFoh)=)C-ndtZUUbel{4!&61|-w$qIpge_uo^&c>loIexd
z%~6GOJ4mKFSBjlIsHjsx<WCig)vP-T&K|{1s<SjC>;~T$;~|Y$Z3QERQv!gJW)aky
z0=%M?61uMS<=QDy9yzR(y<<+;c#Tv+&5zTz5AGg@7!-xhp1Udk?r!lIL=2DPkbjhU
zUB{p<O{bG<frxy;1sgf`2mpim!tg4Eh3_JZMijfzT1`k|btkq3T(;56Ldp}5t#d%b
zJxVq1-}g&u*((yf&aeVxrSp8h&MSrdL3{0Ra-W*PbLEvL$l1Fbxog3B>XR(>_DWu0
ziuO77X}9QocwK|*0z6dQVcmt1Zubx|kW>{ekksclVpwCd`uD^YKhL@FLpP?fAuARg
z#^#0~R63wmfTSrf>R<qzU~=J9RXhG;^!>b(ygdV&(vx>Gd(8g(cmb^%)q(Vpzo=6B
z=JW4u67X02VPH^6n0HV)MQKncnE8J>Px=b4qV@i{gc|WI+j{WW;ZT<c1n0m+!qcfo
zFWzR^6a=ysmb=HL)*j}RDrr##SVOk&XoKFZ>@(*^xJGXZM9E7?sFpo!Yh-xBqb}1f
z#rD?jA}(l5m{#)8Ua8S}!m*r8RZ?J(vXo}auqs=P*hPK>#=93tIVAx}7>8|XQZgDX
zODiKr?4!BU985kPTi7)Jq$P`u6o8l9aWXhlsJ*Wl?H_v&k`B^4*QRth-6F@w$^|Ng
zukqotREyMr+yO~C6brrvGvm!f^W@jWUs(L)c1$2f_Rkj+%()~!M#ak+w_A)tKE>@W
zfPTq4kKOEv@XlucKXb#ni5BuL$Fb%kp}gkiqZ$}eZK*=>5mA+~?nUKCQP`9<rSq3O
zTEspNZv&r>b?lCli9E;!yp}7~!F9JzOM6`rKaJgb^x!0ZTW*E=WmnmEUb%8eIhWyV
z7*7gCHGcJ)Rsq=w!U7#f@QJT%5}a(mNKZscm6XgzBhcF_^J<DlVKC6EZvL=wYy<Xk
zj%9d7s_#o=gMW?rH6+oFQ{g3L8?9g4T1Bs3#Ms42#XvdKjR(W1J2ruP%i2`SH9&Dm
zCg4FuS#Tcg&fB04tw!UR28suY*@fX~-$lla;_sl&`1C;%=!M_6Q@wa1aa?Kq-A(NL
z2(xWr-%W}A)ssP=<o8%n>0w%SBYv!w(T5c#R`gyKVB15>RRt|5pa-=`uz|^Nq##SW
z!{N2<Cmms*F0@R(VwVEA`1C`ab*J=ls&fP+$doVrcplaW`+ETCiz3}mL)#Yl$7mK&
zhHlbm6_E<S6r^-wYqkUP0g@|YnC^J1#uU-^bB5V%6f$U_D&r3lg9S~5M@0Zl1Ycds
z2q^CF@9+r=hHeH&eIv8?+yV(q|N62)A$)Pw1+eH>=IaKX$Qr9cF~HR+WDId1Ee+um
zH-XKblJs>vhXn!r-HxEn#2B#&*rzI+q#I~di5@<)8C=AsfXhQ8j*tpaq<^!y&2cRI
zs&?T4*0dhR(nbt_`P+w%Ty%3YoQTMRZ-;p-X(<R+!KuEBowWGZ1>?!jagSy#1jmUr
zd>1N#@x#T<hlmzsNA(%mh^~%5tt|&9c<hk_f^iuv`W8?tp7wAh?&&}zQm<!3D177@
zTDS}RHTASx0jYRRK-GkDmKB5azlDA5nT%`HqO4j6R6baXE2H4Wz=lk>@v+26?J&7u
zSg`Gdm)Yj>0nG8y!_*}J^8gq?C)v|{yjpDn&ZFfmqlOE)4LUYMEW%nvV8cRVU_!He
zOm3=qg7T$Sim2pv1*VIh1lPIedBqvq-__G`_vVeus>D&&T-}q_z=JAuW*s0KxbuDO
zSJ1?#*dJE=Pd-kBuLFw6lcai|<~<fZ1rzyureU=#^JCRrfe#U*W5eY?@fK@78JR0-
z?5o*46PRVWN}K5@Ys{2+OwldizwSRMyd)NML2rO7&IY|h<rC5=Ty~)?I%`P``iWhQ
z+MTpvW#!iR2Z{*~T)O#SxGecM9H*Kc;$Mi#t@tD5oN>P>CN?2ui>Ux6)NdX#qUrY~
zi4K(RI4M7|GJg6F`iYQkD8qPRL_TXR@x|U^Gzgg2>MVGmSAq*ElQHmYdx^!yTp57y
zLzL>CX@1}^iV3+Lh}Q@DdKFqBwZzU=)G{Y+=13gUlr5ImDJu&Pi<ZEd`%eNQVK)!W
zm0BgU1WPe!xH44iMo%tW7%}ai1~9{RDu)`>hBs!N*cLagVE6Pd*^ze3_NAPO)yHPs
zp@yLQ5&PXa2q~cecv7;3xaXBegzkpBD`V-Oj90);DK*<1QJIgj#~@Le44_48Cp@7>
zU5$YZ5!8r}T9dULCho)HC*ljWXCDX(CtBG$SmYkr?ifi)cPqc@bM5F@iPjj5&Q8&i
z=bd|a&}!j^NR_4?+y)6tYif?vn1@Y{8_qA9%iaQs7=Q|Rpc{qo1_t^<9Q?9&=MSQ6
zTzWX_yJ5k*#crj+D?ZFC1{aRoq+Zdip_-ywxKFK>*~0XEQDj4iPW`xoCo#6!3=29`
z^$DwX%mX7Qo}hjmeRLO}=;Bf=Uxw^DZ;Q}U8bA$gj5I+4VZC&?bnOhKZ`n18(u;JO
zZ1dFMK&|B6-!7vfRq};PmSL<u_}Pl$XwW6!|Ck7;z9&FOe0rLvnOX=34I>BS?s&39
zA^!pSuz$y=o&aB*!B4ttig`rUk$*#%M1UVj3hedff3i(<BcM2%hMx4cw?KhPgCU4Q
z4%IoQaQTAh*u4bbb7J<zdC%8-d^wY4Un(19!mhIHVdPY5<eIYLGOey&g|WVsaUoCd
z2{eZtn6XrUVm`UAlYLM5vFpamZ(BBM%wC_3=aTJ<&a7n|abTe#!t2tW-aEkXf>%wm
z*$5B&WN|xfY>#{_@j~C0lJ20VTsBCqHcfhaY?dxm86~;x$^N!40S&rNJq6Yh&!5Z@
zx)gMjg(Rik3Rjb~OwS7z;azyjHqej$H6lqp(IcKdX`hyz@e9L85I=J#gBxBQPm9$2
z9+q6+wR+nH&ehI9<x#Ue|H0Q7@kK;Gb!9HH2}|QQJKDMY?B2X3UVUmL%J4*y&lSFm
zb|LP<#5~llWb4G2(bmh^@>O(DFUL;eSU+ybmfwZtP^v?WK(!j)t#w_bbK@U<sz%c)
z*(h58;X*}v)Svs$2uU#}+&jR)t>B82NZQtwmkrpMZHCmdtMa>FE|1Tc{OG$s%x3!X
zq&bAUo}T@8nGnmyUI!rh_;S~T%%_Yl-4c!xegT?$Fv9>&6rFXs;Yc)jK<z5D$N;07
z)fPMU)RD~y^V+Ef8}kh3Z`@S1a`-?8p<%Q6-l(}-B%$`y_ZCIt@wV-uWFu@Gbe<4o
zM}OI>5X%Z}Flu32CIw`8Uzg4HpfS%iYGIGq6a)rujU1<k^Pn6);Z&DLES6g%E@e3R
zY(W><-0p&ec-jk#*vF-YtIW|t`JlC?W7c3q-XSF-8Zb>K=3ypn+z|49^eOOJ2DH~k
z>Lp@MtxGuZQ_Z411o+QHayT8m5W?trv2cEk0*JTcK5CazqeJoh_XVELXC}{*8OGb9
z!ehv(xuMV)r`p(<wLI5@nl%jX6q~GwgWE1y5kEviHc3q4NXb*xK4bZ!4@Ywip#SVl
z_Uj;L+1j|OhavXTFfQ=*LpDADw<UIFU~5fj(4la=6Eqx2tHa}$1Y}f>ye(;_7f|@)
zIGnhJZ(?DX@Z?jRKpzq$5DnDv1k27+2xIpnX;;P2Zfvk)PBlG~Cy=9ZB}R<<;wXB%
z;XSCHU%!Iys2Ir{yQi$PlGc2+NJb=5YhAY%ce4<l$V@M$LJU+RMC$uQm`P^DOUO0|
zjIZ51DMRZvG@u4uC5ZZsjBb=!ramWw;~eA_Mg@Ao0j3C`A8rXrQ#A0b6Uq7<sCWSh
zk6F9y?m>7A!5jhW$^A>w)h@@>ym*0}o(_;@7a^y(>Pfbt+c91FxE5u=J7dHMIKgcc
za}&`zK8Zlrq!jz?RCQ_b{lLPkG^#kc8y&4^R$*CknWx1D7qOR2&PCjuBp7+QS&x`?
ztcmZ4oc~z&aPdfXd=EjlL(M*Mo&!4Ek&|t$&6vC6^;7L9v}1=?x#rE2-)*x4S>9}g
z-ZqPIqx3hNnybbi5u$u4BV9h1)<5@!vTlh(zNUu8{EL$)Xe1aLiarp9)B#R_5z1;K
z3_HEadc{Z-C=<K`uQ?n-G)V^?Udw{8IP}4ww{?vi?F2?!`%MDEkTDtu8Ij&!Sx>DP
z6bc{o<W>GWY{z1JE-~pMhEduPvK<8~%X*_jAuiK$w&Vgl0m4bc;vPmg7RU{j$p*w2
zd8zl79H9WfpMAnBZC<$kQa1J+b<j__4tkpEw%@T%V4la@?BEVA>*500aCvOmuRF@0
z9S9*(U^U7_c}S2^TK;slcDN)x@|aJYBX2?pk0m>c+<B^o#FK6!4n;pM@TwEJ=-<{&
zJ-#RP8<;6wm;FXIlcv#-<a;JD))q-3q^m_StE=E3+ZOrrC8ZvPS7!9s%^o2{)hNM-
zv$fNwu9PsXF`MS}yI>*0Lox4nAC9Y$a8L-Qd+TYyNXAN=nTv}Q54X9lOp+&*+J*<J
zynU0&Rk0=Rd?~6Hw|sEZ0_#F56qL}Cj4?FQvdXHnnWFg+X2G|xeG(t;;V?=?x}kAB
z2BM19GZsfj^U}JM#LT8{_L!-XR$^ZUE9}?9mxF0jqu3rV|DHJhEK?r%usKfZb_O|c
zm>s7CKBM>8p$r{qDgUeLmb5}SZ(6IlIT{?T*o2EiQMxS49`GJY0JSfeDsQ$4!Urxt
zbTZ?p1oOsKL`7@?Y6Tkagof)pD#k;h0%A!P6r|^Oq$QlDWYYt*Oug?~99m<}iQ;ZQ
zrcZ3JB1+!BX3XqPeuq?Oz;V_2P$=pu&J6=BF6X1}@<aULb@A8B&0$(`c$5^@!G9zC
zl1TG7Do8YtT}%^&D-{uwcuV&!C!UjZcSLx3@1`4AfI2>aI5yo5K>}*ho$wmmO^vTK
zehZEB<PKMT^{*BBqbQ5@h&K(KvxB^xl-!i*HZ)55?LcqZ#^$U6OvQW+=&R0Z9#{aZ
zpE$#vEQLho)+xzcMrwZGrCwSscu-XuE4N&=q_ewNymM$GOpdxeWm}28d`z%+Lvk1B
z_rUG=zgK45-w|BuOzxB3Q%L7A-XLD~bxv1vtiqiCwNTT84>43FLA^w0hWL|VuMF-M
zZ&H;6jS?UCcC1~%ZJZpz)g=7avNeshH`-Y#7y2HaH@Eo@?iQwt{V1<}kO$=Rkx#LD
zpvpncWSz>j{mgZSRH~7qROPxu;_FPYV9m<+DDeQ8>u64EkBO!Wt?))U7{qD}0kf8f
zeNVywr_!bv<nok&ZC<$NRHM!l8lnRN#1Ts>jFh~uk~V9+bA2KcCgUOXpzbWQ(H+jz
z+!^r;$&mwXjBU57B)y?GCC?Uc4eGYQK6P1&*6Fh1+!L+1#zY8{hq%#u1$*rk0E5zT
zW@X4s^~fM|<lo}#tg++i5*>C4EloI2akxkBrHClgc*|~L6aPlZ{V~aSr=`=&J0mo5
zHLCZ_m0QoHLQMI0_!5p+VT}DX3C6$QUDy=MB_!s$$@m5XodzC=m!{3RE}0L&EHr2-
z5r&KHvrmA7cG$`^9%QL`ll`<8+8<@YAZ^2*(tPTdvbZUqPaLq14L2{e!5-c`RUla0
zuiI+_8mxWA>sJ&QHeXC6CfV`vZ$!~Y>eO57bP`sn<H@iRx)FY6J0m$lxku*sh@am)
ztAR@=-8`W*{%QSI_OlOz_G+w<tfINR+?`3;)8fQ+Ar8FkQ~M)4B@28rHjcG6>3Atr
z8W#V)E);0riK8LE;usmHi2Eu>GWnmHYe|axZS@3gSoBk-q5sM((iliMe9_qN_dXnn
zeeoNP>s5Mr6Di)Ft&A{%A0L@!pTHzyyb=eapF<M(NfFi~M>Ob%QMhBR$E?Y1@<4Aa
z<WSYz8>2c>?-Q6e1~n2yg%d&V02N1eXnKPiJ<--@bC~Xn8<QJ53a4MyZZd<c{W-dS
z@g~J{Dp1vw`Idubr~*%{kVfNjEu5Cu*V(ftVV`n2S($z{=hbE}b=FVVP}j32jKSED
zz&kp5n(NNUU|e^>)P0Y@>4UKj#;9%FV7`}<%Hv3aI2ES?+c!L`rH>`Q2e5EK(8b`4
zHYzGaY2wIw|Mqd?o`YVRULRY9U)jl3S!^+9S{LL0E%@|fUm7IJcdExVnt)YOiOiYo
zXBU*7F&oMoL3~<P<OtrZ!;GIM_p(Aw2~;6`3XBQrY@0klMX<Q|%+k}`OgToIa%3~0
zn{ImF1IcGJit)innndmL*b5zntiQJGumow+rN(ybp8;3#YZ(^4WZ#>-5mL;{7}x4N
z1*cbpZNCnC9zO1t{pA>(zr=*uKK#+%Hyo#N@KUa2=Gcxogt;yAT!qil?hQ+gr+IL@
z;4C_$W^CE_6vRC^l~?~N9_ibg%w@pkwua-TH#CM#qZ4v07n#!Ql~ohHLh}%kaV-8b
zBq;dBK%wix!z5OZ?+5#R4qto>?5f_U^2E*#UDGA52Rx@djU~J#tCQBEe{AxLs+6IL
zViK+MJ<113Q)+mIHA%VJtp?$T1nu91>Yaf(1ea=D{ES9^Q*|6o_~seEvNm{p^ld^H
zJ6z`Nw)C}_M+?hbGxu3R>CZI)id@=zu*D<grkBe){?zdF42dtzRcgi#Kx{Vt!2}M9
z-y1`AK{w**OSO>cgFWF~T=z+kJ}mQCT<LY(O*(uWYy91gw}CTQ_YADZEhMK~(($_p
zaj$5E31sBMOtoBY<7b#g(jJY!KUhsG48+(vB!v$%l+vU!?Wfv0=9b*Zb;@GF?h);5
zpoxw(^}ZoxQL>8Pz0I$=URz8j*b;?RDv=w8Ct@9n98<FnLmC*W)Ic@Ua-gM+(YM*p
zN{=F0+(T1Ue{@g<Y1kYxdkKEdEJ<mlOe>q*jUo*MXpA$}0@g#-O>J@?bfoIGWbfn_
zrNY+IkJG}V<4U&bg#*Bff2-IPs8}T10-(4T7{*_$_C-y}3v2G4!bw0UU^n!yIH|AV
zNe*Sy0x7aA*$ejJ4f6OUU~dcCXt$%z3#x)#{y=TZGhEJ6Ls7ro9`4z2_|d0+Tz(?G
z?jff2Pu%|ucI~8&EhX<no*&zR+TO3hjC5jlG5Z0<RLJ4$z}6O5dzGG4)>x37^(q<P
zikB$CD|>w96&w>3RMH+6GoU{(=%|o{kss^MNc#wB0F0e$*MIVNd7H{Ww=mvrp#SLR
zr@e@R*{Dh${S3TEO_cX0m)gjiD*Ej}ThG;>!w^)>;kxRUvZ)}-u?aF;$<x}TwYo{~
zA-DB#=q9}iNefVzzEQ!9<v#y+QlFC@O<RKvTO}mm&z)=a2q1*k%x6r|Ep~ey9#m3!
zV+>SZ;G{4JLt$jO@lhAYOrlVO%(q{S6o&e33UL{OhTKkEiWXdyye2l(YDLhgCpH8%
zzfm2{JnrC1U4`)IPj9kb_gmBTF?=6ru{yANBY4d^(3&YHOOpL|o)6J(vaK;|yd4>W
zBV!rf1&P(&h*b7O2yDyUFvGT@>eMX~$j?l(TEbJ)h&#e}k%t>{NC3&o^E3fIw@TFY
zmYm3FmqTD)dTLI}^!o3RnHAb}1JU7WVD{Ob_z~o8eq?|ql=v_Ly*0H~NZA|FYgJ-M
zJti8_3*YfytM3bWE>h5}-`TB>P+XW^1MJ_fMo?;ezrG;;lkWQeMuz?;-qM|tKGFPN
zT&U&0=f5D{(&NA7_Wm#8E&mf2%6O)lk(H5ZV}q)fV3?G#o06W9sB)g7rMH`&nxNC$
z)6;{Kfc<XoST~cw9=dUK`Ogo<*a*$c%E$~-R$Bqn!qRG)zP{MxpMTMx%mb!H9cGsc
zOWr(|7bpmfan)2Ci2oKC`v1Xewp3t${|BuR^1lL}gZ=*z-u{1wxBVYr|1hN(j{pB+
z+BS)0nq*>-1wxIozK4NBFFq%X2B}F>N@As<M%oG6d|Un#@V=Wc-g$4*@?Nk$fY!y%
zJP6hRL@pVp5aOX{DP_m(;6>LP!b~kGHrHU@<%B5-py+02`xPGk?b3>pOkWyg5%XL>
zU$8s=&TnDl(Tha2Xc)4Qos^r~-!}l_wGvCIYAu2-i%kwpKQ$i9PB6?!N`#kplAy5r
zCspBgzP+36oirmJpEL_s93R;+%J&n$87K`j+q*+^R*mGbjvGU^lK1NcKv}kbRzv?!
zU2n|SgXQ|yb-w?KMdjdT_J8d9@qh37d8%QO?)kqwS`Yw|)U$W%5evUc#&p^D5*W7i
zkB*I(Xvi!KHnpxU_&*igokyMapCoau|C)mT&&%ZWe<(OpCn4>BQ8cth=ojjA-@is<
zk}!UOMUOBUk)e1VmVo_F2rN<b2dw@RwG9pc2>sv2)H+-L?>cU-+yU%o#!#Sp*a#Aj
z$2kG1j9Dj^WXA<)h@L{w5H6E(xydQnOSz#n`S(mZ-lL$Rp_9`g(%bof*~*Bi<Uh(j
z#!m@6t=qo6`}(O|ym1<fhf$_0_jTHG9z+xr4UNBhrbI8Od3wG1;8cqOM+WGkeEju+
z&gsM?9h|JTEydm#B&evNZQOWpW>Yd32?(hO`#f36dyR2Bq%}dD0Vy;5@21uX)a))s
z>}Tl9+-Xv(6yZ_GE`Z9U-?+GWgVUuqjjZ~b&U)bA&pXZwHtKa&f11(y9P+r>DgkvI
z#v2me`Y6*u)fg87X5M;5>RN~syBS&Vffe9gwJZrl1r8DmI+8FI88hrrzynYQebxR8
zJ**z1GbxmoFnLxz&du&$HH2dBo;r-Hwmfsa19L9wZ2ezHtsNMLkKGZM?#1PE7ye<`
zl&wc`JShT<)VF|xGz_Np!(~>-_KnKahG9pFn5Ed72d4jVqn1n*<p2`{05Ti_0J;Cm
z2{x2>H8poI=U_HuW;Am&{ohUYMHh-%(o;I;`q`(z6k<O!;(vyiJ#NH3?ilVE@?CB+
z+;38%up-|>P(@)${|)h|vJ9jlXrizbAu4jz#Yt##a2%mJw&kwnO^fY|O^dEoXBJgw
zXYJKz?TZS&`b$qlbC*vSJX@Yq?vqd1S6NpCGvZmEIa|4J?i)+K8v#p+u7?y22l0m$
zr>T{5nu3ed_p%pb910X`wua@K-R1eG&hK>>2L=8@(~YP7M?zdrXMcUqW>8GOd|U@*
zb<RmjzgNz)WN6uEF&gCN;Y5X(e0Q!l4|L79kME*oSS#xIbuJR(rEJ;(STi7ykZZL;
zQJ)g2IukFR5>$+lVK<Y{htb@*Yk4q*e0g{{635;^J;^RGlsA?q7;wm-`d}Gl1@5A&
z;4_hU<22y>lUm?PvA|oVN}uIhr)5`1)E+X5qxh)0B+SChtCQb$FPknywI$l%A<Qm_
zDPbtP!?vGsOMF~kl=QGCoeMB;4k_Do4%Q$QNFx`mMR#2TO>{C!9ION}n;}}MbRJc<
zp&j?(V?8pM_6aWszq?!}(3q*)I!Bm?mC|p}hz8oi+6=SA&=4FUWNAqGe&>#k8mr}T
zm=jRrW-PEvpncnC;}+d<n2BY~D$WFf+%>PnxauWfIyaV4vVqDl-pU4E@R&#js@^c_
zeD`lV%i)x5!k78Y<7nM0e87|R92&DU)&Uj|N@YJq7t%Mx*Y5^Uu9bTB_Fzji%B+#~
z@88tSqD2ZD*onjRh{m5G9h|vFU47Mz;4Rzf@5+da*abTp8?N;!&aBP2`R)A*6WBuT
z&v#{Hs;@*|5Qn%QVTxz>X3MNrWL}0@!m@uXp~R8vweCf+N5efkkd^lOqS<2~|I?p7
zNEtO_^M3s5auRZ#7IcZ<^ura@O_iQ%lU7wT>_B(gfQ(SE^t|5K2i3Lakhie8Q;|JN
z8=HX#lkxZcbXC`pG>e2S3od1j1!e^tu%~D8{=lCM<`&jO1!e}dybuN3Y6(zEDm&K2
z7W8sZu#03DuX<vfP#Q`^i}A#a%T9p_s#SKBpy1*XCY9f0Mr77(6P!N9Rl)3tN#_=4
z80(>V58#}?l>YKgIxG$tZq&TxquuAUtR3Zv6+^{yJ~N-SK{#87=-Wx#g(V32KwU?!
zjCW@T9eHAyV0*<2MY4}2SxwUOkz5pLkC$0Yi%pW!c8DL^nQoHEVkef1#_JA|j(0W*
z6fC4*vtX9_E{+C8(bYziFqk!CtqCyLG_<WcL5cirLdnr-xyi9f3ssJ9Chk|;I3x~V
zq4NT1rEKERWQP4MBW~-6Bah0aUXC>SHMhqFf95x%buJKb-Wb|`$WF0M{Oghg&+A9}
z>4Wv5AE<<U8D`B@=8O#}qj}8lpz0@?t0zd;9#X2-4e4C0VUK8b8CbHqG8-m1^bjMG
zz{E;vag0oyt_o+xp8uP!w4R#0AiMPE0>9@2xswL*BS^Kt0=wQF&fdHippbCKEhu3G
z;FU5hTg}d`z6ec5@g;(or+($WqOXdmV^PZ&1ir$1@25=X_A{y&_AsmlXSPCj5j<`r
zFe+a#rmY@X$NGey1OL|9!y>J+%wFwc5QREp@N|F{px(Ia7ZS@BAIX%|<1dv<Jm7r|
zYG~K#iVxU}Y-z>&NwcKfuwyE1W?AKQ4}K=uoJRn!7P133cV{i(JAR_}t`k!JT}*a^
z{^eyF%n+OoI<%Wp6cGfps4sHzyeb*hb=cTe>V|^S$JQ!lBUZIp0$-Ku*Yl>8LpKD^
zCDlh7kAIy+v<{=51_`R#FK8a`v(%}gLEVbbax8WT2&goS{EY-VYK97GSEMc)bK1z`
zFP&*f<<D@y>qRo#3WddT-9awZZP|y7-cC8CH?m=Q`VnNl58V-;zLvqE@N!D;=vl%<
zFnLz2VyPif=L(2!Yp!i*uKM19-HwcGdnS_6_viO`_&q|xVZ>|JIfz~$AmUnKK0Bl<
zk~{IA*+@<(^(xJ5O2TC6i4<@-(-lJ2Gk9>n_UP@DSH|~uTGp!Fl5%2s)33v0Ru~_>
zO_+%o3(Hr7^Z1gyIXC8SUKe+Lxv7^nmm*@})~LXi#S}*-SX_zta}~+FCEGI?qowi4
zKfrT|J%w+KuuJ0w_vq)zlhT$iX<;NsIl?6i`<;*qD07RLq4Ar$3wp+<KZvn(-biNY
zo(hW;7$2c6jWo{OoFi_{aKrK}T~vzsnC0C;It2}3>>T=PE{XlysCUv+nMT~NE~A(k
z;tM>dl#bh!PJ<F=ZR2=@EK%>kWt}Obpz{tEnq*!E1An1hD{2J?cFYh)^4dKdc)F5w
zjPs>!_YkrL1ukV@U{+qC3Q9I6u7ZdziLT{MhR@2vDG?)8n|AfP3~xjbdvw(gs=JC5
z!^{D|ES<dIhjQ5hu#&MQR$K<#QLu)%rTCf!;~-Z;HCk|OE_luatt3*(uCy);#Tl)N
zkQ;Hzf#BPuSSJ*+th##jP#Wd>sj<pDZ_7r#l#TORYytH;LL|k0?Pjk-a>q9KD`+K*
z4d>h9JE=7>%>`6ULqR5XVWq`%ri;K2E&5{TvfK0v2?NDig{=~vhy>O>8O95S46+WV
zZ?22D7#G_Z#8S?$0d`I17%|eGxE572#sZ+CmD8oCLJe}1!apd%PQ3!kwre{s#|Uqz
zr=iMoV&+JK-tA*6620<<>AMYh4X3&r-p02b2p6VlB?0zAjJ;ea7bJGCKY!}k2>t@G
zOCGcrn&eKaCE2*ioOEovc%6FxKdik~Ol55prukP<xEJp3?(Q7i{h)=ryI0}vu7$h1
z>%rZf!rkHE&}1f)$xJ$Z(dmo5v-4f-<hy;>yPk(dmR_>>ucCVs=AmU&M!j*CRKfLq
z9^=BHORm^zOm<qnb?27;z8NRs4)YXq6S8yBHre7a4p-40^VN_-`Kb}5;8@)JLI+!Y
zX9Kl^+$?P2*D%iQu<P%&rSoO}E$)1g`mnT><4O~vrMqGIIdS6T@xrNWQn|BIz#5K4
zwUM~s;KB?Oay@AV2@6H_-$wFMj!l5P1yb>NV<syPwE#UIe*lU4oY*(};v1y&#fun)
z1+W;I_gq(dfKiL)mLYvMi2UMcgD*HyjXoxF?ysgViYG>g@;IM0ERYFS=!h@s5T4Wl
zcj=Cr58hV;nm{xoTU~mO*(oARdzsz1nA9be&35=d#<~COj~L5iP>;^#PdMAf2Q!7j
z;nj2;5sLq=W@sEQo3=8vdB>#|GnLxl<jt{Tx)#&=nlh7}qwhdam!|;~bcm<*iDuNS
z<-u!*crc#*!;X@L+3^ILGlQe{ry$o1%#yNHIolGYZH9=8#F6Y^odOH^iGtc=#jjNv
zV3jsNud3RT=iA<oMNHn%e?}ti-_6BswRxFFlH78TS;)+l@J%>4hEAA!ttjJmnh4_s
zL6GV?cCBaxh4Ngh0!v0f!T{cB45d^PZ?MR|b_Dy7i}yjyz1T=cKtf#e$`axO?WwR{
zJm<C{xK%gQqxY~p1z4U@00)0mdDqEJG09>jk-ix_!<d1ZOS*i?kr6S07RJtL>JdX`
zvT(+(j6tnfu<Dt5pMz6gK4W!B%v8ftIh*p*W;WexGS2(W*AvU26%R0wUsi7s5?oZp
zT(FPZf3gHbe`JBvqpxQzKag?WVeaHaUe0>Q&%#Oa!Z*T<*X^-pu+-(CdM<);r!p~C
zuPDmA$uxC~CS{I1%tyk_XK^V9@2JjN&G`hpKqxi3vye>1Sz{W$>RZ{e#HuPefBR-O
z<H)XlLqV_Jn38BwukFS>M5aeR{s(D9CZ#x%gu%rfu;;2k{rqDM*k!);cW)O(+tk;r
z!Hp^1nrzZ==j_+Pa`-iM#xPG3WZy(ezmj9|UY1fvByXF-hzMaN+5-^RF$O#X1JJT%
zLG~BDH9SQ}RsD#gz63Z6m26-e2=x`QD2`TN9>s+lncZ4WB%Z*PleH5h*LN&zum(w_
zO5(A;@()zLRKwMnr(QcfPuXk(oXnEz=BzKw;^c1rvt1hVHKMwl{lRlb3uo<eN~TMU
zG~kHDCsS@-um&2Bm@;Ls_&4VQX=g^%<c+&&lxDr8z&k10YM>h@ic!>D(SPpf-SKoe
zWkDkVeT9gwv213Jq-;r4tYw#sBs@zXEHUXh=F)<-ZMa*9ocBRG$11kLj9t|(&4Q@&
z!Q+^K_JAU1eoP7%Q*aWZWF>a=uQdoyZ4xnF@yL1=o}&eAD9##bvTJ6Aev8IZF3sxV
z>UVj6;&smN7FJEv;_L)tu6hOM(Q}ZeW2Sj2KVA9*a2V%M5(Qr(q`3Yc{YfJmd(hbT
z*o4(Mtc3-RF2%xgK)R3#7`Z=joupf2e*;9NM6xa;ZrkpK6sXp!s4J;lOX7iD#4|r9
zeP7PHX4mKWZ4!8r^6gejz-qe$p-^Fm0U;ap0f&#|zVeSwkRBR68`e^`S)Di1AE=^{
z!-qu{JNF7`hivDH)b_%|@0VJawLhFRb9T?CI7YuI+&AnCgua~}|CV(}KfG2bM$L9g
zep{XNOp(gzA3(k_3y##0HTDyyqvySYaSTrCQ1VHQ-Q@*`J4!RzqyXnMlCdu~`BP}J
zG`VE`q7C%0>+j*4=J`orou@?X$gHjEt(oGha>wXR8BBHUl~1$sXBu0{)|^8MS<>z$
zXQY=Li}#)P)dy{>srv-v5Tc`Xx4cm{K#-c=vegbD)qS_H7e)9bZ4-PHQ#ZlRqJpuR
zNzN(O?|%3td`?&oBJ-hoDk*x~bL9*wp}Y=pd>b-{t$fs)6ZDZb(a~E22=B$L^5p(-
zAg};CHI}>55bULJO=-XLhz%3s(LvH`#d-QGbu?_}md*tqbn&r%V-{A-#L&nO^|mZ3
zsSAf=>yYE`{1^<BkHjzWt^`f;849}Mq5GdfkC?p3nci@Zh299{pN7W~q~A<`kKy5e
zU>412&FB`YzFr)p>q(tawUKtzOI6gRISBz|g}-8L^>Q}r&F(e$nWhxB{~p6S8PyNj
zQ@h&HTp(UDOI<T<2^>xpn5C}^_u-E1YpX{$BWdU9Z*SAxZgm*N80Ng%RA*8wfuQ)d
z+{$bh_(iNX82rD&fz$r_-I8X2vQ82Cj^j#~w7P<<e>as_K}NUc<`(qBwIOnB{|+sG
zY$aOYh;u<!<!h&l$=7Y|&7({3@AMxMWG|%Yxx|pJram>=O8#}s{xxfw`73=#h=L$#
zVRa}agnqTK=@P*CC^(ZNn$0V#sgHlXkl7`vwE7RMasPLhSX?B<iwO4)x=^@Kwq17X
ztT9p*s5;s2qjDd1j3tH}pjx`@5L-Fg4eiQx&s$}Sun0Ujplo_+XU@Q@N)f@PvzFsa
z4|5v+hknxCMSVkLT6(pjX!g$UCew~O>IZ8inL>7kE2n80(829IPRQEE=xDkP_*!ML
zONH+aJjRlQvppO;loBd&$zE!r%lN*zP`n>h)1Q`FA_QrTbs;gBjOqZCD6LfVuu*M|
z@{u^ik35aM-cN6L3_z3BLh5h&I!&%;;p&h?zbY{b&3(`oteE0ApyF>AQ7e7l=&@#}
z%4<G6neO!uf>j&FU^oV=p9#bxtsqAxz>bjZQWBIjyxEOby(T*nA&t(gKla&f1k+}h
zq)q@^q_~?RI6EX*E`wNQPdCoTQJf3Y|Ml(@nYi+VyD2;(Y4Of_$Sp)WaOV6eB4!OR
zZk2FPH>fF1KPysd1vqEm(7FUSv?x_g#0%uIxv^?r;YOcAUG^$D;iPBJfTcDP8g-(f
zl~!XD=J9p>EfU6AQcKQx_6bB|OY0O9MrQ{EN<RNP58NMTesB~->_f9eR9~RZ6SO#E
zND<X@>{Isnt|R8fCHLj7B+u}DHFQ^C<!U^!BW+ORrH3kdv%H}(%@(nC+x#|WcVj-X
z80{3aKK8=*gdA6F1oJA*?uR&mjX2!r^w)~0^YPcMUdXCT4!gN!?62oF`3KuX7lIPy
z71rNiZ@G3|wTSXD=d9F`bG*>f(Rn=I%X<-Oh?!=VXHpg8I*m)pT-6iF=9b=)rrI<D
zKt|cham@ZPRqJ=BBV)e7CR!mMOo2`(9Wc&Zrc(exd3mp?CscRjKeI-gxX24z$Hbz}
z91;W0anXjI%42fPI)l1Pi7mnN0zzIfg4-<a0I~#{yNFt7Q8qP(>A-bPYcF{>;es7O
zzwZ%TD#ho<znaa|ENQ#q$)p;@bvNvT%hy}?Pl{1G;p7JoBYfao(?0{+cgtG~Xl;*+
zeQ8}|EPo(M*h|M$ZN+y*R+tyJv;12b7|Vpi$&80hTf3_u*xM+A{H~W(_($}BT=QOc
zl{ch;8X>1h)&*OhPkDxSr1YHBAM}^natYt)FRz|eFaUG)J;PHb(t@uPO_v!tuRbeW
zwgs9}v3PijDv9TJ=ekk}azWv#>XXeg4wVs8+?>sH57otSfkS8^POrW)cgP{arG%U*
zS*4N-y;nQ7G*5268W(cvlJk!>Km8;|zRBdW>XA~|=vvV@+V53lwNQbNagzWe{SDD2
zmf57uQ4?*ipO9{$o4_0}{BBc|Gfm1n)}SD6QEl$1P{LBI%Or7Km^0GfSuTj>!s{!n
zwub!BduMj(9Jbnu*j@6}Be3$l^j5+EGm)gWpK?qcZCB749Q?h0I~qnjs&RwBNCLM7
z=E+M(E!5PEr-GT8lOY^16Dn^}&4pNPV3u(I9#2>`_t-?=c03c4%q4;U<PcRuXT#rO
zV_xm*`c<bAr?^2yhgy{htHK&EDu{{=Fm@c*Rr_)3V@sCL8>h@BTLf)%o5^FrcgrL)
z>Qc*G!P|G9dvz-}PpxR;Iw{6vojKW2YQv-z@1W@W<GBQct<Ebtks?4nzW>o<npL1?
z;%=~Kldi^RuPjt#5ZYAlBaGf&Gr&!5!)q^M=NCL(l#SK*jkaUNX_x)Vo^T36ZCY22
zRS~GYEaFny{?Bd_ub4e{{=!`?RUdP>ckGhc5}e=5I?T&DT(#UvI#7y$!A+`L`?V%%
z(((Hq9zUmH!4KZ68hXufKa?HZejkCuxQ~b<rVD120SM@RD!@;}nr!~ne_$_Bb?X|k
zb}qqimi;RTC0jMTkb;VDOuP8%x+8Kd9;}_Ke{TBGC911D6z{G8bLUt-(Y*9fOB|Us
zpx2g5L7haiFeQoxhiTGu>XTiFCvx}=VI;kOD%JIxQO-LJ0%*f=o^L_tiAbuu&Ix-a
z@<TyXwfIJ@*p<yu&Fm;${E(rY(ejS1oHre_R8hrS5cSnTzPWkb>hL*V|CpeX;`^9o
zBYNF!YnaHYVO=uo=7!-~Enes`|M(u;;Mg+^qg0oVqW2$1Een~Be+iLb(ob}zvIpW~
z<s7?AiJ10TYjZP<wvxQby(GTADo#^`Yt|+PEM=}Ky+qK~!a{iG9Ks~KX-=_h8v3(l
z+3tVX87!`-M;OKRmL{f?(j|YF8!anNy~FP|uTJu5P@T-J#J)Arz0+R#5>5E&5bt9?
zxK(wxjy0$9ogzheXnzUV)g41g!GUV*BOUxQfg8VK;Tk4)Qsj5Cd@^+`;RF>?0a_8Q
zhGRJe&0xn{1+fci<mBz(SfKw;p4F~eoleO3gyR;9w`#IM(;LcanSu@1a}&;$aSK@g
zfJ5P)@>>)hpG$|}VPm}FQ#jd@jcD*0f}OoxNIMo`74;Z`1NFUe*hkFBTDg&T1%olN
zuS^NE2-zH>I&^fcgu3~m_S*rQsGPH?2|;184h*NqocZW^<>)v*iS-&J%p~!fDdI+W
z?0|Ek%v(BWt7C)MIgpxGMyB|r9>v?UY`Bzk8UNkL5?V3gz%c1bGUpOkJuA=&#p;<;
z+?YzHl8ib6YSRNtlVm;rsZ0<FVBy4e)P~^Tsp3v@_TMDjCbk<o5jjtO&W`f5Bb|G~
z`5pcv0Z&`Apjj5?W#+9F4naYA!o=?$Y9Dwcij8OzmABxCA{Q!FfaSP?iWi{A!Npj+
z*<A@jQ^Y^eM`n@k@Gobx$x}oBvoj2&cBs^)aJ0rp5TrWb(2<hRzE0UW&Z$2mpi*RO
zi9g8vySBp522Ot>pG6JHXF)r2FkI*d`P<rEX-`liu)UYz8C$bY!Rb*%nJA!*V|tlY
z(X@|dE;8TRq$|Wr(veEWmKT)k928Th=H?t|kL;aH=YkPq)~2X7RX1{WWO<+*W82#W
zpU%ZhUqtkqq5_^i_BC#~PhEKS5Yj2S%5DjXr@(;|3O<uxvaBgGv)jeYyI_gD#MNjR
zn<i>nZ;r7X>80W2xt9`neT6M5JA@rgcE|4C-QhVz>YL(DM7mo4@gF)!ug_y_r<qml
zyxfkCcGe0U7q@8HrTb2LOOf=$mB*{%1J_MA&+v(rQPt5DYy6w0-K+F$Af{^M&@HUn
zM4iH_wB)a`Q!%~vm?E#X^vshiw8A8-+l?k1Inzyk-$i|scZ8&Q`IvbXiCU}{2RU4_
z_pcr>sD8rYDz72QK<Q$w4$Ud~l^e_4)v%(h5{pEKcpHyqW7NC6k$ZA}#>0%;tIH)?
z?qJU+yF}O-yI(+*Gg_64lyae4;CRQTYGg<79gd0-Yf|t>V0KzC=W@e9JK9D~Nc#y)
z@<o6&3$Vj8yyRb$iNqRRgYMUmv89&Bt*UhSsD9~8?1rQJ9~9N8!rZ-+RNKTElk5@8
z(54bV;T6$USm-}@oiei=z&`WcE5%_lZ_VU<YbI=MW9Iif1xMM!Y_U8&1{tTCf5kAP
z9i^QvNRkEHl~oDZCtN!xC6;TWYm22qQ_@<e$mLxnBWKhsSeh9Vv-W>9f<j8?{0Wz$
zCg7>n7#ni%gsYX!X>8=DzqIBfjV4xy*p4S`w95kGmowsWAbfi@0rJj{+0tCJ2@{jM
z)Kh2HEV~M}XvYme3ZVux{T-<}iqi<$e&c?u*pbQ5DurDnYVvd3NEfkLn|TlPp6yr%
z9shi(5@qR@OT2Z)4KM8>_Y?vCiGFK+lb!7aHKq6syBy@CHt@*ap}mbk&Aq=qH9+0H
zs2<5ti9NZ|Z_M!V(zMJIIq-U79-X$mER7CZtl3Mgl};|Wq^<zV?&`<HQYAQ{o<_|a
z+-n%2btH|3b9A?OdomrCpXd@bOC#B{&xeOrHZeavdcqOW;YV1^zD-cL4j2WjSOwdL
zIDy${a`Sz;0mVx*{K<t^lD~#&S~-@*37DvB(MXnh^<W6|&ry%v#+M3QA}K)}LHv*k
zmzCfGw<uPd^d<pO1Hvd6dwnzd)$E*lTE;6-Tl;U*urFvY-7;A2or-#7j!-yf{pZCx
zE-+5dOgYyNNwbi2{F_>ASf#4&ByyZ<h;IFoIg|SjxGT}UnW#Mes?pWJ-H-&meZGFV
ztw!i(6^gSU2LPO>O~^V`Vc~&BHY&i|GUUvLhzquVg@Q3LURLo>c(8Rg$St|iSjozr
z?N2*H<z<s$xL3R}cuT@g8H)?}I=W{i5K0;=i&f`8W>g)|_w5X(4xu~jcFoTvya6TA
z2ziacu%LO?KE?x=a#l%8Jv})`V~@kh@JC`Q#kLEKp}RbQN^T7{BFGic^uCa9l1|^`
zs$m<g5d>YD_*@`nt3e5>wGf0Npev>;EQ1Xm9x1xp*I3m3T{p#s+|-OW^-DT=%`t%9
zeS}`FLzy+ksfms|2G#`7BW!O6YZYDHA<BZez?bPW%k&xpyVqKHl~CizNnr9@*GR&;
z(&{-0VQ`g<pMwuSo7x=%{flzpdP13+d`DJN{D;pt^FZ}pdGznUG2gfd*GKUI`8MR#
zBB^dc7ybGW2*^_}$)YlN!%IHi_(LXnwzh_iU;?R*Z`q;g;%>+Diw%{uZ}MFeIQgA`
z=4q9{D#5udoH1%X3T><YVVoPcNG5d1vv0GpXGTmc<-Vb&8?WbkLZA0XUDG@Mk7u8g
zv1#5`9nTMGKMXo<d5=FouApW8YWM%&*Jr3K=6H{rm6Pn5?DmPY_tnlq-`v7uj{Lh}
z+1Z}IjPGZc{I*}~QDB+8kUw9V7H<-F?MWyVmv4=lN|ZDb#d^=|Uv6}Y+1jx@?gWg`
zKcA;CcXTllfeEEicu5ZRKNXbGReO7sS-2r+KH>(hl;n^`e>qUoNKo}~*)bPlYd*h!
z98>!R+iPSY70@mI;zUG8#KsTl7cUsF4C^+JnyMZ9w{J~OVY71Tge^yDnrx(-Z1E4-
zUzmkk1)YpZoaMScNiA@;Zmc)ZpNgXQu~6$xp~y--LRmS=FT5rk$?dcMw-B%6%3m9W
z9aw_kSpc>MUcG9njhss%FG44@*cfeKuDdxq=B@asVQSJYGM|TzO#Z+V{&rl(?#eOy
zHLWUs4A(#e$?VlX0lut{CVm$e8L?Ke9Yc@t4{@=vAqkpzu6kwIJBIL1b4{1C@O8vL
zwWB}XxX~MC+jxWB>lc4X#a?PDXVG-7T`!4)M(eVa_&dM%u;5}5vf}czhyW+iXl4P|
z;)Wngg@`GvO}w5FZ>5blzg}z@8+YDqohsq)Ca~gRenbA(RI}5>#Hi%J5qruvWpe-Y
zHv;tT$m=uo?$B$zL2cGIdl=bHB#2Oxy6ryIC!PcdqyzOp8o-Vi<2_$9FT%RV3%Ow8
z`SB<-Z`6=_=WjJ6?_F9dqWpm!E{AizC?u?!iR5^~Ll3hWg{W!|XJMB1D<YFTG7zdS
z&3UYTPjcYE>rIU^!kmUbTtR4!a=q#s64ii3L7!>NR+UyWZa>n%`wCuuCbGDYGqquX
zVRB#H*i9B)__U^te)8jz_f?VKWBiey($i+HK>fvfPq|RVxFYY%3KzUHqCnYIvfVTI
zO2bbn?PV*`AcH$4Q;VACZLtFJo1}{#4GvtOz+zb%^v?n1jq6ASQ}y_PK4A{1q}qAI
ze+WR3xuF)w+($_6=K#~M{RTKmrWz+~mAB47hXdac-h?-()cGB)gHC@li0Bw$blx#M
z8h%lk`z$mbSN!CVS_<VGP2A?&tYDv9k_h8aM?`y;@YqfN&ymDt$;a$5C6grXe6Bm3
zAN>+Exah%s=|*zw=yqI1*bFoT>(85Ns!kDPH0R7)!*tW;a;(@@cw7?pE@_++Bxyed
z05k6fyKI%rW<PGIe!`~jp!=o|zNbD5J;&sHb2$=p`9RxYyuJH;uJIRtB7mc^58Gz~
zHXPz^^%B+nB9yGX_(AKotMf$L9SPZk{;|hSs);rG8Px8FvU?{qgjQ8kZdvy{e9xkR
z1v2x6?;+RgbhRXMN7Du-{;e3%%dL#5OgWBkoRl2(MRv2?a|}2Ys%t)|_?uC=Tzf3}
zL{zH#>55-uS~K8<gZwT`&uEa;<WomSA%EqMd}SP1?mC-!?|na$FrOCKW!FU}fbWH6
z7`yd!%J&ql<Tt@^5P-QkhS4*qVjM~7I?ERG%?2IsVtWu+3dfon{U>Yp<^>1gTH}8D
z4f-Vr{9WY}V_^0Q#=`IWvcJ(E{Y2w6UU9LJluP1|6K|L<<Z-MV@1kQ+72Ty_LBXZG
z*SVex+&~B-<^YC=?0iZge`jyqxj;TgrUt+M^)WWX=WjoPci8(PncKO6h3I8vFx0{e
zrSsc#EwL%Gq|@XnzYLQuSim9>*;l&kiBcb!)i~K$?K^J}f#%^NzflZIh7UpY*x$b9
zqMl)Tro!$Cu#cx;0sGR*N4zpDG0419BIe&)@7{c2Ch#nY#Li~$(XalY8qBF){aOs!
z7d&x)Vr934U!t@mL)THq?1?pLIIOsb=zK$?uloql$O?XA=%nk0Onl*9<@!W?$NWP&
zOfzWnn!mP4f$6TSA~W=TG&E`~l>ko&<xe4C{GC-J7qF92LwqS&E1+qhvHO@Kp>R$#
zqn{`3Llmw;b=&jr=_jTaC!sKqielx~pI#q#pt->zXE-2M>LsvMV$Wg0Hq0KDK4H(c
zz~W`sKsi0BO#yhlzihd_D|~Vr^4MnVdlDZ?(_|LUsIkguOY3pKP`W2KM`z#LK(zkz
zdgA-Cv<+v4k>BsC4;+V7Tb^jr_li+i&ZF{QXXVx_!B;b>H`BFHSQ37skre{6`fyMS
zF@y4-K~;6pEH_Moj+i>jEf?q#k4M`<^=+5jF0xz2sDmbLXdgKK!N?YRRSi$eNgv4E
z(4#H7xP_2BWsyr70yj^pu{Q6#kPB@~2eFp2UOWB}vHY6ERV+)x^hrdi8L_ISqo}7l
zjnBB(I-T_!*7<Uiw{wrzi0q4gSmW4N<5(Hu!Q-0syi^<}^Y3`g=Y1)YHP2LaG{!={
zC*}?rDK_sg6~2pg@J;O<QtdLZH9w4Rjh(?e?A^n0(VI#ghbTF<#o|3!cr+evvTY!%
zIr*8G*ujt)=C^`t(Bk;lYGK4!u9wT+52%~vl{%w6Rf?P*afhO#nleVevc=caUC8!1
z)6879UwypOXC8Qv1o`JkOT#jV);Tfp29X~oOUtv+=9k;~j$Fs<X`yUxqcyek=9{P+
z0W>%0a+#Zv^-v;_ZyCvydT!6n35e~<M{(|P&KL1i-Z#5=6XZnklP&xon2q5Q^F!tX
zyQ^OFSScS+pc~n0M^O!pA%%k3cc1InFraOP;>wmRECm=Vck~(S4LU;8PfEM#9fDu2
z=^C}-i>zLIkFU)S&xR@3!EYL-fBbU^(LhbGry9+CFeN2g&m(ZDZP~_(@4vmVwSWaJ
z70XxV7_mmjRqGzWI8Udn)!;Cq7&ujpL9-if-Esq<F}ALTQc->S`%;g6QaID>h2gX%
z2fC}1h=$2Ch5zB(<s9Bh7G7OTC6dg}&@xS19z=8MUi3PW$}8LGG-k~kUh5W0WJoRA
zNyjy$>zP#n=LZQuy`iY?CfvS1!BF7?ix<XLZeomxggXrRWECRpq*puC97<srd?4eK
zY*NY-`P6Snt~iSg8+nKLLL>L8bWbK#p|AIe_l6%UnDLBgu{eF7P#$}~9&f2w+k@MI
z73{e-vy5uJ@}aFuH0Sqc@X1zQws5k5`}dHzrVW@e=KC%!!tk$|QnYO36MtlT5<t($
z+c*8w;KxYC<RrbnvQk1Z1g?&D=6(;zH`)h8gUw9y14%{m{9UZOzAQ|uU_A80O6Q~w
zTK|~e9|D6zenfMja+<|+3YDwZgxdPLxN8ZEA4sY3*wYoGH+!YJ^-o1iY>C5s#d$IE
zz^C?W9lN<bGC9@xCwSi2a@YvHd+1|^<{Co`0W@ZA_pru&e~fM&ZsHVk7bEew>VHK=
z@Ir&b9-L`M`nleJkjYlv(YIVx4pe2{ex5NtlMPnRGx12J+Z)}SC0(;T)1TeB!ece_
zS?zuj8Z;V+&Y*UO9Wb>%z`pbVPJuZF0uhgKS|wQ1U)8^}S+3E2UyWCvxdh?<j*-dj
zpKp-7PuQHT3eK25SmE$7#>pUD?4etG8QG<x19Qez!UpO;ju$;q5z8QnJ+EaxDH*k>
zQP_%<ZSjY@K6^2*IRecOD8dg)aGd!(+Go%p?rW@r7J6*&jp+j4*=HhYoh!?_MygD6
zu-&UyavMbX8$j_hLAOrp-k8~wsqSUKQotioeAhMUo$sW+d;HJ5knRKO0e{B9(i=tL
z0UqMMr<3%>-5FhkCjARx^Cy%nxc~9K<2YWc?&A-kzti2B17(Ox4(Ah6Zy8Y_71t4;
zu3FE%wZ$G<6xvK`(`Aw5fG1N|>;?$&jDFMTfcxi!b18u^(YsV<%w8#0q{?1`<{L*!
zx_+kcxc$zZSEvJ;N4@LMZnX>NhnfN`mQbD}w0gzF4_6bL_(V1f8GH%PbkAf)LjVxV
zFZ9O){OIktbu$&>9vOO;F!blkw+ytUS*BVi^x!YI7-4o0NUQM}{%R(`PeSFAc)3d*
zKJg-cscPCp5WgP0R|ohEl5{!MWb^+mIk5z!*J*R?AmKP8s-r>2OW4I_9q04pfOqcZ
zmVQ_Ak8KZr1Vn#~KcUPuFoQ|-Rn&$ZRnl@-?g^e$me{)!5<@59u|i~ft_8QLU3cn=
zAWwl_ZMmvzsR<8~-|SE6S`7D+E4B1LHk?U>XyCQcI#_}_Iip`78nL6{yD1SarXaB6
zd<xQjKVisGTp8zmmL!3(CYD65UI?0ssal$jDE;8qeWpJ+__qNMZZEV}Arwb4sW1Et
zFmHTw*77O*nReqt!Me<M>gH091aH_qeHEN7dVXuBQIRG!@|D(Jjw0{KBWREJ<$JZu
zf?#={%Y)4@X_*Iu;3Ho~;m-gr9{9Xx95csIl1{K=Lp3s82Cimh%o_sJjfbKMh+%2O
zI<d)~rN#aQJ}>$i@#E)Nxq7a_HH6f+b$?QcJ^g|olYZ57RDTU`-*DdJ9(8Ua9;vhN
zhHK?qk#Slq@923_OKrDH8tGslF@<{MiOidCia(Iz6?q=8Y*{~Z)_h^3Ew2m@m`d%w
zM<SEd--tD8r($a+oAMEkOhez^{dw~t79#rJ`~2*9iz=T8N*ly}j@K#!PHWG%OzG9r
zmvio6Hf+cl9Nrcc)+hw>(DG>Z)ceH8sFx%)2Dl!j1o^bs8b&$l*-S|)aS7d3&L_n+
zE@F0&*elN&z5Ak;2iYto(ffbZ`8L>ifzk{fOS>I4JY)xoJY9Hup<0IB!HBSY8iSrL
zis?kR+;BNY**szOSf{&KgF4Z5DmG2^TAU|K!y~ZcoXTp>c4>W>s`(wUcPc=<Y1Z2v
zU2XKe@KY%!isV;a3cLkcoysAw>L3GKm9A#z=Eq3lc%*R=1#;K7D=w7b{u|g##fymT
ze|~=H8Zu!K<H0PP@fLSAiVkT@H3qoJku>EiKfTjG(y<kd<kZ;5niXq@y60H27n2Ro
zLRv#LsLS9r85`kB8`rD$u=9!{RZ~2<XlwZAcTE{AZ<@PMlB2%u$&{QchFQs^gx1j$
zrgl+dRuX{3?w$NJA~<Q$RF~4V0Bbps<x-P)Qqq&rI%TFHYF#+%As%`sDj_>h`4zdM
zn;~}h9dle3zvO)Fy+?`y);tzN+kjoqsd^RLl;|7(grITO?baIHy<Pabz@dGtL*>VF
zjT_M@P<ZK2`iKws1N=o<K<w!_IT_|E5XEZg)!5xg*-)QeG(>V)LOoA=so^+MHZy=h
zA0t<qQX^F3O7rh_I7MbxC5o-=J`<y!UP4f7)8#9z;~vFIqK0JC_E0dL@90*Mqn)pj
z#LrdEo<P$}`~ek7rzvyRnYKw>yjK;cH_<YPEe=^sp=9DzMjC^H827<WXo~XZiAVLk
zQG4JUl5>a95T?0<puJJ@QO*z-4d7AJ!8WYCcX6C;?7{3M5v5TTKhA2g$3!Y3*5IO{
zGK#I*7(i8FoU^zhT`)Ee8fIV?twK6p4A++S5Cwu)qQ`>_PZC-SO{&x>CNAL;d1lX}
z7NvQsR%{ZJ$pyr-@bRRSGW9jmCoo6xQFJsc<l&8sIBw_??w2rDbER7$&9E6_%jw%)
z?UD_pvnUP!I7(|XGg4C0?BGcAS2_<%YbR5(2rAGx4cY?>&DxqNUF(MNTL$x6ymL_%
z^wip((TpZ;G+b@00;mKfdSptRER19+s%FN`TIeY#7?deQ@+e3pi&W-+CX38U3))-w
zHY@b7D%a5*no^#b<t@(`ON{1}0)M6kB*pbTiU;tn(v@Ms`^s&N1xIqUT>|={>{Tka
z5)B=-uJciDV6`JL>u$>GWOLcq5VV^z&`iVv+M-(MrFpZ%1|lFM(ULsq1k#y)?rM~?
zsnuCQn|H7<Dk#V~;-Ec9TsX<d$H{A1+>GM-)^o<gNVKQ}(N+IS>vvdW-rs+Z-fbvi
zOkzJx!%uflh)ZwPrPBf)6iLZWY}7G$PUOdg8<=U}aoTBu;(?i{XgxGHh{tjHs+mQg
zI>#3f3N22CiD|H3JRrG=8nIEjMhDxia<TW98U-cx5@VD*rdLkA6#9`?nwiRgm9AZJ
zayXKKQ9S8VKV8GmNZYW;RTDNCisgR`Lm+C#CM(|nE!2UxcV4-AtJ#1-?1r&xo)o1i
z6l?0}te*vQrK*X^*rSCiFv!+FA=ydki(oQoX?QIEm;zPu8TB~J<toUKrE^+!VR`Gu
zf;T2-2G{rA>^wbvbZrtO^owMcD33K)I@Q1`Hc=9Y9W0bXrKKEZfvXh}vKlnirNmAc
zOkwlo)Ro08wYE*Xj7zpH8Ws3JSoo|P%4m7p_#mDX)6^oPfsFmnY~6?_zGHcxkJ8Rt
zEDyJ{=AM2Q3*KX*B1xhy2@oAzAz1#<<xyv2oqGDJRE?stX^~+i{}rPu+NZ8@g-Wm&
zepN@!zQ5L1wQW1gRz|m#MDhi;#VC*6d2a|D9jNul!ArFTn)erbmcB)BaU9l4Lj@A%
zr|IW?FxntT=b6f`UupI7dt+K&Qq-KF2@X}f=tuBp)op?EZ5uid^LW??{V{M<KnR$_
zgVpk#VGO(j^x!!~rQx;RACv&wf0%yXw}Cz<glnIeljuUdw&jgrqHAlL2Q5`sgFJa(
zHFMb~M7KS3ci2C0>r05q<s_pj8-0oX(4SEJCK<s3<{%WeLBc_a2U$;%j6T)zTrVtZ
zgWC3->Bp`&G3FjFwQ9jYK#09DLd*u&udPT-Yth{h6Vp*wM|oa_&j8IpYyA%%>-V6G
zvNBKGDWZ951iHPcpO}oDeQ=xCI+fasSyo#gw+%hRb~4g9sVOg9{8jv_G}g<XFw?l7
zsGI#uIj`zt&x8Ri3z3F(-FW?3Czq0MXHxpb+b_scOVMI?s0lq0`REq<9!@))l5g9)
zLUFm>jC-EyfT3bwr^rIH%E>Tp5k%%Ep@790Uf~#arinAIipB>b{1zw%5&07e;Qz<J
zFhWyP{dsxb7|0Un(5AGSlAv+)$^*R*?O-qn|6ngxHhaMMlh<_!wo~kq<%L&(0|w|;
zSn=_*yk_Rei0d<s7MPO3Teq?pd_X7bip3HE@I=-d+FiOAEcel9+zuGS^?ZP5iO%s4
zmHE)Dvzu?ITi;7P-fUi(Nb`9ZVt7%26KVr<W_XEnaR#jqhK@98=!=41`%MzN!zFrn
z3^8xcJ|SM{8aK?!OKU0Y-?+3-BmD=_t=$Uky#&{5O_DGNncm6vkNhL@0n_QO<qNTG
z?w-eFH!Sc~>spUUk7P8C1|?r!&IlcX+%}h!Lvwa}l8f26TQs}xuDGum4-DSdzMZY_
z=nDXvg8okVuHIU|GD)(ccf6d<EW7o%7eQ8`KZ|y>0_0urX$MKm8r|){qxGCm?ekdF
z8fg{pz%7FUez4}kpBl2VWB!fdIQ3Rurs=MUw9X6QMbE8~3z<cI&lKWPe^wJ;UZ+@W
z=TdY_-7V~LadMuZn(N*HDB?>1Aip{T8<}M2=?NDZs9Mjc_oKKCO=}Z&K$b(Zj`!ps
zwPU2foxbEPD%e@~a(9V-WK36yCyK#mPfVbe5*Gn$M^T=mn$d+5sjqmIhQbeV+TzmC
z(X0NX?TfS$^dhzaQr+%%(zGP^3}z1;CPJz)*p8Jv*B&UibiI9@@r#0Qe42HQwJk$n
zXkx(2t90OeM}>#4hnGa%;~urPe<D0GIh0{#{G^DO@c*&`2($emBF+2oO|qNT+XD0A
z_)mns>b(W{g&f5CPQ2ULu9FpY)O!685~cEm$@7B<=`pWk2W<bH`~d05b40Ht_@lx{
z4ye2^&BU5yHuF3z-*3>}=&z*88AhP`U<AiHeDyppf<OOaWhrV6fohIrw`!Cp+((-j
z4s(AK$UbeEa@DO_N4^qa6I-Yb#SyGDzZeg?lQCe+H0;RgV1Xk(!+XBCaa=97G%rL-
z=bAU(G=|En|GkCqv_jakCHYLzuv3{c`OZ=$_{-h<N;%e*2?YTj5i36!+~xEic-CT<
zQ#w(+Fr<5i%YXRD8T5rdL406@?1$o3)x}a-6LNIbhH8M4pq(b><`Rm|@H1xi8qs<N
z#t-zg=EIYbdq9Q_LS1g7I!QO*3PH~a1$rhGO*h;95ly1^bK2ue$?2Xo><ph!-RvU^
zDQ$ZUmGu6Z5Nn}`<Ct95y4Bv)(%YR0b1x@YTIC;f3!o}wChWoDwDv0pTp=TEiwX6!
z@hD%dT09{r^gv_PS>`Xg&y6iQt^Ka8*FG@=k~Gudx}R9jKymP-Ju$s@&$!;FNatIE
zwXxZ{^`Fq5Kfm<X4HC*WZ!&rkI@&T+s)03__N{ap68Hd_)dGOE%0#q1@nh*qwLV85
z!&yp{cO(6C5zP;LqP&^IH@c+{4gbqbDLChG<5$R*T7~#q42zs-gT#LxV~5l}hWHiB
z+}p!wuvfd*+=%1LhV4=NSHz>9Pyh(6o72}c5vH|VA&8Upg?i*V%V+33C(uUW5u(8h
zs+ycZx1{a!SVSRgwF^<$71{G$_|z@H{%&eHgNW-5YSa_gf9lM`gK8~DzuQi}%MD6k
z_qnuaUyN@pT1+dk)+zD<8GWwsJw<T0QeR2q&7HaJYly(9+b?PsH*O&)H-|q8HMAs?
z$+ttOoAadL7`{Mc9Bw?1owL<Gbb(V~tHTO6wlG@0_SdVA^^8nl!6!Pe#}UwV$8bYt
zg0PFG{m$ruk%C1oMCOejdsvFL<f)c|<B@=eJFTMY^Y{9{=3@^3LP&KQBimC|FCWNT
zgAu3WN|=rh!kV`rt4AkCZyD*2Y^~NeVtsE?N$`Aq(&A9szy9B<XqrdZppeQ>SmUiw
zW=(r}&yPrRlxqq<A|mN4m2&lc@)tpoKD@a1$vga8^C9eKKzPH9-Pi|Aw2GaR_7HxT
zY_=4z>3MzSQoHL$=PMz&@+5*~@olU}i&R2;toOZ1V9F`54_6;&4~;)<_K*E(@Jt&O
zoV@l>U9~N`){okICw4yEetrAI&{Jqn+F1q-`56@p=<}mqJ}$A?swbv-bJf;6)CK#<
zas5;kB)?Gh(C~vuy^i1lsUF~n=srY96r+r)`^I_En4*MQrj>?Xl5YKUVx08GYBnz%
z+>XSE_0?C9$M1(1`dP<S+GjU&RK1#%pUPFpc-&Y2?vWP=7$CcvLrw=Z`}FF@spXPz
zGGv9)7*5tF*R6?NuWk*cq`LNrcsa>t&|Wre=iQ!#@q`bRa4=jEmtgTgh1wv*D0|a8
zo5?wau>OY{lPO|DKnQ&BwX`%GiX_l#vX~D@-WHjs%jjP=w7dJ~treY{Xbfk%Y{(a)
z$&g+{Hd{6LU>N0w5RXQgs#-%{?2GuYskN_T|7l%{M;X^d+E;(ywBP*Q^yk(dI#M_E
z<MfH7Rv}K`Tyv|zE+NR*k>#N9oq$t8hPH&Q23qzV5<cO3q#xufT^BQykKw!RPVB3o
zT4cI5+P0|m3!UhcMmI5-<dWJ*)~)jTQ;nNF<RZB#FIRFr@x&EV`O<XT@H-aBK%&#G
z4#dFeU&dk&vK}fz_6@eSS=EH06D|DYn>rgtW?BE>LkKTwwnXod*TL8{JL0p@zDIle
zye7gF)Vlq|lC8F4c2(juXR!)7=@AMt`hAXN5y$8ZS|%#cX%+vxSpB(NDqBc^F{NPk
zyCENq%ppu~?!81JN@_K<R`N3GQi#q7<@LB>Sa_r36FqDWwvZ36nuPC^z)4pZ#15XP
z2jm|7`bYvR4SdTvWxb5spnw8;Zx)<p!ffp;yNQ@v3WlTg`=Ab}{aTCl^m7mKI{0Rp
z_VMNsJSHThXPD)LyD@C3utqf}Z(gTU-&(#YIYPLq7`Tf+ReA?a{o)WbUK_g`hrSS9
zM~^2N-4m(>OBBpuHJ=;r_&cD<-*4kjkMH+B!jo!dj2PLN0)M~XBIJ|oBm}{&va?iZ
z+;W;v(Z{N=NV1-&^VAJTX;O6iNJDO)(m&q!%c7GAcBKd~2Ks?kyyO5P#u<mqr1DLc
z7sj1*bB2nk(T(Y|XK`GV#M`ja984V43)@MSol6jZo5zlib49<Xj@pX3aRgM5Ta*L{
zU(op?w--<f-B!^S?lR3V7j&<`Nul-qV90pnLUP14;;XdtbeHdDdJE7xXrA79e09KY
z5nnJAZ!ysG*_P?~S8ale0*Urkeg?Jp=%kAIFciysbtSfd`^n7eOI4`6(P1CH9oekh
zE?@OHc@;YfW^5lErl}5NS3Jh=-$I>@J1mXF-Mo4eqCbBH>2Yt)tI-<-y9FGArqTjD
zb>Aoa{L-fWwpvl1_HRYv%!Ynb_S$ypK83Mob`L1(SKc>3TLz_Vu+wE!T<Q#CnVc0{
zie~+ktSivmz)VEAi#WeiiiH#WG=yS|bqCy+Zm6pVD+@h#p|2E0thG*)?AN+Q;-rP3
zG@fID$6c~yP8E&RtoRirxkVwkW%l;d3LUf1pHGIVwKO^Dz@)=^7DBXeG2yRvr;_pR
z+bq64r(r9Sxe-Y1GGcSwdppw-M|`Nhy@4-s3+tV(NnfE3vK4gcVY^H6ivtW5$z@*X
zcMjiv{_yp}yFzboP?n3_gi2468(_b&qu4U)>%Y}Iz-8$kRRun1Rbx|!pc~Z%nuAsE
z(XB~oj5S$M=ts?451*l6|9Bf^RtXCR8pm4<uGgcA7jS?YPIjz8KvP;(fRvyxtJK~7
znl_^GLDKR+j&zP?o=Aa?;~pGGBFv`jq|_FoHJ-Bb?lQMY^PIJ15CYF7vv{B-hbB+w
zP-ZLn^e#f1-7cH^c|##iA<IU~P!2j(8#R`z$dW-K>7=3m!of=+t%oi*Bg~`1O?r{5
z_|xG5QUtGh>V{G82EcWnun7UMDcqFz;+w!$vx`FJ)uUD)s%R|)gjZk-*F$*}D6X(x
zuW|evYGeZ)=O|^OkT1L2S9;K`es78T@CT{M1!j+#<1zm*Y)y?Xnoh;Ey3Q%}DKXPo
z_QNk~PHJJKEfva9EGoe{gqD%G%QAw0)oS~97%tm(ats;6>F9Zm=#EGPE^DlqC3Tf-
zYzBk7^2Y<UtY81^agp85v|yl=?1Hf;G8=hrm&QhmQj8blvL3no5TW!%R#dizw|lt|
z2y(++_%J5u()ZAx|L@Dhy2tmbwS2+0(}Y%gi;d8s%{j{nW#B?pTIZs`eY*M$dg3vx
z6JNDd&U|H)>u`QX?G8)^T+H9w?!9Zov1<BuSvHH2ay25c1@l(;gG9O2_c?tvK(yHb
zjbj}6?+b2v3HGJ3>_7c7Je(M$ErXi-bl7Y4ydKDDfi*=6WzB8XMm%V#Y;~Q9s%Cd7
zbPZiI4b#(Y(EH)HMME|J2)|M8DP`_R=Pb>9EBl4F$o73F^FH!Ropb#Ig#)Pkk9M)B
zc-Ry4$`%0dZ&4~<q~I(+gWg`|uu|Ss<3esQnc}wm>_ZM&lmpN*jp7b6Z$d&3TQQst
zKO1dCD13V2OJLg+9a@x=Hcrhx9q${Pw%7bZNDw9{_bURzPyUHj&U_H3yiGjib~ivk
z<ppLfWYj(6v5DwI$@LES<rZgC;R49}7J%!xKqv&_Q!FfhG6JP?|Lx(S__GEIC{F3q
z!tmP1mvM7Pl|r1eAFZLVt>`a;(i8~iz3^sBAb_tlr^{ZE>+}Z+*u`T*`Vm=4SA327
zz3L0<Q4R|-3X+rC5!Ji&O`E?@u&bs2__71W;znH43-rq=pv4y?nl$BLj2DEP)&$4<
zaC=GM^duL_5uH^KR6?FRUlg}W7ngT<MsqLo8~LIh6&fY|#g78GDfBQxxeG&g=-Y?>
zZpgnDCMu_1zEel7B_2v{N?#FMEiJOcOwruI+}=^KkY9*C04<4bW$^N23aDm05C%vW
zo@nG{mC<G)l#yrmCLZ)yl1=?@kqCY^E0(?0N_4~yo|Ix7E6}Eww3qfAnQq0p<|g_5
zsg3}%9HLz?I^EsmqE2p|$CL*4ypUS4nb)b}29|OM2#yZEh(>5Tr<g|s?sChOkS}MQ
zc26IY>&3k*yH9r1F3OcD>}j1NT$aJV;+(7A`?GfNlf17UJC%&g$Q9U%V?5MlC48R$
z0$!OGQ#$(*@7s;6KxZD1Gdk&^4zO$qSgeQRHR{Oq?J{5k7Ch@$CI!ayUMAFTadUUo
zf4+ycOHYxUQh0Ko+T@a$UEZCt__-q|hBn1MtPn4EX7b<uH<0DoYN8UeCg|iDlOd?T
zR}tkqkv^Pv?<kX#m!en*e=4R+dil!X)~m`Zc1mb+j`85{X4L7i9d<+SGr%pS=X0wJ
zKeRWq6#^U&cNX1H(nFo4a?K(E5mhN&lEH)`^*V*M)gcdm*u<>hy<4VbZKAtMWSPSz
z`eAF&BW$|!P4tM~vWU|TEr6;|rCF!uayJm~>{NDmBs>XnEvOdIbg`}26r4_x9Y!a4
ze=L>E`BL?Fv|fp6Ic<krvlH1{Gr!!|mfrnSo^^s>DrLKP%eDE@@G&la&0O<B=-3z&
z`P+|ZO(wS9uS?!lf1NoNl>>smn#T5e3w<+R%r!@%OsQ?Ob7}21{f~jdtD_~>v$cfs
zLpt9!xw?VzC|c7&2d3-6o!p(w8MIm`sPov!=p3t3yZxl{MbXXETdN9)%sju1VtQ2i
zONsgthxH*WB{J*dar26?RGtgK1eU&3Xalpavf9J=J`>I<s0?B3pWcQr=hf$s1J8^N
z+u;T9?{_Qu-PM~H!P3<ug8OO&<`cRHR`2O$8?Z?jS;9}8PUeIDz){3J!#LhSyK3Rw
zxT0H>LmVxCr=T;5C6v?3wV`u`Sq;}9$~wuU`+os8gn0^1+3!r=Et~lDXbCr}jH7&%
zN?Vf3)6JIkNDe6(hbT?fk%Aa~E>9ncSzd8wP6mHFo|veb4QIci9Ca~dQg4)($fCf{
zr#2y9kJPUB`A^9IFNLGZ9N_i^;rLJf>-j%P;czhY{9jQx{x5RU$^S!6iZjqnNKe(r
z*G~c_Xrsqzjs8zyH;ieg9)bTi@TTvA@vPwghl>d`d}3RM{K7uT{vW=Fz4QO-9v3=Y
zR_Myu1bc-ykb_*c2mwuk!YCrMbE5e_$VEB9Dd292E#V)zy-P#-eHxXBBW73A&m9dZ
zY@kasbF-t4ZTD{HsV+9p>x;<?zTqqVoUH5(A+I+#XGbrv*WKIO{qFwTY0Zh1w-<PS
z`e?FiTywspW?P8A{fn6J0-B9}-g&j;&HCt4+$4JE3Z37$R5UnMZRu(Eu6f+bLRv!%
zo(;7QJcjE1-qAhCk2UL(YPWsjI)miEAxIpTsHeXWJ}RgL)T=w&nP?5@sFB`)CYA13
z*o*vmjlP4D6z?)$-oK*PH1=c*he&UBUna#1H{%AHgAgV>A~q=XpC5zvT_Zs;`s!eb
z?l~xDlZ2}sEG$7GiAr*l5i1)_!-$>rKWLJ!)peUI3EPU-3SSou!+&)ZWlWr*PLNGz
zz<|Xci#b1zXhqcEpP>gq&bePW92b{#77W{3WVSjha`KyRRP^T<DC|cw)o-{AqW?xo
zWK0vkF&6a7{VI*s`i*0dpYkUqa{RX*Hbqwk5K5xrr&?4%LZuBm#JAN^%z|`m4a*XX
z#Qs^c^inf;9*MW&24o4JP8(DF?lU*_QB;O_(zH_8o1IFF!Q3;6IwKpTI8g5>_!kDj
zfwX>adTqfz2Aswz;lSaqL;xfuFV`7T+y$gp;($_bE1vQ$xpVF^+1j|MdA7&JLqMSX
zUt_f;&nRJtZz8slyJ`U3-}GOZW>z^jYef;?e_@5vEP^Nt=^?1FQ~!-R0#o0F(`7M!
zll6H`-e6rSgum!q7RhgGnkpu!`3WnBT5Ejk1k+lT$>8|s>~=c%{SqF2#Wx?<fsm0u
zCjZJtmot@^exZkUuSQ8%tM@ZtN)w6ssb^o}gzE~UBI{g`kdxvifBpxJS|ZWMxDlBM
z9)W?QN*KnJVo%mRoHn+)C&3j>;{AnDN2LUr3u*~OK)CzpUDC=CO_loBCYyJ4DAzw9
ziZ3~r8`L=y;w1$zz&V@vmf(U-?DC%(uRT-jLxM@|cSwt0_bvfoA*_ut<kf6`!?u+b
zJ|G-*#XFz_OzQ_EC_v1v6cvQLyWVLjy#QdBFyn9-$hr}YT5rp7F89)jL_OiX<oqT>
z)Lb~C$_79k=gY`zd;p<v;IhWN!3)w=hG`|y!D+B$u0$TXyiX?YkiV!b+I@x9)z-_D
ziC5xTcHL2pXdsTL+|~3Lbx#fZhl$6_vRqRezGdl9{9CDd9XXkKLzrTnfgQq6yo32p
zgMT9XT}D}<ZhR9P#6O)4PgL$}7niS8Q3i!e|7VoZh9nbpIoC+L#YdO7r!%zBE1{Fh
z%wOp{j2=Zji!qYxU#!jLe`_%2hpt}9(ybPJyv_#yp(g(lfiEd6o=u8)P+GrjEeHN1
zNv0wtKPD)(pCBKS14^=tuB^<_A#%7Lxblc&wT?0Xy_juwq8Ty=oAYFF$9}7Ts~kFA
z+!9>V*-s?MlG%vma<{Ld2}dtHuoZ1G>7ZNt1<73Gc!~NPeZwMBNuR985KQJ~ZI7Wx
zk`kn02;-aB#K|y#w2Dg0zdW=x`Z-1`Gk@k0A%9B@<1mZ=wMOAYMW_i|91womQid62
ziNn-Ol0(2(9qdE%QAI<P@|6gz57Pb?{X1bcUDssEo+AR+(zpN@tB^CyiVw`lJI>tw
zAtAS@C?{WK5gLZ)-kO7buTsE4eln8rRM;3(D!qJU1T}Ovlf?>VH?C9*4EeU*e+E54
zf?_Gn`>C1H22bfVm4OuOm=bVRYic23n)~9CD0hKY5W_a97D~O6CXc4BSlT40q5w6X
z)k+K)Kiq`4{Hqhkc!x`3t$B3>k5E=*T}|leJT~U44#6}u&s>Yi&`|L|SbOW(IKFO6
zG=`Y%#Euzb<~C!@%*>2!W@ct)W@ct)Xft!n3^B(%{(W!m)tm3lyYt6fX?5w;*`<>D
zRJYdAuD#b{feAb=K94X>@wm~?w}!%!cJ&Z}UmoDo7JUXNs8o9|R0vkbov<QeHL$*Y
z{P5tPMv+I+HrGdA)fSU>m#{S~l;DQLL)>dK+g!5xPsCh+He+~Oa(q5qAI|<wsY;2y
z!DNN#%a=%{|6Xo!H2W`g)fUpBYqgi1vU?n+c)GM5x4WS8ks;;9x^=|*%9AhDn$bd|
zi$)np;}Qo$$<X2zEXAW|J+8W1Lbg7SkDmuT^&@}Pv@w!AllTcPu8oez_Ls`H-yI)K
zA5ByEnKf4*ts)0DUau!FJ?@VU-*`PA2d3YzZ?8P>-Z6aNFUfgZTkdXODS1t<n~Cv0
z+`}Kces_NCkQ1OEbf<upjWwJPZy$NzUL#_7o`#Nm9#=(mJ^!j>3m)G3r2bZ*-Y7R~
z6S>T2)5wl{1XgFKRNU+*#HD0djQ_$KQ*=9_exjn|J%Lx!;h)1O@9Zz6Q*aX&*R8w>
zPv{mpNFwnn8>g0jW=v9j09xw{QYn6u-C8u0K(sF#$5F~-RcBr(A5&9KAMKE<5MnBy
zO-&_u7rZ_ED#)x;CUALGwtu^H<<<mcT)?bTC4O6HUxaO}&0?-vbbV!(D$V~Z%6*YC
zyIQ^kyC`4|e^S+ap<GIw4XaW{UNedSf1$~fSY5`udR#U+EvMur!T9i~;`Pqe@1Jws
z(tpk|%3h7qo`$G+)q&n&|C%oUXF8?KQ@vP4kE8+4UYXxtBenz%Wx?N5IW`v^wbcLa
zUf^G&GJnXXTs1V3(yyfDKly1Xl)u87^QTdLUm##b7C2Rmuz|Cyf%mh4jgxLlpF0{o
zGQX*oz-KWd2|OHE1Ea*2_$c<YCPH0Ku@E-sA?0@!tq|yreNR?N<6ryYqFi)1<Rd2L
zQBh=@GNj8R*^sd=Uf_)=5hPC7E!1i)7t!)!%3LNro6DRKbP@Dq{yL_YCKvI_bHeUv
z)haTu_BtjMm7YZL7j4p9aY^{KLV+f_rbpmXl(}+fIa$oI{AS8BO*!c5a;{V}>F~y?
z1~}jODjW(I6<FA>rJNe8o>~I;gAx8#SziFC??)vyN<*U*WY$CUPe`rupOB=|KcTXJ
zg%pFtL&GBnD=1XaNK1v?$|?A1$5n-4&E@^5B;+FCl;!<3lnTS=-iQ8)P|5!jDf=ff
zt|X6=zudDi9ToHdg;EZMk}RU(gnhPX%nN!?w1_I4PJU-`V@s4n2X**GvMid9zRQXq
z*PG>=NUoHJ<0-20e4A-r)=F9LEWM4G+<N?)uo*PA)_GsU9M9)0arFdF*v5|-9{AA?
zPTRRuC31c2C}+<r%ftMv&d63%AfmE(R~gLWX3{kgZ~6h?r@Q<q#zL9MZmgW|W)mgC
zVb;_I*u-oj>r0+7DstT`aP$q23XI>?QqGi9&nyZ5K^1YUY$srB=SL+|N&`?Df=wC{
zE@#iHZ&gDpIZ7(s4IJN9_~%-!;_!qopD*Q2#jB~r<s@&E#QinmYa~z>;=z^UbCXfa
z;~|?0#ga%$Nr%cw_-QHSNl%`N{xwo075CjLL_s9M49d^3oaJXCniZsll_J3&E6zSX
zLA(~fyDx_<t~MnR7igR7q3Ww7_9A}TkUHEqLt(B<`1!az%!+-dI{7<}D@i%+70GPX
z(@>ccF3N{b1}!|qOexAuGt@FwI4rPGLaNNDJeZ^NK0*k=CPJg0I$>QmUpauQO{s@0
z$|u{XF+ba^|94vDb^3k_ubk%1QO=D!YY@t4dYNPOj}@2Ul08Lc8t%V`SAXLMOBTd1
zDJGSRU=xbm%9-`+nRS!Mq>|A~=K{y(1eDY1)inwwevnB<D4X>gnT1hFr_#_W7nx-;
zDyI*YOvI|tfs~D5&DFE1B;`Oi%If|aD!Jn*3-toZ$+^jBW%ZCvm1;?(rN*S?H2k#G
z|GG4dN^*Bz?yr~h?oIN3IY|ji9A;-+sCY!+8A*zAJSL1GQKCp72}fOABuk<z%Y40&
z%z<MLxnY<uwge$1at_$qVg42Y3tc=TVd23$YuM;7ho-;>f5$EAv}_qy(!II#ajOD(
z!G0#Ltdp%csTAVYocUH(n@7o`MuA!3`_THB`ErXCq>e=L&kiV(L6OycNxgl^EgA3x
z>D9YNQAYK6W1S;M+UF(gRY*MM`z=}C_Z9D}TUnpm)fa;FE3fP6o_D#f$BY;r+pU+2
zkQ}e~*^->n9Ixk-k$32v_d^o`9}4RnuO08#6Y?H0&7~t<PqKsVo8*O$S2~Xy7jIt8
z7u_C&<-gnWkN32cds~k_A6bjo8r9j<HC)_my3@2X4)FIIMBFWE)3mb=lJ^^ow6fN4
z`v5-cFNF`1t101hLprCOiLPar8r=(WlsO6<MWzy6i3*g;3UGxBB9syLO*O9`J)J`p
zuc2M6FP4B+t>!iZhsDG6Nv8}PRyM15;ucHm^^LkV3y0~$?n#J@5Y`w={PovVa#mR@
z5%x$&%mezhyaYj$5CD>PmQB?{BX}7|A-)J(k}w&Y=0KII235PPMZvsDfRaVQxUg5u
zDz2UgO*lMk0hy9efxD525?g_;P)RH)7M!q&|5Jq)k`hV*`qb+{z(-?i_v++uW-~8D
zP&PydsSEF!{m$w5Xy*40FGH_>?tp6uERrGaFlVF#=05$3e&_%%{eph#fNe+^(zIpv
zdZVN1!OU7-)NO%Ed?G2KBp{iY21_lzW?x&pE#8WBRw8;)f}}}8B`J`qO=+&wNKQOE
zo{iL4qIp4rR9~z*mYEPsAU<IKt2k^tA_=k>(#*KfbG|!rr>V9P01m@0qFQ(^{}8B)
z_vhn!S%TW;8s7?Ujib@!czb3xZ$m$Rz$k<gNfFPxpE=|Ni3u0TIbolAK|i8jJ-`+6
zgv4v?Hy{uqj1+;F%ueH^c38WiA2c8ll2bRX-!y<10*{1&2jI|hZr!zA+(}ln6PgKp
zY`d_X*l8Vr4M{*M!Ik-yX~%wMJGBEb_3PYm<)C(5KW0ED#0%*K4~ZSu@o%gkQV3o&
zJFb)3UhR^8)_{8mA`%XDB#ieIS;!=kCT=sQrNhc)?Z8g@fKvz#QqOkrfHrcCBt}X#
zU{kBT<IeWvWDazYa_4-qH@94rBk7X@3;3Zuu6@a|;X^6>2l*Q=J|~yG>-nF-owCyP
zoi;(q6cs>ENs}Z_N;BX>Ypui4_T+eO%s3DvmcmOvZrliROu+&uY8SUj*{7W|4jKD`
zdgy12V?lZ;gaAS9;5GyotUbmh<4jFEq1YiGU?+$&WgeiZ)!c6BuzFcPU|a$Mr0CYI
z8^?fPQxX88+R<&eE@>x>Ame7x$b?<m8RN8ZHwZi>2oRx#-a%uheyW0s{7MYcO76(C
z<2bADH!cEEr7Qw0w3gaE`#DqY0XSN$?Uwed=k`;^H6WgpH$aYk??EnzFl7KBs#V>t
z<<NF%KX6d|!Wh22ykYOQf3tEBcJ3atgfFX|-InRnwtb`9dTc*-P!B>&IRemfueokG
zx9!<4A240EZMe8?-^eNLFpL{Ej@ZT^<6Cj_lw^r|#3140adWykJ{<2d%oxUw=+*d#
zZXX@c)*mSGtHxO4Z*sf4-rXM`&Hgs@6_t(AF)+P*INoCzG%OvVk5R-g;XZVoxKAaQ
zoJ+l`>ov?9A&MEom*h%zr@2+Ttn07%&iOaDGYHpM)xGjY?V@gYFLMMbCJVoV%h}^R
z?!e;we&wj{_nz+vW=tLa9@nsk)Spy}K+{AIshiYuh9Sf95$2c^d?qfS`^1e7D`n#k
z%EqPC3%i-U*b)60Vti5VXt&CHwWB(nhpu6$m?-=VE>;iA>y^{GNyCm2teA%S-z?D{
znSa<%u6y_LMjT>H@Q=6wu3C4o2_VuK5WXrGz8lv=>#5!3Ui%1QOa_`5631xYY4r$q
z%nN>t&^5jbSGJgQEcZ6kY)OrXWR40?mDk$e#(T4)`K^)wS<xIB9{bnR^ZB8Y3R$+C
zOCD}d_vgEtlZ*M08VdX4qFGh%Dp{W#ah?RP;it$)%15kiCOPk_;h#BuJc6FV&j`0z
zmrVU8h0_!{v-Rjv3_ONjtbZ-<R`(k=OoFB*a+G;0-csa|AXuDVEFV^P8x~EXrZsY$
zc+R|<pDa^)&Cma&?J_Nzq)$8MVDT_{0iPxw(unn!Q_q;DO}eMSbAosxJkeihuGLQ)
zq*#N7@pH0xIv{CtE%;h|k9cSyG`=<>QAOjL&+`{Z^R4@a@WgoHzi?f<o!&}0JN448
z-eR|SUH;v;Z##1MedIUomIKd&;ROgIVg|fuUALV&OdhpQBj#lAcz8WLo!8KR78Tg_
z9_3CG<_z$NdPYCv-f~<z^dA+T!$v9clz2`3&AjIz@Bd~>FhiJY$XDV^OZ4h{2_^_0
z421xXiBC_YXVhEjJKG1M2&p2v_N@cshH(FShjjd7hAmf@@52`iEPw?H3IT;!&#X7!
zcL+=p937GdA(hlf%nLU42R)mfPVcKPC72>a@mHyDsf0>q69||X^gMb3y@S3tzT#jB
z5YAuEtNH9*gY=0ueY?P3z#pJ-Vc0QkDfHBPZ+y{E?Ak_P)X-`<^`v{(e4W6~AXp)B
zk=Th|`1eeE>wNdX$iZcxwBTFuZHc!HdW&-NNA(LwS+^y67kw4MN+6~nGZEP_Y<WJD
zu4YkDU>V@7ps8;w*tN8J9=*}NSzv6&0~Q-_t$4QN+t$5ZzAs?kND?5q;9Or7#`PnI
z6MRd+rb!H;8xgS38M%yvMn_}i2@P39LL!lgjkHGp#!eAx3$zD2!k+*Fu(<`?gYV!@
z@EAFaB*b!Jy9wU}o<XQlV1!iy8X=9aW;k<zLNBq9*l5D6{&3<DPN|vH?~DwETt*6`
zy|Mg+0fNDS2#Bj>Da_3w=P+pm^*@dDNAqKc2r~p&LoDMYu{8ykf-1kQey?XRG9E3D
zT_m&+SPDiHMsi>&4wiyX!)GKm0*+S3t`a&390pG$qzq5RY7=${CI`|Ws-xF)83~Wh
z$C~u}4&TP&5V8qc2W=p={jg`-*BO0_h2HsmykG5IzR$AeYYz^@iW0^MW(PJ#nxf5e
z8ssfZ#Lg3%2uuZM!gCPV6Yd*~7R7?<@tCUvwGi9T?YWI)N4H|V2wwz|0&x>3^X(}1
zZAar`3kXL9qC+xaI4I%5kMWdoIh6FnL~=sAk>7}(MG-<%F=u)51%*aJWsoHgkO=c7
zg_c9DkvECmh3+CLVp6{|uo|Qdw}xILLl8p=BSd4OFfbda4Hky($P32}*he8F6BCOF
zM@D1*uwyjP9efXkK!zq(8BL?#4Wv;=N2}8kAyN^k3|GUe6G^VDV>Qqod=5oK#vx`E
zwv1*+wPW8EyJH;Jk6(BQHAGGzE*6rCN=0YjHV_${2vuhUJPIH?5FLt4M5Geb@iC*+
zQ5e__qNEkoks4SGHiupy{~~f0IS*gKt0On4(KC09IYFLhDpr&_N(C@X6J?7wMw+3i
zetu!&bdj@ERt9au_Mwd!*PmuYyYhp3q2vNl!;0br3>OA#BaWdbm<}wv8iV(tI0Bi&
zlcAc(EyR~X8&Rzoc2v8LgK41+Xd*~VEK|anF|FV27<Lr~4??NydsDF(m<)kt6X9ui
zOyof8iO$p~26zMXVVZDtniS^bL7FIa^ad`V@WgnkGDC-f^N3}Pt_a6OT!Q_?R4O-v
zhoSq>UG&M1dA4F*DIfLxjnhGdNGwz)cA(hAOsYP^fT74>Bq}(w$WUZ7)(<9Dpw`6S
zR9J=|hU120L!;I?xJ;x#^NGgPD25CJ))C9FRqTc;@Uc8>3ucv($}n}D1_01&0&4Y^
zQsXmGn{O3vh5gL{f0!%W4fmD=s5|kV3c&zvh&0AJ!;b4fa%ehHm%7J5ZXh$98P0*{
zKvqp*zmjXmdZ;z=lnS4JiF%>vC}qcRs4!t1mZ0P%?>^C#dd9$JkS&28;f6oYC|rS>
zA@rA_#lU)GBdiU_K^7%+@Yhf}s)Zt50q&A{r6dE<&?stwLZl1>+0Y8=o<gK71K|*7
zR2b#BqNBtthj0q2wxXltEVpnBDz{?g@^MnWOE?eZbMbLXzE3z9B~&qHqCxFUB@@Ze
z4C;Uaf@~efkjM_>P$sH_;)LYe>h8y~{R~{AK$RHvl0>6IRb^j~UwF*8rcz13)GI2y
z0*zE1#n3LQq=J+L1IJJzs-khrv6LhO_fRA%lj4K~!>^$^RFDFVY@Go6oO+r;Qf$0I
zz@MUd;gSj&wyaA|?oaj7r!JPrjgsa#7wP$1<(n_SYzh1h8o9B7WML||!FnOmZz{<r
zU#Z*@8D&Yo$%Si|#*>ppDBolnl~GYtT=%F?wtzI5^P##N4^Jl&luu}8u&S8^ObVwc
zvSvA1unL$|N?K$Hvbs6n9G}k+X0S?`#AJ<Ld?-ORD1u!z2q0OI=7K@(IYCw)=YS*G
znbd3=mEBBQHIrZoCEI~mjh*IR^ssi62lgYWi5b^|YawBXI|sg`^s*~i2wqe&XO*MI
zS>vo(g^y<P(}#M(N3rp^pN2`vFd%9Z+k^SR{Cs|;#*S|<H+UmJB72dY$i)GBWM4#O
zFK}3#W85@zDOQU5Nt&vA*ewbk8-oR4rj?voY^PlDll18A2E8U>H+|SC3dg>32CIgN
z-y~nAZHgo-jWa2=h)L5VdI~?Qi}S_t;SBe?x1;lk60l`TAWN7t!YTQfW>LMkLCd6T
z3aU$B0Vm9qiS^WSZne@u&Lm<=CCiF)-O2g5a>g%-v%*p3ta4VpqCw^;c1k(Rf^*p^
z@Ml&AXN#lt*~V-e_3B`56F*Ks^Ds!oGX7(G$;@PCD>tBzJL`q>n<M@y*PL7B{A_P-
z0>4QBIj5{s_6gUrnc2+vcI^~0-jIBjA?Jvb%yH%dN0EcZQRI|*mIEi*krUupYoV>s
zLH#IviY8zc$^qdJ8%-24sxH-p{<ofApFn^xlm+4vRwdhtUhQmeaRonHz$Fwnf;;vd
z<FVDuNNx>3FGR>Y6a)ekHUcB2CH<gYX&-&S4is@@I1zYC98>{<2u>serUm_!US}U<
zKp4~k6#hCwGtN1^nq}>vURfVwzyZ`RL>0`xmCS0!wQG7&eIx;6P>Kk}*iuZX?+W4D
z9TW7aX0>B_4SnbVRrNcF+1Ol6uGW7{8kHIq8ddJ(&KXy%YCrd_eLMkgP(D*&0Rd1U
zLTFH;2+`QMOzhUS{o93ox_0B+jeS@F4Nyl2Q#e}mt){kP+xA~k`fy%ZHc>axv{ALu
zTo5iL4Cq^J{`79=@jKvSaj~`<*e-6T_c;aNK(QfOV{Neh{jo-|Mzuz{M$JjVNySOY
zNqtClSQbv?U&AEHpvfT0pj*esAjP1?Ai$u&AaRW-LOo0|Of^hNMqNx%OjS%-Obw*y
z9+Mgp7;+tg91;?4Gl<VG$xq2|amNz=gK;N+r+lY)r-~qlpoE}+pn@Qep#1vlf$D+s
zf!aA72DJmt12qCQ0!<S2w_%w<nn9aEkU^0_l0lO}ltGn2)-D}K?n?ej`AYFh^-3Ol
zDSM$>%}Op+G|cG@Eh`l(B`Y-&Xp5zdRzYq<c|&nS)iVP+JS1E+d?cJayd>Nt{3tw`
z3{SdSkURlEb|JNv;YiOxWUOE(Wv6AA-ygay%TdN&#9qZ-z+S;#!d}z*vu&!dX)L3W
z!OX&5-cH<3-A>p}*-qL{+fMKroeix;h=&Yb8a<7MRo#Gb+^BI3^VS%ZA7CgwoEFKJ
zrpGv2xM37OW|TllrYL=r3Y=Rk%n)};z$fFDc2B!wJu#eHCdu(8dy|GrMPSA<Vq7r_
z9g|DYCmWE4!_;S-Gs+rsPe3HgkZw+uVoTF!oUddjDitMckOF2*Fr-;9P8)TPi6lsq
zNlGWDq1J5}1&v80D3eu4FQrzpaG2VU?$?Slh)PAL<1%qr+xPDmj!`Dykg-Wyr=bZ$
zl5VhYSlJKnSB`PC>9Vx_^w`>uA2Uu+s7}>lX#?78qxg;CB~+2EOEqVl(^DFji~$lh
z$vmVU($ASzZR~sZyT`gxWFV`AA#IqJ3^Ij(MLeM6(Qs-w)*oBW4&_%2{c>zLww@g@
z2py7(&_^$z5z&mSJGQxIfl@<g#57`9$uAM6i`YW<qIuT1YcMUl?%mHDBTg70la<O&
z=VEfRo}0^0Qs!6s!|xXH+&E`yH9MSNHDnusj806;s`1H{#tiGbJq#lT)tPuShnkhm
zYKC=71_?u)5%=gH<KPiN=n*vNnlyE4=5-SW0Yml?Xez>SeHfLEYQ_v(2L3}r5%v+{
z=n1sNYEn&q5$9=&H79D>&Fsc^8;7tWaL`j|tJPkUCqM7dtYO)(VqG_2P%;FF*hKfJ
zOp%5Xr~O-V-n3#;_j}iOh(AIQJ%lz|O{=!m^jcP{zSZ1ra<_7bBjOfalr~x&w}oB*
zdRhx(XtZp$I78Yc1D}=K%6;{&`NZ({*Yg@e?TN|!@8T$F%?xMO3oFhQ$42T<xI%R1
zZJT4G`Bh--q;!NPdJFBP+D22WiQVtr7p_`Y)9caQx}hHtCFs*M*%3=t8mq4!8tVv+
zSe6Zoz@|y;j3(AI%cXV8TIz5ZjoMjYf^^#?NyZqfqGj=#R709I)4(D0L}QvU(;%>P
zl0IXHmDo~bEwUcV>K0&VIlLCxfMvtf2h5*D&zNMDv`k*3D}Z@VHf9A`sjO5ssv9${
z1EVLkGTc~it==KvEz#F#>eVe9ejoZy@@ELLhFC?fyyA&kO|9cLa9BI^A4X29XC$ze
zSjw!w#=^56SpwE98&<6w27o1#fQ(I6kMm(o$EyI@-!`!s*0=qR8SA$Z7$Q0;vO1Vg
z7d}t@6sGqL7$Q4W1U>ZOG9MT41+LS-JLx5s-iP1|9Jg9@F^zwuzl6o{?pHbL(pQVO
z)8Rgux6@(YNVL;k>N0Jn+?8iuN_Ie=bXR5-S){9P<rCJVRX=E7XtRu`owFa%LX|z4
zM^bHAuzNlol}5&4*>^@F8%0tW7BY$yMapv+Dve<(Gl?`s%5xhojAF8Un0oF>7N`IV
zRe&VaOz}xrV^fhvCL;EX#2jgHYoE}KHc|$aX%s0$Ro1#MQmHSNt5hk`Rx1~@t1^{n
zES^jAPE~7G04LY(%Z-=I)oWCooukfAIXG6!GC5a^z9AkZAuLg2-EpSHeWLvnPE5lD
zbZ`5T1v<ke{|WEmlK%w%aLIq*XfblS0xL(AiKhnWiT$7OK1on)_$-o^xJXx~QXMPT
zA}vvAIu+1eq+&5ws)`%0e%mHrZ7N?bQ|@tAxv$K`=D61#8=opqO46pLChu^*Cy;H<
zTxR_JN8=Dj2jn?mk)}IKu{Zx8xOE>sx>p(=W$($(U|?*}`S2e8yLYGSW5ztS{lCZ9
zHuL`$liSSyTY$Ei{}V4LM^WeHcFy;wNd+46&XW?04ZD@*>ULG-c1`0f6^o-`_EZOj
z<zu!7l@j@J*fu&(Zf})TPUc5O%5FsxS*qs1<+4vVBI?D{h4O6e3K?gau5u@frHX82
z?h<F`ENr3-n`Xv|yOz`wPlMuwur!&Uz~_>*lSaMbAe&-ii(;R$w5?ho=M2!D1Na=0
z5p>ebyy;E5S-h<8O}_c=*8Zl`E<JD9onh^hYW=~pS-i@~E7sB@)k09R&g^!Gaq^He
zcR>)}e5rb^VDr@D`4vqU$3%^8w7KzP3)D0yCNDhO{pQ}@QEX9MarRPvHowR;f2;33
z=?hajLErl)WQX_l1C|V+hB+W${1xz%L`Z9NxYdq@AvKq5I0z7#U)mQ1j}e{&6BZXl
zMpDgHW4%Yt`Q5PpF4N;odGYM_vg>M#O|gtiWi?r1BpYJ`Wn*_dW16&5gM428Xf@64
zK|Wi0x<S~jc=QNS#Z$6-Ec|Lhw!z#<jCKm6RoCwY?f-osyug^&=2MO<)&9j{f-Ed<
z%ce7W(%E?2Ie6R|e%uLjti|sn<J7fc)RejiS;10StUg)GId}?Lkx)73dWwB(Hc`Tm
zGwk{QJ4n6>djN5C4Q3YMQWpbGD(8@y56ep)QJADVyKS=53Tnx!0$)jM7_HB-%T7q>
z2O!xO2Nf&YdrxV5scM1nGjKFB;BDl!Fz~K?dg^GSynaK{y=dU@F?>jHiI!kFFp2Db
zT&TofGWVuAj<?Son3u&|eourmc!I^N(SUF3)S>bbMHqj1rAdZJ=!N?tPOAc@*lYd;
z?7M}ZQ?L7%FaRrfhVSGTBI4#RntoPa;&sozID>b536wgAdidf2h6-6n!2Sgn^1GxR
zCObIpS2>d4cSR>IpSX8HC)}QN=_5gg1PX?{!A6vj#9k$;a(ns0e3W1m1z0Dhd+J#w
zgF2LQx+*1uC={GTIbGrPq*Qr>9F)_HlT;M)L|2Kh#A7p38OrO*bnb>-u%|3f$KcD1
zZNri47{Xv_R&hAWJiuw5&S{Aovk>t@;S=-2<Fb`$885lXpPd7ali{3#kz+e||KwSA
z=^6X<LbS$sl^wc`#m3;JIZh2(98+Se{7GhMO;c*Cf{Vrg=Q%)n-e%K6?xCKcsiRS8
zjg675t>czly@@*zG^MTiSoL_S+a)lbzqVrg@kh<UQE%(`&#3<nkhsejxZaKUj%NP@
zaB&JfvR9%p5)??bWfukPmoDIV_H>_FvfQFM`JwfVOrnH*L84gq$B>y3VDKa%$Sm$D
zEFAdD=T7xe#iv%Yr_3R&o`gcwDzOAyt4jV?l2{4SX(1!7E@1*q>u<souK4aPbwt{E
zwnzyM7-mpy5go{HPpz+{L5X3nkjij6K}O-~4_?|}T~;7is%N=^?{^VYMMS?!;>1lD
zAjHnOARqMiIethdOnp3MzxxU_Hbw!g6`94#r|ct4W61ef_djDesBQcys^9j^6J4X&
z5d@dv3y&6h?_yxDb9<kI2Zh4$D5XJ6ndZX%xbw-y2$!A0`0V}F^fHBKzvMSXbtKdg
z4~S*Ad<16j*Cby|dGT@s97B=jkvSzs%7LVfG}G}}GGD>nte`0%M!l#z?*N}w?W)9B
z>we_K<R{kw?%TpFJ;hay@0mCayD?=maNgK|xNP_q-k)9`A8^tYa1e^W$t-01f9&my
z1>mboJz-HB0*!|K8trh6Hl!f6$1tmCm&>sTRJ-olQ}Q89v`4?Vh_s%XENwpfPGuG8
z@YWGE5O!p(RIb*sd|%jnr>ON0w^^wC+LgDcdvRkSGf<HprhR+roI?JKsN%96FDjS!
zG{XHg<PZ32VZaWx;l)Dc7pTPRI(;BQgR~Zl-Byq$r{aiE8p;~d8rg)(RQmH1Q@d?8
zO3C5U`DpmX%;mLvfcJ0OmxGmR4hTEbLEtVKux0t+L3~x+fyja0USd;c^CYS6#QAPK
z<PY|0VHS2?u#PX1LIVPB5)+|gX!6(A&{o8<y>2+YB5Qf1a7INsPjOW`1)~o7e9hou
z9kI;IpLRHm%%^H7&;#WR-<`(>Pm{T;bzv_Co$h83sFay{X^yYYoHqgJyu~*zpZ;~C
zG@cD!v)YE&>sybvHWNM8Fy_S_X=WM^5!b6J$L)O{o}U{lfie%Ppz-et9k!e6VP`i*
zTJF`fUJnXlId(Ma@t|-U-9z|OEYWvo6|Hv`7h{x*Io)taqys0HoxoPx;tSnsLh|+K
zmJiov+KxZfu6H7Sw@i}e>nz<S>t|Q@J+5auG4m}x8LxQ%YXK_k(nR+7r_W#3r_bMi
zX923CneBg7xcRRF)bqrIoJ>50#B}U<jYQ4!|1j)C{ddF8of}GNnMp|&mQc1@Xmdkn
zcqA|?Lt{PE@4u}H|7=EDRKVVP5ThTCydoe(B9uy`UeGb<BRlB?3nvB{kJA8yd5XQs
zpdO-t6E0C{sfTYz^@+iDYe)|2VgekIS8Ht(JDC-qjtASBU2j^&wo~ySEizk~X-9ye
zxX42haAG+sO(LBmibOIu^Y}3{IQ}XR_h>y6D!x+-PU@(K0cU)`@^^Ozh!4Ho|4H^&
zN)gwKJ=0+Z-_+it7P@WP{PI8dpA7lu{;OwgQahQ4t6F6ANb_gT07La*EX&);G(b8%
zgSPIpapCBS`DpG$@<j4<a(^(s6cYi1A$%8I?~H?kiS4$Dk>ToXOqyi2F)1`LX!hhB
z>Qt&D3RB%0*Tf{=#O*mMO*G3dCoSr_oP`*av3w3@K!t=r1cgA8>9MFLGtN*qu{Kfk
z(-+8YVL`wY5YGq!!3%-I2&r+uoZ(i4yr(P<Oul+_LmQN1v2W02HhH)1A6$FLgx?3=
zmin^}H%*A9V$^RK=zoDI4*#EZz|~jL4F6UKtQ=<W{P69|7swy~eLpAWe_RFsYtzn)
z)RQX+SoM+hw5>-LtCB2dPVVBYtqVuau3uR?^Cz1!HD)Etj$U=@tEt+NVb@xOLMT7;
z!~A}ttY8N6A#~(|H5d?e3LrNa7!`=<U9Z^&n&zwTULjuR{CRnHyXTsIeDc0&?0L$B
zPVZFoKE#`rE84ofd3f0PmE)G@;&EN<8viHy-sk-Uen|EsayP=~Y1jjp!}{8I6Ys-C
zNByq*QNrtfFRA-_9FpLjbdOE{ap~~!XYQ=)ydo#cv;MyCnT_YVA%~s!wu!fUU-zkm
z{_)+%xwQNC?84?_nC`>(@98()kJk|$*GtK%jdcR|`<AeYD<SWEKOUK_p7)BLo1hr)
z9z5Qjo3_Wt0bTEui`BQQ`;UteUR5u^8$7|w224W_^&9fwYuWPF>+8znYYdOvl4bXw
zs5fXGDj&3!E(WF7H5cyHkM~E6-~GTVuP3j96yAHb8n4#7h~A6_lMm$=$t$n1Hc4;t
zKf1gq@7~ws#!)vHysvK$qBGN;H`L=je{W3uYmB6Ax?Xv&2z}_EN3PzMzUAe7xF2WK
zcs_bWUs(9Ot!Zi>PWRlLujkt9KgUd+jk}P4Y}a%j8c6wwzPG-d%Dz7|{C2-wxAA;s
z>$vj1epxO7W<E**&nA?$FGcc*95Gp!d0<i@6cZ5_qdY48@yL>_9b+egpS<Yy3XqMC
zoz}i9(T^A!w{!BEI&o7|5$^2DNuY~lb*;h}77`IONhbp$v5Z(pT9unNIpv^@;Igb-
z&9IyT(}T0zEasugzY!mR><eML4Ep=!tg0gjCk3z&s}|Af7M1q7e$-gEJCaFv8eR8=
ziPm*xHmjn~Zsy)J48QtSr|4{MX&?`i4o5wby38b=4!r`Pr4%!&L3&n_YMIpsL;528
zAtazrx5MmTcH=Dry}|@-_Z_|w<s;VXv*}_MB@@zdtHP~So>w*&CtGQz(r^?!zc3Wv
z7at0*HSpI93+Rfh-}lTtRaikBqN!PYuwM~$Qkbi&)=%p*mi@-Zc6Fq<M~~gS_xW{8
zb<OXXw={7}%~$T=G|fmWFwjuhX4H1YgqCCmFmN(zwBYX@+!eU^+w)aarI%c!U5qgV
z_c={LIW#I;x^bo15-e$(5<F;zVONu%^hCn)R=IuK^mDqZQOFQ>{E&AdVzbMd;0M&#
zrr8DNBy(u#n)EUrE}edxTonE@?U^Fbx~30=(kWl+%ui8uZXPqYN-lkf_Xr#U4XLwz
z!#<?Tt*}=34X+X&_c(NZBP{~<0rJO-dV_*6`GH?|Q4=~o<(LgWhWRSPB7WEeh^SFy
z6nE59LM=0DhjRW!EkGkCc!ap}|3dW(nreAE+H-$@#6iuG*%9jw+G!YdbrkYo<{PWC
zww$Kjxl2-GV$qaS6LrJjk^hF;W9j!}=NP4AV<&7HNI1|u!Wef`UZ?11I@IvzMvLh@
zWv!kSesfN6+TOFNgbLSQ^I}=AGl%lH8H7v9ROo#N(<t`7`6i6m)K?&jm3Z#3W)9@g
z_a^+7GRu4ff$>^5z9o^l_ijkao}w>5Z-dS1qlq9ozJ<5$nRrvm68Vw3d=H=ZJHS@7
zryN#;3yeD}&&D&lJZ~~3#1AaD%M%ofwn8#*cXz7fJvAlku@;f#NoW!H!>acM_yDVx
zRV_+U79s5w(8`1Ihb3?R6v+CFomFkwZLC#*w0m5X3~5L>>E<VijM-e*&i<m+f{BEH
z&fn#Q%HQo-O*<Ye$;`{OMzjiBZfgZ!H5FehJE%h(T5{~Jchzg1q*|o+#Z%&VIJvh-
zE}PGaUPA53nlj9t-Dh!6;1}sxc}KW&-(2)o1W-;DC4a19_L7m=RdqLhg_D|Niz~3o
zqJ7oI&N<mKJz{W>7GSFD(6jMNvd_q?)IL{k{2<@<qI*S`mOu|I*ASJ*0$d|E(weTY
zi0Z22*^#!*ccAufD6~u5A*8rW(%0Voeq|XX4{W91(D`}+rGcl!n3xBLx0+ZXsP7XO
zTBWuq7I`9MS(Oow^rAnTkAEv(2i0K87D8w_b5Az>%~cRGMTl>27ZwhgaYAHInAv4%
za?u||h0oQp!a{%2X&=uxdQlcP<r8H0SVmQ%^73i<+BV^@mAXuAQMq+-fQWX1hNMjH
zsk5l2{%M=o$&ur@^|x2cKjQxQ!Oot5MBsRvV*a!IN0RW8MK!{gcF_3Px|r2&9S&Fq
z<v-#ogQwep_7?E%N0HlVoZ~fz_acgIe!>QCby;98Sq=Nn;*mIAR~Fe>3dnrB6x(UV
zv~90p&6+07bTsH>Vx_q*%yvQFsbiZnF0d&)wziBbkX`X60%s6M)TRF?5}3ZCK!X{9
zve)5$p<PqkG>f(6%IJME)NkwaKJdkRC)AycF=Duimbd6>No=G_e3P3B+n2TFt0Ns#
zmFa0AUDioKu_I4>!(s7sYy9b7HfzOQDJMVKhy&ua<t(D=t+upA91WLrNq1bad9eiv
zFJ<sY=77l?pdAV0&7X9)>T*#{_-M4;sTeH~4Miy^8jqo1Sw-V6KShOIYc%3OUuz?a
zr5pYIeqf?HuvszM$)@kas!uxIhD1KmD_UsCAj!tT>rvB<U4B*S6FkB0yS4xqv_gB!
zt&~wu>(!bWs()z@D^0ZvXm#PKwHB#s0X{htC@mPC_#M|0F82ygsu2Y2JFowq!Jz&w
zM}E4)L(A{$<*H4|@?^=qrRXHqzU>+a=uBe~ojIA8KQDU2Ait#tU^!8#o=p#MNK%8j
zw2Zc=_`eOI%&?fMm2CSfXZJCiPYE>+SDh%45D-6-!pinzvAfJPic}3jDxabrJ10-*
zeBa{bFx75fe+O4LZ%ox$Cnxj|Gw(9#h`7O^5FA2ZkTP&GGnFB*l6F+OQc2E@lWsd)
zm-vG`t~s;&*SparNzi!70G@7xc5WiaGT;v(v7RMFbHuQ}<noLlVK2xTLOu8w=|J?|
zU22!{>M75w-d`ZKd^$EFGtN>nwt>bLa84u0%ToH1Xx{sL$MT0P`woK)mNWEnBgVJR
z#JK$83T^Q$4eCsskWc|a59)6<?1nNc&c>EhF{9=UobcedRWZRNrqE6h!#`a|$Kf`r
zF2sw?MjeP|b(T}(EMbsr*G>r;)+@(hA9aq%`ft$UEL?s=X0@S>*4tor1dt9m%AeVl
zEpdWd$A6FZoeX#4YzYJPi?Q9u2zz^W(^U4!t?Psngk?ZbGC;L>O3@J;7jv1DgrtWU
zC-rc|b02(#gH|os)P|MDIQ#Q=sTf)^lV=?a_B#84%%d;+;CA4|w|=rlyHeJ9&5(%y
zmz~MPAuXn6n6l3??=n%ajNDha^*)szz3QWZYxazK9OBT0R1q3uwJ{uT*HF``U+?%h
z3vX)@bsSu##VPo7s3(U`5-sBx6IUA1w9#gj(OGHh+eWv7dv&B{mH6UudDlN}Tc|7J
zATTRkvx1W{9={Z_7Dw}!{K!{i>S|-{Fk6U%)a*%*_DXY=ptkb-Uf1k4G6>7x+0u5U
z<#58EUq|mdb(y}hQmpUSc%4--w!?=%9#BP4!C{Sl_9TaTi;8l6SMnSH8=J-#LiRPC
zGo}sh9H#{h5#&TS6j020DUgb?7`Z8jV`P2!^20~7+jp{pux(AlHh{qC-arbA4G`Z)
zwXrcney{!8be<^11%ZipTB}f7AMa~*CN{OEZs|#FZdJTxcMfv7H+L#j@X70`qk>vC
z%55g2I!Ef&nViTRG&Rp1tJ9Vid~&C{I2$v~f_dOtSzJyvbviB*cqfkzvByuK<#-53
zc@3YW;jp#?^WW_+g+e)xTwN)xNPlW+{eNtuLu;FG7#LWfzOq|>IYwWUqyG^ByZqf8
ztl--s0Iz$5$?++66}F(V=_KQDu50nsooxvBB7CTOwcrLSl!GNpg_iKU<`fR%jC|l8
z!_46dP^F8CUZDhA+<cGkhcr%zs{};iI**)K<H)X9cKU=P_KHJ<LJQ2r-TW;oB5!})
z<q!F(s&{Pm8O%PG@XS29V~vLcrS6M99AIbPuS|+Yn5t9+@hWzl^4`~=D)0<vS)PvO
zI(X$tr-f@+Hr}Ff!~zm9!eXn!yC11Tr)NSIjV8p<W*&{F6678;RylfjvRV)oyM<wq
zMHSSkAQ68nj~v6oJFP5^c7JSHBW%eG-&`InM2{MlOC4&ku9E>r!$P6ux(n?gvF$@x
zonRCxWvk(Cya2CFY}YFXg$`UIB=rCkd-L*?UEu(N=y>57vKp-teL4W506l$gi3<ht
zS;W&T%{@iPjEa68XkJX778`j?RlGla-pPxr_fUbdVPXn^Xw06ieI%|cM^X^crM9~-
z)zv|ix4OYpFrW!RiH)^aZ+Vf9j!|Xtlt{a0;Xf6X<Lk8UaF{RaNdMC*+qrP<IQw-R
zt$se3)x(1u|2szVdL0vd;HLAd=k2v$A#MYS^Ol|0utz7YPf!|)+V^DVSAm#0tC37t
zrK3OQM_D|yHbmdyY(&jKOW(t$Z`6)ACN*BEggN#s>(_-MOQ<QqwPD#@B=Mq-|5y(s
zYV)J|dWZ}gkYU?T$X`_-%9BJN0ZFZWe4k2w&VTo3UQWp+^_p%jrkXP&P!M8KphN--
z=hb?Rj>eb2h^AIh7D}N>F*$q|&V*DX&FJGXdr;2@YT*XxE0!7Qa%Iza%q=B5E}&<p
z;+@VEaCCj)nOTo&JL(8o^kOvMy)<@a5<pVIn%HE1cem#pm#w{jT1j?BtDpcOxuBF;
z0uM%~YzSaFHnvBbycpEr+DfM`EWMbC-xC4xs!G(BO0)#07eG9Cdh0bK+^x-%j(9E}
zF4iQaJr7?OlhRa?)^W{DX?;}iZ*kq>*!cb&KC6DC#2iQLgNnbna><ENcjC+PQ~4g!
z^Ad-_^M~?R2&FYuLBvxXD2Obq?a@dyBY$Isz>*QXcby!&2+=*bUF&6tfr#i4pMrj`
z=6;FBSG65tSg@k>516u|`dr-kCQq)43}@8_`2JCYF>Y}|lF+kf^}2@sisvI3Ul#3k
ze&AOdI_i<#k6upEOWGY?#w6WlX=8Q$YQdgygQL?<%r@Jc@74IqJRvQ{puf0z`Zo|G
ztKHE8gP8;$mf_t9d+rAwGZp5q(oenLl!Tbm?+1_#F~@E*xanU>jLsk7+)T{z@?|NO
zk|kb(%#z|3l${S5St0VmOtQkZ^STsaXp)rYo$E$32Ac4jn~N0DHkhrjYRW^$OWIW!
z`i!TIa%029tIxg28Sa$7sYk(?C&naD!0(eit<3(GSFz^QpPzqPs$C{uV>VC$)T2~E
zissD=_khxHQ`U7ZRB2Q;_R*;=m4V8d+LJia{L1d^-A(}}`#ar*)SYJdfW>bhf0$hk
zpALBSKcA`;Ibf`DF#?nonRt_}<e7gP2BAL=N^CONc<-tn-~O#l>_n>X+W9k+r3Crk
zoqG5$+r+dJCIJxgUqp_kq$Ju2iTDi&iHL|47V{I4G~PbargW=}6bu&^z&#`-!%S*#
zp@>c(m}#uI_)^dfB#ozXyE2%LvgsN+%9fWFq)y6Dnz~$RytW!Qmy%rM9{`(*k8?@S
zBbGQhvKX@=Oefb*7sWT<k(S7j^I!<!3BBx4g~i^*?|A(+tK&EAx=B=sYQ#>rCH8$(
zdx(|MPXGLVzwpnq3{gUk-|x$_bQ$T=OLKoL+gYR7*IgsY3~sPRni}}p3X`(UsO48d
z55BC2T0MXnIe^#-+oA_<J%E=Q*kSvvI2ZlT+7VB848b<^UqbKDl56JdKBm9O-hn39
zG}!|VuK3e?Upsv<u8DN|C0()T_Ru>0Rj;XZ1}v}<E`RYjwaYA@aCXVm%%9zHZsy`R
z23XHXw~Mo#kZtB`I!0K}TD42JoUm@@{c;SsoWZ$+>f*PViFuoHfcYI$YGCwtEa>VZ
z8+RZqCWmWDz5VQKkxAEvH^l#%#MD3JMjZa^ppW=`{Cr<<zZmH|>eCsx80wl?8yUOm
zG6I+Y%yiZ^M#c_w^mLzdW`>`mf5mA3pV+@<ja}(<4cq~SK;!>o=JN7?w$5*7=4i(1
zrl;(tt12X>X-xco8|O7X6}eqgZ(Jd|{0AkVU}gg5x+!ibpT^{_%_%nhh;-xA;E{nw
z!81~lGyCvAcPBu3r>*LDH{-eKYOMT-QF^nIpa8gfdw$ZEFJBcUe&SAf$t8L03}^Ue
zswRUYANq69K^2yPQ@P?%VfvpQy&8nlx3DVlVC|EYif?$y@mnxw<WBAJz}|-rfJKeb
z<(gw{yu4pR{9D%U7%@FW>ho`dh4|mHe*b05{-nxpi**s)o~siQ$>vWjk%nfA`E~#n
zHKIt^CJ+|hDP^Iuz>3QtIfO6B95f^q=T8z7$N=*)8`X)*x$=S;(6_{ev@~b+2GKlA
zzXHlu7>4@vc~jHh6m%g!!R2V)<8h2-GRf+G-V!D7y1aaO%YGZ;s{JtNvQ!&3a~jss
zXg*UhR(W(3(i}_oaa_jxC=b7U{d*>wVw2af@bdi;uvV?wP~8SEGs%>h-@Rrnk?CP^
z`bg*f3Jtg3;*<C243^OZ((biAW#{1+`YcSk7OF3C(bfIv-G0wILc-;`d)*-(5RD-`
zzKghFXL&0baz*GgSUX^ojMh&?Fd56LO5%P-y%OkM-nI33<`GHxrWTN$;GZsU=j?B!
zJ6;+)U{ddDPlI*f{%8D-`R7Dwpmx5>8Z<9t{z);`yjI6$BBK1QN(y5Bq6M0|2gr6K
z-72o4*@}$H`K{bcj%`FyYQ6n1hGf}+Li_7&vDOYTKQfgw7ClV_F(Y0UCIQa$b)Ojm
zA8sI@QtlU0@8u6L5`|tCVaH6~>~F*M>AMihDyw|spgz*D1eiD?pbvO@9HFHbQb73K
zpaG;838Ri>Zb&XqIt*VA@o*?%<WOJ@jlR7Eb&VqNXnkM{q<Fu7&zA(laDTQTFkCTg
zb3uM<5GI6TfD>HX(4{5|A*PK2sUIcLm@7A6kBsBBCL-*wyX<X1oG<Ngj7QoZp2=-k
zuE@*TARpe@6(Ek~Rslw3c-@?FLcN)Qez(x35Tt)?P#VRr%3a{E3eS$JCR+bFbi(ef
zsWpeLtft30o2*tPoi4DP0ld~}<7aNFp}HEMY(u7ye%T3~<}#EN1lbx#ICUa-l*UKn
z==y%^M`1fAwq>?g`raby1%w9szVfwmOhI_l%$#4W9D_do{(@pLNIPVVTFkfv5zwLF
zq?I0N<vwvwXVLa-*nhWXCfjGX=$2ups#GhQ(2gSq@Lpl~A2kA)y(NM-w>CmK!51sQ
z-Wz`}X0b&WLk6^(Bq0t2hX$HdTQLVqdZ-6nK>k<<F~8B%C7ATciC2whn<sZ0wT^$-
zj^{I+HR0sE4H_&V^SEs_;1%r3y1=)e1u1J;+-Yb+M8fpf$?Cp)paOr(Gz=tVODH?d
zvxq33=#eW5a;6lw#BGXNRjTBPG|vSbw}iPoW5Xf?!%fE>P8}#5$TWopPH`n<qbKw7
zb4c^bF4tOJeuqQdXc<;<E@pF8955#(p;y+->cCPBQ9N|ed8!(&3}glFq%|mGwHq<R
zOV{P^fylby*?e4EHLopPs2?E*i6CORNbz1Hewq5*ZW9D_<@ZpwIMEChC=!@HW>R!m
zvA9S^AqyKq8o;ojsLzlw941M)YVV7?rGNh!H-Vwi#=Gl%L;|h=|ECR#eteo)mYwcr
z{lE*Ci}hb$z6hoj9&P{Hx6+`63fe+IbKwV_by!UPqmiNGG4KD%0;bPngg-wS!1D9M
z`+v&<4rc#H5;*@4{glk~?2L5P-L#bC>E7O6%zuA<^);NKR)7J9p242s0K?%g85#9o
zXlfx_6VhbT)1;!qqJ0e5Ck#@uqe3e7BMh{C4ANwy!=r1yqKI%mTqHqmc<umK2Se{r
zQ-GBJ{HW6Oh8o*gZzvt~IaMlTMOk)PHkgNZosNOLJ(Olpka}lY7>t^WyqS(T5THf+
zZNf5^(sJmjNO-KQ(u|{M2uq9a(>w4y7|K03*u6OOXVwsDSlidmU^3ugGE^|x;qYN{
z^2GVIxU&10*^nG(5~2`>3FG(xL<LC?1<MdE6AO9K3T+PsTb&Rc8!u@RXpDuOgonL%
zaq+<SQ?Y#}*WfS1ndP^CqZd#nsBh%Ig`N%R|C{OA9SR@4tFs>UU(CGpdM(E!wTpPC
zuziyz#~CF|M_U9w&hf2Ca628rHy!RKg0MewT<*!~XNG;AKSrKEGGD??uD?|J_jF_U
za`CXF?{>`{a?;(VQvs(Y%l_Ei;KDu<-i$(-dnigDbf{19Ql;KgOyASd_Dd5{U_x}i
znwLVS7lVhk@*yc@zx=a+a`2ccsA^S^|F(b)kv~s~p}u?xLH@tx=YK8Wf6dQo(w09&
zQM1#u*E_^*-+e#5Cx`u*&Bfc#a#o<^abeK;W7$dghQRh#EE1PiMtNbB@<dQg7bTUc
zkZ`F&zlY38R0WQMQIsp)D>YG3DFi8%Q7L>)k-(L!xtqL<<R0YBLrGeRI`X=Gm|SPG
z+45k@sOmBHP`Lz@L}nr{G#|y_gHjw$u8k<ew$mpkUvI5uAB~|lpPwGTk@pB_^0z05
zzmVSs6$}OE|3>tBhWQlFX_}(sbVS3!#+)vBAa5`(*!hDFNShf=%T62ksi5O~_-Ctb
zfg$(Ab9{lO`(W7X2<jP~EoVxKQkQ<N&W$(&FdRQLCj_v{{KBtgNYmFVoUt8Z{aya(
z?k%rDsU+eo@+hL{X(MjeuitZwg0Cj+UPEBO9`(Fg11BER=*M%sn<l0*w5>kQ53)t%
zB+15ytsuIGQ19ks(-?V)VjS88MDC_)Cb(sLf9G}KE}IX#1Mc#ch-uK>i4ee0re`UO
zHd=^57f^`6nR?<AZW%T=W(Y;5c1W{|#|BpnxLm@-%%q7jlFI+Hd~a7Lv9XW@&#a(I
zW5Re#ZfnS`#V3<^G!L2Yr}?S5*MG28NfM0{wCl-3?$<*#p5vkeTIWugHZk_MqN_5w
z7*YTmp8euI+ifL%1aeS^5U7@`@0s9{Yk1l~o|hT{Sr%uC0UN?!cQMLo<ML#GwC&3m
zl)iscCtIqV&TciBq;k30XH3#mQo&~50#H?0>rIAN)P`{wo%0ncz*SM_mm_ds9<Z#8
zl}60+h%7U-C(L4rd>coI9AiulN(Z&Raytev)ud_<KVSzKl%p)tVs1_xM}Liwp>L14
zc42Cdx&Ee>h{H%p=oa=MZvB<b07VkBIlvrOOyORBlBC#ztgZ)G!1;3j+a;amxz)+s
zwElRoAsS(nwL?%|Q)69)ef<|_N4sjZ#%9Cty~LG<i>^;cpz6<wYK(8W!2yy#S<cU`
zY1RQ@OfEjZpW@ntZ9s>LR}<(6hD*644Npq2+3{qGtlylv?(u-9YQx6xD?WHXv=bbq
ziQV~@1c~_u1~C%7P@i|E>7ZY~K<IOn#+cgAh-S!dLlV0tMxL?@3Gm>O>G$9kpNEo(
zf+vDwuyl)m?gH{?HA~NoPI(d}w^I#6ND2~X98m9Qs2I8tcyFzP#^&^$PgzDq&yev%
zNEH3+#U7zoD&t=>4e_id<^_#Q*dC4SY|4{;nz70kEutaBleR4*l<o2UM|Ec&57qw0
z@nhd3yU5Z-Qr4_d3ZoE%B)icxq_Je*BFnXRD_dC$Ayjg0*-6ds+LurlB_hQ&k**f&
z?;J`qay8fZ=khw1Kb+6|JkRre&hve~-{W{%J)|8Zx%rO!HUHxy3BlB4{=bo>lJQab
zZzC1G!u#B>U-^RpM?=fONF9unvbJEQ8@D#{X2U&R9t)Nd(o3fw1XPFJ@v07xPZQ;3
zok9n-{6)Vv$z~E!moAl7g$bw#C1a%0r!*})PIs{V7vExrD`tXd=EA;R!R%ax;Wp6=
z?26<VQE`D(xn#SdU^$lJURd&r;@+dlWW3ENa`_B1|A5Dtw%lQCnzXYL7SHC$tLpdn
z3Ls#0x+In4&vy^HlMd_|<RalSl(`Zzq3>E7Jxrcns)abrl)p&RrFHqCY7&1W*HT_z
z?x`or^yMwF9EHV5yKNP`Lv#fX>nwkDpni0eF3_{%6^s=F<2_bgok!k;$jOUA-|NbY
zhwqcB%X*WV7ilNj!KdH2v(GAN+hV}+hH`N))#0b_1mr89-Z8#p>iUJpVI;8MN6)!X
z(Jon4m$B|b1d5j1<>cKvr*?Wyc9NEtb{-X)3%a*JZE9Rc?k@1EtQc#rjd**#87}t<
zJLzM0a>{Y~OiXXa@14UX!u0f$IUE(L<pA%!kh)CwTigq52YU3K0p6Rc$=V^)g5zDc
zI|_J{RD*r^FVCtuXI@rM!_rx*25)CDwr-+5ti&^ArEzdkAt<E(J*VP>SMt_UpE8Eh
zoFq)yEY+Ppm%uVV{uRY^vPMdolhULQ?P{rky?B~a>9<)were`Q0TpSBoD8!;Q8|1g
z&mV2~s+Av!x>Z&#!x`mo&gVO`YX<qLXpUt`(5duDr?r8-4SOftdY>laS&?+1=TFfy
zWo;ka%RR@PE8BX&{PBnTvl8pU;zkCnX6ihr73RT|1+Nn$?9{yJtnzfrl9=wo=hj;M
zvMbq1n`djvF7W2Gg=H7jO4)T@`oB<iu?uT}WmhBRX3MSyeTfB<HB|$pcR&PYYsxOE
zEiSt#p|Yzuj&`~15?C#}nherj>F6B@fJrcld(mi_J(A@DWWkD9?N=`E^y%#)(?W^m
zdS48wTSvn4RDNgP)mBBH_mZPkUvHu1X8P#aTW{fw@X+w0rXw?N0(lsE9`}*CvUbdQ
zOFrBURcIo|!Tw1;mtMSUXA1bw!$##oOxyhqBd6(dOhX-Dn$~N~H{r6tg(U+3J$j;x
zF35Zv7hOg=_S`Vm8!D4mkjJ<mFl1?>Q|PYdnLcpV6V0oT(K760*Dif4*8G&0BdXQ0
zxH6xaDTYlTFo>M$9gK7)fK^aXH^=aNj?a<2!EWsywQWLFW$M(qx25-24&RL}NmYxM
zzE-l|X<%;D*J+?C@dG2aQ|-E$?083pbEk#5i<z$9Q`&uXraigyzRy&u2LMlA<=KgG
z#zEyS-s^qwbu(xE@-?v)Dz;x93f<Pdp2+qb70MqQR5ZbOD6F<-wnsR;Z~V-jM^cPO
z6HXL4VlA2-^=mmoio0cgt1uhh;aoN8SX=$@{Wc9#+x&`w)9}0EUG-B=_g%YgJ*h7I
zi%REB=*3nvWjFGczZ-p5KxQ{35Ns%)Wz`2RCcbMhD>a)-8`n3@N;y%)USQHqrKdh;
zEWI;ODnO4r?^X`tYVOzx$|P}7rYS3%q34PKYmFSx3m1?Q>o?MQ>EJD!Bpn<tq}5_n
zl%FV{`$pa(%-qw$H|Jexet%r9cc5cs8$;)b`8s5Zn@E2H>7Ts?mwPpigtG%}=k6*w
zSDer0ynLcS^uR18ooAm;oGM+bWU<hIz804ZC9nr630J5R9tsIG9}2P=slQ?H<a1bD
zBKifTF)A8i7tS#nm~CAgC>5H}oE=qh`3BQyaJc#i^R1FBX`n52X}kIZnQbCF-?~w&
zU-?4snw7^?oL3YwJV5<X^vI{KA@#5lL}Jicu}7D--!_n}-}4!!95@I!OI~2^GUd8T
z<y_FpSj!g~?7&yHSEgV*m@!o=jkcjE2bQz{$sIa_!_<P+JtLkP0(zQyds{uTI_L+;
zSRe0;WFpON_$Z%U`DnhiO|C&VqJ7&RY9%!upDb^dwTJfHhCkPT$0l>cATn=kEWL$J
zVjG`=Ld-$8&kb+2k6Im)lk=ZfzfgiUYdcfL92$S;Wp;roi{ea#u>Lhq@AuEpvFMEX
zW0o0~RxUd-ZeYcG>`Z9OoOhqIwm`hcPLOd)oPZ-53tFV=jn3x@97-~OS7!K^plQZ{
zp@c$h&smx4)=oUR{E_Uc{WJj|8_st$rK*OTR2>-fok2@FSz6Kr<@#~=<_Kp0`Ql?0
zlDpO@Zh<Els|0`1E`wpqj($_C&sFr$R_71X%$h}&ltQmo>6c5T_2{R@_pjUhoz`zr
z-RBSvsJ_0aTi<(Sdj8Do_EFONF!<=SHG7eB4PvBM;hN9!20qQ>4G$FQ{BMf;U*P{}
z6cIc3rMk3?z3|1P;+N0ZPiez8Uik*7b9(@(fy48Qy#hk;n9Omb7{khV5x*3{o>Q@z
zV{@8MT9U5TIA@hk3so^p)~Srhez8?%WvubGJ$T4}cKBKE(wa1*Mi(o;MYm6>Exdg?
z0AITn|Bc&c_GpX!`^HcoOyUH;sNy|Nt=s>qYf->)wi(Ry+GOOc;0==+*dF}mm}d^l
zTn_l#uV6b#otSo#E6UaZ<>=;WC5!%9PpM6RrI%#T0ydJ^*EEtG&Pcb~pilw8wfo>k
z=j6lmX-uvc41|738kglcl+~E)otrJZln!KKRdxW$6y25~`F>)1`eGS6O78F0tQ4sD
z$nY3#%PhV}t7UInBm4@SYFtEXUt>!<QchZOFeK{wHJj}4cyHFE?1aiu$|NOjUlYNx
z{~fLHnx)|FNSG(xkwFhnk(FMy5-n&K0H6h}wEV-@FKWbC$<xKo4JB^xgj_Kai;e#&
z=y3O<14zhcKp!l-26EAgIOHN+apld4ySZ39y4pM0N^ImBeAS`uGkkkNY3R@)QuUj>
zn6#v%xEsi^iXrgh#j8BC=b--f7oa$}WglSoIOZ&SMjqy%FR!*=OX0Mbm5I8hu0DP!
zn+UyVgRBMx_pvP^7DuAL@9<v-7Ks4J94N^ZR;tqV0|tDC>EEx_S-HAdyP;P61x1L^
zkB0hw^-zd*#kx8|0VEFEOO*)4WnD2a$+|5&=^r8#da|L-)I7#GrU@Xqq?ZIfIUO}~
z>XeP8*13@U=o`P^XuMxq*q!mJWxnbcF$6=~7m8eOEGAb-DA9WiQ!5-8KQ6?PC43EF
z!`ueqKXWD}B%192znU8FFxCyrY1rGUqa_yE4v&)15^HEYuDj1eG9gm@WJ<JgoWTU*
zPnu{EvDV`>?w=Z_?@CX3(3MVkdWuV&m($t1hsGm?RG-A1`|)BotNbryNHP}f=9k#=
z2nq^*s>RREWzR6JVFI_?qbIMDFg7n8#W-`cRhGd##QX-I>{fu?vGlul4OX`xS^nW<
zUIprlFk0<tXv^TD$Lq_@ANY37U1{?0(ZX<B^nG-<>yjRY@d+zUsVJ4W4t*lcveH<3
zEnaHQ(Xi<C$`FNGIy7AUAdx$YH)$f)`Xc|Bky0`rXVTKr{{8bPbJD_#Clj7&E*0q9
zsV|n%9rJyAF7@wD)SSXt*PGK}%<YDMoQzesZMh+f^lmsedbP~tgs?WIJ%Z8Bv_9>z
z?AUzchk<upCQ95l-Ml@yWXFpq=n-+Lbw1rgMeQ6n?rKox$g1==u|&B9roX3alAP}`
zFfh4lIzMcScJ{Z^cNu?a(xVlUl?#j4GV0MPj}U6)dLN;vcv#oL0XzB9Awn@R0+WlW
z^<P?Co;aNKL7acxV?lwF6DilECE=e19JDO`J?+th!3-Tgq&*T+Ub2-zM6pJJ0^kIP
zHT1y*V2?oGmzTd!8d3mcv>TslEi4<}sZ_q5YVAt;@~MA9x%SlDc&EPnPp4j4q$fmR
z?agN=p1|<N0+L|*0L`=R4d%P0S=Lxu5KrO&m{K;}Xt0{7K&5q)6r{9LZ$L_67TePi
z;5o~{Lt<4405Ss;2voAQcRFotFDA2`p*9j-KN}0Jib8>84NiSGyu5GwC<w9ST4zb&
zN!1Z5g}Y^AGqU^fWcPwQ9tgTZTv@^*XYKVKz!Of`oG@-Y{bS^=#1o{SLR*YRg2Uhf
zc&M-HL-9lpzNldn7yea$<u(Ym>cHFJ&&SZ(GEH%n1PU_j=17(uvND9%sXX+wzYi=(
z-f>|kJ<0_DQf@+xoNvhzG6Vu#Q`>|gR^+WtdyCtgSup|u6rRHSVo{(gY{3JcLdBbC
z3a@2X5I|9Ct0OZH5)^|Wh)==mLo9_OxK9oY2m&aI)6J>nK?<cGY!T;6B!wgNpdeus
zHEALV&=kV>2MQQCW{8RlM-PH+VasUo_$-w2!Rk~6FFL9Gh@^m;>4cd=BUV+GL=6Cv
zoS?mvxI^TvKXDLN$K8=AN0h82^bn^Ll29Q0_u}9JIo59-(DGYy2(u9W3vO_M<Q{Dy
z5cIkm!hrDKihv6g)3|j&%kM}a3<&>8a$KOviLC?zXPtxr;XjLv3v_65Yk*dtOePEn
ze}^(IQ2EkU0uh{h@b}H)0u_^x5R(DF?xH2kEcm-maVh>LC7OcZ@r1v7440yUibx8=
z^DzESDO`vXG{iuB+c!m+L-4m7;ZnrW5=}v{L-5y?aVey>6G=h1L-5y7aUq(RiGujL
zqDq)U@RuTSDQH-UrXbiM_$w8-6z4gKq#)cO_+5Ei2$~&4K;ZT0H*(7QQ)e~2nn{A2
z2ol^rI0*8O6TKCXwbLu*RS;-;b$4?h-&)K+nPGh=!0(9TI%OBnrcAL)@a+ej{`f}T
z`M!FC-}1&q>fR!xl?M1mH0#HT0-lQyJ2zpVf7uDZZ-C?4;t3Jzxb5(t2kP2>pzx{{
mpozE|xB|N2gbSxj3Tmy}d!zs-^8g?Ue(quefLqX-D)4`53x25p

literal 0
HcmV?d00001

diff --git a/core/src/test/resources/indices/bwc/repo-2.3.4.zip b/core/src/test/resources/indices/bwc/repo-2.3.4.zip
new file mode 100644
index 0000000000000000000000000000000000000000..ddd92319d16a5e98e149fff7e7856325168535d2
GIT binary patch
literal 77302
zcmbTc1C%Arx-GiOwr$(Cx@_CFtGaA=*|ycyW!pBoY}<8v?{m&O_x$(1fA9C!$XF}K
z8kygiIrEDd5%UWLX%J9oz&}1HK}X;IdGeoMU;tczlc|F}gQ^M?0EEoUSN$K&)dL0q
z1o{930Dz+YxlrMM+-|KE6^8le-+wgd|GM4M&cxK??_zipJKGsLI9u4eI1_Vg6T6r?
zyXZ3!>-zl9*#C*@pFeQ_TWm{XQ|JE=5dZfuSbq)kPpEXv^eptO|35?G{9Q=Ke;M>2
zltTXVO&r5zgvo#c09bGU0RG<x)z@eIJ9;P8ep|1L<91)2kV^kbJd|ksWj)_Uk3|C~
z9<gZ#1>>5!P*rHlYm^cu7-BU(C=(w@8W+q2`8FHVfy%w|h8fhm#0$SPXZr#2YnE|k
zJV$8=+!(<}!*El?o#F(S>(?H?%MTXw?5@`>2?C$X%eRl5k3rtLFQZNyjUh|dAw8{M
zXKJQuPcEX`qZxiK%XnWEk(ck!XA-IQ`Hc&2h)?uuHR_EutuS(vELjCzYj#pu-qxp2
z48HFW(Cf{9`A=>@nN8!mJr1Xw{6eBs<TN*;4W;h-`d>ZUpZQ1dxV-o8J0$%QaYV=W
zQMa6IAEkpHupLHg2OQF|hDosIquJHTe6Ofi!ad8o4&JZ);;9H4K{<(m8H$c>fhPK6
zWk34O8$6t7u@1bh$L?7PC(43#3)I#i1V9T;O0ed2+Aovf6z|kh;R+V5(KNls9X2v-
z<12sJl2f~VR9Gr-3@gj5w;jfjE<01|g58zq?2rf{QM+L=(ngUm<7HzK;7s52TEYtA
z1`Dd>0myupzko=Udf3EVvIKGvh8i+<K~&XN1;@sF$s!UV<B84ufHUHWY<%E@BJT%`
zKz&G=^=$IO^7u0#1-nUx!ik~>gKKFGoup`Ll}Sb#f}24l`vSWGiN=wE9D_i(->|Jj
zgzUyKL6n1Bp<4$pwb_U;?Ul#^sE9{B_~`e@x!!A|BA&hE?}Fm}>4xIG)35m_x1o5W
zFK0vi1ZG#vaBQ{;F{&c#=S&kDEQJlbL^nl21M5Q4DS4`P&3G!mw^ujO1<s)pb!|<p
zId^6^J=NQ1x2WiK0_FDOwM?77@=*`g*ZSocvxN1@Pw4$BM^1&6uXTaeB=$o7`D7Z~
z&}a7~=E%aa%<<0HQ%th}+vwC=v38Cr0%MVt%frq!;1?Ju@+}U2hn!i56_+3iA{>~k
z$~(QnFaG5$)`<iAd242}ZFY-c8EUFpy|M}IIJzJ29a88?D~Q!sDs*#eBb*y}u@dO>
z=kLXAjwn;mpceCFxc<=aVDlPV)=+70&7ccVl=X4e4@QPW^KJ#n>aiTFlrEE&u`h?Q
z0;aPjoZOEAqa`GM&#gwh!d-cHn6|SJRUPYlEp3o!$i8}c{ZDUHv)^)!{mD5}s;={F
z;>srm6v`spsU^+vn-aEFYWd>7=7Nr!BivuHp^(fXEykQrohhBkwMF|+ai!#AC-Vz(
z$@0rD*IL|vM}pt#7*}&I=I~Y?uqG#?SJlqyK~WD<KDIM_S2tei&ko*6Z&bx<Ghv01
ztuNRcC+~vc@bhTVzOi<vc>*0E2Km8DhW8%DW8rtVO%T*s&`s6sN;_DnOknYpMcHo4
z<}MwBBxVe11j&J{IYZ8Lm@MU?yD#aPfk+rXfuYqZu<Lt73akWk-3rb)Hq9!}$w1i9
z|HkWX_v|ki#j?V$8(8;I7P3%DS497-=!?NFBCg=c#MtFn;2)Cw-x?KU_{_2l1^}2#
z{~H>`^miI{q1|bPs)&iVS9Ak5z)=Sq*u*c0C^S1KTmVWc%>J12*d@Lt2%6ioG^E$7
zRs}a=b~XLd-k8ENcxh&CcGUjcy~}y3lg0D;V)BA#_)0G)D|<u0>)p-S(d*Ic?*09K
zcmMsg_QcBD>v4bjXtHx$eZI8zw*YTjM-$)c;B3sx&YLB7)@P^uCcy_s*!;$&y#A?j
zb9b9}?bB8k{2E-yY?yV>F?bJfd)ELj#;i-K-R~3E8F&Xae!?G#x_S!{qx=e*y7gx}
z6D@)5wUQeU#FFg`dr`>Os5^*B@h<ZfeJi?6W6!qGaI|Lk<=?oWXWRzoKm_rR2=&YS
z=Eop>)`$>|GzZOv_Z$?ni6T@E7M8%_g(Wyi2o;T{Aw|ym9@L4~>buMp1Z_p@1h0#S
zVSr}iq)eQ_PY_II9>o?x7jr<5sD)HuULXbropamSnO$7cnb2(O5LoK1NJ(#iC}_{o
z5Lu6AYTmKwgtteDq)Zcm=nH#f49em({IT>4QUX$<#{G3M$vQJM!Nn_)RiXnEs%%(6
zfL2G*3)3;xEK4mC`)1A3%gkW7#NSIA5yZcD*qGvWow=!wBGJVYr<Fn7>{M9{<erJw
z8`;4B80;B+G(f{Ukkso*uPfX~gH}7m?>{t16aypY<~W1@aRKI)*ssvjf}^-g>YTew
zvNkSkp6zk*5EvwzXRNa187&9`BxD=4t0IQ&Pirqyo>jrgTv^Nm2rrUk;zwLa4@H8S
z0*pFproIcNOQZRd^m<L+U|h;YylP(-%l=k3l}}Ld6;uqfRs-sIq_!%T!t%@6?Qrll
z5FCEPH6Paok&-<oedXp`&QxG9Kn?3!jh3uY?W04NBoOgc&A!Bn&=EvJ(7qrdCB{kq
z0ze>@iuclQL?u3sKtfX_45NQzP1ZS_HnzGa!WK^AHbASVPyovvYz~5jz5DE0(#R1`
z{kGjIoqu&G(>EW6D>0WF+%XgC^-WA4>udrj!3C4h1z^T)&k*;JU{VJRW?^ve68I>9
zu`!0Qn$2t2y0XHf2~An~A=ds#?W>7b2$x+Y%nx>Vz0+KFA%<DXh{dKa?M5(a{ac28
zxrat5`Wfdn$DagFec^~ATMYcTKuT8aQxhBuI%~}PaY3@$Fs(EuBn^thmB2%X``P3J
z%z(n8%|}pGWxZUHa3!8;*B!}-3gn2wT}78(=hU!om~gy2%QdwTC`+4cd!_nq<YeX@
zc8YlhY6v&+4icCO_e2_4N>Q$Ud=qn!cRCw}pu)%QM}b0RIXE_Lz$m>9Q6|!Iu8~&r
zcOCBTj<6!HgboTbKLub&U9tuyV|dp*jLqfkHAwSASFdEr77HG3XMF%tvVjmxX;H~+
zQsjfe`fW>v<~C6>1u^L{UYY#_>5z=31k>os${Y<Go9lrqmncTdD4nJkqs>kXUFN{A
zd@1ZPf2GH2hYlCF1lM%d6LFGc7DAca-&ZmCqZb~S@-~??5G{TDBrY=C1ig+v;gPAt
z&(>qGCUdj4#}Fe)36hZb@l7nEBxst{@(Rla9$IRBY@=0~$oT{aK#AdOX7L7VWKI<L
z>QE(t5vR@N=+Tx~3_V0Scsw;B->E*UsR&XA^~Ki*sQrcWCd{Vmn=Dy#grJ*$E<nf0
z<qR|9YG&jgXYPWE%Ph*v$W~i~h2yxl<Y3+_6|#|@jAT3){fsS>Ts|@aA3B@KVt!;b
zu2KmK1^V501~EZ|Xer74rJm6WL+&+|0UzR+5_nZ-Y9Vf#`|6S?bAeJA%QC4FM!AwE
zi=rxD*2J$Q2R@$FLMS$VxCwHZr~QNe4x7kY{ptt?w!GN72H(?pY|K>^gkfl&u@0TC
zv62lk__*Xe$|BYK*08`1981>2TO4M&Ur<-#m0n4$#&@Aoq$d7^9S*CJ{o@P8TWA_t
z5n0#D5Penmo2-|VgK?1*A2c4^UaRHilI{O<#9fSU#tAg%`h9sko-zE-;i4;#GVLEL
z42?hSUruS6|8BTw?9L&fKUL1i%1E`cI#1P4(mv1BN=Qr4NEiWt5_$IieniKqmNZ`R
zxs(h0jSa3A`h_^t*9!ncBc%TViWy}vAV%;!ECKrG>gB(OvtQ=2tnYuY{rofS{+IYz
z{w2Qs|HQYQlAe*co1U7YKhe|EgPDNwZfjpZo532od35RWTy3~*6vr^dKqn;!HpcuO
z1`NLRoG=!sB2F%jo`x7@BWQ)RQsw`?n=sLFZ`Ax=urYw#$;vng)F=mEGC?N5Maxvm
ziq_7JqBDe+T2gGP#<<H4QQ}Y5#mW*A5rK4RK~AjuZNx10xnZGTcjBGb%)p};j$%na
zcr!aGH@ClUV1(O3B&E8g2(m0LIUxPia6CIfKO-p-TE<C?%qEBnmecw6ZmxIAgkWOI
zBtm{-bjKjySM+9}G{9u<4%t~bip%oHIJ||7?;m!}+dgZ+|HA3H(+AcC$e(p0?7zV&
z>)&xYsU~ZKBZitIa203#4W}8dSkgKR=0}BAv!eoaMH{P>$}GC{uVcFLC47viNxPrz
zHOPnwXhZ>lpb(d0zz_bYq@;3XDh_2{LKWK!(u*SSWDqT)kmaS!EbV($iA~ue@ysbr
zEcRP(hu-#E9qD%n7qcR%+H9vY1&*`2I!@YhK1<NFJz9(9@7~J-om&8ReBRgdQ-p2a
zdHmG1<emo$pBl-FHUaaOUb0fT)O55B(@?_1z{Ya~{;=+@*G2O1wNn$SekKHoH04IS
z3*B)$#cqs{p9`!pZQ6U-jvZHa-JQKR@ZwtLHEml)@>TTneb@H2&hv$HoegIYvPYqQ
zlh)u1#)ZBvu=D;!3*dkt6^>&4wd8KD%MtBg+HZNMidw~4%8lrCnhOvLqHh|^cmYiQ
z3OE&oWd=XWPn5wk=&2E}_04uRxB#eK?0{8yA||?ZHQC>`o@^P+LCIuN6Kah$=^W)p
z9J*mT?z-M(m%nP(lgd2<?AoW`W|iN2jI&%Q*GGU?R_kI{#f8Gr0aI30JdSM+C77|w
zb=2IigZa3$vPr@)lZ-#fwiKnXSiu2Q#*Ffu9ppP90XjQ@X8V0PN^{R)N$J*`;2Ng!
z2A1I%hXEyqKLeOT)5c4p+G2AM4}-!*_mK;mikSgZ*<VPa&8+a0c*ib~qH!;<&czjo
zKPwtT_O>7&sfC{AcSOUQ4AKKF==!6PN|hM<sGh}m1GeS0Ge3~$wXl9bIMz)fbQ44d
z*%GwSu-2UhA!JMCL60<+iIuU;Rl+C<LXvha?z@W<DM%{NOu84(z;1z{4$wv-0+)|G
za+I|9zfVrFkH|$#!w-uzb-6w1<+e<RE|$t~C|;tm3MoZaoSfuOz%alQaB?Q_xjc^?
zOH1hUv$OlYi+*pi)}AuyKn_i={9RHfjKcL0L9UqM?s>CpooHR~{!OnyPu~f<Am@U9
z(NYt-#gW3wP}kt*;v58vL0R5lv~Gf_PtBo%h(AN;vPI>LRI6Z1g@Lns(SkMA&>pv(
zFcMvO9m@Z4#9?METR2>^gL~Wi!J5%tPY)`v+y`g1jI?BB^RdGir&}%Q12^b`M}Bjf
zbr7v2{-|3KtFhNyc37*Z>r)`B!DEv7_t&pJ>m`laRI~fMb=}T2+i-yJfuCI8YTWI+
z$-(p?aIKj9;jJNhqhpuB{wimRm&^!(q|8V&7mRN%z3+FzN41E=x0lXeE$9uKS6na8
z7f*S6W*#+4rdCt{rY*tu&s8Ea`GlfVwFH&+WJ;eb(na;eqLVktxrvM)U5>@SZpk{}
zu6hLa*<Q?fQ4zX4&H4C!U-y<*zy2a3?&P$IrhkOl{NEWk*#1sL{yA`*{iiS^>LlnV
zW$gY#o|Vp1G<5!Gbb?mz|5c+6kWDNMOhBbHWzo#cEmmk7ij8Vj$Ez|A7?!jcT`tVI
z^O#=1Ku{-?Q>{S$0$w%3==Bldk6niHZzv%9-+{kKJGlaZ(i~n--+E%REzNf0<}1nG
zx^Us{<jK~XKiQP4wJcqB@u}BbP1B8zxX~dJMJ6nW2p~jW!3-9JZO;R0G$QU0Mrt%N
zDHJ!jS#t=sDA3%!g1gDRetY$N;GKSY@xA@o{gMTd(V^^nh&QcJymfc`__)E7>zVKF
zeN*C*a2@;L_jv*{DE}3`8|C*h<ZYJA{{Hhe!H<`L=2QQxRKV*&TL1kxEYUal0h{sb
z()sIp?yU2?GB@UX!$a>Y2meiDE~mg<lR($L{!1z2)2E+XS=Zg!h5gqM!<YZ_DT4mj
z`>>wJrS#OsI)T?ibHv1zsBb|4zuZ>$XJz+oNStpso<R3)>(f)ezHjQq>c`c?*Tt}a
zx)1#a48hw5WMenY2hzZM`SRBL`^wXM9KXVnP1kkI2ZSEAAKFSMlgj&=JKyTp=M%>7
zKC>&I7oWmZfd`ISpO*Wmp3Fw`FV#2cE1%I;X<v$KeF5Z8-x~_knA;11_m4-3nd$Gh
zG-KV*x8{MhCbIUO?*dmuevGffS076V`MF<S$C<U?pS)u)to=ULv~>@syKm3e^PCJ{
z<EGBW+$p}cYr77OWc(yPTi#CPKOY-^dtI*Ee}Cs_zw*6#TP`)rdXh0an^4ug6wfDi
z!DL_NhfD)ePJ&yE@vaKQBTu$-`7sgt;={04h-7l?x`wF2IBaa%#w}ps%11*@w6m)q
zg)W}mxe94qL`={mn?fIrWx_t(qS~~{tpH&Hoo(x3iRBuc5t{93Jr7=iKyondR0P#&
z)YqqATN6bzDU1bIy@*!7sIt$CQft@dLN423a?=|jQQw*MOC5c7Gw-%>=sln&Rc~`k
z3u%aKDCUXGeJ1I2@SPq)Mme))+`v{^Bdg|M&`?|`jC7oNfGLN^ajdz&M~tBLq1`{K
zV%TndHshOh>4a?js#r_)_bYqrldW_MS!har9t`Er#mAx>E&TPOLWW|y&poT}YV6}&
z5^33jP++insjM~C>!%Hw%K>AfyLvKwBgdY;`$GC<`d0U>TiUo~Rx9_=+LmOM7-*;*
zGa9?!M3?0HF>o?#bzmNxy_9%`+6vUvWtZG#-Ayq>_PH&_b7|GK^yABNq}b9orTEc|
zBd#VNqi#sqKB~6wnh2+>e~KDIjU5V1L~VB35ukvBHO(%tCR;(s)@GFR^Xm24=OGK7
z>CP07uWS2(sGJI>%@9gx^6^`GR`D8wd`96AXvv)I8~4IrZbh`fZ1|M&d&Q#*ndlI3
z_ES7vG#C{|DE9N*$4uz?RbV!v3<*|6M4{LRiEB`1mb5oefiE-bhI2oo7NU_5Jb_#V
z0;qW)sF$Z>zwhr4J8QVGx?tTyxDKJNj({G_Ah5gXDrh^NyC?rlDxPv}qG=pB3f$0m
zDnmSWi&IH4b;YIyg&v=W9p!7v?~s7Y!8Rt`XttQAsxz>~|CJk>zV~V&rN+DWYq7k?
zjZ1aR62v`qD*U0HWdwWQY7_F?6qp&Ctz_Pib}r~3ViSIIxlIAA@K~Lv;F9><XBQ|{
zcQJtQF4VFimKdZRA+lx9+?Pt8IDpI@F>*fO09)CKYDfb*IR3062hZg4yvc$HKe)m%
zU*ub?Exc8mmur>4sRenrowz)IVzclyy8*!L5lSPwMuM_DO4cW+g&+BvEr0&hjQtfm
zyT+!=RHqPr_qaF()R<_}Ge8;%^H+Tv=bKJ5COj;<P^S;7P}lbwy0K7cRsr5M;#H^$
z2V0oxse}^w0X>rN(qk`!s~)>#^<o3Scd6r{l%8UR96?(~DUBC9st7Mmzr{Ua9<sBF
z_DHwhx!A2JGX-_zg3-#^OJ-IN_1%OOZW^vF-r#EMwpDvaw-lR<sDS}GdJBE$?hPL4
zUK5`(r#ywRg93+(t`&Vc0t28tV^n@?V6D7pJBFfS>Z|rw7rIve{<`OI2>19yP)WID
zu%7)sRXKQXY?a>-1qMN7!KWmcm<NY<+E`(zpA#24We&*J`Qqf+RZ&lj5`@2we=FaF
z)MCmPf#|sLO*TH~DT!LZCbYGQiG|I$!f__f?6Nhv8xEkt<muRAp}*+0jb$FasY+T1
zig0=_qpDE*_%(yIP6X<tEz?+6ZCxC|p<STCt5ST|Th!2OOy%w1%5~X#?$HU1dN_V`
zbRr-XKHjFB52J#YCc0!(kMgG*Fg>;_VRu}IHY*1Y9QKgI({D%n2tq(9_FRp3x#99%
zM0O}h+z_ZK56&a6<=k03l4R)2CO=CZx7selc3rV(-D_O4qfNIM2|1Zq`Bfj`uwdxg
zzRevU+!XnvuADcRQ~5RuX8=dSy^jzc$WU3h(UL&b=WxHsv8i>M&CX_J<RJz8x7~R!
z@Zy6j>Q3e;33O%iN9?pTHhdMn`E8}c+uHKgk)DOx^t7lx`=rRXqwn~}Ly{SG_|rW0
zYbBkjCxjd%K?%AF)-esXTROuo#!LESJ092q*dj!ia`?k@W+@xvJ5r{bgba6@3NcOi
zXmosO7|kG!#i__zPvJn>#bfS-5@H^;TJhsxb<riVKLZ0$Fj1X3Y?&S9Gxni0CtdHt
zqF)%5t+nLf<>O%tXc$H>!Bl!hPOy7#tbs*r(LVC3<TTTJbY=z{-r6F{(j0?Y-1+P5
z#Os^QUYrY67K~2<j_Zh)d&DL+2!i(A)_>1n&>$*MobK?`3HkeY=u)x0*zj#ByMAli
z_6Vl$NN1CnIhj{HFaE%wxMQTpa-~*3o9^e5rU7zq9%)Mr{1`-@VYARE-40aE>1DN=
z68$+;eWF52K=MQeCEtg|=|1yQym}B+^%V8kEoDLvaZ7;9Lbq-G6Ij#gXPVwR1yNvx
zRi}A-)GY?3$RPTHjFGFQg&cvctc%K(T1sBLZ0q5=)HTwW_RQ|H?@#w+5z{3j7={hH
zxrtnxplc!$0~?TEQA2^!%QGTGJ>za5nxQ;o{jvA=X`QC4r~K~*&t^1=8Q5^FI7=zm
zMp|2Db6O!jHnNw*^S<XhHYoC(J4|v|ZV<~qF%UYE;tNVDbtSX4XtHp^!i9;vX%K2T
zjpbI{Ol_#+MywjSVSsU~<3dR-AY4I)2t7u|pf{>7BugwuoQY@kmeb;GAmJU>PKlV-
ztHz+7^p42;Zqeec-G75-x1x<S*kiW`k@dSMo;j8;aRb{WAjbMnM!Ir0M~wG<<9LV@
z^ZnjMTh*(ut`|uVkqJV@1m65zhJnPigx88REF;YH&!Q`V?*I%MLcMfT7fKf6?D~1B
z1VTECe;o+wCg+jdyEo_HuK&cpVX{`YO5SbFm{^F%(R|{N4pTcq)$dqfnK)EV0StP*
zSFPKi=BWRMGqVAQBzz%FoYquh6vx*i++vF76CY>cV@;}_i`Swg6`uk1<j_^Bc?@IX
zN-LHw*0L%#JAHlI<W6L-p3JfeUot-bhR~szrYar;v&thoG&%DLP{dvmD^!Y7pv=<Q
z%HD3d5Cf{+oe}Gk?jc2E`yH|Vm*?;Rlu$==>y?i4iBLg3qyN-p#>z^GVZhIu?8?y{
zLHx0xYJy5GJM^;`1=KrK<eU4_*C41L>4IV8U>V$Tt<dK<&1i7rt_*`gCES<7X~>Jw
zn+iB4c86~$e%f9Bla)lRYg!IL1g;N8GFTk+3BA-C8#5FSy1y;viBsKSSxBaJigXR}
zz-qFvX|(mrPU`Zi6KuM2kt%%o(!fJc-cMbWG;)ydvY0iw(yq=F#OEMr`0v?Ww{&1q
zIy5CYSZNomg4fF9b8BcaaN&VF`1MG<3C(QA!ni7G1*MILbe&nBw*f_>xlg>EsV(r=
zb##F!+vpIw=3GWb)~N5CHh^RFMFmEbD5zybE1*JzMS8rhVHTH{AFEJ>RZS<EhjX2a
zr(PU`xEGOwU8{w+;Ne_s*=lq|h}u&)a5IX*drUKjD`sk))Qn1{*pgOzf+(^$VIEQ-
zN$dOy-+m77e#^<2aKT=2j#6rdyttpgLxmIQtG`51oT~oB=A6OoWsA(pS2)&sJW%Pn
z=*2PX=;g_x{0Ui|1}j<3iBr+@9#RdQ=_b$L{;M8Fb<%a=28u(Vcnq$P6o{zAw&)%u
zZSeF=)cU76352Eh&r>N1?-|=%13Y;hkjmYni0I-<nzV88KwIxz<Dz?=Y_7IIY<UxG
z=}iATek?feTDD6)8lcXTei!2+(dGIJ-NA3$hfsQ<$TF(7LtS`5KAYGcSI$c9xWw?9
zLC8*46{)*oK?Jc0Vl(8mI;DmT^l-wAj6J39lt^b$FYmMulwmV!hV|p~-xTSvkw(=e
z`!eQTeRz8gm8cpgrs&~JIdgQ6B=r?Y3!^$UcK2mE+llj6H&_b$wLz$`vGy8lE;7(D
zs;ys===Q7wr($ybUDurt3*=oG30-sCiq?*E-p9}y=0n-Nz4`DFF;dp+Szv-U-QK_7
z-2@cjHj=t+Ir<EFchLETq$6t}rntQe$IaOeXF;hPU0WSx^V8WABSPCtSdK3tMoiyo
z9B)i&y;F;E?b$S}i$<5yPyy>gakxw4#T;MT^(W~Hq56A^4;hhTJ54BF)f_64#vYlG
z+4=duloHM(2C^=v=8^eK|0<!Lvm{UwWmBSp2Z|KXd5?|7SG<U&QBoC6rA;+Id=<+A
zRVK^q<+ps)EHKl-4Kh?NH__+Kq4l0yN^x00&q>2Oohjt%1n|$S$Fv@`hb{UrTkT$&
zy0Hkut6)uRvVM9wagWK@J-n=>xS>^2j>Eenm)n>fj7-@RK(=pek2Lu(X+XD@O<mad
zu#$Wx(I=>@(AcQZ5u9F_;lVK4t(o9%ZJx9z@bdGrC#&pv`@5T$rHQwUX=lkAq5>nt
zcSU032XgtX28@t!9dQmQ^YG?Tkf81)lozB4K4#=64FMMn7OW7-YO9ZvOtmA!v9Y(s
z!qW~v{}2UAf%V;Wb?GER_vZ6xkRvgJLyxjX279*(NHV>uYyXA?CBgWJDKBBj%U59j
z;-SoRR%3)87&8#(86P4IF?-gaZycz6K8yiabkak?uQ7HpAb%LSoMM!AJiLrczRT9d
z>f~w0p7Dfc&`rv*+??;x0%M(!{l;XtxOw^<jFH{uVvWH{iVwx~X@Wh6g2zgY$x~ME
z%9NTIcZzrb+8B52IfI)4Mrv~Y1np^Vg;yX?xs)RH7Gjwkzo6=N$jlCsA7P#yv7O(k
z3`v`;I`39LlG)#c|La$=GTH{KEmmzs_*iM18dI<7v`OBN2+5jr9}1>>RRql#Xse{S
zL`s-_@|Ts_-->E>+=lb>FH3dH6l<(TYV-}r)u0mj^J3lO>A0!udKc=nY8(6LG&ZVc
zs@l4fII=>jUYuR7LFW5AT}3n<miY9G2;+f}yIg+lFq+q$Y8$yg?D28JRFzqHlPwfk
z&y52RUk9c3nH&Q5)lVOP@zB9N{UZkd*dS4VHsJhAkDBA}+=PExA(#K<QD^09|Ip7!
z-OW%}N=nz7_&;5AaBzI|&Ywxp;E*#xb42eq3M?|aaQ8}MN_o4o3f%r;3<&fdP*U3c
zY*+v`w0#VbU@Xu4TO`U`vh-Gk7Zu3K{qPEbdSv3M4%mv{7?X+$9}~IxhZcp$@}u&h
z;f-TL4@UvGaEk+LAkL^WZ=Q_36o?t7FqAS<gt#?W3dzp1S%1nzhwLF*(>{X6w=iW|
zp|)a@hJUeIe&*{ya{M8O=il*(IR7O%$N!_i@c&X=2nE4T)Mpd~n%+?5{|lUIb1^rO
zKTu@<9XMS763)qg*;YUFbrRB3^>Fo)G!wK?e`t*UpKlABKJC;a=s%$KUeKQv{so!{
zOd<#2pA@qBchbo4UqU<oFKAgAIhh&i|0s4$|6hDGTR<dJoiGz5BeN6J0fzH4IXTTU
z1dXVJIavzXX>##l@jeFB3kDhaQ4uxg5yqb^B1<tjJh~Aqj*1Mxg%{z2;S2I`Huep-
zpqB}pA5od!(BK&D31@&fr%nT{EYB&=0rK{(*E3Rd0@n@+(d<Z%fYfkTwA7O{qt_ur
zn6UXlWi$9vEH+wRWyw`Mh@~UAsUUV93hose>Q$0Om_0}q(F)cPN^W+T0v<|!ID}qu
zIAL=$8tOHeeojcuRgEcC$~@K&Qc2oP$u>yG!bVZNLf1{nQ7_8CAwZTqKFY>P%Fo%e
zxOm_{w2v{9XH<t_G5#ML<o>(FWO&F2H}oeR{;lq7^e=h;_q(qaIR2+c+N}pjXW)R;
zKl^YO<t^3AsKSjE<z@h}enL77l9@^9jEwZmJ`CaR#CZN`i{}09SYCz(yAWKA!L0On
z5Ojl+5Lq(-O!-fzobpjf_TCxF^v_aH0Y*9u<YE9XDhH<az@x?tJUx0h3TJF)SK-In
zCohxS@KF%5X3fl-+T(|M2=D(#{VmT5Yvb+n@~>8@dl|}G*&lX-|DCKf{#WcwDgH+`
zD*dDEhW?kbo2?1+q?xIyKOm(R7B)Il5H=6d`MVMsNyaG~E`RZ8xo3F^`By%m=t2M+
zkO{V+7gFOxN41s+H`7#0A=ysU_6Rp%ZH`P&B{e}aH7Xw`4{=H<MLRk%ZOqQhxE&Ie
znqpQGp6rR5QIghv2%f=%<E=@0G9=OF@d}1NllBgbT>vYe#m4-XUTOD%2W#pC0A%I<
z8{%&Ach4X40g;9~*u)R7?pl^}l76|yXo*&m+!VZ-KD7z_(*g}QUTr_C5ei(=pbkKU
zXaFMo%^!pk7@VSRbj2i*=^OD8ArbSKsQ&ub(><i$C*SgKw&|dvn)1sUhtsSlz9SXa
z-_jH+Vc-3hkR-BQR8}^O4@XP!UH6mrV!3xlu$195nVjbgSrCk)Z}{Zb_U^pRJ-T@^
zK3>w788TjZ*v;Y9S8zk9hIVgQ1ioK}mevmGYukAop59PwTaoGVJ$>4#$7Q|@R7(i>
zJhZG*X0^F&Ow{JI8?G(xna|s|6yT&h=3Xyybax8aT7JDeTOZthdcM$c$Z7L_zSsV8
zL!c!Pppl=(<-qBNv)PQ7f4vFsm*;;8M%oLV<aJ$A<blUwGCnLH&US&Vkq91z)FJS^
zt?HNOJpF-1k;{z|VmY%ex&uX#>mozp-Fr7}g(8vjec+qGc4BG66~5036iZ3{7H9U5
zSAq#P&OzIJ8VbGH%T^v&@?znjjS8;3sT?bsSOib<1LzK$jS8xascO<XL8BLE@&oyE
zXYg0H2{p?Zensnvikck1k(?J1NA|VxP}tD?$~T1R99-6y0gP5QsrYuLjcyzqH1c+)
z;mw$X?0b?Sua3h-nYSNoz*iz>V@S|fH`fzRuSAuv7!GeDY6$e(569Rg+Eyfj+P88l
zDvGbX7<ZhVljkvegC37mQdC@muDsH0+4sgBaOx!U_h4Js*WN;MB8o0~TDRg<Y}0BN
zQ7ya*rnk~)wpR4wXY#7IvF>ntZetJnm2!6^X576X&|9XJ#m7Q}H@3XouaF;i1PU%L
z{XAgJ1AdD)HOfix5g5+H*<noIrsuF8#BfNl?Y!XDa5XF%zlg-)<sa#{!G`5IE8B;m
zKcK63S{bebbt;)!gB@*gwPfTkT?>DgJ)&0c6VPDi_`zi)c7$j4{VppuNZR#Kx3btq
zg^Saikh&4`cqxyh%3W&3rPY99aM*BmX<Q!eJ{>iwMy;B~eM-;)`Z#ogtN^uc1fK5&
z)Sn|{gBlbiq$*>jPCJ+A<}YqUIjf13elB!Yt_ou3E|v&HTRs!%xd9#B0+QX0j7(Am
zYZ<d#-CK7CFEy71#ffv=Hv-4b!{cNMuUIsZO7@Nv8G$}tNkU6sN>V61d)|I8q@g=}
zgXUff>-CT^C(-@#2_I)*3eJh^D(yBQ2|BcsvFlM8C_UNOib>vZZv8Gs!#TScJsQO}
z^ef|CPOL_KHV2Yp&<~^dnsw9BctX|zPA{1odj69h*WMh~us{Ts8XYAL!!$A-%Fl9J
z>_?vWE$;#H9{ri8-jhHdp~zUF$V|v_FmK|qIMsX9rKt8~AWBvN;|S*Rr^JGS!EY<c
z35U3&oHPbw+tGQKj&@x}&+fi4qL|4OFQJ@@?EnnJEQaN*<NfaeLz|bVBp+tIRf8~W
zBLyf3<04MoN=;926#NQwBcH&D7*1+K6-gDk4s*3CUToNRHg-1W1Vp6R_PM!<&Dt{F
z2pqZ*z4^Y&o588?2nxhZ>^GdzF<;3QxmmUOJ(13G9_@m&iGxb7O;X7+TuM8);EBl{
zK@EWg_e2*vLC`qV&(l0v4&cFgiu<?Xi6<JOJL_P|bF8=wX*DZy&#>I-!r$02J%(V5
z(Q_)LtEjltkCxgqeC^EXaH(y+WryY0)h^uI8xnREqwQ~jr(OvY$|+_YyHyerjEQ7P
z7_Bi9&z}9zv?tTrML;XwZmvM-43ChO>$(1<oL+GS_Y*A7&q9CUBOAGpZ~3L<9!LWN
zrpeBf`kU(5kAngNDt9;Od=S>?nd+pp^E~0HN)IMX;}MXUi?#7dwfxAIb&aAnu)z3&
z5=$ln_J)lZxM(3%*H8x0hJrt9Hx>u2lN0V5Vpwv+eh_u+DW7v9k=b<pRH2G-3)*|o
zm@Ok9#Cbxn4s$z=KCHv|;op#0)2GpyZ0w{Ttr}@p;@1yqkav1X73te9x%@*eI<Z5>
z$R{&2vBqxnM@L(bH3Fp^H<S7VDjs&#?<n<mT#{CTwg63jsMPMD`=aa){0<1)YgCnM
zQi~wxO3WX%W3N}L6q}VW{TjjV{hW1vE4D*S7nqrH8*U|oXEy+2A=SZ|q;I1+@SABJ
zW2-Q2+%coBT*_UMpIa+~-><6JM-HvCOF$L}_6wnxOaiErXM^_1LDuTQUv&GNu8ykY
z8Tahb>SYL&SCmuQf=45=!G?LDg?-);*|{ZCnKcX11yS3_IyMI0+8y}VhR;sm$BsFL
zdI`hLAeIKFqij@=3k;}pcJznRzmrWrtVZtPubPf&x1qu`wQ(nL5XuVtgt?U-Jx~|H
zTJKbz%mD^@H2L)dYn>|&T|EfG=8Q`06ZPz4i}`$`lJ`q<u#1!7K}bgpT1LjY=}_)?
zp*T81o@i{@kaDZm`GsRNO<URRFm|~%bFSL)r5)PX-F;TUrq+Oxf1Z#~gFBRbQb^3x
z1&l3pZ&TG3D_RJr&4D7+u*qOo$D$I1slA`c<2Q{VZ&X<V52Zwm_Lv9X`QG_vuSXpU
zf>PMGVSgkaXmKNCQWJLZ%RF4112Jv3;dTt^d)ctUMg@zb6}g5~voUi^)$@UeyF7$u
zk?{?_^}O^9;RE?X@n)qT5c@5->f#7&rr6_bHr7?d2o=<i#&~N6C!}$129hb<=^M2p
z8l@FFllvB@MB++a`79s5cI23$$998ybv}hm4=ez`O7Va{+bfI~Vck-yaSO(V>I&Xb
z(I?wW1|BQt8GQmxT@aQQ>9RZKz3b+_ZCJ0%cy{b}+Bw_mc{UgWFez>huWc!9b2PEr
zk;6|5rw~f$2;9|t$00zZ-==}<w3>eU9WGPqtRZ|4`TaUwka54Bg5W>kKer{3d?z5h
z<D}w=-yM_t;c7_>J<ZG`0N~AhSwwHk&IzOuwygYwk&61_kaFJild#PHvyEr)q`UAZ
z<ajN`eL%s+A?wn&Yl-~WaeRi1A#fuN17oN&=Z`<ngM0>JWdWc8qv`L(4jGkCsn1!_
z9ug1Jz=i*07<mE3JG_Iwbe!WbNCjz|)`ea<q5}i;^W4fcfSP_Hvs3r1lZOe~^F=-w
ziy5IU&r5S^Hi(C_WxKyGk9cL3t00&8Tp4LBVNRb(1};Ba2n<kP3xLJm7hWXmS#^P<
z5o8GkE*6`u6-TSH1)5;6f;P=+TLd{dK%tiAqj_~Y@Y!rVj!{l;_F#8)CprA~0(Pyv
zi_}-m-gb?(V^1xR-}4+4$?aRdp?r^4!z(;B7yj}aJD2pbKc*x4=yjnLLG8WFk5eG`
z_n9U8ZiA@TcTs%YSDdw4dlTth64(TLAP3&7KRtlG62k)}wT<oxHSAD)Hrw7>Fp~+I
z>hxl~;_S{_`+}3kmox6e%FoIxM^~y**J~w6{Ny~sqGrx@qB!)rg|ndrO4$HjLUN%r
z-s|DNg7dTBI|^JJc=>@x-5O-#o(e3CR`4*>hg?!o@OQqSqo2VMEQTmy2|#Noaq#4)
zrdQ3a;`@3Kj3{({L7mlFT>R*2H>!4Hbl9j*ZjqcKmZF#v(OO!{XF0~vkwM0b;=xQD
zlE2|(D=KcN#&t_I<EU<N1KR0#l2K<4EbjS%nVXue&>#a)2x6ANKDY@Y;}J2oi;@0J
zhhgK?z}muqDd!>td*esPK3y|V+zn+mx%@G19;H?V7$wLs7j8P(;Ey8{&&z&hz~s}m
zyO$d$9=qCg7miy4f!haYVbvvRN5+uWlzYCQKQt1JeFASvZcU54=!h?QM5)Pni@}jC
zG1Zx`1N}5oO#(rFy|{T+R3H-=7va#GC<n~d1(+tNxn(l`fH<6q2TEWOL8;f27MRfX
z(fO33iNG_^695(VPMm=a0)Er6F51yLeh{K|N6skAIxJFl;WcDjrB7Dv=pcUsTo3`@
zx^s27hI6ySU#eBVWaas8_CLb`)vZCWNF<(#wkUU_igdv2U_mj~>GPlv1e#_+ESy|O
z-1$TjZT&3Iqg2d{b|Ty8A*CLXo@I|N^N=bhd$YWi>MX=Bd_`vGa(qc@U$>!LxWO&C
zqKSz}%|FFBVi^>)r;-@1)C@?R@{eil92t;0e)tH$o4+vBbd)Ch6ij6Z4}3>?ZDyHR
z!cR#|i7F8lxnN0}wL`I{<wNQJt{j`VWVnVR)x6BBT>rR)`<t-hHcwINmlY*1r+Kdi
z*zmR@4y{AHxsX%Fxc_|#2y*qA!D~4<G(6ZG3>(a*&h))<*}h`~HLKVTTH9w!Lv?VT
zB;PA(ErF8wXsV!^M(P{{bak0r3+O9?vGS8&Rd`{P+_OKBAngpUOhMZ6NpR!r;S+bu
zyJCRqyO7nCO_;9hd+BwwF6Xs25<+qPYf)fWmeKKWB^*ZWK-+WT*u|<8*GHmMK`_x~
zy+2pm04-T<ra=zAnQ2S@2p(F=Q9W#m3bYdukh;2+e8J64B4hRXA|doVW>U(Y4*|+{
z0{Wxg-mCqZ;vN)OTe4twJInHPeC!h+s|f=j?2NV~S5XA6;YI)$6Nchs7qbby7WwhY
zNhxfGEUoBcy}je=u$o8Zg$)s9p~fL3Q70<iSIY_6=fiUKI)TSO?{TT6Ge`;gq;oVr
zZDrBFa@Ylk0yoLI<7z)D0T4Ce_`#{VH4nZpKX_oqTDTN~6HAR=6o#{OI^j>wTcME}
z1<vx(H??FRD{*&Ri~edqTky7SW5M0L3XUZJB;2i*`m`6YtQFZ=t0v_)yOSrS*tJj&
zh&-ccHcgfDV5w+NNbZCP@J-DV>h7f-YS4}Dd1#$WnuY`F;1i^wm^S2M2-SmROmow#
zY^lA|Aw0)RmSO&-p7zY7#!AHh1dIA^g*YE4#xUo9j5m|eA#GaM(3;9)tHI(9owRDd
zMpQ~eJG@*iY<6jqJIO|=*8S4i0Y%bJM5nV3pTIIj%k4pu7zb<4ol^NUR~HyOnj~SQ
z`@x^;XPM>$WfNvHhdJ=j+6NVeqXP89jiDM>uU0XD>PW$!Rn|Rp?ilUhMz_pCOQ_p#
z{eb{5#(xaZCDeqL5Ue8U>nH{O5Q__--5(JXY}nJcQ-6QqO`Si-7n{QAGpv+SG;dvp
zac8fVzq^I=dJyi!-AZyUh#`?}Z6CaNi*|Bhqph7`w$XpWU}2g{>JKe=6A_~EHHy3h
zrWb<YY2}q>c6C_pK_cm-Oe-|;fQ6hOonZ6T!%sUq{ahum1M9$Uo_u3_BMDR&RMO^8
z>vG}UVFK3nybX!TO#OZPIX1XY*bXEJOL*}SaPV>DxB2o}abAl9krJC5@>?xGQWLu|
z@_MD3LuP%gpW+K94k_)`4=DZY!(I@5@C$ro2~5_22G~Ro^)>>;hQhzi63YPVR2D1?
zy0oDyTs~tgS{yL|2_Ebh)<p?uyFiF7oFE@y12FV%+cWM0AXWgcG@=bcTI&g2`jCF6
zB)FXq2QsK@f%>`qq)2brSVb9tekJMeA?}&RPn-Av+PXz(K2+R++tG`&-!(qI%tQs%
z1fcG}aU+sJy(+s0z4T(|CRPb9NsCFkKTuc^bi_^3ameCM`T@@xfCB9I;Kg|BYLNzg
zjkuFD5xZ~`CnZxt*{{bVQjr6gyXnytw10b=g(V(%)bj_8GsFWb0fE1m@D992(}RBk
zo-V5}vk%@Wr@**t5Ky##)^k79G$NHSeWFWx767u4R@LgaKSRrthM4vr(TiTb015)>
zN${4b&qeBt?}h=rchdEPQ#ivlOuS0~m}hhhNq0ftIl=d+iFyEQ6ej@Yc2aqtL1fS%
zHohynf7gK4@u&5Hew5U>b3FcN?x_P&MF9=e^;gsM&lq62KhPq%##i^J7Ba|_>G9z=
zMBYr`>7BZyZ25$)^^Buh+UHGPC@che(*s-0M1bAyE_Z&j6R{ckHe-F)dhTknnGerE
zwj%96DzHUa%E#yfAo1kIVfmGv(7N+Js&Ln;Gt@TD1eekdR>}2QKc^Y_Mcwvs7>D-8
zEt_NR6fIA+7QZmlhrV73@p~*afvu(V1+^+ndSumG{8;^<ky{Ly@xi37g06`AEg=RE
z9tELC?N;xIk~niuNPcP7+VF4)Tu3%#0`TSZCYQ&S<~*SYB$Avz2^~%r4WYtY4~!A$
zrtIj1ct@pvc=6y5&t+sSPeq2T!;mIVKW91s1El(y;vDFQstEYh8oqih$86us)Bl**
zzW~u@jCl10$-&n%VD==en%1pIL~HR=ZONXOy3suD;DLj7#9O%U0&w6BW;Aa#c%z+G
zoKrR^%}Mw!77GH$0YJ9Gxl8jq)eRYk3_)`-)>OHDyP=gaW|gOv?!%(#x0TQ8vfiuk
z<`t+?67RsSmE%)bn_)8isf)+=`Lb8rubp05vTID?7weNAep}REh4?Y$S!W;erD<=o
zmiS-<w$SRk00+|cS`?!4SDTPIkr3v4=aq1fstviU)cnj7LUc%^t_mf12j|@xcY}PE
zE<lA>Y(xeqHL8HuCk7cB^%DS)y!fQ3))oh*$$dZB42Lj+Kgm+QQK=_M9CD%~(6>8S
z`bno=iG1=Y&)4I3gs{6z3+9XpGS<hDW2}J%{@MT+l*`4qEf-eISt$8IHvhEPAWatc
z@Dm0@fBQv!4GhYc;!DM(Z9Y^kno{U1dtu05Jq>>CFb;w#e1ziOG32CHt^FNjty<FS
z*Y}O(7EWx{9Lh8&hA05@JRf#Mwv4i!SZ_0TTFaDE-ZLZfmq>zpE|EW{?k)UcWEyc<
zSS5|_XE~YnvcYcaCSvmSUdM(^L2FQ-O+M6i9s*6BzQ4qyt<@+4M!U}wK8Z4F>W@G^
z5hM}VVKl%u^B^#?)FAx;=@y7z7zYu8Lo8A=<VTRh0-a03$ve!nGm>6(oa$OXWOf&C
z>w#G}Vk1$W5a<E!Os9gYx<$*oXLN6L^v%dr=c)snJsSwAr>@-+NK`|g&4h%bgs?j&
z#jsBl?lV<xAj(4mC#WKd?GR)8yZ{bbOh;IgE4jQHb(vVtLoWu0ua`dG#BpaUoy#~T
z0oEhZryq?^mb7M9bT#%h_b-y)#i`^4mOq~EmD(&E`ooz0w~d9y%*p1ja3T{R1;2WK
zReA4PGiDN54c)YM=BhpVRxi#i&H<0MAcbFFP@`}Zj9~=HaG_V8sfG_9wU9?*@Ip^E
zI8%@SN$OwFjvRp{J~ag-A{Az^Z6w9zTk@M7jyTah`)=`Th2X1HO@k+kgo_9sZ>`79
zT*%9OR8tpu@du+Sq^{QX&1Kba?tD*}<6*S8vvsI~fQ9&3MVWuUk0I#VoQlH_ou~L%
zm$so|Xmm6&9_H<zc=}1It1j`YVkF;IN0{XV0q)|i|JsV;s-;I65aT6FHXnYeBQ^A@
zOx`sgk2V6sL)4a8)xd+vre-*CsvG$s8t^@KmQAA{@eKA`Yp!Jz<h6CaTv9;}t(g~v
z7WypYJM;9p9?_+hbKbG6CxsT1khxduO87@R9P2K2+8R^jbCgol@0n^)+B0;gLY<Mt
zW*$JxJ7-p*nSYdl%F)|2EuK5SXL$`C$6X~Ihm1C_-QzNuKXsp)F_cO^wBX`IoYdxg
zL&lYPSIyK@^GjRFY;rD|8GWtes{VL6^oOe&_$0IY%EH$=P>T2$j(x~h_fB=PULBnM
zi=!xf*^i&T?+h{_#2O?Pf%yQ=!VxTXuIQJ2dF`BrhnX@ER8g6cw*y-cold(B*cO0)
z<h!scA-y-&85sl9AlY~it1(oHM(y1Mny5X_fl#4*tPVd?hbi{i@l7xGy}Sm=Ewlcz
z5imB<j(|H;PD|P<4-jnJ9v?lnZe%HPf;ZUVj{#INw7eSXykNNC12e5p*dNr}QH;b6
z60x)mpIqB97pOlu2rYsjJX5c`dR#{(Q3{P8@+wm1@KBRHv3`(x_&c%Oza&whBD&KP
zWy?WltfkX#J8pxUQ#}$Ui?+aTPy$H$CLBfUD08OS{DXUR%Z!QqWb%Hl7N->OHK}RV
zLMT?3UID^<0^{F7za2X&VcVY-w-B{~s4rJVdC;f?idAgZxIyTNH4RhWiXF!|X2n1k
z%;wekAnvRNuBV<J?2|FKQ8}zsZ;ukxD=nA~1ROCGj-+G>RbneJc7z)M$@;kv<@?V{
zt>VoHT42s%z~-6s1U`qI7n;694&t!`YP!Y__-9gkYX}!+jU#85+vF-dmTNi^khiwb
z4ElXwq#fL=&*}Y7s!^UO?(1n34BG$}4R$N$G_LmYbTF95=Ka_l!6d)O8=-^0!G~<Y
zF#0>`=0r52DG7oqN|p;00!XHh7OzDYyM&yH^CI*4!5M*$Es^6$F}B=q>o8<zulI18
zI&2u6wuwsmQJ;fUR#abN;a%wnS#y;D1%&d9c?zE<NqhEZ`wO%qxr&g-VvVq-7s2}d
zNiQJVX5$)Q4PdaQk0mEJ6?=HK%+-Y}kisw?^e^oGGa&|Few0H(X}skc_D&R@xgD|c
zMlJ{wi+$zjTn2sEsFL5#j3VhvfraAz*fR`|YggqM(Ksi9n49%?UZaI`bH#_tew6UB
zfn&;V0ljyiw0Yp`-9ukJ&hx0gY01J%AnaxvPOfJq>6Fs%k4Qqj=g?S9#@#MG&cwv^
z^M1`xQkYZ9IVUq7Tu>W~j1p!w|8d8S4fK&4ms{(cV#w<{B@wpSUvAM)<nB<eFA95`
z-&|2y!LT+1--G*p=4u$_%PnJqo=Q8FD8X+DjYbfzEGNKN$;-Y&GS>l&praKe&cb0v
z!BF$bBD>6a=5ew&g&2h(R+4$v%kCPeyuh{~&+StOKxGDIQEA%n#7ouvZ3N(1MD=DY
zyuq@%dzg_387ywu$#)^ZQQ|eNbMj}df6$=W9yex}L;|^4I+6qLaxFMynE$%wekqbv
z<&sSWf4Mdaof`k#(yERa4bI4*6I$8|NbVs8e#h8XhE8>FV*1!ZE;}(7^hF;DYK|j6
zyu+@!Xjk==j$ePXc>NI7V-U37YpokBw3ewFf;@DDQAFP6yQwPM$*6m?VV+VTvlqto
zRb*;}1X|U+eFJ9zq<_HKLmj$AT2`TV4^>kbE;#>)!;nd7E#9kBU_TfIr(pPP(d?Ns
zawo)oGiJn-s1IE5&U`iSW>tc#46@STybHH3G+gfiSoy{kWxS}>5oc!XDO1t`-BWAP
zbwL{Z8qOu*Uh!I`<?at`4KX&?)Az0ouVo66m!Bc!L<-ik<$kvdH7FU1mDX$*;zU>X
z$tQ%se1MfO&ZAOOJDY6+1Ku~~$ThM3+8?5IYly#{vn|Spp4TqT0>83b7Us&NsA@e5
zrh4pECT;iNo+P4LNv04jpv0^oN$<dHW*HJ#GSArbgn8iqjRno$llg^TMfBOXD-`)m
zIjysBq=mr{_?4P1e|E&kM3^<dFhxxjkXyxtD4<)-0gP%$FL{f89F0kVHo}#yGh~)y
zH&Tb#J9kF7c0Z$6RT@090gRWXy(Lt`Hle|XsWs=9eFm@VAT9IBtTxxBJ4(+~fBOMl
z{&mJ$5_427Oqr)Cm@#eqk;_yhBnH+R`aD242;GuWbVfmzgOmPxhxxQN6SzSj@+KCW
zEPOmLbAaADjP@<q*8^!;9T2kVPZV)8f>9~=tm!R;?Vo0mEf}>`7v%(wNKUTWE_?-_
ziP*}LJdsXwq?+zqFbkRKSDrpopx}z~-~=nYRRnpIE3I=0G58iozp$dj@xK5&K*Yc0
zp<UcwjAlG0oiINF3gOE#lTQqT*uwm|Cm5!@ww*iStx&5Js)}CbraY2d{aEm=lJ_bF
zc@XsZlnU2QSGt2}O5Q$Q$>3ddP*M3J2WS>V=^SkvG|Qrji^tFYaLfBs7mwfO%$ohx
z)^3VM<_ThvYwMfX<TF5M6k}|;y;ir}Ubt@-ew*5Io~_LMYwr&%28bgpH@@XOqX_4X
z)BbO07N!3inkC`CL9@rAZ(LzP47ij5E+8zATuB}QG#KoRi;=ISCj{&no#KxweuE4*
zVvk}<5ciRkZFWfLHTlGyxj~+yj?t+<m0{{9PbDDL>-u;AA+FIr+1pPEBH`xV08iIm
z<#FiwJCi=_jhaiDeK8;HdP_1#QNAAUM(A%btAw9q$!sQ>GI|FSukb3KVs$Iz{)a4d
zD;3x(C;o)+&`)90j!?@$ceUoroYFe%PcmgwijGhrRWEbv6Epx_tOIG6!<TujSyE|@
z|CEr_%Umz1jix&ez~&+<=uk&S?o1%H7(L6&n^knkzlH(TPGxZz#-kO0pDc=sMpj50
zB>%+XPsU*h9tINnDb<2zv0mnmKdXfzOYTm3Ul?Wnz~G;NeqKps`chVlUeztEt@RW@
z+78BiJ6O}QN?a6OFRM&obbx1}Y|aqCZxyv1+(v;uiUN}h(ch?*qm5c4A}r~66c~;-
zX_XkXag%2D28piK-M=bj@2?64qzaP4jIdG_@IxngO3AkZ6v^FcCEW#7WEb8t9I&~g
z(l8Ggbt%Vk!mGx+&F21^j!7kjp8ko<@l>LQeW9O4OX9ymX_+trlk<(x|Fu?>1ebx|
z%WmCpP=G2~h;T4EK@rtoYlYqrjjN+^6-xX-6$)w9ftJe2@oEh<tS_MfL5H)@&3#68
z6SWC1M<qOzz_;1r>F+^dzRqhXu-22G6$mbKOYMhB-zKoR4ElOLH%1%s&%}@m`28O_
zCO!gHL4iE1h`AaK)H0tt!3OX{FZ1fXuZvrAwP0+)x{P=@mtzbr&cbl+j~=-+;^{B+
z3dTPd3(Tbxex-E^40qO4&m^x81x3m^CrDub!#8;rkt-)(X|-@iX>GtvzXaR;M1hH~
zFyLQd<a!8ApQV(2JkMW1l(-4z{Q*!zSk5(kuNS)bEppFcbiLCJ$EyG*`h9qELozYe
zXn5(+VqRVV<X*~^DXj1BZ9zsUT(YZ)B}U;o1-R=t!0Y#P@&`HLK9)&%M#AJ%{#Se_
zY$?oOJ=uwu)6cX<ocsi9<Yh3Gn{!;-4W|BJ@-_}5;_V9k?T$v6+(u^bWdi<tXyR48
z@klafw?X1#z&rh7o|RxW6@cDys=>NGVA${4vJa2l4W=^&hD=tAqNHHbT}3ALwQ9IV
z?F{gFZ}KuM6ICh5ui7)xRF28=XZb#Rk#xVwD*@zxGo~lSK$2fa-^ZGkq<qRLrMz5{
z=W-r}ou*^pPXUl@F9Y>##AAMKWd%-j5|OiKlw2cfjxdUHE@jevOV_W`Q82h@%4q<n
zP-yxtCJOGQJgDDMSi8*iX?Z7o4`B4K1=QHZ)Q6}}x&)#G0b_D-5hLHD$V)phSyXTd
zCtfF~RfakdNSqGdk?x~q0YG_EI~LC4iu_RtkG~76EqzSU`+=t@LU?NzmkVuF*jsaA
z4Q7`-HT!^AMau|T0f@dzhHC>BM70+0Ua`g&GE)}5+9_sXLduxh3ASRN)FyW3StgeU
z+bDkzo4UK!2-6j4#P>R&lb%p2zN*Oub{0gc2g8gCOn`i%hH0i(1P8Qh?L~~jLAipw
zY_MX##`VK+jT;Mc!eCyH7U-q$J_9AYz~(+9xD<n!UV{$vh}*cBF<CA%{cjj-53t@z
zO2EhDG1;TyL=0U!hTNc+aGmlZ{O*D&qfTBAk5P3<N^}V@gh{A%DHA&=5|>U)NngO5
z`p-MMq79aPktx63!6^i&xdM-P1_Z?&qS)WgYeNhtT2?7$p1<S_SpxLvi~MN0mCpiz
zvkFW*0ZsS!gkm}zc)LgFrxu`nFDGsCD+uUA1&Sj$!Q;}DlN~N!2bjKHm8(08PTE20
zq+iz;S8&IcmO7LGTzXd<3hsg6U0PM_Il_SE*s9qzNXlmr-z2ALm)WSnu^d0GV4gde
ze(;FyC>YX4MoSjS3D3liN`6(Ron=*V`zb~L62lV?X=`PO$s*T&N8<Z}*($gJU{PpJ
za|cYzizb)Xj0D&E6$)+`1bL^}_>(oJA2tHF3`5KIqs)yTEBaXUwt1;tAiui21emPQ
z-W|Cg=dG7aIk-!;uhMl{Zba^-kdNnqQU6h<_y%{}{2)`V19*h)76<3Rf}nlUovO4L
znRW@f=K{OScPY)|Rj%uU@YKmV4031PKPac~J`xoY5R!)>)_V?OYHbYQSLDWX^bCGo
zoq4!cE**b9;MdBf<Ic)I_kISkaCI%gulJ`>V3M-H#?St!l_>UgX-q3a?0q1uOHKa+
z;4Xq+E3Y0u&kVY!F5QOL#-IOcY3#IXBVajTGD_zkyK3tWy#N0{{PO=d`ttrg5cxv7
z4Sb<be9~?$_%)Sf3HLs1=qMkUtOjCcUBh})=#_e~n2jbBR)y{WvG4V<<^V#LKxs{5
z(}E^>&2mn%Ge>DOfCso)z-uYX_eMY-^_Z+T5KzPOGDjSieoqWTUqKlKV<?$x2_@`L
zg9_PR+2*iGvM9|o%6qK2uW=je%RN&8@T2$Q!h4MudedM%WE9jL|4(iPeBurYfp)@W
zL-9vBqKf1R^C^=b&n<^PQpovdX3y^-yeD7=;vaz=M?3LWikcgOLuEzegP<^|0gCt|
zdQ5X?NYN{Nv`0WOd?j-~;lW3$6(BzXOGE3SqGdgcmy<=lnC#+ZdIpr@go1!as-2@l
zSh3u#s#Q{*V422kMH5*z-p-WOPKk}6k!9e}M&c5xj9=G*h<^-r@CmCWfQw6^Xg7d)
z{)CHNbSi1(=02*_0ciCB4P=^L4zy~248t}?X(S;9M7^Jo+N~JWz*eg5I9Tb26-;pO
z)68pWN-YNr2{5WZ0>id~wBj^U1Uqw~VwUM9A0{;meFjUk$!U{MIbGP@3NX$x3~3O&
zt_!H`)ZLv@;QI(Vu7IAY8BRDKN_ROHbFd>Y;EoC`bmq0}5u{_e-lNpQ%gDjzBooM=
z)W-ozo#<Nf)jdTRWVWb4USN7nxe#`<%V7GGN(~jP%XB9sIPrEKte%u4<qcB4mApO&
zJG7K^oe2^P&3e31@&#d3>Z}3E`0FAUz<bx@HDQ(Vo?eAjKT9|fk@4^jf|jQ%6)b?P
zO0rOAaw}BcJ9;ewtKJ?GRwmEZS)!bf+?-H=I=&q?I{~`YDLo9f#(~J9&Kg}T(rYNN
zHBHX5IRRjRYDw1U5=w(Ov7kR^_r=ZT4Y|BLL*ZaC`W&+nH{a4*E}%LDuaumjpor|i
zN<nUAihyMW1^FE8=mu5_4k&iV&0-N2Fmrd<uu#P!>U@=}XRB(p*$6!vldp@4y+td#
z{dz47P3uloo-Gvnx4BC6XCt=@$AXttm1j}>_y>lhIPgfx<#S-oI{{-w<H-&BI@rb{
zy#(ZqwkKmd^Rf$2ZYt9n0`Jr{ic({lOvU;mZnrPYGllAgP%iPuC=f76&j8n?LeslD
zyJHlf#d0(VetfxJ@}L!{tkCi|gDGbIQ;H=A|7>IOs1n0%Srrp4gSEI$E2Vc}Ox*oR
zfXPvFk<CR%F?R{K?ob{79wPCfnXQ9MXw%G2HsouoR*Dt^R;BOd3uLtA5{n2q`V{>W
zK<i=~9E$vc%|WWuoZi08-Frs~dg~0?NW<ljsAU!?M6V)_Ejei~HIR^TI9Syu>y0=d
zG!zXXae0fE;*EH85Z{!;!lXrywW7mZ1?zz`hQQ*V-O2R3OQ@3i0+L9c;+uhqmRXHB
zqySL-EuGsV41FDhpTf)TY&7{KSkAQ-Cq5)s%!phxR2APsg#sqw_9BMn1wmP7ariSm
zBX&}=SjpdC>!uQWo{)Q3Xx;H`WjqbjD<OxFOl$9UeO1+o1?9hG9xbM)kW2Sh6g|IE
z&pda=Avt+01?Ik&&*E-n?_%C?)6Z%BZn7A~=PK^cba_)(F(YyEkLb-p8;t%3YAf|l
z?lc&>d=r_9FSNY;(GV^RzzTra-s8YpH&c?*U8>k>!ARh-89_J)9Ak>^cuy1x^4MY)
zWh?uEsCmx`@=P9wqNRf}XQod+?@2cF&l@I1JpJjIW|QKj`)ZUe=~Iej0(vCdbFdw6
zm{elQQ6-4~#(c@w>y6ld1JdH6C-vwLIU{qEw|0QR-es=YueH|Xoi8{=@jnA@dYv=-
zgO0L)7hXZr@)X$9Im4L#iw-9~XIN3@EG}o)AVj<HS}?6*AE(O2d4^)9-wdu-Q)wGZ
zN^!|R2&k?A-NVwC3XEbd5KjD{(NY$_&*?kynm4Ls?`=`2wC?I!IXPObc}|-RXnl2_
z+n<oy21wK}e1nE6@**?aX`m64h4#bvFtNsV{QKIVK*@SVnr56vfW1HrK~5RT#Hf<o
zV^C2Zg`rJNtm$ITN!LU6p~U46M>Bop5GTHTa8d43<?^;7rDD|5J5}=FdOMj+{ubJE
zJm~%Ja;6-pDE0zdbg6V%aMJ5q39W*1X3NEbd}y>{JzglsTR}z|vLLq(!ektn>j=&f
z1C-nz&k-#jSU;h!Rm;cE%WpY$8iBB_nV;)J049nrmbxj-NN%#_**3`4<jdO9O1CG^
z)|9i%Quoxw{QziyYj{fLJDdG=KuP;89~i|m$3BxqDQ}$qfBV3a|BwzACjXue=7!r~
zSezz&^1r-by%cfmiQ)k^I&#T&P}niVzj}Od`jH;+)m}-n$tNyjpz6To5It)u8=Y;i
zg#4KG2skc;hOdwpEdyLGrY+l^*@=m7P&Cx+O~^Brpvvedp_M`AGE3W!z6p2+l;7os
z%g8CWj2V;%Kc44%6KKqOe5O)Lw^#b&h{BG&UUL99LjGZ~vuy(4!MrE+H(<H<@oKVx
z*~hC+c)ZHM5K8`R<&Zc^d4HE`Q2Knf2lEpObbdmiNXiZJc2Gt|{thsv$ATJOOqmc}
zYB_{0%d)*=SWPEh53_fy+DL*uHOi7=8938_0`Tw{)NXn-kyP3ZaJfz;h28*^weCNr
zl#|!^*d#t(u_oy9G=)3i>7ZwCqgGRZj6SMlh;mel4mN0uP~Hi2jS@Z}vt#2FQ1mF3
zvOKd)u3OQ7CPi{na+2@Nx!7Kj<bC&X?F5f0u#&v$eL^V(a-)_7Z1p2?Q|8UP2;eB>
zKnag3V`G0Dr(I<)bNZ_s{e;)z9}DYa@*RCbSyp!&<*=;Q;@i}0R{;zvMssA)d<4yG
z9uKY>(u()T^HQ>0>rR>*T8??b9hikuaJ!lnT9_m6O}&*apX`7aM{kgocl;bE<di(j
z{2N@ZAf$5})WNf~MQ<10h+|^m>zP97Nd9`jvLSe-yBW(B_m#an!J)~u?S${puGNPM
zrSJ#TP1;Qm?x<vQp+jO5|E~&`IceQwh+2Z?j}t%WQEYMv!T{Ry2?wIuQ;JFXa&nQ*
zO7S-(PD+V1-uY(9|1#H?`0->tPg(MHH>Z*Bf^wcI$&Rwr+$WXp#6Q8Xf$zP$A$$Ah
z%I>Q$uXQ8KsI3E%>3eGsAd2_=5ZrF6#VH#ja12UHMHjbfOuHU6ngsj;k_^g}zQ#g{
zRit!lZWnT;aQ_@e_KTdgP?zU}nO>PEd3M}a0O~-8vz)~EF{S2yk+V-%9G@~CDHZQ!
zKH2s7l&^}!o2{GzFo#vUjAcnhlqj63@5s(M9);@s)#P*Cm6wCdpyPMt)*GBMaJhZC
zUJf^~n%5M7&RIZaIc(;sh3>@rC~sxx5}oZ*#a>#?YY3X_lxD;S?DN0QHN5adboi}~
zEKQgEkKPghkcoPhAgDJw8;M6rE-_|8o@6kwR<EIe<jB9*$Y8p0k^wtvN)AJIhGa%D
zhnc`+(e4D=#o|HpyR--62ef7KQ&@~+RVpB>LaBckpPbez_%^+Euu3jVx^~>mwY!mt
zH+aqcZQ0sUK@~;*u*AfJM#GDL0R#P@0R=I`80|JEa3{;AZ(uPFFEIn-vAPrD_8HJe
zBG8ZnH7EQ4z~{jdMv8W8Y@iiLTN6Gc#*v7!Th0L){}0Lkft1Jevd~X0fj*6ZJ@n53
zlD<%sd;=m7&g(SE2?}e^#Qu2hJrE;7ZmDC6<=}Ob2#HBKaB`<A6Ei`7YQT`R`N!Rf
zm*dB=VDki?ULkzAA2=gebSUhOT(*$qceVv!Z|%a9XfK0>N&OTBfhc4<Mt7k&=y)^p
z&^z95+W~{okn_P?{z;C|Q;b%AY<ipOL|?a4b`+Q6(2{4Q6V)hoPS|xO#_<V-3UJFJ
z$n#p=fZ|xBx01!`!9UAo|4r!K8AgE|jnoQpV7n844qS6f8RP<VNZ+c-t2>I0|GV}q
zjt=-+iytz8zOp10N*Wg!=iq6jl`~G;4$I&QwP0DEbd>#MtsZ=#i5sm(5FQFpaB#a)
zZ=S7COL`##(KDFN<qR*@<Y$$lJOIphaMAc@t`_%kYgs<q#2_DP1Md46kcyD5ISBZI
z9Npv^zW}QzAn!AH1MHi88glj*gUhc5F|iVYeGlg4W~iD}F!7?yyzR0^ZrG`Lp>(va
zd!Zj758_1CiN~85o$WBb(MhV_HNr_|4lT%zI+r~KuD{9U_XZcF*1_evD$^hA>P~{`
zdHVoO4lZ+hc1in=?W7L?0sB2ACQk!}{o3kIn}4DhkmEgBV41p94;j>;!h*bs0MGGi
z$-TR{SK5sDg#J!@Kg|YKpJKzd<VF~v_!6+Ho}Q+VI~3%(O8sH-CC<c$Op=TH6iT;C
zIgo!>+Eu2AHz}_*dI?LON-S8OPoM#U^-N!2gS@|H7HCg^6ct$Pwsw@*kgAAs4yJ3&
zfX3Pc2_QM#tHSj4Bc1g3?L{XhZ%BGZp%PNU_d%FSyIqlYl$m-r`OH!`o?~naS&~C+
zB^_6><;H=f{aI2z`OFBl^=GcLEaB`1{6>&$73J-3rRXHNz*?zdH>_G3&7}0RrQ0@T
zZmT)|(m_G@z_tCCnnCxf<x^%+-r4^R*WCZKo$}wa$sP;hZE!7hlm8!Bqg4D}4@e0s
z6{9<O37O>HB%Ia?TaF^QHr8uqA)sE5wq8=?v4kh|C22qrPLI&kr)*@BE}^oJ$|#6O
zl(mc+6lsLT?e+YDY!ji6B1pTnGw)3a>BYc2nQMY}oMjPI)Tnsd3J8?+(9VI8esdSz
zmo~E`kQB_eC!x$XO5dj}5kQ<{X|a#zBfXFuQ3-j-TFKq(SqVb1SCV*q;;|~_ws}t|
zl%>OHk5mF{X7;+x9h5rE=;#Q_JwuL8LSDWQ=6ZAh9gsU#wF%-=cDTQhX<N0vB*{|N
z+{|(Hlu}7AZDB_znWSf%7Aip>Z|!ryDkn_b$0_}dlG5^ypJk7bW6__Fm41Z|<(#8r
z|0N8pYi#OaDga}smN!{119Xf{$)^e89ubTAT;swM@-C=7*fKhrK+!Me-6=+nHsw(A
zA7RxHfH`d%npqF-q~uYiU#B%v@2B8G{j^Pxj>lM|chg*2KjqnuAPIe5O8fOj8eFes
zo15!VF;|XglV~wV(G;FNc`vOm4Z?*w#-5^}>Ai+_tBNNTa+usKF_0pfm1JiEV@1k~
zcA(0crqX?xBT=F7(sUq%9~k6CB+rLp|CE<fh4v3p!IZpWqXK|$Exc8=F6MHGfprP!
zsPzccA)x9~+&Owm=+Bxc*?_x#3`4Pl-~gpgNTHcC1(1QNiQCj#c(*~VOV?`|gvpX*
zNdlbkNR``<D&kg9+}-I=h#wZR<Sp@i%kjTJx0g{&c?YE&k@sZ}t%bniV^qo{jBZo&
zQqPf#mFdSAjWl~iMEe|02KAe-*ZhNv7J&72Ii3lOzN_w_2zAb+V=gdR`koq%Bcz9t
zv&rmbKIlt$K8uYfo3wIJ1w;*-lsQF4X^cekH)~FG7P$1Dnj-BY)9Lym8xnU%YcFxh
zwTg{2sux<}V=y3nC59=Yz=aL=dp#_|08y9M@uIyn#XeKfGhfx~z<x?QuP5WVIM-~%
z9)-xCq_m^`b9$w%LA=$JFOx5V_NYFEp(aNHZE|}`ArWqbSP=Ql3H3D?^c<M=*Jzvp
zNi&BZ`ip|rHvW22$cc%1Rv-@~AI|_2vgkyPL0R*hyoCB%{C*%5@@;LL_9&N$UU{_4
zqwnOQ1gsp>v68$jTu08kzvXDFi#!?l-FsM4g1f%49SdZVZYQ>EL+AI$EqiN@cMR;y
z-n18Zl1?B-K2$I<x?Ik}2q!QK{z-tBpir4Yp*Wd;9N<S<YGV}3Ec@Th*TPG=^*eGN
zWq4noQh_MarrJ*|Zr`DbbA;nvjwK#mW1+MED2hXiwN%&aPZ*_mL;YNW3BS*fbE~a%
z?JVuky_io+L*Ab&8%_$IX`d~0h->;2OwAY!>(Ckm;7BN?F__HEFMhWne5l7;(Vw$O
z<^g5gkE|LB?A<jw3&CElv2&_zxo;2ZbDK?yF2_#;R=mO|dEQxVKo+;t&f^7omI5j^
zf}}PCZGmq2c~3NI@vFAomu*0(r|8$y0;rm{3=j>*Op2Xs5UnqBdGfE(o|UrqmhQx}
z(ZJsTpEQ6^cCoW-+B1n~oFg*g`nRq^_LhqVa~&W*Ii}5ioy(5~DbRuXG)o>#4maA;
z8@$fR^MVQmxnH@p0|>CsNhsMN9Sp8=DKEQUYECkwWRmW(bDFW3{Ge#$U6`<WwCnl{
zyy7m^UM1ugkVeOVs>bVfkfwZ{1UfXu!VU@_qLpMqq9s-(xr9qv0vb@#=YWu%#r)s8
zD53Z+Altiv++uLr^cIk+&LSB3aVn5nDCBh72>TZF^>oJAM|tf6k^5)P-h_cgfqbdT
za$^m7A;viF<nnEsv8L3PGr0<)GXSvqd7ZH_h6ekyTESiN@fx=e)W$^VLN2>Y1^K>1
zn2j-Cp}V##VKJOxaQje=J<x``<Y7?YuVYEB;f&2GsME1d@_AF9JA}(84aN@FsM5pa
zyL_Y#g1OGD#EU>v-V&CogSLmzfj3zg;kA(cQREoC9pm*=$*RG0CQ%^ogVIA8n^DyI
zc~(NIY4&h7#%rRzzXz2Wz>s>s(v7m#aN|dddD$nq{c1tzdj7f5_S>{&oD2&4dtQTN
z-Gn<fIfQhLJ$1*M0*dl+mD>-hqvLo!&+p+_z|?0eFamU>2vk=X?1beCfDMV!?pFs!
zqS@h`Tx*Zz_)9)29XtJ6+6dV_eff6*sE(Vyd<x*zu>;pmnP>VROkduA=^)w;fWN5b
zcq#;ByVUHwaWx?@ptEbI&~K|-_W`8MGlT7@kbWb#TWjMbP~GbP8>Gem0i^l=?o<0G
zbFm6@gwoK7h&EXoy@YdmV#qllD<$}E18>WqHL$*W2%L=Zo6A_3@c!w{!kYEKi|h@o
zl1^pSbUdYpQBbfa1{SCU<a29oAO~7Fo%Ohw$I*VYoQ;j#<>Zj-C8u4lm!TI*X<H7F
z-%N{rGpkaDhd-Tm5kxDn5?A2qn*o+KQ>=3}MYsC-=rGv~zzBn8ujSGOOpBv_UWwQ8
zu^zf$E#$53=-z6jN;&B0Ti%I=t3XHoGM3xO<a31jXl<Yx6wp}e8Yv^`5nh_W$&cr{
zSRSy)vV&4ICh)O7+G$<Rv#oP=_gMw9;wqDUpHDelqs8VuOi{79${4qKo7V7yEwCEl
zqvW~UOE<398-BP2aAXDJJ)sw%Wm{mAmou{w1dl-NPRC0s8c+ae_mv#bW(*d$e<ifg
zmE5E}u4~bumz8ROqI4?;EP+jrx1#-%c`evZwJuU*UWaK~2c7z|;-Cm7Dxwj?qm%vV
z^d<7iW1Qtg&WtnFMvzRTG~c4m>&=T4V+E5}kg42)8nME~I%t>$1_tV{)Q6$kXg|D2
zp(4zUcYz|$R54rU=H$It{=bJZzZZn(N!9V*rUd`Xh*!U;RJ=2`l|&(o$vvp@V4ezK
zE~zD;s}-mj$1BpFh#J=YQo%;Y?i#?r-^0dw{g{+ZRH>hX_3mB(rp2XwmsasMf*G&M
zS;uIy4=i=A!m8m?nnBz(ldSzQS~bi80WB7Jc{w>pXQ&motYtemq*?&#*ixvt?{L7K
zJ17kNDMi-q&??cxK)>UG37(_m^tHU=_VZeFglW&sNv)94V}C8~O6CG#N6$hhKw#!7
zwdiounyWbOo)`<XrsBN=IZV;h;9N?z&sU<ptN?oOR&=xzW3fk}7GJO13%I-v?FshM
z#R62>6CDqh(|XvWU`zsCybU9K7{%*O&Ku7uwRkVx-pu48id>X5Bea}&By{5TFyRw<
zZlr3RkLe#{Lxdrb6nuG#LI!0zZULeY(rFD;63&8AnNmb=5^~i&0(41|jfGtagkDRY
z$k}VSJWt6Ex2YsLibkol0K&^LDf_me8b&iAb?hlTn@YD}Hj-dhFs0~EFuci3)z}r>
zTZO`qKcanFU$j+$@#ISw75{sM))r!e`n&;4V=TEOA5x9WxcpU{Dc8cJK;s&tb57#m
zCDXAM1A`g3H=PV)M!Pj*MX!VnrIrp;EhzaEsLp#dCzi23*9-0{0WX_Oeo$cgWxU~f
z52M%d{S<Q^2~c*TjiTk`vU>s6Y$FqGupQosc0ro3`O)P7OT$%^%%$@m9-uXFiYk6q
z$Vv+Gs66VtV3gg>y5nxwY1xBqInc(cP&T=_ioHUYzbJ4y&*0iCMaS*##0pqL6yk7A
zd{Sw+{?B1gJkX5r9jw>qf%dv=lfVY)GchxV8e|IHJs?a+XsjHaX>BXQa>kktjB<1&
z+Ux$BHa*;9Gg&Azkrq+%SD=vet~`^UR+tcax9h&k?ei7+B9^2#O06W1s1DsGztBPb
zEGAwwaMwD3;uk4#@Ct66h6?LQ73`0<-)}=jwxPyvaLehG*LAuoypC-Jm$xhSkJM^>
z0coG2q(y!hEVuuCA!nh_P<a^b6#%rW(SEO6EkXfyil*hbV>*j+wZ-Lqc{BT^AQua)
z3dzLc3UH%)Up_Bu<XU_|srZ@45+hdCxO@N&a_nxGy!1o!PB;p1_!5JO^AxA~H>i=M
zPG^PXA)Z*2xXZp%lWjl)ySL>adS$GoXK7(iY_G=4=^85V84FTUVdda{$VAiW!ulh{
zbJY-L+%^a$>)Ii&D=5YGFjKTi^qn5_7|_&%HZ$6!lMc7D&T=^#?DrbcXr|W)0z9Ay
zTcFu*k+t*!#Tui^le@UQdN>;Q93b;Z+kO%JqN*|bIjfwUrsBSNv;pM}gUK&-5QC#N
z6s;)0J_1%cc}8;sBWB_~Ty8Ev=RJ_mwTzd&ydcl4LGNIG{nQv{e;|yLz?;{$4+Zvr
zR?#=I?q>QI6{7AW&<yq}rJOb?A>{~Q(r;A&g@-faCsi=_bx@fdnquuT1dGuL0x*-s
z(0mWtc&ZapBI_YEGbgFyy-K$m&gYh`$(uoL=zy!RVL1_P(MnE&f;m{@`c0bS9zUue
zd+7|jwL*-kHQeY;)bMF}uC$o30Y<+^$;*Qs#p;1=T+ec+=IkM=d>h2!O)l;jg*(;_
z1|J48?uN{+GAOp{q|ZRLK9tkOw*!*7Q9LK#f@Flnj-7Vx`18LGx>qkBPbg{?uow{5
zK$hI!J+Sl}Xay|x;k4y+HFov2;|YYVT23{)2Nq{HOry-Rpj9)w;e0xjj4&LYxqKYG
z=)WBn`yaxb{=XkX_EwU9gUH;7&`NJis4D@&8RLbm_-4L0#AQ#4%h9esz{bYSffh*H
zlttp|;YKzlqz#wAR@Sb(*9Qgy^a9`+V4=q+FQ6@Jy+Lz2o%IDf5KQ2W*k4Y8y9t~R
z#Vf`n?j{xnd?Ikpri?#lC)JJ#BHE|6kwTdGvmW6^aRQhYEu*9>7z5zIY8KS&<J)$S
z25<AxlsVrByeu^MLt>k|axouP-0$NQtnK4HfwzE<@zF#l*a-c98Rxxph8h&Hg9j93
zJ0!|`P&Hho5k#BG4fYT*z=gb$bfC%_JSX=1$@Fk>F6ty}cwUmGPABkcxQ18Ljm-KX
zl-V$q=r20$ovJ#a0*IA8XoOnvpH-*;I$$&Tx0azJgh*l9feoPz+cg8aCmcz+=bJzw
z<|<VvBRNu~ZKJMIk0y#rvWeC3YPkj4vP9msz^DKmj7^)qVpGEMNYa`|X~rTzx3%Ea
zc(ObS=6{`V(si`G><+r?T|n3ij2cSFy1xg(U&Jc_B#jix2&?JcY-^@!Twsh%!xt2L
zIm>K;eS=;Nm$NFl>?>NQO)>HK3%wTKnb~vud8H1GP)T68QDn+TVP*Zfit&g#HfaJ$
zKzUrRGkdFE@%Dm=2L+tQvcu}!x41DY=cJS9$k^*C6nYu2#&7Z5W!!oT)M!|?$>IW|
z;=PN0Tt>_JN|DJTAa%oQPD0Pp1&|IpAnWM-*f4yH6A<X*oyH?M=xxFv8;SY8&8x{4
zF!(mgEu#|eTD|IS)pHN1_PL6a5+a=d%(_~U=R!LsH$ZDV2CcCf?LC*0(jBOJZk?-C
zUGyfPli9LU#SR6SyTPm_1G?oZa+wZxtCO;vMH@8FgE<V}=EfMyNoEm1noHQfLo0`r
zSOZ*Ww)PH(&njj2R5dGty`#+<=PF`?K*1xqmN}N+qSFTYMYrc|)(JG-k!Lof{iyt2
zp6Bu@ciQ%9(z9e!fO^FsyWx3HS{Seq03*S}Ii}4uaPwbsG(IE9<@XLRdxvWcDl5o8
z*_^1ocuvXggDCDzfs9Q|l=3Li^cH8Xhc6oX^>m5`#1p8Q%bAp*6VjuB3IVf|cPK7=
zq*jS%phJ+fe3t!Zt9pe(eExedG~a6j9LKZ?kIwX12cShqj|SUWgcCe}hmN7Lsc`)|
z6Bp;g63n7wK%0sI`SQmSzyh$9?-iK%eF-(@DlKuJB0%t;u=+edSx64L888j|n#`C1
zTlFgdhp(tZU+AJM-|Yo?(;lVD8Xdcv7!|DiTJVd4MSAdaEU{^24EA>l4KFy}F3Pp!
zlG+4yHV>fp2A%xzu#|Y6*sTp!Zopd46LkJil>OuB;e+z_heCb_;POn^5m2t?Ca|f6
zv?b){!njkbHBFO53hktg>(w};jFzW3@!oj*$_^%uxA*dz5%1qFM;q|+W|r0=Rsnyc
z#X^?MDKWcP0$0+ZaSy85o)22>%?>&b7R~YA<`w@v;kX?HKM3plSBXk^3Ut*~qDTX5
zCx)0?b-az`0#Q*WhX84O4b)pM$S-NIE{AgX66E5yFy3?kY4AsIqFs3-Y_3u=U24C@
z#lae-lV)gD4y!S-T}Q(U&mhU00I;>6gho6?mFvr}q<fj%ZYb4cS1$iRuIYDZdu8Uh
zt&v4JR3ill4Fr1NH$iilZpvRvDBnQ<y$`6ls|ut_X5^S`!(EqFus-I3MEw?3AmCLO
zTJv%m4j3DIMoxT>&WW3m+va8-(ut?e$T@&10NbZRI9{)0#|LBwC`mpDD1;j08jim!
zH0VP>Keqx6?Ju2xc={>0KHUNV`KV3y4=czubbR7)wF1Ee?iK|(q{ih{1@K5smlKB-
zCAwdS)~rOQQu<@f&dU*K`R}`!aZTBYo0R9wb#6au8FQ5Ugkkm{^Kvt^&O<rC)3W1z
zf>~GzjJ7w8Ak&@j97=<xM3QHV%s2$2eoMYcYgS%93}|?v$hFle?EoVvK|q89Gc=60
z|4<12T!ZBtRsI3Wd78sz9lsw^@AhI%!m!F`GjdXiVIhAx$XXy|I+%`R%dJ!-=%}1M
z1gLA6S^>$dB@;OL@7~7D{Ss_o_7KOrW>AsLqy4t1_)Kpk9yjxHmBH+3rm;kKlJPVp
z=O~$SuLSdj6P*q9>7H$5?KEx%lluql@WOJ*ccHK6U_76w7BgqbdYu`wI-KYasIZhy
zM1jG$KUa%>s9M9+%t0iZMW*!$U^6cc=9n2ayEgN1WiGLx8-EehBIntl(lM)-jz5ba
zT2_T(lZ3IJSxTO<V^;o~M~$Ez#VwB=$dKBx*;5GUO}R9`wL8OR!PsoyCaV0GBgy>c
zf7pll@BXs4;@}%_=EWg-rXqhj<>60fqdokq5d~BjxYJp$k8804$Y=#C^!i}iJybk8
z@>Wnr*$O&W0>G&!j_F?f(MZzitS({SMu(_PS>lu4<KuuJ*TGs}L9mQubRuu0E;-4F
z`uP)q6E-uTbYM3qaTt}M_~)1a;Eh%bfFhk;5xxl8G!M*po?=a)ZNp<#5{@x`6QpMz
z9lp|!#|N7UVNTyxGLI6Fex#ORa2tLzphYl|?A>w@k+r<fZ@P;B@Z!ga3oqj0I1NSK
z`4lS+D2)qhOkc#i=}x-<*&>%-_?TKs7t*~zNf4%Ew9J&3{pXd~Z^HPH(jYVI=Jayt
zDSbYrr$DWpNA9hPMrO!y*R(9J{GMQATu6B}*u<<^YOfbRqSlkLR*lImL2kaxisN4(
zzUaaX%K0d38EnHCIWgHDbR6w}SS^P$)jlu9Smt{@K&(1onCRyi(&x}6pL`*J(7D?+
z`5~Qb)=#I&he9tW7aK5)-=<t+Ujd?~lcpx1hptagcSot28>#G#GY;`xpy{6)h69^R
zJv9=;Nj#;{8+vP?vC)QP8SC>B{F#@4F!r;G0l<^c)=UCZfMU6>JZ;p-%ilvke+r7a
z83sOikRm7zCG+SQ52{XZ3#DFNOF3ozSm`Ut0hA+oQS7Et_TS>MA4Kh%JxMqLRw4c_
zC;QO}Rfpj|-yz549(pC7C;*~=NYUO^XiU3C<17y+Ye15hpbEs<2_wpBdyWQL(~XH3
z`R^A{um}^hf^q#t;D)e^j?ugT%5A$GH2ZeB8cM8-3@2PiTSY?1=KI14$53qOGg#G^
z4KBKY312o^2ElP5QRKR+Qg{Sf)A1p70%V!)k;l=QSI1~Z6^iykBO}B0h_L(fN^XDC
zdIvZgLME?<!jB@uiFPV(19L%WRDz>HN6CnMp>uMP;iseNkX6D);Xb<fxxl=oy#Gba
zn-7?7FW}-bh(+@~u;R~Xf7FtUqIeYdxWNRJ&x3|9%i<tTybo-%|G8el$U5H9Faozg
z;?E@0bT|qQCr9JUK%Q|wuS7q#9Pc6;%XeTCpU73BX94h{wIDG>C5A))P6EuNLwMxw
zFh+Q7E#L6HL-AfZW%5Gu-BCm4;TiD)0+%`$a}~$W3RXb>;yFf9I^;N#znA17W8&1#
zhMyI@-HvgBlnF>hR`5G?2IDS9qDHhA?EC3C4o<<0rETQD#X5hU(mppa<0*>@vPaa@
z7ZveZvEj#5@HnBtJGDysl2VH|w%NN8LT+RB`=a8lN2Bhw3-Sd@HvoA3d8ONr@5++~
z_&J^H`lFn<0lY;o!@M~WX_@#Uqv8uNgS%lH-)ZNPyj_4{keXj%gx+aqg1(+{kSxj#
zKz$=sE+~Ak5vp>hB5jc|j?Fth`MN%U0o%`nIP8#f{1SmK3MTk~m+rF^YRwm99|fYZ
zoJ_Okmo@7*0IqFxZr((0Kw^U5nkLv?sp4&)s|6*^T25|6wFRd`@GX?tGgbL*I<o9x
zRRC=V-vaO$rDBkuwlW_d0F)`eSKxBD!Axuzj6qH@JCBgFTIj|eZM8_~_ChXcl%t=E
z3iO^_Fr+Bw7cn|t=#_Am>O>XVBKrk;|NX(tm|CEMf*eZT$bN3zpp!$^1S;loOeZt$
zx1GcXJ~}g3GP-M-19uJ9>pqaC=M?c?1&VAv)@}lAD_`tj#^Ai;E`+k(7hGutGp1CW
z_yi37D+VVguyM`cTD*^LAFatx2ZJS`odlr$9iVEy(z(n*+NxSD`K7?kPB_zsT>)H1
zt5H<Q{9F<&)(9+ln^TnIhbT6TS$eF7%YqyzpqF=P#yU!)8U;jlr>0ngx{OXKZ^6%#
zhl6u|2x+Cm3TM#{<z0)6+|$O<rCR!N#j0qsX>obc2&}dBg<5Kx^brZmq(<;c^E?pA
zA9O+6J&V3R+R4QB;I@x+n(}kBe2xXp`WLk;kPZd65<+q>i~wc1qP-C`J4rnZ1|4yW
zrZ~t%NkSVThXc<IW1?N6;B-8wL*rONOkWM{yAdGtmxDPi;@nbV^BG8Y{I5+cw{|C~
zVO0M*l#^S}Xi{!mXT2U?AwcdMss9HpPo85MJpCM{%YSI3JwZsQWO=BDHhmZR_#YU*
z!|=WwZ7?c!2hFv@0(%HR%H?S_R!Js^TEeq9!39+&=Zs+5s7~N8KMCg)S~zU{rTHx%
ztX@vx$73=dS#a#?rBlwrUn^V4i_jISY>Q=PUrWmN0JPj__zO2=z>QA6z*DvxxPHo|
zbX5P+mj5hsmC=2;eeCN0g2U4P;3SLxJ-O^NmL}f-vA}KeiPN*gVF$8g{Gs7Dekw36
zGuv>~;{#ibiM;7BuzV71Z*Ksb-je_#jiE?q6>7qszzHl+zA$YBsN#Q0JXlKS$}VAJ
z!t4`G<gW69rFc&tALFLv_{snu0H6bXLFu!keSy2k<K}^`R)Hy6QO9HfC7Zv^$Amu6
zSxW1FIBZqA2=7}?PLk!c{WGE8D034)&2#C#;1tq+m(x-H*)|_g)H-VPguy~y3IC$j
zyb-GPoI=5ZUP`sAG|34<T?aaXnLAZAJY}pGf)G5UIx!(0APzwuh3GG8UjC7`27srM
z+sp&Cz>?ZX-;J68Y<UDM$_UEMCn8T#ouo4>ftuKrj5iU_xpdg+X7q*vYZO)TMuNys
z26a7(Qr55F6?fe>qat82{TGyKv;|`lfSRHJu=rgr7GXUs;#kPvqJ2Ua8WnWY30|c5
zqFd7BOf~cA2e*;7K7t<G$1BM@ycR6r78>c!&5~OF0+x5Z-U$5jV19_9rau!EU_vLj
zL(3?b6F^%Uq>wzky`m{t6b7{H%d{DQND8!gC4}V(UN-E#btk<7#Gm|p8=1TVx^*A=
z;?4+_B0JUebKUY4C+-GW1YW2_?}=LYk^zDy$skdvICd5C7q$>&RuVj<)<E{jw?nQ7
zPt_}ttBm!dU5VL&u5tVHTqV9$t2N)I@S=RlQ0$UM@JH3rMpr2b=oEjoQnXc_;B&=a
zPG4mJP?i^x8*8{~d|asc<BAYy`5=J)Et)tz2V#3DG|5V0ce8RN+~CM)7acec{#yyb
zLSqVmd25Ib&rvGw17O3!Wtoc-0|`p#HqhHMEwtiGlk!Wrcvk>g&|Z21EtNdUIxH(5
z@-D6?ib}MH-iVVB3i9G2mmd=ZgKWf<{}Jgllou$*^)OTW(>;^O&p1UGPg_nxU@+Lv
z<kp?}N=P-RAOaqd_u5eD3wbR$QnNnOGsi4CR7qX{<sGfkh&^v);$a+vC3mH$*ORw-
z72_{|Y?)b>`>O<%-i1osqEl+eRRzT+Fnhb^B=3q$MnSa=9WQ-22ViJ&C;PNVn0Ymy
zA*3*XCfo|Rc`Kpelc8llhfvK?C>WaMF4=$7#<<NU3T`25F^}?|cZZU}=ztbbgtVSs
zC1^vY6UD`%ypFcf18sZ@!$Lk`u9HNv3dn~O85o<87HxcI-T(w&(|U(oj(6^^*TO+I
zM&7~d#=}&di9K{D+yi*+=X8Qa33%jd>4Zd@K5Zipbj5!OAo$ZZ4ESV0-)>WkL3!eK
z1qe8Occy6UqdU<(!0ZvNMSlx3p3D&hD^eg$o&gN+-b86XF!XN}ftqcYomUJet^>{;
z2QxIb#zX`IOV_7iaBeN(rH`q#^j&U!Bpi2XfjQnr=*s=|8c-10L|jgy*f=oTfn1Q_
zQc_AL?(AxW-myeW2rm2*2LHAaN*?Si$g4$09`yyhFZ9y0t=2bG?4KBAucUQ{{{5Y{
z(5Xvyr>2vOWuRPiBKoq2e{r$w9Y<%}d<huWUs3GCwQ@8?bcF{>>ug0nK_~OCRK!b0
ztBM0U1W_fI+FN|GAMz>lhGb!S%+eoAT=cbfg_pHknA`#|^Zh^*yO-%q9Z-DvGeFHB
z4Q0j&^t$V5i@;<eoa=bmMXSPfm7;vW1_%Q?5?28w9tj=uFxKp8su+Svr9-Wb>*8|U
z5GJ0n`oaJN)|63|+rZhR)}Exjx<}HvDu2q0)g6s67!Aa?5a@U~5Z`Fkai50NV@1ew
z@rF(1M(A#6YbA)){d;Y3^|0>Hea5n8sPgza&lXp*3M8AH;1tb4W1z)u(D5q@NTbG|
z>p~yogj4{MThiZZ$BcXqfjd`G?8~|Mt<xQPs}*QZ`ALD>ZIHmLgo6^Z;Wmi?ePwd(
zPj%eXSt?{U!(9%&_`axlXIYdq;kTpGM+GKMvb)3PX*tqk<t`4|Ai$&C0YdzHK>=N0
zF!9(>^yt+h7TM|0!hZ#|3eQz)?&l^c)6WKBoy}D$K|iGK<6+P!KPxbMjWDjqpq;zT
zQ+V%4X0+q6f0w5G@dz&G;;DNMYJ|;QIVnSO1+P}b6BbGaL7;zDfW(7e2vv8Q2g`z-
z+rj10U91Z0FKH1SQNqZAS*Ay7PIDc3)X1s+oFS%T@S7Cj$;c<3(HU)IKw&`|%_;ty
zH*8hQ8%BSbK^A}P+S)z6mHh?yHT!GwhPC34S)N$tH$#<yrQao_wKCg@c?{hSgtj_E
zZxra9XJ!bk71L}%y8nX0;{R|eS(N^}H!Su41;Wxe!$ARgv;eclBykH7ndJZCLxGRR
zkXMT`t$?7C6|9Hu80Ch5WMQ*Or78KyBK$EXB{=0yha#9nn-uZTGVW^x<Z*N2EsQl>
z{Az{~pA;+PA&Zn=^4-XFoP0vdvo9~@DPf2YO_s4S-nQdt^S!>eEZZAZr2|X@+9*TL
zes`Xd;f)X_y`H}U%PCNDFG3X865hkRD1WJp|NDY8m`Ep101NfR;UZop_gj#jPf27{
zMqa?tVel@8T+-h5^IP_!49JJ+&)jHqij}F0yS2VB@+fRduA@3^N7>8kjWB|)ZbVH;
zp1p}x@$TFjZvT%ivm~JNluCT3M(HE4JipO1e_SP5!y8fbJVnDOFTp=w$t~h?G36Wt
zWbgOWt7tU3Ym)o79v!RYCUWxA<()V?opu*vG*}i4^D&y!0-t&C>{+eAQP%vZk?RX%
zkA}kyUr_4!SJ`{hI3x@`KJk~ra9niMrqg!cLC^imX^1F129jCvj?!x3Ow~#a+84y+
z$$7I8qJR!LGFTQ#5br<-Qi)G)@?QXtc;J#*iFXO5s?1a?7$qlcr>{F+L3=>)KH~~5
zsgbW6r3!k3L8C4uH^P8_M^sZo2W&I}*|<E!WX2yd+X6cd-e@m{oM4EZl%R-5E%ooq
zSE4&KqLcM{<SE9hU|+#eP`8^Ypt7797o#9N3Lxqx6CL}NUJ0I68fkV?GWAt%C@}qa
z%;tGkQ9em~7%nnE2O3GZ8m$T`6O8;{cv3AHU$oBv;~W9&D0BkeS<$u={p@FzO86jn
z^5HNB?~x0ksffjeMws+bM2S=qYzx6nCq05}BLz+Hc-rbwivbGzv+Q4P@2L|QNGL;O
z6=hoh(5DaDlyPRty=ZsSuAG3ZyruxDe2Dh&YphD=pxvTTe1cj>X1a<oB^;BSZ9xKm
zXd8HFe1XvlX}n}G0|X}~bpNw<4Ah`}#Rqo6DlN!il-vBdUh%E~E26x*=qgz4PY4EH
zXw*<*{W&l9rXwmxs>XF~PK@<O$-R`CL>^hp=vrPR8wAtJ$}Y9DU7{s-=d++!*?r0A
zj+#el)&xxD|CpKgxDr#sCQV!|&*h3X=#HO>M&zCj&~TXKa(5_iS}^V<MUxncoYe`w
zf3DIUxvpZN-(Lck$ET{fjVyOb(FQCGDCBmkO37(38MIN5G8ir{(pbqV$UDvLfq*MT
zxmV=Id<r1h28_uZh0=(PugpdiP^{XgdaG%3r>bW~+aksfqXAh@U<B<g81yzSK`O|5
z7f{S}551KM)>Fe>TXhUk2}bidg~*>1{GwpWscqbz(OCs~lUrzC4>^C+YxQch6wUaM
z5+z3fRP3&c<+>A`-9b2*x`LK$fLh9yMes+;;+d5*PVQ`^DW%{&sM2|~Tz?;0^4P8-
zaS3^Jo{66o`l6;wUTQk@ngV>6@mljF)mjEbGp;>5l%o<$n7@-~u+GM-9<%7QW~aH4
z5D=|4*TE3bik;n2BpBxaa4`eA?qt=zS%U;~VkN>=8ZO{Fmd`4MGE2lmHB8e3IjmpV
z1~9tHVCDE2n~nxALwQ^-0E~*_pb`pp6_j4a?P+?&pKCbYtL?2Y2MV(nfiO?RIw7R_
z0!;o$(CFtB8#sudb}~{GHM^Tqgej`qJrr8dSH!sXtcbA1ty<PX{rO@e@@J?;r&)$5
zeFi8z0|ak|N=Y+sSGYW|L_tE_c5!<awAN@3)C6w4i{>r@I-pOU$;pFR?j4?c7()ns
zcsM$?a|O$p$T!jWkYb?s165BS)F6vt%B?!+GUiVJWcY*`cORtK*XiQ4p=b?V-;U;e
zT4v&JPOBgb-pFeQF>wG&Jjxx6DYO1YXrHq>P}12|kgSYuZY&!B^B%#-b4Sn)1ZKdV
z28WcOc<C9(=b3SVB9{y+%D0Deiq#wIxcr?%S=*d)0*%K2w#ogwqRc4=u;T9rxj4uv
z%DeHxeRVE=DddNP$t$a5kp2nOKM3gSaTTg=2-F97Y4{b1@c|B#-zOm0*&T(?EA~f1
zz78Jw=XwSpqi{c+e2q+=SLNc&oP%pqzBB}&Z$BOCTdu=P`Y{K$buk0RCt6c6W!os|
zy+7wG0p{@R5#&41`lvlN6i_UR{5jS)Nn?Y8vCk}@awX8y^raL;JZ9Ck{g-@Az--*i
zuLJrz5<#lzlnV9@S{u8Hvh(SRHe!CTGON<PEpvbRwe(dbf^d{H21J%UY-t9?{>x^|
z;6LQ?C;tC~#Qv2u7SY+F`D~wMPt+U$7=c-6q~uBJb@5YnG)AS_$^+Z@P1aa%3eeV*
zLLF7pZ<6j~m%tn@v%MGn+-wA7YKA71iI!|;VaO+LX4V=`-j6zLewB7<M!bI-`Pb-p
z*gl?pMks!Z&e6&!$2rQFm=5Vk=4VJ^Dcb0b=hbwM5~h6024;ga2UwxK6!{w+1lO`g
zOsVn!hbw4@DC{2K+1Qx!{Z}yZPqU%)(LL`WwGqdZ!X8uFez=w!Bh;}mprCawV8d`|
zzX_c34hY_~@~1M5PGQ&P2>OH~!K-jN6A!A5B>SAx+oT8H=A)CisWcLg(%HPrc(z~C
zXww?xI|P2BxSKmetwb|$UqmhfkG5XA(C!tmZzmxwp9SM01^T!jgLFB!N2qdl;{2Ow
zj|Ii2D1$izrj>XVKt;1vqS&`-uvOi0vtN+XA(vaaE6iSp))zO!<t?V{_#GP4Cy6+2
zMwfG|L+kZYRVM?X&!t1J+;2Y8=a0@+F|(55gYjV+4asQVgAIu$08U0^iYQk-Izp>O
zTXnLjS(Zn5o-!uE%W@YQv?qeL_RwqfcX(eEC8KkB;+FO*r5Y_kU+9nyaHx9lju6+-
znIl-p9(@8nv^C=#0L7s|3R0t^{6W@WCE|@(wGBhD>1(1lYmL}HQO)+a(@tOGCR8|x
z^w(6K*+n{Y2t6XF)H{Xjt3Q9p<GKk;{#S$3sjFfRo#Hj0+k>E0&M~rWP<jDRZJuqI
z@)N=uy~W%;IVU)WzN3qF^aG9rC#rM=DD3e=7R%CI^dG8m>^B>R-Orr_s@87H)5Zf*
zy9L=!A(jVK0RYs!Q*>s**X|pe9ox2Tb@V20Y}<HayJOo{$F@4QZ95%y*r)f|XN*1e
zck%82Id|u#))?!eF6OMUW<9lPK0n+L3ej{`JHa!Iqk{8-Lj$_<^AB=+tf9w5A+0*q
zUzy{5XfpcdOCKFl2)BX};Ls^p572of8PMlgg?-%TgT=S8djDKPjri7VJ^Ac#X)6Om
z@?fJ87}R4|?sII4g4l~IJrdIEPx8u@wP*sZp*r`q$3Cp=vlqvCCVm%*Q<Ra?ta;kj
z$?`?U++^HH9Bw{DUDKJctQUNHqs8Eh#C9=NNrOesQC_IPu5LHt6mtqn^eB>XNdb^D
zkJ{3uWi?xtS4WN8$MR-4n0z_5acKU`NR=2bf+&07W^$!c``9!(I`bJO8)kH^PwR5I
zM@fvA4^ocY6d+)$6{{Qb087!KTJ|%TpK2vuqPQdZi!DfD$1=vu`SogowU{EntaLN)
zever!ptRQmreF5KXSZ-Jy1z5@&)l$Kx{YGjakBMPq@cC+v<{X`Tc%iYTwG<ccLn$?
z4wts6eDzvDk2L7%W8mAhh0}FDT>!O=-*&4yyyf0)X|F5huW`_b5t1Tk%d6P9<|g;S
zFJB2I4;syd^`cTz6I8Ej7m}MMD$;R;nEsnfiks^n?S(|Gl9t_UH1@v1x{;<)90Kre
zSUPE*+=jbZWE)+V8T>1@Ex5`07Mg6wt@xU@_iaeqT1Br>%oyaNVgL+x=fgDWjZfm;
zwKmmq3shQ_4SZ4o3NL{__!u;NtJ656gXV)~1u>l-fn?pO`i6C;=8jXwUIqQS)k~+7
zr<A8Y+$F9~u{xHI+?6@syqNS!e@vE?pJe1V6U1v7eOh5*#~#*z?RjdsseB6#>_;mR
zZelSUFUtAe<?z<w%s@1#3nN>o)T0O?Irr3H-Ti$n-8Bko%v2!rYzfW?hcb}tRf)mb
z(6&wCIhIYFsh2EPMXU;J7V3L)d#(fPF|r$Tgzi+k#w_vPpDeTc7?iPJHCaEAnJnm{
zJgWlfq68Y!#>W!+end`NF!eGy>KmDT<QGX{1vFL+ix5bvE`!IuvEDW5MAumrOMu;8
zK*f;^($f*m@)Ft{Du2IA<gy?{c-R-#nVuvu0sm6vkai!NP-aBPZiNuDDdO?eNFbsC
zE7AYGv&VHN_ojC33Er|5!PY^7aPz~LfkJ#|Cz6=hLSUbDGG#RwPSK^Yhm)){?3(%9
zdCId@3(;|UlK?~`G<CAF^AyzvbX1?0i|Xn6+1_@1j?WqWOE@8mP2U1~-OC=n%p((k
zOy>Ou37P<9o*w?1U{gKgUPvZU6Rc($m}ABC{oleq^Gd}tY6Ge^0D#Av2^CcQm^e_G
zHolga>0Kt*Ov|<d2(o)TzF><2jIa$!fC4Z~fQ#HkA%49!A@}Lpu2Ivq{5AuJAvRIH
z5};|hIVh=BAud1NJPCNCl_oB|SA_-Alj6DZx~jV5=-arM@@U<@sY#w-&(}R~4?3>F
zVATO|K)62E{|%o0lK9DP|0TeU`1hD9`aGr4xAl-sK+#0uk!4gZ$NWt7K<HD<=*)2K
zXQIWXZ&vnt2IodD-!xW5zVc2c>Lx370ZVKfMA+jemABNg?$|rnEq9aN3Gke37LQY8
zm%&;ZlX3b$qkcbQ6bNije4?86#A8?rfzMHR$91XOCkaDJZ6_G7<W2~qn%;quE2RN5
zp?&w170-MuOLn04z)f?;&T{q}_7@}D2Euw{M*q=T6^OsbY!WiB*ID+&sD=;$k~8t`
zc}v8{-5QJ$geo`up>x7zmJo42mTV02^DeeRZi`>2s%K5v$&)&vt5~UQ00Kov#mnH$
z17^UY;dV~UmD{Crgv&AMc(PRNCeE)x%vkm>zpx_qt4EsDMz<GSI99fA;STk0IFS!3
zj$~X()F<aXphsYakcK?Ch^V2#@MYwR@vf>*i98Gs)+aN+m~R2yGHSMY;<BF=&%xrd
zSpbXnZUiEYhB^aTV(4*SwH9l6EWD?cFQivmufZ{B+*qJ>h}a{F-5Ij7?rvet*XHS&
zGQBZ4gPoEk-v{sLu+{Q!VpY0M2peQ<t=UB~V?GXfUU>gl9(xO95->EdCx!_~A3%^F
z<Z+m_hhQ*u^Xk*-;BPkkdz^MU{L+(xQV7w69oluxI+|IA<;V1T*<CEJS0xU_*!0g^
z1X5$0orvHQRo{qO#{zH)l4;tviDwVV=^h@X$~CB-tBxow<zHyw&Cw>vW7w}<pq~Aa
z%w4+{aYnIjlRdsVT<G<J$NM!j<Z6LP=?ct^Cx2UUTn&cQN2lpX+DAeRq?eZ^y4mH(
zu~C%3{CzLB7?hu5zMPZ<w9^nP^8_gmEpg9iItuR?(ufG-DM151f-kno?u1mQb1*YL
z_7<qn8L)&gDB(I+RG`0*T?aQ1hc2vsxF3aj&##wqoU0YXEI2im{mk6T%{;SKJf^j^
z8?e^*vLK4g{vdO>U-OpgFRbT}4RRl8P6zJ%g0>YC#+;40_@G=r3|1}UsACHaF@8{I
z=D;teSNvMKon{2M7mNEjV|$b{saM91v`hy*V8t+n+8o*anOUYtb&T|$7w7wt6wKIN
z`URk#Wa)g72vpQn5t@>IFIr35Hn${Pf`9EL*Tgu{XGEHQu17L=-Z>{X?;k;cD0%5Y
z4nMjvl@YB;8Iju9vvJ=8!PCh^<5{=2^eNCB^_Q4&_SRf%2ae8vVWNBOk4NjSWbK8K
zIMWMNAy4Gmw`)ld3+qUyvaO3iR>uH$+uxGoMtM$B$Hpm3j=~;nhjJYfM4FArUah+l
zo!<em7ix6v(#>*3kRTee)1mx-Mo3C=kv@S2?nSrU#NTb*_&EU0xn{`y2P!{?<O>9h
zDNZQ^V0SXtX3QZy^z`g0WkW5S2OIzxQ)@lbvR|^g46C@x1V!IGLYRKx#xPh{8ji<O
z1lDiBi2Y(#v)bjvnLV``XWhKe;9#BS{(+aSR*4YgATnyUG!V0Rk1W!e{?VpnJk_x`
zl4^v5i@_I);us)T6KYwd4Nfa+%c6*a;0N0244(Abq!smyPeWw#(a3Xox(dz{5KRX?
zW3$~G@c`i!a)m*1`MpI+iS*YN@z1MGx7icL3c;H#XY3(J{3FW5bl{pUtfMU2c%c+S
z7_$%yOyAy`X;(?OwQk@gFEmRI5fQ$UDd2VVLWyFRBq9YhiXh+525CX%MkkVmAIp5*
ze^|WA=9%wHiqD{C7e~V5T<YWFHVfR2>ozfc(rj{~j_-Tq#Qc$nIHa*iqNOiX2aOd<
zJ{`?9009g0xo^YV6`ND4o`yK9qj-R~Pr1ZEyterHU%Q*i!w$t$-D9KC^g4Y0DF9|*
z^nF<?qmbfH$I;|n0uu|vq!-`PB*xHSp;&;97kF-tVg#o@X{Rctc5{;*Yr5&B0--#O
z8wpavU#^n(-~7k5OIx=HT~*`RlaJI5R^K(>EK-q()!MhLCEYDVr?WFlX^?`{h>!=r
z5a&~w@sn~5f)eX@&MUt48XC}!-6o0qk56n@Sf>9;g}^;7D2@s8h6kG^gn7CrB1_Z2
zw@xM>bfDn}OM1>Z;PeQ_XA0p8+)5o<jjaWp(emR5aeFyHRa}Q&;HjtBhVRAo6yjM_
z03M8yqTq%1P|Z!m8w8|++)~o)bJNwoOCALk=VZ_%$p6;Sie(p-qmX@BX#z>SW^=FL
z<)y$X$S-)tZDCJSB60t_bq}A2Y$xy(d_U6ao8a|Jhc|krqrDaDV5)Jp^PGP2<Tl^D
zb%xS5H;C=sR^)xB6fZ`9+oiQ;>KQS{k2)IkwYv3nB$9Ja68d*`WHO*Mg^Es!xvAt6
zNkkpY1#FzU)(F#1Z>CWx8Wq|E|JZvGm*|_cgATuCQA7g9aPa$<MxJ&Ov#tFOAyMcg
zorA2{Kv>R8J0_LlCnH5o03XMhga9ZmQ_L_%J4&vrNM%iLVk8taCvQt3#1|-<GAik5
zglmD)WSMF}f?1IMXvq~02KMua=vJE_eyE&-Gf#cYS-y*ruD0VxybHM3*&Zi^187T9
zC>OqfBlm3|_|=6NDg#~zBrZUPiqQ&Su(iV@?N`8h;U0e%NqR2ZU*RoKH6)pF7jr0a
zy2h_f=3#u_GWDcP88R?azN`3wV)mU*Lt5Yusj;?L3K2s+s#!x77x|vp&%ZM2G58fm
z&%K;cA~elX0=T>TgX+pja~ca7E<cKvqdb)gehlKe8HommVtKS*1deB|cbI`dGJL$|
zy0R%=&}!SBXbSc%Cby-QcuVDITD%G&Ez9i7>Cn(3tFp#0$ZIMau4YQ+BUnY~5l5uH
zyrU7+N(>`YdQ8Ms8<%X3j^^bJY023wy_|8gW$h$>OjbB=C$GnIrbhAo-U0myf;pyq
z2oa0iw4F@y5U~3$NdiV6@grF}-(>=BYuhr4<$dU_78mJowc?YmPsAB=Ec+q)sljOd
zz*YHkO%Olv1YuH{C!|=nZ=<T>i_ogR@lR{GEumpP6)Pf@<v>GuQ6jJ6wxpUKf6LaR
z)Z)^bbWN6YcbYr5!Hz2X_&aZAcm5-^N&}v!!Iw%&S7~t+%;IJ#=Ake&06~{vtI`~{
z9hXm8aSP%HVwhBh=V?*0f!s=lIDEO7gw%VcUnR++w1*?&>qjrcuVv`7W5_epy-;L;
zCc`<u!Ncs-dNX=>f){V3>RUj)$WJ9X>}UKrz@i=0&5ZPpY_FkF$`1!d(+&<-4L~~9
zTi{@APV28_u&r}f`194!===s{*_&w1PyF;7t7T7`YGYvAZCfTM$l`-b6LDt3{Uz5*
z;`MWyvloiDNWUL`Kj5P}>+yl`MrY=T?2+nw0rPL9o5Ak6TCNS)t3C@gJ%mt0RZ_GY
z3|7dWS@u8(_e7JL6d2URg!eP;B3|RvD4rJ4KFjtD_JLSe<$Rb&1pfTipLn}iAo~e^
z`(RI~Kc~K>>Om^Uc{2?vclPslSu*KHjxyC-4#{uxrNVXVlrfTluy?WC)}GTX*IJRy
z@~}v?m_lZ4QAb`xfiC4OaVV8(fqVS$f6|S*&*_Md36aJvX)x0Y{+4xE<6r3$o3I#<
zs0a7vm`&_+r{~X0UQ3T3Yh&)Y*QDr;yeoUPL1@r+1P!XoQMJ!il;)pnB{U~Pnmi>;
zJSy62uY)lupJi8v&ex6)vqtxo=H^VE&6VkJN@;1rdr87S^R7n4m?m2Gl9&WEOCL>2
zC%UX&+&mb4qfn#y$X>tqS}n#>_&_M*dK1Mw+LU7MqXcm%RZ2-M_L2(>2f6%uB3Yd?
z=ec1$2Di|lr$!ttwa+~V4BO!V={(8P3ucDsueCobM8~uZ`^yVyUn>%3eZO$QKeyez
zzYX{E7pMZj6T<HA4Ct_rl<wY8VLAM;kXhuWrqGFFPt|F6w-}_X(x+14q;#YF&GyIh
zMDkC~36Q?fy=nog=e>O4bO9MdR`v@|!}e<IQ0(IQ{Jh;MxpR^v4WSPFoU=#cd}Yf5
z^EQt4HktToG#VEF)P*Ab2T3f{-vnmnS(3r(@od4D)_T&?AzM9R8#ev4Iha1#6*>bc
zhre|8M*~l%5`P5^r}V15eTbFrFW1LeAkI!rbI)OuG2ci+Fs`5o{bh(6QKFjkBd9#E
zx8gSCclcnom-A@q9*xl)X^#lan}eGPV<L%R_5n)c`*Z^#&0gQO7V=mgOPf=hyNc)D
z)P83NTL*CUhViE)ax2o*Rrpno&C>*(TOm&*<XgC`ZEbPp&cMCobF;IAwHDOpu68$0
z+t4<$Cr!fIkRmv`c$w?Y%VOU3z&21u;SR!DhhWw>Z?iti$P{p;Kwe1FfFBuN)-%Ra
zJc6<DKr+PPPBbei#%SWo`TX#8=Us$Zo!gq+Kv>_;S6OK@XW5bv{2~0}bo6~pT;M{F
zXCeu^tQv(o)!!~SGixE7KZ@j{qQnuRRfm-zL;iK0mKvZ!^b!;o+}$yAjD~1&{gtDq
zxs!JGZPt;)fMKrX<8VwNt64$-R@x-yfX`m!G<55&ZJ#YzlOa96>*x}&K~T@M{8#Ry
z#RoCXyn=bN!Ap2<UDWpPN&nO5!&-nmlj~mzQI5}mZy(!^bGZ0vce0BdXI!GZmIZF2
zml=<S<;HV-cs=kI-7)jFoQH~%p4`BVKBZHAdy~a1`26-ryv(NNh&c=*uC)?VM!kw!
z;x`yRB66;ke})7_VN6uIAU+m}Mgo60%0&XnN$}f7-|BNaI}A-wLO(=aWd>VhTTVB<
z#n9x;Uz&2J7OI(V-5)W&$eJ>v^Xw_WTK76cCn@^Aq}u&oi-@3FJc6udK~r^HO@!8Y
z|B4O-0*pN(kR2ZDUPtC;+_QyczM02@@b|AxFjV=Bj}VJz>K$*;7QyW3+&rlt-EDf-
zKA6Np;*$wHw4e{B+_G-e%U{)EmQT*4D@olKVaAB;GfCyQDfjP@Q`l1<cKl7;A-b2~
z{qCW8z0!`9Vk86NQ6^B)PxH0%`ORMu8Yzc#0RiB3?XZxOTToQKtkB9cK>9DWEv#Mn
z@w>E@qJvZVg&-3hYuY12>XK9y|3{m!#Q|F^7q~LT4H~iEOfMulRC%UmU50e9H0eQV
zrj-Co8zc0EuWHW{IlL27)c}mKDzb=06wWe&yam$oYT0%U`QJ)(kYkhF=@xLFs_tqt
zM`Nd|?yL4L?lCGHZ9}+id^&FA8{W8J*op5|2SQaV<hx*~9z}+U*BgT|vkIb`2N&>C
zFiAK~L+dW;8~D;AS@i&_981okBLss2K`FTV;tu+~n5&|iV9-zKJq4zl1zKp@_xqDW
z8!msw^v|0w<hMhljG^gA=McAU+W2ycZj`0TedxWTI;?0HR*;z!G)pm;p94onLj7%K
zN=0)~YR;Q<Vmp4a6u;crt#?RVaBx{?MBFd^U&D@yDVT-v9?bO5PzHeb#ZLW~0MPqv
z7KVlKUK8VIuOR((4D5DI>cm&jBU-Y84~5Kj!EDJ72l_^yp**JGS}wN@_p}{Faju;)
zv-JY49eS(Znf(;Do({cax1kw<igUkJu;O{I{#EKL^3yqM@KLLzB!a~&t$rcI@Vcd}
zS%#Hf?~~(d8XwGGwb!_5Orp@3Iqm|qrExQ;v}5LbZ$^qELpH^D%)uk>=b(~hkh1sm
zwpzUy2JQ5=uqHaq>HPCPp3H41pZ?qq``wT=!ypsouQsb=t9Qb;oMWx|O7avr^s7S1
zPLn;21>?QwP+VEd$Q~%{-e%;A7a~AM{<ay86-~EpiBMs7hSe&*nnuDY0Z0LU#35-+
zPJyol>}$77U2oNeoPI47_O-w6yh3m20fkkuLpKNmfe!YF<3$ir{<l-s*tD_$W{{7j
z);bwyGe*5i9GT~IGe+?T!CUQPG2eCCH|q~ht5Z}E%UhuR`|UVto!{TTkjnCqQ2z^U
zYV|fw_8%i<&Hp=i<^N44{tuC|HPHVLQkL!i4@lXQw6v+K-?Hl7<JQB<{He&2IF|pf
zkg|W+F6~hN3pBJThV+601`I3${Xc+)TKso$pVm^e!;rzu%hccOl6CqD+=HPR3F5So
z?Yzuig8>r4VT&d3Qi=|PAFkP^tgMZRz^N5VV_GfCtJ9+p(*Hn<SokM?C%|c|)SlE@
z=;>9$)GFy!pws0DmFpg-Z(@Xgi4=ZIU5h>Sdw8DS<Z;~b;mEG(GxyQBVJ?r!L0@V;
zjVGQ=cRl}YMi;r4H9h_DU_bw24zvCC`hrT+C$1~jnJoKG^AJ`%9A31G?Dq!OaJk<y
zL&p!oBEZL;Eq|tIGB4i0!)DH$8_Ueg93oXQ2t2vlX;@+_IQN}cV(dL0@jHci!{*7K
zQJ^zqU1;#4$Yvf%8eWiOw$A|*(=%lp=$Fdgi*P^#o<4pSHmQ|IU&fq9m%eT#9s~{g
zj?;?PWj^VM54m7owdxQgL78E^C3)%Ms>3=N6JSs;qo>L@KktOGJpb@-O|yyz%9Rl?
zB_j*9)N&xKx&*s#O7%GYbv_oVYKxwod_0$AHU;#p<TAxcvKca$An|9M2SnLLE{vPP
zP-~qqt`qSflri6|;Ns>m#+oS<-K{=3H^^-*r6O{wXfj%|KhQXt3h9XgQcf44ih^{%
z=PeE%@BE~UBM3Y26{ZR5qo2t4FqqsFN}shf54LBivAiBu0UwzM^PlgvS3I3`)rJ(W
zRc`E?5>{?{-9lef7@f2$%lU566!md{Q^k~6sD#mR1T3!jLe-{T`8k`{ZZb{paeu^~
zs;j1f|F3U`zS_ZPI;y%plF#f)v_u7=hM}kmi4XUfYi+zDdR|z1m8~;*9#0x|0x4#k
zJuNKjUl*$oD1@UfLx1EMKg6W!+cFdG_S9J%bTojqGy1m&M`!$RRIL;Ob~-Yz$Y)sx
zXdaVq^0=)bHiX|*o`BPoWwz7}eHO+1?@zlPS&VP(ZZ=kpXTO@_kjA*XB!Id)n;N{E
z0{mT_nzcIHO(Rcow>ln%0bQY*q*Jvxs0HC6@}yi>R}PGu%#j=(0lTk>9a4^yC#tto
z*hr=;1*1)`YVdhU)T-R5Zaq&#78hD0=7?(nL>T(XAVmsq(G>{_(V<^BDSnu5`?D;t
zA7GHid=>FlE^{*3z&$7m&y<)8UP*CbLTcka!m_I$|9mY{AaJ;PWw&>>3Yl~(F3m24
zljZj^Od}|ZQ|4ST9~tS{dXYpP9KyyIjNLD|#$+zhiKHo2gB!oUz^?sF`p7XQvY%R%
zFfZqMF>`jTN(*Skt6H{=gOp9(vx`=D`F9vQfUpOie?xFg`b#V$1P;<a7BUx-1lAt~
zf<B&j%WqTgK?P@QN+?Qkm`zZgvI|5YZ<;?=LrkcT(E<@;(iYegQSI3i$WuU1jCp4p
zJavi0kY#ae)L+O}&^;X384igG3xv^WS3ne+rSNG+j@PBM{nofa9)gY6_`@Rc6Ppie
znEoFSfjir}DTEuNd0i@dyLnwE8<Kd^9Ezt%*FW%Ai6ub<mMp>>i~j06RQDjqI+>AF
zKLET7YKvyi4Pu9G4-cF~jWa&%3h3IGcnDS4DrY2$R{I6NC>J*%o<$mk_qS@JQgcBN
zsbeY`r?J`88l{tXAF+O*-yqZ;Zb$|($Ybt63UgSB_<K)J71V@LMOBh*&0<xm5+hrG
z>1l3Nlr{Yv32>Zf!!QfwD^OvqgV|I3aOH-Kr)#BJD@MolmC*huXwzHSxslI0TO5Ge
z?;B~3K;WYOvs#nL<8~gRqjg?}>OHvT6HZ5~AIgpFpuPFpPTuG&eNu`0;QH3vTK&%9
z-YjXosA2xFg&GOzwgRV9yaSB*pC?O+TQR{W7XLEP8H_kzJX=2Op7LU`e6F52OFYDz
zH118pxg<`q;2P0DJOmwC%Xk7oRDkf#P+I7VH#ltbH!fe>0go};OVNITBa04}fu!S~
z7O9fw%V;$9UN%8o80`&PR|Dzm025q+*f&2?_HQwPodsXG$Zx^X6(mPnL+GA;JV(*R
z?H!!B|2h>(e4fakOkbMbG5%6Iwu;RgtJqm!&np=-%A#7)7E*1k*gmz-Jh*juTsU{l
zEdTp{jJmD181FTYBVWtre&8Aox83OzJEBTye(1aj-cS8y0(}0N?)%?Mz9p*uFROp7
zZWf6Dp!8V&ccsT1*aaA3#Z+W?+76aX#62HVnY2zW%S{N>5Wj$=BU&TpahF%NmvR5r
z63{>6_=t-34TFLKiP6p%+*VdXrSMtqIdN9#Wy|*c!_Qyk`kmWYGJ-l2IM{8=eH>L(
zGBWkyl@`0K=H>nFi(4xW82`l(;~St4aLptk?c!#)Z7cP`Bt=6D@8HFUH=C8cPC`sa
zJmSkqJ#0?cC#wtQ4osUD{4ljnqU8h`ab9Aq@n*=VQbop~fWWF#e&FF149}JSZf4il
zbk&2Pyz06t+HTa@a5kg&J>dh{Dg$&Jrkax82dOj1YB8^c%zX4p)U}YN53+I)f~p{T
z>e-TriyWkub);ddvgSEs0LNolj5SBEjBt9)u4K?!q7*rexW5nj)DTN~`x`KCI||J8
zj?H;!bM?bc+q*DNo_nKiJW4AUuLB}*sM}8y_|k-!Y45>~GcZ{?PuAESJGZOTn?@a}
z;#TA9o>>0({K2nG<xTv{U+#aBKdb*PfB$pKmXe&GL!^?Dg`cF8qI;E@o}@GI55120
zU(o9{`}cH;fN6Oyt{*)0ur{Xdh$!InrshVNXuA$%ciS<x)$k8K6xb)DAOFbgNELD!
z*DP$t=xzq#QYm5P6O0z&!e#!{Fwe0TQy#II87H5ojVa;I8z`>Y-^KZ2ZZ|rn_H(Ms
zJErz>dOr2aoMw_H^{AZyvmj<uS?T9ei>cMvOt~z+SW5YVoU@ml@TA*9!qd$hn51q!
z1clQ+HiC&EyWQUGNm1;o;8!Jq^isih7mkx`(V4>+J#44U#nt~Y|8&Ivn1B7ULt#JX
zWL=M%6=m_#o!L}-1kdhaI+Hnzl}+Dp*1TkF&1S4%Ds3ulHf=DRSb>9t%@nbRrGL)V
z)zWFt(#&-IAwE++&zuUDVsifc3g$wgD;8JVfzZ-2$<pgBHd7`yC_gjycNG`KWcKP6
zxCuQ95;+VKV~)?V7GQ#{VQOQllr+$S*VdMVBP59(5>f<`fE`NbX*Juc6#Y<L)*|ip
z#S80~GM7t}A*bb+!{D!t=N!Z%i-(F}?va)$nGBr9EfZrf$g(I%{A5E`(Sh3AIJW<?
zxtz?I92f-y1B?E*N{GP-!CcIp99gU#%&bk#UH`|^+>?o&nS+^=g_Xt7(E9(lLT*&E
za<bBGZ1!^UbrTG;(hssURZ=o_rof=cyaslku?cEqP1pQx01><1FvQqB-Hn(qaC0FG
zD<8j=ITQ|%A6545GnM&crC69sAtv}xOC|QQkOH$1?xV>DV<wbe-AJ=t3WIS9gV9Bk
z?+bzzve2^35vyb)Q_-ATyOb4st!6?lELVf^dm*F(<owwsa}K|${bj*?O;V`vpW#5u
z!aAx*`34tZ2yIgVn^gw!qg@n5E$=;W<OoL@5my6Ks|M=7geTikp71}`JU0Kw@cgHm
z2ljtz9w+wyKWH9oYUzKPXB_7LtL7O4vV4O7m$*wOi*tcXfPrN>{s+s__P>k!bq}gq
z%1b8q*5#MbEYc7w(*GFZ4tbFddE<EFC=PhZ@z7<&;lwCI(Zu1%{tby}a!h2Q-^Agl
zLRI8xOH;nd!*hk}*jBn#wybonw5+(*Tw2szUUt@AcCIM;>94*JFW$Ud^X+=gdd$4!
z-sapA&P(Qa<?ZIbdu*@zZ3nI<yPZ%q9VebtU8GkpY6`E+J<45Aaw$@6+8S2w^i~#L
zxPCNT9~T9P%r##Oor>_hT=w~0&ZC<C^>rJT)43uor>tIL%hIyXVK&Gwz>SG4qjYUF
z4{|HCPwb&*+AQe`yAlia2AXz(Z5j|t$+z2}s?UnmTuPSC3M<9Qa+)a=!fNh6v^|+Z
zy*@piO5$vzo#z%ADi|ve{&L8o`DB}52OMCi5U`N?;5Om@YXHTQVMDM?|9)9$osnA`
zRe!=PiR!BgN?L$jQm44@T{B&S?nt)5M_gEzP{ss$z;#~oN`2m4m-Ta|TnVx6i~w!A
zhwG4wzN3_E#`fF+Omwo!9IS+~S|QtMbe>iAU>uJS;ytrij)-oCDM2?$bY|+du2JR@
z<&4{O;z71>Hlu8DbcCmfIT|v4l)SMqll6QKi$ZF=%tdxd^zTO;yyE)~^YP3%rP*U+
z9-6ljJdIMY-P>!Zxd0%n5759H0Snnc)dyBx;PHKbEt0xJ^ro<N>Ra!+0N^|`kIpQE
z{TG`jHPBxPMD~vKMri=;R;_1m557tVWRGtAKvy%16)SGyB#F=?p85mj;L0=M=BH+a
zVA;v|P(f0{DcsfEbf-^sX>G<UXzyQ~#1Z;<bs#HSdn@*eG{XA~Te@(#P+_$$`#Q=N
zk?XXInn0n~ei*|U3;)M~ynMj#n?26iKUa(=8KYJl{?D);7ZJBPVNl|ZKc28|`uEw6
z@2YBsT^KIgP*IAOUU%C^V-4+j6m1+HH000Ujm;oJ$OQ+T-PCoY&7$EdLdsd=0Xc!k
zoSE7DP6RU{yrP<Dfb8J5SK<&`Eg@=YpkqUP(Et|}r&w<3h8N~JwV_n31Yg{g+$^}T
zT6I?$DjptDO6BkDsO-8u!i$%L8rXe_@A;)!#(Lj;e&JrdmWTPIoRkKRHf!Ds&>wMI
z)=%)oOQ7MqUYal1AYN`k4(?|hz!8RiqHUp6Cwg#>oqA!K;CLsBL~~B2Sj{jBkX{#Q
zPgPjVNz9PZcS)YuneLFv;UrgzC+d!nO?9^j6)mUXuwhmBtxN>RFx1DAGMTkvZwfKl
zG<9saK#TRYpyuhc{m!$=2nQy%k_@SBpO8eZGkA|_rS0I-Wk>W?kaTn<P{icYu0<P#
zEgte9T>8&zT?s{9HHUYea8m7&gx!$hdpl)bJXxRkkCjoZ!EU<AUUC3rHO~YcRQ;v%
z^@JHZL(A2Ap<GKf?2*iFg33157b1j5p5nxkSlFp8&QM4)RpG5T3x6<_H_}oR<(4}y
z3wk|KxM+|(kEs?};52%`+nW!7DJDJe3QHM*@ynQ2Y~<$GUWccm`Vm7e(Z2ECGS);j
zu&EUa1KtpPj?(7xhnQ6i`<XUEvfE*L2%oo;n1R>K85^h8@xGB)fW8KMIOGksh1&y6
z;&4|?zOJ!l=y%@6<>ZQ$XL4or#2a9lCxV|r9sL$V=`m-CExqIrS&oc5PF%Ik0=vBK
zv2%*eRTSWMIX7tMV8Ifh%NcE8i-@{!CDk1!%-b}CDI|03#BNbZOc<<9eTAFvP1&fS
z%f_}^H+(E}a<gh7YD1eX=ndGoRWJt(-xj`-Rv&LZ3p<Z$A4R(e7FM-i);v36Yf$|L
zeJ@7OwbCUdq|!9r7Y%;e3LV_3L|ZcHvRxopKHrosnB{;!fNZuK4u|cwk5aDNb_5r@
zmv+HuWW)C26l{J3)0LRHnZ>2}dco-ERmMU%b6KilsUg+i1{T}VTHn-KL)nDWiGpH#
zDV8<(^T$-=BVy7?)LYKg7^6^N)Sco|ZfH+5Z}LA2w7hWI4Z4N2q?z(_8Ngb$8>F09
z$na79>H7u0tly7}oDI8GU~*+k*vT0?tgqe<>~x%k<(t7(Vp+kWJ1e^P^}}F(`i;$v
zm_(#C8lY_@&5;ETPb%?BMY>?s_7c`;bt?KN;7aOH5uF)sb*ktQ<0^ILyX9*}1nFs>
zXxZ{nH<Tjk;tE!H;?BXcp7Dhf3AWBV=>o$`afu@HGmNE?#-+P!)V&#AM1dtprBr}b
z!DCFPs40SzOJ5C?JhX@Q@O?Jhi1!UNft4k>%y&WUxJT_WEM?X)g+I&|^8r}XnKcT&
z>SCiy<!3So5XrZqRdnFQ3T39KKg2~~D9gmWTHW&qC0|zL0r~-Q3W`-wbFuIg#dJw^
zEq`bEt}kB@Gt+cv*M7+IM+I}n);Q5TRHYea{{mZJP!RqERxAUmncLzeWpNyZ>qy#4
z@7OSp^QBZ{g*O*N7EQjDMJwBtH$<SieybtkMVfUWM4yr9hDMQ7SFat(px(MLR$1b2
z+isMxaovnBqTND_raG$M8E{DL+Jkr-TaVztMPK<KvnHXthK_40%El?KwwTKX2{~yo
zmbzBlXI@JgDAg<OmhnX;u^-AZUo&Nqce$Xut>9r^?_rY2xV{D2wU}eZefPq%sF^nw
z8Y@}9SbZthpg=ADi5lWEAOy7C+y|W@zN1})1OJ_)APf6+POeK0C>UlQG~qX0=x+NM
z-*+Kin`V>++KVs`@St9k+PyhD>)8m0jd4mJcNUxE&#9%@xXYe*?SZ^6eAwg}Wy%Va
zJX)|%tYfkoO>@2%-#r#GFP(t$CD!9|Gm31xcMOg!xQX{!XINWMT}yW<me25bN)K6Y
zM*vk9Ce*?c35!cz9F5&gv`z~1@FoAQ2)Up16x><6-W1*AFP5s0%Gx-uwIEx27*}19
zCQqL&UC5_Yxhe;5;91q0NDBX2nqxt2B+nvaqpB@vrmW!FmQu7rDVuE0X6K_7ViXVz
zCR1OK0Cz0=jgq;19S2x?mZ0!i=;;hLY17;>X3QO?yguC$2uoIDjEi0<(DXy|!s-G}
z3fRJHv%rg-3dEcsk~`tAJkSat`f0!tiD%`i%O0}2MCItLa+;QrgVH(dN5PmEz#LDR
ztK!g4uN2R@J0*W*i$o%-={h5o6|84zoUNL-Gj;eRq?fT&*x?l}aALca(fgURQe0u~
zL(*1dNC9+7XAMZ^)NB=>H%<uPyaq>|WlC}ri8SYajXPcpyJcaQSEei2mn-iwMO`P4
z<%a1NTOm#r*Pki<Yv9$c(UH=xskP<<KZIJw=8xDj5%Yksl(g63XBkUz&qHOSuv8{A
z<Kh}QXYIG4PS|fDP81%7R@ZfC#~><E<XIP5G0`Rt=AXq<PB-&;7CX|3;v50_9LGIM
zjCBPkCA6-sAV1Mxi0UVD?;1b1>qU6>pH!u3S7j9=AfEnw=;o!GX0wsb+>W1P&O*;8
zU%lbVikias!O3mz8AoBZbjb<CqE#wh_ey`v!>g*Av$-K<sbl*&pZ3;aG23r8$q(-5
zg=5%GAT?A}*=Q9WR$9Yae1t!EzM_r!!iJ#F*vMXWEa$q<+Rcr+n)6ARgO}n>Xo8)n
z*JsOQt;a?4S_<PqV`i#eU7Gzn+uS{toHhES2nD~04OI2KuR3qD;2Zn~sod<rMmCdR
zi*5XFU}Mh~ud3_{4sJ2$%&GpHic!5eE!nD3$DMVALZ5On2xUwztt^_13FIMl=mwyD
zh1$^WvD_&*JV4Vi_p@knXUVjsm^R+OG&)|5yraz;<x7Dcm`WQ|b}l>2QSOT7?*NR6
z5&y(^(#Cg=d!B=n(z0hm4G_OKzQ9CR{Ys#{k#ZF&-@-K%87O5_8n3-MO^7tHxVN54
zK1Zmc=p;&Q>{{Al50y@rAz*(O9QyfIi%@5oe&_N!W49~iVv*XgV0&$mpzs^aer3YX
zgyv=*itm9Q!4`Bup+|}`<cuO9S7lkeF+3SHW6or?w*Z26upnvi!QVE?uw4Q0Pm8x3
z>LrL{m9|z7UOD@8{kfU3q7#C>MaI-vwXj41T9cG%Ii#Y9&J&4BPkT*(+HiJ_4;oMl
zzsMKZCAL^`YdU4wkafTKoRct~&=f4s$Pwa-&%>2%Bu@9*LkZNTkrS0pZPyXG+Au~E
zY*D6r=GGW@=&ThoY_4wus|J(r@&en~HPOp*lT3LU0j}d$!(PtWmKA~wnNQlIcqcMw
zgwo+<jeCse%^aMe6A<x9>j^kZOI$rlC0A0JB4*F1gUOp@y<$gO!!*idn{tx&o!%%R
zYVAsTGQfH=Pux<z#Rb{ND)tSB0Wa`r?ejG7do3ZG-EyQ7z&;aFF8ULm0NLZuAl*=X
z3`P!|m0XJkACw@N(y^1LWi|(oYFMXS*Xs1nk`ss<t($r%7tOqbs~N6waKK~Jkx<0P
z<yoM-2j<BgpbS0NCG}%{+AHmQ-rx}GZ;P;KU3pV~Nd`v#2RP@jlrCl8^!Nk*=SXK+
z7P~a<1&vhP>utd_x*Sa&dH+~LecZ-J#Fj-tGI-Y+F$W4;n?_rf#G3pGMsp@}JxAb0
zPSIR*JH>`;cnMp^qs*M_igVeK>yi2|`&!xoAqAw^SiKz|w5?%iO&|GMr|{Z=d-&^8
z!hfeZ0#q}<!(7FM<Fiv-(`-Ne3CjgsaGt~#BlK0$^miAkm{cP8oe~7L<WAZJXf>x8
zqwQj2cZQHY%h(ku0}zIvg*a((Jd{V^Z$#@V23^POSddSTQ`W1mGT&)q;k$QquLWSs
z&J3D!aN?#$#-P+Ya%jF^JDu5vpFs#>F;Tyez9qU5wJ2r*^duvW{)WC_^P^__AiR|L
zAW{A`K8qp;HxHa3ApF8Eozt4rD^Y#FKF-wteo50o-qrZMx*@|wL`q)t-_LgSDh}K2
z-VMaLmNX8qmml5C>L;A(J)IaJ$hYhYw`_YNrwah9>|Mzb!kJ@z?f7;y{UYQ2eWv@p
zF0%yFf_I1NT$;7^aAHUP&s-4VGEN5^VS#AKY=A+pjD=KXx0qtr*-ubLLvhaD@1HoK
zCij+>R*a+d;R+mkC)Q9q$yRuhJkT{oI++rR4LgU6n9_p#gC|6}OBwp0IP&%MKh5?s
z1<tuf^XAz`nfoGCL@7(_BWdA`>m@BWQrs`Xb9v&q{PLOxgm+8XJu=Gcd(Rq=1qY;(
zVrkwa_z$opq9yX3@;jH!(W=_2(}TV$k3Y_EB=Cb(D^{K2e~$OUy74^n*VrR1Yabs|
zx4d<-W)ak+iQzKXDhOo$a2eghJn!wH{Y_$Cal58u@hRvo*NHyv4{suqMsbO+plKc4
z#p^mr%-+H5Y`!b?uff(K9kDm$3`YjR{$%3hyGS`Gccq0P3u1ez>?pKuFeANO1lkto
zTKZQix|6na#ZM(qJJt3WU+ELV=!@vPqs&g{U<`RJl);vNZ&MrDc)DaTZ_7<0@}G2s
zt7ilaX#_jP)G9u=`fNFA3R_RkXZr&};nha5n9iQnFNG3O)=;C9;KwKqsENv(J{-nt
z-cy}PP{!vrUk2>A!sv6$)2FoCzVo(3ad%0x-Gs8sUu<1XqPdo2?)4v$n7Q#qx&vNN
zwD{*e6_#S1xbuQaN!f!<+oe4+4eKg0FH4o%rChV{=s{skZOS!Mi9-1t?(90Z_^}r-
zH~q>kc$v9#&)>I_nssAgmDl5w7729&tdb_#(#x;-j)=tLD;kuN#^;BG%Kw60g&a+?
zJ~;~`4`5g$t1r<O3R_(=rHSi14*-23bS1ob6#fylDYN|EjXePDJk940<V{Naj4-7i
z*1ze@bH!}kx4|bIep`+$$GU`WPP_^HK}{$#fqR$b^hch;MIIe+DX<~welfb&4_^o6
zaavl(8@+BQKG`ROh{}O$?19fd3Y~gtQB@PJIq73p1Q8YEiv)f*k7BftbFFNz<SM2O
z8aLGWs^>DTZT;mf^%+D`S(R&NaYtt~?GP@frUJiO=tX?7g}PmIpYay5U4n_Ks`|~n
zV0xqXESl{SqOa|plS{kv$PBqB#hdnjo>6Kx7&hEU?+9NN6Z1<D-RJNIQzX$nMAgHJ
zbEq-RhHP@%dMmn%7ViuDLqzeYlwFw`wOXiI)AuA&d~cG}+j0!6+H5~MFGK4_Q2ccg
z<%{5!`8T-ppsKx?-u|r2kKQfL8VXt3Q8un-C$T5G+On*ZZEtO8A{zlOI}tu(<Dr`9
zaH|v=LO-V@Nc@;m^HFb|KfH++DX&x>gsUh3oZ}y>xFQc2E}*quAv7uA*S86i!e0N(
z@{)_T5~#q?V?`}&%!!n5gQZp~8=av^;S22ER4zv?F1b+sWA}<jW5SZKU^m-GbA49q
z6j6fLZvf;CKS8>YRxqddsSIND?!=Yl%P&&nK}}zAh1&2pNMRP3POYpRtALNK7f+yv
zsG+Eb33*AFmNGHe5?^7PPuU(f)A9ZW?H;kMo%f7*(9+^cm-dPCOBlbjK7U*!X(b*s
zP1^9o6{TRF2XeLK?iQz`sVL&nl~Xp4qrN)+fHM8`8Tcr>lQhIiBBSFEjBB9p3B5!>
zd^G69z)D0nZ4w$w;<ds)f9tA;nVItfSXj6iBWP#C6fUcQNY#esiH{x$L{;<8%na-&
zv#}{aNrLAm=o-3P0aja!YPWaqx<B#CnpAXYRatPVZKcMA(Q&0rohS9wpe}svDT??L
zfE@CruqOA}d{zSYEMnuJde&<Gfvf!6dxb?>B{R2a2^QPz>8=Vp7Og}lB|oUw@?l(c
ze(|X^A==5KuRimfVl^`l!$Z4FH33JUNU32&OQWwSW@p_HFQpy7qnLw#*lcMo&Hy-l
z*O<!z=dC0045Zquo*KK7w$7>;sG@VvVVa<fGk)>fLoM9^d$fPzhSmDHsGoh5pMA7u
zwViyZ0tt(kT(AD0d4pM3;3FboUel64qIWIqhVxMb=X2*{6dv;tGM<FqGqaqOkltSa
z;w-$`c7eeYXSu3-&xoySIhLz@fiR4G&FE4ZI-x23^1Iu<*qLOQPQJmF`B#s)9&jYl
z0|58nTs76Y@>EY6oin80kxxaNLbo&{j)8z})^g#STS6dq0**A6IXIK<cE_yXlL4u1
z$8}X?#o&cZuD8hze<=n95LYextybp7;jCtHnkjk0)X8l9z){7YiCv+j;v<ay?xfh-
zx@mLrT4ZoW^pgr=!nzr=;lBNc*t&5;D*N`9@kTvC#F^maA;+(YS2*VHJ-$jlLC#uM
za$S2#(a+?6F<B~~NXvkE4%yOio%6Pq7FZqSh0}*A0s}SN=16z!ElfB-o*Dh*;hm+W
z$nFKCX-@OJGWiV5SIx=;Fx)IQH}q4ivPNq&^J&@gg4Jg0>I<L9hwa<*B06*zOB)IB
z7KTsyTR)O1e_hfe>?ik{-u8*sbb$+$C{G<1lzw&R2=eC;HO{dvLAj7Eqj-d->HRdt
z{T$zHU26nkC3GpRsCMItyyDhp=X-#}H7#oDZdkl_z&F12o_gJG=)|P6HmZ+WieKix
zscYqmx7@DGxYs7F-~)nBMEk1l(fE9CoWe#;3Cb=I<jc2WpD&Rd938?tafoYZCy<<I
zA5DLJ#f@!LnfO#QnWFm1m9vUbEFi1H#@0)#Tb}5A9CL^(xQd$*m6YkiaeFRUj$i#8
zpClx+-GGLhCjBr+-inMLa!r=|$Q<73+9Gugp{19TD?4vQ^YN-2tsq||{4}wKRZ2QG
zPPvuIyTMn_32{NQdF7TgrIGtdK^q0L?TMpFwpsM2QW!<b%7x>!1Ifut#e?h;Y#L!3
z*Bz6DlCLOlA2{pG;E{CoK=?w$*U>6$k%N7ceQ$$DR9uxb71&4Xi-<zC6-%b_5f)Vn
zqG6Y^o&>0POZ7Q{Om$j4lp!_60z!P{mKjb8xLeKsG!4GG!a?gqe7_Nm)%Xg9R+l<)
zrY3f50J<i)3}yxY6x&%547G&N0eoxX4j}Pe){uD>cCdU#i1?!X*m$Vu3vJf!>}Pt#
z)f@o0yodowf;+foSJ{=!2j~`}i)_t$!o6jjY2@trhx1)S;~Lc5T|*pEeNq`fSaB8|
zN@_C=W0$Aa$G|xI{vO0k9#+Oul0d3zM8^2{gw+9c(fJc-m)IJI6%@W=CvF(TY(bgI
zmgwwWkcCh23T3&Q$tW&e%&h(ba}~;4)7xu5wf5aDuDJXNZY;$Er$=v>*9f^^ng<E_
zdgB)u=CJ;N=fr+1yV_Ni0|WiM&2vJ+vUQK%C;2^93X~h4cl8&(yI!I3A9iL{XLFoM
za7~AI+4&G`)##CXc=xFWz`CrA(Zq#>erH^%cSmOSc@9QNip~923!Z}cwxHj#f!QZg
z%A#W2qKb4qPMebgKE>z1?xf+yDXZJUrW8Zv>x~8sm(+J&97{Lj>dK!uWV)og1aw>D
zK2^=U(~EPS7QEg)pjd_DL*LwTQCHkSA#v_lRUUHcrCy=4eY={mec?|88fKhnVW^PY
zj4<xirlC%Zt-A2ebGX#&U|BZpF0aV)y%;m;4TdJYf4}A0>iOJjDppS$S1u)fbM^ka
z7<K_zcy#&RF?Gozf66wptt<t2M|Kkx0rSwUw8)b>Vtsh0I!WcPn_g_shOckVhA0F$
z%a`Oz6zVg{xzz2I!HsuSbc0Z2ig$n3B;}s-?4OrgZ;Wp&SBT8WYF(gK^^}iY(z4-b
zW=+jI25E$bS1be&uf$9t(yB2x<q?S10xjw66lb67Gsc?DY)){U&pGH<g(Pq0Bo&4U
z9o2*=ySwHq@-e2&%pTG&T-kFRsySkvx1`dDHDDPZ$Su)a#wd=OkK!dx%|_O#9HP-v
zUlYc9NG&=n`(O|4C%PB}7b}#hD|SGMwpqXV=}&lPhzQRO+8bLO?5}C5C4X}&Ku_yv
zpE^2qb}*@V3^t|*t9z6-qBtvarZ)Re7{A<@S9+m_+)XWF(sx#7FyKnGdaJcFD1?<a
z6yrGDLQSpwd`@blQ?q>TH;&RemBqk2eOSIfpZ!sk42qejlj%DWAi$`cTAUp}=Zfm`
zCobdMB`VpJ8kbtL39}D(dFG(YFY@D+DqEQoOf9*UF&d?7=USB{Vxg_aAY19zha)b!
zLO*k#Tqy=cQx9{63PJ;Jem)nw$FSRFwg`zE634(f8dxx{=jJuiGv5w(bOxIL_=gf;
zSbbLbq@f*KAQsKr{C2&GubrT80nGPD(JUdK1Xqjy@l#cA8a2T!T(5D(lEniG;a2=`
zE~ZehW_&&5AUsL`NMKN5rx~_Yh3YcYNeaQsE_{=wq~us57hTH7I{ea(ga>|bjfy!r
zQC=x1GR!u2*gdt`RN2OZBdC+<=S_=oq<5m}^NzFw5Qj(mef-cyD1tm*9;YE-!lX7)
z0Q?fJ0jc-L{f0j%vI#BO1a*VSxVUxRG0qd8dR|#eJu@{=<B-e67%DlPY8M1&?4c+{
zqp$%V73wC{^0`!GmdV)SrePne5ei$8{8}twuR%RrZzT*v#8AdiQVAb6I#&8{q_M0Q
z*f7I^+R{oeV<elp;T+89F~+FS1!RwNX<^`vgEy1v6LoZew~4Lo5@*9+63F(QXL*l<
zKWs0#O{#O|CNc}uGm*CaY4e(b^lP0$kV}9tm(~Ld^Pl&Y+c|Z5>H|f2S&)Ef_Oa?C
zaJ*n|!Y?7p?L~4(u>&=|^n0%`$e=MC68gehrnC~#_(p&~@q|T@qoZjnjOcq8cy2_d
zr2E<8a??-x55=A-yrOQY)>)O18sUW;ya`$XDjl1_QM}*o(JYwIm*Dg9mnJN1RellX
zTkltgB7YxGduI0oUM~Mm$7lH1biF=hKpA%3^Pm0wx`mbZuRr=<!`}#b?8!cN8yERY
z`Q39_pWFSVfrX`)JjGAr%FDw5IlsR>io5>pry-S!A^`#!TKp-z_2&^ZJbra*DlxJs
zRGWSC1-zJ4^Ys&jyh&IQL4Pmc9vI@JLy{_D2vVFHzX5=ls{MUHHeN`&uY{pnWd)RR
zBPUuqX_`JB2i6i?&DYPbGg|*JM~xhmVuocSZe&bkT*B}{$>JgFAH9|_GxZaDN4Aty
zc54?dxC+$fsU~`<R(q%cqHMe>m=r9MY<G<*Y9aFt6a66pG*taBC0ZYVQXBOsATY*1
zvMv(E{qJC)2*2}MfgNBUo+xZy3Ri=mQ8nF80aU_|)D0^!K_8OuVabVoFF9_Uo^pUH
z;HfKDH1vmXHzDg_?Tqt|UX?J8XDEtn{&r7DAm^(^5CoEwXqVVG_MH5Zl$aQirc2~$
z1j0WsMRr?if-WOBk%Q{Tzq#{bHqCeNhj}zE8-0(z(E{esb#L6QNDhxT<R}YvL-euX
z;}Ems^R<a-Ph-%{OWjEt4_gDGW^lF%`o?^ew?6#)apCMd_;+<{L?O)JC4cyj1l-Zg
z&ytd&Q)-VnQhxv`12R8|Fngo#E-`x}?g)PAuzxth$#<hbMwm714ygX&OM*l>Ru7?*
z+LvH{6lmo~+Vp&*6i&W68)xN@8By;JR73GOpr;`z8rtV^x)O*%!Ks@{O(Z_?w5S8b
z)Oxy#vUT2)m=#hCqx;ca#T)dch77&`t^*=1Xapb>N7MluRX<Q@hOCMQ%;R=yv|8~8
zQHDO(2#T^%B}H6mjf;&_2O1`R=P*RhYT6m3zO4A%mioUWp86~Q+0GYgyx#1q5~-e4
z;(uG?L3BkHs=Q5gczwRr@K?@w+etRe;!VreqUHNou0{rz0qN7>A%qAmS7yL|8v<_K
zMFY&$6Nd&wxnNT27mfFjw1+K?wT3NyMHIdbu^1gSA;_@Qx!|gNbcZ+{`;GA@eZZtI
z?rR-)2UtMH#))F`PY}=vipxFbVDPySriRy30e@@ax8CQ3`R0@TUz~kqOk`cx<<mwt
zjWzDjxJ%(qBZa#c+PJ$rjWke&yEQcKQn-5+?(U7dyH7uFGGAtrZzhvTR(AF|>*ORS
z_s6|=pOv-2#-WZV_JHu{75~@%_<G6PUxP{}2|PL6m-z3yCFlsSLb_6o<T$YGxbyLt
z=!lo@SJYIUz!Y>xtgC$tW9D*fc;$rL680`BToPm{7=pm@XM+v)GFG$q=QJ3Isq0w2
zslCt1_rmv4nQvTng<M`R*O@OaU+*jZ#qWqe)7Xb?v4CoJ2v$2uq2Ea)tB&8(yKU&)
z(YHfBbYQ(~a*%7{PP_&;`+eEC6ozA#SC&{p@B6L*8n`esU!)F7y;fICQg=*kQ2g)G
ze!Z-+sIsKJ*xC`v0bevX%T33?17T?WcIof5vbm}~$ve^_-B(wleACJvFMPD;Z;Z?a
z33WbD1}gbef3#EMpc2=K^egYH@wlm!pf<ZU3PB<-T*K(qy955aNF~2vrtLtSl|k%|
zUKL{qwd(|X)EgTt!sE4GP|;_$<j5Zx8|M%B$Y&Z?V^1F+f^XicykhrEoPM(KdpqZE
z^g}=1c#%(BtUu}G>&w3Pr#7@9+)VHMJy<!ziD7Q;iM-d5p3CQ+pC~vz*dDS|NqGXT
zozNq}9L{tN0sXT*Jf_zGKjLS^t9+S@$)4%RIprIK=?7}(r?IN9rf8B*BL@O9EV?&>
z7C~sf(#3bw`k;*3k=hF1DFZO3hl~7j0W1;eC%VVh+8H<PILkc^UPqvPEEV9!mtG$H
z$TZ6&^GJ=7b7j48{@479(2`W_aGVJ1^bfkhr0VJSnV(xi`|fva9F|D4)Rq)jI_fwb
z(Iz!JrC0A-pD-DrFM%2vAx})L4DBD{A9xnHUs0ZMevtRk_1Zk<EY48jxGSs3z~2sp
zMhqqs6AFL%kp~=lX4A+5uBTOgJ&~*u)HKl8xXqMMI3gX_&zANf4OgML==gJof#bzR
z@(n~oHGkpHs87&SU*nM37Z@$|5Y#BK=`d{@W{=1iw`rSe@vvc_oSM+206N>6vs~Kv
zwtw;Sw#nFcKQ@%E&McN$V}aS0-ea4oXj5*I!M?MGbP3~Z`0bpu4Ogj=UqJZ_zC*Gt
zZzTD1=>Q_vZprt<66^Vp)A8i<u_^>y2|rOtsi3U>XN2jfUisJHa%d#roJGhHM`y0#
z7)#=IcP+TO>4e8ccC`S#*Q5#a1>Zje%|frd=58+G1&!z9K!Ywp-Ur@d$gGCo`JHOC
z%`@MJ>89E3XiHhIb^mwK0-9gT0keIK38cws(W<5c=m+by_XKA;t<}rcIdUTxlecFm
z95a5nLwKh{xM|`cLz>lmG@K?=&xG|yT}dOA_cTyCV`1NYbBDAfo99oZzB5pyy5<(C
zW*NlF_r@2-&Nu5E?R_zkD@rXpUoxu-#5-^a={($In_yOx^5ap_y+6lUpK{M&Gedt3
z55R+2UM`#O5zgo4p+=jkRGA&(4*9#4#ms)iGmi%wADSmk)3eyWckzuKdJujf&N)Jz
z?UO;V&WwsRfV`N@&P{wYKUvFh<lb9K31xR1sH|c%UqN3Eq&vrwOJ9LhBZxqr(h{ll
z-0th+P?{5WW87n$k7H@P&o>B%DM@2T8U$W&YQrU_;O0FW3tm&WNiPVnbJ+?<Q4I~a
zLhi(~&slUB$hK5*epMEc>IOG!;2!tsBUsZ<O1tj)oq$^18G7m8elJy5MA`yK>^MR#
z0;Xa5dl<7Q25Leb6`1C|NlB4<9znBBb2e7||7;E}22N|KSU$2wiPbtTSa$%2cw1$y
zdizksZj#lQG}}KfSuO)L2A9;nlvW%Boak|kd`mZbU^-~X{Mgn?O2^`vByj!aWRh?`
zgAm$KhAOiTpQCHaepjEo8M%b2^2k0ghEw@O*tq%y^0Nwazvb+M>!DRH*9#d?y(S-e
z9&X<qXQ=Rk%lFAvZg`NCj3*3jf8kx&ey4V*If6pp&9;nBqDc`i<gVM0QgH$g5ps$0
zKqvR8bVVUms;~D-_(T*fly;9|F*9}*R}y`-6l<wi)j`mL8{)Y*K8J2S|Dvr+I_dYL
z_s&*cHg6=CCjc(4X>-FI^>%|Ex$paU5#}$neSb82GGIsl(;NMRkXwjiVuIdpSt;Qt
zVpm5ybH8h}bM0-?-g=g)o`n1<fi|`cUjWO(4I$P}nR7x5y?<1|i(oI@k90CrPP1TA
zp={xdL|b2%U@>my1vNPqZ>)6Se6vWm`YxY^J-&~>AUjGPbk}^QV>h`;A*VWZhr}0M
zf(X{T`nbnbUulRfh{?+19#*^MkKL}r^EJub#YjA+;!l1)l5lUI2Up6jewOzSG>Qdx
ztW{T)ZB>~kj6>#oir%s*7G9}Td!zHigfqZB<Kd+%5^g=e)y6A{L9K!4IC^{7HcR6*
z;zI}U0F-GU2)>QcD#RW8D?>h<;2zlW)p+!ojQiHz(m%3>@$`Z34v(u*!5PN~HykO-
zI1xsG2fwhFk)6%oHfLTSsiFPhc-#>Io`Xqjdd=}m$*4sPAeJVriC<rK*^7Bi5^KI7
zki1ae#F$TEz6SRaJVrZcVMTYIn@;l|zQ&V7T>(oP$ujjJc8^|(P47ycfQqkix=^hv
zV^&WV=tGaCphvv;hHJtz|9*A*(2pr$-D~u1fwb+}C#t+{LX<5}C+XwMLk6%W;{!?k
zD}wAz_w7~7UaS`M<p+ts)8(N9^-q;dt~=DuV$vWQ?p=OewT>%mi%rZ3%<<&9lYGe@
zPnNdmWf;mK<BHKX55}-_5%D+DXQ|ex%_3Zg%4VGA6K7JYe)_i|`}Iq&PzOwpYFCVQ
zwPWY&%3NGPDDN(2wPO5>tBFl)JiCPqk%VWeXCkv95QOU&`u-Ye;9|(So(5%;0xRR&
z$Jd89X_&JUELBceA%8oblKcvWY1Qr_osI|kNvNEBooiD^ia(B>Egv%xBC5XGgaTiK
zC0!0Q+5H0~hi5^II!%r(WSobjP&zEaxD5ifA%0IzB<FS>>1QSX=;n}HVC2it-IvK4
z)*CW?6}3J`m6WXcE8;tqS&sI&_|Op~+@G=?XF_YVuIteJ4+kKxrYzOP<hX0eH}(e%
z4ThVEWm@_#%g$uNbV%BmEr8%wuE>XXwRn+8?bOJ}qwnxyd~#C)?y%*kPK~o)3lnZ|
zhi64jABgJ;Xd3Evsr`_ae8yfl1=fJqZV&WUKdE+Qk{<+^K0WbITFWO1q}vU_Lv&d$
z)y<`DiJ$N~x=Ohk^!yf0BOoS~@@3Xujv~)!{g}5`C7V@qLO1e0C)+Dw(lXZuA-lfJ
z-(CZ`d6Ba3@y#4V$y#q5Ybwwf(g-xmqMndh&OH=OU`(_9*70@rfCl>qr0mFhloyP{
z67?*Dvv*Q&mi);jHuZDgkN8zE(EK*Mc*1{<xz#xb-_m9f_EpKbqTx4KUNZ6~7ul{A
z)-rthfFs-?PikI&Uhsk%tH^teYs>bUx#)`+X?be6%~E9d7J^1reJ)n3os6fMXv$C0
zKlbtB^2d|US7Flszx1``DWYUJIHecwK31z3G^Rb(FsfJ0Si-f5Q?o2*uyc{0SE&%h
zOV6v>QSB2OrCyj&8|b>56ztPrYZ&3EXEQ3P#4UVTHkA-lJA>0gX0JSH^z4gX5^OV@
z!07*1&bQ3Y2a;y;nB8cp<fYhF<ZUC|4AnAh4?#iX*XZ?hQA{Ph;Q5?s^ouvF8uwrW
zw^t_;s$x@Dt;MxJ+t-gc#HFm}Y?so7qngtaeW?N>9Amp!*VV?_3_p-!p-Ozzr6QcB
z*C~VFsP;0km+NY_uDtZe4?&FcsnFWqoN|BZ>pn+JS3Cx<{qgfl)sP7T4}}1@Vl6H!
z6&+G$D-8$|A#^447~ZKbsd$P;a%vod^@>&S_DMFpnMA|GpN*j!w8co8%(b73YM074
z@v`$Fs!5*Q^pyfr8>UQ_=k;x05+mMh$`l?chFQrZg+duglG|u;%7|fNS5AH!U@m%0
z)!9@n;9@4EL~4XkN_r$xr`QxmtNYm+&dcaTBW&j>KQEVm4(D)RHzxr2CFW>v-cseV
zWdjUt12;TJt5s~1BG3Kff`{1F8Y>AlH;^ua{%XK?ly8qT&P5|&;YApX;1{HAq8V9W
z^uZ7%1<nHKi`DF-vAdD7p+2MNPsuq6^=$3gnmveYdLWZNc9t}?MySTA=I^y|s`R$9
zFSfE<EX;a(alwssCy(@wn^g1h8j^Ku@DK*yfz^CRJ6|ISj0LWaAk!0~9u-NaQFFHO
zrV#?dM-`_h(c*WjoU%B=iC>eM=?rqC+<RL;QkC2f->PR1*n^%>om+(AIOYyQ_C|@j
znQ(48;H{>EZCFX?%n<wFwb?`bms(Y#7^|5M6De@C!EsGl1bc-skfzi)b7o#TcW?^U
z$HXdHj=DD!t}X2$3c8t(911qvk88{`DOaZ&KKUHaJ8={-Bh6PnZxf$LDJY&nL@1?{
zuCI|gj59#=MMuLz9?8gv^PC~>Y8HDTOS<uc86Hz~31hRXU813M2DRZ2M`>+VW@>7>
zb$n@oGUq;N?L=yTkOG}kuRSQwtf`*b71~GC(3{iXorSKTr`B|jX*7JU;c8<QNFyZC
zAyepNVI)gcK0ajDz(_^Kq)a7}O+_x5uQG*^C^8`}WN+bHuh7A!45iyKr9L#vo*Oro
z7|1LFVI&78#B|+?2l6d26yqZK%B>EDKscLDfZYi8DrKwjhK^ciIbY5ZwIMjr^J1uM
z7RMs8c3m2#iCAD$L<6HV-><M9@COKHf(L_ODhtMjMhUwb)as*o3xHWcLCz5$^IGEA
zNk%?KUd!TqfWWt!EB2E_gE|OH^|!Qsi$(g?)!WF8ntbL2j)N4URQI@;)J9zfEzowp
zl<e>_l*w~ACo0^)OoNciP7@XjN=L`+pgTv|i^)+<&xb)BA7EeT@zYF9L;PZaiFLFn
zwbGS3cy47go!`}{sPSf*BiwPkGOMMqcC}K>RC=s*?FtgZQ4Ne@$!EJ6YA_(SVUPtA
z_D@uEf2QFuTIM<{-#{(&o~LI%xoWG4z+SwX!86_@rO_|ew2>JYxsyez@rif?c`BdK
ztT8@tkTd3gN~fpewftcUQpsV~<0_FWr9hL;Y}7^MgARr)kBs*&Z9ds~div<v#7XGq
z%gj>mY0kH*-Kf|^NT9R;sNqFLoMu4_rC?bNx{9K&PS`AAQzf)z1r1fUb$raTwhbDk
zL?A?@jC1NpdE3}v-XzoHe50PUt=C_=;5+_3d7qb})(cz@x5N65Zh!^f9%;TLX`2L$
z;d9=N{Eo}5&hirN*lCd(RaxB((|pb&c6p=^w053Gs1s>HN6o&w%2u^$EyGquw~<Wp
z0kOd-o5Oh%eiIp_b<4>|vkIH?7rU3fKz4EL(@H@Hk>sT4XTLDpphaez$}XL1bqaXn
zSe{T-?qdqUOCR*X0vXWNn;hGk)}3r#j$VIkd=(fnPG4_@d}|mJUk~HWq@q&a;>LSw
zpzR+Vzqe~3pM8?WSDX<n;ZEC<+8fd{Yny8=Rab*-d0#bi**g@sO)PiBAD@?IQ4&kY
z22_^2;{89qA_$nIg$SC%zPR<0^h#XIdWxiVsSag%;aVG1HD`|9wmpflc5tgz2n7Lu
z+8ZNBEpvZgh2U6=Zu~Sc9dLD&=TmqM)C{uLf8n)$4n8g}_Ou-(ouWl%*c`>cVQ%gE
zymF>frahBkwfb^V)6r)qBaNS&^uR4pE}%+hJ@@))jNlc0rF%B>QGM{9BoHtSF@&}g
zc4zFLNIo4(=@+a$ph?X}id~||b%1lQEc88`*105~);5G=vf7z9J=KBm0#K{SbiK-c
zA3^@R^j8u=i$TJ?LA-PmX95+C7ZjvbSQHA{D+18}hkqVeQ&atZZps)02y$prT1bl1
z*nQ;vxb@M&pcm=dUaWXxn;C=86^_^{b^>_d6Xg5^a?2}y!6>O5-!<ZXjiCo6rSU=M
zXF|3aWL<FqV4x?O9(-f=N~pv~qjoKDkihdA2@sj-A1d>r3ALN5fi7((@2%9&52yHC
z!<ilwJ_|S9aHV;Pb8`hR^@jG>Y3PgIJa(JJw}(q~@EYQrAHKeOV5nU-D=Dg?vVY>%
zK7jc5Vp+T8*?S2sRhcB<^s+otZ0-7kbAV&1t|imaP41q16z2e>@+GZX)LRNVM}xw@
z-NL~-#92))`#a_w_GHHsF&CJ2Z(Ru<)2^Ak&wN`OpRuNabh+KF@@<_}e#Md$`Ok!z
zD;ai6F%Lp)!arv0=mja;kW#i2<}})yLAy(tui8h^=#|nc-a)GdxdJzu(?2RHiuVMT
z`(o4^`B=u<hErOPLB|~zMlKW<)g7ZKv)vhW{MoHy(XF$Q4bTh3xq`%OAvM>{Z5a4(
zTO#>|aYRUhp{M6(NRVnZv)=oHCQPjr#2#5rO(@^~c5+L9t~+DlQ$&cf?#ae1WB;J8
z5^n^P&!(7Q6*U1e?z*BpX9cqh7iw3*0v(kf%9zE8p`%yze$!w7x8Mh{Wti$(x09wN
zrDq67P#-C3rNLUX<dJqy?uqNg<G5c0QtjP@YqV`KGE*HBVRo4V*E2d2QZ<q!`X<kS
zwf!B*uE~xJ8}lm_c-a4MucB}4KS;^5Uwjkn#`IQi`0)LQLm%~?0{wn&NBa&x+t@Bq
z<hN9Ly%!dx@%@zT_wHwhIg|sq_Iu<7rX$Z8xfth<js)pZd0-ijHc7ANy`H;TW;oZM
zPnI(Tqx)coL_7S|c^-$n{>IHv)Pln*_heTqm51F2>X>%20*GbrR!zB~i`I}wQoOGg
zs_+=%`T7UrUUv#6JeitxSsmOP@N0O-UpHIs23xuZQl%r!b8k9B<%NHqe)2ZHvu986
z8KvW(F=z3epo#OByYiKCgcb`218-5vUYOkF^sjjrqvw)Zzj$Fwcl4FK_mMN`3cY*x
zg8g$Vl%O0MO=C^M*;W;*0ZaJkG(0&Omw!l<HojSl+0iq!t*12=o`}`~GprGAa~se}
zIDeA|yN}D&Gbyb*U+V_fiC#@=56vc~dRB5Uy&|-8^iL->Z8DWH`lr8J3`H5j;kMST
z@TQesYmJ+H*vHi>d107_m7}rX^%kTwpE?i-8);h%s~-+U_-a)V3L{_z8LLjP{`DL@
zx5c8j->~)C`pSeV&9bxPC)UwZ5He!_l~KE6Nbgmo^(oHU*lfxAM`*{7@A^vyam6d=
zX&rGbO=&6>ph_J3Mg|QDBB0DdF3?(KIMV*>ZP9#%K4%xxVN!&5E#rMY-FqU^?D3r^
zhS?Vl|C4m7&(1@}j~^PU6k;#1Eixkw;{SLI?$G)e5|z&Jto30co^Dw4pbX6!Hb-oo
zejV^c0KU^YKX^<LVOh))ez(6gU5y5{y#JW(1Y6GAMKO3lSCcbnm$bbf1m_`ExR8dO
zQrusLk6r-nFGuInNV%U720RJ;M-M$bXcjZ|+wJ7r+z<pe?u+ub#P}B@#kAt9oFLa|
zSd)3rNkSWC`brv4?yOCJFA=zOyG6}nhAf2SCW!_R;IlI6{OcsTnRg0~;nSqX;l@*V
znXAq4WBgoO9k$Pd(*q@ozrDKH4k-kueIm0v9D!|@Oy?9P$Qzj2&&)2^Nw}246y8M9
zJ4KkYo@z<>9&v;OV=B5nzt8@u-{bVp`=Cx|WP70M<@4dfpx<e)>{H7NN##?p)vc4G
zw~X}5FRjL>ulnBPk~dS;2{Z7NKivVUn3}tIu%BhGh{mg-teW=no-YvdFK1MKq@>cP
zDkbV$ln+88U4${uBbP)M=5V}wV0g{A-QdfoNEJILZ8%Yz>@O)$-Tl)1iFVt$&ff~b
zsV6BSz_+#<6QYE9TkU%u$C8<AAFe*c5gL0??;nj}a8Dl*lDK$XQNAX+*p1$KDR$J?
ze0Fii)RAXT-dg<e!)rt|u*;8jZm-Z{wfZaFldHDgj_!?*95;rl5aqG5hlby~<TEfg
zOm!PyME4pjQSfCz-8aUI&h$&DWlB-#3Hd696Z42SZoPS4?-~R<+E-seo~Rp17z4^(
z)MYomTd|OklgypRyw_Fz?2#P=?4dZFL`#L$`*iBYsAZ9HF=d3(8IDvZLKnr(7FOX&
z$*x@@UQV)U^e1&|*%yalyy5UdPNvhZg}A&3p*F}-%HE95W^ztH+5SO~N*A#qCIMah
zT3Q;yA;emB7E^(VYa&w&Y29;%c9(y=wIZ|PjX#^t8S?+sWJ;~1m?-bPHjHpXj>V)-
zR;{Eg@I|>^(c03nf3+?mq>ia0@2bA4+p2$Sim|$hh1&k{c5Gi#D-XYGvcAz^g9PU5
z2-wbhCgxI*p)X{w{3!eU0V(b+#P7o+LmMlCkKwcJdi0}^8YER4b4}FxfkAXsqy6g*
z*$J(YtXtXHs~Qi-&*Q|p>@3Nl_<dI#<rC92!?(CF1Bq5UD2$2AzZhT-vxeuR`G#2A
zENFgW5Y2n#A3f|t11!DpBZueLTVi#{>tL^#?ebe_UtvDIo{{1RX<fYH%2rshx+-y*
z1FU}TcL)a?1svg8#4tO<=16mO8pU6a7cl0EWb=rzM-|KhYH~0s9K!S_p9>`-q!vP}
zBu^4fgc*#!Jnq%>eOqpM#R{85%;P7lAmcwEcGA^_aokKX0<*Tiza-o!^}NYEU^|If
zra}ODujC%2e_GvAb`vqT6bi@e_CfDa`@R_M>F55_Yx|95%FB~S$e^&Wo?(U)!E&Fa
z!XnM2yg5{-y0K(MvY%u@F=&HmwCM69?Sn(`P*wCs4Ayicl#x&}vLjRrku-$UYAP$%
zF(9zc-*5Rxhws}ilKl!+>?ql&Tz|g+5y}w`GUAX%*$Em<9y!gs$URj=R9R2-DcYLd
z6e)%+RJhx#^!w+n;>ZNz4Jl&mo^Fs8A0?2KdE6mAp=1T{z`UMn&Qv-&usn8nFHZ0!
z{vxa>69*st*meZ4egYF{@>utAF6|c8QJXh61|tN!MM%Jigq)AFI)OzWn@XF)UB;Lu
zgRk^gsI=a0!;SloCHq~$e?>cYmpN{x7eKA;`myEPM+c$?@o7Wx1_M2xHJOe-6($5<
zK43nT93srzI;rAZ!=w35PsOGQURl|EY4VhpTkONvAoa@4^5wT9r_lp9%+0-hbQNJ7
zihD%etLPIkJ4FzJmD49-#-m4=9?!~@8lypoTi_0CG$qhe_j%aQFJ<(1qZRc*_bLQ`
zBJ_>2*P2uN!6$oG_rUya<t+ouIatav2SZxviB2D`$zg$|Xa<HPG*@#OCm#6{d~~T4
z{aNVM5P><`9e7o=tgarSEPUIBHJ=Y&Y#bxos&a$ir~Ld<dxU#4<dPwCplGCKC7>wD
zBl?a<W^*eg&oKk*e!q`aOOuP?hP+SDLYV$@RQRLafn==v8o;;XAZ%VT3yj(<BR0vi
zxjrVb%a88c8T24Gz0~TO@VBUgX7#aXr`aX(!Qm4P*-3WjTTb6@f$*ig%RFyyScZ$-
zuu4aQ8*r<(rNA=c@4gl5pgHLdRR#W!s>Y@c!RM;eblVHw1FIv_*o(5T(3i@m4t_(S
z?x7~?j6wi5Cg)QWf!D2y7pRB!v+Q7vpr*8{AUSbfMv=Sw89li6TGH|#&Q#7hUWj1J
zUI)G-DNfz5gyaU&Mc(40_F}gY^UTFL7&7k(t9XzlrzUSJJiU=}Yy-K;ZiC(Zs3s3T
z53t+-&%~l>qQ!L;nKg(fA2IZw-hRlV_t52G{&cHwo|^9}ez&uY3g%N!UN-7n2D)yM
z)FA^`zSU(v_{OnUZhWEe>QJi=RkRicAt|tj>wUQuEGV^Js&xDvYGm^<#!<>dAxCzj
zt7yAj{mK&k`Uh&A%co6N&fA=wuthcgNCp+t3aC@^U3|K;?2BK-q||hOQ!;|1SVWxj
zPkQFBZI)nx1*^4yFaq|q#3%};gMp(=(RC3p0b8_~C2hHEbQ+Vp^2;@?tY7!UUcTM>
zm{5?E?6k2b8ariHo5pg3Qj{0-oF1h-oJ9IKBjT5aw|j{&?8CDAx1Ffq6W<+wf!`0q
zOCE2_7juM~4&oZ^EtW&!E0dPP%Ao1;l-3!+t5o%KtoS`dC;p1R21#Xgu6;RaRqLP9
zK1cn&Xx}_T8LVJzmSwl-FHs{En>KGm+K!i7c%IZ(14f!{(>cc6{C*%vEyO!fmi?n&
zOo$(ax@u5)m5O+#p55_bOmI<=N?CJFwU+Rs)Gu|N;qrQSDJ%_LGY!*&HQ4jcn;Anj
zf3V*G&!{pF#5qGV$I5>C3DUgfWZp%2qI0Cbt+0)d^U^F95sSEwRn`Cm{mxG&ga}Ot
zFzIck_bFwM)=p=IP$;g+Ph4kGL^yyfQ>ZRKWDiT|;VFhQ5dFgJ7Y-jAw#}`miww=r
zOc|o(7>o6dPT8z~AR!JDlKT#R=O=&1CTHG@U(zI=bg>aAsPgb>@#ld1&)YiE7bVwA
zkZuA8y9zf@-Z%FK$_@KMEItYV{74Iu%KEcO_{E<sNKkQ9pZ*h{eQYrgPejqXBaYq0
zFYNRBGq4l|k|{63iNbfFN4kSKFUS((b{t;8AY31uk#NditKX?UtsdbpEu$bgvIef$
zV60mS*vG4o{^8339E|CAP0iIWq53F3Em5Z__sMwrTiv42P!~Zb*=Ie;8A=pq6=ap4
z_pJ|#t3@+&>%0S5CpoqJ5w{AplK$emfjm@t*r7amq3evz@ZZbwk9qOR$tTaW5sUFV
zk}J}uU#(_mI6jTiU4FW_qybPKi(Z2)NiSpwbE0x-#yyaG$fxh<<YkpHC*CQeO>7Qd
z>j4r?{V!07G3phIpQ<EUqI>s?u=f<`Qwy7mI(AK0qg}HS{C-q`ZvZ=($IMQb=UM0@
zt4C2qK^+g|R_x|bRf3=*o<O02-Um^zwsVp>IB0`Mu8?vr<Dh+Phf*)*S=oK0rD{g5
zSYcD^2>GNK=@I`(^~#^EMS$#iVb7_se_SrtRvh~pni2PU^c{3+T0rgW_jSvzfBs|o
zHYKx@9(oU8RnTH7oUm3$u4{t{4>;{vJwGBisP{0ec0rJ}p^ot!)+{|rc0lFHb6}H2
zW_EIU0Pu51iw~`fzMlU&*P1SH@lQ{NXQPQq)S{4+XH=Sy{$^=}?{I2g_LZYdW_FTd
z9@2rBF8RqLr(35gpV$G3$r1Loznf93$6DAqqfZZyl%CIpG7@}qeDx=2DBM|eSxFCl
zg2pw23`Sb6bVBhd6t&YStO<(N_hJ(@Pxx$^lCgs2Dv@E181ILtJ%zmD&Oh8CdI1op
zhfl*Q4y4&eC$pAOF6~s-c_loFvn*()F?I2**cF@(&>RLv_}<SJPWsYxw=|xLX*sR^
zJmVm>w`P5~sw%qtqdeh+G+V@e{FG($zUE~}{EW5ofyA*k3KHN)x+oJ}?bjyns=ve<
zjn4T_pn}f!Y!z!IN6a--qFAYEpmlcfBK3!X!lR=l?!C2y@^vcz8l}2{@c?Gsb_<T{
z_NCmV%;871P+04pk<k%unRfGj*@L2+r?*x)2#s}W?ThJd(RU@<6MVMoup~&v%k9c3
zbCEnZkmW}DM4{=1V}Z^7lkYvrq=E_@d+XrhCu??fCMD?5*svK%kmzcowA)?1e&$BH
zqF-oBjo5rxciZYYwRjmZ0XyT{9hZ}NuRmx2CEGBDuh*{PTUJc|#g`p?Eq|xrLy6fh
z2W5-!BZUbK*I;U>WWv>-KpT>51*c!nEZz+(MAeva=c>#D{M1UTlFDQCmer^ZNohM@
z>Z~C`*j+AnFYy^(F=kE%zgzBDXzC3o{+4po#n8yTzdS(lgI@2Ngnd1d+n%RhKm6~G
zKx(Wt=JVwL{1c(}|6r_P^M5nem`?jxr8noA+2K4Y)TZb1r|n}7`g>)mc73U?lB8p;
zdhD>M{C)scY(j4sHFlhejdaw6_i0;0=<45i`!^7Sb^|q22Pm`6<AwzUmRoL{AIqNC
zT3+j3>qbd3Do<Y<CARH-pY|U+yl!jIgne#%#$L`ZPkpXla06aWXoMRZt}dVGgw4<E
zDTrUaz_)E{t*`4eq&VB{$wPCd+HO0Sx5Cd)kZ7TM*slNWf~0}Z9f&||=h8oAO@(2(
z#Ii}^B)v&HGxpZ3A~U)4Vk<s2Io*1YAAdmAbD!a!o<(>cNljm561SwaJC8-h6DVy^
zb^(rW7vD~#^erA_kbhuH)VwjXGZdp&MN?dzv6Mk^Djp<M%jN=cOqUF3si*a~D3yw{
zmrSIjkiLjr?tBpA&@UD}IW6A0oIUldLom(d&@Y$1ggWIDnCfv_Y384wTBgX0@JD*h
z&}CLA7ZT(LO_GdgT1}V8Nps_uDJbhil9Ei<`A~oq{3-?&lTxz^FXByic1xeGJbwH;
zHYoacj9dI<l6nuL7Y3R6h5u)|<lpJ!Vjoa}#&^n^&ra$hf!Z;JpE0Hbebi%q<6sp9
zUR{g+Co2woU-w%Jo3iKwRmoQoCR+6mh*l!0^k1e)xzI%qR3Y}C9V?N3*dxZu*QHJF
z_U~F<RLGEUT2cn@3~IfhBanG5@UbJunoG6@8g)>Kcu1Ctw#U3AX{3rQ{^PEmzXS7^
zlJYLicSwdA2+7u@FG=V6p~!?tleLRC+9^RA9xXVE<tKh~#D^S*+*>^jd{0$^JPGad
z_}Df|^ejFNh)1R+(%xZ@SgFnepGs92qbfT@Pb4|YU~@@QHYFFMHmT|%rzev|I*B_M
zwv}d6txv$P&ym4--MZ>2F`$&f&zNjTm+FS1W`;lL<@y;J)j}*g$p59Z%KoKf)&8Z5
z|3j&UNQZ$VdP`|Fv8jrHo+Y#*OoN(0L@VVV^fF42&+5v7+G=^=$rsqa488KdO!2?W
zpqes9&RoaxSY*fz0!9e}Mv{cKE5YH6sqoNCf^}rxSkeopCwGDp&QQyb2+kR+{uf2*
zy=u!GbJ_V4FtMtp5ZZYCoNn?`YtbcSWcBt#{7T5cV(WD!N8C3znbZ5XWKAL{;NaKh
zw@j_G<r3$6E=o?qib5O@AU5vGTnUYttFoV*p5|@CaTc%4A}l9gr8wylcuduEJnbVT
zcrEMNm{;(csJoKJP4Yc9b6o<!k->2ry6PE9po~H=COza*-BHxk@dv#^5hJr23^5S~
zEaAznZq&jq+)XUn2p-%}`FE{u0XY7jzjp{0(iQa5O0pMf(t+A>l`<I9ac|Y*ekWm-
z#J#V}lS-s4qJouBiZIbBQ|;g9|0h$Wk`7qS!$6_L3(3i{nGj(opAch0l%pgVD9GH~
zM>&_ix-NNNP+>tSE!s5MLElwI;Y)G9EVr{|iNOJl|9U$I=OVb$99fGMNK{XKLbY7*
zF;?gM9O?f}0UI1@sTS#}6K0bF3=htekt;SS`N`XQ4G~AShiF$b#4j1-s0Rr&sdZ3C
z`e&Nd=44uSucelqr)@O|E9qS9W?gu3g<$lj6}wbm&bz}4H&r<p1@CrF?_zrkr={>{
zN7VBX<MTaBI1H;f3=*m3l5mP9g9j!>)zer(+IcdV)UptD%Wf0PaC-R^Mke)q%M3R4
zwBEwu7!8&obyGwuP$s>s($IxEC{SDD_aMe}wWxa1?<DNv>i2bJT8UIerc@=2B1{bb
zS(-{OyD_D7=PSQ)k+fApS;!fS*BTop4GBJ^OjeJ>`(#X>AQ4Q-3yqE7Oz_~Gsy0#B
zcFCeK4i6xZA)`ajdb7Bbvr5Xz630fCxBbEu-p}t`7km@wvdWN}so+7iIhi(So2x9=
z%`Wh3e>GMvnW8aks*%h7R(7{mbOLxCRvkT6Vx5fIl0frUuua)3v9KlkeM@##;cdM9
z!d0y#8z|0Hf7gZSVU}RwJu%1is-p1Ayx-}iqW|T>BWc>H@A+8Ai&EQddbE(k>f>=}
zmha0%VOCL=@56rI%g3yj9dlBDTDvUYb-$;5nhq(Q*<Ax4>h1Q6r0Lfu7VispKVh9m
zgAU}myFXL6*GzPqtGE8Izh(%uD>4}>1q8Vb#+b&Pk*=4?1sk-+m?oTMub0`F6zx85
zF@LlBC+|jfAsNg9(?4iU@F+gfZl6}7%TnRZw~%Q|P@z*-`J6W`K?k|6t9<hAXoZzN
zg|%@#+AuHZ);AeB&+Md)xTX_waofI7G}zcJEkm2EoyT_CN8Y7}az)#aEIp;ra4FhK
z@I<)aZL=<B$BUVVGNbDKvM-;meOo-F5|>XPOO`~yxUETFiKSQEpkh@gO2?^Un%601
z8(U3|4FrczqtlV82-cF*5vZ`_sYxZqyd|p>84Of=PlurL@xXUm)L(mb<8*&#d?h<n
zOfgg+wT<|J=gM_&cYJMK7|?0>yT>CG5!ILo&KKc~x5YYd7}g`qI&GNJ;}9B-I%bo(
zRO@1~J-(P7d6}ybmq0}(%bdi)h_4k_xuqxF6lY5{ArmztLs=)Ikr+(hq&8V(q9mOe
z$4zA_Q$H<3Whhk<!$F2G8W*(nK^id*g%Vu~b$n3#A;$~7)k4pN`7^E~qyjjZvtwpJ
zeD-=iC&OTWPGU>E$Xn~a_h)<}d)Y9q$0U>vRh8JUn<I1|m7U0pZ`dj2n6=NaqDLU~
z9#z=%M~`SI5EVk4#KY*SwNtfh7}6sXngty+tm`2TMMA|TX6Ds(Yus>{Sx-`R6dw=1
zZ8~-sUT^F{42?%EBvRndaO6337+rri%75fCzg@Lt7~P{E>WlhFjLJjgau*|p8cH0+
zL*%NpSv70;tH&!81(lE?;*;Mfb?6AH4pBXyjr06TRnK}_k83C)YR8{~9zFC*S={6b
z<`v!MmMe$-{mG&2<SV!R&B?j^ELs0#L}pCALA?{+Wq&&089Evf37>$I$I)5udU4Ux
zdXt!JvIcWUVVx{tay|30?qZ9J!~WjnfNAiMRI)JZplR)pOENyQs$M~poKxx%8_YCd
zsDpLfG-l{~G8wa&-p?jvcl=GZS<?(1NAVb#zgg=LUGfyOj&6Okjq}1ub&qM`5OcBt
zbjdV&2r)UHSyC^miO4;5pKZvreyDHQG4+sb%(Q*zZE^@RL>H%p(Ghf@frb7=F{GQ+
zlHtgESlw-!KSZBA!)&cP+w9ZLmwe4ksLR!C<Fs((G-_HoB$WKjoaNNH{d<TkxrbR&
zx1w3sx#`5IXS?8$4g6<r*~xS3VtzaP$SZo5L{TrZDZ{<#&xJwbp3}f~^$>RQE;Ezh
zqQ|mZ)27qhHv37_vb*P>3njI6z@Tw$pF=b{iLIbe;V((=XjBqnK|W8Ho4pOdxN%J1
z_sYPqKfBvZ)!S+!n$dP7D}r7gSJ!*H6Klo+l8Vv#Miy5$dz*k><Dx#+XjPIz!5xp`
z>l7N<$&^!Qr*U>4c{GefRv^iX@lxvq+FkmE@9uYN2$89#SJ{QuF%-U;(T5uSi=;)s
z&HE^J+xqBweiyp78PJCp4JFwWfP2fGrO*ajBzVhRq#OZY#wC3m(fcIq0%l&r7y4Xu
zwU~6Zvnj`p<C`&khS3xxl7dm5W!GA}Q2m=WI6`zJNxA@+x6S$d0d&N;r4K*4rh1Jt
z$~)tXXaBr&GrP|@+MHxpkl91`Dkgr2Dtd@SQ-H)%;HL4wab&Z(4<$MsTMCu8zw4l)
zPcZtCq(S_g#9bg$$}L9l5Bo%6rG#vjhETch;$7{v<?hsKVUVI^mV%Je)4|actguv(
zJL^P9(8ufH>SF(Rs;`pPX)k|5)3037KTBFD-WPr!aZ7iL|A$@4uN?k0t4m1C=jQ|R
zCH@I}w|U+eZPr9JP9#9c*q7_h=4xT9X4yPsOeRZRsPs8m8TB2f+oR3R!bZ)EdE}UO
zmaEXAZ~eVZa;MeNS?UJ+jCtCaOBTKmyRX^(@J%X(;atig`<QwA*xRfSA&3voBjY*f
zphk`>1WuCmOQ_{N({Jl<4FS7COz*Tm)S}Wy5m`-%<jQ|r3J4X7_91x`IQKlblyh_K
zWLmf+Xz)F`TfT1Ebza;3G3J?tB!uhB98S)`{HS~0bl^O)+dPJnl`iD%dvkwO$@=#W
z#IbYt_ZV4LkC3EK)C18a?}>BwZr%}Mq^eM%@915|H7`v!ng!`N*>B@-g#k?DPXQ-y
z#NPfyKt{qNVI}`=(peNR(KSRHT26i*0DU9)&g;V!>K^7e_iux5uK{n~1aTrDAY)K`
zx9rRbfW47@i}RilIfcqZ$`>&Ola>3se&<sF-5b?+1s~+lQpnUShmrAcS%tofcJ>Bb
z1W3P$f9Lk$sN$QGM~ET$N<iD2$G0~hi9Ye*InaIwbzTHuV>mYTebU0N;`=V&xftO3
z=I|ZYdm>aG@<)+Pi%w|3<{O%~3JAJLjU)~fe~dcwf9LcY=Jj*^k?EWXP<>PQZuETy
z3J<P>(BFSV%gD$#>2JA)M$zU8s+hiecSZ&Ldc$qnW4-*jk=TLek6mY5z~dV<%J}yJ
zpFN&b1`Q+N@d1Tz#wd+H)*|EMunCxm_xHyrlNobLghrrKnCSN3#f*~ai8lXqLE2{y
zA`ley`g!$vpO}r;L`Etrrk(6r^kE1q@(o$JsCH;Aq9x%ZbDpnMXiOB@uWm3!D4*PT
z$`>|3o`8u;e`kycS&-PzU}Tg9>ST`k(4$YOq}5+d4Eu9pU}Wi{T%k6xvIII}vms?@
z3ty@MCZ_!*F*9V=qO(7-fvC=$1wZAGQc2h-Ow9VrViw4pMR$G<$0x%_WAw;c#FBy;
zQ9wA=0w%!zsTlL_0QhALAsM%rUC1(O6Q&dQmVW<p%*XY=kN1O<`<Dr>9KGJ27)i2d
zvCQDw2n*~9KBMgE;g~5hbJ5YC8A!aOPGnm~{rNFN)x;bX!MZ3-I8K5liv6oGzGRPL
zsKG=DBRP(=TMqrPF}Y-YqEVq4pLpp$gZGHliFnlv!zHr9+R>lMA0&~(Qt&2(a>T^@
z!W7VDw^7M*WX0#g?9f*zyu_~}XroiU0Jw}&;f-PE=<g^HfXGpJ7yu3<t=_z_b!A{|
zk5eQ%It7IUFd_;M(~-@{p!X&09r{NKjs8^DjbKI)4mMO*f?PwQ46H>Al}IXsavACM
zK7^s56H;&iZK8Ov9C<dRuGj_*<EC%IjM3vM3dH3iQ*Z!+MiRZlVIVf<TTygp@*Rm`
zND3+R8wUoI*2u9JBQ+mNWn|r3A9jq+Pwpmh1fC~`(il~Kw{nTzN1tLZP?g(FVFrwm
zXG+#aSYm7bJ+Nt-#9;~-K+m`twjAy8)slQexpy;+Ml=$xDoqMFHd^d+3ERhW=G@Tk
zy$&N3&47=D>7X}IoQN++HsU(cZ@BcPhSgw8pt5t00yCl;zc>OmRC>3=7^*u{@B!?`
zW|qU?RAP1-GrQr|lzRY@5e}RY3}Q^?Na|&b1mV;Om;r|eQ`7-1Ms9sJ(FPK{!?E#B
z!=ovJ0B>V2*j3a%<`j2<ft)`mXZfHPIRYPxoySaScs#`r&|@so8-ewfLjo2Ng^$V3
zWu`lPmx2huG#)gjhV|QJ5wTO5Sq;~wL;}){xcY3u7YJ&~-wtFGSaWFfm4$-{YnaV^
zhY=Po>9h+i;&W`lwghNKBya(+C($LPnZfW&$~(YEW7GkzaULRP${mYgXv!vl#z+C4
z0p=xkrmmoMn*Z&{wWB+HpMsQgf_1FwBIgL$Q5iN3k5}_m_8P8BIRtPUWy;_{JV~b5
zfTdXJ;&*@sBfGxk@FqfMMU1dsepniowJJ+4(X3UOEPx!=kCm$up#Y$U&0}q<L?{Bt
zV0@9`bc3ocG84SOWGp>Z7ug9xU^JGXYT4XgVvajli0+|aFFD5_{Duyp058F)YP^h{
z5;l(2qk^mm<%LPCv%xa3oK=TqpBFY>=bXkr*NT=)G0e);s+5;?b^HKF59+8D28}*p
zA*nFRL1|$dSh6Z|G5}s!0+y<2!=9WhKoAyz#jZLm1K@{EVhyP<DndnhCPAr2i7|0T
zL1+0>z`{}m?q4T-f`84{?%OycmJ91+-Q}k))i3@r%akE$VN@FENdnS)_Ew8ip=l)T
zf1vkFU{j<*Qv&N1#nDhh)Gsnjis@-f&pSXP4MRE{IS6eoI|sw@>idl2_!aD;=6R#E
zzb5!N@pIWV3L6wif3@>HyF45skK-4yODUQ>`qK?nVu-a>A`dAJ=}a3{9g+UZ=Ie2x
zK9rkCrFR@ptzZ``q~qR}s&v%ZjDqV$dK27InOh1>d*qRYdhwFT%FlUFhZ097@s+!1
zAJ$G-mip@?-M@n3UkgkJe`uQ*!kHsi2)sFNtd6GUD;>XW{{FcfB$GMAL+<WO(6=R_
zu^9}P<{h-in2nKRxR<AIhkHgM5#Vw%Tk0ld6gaAvex=%dzQC!B-$;YIMiM%ejpJ9c
zi<svqG>uaJO65yT$!FIwj~XTU)yDVea&t)Z#m~iUU(KvxRP+~+58|4%$2bEjsL?fV
z8%1anohA&oVCOopnOrDyRx*c-YW%Y0TXJ>VD;xij$XDv3aacA1Dy>o2jTu${Wz9F|
z8vONFI$wi}-Qn^?6T?F9?>Z5}pnCX_hE3e-pIJ-u@zvi!U4p+J`OsWQ4g@AW%cdqe
zf5(fM2hs2;x@PVR%voBF5B{kdMJI+S|1#$5b5+>OnC8uQ*4~X61^sg7quzC8-qW3K
z%5w(of=3yH77(0~cL=a0-$z1I%vsmI|L7780wP$W%;J}E&wsC)=qxA|;SM@M5JdJO
zxMJJ09q;>HDI)wX^abG^G6DfI8=eho@AslE)}VC+iU=_I+vHe;Tx1Et2mqcn>*)8^
zuJ=LV2-^rGOUU(vN32>lRlVPfyV!!Z5%^Iw@b1buv`niOzejda1`QynA{P+Iv8TMK
zfd8}%v!+;94ScWZ!U-y`UPs9!5McMPJ2S6Ut5vDhxKcV|o42j{``y|l6!eVXKl&yp
z2q9D)8$l8|ihzin$IhYqPhOXS<KUm#F8rVxgk9uOLS5EI3x|O}P9HG32%k7tuvW13
zu=KFqkxyidSR3unI{##gIFsNDa5Wk^%=}5~at$Ix;6|~-U*@{QT%=v3U!+@P;G^ZE
z=cD6e*rDGk29pO?vdaQ=0FnR$=r@2IKo=kiPyxuCqew8oY2oy6I%<Xj+5-9lx&j6>
z+V%lCm?+Ex_8uk<Y%+?=Da=XEY4E}ap5b08U#VZIUTGpLA*&&)AZs8iBdb60-_YOC
z-7vU;KVh|Cdt*VcAlR~4YsSS!sYXpkAx8N|iAHrskw)c4zcyHSf6wR4m&_N;SIlSg
z6!GL~RnGsWkNR|Q!Nf(+MaRW}I<(5!#H6CMtiG(etm%{f5gZDZ1oweyz=dFQ@Gdxs
znpnPFj3%C$`dDr;-G!Bx+*HL;&QaGfr#tMAB5yHIK2JGME>9^>Ax~xF*QU|Dx`Fgs
zfTgvQvZJ&k$PwtM?kMl5=O}iL!;Re_E<}wakCV#C1u|kAG^riHyEMfTVK$bBr$%t6
zerKD=TQ-RsFo~z5R+Ya<F`JypOP6+!C!rRU_e#Cu+Bcq@qs;Q7ewIf_LFT|WVVgGz
z8&HZjr0$XbjAzI;Y4U5pD;|Y9UA{g=jyu(mZK{liyhxI)M$RmK7?5hsHfGX3AQ3N5
zEi0dtiUnOZ2^o-ySEnwOpG_&_<h5|>->Q-ZNXkW}5wY{yIdyO44ba6CQgh4OrD6l$
zQ!R7y+B(6v$_99w3^<#<darK94VcEORHW!~Hkmo;Vf+{%jxVQPlB-WYVx==K9AJ)L
zq4t)$NjqX+uy^X*Y9DAzR(M|yd~eS_Ym@=xhuq*0GxBM>RPWhLz;a4q{4O<nb`w2D
zVK60#Ax<u%giZu>&;FbfK?}JS&jc`^QwU^%tm61GK4@RnSQMXkZe<Tp#P?Av%4Mbr
zuzT7~PUa-4i>RH6ctRd(CoOCz;5p?m2M9V21rwL{-*=@cU}>WR&}USULCm<LQ&z8K
z44pNKhw(wKab5@CLPBsLj5s=sP%SIyuu%}q34*NwjP1fLtJN|EtQrNv#34=)X`Fbb
z0xh|^JCrHL0-fP19!tl;janE!gb*j0sY2^1Y2@z?_1ZQy^LEf4qe2)nWCh2&ELk2w
zn(40csBYdIy0#Gj6M=}~gfc~G=~gvboGa>9H(EK4Y?Q%xA(uFkOi>`B1|GxnF<o3(
zfAK^?y1aWj374R)*TPl(zVRjh!y<CkzWLNzL8QD+x*ONAE#JILEkoGnJRFWc_IoB%
z3uZAR@(>-I2Bs6O<+?_5$F+?|fhrG+^ZpGe3=>j_Gsc(+nYGnkc=Fa>LaxQPshKgW
z8zD%q<2tmNU9zcS0Dsc1nlOu(ZyKRYAK+58DOi-NNws6|+4(qJn`+A5YgRPEn!e6O
zVI#2^QH^hV$!u%`UyP{1w`cD%%NfB*AK{X<Nm^vd{q&M#$~9!GF<({-GG$*fiyF~Q
z_vE^?eR+pugR{t34YH|O+X)yENf+Y^wT+s8B9^orT_URCwR7&?i5LN;$8#0hC@ek2
zAaU*5FfZEFEZEibm=%sNr>}5%AHj88PJ<NJ>|@gHF1uaQmo6c=68g!C`gr%p{`cLq
z7S}bn66@up9jsu5*JGqykFm8@R+-rs7*ejwYJ&luDQ4PZc&zYNxr+g7g>*9u(Y;kO
z3&Dj<Gs}qq`%3avNydq6%lrNIvS0buX`s~{vdYwo8@*#a&cW0po^2+C;(MzI`ekb#
zpZndSh**55)(CWy2wLMjHi`TQWnN>o0X%hfiMj}7LF4IuJkD1OpH10Z4YNEAGfD>b
zxJ3Mc(FhZB2`4rR-qhH|zc7e0Q6DPS&R2;ntMZzm2hEgdl&djSsOL6ovKMO49LWoh
zR_K(PjVxZ5n9h}eDmC2PB99Juc^8T^_!jcfP<9iMXBqIX_)=s4qWLde+2N!t&s(y&
z`tZX4f*-u_zaRoH{GZsJi5M%z&(dTUsx<Q<_%HbF$E()-)pW@mr>WCx4wUFp6>8KS
zh#Jh$b6TmD#|~CJ|Djo^E14@+?{L$&F3TX`y*3ya94$#q)MH?vX>q$IRjkjLV_Q4Z
z-XYW<^69ZoHJG5?ocf=*^n&kRtHJwuIx^D%Y&H6?e(<}^D+7N^j?q8=JqG@8{I?kS
z!|~r@=nuz#;Spmu@~Fho?fM`wS6kU_L}sRDqs$8ASYF~-H^^B!(;x0cznxby;ILgL
zlM_o|uYd3PTt@F|wQHj8nJ@E8)5>hF_-`W@hM9xul1#l)1viDZ5?AZl(oA*1LN~Wx
z1mx-V^=!je4JrFRMg{TVsS01s9tu<UYrhwS*cX^u7x)*au2z}xjhlJ#nmvT3hwRsL
ztoSjl6wDd=(X709Hb3h(%TF1%r`!3b*u4s^6fCd_OEq-JHINoAad__F?%!lh9+Sq^
zpJ<+|*xz^fe84s!G}mJ3udjVw9jfb<QU*@6KYKN|6j&FO9zK>FPR+1ST^jn01biwQ
zX6-x+T^D|OMWkld!s`(={lNT{Qe3wm-ss2)Ncl|-4`GhTDe8(u!Ubo23Xct;rmPUC
zwA-ZN`(oUEmEnD;K67|^(ssJaty(Ogv5+Lwmx;TKvAi*uK1NlhO*5suyO8R6qns%}
zRs-}b=-)-r@R4mF0G|#k)>yepF^%Fj8vJ;~{$B&xG47b&H}%*eJ$_zubfB~Yw|@VA
zYwccZ?_Mi-ul3WOu86CGYumg@UCPY+QqHmh&`1?u@4@@h__9fl1A<G-;X*(b+~@z8
zBwd8x4Dq)0X5<qw6fp0XO`>z`loZ}#u*<i$+h?ZcR#BG+KT%b(*&PuS?^Ck&yyuze
zRV{7qJYediuQEd#|I9f4wuz?d6OzZb?;z}c;U6&hPVBcx(ce%%C)m8n`jo))Lja57
zpN!2h-n~EnqqDOBi>llD_y8hGr+|bK0@B?`i1dIoQUgd!cS%SH2-2McLkrR(F(4oV
z(k(HBAQFO<B1nA4_rCAFqW1y5JM*0LoM&SFX05gNn!V52GygSb-f=ni*`9DNACy=f
zR@6De3StS`XJ(*7)<m?p7EF*ln-+?Ci!}%Z|51Qzix()2ogX#LzZXPHR}0b$um#0J
zK7-s)8$dxypD%v{d7}_ts<<HtqP&E!=tw4rN{N1-KIB;4Rn#x$_^vBub1L}LU9LDL
zE@*o-;nVmQ4d!AeRag#T2%*|lSDvpdgBm6kgvA`C8YYp16!G^V_ofn(RZX%8SJPIK
z2^r%(6vE<{t!^pb-g=QLR`mg8Kf}i*WFu|vAz~|vCIrkck9ix)z6v#5E%4-(rW=9n
z^M2bKx3w%1XSCRy|0uN5k(G;BcJz9ZI4B4na7u-dRL5$~b4-s`hl~vg>#9&#(%I(p
z@`Cj&S!{FHs{`GJ*i)g%nvv`!qlcC*riCx<O&uIuX6{#7c$s(i8R+em?yW+aq~7IB
zzIQm;)Nyt(npxiLd@^rN*JNfs1;;riaUx=0U&TNy6zC#?gBWHUBh6dExk3j%UISw`
zGps8_`gjZq1&kvK`4FPFR;KLjD{;YAG5cZR<|2N(+Iw1l<$BFU&S91GOtQA|1*rO^
zsv-$;1-Pp)Zb}GE9BbVyO_FFVeC7cT+ps`{g3uM-?(#g2?tlfIHw9H1TnjMfhy8KN
zYF{>~Qrasrf>4ruiq-t_Wr)>r0$$L?Bd?gF&40ctfq_3HdC8T`m`XF?YYB+}w-ot%
z<<S@WnV-naE)7R}ZFW%*zm`nW##^wC_lOk4zB`Tu`!v$B8+COnyX7FHT{?{FHn^K7
z-TGcD<#1v?_D179YQferPG#7dh$>RnP~id2M>>@mKdAxgNyQ(Q;#ApzE}^)?1i}i4
zV)I+oto^YW%IK(`wirxkosU@>ciDkOc-nO2Qvr;0j4NA#UUT>On)6Ftj?*csK13A_
zVESI)6t$NaIo|)V_l+V|jRHF#PkAKs$;rZeS0MERrG0Xihvudo0o9I_rqfE747$il
z*~W{>Z)i8|)+gnlSr~MJ+-2%kEyj=s{(TvFhT;{pRWuD5?_W$*@ZpajkD1D!gxin2
zKyQMMLe{q*DYuoRh8gUvx+O7+(3Wh}Q$=QruRilSe7cD`0SlaGdAL534!Rt_Rbgz7
zT?N+XbDRm*6IMq^rw~r!PBQdp^`#!{^VB<J5*93teeMijAK2LP3iO?2`|<I8nGl+z
zWxM$Y2J=_rAHT^@JaDFU=5$g(8X{K`DpuTf-#y*DJ^{<P4h=E%$5pGsrcB_WaS2UC
zuM4fiDO!MI8s*uk;)ZjpbNI+>bEug%aOUWR<QvMRZ`^dG;O1S`A;f4a=E8UDYF|wh
zD}!8JZ+G1tz-HFuX<=R7T606PUlY&Yb_b-tWLbTx9uFEk+?tx%+p+Iyw!31T-;iRZ
z`|a7*MACBo8*d+AU?o-LZ5#X!U#-CbITg0Hou}_r#`gG|np~D6>x0<taC^uS)+)K|
zv746uF`v5`;rb9H+y(cetNVOVokRXQq>P4f>iMe^k6N~d%`%T&nSdQ0Me8X(xW&}k
zm#@tpYlcz7ul&*usZOt#Sy6hk<pyXYX<WL%dKufZ*2@^g^|GU>f@D7RDJj&`&`O%n
z(9+VXjpoGT>K>h>^g&8Z)gF$Hpn5ARhFO&FT&7(?<7KrKl}Nf~qG;Ba-IT`jDU+kB
zp=f+;L}{gXrKZV)_3=zKaxB4J^&2}<eQzk?0AWLsr9wLRlxJmYe?5O2A9swA0D6T6
zi{`N-@jba?`CaiRdS$UlM>xF}?Mu1UIfX?(?PfYnlGV-G<Mk&{o2MECv5Rk@HtvWf
zqZF^bqWMWcka&_lJ*3J3x33Mo?n=@ew|+ngC+fHnaakK#L>o@s)mKKSc5PJ2K@ARg
z`PtVtCqLbRN8Omi_)g;+TCl~N`G)5^fp3t-7Hejkvj=tm0(zrA=@zYFtD*-v#{x;?
z6YVW#!?s7)u{T8SxYh##;=)bJFNfE5g^}45E`fFf;Cgw16$WIEp35`4LEC!ps}+7E
zRK(@!#sJ0c<t9n{fvBTC=PR>Og(jxoyMn)*WKy<;MP-SOJ*Zzp&$DQn_We!5mWvG|
zIts|76*dS&d$!uuzqGG7&&nt=vEQEJ9x;Ndz%3neXvsV$6BMC~`(XWGd{Q6P@a4S{
z{#y!oH*rh0HeR@|9~po7Y-dv<pnFBF9sMel8ebN-NR2rN|H@VN={Zac_5`oF8=$<f
z;!Sfuod*q188C1Sx~RRs{V<<x?Dw{}IKo@_=xG9_9<)h<sTHLN%<6Pf&ZxPf#RazE
zv6+6Tr-vdl`RVH;u*3R7V|%!yn{ACs)~)y7a@<MeV|5R+2v6nb{4tefBT~O+qG*jn
zVV2)hF2djz1Ea|i>-${Vc2~R1i@Vk#1=(s#q_<(FShKQVucLQzERE@Hy^5GUnVhJ@
za4u5Xg^u_1NKG=lek5nu1g|-mF~Qgm-KxlY-?zVp8;jG({9d`=ExKfCOwDDAW|I-Y
zr5Td&{lm`u3-Z}yj(Bo--IxhhKY4)mR;1G0k908EVyCX#{ZhGvq2QLuEmmy-#JfBX
z+j@n>_nCAH5!N2kdZ0BqUPJ{pg6pHFwY<LRx5U?K!7nTH!#K3M4{i7()+nh=6Qt=f
zJzXhT$KA?@Mk=7u`RmD@W=eW*uv{J2LaM)(G4cDtcPD053)R**d|*^maGb-%0}{fH
z6uFOO(U3_iULn~f%8_e#C3+DPdhv?vV*!udP&))xoCJsh1^4h;>Qww%Mp*qbrC%jI
zYh<UACFK*ztb0<^;=^m@yT`l4En)@tV|KP<{*pN(+VnxVlQsmIIB7({Fk=&7mW0^J
zEMVc-|FYYS$&s4Yc)wB2wp9L$0K2_S=Hl1;b;MiNqiOhr@2GVi&)|zt8)F<@(PVn#
z8lw=(E^&m?yRKRzZi3)d(KK^sEMYx*p+ycU>Fo=K-Wm+8GN$a;HN$ZCox+J>il2;D
zCDHEepFO0`c8Sq8-oNyU|06$kUDNkla>5DLr8<<(-RoITre-d^njT%qJ4u@g56j?(
zhg_z}B==MtdNiHlc)6SH@+_DhEsPg&htB|B>qDKx@^(H>C*MMS;BLouuaQQnrN(8(
zyj#)Ca6<WnBFzELN~bCsxO{NQ9zy}KUZz&cunJ#)s#o*z{+wVBGl`mtxs?{yeoCGS
z&CBu?g61SSF|ljG26<gjc)qoaRPLxifdIoI$05jR^b3AoYP5U7V!0JfR@1)iU7JYe
z!Xz7ADmmZHo7TEHh}wFXS8ZmE_(auG)@Y_C8%efBuuCq1mll3U6~lAKL6IwA$yfbV
z?_lna!!OUkkV#nbGaW=jXp1SL1C&-xIii3_=xw3++-UJVl-~P(Iy*lyf-)m#y>AX}
z2uR}!Q3}Y(g{#KZ%vHQp=V(yA>V6oMk8jOdAGa$uusaHEO}z>hvKvFA5m!XcAHf$q
zuY$v_VUyMmA7JvLweN(d4s2F1urp&fb&nFIz9V|_LIMgWq->K}k4@Sw<)`B34Jw^R
zdEF#d_&nCtAo7*s_+c5%HP^A0Je_Sz3m>?6<GPfa{PesYhGhBeo7dT{Yu4*sCs}0!
z&Kf3qp*}sClaRuT*(H%xp{|}C5FocDtu7sC&2REw!hntB0=Kw{+6ITMq?6wi)J(8x
zY9oPCCkK=|=Cezkvz@U6n~RyV3xtc^)Y#?BlBd;czaUMlGSn;E{M}N!1znZqrB1Z<
zggm8)et0c7H6F>Xxd-}S1pe&!x&1~M!Y$5H#U((inE4W+l($4W{!?~?pEWOxNT9AY
z0E-ag80G%NXmP~4qXwh*d*I(P+47WU4rkKF_m5`0t9DC{ZE#bv12wwk=?S~EA(;xZ
zQ~UuaEV$CoJ2$NIvAuIJY+(-=684Qqg<9OCWqIcE7Fi77ar-_~Cz4d_X=Q`{wU)PY
za#rwFAsfPpXf6a+XmV^0U#2m85POCVwBbF5vg>r<*6w3F%?hsUTj)(CcdJ$k<Gps_
zFsY(Q{oGd6G&Ao8z1k(g0b)S+_4$(Jk}v*K17yp!tzs5QBBzMX0V=G~LM;0)<0UV~
zIlPf$lU6>yB=(~P;Qm6@q6V^I%>(X-?VQ2*xyfYCD_9X$Q}^C)n<UeA$i2vNinR6#
z;Sy`vsU&Q3pnpDHqDtg@r*XW-Kp^Dy+ii{yhh{2es$HMlIA^GNpV^5D+?gaDD@=*v
zx_o`Wt(r_G^jKDWK;Wx_<U<#o2+4N~h|34xrR^3%$Fx+)TudCwb+_E_=|-j=!wgkN
zGY65hp;TzX5*jFKH(+>)<=IQQg=yth62nDl;U6EZg}mPH$7Ax#riY(=+{^gd@IFy4
zF6x9nPW?94$1)VF`Gcj_v|+Y6l*zz+n+fM}+nXy6#uJqyi8z$3Z^<hum$?x`sJ)>+
z^IZeh(pYhx&zG>C;V_&8QJa#m!2+kNuH^)<6{Td}Cscg<Ea268Dm<HRQHgjeDpoS_
zks@d5GP_zqxd4jUgAnrvscE}nj=^LZ4{CKjbYtr+Vxb~bOj~!l2?wJEl;E7AtcXge
z{%AIPZm}6f3lv8&L|vOXO`yG3fH@)I28Z7+sZen{@aJtsG<=boY9i2^%xt7wO?DAt
zR<1a~8aHDeJ=Do&3#*Zgn=|x_eqBSni~6KZ&s6670rT6MVFGQ^;|?FHeMq?#s-jgq
zGEkp>g=qV`9V49zK2`&s4Dw6*TLTjrBwqhw-*fq$L8SW5a~vg@9=IY*Ov+Yyk=)$P
z(%k^;+fm{wl=6~{;?T($Y{?o9ED#$PP*y+qv~4C;MEMn`Kn6T>IA$-=!p7wjp%JP`
zxTCC4`OXd}Xv3d;+cRISkw-VY2vI}4@8n6VXpZCArsVPb6WiN`De*7C@l3w<=B+8w
z=@WRVRnHndSZog(zoN}#9uwcyy#>{0!uIT@$_#iFNHF81tzv#`KeM!TXw~{C`_;yE
zdcAmE(|)i8zMb%QJ#<2I+PB<MJ?xOo42ou4)2{aZ*rLo^>YjE!+k}Op*U)#vlH#te
zP}~#LwHTq-8xnj+HajZ#a@Y4@Ju2?L=IVP<#QsR(vsvTd+lE+OQc6L(u#x3!DD}lT
zt6WOmS^GT>OASi7kwH6OliCK4x%cb7kwTLZN4IQH7bAZtIz)WB^}w%7442$0Dkg$N
zE0<fKxlH?dAmMU50)C7So};i$ZMDH2QI=dq%yrzg%ON|?@k5d6EsMXQusr*tV8`3n
z`=IUjg12Kn5FZPS)P(@8Tc)!Lo~@aS@deAB*?lqR5l(zQbRKMUHZfk<Tt|}r&3yld
zxa`q%_f3a#CaU;_xGGKQg%El4&v#U}xmud#-}udOY!YpZ;Z%MkH#edv{U+A5SFPgP
zHT5d!!}odL5L`jQ(46JT+?urU74DvKZ;B=qO<Btq1FIHRRDzT%l%py4J_QL^?-u2K
zQwy9E4fCY(8NDIuk=%atW=V^XaTfJ1$pkUZFh-9N`#nrIGQT6zN*>N>+#jSUOqg%>
zUO^hYJt<W>y!GsBWp{k4J?ExphB*QT&9II;5o=pAqH?hZtbwBTsO&`=lctI6P784q
zi~~b>YY`X|`gP=wj5~sZi=>qq?P3;oB;G?YHAUd+<FhAzBLUS@TiyI4(pCtG0DVfj
zS?IVsOHqLLck))6)cV^8ovsQ*!YS5lR_@c3SBl|_^_#Ih)(ed`b)%ouycakr_#QvL
zj(sBRjj}eDGat#M@vyRZ$%wm|W;OLrLhqN)POjFj#%i$b#{uS&pUCT6IwTaV7c_68
z>onh<g&=<%HV{QpHye~4|Mn$kr`|ow3gkQl*ognh`Trxo22o<lZ4&sw;DDey5Dt!p
zzo{ZFF#*nrTqr7f7qpsTM!wSIC>O~mAEXu|7$YbBMojQ7ZgeJ#!=+@~^2lM%x0buT
z%S{4|StgI@q;ar$brNgb8k=7$4?lU2p7Z%?-Ur8O9~|joUIlTm(9Wv6;YjP+%n_RV
zT1MJZ={lcxWSCT`=ZNaAs(}c%GS&xwd`FP5wo`c)2f1)UC7y0~cdGL;WlB*8CqcIA
zpby`{$8ebnQzp_lk~`xb4w7`#T%U*a;zLSB^`^gRlG3XelySx3$HL@xz;t4A19?5#
zj?8E<Y&y%UrVSm*S=g5_67-&TLuD5DNj=@Xy7c5MH2Ha0nhsZUOJu1pBj4!LbY}mE
zN}X`_6s=a2=k^<7PpQ1-{RN_>*`2XI|C_ROsgiSQKA;=i3T*$5{QqMh@l#K_Pc=PB
z=YDz&q!kd6w#@5#dr(5fuw5d=!6h(SLtj%zKT}@CT2m;<BhGP$=U%Xocx@>g563MY
zp_=bcNuy(Qp^sn|FD$mJ^6Ka-=ttQEM<yot1nY=8S(w8(x_ZExW+TkJ97b!FD!q!5
zghr4-gIGvxG7%&_Mkij)$5P7CihE$-L$zAneQ#Y8b7d=nf&8pyzZ(O|hfh#2Hw+EP
zK|pImL8V5+1bhU6hyp*!{dMs74jn}H_Ypp5UGnGQU)sr<8t5RjTjsvnt{zw*luKWM
zmU)R7c=C5sk940s{b}ImhWWQC;I$xFfH<)KM~2zV<D&L}H~ssDbLyj|0gwM<?=M;B
z|HUNvDU%<5H~Fhl^#5X>;*@zlAP}cM@vr7NInD<B4EB}3vbBtp`cw%R0+ZwXJa_X?
zwtj{<{A>UO0?hl*O5=YRfyUo?#?e6_3&r0V?3d4U0s`T>C<C6&l>D>l=>XelKMa7u
z{TYFRK)5f;sI>s|9s>5O|2oh0YpjmWWAI#*@!9E!d>sww3V_AaeyHM};|K3W8SV_M
zab|!Y7JtMlk?<S_-$fakVI~LP@PK|z^t&JGz$0hoga4w8%bxyWZvh|vrOW<nKA?2x
zFa$2jsD|oa5dat>f5Zw-e-1<NqKs0Ddv1V^HwgR(1AOZohR{VBcd_CG{v*=<Ypj;f
zV;EnQVZ0=U(*gMKXFiM=&+)_Lq73&!q926-BM7KYr~8L5(>aWX7iHKTQDK`ofk5e5
ze=u&J!!W%l!}6eu=D+7dB>fzQ*+m&u5jR)qfC~GR;CDZ&GR|R`UzA~iXV7j5<bng3
zQJl^PJM0{W#YGuc$9gi~0gQ}47#n%#Ff1?1=uYo`gZQs)7awSDXVxpLi!##e6)FFF
z&N+(CVOU?3ArmLXhXNFnRQuoaQBZsi<IzPKr+**)Yb^n7oWlTqEq$>u!|7{EzcM%p
zP|nT=aE0k&jMG;-eq|gJp2_(4b&rd2PM>D{m9s*6Cg+d2!o_%}XZgSKCMnP0{V^xJ
y813}@@>g0P_20Dr(eml})3280fNu0xpS*B{(1AJ#0?`0>7%;LDr2)zb=>GtDjWhWG

literal 0
HcmV?d00001


From f295a218a05f75ab5cd219afbd4c402f9984f477 Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Thu, 7 Jul 2016 09:12:09 +0200
Subject: [PATCH 1268/1311] Add notes about sparsity.

---
 docs/reference/how-to.asciidoc         |   2 +
 docs/reference/how-to/general.asciidoc | 104 +++++++++++++++++++++++++
 2 files changed, 106 insertions(+)
 create mode 100644 docs/reference/how-to/general.asciidoc

diff --git a/docs/reference/how-to.asciidoc b/docs/reference/how-to.asciidoc
index ee954553617..f41c3a3bb9c 100644
--- a/docs/reference/how-to.asciidoc
+++ b/docs/reference/how-to.asciidoc
@@ -15,6 +15,8 @@ This section provides guidance about which changes should and shouldn't be
 made.
 --
 
+include::how-to/general.asciidoc[]
+
 include::how-to/indexing-speed.asciidoc[]
 
 include::how-to/search-speed.asciidoc[]
diff --git a/docs/reference/how-to/general.asciidoc b/docs/reference/how-to/general.asciidoc
new file mode 100644
index 00000000000..de37a714de6
--- /dev/null
+++ b/docs/reference/how-to/general.asciidoc
@@ -0,0 +1,104 @@
+[[general-recommendations]]
+== General recommendations
+
+[float]
+[[large-size]]
+=== Don't return large result sets
+
+Elasticsearch is designed as a search engine, which makes it very good at
+getting back the top documents that match a query. However, it is not as good
+for workloads that fall into the database domain, such as retrieving all
+documents that match a particular query. If you need to do this, make sure to
+use the <<search-request-scroll,Scroll>> API.
+
+[float]
+[[sparsity]]
+=== Avoid sparsity
+
+The data-structures behind Lucene, which elasticsearch relies on in order to
+index and store data, work best with dense data, ie. when all documents have the
+same fields. This is especially true for fields that have norms enabled (which
+is the case for `text` fields by default) or doc values enabled (which is the
+case for numerics, `date`, `ip` and `keyword` by default).
+
+The reason is that Lucene internally identifies documents with so-called doc
+ids, which are integers between 0 and the total number of documents in the
+index. These doc ids are used for communication between the internal APIs of
+Lucene: for instance searching on a term with a `match` query produces an
+iterator of doc ids, and these doc ids are then used to retrieve the value of
+the `norm` in order to compute a score for these documents. The way this `norm`
+lookup is implemented currently is by reserving one byte for each document.
+The `norm` value for a given doc id can then be retrieved by reading the
+byte at index `doc_id`. While this is very efficient and helps Lucene quickly
+have access to the `norm` values of every document, this has the drawback that
+documents that do not have a value will also require one byte of storage.
+
+In practice, this means that if an index has `M` documents, norms will require
+`M` bytes of storage *per field*, even for fields that only appear in a small
+fraction of the documents of the index. Although slightly more complex with doc
+values due to the fact that doc values have multiple ways that they can be
+encoded depending on the type of field and on the actual data that the field
+stores, the problem is very similar. In case you wonder: `fielddata`, which was
+used in elasticsearch pre-2.0 before being replaced with doc values, also
+suffered from this issue, except that the impact was only on the memory
+footprint since `fielddata` was not explicitly materialized on disk.
+
+Note that even though the most notable impact of sparsity is on storage
+requirements, it also has an impact on indexing speed and search speed since
+these bytes for documents that do not have a field still need to be written
+at index time and skipped over at search time.
+
+It is totally fine to have a minority of sparse fields in an index. But beware
+that if sparsity becomes the rule rather than the exception, then the index
+will not be as efficient as it could be.
+
+This section mostly focused on `norms` and `doc values` because those are the
+two features that are most affected by sparsity. Sparsity also affect the
+efficiency of the inverted index (used to index `text`/`keyword` fields) and
+dimensional points (used to index `geo_point` and numerics) but to a lesser
+extent.
+
+Here are some recommendations that can help avoid sparsity:
+
+[float]
+==== Avoid putting unrelated data in the same index
+
+You should avoid putting documents that have totally different structures into
+the same index in order to avoid sparsity. It is often better to put these
+documents into different indices, you could also consider giving fewer shards
+to these smaller indices since they will contain fewer documents overall.
+
+Note that this advice does not apply in the case that you need to use
+parent/child relations between your documents since this feature is only
+supported on documents that live in the same index.
+
+[float]
+==== Normalize document structures
+
+Even if you really need to put different kinds of documents in the same index,
+maybe there are opportunities to reduce sparsity. For instance if all documents
+in the index have a timestamp field but some call it `timestamp` and others
+call it `creation_date`, it would help to rename it so that all documents have
+the same field name for the same data.
+
+[float]
+==== Avoid types
+
+Types might sound like a good way to store multiple tenants in a single index.
+They are not: given that types store everything in a single index, having
+multiple types that have different fields in a single index will also cause
+problems due to sparsity as described above. If your types to not have very
+similar mappings, you might want to consider moving them to a dedicated index.
+
+[float]
+==== Disable `norms` and `doc_values` on sparse fields
+
+If none of the above recommendations apply in your case, you might want to
+check whether you actually need `norms` and `doc_values` on your sparse fields.
+`norms` can be disabled if producing scores is not necessary on a field, this is
+typically true for fields that are only used for filtering. `doc_values` can be
+disabled on fields that are neither used for sorting nor for aggregations.
+Beware that this decision should not be made lightly since these parameters
+cannot be changed on a live index, so you would have to reindex if you realize
+that you need `norms` or `doc_values`.
+

From 873661df1796df4fdafab085acba4bed13626c0f Mon Sep 17 00:00:00 2001
From: Adrien Grand <jpountz@gmail.com>
Date: Thu, 7 Jul 2016 17:49:01 +0200
Subject: [PATCH 1269/1311] Fix typo.

---
 docs/reference/how-to/general.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/how-to/general.asciidoc b/docs/reference/how-to/general.asciidoc
index de37a714de6..60f86d821d9 100644
--- a/docs/reference/how-to/general.asciidoc
+++ b/docs/reference/how-to/general.asciidoc
@@ -87,7 +87,7 @@ the same field name for the same data.
 Types might sound like a good way to store multiple tenants in a single index.
 They are not: given that types store everything in a single index, having
 multiple types that have different fields in a single index will also cause
-problems due to sparsity as described above. If your types to not have very
+problems due to sparsity as described above. If your types do not have very
 similar mappings, you might want to consider moving them to a dedicated index.
 
 [float]

From 3267fc4e0ce212516fe452e57d88091ae053590d Mon Sep 17 00:00:00 2001
From: Tanguy Leroux <tlrx.dev@gmail.com>
Date: Thu, 7 Jul 2016 15:48:50 +0200
Subject: [PATCH 1270/1311] Clean up more messy tests

After #13834 many tests that used Groovy scripts (for good or bad reason) in their tests have been moved in the lang-groovy module and the issue #13837 has been created to track these messy tests in order to clean them up.

This commit moves more tests back in core, removes the dependency on Groovy, changes the scripts in order to use the mocked script engine, and change the tests  to integration tests.
---
 .../resources/checkstyle_suppressions.xml     |   5 -
 .../action/IndicesRequestIT.java              |  59 ++++++---
 .../AggregationTestScriptsPlugin.java         |  18 ++-
 .../search/aggregations/EquivalenceIT.java    |  94 +++++++++++---
 .../aggregations/bucket/HistogramIT.java      | 103 ++++++++++++----
 .../aggregations/metrics/CardinalityIT.java   |  91 +++++++++++---
 .../metrics/HDRPercentileRanksIT.java         |  13 +-
 .../metrics/HDRPercentilesIT.java             |  13 +-
 .../metrics/TDigestPercentileRanksIT.java     | 112 ++++++++++-------
 .../metrics/TDigestPercentilesIT.java         | 115 ++++++++++--------
 .../messy/tests/package-info.java             |   6 -
 11 files changed, 423 insertions(+), 206 deletions(-)
 rename modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java => core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java (94%)
 rename modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java => core/src/test/java/org/elasticsearch/search/aggregations/EquivalenceIT.java (83%)
 rename modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java => core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java (90%)
 rename modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/CardinalityTests.java => core/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java (79%)
 rename modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentileRanksTests.java => core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java (83%)
 rename modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentilesTests.java => core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java (81%)

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 574bb1e3b30..f85f998df5b 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -1127,11 +1127,8 @@
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]groovy[/\\]GroovyScriptEngineService.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]BulkTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]DoubleTermsTests.java" checks="LineLength" />
-  <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]EquivalenceTests.java" checks="LineLength" />
-  <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]HistogramTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]IPv4RangeTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]IndexLookupTests.java" checks="LineLength" />
-  <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]IndicesRequestTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]LongTermsTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]MinDocCountTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]MinTests.java" checks="LineLength" />
@@ -1140,8 +1137,6 @@
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]SearchFieldsTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]SimpleSortTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]StringTermsTests.java" checks="LineLength" />
-  <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]TDigestPercentileRanksTests.java" checks="LineLength" />
-  <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]TDigestPercentilesTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]messy[/\\]tests[/\\]package-info.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]groovy[/\\]GroovyScriptTests.java" checks="LineLength" />
   <suppress files="modules[/\\]lang-groovy[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]script[/\\]groovy[/\\]GroovySecurityTests.java" checks="LineLength" />
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java b/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java
similarity index 94%
rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java
rename to core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java
index c0751fe99e2..3f0a901ae32 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java
+++ b/core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java
@@ -17,9 +17,8 @@
  * under the License.
  */
 
-package org.elasticsearch.messy.tests;
+package org.elasticsearch.action;
 
-import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction;
 import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
@@ -77,15 +76,15 @@ import org.elasticsearch.action.termvectors.TermVectorsRequest;
 import org.elasticsearch.action.update.UpdateAction;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateResponse;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.groovy.GroovyPlugin;
+import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.action.SearchTransportService;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -109,6 +108,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Function;
 import java.util.function.Supplier;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -120,7 +120,8 @@ import static org.hamcrest.Matchers.hasItem;
 import static org.hamcrest.Matchers.instanceOf;
 
 @ClusterScope(scope = Scope.SUITE, numClientNodes = 1, minNumDataNodes = 2)
-public class IndicesRequestTests extends ESIntegTestCase {
+public class IndicesRequestIT extends ESIntegTestCase {
+
     private final List<String> indices = new ArrayList<>();
 
     @Override
@@ -148,7 +149,16 @@ public class IndicesRequestTests extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return pluginList(InterceptingTransportService.TestPlugin.class, GroovyPlugin.class);
+        return pluginList(InterceptingTransportService.TestPlugin.class, CustomScriptPlugin.class);
+    }
+
+    public static class CustomScriptPlugin extends MockScriptPlugin {
+
+        @Override
+        @SuppressWarnings("unchecked")
+        protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
+            return Collections.singletonMap("ctx.op='delete'", vars -> ((Map<String, Object>) vars.get("ctx")).put("op", "delete"));
+        }
     }
 
     @Before
@@ -251,7 +261,8 @@ public class IndicesRequestTests extends ESIntegTestCase {
 
         String indexOrAlias = randomIndexOrAlias();
         client().prepareIndex(indexOrAlias, "type", "id").setSource("field", "value").get();
-        UpdateRequest updateRequest = new UpdateRequest(indexOrAlias, "type", "id").script(new Script("ctx.op='delete'"));
+        UpdateRequest updateRequest = new UpdateRequest(indexOrAlias, "type", "id")
+                .script(new Script("ctx.op='delete'", ScriptService.ScriptType.INLINE, CustomScriptPlugin.NAME, Collections.emptyMap()));
         UpdateResponse updateResponse = internalCluster().coordOnlyNodeClient().update(updateRequest).actionGet();
         assertThat(updateResponse.isCreated(), equalTo(false));
 
@@ -360,14 +371,16 @@ public class IndicesRequestTests extends ESIntegTestCase {
     }
 
     public void testFlush() {
-        String[] indexShardActions = new String[]{TransportShardFlushAction.NAME, TransportShardFlushAction.NAME + "[r]", TransportShardFlushAction.NAME + "[p]"};
+        String[] indexShardActions = new String[]{TransportShardFlushAction.NAME, TransportShardFlushAction.NAME + "[r]",
+                TransportShardFlushAction.NAME + "[p]"};
         interceptTransportActions(indexShardActions);
 
         FlushRequest flushRequest = new FlushRequest(randomIndicesOrAliases());
         internalCluster().coordOnlyNodeClient().admin().indices().flush(flushRequest).actionGet();
 
         clearInterceptedActions();
-        String[] indices = new IndexNameExpressionResolver(Settings.EMPTY).concreteIndexNames(client().admin().cluster().prepareState().get().getState(), flushRequest);
+        String[] indices = new IndexNameExpressionResolver(Settings.EMPTY)
+                .concreteIndexNames(client().admin().cluster().prepareState().get().getState(), flushRequest);
         assertIndicesSubset(Arrays.asList(indices), indexShardActions);
     }
 
@@ -383,14 +396,16 @@ public class IndicesRequestTests extends ESIntegTestCase {
     }
 
     public void testRefresh() {
-        String[] indexShardActions = new String[]{TransportShardRefreshAction.NAME, TransportShardRefreshAction.NAME + "[r]", TransportShardRefreshAction.NAME + "[p]"};
+        String[] indexShardActions = new String[]{TransportShardRefreshAction.NAME, TransportShardRefreshAction.NAME + "[r]",
+                TransportShardRefreshAction.NAME + "[p]"};
         interceptTransportActions(indexShardActions);
 
         RefreshRequest refreshRequest = new RefreshRequest(randomIndicesOrAliases());
         internalCluster().coordOnlyNodeClient().admin().indices().refresh(refreshRequest).actionGet();
 
         clearInterceptedActions();
-        String[] indices = new IndexNameExpressionResolver(Settings.EMPTY).concreteIndexNames(client().admin().cluster().prepareState().get().getState(), refreshRequest);
+        String[] indices = new IndexNameExpressionResolver(Settings.EMPTY)
+                .concreteIndexNames(client().admin().cluster().prepareState().get().getState(), refreshRequest);
         assertIndicesSubset(Arrays.asList(indices), indexShardActions);
     }
 
@@ -493,7 +508,9 @@ public class IndicesRequestTests extends ESIntegTestCase {
     public void testPutMapping() {
         interceptTransportActions(PutMappingAction.NAME);
 
-        PutMappingRequest putMappingRequest = new PutMappingRequest(randomUniqueIndicesOrAliases()).type("type").source("field", "type=text");
+        PutMappingRequest putMappingRequest = new PutMappingRequest(randomUniqueIndicesOrAliases())
+                .type("type")
+                .source("field", "type=text");
         internalCluster().coordOnlyNodeClient().admin().indices().putMapping(putMappingRequest).actionGet();
 
         clearInterceptedActions();
@@ -513,7 +530,8 @@ public class IndicesRequestTests extends ESIntegTestCase {
     public void testUpdateSettings() {
         interceptTransportActions(UpdateSettingsAction.NAME);
 
-        UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(randomIndicesOrAliases()).settings(Settings.builder().put("refresh_interval", -1));
+        UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(randomIndicesOrAliases())
+                .settings(Settings.builder().put("refresh_interval", -1));
         internalCluster().coordOnlyNodeClient().admin().indices().updateSettings(updateSettingsRequest).actionGet();
 
         clearInterceptedActions();
@@ -621,8 +639,9 @@ public class IndicesRequestTests extends ESIntegTestCase {
             }
             for (TransportRequest internalRequest : requests) {
                 assertThat(internalRequest, instanceOf(IndicesRequest.class));
-                assertThat(internalRequest.getClass().getName(), ((IndicesRequest)internalRequest).indices(), equalTo(originalRequest.indices()));
-                assertThat(((IndicesRequest)internalRequest).indicesOptions(), equalTo(originalRequest.indicesOptions()));
+                IndicesRequest indicesRequest = (IndicesRequest) internalRequest;
+                assertThat(internalRequest.getClass().getName(), indicesRequest.indices(), equalTo(originalRequest.indices()));
+                assertThat(indicesRequest.indicesOptions(), equalTo(originalRequest.indicesOptions()));
             }
         }
     }
@@ -736,12 +755,16 @@ public class IndicesRequestTests extends ESIntegTestCase {
         }
 
         @Override
-        public <Request extends TransportRequest> void registerRequestHandler(String action, Supplier<Request> request, String executor, boolean forceExecution, boolean canTripCircuitBreaker, TransportRequestHandler<Request> handler) {
-            super.registerRequestHandler(action, request, executor, forceExecution, canTripCircuitBreaker, new InterceptingRequestHandler<>(action, handler));
+        public <Request extends TransportRequest> void registerRequestHandler(String action, Supplier<Request> request, String executor,
+                                                                              boolean forceExecution, boolean canTripCircuitBreaker,
+                                                                              TransportRequestHandler<Request> handler) {
+            super.registerRequestHandler(action, request, executor, forceExecution, canTripCircuitBreaker, new
+                    InterceptingRequestHandler<>(action, handler));
         }
 
         @Override
-        public <Request extends TransportRequest> void registerRequestHandler(String action, Supplier<Request> requestFactory, String executor, TransportRequestHandler<Request> handler) {
+        public <Request extends TransportRequest> void registerRequestHandler(String action, Supplier<Request> requestFactory, String
+                executor, TransportRequestHandler<Request> handler) {
             super.registerRequestHandler(action, requestFactory, executor, new InterceptingRequestHandler<>(action, handler));
         }
 
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationTestScriptsPlugin.java b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationTestScriptsPlugin.java
index 218c3eae0c5..4a0369a4019 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/AggregationTestScriptsPlugin.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/AggregationTestScriptsPlugin.java
@@ -21,16 +21,30 @@ package org.elasticsearch.search.aggregations;
 
 import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.script.MockScriptPlugin;
+import org.elasticsearch.script.Script;
 
 import java.util.HashMap;
 import java.util.Map;
 import java.util.function.Function;
 
+import static java.util.Collections.singletonMap;
+import static org.elasticsearch.script.ScriptService.ScriptType;
+
 /**
  * This class contains various mocked scripts that are used in aggregations integration tests.
  */
 public class AggregationTestScriptsPlugin extends MockScriptPlugin {
 
+    // Equivalent to:
+    //
+    // List values = doc['values'].values;
+    // double[] res = new double[values.size()];
+    // for (int i = 0; i < res.length; i++) {
+    //      res[i] = values.get(i) - dec;
+    // };
+    // return res;
+    public static final Script DECREMENT_ALL_VALUES = new Script("decrement all values", ScriptType.INLINE, NAME, singletonMap("dec", 1));
+
     @Override
     protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
         Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>();
@@ -68,7 +82,7 @@ public class AggregationTestScriptsPlugin extends MockScriptPlugin {
             return doc.get("values");
         });
 
-        scripts.put("decrement all values", vars -> {
+        scripts.put(DECREMENT_ALL_VALUES.getScript(), vars -> {
             int dec = (int) vars.get("dec");
             Map<?, ?> doc = (Map) vars.get("doc");
             ScriptDocValues.Longs values = (ScriptDocValues.Longs) doc.get("values");
@@ -80,6 +94,8 @@ public class AggregationTestScriptsPlugin extends MockScriptPlugin {
             return res;
         });
 
+        scripts.put("_value * -1", vars -> (double) vars.get("_value") * -1);
+
         return scripts;
     }
 }
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/EquivalenceIT.java
similarity index 83%
rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java
rename to core/src/test/java/org/elasticsearch/search/aggregations/EquivalenceIT.java
index 1bca59dc426..72a2b584b12 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/EquivalenceIT.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.elasticsearch.messy.tests;
+package org.elasticsearch.search.aggregations;
 
 import com.carrotsearch.hppc.IntHashSet;
 import org.elasticsearch.action.index.IndexRequestBuilder;
@@ -28,9 +28,9 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService.ScriptType;
-import org.elasticsearch.script.groovy.GroovyPlugin;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
@@ -49,6 +49,7 @@ import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.extendedStats;
@@ -71,11 +72,22 @@ import static org.hamcrest.core.IsNull.notNullValue;
  * Additional tests that aim at testing more complex aggregation trees on larger random datasets, so that things like
  * the growth of dynamic arrays is tested.
  */
-public class EquivalenceTests extends ESIntegTestCase {
+public class EquivalenceIT extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(GroovyPlugin.class);
+        return Collections.singleton(CustomScriptPlugin.class);
+    }
+
+    public static class CustomScriptPlugin extends MockScriptPlugin {
+        @Override
+        protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
+            return Collections.singletonMap("floor(_value / interval)", vars -> {
+                Double value = (Double) vars.get("_value");
+                Integer interval = (Integer) vars.get("interval");
+                return Math.floor(value / interval.doubleValue());
+            });
+        }
     }
 
     // Make sure that unordered, reversed, disjoint and/or overlapping ranges are supported
@@ -102,7 +114,9 @@ public class EquivalenceTests extends ESIntegTestCase {
             source = source.endArray().endObject();
             client().prepareIndex("idx", "type").setSource(source).execute().actionGet();
         }
-        assertNoFailures(client().admin().indices().prepareRefresh("idx").setIndicesOptions(IndicesOptions.lenientExpandOpen()).execute().get());
+        assertNoFailures(client().admin().indices().prepareRefresh("idx").
+                setIndicesOptions(IndicesOptions.lenientExpandOpen())
+                .get());
 
         final int numRanges = randomIntBetween(1, 20);
         final double[][] ranges = new double[numRanges][];
@@ -234,20 +248,48 @@ public class EquivalenceTests extends ESIntegTestCase {
         }
         indexRandom(true, indexingRequests);
 
-        assertNoFailures(client().admin().indices().prepareRefresh("idx").setIndicesOptions(IndicesOptions.lenientExpandOpen()).execute().get());
+        assertNoFailures(client().admin().indices().prepareRefresh("idx")
+                .setIndicesOptions(IndicesOptions.lenientExpandOpen())
+                .execute().get());
 
         TermsAggregatorFactory.ExecutionMode[] globalOrdinalModes = new TermsAggregatorFactory.ExecutionMode[] {
                 TermsAggregatorFactory.ExecutionMode.GLOBAL_ORDINALS_HASH, TermsAggregatorFactory.ExecutionMode.GLOBAL_ORDINALS
         };
 
         SearchResponse resp = client().prepareSearch("idx")
-                .addAggregation(terms("long").field("long_values").size(maxNumTerms).collectMode(randomFrom(SubAggCollectionMode.values())).subAggregation(min("min").field("num")))
-                .addAggregation(terms("double").field("double_values").size(maxNumTerms).collectMode(randomFrom(SubAggCollectionMode.values())).subAggregation(max("max").field("num")))
-                .addAggregation(terms("string_map").field("string_values").collectMode(randomFrom(SubAggCollectionMode.values()))
-                        .executionHint(TermsAggregatorFactory.ExecutionMode.MAP.toString()).size(maxNumTerms)
-                        .subAggregation(stats("stats").field("num")))
-                .addAggregation(terms("string_global_ordinals").field("string_values").collectMode(randomFrom(SubAggCollectionMode.values())).executionHint(globalOrdinalModes[randomInt(globalOrdinalModes.length - 1)].toString()).size(maxNumTerms).subAggregation(extendedStats("stats").field("num")))
-                .addAggregation(terms("string_global_ordinals_doc_values").field("string_values.doc_values").collectMode(randomFrom(SubAggCollectionMode.values())).executionHint(globalOrdinalModes[randomInt(globalOrdinalModes.length - 1)].toString()).size(maxNumTerms).subAggregation(extendedStats("stats").field("num")))
+                    .addAggregation(
+                            terms("long")
+                                    .field("long_values")
+                                    .size(maxNumTerms)
+                                    .collectMode(randomFrom(SubAggCollectionMode.values()))
+                                    .subAggregation(min("min").field("num")))
+                    .addAggregation(
+                            terms("double")
+                                    .field("double_values")
+                                    .size(maxNumTerms)
+                                    .collectMode(randomFrom(SubAggCollectionMode.values()))
+                                    .subAggregation(max("max").field("num")))
+                    .addAggregation(
+                            terms("string_map")
+                                    .field("string_values")
+                                    .collectMode(randomFrom(SubAggCollectionMode.values()))
+                                    .executionHint(TermsAggregatorFactory.ExecutionMode.MAP.toString())
+                                    .size(maxNumTerms)
+                                    .subAggregation(stats("stats").field("num")))
+                    .addAggregation(
+                            terms("string_global_ordinals")
+                                    .field("string_values")
+                                    .collectMode(randomFrom(SubAggCollectionMode.values()))
+                                    .executionHint(globalOrdinalModes[randomInt(globalOrdinalModes.length - 1)].toString())
+                                    .size(maxNumTerms)
+                                    .subAggregation(extendedStats("stats").field("num")))
+                    .addAggregation(
+                            terms("string_global_ordinals_doc_values")
+                                    .field("string_values.doc_values")
+                                    .collectMode(randomFrom(SubAggCollectionMode.values()))
+                                    .executionHint(globalOrdinalModes[randomInt(globalOrdinalModes.length - 1)].toString())
+                                    .size(maxNumTerms)
+                                    .subAggregation(extendedStats("stats").field("num")))
                 .execute().actionGet();
         assertAllSuccessful(resp);
         assertEquals(numDocs, resp.getHits().getTotalHits());
@@ -304,15 +346,25 @@ public class EquivalenceTests extends ESIntegTestCase {
             source = source.endArray().endObject();
             client().prepareIndex("idx", "type").setSource(source).execute().actionGet();
         }
-        assertNoFailures(client().admin().indices().prepareRefresh("idx").setIndicesOptions(IndicesOptions.lenientExpandOpen()).execute().get());
+        assertNoFailures(client().admin().indices().prepareRefresh("idx")
+                .setIndicesOptions(IndicesOptions.lenientExpandOpen())
+                .execute().get());
 
         Map<String, Object> params = new HashMap<>();
         params.put("interval", interval);
+
         SearchResponse resp = client().prepareSearch("idx")
                 .addAggregation(
-                        terms("terms").field("values").collectMode(randomFrom(SubAggCollectionMode.values()))
-                                .script(new Script("floor(_value / interval)", ScriptType.INLINE, null, params)).size(maxNumTerms))
-                .addAggregation(histogram("histo").field("values").interval(interval).minDocCount(1))
+                        terms("terms")
+                                .field("values")
+                                .collectMode(randomFrom(SubAggCollectionMode.values()))
+                                .script(new Script("floor(_value / interval)", ScriptType.INLINE, CustomScriptPlugin.NAME, params))
+                                .size(maxNumTerms))
+                .addAggregation(
+                        histogram("histo")
+                                .field("values")
+                                .interval(interval)
+                                .minDocCount(1))
                 .execute().actionGet();
 
         assertSearchResponse(resp);
@@ -341,7 +393,13 @@ public class EquivalenceTests extends ESIntegTestCase {
         }
         indexRandom(true, indexingRequests);
 
-        SearchResponse response = client().prepareSearch("idx").addAggregation(terms("terms").field("double_value").collectMode(randomFrom(SubAggCollectionMode.values())).subAggregation(percentiles("pcts").field("double_value"))).execute().actionGet();
+        SearchResponse response = client().prepareSearch("idx")
+                .addAggregation(
+                        terms("terms")
+                                .field("double_value")
+                                .collectMode(randomFrom(SubAggCollectionMode.values()))
+                                .subAggregation(percentiles("pcts").field("double_value")))
+                .execute().actionGet();
         assertAllSuccessful(response);
         assertEquals(numDocs, response.getHits().getTotalHits());
     }
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java
similarity index 90%
rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java
rename to core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java
index b4260cf5530..dc439318978 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java
@@ -16,15 +16,17 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.messy.tests;
+package org.elasticsearch.search.aggregations.bucket;
 
 import com.carrotsearch.hppc.LongHashSet;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.groovy.GroovyPlugin;
+import org.elasticsearch.script.ScriptService.ScriptType;
+import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.histogram.ExtendedBounds;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
@@ -38,8 +40,12 @@ import org.hamcrest.Matchers;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
 
+import static java.util.Collections.emptyMap;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.filter;
@@ -56,11 +62,8 @@ import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.core.IsNull.notNullValue;
 
-/**
- *
- */
 @ESIntegTestCase.SuiteScopeTestCase
-public class HistogramTests extends ESIntegTestCase {
+public class HistogramIT extends ESIntegTestCase {
 
     private static final String SINGLE_VALUED_FIELD_NAME = "l_value";
     private static final String MULTI_VALUED_FIELD_NAME = "l_values";
@@ -72,7 +75,32 @@ public class HistogramTests extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(GroovyPlugin.class);
+        return Collections.singleton(CustomScriptPlugin.class);
+    }
+
+    public static class CustomScriptPlugin extends MockScriptPlugin {
+
+        @Override
+        protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
+            Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>();
+
+            scripts.put("_value + 1", vars -> {
+                long value = (long) vars.get("_value");
+                return value + 1L;
+            });
+
+            scripts.put("doc['l_value'].value", vars -> {
+                Map<?, ?> doc = (Map) vars.get("doc");
+                return doc.get(SINGLE_VALUED_FIELD_NAME);
+            });
+
+            scripts.put("doc['l_values']", vars -> {
+                Map<?, ?> doc = (Map) vars.get("doc");
+                return doc.get(MULTI_VALUED_FIELD_NAME);
+            });
+
+            return scripts;
+        }
     }
 
     @Override
@@ -349,15 +377,19 @@ public class HistogramTests extends ESIntegTestCase {
                 }
             }
             assertThat(sum.getValue(), equalTo((double) s));
-            assertThat((long) propertiesKeys[i], equalTo((long) i * interval));
-            assertThat((long) propertiesDocCounts[i], equalTo(valueCounts[i]));
-            assertThat((double) propertiesCounts[i], equalTo((double) s));
+            assertEquals(propertiesKeys[i], (long) i * interval);
+            assertThat(propertiesDocCounts[i], equalTo(valueCounts[i]));
+            assertThat(propertiesCounts[i], equalTo((double) s));
         }
     }
 
     public void testSingleValuedFieldOrderedBySubAggregationAsc() throws Exception {
         SearchResponse response = client().prepareSearch("idx")
-                .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval).order(Histogram.Order.aggregation("sum", true))
+                .addAggregation(
+                        histogram("histo")
+                                .field(SINGLE_VALUED_FIELD_NAME)
+                                .interval(interval)
+                                .order(Histogram.Order.aggregation("sum", true))
                         .subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME)))
                 .execute().actionGet();
 
@@ -397,7 +429,11 @@ public class HistogramTests extends ESIntegTestCase {
 
     public void testSingleValuedFieldOrderedBySubAggregationDesc() throws Exception {
         SearchResponse response = client().prepareSearch("idx")
-                .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval).order(Histogram.Order.aggregation("sum", false))
+                .addAggregation(
+                        histogram("histo")
+                                .field(SINGLE_VALUED_FIELD_NAME)
+                                .interval(interval)
+                                .order(Histogram.Order.aggregation("sum", false))
                         .subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME)))
                 .execute().actionGet();
 
@@ -437,7 +473,11 @@ public class HistogramTests extends ESIntegTestCase {
 
     public void testSingleValuedFieldOrderedByMultiValuedSubAggregationDesc() throws Exception {
         SearchResponse response = client().prepareSearch("idx")
-                .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval).order(Histogram.Order.aggregation("stats.sum", false))
+                .addAggregation(
+                        histogram("histo")
+                                .field(SINGLE_VALUED_FIELD_NAME)
+                                .interval(interval)
+                                .order(Histogram.Order.aggregation("stats.sum", false))
                         .subAggregation(stats("stats").field(SINGLE_VALUED_FIELD_NAME)))
                 .execute().actionGet();
 
@@ -478,9 +518,13 @@ public class HistogramTests extends ESIntegTestCase {
     public void testSingleValuedFieldOrderedBySubAggregationDescDeepOrderPath() throws Exception {
         boolean asc = randomBoolean();
         SearchResponse response = client().prepareSearch("idx")
-                .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval).order(Histogram.Order.aggregation("filter>max", asc))
+                .addAggregation(
+                        histogram("histo")
+                                .field(SINGLE_VALUED_FIELD_NAME)
+                                .interval(interval)
+                                .order(Histogram.Order.aggregation("filter>max", asc))
                         .subAggregation(filter("filter", matchAllQuery())
-                                .subAggregation(max("max").field(SINGLE_VALUED_FIELD_NAME))))
+                        .subAggregation(max("max").field(SINGLE_VALUED_FIELD_NAME))))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -515,7 +559,11 @@ public class HistogramTests extends ESIntegTestCase {
 
     public void testSingleValuedFieldWithValueScript() throws Exception {
         SearchResponse response = client().prepareSearch("idx")
-                .addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).script(new Script("_value + 1")).interval(interval))
+                .addAggregation(
+                        histogram("histo")
+                                .field(SINGLE_VALUED_FIELD_NAME)
+                                .script(new Script("_value + 1", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap()))
+                                .interval(interval))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -588,7 +636,11 @@ public class HistogramTests extends ESIntegTestCase {
 
     public void testMultiValuedFieldWithValueScript() throws Exception {
         SearchResponse response = client().prepareSearch("idx")
-                .addAggregation(histogram("histo").field(MULTI_VALUED_FIELD_NAME).script(new Script("_value + 1")).interval(interval))
+                .addAggregation(
+                        histogram("histo")
+                                .field(MULTI_VALUED_FIELD_NAME)
+                                .script(new Script("_value + 1", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap()))
+                                .interval(interval))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -621,7 +673,10 @@ public class HistogramTests extends ESIntegTestCase {
 
     public void testScriptSingleValue() throws Exception {
         SearchResponse response = client().prepareSearch("idx")
-                .addAggregation(histogram("histo").script(new Script("doc['" + SINGLE_VALUED_FIELD_NAME + "'].value")).interval(interval))
+                .addAggregation(
+                        histogram("histo")
+                            .script(new Script("doc['l_value'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap()))
+                            .interval(interval))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -642,7 +697,10 @@ public class HistogramTests extends ESIntegTestCase {
 
     public void testScriptMultiValued() throws Exception {
         SearchResponse response = client().prepareSearch("idx")
-                .addAggregation(histogram("histo").script(new Script("doc['" + MULTI_VALUED_FIELD_NAME + "']")).interval(interval))
+                .addAggregation(
+                        histogram("histo")
+                                .script(new Script("doc['l_values']", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap()))
+                                .interval(interval))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -701,8 +759,11 @@ public class HistogramTests extends ESIntegTestCase {
         SearchResponse response = client()
                 .prepareSearch("idx", "idx_unmapped")
                 .addAggregation(
-                        histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval)
-                                .extendedBounds(new ExtendedBounds((long) -1 * 2 * interval, (long) valueCounts.length * interval))).execute().actionGet();
+                        histogram("histo")
+                                .field(SINGLE_VALUED_FIELD_NAME)
+                                .interval(interval)
+                                .extendedBounds(new ExtendedBounds((long) -1 * 2 * interval, (long) valueCounts.length * interval)))
+                .get();
 
         assertSearchResponse(response);
 
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/CardinalityTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java
similarity index 79%
rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/CardinalityTests.java
rename to core/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java
index 97447cf2df3..594eba7ddb5 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/CardinalityTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java
@@ -17,14 +17,15 @@
  * under the License.
  */
 
-package org.elasticsearch.messy.tests;
+package org.elasticsearch.search.aggregations.metrics;
 
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
-import org.elasticsearch.script.groovy.GroovyPlugin;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
 import org.elasticsearch.search.aggregations.bucket.global.Global;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
@@ -33,9 +34,14 @@ import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Function;
 
+import static java.util.Collections.emptyMap;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
+import static org.elasticsearch.script.ScriptService.ScriptType;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.cardinality;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.global;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
@@ -45,11 +51,44 @@ import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.notNullValue;
 
 @ESIntegTestCase.SuiteScopeTestCase
-public class CardinalityTests extends ESIntegTestCase {
+public class CardinalityIT extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(GroovyPlugin.class);
+        return Collections.singleton(CustomScriptPlugin.class);
+    }
+
+    public static class CustomScriptPlugin extends MockScriptPlugin {
+
+        @Override
+        protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
+            Map<String, Function<Map<String, Object>, Object>> scripts = new HashMap<>();
+
+            scripts.put("_value", vars -> vars.get("_value"));
+
+            scripts.put("doc['str_value'].value", vars -> {
+                Map<?, ?> doc = (Map) vars.get("doc");
+                return doc.get("str_value");
+            });
+
+            scripts.put("doc['str_values'].values", vars -> {
+                Map<?, ?> doc = (Map) vars.get("doc");
+                ScriptDocValues.Strings strValue = (ScriptDocValues.Strings) doc.get("str_values");
+                return strValue.getValues();
+            });
+
+            scripts.put("doc[' + singleNumericField() + '].value", vars -> {
+                Map<?, ?> doc = (Map) vars.get("doc");
+                return doc.get(singleNumericField());
+            });
+
+            scripts.put("doc[' + multiNumericField(false) + '].values", vars -> {
+                Map<?, ?> doc = (Map) vars.get("doc");
+                return ((ScriptDocValues<?>) doc.get(multiNumericField(false))).getValues();
+            });
+
+            return scripts;
+        }
     }
 
     @Override
@@ -123,11 +162,11 @@ public class CardinalityTests extends ESIntegTestCase {
             assertThat(count.getValue(), greaterThan(0L));
         }
     }
-     private String singleNumericField() {
+     private static String singleNumericField() {
         return randomBoolean() ? "l_value" : "d_value";
     }
 
-    private String multiNumericField(boolean hash) {
+    private static String multiNumericField(boolean hash) {
         return randomBoolean() ? "l_values" : "d_values";
     }
 
@@ -264,7 +303,9 @@ public class CardinalityTests extends ESIntegTestCase {
     public void testSingleValuedStringScript() throws Exception {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .addAggregation(
-                        cardinality("cardinality").precisionThreshold(precisionThreshold).script(new Script("doc['str_value'].value")))
+                        cardinality("cardinality")
+                                .precisionThreshold(precisionThreshold)
+                                .script(new Script("doc['str_value'].value", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap())))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -278,7 +319,9 @@ public class CardinalityTests extends ESIntegTestCase {
     public void testMultiValuedStringScript() throws Exception {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .addAggregation(
-                        cardinality("cardinality").precisionThreshold(precisionThreshold).script(new Script("doc['str_values'].values")))
+                        cardinality("cardinality")
+                                .precisionThreshold(precisionThreshold)
+                                .script(new Script("doc['str_values'].values", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap())))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -290,10 +333,9 @@ public class CardinalityTests extends ESIntegTestCase {
     }
 
     public void testSingleValuedNumericScript() throws Exception {
+        Script script = new Script("doc[' + singleNumericField() + '].value", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap());
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
-                .addAggregation(
-                        cardinality("cardinality").precisionThreshold(precisionThreshold).script(
-                                new Script("doc['" + singleNumericField() + "'].value")))
+                .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).script(script))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -305,10 +347,9 @@ public class CardinalityTests extends ESIntegTestCase {
     }
 
     public void testMultiValuedNumericScript() throws Exception {
+        Script script = new Script("doc[' + multiNumericField(false) + '].values", ScriptType.INLINE, CustomScriptPlugin.NAME, null);
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
-                .addAggregation(
-                        cardinality("cardinality").precisionThreshold(precisionThreshold).script(
-                                new Script("doc['" + multiNumericField(false) + "'].values")))
+                .addAggregation(cardinality("cardinality").precisionThreshold(precisionThreshold).script(script))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -322,7 +363,10 @@ public class CardinalityTests extends ESIntegTestCase {
     public void testSingleValuedStringValueScript() throws Exception {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .addAggregation(
-                        cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_value").script(new Script("_value")))
+                        cardinality("cardinality")
+                                .precisionThreshold(precisionThreshold)
+                                .field("str_value")
+                                .script(new Script("_value", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap())))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -336,7 +380,10 @@ public class CardinalityTests extends ESIntegTestCase {
     public void testMultiValuedStringValueScript() throws Exception {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .addAggregation(
-                        cardinality("cardinality").precisionThreshold(precisionThreshold).field("str_values").script(new Script("_value")))
+                        cardinality("cardinality")
+                                .precisionThreshold(precisionThreshold)
+                                .field("str_values")
+                                .script(new Script("_value", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap())))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -350,8 +397,10 @@ public class CardinalityTests extends ESIntegTestCase {
     public void testSingleValuedNumericValueScript() throws Exception {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .addAggregation(
-                        cardinality("cardinality").precisionThreshold(precisionThreshold).field(singleNumericField())
-                                .script(new Script("_value")))
+                        cardinality("cardinality")
+                                .precisionThreshold(precisionThreshold)
+                                .field(singleNumericField())
+                                .script(new Script("_value", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap())))
                 .execute().actionGet();
 
         assertSearchResponse(response);
@@ -365,8 +414,10 @@ public class CardinalityTests extends ESIntegTestCase {
     public void testMultiValuedNumericValueScript() throws Exception {
         SearchResponse response = client().prepareSearch("idx").setTypes("type")
                 .addAggregation(
-                        cardinality("cardinality").precisionThreshold(precisionThreshold).field(multiNumericField(false))
-                                .script(new Script("_value")))
+                        cardinality("cardinality")
+                                .precisionThreshold(precisionThreshold)
+                                .field(multiNumericField(false))
+                                .script(new Script("_value", ScriptType.INLINE, CustomScriptPlugin.NAME, emptyMap())))
                 .execute().actionGet();
 
         assertSearchResponse(response);
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java
index c1f6847e2b6..57bdc7d5dfc 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentileRanksIT.java
@@ -450,18 +450,7 @@ public class HDRPercentileRanksIT extends AbstractNumericTestCase {
     @Override
     public void testScriptMultiValuedWithParams() throws Exception {
         int sigDigits = randomSignificantDigits();
-        Map<String, Object> params = new HashMap<>();
-        params.put("dec", 1);
-
-        // Equivalent to:
-        //
-        // List values = doc['values'].values;
-        // double[] res = new double[values.size()];
-        // for (int i = 0; i < res.length; i++) {
-        //      res[i] = values.get(i) - dec;
-        // };
-        // return res;
-        Script script = new Script("decrement all values", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params);
+        Script script = AggregationTestScriptsPlugin.DECREMENT_ALL_VALUES;
 
         final double[] pcts = randomPercents(minValues - 1, maxValues - 1);
         SearchResponse searchResponse = client()
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java
index 0dd0ee6799c..8112551f53c 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/HDRPercentilesIT.java
@@ -436,18 +436,7 @@ public class HDRPercentilesIT extends AbstractNumericTestCase {
 
     @Override
     public void testScriptMultiValuedWithParams() throws Exception {
-        Map<String, Object> params = new HashMap<>();
-        params.put("dec", 1);
-
-        // Equivalent to:
-        //
-        // List values = doc['values'].values;
-        // double[] res = new double[values.size()];
-        // for (int i = 0; i < res.length; i++) {
-        //      res[i] = values.get(i) - dec;
-        // };
-        // return res;
-        Script script = new Script("decrement all values", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params);
+        Script script = AggregationTestScriptsPlugin.DECREMENT_ALL_VALUES;
 
         final double[] pcts = randomPercentiles();
         int sigDigits = randomSignificantDigits();
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentileRanksTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java
similarity index 83%
rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentileRanksTests.java
rename to core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java
index 7e76b3f03eb..e50b89d8b96 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentileRanksTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java
@@ -16,21 +16,22 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.messy.tests;
+package org.elasticsearch.search.aggregations.metrics;
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.script.MockScriptPlugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService.ScriptType;
-import org.elasticsearch.script.groovy.GroovyPlugin;
+import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.global.Global;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Order;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
-import org.elasticsearch.search.aggregations.metrics.AbstractNumericTestCase;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
 import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanks;
 import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanksAggregationBuilder;
@@ -42,7 +43,9 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 
+import static java.util.Collections.emptyMap;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.filter;
@@ -57,13 +60,11 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.sameInstance;
 
-/**
- *
- */
-public class TDigestPercentileRanksTests extends AbstractNumericTestCase {
+public class TDigestPercentileRanksIT extends AbstractNumericTestCase {
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(GroovyPlugin.class);
+        return Collections.singleton(AggregationTestScriptsPlugin.class);
     }
 
     private static double[] randomPercents(long minValue, long maxValue) {
@@ -84,7 +85,7 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase {
             }
         }
         Arrays.sort(percents);
-        Loggers.getLogger(TDigestPercentileRanksTests.class).info("Using values={}", Arrays.toString(percents));
+        Loggers.getLogger(TDigestPercentileRanksIT.class).info("Using values={}", Arrays.toString(percents));
         return percents;
     }
 
@@ -198,8 +199,7 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase {
         PercentileRanks values = global.getAggregations().get("percentile_ranks");
         assertThat(values, notNullValue());
         assertThat(values.getName(), equalTo("percentile_ranks"));
-        assertThat((PercentileRanks) global.getProperty("percentile_ranks"), sameInstance(values));
-
+        assertThat(global.getProperty("percentile_ranks"), sameInstance(values));
     }
 
     public void testSingleValuedFieldOutsideRange() throws Exception {
@@ -238,9 +238,12 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase {
         final double[] pcts = randomPercents(minValue - 1, maxValue - 1);
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(randomCompression(percentileRanks("percentile_ranks"))
-                        .field("value").script(new Script("_value - 1"))
-                        .values(pcts))
+                .addAggregation(
+                        randomCompression(
+                                percentileRanks("percentile_ranks"))
+                                .field("value")
+                                .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .values(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -256,10 +259,12 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase {
         final double[] pcts = randomPercents(minValue - 1, maxValue - 1);
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(randomCompression(percentileRanks("percentile_ranks"))
-                        .field("value")
-                                .script(new Script("_value - dec", ScriptType.INLINE, null, params))
-                        .values(pcts))
+                .addAggregation(
+                        randomCompression(
+                                percentileRanks("percentile_ranks"))
+                                .field("value")
+                                .script(new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params))
+                                .values(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -289,9 +294,12 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase {
         final double[] pcts = randomPercents(minValues - 1, maxValues - 1);
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(randomCompression(percentileRanks("percentile_ranks"))
-                        .field("values").script(new Script("_value - 1"))
-                        .values(pcts))
+                .addAggregation(
+                        randomCompression(
+                                percentileRanks("percentile_ranks"))
+                                .field("values")
+                                .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .values(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -304,9 +312,12 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase {
         final double[] pcts = randomPercents(-maxValues, -minValues);
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(randomCompression(percentileRanks("percentile_ranks"))
-                        .field("values").script(new Script("_value * -1"))
-                        .values(pcts))
+                .addAggregation(
+                        randomCompression(
+                                percentileRanks("percentile_ranks"))
+                                .field("values")
+                                .script(new Script("_value * -1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .values(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -322,10 +333,12 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase {
         final double[] pcts = randomPercents(minValues - 1, maxValues - 1);
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(randomCompression(percentileRanks("percentile_ranks"))
-                        .field("values")
-                                .script(new Script("_value - dec", ScriptType.INLINE, null, params))
-                        .values(pcts))
+                .addAggregation(
+                        randomCompression(
+                                percentileRanks("percentile_ranks"))
+                                .field("values")
+                                .script(new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params))
+                                .values(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -339,9 +352,11 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase {
         final double[] pcts = randomPercents(minValue, maxValue);
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(randomCompression(percentileRanks("percentile_ranks"))
-                        .script(new Script("doc['value'].value"))
-                        .values(pcts))
+                .addAggregation(
+                        randomCompression(
+                                percentileRanks("percentile_ranks"))
+                                .script(new Script("doc['value'].value", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .values(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -354,13 +369,17 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase {
     public void testScriptSingleValuedWithParams() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("dec", 1);
+
+        Script script = new Script("doc['value'].value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params);
+
         final double[] pcts = randomPercents(minValue - 1, maxValue - 1);
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(randomCompression(percentileRanks("percentile_ranks"))
-                        .script(
-                                new Script("doc['value'].value - dec", ScriptType.INLINE, null, params))
-                        .values(pcts))
+                .addAggregation(
+                        randomCompression(
+                                percentileRanks("percentile_ranks"))
+                                .script(script)
+                                .values(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -372,11 +391,14 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase {
     @Override
     public void testScriptMultiValued() throws Exception {
         final double[] pcts = randomPercents(minValues, maxValues);
+        Script script = new Script("doc['values'].values", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap());
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(randomCompression(percentileRanks("percentile_ranks"))
-                        .script(new Script("doc['values'].values"))
-                        .values(pcts))
+                .addAggregation(
+                        randomCompression(
+                                percentileRanks("percentile_ranks"))
+                                .script(script)
+                                .values(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -387,16 +409,16 @@ public class TDigestPercentileRanksTests extends AbstractNumericTestCase {
 
     @Override
     public void testScriptMultiValuedWithParams() throws Exception {
-        Map<String, Object> params = new HashMap<>();
-        params.put("dec", 1);
+        Script script = AggregationTestScriptsPlugin.DECREMENT_ALL_VALUES;
+
         final double[] pcts = randomPercents(minValues - 1, maxValues - 1);
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(randomCompression(percentileRanks("percentile_ranks"))
-                                .script(new Script(
-                                        "List values = doc['values'].values; double[] res = new double[values.size()]; for (int i = 0; i < res.length; i++) { res[i] = values.get(i) - dec; }; return res;",
-                                        ScriptType.INLINE, null, params))
-                        .values(pcts))
+                .addAggregation(
+                        randomCompression(
+                                percentileRanks("percentile_ranks"))
+                                .script(script)
+                                .values(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentilesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java
similarity index 81%
rename from modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentilesTests.java
rename to core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java
index 712c9ebd951..b0268351954 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentilesTests.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java
@@ -16,7 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.elasticsearch.messy.tests;
+package org.elasticsearch.search.aggregations.metrics;
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.logging.Loggers;
@@ -24,13 +24,12 @@ import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptService.ScriptType;
-import org.elasticsearch.script.groovy.GroovyPlugin;
+import org.elasticsearch.search.aggregations.AggregationTestScriptsPlugin;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.global.Global;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Order;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
-import org.elasticsearch.search.aggregations.metrics.AbstractNumericTestCase;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentile;
 import org.elasticsearch.search.aggregations.metrics.percentiles.Percentiles;
 import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesAggregationBuilder;
@@ -43,6 +42,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static java.util.Collections.emptyMap;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.search.aggregations.AggregationBuilders.filter;
@@ -57,13 +57,11 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.sameInstance;
 
-/**
- *
- */
-public class TDigestPercentilesTests extends AbstractNumericTestCase {
+public class TDigestPercentilesIT extends AbstractNumericTestCase {
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Collections.singleton(GroovyPlugin.class);
+        return Collections.singleton(AggregationTestScriptsPlugin.class);
     }
 
     private static double[] randomPercentiles() {
@@ -83,7 +81,7 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase {
             }
         }
         Arrays.sort(percentiles);
-        Loggers.getLogger(TDigestPercentilesTests.class).info("Using percentiles={}", Arrays.toString(percentiles));
+        Loggers.getLogger(TDigestPercentilesIT.class).info("Using percentiles={}", Arrays.toString(percentiles));
         return percentiles;
     }
 
@@ -197,8 +195,7 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase {
         Percentiles percentiles = global.getAggregations().get("percentiles");
         assertThat(percentiles, notNullValue());
         assertThat(percentiles.getName(), equalTo("percentiles"));
-        assertThat((Percentiles) global.getProperty("percentiles"), sameInstance(percentiles));
-
+        assertThat(global.getProperty("percentiles"), sameInstance(percentiles));
     }
 
     @Override
@@ -222,9 +219,12 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase {
         final double[] pcts = randomPercentiles();
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(randomCompression(percentiles("percentiles"))
-.field("value").script(new Script("_value - 1"))
-                        .percentiles(pcts))
+                .addAggregation(
+                        randomCompression(
+                                percentiles("percentiles"))
+                                    .field("value")
+                                    .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                    .percentiles(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -240,10 +240,12 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase {
         final double[] pcts = randomPercentiles();
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(randomCompression(percentiles("percentiles"))
-.field("value")
-                                .script(new Script("_value - dec", ScriptType.INLINE, null, params))
-                        .percentiles(pcts))
+                .addAggregation(
+                        randomCompression(
+                                percentiles("percentiles"))
+                                    .field("value")
+                                    .script(new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params))
+                                .percentiles(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -257,9 +259,7 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase {
         final double[] pcts = randomPercentiles();
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(randomCompression(percentiles("percentiles"))
-                        .field("values")
-                        .percentiles(pcts))
+                .addAggregation(randomCompression(percentiles("percentiles")).field("values").percentiles(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -273,9 +273,12 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase {
         final double[] pcts = randomPercentiles();
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(randomCompression(percentiles("percentiles"))
-.field("values").script(new Script("_value - 1"))
-                        .percentiles(pcts))
+                .addAggregation(
+                        randomCompression(
+                                percentiles("percentiles"))
+                                    .field("values")
+                                .script(new Script("_value - 1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                .percentiles(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -288,9 +291,12 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase {
         final double[] pcts = randomPercentiles();
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(randomCompression(percentiles("percentiles"))
-                        .field("values").script(new Script("_value * -1"))
-                        .percentiles(pcts))
+                .addAggregation(
+                        randomCompression(
+                                percentiles("percentiles"))
+                                    .field("values")
+                                    .script(new Script("_value * -1", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap()))
+                                    .percentiles(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -306,10 +312,12 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase {
         final double[] pcts = randomPercentiles();
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(randomCompression(percentiles("percentiles"))
-.field("values")
-                                .script(new Script("_value - dec", ScriptType.INLINE, null, params))
-                        .percentiles(pcts))
+                .addAggregation(
+                        randomCompression(
+                                percentiles("percentiles"))
+                                    .field("values")
+                                    .script(new Script("_value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params))
+                                    .percentiles(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -320,12 +328,15 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase {
 
     @Override
     public void testScriptSingleValued() throws Exception {
+        Script script = new Script("doc['value'].value", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap());
         final double[] pcts = randomPercentiles();
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(randomCompression(percentiles("percentiles"))
-.script(new Script("doc['value'].value"))
-                        .percentiles(pcts))
+                .addAggregation(
+                        randomCompression(
+                                percentiles("percentiles"))
+                                    .script(script)
+                                    .percentiles(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -338,13 +349,17 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase {
     public void testScriptSingleValuedWithParams() throws Exception {
         Map<String, Object> params = new HashMap<>();
         params.put("dec", 1);
+
+        Script script = new Script("doc['value'].value - dec", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, params);
+
         final double[] pcts = randomPercentiles();
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(randomCompression(percentiles("percentiles"))
-                        .script(
-                                new Script("doc['value'].value - dec", ScriptType.INLINE, null, params))
-                        .percentiles(pcts))
+                .addAggregation(
+                        randomCompression(
+                                percentiles("percentiles"))
+                                .script(script)
+                                .percentiles(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -356,11 +371,15 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase {
     @Override
     public void testScriptMultiValued() throws Exception {
         final double[] pcts = randomPercentiles();
+        Script script = new Script("doc['values'].values", ScriptType.INLINE, AggregationTestScriptsPlugin.NAME, emptyMap());
+
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(randomCompression(percentiles("percentiles"))
-.script(new Script("doc['values'].values"))
-                        .percentiles(pcts))
+                .addAggregation(
+                        randomCompression(
+                                percentiles("percentiles"))
+                                    .script(script)
+                                    .percentiles(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
@@ -371,16 +390,16 @@ public class TDigestPercentilesTests extends AbstractNumericTestCase {
 
     @Override
     public void testScriptMultiValuedWithParams() throws Exception {
-        Map<String, Object> params = new HashMap<>();
-        params.put("dec", 1);
+        Script script = AggregationTestScriptsPlugin.DECREMENT_ALL_VALUES;
+
         final double[] pcts = randomPercentiles();
         SearchResponse searchResponse = client().prepareSearch("idx")
                 .setQuery(matchAllQuery())
-                .addAggregation(randomCompression(percentiles("percentiles"))
-                                .script(new Script(
-                                        "List values = doc['values'].values; double[] res = new double[values.size()]; for (int i = 0; i < res.length; i++) { res[i] = values.get(i) - dec; }; return res;",
-                                        ScriptType.INLINE, null, params))
-                        .percentiles(pcts))
+                .addAggregation(
+                        randomCompression(
+                                percentiles("percentiles"))
+                                    .script(script)
+                                    .percentiles(pcts))
                 .execute().actionGet();
 
         assertHitCount(searchResponse, 10);
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java
index a9bf09344f2..867e6d29c88 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/package-info.java
@@ -37,22 +37,18 @@
 /* List of renames that took place:
   renamed:    core/src/test/java/org/elasticsearch/search/aggregations/metrics/AvgIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/AvgTests.java
   renamed:    core/src/test/java/org/elasticsearch/document/BulkIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java
-  renamed:    core/src/test/java/org/elasticsearch/search/aggregations/metrics/CardinalityIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/CardinalityTests.java
   renamed:    core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ChildQuerySearchTests.java
   renamed:    core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ContextAndHeaderTransportTests.java
   ^^^^^ note: the methods from this test using mustache were moved to the mustache module under its messy tests package.
   renamed:    core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DateHistogramTests.java
   renamed:    core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DateRangeTests.java
   renamed:    core/src/test/java/org/elasticsearch/search/aggregations/bucket/DoubleTermsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java
-  renamed:    core/src/test/java/org/elasticsearch/search/aggregations/EquivalenceIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/EquivalenceTests.java
   renamed:    core/src/test/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/ExtendedStatsTests.java
   renamed:    core/src/test/java/org/elasticsearch/search/functionscore/FunctionScoreIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/FunctionScoreTests.java
   renamed:    core/src/test/java/org/elasticsearch/search/geo/GeoDistanceIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/GeoDistanceTests.java
-  renamed:    core/src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/HistogramTests.java
   renamed:    core/src/test/java/org/elasticsearch/search/aggregations/bucket/IPv4RangeIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IPv4RangeTests.java
   renamed:    core/src/test/java/org/elasticsearch/script/IndexLookupIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexLookupTests.java
   renamed:    core/src/test/java/org/elasticsearch/script/IndexedScriptIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndexedScriptTests.java
-  renamed:    core/src/test/java/org/elasticsearch/action/IndicesRequestIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/IndicesRequestTests.java
   renamed:    core/src/test/java/org/elasticsearch/search/innerhits/InnerHitsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/InnerHitsTests.java
   renamed:    core/src/test/java/org/elasticsearch/search/aggregations/bucket/LongTermsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java
   renamed:    core/src/test/java/org/elasticsearch/search/aggregations/metrics/MaxIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/MaxTests.java
@@ -72,8 +68,6 @@
   renamed:    core/src/test/java/org/elasticsearch/search/aggregations/metrics/StatsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StatsTests.java
   renamed:    core/src/test/java/org/elasticsearch/search/aggregations/bucket/StringTermsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/StringTermsTests.java
   renamed:    core/src/test/java/org/elasticsearch/search/aggregations/metrics/SumIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SumTests.java
-  renamed:    core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentileRanksIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentileRanksTests.java
-  renamed:    core/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TDigestPercentilesTests.java
   renamed:    core/src/test/java/org/elasticsearch/search/aggregations/bucket/TopHitsIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TopHitsTests.java
   renamed:    core/src/test/java/org/elasticsearch/index/mapper/TransformOnIndexMapperIT.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/messy/tests/TransformOnIndexMapperTests.java
   renamed:    core/src/main/java/org/elasticsearch/script/groovy/GroovyScriptCompilationException.java -> plugins/lang-groovy/src/test/java/org/elasticsearch/script/groovy/GroovyRestIT.java

From c05f818160cb7423445fafffa37c6eb3c9426e63 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 7 Jul 2016 12:33:27 -0400
Subject: [PATCH 1271/1311] Fix casing of "Elasticsearch" in how-to docs

---
 docs/reference/how-to/general.asciidoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/reference/how-to/general.asciidoc b/docs/reference/how-to/general.asciidoc
index 60f86d821d9..60f0181b2bb 100644
--- a/docs/reference/how-to/general.asciidoc
+++ b/docs/reference/how-to/general.asciidoc
@@ -15,7 +15,7 @@ use the <<search-request-scroll,Scroll>> API.
 [[sparsity]]
 === Avoid sparsity
 
-The data-structures behind Lucene, which elasticsearch relies on in order to
+The data-structures behind Lucene, which Elasticsearch relies on in order to
 index and store data, work best with dense data, ie. when all documents have the
 same fields. This is especially true for fields that have norms enabled (which
 is the case for `text` fields by default) or doc values enabled (which is the
@@ -39,7 +39,7 @@ fraction of the documents of the index. Although slightly more complex with doc
 values due to the fact that doc values have multiple ways that they can be
 encoded depending on the type of field and on the actual data that the field
 stores, the problem is very similar. In case you wonder: `fielddata`, which was
-used in elasticsearch pre-2.0 before being replaced with doc values, also
+used in Elasticsearch pre-2.0 before being replaced with doc values, also
 suffered from this issue, except that the impact was only on the memory
 footprint since `fielddata` was not explicitly materialized on disk.
 

From a20cf8275b99687d453664cdd5a3550d05baa5c5 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 7 Jul 2016 12:43:20 -0400
Subject: [PATCH 1272/1311] Add version 2.3.4 to upgrade from versions

---
 qa/vagrant/versions | 1 +
 1 file changed, 1 insertion(+)

diff --git a/qa/vagrant/versions b/qa/vagrant/versions
index 5f6e0edf99c..98cb5bf8195 100644
--- a/qa/vagrant/versions
+++ b/qa/vagrant/versions
@@ -11,3 +11,4 @@
 2.3.1
 2.3.2
 2.3.3
+2.3.4

From 134b73ab3324f39b72d074b1a0a079ed6fd91ac3 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Thu, 7 Jul 2016 19:10:21 +0200
Subject: [PATCH 1273/1311] Rest Client: HostsSniffer to set http as default
 scheme

 The assumption is HostsSniffer is that all of the arguments have been properly provided and validated through HostsSniffer.Builder, except they weren't, as the scheme didn't have a default value and when not set would cause NPEs down the road. Improved tests to catch this also.
---
 .../java/org/elasticsearch/client/sniff/HostsSniffer.java   | 2 +-
 .../org/elasticsearch/client/sniff/HostsSnifferTests.java   | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
index 7cdb51066cb..f06436c6175 100644
--- a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
+++ b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
@@ -156,7 +156,7 @@ public class HostsSniffer {
 
         private final RestClient restClient;
         private long sniffRequestTimeoutMillis = DEFAULT_SNIFF_REQUEST_TIMEOUT;
-        private Scheme scheme;
+        private Scheme scheme = Scheme.HTTP;
 
         private Builder(RestClient restClient) {
             Objects.requireNonNull(restClient, "restClient cannot be null");
diff --git a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
index 1b38f034262..6e0c3a728d5 100644
--- a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
+++ b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/HostsSnifferTests.java
@@ -89,7 +89,11 @@ public class HostsSnifferTests extends RestClientTestCase {
     public void testSniffNodes() throws IOException, URISyntaxException {
         HttpHost httpHost = new HttpHost(httpServer.getAddress().getHostString(), httpServer.getAddress().getPort());
         try (RestClient restClient = RestClient.builder(httpHost).build()) {
-            HostsSniffer sniffer = new HostsSniffer(restClient, sniffRequestTimeout, scheme);
+            HostsSniffer.Builder builder = HostsSniffer.builder(restClient).setSniffRequestTimeoutMillis(sniffRequestTimeout);
+            if (scheme != HostsSniffer.Scheme.HTTP || randomBoolean()) {
+                builder.setScheme(scheme);
+            }
+            HostsSniffer sniffer = builder.build();
             try {
                 List<HttpHost> sniffedHosts = sniffer.sniffHosts();
                 if (sniffResponse.isFailure) {

From 68676f8cf1faabd8c5c12a88290d57ee64254a46 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 7 Jul 2016 13:31:30 -0400
Subject: [PATCH 1274/1311] Migrate Vagrant tests for Fedora to Fedora 24

This commit migrates the Vagrant box for Fedora for the packaging tests
from Fedora 22 to Fedora 24 as Fedora 22 reached end-of-line upon the
release of Fedora 24.

Relates #19308
---
 Vagrantfile             | 4 ++--
 qa/vagrant/build.gradle | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index 5b51f612eb3..423b50038e0 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -60,8 +60,8 @@ Vagrant.configure(2) do |config|
     config.vm.box = "elastic/oraclelinux-7-x86_64"
     rpm_common config
   end
-  config.vm.define "fedora-22" do |config|
-    config.vm.box = "elastic/fedora-22-x86_64"
+  config.vm.define "fedora-24" do |config|
+    config.vm.box = "elastic/fedora-24-x86_64"
     dnf_common config
   end
   config.vm.define "opensuse-13" do |config|
diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle
index 878e9c5ef7f..86523a734ad 100644
--- a/qa/vagrant/build.gradle
+++ b/qa/vagrant/build.gradle
@@ -30,7 +30,7 @@ List<String> availableBoxes = [
     'centos-6',
     'centos-7',
     'debian-8',
-    'fedora-22',
+    'fedora-24',
     'oel-6',
     'oel-7',
     'opensuse-13',

From d3f8329a3dbd182ff69f74d4e177fc7a4550f1af Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 7 Jul 2016 14:40:05 -0400
Subject: [PATCH 1275/1311] Tighten ensure atomic move cleanup

This commit tightens the cleanup after possible errors while ensuring
the filesystem supports atomic move.

Relates #19309
---
 .../main/java/org/elasticsearch/env/NodeEnvironment.java | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
index dc8b9289d0f..9af22324fd9 100644
--- a/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
+++ b/core/src/main/java/org/elasticsearch/env/NodeEnvironment.java
@@ -897,17 +897,20 @@ public final class NodeEnvironment extends AbstractComponent implements Closeabl
         for (NodePath nodePath : nodePaths) {
             assert Files.isDirectory(nodePath.path) : nodePath.path + " is not a directory";
             final Path src = nodePath.path.resolve("__es__.tmp");
-            Files.createFile(src);
             final Path target = nodePath.path.resolve("__es__.final");
             try {
+                Files.createFile(src);
                 Files.move(src, target, StandardCopyOption.ATOMIC_MOVE);
             } catch (AtomicMoveNotSupportedException ex) {
                 throw new IllegalStateException("atomic_move is not supported by the filesystem on path ["
                         + nodePath.path
                         + "] atomic_move is required for elasticsearch to work correctly.", ex);
             } finally {
-                Files.deleteIfExists(src);
-                Files.deleteIfExists(target);
+                try {
+                    Files.deleteIfExists(src);
+                } finally {
+                    Files.deleteIfExists(target);
+                }
             }
         }
     }

From e86aa29f671d46788c7610691f900b2547b549ff Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 7 Jul 2016 14:44:03 -0400
Subject: [PATCH 1276/1311] Die with dignity

Today when a thread encounters a fatal unrecoverable error that
threatens the stability of the JVM, Elasticsearch marches on. This
includes out of memory errors, stack overflow errors and other errors
that leave the JVM in a questionable state. Instead, the Elasticsearch
JVM should die when these errors are encountered. This commit causes
this to be the case.

Relates #19272
---
 core/build.gradle                             |   2 +-
 .../elasticsearch/bootstrap/Bootstrap.java    |   6 +
 ...ElasticsearchUncaughtExceptionHandler.java |  94 +++++++++++
 .../org/elasticsearch/bootstrap/Security.java |   2 +-
 .../elasticsearch/bootstrap/security.policy   |   2 +-
 ...icsearchUncaughtExceptionHandlerTests.java | 152 ++++++++++++++++++
 distribution/licenses/securesm-1.0.jar.sha1   |   1 -
 distribution/licenses/securesm-1.1.jar.sha1   |   1 +
 .../migration/migrate_5_0/packaging.asciidoc  |   8 +
 docs/reference/setup.asciidoc                 |   2 +
 docs/reference/setup/stopping.asciidoc        |  58 +++++++
 .../bootstrap/BootstrapForTesting.java        |   2 +-
 12 files changed, 325 insertions(+), 5 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java
 create mode 100644 core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandlerTests.java
 delete mode 100644 distribution/licenses/securesm-1.0.jar.sha1
 create mode 100644 distribution/licenses/securesm-1.1.jar.sha1
 create mode 100644 docs/reference/setup/stopping.asciidoc

diff --git a/core/build.gradle b/core/build.gradle
index 513e998ebdc..6fd8c62af3e 100644
--- a/core/build.gradle
+++ b/core/build.gradle
@@ -56,7 +56,7 @@ dependencies {
   compile "org.apache.lucene:lucene-spatial3d:${versions.lucene}"
   compile "org.apache.lucene:lucene-suggest:${versions.lucene}"
 
-  compile 'org.elasticsearch:securesm:1.0'
+  compile 'org.elasticsearch:securesm:1.1'
 
   // utilities
   compile 'net.sf.jopt-simple:jopt-simple:5.0.2'
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
index 3d5347307d6..bdec058b04a 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java
@@ -246,6 +246,12 @@ final class Bootstrap {
             // fail if somebody replaced the lucene jars
             checkLucene();
 
+            // install the default uncaught exception handler; must be done before security is
+            // initialized as we do not want to grant the runtime permission
+            // setDefaultUncaughtExceptionHandler
+            Thread.setDefaultUncaughtExceptionHandler(
+                new ElasticsearchUncaughtExceptionHandler(() -> Node.NODE_NAME_SETTING.get(settings)));
+
             INSTANCE.setup(true, settings, environment);
 
             INSTANCE.start();
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java b/core/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java
new file mode 100644
index 00000000000..405e919fabd
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandler.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.bootstrap;
+
+import org.apache.lucene.index.MergePolicy;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.logging.Loggers;
+
+import java.io.IOError;
+import java.util.Objects;
+import java.util.function.Supplier;
+
+class ElasticsearchUncaughtExceptionHandler implements Thread.UncaughtExceptionHandler {
+
+    private final Supplier<String> loggingPrefixSupplier;
+
+    ElasticsearchUncaughtExceptionHandler(final Supplier<String> loggingPrefixSupplier) {
+        this.loggingPrefixSupplier = Objects.requireNonNull(loggingPrefixSupplier);
+    }
+
+    @Override
+    public void uncaughtException(Thread t, Throwable e) {
+        if (isFatalUncaught(e)) {
+            try {
+                onFatalUncaught(t.getName(), e);
+            } finally {
+                // we use specific error codes in case the above notification failed, at least we
+                // will have some indication of the error bringing us down
+                if (e instanceof InternalError) {
+                    halt(128);
+                } else if (e instanceof OutOfMemoryError) {
+                    halt(127);
+                } else if (e instanceof StackOverflowError) {
+                    halt(126);
+                } else if (e instanceof UnknownError) {
+                    halt(125);
+                } else if (e instanceof IOError) {
+                    halt(124);
+                } else {
+                    halt(1);
+                }
+            }
+        } else {
+            onNonFatalUncaught(t.getName(), e);
+        }
+    }
+
+    // visible for testing
+    static boolean isFatalUncaught(Throwable e) {
+        return isFatalCause(e) || (e instanceof MergePolicy.MergeException && isFatalCause(e.getCause()));
+    }
+
+    private static boolean isFatalCause(Throwable cause) {
+        return cause instanceof Error;
+    }
+
+    // visible for testing
+    void onFatalUncaught(final String threadName, final Throwable t) {
+        final ESLogger logger = Loggers.getLogger(ElasticsearchUncaughtExceptionHandler.class, loggingPrefixSupplier.get());
+        logger.error("fatal error in thread [{}], exiting", t, threadName);
+    }
+
+    // visible for testing
+    void onNonFatalUncaught(final String threadName, final Throwable t) {
+        final ESLogger logger = Loggers.getLogger(ElasticsearchUncaughtExceptionHandler.class, loggingPrefixSupplier.get());
+        logger.warn("uncaught exception in thread [{}]", t, threadName);
+    }
+
+    // visible for testing
+    @SuppressForbidden(reason = "halt")
+    void halt(int status) {
+        // we halt to prevent shutdown hooks from running
+        Runtime.getRuntime().halt(status);
+    }
+
+}
diff --git a/core/src/main/java/org/elasticsearch/bootstrap/Security.java b/core/src/main/java/org/elasticsearch/bootstrap/Security.java
index c44b46c6b0c..05d2c8c1bfd 100644
--- a/core/src/main/java/org/elasticsearch/bootstrap/Security.java
+++ b/core/src/main/java/org/elasticsearch/bootstrap/Security.java
@@ -120,7 +120,7 @@ final class Security {
         Policy.setPolicy(new ESPolicy(createPermissions(environment), getPluginPermissions(environment), filterBadDefaults));
 
         // enable security manager
-        System.setSecurityManager(new SecureSM());
+        System.setSecurityManager(new SecureSM(new String[] { "org.elasticsearch.bootstrap." }));
 
         // do some basic tests
         selfTest();
diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
index dcf37d3dd5c..a4a45585b5f 100644
--- a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
+++ b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy
@@ -24,7 +24,7 @@
 //// SecurityManager impl:
 //// Must have all permissions to properly perform access checks
 
-grant codeBase "${codebase.securesm-1.0.jar}" {
+grant codeBase "${codebase.securesm-1.1.jar}" {
   permission java.security.AllPermission;
 };
 
diff --git a/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandlerTests.java b/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandlerTests.java
new file mode 100644
index 00000000000..e4ff83e9b40
--- /dev/null
+++ b/core/src/test/java/org/elasticsearch/bootstrap/ElasticsearchUncaughtExceptionHandlerTests.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.bootstrap;
+
+import org.apache.lucene.index.MergePolicy;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.Before;
+
+import java.io.IOError;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class ElasticsearchUncaughtExceptionHandlerTests extends ESTestCase {
+
+    private Map<Class<? extends Error>, Integer> expectedStatus;
+
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        Map<Class<? extends Error>, Integer> expectedStatus = new HashMap<>();
+        expectedStatus.put(InternalError.class, 128);
+        expectedStatus.put(OutOfMemoryError.class, 127);
+        expectedStatus.put(StackOverflowError.class, 126);
+        expectedStatus.put(UnknownError.class, 125);
+        expectedStatus.put(IOError.class, 124);
+        this.expectedStatus = Collections.unmodifiableMap(expectedStatus);
+    }
+
+    public void testUncaughtError() throws InterruptedException {
+        final Error error = randomFrom(
+            new InternalError(),
+            new OutOfMemoryError(),
+            new StackOverflowError(),
+            new UnknownError(),
+            new IOError(new IOException("fatal")),
+            new Error() {});
+        final Thread thread = new Thread(() -> { throw error; });
+        final String name = randomAsciiOfLength(10);
+        thread.setName(name);
+        final AtomicBoolean halt = new AtomicBoolean();
+        final AtomicInteger observedStatus = new AtomicInteger();
+        final AtomicReference<String> threadNameReference = new AtomicReference<>();
+        final AtomicReference<Throwable> throwableReference = new AtomicReference<>();
+        thread.setUncaughtExceptionHandler(new ElasticsearchUncaughtExceptionHandler(() -> "testUncaughtError") {
+
+            @Override
+            void halt(int status) {
+                halt.set(true);
+                observedStatus.set(status);
+            }
+
+            @Override
+            void onFatalUncaught(String threadName, Throwable t) {
+                threadNameReference.set(threadName);
+                throwableReference.set(t);
+            }
+
+            @Override
+            void onNonFatalUncaught(String threadName, Throwable t) {
+                fail();
+            }
+
+        });
+        thread.start();
+        thread.join();
+        assertTrue(halt.get());
+        final int status;
+        if (expectedStatus.containsKey(error.getClass())) {
+            status = expectedStatus.get(error.getClass());
+        } else {
+            status = 1;
+        }
+        assertThat(observedStatus.get(), equalTo(status));
+        assertThat(threadNameReference.get(), equalTo(name));
+        assertThat(throwableReference.get(), equalTo(error));
+    }
+
+    public void testUncaughtException() throws InterruptedException {
+        final RuntimeException e = new RuntimeException("boom");
+        final Thread thread = new Thread(() -> { throw e; });
+        final String name = randomAsciiOfLength(10);
+        thread.setName(name);
+        final AtomicReference<String> threadNameReference = new AtomicReference<>();
+        final AtomicReference<Throwable> throwableReference = new AtomicReference<>();
+        thread.setUncaughtExceptionHandler(new ElasticsearchUncaughtExceptionHandler(() -> "testUncaughtException") {
+            @Override
+            void halt(int status) {
+                fail();
+            }
+
+            @Override
+            void onFatalUncaught(String threadName, Throwable t) {
+                fail();
+            }
+
+            @Override
+            void onNonFatalUncaught(String threadName, Throwable t) {
+                threadNameReference.set(threadName);
+                throwableReference.set(t);
+            }
+        });
+        thread.start();
+        thread.join();
+        assertThat(threadNameReference.get(), equalTo(name));
+        assertThat(throwableReference.get(), equalTo(e));
+    }
+
+    public void testIsFatalCause() {
+        assertFatal(new MergePolicy.MergeException(new OutOfMemoryError(), null));
+        assertFatal(new OutOfMemoryError());
+        assertFatal(new StackOverflowError());
+        assertFatal(new InternalError());
+        assertFatal(new UnknownError());
+        assertFatal(new IOError(new IOException()));
+        assertNonFatal(new RuntimeException());
+        assertNonFatal(new UncheckedIOException(new IOException()));
+    }
+
+    private void assertFatal(Throwable cause) {
+        assertTrue(ElasticsearchUncaughtExceptionHandler.isFatalUncaught(cause));
+    }
+
+    private void assertNonFatal(Throwable cause) {
+        assertFalse(ElasticsearchUncaughtExceptionHandler.isFatalUncaught(cause));
+    }
+
+}
diff --git a/distribution/licenses/securesm-1.0.jar.sha1 b/distribution/licenses/securesm-1.0.jar.sha1
deleted file mode 100644
index 96d45d93e66..00000000000
--- a/distribution/licenses/securesm-1.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c0c6cf986ba0057390bfcc80c366a0e3157f944b
diff --git a/distribution/licenses/securesm-1.1.jar.sha1 b/distribution/licenses/securesm-1.1.jar.sha1
new file mode 100644
index 00000000000..9144a082b61
--- /dev/null
+++ b/distribution/licenses/securesm-1.1.jar.sha1
@@ -0,0 +1 @@
+1e423447d020041534be94c0f31a49fbdc1f2950
\ No newline at end of file
diff --git a/docs/reference/migration/migrate_5_0/packaging.asciidoc b/docs/reference/migration/migrate_5_0/packaging.asciidoc
index 977e20a76b1..74faf3bb7d9 100644
--- a/docs/reference/migration/migrate_5_0/packaging.asciidoc
+++ b/docs/reference/migration/migrate_5_0/packaging.asciidoc
@@ -55,3 +55,11 @@ from Elasticsearch.
 Additionally, it was previously possible to set any setting in
 Elasticsearch via JVM system properties. This has been removed from
 Elasticsearch.
+
+==== Dying on fatal errors
+
+Previous versions of Elasticsearch would not halt the JVM if out of memory errors or other fatal
+errors were encountered during the life of the Elasticsearch instance. Because such errors leave
+the JVM in a questionable state, the best course of action is to halt the JVM when this occurs.
+Starting in Elasticsearch 5.x, this is now the case. Operators should consider configuring their
+Elasticsearch services so that they respawn automatically in the case of such a fatal crash.
diff --git a/docs/reference/setup.asciidoc b/docs/reference/setup.asciidoc
index a883b0cc543..ae3c5b3beb4 100644
--- a/docs/reference/setup.asciidoc
+++ b/docs/reference/setup.asciidoc
@@ -47,3 +47,5 @@ include::setup/bootstrap-checks.asciidoc[]
 include::setup/sysconfig.asciidoc[]
 
 include::setup/upgrade.asciidoc[]
+
+include::setup/stopping.asciidoc[]
diff --git a/docs/reference/setup/stopping.asciidoc b/docs/reference/setup/stopping.asciidoc
new file mode 100644
index 00000000000..1aa281f050c
--- /dev/null
+++ b/docs/reference/setup/stopping.asciidoc
@@ -0,0 +1,58 @@
+[[stopping-elasticsearch]]
+=== Stopping Elasticsearch
+
+An orderly shutdown of Elasticsearch ensures that Elasticsearch has a chance to cleanup and close
+outstanding resources. For example, a node that is shutdown in an orderly fashion will remove itself
+from the cluster, sync translogs to disk, and perform other related cleanup activities. You can help
+ensure an orderly shutdown by properly stopping Elasticsearch.
+
+If you're running Elasticsearch as a service, you can stop Elasticsearch via the service management
+functionality provided by your installation.
+
+If you're running Elasticsearch directly, you can stop Elasticsearch by sending control-C if you're
+running Elasticsearch in the console, or by sending `SIGTERM` to the Elasticsearch process on a
+POSIX system. You can obtain the PID to send the signal to via various tools (e.g., `ps` or `jps`):
+
+[source,sh]
+--------------------------------------------------
+$ jps | grep Elasticsearch
+14542 Elasticsearch
+--------------------------------------------------
+
+From the Elasticsearch startup logs:
+
+[source,sh]
+--------------------------------------------------
+[2016-07-07 12:26:18,908][INFO ][node                     ] [Reaper] version[5.0.0-alpha4], pid[15399], build[3f5b994/2016-06-27T16:23:46.861Z], OS[Mac OS X/10.11.5/x86_64], JVM[Oracle Corporation/Java HotSpot(TM) 64-Bit Server VM/1.8.0_92/25.92-b14]
+--------------------------------------------------
+
+Or by specifying a location to write a PID file to on startup (`-p <path>`):
+
+[source,sh]
+--------------------------------------------------
+$ ./bin/elasticsearch -p /tmp/elasticsearch-pid -d
+$ cat /tmp/elasticsearch-pid && echo
+15516
+$ kill -SIGTERM 15516
+--------------------------------------------------
+
+[[fatal-errors]
+[float]
+=== Stopping on Fatal Errors
+
+During the life of the Elasticsearch virtual machine, certain fatal errors could arise that put the
+virtual machine in a questionable state. Such fatal errors include out of memory errors, internal
+errors in virtual machine, and serious I/O errors.
+
+When Elasticsearch detects that the virtual machine has encountered such a fatal error Elasticsearch
+will attempt to log the error and then will halt the virtual machine. When Elasticsearch initiates
+such a shutdown, it does not go through an orderly shutdown as described above. The Elasticsearch
+process will also return with a special status code indicating the nature of the error.
+
+[horizontal]
+JVM internal error:: 128
+Out of memory error:: 127
+Stack overflow error:: 126
+Unknown virtual machine error:: 125
+Serious I/O error:: 124
+Unknown fatal error:: 1
diff --git a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
index 5b79721948e..fe624297e72 100644
--- a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
+++ b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java
@@ -150,7 +150,7 @@ public class BootstrapForTesting {
                         return esPolicy.implies(domain, permission) || testFramework.implies(domain, permission);
                     }
                 });
-                System.setSecurityManager(new SecureSM(true));
+                System.setSecurityManager(SecureSM.createTestSecureSM());
                 Security.selfTest();
 
                 // guarantee plugin classes are initialized first, in case they have one-time hacks.

From 7d13906fcd44a4b3a9ea4deb51a35149777db5bf Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Thu, 7 Jul 2016 15:14:19 -0400
Subject: [PATCH 1277/1311] Add skip to failing ingest-user-agent REST test

This commit adds a skip to a failing ingest-user-agent REST test.
---
 .../rest-api-spec/test/ingest-useragent/30_custom_regex.yaml   | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/plugins/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml b/plugins/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml
index 5613145b664..ea0d8a45b81 100644
--- a/plugins/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml
+++ b/plugins/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml
@@ -1,5 +1,8 @@
 ---
 "Test user agent processor with custom regex file":
+  - skip:
+      version: "all"
+      reason: "awaits fix #19307"
   - do:
       ingest.put_pipeline:
         id: "my_pipeline"

From bf0c362412965857bd256a0a52e900e75e1c17f8 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Thu, 7 Jul 2016 21:29:40 +0200
Subject: [PATCH 1278/1311] ingest: Fixed left over rename to
 'ingest-user-plugin'

Closes #19307
---
 .../elasticsearch/ingest/useragent/IngestUserAgentPlugin.java  | 2 +-
 .../rest-api-spec/test/ingest-useragent/30_custom_regex.yaml   | 3 ---
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java b/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java
index 100183b4532..005f24079ab 100644
--- a/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java
+++ b/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java
@@ -50,7 +50,7 @@ public class IngestUserAgentPlugin extends Plugin implements IngestPlugin {
     @Override
     public Map<String, Processor.Factory> getProcessors(
         Environment env, ScriptService scriptService, TemplateService templateService) {
-        Path userAgentConfigDirectory = env.configFile().resolve("ingest-useragent");
+        Path userAgentConfigDirectory = env.configFile().resolve("ingest-user-agent");
 
         if (Files.exists(userAgentConfigDirectory) == false && Files.isDirectory(userAgentConfigDirectory)) {
             throw new IllegalStateException(
diff --git a/plugins/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml b/plugins/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml
index ea0d8a45b81..5613145b664 100644
--- a/plugins/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml
+++ b/plugins/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml
@@ -1,8 +1,5 @@
 ---
 "Test user agent processor with custom regex file":
-  - skip:
-      version: "all"
-      reason: "awaits fix #19307"
   - do:
       ingest.put_pipeline:
         id: "my_pipeline"

From 593f8bdf0c93a248aa76a89a8703572a0e828bbc Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Thu, 7 Jul 2016 12:54:10 -0700
Subject: [PATCH 1279/1311] Rename repository api methods for clarity and tweak
 documentation.

---
 .../index/shard/StoreRecovery.java            |  2 +-
 .../repositories/Repository.java              | 43 ++++++++++---------
 .../blobstore/BlobStoreRepository.java        | 38 ++++++++--------
 .../repositories/uri/URLRepository.java       |  2 +-
 .../snapshots/RestoreService.java             |  6 +--
 .../snapshots/SnapshotShardsService.java      |  2 +-
 .../snapshots/SnapshotsService.java           | 12 +++---
 .../index/shard/IndexShardTests.java          | 18 ++++----
 .../blobstore/BlobStoreRepositoryTests.java   | 12 +++---
 .../SharedClusterSnapshotRestoreIT.java       |  4 +-
 .../repositories/azure/AzureRepository.java   |  2 +-
 11 files changed, 72 insertions(+), 69 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
index c1016f0f4e3..ff2d1d298a5 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
@@ -397,7 +397,7 @@ final class StoreRecovery {
             if (!shardId.getIndexName().equals(restoreSource.index())) {
                 snapshotShardId = new ShardId(restoreSource.index(), IndexMetaData.INDEX_UUID_NA_VALUE, shardId.id());
             }
-            repository.restore(indexShard, restoreSource.snapshot().getSnapshotId(), restoreSource.version(), snapshotShardId, indexShard.recoveryState());
+            repository.restoreShard(indexShard, restoreSource.snapshot().getSnapshotId(), restoreSource.version(), snapshotShardId, indexShard.recoveryState());
             indexShard.skipTranslogRecovery();
             indexShard.finalizeRecovery();
             indexShard.postRecovery("restore done");
diff --git a/core/src/main/java/org/elasticsearch/repositories/Repository.java b/core/src/main/java/org/elasticsearch/repositories/Repository.java
index 6c4cf93a7a4..2230f046eba 100644
--- a/core/src/main/java/org/elasticsearch/repositories/Repository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/Repository.java
@@ -35,15 +35,16 @@ import java.io.IOException;
 import java.util.List;
 
 /**
- * Snapshot repository interface.
+ * An interface for interacting with a repository in snapshot and restore.
  * <p>
- * Responsible for index and cluster and shard level operations.
+ * Implementations are responsible for reading and writing both metadata and actual shard data to and from
+ * a repository backend.
  * <p>
- * Typical snapshot usage pattern:
+ * To perform a snapshot:
  * <ul>
  * <li>Master calls {@link #initializeSnapshot(SnapshotId, List, org.elasticsearch.cluster.metadata.MetaData)}
  * with list of indices that will be included into the snapshot</li>
- * <li>Data nodes call {@link Repository#snapshot(IndexShard, SnapshotId, IndexCommit, IndexShardSnapshotStatus)}
+ * <li>Data nodes call {@link Repository#snapshotShard(IndexShard, SnapshotId, IndexCommit, IndexShardSnapshotStatus)}
  * for each shard</li>
  * <li>When all shard calls return master calls {@link #finalizeSnapshot} with possible list of failures</li>
  * </ul>
@@ -56,7 +57,7 @@ public interface Repository extends LifecycleComponent {
      * @param snapshotId  snapshot id
      * @return information about snapshot
      */
-    SnapshotInfo readSnapshot(SnapshotId snapshotId);
+    SnapshotInfo getSnapshotInfo(SnapshotId snapshotId);
 
     /**
      * Returns global metadata associate with the snapshot.
@@ -67,7 +68,7 @@ public interface Repository extends LifecycleComponent {
      * @param indices    list of indices
      * @return information about snapshot
      */
-    MetaData readSnapshotMetaData(SnapshotInfo snapshot, List<String> indices) throws IOException;
+    MetaData getSnapshotMetaData(SnapshotInfo snapshot, List<String> indices) throws IOException;
 
     /**
      * Returns the list of snapshots currently stored in the repository that match the given predicate on the snapshot name.
@@ -75,7 +76,7 @@ public interface Repository extends LifecycleComponent {
      *
      * @return snapshot list
      */
-    List<SnapshotId> snapshots();
+    List<SnapshotId> getSnapshots();
 
     /**
      * Starts snapshotting process
@@ -109,12 +110,12 @@ public interface Repository extends LifecycleComponent {
     /**
      * Returns snapshot throttle time in nanoseconds
      */
-    long snapshotThrottleTimeInNanos();
+    long getSnapshotThrottleTimeInNanos();
 
     /**
      * Returns restore throttle time in nanoseconds
      */
-    long restoreThrottleTimeInNanos();
+    long getRestoreThrottleTimeInNanos();
 
 
     /**
@@ -136,11 +137,18 @@ public interface Repository extends LifecycleComponent {
      */
     void endVerification(String verificationToken);
 
+    /**
+     * Verifies repository settings on data node.
+     * @param verificationToken value returned by {@link org.elasticsearch.repositories.Repository#startVerification()}
+     * @param localNode         the local node information, for inclusion in verification errors
+     */
+    void verify(String verificationToken, DiscoveryNode localNode);
+
     /**
      * Returns true if the repository supports only read operations
      * @return true if the repository is read/only
      */
-    boolean readOnly();
+    boolean isReadOnly();
 
     /**
      * Creates a snapshot of the shard based on the index commit point.
@@ -156,7 +164,7 @@ public interface Repository extends LifecycleComponent {
      * @param snapshotIndexCommit commit point
      * @param snapshotStatus      snapshot status
      */
-    void snapshot(IndexShard shard, SnapshotId snapshotId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus);
+    void snapshotShard(IndexShard shard, SnapshotId snapshotId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus);
 
     /**
      * Restores snapshot of the shard.
@@ -169,22 +177,17 @@ public interface Repository extends LifecycleComponent {
      * @param snapshotShardId shard id (in the snapshot)
      * @param recoveryState   recovery state
      */
-    void restore(IndexShard shard, SnapshotId snapshotId, Version version, ShardId snapshotShardId, RecoveryState recoveryState);
+    void restoreShard(IndexShard shard, SnapshotId snapshotId, Version version, ShardId snapshotShardId, RecoveryState recoveryState);
 
     /**
      * Retrieve shard snapshot status for the stored snapshot
      *
      * @param snapshotId snapshot id
-     * @param version   version of elasticsearch that created this snapshot
+     * @param version    version of elasticsearch that created this snapshot
      * @param shardId    shard id
      * @return snapshot status
      */
-    IndexShardSnapshotStatus snapshotStatus(SnapshotId snapshotId, Version version, ShardId shardId);
+    IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, Version version, ShardId shardId);
+
 
-    /**
-     * Verifies repository settings on data node.
-     * @param verificationToken value returned by {@link org.elasticsearch.repositories.Repository#startVerification()}
-     * @param localNode         the local node information, for inclusion in verification errors
-     */
-    void verify(String verificationToken, DiscoveryNode localNode);
 }
diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index 4c328872742..173937ce69c 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -300,13 +300,13 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     @Override
     public void initializeSnapshot(SnapshotId snapshotId, List<String> indices, MetaData metaData) {
-        if (readOnly()) {
+        if (isReadOnly()) {
             throw new RepositoryException(this.repositoryName, "cannot create snapshot in a readonly repository");
         }
         try {
             final String snapshotName = snapshotId.getName();
             // check if the snapshot name already exists in the repository
-            if (snapshots().stream().anyMatch(s -> s.getName().equals(snapshotName))) {
+            if (getSnapshots().stream().anyMatch(s -> s.getName().equals(snapshotName))) {
                 throw new SnapshotCreationException(repositoryName, snapshotId, "snapshot with the same name already exists");
             }
             if (snapshotFormat.exists(snapshotsBlobContainer, blobId(snapshotId)) ||
@@ -328,13 +328,13 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     @Override
     public void deleteSnapshot(SnapshotId snapshotId) {
-        if (readOnly()) {
+        if (isReadOnly()) {
             throw new RepositoryException(this.repositoryName, "cannot delete snapshot from a readonly repository");
         }
         List<String> indices = Collections.emptyList();
         SnapshotInfo snapshot = null;
         try {
-            snapshot = readSnapshot(snapshotId);
+            snapshot = getSnapshotInfo(snapshotId);
             indices = snapshot.indices();
         } catch (SnapshotMissingException ex) {
             throw ex;
@@ -365,7 +365,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
                 globalMetaDataFormat.delete(snapshotsBlobContainer, snapshotName);
             }
             // Delete snapshot from the snapshot list
-            List<SnapshotId> snapshotIds = snapshots().stream().filter(id -> snapshotId.equals(id) == false).collect(Collectors.toList());
+            List<SnapshotId> snapshotIds = getSnapshots().stream().filter(id -> snapshotId.equals(id) == false).collect(Collectors.toList());
             writeSnapshotsToIndexGen(snapshotIds);
 
             // Now delete all indices
@@ -411,7 +411,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
                                                               totalShards,
                                                               shardFailures);
             snapshotFormat.write(blobStoreSnapshot, snapshotsBlobContainer, blobId(snapshotId));
-            List<SnapshotId> snapshotIds = snapshots();
+            List<SnapshotId> snapshotIds = getSnapshots();
             if (!snapshotIds.contains(snapshotId)) {
                 snapshotIds = new ArrayList<>(snapshotIds);
                 snapshotIds.add(snapshotId);
@@ -425,7 +425,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
     }
 
     @Override
-    public List<SnapshotId> snapshots() {
+    public List<SnapshotId> getSnapshots() {
         try {
             return Collections.unmodifiableList(readSnapshotsFromIndex());
         } catch (NoSuchFileException | FileNotFoundException e) {
@@ -437,12 +437,12 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
     }
 
     @Override
-    public MetaData readSnapshotMetaData(SnapshotInfo snapshot, List<String> indices) throws IOException {
+    public MetaData getSnapshotMetaData(SnapshotInfo snapshot, List<String> indices) throws IOException {
         return readSnapshotMetaData(snapshot.snapshotId(), snapshot.version(), indices, false);
     }
 
     @Override
-    public SnapshotInfo readSnapshot(final SnapshotId snapshotId) {
+    public SnapshotInfo getSnapshotInfo(final SnapshotId snapshotId) {
         try {
             return snapshotFormat.read(snapshotsBlobContainer, blobId(snapshotId));
         } catch (FileNotFoundException | NoSuchFileException ex) {
@@ -561,19 +561,19 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
     private static final String UUID = "uuid";
 
     @Override
-    public long snapshotThrottleTimeInNanos() {
+    public long getSnapshotThrottleTimeInNanos() {
         return snapshotRateLimitingTimeInNanos.count();
     }
 
     @Override
-    public long restoreThrottleTimeInNanos() {
+    public long getRestoreThrottleTimeInNanos() {
         return restoreRateLimitingTimeInNanos.count();
     }
 
     @Override
     public String startVerification() {
         try {
-            if (readOnly()) {
+            if (isReadOnly()) {
                 // It's readonly - so there is not much we can do here to verify it
                 return null;
             } else {
@@ -593,7 +593,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     @Override
     public void endVerification(String seed) {
-        if (readOnly()) {
+        if (isReadOnly()) {
             throw new UnsupportedOperationException("shouldn't be called");
         }
         try {
@@ -608,7 +608,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
     }
 
     @Override
-    public boolean readOnly() {
+    public boolean isReadOnly() {
         return readOnly;
     }
 
@@ -618,7 +618,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
     }
 
     protected void writeSnapshotsToIndexGen(final List<SnapshotId> snapshots) throws IOException {
-        assert readOnly() == false; // can not write to a read only repository
+        assert isReadOnly() == false; // can not write to a read only repository
         final BytesReference snapshotsBytes;
         try (BytesStreamOutput bStream = new BytesStreamOutput()) {
             try (StreamOutput stream = new OutputStreamStreamOutput(bStream)) {
@@ -641,7 +641,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         // write the index file
         writeAtomic(SNAPSHOTS_FILE_PREFIX + Long.toString(gen), snapshotsBytes);
         // delete the N-2 index file if it exists, keep the previous one around as a backup
-        if (readOnly() == false && gen - 2 >= 0) {
+        if (isReadOnly() == false && gen - 2 >= 0) {
             final String oldSnapshotIndexFile = SNAPSHOTS_FILE_PREFIX + Long.toString(gen - 2);
             if (snapshotsBlobContainer.blobExists(oldSnapshotIndexFile)) {
                 snapshotsBlobContainer.deleteBlob(oldSnapshotIndexFile);
@@ -797,7 +797,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
     }
 
     @Override
-    public void snapshot(IndexShard shard, SnapshotId snapshotId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus) {
+    public void snapshotShard(IndexShard shard, SnapshotId snapshotId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus) {
         SnapshotContext snapshotContext = new SnapshotContext(shard, snapshotId, snapshotStatus);
         snapshotStatus.startTime(System.currentTimeMillis());
 
@@ -818,7 +818,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
     }
 
     @Override
-    public void restore(IndexShard shard, SnapshotId snapshotId, Version version, ShardId snapshotShardId, RecoveryState recoveryState) {
+    public void restoreShard(IndexShard shard, SnapshotId snapshotId, Version version, ShardId snapshotShardId, RecoveryState recoveryState) {
         final RestoreContext snapshotContext = new RestoreContext(shard, snapshotId, version, snapshotShardId, recoveryState);
         try {
             snapshotContext.restore();
@@ -828,7 +828,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
     }
 
     @Override
-    public IndexShardSnapshotStatus snapshotStatus(SnapshotId snapshotId, Version version, ShardId shardId) {
+    public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, Version version, ShardId shardId) {
         Context context = new Context(snapshotId, version, shardId);
         BlobStoreIndexShardSnapshot snapshot = context.loadSnapshot();
         IndexShardSnapshotStatus status = new IndexShardSnapshotStatus();
diff --git a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java b/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
index f08faa2c0d6..4058baa63ef 100644
--- a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
@@ -141,7 +141,7 @@ public class URLRepository extends BlobStoreRepository {
     }
 
     @Override
-    public boolean readOnly() {
+    public boolean isReadOnly() {
         return true;
     }
 
diff --git a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
index 67a5ae361bb..dedcc6d8d21 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/RestoreService.java
@@ -185,16 +185,16 @@ public class RestoreService extends AbstractComponent implements ClusterStateLis
         try {
             // Read snapshot info and metadata from the repository
             Repository repository = repositoriesService.repository(request.repositoryName);
-            final Optional<SnapshotId> matchingSnapshotId = repository.snapshots().stream()
+            final Optional<SnapshotId> matchingSnapshotId = repository.getSnapshots().stream()
                 .filter(s -> request.snapshotName.equals(s.getName())).findFirst();
             if (matchingSnapshotId.isPresent() == false) {
                 throw new SnapshotRestoreException(request.repositoryName, request.snapshotName, "snapshot does not exist");
             }
             final SnapshotId snapshotId = matchingSnapshotId.get();
-            final SnapshotInfo snapshotInfo = repository.readSnapshot(snapshotId);
+            final SnapshotInfo snapshotInfo = repository.getSnapshotInfo(snapshotId);
             final Snapshot snapshot = new Snapshot(request.repositoryName, snapshotId);
             List<String> filteredIndices = SnapshotUtils.filterIndices(snapshotInfo.indices(), request.indices(), request.indicesOptions());
-            MetaData metaDataIn = repository.readSnapshotMetaData(snapshotInfo, filteredIndices);
+            MetaData metaDataIn = repository.getSnapshotMetaData(snapshotInfo, filteredIndices);
 
             final MetaData metaData;
             if (snapshotInfo.version().before(Version.V_2_0_0_beta1)) {
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
index b24627bb634..7741ef1c0e6 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
@@ -340,7 +340,7 @@ public class SnapshotShardsService extends AbstractLifecycleComponent implements
             // we flush first to make sure we get the latest writes snapshotted
             IndexCommit snapshotIndexCommit = indexShard.snapshotIndex(true);
             try {
-                repository.snapshot(indexShard, snapshot.getSnapshotId(), snapshotIndexCommit, snapshotStatus);
+                repository.snapshotShard(indexShard, snapshot.getSnapshotId(), snapshotIndexCommit, snapshotStatus);
                 if (logger.isDebugEnabled()) {
                     StringBuilder sb = new StringBuilder();
                     sb.append("    index    : version [").append(snapshotStatus.indexVersion()).append("], number_of_files [").append(snapshotStatus.numberOfFiles()).append("] with total_size [").append(new ByteSizeValue(snapshotStatus.totalSize())).append("]\n");
diff --git a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
index 8d2407dd2e4..df56f2a24a6 100644
--- a/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
+++ b/core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
@@ -132,7 +132,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
     public List<SnapshotId> snapshotIds(final String repositoryName) {
         Repository repository = repositoriesService.repository(repositoryName);
         assert repository != null; // should only be called once we've validated the repository exists
-        return repository.snapshots();
+        return repository.getSnapshots();
     }
 
     /**
@@ -148,7 +148,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
         if (!entries.isEmpty()) {
             return inProgressSnapshot(entries.iterator().next());
         }
-        return repositoriesService.repository(repositoryName).readSnapshot(snapshotId);
+        return repositoriesService.repository(repositoryName).getSnapshotInfo(snapshotId);
     }
 
     /**
@@ -174,7 +174,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
         final Repository repository = repositoriesService.repository(repositoryName);
         for (SnapshotId snapshotId : snapshotIdsToIterate) {
             try {
-                snapshotSet.add(repository.readSnapshot(snapshotId));
+                snapshotSet.add(repository.getSnapshotInfo(snapshotId));
             } catch (Exception ex) {
                 if (ignoreUnavailable) {
                     logger.warn("failed to get snapshot [{}]", ex, snapshotId);
@@ -546,7 +546,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                                                                  final SnapshotInfo snapshotInfo) throws IOException {
         Map<ShardId, IndexShardSnapshotStatus> shardStatus = new HashMap<>();
         Repository repository = repositoriesService.repository(repositoryName);
-        MetaData metaData = repository.readSnapshotMetaData(snapshotInfo, snapshotInfo.indices());
+        MetaData metaData = repository.getSnapshotMetaData(snapshotInfo, snapshotInfo.indices());
         for (String index : snapshotInfo.indices()) {
             IndexMetaData indexMetaData = metaData.indices().get(index);
             if (indexMetaData != null) {
@@ -561,7 +561,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
                         shardStatus.put(shardId, shardSnapshotStatus);
                     } else {
                         IndexShardSnapshotStatus shardSnapshotStatus =
-                            repository.snapshotStatus(snapshotInfo.snapshotId(), snapshotInfo.version(), shardId);
+                            repository.getShardSnapshotStatus(snapshotInfo.snapshotId(), snapshotInfo.version(), shardId);
                         shardStatus.put(shardId, shardSnapshotStatus);
                     }
                 }
@@ -953,7 +953,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
     public void deleteSnapshot(final String repositoryName, final String snapshotName, final DeleteSnapshotListener listener) {
         // First, look for the snapshot in the repository
         final Repository repository = repositoriesService.repository(repositoryName);
-        Optional<SnapshotId> matchedEntry = repository.snapshots().stream().filter(s -> s.getName().equals(snapshotName)).findFirst();
+        Optional<SnapshotId> matchedEntry = repository.getSnapshots().stream().filter(s -> s.getName().equals(snapshotName)).findFirst();
         // if nothing found by the same name, then look in the cluster state for current in progress snapshots
         if (matchedEntry.isPresent() == false) {
             matchedEntry = currentSnapshots(repositoryName, Collections.emptyList()).stream()
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index acec53da24b..e51af79df99 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -1184,7 +1184,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         test_target_shard.markAsRecovering("store", new RecoveryState(routing.shardId(), routing.primary(), RecoveryState.Type.SNAPSHOT, routing.restoreSource(), localNode));
         assertTrue(test_target_shard.restoreFromRepository(new RestoreOnlyRepository() {
             @Override
-            public void restore(IndexShard shard, SnapshotId snapshotId, Version version, ShardId snapshotShardId, RecoveryState recoveryState) {
+            public void restoreShard(IndexShard shard, SnapshotId snapshotId, Version version, ShardId snapshotShardId, RecoveryState recoveryState) {
                 try {
                     cleanLuceneIndex(targetStore.directory());
                     for (String file : sourceStore.directory().listAll()) {
@@ -1653,15 +1653,15 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         @Override
         protected void doClose() {}
         @Override
-        public SnapshotInfo readSnapshot(SnapshotId snapshotId) {
+        public SnapshotInfo getSnapshotInfo(SnapshotId snapshotId) {
             return null;
         }
         @Override
-        public MetaData readSnapshotMetaData(SnapshotInfo snapshot, List<String> indices) throws IOException {
+        public MetaData getSnapshotMetaData(SnapshotInfo snapshot, List<String> indices) throws IOException {
             return null;
         }
         @Override
-        public List<SnapshotId> snapshots() {
+        public List<SnapshotId> getSnapshots() {
             return null;
         }
         @Override
@@ -1673,11 +1673,11 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         @Override
         public void deleteSnapshot(SnapshotId snapshotId) {}
         @Override
-        public long snapshotThrottleTimeInNanos() {
+        public long getSnapshotThrottleTimeInNanos() {
             return 0;
         }
         @Override
-        public long restoreThrottleTimeInNanos() {
+        public long getRestoreThrottleTimeInNanos() {
             return 0;
         }
         @Override
@@ -1687,13 +1687,13 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         @Override
         public void endVerification(String verificationToken) {}
         @Override
-        public boolean readOnly() {
+        public boolean isReadOnly() {
             return false;
         }
         @Override
-        public void snapshot(IndexShard shard, SnapshotId snapshotId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus) {}
+        public void snapshotShard(IndexShard shard, SnapshotId snapshotId, IndexCommit snapshotIndexCommit, IndexShardSnapshotStatus snapshotStatus) {}
         @Override
-        public IndexShardSnapshotStatus snapshotStatus(SnapshotId snapshotId, Version version, ShardId shardId) {
+        public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, Version version, ShardId shardId) {
             return null;
         }
         @Override
diff --git a/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java b/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
index 17e2481b7d0..c9d8ff81aa2 100644
--- a/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
+++ b/core/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java
@@ -100,7 +100,7 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
             (BlobStoreRepository) repositoriesService.repository(repositoryName);
         final List<SnapshotId> originalSnapshots = Arrays.asList(snapshotId1, snapshotId2);
 
-        List<SnapshotId> snapshotIds = repository.snapshots().stream()
+        List<SnapshotId> snapshotIds = repository.getSnapshots().stream()
                                                              .sorted((s1, s2) -> s1.getName().compareTo(s2.getName()))
                                                              .collect(Collectors.toList());
         assertThat(snapshotIds, equalTo(originalSnapshots));
@@ -110,9 +110,9 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
         final BlobStoreRepository repository = setupRepo();
 
         // write to and read from a snapshot file with no entries
-        assertThat(repository.snapshots().size(), equalTo(0));
+        assertThat(repository.getSnapshots().size(), equalTo(0));
         repository.writeSnapshotsToIndexGen(Collections.emptyList());
-        assertThat(repository.snapshots().size(), equalTo(0));
+        assertThat(repository.getSnapshots().size(), equalTo(0));
 
         // write to and read from a snapshot file with a random number of entries
         final int numSnapshots = randomIntBetween(1, 1000);
@@ -121,7 +121,7 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
             snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
         }
         repository.writeSnapshotsToIndexGen(snapshotIds);
-        assertThat(repository.snapshots(), equalTo(snapshotIds));
+        assertThat(repository.getSnapshots(), equalTo(snapshotIds));
     }
 
     public void testIndexGenerationalFiles() throws Exception {
@@ -165,7 +165,7 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
             snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), SnapshotId.UNASSIGNED_UUID));
         }
         writeOldFormat(repository, snapshotIds.stream().map(SnapshotId::getName).collect(Collectors.toList()));
-        assertThat(Sets.newHashSet(repository.snapshots()), equalTo(Sets.newHashSet(snapshotIds)));
+        assertThat(Sets.newHashSet(repository.getSnapshots()), equalTo(Sets.newHashSet(snapshotIds)));
 
         // write to and read from a snapshot file with a random number of new entries added
         final int numSnapshots = randomIntBetween(1, 1000);
@@ -173,7 +173,7 @@ public class BlobStoreRepositoryTests extends ESSingleNodeTestCase {
             snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
         }
         repository.writeSnapshotsToIndexGen(snapshotIds);
-        assertThat(Sets.newHashSet(repository.snapshots()), equalTo(Sets.newHashSet(snapshotIds)));
+        assertThat(Sets.newHashSet(repository.getSnapshots()), equalTo(Sets.newHashSet(snapshotIds)));
     }
 
     public void testBlobId() {
diff --git a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
index f6049002852..fb52d0f634b 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
@@ -1397,8 +1397,8 @@ public class SharedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTestCas
         long snapshotPause = 0L;
         long restorePause = 0L;
         for (RepositoriesService repositoriesService : internalCluster().getDataNodeInstances(RepositoriesService.class)) {
-            snapshotPause += repositoriesService.repository("test-repo").snapshotThrottleTimeInNanos();
-            restorePause += repositoriesService.repository("test-repo").restoreThrottleTimeInNanos();
+            snapshotPause += repositoriesService.repository("test-repo").getSnapshotThrottleTimeInNanos();
+            restorePause += repositoriesService.repository("test-repo").getRestoreThrottleTimeInNanos();
         }
 
         if (throttleSnapshot) {
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
index 47ec4128c7c..248c87ed604 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
@@ -185,7 +185,7 @@ public class AzureRepository extends BlobStoreRepository {
     }
 
     @Override
-    public boolean readOnly() {
+    public boolean isReadOnly() {
         return readonly;
     }
 }

From d83e1cccacfa0ceb4d23f7c0a86e289b7e110a84 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 7 Jul 2016 16:40:11 -0400
Subject: [PATCH 1280/1311] Fix checkstyle in test

---
 .../java/org/elasticsearch/index/shard/IndexShardTests.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index e51af79df99..5bc84921a29 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -1642,7 +1642,7 @@ public class IndexShardTests extends ESSingleNodeTestCase {
     }
 
     /** A dummy repository for testing which just needs restore overridden */
-    private static abstract class RestoreOnlyRepository extends AbstractLifecycleComponent implements Repository {
+    private abstract static class RestoreOnlyRepository extends AbstractLifecycleComponent implements Repository {
         public RestoreOnlyRepository() {
             super(Settings.EMPTY);
         }

From 7da753a4d78bcfd682d96065d185f574b9b5d7a5 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Mon, 4 Jul 2016 22:49:24 -0400
Subject: [PATCH 1281/1311] Migrate sampler and missing aggregations to
 NamedWriteable

This is another step down the path to removing aggregation's special
"streams" which reimplement NamedWriteable.
---
 .../elasticsearch/search/SearchModule.java    | 12 +++----
 .../bucket/missing/InternalMissing.java       | 35 +++++--------------
 .../missing/MissingAggregationBuilder.java    |  8 +++--
 .../bucket/sampler/InternalSampler.java       | 35 ++++++-------------
 .../sampler/SamplerAggregationBuilder.java    |  8 +++--
 .../bucket/sampler/UnmappedSampler.java       | 35 ++++++-------------
 .../search/aggregations/bucket/MissingIT.java |  3 --
 7 files changed, 45 insertions(+), 91 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java
index 369750a214b..dd144e413a2 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -551,13 +551,16 @@ public class SearchModule extends AbstractModule {
                 CardinalityAggregationBuilder.AGGREGATION_NAME_FIELD).addResultReader(InternalCardinality::new));
         registerAggregation(new AggregationSpec(GlobalAggregationBuilder::new, GlobalAggregationBuilder::parse,
                 GlobalAggregationBuilder.AGGREGATION_NAME_FIELD).addResultReader(InternalGlobal::new));
-        registerAggregation(MissingAggregationBuilder::new, new MissingParser(), MissingAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(
+                new AggregationSpec(MissingAggregationBuilder::new, new MissingParser(), MissingAggregationBuilder.AGGREGATION_NAME_FIELD)
+                        .addResultReader(InternalMissing::new));
         registerAggregation(new AggregationSpec(FilterAggregationBuilder::new, FilterAggregationBuilder::parse,
                 FilterAggregationBuilder.AGGREGATION_NAME_FIELD).addResultReader(InternalFilter::new));
         registerAggregation(new AggregationSpec(FiltersAggregationBuilder::new, FiltersAggregationBuilder::parse,
                 FiltersAggregationBuilder.AGGREGATION_NAME_FIELD).addResultReader(InternalFilters::new));
-        registerAggregation(SamplerAggregationBuilder::new, SamplerAggregationBuilder::parse,
-                SamplerAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(new AggregationSpec(SamplerAggregationBuilder::new, SamplerAggregationBuilder::parse,
+                SamplerAggregationBuilder.AGGREGATION_NAME_FIELD).addResultReader(InternalSampler.NAME, InternalSampler::new)
+                        .addResultReader(UnmappedSampler.NAME, UnmappedSampler::new));
         registerAggregation(DiversifiedAggregationBuilder::new, new DiversifiedSamplerParser(),
                 DiversifiedAggregationBuilder.AGGREGATION_NAME_FIELD);
         registerAggregation(TermsAggregationBuilder::new, new TermsParser(), TermsAggregationBuilder.AGGREGATION_NAME_FIELD);
@@ -767,9 +770,6 @@ public class SearchModule extends AbstractModule {
 
     static {
         // buckets
-        InternalSampler.registerStreams();
-        UnmappedSampler.registerStreams();
-        InternalMissing.registerStreams();
         StringTerms.registerStreams();
         LongTerms.registerStreams();
         SignificantStringTerms.registerStreams();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/InternalMissing.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/InternalMissing.java
index 78293dd5832..5a46e4ce95f 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/InternalMissing.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/InternalMissing.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.search.aggregations.bucket.missing;
 
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -28,37 +27,21 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
-/**
- *
- */
 public class InternalMissing extends InternalSingleBucketAggregation implements Missing {
-
-    public static final Type TYPE = new Type("missing");
-
-    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public InternalMissing readResult(StreamInput in) throws IOException {
-            InternalMissing missing = new InternalMissing();
-            missing.readFrom(in);
-            return missing;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
-
-    InternalMissing() {
-    }
-
     InternalMissing(String name, long docCount, InternalAggregations aggregations, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
         super(name, docCount, aggregations, pipelineAggregators, metaData);
     }
 
+    /**
+     * Read from a stream.
+     */
+    public InternalMissing(StreamInput in) throws IOException {
+        super(in);
+    }
+
     @Override
-    public Type type() {
-        return TYPE;
+    public String getWriteableName() {
+        return MissingAggregationBuilder.NAME;
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java
index f10f7683841..05868f96df2 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/missing/MissingAggregationBuilder.java
@@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 import org.elasticsearch.search.aggregations.support.ValueType;
@@ -36,18 +37,19 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType;
 import java.io.IOException;
 
 public class MissingAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource, MissingAggregationBuilder> {
-    public static final String NAME = InternalMissing.TYPE.name();
+    public static final String NAME = "missing";
+    public static final Type TYPE = new Type(NAME);
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     public MissingAggregationBuilder(String name, ValueType targetValueType) {
-        super(name, InternalMissing.TYPE, ValuesSourceType.ANY, targetValueType);
+        super(name, TYPE, ValuesSourceType.ANY, targetValueType);
     }
 
     /**
      * Read from a stream.
      */
     public MissingAggregationBuilder(StreamInput in) throws IOException {
-        super(in, InternalMissing.TYPE, ValuesSourceType.ANY);
+        super(in, TYPE, ValuesSourceType.ANY);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java
index c7bbeba38eb..224de07e93d 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/InternalSampler.java
@@ -19,7 +19,6 @@
 package org.elasticsearch.search.aggregations.bucket.sampler;
 
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.bucket.InternalSingleBucketAggregation;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -28,36 +27,22 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
-/**
-*
-*/
 public class InternalSampler extends InternalSingleBucketAggregation implements Sampler {
-
-    public static final Type TYPE = new Type("sampler");
-
-    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public InternalSampler readResult(StreamInput in) throws IOException {
-            InternalSampler result = new InternalSampler();
-            result.readFrom(in);
-            return result;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
-    InternalSampler() {
-    } // for serialization
-
+    public static final String NAME = "mapped_sampler";
     InternalSampler(String name, long docCount, InternalAggregations subAggregations, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
         super(name, docCount, subAggregations, pipelineAggregators, metaData);
     }
 
+    /**
+     * Read from a stream.
+     */
+    public InternalSampler(StreamInput in) throws IOException {
+        super(in);
+    }
+
     @Override
-    public Type type() {
-        return TYPE;
+    public String getWriteableName() {
+        return NAME;
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java
index bdbff785043..1f34d4987e5 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/SamplerAggregationBuilder.java
@@ -29,13 +29,15 @@ import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 
 import java.io.IOException;
 import java.util.Objects;
 
 public class SamplerAggregationBuilder extends AbstractAggregationBuilder<SamplerAggregationBuilder> {
-    public static final String NAME = InternalSampler.TYPE.name();
+    public static final String NAME = "sampler";
+    private static final Type TYPE = new Type(NAME);
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     public static final int DEFAULT_SHARD_SAMPLE_SIZE = 100;
@@ -43,14 +45,14 @@ public class SamplerAggregationBuilder extends AbstractAggregationBuilder<Sample
     private int shardSize = DEFAULT_SHARD_SAMPLE_SIZE;
 
     public SamplerAggregationBuilder(String name) {
-        super(name, InternalSampler.TYPE);
+        super(name, TYPE);
     }
 
     /**
      * Read from a stream.
      */
     public SamplerAggregationBuilder(StreamInput in) throws IOException {
-        super(in, InternalSampler.TYPE);
+        super(in, TYPE);
         shardSize = in.readVInt();
     }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/UnmappedSampler.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/UnmappedSampler.java
index f35f6b4c462..1cf77baa1f0 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/UnmappedSampler.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/UnmappedSampler.java
@@ -20,7 +20,6 @@ package org.elasticsearch.search.aggregations.bucket.sampler;
 
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -29,37 +28,23 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
-/**
- *
- */
 public class UnmappedSampler extends InternalSampler {
-
-    public static final Type TYPE = new Type("sampler", "umsampler");
-
-
-    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public UnmappedSampler readResult(StreamInput in) throws IOException {
-            UnmappedSampler sampler = new UnmappedSampler();
-            sampler.readFrom(in);
-            return sampler;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
-    UnmappedSampler() {
-    }
+    public static final String NAME = "unmapped_sampler";
 
     public UnmappedSampler(String name, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
         super(name, 0, InternalAggregations.EMPTY, pipelineAggregators, metaData);
     }
 
+    /**
+     * Read from a stream.
+     */
+    public UnmappedSampler(StreamInput in) throws IOException {
+        super(in);
+    }
+
     @Override
-    public Type type() {
-        return TYPE;
+    public String getWriteableName() {
+        return NAME;
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java
index c1baf8b3678..ade53a2164a 100644
--- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java
+++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/MissingIT.java
@@ -40,9 +40,6 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.core.IsNull.notNullValue;
 
-/**
- *
- */
 @ESIntegTestCase.SuiteScopeTestCase
 public class MissingIT extends ESIntegTestCase {
 

From fe0f28965a47c29d821964f0594b6b8dabcb2c02 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Tue, 5 Jul 2016 16:58:05 -0400
Subject: [PATCH 1282/1311] Clean up serialization of terms aggregation results

Move to NamedWriteable and remove a lot of duplication.
---
 .../resources/checkstyle_suppressions.xml     |   5 -
 .../elasticsearch/search/SearchModule.java    |  11 +-
 .../aggregations/InternalAggregation.java     |   6 +-
 .../terms/AbstractStringTermsAggregator.java  |   8 +-
 .../bucket/terms/DoubleTerms.java             | 144 +++++---------
 .../bucket/terms/DoubleTermsAggregator.java   |  24 +--
 .../GlobalOrdinalsStringTermsAggregator.java  |  21 +--
 .../bucket/terms/InternalMappedTerms.java     | 113 +++++++++++
 .../bucket/terms/InternalTerms.java           | 175 ++++++++++--------
 .../aggregations/bucket/terms/LongTerms.java  | 139 ++++----------
 .../bucket/terms/LongTermsAggregator.java     |  24 +--
 .../bucket/terms/StringTerms.java             | 144 +++++---------
 .../bucket/terms/StringTermsAggregator.java   |  12 +-
 .../bucket/terms/TermsAggregationBuilder.java |   9 +-
 .../bucket/terms/TermsAggregatorFactory.java  |   4 +-
 .../bucket/terms/UnmappedTerms.java           | 113 ++++++-----
 .../terms/support/BucketPriorityQueue.java    |   8 +-
 .../messy/tests/DoubleTermsTests.java         |  27 +++
 .../messy/tests/LongTermsTests.java           |  27 +++
 19 files changed, 519 insertions(+), 495 deletions(-)
 create mode 100644 core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalMappedTerms.java

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index f85f998df5b..882d12a305f 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -599,19 +599,14 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]heuristics[/\\]ScriptHeuristic.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]significant[/\\]heuristics[/\\]SignificanceHeuristic.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]AbstractTermsParametersParser.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]DoubleTerms.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]DoubleTermsAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]GlobalOrdinalsStringTermsAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]InternalOrder.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]InternalTerms.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]LongTerms.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]LongTermsAggregator.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]StringTerms.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]StringTermsAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]TermsAggregator.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]TermsAggregatorFactory.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]TermsParametersParser.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]UnmappedTerms.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]bucket[/\\]terms[/\\]support[/\\]IncludeExclude.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]ValuesSourceMetricsAggregationBuilder.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]search[/\\]aggregations[/\\]metrics[/\\]cardinality[/\\]CardinalityAggregator.java" checks="LineLength" />
diff --git a/core/src/main/java/org/elasticsearch/search/SearchModule.java b/core/src/main/java/org/elasticsearch/search/SearchModule.java
index dd144e413a2..516984dc477 100644
--- a/core/src/main/java/org/elasticsearch/search/SearchModule.java
+++ b/core/src/main/java/org/elasticsearch/search/SearchModule.java
@@ -563,7 +563,12 @@ public class SearchModule extends AbstractModule {
                         .addResultReader(UnmappedSampler.NAME, UnmappedSampler::new));
         registerAggregation(DiversifiedAggregationBuilder::new, new DiversifiedSamplerParser(),
                 DiversifiedAggregationBuilder.AGGREGATION_NAME_FIELD);
-        registerAggregation(TermsAggregationBuilder::new, new TermsParser(), TermsAggregationBuilder.AGGREGATION_NAME_FIELD);
+        registerAggregation(
+                new AggregationSpec(TermsAggregationBuilder::new, new TermsParser(), TermsAggregationBuilder.AGGREGATION_NAME_FIELD)
+                    .addResultReader(StringTerms.NAME, StringTerms::new)
+                    .addResultReader(UnmappedTerms.NAME, UnmappedTerms::new)
+                    .addResultReader(LongTerms.NAME, LongTerms::new)
+                    .addResultReader(DoubleTerms.NAME, DoubleTerms::new));
         registerAggregation(SignificantTermsAggregationBuilder::new,
                 new SignificantTermsParser(significanceHeuristicParserRegistry, queryParserRegistry),
                 SignificantTermsAggregationBuilder.AGGREGATION_NAME_FIELD);
@@ -770,14 +775,10 @@ public class SearchModule extends AbstractModule {
 
     static {
         // buckets
-        StringTerms.registerStreams();
-        LongTerms.registerStreams();
         SignificantStringTerms.registerStreams();
         SignificantLongTerms.registerStreams();
         UnmappedSignificantTerms.registerStreams();
         InternalGeoHashGrid.registerStreams();
-        DoubleTerms.registerStreams();
-        UnmappedTerms.registerStreams();
         InternalRange.registerStream();
         InternalDateRange.registerStream();
         InternalBinaryRange.registerStream();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
index 93ecfa975b4..d5f0f5222b9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java
@@ -236,9 +236,9 @@ public abstract class InternalAggregation implements Aggregation, ToXContent, St
     }
 
     /**
-     * Reduces the given addAggregation to a single one and returns it. In <b>most</b> cases, the assumption will be the all given
-     * addAggregation are of the same type (the same type as this aggregation). For best efficiency, when implementing,
-     * try reusing an existing get instance (typically the first in the given list) to save on redundant object
+     * Reduces the given aggregations to a single one and returns it. In <b>most</b> cases, the assumption will be the all given
+     * aggregations are of the same type (the same type as this aggregation). For best efficiency, when implementing,
+     * try reusing an existing instance (typically the first in the given list) to save on redundant object
      * construction.
      */
     public final InternalAggregation reduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractStringTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractStringTermsAggregator.java
index 843af201610..5719fe7721b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractStringTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractStringTermsAggregator.java
@@ -27,10 +27,11 @@ import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
+import static java.util.Collections.emptyList;
+
 abstract class AbstractStringTermsAggregator extends TermsAggregator {
 
     protected final boolean showTermDocCountError;
@@ -44,9 +45,8 @@ abstract class AbstractStringTermsAggregator extends TermsAggregator {
 
     @Override
     public InternalAggregation buildEmptyAggregation() {
-        return new StringTerms(name, order, format, bucketCountThresholds.getRequiredSize(), bucketCountThresholds.getShardSize(),
-                bucketCountThresholds.getMinDocCount(), Collections.<InternalTerms.Bucket> emptyList(), showTermDocCountError, 0, 0,
-                pipelineAggregators(), metaData());
+        return new StringTerms(name, order, bucketCountThresholds.getRequiredSize(), bucketCountThresholds.getMinDocCount(),
+                pipelineAggregators(), metaData(), format, bucketCountThresholds.getShardSize(), showTermDocCountError, 0, emptyList(), 0);
     }
 
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java
index ebc06b48ffa..7e3daf5034b 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTerms.java
@@ -22,42 +22,21 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
 /**
- *
+ * Result of the {@link TermsAggregator} when the field is some kind of decimal number like a float, double, or distance.
  */
-public class DoubleTerms extends InternalTerms<DoubleTerms, DoubleTerms.Bucket> {
+public class DoubleTerms extends InternalMappedTerms<DoubleTerms, DoubleTerms.Bucket> {
+    public static final String NAME = "dterms";
 
-    public static final Type TYPE = new Type("terms", "dterms");
-
-    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public DoubleTerms readResult(StreamInput in) throws IOException {
-            DoubleTerms buckets = new DoubleTerms();
-            buckets.readFrom(in);
-            return buckets;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
-    static class Bucket extends InternalTerms.Bucket {
-
-        double term;
-
-        public Bucket(DocValueFormat format, boolean showDocCountError) {
-            super(format, showDocCountError);
-        }
+    static class Bucket extends InternalTerms.Bucket<Bucket> {
+        private final double term;
 
         public Bucket(double term, long docCount, InternalAggregations aggregations, boolean showDocCountError, long docCountError,
                 DocValueFormat format) {
@@ -65,9 +44,22 @@ public class DoubleTerms extends InternalTerms<DoubleTerms, DoubleTerms.Bucket>
             this.term = term;
         }
 
+        /**
+         * Read from a stream.
+         */
+        public Bucket(StreamInput in, DocValueFormat format, boolean showDocCountError) throws IOException {
+            super(in, format, showDocCountError);
+            term = in.readDouble();
+        }
+
+        @Override
+        protected void writeTermTo(StreamOutput out) throws IOException {
+            out.writeDouble(term);
+        }
+
         @Override
         public String getKeyAsString() {
-            return String.valueOf(term);
+            return format.format(term);
         }
 
         @Override
@@ -90,27 +82,6 @@ public class DoubleTerms extends InternalTerms<DoubleTerms, DoubleTerms.Bucket>
             return new Bucket(term, docCount, aggs, showDocCountError, docCountError, format);
         }
 
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            term = in.readDouble();
-            docCount = in.readVLong();
-            docCountError = -1;
-            if (showDocCountError) {
-                docCountError = in.readLong();
-            }
-            aggregations = InternalAggregations.readAggregations(in);
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeDouble(term);
-            out.writeVLong(getDocCount());
-            if (showDocCountError) {
-                out.writeLong(docCountError);
-            }
-            aggregations.writeTo(out);
-        }
-
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
@@ -128,25 +99,29 @@ public class DoubleTerms extends InternalTerms<DoubleTerms, DoubleTerms.Bucket>
         }
     }
 
-    DoubleTerms() {
-    } // for serialization
+    public DoubleTerms(String name, Terms.Order order, int requiredSize, long minDocCount, List<PipelineAggregator> pipelineAggregators,
+            Map<String, Object> metaData, DocValueFormat format, int shardSize, boolean showTermDocCountError, long otherDocCount,
+            List<Bucket> buckets, long docCountError) {
+        super(name, order, requiredSize, minDocCount, pipelineAggregators, metaData, format, shardSize, showTermDocCountError,
+                otherDocCount, buckets, docCountError);
+    }
 
-    public DoubleTerms(String name, Terms.Order order, DocValueFormat format, int requiredSize, int shardSize,
-            long minDocCount, List<? extends InternalTerms.Bucket> buckets, boolean showTermDocCountError, long docCountError,
-            long otherDocCount, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
-        super(name, order, format, requiredSize, shardSize, minDocCount, buckets, showTermDocCountError, docCountError, otherDocCount, pipelineAggregators,
-                metaData);
+    /**
+     * Read from a stream.
+     */
+    public DoubleTerms(StreamInput in) throws IOException {
+        super(in, Bucket::new);
     }
 
     @Override
-    public Type type() {
-        return TYPE;
+    public String getWriteableName() {
+        return NAME;
     }
 
     @Override
     public DoubleTerms create(List<Bucket> buckets) {
-        return new DoubleTerms(this.name, this.order, this.format, this.requiredSize, this.shardSize, this.minDocCount, buckets,
-                this.showTermDocCountError, this.docCountError, this.otherDocCount, this.pipelineAggregators(), this.metaData);
+        return new DoubleTerms(name, order, requiredSize, minDocCount, this.pipelineAggregators(), metaData, format, shardSize,
+                showTermDocCountError, otherDocCount, buckets, docCountError);
     }
 
     @Override
@@ -156,48 +131,9 @@ public class DoubleTerms extends InternalTerms<DoubleTerms, DoubleTerms.Bucket>
     }
 
     @Override
-    protected DoubleTerms create(String name, List<org.elasticsearch.search.aggregations.bucket.terms.InternalTerms.Bucket> buckets,
-            long docCountError, long otherDocCount, InternalTerms prototype) {
-        return new DoubleTerms(name, prototype.order, ((DoubleTerms) prototype).format, prototype.requiredSize, prototype.shardSize,
-                prototype.minDocCount, buckets, prototype.showTermDocCountError, docCountError, otherDocCount, prototype.pipelineAggregators(),
-                prototype.getMetaData());
-    }
-
-    @Override
-    protected void doReadFrom(StreamInput in) throws IOException {
-        this.docCountError = in.readLong();
-        this.order = InternalOrder.Streams.readOrder(in);
-        this.format = in.readNamedWriteable(DocValueFormat.class);
-        this.requiredSize = readSize(in);
-        this.shardSize = readSize(in);
-        this.showTermDocCountError = in.readBoolean();
-        this.minDocCount = in.readVLong();
-        this.otherDocCount = in.readVLong();
-        int size = in.readVInt();
-        List<InternalTerms.Bucket> buckets = new ArrayList<>(size);
-        for (int i = 0; i < size; i++) {
-            Bucket bucket = new Bucket(format, showTermDocCountError);
-            bucket.readFrom(in);
-            buckets.add(bucket);
-        }
-        this.buckets = buckets;
-        this.bucketMap = null;
-    }
-
-    @Override
-    protected void doWriteTo(StreamOutput out) throws IOException {
-        out.writeLong(docCountError);
-        InternalOrder.Streams.writeOrder(order, out);
-        out.writeNamedWriteable(format);
-        writeSize(requiredSize, out);
-        writeSize(shardSize, out);
-        out.writeBoolean(showTermDocCountError);
-        out.writeVLong(minDocCount);
-        out.writeVLong(otherDocCount);
-        out.writeVInt(buckets.size());
-        for (InternalTerms.Bucket bucket : buckets) {
-            bucket.writeTo(out);
-        }
+    protected DoubleTerms create(String name, List<Bucket> buckets, long docCountError, long otherDocCount) {
+        return new DoubleTerms(name, order, requiredSize, minDocCount, pipelineAggregators(), getMetaData(), format,
+                shardSize, showTermDocCountError, otherDocCount, buckets, docCountError);
     }
 
     @Override
@@ -205,11 +141,15 @@ public class DoubleTerms extends InternalTerms<DoubleTerms, DoubleTerms.Bucket>
         builder.field(InternalTerms.DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME, docCountError);
         builder.field(SUM_OF_OTHER_DOC_COUNTS, otherDocCount);
         builder.startArray(CommonFields.BUCKETS);
-        for (InternalTerms.Bucket bucket : buckets) {
+        for (Bucket bucket : buckets) {
             bucket.toXContent(builder, params);
         }
         builder.endArray();
         return builder;
     }
 
+    @Override
+    protected Bucket[] createBucketsArray(int size) {
+        return new Bucket[size];
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTermsAggregator.java
index fcaf73c334e..b2b57f9d060 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/DoubleTermsAggregator.java
@@ -32,9 +32,9 @@ import org.elasticsearch.search.aggregations.support.ValuesSource;
 import org.elasticsearch.search.aggregations.support.ValuesSource.Numeric;
 
 import java.io.IOException;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 /**
  *
@@ -66,20 +66,16 @@ public class DoubleTermsAggregator extends LongTermsAggregator {
         return convertToDouble(terms);
     }
 
-    private static DoubleTerms.Bucket convertToDouble(InternalTerms.Bucket bucket) {
-        final long term = ((Number) bucket.getKey()).longValue();
-        final double value = NumericUtils.sortableLongToDouble(term);
-        return new DoubleTerms.Bucket(value, bucket.docCount, bucket.aggregations, bucket.showDocCountError, bucket.docCountError, bucket.format);
-    }
-
     private static DoubleTerms convertToDouble(LongTerms terms) {
-        final InternalTerms.Bucket[] buckets = terms.getBuckets().toArray(new InternalTerms.Bucket[0]);
-        for (int i = 0; i < buckets.length; ++i) {
-            buckets[i] = convertToDouble(buckets[i]);
-        }
-        return new DoubleTerms(terms.getName(), terms.order, terms.format, terms.requiredSize, terms.shardSize, terms.minDocCount,
-                Arrays.asList(buckets), terms.showTermDocCountError, terms.docCountError, terms.otherDocCount, terms.pipelineAggregators(),
-                terms.getMetaData());
+        List<DoubleTerms.Bucket> buckets = terms.buckets.stream().map(DoubleTermsAggregator::convertToDouble).collect(Collectors.toList());
+        return new DoubleTerms(terms.getName(), terms.order, terms.requiredSize, terms.minDocCount, terms.pipelineAggregators(),
+                terms.getMetaData(), terms.format, terms.shardSize, terms.showTermDocCountError, terms.otherDocCount, buckets,
+                terms.docCountError);
     }
 
+    private static DoubleTerms.Bucket convertToDouble(LongTerms.Bucket bucket) {
+        double value = NumericUtils.sortableLongToDouble(bucket.term);
+        return new DoubleTerms.Bucket(value, bucket.docCount, bucket.aggregations, bucket.showDocCountError, bucket.docCountError,
+                bucket.format);
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java
index 6095826ef4b..d4c138a64aa 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java
@@ -41,7 +41,6 @@ import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
 import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
-import org.elasticsearch.search.aggregations.bucket.terms.InternalTerms.Bucket;
 import org.elasticsearch.search.aggregations.bucket.terms.support.BucketPriorityQueue;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
@@ -156,7 +155,7 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
             size = (int) Math.min(maxBucketOrd(), bucketCountThresholds.getShardSize());
         }
         long otherDocCount = 0;
-        BucketPriorityQueue ordered = new BucketPriorityQueue(size, order.comparator(this));
+        BucketPriorityQueue<OrdBucket> ordered = new BucketPriorityQueue<>(size, order.comparator(this));
         OrdBucket spare = new OrdBucket(-1, 0, null, showTermDocCountError, 0);
         for (long globalTermOrd = 0; globalTermOrd < globalOrds.getValueCount(); ++globalTermOrd) {
             if (includeExclude != null && !acceptedGlobalOrdinals.get(globalTermOrd)) {
@@ -172,7 +171,7 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
             spare.bucketOrd = bucketOrd;
             spare.docCount = bucketDocCount;
             if (bucketCountThresholds.getShardMinDocCount() <= spare.docCount) {
-                spare = (OrdBucket) ordered.insertWithOverflow(spare);
+                spare = ordered.insertWithOverflow(spare);
                 if (spare == null) {
                     spare = new OrdBucket(-1, 0, null, showTermDocCountError, 0);
                 }
@@ -180,7 +179,7 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
         }
 
         // Get the top buckets
-        final InternalTerms.Bucket[] list = new InternalTerms.Bucket[ordered.size()];
+        final StringTerms.Bucket[] list = new StringTerms.Bucket[ordered.size()];
         long survivingBucketOrds[] = new long[ordered.size()];
         for (int i = ordered.size() - 1; i >= 0; --i) {
             final OrdBucket bucket = (OrdBucket) ordered.pop();
@@ -196,20 +195,20 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
 
         //Now build the aggs
         for (int i = 0; i < list.length; i++) {
-            Bucket bucket = list[i];
+            StringTerms.Bucket bucket = list[i];
             bucket.aggregations = bucket.docCount == 0 ? bucketEmptyAggregations() : bucketAggregations(bucket.bucketOrd);
             bucket.docCountError = 0;
         }
 
-        return new StringTerms(name, order, format, bucketCountThresholds.getRequiredSize(), bucketCountThresholds.getShardSize(),
-                bucketCountThresholds.getMinDocCount(), Arrays.asList(list), showTermDocCountError, 0, otherDocCount, pipelineAggregators(),
-                metaData());
+        return new StringTerms(name, order, bucketCountThresholds.getRequiredSize(), bucketCountThresholds.getMinDocCount(),
+                pipelineAggregators(), metaData(), format, bucketCountThresholds.getShardSize(), showTermDocCountError,
+                otherDocCount, Arrays.asList(list), 0);
     }
 
     /**
      * This is used internally only, just for compare using global ordinal instead of term bytes in the PQ
      */
-    static class OrdBucket extends InternalTerms.Bucket {
+    static class OrdBucket extends InternalTerms.Bucket<OrdBucket> {
         long globalOrd;
 
         OrdBucket(long globalOrd, long docCount, InternalAggregations aggregations, boolean showDocCountError, long docCountError) {
@@ -233,7 +232,7 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
         }
 
         @Override
-        Bucket newBucket(long docCount, InternalAggregations aggs, long docCountError) {
+        OrdBucket newBucket(long docCount, InternalAggregations aggs, long docCountError) {
             throw new UnsupportedOperationException();
         }
 
@@ -248,7 +247,7 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
         }
 
         @Override
-        public void writeTo(StreamOutput out) throws IOException {
+        protected void writeTermTo(StreamOutput out) throws IOException {
             throw new UnsupportedOperationException();
         }
 
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalMappedTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalMappedTerms.java
new file mode 100644
index 00000000000..e3f842a08de
--- /dev/null
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalMappedTerms.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.bucket.terms;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Common superclass for results of the terms aggregation on mapped fields.
+ */
+public abstract class InternalMappedTerms<A extends InternalTerms<A, B>, B extends InternalTerms.Bucket<B>> extends InternalTerms<A, B> {
+    protected final DocValueFormat format;
+    protected final int shardSize;
+    protected final boolean showTermDocCountError;
+    protected final long otherDocCount;
+    protected final List<B> buckets;
+    protected Map<String, B> bucketMap;
+
+    protected long docCountError;
+
+    protected InternalMappedTerms(String name, Terms.Order order, int requiredSize, long minDocCount,
+            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData, DocValueFormat format, int shardSize,
+            boolean showTermDocCountError, long otherDocCount, List<B> buckets, long docCountError) {
+        super(name, order, requiredSize, minDocCount, pipelineAggregators, metaData);
+        this.format = format;
+        this.shardSize = shardSize;
+        this.showTermDocCountError = showTermDocCountError;
+        this.otherDocCount = otherDocCount;
+        this.docCountError = docCountError;
+        this.buckets = buckets;
+    }
+
+    /**
+     * Read from a stream.
+     */
+    protected InternalMappedTerms(StreamInput in, Bucket.Reader<B> bucketReader) throws IOException {
+        super(in);
+        docCountError = in.readZLong();
+        format = in.readNamedWriteable(DocValueFormat.class);
+        shardSize = readSize(in);
+        showTermDocCountError = in.readBoolean();
+        otherDocCount = in.readVLong();
+        buckets = in.readList(stream -> bucketReader.read(stream, format, showTermDocCountError));
+    }
+
+    @Override
+    protected final void writeTermTypeInfoTo(StreamOutput out) throws IOException {
+        out.writeZLong(docCountError);
+        out.writeNamedWriteable(format);
+        writeSize(shardSize, out);
+        out.writeBoolean(showTermDocCountError);
+        out.writeVLong(otherDocCount);
+        out.writeList(buckets);
+    }
+
+    @Override
+    protected void setDocCountError(long docCountError) {
+        this.docCountError = docCountError; 
+    }
+
+    @Override
+    protected int getShardSize() {
+        return shardSize;
+    }
+
+    @Override
+    public long getDocCountError() {
+        return docCountError;
+    }
+
+    @Override
+    public long getSumOfOtherDocCounts() {
+        return otherDocCount;
+    }
+
+    @Override
+    public List<B> getBucketsInternal() {
+        return buckets;
+    }
+
+    @Override
+    public B getBucketByKey(String term) {
+        if (bucketMap == null) {
+            bucketMap = buckets.stream().collect(Collectors.toMap(Bucket::getKeyAsString, Function.identity()));
+        }
+        return bucketMap.get(term);
+    }
+}
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
index e134595c594..56d69d814f1 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/InternalTerms.java
@@ -18,7 +18,8 @@
  */
 package org.elasticsearch.search.aggregations.bucket.terms;
 
-import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
@@ -29,45 +30,73 @@ import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
 import org.elasticsearch.search.aggregations.bucket.terms.support.BucketPriorityQueue;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-/**
- *
- */
-public abstract class InternalTerms<A extends InternalTerms, B extends InternalTerms.Bucket> extends InternalMultiBucketAggregation<A, B>
-        implements Terms, ToXContent, Streamable {
+import static java.util.Collections.unmodifiableList;
+
+public abstract class InternalTerms<A extends InternalTerms<A, B>, B extends InternalTerms.Bucket<B>>
+        extends InternalMultiBucketAggregation<A, B> implements Terms, ToXContent {
 
     protected static final String DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME = "doc_count_error_upper_bound";
     protected static final String SUM_OF_OTHER_DOC_COUNTS = "sum_other_doc_count";
 
-    public abstract static class Bucket extends Terms.Bucket {
+    public abstract static class Bucket<B extends Bucket<B>> extends Terms.Bucket {
+        /**
+         * Reads a bucket. Should be a constructor reference.
+         */
+        @FunctionalInterface
+        public interface Reader<B extends Bucket<B>> {
+            B read(StreamInput in, DocValueFormat format, boolean showDocCountError) throws IOException;
+        }
 
         long bucketOrd;
 
         protected long docCount;
         protected long docCountError;
         protected InternalAggregations aggregations;
-        protected boolean showDocCountError;
-        final transient DocValueFormat format;
-
-        protected Bucket(DocValueFormat formatter, boolean showDocCountError) {
-            // for serialization
-            this.showDocCountError = showDocCountError;
-            this.format = formatter;
-        }
+        protected final boolean showDocCountError;
+        protected final DocValueFormat format;
 
         protected Bucket(long docCount, InternalAggregations aggregations, boolean showDocCountError, long docCountError,
                 DocValueFormat formatter) {
-            this(formatter, showDocCountError);
+            this.showDocCountError = showDocCountError;
+            this.format = formatter;
             this.docCount = docCount;
             this.aggregations = aggregations;
             this.docCountError = docCountError;
         }
 
+        /**
+         * Read from a stream.
+         */
+        protected Bucket(StreamInput in, DocValueFormat formatter, boolean showDocCountError) throws IOException {
+            this.showDocCountError = showDocCountError;
+            this.format = formatter;
+            docCount = in.readVLong();
+            docCountError = -1;
+            if (showDocCountError) {
+                docCountError = in.readLong();
+            }
+            aggregations = InternalAggregations.readAggregations(in);
+        }
+
+        @Override
+        public final void writeTo(StreamOutput out) throws IOException {
+            out.writeVLong(getDocCount());
+            if (showDocCountError) {
+                out.writeLong(docCountError);
+            }
+            aggregations.writeTo(out);
+            writeTermTo(out);
+        }
+
+        protected abstract void writeTermTo(StreamOutput out) throws IOException;
+
         @Override
         public long getDocCount() {
             return docCount;
@@ -86,13 +115,13 @@ public abstract class InternalTerms<A extends InternalTerms, B extends InternalT
             return aggregations;
         }
 
-        abstract Bucket newBucket(long docCount, InternalAggregations aggs, long docCountError);
+        abstract B newBucket(long docCount, InternalAggregations aggs, long docCountError);
 
-        public Bucket reduce(List<? extends Bucket> buckets, ReduceContext context) {
+        public B reduce(List<B> buckets, ReduceContext context) {
             long docCount = 0;
             long docCountError = 0;
             List<InternalAggregations> aggregationsList = new ArrayList<>(buckets.size());
-            for (Bucket bucket : buckets) {
+            for (B bucket : buckets) {
                 docCount += bucket.docCount;
                 if (docCountError != -1) {
                     if (bucket.docCountError == -1) {
@@ -108,72 +137,59 @@ public abstract class InternalTerms<A extends InternalTerms, B extends InternalT
         }
     }
 
-    protected Terms.Order order;
-    protected int requiredSize;
-    protected DocValueFormat format;
-    protected int shardSize;
-    protected long minDocCount;
-    protected List<? extends Bucket> buckets;
-    protected Map<String, Bucket> bucketMap;
-    protected long docCountError;
-    protected boolean showTermDocCountError;
-    protected long otherDocCount;
+    protected final Terms.Order order;
+    protected final int requiredSize;
+    protected final long minDocCount;
 
-    protected InternalTerms() {} // for serialization
-
-    protected InternalTerms(String name, Terms.Order order, DocValueFormat format, int requiredSize, int shardSize, long minDocCount,
-            List<? extends Bucket> buckets, boolean showTermDocCountError, long docCountError, long otherDocCount, List<PipelineAggregator> pipelineAggregators,
-            Map<String, Object> metaData) {
+    protected InternalTerms(String name, Terms.Order order, int requiredSize, long minDocCount,
+            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
         super(name, pipelineAggregators, metaData);
         this.order = order;
-        this.format = format;
         this.requiredSize = requiredSize;
-        this.shardSize = shardSize;
         this.minDocCount = minDocCount;
-        this.buckets = buckets;
-        this.showTermDocCountError = showTermDocCountError;
-        this.docCountError = docCountError;
-        this.otherDocCount = otherDocCount;
+    }
+
+    /**
+     * Read from a stream.
+     */
+    protected InternalTerms(StreamInput in) throws IOException {
+       super(in);
+       order = InternalOrder.Streams.readOrder(in);
+       requiredSize = readSize(in);
+       minDocCount = in.readVLong();
     }
 
     @Override
-    public List<Terms.Bucket> getBuckets() {
-        Object o = buckets;
-        return (List<Terms.Bucket>) o;
+    protected final void doWriteTo(StreamOutput out) throws IOException {
+        InternalOrder.Streams.writeOrder(order, out);
+        writeSize(requiredSize, out);
+        out.writeVLong(minDocCount);
+        writeTermTypeInfoTo(out);
     }
 
-    @Override
-    public Terms.Bucket getBucketByKey(String term) {
-        if (bucketMap == null) {
-            bucketMap = new HashMap<>(buckets.size());
-            for (Bucket bucket : buckets) {
-                bucketMap.put(bucket.getKeyAsString(), bucket);
-            }
-        }
-        return bucketMap.get(term);
-    }
+    protected abstract void writeTermTypeInfoTo(StreamOutput out) throws IOException;
 
     @Override
-    public long getDocCountError() {
-        return docCountError;
+    public final List<Terms.Bucket> getBuckets() {
+        return unmodifiableList(getBucketsInternal());
     }
 
+    protected abstract List<B> getBucketsInternal();
+
     @Override
-    public long getSumOfOtherDocCounts() {
-        return otherDocCount;
-    }
+    public abstract B getBucketByKey(String term);
 
     @Override
     public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
-
-        Map<Object, List<InternalTerms.Bucket>> buckets = new HashMap<>();
+        Map<Object, List<B>> buckets = new HashMap<>();
         long sumDocCountError = 0;
         long otherDocCount = 0;
         InternalTerms<A, B> referenceTerms = null;
         for (InternalAggregation aggregation : aggregations) {
+            @SuppressWarnings("unchecked")
             InternalTerms<A, B> terms = (InternalTerms<A, B>) aggregation;
-            if (referenceTerms == null && !terms.getClass().equals(UnmappedTerms.class)) {
-                referenceTerms = (InternalTerms<A, B>) aggregation;
+            if (referenceTerms == null && !aggregation.getClass().equals(UnmappedTerms.class)) {
+                referenceTerms = terms;
             }
             if (referenceTerms != null &&
                     !referenceTerms.getClass().equals(terms.getClass()) &&
@@ -190,10 +206,10 @@ public abstract class InternalTerms<A extends InternalTerms, B extends InternalT
             }
             otherDocCount += terms.getSumOfOtherDocCounts();
             final long thisAggDocCountError;
-            if (terms.buckets.size() < this.shardSize || InternalOrder.isTermOrder(order)) {
+            if (terms.getBucketsInternal().size() < getShardSize() || InternalOrder.isTermOrder(order)) {
                 thisAggDocCountError = 0;
             } else if (InternalOrder.isCountDesc(this.order)) {
-                thisAggDocCountError = terms.buckets.get(terms.buckets.size() - 1).docCount;
+                thisAggDocCountError = terms.getBucketsInternal().get(terms.getBucketsInternal().size() - 1).docCount;
             } else {
                 thisAggDocCountError = -1;
             }
@@ -204,10 +220,10 @@ public abstract class InternalTerms<A extends InternalTerms, B extends InternalT
                     sumDocCountError += thisAggDocCountError;
                 }
             }
-            terms.docCountError = thisAggDocCountError;
-            for (Bucket bucket : terms.buckets) {
+            setDocCountError(thisAggDocCountError);
+            for (B bucket : terms.getBucketsInternal()) {
                 bucket.docCountError = thisAggDocCountError;
-                List<Bucket> bucketList = buckets.get(bucket.getKey());
+                List<B> bucketList = buckets.get(bucket.getKey());
                 if (bucketList == null) {
                     bucketList = new ArrayList<>();
                     buckets.put(bucket.getKey(), bucketList);
@@ -217,9 +233,9 @@ public abstract class InternalTerms<A extends InternalTerms, B extends InternalT
         }
 
         final int size = Math.min(requiredSize, buckets.size());
-        BucketPriorityQueue ordered = new BucketPriorityQueue(size, order.comparator(null));
-        for (List<Bucket> sameTermBuckets : buckets.values()) {
-            final Bucket b = sameTermBuckets.get(0).reduce(sameTermBuckets, reduceContext);
+        BucketPriorityQueue<B> ordered = new BucketPriorityQueue<>(size, order.comparator(null));
+        for (List<B> sameTermBuckets : buckets.values()) {
+            final B b = sameTermBuckets.get(0).reduce(sameTermBuckets, reduceContext);
             if (b.docCountError != -1) {
                 if (sumDocCountError == -1) {
                     b.docCountError = -1;
@@ -228,15 +244,15 @@ public abstract class InternalTerms<A extends InternalTerms, B extends InternalT
                 }
             }
             if (b.docCount >= minDocCount) {
-                Terms.Bucket removed = ordered.insertWithOverflow(b);
+                B removed = ordered.insertWithOverflow(b);
                 if (removed != null) {
                     otherDocCount += removed.getDocCount();
                 }
             }
         }
-        Bucket[] list = new Bucket[ordered.size()];
+        B[] list = createBucketsArray(ordered.size());
         for (int i = ordered.size() - 1; i >= 0; i--) {
-            list[i] = (Bucket) ordered.pop();
+            list[i] = ordered.pop();
         }
         long docCountError;
         if (sumDocCountError == -1) {
@@ -244,10 +260,17 @@ public abstract class InternalTerms<A extends InternalTerms, B extends InternalT
         } else {
             docCountError = aggregations.size() == 1 ? 0 : sumDocCountError;
         }
-        return create(name, Arrays.asList(list), docCountError, otherDocCount, this);
+        return create(name, Arrays.asList(list), docCountError, otherDocCount);
     }
 
-    protected abstract A create(String name, List<InternalTerms.Bucket> buckets, long docCountError, long otherDocCount,
-            InternalTerms prototype);
+    protected abstract void setDocCountError(long docCountError);
 
+    protected abstract int getShardSize();
+
+    protected abstract A create(String name, List<B> buckets, long docCountError, long otherDocCount);
+
+    /**
+     * Create an array to hold some buckets. Used in collecting the results.
+     */
+    protected abstract B[] createBucketsArray(int size);
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java
index 4d95c8b4024..41092dba176 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTerms.java
@@ -22,49 +22,41 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
 /**
- *
+ * Result of the {@link TermsAggregator} when the field is some kind of whole number like a integer, long, or a date.
  */
-public class LongTerms extends InternalTerms<LongTerms, LongTerms.Bucket> {
-
-    public static final Type TYPE = new Type("terms", "lterms");
-
-    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public LongTerms readResult(StreamInput in) throws IOException {
-            LongTerms buckets = new LongTerms();
-            buckets.readFrom(in);
-            return buckets;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
-    static class Bucket extends InternalTerms.Bucket {
+public class LongTerms extends InternalMappedTerms<LongTerms, LongTerms.Bucket> {
+    public static final String NAME = "lterms";
 
+    public static class Bucket extends InternalTerms.Bucket<Bucket> {
         long term;
 
-        public Bucket(DocValueFormat format, boolean showDocCountError) {
-            super(format, showDocCountError);
-        }
-
         public Bucket(long term, long docCount, InternalAggregations aggregations, boolean showDocCountError, long docCountError,
                 DocValueFormat format) {
             super(docCount, aggregations, showDocCountError, docCountError, format);
             this.term = term;
         }
 
+        /**
+         * Read from a stream.
+         */
+        public Bucket(StreamInput in, DocValueFormat format, boolean showDocCountError) throws IOException {
+            super(in, format, showDocCountError);
+            term = in.readLong();
+        }
+
+        @Override
+        protected void writeTermTo(StreamOutput out) throws IOException {
+            out.writeLong(term);
+        }
+
         @Override
         public String getKeyAsString() {
             return format.format(term);
@@ -90,27 +82,6 @@ public class LongTerms extends InternalTerms<LongTerms, LongTerms.Bucket> {
             return new Bucket(term, docCount, aggs, showDocCountError, docCountError, format);
         }
 
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            term = in.readLong();
-            docCount = in.readVLong();
-            docCountError = -1;
-            if (showDocCountError) {
-                docCountError = in.readLong();
-            }
-            aggregations = InternalAggregations.readAggregations(in);
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeLong(term);
-            out.writeVLong(getDocCount());
-            if (showDocCountError) {
-                out.writeLong(docCountError);
-            }
-            aggregations.writeTo(out);
-        }
-
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
@@ -128,24 +99,29 @@ public class LongTerms extends InternalTerms<LongTerms, LongTerms.Bucket> {
         }
     }
 
-    LongTerms() {} // for serialization
+    public LongTerms(String name, Terms.Order order, int requiredSize, long minDocCount, List<PipelineAggregator> pipelineAggregators,
+            Map<String, Object> metaData, DocValueFormat format, int shardSize, boolean showTermDocCountError, long otherDocCount,
+            List<Bucket> buckets, long docCountError) {
+        super(name, order, requiredSize, minDocCount, pipelineAggregators, metaData, format, shardSize, showTermDocCountError,
+                otherDocCount, buckets, docCountError);
+    }
 
-    public LongTerms(String name, Terms.Order order, DocValueFormat format, int requiredSize, int shardSize, long minDocCount,
-            List<? extends InternalTerms.Bucket> buckets, boolean showTermDocCountError, long docCountError, long otherDocCount,
-            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
-        super(name, order, format, requiredSize, shardSize, minDocCount, buckets, showTermDocCountError, docCountError, otherDocCount,
-                pipelineAggregators, metaData);
+    /**
+     * Read from a stream.
+     */
+    public LongTerms(StreamInput in) throws IOException {
+        super(in, Bucket::new);
     }
 
     @Override
-    public Type type() {
-        return TYPE;
+    public String getWriteableName() {
+        return NAME;
     }
 
     @Override
     public LongTerms create(List<Bucket> buckets) {
-        return new LongTerms(this.name, this.order, this.format, this.requiredSize, this.shardSize, this.minDocCount, buckets,
-                this.showTermDocCountError, this.docCountError, this.otherDocCount, this.pipelineAggregators(), this.metaData);
+        return new LongTerms(name, order, requiredSize, minDocCount, pipelineAggregators(), metaData, format, shardSize,
+                showTermDocCountError, otherDocCount, buckets, docCountError);
     }
 
     @Override
@@ -155,48 +131,9 @@ public class LongTerms extends InternalTerms<LongTerms, LongTerms.Bucket> {
     }
 
     @Override
-    protected LongTerms create(String name, List<org.elasticsearch.search.aggregations.bucket.terms.InternalTerms.Bucket> buckets,
-            long docCountError, long otherDocCount, InternalTerms prototype) {
-        return new LongTerms(name, prototype.order, ((LongTerms) prototype).format, prototype.requiredSize, prototype.shardSize,
-                prototype.minDocCount, buckets, prototype.showTermDocCountError, docCountError, otherDocCount, prototype.pipelineAggregators(),
-                prototype.getMetaData());
-    }
-
-    @Override
-    protected void doReadFrom(StreamInput in) throws IOException {
-        this.docCountError = in.readLong();
-        this.order = InternalOrder.Streams.readOrder(in);
-        this.format = in.readNamedWriteable(DocValueFormat.class);
-        this.requiredSize = readSize(in);
-        this.shardSize = readSize(in);
-        this.showTermDocCountError = in.readBoolean();
-        this.minDocCount = in.readVLong();
-        this.otherDocCount = in.readVLong();
-        int size = in.readVInt();
-        List<InternalTerms.Bucket> buckets = new ArrayList<>(size);
-        for (int i = 0; i < size; i++) {
-            Bucket bucket = new Bucket(format, showTermDocCountError);
-            bucket.readFrom(in);
-            buckets.add(bucket);
-        }
-        this.buckets = buckets;
-        this.bucketMap = null;
-    }
-
-    @Override
-    protected void doWriteTo(StreamOutput out) throws IOException {
-        out.writeLong(docCountError);
-        InternalOrder.Streams.writeOrder(order, out);
-        out.writeNamedWriteable(format);
-        writeSize(requiredSize, out);
-        writeSize(shardSize, out);
-        out.writeBoolean(showTermDocCountError);
-        out.writeVLong(minDocCount);
-        out.writeVLong(otherDocCount);
-        out.writeVInt(buckets.size());
-        for (InternalTerms.Bucket bucket : buckets) {
-            bucket.writeTo(out);
-        }
+    protected LongTerms create(String name, List<Bucket> buckets, long docCountError, long otherDocCount) {
+        return new LongTerms(name, order, requiredSize, minDocCount, pipelineAggregators(), getMetaData(), format, shardSize,
+                showTermDocCountError, otherDocCount, buckets, docCountError);
     }
 
     @Override
@@ -204,11 +141,15 @@ public class LongTerms extends InternalTerms<LongTerms, LongTerms.Bucket> {
         builder.field(InternalTerms.DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME, docCountError);
         builder.field(SUM_OF_OTHER_DOC_COUNTS, otherDocCount);
         builder.startArray(CommonFields.BUCKETS);
-        for (InternalTerms.Bucket bucket : buckets) {
+        for (Bucket bucket : buckets) {
             bucket.toXContent(builder, params);
         }
         builder.endArray();
         return builder;
     }
 
+    @Override
+    protected Bucket[] createBucketsArray(int size) {
+        return new Bucket[size];
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsAggregator.java
index 2bcd0df1632..fd7296d07d9 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/LongTermsAggregator.java
@@ -37,10 +37,11 @@ import org.elasticsearch.search.aggregations.support.ValuesSource;
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
+import static java.util.Collections.emptyList;
+
 /**
  *
  */
@@ -124,7 +125,7 @@ public class LongTermsAggregator extends TermsAggregator {
         final int size = (int) Math.min(bucketOrds.size(), bucketCountThresholds.getShardSize());
 
         long otherDocCount = 0;
-        BucketPriorityQueue ordered = new BucketPriorityQueue(size, order.comparator(this));
+        BucketPriorityQueue<LongTerms.Bucket> ordered = new BucketPriorityQueue<>(size, order.comparator(this));
         LongTerms.Bucket spare = null;
         for (long i = 0; i < bucketOrds.size(); i++) {
             if (spare == null) {
@@ -140,7 +141,7 @@ public class LongTermsAggregator extends TermsAggregator {
         }
 
         // Get the top buckets
-        final InternalTerms.Bucket[] list = new InternalTerms.Bucket[ordered.size()];
+        final LongTerms.Bucket[] list = new LongTerms.Bucket[ordered.size()];
         long survivingBucketOrds[] = new long[ordered.size()];
         for (int i = ordered.size() - 1; i >= 0; --i) {
             final LongTerms.Bucket bucket = (LongTerms.Bucket) ordered.pop();
@@ -151,22 +152,21 @@ public class LongTermsAggregator extends TermsAggregator {
 
         runDeferredCollections(survivingBucketOrds);
 
-        //Now build the aggs
+        // Now build the aggs
         for (int i = 0; i < list.length; i++) {
-          list[i].aggregations = bucketAggregations(list[i].bucketOrd);
-          list[i].docCountError = 0;
+            list[i].aggregations = bucketAggregations(list[i].bucketOrd);
+            list[i].docCountError = 0;
         }
 
-        return new LongTerms(name, order, format, bucketCountThresholds.getRequiredSize(), bucketCountThresholds.getShardSize(),
-                bucketCountThresholds.getMinDocCount(), Arrays.asList(list), showTermDocCountError, 0, otherDocCount, pipelineAggregators(),
-                metaData());
+        return new LongTerms(name, order, bucketCountThresholds.getRequiredSize(), bucketCountThresholds.getMinDocCount(),
+                pipelineAggregators(), metaData(), format, bucketCountThresholds.getShardSize(), showTermDocCountError, otherDocCount,
+                Arrays.asList(list), 0);
     }
 
     @Override
     public InternalAggregation buildEmptyAggregation() {
-        return new LongTerms(name, order, format, bucketCountThresholds.getRequiredSize(), bucketCountThresholds.getShardSize(),
-                bucketCountThresholds.getMinDocCount(), Collections.<InternalTerms.Bucket> emptyList(), showTermDocCountError, 0, 0,
-                pipelineAggregators(), metaData());
+        return new LongTerms(name, order, bucketCountThresholds.getRequiredSize(), bucketCountThresholds.getMinDocCount(),
+                pipelineAggregators(), metaData(), format, bucketCountThresholds.getShardSize(), showTermDocCountError, 0, emptyList(), 0);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java
index 45da502cf28..4a40f77b2b2 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTerms.java
@@ -23,50 +23,40 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationStreams;
-import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
 /**
- *
+ * Result of the {@link TermsAggregator} when the field is a String.
  */
-public class StringTerms extends InternalTerms<StringTerms, StringTerms.Bucket> {
-
-    public static final InternalAggregation.Type TYPE = new Type("terms", "sterms");
-
-    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public StringTerms readResult(StreamInput in) throws IOException {
-            StringTerms buckets = new StringTerms();
-            buckets.readFrom(in);
-            return buckets;
-        }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
-    }
-
-    public static class Bucket extends InternalTerms.Bucket {
-
+public class StringTerms extends InternalMappedTerms<StringTerms, StringTerms.Bucket> {
+    public static final String NAME = "sterms";
+    public static class Bucket extends InternalTerms.Bucket<Bucket> {
         BytesRef termBytes;
 
-        public Bucket(DocValueFormat format, boolean showDocCountError) {
-            super(format, showDocCountError);
-        }
-
         public Bucket(BytesRef term, long docCount, InternalAggregations aggregations, boolean showDocCountError, long docCountError,
                 DocValueFormat format) {
             super(docCount, aggregations, showDocCountError, docCountError, format);
             this.termBytes = term;
         }
 
+        /**
+         * Read from a stream.
+         */
+        public Bucket(StreamInput in, DocValueFormat format, boolean showDocCountError) throws IOException {
+            super(in, format, showDocCountError);
+            termBytes = in.readBytesRef();
+        }
+
+        @Override
+        protected void writeTermTo(StreamOutput out) throws IOException {
+            out.writeBytesRef(termBytes);
+        }
+
         @Override
         public Object getKey() {
             return getKeyAsString();
@@ -93,27 +83,6 @@ public class StringTerms extends InternalTerms<StringTerms, StringTerms.Bucket>
             return new Bucket(termBytes, docCount, aggs, showDocCountError, docCountError, format);
         }
 
-        @Override
-        public void readFrom(StreamInput in) throws IOException {
-            termBytes = in.readBytesRef();
-            docCount = in.readVLong();
-            docCountError = -1;
-            if (showDocCountError) {
-                docCountError = in.readLong();
-            }
-            aggregations = InternalAggregations.readAggregations(in);
-        }
-
-        @Override
-        public void writeTo(StreamOutput out) throws IOException {
-            out.writeBytesRef(termBytes);
-            out.writeVLong(getDocCount());
-            if (showDocCountError) {
-                out.writeLong(docCountError);
-            }
-            aggregations.writeTo(out);
-        }
-
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
@@ -128,74 +97,41 @@ public class StringTerms extends InternalTerms<StringTerms, StringTerms.Bucket>
         }
     }
 
-    StringTerms() {
-    } // for serialization
+    public StringTerms(String name, Terms.Order order, int requiredSize, long minDocCount, List<PipelineAggregator> pipelineAggregators,
+            Map<String, Object> metaData, DocValueFormat format, int shardSize, boolean showTermDocCountError, long otherDocCount,
+            List<Bucket> buckets, long docCountError) {
+        super(name, order, requiredSize, minDocCount, pipelineAggregators, metaData, format,
+                shardSize, showTermDocCountError, otherDocCount, buckets, docCountError);
+    }
 
-    public StringTerms(String name, Terms.Order order, DocValueFormat format, int requiredSize, int shardSize, long minDocCount,
-            List<? extends InternalTerms.Bucket> buckets, boolean showTermDocCountError, long docCountError, long otherDocCount,
-            List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
-        super(name, order, format, requiredSize, shardSize, minDocCount, buckets, showTermDocCountError, docCountError, otherDocCount, pipelineAggregators,
-                metaData);
+    /**
+     * Read from a stream.
+     */
+    public StringTerms(StreamInput in) throws IOException {
+        super(in, Bucket::new);
     }
 
     @Override
-    public Type type() {
-        return TYPE;
+    public String getWriteableName() {
+        return NAME;
     }
 
     @Override
     public StringTerms create(List<Bucket> buckets) {
-        return new StringTerms(this.name, this.order, this.format, this.requiredSize, this.shardSize, this.minDocCount, buckets,
-                this.showTermDocCountError, this.docCountError, this.otherDocCount, this.pipelineAggregators(), this.metaData);
+        return new StringTerms(name, order, requiredSize, minDocCount, pipelineAggregators(), metaData, format, shardSize,
+                showTermDocCountError, otherDocCount, buckets, docCountError);
     }
 
     @Override
     public Bucket createBucket(InternalAggregations aggregations, Bucket prototype) {
-        return new Bucket(prototype.termBytes, prototype.docCount, aggregations, prototype.showDocCountError, prototype.docCountError, prototype.format);
+        return new Bucket(prototype.termBytes, prototype.docCount, aggregations, prototype.showDocCountError, prototype.docCountError,
+                prototype.format);
     }
 
     @Override
-    protected StringTerms create(String name, List<org.elasticsearch.search.aggregations.bucket.terms.InternalTerms.Bucket> buckets,
-            long docCountError, long otherDocCount, InternalTerms prototype) {
-        return new StringTerms(name, prototype.order, prototype.format, prototype.requiredSize, prototype.shardSize, prototype.minDocCount, buckets,
-                prototype.showTermDocCountError, docCountError, otherDocCount, prototype.pipelineAggregators(), prototype.getMetaData());
-    }
-
-    @Override
-    protected void doReadFrom(StreamInput in) throws IOException {
-        this.docCountError = in.readLong();
-        this.order = InternalOrder.Streams.readOrder(in);
-        this.format = in.readNamedWriteable(DocValueFormat.class);
-        this.requiredSize = readSize(in);
-        this.shardSize = readSize(in);
-        this.showTermDocCountError = in.readBoolean();
-        this.minDocCount = in.readVLong();
-        this.otherDocCount = in.readVLong();
-        int size = in.readVInt();
-        List<InternalTerms.Bucket> buckets = new ArrayList<>(size);
-        for (int i = 0; i < size; i++) {
-            Bucket bucket = new Bucket(format, showTermDocCountError);
-            bucket.readFrom(in);
-            buckets.add(bucket);
-        }
-        this.buckets = buckets;
-        this.bucketMap = null;
-    }
-
-    @Override
-    protected void doWriteTo(StreamOutput out) throws IOException {
-        out.writeLong(docCountError);
-        InternalOrder.Streams.writeOrder(order, out);
-        out.writeNamedWriteable(format);
-        writeSize(requiredSize, out);
-        writeSize(shardSize, out);
-        out.writeBoolean(showTermDocCountError);
-        out.writeVLong(minDocCount);
-        out.writeVLong(otherDocCount);
-        out.writeVInt(buckets.size());
-        for (InternalTerms.Bucket bucket : buckets) {
-            bucket.writeTo(out);
-        }
+    protected StringTerms create(String name, List<Bucket> buckets, long docCountError, long otherDocCount) {
+        return new StringTerms(name, order, requiredSize, minDocCount, pipelineAggregators(), getMetaData(), format, shardSize,
+                showTermDocCountError, otherDocCount, buckets, docCountError);
     }
 
     @Override
@@ -203,11 +139,15 @@ public class StringTerms extends InternalTerms<StringTerms, StringTerms.Bucket>
         builder.field(InternalTerms.DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME, docCountError);
         builder.field(SUM_OF_OTHER_DOC_COUNTS, otherDocCount);
         builder.startArray(CommonFields.BUCKETS);
-        for (InternalTerms.Bucket bucket : buckets) {
+        for (Bucket bucket : buckets) {
             bucket.toXContent(builder, params);
         }
         builder.endArray();
         return builder;
     }
 
+    @Override
+    protected Bucket[] createBucketsArray(int size) {
+        return new Bucket[size];
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java
index a0f041a0284..7b2b76a0861 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/StringTermsAggregator.java
@@ -129,7 +129,7 @@ public class StringTermsAggregator extends AbstractStringTermsAggregator {
         final int size = (int) Math.min(bucketOrds.size(), bucketCountThresholds.getShardSize());
 
         long otherDocCount = 0;
-        BucketPriorityQueue ordered = new BucketPriorityQueue(size, order.comparator(this));
+        BucketPriorityQueue<StringTerms.Bucket> ordered = new BucketPriorityQueue<>(size, order.comparator(this));
         StringTerms.Bucket spare = null;
         for (int i = 0; i < bucketOrds.size(); i++) {
             if (spare == null) {
@@ -140,12 +140,12 @@ public class StringTermsAggregator extends AbstractStringTermsAggregator {
             otherDocCount += spare.docCount;
             spare.bucketOrd = i;
             if (bucketCountThresholds.getShardMinDocCount() <= spare.docCount) {
-                spare = (StringTerms.Bucket) ordered.insertWithOverflow(spare);
+                spare = ordered.insertWithOverflow(spare);
             }
         }
 
         // Get the top buckets
-        final InternalTerms.Bucket[] list = new InternalTerms.Bucket[ordered.size()];
+        final StringTerms.Bucket[] list = new StringTerms.Bucket[ordered.size()];
         long survivingBucketOrds[] = new long[ordered.size()];
         for (int i = ordered.size() - 1; i >= 0; --i) {
             final StringTerms.Bucket bucket = (StringTerms.Bucket) ordered.pop();
@@ -164,9 +164,9 @@ public class StringTermsAggregator extends AbstractStringTermsAggregator {
           bucket.docCountError = 0;
         }
 
-        return new StringTerms(name, order, format, bucketCountThresholds.getRequiredSize(), bucketCountThresholds.getShardSize(),
-                bucketCountThresholds.getMinDocCount(), Arrays.asList(list), showTermDocCountError, 0, otherDocCount, pipelineAggregators(),
-                metaData());
+        return new StringTerms(name, order, bucketCountThresholds.getRequiredSize(), bucketCountThresholds.getMinDocCount(),
+                pipelineAggregators(), metaData(), format, bucketCountThresholds.getShardSize(), showTermDocCountError, otherDocCount,
+                Arrays.asList(list), 0);
     }
 
     @Override
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
index 87567d64df4..d1a260bc157 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java
@@ -23,8 +23,10 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode;
 import org.elasticsearch.search.aggregations.AggregatorFactories.Builder;
+import org.elasticsearch.search.aggregations.InternalAggregation.Type;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds;
 import org.elasticsearch.search.aggregations.bucket.terms.support.IncludeExclude;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
@@ -39,7 +41,8 @@ import java.util.List;
 import java.util.Objects;
 
 public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource, TermsAggregationBuilder> {
-    public static final String NAME = StringTerms.TYPE.name();
+    public static final String NAME = "terms";
+    private static final InternalAggregation.Type TYPE = new Type("terms");
     public static final ParseField AGGREGATION_NAME_FIELD = new ParseField(NAME);
 
     public static final ParseField EXECUTION_HINT_FIELD_NAME = new ParseField("execution_hint");
@@ -62,14 +65,14 @@ public class TermsAggregationBuilder extends ValuesSourceAggregationBuilder<Valu
     private boolean showTermDocCountError = false;
 
     public TermsAggregationBuilder(String name, ValueType valueType) {
-        super(name, StringTerms.TYPE, ValuesSourceType.ANY, valueType);
+        super(name, TYPE, ValuesSourceType.ANY, valueType);
     }
 
     /**
      * Read from a stream.
      */
     public TermsAggregationBuilder(StreamInput in) throws IOException {
-        super(in, StringTerms.TYPE, ValuesSourceType.ANY);
+        super(in, TYPE, ValuesSourceType.ANY);
         bucketCountThresholds = new BucketCountThresholds(in);
         collectMode = in.readOptionalWriteable(SubAggCollectionMode::readFromStream);
         executionHint = in.readOptionalString();
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
index 8d5ff30c031..c01377d9761 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java
@@ -69,8 +69,8 @@ public class TermsAggregatorFactory extends ValuesSourceAggregatorFactory<Values
     @Override
     protected Aggregator createUnmapped(Aggregator parent, List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData)
             throws IOException {
-        final InternalAggregation aggregation = new UnmappedTerms(name, order, config.format(), bucketCountThresholds.getRequiredSize(),
-                bucketCountThresholds.getShardSize(), bucketCountThresholds.getMinDocCount(), pipelineAggregators, metaData);
+        final InternalAggregation aggregation = new UnmappedTerms(name, order, bucketCountThresholds.getRequiredSize(),
+                bucketCountThresholds.getMinDocCount(), pipelineAggregators, metaData);
         return new NonCollectingAggregator(name, context, parent, factories, pipelineAggregators, metaData) {
             {
                 // even in the case of an unmapped aggregator, validate the
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/UnmappedTerms.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/UnmappedTerms.java
index 4fe26db2e79..7f872b9f4ab 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/UnmappedTerms.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/UnmappedTerms.java
@@ -22,83 +22,69 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.search.DocValueFormat;
-import org.elasticsearch.search.aggregations.AggregationStreams;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
+import static java.util.Collections.emptyList;
+
 /**
- *
+ * Result of the {@link TermsAggregator} when the field is unmapped.
  */
-public class UnmappedTerms extends InternalTerms<UnmappedTerms, InternalTerms.Bucket> {
+public class UnmappedTerms extends InternalTerms<UnmappedTerms, UnmappedTerms.Bucket> {
+    public static final String NAME = "umterms";
 
-    public static final Type TYPE = new Type("terms", "umterms");
-
-    private static final List<Bucket> BUCKETS = Collections.emptyList();
-    private static final Map<String, Bucket> BUCKETS_MAP = Collections.emptyMap();
-
-    public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() {
-        @Override
-        public UnmappedTerms readResult(StreamInput in) throws IOException {
-            UnmappedTerms buckets = new UnmappedTerms();
-            buckets.readFrom(in);
-            return buckets;
+    /**
+     * Concrete type that can't be built because Java needs a concrent type so {@link InternalTerms.Bucket} can have a self type but
+     * {@linkplain UnmappedTerms} doesn't ever need to build it because it never returns any buckets.
+     */
+    protected abstract static class Bucket extends InternalTerms.Bucket<Bucket> {
+        private Bucket(long docCount, InternalAggregations aggregations, boolean showDocCountError, long docCountError,
+                DocValueFormat formatter) {
+            super(docCount, aggregations, showDocCountError, docCountError, formatter);
         }
-    };
-
-    public static void registerStreams() {
-        AggregationStreams.registerStream(STREAM, TYPE.stream());
     }
 
-    UnmappedTerms() {} // for serialization
-
-    public UnmappedTerms(String name, Terms.Order order, DocValueFormat format, int requiredSize, int shardSize, long minDocCount,
+    public UnmappedTerms(String name, Terms.Order order, int requiredSize, long minDocCount,
             List<PipelineAggregator> pipelineAggregators, Map<String, Object> metaData) {
-        super(name, order, format, requiredSize, shardSize, minDocCount, BUCKETS, false, 0, 0, pipelineAggregators, metaData);
+        super(name, order, requiredSize, minDocCount, pipelineAggregators, metaData);
+    }
+
+    /**
+     * Read from a stream.
+     */
+    public UnmappedTerms(StreamInput in) throws IOException {
+        super(in);
     }
 
     @Override
-    public Type type() {
-        return TYPE;
+    protected void writeTermTypeInfoTo(StreamOutput out) throws IOException {
     }
 
     @Override
-    public UnmappedTerms create(List<InternalTerms.Bucket> buckets) {
-        return new UnmappedTerms(this.name, this.order, this.format, this.requiredSize, this.shardSize, this.minDocCount, this.pipelineAggregators(), this.metaData);
+    public String getWriteableName() {
+        return NAME;
     }
 
     @Override
-    public InternalTerms.Bucket createBucket(InternalAggregations aggregations, InternalTerms.Bucket prototype) {
+    public UnmappedTerms create(List<Bucket> buckets) {
+        return new UnmappedTerms(name, order, requiredSize, minDocCount, pipelineAggregators(), metaData);
+    }
+
+    @Override
+    public Bucket createBucket(InternalAggregations aggregations, Bucket prototype) {
         throw new UnsupportedOperationException("not supported for UnmappedTerms");
     }
 
     @Override
-    protected UnmappedTerms create(String name, List<Bucket> buckets, long docCountError, long otherDocCount, InternalTerms prototype) {
+    protected UnmappedTerms create(String name, List<Bucket> buckets, long docCountError, long otherDocCount) {
         throw new UnsupportedOperationException("not supported for UnmappedTerms");
     }
 
-    @Override
-    protected void doReadFrom(StreamInput in) throws IOException {
-        this.docCountError = 0;
-        this.order = InternalOrder.Streams.readOrder(in);
-        this.requiredSize = readSize(in);
-        this.minDocCount = in.readVLong();
-        this.buckets = BUCKETS;
-        this.bucketMap = BUCKETS_MAP;
-    }
-
-    @Override
-    protected void doWriteTo(StreamOutput out) throws IOException {
-        InternalOrder.Streams.writeOrder(order, out);
-        writeSize(requiredSize, out);
-        out.writeVLong(minDocCount);
-    }
-
     @Override
     public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
         for (InternalAggregation agg : aggregations) {
@@ -111,10 +97,43 @@ public class UnmappedTerms extends InternalTerms<UnmappedTerms, InternalTerms.Bu
 
     @Override
     public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
-        builder.field(InternalTerms.DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME, docCountError);
+        builder.field(InternalTerms.DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME, 0);
         builder.field(SUM_OF_OTHER_DOC_COUNTS, 0);
         builder.startArray(CommonFields.BUCKETS).endArray();
         return builder;
     }
 
+    @Override
+    protected void setDocCountError(long docCountError) {
+    }
+
+    @Override
+    protected int getShardSize() {
+        return 0;
+    }
+
+    @Override
+    public long getDocCountError() {
+        return 0;
+    }
+
+    @Override
+    public long getSumOfOtherDocCounts() {
+        return 0;
+    }
+
+    @Override
+    protected List<Bucket> getBucketsInternal() {
+        return emptyList();
+    }
+
+    @Override
+    public Bucket getBucketByKey(String term) {
+        return null;
+    }
+
+    @Override
+    protected Bucket[] createBucketsArray(int size) {
+        return new Bucket[size];
+    }
 }
diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/BucketPriorityQueue.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/BucketPriorityQueue.java
index 4d2205cd524..489f2d676fc 100644
--- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/BucketPriorityQueue.java
+++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/BucketPriorityQueue.java
@@ -23,17 +23,17 @@ import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 
 import java.util.Comparator;
 
-public class BucketPriorityQueue extends PriorityQueue<Terms.Bucket> {
+public class BucketPriorityQueue<B extends Terms.Bucket> extends PriorityQueue<B> {
 
-    private final Comparator<Terms.Bucket> comparator;
+    private final Comparator<? super B> comparator;
 
-    public BucketPriorityQueue(int size, Comparator<Terms.Bucket> comparator) {
+    public BucketPriorityQueue(int size, Comparator<? super B> comparator) {
         super(size);
         this.comparator = comparator;
     }
 
     @Override
-    protected boolean lessThan(Terms.Bucket a, Terms.Bucket b) {
+    protected boolean lessThan(B a, B b) {
         return comparator.compare(a, b) > 0; // reverse, since we reverse again when adding to a list
     }
 }
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java
index 6ed77fffcc8..adbc66c2202 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/DoubleTermsTests.java
@@ -48,6 +48,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -624,6 +625,32 @@ public class DoubleTermsTests extends AbstractTermsTestCase {
         }
     }
 
+    public void testPartiallyUnmappedWithFormat() throws Exception {
+        SearchResponse response = client().prepareSearch("idx_unmapped", "idx").setTypes("type")
+                .addAggregation(terms("terms")
+                        .field(SINGLE_VALUED_FIELD_NAME)
+                        .collectMode(randomFrom(SubAggCollectionMode.values()))
+                        .format("0000.00"))
+                .execute().actionGet();
+
+        assertSearchResponse(response);
+
+
+        Terms terms = response.getAggregations().get("terms");
+        assertThat(terms, notNullValue());
+        assertThat(terms.getName(), equalTo("terms"));
+        assertThat(terms.getBuckets().size(), equalTo(5));
+
+        for (int i = 0; i < 5; i++) {
+            String key = String.format(Locale.ROOT, "%07.2f", (double) i);
+            Terms.Bucket bucket = terms.getBucketByKey(key);
+            assertThat(bucket, notNullValue());
+            assertThat(key(bucket), equalTo(key));
+            assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i));
+            assertThat(bucket.getDocCount(), equalTo(1L));
+        }
+    }
+
     public void testEmptyAggregation() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx")
                 .setQuery(matchAllQuery())
diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java
index fa049429a3a..7bb1ed6fd65 100644
--- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java
+++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/LongTermsTests.java
@@ -47,6 +47,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -629,6 +630,32 @@ public class LongTermsTests extends AbstractTermsTestCase {
         }
     }
 
+    public void testPartiallyUnmappedWithFormat() throws Exception {
+        SearchResponse response = client().prepareSearch("idx_unmapped", "idx").setTypes("type")
+                .addAggregation(terms("terms")
+                        .field(SINGLE_VALUED_FIELD_NAME)
+                        .collectMode(randomFrom(SubAggCollectionMode.values()))
+                        .format("0000"))
+                .execute().actionGet();
+
+        assertSearchResponse(response);
+
+
+        Terms terms = response.getAggregations().get("terms");
+        assertThat(terms, notNullValue());
+        assertThat(terms.getName(), equalTo("terms"));
+        assertThat(terms.getBuckets().size(), equalTo(5));
+
+        for (int i = 0; i < 5; i++) {
+            String key = String.format(Locale.ROOT, "%04d", i);
+            Terms.Bucket bucket = terms.getBucketByKey(key);
+            assertThat(bucket, notNullValue());
+            assertThat(key(bucket), equalTo(key));
+            assertThat(bucket.getKeyAsNumber().intValue(), equalTo(i));
+            assertThat(bucket.getDocCount(), equalTo(1L));
+        }
+    }
+
     public void testEmptyAggregation() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx")
                 .setQuery(matchAllQuery())

From 81fcdfcee97141a475da99680f863fdce5faebed Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Thu, 30 Jun 2016 10:39:45 -0400
Subject: [PATCH 1283/1311] Expose task information from NodeClient

This exposes a method to start an action and return a task from
`NodeClient`. This allows reindex to use the injected `Client` rather
than require injecting `TransportAction`s
---
 .../resources/checkstyle_suppressions.xml     |  1 -
 .../action/support/TransportAction.java       |  4 ++
 .../elasticsearch/client/node/NodeClient.java | 46 +++++++++++++++++--
 .../AbstractBaseReindexRestHandler.java       | 19 ++++----
 .../AbstractBulkByQueryRestHandler.java       |  7 ++-
 ...kIndexByScrollResponseContentListener.java |  6 +--
 .../reindex/RestDeleteByQueryAction.java      |  8 ++--
 .../index/reindex/RestReindexAction.java      |  8 ++--
 .../reindex/RestUpdateByQueryAction.java      |  8 ++--
 9 files changed, 72 insertions(+), 35 deletions(-)

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
index 882d12a305f..f5fc4046e66 100644
--- a/buildSrc/src/main/resources/checkstyle_suppressions.xml
+++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -233,7 +233,6 @@
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]bootstrap[/\\]Security.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]ElasticsearchClient.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]FilterClient.java" checks="LineLength" />
-  <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]node[/\\]NodeClient.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]support[/\\]AbstractClient.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]transport[/\\]TransportClient.java" checks="LineLength" />
   <suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]client[/\\]transport[/\\]support[/\\]TransportProxyClient.java" checks="LineLength" />
diff --git a/core/src/main/java/org/elasticsearch/action/support/TransportAction.java b/core/src/main/java/org/elasticsearch/action/support/TransportAction.java
index 60bd6100089..03408dab77f 100644
--- a/core/src/main/java/org/elasticsearch/action/support/TransportAction.java
+++ b/core/src/main/java/org/elasticsearch/action/support/TransportAction.java
@@ -101,6 +101,10 @@ public abstract class TransportAction<Request extends ActionRequest<Request>, Re
         return task;
     }
 
+    /**
+     * Execute the transport action on the local node, returning the {@link Task} used to track its execution and accepting a
+     * {@link TaskListener} which listens for the completion of the action.
+     */
     public final Task execute(Request request, TaskListener<Response> listener) {
         Task task = taskManager.register("transport", actionName, request);
         execute(task, request, new ActionListener<Response>() {
diff --git a/core/src/main/java/org/elasticsearch/client/node/NodeClient.java b/core/src/main/java/org/elasticsearch/client/node/NodeClient.java
index 59909df428b..e68b902e259 100644
--- a/core/src/main/java/org/elasticsearch/client/node/NodeClient.java
+++ b/core/src/main/java/org/elasticsearch/client/node/NodeClient.java
@@ -26,14 +26,17 @@ import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.GenericAction;
 import org.elasticsearch.action.support.TransportAction;
+import org.elasticsearch.client.Client;
 import org.elasticsearch.client.support.AbstractClient;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskListener;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.Map;
 
 /**
- *
+ * Client that executes actions on the local node.
  */
 public class NodeClient extends AbstractClient {
 
@@ -52,10 +55,43 @@ public class NodeClient extends AbstractClient {
         // nothing really to do
     }
 
-    @SuppressWarnings("unchecked")
     @Override
-    public <Request extends ActionRequest<Request>, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void doExecute(
-            Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
+    public <    Request extends ActionRequest<Request>,
+                Response extends ActionResponse,
+                RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>
+            > void doExecute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
+        // Discard the task because the Client interface doesn't use it.
+        executeLocally(action, request, listener);
+    }
+
+    /**
+     * Execute an {@link Action} locally, returning that {@link Task} used to track it, and linking an {@link ActionListener}. Prefer this
+     * method if you don't need access to the task when listening for the response. This is the method used to implement the {@link Client}
+     * interface.
+     */
+    public <    Request extends ActionRequest<Request>,
+                Response extends ActionResponse
+            > Task executeLocally(GenericAction<Request, Response> action, Request request, ActionListener<Response> listener) {
+        return transportAction(action).execute(request, listener);
+    }
+
+    /**
+     * Execute an {@link Action} locally, returning that {@link Task} used to track it, and linking an {@link TaskListener}. Prefer this
+     * method if you need access to the task when listening for the response.
+     */
+    public <    Request extends ActionRequest<Request>,
+                Response extends ActionResponse
+            > Task executeLocally(GenericAction<Request, Response> action, Request request, TaskListener<Response> listener) {
+        return transportAction(action).execute(request, listener);
+    }
+
+    /**
+     * Get the {@link TransportAction} for an {@link Action}, throwing exceptions if the action isn't available.
+     */
+    @SuppressWarnings("unchecked")
+    private <    Request extends ActionRequest<Request>,
+                Response extends ActionResponse
+            > TransportAction<Request, Response> transportAction(GenericAction<Request, Response> action) {
         if (actions == null) {
             throw new IllegalStateException("NodeClient has not been initialized");
         }
@@ -63,6 +99,6 @@ public class NodeClient extends AbstractClient {
         if (transportAction == null) {
             throw new IllegalStateException("failed to find action [" + action + "] to execute");
         }
-        transportAction.execute(request, listener);
+        return transportAction;
     }
 }
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
index 048e4208fea..7cf1aed2406 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBaseReindexRestHandler.java
@@ -20,9 +20,9 @@
 package org.elasticsearch.index.reindex;
 
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.GenericAction;
 import org.elasticsearch.action.WriteConsistencyLevel;
-import org.elasticsearch.action.support.TransportAction;
-import org.elasticsearch.client.Client;
+import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -43,18 +43,18 @@ import java.util.Map;
 
 public abstract class AbstractBaseReindexRestHandler<
                 Request extends AbstractBulkByScrollRequest<Request>,
-                TA extends TransportAction<Request, BulkIndexByScrollResponse>
+                A extends GenericAction<Request, BulkIndexByScrollResponse>
             > extends BaseRestHandler {
 
     protected final IndicesQueriesRegistry indicesQueriesRegistry;
     protected final AggregatorParsers aggParsers;
     protected final Suggesters suggesters;
     private final ClusterService clusterService;
-    private final TA action;
+    private final A action;
 
     protected AbstractBaseReindexRestHandler(Settings settings, IndicesQueriesRegistry indicesQueriesRegistry,
                                              AggregatorParsers aggParsers, Suggesters suggesters,
-                                             ClusterService clusterService, TA action) {
+                                             ClusterService clusterService, A action) {
         super(settings);
         this.indicesQueriesRegistry = indicesQueriesRegistry;
         this.aggParsers = aggParsers;
@@ -63,9 +63,8 @@ public abstract class AbstractBaseReindexRestHandler<
         this.action = action;
     }
 
-    public void handleRequest(RestRequest request, RestChannel channel,
+    protected void handleRequest(RestRequest request, RestChannel channel, NodeClient client,
                                  boolean includeCreated, boolean includeUpdated) throws IOException {
-
         // Build the internal request
         Request internal = setCommonOptions(request, buildRequest(request));
 
@@ -75,14 +74,14 @@ public abstract class AbstractBaseReindexRestHandler<
             params.put(BulkByScrollTask.Status.INCLUDE_CREATED, Boolean.toString(includeCreated));
             params.put(BulkByScrollTask.Status.INCLUDE_UPDATED, Boolean.toString(includeUpdated));
 
-            action.execute(internal, new BulkIndexByScrollResponseContentListener<>(channel, params));
+            client.executeLocally(action, internal, new BulkIndexByScrollResponseContentListener(channel, params));
             return;
         } else {
             internal.setShouldPersistResult(true);
         }
 
         /*
-         * Lets try and validate before forking so the user gets some error. The
+         * Let's try and validate before forking so the user gets some error. The
          * task can't totally validate until it starts but this is better than
          * nothing.
          */
@@ -91,7 +90,7 @@ public abstract class AbstractBaseReindexRestHandler<
             channel.sendResponse(new BytesRestResponse(channel, validationException));
             return;
         }
-        sendTask(channel, action.execute(internal, LoggingTaskListener.instance()));
+        sendTask(channel, client.executeLocally(action, internal, LoggingTaskListener.instance()));
     }
 
     /**
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java
index 9e4d8fc6d4e..41f103698d6 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/AbstractBulkByQueryRestHandler.java
@@ -19,9 +19,8 @@
 
 package org.elasticsearch.index.reindex;
 
+import org.elasticsearch.action.GenericAction;
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.support.TransportAction;
-import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
@@ -48,11 +47,11 @@ import static org.elasticsearch.index.reindex.AbstractBulkByScrollRequest.SIZE_A
  */
 public abstract class AbstractBulkByQueryRestHandler<
         Request extends AbstractBulkByScrollRequest<Request>,
-        TA extends TransportAction<Request, BulkIndexByScrollResponse>> extends AbstractBaseReindexRestHandler<Request, TA> {
+        A extends GenericAction<Request, BulkIndexByScrollResponse>> extends AbstractBaseReindexRestHandler<Request, A> {
 
     protected AbstractBulkByQueryRestHandler(Settings settings, IndicesQueriesRegistry indicesQueriesRegistry,
                                              AggregatorParsers aggParsers, Suggesters suggesters, ClusterService clusterService,
-                                             TA action) {
+                                             A action) {
         super(settings, indicesQueriesRegistry, aggParsers, suggesters, clusterService, action);
     }
 
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java
index 6cfba3a302d..cc4096f9421 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/BulkIndexByScrollResponseContentListener.java
@@ -35,7 +35,7 @@ import java.util.Map;
 /**
  * RestBuilderListener that returns higher than 200 status if there are any failures and allows to set XContent.Params.
  */
-public class BulkIndexByScrollResponseContentListener<R extends BulkIndexByScrollResponse> extends RestBuilderListener<R> {
+public class BulkIndexByScrollResponseContentListener extends RestBuilderListener<BulkIndexByScrollResponse> {
 
     private final Map<String, String> params;
 
@@ -45,14 +45,14 @@ public class BulkIndexByScrollResponseContentListener<R extends BulkIndexByScrol
     }
 
     @Override
-    public RestResponse buildResponse(R response, XContentBuilder builder) throws Exception {
+    public RestResponse buildResponse(BulkIndexByScrollResponse response, XContentBuilder builder) throws Exception {
         builder.startObject();
         response.toXContent(builder, new ToXContent.DelegatingMapParams(params, channel.request()));
         builder.endObject();
         return new BytesRestResponse(getStatus(response), builder);
     }
 
-    private RestStatus getStatus(R response) {
+    private RestStatus getStatus(BulkIndexByScrollResponse response) {
         /*
          * Return the highest numbered rest status under the assumption that higher numbered statuses are "more error" and thus more
          * interesting to the user.
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java
index 7a1f466c3c0..704bcf2dbaf 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestDeleteByQueryAction.java
@@ -39,13 +39,13 @@ import java.util.function.Consumer;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
-public class RestDeleteByQueryAction extends AbstractBulkByQueryRestHandler<DeleteByQueryRequest, TransportDeleteByQueryAction> {
+public class RestDeleteByQueryAction extends AbstractBulkByQueryRestHandler<DeleteByQueryRequest, DeleteByQueryAction> {
 
     @Inject
     public RestDeleteByQueryAction(Settings settings, RestController controller,
                                    IndicesQueriesRegistry indicesQueriesRegistry, AggregatorParsers aggParsers, Suggesters suggesters,
-                                   ClusterService clusterService, TransportDeleteByQueryAction action) {
-        super(settings, indicesQueriesRegistry, aggParsers, suggesters, clusterService, action);
+                                   ClusterService clusterService) {
+        super(settings, indicesQueriesRegistry, aggParsers, suggesters, clusterService, DeleteByQueryAction.INSTANCE);
         controller.registerHandler(POST, "/{index}/_delete_by_query", this);
         controller.registerHandler(POST, "/{index}/{type}/_delete_by_query", this);
     }
@@ -55,7 +55,7 @@ public class RestDeleteByQueryAction extends AbstractBulkByQueryRestHandler<Dele
         if (false == request.hasContent()) {
             throw new ElasticsearchException("_delete_by_query requires a request body");
         }
-        handleRequest(request, channel, false, false);
+        handleRequest(request, channel, client, false, false);
     }
 
     @Override
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
index 3bbfebf68a7..d40c1ea6622 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestReindexAction.java
@@ -64,7 +64,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
 /**
  * Expose reindex over rest.
  */
-public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexRequest, TransportReindexAction> {
+public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexRequest, ReindexAction> {
     static final ObjectParser<ReindexRequest, ReindexParseContext> PARSER = new ObjectParser<>("reindex");
     private static final Pattern HOST_PATTERN = Pattern.compile("(?<scheme>[^:]+)://(?<host>[^:]+):(?<port>\\d+)");
 
@@ -113,8 +113,8 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexReq
     @Inject
     public RestReindexAction(Settings settings, RestController controller,
             IndicesQueriesRegistry indicesQueriesRegistry, AggregatorParsers aggParsers, Suggesters suggesters,
-            ClusterService clusterService, TransportReindexAction action) {
-        super(settings, indicesQueriesRegistry, aggParsers, suggesters, clusterService, action);
+            ClusterService clusterService) {
+        super(settings, indicesQueriesRegistry, aggParsers, suggesters, clusterService, ReindexAction.INSTANCE);
         controller.registerHandler(POST, "/_reindex", this);
     }
 
@@ -123,7 +123,7 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexReq
         if (false == request.hasContent()) {
             throw new ElasticsearchException("_reindex requires a request body");
         }
-        handleRequest(request, channel, true, true);
+        handleRequest(request, channel, client, true, true);
     }
 
     @Override
diff --git a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
index 81bc0b5d27f..62a49ea0734 100644
--- a/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
+++ b/modules/reindex/src/main/java/org/elasticsearch/index/reindex/RestUpdateByQueryAction.java
@@ -39,20 +39,20 @@ import java.util.function.Consumer;
 
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
-public class RestUpdateByQueryAction extends AbstractBulkByQueryRestHandler<UpdateByQueryRequest, TransportUpdateByQueryAction> {
+public class RestUpdateByQueryAction extends AbstractBulkByQueryRestHandler<UpdateByQueryRequest, UpdateByQueryAction> {
 
     @Inject
     public RestUpdateByQueryAction(Settings settings, RestController controller,
             IndicesQueriesRegistry indicesQueriesRegistry, AggregatorParsers aggParsers, Suggesters suggesters,
-            ClusterService clusterService, TransportUpdateByQueryAction action) {
-        super(settings, indicesQueriesRegistry, aggParsers, suggesters, clusterService, action);
+            ClusterService clusterService) {
+        super(settings, indicesQueriesRegistry, aggParsers, suggesters, clusterService, UpdateByQueryAction.INSTANCE);
         controller.registerHandler(POST, "/{index}/_update_by_query", this);
         controller.registerHandler(POST, "/{index}/{type}/_update_by_query", this);
     }
 
     @Override
     public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception {
-        handleRequest(request, channel, false, true);
+        handleRequest(request, channel, client, false, true);
     }
 
     @Override

From d7099f05b920edff23586b9b83ad4d28f3dd843d Mon Sep 17 00:00:00 2001
From: Glen Smith <github@smithsrock.com>
Date: Thu, 7 Jul 2016 20:46:18 -0400
Subject: [PATCH 1284/1311] slight clarification

---
 docs/reference/indices/shrink-index.asciidoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/reference/indices/shrink-index.asciidoc b/docs/reference/indices/shrink-index.asciidoc
index 2975ef903f3..237b46dfc00 100644
--- a/docs/reference/indices/shrink-index.asciidoc
+++ b/docs/reference/indices/shrink-index.asciidoc
@@ -2,8 +2,8 @@
 == Shrink Index
 
 The shrink index API allows you to shrink an existing index into a new index
-with fewer primary shards. The number of primary shards in the target index
-must be a factor of the shards in the source index. For example an index with
+with fewer primary shards. The requested number of primary shards in the target index
+must be a factor of the number of shards in the source index. For example an index with
 `8` primary shards can be shrunk into `4`, `2` or `1` primary shards or an index
 with `15` primary shards can be shrunk into `5`, `3` or `1`. If the number
 of shards in the index is a prime number it can only be shrunk into a single

From e6be4af0140590dcfd3e18ca40da4f16c94daa66 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Thu, 7 Jul 2016 17:56:12 -0700
Subject: [PATCH 1285/1311] Plugins: Add RepositoryPlugin interface for
 registering snapshot repositories

Repository plugins currently use a lot of custom classes like
RepositoryName and RepositorySettings in order to use guice to construct
repository implementations. But repositories now only really need their
settings to be constructed. Anything else they need (eg a cloud client)
can be constructed within the plugin, instead of via guice.

This change makes repository plugins use the new pull model. It removes
guice from the construction of Repository objects (no more child
injectors) and also from all repository plugins.
---
 .../java/org/elasticsearch/node/Node.java     |   3 +-
 .../RepositoryPlugin.java}                    |  38 +-
 .../repositories/RepositoriesModule.java      |  36 +-
 .../repositories/RepositoriesService.java     | 130 +++----
 .../repositories/Repository.java              |  22 +-
 .../repositories/RepositoryName.java          |  71 ----
 .../repositories/RepositoryTypesRegistry.java |  47 ---
 .../blobstore/BlobStoreRepository.java        |  82 ++--
 .../repositories/fs/FsRepository.java         |  40 +-
 .../repositories/uri/URLRepository.java       |  28 +-
 .../index/shard/IndexShardTests.java          |   5 +
 ...ClusterStateServiceRandomUpdatesTests.java |   2 +-
 .../snapshots/mockstore/MockRepository.java   |  63 ++--
 .../cloud/azure/AzureRepositoryModule.java    |  57 ---
 .../cloud/azure/blobstore/AzureBlobStore.java |  16 +-
 .../azure/storage/AzureStorageService.java    |   2 -
 .../storage/AzureStorageServiceImpl.java      |  47 +--
 .../azure/storage/AzureStorageSettings.java   |  14 +-
 .../azure/AzureRepositoryPlugin.java          |  32 +-
 .../repositories/azure/AzureRepository.java   |  72 ++--
 ...ctAzureRepositoryServiceIntegTestCase.java |  14 +-
 .../storage/AzureStorageServiceMock.java      |  21 +-
 .../storage/AzureStorageServiceTests.java     |  12 -
 .../AzureStorageSettingsFilterTests.java      |   2 +-
 .../azure/AzureBlobStoreTests.java            |   8 +-
 .../azure/AzureSnapshotRestoreTests.java      |   2 +-
 .../gcs/GoogleCloudStorageModule.java         |  31 --
 .../gcs/GoogleCloudStoragePlugin.java         |  20 +-
 .../gcs/GoogleCloudStorageRepository.java     |  34 +-
 .../gcs/GoogleCloudStorageService.java        |   5 +-
 ...eCloudStorageBlobStoreRepositoryTests.java |  20 +-
 .../repositories/hdfs/HdfsPlugin.java         |  13 +-
 .../repositories/hdfs/HdfsRepository.java     |  31 +-
 .../cloud/aws/InternalAwsS3Service.java       |   1 -
 .../org/elasticsearch/cloud/aws/S3Module.java |  38 --
 .../repository/s3/S3RepositoryPlugin.java     |  34 +-
 .../repositories/s3/S3Repository.java         |  68 ++--
 .../cloud/aws/AbstractAwsTestCase.java        |   2 +-
 .../cloud/aws/RepositoryS3SettingsTests.java  | 353 ------------------
 .../cloud/aws/TestAwsS3Service.java           |   9 +-
 .../s3/AbstractS3SnapshotRestoreTest.java     |   4 +-
 41 files changed, 419 insertions(+), 1110 deletions(-)
 rename core/src/main/java/org/elasticsearch/{repositories/RepositorySettings.java => plugins/RepositoryPlugin.java} (50%)
 delete mode 100644 core/src/main/java/org/elasticsearch/repositories/RepositoryName.java
 delete mode 100644 core/src/main/java/org/elasticsearch/repositories/RepositoryTypesRegistry.java
 delete mode 100644 plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/AzureRepositoryModule.java
 delete mode 100644 plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStorageModule.java
 delete mode 100644 plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/S3Module.java
 delete mode 100644 plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java

diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index c3c64818fb0..bfdf5e58084 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -97,6 +97,7 @@ import org.elasticsearch.plugins.IngestPlugin;
 import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.plugins.RepositoryPlugin;
 import org.elasticsearch.plugins.ScriptPlugin;
 import org.elasticsearch.repositories.RepositoriesModule;
 import org.elasticsearch.rest.RestController;
@@ -291,7 +292,7 @@ public class Node implements Closeable {
                     clusterModule.getIndexNameExpressionResolver(), settingsModule.getClusterSettings(),
                     pluginsService.filterPlugins(ActionPlugin.class)));
             modules.add(new GatewayModule());
-            modules.add(new RepositoriesModule());
+            modules.add(new RepositoriesModule(environment, pluginsService.filterPlugins(RepositoryPlugin.class)));
             pluginsService.processModules(modules);
             CircuitBreakerService circuitBreakerService = createCircuitBreakerService(settingsModule.getSettings(),
                 settingsModule.getClusterSettings());
diff --git a/core/src/main/java/org/elasticsearch/repositories/RepositorySettings.java b/core/src/main/java/org/elasticsearch/plugins/RepositoryPlugin.java
similarity index 50%
rename from core/src/main/java/org/elasticsearch/repositories/RepositorySettings.java
rename to core/src/main/java/org/elasticsearch/plugins/RepositoryPlugin.java
index 222d606df28..9306ee37076 100644
--- a/core/src/main/java/org/elasticsearch/repositories/RepositorySettings.java
+++ b/core/src/main/java/org/elasticsearch/plugins/RepositoryPlugin.java
@@ -17,30 +17,28 @@
  * under the License.
  */
 
-package org.elasticsearch.repositories;
+package org.elasticsearch.plugins;
 
-import org.elasticsearch.common.settings.Settings;
+import java.util.Collections;
+import java.util.Map;
+
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.repositories.Repository;
 
 /**
- * Combines repository-specific settings with global settings
+ * An extension point for {@link Plugin} implementations to add custom snapshot repositories.
  */
+public interface RepositoryPlugin {
 
-public class RepositorySettings {
-
-    private final Settings globalSettings;
-
-    private final Settings settings;
-
-    public RepositorySettings(Settings globalSettings, Settings settings) {
-        this.globalSettings = globalSettings;
-        this.settings = settings;
-    }
-
-    public Settings globalSettings() {
-        return globalSettings;
-    }
-
-    public Settings settings() {
-        return settings;
+    /**
+     * Returns repository types added by this plugin.
+     *
+     * @param env The environment for the local node, which may be used for the local settings and path.repo
+     *
+     * The key of the returned {@link Map} is the type name of the repository and
+     * the value is a factory to construct the {@link Repository} interface.
+     */
+    default Map<String, Repository.Factory> getRepositories(Environment env) {
+        return Collections.emptyMap();
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java b/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java
index 0ba9cccddaf..50ab90b4fe1 100644
--- a/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java
+++ b/core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java
@@ -19,8 +19,17 @@
 
 package org.elasticsearch.repositories;
 
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 import org.elasticsearch.action.admin.cluster.snapshots.status.TransportNodesSnapshotsStatus;
 import org.elasticsearch.common.inject.AbstractModule;
+import org.elasticsearch.common.inject.binder.LinkedBindingBuilder;
+import org.elasticsearch.common.inject.multibindings.MapBinder;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.plugins.RepositoryPlugin;
 import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.repositories.uri.URLRepository;
 import org.elasticsearch.snapshots.RestoreService;
@@ -29,21 +38,25 @@ import org.elasticsearch.snapshots.SnapshotsService;
 
 /**
  * Sets up classes for Snapshot/Restore.
- *
- * Plugins can add custom repository types by calling {@link #registerRepository(String, Class)}.
  */
 public class RepositoriesModule extends AbstractModule {
 
-    private final RepositoryTypesRegistry repositoryTypes = new RepositoryTypesRegistry();
+    private final Map<String, Repository.Factory> repositoryTypes;
 
-    public RepositoriesModule() {
-        registerRepository(FsRepository.TYPE, FsRepository.class);
-        registerRepository(URLRepository.TYPE, URLRepository.class);
-    }
+    public RepositoriesModule(Environment env, List<RepositoryPlugin> repoPlugins) {
+        Map<String, Repository.Factory> factories = new HashMap<>();
+        factories.put(FsRepository.TYPE, (metadata) -> new FsRepository(metadata, env));
+        factories.put(URLRepository.TYPE, (metadata) -> new URLRepository(metadata, env));
 
-    /** Registers a custom repository type to the given {@link Repository}. */
-    public void registerRepository(String type, Class<? extends Repository> repositoryType) {
-        repositoryTypes.registerRepository(type, repositoryType);
+        for (RepositoryPlugin repoPlugin : repoPlugins) {
+            Map<String, Repository.Factory> newRepoTypes = repoPlugin.getRepositories(env);
+            for (Map.Entry<String, Repository.Factory> entry : newRepoTypes.entrySet()) {
+                if (factories.put(entry.getKey(), entry.getValue()) != null) {
+                    throw new IllegalArgumentException("Repository type [" + entry.getKey() + "] is already registered");
+                }
+            }
+        }
+        repositoryTypes = Collections.unmodifiableMap(factories);
     }
 
     @Override
@@ -53,6 +66,7 @@ public class RepositoriesModule extends AbstractModule {
         bind(SnapshotShardsService.class).asEagerSingleton();
         bind(TransportNodesSnapshotsStatus.class).asEagerSingleton();
         bind(RestoreService.class).asEagerSingleton();
-        bind(RepositoryTypesRegistry.class).toInstance(repositoryTypes);
+        MapBinder<String, Repository.Factory> typesBinder = MapBinder.newMapBinder(binder(), String.class, Repository.Factory.class);
+        repositoryTypes.forEach((k, v) -> typesBinder.addBinding(k).toInstance(v));
     }
 }
diff --git a/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
index 6efba054c40..076853fd75d 100644
--- a/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
+++ b/core/src/main/java/org/elasticsearch/repositories/RepositoriesService.java
@@ -33,8 +33,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.ModulesBuilder;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.snapshots.RestoreService;
@@ -44,35 +42,30 @@ import org.elasticsearch.transport.TransportService;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.unmodifiableMap;
-import static org.elasticsearch.common.settings.Settings.Builder.EMPTY_SETTINGS;
-
 /**
  * Service responsible for maintaining and providing access to snapshot repositories on nodes.
  */
 public class RepositoriesService extends AbstractComponent implements ClusterStateListener {
 
-    private final RepositoryTypesRegistry typesRegistry;
-
-    private final Injector injector;
+    private final Map<String, Repository.Factory> typesRegistry;
 
     private final ClusterService clusterService;
 
     private final VerifyNodeRepositoryAction verifyAction;
 
-    private volatile Map<String, RepositoryHolder> repositories = emptyMap();
+    private volatile Map<String, Repository> repositories = Collections.emptyMap();
 
     @Inject
-    public RepositoriesService(Settings settings, ClusterService clusterService, TransportService transportService, RepositoryTypesRegistry typesRegistry, Injector injector) {
+    public RepositoriesService(Settings settings, ClusterService clusterService, TransportService transportService,
+                               Map<String, Repository.Factory> typesRegistry) {
         super(settings);
         this.typesRegistry = typesRegistry;
-        this.injector = injector;
         this.clusterService = clusterService;
         // Doesn't make sense to maintain repositories on non-master and non-data nodes
         // Nothing happens there anyway
@@ -270,49 +263,53 @@ public class RepositoriesService extends AbstractComponent implements ClusterSta
 
             logger.trace("processing new index repositories for state version [{}]", event.state().version());
 
-            Map<String, RepositoryHolder> survivors = new HashMap<>();
+            Map<String, Repository> survivors = new HashMap<>();
             // First, remove repositories that are no longer there
-            for (Map.Entry<String, RepositoryHolder> entry : repositories.entrySet()) {
+            for (Map.Entry<String, Repository> entry : repositories.entrySet()) {
                 if (newMetaData == null || newMetaData.repository(entry.getKey()) == null) {
                     logger.debug("unregistering repository [{}]", entry.getKey());
-                    closeRepository(entry.getKey(), entry.getValue());
+                    closeRepository(entry.getValue());
                 } else {
                     survivors.put(entry.getKey(), entry.getValue());
                 }
             }
 
-            Map<String, RepositoryHolder> builder = new HashMap<>();
+            Map<String, Repository> builder = new HashMap<>();
             if (newMetaData != null) {
                 // Now go through all repositories and update existing or create missing
                 for (RepositoryMetaData repositoryMetaData : newMetaData.repositories()) {
-                    RepositoryHolder holder = survivors.get(repositoryMetaData.name());
-                    if (holder != null) {
+                    Repository repository = survivors.get(repositoryMetaData.name());
+                    if (repository != null) {
                         // Found previous version of this repository
-                        if (!holder.type.equals(repositoryMetaData.type()) || !holder.settings.equals(repositoryMetaData.settings())) {
+                        RepositoryMetaData previousMetadata = repository.getMetadata();
+                        if (previousMetadata.type().equals(repositoryMetaData.type()) == false
+                            || previousMetadata.settings().equals(repositoryMetaData.settings()) == false) {
                             // Previous version is different from the version in settings
                             logger.debug("updating repository [{}]", repositoryMetaData.name());
-                            closeRepository(repositoryMetaData.name(), holder);
-                            holder = null;
+                            closeRepository(repository);
+                            repository = null;
                             try {
-                                holder = createRepositoryHolder(repositoryMetaData);
+                                repository = createRepository(repositoryMetaData);
                             } catch (RepositoryException ex) {
+                                // TODO: this catch is bogus, it means the old repo is already closed,
+                                // but we have nothing to replace it
                                 logger.warn("failed to change repository [{}]", ex, repositoryMetaData.name());
                             }
                         }
                     } else {
                         try {
-                            holder = createRepositoryHolder(repositoryMetaData);
+                            repository = createRepository(repositoryMetaData);
                         } catch (RepositoryException ex) {
                             logger.warn("failed to create repository [{}]", ex, repositoryMetaData.name());
                         }
                     }
-                    if (holder != null) {
+                    if (repository != null) {
                         logger.debug("registering repository [{}]", repositoryMetaData.name());
-                        builder.put(repositoryMetaData.name(), holder);
+                        builder.put(repositoryMetaData.name(), repository);
                     }
                 }
             }
-            repositories = unmodifiableMap(builder);
+            repositories = Collections.unmodifiableMap(builder);
         } catch (Exception ex) {
             logger.warn("failure updating cluster state ", ex);
         }
@@ -323,16 +320,16 @@ public class RepositoriesService extends AbstractComponent implements ClusterSta
      * <p>
      * This method is called only on the master node
      *
-     * @param repository repository name
+     * @param repositoryName repository name
      * @return registered repository
      * @throws RepositoryMissingException if repository with such name isn't registered
      */
-    public Repository repository(String repository) {
-        RepositoryHolder holder = repositories.get(repository);
-        if (holder != null) {
-            return holder.repository;
+    public Repository repository(String repositoryName) {
+        Repository repository = repositories.get(repositoryName);
+        if (repository != null) {
+            return repository;
         }
-        throw new RepositoryMissingException(repository);
+        throw new RepositoryMissingException(repositoryName);
     }
 
     /**
@@ -346,55 +343,44 @@ public class RepositoriesService extends AbstractComponent implements ClusterSta
      * @return {@code true} if new repository was added or {@code false} if it was ignored
      */
     private boolean registerRepository(RepositoryMetaData repositoryMetaData) throws IOException {
-        RepositoryHolder previous = repositories.get(repositoryMetaData.name());
+        Repository previous = repositories.get(repositoryMetaData.name());
         if (previous != null) {
-            if (!previous.type.equals(repositoryMetaData.type()) && previous.settings.equals(repositoryMetaData.settings())) {
+            RepositoryMetaData previousMetadata = previous.getMetadata();
+            if (!previousMetadata.type().equals(repositoryMetaData.type()) && previousMetadata.settings().equals(repositoryMetaData.settings())) {
                 // Previous version is the same as this one - ignore it
                 return false;
             }
         }
-        RepositoryHolder holder = createRepositoryHolder(repositoryMetaData);
+        Repository newRepo = createRepository(repositoryMetaData);
         if (previous != null) {
-            // Closing previous version
-            closeRepository(repositoryMetaData.name(), previous);
+            closeRepository(previous);
         }
-        Map<String, RepositoryHolder> newRepositories = new HashMap<>(repositories);
-        newRepositories.put(repositoryMetaData.name(), holder);
+        Map<String, Repository> newRepositories = new HashMap<>(repositories);
+        newRepositories.put(repositoryMetaData.name(), newRepo);
+        repositories = newRepositories;
         return true;
     }
 
-    /**
-     * Closes the repository
-     *
-     * @param name   repository name
-     * @param holder repository holder
-     */
-    private void closeRepository(String name, RepositoryHolder holder) throws IOException {
-        logger.debug("closing repository [{}][{}]", holder.type, name);
-        if (holder.repository != null) {
-            holder.repository.close();
-        }
+    /** Closes the given repository. */
+    private void closeRepository(Repository repository) throws IOException {
+        logger.debug("closing repository [{}][{}]", repository.getMetadata().type(), repository.getMetadata().name());
+        repository.close();
     }
 
     /**
      * Creates repository holder
      */
-    private RepositoryHolder createRepositoryHolder(RepositoryMetaData repositoryMetaData) {
+    private Repository createRepository(RepositoryMetaData repositoryMetaData) {
         logger.debug("creating repository [{}][{}]", repositoryMetaData.type(), repositoryMetaData.name());
-        Injector repositoryInjector = null;
+        Repository.Factory factory = typesRegistry.get(repositoryMetaData.type());
+        if (factory == null) {
+            throw new RepositoryException(repositoryMetaData.name(),
+                "repository type [" + repositoryMetaData.type() + "] does not exist");
+        }
         try {
-            ModulesBuilder modules = new ModulesBuilder();
-            RepositoryName name = new RepositoryName(repositoryMetaData.type(), repositoryMetaData.name());
-            modules.add(b -> {
-                b.bind(RepositoryName.class).toInstance(name);
-                typesRegistry.bindType(b, repositoryMetaData.type());
-                b.bind(RepositorySettings.class).toInstance(new RepositorySettings(settings, repositoryMetaData.settings()));
-            });
-
-            repositoryInjector = modules.createChildInjector(injector);
-            Repository repository = repositoryInjector.getInstance(Repository.class);
+            Repository repository = factory.create(repositoryMetaData);
             repository.start();
-            return new RepositoryHolder(repositoryMetaData.type(), repositoryMetaData.settings(), repository);
+            return repository;
         } catch (Exception e) {
             logger.warn("failed to create repository [{}][{}]", e, repositoryMetaData.type(), repositoryMetaData.name());
             throw new RepositoryException(repositoryMetaData.name(), "failed to create repository", e);
@@ -448,22 +434,6 @@ public class RepositoriesService extends AbstractComponent implements ClusterSta
         }
     }
 
-    /**
-     * Internal data structure for holding repository with its configuration information and injector
-     */
-    private static class RepositoryHolder {
-
-        private final String type;
-        private final Settings settings;
-        private final Repository repository;
-
-        public RepositoryHolder(String type, Settings settings,Repository repository) {
-            this.type = type;
-            this.settings = settings;
-            this.repository = repository;
-        }
-    }
-
     /**
      * Register repository request
      */
@@ -477,7 +447,7 @@ public class RepositoriesService extends AbstractComponent implements ClusterSta
 
         final boolean verify;
 
-        Settings settings = EMPTY_SETTINGS;
+        Settings settings = Settings.EMPTY;
 
         /**
          * Constructs new register repository request
diff --git a/core/src/main/java/org/elasticsearch/repositories/Repository.java b/core/src/main/java/org/elasticsearch/repositories/Repository.java
index 2230f046eba..11a060d73e8 100644
--- a/core/src/main/java/org/elasticsearch/repositories/Repository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/Repository.java
@@ -21,7 +21,10 @@ package org.elasticsearch.repositories;
 import org.apache.lucene.index.IndexCommit;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.snapshots.SnapshotId;
@@ -37,7 +40,7 @@ import java.util.List;
 /**
  * An interface for interacting with a repository in snapshot and restore.
  * <p>
- * Implementations are responsible for reading and writing both metadata and actual shard data to and from
+ * Implementations are responsible for reading and writing both metadata and shard data to and from
  * a repository backend.
  * <p>
  * To perform a snapshot:
@@ -51,6 +54,23 @@ import java.util.List;
  */
 public interface Repository extends LifecycleComponent {
 
+    /**
+     * An factory interface for constructing repositories.
+     * See {@link org.elasticsearch.plugins.RepositoryPlugin}.
+     */
+    interface Factory {
+        /**
+         * Constructs a repository.
+         * @param metadata    metadata for the repository including name and settings
+         */
+        Repository create(RepositoryMetaData metadata) throws Exception;
+    }
+
+    /**
+     * Returns metadata about this repository.
+     */
+    RepositoryMetaData getMetadata();
+
     /**
      * Reads snapshot description from repository.
      *
diff --git a/core/src/main/java/org/elasticsearch/repositories/RepositoryName.java b/core/src/main/java/org/elasticsearch/repositories/RepositoryName.java
deleted file mode 100644
index 7cef6a6bb8a..00000000000
--- a/core/src/main/java/org/elasticsearch/repositories/RepositoryName.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.repositories;
-
-/**
- * Combines together the name and type of the repository
- */
-public class RepositoryName {
-
-    private final String type;
-
-    private final String name;
-
-    public RepositoryName(String type, String name) {
-        this.type = type;
-        this.name = name;
-    }
-
-    public String type() {
-        return this.type;
-    }
-
-    public String getType() {
-        return type();
-    }
-
-    public String name() {
-        return this.name;
-    }
-
-    public String getName() {
-        return name();
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-
-        RepositoryName that = (RepositoryName) o;
-
-        if (name != null ? !name.equals(that.name) : that.name != null) return false;
-        if (type != null ? !type.equals(that.type) : that.type != null) return false;
-
-        return true;
-    }
-
-    @Override
-    public int hashCode() {
-        int result = type != null ? type.hashCode() : 0;
-        result = 31 * result + (name != null ? name.hashCode() : 0);
-        return result;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/repositories/RepositoryTypesRegistry.java b/core/src/main/java/org/elasticsearch/repositories/RepositoryTypesRegistry.java
deleted file mode 100644
index 63a790b77fd..00000000000
--- a/core/src/main/java/org/elasticsearch/repositories/RepositoryTypesRegistry.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.repositories;
-
-import org.elasticsearch.common.inject.Binder;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.ExtensionPoint;
-
-/**
- * A mapping from type name to implementations of {@link Repository}.
- */
-public class RepositoryTypesRegistry {
-    // invariant: repositories and shardRepositories have the same keyset
-    private final ExtensionPoint.SelectedType<Repository> repositoryTypes =
-        new ExtensionPoint.SelectedType<>("repository", Repository.class);
-
-    /** Adds a new repository type to the registry, bound to the given implementation classes. */
-    public void registerRepository(String name, Class<? extends Repository> repositoryType) {
-        repositoryTypes.registerExtension(name, repositoryType);
-    }
-
-    /**
-     * Looks up the given type and binds the implementation into the given binder.
-     * Throws an {@link IllegalArgumentException} if the given type does not exist.
-     */
-    public void bindType(Binder binder, String type) {
-        Settings settings = Settings.builder().put("type", type).build();
-        repositoryTypes.bindType(binder, settings, "type", null);
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
index 173937ce69c..0f93c6e6c53 100644
--- a/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
@@ -37,11 +37,13 @@ import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Numbers;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.store.InputStreamIndexInput;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.iterable.Iterables;
 import org.elasticsearch.index.shard.IndexShard;
 import org.elasticsearch.index.snapshots.IndexShardRestoreFailedException;
@@ -83,7 +85,6 @@ import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.repositories.RepositoryException;
-import org.elasticsearch.repositories.RepositorySettings;
 import org.elasticsearch.repositories.RepositoryVerificationException;
 import org.elasticsearch.snapshots.SnapshotCreationException;
 import org.elasticsearch.snapshots.SnapshotException;
@@ -154,7 +155,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
 
     private BlobContainer snapshotsBlobContainer;
 
-    protected final String repositoryName;
+    protected final RepositoryMetaData metadata;
 
     private static final int BUFFER_SIZE = 4096;
 
@@ -225,16 +226,16 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
     /**
      * Constructs new BlobStoreRepository
      *
-     * @param repositoryName       repository name
-     * @param repositorySettings   repository settings
+     * @param metadata       The metadata for this repository including name and settings
+     * @param globalSettings Settings for the node this repository object is created on
      */
-    protected BlobStoreRepository(String repositoryName, RepositorySettings repositorySettings) {
-        super(repositorySettings.globalSettings());
-        this.repositoryName = repositoryName;
+    protected BlobStoreRepository(RepositoryMetaData metadata, Settings globalSettings) {
+        super(globalSettings);
+        this.metadata = metadata;
         parseFieldMatcher = new ParseFieldMatcher(settings);
-        snapshotRateLimiter = getRateLimiter(repositorySettings, "max_snapshot_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB));
-        restoreRateLimiter = getRateLimiter(repositorySettings, "max_restore_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB));
-        readOnly = repositorySettings.settings().getAsBoolean("readonly", false);
+        snapshotRateLimiter = getRateLimiter(metadata.settings(), "max_snapshot_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB));
+        restoreRateLimiter = getRateLimiter(metadata.settings(), "max_restore_bytes_per_sec", new ByteSizeValue(40, ByteSizeUnit.MB));
+        readOnly = metadata.settings().getAsBoolean("readonly", false);
         indexShardSnapshotFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_CODEC, SNAPSHOT_NAME_FORMAT, BlobStoreIndexShardSnapshot.PROTO, parseFieldMatcher, isCompress());
         indexShardSnapshotLegacyFormat = new LegacyBlobStoreFormat<>(LEGACY_SNAPSHOT_NAME_FORMAT, BlobStoreIndexShardSnapshot.PROTO, parseFieldMatcher);
         indexShardSnapshotsFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_INDEX_CODEC, SNAPSHOT_INDEX_NAME_FORMAT, BlobStoreIndexShardSnapshots.PROTO, parseFieldMatcher, isCompress());
@@ -299,37 +300,42 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
     }
 
     @Override
-    public void initializeSnapshot(SnapshotId snapshotId, List<String> indices, MetaData metaData) {
+    public RepositoryMetaData getMetadata() {
+        return metadata;
+    }
+
+    @Override
+    public void initializeSnapshot(SnapshotId snapshotId, List<String> indices, MetaData clusterMetadata) {
         if (isReadOnly()) {
-            throw new RepositoryException(this.repositoryName, "cannot create snapshot in a readonly repository");
+            throw new RepositoryException(metadata.name(), "cannot create snapshot in a readonly repository");
         }
         try {
             final String snapshotName = snapshotId.getName();
             // check if the snapshot name already exists in the repository
             if (getSnapshots().stream().anyMatch(s -> s.getName().equals(snapshotName))) {
-                throw new SnapshotCreationException(repositoryName, snapshotId, "snapshot with the same name already exists");
+                throw new SnapshotCreationException(metadata.name(), snapshotId, "snapshot with the same name already exists");
             }
             if (snapshotFormat.exists(snapshotsBlobContainer, blobId(snapshotId)) ||
                     snapshotLegacyFormat.exists(snapshotsBlobContainer, snapshotName)) {
-                throw new SnapshotCreationException(repositoryName, snapshotId, "snapshot with such name already exists");
+                throw new SnapshotCreationException(metadata.name(), snapshotId, "snapshot with such name already exists");
             }
             // Write Global MetaData
-            globalMetaDataFormat.write(metaData, snapshotsBlobContainer, snapshotName);
+            globalMetaDataFormat.write(clusterMetadata, snapshotsBlobContainer, snapshotName);
             for (String index : indices) {
-                final IndexMetaData indexMetaData = metaData.index(index);
+                final IndexMetaData indexMetaData = clusterMetadata.index(index);
                 final BlobPath indexPath = basePath().add("indices").add(index);
                 final BlobContainer indexMetaDataBlobContainer = blobStore().blobContainer(indexPath);
                 indexMetaDataFormat.write(indexMetaData, indexMetaDataBlobContainer, snapshotName);
             }
         } catch (IOException ex) {
-            throw new SnapshotCreationException(repositoryName, snapshotId, ex);
+            throw new SnapshotCreationException(metadata.name(), snapshotId, ex);
         }
     }
 
     @Override
     public void deleteSnapshot(SnapshotId snapshotId) {
         if (isReadOnly()) {
-            throw new RepositoryException(this.repositoryName, "cannot delete snapshot from a readonly repository");
+            throw new RepositoryException(metadata.name(), "cannot delete snapshot from a readonly repository");
         }
         List<String> indices = Collections.emptyList();
         SnapshotInfo snapshot = null;
@@ -391,7 +397,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
                 }
             }
         } catch (IOException ex) {
-            throw new RepositoryException(this.repositoryName, "failed to update snapshot in repository", ex);
+            throw new RepositoryException(metadata.name(), "failed to update snapshot in repository", ex);
         }
     }
 
@@ -420,7 +426,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
             }
             return blobStoreSnapshot;
         } catch (IOException ex) {
-            throw new RepositoryException(this.repositoryName, "failed to update snapshot in repository", ex);
+            throw new RepositoryException(metadata.name(), "failed to update snapshot in repository", ex);
         }
     }
 
@@ -432,7 +438,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
             // its a fresh repository, no index file exists, so return an empty list
             return Collections.emptyList();
         } catch (IOException ioe) {
-            throw new RepositoryException(repositoryName, "failed to list snapshots in repository", ioe);
+            throw new RepositoryException(metadata.name(), "failed to list snapshots in repository", ioe);
         }
     }
 
@@ -450,12 +456,12 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
             try {
                 return snapshotLegacyFormat.read(snapshotsBlobContainer, snapshotId.getName());
             } catch (FileNotFoundException | NoSuchFileException ex1) {
-                throw new SnapshotMissingException(repositoryName, snapshotId, ex);
+                throw new SnapshotMissingException(metadata.name(), snapshotId, ex);
             } catch (IOException | NotXContentException ex1) {
-                throw new SnapshotException(repositoryName, snapshotId, "failed to get snapshots", ex1);
+                throw new SnapshotException(metadata.name(), snapshotId, "failed to get snapshots", ex1);
             }
         } catch (IOException | NotXContentException ex) {
-            throw new SnapshotException(repositoryName, snapshotId, "failed to get snapshots", ex);
+            throw new SnapshotException(metadata.name(), snapshotId, "failed to get snapshots", ex);
         }
     }
 
@@ -468,17 +474,17 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
             if (globalMetaDataFormat.exists(snapshotsBlobContainer, snapshotId.getName())) {
                 snapshotVersion = Version.CURRENT;
             } else if (globalMetaDataLegacyFormat.exists(snapshotsBlobContainer, snapshotId.getName())) {
-                throw new SnapshotException(repositoryName, snapshotId, "snapshot is too old");
+                throw new SnapshotException(metadata.name(), snapshotId, "snapshot is too old");
             } else {
-                throw new SnapshotMissingException(repositoryName, snapshotId);
+                throw new SnapshotMissingException(metadata.name(), snapshotId);
             }
         }
         try {
             metaData = globalMetaDataFormat(snapshotVersion).read(snapshotsBlobContainer, snapshotId.getName());
         } catch (FileNotFoundException | NoSuchFileException ex) {
-            throw new SnapshotMissingException(repositoryName, snapshotId, ex);
+            throw new SnapshotMissingException(metadata.name(), snapshotId, ex);
         } catch (IOException ex) {
-            throw new SnapshotException(repositoryName, snapshotId, "failed to get snapshots", ex);
+            throw new SnapshotException(metadata.name(), snapshotId, "failed to get snapshots", ex);
         }
         MetaData.Builder metaDataBuilder = MetaData.builder(metaData);
         for (String index : indices) {
@@ -505,8 +511,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
      * @param defaultRate        default limiting rate
      * @return rate limiter or null of no throttling is needed
      */
-    private RateLimiter getRateLimiter(RepositorySettings repositorySettings, String setting, ByteSizeValue defaultRate) {
-        ByteSizeValue maxSnapshotBytesPerSec = repositorySettings.settings().getAsBytesSize(setting,
+    private RateLimiter getRateLimiter(Settings repositorySettings, String setting, ByteSizeValue defaultRate) {
+        ByteSizeValue maxSnapshotBytesPerSec = repositorySettings.getAsBytesSize(setting,
                 settings.getAsBytesSize(setting, defaultRate));
         if (maxSnapshotBytesPerSec.bytes() <= 0) {
             return null;
@@ -587,7 +593,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
                 return seed;
             }
         } catch (IOException exp) {
-            throw new RepositoryVerificationException(repositoryName, "path " + basePath() + " is not accessible on master node", exp);
+            throw new RepositoryVerificationException(metadata.name(), "path " + basePath() + " is not accessible on master node", exp);
         }
     }
 
@@ -599,7 +605,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         try {
             blobStore().delete(basePath().add(testBlobPrefix(seed)));
         } catch (IOException exp) {
-            throw new RepositoryVerificationException(repositoryName, "cannot delete test data at " + basePath(), exp);
+            throw new RepositoryVerificationException(metadata.name(), "cannot delete test data at " + basePath(), exp);
         }
     }
 
@@ -778,7 +784,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
             } catch (NumberFormatException nfe) {
                 // the index- blob wasn't of the format index-N where N is a number,
                 // no idea what this blob is but it doesn't belong in the repository!
-                logger.debug("[{}] Unknown blob in the repository: {}", repositoryName, blobName);
+                logger.debug("[{}] Unknown blob in the repository: {}", metadata.name(), blobName);
             }
         }
         return latest;
@@ -848,10 +854,10 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
             try  {
                 testBlobContainer.writeBlob("data-" + localNode.getId() + ".dat", new BytesArray(seed));
             } catch (IOException exp) {
-                throw new RepositoryVerificationException(repositoryName, "store location [" + blobStore() + "] is not accessible on the node [" + localNode + "]", exp);
+                throw new RepositoryVerificationException(metadata.name(), "store location [" + blobStore() + "] is not accessible on the node [" + localNode + "]", exp);
             }
         } else {
-            throw new RepositoryVerificationException(repositoryName, "a file written by master to the store [" + blobStore() + "] cannot be accessed on the node [" + localNode + "]. "
+            throw new RepositoryVerificationException(metadata.name(), "a file written by master to the store [" + blobStore() + "] cannot be accessed on the node [" + localNode + "]. "
                 + "This might indicate that the store [" + blobStore() + "] is not shared between this node and the master node or "
                 + "that permissions on the store don't allow reading files written by the master node");
         }
@@ -871,7 +877,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
     @Override
     public String toString() {
         return "BlobStoreRepository[" +
-            "[" + repositoryName +
+            "[" + metadata.name() +
             "], [" + blobStore() + ']' +
             ']';
     }
@@ -1121,7 +1127,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
          * @param snapshotIndexCommit snapshot commit point
          */
         public void snapshot(IndexCommit snapshotIndexCommit) {
-            logger.debug("[{}] [{}] snapshot to [{}] ...", shardId, snapshotId, repositoryName);
+            logger.debug("[{}] [{}] snapshot to [{}] ...", shardId, snapshotId, metadata.name());
             store.incRef();
             try {
                 final Map<String, BlobMetaData> blobs;
@@ -1409,7 +1415,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
         public void restore() throws IOException {
             store.incRef();
             try {
-                logger.debug("[{}] [{}] restoring to [{}] ...", snapshotId, repositoryName, shardId);
+                logger.debug("[{}] [{}] restoring to [{}] ...", snapshotId, metadata.name(), shardId);
                 BlobStoreIndexShardSnapshot snapshot = loadSnapshot();
 
                 if (snapshot.indexFiles().size() == 1
diff --git a/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java b/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
index 5be7115a25b..5888bd07c07 100644
--- a/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/fs/FsRepository.java
@@ -19,17 +19,15 @@
 
 package org.elasticsearch.repositories.fs;
 
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.blobstore.fs.FsBlobStore;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.repositories.RepositoryException;
-import org.elasticsearch.repositories.RepositoryName;
-import org.elasticsearch.repositories.RepositorySettings;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
 
 import java.io.IOException;
@@ -72,62 +70,48 @@ public class FsRepository extends BlobStoreRepository {
     private boolean compress;
 
     /**
-     * Constructs new shared file system repository
-     *
-     * @param name                 repository name
-     * @param repositorySettings   repository settings
+     * Constructs a shared file system repository.
      */
-    @Inject
-    public FsRepository(RepositoryName name, RepositorySettings repositorySettings, Environment environment) throws IOException {
-        super(name.getName(), repositorySettings);
-        Path locationFile;
-        String location = REPOSITORIES_LOCATION_SETTING.get(repositorySettings.settings());
+    public FsRepository(RepositoryMetaData metadata, Environment environment) throws IOException {
+        super(metadata, environment.settings());
+        String location = REPOSITORIES_LOCATION_SETTING.get(metadata.settings());
         if (location.isEmpty()) {
             logger.warn("the repository location is missing, it should point to a shared file system location that is available on all master and data nodes");
-            throw new RepositoryException(name.name(), "missing location");
+            throw new RepositoryException(metadata.name(), "missing location");
         }
-        locationFile = environment.resolveRepoFile(location);
+        Path locationFile = environment.resolveRepoFile(location);
         if (locationFile == null) {
             if (environment.repoFiles().length > 0) {
                 logger.warn("The specified location [{}] doesn't start with any repository paths specified by the path.repo setting: [{}] ", location, environment.repoFiles());
-                throw new RepositoryException(name.name(), "location [" + location + "] doesn't match any of the locations specified by path.repo");
+                throw new RepositoryException(metadata.name(), "location [" + location + "] doesn't match any of the locations specified by path.repo");
             } else {
                 logger.warn("The specified location [{}] should start with a repository path specified by the path.repo setting, but the path.repo setting was not set on this node", location);
-                throw new RepositoryException(name.name(), "location [" + location + "] doesn't match any of the locations specified by path.repo because this setting is empty");
+                throw new RepositoryException(metadata.name(), "location [" + location + "] doesn't match any of the locations specified by path.repo because this setting is empty");
             }
         }
 
         blobStore = new FsBlobStore(settings, locationFile);
-        if (CHUNK_SIZE_SETTING.exists(repositorySettings.settings())) {
-            this.chunkSize = CHUNK_SIZE_SETTING.get(repositorySettings.settings());
+        if (CHUNK_SIZE_SETTING.exists(metadata.settings())) {
+            this.chunkSize = CHUNK_SIZE_SETTING.get(metadata.settings());
         } else if (REPOSITORIES_CHUNK_SIZE_SETTING.exists(settings)) {
             this.chunkSize = REPOSITORIES_CHUNK_SIZE_SETTING.get(settings);
         } else {
             this.chunkSize = null;
         }
-        this.compress = COMPRESS_SETTING.exists(repositorySettings.settings()) ? COMPRESS_SETTING.get(repositorySettings.settings()) : REPOSITORIES_COMPRESS_SETTING.get(settings);
+        this.compress = COMPRESS_SETTING.exists(metadata.settings()) ? COMPRESS_SETTING.get(metadata.settings()) : REPOSITORIES_COMPRESS_SETTING.get(settings);
         this.basePath = BlobPath.cleanPath();
     }
 
-    /**
-     * {@inheritDoc}
-     */
     @Override
     protected BlobStore blobStore() {
         return blobStore;
     }
 
-    /**
-     * {@inheritDoc}
-     */
     @Override
     protected boolean isCompress() {
         return compress;
     }
 
-    /**
-     * {@inheritDoc}
-     */
     @Override
     protected ByteSizeValue chunkSize() {
         return chunkSize;
diff --git a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java b/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
index 4058baa63ef..5ca335c5953 100644
--- a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
+++ b/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java
@@ -19,17 +19,15 @@
 
 package org.elasticsearch.repositories.uri;
 
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.blobstore.url.URLBlobStore;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.util.URIPattern;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.repositories.RepositoryException;
-import org.elasticsearch.repositories.RepositoryName;
-import org.elasticsearch.repositories.RepositorySettings;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
 
 import java.io.IOException;
@@ -77,23 +75,19 @@ public class URLRepository extends BlobStoreRepository {
     private final BlobPath basePath;
 
     /**
-     * Constructs new read-only URL-based repository
-     *
-     * @param name                 repository name
-     * @param repositorySettings   repository settings
+     * Constructs a read-only URL-based repository
      */
-    @Inject
-    public URLRepository(RepositoryName name, RepositorySettings repositorySettings, Environment environment) throws IOException {
-        super(name.getName(), repositorySettings);
+    public URLRepository(RepositoryMetaData metadata, Environment environment) throws IOException {
+        super(metadata, environment.settings());
 
-        if (URL_SETTING.exists(repositorySettings.settings()) == false && REPOSITORIES_URL_SETTING.exists(settings) ==  false) {
-            throw new RepositoryException(name.name(), "missing url");
+        if (URL_SETTING.exists(metadata.settings()) == false && REPOSITORIES_URL_SETTING.exists(settings) ==  false) {
+            throw new RepositoryException(metadata.name(), "missing url");
         }
         supportedProtocols = SUPPORTED_PROTOCOLS_SETTING.get(settings);
         urlWhiteList = ALLOWED_URLS_SETTING.get(settings).toArray(new URIPattern[]{});
         this.environment = environment;
 
-        URL url = URL_SETTING.exists(repositorySettings.settings()) ? URL_SETTING.get(repositorySettings.settings()) : REPOSITORIES_URL_SETTING.get(settings);
+        URL url = URL_SETTING.exists(metadata.settings()) ? URL_SETTING.get(metadata.settings()) : REPOSITORIES_URL_SETTING.get(settings);
         URL normalizedURL = checkURL(url);
         blobStore = new URLBlobStore(settings, normalizedURL);
         basePath = BlobPath.cleanPath();
@@ -115,7 +109,7 @@ public class URLRepository extends BlobStoreRepository {
     private URL checkURL(URL url) {
         String protocol = url.getProtocol();
         if (protocol == null) {
-            throw new RepositoryException(repositoryName, "unknown url protocol from URL [" + url + "]");
+            throw new RepositoryException(getMetadata().name(), "unknown url protocol from URL [" + url + "]");
         }
         for (String supportedProtocol : supportedProtocols) {
             if (supportedProtocol.equals(protocol)) {
@@ -126,18 +120,18 @@ public class URLRepository extends BlobStoreRepository {
                     }
                 } catch (URISyntaxException ex) {
                     logger.warn("cannot parse the specified url [{}]", url);
-                    throw new RepositoryException(repositoryName, "cannot parse the specified url [" + url + "]");
+                    throw new RepositoryException(getMetadata().name(), "cannot parse the specified url [" + url + "]");
                 }
                 // We didn't match white list - try to resolve against path.repo
                 URL normalizedUrl = environment.resolveRepoURL(url);
                 if (normalizedUrl == null) {
                     logger.warn("The specified url [{}] doesn't start with any repository paths specified by the path.repo setting or by {} setting: [{}] ", url, ALLOWED_URLS_SETTING.getKey(), environment.repoFiles());
-                    throw new RepositoryException(repositoryName, "file url [" + url + "] doesn't match any of the locations specified by path.repo or " + ALLOWED_URLS_SETTING.getKey());
+                    throw new RepositoryException(getMetadata().name(), "file url [" + url + "] doesn't match any of the locations specified by path.repo or " + ALLOWED_URLS_SETTING.getKey());
                 }
                 return normalizedUrl;
             }
         }
-        throw new RepositoryException(repositoryName, "unsupported url protocol [" + protocol + "] from URL [" + url + "]");
+        throw new RepositoryException(getMetadata().name(), "unsupported url protocol [" + protocol + "] from URL [" + url + "]");
     }
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index 5bc84921a29..b2b6a657fda 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -72,6 +72,7 @@ import org.elasticsearch.cluster.InternalClusterInfoService;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.routing.AllocationId;
 import org.elasticsearch.cluster.routing.RestoreSource;
@@ -1653,6 +1654,10 @@ public class IndexShardTests extends ESSingleNodeTestCase {
         @Override
         protected void doClose() {}
         @Override
+        public RepositoryMetaData getMetadata() {
+            return null;
+        }
+        @Override
         public SnapshotInfo getSnapshotInfo(SnapshotId snapshotId) {
             return null;
         }
diff --git a/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
index ad20cb577bb..a26cf9124f2 100644
--- a/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java
@@ -270,7 +270,7 @@ public class IndicesClusterStateServiceRandomUpdatesTests extends AbstractIndice
         final TransportService transportService = new TransportService(Settings.EMPTY, null, threadPool);
         final ClusterService clusterService = mock(ClusterService.class);
         final RepositoriesService repositoriesService = new RepositoriesService(Settings.EMPTY, clusterService,
-            transportService, null, null);
+            transportService, null);
         final RecoveryTargetService recoveryTargetService = new RecoveryTargetService(Settings.EMPTY, threadPool,
             transportService, null, clusterService);
         final ShardStateAction shardStateAction = mock(ShardStateAction.class);
diff --git a/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java b/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
index 762be7a3684..09ca55fd2e5 100644
--- a/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
+++ b/core/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
@@ -26,6 +26,7 @@ import java.nio.file.Path;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
@@ -34,34 +35,34 @@ import java.util.concurrent.atomic.AtomicLong;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.PathUtils;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.repositories.RepositoriesModule;
-import org.elasticsearch.repositories.RepositoryName;
-import org.elasticsearch.repositories.RepositorySettings;
+import org.elasticsearch.plugins.RepositoryPlugin;
+import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.snapshots.SnapshotId;
 
 public class MockRepository extends FsRepository {
 
-    public static class Plugin extends org.elasticsearch.plugins.Plugin {
+    public static class Plugin extends org.elasticsearch.plugins.Plugin implements RepositoryPlugin {
 
         public static final Setting<String> USERNAME_SETTING = Setting.simpleString("secret.mock.username", Property.NodeScope);
         public static final Setting<String> PASSWORD_SETTING =
             Setting.simpleString("secret.mock.password", Property.NodeScope, Property.Filtered);
 
-        public void onModule(RepositoriesModule repositoriesModule) {
-            repositoriesModule.registerRepository("mock", MockRepository.class);
+
+        @Override
+        public Map<String, Repository.Factory> getRepositories(Environment env) {
+            return Collections.singletonMap("mock", (metadata) -> new MockRepository(metadata, env));
         }
 
         @Override
@@ -96,45 +97,41 @@ public class MockRepository extends FsRepository {
 
     private volatile boolean blocked = false;
 
-    @Inject
-    public MockRepository(RepositoryName name, RepositorySettings repositorySettings, ClusterService clusterService, Environment environment) throws IOException {
-        super(name, overrideSettings(repositorySettings, clusterService), environment);
-        randomControlIOExceptionRate = repositorySettings.settings().getAsDouble("random_control_io_exception_rate", 0.0);
-        randomDataFileIOExceptionRate = repositorySettings.settings().getAsDouble("random_data_file_io_exception_rate", 0.0);
-        maximumNumberOfFailures = repositorySettings.settings().getAsLong("max_failure_number", 100L);
-        blockOnControlFiles = repositorySettings.settings().getAsBoolean("block_on_control", false);
-        blockOnDataFiles = repositorySettings.settings().getAsBoolean("block_on_data", false);
-        blockOnInitialization = repositorySettings.settings().getAsBoolean("block_on_init", false);
-        randomPrefix = repositorySettings.settings().get("random", "default");
-        waitAfterUnblock = repositorySettings.settings().getAsLong("wait_after_unblock", 0L);
+    public MockRepository(RepositoryMetaData metadata, Environment environment) throws IOException {
+        super(overrideSettings(metadata, environment), environment);
+        randomControlIOExceptionRate = metadata.settings().getAsDouble("random_control_io_exception_rate", 0.0);
+        randomDataFileIOExceptionRate = metadata.settings().getAsDouble("random_data_file_io_exception_rate", 0.0);
+        maximumNumberOfFailures = metadata.settings().getAsLong("max_failure_number", 100L);
+        blockOnControlFiles = metadata.settings().getAsBoolean("block_on_control", false);
+        blockOnDataFiles = metadata.settings().getAsBoolean("block_on_data", false);
+        blockOnInitialization = metadata.settings().getAsBoolean("block_on_init", false);
+        randomPrefix = metadata.settings().get("random", "default");
+        waitAfterUnblock = metadata.settings().getAsLong("wait_after_unblock", 0L);
         logger.info("starting mock repository with random prefix {}", randomPrefix);
         mockBlobStore = new MockBlobStore(super.blobStore());
     }
 
     @Override
-    public void initializeSnapshot(SnapshotId snapshotId, List<String> indices, MetaData metaData) {
+    public void initializeSnapshot(SnapshotId snapshotId, List<String> indices, MetaData clusterMetadata) {
         if (blockOnInitialization ) {
             blockExecution();
         }
-        super.initializeSnapshot(snapshotId, indices, metaData);
+        super.initializeSnapshot(snapshotId, indices, clusterMetadata);
     }
 
-    private static RepositorySettings overrideSettings(RepositorySettings repositorySettings, ClusterService clusterService) {
-        if (repositorySettings.settings().getAsBoolean("localize_location", false)) {
-            return new RepositorySettings(
-                    repositorySettings.globalSettings(),
-                    localizeLocation(repositorySettings.settings(), clusterService));
+    private static RepositoryMetaData overrideSettings(RepositoryMetaData metadata, Environment environment) {
+        // TODO: use another method of testing not being able to read the test file written by the master...
+        // this is super duper hacky
+        if (metadata.settings().getAsBoolean("localize_location", false)) {
+            Path location = PathUtils.get(metadata.settings().get("location"));
+            location = location.resolve(Integer.toString(environment.hashCode()));
+            return new RepositoryMetaData(metadata.name(), metadata.type(),
+                Settings.builder().put(metadata.settings()).put("location", location.toAbsolutePath()).build());
         } else {
-            return repositorySettings;
+            return metadata;
         }
     }
 
-    private static Settings localizeLocation(Settings settings, ClusterService clusterService) {
-        Path location = PathUtils.get(settings.get("location"));
-        location = location.resolve(clusterService.localNode().getId());
-        return Settings.builder().put(settings).put("location", location.toAbsolutePath()).build();
-    }
-
     private long incrementAndGetFailureCount() {
         return failureCounter.incrementAndGet();
     }
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/AzureRepositoryModule.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/AzureRepositoryModule.java
deleted file mode 100644
index 92a87ff8ee1..00000000000
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/AzureRepositoryModule.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.azure;
-
-import org.elasticsearch.cloud.azure.storage.AzureStorageService;
-import org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl;
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.common.settings.Settings;
-
-/**
- * Azure Module
- *
- * <ul>
- * <li>If needed this module will bind azure repository service by default
- * to AzureStorageServiceImpl.</li>
- * </ul>
- *
- * @see org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl
- */
-public class AzureRepositoryModule extends AbstractModule {
-    protected final ESLogger logger;
-
-    // pkg private so it is settable by tests
-    static Class<? extends AzureStorageService> storageServiceImpl = AzureStorageServiceImpl.class;
-
-    @Inject
-    public AzureRepositoryModule(Settings settings) {
-        this.logger = Loggers.getLogger(getClass(), settings);
-    }
-
-    @Override
-    protected void configure() {
-        // If we have settings for azure repository, let's start the azure storage service
-        logger.debug("starting azure repository service");
-        bind(AzureStorageService.class).to(storageServiceImpl).asEagerSingleton();
-    }
-}
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
index 64193e0b2f9..b77cebfa2f2 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java
@@ -23,16 +23,14 @@ import com.microsoft.azure.storage.LocationMode;
 import com.microsoft.azure.storage.StorageException;
 import org.elasticsearch.cloud.azure.storage.AzureStorageService;
 import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage;
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.repositories.RepositoryName;
-import org.elasticsearch.repositories.RepositorySettings;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -53,17 +51,15 @@ public class AzureBlobStore extends AbstractComponent implements BlobStore {
     private final String container;
     private final String repositoryName;
 
-    @Inject
-    public AzureBlobStore(RepositoryName name, Settings settings, RepositorySettings repositorySettings,
+    public AzureBlobStore(RepositoryMetaData metadata, Settings settings,
                           AzureStorageService client) throws URISyntaxException, StorageException {
         super(settings);
         this.client = client;
-        client.start();
-        this.container = getValue(repositorySettings, Repository.CONTAINER_SETTING, Storage.CONTAINER_SETTING);
-        this.repositoryName = name.getName();
-        this.accountName = getValue(repositorySettings, Repository.ACCOUNT_SETTING, Storage.ACCOUNT_SETTING);
+        this.container = getValue(metadata.settings(), settings, Repository.CONTAINER_SETTING, Storage.CONTAINER_SETTING);
+        this.repositoryName = metadata.name();
+        this.accountName = getValue(metadata.settings(), settings, Repository.ACCOUNT_SETTING, Storage.ACCOUNT_SETTING);
 
-        String modeStr = getValue(repositorySettings, Repository.LOCATION_MODE_SETTING, Storage.LOCATION_MODE_SETTING);
+        String modeStr = getValue(metadata.settings(), settings, Repository.LOCATION_MODE_SETTING, Storage.LOCATION_MODE_SETTING);
         if (Strings.hasLength(modeStr)) {
             this.locMode = LocationMode.valueOf(modeStr.toUpperCase(Locale.ROOT));
         } else {
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
index 46972482a6a..3e854ab9c70 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageService.java
@@ -86,6 +86,4 @@ public interface AzureStorageService {
 
     void moveBlob(String account, LocationMode mode, String container, String sourceBlob, String targetBlob)
         throws URISyntaxException, StorageException;
-
-    void start();
 }
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
index 80e62b90ad3..17ff0780a50 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceImpl.java
@@ -33,6 +33,7 @@ import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.collect.Tuple;
+import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -45,15 +46,13 @@ import java.net.URISyntaxException;
 import java.util.HashMap;
 import java.util.Map;
 
-public class AzureStorageServiceImpl extends AbstractLifecycleComponent
-    implements AzureStorageService {
+public class AzureStorageServiceImpl extends AbstractComponent implements AzureStorageService {
 
     final AzureStorageSettings primaryStorageSettings;
     final Map<String, AzureStorageSettings> secondariesStorageSettings;
 
     final Map<String, CloudBlobClient> clients;
 
-    @Inject
     public AzureStorageServiceImpl(Settings settings) {
         super(settings);
 
@@ -62,6 +61,20 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent
         this.secondariesStorageSettings = storageSettings.v2();
 
         this.clients = new HashMap<>();
+
+        logger.debug("starting azure storage client instance");
+
+        // We register the primary client if any
+        if (primaryStorageSettings != null) {
+            logger.debug("registering primary client for account [{}]", primaryStorageSettings.getAccount());
+            createClient(primaryStorageSettings);
+        }
+
+        // We register all secondary clients
+        for (Map.Entry<String, AzureStorageSettings> azureStorageSettingsEntry : secondariesStorageSettings.entrySet()) {
+            logger.debug("registering secondary client for account [{}]", azureStorageSettingsEntry.getKey());
+            createClient(azureStorageSettingsEntry.getValue());
+        }
     }
 
     void createClient(AzureStorageSettings azureStorageSettings) {
@@ -302,32 +315,4 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent
             logger.debug("moveBlob container [{}], sourceBlob [{}], targetBlob [{}] -> done", container, sourceBlob, targetBlob);
         }
     }
-
-    @Override
-    protected void doStart() throws ElasticsearchException {
-        logger.debug("starting azure storage client instance");
-
-        // We register the primary client if any
-        if (primaryStorageSettings != null) {
-            logger.debug("registering primary client for account [{}]", primaryStorageSettings.getAccount());
-            createClient(primaryStorageSettings);
-        }
-
-        // We register all secondary clients
-        for (Map.Entry<String, AzureStorageSettings> azureStorageSettingsEntry : secondariesStorageSettings.entrySet()) {
-            logger.debug("registering secondary client for account [{}]", azureStorageSettingsEntry.getKey());
-            createClient(azureStorageSettingsEntry.getValue());
-        }
-    }
-
-    @Override
-    protected void doStop() throws ElasticsearchException {
-        logger.debug("stopping azure storage client instance");
-        // We should stop all clients but it does sound like CloudBlobClient has
-        // any shutdown method...
-    }
-
-    @Override
-    protected void doClose() throws ElasticsearchException {
-    }
 }
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java
index f64ffed4a03..6d1ed0c1049 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettings.java
@@ -25,7 +25,6 @@ import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.repositories.RepositorySettings;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -172,20 +171,21 @@ public final class AzureStorageSettings {
         return Collections.unmodifiableMap(secondaries);
     }
 
-    public static <T> T getValue(RepositorySettings repositorySettings,
+    public static <T> T getValue(Settings repositorySettings,
+                                 Settings globalSettings,
                                  Setting<T> repositorySetting,
                                  Setting<T> repositoriesSetting) {
-        if (repositorySetting.exists(repositorySettings.settings())) {
-            return repositorySetting.get(repositorySettings.settings());
+        if (repositorySetting.exists(repositorySettings)) {
+            return repositorySetting.get(repositorySettings);
         } else {
-            return repositoriesSetting.get(repositorySettings.globalSettings());
+            return repositoriesSetting.get(globalSettings);
         }
     }
 
-    public static <T> Setting<T> getEffectiveSetting(RepositorySettings repositorySettings,
+    public static <T> Setting<T> getEffectiveSetting(Settings repositorySettings,
                                               Setting<T> repositorySetting,
                                               Setting<T> repositoriesSetting) {
-        if (repositorySetting.exists(repositorySettings.settings())) {
+        if (repositorySetting.exists(repositorySettings)) {
             return repositorySetting;
         } else {
             return repositoriesSetting;
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java b/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
index e7701a828ac..fcd7bf96b2c 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/plugin/repository/azure/AzureRepositoryPlugin.java
@@ -20,42 +20,36 @@
 package org.elasticsearch.plugin.repository.azure;
 
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 
-import org.elasticsearch.cloud.azure.AzureRepositoryModule;
 import org.elasticsearch.cloud.azure.storage.AzureStorageService;
-import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.repositories.RepositoriesModule;
+import org.elasticsearch.plugins.RepositoryPlugin;
+import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.repositories.azure.AzureRepository;
 
 /**
- *
+ * A plugin to add a repository type that writes to and from the Azure cloud storage service.
  */
-public class AzureRepositoryPlugin extends Plugin {
+public class AzureRepositoryPlugin extends Plugin implements RepositoryPlugin {
 
-    private final Settings settings;
-    protected final ESLogger logger = Loggers.getLogger(AzureRepositoryPlugin.class);
-
-    public AzureRepositoryPlugin(Settings settings) {
-        this.settings = settings;
-        logger.trace("starting azure repository plugin...");
+    // overridable for tests
+    protected AzureStorageService createStorageService(Settings settings) {
+        return new AzureStorageServiceImpl(settings);
     }
 
     @Override
-    public Collection<Module> nodeModules() {
-        return Collections.singletonList((Module) new AzureRepositoryModule(settings));
-    }
-
-    public void onModule(RepositoriesModule module) {
-        logger.debug("registering repository type [{}]", AzureRepository.TYPE);
-        module.registerRepository(AzureRepository.TYPE, AzureRepository.class);
+    public Map<String, Repository.Factory> getRepositories(Environment env) {
+        return Collections.singletonMap(AzureRepository.TYPE,
+            (metadata) -> new AzureRepository(metadata, env, createStorageService(env.settings())));
     }
 
     @Override
diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
index 248c87ed604..4b4f7d6ae8e 100644
--- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
+++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
@@ -19,33 +19,33 @@
 
 package org.elasticsearch.repositories.azure;
 
-import com.microsoft.azure.storage.LocationMode;
-import com.microsoft.azure.storage.StorageException;
-import org.elasticsearch.cloud.azure.blobstore.AzureBlobStore;
-import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage;
-import org.elasticsearch.cluster.metadata.MetaData;
-import org.elasticsearch.snapshots.SnapshotId;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.blobstore.BlobPath;
-import org.elasticsearch.common.blobstore.BlobStore;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Setting.Property;
-import org.elasticsearch.common.settings.SettingsException;
-import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.repositories.RepositoryName;
-import org.elasticsearch.repositories.RepositorySettings;
-import org.elasticsearch.repositories.RepositoryVerificationException;
-import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
-import org.elasticsearch.snapshots.SnapshotCreationException;
-
 import java.io.IOException;
 import java.net.URISyntaxException;
 import java.util.List;
 import java.util.Locale;
 import java.util.function.Function;
 
+import com.microsoft.azure.storage.LocationMode;
+import com.microsoft.azure.storage.StorageException;
+import org.elasticsearch.cloud.azure.blobstore.AzureBlobStore;
+import org.elasticsearch.cloud.azure.storage.AzureStorageService;
+import org.elasticsearch.cloud.azure.storage.AzureStorageService.Storage;
+import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.blobstore.BlobPath;
+import org.elasticsearch.common.blobstore.BlobStore;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.settings.SettingsException;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.repositories.RepositoryVerificationException;
+import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
+import org.elasticsearch.snapshots.SnapshotCreationException;
+import org.elasticsearch.snapshots.SnapshotId;
+
 import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getEffectiveSetting;
 import static org.elasticsearch.cloud.azure.storage.AzureStorageSettings.getValue;
 
@@ -83,24 +83,22 @@ public class AzureRepository extends BlobStoreRepository {
     private final boolean compress;
     private final boolean readonly;
 
-    @Inject
-    public AzureRepository(RepositoryName name, RepositorySettings repositorySettings,
-                           AzureBlobStore azureBlobStore) throws IOException, URISyntaxException, StorageException {
-        super(name.getName(), repositorySettings);
+    public AzureRepository(RepositoryMetaData metadata, Environment environment, AzureStorageService storageService)
+        throws IOException, URISyntaxException, StorageException {
+        super(metadata, environment.settings());
 
-        String container = getValue(repositorySettings, Repository.CONTAINER_SETTING, Storage.CONTAINER_SETTING);
-
-        this.blobStore = azureBlobStore;
-        ByteSizeValue configuredChunkSize = getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Storage.CHUNK_SIZE_SETTING);
+        blobStore = new AzureBlobStore(metadata, environment.settings(), storageService);
+        String container = getValue(metadata.settings(), settings, Repository.CONTAINER_SETTING, Storage.CONTAINER_SETTING);
+        ByteSizeValue configuredChunkSize = getValue(metadata.settings(), settings, Repository.CHUNK_SIZE_SETTING, Storage.CHUNK_SIZE_SETTING);
         if (configuredChunkSize.getMb() > MAX_CHUNK_SIZE.getMb()) {
-            Setting<ByteSizeValue> setting = getEffectiveSetting(repositorySettings, Repository.CHUNK_SIZE_SETTING, Storage.CHUNK_SIZE_SETTING);
+            Setting<ByteSizeValue> setting = getEffectiveSetting(metadata.settings(), Repository.CHUNK_SIZE_SETTING, Storage.CHUNK_SIZE_SETTING);
             throw new SettingsException("["  + setting.getKey() + "] must not exceed [" + MAX_CHUNK_SIZE + "] but is set to [" + configuredChunkSize + "].");
         } else {
             this.chunkSize = configuredChunkSize;
         }
 
-        this.compress = getValue(repositorySettings, Repository.COMPRESS_SETTING, Storage.COMPRESS_SETTING);
-        String modeStr = getValue(repositorySettings, Repository.LOCATION_MODE_SETTING, Storage.LOCATION_MODE_SETTING);
+        this.compress = getValue(metadata.settings(), settings, Repository.COMPRESS_SETTING, Storage.COMPRESS_SETTING);
+        String modeStr = getValue(metadata.settings(), settings, Repository.LOCATION_MODE_SETTING, Storage.LOCATION_MODE_SETTING);
         if (Strings.hasLength(modeStr)) {
             LocationMode locationMode = LocationMode.valueOf(modeStr.toUpperCase(Locale.ROOT));
             readonly = locationMode == LocationMode.SECONDARY_ONLY;
@@ -108,7 +106,7 @@ public class AzureRepository extends BlobStoreRepository {
             readonly = false;
         }
 
-        String basePath = getValue(repositorySettings, Repository.BASE_PATH_SETTING, Storage.BASE_PATH_SETTING);
+        String basePath = getValue(metadata.settings(), settings, Repository.BASE_PATH_SETTING, Storage.BASE_PATH_SETTING);
 
         if (Strings.hasLength(basePath)) {
             // Remove starting / if any
@@ -155,16 +153,16 @@ public class AzureRepository extends BlobStoreRepository {
     }
 
     @Override
-    public void initializeSnapshot(SnapshotId snapshotId, List<String> indices, MetaData metaData) {
+    public void initializeSnapshot(SnapshotId snapshotId, List<String> indices, MetaData clusterMetadata) {
         try {
             if (!blobStore.doesContainerExist(blobStore.container())) {
                 logger.debug("container [{}] does not exist. Creating...", blobStore.container());
                 blobStore.createContainer(blobStore.container());
             }
-            super.initializeSnapshot(snapshotId, indices, metaData);
+            super.initializeSnapshot(snapshotId, indices, clusterMetadata);
         } catch (StorageException | URISyntaxException e) {
             logger.warn("can not initialize container [{}]: [{}]", blobStore.container(), e.getMessage());
-            throw new SnapshotCreationException(repositoryName, snapshotId, e);
+            throw new SnapshotCreationException(getMetadata().name(), snapshotId, e);
         }
     }
 
@@ -178,7 +176,7 @@ public class AzureRepository extends BlobStoreRepository {
                 }
             } catch (StorageException | URISyntaxException e) {
                 logger.warn("can not initialize container [{}]: [{}]", blobStore.container(), e.getMessage());
-                throw new RepositoryVerificationException(repositoryName, "can not initialize container " + blobStore.container(), e);
+                throw new RepositoryVerificationException(getMetadata().name(), "can not initialize container " + blobStore.container(), e);
             }
         }
         return super.startVerification();
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceIntegTestCase.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceIntegTestCase.java
index 82af834cd35..dd8251c3d2b 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceIntegTestCase.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/AbstractAzureRepositoryServiceIntegTestCase.java
@@ -40,9 +40,12 @@ import java.util.Collection;
 
 public abstract class AbstractAzureRepositoryServiceIntegTestCase extends AbstractAzureIntegTestCase {
 
-    public static class TestPlugin extends Plugin {
-        public void onModule(AzureRepositoryModule azureRepositoryModule) {
-            AzureRepositoryModule.storageServiceImpl = AzureStorageServiceMock.class;
+    private static final AzureStorageService storageService = new AzureStorageServiceMock();
+
+    public static class TestPlugin extends AzureRepositoryPlugin {
+        @Override
+        protected AzureStorageService createStorageService(Settings settings) {
+            return storageService;
         }
     }
 
@@ -78,7 +81,7 @@ public abstract class AbstractAzureRepositoryServiceIntegTestCase extends Abstra
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return pluginList(AzureRepositoryPlugin.class, TestPlugin.class, MockFSIndexStore.TestPlugin.class);
+        return pluginList(TestPlugin.class, MockFSIndexStore.TestPlugin.class);
     }
 
     @Override
@@ -104,7 +107,6 @@ public abstract class AbstractAzureRepositoryServiceIntegTestCase extends Abstra
     public void cleanRepositoryFiles(String path) throws StorageException, URISyntaxException {
         String container = internalCluster().getInstance(Settings.class).get("repositories.azure.container");
         logger.info("--> remove blobs in container [{}]", container);
-        AzureStorageService client = internalCluster().getInstance(AzureStorageService.class);
-        client.deleteFiles(null, LocationMode.PRIMARY_ONLY, container, path);
+        storageService.deleteFiles(null, LocationMode.PRIMARY_ONLY, container, path);
     }
 }
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java
index 21df62cd768..4ed365b5ac8 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java
@@ -25,6 +25,7 @@ import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.blobstore.BlobMetaData;
 import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
 import org.elasticsearch.common.collect.MapBuilder;
+import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
@@ -43,14 +44,12 @@ import java.util.concurrent.ConcurrentHashMap;
 /**
  * In memory storage for unit tests
  */
-public class AzureStorageServiceMock extends AbstractLifecycleComponent
-        implements AzureStorageService {
+public class AzureStorageServiceMock extends AbstractComponent implements AzureStorageService {
 
     protected Map<String, ByteArrayOutputStream> blobs = new ConcurrentHashMap<>();
 
-    @Inject
-    public AzureStorageServiceMock(Settings settings) {
-        super(settings);
+    public AzureStorageServiceMock() {
+        super(Settings.EMPTY);
     }
 
     @Override
@@ -124,18 +123,6 @@ public class AzureStorageServiceMock extends AbstractLifecycleComponent
         }
     }
 
-    @Override
-    protected void doStart() throws ElasticsearchException {
-    }
-
-    @Override
-    protected void doStop() throws ElasticsearchException {
-    }
-
-    @Override
-    protected void doClose() throws ElasticsearchException {
-    }
-
     /**
      * Test if the given String starts with the specified prefix,
      * ignoring upper/lower case.
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java
index ad2b14e368b..cf7f0cd1abb 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java
@@ -45,7 +45,6 @@ public class AzureStorageServiceTests extends ESTestCase {
 
     public void testGetSelectedClientWithNoPrimaryAndSecondary() {
         AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(Settings.EMPTY);
-        azureStorageService.doStart();
         try {
             azureStorageService.getSelectedClient("whatever", LocationMode.PRIMARY_ONLY);
             fail("we should have raised an IllegalArgumentException");
@@ -59,7 +58,6 @@ public class AzureStorageServiceTests extends ESTestCase {
             .put("cloud.azure.storage.azure1.account", "myaccount1")
             .put("cloud.azure.storage.azure1.key", "mykey1")
             .build());
-        azureStorageService.doStart();
         CloudBlobClient client = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY);
         assertThat(client.getEndpoint(), is(URI.create("https://azure1")));
     }
@@ -69,42 +67,36 @@ public class AzureStorageServiceTests extends ESTestCase {
             .put("cloud.azure.storage.azure1.account", "myaccount1")
             .put("cloud.azure.storage.azure1.key", "mykey1")
             .build());
-        azureStorageService.doStart();
         CloudBlobClient client = azureStorageService.getSelectedClient(null, LocationMode.PRIMARY_ONLY);
         assertThat(client.getEndpoint(), is(URI.create("https://azure1")));
     }
 
     public void testGetSelectedClientPrimary() {
         AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(settings);
-        azureStorageService.doStart();
         CloudBlobClient client = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY);
         assertThat(client.getEndpoint(), is(URI.create("https://azure1")));
     }
 
     public void testGetSelectedClientSecondary1() {
         AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(settings);
-        azureStorageService.doStart();
         CloudBlobClient client = azureStorageService.getSelectedClient("azure2", LocationMode.PRIMARY_ONLY);
         assertThat(client.getEndpoint(), is(URI.create("https://azure2")));
     }
 
     public void testGetSelectedClientSecondary2() {
         AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(settings);
-        azureStorageService.doStart();
         CloudBlobClient client = azureStorageService.getSelectedClient("azure3", LocationMode.PRIMARY_ONLY);
         assertThat(client.getEndpoint(), is(URI.create("https://azure3")));
     }
 
     public void testGetDefaultClientWithPrimaryAndSecondaries() {
         AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(settings);
-        azureStorageService.doStart();
         CloudBlobClient client = azureStorageService.getSelectedClient(null, LocationMode.PRIMARY_ONLY);
         assertThat(client.getEndpoint(), is(URI.create("https://azure1")));
     }
 
     public void testGetSelectedClientNonExisting() {
         AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(settings);
-        azureStorageService.doStart();
         try {
             azureStorageService.getSelectedClient("azure4", LocationMode.PRIMARY_ONLY);
             fail("we should have raised an IllegalArgumentException");
@@ -115,7 +107,6 @@ public class AzureStorageServiceTests extends ESTestCase {
 
     public void testGetSelectedClientDefault() {
         AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(settings);
-        azureStorageService.doStart();
         CloudBlobClient client = azureStorageService.getSelectedClient(null, LocationMode.PRIMARY_ONLY);
         assertThat(client.getEndpoint(), is(URI.create("https://azure1")));
     }
@@ -127,7 +118,6 @@ public class AzureStorageServiceTests extends ESTestCase {
                 .build();
 
         AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(timeoutSettings);
-        azureStorageService.doStart();
         CloudBlobClient client1 = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY);
         assertThat(client1.getDefaultRequestOptions().getTimeoutIntervalInMs(), is(10 * 1000));
         CloudBlobClient client3 = azureStorageService.getSelectedClient("azure3", LocationMode.PRIMARY_ONLY);
@@ -136,7 +126,6 @@ public class AzureStorageServiceTests extends ESTestCase {
 
     public void testGetSelectedClientDefaultTimeout() {
         AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(settings);
-        azureStorageService.doStart();
         CloudBlobClient client1 = azureStorageService.getSelectedClient("azure1", LocationMode.PRIMARY_ONLY);
         assertThat(client1.getDefaultRequestOptions().getTimeoutIntervalInMs(), nullValue());
         CloudBlobClient client3 = azureStorageService.getSelectedClient("azure3", LocationMode.PRIMARY_ONLY);
@@ -150,7 +139,6 @@ public class AzureStorageServiceTests extends ESTestCase {
             .build();
 
         AzureStorageServiceImpl azureStorageService = new AzureStorageServiceMock(timeoutSettings);
-        azureStorageService.doStart();
         CloudBlobClient client1 = azureStorageService.getSelectedClient("azure", LocationMode.PRIMARY_ONLY);
         assertThat(client1.getDefaultRequestOptions().getTimeoutIntervalInMs(), is(nullValue()));
     }
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java
index 4b74d2aa0d1..e82ed3a1875 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java
@@ -46,7 +46,7 @@ public class AzureStorageSettingsFilterTests extends ESTestCase {
             .build();
 
     public void testSettingsFiltering() throws IOException {
-        AzureRepositoryPlugin p = new AzureRepositoryPlugin(Settings.EMPTY);
+        AzureRepositoryPlugin p = new AzureRepositoryPlugin();
         SettingsModule module = new SettingsModule(Settings.EMPTY, p.getSettings(), p.getSettingsFilter());
         SettingsFilter settingsFilter = ModuleTestCase.bindAndGetInstance(module, SettingsFilter.class);
 
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java
index f2155e69e2d..ccd9bf8cf81 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java
@@ -23,11 +23,10 @@ import com.microsoft.azure.storage.StorageException;
 import org.elasticsearch.cloud.azure.blobstore.AzureBlobStore;
 import org.elasticsearch.cloud.azure.storage.AzureStorageService;
 import org.elasticsearch.cloud.azure.storage.AzureStorageServiceImpl;
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.repositories.ESBlobStoreTestCase;
-import org.elasticsearch.repositories.RepositoryName;
-import org.elasticsearch.repositories.RepositorySettings;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.io.IOException;
@@ -44,11 +43,10 @@ public class AzureBlobStoreTests extends ESBlobStoreTestCase {
     @Override
     protected BlobStore newBlobStore() throws IOException {
         try {
-            RepositoryName repositoryName = new RepositoryName("azure", "ittest");
             Settings settings = readSettingsFromFile();
-            RepositorySettings repositorySettings = new RepositorySettings(settings, Settings.builder().build());
+            RepositoryMetaData metadata = new RepositoryMetaData("ittest", "azure", Settings.EMPTY);
             AzureStorageService storageService = new AzureStorageServiceImpl(settings);
-            AzureBlobStore blobStore = new AzureBlobStore(repositoryName, settings, repositorySettings, storageService);
+            AzureBlobStore blobStore = new AzureBlobStore(metadata, settings, storageService);
             blobStore.createContainer(blobStore.container());
             return blobStore;
         } catch (URISyntaxException | StorageException e) {
diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java
index c062e765c32..014014b432c 100644
--- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java
+++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreTests.java
@@ -483,7 +483,7 @@ public class AzureSnapshotRestoreTests extends AbstractAzureWithThirdPartyIntegT
      */
     public void testRemoveAndCreateContainer() throws Exception {
         final String container = getContainerName().concat("-testremove");
-        final AzureStorageService storageService = internalCluster().getInstance(AzureStorageService.class);
+        final AzureStorageService storageService = new AzureStorageServiceImpl(internalCluster().getDefaultSettings());
 
         // It could happen that we run this test really close to a previous one
         // so we might need some time to be able to create the container
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStorageModule.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStorageModule.java
deleted file mode 100644
index 8a4bf88ed74..00000000000
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStorageModule.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.plugin.repository.gcs;
-
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.repositories.gcs.GoogleCloudStorageService;
-
-public class GoogleCloudStorageModule extends AbstractModule {
-
-    @Override
-    protected void configure() {
-        bind(GoogleCloudStorageService.class).to(GoogleCloudStorageService.InternalGoogleCloudStorageService.class).asEagerSingleton();
-    }
-}
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStoragePlugin.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStoragePlugin.java
index 33022ff449e..c0fa38e8b57 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStoragePlugin.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/plugin/repository/gcs/GoogleCloudStoragePlugin.java
@@ -23,6 +23,7 @@ import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Map;
 
 import com.google.api.client.auth.oauth2.TokenRequest;
 import com.google.api.client.auth.oauth2.TokenResponse;
@@ -40,11 +41,16 @@ import com.google.api.services.storage.model.Objects;
 import com.google.api.services.storage.model.StorageObject;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.RepositoryPlugin;
 import org.elasticsearch.repositories.RepositoriesModule;
+import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.repositories.gcs.GoogleCloudStorageRepository;
+import org.elasticsearch.repositories.gcs.GoogleCloudStorageService;
 
-public class GoogleCloudStoragePlugin extends Plugin {
+public class GoogleCloudStoragePlugin extends Plugin implements RepositoryPlugin {
 
     public static final String NAME = "repository-gcs";
 
@@ -108,12 +114,14 @@ public class GoogleCloudStoragePlugin extends Plugin {
         });
     }
 
-    @Override
-    public Collection<Module> nodeModules() {
-        return Collections.singletonList(new GoogleCloudStorageModule());
+    // overridable for tests
+    protected GoogleCloudStorageService createStorageService(Environment environment) {
+        return new GoogleCloudStorageService.InternalGoogleCloudStorageService(environment);
     }
 
-    public void onModule(RepositoriesModule repositoriesModule) {
-        repositoriesModule.registerRepository(GoogleCloudStorageRepository.TYPE, GoogleCloudStorageRepository.class);
+    @Override
+    public Map<String, Repository.Factory> getRepositories(Environment env) {
+        return Collections.singletonMap(GoogleCloudStorageRepository.TYPE,
+            (metadata) -> new GoogleCloudStorageRepository(metadata, env, createStorageService(env)));
     }
 }
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
index df2f054fa91..f7b74d5a4f8 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java
@@ -20,19 +20,18 @@
 package org.elasticsearch.repositories.gcs;
 
 import com.google.api.services.storage.Storage;
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.blobstore.gcs.GoogleCloudStorageBlobStore;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugin.repository.gcs.GoogleCloudStoragePlugin;
 import org.elasticsearch.repositories.RepositoryException;
-import org.elasticsearch.repositories.RepositoryName;
-import org.elasticsearch.repositories.RepositorySettings;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
 
 import java.util.function.Function;
@@ -72,16 +71,15 @@ public class GoogleCloudStorageRepository extends BlobStoreRepository {
     private final BlobPath basePath;
     private final GoogleCloudStorageBlobStore blobStore;
 
-    @Inject
-    public GoogleCloudStorageRepository(RepositoryName repositoryName, RepositorySettings repositorySettings,
+    public GoogleCloudStorageRepository(RepositoryMetaData metadata, Environment environment,
                                         GoogleCloudStorageService storageService) throws Exception {
-        super(repositoryName.getName(), repositorySettings);
+        super(metadata, environment.settings());
 
-        String bucket = get(BUCKET, repositoryName, repositorySettings);
-        String application = get(APPLICATION_NAME, repositoryName, repositorySettings);
-        String serviceAccount = get(SERVICE_ACCOUNT, repositoryName, repositorySettings);
+        String bucket = get(BUCKET, metadata);
+        String application = get(APPLICATION_NAME, metadata);
+        String serviceAccount = get(SERVICE_ACCOUNT, metadata);
 
-        String basePath = BASE_PATH.get(repositorySettings.settings());
+        String basePath = BASE_PATH.get(metadata.settings());
         if (Strings.hasLength(basePath)) {
             BlobPath path = new BlobPath();
             for (String elem : basePath.split("/")) {
@@ -95,18 +93,18 @@ public class GoogleCloudStorageRepository extends BlobStoreRepository {
         TimeValue connectTimeout = null;
         TimeValue readTimeout = null;
 
-        TimeValue timeout = HTTP_CONNECT_TIMEOUT.get(repositorySettings.settings());
+        TimeValue timeout = HTTP_CONNECT_TIMEOUT.get(metadata.settings());
         if ((timeout != null) && (timeout.millis() != NO_TIMEOUT.millis())) {
             connectTimeout = timeout;
         }
 
-        timeout = HTTP_READ_TIMEOUT.get(repositorySettings.settings());
+        timeout = HTTP_READ_TIMEOUT.get(metadata.settings());
         if ((timeout != null) && (timeout.millis() != NO_TIMEOUT.millis())) {
             readTimeout = timeout;
         }
 
-        this.compress = get(COMPRESS, repositoryName, repositorySettings);
-        this.chunkSize = get(CHUNK_SIZE, repositoryName, repositorySettings);
+        this.compress = get(COMPRESS, metadata);
+        this.chunkSize = get(CHUNK_SIZE, metadata);
 
         logger.debug("using bucket [{}], base_path [{}], chunk_size [{}], compress [{}], application [{}]",
                 bucket, basePath, chunkSize, compress, application);
@@ -139,13 +137,13 @@ public class GoogleCloudStorageRepository extends BlobStoreRepository {
     /**
      * Get a given setting from the repository settings, throwing a {@link RepositoryException} if the setting does not exist or is empty.
      */
-    static <T> T get(Setting<T> setting, RepositoryName name, RepositorySettings repositorySettings) {
-        T value = setting.get(repositorySettings.settings());
+    static <T> T get(Setting<T> setting, RepositoryMetaData metadata) {
+        T value = setting.get(metadata.settings());
         if (value == null) {
-            throw new RepositoryException(name.getName(), "Setting [" + setting.getKey() + "] is not defined for repository");
+            throw new RepositoryException(metadata.name(), "Setting [" + setting.getKey() + "] is not defined for repository");
         }
         if ((value instanceof String) && (Strings.hasText((String) value)) == false) {
-            throw new RepositoryException(name.getName(), "Setting [" + setting.getKey() + "] is empty for repository");
+            throw new RepositoryException(metadata.name(), "Setting [" + setting.getKey() + "] is empty for repository");
         }
         return value;
     }
diff --git a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
index 098ce5f1504..55e7813de42 100644
--- a/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
+++ b/plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java
@@ -67,9 +67,8 @@ public interface GoogleCloudStorageService {
 
         private final Environment environment;
 
-        @Inject
-        public InternalGoogleCloudStorageService(Settings settings, Environment environment) {
-            super(settings);
+        public InternalGoogleCloudStorageService(Environment environment) {
+            super(environment.settings());
             this.environment = environment;
         }
 
diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
index c5b57ba6cd6..f9548e7e2ea 100644
--- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
+++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java
@@ -21,18 +21,16 @@ package org.elasticsearch.repositories.gcs;
 
 import com.google.api.services.storage.Storage;
 import org.elasticsearch.common.blobstore.gcs.MockHttpTransport;
-import org.elasticsearch.common.inject.Module;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.plugin.repository.gcs.GoogleCloudStorageModule;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugin.repository.gcs.GoogleCloudStoragePlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.repositories.ESBlobStoreRepositoryIntegTestCase;
 import org.junit.BeforeClass;
 
 import java.util.Collection;
-import java.util.Collections;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -68,25 +66,13 @@ public class GoogleCloudStorageBlobStoreRepositoryTests extends ESBlobStoreRepos
     }
 
     public static class MockGoogleCloudStoragePlugin extends GoogleCloudStoragePlugin {
-
-        public MockGoogleCloudStoragePlugin() {
-        }
-
         @Override
-        public Collection<Module> nodeModules() {
-            return Collections.singletonList(new MockGoogleCloudStorageModule());
-        }
-    }
-
-    public static class MockGoogleCloudStorageModule extends GoogleCloudStorageModule {
-        @Override
-        protected void configure() {
-            bind(GoogleCloudStorageService.class).to(MockGoogleCloudStorageService.class).asEagerSingleton();
+        protected GoogleCloudStorageService createStorageService(Environment environment) {
+            return new MockGoogleCloudStorageService();
         }
     }
 
     public static class MockGoogleCloudStorageService implements GoogleCloudStorageService {
-
         @Override
         public Storage createClient(String serviceAccount, String application, TimeValue connectTimeout, TimeValue readTimeout) throws
                 Exception {
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java
index 3e8f484e3e7..d4af26c3bbc 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsPlugin.java
@@ -23,14 +23,18 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
+import java.util.Collections;
+import java.util.Map;
 
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.RepositoryPlugin;
 import org.elasticsearch.repositories.RepositoriesModule;
+import org.elasticsearch.repositories.Repository;
 
-// Code
-public final class HdfsPlugin extends Plugin {
+public final class HdfsPlugin extends Plugin implements RepositoryPlugin {
 
     // initialize some problematic classes with elevated privileges
     static {
@@ -83,7 +87,8 @@ public final class HdfsPlugin extends Plugin {
         return null;
     }
 
-    public void onModule(RepositoriesModule repositoriesModule) {
-        repositoriesModule.registerRepository("hdfs", HdfsRepository.class);
+    @Override
+    public Map<String, Repository.Factory> getRepositories(Environment env) {
+        return Collections.singletonMap("hdfs", (metadata) -> new HdfsRepository(metadata, env));
     }
 }
diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
index de8fb155abb..b111a5d0d0a 100644
--- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
+++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsRepository.java
@@ -37,21 +37,20 @@ import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.UnsupportedFileSystemException;
 import org.elasticsearch.ElasticsearchGenerationException;
 import org.elasticsearch.SpecialPermission;
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.SuppressForbidden;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
-import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.repositories.RepositoryName;
-import org.elasticsearch.repositories.RepositorySettings;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
 
 public final class HdfsRepository extends BlobStoreRepository {
 
     private final BlobPath basePath = BlobPath.cleanPath();
-    private final RepositorySettings repositorySettings;
     private final ByteSizeValue chunkSize;
     private final boolean compress;
 
@@ -61,18 +60,16 @@ public final class HdfsRepository extends BlobStoreRepository {
     // TODO: why 100KB?
     private static final ByteSizeValue DEFAULT_BUFFER_SIZE = new ByteSizeValue(100, ByteSizeUnit.KB);
 
-    @Inject
-    public HdfsRepository(RepositoryName name, RepositorySettings repositorySettings) throws IOException {
-        super(name.getName(), repositorySettings);
-        this.repositorySettings = repositorySettings;
+    public HdfsRepository(RepositoryMetaData metadata, Environment environment) throws IOException {
+        super(metadata, environment.settings());
 
-        this.chunkSize = repositorySettings.settings().getAsBytesSize("chunk_size", null);
-        this.compress = repositorySettings.settings().getAsBoolean("compress", false);
+        this.chunkSize = metadata.settings().getAsBytesSize("chunk_size", null);
+        this.compress = metadata.settings().getAsBoolean("compress", false);
     }
 
     @Override
     protected void doStart() {
-        String uriSetting = repositorySettings.settings().get("uri");
+        String uriSetting = getMetadata().settings().get("uri");
         if (Strings.hasText(uriSetting) == false) {
             throw new IllegalArgumentException("No 'uri' defined for hdfs snapshot/restore");
         }
@@ -86,13 +83,13 @@ public final class HdfsRepository extends BlobStoreRepository {
                     "Use 'path' option to specify a path [%s], not the uri [%s] for hdfs snapshot/restore", uri.getPath(), uriSetting));
         }
 
-        String pathSetting = repositorySettings.settings().get("path");
+        String pathSetting = getMetadata().settings().get("path");
         // get configuration
         if (pathSetting == null) {
             throw new IllegalArgumentException("No 'path' defined for hdfs snapshot/restore");
         }
         
-        int bufferSize = repositorySettings.settings().getAsBytesSize("buffer_size", DEFAULT_BUFFER_SIZE).bytesAsInt();
+        int bufferSize = getMetadata().settings().getAsBytesSize("buffer_size", DEFAULT_BUFFER_SIZE).bytesAsInt();
 
         try {
             // initialize our filecontext
@@ -103,7 +100,7 @@ public final class HdfsRepository extends BlobStoreRepository {
             FileContext fileContext = AccessController.doPrivileged(new PrivilegedAction<FileContext>() {
                 @Override
                 public FileContext run() {
-                    return createContext(uri, repositorySettings);
+                    return createContext(uri, getMetadata().settings());
                 }
             });
             blobStore = new HdfsBlobStore(fileContext, pathSetting, bufferSize);
@@ -116,12 +113,12 @@ public final class HdfsRepository extends BlobStoreRepository {
     
     // create hadoop filecontext
     @SuppressForbidden(reason = "lesser of two evils (the other being a bunch of JNI/classloader nightmares)")
-    private static FileContext createContext(URI uri, RepositorySettings repositorySettings)  {
-        Configuration cfg = new Configuration(repositorySettings.settings().getAsBoolean("load_defaults", true));
+    private static FileContext createContext(URI uri, Settings repositorySettings)  {
+        Configuration cfg = new Configuration(repositorySettings.getAsBoolean("load_defaults", true));
         cfg.setClassLoader(HdfsRepository.class.getClassLoader());
         cfg.reloadConfiguration();
 
-        Map<String, String> map = repositorySettings.settings().getByPrefix("conf.").getAsMap();
+        Map<String, String> map = repositorySettings.getByPrefix("conf.").getAsMap();
         for (Entry<String, String> entry : map.entrySet()) {
             cfg.set(entry.getKey(), entry.getValue());
         }
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
index 1a0c2992b15..81eb0bfa494 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java
@@ -51,7 +51,6 @@ public class InternalAwsS3Service extends AbstractLifecycleComponent implements
      */
     private Map<Tuple<String, String>, AmazonS3Client> clients = new HashMap<>();
 
-    @Inject
     public InternalAwsS3Service(Settings settings) {
         super(settings);
     }
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/S3Module.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/S3Module.java
deleted file mode 100644
index 11294054c21..00000000000
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/S3Module.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.aws;
-
-import org.elasticsearch.common.inject.AbstractModule;
-
-public class S3Module extends AbstractModule {
-
-
-    // pkg private so it is settable by tests
-    static Class<? extends AwsS3Service> s3ServiceImpl = InternalAwsS3Service.class;
-
-    public static Class<? extends AwsS3Service> getS3ServiceImpl() {
-        return s3ServiceImpl;
-    }
-
-    @Override
-    protected void configure() {
-        bind(AwsS3Service.class).to(s3ServiceImpl).asEagerSingleton();
-    }
-}
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
index fc55e1b0ff1..0d6fac2052c 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/plugin/repository/s3/S3RepositoryPlugin.java
@@ -21,26 +21,26 @@ package org.elasticsearch.plugin.repository.s3;
 
 import java.security.AccessController;
 import java.security.PrivilegedAction;
-import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.cloud.aws.AwsS3Service;
-import org.elasticsearch.cloud.aws.S3Module;
-import org.elasticsearch.common.component.LifecycleComponent;
-import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.cloud.aws.InternalAwsS3Service;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.repositories.RepositoriesModule;
+import org.elasticsearch.plugins.RepositoryPlugin;
+import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.repositories.s3.S3Repository;
 
 /**
- *
+ * A plugin to add a repository type that writes to and from the AWS S3.
  */
-public class S3RepositoryPlugin extends Plugin {
+public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin {
 
     // ClientConfiguration clinit has some classloader problems
     // TODO: fix that
@@ -62,21 +62,15 @@ public class S3RepositoryPlugin extends Plugin {
         });
     }
 
-    @Override
-    public Collection<Module> nodeModules() {
-        Collection<Module> modules = new ArrayList<>();
-        modules.add(new S3Module());
-        return modules;
+    // overridable for tests
+    protected AwsS3Service createStorageService(Settings settings) {
+        return new InternalAwsS3Service(settings);
     }
 
     @Override
-    @SuppressWarnings("rawtypes") // Supertype declaration has raw types
-    public Collection<Class<? extends LifecycleComponent>> nodeServices() {
-        return Collections.<Class<? extends LifecycleComponent>>singleton(S3Module.getS3ServiceImpl());
-    }
-
-    public void onModule(RepositoriesModule repositoriesModule) {
-        repositoriesModule.registerRepository(S3Repository.TYPE, S3Repository.class);
+    public Map<String, Repository.Factory> getRepositories(Environment env) {
+        return Collections.singletonMap(S3Repository.TYPE,
+            (metadata) -> new S3Repository(metadata, env.settings(), createStorageService(env.settings())));
     }
 
     @Override
diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
index 182612041f1..2e108aaf6a7 100644
--- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
+++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java
@@ -24,17 +24,16 @@ import com.amazonaws.Protocol;
 import org.elasticsearch.cloud.aws.AwsS3Service;
 import org.elasticsearch.cloud.aws.AwsS3Service.CLOUD_S3;
 import org.elasticsearch.cloud.aws.blobstore.S3BlobStore;
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
-import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.repositories.RepositoryException;
-import org.elasticsearch.repositories.RepositoryName;
-import org.elasticsearch.repositories.RepositorySettings;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
 
 import java.io.IOException;
@@ -245,59 +244,54 @@ public class S3Repository extends BlobStoreRepository {
     private boolean compress;
 
     /**
-     * Constructs new shared file system repository
-     *
-     * @param name                 repository name
-     * @param repositorySettings   repository settings
-     * @param s3Service            S3 service
+     * Constructs an s3 backed repository
      */
-    @Inject
-    public S3Repository(RepositoryName name, RepositorySettings repositorySettings, AwsS3Service s3Service) throws IOException {
-        super(name.getName(), repositorySettings);
+    public S3Repository(RepositoryMetaData metadata, Settings settings, AwsS3Service s3Service) throws IOException {
+        super(metadata, settings);
 
-        String bucket = getValue(repositorySettings, Repository.BUCKET_SETTING, Repositories.BUCKET_SETTING);
+        String bucket = getValue(metadata.settings(), settings, Repository.BUCKET_SETTING, Repositories.BUCKET_SETTING);
         if (bucket == null) {
-            throw new RepositoryException(name.name(), "No bucket defined for s3 gateway");
+            throw new RepositoryException(metadata.name(), "No bucket defined for s3 gateway");
         }
 
-        String endpoint = getValue(repositorySettings, Repository.ENDPOINT_SETTING, Repositories.ENDPOINT_SETTING);
-        Protocol protocol = getValue(repositorySettings, Repository.PROTOCOL_SETTING, Repositories.PROTOCOL_SETTING);
-        String region = getValue(repositorySettings, Repository.REGION_SETTING, Repositories.REGION_SETTING);
+        String endpoint = getValue(metadata.settings(), settings, Repository.ENDPOINT_SETTING, Repositories.ENDPOINT_SETTING);
+        Protocol protocol = getValue(metadata.settings(), settings, Repository.PROTOCOL_SETTING, Repositories.PROTOCOL_SETTING);
+        String region = getValue(metadata.settings(), settings, Repository.REGION_SETTING, Repositories.REGION_SETTING);
         // If no region is defined either in region, repositories.s3.region, cloud.aws.s3.region or cloud.aws.region
         // we fallback to Default bucket - null
         if (Strings.isEmpty(region)) {
             region = null;
         }
 
-        boolean serverSideEncryption = getValue(repositorySettings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING);
-        ByteSizeValue bufferSize = getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING);
-        Integer maxRetries = getValue(repositorySettings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING);
-        boolean useThrottleRetries = getValue(repositorySettings, Repository.USE_THROTTLE_RETRIES_SETTING, Repositories.USE_THROTTLE_RETRIES_SETTING);
-        this.chunkSize = getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING);
-        this.compress = getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING);
+        boolean serverSideEncryption = getValue(metadata.settings(), settings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING);
+        ByteSizeValue bufferSize = getValue(metadata.settings(), settings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING);
+        Integer maxRetries = getValue(metadata.settings(), settings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING);
+        boolean useThrottleRetries = getValue(metadata.settings(), settings, Repository.USE_THROTTLE_RETRIES_SETTING, Repositories.USE_THROTTLE_RETRIES_SETTING);
+        this.chunkSize = getValue(metadata.settings(), settings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING);
+        this.compress = getValue(metadata.settings(), settings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING);
 
         // We make sure that chunkSize is bigger or equal than/to bufferSize
         if (this.chunkSize.getBytes() < bufferSize.getBytes()) {
-            throw new RepositoryException(name.name(), Repository.CHUNK_SIZE_SETTING.getKey() + " (" + this.chunkSize +
+            throw new RepositoryException(metadata.name(), Repository.CHUNK_SIZE_SETTING.getKey() + " (" + this.chunkSize +
                 ") can't be lower than " + Repository.BUFFER_SIZE_SETTING.getKey() + " (" + bufferSize + ").");
         }
 
         // Parse and validate the user's S3 Storage Class setting
-        String storageClass = getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING);
-        String cannedACL = getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING);
+        String storageClass = getValue(metadata.settings(), settings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING);
+        String cannedACL = getValue(metadata.settings(), settings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING);
 
         logger.debug("using bucket [{}], region [{}], endpoint [{}], protocol [{}], chunk_size [{}], server_side_encryption [{}], " +
             "buffer_size [{}], max_retries [{}], use_throttle_retries [{}], cannedACL [{}], storageClass [{}]",
             bucket, region, endpoint, protocol, chunkSize, serverSideEncryption, bufferSize, maxRetries, useThrottleRetries, cannedACL,
             storageClass);
 
-        String key = getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING);
-        String secret = getValue(repositorySettings, Repository.SECRET_SETTING, Repositories.SECRET_SETTING);
+        String key = getValue(metadata.settings(), settings, Repository.KEY_SETTING, Repositories.KEY_SETTING);
+        String secret = getValue(metadata.settings(), settings, Repository.SECRET_SETTING, Repositories.SECRET_SETTING);
 
         blobStore = new S3BlobStore(settings, s3Service.client(endpoint, protocol, region, key, secret, maxRetries, useThrottleRetries),
                 bucket, region, serverSideEncryption, bufferSize, maxRetries, cannedACL, storageClass);
 
-        String basePath = getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING);
+        String basePath = getValue(metadata.settings(), settings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING);
         if (Strings.hasLength(basePath)) {
             BlobPath path = new BlobPath();
             for(String elem : basePath.split("/")) {
@@ -309,9 +303,6 @@ public class S3Repository extends BlobStoreRepository {
         }
     }
 
-    /**
-     * {@inheritDoc}
-     */
     @Override
     protected BlobStore blobStore() {
         return blobStore;
@@ -322,29 +313,24 @@ public class S3Repository extends BlobStoreRepository {
         return basePath;
     }
 
-    /**
-     * {@inheritDoc}
-     */
     @Override
     protected boolean isCompress() {
         return compress;
     }
 
-    /**
-     * {@inheritDoc}
-     */
     @Override
     protected ByteSizeValue chunkSize() {
         return chunkSize;
     }
 
-    public static <T> T getValue(RepositorySettings repositorySettings,
+    public static <T> T getValue(Settings repositorySettings,
+                                 Settings globalSettings,
                                  Setting<T> repositorySetting,
                                  Setting<T> repositoriesSetting) {
-        if (repositorySetting.exists(repositorySettings.settings())) {
-            return repositorySetting.get(repositorySettings.settings());
+        if (repositorySetting.exists(repositorySettings)) {
+            return repositorySetting.get(repositorySettings);
         } else {
-            return repositoriesSetting.get(repositorySettings.globalSettings());
+            return repositoriesSetting.get(globalSettings);
         }
     }
 }
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTestCase.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTestCase.java
index 9d1768db58b..b7dc7cc525d 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTestCase.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/AbstractAwsTestCase.java
@@ -69,6 +69,6 @@ public abstract class AbstractAwsTestCase extends ESIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return pluginList(S3RepositoryPlugin.class);
+        return pluginList(TestAwsS3Service.TestPlugin.class);
     }
 }
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java
deleted file mode 100644
index c11cb969570..00000000000
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/RepositoryS3SettingsTests.java
+++ /dev/null
@@ -1,353 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.aws;
-
-import com.amazonaws.Protocol;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.ByteSizeUnit;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.repositories.RepositoryException;
-import org.elasticsearch.repositories.RepositoryName;
-import org.elasticsearch.repositories.RepositorySettings;
-import org.elasticsearch.repositories.s3.S3Repository;
-import org.elasticsearch.test.ESTestCase;
-
-import java.io.IOException;
-
-import static org.elasticsearch.repositories.s3.S3Repository.Repositories;
-import static org.elasticsearch.repositories.s3.S3Repository.Repository;
-import static org.elasticsearch.repositories.s3.S3Repository.getValue;
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.isEmptyString;
-
-public class RepositoryS3SettingsTests extends ESTestCase {
-
-    private static final Settings AWS = Settings.builder()
-        .put(AwsS3Service.KEY_SETTING.getKey(), "global-key")
-        .put(AwsS3Service.SECRET_SETTING.getKey(), "global-secret")
-        .put(AwsS3Service.PROTOCOL_SETTING.getKey(), "https")
-        .put(AwsS3Service.PROXY_HOST_SETTING.getKey(), "global-proxy-host")
-        .put(AwsS3Service.PROXY_PORT_SETTING.getKey(), 10000)
-        .put(AwsS3Service.PROXY_USERNAME_SETTING.getKey(), "global-proxy-username")
-        .put(AwsS3Service.PROXY_PASSWORD_SETTING.getKey(), "global-proxy-password")
-        .put(AwsS3Service.SIGNER_SETTING.getKey(), "global-signer")
-        .put(AwsS3Service.REGION_SETTING.getKey(), "global-region")
-        .build();
-
-    private static final Settings S3 = Settings.builder()
-        .put(AwsS3Service.CLOUD_S3.KEY_SETTING.getKey(), "s3-key")
-        .put(AwsS3Service.CLOUD_S3.SECRET_SETTING.getKey(), "s3-secret")
-        .put(AwsS3Service.CLOUD_S3.PROTOCOL_SETTING.getKey(), "http")
-        .put(AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING.getKey(), "s3-proxy-host")
-        .put(AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING.getKey(), 20000)
-        .put(AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING.getKey(), "s3-proxy-username")
-        .put(AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING.getKey(), "s3-proxy-password")
-        .put(AwsS3Service.CLOUD_S3.SIGNER_SETTING.getKey(), "s3-signer")
-        .put(AwsS3Service.CLOUD_S3.REGION_SETTING.getKey(), "s3-region")
-        .put(AwsS3Service.CLOUD_S3.ENDPOINT_SETTING.getKey(), "s3-endpoint")
-        .build();
-
-    private static final Settings REPOSITORIES = Settings.builder()
-        .put(Repositories.KEY_SETTING.getKey(), "repositories-key")
-        .put(Repositories.SECRET_SETTING.getKey(), "repositories-secret")
-        .put(Repositories.BUCKET_SETTING.getKey(), "repositories-bucket")
-        .put(Repositories.PROTOCOL_SETTING.getKey(), "https")
-        .put(Repositories.REGION_SETTING.getKey(), "repositories-region")
-        .put(Repositories.ENDPOINT_SETTING.getKey(), "repositories-endpoint")
-        .put(Repositories.SERVER_SIDE_ENCRYPTION_SETTING.getKey(), true)
-        .put(Repositories.BUFFER_SIZE_SETTING.getKey(), "6mb")
-        .put(Repositories.MAX_RETRIES_SETTING.getKey(), 4)
-        .put(Repositories.CHUNK_SIZE_SETTING.getKey(), "110mb")
-        .put(Repositories.COMPRESS_SETTING.getKey(), true)
-        .put(Repositories.STORAGE_CLASS_SETTING.getKey(), "repositories-class")
-        .put(Repositories.CANNED_ACL_SETTING.getKey(), "repositories-acl")
-        .put(Repositories.BASE_PATH_SETTING.getKey(), "repositories-basepath")
-        .build();
-
-    private static final Settings REPOSITORY = Settings.builder()
-        .put(Repository.KEY_SETTING.getKey(), "repository-key")
-        .put(Repository.SECRET_SETTING.getKey(), "repository-secret")
-        .put(Repository.BUCKET_SETTING.getKey(), "repository-bucket")
-        .put(Repository.PROTOCOL_SETTING.getKey(), "https")
-        .put(Repository.REGION_SETTING.getKey(), "repository-region")
-        .put(Repository.ENDPOINT_SETTING.getKey(), "repository-endpoint")
-        .put(Repository.SERVER_SIDE_ENCRYPTION_SETTING.getKey(), false)
-        .put(Repository.BUFFER_SIZE_SETTING.getKey(), "7mb")
-        .put(Repository.MAX_RETRIES_SETTING.getKey(), 5)
-        .put(Repository.CHUNK_SIZE_SETTING.getKey(), "120mb")
-        .put(Repository.COMPRESS_SETTING.getKey(), false)
-        .put(Repository.STORAGE_CLASS_SETTING.getKey(), "repository-class")
-        .put(Repository.CANNED_ACL_SETTING.getKey(), "repository-acl")
-        .put(Repository.BASE_PATH_SETTING.getKey(), "repository-basepath")
-        .build();
-
-    /**
-     * We test when only cloud.aws settings are set
-     */
-    public void testRepositorySettingsGlobalOnly() {
-        Settings nodeSettings = buildSettings(AWS);
-        RepositorySettings repositorySettings =  new RepositorySettings(nodeSettings, Settings.EMPTY);
-        assertThat(getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING), is("global-key"));
-        assertThat(getValue(repositorySettings, Repository.SECRET_SETTING, Repositories.SECRET_SETTING), is("global-secret"));
-        assertThat(getValue(repositorySettings, Repository.BUCKET_SETTING, Repositories.BUCKET_SETTING), isEmptyString());
-        assertThat(getValue(repositorySettings, Repository.PROTOCOL_SETTING, Repositories.PROTOCOL_SETTING), is(Protocol.HTTPS));
-        assertThat(getValue(repositorySettings, Repository.REGION_SETTING, Repositories.REGION_SETTING), is("global-region"));
-        assertThat(getValue(repositorySettings, Repository.ENDPOINT_SETTING, Repositories.ENDPOINT_SETTING), isEmptyString());
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING.get(nodeSettings), is("global-proxy-host"));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING.get(nodeSettings), is(10000));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING.get(nodeSettings), is("global-proxy-username"));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING.get(nodeSettings), is("global-proxy-password"));
-        assertThat(AwsS3Service.CLOUD_S3.SIGNER_SETTING.get(nodeSettings), is("global-signer"));
-        assertThat(getValue(repositorySettings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING),
-            is(false));
-        assertThat(getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING).getMb(), is(100L));
-        assertThat(getValue(repositorySettings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING), is(3));
-        assertThat(getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING).getGb(), is(1L));
-        assertThat(getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING), is(false));
-        assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING), isEmptyString());
-        assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), isEmptyString());
-        assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), isEmptyString());
-    }
-
-    /**
-     * We test when cloud.aws settings are overloaded by cloud.aws.s3 settings
-     */
-    public void testRepositorySettingsGlobalOverloadedByS3() {
-        Settings nodeSettings = buildSettings(AWS, S3);
-        RepositorySettings repositorySettings =  new RepositorySettings(nodeSettings, Settings.EMPTY);
-        assertThat(getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING), is("s3-key"));
-        assertThat(getValue(repositorySettings, Repository.SECRET_SETTING, Repositories.SECRET_SETTING), is("s3-secret"));
-        assertThat(getValue(repositorySettings, Repository.BUCKET_SETTING, Repositories.BUCKET_SETTING), isEmptyString());
-        assertThat(getValue(repositorySettings, Repository.PROTOCOL_SETTING, Repositories.PROTOCOL_SETTING), is(Protocol.HTTP));
-        assertThat(getValue(repositorySettings, Repository.REGION_SETTING, Repositories.REGION_SETTING), is("s3-region"));
-        assertThat(getValue(repositorySettings, Repository.ENDPOINT_SETTING, Repositories.ENDPOINT_SETTING), is("s3-endpoint"));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING.get(nodeSettings), is("s3-proxy-host"));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING.get(nodeSettings), is(20000));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING.get(nodeSettings), is("s3-proxy-username"));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING.get(nodeSettings), is("s3-proxy-password"));
-        assertThat(AwsS3Service.CLOUD_S3.SIGNER_SETTING.get(nodeSettings), is("s3-signer"));
-        assertThat(getValue(repositorySettings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING),
-            is(false));
-        assertThat(getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING).getMb(), is(100L));
-        assertThat(getValue(repositorySettings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING), is(3));
-        assertThat(getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING).getGb(), is(1L));
-        assertThat(getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING), is(false));
-        assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING), isEmptyString());
-        assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), isEmptyString());
-        assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), isEmptyString());
-    }
-
-    /**
-     * We test when cloud.aws settings are overloaded by repositories.s3 settings
-     */
-    public void testRepositorySettingsGlobalOverloadedByRepositories() {
-        Settings nodeSettings = buildSettings(AWS, REPOSITORIES);
-        RepositorySettings repositorySettings =  new RepositorySettings(nodeSettings, Settings.EMPTY);
-        assertThat(getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING), is("repositories-key"));
-        assertThat(getValue(repositorySettings, Repository.SECRET_SETTING, Repositories.SECRET_SETTING), is("repositories-secret"));
-        assertThat(getValue(repositorySettings, Repository.BUCKET_SETTING, Repositories.BUCKET_SETTING), is("repositories-bucket"));
-        assertThat(getValue(repositorySettings, Repository.PROTOCOL_SETTING, Repositories.PROTOCOL_SETTING), is(Protocol.HTTPS));
-        assertThat(getValue(repositorySettings, Repository.REGION_SETTING, Repositories.REGION_SETTING), is("repositories-region"));
-        assertThat(getValue(repositorySettings, Repository.ENDPOINT_SETTING, Repositories.ENDPOINT_SETTING), is("repositories-endpoint"));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING.get(nodeSettings), is("global-proxy-host"));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING.get(nodeSettings), is(10000));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING.get(nodeSettings), is("global-proxy-username"));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING.get(nodeSettings), is("global-proxy-password"));
-        assertThat(AwsS3Service.CLOUD_S3.SIGNER_SETTING.get(nodeSettings), is("global-signer"));
-        assertThat(getValue(repositorySettings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING),
-            is(true));
-        assertThat(getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING).getMb(), is(6L));
-        assertThat(getValue(repositorySettings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING), is(4));
-        assertThat(getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING).getMb(), is(110L));
-        assertThat(getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING), is(true));
-        assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING),
-            is("repositories-class"));
-        assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), is("repositories-acl"));
-        assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), is("repositories-basepath"));
-    }
-
-    /**
-     * We test when cloud.aws.s3 settings are overloaded by repositories.s3 settings
-     */
-    public void testRepositorySettingsS3OverloadedByRepositories() {
-        Settings nodeSettings = buildSettings(AWS, S3, REPOSITORIES);
-        RepositorySettings repositorySettings =  new RepositorySettings(nodeSettings, Settings.EMPTY);
-        assertThat(getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING), is("repositories-key"));
-        assertThat(getValue(repositorySettings, Repository.SECRET_SETTING, Repositories.SECRET_SETTING), is("repositories-secret"));
-        assertThat(getValue(repositorySettings, Repository.BUCKET_SETTING, Repositories.BUCKET_SETTING), is("repositories-bucket"));
-        assertThat(getValue(repositorySettings, Repository.PROTOCOL_SETTING, Repositories.PROTOCOL_SETTING), is(Protocol.HTTPS));
-        assertThat(getValue(repositorySettings, Repository.REGION_SETTING, Repositories.REGION_SETTING), is("repositories-region"));
-        assertThat(getValue(repositorySettings, Repository.ENDPOINT_SETTING, Repositories.ENDPOINT_SETTING), is("repositories-endpoint"));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING.get(nodeSettings), is("s3-proxy-host"));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING.get(nodeSettings), is(20000));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING.get(nodeSettings), is("s3-proxy-username"));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING.get(nodeSettings), is("s3-proxy-password"));
-        assertThat(AwsS3Service.CLOUD_S3.SIGNER_SETTING.get(nodeSettings), is("s3-signer"));
-        assertThat(getValue(repositorySettings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING),
-            is(true));
-        assertThat(getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING).getMb(), is(6L));
-        assertThat(getValue(repositorySettings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING), is(4));
-        assertThat(getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING).getMb(), is(110L));
-        assertThat(getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING), is(true));
-        assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING),
-            is("repositories-class"));
-        assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), is("repositories-acl"));
-        assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), is("repositories-basepath"));
-    }
-
-    /**
-     * We test when cloud.aws settings are overloaded by single repository settings
-     */
-    public void testRepositorySettingsGlobalOverloadedByRepository() {
-        Settings nodeSettings = buildSettings(AWS);
-        RepositorySettings repositorySettings =  new RepositorySettings(nodeSettings, REPOSITORY);
-        assertThat(getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING), is("repository-key"));
-        assertThat(getValue(repositorySettings, Repository.SECRET_SETTING, Repositories.SECRET_SETTING), is("repository-secret"));
-        assertThat(getValue(repositorySettings, Repository.BUCKET_SETTING, Repositories.BUCKET_SETTING), is("repository-bucket"));
-        assertThat(getValue(repositorySettings, Repository.PROTOCOL_SETTING, Repositories.PROTOCOL_SETTING), is(Protocol.HTTPS));
-        assertThat(getValue(repositorySettings, Repository.REGION_SETTING, Repositories.REGION_SETTING), is("repository-region"));
-        assertThat(getValue(repositorySettings, Repository.ENDPOINT_SETTING, Repositories.ENDPOINT_SETTING), is("repository-endpoint"));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING.get(nodeSettings), is("global-proxy-host"));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING.get(nodeSettings), is(10000));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING.get(nodeSettings), is("global-proxy-username"));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING.get(nodeSettings), is("global-proxy-password"));
-        assertThat(AwsS3Service.CLOUD_S3.SIGNER_SETTING.get(nodeSettings), is("global-signer"));
-        assertThat(getValue(repositorySettings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING),
-            is(false));
-        assertThat(getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING).getMb(), is(7L));
-        assertThat(getValue(repositorySettings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING), is(5));
-        assertThat(getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING).getMb(), is(120L));
-        assertThat(getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING), is(false));
-        assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING),
-            is("repository-class"));
-        assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), is("repository-acl"));
-        assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), is("repository-basepath"));
-    }
-
-    /**
-     * We test when cloud.aws.s3 settings are overloaded by single repository settings
-     */
-    public void testRepositorySettingsS3OverloadedByRepository() {
-        Settings nodeSettings = buildSettings(AWS, S3);
-        RepositorySettings repositorySettings =  new RepositorySettings(nodeSettings, REPOSITORY);
-        assertThat(getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING), is("repository-key"));
-        assertThat(getValue(repositorySettings, Repository.SECRET_SETTING, Repositories.SECRET_SETTING), is("repository-secret"));
-        assertThat(getValue(repositorySettings, Repository.BUCKET_SETTING, Repositories.BUCKET_SETTING), is("repository-bucket"));
-        assertThat(getValue(repositorySettings, Repository.PROTOCOL_SETTING, Repositories.PROTOCOL_SETTING), is(Protocol.HTTPS));
-        assertThat(getValue(repositorySettings, Repository.REGION_SETTING, Repositories.REGION_SETTING), is("repository-region"));
-        assertThat(getValue(repositorySettings, Repository.ENDPOINT_SETTING, Repositories.ENDPOINT_SETTING), is("repository-endpoint"));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING.get(nodeSettings), is("s3-proxy-host"));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING.get(nodeSettings), is(20000));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING.get(nodeSettings), is("s3-proxy-username"));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING.get(nodeSettings), is("s3-proxy-password"));
-        assertThat(AwsS3Service.CLOUD_S3.SIGNER_SETTING.get(nodeSettings), is("s3-signer"));
-        assertThat(getValue(repositorySettings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING),
-            is(false));
-        assertThat(getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING).getMb(), is(7L));
-        assertThat(getValue(repositorySettings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING), is(5));
-        assertThat(getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING).getMb(), is(120L));
-        assertThat(getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING), is(false));
-        assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING),
-            is("repository-class"));
-        assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), is("repository-acl"));
-        assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), is("repository-basepath"));
-    }
-
-    /**
-     * We test when repositories settings are overloaded by single repository settings
-     */
-    public void testRepositorySettingsRepositoriesOverloadedByRepository() {
-        Settings nodeSettings = buildSettings(AWS, S3, REPOSITORIES);
-        RepositorySettings repositorySettings =  new RepositorySettings(nodeSettings, REPOSITORY);
-        assertThat(getValue(repositorySettings, Repository.KEY_SETTING, Repositories.KEY_SETTING), is("repository-key"));
-        assertThat(getValue(repositorySettings, Repository.SECRET_SETTING, Repositories.SECRET_SETTING), is("repository-secret"));
-        assertThat(getValue(repositorySettings, Repository.BUCKET_SETTING, Repositories.BUCKET_SETTING), is("repository-bucket"));
-        assertThat(getValue(repositorySettings, Repository.PROTOCOL_SETTING, Repositories.PROTOCOL_SETTING), is(Protocol.HTTPS));
-        assertThat(getValue(repositorySettings, Repository.REGION_SETTING, Repositories.REGION_SETTING), is("repository-region"));
-        assertThat(getValue(repositorySettings, Repository.ENDPOINT_SETTING, Repositories.ENDPOINT_SETTING), is("repository-endpoint"));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_HOST_SETTING.get(nodeSettings), is("s3-proxy-host"));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_PORT_SETTING.get(nodeSettings), is(20000));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_USERNAME_SETTING.get(nodeSettings), is("s3-proxy-username"));
-        assertThat(AwsS3Service.CLOUD_S3.PROXY_PASSWORD_SETTING.get(nodeSettings), is("s3-proxy-password"));
-        assertThat(AwsS3Service.CLOUD_S3.SIGNER_SETTING.get(nodeSettings), is("s3-signer"));
-        assertThat(getValue(repositorySettings, Repository.SERVER_SIDE_ENCRYPTION_SETTING, Repositories.SERVER_SIDE_ENCRYPTION_SETTING),
-            is(false));
-        assertThat(getValue(repositorySettings, Repository.BUFFER_SIZE_SETTING, Repositories.BUFFER_SIZE_SETTING).getMb(), is(7L));
-        assertThat(getValue(repositorySettings, Repository.MAX_RETRIES_SETTING, Repositories.MAX_RETRIES_SETTING), is(5));
-        assertThat(getValue(repositorySettings, Repository.CHUNK_SIZE_SETTING, Repositories.CHUNK_SIZE_SETTING).getMb(), is(120L));
-        assertThat(getValue(repositorySettings, Repository.COMPRESS_SETTING, Repositories.COMPRESS_SETTING), is(false));
-        assertThat(getValue(repositorySettings, Repository.STORAGE_CLASS_SETTING, Repositories.STORAGE_CLASS_SETTING),
-            is("repository-class"));
-        assertThat(getValue(repositorySettings, Repository.CANNED_ACL_SETTING, Repositories.CANNED_ACL_SETTING), is("repository-acl"));
-        assertThat(getValue(repositorySettings, Repository.BASE_PATH_SETTING, Repositories.BASE_PATH_SETTING), is("repository-basepath"));
-    }
-
-    /**
-     * We test wrong Chunk and Buffer settings
-     */
-    public void testInvalidChunkBufferSizeRepositorySettings() throws IOException {
-        // chunk < buffer should fail
-        internalTestInvalidChunkBufferSizeSettings(new ByteSizeValue(10, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.MB),
-            "chunk_size (5mb) can't be lower than buffer_size (10mb).");
-        // chunk > buffer should pass
-        internalTestInvalidChunkBufferSizeSettings(new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(10, ByteSizeUnit.MB), null);
-        // chunk = buffer should pass
-        internalTestInvalidChunkBufferSizeSettings(new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(5, ByteSizeUnit.MB), null);
-        // buffer < 5mb should fail
-        internalTestInvalidChunkBufferSizeSettings(new ByteSizeValue(4, ByteSizeUnit.MB), new ByteSizeValue(10, ByteSizeUnit.MB),
-            "Failed to parse value [4mb] for setting [buffer_size] must be >= 5mb");
-        // chunk > 5tb should fail
-        internalTestInvalidChunkBufferSizeSettings(new ByteSizeValue(5, ByteSizeUnit.MB), new ByteSizeValue(6, ByteSizeUnit.TB),
-            "Failed to parse value [6tb] for setting [chunk_size] must be <= 5tb");
-    }
-
-    private Settings buildSettings(Settings... global) {
-        Settings.Builder builder = Settings.builder();
-        for (Settings settings : global) {
-            builder.put(settings);
-        }
-        return builder.build();
-    }
-
-    private void internalTestInvalidChunkBufferSizeSettings(ByteSizeValue buffer, ByteSizeValue chunk, String expectedMessage)
-        throws IOException {
-        Settings nodeSettings = buildSettings(AWS, S3, REPOSITORIES);
-        RepositorySettings s3RepositorySettings =  new RepositorySettings(nodeSettings, Settings.builder()
-            .put(Repository.BUFFER_SIZE_SETTING.getKey(), buffer)
-            .put(Repository.CHUNK_SIZE_SETTING.getKey(), chunk)
-            .build());
-
-        try {
-            new S3Repository(new RepositoryName("s3", "s3repo"), s3RepositorySettings, null);
-            fail("We should either raise a NPE or a RepositoryException or a IllegalArgumentException");
-        } catch (RepositoryException e) {
-            assertThat(e.getDetailedMessage(), containsString(expectedMessage));
-        } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), containsString(expectedMessage));
-        } catch (NullPointerException e) {
-            // Because we passed to the CTOR a Null AwsS3Service, we get a NPE which is expected
-            // in the context of this test
-            if (expectedMessage != null) {
-                fail("We should have raised a RepositoryException");
-            }
-        }
-    }
-}
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
index 36608c8b172..05faf7b5069 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/TestAwsS3Service.java
@@ -23,20 +23,21 @@ import com.amazonaws.services.s3.AmazonS3;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.plugin.repository.s3.S3RepositoryPlugin;
 import org.elasticsearch.plugins.Plugin;
 
 import java.util.IdentityHashMap;
 
 public class TestAwsS3Service extends InternalAwsS3Service {
-    public static class TestPlugin extends Plugin {
-        public void onModule(S3Module s3Module) {
-            S3Module.s3ServiceImpl = TestAwsS3Service.class;
+    public static class TestPlugin extends S3RepositoryPlugin {
+        @Override
+        protected AwsS3Service createStorageService(Settings settings) {
+            return new TestAwsS3Service(settings);
         }
     }
 
     IdentityHashMap<AmazonS3, TestAmazonS3> clients = new IdentityHashMap<AmazonS3, TestAmazonS3>();
 
-    @Inject
     public TestAwsS3Service(Settings settings) {
         super(settings);
     }
diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
index d1c43f15adb..80e2c421a33 100644
--- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java
@@ -49,10 +49,8 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.notNullValue;
 
-/**
- */
 @ClusterScope(scope = Scope.SUITE, numDataNodes = 2, numClientNodes = 0, transportClientRatio = 0.0)
-public abstract  class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase {
+public abstract class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase {
 
     @Override
     public Settings nodeSettings(int nodeOrdinal) {

From 1cb1373722b97a23f7f551e9827bbedea97b1424 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simonw@apache.org>
Date: Fri, 8 Jul 2016 10:32:30 +0200
Subject: [PATCH 1286/1311] [TEST] Test analyzer alias works

Relates to #19163
---
 .../indices/analysis/AnalysisModuleTests.java | 30 ++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java b/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java
index 869ac622b39..7ca0df72f8f 100644
--- a/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java
+++ b/core/src/test/java/org/elasticsearch/indices/analysis/AnalysisModuleTests.java
@@ -23,6 +23,7 @@ import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.ar.ArabicNormalizationFilter;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
 import org.apache.lucene.analysis.fa.PersianNormalizationFilter;
 import org.apache.lucene.analysis.hunspell.Dictionary;
@@ -96,7 +97,7 @@ public class AnalysisModuleTests extends ModuleTestCase {
             throw new RuntimeException(e);
         }
     }
-    
+
     private Settings loadFromClasspath(String path) throws IOException {
         return Settings.builder().loadFromStream(path, getClass().getResourceAsStream(path))
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
@@ -121,6 +122,33 @@ public class AnalysisModuleTests extends ModuleTestCase {
         assertTokenFilter("arabic_normalization", ArabicNormalizationFilter.class);
     }
 
+    public void testAnalyzerAlias() throws IOException {
+        Settings settings = Settings.builder()
+            .put("index.analysis.analyzer.foobar.alias","default")
+            .put("index.analysis.analyzer.foobar.type", "keyword")
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_0_0)
+            .build();
+        AnalysisRegistry newRegistry = getNewRegistry(settings);
+        AnalysisService as = getAnalysisService(newRegistry, settings);
+        assertThat(as.analyzer("default").analyzer(), is(instanceOf(KeywordAnalyzer.class)));
+
+    }
+
+    public void testDoubleAlias() throws IOException {
+        Settings settings = Settings.builder()
+            .put("index.analysis.analyzer.foobar.alias","default")
+            .put("index.analysis.analyzer.foobar.type", "keyword")
+            .put("index.analysis.analyzer.barfoo.alias","default")
+            .put("index.analysis.analyzer.barfoo.type","english")
+            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
+            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_0_0)
+            .build();
+        AnalysisRegistry newRegistry = getNewRegistry(settings);
+        String message = expectThrows(IllegalStateException.class, () -> getAnalysisService(newRegistry, settings)).getMessage();
+        assertEquals("already registered analyzer with name: default", message);
+    }
+
     public void testVersionedAnalyzers() throws Exception {
         String yaml = "/org/elasticsearch/index/analysis/test1.yml";
         Settings settings2 = Settings.builder()

From 7b8ae54f0fa25e53c7953a057d863b7a571a6ec6 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Thu, 30 Jun 2016 14:26:32 +0200
Subject: [PATCH 1287/1311] percolator: Also support query term extract for
 queries wrapped inside a FunctionScoreQuery

Additionally for highlighting percolator hits, also extract percolator query from FunctionScoreQuery and DisjunctionMaxQuery
---
 .../search/function/FunctionScoreQuery.java     |  4 ++++
 .../percolator/ExtractQueryTermsService.java    | 17 ++++++++++++++++-
 .../PercolatorHighlightSubFetchPhase.java       | 11 +++++++++++
 .../ExtractQueryTermsServiceTests.java          | 15 +++++++++++++++
 .../PercolatorHighlightSubFetchPhaseTests.java  | 14 ++++++++++++++
 5 files changed, 60 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java
index be98a07a9c1..5e94e82021f 100644
--- a/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java
+++ b/core/src/main/java/org/elasticsearch/common/lucene/search/function/FunctionScoreQuery.java
@@ -73,6 +73,10 @@ public class FunctionScoreQuery extends Query {
         return function;
     }
 
+    public Float getMinScore() {
+        return minScore;
+    }
+
     @Override
     public Query rewrite(IndexReader reader) throws IOException {
         Query rewritten = super.rewrite(reader);
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java
index 147eae6e4d1..44048935304 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java
@@ -49,6 +49,7 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
+import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 import org.elasticsearch.index.mapper.ParseContext;
 
 import java.io.IOException;
@@ -75,7 +76,7 @@ public final class ExtractQueryTermsService {
     static final Map<Class<? extends Query>, Function<Query, Result>> queryProcessors;
 
     static {
-        Map<Class<? extends Query>, Function<Query, Result>> map = new HashMap<>(16);
+        Map<Class<? extends Query>, Function<Query, Result>> map = new HashMap<>(17);
         map.put(MatchNoDocsQuery.class, matchNoDocsQuery());
         map.put(ConstantScoreQuery.class, constantScoreQuery());
         map.put(BoostQuery.class, boostQuery());
@@ -92,6 +93,7 @@ public final class ExtractQueryTermsService {
         map.put(BooleanQuery.class, booleanQuery());
         map.put(DisjunctionMaxQuery.class, disjunctionMaxQuery());
         map.put(SynonymQuery.class, synonymQuery());
+        map.put(FunctionScoreQuery.class, functionScoreQuery());
         queryProcessors = Collections.unmodifiableMap(map);
     }
 
@@ -376,6 +378,19 @@ public final class ExtractQueryTermsService {
         };
     }
 
+    static Function<Query, Result> functionScoreQuery() {
+        return query -> {
+            FunctionScoreQuery functionScoreQuery = (FunctionScoreQuery) query;
+            Result result = extractQueryTerms(functionScoreQuery.getSubQuery());
+            // If min_score is specified we can't guarantee upfront that this percolator query matches,
+            // so in that case we set verified to false.
+            // (if it matches with the percolator document matches with the extracted terms.
+            // Min score filters out docs, which is different than the functions, which just influences the score.)
+            boolean verified = functionScoreQuery.getMinScore() == null;
+            return new Result(verified, result.terms);
+        };
+    }
+
     static Result handleDisjunction(List<Query> disjunctions, int minimumShouldMatch, boolean otherClauses) {
         boolean verified = minimumShouldMatch <= 1 && otherClauses == false;
         Set<Term> terms = new HashSet<>();
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
index 4a110172d77..b9f52a024e6 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java
@@ -25,9 +25,11 @@ import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.text.Text;
 import org.elasticsearch.index.query.ParsedQuery;
@@ -110,10 +112,19 @@ public final class PercolatorHighlightSubFetchPhase extends HighlightPhase {
                     return result;
                 }
             }
+        } else if (query instanceof DisjunctionMaxQuery) {
+            for (Query disjunct : ((DisjunctionMaxQuery) query).getDisjuncts()) {
+                PercolateQuery result = locatePercolatorQuery(disjunct);
+                if (result != null) {
+                    return result;
+                }
+            }
         } else if (query instanceof ConstantScoreQuery) {
             return locatePercolatorQuery(((ConstantScoreQuery) query).getQuery());
         } else if (query instanceof BoostQuery) {
             return locatePercolatorQuery(((BoostQuery) query).getQuery());
+        } else if (query instanceof FunctionScoreQuery) {
+            return locatePercolatorQuery(((FunctionScoreQuery) query).getSubQuery());
         }
 
         return null;
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java
index a050c8eb420..069aac9eda2 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java
@@ -46,6 +46,8 @@ import org.apache.lucene.search.spans.SpanOrQuery;
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
+import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
+import org.elasticsearch.common.lucene.search.function.RandomScoreFunction;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.percolator.ExtractQueryTermsService.Result;
 import org.elasticsearch.test.ESTestCase;
@@ -571,6 +573,19 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         assertTermsEqual(result.terms, new Term("_field", "_value1"), new Term("_field", "_value2"));
     }
 
+    public void testFunctionScoreQuery() {
+        TermQuery termQuery = new TermQuery(new Term("_field", "_value"));
+        FunctionScoreQuery functionScoreQuery = new FunctionScoreQuery(termQuery, new RandomScoreFunction());
+        Result result = extractQueryTerms(functionScoreQuery);
+        assertThat(result.verified, is(true));
+        assertTermsEqual(result.terms, new Term("_field", "_value"));
+
+        functionScoreQuery = new FunctionScoreQuery(termQuery, new RandomScoreFunction(), 1f, null, 10f);
+        result = extractQueryTerms(functionScoreQuery);
+        assertThat(result.verified, is(false));
+        assertTermsEqual(result.terms, new Term("_field", "_value"));
+    }
+
     public void testCreateQueryMetadataQuery() throws Exception {
         MemoryIndex memoryIndex = new MemoryIndex(false);
         memoryIndex.addField("field1", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer());
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
index ec4107fc2ed..69ef623f801 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
@@ -22,9 +22,12 @@ import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
+import org.elasticsearch.common.lucene.search.function.RandomScoreFunction;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.search.Highlighters;
 import org.elasticsearch.search.highlight.SearchContextHighlight;
@@ -32,6 +35,7 @@ import org.elasticsearch.search.internal.SearchContext;
 import org.elasticsearch.test.ESTestCase;
 import org.mockito.Mockito;
 
+import java.util.Arrays;
 import java.util.Collections;
 
 import static org.hamcrest.Matchers.is;
@@ -77,6 +81,16 @@ public class PercolatorHighlightSubFetchPhaseTests extends ESTestCase {
         assertThat(PercolatorHighlightSubFetchPhase.locatePercolatorQuery(boostQuery), nullValue());
         boostQuery = new BoostQuery(percolateQuery, 1f);
         assertThat(PercolatorHighlightSubFetchPhase.locatePercolatorQuery(boostQuery), sameInstance(percolateQuery));
+
+        FunctionScoreQuery functionScoreQuery = new FunctionScoreQuery(new MatchAllDocsQuery(), new RandomScoreFunction());
+        assertThat(PercolatorHighlightSubFetchPhase.locatePercolatorQuery(functionScoreQuery), nullValue());
+        functionScoreQuery = new FunctionScoreQuery(percolateQuery, new RandomScoreFunction());
+        assertThat(PercolatorHighlightSubFetchPhase.locatePercolatorQuery(functionScoreQuery), sameInstance(percolateQuery));
+
+        DisjunctionMaxQuery disjunctionMaxQuery = new DisjunctionMaxQuery(Arrays.asList(new MatchAllDocsQuery()), 1f);
+        assertThat(PercolatorHighlightSubFetchPhase.locatePercolatorQuery(disjunctionMaxQuery), nullValue());
+        disjunctionMaxQuery = new DisjunctionMaxQuery(Arrays.asList(percolateQuery, new MatchAllDocsQuery()), 1f);
+        assertThat(PercolatorHighlightSubFetchPhase.locatePercolatorQuery(disjunctionMaxQuery), sameInstance(percolateQuery));
     }
 
 }

From cfc762b70d86dcadefc5f1ea9b77c38183034602 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 8 Jul 2016 12:25:20 +0200
Subject: [PATCH 1288/1311] Rest Client: add slash to log line when missing
 between host and uri

Closes #19314
---
 .../org/elasticsearch/client/RequestLogger.java    | 14 +++++++++++---
 .../elasticsearch/client/RequestLoggerTests.java   | 11 +++++++++--
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java b/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java
index ad8204be2bb..5b9f4eadb92 100644
--- a/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java
+++ b/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java
@@ -26,6 +26,7 @@ import org.apache.http.HttpEntity;
 import org.apache.http.HttpEntityEnclosingRequest;
 import org.apache.http.HttpHost;
 import org.apache.http.HttpResponse;
+import org.apache.http.RequestLine;
 import org.apache.http.client.methods.HttpUriRequest;
 import org.apache.http.entity.BufferedHttpEntity;
 import org.apache.http.entity.ContentType;
@@ -55,7 +56,7 @@ final class RequestLogger {
      */
     static void logResponse(Log logger, HttpUriRequest request, HttpHost host, HttpResponse httpResponse) {
         if (logger.isDebugEnabled()) {
-            logger.debug("request [" + request.getMethod() + " " + host + request.getRequestLine().getUri() +
+            logger.debug("request [" + request.getMethod() + " " + host + getUri(request.getRequestLine()) +
                     "] returned [" + httpResponse.getStatusLine() + "]");
         }
         if (tracer.isTraceEnabled()) {
@@ -81,7 +82,7 @@ final class RequestLogger {
      * Logs a request that failed
      */
     static void logFailedRequest(Log logger, HttpUriRequest request, HttpHost host, IOException e) {
-        logger.debug("request [" + request.getMethod() + " " + host + request.getRequestLine().getUri() + "] failed", e);
+        logger.debug("request [" + request.getMethod() + " " + host + getUri(request.getRequestLine()) + "] failed", e);
         if (logger.isTraceEnabled()) {
             String traceRequest;
             try {
@@ -98,7 +99,7 @@ final class RequestLogger {
      * Creates curl output for given request
      */
     static String buildTraceRequest(HttpUriRequest request, HttpHost host) throws IOException {
-        String requestLine = "curl -iX " + request.getMethod() + " '" + host + request.getRequestLine().getUri() + "'";
+        String requestLine = "curl -iX " + request.getMethod() + " '" + host + getUri(request.getRequestLine()) + "'";
         if (request instanceof  HttpEntityEnclosingRequest) {
             HttpEntityEnclosingRequest enclosingRequest = (HttpEntityEnclosingRequest) request;
             if (enclosingRequest.getEntity() != null) {
@@ -143,4 +144,11 @@ final class RequestLogger {
         }
         return responseLine;
     }
+
+    private static String getUri(RequestLine requestLine) {
+        if (requestLine.getUri().charAt(0) != '/') {
+            return "/" + requestLine.getUri();
+        }
+        return requestLine.getUri();
+    }
 }
diff --git a/client/rest/src/test/java/org/elasticsearch/client/RequestLoggerTests.java b/client/rest/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
index 190774d46fc..4d3ad75b5e8 100644
--- a/client/rest/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
+++ b/client/rest/src/test/java/org/elasticsearch/client/RequestLoggerTests.java
@@ -50,7 +50,14 @@ public class RequestLoggerTests extends RestClientTestCase {
 
     public void testTraceRequest() throws IOException, URISyntaxException {
         HttpHost host = new HttpHost("localhost", 9200, getRandom().nextBoolean() ? "http" : "https");
-        URI uri = new URI("/index/type/_api");
+
+        String expectedEndpoint = "/index/type/_api";
+        URI uri;
+        if (randomBoolean()) {
+            uri = new URI(expectedEndpoint);
+        } else {
+            uri = new URI("index/type/_api");
+        }
 
         HttpRequestBase request;
         int requestType = RandomInts.randomIntBetween(getRandom(), 0, 7);
@@ -83,7 +90,7 @@ public class RequestLoggerTests extends RestClientTestCase {
                 throw new UnsupportedOperationException();
         }
 
-        String expected = "curl -iX " + request.getMethod() + " '" + host + uri + "'";
+        String expected = "curl -iX " + request.getMethod() + " '" + host + expectedEndpoint + "'";
         boolean hasBody = request instanceof HttpEntityEnclosingRequest && getRandom().nextBoolean();
         String requestBody = "{ \"field\": \"value\" }";
         if (hasBody) {

From c63719d08549433937aee36a0322e5bff44aaa65 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 8 Jul 2016 12:51:48 +0200
Subject: [PATCH 1289/1311] Rest Client: wrap log statement in
 logger.isDebugEnabled

---
 .../src/main/java/org/elasticsearch/client/RequestLogger.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java b/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java
index 5b9f4eadb92..a2b46e92846 100644
--- a/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java
+++ b/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java
@@ -82,7 +82,9 @@ final class RequestLogger {
      * Logs a request that failed
      */
     static void logFailedRequest(Log logger, HttpUriRequest request, HttpHost host, IOException e) {
-        logger.debug("request [" + request.getMethod() + " " + host + getUri(request.getRequestLine()) + "] failed", e);
+        if (logger.isDebugEnabled()) {
+            logger.debug("request [" + request.getMethod() + " " + host + getUri(request.getRequestLine()) + "] failed", e);
+        }
         if (logger.isTraceEnabled()) {
             String traceRequest;
             try {

From 2a91a6ac37a5ddfbf09ee0b5f33179618063bd3c Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 8 Jul 2016 12:52:29 +0200
Subject: [PATCH 1290/1311] Rest Client: check log level against tracer logger
 for trace logging

---
 .../src/main/java/org/elasticsearch/client/RequestLogger.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java b/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java
index a2b46e92846..24e6881fa1e 100644
--- a/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java
+++ b/client/rest/src/main/java/org/elasticsearch/client/RequestLogger.java
@@ -85,7 +85,7 @@ final class RequestLogger {
         if (logger.isDebugEnabled()) {
             logger.debug("request [" + request.getMethod() + " " + host + getUri(request.getRequestLine()) + "] failed", e);
         }
-        if (logger.isTraceEnabled()) {
+        if (tracer.isTraceEnabled()) {
             String traceRequest;
             try {
                 traceRequest = buildTraceRequest(request, host);

From d29d2f8793be097690093a7b7f665f85f036b355 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 8 Jul 2016 07:58:52 -0400
Subject: [PATCH 1291/1311] Disable service in pre-uninstall

Today in the packaging removal scripts, we disable the service in
post-uninstall. Yet, this happens after service files have been
erased. On some systems, this can cause the service disable to fail
leaving behind state causing the service to be enabled on subsequent
installs. This commit moves the service disabling to the pre-uninstall
script to prevent this issue.

Relates #19328
---
 distribution/src/main/packaging/scripts/postrm | 17 -----------------
 distribution/src/main/packaging/scripts/prerm  | 17 +++++++++++++++++
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/distribution/src/main/packaging/scripts/postrm b/distribution/src/main/packaging/scripts/postrm
index 2fa42678d96..19c57eafa2b 100644
--- a/distribution/src/main/packaging/scripts/postrm
+++ b/distribution/src/main/packaging/scripts/postrm
@@ -13,7 +13,6 @@
 
 SOURCE_ENV_FILE=true
 REMOVE_DIRS=false
-REMOVE_SERVICE=false
 REMOVE_USER_AND_GROUP=false
 
 case "$1" in
@@ -21,7 +20,6 @@ case "$1" in
     # Debian ####################################################
     remove)
         REMOVE_DIRS=true
-        REMOVE_SERVICE=true
     ;;
 
     purge)
@@ -34,7 +32,6 @@ case "$1" in
     # RedHat ####################################################
     0)
         REMOVE_DIRS=true
-        REMOVE_SERVICE=true
         REMOVE_USER_AND_GROUP=true
     ;;
     1)
@@ -65,20 +62,6 @@ if [ "$SOURCE_ENV_FILE" = "true" ]; then
     fi
 fi
 
-if [ "$REMOVE_SERVICE" = "true" ]; then
-    if command -v systemctl >/dev/null; then
-        systemctl disable elasticsearch.service > /dev/null 2>&1 || true
-    fi
-
-    if command -v chkconfig >/dev/null; then
-        chkconfig --del elasticsearch 2> /dev/null || true
-    fi
-
-    if command -v update-rc.d >/dev/null; then
-        update-rc.d elasticsearch remove >/dev/null || true
-    fi
-fi
-
 if [ "$REMOVE_DIRS" = "true" ]; then
 
     if [ -d "$LOG_DIR" ]; then
diff --git a/distribution/src/main/packaging/scripts/prerm b/distribution/src/main/packaging/scripts/prerm
index 79523629f77..592f2f9948f 100644
--- a/distribution/src/main/packaging/scripts/prerm
+++ b/distribution/src/main/packaging/scripts/prerm
@@ -12,12 +12,14 @@
 
 
 STOP_REQUIRED=false
+REMOVE_SERVICE=false
 
 case "$1" in
 
     # Debian ####################################################
     remove)
         STOP_REQUIRED=true
+        REMOVE_SERVICE=true
     ;;
     upgrade)
         if [ "$RESTART_ON_UPGRADE" = "true" ]; then
@@ -30,6 +32,7 @@ case "$1" in
     # RedHat ####################################################
     0)
         STOP_REQUIRED=true
+        REMOVE_SERVICE=true
     ;;
     1)
         # Dont do anything on upgrade, because the preun script in redhat gets executed after the postinst (madness!)
@@ -64,6 +67,20 @@ if [ "$STOP_REQUIRED" = "true" ]; then
     echo " OK"
 fi
 
+if [ "$REMOVE_SERVICE" = "true" ]; then
+    if command -v systemctl >/dev/null; then
+        systemctl disable elasticsearch.service > /dev/null 2>&1 || true
+    fi
+
+    if command -v chkconfig >/dev/null; then
+        chkconfig --del elasticsearch 2> /dev/null || true
+    fi
+
+    if command -v update-rc.d >/dev/null; then
+        update-rc.d elasticsearch remove >/dev/null || true
+    fi
+fi
+
 SCRIPTS_DIR="/etc/elasticsearch/scripts"
 # delete the scripts directory if and only if empty
 if [ -d "$SCRIPTS_DIR" ]; then

From f6ac147b1d83d50037d5d03fe0e80b8929134511 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Fri, 8 Jul 2016 14:13:36 +0200
Subject: [PATCH 1292/1311] Add a unit test that sends random requests among 3
 nodes (#19329)

This adds a test that uses transport implementation and sends random requests
to 3 different nodes, the request handlers maybe forwarding the requests to yet another node
etc. until returning the response. This test basically tests that nodes are not deadlocking
in a distributed fashion.
---
 .../AbstractSimpleTransportTestCase.java      | 476 +++++++++++++-----
 1 file changed, 354 insertions(+), 122 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
index c33db33b1f8..b65b313ebc7 100644
--- a/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
+++ b/core/src/test/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java
@@ -19,8 +19,8 @@
 
 package org.elasticsearch.transport;
 
+import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.Version;
-import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -37,6 +37,8 @@ import org.junit.After;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Semaphore;
@@ -51,9 +53,6 @@ import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.notNullValue;
 
-/**
- *
- */
 public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
     protected ThreadPool threadPool;
@@ -62,7 +61,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
     protected DiscoveryNode nodeA;
     protected MockTransportService serviceA;
 
-    protected static final Version version1 = Version.fromId(Version.CURRENT.id+1);
+    protected static final Version version1 = Version.fromId(Version.CURRENT.id + 1);
     protected DiscoveryNode nodeB;
     protected MockTransportService serviceB;
 
@@ -74,24 +73,25 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         super.setUp();
         threadPool = new TestThreadPool(getClass().getName());
         serviceA = build(
-                Settings.builder()
-                    .put("name", "TS_A")
-                    .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "")
-                    .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
-                    .build(),
-                version0);
+            Settings.builder()
+                .put("name", "TS_A")
+                .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "")
+                .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
+                .build(),
+            version0);
         serviceA.acceptIncomingRequests();
         nodeA = new DiscoveryNode("TS_A", serviceA.boundAddress().publishAddress(), emptyMap(), emptySet(), version0);
+        // serviceA.setLocalNode(nodeA);
         serviceB = build(
-                Settings.builder()
-                    .put("name", "TS_B")
-                    .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "")
-                    .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
-                    .build(),
-                version1);
+            Settings.builder()
+                .put("name", "TS_B")
+                .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "")
+                .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
+                .build(),
+            version1);
         serviceB.acceptIncomingRequests();
         nodeB = new DiscoveryNode("TS_B", serviceB.boundAddress().publishAddress(), emptyMap(), emptySet(), version1);
-
+        //serviceB.setLocalNode(nodeB);
         // wait till all nodes are properly connected and the event has been sent, so tests in this class
         // will not get this callback called on the connections done in this setup
         final boolean useLocalNode = randomBoolean();
@@ -140,41 +140,41 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
     public void testHelloWorld() {
         serviceA.registerRequestHandler("sayHello", StringMessageRequest::new, ThreadPool.Names.GENERIC,
             new TransportRequestHandler<StringMessageRequest>() {
-            @Override
-            public void messageReceived(StringMessageRequest request, TransportChannel channel) {
-                assertThat("moshe", equalTo(request.message));
-                try {
-                    channel.sendResponse(new StringMessageResponse("hello " + request.message));
-                } catch (IOException e) {
-                    logger.error("Unexpected failure", e);
-                    fail(e.getMessage());
+                @Override
+                public void messageReceived(StringMessageRequest request, TransportChannel channel) {
+                    assertThat("moshe", equalTo(request.message));
+                    try {
+                        channel.sendResponse(new StringMessageResponse("hello " + request.message));
+                    } catch (IOException e) {
+                        logger.error("Unexpected failure", e);
+                        fail(e.getMessage());
+                    }
                 }
-            }
-        });
+            });
 
         TransportFuture<StringMessageResponse> res = serviceB.submitRequest(nodeA, "sayHello",
-                new StringMessageRequest("moshe"), new TransportResponseHandler<StringMessageResponse>() {
-                    @Override
-                    public StringMessageResponse newInstance() {
-                        return new StringMessageResponse();
-                    }
+            new StringMessageRequest("moshe"), new TransportResponseHandler<StringMessageResponse>() {
+                @Override
+                public StringMessageResponse newInstance() {
+                    return new StringMessageResponse();
+                }
 
-                    @Override
-                    public String executor() {
-                        return ThreadPool.Names.GENERIC;
-                    }
+                @Override
+                public String executor() {
+                    return ThreadPool.Names.GENERIC;
+                }
 
-                    @Override
-                    public void handleResponse(StringMessageResponse response) {
-                        assertThat("hello moshe", equalTo(response.message));
-                    }
+                @Override
+                public void handleResponse(StringMessageResponse response) {
+                    assertThat("hello moshe", equalTo(response.message));
+                }
 
-                    @Override
-                    public void handleException(TransportException exp) {
-                        logger.error("Unexpected failure", exp);
-                        fail("got exception instead of a response: " + exp.getMessage());
-                    }
-                });
+                @Override
+                public void handleException(TransportException exp) {
+                    logger.error("Unexpected failure", exp);
+                    fail("got exception instead of a response: " + exp.getMessage());
+                }
+            });
 
         try {
             StringMessageResponse message = res.get();
@@ -185,27 +185,27 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
         res = serviceB.submitRequest(nodeA, "sayHello", new StringMessageRequest("moshe"),
             TransportRequestOptions.builder().withCompress(true).build(), new TransportResponseHandler<StringMessageResponse>() {
-                    @Override
-                    public StringMessageResponse newInstance() {
-                        return new StringMessageResponse();
-                    }
+                @Override
+                public StringMessageResponse newInstance() {
+                    return new StringMessageResponse();
+                }
 
-                    @Override
-                    public String executor() {
-                        return ThreadPool.Names.GENERIC;
-                    }
+                @Override
+                public String executor() {
+                    return ThreadPool.Names.GENERIC;
+                }
 
-                    @Override
-                    public void handleResponse(StringMessageResponse response) {
-                        assertThat("hello moshe", equalTo(response.message));
-                    }
+                @Override
+                public void handleResponse(StringMessageResponse response) {
+                    assertThat("hello moshe", equalTo(response.message));
+                }
 
-                    @Override
-                    public void handleException(TransportException exp) {
-                        logger.error("Unexpected failure", exp);
-                        fail("got exception instead of a response: " + exp.getMessage());
-                    }
-                });
+                @Override
+                public void handleException(TransportException exp) {
+                    logger.error("Unexpected failure", exp);
+                    fail("got exception instead of a response: " + exp.getMessage());
+                }
+            });
 
         try {
             StringMessageResponse message = res.get();
@@ -335,7 +335,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
             });
 
         TransportFuture<TransportResponse.Empty> res = serviceB.submitRequest(nodeA, "sayHello",
-                TransportRequest.Empty.INSTANCE, TransportRequestOptions.builder().withCompress(true).build(),
+            TransportRequest.Empty.INSTANCE, TransportRequestOptions.builder().withCompress(true).build(),
             new TransportResponseHandler<TransportResponse.Empty>() {
                 @Override
                 public TransportResponse.Empty newInstance() {
@@ -382,10 +382,10 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
                         fail(e.getMessage());
                     }
                 }
-        });
+            });
 
         TransportFuture<StringMessageResponse> res = serviceB.submitRequest(nodeA, "sayHello",
-                new StringMessageRequest("moshe"), TransportRequestOptions.builder().withCompress(true).build(),
+            new StringMessageRequest("moshe"), TransportRequestOptions.builder().withCompress(true).build(),
             new TransportResponseHandler<StringMessageResponse>() {
                 @Override
                 public StringMessageResponse newInstance() {
@@ -427,30 +427,30 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
                     assertThat("moshe", equalTo(request.message));
                     throw new RuntimeException("bad message !!!");
                 }
-        });
+            });
 
         TransportFuture<StringMessageResponse> res = serviceB.submitRequest(nodeA, "sayHelloException",
-                new StringMessageRequest("moshe"), new TransportResponseHandler<StringMessageResponse>() {
-                    @Override
-                    public StringMessageResponse newInstance() {
-                        return new StringMessageResponse();
-                    }
+            new StringMessageRequest("moshe"), new TransportResponseHandler<StringMessageResponse>() {
+                @Override
+                public StringMessageResponse newInstance() {
+                    return new StringMessageResponse();
+                }
 
-                    @Override
-                    public String executor() {
-                        return ThreadPool.Names.GENERIC;
-                    }
+                @Override
+                public String executor() {
+                    return ThreadPool.Names.GENERIC;
+                }
 
-                    @Override
-                    public void handleResponse(StringMessageResponse response) {
-                        fail("got response instead of exception");
-                    }
+                @Override
+                public void handleResponse(StringMessageResponse response) {
+                    fail("got response instead of exception");
+                }
 
-                    @Override
-                    public void handleException(TransportException exp) {
-                        assertThat("runtime_exception: bad message !!!", equalTo(exp.getCause().getMessage()));
-                    }
-                });
+                @Override
+                public void handleException(TransportException exp) {
+                    assertThat("runtime_exception: bad message !!!", equalTo(exp.getCause().getMessage()));
+                }
+            });
 
         try {
             res.txGet();
@@ -497,7 +497,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
                 }
             });
         TransportFuture<TransportResponse.Empty> foobar = serviceB.submitRequest(nodeA, "foobar",
-                new StringMessageRequest(""), TransportRequestOptions.EMPTY, EmptyTransportResponseHandler.INSTANCE_SAME);
+            new StringMessageRequest(""), TransportRequestOptions.EMPTY, EmptyTransportResponseHandler.INSTANCE_SAME);
         latch2.countDown();
         try {
             foobar.txGet();
@@ -519,7 +519,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
             });
 
         TransportFuture<StringMessageResponse> res = serviceB.submitRequest(nodeA, "sayHelloTimeoutNoResponse",
-                new StringMessageRequest("moshe"), TransportRequestOptions.builder().withTimeout(100).build(),
+            new StringMessageRequest("moshe"), TransportRequestOptions.builder().withTimeout(100).build(),
             new TransportResponseHandler<StringMessageResponse>() {
                 @Override
                 public StringMessageResponse newInstance() {
@@ -571,10 +571,10 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
                         fail(e.getMessage());
                     }
                 }
-        });
+            });
         final CountDownLatch latch = new CountDownLatch(1);
         TransportFuture<StringMessageResponse> res = serviceB.submitRequest(nodeA, "sayHelloTimeoutDelayedResponse",
-                new StringMessageRequest("2m"), TransportRequestOptions.builder().withTimeout(100).build(),
+            new StringMessageRequest("2m"), TransportRequestOptions.builder().withTimeout(100).build(),
             new TransportResponseHandler<StringMessageResponse>() {
                 @Override
                 public StringMessageResponse newInstance() {
@@ -611,7 +611,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
             final int counter = i;
             // now, try and send another request, this times, with a short timeout
             res = serviceB.submitRequest(nodeA, "sayHelloTimeoutDelayedResponse",
-                    new StringMessageRequest(counter + "ms"), TransportRequestOptions.builder().withTimeout(3000).build(),
+                new StringMessageRequest(counter + "ms"), TransportRequestOptions.builder().withTimeout(3000).build(),
                 new TransportResponseHandler<StringMessageResponse>() {
                     @Override
                     public StringMessageResponse newInstance() {
@@ -698,7 +698,8 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
         tracer.reset(4);
         boolean timeout = randomBoolean();
-        TransportRequestOptions options = timeout ? TransportRequestOptions.builder().withTimeout(1).build(): TransportRequestOptions.EMPTY;
+        TransportRequestOptions options = timeout ? TransportRequestOptions.builder().withTimeout(1).build() :
+            TransportRequestOptions.EMPTY;
         serviceA.sendRequest(nodeB, "test", new StringMessageRequest("", 10), options, noopResponseHandler);
         requestCompleted.acquire();
         tracer.expectedEvents.get().await();
@@ -1137,27 +1138,27 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         serviceB.addFailToSendNoConnectRule(serviceA);
 
         TransportFuture<StringMessageResponse> res = serviceB.submitRequest(nodeA, "sayHello",
-                new StringMessageRequest("moshe"), new TransportResponseHandler<StringMessageResponse>() {
-                    @Override
-                    public StringMessageResponse newInstance() {
-                        return new StringMessageResponse();
-                    }
+            new StringMessageRequest("moshe"), new TransportResponseHandler<StringMessageResponse>() {
+                @Override
+                public StringMessageResponse newInstance() {
+                    return new StringMessageResponse();
+                }
 
-                    @Override
-                    public String executor() {
-                        return ThreadPool.Names.GENERIC;
-                    }
+                @Override
+                public String executor() {
+                    return ThreadPool.Names.GENERIC;
+                }
 
-                    @Override
-                    public void handleResponse(StringMessageResponse response) {
-                        fail("got response instead of exception");
-                    }
+                @Override
+                public void handleResponse(StringMessageResponse response) {
+                    fail("got response instead of exception");
+                }
 
-                    @Override
-                    public void handleException(TransportException exp) {
-                        assertThat(exp.getCause().getMessage(), endsWith("DISCONNECT: simulated"));
-                    }
-                });
+                @Override
+                public void handleException(TransportException exp) {
+                    assertThat(exp.getCause().getMessage(), endsWith("DISCONNECT: simulated"));
+                }
+            });
 
         try {
             res.txGet();
@@ -1196,7 +1197,7 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
         serviceB.addUnresponsiveRule(serviceA);
 
         TransportFuture<StringMessageResponse> res = serviceB.submitRequest(nodeA, "sayHello",
-                new StringMessageRequest("moshe"), TransportRequestOptions.builder().withTimeout(100).build(),
+            new StringMessageRequest("moshe"), TransportRequestOptions.builder().withTimeout(100).build(),
             new TransportResponseHandler<StringMessageResponse>() {
                 @Override
                 public StringMessageResponse newInstance() {
@@ -1289,18 +1290,18 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
 
     public void testBlockingIncomingRequests() throws Exception {
         TransportService service = build(
-                Settings.builder()
-                    .put("name", "TS_TEST")
-                    .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "")
-                    .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
-                    .build(),
-                version0);
+            Settings.builder()
+                .put("name", "TS_TEST")
+                .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "")
+                .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
+                .build(),
+            version0);
         AtomicBoolean requestProcessed = new AtomicBoolean();
         service.registerRequestHandler("action", TestRequest::new, ThreadPool.Names.SAME,
-                (request, channel) -> {
-                    requestProcessed.set(true);
-                    channel.sendResponse(TransportResponse.Empty.INSTANCE);
-                });
+            (request, channel) -> {
+                requestProcessed.set(true);
+                channel.sendResponse(TransportResponse.Empty.INSTANCE);
+            });
 
         DiscoveryNode node =
             new DiscoveryNode("TS_TEST", "TS_TEST", service.boundAddress().publishAddress(), emptyMap(), emptySet(), version0);
@@ -1340,8 +1341,239 @@ public abstract class AbstractSimpleTransportTestCase extends ESTestCase {
     }
 
     public static class TestRequest extends TransportRequest {
+
+        String info;
+        int resendCount;
+
+        public TestRequest() {
+        }
+
+        public TestRequest(String info) {
+            this.info = info;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            info = in.readOptionalString();
+            resendCount = in.readInt();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeOptionalString(info);
+            out.writeInt(resendCount);
+        }
+
+        @Override
+        public String toString() {
+            return "TestRequest{" +
+                "info='" + info + '\'' +
+                '}';
+        }
     }
 
     private static class TestResponse extends TransportResponse {
+
+        String info;
+
+        public TestResponse() {
+        }
+
+        public TestResponse(String info) {
+            this.info = info;
+        }
+
+        @Override
+        public void readFrom(StreamInput in) throws IOException {
+            super.readFrom(in);
+            info = in.readOptionalString();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            super.writeTo(out);
+            out.writeOptionalString(info);
+        }
+
+        @Override
+        public String toString() {
+            return "TestResponse{" +
+                "info='" + info + '\'' +
+                '}';
+        }
+    }
+
+    public void testSendRandomRequests() throws InterruptedException {
+        TransportService serviceC = build(
+            Settings.builder()
+                .put("name", "TS_TEST")
+                .put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "")
+                .put(TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING")
+                .build(),
+            version0);
+        DiscoveryNode nodeC =
+            new DiscoveryNode("TS_C", "TS_C", serviceC.boundAddress().publishAddress(), emptyMap(), emptySet(), version0);
+        serviceC.acceptIncomingRequests();
+
+        final CountDownLatch latch = new CountDownLatch(5);
+        TransportConnectionListener waitForConnection = new TransportConnectionListener() {
+            @Override
+            public void onNodeConnected(DiscoveryNode node) {
+                latch.countDown();
+            }
+
+            @Override
+            public void onNodeDisconnected(DiscoveryNode node) {
+                fail("disconnect should not be called " + node);
+            }
+        };
+        serviceA.addConnectionListener(waitForConnection);
+        serviceB.addConnectionListener(waitForConnection);
+        serviceC.addConnectionListener(waitForConnection);
+
+        serviceC.connectToNode(nodeA);
+        serviceC.connectToNode(nodeB);
+        serviceA.connectToNode(nodeC);
+        serviceB.connectToNode(nodeC);
+        serviceC.connectToNode(nodeC);
+
+        latch.await();
+        serviceA.removeConnectionListener(waitForConnection);
+        serviceB.removeConnectionListener(waitForConnection);
+        serviceB.removeConnectionListener(waitForConnection);
+
+
+        Map<TransportService, DiscoveryNode> toNodeMap = new HashMap<>();
+        toNodeMap.put(serviceA, nodeA);
+        toNodeMap.put(serviceB, nodeB);
+        toNodeMap.put(serviceC, nodeC);
+        AtomicBoolean fail = new AtomicBoolean(false);
+        class TestRequestHandler implements TransportRequestHandler<TestRequest> {
+
+            private final TransportService service;
+
+            TestRequestHandler(TransportService service) {
+                this.service = service;
+            }
+
+            @Override
+            public void messageReceived(TestRequest request, TransportChannel channel) throws Exception {
+                if (randomBoolean()) {
+                    Thread.sleep(randomIntBetween(10, 50));
+                }
+                if (fail.get()) {
+                    throw new IOException("forced failure");
+                }
+
+                if (randomBoolean() && request.resendCount++ < 20) {
+                    DiscoveryNode node = randomFrom(nodeA, nodeB, nodeC);
+                    logger.debug("send secondary request from {} to {} - {}", toNodeMap.get(service), node, request.info);
+                    service.sendRequest(node, "action1", new TestRequest("secondary " + request.info),
+                        TransportRequestOptions.builder().withCompress(randomBoolean()).build(),
+                        new TransportResponseHandler<TestResponse>() {
+                            @Override
+                            public TestResponse newInstance() {
+                                return new TestResponse();
+                            }
+
+                            @Override
+                            public void handleResponse(TestResponse response) {
+                                try {
+                                    if (randomBoolean()) {
+                                        Thread.sleep(randomIntBetween(10, 50));
+                                    }
+                                    logger.debug("send secondary response {}", response.info);
+
+                                    channel.sendResponse(response);
+                                } catch (Exception e) {
+                                    throw new RuntimeException(e);
+                                }
+                            }
+
+                            @Override
+                            public void handleException(TransportException exp) {
+                                try {
+                                    logger.debug("send secondary exception response for request {}", request.info);
+                                    channel.sendResponse(exp);
+                                } catch (Exception e) {
+                                    throw new RuntimeException(e);
+                                }
+                            }
+
+                            @Override
+                            public String executor() {
+                                return randomBoolean() ? ThreadPool.Names.SAME : ThreadPool.Names.GENERIC;
+                            }
+                        });
+                } else {
+                    logger.debug("send response for {}", request.info);
+                    channel.sendResponse(new TestResponse("Response for: " + request.info));
+                }
+
+            }
+        }
+        serviceB.registerRequestHandler("action1", TestRequest::new, randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC),
+            new TestRequestHandler(serviceB));
+        serviceC.registerRequestHandler("action1", TestRequest::new, randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC),
+            new TestRequestHandler(serviceC));
+        serviceA.registerRequestHandler("action1", TestRequest::new, randomFrom(ThreadPool.Names.SAME, ThreadPool.Names.GENERIC),
+            new TestRequestHandler(serviceA));
+        int iters = randomIntBetween(30, 60);
+        CountDownLatch allRequestsDone = new CountDownLatch(iters);
+        class TestResponseHandler implements TransportResponseHandler<TestResponse> {
+
+            private final int id;
+
+            public TestResponseHandler(int id) {
+                this.id = id;
+            }
+
+            @Override
+            public TestResponse newInstance() {
+                return new TestResponse();
+            }
+
+            @Override
+            public void handleResponse(TestResponse response) {
+                logger.debug("---> received response: {}", response.info);
+                allRequestsDone.countDown();
+            }
+
+            @Override
+            public void handleException(TransportException exp) {
+                logger.debug("---> received exception for id {}", exp, id);
+                allRequestsDone.countDown();
+                Throwable unwrap = ExceptionsHelper.unwrap(exp, IOException.class);
+                assertNotNull(unwrap);
+                assertEquals(IOException.class, unwrap.getClass());
+                assertEquals("forced failure", unwrap.getMessage());
+            }
+
+            @Override
+            public String executor() {
+                return randomBoolean() ? ThreadPool.Names.SAME : ThreadPool.Names.GENERIC;
+            }
+        }
+
+        for (int i = 0; i < iters; i++) {
+            TransportService service = randomFrom(serviceC, serviceB, serviceA);
+            DiscoveryNode node = randomFrom(nodeC, nodeB, nodeA);
+            logger.debug("send from {} to {}", toNodeMap.get(service), node);
+            service.sendRequest(node, "action1", new TestRequest("REQ[" + i + "]"),
+                TransportRequestOptions.builder().withCompress(randomBoolean()).build(), new TestResponseHandler(i));
+        }
+        logger.debug("waiting for response");
+        fail.set(randomBoolean());
+        boolean await = allRequestsDone.await(5, TimeUnit.SECONDS);
+        if (await == false) {
+            logger.debug("now failing forcefully");
+            fail.set(true);
+            assertTrue(allRequestsDone.await(5, TimeUnit.SECONDS));
+        }
+        logger.debug("DONE");
+        serviceC.close();
+
     }
 }

From ff5527f0375b788a6562af96360fc34d0a083043 Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Fri, 8 Jul 2016 12:32:45 +0200
Subject: [PATCH 1293/1311] percolator: Forbid the usage or `range` queries
 with a range based on the current time

If there are percolator queries containing `range` queries with ranges based on the current time then this can lead to incorrect results if the `percolate` query gets cached.  These ranges are changing each time the `percolate` query gets executed and if this query gets cached then the results will be based on how the range was at the time when the `percolate` query got cached.

The ExtractQueryTermsService has been renamed `QueryAnalyzer` and now only deals with analyzing the query (extracting terms and deciding if the entire query is a verified match) . The `PercolatorFieldMapper` is responsible for adding the right fields based on the analysis the `QueryAnalyzer` has performed, because this is highly dependent on the field mappings. Also the `PercolatorFieldMapper` is responsible for creating the percolate query.
---
 .../mapping/types/percolator.asciidoc         |   5 +-
 .../migration/migrate_5_0/percolator.asciidoc |   5 +
 .../percolator/MultiPercolateAction.java      |   1 +
 .../percolator/PercolateAction.java           |   1 +
 .../percolator/PercolateQuery.java            | 120 +----
 .../percolator/PercolateQueryBuilder.java     |  36 +-
 .../percolator/PercolatorFieldMapper.java     | 166 +++++-
 ...ryTermsService.java => QueryAnalyzer.java} | 132 ++---
 .../percolator/RestMultiPercolateAction.java  |   1 +
 .../percolator/RestPercolateAction.java       |   1 +
 .../TransportMultiPercolateAction.java        |   1 +
 .../percolator/TransportPercolateAction.java  |   1 +
 .../percolator/CandidateQueryTests.java       | 436 ++++++++++++++++
 .../percolator/PercolateQueryTests.java       | 492 +++---------------
 .../PercolatorFieldMapperTests.java           | 209 +++++++-
 ...PercolatorHighlightSubFetchPhaseTests.java |  14 +-
 .../percolator/PercolatorIT.java              |  25 -
 .../percolator/PercolatorQuerySearchIT.java   |  97 ++++
 ...viceTests.java => QueryAnalyzerTests.java} | 199 ++-----
 19 files changed, 1063 insertions(+), 879 deletions(-)
 rename modules/percolator/src/main/java/org/elasticsearch/percolator/{ExtractQueryTermsService.java => QueryAnalyzer.java} (73%)
 create mode 100644 modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
 rename modules/percolator/src/test/java/org/elasticsearch/percolator/{ExtractQueryTermsServiceTests.java => QueryAnalyzerTests.java} (74%)

diff --git a/docs/reference/mapping/types/percolator.asciidoc b/docs/reference/mapping/types/percolator.asciidoc
index 4c7ff113fa9..ca8c8386e9b 100644
--- a/docs/reference/mapping/types/percolator.asciidoc
+++ b/docs/reference/mapping/types/percolator.asciidoc
@@ -71,11 +71,14 @@ a percolator query does not exist, it will be handled as a default string field
 fail.
 
 [float]
-==== Important Notes
+==== Limitations
 
 Because the `percolate` query is processing one document at a time, it doesn't support queries and filters that run
 against child documents such as `has_child` and `has_parent`.
 
+The percolator doesn't accepts percolator queries containing `range` queries with ranges that are based on current
+time (using `now`).
+
 There are a number of queries that fetch data via a get call during query parsing. For example the `terms` query when
 using terms lookup, `template` query when using indexed scripts and `geo_shape` when using pre-indexed shapes. When these
 queries are indexed by the `percolator` field type then the get call is executed once. So each time the `percolator`
diff --git a/docs/reference/migration/migrate_5_0/percolator.asciidoc b/docs/reference/migration/migrate_5_0/percolator.asciidoc
index ae2057bddfb..f173a0df958 100644
--- a/docs/reference/migration/migrate_5_0/percolator.asciidoc
+++ b/docs/reference/migration/migrate_5_0/percolator.asciidoc
@@ -48,6 +48,11 @@ the existing document.
 
 The percolate stats have been removed. This is because the percolator no longer caches the percolator queries.
 
+==== Percolator queries containing range queries with now ranges
+
+The percolator no longer accepts percolator queries containing `range` queries with ranges that are based on current
+time (using `now`).
+
 ==== Java client
 
 The percolator is no longer part of the core elasticsearch dependency. It has moved to the percolator module.
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateAction.java
index eefc6c996ba..d6eb5660728 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/MultiPercolateAction.java
@@ -21,6 +21,7 @@ package org.elasticsearch.percolator;
 import org.elasticsearch.action.Action;
 import org.elasticsearch.client.ElasticsearchClient;
 
+@Deprecated
 public class MultiPercolateAction extends Action<MultiPercolateRequest, MultiPercolateResponse, MultiPercolateRequestBuilder> {
 
     public static final MultiPercolateAction INSTANCE = new MultiPercolateAction();
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateAction.java
index 64776f271ae..cebca9ed825 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateAction.java
@@ -22,6 +22,7 @@ package org.elasticsearch.percolator;
 import org.elasticsearch.action.Action;
 import org.elasticsearch.client.ElasticsearchClient;
 
+@Deprecated
 public class PercolateAction extends Action<PercolateRequest, PercolateResponse, PercolateRequestBuilder> {
 
     public static final PercolateAction INSTANCE = new PercolateAction();
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
index 76bc136656c..40218e50a4f 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java
@@ -22,13 +22,11 @@ package org.elasticsearch.percolator;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TwoPhaseIterator;
 import org.apache.lucene.search.Weight;
@@ -36,115 +34,39 @@ import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.Bits;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 
 import java.io.IOException;
 import java.util.Objects;
 import java.util.Set;
 
-import static org.apache.lucene.search.BooleanClause.Occur.FILTER;
-
-public final class PercolateQuery extends Query implements Accountable {
+final class PercolateQuery extends Query implements Accountable {
 
     // cost of matching the query against the document, arbitrary as it would be really complex to estimate
     public static final float MATCH_COST = 1000;
 
-    public static class Builder {
-
-        private final String docType;
-        private final QueryStore queryStore;
-        private final BytesReference documentSource;
-        private final IndexSearcher percolatorIndexSearcher;
-
-        private Query queriesMetaDataQuery;
-        private Query verifiedQueriesQuery = new MatchNoDocsQuery("");
-        private Query percolateTypeQuery;
-
-        /**
-         * @param docType                   The type of the document being percolated
-         * @param queryStore                The lookup holding all the percolator queries as Lucene queries.
-         * @param documentSource            The source of the document being percolated
-         * @param percolatorIndexSearcher   The index searcher on top of the in-memory index that holds the document being percolated
-         */
-        public Builder(String docType, QueryStore queryStore, BytesReference documentSource, IndexSearcher percolatorIndexSearcher) {
-            this.docType = Objects.requireNonNull(docType);
-            this.queryStore = Objects.requireNonNull(queryStore);
-            this.documentSource = Objects.requireNonNull(documentSource);
-            this.percolatorIndexSearcher = Objects.requireNonNull(percolatorIndexSearcher);
-        }
-
-        /**
-         * Optionally sets a query that reduces the number of queries to percolate based on extracted terms from
-         * the document to be percolated.
-         * @param extractedTermsFieldName   The name of the field to get the extracted terms from
-         * @param extractionResultField     The field to indicate for a document whether query term extraction was complete,
-         *                                  partial or failed. If query extraction was complete, the MemoryIndex doesn't
-         */
-        public void extractQueryTermsQuery(String extractedTermsFieldName, String extractionResultField) throws IOException {
-            // We can only skip the MemoryIndex verification when percolating a single document.
-            // When the document being percolated contains a nested object field then the MemoryIndex contains multiple
-            // documents. In this case the term query that indicates whether memory index verification can be skipped
-            // can incorrectly indicate that non nested queries would match, while their nested variants would not.
-            if (percolatorIndexSearcher.getIndexReader().maxDoc() == 1) {
-                this.verifiedQueriesQuery = new TermQuery(new Term(extractionResultField, ExtractQueryTermsService.EXTRACTION_COMPLETE));
-            }
-            this.queriesMetaDataQuery = ExtractQueryTermsService.createQueryTermsQuery(
-                    percolatorIndexSearcher.getIndexReader(), extractedTermsFieldName,
-                    // include extractionResultField:failed, because docs with this term have no extractedTermsField
-                    // and otherwise we would fail to return these docs. Docs that failed query term extraction
-                    // always need to be verified by MemoryIndex:
-                    new Term(extractionResultField, ExtractQueryTermsService.EXTRACTION_FAILED)
-            );
-        }
-
-        /**
-         * @param percolateTypeQuery    A query that identifies all document containing percolator queries
-         */
-        public void setPercolateTypeQuery(Query percolateTypeQuery) {
-            this.percolateTypeQuery = Objects.requireNonNull(percolateTypeQuery);
-        }
-
-        public PercolateQuery build() {
-            if (percolateTypeQuery != null && queriesMetaDataQuery != null) {
-                throw new IllegalStateException("Either filter by deprecated percolator type or by query metadata");
-            }
-            // The query that selects which percolator queries will be evaluated by MemoryIndex:
-            BooleanQuery.Builder queriesQuery = new BooleanQuery.Builder();
-            if (percolateTypeQuery != null) {
-                queriesQuery.add(percolateTypeQuery, FILTER);
-            }
-            if (queriesMetaDataQuery != null) {
-                queriesQuery.add(queriesMetaDataQuery, FILTER);
-            }
-            return new PercolateQuery(docType, queryStore, documentSource, queriesQuery.build(), percolatorIndexSearcher,
-                    verifiedQueriesQuery);
-        }
-
-    }
-
     private final String documentType;
     private final QueryStore queryStore;
     private final BytesReference documentSource;
-    private final Query percolatorQueriesQuery;
-    private final Query verifiedQueriesQuery;
+    private final Query candidateMatchesQuery;
+    private final Query verifiedMatchesQuery;
     private final IndexSearcher percolatorIndexSearcher;
 
-    private PercolateQuery(String documentType, QueryStore queryStore, BytesReference documentSource,
-                           Query percolatorQueriesQuery, IndexSearcher percolatorIndexSearcher, Query verifiedQueriesQuery) {
-        this.documentType = documentType;
-        this.documentSource = documentSource;
-        this.percolatorQueriesQuery = percolatorQueriesQuery;
-        this.queryStore = queryStore;
-        this.percolatorIndexSearcher = percolatorIndexSearcher;
-        this.verifiedQueriesQuery = verifiedQueriesQuery;
+    PercolateQuery(String documentType, QueryStore queryStore, BytesReference documentSource,
+                          Query candidateMatchesQuery, IndexSearcher percolatorIndexSearcher, Query verifiedMatchesQuery) {
+        this.documentType = Objects.requireNonNull(documentType);
+        this.documentSource = Objects.requireNonNull(documentSource);
+        this.candidateMatchesQuery = Objects.requireNonNull(candidateMatchesQuery);
+        this.queryStore = Objects.requireNonNull(queryStore);
+        this.percolatorIndexSearcher = Objects.requireNonNull(percolatorIndexSearcher);
+        this.verifiedMatchesQuery = Objects.requireNonNull(verifiedMatchesQuery);
     }
 
     @Override
     public Query rewrite(IndexReader reader) throws IOException {
-        Query rewritten = percolatorQueriesQuery.rewrite(reader);
-        if (rewritten != percolatorQueriesQuery) {
+        Query rewritten = candidateMatchesQuery.rewrite(reader);
+        if (rewritten != candidateMatchesQuery) {
             return new PercolateQuery(documentType, queryStore, documentSource, rewritten, percolatorIndexSearcher,
-                    verifiedQueriesQuery);
+                    verifiedMatchesQuery);
         } else {
             return this;
         }
@@ -152,8 +74,8 @@ public final class PercolateQuery extends Query implements Accountable {
 
     @Override
     public Weight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
-        final Weight verifiedQueriesQueryWeight = verifiedQueriesQuery.createWeight(searcher, false);
-        final Weight innerWeight = percolatorQueriesQuery.createWeight(searcher, needsScores);
+        final Weight verifiedMatchesWeight = verifiedMatchesQuery.createWeight(searcher, false);
+        final Weight candidateMatchesWeight = candidateMatchesQuery.createWeight(searcher, false);
         return new Weight(this) {
             @Override
             public void extractTerms(Set<Term> set) {
@@ -183,17 +105,17 @@ public final class PercolateQuery extends Query implements Accountable {
 
             @Override
             public float getValueForNormalization() throws IOException {
-                return innerWeight.getValueForNormalization();
+                return candidateMatchesWeight.getValueForNormalization();
             }
 
             @Override
             public void normalize(float v, float v1) {
-                innerWeight.normalize(v, v1);
+                candidateMatchesWeight.normalize(v, v1);
             }
 
             @Override
             public Scorer scorer(LeafReaderContext leafReaderContext) throws IOException {
-                final Scorer approximation = innerWeight.scorer(leafReaderContext);
+                final Scorer approximation = candidateMatchesWeight.scorer(leafReaderContext);
                 if (approximation == null) {
                     return null;
                 }
@@ -226,7 +148,7 @@ public final class PercolateQuery extends Query implements Accountable {
                         }
                     };
                 } else {
-                    Scorer verifiedDocsScorer = verifiedQueriesQueryWeight.scorer(leafReaderContext);
+                    Scorer verifiedDocsScorer = verifiedMatchesWeight.scorer(leafReaderContext);
                     Bits verifiedDocsBits = Lucene.asSequentialAccessBits(leafReaderContext.reader().maxDoc(), verifiedDocsScorer);
                     return new BaseScorer(this, approximation, queries, percolatorIndexSearcher) {
 
@@ -293,7 +215,7 @@ public final class PercolateQuery extends Query implements Accountable {
     @Override
     public String toString(String s) {
         return "PercolateQuery{document_type={" + documentType + "},document_source={" + documentSource.utf8ToString() +
-                "},inner={" + percolatorQueriesQuery.toString(s)  + "}}";
+                "},inner={" + candidateMatchesQuery.toString(s)  + "}}";
     }
 
     @Override
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
index a5db24a71ef..132970d585d 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java
@@ -50,6 +50,7 @@ import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -57,7 +58,6 @@ import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.FieldNameAnalyzer;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentMapperForType;
@@ -406,37 +406,27 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
             docSearcher.setQueryCache(null);
         }
 
-        IndexSettings indexSettings = context.getIndexSettings();
-        boolean mapUnmappedFieldsAsString = indexSettings.getValue(PercolatorFieldMapper.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING);
-        return buildQuery(indexSettings.getIndexVersionCreated(), context, docSearcher, mapUnmappedFieldsAsString);
-    }
-
-    Query buildQuery(Version indexVersionCreated, QueryShardContext context, IndexSearcher docSearcher,
-                     boolean mapUnmappedFieldsAsString) throws IOException {
+        Version indexVersionCreated = context.getIndexSettings().getIndexVersionCreated();
+        boolean mapUnmappedFieldsAsString = context.getIndexSettings()
+                .getValue(PercolatorFieldMapper.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING);
         if (indexVersionCreated.onOrAfter(Version.V_5_0_0_alpha1)) {
             MappedFieldType fieldType = context.fieldMapper(field);
             if (fieldType == null) {
                 throw new QueryShardException(context, "field [" + field + "] does not exist");
             }
 
-            if (!(fieldType instanceof PercolatorFieldMapper.PercolatorFieldType)) {
+            if (!(fieldType instanceof PercolatorFieldMapper.FieldType)) {
                 throw new QueryShardException(context, "expected field [" + field +
                         "] to be of type [percolator], but is of type [" + fieldType.typeName() + "]");
             }
-            PercolatorFieldMapper.PercolatorFieldType pft = (PercolatorFieldMapper.PercolatorFieldType) fieldType;
+            PercolatorFieldMapper.FieldType pft = (PercolatorFieldMapper.FieldType) fieldType;
             PercolateQuery.QueryStore queryStore = createStore(pft, context, mapUnmappedFieldsAsString);
-            PercolateQuery.Builder builder = new PercolateQuery.Builder(
-                    documentType, queryStore, document, docSearcher
-            );
-            builder.extractQueryTermsQuery(pft.getExtractedTermsField(), pft.getExtractionResultFieldName());
-            return builder.build();
+            return pft.percolateQuery(documentType, queryStore, document, docSearcher);
         } else {
             Query percolateTypeQuery = new TermQuery(new Term(TypeFieldMapper.NAME, MapperService.PERCOLATOR_LEGACY_TYPE_NAME));
-            PercolateQuery.Builder builder = new PercolateQuery.Builder(
-                    documentType, createLegacyStore(context, mapUnmappedFieldsAsString), document, docSearcher
-            );
-            builder.setPercolateTypeQuery(percolateTypeQuery);
-            return builder.build();
+            PercolateQuery.QueryStore queryStore = createLegacyStore(context, mapUnmappedFieldsAsString);
+            return new PercolateQuery(documentType, queryStore, document, percolateTypeQuery, docSearcher,
+                    new MatchNoDocsQuery("pre 5.0.0-alpha1 index, no verified matches"));
         }
     }
 
@@ -477,17 +467,17 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBu
         }
     }
 
-    private static PercolateQuery.QueryStore createStore(PercolatorFieldMapper.PercolatorFieldType fieldType,
+    private static PercolateQuery.QueryStore createStore(PercolatorFieldMapper.FieldType fieldType,
                                                          QueryShardContext context,
                                                          boolean mapUnmappedFieldsAsString) {
         return ctx -> {
             LeafReader leafReader = ctx.reader();
-            BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues(fieldType.getQueryBuilderFieldName());
+            BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues(fieldType.queryBuilderField.name());
             if (binaryDocValues == null) {
                 return docId -> null;
             }
 
-            Bits bits = leafReader.getDocsWithField(fieldType.getQueryBuilderFieldName());
+            Bits bits = leafReader.getDocsWithField(fieldType.queryBuilderField.name());
             return docId -> {
                 if (bits.get(docId)) {
                     BytesRef qbSource = binaryDocValues.get(docId);
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
index 1428b8116a8..4cd3bdabbd8 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java
@@ -20,10 +20,22 @@ package org.elasticsearch.percolator;
 
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.Fields;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.queries.TermsQuery;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -38,12 +50,18 @@ import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.core.BinaryFieldMapper;
 import org.elasticsearch.index.mapper.core.KeywordFieldMapper;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.BoostingQueryBuilder;
+import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.index.query.RangeQueryBuilder;
+import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Iterator;
@@ -56,7 +74,12 @@ public class PercolatorFieldMapper extends FieldMapper {
     public static final Setting<Boolean> INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING =
             Setting.boolSetting("index.percolator.map_unmapped_fields_as_string", false, Setting.Property.IndexScope);
     public static final String CONTENT_TYPE = "percolator";
-    private static final PercolatorFieldType FIELD_TYPE = new PercolatorFieldType();
+    private static final FieldType FIELD_TYPE = new FieldType();
+
+    static final byte FIELD_VALUE_SEPARATOR = 0;  // nul code point
+    static final String EXTRACTION_COMPLETE = "complete";
+    static final String EXTRACTION_PARTIAL = "partial";
+    static final String EXTRACTION_FAILED = "failed";
 
     public static final String EXTRACTED_TERMS_FIELD_NAME = "extracted_terms";
     public static final String EXTRACTION_RESULT_FIELD_NAME = "extraction_result";
@@ -74,12 +97,13 @@ public class PercolatorFieldMapper extends FieldMapper {
         @Override
         public PercolatorFieldMapper build(BuilderContext context) {
             context.path().add(name());
+            FieldType fieldType = (FieldType) this.fieldType;
             KeywordFieldMapper extractedTermsField = createExtractQueryFieldBuilder(EXTRACTED_TERMS_FIELD_NAME, context);
-            ((PercolatorFieldType) fieldType).queryTermsField = extractedTermsField.fieldType();
+            fieldType.queryTermsField = extractedTermsField.fieldType();
             KeywordFieldMapper extractionResultField = createExtractQueryFieldBuilder(EXTRACTION_RESULT_FIELD_NAME, context);
-            ((PercolatorFieldType) fieldType).extractionResultField = extractionResultField.fieldType();
+            fieldType.extractionResultField = extractionResultField.fieldType();
             BinaryFieldMapper queryBuilderField = createQueryBuilderFieldBuilder(context);
-            ((PercolatorFieldType) fieldType).queryBuilderField = queryBuilderField.fieldType();
+            fieldType.queryBuilderField = queryBuilderField.fieldType();
             context.path().remove();
             setupFieldType(context);
             return new PercolatorFieldMapper(name(), fieldType, defaultFieldType, context.indexSettings(),
@@ -114,40 +138,28 @@ public class PercolatorFieldMapper extends FieldMapper {
         }
     }
 
-    public static class PercolatorFieldType extends MappedFieldType {
+    public static class FieldType extends MappedFieldType {
 
-        private MappedFieldType queryTermsField;
-        private MappedFieldType extractionResultField;
-        private MappedFieldType queryBuilderField;
+        MappedFieldType queryTermsField;
+        MappedFieldType extractionResultField;
+        MappedFieldType queryBuilderField;
 
-        public PercolatorFieldType() {
+        public FieldType() {
             setIndexOptions(IndexOptions.NONE);
             setDocValuesType(DocValuesType.NONE);
             setStored(false);
         }
 
-        public PercolatorFieldType(PercolatorFieldType ref) {
+        public FieldType(FieldType ref) {
             super(ref);
             queryTermsField = ref.queryTermsField;
             extractionResultField = ref.extractionResultField;
             queryBuilderField = ref.queryBuilderField;
         }
 
-        public String getExtractedTermsField() {
-            return queryTermsField.name();
-        }
-
-        public String getExtractionResultFieldName() {
-            return extractionResultField.name();
-        }
-
-        public String getQueryBuilderFieldName() {
-            return queryBuilderField.name();
-        }
-
         @Override
         public MappedFieldType clone() {
-            return new PercolatorFieldType(this);
+            return new FieldType(this);
         }
 
         @Override
@@ -159,6 +171,52 @@ public class PercolatorFieldMapper extends FieldMapper {
         public Query termQuery(Object value, QueryShardContext context) {
             throw new QueryShardException(context, "Percolator fields are not searchable directly, use a percolate query instead");
         }
+
+        public Query percolateQuery(String documentType, PercolateQuery.QueryStore queryStore, BytesReference documentSource,
+                                    IndexSearcher searcher) throws IOException {
+            IndexReader indexReader = searcher.getIndexReader();
+            Query candidateMatchesQuery = createCandidateQuery(indexReader);
+            Query verifiedMatchesQuery;
+            // We can only skip the MemoryIndex verification when percolating a single document.
+            // When the document being percolated contains a nested object field then the MemoryIndex contains multiple
+            // documents. In this case the term query that indicates whether memory index verification can be skipped
+            // can incorrectly indicate that non nested queries would match, while their nested variants would not.
+            if (indexReader.maxDoc() == 1) {
+                verifiedMatchesQuery = new TermQuery(new Term(extractionResultField.name(), EXTRACTION_COMPLETE));
+            } else {
+                verifiedMatchesQuery = new MatchNoDocsQuery("nested docs, so no verified matches");
+            }
+            return new PercolateQuery(documentType, queryStore, documentSource, candidateMatchesQuery, searcher, verifiedMatchesQuery);
+        }
+
+        Query createCandidateQuery(IndexReader indexReader) throws IOException {
+            List<Term> extractedTerms = new ArrayList<>();
+            // include extractionResultField:failed, because docs with this term have no extractedTermsField
+            // and otherwise we would fail to return these docs. Docs that failed query term extraction
+            // always need to be verified by MemoryIndex:
+            extractedTerms.add(new Term(extractionResultField.name(), EXTRACTION_FAILED));
+
+            LeafReader reader = indexReader.leaves().get(0).reader();
+            Fields fields = reader.fields();
+            for (String field : fields) {
+                Terms terms = fields.terms(field);
+                if (terms == null) {
+                    continue;
+                }
+
+                BytesRef fieldBr = new BytesRef(field);
+                TermsEnum tenum = terms.iterator();
+                for (BytesRef term = tenum.next(); term != null; term = tenum.next()) {
+                    BytesRefBuilder builder = new BytesRefBuilder();
+                    builder.append(fieldBr);
+                    builder.append(FIELD_VALUE_SEPARATOR);
+                    builder.append(term);
+                    extractedTerms.add(new Term(queryTermsField.name(), builder.toBytesRef()));
+                }
+            }
+            return new TermsQuery(extractedTerms);
+        }
+
     }
 
     private final boolean mapUnmappedFieldAsString;
@@ -211,6 +269,7 @@ public class PercolatorFieldMapper extends FieldMapper {
 
         XContentParser parser = context.parser();
         QueryBuilder queryBuilder = parseQueryBuilder(queryShardContext.newParseContext(parser), parser.getTokenLocation());
+        verifyRangeQueries(queryBuilder);
         // Fetching of terms, shapes and indexed scripts happen during this rewrite:
         queryBuilder = queryBuilder.rewrite(queryShardContext);
 
@@ -222,11 +281,34 @@ public class PercolatorFieldMapper extends FieldMapper {
         }
 
         Query query = toQuery(queryShardContext, mapUnmappedFieldAsString, queryBuilder);
-        ExtractQueryTermsService.extractQueryTerms(query, context.doc(), queryTermsField.name(), extractionResultField.name(),
-                queryTermsField.fieldType());
+        processQuery(query, context);
         return null;
     }
 
+    void processQuery(Query query, ParseContext context) {
+        ParseContext.Document doc = context.doc();
+        FieldType pft = (FieldType) this.fieldType();
+        QueryAnalyzer.Result result;
+        try {
+            result = QueryAnalyzer.analyze(query);
+        } catch (QueryAnalyzer.UnsupportedQueryException e) {
+            doc.add(new Field(pft.extractionResultField.name(), EXTRACTION_FAILED, extractionResultField.fieldType()));
+            return;
+        }
+        for (Term term : result.terms) {
+            BytesRefBuilder builder = new BytesRefBuilder();
+            builder.append(new BytesRef(term.field()));
+            builder.append(FIELD_VALUE_SEPARATOR);
+            builder.append(term.bytes());
+            doc.add(new Field(queryTermsField.name(), builder.toBytesRef(), queryTermsField.fieldType()));
+        }
+        if (result.verified) {
+            doc.add(new Field(extractionResultField.name(), EXTRACTION_COMPLETE, extractionResultField.fieldType()));
+        } else {
+            doc.add(new Field(extractionResultField.name(), EXTRACTION_PARTIAL, extractionResultField.fieldType()));
+        }
+    }
+
     public static Query parseQuery(QueryShardContext context, boolean mapUnmappedFieldsAsString, XContentParser parser) throws IOException {
         return toQuery(context, mapUnmappedFieldsAsString, parseQueryBuilder(context.newParseContext(parser), parser.getTokenLocation()));
     }
@@ -273,4 +355,38 @@ public class PercolatorFieldMapper extends FieldMapper {
         return CONTENT_TYPE;
     }
 
+    /**
+     * Fails if a range query with a date range is found based on current time
+     */
+    static void verifyRangeQueries(QueryBuilder queryBuilder) {
+        if (queryBuilder instanceof RangeQueryBuilder) {
+            RangeQueryBuilder rangeQueryBuilder = (RangeQueryBuilder) queryBuilder;
+            if (rangeQueryBuilder.from() instanceof String) {
+                String from = (String) rangeQueryBuilder.from();
+                String to = (String) rangeQueryBuilder.to();
+                if (from.contains("now") || to.contains("now")) {
+                    throw new IllegalArgumentException("Percolator queries containing time range queries based on the " +
+                            "current time are forbidden");
+                }
+            }
+        } else if (queryBuilder instanceof BoolQueryBuilder) {
+            BoolQueryBuilder boolQueryBuilder = (BoolQueryBuilder) queryBuilder;
+            List<QueryBuilder> clauses = new ArrayList<>();
+            clauses.addAll(boolQueryBuilder.filter());
+            clauses.addAll(boolQueryBuilder.must());
+            clauses.addAll(boolQueryBuilder.mustNot());
+            clauses.addAll(boolQueryBuilder.should());
+            for (QueryBuilder clause : clauses) {
+                verifyRangeQueries(clause);
+            }
+        } else if (queryBuilder instanceof ConstantScoreQueryBuilder) {
+            verifyRangeQueries(((ConstantScoreQueryBuilder) queryBuilder).innerQuery());
+        } else if (queryBuilder instanceof FunctionScoreQueryBuilder) {
+            verifyRangeQueries(((FunctionScoreQueryBuilder) queryBuilder).query());
+        } else if (queryBuilder instanceof BoostingQueryBuilder) {
+            verifyRangeQueries(((BoostingQueryBuilder) queryBuilder).negativeQuery());
+            verifyRangeQueries(((BoostingQueryBuilder) queryBuilder).positiveQuery());
+        }
+    }
+
 }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java
similarity index 73%
rename from modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java
rename to modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java
index 44048935304..8154c632907 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/ExtractQueryTermsService.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/QueryAnalyzer.java
@@ -18,15 +18,8 @@
  */
 package org.elasticsearch.percolator;
 
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.index.Fields;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.index.PrefixCodedTerms;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.queries.BlendedTermQuery;
 import org.apache.lucene.queries.CommonTermsQuery;
 import org.apache.lucene.queries.TermsQuery;
@@ -46,37 +39,25 @@ import org.apache.lucene.search.spans.SpanOrQuery;
 import org.apache.lucene.search.spans.SpanQuery;
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
-import org.elasticsearch.index.mapper.ParseContext;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Set;
 import java.util.function.Function;
 
-/**
- * Utility to extract query terms from queries and create queries from documents.
- */
-public final class ExtractQueryTermsService {
+public final class QueryAnalyzer {
 
-    private static final byte FIELD_VALUE_SEPARATOR = 0;  // nul code point
-    public static final String EXTRACTION_COMPLETE = "complete";
-    public static final String EXTRACTION_PARTIAL = "partial";
-    public static final String EXTRACTION_FAILED = "failed";
-
-    static final Map<Class<? extends Query>, Function<Query, Result>> queryProcessors;
+    private static final Map<Class<? extends Query>, Function<Query, Result>> queryProcessors;
 
     static {
-        Map<Class<? extends Query>, Function<Query, Result>> map = new HashMap<>(17);
+        Map<Class<? extends Query>, Function<Query, Result>> map = new HashMap<>();
         map.put(MatchNoDocsQuery.class, matchNoDocsQuery());
         map.put(ConstantScoreQuery.class, constantScoreQuery());
         map.put(BoostQuery.class, boostQuery());
@@ -97,83 +78,34 @@ public final class ExtractQueryTermsService {
         queryProcessors = Collections.unmodifiableMap(map);
     }
 
-    private ExtractQueryTermsService() {
+    private QueryAnalyzer() {
     }
 
     /**
-     * Extracts all terms from the specified query and adds it to the specified document.
+     * Extracts terms from the provided query. These terms are stored with the percolator query and
+     * used by the percolate query's candidate query as fields to be query by. The candidate query
+     * holds the terms from the document to be percolated and allows to the percolate query to ignore
+     * percolator queries that we know would otherwise never match.
      *
-     * @param query                 The query to extract terms from
-     * @param document              The document to add the extracted terms to
-     * @param queryTermsFieldField  The field in the document holding the extracted terms
-     * @param extractionResultField The field contains whether query term extraction was successful, partial or
-     *                              failed. (For example the query contained an unsupported query (e.g. WildcardQuery)
-     *                              then query extraction would fail)
-     * @param fieldType             The field type for the query metadata field
-     */
-    public static void extractQueryTerms(Query query, ParseContext.Document document, String queryTermsFieldField,
-                                         String extractionResultField, FieldType fieldType) {
-        Result result;
-        try {
-            result = extractQueryTerms(query);
-        } catch (UnsupportedQueryException e) {
-            document.add(new Field(extractionResultField, EXTRACTION_FAILED, fieldType));
-            return;
-        }
-        for (Term term : result.terms) {
-            BytesRefBuilder builder = new BytesRefBuilder();
-            builder.append(new BytesRef(term.field()));
-            builder.append(FIELD_VALUE_SEPARATOR);
-            builder.append(term.bytes());
-            document.add(new Field(queryTermsFieldField, builder.toBytesRef(), fieldType));
-        }
-        if (result.verified) {
-            document.add(new Field(extractionResultField, EXTRACTION_COMPLETE, fieldType));
-        } else {
-            document.add(new Field(extractionResultField, EXTRACTION_PARTIAL, fieldType));
-        }
-    }
-
-    /**
-     * Creates a terms query containing all terms from all fields of the specified index reader.
-     */
-    public static Query createQueryTermsQuery(IndexReader indexReader, String queryMetadataField,
-                                              Term... optionalTerms) throws IOException {
-        Objects.requireNonNull(queryMetadataField);
-
-        List<Term> extractedTerms = new ArrayList<>();
-        Collections.addAll(extractedTerms, optionalTerms);
-
-        Fields fields = MultiFields.getFields(indexReader);
-        for (String field : fields) {
-            Terms terms = fields.terms(field);
-            if (terms == null) {
-                continue;
-            }
-
-            BytesRef fieldBr = new BytesRef(field);
-            TermsEnum tenum = terms.iterator();
-            for (BytesRef term = tenum.next(); term != null; term = tenum.next()) {
-                BytesRefBuilder builder = new BytesRefBuilder();
-                builder.append(fieldBr);
-                builder.append(FIELD_VALUE_SEPARATOR);
-                builder.append(term);
-                extractedTerms.add(new Term(queryMetadataField, builder.toBytesRef()));
-            }
-        }
-        return new TermsQuery(extractedTerms);
-    }
-
-    /**
-     * Extracts all query terms from the provided query and adds it to specified list.
      * <p>
-     * From boolean query with no should clauses or phrase queries only the longest term are selected,
+     * When extracting the terms for the specified query, we can also determine if the percolator query is
+     * always going to match. For example if a percolator query just contains a term query or a disjunction
+     * query then when the candidate query matches with that, we know the entire percolator query always
+     * matches. This allows the percolate query to skip the expensive memory index verification step that
+     * it would otherwise have to execute (for example when a percolator query contains a phrase query or a
+     * conjunction query).
+     *
+     * <p>
+     * The query analyzer doesn't always extract all terms from the specified query. For example from a
+     * boolean query with no should clauses or phrase queries only the longest term are selected,
      * since that those terms are likely to be the rarest. Boolean query's must_not clauses are always ignored.
+     *
      * <p>
-     * If from part of the query, no query terms can be extracted then term extraction is stopped and
-     * an UnsupportedQueryException is thrown.
+     * Sometimes the query analyzer can't always extract terms from a sub query, if that happens then
+     * query analysis is stopped and an UnsupportedQueryException is thrown. So that the caller can mark
+     * this query in such a way that the PercolatorQuery always verifies if this query with the MemoryIndex.
      */
-    static Result extractQueryTerms(Query query) {
+    public static Result analyze(Query query) {
         Class queryClass = query.getClass();
         if (queryClass.isAnonymousClass()) {
             // Sometimes queries have anonymous classes in that case we need the direct super class.
@@ -195,14 +127,14 @@ public final class ExtractQueryTermsService {
     static Function<Query, Result> constantScoreQuery() {
         return query -> {
             Query wrappedQuery = ((ConstantScoreQuery) query).getQuery();
-            return extractQueryTerms(wrappedQuery);
+            return analyze(wrappedQuery);
         };
     }
 
     static Function<Query, Result> boostQuery() {
         return query -> {
             Query wrappedQuery = ((BoostQuery) query).getQuery();
-            return extractQueryTerms(wrappedQuery);
+            return analyze(wrappedQuery);
         };
     }
 
@@ -277,7 +209,7 @@ public final class ExtractQueryTermsService {
             Set<Term> bestClauses = null;
             SpanNearQuery spanNearQuery = (SpanNearQuery) query;
             for (SpanQuery clause : spanNearQuery.getClauses()) {
-                Result temp = extractQueryTerms(clause);
+                Result temp = analyze(clause);
                 bestClauses = selectTermListWithTheLongestShortestTerm(temp.terms, bestClauses);
             }
             return new Result(false, bestClauses);
@@ -289,7 +221,7 @@ public final class ExtractQueryTermsService {
             Set<Term> terms = new HashSet<>();
             SpanOrQuery spanOrQuery = (SpanOrQuery) query;
             for (SpanQuery clause : spanOrQuery.getClauses()) {
-                terms.addAll(extractQueryTerms(clause).terms);
+                terms.addAll(analyze(clause).terms);
             }
             return new Result(false, terms);
         };
@@ -297,14 +229,14 @@ public final class ExtractQueryTermsService {
 
     static Function<Query, Result> spanNotQuery() {
         return query -> {
-            Result result = extractQueryTerms(((SpanNotQuery) query).getInclude());
+            Result result = analyze(((SpanNotQuery) query).getInclude());
             return new Result(false, result.terms);
         };
     }
 
     static Function<Query, Result> spanFirstQuery() {
         return query -> {
-            Result result = extractQueryTerms(((SpanFirstQuery) query).getMatch());
+            Result result = analyze(((SpanFirstQuery) query).getMatch());
             return new Result(false, result.terms);
         };
     }
@@ -341,7 +273,7 @@ public final class ExtractQueryTermsService {
 
                     Result temp;
                     try {
-                        temp = extractQueryTerms(clause.getQuery());
+                        temp = analyze(clause.getQuery());
                     } catch (UnsupportedQueryException e) {
                         uqe = e;
                         continue;
@@ -381,7 +313,7 @@ public final class ExtractQueryTermsService {
     static Function<Query, Result> functionScoreQuery() {
         return query -> {
             FunctionScoreQuery functionScoreQuery = (FunctionScoreQuery) query;
-            Result result = extractQueryTerms(functionScoreQuery.getSubQuery());
+            Result result = analyze(functionScoreQuery.getSubQuery());
             // If min_score is specified we can't guarantee upfront that this percolator query matches,
             // so in that case we set verified to false.
             // (if it matches with the percolator document matches with the extracted terms.
@@ -395,7 +327,7 @@ public final class ExtractQueryTermsService {
         boolean verified = minimumShouldMatch <= 1 && otherClauses == false;
         Set<Term> terms = new HashSet<>();
         for (Query disjunct : disjunctions) {
-            Result subResult = extractQueryTerms(disjunct);
+            Result subResult = analyze(disjunct);
             if (subResult.verified == false) {
                 verified = false;
             }
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java
index 5e3a6f90756..3045fa08a09 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestMultiPercolateAction.java
@@ -33,6 +33,7 @@ import org.elasticsearch.rest.action.support.RestToXContentListener;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
+@Deprecated
 public class RestMultiPercolateAction extends BaseRestHandler {
 
     private final boolean allowExplicitIndex;
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java
index bdbe4921f09..1f8c99b2e97 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/RestPercolateAction.java
@@ -35,6 +35,7 @@ import org.elasticsearch.rest.action.support.RestToXContentListener;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
+@Deprecated
 public class RestPercolateAction extends BaseRestHandler {
     @Inject
     public RestPercolateAction(Settings settings, RestController controller) {
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java
index 9968035ec85..0bd8b15bfb7 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportMultiPercolateAction.java
@@ -49,6 +49,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+@Deprecated
 public class TransportMultiPercolateAction extends HandledTransportAction<MultiPercolateRequest, MultiPercolateResponse> {
 
     private final Client client;
diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java
index beca222b755..dbbfd09619b 100644
--- a/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java
+++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/TransportPercolateAction.java
@@ -57,6 +57,7 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
+@Deprecated
 public class TransportPercolateAction extends HandledTransportAction<PercolateRequest, PercolateResponse> {
 
     private final Client client;
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
new file mode 100644
index 00000000000..8ac9890afa8
--- /dev/null
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java
@@ -0,0 +1,436 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.percolator;
+
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.memory.MemoryIndex;
+import org.apache.lucene.queries.BlendedTermQuery;
+import org.apache.lucene.queries.CommonTermsQuery;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.ConstantScoreScorer;
+import org.apache.lucene.search.ConstantScoreWeight;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.FilterScorer;
+import org.apache.lucene.search.FilteredDocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.PrefixQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.search.WildcardQuery;
+import org.apache.lucene.search.spans.SpanNearQuery;
+import org.apache.lucene.search.spans.SpanNotQuery;
+import org.apache.lucene.search.spans.SpanOrQuery;
+import org.apache.lucene.search.spans.SpanTermQuery;
+import org.apache.lucene.store.Directory;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.compress.CompressedXContent;
+import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.DocumentMapper;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESSingleNodeTestCase;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Function;
+
+import static org.hamcrest.Matchers.equalTo;
+
+public class CandidateQueryTests extends ESSingleNodeTestCase {
+
+    private Directory directory;
+    private IndexWriter indexWriter;
+    private DocumentMapper documentMapper;
+    private DirectoryReader directoryReader;
+    private MapperService mapperService;
+
+    private PercolatorFieldMapper fieldMapper;
+    private PercolatorFieldMapper.FieldType fieldType;
+
+    private List<Query> queries;
+    private PercolateQuery.QueryStore queryStore;
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return Collections.singleton(PercolatorPlugin.class);
+    }
+
+    @Before
+    public void init() throws Exception {
+        directory = newDirectory();
+        IndexWriterConfig config = new IndexWriterConfig(new WhitespaceAnalyzer());
+        config.setMergePolicy(NoMergePolicy.INSTANCE);
+        indexWriter = new IndexWriter(directory, config);
+
+        String indexName = "test";
+        IndexService indexService = createIndex(indexName, Settings.EMPTY);
+        mapperService = indexService.mapperService();
+
+        String mapper = XContentFactory.jsonBuilder().startObject().startObject("type")
+                .startObject("properties")
+                .startObject("int_field").field("type", "integer").endObject()
+                .startObject("long_field").field("type", "long").endObject()
+                .startObject("half_float_field").field("type", "half_float").endObject()
+                .startObject("float_field").field("type", "float").endObject()
+                .startObject("double_field").field("type", "double").endObject()
+                .startObject("ip_field").field("type", "ip").endObject()
+                .startObject("field").field("type", "keyword").endObject()
+                .endObject().endObject().endObject().string();
+        documentMapper = mapperService.merge("type", new CompressedXContent(mapper), MapperService.MergeReason.MAPPING_UPDATE, true);
+
+        String queryField = "query_field";
+        String mappingType = "query";
+        String percolatorMapper = XContentFactory.jsonBuilder().startObject().startObject(mappingType)
+                .startObject("properties").startObject(queryField).field("type", "percolator").endObject().endObject()
+                .endObject().endObject().string();
+        mapperService.merge(mappingType, new CompressedXContent(percolatorMapper), MapperService.MergeReason.MAPPING_UPDATE, true);
+        fieldMapper = (PercolatorFieldMapper) mapperService.documentMapper(mappingType).mappers().getMapper(queryField);
+        fieldType = (PercolatorFieldMapper.FieldType) fieldMapper.fieldType();
+
+        queries = new ArrayList<>();
+        queryStore = ctx -> docId -> this.queries.get(docId);
+    }
+
+    @After
+    public void deinit() throws Exception {
+        directoryReader.close();
+        directory.close();
+    }
+
+    public void testDuel() throws Exception {
+        List<Function<String, Query>> queryFunctions = new ArrayList<>();
+        queryFunctions.add((id) -> new PrefixQuery(new Term("field", id)));
+        queryFunctions.add((id) -> new WildcardQuery(new Term("field", id + "*")));
+        queryFunctions.add((id) -> new CustomQuery(new Term("field", id)));
+        queryFunctions.add((id) -> new SpanTermQuery(new Term("field", id)));
+        queryFunctions.add((id) -> new TermQuery(new Term("field", id)));
+        queryFunctions.add((id) -> {
+            BooleanQuery.Builder builder = new BooleanQuery.Builder();
+            return builder.build();
+        });
+        queryFunctions.add((id) -> {
+            BooleanQuery.Builder builder = new BooleanQuery.Builder();
+            builder.add(new TermQuery(new Term("field", id)), BooleanClause.Occur.MUST);
+            if (randomBoolean()) {
+                builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT);
+            }
+            if (randomBoolean()) {
+                builder.add(new CustomQuery(new Term("field", id)), BooleanClause.Occur.MUST);
+            }
+            return builder.build();
+        });
+        queryFunctions.add((id) -> {
+            BooleanQuery.Builder builder = new BooleanQuery.Builder();
+            builder.add(new TermQuery(new Term("field", id)), BooleanClause.Occur.SHOULD);
+            if (randomBoolean()) {
+                builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT);
+            }
+            if (randomBoolean()) {
+                builder.add(new CustomQuery(new Term("field", id)), BooleanClause.Occur.SHOULD);
+            }
+            return builder.build();
+        });
+        queryFunctions.add((id) -> {
+            BooleanQuery.Builder builder = new BooleanQuery.Builder();
+            builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
+            builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
+            if (randomBoolean()) {
+                builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT);
+            }
+            return builder.build();
+        });
+        queryFunctions.add((id) -> {
+            BooleanQuery.Builder builder = new BooleanQuery.Builder();
+            builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
+            builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
+            if (randomBoolean()) {
+                builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT);
+            }
+            return builder.build();
+        });
+        queryFunctions.add((id) -> {
+            BooleanQuery.Builder builder = new BooleanQuery.Builder();
+            builder.setMinimumNumberShouldMatch(randomIntBetween(0, 4));
+            builder.add(new TermQuery(new Term("field", id)), BooleanClause.Occur.SHOULD);
+            builder.add(new CustomQuery(new Term("field", id)), BooleanClause.Occur.SHOULD);
+            return builder.build();
+        });
+        queryFunctions.add((id) -> new MatchAllDocsQuery());
+        queryFunctions.add((id) -> new MatchNoDocsQuery("no reason at all"));
+
+        int numDocs = randomIntBetween(queryFunctions.size(), queryFunctions.size() * 3);
+        List<ParseContext.Document> documents = new ArrayList<>();
+        for (int i = 0; i < numDocs; i++) {
+            String id = Integer.toString(i);
+            Query query = queryFunctions.get(i % queryFunctions.size()).apply(id);
+            addQuery(query, documents);
+        }
+
+        indexWriter.addDocuments(documents);
+        indexWriter.close();
+        directoryReader = DirectoryReader.open(directory);
+        IndexSearcher shardSearcher = newSearcher(directoryReader);
+        // Disable query cache, because ControlQuery cannot be cached...
+        shardSearcher.setQueryCache(null);
+
+        for (int i = 0; i < numDocs; i++) {
+            String id = Integer.toString(i);
+            Iterable<? extends IndexableField> doc = Collections.singleton(new StringField("field", id, Field.Store.NO));
+            MemoryIndex memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer());
+            duelRun(queryStore, memoryIndex, shardSearcher);
+        }
+
+        Iterable<? extends IndexableField> doc = Collections.singleton(new StringField("field", "value", Field.Store.NO));
+        MemoryIndex memoryIndex = MemoryIndex.fromDocument(doc, new WhitespaceAnalyzer());
+        duelRun(queryStore, memoryIndex, shardSearcher);
+        // Empty percolator doc:
+        memoryIndex = new MemoryIndex();
+        duelRun(queryStore, memoryIndex, shardSearcher);
+    }
+
+    public void testDuelSpecificQueries() throws Exception {
+        List<ParseContext.Document> documents = new ArrayList<>();
+
+        CommonTermsQuery commonTermsQuery = new CommonTermsQuery(BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, 128);
+        commonTermsQuery.add(new Term("field", "quick"));
+        commonTermsQuery.add(new Term("field", "brown"));
+        commonTermsQuery.add(new Term("field", "fox"));
+        addQuery(commonTermsQuery, documents);
+
+        BlendedTermQuery blendedTermQuery = BlendedTermQuery.booleanBlendedQuery(new Term[]{new Term("field", "quick"),
+                new Term("field", "brown"), new Term("field", "fox")}, false);
+        addQuery(blendedTermQuery, documents);
+
+        SpanNearQuery spanNearQuery = new SpanNearQuery.Builder("field", true)
+                .addClause(new SpanTermQuery(new Term("field", "quick")))
+                .addClause(new SpanTermQuery(new Term("field", "brown")))
+                .addClause(new SpanTermQuery(new Term("field", "fox")))
+                .build();
+        addQuery(spanNearQuery, documents);
+
+        SpanNearQuery spanNearQuery2 = new SpanNearQuery.Builder("field", true)
+                .addClause(new SpanTermQuery(new Term("field", "the")))
+                .addClause(new SpanTermQuery(new Term("field", "lazy")))
+                .addClause(new SpanTermQuery(new Term("field", "doc")))
+                .build();
+        SpanOrQuery spanOrQuery = new SpanOrQuery(
+                spanNearQuery,
+                spanNearQuery2
+        );
+        addQuery(spanOrQuery, documents);
+
+        SpanNotQuery spanNotQuery = new SpanNotQuery(spanNearQuery, spanNearQuery);
+        addQuery(spanNotQuery, documents);
+
+        long lowerLong = randomIntBetween(0, 256);
+        long upperLong = lowerLong + randomIntBetween(0, 32);
+        addQuery(LongPoint.newRangeQuery("long_field", lowerLong, upperLong), documents);
+
+        indexWriter.addDocuments(documents);
+        indexWriter.close();
+        directoryReader = DirectoryReader.open(directory);
+        IndexSearcher shardSearcher = newSearcher(directoryReader);
+        // Disable query cache, because ControlQuery cannot be cached...
+        shardSearcher.setQueryCache(null);
+
+        Document document = new Document();
+        document.add(new TextField("field", "the quick brown fox jumps over the lazy dog", Field.Store.NO));
+        long randomLong = randomIntBetween((int) lowerLong, (int) upperLong);
+        document.add(new LongPoint("long_field", randomLong));
+        MemoryIndex memoryIndex = MemoryIndex.fromDocument(document, new WhitespaceAnalyzer());
+        duelRun(queryStore, memoryIndex, shardSearcher);
+    }
+
+    private void duelRun(PercolateQuery.QueryStore queryStore, MemoryIndex memoryIndex, IndexSearcher shardSearcher) throws IOException {
+        boolean requireScore = randomBoolean();
+        IndexSearcher percolateSearcher = memoryIndex.createSearcher();
+        Query percolateQuery = fieldType.percolateQuery("type", queryStore, new BytesArray("{}"), percolateSearcher);
+        Query query = requireScore ? percolateQuery : new ConstantScoreQuery(percolateQuery);
+        TopDocs topDocs = shardSearcher.search(query, 10);
+
+        Query controlQuery = new ControlQuery(memoryIndex, queryStore);
+        controlQuery = requireScore ? controlQuery : new ConstantScoreQuery(controlQuery);
+        TopDocs controlTopDocs = shardSearcher.search(controlQuery, 10);
+        assertThat(topDocs.totalHits, equalTo(controlTopDocs.totalHits));
+        assertThat(topDocs.scoreDocs.length, equalTo(controlTopDocs.scoreDocs.length));
+        for (int j = 0; j < topDocs.scoreDocs.length; j++) {
+            assertThat(topDocs.scoreDocs[j].doc, equalTo(controlTopDocs.scoreDocs[j].doc));
+            assertThat(topDocs.scoreDocs[j].score, equalTo(controlTopDocs.scoreDocs[j].score));
+            if (requireScore) {
+                Explanation explain1 = shardSearcher.explain(query, topDocs.scoreDocs[j].doc);
+                Explanation explain2 = shardSearcher.explain(controlQuery, controlTopDocs.scoreDocs[j].doc);
+                assertThat(explain1.isMatch(), equalTo(explain2.isMatch()));
+                assertThat(explain1.getValue(), equalTo(explain2.getValue()));
+            }
+        }
+    }
+
+    private void addQuery(Query query, List<ParseContext.Document> docs) throws IOException {
+        ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(Settings.EMPTY,
+                mapperService.documentMapperParser(), documentMapper, null, null);
+        fieldMapper.processQuery(query, parseContext);
+        docs.add(parseContext.doc());
+        queries.add(query);
+    }
+
+    private static final class CustomQuery extends Query {
+
+        private final Term term;
+
+        private CustomQuery(Term term) {
+            this.term = term;
+        }
+
+        @Override
+        public Query rewrite(IndexReader reader) throws IOException {
+            return new TermQuery(term);
+        }
+
+        @Override
+        public String toString(String field) {
+            return "custom{" + field + "}";
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            return sameClassAs(obj);
+        }
+
+        @Override
+        public int hashCode() {
+            return classHash();
+        }
+    }
+
+    private static final class ControlQuery extends Query {
+
+        private final MemoryIndex memoryIndex;
+        private final PercolateQuery.QueryStore queryStore;
+
+        private ControlQuery(MemoryIndex memoryIndex, PercolateQuery.QueryStore queryStore) {
+            this.memoryIndex = memoryIndex;
+            this.queryStore = queryStore;
+        }
+
+        @Override
+        public Weight createWeight(IndexSearcher searcher, boolean needsScores) {
+            return new ConstantScoreWeight(this) {
+
+                float _score;
+
+                @Override
+                public Explanation explain(LeafReaderContext context, int doc) throws IOException {
+                    Scorer scorer = scorer(context);
+                    if (scorer != null) {
+                        int result = scorer.iterator().advance(doc);
+                        if (result == doc) {
+                            return Explanation.match(scorer.score(), "ControlQuery");
+                        }
+                    }
+                    return Explanation.noMatch("ControlQuery");
+                }
+
+                @Override
+                public String toString() {
+                    return "weight(" + ControlQuery.this + ")";
+                }
+
+                @Override
+                public Scorer scorer(LeafReaderContext context) throws IOException {
+                    DocIdSetIterator allDocs = DocIdSetIterator.all(context.reader().maxDoc());
+                    PercolateQuery.QueryStore.Leaf leaf = queryStore.getQueries(context);
+                    FilteredDocIdSetIterator memoryIndexIterator = new FilteredDocIdSetIterator(allDocs) {
+
+                        @Override
+                        protected boolean match(int doc) {
+                            try {
+                                Query query = leaf.getQuery(doc);
+                                float score = memoryIndex.search(query);
+                                if (score != 0f) {
+                                    if (needsScores) {
+                                        _score = score;
+                                    }
+                                    return true;
+                                } else {
+                                    return false;
+                                }
+                            } catch (IOException e) {
+                                throw new RuntimeException(e);
+                            }
+                        }
+                    };
+                    return new FilterScorer(new ConstantScoreScorer(this, score(), memoryIndexIterator)) {
+
+                        @Override
+                        public float score() throws IOException {
+                            return _score;
+                        }
+                    };
+                }
+            };
+        }
+
+        @Override
+        public String toString(String field) {
+            return "control{" + field + "}";
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            return sameClassAs(obj);
+        }
+
+        @Override
+        public int hashCode() {
+            return classHash();
+        }
+
+    }
+
+}
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java
index c2c2a641a71..eda9ce15c73 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java
@@ -21,60 +21,36 @@ package org.elasticsearch.percolator;
 
 import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.memory.MemoryIndex;
-import org.apache.lucene.queries.BlendedTermQuery;
-import org.apache.lucene.queries.CommonTermsQuery;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
-import org.apache.lucene.search.ConstantScoreScorer;
-import org.apache.lucene.search.ConstantScoreWeight;
-import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.FilterScorer;
-import org.apache.lucene.search.FilteredDocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.PhraseQuery;
-import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.Weight;
-import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.search.spans.SpanNearQuery;
-import org.apache.lucene.search.spans.SpanNotQuery;
-import org.apache.lucene.search.spans.SpanOrQuery;
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.store.Directory;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
-import org.elasticsearch.index.mapper.MapperService;
-import org.elasticsearch.index.mapper.ParseContext;
-import org.elasticsearch.index.mapper.Uid;
-import org.elasticsearch.index.mapper.internal.UidFieldMapper;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.After;
 import org.junit.Before;
 
-import java.io.IOException;
 import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.Collections;
 import java.util.List;
-import java.util.Map;
-import java.util.function.Function;
 
 import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.equalTo;
@@ -82,34 +58,13 @@ import static org.hamcrest.Matchers.is;
 
 public class PercolateQueryTests extends ESTestCase {
 
-    public static final String EXTRACTED_TERMS_FIELD_NAME = "extracted_terms";
-    public static final String UNKNOWN_QUERY_FIELD_NAME = "unknown_query";
-    public static final FieldType EXTRACTED_TERMS_FIELD_TYPE = new FieldType();
-
-    static {
-        EXTRACTED_TERMS_FIELD_TYPE.setTokenized(false);
-        EXTRACTED_TERMS_FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
-        EXTRACTED_TERMS_FIELD_TYPE.freeze();
-    }
-
     private Directory directory;
     private IndexWriter indexWriter;
-    private Map<String, Query> queries;
-    private PercolateQuery.QueryStore queryStore;
     private DirectoryReader directoryReader;
 
     @Before
     public void init() throws Exception {
         directory = newDirectory();
-        queries = new HashMap<>();
-        queryStore = ctx -> docId -> {
-            try {
-                String val = ctx.reader().document(docId).get(UidFieldMapper.NAME);
-                return queries.get(Uid.createUid(val).id());
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-        };
         IndexWriterConfig config = new IndexWriterConfig(new WhitespaceAnalyzer());
         config.setMergePolicy(NoMergePolicy.INSTANCE);
         indexWriter = new IndexWriter(directory, config);
@@ -121,31 +76,38 @@ public class PercolateQueryTests extends ESTestCase {
         directory.close();
     }
 
-    public void testVariousQueries() throws Exception {
-        addPercolatorQuery("1", new TermQuery(new Term("field", "brown")));
-        addPercolatorQuery("2", new TermQuery(new Term("field", "monkey")));
-        addPercolatorQuery("3", new TermQuery(new Term("field", "fox")));
-        BooleanQuery.Builder bq1 = new BooleanQuery.Builder();
-        bq1.add(new TermQuery(new Term("field", "fox")), BooleanClause.Occur.SHOULD);
-        bq1.add(new TermQuery(new Term("field", "monkey")), BooleanClause.Occur.SHOULD);
-        addPercolatorQuery("4", bq1.build());
-        BooleanQuery.Builder bq2 = new BooleanQuery.Builder();
-        bq2.add(new TermQuery(new Term("field", "fox")), BooleanClause.Occur.MUST);
-        bq2.add(new TermQuery(new Term("field", "monkey")), BooleanClause.Occur.MUST);
-        addPercolatorQuery("5", bq2.build());
-        BooleanQuery.Builder bq3 = new BooleanQuery.Builder();
-        bq3.add(new TermQuery(new Term("field", "fox")), BooleanClause.Occur.MUST);
-        bq3.add(new TermQuery(new Term("field", "apes")), BooleanClause.Occur.MUST_NOT);
-        addPercolatorQuery("6", bq3.build());
-        BooleanQuery.Builder bq4 = new BooleanQuery.Builder();
-        bq4.add(new TermQuery(new Term("field", "fox")), BooleanClause.Occur.MUST_NOT);
-        bq4.add(new TermQuery(new Term("field", "apes")), BooleanClause.Occur.MUST);
-        addPercolatorQuery("7", bq4.build());
-        PhraseQuery.Builder pq1 = new PhraseQuery.Builder();
-        pq1.add(new Term("field", "lazy"));
-        pq1.add(new Term("field", "dog"));
-        addPercolatorQuery("8", pq1.build());
+    public void testPercolateQuery() throws Exception {
+        List<Iterable<? extends IndexableField>> docs = new ArrayList<>();
+        List<Query> queries = new ArrayList<>();
+        PercolateQuery.QueryStore queryStore = ctx -> queries::get;
 
+        queries.add(new TermQuery(new Term("field", "fox")));
+        docs.add(Collections.singleton(new StringField("select", "a", Field.Store.NO)));
+
+        SpanNearQuery.Builder snp = new SpanNearQuery.Builder("field", true);
+        snp.addClause(new SpanTermQuery(new Term("field", "jumps")));
+        snp.addClause(new SpanTermQuery(new Term("field", "lazy")));
+        snp.addClause(new SpanTermQuery(new Term("field", "dog")));
+        snp.setSlop(2);
+        queries.add(snp.build());
+        docs.add(Collections.singleton(new StringField("select", "b", Field.Store.NO)));
+
+        PhraseQuery.Builder pq1 = new PhraseQuery.Builder();
+        pq1.add(new Term("field", "quick"));
+        pq1.add(new Term("field", "brown"));
+        pq1.add(new Term("field", "jumps"));
+        pq1.setSlop(1);
+        queries.add(pq1.build());
+        docs.add(Collections.singleton(new StringField("select", "b", Field.Store.NO)));
+
+        BooleanQuery.Builder bq1 = new BooleanQuery.Builder();
+        bq1.add(new TermQuery(new Term("field", "quick")), BooleanClause.Occur.MUST);
+        bq1.add(new TermQuery(new Term("field", "brown")), BooleanClause.Occur.MUST);
+        bq1.add(new TermQuery(new Term("field", "fox")), BooleanClause.Occur.MUST);
+        queries.add(bq1.build());
+        docs.add(Collections.singleton(new StringField("select", "b", Field.Store.NO)));
+
+        indexWriter.addDocuments(docs);
         indexWriter.close();
         directoryReader = DirectoryReader.open(directory);
         IndexSearcher shardSearcher = newSearcher(directoryReader);
@@ -153,26 +115,26 @@ public class PercolateQueryTests extends ESTestCase {
         MemoryIndex memoryIndex = new MemoryIndex();
         memoryIndex.addField("field", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer());
         IndexSearcher percolateSearcher = memoryIndex.createSearcher();
-
-        PercolateQuery.Builder builder = new PercolateQuery.Builder(
-                "docType",
-                queryStore,
-                new BytesArray("{}"),
-                percolateSearcher
-        );
-        builder.extractQueryTermsQuery(EXTRACTED_TERMS_FIELD_NAME, UNKNOWN_QUERY_FIELD_NAME);
         // no scoring, wrapping it in a constant score query:
-        Query query = new ConstantScoreQuery(builder.build());
+        Query query = new ConstantScoreQuery(new PercolateQuery("type", queryStore, new BytesArray("a"),
+                new TermQuery(new Term("select", "a")), percolateSearcher, new MatchNoDocsQuery("")));
         TopDocs topDocs = shardSearcher.search(query, 10);
-        assertThat(topDocs.totalHits, equalTo(5));
-        assertThat(topDocs.scoreDocs.length, equalTo(5));
+        assertThat(topDocs.totalHits, equalTo(1));
+        assertThat(topDocs.scoreDocs.length, equalTo(1));
         assertThat(topDocs.scoreDocs[0].doc, equalTo(0));
         Explanation explanation = shardSearcher.explain(query, 0);
         assertThat(explanation.isMatch(), is(true));
         assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[0].score));
 
+        query = new ConstantScoreQuery(new PercolateQuery("type", queryStore, new BytesArray("b"),
+                new TermQuery(new Term("select", "b")), percolateSearcher, new MatchNoDocsQuery("")));
+        topDocs = shardSearcher.search(query, 10);
+        assertThat(topDocs.totalHits, equalTo(3));
+        assertThat(topDocs.scoreDocs.length, equalTo(3));
+        assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
         explanation = shardSearcher.explain(query, 1);
-        assertThat(explanation.isMatch(), is(false));
+        assertThat(explanation.isMatch(), is(true));
+        assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[0].score));
 
         assertThat(topDocs.scoreDocs[1].doc, equalTo(2));
         explanation = shardSearcher.explain(query, 2);
@@ -180,371 +142,37 @@ public class PercolateQueryTests extends ESTestCase {
         assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[1].score));
 
         assertThat(topDocs.scoreDocs[2].doc, equalTo(3));
-        explanation = shardSearcher.explain(query, 3);
+        explanation = shardSearcher.explain(query, 2);
         assertThat(explanation.isMatch(), is(true));
         assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[2].score));
 
-        explanation = shardSearcher.explain(query, 4);
-        assertThat(explanation.isMatch(), is(false));
+        query = new ConstantScoreQuery(new PercolateQuery("type", queryStore, new BytesArray("c"),
+                new MatchAllDocsQuery(), percolateSearcher, new MatchAllDocsQuery()));
+        topDocs = shardSearcher.search(query, 10);
+        assertThat(topDocs.totalHits, equalTo(4));
 
-        assertThat(topDocs.scoreDocs[3].doc, equalTo(5));
-        explanation = shardSearcher.explain(query, 5);
-        assertThat(explanation.isMatch(), is(true));
-        assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[3].score));
-
-        explanation = shardSearcher.explain(query, 6);
-        assertThat(explanation.isMatch(), is(false));
-
-        assertThat(topDocs.scoreDocs[4].doc, equalTo(7));
-        explanation = shardSearcher.explain(query, 7);
-        assertThat(explanation.isMatch(), is(true));
-        assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[4].score));
-    }
-
-    public void testVariousQueries_withScoring() throws Exception {
-        SpanNearQuery.Builder snp = new SpanNearQuery.Builder("field", true);
-        snp.addClause(new SpanTermQuery(new Term("field", "jumps")));
-        snp.addClause(new SpanTermQuery(new Term("field", "lazy")));
-        snp.addClause(new SpanTermQuery(new Term("field", "dog")));
-        snp.setSlop(2);
-        addPercolatorQuery("1", snp.build());
-        PhraseQuery.Builder pq1 = new PhraseQuery.Builder();
-        pq1.add(new Term("field", "quick"));
-        pq1.add(new Term("field", "brown"));
-        pq1.add(new Term("field", "jumps"));
-        pq1.setSlop(1);
-        addPercolatorQuery("2", pq1.build());
-        BooleanQuery.Builder bq1 = new BooleanQuery.Builder();
-        bq1.add(new TermQuery(new Term("field", "quick")), BooleanClause.Occur.MUST);
-        bq1.add(new TermQuery(new Term("field", "brown")), BooleanClause.Occur.MUST);
-        bq1.add(new TermQuery(new Term("field", "fox")), BooleanClause.Occur.MUST);
-        addPercolatorQuery("3", bq1.build());
-
-        indexWriter.close();
-        directoryReader = DirectoryReader.open(directory);
-        IndexSearcher shardSearcher = newSearcher(directoryReader);
-
-        MemoryIndex memoryIndex = new MemoryIndex();
-        memoryIndex.addField("field", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer());
-        IndexSearcher percolateSearcher = memoryIndex.createSearcher();
-
-        PercolateQuery.Builder builder = new PercolateQuery.Builder(
-                "docType",
-                queryStore,
-                new BytesArray("{}"),
-                percolateSearcher
-        );
-        builder.extractQueryTermsQuery(EXTRACTED_TERMS_FIELD_NAME, UNKNOWN_QUERY_FIELD_NAME);
-        Query query = builder.build();
-        TopDocs topDocs = shardSearcher.search(query, 10);
+        query = new PercolateQuery("type", queryStore, new BytesArray("{}"), new TermQuery(new Term("select", "b")),
+                percolateSearcher, new MatchNoDocsQuery(""));
+        topDocs = shardSearcher.search(query, 10);
         assertThat(topDocs.totalHits, equalTo(3));
-
-        assertThat(topDocs.scoreDocs[0].doc, equalTo(2));
-        Explanation explanation = shardSearcher.explain(query, 2);
+        assertThat(topDocs.scoreDocs.length, equalTo(3));
+        assertThat(topDocs.scoreDocs[0].doc, equalTo(3));
+        explanation = shardSearcher.explain(query, 3);
         assertThat(explanation.isMatch(), is(true));
         assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[0].score));
         assertThat(explanation.getDetails(), arrayWithSize(1));
 
-        assertThat(topDocs.scoreDocs[1].doc, equalTo(1));
-        explanation = shardSearcher.explain(query, 1);
+        assertThat(topDocs.scoreDocs[1].doc, equalTo(2));
+        explanation = shardSearcher.explain(query, 2);
         assertThat(explanation.isMatch(), is(true));
         assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[1].score));
         assertThat(explanation.getDetails(), arrayWithSize(1));
 
-        assertThat(topDocs.scoreDocs[2].doc, equalTo(0));
-        explanation = shardSearcher.explain(query, 0);
+        assertThat(topDocs.scoreDocs[2].doc, equalTo(1));
+        explanation = shardSearcher.explain(query, 1);
         assertThat(explanation.isMatch(), is(true));
         assertThat(explanation.getValue(), equalTo(topDocs.scoreDocs[2].score));
         assertThat(explanation.getDetails(), arrayWithSize(1));
     }
 
-    public void testDuel() throws Exception {
-        List<Function<String, Query>> queries = new ArrayList<>();
-        queries.add((id) -> new PrefixQuery(new Term("field", id)));
-        queries.add((id) -> new WildcardQuery(new Term("field", id + "*")));
-        queries.add((id) -> new CustomQuery(new Term("field", id)));
-        queries.add((id) -> new SpanTermQuery(new Term("field", id)));
-        queries.add((id) -> new TermQuery(new Term("field", id)));
-        queries.add((id) -> {
-            BooleanQuery.Builder builder = new BooleanQuery.Builder();
-            return builder.build();
-        });
-        queries.add((id) -> {
-            BooleanQuery.Builder builder = new BooleanQuery.Builder();
-            builder.add(new TermQuery(new Term("field", id)), BooleanClause.Occur.MUST);
-            if (randomBoolean()) {
-                builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT);
-            }
-            if (randomBoolean()) {
-                builder.add(new CustomQuery(new Term("field", id)), BooleanClause.Occur.MUST);
-            }
-            return builder.build();
-        });
-        queries.add((id) -> {
-            BooleanQuery.Builder builder = new BooleanQuery.Builder();
-            builder.add(new TermQuery(new Term("field", id)), BooleanClause.Occur.SHOULD);
-            if (randomBoolean()) {
-                builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT);
-            }
-            if (randomBoolean()) {
-                builder.add(new CustomQuery(new Term("field", id)), BooleanClause.Occur.SHOULD);
-            }
-            return builder.build();
-        });
-        queries.add((id) -> {
-            BooleanQuery.Builder builder = new BooleanQuery.Builder();
-            builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
-            builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
-            if (randomBoolean()) {
-                builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT);
-            }
-            return builder.build();
-        });
-        queries.add((id) -> {
-            BooleanQuery.Builder builder = new BooleanQuery.Builder();
-            builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
-            builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
-            if (randomBoolean()) {
-                builder.add(new MatchNoDocsQuery("no reason"), BooleanClause.Occur.MUST_NOT);
-            }
-            return builder.build();
-        });
-        queries.add((id) -> {
-            BooleanQuery.Builder builder = new BooleanQuery.Builder();
-            builder.setMinimumNumberShouldMatch(randomIntBetween(0, 4));
-            builder.add(new TermQuery(new Term("field", id)), BooleanClause.Occur.SHOULD);
-            builder.add(new CustomQuery(new Term("field", id)), BooleanClause.Occur.SHOULD);
-            return builder.build();
-        });
-        queries.add((id) -> new MatchAllDocsQuery());
-        queries.add((id) -> new MatchNoDocsQuery("no reason at all"));
-
-        int numDocs = randomIntBetween(queries.size(), queries.size() * 3);
-        for (int i = 0; i < numDocs; i++) {
-            String id = Integer.toString(i);
-            addPercolatorQuery(id, queries.get(i % queries.size()).apply(id));
-        }
-
-        indexWriter.close();
-        directoryReader = DirectoryReader.open(directory);
-        IndexSearcher shardSearcher = newSearcher(directoryReader);
-        // Disable query cache, because ControlQuery cannot be cached...
-        shardSearcher.setQueryCache(null);
-
-        for (int i = 0; i < numDocs; i++) {
-            String id = Integer.toString(i);
-            MemoryIndex memoryIndex = new MemoryIndex();
-            memoryIndex.addField("field", id, new WhitespaceAnalyzer());
-            duelRun(memoryIndex, shardSearcher);
-        }
-
-        MemoryIndex memoryIndex = new MemoryIndex();
-        memoryIndex.addField("field", "value", new WhitespaceAnalyzer());
-        duelRun(memoryIndex, shardSearcher);
-        // Empty percolator doc:
-        memoryIndex = new MemoryIndex();
-        duelRun(memoryIndex, shardSearcher);
-    }
-
-    public void testDuelSpecificQueries() throws Exception {
-        CommonTermsQuery commonTermsQuery = new CommonTermsQuery(BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, 128);
-        commonTermsQuery.add(new Term("field", "quick"));
-        commonTermsQuery.add(new Term("field", "brown"));
-        commonTermsQuery.add(new Term("field", "fox"));
-        addPercolatorQuery("_id1", commonTermsQuery);
-
-        BlendedTermQuery blendedTermQuery = BlendedTermQuery.booleanBlendedQuery(new Term[]{new Term("field", "quick"),
-                new Term("field", "brown"), new Term("field", "fox")}, false);
-        addPercolatorQuery("_id2", blendedTermQuery);
-
-        SpanNearQuery spanNearQuery = new SpanNearQuery.Builder("field", true)
-                .addClause(new SpanTermQuery(new Term("field", "quick")))
-                .addClause(new SpanTermQuery(new Term("field", "brown")))
-                .addClause(new SpanTermQuery(new Term("field", "fox")))
-                .build();
-        addPercolatorQuery("_id3", spanNearQuery);
-
-        SpanNearQuery spanNearQuery2 = new SpanNearQuery.Builder("field", true)
-                .addClause(new SpanTermQuery(new Term("field", "the")))
-                .addClause(new SpanTermQuery(new Term("field", "lazy")))
-                .addClause(new SpanTermQuery(new Term("field", "doc")))
-                .build();
-        SpanOrQuery spanOrQuery = new SpanOrQuery(
-                spanNearQuery,
-                spanNearQuery2
-        );
-        addPercolatorQuery("_id4", spanOrQuery);
-
-        SpanNotQuery spanNotQuery = new SpanNotQuery(spanNearQuery, spanNearQuery);
-        addPercolatorQuery("_id5", spanNotQuery);
-
-        indexWriter.close();
-        directoryReader = DirectoryReader.open(directory);
-        IndexSearcher shardSearcher = newSearcher(directoryReader);
-        // Disable query cache, because ControlQuery cannot be cached...
-        shardSearcher.setQueryCache(null);
-
-        MemoryIndex memoryIndex = new MemoryIndex();
-        memoryIndex.addField("field", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer());
-        duelRun(memoryIndex, shardSearcher);
-    }
-
-    void addPercolatorQuery(String id, Query query, String... extraFields) throws IOException {
-        queries.put(id, query);
-        ParseContext.Document document = new ParseContext.Document();
-        ExtractQueryTermsService.extractQueryTerms(query, document, EXTRACTED_TERMS_FIELD_NAME, UNKNOWN_QUERY_FIELD_NAME,
-                EXTRACTED_TERMS_FIELD_TYPE);
-        document.add(new StoredField(UidFieldMapper.NAME, Uid.createUid(MapperService.PERCOLATOR_LEGACY_TYPE_NAME, id)));
-        assert extraFields.length % 2 == 0;
-        for (int i = 0; i < extraFields.length; i++) {
-            document.add(new StringField(extraFields[i], extraFields[++i], Field.Store.NO));
-        }
-        indexWriter.addDocument(document);
-    }
-
-    private void duelRun(MemoryIndex memoryIndex, IndexSearcher shardSearcher) throws IOException {
-        boolean requireScore = randomBoolean();
-        IndexSearcher percolateSearcher = memoryIndex.createSearcher();
-        PercolateQuery.Builder builder = new PercolateQuery.Builder(
-                "docType",
-                queryStore,
-                new BytesArray("{}"),
-                percolateSearcher
-        );
-        // enables the optimization that prevents queries from being evaluated that don't match
-        builder.extractQueryTermsQuery(EXTRACTED_TERMS_FIELD_NAME, UNKNOWN_QUERY_FIELD_NAME);
-        Query query = requireScore ? builder.build() : new ConstantScoreQuery(builder.build());
-        TopDocs topDocs = shardSearcher.search(query, 10);
-
-        Query controlQuery = new ControlQuery(memoryIndex, queryStore);
-        controlQuery = requireScore ? controlQuery : new ConstantScoreQuery(controlQuery);
-        TopDocs controlTopDocs = shardSearcher.search(controlQuery, 10);
-        assertThat(topDocs.totalHits, equalTo(controlTopDocs.totalHits));
-        assertThat(topDocs.scoreDocs.length, equalTo(controlTopDocs.scoreDocs.length));
-        for (int j = 0; j < topDocs.scoreDocs.length; j++) {
-            assertThat(topDocs.scoreDocs[j].doc, equalTo(controlTopDocs.scoreDocs[j].doc));
-            assertThat(topDocs.scoreDocs[j].score, equalTo(controlTopDocs.scoreDocs[j].score));
-            if (requireScore) {
-                Explanation explain1 = shardSearcher.explain(query, topDocs.scoreDocs[j].doc);
-                Explanation explain2 = shardSearcher.explain(controlQuery, controlTopDocs.scoreDocs[j].doc);
-                assertThat(explain1.isMatch(), equalTo(explain2.isMatch()));
-                assertThat(explain1.getValue(), equalTo(explain2.getValue()));
-            }
-        }
-    }
-
-    private static final class CustomQuery extends Query {
-
-        private final Term term;
-
-        private CustomQuery(Term term) {
-            this.term = term;
-        }
-
-        @Override
-        public Query rewrite(IndexReader reader) throws IOException {
-            return new TermQuery(term);
-        }
-
-        @Override
-        public String toString(String field) {
-            return "custom{" + field + "}";
-        }
-
-        @Override
-        public boolean equals(Object obj) {
-            return sameClassAs(obj);
-        }
-
-        @Override
-        public int hashCode() {
-            return classHash();
-        }
-    }
-
-    private static final class ControlQuery extends Query {
-
-        private final MemoryIndex memoryIndex;
-        private final PercolateQuery.QueryStore queryStore;
-
-        private ControlQuery(MemoryIndex memoryIndex, PercolateQuery.QueryStore queryStore) {
-            this.memoryIndex = memoryIndex;
-            this.queryStore = queryStore;
-        }
-
-        @Override
-        public Weight createWeight(IndexSearcher searcher, boolean needsScores) {
-            return new ConstantScoreWeight(this) {
-
-                float _score;
-
-                @Override
-                public Explanation explain(LeafReaderContext context, int doc) throws IOException {
-                    Scorer scorer = scorer(context);
-                    if (scorer != null) {
-                        int result = scorer.iterator().advance(doc);
-                        if (result == doc) {
-                            return Explanation.match(scorer.score(), "ControlQuery");
-                        }
-                    }
-                    return Explanation.noMatch("ControlQuery");
-                }
-
-                @Override
-                public String toString() {
-                    return "weight(" + ControlQuery.this + ")";
-                }
-
-                @Override
-                public Scorer scorer(LeafReaderContext context) throws IOException {
-                    DocIdSetIterator allDocs = DocIdSetIterator.all(context.reader().maxDoc());
-                    PercolateQuery.QueryStore.Leaf leaf = queryStore.getQueries(context);
-                    FilteredDocIdSetIterator memoryIndexIterator = new FilteredDocIdSetIterator(allDocs) {
-
-                        @Override
-                        protected boolean match(int doc) {
-                            try {
-                                Query query = leaf.getQuery(doc);
-                                float score = memoryIndex.search(query);
-                                if (score != 0f) {
-                                    if (needsScores) {
-                                        _score = score;
-                                    }
-                                    return true;
-                                } else {
-                                    return false;
-                                }
-                            } catch (IOException e) {
-                                throw new RuntimeException(e);
-                            }
-                        }
-                    };
-                    return new FilterScorer(new ConstantScoreScorer(this, score(), memoryIndexIterator)) {
-
-                        @Override
-                        public float score() throws IOException {
-                            return _score;
-                        }
-                    };
-                }
-            };
-        }
-
-        @Override
-        public String toString(String field) {
-            return "control{" + field + "}";
-        }
-
-        @Override
-        public boolean equals(Object obj) {
-            return sameClassAs(obj);
-        }
-
-        @Override
-        public int hashCode() {
-            return classHash();
-        }
-
-    }
-
 }
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
index d6221184b6e..57dda2f55cd 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java
@@ -19,26 +19,51 @@
 
 package org.elasticsearch.percolator;
 
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.PrefixCodedTerms;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.memory.MemoryIndex;
+import org.apache.lucene.queries.TermsQuery;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.IndexService;
+import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.ParsedDocument;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.BoostingQueryBuilder;
+import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardException;
+import org.elasticsearch.index.query.RangeQueryBuilder;
+import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
+import org.elasticsearch.index.query.functionscore.RandomScoreFunctionBuilder;
 import org.elasticsearch.indices.TermsLookup;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESSingleNodeTestCase;
 import org.junit.Before;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
@@ -49,8 +74,9 @@ import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termsLookupQuery;
 import static org.elasticsearch.index.query.QueryBuilders.wildcardQuery;
-import static org.elasticsearch.percolator.ExtractQueryTermsService.EXTRACTION_COMPLETE;
-import static org.elasticsearch.percolator.ExtractQueryTermsService.EXTRACTION_FAILED;
+import static org.elasticsearch.percolator.PercolatorFieldMapper.EXTRACTION_COMPLETE;
+import static org.elasticsearch.percolator.PercolatorFieldMapper.EXTRACTION_FAILED;
+import static org.elasticsearch.percolator.PercolatorFieldMapper.EXTRACTION_PARTIAL;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
@@ -61,7 +87,7 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
     private String fieldName;
     private IndexService indexService;
     private MapperService mapperService;
-    private PercolatorFieldMapper.PercolatorFieldType fieldType;
+    private PercolatorFieldMapper.FieldType fieldType;
 
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
@@ -77,6 +103,10 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
             .startObject("_field_names").field("enabled", false).endObject() // makes testing easier
             .startObject("properties")
                 .startObject("field").field("type", "text").endObject()
+                .startObject("field1").field("type", "text").endObject()
+                .startObject("field2").field("type", "text").endObject()
+                .startObject("_field3").field("type", "text").endObject()
+                .startObject("field4").field("type", "text").endObject()
                 .startObject("number_field").field("type", "long").endObject()
                 .startObject("date_field").field("type", "date").endObject()
             .endObject().endObject().endObject().string();
@@ -90,7 +120,101 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
                 .startObject("properties").startObject(fieldName).field("type", "percolator").endObject().endObject()
                 .endObject().endObject().string();
         mapperService.merge(typeName, new CompressedXContent(percolatorMapper), MapperService.MergeReason.MAPPING_UPDATE, true);
-        fieldType = (PercolatorFieldMapper.PercolatorFieldType) mapperService.fullName(fieldName);
+        fieldType = (PercolatorFieldMapper.FieldType) mapperService.fullName(fieldName);
+    }
+
+    public void testExtractTerms() throws Exception {
+        addQueryMapping();
+        BooleanQuery.Builder bq = new BooleanQuery.Builder();
+        TermQuery termQuery1 = new TermQuery(new Term("field", "term1"));
+        bq.add(termQuery1, BooleanClause.Occur.SHOULD);
+        TermQuery termQuery2 = new TermQuery(new Term("field", "term2"));
+        bq.add(termQuery2, BooleanClause.Occur.SHOULD);
+
+        DocumentMapper documentMapper = mapperService.documentMapper(typeName);
+        PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName);
+        ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(Settings.EMPTY,
+                mapperService.documentMapperParser(), documentMapper, null, null);
+        fieldMapper.processQuery(bq.build(), parseContext);
+        ParseContext.Document document = parseContext.doc();
+
+        PercolatorFieldMapper.FieldType fieldType = (PercolatorFieldMapper.FieldType) fieldMapper.fieldType();
+        assertThat(document.getField(fieldType.extractionResultField.name()).stringValue(), equalTo(EXTRACTION_COMPLETE));
+        List<IndexableField> fields = new ArrayList<>(Arrays.asList(document.getFields(fieldType.queryTermsField.name())));
+        Collections.sort(fields, (field1, field2) -> field1.binaryValue().compareTo(field2.binaryValue()));
+        assertThat(fields.size(), equalTo(2));
+        assertThat(fields.get(0).binaryValue().utf8ToString(), equalTo("field\u0000term1"));
+        assertThat(fields.get(1).binaryValue().utf8ToString(), equalTo("field\u0000term2"));
+    }
+
+    public void testExtractTermsAndRanges_failed() throws Exception {
+        addQueryMapping();
+        TermRangeQuery query = new TermRangeQuery("field1", new BytesRef("a"), new BytesRef("z"), true, true);
+        DocumentMapper documentMapper = mapperService.documentMapper(typeName);
+        PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName);
+        ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(Settings.EMPTY,
+                mapperService.documentMapperParser(), documentMapper, null, null);
+        fieldMapper.processQuery(query, parseContext);
+        ParseContext.Document document = parseContext.doc();
+
+        PercolatorFieldMapper.FieldType fieldType = (PercolatorFieldMapper.FieldType) fieldMapper.fieldType();
+        assertThat(document.getFields().size(), equalTo(1));
+        assertThat(document.getField(fieldType.extractionResultField.name()).stringValue(), equalTo(EXTRACTION_FAILED));
+    }
+
+    public void testExtractTermsAndRanges_partial() throws Exception {
+        addQueryMapping();
+        PhraseQuery phraseQuery = new PhraseQuery("field", "term");
+        DocumentMapper documentMapper = mapperService.documentMapper(typeName);
+        PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName);
+        ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(Settings.EMPTY,
+                mapperService.documentMapperParser(), documentMapper, null, null);
+        fieldMapper.processQuery(phraseQuery, parseContext);
+        ParseContext.Document document = parseContext.doc();
+
+        PercolatorFieldMapper.FieldType fieldType = (PercolatorFieldMapper.FieldType) fieldMapper.fieldType();
+        assertThat(document.getFields().size(), equalTo(2));
+        assertThat(document.getFields().get(0).binaryValue().utf8ToString(), equalTo("field\u0000term"));
+        assertThat(document.getField(fieldType.extractionResultField.name()).stringValue(), equalTo(EXTRACTION_PARTIAL));
+    }
+
+    public void testCreateCandidateQuery() throws Exception {
+        addQueryMapping();
+
+        MemoryIndex memoryIndex = new MemoryIndex(false);
+        memoryIndex.addField("field1", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer());
+        memoryIndex.addField("field2", "some more text", new WhitespaceAnalyzer());
+        memoryIndex.addField("_field3", "unhide me", new WhitespaceAnalyzer());
+        memoryIndex.addField("field4", "123", new WhitespaceAnalyzer());
+        memoryIndex.addField(new LongPoint("number_field", 10L), new WhitespaceAnalyzer());
+
+        IndexReader indexReader = memoryIndex.createSearcher().getIndexReader();
+
+        TermsQuery termsQuery = (TermsQuery) fieldType.createCandidateQuery(indexReader);
+
+        PrefixCodedTerms terms = termsQuery.getTermData();
+        assertThat(terms.size(), equalTo(15L));
+        PrefixCodedTerms.TermIterator termIterator = terms.iterator();
+        assertTermIterator(termIterator, "_field3\u0000me", fieldType.queryTermsField.name());
+        assertTermIterator(termIterator, "_field3\u0000unhide", fieldType.queryTermsField.name());
+        assertTermIterator(termIterator, "field1\u0000brown", fieldType.queryTermsField.name());
+        assertTermIterator(termIterator, "field1\u0000dog", fieldType.queryTermsField.name());
+        assertTermIterator(termIterator, "field1\u0000fox", fieldType.queryTermsField.name());
+        assertTermIterator(termIterator, "field1\u0000jumps", fieldType.queryTermsField.name());
+        assertTermIterator(termIterator, "field1\u0000lazy", fieldType.queryTermsField.name());
+        assertTermIterator(termIterator, "field1\u0000over", fieldType.queryTermsField.name());
+        assertTermIterator(termIterator, "field1\u0000quick", fieldType.queryTermsField.name());
+        assertTermIterator(termIterator, "field1\u0000the", fieldType.queryTermsField.name());
+        assertTermIterator(termIterator, "field2\u0000more", fieldType.queryTermsField.name());
+        assertTermIterator(termIterator, "field2\u0000some", fieldType.queryTermsField.name());
+        assertTermIterator(termIterator, "field2\u0000text", fieldType.queryTermsField.name());
+        assertTermIterator(termIterator, "field4\u0000123", fieldType.queryTermsField.name());
+        assertTermIterator(termIterator, EXTRACTION_FAILED, fieldType.extractionResultField.name());
+    }
+
+    private void assertTermIterator(PrefixCodedTerms.TermIterator termIterator, String expectedValue, String expectedField) {
+        assertThat(termIterator.next().utf8ToString(), equalTo(expectedValue));
+        assertThat(termIterator.field(), equalTo(expectedField));
     }
 
     public void testPercolatorFieldMapper() throws Exception {
@@ -100,12 +224,13 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
             .field(fieldName, queryBuilder)
             .endObject().bytes());
 
-        assertThat(doc.rootDoc().getFields(fieldType.getExtractedTermsField()).length, equalTo(1));
-        assertThat(doc.rootDoc().getFields(fieldType.getExtractedTermsField())[0].binaryValue().utf8ToString(), equalTo("field\0value"));
-        assertThat(doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName()).length, equalTo(1));
-        assertThat(doc.rootDoc().getFields(fieldType.getExtractionResultFieldName()).length, equalTo(1));
-        assertThat(doc.rootDoc().getFields(fieldType.getExtractionResultFieldName())[0].stringValue(), equalTo(EXTRACTION_COMPLETE));
-        BytesRef qbSource = doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName())[0].binaryValue();
+        assertThat(doc.rootDoc().getFields(fieldType.queryTermsField.name()).length, equalTo(1));
+        assertThat(doc.rootDoc().getFields(fieldType.queryTermsField.name())[0].binaryValue().utf8ToString(), equalTo("field\0value"));
+        assertThat(doc.rootDoc().getFields(fieldType.queryBuilderField.name()).length, equalTo(1));
+        assertThat(doc.rootDoc().getFields(fieldType.extractionResultField.name()).length, equalTo(1));
+        assertThat(doc.rootDoc().getFields(fieldType.extractionResultField.name())[0].stringValue(),
+                equalTo(EXTRACTION_COMPLETE));
+        BytesRef qbSource = doc.rootDoc().getFields(fieldType.queryBuilderField.name())[0].binaryValue();
         assertQueryBuilder(qbSource, queryBuilder);
 
         // add an query for which we don't extract terms from
@@ -113,11 +238,12 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
         doc = mapperService.documentMapper(typeName).parse("test", typeName, "1", XContentFactory.jsonBuilder().startObject()
                 .field(fieldName, queryBuilder)
                 .endObject().bytes());
-        assertThat(doc.rootDoc().getFields(fieldType.getExtractionResultFieldName()).length, equalTo(1));
-        assertThat(doc.rootDoc().getFields(fieldType.getExtractionResultFieldName())[0].stringValue(), equalTo(EXTRACTION_FAILED));
-        assertThat(doc.rootDoc().getFields(fieldType.getExtractedTermsField()).length, equalTo(0));
-        assertThat(doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName()).length, equalTo(1));
-        qbSource = doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName())[0].binaryValue();
+        assertThat(doc.rootDoc().getFields(fieldType.extractionResultField.name()).length, equalTo(1));
+        assertThat(doc.rootDoc().getFields(fieldType.extractionResultField.name())[0].stringValue(),
+                equalTo(EXTRACTION_FAILED));
+        assertThat(doc.rootDoc().getFields(fieldType.queryTermsField.name()).length, equalTo(0));
+        assertThat(doc.rootDoc().getFields(fieldType.queryBuilderField.name()).length, equalTo(1));
+        qbSource = doc.rootDoc().getFields(fieldType.queryBuilderField.name())[0].binaryValue();
         assertQueryBuilder(qbSource, queryBuilder);
     }
 
@@ -136,7 +262,7 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
                     XContentFactory.jsonBuilder().startObject()
                     .field(fieldName, query)
                     .endObject().bytes());
-            BytesRef qbSource = doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName())[0].binaryValue();
+            BytesRef qbSource = doc.rootDoc().getFields(fieldType.queryBuilderField.name())[0].binaryValue();
             assertQueryBuilder(qbSource, query);
         }
     }
@@ -148,7 +274,7 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
         ParsedDocument doc = mapperService.documentMapper(typeName).parse("test", typeName, "1", XContentFactory.jsonBuilder().startObject()
                 .field(fieldName, queryBuilder)
                 .endObject().bytes());
-        BytesRef qbSource = doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName())[0].binaryValue();
+        BytesRef qbSource = doc.rootDoc().getFields(fieldType.queryBuilderField.name())[0].binaryValue();
         assertQueryBuilder(qbSource, queryBuilder.rewrite(indexService.newQueryShardContext()));
     }
 
@@ -169,7 +295,7 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
         addQueryMapping();
         ParsedDocument doc = mapperService.documentMapper(typeName).parse("test", typeName, "1", XContentFactory.jsonBuilder().startObject()
             .endObject().bytes());
-        assertThat(doc.rootDoc().getFields(fieldType.getQueryBuilderFieldName()).length, equalTo(0));
+        assertThat(doc.rootDoc().getFields(fieldType.queryBuilderField.name()).length, equalTo(0));
 
         try {
             mapperService.documentMapper(typeName).parse("test", typeName, "1", XContentFactory.jsonBuilder().startObject()
@@ -275,6 +401,53 @@ public class PercolatorFieldMapperTests extends ESSingleNodeTestCase {
         assertThat(e.getCause().getMessage(), equalTo("a document can only contain one percolator query"));
     }
 
+    public void testRangeQueryWithNowRangeIsForbidden() throws Exception {
+        addQueryMapping();
+        MapperParsingException e = expectThrows(MapperParsingException.class, () -> {
+            mapperService.documentMapper(typeName).parse("test", typeName, "1",
+                        jsonBuilder().startObject()
+                                .field(fieldName, rangeQuery("date_field").from("2016-01-01||/D").to("now"))
+                                .endObject().bytes());
+            }
+        );
+        assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
+        e = expectThrows(MapperParsingException.class, () -> {
+            mapperService.documentMapper(typeName).parse("test", typeName, "1",
+                        jsonBuilder().startObject()
+                                .field(fieldName, rangeQuery("date_field").from("2016-01-01||/D").to("now/D"))
+                                .endObject().bytes());
+                }
+        );
+        assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
+        e = expectThrows(MapperParsingException.class, () -> {
+            mapperService.documentMapper(typeName).parse("test", typeName, "1",
+                        jsonBuilder().startObject()
+                                .field(fieldName, rangeQuery("date_field").from("now-1d").to("now"))
+                                .endObject().bytes());
+                }
+        );
+        assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
+    }
+
+    public void testVerifyRangeQueries() {
+        RangeQueryBuilder rangeQuery1 = new RangeQueryBuilder("field").from("2016-01-01||/D").to("2017-01-01||/D");
+        RangeQueryBuilder rangeQuery2 = new RangeQueryBuilder("field").from("2016-01-01||/D").to("now");
+        PercolatorFieldMapper.verifyRangeQueries(rangeQuery1);
+        expectThrows(IllegalArgumentException.class, () -> PercolatorFieldMapper.verifyRangeQueries(rangeQuery2));
+        PercolatorFieldMapper.verifyRangeQueries(new BoolQueryBuilder().must(rangeQuery1));
+        expectThrows(IllegalArgumentException.class, () ->
+                PercolatorFieldMapper.verifyRangeQueries(new BoolQueryBuilder().must(rangeQuery2)));
+        PercolatorFieldMapper.verifyRangeQueries(new ConstantScoreQueryBuilder((rangeQuery1)));
+        expectThrows(IllegalArgumentException.class, () ->
+                PercolatorFieldMapper.verifyRangeQueries(new ConstantScoreQueryBuilder(rangeQuery2)));
+        PercolatorFieldMapper.verifyRangeQueries(new BoostingQueryBuilder(rangeQuery1, new MatchAllQueryBuilder()));
+        expectThrows(IllegalArgumentException.class, () ->
+                PercolatorFieldMapper.verifyRangeQueries(new BoostingQueryBuilder(rangeQuery2, new MatchAllQueryBuilder())));
+        PercolatorFieldMapper.verifyRangeQueries(new FunctionScoreQueryBuilder(rangeQuery1, new RandomScoreFunctionBuilder()));
+        expectThrows(IllegalArgumentException.class, () ->
+                PercolatorFieldMapper.verifyRangeQueries(new FunctionScoreQueryBuilder(rangeQuery2, new RandomScoreFunctionBuilder())));
+    }
+
     private void assertQueryBuilder(BytesRef actual, QueryBuilder expected) throws IOException {
         XContentParser sourceParser = PercolatorFieldMapper.QUERY_BUILDER_CONTENT_TYPE.xContent()
                 .createParser(actual.bytes, actual.offset, actual.length);
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
index 69ef623f801..0870fb2b32c 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhaseTests.java
@@ -45,10 +45,9 @@ import static org.hamcrest.Matchers.sameInstance;
 public class PercolatorHighlightSubFetchPhaseTests extends ESTestCase {
 
     public void testHitsExecutionNeeded() {
-        PercolateQuery percolateQuery = new PercolateQuery.Builder("", ctx -> null, new BytesArray("{}"),
-                Mockito.mock(IndexSearcher.class))
-                .build();
-
+        PercolateQuery percolateQuery = new PercolateQuery(
+                "", ctx -> null, new BytesArray("{}"), new MatchAllDocsQuery(), Mockito.mock(IndexSearcher.class), new MatchAllDocsQuery()
+        );
         PercolatorHighlightSubFetchPhase subFetchPhase = new PercolatorHighlightSubFetchPhase(Settings.EMPTY,
             new Highlighters(Settings.EMPTY));
         SearchContext searchContext = Mockito.mock(SearchContext.class);
@@ -61,10 +60,9 @@ public class PercolatorHighlightSubFetchPhaseTests extends ESTestCase {
     }
 
     public void testLocatePercolatorQuery() {
-        PercolateQuery percolateQuery = new PercolateQuery.Builder("", ctx -> null, new BytesArray("{}"),
-                Mockito.mock(IndexSearcher.class))
-                .build();
-
+        PercolateQuery percolateQuery = new PercolateQuery(
+                "", ctx -> null, new BytesArray("{}"), new MatchAllDocsQuery(), Mockito.mock(IndexSearcher.class), new MatchAllDocsQuery()
+        );
         assertThat(PercolatorHighlightSubFetchPhase.locatePercolatorQuery(new MatchAllDocsQuery()), nullValue());
         BooleanQuery.Builder bq = new BooleanQuery.Builder();
         bq.add(new MatchAllDocsQuery(), BooleanClause.Occur.FILTER);
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
index e4e379c8f60..bdfa49016e9 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java
@@ -1553,31 +1553,6 @@ public class PercolatorIT extends ESIntegTestCase {
         }
     }
 
-    public void testPercolatorQueryWithNowRange() throws Exception {
-        client().admin().indices().prepareCreate(INDEX_NAME)
-                .addMapping("my-type", "timestamp", "type=date,format=epoch_millis")
-                .addMapping(TYPE_NAME, "query", "type=percolator")
-                .get();
-        ensureGreen();
-
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "1")
-                .setSource(jsonBuilder().startObject().field("query", rangeQuery("timestamp").from("now-1d").to("now")).endObject())
-                .get();
-        client().prepareIndex(INDEX_NAME, TYPE_NAME, "2")
-                .setSource(jsonBuilder().startObject().field("query", constantScoreQuery(rangeQuery("timestamp").from("now-1d").to("now"))).endObject())
-                .get();
-        refresh();
-
-        logger.info("--> Percolate doc with field1=b");
-        PercolateResponse response = preparePercolate(client())
-                .setIndices(INDEX_NAME).setDocumentType("my-type")
-                .setPercolateDoc(docBuilder().setDoc("timestamp", System.currentTimeMillis()))
-                .get();
-        assertMatchCount(response, 2L);
-        assertThat(response.getMatches(), arrayWithSize(2));
-        assertThat(convertFromTextArray(response.getMatches(), INDEX_NAME), arrayContainingInAnyOrder("1", "2"));
-    }
-
     void initNestedIndexAndPercolation() throws IOException {
         XContentBuilder mapping = XContentFactory.jsonBuilder();
         mapping.startObject().startObject("properties").startObject("companyname").field("type", "text").endObject()
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
index 5125a7ea5cc..3cc60d75cf2 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java
@@ -44,6 +44,7 @@ import static org.elasticsearch.index.query.QueryBuilders.commonTermsQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery;
+import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
 import static org.elasticsearch.index.query.QueryBuilders.spanNearQuery;
 import static org.elasticsearch.index.query.QueryBuilders.spanNotQuery;
 import static org.elasticsearch.index.query.QueryBuilders.spanTermQuery;
@@ -109,6 +110,102 @@ public class PercolatorQuerySearchIT extends ESSingleNodeTestCase {
         assertThat(response.getHits().getAt(2).getId(), equalTo("3"));
     }
 
+    public void testPercolatorRangeQueries() throws Exception {
+        createIndex("test", client().admin().indices().prepareCreate("test")
+                .addMapping("type", "field1", "type=long", "field2", "type=double", "field3", "type=ip")
+                .addMapping("queries", "query", "type=percolator")
+        );
+
+        client().prepareIndex("test", "queries", "1")
+                .setSource(jsonBuilder().startObject().field("query", rangeQuery("field1").from(10).to(12)).endObject())
+                .get();
+        client().prepareIndex("test", "queries", "2")
+                .setSource(jsonBuilder().startObject().field("query", rangeQuery("field1").from(20).to(22)).endObject())
+                .get();
+        client().prepareIndex("test", "queries", "3")
+                .setSource(jsonBuilder().startObject().field("query", boolQuery()
+                        .must(rangeQuery("field1").from(10).to(12))
+                        .must(rangeQuery("field1").from(12).to(14))
+                ).endObject()).get();
+        client().admin().indices().prepareRefresh().get();
+        client().prepareIndex("test", "queries", "4")
+                .setSource(jsonBuilder().startObject().field("query", rangeQuery("field2").from(10).to(12)).endObject())
+                .get();
+        client().prepareIndex("test", "queries", "5")
+                .setSource(jsonBuilder().startObject().field("query", rangeQuery("field2").from(20).to(22)).endObject())
+                .get();
+        client().prepareIndex("test", "queries", "6")
+                .setSource(jsonBuilder().startObject().field("query", boolQuery()
+                        .must(rangeQuery("field2").from(10).to(12))
+                        .must(rangeQuery("field2").from(12).to(14))
+                ).endObject()).get();
+        client().admin().indices().prepareRefresh().get();
+        client().prepareIndex("test", "queries", "7")
+                .setSource(jsonBuilder().startObject()
+                        .field("query", rangeQuery("field3").from("192.168.1.0").to("192.168.1.5"))
+                        .endObject())
+                .get();
+        client().prepareIndex("test", "queries", "8")
+                .setSource(jsonBuilder().startObject()
+                        .field("query", rangeQuery("field3").from("192.168.1.20").to("192.168.1.30"))
+                        .endObject())
+                .get();
+        client().prepareIndex("test", "queries", "9")
+                .setSource(jsonBuilder().startObject().field("query", boolQuery()
+                        .must(rangeQuery("field3").from("192.168.1.0").to("192.168.1.5"))
+                        .must(rangeQuery("field3").from("192.168.1.5").to("192.168.1.10"))
+                ).endObject()).get();
+        client().admin().indices().prepareRefresh().get();
+
+        // Test long range:
+        BytesReference source = jsonBuilder().startObject().field("field1", 12).endObject().bytes();
+        SearchResponse response = client().prepareSearch()
+                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .get();
+        assertHitCount(response, 2);
+        assertThat(response.getHits().getAt(0).getId(), equalTo("3"));
+        assertThat(response.getHits().getAt(1).getId(), equalTo("1"));
+
+        source = jsonBuilder().startObject().field("field1", 11).endObject().bytes();
+        response = client().prepareSearch()
+                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .get();
+        assertHitCount(response, 1);
+        assertThat(response.getHits().getAt(0).getId(), equalTo("1"));
+
+        // Test double range:
+        source = jsonBuilder().startObject().field("field2", 12).endObject().bytes();
+        response = client().prepareSearch()
+                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .get();
+        assertHitCount(response, 2);
+        assertThat(response.getHits().getAt(0).getId(), equalTo("6"));
+        assertThat(response.getHits().getAt(1).getId(), equalTo("4"));
+
+        source = jsonBuilder().startObject().field("field2", 11).endObject().bytes();
+        response = client().prepareSearch()
+                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .get();
+        assertHitCount(response, 1);
+        assertThat(response.getHits().getAt(0).getId(), equalTo("4"));
+
+        // Test IP range:
+        source = jsonBuilder().startObject().field("field3", "192.168.1.5").endObject().bytes();
+        response = client().prepareSearch()
+                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .get();
+        assertHitCount(response, 2);
+        assertThat(response.getHits().getAt(0).getId(), equalTo("9"));
+        assertThat(response.getHits().getAt(1).getId(), equalTo("7"));
+
+        source = jsonBuilder().startObject().field("field3", "192.168.1.4").endObject().bytes();
+        response = client().prepareSearch()
+                .setQuery(new PercolateQueryBuilder("query", "type", source))
+                .get();
+        assertHitCount(response, 1);
+        assertThat(response.getHits().getAt(0).getId(), equalTo("7"));
+    }
+
     public void testPercolatorQueryExistingDocument() throws Exception {
         createIndex("test", client().admin().indices().prepareCreate("test")
                 .addMapping("type", "field1", "type=keyword", "field2", "type=keyword")
diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java
similarity index 74%
rename from modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java
rename to modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java
index 069aac9eda2..1b8b123aa13 100644
--- a/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java
+++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryAnalyzerTests.java
@@ -18,14 +18,7 @@
  */
 package org.elasticsearch.percolator;
 
-import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.PrefixCodedTerms;
 import org.apache.lucene.index.Term;
-import org.apache.lucene.index.memory.MemoryIndex;
 import org.apache.lucene.queries.BlendedTermQuery;
 import org.apache.lucene.queries.CommonTermsQuery;
 import org.apache.lucene.queries.TermsQuery;
@@ -48,11 +41,9 @@ import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 import org.elasticsearch.common.lucene.search.function.RandomScoreFunction;
-import org.elasticsearch.index.mapper.ParseContext;
-import org.elasticsearch.percolator.ExtractQueryTermsService.Result;
+import org.elasticsearch.percolator.QueryAnalyzer.Result;
 import org.elasticsearch.test.ESTestCase;
 
-
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -60,70 +51,18 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import static org.elasticsearch.percolator.ExtractQueryTermsService.EXTRACTION_COMPLETE;
-import static org.elasticsearch.percolator.ExtractQueryTermsService.EXTRACTION_FAILED;
-import static org.elasticsearch.percolator.ExtractQueryTermsService.EXTRACTION_PARTIAL;
-import static org.elasticsearch.percolator.ExtractQueryTermsService.UnsupportedQueryException;
-import static org.elasticsearch.percolator.ExtractQueryTermsService.extractQueryTerms;
-import static org.elasticsearch.percolator.ExtractQueryTermsService.createQueryTermsQuery;
-import static org.elasticsearch.percolator.ExtractQueryTermsService.selectTermListWithTheLongestShortestTerm;
+import static org.elasticsearch.percolator.QueryAnalyzer.UnsupportedQueryException;
+import static org.elasticsearch.percolator.QueryAnalyzer.analyze;
+import static org.elasticsearch.percolator.QueryAnalyzer.selectTermListWithTheLongestShortestTerm;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.sameInstance;
 
-public class ExtractQueryTermsServiceTests extends ESTestCase {
-
-    public static final String QUERY_TERMS_FIELD = "extracted_terms";
-    public static final String EXTRACTION_RESULT_FIELD = "extraction_result";
-    public static final FieldType QUERY_TERMS_FIELD_TYPE = new FieldType();
-
-    static {
-        QUERY_TERMS_FIELD_TYPE.setTokenized(false);
-        QUERY_TERMS_FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
-        QUERY_TERMS_FIELD_TYPE.freeze();
-    }
-
-    public void testExtractQueryMetadata() {
-        BooleanQuery.Builder bq = new BooleanQuery.Builder();
-        TermQuery termQuery1 = new TermQuery(new Term("field1", "term1"));
-        bq.add(termQuery1, BooleanClause.Occur.SHOULD);
-        TermQuery termQuery2 = new TermQuery(new Term("field2", "term2"));
-        bq.add(termQuery2, BooleanClause.Occur.SHOULD);
-
-        ParseContext.Document document = new ParseContext.Document();
-        extractQueryTerms(bq.build(), document, QUERY_TERMS_FIELD, EXTRACTION_RESULT_FIELD, QUERY_TERMS_FIELD_TYPE);
-        assertThat(document.getField(EXTRACTION_RESULT_FIELD).stringValue(), equalTo(EXTRACTION_COMPLETE));
-        List<IndexableField> fields = new ArrayList<>(Arrays.asList(document.getFields(QUERY_TERMS_FIELD)));
-        Collections.sort(fields, (field1, field2) -> field1.binaryValue().compareTo(field2.binaryValue()));
-        assertThat(fields.size(), equalTo(2));
-        assertThat(fields.get(0).name(), equalTo(QUERY_TERMS_FIELD));
-        assertThat(fields.get(0).binaryValue().utf8ToString(), equalTo("field1\u0000term1"));
-        assertThat(fields.get(1).name(), equalTo(QUERY_TERMS_FIELD));
-        assertThat(fields.get(1).binaryValue().utf8ToString(), equalTo("field2\u0000term2"));
-    }
-
-    public void testExtractQueryMetadata_unsupported() {
-        TermRangeQuery query = new TermRangeQuery("field1", new BytesRef("a"), new BytesRef("z"), true, true);
-        ParseContext.Document document = new ParseContext.Document();
-        extractQueryTerms(query, document, QUERY_TERMS_FIELD, EXTRACTION_RESULT_FIELD, QUERY_TERMS_FIELD_TYPE);
-        assertThat(document.getFields().size(), equalTo(1));
-        assertThat(document.getField(EXTRACTION_RESULT_FIELD).stringValue(), equalTo(EXTRACTION_FAILED));
-    }
-
-    public void testExtractQueryMetadata_notVerified() {
-        PhraseQuery phraseQuery = new PhraseQuery("field", "term");
-
-        ParseContext.Document document = new ParseContext.Document();
-        extractQueryTerms(phraseQuery, document, QUERY_TERMS_FIELD, EXTRACTION_RESULT_FIELD, QUERY_TERMS_FIELD_TYPE);
-        assertThat(document.getFields().size(), equalTo(2));
-        assertThat(document.getFields().get(0).name(), equalTo(QUERY_TERMS_FIELD));
-        assertThat(document.getFields().get(0).binaryValue().utf8ToString(), equalTo("field\u0000term"));
-        assertThat(document.getField(EXTRACTION_RESULT_FIELD).stringValue(), equalTo(EXTRACTION_PARTIAL));
-    }
+public class QueryAnalyzerTests extends ESTestCase {
 
     public void testExtractQueryMetadata_termQuery() {
         TermQuery termQuery = new TermQuery(new Term("_field", "_term"));
-        Result result = extractQueryTerms(termQuery);
+        Result result = analyze(termQuery);
         assertThat(result.verified, is(true));
         List<Term> terms = new ArrayList<>(result.terms);
         assertThat(terms.size(), equalTo(1));
@@ -133,7 +72,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
 
     public void testExtractQueryMetadata_termsQuery() {
         TermsQuery termsQuery = new TermsQuery("_field", new BytesRef("_term1"), new BytesRef("_term2"));
-        Result result = extractQueryTerms(termsQuery);
+        Result result = analyze(termsQuery);
         assertThat(result.verified, is(true));
         List<Term> terms = new ArrayList<>(result.terms);
         Collections.sort(terms);
@@ -145,7 +84,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
 
         // test with different fields
         termsQuery = new TermsQuery(new Term("_field1", "_term1"), new Term("_field2", "_term2"));
-        result = extractQueryTerms(termsQuery);
+        result = analyze(termsQuery);
         assertThat(result.verified, is(true));
         terms = new ArrayList<>(result.terms);
         Collections.sort(terms);
@@ -158,7 +97,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
 
     public void testExtractQueryMetadata_phraseQuery() {
         PhraseQuery phraseQuery = new PhraseQuery("_field", "_term1", "term2");
-        Result result = extractQueryTerms(phraseQuery);
+        Result result = analyze(phraseQuery);
         assertThat(result.verified, is(false));
         List<Term> terms = new ArrayList<>(result.terms);
         assertThat(terms.size(), equalTo(1));
@@ -181,7 +120,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         builder.add(subBuilder.build(), BooleanClause.Occur.SHOULD);
 
         BooleanQuery booleanQuery = builder.build();
-        Result result = extractQueryTerms(booleanQuery);
+        Result result = analyze(booleanQuery);
         assertThat("Should clause with phrase query isn't verified, so entire query can't be verified", result.verified, is(false));
         List<Term> terms = new ArrayList<>(result.terms);
         Collections.sort(terms);
@@ -209,7 +148,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         builder.add(subBuilder.build(), BooleanClause.Occur.SHOULD);
 
         BooleanQuery booleanQuery = builder.build();
-        Result result = extractQueryTerms(booleanQuery);
+        Result result = analyze(booleanQuery);
         assertThat(result.verified, is(true));
         List<Term> terms = new ArrayList<>(result.terms);
         Collections.sort(terms);
@@ -232,7 +171,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         builder.add(phraseQuery, BooleanClause.Occur.SHOULD);
 
         BooleanQuery booleanQuery = builder.build();
-        Result result = extractQueryTerms(booleanQuery);
+        Result result = analyze(booleanQuery);
         assertThat(result.verified, is(false));
         List<Term> terms = new ArrayList<>(result.terms);
         assertThat(terms.size(), equalTo(1));
@@ -246,58 +185,58 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         builder.add(termQuery1, BooleanClause.Occur.SHOULD);
         TermQuery termQuery2 = new TermQuery(new Term("_field", "_term2"));
         builder.add(termQuery2, BooleanClause.Occur.SHOULD);
-        Result result = extractQueryTerms(builder.build());
+        Result result = analyze(builder.build());
         assertThat("All clauses are exact, so candidate matches are verified", result.verified, is(true));
 
         builder = new BooleanQuery.Builder();
         builder.add(termQuery1, BooleanClause.Occur.SHOULD);
         PhraseQuery phraseQuery1 = new PhraseQuery("_field", "_term1", "_term2");
         builder.add(phraseQuery1, BooleanClause.Occur.SHOULD);
-        result = extractQueryTerms(builder.build());
+        result = analyze(builder.build());
         assertThat("Clause isn't exact, so candidate matches are not verified", result.verified, is(false));
 
         builder = new BooleanQuery.Builder();
         builder.add(phraseQuery1, BooleanClause.Occur.SHOULD);
         PhraseQuery phraseQuery2 = new PhraseQuery("_field", "_term3", "_term4");
         builder.add(phraseQuery2, BooleanClause.Occur.SHOULD);
-        result = extractQueryTerms(builder.build());
+        result = analyze(builder.build());
         assertThat("No clause is exact, so candidate matches are not verified", result.verified, is(false));
 
         builder = new BooleanQuery.Builder();
         builder.add(termQuery1, BooleanClause.Occur.MUST_NOT);
         builder.add(termQuery2, BooleanClause.Occur.SHOULD);
-        result = extractQueryTerms(builder.build());
+        result = analyze(builder.build());
         assertThat("There is a must_not clause, so candidate matches are not verified", result.verified, is(false));
 
         builder = new BooleanQuery.Builder();
         builder.setMinimumNumberShouldMatch(randomIntBetween(2, 32));
         builder.add(termQuery1, BooleanClause.Occur.SHOULD);
         builder.add(termQuery2, BooleanClause.Occur.SHOULD);
-        result = extractQueryTerms(builder.build());
+        result = analyze(builder.build());
         assertThat("Minimum match is >= 1, so candidate matches are not verified", result.verified, is(false));
 
         builder = new BooleanQuery.Builder();
         builder.add(termQuery1, randomBoolean() ? BooleanClause.Occur.MUST : BooleanClause.Occur.FILTER);
-        result = extractQueryTerms(builder.build());
+        result = analyze(builder.build());
         assertThat("Single required clause, so candidate matches are verified", result.verified, is(false));
 
         builder = new BooleanQuery.Builder();
         builder.add(termQuery1, randomBoolean() ? BooleanClause.Occur.MUST : BooleanClause.Occur.FILTER);
         builder.add(termQuery2, randomBoolean() ? BooleanClause.Occur.MUST : BooleanClause.Occur.FILTER);
-        result = extractQueryTerms(builder.build());
+        result = analyze(builder.build());
         assertThat("Two or more required clauses, so candidate matches are not verified", result.verified, is(false));
 
         builder = new BooleanQuery.Builder();
         builder.add(termQuery1, randomBoolean() ? BooleanClause.Occur.MUST : BooleanClause.Occur.FILTER);
         builder.add(termQuery2, BooleanClause.Occur.MUST_NOT);
-        result = extractQueryTerms(builder.build());
+        result = analyze(builder.build());
         assertThat("Required and prohibited clauses, so candidate matches are not verified", result.verified, is(false));
     }
 
     public void testExtractQueryMetadata_constantScoreQuery() {
         TermQuery termQuery1 = new TermQuery(new Term("_field", "_term"));
         ConstantScoreQuery constantScoreQuery = new ConstantScoreQuery(termQuery1);
-        Result result = extractQueryTerms(constantScoreQuery);
+        Result result = analyze(constantScoreQuery);
         assertThat(result.verified, is(true));
         List<Term> terms = new ArrayList<>(result.terms);
         assertThat(terms.size(), equalTo(1));
@@ -308,7 +247,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
     public void testExtractQueryMetadata_boostQuery() {
         TermQuery termQuery1 = new TermQuery(new Term("_field", "_term"));
         BoostQuery constantScoreQuery = new BoostQuery(termQuery1, 1f);
-        Result result = extractQueryTerms(constantScoreQuery);
+        Result result = analyze(constantScoreQuery);
         assertThat(result.verified, is(true));
         List<Term> terms = new ArrayList<>(result.terms);
         assertThat(terms.size(), equalTo(1));
@@ -320,7 +259,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         CommonTermsQuery commonTermsQuery = new CommonTermsQuery(BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, 100);
         commonTermsQuery.add(new Term("_field", "_term1"));
         commonTermsQuery.add(new Term("_field", "_term2"));
-        Result result = extractQueryTerms(commonTermsQuery);
+        Result result = analyze(commonTermsQuery);
         assertThat(result.verified, is(false));
         List<Term> terms = new ArrayList<>(result.terms);
         Collections.sort(terms);
@@ -334,7 +273,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
     public void testExtractQueryMetadata_blendedTermQuery() {
         Term[] termsArr = new Term[]{new Term("_field", "_term1"), new Term("_field", "_term2")};
         BlendedTermQuery commonTermsQuery = BlendedTermQuery.booleanBlendedQuery(termsArr, false);
-        Result result = extractQueryTerms(commonTermsQuery);
+        Result result = analyze(commonTermsQuery);
         assertThat(result.verified, is(true));
         List<Term> terms = new ArrayList<>(result.terms);
         Collections.sort(terms);
@@ -358,7 +297,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         // 4) FieldMaskingSpanQuery is a tricky query so we shouldn't optimize this
 
         SpanTermQuery spanTermQuery1 = new SpanTermQuery(new Term("_field", "_short_term"));
-        Result result = extractQueryTerms(spanTermQuery1);
+        Result result = analyze(spanTermQuery1);
         assertThat(result.verified, is(true));
         assertTermsEqual(result.terms, spanTermQuery1.getTerm());
     }
@@ -369,7 +308,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         SpanNearQuery spanNearQuery = new SpanNearQuery.Builder("_field", true)
                 .addClause(spanTermQuery1).addClause(spanTermQuery2).build();
 
-        Result result = extractQueryTerms(spanNearQuery);
+        Result result = analyze(spanNearQuery);
         assertThat(result.verified, is(false));
         assertTermsEqual(result.terms, spanTermQuery2.getTerm());
     }
@@ -378,7 +317,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         SpanTermQuery spanTermQuery1 = new SpanTermQuery(new Term("_field", "_short_term"));
         SpanTermQuery spanTermQuery2 = new SpanTermQuery(new Term("_field", "_very_long_term"));
         SpanOrQuery spanOrQuery = new SpanOrQuery(spanTermQuery1, spanTermQuery2);
-        Result result = extractQueryTerms(spanOrQuery);
+        Result result = analyze(spanOrQuery);
         assertThat(result.verified, is(false));
         assertTermsEqual(result.terms, spanTermQuery1.getTerm(), spanTermQuery2.getTerm());
     }
@@ -386,7 +325,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
     public void testExtractQueryMetadata_spanFirstQuery() {
         SpanTermQuery spanTermQuery1 = new SpanTermQuery(new Term("_field", "_short_term"));
         SpanFirstQuery spanFirstQuery = new SpanFirstQuery(spanTermQuery1, 20);
-        Result result = extractQueryTerms(spanFirstQuery);
+        Result result = analyze(spanFirstQuery);
         assertThat(result.verified, is(false));
         assertTermsEqual(result.terms, spanTermQuery1.getTerm());
     }
@@ -395,27 +334,27 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         SpanTermQuery spanTermQuery1 = new SpanTermQuery(new Term("_field", "_short_term"));
         SpanTermQuery spanTermQuery2 = new SpanTermQuery(new Term("_field", "_very_long_term"));
         SpanNotQuery spanNotQuery = new SpanNotQuery(spanTermQuery1, spanTermQuery2);
-        Result result = extractQueryTerms(spanNotQuery);
+        Result result = analyze(spanNotQuery);
         assertThat(result.verified, is(false));
         assertTermsEqual(result.terms, spanTermQuery1.getTerm());
     }
 
     public void testExtractQueryMetadata_matchNoDocsQuery() {
-        Result result = extractQueryTerms(new MatchNoDocsQuery("sometimes there is no reason at all"));
+        Result result = analyze(new MatchNoDocsQuery("sometimes there is no reason at all"));
         assertThat(result.verified, is(true));
         assertEquals(0, result.terms.size());
 
         BooleanQuery.Builder bq = new BooleanQuery.Builder();
         bq.add(new TermQuery(new Term("field", "value")), BooleanClause.Occur.MUST);
         bq.add(new MatchNoDocsQuery("sometimes there is no reason at all"), BooleanClause.Occur.MUST);
-        result = extractQueryTerms(bq.build());
+        result = analyze(bq.build());
         assertThat(result.verified, is(false));
         assertEquals(0, result.terms.size());
 
         bq = new BooleanQuery.Builder();
         bq.add(new TermQuery(new Term("field", "value")), BooleanClause.Occur.SHOULD);
         bq.add(new MatchNoDocsQuery("sometimes there is no reason at all"), BooleanClause.Occur.SHOULD);
-        result = extractQueryTerms(bq.build());
+        result = analyze(bq.build());
         assertThat(result.verified, is(true));
         assertTermsEqual(result.terms, new Term("field", "value"));
 
@@ -423,18 +362,18 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
                 Arrays.asList(new TermQuery(new Term("field", "value")), new MatchNoDocsQuery("sometimes there is no reason at all")),
                 1f
         );
-        result = extractQueryTerms(disjunctionMaxQuery);
+        result = analyze(disjunctionMaxQuery);
         assertThat(result.verified, is(true));
         assertTermsEqual(result.terms, new Term("field", "value"));
     }
 
     public void testExtractQueryMetadata_matchAllDocsQuery() {
-        expectThrows(UnsupportedQueryException.class, () -> extractQueryTerms(new MatchAllDocsQuery()));
+        expectThrows(UnsupportedQueryException.class, () -> analyze(new MatchAllDocsQuery()));
 
         BooleanQuery.Builder builder = new BooleanQuery.Builder();
         builder.add(new TermQuery(new Term("field", "value")), BooleanClause.Occur.MUST);
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
-        Result result = extractQueryTerms(builder.build());
+        Result result = analyze(builder.build());
         assertThat(result.verified, is(false));
         assertTermsEqual(result.terms, new Term("field", "value"));
 
@@ -443,39 +382,39 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
         BooleanQuery bq1 = builder.build();
-        expectThrows(UnsupportedQueryException.class, () -> extractQueryTerms(bq1));
+        expectThrows(UnsupportedQueryException.class, () -> analyze(bq1));
 
         builder = new BooleanQuery.Builder();
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST_NOT);
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
         BooleanQuery bq2 = builder.build();
-        expectThrows(UnsupportedQueryException.class, () -> extractQueryTerms(bq2));
+        expectThrows(UnsupportedQueryException.class, () -> analyze(bq2));
 
         builder = new BooleanQuery.Builder();
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
         BooleanQuery bq3 = builder.build();
-        expectThrows(UnsupportedQueryException.class, () -> extractQueryTerms(bq3));
+        expectThrows(UnsupportedQueryException.class, () -> analyze(bq3));
 
         builder = new BooleanQuery.Builder();
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST_NOT);
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
         BooleanQuery bq4 = builder.build();
-        expectThrows(UnsupportedQueryException.class, () -> extractQueryTerms(bq4));
+        expectThrows(UnsupportedQueryException.class, () -> analyze(bq4));
 
         builder = new BooleanQuery.Builder();
         builder.add(new TermQuery(new Term("field", "value")), BooleanClause.Occur.SHOULD);
         builder.add(new MatchAllDocsQuery(), BooleanClause.Occur.SHOULD);
         BooleanQuery bq5 = builder.build();
-        expectThrows(UnsupportedQueryException.class, () -> extractQueryTerms(bq5));
+        expectThrows(UnsupportedQueryException.class, () -> analyze(bq5));
     }
 
     public void testExtractQueryMetadata_unsupportedQuery() {
         TermRangeQuery termRangeQuery = new TermRangeQuery("_field", null, null, true, false);
-        UnsupportedQueryException e = expectThrows(UnsupportedQueryException.class, () -> extractQueryTerms(termRangeQuery));
+        UnsupportedQueryException e = expectThrows(UnsupportedQueryException.class, () -> analyze(termRangeQuery));
         assertThat(e.getUnsupportedQuery(), sameInstance(termRangeQuery));
 
         TermQuery termQuery1 = new TermQuery(new Term("_field", "_term"));
@@ -484,7 +423,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         builder.add(termRangeQuery, BooleanClause.Occur.SHOULD);
         BooleanQuery bq = builder.build();
 
-        e = expectThrows(UnsupportedQueryException.class, () -> extractQueryTerms(bq));
+        e = expectThrows(UnsupportedQueryException.class, () -> analyze(bq));
         assertThat(e.getUnsupportedQuery(), sameInstance(termRangeQuery));
     }
 
@@ -497,7 +436,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         builder.add(unsupportedQuery, BooleanClause.Occur.MUST);
         BooleanQuery bq1 = builder.build();
 
-        Result result = extractQueryTerms(bq1);
+        Result result = analyze(bq1);
         assertThat(result.verified, is(false));
         assertTermsEqual(result.terms, termQuery1.getTerm());
 
@@ -507,7 +446,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         builder.add(termQuery2, BooleanClause.Occur.MUST);
         builder.add(unsupportedQuery, BooleanClause.Occur.MUST);
         bq1 = builder.build();
-        result = extractQueryTerms(bq1);
+        result = analyze(bq1);
         assertThat(result.verified, is(false));
         assertTermsEqual(result.terms, termQuery2.getTerm());
 
@@ -515,7 +454,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         builder.add(unsupportedQuery, BooleanClause.Occur.MUST);
         builder.add(unsupportedQuery, BooleanClause.Occur.MUST);
         BooleanQuery bq2 = builder.build();
-        UnsupportedQueryException e = expectThrows(UnsupportedQueryException.class, () -> extractQueryTerms(bq2));
+        UnsupportedQueryException e = expectThrows(UnsupportedQueryException.class, () -> analyze(bq2));
         assertThat(e.getUnsupportedQuery(), sameInstance(unsupportedQuery));
     }
 
@@ -528,7 +467,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
                 Arrays.asList(termQuery1, termQuery2, termQuery3, termQuery4), 0.1f
         );
 
-        Result result = extractQueryTerms(disjunctionMaxQuery);
+        Result result = analyze(disjunctionMaxQuery);
         assertThat(result.verified, is(true));
         List<Term> terms = new ArrayList<>(result.terms);
         Collections.sort(terms);
@@ -546,7 +485,7 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
                 Arrays.asList(termQuery1, termQuery2, termQuery3, new PhraseQuery("_field", "_term4")), 0.1f
         );
 
-        result = extractQueryTerms(disjunctionMaxQuery);
+        result = analyze(disjunctionMaxQuery);
         assertThat(result.verified, is(false));
         terms = new ArrayList<>(result.terms);
         Collections.sort(terms);
@@ -563,12 +502,12 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
 
     public void testSynonymQuery() {
         SynonymQuery query = new SynonymQuery();
-        Result result = extractQueryTerms(query);
+        Result result = analyze(query);
         assertThat(result.verified, is(true));
         assertThat(result.terms.isEmpty(), is(true));
 
         query = new SynonymQuery(new Term("_field", "_value1"), new Term("_field", "_value2"));
-        result = extractQueryTerms(query);
+        result = analyze(query);
         assertThat(result.verified, is(true));
         assertTermsEqual(result.terms, new Term("_field", "_value1"), new Term("_field", "_value2"));
     }
@@ -576,47 +515,16 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
     public void testFunctionScoreQuery() {
         TermQuery termQuery = new TermQuery(new Term("_field", "_value"));
         FunctionScoreQuery functionScoreQuery = new FunctionScoreQuery(termQuery, new RandomScoreFunction());
-        Result result = extractQueryTerms(functionScoreQuery);
+        Result result = analyze(functionScoreQuery);
         assertThat(result.verified, is(true));
         assertTermsEqual(result.terms, new Term("_field", "_value"));
 
         functionScoreQuery = new FunctionScoreQuery(termQuery, new RandomScoreFunction(), 1f, null, 10f);
-        result = extractQueryTerms(functionScoreQuery);
+        result = analyze(functionScoreQuery);
         assertThat(result.verified, is(false));
         assertTermsEqual(result.terms, new Term("_field", "_value"));
     }
 
-    public void testCreateQueryMetadataQuery() throws Exception {
-        MemoryIndex memoryIndex = new MemoryIndex(false);
-        memoryIndex.addField("field1", "the quick brown fox jumps over the lazy dog", new WhitespaceAnalyzer());
-        memoryIndex.addField("field2", "some more text", new WhitespaceAnalyzer());
-        memoryIndex.addField("_field3", "unhide me", new WhitespaceAnalyzer());
-        memoryIndex.addField("field4", "123", new WhitespaceAnalyzer());
-
-        IndexReader indexReader = memoryIndex.createSearcher().getIndexReader();
-        TermsQuery query = (TermsQuery)
-                createQueryTermsQuery(indexReader, QUERY_TERMS_FIELD, new Term(EXTRACTION_RESULT_FIELD, EXTRACTION_FAILED));
-
-        PrefixCodedTerms terms = query.getTermData();
-        assertThat(terms.size(), equalTo(15L));
-        PrefixCodedTerms.TermIterator termIterator = terms.iterator();
-        assertTermIterator(termIterator, "_field3\u0000me", QUERY_TERMS_FIELD);
-        assertTermIterator(termIterator, "_field3\u0000unhide", QUERY_TERMS_FIELD);
-        assertTermIterator(termIterator, "field1\u0000brown", QUERY_TERMS_FIELD);
-        assertTermIterator(termIterator, "field1\u0000dog", QUERY_TERMS_FIELD);
-        assertTermIterator(termIterator, "field1\u0000fox", QUERY_TERMS_FIELD);
-        assertTermIterator(termIterator, "field1\u0000jumps", QUERY_TERMS_FIELD);
-        assertTermIterator(termIterator, "field1\u0000lazy", QUERY_TERMS_FIELD);
-        assertTermIterator(termIterator, "field1\u0000over", QUERY_TERMS_FIELD);
-        assertTermIterator(termIterator, "field1\u0000quick", QUERY_TERMS_FIELD);
-        assertTermIterator(termIterator, "field1\u0000the", QUERY_TERMS_FIELD);
-        assertTermIterator(termIterator, "field2\u0000more", QUERY_TERMS_FIELD);
-        assertTermIterator(termIterator, "field2\u0000some", QUERY_TERMS_FIELD);
-        assertTermIterator(termIterator, "field2\u0000text", QUERY_TERMS_FIELD);
-        assertTermIterator(termIterator, "field4\u0000123", QUERY_TERMS_FIELD);
-        assertTermIterator(termIterator, EXTRACTION_FAILED, EXTRACTION_RESULT_FIELD);
-    }
-
     public void testSelectTermsListWithHighestSumOfTermLength() {
         Set<Term> terms1 = new HashSet<>();
         int shortestTerms1Length = Integer.MAX_VALUE;
@@ -643,11 +551,6 @@ public class ExtractQueryTermsServiceTests extends ESTestCase {
         assertThat(result, sameInstance(expected));
     }
 
-    private void assertTermIterator(PrefixCodedTerms.TermIterator termIterator, String expectedValue, String expectedField) {
-        assertThat(termIterator.next().utf8ToString(), equalTo(expectedValue));
-        assertThat(termIterator.field(), equalTo(expectedField));
-    }
-
     private static void assertTermsEqual(Set<Term> actual, Term... expected) {
         assertEquals(new HashSet<>(Arrays.asList(expected)), actual);
     }

From 8992a5f0f516244bf3a3b622c7861e0c3cb7b6b7 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 8 Jul 2016 10:12:03 -0400
Subject: [PATCH 1294/1311] Add note to issue template regarding supported OS

This commit adds a note to the GitHub issue template noting that bug
reports on OS that we do not support or feature requests for OS that we
do not support will be closed.

Relates #19322
---
 .github/ISSUE_TEMPLATE.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 5ea58046f49..6e784376261 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -3,7 +3,11 @@ GitHub is reserved for bug reports and feature requests. The best place
 to ask a general question is at the Elastic Discourse forums at
 https://discuss.elastic.co. If you are in fact posting a bug report or
 a feature request, please include one and only one of the below blocks
-in your new issue.
+in your new issue. Note that whether you're filing a bug report or a
+feature request, ensure that your submission is for an
+[OS that we support](https://www.elastic.co/support/matrix#show_os).
+Bug reports on OS that we do not support or feature requests specific
+to OS that we do not support will be closed.
 -->
 
 <!--

From 527980c995aed95c7fffa04c09d7a03d0683da33 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 8 Jul 2016 10:43:13 -0400
Subject: [PATCH 1295/1311] Fix nesting of stopping docs

This commit fixes errant nesting of the stopping docs due to using a
section header instead of a chapter header at the top of the stopping
docs.
---
 docs/reference/setup/stopping.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/setup/stopping.asciidoc b/docs/reference/setup/stopping.asciidoc
index 1aa281f050c..45a3b122a90 100644
--- a/docs/reference/setup/stopping.asciidoc
+++ b/docs/reference/setup/stopping.asciidoc
@@ -1,5 +1,5 @@
 [[stopping-elasticsearch]]
-=== Stopping Elasticsearch
+== Stopping Elasticsearch
 
 An orderly shutdown of Elasticsearch ensures that Elasticsearch has a chance to cleanup and close
 outstanding resources. For example, a node that is shutdown in an orderly fashion will remove itself

From 72fa345f5e3e14789c0b424cf2925af6dbfe393e Mon Sep 17 00:00:00 2001
From: Nicholas Knize <nknize@gmail.com>
Date: Fri, 8 Jul 2016 10:01:37 -0500
Subject: [PATCH 1296/1311] Mute GeoDistanceIT while fixing

---
 .../test/java/org/elasticsearch/search/sort/GeoDistanceIT.java   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java
index 2d8944c7963..346e2faa68a 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java
@@ -575,6 +575,7 @@ public class GeoDistanceIT extends ESIntegTestCase {
         return randomDouble() * 180 - 90;
     }
 
+    @AwaitsFix(bugUrl = "http://github.com/elastic/elasticsearch/issues/19263")
     public void testDuelOptimizations() throws Exception {
         Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();

From 982e01d463a4ee6165bca916af2cfd8b195b6dc1 Mon Sep 17 00:00:00 2001
From: Clinton Gormley <clint@traveljury.com>
Date: Fri, 8 Jul 2016 17:13:10 +0200
Subject: [PATCH 1297/1311] Update network.asciidoc

`network.publish_host` defaults to `network.host`, not `network.bind_host`

Closes #19304
---
 docs/reference/modules/network.asciidoc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/reference/modules/network.asciidoc b/docs/reference/modules/network.asciidoc
index 6657e98869f..d6118246b50 100644
--- a/docs/reference/modules/network.asciidoc
+++ b/docs/reference/modules/network.asciidoc
@@ -115,7 +115,7 @@ The publish host is the single interface that the node advertises to other
 nodes in the cluster, so that those nodes can connect to it.   Currently an
 elasticsearch node may be bound to multiple addresses, but only publishes one.
 If not specified, this defaults to the ``best'' address from
-`network.bind_host`, sorted by IPv4/IPv6 stack preference, then by
+`network.host`, sorted by IPv4/IPv6 stack preference, then by
 reachability.
 
 Both of the above settings can be configured just like `network.host` -- they

From 7dff8fbb1d0b7af05ca6007ca031f6e5fbdfa737 Mon Sep 17 00:00:00 2001
From: Yannick Welsch <yannick@welsch.lu>
Date: Fri, 8 Jul 2016 17:30:46 +0200
Subject: [PATCH 1298/1311] Update resiliency docs (#19303)

Adds clarifications about Jepsen tests and new section on issues with versioning.
---
 docs/resiliency/index.asciidoc | 33 ++++++++++++++++++++++++---------
 1 file changed, 24 insertions(+), 9 deletions(-)

diff --git a/docs/resiliency/index.asciidoc b/docs/resiliency/index.asciidoc
index 802c380b4a7..9f8a22c7337 100644
--- a/docs/resiliency/index.asciidoc
+++ b/docs/resiliency/index.asciidoc
@@ -59,9 +59,9 @@ We are committed to tracking down and fixing all the issues that are posted.
 ==== Jepsen Tests
 
 The Jepsen platform is specifically designed to test distributed systems. It is not a single test and is regularly adapted
-to create new scenarios. We have ported all published scenarios to our testing infrastructure. Of course
-as the system evolves, new scenarios can come up that are not yet covered. We are committed to investigating all new scenarios and will
-report issues that we find on this page and in our GitHub repository.
+to create new scenarios. We have currently ported all published Jepsen scenarios that deal with loss of acknowledged writes to our testing
+framework. As the Jepsen tests evolve, we will continue porting new scenarios that are not covered yet. We are committed to investigating
+all new scenarios and will report issues that we find on this page and in our GitHub repository.
 
 [float]
 === Better request retry mechanism when nodes are disconnected (STATUS: ONGOING)
@@ -102,6 +102,19 @@ space.  The following issues have been identified:
 
 Other safeguards are tracked in the meta-issue {GIT}11511[#11511].
 
+[float]
+=== The _version field may not uniquely identify document content during a network partition (STATUS: ONGOING)
+
+When a primary has been partitioned away from the cluster there is a short period of time until it detects this. During that time it will continue
+indexing writes locally, thereby updating document versions. When it tries to replicate the operation, however, it will discover that it is
+partitioned away. It won't acknowledge the write and will wait until the partition is resolved to negotiate with the master on how to proceed.
+The master will decide to either fail any replicas which failed to index the operations on the primary or tell the primary that it has to
+step down because a new primary has been chosen in the meantime. Since the old primary has already written documents, clients may already have read from
+the old primary before it shuts itself down. The version numbers of these reads may not be unique if the new primary has already accepted
+writes for the same document (see {GIT}19269[#19269]).
+
+We are currently implementing Sequence numbers {GIT}10708[#10708] which better track primary changes. Sequence numbers thus provide a basis
+for uniquely identifying writes even in the presence of network partitions and will replace `_version` in operations that require this.
 
 [float]
 === Relocating shards omitted by reporting infrastructure (STATUS: ONGOING)
@@ -119,20 +132,22 @@ in the case of each type of failure. The plan is to have a test case that valida
 [float]
 === Run Jepsen (STATUS: ONGOING)
 
-We have ported all of the known scenarios in the Jepsen blogs to our testing infrastructure. The new tests are run continuously in our
-testing farm and are passing. We are also working on running Jepsen independently to verify that no failures are found.
+We have ported the known scenarios in the Jepsen blogs that check loss of acknowledged writes to our testing infrastructure.
+The new tests are run continuously in our testing farm and are passing. We are also working on running Jepsen independently to verify
+that no failures are found.
 
 
 == Unreleased
 
 [float]
-=== Port Jepsen tests to our testing framework (STATUS: UNRELEASED, V5.0.0)
+=== Port Jepsen tests dealing with loss of acknowledged writes to our testing framework (STATUS: UNRELEASED, V5.0.0)
 
-We have increased our test coverage to include scenarios tested by Jepsen, as described in the Elasticsearch related blogs. We make heavy
-use of randomization to expand on the scenarios that can be tested and to introduce new error conditions.
+We have increased our test coverage to include scenarios tested by Jepsen that demonstrate loss of acknowledged writes, as described in
+the Elasticsearch related blogs. We make heavy use of randomization to expand on the scenarios that can be tested and to introduce
+new error conditions.
 You can follow the work on the master branch of the
 https://github.com/elastic/elasticsearch/blob/master/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java[`DiscoveryWithServiceDisruptionsIT` class],
-where the `testAckedIndexing` test was specifically added to cover known Jepsen related scenarios.
+where the `testAckedIndexing` test was specifically added to check that we don't lose acknowledged writes in various failure scenarios.
 
 
 [float]

From 8dd4a6473ef2a0e124161556ef28eb8c98fd434e Mon Sep 17 00:00:00 2001
From: Nicholas Knize <nknize@gmail.com>
Date: Fri, 8 Jul 2016 12:13:22 -0500
Subject: [PATCH 1299/1311] Remove radial restriction for GeoDistanceQuery

As of lucene 6.1 GeoDistanceQuery no longer requires restricting the radial distance in GeoPointDistanceQuery.

closes #17578
---
 .../elasticsearch/index/query/GeoDistanceQueryBuilder.java   | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java
index 710cb529c07..2f800e9d61a 100644
--- a/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java
+++ b/core/src/main/java/org/elasticsearch/index/query/GeoDistanceQueryBuilder.java
@@ -314,7 +314,10 @@ public class GeoDistanceQueryBuilder extends AbstractQueryBuilder<GeoDistanceQue
         // if index created V_2_3 > use prefix encoded postings format
         final GeoPointField.TermEncoding encoding = (indexVersionCreated.before(Version.V_2_3_0)) ?
             GeoPointField.TermEncoding.NUMERIC : GeoPointField.TermEncoding.PREFIX;
-        normDistance = GeoUtils.maxRadialDistance(center, normDistance);
+        // Lucene 6.0 and earlier requires a radial restriction
+        if (indexVersionCreated.before(Version.V_5_0_0_alpha4)) {
+            normDistance = GeoUtils.maxRadialDistance(center, normDistance);
+        }
         return new GeoPointDistanceQuery(fieldType.name(), encoding, center.lat(), center.lon(), normDistance);
     }
 

From 41874de21d06744c8b287987902e647b0e92fa28 Mon Sep 17 00:00:00 2001
From: Jason Tedor <jason@tedor.me>
Date: Fri, 8 Jul 2016 14:33:36 -0400
Subject: [PATCH 1300/1311] Add indefinite articles to issue template

This commit adds an indefinite article when referring to operating
systems in the issue template comments regarding supported operating
systems.
---
 .github/ISSUE_TEMPLATE.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 6e784376261..53e964188f6 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -6,8 +6,8 @@ a feature request, please include one and only one of the below blocks
 in your new issue. Note that whether you're filing a bug report or a
 feature request, ensure that your submission is for an
 [OS that we support](https://www.elastic.co/support/matrix#show_os).
-Bug reports on OS that we do not support or feature requests specific
-to OS that we do not support will be closed.
+Bug reports on an OS that we do not support or feature requests
+specific to an OS that we do not support will be closed.
 -->
 
 <!--

From 53b8a0c63e830c115dae9787124a2186c05719b7 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Fri, 8 Jul 2016 14:27:08 -0700
Subject: [PATCH 1301/1311] Add back tests for setting reader helper in s3 and
 illegal buffer/chunk size

---
 .../repositories/s3/S3RepositoryTests.java    | 107 ++++++++++++++++++
 1 file changed, 107 insertions(+)
 create mode 100644 plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java

diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java
new file mode 100644
index 00000000000..15b2e8642cb
--- /dev/null
+++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3RepositoryTests.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import java.io.IOException;
+
+import com.amazonaws.Protocol;
+import com.amazonaws.services.s3.AbstractAmazonS3;
+import com.amazonaws.services.s3.AmazonS3;
+import org.elasticsearch.cloud.aws.AwsS3Service;
+import org.elasticsearch.cluster.metadata.RepositoryMetaData;
+import org.elasticsearch.common.component.AbstractLifecycleComponent;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.repositories.RepositoryException;
+import org.elasticsearch.test.ESTestCase;
+
+import static org.elasticsearch.repositories.s3.S3Repository.Repositories;
+import static org.elasticsearch.repositories.s3.S3Repository.Repository;
+import static org.elasticsearch.repositories.s3.S3Repository.getValue;
+import static org.hamcrest.Matchers.containsString;
+
+public class S3RepositoryTests extends ESTestCase {
+
+    private static class DummyS3Client extends AbstractAmazonS3 {
+        @Override
+        public boolean doesBucketExist(String bucketName) {
+            return true;
+        }
+    }
+
+    private static class DummyS3Service extends AbstractLifecycleComponent implements AwsS3Service {
+        public DummyS3Service() {
+            super(Settings.EMPTY);
+        }
+        @Override
+        protected void doStart() {}
+        @Override
+        protected void doStop() {}
+        @Override
+        protected void doClose() {}
+        @Override
+        public AmazonS3 client(String endpoint, Protocol protocol, String region, String account, String key,
+                               Integer maxRetries, boolean useThrottleRetries) {
+            return new DummyS3Client();
+        }
+    }
+
+    public void testSettingsResolution() throws Exception {
+        Settings localSettings = Settings.builder().put(Repository.KEY_SETTING.getKey(), "key1").build();
+        Settings globalSettings = Settings.builder().put(Repositories.KEY_SETTING.getKey(), "key2").build();
+
+        assertEquals("key1", getValue(localSettings, globalSettings, Repository.KEY_SETTING, Repositories.KEY_SETTING));
+        assertEquals("key1", getValue(localSettings, Settings.EMPTY, Repository.KEY_SETTING, Repositories.KEY_SETTING));
+        assertEquals("key2", getValue(Settings.EMPTY, globalSettings, Repository.KEY_SETTING, Repositories.KEY_SETTING));
+        assertEquals("", getValue(Settings.EMPTY, Settings.EMPTY, Repository.KEY_SETTING, Repositories.KEY_SETTING));
+    }
+
+    public void testInvalidChunkBufferSizeSettings() throws IOException {
+        // chunk < buffer should fail
+        assertInvalidBuffer(10, 5, RepositoryException.class, "chunk_size (5mb) can't be lower than buffer_size (10mb).");
+        // chunk > buffer should pass
+        assertValidBuffer(5, 10);
+        // chunk = buffer should pass
+        assertValidBuffer(5, 5);
+        // buffer < 5mb should fail
+        assertInvalidBuffer(4, 10, IllegalArgumentException.class,
+            "Failed to parse value [4mb] for setting [buffer_size] must be >= 5mb");
+        // chunk > 5tb should fail
+        assertInvalidBuffer(5, 6000000, IllegalArgumentException.class,
+            "Failed to parse value [5.7tb] for setting [chunk_size] must be <= 5tb");
+    }
+
+    private void assertValidBuffer(long bufferMB, long chunkMB) throws IOException {
+        RepositoryMetaData metadata = new RepositoryMetaData("dummy-repo", "mock", Settings.builder()
+            .put(Repository.BUFFER_SIZE_SETTING.getKey(), new ByteSizeValue(bufferMB, ByteSizeUnit.MB))
+            .put(Repository.CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(chunkMB, ByteSizeUnit.MB)).build());
+        new S3Repository(metadata, Settings.EMPTY, new DummyS3Service());
+    }
+
+    private void assertInvalidBuffer(int bufferMB, int chunkMB, Class<? extends Exception> clazz, String msg) throws IOException {
+        RepositoryMetaData metadata = new RepositoryMetaData("dummy-repo", "mock", Settings.builder()
+            .put(Repository.BUFFER_SIZE_SETTING.getKey(), new ByteSizeValue(bufferMB, ByteSizeUnit.MB))
+            .put(Repository.CHUNK_SIZE_SETTING.getKey(), new ByteSizeValue(chunkMB, ByteSizeUnit.MB)).build());
+
+        Exception e = expectThrows(clazz, () -> new S3Repository(metadata, Settings.EMPTY, new DummyS3Service()));
+        assertThat(e.getMessage(), containsString(msg));
+    }
+}

From 7e59181e58c61158ac5a933a4624b605142e0fe5 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Fri, 8 Jul 2016 14:57:44 -0700
Subject: [PATCH 1302/1311] Internal: Remove child injectors from guice

This change removes the ability for guice to have child injectors (and
the entire concept of parent injectors) from our fork of guice. The
methodology for removing was simple: I removed createChildInjector, and
continued to remove methods and members that were unused until my head
was spinning. The motivation for this change is to limit what our fork
of guice gives us access to, so we don't regress and start adding back
more complicated uses.
---
 .../elasticsearch/common/inject/Injector.java |  82 -----
 .../common/inject/InjectorBuilder.java        | 102 -------
 .../common/inject/InjectorImpl.java           |  73 +----
 .../common/inject/InjectorShell.java          |  12 +-
 .../common/inject/Injectors.java              | 239 ---------------
 .../common/inject/ModulesBuilder.java         |  14 +-
 .../common/inject/PrivateModule.java          |  69 -----
 .../assistedinject/FactoryProvider.java       | 111 +------
 .../assistedinject/FactoryProvider2.java      | 283 ------------------
 .../inject/assistedinject/Parameter.java      |  28 --
 .../inject/spi/ConvertedConstantBinding.java  |   3 +-
 .../common/inject/spi/Element.java            |   3 -
 .../common/inject/spi/Elements.java           |  12 -
 .../common/inject/spi/LinkedKeyBinding.java   |   3 +-
 .../common/inject/spi/ProviderBinding.java    |   4 +-
 .../common/inject/spi/ProviderKeyBinding.java |   4 +-
 16 files changed, 15 insertions(+), 1027 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/common/inject/Injectors.java
 delete mode 100644 core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider2.java

diff --git a/core/src/main/java/org/elasticsearch/common/inject/Injector.java b/core/src/main/java/org/elasticsearch/common/inject/Injector.java
index bfc1fb42db3..2417e3b1411 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/Injector.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/Injector.java
@@ -38,12 +38,6 @@ import java.util.Map;
  * An injector can also {@link #injectMembers(Object) inject the dependencies} of
  * already-constructed instances. This can be used to interoperate with objects created by other
  * frameworks or services.
- * <p>
- * Injectors can be {@link #createChildInjector(Iterable) hierarchical}. Child injectors inherit
- * the configuration of their parent injectors, but the converse does not hold.
- * <p>
- * The injector's {@link #getBindings() internal bindings} are available for introspection. This
- * enables tools and extensions to operate on an injector reflectively.
  *
  * @author crazybob@google.com (Bob Lee)
  * @author jessewilson@google.com (Jesse Wilson)
@@ -87,41 +81,6 @@ public interface Injector {
      */
     <T> MembersInjector<T> getMembersInjector(Class<T> type);
 
-    /**
-     * Returns all explicit bindings.
-     * <p>
-     * The returned map does not include bindings inherited from a {@link #getParent() parent
-     * injector}, should one exist. The returned map is guaranteed to iterate (for example, with
-     * its {@link java.util.Map#entrySet()} iterator) in the order of insertion. In other words,
-     * the order in which bindings appear in user Modules.
-     * <p>
-     * This method is part of the Guice SPI and is intended for use by tools and extensions.
-     */
-    Map<Key<?>, Binding<?>> getBindings();
-
-    /**
-     * Returns the binding for the given injection key. This will be an explicit bindings if the key
-     * was bound explicitly by a module, or an implicit binding otherwise. The implicit binding will
-     * be created if necessary.
-     * <p>
-     * This method is part of the Guice SPI and is intended for use by tools and extensions.
-     *
-     * @throws ConfigurationException if this injector cannot find or create the binding.
-     */
-    <T> Binding<T> getBinding(Key<T> key);
-
-    /**
-     * Returns the binding for the given type. This will be an explicit bindings if the injection key
-     * was bound explicitly by a module, or an implicit binding otherwise. The implicit binding will
-     * be created if necessary.
-     * <p>
-     * This method is part of the Guice SPI and is intended for use by tools and extensions.
-     *
-     * @throws ConfigurationException if this injector cannot find or create the binding.
-     * @since 2.0
-     */
-    <T> Binding<T> getBinding(Class<T> type);
-
     /**
      * Returns all explicit bindings for {@code type}.
      * <p>
@@ -166,45 +125,4 @@ public interface Injector {
      * @throws ProvisionException     if there was a runtime failure while providing an instance.
      */
     <T> T getInstance(Class<T> type);
-
-    /**
-     * Returns this injector's parent, or {@code null} if this is a top-level injector.
-     *
-     * @since 2.0
-     */
-    Injector getParent();
-
-    /**
-     * Returns a new injector that inherits all state from this injector. All bindings, scopes,
-     * interceptors and type converters are inherited -- they are visible to the child injector.
-     * Elements of the child injector are not visible to its parent.
-     * <p>
-     * Just-in-time bindings created for child injectors will be created in an ancestor injector
-     * whenever possible. This allows for scoped instances to be shared between injectors. Use
-     * explicit bindings to prevent bindings from being shared with the parent injector.
-     * <p>
-     * No key may be bound by both an injector and one of its ancestors. This includes just-in-time
-     * bindings. The lone exception is the key for {@code Injector.class}, which is bound by each
-     * injector to itself.
-     *
-     * @since 2.0
-     */
-    Injector createChildInjector(Iterable<? extends Module> modules);
-
-    /**
-     * Returns a new injector that inherits all state from this injector. All bindings, scopes,
-     * interceptors and type converters are inherited -- they are visible to the child injector.
-     * Elements of the child injector are not visible to its parent.
-     * <p>
-     * Just-in-time bindings created for child injectors will be created in an ancestor injector
-     * whenever possible. This allows for scoped instances to be shared between injectors. Use
-     * explicit bindings to prevent bindings from being shared with the parent injector.
-     * <p>
-     * No key may be bound by both an injector and one of its ancestors. This includes just-in-time
-     * bindings. The lone exception is the key for {@code Injector.class}, which is bound by each
-     * injector to itself.
-     *
-     * @since 2.0
-     */
-    Injector createChildInjector(Module... modules);
 }
diff --git a/core/src/main/java/org/elasticsearch/common/inject/InjectorBuilder.java b/core/src/main/java/org/elasticsearch/common/inject/InjectorBuilder.java
index 2f9a3f1e7cb..93c92d480e2 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/InjectorBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/InjectorBuilder.java
@@ -74,15 +74,6 @@ class InjectorBuilder {
         return this;
     }
 
-    /**
-     * Sets the parent of the injector to-be-constructed. As a side effect, this sets this injector's
-     * stage to the stage of {@code parent}.
-     */
-    InjectorBuilder parentInjector(InjectorImpl parent) {
-        shellBuilder.parent(parent);
-        return stage(parent.getInstance(Stage.class));
-    }
-
     InjectorBuilder addModules(Iterable<? extends Module> modules) {
         shellBuilder.addModules(modules);
         return this;
@@ -102,11 +93,6 @@ class InjectorBuilder {
             initializeStatically();
         }
 
-        // If we're in the tool stage, stop here. Don't eagerly inject or load anything.
-        if (stage == Stage.TOOL) {
-            return new ToolStageInjector(primaryInjector());
-        }
-
         injectDynamically();
 
         return primaryInjector();
@@ -217,92 +203,4 @@ class InjectorBuilder {
             }
         }
     }
-
-    /**
-     * {@link Injector} exposed to users in {@link Stage#TOOL}.
-     */
-    static class ToolStageInjector implements Injector {
-        private final Injector delegateInjector;
-
-        ToolStageInjector(Injector delegateInjector) {
-            this.delegateInjector = delegateInjector;
-        }
-
-        @Override
-        public void injectMembers(Object o) {
-            throw new UnsupportedOperationException(
-                    "Injector.injectMembers(Object) is not supported in Stage.TOOL");
-        }
-
-        @Override
-        public Map<Key<?>, Binding<?>> getBindings() {
-            return this.delegateInjector.getBindings();
-        }
-
-        @Override
-        public <T> Binding<T> getBinding(Key<T> key) {
-            return this.delegateInjector.getBinding(key);
-        }
-
-        @Override
-        public <T> Binding<T> getBinding(Class<T> type) {
-            return this.delegateInjector.getBinding(type);
-        }
-
-        @Override
-        public <T> List<Binding<T>> findBindingsByType(TypeLiteral<T> type) {
-            return this.delegateInjector.findBindingsByType(type);
-        }
-
-        @Override
-        public Injector getParent() {
-            return delegateInjector.getParent();
-        }
-
-        @Override
-        public Injector createChildInjector(Iterable<? extends Module> modules) {
-            return delegateInjector.createChildInjector(modules);
-        }
-
-        @Override
-        public Injector createChildInjector(Module... modules) {
-            return delegateInjector.createChildInjector(modules);
-        }
-
-        @Override
-        public <T> Provider<T> getProvider(Key<T> key) {
-            throw new UnsupportedOperationException(
-                    "Injector.getProvider(Key<T>) is not supported in Stage.TOOL");
-        }
-
-        @Override
-        public <T> Provider<T> getProvider(Class<T> type) {
-            throw new UnsupportedOperationException(
-                    "Injector.getProvider(Class<T>) is not supported in Stage.TOOL");
-        }
-
-        @Override
-        public <T> MembersInjector<T> getMembersInjector(TypeLiteral<T> typeLiteral) {
-            throw new UnsupportedOperationException(
-                    "Injector.getMembersInjector(TypeLiteral<T>) is not supported in Stage.TOOL");
-        }
-
-        @Override
-        public <T> MembersInjector<T> getMembersInjector(Class<T> type) {
-            throw new UnsupportedOperationException(
-                    "Injector.getMembersInjector(Class<T>) is not supported in Stage.TOOL");
-        }
-
-        @Override
-        public <T> T getInstance(Key<T> key) {
-            throw new UnsupportedOperationException(
-                    "Injector.getInstance(Key<T>) is not supported in Stage.TOOL");
-        }
-
-        @Override
-        public <T> T getInstance(Class<T> type) {
-            throw new UnsupportedOperationException(
-                    "Injector.getInstance(Class<T>) is not supported in Stage.TOOL");
-        }
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/inject/InjectorImpl.java b/core/src/main/java/org/elasticsearch/common/inject/InjectorImpl.java
index ae4cd744dd0..817d52ea743 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/InjectorImpl.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/InjectorImpl.java
@@ -64,7 +64,6 @@ import static org.elasticsearch.common.inject.internal.Annotations.findScopeAnno
  */
 class InjectorImpl implements Injector, Lookups {
     final State state;
-    final InjectorImpl parent;
     boolean readOnly;
     BindingsMultimap bindingsMultimap = new BindingsMultimap();
     final Initializer initializer;
@@ -76,16 +75,10 @@ class InjectorImpl implements Injector, Lookups {
 
     Lookups lookups = new DeferredLookups(this);
 
-    InjectorImpl(@Nullable InjectorImpl parent, State state, Initializer initializer) {
-        this.parent = parent;
+    InjectorImpl(State state, Initializer initializer) {
         this.state = state;
         this.initializer = initializer;
-
-        if (parent != null) {
-            localContext = parent.localContext;
-        } else {
-            localContext = new ThreadLocal<>();
-        }
+        localContext = new ThreadLocal<>();
     }
 
     /**
@@ -106,21 +99,6 @@ class InjectorImpl implements Injector, Lookups {
         return bindingsMultimap.getAll(type);
     }
 
-    /**
-     * Returns the binding for {@code key}
-     */
-    @Override
-    public <T> BindingImpl<T> getBinding(Key<T> key) {
-        Errors errors = new Errors(key);
-        try {
-            BindingImpl<T> result = getBindingOrThrow(key, errors);
-            errors.throwConfigurationExceptionIfErrorsExist();
-            return result;
-        } catch (ErrorsException e) {
-            throw new ConfigurationException(errors.merge(e.getErrors()).getMessages());
-        }
-    }
-
     /**
      * Gets a binding implementation.  First, it check to see if the parent has a binding.  If the
      * parent has a binding and the binding is scoped, it will use that binding.  Otherwise, this
@@ -139,29 +117,6 @@ class InjectorImpl implements Injector, Lookups {
         return getJustInTimeBinding(key, errors);
     }
 
-    @Override
-    public <T> Binding<T> getBinding(Class<T> type) {
-        return getBinding(Key.get(type));
-    }
-
-    @Override
-    public Injector getParent() {
-        return parent;
-    }
-
-    @Override
-    public Injector createChildInjector(Iterable<? extends Module> modules) {
-        return new InjectorBuilder()
-                .parentInjector(this)
-                .addModules(modules)
-                .build();
-    }
-
-    @Override
-    public Injector createChildInjector(Module... modules) {
-        return createChildInjector(Arrays.asList(modules));
-    }
-
     /**
      * Returns a just-in-time binding for {@code key}, creating it if necessary.
      *
@@ -171,13 +126,11 @@ class InjectorImpl implements Injector, Lookups {
             throws ErrorsException {
         synchronized (state.lock()) {
             // first try to find a JIT binding that we've already created
-            for (InjectorImpl injector = this; injector != null; injector = injector.parent) {
-                @SuppressWarnings("unchecked") // we only store bindings that match their key
-                        BindingImpl<T> binding = (BindingImpl<T>) injector.jitBindings.get(key);
+            @SuppressWarnings("unchecked") // we only store bindings that match their key
+            BindingImpl<T> binding = (BindingImpl<T>) jitBindings.get(key);
 
-                if (binding != null) {
-                    return binding;
-                }
+            if (binding != null) {
+                return binding;
             }
 
             return createJustInTimeBindingRecursive(key, errors);
@@ -600,14 +553,6 @@ class InjectorImpl implements Injector, Lookups {
      */
     private <T> BindingImpl<T> createJustInTimeBindingRecursive(Key<T> key, Errors errors)
             throws ErrorsException {
-        // ask the parent to create the JIT binding
-        if (parent != null && !parent.readOnly /* ES: don't check on parent if its read only, its already created all the bindings it can*/) {
-            try {
-                return parent.createJustInTimeBindingRecursive(key, new Errors());
-            } catch (ErrorsException ignored) {
-            }
-        }
-
         if (state.isBlacklisted(key)) {
             throw errors.childBindingAlreadySet(key).toException();
         }
@@ -686,12 +631,6 @@ class InjectorImpl implements Injector, Lookups {
         return getBindingOrThrow(key, errors).getInternalFactory();
     }
 
-    // not test-covered
-    @Override
-    public Map<Key<?>, Binding<?>> getBindings() {
-        return state.getExplicitBindingsThisLevel();
-    }
-
     private static class BindingsMultimap {
         final Map<TypeLiteral<?>, List<Binding<?>>> multimap = new HashMap<>();
 
diff --git a/core/src/main/java/org/elasticsearch/common/inject/InjectorShell.java b/core/src/main/java/org/elasticsearch/common/inject/InjectorShell.java
index 7307fb4a6fe..793540aba73 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/InjectorShell.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/InjectorShell.java
@@ -50,18 +50,12 @@ class InjectorShell {
 
     private final List<Element> elements;
     private final InjectorImpl injector;
-    private final PrivateElements privateElements;
 
-    private InjectorShell(Builder builder, List<Element> elements, InjectorImpl injector) {
-        this.privateElements = builder.privateElements;
+    private InjectorShell(List<Element> elements, InjectorImpl injector) {
         this.elements = elements;
         this.injector = injector;
     }
 
-    PrivateElements getPrivateElements() {
-        return privateElements;
-    }
-
     InjectorImpl getInjector() {
         return injector;
     }
@@ -134,7 +128,7 @@ class InjectorShell {
                 throw new IllegalStateException("no state. Did you remember to lock() ?");
             }
 
-            InjectorImpl injector = new InjectorImpl(parent, state, initializer);
+            InjectorImpl injector = new InjectorImpl(state, initializer);
             if (privateElements != null) {
                 privateElements.initInjector(injector);
             }
@@ -167,7 +161,7 @@ class InjectorShell {
             stopwatch.resetAndLog("Binding creation");
 
             List<InjectorShell> injectorShells = new ArrayList<>();
-            injectorShells.add(new InjectorShell(this, elements, injector));
+            injectorShells.add(new InjectorShell(elements, injector));
 
             // recursively build child shells
             PrivateElementProcessor processor = new PrivateElementProcessor(errors, stage);
diff --git a/core/src/main/java/org/elasticsearch/common/inject/Injectors.java b/core/src/main/java/org/elasticsearch/common/inject/Injectors.java
deleted file mode 100644
index 900d2588b52..00000000000
--- a/core/src/main/java/org/elasticsearch/common/inject/Injectors.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.common.inject;
-
-import org.elasticsearch.common.inject.matcher.Matcher;
-import org.elasticsearch.common.inject.name.Names;
-import org.elasticsearch.common.inject.spi.Message;
-
-import java.lang.reflect.Type;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-/**
- *
- */
-public class Injectors {
-
-    public static Throwable getFirstErrorFailure(CreationException e) {
-        if (e.getErrorMessages().isEmpty()) {
-            return e;
-        }
-        // return the first message that has root cause, probably an actual error
-        for (Message message : e.getErrorMessages()) {
-            if (message.getCause() != null) {
-                return message.getCause();
-            }
-        }
-        return e;
-    }
-
-    /**
-     * Returns an instance of the given type with the {@link org.elasticsearch.common.inject.name.Named}
-     * annotation value.
-     * <p>
-     * This method allows you to switch this code
-     * <code>injector.getInstance(Key.get(type, Names.named(name)));</code>
-     * <p>
-     * to the more concise
-     * <code>Injectors.getInstance(injector, type, name);</code>
-     */
-    public static <T> T getInstance(Injector injector, java.lang.Class<T> type, String name) {
-        return injector.getInstance(Key.get(type, Names.named(name)));
-    }
-
-    /**
-     * Returns a collection of all instances of the given base type
-     *
-     * @param baseClass the base type of objects required
-     * @param <T>       the base type
-     * @return a set of objects returned from this injector
-     */
-    public static <T> Set<T> getInstancesOf(Injector injector, Class<T> baseClass) {
-        Set<T> answer = new HashSet<>();
-        Set<Entry<Key<?>, Binding<?>>> entries = injector.getBindings().entrySet();
-        for (Entry<Key<?>, Binding<?>> entry : entries) {
-            Key<?> key = entry.getKey();
-            Class<?> keyType = getKeyType(key);
-            if (keyType != null && baseClass.isAssignableFrom(keyType)) {
-                Binding<?> binding = entry.getValue();
-                Object value = binding.getProvider().get();
-                if (value != null) {
-                    T castValue = baseClass.cast(value);
-                    answer.add(castValue);
-                }
-            }
-        }
-        return answer;
-    }
-
-    /**
-     * Returns a collection of all instances matching the given matcher
-     *
-     * @param matcher matches the types to return instances
-     * @return a set of objects returned from this injector
-     */
-    public static <T> Set<T> getInstancesOf(Injector injector, Matcher<Class> matcher) {
-        Set<T> answer = new HashSet<>();
-        Set<Entry<Key<?>, Binding<?>>> entries = injector.getBindings().entrySet();
-        for (Entry<Key<?>, Binding<?>> entry : entries) {
-            Key<?> key = entry.getKey();
-            Class<?> keyType = getKeyType(key);
-            if (keyType != null && matcher.matches(keyType)) {
-                Binding<?> binding = entry.getValue();
-                Object value = binding.getProvider().get();
-                answer.add((T) value);
-            }
-        }
-        return answer;
-    }
-
-    /**
-     * Returns a collection of all of the providers matching the given matcher
-     *
-     * @param matcher matches the types to return instances
-     * @return a set of objects returned from this injector
-     */
-    public static <T> Set<Provider<T>> getProvidersOf(Injector injector, Matcher<Class> matcher) {
-        Set<Provider<T>> answer = new HashSet<>();
-        Set<Entry<Key<?>, Binding<?>>> entries = injector.getBindings().entrySet();
-        for (Entry<Key<?>, Binding<?>> entry : entries) {
-            Key<?> key = entry.getKey();
-            Class<?> keyType = getKeyType(key);
-            if (keyType != null && matcher.matches(keyType)) {
-                Binding<?> binding = entry.getValue();
-                answer.add((Provider<T>) binding.getProvider());
-            }
-        }
-        return answer;
-    }
-
-    /**
-     * Returns a collection of all providers of the given base type
-     *
-     * @param baseClass the base type of objects required
-     * @param <T>       the base type
-     * @return a set of objects returned from this injector
-     */
-    public static <T> Set<Provider<T>> getProvidersOf(Injector injector, Class<T> baseClass) {
-        Set<Provider<T>> answer = new HashSet<>();
-        Set<Entry<Key<?>, Binding<?>>> entries = injector.getBindings().entrySet();
-        for (Entry<Key<?>, Binding<?>> entry : entries) {
-            Key<?> key = entry.getKey();
-            Class<?> keyType = getKeyType(key);
-            if (keyType != null && baseClass.isAssignableFrom(keyType)) {
-                Binding<?> binding = entry.getValue();
-                answer.add((Provider<T>) binding.getProvider());
-            }
-        }
-        return answer;
-    }
-
-    /**
-     * Returns true if a binding exists for the given matcher
-     */
-    public static boolean hasBinding(Injector injector, Matcher<Class> matcher) {
-        return !getBindingsOf(injector, matcher).isEmpty();
-    }
-
-    /**
-     * Returns true if a binding exists for the given base class
-     */
-    public static boolean hasBinding(Injector injector, Class<?> baseClass) {
-        return !getBindingsOf(injector, baseClass).isEmpty();
-    }
-
-    /**
-     * Returns true if a binding exists for the given key
-     */
-    public static boolean hasBinding(Injector injector, Key<?> key) {
-        Binding<?> binding = getBinding(injector, key);
-        return binding != null;
-    }
-
-    /**
-     * Returns the binding for the given key or null if there is no such binding
-     */
-    public static Binding<?> getBinding(Injector injector, Key<?> key) {
-        Map<Key<?>, Binding<?>> bindings = injector.getBindings();
-        Binding<?> binding = bindings.get(key);
-        return binding;
-    }
-
-    /**
-     * Returns a collection of all of the bindings matching the given matcher
-     *
-     * @param matcher matches the types to return instances
-     * @return a set of objects returned from this injector
-     */
-    public static Set<Binding<?>> getBindingsOf(Injector injector, Matcher<Class> matcher) {
-        Set<Binding<?>> answer = new HashSet<>();
-        Set<Entry<Key<?>, Binding<?>>> entries = injector.getBindings().entrySet();
-        for (Entry<Key<?>, Binding<?>> entry : entries) {
-            Key<?> key = entry.getKey();
-            Class<?> keyType = getKeyType(key);
-            if (keyType != null && matcher.matches(keyType)) {
-                answer.add(entry.getValue());
-            }
-        }
-        return answer;
-    }
-
-    /**
-     * Returns a collection of all bindings of the given base type
-     *
-     * @param baseClass the base type of objects required
-     * @return a set of objects returned from this injector
-     */
-    public static Set<Binding<?>> getBindingsOf(Injector injector, Class<?> baseClass) {
-        Set<Binding<?>> answer = new HashSet<>();
-        Set<Entry<Key<?>, Binding<?>>> entries = injector.getBindings().entrySet();
-        for (Entry<Key<?>, Binding<?>> entry : entries) {
-            Key<?> key = entry.getKey();
-            Class<?> keyType = getKeyType(key);
-            if (keyType != null && baseClass.isAssignableFrom(keyType)) {
-                answer.add(entry.getValue());
-            }
-        }
-        return answer;
-    }
-
-    /**
-     * Returns the key type of the given key
-     */
-    public static <T> Class<?> getKeyType(Key<?> key) {
-        Class<?> keyType = null;
-        TypeLiteral<?> typeLiteral = key.getTypeLiteral();
-        Type type = typeLiteral.getType();
-        if (type instanceof Class) {
-            keyType = (Class<?>) type;
-        }
-        return keyType;
-    }
-
-    public static void cleanCaches(Injector injector) {
-        ((InjectorImpl) injector).clearCache();
-        if (injector.getParent() != null) {
-            cleanCaches(injector.getParent());
-        }
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/common/inject/ModulesBuilder.java b/core/src/main/java/org/elasticsearch/common/inject/ModulesBuilder.java
index 41cdecfbcad..3321b75f4e5 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/ModulesBuilder.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/ModulesBuilder.java
@@ -23,9 +23,6 @@ import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
-/**
- *
- */
 public class ModulesBuilder implements Iterable<Module> {
 
     private final List<Module> modules = new ArrayList<>();
@@ -44,19 +41,10 @@ public class ModulesBuilder implements Iterable<Module> {
 
     public Injector createInjector() {
         Injector injector = Guice.createInjector(modules);
-        Injectors.cleanCaches(injector);
+        ((InjectorImpl) injector).clearCache();
         // in ES, we always create all instances as if they are eager singletons
         // this allows for considerable memory savings (no need to store construction info) as well as cycles
         ((InjectorImpl) injector).readOnlyAllSingletons();
         return injector;
     }
-
-    public Injector createChildInjector(Injector injector) {
-        Injector childInjector = injector.createChildInjector(modules);
-        Injectors.cleanCaches(childInjector);
-        // in ES, we always create all instances as if they are eager singletons
-        // this allows for considerable memory savings (no need to store construction info) as well as cycles
-        ((InjectorImpl) childInjector).readOnlyAllSingletons();
-        return childInjector;
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/inject/PrivateModule.java b/core/src/main/java/org/elasticsearch/common/inject/PrivateModule.java
index 4413028f908..b56c0e11ab1 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/PrivateModule.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/PrivateModule.java
@@ -61,10 +61,6 @@ import java.lang.annotation.Annotation;
  * }
  * </pre>
  * <p>
- * Private modules are implemented using {@link Injector#createChildInjector(Module[]) parent
- * injectors}. When it can satisfy their dependencies, just-in-time bindings will be created in the
- * root environment. Such bindings are shared among all environments in the tree.
- * <p>
  * The scope of a binding is constrained to its environment. A singleton bound in a private
  * module will be unique to its environment. But a binding for the same type in a different private
  * module will yield a different instance.
@@ -143,13 +139,6 @@ public abstract class PrivateModule implements Module {
         return binder;
     }
 
-    /**
-     * @see Binder#bindScope(Class, Scope)
-     */
-    protected final void bindScope(Class<? extends Annotation> scopeAnnotation, Scope scope) {
-        binder.bindScope(scopeAnnotation, scope);
-    }
-
     /**
      * @see Binder#bind(Key)
      */
@@ -171,13 +160,6 @@ public abstract class PrivateModule implements Module {
         return binder.bind(clazz);
     }
 
-    /**
-     * @see Binder#bindConstant()
-     */
-    protected final AnnotatedConstantBindingBuilder bindConstant() {
-        return binder.bindConstant();
-    }
-
     /**
      * @see Binder#install(Module)
      */
@@ -206,34 +188,6 @@ public abstract class PrivateModule implements Module {
         binder.addError(message);
     }
 
-    /**
-     * @see Binder#requestInjection(Object)
-     */
-    protected final void requestInjection(Object instance) {
-        binder.requestInjection(instance);
-    }
-
-    /**
-     * @see Binder#requestStaticInjection(Class[])
-     */
-    protected final void requestStaticInjection(Class<?>... types) {
-        binder.requestStaticInjection(types);
-    }
-
-    /**
-     * Instructs Guice to require a binding to the given key.
-     */
-    protected final void requireBinding(Key<?> key) {
-        binder.getProvider(key);
-    }
-
-    /**
-     * Instructs Guice to require a binding to the given type.
-     */
-    protected final void requireBinding(Class<?> type) {
-        binder.getProvider(type);
-    }
-
     /**
      * @see Binder#getProvider(Key)
      */
@@ -248,21 +202,6 @@ public abstract class PrivateModule implements Module {
         return binder.getProvider(type);
     }
 
-    /**
-     * @see Binder#convertToTypes(org.elasticsearch.common.inject.matcher.Matcher, org.elasticsearch.common.inject.spi.TypeConverter)
-     */
-    protected final void convertToTypes(Matcher<? super TypeLiteral<?>> typeMatcher,
-                                        TypeConverter converter) {
-        binder.convertToTypes(typeMatcher, converter);
-    }
-
-    /**
-     * @see Binder#currentStage()
-     */
-    protected final Stage currentStage() {
-        return binder.currentStage();
-    }
-
     /**
      * @see Binder#getMembersInjector(Class)
      */
@@ -276,12 +215,4 @@ public abstract class PrivateModule implements Module {
     protected <T> MembersInjector<T> getMembersInjector(TypeLiteral<T> type) {
         return binder.getMembersInjector(type);
     }
-
-    /**
-     * @see Binder#bindListener(org.elasticsearch.common.inject.matcher.Matcher, org.elasticsearch.common.inject.spi.TypeListener)
-     */
-    protected void bindListener(Matcher<? super TypeLiteral<?>> typeMatcher,
-                                TypeListener listener) {
-        binder.bindListener(typeMatcher, listener);
-    }
 }
diff --git a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider.java b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider.java
index 0def65b9a13..099ff661595 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider.java
@@ -134,7 +134,7 @@ public class FactoryProvider<F> implements Provider<F>, HasDependencies {
 
     /*
     * This class implements the old @AssistedInject implementation that manually matches constructors
-    * to factory methods. The new child injector implementation lives in FactoryProvider2.
+    * to factory methods.
     */
 
     private Injector injector;
@@ -142,23 +142,6 @@ public class FactoryProvider<F> implements Provider<F>, HasDependencies {
     private final TypeLiteral<F> factoryType;
     private final Map<Method, AssistedConstructor<?>> factoryMethodToConstructor;
 
-    public static <F> Provider<F> newFactory(
-            Class<F> factoryType, Class<?> implementationType) {
-        return newFactory(TypeLiteral.get(factoryType), TypeLiteral.get(implementationType));
-    }
-
-    public static <F> Provider<F> newFactory(
-            TypeLiteral<F> factoryType, TypeLiteral<?> implementationType) {
-        Map<Method, AssistedConstructor<?>> factoryMethodToConstructor
-                = createMethodMapping(factoryType, implementationType);
-
-        if (!factoryMethodToConstructor.isEmpty()) {
-            return new FactoryProvider<>(factoryType, factoryMethodToConstructor);
-        } else {
-            return new FactoryProvider2<>(factoryType, Key.get(implementationType));
-        }
-    }
-
     private FactoryProvider(TypeLiteral<F> factoryType,
                             Map<Method, AssistedConstructor<?>> factoryMethodToConstructor) {
         this.factoryType = factoryType;
@@ -166,22 +149,6 @@ public class FactoryProvider<F> implements Provider<F>, HasDependencies {
         checkDeclaredExceptionsMatch();
     }
 
-    @Inject
-    void setInjectorAndCheckUnboundParametersAreInjectable(Injector injector) {
-        this.injector = injector;
-        for (AssistedConstructor<?> c : factoryMethodToConstructor.values()) {
-            for (Parameter p : c.getAllParameters()) {
-                if (!p.isProvidedByFactory() && !paramCanBeInjected(p, injector)) {
-                    // this is lame - we're not using the proper mechanism to add an
-                    // error to the injector. Throughout this class we throw exceptions
-                    // to add errors, which isn't really the best way in Guice
-                    throw newConfigurationException("Parameter of type '%s' is not injectable or annotated "
-                            + "with @Assisted for Constructor '%s'", p, c);
-                }
-            }
-        }
-    }
-
     private void checkDeclaredExceptionsMatch() {
         for (Map.Entry<Method, AssistedConstructor<?>> entry : factoryMethodToConstructor.entrySet()) {
             for (Class<?> constructorException : entry.getValue().getDeclaredExceptions()) {
@@ -204,82 +171,6 @@ public class FactoryProvider<F> implements Provider<F>, HasDependencies {
         return false;
     }
 
-    private boolean paramCanBeInjected(Parameter parameter, Injector injector) {
-        return parameter.isBound(injector);
-    }
-
-    private static Map<Method, AssistedConstructor<?>> createMethodMapping(
-            TypeLiteral<?> factoryType, TypeLiteral<?> implementationType) {
-        List<AssistedConstructor<?>> constructors = new ArrayList<>();
-
-        for (Constructor<?> constructor : implementationType.getRawType().getConstructors()) {
-            if (constructor.getAnnotation(AssistedInject.class) != null) {
-                @SuppressWarnings("unchecked") // the constructor type and implementation type agree
-                        AssistedConstructor assistedConstructor = new AssistedConstructor(
-                        constructor, implementationType.getParameterTypes(constructor));
-                constructors.add(assistedConstructor);
-            }
-        }
-
-        if (constructors.isEmpty()) {
-            return emptyMap();
-        }
-
-        Method[] factoryMethods = factoryType.getRawType().getMethods();
-
-        if (constructors.size() != factoryMethods.length) {
-            throw newConfigurationException("Constructor mismatch: %s has %s @AssistedInject "
-                    + "constructors, factory %s has %s creation methods", implementationType,
-                    constructors.size(), factoryType, factoryMethods.length);
-        }
-
-        Map<ParameterListKey, AssistedConstructor> paramsToConstructor = new HashMap<>();
-
-        for (AssistedConstructor c : constructors) {
-            if (paramsToConstructor.containsKey(c.getAssistedParameters())) {
-                throw new RuntimeException("Duplicate constructor, " + c);
-            }
-            paramsToConstructor.put(c.getAssistedParameters(), c);
-        }
-
-        Map<Method, AssistedConstructor<?>> result = new HashMap<>();
-        for (Method method : factoryMethods) {
-            if (!method.getReturnType().isAssignableFrom(implementationType.getRawType())) {
-                throw newConfigurationException("Return type of method %s is not assignable from %s",
-                        method, implementationType);
-            }
-
-            List<Type> parameterTypes = new ArrayList<>();
-            for (TypeLiteral<?> parameterType : factoryType.getParameterTypes(method)) {
-                parameterTypes.add(parameterType.getType());
-            }
-            ParameterListKey methodParams = new ParameterListKey(parameterTypes);
-
-            if (!paramsToConstructor.containsKey(methodParams)) {
-                throw newConfigurationException("%s has no @AssistInject constructor that takes the "
-                        + "@Assisted parameters %s in that order. @AssistInject constructors are %s",
-                        implementationType, methodParams, paramsToConstructor.values());
-            }
-
-            method.getParameterAnnotations();
-            for (Annotation[] parameterAnnotations : method.getParameterAnnotations()) {
-                for (Annotation parameterAnnotation : parameterAnnotations) {
-                    if (parameterAnnotation.annotationType() == Assisted.class) {
-                        throw newConfigurationException("Factory method %s has an @Assisted parameter, which "
-                                + "is incompatible with the deprecated @AssistedInject annotation. Please replace "
-                                + "@AssistedInject with @Inject on the %s constructor.",
-                                method, implementationType);
-                    }
-                }
-            }
-
-            AssistedConstructor matchingConstructor = paramsToConstructor.remove(methodParams);
-
-            result.put(method, matchingConstructor);
-        }
-        return result;
-    }
-
     @Override
     public Set<Dependency<?>> getDependencies() {
         Set<Dependency<?>> dependencies = new HashSet<>();
diff --git a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider2.java b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider2.java
deleted file mode 100644
index 29863527202..00000000000
--- a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/FactoryProvider2.java
+++ /dev/null
@@ -1,283 +0,0 @@
-/*
- * Copyright (C) 2008 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.elasticsearch.common.inject.assistedinject;
-
-import org.elasticsearch.common.inject.AbstractModule;
-import org.elasticsearch.common.inject.Binder;
-import org.elasticsearch.common.inject.Binding;
-import org.elasticsearch.common.inject.ConfigurationException;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.inject.Injector;
-import org.elasticsearch.common.inject.Key;
-import org.elasticsearch.common.inject.Module;
-import org.elasticsearch.common.inject.Provider;
-import org.elasticsearch.common.inject.ProvisionException;
-import org.elasticsearch.common.inject.TypeLiteral;
-import org.elasticsearch.common.inject.internal.Errors;
-import org.elasticsearch.common.inject.internal.ErrorsException;
-import org.elasticsearch.common.inject.spi.Message;
-import org.elasticsearch.common.inject.util.Providers;
-
-import java.lang.annotation.Annotation;
-import java.lang.reflect.InvocationHandler;
-import java.lang.reflect.Method;
-import java.lang.reflect.Proxy;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static java.util.Collections.unmodifiableMap;
-import static org.elasticsearch.common.inject.internal.Annotations.getKey;
-
-/**
- * The newer implementation of factory provider. This implementation uses a child injector to
- * create values.
- *
- * @author jessewilson@google.com (Jesse Wilson)
- * @author dtm@google.com (Daniel Martin)
- */
-public final class FactoryProvider2<F> implements InvocationHandler, Provider<F> {
-
-    /**
-     * if a factory method parameter isn't annotated, it gets this annotation.
-     */
-    static final Assisted DEFAULT_ANNOTATION = new Assisted() {
-        @Override
-        public String value() {
-            return "";
-        }
-
-        @Override
-        public Class<? extends Annotation> annotationType() {
-            return Assisted.class;
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            return o instanceof Assisted
-                    && ((Assisted) o).value().equals("");
-        }
-
-        @Override
-        public int hashCode() {
-            return 127 * "value".hashCode() ^ "".hashCode();
-        }
-
-        @Override
-        public String toString() {
-            return "@" + Assisted.class.getName() + "(value=)";
-        }
-    };
-
-    /**
-     * the produced type, or null if all methods return concrete types
-     */
-    private final Key<?> producedType;
-    private final Map<Method, Key<?>> returnTypesByMethod;
-    private final Map<Method, List<Key<?>>> paramTypes;
-
-    /**
-     * the hosting injector, or null if we haven't been initialized yet
-     */
-    private Injector injector;
-
-    /**
-     * the factory interface, implemented and provided
-     */
-    private final F factory;
-
-    /**
-     * @param factoryType  a Java interface that defines one or more create methods.
-     * @param producedType a concrete type that is assignable to the return types of all factory
-     *                     methods.
-     */
-    FactoryProvider2(TypeLiteral<F> factoryType, Key<?> producedType) {
-        this.producedType = producedType;
-
-        Errors errors = new Errors();
-
-        @SuppressWarnings("unchecked") // we imprecisely treat the class literal of T as a Class<T>
-                Class<F> factoryRawType = (Class) factoryType.getRawType();
-
-        try {
-            Map<Method, Key<?>> returnTypesBuilder = new HashMap<>();
-            Map<Method, List<Key<?>>> paramTypesBuilder = new HashMap<>();
-            // TODO: also grab methods from superinterfaces
-            for (Method method : factoryRawType.getMethods()) {
-                Key<?> returnType = getKey(
-                        factoryType.getReturnType(method), method, method.getAnnotations(), errors);
-                returnTypesBuilder.put(method, returnType);
-                List<TypeLiteral<?>> params = factoryType.getParameterTypes(method);
-                Annotation[][] paramAnnotations = method.getParameterAnnotations();
-                int p = 0;
-                List<Key<?>> keys = new ArrayList<>();
-                for (TypeLiteral<?> param : params) {
-                    Key<?> paramKey = getKey(param, method, paramAnnotations[p++], errors);
-                    keys.add(assistKey(method, paramKey, errors));
-                }
-                paramTypesBuilder.put(method, Collections.unmodifiableList(keys));
-            }
-            returnTypesByMethod = unmodifiableMap(returnTypesBuilder);
-            paramTypes = unmodifiableMap(paramTypesBuilder);
-        } catch (ErrorsException e) {
-            throw new ConfigurationException(e.getErrors().getMessages());
-        }
-
-        factory = factoryRawType.cast(Proxy.newProxyInstance(factoryRawType.getClassLoader(),
-                new Class[]{factoryRawType}, this));
-    }
-
-    @Override
-    public F get() {
-        return factory;
-    }
-
-    /**
-     * Returns a key similar to {@code key}, but with an {@literal @}Assisted binding annotation.
-     * This fails if another binding annotation is clobbered in the process. If the key already has
-     * the {@literal @}Assisted annotation, it is returned as-is to preserve any String value.
-     */
-    private <T> Key<T> assistKey(Method method, Key<T> key, Errors errors) throws ErrorsException {
-        if (key.getAnnotationType() == null) {
-            return Key.get(key.getTypeLiteral(), DEFAULT_ANNOTATION);
-        } else if (key.getAnnotationType() == Assisted.class) {
-            return key;
-        } else {
-            errors.withSource(method).addMessage(
-                    "Only @Assisted is allowed for factory parameters, but found @%s",
-                    key.getAnnotationType());
-            throw errors.toException();
-        }
-    }
-
-    /**
-     * At injector-creation time, we initialize the invocation handler. At this time we make sure
-     * all factory methods will be able to build the target types.
-     */
-    @Inject
-    public void initialize(Injector injector) {
-        if (this.injector != null) {
-            throw new ConfigurationException(Collections.singletonList(new Message(FactoryProvider2.class,
-                "Factories.create() factories may only be used in one Injector!")));
-        }
-
-        this.injector = injector;
-
-        for (Method method : returnTypesByMethod.keySet()) {
-            Object[] args = new Object[method.getParameterTypes().length];
-            Arrays.fill(args, "dummy object for validating Factories");
-            getBindingFromNewInjector(method, args); // throws if the binding isn't properly configured
-        }
-    }
-
-    /**
-     * Creates a child injector that binds the args, and returns the binding for the method's result.
-     */
-    public Binding<?> getBindingFromNewInjector(final Method method, final Object[] args) {
-        if (injector == null) {
-            throw new IllegalStateException("Factories.create() factories cannot be used until they're initialized by Guice.");
-        }
-
-        final Key<?> returnType = returnTypesByMethod.get(method);
-
-        Module assistedModule = new AbstractModule() {
-            @Override
-            @SuppressWarnings("unchecked") // raw keys are necessary for the args array and return value
-            protected void configure() {
-                Binder binder = binder().withSource(method);
-
-                int p = 0;
-                for (Key<?> paramKey : paramTypes.get(method)) {
-                    // Wrap in a Provider to cover null, and to prevent Guice from injecting the parameter
-                    binder.bind((Key) paramKey).toProvider(Providers.of(args[p++]));
-                }
-
-                if (producedType != null && !returnType.equals(producedType)) {
-                    binder.bind(returnType).to((Key) producedType);
-                } else {
-                    binder.bind(returnType);
-                }
-            }
-        };
-
-        Injector forCreate = injector.createChildInjector(assistedModule);
-        return forCreate.getBinding(returnType);
-    }
-
-    /**
-     * When a factory method is invoked, we create a child injector that binds all parameters, then
-     * use that to get an instance of the return type.
-     */
-    @Override
-    public Object invoke(Object proxy, final Method method, final Object[] args) throws Throwable {
-        if (method.getDeclaringClass() == Object.class) {
-            return method.invoke(this, args);
-        }
-
-        Provider<?> provider = getBindingFromNewInjector(method, args).getProvider();
-        try {
-            return provider.get();
-        } catch (ProvisionException e) {
-            // if this is an exception declared by the factory method, throw it as-is
-            if (e.getErrorMessages().size() == 1) {
-                Message onlyError = e.getErrorMessages().iterator().next();
-                Throwable cause = onlyError.getCause();
-                if (cause != null && canRethrow(method, cause)) {
-                    throw cause;
-                }
-            }
-            throw e;
-        }
-    }
-
-    @Override
-    public String toString() {
-        return factory.getClass().getInterfaces()[0].getName()
-                + " for " + producedType.getTypeLiteral();
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        return o == this || o == factory;
-    }
-
-    @Override
-    public int hashCode() {
-        // This way both this and its factory hash to the same spot, making hashCode consistent.
-        return factory.hashCode();
-    }
-
-    /**
-     * Returns true if {@code thrown} can be thrown by {@code invoked} without wrapping.
-     */
-    static boolean canRethrow(Method invoked, Throwable thrown) {
-        if (thrown instanceof Error || thrown instanceof RuntimeException) {
-            return true;
-        }
-
-        for (Class<?> declared : invoked.getExceptionTypes()) {
-            if (declared.isInstance(thrown)) {
-                return true;
-            }
-        }
-
-        return false;
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/Parameter.java b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/Parameter.java
index 0fae9dede5b..5ceb086db9f 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/assistedinject/Parameter.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/assistedinject/Parameter.java
@@ -87,34 +87,6 @@ class Parameter {
                 : injector.getInstance(getPrimaryBindingKey());
     }
 
-    public boolean isBound(Injector injector) {
-        return isBound(injector, getPrimaryBindingKey())
-                || isBound(injector, fixAnnotations(getPrimaryBindingKey()));
-    }
-
-    private boolean isBound(Injector injector, Key<?> key) {
-        // This method is particularly lame - we really need an API that can test
-        // for any binding, implicit or explicit
-        try {
-            return injector.getBinding(key) != null;
-        } catch (ConfigurationException e) {
-            return false;
-        }
-    }
-
-    /**
-     * Replace annotation instances with annotation types, this is only
-     * appropriate for testing if a key is bound and not for injecting.
-     * <p>
-     * See Guice bug 125,
-     * http://code.google.com/p/google-guice/issues/detail?id=125
-     */
-    public Key<?> fixAnnotations(Key<?> key) {
-        return key.getAnnotation() == null
-                ? key
-                : Key.get(key.getTypeLiteral(), key.getAnnotation().annotationType());
-    }
-
     Key<?> getPrimaryBindingKey() {
         return isProvider
                 ? getBindingForType(getProvidedType(type))
diff --git a/core/src/main/java/org/elasticsearch/common/inject/spi/ConvertedConstantBinding.java b/core/src/main/java/org/elasticsearch/common/inject/spi/ConvertedConstantBinding.java
index 3c4ec4d9ed5..1963919c0c5 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/spi/ConvertedConstantBinding.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/spi/ConvertedConstantBinding.java
@@ -36,8 +36,7 @@ public interface ConvertedConstantBinding<T> extends Binding<T>, HasDependencies
     T getValue();
 
     /**
-     * Returns the key for the source binding. That binding can e retrieved from an injector using
-     * {@link org.elasticsearch.common.inject.Injector#getBinding(Key) Injector.getBinding(key)}.
+     * Returns the key for the source binding.
      */
     Key<String> getSourceKey();
 
diff --git a/core/src/main/java/org/elasticsearch/common/inject/spi/Element.java b/core/src/main/java/org/elasticsearch/common/inject/spi/Element.java
index f9128d98f56..a2eaf995c70 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/spi/Element.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/spi/Element.java
@@ -25,9 +25,6 @@ import org.elasticsearch.common.inject.Binder;
  * Elements#getElements(org.elasticsearch.common.inject.Module[]) Elements.getElements()} to read the elements
  * from a module, and {@link Elements#getModule(Iterable) Elements.getModule()} to rewrite them.
  * This can be used for static analysis and generation of Guice modules.
- * <p>
- * The elements of an injector can be inspected and exercised. Use {@link
- * org.elasticsearch.common.inject.Injector#getBindings Injector.getBindings()} to reflect on Guice injectors.
  *
  * @author jessewilson@google.com (Jesse Wilson)
  * @author crazybob@google.com (Bob Lee)
diff --git a/core/src/main/java/org/elasticsearch/common/inject/spi/Elements.java b/core/src/main/java/org/elasticsearch/common/inject/spi/Elements.java
index 4b3660fe47a..30911def551 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/spi/Elements.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/spi/Elements.java
@@ -80,13 +80,6 @@ public final class Elements {
         return getElements(Stage.DEVELOPMENT, Arrays.asList(modules));
     }
 
-    /**
-     * Records the elements executed by {@code modules}.
-     */
-    public static List<Element> getElements(Stage stage, Module... modules) {
-        return getElements(stage, Arrays.asList(modules));
-    }
-
     /**
      * Records the elements executed by {@code modules}.
      */
@@ -119,11 +112,6 @@ public final class Elements {
         };
     }
 
-    @SuppressWarnings("unchecked")
-    static <T> BindingTargetVisitor<T, T> getInstanceVisitor() {
-        return (BindingTargetVisitor<T, T>) GET_INSTANCE_VISITOR;
-    }
-
     private static class RecordingBinder implements Binder, PrivateBinder {
         private final Stage stage;
         private final Set<Module> modules;
diff --git a/core/src/main/java/org/elasticsearch/common/inject/spi/LinkedKeyBinding.java b/core/src/main/java/org/elasticsearch/common/inject/spi/LinkedKeyBinding.java
index 82c36c7c2e2..60b7adced3e 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/spi/LinkedKeyBinding.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/spi/LinkedKeyBinding.java
@@ -28,8 +28,7 @@ import org.elasticsearch.common.inject.Key;
 public interface LinkedKeyBinding<T> extends Binding<T> {
 
     /**
-     * Returns the linked key used to resolve injections. That binding can be retrieved from an
-     * injector using {@link org.elasticsearch.common.inject.Injector#getBinding(Key) Injector.getBinding(key)}.
+     * Returns the linked key used to resolve injections.
      */
     Key<? extends T> getLinkedKey();
 
diff --git a/core/src/main/java/org/elasticsearch/common/inject/spi/ProviderBinding.java b/core/src/main/java/org/elasticsearch/common/inject/spi/ProviderBinding.java
index 04fb78d0288..c6ac377c051 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/spi/ProviderBinding.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/spi/ProviderBinding.java
@@ -30,9 +30,7 @@ import org.elasticsearch.common.inject.Provider;
 public interface ProviderBinding<T extends Provider<?>> extends Binding<T> {
 
     /**
-     * Returns the key whose binding is used to {@link Provider#get provide instances}. That binding
-     * can be retrieved from an injector using {@link org.elasticsearch.common.inject.Injector#getBinding(Key)
-     * Injector.getBinding(providedKey)}
+     * Returns the key whose binding is used to {@link Provider#get provide instances}.
      */
     Key<?> getProvidedKey();
 }
\ No newline at end of file
diff --git a/core/src/main/java/org/elasticsearch/common/inject/spi/ProviderKeyBinding.java b/core/src/main/java/org/elasticsearch/common/inject/spi/ProviderKeyBinding.java
index 15aa751f4a6..78191621561 100644
--- a/core/src/main/java/org/elasticsearch/common/inject/spi/ProviderKeyBinding.java
+++ b/core/src/main/java/org/elasticsearch/common/inject/spi/ProviderKeyBinding.java
@@ -30,9 +30,7 @@ import org.elasticsearch.common.inject.Provider;
 public interface ProviderKeyBinding<T> extends Binding<T> {
 
     /**
-     * Returns the key used to resolve the provider's binding. That binding can be retrieved from an
-     * injector using {@link org.elasticsearch.common.inject.Injector#getBinding(Key)
-     * Injector.getBinding(providerKey)}
+     * Returns the key used to resolve the provider's binding.
      */
     Key<? extends Provider<? extends T>> getProviderKey();
 

From 2b9d4bdf857f1b70a9e1c69c4b7347ce28a1fff5 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Fri, 8 Jul 2016 21:37:37 -0700
Subject: [PATCH 1303/1311] Plugins: Remove CustomNodeAttributes extension
 point

The DiscoveryNodeService exists to register CustomNodeAttributes which
plugins can add. This is not necessary, since plugins can already add
additional attributes, and use the node attributes prefix.

This change removes the DiscoveryNodeService, and converts the only
consumer, the ec2 discovery plugin, to add the ec2 availability zone
in additionalSettings().
---
 .../elasticsearch/cluster/ClusterModule.java  |  2 -
 .../cluster/node/DiscoveryNode.java           | 20 +++++
 .../cluster/node/DiscoveryNodeService.java    | 88 -------------------
 .../java/org/elasticsearch/node/Node.java     |  5 +-
 .../node/DiscoveryNodeServiceTests.java       | 85 ------------------
 .../PublishClusterStateActionTests.java       |  6 +-
 .../cloud/aws/AwsEc2ServiceImpl.java          |  5 +-
 .../aws/node/Ec2CustomNodeAttributes.java     | 78 ----------------
 .../discovery/ec2/Ec2DiscoveryPlugin.java     | 69 +++++++++++++--
 .../cloud/aws/AWSSignersTests.java            |  3 +-
 10 files changed, 88 insertions(+), 273 deletions(-)
 delete mode 100644 core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeService.java
 delete mode 100644 core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeServiceTests.java
 delete mode 100644 plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/node/Ec2CustomNodeAttributes.java

diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
index 47dcb6170bc..fac138ce820 100644
--- a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
+++ b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java
@@ -31,7 +31,6 @@ import org.elasticsearch.cluster.metadata.MetaDataIndexStateService;
 import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService;
 import org.elasticsearch.cluster.metadata.MetaDataMappingService;
 import org.elasticsearch.cluster.metadata.MetaDataUpdateSettingsService;
-import org.elasticsearch.cluster.node.DiscoveryNodeService;
 import org.elasticsearch.cluster.routing.DelayedAllocationService;
 import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
@@ -146,7 +145,6 @@ public class ClusterModule extends AbstractModule {
         bind(ClusterInfoService.class).to(clusterInfoServiceImpl).asEagerSingleton();
         bind(GatewayAllocator.class).asEagerSingleton();
         bind(AllocationService.class).asEagerSingleton();
-        bind(DiscoveryNodeService.class).asEagerSingleton();
         bind(ClusterService.class).toInstance(clusterService);
         bind(NodeConnectionsService.class).asEagerSingleton();
         bind(MetaDataCreateIndexService.class).asEagerSingleton();
diff --git a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
index 1cc79f3525e..329405849e9 100644
--- a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
+++ b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java
@@ -35,9 +35,11 @@ import java.io.IOException;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Predicate;
+import java.util.function.Supplier;
 
 import static org.elasticsearch.common.transport.TransportAddressSerializers.addressToStream;
 
@@ -187,6 +189,24 @@ public class DiscoveryNode implements Writeable, ToXContent {
         this.roles = Collections.unmodifiableSet(rolesSet);
     }
 
+    /** Creates a DiscoveryNode representing the local node. */
+    public static DiscoveryNode createLocal(Settings settings, TransportAddress publishAddress, String nodeIdSupplier) {
+        Map<String, String> attributes = new HashMap<>(Node.NODE_ATTRIBUTES.get(settings).getAsMap());
+        Set<DiscoveryNode.Role> roles = new HashSet<>();
+        if (Node.NODE_INGEST_SETTING.get(settings)) {
+            roles.add(DiscoveryNode.Role.INGEST);
+        }
+        if (Node.NODE_MASTER_SETTING.get(settings)) {
+            roles.add(DiscoveryNode.Role.MASTER);
+        }
+        if (Node.NODE_DATA_SETTING.get(settings)) {
+            roles.add(DiscoveryNode.Role.DATA);
+        }
+
+        return new DiscoveryNode(Node.NODE_NAME_SETTING.get(settings), nodeIdSupplier, publishAddress,
+                                 attributes, roles, Version.CURRENT);
+    }
+
     /**
      * Creates a new {@link DiscoveryNode} by reading from the stream provided as argument
      * @param in the stream
diff --git a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeService.java b/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeService.java
deleted file mode 100644
index a91b216eac5..00000000000
--- a/core/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeService.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cluster.node;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.node.Node;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.function.Supplier;
-
-/**
- */
-public class DiscoveryNodeService extends AbstractComponent {
-
-    private final List<CustomAttributesProvider> customAttributesProviders = new CopyOnWriteArrayList<>();
-
-    @Inject
-    public DiscoveryNodeService(Settings settings) {
-        super(settings);
-    }
-
-    public DiscoveryNodeService addCustomAttributeProvider(CustomAttributesProvider customAttributesProvider) {
-        customAttributesProviders.add(customAttributesProvider);
-        return this;
-    }
-
-    public DiscoveryNode buildLocalNode(TransportAddress publishAddress, Supplier<String> nodeIdSupplier) {
-        Map<String, String> attributes = new HashMap<>(Node.NODE_ATTRIBUTES.get(this.settings).getAsMap());
-        Set<DiscoveryNode.Role> roles = new HashSet<>();
-        if (Node.NODE_INGEST_SETTING.get(settings)) {
-            roles.add(DiscoveryNode.Role.INGEST);
-        }
-        if (Node.NODE_MASTER_SETTING.get(settings)) {
-            roles.add(DiscoveryNode.Role.MASTER);
-        }
-        if (Node.NODE_DATA_SETTING.get(settings)) {
-            roles.add(DiscoveryNode.Role.DATA);
-        }
-
-        for (CustomAttributesProvider provider : customAttributesProviders) {
-            try {
-                Map<String, String> customAttributes = provider.buildAttributes();
-                if (customAttributes != null) {
-                    for (Map.Entry<String, String> entry : customAttributes.entrySet()) {
-                        if (!attributes.containsKey(entry.getKey())) {
-                            attributes.put(entry.getKey(), entry.getValue());
-                        }
-                    }
-                }
-            } catch (Exception e) {
-                logger.warn("failed to build custom attributes from provider [{}]", e, provider);
-            }
-        }
-        return new DiscoveryNode(Node.NODE_NAME_SETTING.get(settings), nodeIdSupplier.get(), publishAddress, attributes, roles,
-            Version.CURRENT);
-    }
-
-    public interface CustomAttributesProvider {
-
-        Map<String, String> buildAttributes();
-    }
-}
diff --git a/core/src/main/java/org/elasticsearch/node/Node.java b/core/src/main/java/org/elasticsearch/node/Node.java
index bfdf5e58084..b5ea07fc173 100644
--- a/core/src/main/java/org/elasticsearch/node/Node.java
+++ b/core/src/main/java/org/elasticsearch/node/Node.java
@@ -37,7 +37,6 @@ import org.elasticsearch.cluster.MasterNodeChangePredicate;
 import org.elasticsearch.cluster.NodeConnectionsService;
 import org.elasticsearch.cluster.action.index.MappingUpdatedAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodeService;
 import org.elasticsearch.cluster.routing.RoutingService;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -415,8 +414,8 @@ public class Node implements Closeable {
 
         validateNodeBeforeAcceptingRequests(settings, transportService.boundAddress());
 
-        DiscoveryNode localNode = injector.getInstance(DiscoveryNodeService.class)
-                .buildLocalNode(transportService.boundAddress().publishAddress(), injector.getInstance(NodeEnvironment.class)::nodeId);
+        DiscoveryNode localNode = DiscoveryNode.createLocal(settings,
+            transportService.boundAddress().publishAddress(), injector.getInstance(NodeEnvironment.class).nodeId());
 
         // TODO: need to find a cleaner way to start/construct a service with some initial parameters,
         // playing nice with the life cycle interfaces
diff --git a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeServiceTests.java
deleted file mode 100644
index e9f80426df6..00000000000
--- a/core/src/test/java/org/elasticsearch/cluster/node/DiscoveryNodeServiceTests.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cluster.node;
-
-import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.LocalTransportAddress;
-import org.elasticsearch.test.ESTestCase;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-
-public class DiscoveryNodeServiceTests extends ESTestCase {
-
-    public void testBuildLocalNode() {
-        Map<String, String> expectedAttributes = new HashMap<>();
-        int numCustomSettings = randomIntBetween(0, 5);
-        Settings.Builder builder = Settings.builder();
-        for (int i = 0; i < numCustomSettings; i++) {
-            builder.put("node.attr.attr" + i, "value" + i);
-            expectedAttributes.put("attr" + i, "value" + i);
-        }
-        Set<DiscoveryNode.Role> selectedRoles = new HashSet<>();
-        for (DiscoveryNode.Role role : DiscoveryNode.Role.values()) {
-            if (randomBoolean()) {
-                //test default true for every role
-                selectedRoles.add(role);
-            } else {
-                boolean isRoleEnabled = randomBoolean();
-                builder.put("node." + role.getRoleName(), isRoleEnabled);
-                if (isRoleEnabled) {
-                    selectedRoles.add(role);
-                }
-            }
-        }
-        DiscoveryNodeService discoveryNodeService = new DiscoveryNodeService(builder.build());
-        DiscoveryNode discoveryNode = discoveryNodeService.buildLocalNode(LocalTransportAddress.buildUnique(),
-            () -> UUIDs.randomBase64UUID(random()));
-        assertThat(discoveryNode.getRoles(), equalTo(selectedRoles));
-        assertThat(discoveryNode.getAttributes(), equalTo(expectedAttributes));
-    }
-
-    public void testBuildAttributesWithCustomAttributeServiceProvider() {
-        Map<String, String> expectedAttributes = new HashMap<>();
-        int numCustomSettings = randomIntBetween(0, 5);
-        Settings.Builder builder = Settings.builder();
-        for (int i = 0; i < numCustomSettings; i++) {
-            builder.put("node.attr.attr" + i, "value" + i);
-            expectedAttributes.put("attr" + i, "value" + i);
-        }
-        DiscoveryNodeService discoveryNodeService = new DiscoveryNodeService(builder.build());
-        int numCustomAttributes = randomIntBetween(0, 5);
-        Map<String, String> customAttributes = new HashMap<>();
-        for (int i = 0; i < numCustomAttributes; i++) {
-            customAttributes.put("custom-" + randomAsciiOfLengthBetween(5, 10), randomAsciiOfLengthBetween(1, 10));
-        }
-        expectedAttributes.putAll(customAttributes);
-        discoveryNodeService.addCustomAttributeProvider(() -> customAttributes);
-
-        DiscoveryNode discoveryNode = discoveryNodeService.buildLocalNode(LocalTransportAddress.buildUnique(),
-            () -> UUIDs.randomBase64UUID(random()));
-        assertThat(discoveryNode.getAttributes(), equalTo(expectedAttributes));
-    }
-}
diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java
index 0d4274a5a53..7d72fa5c4dc 100644
--- a/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java
+++ b/core/src/test/java/org/elasticsearch/discovery/zen/publish/PublishClusterStateActionTests.java
@@ -30,7 +30,6 @@ import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodeService;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Randomness;
@@ -161,9 +160,8 @@ public class PublishClusterStateActionTests extends ESTestCase {
                 .build();
 
         MockTransportService service = buildTransportService(settings);
-        DiscoveryNodeService discoveryNodeService = new DiscoveryNodeService(settings);
-        DiscoveryNode discoveryNode = discoveryNodeService.buildLocalNode(service.boundAddress().publishAddress(),
-            () -> NodeEnvironment.generateNodeId(settings));
+        DiscoveryNode discoveryNode = DiscoveryNode.createLocal(settings, service.boundAddress().publishAddress(),
+            NodeEnvironment.generateNodeId(settings));
         MockNode node = new MockNode(discoveryNode, service, listener, logger);
         node.action = buildPublishClusterStateAction(settings, service, () -> node.clusterState, node);
         final CountDownLatch latch = new CountDownLatch(nodes.size() * 2 + 1);
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java
index 2fc82a01f09..649f84b7aed 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java
@@ -35,8 +35,6 @@ import com.amazonaws.services.ec2.AmazonEC2;
 import com.amazonaws.services.ec2.AmazonEC2Client;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cloud.aws.network.Ec2NameResolver;
-import org.elasticsearch.cloud.aws.node.Ec2CustomNodeAttributes;
-import org.elasticsearch.cluster.node.DiscoveryNodeService;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
@@ -56,11 +54,10 @@ public class AwsEc2ServiceImpl extends AbstractLifecycleComponent implements Aws
     private AmazonEC2Client client;
 
     @Inject
-    public AwsEc2ServiceImpl(Settings settings, NetworkService networkService, DiscoveryNodeService discoveryNodeService) {
+    public AwsEc2ServiceImpl(Settings settings, NetworkService networkService) {
         super(settings);
         // add specific ec2 name resolver
         networkService.addCustomNameResolver(new Ec2NameResolver(settings));
-        discoveryNodeService.addCustomAttributeProvider(new Ec2CustomNodeAttributes(settings));
     }
 
     @Override
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/node/Ec2CustomNodeAttributes.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/node/Ec2CustomNodeAttributes.java
deleted file mode 100644
index 9ba1ce650e8..00000000000
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/node/Ec2CustomNodeAttributes.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.cloud.aws.node;
-
-import org.apache.lucene.util.IOUtils;
-import org.elasticsearch.cloud.aws.AwsEc2Service;
-import org.elasticsearch.cloud.aws.AwsEc2ServiceImpl;
-import org.elasticsearch.cluster.node.DiscoveryNodeService;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.net.URL;
-import java.net.URLConnection;
-import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- */
-public class Ec2CustomNodeAttributes extends AbstractComponent implements DiscoveryNodeService.CustomAttributesProvider {
-
-    public Ec2CustomNodeAttributes(Settings settings) {
-        super(settings);
-    }
-
-    @Override
-    public Map<String, String> buildAttributes() {
-        if (AwsEc2Service.AUTO_ATTRIBUTE_SETTING.get(settings) == false) {
-            return null;
-        }
-        Map<String, String> ec2Attributes = new HashMap<>();
-
-        URLConnection urlConnection;
-        InputStream in = null;
-        try {
-            URL url = new URL(AwsEc2ServiceImpl.EC2_METADATA_URL + "placement/availability-zone");
-            logger.debug("obtaining ec2 [placement/availability-zone] from ec2 meta-data url {}", url);
-            urlConnection = url.openConnection();
-            urlConnection.setConnectTimeout(2000);
-            in = urlConnection.getInputStream();
-            BufferedReader urlReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
-
-            String metadataResult = urlReader.readLine();
-            if (metadataResult == null || metadataResult.length() == 0) {
-                logger.error("no ec2 metadata returned from {}", url);
-                return null;
-            }
-            ec2Attributes.put("aws_availability_zone", metadataResult);
-        } catch (IOException e) {
-            logger.debug("failed to get metadata for [placement/availability-zone]", e);
-        } finally {
-            IOUtils.closeWhileHandlingException(in);
-        }
-
-        return ec2Attributes;
-    }
-}
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
index b5f179102a7..ec43fd200fd 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
@@ -19,6 +19,21 @@
 
 package org.elasticsearch.plugin.discovery.ec2;
 
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.UncheckedIOException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.nio.charset.StandardCharsets;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.cloud.aws.AwsEc2Service;
 import org.elasticsearch.cloud.aws.AwsEc2ServiceImpl;
@@ -29,24 +44,19 @@ import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsModule;
 import org.elasticsearch.discovery.DiscoveryModule;
 import org.elasticsearch.discovery.ec2.AwsEc2UnicastHostsProvider;
 import org.elasticsearch.discovery.zen.ZenDiscovery;
+import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
 
-import java.security.AccessController;
-import java.security.PrivilegedAction;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
 /**
  *
  */
 public class Ec2DiscoveryPlugin extends Plugin {
 
+    private static ESLogger logger = Loggers.getLogger(Ec2DiscoveryPlugin.class);
+
     public static final String EC2 = "ec2";
 
     // ClientConfiguration clinit has some classloader problems
@@ -69,6 +79,12 @@ public class Ec2DiscoveryPlugin extends Plugin {
         });
     }
 
+    private Settings settings;
+
+    public Ec2DiscoveryPlugin(Settings settings) {
+        this.settings = settings;
+    }
+
     @Override
     public Collection<Module> nodeModules() {
         Collection<Module> modules = new ArrayList<>();
@@ -123,4 +139,41 @@ public class Ec2DiscoveryPlugin extends Plugin {
         // Register cloud node settings: cloud.node
         AwsEc2Service.AUTO_ATTRIBUTE_SETTING);
     }
+
+    /** Adds a node attribute for the ec2 availability zone. */
+    @Override
+    public Settings additionalSettings() {
+        if (AwsEc2Service.AUTO_ATTRIBUTE_SETTING.get(settings) == false) {
+            return Settings.EMPTY;
+        }
+        Settings.Builder attrs = Settings.builder();
+
+        final URL url;
+        final URLConnection urlConnection;
+        try {
+            url = new URL(AwsEc2ServiceImpl.EC2_METADATA_URL + "placement/availability-zone");
+            logger.debug("obtaining ec2 [placement/availability-zone] from ec2 meta-data url {}", url);
+            urlConnection = url.openConnection();
+            urlConnection.setConnectTimeout(2000);
+        } catch (IOException e) {
+            // should not happen, we know the url is not malformed, and openConnection does not actually hit network
+            throw new UncheckedIOException(e);
+        }
+
+        try (InputStream in = urlConnection.getInputStream();
+             BufferedReader urlReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) {
+
+            String metadataResult = urlReader.readLine();
+            if (metadataResult == null || metadataResult.length() == 0) {
+                throw new IOException("no ec2 metadata returned from " + url);
+            } else {
+                attrs.put(Node.NODE_ATTRIBUTES.getKey() + "aws_availability_zone", metadataResult);
+            }
+        } catch (IOException e) {
+            // this is lenient so the plugin does not fail when installed outside of ec2
+            logger.error("failed to get metadata for [placement/availability-zone]", e);
+        }
+
+        return attrs.build();
+    }
 }
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AWSSignersTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AWSSignersTests.java
index c313f4bd195..555e9f5c10a 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AWSSignersTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/AWSSignersTests.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.cloud.aws;
 
 import com.amazonaws.ClientConfiguration;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugin.discovery.ec2.Ec2DiscoveryPlugin;
 import org.elasticsearch.test.ESTestCase;
 import org.junit.BeforeClass;
@@ -34,7 +35,7 @@ public class AWSSignersTests extends ESTestCase {
      */
     @BeforeClass
     public static void instantiatePlugin() {
-        new Ec2DiscoveryPlugin();
+        new Ec2DiscoveryPlugin(Settings.EMPTY);
     }
 
     public void testSigners() {

From 1fa8ba6c66e8e847e91d8a291c9369d08350ec22 Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Sat, 9 Jul 2016 09:40:08 -0700
Subject: [PATCH 1304/1311] Add unit tests for ec2 AZ node attributes

---
 .../discovery/ec2/Ec2DiscoveryPlugin.java     |  9 +-
 .../ec2/Ec2DiscoveryPluginTests.java          | 83 +++++++++++++++++++
 2 files changed, 90 insertions(+), 2 deletions(-)
 create mode 100644 plugins/discovery-ec2/src/test/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPluginTests.java

diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
index ec43fd200fd..1e314b4b310 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPlugin.java
@@ -143,6 +143,11 @@ public class Ec2DiscoveryPlugin extends Plugin {
     /** Adds a node attribute for the ec2 availability zone. */
     @Override
     public Settings additionalSettings() {
+        return getAvailabilityZoneNodeAttributes(settings, AwsEc2ServiceImpl.EC2_METADATA_URL + "placement/availability-zone");
+    }
+
+    // pkg private for testing
+    static Settings getAvailabilityZoneNodeAttributes(Settings settings, String azMetadataUrl) {
         if (AwsEc2Service.AUTO_ATTRIBUTE_SETTING.get(settings) == false) {
             return Settings.EMPTY;
         }
@@ -151,7 +156,7 @@ public class Ec2DiscoveryPlugin extends Plugin {
         final URL url;
         final URLConnection urlConnection;
         try {
-            url = new URL(AwsEc2ServiceImpl.EC2_METADATA_URL + "placement/availability-zone");
+            url = new URL(azMetadataUrl);
             logger.debug("obtaining ec2 [placement/availability-zone] from ec2 meta-data url {}", url);
             urlConnection = url.openConnection();
             urlConnection.setConnectTimeout(2000);
@@ -165,7 +170,7 @@ public class Ec2DiscoveryPlugin extends Plugin {
 
             String metadataResult = urlReader.readLine();
             if (metadataResult == null || metadataResult.length() == 0) {
-                throw new IOException("no ec2 metadata returned from " + url);
+                throw new IllegalStateException("no ec2 metadata returned from " + url);
             } else {
                 attrs.put(Node.NODE_ATTRIBUTES.getKey() + "aws_availability_zone", metadataResult);
             }
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPluginTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPluginTests.java
new file mode 100644
index 00000000000..14ba0d2339a
--- /dev/null
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/plugin/discovery/ec2/Ec2DiscoveryPluginTests.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.plugin.discovery.ec2;
+
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+
+import org.elasticsearch.cloud.aws.AwsEc2Service;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.test.ESTestCase;
+
+public class Ec2DiscoveryPluginTests extends ESTestCase {
+
+    private Settings getNodeAttributes(Settings settings, String url) {
+        Settings realSettings = Settings.builder()
+            .put(AwsEc2Service.AUTO_ATTRIBUTE_SETTING.getKey(), true)
+            .put(settings).build();
+        return Ec2DiscoveryPlugin.getAvailabilityZoneNodeAttributes(realSettings, url);
+    }
+
+    private void assertNodeAttributes(Settings settings, String url, String expected) {
+        Settings additional = getNodeAttributes(settings, url);
+        if (expected == null) {
+            assertTrue(additional.isEmpty());
+        } else {
+            assertEquals(expected, additional.get(Node.NODE_ATTRIBUTES.getKey() + "aws_availability_zone"));
+        }
+    }
+
+    public void testNodeAttributesDisabled() {
+        Settings settings = Settings.builder()
+            .put(AwsEc2Service.AUTO_ATTRIBUTE_SETTING.getKey(), false).build();
+        assertNodeAttributes(settings, "bogus", null);
+    }
+
+    public void testNodeAttributes() throws Exception {
+        Path zoneUrl = createTempFile();
+        Files.write(zoneUrl, Arrays.asList("us-east-1c"));
+        assertNodeAttributes(Settings.EMPTY, zoneUrl.toUri().toURL().toString(), "us-east-1c");
+    }
+
+    public void testNodeAttributesBogusUrl() {
+        UncheckedIOException e = expectThrows(UncheckedIOException.class, () ->
+            getNodeAttributes(Settings.EMPTY, "bogus")
+        );
+        assertNotNull(e.getCause());
+        String msg = e.getCause().getMessage();
+        assertTrue(msg, msg.contains("no protocol: bogus"));
+    }
+
+    public void testNodeAttributesEmpty() throws Exception {
+        Path zoneUrl = createTempFile();
+        IllegalStateException e = expectThrows(IllegalStateException.class, () ->
+            getNodeAttributes(Settings.EMPTY, zoneUrl.toUri().toURL().toString())
+        );
+        assertTrue(e.getMessage(), e.getMessage().contains("no ec2 metadata returned"));
+    }
+
+    public void testNodeAttributesErrorLenient() throws Exception {
+        Path dne = createTempDir().resolve("dne");
+        assertNodeAttributes(Settings.EMPTY, dne.toUri().toURL().toString(), null);
+    }
+}

From ab8b577aea10b9cf5cff9d0513b4567776aea99c Mon Sep 17 00:00:00 2001
From: Nicholas Knize <nknize@gmail.com>
Date: Sat, 9 Jul 2016 19:08:06 -0500
Subject: [PATCH 1305/1311] Use GeoDistanceIT.testDuelOptimizations for bwc
 testing only

This test is only relevant for testing the GeoDistanceBuilder with old indices (pre GeoPointV2).

closes #19263
---
 .../search/sort/GeoDistanceIT.java            | 24 +++++--------------
 1 file changed, 6 insertions(+), 18 deletions(-)

diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java
index 346e2faa68a..ee4cfbebfaf 100644
--- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java
+++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java
@@ -575,15 +575,10 @@ public class GeoDistanceIT extends ESIntegTestCase {
         return randomDouble() * 180 - 90;
     }
 
-    @AwaitsFix(bugUrl = "http://github.com/elastic/elasticsearch/issues/19263")
     public void testDuelOptimizations() throws Exception {
-        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT);
+        Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_2_1_2);
         Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
-        if (version.before(Version.V_2_2_0)) {
-            assertAcked(prepareCreate("index").setSettings(settings).addMapping("type", "location", "type=geo_point,lat_lon=true"));
-        } else {
-            assertAcked(prepareCreate("index").setSettings(settings).addMapping("type", "location", "type=geo_point"));
-        }
+        assertAcked(prepareCreate("index").setSettings(settings).addMapping("type", "location", "type=geo_point,lat_lon=true"));
         final int numDocs = scaledRandomIntBetween(3000, 10000);
         List<IndexRequestBuilder> docs = new ArrayList<>();
         for (int i = 0; i < numDocs; ++i) {
@@ -602,19 +597,12 @@ public class GeoDistanceIT extends ESIntegTestCase {
                 GeoDistanceQueryBuilder qb = QueryBuilders.geoDistanceQuery("location").point(originLat, originLon).distance(distance)
                         .geoDistance(geoDistance);
                 long matches;
-                if (version.before(Version.V_2_2_0)) {
-                    for (String optimizeBbox : Arrays.asList("none", "memory", "indexed")) {
-                        qb.optimizeBbox(optimizeBbox);
-                        SearchResponse resp = client().prepareSearch("index").setSize(0).setQuery(QueryBuilders.constantScoreQuery(qb))
-                                .execute().actionGet();
-                        matches = assertDuelOptimization(resp);
-                        logger.info("{} -> {} hits", optimizeBbox, matches);
-                    }
-                } else {
+                for (String optimizeBbox : Arrays.asList("none", "memory", "indexed")) {
+                    qb.optimizeBbox(optimizeBbox);
                     SearchResponse resp = client().prepareSearch("index").setSize(0).setQuery(QueryBuilders.constantScoreQuery(qb))
-                            .execute().actionGet();
+                        .execute().actionGet();
                     matches = assertDuelOptimization(resp);
-                    logger.info("{} hits", matches);
+                    logger.info("{} -> {} hits", optimizeBbox, matches);
                 }
             }
         }

From 25ed93dd289b80a757bcc09e7e8daf75abc8c7fb Mon Sep 17 00:00:00 2001
From: Ryan Ernst <ryan@iernst.net>
Date: Sun, 10 Jul 2016 09:11:32 -0700
Subject: [PATCH 1306/1311] Fix test edge case for random bytes reference iter.

Getting an offset to the last byte means we can only stream one byte and
then we are done, we can't get another offset after it.
---
 .../common/bytes/AbstractBytesReferenceTestCase.java           | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java b/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
index c521314f92d..103e263ffa6 100644
--- a/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
+++ b/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java
@@ -271,6 +271,9 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
                 final int offset = randomIntBetween(0, length-1);
                 assertEquals(offset, input.skip(offset));
                 assertEquals(pbr.get(offset), input.readByte());
+                if (offset == length - 1) {
+                    continue; // no more bytes to retrieve!
+                }
                 final int nextOffset = randomIntBetween(offset, length-2);
                 assertEquals(nextOffset - offset, input.skip(nextOffset - offset));
                 assertEquals(pbr.get(nextOffset+1), input.readByte()); // +1 for the one byte we read above

From 306b1d9221f1daa41f3ccfde9e2848812d30495c Mon Sep 17 00:00:00 2001
From: Martijn van Groningen <martijn.v.groningen@gmail.com>
Date: Mon, 11 Jul 2016 08:48:29 +0200
Subject: [PATCH 1307/1311] test: better file names

---
 .../{30_with_ingest.yaml => 30_update_by_query_with_ingest.yaml}  | 0
 .../ingest/{40_with_ingest.yaml => 40_reindex_with_ingest.yaml}   | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/{30_with_ingest.yaml => 30_update_by_query_with_ingest.yaml} (100%)
 rename qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/{40_with_ingest.yaml => 40_reindex_with_ingest.yaml} (100%)

diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/30_with_ingest.yaml b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/30_update_by_query_with_ingest.yaml
similarity index 100%
rename from qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/30_with_ingest.yaml
rename to qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/30_update_by_query_with_ingest.yaml
diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/40_with_ingest.yaml b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/40_reindex_with_ingest.yaml
similarity index 100%
rename from qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/40_with_ingest.yaml
rename to qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/40_reindex_with_ingest.yaml

From 36dbe7250f31eefe900b37aa6d10fbbbe8126fcb Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Mon, 11 Jul 2016 10:21:03 +0200
Subject: [PATCH 1308/1311] Cleanup usage of http.enabled (#19351)

Several tests required http.enabled where it was unnecessary.
We also had RestMainActionIT which tests what two of our REST tests
test already so I removed it.
The explicit use of http.enabled: false is also obsolet since our
test do that by default.
---
 .../RecoveryWithUnsupportedIndicesIT.java     |  4 +-
 .../client/transport/TransportClientIT.java   |  8 +--
 .../DiscoveryWithServiceDisruptionsIT.java    |  1 -
 .../gateway/GatewayServiceTests.java          |  5 +-
 .../index/WaitUntilRefreshIT.java             |  4 --
 .../rest/action/main/RestMainActionIT.java    | 56 -------------------
 6 files changed, 4 insertions(+), 74 deletions(-)
 delete mode 100644 core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java

diff --git a/core/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java b/core/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java
index 9fe83f65c45..429266c4589 100644
--- a/core/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java
+++ b/core/src/test/java/org/elasticsearch/bwcompat/RecoveryWithUnsupportedIndicesIT.java
@@ -18,9 +18,7 @@
  */
 package org.elasticsearch.bwcompat;
 
-import org.elasticsearch.common.network.NetworkModule;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.node.Node;
 
 import static org.hamcrest.Matchers.containsString;
 
@@ -29,7 +27,7 @@ public class RecoveryWithUnsupportedIndicesIT extends StaticIndexBackwardCompati
         String indexName = "unsupported-0.20.6";
 
         logger.info("Checking static index {}", indexName);
-        Settings nodeSettings = prepareBackwardsDataDir(getBwcIndicesPath().resolve(indexName + ".zip"), NetworkModule.HTTP_ENABLED.getKey(), true);
+        Settings nodeSettings = prepareBackwardsDataDir(getBwcIndicesPath().resolve(indexName + ".zip"));
         try {
             internalCluster().startNode(nodeSettings);
             fail();
diff --git a/core/src/test/java/org/elasticsearch/client/transport/TransportClientIT.java b/core/src/test/java/org/elasticsearch/client/transport/TransportClientIT.java
index 5c3cb6d6339..175893455a3 100644
--- a/core/src/test/java/org/elasticsearch/client/transport/TransportClientIT.java
+++ b/core/src/test/java/org/elasticsearch/client/transport/TransportClientIT.java
@@ -51,16 +51,14 @@ public class TransportClientIT extends ESIntegTestCase {
 
     public void testNodeVersionIsUpdated() throws IOException {
         TransportClient client = (TransportClient)  internalCluster().client();
-        Node node = new Node(Settings.builder()
+        try (Node node = new Node(Settings.builder()
                 .put(internalCluster().getDefaultSettings())
                 .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
                 .put("node.name", "testNodeVersionIsUpdated")
                 .put("http.enabled", false)
                 .put(Node.NODE_DATA_SETTING.getKey(), false)
                 .put("cluster.name", "foobar")
-                .build());
-        node.start();
-        try {
+                .build()).start()) {
             TransportAddress transportAddress = node.injector().getInstance(TransportService.class).boundAddress().publishAddress();
             client.addTransportAddress(transportAddress);
             // since we force transport clients there has to be one node started that we connect to.
@@ -79,8 +77,6 @@ public class TransportClientIT extends ESIntegTestCase {
             for (DiscoveryNode discoveryNode : client.filteredNodes()) {
                 assertThat(discoveryNode.getVersion(), equalTo(Version.CURRENT.minimumCompatibilityVersion()));
             }
-        } finally {
-            node.close();
         }
     }
 
diff --git a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
index a1bac928daf..32d02dedc70 100644
--- a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
+++ b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java
@@ -175,7 +175,6 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase {
             .put(FaultDetection.PING_RETRIES_SETTING.getKey(), "1") // for hitting simulated network failures quickly
             .put("discovery.zen.join_timeout", "10s")  // still long to induce failures but to long so test won't time out
             .put(DiscoverySettings.PUBLISH_TIMEOUT_SETTING.getKey(), "1s") // <-- for hitting simulated network failures quickly
-            .put("http.enabled", false) // just to make test quicker
             .build();
 
     @Override
diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java b/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
index 1b62f5d330a..d2085ab9147 100644
--- a/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
+++ b/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java
@@ -35,10 +35,7 @@ public class GatewayServiceTests extends ESTestCase {
         ClusterService clusterService = new ClusterService(Settings.builder().put("cluster.name", "GatewayServiceTests").build(),
                 new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
                 null);
-        return new GatewayService(Settings.builder()
-                .put("http.enabled", "false")
-                .put("discovery.type", "local")
-                .put(settings.build()).build(),
+        return new GatewayService(settings.build(),
                 null, clusterService, null, null, null, new NoopDiscovery(), null, null);
     }
 
diff --git a/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java b/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java
index a0a51d01e03..959aba4c520 100644
--- a/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java
+++ b/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java
@@ -58,10 +58,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSear
  * Tests that requests with RefreshPolicy.WAIT_UNTIL will be visible when they return.
  */
 public class WaitUntilRefreshIT extends ESIntegTestCase {
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(NetworkModule.HTTP_ENABLED.getKey(), true).build();
-    }
 
     @Override
     public Settings indexSettings() {
diff --git a/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java b/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java
deleted file mode 100644
index 72f8733292c..00000000000
--- a/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionIT.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.elasticsearch.rest.action.main;
-
-import org.apache.http.util.EntityUtils;
-import org.elasticsearch.client.Response;
-import org.elasticsearch.common.network.NetworkModule;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.test.ESIntegTestCase;
-
-import java.io.IOException;
-import java.util.Collections;
-
-import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.equalTo;
-
-public class RestMainActionIT extends ESIntegTestCase {
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
-        return Settings.builder()
-            .put(super.nodeSettings(nodeOrdinal))
-            .put(NetworkModule.HTTP_ENABLED.getKey(), true)
-            .build();
-    }
-
-    public void testHeadRequest() throws IOException {
-        try (Response response = getRestClient().performRequest("HEAD", "/", Collections.emptyMap(), null)) {
-            assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
-            assertNull(response.getEntity());
-        }
-    }
-
-    public void testGetRequest() throws IOException {
-        try (Response response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null)) {
-            assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
-            assertNotNull(response.getEntity());
-            assertThat(EntityUtils.toString(response.getEntity()), containsString("cluster_name"));
-        }
-    }
-}

From fd297637a21790517a245321d8f6e55075f07cc6 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 8 Jul 2016 15:35:14 +0200
Subject: [PATCH 1309/1311] Rest Client: add short performRequest method
 variants without params and/or body

Users wanting to send a request by providing only its method and endpoint, effectively the only two required arguments, shouldn't need to pass in an empty map and a null entity for the body. While at it we can also add a variant to send requests by specifying only method, endpoint and params, but not body. Headers remain a vararg as last argument, so they can always optionally be provided.

 Closes #19312
---
 .../org/elasticsearch/client/RestClient.java  | 33 +++++++++++
 .../client/RestClientMultipleHostsTests.java  | 15 ++---
 .../client/RestClientSingleHostTests.java     | 55 +++++++++++++------
 .../client/sniff/HostsSniffer.java            |  2 +-
 4 files changed, 79 insertions(+), 26 deletions(-)

diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java
index a8f46ee6b0e..e3bb1b3c507 100644
--- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java
@@ -117,6 +117,39 @@ public final class RestClient implements Closeable {
         this.blacklist.clear();
     }
 
+    /**
+     * Sends a request to the elasticsearch cluster that the current client points to.
+     * Shortcut to {@link #performRequest(String, String, Map, HttpEntity, Header...)} but without parameters and request body.
+     *
+     * @param method the http method
+     * @param endpoint the path of the request (without host and port)
+     * @param headers the optional request headers
+     * @return the response returned by elasticsearch
+     * @throws IOException in case of a problem or the connection was aborted
+     * @throws ClientProtocolException in case of an http protocol error
+     * @throws ResponseException in case elasticsearch responded with a status code that indicated an error
+     */
+    public Response performRequest(String method, String endpoint, Header... headers) throws IOException {
+        return performRequest(method, endpoint, Collections.<String, String>emptyMap(), null, headers);
+    }
+
+    /**
+     * Sends a request to the elasticsearch cluster that the current client points to.
+     * Shortcut to {@link #performRequest(String, String, Map, HttpEntity, Header...)} but without request body.
+     *
+     * @param method the http method
+     * @param endpoint the path of the request (without host and port)
+     * @param params the query_string parameters
+     * @param headers the optional request headers
+     * @return the response returned by elasticsearch
+     * @throws IOException in case of a problem or the connection was aborted
+     * @throws ClientProtocolException in case of an http protocol error
+     * @throws ResponseException in case elasticsearch responded with a status code that indicated an error
+     */
+    public Response performRequest(String method, String endpoint, Map<String, String> params, Header... headers) throws IOException {
+        return performRequest(method, endpoint, params, null, headers);
+    }
+
     /**
      * Sends a request to the elasticsearch cluster that the current client points to.
      * Selects a host out of the provided ones in a round-robin fashion. Failing hosts are marked dead and retried after a certain
diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
index 9d082466123..b32f848581f 100644
--- a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
+++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.client;
 
-import com.carrotsearch.randomizedtesting.RandomizedTest;
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
 import org.apache.http.HttpHost;
 import org.apache.http.HttpRequest;
@@ -103,7 +102,7 @@ public class RestClientMultipleHostsTests extends RestClientTestCase {
             for (int j = 0; j < httpHosts.length; j++) {
                 int statusCode = randomOkStatusCode(getRandom());
                 try (Response response = restClient.performRequest(randomHttpMethod(getRandom()), "/" + statusCode,
-                        Collections.<String, String>emptyMap(), null)) {
+                        Collections.<String, String>emptyMap())) {
                     assertThat(response.getStatusLine().getStatusCode(), equalTo(statusCode));
                     assertTrue("host not found: " + response.getHost(), hostsSet.remove(response.getHost()));
                 }
@@ -121,8 +120,7 @@ public class RestClientMultipleHostsTests extends RestClientTestCase {
             for (int j = 0; j < httpHosts.length; j++) {
                 String method = randomHttpMethod(getRandom());
                 int statusCode = randomErrorNoRetryStatusCode(getRandom());
-                try (Response response = restClient.performRequest(method, "/" + statusCode,
-                        Collections.<String, String>emptyMap(), null)) {
+                try (Response response = restClient.performRequest(method, "/" + statusCode, Collections.<String, String>emptyMap())) {
                     if (method.equals("HEAD") && statusCode == 404) {
                         //no exception gets thrown although we got a 404
                         assertThat(response.getStatusLine().getStatusCode(), equalTo(404));
@@ -149,7 +147,7 @@ public class RestClientMultipleHostsTests extends RestClientTestCase {
     public void testRoundRobinRetryErrors() throws Exception {
         String retryEndpoint = randomErrorRetryEndpoint();
         try  {
-            restClient.performRequest(randomHttpMethod(getRandom()), retryEndpoint, Collections.<String, String>emptyMap(), null);
+            restClient.performRequest(randomHttpMethod(getRandom()), retryEndpoint, Collections.<String, String>emptyMap());
             fail("request should have failed");
         } catch(ResponseException e) {
             Set<HttpHost> hostsSet = new HashSet<>();
@@ -199,7 +197,7 @@ public class RestClientMultipleHostsTests extends RestClientTestCase {
             for (int j = 0; j < httpHosts.length; j++) {
                 retryEndpoint = randomErrorRetryEndpoint();
                 try  {
-                    restClient.performRequest(randomHttpMethod(getRandom()), retryEndpoint, Collections.<String, String>emptyMap(), null);
+                    restClient.performRequest(randomHttpMethod(getRandom()), retryEndpoint, Collections.<String, String>emptyMap());
                     fail("request should have failed");
                 } catch(ResponseException e) {
                     Response response = e.getResponse();
@@ -226,7 +224,7 @@ public class RestClientMultipleHostsTests extends RestClientTestCase {
                     int statusCode = randomErrorNoRetryStatusCode(getRandom());
                     Response response;
                     try (Response esResponse = restClient.performRequest(randomHttpMethod(getRandom()), "/" + statusCode,
-                            Collections.<String, String>emptyMap(), null)) {
+                            Collections.<String, String>emptyMap())) {
                         response = esResponse;
                     }
                     catch(ResponseException e) {
@@ -245,8 +243,7 @@ public class RestClientMultipleHostsTests extends RestClientTestCase {
                 for (int y = 0; y < i + 1; y++) {
                     retryEndpoint = randomErrorRetryEndpoint();
                     try {
-                        restClient.performRequest(randomHttpMethod(getRandom()), retryEndpoint,
-                                Collections.<String, String>emptyMap(), null);
+                        restClient.performRequest(randomHttpMethod(getRandom()), retryEndpoint, Collections.<String, String>emptyMap());
                         fail("request should have failed");
                     } catch(ResponseException e) {
                         Response response = e.getResponse();
diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
index 59e88b0ed63..b250614b91b 100644
--- a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
+++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
@@ -19,7 +19,6 @@
 
 package org.elasticsearch.client;
 
-import com.carrotsearch.randomizedtesting.RandomizedTest;
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 import org.apache.http.Header;
@@ -156,7 +155,7 @@ public class RestClientSingleHostTests extends RestClientTestCase {
         }
     }
 
-    public void testSetNodes() throws IOException {
+    public void testSetHosts() throws IOException {
         try {
             restClient.setHosts((HttpHost[]) null);
             fail("setHosts should have failed");
@@ -189,8 +188,7 @@ public class RestClientSingleHostTests extends RestClientTestCase {
     public void testOkStatusCodes() throws Exception {
         for (String method : getHttpMethods()) {
             for (int okStatusCode : getOkStatusCodes()) {
-                Response response = restClient.performRequest(method, "/" + okStatusCode,
-                        Collections.<String, String>emptyMap(), null);
+                Response response = performRequest(method, "/" + okStatusCode);
                 assertThat(response.getStatusLine().getStatusCode(), equalTo(okStatusCode));
             }
         }
@@ -204,8 +202,7 @@ public class RestClientSingleHostTests extends RestClientTestCase {
         for (String method : getHttpMethods()) {
             //error status codes should cause an exception to be thrown
             for (int errorStatusCode : getAllErrorStatusCodes()) {
-                try (Response response = restClient.performRequest(method, "/" + errorStatusCode,
-                        Collections.<String, String>emptyMap(), null)) {
+                try (Response response = performRequest(method, "/" + errorStatusCode)) {
                     if (method.equals("HEAD") && errorStatusCode == 404) {
                         //no exception gets thrown although we got a 404
                         assertThat(response.getStatusLine().getStatusCode(), equalTo(errorStatusCode));
@@ -231,14 +228,14 @@ public class RestClientSingleHostTests extends RestClientTestCase {
         for (String method : getHttpMethods()) {
             //IOExceptions should be let bubble up
             try {
-                restClient.performRequest(method, "/coe", Collections.<String, String>emptyMap(), null);
+                performRequest(method, "/coe");
                 fail("request should have failed");
             } catch(IOException e) {
                 assertThat(e, instanceOf(ConnectTimeoutException.class));
             }
             failureListener.assertCalled(httpHost);
             try {
-                restClient.performRequest(method, "/soe", Collections.<String, String>emptyMap(), null);
+                performRequest(method, "/soe");
                 fail("request should have failed");
             } catch(IOException e) {
                 assertThat(e, instanceOf(SocketTimeoutException.class));
@@ -275,8 +272,7 @@ public class RestClientSingleHostTests extends RestClientTestCase {
         }
         for (String method : Arrays.asList("HEAD", "OPTIONS", "TRACE")) {
             try {
-                restClient.performRequest(method, "/" + randomStatusCode(getRandom()),
-                        Collections.<String, String>emptyMap(), entity);
+                restClient.performRequest(method, "/" + randomStatusCode(getRandom()), Collections.<String, String>emptyMap(), entity);
                 fail("request should have failed");
             } catch(UnsupportedOperationException e) {
                 assertThat(e.getMessage(), equalTo(method + " with body is not supported"));
@@ -288,13 +284,13 @@ public class RestClientSingleHostTests extends RestClientTestCase {
         String method = randomHttpMethod(getRandom());
         int statusCode = randomStatusCode(getRandom());
         try {
-            restClient.performRequest(method, "/" + statusCode, Collections.<String, String>emptyMap(), null, (Header[])null);
+            performRequest(method, "/" + statusCode, (Header[])null);
             fail("request should have failed");
         } catch(NullPointerException e) {
             assertEquals("request headers must not be null", e.getMessage());
         }
         try {
-            restClient.performRequest(method, "/" + statusCode, Collections.<String, String>emptyMap(), null, (Header)null);
+            performRequest(method, "/" + statusCode, (Header)null);
             fail("request should have failed");
         } catch(NullPointerException e) {
             assertEquals("request header must not be null", e.getMessage());
@@ -305,7 +301,13 @@ public class RestClientSingleHostTests extends RestClientTestCase {
         String method = randomHttpMethod(getRandom());
         int statusCode = randomStatusCode(getRandom());
         try {
-            restClient.performRequest(method, "/" + statusCode, null, null);
+            restClient.performRequest(method, "/" + statusCode, (Map<String, String>)null);
+            fail("request should have failed");
+        } catch(NullPointerException e) {
+            assertEquals("params must not be null", e.getMessage());
+        }
+        try {
+            restClient.performRequest(method, "/" + statusCode, null, (HttpEntity)null);
             fail("request should have failed");
         } catch(NullPointerException e) {
             assertEquals("params must not be null", e.getMessage());
@@ -352,7 +354,8 @@ public class RestClientSingleHostTests extends RestClientTestCase {
         String uriAsString = "/" + randomStatusCode(getRandom());
         URIBuilder uriBuilder = new URIBuilder(uriAsString);
         Map<String, String> params = Collections.emptyMap();
-        if (getRandom().nextBoolean()) {
+        boolean hasParams = randomBoolean();
+        if (hasParams) {
             int numParams = RandomInts.randomIntBetween(getRandom(), 1, 3);
             params = new HashMap<>(numParams);
             for (int i = 0; i < numParams; i++) {
@@ -395,7 +398,8 @@ public class RestClientSingleHostTests extends RestClientTestCase {
         }
 
         HttpEntity entity = null;
-        if (request instanceof HttpEntityEnclosingRequest && getRandom().nextBoolean()) {
+        boolean hasBody = request instanceof HttpEntityEnclosingRequest && getRandom().nextBoolean();
+        if (hasBody) {
             entity = new StringEntity(RandomStrings.randomAsciiOfLengthBetween(getRandom(), 10, 100));
             ((HttpEntityEnclosingRequest) request).setEntity(entity);
         }
@@ -418,10 +422,29 @@ public class RestClientSingleHostTests extends RestClientTestCase {
         }
 
         try {
-            restClient.performRequest(method, uriAsString, params, entity, headers);
+            if (hasParams == false && hasBody == false && randomBoolean()) {
+                restClient.performRequest(method, uriAsString, headers);
+            } else if (hasBody == false && randomBoolean()) {
+                restClient.performRequest(method, uriAsString, params, headers);
+            } else {
+                restClient.performRequest(method, uriAsString, params, entity, headers);
+            }
         } catch(ResponseException e) {
             //all good
         }
         return request;
     }
+
+    private Response performRequest(String method, String endpoint, Header... headers) throws IOException {
+        switch(randomIntBetween(0, 2)) {
+            case 0:
+                return restClient.performRequest(method, endpoint, headers);
+            case 1:
+                return restClient.performRequest(method, endpoint, Collections.<String, String>emptyMap(), headers);
+            case 2:
+                return restClient.performRequest(method, endpoint, Collections.<String, String>emptyMap(), null, headers);
+            default:
+                throw new UnsupportedOperationException();
+        }
+    }
 }
diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
index f06436c6175..bfe21f5e7d1 100644
--- a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
+++ b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java
@@ -62,7 +62,7 @@ public class HostsSniffer {
      * Calls the elasticsearch nodes info api, parses the response and returns all the found http hosts
      */
     public List<HttpHost> sniffHosts() throws IOException {
-        try (Response response = restClient.performRequest("get", "/_nodes/http", sniffRequestParams, null)) {
+        try (Response response = restClient.performRequest("get", "/_nodes/http", sniffRequestParams)) {
             return readHosts(response.getEntity());
         }
     }

From 942e342662bfecf9c6183ab2635d2112f3bc7002 Mon Sep 17 00:00:00 2001
From: javanna <cavannaluca@gmail.com>
Date: Fri, 8 Jul 2016 20:58:43 +0200
Subject: [PATCH 1310/1311] Rest Client: use short performRequest methods when
 possible

---
 .../client/RestClientMultipleHostsTests.java    | 14 ++++++--------
 .../http/netty/NettyHttpCompressionIT.java      |  5 ++---
 .../DetailedErrorsDisabledIT.java               |  2 +-
 .../detailederrors/DetailedErrorsEnabledIT.java |  4 ++--
 .../plugins/ResponseHeaderPluginIT.java         |  6 ++----
 .../org/elasticsearch/rest/CorsNotSetIT.java    |  6 ++----
 .../org/elasticsearch/rest/CorsRegexIT.java     | 17 +++++++----------
 .../transport/ContextAndHeaderTransportIT.java  |  2 +-
 .../test/rest/client/RestTestClient.java        |  2 +-
 9 files changed, 24 insertions(+), 34 deletions(-)

diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
index b32f848581f..5a43a8d4d9e 100644
--- a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
+++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java
@@ -101,8 +101,7 @@ public class RestClientMultipleHostsTests extends RestClientTestCase {
             Collections.addAll(hostsSet, httpHosts);
             for (int j = 0; j < httpHosts.length; j++) {
                 int statusCode = randomOkStatusCode(getRandom());
-                try (Response response = restClient.performRequest(randomHttpMethod(getRandom()), "/" + statusCode,
-                        Collections.<String, String>emptyMap())) {
+                try (Response response = restClient.performRequest(randomHttpMethod(getRandom()), "/" + statusCode)) {
                     assertThat(response.getStatusLine().getStatusCode(), equalTo(statusCode));
                     assertTrue("host not found: " + response.getHost(), hostsSet.remove(response.getHost()));
                 }
@@ -120,7 +119,7 @@ public class RestClientMultipleHostsTests extends RestClientTestCase {
             for (int j = 0; j < httpHosts.length; j++) {
                 String method = randomHttpMethod(getRandom());
                 int statusCode = randomErrorNoRetryStatusCode(getRandom());
-                try (Response response = restClient.performRequest(method, "/" + statusCode, Collections.<String, String>emptyMap())) {
+                try (Response response = restClient.performRequest(method, "/" + statusCode)) {
                     if (method.equals("HEAD") && statusCode == 404) {
                         //no exception gets thrown although we got a 404
                         assertThat(response.getStatusLine().getStatusCode(), equalTo(404));
@@ -147,7 +146,7 @@ public class RestClientMultipleHostsTests extends RestClientTestCase {
     public void testRoundRobinRetryErrors() throws Exception {
         String retryEndpoint = randomErrorRetryEndpoint();
         try  {
-            restClient.performRequest(randomHttpMethod(getRandom()), retryEndpoint, Collections.<String, String>emptyMap());
+            restClient.performRequest(randomHttpMethod(getRandom()), retryEndpoint);
             fail("request should have failed");
         } catch(ResponseException e) {
             Set<HttpHost> hostsSet = new HashSet<>();
@@ -197,7 +196,7 @@ public class RestClientMultipleHostsTests extends RestClientTestCase {
             for (int j = 0; j < httpHosts.length; j++) {
                 retryEndpoint = randomErrorRetryEndpoint();
                 try  {
-                    restClient.performRequest(randomHttpMethod(getRandom()), retryEndpoint, Collections.<String, String>emptyMap());
+                    restClient.performRequest(randomHttpMethod(getRandom()), retryEndpoint);
                     fail("request should have failed");
                 } catch(ResponseException e) {
                     Response response = e.getResponse();
@@ -223,8 +222,7 @@ public class RestClientMultipleHostsTests extends RestClientTestCase {
                 for (int y = 0; y < iters; y++) {
                     int statusCode = randomErrorNoRetryStatusCode(getRandom());
                     Response response;
-                    try (Response esResponse = restClient.performRequest(randomHttpMethod(getRandom()), "/" + statusCode,
-                            Collections.<String, String>emptyMap())) {
+                    try (Response esResponse = restClient.performRequest(randomHttpMethod(getRandom()), "/" + statusCode)) {
                         response = esResponse;
                     }
                     catch(ResponseException e) {
@@ -243,7 +241,7 @@ public class RestClientMultipleHostsTests extends RestClientTestCase {
                 for (int y = 0; y < i + 1; y++) {
                     retryEndpoint = randomErrorRetryEndpoint();
                     try {
-                        restClient.performRequest(randomHttpMethod(getRandom()), retryEndpoint, Collections.<String, String>emptyMap());
+                        restClient.performRequest(randomHttpMethod(getRandom()), retryEndpoint);
                         fail("request should have failed");
                     } catch(ResponseException e) {
                         Response response = e.getResponse();
diff --git a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java
index 4e69624376d..99a9b9d0f40 100644
--- a/core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java
+++ b/core/src/test/java/org/elasticsearch/http/netty/NettyHttpCompressionIT.java
@@ -62,8 +62,7 @@ public class NettyHttpCompressionIT extends ESIntegTestCase {
         // we need to intercept early, otherwise internal logic in HttpClient will just remove the header and we cannot verify it
         ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor();
         try (RestClient client = createRestClient(HttpClients.custom().addInterceptorFirst(headerExtractor).build())) {
-            try (Response response = client.performRequest("GET", "/", Collections.emptyMap(), null,
-                    new BasicHeader(HttpHeaders.ACCEPT_ENCODING, GZIP_ENCODING))) {
+            try (Response response = client.performRequest("GET", "/", new BasicHeader(HttpHeaders.ACCEPT_ENCODING, GZIP_ENCODING))) {
                 assertEquals(200, response.getStatusLine().getStatusCode());
                 assertTrue(headerExtractor.hasContentEncodingHeader());
                 assertEquals(GZIP_ENCODING, headerExtractor.getContentEncodingHeader().getValue());
@@ -76,7 +75,7 @@ public class NettyHttpCompressionIT extends ESIntegTestCase {
         ContentEncodingHeaderExtractor headerExtractor = new ContentEncodingHeaderExtractor();
         CloseableHttpClient httpClient = HttpClients.custom().disableContentCompression().addInterceptorFirst(headerExtractor).build();
         try (RestClient client = createRestClient(httpClient)) {
-            try (Response response = client.performRequest("GET", "/", Collections.emptyMap(), null)) {
+            try (Response response = client.performRequest("GET", "/")) {
                 assertEquals(200, response.getStatusLine().getStatusCode());
                 assertFalse(headerExtractor.hasContentEncodingHeader());
             }
diff --git a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java
index 21b8706abfb..8b933765dc5 100644
--- a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java
+++ b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsDisabledIT.java
@@ -49,7 +49,7 @@ public class DetailedErrorsDisabledIT extends ESIntegTestCase {
 
     public void testThatErrorTraceParamReturns400() throws Exception {
         try {
-            getRestClient().performRequest("DELETE", "/", Collections.singletonMap("error_trace", "true"), null);
+            getRestClient().performRequest("DELETE", "/", Collections.singletonMap("error_trace", "true"));
             fail("request should have failed");
         } catch(ResponseException e) {
             Response response = e.getResponse();
diff --git a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java
index 269dc72b253..e88fa2e6863 100644
--- a/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java
+++ b/core/src/test/java/org/elasticsearch/options/detailederrors/DetailedErrorsEnabledIT.java
@@ -47,7 +47,7 @@ public class DetailedErrorsEnabledIT extends ESIntegTestCase {
 
     public void testThatErrorTraceWorksByDefault() throws Exception {
         try {
-            getRestClient().performRequest("DELETE", "/", Collections.singletonMap("error_trace", "true"), null);
+            getRestClient().performRequest("DELETE", "/", Collections.singletonMap("error_trace", "true"));
             fail("request should have failed");
         } catch(ResponseException e) {
             Response response = e.getResponse();
@@ -57,7 +57,7 @@ public class DetailedErrorsEnabledIT extends ESIntegTestCase {
         }
 
         try {
-            getRestClient().performRequest("DELETE", "/", Collections.emptyMap(), null);
+            getRestClient().performRequest("DELETE", "/");
             fail("request should have failed");
         } catch(ResponseException e) {
             Response response = e.getResponse();
diff --git a/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java b/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
index 6a3513416f8..59b52f5cf39 100644
--- a/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
+++ b/core/src/test/java/org/elasticsearch/plugins/ResponseHeaderPluginIT.java
@@ -28,7 +28,6 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 
 import java.util.Collection;
-import java.util.Collections;
 
 import static org.hamcrest.Matchers.equalTo;
 
@@ -53,7 +52,7 @@ public class ResponseHeaderPluginIT extends ESIntegTestCase {
     public void testThatSettingHeadersWorks() throws Exception {
         ensureGreen();
         try {
-            getRestClient().performRequest("GET", "/_protected", Collections.emptyMap(), null);
+            getRestClient().performRequest("GET", "/_protected");
             fail("request should have failed");
         } catch(ResponseException e) {
             Response response = e.getResponse();
@@ -61,8 +60,7 @@ public class ResponseHeaderPluginIT extends ESIntegTestCase {
             assertThat(response.getHeader("Secret"), equalTo("required"));
         }
 
-        try (Response authResponse = getRestClient().performRequest("GET", "/_protected", Collections.emptyMap(), null,
-                new BasicHeader("Secret", "password"))) {
+        try (Response authResponse = getRestClient().performRequest("GET", "/_protected", new BasicHeader("Secret", "password"))) {
             assertThat(authResponse.getStatusLine().getStatusCode(), equalTo(200));
             assertThat(authResponse.getHeader("Secret"), equalTo("granted"));
         }
diff --git a/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java b/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java
index 5f272d3e7e2..19ee7c1cf84 100644
--- a/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java
+++ b/core/src/test/java/org/elasticsearch/rest/CorsNotSetIT.java
@@ -26,8 +26,6 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 
-import java.util.Collections;
-
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -46,7 +44,7 @@ public class CorsNotSetIT extends ESIntegTestCase {
 
     public void testCorsSettingDefaultBehaviourDoesNotReturnAnything() throws Exception {
         String corsValue = "http://localhost:9200";
-        try (Response response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null,
+        try (Response response = getRestClient().performRequest("GET", "/",
                 new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue))) {
             assertThat(response.getStatusLine().getStatusCode(), is(200));
             assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue());
@@ -55,7 +53,7 @@ public class CorsNotSetIT extends ESIntegTestCase {
     }
 
     public void testThatOmittingCorsHeaderDoesNotReturnAnything() throws Exception {
-        try (Response response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null)) {
+        try (Response response = getRestClient().performRequest("GET", "/")) {
             assertThat(response.getStatusLine().getStatusCode(), is(200));
             assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue());
             assertThat(response.getHeader("Access-Control-Allow-Credentials"), nullValue());
diff --git a/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java b/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java
index e6c85553231..e96a1dec386 100644
--- a/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java
+++ b/core/src/test/java/org/elasticsearch/rest/CorsRegexIT.java
@@ -30,8 +30,6 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
 import org.elasticsearch.test.ESIntegTestCase.Scope;
 import org.jboss.netty.handler.codec.http.HttpHeaders;
 
-import java.util.Collections;
-
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_METHODS;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN;
@@ -61,12 +59,12 @@ public class CorsRegexIT extends ESIntegTestCase {
 
     public void testThatRegularExpressionWorksOnMatch() throws Exception {
         String corsValue = "http://localhost:9200";
-        try (Response response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null,
+        try (Response response = getRestClient().performRequest("GET", "/",
                 new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue))) {
             assertResponseWithOriginheader(response, corsValue);
         }
         corsValue = "https://localhost:9200";
-        try (Response response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null,
+        try (Response response = getRestClient().performRequest("GET", "/",
                 new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue));) {
             assertResponseWithOriginheader(response, corsValue);
             assertThat(response.getHeader("Access-Control-Allow-Credentials"), is("true"));
@@ -75,7 +73,7 @@ public class CorsRegexIT extends ESIntegTestCase {
 
     public void testThatRegularExpressionReturnsForbiddenOnNonMatch() throws Exception {
         try {
-            getRestClient().performRequest("GET", "/", Collections.emptyMap(), null, new BasicHeader("User-Agent", "Mozilla Bar"),
+            getRestClient().performRequest("GET", "/", new BasicHeader("User-Agent", "Mozilla Bar"),
                     new BasicHeader("Origin", "http://evil-host:9200"));
             fail("request should have failed");
         } catch(ResponseException e) {
@@ -87,15 +85,14 @@ public class CorsRegexIT extends ESIntegTestCase {
     }
 
     public void testThatSendingNoOriginHeaderReturnsNoAccessControlHeader() throws Exception {
-        try (Response response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null,
-                new BasicHeader("User-Agent", "Mozilla Bar"))) {
+        try (Response response = getRestClient().performRequest("GET", "/", new BasicHeader("User-Agent", "Mozilla Bar"))) {
             assertThat(response.getStatusLine().getStatusCode(), is(200));
             assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue());
         }
     }
 
     public void testThatRegularExpressionIsNotAppliedWithoutCorrectBrowserOnMatch() throws Exception {
-        try (Response response = getRestClient().performRequest("GET", "/", Collections.emptyMap(), null)) {
+        try (Response response = getRestClient().performRequest("GET", "/")) {
             assertThat(response.getStatusLine().getStatusCode(), is(200));
             assertThat(response.getHeader("Access-Control-Allow-Origin"), nullValue());
         }
@@ -103,7 +100,7 @@ public class CorsRegexIT extends ESIntegTestCase {
 
     public void testThatPreFlightRequestWorksOnMatch() throws Exception {
         String corsValue = "http://localhost:9200";
-        try (Response response = getRestClient().performRequest("OPTIONS", "/", Collections.emptyMap(), null,
+        try (Response response = getRestClient().performRequest("OPTIONS", "/",
                 new BasicHeader("User-Agent", "Mozilla Bar"), new BasicHeader("Origin", corsValue),
                 new BasicHeader(HttpHeaders.Names.ACCESS_CONTROL_REQUEST_METHOD, "GET"));) {
             assertResponseWithOriginheader(response, corsValue);
@@ -113,7 +110,7 @@ public class CorsRegexIT extends ESIntegTestCase {
 
     public void testThatPreFlightRequestReturnsNullOnNonMatch() throws Exception {
         try {
-            getRestClient().performRequest("OPTIONS", "/", Collections.emptyMap(), null, new BasicHeader("User-Agent", "Mozilla Bar"),
+            getRestClient().performRequest("OPTIONS", "/", new BasicHeader("User-Agent", "Mozilla Bar"),
                     new BasicHeader("Origin", "http://evil-host:9200"),
                     new BasicHeader(HttpHeaders.Names.ACCESS_CONTROL_REQUEST_METHOD, "GET"));
             fail("request should have failed");
diff --git a/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java b/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
index e9ec3787afc..e8a4b22b5a9 100644
--- a/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
+++ b/core/src/test/java/org/elasticsearch/transport/ContextAndHeaderTransportIT.java
@@ -220,7 +220,7 @@ public class ContextAndHeaderTransportIT extends ESIntegTestCase {
         }
 
         try (Response response = getRestClient().performRequest(
-                "GET", "/" + queryIndex + "/_search", Collections.emptyMap(), null,
+                "GET", "/" + queryIndex + "/_search",
                 new BasicHeader(randomHeaderKey, randomHeaderValue), new BasicHeader(relevantHeaderName, randomHeaderValue))) {
             assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
             List<RequestAndHeaders> searchRequests = getRequests(SearchRequest.class);
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java
index 6879e02747d..9b4c4612736 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestClient.java
@@ -113,7 +113,7 @@ public class RestTestClient implements Closeable {
             //we don't really use the urls here, we rely on the client doing round-robin to touch all the nodes in the cluster
             String method = restApi.getMethods().get(0);
             String endpoint = restApi.getPaths().get(0);
-            Response response = restClient.performRequest(method, endpoint, Collections.emptyMap(), null);
+            Response response = restClient.performRequest(method, endpoint);
             RestTestResponse restTestResponse = new RestTestResponse(response);
             Object latestVersion = restTestResponse.evaluate("version.number");
             if (latestVersion == null) {

From 1d03a1409cea5fa0d133fb36eb7f7d88abb733d1 Mon Sep 17 00:00:00 2001
From: Simon Willnauer <simon.willnauer@elasticsearch.com>
Date: Mon, 11 Jul 2016 11:20:56 +0200
Subject: [PATCH 1311/1311] Catch assertion errors on commit and turn it into a
 real exception (#19357)

Lucene IndexWriter asserts on files existing on the filesystem but
some tests throw IOException explicitly on those operatiosn such that
some tests trip asserts. We had this before on InternalEngine#ctor
and added some logic there to catch only a specific assertions based
on some excepition stack analysis. This change applies the same logic
to the IndexWriter#commit part of the engine since it can hit the same
issue.
This also fixes a self-suppression issue in Store.java.

Closes #19356
---
 .../elasticsearch/index/engine/InternalEngine.java | 14 ++++++++++++++
 .../java/org/elasticsearch/index/store/Store.java  |  6 +++---
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
index 779d7629686..1775fea6703 100644
--- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
+++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
@@ -1185,6 +1185,20 @@ public class InternalEngine extends Engine {
                 ex.addSuppressed(inner);
             }
             throw ex;
+        } catch (AssertionError e) {
+            // IndexWriter throws AssertionError on commit, if asserts are enabled, if any files don't exist, but tests that
+            // randomly throw FNFE/NSFE can also hit this:
+            if (ExceptionsHelper.stackTrace(e).contains("org.apache.lucene.index.IndexWriter.filesExist")) {
+                EngineException engineException = new EngineException(shardId, "failed to commit engine", e);
+                try {
+                    failEngine("lucene commit failed", engineException);
+                } catch (Exception inner) {
+                    engineException.addSuppressed(inner);
+                }
+                throw engineException;
+            } else {
+                throw e;
+            }
         }
     }
 
diff --git a/core/src/main/java/org/elasticsearch/index/store/Store.java b/core/src/main/java/org/elasticsearch/index/store/Store.java
index dfbad39ad39..979c834ccc9 100644
--- a/core/src/main/java/org/elasticsearch/index/store/Store.java
+++ b/core/src/main/java/org/elasticsearch/index/store/Store.java
@@ -846,14 +846,14 @@ public class Store extends AbstractIndexShardComponent implements Closeable, Ref
                     logger.warn("failed to build store metadata. checking segment info integrity (with commit [{}])",
                             ex, commit == null ? "no" : "yes");
                     Lucene.checkSegmentInfoIntegrity(directory);
-                    throw ex;
                 } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException cex) {
                     cex.addSuppressed(ex);
                     throw cex;
                 } catch (Exception inner) {
-                    ex.addSuppressed(inner);
-                    throw ex;
+                    inner.addSuppressed(ex);
+                    throw inner;
                 }
+                throw ex;
             }
             return new LoadedMetadata(unmodifiableMap(builder), unmodifiableMap(commitUserDataBuilder), numDocs);
         }